diff --git a/.forgejo/patches/ghc-9.8.patch b/.forgejo/patches/ghc-9.8.patch deleted file mode 100644 index 85796d787d..0000000000 --- a/.forgejo/patches/ghc-9.8.patch +++ /dev/null @@ -1,18 +0,0 @@ -Support ghc-9.8 by widening a lot of constraints. - -This patch can be removed once upstream supports ghc 9.8 offically. - -diff -uprN git-annex-10.20240227.orig/cabal.project git-annex-10.20240227/cabal.project ---- git-annex-10.20240227.orig/cabal.project 1970-01-01 01:00:00.000000000 +0100 -+++ git-annex-10.20240227/cabal.project 2024-04-28 13:30:14.061706299 +0200 -@@ -0,0 +1,10 @@ -+packages: *.cabal -+ -+allow-newer: dav -+allow-newer: haskeline:filepath -+allow-newer: haskeline:directory -+allow-newer: xml-hamlet -+allow-newer: aws:filepath -+allow-newer: dbus:network -+allow-newer: dbus:filepath -+allow-newer: microstache:filepath diff --git a/.forgejo/workflows/generate-lockfile.yml b/.forgejo/workflows/generate-lockfile.yml deleted file mode 100644 index 8dbb579e67..0000000000 --- a/.forgejo/workflows/generate-lockfile.yml +++ /dev/null @@ -1,89 +0,0 @@ -on: - workflow_dispatch: - inputs: - ref_name: - description: 'Tag or commit' - required: true - type: string - - push: - tags: - - '*' - -jobs: - cabal-config-edge: - name: Generate cabal config for edge - runs-on: x86_64 - container: - image: alpine:edge - env: - CI_ALPINE_TARGET_RELEASE: edge - steps: - - name: Environment setup - run: | - apk upgrade -a - apk add nodejs git cabal patch - - name: Repo pull - uses: actions/checkout@v4 - with: - fetch-depth: 1 - ref: ${{ inputs.ref_name }} - - name: Config generation - run: | - patch -p1 -i .forgejo/patches/ghc-9.8.patch - HOME="${{ github.workspace}}"/cabal_cache cabal update - HOME="${{ github.workspace}}"/cabal_cache cabal v2-freeze --shadow-installed-packages --strong-flags --flags="+assistant +webapp +pairing +production +torrentparser +magicmime +benchmark -debuglocks +dbus +networkbsd +gitlfs +httpclientrestricted" - mv cabal.project.freeze git-annex.config - - name: Package upload - uses: forgejo/upload-artifact@v3 - with: - name: cabalconfigedge - path: git-annex*.config - cabal-config-v322: - name: Generate cabal config for v3.22 - runs-on: x86_64 - container: - image: alpine:3.22 - env: - CI_ALPINE_TARGET_RELEASE: v3.22 - steps: - - name: Environment setup - run: | - apk upgrade -a - apk add nodejs git cabal patch - - name: Repo pull - uses: actions/checkout@v4 - with: - fetch-depth: 1 - ref: ${{ inputs.ref_name }} - - name: Config generation - run: | - patch -p1 -i .forgejo/patches/ghc-9.8.patch - HOME="${{ github.workspace }}"/cabal_cache cabal update - HOME="${{ github.workspace }}"/cabal_cache cabal v2-freeze --shadow-installed-packages --strong-flags --flags="+assistant +webapp +pairing +production +torrentparser +magicmime +benchmark -debuglocks +dbus +networkbsd +gitlfs +httpclientrestricted" - mv cabal.project.freeze git-annex.config - - name: Package upload - uses: forgejo/upload-artifact@v3 - with: - name: cabalconfig322 - path: git-annex*.config - upload-tarball: - name: Upload to generic repo - runs-on: x86_64 - needs: [cabal-config-edge,cabal-config-v322] - container: - image: alpine:latest - steps: - - name: Environment setup - run: apk add nodejs curl findutils - - name: Package download - uses: forgejo/download-artifact@v3 - - name: Package deployment - run: | - if test $GITHUB_REF_NAME == "ci" ; then - CI_REF_NAME=${{ inputs.ref_name }} - else - CI_REF_NAME=$GITHUB_REF_NAME - fi - curl --user ${{ vars.CODE_FORGEJO_USER }}:${{ secrets.CODE_FORGEJO_TOKEN }} --upload-file ./cabalconfigedge/git-annex.config ${{ github.server_url }}/api/packages/mirrors/generic/git-annex/$CI_REF_NAME/git-annex-$CI_REF_NAME-edge.cabal - curl --user ${{ vars.CODE_FORGEJO_USER }}:${{ secrets.CODE_FORGEJO_TOKEN }} --upload-file ./cabalconfig322/git-annex.config ${{ github.server_url }}/api/packages/mirrors/generic/git-annex/$CI_REF_NAME/git-annex-$CI_REF_NAME-v322.cabal diff --git a/.forgejo/workflows/mirror-repository.yml b/.forgejo/workflows/mirror-repository.yml deleted file mode 100644 index f44c4668cf..0000000000 --- a/.forgejo/workflows/mirror-repository.yml +++ /dev/null @@ -1,50 +0,0 @@ -on: - workflow_dispatch: - - schedule: - - cron: '@hourly' - -jobs: - mirror: - name: Pull from upstream - runs-on: x86_64 - container: - image: alpine:latest - env: - upstream: https://git.joeyh.name/git/git-annex.git - tags: '10.2025*' - steps: - - name: Environment setup - run: apk add grep git sed coreutils bash nodejs - - name: Fetch destination - uses: actions/checkout@v4 - with: - fetch_depth: 1 - ref: ci - token: ${{ secrets.CODE_FORGEJO_TOKEN }} - - name: Missing tag detecting - run: | - git ls-remote $upstream "refs/tags/$tags" | grep -v '{' | sed 's|.*/||' | sort > upstream_tags - git ls-remote ${{ github.server_url}}/${{ github.repository }} "refs/tags/$tags" | grep -v '{' | sed 's|.*/||' | sort > destination_tags - comm -23 upstream_tags destination_tags > missing_tags - echo "Missing tags:" - cat missing_tags - - name: Missing tag fetch - run: | - git remote add upstream $upstream - while read tag; do - git fetch upstream tag $tag --no-tags - done < missing_tags - - name: Packaging workflow injection - run: | - while read tag; do - git checkout $tag - git tag -d $tag - git checkout ci -- ./.forgejo - git config user.name "forgejo-actions[bot]" - git config user.email "dev@ayakael.net" - git commit -m 'Inject custom workflow' - git tag -a $tag -m $tag - done < missing_tags - - name: Push to destination - run: git push --force origin refs/tags/*:refs/tags/* --tags diff --git a/.ghci b/.ghci new file mode 100644 index 0000000000..c5550cee6e --- /dev/null +++ b/.ghci @@ -0,0 +1 @@ +:load Common diff --git a/.gitattributes b/.gitattributes new file mode 100644 index 0000000000..a81b30b931 --- /dev/null +++ b/.gitattributes @@ -0,0 +1 @@ +CHANGELOG merge=dpkg-mergechangelogs diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000000..16314abf76 --- /dev/null +++ b/.gitignore @@ -0,0 +1,36 @@ +tags +Setup +*.hi +*.o +tmp +test +Build/SysConfig +Build/Version +Build/InstallDesktopFile +Build/EvilSplicer +Build/Standalone +Build/OSXMkLibs +Build/LinuxMkLibs +Build/BuildVersion +Build/MakeMans +git-annex +git-annex-shell +man +git-union-merge +git-union-merge.1 +doc/.ikiwiki +html +*.tix +.hpc +dist +# Sandboxed builds +cabal-dev +.cabal-sandbox +cabal.sandbox.config +.stack-work +# Project-local emacs configuration +.dir-locals.el +# OSX related +.DS_Store +.virthualenv +.tasty-rerun-log diff --git a/.mailmap b/.mailmap new file mode 100644 index 0000000000..2ea582b7c9 --- /dev/null +++ b/.mailmap @@ -0,0 +1,29 @@ +Antoine Beaupré anarcat +Antoine Beaupré https://id.koumbit.net/anarcat +Greg Grossmeier http://grossmeier.net/ +Jimmy Tang jtang +Joachim Breitner http://www.joachim-breitner.de/ +Joey Hess Joey Hess +Joey Hess Joey Hess +Joey Hess Joey Hess +Joey Hess Joey Hess +Joey Hess Joey Hess +Joey Hess Joey Hess +Joey Hess Joey Hess +Joey Hess http://joey.kitenet.net/ +Joey Hess http://joeyh.name/ +Joey Hess http://joeyh.name/ +Joey Hess https://www.google.com/accounts/o8/id?id=AItOawmJfIszzreLNvCqzqzvTayA9_9L6gb9RtY +Johan Kiviniemi http://johan.kiviniemi.name/ +Johan Kiviniemi http://johan.kiviniemi.name/ +Nicolas Pouillard http://ertai.myopenid.com/ +Peter Simons Peter Simons +Peter Simons http://peter-simons.myopenid.com/ +Philipp Kern http://phil.0x539.de/ +Richard Hartmann https://www.google.com/accounts/o8/id?id=AItOawl9sYlePmv1xK-VvjBdN-5doOa_Xw-jH4U +Yaroslav Halchenko +Yaroslav Halchenko http://yarikoptic.myopenid.com/ +Yaroslav Halchenko https://www.google.com/accounts/o8/id?id=AItOawnx8kHW66N3BqmkVpgtXDlYMvr8TJ5VvfY +Yaroslav Halchenko https://me.yahoo.com/a/EbvxpTI_xP9Aod7Mg4cwGhgjrCrdM5s-#7c0f4 +Øyvind A. Holm http://sunny256.sunbase.org/ +Øyvind A. Holm https://sunny256.wordpress.com/ diff --git a/Annex.hs b/Annex.hs new file mode 100644 index 0000000000..bdedec3c69 --- /dev/null +++ b/Annex.hs @@ -0,0 +1,380 @@ +{- git-annex monad + - + - Copyright 2010-2018 Joey Hess + - + - Licensed under the GNU GPL version 3 or higher. + -} + +{-# LANGUAGE GeneralizedNewtypeDeriving, PackageImports, BangPatterns #-} + +module Annex ( + Annex, + AnnexState(..), + new, + run, + eval, + makeRunner, + getState, + changeState, + withState, + setFlag, + setField, + setOutput, + getFlag, + getField, + addCleanup, + gitRepo, + inRepo, + fromRepo, + calcRepo, + getGitConfig, + changeGitConfig, + changeGitRepo, + adjustGitRepo, + getRemoteGitConfig, + withCurrentState, + changeDirectory, + getGitRemotes, + incError, +) where + +import Common +import qualified Git +import qualified Git.Config +import qualified Git.Construct +import Annex.Fixup +import Git.CatFile +import Git.HashObject +import Git.CheckAttr +import Git.CheckIgnore +import qualified Git.Hook +import qualified Git.Queue +import Types.Key +import Types.Backend +import Types.GitConfig +import qualified Types.Remote +import Types.Crypto +import Types.BranchState +import Types.TrustLevel +import Types.Group +import Types.Messages +import Types.Concurrency +import Types.UUID +import Types.FileMatcher +import Types.NumCopies +import Types.LockCache +import Types.DesktopNotify +import Types.CleanupActions +import qualified Database.Keys.Handle as Keys +import Utility.InodeCache +import Utility.Url + +import "mtl" Control.Monad.Reader +import Control.Concurrent +import Control.Concurrent.Async +import Control.Concurrent.STM +import qualified Control.Concurrent.SSem as SSem +import qualified Data.Map.Strict as M +import qualified Data.Set as S + +{- git-annex's monad is a ReaderT around an AnnexState stored in a MVar. + - The MVar is not exposed outside this module. + - + - Note that when an Annex action fails and the exception is caught, + - any changes the action has made to the AnnexState are retained, + - due to the use of the MVar to store the state. + -} +newtype Annex a = Annex { runAnnex :: ReaderT (MVar AnnexState) IO a } + deriving ( + Monad, + MonadIO, + MonadReader (MVar AnnexState), + MonadCatch, + MonadThrow, + MonadMask, + Functor, + Applicative + ) + +-- internal state storage +data AnnexState = AnnexState + { repo :: Git.Repo + , repoadjustment :: (Git.Repo -> IO Git.Repo) + , gitconfig :: GitConfig + , gitremotes :: Maybe [Git.Repo] + , backend :: Maybe (BackendA Annex) + , remotes :: [Types.Remote.RemoteA Annex] + , remoteannexstate :: M.Map UUID AnnexState + , output :: MessageState + , concurrency :: Concurrency + , force :: Bool + , fast :: Bool + , daemon :: Bool + , branchstate :: BranchState + , repoqueue :: Maybe Git.Queue.Queue + , repoqueuesem :: SSem.SSem + , catfilehandles :: M.Map FilePath CatFileHandle + , hashobjecthandle :: Maybe HashObjectHandle + , checkattrhandle :: Maybe CheckAttrHandle + , checkignorehandle :: Maybe (Maybe CheckIgnoreHandle) + , forcebackend :: Maybe String + , globalnumcopies :: Maybe NumCopies + , forcenumcopies :: Maybe NumCopies + , limit :: ExpandableMatcher Annex + , uuidmap :: Maybe UUIDMap + , preferredcontentmap :: Maybe (FileMatcherMap Annex) + , requiredcontentmap :: Maybe (FileMatcherMap Annex) + , forcetrust :: TrustMap + , trustmap :: Maybe TrustMap + , groupmap :: Maybe GroupMap + , ciphers :: M.Map StorableCipher Cipher + , lockcache :: LockCache + , sshstalecleaned :: TMVar Bool + , flags :: M.Map String Bool + , fields :: M.Map String String + , cleanup :: M.Map CleanupAction (Annex ()) + , sentinalstatus :: Maybe SentinalStatus + , useragent :: Maybe String + , errcounter :: Integer + , unusedkeys :: Maybe (S.Set Key) + , tempurls :: M.Map Key URLString + , existinghooks :: M.Map Git.Hook.Hook Bool + , desktopnotify :: DesktopNotify + , workers :: [Either AnnexState (Async AnnexState)] + , activekeys :: TVar (M.Map Key ThreadId) + , activeremotes :: MVar (M.Map (Types.Remote.RemoteA Annex) Integer) + , keysdbhandle :: Maybe Keys.DbHandle + , cachedcurrentbranch :: Maybe Git.Branch + , cachedgitenv :: Maybe [(String, String)] + , urloptions :: Maybe UrlOptions + } + +newState :: GitConfig -> Git.Repo -> IO AnnexState +newState c r = do + emptyactiveremotes <- newMVar M.empty + emptyactivekeys <- newTVarIO M.empty + o <- newMessageState + sc <- newTMVarIO False + qsem <- SSem.new 1 + return $ AnnexState + { repo = r + , repoadjustment = return + , gitconfig = c + , gitremotes = Nothing + , backend = Nothing + , remotes = [] + , remoteannexstate = M.empty + , output = o + , concurrency = NonConcurrent + , force = False + , fast = False + , daemon = False + , branchstate = startBranchState + , repoqueue = Nothing + , repoqueuesem = qsem + , catfilehandles = M.empty + , hashobjecthandle = Nothing + , checkattrhandle = Nothing + , checkignorehandle = Nothing + , forcebackend = Nothing + , globalnumcopies = Nothing + , forcenumcopies = Nothing + , limit = BuildingMatcher [] + , uuidmap = Nothing + , preferredcontentmap = Nothing + , requiredcontentmap = Nothing + , forcetrust = M.empty + , trustmap = Nothing + , groupmap = Nothing + , ciphers = M.empty + , lockcache = M.empty + , sshstalecleaned = sc + , flags = M.empty + , fields = M.empty + , cleanup = M.empty + , sentinalstatus = Nothing + , useragent = Nothing + , errcounter = 0 + , unusedkeys = Nothing + , tempurls = M.empty + , existinghooks = M.empty + , desktopnotify = mempty + , workers = [] + , activekeys = emptyactivekeys + , activeremotes = emptyactiveremotes + , keysdbhandle = Nothing + , cachedcurrentbranch = Nothing + , cachedgitenv = Nothing + , urloptions = Nothing + } + +{- Makes an Annex state object for the specified git repo. + - Ensures the config is read, if it was not already, and performs + - any necessary git repo fixups. -} +new :: Git.Repo -> IO AnnexState +new r = do + r' <- Git.Config.read =<< Git.relPath r + let c = extractGitConfig r' + newState c =<< fixupRepo r' c + +{- Performs an action in the Annex monad from a starting state, + - returning a new state. -} +run :: AnnexState -> Annex a -> IO (a, AnnexState) +run s a = flip run' a =<< newMVar s + +run' :: MVar AnnexState -> Annex a -> IO (a, AnnexState) +run' mvar a = do + r <- runReaderT (runAnnex a) mvar + `onException` (flush =<< readMVar mvar) + s' <- takeMVar mvar + flush s' + return (r, s') + where + flush = maybe noop Keys.flushDbQueue . keysdbhandle + +{- Performs an action in the Annex monad from a starting state, + - and throws away the new state. -} +eval :: AnnexState -> Annex a -> IO a +eval s a = fst <$> run s a + +{- Makes a runner action, that allows diving into IO and from inside + - the IO action, running an Annex action. -} +makeRunner :: Annex (Annex a -> IO a) +makeRunner = do + mvar <- ask + return $ \a -> do + (r, s) <- run' mvar a + putMVar mvar s + return r + +getState :: (AnnexState -> v) -> Annex v +getState selector = do + mvar <- ask + s <- liftIO $ readMVar mvar + return $ selector s + +changeState :: (AnnexState -> AnnexState) -> Annex () +changeState modifier = do + mvar <- ask + liftIO $ modifyMVar_ mvar $ return . modifier + +withState :: (AnnexState -> IO (AnnexState, b)) -> Annex b +withState modifier = do + mvar <- ask + liftIO $ modifyMVar mvar modifier + +{- Sets a flag to True -} +setFlag :: String -> Annex () +setFlag flag = changeState $ \s -> + s { flags = M.insert flag True $ flags s } + +{- Sets a field to a value -} +setField :: String -> String -> Annex () +setField field value = changeState $ \s -> + s { fields = M.insert field value $ fields s } + +{- Adds a cleanup action to perform. -} +addCleanup :: CleanupAction -> Annex () -> Annex () +addCleanup k a = changeState $ \s -> + s { cleanup = M.insert k a $ cleanup s } + +{- Sets the type of output to emit. -} +setOutput :: OutputType -> Annex () +setOutput o = changeState $ \s -> + let m = output s + in s { output = m { outputType = adjustOutputType (outputType m) o } } + +{- Checks if a flag was set. -} +getFlag :: String -> Annex Bool +getFlag flag = fromMaybe False . M.lookup flag <$> getState flags + +{- Gets the value of a field. -} +getField :: String -> Annex (Maybe String) +getField field = M.lookup field <$> getState fields + +{- Returns the annex's git repository. -} +gitRepo :: Annex Git.Repo +gitRepo = getState repo + +{- Runs an IO action in the annex's git repository. -} +inRepo :: (Git.Repo -> IO a) -> Annex a +inRepo a = liftIO . a =<< gitRepo + +{- Extracts a value from the annex's git repisitory. -} +fromRepo :: (Git.Repo -> a) -> Annex a +fromRepo a = a <$> gitRepo + +{- Calculates a value from an annex's git repository and its GitConfig. -} +calcRepo :: (Git.Repo -> GitConfig -> IO a) -> Annex a +calcRepo a = do + s <- getState id + liftIO $ a (repo s) (gitconfig s) + +{- Gets the GitConfig settings. -} +getGitConfig :: Annex GitConfig +getGitConfig = getState gitconfig + +{- Modifies a GitConfig setting. -} +changeGitConfig :: (GitConfig -> GitConfig) -> Annex () +changeGitConfig a = changeState $ \s -> s { gitconfig = a (gitconfig s) } + +{- Changing the git Repo data also involves re-extracting its GitConfig. -} +changeGitRepo :: Git.Repo -> Annex () +changeGitRepo r = do + adjuster <- getState repoadjustment + r' <- liftIO $ adjuster r + changeState $ \s -> s + { repo = r' + , gitconfig = extractGitConfig r' + } + +{- Adds an adjustment to the Repo data. Adjustments persist across reloads + - of the repo's config. -} +adjustGitRepo :: (Git.Repo -> IO Git.Repo) -> Annex () +adjustGitRepo a = do + changeState $ \s -> s { repoadjustment = \r -> repoadjustment s r >>= a } + changeGitRepo =<< gitRepo + +{- Gets the RemoteGitConfig from a remote, given the Git.Repo for that + - remote. -} +getRemoteGitConfig :: Git.Repo -> Annex RemoteGitConfig +getRemoteGitConfig r = do + g <- gitRepo + liftIO $ atomically $ extractRemoteGitConfig g (Git.repoDescribe r) + +{- Converts an Annex action into an IO action, that runs with a copy + - of the current Annex state. + - + - Use with caution; the action should not rely on changing the + - state, as it will be thrown away. -} +withCurrentState :: Annex a -> Annex (IO a) +withCurrentState a = do + s <- getState id + return $ eval s a + +{- It's not safe to use setCurrentDirectory in the Annex monad, + - because the git repo paths are stored relative. + - Instead, use this. + -} +changeDirectory :: FilePath -> Annex () +changeDirectory d = do + r <- liftIO . Git.adjustPath absPath =<< gitRepo + liftIO $ setCurrentDirectory d + r' <- liftIO $ Git.relPath r + changeState $ \s -> s { repo = r' } + +incError :: Annex () +incError = changeState $ \s -> + let ! c = errcounter s + 1 + ! s' = s { errcounter = c } + in s' + +getGitRemotes :: Annex [Git.Repo] +getGitRemotes = do + s <- getState id + case gitremotes s of + Just rs -> return rs + Nothing -> do + rs <- liftIO $ Git.Construct.fromRemotes (repo s) + changeState $ \s' -> s' { gitremotes = Just rs } + return rs diff --git a/Annex/Action.hs b/Annex/Action.hs new file mode 100644 index 0000000000..273c62fa82 --- /dev/null +++ b/Annex/Action.hs @@ -0,0 +1,66 @@ +{- git-annex actions + - + - Copyright 2010-2015 Joey Hess + - + - Licensed under the GNU GPL version 3 or higher. + -} + +{-# LANGUAGE CPP #-} + +module Annex.Action where + +import qualified Data.Map as M +#ifndef mingw32_HOST_OS +import System.Posix.Signals +import System.Posix.Process (getAnyProcessStatus) +import Utility.Exception +#endif + +import Annex.Common +import qualified Annex +import Annex.Content +import Annex.CatFile +import Annex.CheckAttr +import Annex.HashObject +import Annex.CheckIgnore + +{- Actions to perform each time ran. -} +startup :: Annex () +startup = +#ifndef mingw32_HOST_OS + liftIO $ void $ installHandler sigINT Default Nothing +#else + return () +#endif + +{- Cleanup actions. -} +shutdown :: Bool -> Annex () +shutdown nocommit = do + saveState nocommit + sequence_ =<< M.elems <$> Annex.getState Annex.cleanup + stopCoProcesses + liftIO reapZombies -- zombies from long-running git processes + +{- Stops all long-running git query processes. -} +stopCoProcesses :: Annex () +stopCoProcesses = do + catFileStop + checkAttrStop + hashObjectStop + checkIgnoreStop + +{- Reaps any zombie processes that may be hanging around. + - + - Warning: Not thread safe. Anything that was expecting to wait + - on a process and get back an exit status is going to be confused + - if this reap gets there first. -} +reapZombies :: IO () +#ifndef mingw32_HOST_OS +reapZombies = + -- throws an exception when there are no child processes + catchDefaultIO Nothing (getAnyProcessStatus False True) + >>= maybe (return ()) (const reapZombies) + +#else +reapZombies = return () +#endif diff --git a/Annex/AdjustedBranch.hs b/Annex/AdjustedBranch.hs new file mode 100644 index 0000000000..aae8128c63 --- /dev/null +++ b/Annex/AdjustedBranch.hs @@ -0,0 +1,616 @@ +{- adjusted branch + - + - Copyright 2016 Joey Hess + - + - Licensed under the GNU GPL version 3 or higher. + -} + +{-# LANGUAGE BangPatterns #-} + +module Annex.AdjustedBranch ( + Adjustment(..), + OrigBranch, + AdjBranch(..), + originalToAdjusted, + adjustedToOriginal, + fromAdjustedBranch, + getAdjustment, + enterAdjustedBranch, + adjustBranch, + adjustToCrippledFileSystem, + updateAdjustedBranch, + propigateAdjustedCommits, + AdjustedClone(..), + checkAdjustedClone, + isGitVersionSupported, + checkVersionSupported, +) where + +import Annex.Common +import qualified Annex +import Git +import Git.Types +import qualified Git.Branch +import qualified Git.Ref +import qualified Git.Command +import qualified Git.Tree +import qualified Git.DiffTree +import qualified Git.Merge +import Git.Tree (TreeItem(..)) +import Git.Sha +import Git.Env +import Git.Index +import Git.FilePath +import qualified Git.LockFile +import qualified Git.Version +import Annex.Version +import Annex.CatFile +import Annex.Link +import Annex.AutoMerge +import Annex.Content +import Annex.Perms +import Annex.GitOverlay +import Utility.Tmp.Dir +import Utility.CopyFile +import qualified Database.Keys +import Config + +import qualified Data.Map as M + +data Adjustment + = UnlockAdjustment + | LockAdjustment + | FixAdjustment + | UnFixAdjustment + | HideMissingAdjustment + | ShowMissingAdjustment + deriving (Show, Eq) + +reverseAdjustment :: Adjustment -> Adjustment +reverseAdjustment UnlockAdjustment = LockAdjustment +reverseAdjustment LockAdjustment = UnlockAdjustment +reverseAdjustment HideMissingAdjustment = ShowMissingAdjustment +reverseAdjustment ShowMissingAdjustment = HideMissingAdjustment +reverseAdjustment FixAdjustment = UnFixAdjustment +reverseAdjustment UnFixAdjustment = FixAdjustment + +{- How to perform various adjustments to a TreeItem. -} +adjustTreeItem :: Adjustment -> TreeItem -> Annex (Maybe TreeItem) +adjustTreeItem UnlockAdjustment = ifSymlink adjustToPointer noAdjust +adjustTreeItem LockAdjustment = ifSymlink noAdjust adjustToSymlink +adjustTreeItem FixAdjustment = ifSymlink adjustToSymlink noAdjust +adjustTreeItem UnFixAdjustment = ifSymlink (adjustToSymlink' gitAnnexLinkCanonical) noAdjust +adjustTreeItem HideMissingAdjustment = \ti@(TreeItem _ _ s) -> + catKey s >>= \case + Just k -> ifM (inAnnex k) + ( return (Just ti) + , return Nothing + ) + Nothing -> return (Just ti) +adjustTreeItem ShowMissingAdjustment = noAdjust + +ifSymlink :: (TreeItem -> Annex a) -> (TreeItem -> Annex a) -> TreeItem -> Annex a +ifSymlink issymlink notsymlink ti@(TreeItem _f m _s) + | toTreeItemType m == Just TreeSymlink = issymlink ti + | otherwise = notsymlink ti + +noAdjust :: TreeItem -> Annex (Maybe TreeItem) +noAdjust = return . Just + +adjustToPointer :: TreeItem -> Annex (Maybe TreeItem) +adjustToPointer ti@(TreeItem f _m s) = catKey s >>= \case + Just k -> do + Database.Keys.addAssociatedFile k f + Just . TreeItem f (fromTreeItemType TreeFile) + <$> hashPointerFile k + Nothing -> return (Just ti) + +adjustToSymlink :: TreeItem -> Annex (Maybe TreeItem) +adjustToSymlink = adjustToSymlink' gitAnnexLink + +adjustToSymlink' :: (FilePath -> Key -> Git.Repo -> GitConfig -> IO FilePath) -> TreeItem -> Annex (Maybe TreeItem) +adjustToSymlink' gitannexlink ti@(TreeItem f _m s) = catKey s >>= \case + Just k -> do + absf <- inRepo $ \r -> absPath $ + fromTopFilePath f r + linktarget <- calcRepo $ gitannexlink absf k + Just . TreeItem f (fromTreeItemType TreeSymlink) + <$> hashSymlink linktarget + Nothing -> return (Just ti) + +type OrigBranch = Branch +newtype AdjBranch = AdjBranch { adjBranch :: Branch } + +-- This is a hidden branch ref, that's used as the basis for the AdjBranch, +-- since pushes can overwrite the OrigBranch at any time. So, changes +-- are propigated from the AdjBranch to the head of the BasisBranch. +newtype BasisBranch = BasisBranch Ref + +-- The basis for refs/heads/adjusted/master(unlocked) is +-- refs/basis/adjusted/master(unlocked). +basisBranch :: AdjBranch -> BasisBranch +basisBranch (AdjBranch adjbranch) = BasisBranch $ + Ref ("refs/basis/" ++ fromRef (Git.Ref.base adjbranch)) + +adjustedBranchPrefix :: String +adjustedBranchPrefix = "refs/heads/adjusted/" + +serialize :: Adjustment -> String +serialize UnlockAdjustment = "unlocked" +serialize LockAdjustment = "locked" +serialize HideMissingAdjustment = "present" +serialize ShowMissingAdjustment = "showmissing" +serialize FixAdjustment = "fixed" +serialize UnFixAdjustment = "unfixed" + +deserialize :: String -> Maybe Adjustment +deserialize "unlocked" = Just UnlockAdjustment +deserialize "locked" = Just UnlockAdjustment +deserialize "present" = Just HideMissingAdjustment +deserialize "fixed" = Just FixAdjustment +deserialize "unfixed" = Just UnFixAdjustment +deserialize _ = Nothing + +originalToAdjusted :: OrigBranch -> Adjustment -> AdjBranch +originalToAdjusted orig adj = AdjBranch $ Ref $ + adjustedBranchPrefix ++ base ++ '(' : serialize adj ++ ")" + where + base = fromRef (Git.Ref.base orig) + +adjustedToOriginal :: Branch -> Maybe (Adjustment, OrigBranch) +adjustedToOriginal b + | adjustedBranchPrefix `isPrefixOf` bs = do + let (base, as) = separate (== '(') (drop prefixlen bs) + adj <- deserialize (takeWhile (/= ')') as) + Just (adj, Git.Ref.branchRef (Ref base)) + | otherwise = Nothing + where + bs = fromRef b + prefixlen = length adjustedBranchPrefix + +getAdjustment :: Branch -> Maybe Adjustment +getAdjustment = fmap fst . adjustedToOriginal + +fromAdjustedBranch :: Branch -> OrigBranch +fromAdjustedBranch b = maybe b snd (adjustedToOriginal b) + +originalBranch :: Annex (Maybe OrigBranch) +originalBranch = fmap fromAdjustedBranch <$> inRepo Git.Branch.current + +{- Enter an adjusted version of current branch (or, if already in an + - adjusted version of a branch, changes the adjustment of the original + - branch). + - + - Can fail, if no branch is checked out, or if the adjusted branch already + - exists, or perhaps if staged changes conflict with the adjusted branch. + -} +enterAdjustedBranch :: Adjustment -> Annex Bool +enterAdjustedBranch adj = go =<< originalBranch + where + go (Just origbranch) = do + let adjbranch = adjBranch $ originalToAdjusted origbranch adj + ifM (inRepo (Git.Ref.exists adjbranch) <&&> (not <$> Annex.getState Annex.force)) + ( do + mapM_ (warning . unwords) + [ [ "adjusted branch" + , Git.Ref.describe adjbranch + , "already exists." + ] + , [ "Aborting because that branch may have changes that have not yet reached" + , Git.Ref.describe origbranch + ] + , [ "You can check out the adjusted branch manually to enter it," + , "or delete the adjusted branch and re-run this command." + ] + ] + return False + , do + AdjBranch b <- preventCommits $ const $ + adjustBranch adj origbranch + showOutput -- checkout can have output in large repos + inRepo $ Git.Command.runBool + [ Param "checkout" + , Param $ fromRef $ Git.Ref.base b + ] + ) + go Nothing = do + warning "not on any branch!" + return False + +adjustToCrippledFileSystem :: Annex () +adjustToCrippledFileSystem = do + warning "Entering an adjusted branch where files are unlocked as this filesystem does not support locked files." + whenM (isNothing <$> originalBranch) $ + void $ inRepo $ Git.Branch.commitCommand Git.Branch.AutomaticCommit + [ Param "--quiet" + , Param "--allow-empty" + , Param "-m" + , Param "commit before entering adjusted unlocked branch" + ] + unlessM (enterAdjustedBranch UnlockAdjustment) $ + warning "Failed to enter adjusted branch!" + +setBasisBranch :: BasisBranch -> Ref -> Annex () +setBasisBranch (BasisBranch basis) new = + inRepo $ Git.Branch.update' basis new + +setAdjustedBranch :: String -> AdjBranch -> Ref -> Annex () +setAdjustedBranch msg (AdjBranch b) r = inRepo $ Git.Branch.update msg b r + +adjustBranch :: Adjustment -> OrigBranch -> Annex AdjBranch +adjustBranch adj origbranch = do + -- Start basis off with the current value of the origbranch. + setBasisBranch basis origbranch + sha <- adjustCommit adj basis + setAdjustedBranch "entering adjusted branch" adjbranch sha + return adjbranch + where + adjbranch = originalToAdjusted origbranch adj + basis = basisBranch adjbranch + +adjustCommit :: Adjustment -> BasisBranch -> Annex Sha +adjustCommit adj basis = do + treesha <- adjustTree adj basis + commitAdjustedTree treesha basis + +adjustTree :: Adjustment -> BasisBranch -> Annex Sha +adjustTree adj (BasisBranch basis) = do + let toadj = adjustTreeItem adj + treesha <- Git.Tree.adjustTree toadj [] [] basis =<< Annex.gitRepo + return treesha + +type CommitsPrevented = Git.LockFile.LockHandle + +{- Locks git's index file, preventing git from making a commit, merge, + - or otherwise changing the HEAD ref while the action is run. + - + - Throws an IO exception if the index file is already locked. + -} +preventCommits :: (CommitsPrevented -> Annex a) -> Annex a +preventCommits = bracket setup cleanup + where + setup = do + lck <- fromRepo $ indexFileLock . indexFile + liftIO $ Git.LockFile.openLock lck + cleanup = liftIO . Git.LockFile.closeLock + +{- Commits a given adjusted tree, with the provided parent ref. + - + - This should always yield the same value, even if performed in different + - clones of a repo, at different times. The commit message and other + - metadata is based on the parent. + -} +commitAdjustedTree :: Sha -> BasisBranch -> Annex Sha +commitAdjustedTree treesha parent@(BasisBranch b) = + commitAdjustedTree' treesha parent [b] + +commitAdjustedTree' :: Sha -> BasisBranch -> [Ref] -> Annex Sha +commitAdjustedTree' treesha (BasisBranch basis) parents = + go =<< catCommit basis + where + go Nothing = inRepo mkcommit + go (Just basiscommit) = inRepo $ commitWithMetaData + (commitAuthorMetaData basiscommit) + (commitCommitterMetaData basiscommit) + mkcommit + mkcommit = Git.Branch.commitTree Git.Branch.AutomaticCommit + adjustedBranchCommitMessage parents treesha + +{- This message should never be changed. -} +adjustedBranchCommitMessage :: String +adjustedBranchCommitMessage = "git-annex adjusted branch" + +findAdjustingCommit :: AdjBranch -> Annex (Maybe Commit) +findAdjustingCommit (AdjBranch b) = go =<< catCommit b + where + go Nothing = return Nothing + go (Just c) + | commitMessage c == adjustedBranchCommitMessage = return (Just c) + | otherwise = case commitParent c of + [p] -> go =<< catCommit p + _ -> return Nothing + +{- Update the currently checked out adjusted branch, merging the provided + - branch into it. Note that the provided branch should be a non-adjusted + - branch. -} +updateAdjustedBranch :: Branch -> (OrigBranch, Adjustment) -> [Git.Merge.MergeConfig] -> Annex Bool -> Git.Branch.CommitMode -> Annex Bool +updateAdjustedBranch tomerge (origbranch, adj) mergeconfig canresolvemerge commitmode = catchBoolIO $ + join $ preventCommits go + where + adjbranch@(AdjBranch currbranch) = originalToAdjusted origbranch adj + basis = basisBranch adjbranch + + go commitsprevented = + ifM (inRepo $ Git.Branch.changed currbranch tomerge) + ( do + (updatedorig, _) <- propigateAdjustedCommits' + origbranch adj commitsprevented + changestomerge updatedorig + , nochangestomerge + ) + + nochangestomerge = return $ return True + + {- Since the adjusted branch changes files, merging tomerge + - directly into it would likely result in unncessary merge + - conflicts. To avoid those conflicts, instead merge tomerge into + - updatedorig. The result of the merge can the be + - adjusted to yield the final adjusted branch. + - + - In order to do a merge into a ref that is not checked out, + - set the work tree to a temp directory, and set GIT_DIR + - to another temp directory, in which HEAD contains the + - updatedorig sha. GIT_COMMON_DIR is set to point to the real + - git directory, and so git can read and write objects from there, + - but will use GIT_DIR for HEAD and index. + - + - (Doing the merge this way also lets it run even though the main + - index file is currently locked.) + -} + changestomerge (Just updatedorig) = do + misctmpdir <- fromRepo gitAnnexTmpMiscDir + void $ createAnnexDirectory misctmpdir + tmpwt <- fromRepo gitAnnexMergeDir + git_dir <- fromRepo Git.localGitDir + withTmpDirIn misctmpdir "git" $ \tmpgit -> withWorkTreeRelated tmpgit $ + withemptydir tmpwt $ withWorkTree tmpwt $ do + liftIO $ writeFile (tmpgit "HEAD") (fromRef updatedorig) + -- Copy in refs and packed-refs, to work + -- around bug in git 2.13.0, which + -- causes it not to look in GIT_DIR for refs. + refs <- liftIO $ dirContentsRecursive $ + git_dir "refs" + let refs' = (git_dir "packed-refs") : refs + liftIO $ forM_ refs' $ \src -> + whenM (doesFileExist src) $ do + dest <- relPathDirToFile git_dir src + let dest' = tmpgit dest + createDirectoryIfMissing True (takeDirectory dest') + void $ createLinkOrCopy src dest' + -- This reset makes git merge not care + -- that the work tree is empty; otherwise + -- it will think that all the files have + -- been staged for deletion, and sometimes + -- the merge includes these deletions + -- (for an unknown reason). + -- http://thread.gmane.org/gmane.comp.version-control.git/297237 + inRepo $ Git.Command.run [Param "reset", Param "HEAD", Param "--quiet"] + showAction $ "Merging into " ++ fromRef (Git.Ref.base origbranch) + merged <- inRepo (Git.Merge.merge' [] tomerge mergeconfig commitmode) + <||> (resolveMerge (Just updatedorig) tomerge True <&&> commitResolvedMerge commitmode) + if merged + then do + !mergecommit <- liftIO $ extractSha <$> readFile (tmpgit "HEAD") + -- This is run after the commit lock is dropped. + return $ postmerge mergecommit + else return $ return False + changestomerge Nothing = return $ return False + + withemptydir d a = bracketIO setup cleanup (const a) + where + setup = do + whenM (doesDirectoryExist d) $ + removeDirectoryRecursive d + createDirectoryIfMissing True d + cleanup _ = removeDirectoryRecursive d + + {- A merge commit has been made between the basisbranch and + - tomerge. Update the basisbranch and origbranch to point + - to that commit, adjust it to get the new adjusted branch, + - and check it out. + - + - But, there may be unstaged work tree changes that conflict, + - so the check out is done by making a normal merge of + - the new adjusted branch. + -} + postmerge (Just mergecommit) = do + setBasisBranch basis mergecommit + inRepo $ Git.Branch.update' origbranch mergecommit + adjtree <- adjustTree adj (BasisBranch mergecommit) + adjmergecommit <- commitAdjustedTree adjtree (BasisBranch mergecommit) + -- Make currbranch be the parent, so that merging + -- this commit will be a fast-forward. + adjmergecommitff <- commitAdjustedTree' adjtree (BasisBranch mergecommit) [currbranch] + showAction "Merging into adjusted branch" + ifM (autoMergeFrom adjmergecommitff (Just currbranch) mergeconfig canresolvemerge commitmode) + ( reparent adjtree adjmergecommit =<< getcurrentcommit + , return False + ) + postmerge Nothing = return False + + -- Now that the merge into the adjusted branch is complete, + -- take the tree from that merge, and attach it on top of the + -- adjmergecommit, if it's different. + reparent adjtree adjmergecommit (Just currentcommit) = do + if (commitTree currentcommit /= adjtree) + then do + c <- inRepo $ Git.Branch.commitTree Git.Branch.AutomaticCommit + ("Merged " ++ fromRef tomerge) [adjmergecommit] + (commitTree currentcommit) + inRepo $ Git.Branch.update "updating adjusted branch" currbranch c + propigateAdjustedCommits origbranch adj + else inRepo $ Git.Branch.update "updating adjusted branch" currbranch adjmergecommit + return True + reparent _ _ Nothing = return False + + getcurrentcommit = inRepo Git.Branch.currentUnsafe >>= \case + Nothing -> return Nothing + Just c -> catCommit c + +{- Check for any commits present on the adjusted branch that have not yet + - been propigated to the basis branch, and propigate them to the basis + - branch and from there on to the orig branch. + - + - After propigating the commits back to the basis banch, + - rebase the adjusted branch on top of the updated basis branch. + -} +propigateAdjustedCommits :: OrigBranch -> Adjustment -> Annex () +propigateAdjustedCommits origbranch adj = + preventCommits $ \commitsprevented -> + join $ snd <$> propigateAdjustedCommits' origbranch adj commitsprevented + +{- Returns sha of updated basis branch, and action which will rebase + - the adjusted branch on top of the updated basis branch. -} +propigateAdjustedCommits' + :: OrigBranch + -> Adjustment + -> CommitsPrevented + -> Annex (Maybe Sha, Annex ()) +propigateAdjustedCommits' origbranch adj _commitsprevented = + inRepo (Git.Ref.sha basis) >>= \case + Just origsha -> catCommit currbranch >>= \case + Just currcommit -> + newcommits >>= go origsha False >>= \case + Left e -> do + warning e + return (Nothing, return ()) + Right newparent -> return + ( Just newparent + , rebase currcommit newparent + ) + Nothing -> return (Nothing, return ()) + Nothing -> return (Nothing, return ()) + where + (BasisBranch basis) = basisBranch adjbranch + adjbranch@(AdjBranch currbranch) = originalToAdjusted origbranch adj + newcommits = inRepo $ Git.Branch.changedCommits basis currbranch + -- Get commits oldest first, so they can be processed + -- in order made. + [Param "--reverse"] + go parent _ [] = do + setBasisBranch (BasisBranch basis) parent + inRepo $ Git.Branch.update' origbranch parent + return (Right parent) + go parent pastadjcommit (sha:l) = catCommit sha >>= \case + Just c + | commitMessage c == adjustedBranchCommitMessage -> + go parent True l + | pastadjcommit -> + reverseAdjustedCommit parent adj (sha, c) origbranch + >>= \case + Left e -> return (Left e) + Right commit -> go commit pastadjcommit l + _ -> go parent pastadjcommit l + rebase currcommit newparent = do + -- Reuse the current adjusted tree, and reparent it + -- on top of the newparent. + commitAdjustedTree (commitTree currcommit) (BasisBranch newparent) + >>= inRepo . Git.Branch.update rebaseOnTopMsg currbranch + +rebaseOnTopMsg :: String +rebaseOnTopMsg = "rebasing adjusted branch on top of updated original branch" + +{- Reverses an adjusted commit, and commit with provided commitparent, + - yielding a commit sha. + - + - Adjusts the tree of the commitparent, changing only the files that the + - commit changed, and reverse adjusting those changes. + - + - The commit message, and the author and committer metadata are + - copied over from the basiscommit. However, any gpg signature + - will be lost, and any other headers are not copied either. -} +reverseAdjustedCommit :: Sha -> Adjustment -> (Sha, Commit) -> OrigBranch -> Annex (Either String Sha) +reverseAdjustedCommit commitparent adj (csha, basiscommit) origbranch + | length (commitParent basiscommit) > 1 = return $ + Left $ "unable to propigate merge commit " ++ show csha ++ " back to " ++ show origbranch + | otherwise = do + treesha <- reverseAdjustedTree commitparent adj csha + revadjcommit <- inRepo $ commitWithMetaData + (commitAuthorMetaData basiscommit) + (commitCommitterMetaData basiscommit) $ + Git.Branch.commitTree Git.Branch.AutomaticCommit + (commitMessage basiscommit) [commitparent] treesha + return (Right revadjcommit) + +{- Adjusts the tree of the basis, changing only the files that the + - commit changed, and reverse adjusting those changes. + - + - commitDiff does not support merge commits, so the csha must not be a + - merge commit. -} +reverseAdjustedTree :: Sha -> Adjustment -> Sha -> Annex Sha +reverseAdjustedTree basis adj csha = do + (diff, cleanup) <- inRepo (Git.DiffTree.commitDiff csha) + let (adds, others) = partition (\dti -> Git.DiffTree.srcsha dti == nullSha) diff + let (removes, changes) = partition (\dti -> Git.DiffTree.dstsha dti == nullSha) others + adds' <- catMaybes <$> + mapM (adjustTreeItem reverseadj) (map diffTreeToTreeItem adds) + treesha <- Git.Tree.adjustTree + (propchanges changes) + adds' + (map Git.DiffTree.file removes) + basis + =<< Annex.gitRepo + void $ liftIO cleanup + return treesha + where + reverseadj = reverseAdjustment adj + propchanges changes ti@(TreeItem f _ _) = + case M.lookup (norm f) m of + Nothing -> return (Just ti) -- not changed + Just change -> adjustTreeItem reverseadj change + where + m = M.fromList $ map (\i@(TreeItem f' _ _) -> (norm f', i)) $ + map diffTreeToTreeItem changes + norm = normalise . getTopFilePath + +diffTreeToTreeItem :: Git.DiffTree.DiffTreeItem -> TreeItem +diffTreeToTreeItem dti = TreeItem + (Git.DiffTree.file dti) + (Git.DiffTree.dstmode dti) + (Git.DiffTree.dstsha dti) + +data AdjustedClone = InAdjustedClone | NotInAdjustedClone | NeedUpgradeForAdjustedClone + +{- Cloning a repository that has an adjusted branch checked out will + - result in the clone having the same adjusted branch checked out -- but + - the origbranch won't exist in the clone, nor will the basis. So + - to properly set up the adjusted branch, the origbranch and basis need + - to be set. + - + - We can't trust that the origin's origbranch matches up with the currently + - checked out adjusted branch; the origin could have the two branches + - out of sync (eg, due to another branch having been pushed to the origin's + - origbranch), or due to a commit on its adjusted branch not having been + - propigated back to origbranch. + - + - So, find the adjusting commit on the currently checked out adjusted + - branch, and use the parent of that commit as the basis, and set the + - origbranch to it. + - + - The repository may also need to be upgraded to a new version, if the + - current version is too old to support adjusted branches. -} +checkAdjustedClone :: Annex AdjustedClone +checkAdjustedClone = ifM isBareRepo + ( return NotInAdjustedClone + , go =<< inRepo Git.Branch.current + ) + where + go Nothing = return NotInAdjustedClone + go (Just currbranch) = case adjustedToOriginal currbranch of + Nothing -> return NotInAdjustedClone + Just (adj, origbranch) -> do + let basis@(BasisBranch bb) = basisBranch (originalToAdjusted origbranch adj) + unlessM (inRepo $ Git.Ref.exists bb) $ do + unlessM (inRepo $ Git.Ref.exists origbranch) $ do + let remotebranch = Git.Ref.underBase "refs/remotes/origin" origbranch + inRepo $ Git.Branch.update' origbranch remotebranch + aps <- fmap commitParent <$> findAdjustingCommit (AdjBranch currbranch) + case aps of + Just [p] -> setBasisBranch basis p + _ -> giveup $ "Unable to clean up from clone of adjusted branch; perhaps you should check out " ++ Git.Ref.describe origbranch + ifM versionSupportsUnlockedPointers + ( return InAdjustedClone + , return NeedUpgradeForAdjustedClone + ) + +-- git 2.2.0 needed for GIT_COMMON_DIR which is needed +-- by updateAdjustedBranch to use withWorkTreeRelated. +isGitVersionSupported :: IO Bool +isGitVersionSupported = not <$> Git.Version.older "2.2.0" + +checkVersionSupported :: Annex () +checkVersionSupported = do + unlessM versionSupportsAdjustedBranch $ + giveup "Adjusted branches are only supported in v6 or newer repositories." + unlessM (liftIO isGitVersionSupported) $ + giveup "Your version of git is too old; upgrade it to 2.2.0 or newer to use adjusted branches." diff --git a/Annex/AutoMerge.hs b/Annex/AutoMerge.hs new file mode 100644 index 0000000000..904ae2e890 --- /dev/null +++ b/Annex/AutoMerge.hs @@ -0,0 +1,354 @@ +{- git-annex automatic merge conflict resolution + - + - Copyright 2012-2016 Joey Hess + - + - Licensed under the GNU GPL version 3 or higher. + -} + +module Annex.AutoMerge + ( autoMergeFrom + , resolveMerge + , commitResolvedMerge + ) where + +import Annex.Common +import qualified Annex.Queue +import Annex.Direct +import Annex.CatFile +import Annex.Link +import Annex.Content +import qualified Git.LsFiles as LsFiles +import qualified Git.UpdateIndex as UpdateIndex +import qualified Git.Merge +import qualified Git.Ref +import qualified Git +import qualified Git.Branch +import Git.Types (TreeItemType(..), fromTreeItemType) +import Git.FilePath +import Config +import Annex.ReplaceFile +import Annex.VariantFile +import qualified Database.Keys +import Annex.InodeSentinal +import Utility.InodeCache +import Utility.FileMode + +import qualified Data.Set as S +import qualified Data.Map as M +import qualified Data.ByteString.Lazy as L + +{- Merges from a branch into the current branch (which may not exist yet), + - with automatic merge conflict resolution. + - + - Callers should use Git.Branch.changed first, to make sure that + - there are changes from the current branch to the branch being merged in. + -} +autoMergeFrom :: Git.Ref -> Maybe Git.Ref -> [Git.Merge.MergeConfig] -> Annex Bool -> Git.Branch.CommitMode -> Annex Bool +autoMergeFrom branch currbranch mergeconfig canresolvemerge commitmode = do + showOutput + case currbranch of + Nothing -> go Nothing + Just b -> go =<< inRepo (Git.Ref.sha b) + where + go old = ifM isDirect + ( mergeDirect currbranch old branch resolvemerge mergeconfig commitmode + , do + r <- inRepo (Git.Merge.merge branch mergeconfig commitmode) + <||> (resolvemerge <&&> commitResolvedMerge commitmode) + -- Merging can cause new associated files to appear + -- and the smudge filter will add them to the database. + -- To ensure that this process sees those changes, + -- close the database if it was open. + Database.Keys.closeDb + return r + ) + where + resolvemerge = ifM canresolvemerge + ( resolveMerge old branch False + , return False + ) + +{- Resolves a conflicted merge. It's important that any conflicts be + - resolved in a way that itself avoids later merge conflicts, since + - multiple repositories may be doing this concurrently. + - + - Only merge conflicts where at least one side is an annexed file + - is resolved. + - + - This uses the Keys pointed to by the files to construct new + - filenames. So when both sides modified annexed file foo, + - it will be deleted, and replaced with files foo.variant-A and + - foo.variant-B. + - + - On the other hand, when one side deleted foo, and the other modified it, + - it will be deleted, and the modified version stored as file + - foo.variant-A (or B). + - + - It's also possible that one side has foo as an annexed file, and + - the other as a directory or non-annexed file. The annexed file + - is renamed to resolve the merge, and the other object is preserved as-is. + - + - In indirect mode, the merge is resolved in the work tree and files + - staged, to clean up from a conflicted merge that was run in the work + - tree. + - + - In direct mode, the work tree is not touched here; files are staged to + - the index, and written to the gitAnnexMergeDir, for later handling by + - the direct mode merge code. + - + - This is complicated by needing to support merges run in an overlay + - work tree, in which case the CWD won't be within the work tree. + - In this mode, there is no need to update the work tree at all, + - as the overlay work tree will get deleted. + - + - Unlocked files remain unlocked after merging, and locked files + - remain locked. When the merge conflict is between a locked and unlocked + - file, that otherwise point to the same content, the unlocked mode wins. + - This is done because only unlocked files work in filesystems that don't + - support symlinks. + - + - Returns false when there are no merge conflicts to resolve. + - A git merge can fail for other reasons, and this allows detecting + - such failures. + -} +resolveMerge :: Maybe Git.Ref -> Git.Ref -> Bool -> Annex Bool +resolveMerge us them inoverlay = do + top <- if inoverlay + then pure "." + else fromRepo Git.repoPath + (fs, cleanup) <- inRepo (LsFiles.unmerged [top]) + srcmap <- if inoverlay + then pure M.empty + else inodeMap $ pure (map LsFiles.unmergedFile fs, return True) + (mergedks, mergedfs) <- unzip <$> mapM (resolveMerge' srcmap us them inoverlay) fs + let mergedks' = concat mergedks + let mergedfs' = catMaybes mergedfs + let merged = not (null mergedfs') + void $ liftIO cleanup + + unlessM (pure inoverlay <||> isDirect) $ do + (deleted, cleanup2) <- inRepo (LsFiles.deleted [top]) + unless (null deleted) $ + Annex.Queue.addCommand "rm" + [Param "--quiet", Param "-f", Param "--"] + deleted + void $ liftIO cleanup2 + + when merged $ do + Annex.Queue.flush + unlessM (pure inoverlay <||> isDirect) $ do + unstagedmap <- inodeMap $ inRepo $ LsFiles.notInRepo False [top] + cleanConflictCruft mergedks' mergedfs' unstagedmap + showLongNote "Merge conflict was automatically resolved; you may want to examine the result." + return merged + +resolveMerge' :: InodeMap -> Maybe Git.Ref -> Git.Ref -> Bool -> LsFiles.Unmerged -> Annex ([Key], Maybe FilePath) +resolveMerge' _ Nothing _ _ _ = return ([], Nothing) +resolveMerge' unstagedmap (Just us) them inoverlay u = do + kus <- getkey LsFiles.valUs + kthem <- getkey LsFiles.valThem + case (kus, kthem) of + -- Both sides of conflict are annexed files + (Just keyUs, Just keyThem) + | keyUs /= keyThem -> resolveby [keyUs, keyThem] $ do + makeannexlink keyUs LsFiles.valUs + makeannexlink keyThem LsFiles.valThem + -- cleanConflictCruft can't handle unlocked + -- files, so delete here. + unless inoverlay $ + unless (islocked LsFiles.valUs) $ + liftIO $ nukeFile file + | otherwise -> do + -- Only resolve using symlink when both + -- were locked, otherwise use unlocked + -- pointer. + -- In either case, keep original filename. + if islocked LsFiles.valUs && islocked LsFiles.valThem + then makesymlink keyUs file + else makepointer keyUs file (combinedmodes) + return ([keyUs, keyThem], Just file) + -- Our side is annexed file, other side is not. + (Just keyUs, Nothing) -> resolveby [keyUs] $ do + graftin them file LsFiles.valThem LsFiles.valThem LsFiles.valUs + makeannexlink keyUs LsFiles.valUs + -- Our side is not annexed file, other side is. + (Nothing, Just keyThem) -> resolveby [keyThem] $ do + graftin us file LsFiles.valUs LsFiles.valUs LsFiles.valThem + makeannexlink keyThem LsFiles.valThem + -- Neither side is annexed file; cannot resolve. + (Nothing, Nothing) -> return ([], Nothing) + where + file = LsFiles.unmergedFile u + + getkey select = + case select (LsFiles.unmergedSha u) of + Just sha -> catKey sha + Nothing -> return Nothing + + islocked select = select (LsFiles.unmergedTreeItemType u) == Just TreeSymlink + + combinedmodes = case catMaybes [ourmode, theirmode] of + [] -> Nothing + l -> Just (combineModes l) + where + ourmode = fromTreeItemType + <$> LsFiles.valUs (LsFiles.unmergedTreeItemType u) + theirmode = fromTreeItemType + <$> LsFiles.valThem (LsFiles.unmergedTreeItemType u) + + makeannexlink key select + | islocked select = makesymlink key dest + | otherwise = makepointer key dest destmode + where + dest = variantFile file key + destmode = fromTreeItemType <$> select (LsFiles.unmergedTreeItemType u) + + stagefile :: FilePath -> Annex FilePath + stagefile f + | inoverlay = ( f) <$> fromRepo Git.repoPath + | otherwise = pure f + + makesymlink key dest = do + l <- calcRepo $ gitAnnexLink dest key + unless inoverlay $ replacewithsymlink dest l + dest' <- stagefile dest + stageSymlink dest' =<< hashSymlink l + + replacewithsymlink dest link = withworktree dest $ \f -> + replaceFile f $ makeGitLink link + + makepointer key dest destmode = do + unless inoverlay $ + unlessM (reuseOldFile unstagedmap key file dest) $ + linkFromAnnex key dest destmode >>= \case + LinkAnnexFailed -> liftIO $ + writePointerFile dest key destmode + _ -> noop + dest' <- stagefile dest + stagePointerFile dest' destmode =<< hashPointerFile key + unless inoverlay $ + Database.Keys.addAssociatedFile key + =<< inRepo (toTopFilePath dest) + + withworktree f a = ifM isDirect + ( do + d <- fromRepo gitAnnexMergeDir + a (d f) + , a f + ) + + {- Stage a graft of a directory or file from a branch + - and update the work tree. -} + graftin b item selectwant selectwant' selectunwant = do + Annex.Queue.addUpdateIndex + =<< fromRepo (UpdateIndex.lsSubTree b item) + + -- Update the work tree to reflect the graft. + unless inoverlay $ case (selectwant (LsFiles.unmergedTreeItemType u), selectunwant (LsFiles.unmergedTreeItemType u)) of + -- Symlinks are never left in work tree when + -- there's a conflict with anything else. + -- So, when grafting in a symlink, we must create it: + (Just TreeSymlink, _) -> do + case selectwant' (LsFiles.unmergedSha u) of + Nothing -> noop + Just sha -> do + link <- catSymLinkTarget sha + replacewithsymlink item link + -- And when grafting in anything else vs a symlink, + -- the work tree already contains what we want. + (_, Just TreeSymlink) -> noop + _ -> ifM (withworktree item (liftIO . doesDirectoryExist)) + -- a conflict between a file and a directory + -- leaves the directory, so since a directory + -- is there, it must be what was wanted + ( noop + -- probably a file with conflict markers is + -- in the work tree; replace with grafted + -- file content + , case selectwant' (LsFiles.unmergedSha u) of + Nothing -> noop + Just sha -> withworktree item $ \f -> + replaceFile f $ \tmp -> do + c <- catObject sha + liftIO $ L.writeFile tmp c + ) + + resolveby ks a = do + {- Remove conflicted file from index so merge can be resolved. -} + Annex.Queue.addCommand "rm" + [Param "--quiet", Param "-f", Param "--cached", Param "--"] [file] + void a + return (ks, Just file) + +{- git-merge moves conflicting files away to files + - named something like f~HEAD or f~branch or just f, but the + - exact name chosen can vary. Once the conflict is resolved, + - this cruft can be deleted. To avoid deleting legitimate + - files that look like this, only delete files that are + - A) not staged in git and + - B) have a name related to the merged files and + - C) are pointers to or have the content of keys that were involved + - in the merge. + -} +cleanConflictCruft :: [Key] -> [FilePath] -> InodeMap -> Annex () +cleanConflictCruft resolvedks resolvedfs unstagedmap = do + is <- S.fromList . map (inodeCacheToKey Strongly) . concat + <$> mapM Database.Keys.getInodeCaches resolvedks + forM_ (M.toList unstagedmap) $ \(i, f) -> + whenM (matchesresolved is i f) $ + liftIO $ nukeFile f + where + fs = S.fromList resolvedfs + ks = S.fromList resolvedks + inks = maybe False (flip S.member ks) + matchesresolved is i f + | S.member f fs || S.member (conflictCruftBase f) fs = anyM id + [ pure (S.member i is) + , inks <$> isAnnexLink f + , inks <$> liftIO (isPointerFile f) + ] + | otherwise = return False + +conflictCruftBase :: FilePath -> FilePath +conflictCruftBase f = reverse $ drop 1 $ dropWhile (/= '~') $ reverse f + +{- When possible, reuse an existing file from the srcmap as the + - content of a worktree file in the resolved merge. It must have the + - same name as the origfile, or a name that git would use for conflict + - cruft. And, its inode cache must be a known one for the key. -} +reuseOldFile :: InodeMap -> Key -> FilePath -> FilePath -> Annex Bool +reuseOldFile srcmap key origfile destfile = do + is <- map (inodeCacheToKey Strongly) + <$> Database.Keys.getInodeCaches key + liftIO $ go $ mapMaybe (\i -> M.lookup i srcmap) is + where + go [] = return False + go (f:fs) + | f == origfile || conflictCruftBase f == origfile = + ifM (doesFileExist f) + ( do + renameFile f destfile + return True + , go fs + ) + | otherwise = go fs + +commitResolvedMerge :: Git.Branch.CommitMode -> Annex Bool +commitResolvedMerge commitmode = inRepo $ Git.Branch.commitCommand commitmode + [ Param "--no-verify" + , Param "-m" + , Param "git-annex automatic merge conflict fix" + ] + +type InodeMap = M.Map InodeCacheKey FilePath + +inodeMap :: Annex ([FilePath], IO Bool) -> Annex InodeMap +inodeMap getfiles = do + (fs, cleanup) <- getfiles + fsis <- forM fs $ \f -> do + mi <- withTSDelta (liftIO . genInodeCache f) + return $ case mi of + Nothing -> Nothing + Just i -> Just (inodeCacheToKey Strongly i, f) + void $ liftIO cleanup + return $ M.fromList $ catMaybes fsis diff --git a/Annex/BloomFilter.hs b/Annex/BloomFilter.hs new file mode 100644 index 0000000000..af467503a0 --- /dev/null +++ b/Annex/BloomFilter.hs @@ -0,0 +1,53 @@ +{- git-annex bloom filter + - + - Copyright 2010-2015 Joey Hess + - + - Licensed under the GNU GPL version 3 or higher. + -} + +module Annex.BloomFilter where + +import Annex.Common +import qualified Annex +import Utility.Bloom + +import Control.Monad.ST + +{- A bloom filter capable of holding half a million keys with a + - false positive rate of 1 in 10000000 uses around 16 mb of memory, + - so will easily fit on even my lowest memory systems. + -} +bloomCapacity :: Annex Int +bloomCapacity = fromMaybe 500000 . annexBloomCapacity <$> Annex.getGitConfig +bloomAccuracy :: Annex Int +bloomAccuracy = fromMaybe 10000000 . annexBloomAccuracy <$> Annex.getGitConfig +bloomBitsHashes :: Annex (Int, Int) +bloomBitsHashes = do + capacity <- bloomCapacity + accuracy <- bloomAccuracy + case safeSuggestSizing capacity (1 / fromIntegral accuracy) of + Left e -> do + warning $ "bloomfilter " ++ e ++ "; falling back to sane value" + -- precaulculated value for 500000 (1/10000000) + return (16777216,23) + Right v -> return v + +{- Creates a bloom filter, and runs an action to populate it. + - + - The action is passed a callback that it can use to feed values into the + - bloom filter. + - + - Once the action completes, the mutable filter is frozen + - for later use. + -} +genBloomFilter :: Hashable v => ((v -> Annex ()) -> Annex ()) -> Annex (Bloom v) +genBloomFilter populate = do + (numbits, numhashes) <- bloomBitsHashes + bloom <- lift $ newMB (cheapHashes numhashes) numbits + populate $ \v -> lift $ insertMB bloom v + lift $ unsafeFreezeMB bloom + where + lift = liftIO . stToIO + +bloomFilter :: [v] -> Bloom v -> [v] +bloomFilter l bloom = filter (\v -> v `notElemB` bloom) l diff --git a/Annex/Branch.hs b/Annex/Branch.hs new file mode 100644 index 0000000000..e465b75326 --- /dev/null +++ b/Annex/Branch.hs @@ -0,0 +1,662 @@ +{- management of the git-annex branch + - + - Copyright 2011-2018 Joey Hess + - + - Licensed under the GNU GPL version 3 or higher. + -} + +module Annex.Branch ( + fullname, + name, + hasOrigin, + hasSibling, + siblingBranches, + create, + update, + forceUpdate, + updateTo, + get, + getHistorical, + change, + maybeChange, + commitMessage, + commit, + forceCommit, + getBranch, + files, + graftTreeish, + performTransitions, + withIndex, +) where + +import qualified Data.ByteString.Lazy as L +import qualified Data.Set as S +import qualified Data.Map as M +import Data.Function +import Data.Char +import Control.Concurrent (threadDelay) + +import Annex.Common +import Annex.BranchState +import Annex.Journal +import Annex.GitOverlay +import qualified Git +import qualified Git.Command +import qualified Git.Ref +import qualified Git.RefLog +import qualified Git.Sha +import qualified Git.Branch +import qualified Git.UnionMerge +import qualified Git.UpdateIndex +import qualified Git.Tree +import Git.LsTree (lsTreeParams) +import qualified Git.HashObject +import Annex.HashObject +import Git.Types (Ref(..), fromRef, RefDate, TreeItemType(..)) +import Git.FilePath +import Annex.CatFile +import Annex.Perms +import Logs +import Logs.Transitions +import Logs.File +import Logs.Trust.Pure +import Logs.Difference.Pure +import qualified Annex.Queue +import Annex.Branch.Transitions +import qualified Annex +import Annex.Hook +import Utility.FileSystemEncoding +import Utility.Directory.Stream + +{- Name of the branch that is used to store git-annex's information. -} +name :: Git.Ref +name = Git.Ref "git-annex" + +{- Fully qualified name of the branch. -} +fullname :: Git.Ref +fullname = Git.Ref $ "refs/heads/" ++ fromRef name + +{- Branch's name in origin. -} +originname :: Git.Ref +originname = Git.Ref $ "origin/" ++ fromRef name + +{- Does origin/git-annex exist? -} +hasOrigin :: Annex Bool +hasOrigin = inRepo $ Git.Ref.exists originname + +{- Does the git-annex branch or a sibling foo/git-annex branch exist? -} +hasSibling :: Annex Bool +hasSibling = not . null <$> siblingBranches + +{- List of git-annex (shas, branches), including the main one and any + - from remotes. Duplicates are filtered out. -} +siblingBranches :: Annex [(Git.Sha, Git.Branch)] +siblingBranches = inRepo $ Git.Ref.matchingUniq [name] + +{- Creates the branch, if it does not already exist. -} +create :: Annex () +create = void getBranch + +{- Returns the ref of the branch, creating it first if necessary. -} +getBranch :: Annex Git.Ref +getBranch = maybe (hasOrigin >>= go >>= use) return =<< branchsha + where + go True = do + inRepo $ Git.Command.run + [Param "branch", Param $ fromRef name, Param $ fromRef originname] + fromMaybe (error $ "failed to create " ++ fromRef name) + <$> branchsha + go False = withIndex' True $ + inRepo $ Git.Branch.commitAlways Git.Branch.AutomaticCommit "branch created" fullname [] + use sha = do + setIndexSha sha + return sha + branchsha = inRepo $ Git.Ref.sha fullname + +{- Ensures that the branch and index are up-to-date; should be + - called before data is read from it. Runs only once per git-annex run. -} +update :: Annex () +update = runUpdateOnce $ void $ updateTo =<< siblingBranches + +{- Forces an update even if one has already been run. -} +forceUpdate :: Annex Bool +forceUpdate = updateTo =<< siblingBranches + +{- Merges the specified Refs into the index, if they have any changes not + - already in it. The Branch names are only used in the commit message; + - it's even possible that the provided Branches have not been updated to + - point to the Refs yet. + - + - The branch is fast-forwarded if possible, otherwise a merge commit is + - made. + - + - Before Refs are merged into the index, it's important to first stage the + - journal into the index. Otherwise, any changes in the journal would + - later get staged, and might overwrite changes made during the merge. + - This is only done if some of the Refs do need to be merged. + - + - Also handles performing any Transitions that have not yet been + - performed, in either the local branch, or the Refs. + - + - Returns True if any refs were merged in, False otherwise. + -} +updateTo :: [(Git.Sha, Git.Branch)] -> Annex Bool +updateTo pairs = ifM (annexMergeAnnexBranches <$> Annex.getGitConfig) + ( updateTo' pairs + , return False + ) + +updateTo' :: [(Git.Sha, Git.Branch)] -> Annex Bool +updateTo' pairs = do + -- ensure branch exists, and get its current ref + branchref <- getBranch + dirty <- journalDirty + ignoredrefs <- getIgnoredRefs + let unignoredrefs = excludeset ignoredrefs pairs + tomerge <- if null unignoredrefs + then return [] + else do + mergedrefs <- getMergedRefs + filterM isnewer (excludeset mergedrefs unignoredrefs) + if null tomerge + {- Even when no refs need to be merged, the index + - may still be updated if the branch has gotten ahead + - of the index. -} + then do + whenM (needUpdateIndex branchref) $ lockJournal $ \jl -> do + forceUpdateIndex jl branchref + {- When there are journalled changes + - as well as the branch being updated, + - a commit needs to be done. -} + when dirty $ + go branchref True [] jl + else lockJournal $ go branchref dirty tomerge + return $ not $ null tomerge + where + excludeset s = filter (\(r, _) -> S.notMember r s) + isnewer (r, _) = inRepo $ Git.Branch.changed fullname r + go branchref dirty tomerge jl = withIndex $ do + let (refs, branches) = unzip tomerge + cleanjournal <- if dirty then stageJournal jl else return noop + merge_desc <- if null tomerge + then commitMessage + else return $ "merging " ++ + unwords (map Git.Ref.describe branches) ++ + " into " ++ fromRef name + localtransitions <- parseTransitionsStrictly "local" + <$> getLocal transitionsLog + unless (null tomerge) $ do + showSideAction merge_desc + mapM_ checkBranchDifferences refs + mergeIndex jl refs + let commitrefs = nub $ fullname:refs + ifM (handleTransitions jl localtransitions commitrefs) + ( runAnnexHook postUpdateAnnexHook + , do + ff <- if dirty + then return False + else inRepo $ Git.Branch.fastForward fullname refs + if ff + then updateIndex jl branchref + else commitIndex jl branchref merge_desc commitrefs + ) + addMergedRefs tomerge + liftIO cleanjournal + +{- Gets the content of a file, which may be in the journal, or in the index + - (and committed to the branch). + - + - Updates the branch if necessary, to ensure the most up-to-date available + - content is returned. + - + - Returns an empty string if the file doesn't exist yet. -} +get :: FilePath -> Annex String +get file = do + update + getLocal file + +{- Like get, but does not merge the branch, so the info returned may not + - reflect changes in remotes. + - (Changing the value this returns, and then merging is always the + - same as using get, and then changing its value.) -} +getLocal :: FilePath -> Annex String +getLocal file = go =<< getJournalFileStale file + where + go (Just journalcontent) = return journalcontent + go Nothing = getRef fullname file + +{- Gets the content of a file as staged in the branch's index. -} +getStaged :: FilePath -> Annex String +getStaged = getRef indexref + where + -- This makes git cat-file be run with ":file", + -- so it looks at the index. + indexref = Ref "" + +getHistorical :: RefDate -> FilePath -> Annex String +getHistorical date file = + -- This check avoids some ugly error messages when the reflog + -- is empty. + ifM (null <$> inRepo (Git.RefLog.get' [Param (fromRef fullname), Param "-n1"])) + ( giveup ("No reflog for " ++ fromRef fullname) + , getRef (Git.Ref.dateRef fullname date) file + ) + +getRef :: Ref -> FilePath -> Annex String +getRef ref file = withIndex $ decodeBS <$> catFile ref file + +{- Applies a function to modify the content of a file. + - + - Note that this does not cause the branch to be merged, it only + - modifes the current content of the file on the branch. + -} +change :: FilePath -> (String -> String) -> Annex () +change file f = lockJournal $ \jl -> f <$> getLocal file >>= set jl file + +{- Applies a function which can modify the content of a file, or not. -} +maybeChange :: FilePath -> (String -> Maybe String) -> Annex () +maybeChange file f = lockJournal $ \jl -> do + v <- getLocal file + case f v of + Just v' | v' /= v -> set jl file v' + _ -> noop + +{- Records new content of a file into the journal -} +set :: JournalLocked -> FilePath -> String -> Annex () +set = setJournalFile + +{- Commit message used when making a commit of whatever data has changed + - to the git-annex brach. -} +commitMessage :: Annex String +commitMessage = fromMaybe "update" . annexCommitMessage <$> Annex.getGitConfig + +{- Stages the journal, and commits staged changes to the branch. -} +commit :: String -> Annex () +commit = whenM journalDirty . forceCommit + +{- Commits the current index to the branch even without any journalled + - changes. -} +forceCommit :: String -> Annex () +forceCommit message = lockJournal $ \jl -> do + cleanjournal <- stageJournal jl + ref <- getBranch + withIndex $ commitIndex jl ref message [fullname] + liftIO cleanjournal + +{- Commits the staged changes in the index to the branch. + - + - Ensures that the branch's index file is first updated to merge the state + - of the branch at branchref, before running the commit action. This + - is needed because the branch may have had changes pushed to it, that + - are not yet reflected in the index. + - + - The branchref value can have been obtained using getBranch at any + - previous point, though getting it a long time ago makes the race + - more likely to occur. + - + - Note that changes may be pushed to the branch at any point in time! + - So, there's a race. If the commit is made using the newly pushed tip of + - the branch as its parent, and that ref has not yet been merged into the + - index, then the result is that the commit will revert the pushed + - changes, since they have not been merged into the index. This race + - is detected and another commit made to fix it. + - + - (It's also possible for the branch to be overwritten, + - losing the commit made here. But that's ok; the data is still in the + - index and will get committed again later.) + -} +commitIndex :: JournalLocked -> Git.Ref -> String -> [Git.Ref] -> Annex () +commitIndex jl branchref message parents = do + showStoringStateAction + commitIndex' jl branchref message message 0 parents +commitIndex' :: JournalLocked -> Git.Ref -> String -> String -> Integer -> [Git.Ref] -> Annex () +commitIndex' jl branchref message basemessage retrynum parents = do + updateIndex jl branchref + committedref <- inRepo $ Git.Branch.commitAlways Git.Branch.AutomaticCommit message fullname parents + setIndexSha committedref + parentrefs <- commitparents <$> catObject committedref + when (racedetected branchref parentrefs) $ + fixrace committedref parentrefs + where + -- look for "parent ref" lines and return the refs + commitparents = map (Git.Ref . snd) . filter isparent . + map (toassoc . decodeBS) . L.split newline + newline = fromIntegral (ord '\n') + toassoc = separate (== ' ') + isparent (k,_) = k == "parent" + + {- The race can be detected by checking the commit's + - parent, which will be the newly pushed branch, + - instead of the expected ref that the index was updated to. -} + racedetected expectedref parentrefs + | expectedref `elem` parentrefs = False -- good parent + | otherwise = True -- race! + + {- To recover from the race, union merge the lost refs + - into the index. -} + fixrace committedref lostrefs = do + showSideAction "recovering from race" + let retrynum' = retrynum+1 + -- small sleep to let any activity that caused + -- the race settle down + liftIO $ threadDelay (100000 + fromInteger retrynum') + mergeIndex jl lostrefs + let racemessage = basemessage ++ " (recovery from race #" ++ show retrynum' ++ "; expected commit parent " ++ show branchref ++ " but found " ++ show lostrefs ++ " )" + commitIndex' jl committedref racemessage basemessage retrynum' [committedref] + +{- Lists all files on the branch. including ones in the journal + - that have not been committed yet. There may be duplicates in the list. -} +files :: Annex [FilePath] +files = do + update + -- ++ forces the content of the first list to be buffered in memory, + -- so use getJournalledFilesStale which should be much smaller most + -- of the time. branchFiles will stream as the list is consumed. + (++) + <$> getJournalledFilesStale + <*> branchFiles + +{- Files in the branch, not including any from journalled changes, + - and without updating the branch. -} +branchFiles :: Annex [FilePath] +branchFiles = withIndex $ inRepo branchFiles' + +branchFiles' :: Git.Repo -> IO [FilePath] +branchFiles' = Git.Command.pipeNullSplitZombie + (lsTreeParams fullname [Param "--name-only"]) + +{- Populates the branch's index file with the current branch contents. + - + - This is only done when the index doesn't yet exist, and the index + - is used to build up changes to be commited to the branch, and merge + - in changes from other branches. + -} +genIndex :: Git.Repo -> IO () +genIndex g = Git.UpdateIndex.streamUpdateIndex g + [Git.UpdateIndex.lsTree fullname g] + +{- Merges the specified refs into the index. + - Any changes staged in the index will be preserved. -} +mergeIndex :: JournalLocked -> [Git.Ref] -> Annex () +mergeIndex jl branches = do + prepareModifyIndex jl + hashhandle <- hashObjectHandle + ch <- catFileHandle + inRepo $ \g -> Git.UnionMerge.mergeIndex hashhandle ch g branches + +{- Removes any stale git lock file, to avoid git falling over when + - updating the index. + - + - Since all modifications of the index are performed inside this module, + - and only when the journal is locked, the fact that the journal has to be + - locked when this is called ensures that no other process is currently + - modifying the index. So any index.lock file must be stale, caused + - by git running when the system crashed, or the repository's disk was + - removed, etc. + -} +prepareModifyIndex :: JournalLocked -> Annex () +prepareModifyIndex _jl = do + index <- fromRepo gitAnnexIndex + void $ liftIO $ tryIO $ removeFile $ index ++ ".lock" + +{- Runs an action using the branch's index file. -} +withIndex :: Annex a -> Annex a +withIndex = withIndex' False +withIndex' :: Bool -> Annex a -> Annex a +withIndex' bootstrapping a = do + f <- fromRepo gitAnnexIndex + withIndexFile f $ do + checkIndexOnce $ unlessM (liftIO $ doesFileExist f) $ do + unless bootstrapping create + createAnnexDirectory $ takeDirectory f + unless bootstrapping $ inRepo genIndex + a + +{- Updates the branch's index to reflect the current contents of the branch. + - Any changes staged in the index will be preserved. + - + - Compares the ref stored in the lock file with the current + - ref of the branch to see if an update is needed. + -} +updateIndex :: JournalLocked -> Git.Ref -> Annex () +updateIndex jl branchref = whenM (needUpdateIndex branchref) $ + forceUpdateIndex jl branchref + +forceUpdateIndex :: JournalLocked -> Git.Ref -> Annex () +forceUpdateIndex jl branchref = do + withIndex $ mergeIndex jl [fullname] + setIndexSha branchref + +{- Checks if the index needs to be updated. -} +needUpdateIndex :: Git.Ref -> Annex Bool +needUpdateIndex branchref = do + f <- fromRepo gitAnnexIndexStatus + committedref <- Git.Ref . firstLine <$> + liftIO (catchDefaultIO "" $ readFileStrict f) + return (committedref /= branchref) + +{- Record that the branch's index has been updated to correspond to a + - given ref of the branch. -} +setIndexSha :: Git.Ref -> Annex () +setIndexSha ref = do + f <- fromRepo gitAnnexIndexStatus + writeLogFile f $ fromRef ref ++ "\n" + runAnnexHook postUpdateAnnexHook + +{- Stages the journal into the index and returns an action that will + - clean up the staged journal files, which should only be run once + - the index has been committed to the branch. + - + - Before staging, this removes any existing git index file lock. + - This is safe to do because stageJournal is the only thing that + - modifies this index file, and only one can run at a time, because + - the journal is locked. So any existing git index file lock must be + - stale, and the journal must contain any data that was in the process + - of being written to the index file when it crashed. + -} +stageJournal :: JournalLocked -> Annex (IO ()) +stageJournal jl = withIndex $ do + prepareModifyIndex jl + g <- gitRepo + let dir = gitAnnexJournalDir g + (jlogf, jlogh) <- openjlog + h <- hashObjectHandle + withJournalHandle $ \jh -> + Git.UpdateIndex.streamUpdateIndex g + [genstream dir h jh jlogh] + return $ cleanup dir jlogh jlogf + where + genstream dir h jh jlogh streamer = readDirectory jh >>= \case + Nothing -> return () + Just file -> do + unless (dirCruft file) $ do + let path = dir file + sha <- Git.HashObject.hashFile h path + hPutStrLn jlogh file + streamer $ Git.UpdateIndex.updateIndexLine + sha TreeFile (asTopFilePath $ fileJournal file) + genstream dir h jh jlogh streamer + -- Clean up the staged files, as listed in the temp log file. + -- The temp file is used to avoid needing to buffer all the + -- filenames in memory. + cleanup dir jlogh jlogf = do + hFlush jlogh + hSeek jlogh AbsoluteSeek 0 + stagedfs <- lines <$> hGetContents jlogh + mapM_ (removeFile . (dir )) stagedfs + hClose jlogh + nukeFile jlogf + openjlog = do + tmpdir <- fromRepo gitAnnexTmpMiscDir + createAnnexDirectory tmpdir + liftIO $ openTempFile tmpdir "jlog" + +{- This is run after the refs have been merged into the index, + - but before the result is committed to the branch. + - (Which is why it's passed the contents of the local branches's + - transition log before that merge took place.) + - + - When the refs contain transitions that have not yet been done locally, + - the transitions are performed on the index, and a new branch + - is created from the result. + - + - When there are transitions recorded locally that have not been done + - to the remote refs, the transitions are performed in the index, + - and committed to the existing branch. In this case, the untransitioned + - remote refs cannot be merged into the branch (since transitions + - throw away history), so they are added to the list of refs to ignore, + - to avoid re-merging content from them again. + -} +handleTransitions :: JournalLocked -> Transitions -> [Git.Ref] -> Annex Bool +handleTransitions jl localts refs = do + m <- M.fromList <$> mapM getreftransition refs + let remotets = M.elems m + if all (localts ==) remotets + then return False + else do + let allts = combineTransitions (localts:remotets) + let (transitionedrefs, untransitionedrefs) = + partition (\r -> M.lookup r m == Just allts) refs + performTransitionsLocked jl allts (localts /= allts) transitionedrefs + ignoreRefs untransitionedrefs + return True + where + getreftransition ref = do + ts <- parseTransitionsStrictly "remote" . decodeBS + <$> catFile ref transitionsLog + return (ref, ts) + +{- Performs the specified transitions on the contents of the index file, + - commits it to the branch, or creates a new branch. + -} +performTransitions :: Transitions -> Bool -> [Ref] -> Annex () +performTransitions ts neednewlocalbranch transitionedrefs = lockJournal $ \jl -> + performTransitionsLocked jl ts neednewlocalbranch transitionedrefs +performTransitionsLocked :: JournalLocked -> Transitions -> Bool -> [Ref] -> Annex () +performTransitionsLocked jl ts neednewlocalbranch transitionedrefs = do + -- For simplicity & speed, we're going to use the Annex.Queue to + -- update the git-annex branch, while it usually holds changes + -- for the head branch. Flush any such changes. + Annex.Queue.flush + -- Stop any running git cat-files, to ensure that the + -- getStaged calls below use the current index, and not some older + -- one. + catFileStop + withIndex $ do + prepareModifyIndex jl + run $ mapMaybe getTransitionCalculator tlist + Annex.Queue.flush + if neednewlocalbranch + then do + committedref <- inRepo $ Git.Branch.commitAlways Git.Branch.AutomaticCommit message fullname transitionedrefs + setIndexSha committedref + else do + ref <- getBranch + commitIndex jl ref message (nub $ fullname:transitionedrefs) + where + message + | neednewlocalbranch && null transitionedrefs = "new branch for transition " ++ tdesc + | otherwise = "continuing transition " ++ tdesc + tdesc = show $ map describeTransition tlist + tlist = transitionList ts + + {- The changes to make to the branch are calculated and applied to + - the branch directly, rather than going through the journal, + - which would be innefficient. (And the journal is not designed + - to hold changes to every file in the branch at once.) + - + - When a file in the branch is changed by transition code, + - its new content is remembered and fed into the code for subsequent + - transitions. + -} + run [] = noop + run changers = do + trustmap <- calcTrustMap <$> getStaged trustLog + fs <- branchFiles + forM_ fs $ \f -> do + content <- getStaged f + apply changers f content trustmap + apply [] _ _ _ = return () + apply (changer:rest) file content trustmap = + case changer file content trustmap of + RemoveFile -> do + Annex.Queue.addUpdateIndex + =<< inRepo (Git.UpdateIndex.unstageFile file) + -- File is deleted; can't run any other + -- transitions on it. + return () + ChangeFile content' -> do + sha <- hashBlob content' + Annex.Queue.addUpdateIndex $ Git.UpdateIndex.pureStreamer $ + Git.UpdateIndex.updateIndexLine sha TreeFile (asTopFilePath file) + apply rest file content' trustmap + PreserveFile -> + apply rest file content trustmap + +checkBranchDifferences :: Git.Ref -> Annex () +checkBranchDifferences ref = do + theirdiffs <- allDifferences . parseDifferencesLog . decodeBS + <$> catFile ref differenceLog + mydiffs <- annexDifferences <$> Annex.getGitConfig + when (theirdiffs /= mydiffs) $ + giveup "Remote repository is tuned in incompatible way; cannot be merged with local repository." + +ignoreRefs :: [Git.Sha] -> Annex () +ignoreRefs rs = do + old <- getIgnoredRefs + let s = S.unions [old, S.fromList rs] + f <- fromRepo gitAnnexIgnoredRefs + writeLogFile f $ + unlines $ map fromRef $ S.elems s + +getIgnoredRefs :: Annex (S.Set Git.Sha) +getIgnoredRefs = S.fromList . mapMaybe Git.Sha.extractSha . lines <$> content + where + content = do + f <- fromRepo gitAnnexIgnoredRefs + liftIO $ catchDefaultIO "" $ readFile f + +addMergedRefs :: [(Git.Sha, Git.Branch)] -> Annex () +addMergedRefs [] = return () +addMergedRefs new = do + old <- getMergedRefs' + -- Keep only the newest sha for each branch. + let l = nubBy ((==) `on` snd) (new ++ old) + f <- fromRepo gitAnnexMergedRefs + writeLogFile f $ + unlines $ map (\(s, b) -> fromRef s ++ '\t' : fromRef b) l + +getMergedRefs :: Annex (S.Set Git.Sha) +getMergedRefs = S.fromList . map fst <$> getMergedRefs' + +getMergedRefs' :: Annex [(Git.Sha, Git.Branch)] +getMergedRefs' = do + f <- fromRepo gitAnnexMergedRefs + s <- liftIO $ catchDefaultIO "" $ readFile f + return $ map parse $ lines s + where + parse l = + let (s, b) = separate (== '\t') l + in (Ref s, Ref b) + +{- Grafts a treeish into the branch at the specified location, + - and then removes it. This ensures that the treeish won't get garbage + - collected, and will always be available as long as the git-annex branch + - is available. -} +graftTreeish :: Git.Ref -> TopFilePath -> Annex () +graftTreeish treeish graftpoint = lockJournal $ \jl -> do + branchref <- getBranch + updateIndex jl branchref + Git.Tree.Tree t <- inRepo $ Git.Tree.getTree branchref + t' <- inRepo $ Git.Tree.recordTree $ Git.Tree.Tree $ + Git.Tree.RecordedSubTree graftpoint treeish [] : t + c <- inRepo $ Git.Branch.commitTree Git.Branch.AutomaticCommit + "graft" [branchref] t' + origtree <- inRepo $ Git.Tree.recordTree (Git.Tree.Tree t) + c' <- inRepo $ Git.Branch.commitTree Git.Branch.AutomaticCommit + "graft cleanup" [c] origtree + inRepo $ Git.Branch.update' fullname c' + -- The tree in c' is the same as the tree in branchref, + -- and the index was updated to that above, so it's safe to + -- say that the index contains c'. + setIndexSha c' diff --git a/Annex/Branch/Transitions.hs b/Annex/Branch/Transitions.hs new file mode 100644 index 0000000000..c2283f3866 --- /dev/null +++ b/Annex/Branch/Transitions.hs @@ -0,0 +1,78 @@ +{- git-annex branch transitions + - + - Copyright 2013-2018 Joey Hess + - + - Licensed under the GNU GPL version 3 or higher. + -} + +module Annex.Branch.Transitions ( + FileTransition(..), + getTransitionCalculator +) where + +import Logs +import Logs.Transitions +import qualified Logs.UUIDBased as UUIDBased +import qualified Logs.Presence.Pure as Presence +import qualified Logs.Chunk.Pure as Chunk +import qualified Logs.MetaData.Pure as MetaData +import Types.TrustLevel +import Types.UUID +import Types.MetaData + +import qualified Data.Map as M +import qualified Data.Set as S +import Data.Default + +data FileTransition + = ChangeFile String + | RemoveFile + | PreserveFile + +type TransitionCalculator = FilePath -> String -> TrustMap -> FileTransition + +getTransitionCalculator :: Transition -> Maybe TransitionCalculator +getTransitionCalculator ForgetGitHistory = Nothing +getTransitionCalculator ForgetDeadRemotes = Just dropDead + +dropDead :: FilePath -> String -> TrustMap -> FileTransition +dropDead f content trustmap = case getLogVariety f of + Just UUIDBasedLog + -- Don't remove the dead repo from the trust log, + -- because git remotes may still exist, and they need + -- to still know it's dead. + | f == trustLog -> PreserveFile + | otherwise -> ChangeFile $ UUIDBased.showLog id $ dropDeadFromMapLog trustmap id $ UUIDBased.parseLog Just content + Just NewUUIDBasedLog -> ChangeFile $ + UUIDBased.showLogNew id $ dropDeadFromMapLog trustmap id $ UUIDBased.parseLogNew Just content + Just (ChunkLog _) -> ChangeFile $ + Chunk.showLog $ dropDeadFromMapLog trustmap fst $ Chunk.parseLog content + Just (PresenceLog _) -> + let newlog = Presence.compactLog $ dropDeadFromPresenceLog trustmap $ Presence.parseLog content + in if null newlog + then RemoveFile + else ChangeFile $ Presence.showLog newlog + Just RemoteMetaDataLog -> + let newlog = dropDeadFromRemoteMetaDataLog trustmap $ MetaData.simplifyLog $ MetaData.parseLog content + in if S.null newlog + then RemoveFile + else ChangeFile $ MetaData.showLog newlog + Just OtherLog -> PreserveFile + Nothing -> PreserveFile + +dropDeadFromMapLog :: TrustMap -> (k -> UUID) -> M.Map k v -> M.Map k v +dropDeadFromMapLog trustmap getuuid = + M.filterWithKey $ \k _v -> notDead trustmap getuuid k + +{- Presence logs can contain UUIDs or other values. Any line that matches + - a dead uuid is dropped; any other values are passed through. -} +dropDeadFromPresenceLog :: TrustMap -> [Presence.LogLine] -> [Presence.LogLine] +dropDeadFromPresenceLog trustmap = + filter $ notDead trustmap (toUUID . Presence.info) + +dropDeadFromRemoteMetaDataLog :: TrustMap -> MetaData.Log MetaData -> MetaData.Log MetaData +dropDeadFromRemoteMetaDataLog trustmap = + MetaData.filterOutEmpty . MetaData.filterRemoteMetaData (notDead trustmap id) + +notDead :: TrustMap -> (v -> UUID) -> v -> Bool +notDead trustmap a v = M.findWithDefault def (a v) trustmap /= DeadTrusted diff --git a/Annex/BranchState.hs b/Annex/BranchState.hs new file mode 100644 index 0000000000..0550d4224b --- /dev/null +++ b/Annex/BranchState.hs @@ -0,0 +1,41 @@ +{- git-annex branch state management + - + - Runtime state about the git-annex branch. + - + - Copyright 2011-2012 Joey Hess + - + - Licensed under the GNU GPL version 3 or higher. + -} + +module Annex.BranchState where + +import Annex.Common +import Types.BranchState +import qualified Annex + +getState :: Annex BranchState +getState = Annex.getState Annex.branchstate + +changeState :: (BranchState -> BranchState) -> Annex () +changeState changer = Annex.changeState $ \s -> + s { Annex.branchstate = changer (Annex.branchstate s) } + +{- Runs an action to check that the index file exists, if it's not been + - checked before in this run of git-annex. -} +checkIndexOnce :: Annex () -> Annex () +checkIndexOnce a = unlessM (indexChecked <$> getState) $ do + a + changeState $ \s -> s { indexChecked = True } + +{- Runs an action to update the branch, if it's not been updated before + - in this run of git-annex. -} +runUpdateOnce :: Annex () -> Annex () +runUpdateOnce a = unlessM (branchUpdated <$> getState) $ do + a + disableUpdate + +{- Avoids updating the branch. A useful optimisation when the branch + - is known to have not changed, or git-annex won't be relying on info + - from it. -} +disableUpdate :: Annex () +disableUpdate = changeState $ \s -> s { branchUpdated = True } diff --git a/Annex/CatFile.hs b/Annex/CatFile.hs new file mode 100644 index 0000000000..7062f785af --- /dev/null +++ b/Annex/CatFile.hs @@ -0,0 +1,144 @@ +{- git cat-file interface, with handle automatically stored in the Annex monad + - + - Copyright 2011-2015 Joey Hess + - + - Licensed under the GNU GPL version 3 or higher. + -} + +module Annex.CatFile ( + catFile, + catFileDetails, + catObject, + catTree, + catCommit, + catObjectDetails, + catFileHandle, + catObjectMetaData, + catFileStop, + catKey, + catKeyFile, + catKeyFileHEAD, + catSymLinkTarget, +) where + +import qualified Data.ByteString.Lazy as L +import qualified Data.Map as M +import System.PosixCompat.Types + +import Annex.Common +import qualified Git +import qualified Git.CatFile +import qualified Annex +import Git.Types +import Git.FilePath +import Git.Index +import qualified Git.Ref +import Annex.Link +import Utility.FileSystemEncoding + +catFile :: Git.Branch -> FilePath -> Annex L.ByteString +catFile branch file = do + h <- catFileHandle + liftIO $ Git.CatFile.catFile h branch file + +catFileDetails :: Git.Branch -> FilePath -> Annex (Maybe (L.ByteString, Sha, ObjectType)) +catFileDetails branch file = do + h <- catFileHandle + liftIO $ Git.CatFile.catFileDetails h branch file + +catObject :: Git.Ref -> Annex L.ByteString +catObject ref = do + h <- catFileHandle + liftIO $ Git.CatFile.catObject h ref + +catObjectMetaData :: Git.Ref -> Annex (Maybe (Integer, ObjectType)) +catObjectMetaData ref = do + h <- catFileHandle + liftIO $ Git.CatFile.catObjectMetaData h ref + +catTree :: Git.Ref -> Annex [(FilePath, FileMode)] +catTree ref = do + h <- catFileHandle + liftIO $ Git.CatFile.catTree h ref + +catCommit :: Git.Ref -> Annex (Maybe Commit) +catCommit ref = do + h <- catFileHandle + liftIO $ Git.CatFile.catCommit h ref + +catObjectDetails :: Git.Ref -> Annex (Maybe (L.ByteString, Sha, ObjectType)) +catObjectDetails ref = do + h <- catFileHandle + liftIO $ Git.CatFile.catObjectDetails h ref + +{- There can be multiple index files, and a different cat-file is needed + - for each. This is selected by setting GIT_INDEX_FILE in the gitEnv. -} +catFileHandle :: Annex Git.CatFile.CatFileHandle +catFileHandle = do + m <- Annex.getState Annex.catfilehandles + indexfile <- fromMaybe "" . maybe Nothing (lookup indexEnv) + <$> fromRepo gitEnv + case M.lookup indexfile m of + Just h -> return h + Nothing -> do + h <- inRepo Git.CatFile.catFileStart + let m' = M.insert indexfile h m + Annex.changeState $ \s -> s { Annex.catfilehandles = m' } + return h + +{- Stops all running cat-files. Should only be run when it's known that + - nothing is using the handles, eg at shutdown. -} +catFileStop :: Annex () +catFileStop = do + m <- Annex.withState $ pure . \s -> + (s { Annex.catfilehandles = M.empty }, Annex.catfilehandles s) + liftIO $ mapM_ Git.CatFile.catFileStop (M.elems m) + +{- From ref to a symlink or a pointer file, get the key. -} +catKey :: Ref -> Annex (Maybe Key) +catKey ref = go =<< catObjectMetaData ref + where + go (Just (sz, _)) + -- Avoid catting large files, that cannot be symlinks or + -- pointer files, which would require buffering their + -- content in memory, as well as a lot of IO. + | sz <= maxPointerSz = parseLinkOrPointer <$> catObject ref + go _ = return Nothing + +{- Gets a symlink target. -} +catSymLinkTarget :: Sha -> Annex String +catSymLinkTarget sha = fromInternalGitPath . decodeBS <$> get + where + -- Avoid buffering the whole file content, which might be large. + -- 8192 is enough if it really is a symlink. + get = L.take 8192 <$> catObject sha + +{- From a file in the repository back to the key. + - + - Ideally, this should reflect the key that's staged in the index, + - not the key that's committed to HEAD. Unfortunately, git cat-file + - does not refresh the index file after it's started up, so things + - newly staged in the index won't show up. It does, however, notice + - when branches change. + - + - For command-line git-annex use, that doesn't matter. It's perfectly + - reasonable for things staged in the index after the currently running + - git-annex process to not be noticed by it. However, we do want to see + - what's in the index, since it may have uncommitted changes not in HEAD + - + - For the assistant, this is much more of a problem, since it commits + - files and then needs to be able to immediately look up their keys. + - OTOH, the assistant doesn't keep changes staged in the index for very + - long at all before committing them -- and it won't look at the keys + - of files until after committing them. + - + - So, this gets info from the index, unless running as a daemon. + -} +catKeyFile :: FilePath -> Annex (Maybe Key) +catKeyFile f = ifM (Annex.getState Annex.daemon) + ( catKeyFileHEAD f + , catKey $ Git.Ref.fileRef f + ) + +catKeyFileHEAD :: FilePath -> Annex (Maybe Key) +catKeyFileHEAD f = catKey $ Git.Ref.fileFromRef Git.Ref.headRef f diff --git a/Annex/ChangedRefs.hs b/Annex/ChangedRefs.hs new file mode 100644 index 0000000000..edef1c06ca --- /dev/null +++ b/Annex/ChangedRefs.hs @@ -0,0 +1,103 @@ +{- Waiting for changed git refs + - + - Copyright 2014-216 Joey Hess + - + - Licensed under the GNU GPL version 3 or higher. + -} + +module Annex.ChangedRefs + ( ChangedRefs(..) + , ChangedRefsHandle + , waitChangedRefs + , drainChangedRefs + , stopWatchingChangedRefs + , watchChangedRefs + ) where + +import Annex.Common +import Utility.DirWatcher +import Utility.DirWatcher.Types +import qualified Git +import Git.Sha +import qualified Utility.SimpleProtocol as Proto + +import Control.Concurrent +import Control.Concurrent.STM +import Control.Concurrent.STM.TBMChan + +newtype ChangedRefs = ChangedRefs [Git.Ref] + deriving (Show) + +instance Proto.Serializable ChangedRefs where + serialize (ChangedRefs l) = unwords $ map Git.fromRef l + deserialize = Just . ChangedRefs . map Git.Ref . words + +data ChangedRefsHandle = ChangedRefsHandle DirWatcherHandle (TBMChan Git.Sha) + +-- | Wait for one or more git refs to change. +-- +-- When possible, coalesce ref writes that occur closely together +-- in time. Delay up to 0.05 seconds to get more ref writes. +waitChangedRefs :: ChangedRefsHandle -> IO ChangedRefs +waitChangedRefs (ChangedRefsHandle _ chan) = + atomically (readTBMChan chan) >>= \case + Nothing -> return $ ChangedRefs [] + Just r -> do + threadDelay 50000 + rs <- atomically $ loop [] + return $ ChangedRefs (r:rs) + where + loop rs = tryReadTBMChan chan >>= \case + Just (Just r) -> loop (r:rs) + _ -> return rs + +-- | Remove any changes that might be buffered in the channel, +-- without waiting for any new changes. +drainChangedRefs :: ChangedRefsHandle -> IO () +drainChangedRefs (ChangedRefsHandle _ chan) = atomically go + where + go = tryReadTBMChan chan >>= \case + Just (Just _) -> go + _ -> return () + +stopWatchingChangedRefs :: ChangedRefsHandle -> IO () +stopWatchingChangedRefs h@(ChangedRefsHandle wh chan) = do + stopWatchDir wh + atomically $ closeTBMChan chan + drainChangedRefs h + +watchChangedRefs :: Annex (Maybe ChangedRefsHandle) +watchChangedRefs = do + -- This channel is used to accumulate notifications, + -- because the DirWatcher might have multiple threads that find + -- changes at the same time. It is bounded to allow a watcher + -- to be started once and reused, without too many changes being + -- buffered in memory. + chan <- liftIO $ newTBMChanIO 100 + + g <- gitRepo + let refdir = Git.localGitDir g "refs" + liftIO $ createDirectoryIfMissing True refdir + + let notifyhook = Just $ notifyHook chan + let hooks = mkWatchHooks + { addHook = notifyhook + , modifyHook = notifyhook + } + + if canWatch + then do + h <- liftIO $ watchDir refdir (const False) True hooks id + return $ Just $ ChangedRefsHandle h chan + else return Nothing + +notifyHook :: TBMChan Git.Sha -> FilePath -> Maybe FileStatus -> IO () +notifyHook chan reffile _ + | ".lock" `isSuffixOf` reffile = noop + | otherwise = void $ do + sha <- catchDefaultIO Nothing $ + extractSha <$> readFile reffile + -- When the channel is full, there is probably no reader + -- running, or ref changes have been occuring very fast, + -- so it's ok to not write the change to it. + maybe noop (void . atomically . tryWriteTBMChan chan) sha diff --git a/Annex/CheckAttr.hs b/Annex/CheckAttr.hs new file mode 100644 index 0000000000..2be95483ef --- /dev/null +++ b/Annex/CheckAttr.hs @@ -0,0 +1,44 @@ +{- git check-attr interface, with handle automatically stored in the Annex monad + - + - Copyright 2012 Joey Hess + - + - Licensed under the GNU GPL version 3 or higher. + -} + +module Annex.CheckAttr ( + checkAttr, + checkAttrHandle, + checkAttrStop, +) where + +import Annex.Common +import qualified Git.CheckAttr as Git +import qualified Annex + +{- All gitattributes used by git-annex. -} +annexAttrs :: [Git.Attr] +annexAttrs = + [ "annex.backend" + , "annex.numcopies" + , "annex.largefiles" + ] + +checkAttr :: Git.Attr -> FilePath -> Annex String +checkAttr attr file = do + h <- checkAttrHandle + liftIO $ Git.checkAttr h attr file + +checkAttrHandle :: Annex Git.CheckAttrHandle +checkAttrHandle = maybe startup return =<< Annex.getState Annex.checkattrhandle + where + startup = do + h <- inRepo $ Git.checkAttrStart annexAttrs + Annex.changeState $ \s -> s { Annex.checkattrhandle = Just h } + return h + +checkAttrStop :: Annex () +checkAttrStop = maybe noop stop =<< Annex.getState Annex.checkattrhandle + where + stop h = do + liftIO $ Git.checkAttrStop h + Annex.changeState $ \s -> s { Annex.checkattrhandle = Nothing } diff --git a/Annex/CheckIgnore.hs b/Annex/CheckIgnore.hs new file mode 100644 index 0000000000..824f5feeb5 --- /dev/null +++ b/Annex/CheckIgnore.hs @@ -0,0 +1,41 @@ +{- git check-ignore interface, with handle automatically stored in + - the Annex monad + - + - Copyright 2013 Joey Hess + - + - Licensed under the GNU GPL version 3 or higher. + -} + +module Annex.CheckIgnore ( + checkIgnored, + checkIgnoreHandle, + checkIgnoreStop +) where + +import Annex.Common +import qualified Git.CheckIgnore as Git +import qualified Annex + +checkIgnored :: FilePath -> Annex Bool +checkIgnored file = go =<< checkIgnoreHandle + where + go Nothing = return False + go (Just h) = liftIO $ Git.checkIgnored h file + +checkIgnoreHandle :: Annex (Maybe Git.CheckIgnoreHandle) +checkIgnoreHandle = maybe startup return =<< Annex.getState Annex.checkignorehandle + where + startup = do + v <- inRepo Git.checkIgnoreStart + when (isNothing v) $ + warning "The installed version of git is too old for .gitignores to be honored by git-annex." + Annex.changeState $ \s -> s { Annex.checkignorehandle = Just v } + return v + +checkIgnoreStop :: Annex () +checkIgnoreStop = maybe noop stop =<< Annex.getState Annex.checkignorehandle + where + stop (Just h) = do + liftIO $ Git.checkIgnoreStop h + Annex.changeState $ \s -> s { Annex.checkignorehandle = Nothing } + stop Nothing = noop diff --git a/Annex/Common.hs b/Annex/Common.hs new file mode 100644 index 0000000000..bb277df7dd --- /dev/null +++ b/Annex/Common.hs @@ -0,0 +1,14 @@ +{-# LANGUAGE CPP #-} + +module Annex.Common (module X) where + +import Common as X +import Types as X +import Key as X +import Types.UUID as X +import Annex as X (gitRepo, inRepo, fromRepo, calcRepo) +import Annex.Locations as X +import Messages as X +#ifndef mingw32_HOST_OS +import System.Posix.IO as X hiding (createPipe) +#endif diff --git a/Annex/Concurrent.hs b/Annex/Concurrent.hs new file mode 100644 index 0000000000..2e92d18598 --- /dev/null +++ b/Annex/Concurrent.hs @@ -0,0 +1,64 @@ +{- git-annex concurrent state + - + - Copyright 2015 Joey Hess + - + - Licensed under the GNU GPL version 3 or higher. + -} + +module Annex.Concurrent where + +import Annex +import Annex.Common +import Annex.Action +import qualified Annex.Queue + +import qualified Data.Map as M + +{- Allows forking off a thread that uses a copy of the current AnnexState + - to run an Annex action. + - + - The returned IO action can be used to start the thread. + - It returns an Annex action that must be run in the original + - calling context to merge the forked AnnexState back into the + - current AnnexState. + -} +forkState :: Annex a -> Annex (IO (Annex a)) +forkState a = do + st <- dupState + return $ do + (ret, newst) <- run st a + return $ do + mergeState newst + return ret + +{- Returns a copy of the current AnnexState that is safe to be + - used when forking off a thread. + - + - After an Annex action is run using this AnnexState, it + - should be merged back into the current Annex's state, + - by calling mergeState. + -} +dupState :: Annex AnnexState +dupState = do + st <- Annex.getState id + return $ st + { Annex.workers = [] + -- each thread has its own repoqueue, but the repoqueuesem + -- is shared to prevent more than one thread flushing its + -- queue at the same time + , Annex.repoqueue = Nothing + -- avoid sharing eg, open file handles + , Annex.catfilehandles = M.empty + , Annex.checkattrhandle = Nothing + , Annex.checkignorehandle = Nothing + } + +{- Merges the passed AnnexState into the current Annex state. + - Also closes various handles in it. -} +mergeState :: AnnexState -> Annex () +mergeState st = do + st' <- liftIO $ snd <$> run st stopCoProcesses + forM_ (M.toList $ Annex.cleanup st') $ + uncurry addCleanup + Annex.Queue.mergeFrom st' + changeState $ \s -> s { errcounter = errcounter s + errcounter st' } diff --git a/Annex/Content.hs b/Annex/Content.hs new file mode 100644 index 0000000000..5d657cac2a --- /dev/null +++ b/Annex/Content.hs @@ -0,0 +1,970 @@ +{- git-annex file content managing + - + - Copyright 2010-2018 Joey Hess + - + - Licensed under the GNU GPL version 3 or higher. + -} + +{-# LANGUAGE CPP #-} + +module Annex.Content ( + inAnnex, + inAnnex', + inAnnexSafe, + inAnnexCheck, + lockContentShared, + lockContentForRemoval, + ContentRemovalLock, + RetrievalSecurityPolicy(..), + getViaTmp, + getViaTmpFromDisk, + checkDiskSpaceToGet, + prepTmp, + withTmp, + checkDiskSpace, + needMoreDiskSpace, + moveAnnex, + populatePointerFile, + linkToAnnex, + linkFromAnnex, + LinkAnnexResult(..), + unlinkAnnex, + checkedCopyFile, + linkOrCopy, + linkOrCopy', + sendAnnex, + prepSendAnnex, + removeAnnex, + moveBad, + KeyLocation(..), + getKeysPresent, + saveState, + downloadUrl, + preseedTmp, + dirKeys, + withObjectLoc, + staleKeysPrune, + pruneTmpWorkDirBefore, + isUnmodified, + verifyKeyContent, + VerifyConfig(..), + Verification(..), + unVerified, + withTmpWorkDir, +) where + +import System.IO.Unsafe (unsafeInterleaveIO) +import qualified Data.Set as S + +import Annex.Common +import Logs.Location +import Types.Transfer +import Logs.Transfer +import qualified Git +import qualified Annex +import qualified Annex.Queue +import qualified Annex.Branch +import Utility.FileMode +import qualified Annex.Url as Url +import Utility.CopyFile +import Utility.Metered +import Config +import Git.FilePath +import Annex.Perms +import Annex.Link +import qualified Annex.Content.Direct as Direct +import Annex.ReplaceFile +import Annex.LockPool +import Messages.Progress +import Types.Remote (unVerified, Verification(..), RetrievalSecurityPolicy(..)) +import qualified Types.Remote +import qualified Types.Backend +import qualified Backend +import qualified Database.Keys +import Types.NumCopies +import Types.Key +import Annex.UUID +import Annex.InodeSentinal +import Utility.InodeCache +import Annex.Content.LowLevel +import Annex.Content.PointerFile + +{- Checks if a given key's content is currently present. -} +inAnnex :: Key -> Annex Bool +inAnnex key = inAnnexCheck key $ liftIO . doesFileExist + +{- Runs an arbitrary check on a key's content. -} +inAnnexCheck :: Key -> (FilePath -> Annex Bool) -> Annex Bool +inAnnexCheck key check = inAnnex' id False check key + +{- inAnnex that performs an arbitrary check of the key's content. + - + - When the content is unlocked, it must also be unmodified, or the bad + - value will be returned. + - + - In direct mode, at least one of the associated files must pass the + - check. Additionally, the file must be unmodified. + -} +inAnnex' :: (a -> Bool) -> a -> (FilePath -> Annex a) -> Key -> Annex a +inAnnex' isgood bad check key = withObjectLoc key checkindirect checkdirect + where + checkindirect loc = do + r <- check loc + if isgood r + then do + cache <- Database.Keys.getInodeCaches key + if null cache + then return r + else ifM (sameInodeCache loc cache) + ( return r + , return bad + ) + else return bad + checkdirect [] = return bad + checkdirect (loc:locs) = do + r <- check loc + if isgood r + then ifM (Direct.goodContent key loc) + ( return r + , checkdirect locs + ) + else checkdirect locs + +{- A safer check; the key's content must not only be present, but + - is not in the process of being removed. -} +inAnnexSafe :: Key -> Annex (Maybe Bool) +inAnnexSafe key = inAnnex' (fromMaybe True) (Just False) go key + where + is_locked = Nothing + is_unlocked = Just True + is_missing = Just False + + go contentfile = maybe (checkindirect contentfile) (checkdirect contentfile) + =<< contentLockFile key + +#ifndef mingw32_HOST_OS + checkindirect contentfile = checkOr is_missing contentfile + {- In direct mode, the content file must exist, but + - the lock file generally won't exist unless a removal is in + - process. -} + checkdirect contentfile lockfile = + ifM (liftIO $ doesFileExist contentfile) + ( checkOr is_unlocked lockfile + , return is_missing + ) + checkOr d lockfile = checkLocked lockfile >>= return . \case + Nothing -> d + Just True -> is_locked + Just False -> is_unlocked +#else + checkindirect f = liftIO $ ifM (doesFileExist f) + ( lockShared f >>= \case + Nothing -> return is_locked + Just lockhandle -> do + dropLock lockhandle + return is_unlocked + , return is_missing + ) + {- In Windows, see if we can take a shared lock. If so, + - remove the lock file to clean up after ourselves. -} + checkdirect contentfile lockfile = + ifM (liftIO $ doesFileExist contentfile) + ( modifyContent lockfile $ liftIO $ + lockShared lockfile >>= \case + Nothing -> return is_locked + Just lockhandle -> do + dropLock lockhandle + void $ tryIO $ nukeFile lockfile + return is_unlocked + , return is_missing + ) +#endif + +{- Direct mode and especially Windows has to use a separate lock + - file from the content, since locking the actual content file + - would interfere with the user's use of it. -} +contentLockFile :: Key -> Annex (Maybe FilePath) +#ifndef mingw32_HOST_OS +contentLockFile key = ifM isDirect + ( Just <$> calcRepo (gitAnnexContentLock key) + , return Nothing + ) +#else +contentLockFile key = Just <$> calcRepo (gitAnnexContentLock key) +#endif + +{- Prevents the content from being removed while the action is running. + - Uses a shared lock. + - + - If locking fails, or the content is not present, throws an exception + - rather than running the action. + - + - Note that, in direct mode, nothing prevents the user from directly + - editing or removing the content, even while it's locked by this. + -} +lockContentShared :: Key -> (VerifiedCopy -> Annex a) -> Annex a +lockContentShared key a = lockContentUsing lock key $ ifM (inAnnex key) + ( do + u <- getUUID + withVerifiedCopy LockedCopy u (return True) a + , giveup $ "failed to lock content: not present" + ) + where +#ifndef mingw32_HOST_OS + lock contentfile Nothing = tryLockShared Nothing contentfile + lock _ (Just lockfile) = posixLocker tryLockShared lockfile +#else + lock = winLocker lockShared +#endif + +{- Exclusively locks content, while performing an action that + - might remove it. + -} +lockContentForRemoval :: Key -> (ContentRemovalLock -> Annex a) -> Annex a +lockContentForRemoval key a = lockContentUsing lock key $ + a (ContentRemovalLock key) + where +#ifndef mingw32_HOST_OS + {- Since content files are stored with the write bit disabled, have + - to fiddle with permissions to open for an exclusive lock. -} + lock contentfile Nothing = bracket_ + (thawContent contentfile) + (freezeContent contentfile) + (tryLockExclusive Nothing contentfile) + lock _ (Just lockfile) = posixLocker tryLockExclusive lockfile +#else + lock = winLocker lockExclusive +#endif + +{- Passed the object content file, and maybe a separate lock file to use, + - when the content file itself should not be locked. -} +type ContentLocker = FilePath -> Maybe LockFile -> Annex (Maybe LockHandle) + +#ifndef mingw32_HOST_OS +posixLocker :: (Maybe FileMode -> LockFile -> Annex (Maybe LockHandle)) -> LockFile -> Annex (Maybe LockHandle) +posixLocker takelock lockfile = do + mode <- annexFileMode + modifyContent lockfile $ + takelock (Just mode) lockfile + +#else +winLocker :: (LockFile -> IO (Maybe LockHandle)) -> ContentLocker +winLocker takelock _ (Just lockfile) = do + modifyContent lockfile $ + void $ liftIO $ tryIO $ + writeFile lockfile "" + liftIO $ takelock lockfile +-- never reached; windows always uses a separate lock file +winLocker _ _ Nothing = return Nothing +#endif + +lockContentUsing :: ContentLocker -> Key -> Annex a -> Annex a +lockContentUsing locker key a = do + contentfile <- calcRepo $ gitAnnexLocation key + lockfile <- contentLockFile key + bracket + (lock contentfile lockfile) + (unlock lockfile) + (const a) + where + alreadylocked = giveup "content is locked" + failedtolock e = giveup $ "failed to lock content: " ++ show e + + lock contentfile lockfile = + (maybe alreadylocked return + =<< locker contentfile lockfile) + `catchIO` failedtolock + +#ifndef mingw32_HOST_OS + unlock mlockfile lck = do + maybe noop cleanuplockfile mlockfile + liftIO $ dropLock lck +#else + unlock mlockfile lck = do + -- Can't delete a locked file on Windows + liftIO $ dropLock lck + maybe noop cleanuplockfile mlockfile +#endif + + cleanuplockfile lockfile = modifyContent lockfile $ + void $ liftIO $ tryIO $ + nukeFile lockfile + +{- Runs an action, passing it the temp file to get, + - and if the action succeeds, verifies the file matches + - the key and moves the file into the annex as a key's content. -} +getViaTmp :: RetrievalSecurityPolicy -> VerifyConfig -> Key -> (FilePath -> Annex (Bool, Verification)) -> Annex Bool +getViaTmp rsp v key action = checkDiskSpaceToGet key False $ + getViaTmpFromDisk rsp v key action + +{- Like getViaTmp, but does not check that there is enough disk space + - for the incoming key. For use when the key content is already on disk + - and not being copied into place. -} +getViaTmpFromDisk :: RetrievalSecurityPolicy -> VerifyConfig -> Key -> (FilePath -> Annex (Bool, Verification)) -> Annex Bool +getViaTmpFromDisk rsp v key action = checkallowed $ do + tmpfile <- prepTmp key + resuming <- liftIO $ doesFileExist tmpfile + (ok, verification) <- action tmpfile + -- When the temp file already had content, we don't know if + -- that content is good or not, so only trust if it the action + -- Verified it in passing. Otherwise, force verification even + -- if the VerifyConfig normally disables it. + let verification' = if resuming + then case verification of + Verified -> Verified + _ -> MustVerify + else verification + if ok + then ifM (verifyKeyContent rsp v verification' key tmpfile) + ( ifM (pruneTmpWorkDirBefore tmpfile (moveAnnex key)) + ( do + logStatus key InfoPresent + return True + , return False + ) + , do + warning "verification of content failed" + -- The bad content is not retained, because + -- a retry should not try to resume from it + -- since it's apparently corrupted. + -- Also, the bad content could be any data, + -- including perhaps the content of another + -- file than the one that was requested, + -- and so it's best not to keep it on disk. + pruneTmpWorkDirBefore tmpfile (liftIO . nukeFile) + return False + ) + -- On transfer failure, the tmp file is left behind, in case + -- caller wants to resume its transfer + else return False + where + -- Avoid running the action to get the content when the + -- RetrievalSecurityPolicy would cause verification to always fail. + checkallowed a = case rsp of + RetrievalAllKeysSecure -> a + RetrievalVerifiableKeysSecure + | isVerifiable (keyVariety key) -> a + | otherwise -> ifM (annexAllowUnverifiedDownloads <$> Annex.getGitConfig) + ( a + , warnUnverifiableInsecure key >> return False + ) + +{- Verifies that a file is the expected content of a key. + - + - Configuration can prevent verification, for either a + - particular remote or always, unless the RetrievalSecurityPolicy + - requires verification. + - + - Most keys have a known size, and if so, the file size is checked. + - + - When the key's backend allows verifying the content (via checksum), + - it is checked. + - + - If the RetrievalSecurityPolicy requires verification and the key's + - backend doesn't support it, the verification will fail. + -} +verifyKeyContent :: RetrievalSecurityPolicy -> VerifyConfig -> Verification -> Key -> FilePath -> Annex Bool +verifyKeyContent rsp v verification k f = case (rsp, verification) of + (_, Verified) -> return True + (RetrievalVerifiableKeysSecure, _) + | isVerifiable (keyVariety k) -> verify + | otherwise -> ifM (annexAllowUnverifiedDownloads <$> Annex.getGitConfig) + ( verify + , warnUnverifiableInsecure k >> return False + ) + (_, UnVerified) -> ifM (shouldVerify v) + ( verify + , return True + ) + (_, MustVerify) -> verify + where + verify = verifysize <&&> verifycontent + verifysize = case keySize k of + Nothing -> return True + Just size -> do + size' <- liftIO $ catchDefaultIO 0 $ getFileSize f + return (size' == size) + verifycontent = case Types.Backend.verifyKeyContent =<< Backend.maybeLookupBackendVariety (keyVariety k) of + Nothing -> return True + Just verifier -> verifier k f + +warnUnverifiableInsecure :: Key -> Annex () +warnUnverifiableInsecure k = warning $ unwords + [ "Getting " ++ kv ++ " keys with this remote is not secure;" + , "the content cannot be verified to be correct." + , "(Use annex.security.allow-unverified-downloads to bypass" + , "this safety check.)" + ] + where + kv = formatKeyVariety (keyVariety k) + +data VerifyConfig = AlwaysVerify | NoVerify | RemoteVerify Remote | DefaultVerify + +shouldVerify :: VerifyConfig -> Annex Bool +shouldVerify AlwaysVerify = return True +shouldVerify NoVerify = return False +shouldVerify DefaultVerify = annexVerify <$> Annex.getGitConfig +shouldVerify (RemoteVerify r) = + (shouldVerify DefaultVerify + <&&> pure (remoteAnnexVerify (Types.Remote.gitconfig r))) + -- Export remotes are not key/value stores, so always verify + -- content from them even when verification is disabled. + <||> Types.Remote.isExportSupported r + +{- Checks if there is enough free disk space to download a key + - to its temp file. + - + - When the temp file already exists, count the space it is using as + - free, since the download will overwrite it or resume. + - + - Wen there's enough free space, runs the download action. + -} +checkDiskSpaceToGet :: Key -> a -> Annex a -> Annex a +checkDiskSpaceToGet key unabletoget getkey = do + tmp <- fromRepo $ gitAnnexTmpObjectLocation key + + e <- liftIO $ doesFileExist tmp + alreadythere <- liftIO $ if e + then getFileSize tmp + else return 0 + ifM (checkDiskSpace Nothing key alreadythere True) + ( do + -- The tmp file may not have been left writable + when e $ thawContent tmp + getkey + , return unabletoget + ) + +prepTmp :: Key -> Annex FilePath +prepTmp key = do + tmp <- fromRepo $ gitAnnexTmpObjectLocation key + createAnnexDirectory (parentDir tmp) + return tmp + +{- Prepares a temp file for a key, runs an action on it, and cleans up + - the temp file. If the action throws an exception, the temp file is + - left behind, which allows for resuming. + -} +withTmp :: Key -> (FilePath -> Annex a) -> Annex a +withTmp key action = do + tmp <- prepTmp key + res <- action tmp + pruneTmpWorkDirBefore tmp (liftIO . nukeFile) + return res + +{- Moves a key's content into .git/annex/objects/ + - + - When a key has associated pointer files, the object is hard + - linked (or copied) to the files, and the object file is left thawed. + - + - In direct mode, moves the object file to the associated file, or files. + - + - What if the key there already has content? This could happen for + - various reasons; perhaps the same content is being annexed again. + - Perhaps there has been a hash collision generating the keys. + - + - The current strategy is to assume that in this case it's safe to delete + - one of the two copies of the content; and the one already in the annex + - is left there, assuming it's the original, canonical copy. + - + - I considered being more paranoid, and checking that both files had + - the same content. Decided against it because A) users explicitly choose + - a backend based on its hashing properties and so if they're dealing + - with colliding files it's their own fault and B) adding such a check + - would not catch all cases of colliding keys. For example, perhaps + - a remote has a key; if it's then added again with different content then + - the overall system now has two different peices of content for that + - key, and one of them will probably get deleted later. So, adding the + - check here would only raise expectations that git-annex cannot truely + - meet. + - + - May return false, when a particular variety of key is not being + - accepted into the repository. Will display a warning message in this + - case. May also throw exceptions in some cases. + -} +moveAnnex :: Key -> FilePath -> Annex Bool +moveAnnex key src = ifM (checkSecureHashes key) + ( do + withObjectLoc key storeobject storedirect + return True + , return False + ) + where + storeobject dest = ifM (liftIO $ doesFileExist dest) + ( alreadyhave + , modifyContent dest $ do + freezeContent src + liftIO $ moveFile src dest + g <- Annex.gitRepo + fs <- map (`fromTopFilePath` g) + <$> Database.Keys.getAssociatedFiles key + unless (null fs) $ do + ics <- mapM (populatePointerFile (Restage True) key dest) fs + Database.Keys.storeInodeCaches' key [dest] (catMaybes ics) + ) + storeindirect = storeobject =<< calcRepo (gitAnnexLocation key) + + {- In direct mode, the associated file's content may be locally + - modified. In that case, it's preserved. However, the content + - we're moving into the annex may be the only extant copy, so + - it's important we not lose it. So, when the key's content + - cannot be moved to any associated file, it's stored in indirect + - mode. + -} + storedirect = storedirect' storeindirect + storedirect' fallback [] = fallback + storedirect' fallback (f:fs) = do + thawContent src + v <- isAnnexLink f + if Just key == v + then do + Direct.updateInodeCache key src + replaceFile f $ liftIO . moveFile src + chmodContent f + forM_ fs $ + Direct.addContentWhenNotPresent key f + else ifM (Direct.goodContent key f) + ( storedirect' alreadyhave fs + , storedirect' fallback fs + ) + + alreadyhave = liftIO $ removeFile src + +checkSecureHashes :: Key -> Annex Bool +checkSecureHashes key + | cryptographicallySecure (keyVariety key) = return True + | otherwise = ifM (annexSecureHashesOnly <$> Annex.getGitConfig) + ( do + warning $ "annex.securehashesonly blocked adding " ++ formatKeyVariety (keyVariety key) ++ " key to annex objects" + return False + , return True + ) + +data LinkAnnexResult = LinkAnnexOk | LinkAnnexFailed | LinkAnnexNoop + +{- Populates the annex object file by hard linking or copying a source + - file to it. -} +linkToAnnex :: Key -> FilePath -> Maybe InodeCache -> Annex LinkAnnexResult +linkToAnnex key src srcic = ifM (checkSecureHashes key) + ( do + dest <- calcRepo (gitAnnexLocation key) + modifyContent dest $ linkAnnex To key src srcic dest Nothing + , return LinkAnnexFailed + ) + +{- Makes a destination file be a link or copy from the annex object. -} +linkFromAnnex :: Key -> FilePath -> Maybe FileMode -> Annex LinkAnnexResult +linkFromAnnex key dest destmode = do + src <- calcRepo (gitAnnexLocation key) + srcic <- withTSDelta (liftIO . genInodeCache src) + linkAnnex From key src srcic dest destmode + +data FromTo = From | To + +{- Hard links or copies from or to the annex object location. + - Updates inode cache. + - + - Freezes or thaws the destination appropriately. + - + - When a hard link is made, the annex object necessarily has to be thawed + - too. So, adding an object to the annex with a hard link can prevent + - losing the content if the source file is deleted, but does not + - guard against modifications. + - + - Nothing is done if the destination file already exists. + -} +linkAnnex :: FromTo -> Key -> FilePath -> Maybe InodeCache -> FilePath -> Maybe FileMode -> Annex LinkAnnexResult +linkAnnex _ _ _ Nothing _ _ = return LinkAnnexFailed +linkAnnex fromto key src (Just srcic) dest destmode = + withTSDelta (liftIO . genInodeCache dest) >>= \case + Just destic -> do + cs <- Database.Keys.getInodeCaches key + if null cs + then Database.Keys.addInodeCaches key [srcic, destic] + else Database.Keys.addInodeCaches key [srcic] + return LinkAnnexNoop + Nothing -> linkOrCopy key src dest destmode >>= \case + Nothing -> failed + Just r -> do + case fromto of + From -> thawContent dest + To -> case r of + Copied -> freezeContent dest + Linked -> noop + checksrcunchanged + where + failed = do + Database.Keys.addInodeCaches key [srcic] + return LinkAnnexFailed + checksrcunchanged = withTSDelta (liftIO . genInodeCache src) >>= \case + Just srcic' | compareStrong srcic srcic' -> do + destic <- withTSDelta (liftIO . genInodeCache dest) + Database.Keys.addInodeCaches key $ + catMaybes [destic, Just srcic] + return LinkAnnexOk + _ -> do + liftIO $ nukeFile dest + failed + +{- Removes the annex object file for a key. Lowlevel. -} +unlinkAnnex :: Key -> Annex () +unlinkAnnex key = do + obj <- calcRepo $ gitAnnexLocation key + modifyContent obj $ do + secureErase obj + liftIO $ nukeFile obj + +{- Runs an action to transfer an object's content. + - + - In some cases, it's possible for the file to change as it's being sent. + - If this happens, runs the rollback action and returns False. The + - rollback action should remove the data that was transferred. + -} +sendAnnex :: Key -> Annex () -> (FilePath -> Annex Bool) -> Annex Bool +sendAnnex key rollback sendobject = go =<< prepSendAnnex key + where + go Nothing = return False + go (Just (f, checksuccess)) = do + r <- sendobject f + ifM checksuccess + ( return r + , do + rollback + return False + ) + +{- Returns a file that contains an object's content, + - and a check to run after the transfer is complete. + - + - When a file is unlocked (or in direct mode), it's possble for its + - content to change as it's being sent. The check detects this case + - and returns False. + - + - Note that the returned check action is, in some cases, run in the + - Annex monad of the remote that is receiving the object, rather than + - the sender. So it cannot rely on Annex state. + -} +prepSendAnnex :: Key -> Annex (Maybe (FilePath, Annex Bool)) +prepSendAnnex key = withObjectLoc key indirect direct + where + indirect f = do + cache <- Database.Keys.getInodeCaches key + cache' <- if null cache + -- Since no inode cache is in the database, this + -- object is not currently unlocked. But that could + -- change while the transfer is in progress, so + -- generate an inode cache for the starting + -- content. + then maybeToList <$> + withTSDelta (liftIO . genInodeCache f) + else pure cache + return $ if null cache' + then Nothing + else Just (f, sameInodeCache f cache') + direct [] = return Nothing + direct (f:fs) = do + cache <- Direct.recordedInodeCache key + -- check that we have a good file + ifM (sameInodeCache f cache) + ( return $ Just (f, sameInodeCache f cache) + , direct fs + ) + +{- Performs an action, passing it the location to use for a key's content. + - + - In direct mode, the associated files will be passed. But, if there are + - no associated files for a key, the indirect mode action will be + - performed instead. -} +withObjectLoc :: Key -> (FilePath -> Annex a) -> ([FilePath] -> Annex a) -> Annex a +withObjectLoc key indirect direct = ifM isDirect + ( do + fs <- Direct.associatedFiles key + if null fs + then goindirect + else direct fs + , goindirect + ) + where + goindirect = indirect =<< calcRepo (gitAnnexLocation key) + +cleanObjectLoc :: Key -> Annex () -> Annex () +cleanObjectLoc key cleaner = do + file <- calcRepo $ gitAnnexLocation key + void $ tryIO $ thawContentDir file + cleaner + liftIO $ removeparents file (3 :: Int) + where + removeparents _ 0 = noop + removeparents file n = do + let dir = parentDir file + maybe noop (const $ removeparents dir (n-1)) + <=< catchMaybeIO $ removeDirectory dir + +{- Removes a key's file from .git/annex/objects/ + -} +removeAnnex :: ContentRemovalLock -> Annex () +removeAnnex (ContentRemovalLock key) = withObjectLoc key remove removedirect + where + remove file = cleanObjectLoc key $ do + secureErase file + liftIO $ nukeFile file + g <- Annex.gitRepo + mapM_ (\f -> void $ tryIO $ resetpointer $ fromTopFilePath f g) + =<< Database.Keys.getAssociatedFiles key + Database.Keys.removeInodeCaches key + Direct.removeInodeCache key + + -- Check associated pointer file for modifications, and reset if + -- it's unmodified. + resetpointer file = ifM (isUnmodified key file) + ( depopulatePointerFile key file + -- Modified file, so leave it alone. + -- If it was a hard link to the annex object, + -- that object might have been frozen as part of the + -- removal process, so thaw it. + , void $ tryIO $ thawContent file + ) + + -- In direct mode, deletes the associated files or files, and replaces + -- them with symlinks. + removedirect fs = do + cache <- Direct.recordedInodeCache key + Direct.removeInodeCache key + mapM_ (resetfile cache) fs + + resetfile cache f = whenM (Direct.sameInodeCache f cache) $ do + l <- calcRepo $ gitAnnexLink f key + secureErase f + replaceFile f $ makeAnnexLink l + +{- Check if a file contains the unmodified content of the key. + - + - The expensive way to tell is to do a verification of its content. + - The cheaper way is to see if the InodeCache for the key matches the + - file. -} +isUnmodified :: Key -> FilePath -> Annex Bool +isUnmodified key f = go =<< geti + where + go Nothing = return False + go (Just fc) = cheapcheck fc <||> expensivecheck fc + cheapcheck fc = anyM (compareInodeCaches fc) + =<< Database.Keys.getInodeCaches key + expensivecheck fc = ifM (verifyKeyContent RetrievalAllKeysSecure AlwaysVerify UnVerified key f) + -- The file could have been modified while it was + -- being verified. Detect that. + ( geti >>= maybe (return False) (compareInodeCaches fc) + , return False + ) + geti = withTSDelta (liftIO . genInodeCache f) + +{- Moves a key out of .git/annex/objects/ into .git/annex/bad, and + - returns the file it was moved to. -} +moveBad :: Key -> Annex FilePath +moveBad key = do + src <- calcRepo $ gitAnnexLocation key + bad <- fromRepo gitAnnexBadDir + let dest = bad takeFileName src + createAnnexDirectory (parentDir dest) + cleanObjectLoc key $ + liftIO $ moveFile src dest + logStatus key InfoMissing + return dest + +data KeyLocation = InAnnex | InRepository | InAnywhere + +{- List of keys whose content exists in the specified location. + + - InAnnex only lists keys with content in .git/annex/objects, + - while InRepository, in direct mode, also finds keys with content + - in the work tree. InAnywhere lists all keys that have directories + - in .git/annex/objects, whether or not the content is present. + - + - Note that InRepository has to check whether direct mode files + - have goodContent. + -} +getKeysPresent :: KeyLocation -> Annex [Key] +getKeysPresent keyloc = do + direct <- isDirect + dir <- fromRepo gitAnnexObjectDir + s <- getstate direct + depth <- gitAnnexLocationDepth <$> Annex.getGitConfig + liftIO $ walk s direct depth dir + where + walk s direct depth dir = do + contents <- catchDefaultIO [] (dirContents dir) + if depth < 2 + then do + contents' <- filterM (present s direct) contents + let keys = mapMaybe (fileKey . takeFileName) contents' + continue keys [] + else do + let deeper = walk s direct (depth - 1) + continue [] (map deeper contents) + continue keys [] = return keys + continue keys (a:as) = do + {- Force lazy traversal with unsafeInterleaveIO. -} + morekeys <- unsafeInterleaveIO a + continue (morekeys++keys) as + + inanywhere = case keyloc of + InAnywhere -> True + _ -> False + + present _ _ _ | inanywhere = pure True + present _ False d = presentInAnnex d + present s True d = presentDirect s d <||> presentInAnnex d + + presentInAnnex = doesFileExist . contentfile + contentfile d = d takeFileName d + + presentDirect s d = case keyloc of + InAnnex -> return False + InRepository -> case fileKey (takeFileName d) of + Nothing -> return False + Just k -> Annex.eval s $ + anyM (Direct.goodContent k) =<< Direct.associatedFiles k + InAnywhere -> return True + + {- In order to run Annex monad actions within unsafeInterleaveIO, + - the current state is taken and reused. No changes made to this + - state will be preserved. + - + - As an optimsation, call inodesChanged to prime the state with + - a cached value that will be used in the call to goodContent. + -} + getstate direct = do + when direct $ + void inodesChanged + Annex.getState id + +{- Things to do to record changes to content when shutting down. + - + - It's acceptable to avoid committing changes to the branch, + - especially if performing a short-lived action. + -} +saveState :: Bool -> Annex () +saveState nocommit = doSideAction $ do + Annex.Queue.flush + unless nocommit $ + whenM (annexAlwaysCommit <$> Annex.getGitConfig) $ + Annex.Branch.commit =<< Annex.Branch.commitMessage + +{- Downloads content from any of a list of urls. -} +downloadUrl :: Key -> MeterUpdate -> [Url.URLString] -> FilePath -> Annex Bool +downloadUrl k p urls file = + -- Poll the file to handle configurations where an external + -- download command is used. + meteredFile file (Just p) k $ + Url.withUrlOptions $ \uo -> + liftIO $ anyM (\u -> Url.download p u file uo) urls + +{- Copies a key's content, when present, to a temp file. + - This is used to speed up some rsyncs. -} +preseedTmp :: Key -> FilePath -> Annex Bool +preseedTmp key file = go =<< inAnnex key + where + go False = return False + go True = do + ok <- copy + when ok $ thawContent file + return ok + copy = ifM (liftIO $ doesFileExist file) + ( return True + , do + s <- calcRepo $ gitAnnexLocation key + liftIO $ ifM (doesFileExist s) + ( copyFileExternal CopyTimeStamps s file + , return False + ) + ) + +{- Finds files directly inside a directory like gitAnnexBadDir + - (not in subdirectories) and returns the corresponding keys. -} +dirKeys :: (Git.Repo -> FilePath) -> Annex [Key] +dirKeys dirspec = do + dir <- fromRepo dirspec + ifM (liftIO $ doesDirectoryExist dir) + ( do + contents <- liftIO $ getDirectoryContents dir + files <- liftIO $ filterM doesFileExist $ + map (dir ) contents + return $ mapMaybe (fileKey . takeFileName) files + , return [] + ) + +{- Looks in the specified directory for bad/tmp keys, and returns a list + - of those that might still have value, or might be stale and removable. + - + - Also, stale keys that can be proven to have no value + - (ie, their content is already present) are deleted. + -} +staleKeysPrune :: (Git.Repo -> FilePath) -> Bool -> Annex [Key] +staleKeysPrune dirspec nottransferred = do + contents <- dirKeys dirspec + + dups <- filterM inAnnex contents + let stale = contents `exclude` dups + + dir <- fromRepo dirspec + forM_ dups $ \k -> + pruneTmpWorkDirBefore (dir keyFile k) (liftIO . removeFile) + + if nottransferred + then do + inprogress <- S.fromList . map (transferKey . fst) + <$> getTransfers + return $ filter (`S.notMember` inprogress) stale + else return stale + +{- Prune the work dir associated with the specified content file, + - before performing an action that deletes the file, or moves it away. + - + - This preserves the invariant that the workdir never exists without + - the content file. + -} +pruneTmpWorkDirBefore :: FilePath -> (FilePath -> Annex a) -> Annex a +pruneTmpWorkDirBefore f action = do + let workdir = gitAnnexTmpWorkDir f + liftIO $ whenM (doesDirectoryExist workdir) $ + removeDirectoryRecursive workdir + action f + +{- Runs an action, passing it a temporary work directory where + - it can write files while receiving the content of a key. + - + - Preserves the invariant that the workdir never exists without the + - content file, by creating an empty content file first. + - + - On exception, or when the action returns Nothing, + - the temporary work directory is retained (unless + - empty), so anything in it can be used on resume. + -} +withTmpWorkDir :: Key -> (FilePath -> Annex (Maybe a)) -> Annex (Maybe a) +withTmpWorkDir key action = do + -- Create the object file if it does not exist. This way, + -- staleKeysPrune only has to look for object files, and can + -- clean up gitAnnexTmpWorkDir for those it finds. + obj <- prepTmp key + unlessM (liftIO $ doesFileExist obj) $ do + liftIO $ writeFile obj "" + setAnnexFilePerm obj + let tmpdir = gitAnnexTmpWorkDir obj + liftIO $ createDirectoryIfMissing True tmpdir + setAnnexDirPerm tmpdir + res <- action tmpdir + case res of + Just _ -> liftIO $ removeDirectoryRecursive tmpdir + Nothing -> liftIO $ void $ tryIO $ removeDirectory tmpdir + return res + +{- Finds items in the first, smaller list, that are not + - present in the second, larger list. + - + - Constructing a single set, of the list that tends to be + - smaller, appears more efficient in both memory and CPU + - than constructing and taking the S.difference of two sets. -} +exclude :: Ord a => [a] -> [a] -> [a] +exclude [] _ = [] -- optimisation +exclude smaller larger = S.toList $ remove larger $ S.fromList smaller + where + remove a b = foldl (flip S.delete) b a diff --git a/Annex/Content/Direct.hs b/Annex/Content/Direct.hs new file mode 100644 index 0000000000..46fd327ccc --- /dev/null +++ b/Annex/Content/Direct.hs @@ -0,0 +1,181 @@ +{- git-annex file content managing for direct mode + - + - This is deprecated, and will be removed when direct mode gets removed + - from git-annex. + - + - Copyright 2012-2014 Joey Hess + - + - Licensed under the GNU GPL version 3 or higher. + -} + +module Annex.Content.Direct ( + associatedFiles, + associatedFilesRelative, + removeAssociatedFile, + removeAssociatedFileUnchecked, + removeAssociatedFiles, + addAssociatedFile, + goodContent, + recordedInodeCache, + updateInodeCache, + addInodeCache, + writeInodeCache, + compareInodeCaches, + sameInodeCache, + elemInodeCaches, + sameFileStatus, + removeInodeCache, + toInodeCache, + addContentWhenNotPresent, +) where + +import Annex.Common +import Annex.Perms +import qualified Git +import Logs.Location +import Logs.File +import Utility.InodeCache +import Utility.CopyFile +import Annex.ReplaceFile +import Annex.Link +import Annex.InodeSentinal + +{- Absolute FilePaths of Files in the tree that are associated with a key. -} +associatedFiles :: Key -> Annex [FilePath] +associatedFiles key = do + files <- associatedFilesRelative key + top <- fromRepo Git.repoPath + return $ map (top ) files + +{- List of files in the tree that are associated with a key, relative to + - the top of the repo. -} +associatedFilesRelative :: Key -> Annex [FilePath] +associatedFilesRelative key = do + mapping <- calcRepo $ gitAnnexMapping key + liftIO $ catchDefaultIO [] $ withFile mapping ReadMode $ \h -> + -- Read strictly to ensure the file is closed + -- before changeAssociatedFiles tries to write to it. + -- (Especially needed on Windows.) + lines <$> hGetContentsStrict h + +{- Changes the associated files information for a key, applying a + - transformation to the list. Returns new associatedFiles value. -} +changeAssociatedFiles :: Key -> ([FilePath] -> [FilePath]) -> Annex [FilePath] +changeAssociatedFiles key transform = do + mapping <- calcRepo $ gitAnnexMapping key + files <- associatedFilesRelative key + let files' = transform files + when (files /= files') $ + modifyContent mapping $ + writeLogFile mapping $ unlines files' + top <- fromRepo Git.repoPath + return $ map (top ) files' + +{- Removes the list of associated files. -} +removeAssociatedFiles :: Key -> Annex () +removeAssociatedFiles key = do + mapping <- calcRepo $ gitAnnexMapping key + modifyContent mapping $ + liftIO $ nukeFile mapping + +{- Removes an associated file. Returns new associatedFiles value. + - Checks if this was the last copy of the object, and updates location + - log. -} +removeAssociatedFile :: Key -> FilePath -> Annex [FilePath] +removeAssociatedFile key file = do + fs <- removeAssociatedFileUnchecked key file + when (null fs) $ + logStatus key InfoMissing + return fs + +{- Removes an associated file. Returns new associatedFiles value. -} +removeAssociatedFileUnchecked :: Key -> FilePath -> Annex [FilePath] +removeAssociatedFileUnchecked key file = do + file' <- normaliseAssociatedFile file + changeAssociatedFiles key $ filter (/= file') + +{- Adds an associated file. Returns new associatedFiles value. -} +addAssociatedFile :: Key -> FilePath -> Annex [FilePath] +addAssociatedFile key file = do + file' <- normaliseAssociatedFile file + changeAssociatedFiles key $ \files -> + if file' `elem` files + then files + else file':files + +{- Associated files are always stored relative to the top of the repository. + - The input FilePath is relative to the CWD, or is absolute. -} +normaliseAssociatedFile :: FilePath -> Annex FilePath +normaliseAssociatedFile file = do + top <- fromRepo Git.repoPath + liftIO $ relPathDirToFile top file + +{- Checks if a file in the tree, associated with a key, has not been modified. + - + - To avoid needing to fsck the file's content, which can involve an + - expensive checksum, this relies on a cache that contains the file's + - expected mtime and inode. + -} +goodContent :: Key -> FilePath -> Annex Bool +goodContent key file = sameInodeCache file =<< recordedInodeCache key + +{- Gets the recorded inode cache for a key. + - + - A key can be associated with multiple files, so may return more than + - one. -} +recordedInodeCache :: Key -> Annex [InodeCache] +recordedInodeCache key = withInodeCacheFile key $ \f -> + liftIO $ catchDefaultIO [] $ + mapMaybe readInodeCache . lines <$> readFileStrict f + +{- Caches an inode for a file. + - + - Anything else already cached is preserved. + -} +updateInodeCache :: Key -> FilePath -> Annex () +updateInodeCache key file = maybe noop (addInodeCache key) + =<< withTSDelta (liftIO . genInodeCache file) + +{- Adds another inode to the cache for a key. -} +addInodeCache :: Key -> InodeCache -> Annex () +addInodeCache key cache = do + oldcaches <- recordedInodeCache key + unlessM (elemInodeCaches cache oldcaches) $ + writeInodeCache key (cache:oldcaches) + +{- Writes inode cache for a key. -} +writeInodeCache :: Key -> [InodeCache] -> Annex () +writeInodeCache key caches = withInodeCacheFile key $ \f -> + modifyContent f $ + liftIO $ writeFile f $ + unlines $ map showInodeCache caches + +{- Removes an inode cache. -} +removeInodeCache :: Key -> Annex () +removeInodeCache key = withInodeCacheFile key $ \f -> + modifyContent f $ + liftIO $ nukeFile f + +withInodeCacheFile :: Key -> (FilePath -> Annex a) -> Annex a +withInodeCacheFile key a = a =<< calcRepo (gitAnnexInodeCache key) + +{- Checks if a FileStatus matches the recorded InodeCache of a file. -} +sameFileStatus :: Key -> FilePath -> FileStatus -> Annex Bool +sameFileStatus key f status = do + old <- recordedInodeCache key + curr <- withTSDelta $ \delta -> liftIO $ toInodeCache delta f status + case (old, curr) of + (_, Just c) -> elemInodeCaches c old + ([], Nothing) -> return True + _ -> return False + +{- Copies the contentfile to the associated file, if the associated + - file has no content. If the associated file does have content, + - even if the content differs, it's left unchanged. -} +addContentWhenNotPresent :: Key -> FilePath -> FilePath -> Annex () +addContentWhenNotPresent key contentfile associatedfile = do + v <- isAnnexLink associatedfile + when (Just key == v) $ + replaceFile associatedfile $ + liftIO . void . copyFileExternal CopyAllMetaData contentfile + updateInodeCache key associatedfile diff --git a/Annex/Content/LowLevel.hs b/Annex/Content/LowLevel.hs new file mode 100644 index 0000000000..54fc0a4ed8 --- /dev/null +++ b/Annex/Content/LowLevel.hs @@ -0,0 +1,139 @@ +{- git-annex low-level content functions + - + - Copyright 2010-2018 Joey Hess + - + - Licensed under the GNU GPL version 3 or higher. + -} + +{-# LANGUAGE CPP #-} + +module Annex.Content.LowLevel where + +import System.PosixCompat.Files + +import Annex.Common +import Logs.Transfer +import qualified Annex +import Utility.DiskFree +import Utility.FileMode +import Utility.DataUnits +import Utility.CopyFile + +{- Runs the secure erase command if set, otherwise does nothing. + - File may or may not be deleted at the end; caller is responsible for + - making sure it's deleted. -} +secureErase :: FilePath -> Annex () +secureErase file = maybe noop go =<< annexSecureEraseCommand <$> Annex.getGitConfig + where + go basecmd = void $ liftIO $ + boolSystem "sh" [Param "-c", Param $ gencmd basecmd] + gencmd = massReplace [ ("%file", shellEscape file) ] + +data LinkedOrCopied = Linked | Copied + +{- Hard links or copies src to dest, which must not already exist. + - + - Only uses a hard link when annex.thin is enabled and when src is + - not already hardlinked to elsewhere. + - + - Checks disk reserve before copying against the size of the key, + - and will fail if not enough space, or if the dest file already exists. + - + - The FileMode, if provided, influences the mode of the dest file. + - In particular, if it has an execute bit set, the dest file's + - execute bit will be set. The mode is not fully copied over because + - git doesn't support file modes beyond execute. + -} +linkOrCopy :: Key -> FilePath -> FilePath -> Maybe FileMode -> Annex (Maybe LinkedOrCopied) +linkOrCopy = linkOrCopy' (annexThin <$> Annex.getGitConfig) + +linkOrCopy' :: Annex Bool -> Key -> FilePath -> FilePath -> Maybe FileMode -> Annex (Maybe LinkedOrCopied) +linkOrCopy' canhardlink key src dest destmode + | maybe False isExecutable destmode = copy =<< getstat + | otherwise = catchDefaultIO Nothing $ + ifM canhardlink + ( hardlink + , copy =<< getstat + ) + where + hardlink = do + s <- getstat + if linkCount s > 1 + then copy s + else liftIO (createLink src dest >> preserveGitMode dest destmode >> return (Just Linked)) + `catchIO` const (copy s) + copy s = ifM (checkedCopyFile' key src dest destmode s) + ( return (Just Copied) + , return Nothing + ) + getstat = liftIO $ getFileStatus src + +{- Checks disk space before copying. -} +checkedCopyFile :: Key -> FilePath -> FilePath -> Maybe FileMode -> Annex Bool +checkedCopyFile key src dest destmode = catchBoolIO $ + checkedCopyFile' key src dest destmode + =<< liftIO (getFileStatus src) + +checkedCopyFile' :: Key -> FilePath -> FilePath -> Maybe FileMode -> FileStatus -> Annex Bool +checkedCopyFile' key src dest destmode s = catchBoolIO $ + ifM (checkDiskSpace' (fromIntegral $ fileSize s) (Just $ takeDirectory dest) key 0 True) + ( liftIO $ + copyFileExternal CopyAllMetaData src dest + <&&> preserveGitMode dest destmode + , return False + ) + +preserveGitMode :: FilePath -> Maybe FileMode -> IO Bool +preserveGitMode f (Just mode) + | isExecutable mode = catchBoolIO $ do + modifyFileMode f $ addModes executeModes + return True + | otherwise = catchBoolIO $ do + modifyFileMode f $ removeModes executeModes + return True +preserveGitMode _ _ = return True + +{- Checks that there is disk space available to store a given key, + - in a destination directory (or the annex) printing a warning if not. + - + - If the destination is on the same filesystem as the annex, + - checks for any other running downloads, removing the amount of data still + - to be downloaded from the free space. This way, we avoid overcommitting + - when doing concurrent downloads. + -} +checkDiskSpace :: Maybe FilePath -> Key -> Integer -> Bool -> Annex Bool +checkDiskSpace destdir key = checkDiskSpace' (fromMaybe 1 (keySize key)) destdir key + +{- Allows specifying the size of the key, if it's known, which is useful + - as not all keys know their size. -} +checkDiskSpace' :: Integer -> Maybe FilePath -> Key -> Integer -> Bool -> Annex Bool +checkDiskSpace' need destdir key alreadythere samefilesystem = ifM (Annex.getState Annex.force) + ( return True + , do + -- We can't get inprogress and free at the same + -- time, and both can be changing, so there's a + -- small race here. Err on the side of caution + -- by getting inprogress first, so if it takes + -- a while, we'll see any decrease in the free + -- disk space. + inprogress <- if samefilesystem + then sizeOfDownloadsInProgress (/= key) + else pure 0 + dir >>= liftIO . getDiskFree >>= \case + Just have -> do + reserve <- annexDiskReserve <$> Annex.getGitConfig + let delta = need + reserve - have - alreadythere + inprogress + let ok = delta <= 0 + unless ok $ + warning $ needMoreDiskSpace delta + return ok + _ -> return True + ) + where + dir = maybe (fromRepo gitAnnexDir) return destdir + +needMoreDiskSpace :: Integer -> String +needMoreDiskSpace n = "not enough free space, need " ++ + roughSize storageUnits True n ++ " more" ++ forcemsg + where + forcemsg = " (use --force to override this check or adjust annex.diskreserve)" diff --git a/Annex/Content/PointerFile.hs b/Annex/Content/PointerFile.hs new file mode 100644 index 0000000000..1aba305835 --- /dev/null +++ b/Annex/Content/PointerFile.hs @@ -0,0 +1,57 @@ +{- git-annex pointer files + - + - Copyright 2010-2018 Joey Hess + - + - Licensed under the GNU GPL version 3 or higher. + -} + +{-# LANGUAGE CPP #-} + +module Annex.Content.PointerFile where + +import System.PosixCompat.Files + +import Annex.Common +import Annex.Perms +import Annex.Link +import Annex.ReplaceFile +import Annex.InodeSentinal +import Utility.InodeCache +import Annex.Content.LowLevel + +{- Populates a pointer file with the content of a key. + - + - If the file already has some other content, it is not modified. + - + - Returns an InodeCache if it populated the pointer file. + -} +populatePointerFile :: Restage -> Key -> FilePath -> FilePath -> Annex (Maybe InodeCache) +populatePointerFile restage k obj f = go =<< liftIO (isPointerFile f) + where + go (Just k') | k == k' = do + destmode <- liftIO $ catchMaybeIO $ fileMode <$> getFileStatus f + liftIO $ nukeFile f + (ic, populated) <- replaceFile f $ \tmp -> do + ok <- linkOrCopy k obj tmp destmode >>= \case + Just _ -> thawContent tmp >> return True + Nothing -> liftIO (writePointerFile tmp k destmode) >> return False + ic <- withTSDelta (liftIO . genInodeCache tmp) + return (ic, ok) + maybe noop (restagePointerFile restage f) ic + if populated + then return ic + else return Nothing + go _ = return Nothing + +{- Removes the content from a pointer file, replacing it with a pointer. + - + - Does not check if the pointer file is modified. -} +depopulatePointerFile :: Key -> FilePath -> Annex () +depopulatePointerFile key file = do + mode <- liftIO $ catchMaybeIO $ fileMode <$> getFileStatus file + secureErase file + liftIO $ nukeFile file + ic <- replaceFile file $ \tmp -> do + liftIO $ writePointerFile tmp key mode + withTSDelta (liftIO . genInodeCache tmp) + maybe noop (restagePointerFile (Restage True) file) ic diff --git a/Annex/Difference.hs b/Annex/Difference.hs new file mode 100644 index 0000000000..23448192a8 --- /dev/null +++ b/Annex/Difference.hs @@ -0,0 +1,58 @@ +{- git-annex repository differences + - + - Copyright 2015 Joey Hess + - + - Licensed under the GNU GPL version 3 or higher. + -} + +module Annex.Difference ( + module Types.Difference, + setDifferences, +) where + +import Annex.Common +import Types.Difference +import Logs.Difference +import Config +import Annex.UUID +import Logs.UUID +import Annex.Version +import qualified Annex + +import qualified Data.Map as M + +-- Differences are only allowed to be tweaked when initializing a +-- repository for the first time, and then only if there is not another +-- known uuid. If the repository was cloned from elsewhere, it inherits +-- the existing settings. +-- +-- Must be called before setVersion, so it can check if this is the first +-- time the repository is being initialized. +setDifferences :: Annex () +setDifferences = do + u <- getUUID + otherds <- allDifferences <$> recordedDifferences + ds <- mappend otherds . annexDifferences <$> Annex.getGitConfig + when (ds /= mempty) $ do + ds' <- ifM (isJust <$> getVersion) + ( do + oldds <- recordedDifferencesFor u + when (ds /= oldds) $ + warning "Cannot change tunable parameters in already initialized repository." + return oldds + , if otherds == mempty + then ifM (any (/= u) . M.keys <$> uuidMap) + ( do + warning "Cannot change tunable parameters in a clone of an existing repository." + return mempty + , return ds + ) + else if otherds /= ds + then do + warning "The specified tunable parameters differ from values being used in other clones of this repository." + return otherds + else return ds + ) + forM_ (listDifferences ds') $ \d -> + setConfig (ConfigKey $ differenceConfigKey d) (differenceConfigVal d) + recordDifferences ds' u diff --git a/Annex/DirHashes.hs b/Annex/DirHashes.hs new file mode 100644 index 0000000000..f8438484d1 --- /dev/null +++ b/Annex/DirHashes.hs @@ -0,0 +1,98 @@ +{- git-annex file locations + - + - Copyright 2010-2017 Joey Hess + - + - Licensed under the GNU GPL version 3 or higher. + -} + +module Annex.DirHashes ( + Hasher, + HashLevels(..), + objectHashLevels, + branchHashLevels, + branchHashDir, + dirHashes, + hashDirMixed, + hashDirLower, + display_32bits_as_dir +) where + +import Data.Bits +import Data.Word +import Data.Default +import qualified Data.ByteArray + +import Common +import Key +import Types.GitConfig +import Types.Difference +import Utility.FileSystemEncoding +import Utility.Hash + +type Hasher = Key -> FilePath + +-- Number of hash levels to use. 2 is the default. +newtype HashLevels = HashLevels Int + +instance Default HashLevels where + def = HashLevels 2 + +objectHashLevels :: GitConfig -> HashLevels +objectHashLevels = configHashLevels OneLevelObjectHash + +branchHashLevels :: GitConfig -> HashLevels +branchHashLevels = configHashLevels OneLevelBranchHash + +configHashLevels :: Difference -> GitConfig -> HashLevels +configHashLevels d config + | hasDifference d (annexDifferences config) = HashLevels 1 + | otherwise = def + +branchHashDir :: GitConfig -> Key -> String +branchHashDir = hashDirLower . branchHashLevels + +{- Two different directory hashes may be used. The mixed case hash + - came first, and is fine, except for the problem of case-strict + - filesystems such as Linux VFAT (mounted with shortname=mixed), + - which do not allow using a directory "XX" when "xx" already exists. + - To support that, most repositories use the lower case hash for new data. -} +dirHashes :: [HashLevels -> Hasher] +dirHashes = [hashDirLower, hashDirMixed] + +hashDirs :: HashLevels -> Int -> String -> FilePath +hashDirs (HashLevels 1) sz s = addTrailingPathSeparator $ take sz s +hashDirs _ sz s = addTrailingPathSeparator $ take sz s drop sz s + +hashDirLower :: HashLevels -> Hasher +hashDirLower n k = hashDirs n 3 $ take 6 $ show $ md5 $ + encodeBS $ key2file $ nonChunkKey k + +{- This was originally using Data.Hash.MD5 from MissingH. This new version +- is faster, but ugly as it has to replicate the 4 Word32's that produced. -} +hashDirMixed :: HashLevels -> Hasher +hashDirMixed n k = hashDirs n 2 $ take 4 $ concatMap display_32bits_as_dir $ + encodeWord32 $ map fromIntegral $ Data.ByteArray.unpack $ + Utility.Hash.md5 $ encodeBS $ key2file $ nonChunkKey k + where + encodeWord32 (b1:b2:b3:b4:rest) = + (shiftL b4 24 .|. shiftL b3 16 .|. shiftL b2 8 .|. b1) + : encodeWord32 rest + encodeWord32 _ = [] + +{- modified version of display_32bits_as_hex from Data.Hash.MD5 + - in MissingH + - Copyright (C) 2001 Ian Lynagh + - License: Either BSD or GPL + -} +display_32bits_as_dir :: Word32 -> String +display_32bits_as_dir w = trim $ swap_pairs cs + where + -- Need 32 characters to use. To avoid inaverdently making + -- a real word, use letters that appear less frequently. + chars = ['0'..'9'] ++ "zqjxkmvwgpfZQJXKMVWGPF" + cs = map (\x -> getc $ (shiftR w (6*x)) .&. 31) [0..7] + getc n = chars !! fromIntegral n + swap_pairs (x1:x2:xs) = x2:x1:swap_pairs xs + swap_pairs _ = [] + -- Last 2 will always be 00, so omit. + trim = take 6 diff --git a/Annex/Direct.hs b/Annex/Direct.hs new file mode 100644 index 0000000000..5ca9ec14a1 --- /dev/null +++ b/Annex/Direct.hs @@ -0,0 +1,480 @@ +{- git-annex direct mode + - + - This is deprecated, and will be removed when direct mode gets removed + - from git-annex. + - + - Copyright 2012-2014 Joey Hess + - + - Licensed under the GNU GPL version 3 or higher. + -} + +module Annex.Direct where + +import Annex.Common +import qualified Annex +import qualified Git +import qualified Git.LsFiles +import qualified Git.Merge +import qualified Git.DiffTree as DiffTree +import qualified Git.Config +import qualified Git.Ref +import qualified Git.Branch +import Git.Sha +import Git.FilePath +import Git.Types +import Config +import Annex.CatFile +import qualified Annex.Queue +import Logs.Location +import Backend +import Types.KeySource +import Annex.Content +import Annex.Content.Direct +import Annex.Link +import Utility.InodeCache +import Utility.CopyFile +import Annex.Perms +import Annex.ReplaceFile +import Annex.VariantFile +import Git.Index +import Annex.GitOverlay +import Annex.LockFile +import Annex.InodeSentinal + +{- Uses git ls-files to find files that need to be committed, and stages + - them into the index. Returns True if some changes were staged. -} +stageDirect :: Annex Bool +stageDirect = do + Annex.Queue.flush + top <- fromRepo Git.repoPath + (l, cleanup) <- inRepo $ Git.LsFiles.stagedOthersDetails [top] + forM_ l go + void $ liftIO cleanup + staged <- Annex.Queue.size + Annex.Queue.flush + return $ staged /= 0 + where + {- Determine what kind of modified or deleted file this is, as + - efficiently as we can, by getting any key that's associated + - with it in git, as well as its stat info. -} + go (file, Just sha, Just _mode) = withTSDelta $ \delta -> do + shakey <- catKey sha + mstat <- liftIO $ catchMaybeIO $ getSymbolicLinkStatus file + mcache <- liftIO $ maybe (pure Nothing) (toInodeCache delta file) mstat + filekey <- isAnnexLink file + case (shakey, filekey, mstat, mcache) of + (_, Just key, _, _) + | shakey == filekey -> noop + {- A changed symlink. -} + | otherwise -> stageannexlink file key + (Just key, _, _, Just cache) -> do + {- All direct mode files will show as + - modified, so compare the cache to see if + - it really was. -} + oldcache <- recordedInodeCache key + case oldcache of + [] -> modifiedannexed file key cache + _ -> unlessM (elemInodeCaches cache oldcache) $ + modifiedannexed file key cache + (Just key, _, Nothing, _) -> deletedannexed file key + (Nothing, _, Nothing, _) -> deletegit file + (_, _, Just _, _) -> addgit file + go _ = noop + + modifiedannexed file oldkey cache = do + void $ removeAssociatedFile oldkey file + void $ addDirect file cache + + deletedannexed file key = do + void $ removeAssociatedFile key file + deletegit file + + stageannexlink file key = do + l <- calcRepo $ gitAnnexLink file key + stageSymlink file =<< hashSymlink l + void $ addAssociatedFile key file + + addgit file = Annex.Queue.addCommand "add" [Param "-f"] [file] + + deletegit file = Annex.Queue.addCommand "rm" [Param "-qf"] [file] + +{- Run before a commit to update direct mode bookeeping to reflect the + - staged changes being committed. -} +preCommitDirect :: Annex Bool +preCommitDirect = do + (diffs, clean) <- inRepo $ DiffTree.diffIndex Git.Ref.headRef + makeabs <- flip fromTopFilePath <$> gitRepo + forM_ diffs (go makeabs) + liftIO clean + where + go makeabs diff = do + withkey (DiffTree.srcsha diff) (DiffTree.srcmode diff) removeAssociatedFile + withkey (DiffTree.dstsha diff) (DiffTree.dstmode diff) addAssociatedFile + where + withkey sha _mode a = when (sha /= nullSha) $ + catKey sha >>= \case + Nothing -> noop + Just key -> void $ a key $ + makeabs $ DiffTree.file diff + +{- Adds a file to the annex in direct mode. Can fail, if the file is + - modified or deleted while it's being added. -} +addDirect :: FilePath -> InodeCache -> Annex Bool +addDirect file cache = do + showStart "add" file + let source = KeySource + { keyFilename = file + , contentLocation = file + , inodeCache = Just cache + } + got =<< genKey source =<< chooseBackend file + where + got Nothing = do + showEndFail + return False + got (Just (key, _)) = ifM (sameInodeCache file [cache]) + ( do + l <- calcRepo $ gitAnnexLink file key + stageSymlink file =<< hashSymlink l + addInodeCache key cache + void $ addAssociatedFile key file + logStatus key InfoPresent + showEndOk + return True + , do + showEndFail + return False + ) + +{- In direct mode, git merge would usually refuse to do anything, since it + - sees present direct mode files as type changed files. + - + - So, to handle a merge, it's run with the work tree set to a temp + - directory, and the merge is staged into a copy of the index. + - Then the work tree is updated to reflect the merge, and + - finally, the merge is committed and the real index updated. + - + - A lock file is used to avoid races with any other caller of mergeDirect. + - + - To avoid other git processes from making changes to the index while our + - merge is in progress, the index lock file is used as the temp index + - file. This is the same as what git does when updating the index + - normally. + -} +mergeDirect :: Maybe Git.Ref -> Maybe Git.Ref -> Git.Branch -> Annex Bool -> [Git.Merge.MergeConfig] -> Git.Branch.CommitMode -> Annex Bool +mergeDirect startbranch oldref branch resolvemerge mergeconfig commitmode = exclusively $ do + reali <- liftIO . absPath =<< fromRepo indexFile + tmpi <- liftIO . absPath =<< fromRepo (indexFileLock . indexFile) + liftIO $ whenM (doesFileExist reali) $ + copyFile reali tmpi + + d <- fromRepo gitAnnexMergeDir + liftIO $ do + whenM (doesDirectoryExist d) $ + removeDirectoryRecursive d + createDirectoryIfMissing True d + + withIndexFile tmpi $ do + merged <- stageMerge d branch mergeconfig commitmode + ok <- if merged + then return True + else resolvemerge + if ok + then do + mergeDirectCleanup d (fromMaybe Git.Sha.emptyTree oldref) + mergeDirectCommit merged startbranch branch commitmode + liftIO $ whenM (doesFileExist tmpi) $ + rename tmpi reali + else do + liftIO $ nukeFile tmpi + liftIO $ removeDirectoryRecursive d + return ok + where + exclusively = withExclusiveLock gitAnnexMergeLock + +{- Stage a merge into the index, avoiding changing HEAD or the current + - branch. -} +stageMerge :: FilePath -> Git.Branch -> [Git.Merge.MergeConfig] -> Git.Branch.CommitMode -> Annex Bool +stageMerge d branch mergeconfig commitmode = do + -- XXX A bug in git makes stageMerge unsafe to use if the git repo + -- is configured with core.symlinks=false + -- Using merge is not ideal though, since it will + -- update the current branch immediately, before the work tree + -- has been updated, which would leave things in an inconsistent + -- state if mergeDirectCleanup is interrupted. + -- + merger <- ifM (coreSymlinks <$> Annex.getGitConfig) + ( return $ \ref -> Git.Merge.stageMerge ref mergeconfig + , return $ \ref -> Git.Merge.merge ref mergeconfig commitmode + ) + inRepo $ \g -> do + wd <- liftIO $ absPath d + gd <- liftIO $ absPath $ Git.localGitDir g + merger branch $ + g { location = Local { gitdir = gd, worktree = Just (addTrailingPathSeparator wd) } } + +{- Commits after a direct mode merge is complete, and after the work + - tree has been updated by mergeDirectCleanup. + -} +mergeDirectCommit :: Bool -> Maybe Git.Ref -> Git.Branch -> Git.Branch.CommitMode -> Annex () +mergeDirectCommit allowff old branch commitmode = do + void preCommitDirect + d <- fromRepo Git.localGitDir + let merge_head = d "MERGE_HEAD" + let merge_msg = d "MERGE_MSG" + let merge_mode = d "MERGE_MODE" + ifM (pure allowff <&&> canff) + ( inRepo $ Git.Branch.update "merge" Git.Ref.headRef branch -- fast forward + , do + msg <- liftIO $ + catchDefaultIO ("merge " ++ fromRef branch) $ + readFile merge_msg + void $ inRepo $ Git.Branch.commit commitmode False msg + Git.Ref.headRef [Git.Ref.headRef, branch] + ) + liftIO $ mapM_ nukeFile [merge_head, merge_msg, merge_mode] + where + canff = maybe (return False) (\o -> inRepo $ Git.Branch.fastForwardable o branch) old + +mergeDirectCleanup :: FilePath -> Git.Ref -> Annex () +mergeDirectCleanup d oldref = updateWorkTree d oldref False + +{- Updates the direct mode work tree to reflect the changes staged in the + - index by a git command, that was run in a temporary work tree. + - + - Uses diff-index to compare the staged changes with provided ref + - which should be the tree before the merge, and applies those + - changes to the work tree. + - + - There are really only two types of changes: An old item can be deleted, + - or a new item added. Two passes are made, first deleting and then + - adding. This is to handle cases where eg, a file is deleted and a + - directory is added. (The diff-tree output may list these in the opposite + - order, but we cannot add the directory until the file with the + - same name is removed.) + -} +updateWorkTree :: FilePath -> Git.Ref -> Bool -> Annex () +updateWorkTree d oldref force = do + (items, cleanup) <- inRepo $ DiffTree.diffIndex oldref + makeabs <- flip fromTopFilePath <$> gitRepo + let fsitems = zip (map (makeabs . DiffTree.file) items) items + forM_ fsitems $ + go makeabs DiffTree.srcsha moveout moveout_raw + forM_ fsitems $ + go makeabs DiffTree.dstsha movein movein_raw + void $ liftIO cleanup + where + go makeabs getsha a araw (f, item) + | getsha item == nullSha = noop + | otherwise = void $ + tryNonAsync . maybe (araw item makeabs f) (\k -> void $ a item makeabs k f) + =<< catKey (getsha item) + + moveout _ _ = removeDirect + + {- Files deleted by the merge are removed from the work tree. + - Empty work tree directories are removed, per git behavior. -} + moveout_raw _ _ f = liftIO $ do + nukeFile f + void $ tryIO $ removeDirectory $ parentDir f + + {- If the file is already present, with the right content for the + - key, it's left alone. + - + - If the file is already present, and does not exist in the + - oldref, preserve this local file. + - + - Otherwise, create the symlink and then if possible, replace it + - with the content. -} + movein item makeabs k f = unlessM (goodContent k f) $ do + unless force $ preserveUnannexed item makeabs f oldref + l <- calcRepo $ gitAnnexLink f k + replaceFile f $ makeAnnexLink l + toDirect k f + + {- Any new, modified, or renamed files were written to the temp + - directory by the merge, and are moved to the real work tree. -} + movein_raw item makeabs f = do + unless force $ preserveUnannexed item makeabs f oldref + liftIO $ do + createDirectoryIfMissing True $ parentDir f + void $ tryIO $ rename (d getTopFilePath (DiffTree.file item)) f + +{- If the file that's being moved in is already present in the work + - tree, but did not exist in the oldref, preserve this + - local, unannexed file (or directory), as "variant-local". + - + - It's also possible that the file that's being moved in + - is in a directory that collides with an exsting, non-annexed + - file (not a directory), which should be preserved. + -} +preserveUnannexed :: DiffTree.DiffTreeItem -> (TopFilePath -> FilePath) -> FilePath -> Ref -> Annex () +preserveUnannexed item makeabs absf oldref = do + whenM (liftIO (collidingitem absf) <&&> unannexed absf) $ + liftIO $ findnewname absf 0 + checkdirs (DiffTree.file item) + where + checkdirs from = case upFrom (getTopFilePath from) of + Nothing -> noop + Just p -> do + let d = asTopFilePath p + let absd = makeabs d + whenM (liftIO (colliding_nondir absd) <&&> unannexed absd) $ + liftIO $ findnewname absd 0 + checkdirs d + + collidingitem f = isJust + <$> catchMaybeIO (getSymbolicLinkStatus f) + colliding_nondir f = maybe False (not . isDirectory) + <$> catchMaybeIO (getSymbolicLinkStatus f) + + unannexed f = (isNothing <$> isAnnexLink f) + <&&> (isNothing <$> catFileDetails oldref f) + + findnewname :: FilePath -> Int -> IO () + findnewname f n = do + let localf = mkVariant f + ("local" ++ if n > 0 then show n else "") + ifM (collidingitem localf) + ( findnewname f (n+1) + , rename f localf + `catchIO` const (findnewname f (n+1)) + ) + +{- If possible, converts a symlink in the working tree into a direct + - mode file. If the content is not available, leaves the symlink + - unchanged. -} +toDirect :: Key -> FilePath -> Annex () +toDirect k f = fromMaybe noop =<< toDirectGen k f + +toDirectGen :: Key -> FilePath -> Annex (Maybe (Annex ())) +toDirectGen k f = do + loc <- calcRepo $ gitAnnexLocation k + ifM (liftIO $ doesFileExist loc) + ( return $ Just $ fromindirect loc + , do + {- Copy content from another direct file. -} + absf <- liftIO $ absPath f + dlocs <- filterM (goodContent k) =<< + filterM (\l -> isNothing <$> getAnnexLinkTarget l) =<< + (filter (/= absf) <$> addAssociatedFile k f) + case dlocs of + [] -> return Nothing + (dloc:_) -> return $ Just $ fromdirect dloc + ) + where + fromindirect loc = do + {- Move content from annex to direct file. -} + updateInodeCache k loc + void $ addAssociatedFile k f + modifyContent loc $ do + thawContent loc + liftIO (replaceFileFrom loc f) + `catchIO` (\_ -> freezeContent loc) + fromdirect loc = do + replaceFile f $ + liftIO . void . copyFileExternal CopyAllMetaData loc + updateInodeCache k f + +{- Removes a direct mode file, while retaining its content in the annex + - (unless its content has already been changed). -} +removeDirect :: Key -> FilePath -> Annex () +removeDirect k f = do + void $ removeAssociatedFileUnchecked k f + unlessM (inAnnex k) $ + -- If moveAnnex rejects the content of the key, + -- treat that the same as its content having changed. + ifM (goodContent k f) + ( unlessM (moveAnnex k f) $ + logStatus k InfoMissing + , logStatus k InfoMissing + ) + liftIO $ do + nukeFile f + void $ tryIO $ removeDirectory $ parentDir f + +{- Called when a direct mode file has been changed. Its old content may be + - lost. -} +changedDirect :: Key -> FilePath -> Annex () +changedDirect oldk f = do + locs <- removeAssociatedFile oldk f + whenM (pure (null locs) <&&> not <$> inAnnex oldk) $ + logStatus oldk InfoMissing + +{- Git config settings to enable/disable direct mode. -} +setDirect :: Bool -> Annex () +setDirect wantdirect = do + if wantdirect + then do + switchHEAD + setbare + else do + setbare + switchHEADBack + setConfig (annexConfig "direct") val + Annex.changeGitConfig $ \c -> c { annexDirect = wantdirect } + where + val = Git.Config.boolConfig wantdirect + coreworktree = ConfigKey "core.worktree" + indirectworktree = ConfigKey "core.indirect-worktree" + setbare = do + -- core.worktree is not compatable with + -- core.bare; git does not allow both to be set, so + -- unset it when enabling direct mode, caching in + -- core.indirect-worktree + if wantdirect + then moveconfig coreworktree indirectworktree + else moveconfig indirectworktree coreworktree + setConfig (ConfigKey Git.Config.coreBare) val + moveconfig src dest = getConfigMaybe src >>= \case + Nothing -> noop + Just wt -> do + unsetConfig src + setConfig dest wt + reloadConfig + +{- Since direct mode sets core.bare=true, incoming pushes could change + - the currently checked out branch. To avoid this problem, HEAD + - is changed to a internal ref that nothing is going to push to. + - + - For refs/heads/master, use refs/heads/annex/direct/master; + - this way things that show HEAD (eg shell prompts) will + - hopefully show just "master". -} +directBranch :: Ref -> Ref +directBranch orighead = case splitc '/' $ fromRef orighead of + ("refs":"heads":"annex":"direct":_) -> orighead + ("refs":"heads":rest) -> + Ref $ "refs/heads/annex/direct/" ++ intercalate "/" rest + _ -> Ref $ "refs/heads/" ++ fromRef (Git.Ref.base orighead) + +{- Converts a directBranch back to the original branch. + - + - Any other ref is left unchanged. + -} +fromDirectBranch :: Ref -> Ref +fromDirectBranch directhead = case splitc '/' $ fromRef directhead of + ("refs":"heads":"annex":"direct":rest) -> + Ref $ "refs/heads/" ++ intercalate "/" rest + _ -> directhead + +switchHEAD :: Annex () +switchHEAD = maybe noop switch =<< inRepo Git.Branch.currentUnsafe + where + switch orighead = do + let newhead = directBranch orighead + maybe noop (inRepo . Git.Branch.update "entering direct mode" newhead) + =<< inRepo (Git.Ref.sha orighead) + inRepo $ Git.Branch.checkout newhead + +switchHEADBack :: Annex () +switchHEADBack = maybe noop switch =<< inRepo Git.Branch.currentUnsafe + where + switch currhead = do + let orighead = fromDirectBranch currhead + inRepo (Git.Ref.sha currhead) >>= \case + Just headsha + | orighead /= currhead -> do + inRepo $ Git.Branch.update "leaving direct mode" orighead headsha + inRepo $ Git.Branch.checkout orighead + inRepo $ Git.Branch.delete currhead + _ -> inRepo $ Git.Branch.checkout orighead diff --git a/Annex/Drop.hs b/Annex/Drop.hs new file mode 100644 index 0000000000..a2b1322051 --- /dev/null +++ b/Annex/Drop.hs @@ -0,0 +1,134 @@ +{- dropping of unwanted content + - + - Copyright 2012-2014 Joey Hess + - + - Licensed under the GNU GPL version 3 or higher. + -} + +module Annex.Drop where + +import Annex.Common +import qualified Annex +import Logs.Trust +import Annex.NumCopies +import Types.Remote (uuid, appendonly) +import qualified Remote +import qualified Command.Drop +import Command +import Annex.Wanted +import Config +import Annex.Content.Direct +import qualified Database.Keys +import Git.FilePath + +import qualified Data.Set as S +import System.Log.Logger (debugM) + +type Reason = String + +{- Drop a key from local and/or remote when allowed by the preferred content + - and numcopies settings. + - + - Skips trying to drop from remotes that are appendonly, since those drops + - would presumably fail. + - + - The UUIDs are ones where the content is believed to be present. + - The Remote list can include other remotes that do not have the content; + - only ones that match the UUIDs will be dropped from. + - + - If allowed to drop fromhere, that drop will be done last. This is done + - because local drops do not need any LockedCopy evidence, and so dropping + - from local last allows the content to be removed from more remotes. + - + - A VerifiedCopy can be provided as an optimisation when eg, a key + - has just been uploaded to a remote. + - + - In direct mode, all associated files are checked, and only if all + - of them are unwanted are they dropped. + - + - The runner is used to run commands, and so can be either callCommand + - or commandAction. + -} +handleDropsFrom :: [UUID] -> [Remote] -> Reason -> Bool -> Key -> AssociatedFile -> [VerifiedCopy] -> (CommandStart -> CommandCleanup) -> Annex () +handleDropsFrom locs rs reason fromhere key afile preverified runner = do + l <- ifM isDirect + ( associatedFilesRelative key + , do + g <- Annex.gitRepo + map (`fromTopFilePath` g) <$> Database.Keys.getAssociatedFiles key + ) + let fs = case afile of + AssociatedFile (Just f) -> nub (f : l) + AssociatedFile Nothing -> l + n <- getcopies fs + let rs' = filter (not . appendonly) rs + void $ if fromhere && checkcopies n Nothing + then go fs rs' n >>= dropl fs + else go fs rs' n + where + getcopies fs = do + (untrusted, have) <- trustPartition UnTrusted locs + numcopies <- if null fs + then getNumCopies + else maximum <$> mapM getFileNumCopies fs + return (NumCopies (length have), numcopies, S.fromList untrusted) + + {- Check that we have enough copies still to drop the content. + - When the remote being dropped from is untrusted, it was not + - counted as a copy, so having only numcopies suffices. Otherwise, + - we need more than numcopies to safely drop. -} + checkcopies (have, numcopies, _untrusted) Nothing = have > numcopies + checkcopies (have, numcopies, untrusted) (Just u) + | S.member u untrusted = have >= numcopies + | otherwise = have > numcopies + + decrcopies (have, numcopies, untrusted) Nothing = + (NumCopies (fromNumCopies have - 1), numcopies, untrusted) + decrcopies v@(_have, _numcopies, untrusted) (Just u) + | S.member u untrusted = v + | otherwise = decrcopies v Nothing + + go _ [] n = pure n + go fs (r:rest) n + | uuid r `S.notMember` slocs = go fs rest n + | checkcopies n (Just $ Remote.uuid r) = + dropr fs r n >>= go fs rest + | otherwise = pure n + + checkdrop fs n u a + | null fs = check $ -- no associated files; unused content + wantDrop True u (Just key) (AssociatedFile Nothing) + | otherwise = check $ + allM (wantDrop True u (Just key) . AssociatedFile . Just) fs + where + check c = ifM c + ( dodrop n u a + , return n + ) + + dodrop n@(have, numcopies, _untrusted) u a = + ifM (safely $ runner $ a numcopies) + ( do + liftIO $ debugM "drop" $ unwords + [ "dropped" + , case afile of + AssociatedFile Nothing -> key2file key + AssociatedFile (Just af) -> af + , "(from " ++ maybe "here" show u ++ ")" + , "(copies now " ++ show (fromNumCopies have - 1) ++ ")" + , ": " ++ reason + ] + return $ decrcopies n u + , return n + ) + + dropl fs n = checkdrop fs n Nothing $ \numcopies -> + Command.Drop.startLocal afile (mkActionItem afile) numcopies key preverified + + dropr fs r n = checkdrop fs n (Just $ Remote.uuid r) $ \numcopies -> + Command.Drop.startRemote afile (mkActionItem afile) numcopies key r + + slocs = S.fromList locs + + safely a = either (const False) id <$> tryNonAsync a + diff --git a/Annex/Environment.hs b/Annex/Environment.hs new file mode 100644 index 0000000000..6fdac1e498 --- /dev/null +++ b/Annex/Environment.hs @@ -0,0 +1,58 @@ +{- git-annex environment + - + - Copyright 2012, 2013 Joey Hess + - + - Licensed under the GNU GPL version 3 or higher. + -} + +{-# LANGUAGE CPP #-} + +module Annex.Environment where + +import Annex.Common +import Utility.UserInfo +import qualified Git.Config +import Config +import Utility.Env.Set + +{- Checks that the system's environment allows git to function. + - Git requires a GECOS username, or suitable git configuration, or + - environment variables. + - + - Git also requires the system have a hostname containing a dot. + - Otherwise, it tries various methods to find a FQDN, and will fail if it + - does not. To avoid replicating that code here, which would break if its + - methods change, this function does not check the hostname is valid. + - Instead, code that commits can use ensureCommit. + -} +checkEnvironment :: Annex () +checkEnvironment = do + gitusername <- fromRepo $ Git.Config.getMaybe "user.name" + when (isNothing gitusername || gitusername == Just "") $ + liftIO checkEnvironmentIO + +checkEnvironmentIO :: IO () +checkEnvironmentIO = whenM (isNothing <$> myUserGecos) $ do + username <- either (const "unknown") id <$> myUserName + ensureEnv "GIT_AUTHOR_NAME" username + ensureEnv "GIT_COMMITTER_NAME" username + where +#ifndef __ANDROID__ + -- existing environment is not overwritten + ensureEnv var val = setEnv var val False +#else + -- Environment setting is broken on Android, so this is dealt with + -- in runshell instead. + ensureEnv _ _ = noop +#endif + +{- Runs an action that commits to the repository, and if it fails, + - sets user.email and user.name to a dummy value and tries the action again. -} +ensureCommit :: Annex a -> Annex a +ensureCommit a = either retry return =<< tryNonAsync a + where + retry _ = do + name <- liftIO $ either (const "unknown") id <$> myUserName + setConfig (ConfigKey "user.name") name + setConfig (ConfigKey "user.email") name + a diff --git a/Annex/Export.hs b/Annex/Export.hs new file mode 100644 index 0000000000..b5be2efbad --- /dev/null +++ b/Annex/Export.hs @@ -0,0 +1,45 @@ +{- git-annex exports + - + - Copyright 2017 Joey Hess + - + - Licensed under the GNU GPL version 3 or higher. + -} + +module Annex.Export where + +import Annex +import Annex.CatFile +import Types.Key +import Types.Remote +import qualified Git + +import qualified Data.Map as M +import Control.Applicative +import Prelude + +-- An export includes both annexed files and files stored in git. +-- For the latter, a SHA1 key is synthesized. +data ExportKey = AnnexKey Key | GitKey Key + deriving (Show, Eq, Ord) + +asKey :: ExportKey -> Key +asKey (AnnexKey k) = k +asKey (GitKey k) = k + +exportKey :: Git.Sha -> Annex ExportKey +exportKey sha = mk <$> catKey sha + where + mk (Just k) = AnnexKey k + mk Nothing = GitKey $ Key + { keyName = Git.fromRef sha + , keyVariety = SHA1Key (HasExt False) + , keySize = Nothing + , keyMtime = Nothing + , keyChunkSize = Nothing + , keyChunkNum = Nothing + } + +exportTree :: RemoteConfig -> Bool +exportTree c = case M.lookup "exporttree" c of + Just "yes" -> True + _ -> False diff --git a/Annex/FileMatcher.hs b/Annex/FileMatcher.hs new file mode 100644 index 0000000000..44565e321c --- /dev/null +++ b/Annex/FileMatcher.hs @@ -0,0 +1,186 @@ +{- git-annex file matching + - + - Copyright 2012-2016 Joey Hess + - + - Licensed under the GNU GPL version 3 or higher. + -} + +{-# LANGUAGE CPP #-} + +module Annex.FileMatcher ( + GetFileMatcher, + checkFileMatcher, + checkFileMatcher', + checkMatcher, + matchAll, + preferredContentParser, + parsedToMatcher, + mkLargeFilesParser, + largeFilesMatcher, +) where + +import qualified Data.Map as M + +import Annex.Common +import Limit +import Utility.Matcher +import Types.Group +import qualified Annex +import Types.FileMatcher +import Git.FilePath +import Types.Remote (RemoteConfig) +import Annex.CheckAttr +import Git.CheckAttr (unspecifiedAttr) + +#ifdef WITH_MAGICMIME +import Magic +import Utility.Env +#endif + +import Data.Either +import qualified Data.Set as S + +type GetFileMatcher = FilePath -> Annex (FileMatcher Annex) + +checkFileMatcher :: GetFileMatcher -> FilePath -> Annex Bool +checkFileMatcher getmatcher file = checkFileMatcher' getmatcher file (return True) + +-- | Allows running an action when no matcher is configured for the file. +checkFileMatcher' :: GetFileMatcher -> FilePath -> Annex Bool -> Annex Bool +checkFileMatcher' getmatcher file notconfigured = do + matcher <- getmatcher file + checkMatcher matcher Nothing afile S.empty notconfigured d + where + afile = AssociatedFile (Just file) + -- checkMatcher will never use this, because afile is provided. + d = return True + +checkMatcher :: FileMatcher Annex -> Maybe Key -> AssociatedFile -> AssumeNotPresent -> Annex Bool -> Annex Bool -> Annex Bool +checkMatcher matcher mkey afile notpresent notconfigured d + | isEmpty matcher = notconfigured + | otherwise = case (mkey, afile) of + (_, AssociatedFile (Just file)) -> go =<< fileMatchInfo file + (Just key, _) -> go (MatchingKey key) + _ -> d + where + go mi = matchMrun matcher $ \a -> a notpresent mi + +fileMatchInfo :: FilePath -> Annex MatchInfo +fileMatchInfo file = do + matchfile <- getTopFilePath <$> inRepo (toTopFilePath file) + return $ MatchingFile FileInfo + { matchFile = matchfile + , currFile = file + } + +matchAll :: FileMatcher Annex +matchAll = generate [] + +parsedToMatcher :: [ParseResult] -> Either String (FileMatcher Annex) +parsedToMatcher parsed = case partitionEithers parsed of + ([], vs) -> Right $ generate vs + (es, _) -> Left $ unwords $ map ("Parse failure: " ++) es + +data ParseToken + = SimpleToken String ParseResult + | ValueToken String (String -> ParseResult) + +type ParseResult = Either String (Token (MatchFiles Annex)) + +parseToken :: [ParseToken] -> String -> ParseResult +parseToken l t + | t `elem` tokens = Right $ token t + | otherwise = go l + where + go [] = Left $ "near " ++ show t + go (SimpleToken s r : _) | s == t = r + go (ValueToken s mkr : _) | s == k = mkr v + go (_ : ps) = go ps + (k, v) = separate (== '=') t + +commonTokens :: [ParseToken] +commonTokens = + [ SimpleToken "unused" (simply limitUnused) + , SimpleToken "anything" (simply limitAnything) + , SimpleToken "nothing" (simply limitNothing) + , ValueToken "include" (usev limitInclude) + , ValueToken "exclude" (usev limitExclude) + , ValueToken "largerthan" (usev $ limitSize (>)) + , ValueToken "smallerthan" (usev $ limitSize (<)) + ] + +{- This is really dumb tokenization; there's no support for quoted values. + - Open and close parens are always treated as standalone tokens; + - otherwise tokens must be separated by whitespace. -} +tokenizeMatcher :: String -> [String] +tokenizeMatcher = filter (not . null ) . concatMap splitparens . words + where + splitparens = segmentDelim (`elem` "()") + +preferredContentParser :: FileMatcher Annex -> FileMatcher Annex -> Annex GroupMap -> M.Map UUID RemoteConfig -> Maybe UUID -> String -> [ParseResult] +preferredContentParser matchstandard matchgroupwanted getgroupmap configmap mu expr = + map parse $ tokenizeMatcher expr + where + parse = parseToken $ + [ SimpleToken "standard" (call matchstandard) + , SimpleToken "groupwanted" (call matchgroupwanted) + , SimpleToken "present" (simply $ limitPresent mu) + , SimpleToken "inpreferreddir" (simply $ limitInDir preferreddir) + , SimpleToken "securehash" (simply limitSecureHash) + , ValueToken "copies" (usev limitCopies) + , ValueToken "lackingcopies" (usev $ limitLackingCopies False) + , ValueToken "approxlackingcopies" (usev $ limitLackingCopies True) + , ValueToken "inbacked" (usev limitInBackend) + , ValueToken "metadata" (usev limitMetaData) + , ValueToken "inallgroup" (usev $ limitInAllGroup getgroupmap) + ] ++ commonTokens + preferreddir = fromMaybe "public" $ + M.lookup "preferreddir" =<< (`M.lookup` configmap) =<< mu + +mkLargeFilesParser :: Annex (String -> [ParseResult]) +mkLargeFilesParser = do +#ifdef WITH_MAGICMIME + magicmime <- liftIO $ catchMaybeIO $ do + m <- magicOpen [MagicMimeType] + liftIO $ getEnv "GIT_ANNEX_DIR" >>= \case + Nothing -> magicLoadDefault m + Just d -> magicLoad m + (d "magic" "magic.mgc") + return m +#endif + let parse = parseToken $ commonTokens +#ifdef WITH_MAGICMIME + ++ [ ValueToken "mimetype" (usev $ matchMagic magicmime) ] +#else + ++ [ ValueToken "mimetype" (const $ Left "\"mimetype\" not supported; not built with MagicMime support") ] +#endif + return $ map parse . tokenizeMatcher + +{- Generates a matcher for files large enough (or meeting other criteria) + - to be added to the annex, rather than directly to git. -} +largeFilesMatcher :: Annex GetFileMatcher +largeFilesMatcher = go =<< annexLargeFiles <$> Annex.getGitConfig + where + go (Just expr) = do + matcher <- mkmatcher expr + return $ const $ return matcher + go Nothing = return $ \file -> do + expr <- checkAttr "annex.largefiles" file + if null expr || expr == unspecifiedAttr + then return matchAll + else mkmatcher expr + + mkmatcher expr = do + parser <- mkLargeFilesParser + either badexpr return $ parsedToMatcher $ parser expr + badexpr e = giveup $ "bad annex.largefiles configuration: " ++ e + +simply :: MatchFiles Annex -> ParseResult +simply = Right . Operation + +usev :: MkLimit Annex -> String -> ParseResult +usev a v = Operation <$> a v + +call :: FileMatcher Annex -> ParseResult +call sub = Right $ Operation $ \notpresent mi -> + matchMrun sub $ \a -> a notpresent mi diff --git a/Annex/Fixup.hs b/Annex/Fixup.hs new file mode 100644 index 0000000000..049aad3ad5 --- /dev/null +++ b/Annex/Fixup.hs @@ -0,0 +1,148 @@ +{- git-annex repository fixups + - + - Copyright 2013-2018 Joey Hess + - + - Licensed under the GNU GPL version 3 or higher. + -} + +module Annex.Fixup where + +import Git.Types +import Git.Config +import Types.GitConfig +import qualified Git.BuildVersion +import Utility.Path +import Utility.SafeCommand +import Utility.Directory +import Utility.Exception +import Utility.Monad +import Utility.PartialPrelude + +import System.IO +import System.FilePath +import System.PosixCompat.Files +import Data.List +import Control.Monad +import Control.Monad.IfElse +import qualified Data.Map as M +import Control.Applicative +import Prelude + +fixupRepo :: Repo -> GitConfig -> IO Repo +fixupRepo r c = do + let r' = disableWildcardExpansion r + r'' <- fixupUnusualRepos r' c + if annexDirect c + then return (fixupDirect r'') + else return r'' + +{- Disable git's built-in wildcard expansion, which is not wanted + - when using it as plumbing by git-annex. -} +disableWildcardExpansion :: Repo -> Repo +disableWildcardExpansion r + | Git.BuildVersion.older "1.8.1" = r + | otherwise = r + { gitGlobalOpts = gitGlobalOpts r ++ [Param "--literal-pathspecs"] } + +{- Direct mode repos have core.bare=true, but are not really bare. + - Fix up the Repo to be a non-bare repo, and arrange for git commands + - run by git-annex to be passed parameters that override this setting. -} +fixupDirect :: Repo -> Repo +fixupDirect r@(Repo { location = l@(Local { gitdir = d, worktree = Nothing }) }) = do + r + { location = l { worktree = Just (parentDir d) } + , gitGlobalOpts = gitGlobalOpts r ++ + [ Param "-c" + , Param $ coreBare ++ "=" ++ boolConfig False + ] + } +fixupDirect r = r + +{- Submodules have their gitdir containing ".git/modules/", and + - have core.worktree set, and also have a .git file in the top + - of the repo. We need to unset core.worktree, and change the .git + - file into a symlink to the git directory. This way, annex symlinks will be + - of the usual .git/annex/object form, and will consistently work + - whether a repo is used as a submodule or not, and wheverever the + - submodule is mounted. + - + - git-worktree directories have a .git file. + - That needs to be converted to a symlink, and .git/annex made a symlink + - to the main repository's git-annex directory. + - The worktree shares git config with the main repository, so the same + - annex uuid and other configuration will be used in the worktree as in + - the main repository. + - + - git clone or init with --separate-git-dir similarly makes a .git file, + - which in that case points to a different git directory. It's + - also converted to a symlink so links to .git/annex will work. + - + - When the filesystem doesn't support symlinks, we cannot make .git + - into a symlink. But we don't need too, since the repo will use direct + - mode. + -} +fixupUnusualRepos :: Repo -> GitConfig -> IO Repo +fixupUnusualRepos r@(Repo { location = l@(Local { worktree = Just w, gitdir = d }) }) c + | needsSubmoduleFixup r = do + when (coreSymlinks c) $ + (replacedotgit >> unsetcoreworktree) + `catchNonAsync` \_e -> hPutStrLn stderr + "warning: unable to convert submodule to form that will work with git-annex" + return $ r' + { config = M.delete "core.worktree" (config r) + } + | otherwise = ifM (needsGitLinkFixup r) + ( do + when (coreSymlinks c) $ + (replacedotgit >> worktreefixup) + `catchNonAsync` \_e -> hPutStrLn stderr + "warning: unable to convert .git file to symlink that will work with git-annex" + return r' + , return r + ) + where + dotgit = w ".git" + + replacedotgit = whenM (doesFileExist dotgit) $ do + linktarget <- relPathDirToFile w d + nukeFile dotgit + createSymbolicLink linktarget dotgit + + unsetcoreworktree = + maybe (error "unset core.worktree failed") (\_ -> return ()) + =<< Git.Config.unset "core.worktree" r + + worktreefixup = + -- git-worktree sets up a "commondir" file that contains + -- the path to the main git directory. + -- Using --separate-git-dir does not. + catchDefaultIO Nothing (headMaybe . lines <$> readFile (d "commondir")) >>= \case + Just gd -> do + -- Make the worktree's git directory + -- contain an annex symlink to the main + -- repository's annex directory. + let linktarget = gd "annex" + createSymbolicLink linktarget (dotgit "annex") + Nothing -> return () + + -- Repo adjusted, so that symlinks to objects that get checked + -- in will have the usual path, rather than pointing off to the + -- real .git directory. + r' + | coreSymlinks c = r { location = l { gitdir = dotgit } } + | otherwise = r +fixupUnusualRepos r _ = return r + +needsSubmoduleFixup :: Repo -> Bool +needsSubmoduleFixup (Repo { location = (Local { worktree = Just _, gitdir = d }) }) = + (".git" "modules") `isInfixOf` d +needsSubmoduleFixup _ = False + +needsGitLinkFixup :: Repo -> IO Bool +needsGitLinkFixup (Repo { location = (Local { worktree = Just wt, gitdir = d }) }) + -- Optimization: Avoid statting .git in the common case; only + -- when the gitdir is not in the usual place inside the worktree + -- might .git be a file. + | wt ".git" == d = return False + | otherwise = doesFileExist (wt ".git") +needsGitLinkFixup _ = return False diff --git a/Annex/GitOverlay.hs b/Annex/GitOverlay.hs new file mode 100644 index 0000000000..d809e0b23a --- /dev/null +++ b/Annex/GitOverlay.hs @@ -0,0 +1,108 @@ +{- Temporarily changing the files git uses. + - + - Copyright 2014-2016 Joey Hess + - + - Licensed under the GNU GPL version 3 or higher. + -} + +module Annex.GitOverlay where + +import qualified Control.Exception as E + +import Annex.Common +import Git +import Git.Types +import Git.Index +import Git.Env +import qualified Annex +import qualified Annex.Queue + +{- Runs an action using a different git index file. -} +withIndexFile :: FilePath -> Annex a -> Annex a +withIndexFile f a = do + f' <- liftIO $ indexEnvVal f + withAltRepo + (usecachedgitenv $ \g -> liftIO $ addGitEnv g indexEnv f') + (\g g' -> g' { gitEnv = gitEnv g }) + a + where + -- This is an optimisation. Since withIndexFile is run repeatedly, + -- and addGitEnv uses the slow copyGitEnv when gitEnv is Nothing, + -- we cache the copied environment the first time, and reuse it in + -- subsequent calls. + -- + -- (This could be done at another level; eg when creating the + -- Git object in the first place, but it's more efficient to let + -- the enviroment be inherited in all calls to git where it + -- does not need to be modified.) + usecachedgitenv m g = case gitEnv g of + Just _ -> m g + Nothing -> do + e <- Annex.withState $ \s -> case Annex.cachedgitenv s of + Nothing -> do + e <- copyGitEnv + return (s { Annex.cachedgitenv = Just e }, e) + Just e -> return (s, e) + m (g { gitEnv = Just e }) + +{- Runs an action using a different git work tree. + - + - Smudge and clean filters are disabled in this work tree. -} +withWorkTree :: FilePath -> Annex a -> Annex a +withWorkTree d = withAltRepo + (\g -> return $ g { location = modlocation (location g), gitGlobalOpts = gitGlobalOpts g ++ disableSmudgeConfig }) + (\g g' -> g' { location = location g, gitGlobalOpts = gitGlobalOpts g }) + where + modlocation l@(Local {}) = l { worktree = Just d } + modlocation _ = error "withWorkTree of non-local git repo" + disableSmudgeConfig = map Param + [ "-c", "filter.annex.smudge=" + , "-c", "filter.annex.clean=" + ] + +{- Runs an action with the git index file and HEAD, and a few other + - files that are related to the work tree coming from an overlay + - directory other than the usual. This is done by pointing + - GIT_COMMON_DIR at the regular git directory, and GIT_DIR at the + - overlay directory. + - + - Needs git 2.2.0 or newer. + -} +withWorkTreeRelated :: FilePath -> Annex a -> Annex a +withWorkTreeRelated d = withAltRepo modrepo unmodrepo + where + modrepo g = liftIO $ do + g' <- addGitEnv g "GIT_COMMON_DIR" =<< absPath (localGitDir g) + g'' <- addGitEnv g' "GIT_DIR" d + return (g'' { gitEnvOverridesGitDir = True }) + unmodrepo g g' = g' + { gitEnv = gitEnv g + , gitEnvOverridesGitDir = gitEnvOverridesGitDir g + } + +withAltRepo + :: (Repo -> Annex Repo) + -- ^ modify Repo + -> (Repo -> Repo -> Repo) + -- ^ undo modifications; first Repo is the original and second + -- is the one after running the action. + -> Annex a + -> Annex a +withAltRepo modrepo unmodrepo a = do + g <- gitRepo + g' <- modrepo g + q <- Annex.Queue.get + v <- tryNonAsync $ do + Annex.changeState $ \s -> s + { Annex.repo = g' + -- Start a separate queue for any changes made + -- with the modified repo. + , Annex.repoqueue = Nothing + } + a + void $ tryNonAsync Annex.Queue.flush + Annex.changeState $ \s -> s + { Annex.repo = unmodrepo g (Annex.repo s) + , Annex.repoqueue = Just q + } + either E.throw return v diff --git a/Annex/HashObject.hs b/Annex/HashObject.hs new file mode 100644 index 0000000000..16f7414076 --- /dev/null +++ b/Annex/HashObject.hs @@ -0,0 +1,47 @@ +{- git hash-object interface, with handle automatically stored in the Annex monad + - + - Copyright 2016 Joey Hess + - + - Licensed under the GNU GPL version 3 or higher. + -} + +module Annex.HashObject ( + hashFile, + hashBlob, + hashObjectHandle, + hashObjectStop, +) where + +import Annex.Common +import qualified Git.HashObject +import qualified Annex +import Git.Types + +hashObjectHandle :: Annex Git.HashObject.HashObjectHandle +hashObjectHandle = maybe startup return =<< Annex.getState Annex.hashobjecthandle + where + startup = do + h <- inRepo $ Git.HashObject.hashObjectStart + Annex.changeState $ \s -> s { Annex.hashobjecthandle = Just h } + return h + +hashObjectStop :: Annex () +hashObjectStop = maybe noop stop =<< Annex.getState Annex.hashobjecthandle + where + stop h = do + liftIO $ Git.HashObject.hashObjectStop h + Annex.changeState $ \s -> s { Annex.hashobjecthandle = Nothing } + return () + +hashFile :: FilePath -> Annex Sha +hashFile f = do + h <- hashObjectHandle + liftIO $ Git.HashObject.hashFile h f + +{- Note that the content will be written to a temp file. + - So it may be faster to use Git.HashObject.hashObject for large + - blob contents. -} +hashBlob :: String -> Annex Sha +hashBlob content = do + h <- hashObjectHandle + liftIO $ Git.HashObject.hashBlob h content diff --git a/Annex/Hook.hs b/Annex/Hook.hs new file mode 100644 index 0000000000..450ed261b3 --- /dev/null +++ b/Annex/Hook.hs @@ -0,0 +1,80 @@ +{- git-annex git hooks + - + - Note that it's important that the scripts installed by git-annex + - not change, otherwise removing old hooks using an old version of + - the script would fail. + - + - Copyright 2013-2018 Joey Hess + - + - Licensed under the GNU GPL version 3 or higher. + -} + +module Annex.Hook where + +import Annex.Common +import qualified Git.Hook as Git +import Config +import qualified Annex +import Utility.Shell + +import qualified Data.Map as M + +preCommitHook :: Git.Hook +preCommitHook = Git.Hook "pre-commit" (mkHookScript "git annex pre-commit .") [] + +postReceiveHook :: Git.Hook +postReceiveHook = Git.Hook "post-receive" + -- Only run git-annex post-receive when git-annex supports it, + -- to avoid failing if the repository with this hook is used + -- with an older version of git-annex. + (mkHookScript "if git annex post-receive --help >/dev/null 2>&1; then git annex post-receive; fi") + -- This is an old version of the hook script. + [ mkHookScript "git annex post-receive" + ] + +preCommitAnnexHook :: Git.Hook +preCommitAnnexHook = Git.Hook "pre-commit-annex" "" [] + +postUpdateAnnexHook :: Git.Hook +postUpdateAnnexHook = Git.Hook "post-update-annex" "" [] + +mkHookScript :: String -> String +mkHookScript s = unlines + [ shebang_local + , "# automatically configured by git-annex" + , s + ] + +hookWrite :: Git.Hook -> Annex () +hookWrite h = + -- cannot have git hooks in a crippled filesystem (no execute bit) + unlessM crippledFileSystem $ + unlessM (inRepo $ Git.hookWrite h) $ + hookWarning h "already exists, not configuring" + +hookUnWrite :: Git.Hook -> Annex () +hookUnWrite h = unlessM (inRepo $ Git.hookUnWrite h) $ + hookWarning h "contents modified; not deleting. Edit it to remove call to git annex." + +hookWarning :: Git.Hook -> String -> Annex () +hookWarning h msg = do + r <- gitRepo + warning $ Git.hookName h ++ " hook (" ++ Git.hookFile h r ++ ") " ++ msg + +{- Runs a hook. To avoid checking if the hook exists every time, + - the existing hooks are cached. -} +runAnnexHook :: Git.Hook -> Annex () +runAnnexHook hook = do + m <- Annex.getState Annex.existinghooks + case M.lookup hook m of + Just True -> run + Just False -> noop + Nothing -> do + exists <- inRepo $ Git.hookExists hook + Annex.changeState $ \s -> s + { Annex.existinghooks = M.insert hook exists m } + when exists run + where + run = unlessM (inRepo $ Git.runHook hook) $ do + h <- fromRepo $ Git.hookFile hook + warning $ h ++ " failed" diff --git a/Annex/Ingest.hs b/Annex/Ingest.hs new file mode 100644 index 0000000000..aa556b3711 --- /dev/null +++ b/Annex/Ingest.hs @@ -0,0 +1,390 @@ +{- git-annex content ingestion + - + - Copyright 2010-2017 Joey Hess + - + - Licensed under the GNU GPL version 3 or higher. + -} + +module Annex.Ingest ( + LockedDown(..), + LockDownConfig(..), + lockDown, + ingestAdd, + ingestAdd', + ingest, + ingest', + finishIngestDirect, + finishIngestUnlocked, + cleanOldKeys, + addLink, + makeLink, + addUnlocked, + restoreFile, + forceParams, + addAnnexedFile, +) where + +import Annex.Common +import Types.KeySource +import Backend +import Annex.Content +import Annex.Content.Direct +import Annex.Perms +import Annex.Link +import Annex.MetaData +import Annex.Version +import Logs.Location +import qualified Annex +import qualified Annex.Queue +import qualified Database.Keys +import qualified Git +import qualified Git.Branch +import Config +import Utility.InodeCache +import Annex.ReplaceFile +import Utility.Tmp +import Utility.CopyFile +import Utility.Touch +import Git.FilePath +import Annex.InodeSentinal +import Annex.AdjustedBranch + +import Control.Exception (IOException) + +data LockedDown = LockedDown + { lockDownConfig :: LockDownConfig + , keySource :: KeySource + } + deriving (Show) + +data LockDownConfig = LockDownConfig + { lockingFile :: Bool -- ^ write bit removed during lock down + , hardlinkFileTmp :: Bool -- ^ hard link to temp directory + } + deriving (Show) + +{- The file that's being ingested is locked down before a key is generated, + - to prevent it from being modified in between. This lock down is not + - perfect at best (and pretty weak at worst). For example, it does not + - guard against files that are already opened for write by another process. + - So, the InodeCache can be used to detect any changes that might be made + - to the file after it was locked down. + - + - When possible, the file is hard linked to a temp directory. This guards + - against some changes, like deletion or overwrite of the file, and + - allows lsof checks to be done more efficiently when adding a lot of files. + - + - Lockdown can fail if a file gets deleted, and Nothing will be returned. + -} +lockDown :: LockDownConfig -> FilePath -> Annex (Maybe LockedDown) +lockDown cfg file = either + (\e -> warning (show e) >> return Nothing) + (return . Just) + =<< lockDown' cfg file + +lockDown' :: LockDownConfig -> FilePath -> Annex (Either IOException LockedDown) +lockDown' cfg file = ifM (pure (not (hardlinkFileTmp cfg)) <||> crippledFileSystem) + ( withTSDelta $ liftIO . tryIO . nohardlink + , tryIO $ do + tmp <- fromRepo gitAnnexTmpMiscDir + createAnnexDirectory tmp + when (lockingFile cfg) $ + freezeContent file + withTSDelta $ \delta -> liftIO $ do + (tmpfile, h) <- openTempFile tmp $ + relatedTemplate $ takeFileName file + hClose h + nukeFile tmpfile + withhardlink delta tmpfile `catchIO` const (nohardlink delta) + ) + where + nohardlink delta = do + cache <- genInodeCache file delta + return $ LockedDown cfg $ KeySource + { keyFilename = file + , contentLocation = file + , inodeCache = cache + } + withhardlink delta tmpfile = do + createLink file tmpfile + cache <- genInodeCache tmpfile delta + return $ LockedDown cfg $ KeySource + { keyFilename = file + , contentLocation = tmpfile + , inodeCache = cache + } + +{- Ingests a locked down file into the annex. Updates the work tree and + - index. -} +ingestAdd :: Maybe LockedDown -> Annex (Maybe Key) +ingestAdd ld = ingestAdd' ld Nothing + +ingestAdd' :: Maybe LockedDown -> Maybe Key -> Annex (Maybe Key) +ingestAdd' Nothing _ = return Nothing +ingestAdd' ld@(Just (LockedDown cfg source)) mk = do + (mk', mic) <- ingest ld mk + case mk' of + Nothing -> return Nothing + Just k -> do + let f = keyFilename source + if lockingFile cfg + then addLink f k mic + else ifM isDirect + ( do + l <- calcRepo $ gitAnnexLink f k + stageSymlink f =<< hashSymlink l + , do + mode <- liftIO $ catchMaybeIO $ fileMode <$> getFileStatus (contentLocation source) + stagePointerFile f mode =<< hashPointerFile k + ) + return (Just k) + +{- Ingests a locked down file into the annex. Does not update the working + - tree or the index. -} +ingest :: Maybe LockedDown -> Maybe Key -> Annex (Maybe Key, Maybe InodeCache) +ingest ld mk = ingest' Nothing ld mk (Restage True) + +ingest' :: Maybe Backend -> Maybe LockedDown -> Maybe Key -> Restage -> Annex (Maybe Key, Maybe InodeCache) +ingest' _ Nothing _ _ = return (Nothing, Nothing) +ingest' preferredbackend (Just (LockedDown cfg source)) mk restage = withTSDelta $ \delta -> do + k <- case mk of + Nothing -> do + backend <- maybe (chooseBackend $ keyFilename source) (return . Just) preferredbackend + fmap fst <$> genKey source backend + Just k -> return (Just k) + let src = contentLocation source + ms <- liftIO $ catchMaybeIO $ getFileStatus src + mcache <- maybe (pure Nothing) (liftIO . toInodeCache delta src) ms + case (mcache, inodeCache source) of + (_, Nothing) -> go k mcache ms + (Just newc, Just c) | compareStrong c newc -> go k mcache ms + _ -> failure "changed while it was being added" + where + go (Just key) mcache (Just s) + | lockingFile cfg = golocked key mcache s + | otherwise = ifM isDirect + ( godirect key mcache s + , gounlocked key mcache s + ) + go _ _ _ = failure "failed to generate a key" + + golocked key mcache s = + tryNonAsync (moveAnnex key $ contentLocation source) >>= \case + Right True -> do + populateAssociatedFiles key source restage + success key mcache s + Right False -> giveup "failed to add content to annex" + Left e -> restoreFile (keyFilename source) key e + + gounlocked key (Just cache) s = do + -- Remove temp directory hard link first because + -- linkToAnnex falls back to copying if a file + -- already has a hard link. + cleanCruft source + cleanOldKeys (keyFilename source) key + linkToAnnex key (keyFilename source) (Just cache) >>= \case + LinkAnnexFailed -> failure "failed to link to annex" + _ -> do + finishIngestUnlocked' key source restage + success key (Just cache) s + gounlocked _ _ _ = failure "failed statting file" + + godirect key (Just cache) s = do + addInodeCache key cache + finishIngestDirect key source + success key (Just cache) s + godirect _ _ _ = failure "failed statting file" + + success k mcache s = do + genMetaData k (keyFilename source) s + return (Just k, mcache) + + failure msg = do + warning $ keyFilename source ++ " " ++ msg + cleanCruft source + return (Nothing, Nothing) + +finishIngestDirect :: Key -> KeySource -> Annex () +finishIngestDirect key source = do + void $ addAssociatedFile key $ keyFilename source + cleanCruft source + + {- Copy to any other locations using the same key. -} + otherfs <- filter (/= keyFilename source) <$> associatedFiles key + forM_ otherfs $ + addContentWhenNotPresent key (keyFilename source) + +finishIngestUnlocked :: Key -> KeySource -> Annex () +finishIngestUnlocked key source = do + cleanCruft source + finishIngestUnlocked' key source (Restage True) + +finishIngestUnlocked' :: Key -> KeySource -> Restage -> Annex () +finishIngestUnlocked' key source restage = do + Database.Keys.addAssociatedFile key =<< inRepo (toTopFilePath (keyFilename source)) + populateAssociatedFiles key source restage + +{- Copy to any other locations using the same key. -} +populateAssociatedFiles :: Key -> KeySource -> Restage -> Annex () +populateAssociatedFiles key source restage = do + obj <- calcRepo (gitAnnexLocation key) + g <- Annex.gitRepo + ingestedf <- flip fromTopFilePath g + <$> inRepo (toTopFilePath (keyFilename source)) + afs <- map (`fromTopFilePath` g) <$> Database.Keys.getAssociatedFiles key + forM_ (filter (/= ingestedf) afs) $ + populatePointerFile restage key obj + +cleanCruft :: KeySource -> Annex () +cleanCruft source = when (contentLocation source /= keyFilename source) $ + liftIO $ nukeFile $ contentLocation source + +-- If a worktree file was was hard linked to an annex object before, +-- modifying the file would have caused the object to have the wrong +-- content. Clean up from that. +cleanOldKeys :: FilePath -> Key -> Annex () +cleanOldKeys file newkey = do + g <- Annex.gitRepo + ingestedf <- flip fromTopFilePath g <$> inRepo (toTopFilePath file) + topf <- inRepo (toTopFilePath file) + oldkeys <- filter (/= newkey) + <$> Database.Keys.getAssociatedKey topf + forM_ oldkeys $ \key -> + unlessM (isUnmodified key =<< calcRepo (gitAnnexLocation key)) $ do + caches <- Database.Keys.getInodeCaches key + unlinkAnnex key + fs <- filter (/= ingestedf) + . map (`fromTopFilePath` g) + <$> Database.Keys.getAssociatedFiles key + filterM (`sameInodeCache` caches) fs >>= \case + -- If linkToAnnex fails, the associated + -- file with the content is still present, + -- so no need for any recovery. + (f:_) -> do + ic <- withTSDelta (liftIO . genInodeCache f) + void $ linkToAnnex key f ic + _ -> logStatus key InfoMissing + +{- On error, put the file back so it doesn't seem to have vanished. + - This can be called before or after the symlink is in place. -} +restoreFile :: FilePath -> Key -> SomeException -> Annex a +restoreFile file key e = do + whenM (inAnnex key) $ do + liftIO $ nukeFile file + -- The key could be used by other files too, so leave the + -- content in the annex, and make a copy back to the file. + obj <- calcRepo $ gitAnnexLocation key + unlessM (liftIO $ copyFileExternal CopyTimeStamps obj file) $ + warning $ "Unable to restore content of " ++ file ++ "; it should be located in " ++ obj + thawContent file + throwM e + +{- Creates the symlink to the annexed content, returns the link target. -} +makeLink :: FilePath -> Key -> Maybe InodeCache -> Annex String +makeLink file key mcache = flip catchNonAsync (restoreFile file key) $ do + l <- calcRepo $ gitAnnexLink file key + replaceFile file $ makeAnnexLink l + + -- touch symlink to have same time as the original file, + -- as provided in the InodeCache + case mcache of + Just c -> liftIO $ touch file (TimeSpec $ inodeCacheToMtime c) False + Nothing -> noop + + return l + +{- Creates the symlink to the annexed content, and stages it in git. + - + - As long as the filesystem supports symlinks, we use + - git add, rather than directly staging the symlink to git. + - Using git add is best because it allows the queuing to work + - and is faster (staging the symlink runs hash-object commands each time). + - Also, using git add allows it to skip gitignored files, unless forced + - to include them. + -} +addLink :: FilePath -> Key -> Maybe InodeCache -> Annex () +addLink file key mcache = ifM (coreSymlinks <$> Annex.getGitConfig) + ( do + _ <- makeLink file key mcache + ps <- forceParams + Annex.Queue.addCommand "add" (ps++[Param "--"]) [file] + , do + l <- makeLink file key mcache + addAnnexLink l file + ) + +{- Parameters to pass to git add, forcing addition of ignored files. -} +forceParams :: Annex [CommandParam] +forceParams = ifM (Annex.getState Annex.force) + ( return [Param "-f"] + , return [] + ) + +{- Whether a file should be added unlocked or not. Default is to not, + - unless symlinks are not supported. annex.addunlocked can override that. + - Also, when in an adjusted unlocked branch, always add files unlocked. + -} +addUnlocked :: Annex Bool +addUnlocked = isDirect <||> + (versionSupportsUnlockedPointers <&&> + ((not . coreSymlinks <$> Annex.getGitConfig) <||> + (annexAddUnlocked <$> Annex.getGitConfig) <||> + (maybe False (\b -> getAdjustment b == Just UnlockAdjustment) <$> cachedCurrentBranch) + ) + ) + +cachedCurrentBranch :: Annex (Maybe Git.Branch) +cachedCurrentBranch = maybe cache (return . Just) + =<< Annex.getState Annex.cachedcurrentbranch + where + cache :: Annex (Maybe Git.Branch) + cache = inRepo Git.Branch.currentUnsafe >>= \case + Nothing -> return Nothing + Just b -> do + Annex.changeState $ \s -> + s { Annex.cachedcurrentbranch = Just b } + return (Just b) + +{- Adds a file to the work tree for the key, and stages it in the index. + - The content of the key may be provided in a temp file, which will be + - moved into place. + - + - When the content of the key is not accepted into the annex, returns False. + -} +addAnnexedFile :: FilePath -> Key -> Maybe FilePath -> Annex Bool +addAnnexedFile file key mtmp = ifM (addUnlocked <&&> not <$> isDirect) + ( do + mode <- maybe + (pure Nothing) + (\tmp -> liftIO $ catchMaybeIO $ fileMode <$> getFileStatus tmp) + mtmp + stagePointerFile file mode =<< hashPointerFile key + Database.Keys.addAssociatedFile key =<< inRepo (toTopFilePath file) + case mtmp of + Just tmp -> ifM (moveAnnex key tmp) + ( linkunlocked mode >> return True + , writepointer mode >> return False + ) + Nothing -> ifM (inAnnex key) + ( linkunlocked mode >> return True + , writepointer mode >> return True + ) + , do + addLink file key Nothing + whenM isDirect $ do + void $ addAssociatedFile key file + case mtmp of + Just tmp -> do + {- For moveAnnex to work in direct mode, the + - symlink must already exist, so flush the queue. -} + whenM isDirect $ + Annex.Queue.flush + moveAnnex key tmp + Nothing -> return True + ) + where + linkunlocked mode = linkFromAnnex key file mode >>= \case + LinkAnnexFailed -> liftIO $ + writePointerFile file key mode + _ -> return () + writepointer mode = liftIO $ writePointerFile file key mode diff --git a/Annex/Init.hs b/Annex/Init.hs new file mode 100644 index 0000000000..802524c82c --- /dev/null +++ b/Annex/Init.hs @@ -0,0 +1,296 @@ +{- git-annex repository initialization + - + - Copyright 2011-2017 Joey Hess + - + - Licensed under the GNU GPL version 3 or higher. + -} + +{-# LANGUAGE CPP #-} + +module Annex.Init ( + AutoInit(..), + ensureInitialized, + isInitialized, + initialize, + initialize', + uninitialize, + probeCrippledFileSystem, + probeCrippledFileSystem', +) where + +import Annex.Common +import qualified Annex +import qualified Git +import qualified Git.LsFiles +import qualified Git.Config +import qualified Git.Objects +import qualified Annex.Branch +import Logs.UUID +import Logs.Trust.Basic +import Logs.Config +import Types.TrustLevel +import Annex.Version +import Annex.Difference +import Annex.UUID +import Annex.Link +import Annex.WorkTree +import Config +import Config.Smudge +import Annex.Direct +import Annex.AdjustedBranch +import Annex.Environment +import Annex.Hook +import Annex.InodeSentinal +import Upgrade +import Annex.Perms +import Utility.UserInfo +#ifndef mingw32_HOST_OS +import Utility.FileMode +import System.Posix.User +import qualified Utility.LockFile.Posix as Posix +#endif + +newtype AutoInit = AutoInit Bool + +checkCanInitialize :: AutoInit -> Annex a -> Annex a +checkCanInitialize (AutoInit True) a = a +checkCanInitialize (AutoInit False) a = fromRepo Git.repoWorkTree >>= \case + Nothing -> a + Just wt -> liftIO (catchMaybeIO (readFile (wt ".noannex"))) >>= \case + Nothing -> a + Just noannexmsg -> ifM (Annex.getState Annex.force) + ( a + , do + warning "Initialization prevented by .noannex file (use --force to override)" + unless (null noannexmsg) $ + warning noannexmsg + giveup "Not initialized." + ) + +genDescription :: Maybe String -> Annex String +genDescription (Just d) = return d +genDescription Nothing = do + reldir <- liftIO . relHome =<< liftIO . absPath =<< fromRepo Git.repoPath + hostname <- fromMaybe "" <$> liftIO getHostname + let at = if null hostname then "" else "@" + v <- liftIO myUserName + return $ concat $ case v of + Right username -> [username, at, hostname, ":", reldir] + Left _ -> [hostname, ":", reldir] + +initialize :: AutoInit -> Maybe String -> Maybe Version -> Annex () +initialize ai mdescription mversion = checkCanInitialize ai $ do + {- Has to come before any commits are made as the shared + - clone heuristic expects no local objects. -} + sharedclone <- checkSharedClone + + {- This will make the first commit to git, so ensure git is set up + - properly to allow commits when running it. -} + ensureCommit $ Annex.Branch.create + + prepUUID + initialize' (AutoInit True) mversion + + initSharedClone sharedclone + + u <- getUUID + describeUUID u =<< genDescription mdescription + +-- Everything except for uuid setup, shared clone setup, and initial +-- description. +initialize' :: AutoInit -> Maybe Version -> Annex () +initialize' ai mversion = checkCanInitialize ai $ do + checkLockSupport + checkFifoSupport + checkCrippledFileSystem + unlessM isBareRepo $ do + hookWrite preCommitHook + hookWrite postReceiveHook + setDifferences + unlessM (isJust <$> getVersion) $ + setVersion (fromMaybe defaultVersion mversion) + whenM versionSupportsUnlockedPointers $ do + configureSmudgeFilter + scanUnlockedFiles + checkAdjustedClone >>= \case + NeedUpgradeForAdjustedClone -> + void $ upgrade True versionForAdjustedClone + InAdjustedClone -> return () + NotInAdjustedClone -> + ifM (crippledFileSystem <&&> (not <$> isBareRepo)) + ( ifM versionSupportsUnlockedPointers + ( adjustToCrippledFileSystem + , do + enableDirectMode + setDirect True + ) + -- Handle case where this repo was cloned from a + -- direct mode repo + , unlessM isBareRepo + switchHEADBack + ) + propigateSecureHashesOnly + createInodeSentinalFile False + +uninitialize :: Annex () +uninitialize = do + hookUnWrite preCommitHook + hookUnWrite postReceiveHook + removeRepoUUID + removeVersion + +{- Will automatically initialize if there is already a git-annex + - branch from somewhere. Otherwise, require a manual init + - to avoid git-annex accidentally being run in git + - repos that did not intend to use it. + - + - Checks repository version and handles upgrades too. + -} +ensureInitialized :: Annex () +ensureInitialized = getVersion >>= maybe needsinit checkUpgrade + where + needsinit = ifM Annex.Branch.hasSibling + ( initialize (AutoInit True) Nothing Nothing + , giveup "First run: git-annex init" + ) + +{- Checks if a repository is initialized. Does not check version for ugrade. -} +isInitialized :: Annex Bool +isInitialized = maybe Annex.Branch.hasSibling (const $ return True) =<< getVersion + +{- A crippled filesystem is one that does not allow making symlinks, + - or removing write access from files. -} +probeCrippledFileSystem :: Annex Bool +probeCrippledFileSystem = do + tmp <- fromRepo gitAnnexTmpMiscDir + createAnnexDirectory tmp + (r, warnings) <- liftIO $ probeCrippledFileSystem' tmp + mapM_ warning warnings + return r + +probeCrippledFileSystem' :: FilePath -> IO (Bool, [String]) +#ifdef mingw32_HOST_OS +probeCrippledFileSystem' _ = return (True, []) +#else +probeCrippledFileSystem' tmp = do + let f = tmp "gaprobe" + writeFile f "" + r <- probe f + void $ tryIO $ allowWrite f + removeFile f + return r + where + probe f = catchDefaultIO (True, []) $ do + let f2 = f ++ "2" + nukeFile f2 + createSymbolicLink f f2 + nukeFile f2 + preventWrite f + -- Should be unable to write to the file, unless + -- running as root, but some crippled + -- filesystems ignore write bit removals. + ifM ((== 0) <$> getRealUserID) + ( return (False, []) + , do + r <- catchBoolIO $ do + writeFile f "2" + return True + if r + then return (True, ["Filesystem allows writing to files whose write bit is not set."]) + else return (False, []) + ) +#endif + +checkCrippledFileSystem :: Annex () +checkCrippledFileSystem = whenM probeCrippledFileSystem $ do + warning "Detected a crippled filesystem." + setCrippledFileSystem True + + {- Normally git disables core.symlinks itself when the + - filesystem does not support them. But, even if symlinks are + - supported, we don't use them by default in a crippled + - filesystem. -} + whenM (coreSymlinks <$> Annex.getGitConfig) $ do + warning "Disabling core.symlinks." + setConfig (ConfigKey "core.symlinks") + (Git.Config.boolConfig False) + +probeLockSupport :: Annex Bool +probeLockSupport = do +#ifdef mingw32_HOST_OS + return True +#else + tmp <- fromRepo gitAnnexTmpMiscDir + let f = tmp "lockprobe" + createAnnexDirectory tmp + mode <- annexFileMode + liftIO $ do + nukeFile f + ok <- catchBoolIO $ do + Posix.dropLock =<< Posix.lockExclusive (Just mode) f + return True + nukeFile f + return ok +#endif + +probeFifoSupport :: Annex Bool +probeFifoSupport = do +#ifdef mingw32_HOST_OS + return False +#else + tmp <- fromRepo gitAnnexTmpMiscDir + let f = tmp "gaprobe" + let f2 = tmp "gaprobe2" + createAnnexDirectory tmp + liftIO $ do + nukeFile f + nukeFile f2 + ms <- tryIO $ do + createNamedPipe f ownerReadMode + createLink f f2 + getFileStatus f + nukeFile f + nukeFile f2 + return $ either (const False) isNamedPipe ms +#endif + +checkLockSupport :: Annex () +checkLockSupport = unlessM probeLockSupport $ do + warning "Detected a filesystem without POSIX fcntl lock support." + warning "Enabling annex.pidlock." + setConfig (annexConfig "pidlock") (Git.Config.boolConfig True) + +checkFifoSupport :: Annex () +checkFifoSupport = unlessM probeFifoSupport $ do + warning "Detected a filesystem without fifo support." + warning "Disabling ssh connection caching." + setConfig (annexConfig "sshcaching") (Git.Config.boolConfig False) + +enableDirectMode :: Annex () +enableDirectMode = unlessM isDirect $ do + warning "Enabling direct mode." + top <- fromRepo Git.repoPath + (l, clean) <- inRepo $ Git.LsFiles.inRepo [top] + forM_ l $ \f -> + maybe noop (`toDirect` f) =<< isAnnexLink f + void $ liftIO clean + +checkSharedClone :: Annex Bool +checkSharedClone = inRepo Git.Objects.isSharedClone + +initSharedClone :: Bool -> Annex () +initSharedClone False = return () +initSharedClone True = do + showLongNote "Repository was cloned with --shared; setting annex.hardlink=true and making repository untrusted." + u <- getUUID + trustSet u UnTrusted + setConfig (annexConfig "hardlink") (Git.Config.boolConfig True) + +{- Propigate annex.securehashesonly from then global config to local + - config. This makes a clone inherit a parent's setting, but once + - a repository has a local setting, changes to the global config won't + - affect it. -} +propigateSecureHashesOnly :: Annex () +propigateSecureHashesOnly = + maybe noop (setConfig (ConfigKey "annex.securehashesonly")) + =<< getGlobalConfig "annex.securehashesonly" diff --git a/Annex/InodeSentinal.hs b/Annex/InodeSentinal.hs new file mode 100644 index 0000000000..1e763d3170 --- /dev/null +++ b/Annex/InodeSentinal.hs @@ -0,0 +1,96 @@ +{- git-annex inode sentinal file + - + - Copyright 2012-2015 Joey Hess + - + - Licensed under the GNU GPL version 3 or higher. + -} + +{-# LANGUAGE CPP #-} + +module Annex.InodeSentinal where + +import Annex.Common +import qualified Annex +import Utility.InodeCache +import Annex.Perms + +{- If the sendinal shows the inodes have changed, only the size and mtime + - are compared. -} +compareInodeCaches :: InodeCache -> InodeCache -> Annex Bool +compareInodeCaches x y + | compareStrong x y = return True + | otherwise = ifM inodesChanged + ( return $ compareWeak x y + , return False + ) + +compareInodeCachesWith :: Annex InodeComparisonType +compareInodeCachesWith = ifM inodesChanged ( return Weakly, return Strongly ) + +{- Checks if one of the provided old InodeCache matches the current + - version of a file. -} +sameInodeCache :: FilePath -> [InodeCache] -> Annex Bool +sameInodeCache _ [] = return False +sameInodeCache file old = go =<< withTSDelta (liftIO . genInodeCache file) + where + go Nothing = return False + go (Just curr) = elemInodeCaches curr old + +elemInodeCaches :: InodeCache -> [InodeCache] -> Annex Bool +elemInodeCaches _ [] = return False +elemInodeCaches c (l:ls) = ifM (compareInodeCaches c l) + ( return True + , elemInodeCaches c ls + ) + +{- Some filesystems get new inodes each time they are mounted. + - In order to work on such a filesystem, a sentinal file is used to detect + - when the inodes have changed. + - + - If the sentinal file does not exist, we have to assume that the + - inodes have changed. + -} +inodesChanged :: Annex Bool +inodesChanged = sentinalInodesChanged <$> sentinalStatus + +withTSDelta :: (TSDelta -> Annex a) -> Annex a +withTSDelta a = a =<< getTSDelta + +getTSDelta :: Annex TSDelta +#ifdef mingw32_HOST_OS +getTSDelta = sentinalTSDelta <$> sentinalStatus +#else +getTSDelta = pure noTSDelta -- optimisation +#endif + +sentinalStatus :: Annex SentinalStatus +sentinalStatus = maybe check return =<< Annex.getState Annex.sentinalstatus + where + check = do + sc <- liftIO . checkSentinalFile =<< annexSentinalFile + Annex.changeState $ \s -> s { Annex.sentinalstatus = Just sc } + return sc + +{- The sentinal file is only created when first initializing a repository. + - If there are any annexed objects in the repository already, creating + - the file would invalidate their inode caches. -} +createInodeSentinalFile :: Bool -> Annex () +createInodeSentinalFile evenwithobjects = + unlessM (alreadyexists <||> hasobjects) $ do + s <- annexSentinalFile + createAnnexDirectory (parentDir (sentinalFile s)) + liftIO $ writeSentinalFile s + where + alreadyexists = liftIO. sentinalFileExists =<< annexSentinalFile + hasobjects + | evenwithobjects = pure False + | otherwise = liftIO . doesDirectoryExist =<< fromRepo gitAnnexObjectDir + +annexSentinalFile :: Annex SentinalFile +annexSentinalFile = do + sentinalfile <- fromRepo gitAnnexInodeSentinal + sentinalcachefile <- fromRepo gitAnnexInodeSentinalCache + return SentinalFile + { sentinalFile = sentinalfile + , sentinalCacheFile = sentinalcachefile + } diff --git a/Annex/Journal.hs b/Annex/Journal.hs new file mode 100644 index 0000000000..d969a59fc5 --- /dev/null +++ b/Annex/Journal.hs @@ -0,0 +1,109 @@ +{- management of the git-annex journal + - + - The journal is used to queue up changes before they are committed to the + - git-annex branch. Among other things, it ensures that if git-annex is + - interrupted, its recorded data is not lost. + - + - Copyright 2011-2013 Joey Hess + - + - Licensed under the GNU GPL version 3 or higher. + -} + +{-# LANGUAGE CPP #-} + +module Annex.Journal where + +import Annex.Common +import qualified Git +import Annex.Perms +import Annex.LockFile +import Utility.Directory.Stream + +{- Records content for a file in the branch to the journal. + - + - Using the journal, rather than immediatly staging content to the index + - avoids git needing to rewrite the index after every change. + - + - The file in the journal is updated atomically, which allows + - getJournalFileStale to always return a consistent journal file + - content, although possibly not the most current one. + -} +setJournalFile :: JournalLocked -> FilePath -> String -> Annex () +setJournalFile _jl file content = do + tmp <- fromRepo gitAnnexTmpMiscDir + createAnnexDirectory =<< fromRepo gitAnnexJournalDir + createAnnexDirectory tmp + -- journal file is written atomically + jfile <- fromRepo $ journalFile file + let tmpfile = tmp takeFileName jfile + liftIO $ do + withFile tmpfile WriteMode $ \h -> do +#ifdef mingw32_HOST_OS + hSetNewlineMode h noNewlineTranslation +#endif + hPutStr h content + moveFile tmpfile jfile + +{- Gets any journalled content for a file in the branch. -} +getJournalFile :: JournalLocked -> FilePath -> Annex (Maybe String) +getJournalFile _jl = getJournalFileStale + +{- Without locking, this is not guaranteed to be the most recent + - version of the file in the journal, so should not be used as a basis for + - changes. -} +getJournalFileStale :: FilePath -> Annex (Maybe String) +getJournalFileStale file = inRepo $ \g -> catchMaybeIO $ + readFileStrict $ journalFile file g + +{- List of existing journal files, but without locking, may miss new ones + - just being added, or may have false positives if the journal is staged + - as it is run. -} +getJournalledFilesStale :: Annex [FilePath] +getJournalledFilesStale = do + g <- gitRepo + fs <- liftIO $ catchDefaultIO [] $ + getDirectoryContents $ gitAnnexJournalDir g + return $ filter (`notElem` [".", ".."]) $ map fileJournal fs + +withJournalHandle :: (DirectoryHandle -> IO a) -> Annex a +withJournalHandle a = do + d <- fromRepo gitAnnexJournalDir + bracketIO (openDirectory d) closeDirectory (liftIO . a) + +{- Checks if there are changes in the journal. -} +journalDirty :: Annex Bool +journalDirty = do + d <- fromRepo gitAnnexJournalDir + liftIO $ + (not <$> isDirectoryEmpty d) + `catchIO` (const $ doesDirectoryExist d) + +{- Produces a filename to use in the journal for a file on the branch. + - + - The journal typically won't have a lot of files in it, so the hashing + - used in the branch is not necessary, and all the files are put directly + - in the journal directory. + -} +journalFile :: FilePath -> Git.Repo -> FilePath +journalFile file repo = gitAnnexJournalDir repo concatMap mangle file + where + mangle c + | c == pathSeparator = "_" + | c == '_' = "__" + | otherwise = [c] + +{- Converts a journal file (relative to the journal dir) back to the + - filename on the branch. -} +fileJournal :: FilePath -> FilePath +fileJournal = replace [pathSeparator, pathSeparator] "_" . + replace "_" [pathSeparator] + +{- Sentinal value, only produced by lockJournal; required + - as a parameter by things that need to ensure the journal is + - locked. -} +data JournalLocked = ProduceJournalLocked + +{- Runs an action that modifies the journal, using locking to avoid + - contention with other git-annex processes. -} +lockJournal :: (JournalLocked -> Annex a) -> Annex a +lockJournal a = withExclusiveLock gitAnnexJournalLock $ a ProduceJournalLocked diff --git a/Annex/Link.hs b/Annex/Link.hs new file mode 100644 index 0000000000..ba12060a15 --- /dev/null +++ b/Annex/Link.hs @@ -0,0 +1,283 @@ +{- git-annex links to content + - + - On file systems that support them, symlinks are used. + - + - On other filesystems, git instead stores the symlink target in a regular + - file. + - + - Pointer files are used instead of symlinks for unlocked files. + - + - Copyright 2013-2018 Joey Hess + - + - Licensed under the GNU GPL version 3 or higher. + -} + +{-# LANGUAGE CPP, BangPatterns #-} + +module Annex.Link where + +import Annex.Common +import qualified Annex +import qualified Annex.Queue +import qualified Git.Queue +import qualified Git.UpdateIndex +import qualified Git.Index +import qualified Git.LockFile +import qualified Git.Env +import qualified Git +import Git.Types +import Git.FilePath +import Annex.HashObject +import Annex.InodeSentinal +import Utility.FileMode +import Utility.FileSystemEncoding +import Utility.InodeCache +import Utility.Tmp.Dir +import Utility.CopyFile + +import qualified Data.ByteString.Lazy as L + +type LinkTarget = String + +{- Checks if a file is a link to a key. -} +isAnnexLink :: FilePath -> Annex (Maybe Key) +isAnnexLink file = maybe Nothing (fileKey . takeFileName) <$> getAnnexLinkTarget file + +{- Gets the link target of a symlink. + - + - On a filesystem that does not support symlinks, fall back to getting the + - link target by looking inside the file. + - + - Returns Nothing if the file is not a symlink, or not a link to annex + - content. + -} +getAnnexLinkTarget :: FilePath -> Annex (Maybe LinkTarget) +getAnnexLinkTarget f = getAnnexLinkTarget' f + =<< (coreSymlinks <$> Annex.getGitConfig) + +{- Pass False to force looking inside file. -} +getAnnexLinkTarget' :: FilePath -> Bool -> Annex (Maybe LinkTarget) +getAnnexLinkTarget' file coresymlinks = if coresymlinks + then check readSymbolicLink $ + return Nothing + else check readSymbolicLink $ + check probefilecontent $ + return Nothing + where + check getlinktarget fallback = + liftIO (catchMaybeIO $ getlinktarget file) >>= \case + Just l + | isLinkToAnnex (fromInternalGitPath l) -> return (Just l) + | otherwise -> return Nothing + Nothing -> fallback + + probefilecontent f = withFile f ReadMode $ \h -> do + -- The first 8k is more than enough to read; link + -- files are small. + s <- take 8192 <$> hGetContents h + -- If we got the full 8k, the file is too large + if length s == 8192 + then return "" + else + -- If there are any NUL or newline + -- characters, or whitespace, we + -- certianly don't have a link to a + -- git-annex key. + return $ if any (`elem` s) "\0\n\r \t" + then "" + else s + +makeAnnexLink :: LinkTarget -> FilePath -> Annex () +makeAnnexLink = makeGitLink + +{- Creates a link on disk. + - + - On a filesystem that does not support symlinks, writes the link target + - to a file. Note that git will only treat the file as a symlink if + - it's staged as such, so use addAnnexLink when adding a new file or + - modified link to git. + -} +makeGitLink :: LinkTarget -> FilePath -> Annex () +makeGitLink linktarget file = ifM (coreSymlinks <$> Annex.getGitConfig) + ( liftIO $ do + void $ tryIO $ removeFile file + createSymbolicLink linktarget file + , liftIO $ writeFile file linktarget + ) + +{- Creates a link on disk, and additionally stages it in git. -} +addAnnexLink :: LinkTarget -> FilePath -> Annex () +addAnnexLink linktarget file = do + makeAnnexLink linktarget file + stageSymlink file =<< hashSymlink linktarget + +{- Injects a symlink target into git, returning its Sha. -} +hashSymlink :: LinkTarget -> Annex Sha +hashSymlink linktarget = hashBlob (toInternalGitPath linktarget) + +{- Stages a symlink to an annexed object, using a Sha of its target. -} +stageSymlink :: FilePath -> Sha -> Annex () +stageSymlink file sha = + Annex.Queue.addUpdateIndex =<< + inRepo (Git.UpdateIndex.stageSymlink file sha) + +{- Injects a pointer file content into git, returning its Sha. -} +hashPointerFile :: Key -> Annex Sha +hashPointerFile key = hashBlob (formatPointer key) + +{- Stages a pointer file, using a Sha of its content -} +stagePointerFile :: FilePath -> Maybe FileMode -> Sha -> Annex () +stagePointerFile file mode sha = + Annex.Queue.addUpdateIndex =<< + inRepo (Git.UpdateIndex.stageFile sha treeitemtype file) + where + treeitemtype + | maybe False isExecutable mode = TreeExecutable + | otherwise = TreeFile + +writePointerFile :: FilePath -> Key -> Maybe FileMode -> IO () +writePointerFile file k mode = do + writeFile file (formatPointer k) + maybe noop (setFileMode file) mode + +newtype Restage = Restage Bool + +{- Restage pointer file. This is used after updating a worktree file + - when content is added/removed, to prevent git status from showing + - it as modified. + - + - Asks git to refresh its index information for the file. + - That in turn runs the clean filter on the file; when the clean + - filter produces the same pointer that was in the index before, git + - realizes that the file has not actually been modified. + - + - Note that, if the pointer file is staged for deletion, or has different + - content than the current worktree content staged, this won't change + - that. So it's safe to call at any time and any situation. + - + - If the index is known to be locked (eg, git add has run git-annex), + - that would fail. Restage False will prevent the index being updated. + - Will display a message to help the user understand why + - the file will appear to be modified. + - + - This uses the git queue, so the update is not performed immediately, + - and this can be run multiple times cheaply. + - + - The InodeCache is for the worktree file. It is used to detect when + - the worktree file is changed by something else before git update-index + - gets to look at it. + -} +restagePointerFile :: Restage -> FilePath -> InodeCache -> Annex () +restagePointerFile (Restage False) f _ = + toplevelWarning True $ unableToRestage (Just f) +restagePointerFile (Restage True) f orig = withTSDelta $ \tsd -> do + -- update-index is documented as picky about "./file" and it + -- fails on "../../repo/path/file" when cwd is not in the repo + -- being acted on. Avoid these problems with an absolute path. + absf <- liftIO $ absPath f + Annex.Queue.addInternalAction runner [(absf, isunmodified tsd)] + where + isunmodified tsd = genInodeCache f tsd >>= return . \case + Nothing -> False + Just new -> compareStrong orig new + + -- Other changes to the files may have been staged before this + -- gets a chance to run. To avoid a race with any staging of + -- changes, first lock the index file. Then run git update-index + -- on all still-unmodified files, using a copy of the index file, + -- to bypass the lock. Then replace the old index file with the new + -- updated index file. + runner = Git.Queue.InternalActionRunner "restagePointerFile" $ \r l -> do + realindex <- Git.Index.currentIndexFile r + let lock = Git.Index.indexFileLock realindex + lockindex = catchMaybeIO $ Git.LockFile.openLock' lock + unlockindex = maybe noop Git.LockFile.closeLock + showwarning = warningIO $ unableToRestage Nothing + go Nothing = showwarning + go (Just _) = withTmpDirIn (Git.localGitDir r) "annexindex" $ \tmpdir -> do + let tmpindex = tmpdir "index" + let updatetmpindex = do + r' <- Git.Env.addGitEnv r Git.Index.indexEnv + =<< Git.Index.indexEnvVal tmpindex + Git.UpdateIndex.refreshIndex r' $ \feed -> + forM_ l $ \(f', checkunmodified) -> + whenM checkunmodified $ + feed f' + let replaceindex = catchBoolIO $ do + moveFile tmpindex realindex + return True + ok <- createLinkOrCopy realindex tmpindex + <&&> updatetmpindex + <&&> replaceindex + unless ok showwarning + bracket lockindex unlockindex go + +unableToRestage :: Maybe FilePath -> String +unableToRestage mf = unwords + [ "git status will show " ++ fromMaybe "some files" mf + , "to be modified, since content availability has changed" + , "and git-annex was unable to update the index." + , "This is only a cosmetic problem affecting git status; git add," + , "git commit, etc won't be affected." + , "To fix the git status display, you can run:" + , "git update-index -q --refresh " ++ fromMaybe "" mf + ] + +{- Parses a symlink target or a pointer file to a Key. + - Only looks at the first line, as pointer files can have subsequent + - lines. -} +parseLinkOrPointer :: L.ByteString -> Maybe Key +parseLinkOrPointer = parseLinkOrPointer' + . decodeBS . L.take (fromIntegral maxPointerSz) + where + +{- Want to avoid buffering really big files in git into + - memory when reading files that may be pointers. + - + - 8192 bytes is plenty for a pointer to a key. + - Pad some more to allow for any pointer files that might have + - lines after the key explaining what the file is used for. -} +maxPointerSz :: Integer +maxPointerSz = 81920 + +parseLinkOrPointer' :: String -> Maybe Key +parseLinkOrPointer' = go . fromInternalGitPath . takeWhile (not . lineend) + where + go l + | isLinkToAnnex l = fileKey $ takeFileName l + | otherwise = Nothing + lineend '\n' = True + lineend '\r' = True + lineend _ = False + +formatPointer :: Key -> String +formatPointer k = + toInternalGitPath (pathSeparator:objectDir keyFile k) ++ "\n" + +{- Checks if a worktree file is a pointer to a key. + - + - Unlocked files whose content is present are not detected by this. -} +isPointerFile :: FilePath -> IO (Maybe Key) +isPointerFile f = catchDefaultIO Nothing $ bracket open close $ \h -> do + b <- take (fromIntegral maxPointerSz) <$> hGetContents h + -- strict so it reads before the file handle is closed + let !mk = parseLinkOrPointer' b + return mk + where + open = openBinaryFile f ReadMode + close = hClose + +{- Checks a symlink target or pointer file first line to see if it + - appears to point to annexed content. + - + - We only look for paths inside the .git directory, and not at the .git + - directory itself, because GIT_DIR may cause a directory name other + - than .git to be used. + -} +isLinkToAnnex :: FilePath -> Bool +isLinkToAnnex s = (pathSeparator:objectDir) `isInfixOf` s +#ifdef mingw32_HOST_OS + -- '/' is still used inside pointer files on Windows, not the native + -- '\' + || ('/':objectDir) `isInfixOf` s +#endif diff --git a/Annex/Locations.hs b/Annex/Locations.hs new file mode 100644 index 0000000000..7e52dc6115 --- /dev/null +++ b/Annex/Locations.hs @@ -0,0 +1,549 @@ +{- git-annex file locations + - + - Copyright 2010-2017 Joey Hess + - + - Licensed under the GNU GPL version 3 or higher. + -} + +module Annex.Locations ( + keyFile, + fileKey, + keyPaths, + keyPath, + annexDir, + objectDir, + gitAnnexLocation, + gitAnnexLocationDepth, + gitAnnexLink, + gitAnnexLinkCanonical, + gitAnnexContentLock, + gitAnnexMapping, + gitAnnexInodeCache, + gitAnnexInodeSentinal, + gitAnnexInodeSentinalCache, + annexLocations, + gitAnnexDir, + gitAnnexObjectDir, + gitAnnexTmpMiscDir, + gitAnnexTmpObjectDir, + gitAnnexTmpObjectLocation, + gitAnnexTmpWorkDir, + gitAnnexBadDir, + gitAnnexBadLocation, + gitAnnexUnusedLog, + gitAnnexKeysDb, + gitAnnexKeysDbLock, + gitAnnexKeysDbIndexCache, + gitAnnexFsckState, + gitAnnexFsckDbDir, + gitAnnexFsckDbLock, + gitAnnexFsckResultsLog, + gitAnnexExportDbDir, + gitAnnexExportLock, + gitAnnexScheduleState, + gitAnnexTransferDir, + gitAnnexCredsDir, + gitAnnexWebCertificate, + gitAnnexWebPrivKey, + gitAnnexFeedStateDir, + gitAnnexFeedState, + gitAnnexMergeDir, + gitAnnexJournalDir, + gitAnnexJournalLock, + gitAnnexPreCommitLock, + gitAnnexMergeLock, + gitAnnexIndex, + gitAnnexIndexStatus, + gitAnnexViewIndex, + gitAnnexViewLog, + gitAnnexMergedRefs, + gitAnnexIgnoredRefs, + gitAnnexPidFile, + gitAnnexPidLockFile, + gitAnnexDaemonStatusFile, + gitAnnexLogFile, + gitAnnexFuzzTestLogFile, + gitAnnexHtmlShim, + gitAnnexUrlFile, + gitAnnexTmpCfgFile, + gitAnnexSshDir, + gitAnnexRemotesDir, + gitAnnexAssistantDefaultDir, + HashLevels(..), + hashDirMixed, + hashDirLower, + preSanitizeKeyName, + reSanitizeKeyName, + + prop_isomorphic_fileKey +) where + +import Data.Char +import Data.Default + +import Common +import Key +import Types.Key +import Types.UUID +import Types.GitConfig +import Types.Difference +import qualified Git +import qualified Git.Types as Git +import Git.FilePath +import Annex.DirHashes +import Annex.Fixup + +{- Conventions: + - + - Functions ending in "Dir" should always return values ending with a + - trailing path separator. Most code does not rely on that, but a few + - things do. + - + - Everything else should not end in a trailing path sepatator. + - + - Only functions (with names starting with "git") that build a path + - based on a git repository should return full path relative to the git + - repository. Everything else returns path segments. + -} + +{- The directory git annex uses for local state, relative to the .git + - directory -} +annexDir :: FilePath +annexDir = addTrailingPathSeparator "annex" + +{- The directory git annex uses for locally available object content, + - relative to the .git directory -} +objectDir :: FilePath +objectDir = addTrailingPathSeparator $ annexDir "objects" + +{- Annexed file's possible locations relative to the .git directory. + - There are two different possibilities, using different hashes. + - + - Also, some repositories have a Difference in hash directory depth. + -} +annexLocations :: GitConfig -> Key -> [FilePath] +annexLocations config key = map (annexLocation config key) dirHashes + +annexLocation :: GitConfig -> Key -> (HashLevels -> Hasher) -> FilePath +annexLocation config key hasher = objectDir keyPath key (hasher $ objectHashLevels config) + +{- Number of subdirectories from the gitAnnexObjectDir + - to the gitAnnexLocation. -} +gitAnnexLocationDepth :: GitConfig -> Int +gitAnnexLocationDepth config = hashlevels + 1 + where + HashLevels hashlevels = objectHashLevels config + +{- Annexed object's location in a repository. + - + - When there are multiple possible locations, returns the one where the + - file is actually present. + - + - When the file is not present, returns the location where the file should + - be stored. + - + - This does not take direct mode into account, so in direct mode it is not + - the actual location of the file's content. + -} +gitAnnexLocation :: Key -> Git.Repo -> GitConfig -> IO FilePath +gitAnnexLocation key r config = gitAnnexLocation' key r config (annexCrippledFileSystem config) (coreSymlinks config) doesFileExist (Git.localGitDir r) +gitAnnexLocation' :: Key -> Git.Repo -> GitConfig -> Bool -> Bool -> (FilePath -> IO Bool) -> FilePath -> IO FilePath +gitAnnexLocation' key r config crippled symlinkssupported checker gitdir + {- Bare repositories default to hashDirLower for new + - content, as it's more portable. But check all locations. -} + | Git.repoIsLocalBare r = checkall + | hasDifference ObjectHashLower (annexDifferences config) = + only hashDirLower + {- Repositories on crippled filesystems use hashDirLower + - for new content, unless symlinks are supported too. + - Then hashDirMixed is used. But, the content could be + - in either location so check both. -} + | crippled = if symlinkssupported + then check $ map inrepo $ reverse $ annexLocations config key + else checkall + {- Regular repositories only use hashDirMixed, so + - don't need to do any work to check if the file is + - present. -} + | otherwise = only hashDirMixed + where + only = return . inrepo . annexLocation config key + checkall = check $ map inrepo $ annexLocations config key + + inrepo d = gitdir d + check locs@(l:_) = fromMaybe l <$> firstM checker locs + check [] = error "internal" + +{- Calculates a symlink target to link a file to an annexed object. -} +gitAnnexLink :: FilePath -> Key -> Git.Repo -> GitConfig -> IO FilePath +gitAnnexLink file key r config = do + currdir <- getCurrentDirectory + let absfile = absNormPathUnix currdir file + let gitdir = getgitdir currdir + loc <- gitAnnexLocation' key r config False False (\_ -> return True) gitdir + toInternalGitPath <$> relPathDirToFile (parentDir absfile) loc + where + getgitdir currdir + {- This special case is for git submodules on filesystems not + - supporting symlinks; generate link target that will + - work portably. -} + | not (coreSymlinks config) && needsSubmoduleFixup r = + absNormPathUnix currdir $ Git.repoPath r ".git" + | otherwise = Git.localGitDir r + absNormPathUnix d p = toInternalGitPath $ + absPathFrom (toInternalGitPath d) (toInternalGitPath p) + +{- Calculates a symlink target as would be used in a typical git + - repository, with .git in the top of the work tree. -} +gitAnnexLinkCanonical :: FilePath -> Key -> Git.Repo -> GitConfig -> IO FilePath +gitAnnexLinkCanonical file key r config = gitAnnexLink file key r' config' + where + r' = case r of + Git.Repo { Git.location = l@Git.Local { Git.worktree = Just wt } } -> + r { Git.location = l { Git.gitdir = wt ".git" } } + _ -> r + config' = config + { annexCrippledFileSystem = False + , coreSymlinks = True + } + +{- File used to lock a key's content. -} +gitAnnexContentLock :: Key -> Git.Repo -> GitConfig -> IO FilePath +gitAnnexContentLock key r config = do + loc <- gitAnnexLocation key r config + return $ loc ++ ".lck" + +{- File that maps from a key to the file(s) in the git repository. + - Used in direct mode. -} +gitAnnexMapping :: Key -> Git.Repo -> GitConfig -> IO FilePath +gitAnnexMapping key r config = do + loc <- gitAnnexLocation key r config + return $ loc ++ ".map" + +{- File that caches information about a key's content, used to determine + - if a file has changed. + - Used in direct mode. -} +gitAnnexInodeCache :: Key -> Git.Repo -> GitConfig -> IO FilePath +gitAnnexInodeCache key r config = do + loc <- gitAnnexLocation key r config + return $ loc ++ ".cache" + +gitAnnexInodeSentinal :: Git.Repo -> FilePath +gitAnnexInodeSentinal r = gitAnnexDir r "sentinal" + +gitAnnexInodeSentinalCache :: Git.Repo -> FilePath +gitAnnexInodeSentinalCache r = gitAnnexInodeSentinal r ++ ".cache" + +{- The annex directory of a repository. -} +gitAnnexDir :: Git.Repo -> FilePath +gitAnnexDir r = addTrailingPathSeparator $ Git.localGitDir r annexDir + +{- The part of the annex directory where file contents are stored. -} +gitAnnexObjectDir :: Git.Repo -> FilePath +gitAnnexObjectDir r = addTrailingPathSeparator $ Git.localGitDir r objectDir + +{- .git/annex/misctmp/ is used for random temp files -} +gitAnnexTmpMiscDir :: Git.Repo -> FilePath +gitAnnexTmpMiscDir r = addTrailingPathSeparator $ gitAnnexDir r "misctmp" + +{- .git/annex/tmp/ is used for temp files for key's contents -} +gitAnnexTmpObjectDir :: Git.Repo -> FilePath +gitAnnexTmpObjectDir r = addTrailingPathSeparator $ gitAnnexDir r "tmp" + +{- The temp file to use for a given key's content. -} +gitAnnexTmpObjectLocation :: Key -> Git.Repo -> FilePath +gitAnnexTmpObjectLocation key r = gitAnnexTmpObjectDir r keyFile key + +{- Given a temp file such as gitAnnexTmpObjectLocation, makes a name for a + - subdirectory in the same location, that can be used as a work area + - when receiving the key's content. + - + - There are ordering requirements for creating these directories; + - use Annex.Content.withTmpWorkDir to set them up. + -} +gitAnnexTmpWorkDir :: FilePath -> FilePath +gitAnnexTmpWorkDir p = + let (dir, f) = splitFileName p + -- Using a prefix avoids name conflict with any other keys. + in dir "work." ++ f + +{- .git/annex/bad/ is used for bad files found during fsck -} +gitAnnexBadDir :: Git.Repo -> FilePath +gitAnnexBadDir r = addTrailingPathSeparator $ gitAnnexDir r "bad" + +{- The bad file to use for a given key. -} +gitAnnexBadLocation :: Key -> Git.Repo -> FilePath +gitAnnexBadLocation key r = gitAnnexBadDir r keyFile key + +{- .git/annex/foounused is used to number possibly unused keys -} +gitAnnexUnusedLog :: FilePath -> Git.Repo -> FilePath +gitAnnexUnusedLog prefix r = gitAnnexDir r (prefix ++ "unused") + +{- .git/annex/keys/ contains a database of information about keys. -} +gitAnnexKeysDb :: Git.Repo -> FilePath +gitAnnexKeysDb r = gitAnnexDir r "keys" + +{- Lock file for the keys database. -} +gitAnnexKeysDbLock :: Git.Repo -> FilePath +gitAnnexKeysDbLock r = gitAnnexKeysDb r ++ ".lck" + +{- Contains the stat of the last index file that was + - reconciled with rhe keys database. -} +gitAnnexKeysDbIndexCache :: Git.Repo -> FilePath +gitAnnexKeysDbIndexCache r = gitAnnexKeysDb r ++ ".cache" + +{- .git/annex/fsck/uuid/ is used to store information about incremental + - fscks. -} +gitAnnexFsckDir :: UUID -> Git.Repo -> FilePath +gitAnnexFsckDir u r = gitAnnexDir r "fsck" fromUUID u + +{- used to store information about incremental fscks. -} +gitAnnexFsckState :: UUID -> Git.Repo -> FilePath +gitAnnexFsckState u r = gitAnnexFsckDir u r "state" + +{- Directory containing database used to record fsck info. -} +gitAnnexFsckDbDir :: UUID -> Git.Repo -> FilePath +gitAnnexFsckDbDir u r = gitAnnexFsckDir u r "db" + +{- Lock file for the fsck database. -} +gitAnnexFsckDbLock :: UUID -> Git.Repo -> FilePath +gitAnnexFsckDbLock u r = gitAnnexFsckDir u r "fsck.lck" + +{- .git/annex/fsckresults/uuid is used to store results of git fscks -} +gitAnnexFsckResultsLog :: UUID -> Git.Repo -> FilePath +gitAnnexFsckResultsLog u r = gitAnnexDir r "fsckresults" fromUUID u + +{- .git/annex/export/uuid/ is used to store information about + - exports to special remotes. -} +gitAnnexExportDir :: UUID -> Git.Repo -> FilePath +gitAnnexExportDir u r = gitAnnexDir r "export" fromUUID u + +{- Directory containing database used to record export info. -} +gitAnnexExportDbDir :: UUID -> Git.Repo -> FilePath +gitAnnexExportDbDir u r = gitAnnexExportDir u r "db" + +{- Lock file for export state for a special remote. -} +gitAnnexExportLock :: UUID -> Git.Repo -> FilePath +gitAnnexExportLock u r = gitAnnexExportDbDir u r ++ ".lck" + +{- .git/annex/schedulestate is used to store information about when + - scheduled jobs were last run. -} +gitAnnexScheduleState :: Git.Repo -> FilePath +gitAnnexScheduleState r = gitAnnexDir r "schedulestate" + +{- .git/annex/creds/ is used to store credentials to access some special + - remotes. -} +gitAnnexCredsDir :: Git.Repo -> FilePath +gitAnnexCredsDir r = addTrailingPathSeparator $ gitAnnexDir r "creds" + +{- .git/annex/certificate.pem and .git/annex/key.pem are used by the webapp + - when HTTPS is enabled -} +gitAnnexWebCertificate :: Git.Repo -> FilePath +gitAnnexWebCertificate r = gitAnnexDir r "certificate.pem" +gitAnnexWebPrivKey :: Git.Repo -> FilePath +gitAnnexWebPrivKey r = gitAnnexDir r "privkey.pem" + +{- .git/annex/feeds/ is used to record per-key (url) state by importfeeds -} +gitAnnexFeedStateDir :: Git.Repo -> FilePath +gitAnnexFeedStateDir r = addTrailingPathSeparator $ gitAnnexDir r "feedstate" + +gitAnnexFeedState :: Key -> Git.Repo -> FilePath +gitAnnexFeedState k r = gitAnnexFeedStateDir r keyFile k + +{- .git/annex/merge/ is used as a empty work tree for direct mode merges and + - merges in adjusted branches. -} +gitAnnexMergeDir :: Git.Repo -> FilePath +gitAnnexMergeDir r = addTrailingPathSeparator $ gitAnnexDir r "merge" + +{- .git/annex/transfer/ is used to record keys currently + - being transferred, and other transfer bookkeeping info. -} +gitAnnexTransferDir :: Git.Repo -> FilePath +gitAnnexTransferDir r = addTrailingPathSeparator $ gitAnnexDir r "transfer" + +{- .git/annex/journal/ is used to journal changes made to the git-annex + - branch -} +gitAnnexJournalDir :: Git.Repo -> FilePath +gitAnnexJournalDir r = addTrailingPathSeparator $ gitAnnexDir r "journal" + +{- Lock file for the journal. -} +gitAnnexJournalLock :: Git.Repo -> FilePath +gitAnnexJournalLock r = gitAnnexDir r "journal.lck" + +{- Lock file for the pre-commit hook. -} +gitAnnexPreCommitLock :: Git.Repo -> FilePath +gitAnnexPreCommitLock r = gitAnnexDir r "precommit.lck" + +{- Lock file for direct mode merge. -} +gitAnnexMergeLock :: Git.Repo -> FilePath +gitAnnexMergeLock r = gitAnnexDir r "merge.lck" + +{- .git/annex/index is used to stage changes to the git-annex branch -} +gitAnnexIndex :: Git.Repo -> FilePath +gitAnnexIndex r = gitAnnexDir r "index" + +{- Holds the ref of the git-annex branch that the index was last updated to. + - + - The .lck in the name is a historical accident; this is not used as a + - lock. -} +gitAnnexIndexStatus :: Git.Repo -> FilePath +gitAnnexIndexStatus r = gitAnnexDir r "index.lck" + +{- The index file used to generate a filtered branch view._-} +gitAnnexViewIndex :: Git.Repo -> FilePath +gitAnnexViewIndex r = gitAnnexDir r "viewindex" + +{- File containing a log of recently accessed views. -} +gitAnnexViewLog :: Git.Repo -> FilePath +gitAnnexViewLog r = gitAnnexDir r "viewlog" + +{- List of refs that have already been merged into the git-annex branch. -} +gitAnnexMergedRefs :: Git.Repo -> FilePath +gitAnnexMergedRefs r = gitAnnexDir r "mergedrefs" + +{- List of refs that should not be merged into the git-annex branch. -} +gitAnnexIgnoredRefs :: Git.Repo -> FilePath +gitAnnexIgnoredRefs r = gitAnnexDir r "ignoredrefs" + +{- Pid file for daemon mode. -} +gitAnnexPidFile :: Git.Repo -> FilePath +gitAnnexPidFile r = gitAnnexDir r "daemon.pid" + +{- Pid lock file for pidlock mode -} +gitAnnexPidLockFile :: Git.Repo -> FilePath +gitAnnexPidLockFile r = gitAnnexDir r "pidlock" + +{- Status file for daemon mode. -} +gitAnnexDaemonStatusFile :: Git.Repo -> FilePath +gitAnnexDaemonStatusFile r = gitAnnexDir r "daemon.status" + +{- Log file for daemon mode. -} +gitAnnexLogFile :: Git.Repo -> FilePath +gitAnnexLogFile r = gitAnnexDir r "daemon.log" + +{- Log file for fuzz test. -} +gitAnnexFuzzTestLogFile :: Git.Repo -> FilePath +gitAnnexFuzzTestLogFile r = gitAnnexDir r "fuzztest.log" + +{- Html shim file used to launch the webapp. -} +gitAnnexHtmlShim :: Git.Repo -> FilePath +gitAnnexHtmlShim r = gitAnnexDir r "webapp.html" + +{- File containing the url to the webapp. -} +gitAnnexUrlFile :: Git.Repo -> FilePath +gitAnnexUrlFile r = gitAnnexDir r "url" + +{- Temporary file used to edit configuriation from the git-annex branch. -} +gitAnnexTmpCfgFile :: Git.Repo -> FilePath +gitAnnexTmpCfgFile r = gitAnnexDir r "config.tmp" + +{- .git/annex/ssh/ is used for ssh connection caching -} +gitAnnexSshDir :: Git.Repo -> FilePath +gitAnnexSshDir r = addTrailingPathSeparator $ gitAnnexDir r "ssh" + +{- .git/annex/remotes/ is used for remote-specific state. -} +gitAnnexRemotesDir :: Git.Repo -> FilePath +gitAnnexRemotesDir r = addTrailingPathSeparator $ gitAnnexDir r "remotes" + +{- This is the base directory name used by the assistant when making + - repositories, by default. -} +gitAnnexAssistantDefaultDir :: FilePath +gitAnnexAssistantDefaultDir = "annex" + +{- Sanitizes a String that will be used as part of a Key's keyName, + - dealing with characters that cause problems. + - + - This is used when a new Key is initially being generated, eg by getKey. + - Unlike keyFile and fileKey, it does not need to be a reversable + - escaping. Also, it's ok to change this to add more problematic + - characters later. Unlike changing keyFile, which could result in the + - filenames used for existing keys changing and contents getting lost. + - + - It is, however, important that the input and output of this function + - have a 1:1 mapping, to avoid two different inputs from mapping to the + - same key. + -} +preSanitizeKeyName :: String -> String +preSanitizeKeyName = preSanitizeKeyName' False + +preSanitizeKeyName' :: Bool -> String -> String +preSanitizeKeyName' resanitize = concatMap escape + where + escape c + | isAsciiUpper c || isAsciiLower c || isDigit c = [c] + | c `elem` ".-_" = [c] -- common, assumed safe + | c `elem` "/%:" = [c] -- handled by keyFile + -- , is safe and uncommon, so will be used to escape + -- other characters. By itself, it is escaped to + -- doubled form. + | c == ',' = if not resanitize + then ",," + else "," + | otherwise = ',' : show (ord c) + +{- Converts a keyName that has been santizied with an old version of + - preSanitizeKeyName to be sanitized with the new version. -} +reSanitizeKeyName :: String -> String +reSanitizeKeyName = preSanitizeKeyName' True + +{- Converts a key into a filename fragment without any directory. + - + - Escape "/" in the key name, to keep a flat tree of files and avoid + - issues with keys containing "/../" or ending with "/" etc. + - + - "/" is escaped to "%" because it's short and rarely used, and resembles + - a slash + - "%" is escaped to "&s", and "&" to "&a"; this ensures that the mapping + - is one to one. + - ":" is escaped to "&c", because it seemed like a good idea at the time. + - + - Changing what this function escapes and how is not a good idea, as it + - can cause existing objects to get lost. + -} +keyFile :: Key -> FilePath +keyFile = concatMap esc . key2file + where + esc '&' = "&a" + esc '%' = "&s" + esc ':' = "&c" + esc '/' = "%" + esc c = [c] + +{- Reverses keyFile, converting a filename fragment (ie, the basename of + - the symlink target) into a key. -} +fileKey :: FilePath -> Maybe Key +fileKey = file2key . unesc [] + where + unesc r [] = reverse r + unesc r ('%':cs) = unesc ('/':r) cs + unesc r ('&':'c':cs) = unesc (':':r) cs + unesc r ('&':'s':cs) = unesc ('%':r) cs + unesc r ('&':'a':cs) = unesc ('&':r) cs + unesc r (c:cs) = unesc (c:r) cs + +{- for quickcheck -} +prop_isomorphic_fileKey :: String -> Bool +prop_isomorphic_fileKey s + | null s = True -- it's not legal for a key to have no keyName + | otherwise= Just k == fileKey (keyFile k) + where + k = stubKey { keyName = s, keyVariety = OtherKey "test" } + +{- A location to store a key on a special remote that uses a filesystem. + - A directory hash is used, to protect against filesystems that dislike + - having many items in a single directory. + - + - The file is put in a directory with the same name, this allows + - write-protecting the directory to avoid accidental deletion of the file. + -} +keyPath :: Key -> Hasher -> FilePath +keyPath key hasher = hasher key f f + where + f = keyFile key + +{- All possibile locations to store a key in a special remote + - using different directory hashes. + - + - This is compatible with the annexLocations, for interoperability between + - special remotes and git-annex repos. + -} +keyPaths :: Key -> [FilePath] +keyPaths key = map (\h -> keyPath key (h def)) dirHashes diff --git a/Annex/LockFile.hs b/Annex/LockFile.hs new file mode 100644 index 0000000000..1f35444f57 --- /dev/null +++ b/Annex/LockFile.hs @@ -0,0 +1,92 @@ +{- git-annex lock files. + - + - Copyright 2012-2015 Joey Hess + - + - Licensed under the GNU GPL version 3 or higher. + -} + +{-# LANGUAGE CPP #-} + +module Annex.LockFile ( + lockFileCached, + unlockFile, + getLockCache, + fromLockCache, + withExclusiveLock, + tryExclusiveLock, +) where + +import Annex.Common +import Annex +import Types.LockCache +import qualified Git +import Annex.Perms +import Annex.LockPool + +import qualified Data.Map as M + +{- Create a specified lock file, and takes a shared lock, which is retained + - in the cache. -} +lockFileCached :: FilePath -> Annex () +lockFileCached file = go =<< fromLockCache file + where + go (Just _) = noop -- already locked + go Nothing = do +#ifndef mingw32_HOST_OS + mode <- annexFileMode + lockhandle <- noUmask mode $ lockShared (Just mode) file +#else + lockhandle <- liftIO $ waitToLock $ lockShared file +#endif + changeLockCache $ M.insert file lockhandle + +unlockFile :: FilePath -> Annex () +unlockFile file = maybe noop go =<< fromLockCache file + where + go lockhandle = do + liftIO $ dropLock lockhandle + changeLockCache $ M.delete file + +getLockCache :: Annex LockCache +getLockCache = getState lockcache + +fromLockCache :: FilePath -> Annex (Maybe LockHandle) +fromLockCache file = M.lookup file <$> getLockCache + +changeLockCache :: (LockCache -> LockCache) -> Annex () +changeLockCache a = do + m <- getLockCache + changeState $ \s -> s { lockcache = a m } + +{- Runs an action with an exclusive lock held. If the lock is already + - held, blocks until it becomes free. -} +withExclusiveLock :: (Git.Repo -> FilePath) -> Annex a -> Annex a +withExclusiveLock getlockfile a = do + lockfile <- fromRepo getlockfile + createAnnexDirectory $ takeDirectory lockfile + mode <- annexFileMode + bracket (lock mode lockfile) (liftIO . dropLock) (const a) + where +#ifndef mingw32_HOST_OS + lock mode = noUmask mode . lockExclusive (Just mode) +#else + lock _mode = liftIO . waitToLock . lockExclusive +#endif + +{- Tries to take an exclusive lock and run an action. If the lock is + - already held, returns Nothing. -} +tryExclusiveLock :: (Git.Repo -> FilePath) -> Annex a -> Annex (Maybe a) +tryExclusiveLock getlockfile a = do + lockfile <- fromRepo getlockfile + createAnnexDirectory $ takeDirectory lockfile + mode <- annexFileMode + bracket (lock mode lockfile) (liftIO . unlock) go + where +#ifndef mingw32_HOST_OS + lock mode = noUmask mode . tryLockExclusive (Just mode) +#else + lock _mode = liftIO . lockExclusive +#endif + unlock = maybe noop dropLock + go Nothing = return Nothing + go (Just _) = Just <$> a diff --git a/Annex/LockPool.hs b/Annex/LockPool.hs new file mode 100644 index 0000000000..c6a34720ee --- /dev/null +++ b/Annex/LockPool.hs @@ -0,0 +1,17 @@ +{- Wraps Utility.LockPool, making pid locks be used when git-annex is so + - configured. + - + - Copyright 2015 Joey Hess + - + - Licensed under the GNU GPL version 3 or higher. + -} + +{-# LANGUAGE CPP #-} + +module Annex.LockPool (module X) where + +#ifndef mingw32_HOST_OS +import Annex.LockPool.PosixOrPid as X +#else +import Utility.LockPool.Windows as X +#endif diff --git a/Annex/LockPool/PosixOrPid.hs b/Annex/LockPool/PosixOrPid.hs new file mode 100644 index 0000000000..47d2e5144a --- /dev/null +++ b/Annex/LockPool/PosixOrPid.hs @@ -0,0 +1,97 @@ +{- Wraps Utility.LockPool, making pid locks be used when git-annex is so + - configured. + - + - Copyright 2015 Joey Hess + - + - Licensed under the GNU GPL version 3 or higher. + -} + +module Annex.LockPool.PosixOrPid ( + LockFile, + LockHandle, + lockShared, + lockExclusive, + tryLockShared, + tryLockExclusive, + dropLock, + checkLocked, + LockStatus(..), + getLockStatus, + checkSaneLock, +) where + +import Common +import Types +import Annex.Locations +import qualified Annex +import qualified Utility.LockPool.Posix as Posix +import qualified Utility.LockPool.PidLock as Pid +import qualified Utility.LockPool.LockHandle as H +import Utility.LockPool.LockHandle (LockHandle, dropLock) +import Utility.LockFile.Posix (openLockFile) +import Utility.LockPool.STM (LockFile) +import Utility.LockFile.LockStatus + +import System.Posix + +lockShared :: Maybe FileMode -> LockFile -> Annex LockHandle +lockShared m f = pidLock m f $ Posix.lockShared m f + +lockExclusive :: Maybe FileMode -> LockFile -> Annex LockHandle +lockExclusive m f = pidLock m f $ Posix.lockExclusive m f + +tryLockShared :: Maybe FileMode -> LockFile -> Annex (Maybe LockHandle) +tryLockShared m f = tryPidLock m f $ Posix.tryLockShared m f + +tryLockExclusive :: Maybe FileMode -> LockFile -> Annex (Maybe LockHandle) +tryLockExclusive m f = tryPidLock m f $ Posix.tryLockExclusive m f + +checkLocked :: LockFile -> Annex (Maybe Bool) +checkLocked f = Posix.checkLocked f `pidLockCheck` checkpid + where + checkpid pidlock = Pid.checkLocked pidlock >>= \case + -- Only return true when the posix lock file exists. + Just _ -> Posix.checkLocked f + Nothing -> return Nothing + +getLockStatus :: LockFile -> Annex LockStatus +getLockStatus f = Posix.getLockStatus f + `pidLockCheck` Pid.getLockStatus + +checkSaneLock :: LockFile -> LockHandle -> Annex Bool +checkSaneLock f h = H.checkSaneLock f h + `pidLockCheck` flip Pid.checkSaneLock h + +pidLockFile :: Annex (Maybe FilePath) +pidLockFile = ifM (annexPidLock <$> Annex.getGitConfig) + ( Just <$> Annex.fromRepo gitAnnexPidLockFile + , pure Nothing + ) + +pidLockCheck :: IO a -> (LockFile -> IO a) -> Annex a +pidLockCheck posixcheck pidcheck = + liftIO . maybe posixcheck pidcheck =<< pidLockFile + +pidLock :: Maybe FileMode -> LockFile -> IO LockHandle -> Annex LockHandle +pidLock m f posixlock = go =<< pidLockFile + where + go Nothing = liftIO posixlock + go (Just pidlock) = do + timeout <- annexPidLockTimeout <$> Annex.getGitConfig + liftIO $ do + dummyPosixLock m f + Pid.waitLock timeout pidlock + +tryPidLock :: Maybe FileMode -> LockFile -> IO (Maybe LockHandle) -> Annex (Maybe LockHandle) +tryPidLock m f posixlock = liftIO . go =<< pidLockFile + where + go Nothing = posixlock + go (Just pidlock) = do + dummyPosixLock m f + Pid.tryLock pidlock + +-- The posix lock file is created even when using pid locks, in order to +-- avoid complicating any code that might expect to be able to see that +-- lock file. But, it's not locked. +dummyPosixLock :: Maybe FileMode -> LockFile -> IO () +dummyPosixLock m f = closeFd =<< openLockFile ReadLock m f diff --git a/Annex/MakeRepo.hs b/Annex/MakeRepo.hs new file mode 100644 index 0000000000..f80e303596 --- /dev/null +++ b/Annex/MakeRepo.hs @@ -0,0 +1,90 @@ +{- making local repositories (used by webapp mostly) + - + - Copyright 2012-2014 Joey Hess + - + - Licensed under the GNU GPL version 3 or higher. + -} + +module Annex.MakeRepo where + +import Assistant.WebApp.Common +import Annex.Init +import qualified Git.Construct +import qualified Git.Config +import qualified Git.Command +import qualified Git.Branch +import qualified Annex +import Annex.UUID +import Annex.Direct +import Annex.Action +import Types.StandardGroups +import Logs.PreferredContent +import qualified Annex.Branch +import Utility.Process.Transcript + +{- Makes a new git repository. Or, if a git repository already + - exists, returns False. -} +makeRepo :: FilePath -> Bool -> IO Bool +makeRepo path bare = ifM (probeRepoExists path) + ( return False + , do + (transcript, ok) <- + processTranscript "git" (toCommand params) Nothing + unless ok $ + error $ "git init failed!\nOutput:\n" ++ transcript + return True + ) + where + baseparams = [Param "init", Param "--quiet"] + params + | bare = baseparams ++ [Param "--bare", File path] + | otherwise = baseparams ++ [File path] + +{- Runs an action in the git repository in the specified directory. -} +inDir :: FilePath -> Annex a -> IO a +inDir dir a = do + state <- Annex.new =<< Git.Config.read =<< Git.Construct.fromPath dir + Annex.eval state $ a `finally` stopCoProcesses + +{- Creates a new repository, and returns its UUID. -} +initRepo :: Bool -> Bool -> FilePath -> Maybe String -> Maybe StandardGroup -> IO UUID +initRepo True primary_assistant_repo dir desc mgroup = inDir dir $ do + initRepo' desc mgroup + {- Initialize the master branch, so things that expect + - to have it will work, before any files are added. -} + unlessM (Git.Config.isBare <$> gitRepo) $ + void $ inRepo $ Git.Branch.commitCommand Git.Branch.AutomaticCommit + [ Param "--quiet" + , Param "--allow-empty" + , Param "-m" + , Param "created repository" + ] + {- Repositories directly managed by the assistant use direct mode. + - + - Automatic gc is disabled, as it can be slow. Insted, gc is done + - once a day. + -} + when primary_assistant_repo $ do + setDirect True + inRepo $ Git.Command.run + [Param "config", Param "gc.auto", Param "0"] + getUUID +{- Repo already exists, could be a non-git-annex repo though so + - still initialize it. -} +initRepo False _ dir desc mgroup = inDir dir $ do + initRepo' desc mgroup + getUUID + +initRepo' :: Maybe String -> Maybe StandardGroup -> Annex () +initRepo' desc mgroup = unlessM isInitialized $ do + initialize (AutoInit False) desc Nothing + u <- getUUID + maybe noop (defaultStandardGroup u) mgroup + {- Ensure branch gets committed right away so it is + - available for merging immediately. -} + Annex.Branch.commit =<< Annex.Branch.commitMessage + +{- Checks if a git repo exists at a location. -} +probeRepoExists :: FilePath -> IO Bool +probeRepoExists dir = isJust <$> + catchDefaultIO Nothing (Git.Construct.checkForRepo dir) diff --git a/Annex/MetaData.hs b/Annex/MetaData.hs new file mode 100644 index 0000000000..6280e8c810 --- /dev/null +++ b/Annex/MetaData.hs @@ -0,0 +1,114 @@ +{- git-annex metadata + - + - Copyright 2014-2016 Joey Hess + - + - Licensed under the GNU GPL version 3 or higher. + -} + +module Annex.MetaData ( + genMetaData, + dateMetaData, + parseModMeta, + parseMetaDataMatcher, + module X +) where + +import Annex.Common +import qualified Annex +import Types.MetaData as X +import Annex.MetaData.StandardFields as X +import Logs.MetaData +import Annex.CatFile +import Utility.Glob + +import qualified Data.Set as S +import Data.Time.Calendar +import Data.Time.Clock +import Data.Time.Clock.POSIX + +{- Adds metadata for a file that has just been ingested into the + - annex, but has not yet been committed to git. + - + - When the file has been modified, the metadata is copied over + - from the old key to the new key. Note that it looks at the old key as + - committed to HEAD -- the new key may or may not have already been staged + - in the index. + - + - Also, can generate new metadata, if configured to do so. + -} +genMetaData :: Key -> FilePath -> FileStatus -> Annex () +genMetaData key file status = do + catKeyFileHEAD file >>= \case + Nothing -> noop + Just oldkey -> + -- Have to copy first, before adding any + -- more metadata, because copyMetaData does not + -- preserve any metadata already on key. + whenM (copyMetaData oldkey key <&&> (not <$> onlydatemeta oldkey)) $ + warncopied + whenM (annexGenMetaData <$> Annex.getGitConfig) $ do + old <- getCurrentMetaData key + addMetaData key (dateMetaData mtime old) + where + mtime = posixSecondsToUTCTime $ realToFrac $ modificationTime status + warncopied = warning $ + "Copied metadata from old version of " ++ file ++ " to new version. " ++ + "If you don't want this copied metadata, run: git annex metadata --remove-all " ++ file + -- If the only fields copied were date metadata, and they'll + -- be overwritten with the current mtime, no need to warn about + -- copying. + onlydatemeta oldkey = ifM (annexGenMetaData <$> Annex.getGitConfig) + ( null . filter (not . isDateMetaField . fst) . fromMetaData + <$> getCurrentMetaData oldkey + , return False + ) + +{- Generates metadata for a file's date stamp. + - + - Any date fields in the old metadata will be overwritten. + - + - Note that the returned MetaData does not contain all the input MetaData, + - only changes to add the date fields. -} +dateMetaData :: UTCTime -> MetaData -> MetaData +dateMetaData mtime old = modMeta old $ + (SetMeta yearMetaField $ S.singleton $ toMetaValue $ show y) + `ComposeModMeta` + (SetMeta monthMetaField $ S.singleton $ toMetaValue $ show m) + `ComposeModMeta` + (SetMeta dayMetaField $ S.singleton $ toMetaValue $ show d) + where + (y, m, d) = toGregorian $ utctDay mtime + +{- Parses field=value, field+=value, field-=value, field?=value -} +parseModMeta :: String -> Either String ModMeta +parseModMeta p = case lastMaybe f of + Just '+' -> AddMeta <$> mkMetaField f' <*> v + Just '-' -> DelMeta <$> mkMetaField f' <*> (Just <$> v) + Just '?' -> MaybeSetMeta <$> mkMetaField f' <*> v + _ -> SetMeta <$> mkMetaField f <*> (S.singleton <$> v) + where + (f, sv) = separate (== '=') p + f' = beginning f + v = pure (toMetaValue sv) + +{- Parses field=value, fieldvalue, field>=value -} +parseMetaDataMatcher :: String -> Either String (MetaField, MetaValue -> Bool) +parseMetaDataMatcher p = (,) + <$> mkMetaField f + <*> pure matcher + where + (f, op_v) = break (`elem` "=<>") p + matcher = case op_v of + ('=':v) -> checkglob v + ('<':'=':v) -> checkcmp (<=) v + ('<':v) -> checkcmp (<) v + ('>':'=':v) -> checkcmp (>=) v + ('>':v) -> checkcmp (>) v + _ -> checkglob "" + checkglob v = + let cglob = compileGlob v CaseInsensative + in matchGlob cglob . fromMetaValue + checkcmp cmp v v' = case (doubleval v, doubleval (fromMetaValue v')) of + (Just d, Just d') -> d' `cmp` d + _ -> False + doubleval v = readish v :: Maybe Double diff --git a/Annex/MetaData/StandardFields.hs b/Annex/MetaData/StandardFields.hs new file mode 100644 index 0000000000..9a1ecb6d42 --- /dev/null +++ b/Annex/MetaData/StandardFields.hs @@ -0,0 +1,59 @@ +{- git-annex metadata, standard fields + - + - Copyright 2014 Joey Hess + - + - Licensed under the GNU GPL version 3 or higher. + -} + +module Annex.MetaData.StandardFields ( + tagMetaField, + yearMetaField, + monthMetaField, + dayMetaField, + isDateMetaField, + lastChangedField, + mkLastChangedField, + isLastChangedField +) where + +import Types.MetaData + +import Data.List + +tagMetaField :: MetaField +tagMetaField = mkMetaFieldUnchecked "tag" + +yearMetaField :: MetaField +yearMetaField = mkMetaFieldUnchecked "year" + +monthMetaField :: MetaField +monthMetaField = mkMetaFieldUnchecked "month" + +dayMetaField :: MetaField +dayMetaField = mkMetaFieldUnchecked "day" + +isDateMetaField :: MetaField -> Bool +isDateMetaField f + | f == yearMetaField = True + | f == monthMetaField = True + | f == dayMetaField = True + | otherwise = False + +lastChangedField :: MetaField +lastChangedField = mkMetaFieldUnchecked lastchanged + +mkLastChangedField :: MetaField -> MetaField +mkLastChangedField f = mkMetaFieldUnchecked (fromMetaField f ++ lastchangedSuffix) + +isLastChangedField :: MetaField -> Bool +isLastChangedField f + | f == lastChangedField = True + | otherwise = lastchanged `isSuffixOf` s && s /= lastchangedSuffix + where + s = fromMetaField f + +lastchanged :: String +lastchanged = "lastchanged" + +lastchangedSuffix :: String +lastchangedSuffix = "-lastchanged" diff --git a/Annex/Multicast.hs b/Annex/Multicast.hs new file mode 100644 index 0000000000..b05fcce6ef --- /dev/null +++ b/Annex/Multicast.hs @@ -0,0 +1,54 @@ +{- git-annex multicast receive callback + - + - Copyright 2017 Joey Hess + - + - Licensed under the GNU GPL version 3 or higher. + -} + +{-# LANGUAGE CPP #-} + +module Annex.Multicast where + +import Config.Files +import Utility.Env +import Utility.PartialPrelude + +import System.Process +import System.IO +import GHC.IO.Handle.FD +#if ! MIN_VERSION_process(1,4,2) +import System.Posix.IO (handleToFd) +#endif +import Control.Applicative +import Prelude + +multicastReceiveEnv :: String +multicastReceiveEnv = "GIT_ANNEX_MULTICAST_RECEIVE" + +multicastCallbackEnv :: IO (FilePath, [(String, String)], Handle) +multicastCallbackEnv = do + gitannex <- readProgramFile +#if MIN_VERSION_process(1,4,2) + -- This will even work on Windows + (rfd, wfd) <- createPipeFd + rh <- fdToHandle rfd +#else + (rh, wh) <- createPipe + wfd <- handleToFd wh +#endif + environ <- addEntry multicastReceiveEnv (show wfd) <$> getEnvironment + return (gitannex, environ, rh) + +-- This is run when uftpd has received a file. Rather than move +-- the file into the annex here, which would require starting up the +-- Annex monad, parsing git config, and verifying the content, simply +-- output to the specified FD the filename. This keeps the time +-- that uftpd is not receiving the next file as short as possible. +runMulticastReceive :: [String] -> String -> IO () +runMulticastReceive ("-I":_sessionid:fs) hs = case readish hs of + Just fd -> do + h <- fdToHandle fd + mapM_ (hPutStrLn h) fs + hClose h + Nothing -> return () +runMulticastReceive _ _ = return () diff --git a/Annex/Notification.hs b/Annex/Notification.hs new file mode 100644 index 0000000000..ea24be9833 --- /dev/null +++ b/Annex/Notification.hs @@ -0,0 +1,108 @@ +{- git-annex desktop notifications + - + - Copyright 2014 Joey Hess + - + - Licensed under the GNU GPL version 3 or higher. + -} + +{-# LANGUAGE TypeSynonymInstances, FlexibleInstances #-} +{-# LANGUAGE CPP #-} + +module Annex.Notification (NotifyWitness, noNotification, notifyTransfer, notifyDrop) where + +import Annex.Common +import Types.Transfer +#ifdef WITH_DBUS_NOTIFICATIONS +import qualified Annex +import Types.DesktopNotify +import qualified DBus.Notify as Notify +import qualified DBus.Client +#endif + +-- Witness that notification has happened. +data NotifyWitness = NotifyWitness + +-- Only use when no notification should be done. +noNotification :: NotifyWitness +noNotification = NotifyWitness + +{- Wrap around an action that performs a transfer, which may run multiple + - attempts. Displays notification when supported and when the user asked + - for it. -} +notifyTransfer :: Transferrable t => Observable v => Direction -> t -> (NotifyWitness -> Annex v) -> Annex v +#ifdef WITH_DBUS_NOTIFICATIONS +notifyTransfer direction t a = case descTransfrerrable t of + Nothing -> a NotifyWitness + Just desc -> do + wanted <- Annex.getState Annex.desktopnotify + if (notifyStart wanted || notifyFinish wanted) + then do + client <- liftIO DBus.Client.connectSession + startnotification <- liftIO $ if notifyStart wanted + then Just <$> Notify.notify client (startedTransferNote direction desc) + else pure Nothing + res <- a NotifyWitness + let ok = observeBool res + when (notifyFinish wanted) $ liftIO $ void $ maybe + (Notify.notify client $ finishedTransferNote ok direction desc) + (\n -> Notify.replace client n $ finishedTransferNote ok direction desc) + startnotification + return res + else a NotifyWitness +#else +notifyTransfer _ _ a = a NotifyWitness +#endif + +notifyDrop :: AssociatedFile -> Bool -> Annex () +notifyDrop (AssociatedFile Nothing) _ = noop +#ifdef WITH_DBUS_NOTIFICATIONS +notifyDrop (AssociatedFile (Just f)) ok = do + wanted <- Annex.getState Annex.desktopnotify + when (notifyFinish wanted) $ liftIO $ do + client <- DBus.Client.connectSession + void $ Notify.notify client (droppedNote ok f) +#else +notifyDrop (AssociatedFile (Just _)) _ = noop +#endif + +#ifdef WITH_DBUS_NOTIFICATIONS +startedTransferNote :: Direction -> String -> Notify.Note +startedTransferNote Upload = mkNote Notify.Transfer Notify.Low iconUpload + "Uploading" +startedTransferNote Download = mkNote Notify.Transfer Notify.Low iconDownload + "Downloading" + +finishedTransferNote :: Bool -> Direction -> String -> Notify.Note +finishedTransferNote False Upload = mkNote Notify.TransferError Notify.Normal iconFailure + "Failed to upload" +finishedTransferNote False Download = mkNote Notify.TransferError Notify.Normal iconFailure + "Failed to download" +finishedTransferNote True Upload = mkNote Notify.TransferComplete Notify.Low iconSuccess + "Finished uploading" +finishedTransferNote True Download = mkNote Notify.TransferComplete Notify.Low iconSuccess + "Finished downloading" + +droppedNote :: Bool -> String -> Notify.Note +droppedNote False = mkNote Notify.TransferError Notify.Normal iconFailure + "Failed to drop" +droppedNote True = mkNote Notify.TransferComplete Notify.Low iconSuccess + "Dropped" + +iconUpload, iconDownload, iconFailure, iconSuccess :: String +iconUpload = "network-transmit" +iconDownload = "network-receive" +iconFailure = "dialog-error" +iconSuccess = "git-annex" -- Is there a standard icon for success/completion? + +mkNote :: Notify.Category -> Notify.UrgencyLevel -> String -> String -> FilePath -> Notify.Note +mkNote category urgency icon desc path = Notify.blankNote + { Notify.appName = "git-annex" + , Notify.appImage = Just (Notify.Icon icon) + , Notify.summary = desc ++ " " ++ path + , Notify.hints = + [ Notify.Category category + , Notify.Urgency urgency + , Notify.SuppressSound True + ] + } +#endif diff --git a/Annex/NumCopies.hs b/Annex/NumCopies.hs new file mode 100644 index 0000000000..60d918cae5 --- /dev/null +++ b/Annex/NumCopies.hs @@ -0,0 +1,222 @@ +{- git-annex numcopies configuration and checking + - + - Copyright 2014-2015 Joey Hess + - + - Licensed under the GNU GPL version 3 or higher. + -} + +{-# LANGUAGE CPP, ScopedTypeVariables, DeriveDataTypeable #-} + +module Annex.NumCopies ( + module Types.NumCopies, + module Logs.NumCopies, + getFileNumCopies, + getAssociatedFileNumCopies, + getGlobalFileNumCopies, + getNumCopies, + deprecatedNumCopies, + defaultNumCopies, + numCopiesCheck, + numCopiesCheck', + verifyEnoughCopiesToDrop, + verifiableCopies, + UnVerifiedCopy(..), +) where + +import Annex.Common +import qualified Annex +import Types.NumCopies +import Logs.NumCopies +import Logs.Trust +import Annex.CheckAttr +import qualified Remote +import qualified Types.Remote as Remote +import Annex.Content +import Annex.UUID + +import Control.Exception +import qualified Control.Monad.Catch as M +import Data.Typeable + +defaultNumCopies :: NumCopies +defaultNumCopies = NumCopies 1 + +fromSources :: [Annex (Maybe NumCopies)] -> Annex NumCopies +fromSources = fromMaybe defaultNumCopies <$$> getM id + +{- The git config annex.numcopies is deprecated. -} +deprecatedNumCopies :: Annex (Maybe NumCopies) +deprecatedNumCopies = annexNumCopies <$> Annex.getGitConfig + +{- Value forced on the command line by --numcopies. -} +getForcedNumCopies :: Annex (Maybe NumCopies) +getForcedNumCopies = Annex.getState Annex.forcenumcopies + +{- Numcopies value from any of the non-.gitattributes configuration + - sources. -} +getNumCopies :: Annex NumCopies +getNumCopies = fromSources + [ getForcedNumCopies + , getGlobalNumCopies + , deprecatedNumCopies + ] + +{- Numcopies value for a file, from any configuration source, including the + - deprecated git config. -} +getFileNumCopies :: FilePath -> Annex NumCopies +getFileNumCopies f = fromSources + [ getForcedNumCopies + , getFileNumCopies' f + , deprecatedNumCopies + ] + +getAssociatedFileNumCopies :: AssociatedFile -> Annex NumCopies +getAssociatedFileNumCopies (AssociatedFile afile) = + maybe getNumCopies getFileNumCopies afile + +{- This is the globally visible numcopies value for a file. So it does + - not include local configuration in the git config or command line + - options. -} +getGlobalFileNumCopies :: FilePath -> Annex NumCopies +getGlobalFileNumCopies f = fromSources + [ getFileNumCopies' f + ] + +getFileNumCopies' :: FilePath -> Annex (Maybe NumCopies) +getFileNumCopies' file = maybe getGlobalNumCopies (return . Just) =<< getattr + where + getattr = (NumCopies <$$> readish) + <$> checkAttr "annex.numcopies" file + +{- Checks if numcopies are satisfied for a file by running a comparison + - between the number of (not untrusted) copies that are + - belived to exist, and the configured value. + - + - This is good enough for everything except dropping the file, which + - requires active verification of the copies. + -} +numCopiesCheck :: FilePath -> Key -> (Int -> Int -> v) -> Annex v +numCopiesCheck file key vs = do + have <- trustExclude UnTrusted =<< Remote.keyLocations key + numCopiesCheck' file vs have + +numCopiesCheck' :: FilePath -> (Int -> Int -> v) -> [UUID] -> Annex v +numCopiesCheck' file vs have = do + NumCopies needed <- getFileNumCopies file + return $ length have `vs` needed + +data UnVerifiedCopy = UnVerifiedRemote Remote | UnVerifiedHere + deriving (Ord, Eq) + +{- Verifies that enough copies of a key exist amoung the listed remotes, + - to safely drop it, running an action with a proof if so, and + - printing an informative message if not. + -} +verifyEnoughCopiesToDrop + :: String -- message to print when there are no known locations + -> Key + -> Maybe ContentRemovalLock + -> NumCopies + -> [UUID] -- repos to skip considering (generally untrusted remotes) + -> [VerifiedCopy] -- copies already verified to exist + -> [UnVerifiedCopy] -- places to check to see if they have copies + -> (SafeDropProof -> Annex a) -- action to perform the drop + -> Annex a -- action to perform when unable to drop + -> Annex a +verifyEnoughCopiesToDrop nolocmsg key removallock need skip preverified tocheck dropaction nodropaction = + helper [] [] preverified (nub tocheck) + where + helper bad missing have [] = + liftIO (mkSafeDropProof need have removallock) >>= \case + Right proof -> dropaction proof + Left stillhave -> do + notEnoughCopies key need stillhave (skip++missing) bad nolocmsg + nodropaction + helper bad missing have (c:cs) + | isSafeDrop need have removallock = + liftIO (mkSafeDropProof need have removallock) >>= \case + Right proof -> dropaction proof + Left stillhave -> helper bad missing stillhave (c:cs) + | otherwise = case c of + UnVerifiedHere -> lockContentShared key contverified + UnVerifiedRemote r -> checkremote r contverified $ + Remote.hasKey r key >>= \case + Right True -> helper bad missing (mkVerifiedCopy RecentlyVerifiedCopy r : have) cs + Left _ -> helper (r:bad) missing have cs + Right False -> helper bad (Remote.uuid r:missing) have cs + where + contverified vc = helper bad missing (vc : have) cs + + checkremote r cont fallback = case Remote.lockContent r of + Just lockcontent -> do + -- The remote's lockContent will throw an exception + -- when it is unable to lock, in which case the + -- fallback should be run. + -- + -- On the other hand, the continuation could itself + -- throw an exception (ie, the eventual drop action + -- fails), and in this case we don't want to run the + -- fallback since part of the drop action may have + -- already been performed. + -- + -- Differentiate between these two sorts + -- of exceptions by using DropException. + let a = lockcontent key $ \v -> + cont v `catchNonAsync` (throw . DropException) + a `M.catches` + [ M.Handler (\ (e :: AsyncException) -> throwM e) +#if MIN_VERSION_base(4,7,0) + , M.Handler (\ (e :: SomeAsyncException) -> throwM e) +#endif + , M.Handler (\ (DropException e') -> throwM e') + , M.Handler (\ (_e :: SomeException) -> fallback) + ] + Nothing -> fallback + +data DropException = DropException SomeException + deriving (Typeable, Show) + +instance Exception DropException + +notEnoughCopies :: Key -> NumCopies -> [VerifiedCopy] -> [UUID] -> [Remote] -> String -> Annex () +notEnoughCopies key need have skip bad nolocmsg = do + showNote "unsafe" + if length have < fromNumCopies need + then showLongNote $ + "Could only verify the existence of " ++ + show (length have) ++ " out of " ++ show (fromNumCopies need) ++ + " necessary copies" + else do + showLongNote "Unable to lock down 1 copy of file that is required to safely drop it." + showLongNote "(This could have happened because of a concurrent drop, or because a remote has too old a version of git-annex-shell installed.)" + Remote.showTriedRemotes bad + Remote.showLocations True key (map toUUID have++skip) nolocmsg + +{- Finds locations of a key that can be used to get VerifiedCopies, + - in order to allow dropping the key. + - + - Provide a list of UUIDs that the key is being dropped from. + - The returned lists will exclude any of those UUIDs. + - + - The return lists also exclude any repositories that are untrusted, + - since those should not be used for verification. + - + - The UnVerifiedCopy list is cost ordered. + - The VerifiedCopy list contains repositories that are trusted to + - contain the key. + -} +verifiableCopies :: Key -> [UUID] -> Annex ([UnVerifiedCopy], [VerifiedCopy]) +verifiableCopies key exclude = do + locs <- Remote.keyLocations key + (remotes, trusteduuids) <- Remote.remoteLocations locs + =<< trustGet Trusted + untrusteduuids <- trustGet UnTrusted + let exclude' = exclude ++ untrusteduuids + let remotes' = Remote.remotesWithoutUUID remotes (exclude' ++ trusteduuids) + let verified = map (mkVerifiedCopy TrustedCopy) $ + filter (`notElem` exclude') trusteduuids + u <- getUUID + let herec = if u `elem` locs && u `notElem` exclude' + then [UnVerifiedHere] + else [] + return (herec ++ map UnVerifiedRemote remotes', verified) diff --git a/Annex/Path.hs b/Annex/Path.hs new file mode 100644 index 0000000000..0f85b11bff --- /dev/null +++ b/Annex/Path.hs @@ -0,0 +1,36 @@ +{- git-annex program path + - + - Copyright 2013 Joey Hess + - + - Licensed under the GNU GPL version 3 or higher. + -} + +module Annex.Path where + +import Common +import Config.Files +import Utility.Env + +import System.Environment (getExecutablePath) + +{- A fully qualified path to the currently running git-annex program. + - + - getExecutablePath is used when possible. On OSs it supports + - well, it returns the complete path to the program. But, on other OSs, + - it might return just the basename. Fall back to reading the programFile, + - or searching for the command name in PATH. + - + - The standalone build runs git-annex via ld.so, and defeats + - getExecutablePath. It sets GIT_ANNEX_PROGRAMPATH to the correct path + - to the wrapper script to use. + -} +programPath :: IO FilePath +programPath = go =<< getEnv "GIT_ANNEX_PROGRAMPATH" + where + go (Just p) = return p + go Nothing = do + exe <- getExecutablePath + p <- if isAbsolute exe + then return exe + else readProgramFile + maybe cannotFindProgram return =<< searchPath p diff --git a/Annex/Perms.hs b/Annex/Perms.hs new file mode 100644 index 0000000000..b467d4a607 --- /dev/null +++ b/Annex/Perms.hs @@ -0,0 +1,187 @@ +{- git-annex file permissions + - + - Copyright 2012 Joey Hess + - + - Licensed under the GNU GPL version 3 or higher. + -} + +module Annex.Perms ( + FileMode, + setAnnexFilePerm, + setAnnexDirPerm, + annexFileMode, + createAnnexDirectory, + noUmask, + freezeContent, + isContentWritePermOk, + thawContent, + chmodContent, + createContentDir, + freezeContentDir, + thawContentDir, + modifyContent, + withShared, +) where + +import Annex.Common +import Utility.FileMode +import Git.ConfigTypes +import qualified Annex +import Config + +withShared :: (SharedRepository -> Annex a) -> Annex a +withShared a = a =<< coreSharedRepository <$> Annex.getGitConfig + +setAnnexFilePerm :: FilePath -> Annex () +setAnnexFilePerm = setAnnexPerm False + +setAnnexDirPerm :: FilePath -> Annex () +setAnnexDirPerm = setAnnexPerm True + +{- Sets appropriate file mode for a file or directory in the annex, + - other than the content files and content directory. Normally, + - use the default mode, but with core.sharedRepository set, + - allow the group to write, etc. -} +setAnnexPerm :: Bool -> FilePath -> Annex () +setAnnexPerm isdir file = unlessM crippledFileSystem $ + withShared $ liftIO . go + where + go GroupShared = void $ tryIO $ modifyFileMode file $ addModes $ + groupSharedModes ++ + if isdir then [ ownerExecuteMode, groupExecuteMode ] else [] + go AllShared = void $ tryIO $ modifyFileMode file $ addModes $ + readModes ++ + [ ownerWriteMode, groupWriteMode ] ++ + if isdir then executeModes else [] + go _ = noop + +{- Gets the appropriate mode to use for creating a file in the annex + - (other than content files, which are locked down more). -} +annexFileMode :: Annex FileMode +annexFileMode = withShared $ return . go + where + go GroupShared = sharedmode + go AllShared = combineModes (sharedmode:readModes) + go _ = stdFileMode + sharedmode = combineModes groupSharedModes + +{- Creates a directory inside the gitAnnexDir, including any parent + - directories. Makes directories with appropriate permissions. -} +createAnnexDirectory :: FilePath -> Annex () +createAnnexDirectory dir = walk dir [] =<< top + where + top = parentDir <$> fromRepo gitAnnexDir + walk d below stop + | d `equalFilePath` stop = done + | otherwise = ifM (liftIO $ doesDirectoryExist d) + ( done + , walk (parentDir d) (d:below) stop + ) + where + done = forM_ below $ \p -> do + liftIO $ createDirectoryIfMissing True p + setAnnexDirPerm p + +{- Normally, blocks writing to an annexed file, and modifies file + - permissions to allow reading it. + - + - When core.sharedRepository is set, the write bits are not removed from + - the file, but instead the appropriate group write bits are set. This is + - necessary to let other users in the group lock the file. But, in a + - shared repository, the current user may not be able to change a file + - owned by another user, so failure to set this mode is ignored. + -} +freezeContent :: FilePath -> Annex () +freezeContent file = unlessM crippledFileSystem $ + withShared go + where + go GroupShared = liftIO $ void $ tryIO $ modifyFileMode file $ + addModes [ownerReadMode, groupReadMode, ownerWriteMode, groupWriteMode] + go AllShared = liftIO $ void $ tryIO $ modifyFileMode file $ + addModes (readModes ++ writeModes) + go _ = liftIO $ modifyFileMode file $ + removeModes writeModes . + addModes [ownerReadMode] + +isContentWritePermOk :: FilePath -> Annex Bool +isContentWritePermOk file = ifM crippledFileSystem + ( return True + , withShared go + ) + where + go GroupShared = want [ownerWriteMode, groupWriteMode] + go AllShared = want writeModes + go _ = return True + want wantmode = + liftIO (catchMaybeIO $ fileMode <$> getFileStatus file) >>= return . \case + Nothing -> True + Just havemode -> havemode == combineModes (havemode:wantmode) + +{- Adjusts read mode of annexed file per core.sharedRepository setting. -} +chmodContent :: FilePath -> Annex () +chmodContent file = unlessM crippledFileSystem $ + withShared go + where + go GroupShared = liftIO $ void $ tryIO $ modifyFileMode file $ + addModes [ownerReadMode, groupReadMode] + go AllShared = liftIO $ void $ tryIO $ modifyFileMode file $ + addModes readModes + go _ = liftIO $ modifyFileMode file $ + addModes [ownerReadMode] + +{- Allows writing to an annexed file that freezeContent was called on + - before. -} +thawContent :: FilePath -> Annex () +thawContent file = thawPerms $ withShared go + where + go GroupShared = liftIO $ void $ tryIO $ groupWriteRead file + go AllShared = liftIO $ void $ tryIO $ groupWriteRead file + go _ = liftIO $ allowWrite file + +{- Runs an action that thaws a file's permissions. This will probably + - fail on a crippled filesystem. But, if file modes are supported on a + - crippled filesystem, the file may be frozen, so try to thaw it. -} +thawPerms :: Annex () -> Annex () +thawPerms a = ifM crippledFileSystem + ( void $ tryNonAsync a + , a + ) + +{- Blocks writing to the directory an annexed file is in, to prevent the + - file accidentally being deleted. However, if core.sharedRepository + - is set, this is not done, since the group must be allowed to delete the + - file. + -} +freezeContentDir :: FilePath -> Annex () +freezeContentDir file = unlessM crippledFileSystem $ + withShared go + where + dir = parentDir file + go GroupShared = liftIO $ void $ tryIO $ groupWriteRead dir + go AllShared = liftIO $ void $ tryIO $ groupWriteRead dir + go _ = liftIO $ preventWrite dir + +thawContentDir :: FilePath -> Annex () +thawContentDir file = thawPerms $ liftIO $ allowWrite $ parentDir file + +{- Makes the directory tree to store an annexed file's content, + - with appropriate permissions on each level. -} +createContentDir :: FilePath -> Annex () +createContentDir dest = do + unlessM (liftIO $ doesDirectoryExist dir) $ + createAnnexDirectory dir + -- might have already existed with restricted perms + unlessM crippledFileSystem $ + liftIO $ allowWrite dir + where + dir = parentDir dest + +{- Creates the content directory for a file if it doesn't already exist, + - or thaws it if it does, then runs an action to modify the file, and + - finally, freezes the content directory. -} +modifyContent :: FilePath -> Annex a -> Annex a +modifyContent f a = do + createContentDir f -- also thaws it + v <- tryNonAsync a + freezeContentDir f + either throwM return v diff --git a/Annex/Queue.hs b/Annex/Queue.hs new file mode 100644 index 0000000000..c5555caec4 --- /dev/null +++ b/Annex/Queue.hs @@ -0,0 +1,101 @@ +{- git-annex command queue + - + - Copyright 2011, 2012 Joey Hess + - + - Licensed under the GNU GPL version 3 or higher. + -} + +{-# LANGUAGE BangPatterns #-} + +module Annex.Queue ( + addCommand, + addInternalAction, + addUpdateIndex, + flush, + flushWhenFull, + size, + get, + mergeFrom, +) where + +import Annex.Common +import Annex hiding (new) +import qualified Git.Queue +import qualified Git.UpdateIndex + +import qualified Control.Concurrent.SSem as SSem + +{- Adds a git command to the queue. -} +addCommand :: String -> [CommandParam] -> [FilePath] -> Annex () +addCommand command params files = do + q <- get + store <=< flushWhenFull <=< inRepo $ + Git.Queue.addCommand command params files q + +addInternalAction :: Git.Queue.InternalActionRunner -> [(FilePath, IO Bool)] -> Annex () +addInternalAction runner files = do + q <- get + store <=< flushWhenFull <=< inRepo $ + Git.Queue.addInternalAction runner files q + +{- Adds an update-index stream to the queue. -} +addUpdateIndex :: Git.UpdateIndex.Streamer -> Annex () +addUpdateIndex streamer = do + q <- get + store <=< flushWhenFull <=< inRepo $ + Git.Queue.addUpdateIndex streamer q + +{- Runs the queue if it is full. -} +flushWhenFull :: Git.Queue.Queue -> Annex Git.Queue.Queue +flushWhenFull q + | Git.Queue.full q = flush' q + | otherwise = return q + +{- Runs (and empties) the queue. -} +flush :: Annex () +flush = do + q <- get + unless (0 == Git.Queue.size q) $ do + store =<< flush' q + +{- When there are multiple worker threads, each has its own queue. + - + - But, flushing two queues at the same time could lead to failures due to + - git locking files. So, only one queue is allowed to flush at a time. + - The repoqueuesem is shared between threads. + -} +flush' :: Git.Queue.Queue -> Annex Git.Queue.Queue +flush' q = bracket lock unlock go + where + lock = do + s <- getState repoqueuesem + liftIO $ SSem.wait s + return s + unlock = liftIO . SSem.signal + go _ = do + showStoringStateAction + inRepo $ Git.Queue.flush q + +{- Gets the size of the queue. -} +size :: Annex Int +size = Git.Queue.size <$> get + +get :: Annex Git.Queue.Queue +get = maybe new return =<< getState repoqueue + +new :: Annex Git.Queue.Queue +new = do + q <- Git.Queue.new . annexQueueSize <$> getGitConfig + store q + return q + +store :: Git.Queue.Queue -> Annex () +store q = changeState $ \s -> s { repoqueue = Just q } + +mergeFrom :: AnnexState -> Annex () +mergeFrom st = case repoqueue st of + Nothing -> noop + Just newq -> do + q <- get + let !q' = Git.Queue.merge q newq + store =<< flushWhenFull q' diff --git a/Annex/ReplaceFile.hs b/Annex/ReplaceFile.hs new file mode 100644 index 0000000000..be12cbbb08 --- /dev/null +++ b/Annex/ReplaceFile.hs @@ -0,0 +1,56 @@ +{- git-annex file replacing + - + - Copyright 2013-2015 Joey Hess + - + - Licensed under the GNU GPL version 3 or higher. + -} + +{-# LANGUAGE CPP #-} + +module Annex.ReplaceFile where + +import Annex.Common +import Annex.Perms +import Utility.Tmp.Dir +import Utility.Path.Max + +{- Replaces a possibly already existing file with a new version, + - atomically, by running an action. + - + - The action is passed the name of temp file, in a temp directory, + - which it can write to, and once done the temp file is moved into place + - and anything else in the temp directory is deleted. + - + - The action can throw an exception, in which case the temp directory + - will be deleted, and the existing file will be preserved. + - + - Throws an IO exception when it was unable to replace the file. + -} +replaceFile :: FilePath -> (FilePath -> Annex a) -> Annex a +replaceFile file action = do + misctmpdir <- fromRepo gitAnnexTmpMiscDir + void $ createAnnexDirectory misctmpdir +#ifndef mingw32_HOST_OS + -- Use part of the filename as the template for the temp + -- directory. This does not need to be unique, but it + -- makes it more clear what this temp directory is for. + filemax <- liftIO $ fileNameLengthLimit misctmpdir + let basetmp = take (filemax `div` 2) (takeFileName file) +#else + -- Windows has limits on the whole path length, so keep + -- it short. + let basetmp = "t" +#endif + withTmpDirIn misctmpdir basetmp $ \tmpdir -> do + let tmpfile = tmpdir basetmp + r <- action tmpfile + liftIO $ replaceFileFrom tmpfile file + return r + +replaceFileFrom :: FilePath -> FilePath -> IO () +replaceFileFrom src dest = go `catchIO` fallback + where + go = moveFile src dest + fallback _ = do + createDirectoryIfMissing True $ parentDir dest + go diff --git a/Annex/SpecialRemote.hs b/Annex/SpecialRemote.hs new file mode 100644 index 0000000000..64f90852bf --- /dev/null +++ b/Annex/SpecialRemote.hs @@ -0,0 +1,91 @@ +{- git-annex special remote configuration + - + - Copyright 2011-2015 Joey Hess + - + - Licensed under the GNU GPL version 3 or higher. + -} + +module Annex.SpecialRemote where + +import Annex.Common +import Remote (remoteTypes, remoteMap) +import Types.Remote (RemoteConfig, RemoteConfigKey, SetupStage(..), typename, setup) +import Types.GitConfig +import Logs.Remote +import Logs.Trust +import qualified Git.Config +import Git.Types (RemoteName) + +import qualified Data.Map as M +import Data.Ord + +{- See if there's an existing special remote with this name. + - + - Prefer remotes that are not dead when a name appears multiple times. -} +findExisting :: RemoteName -> Annex (Maybe (UUID, RemoteConfig)) +findExisting name = do + t <- trustMap + headMaybe + . sortBy (comparing $ \(u, _c) -> Down $ M.lookup u t) + . findByName name + <$> Logs.Remote.readRemoteLog + +newConfig :: RemoteName -> RemoteConfig +newConfig = M.singleton nameKey + +findByName :: RemoteName -> M.Map UUID RemoteConfig -> [(UUID, RemoteConfig)] +findByName n = filter (matching . snd) . M.toList + where + matching c = case M.lookup nameKey c of + Nothing -> False + Just n' + | n' == n -> True + | otherwise -> False + +specialRemoteMap :: Annex (M.Map UUID RemoteName) +specialRemoteMap = do + m <- Logs.Remote.readRemoteLog + return $ M.fromList $ mapMaybe go (M.toList m) + where + go (u, c) = case M.lookup nameKey c of + Nothing -> Nothing + Just n -> Just (u, n) + +{- find the specified remote type -} +findType :: RemoteConfig -> Either String RemoteType +findType config = maybe unspecified specified $ M.lookup typeKey config + where + unspecified = Left "Specify the type of remote with type=" + specified s = case filter (findtype s) remoteTypes of + [] -> Left $ "Unknown remote type " ++ s + (t:_) -> Right t + findtype s i = typename i == s + +{- The name of a configured remote is stored in its config using this key. -} +nameKey :: RemoteConfigKey +nameKey = "name" + +{- The type of a remote is stored in its config using this key. -} +typeKey :: RemoteConfigKey +typeKey = "type" + +autoEnableKey :: RemoteConfigKey +autoEnableKey = "autoenable" + +autoEnable :: Annex () +autoEnable = do + remotemap <- M.filter configured <$> readRemoteLog + enabled <- remoteMap id + forM_ (M.toList remotemap) $ \(u, c) -> unless (u `M.member` enabled) $ do + case (M.lookup nameKey c, findType c) of + (Just name, Right t) -> whenM (canenable u) $ do + showSideAction $ "Auto enabling special remote " ++ name + dummycfg <- liftIO dummyRemoteGitConfig + tryNonAsync (setup t (Enable c) (Just u) Nothing c dummycfg) >>= \case + Left e -> warning (show e) + Right _ -> return () + _ -> return () + where + configured rc = fromMaybe False $ + Git.Config.isTrue =<< M.lookup autoEnableKey rc + canenable u = (/= DeadTrusted) <$> lookupTrust u diff --git a/Annex/Ssh.hs b/Annex/Ssh.hs new file mode 100644 index 0000000000..b6e5a77483 --- /dev/null +++ b/Annex/Ssh.hs @@ -0,0 +1,429 @@ +{- git-annex ssh interface, with connection caching + - + - Copyright 2012-2017 Joey Hess + - + - Licensed under the GNU GPL version 3 or higher. + -} + +{-# LANGUAGE CPP #-} + +module Annex.Ssh ( + ConsumeStdin(..), + SshCommand, + sshCommand, + sshOptions, + sshCacheDir, + sshReadPort, + forceSshCleanup, + sshOptionsEnv, + sshOptionsTo, + inRepoWithSshOptionsTo, + runSshOptions, + sshAskPassEnv, + runSshAskPass +) where + +import Annex.Common +import Annex.LockFile +import qualified BuildInfo +import qualified Annex +import qualified Git +import qualified Git.Url +import Config +import Annex.Path +import Utility.Env +import Utility.FileSystemEncoding +import Utility.Hash +import Types.CleanupActions +import Types.Concurrency +import Git.Env +import Git.Ssh +#ifndef mingw32_HOST_OS +import Annex.Perms +import Annex.LockPool +#endif + +import Control.Concurrent.STM + +{- Some ssh commands are fed stdin on a pipe and so should be allowed to + - consume it. But ssh commands that are not piped stdin should generally + - not be allowed to consume the process's stdin. -} +data ConsumeStdin = ConsumeStdin | NoConsumeStdin + +{- Generates a command to ssh to a given host (or user@host) on a given + - port. This includes connection caching parameters, and any ssh-options. + - If GIT_SSH or GIT_SSH_COMMAND is enabled, they are used instead. -} +sshCommand :: ConsumeStdin -> (SshHost, Maybe SshPort) -> RemoteGitConfig -> SshCommand -> Annex (FilePath, [CommandParam]) +sshCommand cs (host, port) gc remotecmd = ifM (liftIO safe_GIT_SSH) + ( maybe go return + =<< liftIO (gitSsh' host port remotecmd (consumeStdinParams cs)) + , go + ) + where + go = do + ps <- sshOptions cs (host, port) gc [] + return ("ssh", Param (fromSshHost host):ps++[Param remotecmd]) + +{- Generates parameters to ssh to a given host (or user@host) on a given + - port. This includes connection caching parameters, and any + - ssh-options. Note that the host to ssh to and the command to run + - are not included in the returned options. -} +sshOptions :: ConsumeStdin -> (SshHost, Maybe Integer) -> RemoteGitConfig -> [CommandParam] -> Annex [CommandParam] +sshOptions cs (host, port) gc opts = go =<< sshCachingInfo (host, port) + where + go (Nothing, params) = return $ mkparams cs params + go (Just socketfile, params) = do + prepSocket socketfile host (mkparams NoConsumeStdin params) + + return $ mkparams cs params + mkparams cs' ps = concat + [ ps + , map Param (remoteAnnexSshOptions gc) + , opts + , portParams port + , consumeStdinParams cs' + , [Param "-T"] + ] + +{- Due to passing -n to GIT_SSH and GIT_SSH_COMMAND, some settings + - of those that expect exactly git's parameters will break. So only + - use those if the user set GIT_ANNEX_USE_GIT_SSH to say it's ok. -} +safe_GIT_SSH :: IO Bool +safe_GIT_SSH = (== Just "1") <$> getEnv "GIT_ANNEX_USE_GIT_SSH" + +consumeStdinParams :: ConsumeStdin -> [CommandParam] +consumeStdinParams ConsumeStdin = [] +consumeStdinParams NoConsumeStdin = [Param "-n"] + +{- Returns a filename to use for a ssh connection caching socket, and + - parameters to enable ssh connection caching. -} +sshCachingInfo :: (SshHost, Maybe Integer) -> Annex (Maybe FilePath, [CommandParam]) +sshCachingInfo (host, port) = go =<< sshCacheDir + where + go Nothing = return (Nothing, []) + go (Just dir) = + liftIO (bestSocketPath $ dir hostport2socket host port) >>= return . \case + Nothing -> (Nothing, []) + Just socketfile -> (Just socketfile, sshConnectionCachingParams socketfile) + +{- Given an absolute path to use for a socket file, + - returns whichever is shorter of that or the relative path to the same + - file. + - + - If no path can be constructed that is a valid socket, returns Nothing. -} +bestSocketPath :: FilePath -> IO (Maybe FilePath) +bestSocketPath abssocketfile = do + relsocketfile <- liftIO $ relPathCwdToFile abssocketfile + let socketfile = if length abssocketfile <= length relsocketfile + then abssocketfile + else relsocketfile + return $ if valid_unix_socket_path (socketfile ++ sshgarbage) + then Just socketfile + else Nothing + where + -- ssh appends a 16 char extension to the socket when setting it + -- up, which needs to be taken into account when checking + -- that a valid socket was constructed. + sshgarbage = replicate (1+16) 'X' + +sshConnectionCachingParams :: FilePath -> [CommandParam] +sshConnectionCachingParams socketfile = + [ Param "-S", Param socketfile + , Param "-o", Param "ControlMaster=auto" + , Param "-o", Param "ControlPersist=yes" + ] + +{- ssh connection caching creates sockets, so will not work on a + - crippled filesystem. A GIT_ANNEX_TMP_DIR can be provided to use + - a different filesystem. -} +sshCacheDir :: Annex (Maybe FilePath) +sshCacheDir + | BuildInfo.sshconnectioncaching = + ifM (fromMaybe True . annexSshCaching <$> Annex.getGitConfig) + ( ifM crippledFileSystem + ( maybe (return Nothing) usetmpdir =<< gettmpdir + , Just <$> fromRepo gitAnnexSshDir + ) + , return Nothing + ) + | otherwise = return Nothing + where + gettmpdir = liftIO $ getEnv "GIT_ANNEX_TMP_DIR" + usetmpdir tmpdir = liftIO $ catchMaybeIO $ do + let socktmp = tmpdir "ssh" + createDirectoryIfMissing True socktmp + return socktmp + +portParams :: Maybe Integer -> [CommandParam] +portParams Nothing = [] +portParams (Just port) = [Param "-p", Param $ show port] + +{- Prepare to use a socket file for ssh connection caching. + - + - When concurrency is enabled, this blocks until a ssh connection + - has been made to the host. So, any password prompting by ssh will + - happen in this call, and only one ssh process will prompt at a time. + - + - Locks the socket lock file to prevent other git-annex processes from + - stopping the ssh multiplexer on this socket. + -} +prepSocket :: FilePath -> SshHost -> [CommandParam] -> Annex () +prepSocket socketfile sshhost sshparams = do + -- There could be stale ssh connections hanging around + -- from a previous git-annex run that was interrupted. + -- This must run only once, before we have made any ssh connection, + -- and any other prepSocket calls must block while it's run. + tv <- Annex.getState Annex.sshstalecleaned + join $ liftIO $ atomically $ do + cleaned <- takeTMVar tv + if cleaned + then do + putTMVar tv cleaned + return noop + else return $ do + sshCleanup + liftIO $ atomically $ putTMVar tv True + -- Cleanup at shutdown. + Annex.addCleanup SshCachingCleanup sshCleanup + + liftIO $ createDirectoryIfMissing True $ parentDir socketfile + let socketlock = socket2lock socketfile + + Annex.getState Annex.concurrency >>= \case + Concurrent {} -> makeconnection socketlock + _ -> return () + + lockFileCached socketlock + where + -- When the LockCache already has the socketlock in it, + -- the connection has already been started. Otherwise, + -- get the connection started now. + makeconnection socketlock = + whenM (isNothing <$> fromLockCache socketlock) $ + -- See if ssh can connect in batch mode, + -- if so there's no need to block for a password + -- prompt. + unlessM (tryssh ["-o", "BatchMode=true"]) $ + -- ssh needs to prompt (probably) + -- If the user enters the wrong password, + -- ssh will tell them, so we can ignore + -- failure. + void $ prompt $ tryssh [] + -- Try to ssh to the host quietly. Returns True if ssh apparently + -- connected to the host successfully. If ssh failed to connect, + -- returns False. + -- Even if ssh is forced to run some specific command, this will + -- return True. + -- (Except there's an unlikely false positive where a forced + -- ssh command exits 255.) + tryssh extraps = liftIO $ withNullHandle $ \nullh -> do + let p = proc "ssh" $ concat + [ extraps + , toCommand sshparams + , [fromSshHost sshhost, "true"] + ] + (Nothing, Nothing, Nothing, pid) <- createProcess $ p + { std_out = UseHandle nullh + , std_err = UseHandle nullh + } + exitcode <- waitForProcess pid + return $ case exitcode of + ExitFailure 255 -> False + _ -> True + +{- Find ssh socket files. + - + - The check that the lock file exists makes only socket files + - that were set up by prepSocket be found. On some NFS systems, + - a deleted socket file may linger for a while under another filename; + - and this check makes such files be skipped since the corresponding lock + - file won't exist. + -} +enumSocketFiles :: Annex [FilePath] +enumSocketFiles = liftIO . go =<< sshCacheDir + where + go Nothing = return [] + go (Just dir) = filterM (doesFileExist . socket2lock) + =<< filter (not . isLock) + <$> catchDefaultIO [] (dirContents dir) + +{- Stop any unused ssh connection caching processes. -} +sshCleanup :: Annex () +sshCleanup = mapM_ cleanup =<< enumSocketFiles + where + cleanup socketfile = do +#ifndef mingw32_HOST_OS + -- Drop any shared lock we have, and take an + -- exclusive lock, without blocking. If the lock + -- succeeds, nothing is using this ssh, and it can + -- be stopped. + -- + -- After ssh is stopped cannot remove the lock file; + -- other processes may be waiting on our exclusive + -- lock to use it. + let lockfile = socket2lock socketfile + unlockFile lockfile + mode <- annexFileMode + noUmask mode (tryLockExclusive (Just mode) lockfile) >>= \case + Nothing -> noop + Just lck -> do + forceStopSsh socketfile + liftIO $ dropLock lck +#else + forceStopSsh socketfile +#endif + +{- Stop all ssh connection caching processes, even when they're in use. -} +forceSshCleanup :: Annex () +forceSshCleanup = mapM_ forceStopSsh =<< enumSocketFiles + +forceStopSsh :: FilePath -> Annex () +forceStopSsh socketfile = do + let (dir, base) = splitFileName socketfile + let params = sshConnectionCachingParams base + -- "ssh -O stop" is noisy on stderr even with -q + void $ liftIO $ catchMaybeIO $ + withQuietOutput createProcessSuccess $ + (proc "ssh" $ toCommand $ + [ Param "-O", Param "stop" ] ++ + params ++ [Param "localhost"]) + { cwd = Just dir } + liftIO $ nukeFile socketfile + +{- This needs to be as short as possible, due to limitations on the length + - of the path to a socket file. At the same time, it needs to be unique + - for each host. + -} +hostport2socket :: SshHost -> Maybe Integer -> FilePath +hostport2socket host Nothing = hostport2socket' $ fromSshHost host +hostport2socket host (Just port) = hostport2socket' $ + fromSshHost host ++ "!" ++ show port +hostport2socket' :: String -> FilePath +hostport2socket' s + | length s > lengthofmd5s = show $ md5 $ encodeBS s + | otherwise = s + where + lengthofmd5s = 32 + +socket2lock :: FilePath -> FilePath +socket2lock socket = socket ++ lockExt + +isLock :: FilePath -> Bool +isLock f = lockExt `isSuffixOf` f + +lockExt :: String +lockExt = ".lock" + +{- This is the size of the sun_path component of sockaddr_un, which + - is the limit to the total length of the filename of a unix socket. + - + - On Linux, this is 108. On OSX, 104. TODO: Probe + -} +sizeof_sockaddr_un_sun_path :: Int +sizeof_sockaddr_un_sun_path = 100 + +{- Note that this looks at the true length of the path in bytes, as it will + - appear on disk. -} +valid_unix_socket_path :: FilePath -> Bool +valid_unix_socket_path f = length (decodeW8 f) < sizeof_sockaddr_un_sun_path + +{- Parses the SSH port, and returns the other OpenSSH options. If + - several ports are found, the last one takes precedence. -} +sshReadPort :: [String] -> (Maybe Integer, [String]) +sshReadPort params = (port, reverse args) + where + (port,args) = aux (Nothing, []) params + aux (p,ps) [] = (p,ps) + aux (_,ps) ("-p":p:rest) = aux (readPort p, ps) rest + aux (p,ps) (q:rest) | "-p" `isPrefixOf` q = aux (readPort $ drop 2 q, ps) rest + | otherwise = aux (p,q:ps) rest + readPort p = fmap fst $ listToMaybe $ reads p + +{- When this env var is set, git-annex runs ssh with the specified + - options. (The options are separated by newlines.) + - + - This is a workaround for GIT_SSH not being able to contain + - additional parameters to pass to ssh. (GIT_SSH_COMMAND can, + - but is not supported by older versions of git.) -} +sshOptionsEnv :: String +sshOptionsEnv = "GIT_ANNEX_SSHOPTION" + +toSshOptionsEnv :: [CommandParam] -> String +toSshOptionsEnv = unlines . toCommand + +fromSshOptionsEnv :: String -> [CommandParam] +fromSshOptionsEnv = map Param . lines + +{- Enables ssh caching for git push/pull to a particular + - remote git repo. (Can safely be used on non-ssh remotes.) + - + - Also propigates any configured ssh-options. + - + - Like inRepo, the action is run with the local git repo. + - But here it's a modified version, with gitEnv to set GIT_SSH=git-annex, + - and sshOptionsEnv set so that git-annex will know what socket + - file to use. -} +inRepoWithSshOptionsTo :: Git.Repo -> RemoteGitConfig -> (Git.Repo -> IO a) -> Annex a +inRepoWithSshOptionsTo remote gc a = + liftIO . a =<< sshOptionsTo remote gc =<< gitRepo + +{- To make any git commands be run with ssh caching enabled, + - and configured ssh-options alters the local Git.Repo's gitEnv + - to set GIT_SSH=git-annex, and set sshOptionsEnv when running git + - commands. + - + - If GIT_SSH or GIT_SSH_COMMAND are enabled, this has no effect. -} +sshOptionsTo :: Git.Repo -> RemoteGitConfig -> Git.Repo -> Annex Git.Repo +sshOptionsTo remote gc localr + | not (Git.repoIsUrl remote) || Git.repoIsHttp remote = unchanged + | otherwise = case Git.Url.hostuser remote of + Nothing -> unchanged + Just host -> ifM (liftIO $ safe_GIT_SSH <&&> gitSshEnvSet) + ( unchanged + , do + let port = Git.Url.port remote + let sshhost = either error id (mkSshHost host) + (msockfile, cacheparams) <- sshCachingInfo (sshhost, port) + case msockfile of + Nothing -> use [] + Just sockfile -> do + prepSocket sockfile sshhost $ concat + [ cacheparams + , map Param (remoteAnnexSshOptions gc) + , portParams port + , consumeStdinParams NoConsumeStdin + , [Param "-T"] + ] + use cacheparams + ) + where + unchanged = return localr + + use opts = do + let sshopts = concat + [ opts + , map Param (remoteAnnexSshOptions gc) + ] + if null sshopts + then unchanged + else do + command <- liftIO programPath + liftIO $ do + localr' <- addGitEnv localr sshOptionsEnv + (toSshOptionsEnv sshopts) + addGitEnv localr' gitSshEnv command + +runSshOptions :: [String] -> String -> IO () +runSshOptions args s = do + let args' = toCommand (fromSshOptionsEnv s) ++ args + let p = proc "ssh" args' + exitWith =<< waitForProcess . processHandle =<< createProcess p + +{- When this env var is set, git-annex is being used as a ssh-askpass + - program, and should read the password from the specified location, + - and output it for ssh to read. -} +sshAskPassEnv :: String +sshAskPassEnv = "GIT_ANNEX_SSHASKPASS" + +runSshAskPass :: FilePath -> IO () +runSshAskPass passfile = putStrLn =<< readFile passfile diff --git a/Annex/TaggedPush.hs b/Annex/TaggedPush.hs new file mode 100644 index 0000000000..13c53a3ca0 --- /dev/null +++ b/Annex/TaggedPush.hs @@ -0,0 +1,64 @@ +{- git-annex tagged pushes + - + - Copyright 2012 Joey Hess + - + - Licensed under the GNU GPL version 3 or higher. + -} + +module Annex.TaggedPush where + +import Annex.Common +import qualified Remote +import qualified Annex.Branch +import qualified Git +import qualified Git.Ref +import qualified Git.Command +import qualified Git.Branch +import Utility.Base64 + +{- Converts a git branch into a branch that is tagged with a UUID, typically + - the UUID of the repo that will be pushing it, and possibly with other + - information. + - + - Pushing to branches on the remote that have our uuid in them is ugly, + - but it reserves those branches for pushing by us, and so our pushes will + - never conflict with other pushes. + - + - To avoid cluttering up the branch display, the branch is put under + - refs/synced/, rather than the usual refs/remotes/ + - + - Both UUIDs and Base64 encoded data are always legal to be used in git + - refs, per git-check-ref-format. + -} +toTaggedBranch :: UUID -> Maybe String -> Git.Branch -> Git.Ref +toTaggedBranch u info b = Git.Ref $ intercalate "/" $ catMaybes + [ Just "refs/synced" + , Just $ fromUUID u + , toB64 <$> info + , Just $ Git.fromRef $ Git.Ref.base b + ] + +fromTaggedBranch :: Git.Ref -> Maybe (UUID, Maybe String) +fromTaggedBranch b = case splitc '/' $ Git.fromRef b of + ("refs":"synced":u:info:_base) -> + Just (toUUID u, fromB64Maybe info) + ("refs":"synced":u:_base) -> + Just (toUUID u, Nothing) + _ -> Nothing + +listTaggedBranches :: Annex [(Git.Sha, Git.Ref)] +listTaggedBranches = filter (isJust . fromTaggedBranch . snd) + <$> inRepo Git.Ref.list + +taggedPush :: UUID -> Maybe String -> Git.Ref -> Remote -> Git.Repo -> IO Bool +taggedPush u info branch remote = Git.Command.runBool + [ Param "push" + , Param $ Remote.name remote + {- Using forcePush here is safe because we "own" the tagged branch + - we're pushing; it has no other writers. Ensures it is pushed + - even if it has been rewritten by a transition. -} + , Param $ Git.Branch.forcePush $ refspec Annex.Branch.name + , Param $ refspec branch + ] + where + refspec b = Git.fromRef b ++ ":" ++ Git.fromRef (toTaggedBranch u info b) diff --git a/Annex/Transfer.hs b/Annex/Transfer.hs new file mode 100644 index 0000000000..b207005412 --- /dev/null +++ b/Annex/Transfer.hs @@ -0,0 +1,282 @@ +{- git-annex transfers + - + - Copyright 2012-2018 Joey Hess + - + - Licensed under the GNU GPL version 3 or higher. + -} + +{-# LANGUAGE CPP, BangPatterns #-} + +module Annex.Transfer ( + module X, + upload, + download, + runTransfer, + alwaysRunTransfer, + noRetry, + stdRetry, + pickRemote, +) where + +import Annex.Common +import qualified Annex +import Logs.Transfer as X +import Types.Transfer as X +import Annex.Notification as X +import Annex.Perms +import Utility.Metered +import Utility.ThreadScheduler +import Annex.LockPool +import Types.Key +import qualified Types.Remote as Remote +import Types.Concurrency + +import Control.Concurrent +import qualified Data.Map.Strict as M +import Data.Ord + +upload :: Observable v => UUID -> Key -> AssociatedFile -> RetryDecider -> (MeterUpdate -> Annex v) -> NotifyWitness -> Annex v +upload u key f d a _witness = guardHaveUUID u $ + runTransfer (Transfer Upload u key) f d a + +download :: Observable v => UUID -> Key -> AssociatedFile -> RetryDecider -> (MeterUpdate -> Annex v) -> NotifyWitness -> Annex v +download u key f d a _witness = guardHaveUUID u $ + runTransfer (Transfer Download u key) f d a + +guardHaveUUID :: Observable v => UUID -> Annex v -> Annex v +guardHaveUUID u a + | u == NoUUID = return observeFailure + | otherwise = a + +{- Runs a transfer action. Creates and locks the lock file while the + - action is running, and stores info in the transfer information + - file. + - + - If the transfer action returns False, the transfer info is + - left in the failedTransferDir. + - + - If the transfer is already in progress, returns False. + - + - An upload can be run from a read-only filesystem, and in this case + - no transfer information or lock file is used. + -} +runTransfer :: Observable v => Transfer -> AssociatedFile -> RetryDecider -> (MeterUpdate -> Annex v) -> Annex v +runTransfer = runTransfer' False + +{- Like runTransfer, but ignores any existing transfer lock file for the + - transfer, allowing re-running a transfer that is already in progress. + - + - Note that this may result in confusing progress meter display in the + - webapp, if multiple processes are writing to the transfer info file. -} +alwaysRunTransfer :: Observable v => Transfer -> AssociatedFile -> RetryDecider -> (MeterUpdate -> Annex v) -> Annex v +alwaysRunTransfer = runTransfer' True + +runTransfer' :: Observable v => Bool -> Transfer -> AssociatedFile -> RetryDecider -> (MeterUpdate -> Annex v) -> Annex v +runTransfer' ignorelock t afile retrydecider transferaction = checkSecureHashes t $ do + shouldretry <- retrydecider + info <- liftIO $ startTransferInfo afile + (meter, tfile, createtfile, metervar) <- mkProgressUpdater t info + mode <- annexFileMode + (lck, inprogress) <- prep tfile createtfile mode + if inprogress && not ignorelock + then do + showNote "transfer already in progress, or unable to take transfer lock" + return observeFailure + else do + v <- retry shouldretry info metervar $ transferaction meter + liftIO $ cleanup tfile lck + if observeBool v + then removeFailedTransfer t + else recordFailedTransfer t info + return v + where + prep :: FilePath -> Annex () -> FileMode -> Annex (Maybe LockHandle, Bool) +#ifndef mingw32_HOST_OS + prep tfile createtfile mode = catchPermissionDenied (const prepfailed) $ do + let lck = transferLockFile tfile + createAnnexDirectory $ takeDirectory lck + tryLockExclusive (Just mode) lck >>= \case + Nothing -> return (Nothing, True) + Just lockhandle -> ifM (checkSaneLock lck lockhandle) + ( do + createtfile + return (Just lockhandle, False) + , do + liftIO $ dropLock lockhandle + return (Nothing, True) + ) +#else + prep tfile createtfile _mode = catchPermissionDenied (const prepfailed) $ do + let lck = transferLockFile tfile + createAnnexDirectory $ takeDirectory lck + catchMaybeIO (liftIO $ lockExclusive lck) >>= \case + Nothing -> return (Nothing, False) + Just Nothing -> return (Nothing, True) + Just (Just lockhandle) -> do + createtfile + return (Just lockhandle, False) +#endif + prepfailed = return (Nothing, False) + + cleanup _ Nothing = noop + cleanup tfile (Just lockhandle) = do + let lck = transferLockFile tfile + void $ tryIO $ removeFile tfile +#ifndef mingw32_HOST_OS + void $ tryIO $ removeFile lck + dropLock lockhandle +#else + {- Windows cannot delete the lockfile until the lock + - is closed. So it's possible to race with another + - process that takes the lock before it's removed, + - so ignore failure to remove. + -} + dropLock lockhandle + void $ tryIO $ removeFile lck +#endif + retry shouldretry oldinfo metervar run = tryNonAsync run >>= \case + Right v + | observeBool v -> return v + | otherwise -> checkretry + Left e -> do + warning (show e) + checkretry + where + checkretry = do + b <- getbytescomplete metervar + let newinfo = oldinfo { bytesComplete = Just b } + ifM (shouldretry oldinfo newinfo) + ( retry shouldretry newinfo metervar run + , return observeFailure + ) + getbytescomplete metervar + | transferDirection t == Upload = + liftIO $ readMVar metervar + | otherwise = do + f <- fromRepo $ gitAnnexTmpObjectLocation (transferKey t) + liftIO $ catchDefaultIO 0 $ getFileSize f + +{- Avoid download and upload of keys with insecure content when + - annex.securehashesonly is configured. + - + - This is not a security check. Even if this let the content be + - downloaded, the actual security checks would prevent the content from + - being added to the repository. The only reason this is done here is to + - avoid transferring content that's going to be rejected anyway. + - + - We assume that, if annex.securehashesonly is set and the local repo + - still contains content using an insecure hash, remotes will likewise + - tend to be configured to reject it, so Upload is also prevented. + -} +checkSecureHashes :: Observable v => Transfer -> Annex v -> Annex v +checkSecureHashes t a + | cryptographicallySecure variety = a + | otherwise = ifM (annexSecureHashesOnly <$> Annex.getGitConfig) + ( do + warning $ "annex.securehashesonly blocked transfer of " ++ formatKeyVariety variety ++ " key" + return observeFailure + , a + ) + where + variety = keyVariety (transferKey t) + +type RetryDecider = Annex (TransferInfo -> TransferInfo -> Annex Bool) + +{- The first RetryDecider will be checked first; only if it says not to + - retry will the second one be checked. -} +combineRetryDeciders :: RetryDecider -> RetryDecider -> RetryDecider +combineRetryDeciders a b = do + ar <- a + br <- b + return $ \old new -> ar old new <||> br old new + +noRetry :: RetryDecider +noRetry = pure $ \_ _ -> pure False + +stdRetry :: RetryDecider +stdRetry = combineRetryDeciders forwardRetry configuredRetry + +{- Retries a transfer when it fails, as long as the failed transfer managed + - to send some data. -} +forwardRetry :: RetryDecider +forwardRetry = pure $ \old new -> pure $ + fromMaybe 0 (bytesComplete old) < fromMaybe 0 (bytesComplete new) + +{- Retries a number of times with growing delays in between when enabled + - by git configuration. -} +configuredRetry :: RetryDecider +configuredRetry = do + retrycounter <- liftIO $ newMVar 0 + return $ \_old new -> do + (maxretries, Seconds initretrydelay) <- getcfg $ + Remote.gitconfig <$> transferRemote new + retries <- liftIO $ modifyMVar retrycounter $ + \n -> return (n + 1, n + 1) + if retries < maxretries + then do + let retrydelay = Seconds (initretrydelay * 2^(retries-1)) + showSideAction $ "Delaying " ++ show (fromSeconds retrydelay) ++ "s before retrying." + liftIO $ threadDelaySeconds retrydelay + return True + else return False + where + globalretrycfg = fromMaybe 0 . annexRetry + <$> Annex.getGitConfig + globalretrydelaycfg = fromMaybe (Seconds 1) . annexRetryDelay + <$> Annex.getGitConfig + getcfg Nothing = (,) <$> globalretrycfg <*> globalretrydelaycfg + getcfg (Just gc) = (,) + <$> maybe globalretrycfg return (remoteAnnexRetry gc) + <*> maybe globalretrydelaycfg return (remoteAnnexRetryDelay gc) + +{- Picks a remote from the list and tries a transfer to it. If the transfer + - does not succeed, goes on to try other remotes from the list. + - + - The list should already be ordered by remote cost, and is normally + - tried in order. However, when concurrent jobs are running, they will + - be assigned different remotes of the same cost when possible. This can + - increase total transfer speed. + -} +pickRemote :: Observable v => [Remote] -> (Remote -> Annex v) -> Annex v +pickRemote l a = go l =<< Annex.getState Annex.concurrency + where + go [] _ = return observeFailure + go (r:[]) _ = a r + go rs (Concurrent n) | n > 1 = do + mv <- Annex.getState Annex.activeremotes + active <- liftIO $ takeMVar mv + let rs' = sortBy (lessActiveFirst active) rs + goconcurrent mv active rs' + go (r:rs) _ = do + ok <- a r + if observeBool ok + then return ok + else go rs NonConcurrent + goconcurrent mv active [] = do + liftIO $ putMVar mv active + return observeFailure + goconcurrent mv active (r:rs) = do + let !active' = M.insertWith (+) r 1 active + liftIO $ putMVar mv active' + let getnewactive = do + active'' <- liftIO $ takeMVar mv + let !active''' = M.update (\n -> if n > 1 then Just (n-1) else Nothing) r active'' + return active''' + let removeactive = liftIO . putMVar mv =<< getnewactive + ok <- a r `onException` removeactive + if observeBool ok + then do + removeactive + return ok + else do + active'' <- getnewactive + -- Re-sort the remaining rs + -- because other threads could have + -- been assigned them in the meantime. + let rs' = sortBy (lessActiveFirst active'') rs + goconcurrent mv active'' rs' + +lessActiveFirst :: M.Map Remote Integer -> Remote -> Remote -> Ordering +lessActiveFirst active a b + | Remote.cost a == Remote.cost b = comparing (`M.lookup` active) a b + | otherwise = comparing Remote.cost a b diff --git a/Annex/UUID.hs b/Annex/UUID.hs new file mode 100644 index 0000000000..8a2d884277 --- /dev/null +++ b/Annex/UUID.hs @@ -0,0 +1,124 @@ +{- git-annex uuids + - + - Each git repository used by git-annex has an annex.uuid setting that + - uniquely identifies that repository. + - + - UUIDs of remotes are cached in git config, using keys named + - remote..annex-uuid + - + - Copyright 2010-2016 Joey Hess + - + - Licensed under the GNU GPL version 3 or higher. + -} + +module Annex.UUID ( + getUUID, + getRepoUUID, + getUncachedUUID, + isUUIDConfigured, + prepUUID, + genUUID, + genUUIDInNameSpace, + gCryptNameSpace, + removeRepoUUID, + storeUUID, + storeUUIDIn, + setUUID, + webUUID, + bitTorrentUUID, +) where + +import Annex.Common +import qualified Annex +import qualified Git +import qualified Git.Config +import Config + +import qualified Data.UUID as U +import qualified Data.UUID.V4 as U4 +import qualified Data.UUID.V5 as U5 +import Utility.FileSystemEncoding + +configkey :: ConfigKey +configkey = annexConfig "uuid" + +{- Generates a random UUID, that does not include the MAC address. -} +genUUID :: IO UUID +genUUID = UUID . show <$> U4.nextRandom + +{- Generates a UUID from a given string, using a namespace. + - Given the same namespace, the same string will always result + - in the same UUID. -} +genUUIDInNameSpace :: U.UUID -> String -> UUID +genUUIDInNameSpace namespace = UUID . show . U5.generateNamed namespace . s2w8 + +{- Namespace used for UUIDs derived from git-remote-gcrypt ids. -} +gCryptNameSpace :: U.UUID +gCryptNameSpace = U5.generateNamed U5.namespaceURL $ + s2w8 "http://git-annex.branchable.com/design/gcrypt/" + +{- Get current repository's UUID. -} +getUUID :: Annex UUID +getUUID = annexUUID <$> Annex.getGitConfig + +{- Looks up a remote repo's UUID, caching it in .git/config if + - it's not already. -} +getRepoUUID :: Git.Repo -> Annex UUID +getRepoUUID r = do + c <- toUUID <$> getConfig cachekey "" + let u = getUncachedUUID r + + if c /= u && u /= NoUUID + then do + updatecache u + return u + else return c + where + updatecache u = do + g <- gitRepo + when (g /= r) $ storeUUIDIn cachekey u + cachekey = remoteConfig r "uuid" + +removeRepoUUID :: Annex () +removeRepoUUID = do + unsetConfig configkey + storeUUID NoUUID + +getUncachedUUID :: Git.Repo -> UUID +getUncachedUUID = toUUID . Git.Config.get key "" + where + (ConfigKey key) = configkey + +-- Does the repo's config have a key for the UUID? +-- True even when the key has no value. +isUUIDConfigured :: Git.Repo -> Bool +isUUIDConfigured = isJust . Git.Config.getMaybe key + where + (ConfigKey key) = configkey + +{- Make sure that the repo has an annex.uuid setting. -} +prepUUID :: Annex () +prepUUID = whenM ((==) NoUUID <$> getUUID) $ + storeUUID =<< liftIO genUUID + +storeUUID :: UUID -> Annex () +storeUUID u = do + Annex.changeGitConfig $ \c -> c { annexUUID = u } + storeUUIDIn configkey u + +storeUUIDIn :: ConfigKey -> UUID -> Annex () +storeUUIDIn configfield = setConfig configfield . fromUUID + +{- Only sets the configkey in the Repo; does not change .git/config -} +setUUID :: Git.Repo -> UUID -> IO Git.Repo +setUUID r u = do + let s = show configkey ++ "=" ++ fromUUID u + Git.Config.store s r + +-- Dummy uuid for the whole web. Do not alter. +webUUID :: UUID +webUUID = UUID "00000000-0000-0000-0000-000000000001" + +-- Dummy uuid for bittorrent. Do not alter. +bitTorrentUUID :: UUID +bitTorrentUUID = UUID "00000000-0000-0000-0000-000000000002" diff --git a/Annex/UpdateInstead.hs b/Annex/UpdateInstead.hs new file mode 100644 index 0000000000..e31bcedaf6 --- /dev/null +++ b/Annex/UpdateInstead.hs @@ -0,0 +1,27 @@ +{- git-annex UpdateIntead emulation + - + - Copyright 2017 Joey Hess + - + - Licensed under the GNU GPL version 3 or higher. + -} + +module Annex.UpdateInstead where + +import qualified Annex +import Annex.Common +import Config +import Annex.Version +import Annex.AdjustedBranch +import Git.Branch +import Git.ConfigTypes + +{- receive.denyCurrentBranch=updateInstead does not work in direct mode + - repositories or when an adjusted branch is checked out, so must be + - emulated. -} +needUpdateInsteadEmulation :: Annex Bool +needUpdateInsteadEmulation = updateinsteadset <&&> (isDirect <||> isadjusted) + where + updateinsteadset = (== UpdateInstead) . receiveDenyCurrentBranch + <$> Annex.getGitConfig + isadjusted = versionSupportsUnlockedPointers + <&&> (maybe False (isJust . getAdjustment) <$> inRepo Git.Branch.current) diff --git a/Annex/Url.hs b/Annex/Url.hs new file mode 100644 index 0000000000..7338aecbd1 --- /dev/null +++ b/Annex/Url.hs @@ -0,0 +1,100 @@ +{- Url downloading, with git-annex user agent and configured http + - headers, security restrictions, etc. + - + - Copyright 2013-2018 Joey Hess + - + - Licensed under the GNU GPL version 3 or higher. + -} + +module Annex.Url ( + module U, + withUrlOptions, + getUrlOptions, + getUserAgent, + httpAddressesUnlimited, +) where + +import Annex.Common +import qualified Annex +import Utility.Url as U +import Utility.IPAddress +import Utility.HttpManagerRestricted +import qualified BuildInfo + +import Network.Socket + +defaultUserAgent :: U.UserAgent +defaultUserAgent = "git-annex/" ++ BuildInfo.packageversion + +getUserAgent :: Annex U.UserAgent +getUserAgent = Annex.getState $ + fromMaybe defaultUserAgent . Annex.useragent + +getUrlOptions :: Annex U.UrlOptions +getUrlOptions = Annex.getState Annex.urloptions >>= \case + Just uo -> return uo + Nothing -> do + uo <- mk + Annex.changeState $ \s -> s + { Annex.urloptions = Just uo } + return uo + where + mk = do + (urldownloader, manager) <- checkallowedaddr + mkUrlOptions + <$> (Just <$> getUserAgent) + <*> headers + <*> pure urldownloader + <*> pure manager + <*> (annexAllowedUrlSchemes <$> Annex.getGitConfig) + + headers = annexHttpHeadersCommand <$> Annex.getGitConfig >>= \case + Just cmd -> lines <$> liftIO (readProcess "sh" ["-c", cmd]) + Nothing -> annexHttpHeaders <$> Annex.getGitConfig + + checkallowedaddr = words . annexAllowedHttpAddresses <$> Annex.getGitConfig >>= \case + ["all"] -> do + -- Only allow curl when all are allowed, + -- as its interface does not allow preventing + -- it from accessing specific IP addresses. + curlopts <- map Param . annexWebOptions <$> Annex.getGitConfig + let urldownloader = if null curlopts + then U.DownloadWithCurl curlopts + else U.DownloadWithConduit + manager <- liftIO $ U.newManager U.managerSettings + return (urldownloader, manager) + allowedaddrs -> do + addrmatcher <- liftIO $ + (\l v -> any (\f -> f v) l) . catMaybes + <$> mapM makeAddressMatcher allowedaddrs + -- Default to not allowing access to loopback + -- and private IP addresses to avoid data + -- leakage. + let isallowed addr + | addrmatcher addr = True + | isLoopbackAddress addr = False + | isPrivateAddress addr = False + | otherwise = True + let connectionrestricted = addrConnectionRestricted + ("Configuration of annex.security.allowed-http-addresses does not allow accessing address " ++) + let r = Restriction + { addressRestriction = \addr -> + if isallowed (addrAddress addr) + then Nothing + else Just (connectionrestricted addr) + } + (settings, pr) <- liftIO $ + restrictManagerSettings r U.managerSettings + case pr of + Nothing -> return () + Just ProxyRestricted -> toplevelWarning True + "http proxy settings not used due to annex.security.allowed-http-addresses configuration" + manager <- liftIO $ U.newManager settings + return (U.DownloadWithConduit, manager) + +httpAddressesUnlimited :: Annex Bool +httpAddressesUnlimited = + ("all" == ) . annexAllowedHttpAddresses <$> Annex.getGitConfig + +withUrlOptions :: (U.UrlOptions -> Annex a) -> Annex a +withUrlOptions a = a =<< getUrlOptions diff --git a/Annex/VariantFile.hs b/Annex/VariantFile.hs new file mode 100644 index 0000000000..8365073d5f --- /dev/null +++ b/Annex/VariantFile.hs @@ -0,0 +1,44 @@ +{- git-annex .variant files for automatic merge conflict resolution + - + - Copyright 2014 Joey Hess + - + - Licensed under the GNU GPL version 3 or higher. + -} + +module Annex.VariantFile where + +import Annex.Common +import Utility.FileSystemEncoding +import Utility.Hash + +variantMarker :: String +variantMarker = ".variant-" + +mkVariant :: FilePath -> String -> FilePath +mkVariant file variant = takeDirectory file + dropExtension (takeFileName file) + ++ variantMarker ++ variant + ++ takeExtension file + +{- The filename to use when resolving a conflicted merge of a file, + - that points to a key. + - + - Something derived from the key needs to be included in the filename, + - but rather than exposing the whole key to the user, a very weak hash + - is used. There is a very real, although still unlikely, chance of + - conflicts using this hash. + - + - In the event that there is a conflict with the filename generated + - for some other key, that conflict will itself be handled by the + - conflicted merge resolution code. That case is detected, and the full + - key is used in the filename. + -} +variantFile :: FilePath -> Key -> FilePath +variantFile file key + | doubleconflict = mkVariant file (key2file key) + | otherwise = mkVariant file (shortHash $ key2file key) + where + doubleconflict = variantMarker `isInfixOf` file + +shortHash :: String -> String +shortHash = take 4 . show . md5 . encodeBS diff --git a/Annex/VectorClock.hs b/Annex/VectorClock.hs new file mode 100644 index 0000000000..c7f6d6336d --- /dev/null +++ b/Annex/VectorClock.hs @@ -0,0 +1,46 @@ +{- git-annex vector clocks + - + - We don't have a way yet to keep true distributed vector clocks. + - The next best thing is a timestamp. + - + - Copyright 2017 Joey Hess + - + - Licensed under the GNU GPL version 3 or higher. + -} + +module Annex.VectorClock where + +import Data.Time.Clock.POSIX +import Control.Applicative +import Prelude + +import Utility.Env +import Logs.TimeStamp +import Utility.QuickCheck + +-- | Some very old logs did not have any time stamp at all; +-- Unknown is used for those. +data VectorClock = Unknown | VectorClock POSIXTime + deriving (Eq, Ord) + +-- Unknown is oldest. +prop_VectorClock_sane :: Bool +prop_VectorClock_sane = Unknown < VectorClock 1 + +instance Arbitrary VectorClock where + arbitrary = VectorClock <$> arbitrary + +currentVectorClock :: IO VectorClock +currentVectorClock = go =<< getEnv "GIT_ANNEX_VECTOR_CLOCK" + where + go Nothing = VectorClock <$> getPOSIXTime + go (Just s) = case parsePOSIXTime s of + Just t -> return (VectorClock t) + Nothing -> VectorClock <$> getPOSIXTime + +formatVectorClock :: VectorClock -> String +formatVectorClock Unknown = "0" +formatVectorClock (VectorClock t) = show t + +parseVectorClock :: String -> Maybe VectorClock +parseVectorClock t = VectorClock <$> parsePOSIXTime t diff --git a/Annex/Version.hs b/Annex/Version.hs new file mode 100644 index 0000000000..6fdc2c703b --- /dev/null +++ b/Annex/Version.hs @@ -0,0 +1,68 @@ +{- git-annex repository versioning + - + - Copyright 2010,2013 Joey Hess + - + - Licensed under the GNU GPL version 3 or higher. + -} + +{-# LANGUAGE CPP #-} + +module Annex.Version where + +import Annex.Common +import Config +import qualified Annex + +type Version = String + +defaultVersion :: Version +defaultVersion = "5" + +latestVersion :: Version +latestVersion = "6" + +supportedVersions :: [Version] +supportedVersions = ["3", "5", "6"] + +versionForAdjustedClone :: Version +versionForAdjustedClone = "6" + +upgradableVersions :: [Version] +#ifndef mingw32_HOST_OS +upgradableVersions = ["0", "1", "2", "3", "4", "5"] +#else +upgradableVersions = ["2", "3", "4", "5"] +#endif + +autoUpgradeableVersions :: [Version] +autoUpgradeableVersions = ["3", "4"] + +versionField :: ConfigKey +versionField = annexConfig "version" + +getVersion :: Annex (Maybe Version) +getVersion = annexVersion <$> Annex.getGitConfig + +versionSupportsDirectMode :: Annex Bool +versionSupportsDirectMode = go <$> getVersion + where + go (Just "6") = False + go _ = True + +versionSupportsUnlockedPointers :: Annex Bool +versionSupportsUnlockedPointers = go <$> getVersion + where + go (Just "6") = True + go _ = False + +versionSupportsAdjustedBranch :: Annex Bool +versionSupportsAdjustedBranch = versionSupportsUnlockedPointers + +versionUsesKeysDatabase :: Annex Bool +versionUsesKeysDatabase = versionSupportsUnlockedPointers + +setVersion :: Version -> Annex () +setVersion = setConfig versionField + +removeVersion :: Annex () +removeVersion = unsetConfig versionField diff --git a/Annex/View.hs b/Annex/View.hs new file mode 100644 index 0000000000..22591fc962 --- /dev/null +++ b/Annex/View.hs @@ -0,0 +1,419 @@ +{- metadata based branch views + - + - Copyright 2014 Joey Hess + - + - Licensed under the GNU GPL version 3 or higher. + -} + +module Annex.View where + +import Annex.Common +import Annex.View.ViewedFile +import Types.View +import Types.MetaData +import Annex.MetaData +import qualified Git +import qualified Git.DiffTree as DiffTree +import qualified Git.Branch +import qualified Git.LsFiles +import qualified Git.Ref +import Git.UpdateIndex +import Git.Sha +import Git.Types +import Git.FilePath +import Annex.WorkTree +import Annex.GitOverlay +import Annex.Link +import Annex.CatFile +import Logs.MetaData +import Logs.View +import Utility.Glob +import Types.Command +import CmdLine.Action + +import qualified Data.Set as S +import qualified Data.Map as M +import "mtl" Control.Monad.Writer + +{- Each visible ViewFilter in a view results in another level of + - subdirectory nesting. When a file matches multiple ways, it will appear + - in multiple subdirectories. This means there is a bit of an exponential + - blowup with a single file appearing in a crazy number of places! + - + - Capping the view size to 5 is reasonable; why wants to dig + - through 5+ levels of subdirectories to find anything? + -} +viewTooLarge :: View -> Bool +viewTooLarge view = visibleViewSize view > 5 + +visibleViewSize :: View -> Int +visibleViewSize = length . filter viewVisible . viewComponents + +{- Parses field=value, field!=value, tag, and !tag + - + - Note that the field may not be a legal metadata field name, + - but it's let through anyway. + - This is useful when matching on directory names with spaces, + - which are not legal MetaFields. + -} +parseViewParam :: String -> (MetaField, ViewFilter) +parseViewParam s = case separate (== '=') s of + ('!':tag, []) | not (null tag) -> + ( tagMetaField + , mkExcludeValues tag + ) + (tag, []) -> + ( tagMetaField + , mkFilterValues tag + ) + (field, wanted) + | end field == "!" -> + ( mkMetaFieldUnchecked (beginning field) + , mkExcludeValues wanted + ) + | otherwise -> + ( mkMetaFieldUnchecked field + , mkFilterValues wanted + ) + where + mkFilterValues v + | any (`elem` v) "*?" = FilterGlob v + | otherwise = FilterValues $ S.singleton $ toMetaValue v + mkExcludeValues = ExcludeValues . S.singleton . toMetaValue + +data ViewChange = Unchanged | Narrowing | Widening + deriving (Ord, Eq, Show) + +{- Updates a view, adding new fields to filter on (Narrowing), + - or allowing new values in an existing field (Widening). -} +refineView :: View -> [(MetaField, ViewFilter)] -> (View, ViewChange) +refineView origview = checksize . calc Unchanged origview + where + calc c v [] = (v, c) + calc c v ((f, vf):rest) = + let (v', c') = refine v f vf + in calc (max c c') v' rest + + refine view field vf + | field `elem` map viewField (viewComponents view) = + let (components', viewchanges) = runWriter $ + mapM (\c -> updateViewComponent c field vf) (viewComponents view) + viewchange = if field `elem` map viewField (viewComponents origview) + then maximum viewchanges + else Narrowing + in (view { viewComponents = components' }, viewchange) + | otherwise = + let component = mkViewComponent field vf + view' = view { viewComponents = component : viewComponents view } + in (view', Narrowing) + + checksize r@(v, _) + | viewTooLarge v = giveup $ "View is too large (" ++ show (visibleViewSize v) ++ " levels of subdirectories)" + | otherwise = r + +updateViewComponent :: ViewComponent -> MetaField -> ViewFilter -> Writer [ViewChange] ViewComponent +updateViewComponent c field vf + | viewField c == field = do + let (newvf, viewchange) = combineViewFilter (viewFilter c) vf + tell [viewchange] + return $ mkViewComponent field newvf + | otherwise = return c + +{- Adds an additional filter to a view. This can only result in narrowing + - the view. Multivalued filters are added in non-visible form. -} +filterView :: View -> [(MetaField, ViewFilter)] -> View +filterView v vs = v { viewComponents = viewComponents f' ++ viewComponents v} + where + f = fst $ refineView (v {viewComponents = []}) vs + f' = f { viewComponents = map toinvisible (viewComponents f) } + toinvisible c = c { viewVisible = False } + +{- Combine old and new ViewFilters, yielding a result that matches + - either old+new, or only new. + - + - If we have FilterValues and change to a FilterGlob, + - it's always a widening change, because the glob could match other + - values. OTOH, going the other way, it's a Narrowing change if the old + - glob matches all the new FilterValues. + - + - With two globs, the old one is discarded, and the new one is used. + - We can tell if that's a narrowing change by checking if the old + - glob matches the new glob. For example, "*" matches "foo*", + - so that's narrowing. While "f?o" does not match "f??", so that's + - widening. + -} +combineViewFilter :: ViewFilter -> ViewFilter -> (ViewFilter, ViewChange) +combineViewFilter old@(FilterValues olds) (FilterValues news) + | combined == old = (combined, Unchanged) + | otherwise = (combined, Widening) + where + combined = FilterValues (S.union olds news) +combineViewFilter old@(ExcludeValues olds) (ExcludeValues news) + | combined == old = (combined, Unchanged) + | otherwise = (combined, Narrowing) + where + combined = ExcludeValues (S.union olds news) +combineViewFilter (FilterValues _) newglob@(FilterGlob _) = + (newglob, Widening) +combineViewFilter (FilterGlob oldglob) new@(FilterValues s) + | all (matchGlob (compileGlob oldglob CaseInsensative) . fromMetaValue) (S.toList s) = (new, Narrowing) + | otherwise = (new, Widening) +combineViewFilter (FilterGlob old) newglob@(FilterGlob new) + | old == new = (newglob, Unchanged) + | matchGlob (compileGlob old CaseInsensative) new = (newglob, Narrowing) + | otherwise = (newglob, Widening) +combineViewFilter (FilterGlob _) new@(ExcludeValues _) = (new, Narrowing) +combineViewFilter (ExcludeValues _) new@(FilterGlob _) = (new, Widening) +combineViewFilter (FilterValues _) new@(ExcludeValues _) = (new, Narrowing) +combineViewFilter (ExcludeValues _) new@(FilterValues _) = (new, Widening) + +{- Generates views for a file from a branch, based on its metadata + - and the filename used in the branch. + - + - Note that a file may appear multiple times in a view, when it + - has multiple matching values for a MetaField used in the View. + - + - Of course if its MetaData does not match the View, it won't appear at + - all. + - + - Note that for efficiency, it's useful to partially + - evaluate this function with the view parameter and reuse + - the result. The globs in the view will then be compiled and memoized. + -} +viewedFiles :: View -> MkViewedFile -> FilePath -> MetaData -> [ViewedFile] +viewedFiles view = + let matchers = map viewComponentMatcher (viewComponents view) + in \mkviewedfile file metadata -> + let matches = map (\m -> m metadata) matchers + in if any isNothing matches + then [] + else + let paths = pathProduct $ + map (map toViewPath) (visible matches) + in if null paths + then [mkviewedfile file] + else map ( mkviewedfile file) paths + where + visible = map (fromJust . snd) . + filter (viewVisible . fst) . + zip (viewComponents view) + +{- Checks if metadata matches a ViewComponent filter, and if so + - returns the value, or values that match. Self-memoizing on ViewComponent. -} +viewComponentMatcher :: ViewComponent -> (MetaData -> Maybe [MetaValue]) +viewComponentMatcher viewcomponent = \metadata -> + matcher (currentMetaDataValues metafield metadata) + where + metafield = viewField viewcomponent + matcher = case viewFilter viewcomponent of + FilterValues s -> \values -> setmatches $ + S.intersection s values + FilterGlob glob -> + let cglob = compileGlob glob CaseInsensative + in \values -> setmatches $ + S.filter (matchGlob cglob . fromMetaValue) values + ExcludeValues excludes -> \values -> + if S.null (S.intersection values excludes) + then Just [] + else Nothing + setmatches s + | S.null s = Nothing + | otherwise = Just (S.toList s) + +-- This is '∕', a unicode character that displays the same as '/' but is +-- not it. It is encoded using the filesystem encoding, which allows it +-- to be used even when not in a unicode capable locale. +pseudoSlash :: String +pseudoSlash = "\56546\56456\56469" + +-- And this is '╲' similarly. +pseudoBackslash :: String +pseudoBackslash = "\56546\56469\56498" + +toViewPath :: MetaValue -> FilePath +toViewPath = escapeslash [] . fromMetaValue + where + escapeslash s ('/':cs) = escapeslash (pseudoSlash:s) cs + escapeslash s ('\\':cs) = escapeslash (pseudoBackslash:s) cs + escapeslash s ('%':cs) = escapeslash ("%%":s) cs + escapeslash s (c1:c2:c3:cs) + | [c1,c2,c3] == pseudoSlash = escapeslash ("%":pseudoSlash:s) cs + | [c1,c2,c3] == pseudoBackslash = escapeslash ("%":pseudoBackslash:s) cs + | otherwise = escapeslash ([c1]:s) (c2:c3:cs) + escapeslash s cs = concat (reverse (cs:s)) + +fromViewPath :: FilePath -> MetaValue +fromViewPath = toMetaValue . deescapeslash [] + where + deescapeslash s ('%':escapedc:cs) = deescapeslash ([escapedc]:s) cs + deescapeslash s (c1:c2:c3:cs) + | [c1,c2,c3] == pseudoSlash = deescapeslash ("/":s) cs + | [c1,c2,c3] == pseudoBackslash = deescapeslash ("\\":s) cs + | otherwise = deescapeslash ([c1]:s) (c2:c3:cs) + deescapeslash s cs = concat (reverse (cs:s)) + +prop_viewPath_roundtrips :: MetaValue -> Bool +prop_viewPath_roundtrips v = fromViewPath (toViewPath v) == v + +pathProduct :: [[FilePath]] -> [FilePath] +pathProduct [] = [] +pathProduct (l:ls) = foldl combinel l ls + where + combinel xs ys = [combine x y | x <- xs, y <- ys] + +{- Extracts the metadata from a ViewedFile, based on the view that was used + - to construct it. + - + - Derived metadata is excluded. + -} +fromView :: View -> ViewedFile -> MetaData +fromView view f = MetaData $ + M.fromList (zip fields values) `M.difference` derived + where + visible = filter viewVisible (viewComponents view) + fields = map viewField visible + paths = splitDirectories (dropFileName f) + values = map (S.singleton . fromViewPath) paths + MetaData derived = getViewedFileMetaData f + +{- Constructing a view that will match arbitrary metadata, and applying + - it to a file yields a set of ViewedFile which all contain the same + - MetaFields that were present in the input metadata + - (excluding fields that are not visible). -} +prop_view_roundtrips :: FilePath -> MetaData -> Bool -> Bool +prop_view_roundtrips f metadata visible = null f || viewTooLarge view || + all hasfields (viewedFiles view viewedFileFromReference f metadata) + where + view = View (Git.Ref "master") $ + map (\(mf, mv) -> ViewComponent mf (FilterValues $ S.filter (not . null . fromMetaValue) mv) visible) + (fromMetaData metadata) + visiblefields = sort (map viewField $ filter viewVisible (viewComponents view)) + hasfields fv = sort (map fst (fromMetaData (fromView view fv))) == visiblefields + +{- A directory foo/bar/baz/ is turned into metadata fields + - /=foo, foo/=bar, foo/bar/=baz. + - + - Note that this may generate MetaFields that legalField rejects. + - This is necessary to have a 1:1 mapping between directory names and + - fields. So this MetaData cannot safely be serialized. -} +getDirMetaData :: FilePath -> MetaData +getDirMetaData d = MetaData $ M.fromList $ zip fields values + where + dirs = splitDirectories d + fields = map (mkMetaFieldUnchecked . addTrailingPathSeparator . joinPath) + (inits dirs) + values = map (S.singleton . toMetaValue . fromMaybe "" . headMaybe) + (tails dirs) + +getWorkTreeMetaData :: FilePath -> MetaData +getWorkTreeMetaData = getDirMetaData . dropFileName + +getViewedFileMetaData :: FilePath -> MetaData +getViewedFileMetaData = getDirMetaData . dirFromViewedFile . takeFileName + +{- Applies a view to the currently checked out branch, generating a new + - branch for the view. + -} +applyView :: View -> Annex Git.Branch +applyView = applyView' viewedFileFromReference getWorkTreeMetaData + +{- Generates a new branch for a View, which must be a more narrow + - version of the View originally used to generate the currently + - checked out branch. That is, it must match a subset of the files + - in view, not any others. + -} +narrowView :: View -> Annex Git.Branch +narrowView = applyView' viewedFileReuse getViewedFileMetaData + +{- Go through each staged file. + - If the file is not annexed, skip it, unless it's a dotfile in the top, + - or a file in a dotdir in the top. + - Look up the metadata of annexed files, and generate any ViewedFiles, + - and stage them. + - + - Must be run from top of repository. + -} +applyView' :: MkViewedFile -> (FilePath -> MetaData) -> View -> Annex Git.Branch +applyView' mkviewedfile getfilemetadata view = do + top <- fromRepo Git.repoPath + (l, clean) <- inRepo $ Git.LsFiles.stagedDetails [top] + liftIO . nukeFile =<< fromRepo gitAnnexViewIndex + uh <- withViewIndex $ inRepo Git.UpdateIndex.startUpdateIndex + forM_ l $ \(f, sha, mode) -> do + topf <- inRepo (toTopFilePath f) + go uh topf sha (toTreeItemType =<< mode) =<< lookupFile f + liftIO $ do + void $ stopUpdateIndex uh + void clean + genViewBranch view + where + genviewedfiles = viewedFiles view mkviewedfile -- enables memoization + + go uh topf _sha _mode (Just k) = do + metadata <- getCurrentMetaData k + let f = getTopFilePath topf + let metadata' = getfilemetadata f `unionMetaData` metadata + forM_ (genviewedfiles f metadata') $ \fv -> do + f' <- fromRepo $ fromTopFilePath $ asTopFilePath fv + stagesymlink uh f' =<< calcRepo (gitAnnexLink f' k) + go uh topf (Just sha) (Just treeitemtype) Nothing + | "." `isPrefixOf` getTopFilePath topf = + liftIO $ Git.UpdateIndex.streamUpdateIndex' uh $ + pureStreamer $ updateIndexLine sha treeitemtype topf + go _ _ _ _ _ = noop + + stagesymlink uh f linktarget = do + sha <- hashSymlink linktarget + liftIO . Git.UpdateIndex.streamUpdateIndex' uh + =<< inRepo (Git.UpdateIndex.stageSymlink f sha) + +{- Diff between currently checked out branch and staged changes, and + - update metadata to reflect the changes that are being committed to the + - view. + - + - Adding a file to a directory adds the metadata represented by + - that directory to the file, and removing a file from a directory + - removes the metadata. + - + - Note that removes must be handled before adds. This is so + - that moving a file from x/foo/ to x/bar/ adds back the metadata for x. + -} +withViewChanges :: (ViewedFile -> Key -> CommandStart) -> (ViewedFile -> Key -> CommandStart) -> Annex () +withViewChanges addmeta removemeta = do + (diffs, cleanup) <- inRepo $ DiffTree.diffIndex Git.Ref.headRef + forM_ diffs handleremovals + forM_ diffs handleadds + void $ liftIO cleanup + where + handleremovals item + | DiffTree.srcsha item /= nullSha = + handlechange item removemeta + =<< catKey (DiffTree.srcsha item) + | otherwise = noop + handleadds item + | DiffTree.dstsha item /= nullSha = + handlechange item addmeta + =<< catKey (DiffTree.dstsha item) + | otherwise = noop + handlechange item a = maybe noop + (void . commandAction . a (getTopFilePath $ DiffTree.file item)) + +{- Runs an action using the view index file. + - Note that the file does not necessarily exist, or can contain + - info staged for an old view. -} +withViewIndex :: Annex a -> Annex a +withViewIndex a = do + f <- fromRepo gitAnnexViewIndex + withIndexFile f a + +{- Generates a branch for a view, using the view index file + - to make a commit to the view branch. The view branch is not + - checked out, but entering it will display the view. -} +genViewBranch :: View -> Annex Git.Branch +genViewBranch view = withViewIndex $ do + let branch = branchView view + void $ inRepo $ Git.Branch.commit Git.Branch.AutomaticCommit True (fromRef branch) branch [] + return branch + +withCurrentView :: (View -> Annex a) -> Annex a +withCurrentView a = maybe (giveup "Not in a view.") a =<< currentView diff --git a/Annex/View/ViewedFile.hs b/Annex/View/ViewedFile.hs new file mode 100644 index 0000000000..4bd11f756e --- /dev/null +++ b/Annex/View/ViewedFile.hs @@ -0,0 +1,86 @@ +{- filenames (not paths) used in views + - + - Copyright 2014 Joey Hess + - + - Licensed under the GNU GPL version 3 or higher. + -} + +{-# LANGUAGE CPP #-} + +module Annex.View.ViewedFile ( + ViewedFile, + MkViewedFile, + viewedFileFromReference, + viewedFileReuse, + dirFromViewedFile, + prop_viewedFile_roundtrips, +) where + +import Annex.Common + +type FileName = String +type ViewedFile = FileName + +type MkViewedFile = FilePath -> ViewedFile + +{- Converts a filepath used in a reference branch to the + - filename that will be used in the view. + - + - No two filepaths from the same branch should yeild the same result, + - so all directory structure needs to be included in the output filename + - in some way. + - + - So, from dir/subdir/file.foo, generate file_%dir%subdir%.foo + -} +viewedFileFromReference :: MkViewedFile +viewedFileFromReference f = concat + [ escape base + , if null dirs then "" else "_%" ++ intercalate "%" (map escape dirs) ++ "%" + , escape $ concat extensions + ] + where + (path, basefile) = splitFileName f + dirs = filter (/= ".") $ map dropTrailingPathSeparator (splitPath path) + (base, extensions) = splitShortExtensions basefile + + {- To avoid collisions with filenames or directories that contain + - '%', and to allow the original directories to be extracted + - from the ViewedFile, '%' is escaped. ) + -} + escape :: String -> String + escape = replace "%" (escchar:'%':[]) . replace [escchar] [escchar, escchar] + +escchar :: Char +#ifndef mingw32_HOST_OS +escchar = '\\' +#else +-- \ is path separator on Windows, so instead use ! +escchar = '!' +#endif + +{- For use when operating already within a view, so whatever filepath + - is present in the work tree is already a ViewedFile. -} +viewedFileReuse :: MkViewedFile +viewedFileReuse = takeFileName + +{- Extracts from a ViewedFile the directory where the file is located on + - in the reference branch. -} +dirFromViewedFile :: ViewedFile -> FilePath +dirFromViewedFile = joinPath . drop 1 . sep [] "" + where + sep l _ [] = reverse l + sep l curr (c:cs) + | c == '%' = sep (reverse curr:l) "" cs + | c == escchar = case cs of + (c':cs') -> sep l (c':curr) cs' + [] -> sep l curr cs + | otherwise = sep l (c:curr) cs + +prop_viewedFile_roundtrips :: FilePath -> Bool +prop_viewedFile_roundtrips f + -- Relative filenames wanted, not directories. + | any (isPathSeparator) (end f ++ beginning f) = True + | isAbsolute f = True + | otherwise = dir == dirFromViewedFile (viewedFileFromReference f) + where + dir = joinPath $ beginning $ splitDirectories f diff --git a/Annex/Wanted.hs b/Annex/Wanted.hs new file mode 100644 index 0000000000..d226483a13 --- /dev/null +++ b/Annex/Wanted.hs @@ -0,0 +1,29 @@ +{- git-annex checking whether content is wanted + - + - Copyright 2012 Joey Hess + - + - Licensed under the GNU GPL version 3 or higher. + -} + +module Annex.Wanted where + +import Annex.Common +import Logs.PreferredContent +import Annex.UUID + +import qualified Data.Set as S + +{- Check if a file is preferred content for the local repository. -} +wantGet :: Bool -> Maybe Key -> AssociatedFile -> Annex Bool +wantGet d key file = isPreferredContent Nothing S.empty key file d + +{- Check if a file is preferred content for a remote. -} +wantSend :: Bool -> Maybe Key -> AssociatedFile -> UUID -> Annex Bool +wantSend d key file to = isPreferredContent (Just to) S.empty key file d + +{- Check if a file can be dropped, maybe from a remote. + - Don't drop files that are preferred content. -} +wantDrop :: Bool -> Maybe UUID -> Maybe Key -> AssociatedFile -> Annex Bool +wantDrop d from key file = do + u <- maybe getUUID (return . id) from + not <$> isPreferredContent (Just u) (S.singleton u) key file d diff --git a/Annex/WorkTree.hs b/Annex/WorkTree.hs new file mode 100644 index 0000000000..9301707987 --- /dev/null +++ b/Annex/WorkTree.hs @@ -0,0 +1,90 @@ +{- git-annex worktree files + - + - Copyright 2013-2016 Joey Hess + - + - Licensed under the GNU GPL version 3 or higher. + -} + +module Annex.WorkTree where + +import Annex.Common +import Annex.Link +import Annex.CatFile +import Annex.Version +import Annex.Content +import Annex.ReplaceFile +import Config +import Git.FilePath +import qualified Git.Ref +import qualified Git.Branch +import qualified Git.LsTree +import qualified Git.Types +import Database.Types +import qualified Database.Keys +import qualified Database.Keys.SQL + +{- Looks up the key corresponding to an annexed file in the work tree, + - by examining what the file links to. + - + - An unlocked file will not have a link on disk, so fall back to + - looking for a pointer to a key in git. + -} +lookupFile :: FilePath -> Annex (Maybe Key) +lookupFile file = isAnnexLink file >>= \case + Just key -> makeret key + Nothing -> ifM (versionSupportsUnlockedPointers <||> isDirect) + ( ifM (liftIO $ doesFileExist file) + ( maybe (return Nothing) makeret =<< catKeyFile file + , return Nothing + ) + , return Nothing + ) + where + makeret = return . Just + +{- Modifies an action to only act on files that are already annexed, + - and passes the key on to it. -} +whenAnnexed :: (FilePath -> Key -> Annex (Maybe a)) -> FilePath -> Annex (Maybe a) +whenAnnexed a file = ifAnnexed file (a file) (return Nothing) + +ifAnnexed :: FilePath -> (Key -> Annex a) -> Annex a -> Annex a +ifAnnexed file yes no = maybe no yes =<< lookupFile file + +{- Find all unlocked files and update the keys database for them. + - + - This is expensive, and so normally the associated files are updated + - incrementally when changes are noticed. So, this only needs to be done + - when initializing/upgrading a v6 mode repository. + - + - Also, the content for the unlocked file may already be present as + - an annex object. If so, make the unlocked file use that content. + -} +scanUnlockedFiles :: Annex () +scanUnlockedFiles = whenM (isJust <$> inRepo Git.Branch.current) $ do + showSideAction "scanning for unlocked files" + Database.Keys.runWriter $ + liftIO . Database.Keys.SQL.dropAllAssociatedFiles + (l, cleanup) <- inRepo $ Git.LsTree.lsTree Git.Ref.headRef + forM_ l $ \i -> + when (isregfile i) $ + maybe noop (add i) + =<< catKey (Git.LsTree.sha i) + liftIO $ void cleanup + where + isregfile i = case Git.Types.toTreeItemType (Git.LsTree.mode i) of + Just Git.Types.TreeFile -> True + Just Git.Types.TreeExecutable -> True + _ -> False + add i k = do + let tf = Git.LsTree.file i + Database.Keys.runWriter $ + liftIO . Database.Keys.SQL.addAssociatedFileFast (toIKey k) tf + whenM (inAnnex k) $ do + f <- fromRepo $ fromTopFilePath tf + destmode <- liftIO $ catchMaybeIO $ fileMode <$> getFileStatus f + replaceFile f $ \tmp -> + linkFromAnnex k tmp destmode >>= \case + LinkAnnexOk -> return () + LinkAnnexNoop -> return () + LinkAnnexFailed -> liftIO $ + writePointerFile tmp k destmode diff --git a/Annex/YoutubeDl.hs b/Annex/YoutubeDl.hs new file mode 100644 index 0000000000..28ba26138b --- /dev/null +++ b/Annex/YoutubeDl.hs @@ -0,0 +1,237 @@ +{- youtube-dl integration for git-annex + - + - Copyright 2017-2018 Joey Hess + - + - Licensed under the GNU GPL version 3 or higher. + -} + +module Annex.YoutubeDl ( + youtubeDl, + youtubeDlTo, + youtubeDlSupported, + youtubeDlCheck, + youtubeDlFileName, + youtubeDlFileNameHtmlOnly, +) where + +import Annex.Common +import qualified Annex +import Annex.Content +import Annex.Url +import Utility.Url (URLString) +import Utility.DiskFree +import Utility.HtmlDetect +import Utility.Process.Transcript +import Logs.Transfer + +import Network.URI +import Control.Concurrent.Async + +-- youtube-dl is can follow redirects to anywhere, including potentially +-- localhost or a private address. So, it's only allowed to download +-- content if the user has allowed access to all addresses. +youtubeDlAllowed :: Annex Bool +youtubeDlAllowed = httpAddressesUnlimited + +youtubeDlNotAllowedMessage :: String +youtubeDlNotAllowedMessage = unwords + [ "This url is supported by youtube-dl, but" + , "youtube-dl could potentially access any address, and the" + , "configuration of annex.security.allowed-http-addresses" + , "does not allow that. Not using youtube-dl." + ] + +-- Runs youtube-dl in a work directory, to download a single media file +-- from the url. Reutrns the path to the media file in the work directory. +-- +-- If youtube-dl fails without writing any files to the work directory, +-- or is not installed, returns Right Nothing. +-- +-- The work directory can contain files from a previous run of youtube-dl +-- and it will resume. It should not contain any other files though, +-- and youtube-dl needs to finish up with only one file in the directory +-- so we know which one it downloaded. +-- +-- (Note that we can't use --output to specifiy the file to download to, +-- due to ) +youtubeDl :: URLString -> FilePath -> Annex (Either String (Maybe FilePath)) +youtubeDl url workdir = ifM httpAddressesUnlimited + ( withUrlOptions $ youtubeDl' url workdir + , return $ Left youtubeDlNotAllowedMessage + ) + +youtubeDl' :: URLString -> FilePath -> UrlOptions -> Annex (Either String (Maybe FilePath)) +youtubeDl' url workdir uo + | supportedScheme uo url = ifM (liftIO $ inPath "youtube-dl") + ( runcmd >>= \case + Right True -> workdirfiles >>= \case + (f:[]) -> return (Right (Just f)) + [] -> return nofiles + fs -> return (toomanyfiles fs) + Right False -> workdirfiles >>= \case + [] -> return (Right Nothing) + _ -> return (Left "youtube-dl download is incomplete. Run the command again to resume.") + Left msg -> return (Left msg) + , return (Right Nothing) + ) + | otherwise = return (Right Nothing) + where + nofiles = Left "youtube-dl did not put any media in its work directory, perhaps it's been configured to store files somewhere else?" + toomanyfiles fs = Left $ "youtube-dl downloaded multiple media files; git-annex is only able to deal with one per url: " ++ show fs + workdirfiles = liftIO $ filterM (doesFileExist) =<< dirContents workdir + runcmd = youtubeDlMaxSize workdir >>= \case + Left msg -> return (Left msg) + Right maxsize -> do + quiet <- commandProgressDisabled + opts <- youtubeDlOpts $ dlopts ++ maxsize ++ + if quiet then [ Param "--quiet" ] else [] + ok <- liftIO $ boolSystem' "youtube-dl" opts $ + \p -> p { cwd = Just workdir } + return (Right ok) + dlopts = + [ Param url + -- To make youtube-dl only download one file when given a + -- page with a video and a playlist, download only the video. + , Param "--no-playlist" + -- And when given a page with only a playlist, download only + -- the first video on the playlist. (Assumes the video is + -- somewhat stable, but this is the only way to prevent + -- youtube-dl from downloading the whole playlist.) + , Param "--playlist-items", Param "0" + ] + +-- To honor annex.diskreserve, ask youtube-dl to not download too +-- large a media file. Factors in other downloads that are in progress, +-- and any files in the workdir that it may have partially downloaded +-- before. +youtubeDlMaxSize :: FilePath -> Annex (Either String [CommandParam]) +youtubeDlMaxSize workdir = ifM (Annex.getState Annex.force) + ( return $ Right [] + , liftIO (getDiskFree workdir) >>= \case + Just have -> do + inprogress <- sizeOfDownloadsInProgress (const True) + partial <- liftIO $ sum + <$> (mapM getFileSize =<< dirContents workdir) + reserve <- annexDiskReserve <$> Annex.getGitConfig + let maxsize = have - reserve - inprogress + partial + if maxsize > 0 + then return $ Right + [ Param "--max-filesize" + , Param (show maxsize) + ] + else return $ Left $ + needMoreDiskSpace $ + negate maxsize + 1024 + Nothing -> return $ Right [] + ) + +-- Download a media file to a destination, +youtubeDlTo :: Key -> URLString -> FilePath -> Annex Bool +youtubeDlTo key url dest = do + res <- withTmpWorkDir key $ \workdir -> + youtubeDl url workdir >>= \case + Right (Just mediafile) -> do + liftIO $ renameFile mediafile dest + return (Just True) + Right Nothing -> return (Just False) + Left msg -> do + warning msg + return Nothing + return (fromMaybe False res) + +-- youtube-dl supports downloading urls that are not html pages, +-- but we don't want to use it for such urls, since they can be downloaded +-- without it. So, this first downloads part of the content and checks +-- if it's a html page; only then is youtube-dl used. +htmlOnly :: URLString -> a -> Annex a -> Annex a +htmlOnly url fallback a = withUrlOptions $ \uo -> + liftIO (downloadPartial url uo htmlPrefixLength) >>= \case + Just bs | isHtmlBs bs -> a + _ -> return fallback + +-- Check if youtube-dl supports downloading content from an url. +youtubeDlSupported :: URLString -> Annex Bool +youtubeDlSupported url = either (const False) id + <$> withUrlOptions (youtubeDlCheck' url) + +-- Check if youtube-dl can find media in an url. +-- +-- While this does not download anything, it checks youtubeDlAllowed +-- for symmetry with youtubeDl; the check should not succeed if the +-- download won't succeed. +youtubeDlCheck :: URLString -> Annex (Either String Bool) +youtubeDlCheck url = ifM youtubeDlAllowed + ( withUrlOptions $ youtubeDlCheck' url + , return $ Left youtubeDlNotAllowedMessage + ) + +youtubeDlCheck' :: URLString -> UrlOptions -> Annex (Either String Bool) +youtubeDlCheck' url uo + | supportedScheme uo url = catchMsgIO $ htmlOnly url False $ do + opts <- youtubeDlOpts [ Param url, Param "--simulate" ] + liftIO $ snd <$> processTranscript "youtube-dl" (toCommand opts) Nothing + | otherwise = return (Right False) + +-- Ask youtube-dl for the filename of media in an url. +-- +-- (This is not always identical to the filename it uses when downloading.) +youtubeDlFileName :: URLString -> Annex (Either String FilePath) +youtubeDlFileName url = withUrlOptions go + where + go uo + | supportedScheme uo url = flip catchIO (pure . Left . show) $ + htmlOnly url nomedia (youtubeDlFileNameHtmlOnly' url uo) + | otherwise = return nomedia + nomedia = Left "no media in url" + +-- Does not check if the url contains htmlOnly; use when that's already +-- been verified. +youtubeDlFileNameHtmlOnly :: URLString -> Annex (Either String FilePath) +youtubeDlFileNameHtmlOnly = withUrlOptions . youtubeDlFileNameHtmlOnly' + +youtubeDlFileNameHtmlOnly' :: URLString -> UrlOptions -> Annex (Either String FilePath) +youtubeDlFileNameHtmlOnly' url uo + | supportedScheme uo url = flip catchIO (pure . Left . show) go + | otherwise = return nomedia + where + go = do + -- Sometimes youtube-dl will fail with an ugly backtrace + -- (eg, http://bugs.debian.org/874321) + -- so catch stderr as well as stdout to avoid the user + -- seeing it. --no-warnings avoids warning messages that + -- are output to stdout. + opts <- youtubeDlOpts + [ Param url + , Param "--get-filename" + , Param "--no-warnings" + ] + (Nothing, Just o, Just e, pid) <- liftIO $ createProcess + (proc "youtube-dl" (toCommand opts)) + { std_out = CreatePipe + , std_err = CreatePipe + } + output <- liftIO $ fmap fst $ + hGetContentsStrict o + `concurrently` + hGetContentsStrict e + ok <- liftIO $ checkSuccessProcess pid + return $ case (ok, lines output) of + (True, (f:_)) | not (null f) -> Right f + _ -> nomedia + nomedia = Left "no media in url" + +youtubeDlOpts :: [CommandParam] -> Annex [CommandParam] +youtubeDlOpts addopts = do + opts <- map Param . annexYoutubeDlOptions <$> Annex.getGitConfig + return (opts ++ addopts) + +supportedScheme :: UrlOptions -> URLString -> Bool +supportedScheme uo url = case parseURIRelaxed url of + Nothing -> False + Just u -> case uriScheme u of + -- avoid ugly message from youtube-dl about not supporting file: + "file:" -> False + -- ftp indexes may look like html pages, and there's no point + -- involving youtube-dl in a ftp download + "ftp:" -> False + _ -> allowedScheme uo u diff --git a/Assistant.hs b/Assistant.hs new file mode 100644 index 0000000000..dc358c2bd4 --- /dev/null +++ b/Assistant.hs @@ -0,0 +1,191 @@ +{- git-annex assistant daemon + - + - Copyright 2012-2013 Joey Hess + - + - Licensed under the GNU GPL version 3 or higher. + -} + +{-# LANGUAGE CPP #-} + +module Assistant where + +import qualified Annex +import Assistant.Common +import Assistant.DaemonStatus +import Assistant.NamedThread +import Assistant.Types.ThreadedMonad +import Assistant.Threads.DaemonStatus +import Assistant.Threads.Watcher +import Assistant.Threads.Committer +import Assistant.Threads.Pusher +import Assistant.Threads.Exporter +import Assistant.Threads.Merger +import Assistant.Threads.TransferWatcher +import Assistant.Threads.Transferrer +import Assistant.Threads.RemoteControl +import Assistant.Threads.SanityChecker +import Assistant.Threads.Cronner +import Assistant.Threads.ProblemFixer +#ifndef mingw32_HOST_OS +import Assistant.Threads.MountWatcher +#endif +import Assistant.Threads.NetWatcher +import Assistant.Threads.Upgrader +import Assistant.Threads.UpgradeWatcher +import Assistant.Threads.TransferScanner +import Assistant.Threads.TransferPoller +import Assistant.Threads.ConfigMonitor +import Assistant.Threads.Glacier +#ifdef WITH_WEBAPP +import Assistant.WebApp +import Assistant.Threads.WebApp +#ifdef WITH_PAIRING +import Assistant.Threads.PairListener +#endif +#else +import Assistant.Types.UrlRenderer +#endif +import qualified Utility.Daemon +import Utility.ThreadScheduler +import Utility.HumanTime +import qualified BuildInfo +import Annex.Perms +import Utility.LogFile +#ifdef mingw32_HOST_OS +import Utility.Env +import Annex.Path +import System.Environment (getArgs) +#endif + +import System.Log.Logger +import Network.Socket (HostName) + +stopDaemon :: Annex () +stopDaemon = liftIO . Utility.Daemon.stopDaemon =<< fromRepo gitAnnexPidFile + +{- Starts the daemon. If the daemon is run in the foreground, once it's + - running, can start the browser. + - + - startbrowser is passed the url and html shim file, as well as the original + - stdout and stderr descriptors. -} +startDaemon :: Bool -> Bool -> Maybe Duration -> Maybe String -> Maybe HostName -> Maybe (Maybe Handle -> Maybe Handle -> String -> FilePath -> IO ()) -> Annex () +startDaemon assistant foreground startdelay cannotrun listenhost startbrowser = do + + Annex.changeState $ \s -> s { Annex.daemon = True } + pidfile <- fromRepo gitAnnexPidFile + logfile <- fromRepo gitAnnexLogFile + liftIO $ debugM desc $ "logging to " ++ logfile +#ifndef mingw32_HOST_OS + createAnnexDirectory (parentDir logfile) + logfd <- liftIO $ handleToFd =<< openLog logfile + if foreground + then do + origout <- liftIO $ catchMaybeIO $ + fdToHandle =<< dup stdOutput + origerr <- liftIO $ catchMaybeIO $ + fdToHandle =<< dup stdError + let undaemonize = Utility.Daemon.foreground logfd (Just pidfile) + start undaemonize $ + case startbrowser of + Nothing -> Nothing + Just a -> Just $ a origout origerr + else + start (Utility.Daemon.daemonize logfd (Just pidfile) False) Nothing +#else + -- Windows doesn't daemonize, but does redirect output to the + -- log file. The only way to do so is to restart the program. + when (foreground || not foreground) $ do + let flag = "GIT_ANNEX_OUTPUT_REDIR" + createAnnexDirectory (parentDir logfile) + ifM (liftIO $ isNothing <$> getEnv flag) + ( liftIO $ withFile devNull WriteMode $ \nullh -> do + loghandle <- openLog logfile + e <- getEnvironment + cmd <- programPath + ps <- getArgs + (_, _, _, pid) <- createProcess (proc cmd ps) + { env = Just (addEntry flag "1" e) + , std_in = UseHandle nullh + , std_out = UseHandle loghandle + , std_err = UseHandle loghandle + } + exitWith =<< waitForProcess pid + , start (Utility.Daemon.foreground (Just pidfile)) $ + case startbrowser of + Nothing -> Nothing + Just a -> Just $ a Nothing Nothing + ) +#endif + where + desc + | assistant = "assistant" + | otherwise = "watch" + start daemonize webappwaiter = withThreadState $ \st -> do + checkCanWatch + dstatus <- startDaemonStatus + logfile <- fromRepo gitAnnexLogFile + liftIO $ debugM desc $ "logging to " ++ logfile + liftIO $ daemonize $ + flip runAssistant (go webappwaiter) + =<< newAssistantData st dstatus + +#ifdef WITH_WEBAPP + go webappwaiter = do + d <- getAssistant id +#else + go _webappwaiter = do +#endif + notice ["starting", desc, "version", BuildInfo.packageversion] + urlrenderer <- liftIO newUrlRenderer +#ifdef WITH_WEBAPP + let webappthread = [ assist $ webAppThread d urlrenderer False cannotrun Nothing listenhost webappwaiter ] +#else + let webappthread = [] +#endif + let threads = if isJust cannotrun + then webappthread + else webappthread ++ + [ watch commitThread +#ifdef WITH_WEBAPP +#ifdef WITH_PAIRING + , assist $ pairListenerThread urlrenderer +#endif +#endif + , assist pushThread + , assist pushRetryThread + , assist exportThread + , assist exportRetryThread + , assist mergeThread + , assist transferWatcherThread + , assist transferPollerThread + , assist transfererThread + , assist remoteControlThread + , assist daemonStatusThread + , assist $ sanityCheckerDailyThread urlrenderer + , assist sanityCheckerHourlyThread + , assist $ problemFixerThread urlrenderer +#ifndef mingw32_HOST_OS + , assist $ mountWatcherThread urlrenderer +#endif + , assist netWatcherThread + , assist $ upgraderThread urlrenderer + , assist $ upgradeWatcherThread urlrenderer + , assist netWatcherFallbackThread + , assist $ transferScannerThread urlrenderer + , assist $ cronnerThread urlrenderer + , assist configMonitorThread + , assist glacierThread + , watch watchThread + -- must come last so that all threads that wait + -- on it have already started waiting + , watch $ sanityCheckerStartupThread startdelay + ] + + mapM_ (startthread urlrenderer) threads + liftIO waitForTermination + + watch a = (True, a) + assist a = (False, a) + startthread urlrenderer (watcher, t) + | watcher || assistant = startNamedThread urlrenderer t + | otherwise = noop diff --git a/Assistant/Alert.hs b/Assistant/Alert.hs new file mode 100644 index 0000000000..6db66399c5 --- /dev/null +++ b/Assistant/Alert.hs @@ -0,0 +1,460 @@ +{- git-annex assistant alerts + - + - Copyright 2012-2014 Joey Hess + - + - Licensed under the GNU GPL version 3 or higher. + -} + +{-# LANGUAGE OverloadedStrings, CPP, BangPatterns #-} + +module Assistant.Alert where + +import Annex.Common +import Assistant.Types.Alert +import Assistant.Alert.Utility +import qualified Remote +import Utility.Tense +import Types.Transfer +import Types.Distribution +import Git.Types (RemoteName) + +import Data.String +import qualified Data.Text as T +import qualified Control.Exception as E + +#ifdef WITH_WEBAPP +import Assistant.DaemonStatus +import Assistant.WebApp.Types +import Assistant.WebApp (renderUrl) +#endif +import Assistant.Monad +import Assistant.Types.UrlRenderer + +{- Makes a button for an alert that opens a Route. + - + - If autoclose is set, the button will close the alert it's + - attached to when clicked. -} +#ifdef WITH_WEBAPP +mkAlertButton :: Bool -> T.Text -> UrlRenderer -> Route WebApp -> Assistant AlertButton +mkAlertButton autoclose label urlrenderer route = do + close <- asIO1 removeAlert + url <- liftIO $ renderUrl urlrenderer route [] + return $ AlertButton + { buttonLabel = label + , buttonUrl = url + , buttonAction = if autoclose then Just close else Nothing + , buttonPrimary = True + } +#endif + +renderData :: Alert -> TenseText +renderData = tenseWords . alertData + +baseActivityAlert :: Alert +baseActivityAlert = Alert + { alertClass = Activity + , alertHeader = Nothing + , alertMessageRender = renderData + , alertData = [] + , alertCounter = 0 + , alertBlockDisplay = False + , alertClosable = False + , alertPriority = Medium + , alertIcon = Just ActivityIcon + , alertCombiner = Nothing + , alertName = Nothing + , alertButtons = [] + } + +warningAlert :: String -> String -> Alert +warningAlert name msg = Alert + { alertClass = Warning + , alertHeader = Just $ tenseWords ["warning"] + , alertMessageRender = renderData + , alertData = [UnTensed $ T.pack msg] + , alertCounter = 0 + , alertBlockDisplay = True + , alertClosable = True + , alertPriority = High + , alertIcon = Just ErrorIcon + , alertCombiner = Just $ dataCombiner $ \_old new -> new + , alertName = Just $ WarningAlert name + , alertButtons = [] + } + +errorAlert :: String -> [AlertButton] -> Alert +errorAlert msg buttons = Alert + { alertClass = Error + , alertHeader = Nothing + , alertMessageRender = renderData + , alertData = [UnTensed $ T.pack msg] + , alertCounter = 0 + , alertBlockDisplay = True + , alertClosable = True + , alertPriority = Pinned + , alertIcon = Just ErrorIcon + , alertCombiner = Nothing + , alertName = Nothing + , alertButtons = buttons + } + +activityAlert :: Maybe TenseText -> [TenseChunk] -> Alert +activityAlert header dat = baseActivityAlert + { alertHeader = header + , alertData = dat + } + +startupScanAlert :: Alert +startupScanAlert = activityAlert Nothing + [Tensed "Performing" "Performed", "startup scan"] + +{- Displayed when a shutdown is occurring, so will be seen after shutdown + - has happened. -} +shutdownAlert :: Alert +shutdownAlert = warningAlert "shutdown" "git-annex has been shut down" + +commitAlert :: Alert +commitAlert = activityAlert Nothing + [Tensed "Committing" "Committed", "changes to git"] + +showRemotes :: [RemoteName] -> TenseChunk +showRemotes = UnTensed . T.intercalate ", " . map T.pack + +syncAlert :: [Remote] -> Alert +syncAlert = syncAlert' . map Remote.name + +syncAlert' :: [RemoteName] -> Alert +syncAlert' rs = baseActivityAlert + { alertName = Just SyncAlert + , alertHeader = Just $ tenseWords + [Tensed "Syncing" "Synced", "with", showRemotes rs] + , alertPriority = Low + , alertIcon = Just SyncIcon + } + +syncResultAlert :: [Remote] -> [Remote] -> Alert +syncResultAlert succeeded failed = syncResultAlert' + (map Remote.name succeeded) + (map Remote.name failed) + +syncResultAlert' :: [RemoteName] -> [RemoteName] -> Alert +syncResultAlert' succeeded failed = makeAlertFiller (not $ null succeeded) $ + baseActivityAlert + { alertName = Just SyncAlert + , alertHeader = Just $ tenseWords msg + } + where + msg + | null succeeded = ["Failed to sync with", showRemotes failed] + | null failed = ["Synced with", showRemotes succeeded] + | otherwise = + [ "Synced with", showRemotes succeeded + , "but not with", showRemotes failed + ] + +sanityCheckAlert :: Alert +sanityCheckAlert = activityAlert + (Just $ tenseWords [Tensed "Running" "Ran", "daily sanity check"]) + ["to make sure everything is ok."] + +sanityCheckFixAlert :: String -> Alert +sanityCheckFixAlert msg = Alert + { alertClass = Warning + , alertHeader = Just $ tenseWords ["Fixed a problem"] + , alertMessageRender = render + , alertData = [UnTensed $ T.pack msg] + , alertCounter = 0 + , alertBlockDisplay = True + , alertPriority = High + , alertClosable = True + , alertIcon = Just ErrorIcon + , alertName = Just SanityCheckFixAlert + , alertCombiner = Just $ dataCombiner (++) + , alertButtons = [] + } + where + render alert = tenseWords $ alerthead : alertData alert ++ [alertfoot] + alerthead = "The daily sanity check found and fixed a problem:" + alertfoot = "If these problems persist, consider filing a bug report." + +fsckingAlert :: AlertButton -> Maybe Remote -> Alert +fsckingAlert button mr = baseActivityAlert + { alertData = case mr of + Nothing -> [ UnTensed $ T.pack $ "Consistency check in progress" ] + Just r -> [ UnTensed $ T.pack $ "Consistency check of " ++ Remote.name r ++ " in progress"] + , alertButtons = [button] + } + +showFscking :: UrlRenderer -> Maybe Remote -> IO (Either E.SomeException a) -> Assistant a +showFscking urlrenderer mr a = do +#ifdef WITH_WEBAPP + button <- mkAlertButton False (T.pack "Configure") urlrenderer ConfigFsckR + r <- alertDuring (fsckingAlert button mr) $ + liftIO a +#else + r <- liftIO a +#endif + either (liftIO . E.throwIO) return r + +notFsckedNudge :: UrlRenderer -> Maybe Remote -> Assistant () +#ifdef WITH_WEBAPP +notFsckedNudge urlrenderer mr = do + button <- mkAlertButton True (T.pack "Configure") urlrenderer ConfigFsckR + void $ addAlert (notFsckedAlert mr button) +#else +notFsckedNudge _ _ = noop +#endif + +notFsckedAlert :: Maybe Remote -> AlertButton -> Alert +notFsckedAlert mr button = Alert + { alertHeader = Just $ fromString $ concat + [ "You should enable consistency checking to protect your data" + , maybe "" (\r -> " in " ++ Remote.name r) mr + , "." + ] + , alertIcon = Just InfoIcon + , alertPriority = High + , alertButtons = [button] + , alertClosable = True + , alertClass = Message + , alertMessageRender = renderData + , alertCounter = 0 + , alertBlockDisplay = True + , alertName = Just NotFsckedAlert + , alertCombiner = Just $ dataCombiner $ \_old new -> new + , alertData = [] + } + +baseUpgradeAlert :: [AlertButton] -> TenseText -> Alert +baseUpgradeAlert buttons message = Alert + { alertHeader = Just message + , alertIcon = Just UpgradeIcon + , alertPriority = High + , alertButtons = buttons + , alertClosable = True + , alertClass = Message + , alertMessageRender = renderData + , alertCounter = 0 + , alertBlockDisplay = True + , alertName = Just UpgradeAlert + , alertCombiner = Just $ fullCombiner $ \new _old -> new + , alertData = [] + } + +canUpgradeAlert :: AlertPriority -> GitAnnexVersion -> AlertButton -> Alert +canUpgradeAlert priority version button = + (baseUpgradeAlert [button] $ fromString msg) + { alertPriority = priority + , alertData = [fromString $ " (version " ++ version ++ ")"] + } + where + msg = if priority >= High + then "An important upgrade of git-annex is available!" + else "An upgrade of git-annex is available." + +upgradeReadyAlert :: AlertButton -> Alert +upgradeReadyAlert button = baseUpgradeAlert [button] $ + fromString "A new version of git-annex has been installed." + +upgradingAlert :: Alert +upgradingAlert = activityAlert Nothing [ fromString "Upgrading git-annex" ] + +upgradeFinishedAlert :: Maybe AlertButton -> GitAnnexVersion -> Alert +upgradeFinishedAlert button version = + baseUpgradeAlert (maybeToList button) $ fromString $ + "Finished upgrading git-annex to version " ++ version + +upgradeFailedAlert :: String -> Alert +upgradeFailedAlert msg = (errorAlert msg []) + { alertHeader = Just $ fromString "Upgrade failed." } + +unusedFilesAlert :: [AlertButton] -> String -> Alert +unusedFilesAlert buttons message = Alert + { alertHeader = Just $ fromString $ unwords + [ "Old and deleted files are piling up --" + , message + ] + , alertIcon = Just InfoIcon + , alertPriority = High + , alertButtons = buttons + , alertClosable = True + , alertClass = Message + , alertMessageRender = renderData + , alertCounter = 0 + , alertBlockDisplay = True + , alertName = Just UnusedFilesAlert + , alertCombiner = Just $ fullCombiner $ \new _old -> new + , alertData = [] + } + +brokenRepositoryAlert :: [AlertButton] -> Alert +brokenRepositoryAlert = errorAlert "Serious problems have been detected with your repository. This needs your immediate attention!" + +repairingAlert :: String -> Alert +repairingAlert repodesc = activityAlert Nothing + [ Tensed "Attempting to repair" "Repaired" + , UnTensed $ T.pack repodesc + ] + +pairingAlert :: AlertButton -> Alert +pairingAlert button = baseActivityAlert + { alertData = [ UnTensed "Pairing in progress" ] + , alertPriority = High + , alertButtons = [button] + } + +pairRequestReceivedAlert :: String -> AlertButton -> Alert +pairRequestReceivedAlert who button = Alert + { alertClass = Message + , alertHeader = Nothing + , alertMessageRender = renderData + , alertData = [UnTensed $ T.pack $ who ++ " is sending a pair request."] + , alertCounter = 0 + , alertBlockDisplay = False + , alertPriority = High + , alertClosable = True + , alertIcon = Just InfoIcon + , alertName = Just $ PairAlert who + , alertCombiner = Just $ dataCombiner $ \_old new -> new + , alertButtons = [button] + } + +pairRequestAcknowledgedAlert :: String -> Maybe AlertButton -> Alert +pairRequestAcknowledgedAlert who button = baseActivityAlert + { alertData = ["Pairing with", UnTensed (T.pack who), Tensed "in progress" "complete"] + , alertPriority = High + , alertName = Just $ PairAlert who + , alertCombiner = Just $ dataCombiner $ \_old new -> new + , alertButtons = maybeToList button + } + +connectionNeededAlert :: AlertButton -> Alert +connectionNeededAlert button = Alert + { alertHeader = Just "Share with friends, and keep your devices in sync across the cloud." + , alertIcon = Just ConnectionIcon + , alertPriority = High + , alertButtons = [button] + , alertClosable = True + , alertClass = Message + , alertMessageRender = renderData + , alertCounter = 0 + , alertBlockDisplay = True + , alertName = Just ConnectionNeededAlert + , alertCombiner = Just $ dataCombiner $ \_old new -> new + , alertData = [] + } + +cloudRepoNeededAlert :: Maybe String -> AlertButton -> Alert +cloudRepoNeededAlert friendname button = Alert + { alertHeader = Just $ fromString $ unwords + [ "Unable to download files from" + , (fromMaybe "your other devices" friendname) ++ "." + ] + , alertIcon = Just ErrorIcon + , alertPriority = High + , alertButtons = [button] + , alertClosable = True + , alertClass = Message + , alertMessageRender = renderData + , alertCounter = 0 + , alertBlockDisplay = True + , alertName = Just $ CloudRepoNeededAlert + , alertCombiner = Just $ dataCombiner $ \_old new -> new + , alertData = [] + } + +remoteRemovalAlert :: String -> AlertButton -> Alert +remoteRemovalAlert desc button = Alert + { alertHeader = Just $ fromString $ + "The repository \"" ++ desc ++ + "\" has been emptied, and can now be removed." + , alertIcon = Just InfoIcon + , alertPriority = High + , alertButtons = [button] + , alertClosable = True + , alertClass = Message + , alertMessageRender = renderData + , alertCounter = 0 + , alertBlockDisplay = True + , alertName = Just $ RemoteRemovalAlert desc + , alertCombiner = Just $ dataCombiner $ \_old new -> new + , alertData = [] + } + +{- Show a message that relates to a list of files. + - + - The most recent several files are shown, and a count of any others. -} +fileAlert :: TenseChunk -> [FilePath] -> Alert +fileAlert msg files = (activityAlert Nothing shortfiles) + { alertName = Just $ FileAlert msg + , alertMessageRender = renderer + , alertCounter = counter + , alertCombiner = Just $ fullCombiner combiner + } + where + maxfilesshown = 10 + + (!somefiles, !counter) = splitcounter (dedupadjacent files) + !shortfiles = map (fromString . shortFile . takeFileName) somefiles + + renderer alert = tenseWords $ msg : alertData alert ++ showcounter + where + showcounter = case alertCounter alert of + 0 -> [] + _ -> [fromString $ "and " ++ show (alertCounter alert) ++ " other files"] + + dedupadjacent (x:y:rest) + | x == y = dedupadjacent (y:rest) + | otherwise = x : dedupadjacent (y:rest) + dedupadjacent (x:[]) = [x] + dedupadjacent [] = [] + + {- Note that this ensures the counter is never 1; no need to say + - "1 file" when the filename could be shown. -} + splitcounter l + | length l <= maxfilesshown = (l, 0) + | otherwise = + let (keep, rest) = splitAt (maxfilesshown - 1) l + in (keep, length rest) + + combiner new old = + let (!fs, n) = splitcounter $ + dedupadjacent $ alertData new ++ alertData old + !cnt = n + alertCounter new + alertCounter old + in old + { alertData = fs + , alertCounter = cnt + } + +addFileAlert :: [FilePath] -> Alert +addFileAlert = fileAlert (Tensed "Adding" "Added") + +{- This is only used as a success alert after a transfer, not during it. -} +transferFileAlert :: Direction -> Bool -> FilePath -> Alert +transferFileAlert direction True file + | direction == Upload = fileAlert "Uploaded" [file] + | otherwise = fileAlert "Downloaded" [file] +transferFileAlert direction False file + | direction == Upload = fileAlert "Upload failed" [file] + | otherwise = fileAlert "Download failed" [file] + +dataCombiner :: ([TenseChunk] -> [TenseChunk] -> [TenseChunk]) -> AlertCombiner +dataCombiner combiner = fullCombiner $ + \new old -> old { alertData = alertData new `combiner` alertData old } + +fullCombiner :: (Alert -> Alert -> Alert) -> AlertCombiner +fullCombiner combiner new old + | alertClass new /= alertClass old = Nothing + | alertName new == alertName old = + Just $! new `combiner` old + | otherwise = Nothing + +shortFile :: FilePath -> String +shortFile f + | len < maxlen = f + | otherwise = take half f ++ ".." ++ drop (len - half) f + where + len = length f + maxlen = 20 + half = (maxlen - 2) `div` 2 + diff --git a/Assistant/Alert/Utility.hs b/Assistant/Alert/Utility.hs new file mode 100644 index 0000000000..f3485deab3 --- /dev/null +++ b/Assistant/Alert/Utility.hs @@ -0,0 +1,129 @@ +{- git-annex assistant alert utilities + - + - Copyright 2012, 2013 Joey Hess + - + - Licensed under the GNU GPL version 3 or higher. + -} + +module Assistant.Alert.Utility where + +import Annex.Common +import Assistant.Types.Alert +import Utility.Tense + +import qualified Data.Text as T +import Data.Text (Text) +import qualified Data.Map.Strict as M + +{- This is as many alerts as it makes sense to display at a time. + - A display might be smaller, or larger, the point is to not overwhelm the + - user with a ton of alerts. -} +displayAlerts :: Int +displayAlerts = 6 + +{- This is not a hard maximum, but there's no point in keeping a great + - many filler alerts in an AlertMap, so when there's more than this many, + - they start being pruned, down toward displayAlerts. -} +maxAlerts :: Int +maxAlerts = displayAlerts * 2 + +type AlertPair = (AlertId, Alert) + +{- The desired order is the reverse of: + - + - - Pinned alerts + - - High priority alerts, newest first + - - Medium priority Activity, newest first (mostly used for Activity) + - - Low priority alerts, newest first + - - Filler priorty alerts, newest first + - - Ties are broken by the AlertClass, with Errors etc coming first. + -} +compareAlertPairs :: AlertPair -> AlertPair -> Ordering +compareAlertPairs + (aid, Alert { alertClass = aclass, alertPriority = aprio }) + (bid, Alert { alertClass = bclass, alertPriority = bprio }) + = compare aprio bprio + `mappend` compare aid bid + `mappend` compare aclass bclass + +sortAlertPairs :: [AlertPair] -> [AlertPair] +sortAlertPairs = sortBy compareAlertPairs + +{- Renders an alert's header for display, if it has one. -} +renderAlertHeader :: Alert -> Maybe Text +renderAlertHeader alert = renderTense (alertTense alert) <$> alertHeader alert + +{- Renders an alert's message for display. -} +renderAlertMessage :: Alert -> Text +renderAlertMessage alert = renderTense (alertTense alert) $ + (alertMessageRender alert) alert + +showAlert :: Alert -> String +showAlert alert = T.unpack $ T.unwords $ catMaybes + [ renderAlertHeader alert + , Just $ renderAlertMessage alert + ] + +alertTense :: Alert -> Tense +alertTense alert + | alertClass alert == Activity = Present + | otherwise = Past + +{- Checks if two alerts display the same. -} +effectivelySameAlert :: Alert -> Alert -> Bool +effectivelySameAlert x y = all id + [ alertClass x == alertClass y + , alertHeader x == alertHeader y + , alertData x == alertData y + , alertBlockDisplay x == alertBlockDisplay y + , alertClosable x == alertClosable y + , alertPriority x == alertPriority y + ] + +makeAlertFiller :: Bool -> Alert -> Alert +makeAlertFiller success alert + | isFiller alert = alert + | otherwise = alert + { alertClass = if c == Activity then c' else c + , alertPriority = Filler + , alertClosable = True + , alertButtons = [] + , alertIcon = Just $ if success then SuccessIcon else ErrorIcon + } + where + c = alertClass alert + c' + | success = Success + | otherwise = Error + +isFiller :: Alert -> Bool +isFiller alert = alertPriority alert == Filler + +{- Updates the Alertmap, adding or updating an alert. + - + - Any old filler that looks the same as the alert is removed. + - + - Or, if the alert has an alertCombiner that combines it with + - an old alert, the old alert is replaced with the result, and the + - alert is removed. + - + - Old filler alerts are pruned once maxAlerts is reached. + -} +mergeAlert :: AlertId -> Alert -> AlertMap -> AlertMap +mergeAlert i al m = maybe updatePrune updateCombine (alertCombiner al) + where + pruneSame k al' = k == i || not (effectivelySameAlert al al') + pruneBloat m' + | bloat > 0 = M.fromList $ pruneold $ M.toList m' + | otherwise = m' + where + bloat = M.size m' - maxAlerts + pruneold l = + let (f, rest) = partition (\(_, a) -> isFiller a) l + in drop bloat f ++ rest + updatePrune = pruneBloat $ M.filterWithKey pruneSame $ M.insert i al m + updateCombine combiner = + let combined = M.mapMaybe (combiner al) m + in if M.null combined + then updatePrune + else M.delete i $ M.union combined m diff --git a/Assistant/BranchChange.hs b/Assistant/BranchChange.hs new file mode 100644 index 0000000000..c588c910a7 --- /dev/null +++ b/Assistant/BranchChange.hs @@ -0,0 +1,19 @@ +{- git-annex assistant git-annex branch change tracking + - + - Copyright 2012 Joey Hess + - + - Licensed under the GNU GPL version 3 or higher. + -} + +module Assistant.BranchChange where + +import Assistant.Common +import Assistant.Types.BranchChange + +import Control.Concurrent.MSampleVar + +branchChanged :: Assistant () +branchChanged = flip writeSV () <<~ (fromBranchChangeHandle . branchChangeHandle) + +waitBranchChange :: Assistant () +waitBranchChange = readSV <<~ (fromBranchChangeHandle . branchChangeHandle) diff --git a/Assistant/Changes.hs b/Assistant/Changes.hs new file mode 100644 index 0000000000..6eb9bc28e5 --- /dev/null +++ b/Assistant/Changes.hs @@ -0,0 +1,47 @@ +{- git-annex assistant change tracking + - + - Copyright 2012-2013 Joey Hess + - + - Licensed under the GNU GPL version 3 or higher. + -} + +module Assistant.Changes where + +import Assistant.Common +import Assistant.Types.Changes +import Utility.TList + +import Data.Time.Clock +import Control.Concurrent.STM + +{- Handlers call this when they made a change that needs to get committed. -} +madeChange :: FilePath -> ChangeInfo -> Assistant (Maybe Change) +madeChange f t = Just <$> (Change <$> liftIO getCurrentTime <*> pure f <*> pure t) + +noChange :: Assistant (Maybe Change) +noChange = return Nothing + +{- Indicates an add needs to be done, but has not started yet. -} +pendingAddChange :: FilePath -> Assistant (Maybe Change) +pendingAddChange f = Just <$> (PendingAddChange <$> liftIO getCurrentTime <*> pure f) + +{- Gets all unhandled changes. + - Blocks until at least one change is made. -} +getChanges :: Assistant [Change] +getChanges = (atomically . getTList) <<~ changePool + +{- Gets all unhandled changes, without blocking. -} +getAnyChanges :: Assistant [Change] +getAnyChanges = (atomically . takeTList) <<~ changePool + +{- Puts unhandled changes back into the pool. + - Note: Original order is not preserved. -} +refillChanges :: [Change] -> Assistant () +refillChanges cs = (atomically . flip appendTList cs) <<~ changePool + +{- Records a change to the pool. -} +recordChange :: Change -> Assistant () +recordChange c = (atomically . flip snocTList c) <<~ changePool + +recordChanges :: [Change] -> Assistant () +recordChanges = refillChanges diff --git a/Assistant/Commits.hs b/Assistant/Commits.hs new file mode 100644 index 0000000000..255648c94d --- /dev/null +++ b/Assistant/Commits.hs @@ -0,0 +1,32 @@ +{- git-annex assistant commit tracking + - + - Copyright 2012 Joey Hess + - + - Licensed under the GNU GPL version 3 or higher. + -} + +module Assistant.Commits where + +import Assistant.Common +import Assistant.Types.Commits +import Utility.TList + +import Control.Concurrent.STM + +{- Gets all unhandled commits. + - Blocks until at least one commit is made. -} +getCommits :: Assistant [Commit] +getCommits = (atomically . getTList) <<~ commitChan + +{- Records a commit in the channel. -} +recordCommit :: Assistant () +recordCommit = (atomically . flip consTList Commit) <<~ commitChan + +{- Gets all unhandled export commits. + - Blocks until at least one export commit is made. -} +getExportCommits :: Assistant [Commit] +getExportCommits = (atomically . getTList) <<~ exportCommitChan + +{- Records an export commit in the channel. -} +recordExportCommit :: Assistant () +recordExportCommit = (atomically . flip consTList Commit) <<~ exportCommitChan diff --git a/Assistant/Common.hs b/Assistant/Common.hs new file mode 100644 index 0000000000..d26d2b07a7 --- /dev/null +++ b/Assistant/Common.hs @@ -0,0 +1,14 @@ +{- Common infrastructure for the git-annex assistant. + - + - Copyright 2012 Joey Hess + - + - Licensed under the GNU GPL version 3 or higher. + -} + +module Assistant.Common (module X) where + +import Annex.Common as X +import Assistant.Monad as X +import Assistant.Types.DaemonStatus as X +import Assistant.Types.NamedThread as X +import Assistant.Types.Alert as X diff --git a/Assistant/CredPairCache.hs b/Assistant/CredPairCache.hs new file mode 100644 index 0000000000..ac355b55a5 --- /dev/null +++ b/Assistant/CredPairCache.hs @@ -0,0 +1,53 @@ +{- git-annex assistant CredPair cache. + - + - Copyright 2014 Joey Hess + - + - Licensed under the GNU GPL version 3 or higher. + -} + +{-# LANGUAGE BangPatterns #-} + +module Assistant.CredPairCache ( + cacheCred, + getCachedCred, + expireCachedCred, +) where + +import Assistant.Types.CredPairCache +import Types.Creds +import Assistant.Common +import Utility.ThreadScheduler + +import qualified Data.Map as M +import Control.Concurrent + +{- Caches a CredPair, but only for a limited time, after which it + - will expire. + - + - Note that repeatedly caching the same CredPair + - does not reset its expiry time. + -} +cacheCred :: CredPair -> Seconds -> Assistant () +cacheCred (login, password) expireafter = do + cache <- getAssistant credPairCache + liftIO $ do + changeStrict cache $ M.insert login password + void $ forkIO $ do + threadDelaySeconds expireafter + changeStrict cache $ M.delete login + +getCachedCred :: Login -> Assistant (Maybe Password) +getCachedCred login = do + cache <- getAssistant credPairCache + liftIO $ M.lookup login <$> readMVar cache + +expireCachedCred :: Login -> Assistant () +expireCachedCred login = do + cache <- getAssistant credPairCache + liftIO $ changeStrict cache $ M.delete login + +{- Update map strictly to avoid keeping references to old creds in memory. -} +changeStrict :: CredPairCache -> (M.Map Login Password -> M.Map Login Password) -> IO () +changeStrict cache a = modifyMVar_ cache $ \m -> do + let !m' = a m + return m' diff --git a/Assistant/DaemonStatus.hs b/Assistant/DaemonStatus.hs new file mode 100644 index 0000000000..2f1df5725a --- /dev/null +++ b/Assistant/DaemonStatus.hs @@ -0,0 +1,267 @@ +{- git-annex assistant daemon status + - + - Copyright 2012 Joey Hess + - + - Licensed under the GNU GPL version 3 or higher. + -} + +{-# LANGUAGE BangPatterns #-} + +module Assistant.DaemonStatus where + +import Assistant.Common +import Assistant.Alert.Utility +import Utility.Tmp +import Utility.NotificationBroadcaster +import Types.Transfer +import Logs.Transfer +import Logs.Trust +import Logs.TimeStamp +import qualified Remote +import qualified Types.Remote as Remote +import Config.DynamicConfig +import Annex.Export + +import Control.Concurrent.STM +import System.Posix.Types +import Data.Time.Clock.POSIX +import qualified Data.Map.Strict as M +import qualified Data.Set as S + +getDaemonStatus :: Assistant DaemonStatus +getDaemonStatus = (atomically . readTVar) <<~ daemonStatusHandle + +modifyDaemonStatus_ :: (DaemonStatus -> DaemonStatus) -> Assistant () +modifyDaemonStatus_ a = modifyDaemonStatus $ \s -> (a s, ()) + +modifyDaemonStatus :: (DaemonStatus -> (DaemonStatus, b)) -> Assistant b +modifyDaemonStatus a = do + dstatus <- getAssistant daemonStatusHandle + liftIO $ do + (s, b) <- atomically $ do + r@(!s, _) <- a <$> readTVar dstatus + writeTVar dstatus s + return r + sendNotification $ changeNotifier s + return b + +{- Returns a function that updates the lists of syncable remotes + - and other associated information. -} +calcSyncRemotes :: Annex (DaemonStatus -> DaemonStatus) +calcSyncRemotes = do + rs <- filterM (liftIO . getDynamicConfig . remoteAnnexSync . Remote.gitconfig) + =<< (concat . Remote.byCost <$> Remote.remoteList) + alive <- trustExclude DeadTrusted (map Remote.uuid rs) + let good r = Remote.uuid r `elem` alive + let syncable = filter good rs + contentremotes <- filterM (not <$$> liftIO . getDynamicConfig . remoteAnnexIgnore . Remote.gitconfig) $ + filter (\r -> Remote.uuid r /= NoUUID) syncable + let (exportremotes, dataremotes) = partition (exportTree . Remote.config) contentremotes + + return $ \dstatus -> dstatus + { syncRemotes = syncable + , syncGitRemotes = filter Remote.gitSyncableRemote syncable + , syncDataRemotes = dataremotes + , exportRemotes = exportremotes + , downloadRemotes = contentremotes + , syncingToCloudRemote = any iscloud contentremotes + } + where + iscloud r = not (Remote.readonly r) && Remote.availability r == Remote.GloballyAvailable + +{- Updates the syncRemotes list from the list of all remotes in Annex state. -} +updateSyncRemotes :: Assistant () +updateSyncRemotes = do + modifyDaemonStatus_ =<< liftAnnex calcSyncRemotes + status <- getDaemonStatus + liftIO $ sendNotification $ syncRemotesNotifier status + + when (syncingToCloudRemote status) $ + updateAlertMap $ + M.filter $ \alert -> + alertName alert /= Just CloudRepoNeededAlert + +changeCurrentlyConnected :: (S.Set UUID -> S.Set UUID) -> Assistant () +changeCurrentlyConnected sm = do + modifyDaemonStatus_ $ \ds -> ds + { currentlyConnectedRemotes = sm (currentlyConnectedRemotes ds) + } + v <- currentlyConnectedRemotes <$> getDaemonStatus + debug [show v] + liftIO . sendNotification =<< syncRemotesNotifier <$> getDaemonStatus + +updateScheduleLog :: Assistant () +updateScheduleLog = + liftIO . sendNotification =<< scheduleLogNotifier <$> getDaemonStatus + +{- Load any previous daemon status file, and store it in a MVar for this + - process to use as its DaemonStatus. Also gets current transfer status. -} +startDaemonStatus :: Annex DaemonStatusHandle +startDaemonStatus = do + file <- fromRepo gitAnnexDaemonStatusFile + status <- liftIO $ + flip catchDefaultIO (readDaemonStatusFile file) =<< newDaemonStatus + transfers <- M.fromList <$> getTransfers + addsync <- calcSyncRemotes + liftIO $ atomically $ newTVar $ addsync $ status + { scanComplete = False + , sanityCheckRunning = False + , currentTransfers = transfers + } + +{- Don't just dump out the structure, because it will change over time, + - and parts of it are not relevant. -} +writeDaemonStatusFile :: FilePath -> DaemonStatus -> IO () +writeDaemonStatusFile file status = + viaTmp writeFile file =<< serialized <$> getPOSIXTime + where + serialized now = unlines + [ "lastRunning:" ++ show now + , "scanComplete:" ++ show (scanComplete status) + , "sanityCheckRunning:" ++ show (sanityCheckRunning status) + , "lastSanityCheck:" ++ maybe "" show (lastSanityCheck status) + ] + +readDaemonStatusFile :: FilePath -> IO DaemonStatus +readDaemonStatusFile file = parse <$> newDaemonStatus <*> readFile file + where + parse status = foldr parseline status . lines + parseline line status + | key == "lastRunning" = parseval parsePOSIXTime $ \v -> + status { lastRunning = Just v } + | key == "scanComplete" = parseval readish $ \v -> + status { scanComplete = v } + | key == "sanityCheckRunning" = parseval readish $ \v -> + status { sanityCheckRunning = v } + | key == "lastSanityCheck" = parseval parsePOSIXTime $ \v -> + status { lastSanityCheck = Just v } + | otherwise = status -- unparsable line + where + (key, value) = separate (== ':') line + parseval parser a = maybe status a (parser value) + +{- Checks if a time stamp was made after the daemon was lastRunning. + - + - Some slop is built in; this really checks if the time stamp was made + - at least ten minutes after the daemon was lastRunning. This is to + - ensure the daemon shut down cleanly, and deal with minor clock skew. + - + - If the daemon has never ran before, this always returns False. + -} +afterLastDaemonRun :: EpochTime -> DaemonStatus -> Bool +afterLastDaemonRun timestamp status = maybe False (< t) (lastRunning status) + where + t = realToFrac (timestamp + slop) :: POSIXTime + slop = fromIntegral tenMinutes + +tenMinutes :: Int +tenMinutes = 10 * 60 + +{- Mutates the transfer map. Runs in STM so that the transfer map can + - be modified in the same transaction that modifies the transfer queue. + - Note that this does not send a notification of the change; that's left + - to the caller. -} +adjustTransfersSTM :: DaemonStatusHandle -> (TransferMap -> TransferMap) -> STM () +adjustTransfersSTM dstatus a = do + s <- readTVar dstatus + let !v = a (currentTransfers s) + writeTVar dstatus $ s { currentTransfers = v } + +{- Checks if a transfer is currently running. -} +checkRunningTransferSTM :: DaemonStatusHandle -> Transfer -> STM Bool +checkRunningTransferSTM dstatus t = M.member t . currentTransfers + <$> readTVar dstatus + +{- Alters a transfer's info, if the transfer is in the map. -} +alterTransferInfo :: Transfer -> (TransferInfo -> TransferInfo) -> Assistant () +alterTransferInfo t a = updateTransferInfo' $ M.adjust a t + +{- Updates a transfer's info. Adds the transfer to the map if necessary, + - or if already present, updates it while preserving the old transferTid, + - transferPaused, and bytesComplete values, which are not written to disk. -} +updateTransferInfo :: Transfer -> TransferInfo -> Assistant () +updateTransferInfo t info = updateTransferInfo' $ M.insertWith merge t info + where + merge new old = new + { transferTid = maybe (transferTid new) Just (transferTid old) + , transferPaused = transferPaused new || transferPaused old + , bytesComplete = maybe (bytesComplete new) Just (bytesComplete old) + } + +updateTransferInfo' :: (TransferMap -> TransferMap) -> Assistant () +updateTransferInfo' a = notifyTransfer `after` modifyDaemonStatus_ update + where + update s = s { currentTransfers = a (currentTransfers s) } + +{- Removes a transfer from the map, and returns its info. -} +removeTransfer :: Transfer -> Assistant (Maybe TransferInfo) +removeTransfer t = notifyTransfer `after` modifyDaemonStatus remove + where + remove s = + let (info, ts) = M.updateLookupWithKey + (\_k _v -> Nothing) + t (currentTransfers s) + in (s { currentTransfers = ts }, info) + +{- Send a notification when a transfer is changed. -} +notifyTransfer :: Assistant () +notifyTransfer = do + dstatus <- getAssistant daemonStatusHandle + liftIO $ sendNotification + =<< transferNotifier <$> atomically (readTVar dstatus) + +{- Send a notification when alerts are changed. -} +notifyAlert :: Assistant () +notifyAlert = do + dstatus <- getAssistant daemonStatusHandle + liftIO $ sendNotification + =<< alertNotifier <$> atomically (readTVar dstatus) + +{- Returns the alert's identifier, which can be used to remove it. -} +addAlert :: Alert -> Assistant AlertId +addAlert alert = do + notice [showAlert alert] + notifyAlert `after` modifyDaemonStatus add + where + add s = (s { lastAlertId = i, alertMap = m }, i) + where + !i = nextAlertId $ lastAlertId s + !m = mergeAlert i alert (alertMap s) + +removeAlert :: AlertId -> Assistant () +removeAlert i = updateAlert i (const Nothing) + +updateAlert :: AlertId -> (Alert -> Maybe Alert) -> Assistant () +updateAlert i a = updateAlertMap $ \m -> M.update a i m + +updateAlertMap :: (AlertMap -> AlertMap) -> Assistant () +updateAlertMap a = notifyAlert `after` modifyDaemonStatus_ update + where + update s = + let !m = a (alertMap s) + in s { alertMap = m } + +{- Displays an alert while performing an activity that returns True on + - success. + - + - The alert is left visible afterwards, as filler. + - Old filler is pruned, to prevent the map growing too large. -} +alertWhile :: Alert -> Assistant Bool -> Assistant Bool +alertWhile alert a = alertWhile' alert $ do + r <- a + return (r, r) + +{- Like alertWhile, but allows the activity to return a value too. -} +alertWhile' :: Alert -> Assistant (Bool, a) -> Assistant a +alertWhile' alert a = do + let alert' = alert { alertClass = Activity } + i <- addAlert alert' + (ok, r) <- a + updateAlertMap $ mergeAlert i $ makeAlertFiller ok alert' + return r + +{- Displays an alert while performing an activity, then removes it. -} +alertDuring :: Alert -> Assistant a -> Assistant a +alertDuring alert a = do + i <- addAlert $ alert { alertClass = Activity } + removeAlert i `after` a diff --git a/Assistant/DeleteRemote.hs b/Assistant/DeleteRemote.hs new file mode 100644 index 0000000000..6c88c61f55 --- /dev/null +++ b/Assistant/DeleteRemote.hs @@ -0,0 +1,89 @@ +{- git-annex assistant remote deletion utilities + - + - Copyright 2012 Joey Hess + - + - Licensed under the GNU GPL version 3 or higher. + -} + +{-# LANGUAGE CPP #-} + +module Assistant.DeleteRemote where + +import Assistant.Common +import Assistant.Types.UrlRenderer +import Assistant.TransferQueue +import Types.Transfer +import Logs.Location +import Assistant.DaemonStatus +import qualified Remote +import Remote.List +import qualified Git.Remote.Remove +import Logs.Trust +import qualified Annex + +#ifdef WITH_WEBAPP +import Assistant.WebApp.Types +import Assistant.Alert +import qualified Data.Text as T +#endif + +{- Removes a remote (but leave the repository as-is), and returns the old + - Remote data. -} +disableRemote :: UUID -> Assistant Remote +disableRemote uuid = do + remote <- fromMaybe (error "unknown remote") + <$> liftAnnex (Remote.remoteFromUUID uuid) + liftAnnex $ do + inRepo $ Git.Remote.Remove.remove (Remote.name remote) + void $ remoteListRefresh + updateSyncRemotes + return remote + +{- Removes a remote, marking it dead .-} +removeRemote :: UUID -> Assistant Remote +removeRemote uuid = do + liftAnnex $ trustSet uuid DeadTrusted + disableRemote uuid + +{- Called when a Remote is probably empty, to remove it. + - + - This does one last check for any objects remaining in the Remote, + - and if there are any, queues Downloads of them, and defers removing + - the remote for later. This is to catch any objects not referred to + - in keys in the current branch. + -} +removableRemote :: UrlRenderer -> UUID -> Assistant () +removableRemote urlrenderer uuid = do + keys <- getkeys + if null keys + then finishRemovingRemote urlrenderer uuid + else do + r <- fromMaybe (error "unknown remote") + <$> liftAnnex (Remote.remoteFromUUID uuid) + mapM_ (queueremaining r) keys + where + queueremaining r k = + queueTransferWhenSmall "remaining object in unwanted remote" + (AssociatedFile Nothing) (Transfer Download uuid k) r + {- Scanning for keys can take a long time; do not tie up + - the Annex monad while doing it, so other threads continue to + - run. -} + getkeys = do + a <- liftAnnex $ Annex.withCurrentState $ loggedKeysFor uuid + liftIO a + +{- With the webapp, this asks the user to click on a button to finish + - removing the remote. + - + - Without the webapp, just do the removal now. + -} +finishRemovingRemote :: UrlRenderer -> UUID -> Assistant () +#ifdef WITH_WEBAPP +finishRemovingRemote urlrenderer uuid = do + desc <- liftAnnex $ Remote.prettyUUID uuid + button <- mkAlertButton True (T.pack "Finish deletion process") urlrenderer $ + FinishDeleteRepositoryR uuid + void $ addAlert $ remoteRemovalAlert desc button +#else +finishRemovingRemote _ uuid = void $ removeRemote uuid +#endif diff --git a/Assistant/Drop.hs b/Assistant/Drop.hs new file mode 100644 index 0000000000..5653b7795b --- /dev/null +++ b/Assistant/Drop.hs @@ -0,0 +1,26 @@ +{- git-annex assistant dropping of unwanted content + - + - Copyright 2012 Joey Hess + - + - Licensed under the GNU GPL version 3 or higher. + -} + +module Assistant.Drop ( + handleDrops, + handleDropsFrom, +) where + +import Assistant.Common +import Assistant.DaemonStatus +import Annex.Drop (handleDropsFrom, Reason) +import Logs.Location +import CmdLine.Action +import Types.NumCopies + +{- Drop from local and/or remote when allowed by the preferred content and + - numcopies settings. -} +handleDrops :: Reason -> Bool -> Key -> AssociatedFile -> [VerifiedCopy] -> Assistant () +handleDrops reason fromhere key f preverified = do + syncrs <- syncDataRemotes <$> getDaemonStatus + locs <- liftAnnex $ loggedLocations key + liftAnnex $ handleDropsFrom locs syncrs reason fromhere key f preverified callCommandAction diff --git a/Assistant/Fsck.hs b/Assistant/Fsck.hs new file mode 100644 index 0000000000..9d8848ba98 --- /dev/null +++ b/Assistant/Fsck.hs @@ -0,0 +1,50 @@ +{- git-annex assistant fscking + - + - Copyright 2013 Joey Hess + - + - Licensed under the GNU GPL version 3 or higher. + -} + +module Assistant.Fsck where + +import Assistant.Common +import Types.ScheduledActivity +import qualified Types.Remote as Remote +import Annex.UUID +import Assistant.Alert +import Assistant.Types.UrlRenderer +import Logs.Schedule +import qualified Annex + +import qualified Data.Set as S + +{- Displays a nudge in the webapp if a fsck is not configured for + - the specified remote, or for the local repository. -} +fsckNudge :: UrlRenderer -> Maybe Remote -> Assistant () +fsckNudge urlrenderer mr + | maybe True fsckableRemote mr = + whenM (liftAnnex $ annexFsckNudge <$> Annex.getGitConfig) $ + unlessM (liftAnnex $ checkFscked mr) $ + notFsckedNudge urlrenderer mr + | otherwise = noop + +fsckableRemote :: Remote -> Bool +fsckableRemote = isJust . Remote.remoteFsck + +{- Checks if the remote, or the local repository, has a fsck scheduled. + - Only looks at fscks configured to run via the local repository, not + - other repositories. -} +checkFscked :: Maybe Remote -> Annex Bool +checkFscked mr = any wanted . S.toList <$> (scheduleGet =<< getUUID) + where + wanted = case mr of + Nothing -> isSelfFsck + Just r -> flip isFsckOf (Remote.uuid r) + +isSelfFsck :: ScheduledActivity -> Bool +isSelfFsck (ScheduledSelfFsck _ _) = True +isSelfFsck _ = False + +isFsckOf :: ScheduledActivity -> UUID -> Bool +isFsckOf (ScheduledRemoteFsck u _ _) u' = u == u' +isFsckOf _ _ = False diff --git a/Assistant/Gpg.hs b/Assistant/Gpg.hs new file mode 100644 index 0000000000..34d00a3842 --- /dev/null +++ b/Assistant/Gpg.hs @@ -0,0 +1,36 @@ +{- git-annex assistant gpg stuff + - + - Copyright 2013 Joey Hess + - + - Licensed under the GNU GPL version 3 or higher. + -} + +module Assistant.Gpg where + +import Utility.Gpg +import Utility.UserInfo +import Types.Remote (RemoteConfigKey) + +import qualified Data.Map as M +import Control.Applicative +import Prelude + +{- Generates a gpg user id that is not used by any existing secret key -} +newUserId :: GpgCmd -> IO UserId +newUserId cmd = do + oldkeys <- secretKeys cmd + username <- either (const "unknown") id <$> myUserName + let basekeyname = username ++ "'s git-annex encryption key" + return $ Prelude.head $ filter (\n -> M.null $ M.filter (== n) oldkeys) + ( basekeyname + : map (\n -> basekeyname ++ show n) ([2..] :: [Int]) + ) + +data EnableEncryption = HybridEncryption | SharedEncryption | NoEncryption + deriving (Eq) + +{- Generates Remote configuration for encryption. -} +configureEncryption :: EnableEncryption -> (RemoteConfigKey, String) +configureEncryption SharedEncryption = ("encryption", "shared") +configureEncryption NoEncryption = ("encryption", "none") +configureEncryption HybridEncryption = ("encryption", "hybrid") diff --git a/Assistant/Install.hs b/Assistant/Install.hs new file mode 100644 index 0000000000..9b39de233e --- /dev/null +++ b/Assistant/Install.hs @@ -0,0 +1,194 @@ +{- Assistant installation + - + - Copyright 2012 Joey Hess + - + - Licensed under the GNU GPL version 3 or higher. + -} + +{-# LANGUAGE CPP #-} + +module Assistant.Install where + +import Assistant.Common +import Assistant.Install.AutoStart +import Config.Files +import Utility.FileMode +import Utility.Shell +import Utility.Tmp +import Utility.Env +import Utility.SshConfig + +#ifdef darwin_HOST_OS +import Utility.OSX +#else +import Utility.FreeDesktop +import Assistant.Install.Menu +import Utility.UserInfo +import Utility.Android +#endif + +standaloneAppBase :: IO (Maybe FilePath) +standaloneAppBase = getEnv "GIT_ANNEX_APP_BASE" + +{- The standalone app does not have an installation process. + - So when it's run, it needs to set up autostarting of the assistant + - daemon, as well as writing the programFile, and putting the + - git-annex-shell and git-annex-wrapper wrapper scripts into ~/.ssh + - + - Note that this is done every time it's started, so if the user moves + - it around, the paths this sets up won't break. + - + - File manager hook script installation is done even for + - packaged apps, since it has to go into the user's home directory. + -} +ensureInstalled :: IO () +ensureInstalled = ifM (isJust <$> getEnv "GIT_ANNEX_PACKAGE_INSTALL") + ( go Nothing + , go =<< standaloneAppBase + ) + where + go Nothing = installFileManagerHooks "git-annex" + go (Just base) = do + let program = base "git-annex" + programfile <- programFile + createDirectoryIfMissing True (parentDir programfile) + writeFile programfile program + +#ifdef darwin_HOST_OS + autostartfile <- userAutoStart osxAutoStartLabel + installAutoStart program autostartfile +#else + ifM osAndroid + ( do + -- Integration with the Termux:Boot app. + home <- myHomeDir + let bootfile = home ".termux" "boot" "git-annex" + unlessM (doesFileExist bootfile) $ do + createDirectoryIfMissing True (takeDirectory bootfile) + writeFile bootfile "git-annex assistant --autostart" + , do + menufile <- desktopMenuFilePath "git-annex" <$> userDataDir + icondir <- iconDir <$> userDataDir + installMenu program menufile base icondir + autostartfile <- autoStartPath "git-annex" <$> userConfigDir + installAutoStart program autostartfile + ) +#endif + + sshdir <- sshDir + let runshell var = "exec " ++ base "runshell " ++ var + let rungitannexshell var = runshell $ "git-annex-shell -c \"" ++ var ++ "\"" + + installWrapper (sshdir "git-annex-shell") $ unlines + [ shebang_local + , "set -e" + , "if [ \"x$SSH_ORIGINAL_COMMAND\" != \"x\" ]; then" + , rungitannexshell "$SSH_ORIGINAL_COMMAND" + , "else" + , rungitannexshell "$@" + , "fi" + ] + installWrapper (sshdir "git-annex-wrapper") $ unlines + [ shebang_local + , "set -e" + , runshell "\"$@\"" + ] + + installFileManagerHooks program + +installWrapper :: FilePath -> String -> IO () +installWrapper file content = do + curr <- catchDefaultIO "" $ readFileStrict file + when (curr /= content) $ do + createDirectoryIfMissing True (parentDir file) + viaTmp writeFile file content + modifyFileMode file $ addModes [ownerExecuteMode] + +installFileManagerHooks :: FilePath -> IO () +#ifdef linux_HOST_OS +installFileManagerHooks program = unlessM osAndroid $ do + let actions = ["get", "drop", "undo"] + + -- Gnome + nautilusScriptdir <- (\d -> d "nautilus" "scripts") <$> userDataDir + createDirectoryIfMissing True nautilusScriptdir + forM_ actions $ + genNautilusScript nautilusScriptdir + + -- KDE + userdata <- userDataDir + let kdeServiceMenusdir = userdata "kservices5" "ServiceMenus" + createDirectoryIfMissing True kdeServiceMenusdir + writeFile (kdeServiceMenusdir "git-annex.desktop") + (kdeDesktopFile actions) + where + genNautilusScript scriptdir action = + installscript (scriptdir scriptname action) $ unlines + [ shebang_local + , autoaddedcomment + , "exec " ++ program ++ " " ++ action ++ " --notify-start --notify-finish -- \"$@\"" + ] + scriptname action = "git-annex " ++ action + installscript f c = whenM (safetoinstallscript f) $ do + writeFile f c + modifyFileMode f $ addModes [ownerExecuteMode] + safetoinstallscript f = catchDefaultIO True $ + elem autoaddedcomment . lines <$> readFileStrict f + autoaddedcomment = "# " ++ autoaddedmsg ++ " (To disable, chmod 600 this file.)" + autoaddedmsg = "Automatically added by git-annex, do not edit." + + kdeDesktopFile actions = unlines $ concat $ + kdeDesktopHeader actions : map kdeDesktopAction actions + kdeDesktopHeader actions = + [ "# " ++ autoaddedmsg + , "[Desktop Entry]" + , "Type=Service" + , "ServiceTypes=all/allfiles" + , "MimeType=all/all;" + , "Actions=" ++ intercalate ";" (map kdeDesktopSection actions) + , "X-KDE-Priority=TopLevel" + , "X-KDE-Submenu=Git-Annex" + , "X-KDE-Icon=git-annex" + , "X-KDE-ServiceTypes=KonqPopupMenu/Plugin" + ] + kdeDesktopSection command = "GitAnnex" ++ command + kdeDesktopAction command = + [ "" + , "[Desktop Action " ++ kdeDesktopSection command ++ "]" + , "Name=" ++ command + , "Icon=git-annex" + , unwords + [ "Exec=sh -c 'cd \"$(dirname \"$1\")\" &&" + , program + , command + , "--notify-start --notify-finish -- \"$1\"'" + , "false" -- this becomes $0 in sh, so unused + , "%f" + ] + ] +#else +installFileManagerHooks _ = noop +#endif + +{- Returns a cleaned up environment that lacks settings used to make the + - standalone builds use their bundled libraries and programs. + - Useful when calling programs not included in the standalone builds. + - + - For a non-standalone build, returns Nothing. + -} +cleanEnvironment :: IO (Maybe [(String, String)]) +cleanEnvironment = clean <$> getEnvironment + where + clean environ + | null vars = Nothing + | otherwise = Just $ catMaybes $ map (restoreorig environ) environ + | otherwise = Nothing + where + vars = words $ fromMaybe "" $ + lookup "GIT_ANNEX_STANDLONE_ENV" environ + restoreorig oldenviron p@(k, _v) + | k `elem` vars = case lookup ("ORIG_" ++ k) oldenviron of + (Just v') + | not (null v') -> Just (k, v') + _ -> Nothing + | otherwise = Just p diff --git a/Assistant/Install/AutoStart.hs b/Assistant/Install/AutoStart.hs new file mode 100644 index 0000000000..99c2c0e41c --- /dev/null +++ b/Assistant/Install/AutoStart.hs @@ -0,0 +1,40 @@ +{- Assistant autostart file installation + - + - Copyright 2012 Joey Hess + - + - Licensed under the GNU GPL version 3 or higher. + -} + +{-# LANGUAGE CPP #-} +{-# OPTIONS_GHC -fno-warn-tabs #-} + +module Assistant.Install.AutoStart where + +import Utility.FreeDesktop +#ifdef darwin_HOST_OS +import Utility.OSX +import Utility.Path +import Utility.Directory +#endif + +installAutoStart :: FilePath -> FilePath -> IO () +installAutoStart command file = do +#ifdef darwin_HOST_OS + createDirectoryIfMissing True (parentDir file) + writeFile file $ genOSXAutoStartFile osxAutoStartLabel command + ["assistant", "--autostart"] +#else + writeDesktopMenuFile (fdoAutostart command) file +#endif + +osxAutoStartLabel :: String +osxAutoStartLabel = "com.branchable.git-annex.assistant" + +fdoAutostart :: FilePath -> DesktopEntry +fdoAutostart command = genDesktopEntry + "Git Annex Assistant" + "Autostart" + False + (command ++ " assistant --autostart") + Nothing + [] diff --git a/Assistant/Install/Menu.hs b/Assistant/Install/Menu.hs new file mode 100644 index 0000000000..dd21ee117e --- /dev/null +++ b/Assistant/Install/Menu.hs @@ -0,0 +1,48 @@ +{- Assistant menu installation. + - + - Copyright 2013 Joey Hess + - + - Licensed under the GNU GPL version 3 or higher. + -} + +{-# LANGUAGE CPP #-} +{-# OPTIONS_GHC -fno-warn-tabs #-} + +module Assistant.Install.Menu where + +import Common + +import Utility.FreeDesktop + +installMenu :: FilePath -> FilePath -> FilePath -> FilePath -> IO () +#ifdef darwin_HOST_OS +installMenu _command _menufile _iconsrcdir _icondir = return () +#else +installMenu command menufile iconsrcdir icondir = do + writeDesktopMenuFile (fdoDesktopMenu command) menufile + installIcon (iconsrcdir "logo.svg") $ + iconFilePath (iconBaseName ++ ".svg") "scalable" icondir + installIcon (iconsrcdir "logo_16x16.png") $ + iconFilePath (iconBaseName ++ ".png") "16x16" icondir +#endif + +{- The command can be either just "git-annex", or the full path to use + - to run it. -} +fdoDesktopMenu :: FilePath -> DesktopEntry +fdoDesktopMenu command = genDesktopEntry + "Git Annex" + "Track and sync the files in your Git Annex" + False + (command ++ " webapp") + (Just iconBaseName) + ["Network", "FileTransfer"] + +installIcon :: FilePath -> FilePath -> IO () +installIcon src dest = do + createDirectoryIfMissing True (parentDir dest) + withBinaryFile src ReadMode $ \hin -> + withBinaryFile dest WriteMode $ \hout -> + hGetContents hin >>= hPutStr hout + +iconBaseName :: String +iconBaseName = "git-annex" diff --git a/Assistant/MakeRemote.hs b/Assistant/MakeRemote.hs new file mode 100644 index 0000000000..43b046bc97 --- /dev/null +++ b/Assistant/MakeRemote.hs @@ -0,0 +1,172 @@ +{- git-annex assistant remote creation utilities + - + - Copyright 2012, 2013 Joey Hess + - + - Licensed under the GNU GPL version 3 or higher. + -} + +module Assistant.MakeRemote where + +import Assistant.Common +import Assistant.Ssh +import qualified Types.Remote as R +import qualified Remote +import Remote.List +import qualified Remote.Rsync as Rsync +import qualified Remote.GCrypt as GCrypt +import qualified Git +import qualified Git.Command +import qualified Annex +import qualified Annex.SpecialRemote +import Logs.UUID +import Logs.Remote +import Git.Remote +import Git.Types (RemoteName) +import Creds +import Assistant.Gpg +import Utility.Gpg (KeyId) +import Types.GitConfig + +import qualified Data.Map as M + +{- Sets up a new git or rsync remote, accessed over ssh. -} +makeSshRemote :: SshData -> Annex RemoteName +makeSshRemote sshdata = maker (sshRepoName sshdata) (genSshUrl sshdata) + where + maker + | onlyCapability sshdata RsyncCapable = makeRsyncRemote + | otherwise = makeGitRemote + +{- Runs an action that returns a name of the remote, and finishes adding it. -} +addRemote :: Annex RemoteName -> Annex Remote +addRemote a = do + name <- a + void remoteListRefresh + maybe (error "failed to add remote") return + =<< Remote.byName (Just name) + +{- Inits a rsync special remote, and returns its name. -} +makeRsyncRemote :: RemoteName -> String -> Annex String +makeRsyncRemote name location = makeRemote name location $ const $ void $ + go =<< Annex.SpecialRemote.findExisting name + where + go Nothing = setupSpecialRemote name Rsync.remote config Nothing + (Nothing, R.Init, Annex.SpecialRemote.newConfig name) + go (Just (u, c)) = setupSpecialRemote name Rsync.remote config Nothing + (Just u, R.Enable c, c) + config = M.fromList + [ ("encryption", "shared") + , ("rsyncurl", location) + , ("type", "rsync") + ] + +{- Inits a gcrypt special remote, and returns its name. -} +makeGCryptRemote :: RemoteName -> String -> KeyId -> Annex RemoteName +makeGCryptRemote remotename location keyid = + initSpecialRemote remotename GCrypt.remote Nothing $ M.fromList + [ ("type", "gcrypt") + , ("gitrepo", location) + , configureEncryption HybridEncryption + , ("keyid", keyid) + ] + +type SpecialRemoteMaker = RemoteName -> RemoteType -> Maybe CredPair -> R.RemoteConfig -> Annex RemoteName + +{- Inits a new special remote. The name is used as a suggestion, but + - will be changed if there is already a special remote with that name. -} +initSpecialRemote :: SpecialRemoteMaker +initSpecialRemote name remotetype mcreds config = go 0 + where + go :: Int -> Annex RemoteName + go n = do + let fullname = if n == 0 then name else name ++ show n + Annex.SpecialRemote.findExisting fullname >>= \case + Nothing -> setupSpecialRemote fullname remotetype config mcreds + (Nothing, R.Init, Annex.SpecialRemote.newConfig fullname) + Just _ -> go (n + 1) + +{- Enables an existing special remote. -} +enableSpecialRemote :: SpecialRemoteMaker +enableSpecialRemote name remotetype mcreds config = + Annex.SpecialRemote.findExisting name >>= \case + Nothing -> error $ "Cannot find a special remote named " ++ name + Just (u, c) -> setupSpecialRemote' False name remotetype config mcreds (Just u, R.Enable c, c) + +setupSpecialRemote :: RemoteName -> RemoteType -> R.RemoteConfig -> Maybe CredPair -> (Maybe UUID, R.SetupStage, R.RemoteConfig) -> Annex RemoteName +setupSpecialRemote = setupSpecialRemote' True + +setupSpecialRemote' :: Bool -> RemoteName -> RemoteType -> R.RemoteConfig -> Maybe CredPair -> (Maybe UUID, R.SetupStage, R.RemoteConfig) -> Annex RemoteName +setupSpecialRemote' setdesc name remotetype config mcreds (mu, ss, c) = do + {- Currently, only 'weak' ciphers can be generated from the + - assistant, because otherwise GnuPG may block once the entropy + - pool is drained, and as of now there's no way to tell the user + - to perform IO actions to refill the pool. -} + let weakc = M.insert "highRandomQuality" "false" $ M.union config c + dummycfg <- liftIO dummyRemoteGitConfig + (c', u) <- R.setup remotetype ss mu mcreds weakc dummycfg + configSet u c' + when setdesc $ + whenM (isNothing . M.lookup u <$> uuidMap) $ + describeUUID u name + return name + +{- Returns the name of the git remote it created. If there's already a + - remote at the location, returns its name. -} +makeGitRemote :: String -> String -> Annex RemoteName +makeGitRemote basename location = makeRemote basename location $ \name -> + void $ inRepo $ Git.Command.runBool + [Param "remote", Param "add", Param name, Param location] + +{- If there's not already a remote at the location, adds it using the + - action, which is passed the name of the remote to make. + - + - Returns the name of the remote. -} +makeRemote :: String -> String -> (RemoteName -> Annex ()) -> Annex RemoteName +makeRemote basename location a = do + rs <- Annex.getGitRemotes + if not (any samelocation rs) + then do + let name = uniqueRemoteName basename 0 rs + a name + return name + else return basename + where + samelocation x = Git.repoLocation x == location + +{- Given a list of all remotes, generate an unused name for a new + - remote, adding a number if necessary. + - + - Ensures that the returned name is a legal git remote name. -} +uniqueRemoteName :: String -> Int -> [Git.Repo] -> RemoteName +uniqueRemoteName basename n rs + | null namecollision = name + | otherwise = uniqueRemoteName legalbasename (succ n) rs + where + namecollision = filter samename rs + samename x = Git.remoteName x == Just name + name + | n == 0 = legalbasename + | otherwise = legalbasename ++ show n + legalbasename = makeLegalName basename + +{- Finds a CredPair belonging to any Remote that is of a given type + - and matches some other criteria. + - + - This can be used as a default when another repository is being set up + - using the same service. + - + - A function must be provided that returns the CredPairStorage + - to use for a particular Remote's uuid. + -} +previouslyUsedCredPair + :: (UUID -> CredPairStorage) + -> RemoteType + -> (Remote -> Bool) + -> Annex (Maybe CredPair) +previouslyUsedCredPair getstorage remotetype criteria = + getM fromstorage =<< filter criteria . filter sametype <$> remoteList + where + sametype r = R.typename (R.remotetype r) == R.typename remotetype + fromstorage r = do + let storage = getstorage (R.uuid r) + getRemoteCredPair (R.config r) (R.gitconfig r) storage diff --git a/Assistant/Monad.hs b/Assistant/Monad.hs new file mode 100644 index 0000000000..403ee16a8c --- /dev/null +++ b/Assistant/Monad.hs @@ -0,0 +1,148 @@ +{- git-annex assistant monad + - + - Copyright 2012 Joey Hess + - + - Licensed under the GNU GPL version 3 or higher. + -} + +{-# LANGUAGE GeneralizedNewtypeDeriving, MultiParamTypeClasses #-} + +module Assistant.Monad ( + Assistant, + AssistantData(..), + newAssistantData, + runAssistant, + getAssistant, + LiftAnnex, + liftAnnex, + (<~>), + (<<~), + asIO, + asIO1, + asIO2, + ThreadName, + debug, + notice +) where + +import "mtl" Control.Monad.Reader +import System.Log.Logger + +import Annex.Common +import Assistant.Types.ThreadedMonad +import Assistant.Types.DaemonStatus +import Assistant.Types.ScanRemotes +import Assistant.Types.TransferQueue +import Assistant.Types.TransferSlots +import Assistant.Types.TransferrerPool +import Assistant.Types.Pushes +import Assistant.Types.BranchChange +import Assistant.Types.Commits +import Assistant.Types.Changes +import Assistant.Types.RepoProblem +import Assistant.Types.ThreadName +import Assistant.Types.RemoteControl +import Assistant.Types.CredPairCache + +newtype Assistant a = Assistant { mkAssistant :: ReaderT AssistantData IO a } + deriving ( + Monad, + MonadIO, + MonadReader AssistantData, + Functor, + Applicative + ) + +data AssistantData = AssistantData + { threadName :: ThreadName + , threadState :: ThreadState + , daemonStatusHandle :: DaemonStatusHandle + , scanRemoteMap :: ScanRemoteMap + , transferQueue :: TransferQueue + , transferSlots :: TransferSlots + , transferrerPool :: TransferrerPool + , failedPushMap :: FailedPushMap + , failedExportMap :: FailedPushMap + , commitChan :: CommitChan + , exportCommitChan :: CommitChan + , changePool :: ChangePool + , repoProblemChan :: RepoProblemChan + , branchChangeHandle :: BranchChangeHandle + , remoteControl :: RemoteControl + , credPairCache :: CredPairCache + } + +newAssistantData :: ThreadState -> DaemonStatusHandle -> IO AssistantData +newAssistantData st dstatus = AssistantData + <$> pure (ThreadName "main") + <*> pure st + <*> pure dstatus + <*> newScanRemoteMap + <*> newTransferQueue + <*> newTransferSlots + <*> newTransferrerPool (checkNetworkConnections dstatus) + <*> newFailedPushMap + <*> newFailedPushMap + <*> newCommitChan + <*> newCommitChan + <*> newChangePool + <*> newRepoProblemChan + <*> newBranchChangeHandle + <*> newRemoteControl + <*> newCredPairCache + +runAssistant :: AssistantData -> Assistant a -> IO a +runAssistant d a = runReaderT (mkAssistant a) d + +getAssistant :: (AssistantData -> a) -> Assistant a +getAssistant = reader + +{- Using a type class for lifting into the annex monad allows + - easily lifting to it from multiple different monads. -} +class LiftAnnex m where + liftAnnex :: Annex a -> m a + +{- Runs an action in the git-annex monad. Note that the same monad state + - is shared among all assistant threads, so only one of these can run at + - a time. Therefore, long-duration actions should be avoided. -} +instance LiftAnnex Assistant where + liftAnnex a = do + st <- reader threadState + liftIO $ runThreadState st a + +{- Runs an IO action, passing it an IO action that runs an Assistant action. -} +(<~>) :: (IO a -> IO b) -> Assistant a -> Assistant b +io <~> a = do + d <- reader id + liftIO $ io $ runAssistant d a + +{- Creates an IO action that will run an Assistant action when run. -} +asIO :: Assistant a -> Assistant (IO a) +asIO a = do + d <- reader id + return $ runAssistant d a + +asIO1 :: (a -> Assistant b) -> Assistant (a -> IO b) +asIO1 a = do + d <- reader id + return $ \v -> runAssistant d $ a v + +asIO2 :: (a -> b -> Assistant c) -> Assistant (a -> b -> IO c) +asIO2 a = do + d <- reader id + return $ \v1 v2 -> runAssistant d (a v1 v2) + +{- Runs an IO action on a selected field of the AssistantData. -} +(<<~) :: (a -> IO b) -> (AssistantData -> a) -> Assistant b +io <<~ v = reader v >>= liftIO . io + +debug :: [String] -> Assistant () +debug = logaction debugM + +notice :: [String] -> Assistant () +notice = logaction noticeM + +logaction :: (String -> String -> IO ()) -> [String] -> Assistant () +logaction a ws = do + ThreadName name <- getAssistant threadName + liftIO $ a name $ unwords $ (name ++ ":") : ws diff --git a/Assistant/NamedThread.hs b/Assistant/NamedThread.hs new file mode 100644 index 0000000000..22125810ca --- /dev/null +++ b/Assistant/NamedThread.hs @@ -0,0 +1,99 @@ +{- git-annex assistant named threads. + - + - Copyright 2012 Joey Hess + - + - Licensed under the GNU GPL version 3 or higher. + -} + +{-# LANGUAGE CPP #-} + +module Assistant.NamedThread where + +import Annex.Common +import Assistant.Types.NamedThread +import Assistant.Types.ThreadName +import Assistant.Types.DaemonStatus +import Assistant.Types.UrlRenderer +import Assistant.DaemonStatus +import Assistant.Monad +import Utility.NotificationBroadcaster + +import Control.Concurrent +import Control.Concurrent.Async +import qualified Data.Map.Strict as M +import qualified Control.Exception as E + +#ifdef WITH_WEBAPP +import Assistant.WebApp.Types +import Assistant.Types.Alert +import Assistant.Alert +import qualified Data.Text as T +#endif + +{- Starts a named thread, if it's not already running. + - + - Named threads are run by a management thread, so if they crash + - an alert is displayed, allowing the thread to be restarted. -} +startNamedThread :: UrlRenderer -> NamedThread -> Assistant () +startNamedThread urlrenderer (NamedThread afterstartupsanitycheck name a) = + M.lookup name . startedThreads <$> getDaemonStatus >>= \case + Nothing -> start + Just (aid, _) -> do + r <- liftIO (E.try (poll aid) :: IO (Either E.SomeException (Maybe (Either E.SomeException ())))) + case r of + Right Nothing -> noop + _ -> start + where + start + | afterstartupsanitycheck = do + status <- getDaemonStatus + h <- liftIO $ newNotificationHandle False $ + startupSanityCheckNotifier status + startwith $ runmanaged $ + liftIO $ waitNotification h + | otherwise = startwith $ runmanaged noop + startwith runner = do + d <- getAssistant id + aid <- liftIO $ runner $ d { threadName = name } + restart <- asIO $ startNamedThread urlrenderer (NamedThread False name a) + modifyDaemonStatus_ $ \s -> s + { startedThreads = M.insert name (aid, restart) (startedThreads s) } + runmanaged first d = do + aid <- async $ runAssistant d $ do + void first + a + void $ forkIO $ manager d aid + return aid + manager d aid = (E.try (wait aid) :: IO (Either E.SomeException ())) >>= \case + Right _ -> noop + Left e -> do + let msg = unwords + [ fromThreadName $ threadName d + , "crashed:", show e + ] + hPutStrLn stderr msg +#ifdef WITH_WEBAPP + button <- runAssistant d $ mkAlertButton True + (T.pack "Restart Thread") + urlrenderer + (RestartThreadR name) + runAssistant d $ void $ addAlert $ + (warningAlert (fromThreadName name) msg) + { alertButtons = [button] } +#endif + +namedThreadId :: NamedThread -> Assistant (Maybe ThreadId) +namedThreadId (NamedThread _ name _) = do + m <- startedThreads <$> getDaemonStatus + return $ asyncThreadId . fst <$> M.lookup name m + +{- Waits for all named threads that have been started to finish. + - + - Note that if a named thread crashes, it will probably + - cause this to crash as well. Also, named threads that are started + - after this is called will not be waited on. -} +waitNamedThreads :: Assistant () +waitNamedThreads = do + m <- startedThreads <$> getDaemonStatus + liftIO $ mapM_ (wait . fst) $ M.elems m + diff --git a/Assistant/Pairing.hs b/Assistant/Pairing.hs new file mode 100644 index 0000000000..34c5aab1f6 --- /dev/null +++ b/Assistant/Pairing.hs @@ -0,0 +1,103 @@ +{- git-annex assistant repo pairing, core data types + - + - Copyright 2012 Joey Hess + - + - Licensed under the GNU GPL version 3 or higher. + -} + +{-# LANGUAGE CPP #-} + +module Assistant.Pairing where + +import Annex.Common +import Utility.Verifiable +import Assistant.Ssh + +import Control.Concurrent +import Network.Socket +import Data.Char +import qualified Data.Text as T + +data PairStage + {- "I'll pair with anybody who shares the secret that can be used + - to verify this request." -} + = PairReq + {- "I've verified your request, and you can verify this to see + - that I know the secret. I set up your ssh key already. + - Here's mine for you to set up." -} + | PairAck + {- "I saw your PairAck; you can stop sending them." -} + | PairDone + deriving (Eq, Read, Show, Ord, Enum) + +newtype PairMsg = PairMsg (Verifiable (PairStage, PairData, SomeAddr)) + deriving (Eq, Read, Show) + +verifiedPairMsg :: PairMsg -> PairingInProgress -> Bool +verifiedPairMsg (PairMsg m) pip = verify m $ inProgressSecret pip + +fromPairMsg :: PairMsg -> Verifiable (PairStage, PairData, SomeAddr) +fromPairMsg (PairMsg m) = m + +pairMsgStage :: PairMsg -> PairStage +pairMsgStage (PairMsg (Verifiable (s, _, _) _)) = s + +pairMsgData :: PairMsg -> PairData +pairMsgData (PairMsg (Verifiable (_, d, _) _)) = d + +pairMsgAddr :: PairMsg -> SomeAddr +pairMsgAddr (PairMsg (Verifiable (_, _, a) _)) = a + +data PairData = PairData + -- uname -n output, not a full domain name + { remoteHostName :: Maybe HostName + , remoteUserName :: UserName + , remoteDirectory :: FilePath + , remoteSshPubKey :: SshPubKey + , pairUUID :: UUID + } + deriving (Eq, Read, Show) + +checkSane :: PairData -> Bool +checkSane p = all (not . any isControl) + [ fromMaybe "" (remoteHostName p) + , remoteUserName p + , remoteDirectory p + , remoteSshPubKey p + , fromUUID (pairUUID p) + ] + +type UserName = String + +{- A pairing that is in progress has a secret, a thread that is + - broadcasting pairing messages, and a SshKeyPair that has not yet been + - set up on disk. -} +data PairingInProgress = PairingInProgress + { inProgressSecret :: Secret + , inProgressThreadId :: Maybe ThreadId + , inProgressSshKeyPair :: SshKeyPair + , inProgressPairData :: PairData + , inProgressPairStage :: PairStage + } + deriving (Show) + +data AddrClass = IPv4AddrClass | IPv6AddrClass + +data SomeAddr = IPv4Addr HostAddress +{- My Android build of the Network library does not currently have IPV6 + - support. -} +#ifndef __ANDROID__ + | IPv6Addr HostAddress6 +#endif + deriving (Ord, Eq, Read, Show) + +{- This contains the whole secret, just lightly obfuscated to make it not + - too obvious. It's only displayed in the user's web browser. -} +newtype SecretReminder = SecretReminder [Int] + deriving (Show, Eq, Ord, Read) + +toSecretReminder :: T.Text -> SecretReminder +toSecretReminder = SecretReminder . map ord . T.unpack + +fromSecretReminder :: SecretReminder -> T.Text +fromSecretReminder (SecretReminder s) = T.pack $ map chr s diff --git a/Assistant/Pairing/MakeRemote.hs b/Assistant/Pairing/MakeRemote.hs new file mode 100644 index 0000000000..22baffe67f --- /dev/null +++ b/Assistant/Pairing/MakeRemote.hs @@ -0,0 +1,98 @@ +{- git-annex assistant pairing remote creation + - + - Copyright 2012 Joey Hess + - + - Licensed under the GNU GPL version 3 or higher. + -} + +module Assistant.Pairing.MakeRemote where + +import Assistant.Common +import Assistant.Ssh +import Assistant.Pairing +import Assistant.Pairing.Network +import Assistant.MakeRemote +import Assistant.Sync +import Config.Cost +import Config +import qualified Types.Remote as Remote + +import Network.Socket +import qualified Data.Text as T + +{- Authorized keys are set up before pairing is complete, so that the other + - side can immediately begin syncing. -} +setupAuthorizedKeys :: PairMsg -> FilePath -> IO () +setupAuthorizedKeys msg repodir = case validateSshPubKey $ remoteSshPubKey $ pairMsgData msg of + Left err -> error err + Right pubkey -> do + absdir <- absPath repodir + unlessM (liftIO $ addAuthorizedKeys True absdir pubkey) $ + error "failed setting up ssh authorized keys" + +{- When local pairing is complete, this is used to set up the remote for + - the host we paired with. -} +finishedLocalPairing :: PairMsg -> SshKeyPair -> Assistant () +finishedLocalPairing msg keypair = do + sshdata <- liftIO $ installSshKeyPair keypair =<< pairMsgToSshData msg + {- Ensure that we know the ssh host key for the host we paired with. + - If we don't, ssh over to get it. -} + liftIO $ unlessM (knownHost $ sshHostName sshdata) $ + void $ sshTranscript + [ sshOpt "StrictHostKeyChecking" "no" + , sshOpt "NumberOfPasswordPrompts" "0" + , "-n" + ] + (genSshHost (sshHostName sshdata) (sshUserName sshdata)) + ("git-annex-shell -c configlist " ++ T.unpack (sshDirectory sshdata)) + Nothing + r <- liftAnnex $ addRemote $ makeSshRemote sshdata + repo <- liftAnnex $ Remote.getRepo r + liftAnnex $ setRemoteCost repo semiExpensiveRemoteCost + syncRemote r + +{- Mostly a straightforward conversion. Except: + - * Determine the best hostname to use to contact the host. + - * Strip leading ~/ from the directory name. + -} +pairMsgToSshData :: PairMsg -> IO SshData +pairMsgToSshData msg = do + let d = pairMsgData msg + hostname <- liftIO $ bestHostName msg + let dir = case remoteDirectory d of + ('~':'/':v) -> v + v -> v + return SshData + { sshHostName = T.pack hostname + , sshUserName = Just (T.pack $ remoteUserName d) + , sshDirectory = T.pack dir + , sshRepoName = genSshRepoName hostname dir + , sshPort = 22 + , needsPubKey = True + , sshCapabilities = [GitAnnexShellCapable, GitCapable, RsyncCapable] + , sshRepoUrl = Nothing + } + +{- Finds the best hostname to use for the host that sent the PairMsg. + - + - If remoteHostName is set, tries to use a .local address based on it. + - That's the most robust, if this system supports .local. + - Otherwise, looks up the hostname in the DNS for the remoteAddress, + - if any. May fall back to remoteAddress if there's no DNS. Ugh. -} +bestHostName :: PairMsg -> IO HostName +bestHostName msg = case remoteHostName $ pairMsgData msg of + Just h -> do + let localname = h ++ ".local" + addrs <- catchDefaultIO [] $ + getAddrInfo Nothing (Just localname) Nothing + maybe fallback (const $ return localname) (headMaybe addrs) + Nothing -> fallback + where + fallback = do + let a = pairMsgAddr msg + let sockaddr = case a of + IPv4Addr addr -> SockAddrInet (fromInteger 0) addr + IPv6Addr addr -> SockAddrInet6 (fromInteger 0) 0 addr 0 + fromMaybe (showAddr a) + <$> catchDefaultIO Nothing + (fst <$> getNameInfo [] True False sockaddr) diff --git a/Assistant/Pairing/Network.hs b/Assistant/Pairing/Network.hs new file mode 100644 index 0000000000..09f0fc3207 --- /dev/null +++ b/Assistant/Pairing/Network.hs @@ -0,0 +1,132 @@ +{- git-annex assistant pairing network code + - + - All network traffic is sent over multicast UDP. For reliability, + - each message is repeated until acknowledged. This is done using a + - thread, that gets stopped before the next message is sent. + - + - Copyright 2012 Joey Hess + - + - Licensed under the GNU GPL version 3 or higher. + -} + +module Assistant.Pairing.Network where + +import Assistant.Common +import Assistant.Pairing +import Assistant.DaemonStatus +import Utility.ThreadScheduler +import Utility.Verifiable + +import Network.Multicast +import Network.Info +import Network.Socket +import qualified Network.Socket.ByteString as B +import qualified Data.ByteString.UTF8 as BU8 +import qualified Data.Map as M +import Control.Concurrent + +{- This is an arbitrary port in the dynamic port range, that could + - conceivably be used for some other broadcast messages. + - If so, hope they ignore the garbage from us; we'll certianly + - ignore garbage from them. Wild wild west. -} +pairingPort :: PortNumber +pairingPort = 55556 + +{- Goal: Reach all hosts on the same network segment. + - Method: Use same address that avahi uses. Other broadcast addresses seem + - to not be let through some routers. -} +multicastAddress :: AddrClass -> HostName +multicastAddress IPv4AddrClass = "224.0.0.251" +multicastAddress IPv6AddrClass = "ff02::fb" + +{- Multicasts a message repeatedly on all interfaces, with a 2 second + - delay between each transmission. The message is repeated forever + - unless a number of repeats is specified. + - + - The remoteHostAddress is set to the interface's IP address. + - + - Note that new sockets are opened each time. This is hardly efficient, + - but it allows new network interfaces to be used as they come up. + - On the other hand, the expensive DNS lookups are cached. + -} +multicastPairMsg :: Maybe Int -> Secret -> PairData -> PairStage -> IO () +multicastPairMsg repeats secret pairdata stage = go M.empty repeats + where + go _ (Just 0) = noop + go cache n = do + addrs <- activeNetworkAddresses + let cache' = updatecache cache addrs + mapM_ (sendinterface cache') addrs + threadDelaySeconds (Seconds 2) + go cache' $ pred <$> n + {- The multicast library currently chokes on ipv6 addresses. -} + sendinterface _ (IPv6Addr _) = noop + sendinterface cache i = void $ tryIO $ + withSocketsDo $ bracket setup cleanup use + where + setup = multicastSender (multicastAddress IPv4AddrClass) pairingPort + cleanup (sock, _) = close sock -- FIXME does not work + use (sock, addr) = do + setInterface sock (showAddr i) + maybe noop + (\s -> void $ B.sendTo sock (BU8.fromString s) addr) + (M.lookup i cache) + updatecache cache [] = cache + updatecache cache (i:is) + | M.member i cache = updatecache cache is + | otherwise = updatecache (M.insert i (show $ mkmsg i) cache) is + mkmsg addr = PairMsg $ + mkVerifiable (stage, pairdata, addr) secret + +startSending :: PairingInProgress -> PairStage -> (PairStage -> IO ()) -> Assistant () +startSending pip stage sender = do + a <- asIO start + void $ liftIO $ forkIO a + where + start = do + tid <- liftIO myThreadId + let pip' = pip { inProgressPairStage = stage, inProgressThreadId = Just tid } + oldpip <- modifyDaemonStatus $ + \s -> (s { pairingInProgress = Just pip' }, pairingInProgress s) + maybe noop stopold oldpip + liftIO $ sender stage + stopold = maybe noop (liftIO . killThread) . inProgressThreadId + +stopSending :: PairingInProgress -> Assistant () +stopSending pip = do + maybe noop (liftIO . killThread) $ inProgressThreadId pip + modifyDaemonStatus_ $ \s -> s { pairingInProgress = Nothing } + +class ToSomeAddr a where + toSomeAddr :: a -> SomeAddr + +instance ToSomeAddr IPv4 where + toSomeAddr (IPv4 a) = IPv4Addr a + +instance ToSomeAddr IPv6 where + toSomeAddr (IPv6 o1 o2 o3 o4) = IPv6Addr (o1, o2, o3, o4) + +showAddr :: SomeAddr -> HostName +showAddr (IPv4Addr a) = show $ IPv4 a +showAddr (IPv6Addr (o1, o2, o3, o4)) = show $ IPv6 o1 o2 o3 o4 + +activeNetworkAddresses :: IO [SomeAddr] +activeNetworkAddresses = filter (not . all (`elem` "0.:") . showAddr) + . concatMap (\ni -> [toSomeAddr $ ipv4 ni, toSomeAddr $ ipv6 ni]) + <$> getNetworkInterfaces + +{- A human-visible description of the repository being paired with. + - Note that the repository's description is not shown to the user, because + - it could be something like "my repo", which is confusing when pairing + - with someone else's repo. However, this has the same format as the + - default decription of a repo. -} +pairRepo :: PairMsg -> String +pairRepo msg = concat + [ remoteUserName d + , "@" + , fromMaybe (showAddr $ pairMsgAddr msg) (remoteHostName d) + , ":" + , remoteDirectory d + ] + where + d = pairMsgData msg diff --git a/Assistant/Pushes.hs b/Assistant/Pushes.hs new file mode 100644 index 0000000000..61891ea28c --- /dev/null +++ b/Assistant/Pushes.hs @@ -0,0 +1,37 @@ +{- git-annex assistant push tracking + - + - Copyright 2012 Joey Hess + - + - Licensed under the GNU GPL version 3 or higher. + -} + +module Assistant.Pushes where + +import Assistant.Common +import Assistant.Types.Pushes + +import Control.Concurrent.STM +import Data.Time.Clock +import qualified Data.Map as M + +{- Blocks until there are failed pushes. + - Returns Remotes whose pushes failed a given time duration or more ago. + - (This may be an empty list.) -} +getFailedPushesBefore :: NominalDiffTime -> FailedPushMap -> Assistant [Remote] +getFailedPushesBefore duration v = liftIO $ do + m <- atomically $ readTMVar v + now <- getCurrentTime + return $ M.keys $ M.filter (not . toorecent now) m + where + toorecent now time = now `diffUTCTime` time < duration + +{- Modifies the map. -} +changeFailedPushMap :: FailedPushMap -> (PushMap -> PushMap) -> Assistant () +changeFailedPushMap v f = liftIO $ atomically $ + store . f . fromMaybe M.empty =<< tryTakeTMVar v + where + {- tryTakeTMVar empties the TMVar; refill it only if + - the modified map is not itself empty -} + store m + | m == M.empty = noop + | otherwise = putTMVar v $! m diff --git a/Assistant/RemoteControl.hs b/Assistant/RemoteControl.hs new file mode 100644 index 0000000000..1016f1169b --- /dev/null +++ b/Assistant/RemoteControl.hs @@ -0,0 +1,21 @@ +{- git-annex assistant RemoteDaemon control + - + - Copyright 2014 Joey Hess + - + - Licensed under the GNU GPL version 3 or higher. + -} + +module Assistant.RemoteControl ( + sendRemoteControl, + RemoteDaemon.Consumed(..) +) where + +import Assistant.Common +import qualified RemoteDaemon.Types as RemoteDaemon + +import Control.Concurrent + +sendRemoteControl :: RemoteDaemon.Consumed -> Assistant () +sendRemoteControl msg = do + clicker <- getAssistant remoteControl + liftIO $ writeChan clicker msg diff --git a/Assistant/Repair.hs b/Assistant/Repair.hs new file mode 100644 index 0000000000..29bdc44f1c --- /dev/null +++ b/Assistant/Repair.hs @@ -0,0 +1,159 @@ +{- git-annex assistant repository repair + - + - Copyright 2013 Joey Hess + - + - Licensed under the GNU GPL version 3 or higher. + -} + +{-# LANGUAGE CPP #-} + +module Assistant.Repair where + +import Assistant.Common +import Command.Repair (repairAnnexBranch, trackingOrSyncBranch) +import Git.Fsck (FsckResults, foundBroken) +import Git.Repair (runRepairOf) +import qualified Git +import qualified Remote +import qualified Types.Remote as Remote +import Logs.FsckResults +import Annex.UUID +import Utility.Batch +import Annex.Path +import Assistant.Sync +import Assistant.Alert +import Assistant.DaemonStatus +import Assistant.Types.UrlRenderer +#ifdef WITH_WEBAPP +import Assistant.WebApp.Types +import qualified Data.Text as T +#endif +import qualified Utility.Lsof as Lsof +import Utility.ThreadScheduler + +import Control.Concurrent.Async + +{- When the FsckResults require a repair, tries to do a non-destructive + - repair. If that fails, pops up an alert. -} +repairWhenNecessary :: UrlRenderer -> UUID -> Maybe Remote -> FsckResults -> Assistant Bool +repairWhenNecessary urlrenderer u mrmt fsckresults + | foundBroken fsckresults = do + liftAnnex $ writeFsckResults u fsckresults + repodesc <- liftAnnex $ Remote.prettyUUID u + ok <- alertWhile (repairingAlert repodesc) + (runRepair u mrmt False) +#ifdef WITH_WEBAPP + unless ok $ do + button <- mkAlertButton True (T.pack "Click Here") urlrenderer $ + RepairRepositoryR u + void $ addAlert $ brokenRepositoryAlert [button] +#endif + return ok + | otherwise = return False + +runRepair :: UUID -> Maybe Remote -> Bool -> Assistant Bool +runRepair u mrmt destructiverepair = do + fsckresults <- liftAnnex $ readFsckResults u + myu <- liftAnnex getUUID + ok <- if u == myu + then localrepair fsckresults + else remoterepair fsckresults + liftAnnex $ clearFsckResults u + debug [ "Repaired", show u, show ok ] + + return ok + where + localrepair fsckresults = do + -- Stop the watcher from running while running repairs. + changeSyncable Nothing False + + -- This intentionally runs the repair inside the Annex + -- monad, which is not strictly necessary, but keeps + -- other threads that might be trying to use the Annex + -- from running until it completes. + ok <- liftAnnex $ repair fsckresults Nothing + + -- Run a background fast fsck if a destructive repair had + -- to be done, to ensure that the git-annex branch + -- reflects the current state of the repo. + when destructiverepair $ + backgroundfsck [ Param "--fast" ] + + -- Start the watcher running again. This also triggers it to + -- do a startup scan, which is especially important if the + -- git repo repair removed files from the index file. Those + -- files will be seen as new, and re-added to the repository. + when (ok || destructiverepair) $ + changeSyncable Nothing True + + return ok + + remoterepair fsckresults = case Remote.repairRepo =<< mrmt of + Nothing -> return False + Just mkrepair -> do + thisrepopath <- liftIO . absPath + =<< liftAnnex (fromRepo Git.repoPath) + a <- liftAnnex $ mkrepair $ + repair fsckresults (Just thisrepopath) + liftIO $ catchBoolIO a + + repair fsckresults referencerepo = do + (ok, modifiedbranches) <- inRepo $ + runRepairOf fsckresults trackingOrSyncBranch destructiverepair referencerepo + when destructiverepair $ + repairAnnexBranch modifiedbranches + return ok + + backgroundfsck params = liftIO $ void $ async $ do + program <- programPath + batchCommand program (Param "fsck" : params) + +{- Detect when a git lock file exists and has no git process currently + - writing to it. This strongly suggests it is a stale lock file. + - + - However, this could be on a network filesystem. Which is not very safe + - anyway (the assistant relies on being able to check when files have + - no writers to know when to commit them). Also, a few lock-file-ish + - things used by git are not kept open, particularly MERGE_HEAD. + - + - So, just in case, when the lock file appears stale, we delay for one + - minute, and check its size. If the size changed, delay for another + - minute, and so on. This will at work to detect when another machine + - is writing out a new index file, since git does so by writing the + - new content to index.lock. + - + - Returns true if locks were cleaned up. + -} +repairStaleGitLocks :: Git.Repo -> Assistant Bool +repairStaleGitLocks r = do + lockfiles <- liftIO $ filter islock <$> findgitfiles r + repairStaleLocks lockfiles + return $ not $ null lockfiles + where + findgitfiles = dirContentsRecursiveSkipping (== dropTrailingPathSeparator annexDir) True . Git.localGitDir + islock f + | "gc.pid" `isInfixOf` f = False + | ".lock" `isSuffixOf` f = True + | takeFileName f == "MERGE_HEAD" = True + | otherwise = False + +repairStaleLocks :: [FilePath] -> Assistant () +repairStaleLocks lockfiles = go =<< getsizes + where + getsize lf = catchMaybeIO $ (\s -> (lf, s)) <$> getFileSize lf + getsizes = liftIO $ catMaybes <$> mapM getsize lockfiles + go [] = return () + go l = ifM (liftIO $ null <$> Lsof.query ("--" : map fst l)) + ( do + waitforit "to check stale git lock file" + l' <- getsizes + if l' == l + then liftIO $ mapM_ nukeFile (map fst l) + else go l' + , do + waitforit "for git lock file writer" + go =<< getsizes + ) + waitforit why = do + notice ["Waiting for 60 seconds", why] + liftIO $ threadDelaySeconds $ Seconds 60 diff --git a/Assistant/RepoProblem.hs b/Assistant/RepoProblem.hs new file mode 100644 index 0000000000..32595916ec --- /dev/null +++ b/Assistant/RepoProblem.hs @@ -0,0 +1,34 @@ +{- git-annex assistant remote problem handling + - + - Copyright 2013 Joey Hess + - + - Licensed under the GNU GPL version 3 or higher. + -} + +module Assistant.RepoProblem where + +import Assistant.Common +import Assistant.Types.RepoProblem +import Utility.TList + +import Control.Concurrent.STM + +{- Gets all repositories that have problems. Blocks until there is at + - least one. -} +getRepoProblems :: Assistant [RepoProblem] +getRepoProblems = nubBy sameRepoProblem + <$> (atomically . getTList) <<~ repoProblemChan + +{- Indicates that there was a problem with a repository, and the problem + - appears to not be a transient (eg network connection) problem. + - + - If the problem is able to be repaired, the passed action will be run. + - (However, if multiple problems are reported with a single repository, + - only a single action will be run.) + -} +repoHasProblem :: UUID -> Assistant () -> Assistant () +repoHasProblem u afterrepair = do + rp <- RepoProblem + <$> pure u + <*> asIO afterrepair + (atomically . flip consTList rp) <<~ repoProblemChan diff --git a/Assistant/Restart.hs b/Assistant/Restart.hs new file mode 100644 index 0000000000..51ad7748a0 --- /dev/null +++ b/Assistant/Restart.hs @@ -0,0 +1,117 @@ +{- git-annex assistant restarting + - + - Copyright 2013 Joey Hess + - + - Licensed under the GNU GPL version 3 or higher. + -} + +{-# LANGUAGE CPP #-} + +module Assistant.Restart where + +import Assistant.Common +import Assistant.Threads.Watcher +import Assistant.DaemonStatus +import Assistant.NamedThread +import Utility.ThreadScheduler +import Utility.NotificationBroadcaster +import Utility.Url +import Utility.PID +import qualified Git.Construct +import qualified Git.Config +import qualified Annex +import qualified Git +import Annex.Path + +import Control.Concurrent +#ifndef mingw32_HOST_OS +import System.Posix (signalProcess, sigTERM) +#else +import System.Win32.Process (terminateProcessById) +#endif +import Network.URI + +{- Before the assistant can be restarted, have to remove our + - gitAnnexUrlFile and our gitAnnexPidFile. Pausing the watcher is also + - a good idea, to avoid fighting when two assistants are running in the + - same repo. + -} +prepRestart :: Assistant () +prepRestart = do + liftIO . maybe noop (`throwTo` PauseWatcher) =<< namedThreadId watchThread + liftIO . nukeFile =<< liftAnnex (fromRepo gitAnnexUrlFile) + liftIO . nukeFile =<< liftAnnex (fromRepo gitAnnexPidFile) + +{- To finish a restart, send a global redirect to the new url + - to any web browsers that are displaying the webapp. + - + - Wait for browser to update before terminating this process. -} +postRestart :: URLString -> Assistant () +postRestart url = do + modifyDaemonStatus_ $ \status -> status { globalRedirUrl = Just url } + liftIO . sendNotification . globalRedirNotifier =<< getDaemonStatus + void $ liftIO $ forkIO $ do + threadDelaySeconds (Seconds 120) + terminateSelf + +terminateSelf :: IO () +terminateSelf = +#ifndef mingw32_HOST_OS + signalProcess sigTERM =<< getPID +#else + terminateProcessById =<< getPID +#endif + +runRestart :: Assistant URLString +runRestart = liftIO . newAssistantUrl + =<< liftAnnex (Git.repoLocation <$> Annex.gitRepo) + +{- Starts up the assistant in the repository, and waits for it to create + - a gitAnnexUrlFile. Waits for the assistant to be up and listening for + - connections by testing the url. -} +newAssistantUrl :: FilePath -> IO URLString +newAssistantUrl repo = do + startAssistant repo + geturl + where + geturl = do + r <- Git.Config.read =<< Git.Construct.fromPath repo + waiturl $ gitAnnexUrlFile r + waiturl urlfile = do + v <- tryIO $ readFile urlfile + case v of + Left _ -> delayed $ waiturl urlfile + Right url -> ifM (assistantListening url) + ( return url + , delayed $ waiturl urlfile + ) + delayed a = do + threadDelay 100000 -- 1/10th of a second + a + +{- Checks if the assistant is listening on an url. + - + - Always checks http, because https with self-signed cert is problematic. + - warp-tls listens to http, in order to show an error page, so this works. + -} +assistantListening :: URLString -> IO Bool +assistantListening url = catchBoolIO $ exists url' =<< defUrlOptions + where + url' = case parseURI url of + Nothing -> url + Just uri -> show $ uri + { uriScheme = "http:" + } + +{- Does not wait for assistant to be listening for web connections. + - + - On windows, the assistant does not daemonize, which is why the forkIO is + - done. + -} +startAssistant :: FilePath -> IO () +startAssistant repo = void $ forkIO $ do + program <- programPath + (_, _, _, pid) <- + createProcess $ + (proc program ["assistant"]) { cwd = Just repo } + void $ checkSuccessProcess pid diff --git a/Assistant/ScanRemotes.hs b/Assistant/ScanRemotes.hs new file mode 100644 index 0000000000..0ce7a47ccf --- /dev/null +++ b/Assistant/ScanRemotes.hs @@ -0,0 +1,41 @@ +{- git-annex assistant remotes needing scanning + - + - Copyright 2012 Joey Hess + - + - Licensed under the GNU GPL version 3 or higher. + -} + +module Assistant.ScanRemotes where + +import Assistant.Common +import Assistant.Types.ScanRemotes +import qualified Types.Remote as Remote + +import Data.Function +import Control.Concurrent.STM +import qualified Data.Map as M + +{- Blocks until there is a remote or remotes that need to be scanned. + - + - The list has higher priority remotes listed first. -} +getScanRemote :: Assistant [(Remote, ScanInfo)] +getScanRemote = do + v <- getAssistant scanRemoteMap + liftIO $ atomically $ + reverse . sortBy (compare `on` scanPriority . snd) . M.toList + <$> takeTMVar v + +{- Adds new remotes that need scanning. -} +addScanRemotes :: Bool -> [Remote] -> Assistant () +addScanRemotes _ [] = noop +addScanRemotes full rs = do + v <- getAssistant scanRemoteMap + liftIO $ atomically $ do + m <- fromMaybe M.empty <$> tryTakeTMVar v + putTMVar v $ M.unionWith merge (M.fromList $ zip rs (map info rs)) m + where + info r = ScanInfo (-1 * Remote.cost r) full + merge x y = ScanInfo + { scanPriority = max (scanPriority x) (scanPriority y) + , fullScan = fullScan x || fullScan y + } diff --git a/Assistant/Ssh.hs b/Assistant/Ssh.hs new file mode 100644 index 0000000000..8528446160 --- /dev/null +++ b/Assistant/Ssh.hs @@ -0,0 +1,409 @@ +{- git-annex assistant ssh utilities + - + - Copyright 2012-2013 Joey Hess + - + - Licensed under the GNU GPL version 3 or higher. + -} + +module Assistant.Ssh where + +import Annex.Common +import Utility.Tmp +import Utility.Tmp.Dir +import Utility.Shell +import Utility.Rsync +import Utility.FileMode +import Utility.SshConfig +import Git.Remote +import Utility.SshHost +import Utility.Process.Transcript + +import Data.Text (Text) +import qualified Data.Text as T +import Data.Char +import Network.URI + +data SshData = SshData + { sshHostName :: Text + , sshUserName :: Maybe Text + , sshDirectory :: Text + , sshRepoName :: String + , sshPort :: Int + , needsPubKey :: Bool + , sshCapabilities :: [SshServerCapability] + , sshRepoUrl :: Maybe String + } + deriving (Read, Show, Eq) + +data SshServerCapability + = GitAnnexShellCapable -- server has git-annex-shell installed + | GitCapable -- server has git installed + | RsyncCapable -- server supports raw rsync access (not only via git-annex-shell) + | PushCapable -- repo on server is set up already, and ready to accept pushes + deriving (Read, Show, Eq) + +hasCapability :: SshData -> SshServerCapability -> Bool +hasCapability d c = c `elem` sshCapabilities d + +addCapability :: SshData -> SshServerCapability -> SshData +addCapability d c = d { sshCapabilities = c : sshCapabilities d } + +onlyCapability :: SshData -> SshServerCapability -> Bool +onlyCapability d c = all (== c) (sshCapabilities d) + +type SshPubKey = String +type SshPrivKey = String + +data SshKeyPair = SshKeyPair + { sshPubKey :: SshPubKey + , sshPrivKey :: SshPrivKey + } + +instance Show SshKeyPair where + show = sshPubKey + +{- ssh -ofoo=bar command-line option -} +sshOpt :: String -> String -> String +sshOpt k v = concat ["-o", k, "=", v] + +{- user@host or host -} +genSshHost :: Text -> Maybe Text -> SshHost +genSshHost host user = either error id $ mkSshHost $ + maybe "" (\v -> T.unpack v ++ "@") user ++ T.unpack host + +{- Generates a ssh or rsync url from a SshData. -} +genSshUrl :: SshData -> String +genSshUrl sshdata = case sshRepoUrl sshdata of + Just repourl -> repourl + Nothing -> addtrailingslash $ T.unpack $ T.concat $ + if (onlyCapability sshdata RsyncCapable) + then [u, h, T.pack ":", sshDirectory sshdata] + else [T.pack "ssh://", u, h, d] + where + u = maybe (T.pack "") (\v -> T.concat [v, T.pack "@"]) $ sshUserName sshdata + h = sshHostName sshdata + d + | T.pack "/" `T.isPrefixOf` sshDirectory sshdata = sshDirectory sshdata + | T.pack "~/" `T.isPrefixOf` sshDirectory sshdata = T.concat [T.pack "/", sshDirectory sshdata] + | otherwise = T.concat [T.pack "/~/", sshDirectory sshdata] + addtrailingslash s + | "/" `isSuffixOf` s = s + | otherwise = s ++ "/" + +{- Reverses genSshUrl -} +parseSshUrl :: String -> Maybe SshData +parseSshUrl u + | "ssh://" `isPrefixOf` u = fromssh (drop (length "ssh://") u) + | otherwise = fromrsync u + where + mkdata (userhost, dir) = Just $ SshData + { sshHostName = T.pack host + , sshUserName = if null user then Nothing else Just $ T.pack user + , sshDirectory = T.pack dir + , sshRepoName = genSshRepoName host dir + -- dummy values, cannot determine from url + , sshPort = 22 + , needsPubKey = True + , sshCapabilities = [] + , sshRepoUrl = Nothing + } + where + (user, host) = if '@' `elem` userhost + then separate (== '@') userhost + else ("", userhost) + fromrsync s + | not (rsyncUrlIsShell u) = Nothing + | otherwise = mkdata $ separate (== ':') s + fromssh = mkdata . break (== '/') + +{- Generates a git remote name, like host_dir or host -} +genSshRepoName :: String -> FilePath -> String +genSshRepoName host dir + | null dir = makeLegalName host + | otherwise = makeLegalName $ host ++ "_" ++ dir + +{- The output of ssh, including both stdout and stderr. -} +sshTranscript :: [String] -> SshHost -> String -> (Maybe String) -> IO (String, Bool) +sshTranscript opts sshhost cmd input = processTranscript "ssh" + (opts ++ [fromSshHost sshhost, cmd]) input + +{- Ensure that the ssh public key doesn't include any ssh options, like + - command=foo, or other weirdness. + - + - The returned version of the key has its comment removed. + -} +validateSshPubKey :: SshPubKey -> Either String SshPubKey +validateSshPubKey pubkey + | length (lines pubkey) == 1 = check $ words pubkey + | otherwise = Left "too many lines in ssh public key" + where + check (prefix:key:_) = checkprefix prefix (unwords [prefix, key]) + check _ = err "wrong number of words in ssh public key" + + err msg = Left $ unwords [msg, pubkey] + + checkprefix prefix validpubkey + | ssh == "ssh" && all isAlphaNum keytype = Right validpubkey + | otherwise = err "bad ssh public key prefix" + where + (ssh, keytype) = separate (== '-') prefix + +addAuthorizedKeys :: Bool -> FilePath -> SshPubKey -> IO Bool +addAuthorizedKeys gitannexshellonly dir pubkey = boolSystem "sh" + [ Param "-c" , Param $ addAuthorizedKeysCommand gitannexshellonly dir pubkey ] + +{- Should only be used within the same process that added the line; + - the layout of the line is not kepy stable across versions. -} +removeAuthorizedKeys :: Bool -> FilePath -> SshPubKey -> IO () +removeAuthorizedKeys gitannexshellonly dir pubkey = do + let keyline = authorizedKeysLine gitannexshellonly dir pubkey + sshdir <- sshDir + let keyfile = sshdir "authorized_keys" + ls <- lines <$> readFileStrict keyfile + viaTmp writeSshConfig keyfile $ unlines $ filter (/= keyline) ls + +{- Implemented as a shell command, so it can be run on remote servers over + - ssh. + - + - The ~/.ssh/git-annex-shell wrapper script is created if not already + - present. + -} +addAuthorizedKeysCommand :: Bool -> FilePath -> SshPubKey -> String +addAuthorizedKeysCommand gitannexshellonly dir pubkey = intercalate "&&" + [ "mkdir -p ~/.ssh" + , intercalate "; " + [ "if [ ! -e " ++ wrapper ++ " ]" + , "then (" ++ intercalate ";" (map echoval script) ++ ") > " ++ wrapper + , "fi" + ] + , "chmod 700 " ++ wrapper + , "touch ~/.ssh/authorized_keys" + , "chmod 600 ~/.ssh/authorized_keys" + , unwords + [ "echo" + , shellEscape $ authorizedKeysLine gitannexshellonly dir pubkey + , ">>~/.ssh/authorized_keys" + ] + ] + where + echoval v = "echo " ++ shellEscape v + wrapper = "~/.ssh/git-annex-shell" + script = + [ shebang_portable + , "set -e" + , "if [ \"x$SSH_ORIGINAL_COMMAND\" != \"x\" ]; then" + , runshell "$SSH_ORIGINAL_COMMAND" + , "else" + , runshell "$@" + , "fi" + ] + runshell var = "exec git-annex-shell -c \"" ++ var ++ "\"" + +authorizedKeysLine :: Bool -> FilePath -> SshPubKey -> String +authorizedKeysLine gitannexshellonly dir pubkey + | gitannexshellonly = limitcommand ++ pubkey + {- TODO: Locking down rsync is difficult, requiring a rather + - long perl script. -} + | otherwise = pubkey + where + limitcommand = "command=\"env GIT_ANNEX_SHELL_DIRECTORY="++shellEscape dir++" ~/.ssh/git-annex-shell\",no-agent-forwarding,no-port-forwarding,no-X11-forwarding,no-pty " + +{- Generates a ssh key pair. -} +genSshKeyPair :: IO SshKeyPair +genSshKeyPair = withTmpDir "git-annex-keygen" $ \dir -> do + ok <- boolSystem "ssh-keygen" + [ Param "-P", Param "" -- no password + , Param "-f", File $ dir "key" + ] + unless ok $ + error "ssh-keygen failed" + SshKeyPair + <$> readFile (dir "key.pub") + <*> readFile (dir "key") + +{- Installs a ssh key pair, and sets up ssh config with a mangled hostname + - that will enable use of the key. This way we avoid changing the user's + - regular ssh experience at all. Returns a modified SshData containing the + - mangled hostname. + - + - Note that the key files are put in ~/.ssh/git-annex/, rather than directly + - in ssh because of an **INSANE** behavior of gnome-keyring: It loads + - ~/.ssh/ANYTHING.pub, and uses them indiscriminately. But using this key + - for a normal login to the server will force git-annex-shell to run, + - and locks the user out. Luckily, it does not recurse into subdirectories. + - + - Similarly, IdentitiesOnly is set in the ssh config to prevent the + - ssh-agent from forcing use of a different key. + - + - Force strict host key checking to avoid repeated prompts + - when git-annex and git try to access the remote, if its + - host key has changed. + -} +installSshKeyPair :: SshKeyPair -> SshData -> IO SshData +installSshKeyPair sshkeypair sshdata = do + sshdir <- sshDir + createDirectoryIfMissing True $ parentDir $ sshdir sshPrivKeyFile sshdata + + unlessM (doesFileExist $ sshdir sshPrivKeyFile sshdata) $ + writeFileProtected (sshdir sshPrivKeyFile sshdata) (sshPrivKey sshkeypair) + unlessM (doesFileExist $ sshdir sshPubKeyFile sshdata) $ + writeFile (sshdir sshPubKeyFile sshdata) (sshPubKey sshkeypair) + + setSshConfig sshdata + [ ("IdentityFile", "~/.ssh/" ++ sshPrivKeyFile sshdata) + , ("IdentitiesOnly", "yes") + , ("StrictHostKeyChecking", "yes") + ] + +sshPrivKeyFile :: SshData -> FilePath +sshPrivKeyFile sshdata = "git-annex" "key." ++ mangleSshHostName sshdata + +sshPubKeyFile :: SshData -> FilePath +sshPubKeyFile sshdata = sshPrivKeyFile sshdata ++ ".pub" + +{- Generates an installs a new ssh key pair if one is not already + - installed. Returns the modified SshData that will use the key pair, + - and the key pair. -} +setupSshKeyPair :: SshData -> IO (SshData, SshKeyPair) +setupSshKeyPair sshdata = do + sshdir <- sshDir + mprivkey <- catchMaybeIO $ readFile (sshdir sshPrivKeyFile sshdata) + mpubkey <- catchMaybeIO $ readFile (sshdir sshPubKeyFile sshdata) + keypair <- case (mprivkey, mpubkey) of + (Just privkey, Just pubkey) -> return $ SshKeyPair + { sshPubKey = pubkey + , sshPrivKey = privkey + } + _ -> genSshKeyPair + sshdata' <- installSshKeyPair keypair sshdata + return (sshdata', keypair) + +{- Fixes git-annex ssh key pairs configured in .ssh/config + - by old versions to set IdentitiesOnly. + - + - Strategy: Search for IdentityFile lines with key.git-annex + - in their names. These are for git-annex ssh key pairs. + - Add the IdentitiesOnly line immediately after them, if not already + - present. + -} +fixSshKeyPairIdentitiesOnly :: IO () +fixSshKeyPairIdentitiesOnly = changeUserSshConfig $ unlines . go [] . lines + where + go c [] = reverse c + go c (l:[]) + | all (`isInfixOf` l) indicators = go (fixedline l:l:c) [] + | otherwise = go (l:c) [] + go c (l:next:rest) + | all (`isInfixOf` l) indicators && not ("IdentitiesOnly" `isInfixOf` next) = + go (fixedline l:l:c) (next:rest) + | otherwise = go (l:c) (next:rest) + indicators = ["IdentityFile", "key.git-annex"] + fixedline tmpl = takeWhile isSpace tmpl ++ "IdentitiesOnly yes" + +{- Add StrictHostKeyChecking to any ssh config stanzas that were written + - by git-annex. -} +fixUpSshRemotes :: IO () +fixUpSshRemotes = modifyUserSshConfig (map go) + where + go c@(HostConfig h _) + | "git-annex-" `isPrefixOf` h = fixupconfig c + | otherwise = c + go other = other + + fixupconfig c = case findHostConfigKey c "StrictHostKeyChecking" of + Nothing -> addToHostConfig c "StrictHostKeyChecking" "yes" + Just _ -> c + +{- Setups up a ssh config with a mangled hostname. + - Returns a modified SshData containing the mangled hostname. -} +setSshConfig :: SshData -> [(String, String)] -> IO SshData +setSshConfig sshdata config = do + sshdir <- sshDir + createDirectoryIfMissing True sshdir + let configfile = sshdir "config" + unlessM (catchBoolIO $ isInfixOf mangledhost <$> readFile configfile) $ do + appendFile configfile $ unlines $ + [ "" + , "# Added automatically by git-annex" + , "Host " ++ mangledhost + ] ++ map (\(k, v) -> "\t" ++ k ++ " " ++ v) + (settings ++ config) + setSshConfigMode configfile + + return $ sshdata + { sshHostName = T.pack mangledhost + , sshRepoUrl = replace orighost mangledhost + <$> sshRepoUrl sshdata + } + where + orighost = T.unpack $ sshHostName sshdata + mangledhost = mangleSshHostName sshdata + settings = + [ ("Hostname", orighost) + , ("Port", show $ sshPort sshdata) + ] + +{- This hostname is specific to a given repository on the ssh host, + - so it is based on the real hostname, the username, and the directory. + - + - The mangled hostname has the form: + - "git-annex-realhostname-username_port_dir" + - Note that "-" is only used in the realhostname and as a separator; + - this is necessary to allow unMangleSshHostName to work. + - + - Unusual characters are url encoded, but using "." rather than "%" + - (the latter has special meaning to ssh). + - + - In the username and directory, unusual characters are any + - non-alphanumerics, other than "_" + - + - The real hostname is not normally encoded at all. This is done for + - backwards compatability and to avoid unnecessary ugliness in the + - filename. However, when it contains special characters + - (notably ":" which cannot be used on some filesystems), it is url + - encoded. To indicate it was encoded, the mangled hostname + - has the form + - "git-annex-.encodedhostname-username_port_dir" + -} +mangleSshHostName :: SshData -> String +mangleSshHostName sshdata = intercalate "-" + [ "git-annex" + , escapehostname (T.unpack (sshHostName sshdata)) + , escape extra + ] + where + extra = intercalate "_" $ map T.unpack $ catMaybes + [ sshUserName sshdata + , Just $ T.pack $ show $ sshPort sshdata + , Just $ sshDirectory sshdata + ] + safe c + | isAlphaNum c = True + | c == '_' = True + | otherwise = False + escape s = replace "%" "." $ escapeURIString safe s + escapehostname s + | all (\c -> c == '.' || safe c) s = s + | otherwise = '.' : escape s + +{- Extracts the real hostname from a mangled ssh hostname. -} +unMangleSshHostName :: String -> String +unMangleSshHostName h = case splitc '-' h of + ("git":"annex":rest) -> unescape (intercalate "-" (beginning rest)) + _ -> h + where + unescape ('.':s) = unEscapeString (replace "." "%" s) + unescape s = s + +{- Does ssh have known_hosts data for a hostname? -} +knownHost :: Text -> IO Bool +knownHost hostname = do + sshdir <- sshDir + ifM (doesFileExist $ sshdir "known_hosts") + ( not . null <$> checkhost + , return False + ) + where + {- ssh-keygen -F can crash on some old known_hosts file -} + checkhost = catchDefaultIO "" $ + readProcess "ssh-keygen" ["-F", T.unpack hostname] diff --git a/Assistant/Sync.hs b/Assistant/Sync.hs new file mode 100644 index 0000000000..6792c13033 --- /dev/null +++ b/Assistant/Sync.hs @@ -0,0 +1,274 @@ +{- git-annex assistant repo syncing + - + - Copyright 2012 Joey Hess + - + - Licensed under the GNU GPL version 3 or higher. + -} + +module Assistant.Sync where + +import Assistant.Common +import Assistant.Pushes +import Assistant.Alert +import Assistant.Alert.Utility +import Assistant.DaemonStatus +import Assistant.ScanRemotes +import Assistant.RemoteControl +import qualified Command.Sync +import Utility.Parallel +import qualified Git +import qualified Git.Command +import qualified Remote +import qualified Types.Remote as Remote +import qualified Remote.List as Remote +import qualified Annex.Branch +import Annex.UUID +import Annex.TaggedPush +import Annex.Ssh +import qualified Config +import Git.Config +import Config.DynamicConfig +import Assistant.NamedThread +import Assistant.Threads.Watcher (watchThread, WatcherControl(..)) +import Assistant.TransferSlots +import Assistant.TransferQueue +import Assistant.RepoProblem +import Assistant.Commits +import Types.Transfer +import Database.Export + +import Data.Time.Clock +import qualified Data.Map as M +import Control.Concurrent + +{- Syncs with remotes that may have been disconnected for a while. + - + - First gets git in sync, and then prepares any necessary file transfers. + - + - An expensive full scan is queued when the git-annex branches of some of + - the remotes have diverged from the local git-annex branch. Otherwise, + - it's sufficient to requeue failed transfers. + - + - Also handles signaling any connectRemoteNotifiers, after the syncing is + - done, and records an export commit to make any exports be updated. + -} +reconnectRemotes :: [Remote] -> Assistant () +reconnectRemotes [] = recordExportCommit +reconnectRemotes rs = void $ do + rs' <- liftIO $ filterM (Remote.checkAvailable True) rs + unless (null rs') $ do + failedrs <- syncAction rs' (const go) + forM_ failedrs $ \r -> + whenM (liftIO $ Remote.checkAvailable False r) $ + repoHasProblem (Remote.uuid r) (syncRemote r) + mapM_ signal $ filter (`notElem` failedrs) rs' + recordExportCommit + where + gitremotes = liftAnnex $ + filterM (notspecialremote <$$> Remote.getRepo) rs + notspecialremote r + | Git.repoIsUrl r = True + | Git.repoIsLocal r = True + | Git.repoIsLocalUnknown r = True + | otherwise = False + sync currentbranch@(Just _, _) = do + (failedpull, diverged) <- manualPull currentbranch =<< gitremotes + now <- liftIO getCurrentTime + failedpush <- pushToRemotes' now =<< gitremotes + return (nub $ failedpull ++ failedpush, diverged) + {- No local branch exists yet, but we can try pulling. -} + sync (Nothing, _) = manualPull (Nothing, Nothing) =<< gitremotes + go = do + (failed, diverged) <- sync + =<< liftAnnex (join Command.Sync.getCurrBranch) + addScanRemotes diverged =<< + filterM (not <$$> liftIO . getDynamicConfig . remoteAnnexIgnore . Remote.gitconfig) rs + return failed + signal r = liftIO . mapM_ (flip tryPutMVar ()) + =<< fromMaybe [] . M.lookup (Remote.uuid r) . connectRemoteNotifiers + <$> getDaemonStatus + +{- Pushes the local sync branch to all remotes, in + - parallel, along with the git-annex branch. This is the same + - as "git annex sync", except in parallel, and will co-exist with use of + - "git annex sync". + - + - Avoids running possibly long-duration commands in the Annex monad, so + - as not to block other threads. + - + - This can fail, when the remote's sync branch (or git-annex branch) has + - been updated by some other remote pushing into it, or by the remote + - itself. To handle failure, a manual pull and merge is done, and the push + - is retried. + - + - When there's a lot of activity, we may fail more than once. + - On the other hand, we may fail because the remote is not available. + - Rather than retrying indefinitely, after the first retry we enter a + - fallback mode, where our push is guarenteed to succeed if the remote is + - reachable. If the fallback fails, the push is queued to be retried + - later. + - + - Returns any remotes that it failed to push to. + -} +pushToRemotes :: [Remote] -> Assistant [Remote] +pushToRemotes remotes = do + now <- liftIO getCurrentTime + let remotes' = filter (wantpush . Remote.gitconfig) remotes + syncAction remotes' (pushToRemotes' now) + where + wantpush gc + | remoteAnnexReadOnly gc = False + | not (remoteAnnexPush gc) = False + | otherwise = True + +pushToRemotes' :: UTCTime -> [Remote] -> Assistant [Remote] +pushToRemotes' now remotes = do + (g, branch, u) <- liftAnnex $ do + Annex.Branch.commit =<< Annex.Branch.commitMessage + (,,) + <$> gitRepo + <*> join Command.Sync.getCurrBranch + <*> getUUID + ret <- go True branch g u remotes + return ret + where + go _ (Nothing, _) _ _ _ = return [] -- no branch, so nothing to do + go _ _ _ _ [] = return [] -- no remotes, so nothing to do + go shouldretry currbranch@(Just branch, _) g u rs = do + debug ["pushing to", show rs] + (succeeded, failed) <- parallelPush g rs (push branch) + updatemap succeeded [] + if null failed + then return [] + else if shouldretry + then retry currbranch g u failed + else fallback branch g u failed + + updatemap succeeded failed = do + v <- getAssistant failedPushMap + changeFailedPushMap v $ \m -> + M.union (makemap failed) $ + M.difference m (makemap succeeded) + makemap l = M.fromList $ zip l (repeat now) + + retry currbranch g u rs = do + debug ["trying manual pull to resolve failed pushes"] + void $ manualPull currbranch rs + go False currbranch g u rs + + fallback branch g u rs = do + debug ["fallback pushing to", show rs] + (succeeded, failed) <- parallelPush g rs (taggedPush u Nothing branch) + updatemap succeeded failed + return failed + + push branch remote = Command.Sync.pushBranch remote branch + +parallelPush :: Git.Repo -> [Remote] -> (Remote -> Git.Repo -> IO Bool)-> Assistant ([Remote], [Remote]) +parallelPush g rs a = do + rgs <- liftAnnex $ mapM topush rs + (succeededrgs, failedrgs) <- liftIO $ inParallel (uncurry a) rgs + return (map fst succeededrgs, map fst failedrgs) + where + topush r = (,) + <$> pure r + <*> (Remote.getRepo r >>= \repo -> + sshOptionsTo repo (Remote.gitconfig r) g) + +{- Displays an alert while running an action that syncs with some remotes, + - and returns any remotes that it failed to sync with. + - + - Readonly remotes are also hidden (to hide the web special remote). + -} +syncAction :: [Remote] -> ([Remote] -> Assistant [Remote]) -> Assistant [Remote] +syncAction rs a + | null visibleremotes = a rs + | otherwise = do + i <- addAlert $ syncAlert visibleremotes + failed <- a rs + failed' <- filterM (not . Git.repoIsLocalUnknown <$$> liftAnnex . Remote.getRepo) failed + let succeeded = filter (`notElem` failed) visibleremotes + if null succeeded && null failed' + then removeAlert i + else updateAlertMap $ mergeAlert i $ + syncResultAlert succeeded failed' + return failed + where + visibleremotes = filter (not . Remote.readonly) rs + +{- Manually pull from remotes and merge their branches. Returns any + - remotes that it failed to pull from, and a Bool indicating + - whether the git-annex branches of the remotes and local had + - diverged before the pull. + -} +manualPull :: Command.Sync.CurrBranch -> [Remote] -> Assistant ([Remote], Bool) +manualPull currentbranch remotes = do + g <- liftAnnex gitRepo + failed <- forM remotes $ \r -> if wantpull $ Remote.gitconfig r + then do + g' <- liftAnnex $ do + repo <- Remote.getRepo r + sshOptionsTo repo (Remote.gitconfig r) g + ifM (liftIO $ Git.Command.runBool [Param "fetch", Param $ Remote.name r] g') + ( return Nothing + , return $ Just r + ) + else return Nothing + haddiverged <- liftAnnex Annex.Branch.forceUpdate + forM_ remotes $ \r -> + liftAnnex $ Command.Sync.mergeRemote r + currentbranch Command.Sync.mergeConfig def + when haddiverged $ + updateExportTreeFromLogAll + return (catMaybes failed, haddiverged) + where + wantpull gc = remoteAnnexPull gc + +{- Start syncing a remote, using a background thread. -} +syncRemote :: Remote -> Assistant () +syncRemote remote = do + updateSyncRemotes + thread <- asIO $ do + reconnectRemotes [remote] + addScanRemotes True [remote] + void $ liftIO $ forkIO $ thread + +{- Use Nothing to change autocommit setting; or a remote to change + - its sync setting. -} +changeSyncable :: Maybe Remote -> Bool -> Assistant () +changeSyncable Nothing enable = do + liftAnnex $ Config.setConfig key (boolConfig enable) + liftIO . maybe noop (`throwTo` signal) + =<< namedThreadId watchThread + where + key = Config.annexConfig "autocommit" + signal + | enable = ResumeWatcher + | otherwise = PauseWatcher +changeSyncable (Just r) True = do + liftAnnex $ changeSyncFlag r True + syncRemote r + sendRemoteControl RELOAD +changeSyncable (Just r) False = do + liftAnnex $ changeSyncFlag r False + updateSyncRemotes + {- Stop all transfers to or from this remote. + - XXX Can't stop any ongoing scan, or git syncs. -} + void $ dequeueTransfers tofrom + mapM_ (cancelTransfer False) =<< + filter tofrom . M.keys . currentTransfers <$> getDaemonStatus + where + tofrom t = transferUUID t == Remote.uuid r + +changeSyncFlag :: Remote -> Bool -> Annex () +changeSyncFlag r enabled = do + repo <- Remote.getRepo r + let key = Config.remoteConfig repo "sync" + Config.setConfig key (boolConfig enabled) + void Remote.remoteListRefresh + +updateExportTreeFromLogAll :: Assistant () +updateExportTreeFromLogAll = do + rs <- exportRemotes <$> getDaemonStatus + forM_ rs $ \r -> liftAnnex $ + openDb (Remote.uuid r) >>= updateExportTreeFromLog diff --git a/Assistant/Threads/Committer.hs b/Assistant/Threads/Committer.hs new file mode 100644 index 0000000000..aa57d26a86 --- /dev/null +++ b/Assistant/Threads/Committer.hs @@ -0,0 +1,513 @@ +{- git-annex assistant commit thread + - + - Copyright 2012 Joey Hess + - + - Licensed under the GNU GPL version 3 or higher. + -} + +{-# LANGUAGE CPP #-} + +module Assistant.Threads.Committer where + +import Assistant.Common +import Assistant.Changes +import Assistant.Types.Changes +import Assistant.Commits +import Assistant.Alert +import Assistant.DaemonStatus +import Assistant.TransferQueue +import Assistant.Drop +import Types.Transfer +import Logs.Location +import qualified Annex.Queue +import qualified Git.LsFiles +import Utility.ThreadScheduler +import qualified Utility.Lsof as Lsof +import qualified Utility.DirWatcher as DirWatcher +import Types.KeySource +import Config +import Annex.Content +import Annex.Ingest +import Annex.Link +import Annex.CatFile +import Annex.InodeSentinal +import Annex.Version +import qualified Annex +import Utility.InodeCache +import Annex.Content.Direct +import qualified Database.Keys +import qualified Command.Sync +import qualified Git.Branch +import Utility.Tuple + +import Data.Time.Clock +import qualified Data.Set as S +import qualified Data.Map as M +import Data.Either +import Control.Concurrent + +{- This thread makes git commits at appropriate times. -} +commitThread :: NamedThread +commitThread = namedThread "Committer" $ do + havelsof <- liftIO $ inPath "lsof" + delayadd <- liftAnnex $ + maybe delayaddDefault (return . Just . Seconds) + =<< annexDelayAdd <$> Annex.getGitConfig + msg <- liftAnnex Command.Sync.commitMsg + waitChangeTime $ \(changes, time) -> do + readychanges <- handleAdds havelsof delayadd $ + simplifyChanges changes + if shouldCommit False time (length readychanges) readychanges + then do + debug + [ "committing" + , show (length readychanges) + , "changes" + ] + void $ alertWhile commitAlert $ + liftAnnex $ commitStaged msg + recordCommit + recordExportCommit + let numchanges = length readychanges + mapM_ checkChangeContent readychanges + return numchanges + else do + refill readychanges + return 0 + +refill :: [Change] -> Assistant () +refill [] = noop +refill cs = do + debug ["delaying commit of", show (length cs), "changes"] + refillChanges cs + +{- Wait for one or more changes to arrive to be committed, and then + - runs an action to commit them. If more changes arrive while this is + - going on, they're handled intelligently, batching up changes into + - large commits where possible, doing rename detection, and + - commiting immediately otherwise. -} +waitChangeTime :: (([Change], UTCTime) -> Assistant Int) -> Assistant () +waitChangeTime a = waitchanges 0 + where + waitchanges lastcommitsize = do + -- Wait one one second as a simple rate limiter. + liftIO $ threadDelaySeconds (Seconds 1) + -- Now, wait until at least one change is available for + -- processing. + cs <- getChanges + handlechanges cs lastcommitsize + handlechanges changes lastcommitsize = do + let len = length changes + -- See if now's a good time to commit. + now <- liftIO getCurrentTime + scanning <- not . scanComplete <$> getDaemonStatus + case (lastcommitsize >= maxCommitSize, shouldCommit scanning now len changes, possiblyrename changes) of + (True, True, _) + | len > maxCommitSize -> + a (changes, now) >>= waitchanges + | otherwise -> aftermaxcommit changes + (_, True, False) -> + a (changes, now) >>= waitchanges + (_, True, True) -> do + morechanges <- getrelatedchanges changes + a (changes ++ morechanges, now) >>= waitchanges + _ -> do + refill changes + waitchanges lastcommitsize + + {- Did we perhaps only get one of the AddChange and RmChange pair + - that make up a file rename? Or some of the pairs that make up + - a directory rename? + -} + possiblyrename = all renamepart + + renamepart (PendingAddChange _ _) = True + renamepart c = isRmChange c + + {- Gets changes related to the passed changes, without blocking + - very long. + - + - If there are multiple RmChanges, this is probably a directory + - rename, in which case it may be necessary to wait longer to get + - all the Changes involved. + -} + getrelatedchanges oldchanges + | length (filter isRmChange oldchanges) > 1 = + concat <$> getbatchchanges [] + | otherwise = do + liftIO humanImperceptibleDelay + getAnyChanges + getbatchchanges cs = do + liftIO $ threadDelay $ fromIntegral $ oneSecond `div` 10 + cs' <- getAnyChanges + if null cs' + then return cs + else getbatchchanges (cs':cs) + + {- The last commit was maximum size, so it's very likely there + - are more changes and we'd like to ensure we make another commit + - of maximum size if possible. + - + - But, it can take a while for the Watcher to wake back up + - after a commit. It can get blocked by another thread + - that is using the Annex state, such as a git-annex branch + - commit. Especially after such a large commit, this can + - take several seconds. When this happens, it defeats the + - normal commit batching, which sees some old changes the + - Watcher found while the commit was being prepared, and sees + - no recent ones, and wants to commit immediately. + - + - All that we need to do, then, is wait for the Watcher to + - wake up, and queue up one more change. + - + - However, it's also possible that we're at the end of changes for + - now. So to avoid waiting a really long time before committing + - those changes we have, poll for up to 30 seconds, and then + - commit them. + - + - Also, try to run something in Annex, to ensure we block + - longer if the Annex state is indeed blocked. + -} + aftermaxcommit oldchanges = loop (30 :: Int) + where + loop 0 = continue oldchanges + loop n = do + liftAnnex noop -- ensure Annex state is free + liftIO $ threadDelaySeconds (Seconds 1) + changes <- getAnyChanges + if null changes + then loop (n - 1) + else continue (oldchanges ++ changes) + continue cs + | null cs = waitchanges 0 + | otherwise = handlechanges cs 0 + +isRmChange :: Change -> Bool +isRmChange (Change { changeInfo = i }) | i == RmChange = True +isRmChange _ = False + +{- An amount of time that is hopefully imperceptably short for humans, + - while long enough for a computer to get some work done. + - Note that 0.001 is a little too short for rename change batching to + - work. -} +humanImperceptibleInterval :: NominalDiffTime +humanImperceptibleInterval = 0.01 + +humanImperceptibleDelay :: IO () +humanImperceptibleDelay = threadDelay $ + truncate $ humanImperceptibleInterval * fromIntegral oneSecond + +maxCommitSize :: Int +maxCommitSize = 5000 + +{- Decide if now is a good time to make a commit. + - Note that the list of changes has a random order. + - + - Current strategy: If there have been 10 changes within the past second, + - a batch activity is taking place, so wait for later. + -} +shouldCommit :: Bool -> UTCTime -> Int -> [Change] -> Bool +shouldCommit scanning now len changes + | scanning = len >= maxCommitSize + | len == 0 = False + | len >= maxCommitSize = True + | length recentchanges < 10 = True + | otherwise = False -- batch activity + where + thissecond c = timeDelta c <= 1 + recentchanges = filter thissecond changes + timeDelta c = now `diffUTCTime` changeTime c + +commitStaged :: String -> Annex Bool +commitStaged msg = do + {- This could fail if there's another commit being made by + - something else. -} + v <- tryNonAsync Annex.Queue.flush + case v of + Left _ -> return False + Right _ -> do + ok <- Command.Sync.commitStaged Git.Branch.AutomaticCommit msg + when ok $ + Command.Sync.updateSyncBranch =<< join Command.Sync.getCurrBranch + return ok + +{- OSX needs a short delay after a file is added before locking it down, + - as pasting a file seems to try to set file permissions or otherwise + - access the file after closing it. -} +delayaddDefault :: Annex (Maybe Seconds) +#ifdef darwin_HOST_OS +delayaddDefault = ifM (isDirect <||> versionSupportsUnlockedPointers) + ( return Nothing + , return $ Just $ Seconds 1 + ) +#else +delayaddDefault = return Nothing +#endif + +{- If there are PendingAddChanges, or InProcessAddChanges, the files + - have not yet actually been added to the annex, and that has to be done + - now, before committing. + - + - Deferring the adds to this point causes batches to be bundled together, + - which allows faster checking with lsof that the files are not still open + - for write by some other process, and faster checking with git-ls-files + - that the files are not already checked into git. + - + - When a file is added in locked mode, Inotify will notice the new symlink. + - So this waits for additional Changes to arrive, so that the symlink has + - hopefully been staged before returning, and will be committed immediately. + - (OTOH, for kqueue, eventsCoalesce, so instead the symlink is directly + - created and staged.) + - + - Returns a list of all changes that are ready to be committed. + - Any pending adds that are not ready yet are put back into the ChangeChan, + - where they will be retried later. + -} +handleAdds :: Bool -> Maybe Seconds -> [Change] -> Assistant [Change] +handleAdds havelsof delayadd cs = returnWhen (null incomplete) $ do + let (pending, inprocess) = partition isPendingAddChange incomplete + direct <- liftAnnex isDirect + unlocked <- liftAnnex versionSupportsUnlockedPointers + let lockingfiles = not (unlocked || direct) + let lockdownconfig = LockDownConfig + { lockingFile = lockingfiles + , hardlinkFileTmp = True + } + (pending', cleanup) <- if unlocked || direct + then return (pending, noop) + else findnew pending + (postponed, toadd) <- partitionEithers + <$> safeToAdd lockdownconfig havelsof delayadd pending' inprocess + cleanup + + unless (null postponed) $ + refillChanges postponed + + returnWhen (null toadd) $ do + added <- addaction toadd $ + catMaybes <$> + if not lockingfiles + then addunlocked direct toadd + else forM toadd (add lockdownconfig) + if DirWatcher.eventsCoalesce || null added || unlocked || direct + then return $ added ++ otherchanges + else do + r <- handleAdds havelsof delayadd =<< getChanges + return $ r ++ added ++ otherchanges + where + (incomplete, otherchanges) = partition (\c -> isPendingAddChange c || isInProcessAddChange c) cs + + -- Find files that are actually new, and not unlocked annexed + -- files. The ls-files is run on a batch of files. + findnew [] = return ([], noop) + findnew pending@(exemplar:_) = do + let segments = segmentXargsUnordered $ map changeFile pending + rs <- liftAnnex $ forM segments $ \fs -> + inRepo (Git.LsFiles.notInRepo False fs) + let (newfiles, cleanup) = foldl' + (\(l1, a1) (l2, a2) -> (l1 ++ l2, a1 >> a2)) + ([], return True) rs + -- note: timestamp info is lost here + let ts = changeTime exemplar + return (map (PendingAddChange ts) newfiles, void $ liftIO cleanup) + + returnWhen c a + | c = return otherchanges + | otherwise = a + + add :: LockDownConfig -> Change -> Assistant (Maybe Change) + add lockdownconfig change@(InProcessAddChange { lockedDown = ld }) = + catchDefaultIO Nothing <~> doadd + where + ks = keySource ld + doadd = sanitycheck ks $ do + (mkey, mcache) <- liftAnnex $ do + showStart "add" $ keyFilename ks + ingest (Just $ LockedDown lockdownconfig ks) Nothing + maybe (failedingest change) (done change mcache $ keyFilename ks) mkey + add _ _ = return Nothing + + {- Avoid overhead of re-injesting a renamed unlocked file, by + - examining the other Changes to see if a removed file has the + - same InodeCache as the new file. If so, we can just update + - bookkeeping, and stage the file in git. + -} + addunlocked :: Bool -> [Change] -> Assistant [Maybe Change] + addunlocked isdirect toadd = do + ct <- liftAnnex compareInodeCachesWith + m <- liftAnnex $ removedKeysMap isdirect ct cs + delta <- liftAnnex getTSDelta + let cfg = LockDownConfig + { lockingFile = False + , hardlinkFileTmp = True + } + if M.null m + then forM toadd (add cfg) + else forM toadd $ \c -> do + mcache <- liftIO $ genInodeCache (changeFile c) delta + case mcache of + Nothing -> add cfg c + Just cache -> + case M.lookup (inodeCacheToKey ct cache) m of + Nothing -> add cfg c + Just k -> fastadd isdirect c k + + fastadd :: Bool -> Change -> Key -> Assistant (Maybe Change) + fastadd isdirect change key = do + let source = keySource $ lockedDown change + liftAnnex $ if isdirect + then finishIngestDirect key source + else finishIngestUnlocked key source + done change Nothing (keyFilename source) key + + removedKeysMap :: Bool -> InodeComparisonType -> [Change] -> Annex (M.Map InodeCacheKey Key) + removedKeysMap isdirect ct l = do + mks <- forM (filter isRmChange l) $ \c -> + catKeyFile $ changeFile c + M.fromList . concat <$> mapM mkpairs (catMaybes mks) + where + mkpairs k = map (\c -> (inodeCacheToKey ct c, k)) <$> + if isdirect + then recordedInodeCache k + else Database.Keys.getInodeCaches k + + failedingest change = do + refill [retryChange change] + liftAnnex showEndFail + return Nothing + + done change mcache file key = liftAnnex $ do + logStatus key InfoPresent + ifM versionSupportsUnlockedPointers + ( do + mode <- liftIO $ catchMaybeIO $ fileMode <$> getFileStatus file + stagePointerFile file mode =<< hashPointerFile key + , do + link <- ifM isDirect + ( calcRepo $ gitAnnexLink file key + , makeLink file key mcache + ) + whenM (pure DirWatcher.eventsCoalesce <||> isDirect) $ + stageSymlink file =<< hashSymlink link + ) + showEndOk + return $ Just $ finishedChange change key + + {- Check that the keysource's keyFilename still exists, + - and is still a hard link to its contentLocation, + - before ingesting it. -} + sanitycheck keysource a = do + fs <- liftIO $ getSymbolicLinkStatus $ keyFilename keysource + ks <- liftIO $ getSymbolicLinkStatus $ contentLocation keysource + if deviceID ks == deviceID fs && fileID ks == fileID fs + then a + else do + -- remove the hard link + when (contentLocation keysource /= keyFilename keysource) $ + void $ liftIO $ tryIO $ removeFile $ contentLocation keysource + return Nothing + + {- Shown an alert while performing an action to add a file or + - files. When only a few files are added, their names are shown + - in the alert. When it's a batch add, the number of files added + - is shown. + - + - Add errors tend to be transient and will be + - automatically dealt with, so the alert is always told + - the add succeeded. + -} + addaction [] a = a + addaction toadd a = alertWhile' (addFileAlert $ map changeFile toadd) $ + (,) + <$> pure True + <*> a + +{- Files can Either be Right to be added now, + - or are unsafe, and must be Left for later. + - + - Check by running lsof on the repository. + -} +safeToAdd :: LockDownConfig -> Bool -> Maybe Seconds -> [Change] -> [Change] -> Assistant [Either Change Change] +safeToAdd _ _ _ [] [] = return [] +safeToAdd lockdownconfig havelsof delayadd pending inprocess = do + maybe noop (liftIO . threadDelaySeconds) delayadd + liftAnnex $ do + lockeddown <- forM pending $ lockDown lockdownconfig . changeFile + let inprocess' = inprocess ++ mapMaybe mkinprocess (zip pending lockeddown) + openfiles <- if havelsof + then S.fromList . map fst3 . filter openwrite <$> + findopenfiles (map (keySource . lockedDown) inprocess') + else pure S.empty + let checked = map (check openfiles) inprocess' + + {- If new events are received when files are closed, + - there's no need to retry any changes that cannot + - be done now. -} + if DirWatcher.closingTracked + then do + mapM_ canceladd $ lefts checked + allRight $ rights checked + else return checked + where + check openfiles change@(InProcessAddChange { lockedDown = ld }) + | S.member (contentLocation (keySource ld)) openfiles = Left change + check _ change = Right change + + mkinprocess (c, Just ld) = Just InProcessAddChange + { changeTime = changeTime c + , lockedDown = ld + } + mkinprocess (_, Nothing) = Nothing + + canceladd (InProcessAddChange { lockedDown = ld }) = do + let ks = keySource ld + warning $ keyFilename ks + ++ " still has writers, not adding" + -- remove the hard link + when (contentLocation ks /= keyFilename ks) $ + void $ liftIO $ tryIO $ removeFile $ contentLocation ks + canceladd _ = noop + + openwrite (_file, mode, _pid) + | mode == Lsof.OpenWriteOnly = True + | mode == Lsof.OpenReadWrite = True + | mode == Lsof.OpenUnknown = True + | otherwise = False + + allRight = return . map Right + + {- Normally the KeySources are locked down inside the temp directory, + - so can just lsof that, which is quite efficient. + - + - In crippled filesystem mode, there is no lock down, so must run lsof + - on each individual file. + -} + findopenfiles keysources = ifM crippledFileSystem + ( liftIO $ do + let segments = segmentXargsUnordered $ map keyFilename keysources + concat <$> forM segments (\fs -> Lsof.query $ "--" : fs) + , do + tmpdir <- fromRepo gitAnnexTmpMiscDir + liftIO $ Lsof.queryDir tmpdir + ) + +{- After a Change is committed, queue any necessary transfers or drops + - of the content of the key. + - + - This is not done during the startup scan, because the expensive + - transfer scan does the same thing then. + -} +checkChangeContent :: Change -> Assistant () +checkChangeContent change@(Change { changeInfo = i }) = + case changeInfoKey i of + Nothing -> noop + Just k -> whenM (scanComplete <$> getDaemonStatus) $ do + present <- liftAnnex $ inAnnex k + void $ if present + then queueTransfers "new file created" Next k af Upload + else queueTransfers "new or renamed file wanted" Next k af Download + handleDrops "file renamed" present k af [] + where + f = changeFile change + af = AssociatedFile (Just f) +checkChangeContent _ = noop diff --git a/Assistant/Threads/ConfigMonitor.hs b/Assistant/Threads/ConfigMonitor.hs new file mode 100644 index 0000000000..d63faff5e9 --- /dev/null +++ b/Assistant/Threads/ConfigMonitor.hs @@ -0,0 +1,92 @@ +{- git-annex assistant config monitor thread + - + - Copyright 2012 Joey Hess + - + - Licensed under the GNU GPL version 3 or higher. + -} + +module Assistant.Threads.ConfigMonitor where + +import Assistant.Common +import Assistant.BranchChange +import Assistant.DaemonStatus +import Assistant.Commits +import Utility.ThreadScheduler +import Logs +import Logs.UUID +import Logs.Trust +import Logs.PreferredContent +import Logs.Group +import Logs.NumCopies +import Remote.List (remoteListRefresh) +import qualified Git.LsTree as LsTree +import Git.Types +import Git.FilePath +import qualified Annex.Branch + +import qualified Data.Set as S + +{- This thread detects when configuration changes have been made to the + - git-annex branch and reloads cached configuration. + - + - If the branch is frequently changing, it's checked for configuration + - changes no more often than once every 60 seconds. On the other hand, + - if the branch has not changed in a while, configuration changes will + - be detected immediately. + -} +configMonitorThread :: NamedThread +configMonitorThread = namedThread "ConfigMonitor" $ loop =<< getConfigs + where + loop old = do + waitBranchChange + new <- getConfigs + when (old /= new) $ do + let changedconfigs = new `S.difference` old + debug $ "reloading config" : + map fst (S.toList changedconfigs) + reloadConfigs new + {- Record a commit to get this config + - change pushed out to remotes. -} + recordCommit + liftIO $ threadDelaySeconds (Seconds 60) + loop new + +{- Config files, and their checksums. -} +type Configs = S.Set (FilePath, Sha) + +{- All git-annex's config files, and actions to run when they change. -} +configFilesActions :: [(FilePath, Assistant ())] +configFilesActions = + [ (uuidLog, void $ liftAnnex uuidMapLoad) + , (remoteLog, void $ liftAnnex remoteListRefresh) + , (trustLog, void $ liftAnnex trustMapLoad) + , (groupLog, void $ liftAnnex groupMapLoad) + , (numcopiesLog, void $ liftAnnex globalNumCopiesLoad) + , (scheduleLog, void updateScheduleLog) + -- Preferred and required content settings depend on most of the + -- other configs, so will be reloaded whenever any configs change. + , (preferredContentLog, noop) + , (requiredContentLog, noop) + , (groupPreferredContentLog, noop) + ] + +reloadConfigs :: Configs -> Assistant () +reloadConfigs changedconfigs = do + sequence_ as + void $ liftAnnex preferredRequiredMapsLoad + {- Changes to the remote log, or the trust log, can affect the + - syncRemotes list. Changes to the uuid log may affect its + - display so are also included. -} + when (any (`elem` fs) [remoteLog, trustLog, uuidLog]) + updateSyncRemotes + where + (fs, as) = unzip $ filter (flip S.member changedfiles . fst) + configFilesActions + changedfiles = S.map fst changedconfigs + +getConfigs :: Assistant Configs +getConfigs = S.fromList . map extract + <$> liftAnnex (inRepo $ LsTree.lsTreeFiles Annex.Branch.fullname files) + where + files = map fst configFilesActions + extract treeitem = (getTopFilePath $ LsTree.file treeitem, LsTree.sha treeitem) diff --git a/Assistant/Threads/Cronner.hs b/Assistant/Threads/Cronner.hs new file mode 100644 index 0000000000..3e21531380 --- /dev/null +++ b/Assistant/Threads/Cronner.hs @@ -0,0 +1,225 @@ +{- git-annex assistant sceduled jobs runner + - + - Copyright 2013 Joey Hess + - + - Licensed under the GNU GPL version 3 or higher. + -} + +{-# LANGUAGE DeriveDataTypeable #-} + +module Assistant.Threads.Cronner ( + cronnerThread +) where + +import Assistant.Common +import Assistant.DaemonStatus +import Utility.NotificationBroadcaster +import Annex.UUID +import Annex.Path +import Logs.Schedule +import Utility.Scheduled +import Types.ScheduledActivity +import Utility.ThreadScheduler +import Utility.HumanTime +import Utility.Batch +import Assistant.TransferQueue +import Annex.Content +import Types.Transfer +import Assistant.Types.UrlRenderer +import Assistant.Alert +import Remote +import qualified Types.Remote as Remote +import qualified Git +import qualified Git.Fsck +import Assistant.Fsck +import Assistant.Repair + +import Control.Concurrent.Async +import Control.Concurrent.MVar +import Data.Time.LocalTime +import Data.Time.Clock +import qualified Data.Map as M +import qualified Data.Set as S + +{- Loads schedules for this repository, and fires off one thread for each + - scheduled event that runs on this repository. Each thread sleeps until + - its event is scheduled to run. + - + - To handle events that run on remotes, which need to only run when + - their remote gets connected, threads are also started, and are passed + - a MVar to wait on, which is stored in the DaemonStatus's + - connectRemoteNotifiers. + - + - In the meantime the main thread waits for any changes to the + - schedules. When there's a change, compare the old and new list of + - schedules to find deleted and added ones. Start new threads for added + - ones, and kill the threads for deleted ones. -} +cronnerThread :: UrlRenderer -> NamedThread +cronnerThread urlrenderer = namedThreadUnchecked "Cronner" $ do + fsckNudge urlrenderer Nothing + dstatus <- getDaemonStatus + h <- liftIO $ newNotificationHandle False (scheduleLogNotifier dstatus) + go h M.empty M.empty + where + go h amap nmap = do + activities <- liftAnnex $ scheduleGet =<< getUUID + + let addedactivities = activities `S.difference` M.keysSet amap + let removedactivities = M.keysSet amap `S.difference` activities + + forM_ (S.toList removedactivities) $ \activity -> + case M.lookup activity amap of + Just a -> do + debug ["stopping removed job for", fromScheduledActivity activity, show (asyncThreadId a)] + liftIO $ cancel a + Nothing -> noop + + lastruntimes <- liftAnnex getLastRunTimes + started <- startactivities (S.toList addedactivities) lastruntimes + let addedamap = M.fromList $ map fst started + let addednmap = M.fromList $ catMaybes $ map snd started + + let removefiltered = M.filterWithKey (\k _ -> S.member k removedactivities) + let amap' = M.difference (M.union addedamap amap) (removefiltered amap) + let nmap' = M.difference (M.union addednmap nmap) (removefiltered nmap) + modifyDaemonStatus_ $ \s -> s { connectRemoteNotifiers = M.fromListWith (++) (M.elems nmap') } + + liftIO $ waitNotification h + debug ["reloading changed activities"] + go h amap' nmap' + startactivities as lastruntimes = forM as $ \activity -> + case connectActivityUUID activity of + Nothing -> do + runner <- asIO2 (sleepingActivityThread urlrenderer) + a <- liftIO $ async $ + runner activity (M.lookup activity lastruntimes) + return ((activity, a), Nothing) + Just u -> do + mvar <- liftIO newEmptyMVar + runner <- asIO2 (remoteActivityThread urlrenderer mvar) + a <- liftIO $ async $ + runner activity (M.lookup activity lastruntimes) + return ((activity, a), Just (activity, (u, [mvar]))) + +{- Calculate the next time the activity is scheduled to run, then + - sleep until that time, and run it. Then call setLastRunTime, and + - loop. + -} +sleepingActivityThread :: UrlRenderer -> ScheduledActivity -> Maybe LocalTime -> Assistant () +sleepingActivityThread urlrenderer activity lasttime = go lasttime =<< getnexttime lasttime + where + getnexttime = liftIO . nextTime schedule + go _ Nothing = debug ["no scheduled events left for", desc] + go l (Just (NextTimeExactly t)) = waitrun l t Nothing + go l (Just (NextTimeWindow windowstart windowend)) = + waitrun l windowstart (Just windowend) + desc = fromScheduledActivity activity + schedule = getSchedule activity + waitrun l t mmaxt = do + seconds <- liftIO $ secondsUntilLocalTime t + when (seconds > Seconds 0) $ do + debug ["waiting", show seconds, "for next scheduled", desc] + liftIO $ threadDelaySeconds seconds + now <- liftIO getCurrentTime + tz <- liftIO $ getTimeZone now + let nowt = utcToLocalTime tz now + if tolate nowt tz + then do + debug ["too late to run scheduled", desc] + go l =<< getnexttime l + else run nowt + where + tolate nowt tz = case mmaxt of + Just maxt -> nowt > maxt + -- allow the job to start 10 minutes late + Nothing ->diffUTCTime + (localTimeToUTC tz nowt) + (localTimeToUTC tz t) > 600 + run nowt = do + runActivity urlrenderer activity nowt + go (Just nowt) =<< getnexttime (Just nowt) + +{- Wait for the remote to become available by waiting on the MVar. + - Then check if the time is within a time window when activity + - is scheduled to run, and if so run it. + - Otherwise, just wait again on the MVar. + -} +remoteActivityThread :: UrlRenderer -> MVar () -> ScheduledActivity -> Maybe LocalTime -> Assistant () +remoteActivityThread urlrenderer mvar activity lasttime = do + liftIO $ takeMVar mvar + go =<< liftIO (nextTime (getSchedule activity) lasttime) + where + go (Just (NextTimeWindow windowstart windowend)) = do + now <- liftIO getCurrentTime + tz <- liftIO $ getTimeZone now + if now >= localTimeToUTC tz windowstart && now <= localTimeToUTC tz windowend + then do + let nowt = utcToLocalTime tz now + runActivity urlrenderer activity nowt + loop (Just nowt) + else loop lasttime + go _ = noop -- running at exact time not handled here + loop = remoteActivityThread urlrenderer mvar activity + +secondsUntilLocalTime :: LocalTime -> IO Seconds +secondsUntilLocalTime t = do + now <- getCurrentTime + tz <- getTimeZone now + let secs = truncate $ diffUTCTime (localTimeToUTC tz t) now + return $ if secs > 0 + then Seconds secs + else Seconds 0 + +runActivity :: UrlRenderer -> ScheduledActivity -> LocalTime -> Assistant () +runActivity urlrenderer activity nowt = do + debug ["starting", desc] + runActivity' urlrenderer activity + debug ["finished", desc] + liftAnnex $ setLastRunTime activity nowt + where + desc = fromScheduledActivity activity + +runActivity' :: UrlRenderer -> ScheduledActivity -> Assistant () +runActivity' urlrenderer (ScheduledSelfFsck _ d) = do + program <- liftIO programPath + g <- liftAnnex gitRepo + fsckresults <- showFscking urlrenderer Nothing $ tryNonAsync $ do + void $ batchCommand program (Param "fsck" : annexFsckParams d) + Git.Fsck.findBroken True g + u <- liftAnnex getUUID + void $ repairWhenNecessary urlrenderer u Nothing fsckresults + mapM_ reget =<< liftAnnex (dirKeys gitAnnexBadDir) + where + reget k = queueTransfers "fsck found bad file; redownloading" Next k (AssociatedFile Nothing) Download +runActivity' urlrenderer (ScheduledRemoteFsck u s d) = dispatch =<< liftAnnex (remoteFromUUID u) + where + dispatch Nothing = debug ["skipping remote fsck of uuid without a configured remote", fromUUID u, fromSchedule s] + dispatch (Just rmt) = void $ case Remote.remoteFsck rmt of + Nothing -> go rmt $ do + program <- programPath + void $ batchCommand program $ + [ Param "fsck" + -- avoid downloading files + , Param "--fast" + , Param "--from" + , Param $ Remote.name rmt + ] ++ annexFsckParams d + Just mkfscker -> do + {- Note that having mkfsker return an IO action + - avoids running a long duration fsck in the + - Annex monad. -} + go rmt =<< liftAnnex (mkfscker (annexFsckParams d)) + go rmt annexfscker = do + repo <- liftAnnex $ Remote.getRepo rmt + fsckresults <- showFscking urlrenderer (Just rmt) $ tryNonAsync $ do + void annexfscker + if Git.repoIsLocal repo && not (Git.repoIsLocalUnknown repo) + then Just <$> Git.Fsck.findBroken True repo + else pure Nothing + maybe noop (void . repairWhenNecessary urlrenderer u (Just rmt)) fsckresults + +annexFsckParams :: Duration -> [CommandParam] +annexFsckParams d = + [ Param "--incremental-schedule=1d" + , Param $ "--time-limit=" ++ fromDuration d + ] diff --git a/Assistant/Threads/DaemonStatus.hs b/Assistant/Threads/DaemonStatus.hs new file mode 100644 index 0000000000..d5b2cc25d6 --- /dev/null +++ b/Assistant/Threads/DaemonStatus.hs @@ -0,0 +1,29 @@ +{- git-annex assistant daemon status thread + - + - Copyright 2012 Joey Hess + - + - Licensed under the GNU GPL version 3 or higher. + -} + +module Assistant.Threads.DaemonStatus where + +import Assistant.Common +import Assistant.DaemonStatus +import Utility.ThreadScheduler +import Utility.NotificationBroadcaster + +{- This writes the daemon status to disk, when it changes, but no more + - frequently than once every ten minutes. + -} +daemonStatusThread :: NamedThread +daemonStatusThread = namedThread "DaemonStatus" $ do + notifier <- liftIO . newNotificationHandle False + =<< changeNotifier <$> getDaemonStatus + checkpoint + runEvery (Seconds tenMinutes) <~> do + liftIO $ waitNotification notifier + checkpoint + where + checkpoint = do + file <- liftAnnex $ fromRepo gitAnnexDaemonStatusFile + liftIO . writeDaemonStatusFile file =<< getDaemonStatus diff --git a/Assistant/Threads/Exporter.hs b/Assistant/Threads/Exporter.hs new file mode 100644 index 0000000000..6a43954bcb --- /dev/null +++ b/Assistant/Threads/Exporter.hs @@ -0,0 +1,79 @@ +{- git-annex assistant export updating thread + - + - Copyright 2017 Joey Hess + - + - Licensed under the GNU GPL version 3 or higher. + -} + +module Assistant.Threads.Exporter where + +import Assistant.Common +import Assistant.Commits +import Assistant.Pushes +import Assistant.DaemonStatus +import Annex.Concurrent +import Utility.ThreadScheduler +import qualified Annex +import qualified Remote +import qualified Types.Remote as Remote +import qualified Command.Sync + +import Control.Concurrent.Async +import Data.Time.Clock +import qualified Data.Map as M + +{- This thread retries exports that failed before. -} +exportRetryThread :: NamedThread +exportRetryThread = namedThread "ExportRetrier" $ runEvery (Seconds halfhour) <~> do + -- We already waited half an hour, now wait until there are failed + -- exports to retry. + toexport <- getFailedPushesBefore (fromIntegral halfhour) + =<< getAssistant failedExportMap + unless (null toexport) $ do + debug ["retrying", show (length toexport), "failed exports"] + void $ exportToRemotes toexport + where + halfhour = 1800 + +{- This thread updates exports soon after git commits are made. -} +exportThread :: NamedThread +exportThread = namedThread "Exporter" $ runEvery (Seconds 30) <~> do + -- We already waited two seconds as a simple rate limiter. + -- Next, wait until at least one commit has been made + void getExportCommits + -- Now see if now's a good time to push. + void $ exportToRemotes =<< exportTargets + +{- We want to avoid exporting to remotes that are marked readonly. + - + - Also, avoid exporting to local remotes we can easily tell are not available, + - to avoid ugly messages when a removable drive is not attached. + -} +exportTargets :: Assistant [Remote] +exportTargets = liftIO . filterM (Remote.checkAvailable True) + =<< candidates <$> getDaemonStatus + where + candidates = filter (not . Remote.readonly) . exportRemotes + +exportToRemotes :: [Remote] -> Assistant () +exportToRemotes rs = do + -- This is a long-duration action which runs in the Annex monad, + -- so don't just liftAnnex to run it; fork the Annex state. + runner <- liftAnnex $ forkState $ + forM rs $ \r -> do + Annex.changeState $ \st -> st { Annex.errcounter = 0 } + start <- liftIO getCurrentTime + void $ Command.Sync.seekExportContent rs + =<< join Command.Sync.getCurrBranch + -- Look at command error counter to see if the export + -- didn't work. + failed <- (> 0) <$> Annex.getState Annex.errcounter + Annex.changeState $ \st -> st { Annex.errcounter = 0 } + return $ if failed + then Just (r, start) + else Nothing + failed <- catMaybes + <$> (liftAnnex =<< liftIO . wait =<< liftIO (async runner)) + unless (null failed) $ do + v <- getAssistant failedExportMap + changeFailedPushMap v $ M.union $ M.fromList failed diff --git a/Assistant/Threads/Glacier.hs b/Assistant/Threads/Glacier.hs new file mode 100644 index 0000000000..2fd025df17 --- /dev/null +++ b/Assistant/Threads/Glacier.hs @@ -0,0 +1,44 @@ +{- git-annex assistant Amazon Glacier retrieval + - + - Copyright 2012 Joey Hess + - + - Licensed under the GNU GPL version 3 or higher. + -} + +{-# LANGUAGE CPP #-} +{-# LANGUAGE OverloadedStrings #-} + +module Assistant.Threads.Glacier where + +import Assistant.Common +import Utility.ThreadScheduler +import qualified Types.Remote as Remote +import qualified Remote.Glacier as Glacier +import Types.Transfer +import Logs.Transfer +import Assistant.DaemonStatus +import Assistant.TransferQueue + +import qualified Data.Set as S + +{- Wakes up every half hour and checks if any glacier remotes have failed + - downloads. If so, runs glacier-cli to check if the files are now + - available, and queues the downloads. -} +glacierThread :: NamedThread +glacierThread = namedThread "Glacier" $ runEvery (Seconds 3600) <~> go + where + isglacier r = Remote.remotetype r == Glacier.remote + go = do + rs <- filter isglacier . downloadRemotes <$> getDaemonStatus + forM_ rs $ \r -> + check r =<< liftAnnex (getFailedTransfers $ Remote.uuid r) + check _ [] = noop + check r l = do + let keys = map getkey l + (availkeys, failedkeys) <- liftAnnex $ Glacier.jobList r keys + let s = S.fromList (failedkeys ++ availkeys) + let l' = filter (\p -> S.member (getkey p) s) l + forM_ l' $ \(t, info) -> do + liftAnnex $ removeFailedTransfer t + queueTransferWhenSmall "object available from glacier" (associatedFile info) t r + getkey = transferKey . fst diff --git a/Assistant/Threads/Merger.hs b/Assistant/Threads/Merger.hs new file mode 100644 index 0000000000..ff2fdc9a00 --- /dev/null +++ b/Assistant/Threads/Merger.hs @@ -0,0 +1,109 @@ +{- git-annex assistant git merge thread + - + - Copyright 2012-2017 Joey Hess + - + - Licensed under the GNU GPL version 3 or higher. + -} + +module Assistant.Threads.Merger where + +import Assistant.Common +import Assistant.TransferQueue +import Assistant.BranchChange +import Assistant.Sync +import Utility.DirWatcher +import Utility.DirWatcher.Types +import qualified Annex.Branch +import qualified Git +import qualified Git.Branch +import qualified Git.Ref +import qualified Command.Sync + +{- This thread watches for changes to .git/refs/, and handles incoming + - pushes. -} +mergeThread :: NamedThread +mergeThread = namedThread "Merger" $ do + g <- liftAnnex gitRepo + let dir = Git.localGitDir g "refs" + liftIO $ createDirectoryIfMissing True dir + let hook a = Just <$> asIO2 (runHandler a) + changehook <- hook onChange + errhook <- hook onErr + let hooks = mkWatchHooks + { addHook = changehook + , modifyHook = changehook + , errHook = errhook + } + void $ liftIO $ watchDir dir (const False) True hooks id + debug ["watching", dir] + +type Handler = FilePath -> Assistant () + +{- Runs an action handler. + - + - Exceptions are ignored, otherwise a whole thread could be crashed. + -} +runHandler :: Handler -> FilePath -> Maybe FileStatus -> Assistant () +runHandler handler file _filestatus = + either (liftIO . print) (const noop) =<< tryIO <~> handler file + +{- Called when there's an error with inotify. -} +onErr :: Handler +onErr = error + +{- Called when a new branch ref is written, or a branch ref is modified. + - + - At startup, synthetic add events fire, causing this to run, but that's + - ok; it ensures that any changes pushed since the last time the assistant + - ran are merged in. + -} +onChange :: Handler +onChange file + | ".lock" `isSuffixOf` file = noop + | isAnnexBranch file = do + branchChanged + diverged <- liftAnnex Annex.Branch.forceUpdate + when diverged $ do + updateExportTreeFromLogAll + queueDeferredDownloads "retrying deferred download" Later + | otherwise = mergecurrent + where + changedbranch = fileToBranch file + + mergecurrent = + mergecurrent' =<< liftAnnex (join Command.Sync.getCurrBranch) + mergecurrent' currbranch@(Just b, _) + | changedbranch `isRelatedTo` b = + whenM (liftAnnex $ inRepo $ Git.Branch.changed b changedbranch) $ do + debug + [ "merging", Git.fromRef changedbranch + , "into", Git.fromRef b + ] + void $ liftAnnex $ Command.Sync.merge + currbranch Command.Sync.mergeConfig + def + Git.Branch.AutomaticCommit + changedbranch + mergecurrent' _ = noop + +{- Is the first branch a synced branch or remote tracking branch related + - to the second branch, which should be merged into it? -} +isRelatedTo :: Git.Ref -> Git.Ref -> Bool +isRelatedTo x y + | basex /= takeDirectory basex ++ "/" ++ basey = False + | "/synced/" `isInfixOf` Git.fromRef x = True + | "refs/remotes/" `isPrefixOf` Git.fromRef x = True + | otherwise = False + where + basex = Git.fromRef $ Git.Ref.base x + basey = Git.fromRef $ Git.Ref.base y + +isAnnexBranch :: FilePath -> Bool +isAnnexBranch f = n `isSuffixOf` f + where + n = '/' : Git.fromRef Annex.Branch.name + +fileToBranch :: FilePath -> Git.Ref +fileToBranch f = Git.Ref $ "refs" base + where + base = Prelude.last $ split "/refs/" f diff --git a/Assistant/Threads/MountWatcher.hs b/Assistant/Threads/MountWatcher.hs new file mode 100644 index 0000000000..c5d075f86f --- /dev/null +++ b/Assistant/Threads/MountWatcher.hs @@ -0,0 +1,182 @@ +{- git-annex assistant mount watcher, using either dbus or mtab polling + - + - Copyright 2012 Joey Hess + - + - Licensed under the GNU GPL version 3 or higher. + -} + +{-# LANGUAGE CPP #-} +{-# LANGUAGE OverloadedStrings #-} + +module Assistant.Threads.MountWatcher where + +import Assistant.Common +import Assistant.DaemonStatus +import Assistant.Sync +import qualified Annex +import qualified Git +import Utility.ThreadScheduler +import Utility.Mounts +import Remote.List +import qualified Types.Remote as Remote +import Assistant.Types.UrlRenderer +import Assistant.Fsck + +import qualified Data.Set as S + +#if WITH_DBUS +import Utility.DBus +import DBus.Client +import DBus +import Data.Word (Word32) +import Control.Concurrent +import qualified Control.Exception as E +#else +#ifdef linux_HOST_OS +#warning Building without dbus support; will use mtab polling +#endif +#endif + +mountWatcherThread :: UrlRenderer -> NamedThread +mountWatcherThread urlrenderer = namedThread "MountWatcher" $ +#if WITH_DBUS + dbusThread urlrenderer +#else + pollingThread urlrenderer +#endif + +#if WITH_DBUS + +dbusThread :: UrlRenderer -> Assistant () +dbusThread urlrenderer = do + runclient <- asIO1 go + r <- liftIO $ E.try $ runClient getSystemAddress runclient + either onerr (const noop) r + where + go client = ifM (checkMountMonitor client) + ( do + {- Store the current mount points in an MVar, to be + - compared later. We could in theory work out the + - mount point from the dbus message, but this is + - easier. -} + mvar <- liftIO $ newMVar =<< currentMountPoints + handleevent <- asIO1 $ \_event -> do + nowmounted <- liftIO $ currentMountPoints + wasmounted <- liftIO $ swapMVar mvar nowmounted + handleMounts urlrenderer wasmounted nowmounted + liftIO $ forM_ mountChanged $ \matcher -> + void $ addMatch client matcher handleevent + , do + liftAnnex $ + warning "No known volume monitor available through dbus; falling back to mtab polling" + pollingThread urlrenderer + ) + onerr :: E.SomeException -> Assistant () + onerr e = do + liftAnnex $ + warning $ "dbus failed; falling back to mtab polling (" ++ show e ++ ")" + pollingThread urlrenderer + +{- Examine the list of services connected to dbus, to see if there + - are any we can use to monitor mounts. If not, will attempt to start one. -} +checkMountMonitor :: Client -> Assistant Bool +checkMountMonitor client = do + running <- filter (`elem` usableservices) + <$> liftIO (listServiceNames client) + case running of + [] -> startOneService client startableservices + (service:_) -> do + debug [ "Using running DBUS service" + , service + , "to monitor mount events." + ] + return True + where + startableservices = [udisks2] + usableservices = startableservices + udisks2 = "org.freedesktop.UDisks2" + +startOneService :: Client -> [ServiceName] -> Assistant Bool +startOneService _ [] = return False +startOneService client (x:xs) = do + _ <- liftIO $ tryNonAsync $ callDBus client "StartServiceByName" + [toVariant x, toVariant (0 :: Word32)] + ifM (liftIO $ elem x <$> listServiceNames client) + ( do + debug + [ "Started DBUS service", x + , "to monitor mount events." + ] + return True + , startOneService client xs + ) + +{- Filter matching events recieved when drives are mounted and unmounted. -} +mountChanged :: [MatchRule] +mountChanged = [udisks2mount, udisks2umount] + where + udisks2mount = matchAny + { matchPath = Just "/org/freedesktop/UDisks2" + , matchInterface = Just "org.freedesktop.DBus.ObjectManager" + , matchMember = Just "InterfacesAdded" + } + udisks2umount = matchAny + { matchPath = Just "/org/freedesktop/UDisks2" + , matchInterface = Just "org.freedesktop.DBus.ObjectManager" + , matchMember = Just "InterfacesRemoved" + } +#endif + +pollingThread :: UrlRenderer -> Assistant () +pollingThread urlrenderer = go =<< liftIO currentMountPoints + where + go wasmounted = do + liftIO $ threadDelaySeconds (Seconds 10) + nowmounted <- liftIO currentMountPoints + handleMounts urlrenderer wasmounted nowmounted + go nowmounted + +handleMounts :: UrlRenderer -> MountPoints -> MountPoints -> Assistant () +handleMounts urlrenderer wasmounted nowmounted = + mapM_ (handleMount urlrenderer . mnt_dir) $ + S.toList $ newMountPoints wasmounted nowmounted + +handleMount :: UrlRenderer -> FilePath -> Assistant () +handleMount urlrenderer dir = do + debug ["detected mount of", dir] + rs <- filterM (Git.repoIsLocal <$$> liftAnnex . Remote.getRepo) + =<< remotesUnder dir + mapM_ (fsckNudge urlrenderer . Just) rs + reconnectRemotes rs + +{- Finds remotes located underneath the mount point. + - + - Updates state to include the remotes. + - + - The config of git remotes is re-read, as it may not have been available + - at startup time, or may have changed (it could even be a different + - repository at the same remote location..) + -} +remotesUnder :: FilePath -> Assistant [Remote] +remotesUnder dir = do + repotop <- liftAnnex $ fromRepo Git.repoPath + rs <- liftAnnex remoteList + pairs <- liftAnnex $ mapM (checkremote repotop) rs + let (waschanged, rs') = unzip pairs + when (or waschanged) $ do + liftAnnex $ Annex.changeState $ \s -> s { Annex.remotes = catMaybes rs' } + updateSyncRemotes + return $ mapMaybe snd $ filter fst pairs + where + checkremote repotop r = case Remote.localpath r of + Just p | dirContains dir (absPathFrom repotop p) -> + (,) <$> pure True <*> updateRemote r + _ -> return (False, Just r) + +type MountPoints = S.Set Mntent + +currentMountPoints :: IO MountPoints +currentMountPoints = S.fromList <$> getMounts + +newMountPoints :: MountPoints -> MountPoints -> MountPoints +newMountPoints old new = S.difference new old diff --git a/Assistant/Threads/NetWatcher.hs b/Assistant/Threads/NetWatcher.hs new file mode 100644 index 0000000000..4dc8721b1e --- /dev/null +++ b/Assistant/Threads/NetWatcher.hs @@ -0,0 +1,202 @@ +{- git-annex assistant network connection watcher, using dbus + - + - Copyright 2012 Joey Hess + - + - Licensed under the GNU GPL version 3 or higher. + -} + +{-# LANGUAGE CPP #-} +{-# LANGUAGE OverloadedStrings #-} + +module Assistant.Threads.NetWatcher where + +import Assistant.Common +import Assistant.Sync +import Utility.ThreadScheduler +import qualified Types.Remote as Remote +import Assistant.DaemonStatus +import Utility.NotificationBroadcaster + +#if WITH_DBUS +import Assistant.RemoteControl +import Utility.DBus +import DBus.Client +import DBus +#else +#ifdef linux_HOST_OS +#warning Building without dbus support; will poll for network connection changes +#endif +#endif + +netWatcherThread :: NamedThread +#if WITH_DBUS +netWatcherThread = thread dbusThread +#else +netWatcherThread = thread noop +#endif + where + thread = namedThread "NetWatcher" + +{- This is a fallback for when dbus cannot be used to detect + - network connection changes, but it also ensures that + - any networked remotes that may have not been routable for a + - while (despite the local network staying up), are synced with + - periodically. + - + - Note that it does not signal the RemoteControl, because it doesn't + - know that the network has changed. + -} +netWatcherFallbackThread :: NamedThread +netWatcherFallbackThread = namedThread "NetWatcherFallback" $ + runEvery (Seconds 3600) <~> handleConnection + +#if WITH_DBUS + +dbusThread :: Assistant () +dbusThread = do + handleerr <- asIO2 onerr + runclient <- asIO1 go + liftIO $ persistentClient getSystemAddress () handleerr runclient + where + go client = ifM (checkNetMonitor client) + ( do + callback <- asIO1 connchange + liftIO $ do + listenNMConnections client callback + listenNDConnections client callback + listenWicdConnections client callback + , do + liftAnnex $ + warning "No known network monitor available through dbus; falling back to polling" + ) + connchange False = do + debug ["detected network disconnection"] + sendRemoteControl LOSTNET + connchange True = do + debug ["detected network connection"] + handleConnection + sendRemoteControl RESUME + onerr e _ = do + liftAnnex $ + warning $ "lost dbus connection; falling back to polling (" ++ show e ++ ")" + {- Wait, in hope that dbus will come back -} + liftIO $ threadDelaySeconds (Seconds 60) + +{- Examine the list of services connected to dbus, to see if there + - are any we can use to monitor network connections. -} +checkNetMonitor :: Client -> Assistant Bool +checkNetMonitor client = do + running <- liftIO $ filter (`elem` manager_addresses) + <$> listServiceNames client + case running of + [] -> return False + (service:_) -> do + debug [ "Using running DBUS service" + , service + , "to monitor network connection events." + ] + return True + where + manager_addresses = [networkmanager, networkd, wicd] + networkmanager = "org.freedesktop.NetworkManager" + networkd = "org.freedesktop.network1" + wicd = "org.wicd.daemon" + +{- Listens for systemd-networkd connections and diconnections. + - + - Connection example (once fully connected): + - [Variant {"OperationalState": Variant "routable"}] + - + - Disconnection example: + - [Variant {"OperationalState": Variant _}] + -} +listenNDConnections :: Client -> (Bool -> IO ()) -> IO () +listenNDConnections client setconnected = + void $ addMatch client matcher + $ \event -> mapM_ handleevent + (map dictionaryItems $ mapMaybe fromVariant $ signalBody event) + where + matcher = matchAny + { matchInterface = Just "org.freedesktop.DBus.Properties" + , matchMember = Just "PropertiesChanged" + } + operational_state_key = toVariant ("OperationalState" :: String) + routable = toVariant $ toVariant ("routable" :: String) + handleevent m = case lookup operational_state_key m of + Just state -> if state == routable + then setconnected True + else setconnected False + Nothing -> noop + +{- Listens for NetworkManager connections and diconnections. + - + - Connection example (once fully connected): + - [Variant {"ActivatingConnection": Variant (ObjectPath "/"), "PrimaryConnection": Variant (ObjectPath "/org/freedesktop/NetworkManager/ActiveConnection/34"), "State": Variant 70}] + - + - Disconnection example: + - [Variant {"ActiveConnections": Variant []}] + -} +listenNMConnections :: Client -> (Bool -> IO ()) -> IO () +listenNMConnections client setconnected = + void $ addMatch client matcher + $ \event -> mapM_ handleevent + (map dictionaryItems $ mapMaybe fromVariant $ signalBody event) + where + matcher = matchAny + { matchInterface = Just "org.freedesktop.NetworkManager" + , matchMember = Just "PropertiesChanged" + } + nm_active_connections_key = toVariant ("ActiveConnections" :: String) + nm_activatingconnection_key = toVariant ("ActivatingConnection" :: String) + noconnections = Just $ toVariant $ toVariant ([] :: [ObjectPath]) + rootconnection = Just $ toVariant $ toVariant $ objectPath_ "/" + handleevent m + | lookup nm_active_connections_key m == noconnections = + setconnected False + | lookup nm_activatingconnection_key m == rootconnection = + setconnected True + | otherwise = noop + +{- Listens for Wicd connections and disconnections. + - + - Connection example: + - ConnectResultsSent: + - Variant "success" + - + - Diconnection example: + - StatusChanged + - [Variant 0, Variant [Varient ""]] + -} +listenWicdConnections :: Client -> (Bool -> IO ()) -> IO () +listenWicdConnections client setconnected = do + match connmatcher $ \event -> + when (any (== wicd_success) (signalBody event)) $ + setconnected True + match statusmatcher $ \event -> handleevent (signalBody event) + where + connmatcher = matchAny + { matchInterface = Just "org.wicd.daemon" + , matchMember = Just "ConnectResultsSent" + } + statusmatcher = matchAny + { matchInterface = Just "org.wicd.daemon" + , matchMember = Just "StatusChanged" + } + wicd_success = toVariant ("success" :: String) + wicd_disconnected = toVariant [toVariant ("" :: String)] + handleevent status + | any (== wicd_disconnected) status = setconnected False + | otherwise = noop + match matcher a = + void $ addMatch client matcher a +#endif + +handleConnection :: Assistant () +handleConnection = do + liftIO . sendNotification . networkConnectedNotifier =<< getDaemonStatus + reconnectRemotes =<< networkRemotes + +{- Network remotes to sync with. -} +networkRemotes :: Assistant [Remote] +networkRemotes = filter (isNothing . Remote.localpath) . syncRemotes + <$> getDaemonStatus diff --git a/Assistant/Threads/PairListener.hs b/Assistant/Threads/PairListener.hs new file mode 100644 index 0000000000..09eaf1fe8a --- /dev/null +++ b/Assistant/Threads/PairListener.hs @@ -0,0 +1,154 @@ +{- git-annex assistant thread to listen for incoming pairing traffic + - + - Copyright 2012 Joey Hess + - + - Licensed under the GNU GPL version 3 or higher. + -} + +module Assistant.Threads.PairListener where + +import Assistant.Common +import Assistant.Pairing +import Assistant.Pairing.Network +import Assistant.Pairing.MakeRemote +import Assistant.WebApp (UrlRenderer) +import Assistant.WebApp.Types +import Assistant.Alert +import Assistant.DaemonStatus +import Utility.ThreadScheduler +import Git + +import Network.Multicast +import Network.Socket +import qualified Data.ByteString as B +import qualified Data.ByteString.UTF8 as BU8 +import qualified Network.Socket.ByteString as B +import qualified Data.Text as T + +pairListenerThread :: UrlRenderer -> NamedThread +pairListenerThread urlrenderer = namedThread "PairListener" $ do + listener <- asIO1 $ go [] [] + liftIO $ withSocketsDo $ + runEvery (Seconds 60) $ void $ tryIO $ + listener =<< getsock + where + {- Note this can crash if there's no network interface, + - or only one like lo that doesn't support multicast. -} + getsock = multicastReceiver (multicastAddress IPv4AddrClass) pairingPort + + go reqs cache sock = liftIO (getmsg sock B.empty) >>= \msg -> case readish (BU8.toString msg) of + Nothing -> go reqs cache sock + Just m -> do + debug ["received", show msg] + (pip, verified) <- verificationCheck m + =<< (pairingInProgress <$> getDaemonStatus) + let wrongstage = maybe False (\p -> pairMsgStage m <= inProgressPairStage p) pip + let fromus = maybe False (\p -> remoteSshPubKey (pairMsgData m) == remoteSshPubKey (inProgressPairData p)) pip + case (wrongstage, fromus, checkSane (pairMsgData m), pairMsgStage m) of + (_, True, _, _) -> do + debug ["ignoring message that looped back"] + go reqs cache sock + (_, _, False, _) -> do + liftAnnex $ warning $ + "illegal control characters in pairing message; ignoring (" ++ show (pairMsgData m) ++ ")" + go reqs cache sock + -- PairReq starts a pairing process, so a + -- new one is always heeded, even if + -- some other pairing is in process. + (_, _, _, PairReq) -> if m `elem` reqs + then go reqs (invalidateCache m cache) sock + else do + pairReqReceived verified urlrenderer m + go (m:take 10 reqs) (invalidateCache m cache) sock + (True, _, _, _) -> do + debug + ["ignoring out of order message" + , show (pairMsgStage m) + , "expected" + , show (succ . inProgressPairStage <$> pip) + ] + go reqs cache sock + (_, _, _, PairAck) -> do + cache' <- pairAckReceived verified pip m cache + go reqs cache' sock + (_,_ , _, PairDone) -> do + pairDoneReceived verified pip m + go reqs cache sock + + {- As well as verifying the message using the shared secret, + - check its UUID against the UUID we have stored. If + - they're the same, someone is sending bogus messages, + - which could be an attempt to brute force the shared secret. -} + verificationCheck _ Nothing = return (Nothing, False) + verificationCheck m (Just pip) + | not verified && sameuuid = do + liftAnnex $ warning + "detected possible pairing brute force attempt; disabled pairing" + stopSending pip + return (Nothing, False) + | otherwise = return (Just pip, verified && sameuuid) + where + verified = verifiedPairMsg m pip + sameuuid = pairUUID (inProgressPairData pip) == pairUUID (pairMsgData m) + + {- PairReqs invalidate the cache of recently finished pairings. + - This is so that, if a new pairing is started with the + - same secret used before, a bogus PairDone is not sent. -} + invalidateCache msg = filter (not . verifiedPairMsg msg) + + getmsg sock c = do + (msg, _) <- B.recvFrom sock chunksz + if B.length msg < chunksz + then return $ c <> msg + else getmsg sock $ c <> msg + where + chunksz = 1024 + +{- Show an alert when a PairReq is seen. -} +pairReqReceived :: Bool -> UrlRenderer -> PairMsg -> Assistant () +pairReqReceived True _ _ = noop -- ignore our own PairReq +pairReqReceived False urlrenderer msg = do + button <- mkAlertButton True (T.pack "Respond") urlrenderer (FinishLocalPairR msg) + void $ addAlert $ pairRequestReceivedAlert repo button + where + repo = pairRepo msg + +{- When a verified PairAck is seen, a host is ready to pair with us, and has + - already configured our ssh key. Stop sending PairReqs, finish the pairing, + - and send a single PairDone. -} +pairAckReceived :: Bool -> Maybe PairingInProgress -> PairMsg -> [PairingInProgress] -> Assistant [PairingInProgress] +pairAckReceived True (Just pip) msg cache = do + stopSending pip + repodir <- repoPath <$> liftAnnex gitRepo + liftIO $ setupAuthorizedKeys msg repodir + finishedLocalPairing msg (inProgressSshKeyPair pip) + startSending pip PairDone $ multicastPairMsg + (Just 1) (inProgressSecret pip) (inProgressPairData pip) + return $ pip : take 10 cache +{- A stale PairAck might also be seen, after we've finished pairing. + - Perhaps our PairDone was not received. To handle this, we keep + - a cache of recently finished pairings, and re-send PairDone in + - response to stale PairAcks for them. -} +pairAckReceived _ _ msg cache = do + let pips = filter (verifiedPairMsg msg) cache + unless (null pips) $ + forM_ pips $ \pip -> + startSending pip PairDone $ multicastPairMsg + (Just 1) (inProgressSecret pip) (inProgressPairData pip) + return cache + +{- If we get a verified PairDone, the host has accepted our PairAck, and + - has paired with us. Stop sending PairAcks, and finish pairing with them. + - + - TODO: Should third-party hosts remove their pair request alert when they + - see a PairDone? + - Complication: The user could have already clicked on the alert and be + - entering the secret. Would be better to start a fresh pair request in this + - situation. + -} +pairDoneReceived :: Bool -> Maybe PairingInProgress -> PairMsg -> Assistant () +pairDoneReceived False _ _ = noop -- not verified +pairDoneReceived True Nothing _ = noop -- not in progress +pairDoneReceived True (Just pip) msg = do + stopSending pip + finishedLocalPairing msg (inProgressSshKeyPair pip) diff --git a/Assistant/Threads/ProblemFixer.hs b/Assistant/Threads/ProblemFixer.hs new file mode 100644 index 0000000000..19f7ccca21 --- /dev/null +++ b/Assistant/Threads/ProblemFixer.hs @@ -0,0 +1,73 @@ +{- git-annex assistant thread to handle fixing problems with repositories + - + - Copyright 2013 Joey Hess + - + - Licensed under the GNU GPL version 3 or higher. + -} + +module Assistant.Threads.ProblemFixer ( + problemFixerThread +) where + +import Assistant.Common +import Assistant.Types.RepoProblem +import Assistant.RepoProblem +import Assistant.Types.UrlRenderer +import Assistant.Alert +import Remote +import qualified Types.Remote as Remote +import qualified Git.Fsck +import Assistant.Repair +import qualified Git +import Annex.UUID +import Utility.ThreadScheduler + +{- Waits for problems with a repo, and tries to fsck the repo and repair + - the problem. -} +problemFixerThread :: UrlRenderer -> NamedThread +problemFixerThread urlrenderer = namedThread "ProblemFixer" $ + go =<< getRepoProblems + where + go problems = do + mapM_ (handleProblem urlrenderer) problems + liftIO $ threadDelaySeconds (Seconds 60) + -- Problems may have been re-reported while they were being + -- fixed, so ignore those. If a new unique problem happened + -- 60 seconds after the last was fixed, we're unlikely + -- to do much good anyway. + go =<< filter (\p -> not (any (sameRepoProblem p) problems)) + <$> getRepoProblems + +handleProblem :: UrlRenderer -> RepoProblem -> Assistant () +handleProblem urlrenderer repoproblem = do + fixed <- ifM ((==) (problemUUID repoproblem) <$> liftAnnex getUUID) + ( handleLocalRepoProblem urlrenderer + , maybe (return False) (handleRemoteProblem urlrenderer) + =<< liftAnnex (remoteFromUUID $ problemUUID repoproblem) + ) + when fixed $ + liftIO $ afterFix repoproblem + +handleRemoteProblem :: UrlRenderer -> Remote -> Assistant Bool +handleRemoteProblem urlrenderer rmt = do + repo <- liftAnnex $ Remote.getRepo rmt + handleRemoteProblem' repo urlrenderer rmt + +handleRemoteProblem' :: Git.Repo -> UrlRenderer -> Remote -> Assistant Bool +handleRemoteProblem' repo urlrenderer rmt + | Git.repoIsLocal repo && not (Git.repoIsLocalUnknown repo) = + ifM (liftIO $ checkAvailable True rmt) + ( do + fixedlocks <- repairStaleGitLocks repo + fsckresults <- showFscking urlrenderer (Just rmt) $ tryNonAsync $ + Git.Fsck.findBroken True repo + repaired <- repairWhenNecessary urlrenderer (Remote.uuid rmt) (Just rmt) fsckresults + return $ fixedlocks || repaired + , return False + ) + | otherwise = return False + +{- This is not yet used, and should probably do a fsck. -} +handleLocalRepoProblem :: UrlRenderer -> Assistant Bool +handleLocalRepoProblem _urlrenderer = do + repairStaleGitLocks =<< liftAnnex gitRepo diff --git a/Assistant/Threads/Pusher.hs b/Assistant/Threads/Pusher.hs new file mode 100644 index 0000000000..82bd39d893 --- /dev/null +++ b/Assistant/Threads/Pusher.hs @@ -0,0 +1,50 @@ +{- git-annex assistant git pushing thread + - + - Copyright 2012 Joey Hess + - + - Licensed under the GNU GPL version 3 or higher. + -} + +module Assistant.Threads.Pusher where + +import Assistant.Common +import Assistant.Commits +import Assistant.Pushes +import Assistant.DaemonStatus +import Assistant.Sync +import Utility.ThreadScheduler +import qualified Remote +import qualified Types.Remote as Remote + +{- This thread retries pushes that failed before. -} +pushRetryThread :: NamedThread +pushRetryThread = namedThread "PushRetrier" $ runEvery (Seconds halfhour) <~> do + -- We already waited half an hour, now wait until there are failed + -- pushes to retry. + topush <- getFailedPushesBefore (fromIntegral halfhour) + =<< getAssistant failedPushMap + unless (null topush) $ do + debug ["retrying", show (length topush), "failed pushes"] + void $ pushToRemotes topush + where + halfhour = 1800 + +{- This thread pushes git commits out to remotes soon after they are made. -} +pushThread :: NamedThread +pushThread = namedThread "Pusher" $ runEvery (Seconds 2) <~> do + -- We already waited two seconds as a simple rate limiter. + -- Next, wait until at least one commit has been made + void getCommits + -- Now see if now's a good time to push. + void $ pushToRemotes =<< pushTargets + +{- We want to avoid pushing to remotes that are marked readonly. + - + - Also, avoid pushing to local remotes we can easily tell are not available, + - to avoid ugly messages when a removable drive is not attached. + -} +pushTargets :: Assistant [Remote] +pushTargets = liftIO . filterM (Remote.checkAvailable True) + =<< candidates <$> getDaemonStatus + where + candidates = filter (not . Remote.readonly) . syncGitRemotes diff --git a/Assistant/Threads/RemoteControl.hs b/Assistant/Threads/RemoteControl.hs new file mode 100644 index 0000000000..2a411ef31c --- /dev/null +++ b/Assistant/Threads/RemoteControl.hs @@ -0,0 +1,121 @@ +{- git-annex assistant communication with remotedaemon + - + - Copyright 2014 Joey Hess + - + - Licensed under the GNU GPL version 3 or higher. + -} + +module Assistant.Threads.RemoteControl where + +import Assistant.Common +import RemoteDaemon.Types +import Annex.Path +import Utility.Batch +import Utility.SimpleProtocol +import Assistant.Alert +import Assistant.Alert.Utility +import Assistant.DaemonStatus +import qualified Git +import qualified Git.Types as Git +import qualified Remote +import qualified Types.Remote as Remote + +import Control.Concurrent +import Control.Concurrent.Async +import Network.URI +import qualified Data.Map as M +import qualified Data.Set as S + +remoteControlThread :: NamedThread +remoteControlThread = namedThread "RemoteControl" $ do + program <- liftIO programPath + (cmd, params) <- liftIO $ toBatchCommand + (program, [Param "remotedaemon", Param "--foreground"]) + let p = proc cmd (toCommand params) + (Just toh, Just fromh, _, pid) <- liftIO $ createProcess p + { std_in = CreatePipe + , std_out = CreatePipe + } + + urimap <- liftIO . newMVar =<< liftAnnex getURIMap + + controller <- asIO $ remoteControllerThread toh + responder <- asIO $ remoteResponderThread fromh urimap + + -- run controller and responder until the remotedaemon dies + liftIO $ void $ tryNonAsync $ controller `concurrently` responder + debug ["remotedaemon exited"] + liftIO $ forceSuccessProcess p pid + +-- feed from the remoteControl channel into the remotedaemon +remoteControllerThread :: Handle -> Assistant () +remoteControllerThread toh = do + clicker <- getAssistant remoteControl + forever $ do + msg <- liftIO $ readChan clicker + debug [show msg] + liftIO $ do + hPutStrLn toh $ unwords $ formatMessage msg + hFlush toh + +-- read status messages emitted by the remotedaemon and handle them +remoteResponderThread :: Handle -> MVar (M.Map URI Remote) -> Assistant () +remoteResponderThread fromh urimap = go M.empty + where + go syncalerts = do + l <- liftIO $ hGetLine fromh + debug [l] + case parseMessage l of + Just (CONNECTED uri) -> changeconnected S.insert uri + Just (DISCONNECTED uri) -> changeconnected S.delete uri + Just (SYNCING uri) -> withr uri $ \r -> + if M.member (Remote.uuid r) syncalerts + then go syncalerts + else do + i <- addAlert $ syncAlert [r] + go (M.insert (Remote.uuid r) i syncalerts) + Just (DONESYNCING uri status) -> withr uri $ \r -> + case M.lookup (Remote.uuid r) syncalerts of + Nothing -> cont + Just i -> do + let (succeeded, failed) = if status + then ([r], []) + else ([], [r]) + updateAlertMap $ mergeAlert i $ + syncResultAlert succeeded failed + go (M.delete (Remote.uuid r) syncalerts) + Just (WARNING (RemoteURI uri) msg) -> do + void $ addAlert $ + warningAlert ("RemoteControl "++ show uri) msg + cont + Nothing -> do + debug ["protocol error from remotedaemon: ", l] + cont + where + cont = go syncalerts + withr uri = withRemote uri urimap cont + changeconnected sm uri = withr uri $ \r -> do + changeCurrentlyConnected $ sm $ Remote.uuid r + cont + +getURIMap :: Annex (M.Map URI Remote) +getURIMap = Remote.remoteMap' id (\r -> mkk . Git.location <$> Remote.getRepo r) + where + mkk (Git.Url u) = Just u + mkk _ = Nothing + +withRemote + :: RemoteURI + -> MVar (M.Map URI Remote) + -> Assistant a + -> (Remote -> Assistant a) + -> Assistant a +withRemote (RemoteURI uri) remotemap noremote a = do + m <- liftIO $ readMVar remotemap + case M.lookup uri m of + Just r -> a r + Nothing -> do + {- Reload map, in case a new remote has been added. -} + m' <- liftAnnex getURIMap + void $ liftIO $ swapMVar remotemap $ m' + maybe noremote a (M.lookup uri m') diff --git a/Assistant/Threads/SanityChecker.hs b/Assistant/Threads/SanityChecker.hs new file mode 100644 index 0000000000..43812e5d41 --- /dev/null +++ b/Assistant/Threads/SanityChecker.hs @@ -0,0 +1,327 @@ +{- git-annex assistant sanity checker + - + - Copyright 2012, 2013 Joey Hess + - + - Licensed under the GNU GPL version 3 or higher. + -} + +{-# LANGUAGE CPP #-} + +module Assistant.Threads.SanityChecker ( + sanityCheckerStartupThread, + sanityCheckerDailyThread, + sanityCheckerHourlyThread +) where + +import Assistant.Common +import Assistant.DaemonStatus +import Assistant.Alert +import Assistant.Repair +import Assistant.Drop +import Assistant.Ssh +import Assistant.TransferQueue +import Assistant.Types.UrlRenderer +import Assistant.Restart +import qualified Annex.Branch +import qualified Git +import qualified Git.LsFiles +import qualified Git.Command.Batch +import qualified Git.Config +import Utility.ThreadScheduler +import qualified Assistant.Threads.Watcher as Watcher +import Utility.Batch +import Utility.NotificationBroadcaster +import Config +import Utility.HumanTime +import Utility.Tense +import Git.Repair +import Git.Index +import Assistant.Unused +import Logs.Unused +import Types.Transfer +import Types.Key +import Annex.Path +import qualified Annex +#ifdef WITH_WEBAPP +import Assistant.WebApp.Types +#endif +#ifndef mingw32_HOST_OS +import Utility.LogFile +import Utility.DiskFree +#endif + +import Data.Time.Clock.POSIX +import qualified Data.Text as T + +{- This thread runs once at startup, and most other threads wait for it + - to finish. (However, the webapp thread does not, to prevent the UI + - being nonresponsive.) -} +sanityCheckerStartupThread :: Maybe Duration -> NamedThread +sanityCheckerStartupThread startupdelay = namedThreadUnchecked "SanityCheckerStartup" $ do + {- Stale git locks can prevent commits from happening, etc. -} + void $ repairStaleGitLocks =<< liftAnnex gitRepo + + {- A corrupt index file can prevent the assistant from working at + - all, so detect and repair. -} + ifM (not <$> liftAnnex (inRepo checkIndexFast)) + ( do + notice ["corrupt index file found at startup; removing and restaging"] + liftAnnex $ inRepo $ nukeFile . indexFile + {- Normally the startup scan avoids re-staging files, + - but with the index deleted, everything needs to be + - restaged. -} + modifyDaemonStatus_ $ \s -> s { forceRestage = True } + , whenM (liftAnnex $ inRepo missingIndex) $ do + debug ["no index file; restaging"] + modifyDaemonStatus_ $ \s -> s { forceRestage = True } + ) + {- If the git-annex index file is corrupt, it's ok to remove it; + - the data from the git-annex branch will be used, and the index + - will be automatically regenerated. -} + unlessM (liftAnnex $ Annex.Branch.withIndex $ inRepo $ Git.Repair.checkIndexFast) $ do + notice ["corrupt annex/index file found at startup; removing"] + liftAnnex $ liftIO . nukeFile =<< fromRepo gitAnnexIndex + + {- Fix up ssh remotes set up by past versions of the assistant. -} + liftIO $ fixUpSshRemotes + + {- Clean up old temp files. -} + void $ liftAnnex $ tryNonAsync $ do + cleanOldTmpMisc + cleanReallyOldTmp + + {- If there's a startup delay, it's done here. -} + liftIO $ maybe noop (threadDelaySeconds . Seconds . fromIntegral . durationSeconds) startupdelay + + {- Notify other threads that the startup sanity check is done. -} + status <- getDaemonStatus + liftIO $ sendNotification $ startupSanityCheckNotifier status + +{- This thread wakes up hourly for inxepensive frequent sanity checks. -} +sanityCheckerHourlyThread :: NamedThread +sanityCheckerHourlyThread = namedThread "SanityCheckerHourly" $ forever $ do + liftIO $ threadDelaySeconds $ Seconds oneHour + hourlyCheck + +{- This thread wakes up daily to make sure the tree is in good shape. -} +sanityCheckerDailyThread :: UrlRenderer -> NamedThread +sanityCheckerDailyThread urlrenderer = namedThread "SanityCheckerDaily" $ forever $ do + waitForNextCheck + + debug ["starting sanity check"] + void $ alertWhile sanityCheckAlert go + debug ["sanity check complete"] + where + go = do + modifyDaemonStatus_ $ \s -> s { sanityCheckRunning = True } + + now <- liftIO getPOSIXTime -- before check started + r <- either showerr return + =<< (tryIO . batch) <~> dailyCheck urlrenderer + + modifyDaemonStatus_ $ \s -> s + { sanityCheckRunning = False + , lastSanityCheck = Just now + } + + return r + + showerr e = do + liftAnnex $ warning $ show e + return False + +{- Only run one check per day, from the time of the last check. -} +waitForNextCheck :: Assistant () +waitForNextCheck = do + v <- lastSanityCheck <$> getDaemonStatus + now <- liftIO getPOSIXTime + liftIO $ threadDelaySeconds $ Seconds $ calcdelay now v + where + calcdelay _ Nothing = oneDay + calcdelay now (Just lastcheck) + | lastcheck < now = max oneDay $ + oneDay - truncate (now - lastcheck) + | otherwise = oneDay + +{- It's important to stay out of the Annex monad as much as possible while + - running potentially expensive parts of this check, since remaining in it + - will block the watcher. -} +dailyCheck :: UrlRenderer -> Assistant Bool +dailyCheck urlrenderer = do + checkRepoExists + + g <- liftAnnex gitRepo + batchmaker <- liftIO getBatchCommandMaker + + -- Find old unstaged symlinks, and add them to git. + (unstaged, cleanup) <- liftIO $ Git.LsFiles.notInRepo False ["."] g + now <- liftIO getPOSIXTime + forM_ unstaged $ \file -> do + ms <- liftIO $ catchMaybeIO $ getSymbolicLinkStatus file + case ms of + Just s | toonew (statusChangeTime s) now -> noop + | isSymbolicLink s -> addsymlink file ms + _ -> noop + liftIO $ void cleanup + + {- Allow git-gc to run once per day. More frequent gc is avoided + - by default to avoid slowing things down. Only run repacks when 100x + - the usual number of loose objects are present; we tend + - to have a lot of small objects and they should not be a + - significant size. -} + when (Git.Config.getMaybe "gc.auto" g == Just "0") $ + liftIO $ void $ Git.Command.Batch.run batchmaker + [ Param "-c", Param "gc.auto=670000" + , Param "gc" + , Param "--auto" + ] g + + {- Check if the unused files found last time have been dealt with. -} + checkOldUnused urlrenderer + + {- Run git-annex unused once per day. This is run as a separate + - process to stay out of the annex monad and so it can run as a + - batch job. -} + program <- liftIO programPath + let (program', params') = batchmaker (program, [Param "unused"]) + void $ liftIO $ boolSystem program' params' + {- Invalidate unused keys cache, and queue transfers of all unused + - keys, or if no transfers are called for, drop them. -} + unused <- liftAnnex unusedKeys' + void $ liftAnnex $ setUnusedKeys unused + forM_ unused $ \k -> do + unlessM (queueTransfers "unused" Later k (AssociatedFile Nothing) Upload) $ + handleDrops "unused" True k (AssociatedFile Nothing) [] + + return True + where + toonew timestamp now = now < (realToFrac (timestamp + slop) :: POSIXTime) + slop = fromIntegral tenMinutes + insanity msg = do + liftAnnex $ warning msg + void $ addAlert $ sanityCheckFixAlert msg + addsymlink file s = do + isdirect <- liftAnnex isDirect + Watcher.runHandler (Watcher.onAddSymlink isdirect) file s + insanity $ "found unstaged symlink: " ++ file + +hourlyCheck :: Assistant () +hourlyCheck = do + checkRepoExists +#ifndef mingw32_HOST_OS + checkLogSize 0 +#else + noop +#endif + +#ifndef mingw32_HOST_OS +{- Rotate logs once when total log file size is > 2 mb. + - + - If total log size is larger than the amount of free disk space, + - continue rotating logs until size is < 2 mb, even if this + - results in immediately losing the just logged data. + -} +checkLogSize :: Int -> Assistant () +checkLogSize n = do + f <- liftAnnex $ fromRepo gitAnnexLogFile + logs <- liftIO $ listLogs f + totalsize <- liftIO $ sum <$> mapM getFileSize logs + when (totalsize > 2 * oneMegabyte) $ do + notice ["Rotated logs due to size:", show totalsize] + liftIO $ openLog f >>= handleToFd >>= redirLog + when (n < maxLogs + 1) $ do + df <- liftIO $ getDiskFree $ takeDirectory f + case df of + Just free + | free < fromIntegral totalsize -> + checkLogSize (n + 1) + _ -> noop + where + oneMegabyte :: Integer + oneMegabyte = 1000000 +#endif + +oneHour :: Int +oneHour = 60 * 60 + +oneDay :: Int +oneDay = 24 * oneHour + +{- If annex.expireunused is set, find any keys that have lingered unused + - for the specified duration, and remove them. + - + - Otherwise, check to see if unused keys are piling up, and let the user + - know. -} +checkOldUnused :: UrlRenderer -> Assistant () +checkOldUnused urlrenderer = go =<< annexExpireUnused <$> liftAnnex Annex.getGitConfig + where + go (Just Nothing) = noop + go (Just (Just expireunused)) = expireUnused (Just expireunused) + go Nothing = maybe noop promptconfig =<< describeUnusedWhenBig + + promptconfig msg = +#ifdef WITH_WEBAPP + do + button <- mkAlertButton True (T.pack "Configure") urlrenderer ConfigUnusedR + void $ addAlert $ unusedFilesAlert [button] $ T.unpack $ renderTense Present msg +#else + debug [show $ renderTense Past msg] +#endif + +{- Files may be left in misctmp by eg, an interrupted add of files + - by the assistant, which hard links files to there as part of lockdown + - checks. Delete these files if they're more than a day old. + - + - Note that this is not safe to run after the Watcher starts up, since it + - will create such files, and due to hard linking they may have old + - mtimes. So, this should only be called from the + - sanityCheckerStartupThread, which runs before the Watcher starts up. + - + - Also, if a git-annex add is being run at the same time the assistant + - starts up, its tmp files could be deleted. However, the watcher will + - come along and add everything once it starts up anyway, so at worst + - this would make the git-annex add fail unexpectedly. + -} +cleanOldTmpMisc :: Annex () +cleanOldTmpMisc = do + now <- liftIO getPOSIXTime + let oldenough = now - (60 * 60 * 24) + tmp <- fromRepo gitAnnexTmpMiscDir + liftIO $ mapM_ (cleanOld (<= oldenough)) =<< dirContentsRecursive tmp + +{- While .git/annex/tmp is now only used for storing partially transferred + - objects, older versions of git-annex used it for misctemp. Clean up any + - files that might be left from that, by looking for files whose names + - cannot be the key of an annexed object. Only delete files older than + - 1 week old. + - + - Also, some remotes such as rsync may use this temp directory for storing + - eg, encrypted objects that are being transferred. So, delete old + - objects that use a GPGHMAC backend. + -} +cleanReallyOldTmp :: Annex () +cleanReallyOldTmp = do + now <- liftIO getPOSIXTime + let oldenough = now - (60 * 60 * 24 * 7) + tmp <- fromRepo gitAnnexTmpObjectDir + liftIO $ mapM_ (cleanjunk (<= oldenough)) =<< dirContentsRecursive tmp + where + cleanjunk check f = case fileKey (takeFileName f) of + Nothing -> cleanOld check f + Just k + | "GPGHMAC" `isPrefixOf` formatKeyVariety (keyVariety k) -> + cleanOld check f + | otherwise -> noop + +cleanOld :: (POSIXTime -> Bool) -> FilePath -> IO () +cleanOld check f = go =<< catchMaybeIO getmtime + where + getmtime = realToFrac . modificationTime <$> getSymbolicLinkStatus f + go (Just mtime) | check mtime = nukeFile f + go _ = noop + +checkRepoExists :: Assistant () +checkRepoExists = do + g <- liftAnnex gitRepo + liftIO $ unlessM (doesDirectoryExist $ Git.repoPath g) $ + terminateSelf diff --git a/Assistant/Threads/TransferPoller.hs b/Assistant/Threads/TransferPoller.hs new file mode 100644 index 0000000000..f5d6890c8f --- /dev/null +++ b/Assistant/Threads/TransferPoller.hs @@ -0,0 +1,56 @@ +{- git-annex assistant transfer polling thread + - + - Copyright 2012 Joey Hess + - + - Licensed under the GNU GPL version 3 or higher. + -} + +module Assistant.Threads.TransferPoller where + +import Assistant.Common +import Assistant.DaemonStatus +import Types.Transfer +import Logs.Transfer +import Utility.NotificationBroadcaster +import qualified Assistant.Threads.TransferWatcher as TransferWatcher + +import Control.Concurrent +import qualified Data.Map as M + +{- This thread polls the status of ongoing transfers, determining how much + - of each transfer is complete. -} +transferPollerThread :: NamedThread +transferPollerThread = namedThread "TransferPoller" $ do + g <- liftAnnex gitRepo + tn <- liftIO . newNotificationHandle True =<< + transferNotifier <$> getDaemonStatus + forever $ do + liftIO $ threadDelay 500000 -- 0.5 seconds + ts <- currentTransfers <$> getDaemonStatus + if M.null ts + -- block until transfers running + then liftIO $ waitNotification tn + else mapM_ (poll g) $ M.toList ts + where + poll g (t, info) + {- Downloads are polled by checking the size of the + - temp file being used for the transfer. -} + | transferDirection t == Download = do + let f = gitAnnexTmpObjectLocation (transferKey t) g + sz <- liftIO $ catchMaybeIO $ getFileSize f + newsize t info sz + {- Uploads don't need to be polled for when the TransferWatcher + - thread can track file modifications. -} + | TransferWatcher.watchesTransferSize = noop + {- Otherwise, this code polls the upload progress + - by reading the transfer info file. -} + | otherwise = do + let f = transferFile t g + mi <- liftIO $ catchDefaultIO Nothing $ + readTransferInfoFile Nothing f + maybe noop (newsize t info . bytesComplete) mi + + newsize t info sz + | bytesComplete info /= sz && isJust sz = + alterTransferInfo t $ \i -> i { bytesComplete = sz } + | otherwise = noop diff --git a/Assistant/Threads/TransferScanner.hs b/Assistant/Threads/TransferScanner.hs new file mode 100644 index 0000000000..e76c0b4fbc --- /dev/null +++ b/Assistant/Threads/TransferScanner.hs @@ -0,0 +1,195 @@ +{- git-annex assistant thread to scan remotes to find needed transfers + - + - Copyright 2012 Joey Hess + - + - Licensed under the GNU GPL version 3 or higher. + -} + +module Assistant.Threads.TransferScanner where + +import Assistant.Common +import Assistant.Types.ScanRemotes +import Assistant.ScanRemotes +import Assistant.TransferQueue +import Assistant.DaemonStatus +import Assistant.Drop +import Assistant.Sync +import Assistant.DeleteRemote +import Assistant.Types.UrlRenderer +import Types.Transfer +import Logs.Transfer +import Logs.Location +import Logs.Group +import qualified Remote +import qualified Types.Remote as Remote +import Utility.ThreadScheduler +import Utility.NotificationBroadcaster +import Utility.Batch +import qualified Git.LsFiles as LsFiles +import Annex.WorkTree +import Annex.Content +import Annex.Wanted +import CmdLine.Action + +import qualified Data.Set as S +import Control.Concurrent + +{- This thread waits until a remote needs to be scanned, to find transfers + - that need to be made, to keep data in sync. + -} +transferScannerThread :: UrlRenderer -> NamedThread +transferScannerThread urlrenderer = namedThread "TransferScanner" $ do + startupScan + go S.empty + where + go scanned = do + scanrunning False + liftIO $ threadDelaySeconds (Seconds 2) + (rs, infos) <- unzip <$> getScanRemote + scanrunning True + if any fullScan infos || any (`S.notMember` scanned) rs + then do + expensiveScan urlrenderer rs + go $ scanned `S.union` S.fromList rs + else do + mapM_ failedTransferScan rs + go scanned + scanrunning b = do + ds <- modifyDaemonStatus $ \s -> + (s { transferScanRunning = b }, s) + liftIO $ sendNotification $ transferNotifier ds + + {- All git remotes are synced, all exports are updated, + - and all available remotes are scanned in full on startup, + - for multiple reasons, including: + - + - * This may be the first run, and there may be remotes + - already in place, that need to be synced. + - * Changes may have been made last time we run, but remotes were + - not available to be synced with. + - * Changes may have been made to remotes while we were down. + - * We may have run before, and scanned a remote, but + - only been in a subdirectory of the git remote, and so + - not synced it all. + - * We may have run before, and had transfers queued, + - and then the system (or us) crashed, and that info was + - lost. + - * A remote may be in the unwanted group, and this is a chance + - to determine if the remote has been emptied. + -} + startupScan = do + reconnectRemotes =<< syncGitRemotes <$> getDaemonStatus + addScanRemotes True =<< scannableRemotes + +{- This is a cheap scan for failed transfers involving a remote. -} +failedTransferScan :: Remote -> Assistant () +failedTransferScan r = do + failed <- liftAnnex $ clearFailedTransfers (Remote.uuid r) + mapM_ retry failed + where + retry (t, info) + | transferDirection t == Download = + {- Check if the remote still has the key. + - If not, relies on the expensiveScan to + - get it queued from some other remote. -} + whenM (liftAnnex $ remoteHas r $ transferKey t) $ + requeue t info + | otherwise = + {- The Transferrer checks when uploading + - that the remote doesn't already have the + - key, so it's not redundantly checked here. -} + requeue t info + requeue t info = queueTransferWhenSmall "retrying failed transfer" (associatedFile info) t r + +{- This is a expensive scan through the full git work tree, finding + - files to transfer. The scan is blocked when the transfer queue gets + - too large. + - + - This also finds files that are present either here or on a remote + - but that are not preferred content, and drops them. Searching for files + - to drop is done concurrently with the scan for transfers. + - + - TODO: It would be better to first drop as much as we can, before + - transferring much, to minimise disk use. + - + - During the scan, we'll also check if any unwanted repositories are empty, + - and can be removed. While unrelated, this is a cheap place to do it, + - since we need to look at the locations of all keys anyway. + -} +expensiveScan :: UrlRenderer -> [Remote] -> Assistant () +expensiveScan urlrenderer rs = batch <~> do + debug ["starting scan of", show visiblers] + + let us = map Remote.uuid rs + + mapM_ (liftAnnex . clearFailedTransfers) us + + unwantedrs <- liftAnnex $ S.fromList + <$> filterM inUnwantedGroup us + + g <- liftAnnex gitRepo + (files, cleanup) <- liftIO $ LsFiles.inRepo [] g + removablers <- scan unwantedrs files + void $ liftIO cleanup + + debug ["finished scan of", show visiblers] + + remove <- asIO1 $ removableRemote urlrenderer + liftIO $ mapM_ (void . tryNonAsync . remove) $ S.toList removablers + where + visiblers = let rs' = filter (not . Remote.readonly) rs + in if null rs' then rs else rs' + + scan unwanted [] = return unwanted + scan unwanted (f:fs) = do + (unwanted', ts) <- maybe + (return (unwanted, [])) + (findtransfers f unwanted) + =<< liftAnnex (lookupFile f) + mapM_ (enqueue f) ts + + {- Delay for a short time to avoid using too much CPU. -} + liftIO $ threadDelay $ fromIntegral $ oneSecond `div` 200 + + scan unwanted' fs + + enqueue f (r, t) = + queueTransferWhenSmall "expensive scan found missing object" + (AssociatedFile (Just f)) t r + findtransfers f unwanted key = do + let af = AssociatedFile (Just f) + locs <- liftAnnex $ loggedLocations key + present <- liftAnnex $ inAnnex key + let slocs = S.fromList locs + + {- The remotes may have changed since this scan began. -} + syncrs <- syncDataRemotes <$> getDaemonStatus + let use l a = mapMaybe (a key slocs) . l <$> getDaemonStatus + + liftAnnex $ handleDropsFrom locs syncrs + "expensive scan found too many copies of object" + present key af [] callCommandAction + ts <- if present + then liftAnnex . filterM (wantSend True (Just key) af . Remote.uuid . fst) + =<< use syncDataRemotes (genTransfer Upload False) + else ifM (liftAnnex $ wantGet True (Just key) af) + ( use downloadRemotes (genTransfer Download True) , return [] ) + let unwanted' = S.difference unwanted slocs + return (unwanted', ts) + +-- Both syncDataRemotes and exportRemotes can be scanned. +-- The downloadRemotes list contains both. +scannableRemotes :: Assistant [Remote] +scannableRemotes = downloadRemotes <$> getDaemonStatus + +genTransfer :: Direction -> Bool -> Key -> S.Set UUID -> Remote -> Maybe (Remote, Transfer) +genTransfer direction want key slocs r + | direction == Upload && Remote.readonly r = Nothing + | S.member (Remote.uuid r) slocs == want = Just + (r, Transfer direction (Remote.uuid r) key) + | otherwise = Nothing + +remoteHas :: Remote -> Key -> Annex Bool +remoteHas r key = elem + <$> pure (Remote.uuid r) + <*> loggedLocations key diff --git a/Assistant/Threads/TransferWatcher.hs b/Assistant/Threads/TransferWatcher.hs new file mode 100644 index 0000000000..a04c6c01c8 --- /dev/null +++ b/Assistant/Threads/TransferWatcher.hs @@ -0,0 +1,105 @@ +{- git-annex assistant transfer watching thread + - + - Copyright 2012 Joey Hess + - + - Licensed under the GNU GPL version 3 or higher. + -} + +module Assistant.Threads.TransferWatcher where + +import Assistant.Common +import Assistant.DaemonStatus +import Assistant.TransferSlots +import Types.Transfer +import Logs.Transfer +import Utility.DirWatcher +import Utility.DirWatcher.Types +import qualified Remote + +import Control.Concurrent +import qualified Data.Map as M + +{- This thread watches for changes to the gitAnnexTransferDir, + - and updates the DaemonStatus's map of ongoing transfers. -} +transferWatcherThread :: NamedThread +transferWatcherThread = namedThread "TransferWatcher" $ do + dir <- liftAnnex $ gitAnnexTransferDir <$> gitRepo + liftIO $ createDirectoryIfMissing True dir + let hook a = Just <$> asIO2 (runHandler a) + addhook <- hook onAdd + delhook <- hook onDel + modifyhook <- hook onModify + errhook <- hook onErr + let hooks = mkWatchHooks + { addHook = addhook + , delHook = delhook + , modifyHook = modifyhook + , errHook = errhook + } + void $ liftIO $ watchDir dir (const False) True hooks id + debug ["watching for transfers"] + +type Handler = FilePath -> Assistant () + +{- Runs an action handler. + - + - Exceptions are ignored, otherwise a whole thread could be crashed. + -} +runHandler :: Handler -> FilePath -> Maybe FileStatus -> Assistant () +runHandler handler file _filestatus = + either (liftIO . print) (const noop) =<< tryIO <~> handler file + +{- Called when there's an error with inotify. -} +onErr :: Handler +onErr = error + +{- Called when a new transfer information file is written. -} +onAdd :: Handler +onAdd file = case parseTransferFile file of + Nothing -> noop + Just t -> go t =<< liftAnnex (checkTransfer t) + where + go _ Nothing = noop -- transfer already finished + go t (Just info) = do + debug [ "transfer starting:", describeTransfer t info ] + r <- liftAnnex $ Remote.remoteFromUUID $ transferUUID t + updateTransferInfo t info { transferRemote = r } + +{- Called when a transfer information file is updated. + - + - The only thing that should change in the transfer info is the + - bytesComplete, so that's the only thing updated in the DaemonStatus. -} +onModify :: Handler +onModify file = case parseTransferFile file of + Nothing -> noop + Just t -> go t =<< liftIO (readTransferInfoFile Nothing file) + where + go _ Nothing = noop + go t (Just newinfo) = alterTransferInfo t $ + \i -> i { bytesComplete = bytesComplete newinfo } + +{- This thread can only watch transfer sizes when the DirWatcher supports + - tracking modificatons to files. -} +watchesTransferSize :: Bool +watchesTransferSize = modifyTracked + +{- Called when a transfer information file is removed. -} +onDel :: Handler +onDel file = case parseTransferFile file of + Nothing -> noop + Just t -> do + debug [ "transfer finishing:", show t] + minfo <- removeTransfer t + + -- Run transfer hook. + m <- transferHook <$> getDaemonStatus + maybe noop (\hook -> void $ liftIO $ forkIO $ hook t) + (M.lookup (transferKey t) m) + + finished <- asIO2 finishedTransfer + void $ liftIO $ forkIO $ do + {- XXX race workaround delay. The location + - log needs to be updated before finishedTransfer + - runs. -} + threadDelay 10000000 -- 10 seconds + finished t minfo diff --git a/Assistant/Threads/Transferrer.hs b/Assistant/Threads/Transferrer.hs new file mode 100644 index 0000000000..293ce41c27 --- /dev/null +++ b/Assistant/Threads/Transferrer.hs @@ -0,0 +1,27 @@ +{- git-annex assistant data transferrer thread + - + - Copyright 2012 Joey Hess + - + - Licensed under the GNU GPL version 3 or higher. + -} + +module Assistant.Threads.Transferrer where + +import Assistant.Common +import Assistant.TransferQueue +import Assistant.TransferSlots +import Types.Transfer +import Annex.Path +import Utility.Batch + +{- Dispatches transfers from the queue. -} +transfererThread :: NamedThread +transfererThread = namedThread "Transferrer" $ do + program <- liftIO programPath + batchmaker <- liftIO getBatchCommandMaker + forever $ inTransferSlot program batchmaker $ + maybe (return Nothing) (uncurry genTransfer) + =<< getNextTransfer notrunning + where + {- Skip transfers that are already running. -} + notrunning = isNothing . startedTime diff --git a/Assistant/Threads/UpgradeWatcher.hs b/Assistant/Threads/UpgradeWatcher.hs new file mode 100644 index 0000000000..a50c845969 --- /dev/null +++ b/Assistant/Threads/UpgradeWatcher.hs @@ -0,0 +1,109 @@ +{- git-annex assistant thread to detect when git-annex is upgraded + - + - Copyright 2013 Joey Hess + - + - Licensed under the GNU GPL version 3 or higher. + -} + +{-# LANGUAGE CPP #-} + +module Assistant.Threads.UpgradeWatcher ( + upgradeWatcherThread +) where + +import Assistant.Common +import Assistant.Upgrade +import Utility.DirWatcher +import Utility.DirWatcher.Types +import Utility.ThreadScheduler +import Assistant.Types.UrlRenderer +import Assistant.Alert +import Assistant.DaemonStatus +#ifdef WITH_WEBAPP +import Assistant.WebApp.Types +import qualified BuildInfo +#endif + +import Control.Concurrent.MVar +import qualified Data.Text as T + +data WatcherState = InStartupScan | Started | Upgrading + deriving (Eq) + +upgradeWatcherThread :: UrlRenderer -> NamedThread +upgradeWatcherThread urlrenderer = namedThread "UpgradeWatcher" $ do + whenM (liftIO checkSuccessfulUpgrade) $ + showSuccessfulUpgrade urlrenderer + go =<< liftIO upgradeFlagFile + where + go flagfile = do + mvar <- liftIO $ newMVar InStartupScan + changed <- Just <$> asIO2 (changedFile urlrenderer mvar flagfile) + let hooks = mkWatchHooks + { addHook = changed + , delHook = changed + , addSymlinkHook = changed + , modifyHook = changed + , delDirHook = changed + } + let dir = parentDir flagfile + let depth = length (splitPath dir) + 1 + let nosubdirs f = length (splitPath f) == depth + void $ liftIO $ watchDir dir nosubdirs False hooks (startup mvar) + -- Ignore bogus events generated during the startup scan. + -- We ask the watcher to not generate them, but just to be safe.. + startup mvar scanner = do + r <- scanner + void $ swapMVar mvar Started + return r + +changedFile :: UrlRenderer -> MVar WatcherState -> FilePath -> FilePath -> Maybe FileStatus -> Assistant () +changedFile urlrenderer mvar flagfile file _status + | flagfile /= file = noop + | otherwise = do + state <- liftIO $ readMVar mvar + when (state == Started) $ do + setstate Upgrading + ifM (liftIO upgradeSanityCheck) + ( handleUpgrade urlrenderer + , do + debug ["new version failed sanity check; not using"] + setstate Started + ) + where + setstate = void . liftIO . swapMVar mvar + +handleUpgrade :: UrlRenderer -> Assistant () +handleUpgrade urlrenderer = do + -- Wait 2 minutes for any final upgrade changes to settle. + -- (For example, other associated files may be being put into + -- place.) Not needed when using a distribution bundle, because + -- in that case git-annex handles the upgrade in a non-racy way. + liftIO $ unlessM usingDistribution $ + threadDelaySeconds (Seconds 120) + ifM autoUpgradeEnabled + ( do + debug ["starting automatic upgrade"] + unattendedUpgrade +#ifdef WITH_WEBAPP + , do + button <- mkAlertButton True (T.pack "Finish Upgrade") urlrenderer ConfigFinishUpgradeR + void $ addAlert $ upgradeReadyAlert button +#else + , noop +#endif + ) + +showSuccessfulUpgrade :: UrlRenderer -> Assistant () +showSuccessfulUpgrade urlrenderer = do +#ifdef WITH_WEBAPP + button <- ifM autoUpgradeEnabled + ( pure Nothing + , Just <$> mkAlertButton True + (T.pack "Enable Automatic Upgrades") + urlrenderer ConfigEnableAutomaticUpgradeR + ) + void $ addAlert $ upgradeFinishedAlert button BuildInfo.packageversion +#else + noop +#endif diff --git a/Assistant/Threads/Upgrader.hs b/Assistant/Threads/Upgrader.hs new file mode 100644 index 0000000000..75c4353b99 --- /dev/null +++ b/Assistant/Threads/Upgrader.hs @@ -0,0 +1,85 @@ +{- git-annex assistant thread to detect when upgrade is available + - + - Copyright 2013 Joey Hess + - + - Licensed under the GNU GPL version 3 or higher. + -} + +{-# LANGUAGE CPP #-} + +module Assistant.Threads.Upgrader ( + upgraderThread +) where + +import Assistant.Common +import Assistant.Upgrade + +import Assistant.Types.UrlRenderer +import Assistant.DaemonStatus +import Assistant.Alert +import Utility.NotificationBroadcaster +import qualified Annex +import qualified BuildInfo +import qualified Utility.DottedVersion as DottedVersion +import Types.Distribution +#ifdef WITH_WEBAPP +import Assistant.WebApp.Types +#endif + +import Data.Time.Clock +import qualified Data.Text as T + +upgraderThread :: UrlRenderer -> NamedThread +upgraderThread urlrenderer = namedThread "Upgrader" $ + when (isJust BuildInfo.upgradelocation) $ do + {- Check for upgrade on startup, unless it was just + - upgraded. -} + unlessM (liftIO checkSuccessfulUpgrade) $ + checkUpgrade urlrenderer + h <- liftIO . newNotificationHandle False . networkConnectedNotifier =<< getDaemonStatus + go h =<< liftIO getCurrentTime + where + {- Wait for a network connection event. Then see if it's been + - half a day since the last upgrade check. If so, proceed with + - check. -} + go h lastchecked = do + liftIO $ waitNotification h + autoupgrade <- liftAnnex $ annexAutoUpgrade <$> Annex.getGitConfig + if autoupgrade == NoAutoUpgrade + then go h lastchecked + else do + now <- liftIO getCurrentTime + if diffUTCTime now lastchecked > halfday + then do + checkUpgrade urlrenderer + go h =<< liftIO getCurrentTime + else go h lastchecked + halfday = 12 * 60 * 60 + +checkUpgrade :: UrlRenderer -> Assistant () +checkUpgrade urlrenderer = do + debug [ "Checking if an upgrade is available." ] + go =<< downloadDistributionInfo + where + go Nothing = debug [ "Failed to check if upgrade is available." ] + go (Just d) = do + let installed = DottedVersion.normalize BuildInfo.packageversion + let avail = DottedVersion.normalize $ distributionVersion d + let old = DottedVersion.normalize <$> distributionUrgentUpgrade d + if Just installed <= old + then canUpgrade High urlrenderer d + else if installed < avail + then canUpgrade Low urlrenderer d + else debug [ "No new version found." ] + +canUpgrade :: AlertPriority -> UrlRenderer -> GitAnnexDistribution -> Assistant () +canUpgrade urgency urlrenderer d = ifM autoUpgradeEnabled + ( startDistributionDownload d + , do +#ifdef WITH_WEBAPP + button <- mkAlertButton True (T.pack "Upgrade") urlrenderer (ConfigStartUpgradeR d) + void $ addAlert (canUpgradeAlert urgency (distributionVersion d) button) +#else + noop +#endif + ) diff --git a/Assistant/Threads/Watcher.hs b/Assistant/Threads/Watcher.hs new file mode 100644 index 0000000000..742b8c88fa --- /dev/null +++ b/Assistant/Threads/Watcher.hs @@ -0,0 +1,416 @@ +{- git-annex assistant tree watcher + - + - Copyright 2012-2015 Joey Hess + - + - Licensed under the GNU GPL version 3 or higher. + -} + +{-# LANGUAGE DeriveDataTypeable, CPP #-} + +module Assistant.Threads.Watcher ( + watchThread, + WatcherControl(..), + checkCanWatch, + needLsof, + onAddSymlink, + runHandler, +) where + +import Assistant.Common +import Assistant.DaemonStatus +import Assistant.Changes +import Assistant.Types.Changes +import Assistant.Alert +import Utility.DirWatcher +import Utility.DirWatcher.Types +import qualified Annex +import qualified Annex.Queue +import qualified Git +import qualified Git.UpdateIndex +import qualified Git.LsFiles as LsFiles +import Annex.WorkTree +import Annex.Direct +import Annex.Content.Direct +import Annex.CatFile +import Annex.CheckIgnore +import Annex.Link +import Annex.FileMatcher +import Annex.Content +import Annex.ReplaceFile +import Annex.Version +import Annex.InodeSentinal +import Git.Types +import Git.FilePath +import Config +import Config.GitConfig +import Utility.ThreadScheduler +import Utility.FileSystemEncoding +import Logs.Location +import qualified Database.Keys +#ifndef mingw32_HOST_OS +import qualified Utility.Lsof as Lsof +#endif + +import Data.Typeable +import qualified Data.ByteString.Lazy as L +import qualified Control.Exception as E +import Data.Time.Clock + +checkCanWatch :: Annex () +checkCanWatch + | canWatch = do +#ifndef mingw32_HOST_OS + liftIO Lsof.setup + unlessM (liftIO (inPath "lsof") <||> Annex.getState Annex.force) + needLsof +#else + noop +#endif + | otherwise = giveup "watch mode is not available on this system" + +needLsof :: Annex () +needLsof = giveup $ unlines + [ "The lsof command is needed for watch mode to be safe, and is not in PATH." + , "To override lsof checks to ensure that files are not open for writing" + , "when added to the annex, you can use --force" + , "Be warned: This can corrupt data in the annex, and make fsck complain." + ] + +{- A special exception that can be thrown to pause or resume the watcher. -} +data WatcherControl = PauseWatcher | ResumeWatcher + deriving (Show, Eq, Typeable) + +instance E.Exception WatcherControl + +watchThread :: NamedThread +watchThread = namedThread "Watcher" $ + ifM (liftAnnex $ getGitConfigVal annexAutoCommit) + ( runWatcher + , waitFor ResumeWatcher runWatcher + ) + +runWatcher :: Assistant () +runWatcher = do + startup <- asIO1 startupScan + matcher <- liftAnnex largeFilesMatcher + direct <- liftAnnex isDirect + unlocked <- liftAnnex versionSupportsUnlockedPointers + symlinkssupported <- liftAnnex $ coreSymlinks <$> Annex.getGitConfig + addhook <- hook $ if unlocked + then onAddUnlocked symlinkssupported matcher + else if direct + then onAddDirect symlinkssupported matcher + else onAdd matcher + delhook <- hook onDel + addsymlinkhook <- hook $ onAddSymlink direct + deldirhook <- hook onDelDir + errhook <- hook onErr + let hooks = mkWatchHooks + { addHook = addhook + , delHook = delhook + , addSymlinkHook = addsymlinkhook + , delDirHook = deldirhook + , errHook = errhook + } + scanevents <- liftAnnex $ annexStartupScan <$> Annex.getGitConfig + h <- liftIO $ watchDir "." ignored scanevents hooks startup + debug [ "watching", "."] + + {- Let the DirWatcher thread run until signalled to pause it, + - then wait for a resume signal, and restart. -} + waitFor PauseWatcher $ do + liftIO $ stopWatchDir h + waitFor ResumeWatcher runWatcher + where + hook a = Just <$> asIO2 (runHandler a) + +waitFor :: WatcherControl -> Assistant () -> Assistant () +waitFor sig next = do + r <- liftIO (E.try pause :: IO (Either E.SomeException ())) + case r of + Left e -> case E.fromException e of + Just s + | s == sig -> next + _ -> noop + _ -> noop + where + pause = runEvery (Seconds 86400) noop + +{- Initial scartup scan. The action should return once the scan is complete. -} +startupScan :: IO a -> Assistant a +startupScan scanner = do + liftAnnex $ showAction "scanning" + alertWhile' startupScanAlert $ do + r <- liftIO scanner + + -- Notice any files that were deleted before + -- watching was started. + top <- liftAnnex $ fromRepo Git.repoPath + (fs, cleanup) <- liftAnnex $ inRepo $ LsFiles.deleted [top] + forM_ fs $ \f -> do + liftAnnex $ onDel' f + maybe noop recordChange =<< madeChange f RmChange + void $ liftIO cleanup + + liftAnnex $ showAction "started" + liftIO $ putStrLn "" + + modifyDaemonStatus_ $ \s -> s { scanComplete = True } + + -- Ensure that the Committer sees any changes + -- that it did not process, and acts on them now that + -- the scan is complete. + refillChanges =<< getAnyChanges + + return (True, r) + +{- Hardcoded ignores, passed to the DirWatcher so it can avoid looking + - at the entire .git directory. Does not include .gitignores. -} +ignored :: FilePath -> Bool +ignored = ig . takeFileName + where + ig ".git" = True + ig ".gitignore" = True + ig ".gitattributes" = True +#ifdef darwin_HOST_OS + ig ".DS_Store" = True +#endif + ig _ = False + +unlessIgnored :: FilePath -> Assistant (Maybe Change) -> Assistant (Maybe Change) +unlessIgnored file a = ifM (liftAnnex $ checkIgnored file) + ( noChange + , a + ) + +type Handler = FilePath -> Maybe FileStatus -> Assistant (Maybe Change) + +{- Runs an action handler, and if there was a change, adds it to the ChangeChan. + - + - Exceptions are ignored, otherwise a whole watcher thread could be crashed. + -} +runHandler :: Handler -> FilePath -> Maybe FileStatus -> Assistant () +runHandler handler file filestatus = void $ do + r <- tryIO <~> handler (normalize file) filestatus + case r of + Left e -> liftIO $ warningIO $ show e + Right Nothing -> noop + Right (Just change) -> recordChange change + where + normalize f + | "./" `isPrefixOf` file = drop 2 f + | otherwise = f + +{- Small files are added to git as-is, while large ones go into the annex. -} +add :: GetFileMatcher -> FilePath -> Assistant (Maybe Change) +add largefilematcher file = ifM (liftAnnex $ checkFileMatcher largefilematcher file) + ( pendingAddChange file + , do + liftAnnex $ Annex.Queue.addCommand "add" + [Param "--force", Param "--"] [file] + madeChange file AddFileChange + ) + +onAdd :: GetFileMatcher -> Handler +onAdd matcher file filestatus + | maybe False isRegularFile filestatus = + unlessIgnored file $ + add matcher file + | otherwise = noChange + +shouldRestage :: DaemonStatus -> Bool +shouldRestage ds = scanComplete ds || forceRestage ds + +onAddUnlocked :: Bool -> GetFileMatcher -> Handler +onAddUnlocked symlinkssupported matcher f fs = do + mk <- liftIO $ isPointerFile f + case mk of + Nothing -> onAddUnlocked' False contentchanged addassociatedfile addlink samefilestatus symlinkssupported matcher f fs + Just k -> addlink f k + where + addassociatedfile key file = + Database.Keys.addAssociatedFile key + =<< inRepo (toTopFilePath file) + samefilestatus key file status = do + cache <- Database.Keys.getInodeCaches key + curr <- withTSDelta $ \delta -> liftIO $ toInodeCache delta file status + case (cache, curr) of + (_, Just c) -> elemInodeCaches c cache + ([], Nothing) -> return True + _ -> return False + contentchanged oldkey file = do + Database.Keys.removeAssociatedFile oldkey + =<< inRepo (toTopFilePath file) + unlessM (inAnnex oldkey) $ + logStatus oldkey InfoMissing + addlink file key = do + mode <- liftIO $ catchMaybeIO $ fileMode <$> getFileStatus file + liftAnnex $ stagePointerFile file mode =<< hashPointerFile key + madeChange file $ LinkChange (Just key) + +{- In direct mode, add events are received for both new files, and + - modified existing files. + -} +onAddDirect :: Bool -> GetFileMatcher -> Handler +onAddDirect = onAddUnlocked' True changedDirect addassociatedfile addlink sameFileStatus + where + addassociatedfile key file = void $ addAssociatedFile key file + addlink file key = do + link <- liftAnnex $ calcRepo $ gitAnnexLink file key + addLink file link (Just key) + +onAddUnlocked' + :: Bool + -> (Key -> FilePath -> Annex ()) + -> (Key -> FilePath -> Annex ()) + -> (FilePath -> Key -> Assistant (Maybe Change)) + -> (Key -> FilePath -> FileStatus -> Annex Bool) + -> Bool + -> GetFileMatcher + -> Handler +onAddUnlocked' isdirect contentchanged addassociatedfile addlink samefilestatus symlinkssupported matcher file fs = do + v <- liftAnnex $ catKeyFile file + case (v, fs) of + (Just key, Just filestatus) -> + ifM (liftAnnex $ samefilestatus key file filestatus) + {- It's possible to get an add event for + - an existing file that is not + - really modified, but it might have + - just been deleted and been put back, + - so its annex link is restaged to make sure. -} + ( ifM (shouldRestage <$> getDaemonStatus) + ( addlink file key + , noChange + ) + , guardSymlinkStandin (Just key) $ do + debug ["changed", file] + liftAnnex $ contentchanged key file + add matcher file + ) + _ -> unlessIgnored file $ + guardSymlinkStandin Nothing $ do + debug ["add", file] + add matcher file + where + {- On a filesystem without symlinks, we'll get changes for regular + - files that git uses to stand-in for symlinks. Detect when + - this happens, and stage the symlink, rather than annexing the + - file. -} + guardSymlinkStandin mk a + | symlinkssupported = a + | otherwise = do + linktarget <- liftAnnex $ getAnnexLinkTarget file + case linktarget of + Nothing -> a + Just lt -> do + case fileKey $ takeFileName lt of + Nothing -> noop + Just key -> liftAnnex $ + addassociatedfile key file + onAddSymlink' linktarget mk isdirect file fs + +{- A symlink might be an arbitrary symlink, which is just added. + - Or, if it is a git-annex symlink, ensure it points to the content + - before adding it. + -} +onAddSymlink :: Bool -> Handler +onAddSymlink isdirect file filestatus = unlessIgnored file $ do + linktarget <- liftIO (catchMaybeIO $ readSymbolicLink file) + kv <- liftAnnex (lookupFile file) + onAddSymlink' linktarget kv isdirect file filestatus + +onAddSymlink' :: Maybe String -> Maybe Key -> Bool -> Handler +onAddSymlink' linktarget mk isdirect file filestatus = go mk + where + go (Just key) = do + when isdirect $ + liftAnnex $ void $ addAssociatedFile key file + link <- liftAnnex $ calcRepo $ gitAnnexLink file key + if linktarget == Just link + then ensurestaged (Just link) =<< getDaemonStatus + else do + unless isdirect $ + liftAnnex $ replaceFile file $ + makeAnnexLink link + addLink file link (Just key) + -- other symlink, not git-annex + go Nothing = ensurestaged linktarget =<< getDaemonStatus + + {- This is often called on symlinks that are already + - staged correctly. A symlink may have been deleted + - and being re-added, or added when the watcher was + - not running. So they're normally restaged to make sure. + - + - As an optimisation, during the startup scan, avoid + - restaging everything. Only links that were created since + - the last time the daemon was running are staged. + - (If the daemon has never ran before, avoid staging + - links too.) + -} + ensurestaged (Just link) daemonstatus + | shouldRestage daemonstatus = addLink file link mk + | otherwise = case filestatus of + Just s + | not (afterLastDaemonRun (statusChangeTime s) daemonstatus) -> noChange + _ -> addLink file link mk + ensurestaged Nothing _ = noChange + +{- For speed, tries to reuse the existing blob for symlink target. -} +addLink :: FilePath -> FilePath -> Maybe Key -> Assistant (Maybe Change) +addLink file link mk = do + debug ["add symlink", file] + liftAnnex $ do + v <- catObjectDetails $ Ref $ ':':file + case v of + Just (currlink, sha, _type) + | s2w8 link == L.unpack currlink -> + stageSymlink file sha + _ -> stageSymlink file =<< hashSymlink link + madeChange file $ LinkChange mk + +onDel :: Handler +onDel file _ = do + debug ["file deleted", file] + liftAnnex $ onDel' file + madeChange file RmChange + +onDel' :: FilePath -> Annex () +onDel' file = do + topfile <- inRepo (toTopFilePath file) + ifM versionSupportsUnlockedPointers + ( withkey $ flip Database.Keys.removeAssociatedFile topfile + , whenM isDirect $ + withkey $ \key -> void $ removeAssociatedFile key file + ) + Annex.Queue.addUpdateIndex =<< + inRepo (Git.UpdateIndex.unstageFile file) + where + withkey a = maybe noop a =<< catKeyFile file + +{- A directory has been deleted, or moved, so tell git to remove anything + - that was inside it from its cache. Since it could reappear at any time, + - use --cached to only delete it from the index. + - + - This queues up a lot of RmChanges, which assists the Committer in + - pairing up renamed files when the directory was renamed. -} +onDelDir :: Handler +onDelDir dir _ = do + debug ["directory deleted", dir] + (fs, clean) <- liftAnnex $ inRepo $ LsFiles.deleted [dir] + + liftAnnex $ mapM_ onDel' fs + + -- Get the events queued up as fast as possible, so the + -- committer sees them all in one block. + now <- liftIO getCurrentTime + recordChanges $ map (\f -> Change now f RmChange) fs + + void $ liftIO clean + noChange + +{- Called when there's an error with inotify or kqueue. -} +onErr :: Handler +onErr msg _ = do + liftAnnex $ warning msg + void $ addAlert $ warningAlert "watcher" msg + noChange diff --git a/Assistant/Threads/WebApp.hs b/Assistant/Threads/WebApp.hs new file mode 100644 index 0000000000..dfb631bc6f --- /dev/null +++ b/Assistant/Threads/WebApp.hs @@ -0,0 +1,137 @@ +{- git-annex assistant webapp thread + - + - Copyright 2012-2014 Joey Hess + - + - Licensed under the GNU GPL version 3 or higher. + -} + +{-# LANGUAGE TemplateHaskell, MultiParamTypeClasses #-} +{-# LANGUAGE ViewPatterns, OverloadedStrings #-} +{-# LANGUAGE CPP #-} +{-# OPTIONS_GHC -fno-warn-orphans #-} + +module Assistant.Threads.WebApp where + +import Assistant.Common +import Assistant.WebApp +import Assistant.WebApp.Types +import Assistant.WebApp.DashBoard +import Assistant.WebApp.SideBar +import Assistant.WebApp.Notifications +import Assistant.WebApp.RepoList +import Assistant.WebApp.Configurators +import Assistant.WebApp.Configurators.Local +import Assistant.WebApp.Configurators.Ssh +import Assistant.WebApp.Configurators.Pairing +import Assistant.WebApp.Configurators.AWS +import Assistant.WebApp.Configurators.IA +import Assistant.WebApp.Configurators.WebDAV +import Assistant.WebApp.Configurators.Preferences +import Assistant.WebApp.Configurators.Unused +import Assistant.WebApp.Configurators.Edit +import Assistant.WebApp.Configurators.Delete +import Assistant.WebApp.Configurators.Fsck +import Assistant.WebApp.Configurators.Upgrade +import Assistant.WebApp.Documentation +import Assistant.WebApp.Control +import Assistant.WebApp.OtherRepos +import Assistant.WebApp.Repair +import Assistant.WebApp.Pairing +import Assistant.Types.ThreadedMonad +import Utility.WebApp +import Utility.AuthToken +import Utility.Tmp +import Utility.FileMode +import Git +import qualified Annex + +import Yesod +import Network.Socket (SockAddr, HostName) +import Data.Text (pack, unpack) +import qualified Network.Wai.Handler.WarpTLS as TLS +import Network.Wai.Middleware.RequestLogger + +mkYesodDispatch "WebApp" $(parseRoutesFile "Assistant/WebApp/routes") + +type Url = String + +webAppThread + :: AssistantData + -> UrlRenderer + -> Bool + -> Maybe String + -> Maybe (IO Url) + -> Maybe HostName + -> Maybe (Url -> FilePath -> IO ()) + -> NamedThread +webAppThread assistantdata urlrenderer noannex cannotrun postfirstrun listenhost onstartup = thread $ liftIO $ do + listenhost' <- if isJust listenhost + then pure listenhost + else getAnnex $ annexListen <$> Annex.getGitConfig + tlssettings <- getAnnex getTlsSettings +#ifdef __ANDROID__ + when (isJust listenhost') $ + -- See Utility.WebApp + giveup "Sorry, --listen is not currently supported on Android" +#endif + webapp <- WebApp + <$> pure assistantdata + <*> genAuthToken 128 + <*> getreldir + <*> pure staticRoutes + <*> pure postfirstrun + <*> pure cannotrun + <*> pure noannex + <*> pure listenhost' + <*> newWormholePairingState + setUrlRenderer urlrenderer $ yesodRender webapp (pack "") + app <- toWaiAppPlain webapp + app' <- ifM debugEnabled + ( return $ logStdout app + , return app + ) + runWebApp tlssettings listenhost' app' $ \addr -> if noannex + then withTmpFile "webapp.html" $ \tmpfile h -> do + hClose h + go tlssettings addr webapp tmpfile Nothing + else do + htmlshim <- getAnnex' $ fromRepo gitAnnexHtmlShim + urlfile <- getAnnex' $ fromRepo gitAnnexUrlFile + go tlssettings addr webapp htmlshim (Just urlfile) + where + -- The webapp thread does not wait for the startupSanityCheckThread + -- to finish, so that the user interface remains responsive while + -- that's going on. + thread = namedThreadUnchecked "WebApp" + getreldir + | noannex = return Nothing + | otherwise = Just <$> + (relHome =<< absPath + =<< getAnnex' (fromRepo repoPath)) + go tlssettings addr webapp htmlshim urlfile = do + let url = myUrl tlssettings webapp addr + maybe noop (`writeFileProtected` url) urlfile + writeHtmlShim "Starting webapp..." url htmlshim + maybe noop (\a -> a url htmlshim) onstartup + + getAnnex a + | noannex = pure Nothing + | otherwise = getAnnex' a + getAnnex' = runThreadState (threadState assistantdata) + +myUrl :: Maybe TLS.TLSSettings -> WebApp -> SockAddr -> Url +myUrl tlssettings webapp addr = unpack $ yesodRender webapp urlbase DashboardR [] + where + urlbase = pack $ proto ++ "://" ++ show addr + proto + | isJust tlssettings = "https" + | otherwise = "http" + +getTlsSettings :: Annex (Maybe TLS.TLSSettings) +getTlsSettings = do + cert <- fromRepo gitAnnexWebCertificate + privkey <- fromRepo gitAnnexWebPrivKey + ifM (liftIO $ allM doesFileExist [cert, privkey]) + ( return $ Just $ TLS.tlsSettings cert privkey + , return Nothing + ) diff --git a/Assistant/TransferQueue.hs b/Assistant/TransferQueue.hs new file mode 100644 index 0000000000..6a44732622 --- /dev/null +++ b/Assistant/TransferQueue.hs @@ -0,0 +1,233 @@ +{- git-annex assistant pending transfer queue + - + - Copyright 2012-2014 Joey Hess + - + - Licensed under the GNU GPL version 3 or higher. + -} + +{-# LANGUAGE BangPatterns #-} + +module Assistant.TransferQueue ( + TransferQueue, + Schedule(..), + newTransferQueue, + getTransferQueue, + queueTransfers, + queueTransfersMatching, + queueDeferredDownloads, + queueTransfer, + queueTransferAt, + queueTransferWhenSmall, + getNextTransfer, + getMatchingTransfers, + dequeueTransfers, +) where + +import Assistant.Common +import Assistant.DaemonStatus +import Assistant.Types.TransferQueue +import Types.Transfer +import Logs.Transfer +import Types.Remote +import qualified Remote +import qualified Types.Remote as Remote +import Annex.Wanted +import Utility.TList + +import Control.Concurrent.STM +import qualified Data.Map.Strict as M +import qualified Data.Set as S + +type Reason = String + +{- Reads the queue's content without blocking or changing it. -} +getTransferQueue :: Assistant [(Transfer, TransferInfo)] +getTransferQueue = (atomically . readTList . queuelist) <<~ transferQueue + +stubInfo :: AssociatedFile -> Remote -> TransferInfo +stubInfo f r = stubTransferInfo + { transferRemote = Just r + , associatedFile = f + } + +{- Adds transfers to queue for some of the known remotes. + - Honors preferred content settings, only transferring wanted files. -} +queueTransfers :: Reason -> Schedule -> Key -> AssociatedFile -> Direction -> Assistant Bool +queueTransfers = queueTransfersMatching (const True) + +{- Adds transfers to queue for some of the known remotes, that match a + - condition. Honors preferred content settings. -} +queueTransfersMatching :: (UUID -> Bool) -> Reason -> Schedule -> Key -> AssociatedFile -> Direction -> Assistant Bool +queueTransfersMatching matching reason schedule k f direction + | direction == Download = ifM (liftAnnex $ wantGet True (Just k) f) + ( go + , return False + ) + | otherwise = go + where + go = do + rs <- liftAnnex . selectremotes =<< getDaemonStatus + let matchingrs = filter (matching . Remote.uuid) rs + if null matchingrs + then do + defer + return False + else do + forM_ matchingrs $ \r -> + enqueue reason schedule (gentransfer r) (stubInfo f r) + return True + selectremotes st + {- Queue downloads from all remotes that + - have the key. The list of remotes is ordered with + - cheapest first. More expensive ones will only be tried + - if downloading from a cheap one fails. -} + | direction == Download = do + s <- locs + return $ filter (inset s) (downloadRemotes st) + {- Upload to all remotes that want the content and don't + - already have it. -} + | otherwise = do + s <- locs + filterM (wantSend True (Just k) f . Remote.uuid) $ + filter (\r -> not (inset s r || Remote.readonly r)) + (syncDataRemotes st) + where + locs = S.fromList . map Remote.uuid <$> Remote.keyPossibilities k + inset s r = S.member (Remote.uuid r) s + gentransfer r = Transfer + { transferDirection = direction + , transferKey = k + , transferUUID = Remote.uuid r + } + defer + {- Defer this download, as no known remote has the key. -} + | direction == Download = do + q <- getAssistant transferQueue + void $ liftIO $ atomically $ + consTList (deferreddownloads q) (k, f) + | otherwise = noop + +{- Queues any deferred downloads that can now be accomplished, leaving + - any others in the list to try again later. -} +queueDeferredDownloads :: Reason -> Schedule -> Assistant () +queueDeferredDownloads reason schedule = do + q <- getAssistant transferQueue + l <- liftIO $ atomically $ readTList (deferreddownloads q) + rs <- downloadRemotes <$> getDaemonStatus + left <- filterM (queue rs) l + unless (null left) $ + liftIO $ atomically $ appendTList (deferreddownloads q) left + where + queue rs (k, f) = do + uuids <- liftAnnex $ Remote.keyLocations k + let sources = filter (\r -> uuid r `elem` uuids) rs + unless (null sources) $ + forM_ sources $ \r -> + enqueue reason schedule + (gentransfer r) (stubInfo f r) + return $ null sources + where + gentransfer r = Transfer + { transferDirection = Download + , transferKey = k + , transferUUID = Remote.uuid r + } + +enqueue :: Reason -> Schedule -> Transfer -> TransferInfo -> Assistant () +enqueue reason schedule t info + | schedule == Next = go consTList + | otherwise = go snocTList + where + go modlist = whenM (add modlist) $ do + debug [ "queued", describeTransfer t info, ": " ++ reason ] + notifyTransfer + add modlist = do + q <- getAssistant transferQueue + dstatus <- getAssistant daemonStatusHandle + liftIO $ atomically $ ifM (checkRunningTransferSTM dstatus t) + ( return False + , do + l <- readTList (queuelist q) + if (t `notElem` map fst l) + then do + void $ modifyTVar' (queuesize q) succ + void $ modlist (queuelist q) (t, info) + return True + else return False + ) + +{- Adds a transfer to the queue. -} +queueTransfer :: Reason -> Schedule -> AssociatedFile -> Transfer -> Remote -> Assistant () +queueTransfer reason schedule f t remote = + enqueue reason schedule t (stubInfo f remote) + +{- Blocks until the queue is no larger than a given size, and then adds a + - transfer to the queue. -} +queueTransferAt :: Int -> Reason -> Schedule -> AssociatedFile -> Transfer -> Remote -> Assistant () +queueTransferAt wantsz reason schedule f t remote = do + q <- getAssistant transferQueue + liftIO $ atomically $ do + sz <- readTVar (queuesize q) + unless (sz <= wantsz) $ + retry -- blocks until queuesize changes + enqueue reason schedule t (stubInfo f remote) + +queueTransferWhenSmall :: Reason -> AssociatedFile -> Transfer -> Remote -> Assistant () +queueTransferWhenSmall reason = queueTransferAt 10 reason Later + +{- Blocks until a pending transfer is available in the queue, + - and removes it. + - + - Checks that it's acceptable, before adding it to the + - currentTransfers map. If it's not acceptable, it's discarded. + - + - This is done in a single STM transaction, so there is no window + - where an observer sees an inconsistent status. -} +getNextTransfer :: (TransferInfo -> Bool) -> Assistant (Maybe (Transfer, TransferInfo)) +getNextTransfer acceptable = do + q <- getAssistant transferQueue + dstatus <- getAssistant daemonStatusHandle + liftIO $ atomically $ do + sz <- readTVar (queuesize q) + if sz < 1 + then retry -- blocks until queuesize changes + else do + (r@(t,info):rest) <- readTList (queuelist q) + void $ modifyTVar' (queuesize q) pred + setTList (queuelist q) rest + if acceptable info + then do + adjustTransfersSTM dstatus $ + M.insert t info + return $ Just r + else return Nothing + +{- Moves transfers matching a condition from the queue, to the + - currentTransfers map. -} +getMatchingTransfers :: (Transfer -> Bool) -> Assistant [(Transfer, TransferInfo)] +getMatchingTransfers c = do + q <- getAssistant transferQueue + dstatus <- getAssistant daemonStatusHandle + liftIO $ atomically $ do + ts <- dequeueTransfersSTM q c + unless (null ts) $ + adjustTransfersSTM dstatus $ \m -> M.union m $ M.fromList ts + return ts + +{- Removes transfers matching a condition from the queue, and returns the + - removed transfers. -} +dequeueTransfers :: (Transfer -> Bool) -> Assistant [(Transfer, TransferInfo)] +dequeueTransfers c = do + q <- getAssistant transferQueue + removed <- liftIO $ atomically $ dequeueTransfersSTM q c + unless (null removed) $ + notifyTransfer + return removed + +dequeueTransfersSTM :: TransferQueue -> (Transfer -> Bool) -> STM [(Transfer, TransferInfo)] +dequeueTransfersSTM q c = do + !(removed, ts) <- partition (c . fst) <$> readTList (queuelist q) + let !len = length ts + void $ writeTVar (queuesize q) len + setTList (queuelist q) ts + return removed diff --git a/Assistant/TransferSlots.hs b/Assistant/TransferSlots.hs new file mode 100644 index 0000000000..3c7a01bec1 --- /dev/null +++ b/Assistant/TransferSlots.hs @@ -0,0 +1,300 @@ +{- git-annex assistant transfer slots + - + - Copyright 2012 Joey Hess + - + - Licensed under the GNU GPL version 3 or higher. + -} + +{-# LANGUAGE CPP #-} + +module Assistant.TransferSlots where + +import Assistant.Common +import Utility.ThreadScheduler +import Assistant.Types.TransferSlots +import Assistant.DaemonStatus +import Assistant.TransferrerPool +import Assistant.Types.TransferrerPool +import Assistant.Types.TransferQueue +import Assistant.TransferQueue +import Assistant.Alert +import Assistant.Alert.Utility +import Assistant.Commits +import Assistant.Drop +import Types.Transfer +import Logs.Transfer +import Logs.Location +import qualified Git +import qualified Remote +import qualified Types.Remote as Remote +import Annex.Content +import Annex.Wanted +import Annex.Path +import Utility.Batch +import Types.NumCopies + +import qualified Data.Map as M +import qualified Control.Exception as E +import Control.Concurrent +import qualified Control.Concurrent.MSemN as MSemN +#ifndef mingw32_HOST_OS +import System.Posix.Process (getProcessGroupIDOf) +import System.Posix.Signals (signalProcessGroup, sigTERM, sigKILL) +#else +import System.Win32.Process (terminateProcessById) +#endif + +type TransferGenerator = Assistant (Maybe (Transfer, TransferInfo, Transferrer -> Assistant ())) + +{- Waits until a transfer slot becomes available, then runs a + - TransferGenerator, and then runs the transfer action in its own thread. + -} +inTransferSlot :: FilePath -> BatchCommandMaker -> TransferGenerator -> Assistant () +inTransferSlot program batchmaker gen = do + flip MSemN.wait 1 <<~ transferSlots + runTransferThread program batchmaker =<< gen + +{- Runs a TransferGenerator, and its transfer action, + - without waiting for a slot to become available. -} +inImmediateTransferSlot :: FilePath -> BatchCommandMaker -> TransferGenerator -> Assistant () +inImmediateTransferSlot program batchmaker gen = do + flip MSemN.signal (-1) <<~ transferSlots + runTransferThread program batchmaker =<< gen + +{- Runs a transfer action, in an already allocated transfer slot. + - Once it finishes, frees the transfer slot. + - + - Note that the action is subject to being killed when the transfer + - is canceled or paused. + - + - A PauseTransfer exception is handled by letting the action be killed, + - then pausing the thread until a ResumeTransfer exception is raised, + - then rerunning the action. + -} +runTransferThread :: FilePath -> BatchCommandMaker -> Maybe (Transfer, TransferInfo, Transferrer -> Assistant ()) -> Assistant () +runTransferThread _ _ Nothing = flip MSemN.signal 1 <<~ transferSlots +runTransferThread program batchmaker (Just (t, info, a)) = do + d <- getAssistant id + aio <- asIO1 a + tid <- liftIO $ forkIO $ runTransferThread' program batchmaker d aio + updateTransferInfo t $ info { transferTid = Just tid } + +runTransferThread' :: FilePath -> BatchCommandMaker -> AssistantData -> (Transferrer -> IO ()) -> IO () +runTransferThread' program batchmaker d run = go + where + go = catchPauseResume $ + withTransferrer program batchmaker (transferrerPool d) + run + pause = catchPauseResume $ + runEvery (Seconds 86400) noop + {- Note: This must use E.try, rather than E.catch. + - When E.catch is used, and has called go in its exception + - handler, Control.Concurrent.throwTo will block sometimes + - when signaling. Using E.try avoids the problem. -} + catchPauseResume a' = do + r <- E.try a' :: IO (Either E.SomeException ()) + case r of + Left e -> case E.fromException e of + Just PauseTransfer -> pause + Just ResumeTransfer -> go + _ -> done + _ -> done + done = runAssistant d $ + flip MSemN.signal 1 <<~ transferSlots + +{- By the time this is called, the daemonstatus's currentTransfers map should + - already have been updated to include the transfer. -} +genTransfer :: Transfer -> TransferInfo -> TransferGenerator +genTransfer t info = case transferRemote info of + Just remote -> ifM (unpluggedremovabledrive remote) + ( do + -- optimisation, since the transfer would fail + liftAnnex $ recordFailedTransfer t info + void $ removeTransfer t + return Nothing + , ifM (liftAnnex $ shouldTransfer t info) + ( do + debug [ "Transferring:" , describeTransfer t info ] + notifyTransfer + return $ Just (t, info, go remote) + , do + debug [ "Skipping unnecessary transfer:", + describeTransfer t info ] + void $ removeTransfer t + finishedTransfer t (Just info) + return Nothing + ) + ) + _ -> return Nothing + where + direction = transferDirection t + isdownload = direction == Download + + unpluggedremovabledrive remote = Git.repoIsLocalUnknown + <$> liftAnnex (Remote.getRepo remote) + + {- Alerts are only shown for successful transfers. + - Transfers can temporarily fail for many reasons, + - so there's no point in bothering the user about + - those. The assistant should recover. + - + - After a successful upload, handle dropping it from + - here, if desired. In this case, the remote it was + - uploaded to is known to have it. + - + - Also, after a successful transfer, the location + - log has changed. Indicate that a commit has been + - made, in order to queue a push of the git-annex + - branch out to remotes that did not participate + - in the transfer. + - + - If the process failed, it could have crashed, + - so remove the transfer from the list of current + - transfers, just in case it didn't stop + - in a way that lets the TransferWatcher do its + - usual cleanup. However, first check if something else is + - running the transfer, to avoid removing active transfers. + -} + go remote transferrer = ifM (liftIO $ performTransfer transferrer t info) + ( do + case associatedFile info of + AssociatedFile Nothing -> noop + AssociatedFile (Just af) -> void $ + addAlert $ makeAlertFiller True $ + transferFileAlert direction True af + unless isdownload $ + handleDrops + ("object uploaded to " ++ show remote) + True (transferKey t) + (associatedFile info) + [mkVerifiedCopy RecentlyVerifiedCopy remote] + void recordCommit + , whenM (liftAnnex $ isNothing <$> checkTransfer t) $ + void $ removeTransfer t + ) + +{- Called right before a transfer begins, this is a last chance to avoid + - unnecessary transfers. + - + - For downloads, we obviously don't need to download if the already + - have the object. + - + - Smilarly, for uploads, check if the remote is known to already have + - the object. + - + - Also, uploads get queued to all remotes, in order of cost. + - This may mean, for example, that an object is uploaded over the LAN + - to a locally paired client, and once that upload is done, a more + - expensive transfer remote no longer wants the object. (Since + - all the clients have it already.) So do one last check if this is still + - preferred content. + - + - We'll also do one last preferred content check for downloads. An + - example of a case where this could be needed is if a download is queued + - for a file that gets moved out of an archive directory -- but before + - that download can happen, the file is put back in the archive. + -} +shouldTransfer :: Transfer -> TransferInfo -> Annex Bool +shouldTransfer t info + | transferDirection t == Download = + (not <$> inAnnex key) <&&> wantGet True (Just key) file + | transferDirection t == Upload = case transferRemote info of + Nothing -> return False + Just r -> notinremote r + <&&> wantSend True (Just key) file (Remote.uuid r) + | otherwise = return False + where + key = transferKey t + file = associatedFile info + + {- Trust the location log to check if the remote already has + - the key. This avoids a roundtrip to the remote. -} + notinremote r = notElem (Remote.uuid r) <$> loggedLocations key + +{- Queue uploads of files downloaded to us, spreading them + - out to other reachable remotes. + - + - Downloading a file may have caused a remote to not want it; + - so check for drops from remotes. + - + - Uploading a file may cause the local repo, or some other remote to not + - want it; handle that too. + -} +finishedTransfer :: Transfer -> Maybe TransferInfo -> Assistant () +finishedTransfer t (Just info) + | transferDirection t == Download = + whenM (liftAnnex $ inAnnex $ transferKey t) $ do + dodrops False + void $ queueTransfersMatching (/= transferUUID t) + "newly received object" + Later (transferKey t) (associatedFile info) Upload + | otherwise = dodrops True + where + dodrops fromhere = handleDrops + ("drop wanted after " ++ describeTransfer t info) + fromhere (transferKey t) (associatedFile info) [] +finishedTransfer _ _ = noop + +{- Pause a running transfer. -} +pauseTransfer :: Transfer -> Assistant () +pauseTransfer = cancelTransfer True + +{- Cancel a running transfer. -} +cancelTransfer :: Bool -> Transfer -> Assistant () +cancelTransfer pause t = do + m <- getCurrentTransfers + unless pause $ + {- remove queued transfer -} + void $ dequeueTransfers $ equivilantTransfer t + {- stop running transfer -} + maybe noop stop (M.lookup t m) + where + stop info = do + {- When there's a thread associated with the + - transfer, it's signaled first, to avoid it + - displaying any alert about the transfer having + - failed when the transfer process is killed. -} + liftIO $ maybe noop signalthread $ transferTid info + liftIO $ maybe noop killproc $ transferPid info + if pause + then void $ alterTransferInfo t $ + \i -> i { transferPaused = True } + else void $ removeTransfer t + signalthread tid + | pause = throwTo tid PauseTransfer + | otherwise = killThread tid + killproc pid = void $ tryIO $ do +#ifndef mingw32_HOST_OS + {- In order to stop helper processes like rsync, + - kill the whole process group of the process + - running the transfer. -} + g <- getProcessGroupIDOf pid + let signal sig = void $ tryIO $ signalProcessGroup sig g + signal sigTERM + threadDelay 50000 -- 0.05 second grace period + signal sigKILL +#else + terminateProcessById pid +#endif + +{- Start or resume a transfer. -} +startTransfer :: Transfer -> Assistant () +startTransfer t = do + m <- getCurrentTransfers + maybe startqueued go (M.lookup t m) + where + go info = maybe (start info) resume $ transferTid info + startqueued = do + is <- map snd <$> getMatchingTransfers (== t) + maybe noop start $ headMaybe is + resume tid = do + alterTransferInfo t $ \i -> i { transferPaused = False } + liftIO $ throwTo tid ResumeTransfer + start info = do + program <- liftIO programPath + batchmaker <- liftIO getBatchCommandMaker + inImmediateTransferSlot program batchmaker $ + genTransfer t info + +getCurrentTransfers :: Assistant TransferMap +getCurrentTransfers = currentTransfers <$> getDaemonStatus diff --git a/Assistant/TransferrerPool.hs b/Assistant/TransferrerPool.hs new file mode 100644 index 0000000000..892e156e8b --- /dev/null +++ b/Assistant/TransferrerPool.hs @@ -0,0 +1,94 @@ +{- A pool of "git-annex transferkeys" processes + - + - Copyright 2013 Joey Hess + - + - Licensed under the GNU GPL version 3 or higher. + -} + +module Assistant.TransferrerPool where + +import Assistant.Common +import Assistant.Types.TransferrerPool +import Types.Transfer +import Utility.Batch + +import qualified Command.TransferKeys as T + +import Control.Concurrent.STM hiding (check) +import Control.Exception (throw) +import Control.Concurrent + +{- Runs an action with a Transferrer from the pool. + - + - Only one Transferrer is left running in the pool at a time. + - So if this needed to start a new Transferrer, it's stopped when done. + -} +withTransferrer :: FilePath -> BatchCommandMaker -> TransferrerPool -> (Transferrer -> IO a) -> IO a +withTransferrer program batchmaker pool a = do + (mi, leftinpool) <- atomically (popTransferrerPool pool) + i@(TransferrerPoolItem (Just t) check) <- case mi of + Nothing -> mkTransferrerPoolItem pool =<< mkTransferrer program batchmaker + Just i -> checkTransferrerPoolItem program batchmaker i + v <- tryNonAsync $ a t + if leftinpool == 0 + then atomically $ pushTransferrerPool pool i + else do + void $ forkIO $ stopTransferrer t + atomically $ pushTransferrerPool pool $ TransferrerPoolItem Nothing check + either throw return v + +{- Check if a Transferrer from the pool is still ok to be used. + - If not, stop it and start a new one. -} +checkTransferrerPoolItem :: FilePath -> BatchCommandMaker -> TransferrerPoolItem -> IO TransferrerPoolItem +checkTransferrerPoolItem program batchmaker i = case i of + TransferrerPoolItem (Just t) check -> ifM check + ( return i + , do + stopTransferrer t + new check + ) + TransferrerPoolItem Nothing check -> new check + where + new check = do + t <- mkTransferrer program batchmaker + return $ TransferrerPoolItem (Just t) check + +{- Requests that a Transferrer perform a Transfer, and waits for it to + - finish. -} +performTransfer :: Transferrer -> Transfer -> TransferInfo -> IO Bool +performTransfer transferrer t info = catchBoolIO $ do + T.sendRequest t info (transferrerWrite transferrer) + T.readResponse (transferrerRead transferrer) + +{- Starts a new git-annex transferkeys process, setting up handles + - that will be used to communicate with it. -} +mkTransferrer :: FilePath -> BatchCommandMaker -> IO Transferrer +mkTransferrer program batchmaker = do + {- It runs as a batch job. -} + let (program', params') = batchmaker (program, [Param "transferkeys"]) + {- It's put into its own group so that the whole group can be + - killed to stop a transfer. -} + (Just writeh, Just readh, _, pid) <- createProcess + (proc program' $ toCommand params') + { create_group = True + , std_in = CreatePipe + , std_out = CreatePipe + } + return $ Transferrer + { transferrerRead = readh + , transferrerWrite = writeh + , transferrerHandle = pid + } + +{- Checks if a Transferrer is still running. If not, makes a new one. -} +checkTransferrer :: FilePath -> BatchCommandMaker -> Transferrer -> IO Transferrer +checkTransferrer program batchmaker t = + maybe (return t) (const $ mkTransferrer program batchmaker) + =<< getProcessExitCode (transferrerHandle t) + +{- Closing the fds will stop the transferrer. -} +stopTransferrer :: Transferrer -> IO () +stopTransferrer t = do + hClose $ transferrerRead t + hClose $ transferrerWrite t + void $ waitForProcess $ transferrerHandle t diff --git a/Assistant/Types/Alert.hs b/Assistant/Types/Alert.hs new file mode 100644 index 0000000000..a2e5d5c822 --- /dev/null +++ b/Assistant/Types/Alert.hs @@ -0,0 +1,79 @@ +{- git-annex assistant alert types + - + - Copyright 2013 Joey Hess + - + - Licensed under the GNU GPL version 3 or higher. + -} + +module Assistant.Types.Alert where + +import Utility.Tense + +import Data.Text (Text) +import qualified Data.Map as M + +{- Different classes of alerts are displayed differently. -} +data AlertClass = Success | Message | Activity | Warning | Error + deriving (Eq, Ord) + +data AlertPriority = Filler | Low | Medium | High | Pinned + deriving (Eq, Ord) + +{- An alert can have an name, which is used to combine it with other similar + - alerts. -} +data AlertName + = FileAlert TenseChunk + | SanityCheckFixAlert + | WarningAlert String + | PairAlert String + | ConnectionNeededAlert + | RemoteRemovalAlert String + | CloudRepoNeededAlert + | SyncAlert + | NotFsckedAlert + | UpgradeAlert + | UnusedFilesAlert + deriving (Eq) + +{- The first alert is the new alert, the second is an old alert. + - Should return a modified version of the old alert. -} +type AlertCombiner = Alert -> Alert -> Maybe Alert + +data Alert = Alert + { alertClass :: AlertClass + , alertHeader :: Maybe TenseText + , alertMessageRender :: Alert -> TenseText + , alertData :: [TenseChunk] + , alertCounter :: Int + , alertBlockDisplay :: Bool + , alertClosable :: Bool + , alertPriority :: AlertPriority + , alertIcon :: Maybe AlertIcon + , alertCombiner :: Maybe AlertCombiner + , alertName :: Maybe AlertName + , alertButtons :: [AlertButton] + } + +data AlertIcon = ActivityIcon | SyncIcon | SuccessIcon | ErrorIcon | InfoIcon | UpgradeIcon | ConnectionIcon + +type AlertMap = M.Map AlertId Alert + +{- Higher AlertId indicates a more recent alert. -} +newtype AlertId = AlertId Integer + deriving (Read, Show, Eq, Ord) + +firstAlertId :: AlertId +firstAlertId = AlertId 0 + +nextAlertId :: AlertId -> AlertId +nextAlertId (AlertId i) = AlertId $ succ i + +{- When clicked, a button always redirects to a URL + - It may also run an IO action in the background, which is useful + - to make the button close or otherwise change the alert. -} +data AlertButton = AlertButton + { buttonLabel :: Text + , buttonUrl :: Text + , buttonAction :: Maybe (AlertId -> IO ()) + , buttonPrimary :: Bool + } diff --git a/Assistant/Types/BranchChange.hs b/Assistant/Types/BranchChange.hs new file mode 100644 index 0000000000..4c15f133ae --- /dev/null +++ b/Assistant/Types/BranchChange.hs @@ -0,0 +1,20 @@ +{- git-annex assistant git-annex branch change tracking + - + - Copyright 2012 Joey Hess + - + - Licensed under the GNU GPL version 3 or higher. + -} + +module Assistant.Types.BranchChange where + +import Control.Concurrent.MSampleVar +import Control.Applicative +import Prelude + +newtype BranchChangeHandle = BranchChangeHandle (MSampleVar ()) + +newBranchChangeHandle :: IO BranchChangeHandle +newBranchChangeHandle = BranchChangeHandle <$> newEmptySV + +fromBranchChangeHandle :: BranchChangeHandle -> MSampleVar () +fromBranchChangeHandle (BranchChangeHandle v) = v diff --git a/Assistant/Types/Changes.hs b/Assistant/Types/Changes.hs new file mode 100644 index 0000000000..70c40523a0 --- /dev/null +++ b/Assistant/Types/Changes.hs @@ -0,0 +1,100 @@ +{- git-annex assistant change tracking + - + - Copyright 2012-2015 Joey Hess + - + - Licensed under the GNU GPL version 3 or higher. + -} + +{-# LANGUAGE BangPatterns #-} + +module Assistant.Types.Changes where + +import Types.KeySource +import Types.Key +import Utility.TList +import Annex.Ingest + +import Control.Concurrent.STM +import Data.Time.Clock +import qualified Data.Set as S + +{- An un-ordered pool of Changes that have been noticed and should be + - staged and committed. Changes will typically be in order, but ordering + - may be lost. In any case, order should not matter, as any given Change + - may later be reverted by a later Change (ie, a file is added and then + - deleted). Code that processes the changes needs to deal with such + - scenarios. + -} +type ChangePool = TList Change + +newChangePool :: IO ChangePool +newChangePool = atomically newTList + +data Change + = Change + { changeTime :: UTCTime + , _changeFile :: FilePath + , changeInfo :: ChangeInfo + } + | PendingAddChange + { changeTime ::UTCTime + , _changeFile :: FilePath + } + | InProcessAddChange + { changeTime ::UTCTime + , lockedDown :: LockedDown + } + deriving (Show) + +data ChangeInfo = AddKeyChange Key | AddFileChange | LinkChange (Maybe Key) | RmChange + deriving (Show, Eq, Ord) + +changeInfoKey :: ChangeInfo -> Maybe Key +changeInfoKey (AddKeyChange k) = Just k +changeInfoKey (LinkChange (Just k)) = Just k +changeInfoKey _ = Nothing + +changeFile :: Change -> FilePath +changeFile (Change _ f _) = f +changeFile (PendingAddChange _ f) = f +changeFile (InProcessAddChange _ ld) = keyFilename $ keySource ld + +isPendingAddChange :: Change -> Bool +isPendingAddChange (PendingAddChange {}) = True +isPendingAddChange _ = False + +isInProcessAddChange :: Change -> Bool +isInProcessAddChange (InProcessAddChange {}) = True +isInProcessAddChange _ = False + +retryChange :: Change -> Change +retryChange c@(InProcessAddChange time _) = + PendingAddChange time $ changeFile c +retryChange c = c + +finishedChange :: Change -> Key -> Change +finishedChange c@(InProcessAddChange {}) k = Change + { changeTime = changeTime c + , _changeFile = changeFile c + , changeInfo = AddKeyChange k + } +finishedChange c _ = c + +{- Combine PendingAddChanges that are for the same file. + - Multiple such often get noticed when eg, a file is opened and then + - closed in quick succession. -} +simplifyChanges :: [Change] -> [Change] +simplifyChanges [c] = [c] +simplifyChanges cl = go cl S.empty [] + where + go [] _ l = reverse l + go (c:cs) seen l + | isPendingAddChange c = + if S.member f seen + then go cs seen l + else + let !seen' = S.insert f seen + in go cs seen' (c:l) + | otherwise = go cs seen (c:l) + where + f = changeFile c diff --git a/Assistant/Types/Commits.hs b/Assistant/Types/Commits.hs new file mode 100644 index 0000000000..bf83fc486e --- /dev/null +++ b/Assistant/Types/Commits.hs @@ -0,0 +1,19 @@ +{- git-annex assistant commit tracking + - + - Copyright 2012 Joey Hess + - + - Licensed under the GNU GPL version 3 or higher. + -} + +module Assistant.Types.Commits where + +import Utility.TList + +import Control.Concurrent.STM + +type CommitChan = TList Commit + +data Commit = Commit + +newCommitChan :: IO CommitChan +newCommitChan = atomically newTList diff --git a/Assistant/Types/CredPairCache.hs b/Assistant/Types/CredPairCache.hs new file mode 100644 index 0000000000..9777e29ee0 --- /dev/null +++ b/Assistant/Types/CredPairCache.hs @@ -0,0 +1,18 @@ +{- git-annex assistant CredPair cache. + - + - Copyright 2014 Joey Hess + - + - Licensed under the GNU GPL version 3 or higher. + -} + +module Assistant.Types.CredPairCache where + +import Types.Creds + +import Control.Concurrent +import qualified Data.Map as M + +type CredPairCache = MVar (M.Map Login Password) + +newCredPairCache :: IO CredPairCache +newCredPairCache = newMVar M.empty diff --git a/Assistant/Types/DaemonStatus.hs b/Assistant/Types/DaemonStatus.hs new file mode 100644 index 0000000000..1166cd18ad --- /dev/null +++ b/Assistant/Types/DaemonStatus.hs @@ -0,0 +1,119 @@ +{- git-annex assistant daemon status + - + - Copyright 2012 Joey Hess + - + - Licensed under the GNU GPL version 3 or higher. + -} + +module Assistant.Types.DaemonStatus where + +import Annex.Common +import Assistant.Pairing +import Utility.NotificationBroadcaster +import Types.Transfer +import Assistant.Types.ThreadName +import Assistant.Types.Alert +import Utility.Url + +import Control.Concurrent.STM +import Control.Concurrent.MVar +import Control.Concurrent.Async +import Data.Time.Clock.POSIX +import qualified Data.Map as M +import qualified Data.Set as S + +data DaemonStatus = DaemonStatus + -- All the named threads that comprise the daemon, + -- and actions to run to restart them. + { startedThreads :: M.Map ThreadName (Async (), IO ()) + -- False when the daemon is performing its startup scan + , scanComplete :: Bool + -- True when all files should be restaged. + , forceRestage :: Bool + -- Time when a previous process of the daemon was running ok + , lastRunning :: Maybe POSIXTime + -- True when the daily sanity checker is running + , sanityCheckRunning :: Bool + -- Last time the daily sanity checker ran + , lastSanityCheck :: Maybe POSIXTime + -- True when a scan for file transfers is running + , transferScanRunning :: Bool + -- Currently running file content transfers + , currentTransfers :: TransferMap + -- Messages to display to the user. + , alertMap :: AlertMap + , lastAlertId :: AlertId + -- Ordered list of all remotes that can be synced with + , syncRemotes :: [Remote] + -- Ordered list of remotes to sync git with + , syncGitRemotes :: [Remote] + -- Ordered list of remotes to sync data with + , syncDataRemotes :: [Remote] + -- Ordered list of remotes to export to + , exportRemotes :: [Remote] + -- Ordered list of remotes that data can be downloaded from + , downloadRemotes :: [Remote] + -- Are we syncing to any cloud remotes? + , syncingToCloudRemote :: Bool + -- Set of uuids of remotes that are currently connected. + , currentlyConnectedRemotes :: S.Set UUID + -- Pairing request that is in progress. + , pairingInProgress :: Maybe PairingInProgress + -- Broadcasts notifications about all changes to the DaemonStatus. + , changeNotifier :: NotificationBroadcaster + -- Broadcasts notifications when queued or current transfers change. + , transferNotifier :: NotificationBroadcaster + -- Broadcasts notifications when there's a change to the alerts. + , alertNotifier :: NotificationBroadcaster + -- Broadcasts notifications when the syncRemotes change. + , syncRemotesNotifier :: NotificationBroadcaster + -- Broadcasts notifications when the scheduleLog changes. + , scheduleLogNotifier :: NotificationBroadcaster + -- Broadcasts a notification once the startup sanity check has run. + , startupSanityCheckNotifier :: NotificationBroadcaster + -- Broadcasts notifications when the network is connected. + , networkConnectedNotifier :: NotificationBroadcaster + -- Broadcasts notifications when a global redirect is needed. + , globalRedirNotifier :: NotificationBroadcaster + , globalRedirUrl :: Maybe URLString + -- Actions to run after a Key is transferred. + , transferHook :: M.Map Key (Transfer -> IO ()) + -- MVars to signal when a remote gets connected. + , connectRemoteNotifiers :: M.Map UUID [MVar ()] + } + +type TransferMap = M.Map Transfer TransferInfo + +type DaemonStatusHandle = TVar DaemonStatus + +newDaemonStatus :: IO DaemonStatus +newDaemonStatus = DaemonStatus + <$> pure M.empty + <*> pure False + <*> pure False + <*> pure Nothing + <*> pure False + <*> pure Nothing + <*> pure False + <*> pure M.empty + <*> pure M.empty + <*> pure firstAlertId + <*> pure [] + <*> pure [] + <*> pure [] + <*> pure [] + <*> pure [] + <*> pure False + <*> pure S.empty + <*> pure Nothing + <*> newNotificationBroadcaster + <*> newNotificationBroadcaster + <*> newNotificationBroadcaster + <*> newNotificationBroadcaster + <*> newNotificationBroadcaster + <*> newNotificationBroadcaster + <*> newNotificationBroadcaster + <*> newNotificationBroadcaster + <*> pure Nothing + <*> pure M.empty + <*> pure M.empty diff --git a/Assistant/Types/NamedThread.hs b/Assistant/Types/NamedThread.hs new file mode 100644 index 0000000000..b07b322ad9 --- /dev/null +++ b/Assistant/Types/NamedThread.hs @@ -0,0 +1,21 @@ +{- named threads + - + - Copyright 2012 Joey Hess + - + - Licensed under the GNU GPL version 3 or higher. + -} + +module Assistant.Types.NamedThread where + +import Assistant.Monad +import Assistant.Types.ThreadName + +{- Information about a named thread that can be run. -} +data NamedThread = NamedThread Bool ThreadName (Assistant ()) + +namedThread :: String -> Assistant () -> NamedThread +namedThread = NamedThread True . ThreadName + +{- A named thread that can start running before the startup sanity check. -} +namedThreadUnchecked :: String -> Assistant () -> NamedThread +namedThreadUnchecked = NamedThread False . ThreadName diff --git a/Assistant/Types/Pushes.hs b/Assistant/Types/Pushes.hs new file mode 100644 index 0000000000..ff860a0c11 --- /dev/null +++ b/Assistant/Types/Pushes.hs @@ -0,0 +1,24 @@ +{- git-annex assistant push tracking + - + - Copyright 2012 Joey Hess + - + - Licensed under the GNU GPL version 3 or higher. + -} + +module Assistant.Types.Pushes where + +import Annex.Common + +import Control.Concurrent.STM +import Data.Time.Clock +import qualified Data.Map as M + +{- Track the most recent push failure for each remote. -} +type PushMap = M.Map Remote UTCTime +type FailedPushMap = TMVar PushMap + +{- The TMVar starts empty, and is left empty when there are no + - failed pushes. This way we can block until there are some failed pushes. + -} +newFailedPushMap :: IO FailedPushMap +newFailedPushMap = atomically newEmptyTMVar diff --git a/Assistant/Types/RemoteControl.hs b/Assistant/Types/RemoteControl.hs new file mode 100644 index 0000000000..42cb4a5aa4 --- /dev/null +++ b/Assistant/Types/RemoteControl.hs @@ -0,0 +1,16 @@ +{- git-annex assistant RemoteDaemon control + - + - Copyright 2014 Joey Hess + - + - Licensed under the GNU GPL version 3 or higher. + -} + +module Assistant.Types.RemoteControl where + +import qualified RemoteDaemon.Types as RemoteDaemon +import Control.Concurrent + +type RemoteControl = Chan RemoteDaemon.Consumed + +newRemoteControl :: IO RemoteControl +newRemoteControl = newChan diff --git a/Assistant/Types/RepoProblem.hs b/Assistant/Types/RepoProblem.hs new file mode 100644 index 0000000000..3b9c72cf81 --- /dev/null +++ b/Assistant/Types/RepoProblem.hs @@ -0,0 +1,28 @@ +{- git-annex assistant repository problem tracking + - + - Copyright 2013 Joey Hess + - + - Licensed under the GNU GPL version 3 or higher. + -} + +module Assistant.Types.RepoProblem where + +import Types +import Utility.TList + +import Control.Concurrent.STM +import Data.Function + +data RepoProblem = RepoProblem + { problemUUID :: UUID + , afterFix :: IO () + } + +{- The afterFix actions are assumed to all be equivilant. -} +sameRepoProblem :: RepoProblem -> RepoProblem -> Bool +sameRepoProblem = (==) `on` problemUUID + +type RepoProblemChan = TList RepoProblem + +newRepoProblemChan :: IO RepoProblemChan +newRepoProblemChan = atomically newTList diff --git a/Assistant/Types/ScanRemotes.hs b/Assistant/Types/ScanRemotes.hs new file mode 100644 index 0000000000..84a9ee6592 --- /dev/null +++ b/Assistant/Types/ScanRemotes.hs @@ -0,0 +1,25 @@ +{- git-annex assistant remotes needing scanning + - + - Copyright 2012 Joey Hess + - + - Licensed under the GNU GPL version 3 or higher. + -} + +module Assistant.Types.ScanRemotes where + +import Annex.Common + +import Control.Concurrent.STM +import qualified Data.Map as M + +data ScanInfo = ScanInfo + { scanPriority :: Float + , fullScan :: Bool + } + +type ScanRemoteMap = TMVar (M.Map Remote ScanInfo) + +{- The TMVar starts empty, and is left empty when there are no remotes + - to scan. -} +newScanRemoteMap :: IO ScanRemoteMap +newScanRemoteMap = atomically newEmptyTMVar diff --git a/Assistant/Types/ThreadName.hs b/Assistant/Types/ThreadName.hs new file mode 100644 index 0000000000..57c704dad5 --- /dev/null +++ b/Assistant/Types/ThreadName.hs @@ -0,0 +1,14 @@ +{- name of a thread + - + - Copyright 2012 Joey Hess + - + - Licensed under the GNU GPL version 3 or higher. + -} + +module Assistant.Types.ThreadName where + +newtype ThreadName = ThreadName String + deriving (Eq, Read, Show, Ord) + +fromThreadName :: ThreadName -> String +fromThreadName (ThreadName n) = n diff --git a/Assistant/Types/ThreadedMonad.hs b/Assistant/Types/ThreadedMonad.hs new file mode 100644 index 0000000000..ccb35d023e --- /dev/null +++ b/Assistant/Types/ThreadedMonad.hs @@ -0,0 +1,38 @@ +{- making the Annex monad available across threads + - + - Copyright 2012 Joey Hess + - + - Licensed under the GNU GPL version 3 or higher. + -} + +module Assistant.Types.ThreadedMonad where + +import Annex.Common +import qualified Annex + +import Control.Concurrent +import Data.Tuple + +{- The Annex state is stored in a MVar, so that threaded actions can access + - it. -} +type ThreadState = MVar Annex.AnnexState + +{- Stores the Annex state in a MVar. + - + - Once the action is finished, retrieves the state from the MVar. + -} +withThreadState :: (ThreadState -> Annex a) -> Annex a +withThreadState a = do + state <- Annex.getState id + mvar <- liftIO $ newMVar state + r <- a mvar + newstate <- liftIO $ takeMVar mvar + Annex.changeState (const newstate) + return r + +{- Runs an Annex action, using the state from the MVar. + - + - This serializes calls by threads; only one thread can run in Annex at a + - time. -} +runThreadState :: ThreadState -> Annex a -> IO a +runThreadState mvar a = modifyMVar mvar $ \state -> swap <$> Annex.run state a diff --git a/Assistant/Types/TransferQueue.hs b/Assistant/Types/TransferQueue.hs new file mode 100644 index 0000000000..7e2b4ce3bb --- /dev/null +++ b/Assistant/Types/TransferQueue.hs @@ -0,0 +1,29 @@ +{- git-annex assistant pending transfer queue + - + - Copyright 2012 Joey Hess + - + - Licensed under the GNU GPL version 3 or higher. + -} + +module Assistant.Types.TransferQueue where + +import Annex.Common +import Types.Transfer + +import Control.Concurrent.STM +import Utility.TList + +data TransferQueue = TransferQueue + { queuesize :: TVar Int + , queuelist :: TList (Transfer, TransferInfo) + , deferreddownloads :: TList (Key, AssociatedFile) + } + +data Schedule = Next | Later + deriving (Eq) + +newTransferQueue :: IO TransferQueue +newTransferQueue = atomically $ TransferQueue + <$> newTVar 0 + <*> newTList + <*> newTList diff --git a/Assistant/Types/TransferSlots.hs b/Assistant/Types/TransferSlots.hs new file mode 100644 index 0000000000..5fa1219a7f --- /dev/null +++ b/Assistant/Types/TransferSlots.hs @@ -0,0 +1,34 @@ +{- git-annex assistant transfer slots + - + - Copyright 2012 Joey Hess + - + - Licensed under the GNU GPL version 3 or higher. + -} + +{-# LANGUAGE DeriveDataTypeable #-} + +module Assistant.Types.TransferSlots where + +import qualified Control.Exception as E +import qualified Control.Concurrent.MSemN as MSemN +import Data.Typeable + +type TransferSlots = MSemN.MSemN Int + +{- A special exception that can be thrown to pause or resume a transfer, while + - keeping its slot in use. -} +data TransferException = PauseTransfer | ResumeTransfer + deriving (Show, Eq, Typeable) + +instance E.Exception TransferException + +{- Number of concurrent transfers allowed to be run from the assistant. + - + - Transfers launched by other means, including by remote assistants, + - do not currently take up slots. + -} +numSlots :: Int +numSlots = 1 + +newTransferSlots :: IO TransferSlots +newTransferSlots = MSemN.new numSlots diff --git a/Assistant/Types/TransferrerPool.hs b/Assistant/Types/TransferrerPool.hs new file mode 100644 index 0000000000..742d8437c1 --- /dev/null +++ b/Assistant/Types/TransferrerPool.hs @@ -0,0 +1,64 @@ +{- A pool of "git-annex transferkeys" processes available for use + - + - Copyright 2013 Joey Hess + - + - Licensed under the GNU GPL version 3 or higher. + -} + +module Assistant.Types.TransferrerPool where + +import Annex.Common +import Utility.NotificationBroadcaster +import Assistant.Types.DaemonStatus + +import Control.Concurrent.STM hiding (check) + +type TransferrerPool = TVar (MkCheckTransferrer, [TransferrerPoolItem]) + +type CheckTransferrer = IO Bool +type MkCheckTransferrer = IO (IO Bool) + +{- Each item in the pool may have a transferrer running, and has an + - IO action that can be used to check if it's still ok to use the + - transferrer. -} +data TransferrerPoolItem = TransferrerPoolItem (Maybe Transferrer) CheckTransferrer + +data Transferrer = Transferrer + { transferrerRead :: Handle + , transferrerWrite :: Handle + , transferrerHandle :: ProcessHandle + } + +newTransferrerPool :: MkCheckTransferrer -> IO TransferrerPool +newTransferrerPool c = newTVarIO (c, []) + +popTransferrerPool :: TransferrerPool -> STM (Maybe TransferrerPoolItem, Int) +popTransferrerPool p = do + (c, l) <- readTVar p + case l of + [] -> return (Nothing, 0) + (i:is) -> do + writeTVar p (c, is) + return $ (Just i, length is) + +pushTransferrerPool :: TransferrerPool -> TransferrerPoolItem -> STM () +pushTransferrerPool p i = do + (c, l) <- readTVar p + let l' = i:l + writeTVar p (c, l') + +{- Note that making a CheckTransferrer may allocate resources, + - such as a NotificationHandle, so it's important that the returned + - TransferrerPoolItem is pushed into the pool, and not left to be + - garbage collected. -} +mkTransferrerPoolItem :: TransferrerPool -> Transferrer -> IO TransferrerPoolItem +mkTransferrerPoolItem p t = do + mkcheck <- atomically $ fst <$> readTVar p + check <- mkcheck + return $ TransferrerPoolItem (Just t) check + +checkNetworkConnections :: DaemonStatusHandle -> MkCheckTransferrer +checkNetworkConnections dstatushandle = do + dstatus <- atomically $ readTVar dstatushandle + h <- newNotificationHandle False (networkConnectedNotifier dstatus) + return $ not <$> checkNotification h diff --git a/Assistant/Types/UrlRenderer.hs b/Assistant/Types/UrlRenderer.hs new file mode 100644 index 0000000000..68c238d6a1 --- /dev/null +++ b/Assistant/Types/UrlRenderer.hs @@ -0,0 +1,26 @@ +{- webapp url renderer access from the assistant + - + - Copyright 2013 Joey Hess + - + - Licensed under the GNU GPL version 3 or higher. + -} + +{-# LANGUAGE CPP #-} + +module Assistant.Types.UrlRenderer ( + UrlRenderer, + newUrlRenderer +) where + +#ifdef WITH_WEBAPP + +import Assistant.WebApp (UrlRenderer, newUrlRenderer) + +#else + +data UrlRenderer = UrlRenderer -- dummy type + +newUrlRenderer :: IO UrlRenderer +newUrlRenderer = return UrlRenderer + +#endif diff --git a/Assistant/Unused.hs b/Assistant/Unused.hs new file mode 100644 index 0000000000..26d59cf696 --- /dev/null +++ b/Assistant/Unused.hs @@ -0,0 +1,85 @@ +{- git-annex assistant unused files + - + - Copyright 2014 Joey Hess + - + - Licensed under the GNU GPL version 3 or higher. + -} + +{-# LANGUAGE OverloadedStrings #-} + +module Assistant.Unused where + +import qualified Data.Map as M + +import Assistant.Common +import qualified Git +import Logs.Unused +import Logs.Location +import Annex.Content +import Utility.DataUnits +import Utility.DiskFree +import Utility.HumanTime +import Utility.Tense + +import Data.Time.Clock.POSIX +import qualified Data.Text as T + +describeUnused :: Assistant (Maybe TenseText) +describeUnused = describeUnused' False + +describeUnusedWhenBig :: Assistant (Maybe TenseText) +describeUnusedWhenBig = describeUnused' True + +{- This uses heuristics: 1000 unused keys, or more unused keys + - than the remaining free disk space, or more than 1/10th the total + - disk space being unused keys all suggest a problem. -} +describeUnused' :: Bool -> Assistant (Maybe TenseText) +describeUnused' whenbig = liftAnnex $ go =<< readUnusedLog "" + where + go m = do + let num = M.size m + let diskused = foldl' sumkeysize 0 (M.keys m) + df <- forpath getDiskFree + disksize <- forpath getDiskSize + return $ if num == 0 + then Nothing + else if not whenbig || moreused df diskused || tenthused disksize diskused + then Just $ tenseWords + [ UnTensed $ T.pack $ roughSize storageUnits False diskused + , Tensed "are" "were" + , "taken up by unused files" + ] + else if num > 1000 + then Just $ tenseWords + [ UnTensed $ T.pack $ show num ++ " unused files" + , Tensed "exist" "existed" + ] + else Nothing + + moreused Nothing _ = False + moreused (Just df) used = df <= used + + tenthused Nothing _ = False + tenthused (Just disksize) used = used >= disksize `div` 10 + + sumkeysize s k = s + fromMaybe 0 (keySize k) + + forpath a = inRepo $ liftIO . a . Git.repoPath + +{- With a duration, expires all unused files that are older. + - With Nothing, expires *all* unused files. -} +expireUnused :: Maybe Duration -> Assistant () +expireUnused duration = do + m <- liftAnnex $ readUnusedLog "" + now <- liftIO getPOSIXTime + let oldkeys = M.keys $ M.filter (tooold now) m + forM_ oldkeys $ \k -> do + debug ["removing old unused key", key2file k] + liftAnnex $ do + lockContentForRemoval k removeAnnex + logStatus k InfoMissing + where + boundry = durationToPOSIXTime <$> duration + tooold now (_, mt) = case boundry of + Nothing -> True + Just b -> maybe False (\t -> now - t >= b) mt diff --git a/Assistant/Upgrade.hs b/Assistant/Upgrade.hs new file mode 100644 index 0000000000..c63897150b --- /dev/null +++ b/Assistant/Upgrade.hs @@ -0,0 +1,363 @@ +{- git-annex assistant upgrading + - + - Copyright 2013 Joey Hess + - + - Licensed under the GNU GPL version 3 or higher. + -} + +{-# LANGUAGE CPP #-} + +module Assistant.Upgrade where + +import Assistant.Common +import Assistant.Restart +import qualified Annex +import Assistant.Alert +import Assistant.DaemonStatus +import Utility.Env +import Utility.Env.Set +import Types.Distribution +import Types.Transfer +import Logs.Web +import Logs.Presence +import Logs.Location +import Annex.Content +import Annex.UUID +import qualified Backend +import qualified Types.Backend +import qualified Types.Key +import Assistant.TransferQueue +import Assistant.TransferSlots +import Remote (remoteFromUUID) +import Annex.Path +import Config.Files +import Utility.ThreadScheduler +import Utility.Tmp.Dir +import Utility.UserInfo +import Utility.Gpg +import Utility.FileMode +import Utility.Metered +import qualified Utility.Lsof as Lsof +import qualified BuildInfo +import qualified Utility.Url as Url +import qualified Annex.Url as Url +import Utility.Tuple + +import qualified Data.Map as M + +{- Upgrade without interaction in the webapp. -} +unattendedUpgrade :: Assistant () +unattendedUpgrade = do + prepUpgrade + url <- runRestart + postUpgrade url + +prepUpgrade :: Assistant () +prepUpgrade = do + void $ addAlert upgradingAlert + liftIO $ setEnv upgradedEnv "1" True + prepRestart + +postUpgrade :: URLString -> Assistant () +postUpgrade = postRestart + +autoUpgradeEnabled :: Assistant Bool +autoUpgradeEnabled = liftAnnex $ (==) AutoUpgrade . annexAutoUpgrade <$> Annex.getGitConfig + +checkSuccessfulUpgrade :: IO Bool +checkSuccessfulUpgrade = isJust <$> getEnv upgradedEnv + +upgradedEnv :: String +upgradedEnv = "GIT_ANNEX_UPGRADED" + +{- Start downloading the distribution key from the web. + - Install a hook that will be run once the download is complete, + - and finishes the upgrade. + - + - Creates the destination directory where the upgrade will be installed + - early, in order to check if another upgrade has happened (or is + - happending). On failure, the directory is removed. + -} +startDistributionDownload :: GitAnnexDistribution -> Assistant () +startDistributionDownload d = go =<< liftIO . newVersionLocation d =<< liftIO oldVersionLocation + where + go Nothing = debug ["Skipping redundant upgrade"] + go (Just dest) = do + liftAnnex $ setUrlPresent webUUID k u + hook <- asIO1 $ distributionDownloadComplete d dest cleanup + modifyDaemonStatus_ $ \s -> s + { transferHook = M.insert k hook (transferHook s) } + maybe noop (queueTransfer "upgrade" Next (AssociatedFile (Just f)) t) + =<< liftAnnex (remoteFromUUID webUUID) + startTransfer t + k = distributionKey d + u = distributionUrl d + f = takeFileName u ++ " (for upgrade)" + t = Transfer + { transferDirection = Download + , transferUUID = webUUID + , transferKey = k + } + cleanup = liftAnnex $ do + lockContentForRemoval k removeAnnex + setUrlMissing webUUID k u + logStatus k InfoMissing + +{- Called once the download is done. + - Passed an action that can be used to clean up the downloaded file. + - + - Verifies the content of the downloaded key. + -} +distributionDownloadComplete :: GitAnnexDistribution -> FilePath -> Assistant () -> Transfer -> Assistant () +distributionDownloadComplete d dest cleanup t + | transferDirection t == Download = do + debug ["finished downloading git-annex distribution"] + maybe (failedupgrade "bad download") go + =<< liftAnnex (withObjectLoc k fsckit (getM fsckit)) + | otherwise = cleanup + where + k = distributionKey d + fsckit f = case Backend.maybeLookupBackendVariety (Types.Key.keyVariety k) of + Nothing -> return $ Just f + Just b -> case Types.Backend.verifyKeyContent b of + Nothing -> return $ Just f + Just verifier -> ifM (verifier k f) + ( return $ Just f + , return Nothing + ) + go f = do + ua <- asIO $ upgradeToDistribution dest cleanup f + fa <- asIO1 failedupgrade + liftIO $ ua `catchNonAsync` (fa . show) + failedupgrade msg = do + void $ addAlert $ upgradeFailedAlert msg + cleanup + liftIO $ void $ tryIO $ removeDirectoryRecursive dest + +{- The upgrade method varies by OS. + - + - In general, find where the distribution was installed before, + - and unpack the new distribution next to it (in a versioned directory). + - Then update the programFile to point to the new version. + -} +upgradeToDistribution :: FilePath -> Assistant () -> FilePath -> Assistant () +upgradeToDistribution newdir cleanup distributionfile = do + liftIO $ createDirectoryIfMissing True newdir + (program, deleteold) <- unpack + changeprogram program + cleanup + prepUpgrade + url <- runRestart + {- At this point, the new assistant is fully running, so + - it's safe to delete the old version. -} + liftIO $ void $ tryIO deleteold + postUpgrade url + where + changeprogram program = liftIO $ do + unlessM (boolSystem program [Param "version"]) $ + giveup "New git-annex program failed to run! Not using." + pf <- programFile + liftIO $ writeFile pf program + +#ifdef darwin_HOST_OS + {- OS X uses a dmg, so mount it, and copy the contents into place. -} + unpack = liftIO $ do + olddir <- oldVersionLocation + withTmpDirIn (parentDir newdir) "git-annex.upgrade" $ \tmpdir -> do + void $ boolSystem "hdiutil" + [ Param "attach", File distributionfile + , Param "-mountpoint", File tmpdir + ] + void $ boolSystem "cp" + [ Param "-R" + , File $ tmpdir installBase "Contents" + , File $ newdir + ] + void $ boolSystem "hdiutil" + [ Param "eject" + , File tmpdir + ] + sanitycheck newdir + let deleteold = do + deleteFromManifest $ olddir "Contents" "MacOS" + makeorigsymlink olddir + return (newdir "Contents" "MacOS" "git-annex", deleteold) +#else + {- Linux uses a tarball (so could other POSIX systems), so + - untar it (into a temp directory) and move the directory + - into place. -} + unpack = liftIO $ do + olddir <- oldVersionLocation + withTmpDirIn (parentDir newdir) "git-annex.upgrade" $ \tmpdir -> do + let tarball = tmpdir "tar" + -- Cannot rely on filename extension, and this also + -- avoids problems if tar doesn't support transparent + -- decompression. + void $ boolSystem "sh" + [ Param "-c" + , Param $ "zcat < " ++ shellEscape distributionfile ++ + " > " ++ shellEscape tarball + ] + tarok <- boolSystem "tar" + [ Param "xf" + , Param tarball + , Param "--directory", File tmpdir + ] + unless tarok $ + error $ "failed to untar " ++ distributionfile + sanitycheck $ tmpdir installBase + installby rename newdir (tmpdir installBase) + let deleteold = do + deleteFromManifest olddir + makeorigsymlink olddir + return (newdir "git-annex", deleteold) + installby a dstdir srcdir = + mapM_ (\x -> a x (dstdir takeFileName x)) + =<< dirContents srcdir +#endif + sanitycheck dir = + unlessM (doesDirectoryExist dir) $ + error $ "did not find " ++ dir ++ " in " ++ distributionfile + makeorigsymlink olddir = do + let origdir = parentDir olddir installBase + nukeFile origdir + createSymbolicLink newdir origdir + +{- Finds where the old version was installed. -} +oldVersionLocation :: IO FilePath +oldVersionLocation = do + pdir <- parentDir <$> readProgramFile +#ifdef darwin_HOST_OS + let dirs = splitDirectories pdir + {- It will probably be deep inside a git-annex.app directory. -} + let olddir = case findIndex ("git-annex.app" `isPrefixOf`) dirs of + Nothing -> pdir + Just i -> joinPath (take (i + 1) dirs) +#else + let olddir = pdir +#endif + when (null olddir) $ + error $ "Cannot find old distribution bundle; not upgrading. (Looked in " ++ pdir ++ ")" + return olddir + +{- Finds a place to install the new version. + - Generally, put it in the parent directory of where the old version was + - installed, and use a version number in the directory name. + - If unable to write to there, instead put it in the home directory. + - + - The directory is created. If it already exists, returns Nothing. + -} +newVersionLocation :: GitAnnexDistribution -> FilePath -> IO (Maybe FilePath) +newVersionLocation d olddir = + trymkdir newloc $ do + home <- myHomeDir + trymkdir (home s) $ + return Nothing + where + s = installBase ++ "." ++ distributionVersion d + topdir = parentDir olddir + newloc = topdir s + trymkdir dir fallback = + (createDirectory dir >> return (Just dir)) + `catchIO` const fallback + +installBase :: String +installBase = "git-annex." ++ +#ifdef linux_HOST_OS + "linux" +#else +#ifdef darwin_HOST_OS + "app" +#else + "dir" +#endif +#endif + +deleteFromManifest :: FilePath -> IO () +deleteFromManifest dir = do + fs <- map (dir ) . lines <$> catchDefaultIO "" (readFile manifest) + mapM_ nukeFile fs + nukeFile manifest + removeEmptyRecursive dir + where + manifest = dir "git-annex.MANIFEST" + +removeEmptyRecursive :: FilePath -> IO () +removeEmptyRecursive dir = do + mapM_ removeEmptyRecursive =<< dirContents dir + void $ tryIO $ removeDirectory dir + +{- This is a file that the UpgradeWatcher can watch for modifications to + - detect when git-annex has been upgraded. + -} +upgradeFlagFile :: IO FilePath +upgradeFlagFile = programPath + +{- Sanity check to see if an upgrade is complete and the program is ready + - to be run. -} +upgradeSanityCheck :: IO Bool +upgradeSanityCheck = ifM usingDistribution + ( doesFileExist =<< programFile + , do + -- Ensure that the program is present, and has no writers, + -- and can be run. This should handle distribution + -- upgrades, manual upgrades, etc. + program <- programPath + untilM (doesFileExist program <&&> nowriter program) $ + threadDelaySeconds (Seconds 60) + boolSystem program [Param "version"] + ) + where + nowriter f = null + . filter (`elem` [Lsof.OpenReadWrite, Lsof.OpenWriteOnly]) + . map snd3 + <$> Lsof.query [f] + +usingDistribution :: IO Bool +usingDistribution = isJust <$> getEnv "GIT_ANNEX_STANDLONE_ENV" + +downloadDistributionInfo :: Assistant (Maybe GitAnnexDistribution) +downloadDistributionInfo = do + uo <- liftAnnex Url.getUrlOptions + gpgcmd <- liftAnnex $ gpgCmd <$> Annex.getGitConfig + liftIO $ withTmpDir "git-annex.tmp" $ \tmpdir -> do + let infof = tmpdir "info" + let sigf = infof ++ ".sig" + ifM (Url.download nullMeterUpdate distributionInfoUrl infof uo + <&&> Url.download nullMeterUpdate distributionInfoSigUrl sigf uo + <&&> verifyDistributionSig gpgcmd sigf) + ( parseInfoFile <$> readFileStrict infof + , return Nothing + ) + +distributionInfoUrl :: String +distributionInfoUrl = fromJust BuildInfo.upgradelocation ++ ".info" + +distributionInfoSigUrl :: String +distributionInfoSigUrl = distributionInfoUrl ++ ".sig" + +{- Verifies that a file from the git-annex distribution has a valid + - signature. Pass the detached .sig file; the file to be verified should + - be located next to it. + - + - The gpg keyring used to verify the signature is located in + - trustedkeys.gpg, next to the git-annex program. + -} +verifyDistributionSig :: GpgCmd -> FilePath -> IO Bool +verifyDistributionSig gpgcmd sig = do + p <- readProgramFile + if isAbsolute p + then withUmask 0o0077 $ withTmpDir "git-annex-gpg.tmp" $ \gpgtmp -> do + let trustedkeys = takeDirectory p "trustedkeys.gpg" + boolGpgCmd gpgcmd + [ Param "--no-default-keyring" + , Param "--no-auto-check-trustdb" + , Param "--no-options" + , Param "--homedir" + , File gpgtmp + , Param "--keyring" + , File trustedkeys + , Param "--verify" + , File sig + ] + else return False diff --git a/Assistant/WebApp.hs b/Assistant/WebApp.hs new file mode 100644 index 0000000000..3d0b632717 --- /dev/null +++ b/Assistant/WebApp.hs @@ -0,0 +1,74 @@ +{- git-annex assistant webapp core + - + - Copyright 2012, 2013 Joey Hess + - + - Licensed under the GNU AGPL version 3 or higher. + -} + +{-# LANGUAGE TypeFamilies, QuasiQuotes, MultiParamTypeClasses #-} +{-# LANGUAGE TemplateHaskell, OverloadedStrings, RankNTypes #-} + +module Assistant.WebApp where + +import Assistant.WebApp.Types +import Assistant.Common +import Utility.NotificationBroadcaster +import Utility.Yesod +import Utility.AuthToken + +import Data.Text (Text) +import Control.Concurrent +import qualified Network.Wai as W +import qualified Data.ByteString.Char8 as S8 +import qualified Data.Text as T + +waitNotifier :: Assistant NotificationBroadcaster -> NotificationId -> Handler () +waitNotifier getbroadcaster nid = liftAssistant $ do + b <- getbroadcaster + liftIO $ waitNotification $ notificationHandleFromId b nid + +newNotifier :: Assistant NotificationBroadcaster -> Handler NotificationId +newNotifier getbroadcaster = liftAssistant $ do + b <- getbroadcaster + liftIO $ notificationHandleToId <$> newNotificationHandle True b + +{- Adds the auth parameter as a hidden field on a form. Must be put into + - every form. -} +webAppFormAuthToken :: Widget +webAppFormAuthToken = do + webapp <- liftH getYesod + [whamlet||] + +{- A button with an icon, and maybe label or tooltip, that can be + - clicked to perform some action. + - With javascript, clicking it POSTs the Route, and remains on the same + - page. + - With noscript, clicking it GETs the Route. -} +actionButton :: Route WebApp -> (Maybe String) -> (Maybe String) -> String -> String -> Widget +actionButton route label tooltip buttonclass iconclass = $(widgetFile "actionbutton") + +type UrlRenderFunc = Route WebApp -> [(Text, Text)] -> Text +type UrlRenderer = MVar (UrlRenderFunc) + +newUrlRenderer :: IO UrlRenderer +newUrlRenderer = newEmptyMVar + +setUrlRenderer :: UrlRenderer -> (UrlRenderFunc) -> IO () +setUrlRenderer = putMVar + +inFirstRun :: Handler Bool +inFirstRun = isNothing . relDir <$> getYesod + +{- Blocks until the webapp is running and has called setUrlRenderer. -} +renderUrl :: UrlRenderer -> Route WebApp -> [(Text, Text)] -> IO Text +renderUrl urlrenderer route params = do + r <- readMVar urlrenderer + return $ r route params + +{- Redirects back to the referring page, or if there's none, DashboardR -} +redirectBack :: Handler () +redirectBack = do + mr <- lookup "referer" . W.requestHeaders <$> waiRequest + case mr of + Nothing -> redirect DashboardR + Just r -> redirect $ T.pack $ S8.unpack r diff --git a/Assistant/WebApp/Common.hs b/Assistant/WebApp/Common.hs new file mode 100644 index 0000000000..a148dcabdb --- /dev/null +++ b/Assistant/WebApp/Common.hs @@ -0,0 +1,17 @@ +{- git-annex assistant webapp, common imports + - + - Copyright 2012 Joey Hess + - + - Licensed under the GNU AGPL version 3 or higher. + -} + +module Assistant.WebApp.Common (module X) where + +import Assistant.Common as X +import Assistant.WebApp as X +import Assistant.WebApp.Page as X +import Assistant.WebApp.Form as X +import Assistant.WebApp.Types as X +import Assistant.WebApp.RepoId as X +import Utility.Yesod as X hiding (textField, passwordField, insertBy, replace, joinPath, deleteBy, delete, insert, Key, Option, PermissionDenied) +import Data.Text as X (Text) diff --git a/Assistant/WebApp/Configurators.hs b/Assistant/WebApp/Configurators.hs new file mode 100644 index 0000000000..0042638b15 --- /dev/null +++ b/Assistant/WebApp/Configurators.hs @@ -0,0 +1,44 @@ +{- git-annex assistant webapp configurators + - + - Copyright 2012 Joey Hess + - + - Licensed under the GNU AGPL version 3 or higher. + -} + +{-# LANGUAGE QuasiQuotes, TemplateHaskell, OverloadedStrings #-} + +module Assistant.WebApp.Configurators where + +import Assistant.WebApp.Common +import Assistant.WebApp.RepoList + +{- The main configuration screen. -} +getConfigurationR :: Handler Html +getConfigurationR = ifM inFirstRun + ( redirect FirstRepositoryR + , page "Configuration" (Just Configuration) $ do + $(widgetFile "configurators/main") + ) + +getAddRepositoryR :: Handler Html +getAddRepositoryR = page "Add Repository" (Just Configuration) $ do + let repolist = repoListDisplay mainRepoSelector + $(widgetFile "configurators/addrepository") + +makeMiscRepositories :: Widget +makeMiscRepositories = $(widgetFile "configurators/addrepository/misc") + +makeCloudRepositories :: Widget +makeCloudRepositories = $(widgetFile "configurators/addrepository/cloud") + +makeWormholePairing :: Widget +makeWormholePairing = $(widgetFile "configurators/addrepository/wormholepairing") + +makeSshRepository :: Widget +makeSshRepository = $(widgetFile "configurators/addrepository/ssh") + +makeConnectionRepositories :: Widget +makeConnectionRepositories = $(widgetFile "configurators/addrepository/connection") + +makeArchiveRepositories :: Widget +makeArchiveRepositories = $(widgetFile "configurators/addrepository/archive") diff --git a/Assistant/WebApp/Configurators/AWS.hs b/Assistant/WebApp/Configurators/AWS.hs new file mode 100644 index 0000000000..981477e114 --- /dev/null +++ b/Assistant/WebApp/Configurators/AWS.hs @@ -0,0 +1,234 @@ +{- git-annex assistant webapp configurators for Amazon AWS services + - + - Copyright 2012 Joey Hess + - + - Licensed under the GNU AGPL version 3 or higher. + -} + +{-# LANGUAGE CPP, QuasiQuotes, TemplateHaskell, OverloadedStrings #-} + +module Assistant.WebApp.Configurators.AWS where + +import Assistant.WebApp.Common +import Assistant.WebApp.MakeRemote +#ifdef WITH_S3 +import qualified Remote.S3 as S3 +import Logs.Remote +import qualified Remote +import qualified Types.Remote as Remote +#endif +import qualified Remote.Glacier as Glacier +import qualified Remote.Helper.AWS as AWS +import Types.Remote (RemoteConfig) +import Types.StandardGroups +import Creds +import Assistant.Gpg +import Git.Types (RemoteName) + +import qualified Data.Text as T +import qualified Data.Map as M +import Data.Char + +awsConfigurator :: Widget -> Handler Html +awsConfigurator = page "Add an Amazon repository" (Just Configuration) + +glacierConfigurator :: Widget -> Handler Html +glacierConfigurator a = do + ifM (liftIO $ inPath "glacier") + ( awsConfigurator a + , awsConfigurator needglaciercli + ) + where + needglaciercli = $(widgetFile "configurators/needglaciercli") + +data StorageClass = StandardRedundancy | StandardInfrequentAccess | ReducedRedundancy + deriving (Eq, Enum, Bounded) + +instance Show StorageClass where + show StandardRedundancy = "STANDARD" + show StandardInfrequentAccess = "STANDARD_IA" + show ReducedRedundancy = "REDUCED_REDUNDANCY" + +data AWSInput = AWSInput + { accessKeyID :: Text + , secretAccessKey :: Text + , datacenter :: Text + -- Only used for S3, not Glacier. + , storageClass :: StorageClass + , repoName :: Text + , enableEncryption :: EnableEncryption + } + +data AWSCreds = AWSCreds Text Text + +extractCreds :: AWSInput -> AWSCreds +extractCreds i = AWSCreds (accessKeyID i) (secretAccessKey i) + +s3InputAForm :: Maybe CredPair -> MkAForm AWSInput +s3InputAForm defcreds = AWSInput + <$> accessKeyIDFieldWithHelp (T.pack . fst <$> defcreds) + <*> secretAccessKeyField (T.pack . snd <$> defcreds) + <*> datacenterField AWS.S3 + <*> areq (selectFieldList storageclasses) (bfs "Storage class") (Just StandardRedundancy) + <*> areq textField (bfs "Repository name") (Just "S3") + <*> enableEncryptionField + where + storageclasses :: [(Text, StorageClass)] + storageclasses = + [ ("Standard redundancy", StandardRedundancy) +#ifdef WITH_S3 +#if MIN_VERSION_aws(0,13,0) + , ("Infrequent access (cheaper for backups and archives)", StandardInfrequentAccess) +#endif +#endif + , ("Reduced redundancy (costs less)", ReducedRedundancy) + ] + +glacierInputAForm :: Maybe CredPair -> MkAForm AWSInput +glacierInputAForm defcreds = AWSInput + <$> accessKeyIDFieldWithHelp (T.pack . fst <$> defcreds) + <*> secretAccessKeyField (T.pack . snd <$> defcreds) + <*> datacenterField AWS.Glacier + <*> pure StandardRedundancy + <*> areq textField (bfs "Repository name") (Just "glacier") + <*> enableEncryptionField + +awsCredsAForm :: Maybe CredPair -> MkAForm AWSCreds +awsCredsAForm defcreds = AWSCreds + <$> accessKeyIDFieldWithHelp (T.pack . fst <$> defcreds) + <*> secretAccessKeyField (T.pack . snd <$> defcreds) + +accessKeyIDField :: Widget -> Maybe Text -> MkAForm Text +accessKeyIDField help = areq (textField `withNote` help) (bfs "Access Key ID") + +accessKeyIDFieldWithHelp :: Maybe Text -> MkAForm Text +accessKeyIDFieldWithHelp = accessKeyIDField help + where + help = [whamlet| + + Get Amazon access keys +|] + +secretAccessKeyField :: Maybe Text -> MkAForm Text +secretAccessKeyField = areq passwordField (bfs "Secret Access Key") + +datacenterField :: AWS.Service -> MkAForm Text +datacenterField service = areq (selectFieldList list) (bfs "Datacenter") defregion + where + list = M.toList $ AWS.regionMap service + defregion = Just $ AWS.defaultRegion service + +getAddS3R :: Handler Html +getAddS3R = postAddS3R + +postAddS3R :: Handler Html +#ifdef WITH_S3 +postAddS3R = awsConfigurator $ do + defcreds <- liftAnnex previouslyUsedAWSCreds + ((result, form), enctype) <- liftH $ + runFormPostNoToken $ renderBootstrap3 bootstrapFormLayout $ s3InputAForm defcreds + case result of + FormSuccess input -> liftH $ do + let name = T.unpack $ repoName input + makeAWSRemote initSpecialRemote S3.remote TransferGroup (extractCreds input) name $ M.fromList + [ configureEncryption $ enableEncryption input + , ("type", "S3") + , ("datacenter", T.unpack $ datacenter input) + , ("storageclass", show $ storageClass input) + , ("chunk", "1MiB") + ] + _ -> $(widgetFile "configurators/adds3") +#else +postAddS3R = giveup "S3 not supported by this build" +#endif + +getAddGlacierR :: Handler Html +getAddGlacierR = postAddGlacierR + +postAddGlacierR :: Handler Html +#ifdef WITH_S3 +postAddGlacierR = glacierConfigurator $ do + defcreds <- liftAnnex previouslyUsedAWSCreds + ((result, form), enctype) <- liftH $ + runFormPostNoToken $ renderBootstrap3 bootstrapFormLayout $ glacierInputAForm defcreds + case result of + FormSuccess input -> liftH $ do + let name = T.unpack $ repoName input + makeAWSRemote initSpecialRemote Glacier.remote SmallArchiveGroup (extractCreds input) name $ M.fromList + [ configureEncryption $ enableEncryption input + , ("type", "glacier") + , ("datacenter", T.unpack $ datacenter input) + ] + _ -> $(widgetFile "configurators/addglacier") +#else +postAddGlacierR = giveup "S3 not supported by this build" +#endif + +getEnableS3R :: UUID -> Handler Html +#ifdef WITH_S3 +getEnableS3R uuid = do + m <- liftAnnex readRemoteLog + if maybe False S3.configIA (M.lookup uuid m) + then redirect $ EnableIAR uuid + else postEnableS3R uuid +#else +getEnableS3R = postEnableS3R +#endif + +postEnableS3R :: UUID -> Handler Html +#ifdef WITH_S3 +postEnableS3R uuid = awsConfigurator $ enableAWSRemote S3.remote uuid +#else +postEnableS3R _ = giveup "S3 not supported by this build" +#endif + +getEnableGlacierR :: UUID -> Handler Html +getEnableGlacierR = postEnableGlacierR + +postEnableGlacierR :: UUID -> Handler Html +postEnableGlacierR = glacierConfigurator . enableAWSRemote Glacier.remote + +enableAWSRemote :: RemoteType -> UUID -> Widget +#ifdef WITH_S3 +enableAWSRemote remotetype uuid = do + defcreds <- liftAnnex previouslyUsedAWSCreds + ((result, form), enctype) <- liftH $ + runFormPostNoToken $ renderBootstrap3 bootstrapFormLayout $ awsCredsAForm defcreds + case result of + FormSuccess creds -> liftH $ do + m <- liftAnnex readRemoteLog + let name = fromJust $ M.lookup "name" $ + fromJust $ M.lookup uuid m + makeAWSRemote enableSpecialRemote remotetype SmallArchiveGroup creds name M.empty + _ -> do + description <- liftAnnex $ + T.pack <$> Remote.prettyUUID uuid + $(widgetFile "configurators/enableaws") +#else +enableAWSRemote _ _ = giveup "S3 not supported by this build" +#endif + +makeAWSRemote :: SpecialRemoteMaker -> RemoteType -> StandardGroup -> AWSCreds -> RemoteName -> RemoteConfig -> Handler () +makeAWSRemote maker remotetype defaultgroup (AWSCreds ak sk) name config = + setupCloudRemote defaultgroup Nothing $ + maker hostname remotetype (Just creds) config + where + creds = (T.unpack ak, T.unpack sk) + {- AWS services use the remote name as the basis for a host + - name, so filter it to contain valid characters. -} + hostname = case filter isAlphaNum name of + [] -> "aws" + n -> n + +getRepoInfo :: RemoteConfig -> Widget +getRepoInfo c = [whamlet|S3 remote using bucket: #{bucket}|] + where + bucket = fromMaybe "" $ M.lookup "bucket" c + +#ifdef WITH_S3 +previouslyUsedAWSCreds :: Annex (Maybe CredPair) +previouslyUsedAWSCreds = getM gettype [S3.remote, Glacier.remote] + where + gettype t = previouslyUsedCredPair AWS.creds t $ + not . S3.configIA . Remote.config +#endif diff --git a/Assistant/WebApp/Configurators/Delete.hs b/Assistant/WebApp/Configurators/Delete.hs new file mode 100644 index 0000000000..117d4b4272 --- /dev/null +++ b/Assistant/WebApp/Configurators/Delete.hs @@ -0,0 +1,120 @@ +{- git-annex assistant webapp repository deletion + - + - Copyright 2013 Joey Hess + - + - Licensed under the GNU AGPL version 3 or higher. + -} + +{-# LANGUAGE QuasiQuotes, TemplateHaskell, OverloadedStrings #-} + +module Assistant.WebApp.Configurators.Delete where + +import Assistant.WebApp.Common +import Assistant.DeleteRemote +import Assistant.DaemonStatus +import Assistant.ScanRemotes +import Assistant.Sync +import qualified Remote +import qualified Git +import Config.Files +import Logs.Trust +import Logs.Remote +import Logs.PreferredContent +import Types.StandardGroups +import Annex.UUID +import Command.Uninit (prepareRemoveAnnexDir) + +import qualified Data.Text as T +import qualified Data.Map as M + +notCurrentRepo :: UUID -> Handler Html -> Handler Html +notCurrentRepo uuid a = do + u <- liftAnnex getUUID + if u == uuid + then redirect DeleteCurrentRepositoryR + else go =<< liftAnnex (Remote.remoteFromUUID uuid) + where + go Nothing = error "Unknown UUID" + go (Just _) = a + +getDeleteRepositoryR :: UUID -> Handler Html +getDeleteRepositoryR uuid = notCurrentRepo uuid $ do + deletionPage $ do + reponame <- liftAnnex $ Remote.prettyUUID uuid + $(widgetFile "configurators/delete/start") + +getStartDeleteRepositoryR :: UUID -> Handler Html +getStartDeleteRepositoryR uuid = do + remote <- fromMaybe (error "unknown remote") + <$> liftAnnex (Remote.remoteFromUUID uuid) + liftAnnex $ do + trustSet uuid UnTrusted + setStandardGroup uuid UnwantedGroup + liftAssistant $ addScanRemotes True [remote] + redirect DashboardR + +getFinishDeleteRepositoryR :: UUID -> Handler Html +getFinishDeleteRepositoryR uuid = deletionPage $ do + void $ liftAssistant $ removeRemote uuid + + reponame <- liftAnnex $ Remote.prettyUUID uuid + {- If it's not listed in the remote log, it must be a git repo. -} + gitrepo <- liftAnnex $ M.notMember uuid <$> readRemoteLog + $(widgetFile "configurators/delete/finished") + +getDeleteCurrentRepositoryR :: Handler Html +getDeleteCurrentRepositoryR = deleteCurrentRepository + +postDeleteCurrentRepositoryR :: Handler Html +postDeleteCurrentRepositoryR = deleteCurrentRepository + +deleteCurrentRepository :: Handler Html +deleteCurrentRepository = dangerPage $ do + reldir <- fromJust . relDir <$> liftH getYesod + havegitremotes <- haveremotes syncGitRemotes + havedataremotes <- haveremotes downloadRemotes + ((result, form), enctype) <- liftH $ + runFormPostNoToken $ renderBootstrap3 bootstrapFormLayout $ + sanityVerifierAForm $ SanityVerifier magicphrase + case result of + FormSuccess _ -> liftH $ do + dir <- liftAnnex $ fromRepo Git.repoPath + liftIO $ removeAutoStartFile dir + + {- Disable syncing to this repository, and all + - remotes. This stops all transfers, and all + - file watching. -} + liftAssistant $ do + changeSyncable Nothing False + rs <- syncRemotes <$> getDaemonStatus + mapM_ (\r -> changeSyncable (Just r) False) rs + + liftAnnex $ prepareRemoveAnnexDir dir + liftIO $ removeDirectoryRecursive =<< absPath dir + + redirect ShutdownConfirmedR + _ -> $(widgetFile "configurators/delete/currentrepository") + where + haveremotes selector = not . null . selector + <$> liftAssistant getDaemonStatus + +data SanityVerifier = SanityVerifier T.Text + deriving (Eq) + +sanityVerifierAForm :: SanityVerifier -> MkAForm SanityVerifier +sanityVerifierAForm template = SanityVerifier + <$> areq checksanity (bfs "Confirm deletion?") Nothing + where + checksanity = checkBool (\input -> SanityVerifier input == template) + insane textField + + insane = "Maybe this is not a good idea..." :: Text + +deletionPage :: Widget -> Handler Html +deletionPage = page "Delete repository" (Just Configuration) + +dangerPage :: Widget -> Handler Html +dangerPage = page "Danger danger danger" (Just Configuration) + +magicphrase :: Text +magicphrase = "Yes, please do as I say!" diff --git a/Assistant/WebApp/Configurators/Edit.hs b/Assistant/WebApp/Configurators/Edit.hs new file mode 100644 index 0000000000..fe30d1b340 --- /dev/null +++ b/Assistant/WebApp/Configurators/Edit.hs @@ -0,0 +1,317 @@ +{- git-annex assistant webapp configurator for editing existing repos + - + - Copyright 2012 Joey Hess + - + - Licensed under the GNU AGPL version 3 or higher. + -} + +{-# LANGUAGE CPP, QuasiQuotes, TemplateHaskell, OverloadedStrings #-} +{-# LANGUAGE FlexibleContexts #-} + +module Assistant.WebApp.Configurators.Edit where + +import Assistant.WebApp.Common +import Assistant.WebApp.Gpg +import Assistant.WebApp.Configurators +import Assistant.DaemonStatus +import Assistant.WebApp.MakeRemote (uniqueRemoteName) +import Assistant.ScanRemotes +import Assistant.Sync +import Assistant.Alert +import qualified Assistant.WebApp.Configurators.AWS as AWS +#ifdef WITH_S3 +import qualified Assistant.WebApp.Configurators.IA as IA +import qualified Remote.S3 as S3 +#endif +import qualified Remote +import qualified Types.Remote as Remote +import qualified Remote.List as Remote +import Logs.UUID +import Logs.Group +import Logs.PreferredContent +import Logs.Remote +import Types.StandardGroups +import qualified Git +import qualified Git.Types as Git +import qualified Git.Command +import qualified Git.Config +import qualified Annex +import Git.Remote +import Remote.Helper.Encryptable (extractCipher) +import Types.Crypto +import Utility.Gpg +import Annex.UUID +import Assistant.Ssh +import Config +import Config.GitConfig +import Config.DynamicConfig + +import qualified Data.Text as T +import qualified Data.Map as M +import qualified Data.Set as S + +data RepoGroup = RepoGroupCustom String | RepoGroupStandard StandardGroup + deriving (Show, Eq) + +data RepoConfig = RepoConfig + { repoName :: Text + , repoDescription :: Maybe Text + , repoGroup :: RepoGroup + , repoAssociatedDirectory :: Maybe Text + , repoSyncable :: Bool + } + deriving (Show) + +getRepoConfig :: UUID -> Maybe Remote -> Annex RepoConfig +getRepoConfig uuid mremote = do + -- Ensure we're editing current data by discarding caches. + void groupMapLoad + void uuidMapLoad + + groups <- lookupGroups uuid + remoteconfig <- M.lookup uuid <$> readRemoteLog + let (repogroup, associateddirectory) = case getStandardGroup groups of + Nothing -> (RepoGroupCustom $ unwords $ S.toList groups, Nothing) + Just g -> (RepoGroupStandard g, associatedDirectory remoteconfig g) + + description <- fmap T.pack . M.lookup uuid <$> uuidMap + + syncable <- case mremote of + Just r -> liftIO $ getDynamicConfig $ remoteAnnexSync $ Remote.gitconfig r + Nothing -> getGitConfigVal annexAutoCommit + + return $ RepoConfig + (T.pack $ maybe "here" Remote.name mremote) + description + repogroup + (T.pack <$> associateddirectory) + syncable + +setRepoConfig :: UUID -> Maybe Remote -> RepoConfig -> RepoConfig -> Handler () +setRepoConfig uuid mremote oldc newc = do + when descriptionChanged $ liftAnnex $ do + maybe noop (describeUUID uuid . T.unpack) (repoDescription newc) + void uuidMapLoad + when nameChanged $ do + liftAnnex $ do + name <- uniqueRemoteName (legalName newc) 0 <$> Annex.getGitRemotes + {- git remote rename expects there to be a + - remote..fetch, and exits nonzero if + - there's not. Special remotes don't normally + - have that, and don't use it. Temporarily add + - it if it's missing. -} + let remotefetch = "remote." ++ T.unpack (repoName oldc) ++ ".fetch" + needfetch <- isNothing <$> fromRepo (Git.Config.getMaybe remotefetch) + when needfetch $ + inRepo $ Git.Command.run + [Param "config", Param remotefetch, Param ""] + inRepo $ Git.Command.run + [ Param "remote" + , Param "rename" + , Param $ T.unpack $ repoName oldc + , Param name + ] + void Remote.remoteListRefresh + liftAssistant updateSyncRemotes + when associatedDirectoryChanged $ case repoAssociatedDirectory newc of + Nothing -> noop + Just t + | T.null t -> noop + | otherwise -> liftAnnex $ do + let dir = takeBaseName $ T.unpack t + m <- readRemoteLog + case M.lookup uuid m of + Nothing -> noop + Just remoteconfig -> configSet uuid $ + M.insert "preferreddir" dir remoteconfig + when groupChanged $ do + liftAnnex $ case repoGroup newc of + RepoGroupStandard g -> setStandardGroup uuid g + RepoGroupCustom s -> groupSet uuid $ S.fromList $ words s + {- Enabling syncing will cause a scan, + - so avoid queueing a duplicate scan. -} + when (repoSyncable newc && not syncableChanged) $ liftAssistant $ + case mremote of + Just remote -> addScanRemotes True [remote] + Nothing -> addScanRemotes True + =<< syncDataRemotes <$> getDaemonStatus + when syncableChanged $ + liftAssistant $ changeSyncable mremote (repoSyncable newc) + where + syncableChanged = repoSyncable oldc /= repoSyncable newc + associatedDirectoryChanged = repoAssociatedDirectory oldc /= repoAssociatedDirectory newc + groupChanged = repoGroup oldc /= repoGroup newc + nameChanged = isJust mremote && legalName oldc /= legalName newc + descriptionChanged = repoDescription oldc /= repoDescription newc + + legalName = makeLegalName . T.unpack . repoName + +editRepositoryAForm :: Maybe Git.Repo -> Maybe Remote -> RepoConfig -> MkAForm RepoConfig +editRepositoryAForm mrepo mremote d = RepoConfig + <$> areq (if ishere then readonlyTextField else textField) + (bfs "Name") (Just $ repoName d) + <*> aopt textField (bfs "Description") (Just $ repoDescription d) + <*> areq (selectFieldList groups `withNote` help) (bfs "Repository group") (Just $ repoGroup d) + <*> associateddirectory + <*> areq checkBoxField "Syncing enabled" (Just $ repoSyncable d) + where + ishere = isNothing mremote + isspecial = maybe False ((== Git.Unknown) . Git.location) mrepo + groups = customgroups ++ standardgroups + standardgroups :: [(Text, RepoGroup)] + standardgroups = map (\g -> (T.pack $ descStandardGroup g , RepoGroupStandard g)) $ + filter sanegroup [minBound..maxBound] + sanegroup + | isspecial = const True + | otherwise = not . specialRemoteOnly + customgroups :: [(Text, RepoGroup)] + customgroups = case repoGroup d of + RepoGroupCustom s -> [(T.pack s, RepoGroupCustom s)] + _ -> [] + help = [whamlet|What's this?|] + + associateddirectory = case repoAssociatedDirectory d of + Nothing -> aopt hiddenField "" Nothing + Just dir -> aopt textField (bfs "Associated directory") (Just $ Just dir) + +getEditRepositoryR :: RepoId -> Handler Html +getEditRepositoryR = postEditRepositoryR + +postEditRepositoryR :: RepoId -> Handler Html +postEditRepositoryR = editForm False + +getEditNewRepositoryR :: UUID -> Handler Html +getEditNewRepositoryR = postEditNewRepositoryR + +postEditNewRepositoryR :: UUID -> Handler Html +postEditNewRepositoryR = editForm True . RepoUUID + +getEditNewCloudRepositoryR :: UUID -> Handler Html +getEditNewCloudRepositoryR = postEditNewCloudRepositoryR + +postEditNewCloudRepositoryR :: UUID -> Handler Html +postEditNewCloudRepositoryR uuid = connectionNeeded >> editForm True (RepoUUID uuid) + +editForm :: Bool -> RepoId -> Handler Html +editForm new (RepoUUID uuid) + | uuid == webUUID || uuid == bitTorrentUUID = page "The web" (Just Configuration) $ do + $(widgetFile "configurators/edit/webrepository") + | otherwise = page "Edit repository" (Just Configuration) $ do + mremote <- liftAnnex $ Remote.remoteFromUUID uuid + when (mremote == Nothing) $ + whenM ((/=) uuid <$> liftAnnex getUUID) $ + error "unknown remote" + curr <- liftAnnex $ getRepoConfig uuid mremote + liftAnnex $ checkAssociatedDirectory curr mremote + mrepo <- liftAnnex $ + maybe (pure Nothing) (Just <$$> Remote.getRepo) mremote + ((result, form), enctype) <- liftH $ + runFormPostNoToken $ renderBootstrap3 bootstrapFormLayout $ + editRepositoryAForm mrepo mremote curr + case result of + FormSuccess input -> liftH $ do + setRepoConfig uuid mremote curr input + liftAnnex $ checkAssociatedDirectory input mremote + redirect DashboardR + _ -> do + let istransfer = repoGroup curr == RepoGroupStandard TransferGroup + config <- liftAnnex $ M.lookup uuid <$> readRemoteLog + let repoInfo = getRepoInfo mremote config + let repoEncryption = getRepoEncryption mremote config + $(widgetFile "configurators/edit/repository") +editForm _new r@(RepoName _) = page "Edit repository" (Just Configuration) $ do + mr <- liftAnnex (repoIdRemote r) + let repoInfo = getRepoInfo mr Nothing + g <- liftAnnex gitRepo + mrepo <- liftAnnex $ maybe (pure Nothing) (Just <$$> Remote.getRepo) mr + let sshrepo = maybe False (remoteLocationIsSshUrl . flip parseRemoteLocation g . Git.repoLocation) mrepo + $(widgetFile "configurators/edit/nonannexremote") + +{- Makes any directory associated with the repository. -} +checkAssociatedDirectory :: RepoConfig -> Maybe Remote -> Annex () +checkAssociatedDirectory _ Nothing = noop +checkAssociatedDirectory cfg (Just r) = do + repoconfig <- M.lookup (Remote.uuid r) <$> readRemoteLog + case repoGroup cfg of + RepoGroupStandard gr -> case associatedDirectory repoconfig gr of + Just d -> inRepo $ \g -> + createDirectoryIfMissing True $ + Git.repoPath g d + Nothing -> noop + _ -> noop + +getRepoInfo :: Maybe Remote.Remote -> Maybe Remote.RemoteConfig -> Widget +getRepoInfo (Just r) (Just c) = case M.lookup "type" c of + Just "S3" +#ifdef WITH_S3 + | S3.configIA c -> IA.getRepoInfo c +#endif + | otherwise -> AWS.getRepoInfo c + Just t + | t /= "git" -> [whamlet|#{t} remote|] + _ -> getGitRepoInfo =<< liftAnnex (Remote.getRepo r) +getRepoInfo (Just r) _ = getRepoInfo (Just r) (Just $ Remote.config r) +getRepoInfo _ _ = [whamlet|git repository|] + +getGitRepoInfo :: Git.Repo -> Widget +getGitRepoInfo r = do + let loc = Git.repoLocation r + [whamlet|git repository located at #{loc}|] + +getRepoEncryption :: Maybe Remote.Remote -> Maybe Remote.RemoteConfig -> Widget +getRepoEncryption (Just _) (Just c) = case extractCipher c of + Nothing -> + [whamlet|not encrypted|] + (Just (SharedCipher _)) -> + [whamlet|encrypted: encryption key stored in git repository|] + (Just (EncryptedCipher _ _ ks)) -> desckeys ks + (Just (SharedPubKeyCipher _ ks)) -> desckeys ks + where + desckeys (KeyIds { keyIds = ks }) = do + cmd <- liftAnnex $ gpgCmd <$> Annex.getGitConfig + knownkeys <- liftIO (secretKeys cmd) + [whamlet| +encrypted using gpg key: +
    + $forall k <- ks +
  • + ^{gpgKeyDisplay k (M.lookup k knownkeys)} +|] +getRepoEncryption _ _ = return () -- local repo + +getUpgradeRepositoryR :: RepoId -> Handler () +getUpgradeRepositoryR (RepoUUID _) = redirect DashboardR +getUpgradeRepositoryR r = go =<< liftAnnex (repoIdRemote r) + where + go Nothing = redirect DashboardR + go (Just rmt) = do + liftIO fixSshKeyPairIdentitiesOnly + liftAnnex $ do + repo <- Remote.getRepo rmt + setConfig + (remoteConfig repo "ignore") + (Git.Config.boolConfig False) + liftAnnex $ void Remote.remoteListRefresh + liftAssistant updateSyncRemotes + liftAssistant $ syncRemote rmt + redirect DashboardR + +{- If there is no currently connected remote, display an alert suggesting + - to set up one. -} +connectionNeeded :: Handler () +connectionNeeded = whenM noconnection $ do + urlrender <- getUrlRender + void $ liftAssistant $ do + close <- asIO1 removeAlert + addAlert $ connectionNeededAlert $ AlertButton + { buttonLabel = "Connect" + , buttonUrl = urlrender ConnectionNeededR + , buttonAction = Just close + , buttonPrimary = True + } + where + noconnection = S.null . currentlyConnectedRemotes <$> liftAssistant getDaemonStatus + +getConnectionNeededR :: Handler Html +getConnectionNeededR = page "Connection needed" (Just Configuration) $ do + $(widgetFile "configurators/needconnection") diff --git a/Assistant/WebApp/Configurators/Fsck.hs b/Assistant/WebApp/Configurators/Fsck.hs new file mode 100644 index 0000000000..ee6ab1d91b --- /dev/null +++ b/Assistant/WebApp/Configurators/Fsck.hs @@ -0,0 +1,196 @@ +{- git-annex assistant fsck configuration + - + - Copyright 2013 Joey Hess + - + - Licensed under the GNU AGPL version 3 or higher. + -} + +{-# LANGUAGE QuasiQuotes, TemplateHaskell, OverloadedStrings #-} + +module Assistant.WebApp.Configurators.Fsck where + +import qualified Data.Map as M +import qualified Data.Set as S +import qualified Data.Text as T + +import Assistant.WebApp.Common +import Types.ScheduledActivity +import Utility.HumanTime +import Utility.Scheduled +import Logs.Schedule +import Annex.UUID +import qualified Remote +import Assistant.DaemonStatus +import qualified Annex.Branch +import Assistant.Fsck +import Config +import Git.Config +import qualified Annex + +{- This adds a form to the page. It does not handle posting of the form, + - because unlike a typical yesod form that posts using the same url + - that generated it, this form posts using one of two other routes. -} +showFsckForm :: Bool -> ScheduledActivity -> Widget +showFsckForm new activity = do + u <- liftAnnex getUUID + let action = if new + then AddActivityR u + else ChangeActivityR u activity + ((res, form), enctype) <- liftH $ runFsckForm new activity + case res of + FormSuccess _ -> noop + _ -> $(widgetFile "configurators/fsck/form") + +{- This does not display a form, but it does get it from a post, and run + - some Annex action on it. -} +withFsckForm :: (ScheduledActivity -> Annex ()) -> Handler () +withFsckForm a = do + ((res, _form), _enctype) <- runFsckForm False $ defaultFsck Nothing + case res of + FormSuccess activity -> liftAnnex $ a activity + _ -> noop + +mkFsck :: UUID -> UUID -> Schedule -> Duration -> ScheduledActivity +mkFsck hereu u s d + | u == hereu = ScheduledSelfFsck s d + | otherwise = ScheduledRemoteFsck u s d + +runFsckForm :: Bool -> ScheduledActivity -> Handler ((FormResult ScheduledActivity, Widget), Enctype) +runFsckForm new activity = case activity of + ScheduledSelfFsck s d -> go s d =<< liftAnnex getUUID + ScheduledRemoteFsck ru s d -> go s d ru + where + go (Schedule r t) d ru = do + u <- liftAnnex getUUID + repolist <- liftAssistant (getrepolist ru) + runFormPostNoToken $ \msg -> do + (reposRes, reposView) <- mreq (selectFieldList repolist) (bfs "") (Just ru) + (durationRes, durationView) <- mreq intField (bfs "") (Just $ durationSeconds d `quot` 60 ) + (timeRes, timeView) <- mreq (selectFieldList times) (bfs "") (Just t) + (recurranceRes, recurranceView) <- mreq (selectFieldList recurrances) (bfs "") (Just r) + let form = do + webAppFormAuthToken + $(widgetFile "configurators/fsck/formcontent") + let formresult = mkFsck + <$> pure u + <*> reposRes + <*> (Schedule <$> recurranceRes <*> timeRes) + <*> (Duration <$> ((60 *) <$> durationRes)) + return (formresult, form) + where + times :: [(Text, ScheduledTime)] + times = ensurevalue t (T.pack $ fromScheduledTime t) $ + map (\x -> (T.pack $ fromScheduledTime x, x)) $ + AnyTime : map (\h -> SpecificTime h 0) [0..23] + recurrances :: [(Text, Recurrance)] + recurrances = ensurevalue r (T.pack $ fromRecurrance r) $ + [ ("every day", Daily) + , ("every Sunday", Weekly $ Just 1) + , ("every Monday", Weekly $ Just 2) + , ("every Tuesday", Weekly $ Just 3) + , ("every Wednesday", Weekly $ Just 4) + , ("every Thursday", Weekly $ Just 5) + , ("every Friday", Weekly $ Just 6) + , ("every Saturday", Weekly $ Just 7) + , ("monthly", Monthly Nothing) + , ("twice a month", Divisible 2 (Weekly Nothing)) + , ("yearly", Yearly Nothing) + , ("twice a year", Divisible 6 (Monthly Nothing)) + , ("quarterly", Divisible 4 (Monthly Nothing)) + ] + ensurevalue v desc l = case M.lookup v (M.fromList $ map (\(x,y) -> (y,x)) l) of + Just _ -> l + Nothing -> (desc, v) : l + getrepolist :: UUID -> Assistant [(Text, UUID)] + getrepolist ensureu = do + -- It is possible to have fsck jobs for remotes that + -- do not implement remoteFsck, but it's not too useful, + -- so omit them from the UI normally. + remotes <- filter (\r -> Remote.uuid r == ensureu || isJust (Remote.remoteFsck r)) . syncRemotes + <$> getDaemonStatus + u <- liftAnnex getUUID + let us = u : (map Remote.uuid remotes) + liftAnnex $ + zip <$> (map T.pack <$> Remote.prettyListUUIDs us) <*> pure us + +defaultFsck :: Maybe Remote -> ScheduledActivity +defaultFsck Nothing = ScheduledSelfFsck (Schedule Daily AnyTime) (Duration $ 60*60) +defaultFsck (Just r) = ScheduledRemoteFsck (Remote.uuid r) (Schedule Daily AnyTime) (Duration $ 60*60) + +showFsckStatus :: ScheduledActivity -> Widget +showFsckStatus activity = do + m <- liftAnnex getLastRunTimes + let lastrun = M.lookup activity m + $(widgetFile "configurators/fsck/status") + +getConfigFsckR :: Handler Html +getConfigFsckR = postConfigFsckR +postConfigFsckR :: Handler Html +postConfigFsckR = page "Consistency checks" (Just Configuration) $ do + scheduledchecks <- liftAnnex $ + S.toList <$> (scheduleGet =<< getUUID) + rs <- liftAssistant $ + filter fsckableRemote . syncRemotes <$> getDaemonStatus + recommendedchecks <- liftAnnex $ map defaultFsck + <$> filterM (not <$$> checkFscked) (Nothing : map Just rs) + $(widgetFile "configurators/fsck") + +changeSchedule :: Handler () -> Handler Html +changeSchedule a = do + a + liftAnnex $ Annex.Branch.commit =<< Annex.Branch.commitMessage + redirect ConfigFsckR + +getRemoveActivityR :: UUID -> ScheduledActivity -> Handler Html +getRemoveActivityR u activity = changeSchedule $ + liftAnnex $ scheduleRemove u activity + +getAddActivityR :: UUID -> Handler Html +getAddActivityR = postAddActivityR +postAddActivityR :: UUID -> Handler Html +postAddActivityR u = changeSchedule $ + withFsckForm $ scheduleAdd u + +getChangeActivityR :: UUID -> ScheduledActivity -> Handler Html +getChangeActivityR = postChangeActivityR +postChangeActivityR :: UUID -> ScheduledActivity -> Handler Html +postChangeActivityR u oldactivity = changeSchedule $ + withFsckForm $ \newactivity -> scheduleChange u $ + S.insert newactivity . S.delete oldactivity + +data FsckPreferences = FsckPreferences + { enableFsckNudge :: Bool + } + +getFsckPreferences :: Annex FsckPreferences +getFsckPreferences = FsckPreferences + <$> (annexFsckNudge <$> Annex.getGitConfig) + +fsckPreferencesAForm :: FsckPreferences -> MkAForm FsckPreferences +fsckPreferencesAForm d = FsckPreferences + <$> areq (checkBoxField `withNote` nudgenote) "Reminders" (Just $ enableFsckNudge d) + where + nudgenote = [whamlet|Remind me when using repositories that lack consistency checks.|] + +runFsckPreferencesForm :: Handler ((FormResult FsckPreferences, Widget), Enctype) +runFsckPreferencesForm = do + prefs <- liftAnnex getFsckPreferences + runFormPostNoToken $ renderBootstrap3 formLayout $ fsckPreferencesAForm prefs + where formLayout = BootstrapHorizontalForm (ColSm 0) (ColSm 2) (ColSm 0) (ColSm 10) + +showFsckPreferencesForm :: Widget +showFsckPreferencesForm = do + ((res, form), enctype) <- liftH $ runFsckPreferencesForm + case res of + FormSuccess _ -> noop + _ -> $(widgetFile "configurators/fsck/preferencesform") + +postConfigFsckPreferencesR :: Handler Html +postConfigFsckPreferencesR = do + ((res, _form), _enctype) <- runFsckPreferencesForm + case res of + FormSuccess prefs -> + liftAnnex $ setConfig (annexConfig "fscknudge") + (boolConfig $ enableFsckNudge prefs) + _ -> noop + redirect ConfigFsckR diff --git a/Assistant/WebApp/Configurators/IA.hs b/Assistant/WebApp/Configurators/IA.hs new file mode 100644 index 0000000000..32164339f5 --- /dev/null +++ b/Assistant/WebApp/Configurators/IA.hs @@ -0,0 +1,210 @@ +{- git-annex assistant webapp configurators for Internet Archive + - + - Copyright 2013 Joey Hess + - + - Licensed under the GNU AGPL version 3 or higher. + -} + +{-# LANGUAGE CPP, QuasiQuotes, TemplateHaskell, OverloadedStrings #-} + +module Assistant.WebApp.Configurators.IA where + +import Assistant.WebApp.Common +import qualified Assistant.WebApp.Configurators.AWS as AWS +#ifdef WITH_S3 +import qualified Remote.S3 as S3 +import qualified Remote.Helper.AWS as AWS +import Assistant.WebApp.MakeRemote +import qualified Remote +import qualified Types.Remote as Remote +import Types.StandardGroups +import Logs.Remote +import Assistant.Gpg +#endif +import Types.Remote (RemoteConfig) +import qualified Annex.Url as Url +import Creds + +import qualified Data.Text as T +import qualified Data.Map as M +import Data.Char +import Network.URI + +iaConfigurator :: Widget -> Handler Html +iaConfigurator = page "Add an Internet Archive repository" (Just Configuration) + +data IAInput = IAInput + { accessKeyID :: Text + , secretAccessKey :: Text + , mediaType :: MediaType + , itemName :: Text + } + +extractCreds :: IAInput -> AWS.AWSCreds +extractCreds i = AWS.AWSCreds (accessKeyID i) (secretAccessKey i) + +{- IA defines only a few media types currently, or the media type + - may be omitted + - + - We add a few other common types, mapped to what we've been told + - is the closest match. + -} +data MediaType = MediaImages | MediaAudio | MediaVideo | MediaText | MediaSoftware | MediaOmitted + deriving (Eq, Ord, Enum, Bounded) + +{- Format a MediaType for entry into the IA metadata -} +formatMediaType :: MediaType -> String +formatMediaType MediaText = "texts" +formatMediaType MediaImages = "image" +formatMediaType MediaSoftware = "software" +formatMediaType MediaVideo = "movies" +formatMediaType MediaAudio = "audio" +formatMediaType MediaOmitted = "" + +{- A default collection to use for each Mediatype. -} +collectionMediaType :: MediaType -> Maybe String +collectionMediaType MediaText = Just "opensource" +collectionMediaType MediaImages = Just "opensource" -- not ideal +collectionMediaType MediaSoftware = Just "opensource" -- not ideal +collectionMediaType MediaVideo = Just "opensource_movies" +collectionMediaType MediaAudio = Just "opensource_audio" +collectionMediaType MediaOmitted = Just "opensource" + +{- Format a MediaType for user display. -} +showMediaType :: MediaType -> String +showMediaType MediaText = "texts" +showMediaType MediaImages = "photos & images" +showMediaType MediaSoftware = "software" +showMediaType MediaVideo = "videos & movies" +showMediaType MediaAudio = "audio & music" +showMediaType MediaOmitted = "other" + +iaInputAForm :: Maybe CredPair -> MkAForm IAInput +iaInputAForm defcreds = IAInput + <$> accessKeyIDFieldWithHelp (T.pack . fst <$> defcreds) + <*> AWS.secretAccessKeyField (T.pack . snd <$> defcreds) + <*> areq (selectFieldList mediatypes) (bfs "Media Type") (Just MediaOmitted) + <*> areq (textField `withExpandableNote` ("Help", itemNameHelp)) (bfs "Item Name") Nothing + where + mediatypes :: [(Text, MediaType)] + mediatypes = map (\t -> (T.pack $ showMediaType t, t)) [minBound..] + +itemNameHelp :: Widget +itemNameHelp = [whamlet| +
    + Each item stored in the Internet Archive must have a unique name. +
    + Once you create the item, a special directory will appear # + with a name matching the item name. Files you put in that directory # + will be uploaded to your Internet Archive item. +|] + +iaCredsAForm :: Maybe CredPair -> MkAForm AWS.AWSCreds +iaCredsAForm defcreds = AWS.AWSCreds + <$> accessKeyIDFieldWithHelp (T.pack . fst <$> defcreds) + <*> AWS.secretAccessKeyField (T.pack . snd <$> defcreds) + +#ifdef WITH_S3 +previouslyUsedIACreds :: Annex (Maybe CredPair) +previouslyUsedIACreds = previouslyUsedCredPair AWS.creds S3.remote $ + S3.configIA . Remote.config +#endif + +accessKeyIDFieldWithHelp :: Maybe Text -> MkAForm Text +accessKeyIDFieldWithHelp = AWS.accessKeyIDField help + where + help = [whamlet| + + Get Internet Archive access keys +|] + +getAddIAR :: Handler Html +getAddIAR = postAddIAR + +postAddIAR :: Handler Html +#ifdef WITH_S3 +postAddIAR = iaConfigurator $ do + defcreds <- liftAnnex previouslyUsedIACreds + ((result, form), enctype) <- liftH $ + runFormPostNoToken $ renderBootstrap3 bootstrapFormLayout $ iaInputAForm defcreds + case result of + FormSuccess input -> liftH $ do + let name = escapeBucket $ T.unpack $ itemName input + AWS.makeAWSRemote initSpecialRemote S3.remote PublicGroup (extractCreds input) name $ + M.fromList $ catMaybes + [ Just $ configureEncryption NoEncryption + , Just ("type", "S3") + , Just ("host", S3.iaHost) + , Just ("bucket", escapeHeader name) + , Just ("x-archive-meta-title", escapeHeader $ T.unpack $ itemName input) + , if mediaType input == MediaOmitted + then Nothing + else Just ("x-archive-mediatype", formatMediaType $ mediaType input) + , (,) <$> pure "x-archive-meta-collection" <*> collectionMediaType (mediaType input) + -- Make item show up ASAP. + , Just ("x-archive-interactive-priority", "1") + , Just ("preferreddir", name) + ] + _ -> $(widgetFile "configurators/addia") +#else +postAddIAR = giveup "S3 not supported by this build" +#endif + +getEnableIAR :: UUID -> Handler Html +getEnableIAR = postEnableIAR + +postEnableIAR :: UUID -> Handler Html +#ifdef WITH_S3 +postEnableIAR = iaConfigurator . enableIARemote +#else +postEnableIAR _ = giveup "S3 not supported by this build" +#endif + +#ifdef WITH_S3 +enableIARemote :: UUID -> Widget +enableIARemote uuid = do + defcreds <- liftAnnex previouslyUsedIACreds + ((result, form), enctype) <- liftH $ + runFormPostNoToken $ renderBootstrap3 bootstrapFormLayout $ iaCredsAForm defcreds + case result of + FormSuccess creds -> liftH $ do + m <- liftAnnex readRemoteLog + let name = fromJust $ M.lookup "name" $ + fromJust $ M.lookup uuid m + AWS.makeAWSRemote enableSpecialRemote S3.remote PublicGroup creds name M.empty + _ -> do + description <- liftAnnex $ + T.pack <$> Remote.prettyUUID uuid + $(widgetFile "configurators/enableia") +#endif + +{- Convert a description into a bucket item name, which will also be + - used as the repository name, and the preferreddir. + - IA seems to need only lower case, and no spaces. -} +escapeBucket :: String -> String +escapeBucket = map toLower . replace " " "-" + +{- IA S3 API likes headers to be URI escaped, escaping spaces looks ugly. -} +escapeHeader :: String -> String +escapeHeader = escapeURIString (\c -> isUnescapedInURI c && c /= ' ') + +getRepoInfo :: RemoteConfig -> Widget +getRepoInfo c = do + uo <- liftAnnex Url.getUrlOptions + exists <- liftIO $ catchDefaultIO False $ Url.exists url uo + [whamlet| + + Internet Archive item +$if (not exists) +

    + The page will only be available once some files # + have been uploaded, and the Internet Archive has processed them. +|] + where + bucket = fromMaybe "" $ M.lookup "bucket" c +#ifdef WITH_S3 + url = S3.iaItemUrl bucket +#else + url = case bucket of + _ -> "" +#endif diff --git a/Assistant/WebApp/Configurators/Local.hs b/Assistant/WebApp/Configurators/Local.hs new file mode 100644 index 0000000000..e4bfbfebee --- /dev/null +++ b/Assistant/WebApp/Configurators/Local.hs @@ -0,0 +1,438 @@ +{- git-annex assistant webapp configurators for making local repositories + - + - Copyright 2012-2014 Joey Hess + - + - Licensed under the GNU AGPL version 3 or higher. + -} + +{-# LANGUAGE CPP, QuasiQuotes, TemplateHaskell, OverloadedStrings #-} +{-# LANGUAGE RankNTypes, KindSignatures, TypeFamilies, FlexibleContexts #-} + +module Assistant.WebApp.Configurators.Local where + +import Assistant.WebApp.Common +import Assistant.WebApp.Gpg +import Assistant.WebApp.MakeRemote +import Assistant.Sync +import Assistant.Restart +import Annex.MakeRepo +import qualified Annex +import qualified Git +import qualified Git.Config +import qualified Git.Command +import qualified Command.Sync +import Config.Files +import Utility.FreeDesktop +import Utility.DiskFree +#ifndef mingw32_HOST_OS +import Utility.Mounts +#endif +import Utility.DataUnits +import Remote (prettyUUID) +import Annex.UUID +import Types.StandardGroups +import Logs.PreferredContent +import Logs.UUID +import Utility.UserInfo +import Config +import Utility.Gpg +import qualified Remote.GCrypt as GCrypt +import qualified Types.Remote +import Utility.Android + +import qualified Data.Text as T +import qualified Data.Map as M +import Data.Char +import Data.Ord +import qualified Text.Hamlet as Hamlet + +data RepositoryPath = RepositoryPath Text + deriving Show + +{- Custom field display for a RepositoryPath, with an icon etc. + - + - Validates that the path entered is not empty, and is a safe value + - to use as a repository. -} +repositoryPathField :: forall (m :: * -> *). (MonadIO m, HandlerSite m ~ WebApp) => Bool -> Field m Text +repositoryPathField autofocus = Field + { fieldParse = \l _ -> parse l + , fieldEnctype = UrlEncoded + , fieldView = view + } + where + view idAttr nameAttr attrs val isReq = + [whamlet||] + + parse [path] + | T.null path = nopath + | otherwise = liftIO $ checkRepositoryPath path + parse [] = return $ Right Nothing + parse _ = nopath + + nopath = return $ Left "Enter a location for the repository" + +{- As well as checking the path for a lot of silly things, tilde is + - expanded in the returned path. -} +checkRepositoryPath :: Text -> IO (Either (SomeMessage WebApp) (Maybe Text)) +checkRepositoryPath p = do + home <- myHomeDir + let basepath = expandTilde home $ T.unpack p + path <- absPath basepath + let parent = parentDir path + problems <- catMaybes <$> mapM runcheck + [ (return $ path == "/", "Enter the full path to use for the repository.") + , (return $ all isSpace basepath, "A blank path? Seems unlikely.") + , (doesFileExist path, "A file already exists with that name.") + , (return $ path == home, "Sorry, using git-annex for your whole home directory is not currently supported.") + , (not <$> doesDirectoryExist parent, "Parent directory does not exist.") + , (not <$> canWrite path, "Cannot write a repository there.") + ] + return $ + case headMaybe problems of + Nothing -> Right $ Just $ T.pack basepath + Just prob -> Left prob + where + runcheck (chk, msg) = ifM chk ( return $ Just msg, return Nothing ) + expandTilde home ('~':'/':path) = home path + expandTilde _ path = path + +{- On first run, if run in the home directory, default to putting it in + - ~/Desktop/annex, when a Desktop directory exists, and ~/annex otherwise. + - + - When on Android, default to ~/storage/shared/annex, which termux sets up + - as a link to the sdcard. + - + - If run in another directory, that the user can write to, + - the user probably wants to put it there. Unless that directory + - contains a git-annex file, in which case the user has probably + - browsed to a directory with git-annex and run it from there. -} +defaultRepositoryPath :: Bool -> IO FilePath +defaultRepositoryPath firstrun = do +#ifndef mingw32_HOST_OS + home <- myHomeDir + currdir <- liftIO getCurrentDirectory + if home == currdir && firstrun + then inhome + else ifM (legit currdir <&&> canWrite currdir) + ( return currdir + , inhome + ) +#else + -- On Windows, always default to ~/Desktop/annex or ~/annex, + -- no cwd handling because the user might be able to write + -- to the entire drive. + if firstrun then inhome else inhome +#endif + where + inhome = ifM osAndroid + ( do + home <- myHomeDir + let storageshared = home "storage" "shared" + ifM (doesDirectoryExist storageshared) + ( relHome $ storageshared gitAnnexAssistantDefaultDir + , return $ "~" gitAnnexAssistantDefaultDir + ) + , do + desktop <- userDesktopDir + ifM (doesDirectoryExist desktop <&&> canWrite desktop) + ( relHome $ desktop gitAnnexAssistantDefaultDir + , return $ "~" gitAnnexAssistantDefaultDir + ) + ) +#ifndef mingw32_HOST_OS + -- Avoid using eg, standalone build's git-annex.linux/ directory + -- when run from there. + legit d = not <$> doesFileExist (d "git-annex") +#endif + +newRepositoryForm :: FilePath -> Hamlet.Html -> MkMForm RepositoryPath +newRepositoryForm defpath msg = do + (pathRes, pathView) <- mreq (repositoryPathField True) (bfs "") + (Just $ T.pack $ addTrailingPathSeparator defpath) + let (err, errmsg) = case pathRes of + FormMissing -> (False, "") + FormFailure l -> (True, concatMap T.unpack l) + FormSuccess _ -> (False, "") + let form = do + webAppFormAuthToken + $(widgetFile "configurators/newrepository/form") + return (RepositoryPath <$> pathRes, form) + +{- Making the first repository, when starting the webapp for the first time. -} +getFirstRepositoryR :: Handler Html +getFirstRepositoryR = postFirstRepositoryR +postFirstRepositoryR :: Handler Html +postFirstRepositoryR = page "Getting started" (Just Configuration) $ do + unlessM (liftIO $ inPath "git") $ + giveup "You need to install git in order to use git-annex!" +#ifdef __ANDROID__ + androidspecial <- liftIO $ doesDirectoryExist "/sdcard/DCIM" + let path = "/sdcard/annex" +#else + androidspecial <- liftIO osAndroid + path <- liftIO . defaultRepositoryPath =<< liftH inFirstRun +#endif + ((res, form), enctype) <- liftH $ runFormPostNoToken $ newRepositoryForm path + case res of + FormSuccess (RepositoryPath p) -> liftH $ + startFullAssistant (T.unpack p) ClientGroup Nothing + _ -> $(widgetFile "configurators/newrepository/first") + +getAndroidCameraRepositoryR :: Handler () +getAndroidCameraRepositoryR = do +#ifdef __ANDROID__ + let dcim = "/sdcard/DCIM" +#else + home <- liftIO myHomeDir + let dcim = home "storage" "dcim" +#endif + startFullAssistant dcim SourceGroup $ Just addignore + where + addignore = do + liftIO $ unlessM (doesFileExist ".gitignore") $ + writeFile ".gitignore" ".thumbnails" + void $ inRepo $ + Git.Command.runBool [Param "add", File ".gitignore"] + +{- Adding a new local repository, which may be entirely separate, or may + - be connected to the current repository. -} +getNewRepositoryR :: Handler Html +getNewRepositoryR = postNewRepositoryR +postNewRepositoryR :: Handler Html +postNewRepositoryR = page "Add another repository" (Just Configuration) $ do + home <- liftIO myHomeDir + ((res, form), enctype) <- liftH $ runFormPostNoToken $ newRepositoryForm home + case res of + FormSuccess (RepositoryPath p) -> do + let path = T.unpack p + isnew <- liftIO $ makeRepo path False + u <- liftIO $ initRepo isnew True path Nothing (Just ClientGroup) + liftIO $ addAutoStartFile path + liftIO $ startAssistant path + askcombine u path + _ -> $(widgetFile "configurators/newrepository") + where + askcombine newrepouuid newrepopath = do + newrepo <- liftIO $ relHome newrepopath + mainrepo <- fromJust . relDir <$> liftH getYesod + $(widgetFile "configurators/newrepository/combine") + +{- Ensure that a remote's description, group, etc are available by + - immediately pulling from it. Also spawns a sync to push to it as well. -} +immediateSyncRemote :: Remote -> Assistant () +immediateSyncRemote r = do + currentbranch <- liftAnnex $ join Command.Sync.getCurrBranch + void $ manualPull currentbranch [r] + syncRemote r + +getCombineRepositoryR :: FilePath -> UUID -> Handler Html +getCombineRepositoryR newrepopath newrepouuid = do + liftAssistant . immediateSyncRemote =<< combineRepos newrepopath remotename + redirect $ EditRepositoryR $ RepoUUID newrepouuid + where + remotename = takeFileName newrepopath + +selectDriveForm :: [RemovableDrive] -> Hamlet.Html -> MkMForm RemovableDrive +selectDriveForm drives = renderBootstrap3 bootstrapFormLayout $ RemovableDrive + <$> pure Nothing + <*> areq (selectFieldList pairs `withNote` onlywritable) (bfs "Select drive:") Nothing + <*> areq textField (bfs "Use this directory on the drive:") + (Just $ T.pack gitAnnexAssistantDefaultDir) + where + pairs = zip (map describe drives) (map mountPoint drives) + describe drive = case diskFree drive of + Nothing -> mountPoint drive + Just free -> + let sz = roughSize storageUnits True free + in T.unwords + [ mountPoint drive + , T.concat ["(", T.pack sz] + , "free)" + ] + onlywritable = [whamlet|This list only includes drives you can write to.|] + +removableDriveRepository :: RemovableDrive -> FilePath +removableDriveRepository drive = + T.unpack (mountPoint drive) T.unpack (driveRepoPath drive) + +{- Adding a removable drive. -} +getAddDriveR :: Handler Html +getAddDriveR = postAddDriveR +postAddDriveR :: Handler Html +postAddDriveR = page "Add a removable drive" (Just Configuration) $ do + removabledrives <- liftIO driveList + writabledrives <- liftIO $ + filterM (canWrite . T.unpack . mountPoint) removabledrives + ((res, form), enctype) <- liftH $ runFormPostNoToken $ + selectDriveForm (sort writabledrives) + case res of + FormSuccess drive -> liftH $ redirect $ ConfirmAddDriveR drive + _ -> $(widgetFile "configurators/adddrive") + +{- The repo may already exist, when adding removable media + - that has already been used elsewhere. If so, check + - the UUID of the repo and see if it's one we know. If not, + - the user must confirm the repository merge. + - + - If the repo does not already exist on the drive, prompt about + - encryption. -} +getConfirmAddDriveR :: RemovableDrive -> Handler Html +getConfirmAddDriveR drive = ifM (liftIO $ probeRepoExists dir) + ( do + mu <- liftIO $ probeUUID dir + case mu of + Nothing -> maybe askcombine isknownuuid + =<< liftAnnex (probeGCryptRemoteUUID dir) + Just driveuuid -> isknownuuid driveuuid + , newrepo + ) + where + dir = removableDriveRepository drive + newrepo = do + cmd <- liftAnnex $ gpgCmd <$> Annex.getGitConfig + secretkeys <- sortBy (comparing snd) . M.toList + <$> liftIO (secretKeys cmd) + page "Encrypt repository?" (Just Configuration) $ + $(widgetFile "configurators/adddrive/encrypt") + knownrepo = getFinishAddDriveR drive NoRepoKey + askcombine = page "Combine repositories?" (Just Configuration) $ + $(widgetFile "configurators/adddrive/combine") + isknownuuid driveuuid = + ifM (M.member driveuuid <$> liftAnnex uuidMap) + ( knownrepo + , askcombine + ) + +setupDriveModal :: Widget +setupDriveModal = $(widgetFile "configurators/adddrive/setupmodal") + +getGenKeyForDriveR :: RemovableDrive -> Handler Html +getGenKeyForDriveR drive = withNewSecretKey $ \keyid -> + {- Generating a key takes a long time, and + - the removable drive may have been disconnected + - in the meantime. Check that it is still mounted + - before finishing. -} + ifM (liftIO $ any (\d -> mountPoint d == mountPoint drive) <$> driveList) + ( getFinishAddDriveR drive (RepoKey keyid) + , getAddDriveR + ) + +getFinishAddDriveR :: RemovableDrive -> RepoKey -> Handler Html +getFinishAddDriveR drive = go + where + go (RepoKey keyid) = whenGcryptInstalled $ makewith $ const $ do + r <- liftAnnex $ addRemote $ + makeGCryptRemote remotename dir keyid + return (Types.Remote.uuid r, r) + go NoRepoKey = checkGCryptRepoEncryption dir makeunencrypted makeunencrypted $ do + mu <- liftAnnex $ probeGCryptRemoteUUID dir + case mu of + Just u -> enableexistinggcryptremote u + Nothing -> giveup "The drive contains a gcrypt repository that is not a git-annex special remote. This is not supported." + enableexistinggcryptremote u = do + remotename' <- liftAnnex $ getGCryptRemoteName u dir + makewith $ const $ do + r <- liftAnnex $ addRemote $ + enableSpecialRemote remotename' GCrypt.remote Nothing $ M.fromList + [("gitrepo", dir)] + return (u, r) + {- Making a new unencrypted repo, or combining with an existing one. -} + makeunencrypted = makewith $ \isnew -> (,) + <$> liftIO (initRepo isnew False dir (Just remotename) Nothing) + <*> combineRepos dir remotename + makewith a = do + liftIO $ createDirectoryIfMissing True dir + isnew <- liftIO $ makeRepo dir True + {- Removable drives are not reliable media, so enable fsync. -} + liftIO $ inDir dir $ + setConfig (ConfigKey "core.fsyncobjectfiles") + (Git.Config.boolConfig True) + (u, r) <- a isnew + when isnew $ + liftAnnex $ defaultStandardGroup u TransferGroup + liftAssistant $ immediateSyncRemote r + redirect $ EditNewRepositoryR u + mountpoint = T.unpack (mountPoint drive) + dir = removableDriveRepository drive + remotename = takeFileName mountpoint + +{- Each repository is made a remote of the other. + - Next call syncRemote to get them in sync. -} +combineRepos :: FilePath -> String -> Handler Remote +combineRepos dir name = liftAnnex $ do + hostname <- fromMaybe "host" <$> liftIO getHostname + mylocation <- fromRepo Git.repoLocation + mypath <- liftIO $ relPathDirToFile dir mylocation + liftIO $ inDir dir $ void $ makeGitRemote hostname mypath + addRemote $ makeGitRemote name dir + +getEnableDirectoryR :: UUID -> Handler Html +getEnableDirectoryR uuid = page "Enable a repository" (Just Configuration) $ do + description <- liftAnnex $ T.pack <$> prettyUUID uuid + $(widgetFile "configurators/enabledirectory") + +{- List of removable drives. -} +driveList :: IO [RemovableDrive] +#ifdef mingw32_HOST_OS +-- Just enumerate all likely drive letters for Windows. +-- Could use wmic, but it only works for administrators. +driveList = mapM (\d -> genRemovableDrive $ d:":\\") ['A'..'Z'] +#else +driveList = mapM (genRemovableDrive . mnt_dir) =<< filter sane <$> getMounts + where + -- filter out some things that are surely not removable drives + sane Mntent { mnt_dir = dir, mnt_fsname = dev } + {- We want real disks like /dev/foo, not + - dummy mount points like proc or tmpfs or + - gvfs-fuse-daemon. -} + | not ('/' `elem` dev) = False + {- Just in case: These mount points are surely not + - removable disks. -} + | dir == "/" = False + | dir == "/tmp" = False + | dir == "/run/shm" = False + | dir == "/run/lock" = False +#ifdef __ANDROID__ + | dir == "/mnt/sdcard" = False + | dir == "/sdcard" = False +#endif + | otherwise = True +#endif + +genRemovableDrive :: FilePath -> IO RemovableDrive +genRemovableDrive dir = RemovableDrive + <$> getDiskFree dir + <*> pure (T.pack dir) + <*> pure (T.pack gitAnnexAssistantDefaultDir) + +{- Bootstraps from first run mode to a fully running assistant in a + - repository, by running the postFirstRun callback, which returns the + - url to the new webapp. -} +startFullAssistant :: FilePath -> StandardGroup -> Maybe (Annex ())-> Handler () +startFullAssistant path repogroup setup = do + webapp <- getYesod + url <- liftIO $ do + isnew <- makeRepo path False + void $ initRepo isnew True path Nothing (Just repogroup) + inDir path $ fromMaybe noop setup + addAutoStartFile path + setCurrentDirectory path + fromJust $ postFirstRun webapp + redirect $ T.pack url + +{- Checks if the user can write to a directory. + - + - The directory may be in the process of being created; if so + - the parent directory is checked instead. -} +canWrite :: FilePath -> IO Bool +canWrite dir = do + tocheck <- ifM (doesDirectoryExist dir) + (return dir, return $ parentDir dir) + catchBoolIO $ fileAccess tocheck False True False + +{- Gets the UUID of the git repo at a location, which may not exist, or + - not be a git-annex repo. -} +probeUUID :: FilePath -> IO (Maybe UUID) +probeUUID dir = catchDefaultIO Nothing $ inDir dir $ do + u <- getUUID + return $ if u == NoUUID then Nothing else Just u diff --git a/Assistant/WebApp/Configurators/Pairing.hs b/Assistant/WebApp/Configurators/Pairing.hs new file mode 100644 index 0000000000..5fcc42b28b --- /dev/null +++ b/Assistant/WebApp/Configurators/Pairing.hs @@ -0,0 +1,321 @@ +{- git-annex assistant webapp configurator for pairing + - + - Copyright 2012,2016 Joey Hess + - + - Licensed under the GNU AGPL version 3 or higher. + -} + +{-# LANGUAGE TypeFamilies, QuasiQuotes, TemplateHaskell, OverloadedStrings, FlexibleContexts #-} +{-# LANGUAGE CPP #-} + +module Assistant.WebApp.Configurators.Pairing where + +import Assistant.Pairing +import Assistant.WebApp.Common +import Annex.UUID +#ifdef WITH_PAIRING +import Assistant.DaemonStatus +import Assistant.Pairing.MakeRemote +import Assistant.Pairing.Network +import Assistant.Ssh +import Utility.Verifiable +#endif +import Utility.UserInfo +import Utility.Tor +import Utility.Su +import Assistant.WebApp.Pairing +import Assistant.Alert +import qualified Utility.MagicWormhole as Wormhole +import Assistant.MakeRemote +import Assistant.RemoteControl +import Assistant.Sync +import Assistant.WebApp.SideBar +import Command.P2P (unusedPeerRemoteName, PairingResult(..)) +import P2P.Address +import Git +import Config.Files +import Utility.Process.Transcript + +import qualified Data.Map as M +import qualified Data.Text as T +#ifdef WITH_PAIRING +import qualified Data.Text.Encoding as T +import qualified Data.ByteString as B +import Data.Char +import qualified Control.Exception as E +import Control.Concurrent +#endif +import Control.Concurrent.STM hiding (check) + +getStartWormholePairFriendR :: Handler Html +getStartWormholePairFriendR = startWormholePairR PairingWithFriend + +getStartWormholePairSelfR :: Handler Html +getStartWormholePairSelfR = startWormholePairR PairingWithSelf + +startWormholePairR :: PairingWith -> Handler Html +startWormholePairR pairingwith = whenTorInstalled $ whenWormholeInstalled $ + pairPage $ do + sucommand <- liftIO $ mkSuCommand "git-annex" [Param "enable-tor"] + $(widgetFile "configurators/pairing/wormhole/start") + +getPrepareWormholePairR :: PairingWith -> Handler Html +getPrepareWormholePairR pairingwith = do + enableTor + myaddrs <- liftAnnex loadP2PAddresses + remotename <- liftAnnex unusedPeerRemoteName + h <- liftAssistant $ + startWormholePairing pairingwith remotename myaddrs + i <- liftIO . addWormholePairingState h + =<< wormholePairingState <$> getYesod + redirect $ RunningWormholePairR i + +enableTor :: Handler () +enableTor = do + gitannex <- liftIO readProgramFile + (transcript, ok) <- liftIO $ processTranscript gitannex ["enable-tor"] Nothing + if ok + -- Reload remotedameon so it's serving the tor hidden + -- service. + then liftAssistant $ sendRemoteControl RELOAD + else giveup $ "Failed to enable tor\n\n" ++ transcript + +getRunningWormholePairR :: WormholePairingId -> Handler Html +getRunningWormholePairR = runningWormholePairR + +postRunningWormholePairR :: WormholePairingId -> Handler Html +postRunningWormholePairR = runningWormholePairR + +runningWormholePairR :: WormholePairingId -> Handler Html +runningWormholePairR i = go =<< getWormholePairingHandle i + where + go Nothing = redirect StartWormholePairFriendR + go (Just h) = pairPage $ withPairingWith h $ \pairingwith -> do + ourcode <- liftIO $ getOurWormholeCode h + let codeprompt = case pairingwith of + PairingWithFriend -> "Your friend's pairing code" + PairingWithSelf -> "The other device's pairing code" + ((result, form), enctype) <- liftH $ + runFormPostNoToken $ renderBootstrap3 bootstrapFormLayout $ + areq (checkwormholecode ourcode pairingwith textField) (bfs codeprompt) Nothing + case result of + FormSuccess t -> case Wormhole.toCode (T.unpack t) of + Nothing -> giveup invalidcode + Just theircode -> finish h theircode + _ -> showform form enctype ourcode pairingwith + + showform form enctype ourcode pairingwith = + $(widgetFile "configurators/pairing/wormhole/prompt") + + checkwormholecode ourcode pairingwith = check $ \t -> + case Wormhole.toCode (T.unpack t) of + Nothing -> Left (T.pack invalidcode) + Just theircode + | theircode == ourcode -> Left $ + case pairingwith of + PairingWithSelf -> "Oops -- You entered this repository's pairing code. Enter the pairing code of the *other* repository." + PairingWithFriend -> "Oops -- You entered your pairing code. Enter your friend's pairing code." + | otherwise -> Right t + + invalidcode = "That does not look like a valid pairing code. Try again..." + + finish h theircode = do + void $ liftIO $ sendTheirWormholeCode h theircode + res <- liftAssistant $ finishWormholePairing h + case res of + SendFailed -> giveup "Failed sending data to pair." + ReceiveFailed -> giveup "Failed receiving data from pair." + LinkFailed e -> giveup $ "Failed linking to pair: " ++ e + PairSuccess -> withRemoteName h $ \remotename -> do + r <- liftAnnex $ addRemote (return remotename) + liftAssistant $ syncRemote r + liftAssistant $ sendRemoteControl RELOAD + redirect DashboardR + +getWormholePairingHandle :: WormholePairingId -> Handler (Maybe WormholePairingHandle) +getWormholePairingHandle i = do + s <- wormholePairingState <$> getYesod + liftIO $ atomically $ M.lookup i <$> readTVar s + +whenTorInstalled :: Handler Html -> Handler Html +whenTorInstalled a = ifM (liftIO torIsInstalled) + ( a + , page "Need Tor" (Just Configuration) $ + $(widgetFile "configurators/needtor") + ) + +whenWormholeInstalled :: Handler Html -> Handler Html +whenWormholeInstalled a = ifM (liftIO Wormhole.isInstalled) + ( a + , page "Need Magic Wormhole" (Just Configuration) $ + $(widgetFile "configurators/needmagicwormhole") + ) + +{- Starts local pairing. -} +getStartLocalPairR :: Handler Html +getStartLocalPairR = postStartLocalPairR +postStartLocalPairR :: Handler Html +#ifdef WITH_PAIRING +postStartLocalPairR = promptSecret Nothing $ + startLocalPairing PairReq noop pairingAlert Nothing +#else +postStartLocalPairR = noLocalPairing + +noLocalPairing :: Handler Html +noLocalPairing = noPairing "local" +#endif + +{- Runs on the system that responds to a local pair request; sets up the ssh + - authorized key first so that the originating host can immediately sync + - with us. -} +getFinishLocalPairR :: PairMsg -> Handler Html +getFinishLocalPairR = postFinishLocalPairR +postFinishLocalPairR :: PairMsg -> Handler Html +#ifdef WITH_PAIRING +postFinishLocalPairR msg = promptSecret (Just msg) $ \_ secret -> do + repodir <- liftH $ repoPath <$> liftAnnex gitRepo + liftIO $ setup repodir + startLocalPairing PairAck (cleanup repodir) alert uuid "" secret + where + alert = pairRequestAcknowledgedAlert (pairRepo msg) . Just + setup repodir = setupAuthorizedKeys msg repodir + cleanup repodir = removeAuthorizedKeys True repodir $ + remoteSshPubKey $ pairMsgData msg + uuid = Just $ pairUUID $ pairMsgData msg +#else +postFinishLocalPairR _ = noLocalPairing +#endif + +getRunningLocalPairR :: SecretReminder -> Handler Html +#ifdef WITH_PAIRING +getRunningLocalPairR s = pairPage $ do + let secret = fromSecretReminder s + $(widgetFile "configurators/pairing/local/inprogress") +#else +getRunningLocalPairR _ = noLocalPairing +#endif + +#ifdef WITH_PAIRING + +{- Starts local pairing, at either the PairReq (initiating host) or + - PairAck (responding host) stage. + - + - Displays an alert, and starts a thread sending the pairing message, + - which will continue running until the other host responds, or until + - canceled by the user. If canceled by the user, runs the oncancel action. + - + - Redirects to the pairing in progress page. + -} +startLocalPairing :: PairStage -> IO () -> (AlertButton -> Alert) -> Maybe UUID -> Text -> Secret -> Widget +startLocalPairing stage oncancel alert muuid displaysecret secret = do + urlrender <- liftH getUrlRender + reldir <- fromJust . relDir <$> liftH getYesod + + sendrequests <- liftAssistant $ asIO2 $ mksendrequests urlrender + {- Generating a ssh key pair can take a while, so do it in the + - background. -} + thread <- liftAssistant $ asIO $ do + keypair <- liftIO $ genSshKeyPair + let pubkey = either error id $ validateSshPubKey $ sshPubKey keypair + pairdata <- liftIO $ PairData + <$> getHostname + <*> (either error id <$> myUserName) + <*> pure reldir + <*> pure pubkey + <*> (maybe genUUID return muuid) + let sender = multicastPairMsg Nothing secret pairdata + let pip = PairingInProgress secret Nothing keypair pairdata stage + startSending pip stage $ sendrequests sender + void $ liftIO $ forkIO thread + + liftH $ redirect $ RunningLocalPairR $ toSecretReminder displaysecret + where + {- Sends pairing messages until the thread is killed, + - and shows an activity alert while doing it. + - + - The cancel button returns the user to the DashboardR. This is + - not ideal, but they have to be sent somewhere, and could + - have been on a page specific to the in-process pairing + - that just stopped, so can't go back there. + -} + mksendrequests urlrender sender _stage = do + tid <- liftIO myThreadId + let selfdestruct = AlertButton + { buttonLabel = "Cancel" + , buttonPrimary = True + , buttonUrl = urlrender DashboardR + , buttonAction = Just $ const $ do + oncancel + killThread tid + } + alertDuring (alert selfdestruct) $ liftIO $ do + _ <- E.try (sender stage) :: IO (Either E.SomeException ()) + return () + +data InputSecret = InputSecret { secretText :: Maybe Text } + +{- If a PairMsg is passed in, ensures that the user enters a secret + - that can validate it. -} +promptSecret :: Maybe PairMsg -> (Text -> Secret -> Widget) -> Handler Html +promptSecret msg cont = pairPage $ do + ((result, form), enctype) <- liftH $ + runFormPostNoToken $ renderBootstrap3 bootstrapFormLayout $ + InputSecret <$> aopt textField (bfs "Secret phrase") Nothing + case result of + FormSuccess v -> do + let rawsecret = fromMaybe "" $ secretText v + let secret = toSecret rawsecret + case msg of + Nothing -> case secretProblem secret of + Nothing -> cont rawsecret secret + Just problem -> + showform form enctype $ Just problem + Just m -> + if verify (fromPairMsg m) secret + then cont rawsecret secret + else showform form enctype $ Just + "That's not the right secret phrase." + _ -> showform form enctype Nothing + where + showform form enctype mproblem = do + let start = isNothing msg + let badphrase = isJust mproblem + let problem = fromMaybe "" mproblem + let (username, hostname) = maybe ("", "") + (\(_, v, a) -> (T.pack $ remoteUserName v, T.pack $ fromMaybe (showAddr a) (remoteHostName v))) + (verifiableVal . fromPairMsg <$> msg) + u <- liftIO myUserName + let sameusername = Right username == (T.pack <$> u) + $(widgetFile "configurators/pairing/local/prompt") + +{- This counts unicode characters as more than one character, + - but that's ok; they *do* provide additional entropy. -} +secretProblem :: Secret -> Maybe Text +secretProblem s + | B.null s = Just "The secret phrase cannot be left empty. (Remember that punctuation and white space is ignored.)" + | B.length s < 6 = Just "Enter a longer secret phrase, at least 6 characters, but really, a phrase is best! This is not a password you'll need to enter every day." + | s == toSecret sampleQuote = Just "Speaking of foolishness, don't paste in the example I gave. Enter a different phrase, please!" + | otherwise = Nothing + +toSecret :: Text -> Secret +toSecret s = T.encodeUtf8 $ T.toLower $ T.filter isAlphaNum s + +{- From Dickens -} +sampleQuote :: Text +sampleQuote = T.unwords + [ "It was the best of times," + , "it was the worst of times," + , "it was the age of wisdom," + , "it was the age of foolishness." + ] + +#else + +#endif + +pairPage :: Widget -> Handler Html +pairPage = page "Pairing" (Just Configuration) + +noPairing :: Text -> Handler Html +noPairing pairingtype = pairPage $ + $(widgetFile "configurators/pairing/disabled") diff --git a/Assistant/WebApp/Configurators/Preferences.hs b/Assistant/WebApp/Configurators/Preferences.hs new file mode 100644 index 0000000000..54b4add376 --- /dev/null +++ b/Assistant/WebApp/Configurators/Preferences.hs @@ -0,0 +1,122 @@ +{- git-annex assistant general preferences + - + - Copyright 2013 Joey Hess + - + - Licensed under the GNU AGPL version 3 or higher. + -} + +{-# LANGUAGE QuasiQuotes, TemplateHaskell, OverloadedStrings #-} + +module Assistant.WebApp.Configurators.Preferences ( + getPreferencesR, + postPreferencesR +) where + +import Assistant.WebApp.Common +import qualified Annex +import qualified Git +import Config +import Config.Files +import Annex.NumCopies +import Utility.DataUnits +import Git.Config +import Types.Distribution +import qualified BuildInfo + +import qualified Data.Text as T + +data PrefsForm = PrefsForm + { diskReserve :: Text + , numCopies :: Int + , autoStart :: Bool + , autoUpgrade :: AutoUpgrade + , enableDebug :: Bool + } + +prefsAForm :: PrefsForm -> MkAForm PrefsForm +prefsAForm d = PrefsForm + <$> areq (storageField `withNote` diskreservenote) + (bfs "Disk reserve") (Just $ diskReserve d) + <*> areq (positiveIntField `withNote` numcopiesnote) + (bfs "Number of copies") (Just $ numCopies d) + <*> areq (checkBoxField `withNote` autostartnote) + "Auto start" (Just $ autoStart d) + <*> areq (selectFieldList autoUpgradeChoices) + (bfs autoUpgradeLabel) (Just $ autoUpgrade d) + <*> areq (checkBoxField `withNote` debugnote) + "Enable debug logging" (Just $ enableDebug d) + where + diskreservenote = [whamlet|
    Avoid downloading files from other repositories when there is too little free disk space.|] + numcopiesnote = [whamlet|
    Only drop a file after verifying that other repositories contain this many copies.|] + debugnote = [whamlet|    View Log|] + autostartnote = [whamlet|Start the git-annex assistant at boot or on login.|] + + autoUpgradeChoices :: [(Text, AutoUpgrade)] + autoUpgradeChoices = + [ ("ask me", AskUpgrade) + , ("enabled", AutoUpgrade) + , ("disabled", NoAutoUpgrade) + ] + autoUpgradeLabel + | isJust BuildInfo.upgradelocation = "Auto upgrade" + | otherwise = "Auto restart on upgrade" + + positiveIntField = check isPositive intField + where + isPositive i + | i > 0 = Right i + | otherwise = Left notPositive + notPositive :: Text + notPositive = "This should be 1 or more!" + + storageField = check validStorage textField + where + validStorage t + | T.null t = Right t + | otherwise = case readSize dataUnits $ T.unpack t of + Nothing -> Left badParse + Just _ -> Right t + badParse :: Text + badParse = "Parse error. Expected something like \"100 megabytes\" or \"2 gb\"" + +getPrefs :: Annex PrefsForm +getPrefs = PrefsForm + <$> (T.pack . roughSize storageUnits False . annexDiskReserve <$> Annex.getGitConfig) + <*> (fromNumCopies <$> getNumCopies) + <*> inAutoStartFile + <*> (annexAutoUpgrade <$> Annex.getGitConfig) + <*> (annexDebug <$> Annex.getGitConfig) + +storePrefs :: PrefsForm -> Annex () +storePrefs p = do + setConfig (annexConfig "diskreserve") (T.unpack $ diskReserve p) + setGlobalNumCopies (NumCopies $ numCopies p) + unsetConfig (annexConfig "numcopies") -- deprecated + setConfig (annexConfig "autoupgrade") (fromAutoUpgrade $ autoUpgrade p) + unlessM ((==) <$> pure (autoStart p) <*> inAutoStartFile) $ do + here <- fromRepo Git.repoPath + liftIO $ if autoStart p + then addAutoStartFile here + else removeAutoStartFile here + setConfig (annexConfig "debug") (boolConfig $ enableDebug p) + liftIO $ if enableDebug p + then enableDebugOutput + else disableDebugOutput + +getPreferencesR :: Handler Html +getPreferencesR = postPreferencesR +postPreferencesR :: Handler Html +postPreferencesR = page "Preferences" (Just Configuration) $ do + ((result, form), enctype) <- liftH $ do + current <- liftAnnex getPrefs + runFormPostNoToken $ renderBootstrap3 bootstrapFormLayout $ prefsAForm current + case result of + FormSuccess new -> liftH $ do + liftAnnex $ storePrefs new + redirect ConfigurationR + _ -> $(widgetFile "configurators/preferences") + +inAutoStartFile :: Annex Bool +inAutoStartFile = do + here <- liftIO . absPath =<< fromRepo Git.repoPath + any (`equalFilePath` here) <$> liftIO readAutoStartFile diff --git a/Assistant/WebApp/Configurators/Ssh.hs b/Assistant/WebApp/Configurators/Ssh.hs new file mode 100644 index 0000000000..52e8247aa1 --- /dev/null +++ b/Assistant/WebApp/Configurators/Ssh.hs @@ -0,0 +1,789 @@ +{- git-annex assistant webapp configurator for ssh-based remotes + - + - Copyright 2012-2015 Joey Hess + - + - Licensed under the GNU AGPL version 3 or higher. + -} + +{-# LANGUAGE QuasiQuotes, TemplateHaskell, OverloadedStrings #-} +{-# LANGUAGE CPP, FlexibleContexts #-} + +module Assistant.WebApp.Configurators.Ssh where + +import Assistant.WebApp.Common +import Assistant.WebApp.Gpg +import Assistant.Ssh +import Annex.Ssh +import Assistant.WebApp.MakeRemote +import Logs.Remote +import Remote +import Types.StandardGroups +import Utility.UserInfo +import Utility.Gpg +import Types.Remote (RemoteConfig) +import Git.Types (RemoteName, fromRef) +import qualified Remote.GCrypt as GCrypt +import qualified Annex +import qualified Git.Construct +import qualified Git.Config +import qualified Git.Command +import qualified Remote.Helper.Ssh +import qualified Annex.Branch +import Annex.UUID +import Logs.UUID +import Assistant.RemoteControl +import Types.Creds +import Assistant.CredPairCache +import Annex.Path +import Utility.Tmp +import Utility.FileMode +import Utility.ThreadScheduler +import Utility.Env +import Utility.SshHost +import Utility.Process.Transcript + +import qualified Data.Text as T +import qualified Data.Map as M +import Network.Socket +import Data.Ord + +sshConfigurator :: Widget -> Handler Html +sshConfigurator = page "Add a remote server" (Just Configuration) + +data SshInput = SshInput + { inputHostname :: Maybe Text + , inputUsername :: Maybe Text + , inputAuthMethod :: AuthMethod + , inputPassword :: Maybe Text + , inputDirectory :: Maybe Text + , inputPort :: Int + } + +data AuthMethod + = Password + | CachedPassword + | ExistingSshKey + deriving (Eq, Show) + +-- Is a repository a new one that's being created, or did it already exist +-- and is just being added. +data RepoStatus = NewRepo | ExistingRepo + +{- SshInput is only used for applicative form prompting, this converts + - the result of such a form into a SshData. -} +mkSshData :: SshInput -> SshData +mkSshData s = SshData + { sshHostName = fromMaybe "" $ inputHostname s + , sshUserName = inputUsername s + , sshDirectory = fromMaybe "" $ inputDirectory s + , sshRepoName = genSshRepoName + (T.unpack $ fromJust $ inputHostname s) + (maybe "" T.unpack $ inputDirectory s) + , sshPort = inputPort s + , needsPubKey = False + , sshCapabilities = [] -- untested + , sshRepoUrl = Nothing + } + +mkSshInput :: SshData -> SshInput +mkSshInput s = SshInput + { inputHostname = Just $ sshHostName s + , inputUsername = sshUserName s + , inputAuthMethod = if needsPubKey s then CachedPassword else ExistingSshKey + , inputPassword = Nothing + , inputDirectory = Just $ sshDirectory s + , inputPort = sshPort s + } + +sshInputAForm :: Field Handler Text -> SshInput -> AForm Handler SshInput +sshInputAForm hostnamefield d = normalize <$> gen + where + gen = SshInput + <$> aopt check_hostname (bfs "Host name") (Just $ inputHostname d) + <*> aopt check_username (bfs "User name") (Just $ inputUsername d) + <*> areq (selectFieldList authmethods) (bfs "Authenticate with") (Just $ inputAuthMethod d) + <*> aopt passwordField (bfs "Password") Nothing + <*> aopt textField (bfs "Directory") (Just $ Just $ fromMaybe (T.pack gitAnnexAssistantDefaultDir) $ inputDirectory d) + <*> areq intField (bfs "Port") (Just $ inputPort d) + + authmethods :: [(Text, AuthMethod)] + authmethods = + [ ("password", Password) + , ("existing ssh key", ExistingSshKey) + ] + + check_username = checkBool (all (`notElem` ("/:@ \t" :: String)) . T.unpack) + bad_username textField + + bad_username = "bad user name" :: Text +#ifndef __ANDROID__ + bad_hostname = "cannot resolve host name" :: Text + + check_hostname = checkM (liftIO . checkdns) hostnamefield + checkdns t = do + let h = T.unpack t + let canonname = Just $ defaultHints { addrFlags = [AI_CANONNAME] } + r <- catchMaybeIO $ getAddrInfo canonname (Just h) Nothing + return $ case mapMaybe addrCanonName <$> r of + -- canonicalize input hostname if it had no dot + Just (fullname:_) + | '.' `elem` h -> Right t + | otherwise -> Right $ T.pack fullname + Just [] -> Right t + Nothing -> Left bad_hostname +#else + -- getAddrInfo currently broken on Android + check_hostname = hostnamefield -- unchecked +#endif + + -- The directory is implicitly in home, so remove any leading ~/ + normalize i = i { inputDirectory = normalizedir <$> inputDirectory i } + normalizedir dir + | "~/" `T.isPrefixOf` dir = T.drop 2 dir + | "/~/" `T.isPrefixOf` dir = T.drop 3 dir + | otherwise = dir + +data ServerStatus + = UntestedServer + | UnusableServer Text -- reason why it's not usable + | ServerNeedsPubKey SshPubKey + | UsableServer [SshServerCapability] + deriving (Eq) + +capabilities :: ServerStatus -> [SshServerCapability] +capabilities (UsableServer cs) = cs +capabilities _ = [] + +getAddSshR :: Handler Html +getAddSshR = postAddSshR +postAddSshR :: Handler Html +postAddSshR = sshConfigurator $ do + username <- liftIO $ either (const Nothing) (Just . T.pack) <$> myUserName + ((result, form), enctype) <- liftH $ + runFormPostNoToken $ renderBootstrap3 bootstrapFormLayout $ sshInputAForm textField $ + SshInput Nothing username Password Nothing Nothing 22 + case result of + FormSuccess sshinput -> do + s <- liftAssistant $ testServer sshinput + case s of + Left status -> showform form enctype status + Right (sshdata, u) -> liftH $ redirect $ ConfirmSshR sshdata u + _ -> showform form enctype UntestedServer + where + showform form enctype status = $(widgetFile "configurators/ssh/add") + +sshTestModal :: Widget +sshTestModal = $(widgetFile "configurators/ssh/testmodal") + +sshSetupModal :: SshData -> Widget +sshSetupModal sshdata = $(widgetFile "configurators/ssh/setupmodal") + +getEnableRsyncR :: UUID -> Handler Html +getEnableRsyncR = postEnableRsyncR +postEnableRsyncR :: UUID -> Handler Html +postEnableRsyncR = enableSshRemote getsshinput enableRsyncNet enablersync + where + enablersync sshdata u = redirect $ ConfirmSshR + (sshdata { sshCapabilities = [RsyncCapable] }) u + getsshinput = parseSshUrl <=< M.lookup "rsyncurl" + +{- This only handles gcrypt repositories that are located on ssh servers; + - ones on local drives are handled via another part of the UI. -} +getEnableSshGCryptR :: UUID -> Handler Html +getEnableSshGCryptR = postEnableSshGCryptR +postEnableSshGCryptR :: UUID -> Handler Html +postEnableSshGCryptR u = whenGcryptInstalled $ + enableSshRemote getsshinput enableRsyncNetGCrypt enablegcrypt u + where + enablegcrypt sshdata _ = prepSsh False sshdata $ \sshdata' -> + sshConfigurator $ + checkExistingGCrypt sshdata' $ + giveup "Expected to find an encrypted git repository, but did not." + getsshinput = parseSshUrl <=< M.lookup "gitrepo" + +getEnableSshGitRemoteR :: UUID -> Handler Html +getEnableSshGitRemoteR = postEnableSshGitRemoteR +postEnableSshGitRemoteR :: UUID -> Handler Html +postEnableSshGitRemoteR = enableSshRemote getsshinput enableRsyncNet enablesshgitremote + where + enablesshgitremote sshdata u = redirect $ ConfirmSshR sshdata u + + getsshinput = parseSshUrl <=< M.lookup "location" + +{- To enable a remote that uses ssh as its transport, + - parse a config key to get its url, and display a form + - to prompt for its password. + -} +enableSshRemote :: (RemoteConfig -> Maybe SshData) -> (SshInput -> RemoteName -> Handler Html) -> (SshData -> UUID -> Handler Html) -> UUID -> Handler Html +enableSshRemote getsshdata rsyncnetsetup genericsetup u = do + m <- fromMaybe M.empty . M.lookup u <$> liftAnnex readRemoteLog + case (unmangle <$> getsshdata m, M.lookup "name" m) of + (Just sshdata, Just reponame) + | isGitLab sshdata -> enableGitLab sshdata + | otherwise -> sshConfigurator $ do + ((result, form), enctype) <- liftH $ + runFormPostNoToken $ renderBootstrap3 bootstrapFormLayout $ + sshInputAForm textField $ mkSshInput sshdata + case result of + FormSuccess sshinput + | isRsyncNet (inputHostname sshinput) -> + void $ liftH $ rsyncnetsetup sshinput reponame + | otherwise -> do + s <- liftAssistant $ testServer sshinput + case s of + Left status -> showform form enctype status + Right (sshdata', _u) -> void $ liftH $ genericsetup + ( sshdata' { sshRepoName = reponame } ) u + _ -> showform form enctype UntestedServer + _ -> redirect AddSshR + where + unmangle sshdata = sshdata + { sshHostName = T.pack $ unMangleSshHostName $ + T.unpack $ sshHostName sshdata + } + showform form enctype status = do + description <- liftAnnex $ T.pack <$> prettyUUID u + $(widgetFile "configurators/ssh/enable") + +{- To deal with git-annex and possibly even git and rsync not being + - available in the remote server's PATH, when git-annex was installed + - from the standalone tarball etc, look for a ~/.ssh/git-annex-wrapper + - and if it's there, use it to run a command. -} +wrapCommand :: String -> String +wrapCommand cmd = "if [ -x " ++ commandWrapper ++ " ]; then " ++ commandWrapper ++ " " ++ cmd ++ "; else " ++ cmd ++ "; fi" + +commandWrapper :: String +commandWrapper = "~/.ssh/git-annex-wrapper" + +{- Test if we can ssh into the server, using the specified AuthMethod. + - + - Once logged into the server, probe to see if git-annex-shell, + - git, and rsync are available. + - + - Note that ~/.ssh/git-annex-shell may be present, while + - git-annex-shell is not in PATH. + - Also, git and rsync may not be in PATH; as long as the commandWrapper + - is present, assume it is able to be used to run them. + - + - Also probe to see if there is already a git repository at the location + - with either an annex-uuid or a gcrypt-id set. (If not, returns NoUUID.) + -} +testServer :: SshInput -> Assistant (Either ServerStatus (SshData, UUID)) +testServer (SshInput { inputHostname = Nothing }) = return $ + Left $ UnusableServer "Please enter a host name." +testServer sshinput@(SshInput { inputHostname = Just hn }) = do + (status, u) <- probe + case capabilities status of + [] -> return $ Left status + cs -> do + let sshdata = (mkSshData sshinput) + { needsPubKey = inputAuthMethod sshinput /= ExistingSshKey + , sshCapabilities = cs + } + return $ Right (sshdata, u) + where + probe = do + let remotecommand = shellWrap $ intercalate ";" + [ report "loggedin" + , checkcommand "git-annex-shell" + , checkcommand "git" + , checkcommand "rsync" + , checkcommand shim + , checkcommand commandWrapper + , getgitconfig (T.unpack <$> inputDirectory sshinput) + ] + knownhost <- liftIO $ knownHost hn + let sshopts = + {- If this is an already known host, let + - ssh check it as usual. + - Otherwise, trust the host key. -} + [ sshOpt "StrictHostKeyChecking" $ + if knownhost then "yes" else "no" + , "-n" -- don't read from stdin + , "-p", show (inputPort sshinput) + ] + let sshhost = genSshHost + (fromJust $ inputHostname sshinput) + (inputUsername sshinput) + parsetranscript . fst <$> sshAuthTranscript sshinput sshopts sshhost remotecommand Nothing + parsetranscript s = + let cs = map snd $ filter (reported . fst) + [ ("git-annex-shell", GitAnnexShellCapable) + , (shim, GitAnnexShellCapable) + , ("git", GitCapable) + , ("rsync", RsyncCapable) + , (commandWrapper, GitCapable) + , (commandWrapper, RsyncCapable) + ] + u = fromMaybe NoUUID $ headMaybe $ mapMaybe finduuid $ + map (separate (== '=')) $ lines s + in if null cs + then (UnusableServer unusablereason, u) + else (UsableServer cs, u) + where + reported r = token r `isInfixOf` s + unusablereason = if reported "loggedin" + then "Neither rsync nor git-annex are installed on the server. Perhaps you should go install them?" + else T.pack $ "Failed to ssh to the server. Transcript: " ++ s + finduuid (k, v) + | k == "annex.uuid" = Just $ toUUID v + | k == GCrypt.coreGCryptId = Just $ genUUIDInNameSpace gCryptNameSpace v + | otherwise = Nothing + + checkcommand c = "if which " ++ c ++ "; then " ++ report c ++ "; fi" + token r = "git-annex-probe " ++ r + report r = "echo " ++ shellEscape (token r) + shim = "~/.ssh/git-annex-shell" + getgitconfig (Just d) + | not (null d) = "cd " ++ shellEscape d ++ " && git config --list" + getgitconfig _ = "echo" + +{- Runs a ssh command to set up the repository; if it fails shows + - the user the transcript, and if it succeeds, runs an action. -} +sshSetup :: SshInput -> [String] -> SshHost -> String -> Maybe String -> Handler Html -> Handler Html +sshSetup sshinput opts sshhost cmd input a = do + (transcript, ok) <- liftAssistant $ sshAuthTranscript sshinput opts sshhost cmd input + if ok + then do + liftAssistant $ expireCachedCred $ getLogin sshinput + a + else sshErr sshinput transcript + +sshErr :: SshInput -> String -> Handler Html +sshErr sshinput msg + | inputAuthMethod sshinput == CachedPassword = + ifM (liftAssistant $ isNothing <$> getCachedCred (getLogin sshinput)) + ( sshConfigurator $ + $(widgetFile "configurators/ssh/expiredpassword") + , showerr + ) + | otherwise = showerr + where + showerr = sshConfigurator $ + $(widgetFile "configurators/ssh/error") + +{- Runs a ssh command, returning a transcript of its output. + - + - Depending on the SshInput, avoids using a password, or uses a + - cached password. ssh is coaxed to use git-annex as SSH_ASKPASS + - to get the password. + -} +sshAuthTranscript :: SshInput -> [String] -> SshHost -> String -> (Maybe String) -> Assistant (String, Bool) +sshAuthTranscript sshinput opts sshhost cmd input = case inputAuthMethod sshinput of + ExistingSshKey -> liftIO $ go [passwordprompts 0] Nothing + CachedPassword -> setupAskPass + Password -> do + cacheCred (login, geti inputPassword) (Seconds $ 60 * 10) + setupAskPass + where + login = getLogin sshinput + geti f = maybe "" T.unpack (f sshinput) + + go extraopts environ = processTranscript' + (askPass environ (proc "ssh" (extraopts ++ opts ++ [fromSshHost sshhost, cmd]))) + -- Always provide stdin, even when empty. + (Just (fromMaybe "" input)) + + {- ssh will only use SSH_ASKPASS when DISPLAY is set and there + - is no controlling terminal. -} + askPass environ p = p + { env = environ +#if MIN_VERSION_process(1,3,0) + , detach_console = True + , new_session = True +#endif + } + + setupAskPass = do + program <- liftIO programPath + v <- getCachedCred login + liftIO $ case v of + Nothing -> go [passwordprompts 0] Nothing + Just pass -> withTmpFile "ssh" $ \passfile h -> do + hClose h + writeFileProtected passfile pass + environ <- getEnvironment + let environ' = addEntries + [ ("SSH_ASKPASS", program) + , (sshAskPassEnv, passfile) + , ("DISPLAY", ":0") + ] environ + go [passwordprompts 1] (Just environ') + + passwordprompts :: Int -> String + passwordprompts = sshOpt "NumberOfPasswordPrompts" . show + +getLogin :: SshInput -> Login +getLogin sshinput = geti inputUsername ++ "@" ++ geti inputHostname + where + geti f = maybe "" T.unpack (f sshinput) + +{- The UUID will be NoUUID when the repository does not already exist, + - or was not a git-annex repository before. -} +getConfirmSshR :: SshData -> UUID -> Handler Html +getConfirmSshR sshdata u + | u == NoUUID = handlenew + | otherwise = handleexisting =<< (M.lookup u <$> liftAnnex uuidMap) + where + handlenew = sshConfigurator $ do + cmd <- liftAnnex $ gpgCmd <$> Annex.getGitConfig + secretkeys <- sortBy (comparing snd) . M.toList + <$> liftIO (secretKeys cmd) + $(widgetFile "configurators/ssh/confirm") + handleexisting Nothing = sshConfigurator $ + -- Not a UUID we know, so prompt about combining. + $(widgetFile "configurators/ssh/combine") + handleexisting (Just _) = prepSsh False sshdata $ \sshdata' -> do + m <- liftAnnex readRemoteLog + case M.lookup "type" =<< M.lookup u m of + Just "gcrypt" -> combineExistingGCrypt sshdata' u + _ -> makeSshRepo ExistingRepo sshdata' + +{- The user has confirmed they want to combine with a ssh repository, + - which is not known to us. So it might be using gcrypt. -} +getCombineSshR :: SshData -> Handler Html +getCombineSshR sshdata = prepSsh False sshdata $ \sshdata' -> + sshConfigurator $ + checkExistingGCrypt sshdata' $ + void $ liftH $ makeSshRepo ExistingRepo sshdata' + +getRetrySshR :: SshData -> Handler () +getRetrySshR sshdata = do + s <- liftAssistant $ testServer $ mkSshInput sshdata + redirect $ either (const $ ConfirmSshR sshdata NoUUID) (uncurry ConfirmSshR) s + +{- Making a new git repository. -} +getMakeSshGitR :: SshData -> Handler Html +getMakeSshGitR sshdata = prepSsh True sshdata (makeSshRepo NewRepo) + +getMakeSshRsyncR :: SshData -> Handler Html +getMakeSshRsyncR sshdata = prepSsh False (rsyncOnly sshdata) (makeSshRepo NewRepo) + +rsyncOnly :: SshData -> SshData +rsyncOnly sshdata = sshdata { sshCapabilities = [RsyncCapable] } + +getMakeSshGCryptR :: SshData -> RepoKey -> Handler Html +getMakeSshGCryptR sshdata NoRepoKey = whenGcryptInstalled $ + withNewSecretKey $ getMakeSshGCryptR sshdata . RepoKey +getMakeSshGCryptR sshdata (RepoKey keyid) = whenGcryptInstalled $ + prepSsh False sshdata $ makeGCryptRepo NewRepo keyid + +{- Detect if the user entered a location with an existing, known + - gcrypt repository, and enable it. Otherwise, runs the action. -} +checkExistingGCrypt :: SshData -> Widget -> Widget +checkExistingGCrypt sshdata nope = checkGCryptRepoEncryption repourl nope nope $ do + mu <- liftAnnex $ probeGCryptRemoteUUID repourl + case mu of + Just u -> void $ liftH $ + combineExistingGCrypt sshdata u + Nothing -> giveup "The location contains a gcrypt repository that is not a git-annex special remote. This is not supported." + where + repourl = genSshUrl sshdata + +{- Enables an existing gcrypt special remote. -} +enableGCrypt :: SshData -> RemoteName -> Handler Html +enableGCrypt sshdata reponame = setupRemote postsetup Nothing Nothing mk + where + mk = enableSpecialRemote reponame GCrypt.remote Nothing $ + M.fromList [("gitrepo", genSshUrl sshdata)] + postsetup _ = redirect DashboardR + +{- Combining with a gcrypt repository that may not be + - known in remote.log, so probe the gcrypt repo. -} +combineExistingGCrypt :: SshData -> UUID -> Handler Html +combineExistingGCrypt sshdata u = do + reponame <- liftAnnex $ getGCryptRemoteName u repourl + enableGCrypt sshdata reponame + where + repourl = genSshUrl sshdata + +{- Sets up remote repository for ssh, or directory for rsync. -} +prepSsh :: Bool -> SshData -> (SshData -> Handler Html) -> Handler Html +prepSsh needsinit sshdata a + | needsPubKey sshdata = do + (sshdata', keypair) <- liftIO $ setupSshKeyPair sshdata + prepSsh' needsinit sshdata sshdata' (Just keypair) a + | sshPort sshdata /= 22 = do + sshdata' <- liftIO $ setSshConfig sshdata [] + prepSsh' needsinit sshdata sshdata' Nothing a + | otherwise = prepSsh' needsinit sshdata sshdata Nothing a + +prepSsh' :: Bool -> SshData -> SshData -> Maybe SshKeyPair -> (SshData -> Handler Html) -> Handler Html +prepSsh' needsinit origsshdata sshdata keypair a + | hasCapability sshdata PushCapable = do + {- To ensure the repository is initialized, try to push the + - git-annex branch to it. Then git-annex-shell will see + - the branch and auto-initialize. -} + when needsinit $ do + void $ liftAnnex $ inRepo $ Git.Command.runBool + [ Param "push" + , Param (genSshUrl sshdata) + , Param (fromRef Annex.Branch.name) + ] + a sshdata + | otherwise = sshSetup (mkSshInput origsshdata) + [ "-p", show (sshPort origsshdata) + ] + (genSshHost (sshHostName origsshdata) (sshUserName origsshdata)) + remoteCommand + Nothing (a sshdata) + where + remotedir = T.unpack $ sshDirectory sshdata + remoteCommand = shellWrap $ intercalate "&&" $ catMaybes + [ Just $ "mkdir -p " ++ shellEscape remotedir + , Just $ "cd " ++ shellEscape remotedir + , if rsynconly then Nothing else Just $ unwords + [ "if [ ! -d .git ]; then" + , wrapCommand "git init --bare --shared" + , "&&" + , wrapCommand "git config receive.denyNonFastforwards" + , ";fi" + ] + , if needsinit then Just (wrapCommand "git annex init") else Nothing + , if needsPubKey origsshdata + then addAuthorizedKeysCommand (hasCapability origsshdata GitAnnexShellCapable) remotedir . sshPubKey <$> keypair + else Nothing + ] + rsynconly = onlyCapability origsshdata RsyncCapable + +makeSshRepo :: RepoStatus -> SshData -> Handler Html +makeSshRepo rs sshdata + | onlyCapability sshdata RsyncCapable = setupCloudRemote TransferGroup Nothing mk + | otherwise = makeSshRepoConnection rs mk setup + where + mk = makeSshRemote sshdata + -- Record the location of the ssh remote in the remote log, so it + -- can easily be enabled elsewhere using the webapp. + setup r = do + m <- readRemoteLog + let c = fromMaybe M.empty (M.lookup (Remote.uuid r) m) + let c' = M.insert "location" (genSshUrl sshdata) $ + M.insert "type" "git" $ + M.insert "name" (fromMaybe (Remote.name r) (M.lookup "name" c)) c + configSet (Remote.uuid r) c' + +makeSshRepoConnection :: RepoStatus -> Annex RemoteName -> (Remote -> Annex ()) -> Handler Html +makeSshRepoConnection rs mk setup = setupRemote postsetup mgroup Nothing mk + where + mgroup = case rs of + NewRepo -> Just TransferGroup + ExistingRepo -> Nothing + postsetup r = do + liftAssistant $ sendRemoteControl RELOAD + liftAnnex $ setup r + case rs of + NewRepo -> redirect $ EditNewRepositoryR (Remote.uuid r) + ExistingRepo -> redirect DashboardR + +makeGCryptRepo :: RepoStatus -> KeyId -> SshData -> Handler Html +makeGCryptRepo rs keyid sshdata = makeSshRepoConnection rs mk (const noop) + where + mk = makeGCryptRemote (sshRepoName sshdata) (genSshUrl sshdata) keyid + +getAddRsyncNetR :: Handler Html +getAddRsyncNetR = postAddRsyncNetR +postAddRsyncNetR :: Handler Html +postAddRsyncNetR = do + ((result, form), enctype) <- runFormPostNoToken $ + renderBootstrap3 bootstrapFormLayout $ sshInputAForm hostnamefield $ + SshInput Nothing Nothing Password Nothing Nothing 22 + let showform status = inpage $ + $(widgetFile "configurators/rsync.net/add") + case result of + FormSuccess sshinput + | isRsyncNet (inputHostname sshinput) -> + go sshinput + | otherwise -> + showform $ UnusableServer + "That is not a rsync.net host name." + _ -> showform UntestedServer + where + inpage = page "Add a Rsync.net repository" (Just Configuration) + hostnamefield = textField `withExpandableNote` ("Help", help) + help = [whamlet| +

    + When you sign up for a Rsync.net account, you should receive an # + email from them with the host name and user name to put here. +
    + The host name will be something like "usw-s001.rsync.net", and the # + user name something like "7491" +|] + go sshinput = do + let reponame = genSshRepoName "rsync.net" + (maybe "" T.unpack $ inputDirectory sshinput) + + prepRsyncNet sshinput reponame $ \sshdata -> inpage $ + checkExistingGCrypt sshdata $ do + cmd <- liftAnnex $ gpgCmd <$> Annex.getGitConfig + secretkeys <- sortBy (comparing snd) . M.toList + <$> liftIO (secretKeys cmd) + $(widgetFile "configurators/rsync.net/encrypt") + +getMakeRsyncNetSharedR :: SshData -> Handler Html +getMakeRsyncNetSharedR = makeSshRepo NewRepo . rsyncOnly + +{- Make a new gcrypt special remote on rsync.net. -} +getMakeRsyncNetGCryptR :: SshData -> RepoKey -> Handler Html +getMakeRsyncNetGCryptR sshdata NoRepoKey = whenGcryptInstalled $ + withNewSecretKey $ getMakeRsyncNetGCryptR sshdata . RepoKey +getMakeRsyncNetGCryptR sshdata (RepoKey keyid) = whenGcryptInstalled $ + sshSetup (mkSshInput sshdata) [] sshhost gitinit Nothing $ + makeGCryptRepo NewRepo keyid sshdata + where + sshhost = genSshHost (sshHostName sshdata) (sshUserName sshdata) + gitinit = "git init --bare " ++ T.unpack (sshDirectory sshdata) + +enableRsyncNet :: SshInput -> String -> Handler Html +enableRsyncNet sshinput reponame = + prepRsyncNet sshinput reponame $ makeSshRepo ExistingRepo . rsyncOnly + +enableRsyncNetGCrypt :: SshInput -> RemoteName -> Handler Html +enableRsyncNetGCrypt sshinput reponame = + prepRsyncNet sshinput reponame $ \sshdata -> whenGcryptInstalled $ + checkGCryptRepoEncryption (genSshUrl sshdata) notencrypted notinstalled $ + enableGCrypt sshdata reponame + where + notencrypted = giveup "Unexpectedly found a non-encrypted git repository, instead of the expected encrypted git repository." + notinstalled = error "internal" + +{- Prepares rsync.net ssh key and creates the directory that will be + - used on rsync.net. If successful, runs an action with its SshData. + - + - To append the ssh key to rsync.net's authorized_keys, their + - documentation recommends a dd methodd, where the line is fed + - in to ssh over stdin. + -} +prepRsyncNet :: SshInput -> String -> (SshData -> Handler Html) -> Handler Html +prepRsyncNet sshinput reponame a = do + knownhost <- liftIO $ maybe (return False) knownHost (inputHostname sshinput) + (sshdata, keypair) <- liftIO $ setupSshKeyPair $ + (mkSshData sshinput) + { sshRepoName = reponame + , needsPubKey = True + , sshCapabilities = [RsyncCapable] + } + let sshhost = genSshHost (sshHostName sshdata) (sshUserName sshdata) + let torsyncnet + | knownhost = [] + | otherwise = [sshOpt "StrictHostKeyChecking" "no"] + {- I'd prefer to separate commands with && , but + - rsync.net's shell does not support that. -} + let remotecommand = intercalate ";" + [ "mkdir -p .ssh" + , "touch .ssh/authorized_keys" + , "dd of=.ssh/authorized_keys oflag=append conv=notrunc" + , "mkdir -p " ++ T.unpack (sshDirectory sshdata) + ] + sshSetup sshinput torsyncnet sshhost remotecommand + (Just $ sshPubKey keypair) (a sshdata) + +isRsyncNet :: Maybe Text -> Bool +isRsyncNet Nothing = False +isRsyncNet (Just host) = ".rsync.net" `T.isSuffixOf` T.toLower host + +data GitLabUrl = GitLabUrl { unGitLabUrl :: Text } + +badGitLabUrl :: Text +badGitLabUrl = "Bad SSH clone url. Expected something like: git@gitlab.com:yourlogin/annex.git" + +parseGitLabUrl :: GitLabUrl -> Maybe SshData +parseGitLabUrl (GitLabUrl t) = + let (u, r) = separate (== '@') (T.unpack t) + (h, p) = separate (== ':') r + in if null u || null h || null p + then Nothing + else Just $ SshData + { sshHostName = T.pack h + , sshUserName = Just (T.pack u) + , sshDirectory = T.pack p + , sshRepoName = genSshRepoName h p + , sshPort = 22 + , needsPubKey = False + , sshCapabilities = + [ GitAnnexShellCapable + , GitCapable + , PushCapable + ] + , sshRepoUrl = Just (T.unpack t) + } + +isGitLab :: SshData -> Bool +isGitLab d = T.pack "gitlab.com" `T.isSuffixOf` (T.toLower (sshHostName d)) + +toGitLabUrl :: SshData -> GitLabUrl +toGitLabUrl d = GitLabUrl $ T.concat + [ fromMaybe (T.pack "git") (sshUserName d) + , T.pack "@" + , sshHostName d + , T.pack ":" + , sshDirectory d + ] + +{- Try to ssh into the gitlab server, verify we can access the repository, + - and get the uuid of the repository, if it already has one. + - + - A repository on gitlab won't be initialized as a git-annex repo + - unless a git-annex branch was already pushed to it. So, if + - git-annex-shell fails to work that's probably why; verify if + - the server is letting us ssh in by running git send-pack + - (in dry run mode). -} +testGitLabUrl :: GitLabUrl -> Annex (ServerStatus, Maybe SshData, UUID) +testGitLabUrl glu = case parseGitLabUrl glu of + Nothing -> return (UnusableServer badGitLabUrl, Nothing, NoUUID) + Just sshdata -> + checkor sshdata $ do + (sshdata', keypair) <- liftIO $ setupSshKeyPair sshdata + checkor sshdata' $ + return (ServerNeedsPubKey (sshPubKey keypair), Just sshdata', NoUUID) + where + checkor sshdata ora = do + u <- probeuuid sshdata + if u /= NoUUID + then return (UsableServer (sshCapabilities sshdata), Just sshdata, u) + else ifM (verifysshworks sshdata) + ( return (UsableServer (sshCapabilities sshdata), Just sshdata, NoUUID) + , ora + ) + probeuuid sshdata = do + r <- inRepo $ Git.Construct.fromRemoteLocation (fromJust $ sshRepoUrl sshdata) + getUncachedUUID . either (const r) fst <$> + Remote.Helper.Ssh.onRemote NoConsumeStdin r + (Git.Config.fromPipe r, return (Left $ error "configlist failed")) + "configlist" [] [] + verifysshworks sshdata = inRepo $ Git.Command.runBool + [ Param "send-pack" + , Param (fromJust $ sshRepoUrl sshdata) + , Param "--dry-run" + , Param "--force" + , Param (fromRef Annex.Branch.name) + ] + +gitLabUrlAForm :: Maybe GitLabUrl -> AForm Handler GitLabUrl +gitLabUrlAForm defval = GitLabUrl <$> areq check_input (bfs "SSH clone url") (unGitLabUrl <$> defval) + where + check_input = checkBool (isJust . parseGitLabUrl . GitLabUrl) + badGitLabUrl textField + +getAddGitLabR :: Handler Html +getAddGitLabR = postAddGitLabR +postAddGitLabR :: Handler Html +postAddGitLabR = promptGitLab Nothing + +promptGitLab :: Maybe GitLabUrl -> Handler Html +promptGitLab defval = sshConfigurator $ do + ((result, form), enctype) <- liftH $ + runFormPostNoToken $ renderBootstrap3 bootstrapFormLayout $ + gitLabUrlAForm defval + case result of + FormSuccess gitlaburl -> do + (status, msshdata, u) <- liftAnnex $ testGitLabUrl gitlaburl + case (status, msshdata) of + (UsableServer _, Just sshdata) -> + liftH $ redirect $ ConfirmSshR sshdata u + _ -> showform form enctype status + _ -> showform form enctype UntestedServer + where + showform form enctype status = $(widgetFile "configurators/gitlab.com/add") + +enableGitLab :: SshData -> Handler Html +enableGitLab = promptGitLab . Just . toGitLabUrl diff --git a/Assistant/WebApp/Configurators/Unused.hs b/Assistant/WebApp/Configurators/Unused.hs new file mode 100644 index 0000000000..11f60e3127 --- /dev/null +++ b/Assistant/WebApp/Configurators/Unused.hs @@ -0,0 +1,80 @@ +{- git-annex assistant unused file preferences + - + - Copyright 2014 Joey Hess + - + - Licensed under the GNU AGPL version 3 or higher. + -} + +{-# LANGUAGE QuasiQuotes, TemplateHaskell, OverloadedStrings #-} + +module Assistant.WebApp.Configurators.Unused where + +import Assistant.WebApp.Common +import qualified Annex +import Utility.HumanTime +import Assistant.Unused +import Config +import Git.Config +import Logs.Unused +import Utility.Tense + +import qualified Text.Hamlet as Hamlet + +data UnusedForm = UnusedForm + { enableExpire :: Bool + , expireWhen :: Integer + } + +unusedForm :: UnusedForm -> Hamlet.Html -> MkMForm UnusedForm +unusedForm d msg = do + (enableRes, enableView) <- mreq (selectFieldList enabledisable) (bfs "") + (Just $ enableExpire d) + (whenRes, whenView) <- mreq intField (bfs "") + (Just $ expireWhen d) + let form = do + webAppFormAuthToken + $(widgetFile "configurators/unused/form") + return (UnusedForm <$> enableRes <*> whenRes, form) + where + enabledisable :: [(Text, Bool)] + enabledisable = [("Disable expiry", False), ("Enable expiry", True)] + +getConfigUnusedR :: Handler Html +getConfigUnusedR = postConfigUnusedR +postConfigUnusedR :: Handler Html +postConfigUnusedR = page "Unused files" (Just Configuration) $ do + current <- liftAnnex getUnused + ((res, form), enctype) <- liftH $ runFormPostNoToken $ unusedForm current + case res of + FormSuccess new -> liftH $ do + liftAnnex $ storeUnused new + redirect ConfigurationR + _ -> do + munuseddesc <- liftAssistant describeUnused + ts <- liftAnnex $ dateUnusedLog "" + mlastchecked <- case ts of + Nothing -> pure Nothing + Just t -> Just <$> liftIO (durationSince t) + $(widgetFile "configurators/unused") + +getUnused :: Annex UnusedForm +getUnused = convert . annexExpireUnused <$> Annex.getGitConfig + where + convert Nothing = noexpire + convert (Just Nothing) = noexpire + convert (Just (Just n)) = UnusedForm True $ durationToDays n + + -- The 7 is so that, if they enable expiry, they have to change + -- it to get faster than a week. + noexpire = UnusedForm False 7 + +storeUnused :: UnusedForm -> Annex () +storeUnused f = setConfig (annexConfig "expireunused") $ + if not (enableExpire f) || expireWhen f < 0 + then boolConfig False + else fromDuration $ daysToDuration $ expireWhen f + +getCleanupUnusedR :: Handler Html +getCleanupUnusedR = do + liftAssistant $ expireUnused Nothing + redirect ConfigUnusedR diff --git a/Assistant/WebApp/Configurators/Upgrade.hs b/Assistant/WebApp/Configurators/Upgrade.hs new file mode 100644 index 0000000000..3ef361c432 --- /dev/null +++ b/Assistant/WebApp/Configurators/Upgrade.hs @@ -0,0 +1,48 @@ +{- git-annex assistant webapp upgrade UI + - + - Copyright 2013 Joey Hess + - + - Licensed under the GNU AGPL version 3 or higher. + -} + +{-# LANGUAGE CPP, QuasiQuotes, TemplateHaskell, OverloadedStrings #-} + +module Assistant.WebApp.Configurators.Upgrade where + +import Assistant.WebApp.Common +import Types.Distribution +import Assistant.Upgrade +import Assistant.Restart +import Config + +{- On Android, just point the user at the apk file to download. + - Installation will be handled by selecting the downloaded file. + - + - Otherwise, start the upgrade process, which will run fully + - noninteractively. + - -} +getConfigStartUpgradeR :: GitAnnexDistribution -> Handler Html +getConfigStartUpgradeR d = do +#ifdef ANDROID_SPLICES + let url = distributionUrl d + page "Upgrade" (Just Configuration) $ + $(widgetFile "configurators/upgrade/android") +#else + liftAssistant $ startDistributionDownload d + redirect DashboardR +#endif + +{- Finish upgrade by starting the new assistant in the same repository this + - one is running in, and redirecting to it. -} +getConfigFinishUpgradeR :: Handler Html +getConfigFinishUpgradeR = do + liftAssistant prepUpgrade + url <- liftAssistant runRestart + liftAssistant $ postUpgrade url + redirect url + +getConfigEnableAutomaticUpgradeR :: Handler Html +getConfigEnableAutomaticUpgradeR = do + liftAnnex $ setConfig (annexConfig "autoupgrade") + (fromAutoUpgrade AutoUpgrade) + redirect DashboardR diff --git a/Assistant/WebApp/Configurators/WebDAV.hs b/Assistant/WebApp/Configurators/WebDAV.hs new file mode 100644 index 0000000000..4a8da2067d --- /dev/null +++ b/Assistant/WebApp/Configurators/WebDAV.hs @@ -0,0 +1,145 @@ +{- git-annex assistant webapp configurators for WebDAV remotes + - + - Copyright 2012, 2013 Joey Hess + - + - Licensed under the GNU AGPL version 3 or higher. + -} + +{-# LANGUAGE CPP, TemplateHaskell, OverloadedStrings #-} + +module Assistant.WebApp.Configurators.WebDAV where + +import Assistant.WebApp.Common +import Creds +#ifdef WITH_WEBDAV +import qualified Remote.WebDAV as WebDAV +import Assistant.WebApp.MakeRemote +import qualified Remote +import Types.Remote (RemoteConfig) +import Types.StandardGroups +import Logs.Remote +import Git.Types (RemoteName) +import Assistant.Gpg +import Types.GitConfig + +import qualified Data.Map as M +#endif +import qualified Data.Text as T +import Network.URI + +webDAVConfigurator :: Widget -> Handler Html +webDAVConfigurator = page "Add a WebDAV repository" (Just Configuration) + +boxConfigurator :: Widget -> Handler Html +boxConfigurator = page "Add a Box.com repository" (Just Configuration) + +data WebDAVInput = WebDAVInput + { user :: Text + , password :: Text + , embedCreds :: Bool + , directory :: Text + , enableEncryption :: EnableEncryption + } + +toCredPair :: WebDAVInput -> CredPair +toCredPair input = (T.unpack $ user input, T.unpack $ password input) + +boxComAForm :: Maybe CredPair -> MkAForm WebDAVInput +boxComAForm defcreds = WebDAVInput + <$> areq textField (bfs "Username or Email") (T.pack . fst <$> defcreds) + <*> areq passwordField (bfs "Box.com Password") (T.pack . snd <$> defcreds) + <*> areq checkBoxField "Share this account with other devices and friends?" (Just True) + <*> areq textField (bfs "Directory") (Just "annex") + <*> enableEncryptionField + +webDAVCredsAForm :: Maybe CredPair -> MkAForm WebDAVInput +webDAVCredsAForm defcreds = WebDAVInput + <$> areq textField (bfs "Username or Email") (T.pack . fst <$> defcreds) + <*> areq passwordField (bfs "Password") (T.pack . snd <$> defcreds) + <*> pure False + <*> pure T.empty + <*> pure NoEncryption -- not used! + +getAddBoxComR :: Handler Html +getAddBoxComR = postAddBoxComR +postAddBoxComR :: Handler Html +#ifdef WITH_WEBDAV +postAddBoxComR = boxConfigurator $ do + defcreds <- liftAnnex $ previouslyUsedWebDAVCreds "box.com" + ((result, form), enctype) <- liftH $ + runFormPostNoToken $ renderBootstrap3 bootstrapFormLayout + $ boxComAForm defcreds + case result of + FormSuccess input -> liftH $ + makeWebDavRemote initSpecialRemote "box.com" (toCredPair input) $ M.fromList + [ configureEncryption $ enableEncryption input + , ("embedcreds", if embedCreds input then "yes" else "no") + , ("type", "webdav") + , ("url", "https://dav.box.com/dav/" ++ T.unpack (directory input)) + -- Box.com has a max file size of 100 mb, but + -- using smaller chunks has better memory + -- performance. + , ("chunk", "10mb") + ] + _ -> $(widgetFile "configurators/addbox.com") +#else +postAddBoxComR = giveup "WebDAV not supported by this build" +#endif + +getEnableWebDAVR :: UUID -> Handler Html +getEnableWebDAVR = postEnableWebDAVR +postEnableWebDAVR :: UUID -> Handler Html +#ifdef WITH_WEBDAV +postEnableWebDAVR uuid = do + m <- liftAnnex readRemoteLog + let c = fromJust $ M.lookup uuid m + let name = fromJust $ M.lookup "name" c + let url = fromJust $ M.lookup "url" c + mcreds <- liftAnnex $ do + dummycfg <- liftIO dummyRemoteGitConfig + getRemoteCredPairFor "webdav" c dummycfg (WebDAV.davCreds uuid) + case mcreds of + Just creds -> webDAVConfigurator $ liftH $ + makeWebDavRemote enableSpecialRemote name creds M.empty + Nothing + | "box.com/" `isInfixOf` url -> + boxConfigurator $ showform name url + | otherwise -> + webDAVConfigurator $ showform name url + where + showform name url = do + defcreds <- liftAnnex $ + maybe (pure Nothing) previouslyUsedWebDAVCreds $ + urlHost url + ((result, form), enctype) <- liftH $ + runFormPostNoToken $ renderBootstrap3 bootstrapFormLayout $ + webDAVCredsAForm defcreds + case result of + FormSuccess input -> liftH $ + makeWebDavRemote enableSpecialRemote name (toCredPair input) M.empty + _ -> do + description <- liftAnnex $ + T.pack <$> Remote.prettyUUID uuid + $(widgetFile "configurators/enablewebdav") +#else +postEnableWebDAVR _ = giveup "WebDAV not supported by this build" +#endif + +#ifdef WITH_WEBDAV +makeWebDavRemote :: SpecialRemoteMaker -> RemoteName -> CredPair -> RemoteConfig -> Handler () +makeWebDavRemote maker name creds config = + setupCloudRemote TransferGroup Nothing $ + maker name WebDAV.remote (Just creds) config + +{- Only returns creds previously used for the same hostname. -} +previouslyUsedWebDAVCreds :: String -> Annex (Maybe CredPair) +previouslyUsedWebDAVCreds hostname = + previouslyUsedCredPair WebDAV.davCreds WebDAV.remote samehost + where + samehost url = case urlHost =<< WebDAV.configUrl url of + Nothing -> False + Just h -> h == hostname +#endif + +urlHost :: String -> Maybe String +urlHost url = uriRegName <$> (uriAuthority =<< parseURI url) diff --git a/Assistant/WebApp/Control.hs b/Assistant/WebApp/Control.hs new file mode 100644 index 0000000000..8a0e82119d --- /dev/null +++ b/Assistant/WebApp/Control.hs @@ -0,0 +1,78 @@ +{- git-annex assistant webapp control + - + - Copyright 2012, 2013 Joey Hess + - + - Licensed under the GNU AGPL version 3 or higher. + -} + +{-# LANGUAGE TypeFamilies, QuasiQuotes, MultiParamTypeClasses, TemplateHaskell, OverloadedStrings, RankNTypes #-} + +module Assistant.WebApp.Control where + +import Assistant.WebApp.Common +import Assistant.DaemonStatus +import Assistant.Alert +import Assistant.TransferSlots +import Assistant.Restart +import Utility.LogFile +import Utility.NotificationBroadcaster + +import Control.Concurrent +import qualified Data.Map as M +import qualified Data.Text as T + +getShutdownR :: Handler Html +getShutdownR = page "Shutdown" Nothing $ + $(widgetFile "control/shutdown") + +getShutdownConfirmedR :: Handler Html +getShutdownConfirmedR = do + liftAssistant $ do + {- Remove all alerts for currently running activities. -} + updateAlertMap $ M.filter $ \a -> alertClass a /= Activity + void $ addAlert shutdownAlert + {- Stop transfers the assistant is running, + - otherwise they would continue past shutdown. + - Pausing transfers prevents more being started up (and stops + - the transfer processes). -} + ts <- M.keys . currentTransfers <$> getDaemonStatus + mapM_ pauseTransfer ts + webapp <- getYesod + let url = T.unpack $ yesodRender webapp (T.pack "") NotRunningR [] + {- Signal any other web browsers. -} + liftAssistant $ do + modifyDaemonStatus_ $ \status -> status { globalRedirUrl = Just url } + liftIO . sendNotification . globalRedirNotifier =<< getDaemonStatus + {- Wait 2 seconds before shutting down, to give the web + - page time to load in the browser. -} + void $ liftIO $ forkIO $ do + threadDelay 2000000 + terminateSelf + redirect NotRunningR + +{- Use a custom page to avoid putting long polling elements on it that will + - fail and cause thet web browser to show an error once the webapp is + - truely stopped. -} +getNotRunningR :: Handler Html +getNotRunningR = customPage' False Nothing $ + $(widgetFile "control/notrunning") + +getRestartR :: Handler Html +getRestartR = do + liftAssistant prepRestart + url <- liftAssistant runRestart + liftAssistant $ postRestart url + redirect url + +getRestartThreadR :: ThreadName -> Handler () +getRestartThreadR name = do + m <- liftAssistant $ startedThreads <$> getDaemonStatus + liftIO $ maybe noop snd $ M.lookup name m + redirectBack + +getLogR :: Handler Html +getLogR = page "Logs" Nothing $ do + logfile <- liftAnnex $ fromRepo gitAnnexLogFile + logs <- liftIO $ listLogs logfile + logcontent <- liftIO $ concat <$> mapM readFile logs + $(widgetFile "control/log") diff --git a/Assistant/WebApp/DashBoard.hs b/Assistant/WebApp/DashBoard.hs new file mode 100644 index 0000000000..0ed6978da2 --- /dev/null +++ b/Assistant/WebApp/DashBoard.hs @@ -0,0 +1,178 @@ +{- git-annex assistant webapp dashboard + - + - Copyright 2012 Joey Hess + - + - Licensed under the GNU AGPL version 3 or higher. + -} + +{-# LANGUAGE CPP, TypeFamilies, QuasiQuotes, MultiParamTypeClasses, TemplateHaskell, OverloadedStrings, RankNTypes #-} + +module Assistant.WebApp.DashBoard where + +import Assistant.WebApp.Common +import Assistant.WebApp.RepoList +import Assistant.WebApp.Notifications +import Assistant.TransferQueue +import Assistant.TransferSlots +import Assistant.DaemonStatus +import Utility.NotificationBroadcaster +import Types.Transfer +import Logs.Transfer +import Utility.Percentage +import Utility.DataUnits +import qualified Remote +import qualified Git + +import qualified Text.Hamlet as Hamlet +import qualified Data.Map as M +import Control.Concurrent + +{- A display of currently running and queued transfers. -} +transfersDisplay :: Widget +transfersDisplay = do + current <- liftAssistant $ M.toList <$> getCurrentTransfers + queued <- take 10 <$> liftAssistant getTransferQueue + autoUpdate ident NotifierTransfersR (10 :: Int) (10 :: Int) + let transfers = simplifyTransfers $ current ++ queued + let transfersrunning = not $ null transfers + scanrunning <- if transfersrunning + then return False + else liftAssistant $ transferScanRunning <$> getDaemonStatus + $(widgetFile "dashboard/transfers") + where + ident = "transfers" + isrunning info = not $ + transferPaused info || isNothing (startedTime info) + desc transfer info = case associatedFile info of + AssociatedFile Nothing -> key2file $ transferKey transfer + AssociatedFile (Just af) -> af + +{- Simplifies a list of transfers, avoiding display of redundant + - equivilant transfers. -} +simplifyTransfers :: [(Transfer, TransferInfo)] -> [(Transfer, TransferInfo)] +simplifyTransfers [] = [] +simplifyTransfers (x:[]) = [x] +simplifyTransfers (v@(t1, _):r@((t2, _):l)) + | equivilantTransfer t1 t2 = simplifyTransfers (v:l) + | otherwise = v : simplifyTransfers r + +{- Called by client to get a display of currently in process transfers. + - + - Returns a div, which will be inserted into the calling page. + - + - Note that the head of the widget is not included, only its + - body is. To get the widget head content, the widget is also + - inserted onto the getDashboardR page. + -} +getTransfersR :: NotificationId -> Handler Html +getTransfersR nid = do + waitNotifier getTransferBroadcaster nid + + p <- widgetToPageContent transfersDisplay + withUrlRenderer $ [hamlet|^{pageBody p}|] + +{- The main dashboard. -} +dashboard :: Bool -> Widget +dashboard warnNoScript = do + let repolist = repoListDisplay $ + mainRepoSelector { nudgeAddMore = True } + let transferlist = transfersDisplay + $(widgetFile "dashboard/main") + +getDashboardR :: Handler Html +getDashboardR = ifM inFirstRun + ( redirect ConfigurationR + , page "" (Just DashBoard) $ dashboard True + ) + +{- Used to test if the webapp is running. -} +headDashboardR :: Handler () +headDashboardR = noop + +{- Same as DashboardR, except no autorefresh at all (and no noscript warning). -} +getNoScriptR :: Handler Html +getNoScriptR = page "" (Just DashBoard) $ dashboard False + +{- Same as DashboardR, except with autorefreshing via meta refresh. -} +getNoScriptAutoR :: Handler Html +getNoScriptAutoR = page "" (Just DashBoard) $ do + let delayseconds = 3 :: Int + let this = NoScriptAutoR + toWidgetHead $(Hamlet.hamletFile $ hamletTemplate "dashboard/metarefresh") + dashboard False + +{- The javascript code does a post. -} +postFileBrowserR :: Handler () +postFileBrowserR = void openFileBrowser + +{- Used by non-javascript browsers, where clicking on the link actually + - opens this page, so we redirect back to the referrer. -} +getFileBrowserR :: Handler () +getFileBrowserR = whenM openFileBrowser redirectBack + +{- Opens the system file browser on the repo, or, as a fallback, + - goes to a file:// url. Returns True if it's ok to redirect away + - from the page (ie, the system file browser was opened). + - + - Note that the command is opened using a different thread, to avoid + - blocking the response to the browser on it. -} +openFileBrowser :: Handler Bool +openFileBrowser = do + path <- liftIO . absPath =<< liftAnnex (fromRepo Git.repoPath) +#ifdef darwin_HOST_OS + let cmd = "open" + let p = proc cmd [path] +#else +#ifdef mingw32_HOST_OS + {- Changing to the directory and then opening . works around + - spaces in directory name, etc. -} + let cmd = "cmd" + let p = (proc cmd ["/c start ."]) { cwd = Just path } +#else + let cmd = "xdg-open" + let p = proc cmd [path] +#endif +#endif + ifM (liftIO $ inPath cmd) + ( do + let run = void $ liftIO $ forkIO $ do + (Nothing, Nothing, Nothing, pid) <- createProcess p + void $ waitForProcess pid + run +#ifdef mingw32_HOST_OS + {- On windows, if the file browser is not + - already open, it comes up below the + - web browser when started. + - + - Running it a second time brings it + - to the foreground. + - + - Seems to need a delay long enough for the file + - browser to be open in order to work. Here 1 + - second. -} + liftIO $ threadDelay 1000000 + run +#endif + return True + , do + void $ redirect $ "file://" ++ path + return False + ) + +{- Transfer controls. The GET is done in noscript mode and redirects back + - to the referring page. The POST is called by javascript. -} +getPauseTransferR :: Transfer -> Handler () +getPauseTransferR = noscript postPauseTransferR +postPauseTransferR :: Transfer -> Handler () +postPauseTransferR = liftAssistant . pauseTransfer +getStartTransferR :: Transfer -> Handler () +getStartTransferR = noscript postStartTransferR +postStartTransferR :: Transfer -> Handler () +postStartTransferR = liftAssistant . startTransfer +getCancelTransferR :: Transfer -> Handler () +getCancelTransferR = noscript postCancelTransferR +postCancelTransferR :: Transfer -> Handler () +postCancelTransferR = liftAssistant . cancelTransfer False + +noscript :: (Transfer -> Handler ()) -> Transfer -> Handler () +noscript a t = a t >> redirectBack diff --git a/Assistant/WebApp/Documentation.hs b/Assistant/WebApp/Documentation.hs new file mode 100644 index 0000000000..63c4f7cb98 --- /dev/null +++ b/Assistant/WebApp/Documentation.hs @@ -0,0 +1,42 @@ +{- git-annex assistant webapp documentation + - + - Copyright 2012 Joey Hess + - + - Licensed under the GNU AGPL version 3 or higher. + -} + +{-# LANGUAGE QuasiQuotes, TemplateHaskell, OverloadedStrings #-} + +module Assistant.WebApp.Documentation where + +import Assistant.WebApp.Common +import Assistant.Install (standaloneAppBase) +import BuildInfo (packageversion) +import BuildFlags + +{- The full license info may be included in a file on disk that can + - be read in and displayed. -} +licenseFile :: IO (Maybe FilePath) +licenseFile = do + base <- standaloneAppBase + return $ ( "LICENSE") <$> base + +getAboutR :: Handler Html +getAboutR = page "About git-annex" (Just About) $ do + builtinlicense <- isJust <$> liftIO licenseFile + $(widgetFile "documentation/about") + +getLicenseR :: Handler Html +getLicenseR = do + v <- liftIO licenseFile + case v of + Nothing -> redirect AboutR + Just f -> customPage (Just About) $ do + -- no sidebar, just pages of legalese.. + setTitle "License" + license <- liftIO $ readFile f + $(widgetFile "documentation/license") + +getRepoGroupR :: Handler Html +getRepoGroupR = page "About repository groups" (Just About) $ + $(widgetFile "documentation/repogroup") diff --git a/Assistant/WebApp/Form.hs b/Assistant/WebApp/Form.hs new file mode 100644 index 0000000000..43b367d4ca --- /dev/null +++ b/Assistant/WebApp/Form.hs @@ -0,0 +1,99 @@ +{- git-annex assistant webapp form utilities + - + - Copyright 2012 Joey Hess + - + - Licensed under the GNU AGPL version 3 or higher. + -} + +{-# LANGUAGE FlexibleContexts, TypeFamilies, QuasiQuotes #-} +{-# LANGUAGE MultiParamTypeClasses, TemplateHaskell #-} +{-# LANGUAGE CPP #-} +{-# LANGUAGE OverloadedStrings, RankNTypes #-} + +module Assistant.WebApp.Form where + +import Assistant.WebApp.Types +import Assistant.Gpg + +import Yesod hiding (textField, passwordField) +import Yesod.Form.Fields as F +import Yesod.Form.Bootstrap3 as Y hiding (bfs) +import Data.Text (Text) + +{- Yesod's textField sets the required attribute for required fields. + - We don't want this, because many of the forms used in this webapp + - display a modal dialog when submitted, which interacts badly with + - required field handling by the browser. + - + - Required fields are still checked by Yesod. + -} +textField :: MkField Text +textField = F.textField + { fieldView = \theId name attrs val _isReq -> [whamlet| + +|] + } + +readonlyTextField :: MkField Text +readonlyTextField = F.textField + { fieldView = \theId name attrs val _isReq -> [whamlet| + +|] + } + +{- Also without required attribute. -} +passwordField :: MkField Text +passwordField = F.passwordField + { fieldView = \theId name attrs val _isReq -> toWidget [hamlet| + +|] + } + +{- Makes a note widget be displayed after a field. -} +withNote :: (ToWidget (HandlerSite m) a) => Field m v -> a -> Field m v +withNote field note = field { fieldView = newview } + where + newview theId name attrs val isReq = + let fieldwidget = (fieldView field) theId name attrs val isReq + in [whamlet|^{fieldwidget}  ^{note}|] + +{- Note that the toggle string must be unique on the form. -} +withExpandableNote :: (ToWidget (HandlerSite m) w) => Field m v -> (String, w) -> Field m v +withExpandableNote field (toggle, note) = withNote field $ [whamlet| +#{toggle} +
    + ^{note} +|] + where + ident = "toggle_" ++ toggle + +{- Adds a check box to an AForm to control encryption. -} +#if MIN_VERSION_yesod_core(1,6,0) +enableEncryptionField :: (RenderMessage site FormMessage) => AForm (HandlerFor site) EnableEncryption +#else +enableEncryptionField :: (RenderMessage site FormMessage) => AForm (HandlerT site IO) EnableEncryption +#endif +enableEncryptionField = areq (selectFieldList choices) (bfs "Encryption") (Just SharedEncryption) + where + choices :: [(Text, EnableEncryption)] + choices = + [ ("Encrypt all data", SharedEncryption) + , ("Disable encryption", NoEncryption) + ] + +{- Defines the layout used by the Bootstrap3 form helper -} +bootstrapFormLayout :: BootstrapFormLayout +bootstrapFormLayout = BootstrapHorizontalForm (ColSm 0) (ColSm 2) (ColSm 0) (ColSm 10) + +{- Adds the form-control class used by Bootstrap3 for layout to a field + - This is the same as Yesod.Form.Bootstrap3.bfs except it takes just a Text + - parameter as I couldn't get the original bfs to compile due to type ambiguities. + -} +bfs :: Text -> FieldSettings master +bfs msg = FieldSettings + { fsLabel = SomeMessage msg + , fsName = Nothing + , fsId = Nothing + , fsAttrs = [("class", "form-control")] + , fsTooltip = Nothing + } diff --git a/Assistant/WebApp/Gpg.hs b/Assistant/WebApp/Gpg.hs new file mode 100644 index 0000000000..22285cf451 --- /dev/null +++ b/Assistant/WebApp/Gpg.hs @@ -0,0 +1,113 @@ +{- git-annex webapp gpg stuff + - + - Copyright 2013 Joey Hess + - + - Licensed under the GNU AGPL version 3 or higher. + -} + +{-# LANGUAGE QuasiQuotes, TemplateHaskell, OverloadedStrings #-} + +module Assistant.WebApp.Gpg where + +import Assistant.WebApp.Common +import Assistant.Gpg +import Utility.Gpg +import qualified Annex +import qualified Git.Command +import qualified Git.Remote.Remove +import qualified Git.Construct +import qualified Annex.Branch +import qualified Git.GCrypt +import qualified Remote.GCrypt as GCrypt +import Git.Types (RemoteName) +import Assistant.WebApp.MakeRemote +import Logs.Remote + +import qualified Data.Map as M + +gpgKeyDisplay :: KeyId -> Maybe UserId -> Widget +gpgKeyDisplay keyid userid = [whamlet| + + + \ + $maybe name <- userid + #{name} + $nothing + key id #{keyid} +|] + +genKeyModal :: Widget +genKeyModal = $(widgetFile "configurators/genkeymodal") + +isGcryptInstalled :: IO Bool +isGcryptInstalled = inPath "git-remote-gcrypt" + +whenGcryptInstalled :: Handler Html -> Handler Html +whenGcryptInstalled a = ifM (liftIO isGcryptInstalled) + ( a + , page "Need git-remote-gcrypt" (Just Configuration) $ + $(widgetFile "configurators/needgcrypt") + ) + +withNewSecretKey :: (KeyId -> Handler Html) -> Handler Html +withNewSecretKey use = do + cmd <- liftAnnex $ gpgCmd <$> Annex.getGitConfig + userid <- liftIO $ newUserId cmd + liftIO $ genSecretKey cmd RSA "" userid maxRecommendedKeySize + results <- M.keys . M.filter (== userid) <$> liftIO (secretKeys cmd) + case results of + [] -> giveup "Failed to generate gpg key!" + (key:_) -> use key + +{- Tries to find the name used in remote.log for a gcrypt repository + - with a given uuid. + - + - The gcrypt remote may not be on that is listed in the local remote.log + - (or the info may be out of date), so this actually fetches the git-annex + - branch from the gcrypt remote and merges it in, and then looks up + - the name. + -} +getGCryptRemoteName :: UUID -> String -> Annex RemoteName +getGCryptRemoteName u repoloc = do + tmpremote <- uniqueRemoteName "tmpgcryptremote" 0 <$> Annex.getGitRemotes + void $ inRepo $ Git.Command.runBool + [ Param "remote" + , Param "add" + , Param tmpremote + , Param $ Git.GCrypt.urlPrefix ++ repoloc + ] + mname <- ifM (inRepo $ Git.Command.runBool [Param "fetch", Param tmpremote]) + ( do + void Annex.Branch.forceUpdate + (M.lookup "name" <=< M.lookup u) <$> readRemoteLog + , return Nothing + ) + void $ inRepo $ Git.Remote.Remove.remove tmpremote + maybe missing return mname + where + missing = giveup $ "Cannot find configuration for the gcrypt remote at " ++ repoloc + +{- Checks to see if a repo is encrypted with gcrypt, and runs one action if + - it's not an another if it is. + - + - Since the probing requires gcrypt to be installed, a third action must + - be provided to run if it's not installed. + -} +checkGCryptRepoEncryption :: (Monad m, LiftAnnex m) => String -> m a -> m a -> m a -> m a +checkGCryptRepoEncryption location notencrypted notinstalled encrypted = + ifM (liftAnnex $ liftIO isGcryptInstalled) + ( dispatch =<< liftAnnex (inRepo $ Git.GCrypt.probeRepo location) + , notinstalled + ) + where + dispatch Git.GCrypt.Decryptable = encrypted + dispatch Git.GCrypt.NotEncrypted = notencrypted + dispatch Git.GCrypt.NotDecryptable = + giveup "This git repository is encrypted with a GnuPG key that you do not have." + +{- Gets the UUID of the gcrypt repo at a location, which may not exist. + - Only works if the gcrypt repo was created as a git-annex remote. -} +probeGCryptRemoteUUID :: String -> Annex (Maybe UUID) +probeGCryptRemoteUUID repolocation = do + r <- inRepo $ Git.Construct.fromRemoteLocation repolocation + GCrypt.getGCryptUUID False r diff --git a/Assistant/WebApp/MakeRemote.hs b/Assistant/WebApp/MakeRemote.hs new file mode 100644 index 0000000000..2575febeac --- /dev/null +++ b/Assistant/WebApp/MakeRemote.hs @@ -0,0 +1,46 @@ +{- git-annex assistant webapp making remotes + - + - Copyright 2012 Joey Hess + - + - Licensed under the GNU AGPL version 3 or higher. + -} + +module Assistant.WebApp.MakeRemote ( + module Assistant.MakeRemote, + module Assistant.WebApp.MakeRemote +) where + +import Assistant.Common +import Assistant.WebApp.Types +import Assistant.Sync +import qualified Remote +import qualified Types.Remote as Remote +import qualified Config +import Config.Cost +import Types.StandardGroups +import Git.Types (RemoteName) +import Logs.PreferredContent +import Assistant.MakeRemote + +import Utility.Yesod + +{- Runs an action that creates or enables a cloud remote, + - and finishes setting it up, then starts syncing with it, + - and finishes by displaying the page to edit it. + - + - This includes displaying the connectionNeeded nudge if appropariate. + -} +setupCloudRemote :: StandardGroup -> Maybe Cost -> Annex RemoteName -> Handler a +setupCloudRemote = setupRemote postsetup . Just + where + postsetup = redirect . EditNewCloudRepositoryR . Remote.uuid + +setupRemote :: (Remote -> Handler a) -> Maybe StandardGroup -> Maybe Cost -> Annex RemoteName -> Handler a +setupRemote postsetup mgroup mcost getname = do + r <- liftAnnex $ addRemote getname + repo <- liftAnnex $ Remote.getRepo r + liftAnnex $ do + maybe noop (defaultStandardGroup (Remote.uuid r)) mgroup + maybe noop (Config.setRemoteCost repo) mcost + liftAssistant $ syncRemote r + postsetup r diff --git a/Assistant/WebApp/Notifications.hs b/Assistant/WebApp/Notifications.hs new file mode 100644 index 0000000000..5fb66a2240 --- /dev/null +++ b/Assistant/WebApp/Notifications.hs @@ -0,0 +1,86 @@ +{- git-annex assistant webapp notifications + - + - Copyright 2012 Joey Hess + - + - Licensed under the GNU AGPL version 3 or higher. + -} + +{-# LANGUAGE QuasiQuotes, TemplateHaskell, OverloadedStrings #-} + +module Assistant.WebApp.Notifications where + +import Assistant.Common +import Assistant.WebApp +import Assistant.WebApp.Types +import Assistant.DaemonStatus +import Utility.NotificationBroadcaster +import Utility.Yesod +import Utility.AuthToken + +import Data.Text (Text) +import qualified Data.Text as T +import qualified Data.Aeson.Types as Aeson + +{- Add to any widget to make it auto-update using long polling. + - + - The widget should have a html element with an id=ident, which will be + - replaced when it's updated. + - + - The geturl route should return the notifier url to use for polling. + - + - ms_delay is how long to delay between AJAX updates + - ms_startdelay is how long to delay before updating with AJAX at the start + -} +autoUpdate :: Text -> Route WebApp -> Int -> Int -> Widget +autoUpdate tident geturl ms_delay ms_startdelay = do + let delay = Aeson.String (T.pack (show ms_delay)) + let startdelay = Aeson.String (T.pack (show ms_startdelay)) + let ident = Aeson.String tident + $(widgetFile "notifications/longpolling") + +{- Notifier urls are requested by the javascript, to avoid allocation + - of NotificationIds when noscript pages are loaded. This constructs a + - notifier url for a given Route and NotificationBroadcaster. + -} +notifierUrl :: (NotificationId -> Route WebApp) -> Assistant NotificationBroadcaster -> Handler RepPlain +notifierUrl route broadcaster = do + (urlbits, _params) <- renderRoute . route <$> newNotifier broadcaster + webapp <- getYesod + return $ RepPlain $ toContent $ T.concat + [ "/" + , T.intercalate "/" urlbits + , "?auth=" + , fromAuthToken (authToken webapp) + ] + +getNotifierTransfersR :: Handler RepPlain +getNotifierTransfersR = notifierUrl TransfersR getTransferBroadcaster + +getNotifierSideBarR :: Handler RepPlain +getNotifierSideBarR = notifierUrl SideBarR getAlertBroadcaster + +getNotifierRepoListR :: RepoSelector -> Handler RepPlain +getNotifierRepoListR reposelector = notifierUrl route getRepoListBroadcaster + where + route nid = RepoListR nid reposelector + +getNotifierGlobalRedirR :: Handler RepPlain +getNotifierGlobalRedirR = notifierUrl GlobalRedirR getGlobalRedirBroadcaster + +getTransferBroadcaster :: Assistant NotificationBroadcaster +getTransferBroadcaster = transferNotifier <$> getDaemonStatus + +getAlertBroadcaster :: Assistant NotificationBroadcaster +getAlertBroadcaster = alertNotifier <$> getDaemonStatus + +getRepoListBroadcaster :: Assistant NotificationBroadcaster +getRepoListBroadcaster = syncRemotesNotifier <$> getDaemonStatus + +getGlobalRedirBroadcaster :: Assistant NotificationBroadcaster +getGlobalRedirBroadcaster = globalRedirNotifier <$> getDaemonStatus + +getGlobalRedirR :: NotificationId -> Handler RepPlain +getGlobalRedirR nid = do + waitNotifier getGlobalRedirBroadcaster nid + maybe (getGlobalRedirR nid) (return . RepPlain . toContent . T.pack) + =<< globalRedirUrl <$> liftAssistant getDaemonStatus diff --git a/Assistant/WebApp/OtherRepos.hs b/Assistant/WebApp/OtherRepos.hs new file mode 100644 index 0000000000..96c2a02d87 --- /dev/null +++ b/Assistant/WebApp/OtherRepos.hs @@ -0,0 +1,38 @@ +{- git-annex assistant webapp switching to other repos + - + - Copyright 2012 Joey Hess + - + - Licensed under the GNU AGPL version 3 or higher. + -} + +{-# LANGUAGE QuasiQuotes, TemplateHaskell, OverloadedStrings #-} + +module Assistant.WebApp.OtherRepos where + +import Assistant.Common +import Assistant.WebApp.Types +import Assistant.WebApp.Page +import Config.Files +import Utility.Yesod +import Assistant.Restart + +getRepositorySwitcherR :: Handler Html +getRepositorySwitcherR = page "Switch repository" Nothing $ do + repolist <- liftIO listOtherRepos + $(widgetFile "control/repositoryswitcher") + +listOtherRepos :: IO [(String, String)] +listOtherRepos = do + dirs <- readAutoStartFile + pwd <- getCurrentDirectory + gooddirs <- filterM isrepo $ + filter (\d -> not $ d `dirContains` pwd) dirs + names <- mapM relHome gooddirs + return $ sort $ zip names gooddirs + where + isrepo d = doesDirectoryExist (d ".git") + +getSwitchToRepositoryR :: FilePath -> Handler Html +getSwitchToRepositoryR repo = do + liftIO $ addAutoStartFile repo -- make this the new default repo + redirect =<< liftIO (newAssistantUrl repo) diff --git a/Assistant/WebApp/Page.hs b/Assistant/WebApp/Page.hs new file mode 100644 index 0000000000..0e5f4fae3e --- /dev/null +++ b/Assistant/WebApp/Page.hs @@ -0,0 +1,82 @@ +{- git-annex assistant webapp page display + - + - Copyright 2012 Joey Hess + - + - Licensed under the GNU AGPL version 3 or higher. + -} + +{-# LANGUAGE TypeFamilies, QuasiQuotes, MultiParamTypeClasses, TemplateHaskell, OverloadedStrings, RankNTypes, CPP #-} + +module Assistant.WebApp.Page where + +import Assistant.Common +import Assistant.WebApp +import Assistant.WebApp.Types +import Assistant.WebApp.SideBar +import Utility.Yesod + +import qualified Text.Hamlet as Hamlet +import Data.Text (Text) + +data NavBarItem = DashBoard | Configuration | About + deriving (Eq, Ord, Enum, Bounded) + +navBarName :: NavBarItem -> Text +navBarName DashBoard = "Dashboard" +navBarName Configuration = "Configuration" +navBarName About = "About" + +navBarRoute :: NavBarItem -> Route WebApp +navBarRoute DashBoard = DashboardR +navBarRoute Configuration = ConfigurationR +navBarRoute About = AboutR + +defaultNavBar :: [NavBarItem] +defaultNavBar = [minBound .. maxBound] + +firstRunNavBar :: [NavBarItem] +firstRunNavBar = [Configuration, About] + +selectNavBar :: Handler [NavBarItem] +selectNavBar = ifM inFirstRun (return firstRunNavBar, return defaultNavBar) + +{- A standard page of the webapp, with a title, a sidebar, and that may + - be highlighted on the navbar. -} +page :: Hamlet.Html -> Maybe NavBarItem -> Widget -> Handler Html +page title navbaritem content = customPage navbaritem $ do + setTitle title + content + sideBarDisplay + +{- A custom page, with no title or sidebar set. -} +customPage :: Maybe NavBarItem -> Widget -> Handler Html +customPage = customPage' True + +customPage' :: Bool -> Maybe NavBarItem -> Widget -> Handler Html +customPage' with_longpolling navbaritem content = do + webapp <- getYesod + case cannotRun webapp of + Nothing -> do + navbar <- map navdetails <$> selectNavBar + pageinfo <- widgetToPageContent $ do + addStylesheet $ StaticR css_bootstrap_css + addStylesheet $ StaticR css_bootstrap_theme_css + addScript $ StaticR js_jquery_full_js + addScript $ StaticR js_bootstrap_js + when with_longpolling $ + addScript $ StaticR js_longpolling_js + $(widgetFile "page") + withUrlRenderer $(Hamlet.hamletFile $ hamletTemplate "bootstrap") + Just msg -> error msg + where + navdetails i = (navBarName i, navBarRoute i, Just i == navbaritem) + +hasFileBrowser :: Bool +#ifdef ANDROID_SPLICES +hasFileBrowser = False +#else +hasFileBrowser = True +#endif + +controlMenu :: Widget +controlMenu = $(widgetFile "controlmenu") diff --git a/Assistant/WebApp/Pairing.hs b/Assistant/WebApp/Pairing.hs new file mode 100644 index 0000000000..2c4086a5cf --- /dev/null +++ b/Assistant/WebApp/Pairing.hs @@ -0,0 +1,79 @@ +{- git-annex assistant pairing + - + - Copyright 2016 Joey Hess + - + - Licensed under the GNU AGPL version 3 or higher. + -} + +module Assistant.WebApp.Pairing where + +import Assistant.Common +import qualified Utility.MagicWormhole as Wormhole +import Command.P2P (wormholePairing, PairingResult(..)) +import P2P.Address +import Annex.Concurrent +import Git.Types + +import Control.Concurrent +import Control.Concurrent.Async +import Control.Concurrent.STM +import qualified Data.Map.Strict as M + +data PairingWith = PairingWithSelf | PairingWithFriend + deriving (Eq, Show, Read) + +type WormholePairingState = TVar (M.Map WormholePairingId WormholePairingHandle) + +type WormholePairingHandle = (PairingWith, RemoteName, MVar Wormhole.CodeObserver, MVar Wormhole.Code, Async (Annex PairingResult)) + +newtype WormholePairingId = WormholePairingId Int + deriving (Ord, Eq, Show, Read) + +newWormholePairingState :: IO WormholePairingState +newWormholePairingState = newTVarIO M.empty + +addWormholePairingState :: WormholePairingHandle -> WormholePairingState -> IO WormholePairingId +addWormholePairingState h tv = atomically $ do + m <- readTVar tv + -- use of head is safe because allids is infinite + let i = Prelude.head $ filter (`notElem` M.keys m) allids + writeTVar tv (M.insert i h m) + return i + where + allids = map WormholePairingId [1..] + +-- | Starts the wormhole pairing processes. +startWormholePairing :: PairingWith -> RemoteName -> [P2PAddress] -> Assistant WormholePairingHandle +startWormholePairing pairingwith remotename ouraddrs = do + observerrelay <- liftIO newEmptyMVar + producerrelay <- liftIO newEmptyMVar + -- wormholePairing needs to run in the Annex monad, and is a + -- long-duration action. So, don't just liftAnnex to run it; + -- fork the Annex state. + runner <- liftAnnex $ forkState $ + wormholePairing remotename ouraddrs $ \observer producer -> do + putMVar observerrelay observer + theircode <- takeMVar producerrelay + Wormhole.sendCode producer theircode + tid <- liftIO $ async runner + return (pairingwith, remotename, observerrelay, producerrelay, tid) + +-- | Call after sendTheirWormholeCode. This can take some time to return. +finishWormholePairing :: WormholePairingHandle -> Assistant PairingResult +finishWormholePairing (_, _, _, _, tid) = liftAnnex =<< liftIO (wait tid) + +-- | Waits for wormhole to produce our code. Can be called repeatedly, safely. +getOurWormholeCode :: WormholePairingHandle -> IO Wormhole.Code +getOurWormholeCode (_, _, observerrelay, _, _) = + readMVar observerrelay >>= Wormhole.waitCode + +-- | Sends their code to wormhole. If their code has already been sent, +-- avoids blocking and returns False. +sendTheirWormholeCode :: WormholePairingHandle -> Wormhole.Code -> IO Bool +sendTheirWormholeCode (_, _, _, producerrelay, _) = tryPutMVar producerrelay + +withPairingWith :: WormholePairingHandle -> (PairingWith -> a) -> a +withPairingWith (pairingwith, _, _, _, _) a = a pairingwith + +withRemoteName :: WormholePairingHandle -> (RemoteName -> a) -> a +withRemoteName (_, remotename, _, _, _) a = a remotename diff --git a/Assistant/WebApp/Repair.hs b/Assistant/WebApp/Repair.hs new file mode 100644 index 0000000000..ded075cedc --- /dev/null +++ b/Assistant/WebApp/Repair.hs @@ -0,0 +1,35 @@ +{- git-annex assistant repository repair + - + - Copyright 2013 Joey Hess + - + - Licensed under the GNU AGPL version 3 or higher. + -} + +{-# LANGUAGE QuasiQuotes, TemplateHaskell, OverloadedStrings #-} + +module Assistant.WebApp.Repair where + +import Assistant.WebApp.Common +import Assistant.WebApp.RepoList +import Remote (prettyUUID, remoteFromUUID) +import Annex.UUID (getUUID) +import Assistant.Repair + +getRepairRepositoryR :: UUID -> Handler Html +getRepairRepositoryR = postRepairRepositoryR +postRepairRepositoryR :: UUID -> Handler Html +postRepairRepositoryR u = page "Repair repository" Nothing $ do + repodesc <- liftAnnex $ prettyUUID u + repairingmainrepo <- (==) u <$> liftAnnex getUUID + $(widgetFile "control/repairrepository") + +getRepairRepositoryRunR :: UUID -> Handler Html +getRepairRepositoryRunR = postRepairRepositoryRunR +postRepairRepositoryRunR :: UUID -> Handler Html +postRepairRepositoryRunR u = do + r <- liftAnnex $ remoteFromUUID u + void $ liftAssistant $ runRepair u r True + page "Repair repository" Nothing $ do + let repolist = repoListDisplay $ + mainRepoSelector { nudgeAddMore = True } + $(widgetFile "control/repairrepository/done") diff --git a/Assistant/WebApp/RepoId.hs b/Assistant/WebApp/RepoId.hs new file mode 100644 index 0000000000..99cdb85113 --- /dev/null +++ b/Assistant/WebApp/RepoId.hs @@ -0,0 +1,40 @@ +{- git-annex assistant webapp RepoId type + - + - Copyright 2012,2013 Joey Hess + - + - Licensed under the GNU AGPL version 3 or higher. + -} + +module Assistant.WebApp.RepoId where + +import Annex.Common +import Git.Types (RemoteName) +import qualified Remote + +{- Parts of the webapp need to be able to act on repositories that may or + - may not have a UUID. -} +data RepoId + = RepoUUID UUID + | RepoName RemoteName + deriving (Eq, Ord, Show, Read) + +mkRepoId :: Remote -> RepoId +mkRepoId r = case Remote.uuid r of + NoUUID -> RepoName (Remote.name r) + u -> RepoUUID u + + +describeRepoId :: RepoId -> Annex String +describeRepoId (RepoUUID u) = Remote.prettyUUID u +describeRepoId (RepoName n) = return n + +repoIdRemote :: RepoId -> Annex (Maybe Remote) +repoIdRemote (RepoUUID u) = Remote.remoteFromUUID u +repoIdRemote (RepoName n) = Remote.byNameOnly n + +lacksUUID :: RepoId -> Bool +lacksUUID r = asUUID r == NoUUID + +asUUID :: RepoId -> UUID +asUUID (RepoUUID u) = u +asUUID _ = NoUUID diff --git a/Assistant/WebApp/RepoList.hs b/Assistant/WebApp/RepoList.hs new file mode 100644 index 0000000000..092557d578 --- /dev/null +++ b/Assistant/WebApp/RepoList.hs @@ -0,0 +1,269 @@ +{- git-annex assistant webapp repository list + - + - Copyright 2012,2013 Joey Hess + - + - Licensed under the GNU AGPL version 3 or higher. + -} + +{-# LANGUAGE QuasiQuotes, TemplateHaskell, OverloadedStrings, CPP #-} + +module Assistant.WebApp.RepoList where + +import Assistant.WebApp.Common +import Assistant.DaemonStatus +import Assistant.WebApp.Notifications +import qualified Remote +import qualified Types.Remote as Remote +import Remote.List (remoteListRefresh) +import Annex.UUID (getUUID) +import Logs.Remote +import Logs.Trust +import Logs.Group +import Config +import Config.GitConfig +import Git.Remote +import Assistant.Sync +import Config.Cost +import Utility.NotificationBroadcaster +import qualified Git + +import qualified Data.Map as M +import qualified Data.Set as S +import qualified Data.Text as T +import Data.Function +import Control.Concurrent + +type RepoList = [(RepoDesc, RepoId, CurrentlyConnected, Actions)] + +type RepoDesc = String +type CurrentlyConnected = Bool + +{- Actions that can be performed on a repo in the list. -} +data Actions + = DisabledRepoActions + { setupRepoLink :: Route WebApp } + | SyncingRepoActions + { setupRepoLink :: Route WebApp + , syncToggleLink :: Route WebApp + } + | NotSyncingRepoActions + { setupRepoLink :: Route WebApp + , syncToggleLink :: Route WebApp + } + | UnwantedRepoActions + { setupRepoLink :: Route WebApp } + +mkSyncingRepoActions :: RepoId -> Actions +mkSyncingRepoActions repoid = SyncingRepoActions + { setupRepoLink = EditRepositoryR repoid + , syncToggleLink = DisableSyncR repoid + } + +mkNotSyncingRepoActions :: RepoId -> Actions +mkNotSyncingRepoActions repoid = NotSyncingRepoActions + { setupRepoLink = EditRepositoryR repoid + , syncToggleLink = EnableSyncR repoid + } + +mkUnwantedRepoActions :: RepoId -> Actions +mkUnwantedRepoActions repoid = UnwantedRepoActions + { setupRepoLink = EditRepositoryR repoid + } + +needsEnabled :: Actions -> Bool +needsEnabled (DisabledRepoActions _) = True +needsEnabled _ = False + +notSyncing :: Actions -> Bool +notSyncing (SyncingRepoActions _ _) = False +notSyncing _ = True + +notWanted :: Actions -> Bool +notWanted (UnwantedRepoActions _) = True +notWanted _ = False + +{- Called by client to get a list of repos, that refreshes + - when new repos are added. + - + - Returns a div, which will be inserted into the calling page. + -} +getRepoListR :: NotificationId -> RepoSelector -> Handler Html +getRepoListR nid reposelector = do + waitNotifier getRepoListBroadcaster nid + p <- widgetToPageContent $ repoListDisplay reposelector + withUrlRenderer $ [hamlet|^{pageBody p}|] + +mainRepoSelector :: RepoSelector +mainRepoSelector = RepoSelector + { onlyCloud = False + , onlyConfigured = False + , includeHere = True + , nudgeAddMore = False + } + +{- List of cloud repositories, configured and not. -} +cloudRepoList :: Widget +cloudRepoList = repoListDisplay RepoSelector + { onlyCloud = True + , onlyConfigured = False + , includeHere = False + , nudgeAddMore = False + } + +repoListDisplay :: RepoSelector -> Widget +repoListDisplay reposelector = do + autoUpdate ident (NotifierRepoListR reposelector) (10 :: Int) (10 :: Int) + addScript $ StaticR js_jquery_ui_core_js + addScript $ StaticR js_jquery_ui_widget_js + addScript $ StaticR js_jquery_ui_mouse_js + addScript $ StaticR js_jquery_ui_sortable_js + + repolist <- liftH $ repoList reposelector + let addmore = nudgeAddMore reposelector + let nootherrepos = length repolist < 2 + + $(widgetFile "repolist") + where + ident = "repolist" + +{- A list of known repositories, with actions that can be taken on them. -} +repoList :: RepoSelector -> Handler RepoList +repoList reposelector + | onlyConfigured reposelector = list =<< configured + | otherwise = list =<< (++) <$> configured <*> unconfigured + where + configured = do + syncremotes <- syncRemotes <$> liftAssistant getDaemonStatus + let syncing = S.fromList $ map mkRepoId syncremotes + liftAnnex $ do + unwanted <- S.fromList + <$> filterM inUnwantedGroup (map Remote.uuid syncremotes) + trustmap <- trustMap + allrs <- concat . Remote.byCost <$> Remote.remoteList + rs <- filter (\r -> M.lookup (Remote.uuid r) trustmap /= Just DeadTrusted) + . map fst + . filter selectedrepo + <$> forM allrs (\r -> (,) <$> pure r <*> Remote.getRepo r) + let l = flip map (map mkRepoId rs) $ \r -> case r of + (RepoUUID u) + | u `S.member` unwanted -> (r, mkUnwantedRepoActions r) + _ + | r `S.member` syncing -> (r, mkSyncingRepoActions r) + | otherwise -> (r, mkNotSyncingRepoActions r) + if includeHere reposelector + then do + r <- RepoUUID <$> getUUID + autocommit <- getGitConfigVal annexAutoCommit + let hereactions = if autocommit + then mkSyncingRepoActions r + else mkNotSyncingRepoActions r + let here = (r, hereactions) + return $ here : l + else return l + unconfigured = liftAnnex $ do + m <- readRemoteLog + g <- gitRepo + map snd . catMaybes . filter selectedremote + . map (findinfo m g) + <$> trustExclude DeadTrusted (M.keys m) + selectedrepo (r, repo) + | Remote.readonly r = False + | onlyCloud reposelector = Git.repoIsUrl repo + && Remote.uuid r /= NoUUID + | otherwise = True + selectedremote Nothing = False + selectedremote (Just (iscloud, _)) + | onlyCloud reposelector = iscloud + | otherwise = True + findinfo m g u = case getconfig "type" of + Just "rsync" -> val True EnableRsyncR + Just "directory" -> val False EnableDirectoryR +#ifdef WITH_S3 + Just "S3" -> val True EnableS3R +#endif + Just "glacier" -> val True EnableGlacierR +#ifdef WITH_WEBDAV + Just "webdav" -> val True EnableWebDAVR +#endif + Just "gcrypt" -> + -- Skip gcrypt repos on removable drives; + -- handled separately. + case getconfig "gitrepo" of + Just rr | remoteLocationIsUrl (parseRemoteLocation rr g) -> + val True EnableSshGCryptR + _ -> Nothing + Just "git" -> + case getconfig "location" of + Just loc | remoteLocationIsSshUrl (parseRemoteLocation loc g) -> + val True EnableSshGitRemoteR + _ -> Nothing + _ -> Nothing + where + getconfig k = M.lookup k =<< M.lookup u m + val iscloud r = Just (iscloud, (RepoUUID u, DisabledRepoActions $ r u)) + list l = do + cc <- currentlyConnectedRemotes <$> liftAssistant getDaemonStatus + forM (nubBy ((==) `on` fst) l) $ \(repoid, actions) -> + (,,,) + <$> liftAnnex (describeRepoId repoid) + <*> pure repoid + <*> pure (getCurrentlyConnected repoid cc) + <*> pure actions + +getCurrentlyConnected :: RepoId -> S.Set UUID -> CurrentlyConnected +getCurrentlyConnected (RepoUUID u) cc = S.member u cc +getCurrentlyConnected _ _ = False + +getEnableSyncR :: RepoId -> Handler () +getEnableSyncR = flipSync True + +getDisableSyncR :: RepoId -> Handler () +getDisableSyncR = flipSync False + +flipSync :: Bool -> RepoId -> Handler () +flipSync enable repoid = do + mremote <- liftAnnex $ repoIdRemote repoid + liftAssistant $ changeSyncable mremote enable + redirectBack + +getRepositoriesReorderR :: Handler () +getRepositoriesReorderR = do + {- Get uuid of the moved item, and the list it was moved within. -} + moved <- fromjs <$> runInputGet (ireq textField "moved") + list <- map fromjs <$> lookupGetParams "list[]" + liftAnnex $ go list =<< repoIdRemote moved + liftAssistant updateSyncRemotes + where + go _ Nothing = noop + go list (Just remote) = do + rs <- catMaybes <$> mapM repoIdRemote list + forM_ (reorderCosts remote rs) $ \(r, newcost) -> + when (Remote.cost r /= newcost) $ do + repo <- Remote.getRepo r + setRemoteCost repo newcost + void remoteListRefresh + fromjs = fromMaybe (RepoUUID NoUUID) . readish . T.unpack + +reorderCosts :: Remote -> [Remote] -> [(Remote, Cost)] +reorderCosts remote rs = zip rs'' (insertCostAfter costs i) + where + {- Find the index of the remote in the list that the remote + - was moved to be after. + - If it was moved to the start of the list, -1 -} + i = fromMaybe 0 (elemIndex remote rs) - 1 + rs' = filter (\r -> Remote.uuid r /= Remote.uuid remote) rs + costs = map Remote.cost rs' + rs'' = (\(x, y) -> x ++ [remote] ++ y) $ splitAt (i + 1) rs' + +getSyncNowRepositoryR :: UUID -> Handler () +getSyncNowRepositoryR uuid = do + u <- liftAnnex getUUID + if u == uuid + then do + thread <- liftAssistant $ asIO $ + reconnectRemotes + =<< (syncRemotes <$> getDaemonStatus) + void $ liftIO $ forkIO thread + else maybe noop (liftAssistant . syncRemote) + =<< liftAnnex (Remote.remoteFromUUID uuid) + redirectBack diff --git a/Assistant/WebApp/SideBar.hs b/Assistant/WebApp/SideBar.hs new file mode 100644 index 0000000000..51279f24a2 --- /dev/null +++ b/Assistant/WebApp/SideBar.hs @@ -0,0 +1,109 @@ +{- git-annex assistant webapp sidebar + - + - Copyright 2012 Joey Hess + - + - Licensed under the GNU AGPL version 3 or higher. + -} + +{-# LANGUAGE QuasiQuotes, TemplateHaskell, OverloadedStrings, FlexibleContexts #-} + +module Assistant.WebApp.SideBar where + +import Assistant.Common +import Assistant.WebApp +import Assistant.WebApp.Types +import Assistant.WebApp.Notifications +import Assistant.Alert.Utility +import Assistant.DaemonStatus +import Utility.NotificationBroadcaster +import Utility.Yesod + +import Data.Text (Text) +import qualified Data.Text as T +import qualified Data.Map as M +import Control.Concurrent + +sideBarDisplay :: Widget +sideBarDisplay = do + let content = do + {- Add newest alerts to the sidebar. -} + alertpairs <- liftH $ M.toList . alertMap + <$> liftAssistant getDaemonStatus + mapM_ renderalert $ + take displayAlerts $ reverse $ sortAlertPairs alertpairs + let ident = "sidebar" + $(widgetFile "sidebar/main") + autoUpdate ident NotifierSideBarR (10 :: Int) (10 :: Int) + where + bootstrapclass :: AlertClass -> Text + bootstrapclass Activity = "alert-info" + bootstrapclass Warning = "alert" + bootstrapclass Error = "alert-danger" + bootstrapclass Success = "alert-success" + bootstrapclass Message = "alert-info" + + renderalert (aid, alert) = do + let alertid = show aid + let closable = alertClosable alert + let block = alertBlockDisplay alert + let divclass = bootstrapclass $ alertClass alert + let message = renderAlertMessage alert + let messagelines = T.lines message + let multiline = length messagelines > 1 + let buttons = zip (alertButtons alert) [1..] + $(widgetFile "sidebar/alert") + +{- Called by client to get a sidebar display. + - + - Returns a div, which will be inserted into the calling page. + - + - Note that the head of the widget is not included, only its + - body is. To get the widget head content, the widget is also + - inserted onto all pages. + -} +getSideBarR :: NotificationId -> Handler Html +getSideBarR nid = do + waitNotifier getAlertBroadcaster nid + + {- This 0.1 second delay avoids very transient notifications from + - being displayed and churning the sidebar unnecesarily. + - + - This needs to be below the level perceptable by the user, + - to avoid slowing down user actions like closing alerts. -} + liftIO $ threadDelay 100000 + + page <- widgetToPageContent sideBarDisplay + withUrlRenderer $ [hamlet|^{pageBody page}|] + +{- Called by the client to close an alert. -} +getCloseAlert :: AlertId -> Handler () +getCloseAlert = liftAssistant . removeAlert + +{- When an alert with a button is clicked on, the button takes us here. -} +getClickAlert :: AlertId -> Int -> Handler () +getClickAlert i bnum = do + m <- alertMap <$> liftAssistant getDaemonStatus + case M.lookup i m of + Just (Alert { alertButtons = bs }) + | length bs >= bnum -> do + let b = bs !! (bnum - 1) + {- Spawn a thread to run the action + - while redirecting. -} + case buttonAction b of + Nothing -> noop + Just a -> liftIO $ void $ forkIO $ a i + redirect $ buttonUrl b + | otherwise -> redirectBack + _ -> redirectBack + +htmlIcon :: AlertIcon -> Widget +htmlIcon ActivityIcon = [whamlet||] +htmlIcon SyncIcon = [whamlet||] +htmlIcon InfoIcon = bootstrapIcon "info-sign" +htmlIcon SuccessIcon = bootstrapIcon "ok" +htmlIcon ErrorIcon = bootstrapIcon "exclamation-sign" +htmlIcon UpgradeIcon = bootstrapIcon "arrow-up" +htmlIcon ConnectionIcon = bootstrapIcon "signal" + +bootstrapIcon :: Text -> Widget +bootstrapIcon name = [whamlet||] diff --git a/Assistant/WebApp/Types.hs b/Assistant/WebApp/Types.hs new file mode 100644 index 0000000000..11da2722bc --- /dev/null +++ b/Assistant/WebApp/Types.hs @@ -0,0 +1,205 @@ +{- git-annex assistant webapp types + - + - Copyright 2012 Joey Hess + - + - Licensed under the GNU AGPL version 3 or higher. + -} + +{-# LANGUAGE TypeFamilies, QuasiQuotes, MultiParamTypeClasses #-} +{-# LANGUAGE TemplateHaskell, OverloadedStrings, RankNTypes #-} +{-# LANGUAGE FlexibleInstances, FlexibleContexts, ViewPatterns #-} +{-# LANGUAGE CPP #-} +{-# OPTIONS_GHC -fno-warn-orphans #-} + +module Assistant.WebApp.Types ( + module Assistant.WebApp.Types, + Route +) where + +import Assistant.Common +import Assistant.Ssh +import Assistant.Pairing +import Utility.NotificationBroadcaster +import Utility.AuthToken +import Utility.WebApp +import Utility.Yesod +import Types.Transfer +import Utility.Gpg (KeyId) +import BuildInfo (packageversion) +import Types.ScheduledActivity +import Assistant.WebApp.RepoId +import Assistant.WebApp.Pairing +import Types.Distribution + +import Yesod.Static +import Text.Hamlet +import Data.Text (Text, pack, unpack) +import Network.Socket (HostName) + +publicFiles "static" + +staticRoutes :: Static +staticRoutes = $(embed "static") + +data WebApp = WebApp + { assistantData :: AssistantData + , authToken :: AuthToken + , relDir :: Maybe FilePath + , getStatic :: Static + , postFirstRun :: Maybe (IO String) + , cannotRun :: Maybe String + , noAnnex :: Bool + , listenHost ::Maybe HostName + , wormholePairingState :: WormholePairingState + } + +mkYesodData "WebApp" $(parseRoutesFile "Assistant/WebApp/routes") + +excludeStatic :: [Text] -> Bool +excludeStatic [] = True +excludeStatic (p:_) = p /= "static" + +instance Yesod WebApp where + {- Require an auth token be set when accessing any (non-static) route -} + isAuthorized r _ = checkAuthToken authToken r excludeStatic + + {- Add the auth token to every url generated, except static subsite + - urls (which can show up in Permission Denied pages). -} + joinPath = insertAuthToken authToken excludeStatic + + makeSessionBackend = webAppSessionBackend + jsLoader _ = BottomOfHeadBlocking + + {- The webapp does not use defaultLayout, so this is only used + - for error pages or any other built-in yesod page. + - + - This can use static routes, but should use no other routes, + - as that would expose the auth token. + -} + defaultLayout content = do + webapp <- getYesod + pageinfo <- widgetToPageContent $ do + addStylesheet $ StaticR css_bootstrap_css + addStylesheet $ StaticR css_bootstrap_theme_css + addScript $ StaticR js_jquery_full_js + addScript $ StaticR js_bootstrap_js + $(widgetFile "error") + withUrlRenderer $(hamletFile $ hamletTemplate "bootstrap") + +instance RenderMessage WebApp FormMessage where + renderMessage _ _ = defaultFormMessage + +instance LiftAnnex Handler where + liftAnnex a = ifM (noAnnex <$> getYesod) + ( error "internal liftAnnex" + , liftAssistant $ liftAnnex a + ) + +#if MIN_VERSION_yesod_core(1,6,0) +instance LiftAnnex (WidgetFor WebApp) where +#else +instance LiftAnnex (WidgetT WebApp IO) where +#endif + liftAnnex = liftH . liftAnnex + +class LiftAssistant m where + liftAssistant :: Assistant a -> m a + +instance LiftAssistant Handler where + liftAssistant a = liftIO . flip runAssistant a + =<< assistantData <$> getYesod + +#if MIN_VERSION_yesod_core(1,6,0) +instance LiftAssistant (WidgetFor WebApp) where +#else +instance LiftAssistant (WidgetT WebApp IO) where +#endif + liftAssistant = liftH . liftAssistant + +type MkMForm x = MForm Handler (FormResult x, Widget) + +type MkAForm x = AForm Handler x + +type MkField x = forall m. Monad m => RenderMessage (HandlerSite m) FormMessage => Field m x + +data RepoSelector = RepoSelector + { onlyCloud :: Bool + , onlyConfigured :: Bool + , includeHere :: Bool + , nudgeAddMore :: Bool + } + deriving (Read, Show, Eq) + +data RemovableDrive = RemovableDrive + { diskFree :: Maybe Integer + , mountPoint :: Text + , driveRepoPath :: Text + } + deriving (Read, Show, Eq, Ord) + +data RepoKey = RepoKey KeyId | NoRepoKey + deriving (Read, Show, Eq, Ord) + +instance PathPiece RemovableDrive where + toPathPiece = pack . show + fromPathPiece = readish . unpack + +instance PathPiece RepoKey where + toPathPiece = pack . show + fromPathPiece = readish . unpack + +instance PathPiece SshData where + toPathPiece = pack . show + fromPathPiece = readish . unpack + +instance PathPiece NotificationId where + toPathPiece = pack . show + fromPathPiece = readish . unpack + +instance PathPiece AlertId where + toPathPiece = pack . show + fromPathPiece = readish . unpack + +instance PathPiece Transfer where + toPathPiece = pack . show + fromPathPiece = readish . unpack + +instance PathPiece PairMsg where + toPathPiece = pack . show + fromPathPiece = readish . unpack + +instance PathPiece SecretReminder where + toPathPiece = pack . show + fromPathPiece = readish . unpack + +instance PathPiece UUID where + toPathPiece = pack . show + fromPathPiece = readish . unpack + +instance PathPiece RepoSelector where + toPathPiece = pack . show + fromPathPiece = readish . unpack + +instance PathPiece ThreadName where + toPathPiece = pack . show + fromPathPiece = readish . unpack + +instance PathPiece ScheduledActivity where + toPathPiece = pack . show + fromPathPiece = readish . unpack + +instance PathPiece RepoId where + toPathPiece = pack . show + fromPathPiece = readish . unpack + +instance PathPiece GitAnnexDistribution where + toPathPiece = pack . show + fromPathPiece = readish . unpack + +instance PathPiece PairingWith where + toPathPiece = pack . show + fromPathPiece = readish . unpack + +instance PathPiece WormholePairingId where + toPathPiece = pack . show + fromPathPiece = readish . unpack diff --git a/Assistant/WebApp/routes b/Assistant/WebApp/routes new file mode 100644 index 0000000000..9be10ba940 --- /dev/null +++ b/Assistant/WebApp/routes @@ -0,0 +1,117 @@ +/ DashboardR GET HEAD + +/noscript NoScriptR GET +/noscript/auto NoScriptAutoR GET + +/about AboutR GET +/about/license LicenseR GET +/about/repogroups RepoGroupR GET + +/shutdown ShutdownR GET +/shutdown/confirm ShutdownConfirmedR GET +/shutdown/complete NotRunningR GET +/restart RestartR GET +/restart/thread/#ThreadName RestartThreadR GET +/log LogR GET + +/config ConfigurationR GET +/config/preferences PreferencesR GET POST +/config/needconnection ConnectionNeededR GET +/config/fsck ConfigFsckR GET POST +/config/fsck/preferences ConfigFsckPreferencesR POST +/config/upgrade/start/#GitAnnexDistribution ConfigStartUpgradeR GET +/config/upgrade/finish ConfigFinishUpgradeR GET +/config/upgrade/automatically ConfigEnableAutomaticUpgradeR GET +/config/unused ConfigUnusedR GET POST + +/config/addrepository AddRepositoryR GET +/config/repository/new NewRepositoryR GET POST +/config/repository/new/first FirstRepositoryR GET POST +/config/repository/new/androidcamera AndroidCameraRepositoryR GET +/config/repository/switcher RepositorySwitcherR GET +/config/repository/switchto/#FilePath SwitchToRepositoryR GET +/config/repository/combine/#FilePath/#UUID CombineRepositoryR GET +/config/repository/edit/#RepoId EditRepositoryR GET POST +/config/repository/edit/new/#UUID EditNewRepositoryR GET POST +/config/repository/edit/new/cloud/#UUID EditNewCloudRepositoryR GET POST +/config/repository/sync/disable/#RepoId DisableSyncR GET +/config/repository/sync/enable/#RepoId EnableSyncR GET +/config/repository/upgrade/#RepoId UpgradeRepositoryR GET + +/config/repository/add/drive AddDriveR GET POST +/config/repository/add/drive/confirm/#RemovableDrive ConfirmAddDriveR GET +/config/repository/add/drive/genkey/#RemovableDrive GenKeyForDriveR GET +/config/repository/add/drive/finish/#RemovableDrive/#RepoKey FinishAddDriveR GET +/config/repository/add/ssh AddSshR GET POST +/config/repository/add/ssh/confirm/#SshData/#UUID ConfirmSshR GET +/config/repository/add/ssh/retry/#SshData RetrySshR GET +/config/repository/add/ssh/make/git/#SshData MakeSshGitR GET +/config/repository/add/ssh/make/rsync/#SshData MakeSshRsyncR GET +/config/repository/add/ssh/make/gcrypt/#SshData/#RepoKey MakeSshGCryptR GET +/config/repository/add/ssh/combine/#SshData CombineSshR GET +/config/repository/add/cloud/rsync.net AddRsyncNetR GET POST +/config/repository/add/cloud/rsync.net/shared/#SshData MakeRsyncNetSharedR GET +/config/repository/add/cloud/rsync.net/gcrypt/#SshData/#RepoKey MakeRsyncNetGCryptR GET +/config/repository/add/cloud/S3 AddS3R GET POST +/config/repository/add/cloud/IA AddIAR GET POST +/config/repository/add/cloud/glacier AddGlacierR GET POST +/config/repository/add/cloud/box.com AddBoxComR GET POST +/config/repository/add/cloud/gitlab.com AddGitLabR GET POST + +/config/repository/pair/local/start StartLocalPairR GET POST +/config/repository/pair/local/running/#SecretReminder RunningLocalPairR GET +/config/repository/pair/local/finish/#PairMsg FinishLocalPairR GET POST + +/config/repository/pair/wormhole/start/self StartWormholePairSelfR GET +/config/repository/pair/wormhole/start/friend StartWormholePairFriendR GET +/config/repository/pair/wormhole/prepare/#PairingWith PrepareWormholePairR GET +/config/repository/pair/wormhole/running/#WormholePairingId RunningWormholePairR GET POST + +/config/repository/enable/rsync/#UUID EnableRsyncR GET POST +/config/repository/enable/gcrypt/#UUID EnableSshGCryptR GET POST +/config/repository/enable/directory/#UUID EnableDirectoryR GET +/config/repository/enable/S3/#UUID EnableS3R GET POST +/config/repository/enable/IA/#UUID EnableIAR GET POST +/config/repository/enable/glacier/#UUID EnableGlacierR GET POST +/config/repository/enable/webdav/#UUID EnableWebDAVR GET POST +/config/repository/enable/sshgitremote/#UUID EnableSshGitRemoteR GET POST + +/config/repository/reorder RepositoriesReorderR GET + +/config/repository/syncnow/#UUID SyncNowRepositoryR GET + +/config/repository/delete/confirm/#UUID DeleteRepositoryR GET +/config/repository/delete/start/#UUID StartDeleteRepositoryR GET +/config/repository/delete/finish/#UUID FinishDeleteRepositoryR GET +/config/repository/delete/here DeleteCurrentRepositoryR GET POST + +/config/activity/add/#UUID AddActivityR GET POST +/config/activity/change/#UUID/#ScheduledActivity ChangeActivityR GET POST +/config/activity/remove/#UUID/#ScheduledActivity RemoveActivityR GET + +/transfers/#NotificationId TransfersR GET +/notifier/transfers NotifierTransfersR GET + +/sidebar/#NotificationId SideBarR GET +/notifier/sidebar NotifierSideBarR GET + +/repolist/#NotificationId/#RepoSelector RepoListR GET +/notifier/repolist/#RepoSelector NotifierRepoListR GET + +/globalredir/#NotificationId GlobalRedirR GET +/notifier/globalredir NotifierGlobalRedirR GET + +/alert/close/#AlertId CloseAlert GET +/alert/click/#AlertId/#Int ClickAlert GET +/filebrowser FileBrowserR GET POST + +/transfer/pause/#Transfer PauseTransferR GET POST +/transfer/start/#Transfer StartTransferR GET POST +/transfer/cancel/#Transfer CancelTransferR GET POST + +/repair/#UUID RepairRepositoryR GET POST +/repair/run/#UUID RepairRepositoryRunR GET POST + +/unused/cleanup CleanupUnusedR GET + +/static StaticR Static getStatic diff --git a/Backend.hs b/Backend.hs new file mode 100644 index 0000000000..af033a63b6 --- /dev/null +++ b/Backend.hs @@ -0,0 +1,96 @@ +{- git-annex key/value backends + - + - Copyright 2010-2017 Joey Hess + - + - Licensed under the GNU GPL version 3 or higher. + -} + +module Backend ( + list, + defaultBackend, + genKey, + getBackend, + chooseBackend, + lookupBackendVariety, + maybeLookupBackendVariety, + isStableKey, +) where + +import Annex.Common +import qualified Annex +import Annex.CheckAttr +import Types.Key +import Types.KeySource +import qualified Types.Backend as B + +-- When adding a new backend, import it here and add it to the list. +import qualified Backend.Hash +import qualified Backend.WORM +import qualified Backend.URL + +import qualified Data.Map as M + +list :: [Backend] +list = Backend.Hash.backends ++ Backend.WORM.backends ++ Backend.URL.backends + +{- Backend to use by default when generating a new key. -} +defaultBackend :: Annex Backend +defaultBackend = maybe cache return =<< Annex.getState Annex.backend + where + cache = do + n <- maybe (annexBackend <$> Annex.getGitConfig) (return . Just) + =<< Annex.getState Annex.forcebackend + let b = case n of + Just name | valid name -> lookupname name + _ -> Prelude.head list + Annex.changeState $ \s -> s { Annex.backend = Just b } + return b + valid name = not (null name) + lookupname = lookupBackendVariety . parseKeyVariety + +{- Generates a key for a file. -} +genKey :: KeySource -> Maybe Backend -> Annex (Maybe (Key, Backend)) +genKey source preferredbackend = do + b <- maybe defaultBackend return preferredbackend + B.getKey b source >>= return . \case + Nothing -> Nothing + Just k -> Just (makesane k, b) + where + -- keyNames should not contain newline characters. + makesane k = k { keyName = map fixbadchar (keyName k) } + fixbadchar c + | c == '\n' = '_' + | otherwise = c + +getBackend :: FilePath -> Key -> Annex (Maybe Backend) +getBackend file k = case maybeLookupBackendVariety (keyVariety k) of + Just backend -> return $ Just backend + Nothing -> do + warning $ "skipping " ++ file ++ " (unknown backend " ++ formatKeyVariety (keyVariety k) ++ ")" + return Nothing + +{- Looks up the backend that should be used for a file. + - That can be configured on a per-file basis in the gitattributes file, + - or forced with --backend. -} +chooseBackend :: FilePath -> Annex (Maybe Backend) +chooseBackend f = Annex.getState Annex.forcebackend >>= go + where + go Nothing = maybeLookupBackendVariety . parseKeyVariety + <$> checkAttr "annex.backend" f + go (Just _) = Just <$> defaultBackend + +{- Looks up a backend by variety. May fail if unsupported or disabled. -} +lookupBackendVariety :: KeyVariety -> Backend +lookupBackendVariety v = fromMaybe unknown $ maybeLookupBackendVariety v + where + unknown = giveup $ "unknown backend " ++ formatKeyVariety v + +maybeLookupBackendVariety :: KeyVariety -> Maybe Backend +maybeLookupBackendVariety v = M.lookup v varietyMap + +varietyMap :: M.Map KeyVariety Backend +varietyMap = M.fromList $ zip (map B.backendVariety list) list + +isStableKey :: Key -> Bool +isStableKey k = maybe False (`B.isStableKey` k) + (maybeLookupBackendVariety (keyVariety k)) diff --git a/Backend/Hash.hs b/Backend/Hash.hs new file mode 100644 index 0000000000..7d8e335854 --- /dev/null +++ b/Backend/Hash.hs @@ -0,0 +1,289 @@ +{- git-annex hashing backends + - + - Copyright 2011-2018 Joey Hess + - + - Licensed under the GNU GPL version 3 or higher. + -} + +{-# LANGUAGE CPP #-} + +module Backend.Hash ( + backends, + testKeyBackend, +) where + +import Annex.Common +import qualified Annex +import Types.Key +import Types.Backend +import Types.KeySource +import Utility.Hash + +import qualified Data.ByteString.Lazy as L +import Data.Char + +data Hash + = MD5Hash + | SHA1Hash + | SHA2Hash HashSize + | SHA3Hash HashSize + | SkeinHash HashSize +#if MIN_VERSION_cryptonite(0,23,0) + | Blake2bHash HashSize + | Blake2sHash HashSize + | Blake2spHash HashSize +#endif + +{- Order is slightly significant; want SHA256 first, and more general + - sizes earlier. -} +hashes :: [Hash] +hashes = concat + [ map (SHA2Hash . HashSize) [256, 512, 224, 384] + , map (SHA3Hash . HashSize) [256, 512, 224, 384] + , map (SkeinHash . HashSize) [256, 512] +#if MIN_VERSION_cryptonite(0,23,0) + , map (Blake2bHash . HashSize) [256, 512, 160, 224, 384] + , map (Blake2sHash . HashSize) [256, 160, 224] + , map (Blake2spHash . HashSize) [256, 224] +#endif + , [SHA1Hash] + , [MD5Hash] + ] + +{- The SHA256E backend is the default, so genBackendE comes first. -} +backends :: [Backend] +backends = concatMap (\h -> [genBackendE h, genBackend h]) hashes + +genBackend :: Hash -> Backend +genBackend hash = Backend + { backendVariety = hashKeyVariety hash (HasExt False) + , getKey = keyValue hash + , verifyKeyContent = Just $ checkKeyChecksum hash + , canUpgradeKey = Just needsUpgrade + , fastMigrate = Just trivialMigrate + , isStableKey = const True + } + +genBackendE :: Hash -> Backend +genBackendE hash = (genBackend hash) + { backendVariety = hashKeyVariety hash (HasExt True) + , getKey = keyValueE hash + } + +hashKeyVariety :: Hash -> HasExt -> KeyVariety +hashKeyVariety MD5Hash = MD5Key +hashKeyVariety SHA1Hash = SHA1Key +hashKeyVariety (SHA2Hash size) = SHA2Key size +hashKeyVariety (SHA3Hash size) = SHA3Key size +hashKeyVariety (SkeinHash size) = SKEINKey size +#if MIN_VERSION_cryptonite(0,23,0) +hashKeyVariety (Blake2bHash size) = Blake2bKey size +hashKeyVariety (Blake2sHash size) = Blake2sKey size +hashKeyVariety (Blake2spHash size) = Blake2spKey size +#endif + +{- A key is a hash of its contents. -} +keyValue :: Hash -> KeySource -> Annex (Maybe Key) +keyValue hash source = do + let file = contentLocation source + filesize <- liftIO $ getFileSize file + s <- hashFile hash file + return $ Just $ stubKey + { keyName = s + , keyVariety = hashKeyVariety hash (HasExt False) + , keySize = Just filesize + } + +{- Extension preserving keys. -} +keyValueE :: Hash -> KeySource -> Annex (Maybe Key) +keyValueE hash source = keyValue hash source >>= maybe (return Nothing) addE + where + addE k = do + maxlen <- annexMaxExtensionLength <$> Annex.getGitConfig + let ext = selectExtension maxlen (keyFilename source) + return $ Just $ k + { keyName = keyName k ++ ext + , keyVariety = hashKeyVariety hash (HasExt True) + } + +selectExtension :: Maybe Int -> FilePath -> String +selectExtension maxlen f + | null es = "" + | otherwise = intercalate "." ("":es) + where + es = filter (not . null) $ reverse $ + take 2 $ filter (all validInExtension) $ + takeWhile shortenough $ + reverse $ splitc '.' $ takeExtensions f + shortenough e = length e <= fromMaybe maxExtensionLen maxlen + +maxExtensionLen :: Int +maxExtensionLen = 4 -- long enough for "jpeg" + +{- A key's checksum is checked during fsck when it's content is present + - except for in fast mode. -} +checkKeyChecksum :: Hash -> Key -> FilePath -> Annex Bool +checkKeyChecksum hash key file = catchIOErrorType HardwareFault hwfault $ do + fast <- Annex.getState Annex.fast + exists <- liftIO $ doesFileExist file + case (exists, fast) of + (True, False) -> do + showAction "checksum" + check <$> hashFile hash file + _ -> return True + where + expected = keyHash key + check s + | s == expected = True + {- A bug caused checksums to be prefixed with \ in some + - cases; still accept these as legal now that the bug has been + - fixed. -} + | '\\' : s == expected = True + | otherwise = False + + hwfault e = do + warning $ "hardware fault: " ++ show e + return False + +keyHash :: Key -> String +keyHash key = dropExtensions (keyName key) + +validInExtension :: Char -> Bool +validInExtension c + | isAlphaNum c = True + | c == '.' = True + | otherwise = False + +{- Upgrade keys that have the \ prefix on their hash due to a bug, or + - that contain non-alphanumeric characters in their extension. + - + - Also, for a while migrate from eg SHA256E to SHA256 resulted in a SHA256 + - key that contained an extension inside its keyName. Upgrade those + - keys, removing the extension. + -} +needsUpgrade :: Key -> Bool +needsUpgrade key = or + [ "\\" `isPrefixOf` keyHash key + , any (not . validInExtension) (takeExtensions $ keyName key) + , not (hasExt (keyVariety key)) && keyHash key /= keyName key + ] + +trivialMigrate :: Key -> Backend -> AssociatedFile -> Annex (Maybe Key) +trivialMigrate oldkey newbackend afile = trivialMigrate' oldkey newbackend afile + <$> (annexMaxExtensionLength <$> Annex.getGitConfig) + +trivialMigrate' :: Key -> Backend -> AssociatedFile -> Maybe Int -> Maybe Key +trivialMigrate' oldkey newbackend afile maxextlen + {- Fast migration from hashE to hash backend. -} + | migratable && hasExt oldvariety = Just $ oldkey + { keyName = keyHash oldkey + , keyVariety = newvariety + } + {- Fast migration from hash to hashE backend. -} + | migratable && hasExt newvariety = case afile of + AssociatedFile Nothing -> Nothing + AssociatedFile (Just file) -> Just $ oldkey + { keyName = keyHash oldkey + ++ selectExtension maxextlen file + , keyVariety = newvariety + } + {- Upgrade to fix bad previous migration that created a + - non-extension preserving key, with an extension + - in its keyName. -} + | newvariety == oldvariety && not (hasExt oldvariety) && + keyHash oldkey /= keyName oldkey = Just $ oldkey + { keyName = keyHash oldkey + } + | otherwise = Nothing + where + migratable = oldvariety /= newvariety + && sameExceptExt oldvariety newvariety + oldvariety = keyVariety oldkey + newvariety = backendVariety newbackend + +hashFile :: Hash -> FilePath -> Annex String +hashFile hash file = liftIO $ do + h <- hasher <$> L.readFile file + -- Force full evaluation so file is read and closed. + return (length h `seq` h) + where + hasher = case hash of + MD5Hash -> md5Hasher + SHA1Hash -> sha1Hasher + SHA2Hash hashsize -> sha2Hasher hashsize + SHA3Hash hashsize -> sha3Hasher hashsize + SkeinHash hashsize -> skeinHasher hashsize +#if MIN_VERSION_cryptonite(0,23,0) + Blake2bHash hashsize -> blake2bHasher hashsize + Blake2sHash hashsize -> blake2sHasher hashsize + Blake2spHash hashsize -> blake2spHasher hashsize +#endif + +sha2Hasher :: HashSize -> (L.ByteString -> String) +sha2Hasher (HashSize hashsize) + | hashsize == 256 = use sha2_256 + | hashsize == 224 = use sha2_224 + | hashsize == 384 = use sha2_384 + | hashsize == 512 = use sha2_512 + | otherwise = error $ "unsupported SHA size " ++ show hashsize + where + use hasher = show . hasher + +sha3Hasher :: HashSize -> (L.ByteString -> String) +sha3Hasher (HashSize hashsize) + | hashsize == 256 = show . sha3_256 + | hashsize == 224 = show . sha3_224 + | hashsize == 384 = show . sha3_384 + | hashsize == 512 = show . sha3_512 + | otherwise = error $ "unsupported SHA3 size " ++ show hashsize + +skeinHasher :: HashSize -> (L.ByteString -> String) +skeinHasher (HashSize hashsize) + | hashsize == 256 = show . skein256 + | hashsize == 512 = show . skein512 + | otherwise = error $ "unsupported SKEIN size " ++ show hashsize + +#if MIN_VERSION_cryptonite(0,23,0) +blake2bHasher :: HashSize -> (L.ByteString -> String) +blake2bHasher (HashSize hashsize) + | hashsize == 256 = show . blake2b_256 + | hashsize == 512 = show . blake2b_512 + | hashsize == 160 = show . blake2b_160 + | hashsize == 224 = show . blake2b_224 + | hashsize == 384 = show . blake2b_384 + | otherwise = error $ "unsupported BLAKE2B size " ++ show hashsize + +blake2sHasher :: HashSize -> (L.ByteString -> String) +blake2sHasher (HashSize hashsize) + | hashsize == 256 = show . blake2s_256 + | hashsize == 160 = show . blake2s_160 + | hashsize == 224 = show . blake2s_224 + | otherwise = error $ "unsupported BLAKE2S size " ++ show hashsize + +blake2spHasher :: HashSize -> (L.ByteString -> String) +blake2spHasher (HashSize hashsize) + | hashsize == 256 = show . blake2sp_256 + | hashsize == 224 = show . blake2sp_224 + | otherwise = error $ "unsupported BLAKE2SP size " ++ show hashsize +#endif + +sha1Hasher :: L.ByteString -> String +sha1Hasher = show . sha1 + +md5Hasher :: L.ByteString -> String +md5Hasher = show . md5 + +{- A varient of the SHA256E backend, for testing that needs special keys + - that cannot collide with legitimate keys in the repository. + - + - This is accomplished by appending a special extension to the key, + - that is not one that selectExtension would select (due to being too + - long). + -} +testKeyBackend :: Backend +testKeyBackend = + let b = genBackendE (SHA2Hash (HashSize 256)) + in b { getKey = (fmap addE) <$$> getKey b } + where + addE k = k { keyName = keyName k ++ longext } + longext = ".this-is-a-test-key" diff --git a/Backend/URL.hs b/Backend/URL.hs new file mode 100644 index 0000000000..b9d8264d6c --- /dev/null +++ b/Backend/URL.hs @@ -0,0 +1,39 @@ +{- git-annex "URL" backend -- keys whose content is available from urls. + - + - Copyright 2011 Joey Hess + - + - Licensed under the GNU GPL version 3 or higher. + -} + +module Backend.URL ( + backends, + fromUrl +) where + +import Annex.Common +import Types.Key +import Types.Backend +import Backend.Utilities + +backends :: [Backend] +backends = [backend] + +backend :: Backend +backend = Backend + { backendVariety = URLKey + , getKey = const $ return Nothing + , verifyKeyContent = Nothing + , canUpgradeKey = Nothing + , fastMigrate = Nothing + -- The content of an url can change at any time, so URL keys are + -- not stable. + , isStableKey = const False + } + +{- Every unique url has a corresponding key. -} +fromUrl :: String -> Maybe Integer -> Key +fromUrl url size = stubKey + { keyName = genKeyName url + , keyVariety = URLKey + , keySize = size + } diff --git a/Backend/Utilities.hs b/Backend/Utilities.hs new file mode 100644 index 0000000000..1691fa2b22 --- /dev/null +++ b/Backend/Utilities.hs @@ -0,0 +1,31 @@ +{- git-annex backend utilities + - + - Copyright 2012 Joey Hess + - + - Licensed under the GNU GPL version 3 or higher. + -} + +module Backend.Utilities where + +import Annex.Common +import Utility.FileSystemEncoding +import Utility.Hash + +{- Generates a keyName from an input string. Takes care of sanitizing it. + - If it's not too long, the full string is used as the keyName. + - Otherwise, it's truncated, and its md5 is prepended to ensure a unique + - key. -} +genKeyName :: String -> String +genKeyName s + -- Avoid making keys longer than the length of a SHA256 checksum. + | bytelen > sha256len = + truncateFilePath (sha256len - md5len - 1) s' ++ "-" ++ + show (md5 (encodeBS s)) + | otherwise = s' + where + s' = preSanitizeKeyName s + bytelen = length (decodeW8 s') + + sha256len = 64 + md5len = 32 + diff --git a/Backend/WORM.hs b/Backend/WORM.hs new file mode 100644 index 0000000000..689cc1d904 --- /dev/null +++ b/Backend/WORM.hs @@ -0,0 +1,58 @@ +{- git-annex "WORM" backend -- Write Once, Read Many + - + - Copyright 2010 Joey Hess + - + - Licensed under the GNU GPL version 3 or higher. + -} + +module Backend.WORM (backends) where + +import Annex.Common +import Types.Key +import Types.Backend +import Types.KeySource +import Backend.Utilities +import Git.FilePath + +backends :: [Backend] +backends = [backend] + +backend :: Backend +backend = Backend + { backendVariety = WORMKey + , getKey = keyValue + , verifyKeyContent = Nothing + , canUpgradeKey = Just needsUpgrade + , fastMigrate = Just removeSpaces + , isStableKey = const True + } + +{- The key includes the file size, modification time, and the + - original filename relative to the top of the git repository. + -} +keyValue :: KeySource -> Annex (Maybe Key) +keyValue source = do + let f = contentLocation source + stat <- liftIO $ getFileStatus f + sz <- liftIO $ getFileSize' f stat + relf <- getTopFilePath <$> inRepo (toTopFilePath $ keyFilename source) + return $ Just $ stubKey + { keyName = genKeyName relf + , keyVariety = WORMKey + , keySize = Just sz + , keyMtime = Just $ modificationTime stat + } + +{- Old WORM keys could contain spaces, and can be upgraded to remove them. -} +needsUpgrade :: Key -> Bool +needsUpgrade key = ' ' `elem` keyName key + +removeSpaces :: Key -> Backend -> AssociatedFile -> Annex (Maybe Key) +removeSpaces oldkey newbackend _ + | migratable = return $ Just $ oldkey + { keyName = reSanitizeKeyName (keyName oldkey) } + | otherwise = return Nothing + where + migratable = oldvariety == newvariety + oldvariety = keyVariety oldkey + newvariety = backendVariety newbackend diff --git a/Build/BuildVersion.hs b/Build/BuildVersion.hs new file mode 100644 index 0000000000..dbd819fff7 --- /dev/null +++ b/Build/BuildVersion.hs @@ -0,0 +1,13 @@ +{- For use by autobuilders, this outputs the version of git-annex that + - is being built, and also updates the Build/Version file so the build + - will report the right version, including the current git rev. -} + +{-# OPTIONS_GHC -fno-warn-tabs #-} + +import Build.Version + +main :: IO () +main = do + ver <- getVersion + writeVersion ver + putStr ver diff --git a/Build/BundledPrograms.hs b/Build/BundledPrograms.hs new file mode 100644 index 0000000000..f593c98103 --- /dev/null +++ b/Build/BundledPrograms.hs @@ -0,0 +1,95 @@ +{- Bundled programs + - + - Copyright 2013 Joey Hess + - + - Licensed under the GNU GPL version 3 or higher. + -} + +{-# LANGUAGE CPP #-} + +module Build.BundledPrograms where + +import Data.Maybe + +import BuildInfo + +{- Programs that git-annex uses, to include in the bundle. + - + - These may be just the command name, or the full path to it. -} +bundledPrograms :: [FilePath] +bundledPrograms = preferredBundledPrograms ++ extraBundledPrograms + +{- Programs that are only included in the bundle in case the system + - doesn't have them. These come after the system PATH. + -} +extraBundledPrograms :: [FilePath] +extraBundledPrograms = catMaybes + -- The system gpg is probably better, because it may better + -- integrate with the system gpg-agent, etc. + -- On Windows, gpg is bundled with git for windows. +#ifndef mingw32_HOST_OS + [ BuildInfo.gpg +#else + [ +#endif +#ifndef darwin_HOST_OS +#ifndef mingw32_HOST_OS + -- OS X has ssh installed by default. + -- On Windows, git provides ssh. + -- Linux probably has ssh installed system wide, + -- and if so the user probably wants to use that one. + , Just "ssh" + , Just "ssh-keygen" +#endif +#endif + ] + +{- Programs that should be preferred for use from the bundle, over + - any that might be installed on the system otherwise. These come before + - the system PATH. + - + - For example, git-annex is built for a specific version of git. + -} +preferredBundledPrograms :: [FilePath] +preferredBundledPrograms = catMaybes + [ Nothing +#ifndef mingw32_HOST_OS + -- git is not included in the windows bundle; git for windows is used + , Just "git" + -- Not strictly needed in PATH by git-annex, but called + -- by git when it sshes to a remote. + , Just "git-upload-pack" + , Just "git-receive-pack" + , Just "git-shell" +#endif +#ifndef mingw32_HOST_OS + -- using xargs on windows led to problems, so it's not used there + , Just "xargs" +#endif + , Just "rsync" +#ifndef mingw32_HOST_OS + , Just "sh" + -- used by git-annex when available + , Just "uname" +#endif + , BuildInfo.lsof + , BuildInfo.gcrypt +#ifndef mingw32_HOST_OS + -- These utilities are included in git for Windows + , ifset BuildInfo.curl "curl" + , Just "cp" +#endif +#ifdef linux_HOST_OS + -- used to unpack the tarball when upgrading + , Just "gunzip" + , Just "tar" + -- used by runshell to generate locales + , Just "localedef" +#endif + -- nice, ionice, and nocache are not included in the bundle; + -- we rely on the system's own version, which may better match + -- its kernel, and avoid using them if not available. + ] + where + ifset True s = Just s + ifset False _ = Nothing diff --git a/Build/Configure.hs b/Build/Configure.hs new file mode 100644 index 0000000000..6157921b58 --- /dev/null +++ b/Build/Configure.hs @@ -0,0 +1,102 @@ +{- Checks system configuration and generates Build/SysConfig and Build/Version. -} + +{-# OPTIONS_GHC -fno-warn-tabs #-} + +module Build.Configure where + +import Build.TestConfig +import Build.Version +import Utility.SafeCommand +import Utility.Env.Basic +import qualified Git.Version +import Utility.Directory + +import Control.Monad +import Control.Applicative +import Prelude + +tests :: [TestCase] +tests = + [ TestCase "UPGRADE_LOCATION" getUpgradeLocation + , TestCase "git" $ testCmd "git" "git --version >/dev/null" + , TestCase "git version" getGitVersion + , testCp "cp_a" "-a" + , testCp "cp_p" "-p" + , testCp "cp_preserve_timestamps" "--preserve=timestamps" + , testCp "cp_reflink_auto" "--reflink=auto" + , TestCase "xargs -0" $ testCmd "xargs_0" "xargs -0 /dev/null" + , TestCase "curl" $ testCmd "curl" "curl --version >/dev/null" + , TestCase "bup" $ testCmd "bup" "bup --version >/dev/null" + , TestCase "nice" $ testCmd "nice" "nice true >/dev/null" + , TestCase "ionice" $ testCmd "ionice" "ionice -c3 true >/dev/null" + , TestCase "nocache" $ testCmd "nocache" "nocache true >/dev/null" + , TestCase "gpg" $ maybeSelectCmd "gpg" + [ ("gpg", "--version >/dev/null") + , ("gpg2", "--version >/dev/null") ] + , TestCase "lsof" $ findCmdPath "lsof" "lsof" + , TestCase "git-remote-gcrypt" $ findCmdPath "gcrypt" "git-remote-gcrypt" + , TestCase "ssh connection caching" getSshConnectionCaching + ] + +tmpDir :: String +tmpDir = "tmp" + +testFile :: String +testFile = tmpDir ++ "/testfile" + +testCp :: ConfigKey -> String -> TestCase +testCp k option = TestCase cmd $ testCmd k cmdline + where + cmd = "cp " ++ option + cmdline = cmd ++ " " ++ testFile ++ " " ++ testFile ++ ".new" + +getUpgradeLocation :: Test +getUpgradeLocation = do + e <- getEnv "UPGRADE_LOCATION" + return $ Config "upgradelocation" $ MaybeStringConfig e + +getGitVersion :: Test +getGitVersion = go =<< getEnv "FORCE_GIT_VERSION" + where + go (Just s) = return $ Config "gitversion" $ StringConfig s + go Nothing = do + v <- Git.Version.installed + let oldestallowed = Git.Version.normalize "1.7.1.0" + when (v < oldestallowed) $ + error $ "installed git version " ++ show v ++ " is too old! (Need " ++ show oldestallowed ++ " or newer)" + return $ Config "gitversion" $ StringConfig $ show v + +getSshConnectionCaching :: Test +getSshConnectionCaching = Config "sshconnectioncaching" . BoolConfig <$> + boolSystem "sh" [Param "-c", Param "ssh -o ControlPersist=yes -V >/dev/null 2>/dev/null"] + +setup :: IO () +setup = do + createDirectoryIfMissing True tmpDir + writeFile testFile "test file contents" + +cleanup :: IO () +cleanup = removeDirectoryRecursive tmpDir + +run :: [TestCase] -> IO () +run ts = do + setup + config <- runTests ts + v <- getEnv "CROSS_COMPILE" + case v of + Just "Android" -> writeSysConfig $ androidConfig config + _ -> writeSysConfig config + writeVersion =<< getVersion + cleanup + +{- Hard codes some settings to cross-compile for Android. -} +androidConfig :: [Config] -> [Config] +androidConfig c = overrides ++ filter (not . overridden) c + where + overrides = + [ Config "cp_reflink_auto" $ BoolConfig False + , Config "curl" $ BoolConfig False + ] + overridden (Config k _) = k `elem` overridekeys + overridekeys = map (\(Config k _) -> k) overrides diff --git a/Build/DesktopFile.hs b/Build/DesktopFile.hs new file mode 100644 index 0000000000..a54f45d338 --- /dev/null +++ b/Build/DesktopFile.hs @@ -0,0 +1,86 @@ +{- Generating and installing a desktop menu entry file and icon, + - and a desktop autostart file. (And OSX equivilants.) + - + - Copyright 2012 Joey Hess + - + - Licensed under the GNU GPL version 3 or higher. + -} + +{-# LANGUAGE CPP #-} +{-# OPTIONS_GHC -fno-warn-tabs #-} + +module Build.DesktopFile where + +import Utility.Exception +import Utility.FreeDesktop +import Utility.Path +import Utility.Monad +import Utility.Directory +import Config.Files +import Utility.OSX +import Assistant.Install.AutoStart +import Assistant.Install.Menu + +import System.Environment +import System.PosixCompat.User +import Data.Maybe +import Control.Applicative +import Prelude + +systemwideInstall :: IO Bool +#ifndef mingw32_HOST_OS +systemwideInstall = isroot <||> destdirset + where + isroot = do + uid <- fromIntegral <$> getRealUserID + return $ uid == (0 :: Int) + destdirset = isJust <$> catchMaybeIO (getEnv "DESTDIR") +#else +systemwideInstall = return False +#endif + +inDestDir :: FilePath -> IO FilePath +inDestDir f = do + destdir <- catchDefaultIO "" (getEnv "DESTDIR") + return $ destdir ++ "/" ++ f + +writeFDODesktop :: FilePath -> IO () +writeFDODesktop command = do + systemwide <- systemwideInstall + + datadir <- if systemwide then return systemDataDir else userDataDir + menufile <- inDestDir (desktopMenuFilePath "git-annex" datadir) + icondir <- inDestDir (iconDir datadir) + installMenu command menufile "doc" icondir + + configdir <- if systemwide then return systemConfigDir else userConfigDir + installAutoStart command + =<< inDestDir (autoStartPath "git-annex" configdir) + +writeOSXDesktop :: FilePath -> IO () +writeOSXDesktop command = do + installAutoStart command =<< inDestDir =<< ifM systemwideInstall + ( return $ systemAutoStart osxAutoStartLabel + , userAutoStart osxAutoStartLabel + ) + +install :: FilePath -> IO () +install command = do +#ifdef darwin_HOST_OS + writeOSXDesktop command +#else + writeFDODesktop command +#endif + ifM systemwideInstall + ( return () + , do + programfile <- inDestDir =<< programFile + createDirectoryIfMissing True (parentDir programfile) + writeFile programfile command + ) + +installUser :: FilePath -> IO () +installUser command = ifM systemwideInstall + ( return () + , install command + ) diff --git a/Build/DistributionUpdate.hs b/Build/DistributionUpdate.hs new file mode 100644 index 0000000000..920cadb3e8 --- /dev/null +++ b/Build/DistributionUpdate.hs @@ -0,0 +1,209 @@ +{- Downloads git-annex autobuilds and installs them into the git-annex + - repository in ~/lib/downloads that is used to distribute git-annex + - releases. + - + - Generates info files, containing the version (of the corresponding file + - from the autobuild). + - + - Also gpg signs the files. + -} + +import Annex.Common +import Types.Distribution +import Build.Version (getChangelogVersion, Version) +import Utility.UserInfo +import Utility.Url +import Utility.Tmp.Dir +import Utility.FileSystemEncoding +import Utility.Metered +import qualified Git.Construct +import qualified Annex +import Annex.Content +import Annex.WorkTree +import Git.Command + +import Data.Time.Clock +import Data.Char +import System.Posix.Directory + +-- git-annex distribution signing key (for Joey Hess) +signingKey :: String +signingKey = "89C809CB" + +-- URL to an autobuilt git-annex file, and the place to install +-- it in the repository. +autobuilds :: [(URLString, FilePath)] +autobuilds = + (map linuxarch ["i386", "amd64", "armel", "i386-ancient"]) ++ + (map androidversion ["4.0", "4.3", "5.0"]) ++ + [ (autobuild "x86_64-apple-yosemite/git-annex.dmg", "git-annex/OSX/current/10.10_Yosemite/git-annex.dmg") + , (autobuild "windows/git-annex-installer.exe", "git-annex/windows/current/git-annex-installer.exe") + ] + where + linuxarch a = + ( autobuild (a ++ "/git-annex-standalone-" ++ a ++ ".tar.gz") + , "git-annex/linux/current/git-annex-standalone-" ++ a ++ ".tar.gz" + ) + androidversion v = + ( autobuild ("android/" ++ v ++ "/git-annex.apk") + , "git-annex/android/current/" ++ v ++ "/git-annex.apk" + ) + autobuild f = "https://downloads.kitenet.net/git-annex/autobuild/" ++ f + +main :: IO () +main = do + useFileSystemEncoding + version <- liftIO getChangelogVersion + repodir <- getRepoDir + changeWorkingDirectory repodir + updated <- catMaybes <$> mapM (getbuild repodir) autobuilds + state <- Annex.new =<< Git.Construct.fromPath "." + Annex.eval state (makeinfos updated version) + +-- Download a build from the autobuilder, virus check it, and return its +-- version. +-- It's very important that the version matches the build, otherwise +-- auto-upgrades can loop reatedly. So, check build-version before +-- and after downloading the file. +getbuild :: FilePath -> (URLString, FilePath) -> IO (Maybe (FilePath, Version)) +getbuild repodir (url, f) = do + bv1 <- getbv + let dest = repodir f + let tmp = dest ++ ".tmp" + nukeFile tmp + createDirectoryIfMissing True (parentDir dest) + let oops s = do + nukeFile tmp + putStrLn $ "*** " ++ s + return Nothing + uo <- defUrlOptions + ifM (download nullMeterUpdate url tmp uo) + ( ifM (liftIO $ virusFree tmp) + ( do + bv2 <- getbv + case bv2 of + Nothing -> oops $ "no build-version file for " ++ url + (Just v) + | bv2 == bv1 -> do + nukeFile dest + renameFile tmp dest + -- remove git rev part of version + let v' = takeWhile (/= '-') v + return $ Just (f, v') + | otherwise -> oops $ "build version changed while downloading " ++ url ++ " " ++ show (bv1, bv2) + , oops $ "VIRUS detected in " ++ url + ) + , oops $ "failed to download " ++ url + ) + where + bvurl = takeDirectory url ++ "/build-version" + getbv = do + bv <- catchDefaultIO "" $ readProcess "curl" ["--silent", bvurl] + return $ if null bv || any (not . versionchar) bv then Nothing else Just bv + versionchar c = isAlphaNum c || c == '.' || c == '-' + +makeinfos :: [(FilePath, Version)] -> Version -> Annex () +makeinfos updated version = do + mapM_ (\f -> inRepo $ runBool [Param "annex", Param "add", File f]) (map fst updated) + void $ inRepo $ runBool + [ Param "commit" + , Param "-a" + , Param ("-S" ++ signingKey) + , Param "-m" + , Param $ "publishing git-annex " ++ version + ] + now <- liftIO getCurrentTime + liftIO $ putStrLn $ "building info files" + forM_ updated $ \(f, bv) -> do + v <- lookupFile f + case v of + Nothing -> noop + Just k -> whenM (inAnnex k) $ do + liftIO $ putStrLn f + let infofile = f ++ ".info" + let d = GitAnnexDistribution + { distributionUrl = mkUrl f + , distributionKey = k + , distributionVersion = bv + , distributionReleasedate = now + , distributionUrgentUpgrade = Just "6.20180626" + } + liftIO $ writeFile infofile $ formatInfoFile d + void $ inRepo $ runBool [Param "add", File infofile] + signFile infofile + signFile f + void $ inRepo $ runBool + [ Param "commit" + , Param ("-S" ++ signingKey) + , Param "-m" + , Param $ "updated info files for git-annex " ++ version + ] + void $ inRepo $ runBool + [ Param "annex" + , Param "move" + , Param "--to" + , Param "website" + ] + void $ inRepo $ runBool + [ Param "annex" + , Param "sync" + ] + + -- Check for out of date info files. + infos <- liftIO $ filter (".info" `isSuffixOf`) + <$> dirContentsRecursive "git-annex" + ds <- liftIO $ forM infos (readish <$$> readFile) + let dis = zip infos ds + let ood = filter outofdate dis + unless (null ood) $ + error $ "Some info files are out of date: " ++ show (map fst ood) + where + outofdate (_, md) = case md of + Nothing -> True + Just d -> distributionVersion d /= version + +getRepoDir :: IO FilePath +getRepoDir = do + home <- liftIO myHomeDir + return $ home "lib" "downloads" + +mkUrl :: FilePath -> String +mkUrl f = "https://downloads.kitenet.net/" ++ f + +signFile :: FilePath -> Annex () +signFile f = do + void $ liftIO $ boolSystem "gpg" + [ Param "-a" + , Param $ "--default-key=" ++ signingKey + , Param "--detach-sign" + , File f + ] + liftIO $ rename (f ++ ".asc") (f ++ ".sig") + void $ inRepo $ runBool [Param "add", File (f ++ ".sig")] + +-- clamscan should handle unpacking archives, but did not in my +-- testing, so do it manually. +virusFree :: FilePath -> IO Bool +virusFree f + | ".tar.gz" `isSuffixOf` f = unpack $ \tmpdir -> + boolSystem "tar" [ Param "xf", File f, Param "-C", File tmpdir ] + | ".dmg" `isSuffixOf` f = unpack $ \tmpdir -> do + -- 7z can extract partitions from a dmg, and then + -- run on partitions can extract their files + unhfs tmpdir f + parts <- filter (".hfs" `isSuffixOf`) <$> getDirectoryContents tmpdir + forM_ parts $ unhfs tmpdir + return True + | otherwise = clamscan f + where + clamscan f' = boolSystem "clamscan" + [ Param "--no-summary" + , Param "-r" + , Param f' + ] + unpack unpacker = withTmpDir "clamscan" $ \tmpdir -> do + unlessM (unpacker tmpdir) $ + error $ "Failed to unpack " ++ f ++ " for virus scan" + clamscan tmpdir + unhfs dest f' = unlessM (boolSystem "7z" [ Param "x", Param ("-o" ++ dest), File f' ]) $ + error $ "Failed extracting hfs " ++ f' diff --git a/Build/EvilSplicer.hs b/Build/EvilSplicer.hs new file mode 100644 index 0000000000..e07034c5b0 --- /dev/null +++ b/Build/EvilSplicer.hs @@ -0,0 +1,738 @@ +{- Expands template haskell splices + - + - You should probably just use http://hackage.haskell.org/package/zeroth + - instead. I wish I had known about it before writing this. + - + - First, the code must be built with a ghc that supports TH, + - and the splices dumped to a log. For example: + - cabal build --ghc-options=-ddump-splices 2>&1 | tee log + - + - Along with the log, a headers file may also be provided, containing + - additional imports needed by the template haskell code. + - + - This program will parse the log, and expand all splices therein, + - writing files to the specified destdir (which can be "." to modify + - the source tree directly). They can then be built a second + - time, with a ghc that does not support TH. + - + - Note that template haskell code may refer to symbols that are not + - exported by the library that defines the TH code. In this case, + - the library has to be modifed to export those symbols. + - + - There can also be other problems with the generated code; it may + - need modifications to compile. + - + - + - Copyright 2013 Joey Hess + - + - Licensed under the GNU GPL version 3 or higher. + -} + +module Main where + +import Text.Parsec +import Text.Parsec.String +import Control.Applicative ((<$>)) +import Data.Either +import Data.List hiding (find) +import Data.Char +import System.Environment +import System.FilePath +import System.IO +import Control.Monad +import Prelude hiding (log) + +import Utility.Monad +import Utility.Misc +import Utility.Exception hiding (try) +import Utility.Path +import Utility.FileSystemEncoding +import Utility.Directory +import Utility.Split + +data Coord = Coord + { coordLine :: Int + , coordColumn :: Int + } + deriving (Read, Show) + +offsetCoord :: Coord -> Coord -> Coord +offsetCoord a b = Coord + (coordLine a - coordLine b) + (coordColumn a - coordColumn b) + +data SpliceType = SpliceExpression | SpliceDeclaration + deriving (Read, Show, Eq) + +data Splice = Splice + { splicedFile :: FilePath + , spliceStart :: Coord + , spliceEnd :: Coord + , splicedExpression :: String + , splicedCode :: String + , spliceType :: SpliceType + } + deriving (Read, Show) + +isExpressionSplice :: Splice -> Bool +isExpressionSplice s = spliceType s == SpliceExpression + +number :: Parser Int +number = read <$> many1 digit + +{- A pair of Coords is written in one of three ways: + - "95:21-73", "1:1", or "(92,25)-(94,2)" + -} +coordsParser :: Parser (Coord, Coord) +coordsParser = (try singleline <|> try weird <|> multiline) "Coords" + where + singleline = do + line <- number + void $ char ':' + startcol <- number + void $ char '-' + endcol <- number + return $ (Coord line startcol, Coord line endcol) + + weird = do + line <- number + void $ char ':' + col <- number + return $ (Coord line col, Coord line col) + + multiline = do + start <- fromparens + void $ char '-' + end <- fromparens + return $ (start, end) + + fromparens = between (char '(') (char ')') $ do + line <- number + void $ char ',' + col <- number + return $ Coord line col + +indent :: Parser String +indent = many1 $ char ' ' + +restOfLine :: Parser String +restOfLine = newline `after` many (noneOf "\n") + +indentedLine :: Parser String +indentedLine = indent >> restOfLine + +spliceParser :: Parser Splice +spliceParser = do + file <- many1 (noneOf ":\n") + void $ char ':' + (start, end) <- coordsParser + void $ string ": Splicing " + splicetype <- tosplicetype + <$> (string "expression" <|> string "declarations") + void newline + + getthline <- expressionextractor + expression <- unlines <$> many1 getthline + + void indent + void $ string "======>" + void newline + + getcodeline <- expressionextractor + realcoords <- try (Right <$> getrealcoords file) <|> (Left <$> getcodeline) + codelines <- many getcodeline + return $ case realcoords of + Left firstcodeline -> + Splice file start end expression + (unlines $ firstcodeline:codelines) + splicetype + Right (realstart, realend) -> + Splice file realstart realend expression + (unlines codelines) + splicetype + where + tosplicetype "declarations" = SpliceDeclaration + tosplicetype "expression" = SpliceExpression + tosplicetype s = error $ "unknown splice type: " ++ s + + {- All lines of the indented expression start with the same + - indent, which is stripped. Any other indentation is preserved. -} + expressionextractor = do + i <- lookAhead indent + return $ try $ do + void $ string i + restOfLine + + {- When splicing declarations, GHC will output a splice + - at 1:1, and then inside the splice code block, + - the first line will give the actual coordinates of the + - line that was spliced. -} + getrealcoords file = do + void indent + void $ string file + void $ char ':' + char '\n' `after` coordsParser + +{- Extracts the splices, ignoring the rest of the compiler output. -} +splicesExtractor :: Parser [Splice] +splicesExtractor = rights <$> many extract + where + extract = try (Right <$> spliceParser) <|> (Left <$> compilerJunkLine) + compilerJunkLine = restOfLine + +{- Modifies the source file, expanding the splices, which all must + - have the same splicedFile. Writes the new file to the destdir. + - + - Each splice's Coords refer to the original position in the file, + - and not to its position after any previous splices may have inserted + - or removed lines. + - + - To deal with this complication, the file is broken into logical lines + - (which can contain any String, including a multiline or empty string). + - Each splice is assumed to be on its own block of lines; two + - splices on the same line is not currently supported. + - This means that a splice can modify the logical lines within its block + - as it likes, without interfering with the Coords of other splices. + - + - As well as expanding splices, this can add a block of imports to the + - file. These are put right before the first line in the file that + - starts with "import " + -} +applySplices :: FilePath -> Maybe String -> [Splice] -> IO () +applySplices _ _ [] = noop +applySplices destdir imports splices@(first:_) = do + let f = splicedFile first + let dest = (destdir f) + lls <- map (++ "\n") . lines <$> readFileStrict f + createDirectoryIfMissing True (parentDir dest) + let newcontent = concat $ addimports $ expand lls splices + oldcontent <- catchMaybeIO $ readFileStrict dest + when (oldcontent /= Just newcontent) $ do + putStrLn $ "splicing " ++ f + withFile dest WriteMode $ \h -> do + hPutStr h newcontent + hClose h + where + expand lls [] = lls + expand lls (s:rest) + | isExpressionSplice s = expand (expandExpressionSplice s lls) rest + | otherwise = expand (expandDeclarationSplice s lls) rest + + addimports lls = case imports of + Nothing -> lls + Just v -> + let (start, end) = break ("import " `isPrefixOf`) lls + in if null end + then start + else concat + [ start + , [v] + , end + ] + +{- Declaration splices are expanded to replace their whole line. -} +expandDeclarationSplice :: Splice -> [String] -> [String] +expandDeclarationSplice s lls = concat [before, [splice], end] + where + cs = spliceStart s + ce = spliceEnd s + + (before, rest) = splitAt (coordLine cs - 1) lls + (_oldlines, end) = splitAt (1 + coordLine (offsetCoord ce cs)) rest + splice = mangleCode $ splicedCode s + +{- Expression splices are expanded within their line. -} +expandExpressionSplice :: Splice -> [String] -> [String] +expandExpressionSplice sp lls = concat [before, spliced:padding, end] + where + cs = spliceStart sp + ce = spliceEnd sp + + (before, rest) = splitAt (coordLine cs - 1) lls + (oldlines, end) = splitAt (1 + coordLine (offsetCoord ce cs)) rest + (splicestart, padding, spliceend) = case map expandtabs oldlines of + ss:r + | null r -> (ss, [], ss) + | otherwise -> (ss, take (length r) (repeat []), last r) + _ -> ([], [], []) + spliced = concat + [ joinsplice $ deqqstart $ take (coordColumn cs - 1) splicestart + , addindent (findindent splicestart) (mangleCode $ splicedCode sp) + , deqqend $ drop (coordColumn ce) spliceend + ] + + {- coordinates assume tabs are expanded to 8 spaces -} + expandtabs = replace "\t" (take 8 $ repeat ' ') + + {- splicing leaves $() quasiquote behind; remove it -} + deqqstart s = case reverse s of + ('(':'$':restq) -> reverse restq + _ -> s + deqqend (')':s) = s + deqqend s = s + + {- Prepare the code that comes just before the splice so + - the splice will combine with it appropriately. -} + joinsplice s + -- all indentation? Skip it, we'll use the splice's indentation + | all isSpace s = "" + -- function definition needs no preparation + -- ie: foo = $(splice) + | "=" `isSuffixOf` s' = s + -- nor does lambda definition or case expression + | "->" `isSuffixOf` s' = s + -- nor does a let .. in declaration + | "in" `isSuffixOf` s' = s + -- already have a $ to set off the splice + -- ie: foo $ $(splice) + | "$" `isSuffixOf` s' = s + -- need to add a $ to set off the splice + -- ie: bar $(splice) + | otherwise = s ++ " $ " + where + s' = filter (not . isSpace) s + + findindent = length . takeWhile isSpace + addindent n = unlines . map (i ++) . lines + where + i = take n $ repeat ' ' + +{- Tweaks code output by GHC in splices to actually build. Yipes. -} +mangleCode :: String -> String +mangleCode = flip_colon + . persist_dequalify_hack + . let_do + . remove_unnecessary_type_signatures + . lambdaparenhackyesod + . lambdaparenhackpersistent + . lambdaparens + . declaration_parens + . case_layout + . case_layout_multiline + . yesod_url_render_hack + . text_builder_hack + . nested_instances + . boxed_fileembed + . collapse_multiline_strings + . remove_package_version + . emptylambda + where + {- Lambdas are often output without parens around them. + - This breaks when the lambda is immediately applied to a + - parameter. + - + - For example: + - + - renderRoute (StaticR sub_a1nUH) + - = \ (a_a1nUI, b_a1nUJ) + - -> (((pack "static") : a_a1nUI), + - b_a1nUJ) + - (renderRoute sub_a1nUH) + - + - There are sometimes many lines of lambda code that need to be + - parenthesised. Approach: find the "->" and scan down the + - column to the first non-whitespace. This is assumed + - to be the expression after the lambda. + - + - Runs recursively on the body of the lambda, to handle nested + - lambdas. + -} + lambdaparens = parsecAndReplace $ do + -- skip lambdas inside tuples or parens + prefix <- noneOf "(, \n" + preindent <- many1 $ oneOf " \n" + void $ string "\\ " + lambdaparams <- restofline + continuedlambdaparams <- many $ try $ do + indent1 <- many1 $ char ' ' + p <- satisfy isLetter + aram <- many $ satisfy isAlphaNum <|> oneOf "_" + void newline + return $ indent1 ++ p:aram ++ "\n" + indent1 <- many1 $ char ' ' + void $ string "-> " + firstline <- restofline + lambdalines <- many $ try $ do + void $ string indent1 + void $ char ' ' + l <- restofline + return $ indent1 ++ " " ++ l + return $ concat + [ prefix:preindent + , "(\\ " ++ lambdaparams ++ "\n" + , concat continuedlambdaparams + , indent1 ++ "-> " + , lambdaparens $ intercalate "\n" (firstline:lambdalines) + , ")\n" + ] + + {- Hack to add missing parens in a specific case in yesod + - static route code. + - + - StaticR + - yesod_dispatch_env_a4iDV + - (\ p_a4iE2 r_a4iE3 + - -> r_a4iE3 + - {Network.Wai.pathInfo = p_a4iE2} + - xrest_a4iDT req_a4iDW)) } + - + - Need to add another paren around the lambda, and close it + - before its parameters. lambdaparens misses this one because + - there is already one paren present. + - + - Note that the { } may be on the same line, or wrapped to next. + - + - FIXME: This is a hack. lambdaparens could just always add a + - layer of parens even when a lambda seems to be in parent. + -} + lambdaparenhackyesod = parsecAndReplace $ do + indent1 <- many1 $ char ' ' + staticr <- string "StaticR" + void newline + void $ string indent1 + yesod_dispatch_env <- restofline + void $ string indent1 + lambdaprefix <- string "(\\ " + l1 <- restofline + void $ string indent1 + lambdaarrow <- string " ->" + l2 <- restofline + l3 <- if '{' `elem` l2 && '}' `elem` l2 + then return "" + else do + void $ string indent1 + restofline + return $ unlines + [ indent1 ++ staticr + , indent1 ++ yesod_dispatch_env + , indent1 ++ "(" ++ lambdaprefix ++ l1 + , indent1 ++ lambdaarrow ++ l2 ++ l3 ++ ")" + ] + + {- Hack to reorder misplaced paren in persistent code. + - + - = ((Right Fscked) + - (\ persistValue_a36iM + - -> case fromPersistValue persistValue_a36iM of { + - Right r_a36iN -> Right r_a36iN + - Left err_a36iO + - -> (Left + - $ ((("field " `Data.Monoid.mappend` (packPTH "key")) + - `Data.Monoid.mappend` ": ") + - `Data.Monoid.mappend` err_a36iO)) } + - x_a36iL)) + - + - Fixed by adding another level of params around the lambda + - (lambdaparams should be generalized to cover this case). + -} + lambdaparenhackpersistent = parsecAndReplace $ do + indent1 <- many1 $ char ' ' + start <- do + s1 <- string "(\\ " + s2 <- string "persistValue_" + s3 <- restofline + return $ s1 ++ s2 ++ s3 + void $ string indent1 + indent2 <- many1 $ char ' ' + void $ string "-> " + l1 <- restofline + lambdalines <- many $ try $ do + void $ string $ indent1 ++ indent2 ++ " " + l <- restofline + return $ indent1 ++ indent2 ++ " " ++ l + return $ concat + [ indent1 ++ "(" ++ start ++ "\n" + , indent1 ++ indent2 ++ "-> " ++ l1 ++ "\n" + , intercalate "\n" lambdalines + , ")\n" + ] + + restofline = manyTill (noneOf "\n") newline + + {- For some reason, GHC sometimes doesn't like the multiline + - strings it creates. It seems to get hung up on \{ at the + - start of a new line sometimes, wanting it to not be escaped. + - + - To work around what is likely a GHC bug, just collapse + - multiline strings. -} + collapse_multiline_strings = parsecAndReplace $ do + void $ string "\\\n" + void $ many1 $ oneOf " \t" + void $ string "\\" + return "\\n" + + {- GHC outputs splices using explicit braces rather than layout. + - For a case expression, it does something weird: + - + - case foo of { + - xxx -> blah + - yyy -> blah }; + - + - This is not legal Haskell; the statements in the case must be + - separated by ';' + - + - To fix, we could just put a semicolon at the start of every line + - containing " -> " ... Except that lambdas also contain that. + - But we can get around that: GHC outputs lambdas like this: + - + - \ foo + - -> bar + - + - Or like this: + - + - \ foo -> bar + - + - So, we can put the semicolon at the start of every line + - containing " -> " unless there's a "\ " first, or it's + - all whitespace up until it. + -} + case_layout = skipfree $ parsecAndReplace $ do + void newline + indent1 <- many1 $ char ' ' + prefix <- manyTill (noneOf "\n") (try (string "-> ")) + if length prefix > 20 + then unexpected "too long a prefix" + else if "\\ " `isInfixOf` prefix + then unexpected "lambda expression" + else if null prefix + then unexpected "second line of lambda" + else return $ "\n" ++ indent1 ++ "; " ++ prefix ++ " -> " + {- Sometimes cases themselves span multiple lines: + - + - Nothing + - -> foo + - + - -- This is not yet handled! + - ComplexConstructor var var + - var var + - -> foo + -} + case_layout_multiline = skipfree $ parsecAndReplace $ do + void newline + indent1 <- many1 $ char ' ' + firstline <- restofline + + void $ string indent1 + indent2 <- many1 $ char ' ' + void $ string "-> " + if "\\ " `isInfixOf` firstline + then unexpected "lambda expression" + else return $ "\n" ++ indent1 ++ "; " ++ firstline ++ "\n" + ++ indent1 ++ indent2 ++ "-> " + + {- Type definitions for free monads triggers the case_* hacks, avoid. -} + skipfree f s + | "MonadFree" `isInfixOf` s = s + | otherwise = f s + + {- (foo, \ -> bar) is not valid haskell, GHC. + - Change to (foo, bar) + - + - (Does this ever happen outside a tuple? Only saw + - it inside them.. + -} + emptylambda = replace ", \\ -> " ", " + + {- GHC may output this: + - + - instance RenderRoute WebApp where + - data instance Route WebApp + - ^^^^^^^^ + - The marked word should not be there. + - + - FIXME: This is a yesod and persistent-specific hack, + - it should look for the outer instance. + -} + nested_instances = replace " data instance Route" " data Route" + . replace " data instance Unique" " data Unique" + . replace " data instance EntityField" " data EntityField" + . replace " type instance PersistEntityBackend" " type PersistEntityBackend" + + {- GHC does not properly parenthesise generated data type + - declarations. -} + declaration_parens = replace "StaticR Route Static" "StaticR (Route Static)" + + {- A type signature is sometimes given for an entire lambda, + - which is not properly parenthesized or laid out. This is a + - hack to remove one specific case where this happens and the + - signature is easily inferred, so is just removed. + -} + remove_unnecessary_type_signatures = parsecAndReplace $ do + void $ string " ::" + void newline + void $ many1 $ char ' ' + void $ string "Text.Css.Block Text.Css.Resolved" + void newline + return "" + + {- GHC may add full package and version qualifications for + - symbols from unimported modules. We don't want these. + - + - Examples: + - "blaze-html-0.4.3.1:Text.Blaze.Internal.preEscapedText" + - "ghc-prim:GHC.Types.:" + -} + remove_package_version = parsecAndReplace $ + mangleSymbol <$> qualifiedSymbol + + mangleSymbol "GHC.Types." = "" + mangleSymbol "GHC.Tuple." = "" + mangleSymbol s = s + + qualifiedSymbol :: Parser String + qualifiedSymbol = do + s <- hstoken + void $ char ':' + if length s < 5 + then unexpected "too short to be a namespace" + else do + t <- hstoken + case t of + (c:r) | isUpper c && "." `isInfixOf` r -> return t + _ -> unexpected "not a module qualified symbol" + + hstoken :: Parser String + hstoken = do + t <- satisfy isLetter + oken <- many $ satisfy isAlphaNum <|> oneOf "-.'" + return $ t:oken + + {- This works when it's "GHC.Types.:", but we strip + - that above, so have to fix up after it here. + - The ; is added by case_layout. -} + flip_colon = replace "; : _ " "; _ : " + + {- TH for persistent has some qualified symbols in places + - that are not allowed. -} + persist_dequalify_hack = replace "Database.Persist.TH.++" "`Data.Text.append`" + . replace "Database.Persist.Sql.Class.sqlType" "sqlType" + . replace "Database.Persist.Class.PersistField.toPersistValue" "toPersistValue" + . replace "Database.Persist.Class.PersistField.fromPersistValue" "fromPersistValue" + + {- Sometimes generates invalid bracketed code with a let + - expression: + - + - foo = do { let x = foo; + - use foo } + - + - Fix by converting the "let x = " to "x <- return $" + -} + let_do = parsecAndReplace $ do + void $ string "= do { let " + x <- many $ noneOf "=\r\n" + _ <- many1 $ oneOf " \t\r\n" + void $ string "= " + return $ "= do { " ++ x ++ " <- return $ " + +{- Embedded files use unsafe packing, which is problematic + - for several reasons, including that GHC sometimes omits trailing + - newlines in the file content, which leads to the wrong byte + - count. Also, GHC sometimes outputs unicode characters, which + - are not legal in unboxed strings. + - + - Avoid problems by converting: + - GHC.IO.unsafePerformIO + - (Data.ByteString.Unsafe.unsafePackAddressLen + - lllll + - "blabblah"#)), + - to: + - Data.ByteString.Char8.pack "blabblah"), + - + - Note that the string is often multiline. This only works if + - collapse_multiline_strings has run first. + -} +boxed_fileembed :: String -> String +boxed_fileembed = parsecAndReplace $ do + i <- indent + void $ string "GHC.IO.unsafePerformIO" + void newline + void indent + void $ string "(Data.ByteString.Unsafe.unsafePackAddressLen" + void newline + void indent + void number + void newline + void indent + void $ char '"' + s <- restOfLine + let s' = take (length s - 5) s + if "\"#))," `isSuffixOf` s + then return (i ++ "Data.ByteString.Char8.pack \"" ++ s' ++ "\"),\n") + else fail "not an unboxed string" + +{- This works around a problem in the expanded template haskell for Yesod + - type-safe url rendering. + - + - It generates code like this: + - + - (toHtml + - (\ u_a2ehE -> urender_a2ehD u_a2ehE [] + - (CloseAlert aid))))); + - + - Where urender_a2ehD is the function returned by getUrlRenderParams. + - But, that function that only takes 2 params, not 3. + - And toHtml doesn't take a parameter at all! + - + - So, this modifes the code, to look like this: + - + - (toHtml + - (flip urender_a2ehD [] + - (CloseAlert aid))))); + - + - FIXME: Investigate and fix this properly. + -} +yesod_url_render_hack :: String -> String +yesod_url_render_hack = parsecAndReplace $ do + void $ string "(toHtml" + void whitespace + void $ string "(\\" + void whitespace + wtf <- hstoken + void whitespace + void $ string "->" + void whitespace + renderer <- hstoken + void whitespace + void $ string wtf + void whitespace + return $ "(toHtml (flip " ++ renderer ++ " " + where + whitespace :: Parser String + whitespace = many $ oneOf " \t\r\n" + + hstoken :: Parser String + hstoken = many1 $ satisfy isAlphaNum <|> oneOf "_" + +{- Use exported symbol. -} +text_builder_hack :: String -> String +text_builder_hack = replace "Data.Text.Lazy.Builder.Internal.fromText" "Data.Text.Lazy.Builder.fromText" + +{- Given a Parser that finds strings it wants to modify, + - and returns the modified string, does a mass + - find and replace throughout the input string. + - Rather slow, but crazy powerful. -} +parsecAndReplace :: Parser String -> String -> String +parsecAndReplace p s = case parse find "" s of + Left _e -> s + Right l -> concatMap (either return id) l + where + find :: Parser [Either Char String] + find = many $ try (Right <$> p) <|> (Left <$> anyChar) + +main :: IO () +main = do + useFileSystemEncoding + go =<< getArgs + where + go (destdir:log:header:[]) = run destdir log (Just header) + go (destdir:log:[]) = run destdir log Nothing + go _ = error "usage: EvilSplicer destdir logfile [headerfile]" + + run destdir log mheader = do + r <- parseFromFile splicesExtractor log + case r of + Left e -> error $ show e + Right splices -> do + let groups = groupBy (\a b -> splicedFile a == splicedFile b) splices + imports <- maybe (return Nothing) (catchMaybeIO . readFile) mheader + mapM_ (applySplices destdir imports) groups diff --git a/Build/InstallDesktopFile.hs b/Build/InstallDesktopFile.hs new file mode 100644 index 0000000000..d6a94added --- /dev/null +++ b/Build/InstallDesktopFile.hs @@ -0,0 +1,19 @@ +{- Generating and installing a desktop menu entry file and icon, + - and a desktop autostart file. (And OSX equivilants.) + - + - Copyright 2012 Joey Hess + - + - Licensed under the GNU GPL version 3 or higher. + -} + +module Main where + +import Build.DesktopFile + +import System.Environment + +main :: IO () +main = getArgs >>= go + where + go [] = error "specify git-annex command" + go (command:_) = install command diff --git a/Build/LinuxMkLibs.hs b/Build/LinuxMkLibs.hs new file mode 100644 index 0000000000..ba40206fdf --- /dev/null +++ b/Build/LinuxMkLibs.hs @@ -0,0 +1,114 @@ +{- Linux library copier and binary shimmer + - + - Copyright 2013 Joey Hess + - + - Licensed under the GNU GPL version 3 or higher. + -} + +module Main where + +import System.Environment +import Data.Maybe +import System.FilePath +import Control.Monad +import Data.List +import System.Posix.Files +import Control.Monad.IfElse +import Control.Applicative +import Prelude + +import Utility.LinuxMkLibs +import Utility.Directory +import Utility.Process +import Utility.Monad +import Utility.Path +import Utility.FileMode +import Utility.CopyFile + +main :: IO () +main = getArgs >>= go + where + go [] = error "specify LINUXSTANDALONE_DIST" + go (top:_) = mklibs top + +mklibs :: FilePath -> IO () +mklibs top = do + fs <- dirContentsRecursive top + exes <- filterM checkExe fs + libs <- parseLdd <$> readProcess "ldd" exes + glibclibs <- glibcLibs + let libs' = nub $ libs ++ glibclibs + libdirs <- nub . catMaybes <$> mapM (installLib installFile top) libs' + + -- Various files used by runshell to set up env vars used by the + -- linker shims. + writeFile (top "libdirs") (unlines libdirs) + writeFile (top "gconvdir") + (parentDir $ Prelude.head $ filter ("/gconv/" `isInfixOf`) glibclibs) + + let linker = Prelude.head $ filter ("ld-linux" `isInfixOf`) libs' + mapM_ (installLinkerShim top linker) exes + +{- Installs a linker shim script around a binary. + - + - Note that each binary is put into its own separate directory, + - to avoid eg git looking for binaries in its directory rather + - than in PATH. + - + - The linker is symlinked to a file with the same basename as the binary, + - since that looks better in ps than "ld-linux.so". + -} +installLinkerShim :: FilePath -> FilePath -> FilePath -> IO () +installLinkerShim top linker exe = do + createDirectoryIfMissing True (top shimdir) + createDirectoryIfMissing True (top exedir) + ifM (isSymbolicLink <$> getSymbolicLinkStatus exe) + ( do + sl <- readSymbolicLink exe + nukeFile exe + nukeFile exedest + -- Assume that for a symlink, the destination + -- will also be shimmed. + let sl' = ".." takeFileName sl takeFileName sl + createSymbolicLink sl' exedest + , renameFile exe exedest + ) + link <- relPathDirToFile (top exedir) (top ++ linker) + unlessM (doesFileExist (top exelink)) $ + createSymbolicLink link (top exelink) + writeFile exe $ unlines + [ "#!/bin/sh" + , "GIT_ANNEX_PROGRAMPATH=\"$0\"" + , "export GIT_ANNEX_PROGRAMPATH" + , "exec \"$GIT_ANNEX_DIR/" ++ exelink ++ "\" --library-path \"$GIT_ANNEX_LD_LIBRARY_PATH\" \"$GIT_ANNEX_DIR/shimmed/" ++ base ++ "/" ++ base ++ "\" \"$@\"" + ] + modifyFileMode exe $ addModes executeModes + where + base = takeFileName exe + shimdir = "shimmed" base + exedir = "exe" + exedest = top shimdir base + exelink = exedir base + +installFile :: FilePath -> FilePath -> IO () +installFile top f = do + createDirectoryIfMissing True destdir + void $ copyFileExternal CopyTimeStamps f destdir + where + destdir = inTop top $ parentDir f + +checkExe :: FilePath -> IO Bool +checkExe f + | ".so" `isSuffixOf` f = return False + | otherwise = ifM (isExecutable . fileMode <$> getFileStatus f) + ( checkFileExe <$> readProcess "file" ["-L", f] + , return False + ) + +{- Check that file(1) thinks it's a Linux ELF executable, or possibly + - a shared library (a few executables like ssh appear as shared libraries). -} +checkFileExe :: String -> Bool +checkFileExe s = and + [ "ELF" `isInfixOf` s + , "executable" `isInfixOf` s || "shared object" `isInfixOf` s + ] diff --git a/Build/MakeMans.hs b/Build/MakeMans.hs new file mode 100644 index 0000000000..25e09c4aa7 --- /dev/null +++ b/Build/MakeMans.hs @@ -0,0 +1,15 @@ +{- Build man pages, for use by Makefile + - + - Copyright 2016 Joey Hess + - + - Licensed under the GNU GPL version 3 or higher. + -} + +{-# OPTIONS_GHC -fno-warn-tabs #-} + +module Main where + +import Build.Mans + +main :: IO () +main = buildMansOrWarn diff --git a/Build/Mans.hs b/Build/Mans.hs new file mode 100644 index 0000000000..2ea9b4197c --- /dev/null +++ b/Build/Mans.hs @@ -0,0 +1,63 @@ +{- Build man pages. + - + - Copyright 2016 Joey Hess + - + - Licensed under the GNU GPL version 3 or higher. + -} + +{-# OPTIONS_GHC -fno-warn-tabs #-} + +module Build.Mans where + +import System.Directory +import System.FilePath +import Data.List +import Control.Monad +import System.Process +import System.Exit +import Data.Maybe +import Utility.Exception +import Control.Applicative +import Prelude + +buildMansOrWarn :: IO () +buildMansOrWarn = do + mans <- buildMans + when (any isNothing mans) $ + error "mdwn2man failed" + +buildMans :: IO [Maybe FilePath] +buildMans = do + mansrc <- filter isManSrc <$> getDirectoryContents "doc" + createDirectoryIfMissing False "man" + forM mansrc $ \f -> do + let src = "doc" f + let dest = srcToDest src + srcm <- getModificationTime src + destm <- catchMaybeIO $ getModificationTime dest + if (Just srcm > destm) + then do + r <- system $ unwords + [ "./Build/mdwn2man" + , progName src + , "1" + , src + , "> " ++ dest + ] + if r == ExitSuccess + then return (Just dest) + else return Nothing + else return (Just dest) + +isManSrc :: FilePath -> Bool +isManSrc s + | not (takeExtension s == ".mdwn") = False + | otherwise = "git-annex" `isPrefixOf` f || "git-remote-" `isPrefixOf` f + where + f = takeFileName s + +srcToDest :: FilePath -> FilePath +srcToDest s = "man" progName s ++ ".1" + +progName :: FilePath -> FilePath +progName = dropExtension . takeFileName diff --git a/Build/NullSoftInstaller.hs b/Build/NullSoftInstaller.hs new file mode 100644 index 0000000000..0f56eef12b --- /dev/null +++ b/Build/NullSoftInstaller.hs @@ -0,0 +1,199 @@ +{- Generates a NullSoft installer program for git-annex on Windows. + - + - This uses the Haskell nsis package to generate a .nsi file, + - which is then used to produce git-annex-installer.exe + - + - The installer includes git-annex, and utilities it uses, with the + - exception of git and some utilities that are bundled with git. + - The user needs to install git separately, and the installer checks + - for that. + - + - To build the installer, git-annex should already be built to + - ./git-annex.exe and the necessary utility programs + - (specifically rsync) + - already installed in PATH from msys32. + - + - Copyright 2013-2015 Joey Hess + - + - Licensed under the GNU GPL version 3 or higher. + -} + +{-# LANGUAGE OverloadedStrings, FlexibleContexts #-} + +import Development.NSIS +import System.FilePath +import Control.Monad +import Control.Applicative +import Data.String +import Data.Maybe +import Data.Char +import Data.List (nub, isPrefixOf) + +import Utility.Tmp.Dir +import Utility.Path +import Utility.CopyFile +import Utility.SafeCommand +import Utility.Process +import Utility.Exception +import Utility.Directory +import Build.BundledPrograms + +main = do + withTmpDir "nsis-build" $ \tmpdir -> do + let gitannex = tmpdir gitannexprogram + mustSucceed "ln" [File "git-annex.exe", File gitannex] + let license = tmpdir licensefile + mustSucceed "sh" [Param "-c", Param $ "zcat standalone/licences.gz > '" ++ license ++ "'"] + webappscript <- vbsLauncher tmpdir "git-annex-webapp" "git annex webapp" + autostartscript <- vbsLauncher tmpdir "git-annex-autostart" "git annex assistant --autostart" + let htmlhelp = tmpdir "git-annex.html" + writeFile htmlhelp htmlHelpText + let gitannexcmd = tmpdir "git-annex.cmd" + writeFile gitannexcmd "git annex %*" + writeFile nsifile $ makeInstaller + gitannex gitannexcmd license htmlhelp winPrograms + [ webappscript, autostartscript ] + mustSucceed "makensis" [File nsifile] + removeFile nsifile -- left behind if makensis fails + where + nsifile = "git-annex.nsi" + mustSucceed cmd params = do + r <- boolSystem cmd params + case r of + True -> return () + False -> error $ cmd ++ " failed" + +{- Generates a .vbs launcher which runs a command without any visible DOS + - box. It expects to be passed the directory where git-annex is installed. -} +vbsLauncher :: FilePath -> String -> String -> IO String +vbsLauncher tmpdir basename cmd = do + let f = tmpdir basename ++ ".vbs" + writeFile f $ unlines + [ "Set objshell=CreateObject(\"Wscript.Shell\")" + , "objShell.CurrentDirectory = Wscript.Arguments.item(0)" + , "objShell.Run(\"" ++ cmd ++ "\"), 0, False" + ] + return f + +gitannexprogram :: FilePath +gitannexprogram = "git-annex.exe" + +licensefile :: FilePath +licensefile = "git-annex-licenses.txt" + +installer :: FilePath +installer = "git-annex-installer.exe" + +uninstaller :: FilePath +uninstaller = "git-annex-uninstall.exe" + +gitInstallDir :: Exp FilePath +gitInstallDir = fromString "$PROGRAMFILES\\Git" + +-- This intentionally has a different name than git-annex or +-- git-annex-webapp, since it is itself treated as an executable file. +-- Also, on XP, the filename is displayed, not the description. +startMenuItem :: Exp FilePath +startMenuItem = "$SMPROGRAMS/Git Annex (Webapp).lnk" + +oldStartMenuItem :: Exp FilePath +oldStartMenuItem = "$SMPROGRAMS/git-annex.lnk" + +autoStartItem :: Exp FilePath +autoStartItem = "$SMSTARTUP/git-annex-autostart.lnk" + +needGit :: Exp String +needGit = strConcat + [ fromString "You need git installed to use git-annex. Looking at " + , gitInstallDir + , fromString " , it seems to not be installed, " + , fromString "or may be installed in another location. " + , fromString "You can install git from http:////git-scm.com//" + ] + +makeInstaller :: FilePath -> FilePath -> FilePath -> FilePath -> [FilePath] -> [FilePath] -> String +makeInstaller gitannex gitannexcmd license htmlhelp extrabins launchers = nsis $ do + name "git-annex" + outFile $ str installer + {- Installing into the same directory as git avoids needing to modify + - path myself, since the git installer already does it. -} + installDir gitInstallDir + requestExecutionLevel Admin + + iff (fileExists gitInstallDir) + (return ()) + (alert needGit) + + -- Pages to display + page Directory -- Pick where to install + page (License license) + page InstFiles -- Give a progress bar while installing + -- Start menu shortcut + Development.NSIS.createDirectory "$SMPROGRAMS" + createShortcut startMenuItem + [ Target "wscript.exe" + , Parameters "\"$INSTDIR/cmd/git-annex-webapp.vbs\" \"$INSTDIR/cmd\"" + , StartOptions "SW_SHOWNORMAL" + , IconFile "$INSTDIR/usr/bin/git-annex.exe" + , IconIndex 2 + , Description "Git Annex (Webapp)" + ] + delete [RebootOK] $ oldStartMenuItem + createShortcut autoStartItem + [ Target "wscript.exe" + , Parameters "\"$INSTDIR/cmd/git-annex-autostart.vbs\" \"$INSTDIR/cmd\"" + , StartOptions "SW_SHOWNORMAL" + , IconFile "$INSTDIR/usr/bin/git-annex.exe" + , IconIndex 2 + , Description "git-annex autostart" + ] + section "cmd" [] $ do + -- Remove old files no longer installed in the cmd + -- directory. + removefilesFrom "$INSTDIR/cmd" (gitannex:extrabins) + -- Install everything to the same location git puts its + -- bins. This makes "git annex" work in the git bash + -- shell, since git expects to find the git-annex binary + -- there. + setOutPath "$INSTDIR\\usr\\bin" + mapM_ addfile (gitannex:extrabins) + -- This little wrapper is installed in the cmd directory, + -- so that "git-annex" works (as well as "git annex"), + -- when only that directory is in PATH (ie, in a ms-dos + -- prompt window). + setOutPath "$INSTDIR\\cmd" + addfile gitannexcmd + section "meta" [] $ do + -- git opens this file when git annex --help is run. + -- (Program Files/Git/mingw32/share/doc/git-doc/git-annex.html) + setOutPath "$INSTDIR\\mingw32\\share\\doc\\git-doc" + addfile htmlhelp + setOutPath "$INSTDIR" + addfile license + setOutPath "$INSTDIR\\cmd" + mapM_ addfile launchers + writeUninstaller $ str uninstaller + uninstall $ do + delete [RebootOK] $ startMenuItem + delete [RebootOK] $ autoStartItem + removefilesFrom "$INSTDIR/usr/bin" (gitannex:extrabins) + removefilesFrom "$INSTDIR/cmd" (gitannexcmd:launchers) + removefilesFrom "$INSTDIR\\mingw32\\share\\doc\\git-doc" [htmlhelp] + removefilesFrom "$INSTDIR" [license, uninstaller] + where + addfile f = file [] (str f) + removefilesFrom d = mapM_ (\f -> delete [RebootOK] $ fromString $ d ++ "/" ++ takeFileName f) + +winPrograms :: [FilePath] +winPrograms = map (\p -> p ++ ".exe") bundledPrograms + +htmlHelpText :: String +htmlHelpText = unlines + [ "" + , "git-annex help" + , "" + , "For help on git-annex, run \"git annex help\", or" + , "read the man page." + , "" + , " + - + - Licensed under the GNU GPL version 3 or higher. + -} + +module Main where + +import System.Environment (getArgs) +import Data.Maybe +import System.FilePath +import Control.Monad +import Control.Monad.IfElse +import Data.List +import Control.Applicative +import Prelude + +import Utility.PartialPrelude +import Utility.Directory +import Utility.Process +import Utility.Monad +import Utility.SafeCommand +import Utility.Path +import Utility.Exception +import Utility.Env +import Utility.Split + +import qualified Data.Map as M +import qualified Data.Set as S + +type LibMap = M.Map FilePath String + +{- Recursively find and install libs, until nothing new to install is found. -} +mklibs :: FilePath -> [FilePath] -> [(FilePath, FilePath)] -> LibMap -> IO () +mklibs appbase libdirs replacement_libs libmap = do + (new, replacement_libs', libmap') <- installLibs appbase replacement_libs libmap + unless (null new) $ + mklibs appbase (libdirs++new) replacement_libs' libmap' + +{- Returns directories into which new libs were installed. -} +installLibs :: FilePath -> [(FilePath, FilePath)] -> LibMap -> IO ([FilePath], [(FilePath, FilePath)], LibMap) +installLibs appbase replacement_libs libmap = do + (needlibs, replacement_libs', libmap') <- otool appbase replacement_libs libmap + libs <- forM needlibs $ \lib -> do + pathlib <- findLibPath lib + let shortlib = fromMaybe (error "internal") (M.lookup lib libmap') + let fulllib = dropWhile (== '/') lib + let dest = appbase fulllib + let symdest = appbase shortlib + -- This is a hack; libraries need to be in the same + -- directory as the program, so also link them into the + -- extra directory. + let symdestextra = appbase "extra" shortlib + ifM (doesFileExist dest) + ( return Nothing + , do + createDirectoryIfMissing True (parentDir dest) + putStrLn $ "installing " ++ pathlib ++ " as " ++ shortlib + unlessM (boolSystem "cp" [File pathlib, File dest] + <&&> boolSystem "chmod" [Param "644", File dest] + <&&> boolSystem "ln" [Param "-s", File fulllib, File symdest] + <&&> boolSystem "ln" [Param "-s", File (".." fulllib), File symdestextra]) $ + error "library install failed" + return $ Just appbase + ) + return (catMaybes libs, replacement_libs', libmap') + +{- Returns libraries to install. + - + - Note that otool -L ignores DYLD_LIBRARY_PATH, so the + - library files returned may need to be run through findLibPath + - to find the actual libraries to install. + -} +otool :: FilePath -> [(FilePath, FilePath)] -> LibMap -> IO ([FilePath], [(FilePath, FilePath)], LibMap) +otool appbase replacement_libs libmap = do + files <- filterM doesFileExist =<< dirContentsRecursive appbase + process [] files replacement_libs libmap + where + want s = not ("@executable_path" `isInfixOf` s) + -- OSX framekworks such as Cocoa are too tightly tied to + -- a specific OSX version, so don't bundle. + && not (".framework" `isInfixOf` s) + -- libSystem.B is tightly tied to frameworks. + && not ("libSystem.B" `isInfixOf` s) + -- ImageIO.framework uses libPng which is built against a + -- specific version of libz; other versions lack the + -- _inflateValidate symbol. So, avoid bundling libz unless + -- this incompatability is resolved. + && not ("libz." `isInfixOf` s) + process c [] rls m = return (nub $ concat c, rls, m) + process c (file:rest) rls m = do + _ <- boolSystem "chmod" [Param "755", File file] + libs <- filter want . parseOtool + <$> readProcess "otool" ["-L", file] + expanded_libs <- expand_rpath libs replacement_libs file + let rls' = nub $ rls ++ (zip libs expanded_libs) + m' <- install_name_tool file libs expanded_libs m + process (expanded_libs:c) rest rls' m' + +findLibPath :: FilePath -> IO FilePath +findLibPath l = go =<< getEnv "DYLD_LIBRARY_PATH" + where + go Nothing = return l + go (Just p) = fromMaybe l + <$> firstM doesFileExist (map ( f) (splitc ':' p)) + f = takeFileName l + +{- Expands any @rpath in the list of libraries. + - + - This is done by the nasty method of running the command with a dummy + - option (so it doesn't do anything.. hopefully!) and asking the dynamic + - linker to print expanded rpaths. + -} +expand_rpath :: [String] -> [(FilePath, FilePath)] -> FilePath -> IO [String] +expand_rpath libs replacement_libs cmd + | any ("@rpath" `isInfixOf`) libs = do + installed <- M.fromList . Prelude.read + <$> readFile "tmp/standalone-installed" + let origcmd = case M.lookup cmd installed of + Nothing -> cmd + Just cmd' -> cmd' + s <- catchDefaultIO "" $ readProcess "sh" ["-c", probe origcmd] + let m = if (null s) + then M.fromList replacement_libs + else M.fromList $ mapMaybe parse $ lines s + return $ map (replacem m) libs + | otherwise = return libs + where + probe c = "DYLD_PRINT_RPATHS=1 " ++ c ++ " --getting-rpath-dummy-option 2>&1 | grep RPATH" + parse s = case words s of + ("RPATH":"successful":"expansion":"of":old:"to:":new:[]) -> + Just (old, new) + _ -> Nothing + replacem m l = fromMaybe l $ M.lookup l m + +parseOtool :: String -> [FilePath] +parseOtool = catMaybes . map parse . lines + where + parse l + | "\t" `isPrefixOf` l = headMaybe $ words l + | otherwise = Nothing + +{- Adjusts binaries to use libraries bundled with it, rather than the + - system libraries. -} +install_name_tool :: FilePath -> [FilePath] -> [FilePath] -> LibMap -> IO LibMap +install_name_tool _ [] _ libmap = return libmap +install_name_tool binary libs expanded_libs libmap = do + let (libnames, libmap') = getLibNames expanded_libs libmap + let params = concatMap change $ zip libs libnames + ok <- boolSystem "install_name_tool" $ params ++ [File binary] + unless ok $ + error $ "install_name_tool failed for " ++ binary + return libmap' + where + change (lib, libname) = + [ Param "-change" + , File lib + , Param $ "@executable_path/" ++ libname + ] + +getLibNames :: [FilePath] -> LibMap -> ([FilePath], LibMap) +getLibNames libs libmap = go [] libs libmap + where + go c [] m = (reverse c, m) + go c (l:rest) m = + let (f, m') = getLibName l m + in go (f:c) rest m' + +{- Uses really short names for the library files it installs, because + - binaries have arbitrarily short RPATH field limits. -} +getLibName :: FilePath -> LibMap -> (FilePath, LibMap) +getLibName lib libmap = case M.lookup lib libmap of + Just n -> (n, libmap) + Nothing -> (nextfreename, M.insert lib nextfreename libmap) + where + names = map pure ['A' .. 'Z'] ++ + [[n, l] | n <- ['0' .. '9'], l <- ['A' .. 'Z']] + used = S.fromList $ M.elems libmap + nextfreename = fromMaybe (error "ran out of short library names!") $ + headMaybe $ dropWhile (`S.member` used) names + +main :: IO () +main = getArgs >>= go + where + go [] = error "specify OSXAPP_BASE" + go (appbase:_) = mklibs appbase [] [] M.empty diff --git a/Build/Standalone.hs b/Build/Standalone.hs new file mode 100644 index 0000000000..5cb09cd609 --- /dev/null +++ b/Build/Standalone.hs @@ -0,0 +1,52 @@ +{- Makes standalone bundle. + - + - Copyright 2012 Joey Hess + - + - Licensed under the GNU GPL version 3 or higher. + -} + +{-# LANGUAGE CPP #-} + +module Main where + +import Control.Monad.IfElse +import System.Environment +import System.FilePath +import Control.Monad +import Build.BundledPrograms + +import Utility.SafeCommand +import Utility.Path +import Utility.Directory + +progDir :: FilePath -> FilePath +#ifdef darwin_HOST_OS +progDir topdir = topdir +#else +progDir topdir = topdir "bin" +#endif + +extraProgDir :: FilePath -> FilePath +extraProgDir topdir = topdir "extra" + +installProg :: FilePath -> FilePath -> IO (FilePath, FilePath) +installProg dir prog = searchPath prog >>= go + where + go Nothing = error $ "cannot find " ++ prog ++ " in PATH" + go (Just f) = do + let dest = dir takeFileName f + unlessM (boolSystem "install" [File f, File dest]) $ + error $ "install failed for " ++ prog + return (dest, f) + +main :: IO () +main = getArgs >>= go + where + go [] = error "specify topdir" + go (topdir:_) = do + installed <- forM + [ (progDir topdir, preferredBundledPrograms) + , (extraProgDir topdir, extraBundledPrograms) ] $ \(dir, progs) -> do + createDirectoryIfMissing True dir + forM progs $ installProg dir + writeFile "tmp/standalone-installed" (show (concat installed)) diff --git a/Build/TestConfig.hs b/Build/TestConfig.hs new file mode 100644 index 0000000000..2f7213f460 --- /dev/null +++ b/Build/TestConfig.hs @@ -0,0 +1,130 @@ +{- Tests the system and generates SysConfig. -} + +{-# OPTIONS_GHC -fno-warn-tabs #-} + +module Build.TestConfig where + +import Utility.Path +import Utility.Monad +import Utility.SafeCommand +import Utility.Directory + +import System.IO +import System.FilePath + +type ConfigKey = String +data ConfigValue = + BoolConfig Bool | + StringConfig String | + MaybeStringConfig (Maybe String) | + MaybeBoolConfig (Maybe Bool) +data Config = Config ConfigKey ConfigValue + +type Test = IO Config +type TestName = String +data TestCase = TestCase TestName Test + +instance Show ConfigValue where + show (BoolConfig b) = show b + show (StringConfig s) = show s + show (MaybeStringConfig s) = show s + show (MaybeBoolConfig s) = show s + +instance Show Config where + show (Config key value) = unlines + [ key ++ " :: " ++ valuetype value + , key ++ " = " ++ show value + ] + where + valuetype (BoolConfig _) = "Bool" + valuetype (StringConfig _) = "String" + valuetype (MaybeStringConfig _) = "Maybe String" + valuetype (MaybeBoolConfig _) = "Maybe Bool" + +writeSysConfig :: [Config] -> IO () +writeSysConfig config = writeFile "Build/SysConfig" body + where + body = unlines $ header ++ map show config ++ footer + header = [ + "{- Automatically generated. -}" + , "" + ] + footer = [] + +runTests :: [TestCase] -> IO [Config] +runTests [] = return [] +runTests (TestCase tname t : ts) = do + testStart tname + c <- t + testEnd c + rest <- runTests ts + return $ c:rest + +{- Checks if a command is available by running a command line. -} +testCmd :: ConfigKey -> String -> Test +testCmd k cmdline = do + ok <- boolSystem "sh" [ Param "-c", Param $ quiet cmdline ] + return $ Config k (BoolConfig ok) + +{- Ensures that one of a set of commands is available by running each in + - turn. The Config is set to the first one found. -} +selectCmd :: ConfigKey -> [(String, String)] -> Test +selectCmd k = searchCmd + (return . Config k . StringConfig) + (\cmds -> do + testEnd $ Config k $ BoolConfig False + error $ "* need one of these commands, but none are available: " ++ show cmds + ) + +maybeSelectCmd :: ConfigKey -> [(String, String)] -> Test +maybeSelectCmd k = searchCmd + (return . Config k . MaybeStringConfig . Just) + (\_ -> return $ Config k $ MaybeStringConfig Nothing) + +searchCmd :: (String -> Test) -> ([String] -> Test) -> [(String, String)] -> Test +searchCmd success failure cmdsparams = search cmdsparams + where + search [] = failure $ fst $ unzip cmdsparams + search ((c, params):cs) = do + ok <- boolSystem "sh" [ Param "-c", Param $ quiet $ c ++ " " ++ params ] + if ok + then success c + else search cs + +{- Finds a command, either in PATH or perhaps in a sbin directory not in + - PATH. If it's in PATH the config is set to just the command name, + - but if it's found outside PATH, the config is set to the full path to + - the command. -} +findCmdPath :: ConfigKey -> String -> Test +findCmdPath k command = do + ifM (inPath command) + ( return $ Config k $ MaybeStringConfig $ Just command + , do + r <- getM find ["/usr/sbin", "/sbin", "/usr/local/sbin"] + return $ Config k $ MaybeStringConfig r + ) + where + find d = + let f = d command + in ifM (doesFileExist f) ( return (Just f), return Nothing ) + +quiet :: String -> String +quiet s = s ++ " >/dev/null 2>&1" + +testStart :: TestName -> IO () +testStart s = do + putStr $ " checking " ++ s ++ "..." + hFlush stdout + +testEnd :: Config -> IO () +testEnd (Config _ (BoolConfig True)) = status "yes" +testEnd (Config _ (BoolConfig False)) = status "no" +testEnd (Config _ (StringConfig s)) = status s +testEnd (Config _ (MaybeStringConfig (Just s))) = status s +testEnd (Config _ (MaybeStringConfig Nothing)) = status "not available" +testEnd (Config _ (MaybeBoolConfig (Just True))) = status "yes" +testEnd (Config _ (MaybeBoolConfig (Just False))) = status "no" +testEnd (Config _ (MaybeBoolConfig Nothing)) = status "unknown" + +status :: String -> IO () +status s = putStrLn $ ' ':s diff --git a/Build/Version.hs b/Build/Version.hs new file mode 100644 index 0000000000..69752a0897 --- /dev/null +++ b/Build/Version.hs @@ -0,0 +1,69 @@ +{- Package version determination. -} + +{-# OPTIONS_GHC -fno-warn-tabs #-} + +module Build.Version where + +import Data.List +import System.Environment +import Data.Char +import System.Process +import Control.Applicative +import Prelude + +import Utility.Monad +import Utility.Exception + +type Version = String + +{- Set when making an official release. (Distribution vendors should set + - this too.) -} +isReleaseBuild :: IO Bool +isReleaseBuild = (== Just "1") <$> catchMaybeIO (getEnv "RELEASE_BUILD") + +{- Version comes from the CHANGELOG, plus the git rev of the last commit. + - This works for autobuilds, ad-hoc builds, etc. + - + - If git or a git repo is not available, or something goes wrong, + - or this is a release build, just use the version from the CHANGELOG. -} +getVersion :: IO Version +getVersion = do + changelogversion <- getChangelogVersion + ifM (isReleaseBuild) + ( return changelogversion + , catchDefaultIO changelogversion $ do + gitversion <- takeWhile (\c -> isAlphaNum c) <$> readProcess "sh" + [ "-c" + , "git log -n 1 --format=format:'%h'" + ] "" + return $ if null gitversion + then changelogversion + else concat + [ changelogversion + , "-g" + , gitversion + ] + ) + +getChangelogVersion :: IO Version +getChangelogVersion = do + changelog <- readFile "CHANGELOG" + let verline = takeWhile (/= '\n') changelog + return $ middle (words verline !! 1) + where + middle = drop 1 . init + +writeVersion :: Version -> IO () +writeVersion = writeFile "Build/Version" . body + where + body ver = unlines $ concat + [ header + , ["packageversion :: String"] + , ["packageversion = \"" ++ ver ++ "\""] + , footer + ] + header = [ + "{- Automatically generated. -}" + , "" + ] + footer = [] diff --git a/Build/collect-ghc-options.sh b/Build/collect-ghc-options.sh new file mode 100755 index 0000000000..4f75a7202e --- /dev/null +++ b/Build/collect-ghc-options.sh @@ -0,0 +1,12 @@ +#!/bin/sh +# Generate --ghc-options to pass LDFLAGS, CFLAGS, and CPPFLAGS through ghc +# and on to ld, cc, and cpp. +for w in $LDFLAGS; do + printf -- "-optl%s\n" "$w" +done +for w in $CFLAGS; do + printf -- "-optc%s\n" "$w" +done +for w in $CPPFLAGS; do + printf -- "-optc-Wp,%s\n" "$w" +done diff --git a/Build/mdwn2man b/Build/mdwn2man new file mode 100755 index 0000000000..d50c2a0111 --- /dev/null +++ b/Build/mdwn2man @@ -0,0 +1,57 @@ +#!/usr/bin/env perl +# Warning: hack + +my $prog=shift; +my $section=shift; + +print ".TH $prog $section\n"; + +while (<>) { + s{(\\?)\[\[([^\s\|\]]+)(\|[^\s\]]+)?\]\]}{$1 ? "[[$2]]" : $2}eg; + s/\`([^\`]*)\`/\\fB$1\\fP/g; + s/\`//g; + s/^ *\./\\&./g; + if (/^#\s/) { + s/^#\s/.SH /; + <>; # blank; + } + s/^[ \n]+//; + s/^\t/ /; + s/-/\\-/g; + s/git\\-annex/git-annex/g; + s/^Warning:.*mdwn2man.*//g; + s/^$/.PP\n/; + s/^\*\s+(.*)/.IP "$1"/; + next if $_ eq ".PP\n" && $skippara; + if (/^.IP /) { + $inlist=1; + $spippara=0; + } + elsif (/^.SH/) { + $skippara=0; + $inlist=0; + } + elsif (/^\./) { + $skippara=1; + } + else { + $skippara=0; + } + if ($inlist && $_ eq ".PP\n") { + $_=".IP\n"; + } + + if ($inNAME) { + # make lexgrog happy + s/^git-annex (\w)/git-annex-$1/; + } + if ($_ eq ".SH NAME\n") { + $inNAME=1; + } + else { + $inNAME=0; + } + s/\\"/\\\\"/g; # hack for git-annex-shell's quotes around SSH_ORIGINAL_COMMAND + + print $_; +} diff --git a/BuildFlags.hs b/BuildFlags.hs new file mode 100644 index 0000000000..e750506e6e --- /dev/null +++ b/BuildFlags.hs @@ -0,0 +1,109 @@ +{- git-annex build flags + - + - Copyright 2013-2017 Joey Hess + - + - Licensed under the GNU GPL version 3 or higher. + -} + +{-# LANGUAGE CPP #-} + +module BuildFlags where + +import Data.List +import Data.Ord +import qualified Data.CaseInsensitive as CI + +buildFlags :: [String] +buildFlags = filter (not . null) + [ "" +#ifdef WITH_ASSISTANT + , "Assistant" +#else +#warning Building without the assistant. +#endif +#ifdef WITH_WEBAPP + , "Webapp" +#else +#warning Building without the webapp. You probably need to install Yesod.. +#endif +#ifdef WITH_PAIRING + , "Pairing" +#else +#warning Building without local pairing. +#endif +#ifdef WITH_S3 + , "S3" +#if MIN_VERSION_aws(0,10,6) + ++ "(multipartupload)" +#endif +#if MIN_VERSION_aws(0,13,0) + ++ "(storageclasses)" +#endif +#else +#warning Building without S3. +#endif +#ifdef WITH_WEBDAV + , "WebDAV" +#else +#warning Building without WebDAV. +#endif +#ifdef WITH_INOTIFY + , "Inotify" +#endif +#ifdef WITH_FSEVENTS + , "FsEvents" +#endif +#ifdef WITH_KQUEUE + , "Kqueue" +#endif +#ifdef WITH_DBUS + , "DBus" +#endif +#ifdef WITH_DESKTOP_NOTIFY + , "DesktopNotify" +#endif +#ifdef WITH_CONCURRENTOUTPUT + , "ConcurrentOutput" +#else +#warning Building without ConcurrentOutput +#endif +#ifdef WITH_TORRENTPARSER + , "TorrentParser" +#endif +#ifdef WITH_MAGICMIME + , "MagicMime" +#endif + -- Always enabled now, but users may be used to seeing these flags + -- listed. + , "Feeds" + , "Testsuite" + ] + +-- Not a complete list, let alone a listing transitive deps, but only +-- the ones that are often interesting to know. +dependencyVersions :: [String] +dependencyVersions = map fmt $ sortBy (comparing (CI.mk . fst)) + [ ("feed", VERSION_feed) + , ("uuid", VERSION_uuid) + , ("bloomfilter", VERSION_bloomfilter) + , ("http-client", VERSION_http_client) + , ("persistent-sqlite", VERSION_persistent_sqlite) + , ("cryptonite", VERSION_cryptonite) +#ifdef WITH_S3 + , ("aws", VERSION_aws) +#endif +#ifdef WITH_WEBDAV + , ("DAV", VERSION_DAV) +#endif +#ifdef WITH_TORRENTPARSER + , ("torrent", VERSION_torrent) +#endif +#ifdef WITH_WEBAPP + , ("yesod", VERSION_yesod) +#endif +#ifdef TOOL_VERSION_ghc + , ("ghc", TOOL_VERSION_ghc) +#endif + ] + where + fmt (p, v) = p ++ "-" ++ v diff --git a/BuildInfo.hs b/BuildInfo.hs new file mode 100644 index 0000000000..465ac372ef --- /dev/null +++ b/BuildInfo.hs @@ -0,0 +1,16 @@ +{- git-annex build info + - + - Copyright 2013-2017 Joey Hess + - + - Licensed under the GNU GPL version 3 or higher. + -} + +{-# LANGUAGE CPP #-} + +module BuildInfo where + +-- This file is generated by the configure program with the results of its +-- probing. +#include "Build/SysConfig" +-- This file is automatically generated from the CHANGELOG version +#include "Build/Version" diff --git a/CHANGELOG b/CHANGELOG new file mode 100644 index 0000000000..6addfe4b5c --- /dev/null +++ b/CHANGELOG @@ -0,0 +1,5815 @@ +git-annex (6.20180927) UNRELEASED; urgency=medium + + * Multiple --from and --to options can be used in a single command, + eg git annex copy -J2 --to foo --to bar will copy files to two remotes + at the same time. + * sync: Warn when a remote's export is not updated to the current + tree because export tracking is not configured. + + -- Joey Hess Thu, 27 Sep 2018 15:27:20 -0400 + +git-annex (6.20180926) upstream; urgency=medium + + [ Joey Hess ] + * Fixes a reversion in the last release that broke interoperation with + older versions of git-annex-shell. + * init: Improve generated post-receive hook, so it won't fail when + run on a system whose git-annex is too old to support git-annex post-receive + * init: Update the post-receive hook when re-run in an existing repository. + * S3: Fix url construction bug when the publicurl has been set to an url + that does not end with a slash. + * --debug shows urls accessed by git-annex, like it used to do when + git-annex used wget and curl. + * Fix support for filenames containing newlines when querying git + cat-file, though less efficiently than other filenames. + This should make git-annex fully support filenames containing newlines + as the rest of git's interface is used in newline-safe ways. + * Added -z option to git-annex commands that use --batch, useful for + supporting filenames containing newlines. + * Added annex.maxextensionlength for use cases where extensions longer + than 4 characters are needed. + * Added remote.name.annex-security-allow-unverified-downloads, a + per-remote setting for annex.security.allow-unverified-downloads. + * More FreeBSD build fixes. + + [ Yaroslav Halchenko ] + * debian/control + + add netbase to Depends: since required for basic tcp interactions + (see e.g. https://github.com/nipy/heudiconv/issues/260) + + -- Joey Hess Wed, 26 Sep 2018 12:56:49 -0400 + +git-annex (6.20180913) upstream; urgency=medium + + * When --batch is used with matching options like --in, --metadata, + etc, only operate on the provided files when they match those options. + Otherwise, a blank line is output in the batch protocol. + Affected commands: find, add, whereis, drop, copy, move, get + * Make metadata --batch combined with matching options refuse to run, + since it does not seem worth supporting that combination. + * v6 add: Take advantage of improved SIGPIPE handler in git 2.5 to + speed up the clean filter by not reading the file content from the + pipe. This also avoids git buffering the whole file content in memory. + * v6: After updating the worktree for an add/drop, update git's index, + so git status will not show the files as modified. + * v6: When annex.largefiles is not configured for a file, running git + add or git commit, or otherwise using git to stage a file + will add it to the annex if the file was in the annex before, + and to git otherwise. This is to avoid accidental conversion. + Note that git-annex add's behavior has not changed. + * v6: Update associated files database when git has staged changes + to pointer files. + * v6: Fix some race conditions. + * v6: Fix annex object file permissions when git-annex add is run + on a modified unlocked file, and in some related cases. + * v6: When a file is unlocked but has not been modified, + and the unlocking is only staged, git-annex add did not lock it. + Now it will, for consistency with how modified files are handled and + with v5. + * Fix git command queue to be concurrency safe. + * linux standalone: When LOCPATH is already set, use it instead of the + bundled locales. It can be set to an empty string to use the system + locales too. + * Stop using external hash programs, since cryptonite is faster. + * Fix build on FreeBSD. + * S3: Support buckets with versioning enabled. When a remote + is configured with exporttree=yes versioning=yes, git-annex can + download past versions of exported files from it. + * S3: Multipart uploads are now only supported when git-annex is built + with aws-0.16.0 or later, as earlier versions of the library don't + support versioning with multipart uploads. + * S3: Support AWS_SESSION_TOKEN. + * Don't use GIT_PREFIX when GIT_WORK_TREE=. because it seems git + does not intend GIT_WORK_TREE to be relative to GIT_PREFIX in that + case, despite GIT_WORK_TREE=.. being relative to GIT_PREFIX. + * Don't use GIT_PREFIX to fix up a relative GIT_DIR, because + git 2.11 sets GIT_PREFIX to a path it's not relative to. + and apparently GIT_DIR is never relative to GIT_PREFIX. + * git-annex.cabal: Fix build without assistant, and some other refinements. + Thanks fftehnik. + + -- Joey Hess Thu, 13 Sep 2018 15:50:38 -0400 + +git-annex (6.20180807) upstream; urgency=medium + + * S3: Support credential-less download from remotes configured + with public=yes exporttree=yes. + * Fix reversion in display of http 404 errors. + * Added remote.name.annex-speculate-present config that can be used to + make cache remotes. + * Added --accessedwithin matching option. + * Added annex.commitmessage config that can specify a commit message + for the git-annex branch instead of the usual "update". + * Fix wrong sorting of remotes when using -J, it was sorting by uuid, + rather than cost. + * addurl: Include filename in --json-progress output. + * Fix git-annex branch data loss that could occur after + git-annex forget --drop-dead. + + -- Joey Hess Tue, 07 Aug 2018 16:22:05 -0400 + +git-annex (6.20180719) upstream; urgency=medium + + * Support working trees set up by git-worktree. + * Improve support for repositories created with --separate-git-dir. + * Support configuring remote.web.annex-cost and remote.bittorrent.annex-cost + * addurl: When security configuration prevents downloads with youtube-dl, + still check if the url is one that it supports, and fail downloading + it, instead of downloading the raw web page. + * Send User-Agent and any configured annex.http-headers when downloading + with http, fixes reversion introduced when switching to http-client. + * Fix reversion introduced in version 6.20180316 that caused git-annex to + stop processing files when unable to contact a ssh remote. + * v6: Work around git bug that runs smudge/clean filters at the top of the + repository while passing them a relative GIT_WORK_TREE that may point + outside of the repository, by using GIT_PREFIX to get back to the + subdirectory where a relative GIT_WORK_TREE is valid. + * p2p --pair: Fix interception of the magic-wormhole pairing code, + which since 0.8.2 it has sent to stderr rather than stdout. + * info: Display uuid and description when a repository is identified by + uuid, and for "here". + * unused --from: Allow specifiying a repository by uuid or description. + * linux standalone: Generate locale files in ~/.cache/git-annex/locales/ + so they're available even when the standalone tarball is installed + in a directory owned by root. Note that this prevents using the + standalone bundle in environments where HOME is not writable. + * Include uname command in standalone builds since git-annex uses it. + * git-annex.cabal: Fix network version. + + -- Joey Hess Thu, 19 Jul 2018 13:53:45 -0400 + +git-annex (6.20180626) upstream; urgency=high + + Security fix release for CVE-2018-10857 and CVE-2018-10859 + https://git-annex.branchable.com/security/CVE-2018-10857_and_CVE-2018-10859/ + + * Refuse to download content, that cannot be verified with a hash, + from encrypted special remotes (for CVE-2018-10859), + and from all external special remotes and glacier (for CVE-2018-10857). + In particular, URL and WORM keys stored on such remotes won't + be downloaded. If this affects your files, you can run + `git-annex migrate` on the affected files, to convert them + to use a hash. + * Added annex.security.allow-unverified-downloads, which can override + the above. + * Added annex.security.allowed-url-schemes setting, which defaults + to only allowing http, https, and ftp URLs. Note especially that file:/ + is no longer enabled by default. + * Removed annex.web-download-command, since its interface does not allow + supporting annex.security.allowed-url-schemes across redirects. + If you used this setting, you may want to instead use annex.web-options + to pass options to curl. + * git-annex will refuse to download content from http servers on + localhost, or any private IP addresses, to prevent accidental + exposure of internal data. This can be overridden with the + annex.security.allowed-http-addresses setting. + * Local http proxies will not be used unless allowed by the + annex.security.allowed-http-addresses setting. + * Since the interfaces to curl and youtube-dl do not have a way to + prevent them from accessing localhost or private IP addresses, + they default to not being used for url downloads. + Only when annex.security.allowed-http-addresses=all will curl and + youtube-dl be used. + + Non-security fix changes: + + * Fix build with ghc 8.4+, which broke due to the Semigroup Monoid change. + * version: Show operating system and repository version list + when run outside a git repo too. + * Fix annex-checkuuid implementation, so that remotes configured that + way can be used. + * Fix problems accessing repositories over http when annex.tune.* + is configured. + * External special remotes can now add info to `git annex info $remote`, + by replying to the GETINFO message. + * adb: Android serial numbers are not all 16 characters long, so accept + other lengths. + * Display error messages that come from git-annex-shell when the p2p + protocol is used, so that diskreserve messages, IO errors, etc from + the remote side are visible again. + * When content has been lost from an export remote and + git-annex fsck --from remote has noticed it's gone, re-running + git-annex export or git-annex sync --content will re-upload it. + + -- Joey Hess Fri, 22 Jun 2018 10:36:22 -0400 + +git-annex (6.20180529) upstream; urgency=medium + + * Prevent haskell http-client from decompressing gzip files, so downloads + of such files works the same as it used to with wget and curl. + * Workaround for bug in an old version of cryptonite that broke https + downloads, by using curl for downloads when git-annex is built with it. + * view, vadd: Fix crash when a git submodule has a name starting with a dot. + * Don't allow entering a view with staged or unstaged changes. + * move: --force was accidentially enabling two unrelated behaviors + since 6.20180427. The older behavior, which has never been well + documented and seems almost entirely useless, has been removed. + * copy: --force no longer does anything. + * migrate: Fix bug in migration between eg SHA256 and SHA256E, + that caused the extension to be included in SHA256 keys, + and omitted from SHA256E keys. + (Bug introduced in version 6.20170214) + * migrate: Check for above bug when migrating from SHA256 to SHA256 + (and same for SHA1 to SHA1 etc), and remove the extension that should + not be in the SHA256 key. + * fsck: Detect and warn when keys need an upgrade, either to fix up + from the above migrate bug, or to add missing size information + (a long ago transition), or because of a few other past key related + bugs. + * git-annex-shell: GIT_ANNEX_SHELL_APPENDONLY makes it allow writes, + but not deletion of annexed content. Note that securing pushes to + the git repository is left up to the user. + * setpresentkey: Added --batch support. + + -- Joey Hess Tue, 29 May 2018 13:05:26 -0400 + +git-annex (6.20180509) upstream; urgency=medium + + * The old git-annex Android app is now deprecated in favor of running + git-annex in termux. + * runshell: Use proot when running on Android, to work around + Android 8's ill-advised seccomp filtering of system calls, + including ones crucial for reliable thread locking. + (This will only work with termux's version of proot.) + * Fix bug in last release that crashes when using + --all or running git-annex in a bare repository. May have also + affected git-annex unused and git-annex info. + * Fix bug in last release that prevented the webapp opening on + non-Linux systems. + * Support building with hinotify-0.3.10. + * Display error message when http download fails. + * Avoid forward retry when 0 bytes were received. + + -- Joey Hess Wed, 09 May 2018 16:20:26 -0400 + +git-annex (6.20180427) upstream; urgency=medium + + * move: Now takes numcopies configuration, and required content + configuration into account, and refuses to reduce the current + number of copies of a file, or remove content that a repository + requires. --force can override these checks. + Note that it's still allowed to move the content of a file + from one repository to another when numcopies is not satisfied, as long + as the move does not result in there being fewer copies. + * Fix mangling of --json output of utf-8 characters when not + running in a utf-8 locale. + * Fix build with yesod 1.6. + * Clean up some build warnings with newer versions of ghc and haskell + libraries. + * runshell: Unset LD_PRELOAD since preloaded libraries from the host + system may not get along with the bundled linker. + * runshell: Added some tweaks to make git-annex work in termux on + Android. The regular arm standalone tarball now works in termux. + * Webapp: Support being run inside termux on Android, and offer to set up + a repository on the sdcard. + * Assistant: Integrate with Termux:Boot, so when it's installed, the + assistant is autostarted on boot. + * Assistant: Fix installation of menus, icons, etc when run + from within runshell. + * import: Avoid buffering all filenames to be imported in memory. + * Improve memory use and speed of --all and git-annex info remote, + by not buffering list of all keys. + + -- Joey Hess Fri, 27 Apr 2018 12:36:20 -0400 + +git-annex (6.20180409) upstream; urgency=medium + + * Added adb special remote which allows exporting files to Android devices. + * For url downloads, git-annex now defaults to using a http library, + rather than wget or curl. But, if annex.web-options is set, it will + use curl. To use the .netrc file, run: + git config annex.web-options --netrc + * git-annex no longer uses wget (and wget is no longer shipped with + git-annex builds). + * Enable HTTP connection reuse across multiple files for improved speed. + * Fix calculation of estimated completion for progress meter. + * OSX app: Work around libz/libPng/ImageIO.framework version skew + by not bundling libz, assuming OSX includes a suitable libz.1.dylib. + * Added annex.retry, annex.retry-delay, and per-remote versions + to configure transfer retries. + * Also do forward retrying in cases where no exception is thrown, + but the transfer failed. + * When adding a new version of a file, and annex.genmetadata is enabled, + don't copy the data metadata from the old version of the file, + instead use the mtime of the file. + * Avoid running annex.http-headers-command more than once. + * info: Added "combined size of repositories containing these files" + stat when run on a directory. + * info: Changed sorting of numcopies stats table, so it's ordered + by the variance from the desired number of copies. + * Fix resuming a download when using curl. + + -- Joey Hess Mon, 09 Apr 2018 13:03:15 -0400 + +git-annex (6.20180316) upstream; urgency=medium + + * New protocol for communicating with git-annex-shell increases speed + of operations involving ssh remotes. When not transferring large files, + git-annex is between 200% and 400% faster using the new protocol, + and it's just as fast as before when transferring large files. + (When the remote has an old git-annex-shell, git-annex falls back + to the old slower code. This fallback is planned to be removed + after 5 years or so.) + * Note that, due to not using rsync to transfer files over ssh + any longer, permissions and other file metadata of annexed files + will no longer be preserved when copying them to and from ssh remotes. + Other remotes never supported preserving that information, so + this is not considered a regression. + * Fix data loss bug in content locking over tor, when the remote + repository is in direct mode, it neglected to check that the content + was actually present when locking it. This could cause git annex drop + to remove the only copy of a file when it thought the tor remote had + a copy. + * Fix data loss bug when the local repository uses direct mode, and a + locally modified file is dropped from a remote repsitory. The bug + caused the modified file to be counted as a copy of the original file. + (This is not a severe bug because in such a situation, dropping + from the remote and then modifying the file is allowed and has the same + end result.) + * Some downloads will be verified, even when annex.verify=false. + This is done in some edge cases where there's a likelyhood than an + object was downloaded incorrectly. + * Support exporttree=yes for rsync special remotes. + * Added backends for the BLAKE2 family of hashes, when built with + a new enough version of cryptonite. + * Improve SHA*E extension extraction code to not treat parts of the + filename that contain punctuation or other non-alphanumeric characters + as extensions. Before, such characters were filtered out. + * Better ssh connection warmup when using -J for concurrency. + Avoids ugly messages when forced ssh command is not git-annex-shell. + * Fix race condition in ssh warmup that caused git-annex to get + stuck and never process some files when run with high levels of + concurrency. + * Fix reversion introduced in 6.20171214 that caused concurrent + transfers to incorrectly fail with "transfer already in progress". + * Note that Remote/Git.hs now contains AGPL licensed code, + thus the license of git-annex as a whole is AGPL. This was already + the case when git-annex was built with the webapp enabled. + * Include amount of data transferred in progress display. + * Dial back optimisation when building on arm, which prevents + ghc and llc from running out of memory when optimising some files. + (Unfortunately this fix is incomplete due to a ghc bug.) + + -- Joey Hess Fri, 16 Mar 2018 12:10:40 -0400 + +git-annex (6.20180227) upstream; urgency=medium + + * inprogress: Avoid showing failures for files not in progress. + * Added INFO to external special remote protocol. + * Added EXTENSIONS to external special remote protocol. + * datalad < 0.9.1 had a problem in its special remote protocol handling + which is broken by EXTENSIONS. Make the debian git-annex package + conflict with the problem version of datalad. + * fsck: Warn when required content is not present in the repository that + requires it. + * Add gpg-agent to Build-Depends. + Needed to run the test suite. + * --json: When there are multiple lines of notes about a file, make the note + field multiline, rather than the old behavior of only including the + last line. + * git-annex.cabal: Once more try to not build the assistant on the hurd, + hopefully hackage finally recognises that OS. + * Split Test.hs and avoid optimising it much, to need less memory to + compile. + * Fix behavior of --json-progress followed by --json, the latter option + used to disable the former. + * Added --json-error-messages option, which makes messages + that would normally be output to standard error be included in + the json output. + * Remove temporary code added in 6.20160619 to prime the mergedrefs + log. + * importfeed: Fix a failure when downloading with youtube-dl + and the destination subdirectory does not exist yet. + * Added annex.merge-annex-branches config setting which + can be used to disable automatic merge of git-annex branches. + * tips/automatically_adding_metadata/pre-commit-annex: Fix to not + silently skip filenames containing non-ascii characters. + * sync: Fix bug that prevented pulling changes into direct mode + repositories that were committed to remotes using git commit + rather than git-annex sync. + * Makefile: Remove chrpath workaround for bug in cabal, + which is no longer needed. + + -- Joey Hess Tue, 27 Feb 2018 12:04:52 -0400 + +git-annex (6.20180112) upstream; urgency=medium + + * Added inprogress command for accessing files as they are being + downloaded. + * Fix bug introduced in version 6.20171018 that caused some commands + to print out "ok" twice after processing a file. + * addurl: When the file youtube-dl will download is already an annexed + file, don't download it again and fail to overwrite it, instead just do + nothing, like it used to when quvi was used. + * addurl: Fix encoding of filename queried from youtube-dl when in + --fast mode. + * Fix several places where files in .git/annex/ were written with modes + that did not take the core.sharedRepository config into account. + * Improve startup time for commands that do not operate on remotes, + and for tab completion, by not unnessessarily statting paths to + remotes, which used to cause eg, spin-up of removable drives. + * Added remote..annex-checkuuid config, which can be set to false + to disable the default checking of the uuid of remotes that point to + directories. This can be useful to avoid unncessary drive spin-ups and + automounting. + * git-annex.cabal: Add back custom-setup stanza, so cabal new-build works. + * git-annex.cabal: Removed the testsuite build flag; test suite is always + included. + + -- Joey Hess Fri, 12 Jan 2018 15:45:48 -0400 + +git-annex (6.20171214) upstream; urgency=medium + + * Use youtube-dl rather than quvi to download media from web pages, + since quvi is not being actively developed and youtube-dl supports + many more sites. + * addurl --relaxed got slower, since youtube-dl has to hit the network + to check for embedded media. If you relied on --relaxed not hitting the + network for speed reasons, using --relaxed --raw will get the old level + of speed, but can't be used for urls with embedded videos. + * importfeed now downloads things linked to by feeds, even when they are + not media files. + * Removed no longer needed dependency on yesod-default. + * Allow exporttree remotes to be marked as dead. + * initremote, enableremote: Really support gpg subkeys suffixed with an + exclamation mark, which forces gpg to use a specific subkey. + (Previous try had a bug.) + * lookupkey: Support being given an absolute filename to a file + within the current git repository. + * A top-level .noannex file will prevent git-annex init from being used + in a repository. This is useful for repositories that have a policy + reason not to use git-annex. The content of the file will be displayed + to the user who tries to run git-annex init. + + -- Joey Hess Thu, 14 Dec 2017 11:50:48 -0400 + +git-annex (6.20171124) unstable; urgency=medium + + * Display progress meter when uploading a key without size information, + getting the size by statting the content file. + * Fix build with dns-3.0. + + -- Joey Hess Fri, 24 Nov 2017 10:49:36 -0400 + +git-annex (6.20171109) unstable; urgency=medium + + * Fix export of subdir of a branch. + * Fix exporting of non-annexed files to external special remotes. + * unlock, lock: Support --json. + * When there are multiple urls for a file, still treat it as being present + in the web when some urls don't work, as long as at least one url does + work. + * Makefile improvement for sudo make install. + Thanks, Eric Siegerman + * Makefile improvement for BUILDER=stack, use stack to run ghc. + * testremote: Test exporttree. + * Fix directory special remote's cleanup of empty export directories. + + -- Joey Hess Thu, 09 Nov 2017 12:21:49 -0400 + +git-annex (6.20171026) unstable; urgency=medium + + * Windows: Fix reversion that caused the path used to link + to annexed content to include the drive letter and full path, rather + than being relative. (`git annex fix` will fix up after this problem). + * Windows build fixed, and changed to use stack for more reliable build + environment. + * Windows: Remove wget from bundle; it needs libraries that are not + included, and git for windows includes curl which git-annex will use + instead. + * Add day to metadata when annex.genmetadata is enabled. + Thanks, Sean T Parsons + * stack.yaml: Added nix packages section. + Thanks, Sean T Parsons + + -- Joey Hess Thu, 26 Oct 2017 13:56:18 -0400 + +git-annex (6.20171018) unstable; urgency=medium + + * add: Replace work tree file atomically on systems supporting hard + links. Avoids a window where interrupting an add could result in + the file being moved into the annex, with no symlink yet created. + * webdav: Avoid unncessisarily creating the collection at the top + of the repository when storing files there, since that collection + is created by initremote. + (This seems to work around some brokenness of the box.com webdav + server, which caused uploads to be very slow or sometimes fail.) + * webdav: Make --debug show all webdav operations. + * get -J/move -J/copy -J/mirror -J/sync -J: Avoid "transfer already in + progress" errors when two files use the same key. + * Konqueror desktop file location changed to one used by plasma 5. + Thanks, Félix Sipma. + * Avoid repeated checking that files passed on the command line exist. + * Fix build with aws-0.17. + * stack.yaml: Update to lts-9.9. + + -- Joey Hess Wed, 18 Oct 2017 15:40:06 -0400 + +git-annex (6.20171003) unstable; urgency=medium + + * webdav: Improve error message for failed request to include the request + method and path. + * metadata: Added --remove-all. + * Warn when metadata is inherited from a previous version of a file, + to avoid the user being surprised in cases where that behavior is not + desired or expected. + * sync: Added --cleanup, which removes local and remote synced/ branches. + * external: When the external special remote program crashed, a newline + could be output, which messed up the expected output for --batch mode. + * external: Avoid checking EXPORTSUPPORTED for special remotes that are + not configured to use exports. + * test: Fix reversion that made it only run inside a git repository. + * copy, move: Behave same with --fast when sending to remotes located + on a local disk as when sending to other remotes. + * Fix process and file descriptor leak that was exposed when + git-annex was built with ghc 8.2.1. Broke git-annex test on OSX + due to running out of FDs, and may have also leaked in other situations. + * info: Improve cleanup of stale transfer info files. + + -- Joey Hess Tue, 03 Oct 2017 13:18:15 -0400 + +git-annex (6.20170925) unstable; urgency=medium + + * git-annex export: New command, can create and efficiently update + exports of trees to special remotes. + * Use git-annex initremote with exporttree=yes to set up a special remote + for use by git-annex export. + * Implemented export to directory, S3, and webdav special remotes. + * External special remote protocol extended to support export. + Developers of external special remotes should consider if export makes + sense for them and add support. + * sync, assistant: Update tracking exports. + * Support building with feed-1.0, while still supporting older versions. + * init: Display an additional message when it detects a filesystem that + allows writing to files whose write bit is not set. + * S3: Allow removing files from IA. + * webdav: Checking if a non-existent file is present on Box.com + triggered a bug in its webdav support that generates an infinite series + of redirects. Deal with such problems by assuming such behavior means + the file is not present. + * webdav: Fix lack of url-escaping of filenames. Mostly impacted exports + of filenames containing eg spaces. + * webdav: Changed path used on webdav server for temporary files. + + -- Joey Hess Mon, 25 Sep 2017 11:13:58 -0400 + +git-annex (6.20170818) unstable; urgency=high + + * Security fix: Disallow hostname starting with a dash, which + would get passed to ssh and be treated an option. This could + be used by an attacker who provides a crafted repository url + to cause the victim to execute arbitrary code via -oProxyCommand. + CVE-2017-12976 + (The same class of security hole recently affected git itself.) + * git-annex.cabal: Deal with breaking changes in Cabal 2.0. + * Fix build with QuickCheck 2.10. + * fsck: Support --json. + * move, copy: Support --batch. + * Added GIT_ANNEX_VECTOR_CLOCK environment variable, which can be used to + override the default timestamps used in log files in the git-annex + branch. This is a dangerous environment variable; use with caution. + * Fix a git-annex test failure when run on NFS due to NFS lock files + preventing directory removal. + * test: Avoid most situations involving failure to delete test + directories, by forking a worker process and only deleting the test + directory once it exits. + * Disable http-client's default 30 second response timeout when HEADing + an url to check if it exists. Some web servers take quite a long time + to answer a HEAD request. + * Added remote configuration settings annex-ignore-command and + annex-sync-command, which are dynamic equivilants of the annex-ignore + and annex-sync configurations. + * Prevent spaces from being embedded in the name of new WORM keys, + as that handing spaces in keys would complicate things like the + external special remote protocol. + * migrate: WORM keys containing spaces will be migrated to not contain + spaces anymore. + * External special remotes will refuse to operate on keys with spaces in + their names. That has never worked correctly due to the design of the + external special remote protocol. Display an error message suggesting + migration. + * Fix incorrect external special remote documentation, which said that + the filename parameter to the TRANSFER command could not contain + spaces. It can in fact contain spaces. Special remotes implementors + that relied on that may need to fix bugs in their special remotes. + * Fix the external special remotes git-annex-remote-ipfs, + git-annex-remote-torrent and the example.sh template to correctly + support filenames with spaces. + * Windows: Win32 package has subsumed Win32-extras; update dependency. + + -- Joey Hess Fri, 18 Aug 2017 11:19:06 -0400 + +git-annex (6.20170520) unstable; urgency=medium + + * move --to=here moves from all reachable remotes to the local repository. + * initremote, enableremote: Support gpg subkeys suffixed with an + exclamation mark, which forces gpg to use a specific subkey. + * Improve progress display when watching file size, in cases where + a transfer does not resume. + * Fix transfer log file locking problem when running concurrent + transfers. + * Avoid concurrent git-config setting problem when running concurrent + threads. + * metadata: When setting metadata of a file that did not exist, + no error message was displayed, unlike getting metadata and most other + git-annex commands. Fixed this oversight. + * Added annex.resolvemerge configuration, which can be set to false to + disable the usual automatic merge conflict resolution done by git-annex + sync and the assistant. + * sync: Added --no-resolvemerge option. + * Avoid error about git-annex-shell not being found when + syncing with -J with a git remote where git-annex-shell is not + installed. + * Fix bug that prevented transfer locks from working when + run on SMB or other filesystem that does not support fcntl locks + and hard links. + * assistant: Merge changes from refs/remotes/foo/master into master. + Previously, only sync branches were merged. This makes regular git push + into a repository watched by the assistant auto-merge. + * Makefile: Install completions for the fish and zsh shells + when git-annex is built with optparse-applicative-0.14. + * assistant: Don't trust OSX FSEvents's eventFlagItemModified to be called + when the last writer of a file closes it; apparently that sometimes + does not happen, which prevented files from being quickly added. + + -- Joey Hess Mon, 12 Jun 2017 13:37:16 -0400 + +git-annex (6.20170519) unstable; urgency=medium + + * Ssh password prompting improved when using -J for concurrency. + When ssh connection caching is enabled (and when GIT_ANNEX_USE_GIT_SSH + is not set), only one ssh password prompt will be made per host, and + only one ssh password prompt will be made at a time. + * When built with concurrent-output 1.9, ssh password prompts will no + longer interfere with the -J display. + * Removed dependency on MissingH, instead depending on the split library. + * Progress is displayed for transfers of files of unknown size. + * Work around bug in git 2.13.0 involving GIT_COMMON_DIR that broke + merging changes into adjusted branches. + + -- Joey Hess Fri, 19 May 2017 10:37:57 -0400 + +git-annex (6.20170510) unstable; urgency=medium + + * When a http remote does not expose an annex.uuid config, only warn + about it once, not every time git-annex is run. + * multicast: New command, uses uftp to multicast annexed files, for eg + a classroom setting. + * Added remote..annex-push and remote..annex-pull + which can be useful to make remotes that don't get fully synced with + local changes. + * Disable git-annex's support for GIT_SSH and GIT_SSH_COMMAND, unless + GIT_ANNEX_USE_GIT_SSH=1 is also set in the environment. This is + necessary because as feared, the extra -n parameter that git-annex + passes breaks uses of these environment variables that expect exactly + the parameters that git passes. + * enableremote: When enabling a non-special remote, param=value + parameters can't be used, so error out if any are provided. + * enableremote: Fix re-enabling of special remotes that have a git + url, so that eg, encryption key changes take effect. They were silently + ignored, a reversion introduced in 6.20160527. + * gcrypt: Support re-enabling to change eg, encryption parameters. + This was never supported before. + * git annex add -u now supported, analagous to git add -u + * version: Added "dependency versions" line. + * Keys marked as dead are now skipped by --all. + * annex.backend is the new name for what was annex.backends, and takes + a single key-value backend, rather than the unncessary and confusing + list. The old option still works if set. + + -- Joey Hess Wed, 10 May 2017 15:05:22 -0400 + +git-annex (6.20170321) unstable; urgency=medium + + * Bugfix: Passing a command a filename that does not exist sometimes + did not display an error, when a path to a directory was also passed. + * status: Propigate nonzero exit code from git status. + * Linux standalone builds put the bundled ssh last in PATH, + so any system ssh will be preferred over it. + * assistant: Add 1/200th second delay between checking each file + in the full transfer scan, to avoid using too much CPU. + * get -J: Improve distribution of jobs amoung remotes when there are more + jobs than remotes. + * fsck -q: When a file has bad content, include the name of the file + in the warning message. + * Windows: Improve handling of shebang in external special remote + program, searching for the program in the PATH. + * Drop support for building with old versions of dns, http-conduit, + directory, feed, and http-types. + * Windows: Fix bug in shell script shebang lookup code that + caused a "delayed read on closed handle" error. + * git-annex-shell: Fix bug when used with a recently cloned repository, + where "merging" messages were included in the output of configlist + (and perhaps other commands) and caused a "Failed to get annex.uuid + configuration" error. + * Support GIT_SSH and GIT_SSH_COMMAND, which are handled close the same + as they are by git. However, unlike git, git-annex sometimes needs to + pass the -n parameter when using these. + * sync --content-of=path (-C path) added for when you want to sync + only some files' contents, not the whole working tree. + + -- Joey Hess Tue, 21 Mar 2017 11:27:38 -0400 + +git-annex (6.20170301.1) unstable; urgency=medium + + * Fix reversion in yesterday's release that made SHA1E and MD5E backends + not work. + + -- Joey Hess Wed, 01 Mar 2017 12:46:03 -0400 + +git-annex (6.20170301) unstable; urgency=medium + + * No changes from 6.20170228; a new version number was needed due + to a problem with Hackage. + + -- Joey Hess Wed, 01 Mar 2017 12:06:02 -0400 + +git-annex (6.20170228) unstable; urgency=medium + + * Cryptographically secure hashes can be forced to be used in a + repository, by setting annex.securehashesonly. + This does not prevent the git repository from containing links + to insecure hashes, but it does prevent the content of such files + from being added to .git/annex/objects by any method. + * Tighten key parser to prevent SHA1 collision attacks generating + two keys that have the same SHA1. (Only done for keys that contain + a hash). This ensures that signed git commits of annexed files + will remain secure, as long as git-annex is using a secure hashing + backend. + * fsck: Warn about any files whose content is present, that don't + use secure hashes, when annex.securehashesonly is set. + * init: When annex.securehashesonly has been set with git-annex config, + copy that value to the annex.securehashesonly git config. + * Added --securehash option to match files using a secure hash function, + and corresponding securehash preferred content expression. + * sync, merge: Fail when the current branch has no commits yet, instead + of not merging in anything from remotes and appearing to succeed. + * Run ssh with -n whenever input is not being piped into it, + to avoid it consuming stdin that it shouldn't. + This fixes git-annex-checkpresentkey --batch remote, + which didn't output results for all keys passed into it. Other + git-annex commands that communicate with a remote over ssh may also + have been consuming stdin that they shouldn't have, which could have + impacted using them in eg, shell scripts. + * sync: Improve integration with receive.denyCurrentBranch=updateInstead, + displaying error messages from the remote then it fails to update + its checked out branch. + * Added post-recieve hook, which makes updateInstead work with direct + mode and adjusted branches. + * init: Set up the post-receive hook. + * sync: When syncing with a local repository located on a crippled + filesystem, run the post-receive hook there, since it wouldn't get run + otherwise. This makes pushing to repos on FAT-formatted removable + drives update them when receive.denyCurrentBranch=updateInstead. + * config group groupwanted numcopies schedule wanted required: + Avoid displaying extraneous messages about repository auto-init, + git-annex branch merging, etc, when being used to get information. + * adjust: Fix behavior when used in a repository that contains + submodules. + * Run wget with -nv instead of -q, so it will display HTTP errors. + * Run curl with -S, so HTTP errors are displayed, even when + it's otherwise silent. + * When downloading in --json or --quiet mode, use curl in preference + to wget, since curl is able to display only errors to stderr, unlike + wget. + * status: Pass --ignore-submodules=when option on to git status. + * config --set: As well as setting value in git-annex branch, + set local gitconfig. This is needed especially for + annex.securehashesonly, which is read only from local gitconfig and not + the git-annex branch. + * Removed support for building with the old cryptohash library. + Building with that library made git-annex not support SHA3; it's time + for that to always be supported in case SHA2 dominoes. + * git-annex.cabal: Make crypto-api a dependency even when built w/o + webapp and test suite. + + -- Joey Hess Tue, 28 Feb 2017 14:39:47 -0400 + +git-annex (6.20170214) unstable; urgency=medium + + * Increase default cost for p2p remotes from 200 to 1000. + This makes git-annex prefer transferring data from special + remotes when possible. + * Remove -j short option for --json-progress; that option was already + taken for --json. + * vicfg: Include the numcopies configuation. + * config: New command for storing configuration in the git-annex branch. + * annex.autocommit can be configured via git-annex config, to control + the default behavior in all clones of a repository. + * New annex.synccontent config setting, which can be set to true to make + git annex sync default to --content. This may become the default at + some point in the future. As well as being configuable by git config, + it can be configured by git-annex config to control the default + behavior in all clones of a repository. + * stack.yaml: Update to lts-7.18. + * Some optimisations to string splitting code. + * unused: When large files are checked right into git, avoid buffering + their contents in memory. + * unused: Improved memory use significantly when there are a lot + of differences between branches. + * Wormhole pairing will start to provide an appid to wormhole on + 2021-12-31. An appid can't be provided now because Debian stable is going + to ship a older version of git-annex that does not provide an appid. + Assumption is that by 2021-12-31, this version of git-annex will be + shipped in a Debian stable release. If that turns out to not be the + case, this change will need to be cherry-picked into the git-annex in + Debian stable, or its wormhole pairing will break. + * Fix build with aws 0.16. Thanks, aristidb. + * assistant: Make --autostart --foreground wait for the children it + starts. Before, the --foreground was ignored when autostarting. + * initremote: When a uuid= parameter is passed, use the specified + UUID for the new special remote, instead of generating a UUID. + This can be useful in some situations, eg when the same data can be + accessed via two different special remote backends. + * import: Changed how --deduplicate, --skip-duplicates, and + --clean-duplicates determine if a file is a duplicate. + Before, only content known to be present somewhere was considered + a duplicate. Now, any content that has been annexed before will be + considered a duplicate, even if all annexed copies of the data have + been lost. + Note that --clean-duplicates and --deduplicate still check + numcopies, so won't delete duplicate files unless there's an annexed + copy. + * import: --deduplicate and --skip-duplicates were implemented + inneficiently; they unncessarily hashed each file twice. They have + been improved to only hash once. + * import: Added --reinject-duplicates. + * Added git template directory to Linux standalone tarball and OSX + app bundle. + * Improve pid locking code to work on filesystems that don't support hard + links. + * S3: Fix check of uuid file stored in bucket, which was not working. + * Work around sqlite's incorrect handling of umask when creating + databases. + + -- Joey Hess Tue, 14 Feb 2017 14:22:00 -0400 + +git-annex (6.20170101) unstable; urgency=medium + + * XMPP support has been removed from the assistant in this release. + If your repositories used XMPP to keep in sync, that will no longer + work, and you should enable some other remote to keep them in sync. + A ssh server is one way, or use the new Tor pairing feature. + * p2p --pair makes it easy to pair repositories, over Tor, using + Magic Wormhole codes to find the other repository. + See http://git-annex.branchable.com/tips/peer_to_peer_network_with_tor/ + * webapp: The "Share with a friend" and "Share with your other devices" + pages have been changed to pair repositories using Tor and Magic Wormhole. + * metadata --batch: Fix bug when conflicting metadata changes were + made in the same batch run. + * Pass annex.web-options to wget and curl after other options, so that + eg --no-show-progress can be set by the user to disable the default + --show-progress. + * Revert ServerAliveInterval change in 6.20161111, which caused problems + with too many old versions of ssh and unusual ssh configurations. + It should have not been needed anyway since ssh is supposted to + have TCPKeepAlive enabled by default. + * Make all --batch input, as well as fromkey and registerurl stdin + be processed without requiring it to be in the current encoding. + * p2p: --link no longer takes a remote name, instead the --name + option can be used. + * Linux standalone: Improve generation of locale definition files, + supporting locales such as en_GB.UTF-8. + * rekey --force: Incorrectly marked the new key's content as being + present in the local repo even when it was not. + * enable-tor: Put tor sockets in /var/lib/tor-annex/, rather + than in /etc/tor/hidden_service/. + * enable-tor: No longer needs to be run as root. + * enable-tor: When run as a regular user, also tests a connection back to + the hidden service over tor. + * Support all common locations of the torrc file. + * Always use filesystem encoding for all file and handle reads and + writes. + * Fix build with directory-1.3. + * Debian: Suggest tor and magic-wormhole. + * Debian: Build webapp on armel. + + -- Joey Hess Sat, 31 Dec 2016 15:11:04 -0400 + +git-annex (6.20161210) unstable; urgency=medium + + * Linux standalone: Updated ghc to fix its "unable to decommit memory" + bug, which may have resulted in data loss when these builds were used + with Linux kernels older than 4.5. + * enable-tor: New command, enables tor hidden service for P2P syncing. + * p2p: New command, allows linking repositories using a P2P network. + * remotedaemon: Serve tor hidden service. + * Added git-remote-tor-annex, which allows git pull and push to the tor + hidden service. + * remotedaemon: Fork to background by default. Added --foreground switch + to enable old behavior. + * addurl: Fix bug in checking annex.largefiles expressions using + largerthan, mimetype, and smallerthan; the first two always failed + to match, and the latter always matched. + * Relicense 5 source files that are not part of the webapp from AGPL to GPL. + * map: Run xdot if it's available in PATH. On OSX, the dot command + does not support graphical display, while xdot does. + * Debian: xdot is a better interactive viewer than dot, so Suggest + xdot, rather than graphviz. + * rmurl: Multiple pairs of files and urls can be provided on the + command line. + * rmurl: Added --batch mode. + * fromkey: Accept multiple pairs of files and keys. + Thanks, Daniel Brooks. + * rekey: Added --batch mode. + * add: Stage modified non-large files when running in indirect mode. + (This was already done in v6 mode and direct mode.) + * git-annex-shell, remotedaemon, git remote: Fix some memory DOS attacks. + * Fix build with http-client 0.5. + Thanks, Alper Nebi Yasak. + + -- Joey Hess Sat, 10 Dec 2016 11:56:25 -0400 + +git-annex (6.20161118) unstable; urgency=medium + + * git-annex.cabal: Loosen bounds on persistent to allow 2.5, which + on Debian has been patched to work with esqueleto. + This may break cabal's resolver on non-Debian systems; + if so, either use stack to build, or run cabal with + --constraint='persistent ==2.2.4.1' + Hopefully this mess with esqueleto will be resolved soon. + * sync: Pass --allow-unrelated-histories to git merge when used with git + git 2.9.0 or newer. This makes merging a remote into a freshly created + direct mode repository work the same as it works in indirect mode. + * Avoid backtraces on expected failures when built with ghc 8; + only use backtraces for unexpected errors. + * fsck --all --from was checking the existence and content of files + in the local repository, rather than on the special remote. Oops. + * Linux arm standalone: Build with a 32kb page size, which is needed + on several ARM NAS devices, including Drobo 5N, and WD NAS. + + -- Joey Hess Fri, 18 Nov 2016 11:43:14 -0400 + +git-annex (6.20161111) unstable; urgency=medium + + * Restarting a crashing git process could result in filename encoding + issues when not in a unicode locale, as the restarted processes's + handles were not read in raw mode. + * Make .git/annex/ssh.config file work with versions of ssh older than + 7.3, which don't support Include. When used with an older version + of ssh, any ServerAliveInterval in ~/.ssh/config will be overridden + by .git/annex/ssh.config. + * S3: Support the special case endpoint needed for the cn-north-1 region. + * Webapp: Don't list the Frankfurt S3 region, as this (and some other new + regions) need V4 authorization which the aws library does not yet use. + * reinject --known: Avoid second, unncessary checksum of file. + * OSX: Remove RPATHs from git-annex binary, which are not needed, + slow down startup, and break the OSX Sierra linker. + * webapp: Explicitly avoid checking for auth in static subsite + requests. Yesod didn't used to do auth checks for that, but this may + have changed. + * Linux standalone: Avoid using hard links in the tarball so it can be + untarred on eg, afs which does not support them. + + -- Joey Hess Fri, 11 Nov 2016 14:46:39 -0400 + +git-annex (6.20161031) unstable; urgency=medium + + * Assistant, repair: Fix ignoring of git fsck errors due to + duplicate file entries in tree objects. + * Linux standalone: Fix location of locale files in the bundle. + * Fix reversion in 6.20161012 that prevented adding files with a space + in their name. + + -- Joey Hess Mon, 31 Oct 2016 18:55:59 -0400 + +git-annex (6.20161027) unstable; urgency=medium + + * lock, smudge: Fix edge cases where data loss could occur in v6 mode + when the keys database was not populated. + * upgrade: Handle upgrade to v6 when the repository already contains + v6 unlocked files whose content is already present. + * Improve style of offline html build of website. + * importfeed: Drop URL parameters from file extension. + Thanks, James MacMahon. + * Assistant, repair: Improved filtering out of git fsck lines about + duplicate file entries in tree objects. + * test: Deal with gpg-agent behavior change that broke the test suite. + * Improve ssh socket cleanup code to skip over the cruft that + NFS sometimes puts in a directory when a file is being deleted. + * If a transfer fails for some reason, but some data managed to be sent, + the transfer will be retried. (The assistant already did this.) + * Run ssh with ServerAliveInterval 60, so that stalled transfers will + be noticed within about 3 minutes. + (Any setting in your ~/.ssh/config or /etc/ssh/ssh_config + overrides this.) + + -- Joey Hess Thu, 27 Oct 2016 15:21:58 -0400 + +git-annex (6.20161012) unstable; urgency=medium + + * Optimisations to time it takes git-annex to walk working tree and find + files to work on. Sped up by around 18%. + * Optimisations to git-annex branch query and setting, avoiding repeated + copies of the environment. Speeds up commands like + "git-annex find --in remote" by over 50%. + * Optimised git-annex branch log file timestamp parsing. + * Add "total-size" field to --json-progress output. + * Make --json-progress output be shown even when the size of a object + is not known. + * Multiple external special remote processes for the same remote will be + started as needed when using -J. This should not beak any existing + external special remotes, because running multiple git-annex commands + at the same time could already start multiple processes for the same + external special remotes. + * Linux standalone: Include locale files in the bundle, and generate + locale definition files for the locales in use when starting runshell. + (Currently only done for utf-8 locales.) + * Avoid using a lot of memory when large objects are present in the git + repository and have to be checked to see if they are a pointed to an + annexed file. Cases where such memory use could occur included, but + were not limited to: + - git commit -a of a large unlocked file (in v5 mode) + - git-annex adjust when a large file was checked into git directly + * When auto-upgrading a v3 remote, avoid upgrading to version 6, + instead keep it at version 5. + * Support using v3 repositories without upgrading them to v5. + * sync: Fix bug in adjusted branch merging that could cause recently + added files to be lost when updating the adjusted branch. + + -- Joey Hess Wed, 12 Oct 2016 09:37:41 -0400 + +git-annex (6.20160923) unstable; urgency=medium + + * Rate limit console progress display updates to 10 per second. + Was updating as frequently as changes were reported, up to hundreds of + times per second, which used unncessary bandwidth when running git-annex + over ssh etc. + * Make --json and --quiet work when used with -J. + Previously, -J override the other options. + * addurl, get: Added --json-progress option, which adds progress + objects to the json output. + * Remove key:null from git-annex add --json output. + * copy, move, mirror: Support --json and --json-progress. + * Improve gpg secret key list parser to deal with changes in gpg 2.1.15. + Fixes key name display in webapp. + * info: Support being passed a treeish, and show info about the annexed + files in it similar to how a directory is handled. + * sync: Previously, when run in a branch with a slash in its name, + such as "foo/bar", the sync branch was "synced/bar". That conflicted + with the sync branch used for branch "bar", so has been changed to + "synced/foo/bar". + * Note that if you're using an old version of git-annex to sync with + a branch with a slash in its name, it won't see some changes synced by + this version, and this version won't see some changes synced by the older + version. This is not a problem if there's a central bare repository, + but may impact other configurations until git-annex is upgraded to this + version. + * adjust: Previously, when adjusting a branch with a slash in its name, + such as "foo/bar", the adjusted branch was "adjusted/bar(unlocked)". + That conflicted with the adjusted branch used for branch "bar", + so has been changed to "adjusted/foo/bar(unlocked)" + * Also, running sync in an adjusted branch did not correctly sync + changes back to the parent branch when it had a slash in its name. + This bug has been fixed. + * addurl, importfeed: Improve behavior when file being added is gitignored. + + -- Joey Hess Fri, 23 Sep 2016 09:43:26 -0400 + +git-annex (6.20160907) unstable; urgency=medium + + * Windows: Handle shebang in external special remote program. + * Fix formatting of git-annex-smudge man page, and improve mdwn2man. + Thanks, Jim Paris. + * examimekey: Allow being run in a git repo that is not initialized by + git-annex yet. + * Android: Fix disabling use of cp --reflink=auto, curl, sha224, and sha384. + * Make --json and --quiet suppress automatic init messages, and any + other messages that might be output before a command starts. + Fixes a reversion introduced in version 5.20150727. + * Assistant, repair: Filter out git fsck lines about duplicate file + entries in tree objects. + * get -J, sync --content -J: Download different files from different + remotes when the remotes have the same costs. + + -- Joey Hess Wed, 07 Sep 2016 11:12:11 -0400 + +git-annex (6.20160808) unstable; urgency=medium + + * metadata --json output format has changed, adding a inner json object + named "fields" which contains only the fields and their values. + This should be easier to parse than the old format, which mixed up + metadata fields with other keys in the json object. + Any consumers of the old format will need to be updated. + * Added metadata --batch option, which allows getting, setting, deleting, + and modifying metadata for multiple files/keys. + * Added --branch option to copy, drop, fsck, get, metadata, mirror, move, + and whereis commands. This option makes git-annex operate on files that + are included in a specified branch (or other treeish). + * git-annex.cabal: Temporarily limit to http-conduit <2.2.0 + since aws 0.14.0 is not compatible with the newer version. + * git-annex.cabal: Temporarily limit to persistent <2.5 + since esqueleto 2.4.3 is not compatible with the newer version. + * Removed dependency on json library; all JSON is now handled by aeson. + * When built with uuid-1.3.12, generate more random UUIDs than before. + (However, this did not impact git-annex much, so a hard depedency has + not been added on uuid-1.3.12.) + * info: When run on a file now includes an indication of whether + the content is present locally. + * get, move, copy, mirror: Added --failed switch which retries + failed copies/moves. + * Re-enable accumulating transfer failure log files for command-line + actions (disabled in 5.20150522), and remove the log files after + successful transfers. + + -- Joey Hess Mon, 08 Aug 2016 11:42:17 -0400 + +git-annex (6.20160619) unstable; urgency=medium + + * get, drop: Add --batch and --json options. + * testremote: Fix crash when testing a freshly made external special remote. + * Remove unnecessary rpaths in the git-annex binary, but only when + it's built using make, not cabal. + This speeds up git-annex startup time by around 50%. + * Speed up startup time by caching the refs that have been merged into + the git-annex branch. + This can speed up git-annex commands by as much as a second, + depending on the number of remotes. + * fsck: Fix a reversion in direct mode fsck of a file that is + present when the location log thinks it is not. Reversion introduced + in version 5.20151208. + * uninit: Fix crash due to trying to write to deleted keys db. + Reversion introduced by v6 mode support, affects v5 too. + * Fix a similar crash when the webapp is used to delete a repository. + * Support checking presence of content at a http url that redirects to + a ftp url. + * log: Added --all option. + * New url for git-remote-gcrypt, now maintained by spwhitton. + * webapp: Don't allow deleting a remote that has syncing disabled, + as such a deletion will never finish. + Thanks, Farhan Kathawala. + * webapp: Escape unusual characters in ssh hostnames when generating + mangled hostnames. This allows IPv6 addresses to be used on filesystems + not supporting : in filenames. + * Avoid any access to keys database in v5 mode repositories, which + are not supposed to use that database. + * Remove the EKG build flag, since Gentoo for some reason decided to + enable this flag, depsite it not being intended for production use and + so disabled by default. + + -- Joey Hess Tue, 19 Jul 2016 14:17:54 -0400 + +git-annex (6.20160613) unstable; urgency=medium + + * Improve SHA*E extension extraction code. + * Windows: Avoid terminating git-annex branch lines with \r\n when + union merging and performing transitions. + * Remove Makefile from cabal tarball; man page building is now handled by + a small haskell program. + * sync --content: Fix bug that caused transfers of files to be made + to a git remote that does not have a UUID. This particularly impacted + clones from gcrypt repositories. + * Pass -S to git commit-tree when commit.gpgsign is set and when + making a non-automatic commit, in order to preserve current behavior + when used with git 2.9, which has stopped doing this itself. + * remotedaemon: Fixed support for notifications of changes to gcrypt + remotes, which was never tested and didn't quite work before. + * list: Do not include dead repositories. + * move --to: Better behavior when system is completely out of disk space; + drop content from disk before writing location log. + * Avoid a crash if getpwuid does not work, when querying the user's full + name. + * Automatically enable v6 mode when initializing in a clone from a repo + that has an adjusted branch checked out. + * v6: Fix initialization of a bare clone of a repo that has an adjusted + branch checked out. + * v6: Fix bad automatic merge conflict resolution between an annexed file + and a directory with the same name when in an adjusted branch. + * v6: Fix bad merge in an adjusted branch that resulted in an empty tree. + * v6: Fix bug in initialization of clone from a repo with an adjusted branch + that had not been synced back to master. + (This bug caused broken tree objects to get built by a later git annex + sync.) + * v6: Make lock and unlock work on files whose content is not present. + * v6: Fix update of associated files db when unlocking a file. + * v6: Make git clean filter preserve the backend that was used for a file. + + -- Joey Hess Mon, 13 Jun 2016 14:57:38 -0400 + +git-annex (6.20160527) unstable; urgency=medium + + * Split lines in the git-annex branch on \r as well as \n, to deal + with \r\n terminated lines written by some versions of git-annex on + Windows. This fixes strange displays in some cases. + * assistant: Fix bug that caused v6 pointer files to be annexed by the + assistant. + * assistant: Fix race in v6 mode that caused downloaded file content to + sometimes not replace pointer files. + * add: Adding a v6 pointer file used to annex it; now the pointer file is + added to git as-is. (git add of a pointer file already did the right + thing) + * enableremote: Can now be used to explicitly enable git-annex to use + git remotes. Using the command this way prevents other git-annex + commands from probing new git remotes to auto-enable them. + * enableremote: Remove annex-ignore configuration from a remote. + * Change git annex info remote encryption description to use wording + closer to what's used in initremote. + * Pass the various gnupg-options configs to gpg in several cases where + they were not before. Most notably, gnupg-decrypt-options is now + passed when decrypting an encrypted cipher. + * adjust: Add --fix adjustment, which is useful when the git directory + is in a nonstandard place. + * adjust: If the adjusted branch already exists, avoid overwriting it, + since it might contain changes that have not yet been propigated to the + original branch. + * Work around git weirdness in handling of relative path to GIT_INDEX_FILE + when in a subdirectory of the repository. This affected git annex view. + * Fix crash when entering/changing view in a subdirectory of a repo that + has a dotfile in its root. + * webapp: Avoid confusing display of dead remotes. + * Support building with ghc 8.0.1. + * Updated cabal file explicitly lists source files. The tarball + on hackage will include only the files needed for cabal install; + it is NOT the full git-annex source tree. + * debian/changelog, debian/NEWS, debian/copyright: Converted to symlinks + to CHANGELOG, NEWS, and COPYRIGHT, which used to symlink to these instead. + + -- Joey Hess Fri, 27 May 2016 11:48:36 -0400 + +git-annex (6.20160511) unstable; urgency=medium + + * Fix bug that sometimes prevented git-annex smudge --clean from consuming + all its input, which resulted in git add bypassing git-annex. + * Fix build with directory-1.2.6.2. + * Improve behavior when a just added http remote is not available + during uuid probe. Do not mark it as annex-ignore, so it will be tried + again later. + * Android: Icon refresh. + Thanks, freewheelinfranks. + * Added DIRHASH-LOWER to external special remote protocol. + * git-annex.cabal: Add Setup-Depends. + * stack.yaml: Enable explicit-setup-deps. + * Windows: Fix several bugs in propigation of changes from the adjusted + branch back to the master branch. + * Windows: Fix an over-long temp directory name. + * map: Hide dead repositories that are not connected to the graph. + * map: Changed colors; red is used for untrusted repositories and grey + for dead. + * version: Display OS version and architecture too. + * Propigate GIT_DIR and GIT_WORK_TREE environment to external special + remotes. + * Added annex.gnupg-decrypt-options and + remote..annex-gnupg-decrypt-options, which are passed to gpg + when it's decrypting data. + * fsck: When a key is not previously known in the location log, + record something so that reinject --known will work. + * In the unusual configuration where annex.crippledfilesystem=true but + core.symlinks=true, store object contents in mixed case hash + directories so that symlinks will point to them. + * Added new encryption=sharedpubkey mode for special remotes. + This is useful for makking a special remote that anyone with a clone + of the repo and your public keys can upload files to, but only you can + decrypt the files stored in it. + + -- Joey Hess Wed, 11 May 2016 12:41:42 -0400 + +git-annex (6.20160419) unstable; urgency=medium + + * Fix bug that prevented resuming of uploads to encrypted special remotes + that used chunking. + * That bug could also expose the names of keys to such remotes, so it is a + minor security issue. + * Fix duplicate progress meter display when downloading from a git remote + over http with -J. + * reinject: When src file's content cannot be verified, leave it alone, + instead of deleting it. + * reinject: Added new mode which can reinject known files into the annex. + For example: git-annex reinject --known /mnt/backup/* + * calckey: New plumbing command, calculates the key that would be used + to refer to a file. + * Fix bug that prevented annex.sshcaching=false configuration from taking + effect when on a crippled filesystem. Thanks, divergentdave. + * git 2.9.0 is going to prevent git merge from merging in unrelated + branches. Since the webapp's pairing etc features often combine + together repositories with unrelated histories, work around + this behavior change when the assistant merges, by passing + --allow-unrelated-histories. Note though that this is not done + for git annex sync's merges, so it will follow git's default or + configured behavior. + * When git-annex is used with a git version older than 2.2.0, disable + support for adjusted branches, since GIT_COMMON_DIR is needed to update + them and was first added in that version of git. + * Avoid setting LOCPATH in linux standalone builds that are built with + a ghc that has been fixed to not hang when it cannot find locale files. + * Isolate test suite from global git config settings. + + -- Joey Hess Thu, 28 Apr 2016 09:31:14 -0400 + +git-annex (6.20160418) unstable; urgency=medium + + * smudge: Print a warning when annex.thin is set, as git's smudge + interface does not allow honoring that configuration. + * webapp: When $HOME is a git repository, and has been initialized for + use by git-annex, opening the webapp went ahead and ran the assistant + there, annexing all files. Since this is almost certianly not + desirable, especially when the user is just opening the webapp from + a dekstop menu which happens to run it in $HOME, the webapp will now not + treat such a $HOME git repository as a git-annex repository. + * webapp: Update url to add gitlab.com ssh key. + * Fix bug in v6 mode that prevented treating unlocked executable files + as annexed. If you have such files, run git annex init --version=6 + to update the cache after upgrading to this version of git-annex. + * Preserve execute bits of unlocked files in v6 mode. + * fsck: Warn when core.sharedRepository is set and an annex object file's + write bit is not set and cannot be set due to the file being owned + by a different user. + * Fix hang when dropping content needs to lock the content on a + ssh remote, which occurred when the remote has git-annex version + 5.20151019 or newer. (The bug was in the client side; the remote + git-annex-shell does not need to be upgraded.) + + -- Joey Hess Mon, 18 Apr 2016 18:33:52 -0400 + +git-annex (6.20160412) unstable; urgency=medium + + * adjust --unlock: Enters an adjusted branch in which all annexed files + are unlocked. The v6 equivilant of direct mode, but much cleaner! + * Upgrading a direct mode repository to v6 has changed to enter + an adjusted unlocked branch. This makes the direct mode to v6 upgrade + able to be performed in one clone of a repository without affecting + other clones, which can continue using v5 and direct mode. + * init --version=6: Automatically enter the adjusted unlocked branch + when filesystem doesn't support symlinks. + * ddar remote: fix ssh calls + Thanks, Robie Basak + * log: Display time with time zone. + * log --raw-date: Use to display seconds from unix epoch. + * v6: Close pointer file handles more quickly, to avoid problems on Windows. + * sync: Show output of git commit. + * annex.thin and annex.hardlink are now supported on Windows. + * unannex --fast now makes hard links on Windows. + * Fix bug in annex.largefiles mimetype= matching when git-annex + is run in a subdirectory of the repository. + * Fix build with ghc v7.11. Thanks, Gabor Greif. + + -- Joey Hess Tue, 12 Apr 2016 14:53:22 -0400 + +git-annex (6.20160318) unstable; urgency=medium + + * metadata: Added -r to remove all current values of a field. + * Fix data loss that can occur when annex.pidlock is set in a repository. + * Fix bug preventing moving files to/from a repository with annex.pidlock set. + * Fix shared lock file FD leak. + * Fix metadata hook behavior when multiple files are added at once. + Thanks, Klaus Ethgen. + * Added dependencies on haskell mountpoints and disk-free-space + libraries, removing FFI code from git-annex. + * dropkey: Add --batch and --json. + * Fix OSX dmg to include libraries needed by bundled gpg, + lost in last release. + * Always try to thaw content, even when annex.crippledfilesystem is set. + * Correct git-annex info to include unlocked files in v6 repository. + * Sped up git-annex add in direct mode and v6 by using + git hash-object --stdin-paths. + * Sped up git-annex merge by using git hash-object --stdin-paths. + + -- Joey Hess Fri, 18 Mar 2016 11:30:36 -0400 + +git-annex (6.20160229) unstable; urgency=medium + + * Update perlmagick build dependency. Closes: #789225 + * Fix memory leak in last release, which affected commands like + git-annex status when a large non-annexed file is present in the work + tree. + * fsck: When the only copy of a file is in a dead repository, mention + the repository. + * info: Mention when run in a dead repository. + * Linux and OSX standalone builds put the bundled gpg last in PATH, + so any system gpg will be preferred over it. + * Avoid crashing when built with MagicMime support, but when the magic + database cannot be loaded. + * Include magic database in the linux and OSX standalone builds. + * Fix memory leak when hashing files, which triggered during fsck + when an external hash program was not used. + (This leak was introduced in version 6.20160114.) + * Support --metadata fieldnumber etc + to match ranges of numeric values. + * Similarly, support preferred content expressions like + metadata=fieldnumber + * The pre-commit-annex hook script that automatically extracts + metadata has been updated to also use exiftool. + Thanks, Klaus Ethgen. + + -- Joey Hess Mon, 29 Feb 2016 12:41:49 -0400 + +git-annex (6.20160217) unstable; urgency=medium + + * Support getting files from read-only repositories. + * checkpresentkey: Allow to be run without an explicit remote. + * checkpresentkey: Added --batch. + * Work around problem with concurrent-output when in a non-unicode locale + by avoiding use of it in such a locale. Instead -J will behave as if + it was built without concurrent-output support in this situation. + * Fix storing of filenames of v6 unlocked files when the filename is not + representable in the current locale. + * fsck: Detect and fix missing associated file mappings in v6 repositories. + * fsck: Populate unlocked files in v6 repositories whose content is + present in annex/objects but didn't reach the work tree. + * When initializing a v6 repo on a crippled filesystem, don't force it + into direct mode. + * Windows: Fix v6 unlocked files to actually work. + * add, addurl, import, importfeed: When in a v6 repository on a crippled + filesystem, add files unlocked. + * annex.addunlocked: New configuration setting, makes files always be + added unlocked. (v6 only) + * Improve format of v6 unlocked pointer files to support keys containing + slashes. + + -- Joey Hess Wed, 17 Feb 2016 14:48:51 -0400 + +git-annex (6.20160211) unstable; urgency=medium + + * annex.addsmallfiles: New option controlling what is done when + adding files not matching annex.largefiles. + * Fix reversion in lookupkey, contentlocation, and examinekey which + caused them to sometimes output side messages. + * webapp: Fix deletion of current repository directory. + * Added "nothing" to preferred content expression syntax. + * annex.largefiles can be configured in .gitattributes too; + this is particulary useful for v6 repositories, since the + .gitattributes configuration will apply in all clones of the + repository. + * Limit annex.largefiles parsing to the subset of preferred content + expressions that make sense in its context. So, not "standard" + or "lackingcopies", etc. + * annex.largefiles: Add support for mimetype=text/* etc, when git-annex + is linked with libmagic. + * matchexpression: Added --largefiles option to parse an annex.largefiles + expression. + * Brought back the dbus and xmpp build flags, so build from source can be + done without C libraries that may be hard to install. + * init: Fix bugs in submodule .git symlink fixup, that occurred when + initializing in a subdirectory of a submodule and a submodule of a + submodule. + * WebDAV: Set depth 1 in PROPFIND request, for better compatibility with + some servers. Thanks, wzhd. + * WebDAV: Remove a bogus trailing slash from the end of the url to the + temporary store location for a key. Thanks, wzhd. + * S3: Allow configuring with requeststyle=path to use path-style bucket + access instead of the default DNS-style access. + + -- Joey Hess Thu, 11 Feb 2016 11:42:19 -0400 + +git-annex (6.20160126) unstable; urgency=medium + + * Fix nasty reversion in the last release that broke sync --content's + handling of many preferred content expressions. + * whereis --json: Urls are now listed inside the remote that claims them, + rather than all together at the end. + * info, add, whereis, find: Support --batch mode. + * Force output to be line-buffered, even when it's not connected to the + terminal. This is particuarly important for commands with --batch + output, which was not always being flushed at an appropriate time. + * add, import: Support --json output. + * addurl --json: Include field for added key (unless the file was + added directly to git due to annex.largefiles configuration.) + (Also done by add --json and import --json) + * registerurl: Check if a remote claims the url, same as addurl does. + * Bug fix: Git config settings passed to git-annex -c did not always take + effect. + * assistant: Use udisks2 dbus events to detect when disks are mounted, + instead of relying on gnome/kde stuff that is not stable. + * Fix build with QuickCheck 2.8.2 + * matchexpression: New plumbing command to check if a preferred content + expression matches some data. + * Removed the webapp-secure build flag, rolling it into the webapp build + flag. + * Removed the quvi, tahoe, feed, and tfds build flags, adding + aeson feed and regex-tdfa to the core dependencies. + * Roll the dns build flag into the assistant build flag. + * Debian: Avoid building debug package, since gdb is not often useful + to debug haskell programs. + + -- Joey Hess Tue, 26 Jan 2016 14:57:42 -0400 + +git-annex (6.20160114) unstable; urgency=medium + + "hexapodia as the key insight" + + * Added v6 repository mode, but v5 is still the default for now. + * unlock, lock: In v6 mode, unlocking a file changes it from a symlink to a + pointer file, and this change can be committed to the git repository. + For details, see http://git-annex.branchable.com/tips/unlocked_files/ + * The upgrade to version 6 is not done fully automatically yet, because + upgrading a direct mode repository to version 6 will prevent old + versions of git-annex from working in other clones of that repository. + For details, see http://git-annex.branchable.com/upgrades/ + * init: --version parameter added to control which supported repository + version to use. + * init, upgrade: Configure .git/info/attributes to use git-annex + as a smudge filter. In v6 repository mode, this makes git add + add files to the annex in unlocked mode, unless overridden by + annex.largefiles configuration. + * assistant: In v6 mode, adds files in unlocked mode, so they can + continue to be modified. + * Added annex.thin setting, which makes unlocked files in v6 repositories + be hard linked to their content, instead of a copy. This saves disk + space but means any modification of an unlocked file will lose the local + (and possibly only) copy of the old version. + * Enable annex.thin by default on upgrade from direct mode to v6, since + direct mode made the same tradeoff. + * fix: Adjusts unlocked files as configured by annex.thin. + * persistent-sqlite is now a hard build dependency, since v6 repository + mode needs it. + + + * status: On crippled filesystems, was displaying M for all annexed files + that were present. Probably caused by a change to what git status + displays in this situation. Fixed by treating files git thinks are + modified the same as typechanged files. + * addurl: Added --batch and --with-files options. + * addurl: Support --json, particularly useful in --batch mode. + * addurl: Refuse to overwrite any existing, non-annexed file. + * Debian: Adjust build dependencies for webapp, DAV. Now available on + mips, mipsel, but temporarily removed armel since build is failing + there. + * info: Fix "backend usage" numbers, which were counting present keys + twice. + * info --json: Improve json for "backend usage", using a nested object + with fields for each backend instead of the previous weird nested lists. + This may break existing parsers of this json output, if there were any. + * whereis --json: Make url list be included in machine-parseable form. + * test: Added --keep-failures option. + * unused: Bug fix when a new file was added to the annex, and then + removed (but not git rmed). git still has the add staged in this case, + so the content should not be unused and was wrongly treated as such. + * migrate: Copy over metadata to new key. + * rekey: No longer copies over urls from the old to the new key. + It makes sense for migrate to do that, but not for this low-level + (and little used) plumbing command to. + * view: Fix crash in non-unicode capable locale when entering a view + of metadata containing a slash or backslash. + * When annex.http-headers is used to set the User-Agent header, avoid + sending User-Agent: git-annex + * Windows: Fix rsync cross-drive hack to work with msys2 rsync. + Thanks, Pieter Kitslaar. + + -- Joey Hess Thu, 14 Jan 2016 10:14:19 -0400 + +git-annex (5.20151218) unstable; urgency=medium + + * Add S3 features to git-annex version output. + * webdav: When testing the WebDAV server, send a file with content. + The empty file it was sending tickled bugs in some php WebDAV server. + * fsck: Failed to honor annex.diskreserve when checking a remote. + * Debian: Build depend on concurrent-output. + * Fix insecure temporary permissions when git-annex repair is used in + in a corrupted git repository. + * Fix potential denial of service attack when creating temp dirs. + + -- Joey Hess Fri, 18 Dec 2015 12:09:33 -0400 + +git-annex (5.20151208) unstable; urgency=medium + + * Build with -j1 again to get reproducible build. + * Display progress meter in -J mode when copying from a local git repo, + to a local git repo, and from a remote git repo. + * Display progress meter in -J mode when downloading from the web. + * map: Improve display of git remotes with non-ssh urls, including http + and gcrypt. + * When core.sharedRepository is set, annex object files are not made mode + 444, since that prevents a user other than the file owner from locking + them. Instead, a mode such as 664 is used in this case. + * tahoe: Include tahoe capabilities in whereis display. + * import: Changed to honor annex.largefiles settings. + * addurl, importfeed: Changed to honor annex.largefiles settings, + when the content of the url is downloaded. (Not when using --fast or + --relaxed.) + * webapp: Fix bugs that could result in a relative path such as "." + being written to ~/.config/git-annex/autostart, and ignore any such + relative paths in the file. + This was a reversion caused by the relative path changes in 5.20150113. + * dropunused: Make more robust when trying to drop an object that has + already been dropped. + * Fix reversion in handling of long filenames, particularly when using + addurl/importfeed, which was introduced in the previous release. + + -- Joey Hess Tue, 08 Dec 2015 11:14:03 -0400 + +git-annex (5.20151116) unstable; urgency=medium + + * Use concurrent-output library when configured with -fConcurrentOutput. + This allows nicely displayed messages when using the -J flag. + * Additional commands now support the -J flag: + fsck, drop, add, addurl, import + * import: Avoid very ugly error messages when the directory files + are imported to is not a directort, but perhaps an annexed file. + * Concurrent progress bars are now displayed when using -J with a command + that moves file contents around. + * Fix race that could result in an annexed file's symlink not being + created, when eg, running concurrent git-annex adds. + * add: Fix error recovery rollback to not move the injested file content + out of the annex back to the file, because other files may point to + that same content. Instead, copy the injected file content out to + recover. + * quvi may output utf-8 encoded data when the conifigured locale doesn't + support that; avoid crashing on such invalid encoding. + * runshell: Avoid failing when $HOME/.ssh does not exist and cannot be + created. + * Make the git-annex-standalone.deb prevent runshell from installing + wrappers into $HOME/.ssh + * Make git-annex-standalone.deb include the git-annex html documentation, + desktop file, and base completion file, same as the regular git-annex.deb. + * fsck: When fscking a dead repo, avoid incorrect "fixing location log" + message, and display a warning about it being dead, since it's unusual + to have access to a dead repo. + * assistant: Pass ssh-options through 3 more git pull/push calls + that were missed before. + * Added annex.pidlock and annex.pidlocktimeout configuration to support + filesystems where POSIX fcntl locks cannot be used. + * init: Automatically enable annex.pidlock when necessary. + + -- Joey Hess Mon, 16 Nov 2015 14:17:40 -0400 + +git-annex (5.20151102.1) unstable; urgency=medium + + * Avoid installing desktop file and program file if cabal install + git-annex is run as root, since that is not a systemwide install, + but to /root, and so generating a systemwide desktop file is not right. + * When cabal install is run with the desktop file location not writable, + display a warning, but continue successfully. + + -- Joey Hess Tue, 03 Nov 2015 12:08:38 -0400 + +git-annex (5.20151102) unstable; urgency=medium + + * Use statvfs on OSX. + * Symlink timestamp preservation code uses functions + from unix-2.7.0 when available, which should be more portable. + * enableremote: List uuids and descriptions of remotes that can be + enabled, and accept either the uuid or the description in leu if the + name. + * Catch up with current git behavior when both repo and repo.git exist; + it seems it now prefers repo in this case, although historically it may + have preferred repo.git. + * Fix failure to build with aws-0.13.0. + * When built with aws-0.13.0, the S3 special remote can be used to create + google nearline buckets, by setting storageclass=NEARLINE. + + -- Joey Hess Mon, 02 Nov 2015 12:41:20 -0400 + +git-annex (5.20151019) unstable; urgency=medium + + * Fix a longstanding, but unlikely to occur bug, where dropping + a file from a remote could race with other drops of the same file, + and result in all copies of its content being lost. + * git-annex-shell: Added lockcontent command, to prevent dropping of + a key's content. This is necessary due to the above bugfix. + * In some cases, the above bugfix changes what git-annex allows you to + drop: + - When a file is present in several special remotes, + but not in any accessible git repositories, dropping it from one of + the special remotes will now fail. Instead, the file has to be + moved from one of the special remotes to the git repository, and can + then safely be dropped from the git repository. + - If a git remote has too old a version of git-annex-shell installed, + git-annex won't trust it to hold onto a copy of a file when dropping + that file from the local git repository. + * Changed drop ordering when using git annex sync --content or the + assistant, to drop from remotes first and from the local repo last. + This works better with the behavior changes to drop in many cases. + * Do verification of checksums of annex objects downloaded from remotes. + * When annex objects are received into git repositories from other git + repos, their checksums are verified then too. + * To get the old, faster, behavior of not verifying checksums, set + annex.verify=false, or remote..annex-verify=false. + * setkey, rekey: These commands also now verify that the provided file + matches the expected checksum of the key, unless annex.verify=false. + * reinject: Already verified content; this can now be disabled by + setting annex.verify=false. + * sync, merge, assistant: When git merge failed for a reason other + than a conflicted merge, such as a crippled filesystem not allowing + particular characters in filenames, git-annex would make a merge commit + that could omit such files or otherwise be bad. Fixed by aborting the + whole merge process when git merge fails for any reason other than a + merge conflict. + * Allow building with S3 disabled again. + * Ported disk free space checking code to work on Solaris. + * Windows webapp: Fix support for entering password when setting + up a ssh remote. + * copy --auto was checking the wrong repo's preferred content. + (--from was checking what --to should, and vice-versa.) + Fixed this bug, which was introduced in version 5.20150727. + * Avoid unncessary write to the location log when a file is unlocked + and then added back with unchanged content. + * S3: Fix support for using https. + * Avoid displaying network transport warning when a ssh remote + does not yet have an annex.uuid set. + * Debian: Add torrent library to build-depends as it's packaged now, + and stop recommending bittornado | bittorrent. + * Debian: Remove build dependency on transformers library, as it is now + included in ghc. + * Debian: Remove menu file, since a desktop file is provided and + lintian says there can be only one. + + -- Joey Hess Mon, 19 Oct 2015 13:59:01 -0400 + +git-annex (5.20150930) unstable; urgency=medium + + * Added new linux standalone "ancient" build to support kernels + like 2.6.32. + * info: Don't allow use in a non-git-annex repository, since it + uses the git-annex branch and would create it if it were missing. + * assistant: When updating ~/.ssh/config, preserve any symlinks. + * webapp: Remove the "disable remote" feature from the UI. + * S3: When built with aws-0.13.0, supports using more storage classes. + In particular, storageclass=STANDARD_IA to use Amazon's + new Infrequently Accessed storage, and storageclass=NEARLINE + to use Google's NearLine storage. + * Improve ~/.ssh/config modification code to not add trailing spaces + to lines it cannot parse. + * Fix a crash at direct mode merge time when .git/index doesn't exist + yet. Triggered by eg, git-annex sync --no-commit in a fresh clone of + a repository. + * status: Show added but not yet committed files. + * Added stack.yaml to support easy builds from source with stack. + + -- Joey Hess Wed, 30 Sep 2015 14:31:52 -0400 + +git-annex (5.20150916) unstable; urgency=medium + + * Fix Windows build to work with ghc 7.10. + * init: Fix reversion in detection of repo made with git clone --shared + * info: Support querying info of individual files in direct mode. + * unused: Fix reversion in 5.20150727 that broke parsing of the + --unused-refspec option. Thanks, Øyvind A. Holm. + * Make full option parsing be done when not in a git repo, so --help + can be displayed for commands that require a git repo, etc. + * fsck: Work around bug in persistent that broke display of + problematically encoded filenames on stderr when using --incremental. + * When gpg.program is configured, it's used to get the command to run + for gpg. Useful on systems that have only a gpg2 command or want to + use it instead of the gpg command. + * Windows: Switched to using git for Windows, rather than msysgit. + Using msysgit with git-annex is no longer supported. + * Windows: Even when the user neglects to tell the git installer to + add git to PATH, git-annex will still work from within the git bash + shell, and the webapp can be used too. + * sync: Add --no-commit, --no-pull, --no-push options to turn off parts of + the sync process, as well as supporting --commit, --pull, --push, and + --no-content options to specify the (current) default behavior. + * annex.hardlink extended to also try to use hard links when copying from + the repository to a remote. + * Improve bash completion, so it completes names of remotes and backends + in appropriate places. + * Special remotes configured with autoenable=true will be automatically + enabled when git-annex init is run. + * Fix bug in combination of preferred and required content settings. + When one was set to the empty string and the other set to some expression, + this bug caused all files to be wanted, instead of only files matching + the expression. + + -- Joey Hess Wed, 16 Sep 2015 10:31:24 -0400 + +git-annex (5.20150824) unstable; urgency=medium + + * Sped up downloads of files from ssh remotes, reducing the + non-data-transfer overhead 6x. + * sync: Support --jobs + * sync --content: Avoid unnecessary second pull from remotes when + no file transfers are made. + * External special remotes can now be built that can be used in readonly + mode, where git-annex downloads content from the remote using regular + http. + * Added WHEREIS to external special remote protocol. + * importfeed --relaxed: Avoid hitting the urls of items in the feed. + * Fix reversion in init when ran as root, introduced in version 5.20150731. + * Reorder declaration to fix build with yesod-core > 1.4.13. + Thanks, Michael Alan Dorman. + * Fix building without quvi and without database. + Thanks, Ben Boeckel. + * Avoid building the assistant on the hurd, since an inotify equivalent + is not yet implemented in git-annex for the hurd. + * --debug log messages are now timestamped with fractional seconds. + * --debug is passed along to git-annex-shell when git-annex is in debug mode. + * Makefile: Pass LDFLAGS, CFLAGS, and CPPFLAGS through ghc and on to + ld, cc, and cpp. + * As a result of the Makefile changes, the Debian package is built + with various hardening options. Although their benefit to a largely + haskell program is unknown. + + -- Joey Hess Mon, 24 Aug 2015 14:11:05 -0700 + +git-annex (5.20150812) unstable; urgency=medium + + * Added support for SHA3 hashed keys (in 8 varieties), when git-annex is + built using the cryptonite library. + * metadata: Fix reversion introduced in 5.20150727 that caused recursive + display of metadata to not work. + * Windows: Fix bug that caused git-annex sync to fail due to missing + environment variable. + * Fix setting/setting/viewing metadata that contains unicode or other + special characters, when in a non-unicode locale. + * Simplify setup process for a ssh remote. Now it suffices to run git + remote add, followed by git-annex sync. Now the remote is automatically + initialized for use by git-annex, where before the git-annex branch had + to manually be pushed before using git-annex sync. Note that this + involved changes to git-annex-shell, so if the remote is using an old + version, the manual push is still needed. + * git-annex-shell: Don't let configlist auto-init repository when in + readonly mode. + * Perform a clean shutdown when --time-limit is reached. + This includes running queued git commands, and cleanup actions normally + run when a command is finished. + * fsck: Commit incremental fsck database when --time-limit is reached. + Previously, some of the last files fscked did not make it into the + database when using --time-limit. + * fsck: Commit incremental fsck database after every 1000 files + fscked, or every 5 minutes, whichever comes first. Previously, + commits were made every 1000 files fscked. + * Linux standalone: Work around problem that prevented it from working + properly if unpacked into a directory that contains ":" or ";" in its + name. + * proxy: Fix proxy git commit of non-annexed files in direct mode. + * proxy: If a non-proxied git command, such as git revert + would normally fail because of unstaged files in the work tree, + make the proxied command fail the same way. + * proxy: Fix removal of files deleted by the proxied command. + * proxy: Fix behavior when run in subdirectory of git repo. + * Improve Setup.hs file so that cabal copy --destdir works. + Thanks, Magnus Therning. + * Tighten dependency on optparse-applicative to 0.11.0. + * Added back debian/cabal-wrapper, since it still seems needed after all. + + -- Joey Hess Wed, 12 Aug 2015 11:14:58 -0400 + +git-annex (5.20150731) unstable; urgency=medium + + * webapp: Support enabling known gitlab.com remotes. + * Fix rsync special remote to work when -Jn is used for concurrent + uploads. + * The last release accidentally removed a number of options from the + copy command. (-J, file matching options, etc). These have been added + back. + * init: Detect when the filesystem is crippled such that it ignores + attempts to remove the write bit from a file, and enable direct mode. + Seen with eg, NTFS fuse on linux. + * Fix man page installation by cabal install; all the new man pages are + now installed. + + -- Joey Hess Fri, 31 Jul 2015 11:34:36 -0400 + +git-annex (5.20150727) unstable; urgency=medium + + * Fix bug that prevented uploads to remotes using new-style chunking + from resuming after the last successfully uploaded chunk. + * Switched option parsing to use optparse-applicative. This was a very large + and invasive change, and may have caused some minor behavior changes to + edge cases of option parsing. (For example, the metadata command no + longer accepts the combination of --get and --set, which never actually + worked.) + * Bash completion file is now included in the git-annex source tree, + and installed into Debian package (and any other packages built using make + install). This bash completion is generated by the option parser, so it + covers all commands, all options, and will never go out of date! + * As well as tab completing "git-annex" commands, "git annex" will also tab + complete. However, git's bash completion script needs a patch, + which I've submitted, for this to work prefectly. + * version --raw now works when run outside a git repository. + * assistant --startdelay now works when run outside a git repository. + * dead now accepts multiple --key options. + * addurl now accepts --prefix and --suffix options to adjust the + filenames used. + * sync --content: Fix bug that caused files to be uploaded to eg, + more archive remotes than wanted copies, only to later be dropped + to satisfy the preferred content settings. + * importfeed: Improve detection of known items whose url has changed, + and avoid adding redundant files. Where before this only looked at + permalinks in rss feeds, it now also looks at guids. + * importfeed: Look at not only permalinks, but now also guids + to identify previously downloaded files. + * Webapp: Now features easy setup of git-annex repositories on gitlab.com. + * Adjust debian build deps: The webapp can now build on arm64, s390x + and hurd-i386. WebDAV support is also available on those architectures. + * Debian package now maintained by Richard Hartmann. + * Support building without persistent database on for systems that + lack TH. This removes support for incremental fsck. + + -- Joey Hess Mon, 27 Jul 2015 12:24:49 -0400 + +git-annex (5.20150710) unstable; urgency=medium + + * add: Stage symlinks the same as git add would, even if they are not a + link to annexed content. + * sync: When annex.autocommit=false, avoid making any commit of local + changes, while still merging with remote to the extent possible. + * unused: --used-refspec can now be configured to look at refs in the + reflog. This provides a way to not consider old versions of files to be + unused after they have reached a specified age, when the old refs in + the reflog expire. + * log: Fix reversion introduced in version 5.20150528 that broke this command. + * assistant --autostart: First stop any daemons that are already running, + which might be left over from a previous login session and so unable to + use the ssh agent of a new login session. + * assistant: Fix local pairing to not include newline in ssh pubkey, + which is rejected on the other end for security reasons. + * assistant: Fix ANNEX_SHELL_DIR written to ~/.ssh/authorized_keys + in local pairing to be the absolute path to the repository, not "." + This was a reversion caused by the relative path changes in 5.20150113. + * Brought back the setkey plumbing command that was removed in 2011, since + we found a use case for it. Note that the command's syntax was changed + for consistency. + * bugfix: Pass --full-tree when using git ls-files to get a list of files + on the git-annex branch, so it works when run in a subdirectory. + This bug affected git-annex unused, and potentially also transitions + running code and other things. + * Support git's undocumented core.sharedRepository=2 value, which + is equivalent to "world", and is set when a repo was created using + git init --shared=world. + * When building on linux, pass --as-needed to linker to avoid linking + with unused shared libraries including libyaml. + * import: Fix failure of cross-device import on Windows. + * merge: Avoid creating the synced/master branch. + * Removed support for optparse-applicative versions older than 0.10. + + -- Joey Hess Fri, 10 Jul 2015 16:36:42 -0400 + +git-annex (5.20150617) unstable; urgency=medium + + * Now supports git annex sync --all --content to sync all versions of all + files with all repos that want them. + * Added new "anything" preferred content expression, which matches all + versions of all files. + * Standard preferred content for client, backup, incremental backup, + and unwanted groups have been adjusted to work better when used + with git annex sync --all --content. + * fromkey, registerurl: Improve handling of urls that happen to also + be parsable as strange keys. + * sync, remotedaemon: Pass configured ssh-options even when + annex.sshcaching is disabled. + * assistant: Consume systemd-networkd dbus events to learn about + changes to network connections, as was already done with + network-manager and wicd. + Thanks to Sebastian Reuße for the patches. + * get --incomplete: New option to resume any interrupted downloads. + * dead --key: Can be used to mark a key as dead. + * fsck: Ignore keys that are known to be dead when running in + --all/--unused/--key mode or a in a bare repo. Closes: #753888 + Otherwise, still reports files with lost contents, even if the content + is dead. + * S3: Special remotes can be configured with public=yes to allow + the public to access the bucket's content. + * S3: Publically accessible buckets can be used without creds. + * import --clean-duplicates: Fix bug that didn't count local or trusted + repo's copy of a file as one of the necessary copies to allow removing + it from the import location. + * tahoe: Use ~/.tahoe-git-annex/ rather than ~/.tahoe/git-annex/ + when setting up a tahoe special remote to avoid old versions of + tahoe create-client choking. + * Fix bug that prevented enumerating locally present objects in repos + tuned with annex.tune.objecthash1=true. + Fixes: unused, object count in info, unannex. + * Improve url parsing to handle some urls containing illegal [] + characters in their paths. + * info: Added json output for "backend usage", "numcopies stats", + "repositories containing these files", and "transfers in progress". + * Fix incremental backup standard preferred content expression to match + its documentation, which says it does not want files that have reached + a backup repository. + * Increased the default annex.bloomaccuracy from 1000 to 10000000. + This makes git annex unused use up to 16 mb more memory than it did + before, but the massive increase in accuracy makes this worthwhile + for all but the smallest systems. + * Build documentation with deterministic=1 for reproducible builds. + (A new ikiwiki feature.) Closes: #785736 + * Re-remove dependency on obsolete hamlet package. Closes: #786659 + * debian/cabal-wrapper: Removed this hack which should not be needed anymore. + + -- Joey Hess Wed, 17 Jun 2015 13:50:35 -0400 + +git-annex (5.20150528) unstable; urgency=medium + + * fromkey, registerurl: Allow urls to be specified instead of keys, + and generate URL keys. + * Linux standalone, OSX app: Improve runshell script to always quote + shell vars, so that it will work when eg, untarred into a directory + path with spaces in its name. + * Revert removal dependency on obsolete hamlet package, since the + autobuilders are not ready for this change yet and it prevented them + from building the webapp. Reopens: #786659 + * fsck: When checksumming a file fails due to a hardware fault, + the file is now moved to the bad directory, and the fsck proceeds. + Before, the fsck immediately failed. + * Linux standalone: The webapp was not built in the previous release, + this release fixes that oversight. + + -- Joey Hess Thu, 28 May 2015 10:48:03 -0400 + +git-annex (5.20150522) unstable; urgency=medium + + * import: Refuse to import files that are within the work tree, as that + does not make sense and could cause data loss. + * drop: Now supports --all, --unused, and --key. + * drop: Now defaults to --all when run in a bare repository. + (Previously, did nothing when run in a bare repository.) + * get, move, copy, mirror: Concurrent transfers are now supported! + For example: git-annex get -J10 + However, progress bars are not yet displayed for concurrent transfers, + pending an updated version of the ascii-progress library. + * --quiet now makes progress output by rsync, wget, etc be quiet too. + * Take space that will be used by other running downloads into account when + checking annex.diskreserve. + * Avoid accumulating transfer failure log files unless the assistant is + being used. + * Fix an unlikely race that could result in two transfers of the same key + running at once. + * Stale transfer lock and info files will be cleaned up automatically + when get/unused/info commands are run. + * unused: Add --used-refspec option and annex.used-refspec, which can + specify a set of refs to consider used, rather than the default of + considering all refs used. + * webapp: Fix zombie xdg-open process left when opening file browser. + Closes: #785498 + * Safer posix fctnl locking implementation, using lock pools and STM. + * Build documentation with TZ=UTC for reproducible builds. See #785736. + * OSX: Corrected the location of trustedkeys.gpg, so the built-in + upgrade code will find it. Fixes OSX upgrade going forward, but + older versions won't upgrade themselves due to this problem. + * Remove dependency on obsolete hamlet package. Closes: #786659 + + -- Joey Hess Fri, 22 May 2015 14:20:18 -0400 + +git-annex (5.20150508.1) unstable; urgency=medium + + * Now builds cleanly using ghc 7.10 (as well as ghc back to 7.6). + * Imrovements to the git-annex-standalone.deb build process. + (Thanks, Yaroslav Halchenko) + + -- Joey Hess Mon, 11 May 2015 12:08:58 -0400 + +git-annex (5.20150508) unstable; urgency=medium + + * Improve behavior when a git-annex command is told to operate + on a file that doesn't exist. It will now continue to other + files specified after that on the command line, and only error out at + the end. + * S3: Enable debug logging when annex.debug or --debug is set. + * S3: git annex info will show additional information about a S3 remote + (endpoint, port, storage class) + * S3: Let git annex enableremote be used, without trying to recreate + a bucket that should already exist. + * S3: Fix incompatibility with bucket names used by hS3; the aws library + cannot handle upper-case bucket names. git-annex now converts them to + lower case automatically. + * import: Check for gitignored files before moving them into the tree. + (Needs git 1.8.4 or newer.) + * import: Don't stop entire import when one file fails due to being + gitignored or conflicting with something in the work tree. + * import: Before removing a duplicate file in --deduplicate or + --clean-duplicates mode, verify that enough copies of its content still + exist. + * Improve integration with KDE's file manager to work with dolphin + version 14.12.3 while still being compatable with 4.14.2. + Thanks, silvio. + * assistant: Added --autostop to complement --autostart. + * Work around wget bug #784348 which could cause it to clobber git-annex + symlinks when downloading from ftp. + * Support checking ftp urls for file presence. + * Fix bogus failure of fsck --fast. + * fsck: Ignore error recording the fsck in the activity log, + which can happen when running fsck in a read-only repository. + Closes: #698559 + (fsck can still need to write to the repository if it find problems, + but a successful fsck can be done read-only) + * Improve quvi 0.4 output parsing to handle cases wher there is no known + filename extension. This is currently the case when using quvi with + youtube. In this case, the extension ".m" will be used. + * Dropped support for older versions of yesod, warp, and dbus than the ones + in Debian Jessie. + * Switch from the obsolete dataenc library for base64 encoding to sandi. + (Thanks, Magnus Therning) + * Debian's ghc now supports TH on arm! Adjust build dependencies + to build the webapp on arm, and enable DAV support on arm. \o/ + * Adjust some other arch specific build dependencies that are now + available on more architectures in Devian unstable. + * Windows: Remove cygwin ssh, the newer version of which has stopped + honoring the setting of HOME. Instead, copy msysgit's ssh into PATH. + Note that setting up a remote ssh server using password authentication + is known to be broken in this release on Windows. + * Windows: Roll back to an older version of rsync from cygwin. + The newer version has some dependency on a newer ssh from cygwin. + + -- Joey Hess Fri, 08 May 2015 13:42:30 -0400 + +git-annex (5.20150420) unstable; urgency=medium + + * Fix activity log parsing, which caused the log to not retain + activity from other uuids. + * Union merge could fall over if there was a file in the repository + with the same name as a git ref. Now fixed. + * info dir: Added information about repositories that + contain files in the specified directory. + * info: Added --bytes option. + * bittorrent: Fix handling of magnet links. + * When a key's size is unknown, still check the annex.diskreserve, + and avoid getting content if the disk is too full. + * Fix fsck --from a git remote in a local directory, and from + a directory special remote. + This was a reversion caused by the relative path changes in 5.20150113. + * fsck --from remote: When bad content is found in the remote, + and the local repo does not have a copy of the content, preserve + the bad content in .git/annex/bad/ to avoid further data loss. + * fsck --from remote: Avoid downloading a key if it would go over + the annex.diskreserve limit. + * required: New command, like wanted, but for required content. + * Removed dependency on haskell SHA library, + instead using cryptohash >= 0.11.0. + * Make repo init more robust. + * New debian/rules build-standalone target, which generates a + git-annex-standalone.deb that should work on many old Debian etc + systems. Thanks, Yaroslav Halchenko. + * Windows: Renamed start menu file to avoid loop in some versions + of Windows where the menu file is treated as a git-annex program. + * Windows: Fixed support of remotes on other drives. + (A reversion introduced in version 5.20150113.) + * Windows: Bundled versions of rsync, wget, ssh, and gpg from + cygwin all updated. Thanks, Yury V. Zaytsev. + + -- Joey Hess Mon, 20 Apr 2015 14:44:04 -0400 + +git-annex (5.20150409) unstable; urgency=medium + + * This fixes a bug in the assistant introduced by the literal pathspec + changes in version 5.20150406. + * --quiet now suppresses progress displays from eg, rsync. + (Second time's the charm..) + * fromkey, registerurl: When reading from stdin, allow the + filename and url, respectively, to contain whitespace. + * add: If annex.largefiles is set and does not match a file that's being + added, the file will be checked into git rather than being added to the + annex. Previously, git annex add skipped over such files; this new + behavior is more useful in direct mode. + * proxy: Made it work when run in a new repository before initial + commit. + * info: Display repository mode: bare when in a bare (non-direct mode) + repo. + * importfeed: Fix feed download when curl is used. + * importfeed: Error out when passed a non-url. + * webapp: When adding another local repository, and combining it + with the current repository, the new repository's remote path + was set to "." rather than the path to the current repository. + This was a reversion caused by the relative path changes in 5.20150113. + * contentlocationn: New plumbing command. + + -- Joey Hess Thu, 09 Apr 2015 15:06:38 -0400 + +git-annex (5.20150406.1) unstable; urgency=medium + + * Fixes a bug in the last release that caused rsync and possibly + other commands to hang at the end of a file transfer. + (--quiet is back to not blocking progress displays until + that code can be fixed properly.) + + -- Joey Hess Mon, 06 Apr 2015 17:13:13 -0400 + +git-annex (5.20150406) unstable; urgency=medium + + * Prevent git-ls-files from double-expanding wildcards when an + unexpanded wildcard is passed to a git-annex command like add or find. + * Fix make build target. Thanks, Justin Geibel. + * Fix GETURLS in external special remote protocol to strip + downloader prefix from logged url info before checking for the + specified prefix. + * importfeed: Avoid downloading a redundant item from a feed whose + permalink has been seen before, even when the url has changed. + * importfeed: Always store itemid in metadata; before this was only + done when annex.genmetadata was set. + * Relax debian package dependencies to git >= 1:1.8.1 rather + than needing >= 1:2.0. + * test: Fix --list-tests + * addurl --file: When used with a special remote that claims + urls and checks their contents, don't override the user's provided + filename with filenames that the special remote suggests. Also, + don't allow adding the url if the special remote says it contains + multiple files. + * import: --deduplicate and --cleanduplicates now output the keys + corresponding to duplicated files they process. + * expire: New command, for expiring inactive repositories. + * fsck: Record fsck activity for use by expire command. + * Fix truncation of parameters that could occur when using xargs git-annex. + * Significantly sped up processing of large numbers of directories + passed to a single git-annex command. + * version: Add --raw + * init: Improve fifo test to detect NFS systems that support fifos + but not well enough for sshcaching. + * --quiet now suppresses progress displays from eg, rsync. + (The option already suppressed git-annex's own built-in progress + displays.) + + -- Joey Hess Mon, 06 Apr 2015 12:48:48 -0400 + +git-annex (5.20150327) unstable; urgency=medium + + * readpresentkey: New plumbing command for checking location log. + * checkpresentkey: New plumbing command to check if a key can be verified + to be present on a remote. + * Added a post-update-annex hook, which is run after the git-annex branch + is updated. Needed for git update-server-info. + * migrate: --force will force migration of keys already using the + destination backend. Useful in rare cases. + * Man pages for individual commands now available, and can be + opened using "git annex help " + * --auto is no longer a global option; only get, drop, and copy + accept it. (Not a behavior change unless you were passing it to a + command that ignored it.) + * Improve error message when --in @date is used and there is no + reflog for the git-annex branch. + * assistant: Committing a whole lot of files at once could overflow + command-line length limits and cause the commit to fail. This + only happened when using the assistant in an indirect mode repository. + * Work around curl bug when asked to download an empty url to a file. + * Fix bug introduced in the last release that broke git-annex sync + when git-annex was installed from the standalone tarball. + + -- Joey Hess Fri, 27 Mar 2015 13:10:59 -0400 + +git-annex (5.20150317) unstable; urgency=medium + + * fsck: Incremental fsck uses sqlite to store its records, instead + of abusing the sticky bit. Existing sticky bits are ignored; + incremental fscks started by old versions won't be resumed by + this version. + * fsck: Multiple incremental fscks of different repos (including remotes) + can now be running at the same time in the same repo without it + getting confused about which files have been checked for which remotes. + * unannex: Refuse to unannex when repo is too new to have a HEAD, + since in this case there must be staged changes in the index + (if there is anything to unannex), and the unannex code path + needs to run with a clean index. + * Linux standalone: Set LOCPATH=/dev/null to work around + https://ghc.haskell.org/trac/ghc/ticket/7695 + This prevents localization from working, but git-annex + is not localized anyway. + * sync: As well as the synced/git-annex push, attempt a + git-annex:git-annex push, as long as the remote branch + is an ancestor of the local branch, to better support bare git repos. + (This used to be done, but it forgot to do it since version 4.20130909.) + * When re-execing git-annex, use current program location, rather than + ~/.config/git-annex/program, when possible. + * Submodules are now supported by git-annex! + * metadata: Fix encoding problem that led to mojibake when storing + metadata strings that contained both unicode characters and a space + (or '!') character. + * Also potentially fixes encoding problem when embedding credentials + that contain unicode characters. + * sync: Fix committing when in a direct mode repo that has no HEAD ref. + (For example, a newly checked out git submodule.) + * Added SETURIPRESENT and SETURIMISSING to external special remote protocol, + useful for things like ipfs that don't use regular urls. + * addurl: Added --raw option, which bypasses special handling of quvi, + bittorrent etc urls. + * git-annex-shell: Improve error message when the specified repository + doesn't exist or git config fails for some reason. + * fromkey --force: Skip test that the key has its content in the annex. + * fromkey: Add stdin mode. + * registerurl: New plumbing command for mass-adding urls to keys. + * remotedaemon: Fixed support for notifications of changes to gcrypt + remotes, which was never tested and didn't quite work before. + + -- Joey Hess Tue, 17 Mar 2015 13:02:36 -0400 + +git-annex (5.20150219) unstable; urgency=medium + + * glacier: Detect when the glacier command in PATH is the wrong one, + from boto, rather than from glacier-cli, and refuse to use it, + since the boto program fails to fail when passed + parameters it does not understand. + * groupwanted: New command to set the groupwanted preferred content + expression. + * import: Support file matching options such as --exclude, --include, + --smallerthan, --largerthan + * The file matching options are now only accepted by commands that + can actually use them, instead of by all commands. + * import: Avoid checksumming file twice when run in the default + or --duplicate mode. + * Windows: Fix bug in dropping an annexed file, which + caused a symlink to be staged that contained backslashes. + * webapp: Fix reversion in opening webapp when starting it manually + inside a repository. + * assistant: Improve sanity check for control characters when pairing. + * Improve race recovery code when committing to git-annex branch. + * addurl: Avoid crash if quvi is not installed, when git-annex was + built with process-1.2 + * bittorrent: Fix mojibake introduced in parsing arai2c progress output. + * fsck --from: If a download from a remote fails, propagate the failure. + * metadata: When setting metadata, do not recurse into directories by + default, since that can be surprising behavior and difficult to recover + from. The old behavior is available by using --force. + * sync, assistant: Include repository name in head branch commit message. + * The ssh-options git config is now used by gcrypt, rsync, and ddar + special remotes that use ssh as a transport. + * sync, assistant: Use the ssh-options git config when doing git pull + and push. + * remotedaemon: Use the ssh-options git config. + * Linux standalone: Improved process names of linker shimmed programs. + + -- Joey Hess Thu, 19 Feb 2015 14:16:03 -0400 + +git-annex (5.20150205) unstable; urgency=medium + + * info: Can now display info about a given uuid. + * Added to remote/uuid info: Count of the number of keys present + on the remote, and their size. This is rather expensive to calculate, + so comes last and --fast will disable it. + * info remote: Include the date of the last sync with the remote. + * sync: Added --message/-m option like git commit. + * remotedaemon: Fix problem that could prevent ssh connections being + made after two LOSTNET messages were received in a row (perhaps due to + two different network interfaces being brought down). + * Fix build failure when wget is not installed. + * Fix wording of message displayed when unable to get a file that + is available in untrusted repositories. + * addurl: When a Content-Disposition header suggests a filename to use, + addurl will consider using it, if it's reasonable and doesn't conflict + with an existing file. (--file overrides this) + * Fix default repository description created by git annex init, + which got broken by the relative path changes in the last release. + * init: Repository tuning parameters can now be passed when initializing a + repository for the first time. For details, see + http://git-annex.branchable.com/tuning/ + * merge: Refuse to merge changes from a git-annex branch of a repo + that has been tuned in incompatible ways. + * Support annex.tune.objecthash1, annex.tune.objecthashlower, and + annex.tune.branchhash1. + * Remove support for building without cryptohash. + * Added MD5 and MD5E backends. + * assistant: Fix local pairing when ssh pubkey comment contains spaces. + * Avoid using fileSize which maxes out at just 2 gb on Windows. + Instead, use hFileSize, which doesn't have a bounded size. + Fixes support for files > 2 gb on Windows. + * Windows: Fix running of the pre-commit-annex hook. + * Windows: Fix S3 special remote; need to call withSocketsDo. Thanks, Trent. + + -- Joey Hess Thu, 05 Feb 2015 14:08:33 -0400 + +git-annex (5.20150113) unstable; urgency=medium + + * unlock: Don't allow unlocking files that have never been committed to git + before, to avoid an intractable problem that prevents the pre-commit + hook from telling if such a file is intended to be an annexed file or not. + * Avoid re-checksumming when migrating from hash to hashE backend. + Closes: #774494 + * Fix build with process 1.2.1.0. + * Android: Provide a version built with -fPIE -pie to support Android 5.0. + * sync: Fix an edge case where syncing in a bare repository would try to + merge and so fail. + * Check git version at runtime, rather than assuming it will be the same + as the git version used at build time when running git-checkattr and + git-branch remove. + * Switch to using relative paths to the git repository. + - This allows the git repository to be moved while git-annex is running in + it, with fewer problems. + - On Windows, this avoids some of the problems with the absurdly small + MAX_PATH of 260 bytes. In particular, git-annex repositories should + work in deeper/longer directory structures than before. + * Generate shorter keys for WORM and URL, avoiding keys that are longer + than used for SHA256, so as to not break on systems like Windows that + have very small maximum path length limits. + * Bugfix: A file named HEAD in the work tree could confuse some git commands + run by git-annex. + + -- Joey Hess Tue, 13 Jan 2015 12:10:08 -0400 + +git-annex (5.20141231) unstable; urgency=medium + + * vicfg: Avoid crashing on badly encoded config data. + * Work around statfs() overflow on some XFS systems. + * sync: Now supports remote groups, the same way git remote update does. + * setpresentkey: A new plumbing-level command. + * Run shutdown cleanup actions even if there were failures processing + the command. Among other fixes, this means that addurl will stage + added files even if adding one of the urls fails. + * bittorrent: Fix locking problem when using addurl file:// + * Windows: Fix local rsync filepath munging (fixes 26 test suite failures). + * Windows: Got the rsync special remote working. + * Windows: Fix handling of views of filenames containing '%' + * OSX: Switched away from deprecated statfs64 interface. + + -- Joey Hess Wed, 31 Dec 2014 15:15:46 -0400 + +git-annex (5.20141219) unstable; urgency=medium + + * Webapp: When adding a new box.com remote, use the new style chunking. + Thanks, Jon Ander Peñalba. + * External special remote protocol now includes commands for setting + and getting the urls associated with a key. + * Urls can now be claimed by remotes. This will allow creating, + for example, a external special remote that handles magnet: and + *.torrent urls. + * Use wget -q --show-progress for less verbose wget output, + when built with wget 1.16. + * Added bittorrent special remote. + * addurl behavior change: When downloading an url ending in .torrent, + it will download files from bittorrent, instead of the old behavior + of adding the torrent file to the repository. + * Added Recommends on aria2. + * When possible, build with the haskell torrent library for parsing + torrent files. As a fallback, can instead use btshowmetainfo from + bittornado | bittorrent. + * Fix build with -f-S3. + + -- Joey Hess Fri, 19 Dec 2014 16:53:26 -0400 + +git-annex (5.20141203) unstable; urgency=medium + + * proxy: New command for direct mode repositories, allows bypassing + the direct mode guard in a safe way to do all sorts of things + including git revert, git mv, git checkout ... + * undo: New command to undo the most recent change to a file + or to the contents of a directory. + * Add undo action to nautilus and konqueror integration. + * diffdriver: New git-annex command, to make git external diff drivers + work with annexed files. + * pre-commit: Block partial commit of unlocked annexed file, since + that left a typechange staged in index due to some infelicity of git's + handling of partial commits. + * Work around behavior change in lsof 4.88's -F output format. + * S3: Switched to using the haskell aws library. + * S3: No longer buffers entire files in memory when uploading without + chunking. + * S3: When built with a new enough version of the haskell aws library, + supports doing multipart uploads, in order to store extremely large + files in S3 when not using chunking. + * Don't show "(gpg)" when decrypting the remote encryption cipher, + since this could be taken to read that's the only time git-annex + runs gpg, which is not the case. + * Debian package is now maintained by Gergely Nagy. + * Windows: Remove Alt+A keyboard shortcut, which turns out to have scope + outside the menus. + * Windows: Install ssh and other bundled programs to Git/cmd, + instead of Git/bin, since the latter is not in the default msysgit PATH. + + -- Joey Hess Wed, 03 Dec 2014 15:16:52 -0400 + +git-annex (5.20141125) unstable; urgency=medium + + * Remove fixup code for bad bare repositories created by + versions 5.20131118 through 5.20131127. That fixup code would + accidentally fire when --git-dir was incorrectly + pointed at the working tree of a git-annex repository, + possibly resulting in data loss. Closes: #768093 + * Windows: Fix crash when user.name is not set in git config. + + -- Joey Hess Wed, 05 Nov 2014 11:41:51 -0400 + +git-annex (5.20141024) unstable; urgency=medium + + * vicfg: Deleting configurations now resets to the default, where + before it has no effect. + * Remove hurd stuff from cabal file, since hackage currently rejects + it, and the test suite fails on hurd. + * initremote: Don't allow creating a special remote that has the same + name as an existing git remote. + * Windows: Use haskell setenv library to clean up several ugly workarounds + for inability to manipulate the environment on windows. This includes + making git-annex not re-exec itself on start on windows, and making the + test suite on Windows run tests without forking. + * glacier: Fix pipe setup when calling glacier-cli to retrieve an object. + * info: When run on a single annexed file, displays some info about the + file, including its key and size. + * info: When passed the name or uuid of a remote, displays info about that + remote. Remotes that support encryption, chunking, or embedded + creds will include that in their info. + * enableremote: When the remote has creds, update the local creds cache + file. Before, the old version of the creds could be left there, and + would continue to be used. + + -- Joey Hess Fri, 24 Oct 2014 13:03:29 -0400 + +git-annex (5.20141013) unstable; urgency=medium + + * Adjust cabal file to support building w/o assistant on the hurd. + * Support building with yesod 1.4. + * S3: Fix embedcreds=yes handling for the Internet Archive. + * map: Handle .git prefixed remote repos. Closes: #614759 + * repair: Prevent auto gc from happening when fetching from a remote. + + -- Joey Hess Mon, 13 Oct 2014 10:13:06 -0400 + +git-annex (5.20140927) unstable; urgency=medium + + * Really depend (not just build-depend) on new enough git for --no-gpg-sign + to work. Closes: #763057 + * Add temporary workaround for bug #763078 which broke building on armel + and armhf. + + -- Joey Hess Sat, 27 Sep 2014 14:25:09 -0400 + +git-annex (5.20140926) unstable; urgency=high + + * Depend on new enough git for --no-gpg-sign to work. Closes: #762446 + * Work around failure to build on mips by using cabal, not Setup, + to build in debian/rules. + + -- Joey Hess Fri, 26 Sep 2014 15:09:02 -0400 + +git-annex (5.20140919) unstable; urgency=high + + * Security fix for S3 and glacier when using embedcreds=yes with + encryption=pubkey or encryption=hybrid. CVE-2014-6274 + The creds embedded in the git repo were *not* encrypted. + git-annex enableremote will warn when used on a remote that has + this problem. For details, see: + https://git-annex.branchable.com/upgrades/insecure_embedded_creds/ + * assistant: Detect when repository has been deleted or moved, and + automatically shut down the assistant. Closes: #761261 + * Windows: Avoid crashing trying to list gpg secret keys, for gcrypt + which is not yet supported on Windows. + * WebDav: Fix enableremote crash when the remote already exists. + (Bug introduced in version 5.20140817.) + * add: In direct mode, adding an annex symlink will check it into git, + as was already done in indirect mode. + + -- Joey Hess Fri, 19 Sep 2014 12:53:42 -0400 + +git-annex (5.20140915) unstable; urgency=medium + + * New annex.hardlink setting. Closes: #758593 + * init: Automatically detect when a repository was cloned with --shared, + and set annex.hardlink=true, as well as marking the repository as + untrusted. + * Fix parsing of ipv6 address in git remote address when it was not + formatted as an url. + * The annex-rsync-transport configuration is now also used when checking + if a key is present on a rsync remote, and when dropping a key from + the remote. + * Promote file not found warning message to an error. + * Fix transfer lock file FD leak that could occur when two separate + git-annex processes were both working to perform the same set of + transfers. + * sync: Ensure that pending changes to git-annex branch are committed + before push when in direct mode. (Fixing a very minor reversion.) + * WORM backend: Switched to include the relative path to the file inside + the repository, rather than just the file's base name. Note that if you're + relying on such things to keep files separate with WORM, you should really + be using a better backend. + * Rather than crashing when there's a problem with the requested bloomfilter + capacity/accuracy, fall back to a reasonable default bloom filter size. + * Fix build with optparse-applicative 0.10. Closes: #761484 + * webapp: Fixed visual glitch in xmpp pairing that was reported live by a + user who tracked me down in front of a coffee cart in Portland. + (New bug reporting method of choice?) + + -- Joey Hess Mon, 15 Sep 2014 10:45:00 -0400 + +git-annex (5.20140831) unstable; urgency=medium + + * Make --help work when not in a git repository. Closes: #758592 + * Ensure that all lock fds are close-on-exec, fixing various problems with + them being inherited by child processes such as git commands. + * When accessing a local remote, shut down git-cat-file processes + afterwards, to ensure that remotes on removable media can be unmounted. + Closes: #758630 + * Fix handing of autocorrection when running outside a git repository. + * Fix stub git-annex test support when built without tasty. + * Do not preserve permissions and acls when copying files from + one local git repository to another. Timestamps are still preserved + as long as cp --preserve=timestamps is supported. Closes: #729757 + + -- Joey Hess Sun, 31 Aug 2014 12:30:08 -0700 + +git-annex (5.20140817) unstable; urgency=medium + + * New chunk= option to chunk files stored in special remotes. + Supported by: directory, S3, webdav, gcrypt, rsync, and all external + and hook special remotes. + * Partially transferred files are automatically resumed when using + chunked remotes! + * The old chunksize= option is deprecated. Do not use for new remotes. + * Legacy code for directory remotes using the old chunksize= option + will keep them working, but more slowly than before. + * webapp: Automatically install Konqueror integration scripts + to get and drop files. + * repair: Removing bad objects could leave fsck finding no more + unreachable objects, but some branches no longer accessible. + Fix this, including support for fixing up repositories that + were incompletely repaired before. + * Fix cost calculation for non-encrypted remotes. + * Display exception message when a transfer fails due to an exception. + * WebDAV: Sped up by avoiding making multiple http connections + when storing a file. + * WebDAV: Avoid buffering whole file in memory when uploading and + downloading. + * WebDAV: Dropped support for DAV before 1.0. + * testremote: New command to test uploads/downloads to a remote. + * Dropping an object from a bup special remote now deletes the git branch + for the object, although of course the object's content cannot be deleted + due to the nature of bup. + * unlock: Better error handling; continue past files that are not available + or cannot be unlocked due to disk space, and try all specified files. + * Windows: Now uses actual inode equivilants in new direct mode + repositories, for safer detection of eg, renaming of files with the same + size and mtime. + * direct: Fix ugly warning messages. + * WORM backend: When adding a file in a subdirectory, avoid including the + subdirectory in the key name. + * S3, Glacier, WebDAV: Fix bug that prevented accessing the creds + when the repository was configured with encryption=shared embedcreds=yes. + * direct: Avoid leaving file content in misctemp if interrupted. + * git-annex-shell sendkey: Don't fail if a remote asks for a key to be sent + that already has a transfer lock file indicating it's being sent to that + remote. The remote may have moved between networks, or reconnected. + * Switched from the old haskell HTTP library to http-conduit. + + -- Joey Hess Sun, 17 Aug 2014 10:30:58 -0400 + +git-annex (5.20140717) unstable; urgency=high + + * Fix minor FD leak in journal code. Closes: #754608 + * direct: Fix handling of case where a work tree subdirectory cannot + be written to due to permissions. + * migrate: Avoid re-checksumming when migrating from hashE to hash backend. + * uninit: Avoid failing final removal in some direct mode repositories + due to file modes. + * S3: Deal with AWS ACL configurations that do not allow creating or + checking the location of a bucket, but only reading and writing content to + it. + * resolvemerge: New plumbing command that runs the automatic merge conflict + resolver. + * Deal with change in git 2.0 that made indirect mode merge conflict + resolution leave behind old files. + * sync: Fix git sync with local git remotes even when they don't have an + annex.uuid set. (The assistant already did so.) + * Set gcrypt-publish-participants when setting up a gcrypt repository, + to avoid unncessary passphrase prompts. + This is a security/usability tradeoff. To avoid exposing the gpg key + ids who can decrypt the repository, users can unset + gcrypt-publish-participants. + * Install nautilus hooks even when ~/.local/share/nautilus/ does not yet + exist, since it is not automatically created for Gnome 3 users. + * Windows: Move .vbs files out of git\bin, to avoid that being in the + PATH, which caused some weird breakage. (Thanks, divB) + * Windows: Fix locking issue that prevented the webapp starting + (since 5.20140707). + + -- Joey Hess Thu, 17 Jul 2014 11:27:25 -0400 + +git-annex (5.20140709) unstable; urgency=medium + + * Fix race in direct mode merge code that could cause all files in the + repository to be removed. It should be able to recover repositories + experiencing this bug without data loss. See: + http://git-annex.branchable.com/bugs/bad_merge_commit_deleting_all_files/ + * Fix git version that supported --no-gpg-sign. + * Fix bug in automatic merge conflict resolution, when one side is an + annexed symlink, and the other side is a non-annexed symlink. + * Really fix bug that caused the assistant to make many unncessary + empty merge commits. + + -- Joey Hess Wed, 09 Jul 2014 15:28:03 -0400 + +git-annex (5.20140707) unstable; urgency=medium + + * assistant: Fix bug, introduced in last release, that caused the assistant + to make many unncessary empty merge commits. + * assistant: Fix one-way assistant->assistant sync in direct mode. + * Fix bug in annex.queuesize calculation that caused much more + queue flushing than necessary. + * importfeed: When annex.genmetadata is set, metadata from the feed + is added to files that are imported from it. + * Support users who have set commit.gpgsign, by disabling gpg signatures + for git-annex branch commits and commits made by the assistant. + * Fix memory leak when committing millions of changes to the git-annex + branch, eg after git-annex add has run on 2 million files in one go. + * Support building with bloomfilter 2.0.0. + * Run standalone install process when the assistant is started + (was only being run when the webapp was opened). + * Android: patch git to avoid fchmod, which fails on /sdcard. + * Windows: Got rid of that pesky DOS box when starting the webapp. + * Windows: Added Startup menu item so assistant starts automatically + on login. + * Windows: Fix opening file browser from webapp when repo is in a + directory with spaces. + * Windows: Assistant now logs to daemon.log. + + -- Joey Hess Mon, 07 Jul 2014 12:24:13 -0400 + +git-annex (5.20140613) unstable; urgency=medium + + * Ignore setsid failures. + * Avoid leaving behind .tmp files when failing in some cases, including + importing files to a disk that is full. + * Avoid bad commits after interrupted direct mode sync (or merge). + * Fix build with wai 0.3.0. + * Deal with FAT's low resolution timestamps, which in combination with + Linux's caching of higher res timestamps while a FAT is mounted, caused + direct mode repositories on FAT to seem to have modified files after + they were unmounted and remounted. + * Windows: Fix opening webapp when repository is in a directory with + spaces in the path. + * Detect when Windows has lost its mind in a timezone change, and + automatically apply a delta to the timestamps it returns, to get back to + sane values. + + -- Joey Hess Fri, 13 Jun 2014 09:58:07 -0400 + +git-annex (5.20140606) unstable; urgency=medium + + * webapp: When adding a new local repository, fix bug that caused its + group and preferred content to be set in the current repository, + even when not combining. + * webapp: Avoid stomping on existing description, group and + preferred content settings when enabling or combining with + an already existing remote. + * assistant: Make sanity checker tmp dir cleanup code more robust. + * unused: Avoid checking view branches for unused files. + * webapp: Include ssh port in mangled hostname. + * Windows: Fix bug introduced in last release that caused files + in the git-annex branch to have lines teminated with \r. + * Windows: Fix retrieving of files from local bare git repositories. + + -- Joey Hess Fri, 06 Jun 2014 12:54:06 -0400 + +git-annex (5.20140529) unstable; urgency=medium + + * Fix encoding of data written to git-annex branch. Avoid truncating + unicode characters to 8 bits. Allow any encoding to be used, as with + filenames (but utf8 is the sane choice). Affects metadata and repository + descriptions, and preferred content expressions. + * assistant: When there are multiple remotes giving different ways + to access the same repository, honor remote cost settings and use + the cheapest available. + * webapp: More robust startup when annex directory is not a git repo. + * initremote/enableremote: Basic support for using with regular git remotes; + initremote stores the location of an already existing git remote, + and enableremote setups up a remote using its stored location. + * webapp: Support for enabling known git repositories on ssh servers. + The repository must have been added using initremote. + * webapp: When setting up a ssh remote, record it using initremote, + so that it can be easily enabled elsewhere. + * webapp: When setting up a ssh remote, if the user inputs ~/foo, + normalize that to foo, since it's in the home directory by default. + * Use exceptions in place of deprecated MonadCatchIO-transformers + Thanks, Ben Gamari. + * android: Run busybox install with -s, since some versions of Android + prohibit making hard links. + * Android webapp: Fix EvilSplicer bugs that mangled the css files, + preventing icons from displaying, and also slightly broke the js files. + + -- Joey Hess Thu, 29 May 2014 14:41:56 -0400 + +git-annex (5.20140517) unstable; urgency=medium + + * webapp: Switched to bootstrap 3. + Thanks, Sören Brunk. + * Standalone builds now check gpg signatures before upgrading. + * Simplified repository description line format. The remote name, + if any, is always in square brackets after the description. + * assistant: Clean up stale tmp files on startup. + * webapp: Better ssh password prompting. + * Depend on git-remote-gcrypt 0.20130908-6. Older versions + fail when the assistant is run with no controlling tty. + * Added ddar special remote. + Thanks, Robie Basak. + * webapp: Fixed drag and drop to reorder the list of remotes. + * group: When no groups are specified to set, lists the current groups + of a repository. + * Add remote.$name.annex-shell configuration. + Thanks, Fraser Tweedale + * Support symlinking git-annex and git-annex-shell + from the Linux standalone bundle into PATH. + Thanks, jlebar. + + -- Joey Hess Sat, 17 May 2014 13:30:39 -0400 + +git-annex (5.20140421) unstable; urgency=medium + + * assistant: Now detects immediately when other repositories push + changes to a ssh remote, and pulls. + ** XMPP is no longer needed in this configuration! ** + This requires the remote server have git-annex-shell with + notifychanges support (>= 5.20140405) + * webapp: Show a network signal icon next to ssh and xmpp remotes that + it's currently connected with. + * webapp: Rework xmpp nudge to prompt for either xmpp or a ssh remote + to be set up. + * sync, assistant, remotedaemon: Use ssh connection caching for git pushes + and pulls. + * remotedaemon: When network connection is lost, close all cached ssh + connections. + * Improve handling of monthly/yearly scheduling. + * Avoid depending on shakespeare except for when building the webapp. + * uninit: Avoid making unncessary copies of files. + * info: Allow use in a repository where annex.uuid is not set. + * reinit: New command that can initialize a new repository using + the configuration of a previously known repository. + Useful if a repository got deleted and you want + to clone it back the way it was. + * drop --from: When local repository is untrusted, its copy of a file does + not count. + * Bring back rsync -p, but only when git-annex is running on a non-crippled + file system. This is a better approach to fix #700282 while not + unncessarily losing file permissions on non-crippled systems. + * webapp: Start even if the current directory is listed in + ~/.config/git-annex/autostart but no longer has a git repository in it. + * findref: New command, like find but shows files in a specified git ref. + * webapp: Fix UI for removing XMPP connection. + * When init detects that git is not configured to commit, and sets + user.email to work around the problem, also make it set user.name. + * webapp: Support using git-annex on a remote server, which was installed + from the standalone tarball or OSX app, and so does not have + git-annex in PATH (and may also not have git or rsync in PATH). + * standalone tarball, OSX app: Install a ~/.ssh/git-annex-wrapper, which + can be used to run git-annex, git, rsync, etc. + + -- Joey Hess Sun, 20 Apr 2014 19:43:14 -0400 + +git-annex (5.20140412) unstable; urgency=high + + * Last release didn't quite fix the high cpu issue in all cases, this should. + + -- Joey Hess Fri, 11 Apr 2014 17:14:38 -0400 + +git-annex (5.20140411) unstable; urgency=high + + * importfeed: Filename template can now contain an itempubdate variable. + Needs feed 0.3.9.2. + * Fix rsync progress parsing in locales that use comma in number display. + Closes: #744148 + * assistant: Fix high CPU usage triggered when a monthly fsck is scheduled, + and the last time the job ran was a day of the month > 12. This caused a + runaway loop. Thanks to Anarcat for his assistance, and to Maximiliano + Curia for identifying the cause of this bug. + * Remove wget from OSX dmg, due to issues with cert paths that broke + git-annex automatic upgrading. Instead, curl is used, unless the + OSX system has wget installed, which will then be used. + + -- Joey Hess Fri, 11 Apr 2014 14:59:49 -0400 + +git-annex (5.20140405) unstable; urgency=medium + + * git-annex-shell: Added notifychanges command. + * Improve display of dbus notifications. Thanks, Johan Kiviniemi. + * Fix nautilus script installation to not crash when the nautilus script dir + does not exist. Instead, only install scripts when the directory already + exists. + + -- Joey Hess Sat, 05 Apr 2014 16:54:33 -0400 + +git-annex (5.20140402) unstable; urgency=medium + + * unannex, uninit: Avoid committing after every file is unannexed, + for massive speedup. + * --notify-finish switch will cause desktop notifications after each + file upload/download/drop completes + (using the dbus Desktop Notifications Specification) + * --notify-start switch will show desktop notifications when each + file upload/download starts. + * webapp: Automatically install Nautilus integration scripts + to get and drop files. + * tahoe: Pass -d parameter before subcommand; putting it after + the subcommand no longer works with tahoe-lafs version 1.10. + (Thanks, Alberto Berti) + * forget --drop-dead: Avoid removing the dead remote from the trust.log, + so that if git remotes for it still exist anywhere, git annex info + will still know it's dead and not show it. + * git-annex-shell: Make configlist automatically initialize + a remote git repository, as long as a git-annex branch has + been pushed to it, to simplify setup of remote git repositories, + including via gitolite. + * add --include-dotfiles: New option, perhaps useful for backups. + * Version 5.20140227 broke creation of glacier repositories, + not including the datacenter and vault in their configuration. + This bug is fixed, but glacier repositories set up with the broken + version of git-annex need to have the datacenter and vault set + in order to be usable. This can be done using git annex enableremote + to add the missing settings. For details, see + http://git-annex.branchable.com/bugs/problems_with_glacier/ + * Added required content configuration. + * assistant: Improve ssh authorized keys line generated in local pairing + or for a remote ssh server to set environment variables in an + alternative way that works with the non-POSIX fish shell, as well + as POSIX shells. + + -- Joey Hess Wed, 02 Apr 2014 16:42:53 -0400 + +git-annex (5.20140320) unstable; urgency=medium + + * Fix zombie leak and general inneficiency when copying files to a + local git repo. + * Fix ssh connection caching stop method to work with openssh 6.5p1, + which broke the old method. + * webapp: Added a "Sync now" item to each repository's menu. + * webapp: Use securemem for constant time auth token comparisons. + * copy --fast --to remote: Avoid printing anything for files that + are already believed to be present on the remote. + * Commands that allow specifying which repository to act on using + the repository's description will now fail when multiple repositories + match, rather than picking a repository at random. + (So will --in=) + * Better workaround for problem umasks when eg, setting up ssh keys. + * "standard" can now be used as a first-class keyword in preferred content + expressions. For example "standard or (include=otherdir/*)" + * groupwanted can be used in preferred content expressions. + * vicfg: Allows editing preferred content expressions for groups. + * Improve behavior when unable to parse a preferred content expression + (thanks, ion). + * metadata: Add --get + * metadata: Support --key option (and some other ones like --all) + * For each metadata field, there's now an automatically maintained + "$field-lastchanged" that gives the date of the last change to that + field. Also the "lastchanged" field for the date of the last change + to any of a file's metadata. + * unused: In direct mode, files that are deleted from the work tree + and so have no content present are no longer incorrectly detected as + unused. + * Avoid encoding errors when using the unused log file. + * map: Fix crash when one of the remotes of a repo is a local directory + that does not exist, or is not a git repo. + * repair: Improve memory usage when git fsck finds a great many broken + objects. + * Windows: Fix some filename encoding bugs. + * rsync special remote: Fix slashes when used on Windows. + + -- Joey Hess Thu, 20 Mar 2014 13:21:12 -0400 + +git-annex (5.20140306) unstable; urgency=high + + * sync: Fix bug in direct mode that caused a file that was not + checked into git to be deleted when there was a conflicting + merge with a remote. + * webapp: Now supports HTTPS. + * webapp: No longer supports a port specified after --listen, since + it was buggy, and that use case is better supported by setting up HTTPS. + * annex.listen can be configured, instead of using --listen + * annex.startupscan can be set to false to disable the assistant's startup + scan. + * Probe for quvi version at run time. + * webapp: Filter out from Switch Repository list any + repositories listed in autostart file that don't have a + git directory anymore. (Or are bare) + * webapp: Refuse to start in a bare git repository. + * assistant --autostart: Refuse to start in a bare git repository. + * webapp: Don't list the public repository group when editing a + git repository; it only makes sense for special remotes. + * view, vfilter: Add support for filtering tags and values out of a view, + using !tag and field!=value. + * vadd: Allow listing multiple desired values for a field. + * view: Refuse to enter a view when no branch is currently checked out. + * metadata: To only set a field when it's not already got a value, use + -s field?=value + * Run .git/hooks/pre-commit-annex whenever a commit is made. + * sync: Automatically resolve merge conflict between and annexed file + and a regular git file. + * glacier: Pass --region to glacier checkpresent. + * webdav: When built with a new enough haskell DAV (0.6), disable + the http response timeout, which was only 5 seconds. + * webapp: Include no-pty in ssh authorized_keys lines. + * assistant: Smarter log file rotation, which takes free disk space + into account. + + -- Joey Hess Thu, 06 Mar 2014 12:28:04 -0400 + +git-annex (5.20140227) unstable; urgency=medium + + * metadata: Field names limited to alphanumerics and a few whitelisted + punctuation characters to avoid issues with views, etc. + * metadata: Field names are now case insensative. + * When constructing views, metadata is available about the location of the + file in the view's reference branch. Allows incorporating parts of the + directory hierarchy in a view. + For example `git annex view tag=* podcasts/=*` makes a view in the form + tag/showname. + * --metadata field=value can now use globs to match, and matches + case insensatively, the same as git annex view field=value does. + * annex.genmetadata can be set to make git-annex automatically set + metadata (year and month) when adding files. + * Make annex.web-options be used in several places that call curl. + * Fix handling of rsync remote urls containing a username, + including rsync.net. + * Preserve metadata when staging a new version of an annexed file. + * metadata: Support --json + * webapp: Fix creation of box.com and Amazon S3 and Glacier + repositories, broken in 5.20140221. + * webdav: When built with DAV 0.6.0, use the new DAV monad to avoid + locking files, which is not needed by git-annex's use of webdav, and + does not work on Box.com. + * webdav: Fix path separator bug when used on Windows. + * repair: Optimise unpacking of pack files, and avoid repeated error + messages about corrupt pack files. + * Add build dep on regex-compat to fix build on mipsel, which lacks + regex-tdfa. + * Disable test suite on sparc, which is missing optparse-applicative. + * Put non-object tmp files in .git/annex/misctmp, leaving .git/annex/tmp + for only partially transferred objects. + + -- Joey Hess Thu, 27 Feb 2014 11:34:19 -0400 + +git-annex (5.20140221) unstable; urgency=medium + + * metadata: New command that can attach metadata to files. + * --metadata can be used to limit commands to acting on files + that have particular metadata. + * Preferred content expressions can use metadata=field=value + to limit them to acting on files that have particular metadata. + * view: New command that creates and checks out a branch that provides + a structured view of selected metadata. + * vfilter, vadd, vpop, vcycle: New commands for operating within views. + * pre-commit: Update metadata when committing changes to locations + of annexed files within a view. + * Add progress display for transfers to/from external special remotes. + * unused: Fix to actually detect unused keys when in direct mode. + * fsck: When run with --all or --unused, while .gitattributes + annex.numcopies cannot be honored since it's operating on keys + instead of files, make it honor the global numcopies setting, + and the annex.numcopies git config setting. + * trust, untrust, semitrust, dead: Warn when the trust level is + overridden in .git/config. + * glacier: Do not try to run glacier value create when an existing glacier + remote is enabled. + * fsck: Refuse to do anything if more than one of --incremental, --more, + and --incremental-schedule are given, since it's not clear which option + should win. + * Windows webapp: Can set up box.com, Amazon S3, and rsync.net remotes + * Windows webapp: Can create repos on removable drives. + * Windows: Ensure HOME is set, as needed by bundled cygwin utilities. + + -- Joey Hess Fri, 21 Feb 2014 11:23:59 -0400 + +git-annex (5.20140210) unstable; urgency=medium + + * --in can now refer to files that were located in a repository at + some past date. For example, --in="here@{yesterday}" + * Fixed direct mode annexed content locking code, which is used to + guard against recursive file drops. + * This is the first beta-level release of the Windows port with important + fixes (see below). + (The webapp and assistant are still alpha-level on Windows.) + * sync --content: Honor annex-ignore configuration. + * sync: Don't try to sync with xmpp remotes, which are only currently + supported when using the assistant. + * sync --content: Re-pull from remotes after downloading content, + since that can take a while and other changes may be pushed in the + meantime. + * sync --content: Reuse smart copy code from copy command, including + handling and repairing out of date location tracking info. + Closes: #737480 + * sync --content: Drop files from remotes that don't want them after + getting them. + * sync: Fix bug in automatic merge conflict resolution code when used + on a filesystem not supporting symlinks, which resulted in it losing + track of the symlink bit of annexed files. + * Added ways to configure rsync options to be used only when uploading + or downloading from a remote. Useful to eg limit upload bandwidth. + * Fix initremote with encryption=pubkey to work with S3, glacier, webdav, + and external special remotes. + * Avoid building with DAV 0.6 which is badly broken (see #737902). + * Fix dropping of unused keys with spaces in their name. + * Fix build on platforms not supporting the webapp. + * Document in man page that sshcaching uses ssh ControlMaster. + Closes: #737476 + * Windows: It's now safe to run multiple git-annex processes concurrently + on Windows; the lock files have been sorted out. + * Windows: Avoid using unix-compat's rename, which refuses to rename + directories. + * Windows: Fix deletion of repositories by test suite and webapp. + * Windows: Test suite 100% passes again. + * Windows: Fix bug in symlink calculation code. + * Windows: Fix handling of absolute unix-style git repository paths. + * Android: Avoid crashing when unable to set file mode for ssh config file + due to Android filesystem horribleness. + + -- Joey Hess Mon, 10 Feb 2014 12:54:57 -0400 + +git-annex (5.20140127) unstable; urgency=medium + + * sync --content: New option that makes the content of annexed files be + transferred. Similar to the assistant, this honors any configured + preferred content expressions. + * Remove --json option from commands not supporting it. + * status: Support --json. + * list: Fix specifying of files to list. + * Allow --all to be mixed with matching options like --copies and --in + (but not --include and --exclude). + * numcopies: New command, sets global numcopies value that is seen by all + clones of a repository. + * The annex.numcopies git config setting is deprecated. Once the numcopies + command is used to set the global number of copies, any annex.numcopies + git configs will be ignored. + * assistant: Make the prefs page set the global numcopies. + * Add lackingcopies, approxlackingcopies, and unused to + preferred content expressions. + * Client, transfer, incremental backup, and archive repositories + now want to get content that does not yet have enough copies. + * Client, transfer, and source repositories now do not want to retain + unused file contents. + * assistant: Checks daily for unused file contents, and when possible + moves them to a repository (such as a backup repository) that + wants to retain them. + * assistant: annex.expireunused can be configured to cause unused + file contents to be deleted after some period of time. + * webapp: Nudge user to see if they want to expire old unused file + contents when a lot of them seem to be piling up in the repository. + * repair: Check git version at run time. + * assistant: Run the periodic git gc in batch mode. + * added annex.secure-erase-command config option. + * test suite: Use tasty-rerun, and expose tasty command-line options. + * Optimise non-bare http remotes; no longer does a 404 to the wrong + url every time before trying the right url. Needs annex-bare to be + set to false, which is done when initially probing the uuid of a + http remote. + * webapp: After upgrading a git repository to git-annex, fix + bug that made it temporarily not be synced with. + * whereis: Support --all. + * All commands that support --all also support a --key option, + which limits them to acting on a single key. + + -- Joey Hess Mon, 27 Jan 2014 13:43:28 -0400 + +git-annex (5.20140117) unstable; urgency=medium + + * Really fix FTBFS on mipsel and sparc due to test suite not being available + on those architectures. + + -- Joey Hess Fri, 17 Jan 2014 14:46:27 -0400 + +git-annex (5.20140116) unstable; urgency=medium + + * Added tahoe special remote. + * external special remote protocol: Added GETGITDIR, and GETAVAILABILITY. + * Refuse to build with git older than 1.7.1.1, which is needed for + git checkout -B + * map: Fix display of v5 direct mode repos. + * repair: Support old git versions from before git fsck --no-dangling was + implemented. + * Fix a long-standing bug that could cause the wrong index file to be used + when committing to the git-annex branch, if GIT_INDEX_FILE is set in the + environment. This typically resulted in git-annex branch log files being + committed to the master branch and later showing up in the work tree. + (These log files can be safely removed.) + * assistant: Detect if .git/annex/index is corrupt at startup, and + recover. + * repair: Fix bug in packed refs file exploding code that caused a .gitrefs + directory to be created instead of .git/refs + * Fix FTBFS on mipsel and sparc due to test suite not being available + on those architectures. + * Android: Avoid passing --clobber to busybox wget. + + -- Joey Hess Thu, 16 Jan 2014 11:34:54 -0400 + +git-annex (5.20140107) unstable; urgency=medium + + * mirror: Support --all (and --unused). + * external special remote protocol: Added GETUUID, GETWANTED, SETWANTED, + SETSTATE, GETSTATE, DEBUG. + * Windows: Fix bug in direct mode merge code that could cause files + in subdirectories to go missing. + * Windows: Avoid eating stdin when running ssh to add a authorized key, + since this is used for password prompting. + * Avoid looping if long-running git cat-file or git hash-object crashes + and keeps crashing when restarted. + * Assistant: Remove stale MERGE_HEAD files in lockfile cleanup. + * Remotes can now be made read-only, by setting remote..annex-readonly + * wanted, schedule: Avoid printing "ok" after requested value. + * assistant: Ensure that .ssh/config and .ssh/authorized_keys are not + group or world writable when writing to those files, as that can make + ssh refuse to use them, if it allows another user to write to them. + * addurl, importfeed: Honor annex.diskreserve as long as the size of the + url can be checked. + * add: Fix rollback when disk is completely full. + * assistant: Fixed several minor memory leaks that manifested when + adding a large number of files. + * assistant: Start a new git-annex transferkeys process + after a network connection change, so that remotes that use a persistent + network connection are restarted. + * Adjust Debian build deps to match current state of sparc, mipsel. + + -- Joey Hess Tue, 07 Jan 2014 12:22:18 -0400 + +git-annex (5.20131230) unstable; urgency=medium + + * Added new external special remote interface. + * importfeed: Support youtube playlists. + * Add tasty to build-depends, so that test suite builds again. + (tasty was stuck in incoming.) + * Fix typo in test suite. + * Fix bug in Linux standalone build's shimming that broke git-annex-shell. + * Include git-receive-pack, git-upload-pack, git, and git-shell wrappers + in the Linux standalone build, and OSX app, so they will be available + when it's added to PATH. + * addurl, importfeed: Sanitize | and some other symbols and special + characters. + * Auto-upgrade v3 indirect repos to v5 with no changes. + This also fixes a problem when a direct mode repo was somehow set to v3 + rather than v4, and so the automatic direct mode upgrade to v5 was not + done. + * Android: Avoid trying to use Android's own ionice, which does not + allow specifying a command to run. Fixes transferring files to/from + android and probably a few other things. + + -- Joey Hess Mon, 30 Dec 2013 14:13:40 -0400 + +git-annex (5.20131221) unstable; urgency=low + + * assistant: Fix OSX-specific bug that caused the startup scan to try to + follow symlinks to other directories, and add their contents to the annex. + * assistant: Set StrictHostKeyChecking yes when creating ssh remotes, + and add it to the configuration for any ssh remotes previously created + by the assistant. This avoids repeated prompts by ssh if the host key + changes, instead syncing with such a remote will fail. Closes: #732602 + * Fix test suite to cover lock --force change. + * Add plumbing-level lookupkey and examinekey commands. + * find --format: Added hashdirlower, hashdirmixed, keyname, and mtime + format variables. + * assistant: Always batch changes found in startup scan. + * An armel Linux standalone build is now available, which includes the + webapp. + * Programs from Linux and OSX standalone builds can now be symlinked + into a directory in PATH as an alternative installation method, and will + use readlink to find where the build was unpacked. + * Include man pages in Linux and OSX standalone builds. + * Linux standalone build now includes its own glibc and forces the linker to + use it, to remove dependence on the host glibc. + + -- Joey Hess Sat, 21 Dec 2013 12:00:17 -0400 + +git-annex (5.20131213) unstable; urgency=low + + * Avoid using git commit in direct mode, since in some situations + it will read the full contents of files in the tree. + * assistant: Batch jobs are now run with ionice and nocache, when + those commands are available. + * assistant: Run transferkeys as batch jobs. + * Automatically fix up bad bare repositories created by + versions 5.20131118 through 5.20131127. + * rsync special remote: Fix fallback mode for rsync remotes that + use hashDirMixed. Closes: #731142 + * copy --from, get --from: When --force is used, ignore the + location log and always try to get the file from the remote. + * Deal with box.com changing the url of their webdav endpoint. + * Android: Fix SRV record lookups for XMPP to use android getprop + command to find DNS server, since there is no resolv.conf. + * import: Add --skip-duplicates option. + * lock: Require --force. Closes: #731606 + * import: better handling of overwriting an existing file/directory/broken + link when importing + * Windows: assistant and webapp work! (very experimental) + * Windows: Support annex.diskreserve. + * Fix bad behavior in Firefox, which was caused by an earlier fix to + bad behavior in Chromium. + * repair: Improve repair of git-annex index file. + * repair: Remove damaged git-annex sync branches. + * status: Ignore new files that are gitignored. + * Fix direct mode's handling when modifications to non-annexed files + are pulled from a remote. A bug prevented the files from being updated + in the work tree, and this caused the modification to be reverted. + * OSX: Remove ssh and ssh-keygen from dmg as they're included in OSX by + default. + + -- Joey Hess Fri, 13 Dec 2013 14:20:32 -0400 + +git-annex (5.20131130) unstable; urgency=low + + * init: Fix a bug that caused git annex init, when run in a bare + repository, to set core.bare=false. + + -- Joey Hess Sat, 30 Nov 2013 16:32:35 -0400 + +git-annex (5.20131127.1) unstable; urgency=low + + * Rebuild that does not try to use quvi 0.9 from experimental. + + -- Joey Hess Thu, 28 Nov 2013 07:57:36 -0400 + +git-annex (5.20131127) unstable; urgency=low + + * webapp: Detect when upgrades are available, and upgrade if the user + desires. + (Only when git-annex is installed using the prebuilt binaries + from git-annex upstream, not from eg Debian.) + * assistant: Detect when the git-annex binary is modified or replaced, + and either prompt the user to restart the program, or automatically + restart it. + * annex.autoupgrade configures both the above upgrade behaviors. + * Added support for quvi 0.9. Slightly suboptimal due to limitations in its + interface compared with the old version. + * Bug fix: annex.version did not get set on automatic upgrade to v5 direct + mode repo, so the upgrade was performed repeatedly, slowing commands down. + * webapp: Fix bug that broke switching between local repositories + that use the new guarded direct mode. + * Android: Fix stripping of the git-annex binary. + * Android: Make terminal app show git-annex version number. + * Android: Re-enable XMPP support. + * reinject: Allow to be used in direct mode. + * Futher improvements to git repo repair. Has now been tested in tens + of thousands of intentionally damaged repos, and successfully + repaired them all. + * Allow use of --unused in bare repository. + + -- Joey Hess Wed, 27 Nov 2013 18:41:44 -0400 + +git-annex (5.20131120) unstable; urgency=low + + * Fix Debian package to not try to run test suite, since haskell-tasty + is not out of new or in Build-Depends yet. + * dropunused, addunused: Allow "all" instead of a range to + act on all unused data. + * Ensure execute bit is set on directories when core.sharedrepository is set. + * Ensure that core.sharedrepository is honored when creating the .git/annex + directory. + * Improve repair code in the case where the index file is corrupt, + and this hides other problems from git fsck. + + -- Joey Hess Wed, 20 Nov 2013 12:54:18 -0400 + +git-annex (5.20131118) unstable; urgency=low + + * Direct mode repositories now have core.bare=true set, to prevent + accidentally running git commands that try to operate on the work tree, + and so do the wrong thing in direct mode. + * annex.version is now set to 5 for direct mode repositories. + This upgrade is handled fully automatically, no need to run + git annex upgrade + * The "status" command has been renamed to "info", to allow + "git annex status" to be used in direct mode repositories, now that + "git status" won't work in them. + * The -c option now not only modifies the git configuration seen by + git-annex, but it is passed along to every git command git-annex runs. + * watcher: Avoid loop when adding a file owned by someone else fails + in indirect mode because its permissions cannot be modified. + * webapp: Avoid encoding problems when displaying the daemon log file. + * webapp: Improve UI around remote that have no annex.uuid set, + either because setup of them is incomplete, or because the remote + git repository is not a git-annex repository. + * Include ssh-keygen in standalone bundle. + * Allow optionally configuring git-annex with -fEKG to enable awesome + remote monitoring interfaceat http://localhost:4242/ + * Fix bug that caused bad information to be written to the git-annex branch + when running describe or other commands with a remote that has no uuid. + * Work around Android linker problem that had prevented git-annex from + running on Android 4.3 and 4.4. + * repair: Handle case where index file is corrupt, but all objects are ok. + * assistant: Notice on startup when the index file is corrupt, and + auto-repair. + * Fix direct mode merge bug when a direct mode file was deleted and replaced + with a directory. An ordering problem caused the directory to not get + created in this case. + Thanks to Tim for the test case. + * Direct mode .git/annex/objects directories are no longer left writable, + because that allowed writing to symlinks of files that are not present, + which followed the link and put bad content in an object location. + Thanks to Tim for the test case. + * fsck: Fix up .git/annex/object directory permissions. + * Switched to the tasty test framework. + * Android: Adjust default .gitignore to ignore .thumbnails at any location + in the tree, not just at its top. + * webapp: Check annex.version. + + -- Joey Hess Mon, 18 Nov 2013 10:45:43 -0400 + +git-annex (4.20131106) unstable; urgency=low + + * Improve local pairing behavior when two computers both try to start + the pairing process separately. + * sync: Work even when the local git repository is new and empty, + with no master branch. + * gcrypt, bup: Fix bug that prevented using these special remotes + with encryption=pubkey. + * Fix enabling of gcrypt repository accessed over ssh; + git-annex-shell gcryptsetup had a bug that caused it to fail + with permission denied. + * Fix zombie process that occurred when switching between repository + views in the webapp. + * map: Work when there are gcrypt remotes. + * Fix build w/o webapp. + * Fix exception handling bug that could cause .git/annex/index to be used + for git commits outside the git-annex branch. Known to affect git-annex + when used with the git shipped with Ubuntu 13.10. + + -- Joey Hess Wed, 06 Nov 2013 11:17:47 -0400 + +git-annex (4.20131101) unstable; urgency=low + + * The "git annex content" command is renamed to "git annex wanted". + * New --want-get and --want-drop options which can be used to + test preferred content settings. + For example, "git annex find --in . --want-drop" + * assistant: When autostarted, wait 5 seconds before running the startup + scan, to avoid contending with the user's desktop login process. + * webapp: When setting up a bare shared repository, enable non-fast-forward + pushes. + * sync: Show a hint about receive.denyNonFastForwards when a push fails. + * directory, webdav: Fix bug introduced in version 4.20131002 that + caused the chunkcount file to not be written. Work around repositories + without such a file, so files can still be retreived from them. + * assistant: Automatically repair damanged git repository, if it can + be done without losing data. + * assistant: Support repairing git remotes that are locally accessible + (eg, on removable drives). + * add: Fix reversion in 4.20130827 when adding unlocked files that have + not yet been committed. + * unannex: New, much slower, but more safe behavior: Copies files out of + the annex. This avoids an unannex of one file breaking other files that + link to the same content. Also, it means that the content + remains in the annex using up space until cleaned up with + "git annex unused". + (The behavior of unannex --fast has not changed; it still hard links + to content in the annex. --fast was not made the default because it is + potentially unsafe; editing such a hard linked file can unexpectedly + change content stored in the annex.) + + -- Joey Hess Fri, 01 Nov 2013 11:34:27 -0400 + +git-annex (4.20131024) unstable; urgency=low + + * webapp: Fix bug when adding a remote and git-remote-gcrypt + is not installed. + * The assitant can now run scheduled incremental fsck jobs on the local + repository and remotes. These can be configured using vicfg or with the + webapp. + * repair: New command, which can repair damaged git repositories + (even ones not using git-annex). + * webapp: When git repository damange is detected, repairs can be + done using the webapp UI. + * Automatically and safely detect and recover from dangling + .git/annex/index.lock files, which would prevent git from + committing to the git-annex branch, eg after a crash. + * assistant: Detect stale git lock files at startup time, and remove them. + * addurl: Better sanitization of generated filenames. + * Better sanitization of problem characters when generating URL and WORM + keys. + * The control socket path passed to ssh needs to be 17 characters + shorter than the maximum unix domain socket length, because ssh + appends stuff to it to make a temporary filename. Closes: #725512 + * status: Fix space leak in local mode, introduced in version 4.20130920. + * import: Skip .git directories. + * Remove bogus runshell loop check. + * addurl: Improve message when adding url with wrong size to existing file. + * Fixed handling of URL keys that have no recorded size. + * status: Fix a crash if a temp file went away while its size was + being checked for status. + * Deal with git check-attr -z output format change in git 1.8.5. + * Work around sed output difference that led to version containing a newline + on OSX. + * sync: Fix automatic resolution of merge conflicts where one side is an + annexed file, and the other side is a non-annexed file, or a directory. + * S3: Try to ensure bucket name is valid for archive.org. + * assistant: Bug fix: When run in a subdirectory, files from incoming merges + were wrongly added to that subdirectory, and removed from their original + locations. + * Windows: Deal with strange msysgit 1.8.4 behavior of not understanding + DOS formatted paths for --git-dir and --work-tree. + * Removed workaround for bug in git 1.8.4r0. + * Added git-recover-repository command to git-annex source + (not built by default; this needs to move to someplace else). + * webapp: Move sidebar to the right hand side of the screen. + + -- Joey Hess Thu, 24 Oct 2013 12:59:55 -0400 + +git-annex (4.20131002) unstable; urgency=low + + * Note that the layout of gcrypt repositories has changed, and + if you created one you must manually upgrade it. + See http://git-annex.branchable.com/upgrades/gcrypt/ + * webapp: Support setting up and using encrypted git repositories on + any ssh server, as well as on rsync.net. + * git-annex-shell: Added support for operating inside gcrypt repositories. + * Disable receive.denyNonFastForwards when setting up a gcrypt special + remote, since gcrypt needs to be able to fast-forward the master branch. + * import: Preserve top-level directory structure. + * Use cryptohash rather than SHA for hashing when no external hash program + is available. This is a significant speedup for SHA256 on OSX, for + example. + * Added SKEIN256 and SKEIN512 backends. + * Android build redone from scratch, many dependencies updated, + and entire build can now be done using provided scripts. + * assistant: Clear the list of failed transfers when doing a full transfer + scan. This prevents repeated retries to download files that are not + available, or are not referenced by the current git tree. + * indirect, direct: Better behavior when a file is not owned by + the user running the conversion. + * add, import, assistant: Better preserve the mtime of symlinks, + when when adding content that gets deduplicated. + * Send a git-annex user-agent when downloading urls. + Overridable with --user-agent option. + (Not yet done for S3 or WebDAV due to limitations of libraries used.) + * webapp: Fixed a bug where when a new remote is added, one file + may fail to sync to or from it due to the transferrer process not + yet knowing about the new remote. + * OSX: Bundled gpg upgraded, now compatible with config files + written by MacGPG. + * assistant: More robust inotify handling; avoid crashing if a directory + cannot be read. + * Moved list of backends and remote types from status to version + command. + + -- Joey Hess Wed, 02 Oct 2013 16:00:39 -0400 + +git-annex (4.20130920) unstable; urgency=low + + * webapp: Initial support for setting up encrypted removable drives. + * Recommend using my patched gcrypt, which fixes some bugs: + https://github.com/joeyh/git-remote-gcrypt + * Support hot-swapping of removable drives containing gcrypt repositories. + * list: New command, displays a compact table of remotes that + contain files. + (Thanks, anarcat for display code and mastensg for inspiration.) + * fsck: Fix detection and fixing of present direct mode files that are + wrongly represented as standin symlinks on crippled filesystems. + * sync: Fix bug that caused direct mode mappings to not be updated + when merging files into the tree on Windows. + * sync: Don't fail if the directory it is run in gets removed by the + sync. + * addurl: Fix quvi audodetection, broken in last release. + * status: In local mode, displays information about variance from configured + numcopies levels. (--fast avoids calculating these) + * gcrypt: Ensure that signing key is set to one of the participants keys. + * webapp: Show encryption information when editing a remote. + * Avoid unnecessarily catting non-symlink files from git, which can be + so large it runs out of memory. + + -- Joey Hess Fri, 20 Sep 2013 10:34:51 -0400 + +git-annex (4.20130911) unstable; urgency=low + + * Fix problem with test suite in non-unicode locale. + + -- Joey Hess Wed, 11 Sep 2013 12:14:16 -0400 + +git-annex (4.20130909) unstable; urgency=low + + * initremote: Syntax change when setting up an encrypted special remote. + Now use keyid=$KEYID rather than the old encryption=$KEYID + * forget: New command, causes git-annex branch history to be forgotten + in a way that will spread to other clones of the repository. + (As long as they're running this version or newer of git-annex.) + * forget --drop-dead: Completely removes mentions of repositories that + have been marked as dead from the git-annex branch. + * sync, assistant: Force push of the git-annex branch. Necessary + to ensure it gets pushed to remotes after being rewritten by forget. + * Added gcrypt support. This combines a fully encrypted git + repository (using git-remote-gcrypt) with an encrypted git-annex special + remote. + * sync: Support syncing with gcrypt remotes. + * importfeed: Also ignore transient problems with downloading content + from feeds. + * Honor core.sharedrepository when receiving and adding files in direct + mode. + * enableremote: gpg keys can be removed from those a remote encrypts + to by passing "keyid-=$KEYID". keyid+= is also provided. + (Thanks, guilhem for the patch.) + * Added encryption=pubkey scheme, which encrypts to public keys directly + rather than the hybrid approach. See documentation for advantages + and disadvantages, but encryption=hybrid is the recommended scheme still. + (Thanks, guilhem for the patch.) + * Fix Feeds display in build flags. + * Remind user when annex-ignore is set for some remotes, if unable to + get or drop a file, possibly because it's on an ignored remote. + * gpg: Force --no-textmode in case the user has it turned on in config. + * webapp: Improve javascript's handling of longpolling connection + failures, by reloading the current page in this case. + Works around chromium behavior where ajax connections to urls + that were already accessed are denied after navigating back to + a previous page. + * Allow building without quvi support. + + -- Joey Hess Mon, 09 Sep 2013 09:47:02 -0400 + +git-annex (4.20130827) unstable; urgency=low + + * Youtube support! (And 53 other video hosts). When quvi is installed, + git-annex addurl automatically uses it to detect when an page is + a video, and downloads the video file. + * web special remote: Also support using quvi, for getting files, + or checking if files exist in the web. + * unused: Is now a minimum of 30 times faster, and typically many + more times than that (when a repository has several branches). + (Thanks, guilhem for the patch.) + * unused: Fix bugs in two edge cases involving manually staged changes. + (Thanks, guilhem for the patch.) + * Android: Fix bug in terminal app that caused it to spin using much + CPU and battery. This problem was introduced in version 4.20130601. + * sync, merge: Bug fix: Don't try to merge into master when in a bare repo. + * import: Add options to control handling of duplicate files: + --duplicate, --deduplicate, and --clean-duplicates + * mirror: New command, makes two repositories contain the same set of files. + * Set --clobber when running wget to ensure resuming works properly. + * Unescape characters in 'file://...' URIs. (Thanks, guilhem for the patch.) + * Better error message when trying to use a git remote that has annex.ignore + set. + * Fix bug that caused typechanged symlinks to be assumed to be unlocked + files, so they were added to the annex by the pre-commit hook. + * Debian: Run the builtin test suite as an autopkgtest. + * Debian: Recommend ssh-askpass, which ssh will use when the assistant + is run w/o a tty. Closes: #719832 + + -- Joey Hess Tue, 27 Aug 2013 11:03:00 -0400 + +git-annex (4.20130815) unstable; urgency=low + + * assistant, watcher: .gitignore files and other git ignores are now + honored, when git 1.8.4 or newer is installed. + (Thanks, Adam Spiers, for getting the necessary support into git for this.) + * importfeed: Ignores transient problems with feeds. Only exits nonzero + when a feed has repeatedly had a problems for at least 1 day. + * importfeed: Fix handling of dots in extensions. + * Windows: Added support for encrypted special remotes. + * Windows: Fixed permissions problem that prevented removing files + from directory special remote. Directory special remotes now fully usable. + + -- Joey Hess Thu, 15 Aug 2013 10:14:33 +0200 + +git-annex (4.20130802) unstable; urgency=low + + * dropunused behavior change: Now refuses to drop the last copy of a + file, unless you use the --force. + This was the last place in git-annex that could remove data referred + to by the git history, without being forced. + Like drop, dropunused checks remotes, and honors the global + annex.numcopies setting. (However, .gitattributes settings cannot + apply to unused files.) + * Fix inverted logic in last release's fix for data loss bug, + that caused git-annex sync on FAT or other crippled filesystems to add + symlink standin files to the annex. + * importfeed can be used to import files from podcast feeds. + * webapp: When setting up a dedicated ssh key to access the annex + on a host, set IdentitiesOnly to prevent the ssh-agent from forcing + use of a different ssh key. That could result in unncessary password + prompts, or prevent git-annex-shell from being run on the remote host. + * webapp: Improve handling of remotes whose setup has stalled. + * Add status message to XMPP presence tag, to identify to others that + the client is a git-annex client. Closes: #717652 + * webapp: When creating a repository on a removable drive, set + core.fsyncobjectfiles, to help prevent data loss when the drive is yanked. + * Always build with -threaded, to avoid a deadlock when communicating with + gpg. + * unused: No longer shows as unused tmp files that are actively being + transferred. + * assistant: Fix NetWatcher to not sync with remotes that have + remote..annex-sync set to false. + * assistant: Fix deadlock that could occur when adding a lot of files + at once in indirect mode. + * assistant: Fix bug that caused it to stall when adding a very large + number of files at once (around 5 thousand). + * OSX: Make git-annex-webapp run in the background, so that the app icon + can be clicked on the open a new webapp when the assistant is already + running. + * Improve test suite on Windows; now tests git annex sync. + * Fix a few bugs involving filenames that are at or near the filesystem's + maximum filename length limit. + * find: Avoid polluting stdout with progress messages. Closes: #718186 + * Escape ':' in file/directory names to avoid it being treated + as a pathspec by some git commands. Closes: #718185 + * Slow and ugly work around for bug #718517 in git 1.8.4~rc0, which broke + git-cat-file --batch for filenames containing spaces. + (Will be reverted after next git pre-release fixes the problem.) + + -- Joey Hess Fri, 02 Aug 2013 11:35:16 -0400 + +git-annex (4.20130723) unstable; urgency=low + + * Fix data loss bug when adding an (uncompressed) tarball of a + git-annex repository, or other file that begins with something + that can be mistaken for a git-annex link. Closes: #717456 + * New improved version of the git-annex logo, contributed by + John Lawrence. + * Rsync.net have committed to support git-annex and offer a special + discounted rate for git-annex users. Updated the webapp to reflect this. + http://www.rsync.net/products/git-annex-pricing.html + * Install XDG desktop icon files. + * Support unannex and uninit in direct mode. + * Support import in direct mode. + * webapp: Better display of added files. + * fix: Preserve the original mtime of fixed symlinks. + * uninit: Preserve .git/annex/objects at the end, if it still + has content, so that old versions of files and deleted files + are not deleted. Print a message with some suggested actions. + * When a transfer is already being run by another process, + proceed on to the next file, rather than dying. + * Fix checking when content is present in a non-bare repository + accessed via http. + * Display byte sizes with more precision. + * watcher: Fixed a crash that could occur when a directory was renamed + or deleted before it could be scanned. + * watcher: Partially worked around a bug in hinotify, no longer crashes + if hinotify cannot process a directory (but can't detect changes in it) + * directory special remote: Fix checking that there is enough disk space + to hold an object, was broken when using encryption. + * webapp: Differentiate between creating a new S3/Glacier/WebDav remote, + and initializing an existing remote. When creating a new remote, avoid + conflicts with other existing (or deleted) remotes with the same name. + * When an XMPP server has SRV records, try them, but don't then fall + back to the regular host if they all fail. + * For long hostnames, use a hash of the hostname to generate the socket + file for ssh connection caching. + + -- Joey Hess Tue, 23 Jul 2013 10:46:05 -0400 + +git-annex (4.20130709) unstable; urgency=low + + * --all: New switch that makes git-annex operate on all data stored + in the git annex, including old versions of files. Supported by + fsck, get, move, copy. + * --unused: New switch that makes git-annex operate on all data found + by the last run of git annex unused. Supported by fsck, move, copy. + * get, move, copy: Can now be run in a bare repository, + like fsck already could. --all is enabled automatically in this case. + * merge: Now also merges synced/master or similar branches, which + makes it useful to put in a post-receive hook to make a repository + automatically update its working copy when git annex sync or the assistant + sync with it. + * webapp: Fix ssh setup with nonstandard port, broken in last release. + * init: Detect systems on which git commit fails due to not being able to + determine the FQDN, and put in a workaround so committing to the git-annex + branch works. + * addurl --pathdepth: Fix failure when the pathdepth specified is deeper + than the urls's path. + * Windows: Look for .exe extension when searching for a command in path. + * Pass -f to curl when downloading a file with it, so it propigates failure. + * Windows: Fix url to object when using a http remote. + * webapp: Fix authorized_keys line added when setting up a rsync remote + on a server that also supports git-annex, to not force running + git-annex-shell. + * OSX Mountain Lion: Fixed gpg bundled in dmg to not fail due to a missing + gpg-agent. + * Android: gpg is built without --enable-minimal, so it interoperates + better with other gpg builds that may default to using other algorithms + for encryption. + * dropunused, addunused: Complain when asked to operate on a number that + does not correspond to any unused key. + * fsck: Don't claim to fix direct mode when run on a symlink whose content + is not present. + * Make --numcopies override annex.numcopies set in .gitattributes. + + -- Joey Hess Tue, 09 Jul 2013 13:55:39 -0400 + +git-annex (4.20130627) unstable; urgency=low + + * assistant --autostart: Automatically ionices the daemons it starts. + * assistant: Daily sanity check thread is run niced. + * bup: Handle /~/ in bup remote paths. + Thanks, Oliver Matthews + * fsck: Ensures that direct mode is used for files when it's enabled. + * webapp: Fix bug when setting up a remote ssh repo repeatedly on the same + server. + * webapp: Ensure that ssh keys generated for different directories + on a server are always different. + * webapp: Fix bug setting up ssh repo if the user enters "~/" at the start + of the path. + * assistant: Fix bug that prevented adding files written by gnucash, + and more generally support adding hard links to files. However, + other operations on hard links are still unsupported. + * webapp: Fix bug that caused the webapp to hang when built with yesod 1.2. + + -- Joey Hess Thu, 27 Jun 2013 14:21:55 -0400 + +git-annex (4.20130621) unstable; urgency=low + + * Supports indirect mode on encfs in paranoia mode, and other + filesystems that do not support hard links, but do support + symlinks and other POSIX filesystem features. + * Android: Add .thumbnails to .gitignore when setting up a camera + repository. + * Android: Make the "Open webapp" menu item open the just created + repository when a new repo is made. + * webapp: When the user switches to display a different repository, + that repository becomes the default repository to be displayed next time + the webapp gets started. + * glacier: Better handling of the glacier inventory, which avoids + duplicate uploads to the same glacier repository by `git annex copy`. + * Direct mode: No longer temporarily remove write permission bit of files + when adding them. + * sync: Better support for bare git remotes. Now pushes directly to the + master branch on such a remote, instead of to synced/master. This + makes it easier to clone from a bare git remote that has been populated + with git annex sync or by the assistant. + * Android: Fix use of cp command to not try to use features present + only on build system. + * Windows: Fix hang when adding several files at once. + * assistant: In direct mode, objects are now only dropped when all + associated files are unwanted. This avoids a repreated drop/get loop + of a file that has a copy in an archive directory, and a copy not in an + archive directory. (Indirect mode still has some buggy behavior in this + area, since it does not keep track of associated files.) + Closes: #712060 + * status: No longer shows dead repositories. + * annex.debug can now be set to enable debug logging by default. + The webapp's debugging check box does this. + * fsck: Avoid getting confused by Windows path separators + * Windows: Multiple bug fixes, including fixing the data written to the + git-annex branch. + * Windows: The test suite now passes on Windows (a few broken parts are + disabled). + * assistant: On Linux, the expensive transfer scan is run niced. + * Enable assistant and WebDAV support on powerpc and sparc architectures, + which now have the necessary dependencies built. + + -- Joey Hess Fri, 21 Jun 2013 10:18:41 -0400 + +git-annex (4.20130601) unstable; urgency=medium + + * XMPP: Git push over xmpp made much more robust. + * XMPP: Avoid redundant and unncessary pushes. Note that this breaks + compatibility with previous versions of git-annex, which will refuse + to accept any XMPP pushes from this version. + * XMPP: Send pings and use them to detect when contact with the server + is lost. + * hook special remote: Added combined hook program support. + * Android app: Avoid using hard links to app's lib directory, which + is sometimes on a different filesystem than the data directory. + * Fix bug in parsing of parens in some preferred content expressions. + This fixes the behavior of the manual mode group. + * assistant: Work around git-cat-file's not reloading the index after files + are staged. + * Improve error handling when getting uuid of http remotes to auto-ignore, + like with ssh remotes. + * content: New command line way to view and configure a repository's + preferred content settings. + * sync: Fix double merge conflict resolution handling. + * XMPP: Fix a file descriptor leak. + * Android: Added an "Open WebApp" item to the terminal's menu. + * Android: Work around Android devices where the `am` command doesn't work. + * Can now restart certain long-running git processes if they crash, and + continue working. + + -- Joey Hess Sat, 01 Jun 2013 19:16:04 -0400 + +git-annex (4.20130521) unstable; urgency=low + + * Sanitize debian changelog version before putting it into cabal file. + Closes: #708619 + * Switch to MonadCatchIO-transformers for better handling of state while + catching exceptions. + * Fix a zombie that could result when running a process like gpg to + read and write to it. + * Allow building with gpg2. + * Disable building with the haskell threaded runtime when the webapp + is not built. This may fix builds on mips, s390x and sparc, which are + failing to link -lHSrts_thr + * Temporarily build without webapp on kfreebsd-i386, until yesod is + installable there again. + * Direct mode bug fix: After a conflicted merge was automatically resolved, + the content of a file that was already present could incorrectly + be replaced with a symlink. + * Fix a bug in the git-annex branch handling code that could + cause info from a remote to not be merged and take effect immediately. + * Direct mode is now fully tested by the test suite. + * Detect bad content in ~/.config/git-annex/program and look in PATH instead. + * OSX: Fixed gpg included in dmg. + * Linux standalone: Back to being built with glibc 2.13 for maximum + portability. + + -- Joey Hess Tue, 21 May 2013 13:10:26 -0400 + +git-annex (4.20130516) unstable; urgency=low + + * Android: The webapp is ported and working. + * Windows: There is a very rough Windows port. Do not trust it with + important data. + * git-annex-shell: Ensure that received files can be read. Files + transferred from some Android devices may have very broken permissions + as received. + * direct mode: Direct mode commands now work on files staged in the index, + they do not need to be committed to git. + * Temporarily add an upper bound to the version of yesod that can be built + with, since yesod 1.2 has a great many changes that will require extensive + work on the webapp. + * Disable building with the haskell threaded runtime when the assistant + is not built. This may fix builds on s390x and sparc, which are failing + to link -lHSrts_thr + * Avoid depending on regex-tdfa on mips, mipsel, and s390, where it fails + to build. + * direct: Fix a bug that could cause some files to be left in indirect mode. + * When initializing a directory special remote with a relative path, + the path is made absolute. + * SHA: Add a runtime sanity check that sha commands output something + that appears to be a real sha. + * configure: Better checking that sha commands output in the desired format. + * rsync special remotes: When sending from a crippled filesystem, use + the destination's default file permissions, as the local ones can + be arbitrarily broken. (Ie, ----rwxr-x for files on Android) + * migrate: Detect if a file gets corrupted while it's being migrated. + * Debian: Add a menu file. + + -- Joey Hess Thu, 16 May 2013 11:03:35 -0400 + +git-annex (4.20130501) unstable; urgency=low + + * sync, assistant: Behavior changes: Sync with remotes that have + annex-ignore set, so that git remotes on servers without git-annex + installed can be used to keep clients' git repos in sync. + * assistant: Work around misfeature in git 1.8.2 that makes + `git commit --alow-empty -m ""` run an editor. + * sync: Bug fix, avoid adding to the annex the + dummy symlinks used on crippled filesystems. + * Add public repository group. + (And inpreferreddir to preferred content expressions.) + * webapp: Can now set up Internet Archive repositories. + * S3: Dropping content from the Internet Archive doesn't work, but + their API indicates it does. Always refuse to drop from there. + * Automatically register public urls for files uploaded to the + Internet Archive. + * To enable an existing special remote, the new enableremote command + must be used. The initremote command now is used only to create + new special remotes. + * initremote: If two existing remotes have the same name, + prefer the one with a higher trust level. + * assistant: Improved XMPP protocol to better support multiple repositories + using the same XMPP account. Fixes bad behavior when sharing with a friend + when you or the friend have multiple reposotories on an XMPP account. + Note that XMPP pairing with your own devices still pairs with all + repositories using your XMPP account. + * assistant: Fix bug that could cause incoming pushes to not get + merged into the local tree. Particularly affected XMPP pushes. + * webapp: Display some additional information about a repository on + its edit page. + * webapp: Install FDO desktop menu file when started in standalone mode. + * webapp: Don't default to making repository in cwd when started + from within a directory containing a git-annex file (eg, standalone + tarball directory). + * Detect systems that have no user name set in GECOS, and also + don't have user.name set in git config, and put in a workaround + so that commits to the git-annex branch (and the assistant) + will still succeed despite git not liking the system configuration. + * webapp: When told to add a git repository on a remote server, and + the repository already exists as a non-bare repository, use it, + rather than initializing a bare repository in the same directory. + * direct, indirect: Refuse to do anything when the assistant + or git-annex watch daemon is running. + * assistant: When built with git before 1.8.0, use `git remote rm` + to delete a remote. Newer git uses `git remote remove`. + * rmurl: New command, removes one of the recorded urls for a file. + * Detect when the remote is broken like bitbucket is, and exits 0 when + it fails to run git-annex-shell. + * assistant: Several improvements to performance and behavior when + performing bulk adds of a large number of files (tens to hundreds + of thousands). + * assistant: Sanitize XMPP presence information logged for debugging. + * webapp: Now automatically fills in any creds used by an existing remote + when creating a new remote of the same type. Done for Internet Archive, + S3, Glacier, and Box.com remotes. + * Store an annex-uuid file in the bucket when setting up a new S3 remote. + * Support building with DAV 0.4. + + -- Joey Hess Wed, 01 May 2013 01:42:46 -0400 + +git-annex (4.20130417) unstable; urgency=low + + * initremote: Generates encryption keys with high quality entropy. + This can be disabled using --fast to get the old behavior. + The assistant still uses low-quality entropy when creating encrypted + remotes, to avoid delays. (Thanks, guilhem for the patch.) + * Bugfix: Direct mode no longer repeatedly checksums duplicated files. + * assistant: Work around horrible, terrible, very bad behavior of + gnome-keyring, by not storing special-purpose ssh keys in ~/.ssh/*.pub. + Apparently gnome-keyring apparently will load and indiscriminately use + such keys in some cases, even if they are not using any of the standard + ssh key names. Instead store the keys in ~/.ssh/annex/, + which gnome-keyring will not check. + * addurl: Bugfix: Did not properly add file in direct mode. + * assistant: Bug fix to avoid annexing the files that git uses + to stand in for symlinks on FAT and other filesystem not supporting + symlinks. + * Adjust preferred content expressions so that content in archive + directories is preferred until it has reached an archive or smallarchive + repository. + * webapp: New --listen= option allows running the webapp on one computer + and connecting to it from another. (Note: Does not yet use HTTPS.) + * Added annex.web-download-command setting. + * Added per-remote annex-rsync-transport option. (guilhem again) + * Ssh connection caching is now also used by rsync special remotes. + (guilhem yet again) + * The version number is now derived from git, unless built with + VERSION_FROM_CHANGELOG. + * assistant: Stop any transfers the assistant initiated on shutdown. + * assistant: Added sequence numbers to XMPP git push packets. (Not yet used.) + * addurl: Register transfer so the webapp can see it. + * addurl: Automatically retry downloads that fail, as long as some + additional content was downloaded. + * webapp: Much improved progress bar display for downloads from encrypted + remotes. + * Avoid using runghc, as that needs ghci. + * webapp: When a repository's group is changed, rescan for transfers. + * webapp: Added animations. + * webapp: Include the repository directory in the mangled hostname and + ssh key name, so that a locked down ssh key for one repository is not + re-used when setting up additional repositories on the same server. + * Fall back to internal url downloader when built without curl. + * fsck: Check content of direct mode files (only when the inode cache + thinks they are unmodified). + + -- Joey Hess Wed, 17 Apr 2013 09:07:38 -0400 + +git-annex (4.20130405) unstable; urgency=low + + * Group subcommands into sections in usage. Closes: #703797 + * Per-command usage messages. + * webapp: Fix a race that sometimes caused alerts or other notifications + to be missed if they occurred while a page was loading. + * webapp: Progess bar fixes for many types of special remotes. + * Build debian package without using cabal, which writes to HOME. + Closes: #704205 + * webapp: Run ssh server probes in a way that will work when the + login shell is a monstrosity that should have died 25 years ago, + such as csh. + * New annex.largefiles setting, which configures which files + `git annex add` and the assistant add to the annex. + * assistant: Check small files into git directly. + * Remotes can be configured to use other MAC algorithms than HMACSHA1 + to encrypt filenames. + Thanks, guilhem for the patch. + * git-annex-shell: Passes rsync --bwlimit options on rsync. + Thanks, guilhem for the patch. + * webapp: Added UI to delete repositories. Closes: #689847 + * Adjust built-in preferred content expressions to make most types + of repositories want content that is only located on untrusted, dead, + and unwanted repositories. + * drop --auto: Fix bug that prevented dropping files from untrusted + repositories. + * assistant: Fix bug that could cause direct mode files to be unstaged + from git. + * Update working tree files fully atomically. + * webapp: Improved transfer queue management. + * init: Probe whether the filesystem supports fifos, and if not, + disable ssh connection caching. + * Use lower case hash directories for storing files on crippled filesystems, + same as is already done for bare repositories. + + -- Joey Hess Fri, 05 Apr 2013 10:42:18 -0400 + +git-annex (4.20130323) unstable; urgency=low + + * webapp: Repository list is now included in the dashboard, and other + UI tweaks. + * webapp: Improved UI for pairing your own devices together using XMPP. + * webapp: Display an alert when there are XMPP remotes, and a cloud + transfer repository needs to be configured. + * Add incrementalbackup repository group. + * webapp: Encourage user to install git-annex on a server when adding + a ssh server, rather than just funneling them through to rsync. + * xmpp: --debug now enables a sanitized dump of the XMPP protocol + * xmpp: Try harder to detect presence of clients when there's a git push + to send. + * xmpp: Re-enable XA flag, since disabling it did not turn out to help + with the problems Google Talk has with not always sending presence + messages to clients. + * map: Combine duplicate repositories, for a nicer looking map. + * Fix several bugs caused by a bad Ord instance for Remote. + * webapp: Switch all forms to POST. + * assistant: Avoid syncing with annex-ignored remotes when reconnecting + to the network, or connecting a drive. + * assistant: Fix OSX bug that prevented committing changed files to a + repository when in indirect mode. + * webapp: Improved alerts displayed when syncing with remotes, and + when syncing with a remote fails. + * webapp: Force wrap long filenames in transfer display. + * assistant: The ConfigMonitor left one zombie behind each time + it checked for changes, now fixed. + * get, copy, move: Display an error message when an identical transfer + is already in progress, rather than failing with no indication why. + * assistant: Several optimisations to file transfers. + * OSX app and standalone Linux tarball now both support being added to + PATH; no need to use runshell to start git-annex. + * webapp: When adding a removable drive, you can now specify the + directory inside it to use. + * webapp: Confirm whether user wants to combine repositories when + adding a removable drive that already has a repository on it. + + -- Joey Hess Fri, 22 Mar 2013 18:54:05 -0400 + +git-annex (4.20130314) unstable; urgency=low + + * Bugfix: git annex add, when ran without any file or directory specified, + should add files in the current directory, but not act on unlocked files + elsewhere in the tree. + * Bugfix: drop --from an unavailable remote no longer updates the location + log, incorrectly, to say the remote does not have the key. + * Bugfix: If the UUID of a remote is not known, prevent --from, --to, + and other ways of specifying remotes by name from selecting it, + since it is not possible to sanely use it. + * Bugfix: Fix bug in inode cache sentinal check, which broke + copying to local repos if the repo being copied from had moved + to a different filesystem or otherwise changed all its inodes + + * Switch from using regex-compat to regex-tdfa, as the C regex library + is rather buggy. + * status: Can now be run with a directory path to show only the + status of that directory, rather than the whole annex. + * Added remote..annex-gnupg-options setting. + Thanks, guilhem for the patch. + * addurl: Add --relaxed option. + * addurl: Escape invalid characters in urls, rather than failing to + use an invalid url. + * addurl: Properly handle url-escaped characters in file:// urls. + + * assistant: Fix dropping content when a file is moved to an archive + directory, and getting contennt when a file is moved back out. + * assistant: Fix bug in direct mode that could occur when a symlink is + moved out of an archive directory, and resulted in the file not being + set to direct mode when it was transferred. + * assistant: Generate better commits for renames. + * assistant: Logs are rotated to avoid them using too much disk space. + * assistant: Avoid noise in logs from git commit about typechanged + files in direct mode repositories. + * assistant: Set gc.auto=0 when creating repositories to prevent + automatic commits from causing git-gc runs. + * assistant: If gc.auto=0, run git-gc once a day, packing loose objects + very non-aggressively. + * assistant: XMPP git pull and push requests are cached and sent when + presence of a new client is detected. + * assistant: Sync with all git remotes on startup. + * assistant: Get back in sync with XMPP remotes after network reconnection, + and on startup. + * assistant: Fix syncing after XMPP pairing. + * assistant: Optimised handling of renamed files in direct mode, + avoiding re-checksumming. + * assistant: Detects most renames, including directory renames, and + combines all their changes into a single commit. + * assistant: Fix ~/.ssh/git-annex-shell wrapper to work when the + ssh key does not force a command. + * assistant: Be smarter about avoiding unncessary transfers. + + * webapp: Work around bug in Warp's slowloris attack prevention code, + that caused regular browsers to stall when they reuse a connection + after leaving it idle for 30 seconds. + (See https://github.com/yesodweb/wai/issues/146) + * webapp: New preferences page allows enabling/disabling debug logging + at runtime, as well as configuring numcopies and diskreserve. + * webapp: Repository costs can be configured by dragging repositories around + in the repository list. + * webapp: Proceed automatically on from "Configure jabber account" + to pairing. + * webapp: Only show up to 10 queued transfers. + * webapp: DTRT when told to create a git repo that already exists. + * webapp: Set locally paired repositories to a lower cost than other + network remotes. + + * Run ssh with -T to avoid tty allocation and any login scripts that + may do undesired things with it. + * Several improvements to Makefile and cabal file. Thanks, Peter Simmons + * Stop depending on testpack. + * Android: Enable test suite. + + -- Joey Hess Thu, 14 Mar 2013 15:29:20 -0400 + +git-annex (4.20130227) unstable; urgency=low + + * annex.version is now set to 4 for direct mode repositories. + * Should now fully support git repositories with core.symlinks=false; + always using git's pseudosymlink files in such repositories. + * webapp: Allow creating repositories on filesystems that lack support for + symlinks. + * webapp: Can now add a new local repository, and make it sync with + the main local repository. + * Android: Bundle now includes openssh. + * Android: Support ssh connection caching. + * Android: Assistant is fully working. (But no webapp yet.) + * Direct mode: Support filesystems like FAT which can change their inodes + each time they are mounted. + * Direct mode: Fix support for adding a modified file. + * Avoid passing -p to rsync, to interoperate with crippled filesystems. + Closes: #700282 + * Additional GIT_DIR support bugfixes. May actually work now. + * webapp: Display any error message from git init if it fails to create + a repository. + * Fix a reversion in matching globs introduced in the last release, + where "*" did not match files inside subdirectories. No longer uses + the Glob library. + * copy: Update location log when no copy was performed, if the location + log was out of date. + * Makefile now builds using cabal, taking advantage of cabal's automatic + detection of appropriate build flags. + * test: The test suite is now built into the git-annex binary, and can + be run at any time. + + -- Joey Hess Wed, 27 Feb 2013 14:07:24 -0400 + +git-annex (3.20130216) unstable; urgency=low + + * Now uses the Haskell uuid library, rather than needing a uuid program. + * Now uses the Haskell Glob library, rather than pcre-light, avoiding + the need to install libpcre. Currently done only for Cabal or when + the Makefile is made to use -DWITH_GLOB + * Android port now available (command-line only). + * New annex.crippledfilesystem setting, allows use of git-annex + repositories on FAT and even worse filesystems; avoiding use of + hard links and locked down permissions settings. (Support is incomplete.) + * init: Detect when the repository is on a filesystem that does not + support hard links, or symlinks, or unix permissions, and set + annex.crippledfilesystem, as well as annex.direct. + * add: Improved detection of files that are modified while being added. + * Fix a bug in direct mode, introduced in the previous release, where + if a file was dropped and then got back, it would be stored in indirect + mode. + + -- Joey Hess Sat, 16 Feb 2013 10:03:26 -0400 + +git-annex (3.20130207) unstable; urgency=low + + * webapp: Now allows restarting any threads that crash. + * Adjust debian package to only build-depend on DAV on architectures + where it is available. + * addurl --fast: Use curl, rather than haskell HTTP library, to support https. + * annex.autocommit: New setting, can be used to disable autocommit + of changed files by the assistant, while it still does data syncing + and other tasks. + * assistant: Ignore .DS_Store on OSX. + * assistant: Fix location log when adding new file in direct mode. + * Deal with stale mappings for deleted file in direct mode. + * pre-commit: Update direct mode mappings. + * uninit, unannex --fast: If hard link creation fails, fall back to slow + mode. + * Clean up direct mode cache and mapping info when dropping keys. + * dropunused: Clean up stale direct mode cache and mapping info not + removed before. + + -- Joey Hess Thu, 07 Feb 2013 12:45:25 -0400 + +git-annex (3.20130124) unstable; urgency=low + + * Added source repository group, that only retains files until they've + been transferred to another repository. Useful for things like + repositories on cameras. + * Added manual repository group. Use to prevent the assistant from + downloading any file contents to keep things in sync. Instead + `git annex get`, `git annex drop` etc can be used manually as desired. + * webapp: More adjustments to longpoll code to deal with changes in + variable quoting in different versions of shakespeare-js. + * webapp: Avoid an error if a transfer is stopped just as it finishes. + Closes: #698184 + * webapp: Now always logs to .git/annex/daemon.log + * webapp: Has a page to view the log, accessed from the control menu. + * webapp: Fix crash adding removable drive that has an annex directory + in it that is not a git repository. + * Deal with incompatibility in gpg2, which caused prompts for encryption + passphrases rather than using the supplied --passphrase-fd. + * bugfix: Union merges involving two or more repositories could sometimes + result in data from one repository getting lost. This could result + in the location log data becoming wrong, and fsck being needed to fix it. + * sync: Automatic merge conflict resolution now stages deleted files. + * Depend on git 1.7.7.6 for --no-edit. Closes: #698399 + * Fix direct mode mapping code to always store direct mode filenames + relative to the top of the repository, even when operating inside a + subdirectory. + * fsck: Detect and fix consistency errors in direct mode mapping files. + * Avoid filename encoding errors when writing direct mode mappings. + + -- Joey Hess Tue, 22 Jan 2013 07:11:59 +1100 + +git-annex (3.20130114) unstable; urgency=low + + * Now handles the case where a file that's being transferred to a remote + is modified in place, which direct mode allows. When this + happens, the transfer now fails, rather than allow possibly corrupt + data into the remote. + * fsck: Better checking of file content in direct mode. + * drop: Suggest using git annex move when numcopies prevents dropping a file. + * webapp: Repo switcher filters out repos that do not exist any more + (or are on a drive that's not mounted). + * webapp: Use IP address, rather than localhost, since some systems may + have configuration problems or other issues that prevent web browsers + from connecting to the right localhost IP for the webapp. + * webapp: Adjust longpoll code to work with recent versions of + shakespeare-js. + * assistant: Support new gvfs dbus names used in Gnome 3.6. + * In direct mode, files with the same key are no longer hardlinked, as + that would cause a surprising behavior if modifying one, where the other + would also change. + * webapp: Avoid illegal characters in hostname when creating S3 or + Glacier remote. + * assistant: Avoid committer crashing if a file is deleted at the wrong + instant. + + -- Joey Hess Mon, 14 Jan 2013 15:25:18 -0400 + +git-annex (3.20130107) unstable; urgency=low + + * webapp: Add UI to stop and restart assistant. + * committer: Fix a file handle leak. + * assistant: Make expensive transfer scan work fully in direct mode. + * More commands work in direct mode repositories: find, whereis, move, copy, + drop, log, fsck, add, addurl. + * sync: No longer automatically adds files in direct mode. + * assistant: Detect when system is not configured with a user name, + and set environment to prevent git from failing. + * direct: Avoid hardlinking symlinks that point to the same content + when the content is not present. + * Fix transferring files to special remotes in direct mode. + + -- Joey Hess Mon, 07 Jan 2013 01:01:41 -0400 + +git-annex (3.20130102) unstable; urgency=low + + * direct, indirect: New commands, that switch a repository to and from + direct mode. In direct mode, files are accessed directly, rather than + via symlinks. Note that direct mode is currently experimental. Many + git-annex commands do not work in direct mode. Some git commands can + cause data loss when used in direct mode repositories. + * assistant: Now uses direct mode by default when setting up a new + local repository. + * OSX assistant: Uses the FSEvents API to detect file changes. + This avoids issues with running out of file descriptors on large trees, + as well as allowing detection of modification of files in direct mode. + Other BSD systems still use kqueue. + * kqueue: Fix bug that made broken symlinks not be noticed. + * vicfg: Quote filename. Closes: #696193 + * Bugfix: Fixed bug parsing transfer info files, where the newline after + the filename was included in it. This was generally benign, but in + the assistant, it caused unexpected dropping of preferred content. + * Bugfix: Remove leading \ from checksums output by sha*sum commands, + when the filename contains \ or a newline. Closes: #696384 + * fsck: Still accept checksums with a leading \ as valid, now that + above bug is fixed. + * SHA*E backends: Exclude non-alphanumeric characters from extensions. + * migrate: Remove leading \ in SHA* checksums, and non-alphanumerics + from extensions of SHA*E keys. + + -- Joey Hess Wed, 02 Jan 2013 13:21:34 -0400 + +git-annex (3.20121211) unstable; urgency=low + + * webapp: Defaults to sharing box.com account info with friends, allowing + one-click enabling of the repository. + * Fix broken .config/git-annex/program installed by standalone tarball. + * assistant: Retrival from glacier now handled. + * Include ssh in standalone tarball and OSX app. + * watch: Avoid leaving hard links to files behind in .git/annex/tmp + if a file is deleted or moved while it's being quarantined in preparation + to being added to the annex. + * Allow `git annex drop --from web`; of course this does not remove + any file from the web, but it does make git-annex remove all urls + associated with a file. + * webapp: S3 and Glacier forms now have a select list of all + currently-supported AWS regions. + * webdav: Avoid trying to set props, avoiding incompatibility with + livedrive.com. Needs DAV version 0.3. + * webapp: Prettify error display. + * webapp: Fix bad interaction between required fields and modals. + * webapp: Added help buttons and links next to fields that require + explanations. + * webapp: Encryption can be disabled when setting up remotes. + * assistant: Avoid trying to drop content from remotes that don't have it. + * assistant: Allow periods in ssh key comments. + * get/copy --auto: Transfer data even if it would exceed numcopies, + when preferred content settings want it. + * drop --auto: Fix dropping content when there are no preferred content + settings. + * webapp: Allow user to specify the port when setting up a ssh or rsync + remote. + * assistant: Fix syncing to just created ssh remotes. + * Enable WebDAV support in Debian package. Closes: #695532 + + -- Joey Hess Tue, 11 Dec 2012 11:25:03 -0400 + +git-annex (3.20121127) unstable; urgency=low + + * Fix dirContentsRecursive, which had missed some files in deeply nested + subdirectories. Could affect various parts of git-annex. + * rsync: Fix bug introduced in last release that broke encrypted rsync + special remotes. + * The standalone builds now unset their special path and library path + variables before running the system web browser. + + -- Joey Hess Tue, 27 Nov 2012 17:07:32 -0400 + +git-annex (3.20121126) unstable; urgency=low + + * New webdav and Amazon glacier special remotes. + * Display a warning when a non-existing file or directory is specified. + * webapp: Added configurator for Box.com. + * webapp: Show error messages to user when testing XMPP creds. + * Fix build of assistant without yesod. + * webapp: The list of repositiories refreshes when new repositories are + added, including when new repository configurations are pushed in from + remotes. + * OSX: Fix RunAtLoad value in plist file. + * Getting a file from chunked directory special remotes no longer buffers + it all in memory. + * S3: Added progress display for uploading and downloading. + * directory special remote: Made more efficient and robust. + * Bugfix: directory special remote could loop forever storing a key + when a too small chunksize was configured. + * Allow controlling whether login credentials for S3 and webdav are + committed to the repository, by setting embedcreds=yes|no when running + initremote. + * Added smallarchive repository group, that only archives files that are + in archive directories. Used by default for glacier when set up in the + webapp. + * assistant: Fixed handling of toplevel archive directory and + client repository group. + * assistant: Apply preferred content settings when a new symlink + is created, or a symlink gets renamed. Made archive directories work. + + -- Joey Hess Mon, 26 Nov 2012 11:37:49 -0400 + +git-annex (3.20121112) unstable; urgency=low + + * assistant: Can use XMPP to notify other nodes about pushes made to other + repositories, as well as pushing to them directly over XMPP. + * wepapp: Added an XMPP configuration interface. + * webapp: Supports pairing over XMPP, with both friends, and other repos + using the same account. + * assistant: Drops non-preferred content when possible. + * assistant: Notices, and applies config changes as they are made to + the git-annex branch, including config changes pushed in from remotes. + * git-annex-shell: GIT_ANNEX_SHELL_DIRECTORY can be set to limit it + to operating on a specified directory. + * webapp: When setting up authorized_keys, use GIT_ANNEX_SHELL_DIRECTORY. + * Preferred content path matching bugfix. + * Preferred content expressions cannot use "in=". + * Preferred content expressions can use "present". + * Fix handling of GIT_DIR when it refers to a git submodule. + * Depend on and use the Haskell SafeSemaphore library, which provides + exception-safe versions of SampleVar and QSemN. + Thanks, Ben Gamari for an excellent patch set. + * file:/// URLs can now be used with the web special remote. + * webapp: Allow dashes in ssh key comments when pairing. + * uninit: Check and abort if there are symlinks to annexed content that + are not checked into git. + * webapp: Switched to using the same multicast IP address that avahi uses. + * bup: Don't pass - to bup-split to make it read stdin; bup 0.25 + does not accept that. + * bugfix: Don't fail transferring content from read-only repos. + Closes: #691341 + * configure: Check that checksum programs produce correct checksums. + * Re-enable dbus, using a new version of the library that fixes the memory + leak. + * NetWatcher: When dbus connection is lost, try to reconnect. + * Use USER and HOME environment when set, and only fall back to getpwent, + which doesn't work with LDAP or NIS. + * rsync special remote: Include annex-rsync-options when running rsync + to test a key's presence. + * The standalone tarball's runshell now takes care of installing a + ~/.ssh/git-annex-shell wrapper the first time it's run. + * webapp: Make an initial, empty commit so there is a master branch + * assistant: Fix syncing local drives. + * webapp: Fix creation of rsync.net repositories. + * webapp: Fix renaming of special remotes. + * webapp: Generate better git remote names. + * webapp: Ensure that rsync special remotes are enabled using the same + name they were originally created using. + * Bugfix: Fix hang in webapp when setting up a ssh remote with an absolute + path. + + -- Joey Hess Mon, 12 Nov 2012 10:39:47 -0400 + +git-annex (3.20121017) unstable; urgency=low + + * Fix zombie cleanup reversion introduced in 3.20121009. + * Additional fix to support git submodules. + + -- Joey Hess Tue, 16 Oct 2012 21:10:14 -0400 + +git-annex (3.20121016) unstable; urgency=low + + * vicfg: New file format, avoids ambiguity with repos that have the same + description, or no description. + * Bug fix: A recent change caused git-annex-shell to crash. + * Better preferred content expression for transfer repos. + * webapp: Repository edit form can now edit the name of a repository. + * webapp: Make bare repositories on removable drives, as there is nothing + to ensure non-bare repos get updated when syncing. + * webapp: Better behavior when pausing syncing to a remote when a transfer + scan is running and queueing new transfers for that remote. + * The standalone binaries are now built to not use ssh connection caching, + in order to work with old versions of ssh. + * A relative core.worktree is relative to the gitdir. Now that this is + handled correctly, git-annex can be used in git submodules. + * Temporarily disable use of dbus, as the haskell dbus library blows up + when losing connection, which will need to be fixed upstream. + + -- Joey Hess Tue, 16 Oct 2012 15:25:22 -0400 + +git-annex (3.20121010) unstable; urgency=low + + * Renamed --ingroup to --inallgroup. + * Standard groups changed to client, transfer, archive, and backup. + Each of these has its own standard preferred content setting. + * dead: Remove dead repository from all groups. + * Avoid unsetting HOME when running certian git commands. Closes: #690193 + * test: Fix threaded runtime hang. + * Makefile: Avoid building with -threaded if the ghc threaded runtime does + not exist. + * webapp: Improve wording of intro display. Closes: #689848 + * webapp: Repositories can now be configured, to change their description, + their group, or even to disable syncing to them. + * git config remote.name.annex-sync can be used to control whether + a remote gets synced. + * Fix a crash when merging files in the git-annex branch that contain + invalid utf8. + * Automatically detect when a ssh remote does not have git-annex-shell + installed, and set annex-ignore. + + -- Joey Hess Fri, 12 Oct 2012 13:45:21 -0400 + +git-annex (3.20121009) unstable; urgency=low + + * watch, assistant: It's now safe to git annex unlock files while + the watcher is running, as well as modify files checked into git + as normal files. Additionally, .gitignore settings are now honored. + Closes: #689979 + * group, ungroup: New commands to indicate groups of repositories. + * webapp: Adds newly created repositories to one of these groups: + clients, drives, servers + * vicfg: New command, allows editing (or simply viewing) most + of the repository configuration settings stored in the git-annex branch. + * Added preferred content expressions, configurable using vicfg. + * get --auto: If the local repository has preferred content + configured, only get that content. + * drop --auto: If the repository the content is dropped from has + preferred content configured, drop only content that is not preferred. + * copy --auto: Only transfer content that the destination repository prefers. + * assistant: Now honors preferred content settings when deciding what to + transfer. + * --copies=group:number can now be used to match files that are present + in a specified number of repositories in a group. + * Added --smallerthan, --largerthan, and --inall limits. + * Only build-depend on libghc-clientsession-dev on arches that will have + the webapp. + * uninit: Unset annex.version. Closes: #689852 + + -- Joey Hess Tue, 09 Oct 2012 15:13:23 -0400 + +git-annex (3.20121001) unstable; urgency=low + + * fsck: Now has an incremental mode. Start a new incremental fsck pass + with git annex fsck --incremental. Now the fsck can be interrupted + as desired, and resumed with git annex fsck --more. + Thanks, Justin Azoff + * New --time-limit option, makes long git-annex commands stop after + a specified amount of time. + * fsck: New --incremental-schedule option which is nice for scheduling + eg, monthly incremental fsck runs in cron jobs. + * Fix fallback to ~/Desktop when xdg-user-dir is not available. + Closes: #688833 + * S3: When using a shared cipher, S3 credentials are not stored encrypted + in the git repository, as that would allow anyone with access to + the repository access to the S3 account. Instead, they're stored + in a 600 mode file in the local git repo. + * webapp: Avoid crashing when ssh-keygen -F chokes on an invalid known_hosts + file. + * Always do a system wide installation when DESTDIR is set. Closes: #689052 + * The Makefile now builds with the new yesod by default. + Systems like Debian that have the old yesod 1.0.1 should set + GIT_ANNEX_LOCAL_FEATURES=-DWITH_OLD_YESOD + * copy: Avoid updating the location log when no copy is performed. + * configure: Test that uuid -m works, falling back to plain uuid if not. + * Avoid building the webapp on Debian architectures that do not yet + have template haskell and thus yesod. (Should be available for arm soonish + I hope). + + -- Joey Hess Mon, 01 Oct 2012 13:56:55 -0400 + +git-annex (3.20120924) unstable; urgency=low + + * assistant: New command, a daemon which does everything watch does, + as well as automatically syncing file contents between repositories. + * webapp: An interface for managing and configuring the assistant. + * The default backend used when adding files to the annex is changed + from SHA256 to SHA256E, to simplify interoperability with OSX, media + players, and various programs that needlessly look at symlink targets. + To get old behavior, add a .gitattributes containing: * annex.backend=SHA256 + * init: If no description is provided for a new repository, one will + automatically be generated, like "joey@gnu:~/foo" + * test: Set a lot of git environment variables so testing works in strange + environments that normally need git config to set names, etc. + Closes: #682351 Thanks, gregor herrmann + * Disable ssh connection caching if the path to the control socket would be + too long (and use relative path to minimise path to the control socket). + * migrate: Check content before generating the new key, to avoid generating + a key for corrupt data. + * Support repositories created with --separate-git-dir. Closes: #684405 + * reinject: When the provided file doesn't match, leave it where it is, + rather than moving to .git/annex/bad/ + * Avoid crashing on encoding errors in filenames when writing transfer info + files and reading from checksum commands. + * sync: Pushes the git-annex branch to remote/synced/git-annex, rather + than directly to remote/git-annex. + * Now supports matching files that are present on a number of remotes + with a specified trust level. Example: --copies=trusted:2 + Thanks, Nicolas Pouillard + + -- Joey Hess Mon, 24 Sep 2012 13:47:48 -0400 + +git-annex (3.20120825) unstable; urgency=low + + * S3: Add fileprefix setting. + * Pass --use-agent to gpg when in no tty mode. Thanks, Eskild Hustvedt. + * Bugfix: Fix fsck in SHA*E backends, when the key contains composite + extensions, as added in 3.20120721. + + -- Joey Hess Sat, 25 Aug 2012 10:00:10 -0400 + +git-annex (3.20120807) unstable; urgency=low + + * initremote: Avoid recording remote's description before checking + that its config is valid. + * unused, status: Avoid crashing when ran in bare repo. + * Avoid crashing when "git annex get" fails to download from one + location, and falls back to downloading from a second location. + + -- Joey Hess Tue, 07 Aug 2012 13:35:07 -0400 + +git-annex (3.20120721) unstable; urgency=low + + * get, move, copy: Now refuse to do anything when the requested file + transfer is already in progress by another process. + * status: Lists transfers that are currently in progress. + * Fix passing --uuid to git-annex-shell. + * When shaNsum commands cannot be found, use the Haskell SHA library + (already a dependency) to do the checksumming. This may be slower, + but avoids portability problems. + * Use SHA library for files less than 50 kb in size, at which point it's + faster than forking the more optimised external program. + * SHAnE backends are now smarter about composite extensions, such as + .tar.gz Closes: #680450 + * map: Write map.dot to .git/annex, which avoids watch trying to annex it. + + -- Joey Hess Sat, 21 Jul 2012 16:52:48 -0400 + +git-annex (3.20120629) unstable; urgency=low + + * cabal: Only try to use inotify on Linux. + * Version build dependency on STM, and allow building without it, + which disables the watch command. + * Avoid ugly failure mode when moving content from a local repository + that is not available. + * Got rid of the last place that did utf8 decoding. + * Accept arbitrarily encoded repository filepaths etc when reading + git config output. This fixes support for remotes with unusual characters + in their names. + * sync: Automatically resolves merge conflicts. + + -- Joey Hess Fri, 29 Jun 2012 10:17:49 -0400 + +git-annex (3.20120624) unstable; urgency=low + + * watch: New subcommand, a daemon which notices changes to + files and automatically annexes new files, etc, so you don't + need to manually run git commands when manipulating files. + Available on Linux, BSDs, and OSX! + * Enable diskfree on kfreebsd, using kqueue. + * unused: Fix crash when key names contain invalid utf8. + * sync: Avoid recent git's interactive merge. + + -- Joey Hess Sun, 24 Jun 2012 12:36:50 -0400 + +git-annex (3.20120614) unstable; urgency=medium + + * addurl: Was broken by a typo introduced 2 released ago, now fixed. + Closes: #677576 + * Install man page when run by cabal, in a location where man will + find it, even when installing under $HOME. Thanks, Nathan Collins + + -- Joey Hess Thu, 14 Jun 2012 20:21:29 -0400 + +git-annex (3.20120611) unstable; urgency=medium + + * add: Prevent (most) modifications from being made to a file while it + is being added to the annex. + * initremote: Automatically describe a remote when creating it. + * uninit: Refuse to run in a subdirectory. Closes: #677076 + + -- Joey Hess Mon, 11 Jun 2012 10:32:01 -0400 + +git-annex (3.20120605) unstable; urgency=low + + * sync: Show a nicer message if a user tries to sync to a special remote. + * lock: Reset unlocked file to index, rather than to branch head. + * import: New subcommand, pulls files from a directory outside the annex + and adds them. + * Fix display of warning message when encountering a file that uses an + unsupported backend. + * Require that the SHA256 backend can be used when building, since it's the + default. + * Preserve parent environment when running hooks of the hook special remote. + + -- Joey Hess Tue, 05 Jun 2012 14:03:39 -0400 + +git-annex (3.20120522) unstable; urgency=low + + * Pass -a to cp even when it supports --reflink=auto, to preserve + permissions. + * Clean up handling of git directory and git worktree. + * Add support for core.worktree, and fix support for GIT_WORK_TREE and + GIT_DIR. + + -- Joey Hess Tue, 22 May 2012 11:16:13 -0400 + +git-annex (3.20120511) unstable; urgency=low + + * Rsync special remotes can be configured with shellescape=no + to avoid shell quoting that is normally done when using rsync over ssh. + This is known to be needed for certian rsync hosting providers + (specificially hidrive.strato.com) that use rsync over ssh but do not + pass it through the shell. + * dropunused: Allow specifying ranges to drop. + * addunused: New command, the opposite of dropunused, it relinks unused + content into the git repository. + * Fix use of several config settings: annex.ssh-options, + annex.rsync-options, annex.bup-split-options. (And adjust types to avoid + the bugs that broke several config settings.) + + -- Joey Hess Fri, 11 May 2012 12:29:30 -0400 + +git-annex (3.20120430) unstable; urgency=low + + * Fix use of annex.diskreserve config setting. + * Directory special remotes now check annex.diskreserve. + * Support git's core.sharedRepository configuration. + * Add annex.http-headers and annex.http-headers-command config + settings, to allow custom headers to be sent with all HTTP requests. + (Requested by the Internet Archive) + * uninit: Clear annex.uuid from .git/config. Closes: #670639 + * Added shared cipher mode to encryptable special remotes. This option + avoids gpg key distribution, at the expense of flexibility, and with + the requirement that all clones of the git repository be equally trusted. + + -- Joey Hess Mon, 30 Apr 2012 13:16:10 -0400 + +git-annex (3.20120418) unstable; urgency=low + + * bugfix: Adding a dotfile also caused all non-dotfiles to be added. + * bup: Properly handle key names with spaces or other things that are + not legal git refs. + * git-annex (but not git-annex-shell) supports the git help.autocorrect + configuration setting, doing fuzzy matching using the restricted + Damerau-Levenshtein edit distance, just as git does. This adds a build + dependency on the haskell edit-distance library. + * Renamed diskfree.c to avoid OSX case insensativity bug. + * cabal now installs git-annex-shell as a symlink to git-annex. + * cabal file now autodetects whether S3 support is available. + + -- Joey Hess Wed, 18 Apr 2012 12:11:32 -0400 + +git-annex (3.20120406) unstable; urgency=low + + * Disable diskfree on kfreebsd, as I have a build failure on kfreebsd-i386 + that is quite likely caused by it. + + -- Joey Hess Sat, 07 Apr 2012 15:50:36 -0400 + +git-annex (3.20120405) unstable; urgency=low + + * Rewrote free disk space checking code, moving the portability + handling into a small C library. + * status: Display amount of free disk space. + + -- Joey Hess Thu, 05 Apr 2012 16:19:10 -0400 + +git-annex (3.20120315) unstable; urgency=low + + * fsck: Fix up any broken links and misplaced content caused by the + directory hash calculation bug fixed in the last release. + * sync: Sync to lower cost remotes first. + * status: Fixed to run in constant space. + * status: More accurate display of sizes of tmp and bad keys. + * unused: Now uses a bloom filter, and runs in constant space. + Use of a bloom filter does mean it will not notice a small + number of unused keys. For repos with up to half a million keys, + it will miss one key in 1000. + * Added annex.bloomcapacity and annex.bloomaccuracy, which can be + adjusted as desired to tune the bloom filter. + * status: Display amount of memory used by bloom filter, and + detect when it's too small for the number of keys in a repository. + * git-annex-shell: Runs hooks/annex-content after content is received + or dropped. + * Work around a bug in rsync (IMHO) introduced by openSUSE's SIP patch. + * git-annex now behaves as git-annex-shell if symlinked to and run by that + name. The Makefile sets this up, saving some 8 mb of installed size. + * git-union-merge is a demo program, so it is no longer built by default. + + -- Joey Hess Thu, 15 Mar 2012 11:05:28 -0400 + +git-annex (3.20120309) unstable; urgency=low + + * Fix key directory hash calculation code to behave as it did before + version 3.20120227 when a key contains non-ascii characters (only + WORM backend is likely to have been affected). + + -- Joey Hess Fri, 09 Mar 2012 20:05:09 -0400 + +git-annex (3.20120230) unstable; urgency=low + + * "here" can be used to refer to the current repository, + which can read better than the old "." (which still works too). + * Directory special remotes now support chunking files written to them, + avoiding writing files larger than a specified size. + * Add progress bar display to the directory special remote. + * Add configurable hooks that are run when git-annex starts and stops + using a remote: remote.name.annex-start-command and + remote.name.annex-stop-command + * Fix a bug in symlink calculation code, that triggered in rare + cases where an annexed file is in a subdirectory that nearly + matched to the .git/annex/object/xx/yy subdirectories. + + -- Joey Hess Mon, 05 Mar 2012 13:38:13 -0400 + +git-annex (3.20120229) unstable; urgency=low + + * Fix test suite to not require a unicode locale. + * Fix cabal build failure. Thanks, Sergei Trofimovich + + -- Joey Hess Wed, 29 Feb 2012 02:31:31 -0400 + +git-annex (3.20120227) unstable; urgency=low + + * Modifications to support ghc 7.4's handling of filenames. + This version can only be built with ghc 7.4 or newer. See the ghc7.0 + branch for older ghcs. + * S3: Fix irrefutable pattern failure when accessing encrypted S3 + credentials. + * Use the haskell IfElse library. + * Fix teardown of stale cached ssh connections. + * Fixed to use the strict state monad, to avoid leaking all kinds of memory + due to lazy state update thunks when adding/fixing many files. + * Fixed some memory leaks that occurred when committing journal files. + * Added a annex.queuesize setting, useful when adding hundreds of thousands + of files on a system with plenty of memory. + * whereis: Prints the urls of files that the web special remote knows about. + * addurl --fast: Verifies that the url can be downloaded (only getting + its head), and records the size in the key. + * When checking that an url has a key, verify that the Content-Length, + if available, matches the size of the key. + * addurl: Added a --file option, which can be used to specify what + file the url is added to. This can be used to override the default + filename that is used when adding an url, which is based on the url. + Or, when the file already exists, the url is recorded as another + location of the file. + * addurl: Normalize badly encoded urls. + * addurl: Add --pathdepth option. + * rekey: New plumbing level command, can be used to change the keys used + for files en masse. + * Store web special remote url info in a more efficient location. + (Urls stored with this version will not be visible to older versions.) + * Deal with NFS problem that caused a failure to remove a directory + when removing content from the annex. + * Make a single location log commit after a remote has received or + dropped files. Uses a new "git-annex-shell commit" command when available. + * To avoid commits of data to the git-annex branch after each command + is run, set annex.alwayscommit=false. Its data will then be committed + less frequently, when a merge or sync is done. + * configure: Check if ssh connection caching is supported by the installed + version of ssh and default annex.sshcaching accordingly. + * move --from, copy --from: Now 10 times faster when scanning to find + files in a remote on a local disk; rather than go through the location log + to see which files are present on the remote, it simply looks at the + disk contents directly. + + -- Joey Hess Mon, 27 Feb 2012 12:58:21 -0400 + +git-annex (3.20120123) unstable; urgency=low + + * fsck --from: Fscking a remote is now supported. It's done by retrieving + the contents of the specified files from the remote, and checking them, + so can be an expensive operation. Still, if the remote is a special + remote, or a git repository that you cannot run fsck in locally, it's + nice to have the ability to fsck it. + * If you have any directory special remotes, now would be a good time to + fsck them, in case you were hit by the data loss bug fixed in the + previous release! + * fsck --from remote --fast: Avoids expensive file transfers, at the + expense of not checking file size and/or contents. + * Ssh connection caching is now enabled automatically by git-annex. + Only one ssh connection is made to each host per git-annex run, which + can speed some things up a lot, as well as avoiding repeated password + prompts. Concurrent git-annex processes also share ssh connections. + Cached ssh connections are shut down when git-annex exits. + * To disable the ssh caching (if for example you have your own broader + ssh caching configuration), set annex.sshcaching=false. + + -- Joey Hess Mon, 23 Jan 2012 13:48:48 -0400 + +git-annex (3.20120116) unstable; urgency=medium + + * Fix data loss bug in directory special remote, when moving a file + to the remote failed, and partially transferred content was left + behind in the directory, re-running the same move would think it + succeeded and delete the local copy. + + -- Joey Hess Mon, 16 Jan 2012 16:43:45 -0400 + +git-annex (3.20120115) unstable; urgency=low + + * Add a sanity check for bad StatFS results. On architectures + where StatFS does not currently work (s390, mips, powerpc, sparc), + this disables the diskreserve checking code, and attempting to + configure an annex.diskreserve will result in an error. + * Fix QuickCheck dependency in cabal file. + * Minor optimisations. + + -- Joey Hess Sun, 15 Jan 2012 13:54:20 -0400 + +git-annex (3.20120113) unstable; urgency=low + + * log: Add --gource mode, which generates output usable by gource. + * map: Fix display of remote repos + * Add annex-trustlevel configuration settings, which can be used to + override the trust level of a remote. + * git-annex, git-union-merge: Support GIT_DIR and GIT_WORK_TREE. + * Add libghc-testpack-dev to build depends on all arches. + + -- Joey Hess Fri, 13 Jan 2012 15:35:17 -0400 + +git-annex (3.20120106) unstable; urgency=low + + * Support unescaped repository urls, like git does. + * log: New command that displays the location log for files, + showing each repository they were added to and removed from. + * Fix overbroad gpg --no-tty fix from last release. + + -- Joey Hess Sat, 07 Jan 2012 13:16:23 -0400 + +git-annex (3.20120105) unstable; urgency=low + + * Added annex-web-options configuration settings, which can be + used to provide parameters to whichever of wget or curl git-annex uses + (depends on which is available, but most of their important options + suitable for use here are the same). + * Dotfiles, and files inside dotdirs are not added by "git annex add" + unless the dotfile or directory is explicitly listed. So "git annex add ." + will add all untracked files in the current directory except for those in + dotdirs. + * Added quickcheck to build dependencies, and fail if test suite cannot be + built. + * fsck: Do backend-specific check before checking numcopies is satisfied. + * Run gpg with --no-tty. Closes: #654721 + + -- Joey Hess Thu, 05 Jan 2012 13:44:12 -0400 + +git-annex (3.20111231) unstable; urgency=low + + * sync: Improved to work well without a central bare repository. + Thanks to Joachim Breitner. + * Rather than manually committing, pushing, pulling, merging, and git annex + merging, we encourage you to give "git annex sync" a try. + * sync --fast: Selects some of the remotes with the lowest annex.cost + and syncs those, in addition to any specified at the command line. + * Union merge now finds the least expensive way to represent the merge. + * reinject: Add a sanity check for using an annexed file as the source file. + * Properly handle multiline git config values. + * Fix the hook special remote, which bitrotted a while ago. + * map: --fast disables use of dot to display map + * Test suite improvements. Current top-level test coverage: 75% + * Improve deletion of files from rsync special remotes. Closes: #652849 + * Add --include, which is the same as --not --exclude. + * Format strings can be specified using the new --format option, to control + what is output by git annex find. + * Support git annex find --json + * Fixed behavior when multiple insteadOf configs are provided for the + same url base. + * Can now be built with older git versions (before 1.7.7); the resulting + binary should only be used with old git. + * Updated to build with monad-control 0.3. + + -- Joey Hess Sat, 31 Dec 2011 14:55:29 -0400 + +git-annex (3.20111211) unstable; urgency=medium + + * Fix bug in last version in getting contents from bare repositories. + * Ensure that git-annex branch changes are merged into git-annex's index, + which fixes a bug that could cause changes that were pushed to the + git-annex branch to get reverted. As a side effect, it's now safe + for users to check out and commit changes directly to the git-annex + branch. + * map: Fix a failure to detect a loop when both repositories are local + and refer to each other with relative paths. + * Prevent key names from containing newlines. + * add: If interrupted, add can leave files converted to symlinks but not + yet added to git. Running the add again will now clean up this situtation. + * Fix caching of decrypted ciphers, which failed when drop had to check + multiple different encrypted special remotes. + * unannex: Can be run on files that have been added to the annex, but not + yet committed. + * sync: New command that synchronises the local repository and default + remote, by running git commit, pull, and push for you. + * Version monad-control dependency in cabal file. + + -- Joey Hess Sun, 11 Dec 2011 21:24:39 -0400 + +git-annex (3.20111203) unstable; urgency=low + + * The VFAT filesystem on recent versions of Linux, when mounted with + shortname=mixed, does not get along well with git-annex's mixed case + .git/annex/objects hash directories. To avoid this problem, new content + is now stored in all-lowercase hash directories. Except for non-bare + repositories which would be a pain to transition and cannot be put on FAT. + (Old mixed-case hash directories are still tried for backwards + compatibility.) + * Flush json output, avoiding a buffering problem that could result in + doubled output. + * Avoid needing haskell98 and other fixes for new ghc. Thanks, Mark Wright. + * Bugfix: dropunused did not drop keys with two spaces in their name. + * Support for storing .git/annex on a different device than the rest of the + git repository. + * --inbackend can be used to make git-annex only operate on files + whose content is stored using a specified key-value backend. + * dead: A command which says that a repository is gone for good + and you don't want git-annex to mention it again. + + -- Joey Hess Sat, 03 Dec 2011 21:01:45 -0400 + +git-annex (3.20111122) unstable; urgency=low + + * merge: Improve commit messages to mention what was merged. + * Avoid doing auto-merging in commands that don't need fully current + information from the git-annex branch. In particular, git annex add + no longer needs to auto-merge. + * init: When run in an already initalized repository, and without + a description specified, don't delete the old description. + * Optimised union merging; now only runs git cat-file once, and runs + in constant space. + * status: Now displays trusted, untrusted, and semitrusted repositories + separately. + * status: Include all special remotes in the list of repositories. + * status: Fix --json mode. + * status: --fast is back + * Fix support for insteadOf url remapping. Closes: #644278 + * When not run in a git repository, git-annex can still display a usage + message, and "git annex version" even works. + * migrate: Don't fall over a stale temp file. + * Avoid excessive escaping for rsync special remotes that are not accessed + over ssh. + * find: Support --print0 + + -- Joey Hess Tue, 22 Nov 2011 14:31:45 -0400 + +git-annex (3.20111111) unstable; urgency=low + + * Handle a case where an annexed file is moved into a gitignored directory, + by having fix --force add its change. + * Avoid cyclic drop problems. + * Optimized copy --from and get --from to avoid checking the location log + for files that are already present. + * Automatically fix up badly formatted uuid.log entries produced by + 3.20111105, whenever the uuid.log is changed (ie, by init or describe). + * map: Support remotes with /~/ and /~user/ + + -- Joey Hess Fri, 11 Nov 2011 13:44:18 -0400 + +git-annex (3.20111107) unstable; urgency=low + + * merge: Use fast-forward merges when possible. + Thanks Valentin Haenel for a test case showing how non-fast-forward + merges could result in an ongoing pull/merge/push cycle. + * Don't try to read config from repos with annex-ignore set. + * Bugfix: In the past two releases, git-annex init has written the uuid.log + in the wrong format, with the UUID and description flipped. + + -- Joey Hess Mon, 07 Nov 2011 12:47:44 -0400 + +git-annex (3.20111105) unstable; urgency=low + + * The default backend used when adding files to the annex is changed + from WORM to SHA256. + To get old behavior, add a .gitattributes containing: * annex.backend=WORM + * Sped up some operations on remotes that are on the same host. + * copy --to: Fixed leak when copying many files to a remote on the same + host. + * uninit: Add guard against being run with the git-annex branch checked out. + * Fail if --from or --to is passed to commands that do not support them. + * drop --from is now supported to remove file content from a remote. + * status: Now always shows the current repository, even when it does not + appear in uuid.log. + * fsck: Now works in bare repositories. Checks location log information, + and file contents. Does not check that numcopies is satisfied, as + .gitattributes information about numcopies is not available in a bare + repository. + * unused, dropunused: Now work in bare repositories. + * Removed the setkey command, and added a reinject command with a more + useful interface. + * The fromkey command now takes the key as its first parameter. The --key + option is no longer used. + * Built without any filename containing .git being excluded. Closes: #647215 + * Record uuid when auto-initializing a remote so it shows in status. + * Bugfix: Fixed git-annex init crash in a bare repository when there was + already an existing git-annex branch. + * Pass -t to rsync to preserve timestamps. + + -- Joey Hess Sat, 05 Nov 2011 15:47:52 -0400 + +git-annex (3.20111025) unstable; urgency=low + + * A remote can have a annexUrl configured, that is used by git-annex + instead of its usual url. (Similar to pushUrl.) + * migrate: Copy url logs for keys when migrating. + * git-annex-shell: GIT_ANNEX_SHELL_READONLY and GIT_ANNEX_SHELL_LIMITED + environment variables can be set to limit what commands can be run. + This is used by gitolite's new git-annex support! + + -- Joey Hess Tue, 25 Oct 2011 13:03:08 -0700 + +git-annex (3.20111011) unstable; urgency=low + + * This version of git-annex only works with git 1.7.7 and newer. + The breakage with old versions is subtle, and affects the + annex.numcopies settings in .gitattributes, so be sure to upgrade git + to 1.7.7. (Debian package now depends on that version.) + * Don't pass absolute paths to git show-attr, as it started following + symlinks when that's done in 1.7.7. Instead, use relative paths, + which show-attr only handles 100% correctly in 1.7.7. Closes: #645046 + * Fix referring to remotes by uuid. + * New or changed repository descriptions in uuid.log now have a timestamp, + which is used to ensure the newest description is used when the uuid.log + has been merged. + * Note that older versions of git-annex will display the timestamp as part + of the repository description, which is ugly but otherwise harmless. + * Add timestamps to trust.log and remote.log too. + * git-annex-shell: Added the --uuid option. + * git-annex now asks git-annex-shell to verify that it's operating in + the expected repository. + * Note that this git-annex will not interoperate with remotes using + older versions of git-annex-shell. + * Now supports git's insteadOf configuration, to modify the url + used to access a remote. Note that pushInsteadOf is not used; + that and pushurl are reserved for actual git pushes. Closes: #644278 + * status: List all known repositories. + * When displaying a list of repositories, show git remote names + in addition to their descriptions. + * Add locking to avoid races when changing the git-annex branch. + * Various speed improvements gained by using ByteStrings. + * Contain the zombie hordes. + + -- Joey Hess Tue, 11 Oct 2011 23:00:02 -0400 + +git-annex (3.20110928) unstable; urgency=low + + * --in can be used to make git-annex only operate on files + believed to be present in a given repository. + * Arbitrarily complex expressions can be built to limit the files git-annex + operates on, by combining the options --not --and --or -( and -) + Example: git annex get --exclude '*.mp3' --and --not -( --in usbdrive --or --in archive -) + * --copies=N can be used to make git-annex only operate on files with + the specified number of copies. (And --not --copies=N for the inverse.) + * find: Rather than only showing files whose contents are present, + when used with --exclude --copies or --in, displays all files that + match the specified conditions. + * Note that this is a behavior change for git-annex find! Old behavior + can be gotten by using: git-annex find --in . + * status: Massively sped up; remove --fast mode. + * unused: File contents used by branches and tags are no longer + considered unused, even when not used by the current branch. This is + the final piece of the puzzle needed for git-annex to to play nicely + with branches. + + -- Joey Hess Wed, 28 Sep 2011 18:14:02 -0400 + +git-annex (3.20110915) unstable; urgency=low + + * whereis: Show untrusted locations separately and do not include in + location count. + * Fix build without S3. + * addurl: Always use whole url as destination filename, rather than + only its file component. + * get, drop, copy: Added --auto option, which decides whether + to get/drop content as needed to work toward the configured numcopies. + * bugfix: drop and fsck did not honor --exclude + + -- Joey Hess Thu, 15 Sep 2011 22:25:46 -0400 + +git-annex (3.20110906) unstable; urgency=low + + * Improve display of newlines around error and warning messages. + * Fix Makefile to work with cabal again. + + -- Joey Hess Tue, 06 Sep 2011 13:45:16 -0400 + +git-annex (3.20110902) unstable; urgency=low + + * Set EMAIL when running test suite so that git does not need to be + configured first. Closes: #638998 + * The wget command will now be used in preference to curl, if available. + * init: Make description an optional parameter. + * unused, status: Sped up by avoiding unnecessary stats of annexed files. + * unused --remote: Reduced memory use to 1/4th what was used before. + * Add --json switch, to produce machine-consumable output. + + -- Joey Hess Fri, 02 Sep 2011 21:20:37 -0400 + +git-annex (3.20110819) unstable; urgency=low + + * Now "git annex init" only has to be run once, when a git repository + is first being created. Clones will automatically notice that git-annex + is in use and automatically perform a basic initalization. It's + still recommended to run "git annex init" in any clones, to describe them. + * Added annex-cost-command configuration, which can be used to vary the + cost of a remote based on the output of a shell command. + * Fix broken upgrade from V1 repository. Closes: #638584 + + -- Joey Hess Fri, 19 Aug 2011 20:34:09 -0400 + +git-annex (3.20110817) unstable; urgency=low + + * Fix shell escaping in rsync special remote. + * addurl: --fast can be used to avoid immediately downloading the url. + * Added support for getting content from git remotes using http (and https). + * Added curl to Debian package dependencies. + + -- Joey Hess Wed, 17 Aug 2011 01:29:02 -0400 + +git-annex (3.20110719) unstable; urgency=low + + * add: Be even more robust to avoid ever leaving the file seemingly deleted. + Closes: #634233 + * Bugfix: Make add ../ work. + * Support the standard git -c name=value + * unannex: Clean up use of git commit -a. + + -- Joey Hess Tue, 19 Jul 2011 23:39:53 -0400 + +git-annex (3.20110707) unstable; urgency=low + + * Fix sign bug in disk free space checking. + * Bugfix: Forgot to de-escape keys when upgrading. Could result in + bad location log data for keys that contain [&:%] in their names. + (A workaround for this problem is to run git annex fsck.) + * add: Avoid a failure mode that resulted in the file seemingly being + deleted (content put in the annex but no symlink present). + + -- Joey Hess Thu, 07 Jul 2011 19:29:39 -0400 + +git-annex (3.20110705) unstable; urgency=low + + * uninit: Delete the git-annex branch and .git/annex/ + * unannex: In --fast mode, file content is left in the annex, and a + hard link made to it. + * uninit: Use unannex in --fast mode, to support unannexing multiple + files that link to the same content. + * Drop the dependency on the haskell curl bindings, use regular haskell HTTP. + * Fix a pipeline stall when upgrading (caused by #624389). + + -- Joey Hess Tue, 05 Jul 2011 14:37:39 -0400 + +git-annex (3.20110702) unstable; urgency=low + + * Now the web can be used as a special remote. + This feature replaces the old URL backend. + * addurl: New command to download an url and store it in the annex. + * Sped back up fsck, copy --from, and other commands that often + have to read a lot of information from the git-annex branch. Such + commands are now faster than they were before introduction of the + git-annex branch. + * Always ensure git-annex branch exists. + * Modify location log parser to allow future expansion. + * --force will cause add, etc, to operate on ignored files. + * Avoid mangling encoding when storing the description of repository + and other content. + * cabal can now be used to build git-annex. This is substantially + slower than using make, does not build or install documentation, + does not run the test suite, and is not particularly recommended, + but could be useful to some. + + -- Joey Hess Sat, 02 Jul 2011 15:00:18 -0400 + +git-annex (3.20110624) experimental; urgency=low + + * New repository format, annex.version=3. Use `git annex upgrade` to migrate. + * git-annex now stores its logs in a git-annex branch. + * merge: New subcommand. Auto-merges the new git-annex branch. + * Improved handling of bare git repos with annexes. Many more commands will + work in them. + * git-annex is now more robust; it will never leave state files + uncommitted when some other git process comes along and locks the index + at an inconvenient time. + * rsync is now used when copying files from repos on other filesystems. + cp is still used when copying file from repos on the same filesystem, + since --reflink=auto can make it significantly faster on filesystems + such as btrfs. + * Allow --trust etc to specify a repository by name, for temporarily + trusting repositories that are not configured remotes. + * unlock: Made atomic. + * git-union-merge: New git subcommand, that does a generic union merge + operation, and operates efficiently without touching the working tree. + + -- Joey Hess Fri, 24 Jun 2011 14:32:18 -0400 + +git-annex (0.20110610) unstable; urgency=low + + * Add --numcopies option. + * Add --trust, --untrust, and --semitrust options. + * get --from is the same as copy --from + * Bugfix: Fix fsck to not think all SHAnE keys are bad. + + -- Joey Hess Fri, 10 Jun 2011 11:48:40 -0400 + +git-annex (0.20110601) unstable; urgency=low + + * Minor bugfixes and error message improvements. + * Massively sped up `git annex lock` by avoiding use of the uber-slow + `git reset`, and only running `git checkout` once, even when many files + are being locked. + * Fix locking of files with staged changes. + * Somewhat sped up `git commit` of modifications to unlocked files. + * Build fix for older ghc. + + -- Joey Hess Wed, 01 Jun 2011 11:50:47 -0400 + +git-annex (0.20110522) unstable; urgency=low + + * Closer emulation of git's behavior when told to use "foo/.git" as a + git repository instead of just "foo". Closes: #627563 + * Fix bug in --exclude introduced in 0.20110516. + + -- Joey Hess Fri, 27 May 2011 20:20:41 -0400 + +git-annex (0.20110521) unstable; urgency=low + + * status: New subcommand to show info about an annex, including its size. + * --backend now overrides any backend configured in .gitattributes files. + * Add --debug option. Closes: #627499 + + -- Joey Hess Sat, 21 May 2011 11:52:53 -0400 + +git-annex (0.20110516) unstable; urgency=low + + * Add a few tweaks to make it easy to use the Internet Archive's variant + of S3. In particular, munge key filenames to comply with the IA's filename + limits, disable encryption, support their nonstandard way of creating + buckets, and allow x-archive-* headers to be specified in initremote to + set item metadata. + * Added filename extension preserving variant backends SHA1E, SHA256E, etc. + * migrate: Use current filename when generating new key, for backends + where the filename affects the key name. + * Work around a bug in Network.URI's handling of bracketed ipv6 addresses. + + -- Joey Hess Mon, 16 May 2011 14:16:52 -0400 + +git-annex (0.20110503) unstable; urgency=low + + * Fix hasKeyCheap setting for bup and rsync special remotes. + * Add hook special remotes. + * Avoid crashing when an existing key is readded to the annex. + * unused: Now also lists files fsck places in .git/annex/bad/ + * S3: When encryption is enabled, the Amazon S3 login credentials + are stored, encrypted, in .git-annex/remotes.log, so environment + variables need not be set after the remote is initialized. + + -- Joey Hess Tue, 03 May 2011 20:56:01 -0400 + +git-annex (0.20110427) unstable; urgency=low + + * Switch back to haskell SHA library, so git-annex remains buildable on + Debian stable. + * Added rsync special remotes. This could be used, for example, to + store annexed content on rsync.net (encrypted naturally). Or anywhere else. + * Bugfix: Avoid pipeline stall when running git annex drop or fsck on a + lot of files. Possibly only occured with ghc 7. + + -- Joey Hess Wed, 27 Apr 2011 22:50:26 -0400 + +git-annex (0.20110425) unstable; urgency=low + + * Use haskell Crypto library instead of haskell SHA library. + * Remove testpack from build depends for non x86 architectures where it + is not available. The test suite will not be run if it cannot be compiled. + * Avoid using absolute paths when staging location log, as that can + confuse git when a remote's path contains a symlink. Closes: #621386 + + -- Joey Hess Mon, 25 Apr 2011 15:47:00 -0400 + +git-annex (0.20110420) unstable; urgency=low + + * Update Debian build dependencies for ghc 7. + * Debian package is now built with S3 support. + Thanks Joachim Breitner for making this possible. + * Somewhat improved memory usage of S3, still work to do. + Thanks Greg Heartsfield for ongoing work to improve the hS3 library + for git-annex. + + -- Joey Hess Thu, 21 Apr 2011 15:00:48 -0400 + +git-annex (0.20110419) unstable; urgency=low + + * Don't run gpg in batch mode, so it can prompt for passphrase when + there is no agent. + * Add missing build dep on dataenc. + * S3: Fix stalls when transferring encrypted data. + * bup: Avoid memory leak when transferring encrypted data. + + -- Joey Hess Tue, 19 Apr 2011 21:26:51 -0400 + +git-annex (0.20110417) unstable; urgency=low + + * bup is now supported as a special type of remote. + * The data sent to special remotes (Amazon S3, bup, etc) can be encrypted + using GPG for privacy. + * Use lowercase hash directories for locationlog files, to avoid + some issues with git on OSX with the mixed-case directories. + No migration is needed; the old mixed case hash directories are still + read; new information is written to the new directories. + * Unused files on remotes, particulary special remotes, can now be + identified and dropped, by using "--from remote" with git annex unused + and git annex dropunused. + * Clear up short option confusion between --from and --force (-f is now + --from, and there is no short option for --force). + * Add build depend on perlmagick so docs are consistently built. + Closes: #621410 + * Add doc-base file. Closes: #621408 + * Periodically flush git command queue, to avoid boating memory usage + too much. + * Support "sha1" and "sha512" commands on FreeBSD, and allow building + if any/all SHA commands are not available. Thanks, Fraser Tweedale + + -- Joey Hess Sun, 17 Apr 2011 12:00:24 -0400 + +git-annex (0.20110401) experimental; urgency=low + + * Amazon S3 is now supported as a special type of remote. + Warning: Encrypting data before sending it to S3 is not yet supported. + * Note that Amazon S3 support is not built in by default on Debian yet, + as hS3 is not packaged. + * fsck: Ensure that files and directories in .git/annex/objects + have proper permissions. + * Added a special type of remote called a directory remote, which + simply stores files in an arbitrary local directory. + * Bugfix: copy --to --fast never really copied, fixed. + + -- Joey Hess Fri, 01 Apr 2011 21:27:22 -0400 + +git-annex (0.20110328) experimental; urgency=low + + * annex.diskreserve can be given in arbitrary units (ie "0.5 gigabytes") + * Generalized remotes handling, laying groundwork for remotes that are + not regular git remotes. (Think Amazon S3.) + * Provide a less expensive version of `git annex copy --to`, enabled + via --fast. This assumes that location tracking information is correct, + rather than contacting the remote for every file. + * Bugfix: Keys could be received into v1 annexes from v2 annexes, via + v1 git-annex-shell. This results in some oddly named keys in the v1 + annex. Recognise and fix those keys when upgrading, instead of crashing. + + -- Joey Hess Mon, 28 Mar 2011 10:47:29 -0400 + +git-annex (0.20110325) experimental; urgency=low + + * Free space checking is now done, for transfers of data for keys + that have free space metadata. (Notably, not for SHA* keys generated + with git-annex 0.2x or earlier.) The code is believed to work on + Linux, FreeBSD, and OSX; check compile-time messages to see if it + is not enabled for your OS. + * Add annex.diskreserve config setting, to control how much free space + to reserve for other purposes and avoid using (defaults to 1 mb). + * Add --fast flag, that can enable less expensive, but also less thorough + versions of some commands. + * fsck: In fast mode, avoid checking checksums. + * unused: In fast mode, just show all existing temp files as unused, + and avoid expensive scan for other unused content. + * migrate: Support migrating v1 SHA keys to v2 SHA keys with + size information that can be used for free space checking. + * Fix space leak in fsck and drop commands. + * migrate: Bugfix for case when migrating a file results in a key that + is already present in .git/annex/objects. + * dropunused: Significantly sped up; only read unused log file once. + + -- Joey Hess Fri, 25 Mar 2011 00:47:37 -0400 + +git-annex (0.20110320) experimental; urgency=low + + * Fix dropping of files using the URL backend. + * Fix support for remotes with '.' in their names. + * Add version command to show git-annex version as well as repository + version information. + * No longer auto-upgrade to repository format 2, to avoid accidental + upgrades, etc. Use git-annex upgrade when you're ready to run this + version. + + -- Joey Hess Sun, 20 Mar 2011 16:36:33 -0400 + +git-annex (0.20110316) experimental; urgency=low + + * New repository format, annex.version=2. + * The first time git-annex is run in an old format repository, it + will automatically upgrade it to the new format, staging all + necessary changes to git. Also added a "git annex upgrade" command. + * Colons are now avoided in filenames, so bare clones of git repos + can be put on USB thumb drives formatted with vFAT or similar + filesystems. + * Added two levels of hashing to object directory and .git-annex logs, + to improve scalability with enormous numbers of annexed + objects. (With one hundred million annexed objects, each + directory would contain fewer than 1024 files.) + * The setkey, fromkey, and dropkey subcommands have changed how + the key is specified. --backend is no longer used with these. + + -- Joey Hess Wed, 16 Mar 2011 16:20:23 -0400 + +git-annex (0.24) unstable; urgency=low + + Branched the 0.24 series, which will be maintained for a while to + support v1 git-annex repos, while main development moves to the 0.2011 + series, with v2 git-annex repos. + + * Add Suggests on graphviz. Closes: #618039 + * When adding files to the annex, the symlinks pointing at the annexed + content are made to have the same mtime as the original file. + While git does not preserve that information, this allows a tool + like metastore to be used with annexed files. + (Currently this is only done on systems supporting POSIX 200809.) + + -- Joey Hess Wed, 16 Mar 2011 18:35:13 -0400 + +git-annex (0.23) unstable; urgency=low + + * Support ssh remotes with a port specified. + * whereis: New subcommand to show where a file's content has gotten to. + * Rethink filename encoding handling for display. Since filename encoding + may or may not match locale settings, any attempt to decode filenames + will fail for some files. So instead, do all output in binary mode. + + -- Joey Hess Sat, 12 Mar 2011 15:02:49 -0400 + +git-annex (0.22) unstable; urgency=low + + * Git annexes can now be attached to bare git repositories. + (Both the local and remote host must have this version of git-annex + installed for it to work.) + * Support filenames that start with a dash; when such a file is passed + to a utility it will be escaped to avoid it being interpreted as an + option. (I went a little overboard and got the type checker involved + in this, so such files are rather comprehensively supported now.) + * New backends: SHA512 SHA384 SHA256 SHA224 + (Supported on systems where corresponding shaNsum commands are available.) + * describe: New subcommand that can set or change the description of + a repository. + * Fix test suite to reap zombies. + (Zombies can be particularly annoying on OSX; thanks to Jimmy Tang + for his help eliminating the infestation... for now.) + * Make test suite not rely on a working cp -pr. + (The Unix wars are still ON!) + * Look for dir.git directories the same as git does. + * Support remote urls specified as relative paths. + * Support non-ssh remote paths that contain tilde expansions. + * fsck: Check for and repair location log damage. + * Bugfix: When fsck detected and moved away corrupt file content, it did + not update the location log. + + -- Joey Hess Fri, 04 Mar 2011 15:10:57 -0400 + +git-annex (0.21) unstable; urgency=low + + * test: Don't rely on chmod -R working. + * unannex: Fix recently introduced bug when attempting to unannex more + than one file at a time. + * test: Set git user name and email in case git can't guess values. + * Fix display of unicode filenames. + + -- Joey Hess Fri, 11 Feb 2011 23:21:08 -0400 + +git-annex (0.20) unstable; urgency=low + + * Preserve specified file ordering when instructed to act on multiple + files or directories. For example, "git annex get a b" will now always + get "a" before "b". Previously it could operate in either order. + * unannex: Commit staged changes at end, to avoid some confusing behavior + with the pre-commit hook, which would see some types of commits after + an unannex as checking in of an unlocked file. + * map: New subcommand that uses graphviz to display a nice map of + the git repository network. + * Deal with the mtl/monads-fd conflict. + * configure: Check for sha1sum. + + -- Joey Hess Tue, 08 Feb 2011 18:57:24 -0400 + +git-annex (0.19) unstable; urgency=low + + * configure: Support using the uuidgen command if the uuid command is + not available. + * Allow --exclude to be specified more than once. + * There are now three levels of repository trust. + * untrust: Now marks the current repository as untrusted. + * semitrust: Now restores the default trust level. (What untrust used to do.) + * fsck, drop: Take untrusted repositories into account. + * Bugfix: Files were copied from trusted remotes first even if their + annex.cost was higher than other remotes. + * Improved temp file handling. Transfers of content can now be resumed + from temp files later; the resume does not have to be the immediate + next git-annex run. + * unused: Include partially transferred content in the list. + * Bugfix: Running a second git-annex while a first has a transfer in + progress no longer deletes the first processes's temp file. + + -- Joey Hess Fri, 28 Jan 2011 14:31:37 -0400 + +git-annex (0.18) unstable; urgency=low + + * Bugfix: `copy --to` and `move --to` forgot to stage location log changes + after transferring the file to the remote repository. + (Did not affect ssh remotes.) + * fsck: Fix bug in moving of corrupted files to .git/annex/bad/ + * migrate: Fix support for --backend option. + * unlock: Fix behavior when file content is not present. + * Test suite improvements. Current top-level test coverage: 80% + + -- Joey Hess Fri, 14 Jan 2011 14:17:44 -0400 + +git-annex (0.17) unstable; urgency=low + + * unannex: Now skips files whose content is not present, rather than + it being an error. + * New migrate subcommand can be used to switch files to using a different + backend, safely and with no duplication of content. + * bugfix: Fix crash caused by empty key name. (Thanks Henrik for reporting.) + + -- Joey Hess Sun, 09 Jan 2011 10:04:11 -0400 + +git-annex (0.16) unstable; urgency=low + + * git-annex-shell: Avoid exposing any git repo config except for the + annex.uuid when doing configlist. + * bugfix: Running `move --to` with a remote whose UUID was not yet known + could result in git-annex not recording on the local side where the + file was moved to. This could not result in data loss, or even a + significant problem, since the remote *did* record that it had the file. + * Also, add a general guard to detect attempts to record information + about repositories with missing UUIDs. + * bugfix: Running `move --to` with a non-ssh remote failed. + * bugfix: Running `copy --to` with a non-ssh remote actually did a move. + * Many test suite improvements. Current top-level test coverage: 65% + + -- Joey Hess Fri, 07 Jan 2011 14:33:13 -0400 + +git-annex (0.15) unstable; urgency=low + + * Support scp-style urls for remotes (host:path). + * Support ssh urls containing "~". + * Add trust and untrust subcommands, to allow configuring repositories + that are trusted to retain files without explicit checking. + * Fix bug in numcopies handling when multiple remotes pointed to the + same repository. + * Introduce the git-annex-shell command. It's now possible to make + a user have it as a restricted login shell, similar to git-shell. + * Note that git-annex will always use git-annex-shell when accessing + a ssh remote, so all of your remotes need to be upgraded to this + version of git-annex at the same time. + * Now rsync is exclusively used for copying files to and from remotes. + scp is not longer supported. + + -- Joey Hess Fri, 31 Dec 2010 22:00:52 -0400 + +git-annex (0.14) unstable; urgency=low + + * Bugfix to git annex unused in a repository with nothing yet annexed. + * Support upgrading from a v0 annex with nothing in it. + * Avoid multiple calls to git ls-files when passed eg, "*". + + -- Joey Hess Fri, 24 Dec 2010 17:38:48 -0400 + +git-annex (0.13) unstable; urgency=low + + * Makefile: Install man page and html (when built). + * Makefile: Add GHCFLAGS variable. + * Fix upgrade from 0.03. + * Support remotes using git+ssh and ssh+git as protocol. + Closes: #607056 + + -- Joey Hess Tue, 14 Dec 2010 13:05:10 -0400 + +git-annex (0.12) unstable; urgency=low + + * Add --exclude option to exclude files from processing. + * mwdn2man: Fix a bug in newline supression. Closes: #606578 + * Bugfix to git annex add of an unlocked file in a subdir. Closes: #606579 + * Makefile: Add PREFIX variable. + + -- Joey Hess Sat, 11 Dec 2010 17:32:00 -0400 + +git-annex (0.11) unstable; urgency=low + + * If available, rsync will be used for file transfers from remote + repositories. This allows resuming interrupted transfers. + * Added remote.annex-rsync-options. + * Avoid deleting temp files when rsync fails. + * Improve detection of version 0 repos. + * Add uninit subcommand. Closes: #605749 + + -- Joey Hess Sat, 04 Dec 2010 17:27:42 -0400 + +git-annex (0.10) unstable; urgency=low + + * In .gitattributes, the annex.numcopies attribute can be used + to control the number of copies to retain of different types of files. + * Bugfix: Always correctly handle gitattributes when in a subdirectory of + the repository. (Had worked ok for ones like "*.mp3", but failed for + ones like "dir/*".) + * fsck: Fix warning about not enough copies of a file, when locations + are known, but are not available in currently configured remotes. + * precommit: Optimise to avoid calling git-check-attr more than once. + * The git-annex-backend attribute has been renamed to annex.backend. + + -- Joey Hess Sun, 28 Nov 2010 19:28:05 -0400 + +git-annex (0.09) unstable; urgency=low + + * Add copy subcommand. + * Fix bug in setkey subcommand triggered by move --to. + + -- Joey Hess Sat, 27 Nov 2010 17:14:59 -0400 + +git-annex (0.08) unstable; urgency=low + + * Fix `git annex add ../foo` (when ran in a subdir of the repo). + * Add configure step to build process. + * Only use cp -a if it is supported, falling back to cp -p or plain cp + as needed for portability. + * cp --reflink=auto is used if supported, and will make git annex unlock + much faster on filesystems like btrfs that support copy on write. + + -- Joey Hess Sun, 21 Nov 2010 13:45:44 -0400 + +git-annex (0.07) unstable; urgency=low + + * find: New subcommand. + * unused: New subcommand, finds unused data. (Split out from fsck.) + * dropunused: New subcommand, provides for easy dropping of unused keys + by number, as listed by the unused subcommand. + * fsck: Print warnings to stderr; --quiet can now be used to only see + problems. + + -- Joey Hess Mon, 15 Nov 2010 18:41:50 -0400 + +git-annex (0.06) unstable; urgency=low + + * fsck: Check if annex.numcopies is satisfied. + * fsck: Verify the sha1 of files when the SHA1 backend is used. + * fsck: Verify the size of files when the WORM backend is used. + * fsck: Allow specifying individual files if fscking everything + is not desired. + * fsck: Fix bug, introduced in 0.04, in detection of unused data. + + -- Joey Hess Sat, 13 Nov 2010 16:24:29 -0400 + +git-annex (0.05) unstable; urgency=low + + * Optimize both pre-commit and lock subcommands to not call git diff + on every file being committed/locked. + (This actually also works around a bug in ghc, that caused + git-annex 0.04 pre-commit to sometimes corrupt filename being read + from git ls-files and fail. + See + The excessive number of calls made by pre-commit exposed the ghc bug. + Thanks Josh Triplett for the debugging.) + * Build with -O2. + + -- Joey Hess Thu, 11 Nov 2010 18:31:09 -0400 + +git-annex (0.04) unstable; urgency=low + + * Add unlock subcommand, which replaces the symlink with a copy of + the file's content in preparation of changing it. The "edit" subcommand + is an alias for unlock. + * Add lock subcommand. + * Unlocked files will now automatically be added back into the annex when + committed (and the updated symlink committed), by some magic in the + pre-commit hook. + * The SHA1 backend is now fully usable. + * Add annex.version, which will be used to automate upgrades + between incompatible versions. + * Reorganised the layout of .git/annex/ + * The new layout will be automatically upgraded to the first time + git-annex is used in a repository with the old layout. + * Note that git-annex 0.04 cannot transfer content from old repositories + that have not yet been upgraded. + * Annexed file contents are now made unwritable and put in unwriteable + directories, to avoid them accidentally being removed or modified. + (Thanks Josh Triplett for the idea.) + * Add build dep on libghc6-testpack-dev. Closes: #603016 + * Avoid using runghc to run test suite as it is not available on all + architectures. Closes: #603006 + + -- Joey Hess Wed, 10 Nov 2010 14:23:23 -0400 + +git-annex (0.03) unstable; urgency=low + + * Fix support for file:// remotes. + * Add --verbose + * Fix SIGINT handling. + * Fix handling of files with unusual characters in their name. + * Fixed memory leak; git-annex no longer reads the whole file list + from git before starting, and will be much faster with large repos. + * Fix crash on unknown symlinks. + * Added remote.annex-scp-options and remote.annex-ssh-options. + * The backends to use when adding different sets of files can be configured + via gitattributes. + * In .gitattributes, the git-annex-backend attribute can be set to the + names of backends to use when adding different types of files. + * Add fsck subcommand. (For now it only finds unused key contents in the + annex.) + + -- Joey Hess Sun, 07 Nov 2010 18:26:04 -0400 + +git-annex (0.02) unstable; urgency=low + + * Can scp annexed files from remote hosts, and check remote hosts for + file content when dropping files. + * New move subcommand, that makes it easy to move file contents from + or to a remote. + * New fromkey subcommand, for registering urls, etc. + * git-annex init will now set up a pre-commit hook that fixes up symlinks + before they are committed, to ensure that moving symlinks around does not + break them. + * More intelligent and fast staging of modified files; git add coalescing. + * Add remote.annex-ignore git config setting to allow completly disabling + a given remote. + * --from/--to can be used to control the remote repository that git-annex + uses. + * --quiet can be used to avoid verbose output + * New plumbing-level dropkey and addkey subcommands. + * Lots of bug fixes. + + -- Joey Hess Wed, 27 Oct 2010 16:39:29 -0400 + +git-annex (0.01) unstable; urgency=low + + * First prerelease. + + -- Joey Hess Wed, 20 Oct 2010 12:54:24 -0400 diff --git a/COPYRIGHT b/COPYRIGHT new file mode 100644 index 0000000000..3c1c93da90 --- /dev/null +++ b/COPYRIGHT @@ -0,0 +1,824 @@ +Format: http://www.debian.org/doc/packaging-manuals/copyright-format/1.0/ +Source: native package + +Files: * +Copyright: © 2010-2018 Joey Hess +License: GPL-3+ + +Files: Assistant/WebApp.hs Assistant/WebApp/* templates/* static/* +Copyright: © 2012-2017 Joey Hess + © 2014 Sören Brunk +License: AGPL-3+ + +Files: Remote/Git.hs Remote/Helper/Ssh.hs Remote/Adb.hs Remote/External.hs Remote/Extermal/Types.hs +Copyright: © 2011-2018 Joey Hess +License: AGPL-3+ + +Files: Remote/Ddar.hs +Copyright: © 2011 Joey Hess + © 2014 Robie Basak +License: GPL-3+ + +Files: Utility/ThreadScheduler.hs +Copyright: 2011 Bas van Dijk & Roel van Dijk + 2012, 2013 Joey Hess +License: BSD-2-clause + +Files: Utility/HttpManagerRestricted.hs +Copyright: 2018 Joey Hess + 2013 Michael Snoyman +License: Expat + +Files: Utility/* +Copyright: 2012-2018 Joey Hess +License: BSD-2-clause + +Files: doc/logo* */favicon.ico standalone/osx/git-annex.app/Contents/Resources/git-annex.icns standalone/android/icons/* +Copyright: 2007 Henrik Nyh + 2010 Joey Hess + 2013 John Lawrence +License: icon-license + Free to modify and redistribute with due credit, and obviously free to use. + +Files: Annex/DirHashes.hs +Copyright: 2001 Ian Lynagh + 2010-2015 Joey Hess +License: GPL-3+ + +Files: doc/tips/automatically_adding_metadata/pre-commit-annex +Copyright: 2014 Joey Hess + 2016 Klaus Ethgen +License: GPL-3+ + +Files: static/jquery* +Copyright: © 2005-2011 by John Resig, Branden Aaron & Jörn Zaefferer + © 2011 The Dojo Foundation +License: Expat or GPL-2 + The full text of version 2 of the GPL is distributed in + /usr/share/common-licenses/GPL-2 on Debian systems. The text of the Expat + license is in the Expat section below. + +Files: static/*/bootstrap* static/*/glyphicons-halflings* +Copyright: 2012-2014 Twitter, Inc. +License: MIT-twitter + Copyright (c) 2011-2014 Twitter, Inc + . + Permission is hereby granted, free of charge, to any person obtaining a copy + of this software and associated documentation files (the "Software"), to deal + in the Software without restriction, including without limitation the rights + to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + copies of the Software, and to permit persons to whom the Software is + furnished to do so, subject to the following conditions: + . + The above copyright notice and this permission notice shall be included in + all copies or substantial portions of the Software. + . + THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN + THE SOFTWARE. + +Files: Logs/Line.hs +Copyright: 2001, The University Court of the University of Glasgow. +License: ghc-license + All rights reserved. + . + Redistribution and use in source and binary forms, with or without + modification, are permitted provided that the following conditions are met: + . + - Redistributions of source code must retain the above copyright notice, + this list of conditions and the following disclaimer. + . + - Redistributions in binary form must reproduce the above copyright notice, + this list of conditions and the following disclaimer in the documentation + and/or other materials provided with the distribution. + . + - Neither name of the University nor the names of its contributors may be + used to endorse or promote products derived from this software without + specific prior written permission. + . + THIS SOFTWARE IS PROVIDED BY THE UNIVERSITY COURT OF THE UNIVERSITY OF + GLASGOW AND THE CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, + INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND + FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE + UNIVERSITY COURT OF THE UNIVERSITY OF GLASGOW OR THE CONTRIBUTORS BE LIABLE + FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR + SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER + CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT + LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY + OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH + DAMAGE. + +License: GPL-3+ + The full text of version 3 of the GPL is distributed as doc/license/GPL in + this package's source, or in /usr/share/common-licenses/GPL-3 on + Debian systems. + +License: BSD-2-clause + Redistribution and use in source and binary forms, with or without + modification, are permitted provided that the following conditions + are met: + 1. Redistributions of source code must retain the above copyright + notice, this list of conditions and the following disclaimer. + 2. Redistributions in binary form must reproduce the above copyright + notice, this list of conditions and the following disclaimer in the + documentation and/or other materials provided with the distribution. + . + THIS SOFTWARE IS PROVIDED BY AUTHORS AND CONTRIBUTORS ``AS IS'' AND + ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHORS OR CONTRIBUTORS BE LIABLE + FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS + OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) + HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT + LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY + OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF + SUCH DAMAGE. + +License: Expat + Permission is hereby granted, free of charge, to any person obtaining + a copy of this software and associated documentation files (the + "Software"), to deal in the Software without restriction, including + without limitation the rights to use, copy, modify, merge, publish, + distribute, sublicense, and/or sell copies of the Software, and to + permit persons to whom the Software is furnished to do so, subject to + the following conditions: + . + The above copyright notice and this permission notice shall be + included in all copies or substantial portions of the Software. + . + THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, + EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF + MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND + NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE + LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION + OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION + WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + +License: AGPL-3+ + GNU AFFERO GENERAL PUBLIC LICENSE + Version 3, 19 November 2007 + . + Copyright (C) 2007 Free Software Foundation, Inc. + Everyone is permitted to copy and distribute verbatim copies + of this license document, but changing it is not allowed. + . + Preamble + . + The GNU Affero General Public License is a free, copyleft license for + software and other kinds of works, specifically designed to ensure + cooperation with the community in the case of network server software. + . + The licenses for most software and other practical works are designed + to take away your freedom to share and change the works. By contrast, + our General Public Licenses are intended to guarantee your freedom to + share and change all versions of a program--to make sure it remains free + software for all its users. + . + When we speak of free software, we are referring to freedom, not + price. Our General Public Licenses are designed to make sure that you + have the freedom to distribute copies of free software (and charge for + them if you wish), that you receive source code or can get it if you + want it, that you can change the software or use pieces of it in new + free programs, and that you know you can do these things. + . + Developers that use our General Public Licenses protect your rights + with two steps: (1) assert copyright on the software, and (2) offer + you this License which gives you legal permission to copy, distribute + and/or modify the software. + . + A secondary benefit of defending all users' freedom is that + improvements made in alternate versions of the program, if they + receive widespread use, become available for other developers to + incorporate. Many developers of free software are heartened and + encouraged by the resulting cooperation. However, in the case of + software used on network servers, this result may fail to come about. + The GNU General Public License permits making a modified version and + letting the public access it on a server without ever releasing its + source code to the public. + . + The GNU Affero General Public License is designed specifically to + ensure that, in such cases, the modified source code becomes available + to the community. It requires the operator of a network server to + provide the source code of the modified version running there to the + users of that server. Therefore, public use of a modified version, on + a publicly accessible server, gives the public access to the source + code of the modified version. + . + An older license, called the Affero General Public License and + published by Affero, was designed to accomplish similar goals. This is + a different license, not a version of the Affero GPL, but Affero has + released a new version of the Affero GPL which permits relicensing under + this license. + . + The precise terms and conditions for copying, distribution and + modification follow. + . + TERMS AND CONDITIONS + . + 0. Definitions. + . + "This License" refers to version 3 of the GNU Affero General Public License. + . + "Copyright" also means copyright-like laws that apply to other kinds of + works, such as semiconductor masks. + . + "The Program" refers to any copyrightable work licensed under this + License. Each licensee is addressed as "you". "Licensees" and + "recipients" may be individuals or organizations. + . + To "modify" a work means to copy from or adapt all or part of the work + in a fashion requiring copyright permission, other than the making of an + exact copy. The resulting work is called a "modified version" of the + earlier work or a work "based on" the earlier work. + . + A "covered work" means either the unmodified Program or a work based + on the Program. + . + To "propagate" a work means to do anything with it that, without + permission, would make you directly or secondarily liable for + infringement under applicable copyright law, except executing it on a + computer or modifying a private copy. Propagation includes copying, + distribution (with or without modification), making available to the + public, and in some countries other activities as well. + . + To "convey" a work means any kind of propagation that enables other + parties to make or receive copies. Mere interaction with a user through + a computer network, with no transfer of a copy, is not conveying. + . + An interactive user interface displays "Appropriate Legal Notices" + to the extent that it includes a convenient and prominently visible + feature that (1) displays an appropriate copyright notice, and (2) + tells the user that there is no warranty for the work (except to the + extent that warranties are provided), that licensees may convey the + work under this License, and how to view a copy of this License. If + the interface presents a list of user commands or options, such as a + menu, a prominent item in the list meets this criterion. + . + 1. Source Code. + . + The "source code" for a work means the preferred form of the work + for making modifications to it. "Object code" means any non-source + form of a work. + . + A "Standard Interface" means an interface that either is an official + standard defined by a recognized standards body, or, in the case of + interfaces specified for a particular programming language, one that + is widely used among developers working in that language. + . + The "System Libraries" of an executable work include anything, other + than the work as a whole, that (a) is included in the normal form of + packaging a Major Component, but which is not part of that Major + Component, and (b) serves only to enable use of the work with that + Major Component, or to implement a Standard Interface for which an + implementation is available to the public in source code form. A + "Major Component", in this context, means a major essential component + (kernel, window system, and so on) of the specific operating system + (if any) on which the executable work runs, or a compiler used to + produce the work, or an object code interpreter used to run it. + . + The "Corresponding Source" for a work in object code form means all + the source code needed to generate, install, and (for an executable + work) run the object code and to modify the work, including scripts to + control those activities. However, it does not include the work's + System Libraries, or general-purpose tools or generally available free + programs which are used unmodified in performing those activities but + which are not part of the work. For example, Corresponding Source + includes interface definition files associated with source files for + the work, and the source code for shared libraries and dynamically + linked subprograms that the work is specifically designed to require, + such as by intimate data communication or control flow between those + subprograms and other parts of the work. + . + The Corresponding Source need not include anything that users + can regenerate automatically from other parts of the Corresponding + Source. + . + The Corresponding Source for a work in source code form is that + same work. + . + 2. Basic Permissions. + . + All rights granted under this License are granted for the term of + copyright on the Program, and are irrevocable provided the stated + conditions are met. This License explicitly affirms your unlimited + permission to run the unmodified Program. The output from running a + covered work is covered by this License only if the output, given its + content, constitutes a covered work. This License acknowledges your + rights of fair use or other equivalent, as provided by copyright law. + . + You may make, run and propagate covered works that you do not + convey, without conditions so long as your license otherwise remains + in force. You may convey covered works to others for the sole purpose + of having them make modifications exclusively for you, or provide you + with facilities for running those works, provided that you comply with + the terms of this License in conveying all material for which you do + not control copyright. Those thus making or running the covered works + for you must do so exclusively on your behalf, under your direction + and control, on terms that prohibit them from making any copies of + your copyrighted material outside their relationship with you. + . + Conveying under any other circumstances is permitted solely under + the conditions stated below. Sublicensing is not allowed; section 10 + makes it unnecessary. + . + 3. Protecting Users' Legal Rights From Anti-Circumvention Law. + . + No covered work shall be deemed part of an effective technological + measure under any applicable law fulfilling obligations under article + 11 of the WIPO copyright treaty adopted on 20 December 1996, or + similar laws prohibiting or restricting circumvention of such + measures. + . + When you convey a covered work, you waive any legal power to forbid + circumvention of technological measures to the extent such circumvention + is effected by exercising rights under this License with respect to + the covered work, and you disclaim any intention to limit operation or + modification of the work as a means of enforcing, against the work's + users, your or third parties' legal rights to forbid circumvention of + technological measures. + . + 4. Conveying Verbatim Copies. + . + You may convey verbatim copies of the Program's source code as you + receive it, in any medium, provided that you conspicuously and + appropriately publish on each copy an appropriate copyright notice; + keep intact all notices stating that this License and any + non-permissive terms added in accord with section 7 apply to the code; + keep intact all notices of the absence of any warranty; and give all + recipients a copy of this License along with the Program. + . + You may charge any price or no price for each copy that you convey, + and you may offer support or warranty protection for a fee. + . + 5. Conveying Modified Source Versions. + . + You may convey a work based on the Program, or the modifications to + produce it from the Program, in the form of source code under the + terms of section 4, provided that you also meet all of these conditions: + . + a) The work must carry prominent notices stating that you modified + it, and giving a relevant date. + . + b) The work must carry prominent notices stating that it is + released under this License and any conditions added under section + 7. This requirement modifies the requirement in section 4 to + "keep intact all notices". + . + c) You must license the entire work, as a whole, under this + License to anyone who comes into possession of a copy. This + License will therefore apply, along with any applicable section 7 + additional terms, to the whole of the work, and all its parts, + regardless of how they are packaged. This License gives no + permission to license the work in any other way, but it does not + invalidate such permission if you have separately received it. + . + d) If the work has interactive user interfaces, each must display + Appropriate Legal Notices; however, if the Program has interactive + interfaces that do not display Appropriate Legal Notices, your + work need not make them do so. + . + A compilation of a covered work with other separate and independent + works, which are not by their nature extensions of the covered work, + and which are not combined with it such as to form a larger program, + in or on a volume of a storage or distribution medium, is called an + "aggregate" if the compilation and its resulting copyright are not + used to limit the access or legal rights of the compilation's users + beyond what the individual works permit. Inclusion of a covered work + in an aggregate does not cause this License to apply to the other + parts of the aggregate. + . + 6. Conveying Non-Source Forms. + . + You may convey a covered work in object code form under the terms + of sections 4 and 5, provided that you also convey the + machine-readable Corresponding Source under the terms of this License, + in one of these ways: + . + a) Convey the object code in, or embodied in, a physical product + (including a physical distribution medium), accompanied by the + Corresponding Source fixed on a durable physical medium + customarily used for software interchange. + . + b) Convey the object code in, or embodied in, a physical product + (including a physical distribution medium), accompanied by a + written offer, valid for at least three years and valid for as + long as you offer spare parts or customer support for that product + model, to give anyone who possesses the object code either (1) a + copy of the Corresponding Source for all the software in the + product that is covered by this License, on a durable physical + medium customarily used for software interchange, for a price no + more than your reasonable cost of physically performing this + conveying of source, or (2) access to copy the + Corresponding Source from a network server at no charge. + . + c) Convey individual copies of the object code with a copy of the + written offer to provide the Corresponding Source. This + alternative is allowed only occasionally and noncommercially, and + only if you received the object code with such an offer, in accord + with subsection 6b. + . + d) Convey the object code by offering access from a designated + place (gratis or for a charge), and offer equivalent access to the + Corresponding Source in the same way through the same place at no + further charge. You need not require recipients to copy the + Corresponding Source along with the object code. If the place to + copy the object code is a network server, the Corresponding Source + may be on a different server (operated by you or a third party) + that supports equivalent copying facilities, provided you maintain + clear directions next to the object code saying where to find the + Corresponding Source. Regardless of what server hosts the + Corresponding Source, you remain obligated to ensure that it is + available for as long as needed to satisfy these requirements. + . + e) Convey the object code using peer-to-peer transmission, provided + you inform other peers where the object code and Corresponding + Source of the work are being offered to the general public at no + charge under subsection 6d. + . + A separable portion of the object code, whose source code is excluded + from the Corresponding Source as a System Library, need not be + included in conveying the object code work. + . + A "User Product" is either (1) a "consumer product", which means any + tangible personal property which is normally used for personal, family, + or household purposes, or (2) anything designed or sold for incorporation + into a dwelling. In determining whether a product is a consumer product, + doubtful cases shall be resolved in favor of coverage. For a particular + product received by a particular user, "normally used" refers to a + typical or common use of that class of product, regardless of the status + of the particular user or of the way in which the particular user + actually uses, or expects or is expected to use, the product. A product + is a consumer product regardless of whether the product has substantial + commercial, industrial or non-consumer uses, unless such uses represent + the only significant mode of use of the product. + . + "Installation Information" for a User Product means any methods, + procedures, authorization keys, or other information required to install + and execute modified versions of a covered work in that User Product from + a modified version of its Corresponding Source. The information must + suffice to ensure that the continued functioning of the modified object + code is in no case prevented or interfered with solely because + modification has been made. + . + If you convey an object code work under this section in, or with, or + specifically for use in, a User Product, and the conveying occurs as + part of a transaction in which the right of possession and use of the + User Product is transferred to the recipient in perpetuity or for a + fixed term (regardless of how the transaction is characterized), the + Corresponding Source conveyed under this section must be accompanied + by the Installation Information. But this requirement does not apply + if neither you nor any third party retains the ability to install + modified object code on the User Product (for example, the work has + been installed in ROM). + . + The requirement to provide Installation Information does not include a + requirement to continue to provide support service, warranty, or updates + for a work that has been modified or installed by the recipient, or for + the User Product in which it has been modified or installed. Access to a + network may be denied when the modification itself materially and + adversely affects the operation of the network or violates the rules and + protocols for communication across the network. + . + Corresponding Source conveyed, and Installation Information provided, + in accord with this section must be in a format that is publicly + documented (and with an implementation available to the public in + source code form), and must require no special password or key for + unpacking, reading or copying. + . + 7. Additional Terms. + . + "Additional permissions" are terms that supplement the terms of this + License by making exceptions from one or more of its conditions. + Additional permissions that are applicable to the entire Program shall + be treated as though they were included in this License, to the extent + that they are valid under applicable law. If additional permissions + apply only to part of the Program, that part may be used separately + under those permissions, but the entire Program remains governed by + this License without regard to the additional permissions. + . + When you convey a copy of a covered work, you may at your option + remove any additional permissions from that copy, or from any part of + it. (Additional permissions may be written to require their own + removal in certain cases when you modify the work.) You may place + additional permissions on material, added by you to a covered work, + for which you have or can give appropriate copyright permission. + . + Notwithstanding any other provision of this License, for material you + add to a covered work, you may (if authorized by the copyright holders of + that material) supplement the terms of this License with terms: + . + a) Disclaiming warranty or limiting liability differently from the + terms of sections 15 and 16 of this License; or + . + b) Requiring preservation of specified reasonable legal notices or + author attributions in that material or in the Appropriate Legal + Notices displayed by works containing it; or + . + c) Prohibiting misrepresentation of the origin of that material, or + requiring that modified versions of such material be marked in + reasonable ways as different from the original version; or + . + d) Limiting the use for publicity purposes of names of licensors or + authors of the material; or + . + e) Declining to grant rights under trademark law for use of some + trade names, trademarks, or service marks; or + . + f) Requiring indemnification of licensors and authors of that + material by anyone who conveys the material (or modified versions of + it) with contractual assumptions of liability to the recipient, for + any liability that these contractual assumptions directly impose on + those licensors and authors. + . + All other non-permissive additional terms are considered "further + restrictions" within the meaning of section 10. If the Program as you + received it, or any part of it, contains a notice stating that it is + governed by this License along with a term that is a further + restriction, you may remove that term. If a license document contains + a further restriction but permits relicensing or conveying under this + License, you may add to a covered work material governed by the terms + of that license document, provided that the further restriction does + not survive such relicensing or conveying. + . + If you add terms to a covered work in accord with this section, you + must place, in the relevant source files, a statement of the + additional terms that apply to those files, or a notice indicating + where to find the applicable terms. + . + Additional terms, permissive or non-permissive, may be stated in the + form of a separately written license, or stated as exceptions; + the above requirements apply either way. + . + 8. Termination. + . + You may not propagate or modify a covered work except as expressly + provided under this License. Any attempt otherwise to propagate or + modify it is void, and will automatically terminate your rights under + this License (including any patent licenses granted under the third + paragraph of section 11). + . + However, if you cease all violation of this License, then your + license from a particular copyright holder is reinstated (a) + provisionally, unless and until the copyright holder explicitly and + finally terminates your license, and (b) permanently, if the copyright + holder fails to notify you of the violation by some reasonable means + prior to 60 days after the cessation. + . + Moreover, your license from a particular copyright holder is + reinstated permanently if the copyright holder notifies you of the + violation by some reasonable means, this is the first time you have + received notice of violation of this License (for any work) from that + copyright holder, and you cure the violation prior to 30 days after + your receipt of the notice. + . + Termination of your rights under this section does not terminate the + licenses of parties who have received copies or rights from you under + this License. If your rights have been terminated and not permanently + reinstated, you do not qualify to receive new licenses for the same + material under section 10. + . + 9. Acceptance Not Required for Having Copies. + . + You are not required to accept this License in order to receive or + run a copy of the Program. Ancillary propagation of a covered work + occurring solely as a consequence of using peer-to-peer transmission + to receive a copy likewise does not require acceptance. However, + nothing other than this License grants you permission to propagate or + modify any covered work. These actions infringe copyright if you do + not accept this License. Therefore, by modifying or propagating a + covered work, you indicate your acceptance of this License to do so. + . + 10. Automatic Licensing of Downstream Recipients. + . + Each time you convey a covered work, the recipient automatically + receives a license from the original licensors, to run, modify and + propagate that work, subject to this License. You are not responsible + for enforcing compliance by third parties with this License. + . + An "entity transaction" is a transaction transferring control of an + organization, or substantially all assets of one, or subdividing an + organization, or merging organizations. If propagation of a covered + work results from an entity transaction, each party to that + transaction who receives a copy of the work also receives whatever + licenses to the work the party's predecessor in interest had or could + give under the previous paragraph, plus a right to possession of the + Corresponding Source of the work from the predecessor in interest, if + the predecessor has it or can get it with reasonable efforts. + . + You may not impose any further restrictions on the exercise of the + rights granted or affirmed under this License. For example, you may + not impose a license fee, royalty, or other charge for exercise of + rights granted under this License, and you may not initiate litigation + (including a cross-claim or counterclaim in a lawsuit) alleging that + any patent claim is infringed by making, using, selling, offering for + sale, or importing the Program or any portion of it. + . + 11. Patents. + . + A "contributor" is a copyright holder who authorizes use under this + License of the Program or a work on which the Program is based. The + work thus licensed is called the contributor's "contributor version". + . + A contributor's "essential patent claims" are all patent claims + owned or controlled by the contributor, whether already acquired or + hereafter acquired, that would be infringed by some manner, permitted + by this License, of making, using, or selling its contributor version, + but do not include claims that would be infringed only as a + consequence of further modification of the contributor version. For + purposes of this definition, "control" includes the right to grant + patent sublicenses in a manner consistent with the requirements of + this License. + . + Each contributor grants you a non-exclusive, worldwide, royalty-free + patent license under the contributor's essential patent claims, to + make, use, sell, offer for sale, import and otherwise run, modify and + propagate the contents of its contributor version. + . + In the following three paragraphs, a "patent license" is any express + agreement or commitment, however denominated, not to enforce a patent + (such as an express permission to practice a patent or covenant not to + sue for patent infringement). To "grant" such a patent license to a + party means to make such an agreement or commitment not to enforce a + patent against the party. + . + If you convey a covered work, knowingly relying on a patent license, + and the Corresponding Source of the work is not available for anyone + to copy, free of charge and under the terms of this License, through a + publicly available network server or other readily accessible means, + then you must either (1) cause the Corresponding Source to be so + available, or (2) arrange to deprive yourself of the benefit of the + patent license for this particular work, or (3) arrange, in a manner + consistent with the requirements of this License, to extend the patent + license to downstream recipients. "Knowingly relying" means you have + actual knowledge that, but for the patent license, your conveying the + covered work in a country, or your recipient's use of the covered work + in a country, would infringe one or more identifiable patents in that + country that you have reason to believe are valid. + . + If, pursuant to or in connection with a single transaction or + arrangement, you convey, or propagate by procuring conveyance of, a + covered work, and grant a patent license to some of the parties + receiving the covered work authorizing them to use, propagate, modify + or convey a specific copy of the covered work, then the patent license + you grant is automatically extended to all recipients of the covered + work and works based on it. + . + A patent license is "discriminatory" if it does not include within + the scope of its coverage, prohibits the exercise of, or is + conditioned on the non-exercise of one or more of the rights that are + specifically granted under this License. You may not convey a covered + work if you are a party to an arrangement with a third party that is + in the business of distributing software, under which you make payment + to the third party based on the extent of your activity of conveying + the work, and under which the third party grants, to any of the + parties who would receive the covered work from you, a discriminatory + patent license (a) in connection with copies of the covered work + conveyed by you (or copies made from those copies), or (b) primarily + for and in connection with specific products or compilations that + contain the covered work, unless you entered into that arrangement, + or that patent license was granted, prior to 28 March 2007. + . + Nothing in this License shall be construed as excluding or limiting + any implied license or other defenses to infringement that may + otherwise be available to you under applicable patent law. + . + 12. No Surrender of Others' Freedom. + . + If conditions are imposed on you (whether by court order, agreement or + otherwise) that contradict the conditions of this License, they do not + excuse you from the conditions of this License. If you cannot convey a + covered work so as to satisfy simultaneously your obligations under this + License and any other pertinent obligations, then as a consequence you may + not convey it at all. For example, if you agree to terms that obligate you + to collect a royalty for further conveying from those to whom you convey + the Program, the only way you could satisfy both those terms and this + License would be to refrain entirely from conveying the Program. + . + 13. Remote Network Interaction; Use with the GNU General Public License. + . + Notwithstanding any other provision of this License, if you modify the + Program, your modified version must prominently offer all users + interacting with it remotely through a computer network (if your version + supports such interaction) an opportunity to receive the Corresponding + Source of your version by providing access to the Corresponding Source + from a network server at no charge, through some standard or customary + means of facilitating copying of software. This Corresponding Source + shall include the Corresponding Source for any work covered by version 3 + of the GNU General Public License that is incorporated pursuant to the + following paragraph. + . + Notwithstanding any other provision of this License, you have + permission to link or combine any covered work with a work licensed + under version 3 of the GNU General Public License into a single + combined work, and to convey the resulting work. The terms of this + License will continue to apply to the part which is the covered work, + but the work with which it is combined will remain governed by version + 3 of the GNU General Public License. + . + 14. Revised Versions of this License. + . + The Free Software Foundation may publish revised and/or new versions of + the GNU Affero General Public License from time to time. Such new versions + will be similar in spirit to the present version, but may differ in detail to + address new problems or concerns. + . + Each version is given a distinguishing version number. If the + Program specifies that a certain numbered version of the GNU Affero General + Public License "or any later version" applies to it, you have the + option of following the terms and conditions either of that numbered + version or of any later version published by the Free Software + Foundation. If the Program does not specify a version number of the + GNU Affero General Public License, you may choose any version ever published + by the Free Software Foundation. + . + If the Program specifies that a proxy can decide which future + versions of the GNU Affero General Public License can be used, that proxy's + public statement of acceptance of a version permanently authorizes you + to choose that version for the Program. + . + Later license versions may give you additional or different + permissions. However, no additional obligations are imposed on any + author or copyright holder as a result of your choosing to follow a + later version. + . + 15. Disclaimer of Warranty. + . + THERE IS NO WARRANTY FOR THE PROGRAM, TO THE EXTENT PERMITTED BY + APPLICABLE LAW. EXCEPT WHEN OTHERWISE STATED IN WRITING THE COPYRIGHT + HOLDERS AND/OR OTHER PARTIES PROVIDE THE PROGRAM "AS IS" WITHOUT WARRANTY + OF ANY KIND, EITHER EXPRESSED OR IMPLIED, INCLUDING, BUT NOT LIMITED TO, + THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR + PURPOSE. THE ENTIRE RISK AS TO THE QUALITY AND PERFORMANCE OF THE PROGRAM + IS WITH YOU. SHOULD THE PROGRAM PROVE DEFECTIVE, YOU ASSUME THE COST OF + ALL NECESSARY SERVICING, REPAIR OR CORRECTION. + . + 16. Limitation of Liability. + . + IN NO EVENT UNLESS REQUIRED BY APPLICABLE LAW OR AGREED TO IN WRITING + WILL ANY COPYRIGHT HOLDER, OR ANY OTHER PARTY WHO MODIFIES AND/OR CONVEYS + THE PROGRAM AS PERMITTED ABOVE, BE LIABLE TO YOU FOR DAMAGES, INCLUDING ANY + GENERAL, SPECIAL, INCIDENTAL OR CONSEQUENTIAL DAMAGES ARISING OUT OF THE + USE OR INABILITY TO USE THE PROGRAM (INCLUDING BUT NOT LIMITED TO LOSS OF + DATA OR DATA BEING RENDERED INACCURATE OR LOSSES SUSTAINED BY YOU OR THIRD + PARTIES OR A FAILURE OF THE PROGRAM TO OPERATE WITH ANY OTHER PROGRAMS), + EVEN IF SUCH HOLDER OR OTHER PARTY HAS BEEN ADVISED OF THE POSSIBILITY OF + SUCH DAMAGES. + . + 17. Interpretation of Sections 15 and 16. + . + If the disclaimer of warranty and limitation of liability provided + above cannot be given local legal effect according to their terms, + reviewing courts shall apply local law that most closely approximates + an absolute waiver of all civil liability in connection with the + Program, unless a warranty or assumption of liability accompanies a + copy of the Program in return for a fee. + . + END OF TERMS AND CONDITIONS + . + How to Apply These Terms to Your New Programs + . + If you develop a new program, and you want it to be of the greatest + possible use to the public, the best way to achieve this is to make it + free software which everyone can redistribute and change under these terms. + . + To do so, attach the following notices to the program. It is safest + to attach them to the start of each source file to most effectively + state the exclusion of warranty; and each file should have at least + the "copyright" line and a pointer to where the full notice is found. + . + + Copyright (C) + . + This program is free software: you can redistribute it and/or modify + it under the terms of the GNU Affero General Public License as published by + the Free Software Foundation, either version 3 of the License, or + (at your option) any later version. + . + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU Affero General Public License for more details. + . + You should have received a copy of the GNU Affero General Public License + along with this program. If not, see . + . + Also add information on how to contact you by electronic and paper mail. + . + If your software can interact with users remotely through a computer + network, you should also make sure that it provides a way for users to + get its source. For example, if your program is a web application, its + interface could display a "Source" link that leads users to an archive + of the code. There are many ways you could offer source, and different + solutions will be better for different programs; see section 13 for the + specific requirements. + . + You should also get your employer (if you work as a programmer) or school, + if any, to sign a "copyright disclaimer" for the program, if necessary. + For more information on this, and how to apply and follow the GNU AGPL, see + . diff --git a/CmdLine.hs b/CmdLine.hs new file mode 100644 index 0000000000..c9de90ec07 --- /dev/null +++ b/CmdLine.hs @@ -0,0 +1,125 @@ +{- git-annex command line parsing and dispatch + - + - Copyright 2010-2015 Joey Hess + - + - Licensed under the GNU GPL version 3 or higher. + -} + +module CmdLine ( + dispatch, + usage, +) where + +import qualified Options.Applicative as O +import qualified Options.Applicative.Help as H +import qualified Control.Exception as E +import Control.Exception (throw) + +import Annex.Common +import qualified Annex +import qualified Git +import qualified Git.AutoCorrect +import qualified Git.Config +import Annex.Action +import Annex.Environment +import Command +import Types.Messages + +{- Runs the passed command line. -} +dispatch :: Bool -> CmdParams -> [Command] -> [GlobalOption] -> [(String, String)] -> IO Git.Repo -> String -> String -> IO () +dispatch fuzzyok allargs allcmds globaloptions fields getgitrepo progname progdesc = do + setupConsole + go =<< (E.try getgitrepo :: IO (Either E.SomeException Git.Repo)) + where + go (Right g) = do + state <- Annex.new g + Annex.eval state $ do + checkEnvironment + forM_ fields $ uncurry Annex.setField + (cmd, seek, globalconfig) <- parsewith False cmdparser + (\a -> inRepo $ a . Just) + (liftIO . O.handleParseResult) + when (cmdnomessages cmd) $ do + Annex.setOutput QuietOutput + Annex.changeState $ \s -> s + { Annex.output = (Annex.output s) { implicitMessages = False } } + getParsed globalconfig + whenM (annexDebug <$> Annex.getGitConfig) $ + liftIO enableDebugOutput + startup + performCommandAction cmd seek $ + shutdown $ cmdnocommit cmd + go (Left norepo) = do + let ingitrepo = \a -> a =<< Git.Config.global + -- Parse command line with full cmdparser first, + -- so that help can be displayed for bad parses + -- even when not run in a repo. + res <- parsewith False cmdparser ingitrepo return + case res of + Failure _ -> void (O.handleParseResult res) + _ -> do + -- Parse command line in norepo mode. + (_, a, _globalconfig) <- parsewith True + (fromMaybe (throw norepo) . cmdnorepo) + ingitrepo + O.handleParseResult + a + + parsewith secondrun getparser ingitrepo handleresult = + case parseCmd progname progdesc globaloptions allargs allcmds getparser of + O.Failure _ -> do + -- parse failed, so fall back to + -- fuzzy matching, or to showing usage + when (fuzzy && not secondrun) $ + ingitrepo autocorrect + handleresult (parseCmd progname progdesc globaloptions correctedargs allcmds getparser) + res -> handleresult res + where + autocorrect = Git.AutoCorrect.prepare (fromJust inputcmdname) cmdname cmds + (fuzzy, cmds, inputcmdname, args) = findCmd fuzzyok allargs allcmds + name + | fuzzy = case cmds of + (c:_) -> Just (cmdname c) + _ -> inputcmdname + | otherwise = inputcmdname + correctedargs = case name of + Nothing -> allargs + Just n -> n:args + +{- Parses command line, selecting one of the commands from the list. -} +parseCmd :: String -> String -> [GlobalOption] -> CmdParams -> [Command] -> (Command -> O.Parser v) -> O.ParserResult (Command, v, GlobalSetter) +parseCmd progname progdesc globaloptions allargs allcmds getparser = + O.execParserPure (O.prefs O.idm) pinfo allargs + where + pinfo = O.info (O.helper <*> subcmds) (O.progDescDoc (Just intro)) + subcmds = O.hsubparser $ mconcat $ map mkcommand allcmds + mkcommand c = O.command (cmdname c) $ O.info (mkparser c) $ O.fullDesc + <> O.header (synopsis (progname ++ " " ++ cmdname c) (cmddesc c)) + <> O.footer ("For details, run: " ++ progname ++ " help " ++ cmdname c) + mkparser c = (,,) + <$> pure c + <*> getparser c + <*> combineGlobalOptions (globaloptions ++ cmdglobaloptions c) + synopsis n d = n ++ " - " ++ d + intro = mconcat $ concatMap (\l -> [H.text l, H.line]) + (synopsis progname progdesc : commandList allcmds) + +{- Parses command line params far enough to find the Command to run, and + - returns the remaining params. + - Does fuzzy matching if necessary, which may result in multiple Commands. -} +findCmd :: Bool -> CmdParams -> [Command] -> (Bool, [Command], Maybe String, CmdParams) +findCmd fuzzyok argv cmds + | not (null exactcmds) = ret (False, exactcmds) + | fuzzyok && not (null inexactcmds) = ret (True, inexactcmds) + | otherwise = ret (False, []) + where + ret (fuzzy, matches) = (fuzzy, matches, name, args) + (name, args) = findname argv [] + findname [] c = (Nothing, reverse c) + findname (a:as) c + | "-" `isPrefixOf` a = findname as (a:c) + | otherwise = (Just a, reverse c ++ as) + exactcmds = filter (\c -> name == Just (cmdname c)) cmds + inexactcmds = case name of + Nothing -> [] + Just n -> Git.AutoCorrect.fuzzymatches n cmdname cmds diff --git a/CmdLine/Action.hs b/CmdLine/Action.hs new file mode 100644 index 0000000000..d4775a1a7d --- /dev/null +++ b/CmdLine/Action.hs @@ -0,0 +1,221 @@ +{- git-annex command-line actions + - + - Copyright 2010-2017 Joey Hess + - + - Licensed under the GNU GPL version 3 or higher. + -} + +{-# LANGUAGE CPP #-} + +module CmdLine.Action where + +import Annex.Common +import qualified Annex +import Annex.Concurrent +import Types.Command +import Types.Concurrency +import Messages.Concurrent +import Types.Messages +import Remote.List + +import Control.Concurrent +import Control.Concurrent.Async +import Control.Concurrent.STM +import Control.Exception (throwIO) +import Data.Either +import qualified Data.Map.Strict as M + +#ifdef WITH_CONCURRENTOUTPUT +import qualified System.Console.Regions as Regions +#endif + +{- Runs a command, starting with the check stage, and then + - the seek stage. Finishes by running the continutation, and + - then showing a count of any failures. -} +performCommandAction :: Command -> CommandSeek -> Annex () -> Annex () +performCommandAction Command { cmdcheck = c, cmdname = name } seek cont = do + mapM_ runCheck c + Annex.changeState $ \s -> s { Annex.errcounter = 0 } + seek + finishCommandActions + cont + showerrcount =<< Annex.getState Annex.errcounter + where + showerrcount 0 = noop + showerrcount cnt = giveup $ name ++ ": " ++ show cnt ++ " failed" + +{- Runs one of the actions needed to perform a command. + - Individual actions can fail without stopping the whole command, + - including by throwing IO errors (but other errors terminate the whole + - command). + - + - When concurrency is enabled, a thread is forked off to run the action + - in the background, as soon as a free slot is available. + + - This should only be run in the seek stage. + -} +commandAction :: CommandStart -> Annex () +commandAction a = go =<< Annex.getState Annex.concurrency + where + go (Concurrent n) = do + ws <- Annex.getState Annex.workers + (st, ws') <- if null ws + then do + -- Generate the remote list now, to avoid + -- each thread generating it, which would + -- be more expensive and could cause + -- threads to contend over eg, calls to + -- setConfig. + _ <- remoteList + st <- dupState + return (st, replicate (n-1) (Left st)) + else do + l <- liftIO $ drainTo (n-1) ws + findFreeSlot l + w <- liftIO $ async + $ snd <$> Annex.run st (inOwnConsoleRegion (Annex.output st) run) + Annex.changeState $ \s -> s { Annex.workers = Right w:ws' } + go NonConcurrent = run + run = void $ includeCommandAction a + +commandActions :: [CommandStart] -> Annex () +commandActions = mapM_ commandAction + +{- Waits for any forked off command actions to finish. + - + - Merge together the cleanup actions of all the AnnexStates used by + - threads, into the current Annex's state, so they'll run at shutdown. + - + - Also merge together the errcounters of the AnnexStates. + -} +finishCommandActions :: Annex () +finishCommandActions = do + ws <- Annex.getState Annex.workers + Annex.changeState $ \s -> s { Annex.workers = [] } + l <- liftIO $ drainTo 0 ws + forM_ (lefts l) mergeState + +{- Wait for Asyncs from the list to finish, replacing them with their + - final AnnexStates, until the list of remaining Asyncs is not larger + - than the specified size, then returns the new list. + - + - If the action throws an exception, it is propigated, but first + - all other actions are waited for, to allow for a clean shutdown. + -} +drainTo + :: Int + -> [Either Annex.AnnexState (Async Annex.AnnexState)] + -> IO [Either Annex.AnnexState (Async Annex.AnnexState)] +drainTo sz l + | null as || sz >= length as = return l + | otherwise = do + (done, ret) <- waitAnyCatch as + let as' = filter (/= done) as + case ret of + Left e -> do + void $ drainTo 0 (map Left sts ++ map Right as') + throwIO e + Right st -> do + drainTo sz $ map Left (st:sts) ++ map Right as' + where + (sts, as) = partitionEithers l + +findFreeSlot :: [Either Annex.AnnexState (Async Annex.AnnexState)] -> Annex (Annex.AnnexState, [Either Annex.AnnexState (Async Annex.AnnexState)]) +findFreeSlot = go [] + where + go c [] = do + st <- dupState + return (st, c) + go c (Left st:rest) = return (st, c ++ rest) + go c (v:rest) = go (v:c) rest + +{- Like commandAction, but without the concurrency. -} +includeCommandAction :: CommandStart -> CommandCleanup +includeCommandAction a = account =<< tryIO (callCommandAction a) + where + account (Right True) = return True + account (Right False) = incerr + account (Left err) = do + toplevelWarning True (show err) + implicitMessage showEndFail + incerr + incerr = do + Annex.incError + return False + +{- Runs a single command action through the start, perform and cleanup + - stages, without catching errors. Useful if one command wants to run + - part of another command. -} +callCommandAction :: CommandStart -> CommandCleanup +callCommandAction = fromMaybe True <$$> callCommandAction' + +{- Like callCommandAction, but returns Nothing when the command did not + - perform any action. -} +callCommandAction' :: CommandStart -> Annex (Maybe Bool) +callCommandAction' a = callCommandActionQuiet a >>= \case + Nothing -> return Nothing + Just r -> implicitMessage (showEndResult r) >> return (Just r) + +callCommandActionQuiet :: CommandStart -> Annex (Maybe Bool) +callCommandActionQuiet = start + where + start = stage $ maybe skip perform + perform = stage $ maybe failure cleanup + cleanup = stage $ status + stage = (=<<) + skip = return Nothing + failure = return (Just False) + status = return . Just + +{- Do concurrent output when that has been requested. -} +allowConcurrentOutput :: Annex a -> Annex a +#ifdef WITH_CONCURRENTOUTPUT +allowConcurrentOutput a = go =<< Annex.getState Annex.concurrency + where + go NonConcurrent = a + go (Concurrent _) = ifM (liftIO concurrentOutputSupported) + ( Regions.displayConsoleRegions $ + goconcurrent True + , goconcurrent False + ) + goconcurrent b = bracket_ (setup b) cleanup a + setup = setconcurrentenabled + cleanup = do + finishCommandActions + setconcurrentenabled False + setconcurrentenabled b = Annex.changeState $ \s -> + s { Annex.output = (Annex.output s) { concurrentOutputEnabled = b } } +#else +allowConcurrentOutput = id +#endif + +{- Ensures that only one thread processes a key at a time. + - Other threads will block until it's done. -} +onlyActionOn :: Key -> CommandStart -> CommandStart +onlyActionOn k a = onlyActionOn' k run + where + -- Run whole action, not just start stage, so other threads + -- block until it's done. + run = callCommandActionQuiet a >>= \case + Nothing -> return Nothing + Just r' -> return $ Just $ return $ Just $ return r' + +onlyActionOn' :: Key -> Annex a -> Annex a +onlyActionOn' k a = go =<< Annex.getState Annex.concurrency + where + go NonConcurrent = a + go (Concurrent _) = do + tv <- Annex.getState Annex.activekeys + bracket (setup tv) id (const a) + setup tv = liftIO $ do + mytid <- myThreadId + atomically $ do + m <- readTVar tv + case M.lookup k m of + Just tid + | tid /= mytid -> retry + | otherwise -> return (return ()) + Nothing -> do + writeTVar tv $! M.insert k mytid m + return $ liftIO $ atomically $ + modifyTVar tv $ M.delete k diff --git a/CmdLine/Batch.hs b/CmdLine/Batch.hs new file mode 100644 index 0000000000..324552dd58 --- /dev/null +++ b/CmdLine/Batch.hs @@ -0,0 +1,108 @@ +{- git-annex batch commands + - + - Copyright 2015 Joey Hess + - + - Licensed under the GNU GPL version 3 or higher. + -} + +module CmdLine.Batch where + +import Annex.Common +import Types.Command +import CmdLine.Action +import CmdLine.GitAnnex.Options +import Options.Applicative +import Limit +import Types.FileMatcher + +data BatchMode = Batch BatchFormat | NoBatch + +data BatchFormat = BatchLine | BatchNull + +parseBatchOption :: Parser BatchMode +parseBatchOption = go + <$> switch + ( long "batch" + <> help "enable batch mode" + ) + <*> switch + ( short 'z' + <> help "null delimited batch input" + ) + where + go True False = Batch BatchLine + go True True = Batch BatchNull + go False _ = NoBatch + +-- A batchable command can run in batch mode, or not. +-- In batch mode, one line at a time is read, parsed, and a reply output to +-- stdout. In non batch mode, the command's parameters are parsed and +-- a reply output for each. +batchable :: (opts -> String -> Annex Bool) -> Parser opts -> CmdParamsDesc -> CommandParser +batchable handler parser paramdesc = batchseeker <$> batchparser + where + batchparser = (,,) + <$> parser + <*> parseBatchOption + <*> cmdParams paramdesc + + batchseeker (opts, NoBatch, params) = + mapM_ (go NoBatch opts) params + batchseeker (opts, batchmode@(Batch fmt), _) = + batchInput fmt Right (go batchmode opts) + + go batchmode opts p = + unlessM (handler opts p) $ + batchBadInput batchmode + +-- bad input is indicated by an empty line in batch mode. In non batch +-- mode, exit on bad input. +batchBadInput :: BatchMode -> Annex () +batchBadInput NoBatch = liftIO exitFailure +batchBadInput (Batch _) = liftIO $ putStrLn "" + +-- Reads lines of batch mode input and passes to the action to handle. +batchInput :: BatchFormat -> (String -> Either String a) -> (a -> Annex ()) -> Annex () +batchInput fmt parser a = go =<< batchLines fmt + where + go [] = return () + go (l:rest) = do + either parseerr a (parser l) + go rest + parseerr s = giveup $ "Batch input parse failure: " ++ s + +batchLines :: BatchFormat -> Annex [String] +batchLines fmt = liftIO $ splitter <$> getContents + where + splitter = case fmt of + BatchLine -> lines + BatchNull -> splitc '\0' + +-- Runs a CommandStart in batch mode. +-- +-- The batch mode user expects to read a line of output, and it's up to the +-- CommandStart to generate that output as it succeeds or fails to do its +-- job. However, if it stops without doing anything, it won't generate +-- any output, so in that case, batchBadInput is used to provide the caller +-- with an empty line. +batchCommandAction :: CommandStart -> Annex () +batchCommandAction a = maybe (batchBadInput (Batch BatchLine)) (const noop) + =<< callCommandAction' a + +-- Reads lines of batch input and passes the filepaths to a CommandStart +-- to handle them. +-- +-- File matching options are not checked. +allBatchFiles :: BatchFormat -> (FilePath -> CommandStart) -> Annex () +allBatchFiles fmt a = batchInput fmt Right $ batchCommandAction . a + +-- Like allBatchFiles, but checks the file matching options +-- and skips non-matching files. +batchFilesMatching :: BatchFormat -> (FilePath -> CommandStart) -> Annex () +batchFilesMatching fmt a = do + matcher <- getMatcher + allBatchFiles fmt $ \f -> + ifM (matcher $ MatchingFile $ FileInfo f f) + ( a f + , return Nothing + ) diff --git a/CmdLine/GitAnnex.hs b/CmdLine/GitAnnex.hs new file mode 100644 index 0000000000..e2f28e39c3 --- /dev/null +++ b/CmdLine/GitAnnex.hs @@ -0,0 +1,249 @@ +{- git-annex main program + - + - Copyright 2010-2015 Joey Hess + - + - Licensed under the GNU GPL version 3 or higher. + -} + +{-# LANGUAGE CPP, OverloadedStrings #-} + +module CmdLine.GitAnnex where + +import qualified Git.CurrentRepo +import CmdLine +import Command +import Utility.Env +import Annex.Ssh +import Annex.Multicast +import Types.Test + +import qualified Command.Help +import qualified Command.Add +import qualified Command.Unannex +import qualified Command.Drop +import qualified Command.Move +import qualified Command.Copy +import qualified Command.Get +import qualified Command.Fsck +import qualified Command.LookupKey +import qualified Command.CalcKey +import qualified Command.ContentLocation +import qualified Command.ExamineKey +import qualified Command.MatchExpression +import qualified Command.FromKey +import qualified Command.RegisterUrl +import qualified Command.SetKey +import qualified Command.DropKey +import qualified Command.TransferKey +import qualified Command.TransferKeys +import qualified Command.SetPresentKey +import qualified Command.ReadPresentKey +import qualified Command.CheckPresentKey +import qualified Command.ReKey +import qualified Command.Adjust +import qualified Command.MetaData +import qualified Command.View +import qualified Command.VAdd +import qualified Command.VFilter +import qualified Command.VPop +import qualified Command.VCycle +import qualified Command.Reinject +import qualified Command.Fix +import qualified Command.Init +import qualified Command.Describe +import qualified Command.InitRemote +import qualified Command.EnableRemote +import qualified Command.EnableTor +import qualified Command.Multicast +import qualified Command.Expire +import qualified Command.Repair +import qualified Command.Unused +import qualified Command.DropUnused +import qualified Command.AddUnused +import qualified Command.Unlock +import qualified Command.Lock +import qualified Command.PreCommit +import qualified Command.PostReceive +import qualified Command.Find +import qualified Command.FindRef +import qualified Command.Whereis +import qualified Command.List +import qualified Command.Log +import qualified Command.Merge +import qualified Command.ResolveMerge +import qualified Command.Info +import qualified Command.Status +import qualified Command.Inprogress +import qualified Command.Migrate +import qualified Command.Uninit +import qualified Command.Reinit +import qualified Command.NumCopies +import qualified Command.Trust +import qualified Command.Untrust +import qualified Command.Semitrust +import qualified Command.Dead +import qualified Command.Group +import qualified Command.Wanted +import qualified Command.GroupWanted +import qualified Command.Required +import qualified Command.Schedule +import qualified Command.Ungroup +import qualified Command.Config +import qualified Command.Vicfg +import qualified Command.Sync +import qualified Command.Mirror +import qualified Command.AddUrl +import qualified Command.ImportFeed +import qualified Command.RmUrl +import qualified Command.Import +import qualified Command.Export +import qualified Command.Map +import qualified Command.Direct +import qualified Command.Indirect +import qualified Command.Upgrade +import qualified Command.Forget +import qualified Command.P2P +import qualified Command.Proxy +import qualified Command.DiffDriver +import qualified Command.Smudge +import qualified Command.Undo +import qualified Command.Version +import qualified Command.RemoteDaemon +#ifdef WITH_ASSISTANT +import qualified Command.Watch +import qualified Command.Assistant +#ifdef WITH_WEBAPP +import qualified Command.WebApp +#endif +#endif +import qualified Command.Test +import qualified Command.FuzzTest +import qualified Command.TestRemote +#ifdef WITH_BENCHMARK +import qualified Command.Benchmark +#endif + +cmds :: Parser TestOptions -> Maybe TestRunner -> [Command] +cmds testoptparser testrunner = + [ Command.Help.cmd + , Command.Add.cmd + , Command.Get.cmd + , Command.Drop.cmd + , Command.Move.cmd + , Command.Copy.cmd + , Command.Fsck.cmd + , Command.Unlock.cmd + , Command.Unlock.editcmd + , Command.Lock.cmd + , Command.Sync.cmd + , Command.Mirror.cmd + , Command.AddUrl.cmd + , Command.ImportFeed.cmd + , Command.RmUrl.cmd + , Command.Import.cmd + , Command.Export.cmd + , Command.Init.cmd + , Command.Describe.cmd + , Command.InitRemote.cmd + , Command.EnableRemote.cmd + , Command.EnableTor.cmd + , Command.Multicast.cmd + , Command.Reinject.cmd + , Command.Unannex.cmd + , Command.Uninit.cmd + , Command.Reinit.cmd + , Command.PreCommit.cmd + , Command.PostReceive.cmd + , Command.NumCopies.cmd + , Command.Trust.cmd + , Command.Untrust.cmd + , Command.Semitrust.cmd + , Command.Dead.cmd + , Command.Group.cmd + , Command.Wanted.cmd + , Command.GroupWanted.cmd + , Command.Required.cmd + , Command.Schedule.cmd + , Command.Ungroup.cmd + , Command.Config.cmd + , Command.Vicfg.cmd + , Command.LookupKey.cmd + , Command.CalcKey.cmd + , Command.ContentLocation.cmd + , Command.ExamineKey.cmd + , Command.MatchExpression.cmd + , Command.FromKey.cmd + , Command.RegisterUrl.cmd + , Command.SetKey.cmd + , Command.DropKey.cmd + , Command.TransferKey.cmd + , Command.TransferKeys.cmd + , Command.SetPresentKey.cmd + , Command.ReadPresentKey.cmd + , Command.CheckPresentKey.cmd + , Command.ReKey.cmd + , Command.Adjust.cmd + , Command.MetaData.cmd + , Command.View.cmd + , Command.VAdd.cmd + , Command.VFilter.cmd + , Command.VPop.cmd + , Command.VCycle.cmd + , Command.Fix.cmd + , Command.Expire.cmd + , Command.Repair.cmd + , Command.Unused.cmd + , Command.DropUnused.cmd + , Command.AddUnused.cmd + , Command.Find.cmd + , Command.FindRef.cmd + , Command.Whereis.cmd + , Command.List.cmd + , Command.Log.cmd + , Command.Merge.cmd + , Command.ResolveMerge.cmd + , Command.Info.cmd + , Command.Status.cmd + , Command.Inprogress.cmd + , Command.Migrate.cmd + , Command.Map.cmd + , Command.Direct.cmd + , Command.Indirect.cmd + , Command.Upgrade.cmd + , Command.Forget.cmd + , Command.P2P.cmd + , Command.Proxy.cmd + , Command.DiffDriver.cmd + , Command.Smudge.cmd + , Command.Undo.cmd + , Command.Version.cmd + , Command.RemoteDaemon.cmd +#ifdef WITH_ASSISTANT + , Command.Watch.cmd + , Command.Assistant.cmd +#ifdef WITH_WEBAPP + , Command.WebApp.cmd +#endif +#endif + , Command.Test.cmd testoptparser testrunner + , Command.FuzzTest.cmd + , Command.TestRemote.cmd +#ifdef WITH_BENCHMARK + , Command.Benchmark.cmd +#endif + ] + +run :: Parser TestOptions -> Maybe TestRunner -> [String] -> IO () +run testoptparser testrunner args = go envmodes + where + go [] = dispatch True args + (cmds testoptparser testrunner) + gitAnnexGlobalOptions [] Git.CurrentRepo.get + "git-annex" + "manage files with git, without checking their contents in" + go ((v, a):rest) = maybe (go rest) a =<< getEnv v + envmodes = + [ (sshOptionsEnv, runSshOptions args) + , (sshAskPassEnv, runSshAskPass) + , (multicastReceiveEnv, runMulticastReceive args) + ] diff --git a/CmdLine/GitAnnex/Options.hs b/CmdLine/GitAnnex/Options.hs new file mode 100644 index 0000000000..49a0d66f9b --- /dev/null +++ b/CmdLine/GitAnnex/Options.hs @@ -0,0 +1,416 @@ +{- git-annex command-line option parsing + - + - Copyright 2010-2018 Joey Hess + - + - Licensed under the GNU GPL version 3 or higher. + -} + +{-# LANGUAGE TypeSynonymInstances, FlexibleInstances, CPP #-} + +module CmdLine.GitAnnex.Options where + +import Options.Applicative +#if ! MIN_VERSION_optparse_applicative(0,14,1) +import Options.Applicative.Builder.Internal +#endif +import Control.Concurrent +import qualified Data.Map as M + +import Annex.Common +import qualified Git.Config +import qualified Git.Construct +import Git.Remote +import Git.Types +import Types.Key +import Types.TrustLevel +import Types.NumCopies +import Types.Messages +import Types.Command +import Types.DeferredParse +import Types.DesktopNotify +import Types.Concurrency +import qualified Annex +import qualified Remote +import qualified Limit +import qualified Limit.Wanted +import CmdLine.Option +import CmdLine.Usage +import CmdLine.GlobalSetter +import qualified Backend +import qualified Types.Backend as Backend +import Utility.HumanTime + +-- Global options that are accepted by all git-annex sub-commands, +-- although not always used. +gitAnnexGlobalOptions :: [GlobalOption] +gitAnnexGlobalOptions = commonGlobalOptions ++ + [ globalSetter setnumcopies $ option auto + ( long "numcopies" <> short 'N' <> metavar paramNumber + <> help "override default number of copies" + <> hidden + ) + , globalSetter (Remote.forceTrust Trusted) $ strOption + ( long "trust" <> metavar paramRemote + <> help "override trust setting" + <> hidden + <> completeRemotes + ) + , globalSetter (Remote.forceTrust SemiTrusted) $ strOption + ( long "semitrust" <> metavar paramRemote + <> help "override trust setting back to default" + <> hidden + <> completeRemotes + ) + , globalSetter (Remote.forceTrust UnTrusted) $ strOption + ( long "untrust" <> metavar paramRemote + <> help "override trust setting to untrusted" + <> hidden + <> completeRemotes + ) + , globalSetter setgitconfig $ strOption + ( long "config" <> short 'c' <> metavar "NAME=VALUE" + <> help "override git configuration setting" + <> hidden + ) + , globalSetter setuseragent $ strOption + ( long "user-agent" <> metavar paramName + <> help "override default User-Agent" + <> hidden + ) + , globalFlag (Annex.setFlag "trustglacier") + ( long "trust-glacier" + <> help "Trust Amazon Glacier inventory" + <> hidden + ) + , globalFlag (setdesktopnotify mkNotifyFinish) + ( long "notify-finish" + <> help "show desktop notification after transfer finishes" + <> hidden + ) + , globalFlag (setdesktopnotify mkNotifyStart) + ( long "notify-start" + <> help "show desktop notification after transfer starts" + <> hidden + ) + ] + where + setnumcopies n = Annex.changeState $ \s -> s { Annex.forcenumcopies = Just $ NumCopies n } + setuseragent v = Annex.changeState $ \s -> s { Annex.useragent = Just v } + setgitconfig v = Annex.adjustGitRepo $ \r -> Git.Config.store v $ + r { gitGlobalOpts = gitGlobalOpts r ++ [Param "-c", Param v] } + setdesktopnotify v = Annex.changeState $ \s -> s { Annex.desktopnotify = Annex.desktopnotify s <> v } + +{- Parser that accepts all non-option params. -} +cmdParams :: CmdParamsDesc -> Parser CmdParams +cmdParams paramdesc = many $ argument str + ( metavar paramdesc + <> action "file" + ) + +parseAutoOption :: Parser Bool +parseAutoOption = switch + ( long "auto" <> short 'a' + <> help "automatic mode" + ) + +parseRemoteOption :: RemoteName -> DeferredParse Remote +parseRemoteOption = DeferredParse + . (fromJust <$$> Remote.byNameWithUUID) + . Just + +-- | From or To a remote. +data FromToOptions r = From r | To r + +instance DeferredParseClass (FromToOptions [DeferredParse r]) where + finishParse (From l) = From <$> finishParse l + finishParse (To l) = To <$> finishParse l + +parseFromToOptions :: Parser (FromToOptions [DeferredParse Remote]) +parseFromToOptions = + (From . map parseRemoteOption <$> some parseFromOption) + <|> (To . map parseRemoteOption <$> some parseToOption) + +parseFromOption :: Parser RemoteName +parseFromOption = strOption + ( long "from" <> short 'f' <> metavar paramRemote + <> help "source remote" + <> completeRemotes + ) + +parseToOption :: Parser RemoteName +parseToOption = strOption + ( long "to" <> short 't' <> metavar paramRemote + <> help "destination remote" + <> completeRemotes + ) + +-- | Like FromToOptions, but with a special --to=here +type FromToHereOptions r = Either ToHere (FromToOptions r) + +data ToHere = ToHere + +parseFromToHereOptions :: Parser (FromToHereOptions [DeferredParse Remote]) +parseFromToHereOptions = parsefrom <|> parseto + where + parsefrom = Right . From . map parseRemoteOption <$> some parseFromOption + parseto = herespecialcase <$> some parseToOption + where + herespecialcase l + | any ishere l && all ishere l = Left ToHere + | any ishere l = fail "Cannot mix --to=here with --to=remote" + | otherwise = Right $ To $ map parseRemoteOption l + ishere "here" = True + ishere "." = True + ishere _ = False + +instance DeferredParseClass (FromToHereOptions [DeferredParse r]) where + finishParse = either (pure . Left) (Right <$$> finishParse) + +-- Options for acting on keys, rather than work tree files. +data KeyOptions + = WantAllKeys + | WantUnusedKeys + | WantFailedTransfers + | WantSpecificKey Key + | WantIncompleteKeys + | WantBranchKeys [Branch] + +parseKeyOptions :: Parser KeyOptions +parseKeyOptions = parseAllOption + <|> WantBranchKeys <$> some (option (str >>= pure . Ref) + ( long "branch" <> metavar paramRef + <> help "operate on files in the specified branch or treeish" + )) + <|> flag' WantUnusedKeys + ( long "unused" <> short 'U' + <> help "operate on files found by last run of git-annex unused" + ) + <|> (WantSpecificKey <$> option (str >>= parseKey) + ( long "key" <> metavar paramKey + <> help "operate on specified key" + )) + +parseFailedTransfersOption :: Parser KeyOptions +parseFailedTransfersOption = flag' WantFailedTransfers + ( long "failed" + <> help "operate on files that recently failed to be transferred" + ) + +parseIncompleteOption :: Parser KeyOptions +parseIncompleteOption = flag' WantIncompleteKeys + ( long "incomplete" + <> help "resume previous downloads" + ) + +parseAllOption :: Parser KeyOptions +parseAllOption = flag' WantAllKeys + ( long "all" <> short 'A' + <> help "operate on all versions of all files" + ) + +parseKey :: Monad m => String -> m Key +parseKey = maybe (fail "invalid key") return . file2key + +-- Options to match properties of annexed files. +annexedMatchingOptions :: [GlobalOption] +annexedMatchingOptions = concat + [ nonWorkTreeMatchingOptions' + , fileMatchingOptions' + , combiningOptions + , timeLimitOption + ] + +-- Matching options that don't need to examine work tree files. +nonWorkTreeMatchingOptions :: [GlobalOption] +nonWorkTreeMatchingOptions = nonWorkTreeMatchingOptions' ++ combiningOptions + +nonWorkTreeMatchingOptions' :: [GlobalOption] +nonWorkTreeMatchingOptions' = + [ globalSetter Limit.addIn $ strOption + ( long "in" <> short 'i' <> metavar paramRemote + <> help "match files present in a remote" + <> hidden + <> completeRemotes + ) + , globalSetter Limit.addCopies $ strOption + ( long "copies" <> short 'C' <> metavar paramRemote + <> help "skip files with fewer copies" + <> hidden + ) + , globalSetter (Limit.addLackingCopies False) $ strOption + ( long "lackingcopies" <> metavar paramNumber + <> help "match files that need more copies" + <> hidden + ) + , globalSetter (Limit.addLackingCopies True) $ strOption + ( long "approxlackingcopies" <> metavar paramNumber + <> help "match files that need more copies (faster)" + <> hidden + ) + , globalSetter Limit.addInBackend $ strOption + ( long "inbackend" <> short 'B' <> metavar paramName + <> help "match files using a key-value backend" + <> hidden + <> completeBackends + ) + , globalFlag Limit.addSecureHash + ( long "securehash" + <> help "match files using a cryptographically secure hash" + <> hidden + ) + , globalSetter Limit.addInAllGroup $ strOption + ( long "inallgroup" <> metavar paramGroup + <> help "match files present in all remotes in a group" + <> hidden + ) + , globalSetter Limit.addMetaData $ strOption + ( long "metadata" <> metavar "FIELD=VALUE" + <> help "match files with attached metadata" + <> hidden + ) + , globalFlag Limit.Wanted.addWantGet + ( long "want-get" + <> help "match files the repository wants to get" + <> hidden + ) + , globalFlag Limit.Wanted.addWantDrop + ( long "want-drop" + <> help "match files the repository wants to drop" + <> hidden + ) + , globalSetter Limit.addAccessedWithin $ option (str >>= parseDuration) + ( long "accessedwithin" + <> metavar paramTime + <> help "match files accessed within a time interval" + <> hidden + ) + ] + +-- Options to match files which may not yet be annexed. +fileMatchingOptions :: [GlobalOption] +fileMatchingOptions = fileMatchingOptions' ++ combiningOptions + +fileMatchingOptions' :: [GlobalOption] +fileMatchingOptions' = + [ globalSetter Limit.addExclude $ strOption + ( long "exclude" <> short 'x' <> metavar paramGlob + <> help "skip files matching the glob pattern" + <> hidden + ) + , globalSetter Limit.addInclude $ strOption + ( long "include" <> short 'I' <> metavar paramGlob + <> help "limit to files matching the glob pattern" + <> hidden + ) + , globalSetter Limit.addLargerThan $ strOption + ( long "largerthan" <> metavar paramSize + <> help "match files larger than a size" + <> hidden + ) + , globalSetter Limit.addSmallerThan $ strOption + ( long "smallerthan" <> metavar paramSize + <> help "match files smaller than a size" + <> hidden + ) + ] + +combiningOptions :: [GlobalOption] +combiningOptions = + [ longopt "not" "negate next option" + , longopt "and" "both previous and next option must match" + , longopt "or" "either previous or next option must match" + , shortopt '(' "open group of options" + , shortopt ')' "close group of options" + ] + where + longopt o h = globalFlag (Limit.addToken o) ( long o <> help h <> hidden ) + shortopt o h = globalFlag (Limit.addToken [o]) ( short o <> help h <> hidden ) + +jsonOptions :: [GlobalOption] +jsonOptions = + [ globalFlag (Annex.setOutput (JSONOutput stdjsonoptions)) + ( long "json" <> short 'j' + <> help "enable JSON output" + <> hidden + ) + , globalFlag (Annex.setOutput (JSONOutput jsonerrormessagesoptions)) + ( long "json-error-messages" + <> help "include error messages in JSON" + <> hidden + ) + ] + where + stdjsonoptions = JSONOptions + { jsonProgress = False + , jsonErrorMessages = False + } + jsonerrormessagesoptions = stdjsonoptions { jsonErrorMessages = True } + +jsonProgressOption :: [GlobalOption] +jsonProgressOption = + [ globalFlag (Annex.setOutput (JSONOutput jsonoptions)) + ( long "json-progress" + <> help "include progress in JSON output" + <> hidden + ) + ] + where + jsonoptions = JSONOptions + { jsonProgress = True + , jsonErrorMessages = False + } + +-- Note that a command that adds this option should wrap its seek +-- action in `allowConcurrentOutput`. +jobsOption :: [GlobalOption] +jobsOption = + [ globalSetter set $ + option auto + ( long "jobs" <> short 'J' <> metavar paramNumber + <> help "enable concurrent jobs" + <> hidden + ) + ] + where + set n = do + Annex.changeState $ \s -> s { Annex.concurrency = Concurrent n } + c <- liftIO getNumCapabilities + when (n > c) $ + liftIO $ setNumCapabilities n + +timeLimitOption :: [GlobalOption] +timeLimitOption = + [ globalSetter Limit.addTimeLimit $ option (str >>= parseDuration) + ( long "time-limit" <> short 'T' <> metavar paramTime + <> help "stop after the specified amount of time" + <> hidden + ) + ] + +data DaemonOptions = DaemonOptions + { foregroundDaemonOption :: Bool + , stopDaemonOption :: Bool + } + +parseDaemonOptions :: Parser DaemonOptions +parseDaemonOptions = DaemonOptions + <$> switch + ( long "foreground" + <> help "do not daemonize" + ) + <*> switch + ( long "stop" + <> help "stop daemon" + ) +completeRemotes :: HasCompleter f => Mod f a +completeRemotes = completer $ mkCompleter $ \input -> do + r <- maybe (pure Nothing) (Just <$$> Git.Config.read) + =<< Git.Construct.fromCwd + return $ filter (input `isPrefixOf`) $ + map remoteKeyToRemoteName $ + filter isRemoteKey $ + maybe [] (M.keys . config) r + +completeBackends :: HasCompleter f => Mod f a +completeBackends = completeWith $ + map (formatKeyVariety . Backend.backendVariety) Backend.list diff --git a/CmdLine/GitAnnexShell.hs b/CmdLine/GitAnnexShell.hs new file mode 100644 index 0000000000..c18aa1f59e --- /dev/null +++ b/CmdLine/GitAnnexShell.hs @@ -0,0 +1,174 @@ +{- git-annex-shell main program + - + - Copyright 2010-2018 Joey Hess + - + - Licensed under the GNU GPL version 3 or higher. + -} + +module CmdLine.GitAnnexShell where + +import Annex.Common +import qualified Git.Construct +import qualified Git.Config +import CmdLine +import CmdLine.GlobalSetter +import Command +import Annex.UUID +import CmdLine.GitAnnexShell.Checks +import CmdLine.GitAnnexShell.Fields +import Remote.GCrypt (getGCryptUUID) +import P2P.Protocol (ServerMode(..)) + +import qualified Command.ConfigList +import qualified Command.InAnnex +import qualified Command.LockContent +import qualified Command.DropKey +import qualified Command.RecvKey +import qualified Command.SendKey +import qualified Command.TransferInfo +import qualified Command.Commit +import qualified Command.NotifyChanges +import qualified Command.GCryptSetup +import qualified Command.P2PStdIO + +import qualified Data.Map as M + +cmdsMap :: M.Map ServerMode [Command] +cmdsMap = M.fromList $ map mk + [ (ServeReadOnly, readonlycmds) + , (ServeAppendOnly, appendcmds) + , (ServeReadWrite, allcmds) + ] + where + readonlycmds = + [ Command.ConfigList.cmd + , gitAnnexShellCheck Command.InAnnex.cmd + , gitAnnexShellCheck Command.LockContent.cmd + , gitAnnexShellCheck Command.SendKey.cmd + , gitAnnexShellCheck Command.TransferInfo.cmd + , gitAnnexShellCheck Command.NotifyChanges.cmd + -- p2pstdio checks the enviroment variables to + -- determine the security policy to use + , gitAnnexShellCheck Command.P2PStdIO.cmd + ] + appendcmds = readonlycmds ++ + [ gitAnnexShellCheck Command.RecvKey.cmd + , gitAnnexShellCheck Command.Commit.cmd + ] + allcmds = + [ gitAnnexShellCheck Command.DropKey.cmd + , Command.GCryptSetup.cmd + ] + + mk (s, l) = (s, map (adddirparam . noMessages) l) + adddirparam c = c { cmdparamdesc = "DIRECTORY " ++ cmdparamdesc c } + +cmdsFor :: ServerMode -> [Command] +cmdsFor = fromMaybe [] . flip M.lookup cmdsMap + +cmdsList :: [Command] +cmdsList = concat $ M.elems cmdsMap + +globalOptions :: [GlobalOption] +globalOptions = + globalSetter checkUUID (strOption + ( long "uuid" <> metavar paramUUID + <> help "local repository uuid" + )) + : commonGlobalOptions + where + checkUUID expected = getUUID >>= check + where + check u | u == toUUID expected = noop + check NoUUID = checkGCryptUUID expected + check u = unexpectedUUID expected u + checkGCryptUUID expected = check =<< getGCryptUUID True =<< gitRepo + where + check (Just u) | u == toUUID expected = noop + check Nothing = unexpected expected "uninitialized repository" + check (Just u) = unexpectedUUID expected u + unexpectedUUID expected u = unexpected expected $ "UUID " ++ fromUUID u + unexpected expected s = giveup $ + "expected repository UUID " ++ expected ++ " but found " ++ s + +run :: [String] -> IO () +run [] = failure +-- skip leading -c options, passed by eg, ssh +run ("-c":p) = run p +-- a command can be either a builtin or something to pass to git-shell +run c@(cmd:dir:params) + | cmd `elem` builtins = builtin cmd dir params + | otherwise = external c +run c@(cmd:_) + -- Handle the case of being the user's login shell. It will be passed + -- a single string containing all the real parameters. + | "git-annex-shell " `isPrefixOf` cmd = run $ drop 1 $ shellUnEscape cmd + | cmd `elem` builtins = failure + | otherwise = external c + +builtins :: [String] +builtins = map cmdname cmdsList + +builtin :: String -> String -> [String] -> IO () +builtin cmd dir params = do + unless (cmd `elem` map cmdname (cmdsFor ServeReadOnly)) + checkNotReadOnly + unless (cmd `elem` map cmdname (cmdsFor ServeAppendOnly)) + checkNotAppendOnly + checkDirectory $ Just dir + let (params', fieldparams, opts) = partitionParams params + rsyncopts = ("RsyncOptions", unwords opts) + fields = rsyncopts : filter checkField (parseFields fieldparams) + dispatch False (cmd : params') cmdsList globalOptions fields mkrepo + "git-annex-shell" + "Restricted login shell for git-annex only SSH access" + where + mkrepo = do + r <- Git.Construct.repoAbsPath dir >>= Git.Construct.fromAbsPath + Git.Config.read r + `catchIO` \_ -> do + hn <- fromMaybe "unknown" <$> getHostname + giveup $ "failed to read git config of git repository in " ++ hn ++ " on " ++ dir ++ "; perhaps this repository is not set up correctly or has moved" + +external :: [String] -> IO () +external params = do + {- Normal git-shell commands all have the directory as their last + - parameter. -} + let lastparam = lastMaybe =<< shellUnEscape <$> lastMaybe params + (params', _, _) = partitionParams params + checkDirectory lastparam + checkNotLimited + unlessM (boolSystem "git-shell" $ map Param $ "-c":params') $ + giveup "git-shell failed" + +{- Split the input list into 3 groups separated with a double dash --. + - Parameters between two -- markers are field settings, in the form: + - field=value field=value + - + - Parameters after the last -- are the command itself and its arguments e.g., + - rsync --bandwidth=100. + -} +partitionParams :: [String] -> ([String], [String], [String]) +partitionParams ps = case segment (== "--") ps of + params:fieldparams:rest -> ( params, fieldparams, intercalate ["--"] rest ) + [params] -> (params, [], []) + _ -> ([], [], []) + +parseFields :: [String] -> [(String, String)] +parseFields = map (separate (== '=')) + +{- Only allow known fields to be set, ignore others. + - Make sure that field values make sense. -} +checkField :: (String, String) -> Bool +checkField (field, val) + | field == fieldName remoteUUID = fieldCheck remoteUUID val + | field == fieldName associatedFile = fieldCheck associatedFile val + | field == fieldName unlocked = fieldCheck unlocked val + | field == fieldName direct = fieldCheck direct val + | field == fieldName autoInit = fieldCheck autoInit val + | otherwise = False + +failure :: IO () +failure = giveup $ "bad parameters\n\n" ++ usage h cmdsList + where + h = "git-annex-shell [-c] command [parameters ...] [option ...]" diff --git a/CmdLine/GitAnnexShell/Checks.hs b/CmdLine/GitAnnexShell/Checks.hs new file mode 100644 index 0000000000..bcef88ce28 --- /dev/null +++ b/CmdLine/GitAnnexShell/Checks.hs @@ -0,0 +1,82 @@ +{- git-annex-shell checks + - + - Copyright 2012 Joey Hess + - + - Licensed under the GNU GPL version 3 or higher. + -} + +module CmdLine.GitAnnexShell.Checks where + +import Annex.Common +import Command +import qualified Annex +import Annex.Init +import Utility.UserInfo +import Utility.Env + +limitedEnv :: String +limitedEnv = "GIT_ANNEX_SHELL_LIMITED" + +checkNotLimited :: IO () +checkNotLimited = checkEnv limitedEnv + +readOnlyEnv :: String +readOnlyEnv = "GIT_ANNEX_SHELL_READONLY" + +checkNotReadOnly :: IO () +checkNotReadOnly = checkEnv readOnlyEnv + +appendOnlyEnv :: String +appendOnlyEnv = "GIT_ANNEX_SHELL_APPENDONLY" + +checkNotAppendOnly :: IO () +checkNotAppendOnly = checkEnv appendOnlyEnv + +checkEnv :: String -> IO () +checkEnv var = checkEnvSet var >>= \case + False -> noop + True -> giveup $ "Action blocked by " ++ var + +checkEnvSet :: String -> IO Bool +checkEnvSet var = getEnv var >>= return . \case + Nothing -> False + Just "" -> False + Just _ -> True + +checkDirectory :: Maybe FilePath -> IO () +checkDirectory mdir = do + v <- getEnv "GIT_ANNEX_SHELL_DIRECTORY" + case (v, mdir) of + (Nothing, _) -> noop + (Just d, Nothing) -> req d Nothing + (Just d, Just dir) + | d `equalFilePath` dir -> noop + | otherwise -> do + home <- myHomeDir + d' <- canondir home d + dir' <- canondir home dir + if d' `equalFilePath` dir' + then noop + else req d' (Just dir') + where + req d mdir' = giveup $ unwords + [ "Only allowed to access" + , d + , maybe "and could not determine directory from command line" ("not " ++) mdir' + ] + + {- A directory may start with ~/ or in some cases, even /~/, + - or could just be relative to home, or of course could + - be absolute. -} + canondir home d + | "~/" `isPrefixOf` d = return d + | "/~/" `isPrefixOf` d = return $ drop 1 d + | otherwise = relHome $ absPathFrom home d + +{- Modifies a Command to check that it is run in either a git-annex + - repository, or a repository with a gcrypt-id set. -} +gitAnnexShellCheck :: Command -> Command +gitAnnexShellCheck = addCheck okforshell . dontCheck repoExists + where + okforshell = unlessM (isInitialized <||> isJust . gcryptId <$> Annex.getGitConfig) $ + giveup "Not a git-annex or gcrypt repository." diff --git a/CmdLine/GitAnnexShell/Fields.hs b/CmdLine/GitAnnexShell/Fields.hs new file mode 100644 index 0000000000..efe22971dd --- /dev/null +++ b/CmdLine/GitAnnexShell/Fields.hs @@ -0,0 +1,42 @@ +{- git-annex-shell fields + - + - Copyright 2012 Joey Hess + - + - Licensed under the GNU GPL version 3 or higher. + -} + +module CmdLine.GitAnnexShell.Fields where + +import Annex.Common +import qualified Annex +import Git.FilePath + +import Data.Char + +{- A field, stored in Annex state, with a value sanity checker. -} +data Field = Field + { fieldName :: String + , fieldCheck :: String -> Bool + } + +getField :: Field -> Annex (Maybe String) +getField = Annex.getField . fieldName + +remoteUUID :: Field +remoteUUID = Field "remoteuuid" $ + -- does it look like a UUID? + all (\c -> isAlphaNum c || c == '-') + +associatedFile :: Field +associatedFile = Field "associatedfile" $ \f -> + -- is the file a safe relative filename? + not (absoluteGitPath f) && not ("../" `isPrefixOf` f) + +direct :: Field +direct = Field "direct" $ \f -> f == "1" + +unlocked :: Field +unlocked = Field "unlocked" $ \f -> f == "1" + +autoInit :: Field +autoInit = Field "autoinit" $ \f -> f == "1" diff --git a/CmdLine/GitRemoteTorAnnex.hs b/CmdLine/GitRemoteTorAnnex.hs new file mode 100644 index 0000000000..7425c5b622 --- /dev/null +++ b/CmdLine/GitRemoteTorAnnex.hs @@ -0,0 +1,64 @@ +{- git-remote-tor-annex program + - + - Copyright 2016 Joey Hess + - + - Licensed under the GNU GPL version 3 or higher. + -} + +module CmdLine.GitRemoteTorAnnex where + +import Common +import qualified Annex +import qualified Git.CurrentRepo +import P2P.Protocol +import P2P.IO +import Utility.Tor +import Utility.AuthToken +import Annex.UUID +import P2P.Address +import P2P.Auth + +run :: [String] -> IO () +run (_remotename:address:[]) = forever $ + getLine >>= \case + "capabilities" -> putStrLn "connect" >> ready + "connect git-upload-pack" -> go UploadPack + "connect git-receive-pack" -> go ReceivePack + l -> error $ "git-remote-helpers protocol error at " ++ show l + where + (onionaddress, onionport) + | '/' `elem` address = parseAddressPort $ + reverse $ takeWhile (/= '/') $ reverse address + | otherwise = parseAddressPort address + go service = do + ready + connectService onionaddress onionport service >>= \case + Right exitcode -> exitWith exitcode + Left e -> giveup $ describeProtoFailure e + ready = do + putStrLn "" + hFlush stdout + +run (_remotename:[]) = giveup "remote address not configured" +run _ = giveup "expected remote name and address parameters" + +parseAddressPort :: String -> (OnionAddress, OnionPort) +parseAddressPort s = + let (a, sp) = separate (== ':') s + in case readish sp of + Nothing -> giveup "onion address must include port number" + Just p -> (OnionAddress a, p) + +connectService :: OnionAddress -> OnionPort -> Service -> IO (Either ProtoFailure ExitCode) +connectService address port service = do + state <- Annex.new =<< Git.CurrentRepo.get + Annex.eval state $ do + authtoken <- fromMaybe nullAuthToken + <$> loadP2PRemoteAuthToken (TorAnnex address port) + myuuid <- getUUID + g <- Annex.gitRepo + conn <- liftIO $ connectPeer g (TorAnnex address port) + runst <- liftIO $ mkRunState Client + liftIO $ runNetProto runst conn $ auth myuuid authtoken noop >>= \case + Just _theiruuid -> connect service stdin stdout + Nothing -> giveup $ "authentication failed, perhaps you need to set " ++ p2pAuthTokenEnv diff --git a/CmdLine/GlobalSetter.hs b/CmdLine/GlobalSetter.hs new file mode 100644 index 0000000000..b0447c6c83 --- /dev/null +++ b/CmdLine/GlobalSetter.hs @@ -0,0 +1,24 @@ +{- git-annex global options + - + - Copyright 2015 Joey Hess + - + - Licensed under the GNU GPL version 3 or higher. + -} + +module CmdLine.GlobalSetter where + +import Types.DeferredParse +import Common +import Annex + +import Options.Applicative + +globalFlag :: Annex () -> Mod FlagFields GlobalSetter -> GlobalOption +globalFlag setter = flag' (DeferredParse setter) + +globalSetter :: (v -> Annex ()) -> Parser v -> GlobalOption +globalSetter setter parser = DeferredParse . setter <$> parser + +combineGlobalOptions :: [GlobalOption] -> Parser GlobalSetter +combineGlobalOptions l = DeferredParse . mapM_ getParsed + <$> many (foldl1 (<|>) l) diff --git a/CmdLine/Option.hs b/CmdLine/Option.hs new file mode 100644 index 0000000000..b0fec3f092 --- /dev/null +++ b/CmdLine/Option.hs @@ -0,0 +1,63 @@ +{- common command-line options + - + - Copyright 2010-2011 Joey Hess + - + - Licensed under the GNU GPL version 3 or higher. + -} + +module CmdLine.Option where + +import Options.Applicative + +import Annex.Common +import CmdLine.Usage +import CmdLine.GlobalSetter +import qualified Annex +import Types.Messages +import Types.DeferredParse + +-- Global options accepted by both git-annex and git-annex-shell sub-commands. +commonGlobalOptions :: [GlobalOption] +commonGlobalOptions = + [ globalFlag (setforce True) + ( long "force" + <> help "allow actions that may lose annexed data" + <> hidden + ) + , globalFlag (setfast True) + ( long "fast" <> short 'F' + <> help "avoid slow operations" + <> hidden + ) + , globalFlag (Annex.setOutput QuietOutput) + ( long "quiet" <> short 'q' + <> help "avoid verbose output" + <> hidden + ) + , globalFlag (Annex.setOutput NormalOutput) + ( long "verbose" <> short 'v' + <> help "allow verbose output (default)" + <> hidden + ) + , globalFlag setdebug + ( long "debug" <> short 'd' + <> help "show debug messages" + <> hidden + ) + , globalFlag unsetdebug + ( long "no-debug" + <> help "don't show debug messages" + <> hidden + ) + , globalSetter setforcebackend $ strOption + ( long "backend" <> short 'b' <> metavar paramName + <> help "specify key-value backend to use" + <> hidden + ) + ] + where + setforce v = Annex.changeState $ \s -> s { Annex.force = v } + setfast v = Annex.changeState $ \s -> s { Annex.fast = v } + setforcebackend v = Annex.changeState $ \s -> s { Annex.forcebackend = Just v } + setdebug = Annex.changeGitConfig $ \c -> c { annexDebug = True } + unsetdebug = Annex.changeGitConfig $ \c -> c { annexDebug = False } diff --git a/CmdLine/Seek.hs b/CmdLine/Seek.hs new file mode 100644 index 0000000000..677fc4b684 --- /dev/null +++ b/CmdLine/Seek.hs @@ -0,0 +1,286 @@ +{- git-annex command seeking + - + - These functions find appropriate files or other things based on + - the values a user passes to a command, and prepare actions operating + - on them. + - + - Copyright 2010-2018 Joey Hess + - + - Licensed under the GNU GPL version 3 or higher. + -} + +module CmdLine.Seek where + +import Annex.Common +import Types.Command +import Types.FileMatcher +import qualified Annex +import qualified Git +import qualified Git.Command +import qualified Git.LsFiles as LsFiles +import qualified Git.LsTree as LsTree +import Git.FilePath +import qualified Limit +import CmdLine.GitAnnex.Options +import Logs.Location +import Logs.Unused +import Types.Transfer +import Logs.Transfer +import Remote.List +import qualified Remote +import Annex.CatFile +import Annex.Content +import Annex.InodeSentinal +import qualified Database.Keys + +withFilesInGit :: (FilePath -> CommandSeek) -> [WorkTreeItem] -> CommandSeek +withFilesInGit a l = seekActions $ prepFiltered a $ + seekHelper LsFiles.inRepo l + +withFilesInGitNonRecursive :: String -> (FilePath -> CommandSeek) -> [WorkTreeItem] -> CommandSeek +withFilesInGitNonRecursive needforce a l = ifM (Annex.getState Annex.force) + ( withFilesInGit a l + , if null l + then giveup needforce + else seekActions $ prepFiltered a (getfiles [] l) + ) + where + getfiles c [] = return (reverse c) + getfiles c ((WorkTreeItem p):ps) = do + (fs, cleanup) <- inRepo $ LsFiles.inRepo [p] + case fs of + [f] -> do + void $ liftIO $ cleanup + getfiles (f:c) ps + [] -> do + void $ liftIO $ cleanup + getfiles c ps + _ -> giveup needforce + +withFilesNotInGit :: Bool -> (FilePath -> CommandSeek) -> [WorkTreeItem] -> CommandSeek +withFilesNotInGit skipdotfiles a l + | skipdotfiles = do + {- dotfiles are not acted on unless explicitly listed -} + files <- filter (not . dotfile) <$> + seekunless (null ps && not (null l)) ps + dotfiles <- seekunless (null dotps) dotps + go (files++dotfiles) + | otherwise = go =<< seekunless False l + where + (dotps, ps) = partition (\(WorkTreeItem f) -> dotfile f) l + seekunless True _ = return [] + seekunless _ l' = do + force <- Annex.getState Annex.force + g <- gitRepo + liftIO $ Git.Command.leaveZombie + <$> LsFiles.notInRepo force (map (\(WorkTreeItem f) -> f) l') g + go fs = seekActions $ prepFiltered a $ + return $ concat $ segmentPaths (map (\(WorkTreeItem f) -> f) l) fs + +withFilesInRefs :: ((FilePath, Key) -> CommandSeek) -> [Git.Ref] -> CommandSeek +withFilesInRefs a = mapM_ go + where + go r = do + matcher <- Limit.getMatcher + (l, cleanup) <- inRepo $ LsTree.lsTree r + forM_ l $ \i -> do + let f = getTopFilePath $ LsTree.file i + catKey (LsTree.sha i) >>= \case + Nothing -> noop + Just k -> whenM (matcher $ MatchingKey k) $ + a (f, k) + liftIO $ void cleanup + +withPathContents :: ((FilePath, FilePath) -> CommandSeek) -> CmdParams -> CommandSeek +withPathContents a params = do + matcher <- Limit.getMatcher + forM_ params $ \p -> do + fs <- liftIO $ get p + forM fs $ \f -> + whenM (checkmatch matcher f) $ + a f + where + get p = ifM (isDirectory <$> getFileStatus p) + ( map (\f -> (f, makeRelative (parentDir p) f)) + <$> dirContentsRecursiveSkipping (".git" `isSuffixOf`) True p + , return [(p, takeFileName p)] + ) + checkmatch matcher (f, relf) = matcher $ MatchingFile $ FileInfo + { currFile = f + , matchFile = relf + } + +withWords :: ([String] -> CommandSeek) -> CmdParams -> CommandSeek +withWords a params = seekActions $ return [a params] + +withStrings :: (String -> CommandSeek) -> CmdParams -> CommandSeek +withStrings a params = seekActions $ return $ map a params + +withPairs :: ((String, String) -> CommandSeek) -> CmdParams -> CommandSeek +withPairs a params = seekActions $ return $ map a $ pairs [] params + where + pairs c [] = reverse c + pairs c (x:y:xs) = pairs ((x,y):c) xs + pairs _ _ = giveup "expected pairs" + +withFilesToBeCommitted :: (FilePath -> CommandSeek) -> [WorkTreeItem] -> CommandSeek +withFilesToBeCommitted a l = seekActions $ prepFiltered a $ + seekHelper LsFiles.stagedNotDeleted l + +withFilesOldUnlocked :: (FilePath -> CommandSeek) -> [WorkTreeItem] -> CommandSeek +withFilesOldUnlocked = withFilesOldUnlocked' LsFiles.typeChanged + +{- Unlocked files before v6 have changed type from a symlink to a regular file. + - + - Furthermore, unlocked files used to be a git-annex symlink, + - not some other sort of symlink. + -} +withFilesOldUnlocked' :: ([FilePath] -> Git.Repo -> IO ([FilePath], IO Bool)) -> (FilePath -> CommandSeek) -> [WorkTreeItem] -> CommandSeek +withFilesOldUnlocked' typechanged a l = seekActions $ + prepFiltered a unlockedfiles + where + unlockedfiles = filterM isOldUnlocked =<< seekHelper typechanged l + +isOldUnlocked :: FilePath -> Annex Bool +isOldUnlocked f = liftIO (notSymlink f) <&&> + (isJust <$> catKeyFile f <||> isJust <$> catKeyFileHEAD f) + +withFilesOldUnlockedToBeCommitted :: (FilePath -> CommandSeek) -> [WorkTreeItem] -> CommandSeek +withFilesOldUnlockedToBeCommitted = withFilesOldUnlocked' LsFiles.typeChangedStaged + +{- v6 unlocked pointer files that are staged, and whose content has not been + - modified-} +withUnmodifiedUnlockedPointers :: (FilePath -> CommandSeek) -> [WorkTreeItem] -> CommandSeek +withUnmodifiedUnlockedPointers a l = seekActions $ + prepFiltered a unlockedfiles + where + unlockedfiles = filterM isV6UnmodifiedUnlocked + =<< seekHelper LsFiles.typeChangedStaged l + +isV6UnmodifiedUnlocked :: FilePath -> Annex Bool +isV6UnmodifiedUnlocked f = catKeyFile f >>= \case + Nothing -> return False + Just k -> sameInodeCache f =<< Database.Keys.getInodeCaches k + +{- Finds files that may be modified. -} +withFilesMaybeModified :: (FilePath -> CommandSeek) -> [WorkTreeItem] -> CommandSeek +withFilesMaybeModified a params = seekActions $ + prepFiltered a $ seekHelper LsFiles.modified params + +withKeys :: (Key -> CommandSeek) -> CmdParams -> CommandSeek +withKeys a l = seekActions $ return $ map (a . parse) l + where + parse p = fromMaybe (giveup "bad key") $ file2key p + +withNothing :: CommandSeek -> CmdParams -> CommandSeek +withNothing a [] = a +withNothing _ _ = giveup "This command takes no parameters." + +{- Handles the --all, --branch, --unused, --failed, --key, and + - --incomplete options, which specify particular keys to run an + - action on. + - + - In a bare repo, --all is the default. + - + - Otherwise falls back to a regular CommandSeek action on + - whatever params were passed. + -} +withKeyOptions + :: Maybe KeyOptions + -> Bool + -> ((Key, ActionItem) -> CommandSeek) + -> ([WorkTreeItem] -> CommandSeek) + -> [WorkTreeItem] + -> CommandSeek +withKeyOptions ko auto keyaction = withKeyOptions' ko auto mkkeyaction + where + mkkeyaction = do + matcher <- Limit.getMatcher + return $ \v -> + whenM (matcher $ MatchingKey $ fst v) $ + keyaction v + +withKeyOptions' + :: Maybe KeyOptions + -> Bool + -> Annex ((Key, ActionItem) -> Annex ()) + -> ([WorkTreeItem] -> CommandSeek) + -> [WorkTreeItem] + -> CommandSeek +withKeyOptions' ko auto mkkeyaction fallbackaction params = do + bare <- fromRepo Git.repoIsLocalBare + when (auto && bare) $ + giveup "Cannot use --auto in a bare repository" + case (null params, ko) of + (True, Nothing) + | bare -> noauto $ runkeyaction finishCheck loggedKeys + | otherwise -> fallbackaction params + (False, Nothing) -> fallbackaction params + (True, Just WantAllKeys) -> noauto $ runkeyaction finishCheck loggedKeys + (True, Just WantUnusedKeys) -> noauto $ runkeyaction (pure . Just) unusedKeys' + (True, Just WantFailedTransfers) -> noauto runfailedtransfers + (True, Just (WantSpecificKey k)) -> noauto $ runkeyaction (pure . Just) (return [k]) + (True, Just WantIncompleteKeys) -> noauto $ runkeyaction (pure . Just) incompletekeys + (True, Just (WantBranchKeys bs)) -> noauto $ runbranchkeys bs + (False, Just _) -> giveup "Can only specify one of file names, --all, --branch, --unused, --failed, --key, or --incomplete" + where + noauto a + | auto = giveup "Cannot use --auto with --all or --branch or --unused or --key or --incomplete" + | otherwise = a + incompletekeys = staleKeysPrune gitAnnexTmpObjectDir True + runkeyaction checker getks = do + keyaction <- mkkeyaction + ks <- getks + forM_ ks $ checker >=> maybe noop + (\k -> keyaction (k, mkActionItem k)) + runbranchkeys bs = do + keyaction <- mkkeyaction + forM_ bs $ \b -> do + (l, cleanup) <- inRepo $ LsTree.lsTree b + forM_ l $ \i -> do + let bfp = mkActionItem $ BranchFilePath b (LsTree.file i) + maybe noop (\k -> keyaction (k, bfp)) + =<< catKey (LsTree.sha i) + unlessM (liftIO cleanup) $ + error ("git ls-tree " ++ Git.fromRef b ++ " failed") + runfailedtransfers = do + keyaction <- mkkeyaction + rs <- remoteList + ts <- concat <$> mapM (getFailedTransfers . Remote.uuid) rs + forM_ ts $ \(t, i) -> + keyaction (transferKey t, mkActionItem (t, i)) + +prepFiltered :: (FilePath -> CommandSeek) -> Annex [FilePath] -> Annex [CommandSeek] +prepFiltered a fs = do + matcher <- Limit.getMatcher + map (process matcher) <$> fs + where + process matcher f = whenM (matcher $ MatchingFile $ FileInfo f f) $ a f + +seekActions :: Annex [CommandSeek] -> Annex () +seekActions gen = sequence_ =<< gen + +seekHelper :: ([FilePath] -> Git.Repo -> IO ([FilePath], IO Bool)) -> [WorkTreeItem] -> Annex [FilePath] +seekHelper a l = inRepo $ \g -> + concat . concat <$> forM (segmentXargsOrdered l') + (runSegmentPaths (\fs -> Git.Command.leaveZombie <$> a fs g)) + where + l' = map (\(WorkTreeItem f) -> f) l + +-- An item in the work tree, which may be a file or a directory. +newtype WorkTreeItem = WorkTreeItem FilePath + +-- Many git commands seek work tree items matching some criteria, +-- and silently skip over anything that does not exist. But users expect +-- an error message when one of the files they provided as a command-line +-- parameter doesn't exist, so this checks that each exists. +workTreeItems :: CmdParams -> Annex [WorkTreeItem] +workTreeItems ps = do + forM_ ps $ \p -> + unlessM (isJust <$> liftIO (catchMaybeIO $ getSymbolicLinkStatus p)) $ do + toplevelWarning False (p ++ " not found") + Annex.incError + return (map WorkTreeItem ps) + +notSymlink :: FilePath -> IO Bool +notSymlink f = liftIO $ not . isSymbolicLink <$> getSymbolicLinkStatus f diff --git a/CmdLine/Usage.hs b/CmdLine/Usage.hs new file mode 100644 index 0000000000..6dd2d053d9 --- /dev/null +++ b/CmdLine/Usage.hs @@ -0,0 +1,110 @@ +{- git-annex usage messages + - + - Copyright 2010-2015 Joey Hess + - + - Licensed under the GNU GPL version 3 or higher. + -} + +module CmdLine.Usage where + +import Annex.Common +import Types.Command + +usageMessage :: String -> String +usageMessage s = "Usage: " ++ s + +usage :: String -> [Command] -> String +usage header cmds = unlines $ usageMessage header : commandList cmds + +{- Commands listed by section, with breif usage and description. -} +commandList :: [Command] -> [String] +commandList cmds = concatMap go [minBound..] + where + go section + | null cs = [] + | otherwise = + [ "" + , descSection section ++ ":" + , "" + ] ++ map cmdline cs + where + cs = filter (\c -> cmdsection c == section) scmds + cmdline c = concat + [ cmdname c + , namepad (cmdname c) + , cmdparamdesc c + , descpad (cmdparamdesc c) + , cmddesc c + ] + pad n s = replicate (n - length s) ' ' + namepad = pad $ longest cmdname + 1 + descpad = pad $ longest cmdparamdesc + 2 + longest f = foldl max 0 $ map (length . f) cmds + scmds = sort cmds + + +{- Descriptions of params used in usage messages. -} +paramPaths :: String +paramPaths = paramRepeating paramPath -- most often used +paramPath :: String +paramPath = "PATH" +paramKey :: String +paramKey = "KEY" +paramDesc :: String +paramDesc = "DESC" +paramUrl :: String +paramUrl = "URL" +paramNumber :: String +paramNumber = "NUMBER" +paramNumRange :: String +paramNumRange = "NUM|RANGE" +paramRemote :: String +paramRemote = "REMOTE" +paramField :: String +paramField = "FIELD" +paramGlob :: String +paramGlob = "GLOB" +paramName :: String +paramName = "NAME" +paramValue :: String +paramValue = "VALUE" +paramUUID :: String +paramUUID = "UUID" +paramType :: String +paramType = "TYPE" +paramDate :: String +paramDate = "DATE" +paramTime :: String +paramTime = "TIME" +paramFormat :: String +paramFormat = "FORMAT" +paramFile :: String +paramFile = "FILE" +paramRef :: String +paramRef = "REF" +paramRefSpec :: String +paramRefSpec = "REFSPEC" +paramGroup :: String +paramGroup = "GROUP" +paramExpression :: String +paramExpression = "EXPR" +paramSize :: String +paramSize = "SIZE" +paramAddress :: String +paramAddress = "ADDRESS" +paramItem :: String +paramItem = "ITEM" +paramTreeish :: String +paramTreeish = "TREEISH" +paramKeyValue :: String +paramKeyValue = "K=V" +paramNothing :: String +paramNothing = "" +paramRepeating :: String -> String +paramRepeating s = s ++ " ..." +paramOptional :: String -> String +paramOptional s = s +paramPair :: String -> String -> String +paramPair a b = a ++ " " ++ b +paramOr :: String -> String -> String +paramOr a b = a ++ " | " ++ b diff --git a/Command.hs b/Command.hs new file mode 100644 index 0000000000..b886e4fe27 --- /dev/null +++ b/Command.hs @@ -0,0 +1,135 @@ +{- git-annex command infrastructure + - + - Copyright 2010-2016 Joey Hess + - + - Licensed under the GNU GPL version 3 or higher. + -} + +module Command ( + module Command, + module ReExported +) where + +import Annex.Common as ReExported +import Annex.WorkTree as ReExported (whenAnnexed, ifAnnexed) +import Types.Command as ReExported +import Types.DeferredParse as ReExported +import CmdLine.Seek as ReExported +import CmdLine.Usage as ReExported +import CmdLine.Action as ReExported +import CmdLine.Option as ReExported +import CmdLine.GlobalSetter as ReExported +import CmdLine.GitAnnex.Options as ReExported +import CmdLine.Batch as ReExported +import Options.Applicative as ReExported hiding (command) +import qualified Annex +import qualified Git +import Annex.Init +import Config +import Utility.Daemon +import Types.Transfer +import Types.ActionItem +import Types.Messages + +{- Generates a normal Command -} +command :: String -> CommandSection -> String -> CmdParamsDesc -> (CmdParamsDesc -> CommandParser) -> Command +command name section desc paramdesc mkparser = + Command commonChecks False False name paramdesc + section desc (mkparser paramdesc) [] Nothing + +{- Simple option parser that takes all non-option params as-is. -} +withParams :: (CmdParams -> v) -> CmdParamsDesc -> Parser v +withParams mkseek paramdesc = mkseek <$> cmdParams paramdesc + +{- Uses the supplied option parser, which yields a deferred parse, + - and calls finishParse on the result before passing it to the + - CommandSeek constructor. -} +(<--<) :: DeferredParseClass a + => (a -> CommandSeek) + -> (CmdParamsDesc -> Parser a) + -> CmdParamsDesc + -> Parser CommandSeek +(<--<) mkseek optparser paramsdesc = + (mkseek <=< finishParse) <$> optparser paramsdesc + +{- Indicates that a command doesn't need to commit any changes to + - the git-annex branch. -} +noCommit :: Command -> Command +noCommit c = c { cmdnocommit = True } + +{- Indicates that a command should not output the usual messages when + - starting or stopping processing a file or other item. Unless --json mode + - is enabled, this also enables quiet output mode, so only things + - explicitly output by the command are shown and not progress messages + - etc. -} +noMessages :: Command -> Command +noMessages c = c { cmdnomessages = True } + +{- Undoes noMessages -} +allowMessages :: Annex () +allowMessages = do + outputType <$> Annex.getState Annex.output >>= \case + QuietOutput -> Annex.setOutput NormalOutput + _ -> noop + Annex.changeState $ \s -> s + { Annex.output = (Annex.output s) { implicitMessages = True } } + +{- Adds a fallback action to a command, that will be run if it's used + - outside a git repository. -} +noRepo :: (String -> Parser (IO ())) -> Command -> Command +noRepo a c = c { cmdnorepo = Just (a (cmdparamdesc c)) } + +{- Adds global options to a command. -} +withGlobalOptions :: [[GlobalOption]] -> Command -> Command +withGlobalOptions os c = c { cmdglobaloptions = cmdglobaloptions c ++ concat os } + +{- For start and perform stages to indicate what step to run next. -} +next :: a -> Annex (Maybe a) +next a = return $ Just a + +{- Or to indicate nothing needs to be done. -} +stop :: Annex (Maybe a) +stop = return Nothing + +{- Stops unless a condition is met. -} +stopUnless :: Annex Bool -> Annex (Maybe a) -> Annex (Maybe a) +stopUnless c a = ifM c ( a , stop ) + +{- When acting on a failed transfer, stops unless it was in the specified + - direction. -} +checkFailedTransferDirection :: ActionItem -> Direction -> Annex (Maybe a) -> Annex (Maybe a) +checkFailedTransferDirection ai d = stopUnless (pure check) + where + check = case actionItemTransferDirection ai of + Nothing -> True + Just d' -> d' == d + +commonChecks :: [CommandCheck] +commonChecks = [repoExists] + +repoExists :: CommandCheck +repoExists = CommandCheck 0 ensureInitialized + +notDirect :: Command -> Command +notDirect = addCheck $ whenM isDirect $ + giveup "You cannot run this command in a direct mode repository." + +notBareRepo :: Command -> Command +notBareRepo = addCheck $ whenM (fromRepo Git.repoIsLocalBare) $ + giveup "You cannot run this command in a bare repository." + +noDaemonRunning :: Command -> Command +noDaemonRunning = addCheck $ whenM (isJust <$> daemonpid) $ + giveup "You cannot run this command while git-annex watch or git-annex assistant is running." + where + daemonpid = liftIO . checkDaemon =<< fromRepo gitAnnexPidFile + +dontCheck :: CommandCheck -> Command -> Command +dontCheck check cmd = mutateCheck cmd $ \c -> filter (/= check) c + +addCheck :: Annex () -> Command -> Command +addCheck check cmd = mutateCheck cmd $ \c -> + CommandCheck (length c + 100) check : c + +mutateCheck :: Command -> ([CommandCheck] -> [CommandCheck]) -> Command +mutateCheck cmd@(Command { cmdcheck = c }) a = cmd { cmdcheck = a c } diff --git a/Command/Add.hs b/Command/Add.hs new file mode 100644 index 0000000000..840adc8f25 --- /dev/null +++ b/Command/Add.hs @@ -0,0 +1,156 @@ +{- git-annex command + - + - Copyright 2010-2017 Joey Hess + - + - Licensed under the GNU GPL version 3 or higher. + -} + +module Command.Add where + +import Command +import Annex.Ingest +import Logs.Location +import Annex.Content +import Annex.Content.Direct +import qualified Annex +import qualified Annex.Queue +import qualified Database.Keys +import Config +import Annex.FileMatcher +import Annex.Link +import Annex.Version +import Git.FilePath + +cmd :: Command +cmd = notBareRepo $ + withGlobalOptions [jobsOption, jsonOptions, fileMatchingOptions] $ + command "add" SectionCommon "add files to annex" + paramPaths (seek <$$> optParser) + +data AddOptions = AddOptions + { addThese :: CmdParams + , includeDotFiles :: Bool + , batchOption :: BatchMode + , updateOnly :: Bool + } + +optParser :: CmdParamsDesc -> Parser AddOptions +optParser desc = AddOptions + <$> cmdParams desc + <*> switch + ( long "include-dotfiles" + <> help "don't skip dotfiles" + ) + <*> parseBatchOption + <*> switch + ( long "update" + <> short 'u' + <> help "only update tracked files" + ) + +seek :: AddOptions -> CommandSeek +seek o = allowConcurrentOutput $ do + matcher <- largeFilesMatcher + let gofile file = ifM (checkFileMatcher matcher file <||> Annex.getState Annex.force) + ( start file + , ifM (annexAddSmallFiles <$> Annex.getGitConfig) + ( startSmall file + , stop + ) + ) + case batchOption o of + Batch fmt + | updateOnly o -> + giveup "--update --batch is not supported" + | otherwise -> batchFilesMatching fmt gofile + NoBatch -> do + l <- workTreeItems (addThese o) + let go a = a (commandAction . gofile) l + unless (updateOnly o) $ + go (withFilesNotInGit (not $ includeDotFiles o)) + go withFilesMaybeModified + ifM versionSupportsUnlockedPointers + ( go withUnmodifiedUnlockedPointers + , unlessM isDirect $ + go withFilesOldUnlocked + ) + +{- Pass file off to git-add. -} +startSmall :: FilePath -> CommandStart +startSmall file = do + showStart "add" file + next $ next $ addSmall file + +addSmall :: FilePath -> Annex Bool +addSmall file = do + showNote "non-large file; adding content to git repository" + addFile file + +addFile :: FilePath -> Annex Bool +addFile file = do + ps <- forceParams + Annex.Queue.addCommand "add" (ps++[Param "--"]) [file] + return True + +start :: FilePath -> CommandStart +start file = do + ifM versionSupportsUnlockedPointers + ( do + mk <- liftIO $ isPointerFile file + maybe go fixuppointer mk + , go + ) + where + go = ifAnnexed file addpresent add + add = liftIO (catchMaybeIO $ getSymbolicLinkStatus file) >>= \case + Nothing -> stop + Just s + | not (isRegularFile s) && not (isSymbolicLink s) -> stop + | otherwise -> do + showStart "add" file + next $ if isSymbolicLink s + then next $ addFile file + else perform file + addpresent key = ifM versionSupportsUnlockedPointers + ( liftIO (catchMaybeIO $ getSymbolicLinkStatus file) >>= \case + Just s | isSymbolicLink s -> fixuplink key + _ -> add + , ifM isDirect + ( liftIO (catchMaybeIO $ getSymbolicLinkStatus file) >>= \case + Just s | isSymbolicLink s -> fixuplink key + _ -> ifM (goodContent key file) + ( stop , add ) + , fixuplink key + ) + ) + fixuplink key = do + -- the annexed symlink is present but not yet added to git + showStart "add" file + liftIO $ removeFile file + addLink file key Nothing + next $ next $ + cleanup key =<< inAnnex key + fixuppointer key = do + -- the pointer file is present, but not yet added to git + showStart "add" file + Database.Keys.addAssociatedFile key =<< inRepo (toTopFilePath file) + next $ next $ addFile file + +perform :: FilePath -> CommandPerform +perform file = do + lockingfile <- not <$> addUnlocked + let cfg = LockDownConfig + { lockingFile = lockingfile + , hardlinkFileTmp = True + } + lockDown cfg file >>= ingestAdd >>= finish + where + finish (Just key) = next $ cleanup key True + finish Nothing = stop + +cleanup :: Key -> Bool -> CommandCleanup +cleanup key hascontent = do + maybeShowJSON $ JSONChunk [("key", key2file key)] + when hascontent $ + logStatus key InfoPresent + return True diff --git a/Command/AddUnused.hs b/Command/AddUnused.hs new file mode 100644 index 0000000000..c83c74e726 --- /dev/null +++ b/Command/AddUnused.hs @@ -0,0 +1,41 @@ +{- git-annex command + - + - Copyright 2012 Joey Hess + - + - Licensed under the GNU GPL version 3 or higher. + -} + +module Command.AddUnused where + +import Logs.Location +import Command +import Annex.Ingest +import Command.Unused (withUnusedMaps, UnusedMaps(..), startUnused) + +cmd :: Command +cmd = notDirect $ + command "addunused" SectionMaintenance + "add back unused files" + (paramRepeating paramNumRange) (withParams seek) + +seek :: CmdParams -> CommandSeek +seek = withUnusedMaps start + +start :: UnusedMaps -> Int -> CommandStart +start = startUnused "addunused" perform + (performOther "bad") + (performOther "tmp") + +perform :: Key -> CommandPerform +perform key = next $ do + logStatus key InfoPresent + addLink file key Nothing + return True + where + file = "unused." ++ key2file key + +{- The content is not in the annex, but in another directory, and + - it seems better to error out, rather than moving bad/tmp content into + - the annex. -} +performOther :: String -> Key -> CommandPerform +performOther other _ = giveup $ "cannot addunused " ++ other ++ "content" diff --git a/Command/AddUrl.hs b/Command/AddUrl.hs new file mode 100644 index 0000000000..815127a926 --- /dev/null +++ b/Command/AddUrl.hs @@ -0,0 +1,476 @@ +{- git-annex command + - + - Copyright 2011-2017 Joey Hess + - + - Licensed under the GNU GPL version 3 or higher. + -} + +module Command.AddUrl where + +import Network.URI + +import Command +import Backend +import qualified Annex +import qualified Annex.Url as Url +import qualified Backend.URL +import qualified Remote +import qualified Types.Remote as Remote +import qualified Command.Add +import Annex.Content +import Annex.Ingest +import Annex.CheckIgnore +import Annex.UUID +import Annex.YoutubeDl +import Logs.Web +import Types.KeySource +import Types.UrlContents +import Annex.FileMatcher +import Logs.Location +import Utility.Metered +import Utility.FileSystemEncoding +import Utility.HtmlDetect +import Utility.Path.Max +import qualified Annex.Transfer as Transfer + +cmd :: Command +cmd = notBareRepo $ withGlobalOptions [jobsOption, jsonOptions, jsonProgressOption] $ + command "addurl" SectionCommon "add urls to annex" + (paramRepeating paramUrl) (seek <$$> optParser) + +data AddUrlOptions = AddUrlOptions + { addUrls :: CmdParams + , pathdepthOption :: Maybe Int + , prefixOption :: Maybe String + , suffixOption :: Maybe String + , downloadOptions :: DownloadOptions + , batchOption :: BatchMode + , batchFilesOption :: Bool + } + +data DownloadOptions = DownloadOptions + { relaxedOption :: Bool + , rawOption :: Bool + , fileOption :: Maybe FilePath + } + +optParser :: CmdParamsDesc -> Parser AddUrlOptions +optParser desc = AddUrlOptions + <$> cmdParams desc + <*> optional (option auto + ( long "pathdepth" <> metavar paramNumber + <> help "number of url path components to use in filename" + )) + <*> optional (strOption + ( long "prefix" <> metavar paramValue + <> help "add a prefix to the filename" + )) + <*> optional (strOption + ( long "suffix" <> metavar paramValue + <> help "add a suffix to the filename" + )) + <*> parseDownloadOptions True + <*> parseBatchOption + <*> switch + ( long "with-files" + <> help "parse batch mode lines of the form \"$url $file\"" + ) + +parseDownloadOptions :: Bool -> Parser DownloadOptions +parseDownloadOptions withfileoption = DownloadOptions + <$> switch + ( long "relaxed" + <> help "skip size check" + ) + <*> switch + ( long "raw" + <> help "disable special handling for torrents, youtube-dl, etc" + ) + <*> if withfileoption + then optional (strOption + ( long "file" <> metavar paramFile + <> help "specify what file the url is added to" + )) + else pure Nothing + +seek :: AddUrlOptions -> CommandSeek +seek o = allowConcurrentOutput $ do + forM_ (addUrls o) (\u -> go (o, u)) + case batchOption o of + Batch fmt -> batchInput fmt (parseBatchInput o) go + NoBatch -> noop + where + go (o', u) = do + r <- Remote.claimingUrl u + if Remote.uuid r == webUUID || rawOption (downloadOptions o') + then void $ commandAction $ startWeb o' u + else checkUrl r o' u + +parseBatchInput :: AddUrlOptions -> String -> Either String (AddUrlOptions, URLString) +parseBatchInput o s + | batchFilesOption o = + let (u, f) = separate (== ' ') s + in if null u || null f + then Left ("parsed empty url or filename in input: " ++ s) + else Right (o { downloadOptions = (downloadOptions o) { fileOption = Just f } }, u) + | otherwise = Right (o, s) + +checkUrl :: Remote -> AddUrlOptions -> URLString -> Annex () +checkUrl r o u = do + pathmax <- liftIO $ fileNameLengthLimit "." + let deffile = fromMaybe (urlString2file u (pathdepthOption o) pathmax) (fileOption (downloadOptions o)) + go deffile =<< maybe + (error $ "unable to checkUrl of " ++ Remote.name r) + (tryNonAsync . flip id u) + (Remote.checkUrl r) + where + + go _ (Left e) = void $ commandAction $ do + showStartAddUrl u o + warning (show e) + next $ next $ return False + go deffile (Right (UrlContents sz mf)) = do + let f = adjustFile o (fromMaybe (maybe deffile fromSafeFilePath mf) (fileOption (downloadOptions o))) + void $ commandAction $ startRemote r o f u sz + go deffile (Right (UrlMulti l)) + | isNothing (fileOption (downloadOptions o)) = + forM_ l $ \(u', sz, f) -> do + let f' = adjustFile o (deffile fromSafeFilePath f) + void $ commandAction $ + startRemote r o f' u' sz + | otherwise = giveup $ unwords + [ "That url contains multiple files according to the" + , Remote.name r + , " remote; cannot add it to a single file." + ] + +startRemote :: Remote -> AddUrlOptions -> FilePath -> URLString -> Maybe Integer -> CommandStart +startRemote r o file uri sz = do + pathmax <- liftIO $ fileNameLengthLimit "." + let file' = joinPath $ map (truncateFilePath pathmax) $ splitDirectories file + showStartAddUrl uri o + showNote $ "from " ++ Remote.name r + showDestinationFile file' + next $ performRemote r o uri file' sz + +performRemote :: Remote -> AddUrlOptions -> URLString -> FilePath -> Maybe Integer -> CommandPerform +performRemote r o uri file sz = ifAnnexed file adduri geturi + where + loguri = setDownloader uri OtherDownloader + adduri = addUrlChecked o loguri file (Remote.uuid r) checkexistssize + checkexistssize key = return $ case sz of + Nothing -> (True, True, loguri) + Just n -> (True, n == fromMaybe n (keySize key), loguri) + geturi = next $ isJust <$> downloadRemoteFile r (downloadOptions o) uri file sz + +downloadRemoteFile :: Remote -> DownloadOptions -> URLString -> FilePath -> Maybe Integer -> Annex (Maybe Key) +downloadRemoteFile r o uri file sz = checkCanAdd file $ do + let urlkey = Backend.URL.fromUrl uri sz + liftIO $ createDirectoryIfMissing True (parentDir file) + ifM (Annex.getState Annex.fast <||> pure (relaxedOption o)) + ( do + addWorkTree (Remote.uuid r) loguri file urlkey Nothing + return (Just urlkey) + , do + -- Set temporary url for the urlkey + -- so that the remote knows what url it + -- should use to download it. + setTempUrl urlkey loguri + let downloader = \dest p -> fst + <$> Remote.retrieveKeyFile r urlkey + (AssociatedFile (Just file)) dest p + ret <- downloadWith downloader urlkey (Remote.uuid r) loguri file + removeTempUrl urlkey + return ret + ) + where + loguri = setDownloader uri OtherDownloader + +startWeb :: AddUrlOptions -> URLString -> CommandStart +startWeb o urlstring = go $ fromMaybe bad $ parseURI urlstring + where + bad = fromMaybe (giveup $ "bad url " ++ urlstring) $ + Url.parseURIRelaxed $ urlstring + go url = do + showStartAddUrl urlstring o + pathmax <- liftIO $ fileNameLengthLimit "." + urlinfo <- if relaxedOption (downloadOptions o) + then pure Url.assumeUrlExists + else Url.withUrlOptions $ + liftIO . Url.getUrlInfo urlstring + file <- adjustFile o <$> case fileOption (downloadOptions o) of + Just f -> pure f + Nothing -> case Url.urlSuggestedFile urlinfo of + Nothing -> pure $ url2file url (pathdepthOption o) pathmax + Just sf -> do + let f = truncateFilePath pathmax $ + sanitizeFilePath sf + ifM (liftIO $ doesFileExist f <||> doesDirectoryExist f) + ( pure $ url2file url (pathdepthOption o) pathmax + , pure f + ) + next $ performWeb o urlstring file urlinfo + +performWeb :: AddUrlOptions -> URLString -> FilePath -> Url.UrlInfo -> CommandPerform +performWeb o url file urlinfo = ifAnnexed file addurl geturl + where + geturl = next $ isJust <$> addUrlFile (downloadOptions o) url urlinfo file + addurl = addUrlChecked o url file webUUID $ \k -> + ifM (pure (not (rawOption (downloadOptions o))) <&&> youtubeDlSupported url) + ( return (True, True, setDownloader url YoutubeDownloader) + , return (Url.urlExists urlinfo, Url.urlSize urlinfo == keySize k, url) + ) + +{- Check that the url exists, and has the same size as the key, + - and add it as an url to the key. -} +addUrlChecked :: AddUrlOptions -> URLString -> FilePath -> UUID -> (Key -> Annex (Bool, Bool, URLString)) -> Key -> CommandPerform +addUrlChecked o url file u checkexistssize key = + ifM ((elem url <$> getUrls key) <&&> (elem u <$> loggedLocations key)) + ( do + showDestinationFile file + next $ return True + , do + (exists, samesize, url') <- checkexistssize key + if exists && (samesize || relaxedOption (downloadOptions o)) + then do + setUrlPresent u key url' + next $ return True + else do + warning $ "while adding a new url to an already annexed file, " ++ if exists + then "url does not have expected file size (use --relaxed to bypass this check) " ++ url + else "failed to verify url exists: " ++ url + stop + ) + +{- Downloads an url (except in fast or relaxed mode) and adds it to the + - repository, normally at the specified FilePath. + - But, if youtube-dl supports the url, it will be written to a + - different file, based on the title of the media. Unless the user + - specified fileOption, which then forces using the FilePath. + -} +addUrlFile :: DownloadOptions -> URLString -> Url.UrlInfo -> FilePath -> Annex (Maybe Key) +addUrlFile o url urlinfo file = + ifM (Annex.getState Annex.fast <||> pure (relaxedOption o)) + ( nodownloadWeb o url urlinfo file + , downloadWeb o url urlinfo file + ) + +downloadWeb :: DownloadOptions -> URLString -> Url.UrlInfo -> FilePath -> Annex (Maybe Key) +downloadWeb o url urlinfo file = + go =<< downloadWith' downloader urlkey webUUID url (AssociatedFile (Just file)) + where + urlkey = addSizeUrlKey urlinfo $ Backend.URL.fromUrl url Nothing + downloader f p = downloadUrl urlkey p [url] f + go Nothing = return Nothing + -- If we downloaded a html file, try to use youtube-dl to + -- extract embedded media. + go (Just tmp) = ifM (pure (not (rawOption o)) <&&> liftIO (isHtml <$> readFile tmp)) + ( tryyoutubedl tmp + , normalfinish tmp + ) + normalfinish tmp = checkCanAdd file $ do + showDestinationFile file + liftIO $ createDirectoryIfMissing True (parentDir file) + finishDownloadWith tmp webUUID url file + tryyoutubedl tmp + | isJust (fileOption o) = dl file + -- Ask youtube-dl what filename it will download + -- first, and check if that is already an annexed file, + -- to avoid unnecessary work in that case. + | otherwise = youtubeDlFileNameHtmlOnly url >>= \case + Right dest -> ifAnnexed dest + (alreadyannexed dest) + (dl dest) + Left _ -> normalfinish tmp + where + dl dest = withTmpWorkDir mediakey $ \workdir -> do + let cleanuptmp = pruneTmpWorkDirBefore tmp (liftIO . nukeFile) + Transfer.notifyTransfer Transfer.Download url $ + Transfer.download webUUID mediakey (AssociatedFile Nothing) Transfer.noRetry $ \_p -> + youtubeDl url workdir >>= \case + Right (Just mediafile) -> do + cleanuptmp + checkCanAdd dest $ do + showDestinationFile dest + addWorkTree webUUID mediaurl dest mediakey (Just mediafile) + return $ Just mediakey + Right Nothing -> normalfinish tmp + Left msg -> do + cleanuptmp + warning msg + return Nothing + mediaurl = setDownloader url YoutubeDownloader + mediakey = Backend.URL.fromUrl mediaurl Nothing + -- Does the already annexed file have the mediaurl + -- as an url? If so nothing to do. + alreadyannexed dest k = do + us <- getUrls k + if mediaurl `elem` us + then return (Just k) + else do + warning $ dest ++ " already exists; not overwriting" + return Nothing + +{- The destination file is not known at start time unless the user provided + - a filename. It's not displayed then for output consistency, + - but is added to the json when available. -} +showStartAddUrl :: URLString -> AddUrlOptions -> Annex () +showStartAddUrl url o = do + showStart' "addurl" (Just url) + case fileOption (downloadOptions o) of + Nothing -> noop + Just file -> maybeShowJSON $ JSONChunk [("file", file)] + +showDestinationFile :: FilePath -> Annex () +showDestinationFile file = do + showNote ("to " ++ file) + maybeShowJSON $ JSONChunk [("file", file)] + +{- The Key should be a dummy key, based on the URL, which is used + - for this download, before we can examine the file and find its real key. + - For resuming downloads to work, the dummy key for a given url should be + - stable. For disk space checking to work, the dummy key should have + - the size of the url already set. + - + - Downloads the url, sets up the worktree file, and returns the + - real key. + -} +downloadWith :: (FilePath -> MeterUpdate -> Annex Bool) -> Key -> UUID -> URLString -> FilePath -> Annex (Maybe Key) +downloadWith downloader dummykey u url file = + go =<< downloadWith' downloader dummykey u url afile + where + afile = AssociatedFile (Just file) + go Nothing = return Nothing + go (Just tmp) = finishDownloadWith tmp u url file + +{- Like downloadWith, but leaves the dummy key content in + - the returned location. -} +downloadWith' :: (FilePath -> MeterUpdate -> Annex Bool) -> Key -> UUID -> URLString -> AssociatedFile -> Annex (Maybe FilePath) +downloadWith' downloader dummykey u url afile = + checkDiskSpaceToGet dummykey Nothing $ do + tmp <- fromRepo $ gitAnnexTmpObjectLocation dummykey + ok <- Transfer.notifyTransfer Transfer.Download url $ + Transfer.download u dummykey afile Transfer.stdRetry $ \p -> do + liftIO $ createDirectoryIfMissing True (parentDir tmp) + downloader tmp p + if ok + then return (Just tmp) + else return Nothing + +finishDownloadWith :: FilePath -> UUID -> URLString -> FilePath -> Annex (Maybe Key) +finishDownloadWith tmp u url file = do + backend <- chooseBackend file + let source = KeySource + { keyFilename = file + , contentLocation = tmp + , inodeCache = Nothing + } + genKey source backend >>= \case + Nothing -> return Nothing + Just (key, _) -> do + addWorkTree u url file key (Just tmp) + return (Just key) + +{- Adds the url size to the Key. -} +addSizeUrlKey :: Url.UrlInfo -> Key -> Key +addSizeUrlKey urlinfo key = key { keySize = Url.urlSize urlinfo } + +{- Adds worktree file to the repository. -} +addWorkTree :: UUID -> URLString -> FilePath -> Key -> Maybe FilePath -> Annex () +addWorkTree u url file key mtmp = case mtmp of + Nothing -> go + Just tmp -> do + -- Move to final location for large file check. + pruneTmpWorkDirBefore tmp $ \_ -> liftIO $ do + createDirectoryIfMissing True (takeDirectory file) + renameFile tmp file + largematcher <- largeFilesMatcher + large <- checkFileMatcher largematcher file + if large + then do + -- Move back to tmp because addAnnexedFile + -- needs the file in a different location + -- than the work tree file. + liftIO $ renameFile file tmp + go + else void $ Command.Add.addSmall file + where + go = do + maybeShowJSON $ JSONChunk [("key", key2file key)] + setUrlPresent u key url + ifM (addAnnexedFile file key mtmp) + ( do + when (isJust mtmp) $ + logStatus key InfoPresent + , maybe noop (\tmp -> pruneTmpWorkDirBefore tmp (liftIO . nukeFile)) mtmp + ) + +nodownloadWeb :: DownloadOptions -> URLString -> Url.UrlInfo -> FilePath -> Annex (Maybe Key) +nodownloadWeb o url urlinfo file + | Url.urlExists urlinfo = if rawOption o + then nomedia + else either (const nomedia) usemedia + =<< youtubeDlFileName url + | otherwise = do + warning $ "unable to access url: " ++ url + return Nothing + where + nomedia = do + let key = Backend.URL.fromUrl url (Url.urlSize urlinfo) + nodownloadWeb' url key file + usemedia mediafile = do + let dest = if isJust (fileOption o) + then file + else takeFileName mediafile + let mediaurl = setDownloader url YoutubeDownloader + let mediakey = Backend.URL.fromUrl mediaurl Nothing + nodownloadWeb' mediaurl mediakey dest + +nodownloadWeb' :: URLString -> Key -> FilePath -> Annex (Maybe Key) +nodownloadWeb' url key file = checkCanAdd file $ do + showDestinationFile file + liftIO $ createDirectoryIfMissing True (parentDir file) + addWorkTree webUUID url file key Nothing + return (Just key) + +url2file :: URI -> Maybe Int -> Int -> FilePath +url2file url pathdepth pathmax = case pathdepth of + Nothing -> truncateFilePath pathmax $ sanitizeFilePath fullurl + Just depth + | depth >= length urlbits -> frombits id + | depth > 0 -> frombits $ drop depth + | depth < 0 -> frombits $ reverse . take (negate depth) . reverse + | otherwise -> giveup "bad --pathdepth" + where + fullurl = concat + [ maybe "" uriRegName (uriAuthority url) + , uriPath url + , uriQuery url + ] + frombits a = intercalate "/" $ a urlbits + urlbits = map (truncateFilePath pathmax . sanitizeFilePath) $ + filter (not . null) $ splitc '/' fullurl + +urlString2file :: URLString -> Maybe Int -> Int -> FilePath +urlString2file s pathdepth pathmax = case Url.parseURIRelaxed s of + Nothing -> giveup $ "bad uri " ++ s + Just u -> url2file u pathdepth pathmax + +adjustFile :: AddUrlOptions -> FilePath -> FilePath +adjustFile o = addprefix . addsuffix + where + addprefix f = maybe f (++ f) (prefixOption o) + addsuffix f = maybe f (f ++) (suffixOption o) + +checkCanAdd :: FilePath -> Annex (Maybe a) -> Annex (Maybe a) +checkCanAdd file a = ifM (isJust <$> (liftIO $ catchMaybeIO $ getSymbolicLinkStatus file)) + ( do + warning $ file ++ " already exists; not overwriting" + return Nothing + , ifM ((not <$> Annex.getState Annex.force) <&&> checkIgnored file) + ( do + warning $ "not adding " ++ file ++ " which is .gitignored (use --force to override)" + return Nothing + , a + ) + ) diff --git a/Command/Adjust.hs b/Command/Adjust.hs new file mode 100644 index 0000000000..0fef3f9365 --- /dev/null +++ b/Command/Adjust.hs @@ -0,0 +1,42 @@ +{- git-annex command + - + - Copyright 2016 Joey Hess + - + - Licensed under the GNU GPL version 3 or higher. + -} + +module Command.Adjust where + +import Command +import Annex.AdjustedBranch + +cmd :: Command +cmd = notBareRepo $ notDirect $ noDaemonRunning $ + command "adjust" SectionSetup "enter adjusted branch" + paramNothing (seek <$$> optParser) + +optParser :: CmdParamsDesc -> Parser Adjustment +optParser _ = + flag' UnlockAdjustment + ( long "unlock" + <> help "unlock annexed files" + ) + <|> flag' FixAdjustment + ( long "fix" + <> help "fix symlinks to annnexed files" + ) + {- Not ready yet + <|> flag' HideMissingAdjustment + ( long "hide-missing" + <> help "omit annexed files whose content is not present" + ) + -} + +seek :: Adjustment -> CommandSeek +seek = commandAction . start + +start :: Adjustment -> CommandStart +start adj = do + checkVersionSupported + showStart' "adjust" Nothing + next $ next $ enterAdjustedBranch adj diff --git a/Command/Assistant.hs b/Command/Assistant.hs new file mode 100644 index 0000000000..70088674da --- /dev/null +++ b/Command/Assistant.hs @@ -0,0 +1,135 @@ +{- git-annex assistant + - + - Copyright 2012-2017 Joey Hess + - + - Licensed under the GNU GPL version 3 or higher. + -} + +module Command.Assistant where + +import Command +import qualified Command.Watch +import Annex.Init +import Annex.Path +import Config.Files +import qualified BuildInfo +import Utility.HumanTime +import Assistant.Install + +import Control.Concurrent.Async + +cmd :: Command +cmd = dontCheck repoExists $ notBareRepo $ + noRepo (startNoRepo <$$> optParser) $ + command "assistant" SectionCommon + "automatically sync changes" + paramNothing (seek <$$> optParser) + +data AssistantOptions = AssistantOptions + { daemonOptions :: DaemonOptions + , autoStartOption :: Bool + , startDelayOption :: Maybe Duration + , autoStopOption :: Bool + } + +optParser :: CmdParamsDesc -> Parser AssistantOptions +optParser _ = AssistantOptions + <$> parseDaemonOptions + <*> switch + ( long "autostart" + <> help "start in known repositories" + ) + <*> optional (option (str >>= parseDuration) + ( long "startdelay" <> metavar paramNumber + <> help "delay before running startup scan" + )) + <*> switch + ( long "autostop" + <> help "stop in known repositories" + ) + +seek :: AssistantOptions -> CommandSeek +seek = commandAction . start + +start :: AssistantOptions -> CommandStart +start o + | autoStartOption o = do + liftIO $ autoStart o + stop + | autoStopOption o = do + liftIO autoStop + stop + | otherwise = do + liftIO ensureInstalled + ensureInitialized + Command.Watch.start True (daemonOptions o) (startDelayOption o) + +startNoRepo :: AssistantOptions -> IO () +startNoRepo o + | autoStartOption o = autoStart o + | autoStopOption o = autoStop + | otherwise = giveup "Not in a git repository." + +-- Does not return +autoStart :: AssistantOptions -> IO () +autoStart o = do + dirs <- liftIO readAutoStartFile + when (null dirs) $ do + f <- autoStartFile + giveup $ "Nothing listed in " ++ f + program <- programPath + haveionice <- pure BuildInfo.ionice <&&> inPath "ionice" + pids <- forM dirs $ \d -> do + putStrLn $ "git-annex autostart in " ++ d + mpid <- catchMaybeIO $ go haveionice program d + if foregroundDaemonOption (daemonOptions o) + then return mpid + else do + case mpid of + Nothing -> putStrLn "failed" + Just pid -> ifM (checkSuccessProcess pid) + ( putStrLn "ok" + , putStrLn "failed" + ) + return Nothing + -- Wait for any foreground jobs to finish and propigate exit status. + ifM (all (== True) <$> mapConcurrently checkSuccessProcess (catMaybes pids)) + ( exitSuccess + , exitFailure + ) + where + go haveionice program dir = do + setCurrentDirectory dir + -- First stop any old daemon running in this directory, which + -- might be a leftover from an old login session. Such a + -- leftover might be left in an environment where it is + -- unable to use the ssh agent or other login session + -- resources. + void $ boolSystem program [Param "assistant", Param "--stop"] + (Nothing, Nothing, Nothing, pid) <- createProcess p + return pid + where + p + | haveionice = proc "ionice" + (toCommand $ Param "-c3" : Param program : baseparams) + | otherwise = proc program + (toCommand baseparams) + baseparams = catMaybes + [ Just $ Param "assistant" + , Just $ Param $ "--startdelay=" ++ fromDuration (fromMaybe (Duration 5) (startDelayOption o)) + , if foregroundDaemonOption (daemonOptions o) + then Just $ Param "--foreground" + else Nothing + ] + +autoStop :: IO () +autoStop = do + dirs <- liftIO readAutoStartFile + program <- programPath + forM_ dirs $ \d -> do + putStrLn $ "git-annex autostop in " ++ d + setCurrentDirectory d + ifM (boolSystem program [Param "assistant", Param "--stop"]) + ( putStrLn "ok" + , putStrLn "failed" + ) diff --git a/Command/Benchmark.hs b/Command/Benchmark.hs new file mode 100644 index 0000000000..bcfecc2dc1 --- /dev/null +++ b/Command/Benchmark.hs @@ -0,0 +1,123 @@ +{- git-annex benchmark + - + - Copyright 2016 Joey Hess + - + - Licensed under the GNU GPL version 3 or higher. + -} + +{-# OPTIONS_GHC -fno-warn-orphans #-} + +module Command.Benchmark where + +import Command +import Database.Types +import qualified Database.Keys.SQL as SQL +import qualified Database.Queue as H +import Utility.Tmp +import Git.FilePath + +import Criterion.Main +import Criterion.Internal (runAndAnalyse) +import Criterion.Monad +import Control.Monad.IO.Class (liftIO) +import Control.Monad +import Control.DeepSeq +import System.FilePath +import System.Random + +cmd :: Command +cmd = noRepo (withParams benchmark) $ + dontCheck repoExists $ + command "benchmark" SectionTesting + "run benchmarks" + paramNothing + (withParams (liftIO . benchmark)) + +benchmark :: CmdParams -> IO () +benchmark _ = withTmpDirIn "." "benchmark" $ \tmpdir -> do + -- benchmark different sizes of databases + dbs <- mapM (benchDb tmpdir) + [ 1000 + , 10000 + -- , 100000 + ] + runCriterion $ + bgroup "keys database" $ flip concatMap dbs $ \db -> + [ getAssociatedFilesHitBench db + , getAssociatedFilesMissBench db + , getAssociatedKeyHitBench db + , getAssociatedKeyMissBench db + , addAssociatedFileOldBench db + , addAssociatedFileNewBench db + ] + +getAssociatedFilesHitBench :: BenchDb -> Benchmark +getAssociatedFilesHitBench ( BenchDb h num) = bench ("getAssociatedFiles from " ++ show num ++ " (hit)") $ nfIO $ do + n <- getStdRandom (randomR (1,num)) + SQL.getAssociatedFiles (keyN n) (SQL.ReadHandle h) + +getAssociatedFilesMissBench :: BenchDb -> Benchmark +getAssociatedFilesMissBench ( BenchDb h num) = bench ("getAssociatedFiles from " ++ show num ++ " (miss)") $ nfIO $ + SQL.getAssociatedFiles keyMiss (SQL.ReadHandle h) + +getAssociatedKeyHitBench :: BenchDb -> Benchmark +getAssociatedKeyHitBench (BenchDb h num) = bench ("getAssociatedKey from " ++ show num ++ " (hit)") $ nfIO $ do + n <- getStdRandom (randomR (1,num)) + SQL.getAssociatedKey (fileN n) (SQL.ReadHandle h) + +getAssociatedKeyMissBench :: BenchDb -> Benchmark +getAssociatedKeyMissBench (BenchDb h num) = bench ("getAssociatedKey from " ++ show num ++ " (miss)") $ nfIO $ + SQL.getAssociatedKey fileMiss (SQL.ReadHandle h) + +addAssociatedFileOldBench :: BenchDb -> Benchmark +addAssociatedFileOldBench ( BenchDb h num) = bench ("addAssociatedFile to " ++ show num ++ " (old)") $ nfIO $ do + n <- getStdRandom (randomR (1,num)) + SQL.addAssociatedFile (keyN n) (fileN n) (SQL.WriteHandle h) + H.flushDbQueue h + +addAssociatedFileNewBench :: BenchDb -> Benchmark +addAssociatedFileNewBench ( BenchDb h num) = bench ("addAssociatedFile to " ++ show num ++ " (new)") $ nfIO $ do + n <- getStdRandom (randomR (1,num)) + SQL.addAssociatedFile (keyN n) (fileN (n+1)) (SQL.WriteHandle h) + H.flushDbQueue h + +populateAssociatedFiles :: H.DbQueue -> Int -> IO () +populateAssociatedFiles h num = do + forM_ [1..num] $ \n -> + SQL.addAssociatedFile (keyN n) (fileN n) (SQL.WriteHandle h) + H.flushDbQueue h + +keyN :: Int -> IKey +keyN n = IKey ("key" ++ show n) + +fileN :: Int -> TopFilePath +fileN n = asTopFilePath ("file" ++ show n) + +keyMiss :: IKey +keyMiss = keyN 0 -- 0 is never stored + +fileMiss :: TopFilePath +fileMiss = fileN 0 -- 0 is never stored + +data BenchDb = BenchDb H.DbQueue Int + +benchDb :: FilePath -> Int -> IO BenchDb +benchDb tmpdir num = do + putStrLn $ "setting up database with " ++ show num + H.initDb f SQL.createTables + h <- H.openDbQueue f SQL.containedTable + populateAssociatedFiles h num + return (BenchDb h num) + where + f = tmpdir "db" ++ show num + +instance NFData TopFilePath where + rnf = rnf . getTopFilePath + +instance NFData IKey where + rnf (IKey s) = rnf s + +-- can't use Criterion's defaultMain here because it looks at +-- command-line parameters +runCriterion :: Benchmark -> IO () +runCriterion = withConfig defaultConfig . runAndAnalyse (const True) diff --git a/Command/CalcKey.hs b/Command/CalcKey.hs new file mode 100644 index 0000000000..57e6f40c96 --- /dev/null +++ b/Command/CalcKey.hs @@ -0,0 +1,26 @@ +{- git-annex command + - + - Copyright 2016 Joey Hess + - + - Licensed under the GNU GPL version 3 or higher. + -} + +module Command.CalcKey where + +import Command +import Backend (genKey) +import Types.KeySource + +cmd :: Command +cmd = noCommit $ noMessages $ dontCheck repoExists $ + command "calckey" SectionPlumbing + "calculates the key that would be used to refer to a file" + (paramRepeating paramFile) + (batchable run (pure ())) + +run :: () -> String -> Annex Bool +run _ file = genKey (KeySource file file Nothing) Nothing >>= \case + Just (k, _) -> do + liftIO $ putStrLn $ key2file k + return True + Nothing -> return False diff --git a/Command/CheckPresentKey.hs b/Command/CheckPresentKey.hs new file mode 100644 index 0000000000..fb8f9e53e2 --- /dev/null +++ b/Command/CheckPresentKey.hs @@ -0,0 +1,76 @@ +{- git-annex command + - + - Copyright 2015-2016 Joey Hess + - + - Licensed under the GNU GPL version 3 or higher. + -} + +module Command.CheckPresentKey where + +import Command +import qualified Remote + +cmd :: Command +cmd = noCommit $ noMessages $ + command "checkpresentkey" SectionPlumbing + "check if key is present in remote" + (paramPair paramKey (paramOptional paramRemote)) + (seek <$$> optParser) + +data CheckPresentKeyOptions = CheckPresentKeyOptions + { params :: CmdParams + , batchOption :: BatchMode + } + +optParser :: CmdParamsDesc -> Parser CheckPresentKeyOptions +optParser desc = CheckPresentKeyOptions + <$> cmdParams desc + <*> parseBatchOption + +seek :: CheckPresentKeyOptions -> CommandSeek +seek o = case batchOption o of + NoBatch -> case params o of + (ks:rn:[]) -> toRemote rn >>= (check ks . Just) >>= exitResult + (ks:[]) -> check ks Nothing >>= exitResult + _ -> wrongnumparams + Batch fmt -> do + checker <- case params o of + (rn:[]) -> toRemote rn >>= \r -> return (flip check (Just r)) + [] -> return (flip check Nothing) + _ -> wrongnumparams + batchInput fmt Right $ checker >=> batchResult + where + wrongnumparams = giveup "Wrong number of parameters" + +data Result = Present | NotPresent | CheckFailure String + +check :: String -> Maybe Remote -> Annex Result +check ks mr = case mr of + Nothing -> go Nothing =<< Remote.keyPossibilities k + Just r -> go Nothing [r] + where + k = toKey ks + go Nothing [] = return NotPresent + go (Just e) [] = return $ CheckFailure e + go olderr (r:rs) = Remote.hasKey r k >>= \case + Right True -> return Present + Right False -> go olderr rs + Left e -> go (Just e) rs + +exitResult :: Result -> Annex a +exitResult Present = liftIO exitSuccess +exitResult NotPresent = liftIO exitFailure +exitResult (CheckFailure msg) = liftIO $ do + hPutStrLn stderr msg + exitWith $ ExitFailure 100 + +batchResult :: Result -> Annex () +batchResult Present = liftIO $ putStrLn "1" +batchResult _ = liftIO $ putStrLn "0" + +toKey :: String -> Key +toKey = fromMaybe (giveup "Bad key") . file2key + +toRemote :: String -> Annex Remote +toRemote rn = maybe (giveup "Unknown remote") return + =<< Remote.byNameWithUUID (Just rn) diff --git a/Command/Commit.hs b/Command/Commit.hs new file mode 100644 index 0000000000..c2943aebe6 --- /dev/null +++ b/Command/Commit.hs @@ -0,0 +1,29 @@ +{- git-annex command + - + - Copyright 2012 Joey Hess + - + - Licensed under the GNU GPL version 3 or higher. + -} + +module Command.Commit where + +import Command +import qualified Annex.Branch +import qualified Git + +cmd :: Command +cmd = command "commit" SectionPlumbing + "commits any staged changes to the git-annex branch" + paramNothing (withParams seek) + +seek :: CmdParams -> CommandSeek +seek = withNothing (commandAction start) + +start :: CommandStart +start = next $ next $ do + Annex.Branch.commit =<< Annex.Branch.commitMessage + _ <- runhook <=< inRepo $ Git.hookPath "annex-content" + return True + where + runhook (Just hook) = liftIO $ boolSystem hook [] + runhook Nothing = return True diff --git a/Command/Config.hs b/Command/Config.hs new file mode 100644 index 0000000000..a79a4f0772 --- /dev/null +++ b/Command/Config.hs @@ -0,0 +1,70 @@ +{- git-annex command + - + - Copyright 2017 Joey Hess + - + - Licensed under the GNU GPL version 3 or higher. + -} + +module Command.Config where + +import Command +import Logs.Config +import Config + +cmd :: Command +cmd = noMessages $ command "config" SectionSetup + "configuration stored in git-annex branch" + paramNothing (seek <$$> optParser) + +data Action + = SetConfig ConfigName ConfigValue + | GetConfig ConfigName + | UnsetConfig ConfigName + +type Name = String +type Value = String + +optParser :: CmdParamsDesc -> Parser Action +optParser _ = setconfig <|> getconfig <|> unsetconfig + where + setconfig = SetConfig + <$> strOption + ( long "set" + <> help "set configuration" + <> metavar paramName + ) + <*> strArgument + ( metavar paramValue + ) + getconfig = GetConfig <$> strOption + ( long "get" + <> help "get configuration" + <> metavar paramName + ) + unsetconfig = UnsetConfig <$> strOption + ( long "unset" + <> help "unset configuration" + <> metavar paramName + ) + +seek :: Action -> CommandSeek +seek (SetConfig name val) = commandAction $ do + allowMessages + showStart' name (Just val) + next $ next $ do + setGlobalConfig name val + setConfig (ConfigKey name) val + return True +seek (UnsetConfig name) = commandAction $ do + allowMessages + showStart' name (Just "unset") + next $ next $ do + unsetGlobalConfig name + unsetConfig (ConfigKey name) + return True +seek (GetConfig name) = commandAction $ + getGlobalConfig name >>= \case + Nothing -> stop + Just v -> do + liftIO $ putStrLn v + stop diff --git a/Command/ConfigList.hs b/Command/ConfigList.hs new file mode 100644 index 0000000000..df674a9add --- /dev/null +++ b/Command/ConfigList.hs @@ -0,0 +1,51 @@ +{- git-annex command + - + - Copyright 2010-2014 Joey Hess + - + - Licensed under the GNU GPL version 3 or higher. + -} + +module Command.ConfigList where + +import Command +import Annex.UUID +import Annex.Init +import qualified Annex.Branch +import qualified Git.Config +import Remote.GCrypt (coreGCryptId) +import qualified CmdLine.GitAnnexShell.Fields as Fields +import CmdLine.GitAnnexShell.Checks + +cmd :: Command +cmd = noCommit $ dontCheck repoExists $ + command "configlist" SectionPlumbing + "outputs relevant git configuration" + paramNothing (withParams seek) + +seek :: CmdParams -> CommandSeek +seek = withNothing (commandAction start) + +start :: CommandStart +start = do + u <- findOrGenUUID + showConfig "annex.uuid" $ fromUUID u + showConfig coreGCryptId =<< fromRepo (Git.Config.get coreGCryptId "") + stop + where + showConfig k v = liftIO $ putStrLn $ k ++ "=" ++ v + +{- The repository may not yet have a UUID; automatically initialize it + - when there's a git-annex branch available or if the autoinit field was + - set. -} +findOrGenUUID :: Annex UUID +findOrGenUUID = do + u <- getUUID + if u /= NoUUID + then return u + else ifM (Annex.Branch.hasSibling <||> (isJust <$> Fields.getField Fields.autoInit)) + ( do + liftIO checkNotReadOnly + initialize (AutoInit True) Nothing Nothing + getUUID + , return NoUUID + ) diff --git a/Command/ContentLocation.hs b/Command/ContentLocation.hs new file mode 100644 index 0000000000..202d76a21d --- /dev/null +++ b/Command/ContentLocation.hs @@ -0,0 +1,29 @@ +{- git-annex command + - + - Copyright 2015 Joey Hess + - + - Licensed under the GNU GPL version 3 or higher. + -} + +module Command.ContentLocation where + +import Command +import Annex.Content + +cmd :: Command +cmd = noCommit $ noMessages $ + command "contentlocation" SectionPlumbing + "looks up content for a key" + (paramRepeating paramKey) + (batchable run (pure ())) + +run :: () -> String -> Annex Bool +run _ p = do + let k = fromMaybe (giveup "bad key") $ file2key p + maybe (return False) (\f -> liftIO (putStrLn f) >> return True) + =<< inAnnex' (pure True) Nothing check k + where + check f = ifM (liftIO (doesFileExist f)) + ( return (Just f) + , return Nothing + ) diff --git a/Command/Copy.hs b/Command/Copy.hs new file mode 100644 index 0000000000..12e1af1621 --- /dev/null +++ b/Command/Copy.hs @@ -0,0 +1,74 @@ +{- git-annex command + - + - Copyright 2010 Joey Hess + - + - Licensed under the GNU GPL version 3 or higher. + -} + +module Command.Copy where + +import Command +import qualified Command.Move +import qualified Remote +import Annex.Wanted +import Annex.NumCopies + +cmd :: Command +cmd = withGlobalOptions [jobsOption, jsonOptions, jsonProgressOption, annexedMatchingOptions] $ + command "copy" SectionCommon + "copy content of files to/from another repository" + paramPaths (seek <--< optParser) + +data CopyOptions = CopyOptions + { copyFiles :: CmdParams + , fromToOptions :: FromToHereOptions + , keyOptions :: Maybe KeyOptions + , autoMode :: Bool + , batchOption :: BatchMode + } + +optParser :: CmdParamsDesc -> Parser CopyOptions +optParser desc = CopyOptions + <$> cmdParams desc + <*> parseFromToHereOptions + <*> optional (parseKeyOptions <|> parseFailedTransfersOption) + <*> parseAutoOption + <*> parseBatchOption + +instance DeferredParseClass CopyOptions where + finishParse v = CopyOptions + <$> pure (copyFiles v) + <*> finishParse (fromToOptions v) + <*> pure (keyOptions v) + <*> pure (autoMode v) + <*> pure (batchOption v) + +seek :: CopyOptions -> CommandSeek +seek o = allowConcurrentOutput $ do + let go = whenAnnexed $ start o + case batchOption o of + Batch fmt -> batchFilesMatching fmt go + NoBatch -> withKeyOptions + (keyOptions o) (autoMode o) + (commandAction . Command.Move.startKey (fromToOptions o) Command.Move.RemoveNever) + (withFilesInGit $ commandAction . go) + =<< workTreeItems (copyFiles o) + +{- A copy is just a move that does not delete the source file. + - However, auto mode avoids unnecessary copies, and avoids getting or + - sending non-preferred content. -} +start :: CopyOptions -> FilePath -> Key -> CommandStart +start o file key = stopUnless shouldCopy $ + Command.Move.start (fromToOptions o) Command.Move.RemoveNever file key + where + shouldCopy + | autoMode o = want <||> numCopiesCheck file key (<) + | otherwise = return True + want = case fromToOptions o of + Right (ToRemote dest) -> + (Remote.uuid <$> getParsed dest) >>= checkwantsend + Right (FromRemote _) -> checkwantget + Left ToHere -> checkwantget + + checkwantsend = wantSend False (Just key) (AssociatedFile (Just file)) + checkwantget = wantGet False (Just key) (AssociatedFile (Just file)) diff --git a/Command/Dead.hs b/Command/Dead.hs new file mode 100644 index 0000000000..b750ff7dec --- /dev/null +++ b/Command/Dead.hs @@ -0,0 +1,44 @@ +{- git-annex command + - + - Copyright 2011, 2015 Joey Hess + - + - Licensed under the GNU GPL version 3 or higher. + -} + +module Command.Dead where + +import Command +import Types.TrustLevel +import Command.Trust (trustCommand) +import Logs.Location +import Remote (keyLocations) +import Git.Types + +cmd :: Command +cmd = command "dead" SectionSetup "hide a lost repository or key" + (paramRepeating paramRemote) (seek <$$> optParser) + +data DeadOptions = DeadRemotes [RemoteName] | DeadKeys [Key] + +optParser :: CmdParamsDesc -> Parser DeadOptions +optParser desc = (DeadRemotes <$> cmdParams desc) + <|> (DeadKeys <$> many (option (str >>= parseKey) + ( long "key" <> metavar paramKey + <> help "keys whose content has been irretrievably lost" + ))) + +seek :: DeadOptions -> CommandSeek +seek (DeadRemotes rs) = trustCommand "dead" DeadTrusted rs +seek (DeadKeys ks) = commandActions $ map startKey ks + +startKey :: Key -> CommandStart +startKey key = do + showStart' "dead" (Just $ key2file key) + keyLocations key >>= \case + [] -> next $ performKey key + _ -> giveup "This key is still known to be present in some locations; not marking as dead." + +performKey :: Key -> CommandPerform +performKey key = do + setDead key + next $ return True diff --git a/Command/Describe.hs b/Command/Describe.hs new file mode 100644 index 0000000000..eb87db968b --- /dev/null +++ b/Command/Describe.hs @@ -0,0 +1,33 @@ +{- git-annex command + - + - Copyright 2011 Joey Hess + - + - Licensed under the GNU GPL version 3 or higher. + -} + +module Command.Describe where + +import Command +import qualified Remote +import Logs.UUID + +cmd :: Command +cmd = command "describe" SectionSetup + "change description of a repository" + (paramPair paramRemote paramDesc) + (withParams seek) + +seek :: CmdParams -> CommandSeek +seek = withWords (commandAction . start) + +start :: [String] -> CommandStart +start (name:description) = do + showStart' "describe" (Just name) + u <- Remote.nameToUUID name + next $ perform u $ unwords description +start _ = giveup "Specify a repository and a description." + +perform :: UUID -> String -> CommandPerform +perform u description = do + describeUUID u description + next $ return True diff --git a/Command/DiffDriver.hs b/Command/DiffDriver.hs new file mode 100644 index 0000000000..2949a994c8 --- /dev/null +++ b/Command/DiffDriver.hs @@ -0,0 +1,102 @@ +{- git-annex command + - + - Copyright 2014 Joey Hess + - + - Licensed under the GNU GPL version 3 or higher. + -} + +module Command.DiffDriver where + +import Command +import Annex.Content +import Annex.Link +import Git.Types + +cmd :: Command +cmd = dontCheck repoExists $ + command "diffdriver" SectionPlumbing + "external git diff driver shim" + ("-- cmd --") (withParams seek) + +seek :: CmdParams -> CommandSeek +seek = withWords (commandAction . start) + +start :: [String] -> CommandStart +start opts = do + let (req, differ) = parseReq opts + void $ liftIO . exitBool =<< liftIO . differ =<< fixupReq req + stop + +data Req + = Req + { rPath :: FilePath + , rOldFile :: FilePath + , rOldHex :: String + , rOldMode :: String + , rNewFile :: FilePath + , rNewHex :: String + , rNewMode ::String + } + | UnmergedReq + { rPath :: FilePath + } + +type Differ = Req -> IO Bool + +serializeReq :: Req -> [CommandParam] +serializeReq req@(UnmergedReq {}) = [Param $ rPath req] +serializeReq req@(Req {}) = map Param + [ rPath req + , rOldFile req + , rOldHex req + , rOldMode req + , rNewFile req + , rNewHex req + , rNewMode req + ] + +parseReq :: [String] -> (Req, Differ) +parseReq opts = case separate (== "--") opts of + (c:ps, l) -> (mk l, externalDiffer c ps) + ([],_) -> badopts + where + mk (path:old_file:old_hex:old_mode:new_file:new_hex:new_mode:[]) = + Req + { rPath = path + , rOldFile = old_file + , rOldHex = old_hex + , rOldMode = old_mode + , rNewFile = new_file + , rNewHex = new_hex + , rNewMode = new_mode + } + mk (unmergedpath:[]) = UnmergedReq { rPath = unmergedpath } + mk _ = badopts + + badopts = giveup $ "Unexpected input: " ++ unwords opts + +{- Check if either file is a symlink to a git-annex object, + - which git-diff will leave as a normal file containing the link text. + - Adjust the Req to instead point to the actual location of the annexed + - object (which may or may not exist). -} +fixupReq :: Req -> Annex Req +fixupReq req@(UnmergedReq {}) = return req +fixupReq req@(Req {}) = + check rOldFile rOldMode (\r f -> r { rOldFile = f }) req + >>= check rNewFile rNewMode (\r f -> r { rNewFile = f }) + where + check getfile getmode setfile r = case readTreeItemType (getmode r) of + Just TreeSymlink -> do + v <- getAnnexLinkTarget' (getfile r) False + case fileKey . takeFileName =<< v of + Nothing -> return r + Just k -> setfile r <$> + withObjectLoc k + -- indirect mode + return + -- direct mode + (return . Prelude.head) + _ -> return r + +externalDiffer :: String -> [String] -> Differ +externalDiffer c ps = \req -> boolSystem c (map Param ps ++ serializeReq req ) diff --git a/Command/Direct.hs b/Command/Direct.hs new file mode 100644 index 0000000000..093999c805 --- /dev/null +++ b/Command/Direct.hs @@ -0,0 +1,68 @@ +{- git-annex command + - + - Copyright 2012 Joey Hess + - + - Licensed under the GNU GPL version 3 or higher. + -} + +module Command.Direct where + +import Command +import qualified Git +import qualified Git.LsFiles +import qualified Git.Branch +import Config +import Annex.Direct +import Annex.Version + +cmd :: Command +cmd = notBareRepo $ noDaemonRunning $ + command "direct" SectionSetup "switch repository to direct mode" + paramNothing (withParams seek) + +seek :: CmdParams -> CommandSeek +seek = withNothing (commandAction start) + +start :: CommandStart +start = ifM versionSupportsDirectMode + ( ifM isDirect ( stop , next perform ) + , giveup "Direct mode is not supported by this repository version. Use git-annex unlock instead." + ) + +perform :: CommandPerform +perform = do + showStart' "commit" Nothing + showOutput + _ <- inRepo $ Git.Branch.commitCommand Git.Branch.ManualCommit + [ Param "-a" + , Param "-m" + , Param "commit before switching to direct mode" + ] + showEndOk + + top <- fromRepo Git.repoPath + (l, clean) <- inRepo $ Git.LsFiles.inRepo [top] + forM_ l go + void $ liftIO clean + next cleanup + where + go = whenAnnexed $ \f k -> do + toDirectGen k f >>= \case + Nothing -> noop + Just a -> do + showStart "direct" f + tryNonAsync a >>= \case + Left e -> warnlocked e + Right _ -> showEndOk + return Nothing + + warnlocked :: SomeException -> Annex () + warnlocked e = do + warning $ show e + warning "leaving this file as-is; correct this problem and run git annex fsck on it" + +cleanup :: CommandCleanup +cleanup = do + showStart' "direct" Nothing + setDirect True + return True diff --git a/Command/Drop.hs b/Command/Drop.hs new file mode 100644 index 0000000000..6de808de9c --- /dev/null +++ b/Command/Drop.hs @@ -0,0 +1,217 @@ +{- git-annex command + - + - Copyright 2010 Joey Hess + - + - Licensed under the GNU GPL version 3 or higher. + -} + +module Command.Drop where + +import Command +import qualified Remote +import qualified Annex +import Annex.UUID +import Logs.Location +import Logs.Trust +import Logs.PreferredContent +import Annex.NumCopies +import Annex.Content +import Annex.Wanted +import Annex.Notification + +import System.Log.Logger (debugM) +import qualified Data.Set as S + +cmd :: Command +cmd = withGlobalOptions [jobsOption, jsonOptions, annexedMatchingOptions] $ + command "drop" SectionCommon + "remove content of files from repository" + paramPaths (seek <$$> optParser) + +data DropOptions = DropOptions + { dropFiles :: CmdParams + , dropFrom :: Maybe (DeferredParse Remote) + , autoMode :: Bool + , keyOptions :: Maybe KeyOptions + , batchOption :: BatchMode + } + +optParser :: CmdParamsDesc -> Parser DropOptions +optParser desc = DropOptions + <$> cmdParams desc + <*> optional parseDropFromOption + <*> parseAutoOption + <*> optional parseKeyOptions + <*> parseBatchOption + +parseDropFromOption :: Parser (DeferredParse Remote) +parseDropFromOption = parseRemoteOption <$> strOption + ( long "from" <> short 'f' <> metavar paramRemote + <> help "drop content from a remote" + <> completeRemotes + ) + +seek :: DropOptions -> CommandSeek +seek o = allowConcurrentOutput $ + case batchOption o of + Batch fmt -> batchFilesMatching fmt go + NoBatch -> withKeyOptions (keyOptions o) (autoMode o) + (commandAction . startKeys o) + (withFilesInGit (commandAction . go)) + =<< workTreeItems (dropFiles o) + where + go = whenAnnexed $ start o + +start :: DropOptions -> FilePath -> Key -> CommandStart +start o file key = start' o key afile (mkActionItem afile) + where + afile = AssociatedFile (Just file) + +start' :: DropOptions -> Key -> AssociatedFile -> ActionItem -> CommandStart +start' o key afile ai = do + from <- maybe (pure Nothing) (Just <$$> getParsed) (dropFrom o) + checkDropAuto (autoMode o) from afile key $ \numcopies -> + stopUnless (want from) $ + case from of + Nothing -> startLocal afile ai numcopies key [] + Just remote -> do + u <- getUUID + if Remote.uuid remote == u + then startLocal afile ai numcopies key [] + else startRemote afile ai numcopies key remote + where + want from + | autoMode o = wantDrop False (Remote.uuid <$> from) (Just key) afile + | otherwise = return True + +startKeys :: DropOptions -> (Key, ActionItem) -> CommandStart +startKeys o (key, ai) = start' o key (AssociatedFile Nothing) ai + +startLocal :: AssociatedFile -> ActionItem -> NumCopies -> Key -> [VerifiedCopy] -> CommandStart +startLocal afile ai numcopies key preverified = stopUnless (inAnnex key) $ do + showStartKey "drop" key ai + next $ performLocal key afile numcopies preverified + +startRemote :: AssociatedFile -> ActionItem -> NumCopies -> Key -> Remote -> CommandStart +startRemote afile ai numcopies key remote = do + showStartKey ("drop " ++ Remote.name remote) key ai + next $ performRemote key afile numcopies remote + +performLocal :: Key -> AssociatedFile -> NumCopies -> [VerifiedCopy] -> CommandPerform +performLocal key afile numcopies preverified = lockContentForRemoval key $ \contentlock -> do + u <- getUUID + (tocheck, verified) <- verifiableCopies key [u] + doDrop u (Just contentlock) key afile numcopies [] (preverified ++ verified) tocheck + ( \proof -> do + liftIO $ debugM "drop" $ unwords + [ "Dropping from here" + , "proof:" + , show proof + ] + removeAnnex contentlock + notifyDrop afile True + next $ cleanupLocal key + , do + notifyDrop afile False + stop + ) + +performRemote :: Key -> AssociatedFile -> NumCopies -> Remote -> CommandPerform +performRemote key afile numcopies remote = do + -- Filter the remote it's being dropped from out of the lists of + -- places assumed to have the key, and places to check. + -- When the local repo has the key, that's one additional copy, + -- as long as the local repo is not untrusted. + (tocheck, verified) <- verifiableCopies key [uuid] + doDrop uuid Nothing key afile numcopies [uuid] verified tocheck + ( \proof -> do + liftIO $ debugM "drop" $ unwords + [ "Dropping from remote" + , show remote + , "proof:" + , show proof + ] + ok <- Remote.removeKey remote key + next $ cleanupRemote key remote ok + , stop + ) + where + uuid = Remote.uuid remote + +cleanupLocal :: Key -> CommandCleanup +cleanupLocal key = do + logStatus key InfoMissing + return True + +cleanupRemote :: Key -> Remote -> Bool -> CommandCleanup +cleanupRemote key remote ok = do + when ok $ + Remote.logStatus remote key InfoMissing + return ok + +{- Before running the dropaction, checks specified remotes to + - verify that enough copies of a key exist to allow it to be + - safely removed (with no data loss). + - + - Also checks if it's required content, and refuses to drop if so. + - + - --force overrides and always allows dropping. + -} +doDrop + :: UUID + -> Maybe ContentRemovalLock + -> Key + -> AssociatedFile + -> NumCopies + -> [UUID] + -> [VerifiedCopy] + -> [UnVerifiedCopy] + -> (Maybe SafeDropProof -> CommandPerform, CommandPerform) + -> CommandPerform +doDrop dropfrom contentlock key afile numcopies skip preverified check (dropaction, nodropaction) = + ifM (Annex.getState Annex.force) + ( dropaction Nothing + , ifM (checkRequiredContent dropfrom key afile) + ( verifyEnoughCopiesToDrop nolocmsg key + contentlock numcopies + skip preverified check + (dropaction . Just) + (forcehint nodropaction) + , stop + ) + ) + where + nolocmsg = "Rather than dropping this file, try using: git annex move" + forcehint a = do + showLongNote "(Use --force to override this check, or adjust numcopies.)" + a + +checkRequiredContent :: UUID -> Key -> AssociatedFile -> Annex Bool +checkRequiredContent u k afile = + ifM (isRequiredContent (Just u) S.empty (Just k) afile False) + ( requiredContent + , return True + ) + +requiredContent :: Annex Bool +requiredContent = do + showLongNote "That file is required content, it cannot be dropped!" + showLongNote "(Use --force to override this check, or adjust required content configuration.)" + return False + +{- In auto mode, only runs the action if there are enough + - copies on other semitrusted repositories. -} +checkDropAuto :: Bool -> Maybe Remote -> AssociatedFile -> Key -> (NumCopies -> CommandStart) -> CommandStart +checkDropAuto automode mremote afile key a = + go =<< getAssociatedFileNumCopies afile + where + go numcopies + | automode = do + locs <- Remote.keyLocations key + uuid <- getUUID + let remoteuuid = fromMaybe uuid $ Remote.uuid <$> mremote + locs' <- trustExclude UnTrusted $ filter (/= remoteuuid) locs + if NumCopies (length locs') >= numcopies + then a numcopies + else stop + | otherwise = a numcopies diff --git a/Command/DropKey.hs b/Command/DropKey.hs new file mode 100644 index 0000000000..df85803062 --- /dev/null +++ b/Command/DropKey.hs @@ -0,0 +1,59 @@ +{- git-annex command + - + - Copyright 2010,2016 Joey Hess + - + - Licensed under the GNU GPL version 3 or higher. + -} + +module Command.DropKey where + +import Command +import qualified Annex +import Logs.Location +import Annex.Content + +cmd :: Command +cmd = noCommit $ withGlobalOptions [jsonOptions] $ + command "dropkey" SectionPlumbing + "drops annexed content for specified keys" + (paramRepeating paramKey) + (seek <$$> optParser) + +data DropKeyOptions = DropKeyOptions + { toDrop :: [String] + , batchOption :: BatchMode + } + +optParser :: CmdParamsDesc -> Parser DropKeyOptions +optParser desc = DropKeyOptions + <$> cmdParams desc + <*> parseBatchOption + +seek :: DropKeyOptions -> CommandSeek +seek o = do + unlessM (Annex.getState Annex.force) $ + giveup "dropkey can cause data loss; use --force if you're sure you want to do this" + withKeys (commandAction . start) (toDrop o) + case batchOption o of + Batch fmt -> batchInput fmt parsekey $ batchCommandAction . start + NoBatch -> noop + where + parsekey = maybe (Left "bad key") Right . file2key + +start :: Key -> CommandStart +start key = do + showStartKey "dropkey" key (mkActionItem key) + next $ perform key + +perform :: Key -> CommandPerform +perform key = ifM (inAnnex key) + ( lockContentForRemoval key $ \contentlock -> do + removeAnnex contentlock + next $ cleanup key + , next $ return True + ) + +cleanup :: Key -> CommandCleanup +cleanup key = do + logStatus key InfoMissing + return True diff --git a/Command/DropUnused.hs b/Command/DropUnused.hs new file mode 100644 index 0000000000..c5a61d7391 --- /dev/null +++ b/Command/DropUnused.hs @@ -0,0 +1,59 @@ +{- git-annex command + - + - Copyright 2010,2012 Joey Hess + - + - Licensed under the GNU GPL version 3 or higher. + -} + +module Command.DropUnused where + +import Command +import qualified Command.Drop +import qualified Remote +import qualified Git +import Command.Unused (withUnusedMaps, UnusedMaps(..), startUnused) +import Annex.NumCopies +import Annex.Content + +cmd :: Command +cmd = command "dropunused" SectionMaintenance + "drop unused file content" + (paramRepeating paramNumRange) (seek <$$> optParser) + +data DropUnusedOptions = DropUnusedOptions + { rangesToDrop :: CmdParams + , dropFrom :: Maybe (DeferredParse Remote) + } + +optParser :: CmdParamsDesc -> Parser DropUnusedOptions +optParser desc = DropUnusedOptions + <$> cmdParams desc + <*> optional (Command.Drop.parseDropFromOption) + +seek :: DropUnusedOptions -> CommandSeek +seek o = do + numcopies <- getNumCopies + from <- maybe (pure Nothing) (Just <$$> getParsed) (dropFrom o) + withUnusedMaps (start from numcopies) (rangesToDrop o) + +start :: Maybe Remote -> NumCopies -> UnusedMaps -> Int -> CommandStart +start from numcopies = startUnused "dropunused" + (perform from numcopies) + (performOther gitAnnexBadLocation) + (performOther gitAnnexTmpObjectLocation) + +perform :: Maybe Remote -> NumCopies -> Key -> CommandPerform +perform from numcopies key = case from of + Just r -> do + showAction $ "from " ++ Remote.name r + Command.Drop.performRemote key (AssociatedFile Nothing) numcopies r + Nothing -> ifM (inAnnex key) + ( Command.Drop.performLocal key (AssociatedFile Nothing) numcopies [] + , next (return True) + ) + +performOther :: (Key -> Git.Repo -> FilePath) -> Key -> CommandPerform +performOther filespec key = do + f <- fromRepo $ filespec key + pruneTmpWorkDirBefore f (liftIO . nukeFile) + next $ return True diff --git a/Command/EnableRemote.hs b/Command/EnableRemote.hs new file mode 100644 index 0000000000..e114707e2f --- /dev/null +++ b/Command/EnableRemote.hs @@ -0,0 +1,118 @@ +{- git-annex command + - + - Copyright 2013-2016 Joey Hess + - + - Licensed under the GNU GPL version 3 or higher. + -} + +module Command.EnableRemote where + +import Command +import qualified Annex +import qualified Logs.Remote +import qualified Types.Remote as R +import qualified Git +import qualified Git.Types as Git +import qualified Annex.SpecialRemote +import qualified Remote +import qualified Types.Remote as Remote +import qualified Remote.Git +import Logs.UUID +import Annex.UUID +import Config +import Config.DynamicConfig +import Types.GitConfig + +import qualified Data.Map as M + +cmd :: Command +cmd = command "enableremote" SectionSetup + "enables git-annex to use a remote" + (paramPair paramName $ paramOptional $ paramRepeating paramKeyValue) + (withParams seek) + +seek :: CmdParams -> CommandSeek +seek = withWords (commandAction . start) + +start :: [String] -> CommandStart +start [] = unknownNameError "Specify the remote to enable." +start (name:rest) = go =<< filter matchingname <$> Annex.getGitRemotes + where + matchingname r = Git.remoteName r == Just name + go [] = startSpecialRemote name (Logs.Remote.keyValToConfig rest) + =<< Annex.SpecialRemote.findExisting name + go (r:_) = do + -- This could be either a normal git remote or a special + -- remote that has an url (eg gcrypt). + rs <- Remote.remoteList + case filter (\rmt -> Remote.name rmt == name) rs of + (rmt:_) | Remote.remotetype rmt == Remote.Git.remote -> + startNormalRemote name rest r + _ -> go [] + +-- Normal git remotes are special-cased; enableremote retries probing +-- the remote uuid. +startNormalRemote :: Git.RemoteName -> [String] -> Git.Repo -> CommandStart +startNormalRemote name restparams r + | null restparams = do + showStart' "enableremote" (Just name) + next $ next $ do + setRemoteIgnore r False + r' <- Remote.Git.configRead False r + u <- getRepoUUID r' + return $ u /= NoUUID + | otherwise = giveup $ + "That is a normal git remote; passing these parameters does not make sense: " ++ unwords restparams + +startSpecialRemote :: Git.RemoteName -> Remote.RemoteConfig -> Maybe (UUID, Remote.RemoteConfig) -> CommandStart +startSpecialRemote name config Nothing = do + m <- Annex.SpecialRemote.specialRemoteMap + confm <- Logs.Remote.readRemoteLog + Remote.nameToUUID' name >>= \case + Right u | u `M.member` m -> + startSpecialRemote name config $ + Just (u, fromMaybe M.empty (M.lookup u confm)) + _ -> unknownNameError "Unknown remote name." +startSpecialRemote name config (Just (u, c)) = do + let fullconfig = config `M.union` c + t <- either giveup return (Annex.SpecialRemote.findType fullconfig) + showStart' "enableremote" (Just name) + gc <- maybe (liftIO dummyRemoteGitConfig) + (return . Remote.gitconfig) + =<< Remote.byUUID u + next $ performSpecialRemote t u c fullconfig gc + +performSpecialRemote :: RemoteType -> UUID -> R.RemoteConfig -> R.RemoteConfig -> RemoteGitConfig -> CommandPerform +performSpecialRemote t u oldc c gc = do + (c', u') <- R.setup t (R.Enable oldc) (Just u) Nothing c gc + next $ cleanupSpecialRemote u' c' + +cleanupSpecialRemote :: UUID -> R.RemoteConfig -> CommandCleanup +cleanupSpecialRemote u c = do + Logs.Remote.configSet u c + Remote.byUUID u >>= \case + Nothing -> noop + Just r -> do + repo <- R.getRepo r + setRemoteIgnore repo False + return True + +unknownNameError :: String -> Annex a +unknownNameError prefix = do + m <- Annex.SpecialRemote.specialRemoteMap + descm <- M.unionWith Remote.addName <$> uuidMap <*> pure m + specialmsg <- if M.null m + then pure "(No special remotes are currently known; perhaps use initremote instead?)" + else Remote.prettyPrintUUIDsDescs + "known special remotes" + descm (M.keys m) + disabledremotes <- filterM isdisabled =<< Annex.getGitRemotes + let remotesmsg = unlines $ map ("\t" ++) $ + mapMaybe Git.remoteName disabledremotes + giveup $ concat $ filter (not . null) [prefix ++ "\n", remotesmsg, specialmsg] + where + isdisabled r = anyM id + [ (==) NoUUID <$> getRepoUUID r + , liftIO . getDynamicConfig . remoteAnnexIgnore + =<< Annex.getRemoteGitConfig r + ] diff --git a/Command/EnableTor.hs b/Command/EnableTor.hs new file mode 100644 index 0000000000..ea21baf2bb --- /dev/null +++ b/Command/EnableTor.hs @@ -0,0 +1,134 @@ +{- git-annex command + - + - Copyright 2016 Joey Hess + - + - Licensed under the GNU GPL version 3 or higher. + -} + +{-# LANGUAGE CPP #-} + +module Command.EnableTor where + +import Command +import qualified Annex +import P2P.Address +import P2P.Annex +import Utility.Tor +import Annex.UUID +#ifndef mingw32_HOST_OS +import Config.Files +#endif +import P2P.IO +import qualified P2P.Protocol as P2P +import Utility.ThreadScheduler +import RemoteDaemon.Transport.Tor + +import Control.Concurrent.Async +import qualified Network.Socket as S +#ifndef mingw32_HOST_OS +import Utility.Su +import System.Posix.User +#endif + +cmd :: Command +cmd = noCommit $ dontCheck repoExists $ + command "enable-tor" SectionSetup "enable tor hidden service" + "uid" (withParams seek) + +seek :: CmdParams -> CommandSeek +seek = withWords (commandAction . start) + +-- This runs as root, so avoid making any commits or initializing +-- git-annex, or doing other things that create root-owned files. +start :: [String] -> CommandStart +start os = do + uuid <- getUUID + when (uuid == NoUUID) $ + giveup "This can only be run in a git-annex repository." +#ifndef mingw32_HOST_OS + curruserid <- liftIO getEffectiveUserID + if curruserid == 0 + then case readish =<< headMaybe os of + Nothing -> giveup "Need user-id parameter." + Just userid -> go uuid userid + else do + showStart' "enable-tor" Nothing + gitannex <- liftIO readProgramFile + let ps = [Param (cmdname cmd), Param (show curruserid)] + sucommand <- liftIO $ mkSuCommand gitannex ps + maybe noop showLongNote + (describePasswordPrompt' sucommand) + ifM (liftIO $ runSuCommand sucommand) + ( next $ next checkHiddenService + , giveup $ unwords $ + [ "Failed to run as root:" , gitannex ] ++ toCommand ps + ) +#else + go uuid 0 +#endif + where + go uuid userid = do + (onionaddr, onionport) <- liftIO $ + addHiddenService torAppName userid (fromUUID uuid) + storeP2PAddress $ TorAnnex onionaddr onionport + stop + +checkHiddenService :: CommandCleanup +checkHiddenService = bracket setup cleanup go + where + setup = do + showLongNote "Tor hidden service is configured. Checking connection to it. This may take a few minutes." + startlistener + + cleanup = liftIO . cancel + + go _ = check (150 :: Int) =<< filter istoraddr <$> loadP2PAddresses + + istoraddr (TorAnnex _ _) = True + + check 0 _ = giveup "Still unable to connect to hidden service. It might not yet be usable by others. Please check Tor's logs for details." + check _ [] = giveup "Somehow didn't get an onion address." + check n addrs@(addr:_) = do + g <- Annex.gitRepo + -- Connect but don't bother trying to auth, + -- we just want to know if the tor circuit works. + liftIO (tryNonAsync $ connectPeer g addr) >>= \case + Left e -> do + warning $ "Unable to connect to hidden service. It may not yet have propigated to the Tor network. (" ++ show e ++ ") Will retry.." + liftIO $ threadDelaySeconds (Seconds 2) + check (n-1) addrs + Right conn -> do + liftIO $ closeConnection conn + showLongNote "Tor hidden service is working." + return True + + -- Unless the remotedaemon is already listening on the hidden + -- service's socket, start a listener. This is only run during the + -- check, and it refuses all auth attempts. + startlistener = do + r <- Annex.gitRepo + u <- getUUID + msock <- torSocketFile + case msock of + Just sockfile -> ifM (liftIO $ haslistener sockfile) + ( liftIO $ async $ return () + , liftIO $ async $ runlistener sockfile u r + ) + Nothing -> giveup "Could not find socket file in Tor configuration!" + + runlistener sockfile u r = serveUnixSocket sockfile $ \h -> do + let conn = P2PConnection + { connRepo = r + , connCheckAuth = const False + , connIhdl = h + , connOhdl = h + } + runst <- mkRunState Client + void $ runNetProto runst conn $ P2P.serveAuth u + hClose h + + haslistener sockfile = catchBoolIO $ do + soc <- S.socket S.AF_UNIX S.Stream S.defaultProtocol + S.connect soc (S.SockAddrUnix sockfile) + S.close soc + return True diff --git a/Command/ExamineKey.hs b/Command/ExamineKey.hs new file mode 100644 index 0000000000..2c79c1a658 --- /dev/null +++ b/Command/ExamineKey.hs @@ -0,0 +1,26 @@ +{- git-annex command + - + - Copyright 2013 Joey Hess + - + - Licensed under the GNU GPL version 3 or higher. + -} + +module Command.ExamineKey where + +import Command +import qualified Utility.Format +import Command.Find (parseFormatOption, showFormatted, keyVars) + +cmd :: Command +cmd = noCommit $ noMessages $ dontCheck repoExists $ + withGlobalOptions [jsonOptions] $ + command "examinekey" SectionPlumbing + "prints information from a key" + (paramRepeating paramKey) + (batchable run (optional parseFormatOption)) + +run :: Maybe Utility.Format.Format -> String -> Annex Bool +run format p = do + let k = fromMaybe (giveup "bad key") $ file2key p + showFormatted format (key2file k) (keyVars k) + return True diff --git a/Command/Expire.hs b/Command/Expire.hs new file mode 100644 index 0000000000..1329171c80 --- /dev/null +++ b/Command/Expire.hs @@ -0,0 +1,116 @@ +{- git-annex command + - + - Copyright 2015 Joey Hess + - + - Licensed under the GNU GPL version 3 or higher. + -} + +module Command.Expire where + +import Command +import Logs.Activity +import Logs.UUID +import Logs.MapLog +import Logs.Trust +import Annex.UUID +import Annex.VectorClock +import qualified Remote +import Utility.HumanTime + +import Data.Time.Clock.POSIX +import qualified Data.Map as M + +cmd :: Command +cmd = command "expire" SectionMaintenance + "expire inactive repositories" + paramExpire (seek <$$> optParser) + +paramExpire :: String +paramExpire = (paramRepeating $ paramOptional paramRemote ++ ":" ++ paramTime) + +data ExpireOptions = ExpireOptions + { expireParams :: CmdParams + , activityOption :: Maybe Activity + , noActOption :: Bool + } + +optParser :: CmdParamsDesc -> Parser ExpireOptions +optParser desc = ExpireOptions + <$> cmdParams desc + <*> optional (option (str >>= parseActivity) + ( long "activity" <> metavar paramName + <> help "specify activity that prevents expiry" + )) + <*> switch + ( long "no-act" + <> help "don't really do anything" + ) + +seek :: ExpireOptions -> CommandSeek +seek o = do + expire <- parseExpire (expireParams o) + actlog <- lastActivities (activityOption o) + u <- getUUID + us <- filter (/= u) . M.keys <$> uuidMap + descs <- uuidMap + commandActions $ map (start expire (noActOption o) actlog descs) us + +start :: Expire -> Bool -> Log Activity -> M.Map UUID String -> UUID -> CommandStart +start (Expire expire) noact actlog descs u = + case lastact of + Just ent | notexpired ent -> checktrust (== DeadTrusted) $ do + showStart' "unexpire" (Just desc) + showNote =<< whenactive + unless noact $ + trustSet u SemiTrusted + _ -> checktrust (/= DeadTrusted) $ do + showStart' "expire" (Just desc) + showNote =<< whenactive + unless noact $ + trustSet u DeadTrusted + where + lastact = changed <$> M.lookup u actlog + whenactive = case lastact of + Just (VectorClock c) -> do + d <- liftIO $ durationSince $ posixSecondsToUTCTime c + return $ "last active: " ++ fromDuration d ++ " ago" + _ -> return "no activity" + desc = fromUUID u ++ " " ++ fromMaybe "" (M.lookup u descs) + notexpired ent = case ent of + Unknown -> False + VectorClock c -> case lookupexpire of + Just (Just expiretime) -> c >= expiretime + _ -> True + lookupexpire = headMaybe $ catMaybes $ + map (`M.lookup` expire) [Just u, Nothing] + checktrust want a = ifM (want <$> lookupTrust u) + ( do + void a + next $ next $ return True + , stop + ) + +data Expire = Expire (M.Map (Maybe UUID) (Maybe POSIXTime)) + +parseExpire :: [String] -> Annex Expire +parseExpire [] = giveup "Specify an expire time." +parseExpire ps = do + now <- liftIO getPOSIXTime + Expire . M.fromList <$> mapM (parse now) ps + where + parse now s = case separate (== ':') s of + (t, []) -> return (Nothing, parsetime now t) + (n, t) -> do + r <- Remote.nameToUUID n + return (Just r, parsetime now t) + parsetime _ "never" = Nothing + parsetime now s = case parseDuration s of + Nothing -> giveup $ "bad expire time: " ++ s + Just d -> Just (now - durationToPOSIXTime d) + +parseActivity :: Monad m => String -> m Activity +parseActivity s = case readish s of + Nothing -> fail $ "Unknown activity. Choose from: " ++ + unwords (map show [minBound..maxBound :: Activity]) + Just v -> return v + diff --git a/Command/Export.hs b/Command/Export.hs new file mode 100644 index 0000000000..4c9dbadfb2 --- /dev/null +++ b/Command/Export.hs @@ -0,0 +1,379 @@ +{- git-annex command + - + - Copyright 2017 Joey Hess + - + - Licensed under the GNU GPL version 3 or higher. + -} + +{-# LANGUAGE TupleSections, BangPatterns #-} + +module Command.Export where + +import Command +import qualified Annex +import qualified Git +import qualified Git.DiffTree +import qualified Git.LsTree +import qualified Git.Ref +import Git.Types +import Git.FilePath +import Git.Sha +import Types.Remote +import Types.Export +import Annex.Export +import Annex.Content +import Annex.Transfer +import Annex.CatFile +import Annex.LockFile +import Logs.Location +import Logs.Export +import Database.Export +import Messages.Progress +import Config +import Utility.Tmp +import Utility.Metered + +import qualified Data.ByteString.Lazy as L +import qualified Data.Map as M +import Control.Concurrent + +cmd :: Command +cmd = command "export" SectionCommon + "export content to a remote" + paramTreeish (seek <$$> optParser) + +data ExportOptions = ExportOptions + { exportTreeish :: Git.Ref + , exportRemote :: DeferredParse Remote + , exportTracking :: Bool + } + +optParser :: CmdParamsDesc -> Parser ExportOptions +optParser _ = ExportOptions + <$> (Git.Ref <$> parsetreeish) + <*> (parseRemoteOption <$> parseToOption) + <*> parsetracking + where + parsetreeish = argument str + ( metavar paramTreeish + ) + parsetracking = switch + ( long "tracking" + <> help ("track changes to the " ++ paramTreeish) + ) + +-- To handle renames which swap files, the exported file is first renamed +-- to a stable temporary name based on the key. +exportTempName :: ExportKey -> ExportLocation +exportTempName ek = mkExportLocation $ + ".git-annex-tmp-content-" ++ key2file (asKey (ek)) + +seek :: ExportOptions -> CommandSeek +seek o = do + r <- getParsed (exportRemote o) + unlessM (isExportSupported r) $ + giveup "That remote does not support exports." + when (exportTracking o) $ + setConfig (remoteConfig r "export-tracking") + (fromRef $ exportTreeish o) + new <- fromMaybe (giveup "unknown tree") <$> + -- Dereference the tree pointed to by the branch, commit, + -- or tag. + inRepo (Git.Ref.tree (exportTreeish o)) + withExclusiveLock (gitAnnexExportLock (uuid r)) $ do + db <- openDb (uuid r) + ea <- exportActions r + changeExport r ea db new + unlessM (Annex.getState Annex.fast) $ + void $ fillExport r ea db new + closeDb db + +-- | Changes what's exported to the remote. Does not upload any new +-- files, but does delete and rename files already exported to the remote. +changeExport :: Remote -> ExportActions Annex -> ExportHandle -> Git.Ref -> CommandSeek +changeExport r ea db new = do + old <- getExport (uuid r) + recordExportBeginning (uuid r) new + + -- Clean up after incomplete export of a tree, in which + -- the next block of code below may have renamed some files to + -- temp files. Diff from the incomplete tree to the new tree, + -- and delete any temp files that the new tree can't use. + let recover diff = commandAction $ + startRecoverIncomplete r ea db + (Git.DiffTree.srcsha diff) + (Git.DiffTree.file diff) + forM_ (concatMap incompleteExportedTreeish old) $ \incomplete -> + mapdiff recover incomplete new + + -- Diff the old and new trees, and delete or rename to new name all + -- changed files in the export. After this, every file that remains + -- in the export will have the content from the new treeish. + -- + -- When there was an export conflict, this resolves it. + -- + -- The ExportTree is also updated here to reflect the new tree. + case map exportedTreeish old of + [] -> updateExportTree db emptyTree new + [oldtreesha] -> do + diffmap <- mkDiffMap oldtreesha new db + let seekdiffmap a = commandActions $ + map a (M.toList diffmap) + -- Rename old files to temp, or delete. + seekdiffmap $ \(ek, (moldf, mnewf)) -> do + case (moldf, mnewf) of + (Just oldf, Just _newf) -> + startMoveToTempName r ea db oldf ek + (Just oldf, Nothing) -> + startUnexport' r ea db oldf ek + _ -> stop + -- Rename from temp to new files. + seekdiffmap $ \(ek, (moldf, mnewf)) -> + case (moldf, mnewf) of + (Just _oldf, Just newf) -> + startMoveFromTempName r ea db ek newf + _ -> stop + ts -> do + warning "Export conflict detected. Different trees have been exported to the same special remote. Resolving.." + forM_ ts $ \oldtreesha -> do + -- Unexport both the srcsha and the dstsha, + -- because the wrong content may have + -- been renamed to the dstsha due to the + -- export conflict. + let unexportboth d = + [ Git.DiffTree.srcsha d + , Git.DiffTree.dstsha d + ] + -- Don't rename to temp, because the + -- content is unknown; delete instead. + mapdiff + (\diff -> commandAction $ startUnexport r ea db (Git.DiffTree.file diff) (unexportboth diff)) + oldtreesha new + updateExportTree db emptyTree new + liftIO $ recordExportTreeCurrent db new + + -- Waiting until now to record the export guarantees that, + -- if this export is interrupted, there are no files left over + -- from a previous export, that are not part of this export. + c <- Annex.getState Annex.errcounter + when (c == 0) $ do + recordExport (uuid r) $ ExportChange + { oldTreeish = map exportedTreeish old + , newTreeish = new + } + where + mapdiff a oldtreesha newtreesha = do + (diff, cleanup) <- inRepo $ + Git.DiffTree.diffTreeRecursive oldtreesha newtreesha + seekActions $ pure $ map a diff + void $ liftIO cleanup + +-- Map of old and new filenames for each changed ExportKey in a diff. +type DiffMap = M.Map ExportKey (Maybe TopFilePath, Maybe TopFilePath) + +mkDiffMap :: Git.Ref -> Git.Ref -> ExportHandle -> Annex DiffMap +mkDiffMap old new db = do + (diff, cleanup) <- inRepo $ Git.DiffTree.diffTreeRecursive old new + diffmap <- M.fromListWith combinedm . concat <$> forM diff mkdm + void $ liftIO cleanup + return diffmap + where + combinedm (srca, dsta) (srcb, dstb) = (srca <|> srcb, dsta <|> dstb) + mkdm i = do + srcek <- getek (Git.DiffTree.srcsha i) + dstek <- getek (Git.DiffTree.dstsha i) + updateExportTree' db srcek dstek i + return $ catMaybes + [ (, (Just (Git.DiffTree.file i), Nothing)) <$> srcek + , (, (Nothing, Just (Git.DiffTree.file i))) <$> dstek + ] + getek sha + | sha == nullSha = return Nothing + | otherwise = Just <$> exportKey sha + +-- | Upload all exported files that are not yet in the remote, +-- Returns True when files were uploaded. +fillExport :: Remote -> ExportActions Annex -> ExportHandle -> Git.Ref -> Annex Bool +fillExport r ea db new = do + (l, cleanup) <- inRepo $ Git.LsTree.lsTree new + cvar <- liftIO $ newMVar False + commandActions $ map (startExport r ea db cvar) l + void $ liftIO $ cleanup + liftIO $ takeMVar cvar + +startExport :: Remote -> ExportActions Annex -> ExportHandle -> MVar Bool -> Git.LsTree.TreeItem -> CommandStart +startExport r ea db cvar ti = do + ek <- exportKey (Git.LsTree.sha ti) + stopUnless (notpresent ek) $ do + showStart ("export " ++ name r) f + liftIO $ modifyMVar_ cvar (pure . const True) + next $ performExport r ea db ek af (Git.LsTree.sha ti) loc + where + loc = mkExportLocation f + f = getTopFilePath (Git.LsTree.file ti) + af = AssociatedFile (Just f) + notpresent ek = (||) + <$> liftIO (notElem loc <$> getExportedLocation db (asKey ek)) + -- If content was removed from the remote, the export db + -- will still list it, so also check location tracking. + <*> (notElem (uuid r) <$> loggedLocations (asKey ek)) + +performExport :: Remote -> ExportActions Annex -> ExportHandle -> ExportKey -> AssociatedFile -> Sha -> ExportLocation -> CommandPerform +performExport r ea db ek af contentsha loc = do + let storer = storeExport ea + sent <- case ek of + AnnexKey k -> ifM (inAnnex k) + ( notifyTransfer Upload af $ + -- Using noRetry here because interrupted + -- exports cannot be resumed. + upload (uuid r) k af noRetry $ \pm -> do + let rollback = void $ + performUnexport r ea db [ek] loc + sendAnnex k rollback $ \f -> + metered Nothing k (return $ Just f) $ \_ m -> do + let m' = combineMeterUpdate pm m + storer f k loc m' + , do + showNote "not available" + return False + ) + -- Sending a non-annexed file. + GitKey sha1k -> metered Nothing sha1k (return Nothing) $ \_ m -> + withTmpFile "export" $ \tmp h -> do + b <- catObject contentsha + liftIO $ L.hPut h b + liftIO $ hClose h + storer tmp sha1k loc m + if sent + then next $ cleanupExport r db ek loc + else stop + +cleanupExport :: Remote -> ExportHandle -> ExportKey -> ExportLocation -> CommandCleanup +cleanupExport r db ek loc = do + liftIO $ addExportedLocation db (asKey ek) loc + logChange (asKey ek) (uuid r) InfoPresent + return True + +startUnexport :: Remote -> ExportActions Annex -> ExportHandle -> TopFilePath -> [Git.Sha] -> CommandStart +startUnexport r ea db f shas = do + eks <- forM (filter (/= nullSha) shas) exportKey + if null eks + then stop + else do + showStart ("unexport " ++ name r) f' + next $ performUnexport r ea db eks loc + where + loc = mkExportLocation f' + f' = getTopFilePath f + +startUnexport' :: Remote -> ExportActions Annex -> ExportHandle -> TopFilePath -> ExportKey -> CommandStart +startUnexport' r ea db f ek = do + showStart ("unexport " ++ name r) f' + next $ performUnexport r ea db [ek] loc + where + loc = mkExportLocation f' + f' = getTopFilePath f + +performUnexport :: Remote -> ExportActions Annex -> ExportHandle -> [ExportKey] -> ExportLocation -> CommandPerform +performUnexport r ea db eks loc = do + ifM (allM (\ek -> removeExport ea (asKey ek) loc) eks) + ( next $ cleanupUnexport r ea db eks loc + , stop + ) + +cleanupUnexport :: Remote -> ExportActions Annex -> ExportHandle -> [ExportKey] -> ExportLocation -> CommandCleanup +cleanupUnexport r ea db eks loc = do + liftIO $ do + forM_ eks $ \ek -> + removeExportedLocation db (asKey ek) loc + flushDbQueue db + + -- An appendonly remote can support removeExportLocation to remove + -- the file from the exported tree, but still retain the content + -- and allow retrieving it. + unless (appendonly r) $ do + remaininglocs <- liftIO $ + concat <$> forM eks (\ek -> getExportedLocation db (asKey ek)) + when (null remaininglocs) $ + forM_ eks $ \ek -> + logChange (asKey ek) (uuid r) InfoMissing + + removeEmptyDirectories ea db loc (map asKey eks) + +startRecoverIncomplete :: Remote -> ExportActions Annex -> ExportHandle -> Git.Sha -> TopFilePath -> CommandStart +startRecoverIncomplete r ea db sha oldf + | sha == nullSha = stop + | otherwise = do + ek <- exportKey sha + let loc = exportTempName ek + showStart ("unexport " ++ name r) (fromExportLocation loc) + liftIO $ removeExportedLocation db (asKey ek) oldloc + next $ performUnexport r ea db [ek] loc + where + oldloc = mkExportLocation oldf' + oldf' = getTopFilePath oldf + +startMoveToTempName :: Remote -> ExportActions Annex -> ExportHandle -> TopFilePath -> ExportKey -> CommandStart +startMoveToTempName r ea db f ek = do + showStart ("rename " ++ name r) (f' ++ " -> " ++ fromExportLocation tmploc) + next $ performRename r ea db ek loc tmploc + where + loc = mkExportLocation f' + f' = getTopFilePath f + tmploc = exportTempName ek + +startMoveFromTempName :: Remote -> ExportActions Annex -> ExportHandle -> ExportKey -> TopFilePath -> CommandStart +startMoveFromTempName r ea db ek f = do + let tmploc = exportTempName ek + stopUnless (liftIO $ elem tmploc <$> getExportedLocation db (asKey ek)) $ do + showStart ("rename " ++ name r) (fromExportLocation tmploc ++ " -> " ++ f') + next $ performRename r ea db ek tmploc loc + where + loc = mkExportLocation f' + f' = getTopFilePath f + +performRename :: Remote -> ExportActions Annex -> ExportHandle -> ExportKey -> ExportLocation -> ExportLocation -> CommandPerform +performRename r ea db ek src dest = do + ifM (renameExport ea (asKey ek) src dest) + ( next $ cleanupRename ea db ek src dest + -- In case the special remote does not support renaming, + -- unexport the src instead. + , do + warning "rename failed; deleting instead" + performUnexport r ea db [ek] src + ) + +cleanupRename :: ExportActions Annex -> ExportHandle -> ExportKey -> ExportLocation -> ExportLocation -> CommandCleanup +cleanupRename ea db ek src dest = do + liftIO $ do + removeExportedLocation db (asKey ek) src + addExportedLocation db (asKey ek) dest + flushDbQueue db + if exportDirectories src /= exportDirectories dest + then removeEmptyDirectories ea db src [asKey ek] + else return True + +-- | Remove empty directories from the export. Call after removing an +-- exported file, and after calling removeExportLocation and flushing the +-- database. +removeEmptyDirectories :: ExportActions Annex -> ExportHandle -> ExportLocation -> [Key] -> Annex Bool +removeEmptyDirectories ea db loc ks + | null (exportDirectories loc) = return True + | otherwise = case removeExportDirectory ea of + Nothing -> return True + Just removeexportdirectory -> do + ok <- allM (go removeexportdirectory) + (reverse (exportDirectories loc)) + unless ok $ liftIO $ do + -- Add location back to export database, + -- so this is tried again next time. + forM_ ks $ \k -> + addExportedLocation db k loc + flushDbQueue db + return ok + where + go removeexportdirectory d = + ifM (liftIO $ isExportDirectoryEmpty db d) + ( removeexportdirectory d + , return True + ) diff --git a/Command/Find.hs b/Command/Find.hs new file mode 100644 index 0000000000..ab4707c910 --- /dev/null +++ b/Command/Find.hs @@ -0,0 +1,91 @@ +{- git-annex command + - + - Copyright 2010-2012 Joey Hess + - + - Licensed under the GNU GPL version 3 or higher. + -} + +module Command.Find where + +import Data.Default +import qualified Data.Map as M + +import Command +import Annex.Content +import Limit +import Types.Key +import qualified Utility.Format +import Utility.DataUnits + +cmd :: Command +cmd = withGlobalOptions [annexedMatchingOptions] $ mkCommand $ + command "find" SectionQuery "lists available files" + paramPaths (seek <$$> optParser) + +mkCommand :: Command -> Command +mkCommand = noCommit . noMessages . withGlobalOptions [jsonOptions] + +data FindOptions = FindOptions + { findThese :: CmdParams + , formatOption :: Maybe Utility.Format.Format + , batchOption :: BatchMode + } + +optParser :: CmdParamsDesc -> Parser FindOptions +optParser desc = FindOptions + <$> cmdParams desc + <*> optional parseFormatOption + <*> parseBatchOption + +parseFormatOption :: Parser Utility.Format.Format +parseFormatOption = + option (Utility.Format.gen <$> str) + ( long "format" <> metavar paramFormat + <> help "control format of output" + ) + <|> flag' (Utility.Format.gen "${file}\0") + ( long "print0" + <> help "output filenames terminated with nulls" + ) + +seek :: FindOptions -> CommandSeek +seek o = case batchOption o of + NoBatch -> withFilesInGit (commandAction . go) + =<< workTreeItems (findThese o) + Batch fmt -> batchFilesMatching fmt go + where + go = whenAnnexed $ start o + +-- only files inAnnex are shown, unless the user has requested +-- others via a limit +start :: FindOptions -> FilePath -> Key -> CommandStart +start o file key = ifM (limited <||> inAnnex key) + ( do + showFormatted (formatOption o) file $ ("file", file) : keyVars key + next $ next $ return True + , stop + ) + +showFormatted :: Maybe Utility.Format.Format -> String -> [(String, String)] -> Annex () +showFormatted format unformatted vars = + unlessM (showFullJSON $ JSONChunk vars) $ + case format of + Nothing -> liftIO $ putStrLn unformatted + Just formatter -> liftIO $ putStr $ + Utility.Format.format formatter $ + M.fromList vars + +keyVars :: Key -> [(String, String)] +keyVars key = + [ ("key", key2file key) + , ("backend", formatKeyVariety $ keyVariety key) + , ("bytesize", size show) + , ("humansize", size $ roughSize storageUnits True) + , ("keyname", keyName key) + , ("hashdirlower", hashDirLower def key) + , ("hashdirmixed", hashDirMixed def key) + , ("mtime", whenavail show $ keyMtime key) + ] + where + size c = whenavail c $ keySize key + whenavail = maybe "unknown" diff --git a/Command/FindRef.hs b/Command/FindRef.hs new file mode 100644 index 0000000000..b829a070f2 --- /dev/null +++ b/Command/FindRef.hs @@ -0,0 +1,22 @@ +{- git-annex command + - + - Copyright 2014 Joey Hess + - + - Licensed under the GNU GPL version 3 or higher. + -} + +module Command.FindRef where + +import Command +import qualified Command.Find as Find +import qualified Git + +cmd :: Command +cmd = withGlobalOptions [nonWorkTreeMatchingOptions] $ Find.mkCommand $ + command "findref" SectionPlumbing + "lists files in a git ref" + paramRef (seek <$$> Find.optParser) + +seek :: Find.FindOptions -> CommandSeek +seek o = (commandAction . uncurry (Find.start o)) + `withFilesInRefs` (map Git.Ref $ Find.findThese o) diff --git a/Command/Fix.hs b/Command/Fix.hs new file mode 100644 index 0000000000..4439616761 --- /dev/null +++ b/Command/Fix.hs @@ -0,0 +1,110 @@ +{- git-annex command + - + - Copyright 2010-2015 Joey Hess + - + - Licensed under the GNU GPL version 3 or higher. + -} + +{-# LANGUAGE CPP #-} + +module Command.Fix where + +import Command +import Config +import qualified Annex +import Annex.Version +import Annex.ReplaceFile +import Annex.Content +import Annex.Perms +import qualified Annex.Queue +import qualified Database.Keys +#if ! defined(mingw32_HOST_OS) && ! defined(__ANDROID__) +import Utility.Touch +#endif + +cmd :: Command +cmd = notDirect $ noCommit $ withGlobalOptions [annexedMatchingOptions] $ + command "fix" SectionMaintenance + "fix up links to annexed content" + paramPaths (withParams seek) + +seek :: CmdParams -> CommandSeek +seek ps = unlessM crippledFileSystem $ do + fixwhat <- ifM versionSupportsUnlockedPointers + ( return FixAll + , return FixSymlinks + ) + withFilesInGit + (commandAction . (whenAnnexed $ start fixwhat)) + =<< workTreeItems ps + +data FixWhat = FixSymlinks | FixAll + +start :: FixWhat -> FilePath -> Key -> CommandStart +start fixwhat file key = do + currlink <- liftIO $ catchMaybeIO $ readSymbolicLink file + wantlink <- calcRepo $ gitAnnexLink file key + case currlink of + Just l + | l /= wantlink -> fixby $ fixSymlink file wantlink + | otherwise -> stop + Nothing -> case fixwhat of + FixAll -> fixthin + FixSymlinks -> stop + where + fixby a = do + showStart "fix" file + next a + fixthin = do + obj <- calcRepo $ gitAnnexLocation key + stopUnless (isUnmodified key file <&&> isUnmodified key obj) $ do + thin <- annexThin <$> Annex.getGitConfig + fs <- liftIO $ catchMaybeIO $ getFileStatus file + os <- liftIO $ catchMaybeIO $ getFileStatus obj + case (linkCount <$> fs, linkCount <$> os, thin) of + (Just 1, Just 1, True) -> + fixby $ makeHardLink file key + (Just n, Just n', False) | n > 1 && n == n' -> + fixby $ breakHardLink file key obj + _ -> stop + +breakHardLink :: FilePath -> Key -> FilePath -> CommandPerform +breakHardLink file key obj = do + replaceFile file $ \tmp -> do + mode <- liftIO $ catchMaybeIO $ fileMode <$> getFileStatus file + unlessM (checkedCopyFile key obj tmp mode) $ + error "unable to break hard link" + thawContent tmp + modifyContent obj $ freezeContent obj + Database.Keys.storeInodeCaches key [file] + next $ return True + +makeHardLink :: FilePath -> Key -> CommandPerform +makeHardLink file key = do + replaceFile file $ \tmp -> do + mode <- liftIO $ catchMaybeIO $ fileMode <$> getFileStatus file + linkFromAnnex key tmp mode >>= \case + LinkAnnexFailed -> error "unable to make hard link" + _ -> noop + next $ return True + +fixSymlink :: FilePath -> FilePath -> CommandPerform +fixSymlink file link = do + liftIO $ do +#if ! defined(mingw32_HOST_OS) && ! defined(__ANDROID__) + -- preserve mtime of symlink + mtime <- catchMaybeIO $ TimeSpec . modificationTime + <$> getSymbolicLinkStatus file +#endif + createDirectoryIfMissing True (parentDir file) + removeFile file + createSymbolicLink link file +#if ! defined(mingw32_HOST_OS) && ! defined(__ANDROID__) + maybe noop (\t -> touch file t False) mtime +#endif + next $ cleanupSymlink file + +cleanupSymlink :: FilePath -> CommandCleanup +cleanupSymlink file = do + Annex.Queue.addCommand "add" [Param "--force", Param "--"] [file] + return True diff --git a/Command/Forget.hs b/Command/Forget.hs new file mode 100644 index 0000000000..40ac98d4bf --- /dev/null +++ b/Command/Forget.hs @@ -0,0 +1,54 @@ +{- git-annex command + - + - Copyright 2013 Joey Hess + - + - Licensed under the GNU GPL version 3 or higher. + -} + +module Command.Forget where + +import Command +import qualified Annex.Branch as Branch +import Logs.Transitions +import qualified Annex +import Annex.VectorClock + +cmd :: Command +cmd = command "forget" SectionMaintenance + "prune git-annex branch history" + paramNothing (seek <$$> optParser) + +data ForgetOptions = ForgetOptions + { dropDead :: Bool + } + +optParser :: CmdParamsDesc -> Parser ForgetOptions +optParser _ = ForgetOptions + <$> switch + ( long "drop-dead" + <> help "drop references to dead repositories" + ) + +seek :: ForgetOptions -> CommandSeek +seek = commandAction . start + +start :: ForgetOptions -> CommandStart +start o = do + showStart' "forget" (Just "git-annex") + c <- liftIO currentVectorClock + let basets = addTransition c ForgetGitHistory noTransitions + let ts = if dropDead o + then addTransition c ForgetDeadRemotes basets + else basets + next $ perform ts =<< Annex.getState Annex.force + +perform :: Transitions -> Bool -> CommandPerform +perform ts True = do + recordTransitions Branch.change ts + -- get branch committed before contining with the transition + Branch.update + void $ Branch.performTransitions ts True [] + next $ return True +perform _ False = do + showLongNote "To forget git-annex branch history, you must specify --force. This deletes metadata!" + stop diff --git a/Command/FromKey.hs b/Command/FromKey.hs new file mode 100644 index 0000000000..4b48f4e5c2 --- /dev/null +++ b/Command/FromKey.hs @@ -0,0 +1,97 @@ +{- git-annex command + - + - Copyright 2010-2018 Joey Hess + - + - Licensed under the GNU GPL version 3 or higher. + -} + +{-# LANGUAGE BangPatterns #-} + +module Command.FromKey where + +import Command +import qualified Annex.Queue +import Annex.Content +import qualified Annex +import qualified Backend.URL + +import Network.URI + +cmd :: Command +cmd = notDirect $ notBareRepo $ + command "fromkey" SectionPlumbing "adds a file using a specific key" + (paramRepeating (paramPair paramKey paramPath)) + (seek <$$> optParser) + +data FromKeyOptions = FromKeyOptions + { keyFilePairs :: CmdParams + , batchOption :: BatchMode + } + +optParser :: CmdParamsDesc -> Parser FromKeyOptions +optParser desc = FromKeyOptions + <$> cmdParams desc + <*> parseBatchOption + +seek :: FromKeyOptions -> CommandSeek +seek o = case (batchOption o, keyFilePairs o) of + (Batch fmt, _) -> commandAction $ startMass fmt + -- older way of enabling batch input, does not support BatchNull + (NoBatch, []) -> commandAction $ startMass BatchLine + (NoBatch, ps) -> do + force <- Annex.getState Annex.force + withPairs (commandAction . start force) ps + +start :: Bool -> (String, FilePath) -> CommandStart +start force (keyname, file) = do + let key = mkKey keyname + unless force $ do + inbackend <- inAnnex key + unless inbackend $ giveup $ + "key ("++ keyname ++") is not present in backend (use --force to override this sanity check)" + showStart "fromkey" file + next $ perform key file + +startMass :: BatchFormat -> CommandStart +startMass fmt = do + showStart' "fromkey" (Just "stdin") + next (massAdd fmt) + +massAdd :: BatchFormat -> CommandPerform +massAdd fmt = go True =<< map (separate (== ' ')) <$> batchLines fmt + where + go status [] = next $ return status + go status ((keyname,f):rest) | not (null keyname) && not (null f) = do + let key = mkKey keyname + ok <- perform' key f + let !status' = status && ok + go status' rest + go _ _ = giveup "Expected pairs of key and file on stdin, but got something else." + +-- From user input to a Key. +-- User can input either a serialized key, or an url. +-- +-- In some cases, an input can be parsed as both a key and as an uri. +-- For example, "WORM--a:a" parses as an uri. To disambiguate, check +-- the uri scheme, to see if it looks like the prefix of a key. This relies +-- on key backend names never containing a ':'. +mkKey :: String -> Key +mkKey s = case parseURI s of + Just u | not (isKeyPrefix (uriScheme u)) -> + Backend.URL.fromUrl s Nothing + _ -> case file2key s of + Just k -> k + Nothing -> giveup $ "bad key/url " ++ s + +perform :: Key -> FilePath -> CommandPerform +perform key file = do + ok <- perform' key file + next $ return ok + +perform' :: Key -> FilePath -> Annex Bool +perform' key file = do + link <- calcRepo $ gitAnnexLink file key + liftIO $ createDirectoryIfMissing True (parentDir file) + liftIO $ createSymbolicLink link file + Annex.Queue.addCommand "add" [Param "--"] [file] + return True diff --git a/Command/Fsck.hs b/Command/Fsck.hs new file mode 100644 index 0000000000..6608b71502 --- /dev/null +++ b/Command/Fsck.hs @@ -0,0 +1,728 @@ +{- git-annex command + - + - Copyright 2010-2018 Joey Hess + - + - Licensed under the GNU GPL version 3 or higher. + -} + +{-# LANGUAGE CPP #-} + +module Command.Fsck where + +import Command +import qualified Annex +import qualified Remote +import qualified Types.Backend +import qualified Backend +import Annex.Content +import qualified Annex.Content.Direct as Direct +import Annex.Direct +import Annex.Perms +import Annex.Link +import Logs.Location +import Logs.Trust +import Logs.Activity +import Logs.TimeStamp +import Logs.PreferredContent +import Annex.NumCopies +import Annex.UUID +import Annex.ReplaceFile +import Utility.DataUnits +import Config +import Utility.HumanTime +import Utility.CopyFile +import Git.FilePath +import Utility.PID +import qualified Database.Keys +import qualified Database.Fsck as FsckDb +import Types.CleanupActions +import Types.Key +import Types.ActionItem + +import Data.Time.Clock.POSIX +import System.Posix.Types (EpochTime) +import qualified Data.Set as S +import qualified Data.Map as M + +cmd :: Command +cmd = withGlobalOptions [jobsOption, jsonOptions, annexedMatchingOptions] $ + command "fsck" SectionMaintenance + "find and fix problems" + paramPaths (seek <$$> optParser) + +data FsckOptions = FsckOptions + { fsckFiles :: CmdParams + , fsckFromOption :: Maybe (DeferredParse Remote) + , incrementalOpt :: Maybe IncrementalOpt + , keyOptions :: Maybe KeyOptions + } + +data IncrementalOpt + = StartIncrementalO + | MoreIncrementalO + | ScheduleIncrementalO Duration + +optParser :: CmdParamsDesc -> Parser FsckOptions +optParser desc = FsckOptions + <$> cmdParams desc + <*> optional (parseRemoteOption <$> strOption + ( long "from" <> short 'f' <> metavar paramRemote + <> help "check remote" + <> completeRemotes + )) + <*> optional parseincremental + <*> optional parseKeyOptions + where + parseincremental = + flag' StartIncrementalO + ( long "incremental" <> short 'S' + <> help "start an incremental fsck" + ) + <|> flag' MoreIncrementalO + ( long "more" <> short 'm' + <> help "continue an incremental fsck" + ) + <|> (ScheduleIncrementalO <$> option (str >>= parseDuration) + ( long "incremental-schedule" <> metavar paramTime + <> help "schedule incremental fscking" + )) + +seek :: FsckOptions -> CommandSeek +seek o = allowConcurrentOutput $ do + from <- maybe (pure Nothing) (Just <$$> getParsed) (fsckFromOption o) + u <- maybe getUUID (pure . Remote.uuid) from + checkDeadRepo u + i <- prepIncremental u (incrementalOpt o) + withKeyOptions (keyOptions o) False + (\kai -> commandAction . startKey from i kai =<< getNumCopies) + (withFilesInGit $ commandAction . (whenAnnexed (start from i))) + =<< workTreeItems (fsckFiles o) + cleanupIncremental i + void $ tryIO $ recordActivity Fsck u + +checkDeadRepo :: UUID -> Annex () +checkDeadRepo u = + whenM ((==) DeadTrusted <$> lookupTrust u) $ + earlyWarning "Warning: Fscking a repository that is currently marked as dead." + +start :: Maybe Remote -> Incremental -> FilePath -> Key -> CommandStart +start from inc file key = Backend.getBackend file key >>= \case + Nothing -> stop + Just backend -> do + numcopies <- getFileNumCopies file + case from of + Nothing -> go $ perform key file backend numcopies + Just r -> go $ performRemote key afile backend numcopies r + where + go = runFsck inc (mkActionItem afile) key + afile = AssociatedFile (Just file) + +perform :: Key -> FilePath -> Backend -> NumCopies -> Annex Bool +perform key file backend numcopies = do + keystatus <- getKeyFileStatus key file + check + -- order matters + [ fixLink key file + , verifyLocationLog key keystatus ai + , verifyRequiredContent key ai + , verifyAssociatedFiles key keystatus file + , verifyWorkTree key file + , checkKeySize key keystatus ai + , checkBackend backend key keystatus afile + , checkKeyUpgrade backend key ai afile + , checkKeyNumCopies key afile numcopies + ] + where + afile = AssociatedFile (Just file) + ai = ActionItemAssociatedFile afile + +{- To fsck a remote, the content is retrieved to a tmp file, + - and checked locally. -} +performRemote :: Key -> AssociatedFile -> Backend -> NumCopies -> Remote -> Annex Bool +performRemote key afile backend numcopies remote = + dispatch =<< Remote.hasKey remote key + where + dispatch (Left err) = do + showNote err + return False + dispatch (Right True) = withtmp $ \tmpfile -> + getfile tmpfile >>= \case + Nothing -> go True Nothing + Just True -> go True (Just tmpfile) + Just False -> do + warning "failed to download file from remote" + void $ go True Nothing + return False + dispatch (Right False) = go False Nothing + go present localcopy = check + [ verifyLocationLogRemote key ai remote present + , verifyRequiredContent key ai + , withLocalCopy localcopy $ checkKeySizeRemote key remote ai + , withLocalCopy localcopy $ checkBackendRemote backend key remote ai + , checkKeyNumCopies key afile numcopies + ] + ai = ActionItemAssociatedFile afile + withtmp a = do + pid <- liftIO getPID + t <- fromRepo gitAnnexTmpObjectDir + createAnnexDirectory t + let tmp = t "fsck" ++ show pid ++ "." ++ keyFile key + let cleanup = liftIO $ catchIO (removeFile tmp) (const noop) + cleanup + cleanup `after` a tmp + getfile tmp = ifM (checkDiskSpace (Just (takeDirectory tmp)) key 0 True) + ( ifM (Remote.retrieveKeyFileCheap remote key afile tmp) + ( return (Just True) + , ifM (Annex.getState Annex.fast) + ( return Nothing + , Just . fst <$> + Remote.retrieveKeyFile remote key (AssociatedFile Nothing) tmp dummymeter + ) + ) + , return (Just False) + ) + dummymeter _ = noop + +startKey :: Maybe Remote -> Incremental -> (Key, ActionItem) -> NumCopies -> CommandStart +startKey from inc (key, ai) numcopies = + case Backend.maybeLookupBackendVariety (keyVariety key) of + Nothing -> stop + Just backend -> runFsck inc ai key $ + case from of + Nothing -> performKey key backend numcopies + Just r -> performRemote key (AssociatedFile Nothing) backend numcopies r + +performKey :: Key -> Backend -> NumCopies -> Annex Bool +performKey key backend numcopies = do + keystatus <- getKeyStatus key + check + [ verifyLocationLog key keystatus (mkActionItem key) + , checkKeySize key keystatus (mkActionItem key) + , checkBackend backend key keystatus (AssociatedFile Nothing) + , checkKeyNumCopies key (AssociatedFile Nothing) numcopies + ] + +check :: [Annex Bool] -> Annex Bool +check cs = and <$> sequence cs + +{- Checks that symlinks points correctly to the annexed content. + -} +fixLink :: Key -> FilePath -> Annex Bool +fixLink key file = do + want <- calcRepo $ gitAnnexLink file key + have <- getAnnexLinkTarget file + maybe noop (go want) have + return True + where + go want have + | want /= fromInternalGitPath have = do + showNote "fixing link" + liftIO $ createDirectoryIfMissing True (parentDir file) + liftIO $ removeFile file + addAnnexLink want file + | otherwise = noop + +{- Checks that the location log reflects the current status of the key, + - in this repository only. -} +verifyLocationLog :: Key -> KeyStatus -> ActionItem -> Annex Bool +verifyLocationLog key keystatus ai = do + direct <- isDirect + obj <- calcRepo $ gitAnnexLocation key + present <- if not direct && isKeyUnlocked keystatus + then liftIO (doesFileExist obj) + else inAnnex key + u <- getUUID + + {- Since we're checking that a key's object file is present, throw + - in a permission fixup here too. -} + when (present && not direct) $ do + void $ tryIO $ if isKeyUnlocked keystatus + then thawContent obj + else freezeContent obj + unlessM (isContentWritePermOk obj) $ + warning $ "** Unable to set correct write mode for " ++ obj ++ " ; perhaps you don't own that file" + whenM (liftIO $ doesDirectoryExist $ parentDir obj) $ + freezeContentDir obj + + {- Warn when annex.securehashesonly is set and content using an + - insecure hash is present. This should only be able to happen + - if the repository already contained the content before the + - config was set. -} + when (present && not (cryptographicallySecure (keyVariety key))) $ + whenM (annexSecureHashesOnly <$> Annex.getGitConfig) $ + warning $ "** Despite annex.securehashesonly being set, " ++ obj ++ " has content present in the annex using an insecure " ++ formatKeyVariety (keyVariety key) ++ " key" + + {- In direct mode, modified files will show up as not present, + - but that is expected and not something to do anything about. -} + if direct && not present + then return True + else verifyLocationLog' key ai present u (logChange key u) + +verifyLocationLogRemote :: Key -> ActionItem -> Remote -> Bool -> Annex Bool +verifyLocationLogRemote key ai remote present = + verifyLocationLog' key ai present (Remote.uuid remote) + (Remote.logStatus remote key) + +verifyLocationLog' :: Key -> ActionItem -> Bool -> UUID -> (LogStatus -> Annex ()) -> Annex Bool +verifyLocationLog' key ai present u updatestatus = do + uuids <- loggedLocations key + case (present, u `elem` uuids) of + (True, False) -> do + fix InfoPresent + -- There is no data loss, so do not fail. + return True + (False, True) -> do + fix InfoMissing + warning $ + "** Based on the location log, " ++ + actionItemDesc ai key ++ + "\n** was expected to be present, " ++ + "but its content is missing." + return False + (False, False) -> do + -- When the location log for the key is not present, + -- create it, so that the key will be known. + when (null uuids) $ + whenM (not <$> isKnownKey key) $ + updatestatus InfoMissing + return True + (True, True) -> return True + where + fix s = do + showNote "fixing location log" + updatestatus s + +{- Verifies that all repos that are required to contain the content do, + - checking against the location log. -} +verifyRequiredContent :: Key -> ActionItem -> Annex Bool +verifyRequiredContent key ai@(ActionItemAssociatedFile afile) = do + requiredlocs <- S.fromList . M.keys <$> requiredContentMap + if S.null requiredlocs + then return True + else do + presentlocs <- S.fromList <$> loggedLocations key + missinglocs <- filterM + (\u -> isRequiredContent (Just u) S.empty (Just key) afile False) + (S.toList $ S.difference requiredlocs presentlocs) + if null missinglocs + then return True + else do + missingrequired <- Remote.prettyPrintUUIDs "missingrequired" missinglocs + warning $ + "** Required content " ++ + actionItemDesc ai key ++ + " is missing from these repositories:\n" ++ + missingrequired + return False +verifyRequiredContent _ _ = return True + +{- Verifies the associated file records. -} +verifyAssociatedFiles :: Key -> KeyStatus -> FilePath -> Annex Bool +verifyAssociatedFiles key keystatus file = do + ifM isDirect (godirect, goindirect) + return True + where + godirect = do + fs <- Direct.addAssociatedFile key file + forM_ fs $ \f -> + unlessM (liftIO $ doesFileExist f) $ + void $ Direct.removeAssociatedFile key f + goindirect = case keystatus of + KeyUnlocked -> do + f <- inRepo $ toTopFilePath file + afs <- Database.Keys.getAssociatedFiles key + unless (getTopFilePath f `elem` map getTopFilePath afs) $ + Database.Keys.addAssociatedFile key f + _ -> return () + +verifyWorkTree :: Key -> FilePath -> Annex Bool +verifyWorkTree key file = do + ifM isDirect ( godirect, goindirect ) + return True + where + {- Ensures that files whose content is available are in direct mode. -} + godirect = whenM (isJust <$> isAnnexLink file) $ do + v <- toDirectGen key file + case v of + Nothing -> noop + Just a -> do + showNote "fixing direct mode" + a + {- Make sure that a pointer file is replaced with its content, + - when the content is available. -} + goindirect = do + mk <- liftIO $ isPointerFile file + case mk of + Just k | k == key -> whenM (inAnnex key) $ do + showNote "fixing worktree content" + replaceFile file $ \tmp -> do + mode <- liftIO $ catchMaybeIO $ fileMode <$> getFileStatus file + ifM (annexThin <$> Annex.getGitConfig) + ( void $ linkFromAnnex key tmp mode + , do + obj <- calcRepo $ gitAnnexLocation key + void $ checkedCopyFile key obj tmp mode + thawContent tmp + ) + Database.Keys.storeInodeCaches key [file] + _ -> return () + +{- The size of the data for a key is checked against the size encoded in + - the key's metadata, if available. + - + - Not checked when a file is unlocked, or in direct mode. + -} +checkKeySize :: Key -> KeyStatus -> ActionItem -> Annex Bool +checkKeySize _ KeyUnlocked _ = return True +checkKeySize key _ ai = do + file <- calcRepo $ gitAnnexLocation key + ifM (liftIO $ doesFileExist file) + ( checkKeySizeOr badContent key file ai + , return True + ) + +withLocalCopy :: Maybe FilePath -> (FilePath -> Annex Bool) -> Annex Bool +withLocalCopy Nothing _ = return True +withLocalCopy (Just localcopy) f = f localcopy + +checkKeySizeRemote :: Key -> Remote -> ActionItem -> FilePath -> Annex Bool +checkKeySizeRemote key remote ai localcopy = + checkKeySizeOr (badContentRemote remote localcopy) key localcopy ai + +checkKeySizeOr :: (Key -> Annex String) -> Key -> FilePath -> ActionItem -> Annex Bool +checkKeySizeOr bad key file ai = case keySize key of + Nothing -> return True + Just size -> do + size' <- liftIO $ getFileSize file + comparesizes size size' + where + comparesizes a b = do + let same = a == b + unless same $ badsize a b + return same + badsize a b = do + msg <- bad key + warning $ concat + [ actionItemDesc ai key + , ": Bad file size (" + , compareSizes storageUnits True a b + , "); " + , msg + ] + +{- Check for keys that are upgradable. + - + - Warns and suggests the user migrate, but does not migrate itself, + - because migration can cause more disk space to be used, and makes + - worktree changes that need to be committed. + -} +checkKeyUpgrade :: Backend -> Key -> ActionItem -> AssociatedFile -> Annex Bool +checkKeyUpgrade backend key ai (AssociatedFile (Just file)) = + case Types.Backend.canUpgradeKey backend of + Just a | a key -> do + warning $ concat + [ actionItemDesc ai key + , ": Can be upgraded to an improved key format. " + , "You can do so by running: git annex migrate --backend=" + , formatKeyVariety (keyVariety key) ++ " " + , file + ] + return True + _ -> return True +checkKeyUpgrade _ _ _ (AssociatedFile Nothing) = + -- Don't suggest migrating without a filename, because + -- while possible to do, there is no actual benefit from + -- doing that in this situation. + return True + +{- Runs the backend specific check on a key's content object. + - + - When a file is unlocked, it may be a hard link to the object, + - thus when the user modifies the file, the object will be modified and + - not pass the check, and we don't want to find an error in this case. + - So, skip the check if the key is unlocked and modified. + - + - In direct mode this is not done if the file has clearly been modified, + - because modification of direct mode files is allowed. It's still done + - if the file does not appear modified, to catch disk corruption, etc. + -} +checkBackend :: Backend -> Key -> KeyStatus -> AssociatedFile -> Annex Bool +checkBackend backend key keystatus afile = go =<< isDirect + where + go False = do + content <- calcRepo $ gitAnnexLocation key + ifM (pure (isKeyUnlocked keystatus) <&&> (not <$> isUnmodified key content)) + ( nocheck + , checkBackendOr badContent backend key content (mkActionItem afile) + ) + go True = case afile of + AssociatedFile Nothing -> nocheck + AssociatedFile (Just f) -> checkdirect f + checkdirect file = ifM (Direct.goodContent key file) + ( checkBackendOr' (badContentDirect file) backend key file (mkActionItem afile) + (Direct.goodContent key file) + , nocheck + ) + nocheck = return True + +checkBackendRemote :: Backend -> Key -> Remote -> ActionItem -> FilePath -> Annex Bool +checkBackendRemote backend key remote ai localcopy = + checkBackendOr (badContentRemote remote localcopy) backend key localcopy ai + +checkBackendOr :: (Key -> Annex String) -> Backend -> Key -> FilePath -> ActionItem -> Annex Bool +checkBackendOr bad backend key file ai = + checkBackendOr' bad backend key file ai (return True) + +-- The postcheck action is run after the content is verified, +-- in order to detect situations where the file is changed while being +-- verified (particularly in direct mode). +checkBackendOr' :: (Key -> Annex String) -> Backend -> Key -> FilePath -> ActionItem -> Annex Bool -> Annex Bool +checkBackendOr' bad backend key file ai postcheck = + case Types.Backend.verifyKeyContent backend of + Nothing -> return True + Just verifier -> do + ok <- verifier key file + ifM postcheck + ( do + unless ok $ do + msg <- bad key + warning $ concat + [ actionItemDesc ai key + , ": Bad file content; " + , msg + ] + return ok + , return True + ) + +checkKeyNumCopies :: Key -> AssociatedFile -> NumCopies -> Annex Bool +checkKeyNumCopies key afile numcopies = do + let (desc, hasafile) = case afile of + AssociatedFile Nothing -> (key2file key, False) + AssociatedFile (Just af) -> (af, True) + locs <- loggedLocations key + (untrustedlocations, otherlocations) <- trustPartition UnTrusted locs + (deadlocations, safelocations) <- trustPartition DeadTrusted otherlocations + let present = NumCopies (length safelocations) + if present < numcopies + then ifM (pure (not hasafile) <&&> checkDead key) + ( do + showLongNote $ "This key is dead, skipping." + return True + , do + untrusted <- Remote.prettyPrintUUIDs "untrusted" untrustedlocations + dead <- Remote.prettyPrintUUIDs "dead" deadlocations + warning $ missingNote desc present numcopies untrusted dead + when (fromNumCopies present == 0 && not hasafile) $ + showLongNote "(Avoid this check by running: git annex dead --key )" + return False + ) + else return True + +missingNote :: String -> NumCopies -> NumCopies -> String -> String -> String +missingNote file (NumCopies 0) _ [] dead = + "** No known copies exist of " ++ file ++ honorDead dead +missingNote file (NumCopies 0) _ untrusted dead = + "Only these untrusted locations may have copies of " ++ file ++ + "\n" ++ untrusted ++ + "Back it up to trusted locations with git-annex copy." ++ honorDead dead +missingNote file present needed [] _ = + "Only " ++ show (fromNumCopies present) ++ " of " ++ show (fromNumCopies needed) ++ + " trustworthy copies exist of " ++ file ++ + "\nBack it up with git-annex copy." +missingNote file present needed untrusted dead = + missingNote file present needed [] dead ++ + "\nThe following untrusted locations may also have copies: " ++ + "\n" ++ untrusted + +honorDead :: String -> String +honorDead dead + | null dead = "" + | otherwise = "\nThese dead repositories used to have copies\n" ++ dead + +{- Bad content is moved aside. -} +badContent :: Key -> Annex String +badContent key = do + dest <- moveBad key + return $ "moved to " ++ dest + +{- Bad content is left where it is, but we touch the file, so it'll be + - committed to a new key. -} +badContentDirect :: FilePath -> Key -> Annex String +badContentDirect file key = do + void $ liftIO $ catchMaybeIO $ touchFile file + logStatus key InfoMissing + return "left in place for you to examine" + +{- Bad content is dropped from the remote. We have downloaded a copy + - from the remote to a temp file already (in some cases, it's just a + - symlink to a file in the remote). To avoid any further data loss, + - that temp file is moved to the bad content directory unless + - the local annex has a copy of the content. -} +badContentRemote :: Remote -> FilePath -> Key -> Annex String +badContentRemote remote localcopy key = do + bad <- fromRepo gitAnnexBadDir + let destbad = bad key2file key + movedbad <- ifM (inAnnex key <||> liftIO (doesFileExist destbad)) + ( return False + , do + createAnnexDirectory (parentDir destbad) + liftIO $ catchDefaultIO False $ + ifM (isSymbolicLink <$> getSymbolicLinkStatus localcopy) + ( copyFileExternal CopyTimeStamps localcopy destbad + , do + moveFile localcopy destbad + return True + ) + ) + + dropped <- Remote.removeKey remote key + when dropped $ + Remote.logStatus remote key InfoMissing + return $ case (movedbad, dropped) of + (True, True) -> "moved from " ++ Remote.name remote ++ + " to " ++ destbad + (False, True) -> "dropped from " ++ Remote.name remote + (_, False) -> "failed to drop from" ++ Remote.name remote + +runFsck :: Incremental -> ActionItem -> Key -> Annex Bool -> CommandStart +runFsck inc ai key a = ifM (needFsck inc key) + ( do + showStartKey "fsck" key ai + next $ do + ok <- a + when ok $ + recordFsckTime inc key + next $ return ok + , stop + ) + +{- Check if a key needs to be fscked, with support for incremental fscks. -} +needFsck :: Incremental -> Key -> Annex Bool +needFsck (ScheduleIncremental _ _ i) k = needFsck i k +needFsck (ContIncremental h) key = liftIO $ not <$> FsckDb.inDb h key +needFsck _ _ = return True + +recordFsckTime :: Incremental -> Key -> Annex () +recordFsckTime inc key = withFsckDb inc $ \h -> liftIO $ FsckDb.addDb h key + +{- Records the start time of an incremental fsck. + - + - To guard against time stamp damange (for example, if an annex directory + - is copied without -a), the fsckstate file contains a time that should + - be identical to its modification time. + - (This is not possible to do on Windows, and so the timestamp in + - the file will only be equal or greater than the modification time.) + -} +recordStartTime :: UUID -> Annex () +recordStartTime u = do + f <- fromRepo (gitAnnexFsckState u) + createAnnexDirectory $ parentDir f + liftIO $ nukeFile f + liftIO $ withFile f WriteMode $ \h -> do +#ifndef mingw32_HOST_OS + t <- modificationTime <$> getFileStatus f +#else + t <- getPOSIXTime +#endif + hPutStr h $ showTime $ realToFrac t + setAnnexFilePerm f + where + showTime :: POSIXTime -> String + showTime = show + +resetStartTime :: UUID -> Annex () +resetStartTime u = liftIO . nukeFile =<< fromRepo (gitAnnexFsckState u) + +{- Gets the incremental fsck start time. -} +getStartTime :: UUID -> Annex (Maybe EpochTime) +getStartTime u = do + f <- fromRepo (gitAnnexFsckState u) + liftIO $ catchDefaultIO Nothing $ do + timestamp <- modificationTime <$> getFileStatus f + let fromstatus = Just (realToFrac timestamp) + fromfile <- parsePOSIXTime <$> readFile f + return $ if matchingtimestamp fromfile fromstatus + then Just timestamp + else Nothing + where + matchingtimestamp fromfile fromstatus = +#ifndef mingw32_HOST_OS + fromfile == fromstatus +#else + fromfile >= fromstatus +#endif + +data Incremental + = NonIncremental + | ScheduleIncremental Duration UUID Incremental + | StartIncremental FsckDb.FsckHandle + | ContIncremental FsckDb.FsckHandle + +prepIncremental :: UUID -> Maybe IncrementalOpt -> Annex Incremental +prepIncremental _ Nothing = pure NonIncremental +prepIncremental u (Just StartIncrementalO) = do + recordStartTime u + ifM (FsckDb.newPass u) + ( StartIncremental <$> openFsckDb u + , giveup "Cannot start a new --incremental fsck pass; another fsck process is already running." + ) +prepIncremental u (Just MoreIncrementalO) = + ContIncremental <$> openFsckDb u +prepIncremental u (Just (ScheduleIncrementalO delta)) = do + started <- getStartTime u + i <- prepIncremental u $ Just $ case started of + Nothing -> StartIncrementalO + Just _ -> MoreIncrementalO + return (ScheduleIncremental delta u i) + +cleanupIncremental :: Incremental -> Annex () +cleanupIncremental (ScheduleIncremental delta u i) = do + v <- getStartTime u + case v of + Nothing -> noop + Just started -> do + now <- liftIO getPOSIXTime + when (now - realToFrac started >= durationToPOSIXTime delta) $ + resetStartTime u + cleanupIncremental i +cleanupIncremental _ = return () + +openFsckDb :: UUID -> Annex FsckDb.FsckHandle +openFsckDb u = do + h <- FsckDb.openDb u + Annex.addCleanup FsckCleanup $ + FsckDb.closeDb h + return h + +withFsckDb :: Incremental -> (FsckDb.FsckHandle -> Annex ()) -> Annex () +withFsckDb (ContIncremental h) a = a h +withFsckDb (StartIncremental h) a = a h +withFsckDb NonIncremental _ = noop +withFsckDb (ScheduleIncremental _ _ i) a = withFsckDb i a + +data KeyStatus = KeyLocked | KeyUnlocked | KeyMissing + +isKeyUnlocked :: KeyStatus -> Bool +isKeyUnlocked KeyUnlocked = True +isKeyUnlocked KeyLocked = False +isKeyUnlocked KeyMissing = False + +getKeyStatus :: Key -> Annex KeyStatus +getKeyStatus key = ifM isDirect + ( return KeyUnlocked + , catchDefaultIO KeyMissing $ do + unlocked <- not . null <$> Database.Keys.getAssociatedFiles key + return $ if unlocked then KeyUnlocked else KeyLocked + ) + +getKeyFileStatus :: Key -> FilePath -> Annex KeyStatus +getKeyFileStatus key file = do + s <- getKeyStatus key + case s of + KeyLocked -> catchDefaultIO KeyLocked $ + ifM (isJust <$> isAnnexLink file) + ( return KeyLocked + , return KeyUnlocked + ) + _ -> return s diff --git a/Command/FuzzTest.hs b/Command/FuzzTest.hs new file mode 100644 index 0000000000..2e5c7536e6 --- /dev/null +++ b/Command/FuzzTest.hs @@ -0,0 +1,281 @@ +{- git-annex fuzz generator + - + - Copyright 2013 Joey Hess + - + - Licensed under the GNU GPL version 3 or higher. + -} + +module Command.FuzzTest where + +import Command +import qualified Annex +import qualified Git.Config +import Config +import Utility.ThreadScheduler +import Utility.DiskFree + +import Data.Time.Clock +import System.Random (getStdRandom, random, randomR) +import Test.QuickCheck +import Control.Concurrent + +cmd :: Command +cmd = notBareRepo $ + command "fuzztest" SectionTesting + "generates fuzz test files" + paramNothing (withParams seek) + +seek :: CmdParams -> CommandSeek +seek = withNothing (commandAction start) + +start :: CommandStart +start = do + guardTest + logf <- fromRepo gitAnnexFuzzTestLogFile + showStart "fuzztest" logf + logh <- liftIO $ openFile logf WriteMode + void $ forever $ fuzz logh + stop + +guardTest :: Annex () +guardTest = unlessM (fromMaybe False . Git.Config.isTrue <$> getConfig key "") $ + giveup $ unlines + [ "Running fuzz tests *writes* to and *deletes* files in" + , "this repository, and pushes those changes to other" + , "repositories! This is a developer tool, not something" + , "to play with." + , "" + , "Refusing to run fuzz tests, since " ++ keyname ++ " is not set!" + ] + where + key = annexConfig "eat-my-repository" + (ConfigKey keyname) = key + + +fuzz :: Handle -> Annex () +fuzz logh = do + fuzzer <- genFuzzAction + record logh $ flip Started fuzzer + result <- tryNonAsync $ runFuzzAction fuzzer + record logh $ flip Finished $ + either (const False) (const True) result + +record :: Handle -> (UTCTime -> TimeStampedFuzzAction) -> Annex () +record h tmpl = liftIO $ do + now <- getCurrentTime + let s = show $ tmpl now + print s + hPrint h s + hFlush h + +{- Delay for either a fraction of a second, or a few seconds, or up + - to 1 minute. + - + - The MinutesDelay is used as an opportunity to do housekeeping tasks. + -} +randomDelay :: Delay -> Annex () +randomDelay TinyDelay = liftIO $ + threadDelay =<< getStdRandom (randomR (10000, 1000000)) +randomDelay SecondsDelay = liftIO $ + threadDelaySeconds =<< Seconds <$> getStdRandom (randomR (1, 10)) +randomDelay MinutesDelay = do + liftIO $ threadDelaySeconds =<< Seconds <$> getStdRandom (randomR (1, 60)) + reserve <- annexDiskReserve <$> Annex.getGitConfig + free <- liftIO $ getDiskFree "." + case free of + Just have | have < reserve -> do + warning "Low disk space; fuzz test paused." + liftIO $ threadDelaySeconds (Seconds 60) + randomDelay MinutesDelay + _ -> noop + +data Delay + = TinyDelay + | SecondsDelay + | MinutesDelay + deriving (Read, Show, Eq) + +instance Arbitrary Delay where + arbitrary = elements [TinyDelay, SecondsDelay, MinutesDelay] + +data FuzzFile = FuzzFile FilePath + deriving (Read, Show, Eq) + +data FuzzDir = FuzzDir FilePath + deriving (Read, Show, Eq) + +instance Arbitrary FuzzFile where + arbitrary = FuzzFile <$> arbitrary + +instance Arbitrary FuzzDir where + arbitrary = FuzzDir <$> arbitrary + +class ToFilePath a where + toFilePath :: a -> FilePath + +instance ToFilePath FuzzFile where + toFilePath (FuzzFile f) = f + +instance ToFilePath FuzzDir where + toFilePath (FuzzDir d) = d + +isFuzzFile :: FilePath -> Bool +isFuzzFile f = "fuzzfile_" `isPrefixOf` takeFileName f + +isFuzzDir :: FilePath -> Bool +isFuzzDir d = "fuzzdir_" `isPrefixOf` d + +mkFuzzFile :: FilePath -> [FuzzDir] -> FuzzFile +mkFuzzFile file dirs = FuzzFile $ joinPath (map toFilePath dirs) ("fuzzfile_" ++ file) + +mkFuzzDir :: Int -> FuzzDir +mkFuzzDir n = FuzzDir $ "fuzzdir_" ++ show n + +{- File is placed inside a directory hierarchy up to 4 subdirectories deep. -} +genFuzzFile :: IO FuzzFile +genFuzzFile = do + n <- getStdRandom $ randomR (0, 4) + dirs <- replicateM n genFuzzDir + file <- show <$> (getStdRandom random :: IO Int) + return $ mkFuzzFile file dirs + +{- Only 16 distinct subdirectories are used. When nested 4 deep, this + - yields 69904 total directories max, which is below the default Linux + - inotify limit of 81920. The goal is not to run the assistant out of + - inotify descriptors. -} +genFuzzDir :: IO FuzzDir +genFuzzDir = mkFuzzDir <$> (getStdRandom (randomR (1,16)) :: IO Int) + +data TimeStampedFuzzAction + = Started UTCTime FuzzAction + | Finished UTCTime Bool + deriving (Read, Show) + +data FuzzAction + = FuzzAdd FuzzFile + | FuzzDelete FuzzFile + | FuzzMove FuzzFile FuzzFile + | FuzzModify FuzzFile + | FuzzDeleteDir FuzzDir + | FuzzMoveDir FuzzDir FuzzDir + | FuzzPause Delay + deriving (Read, Show, Eq) + +instance Arbitrary FuzzAction where + arbitrary = frequency + [ (50, FuzzAdd <$> arbitrary) + , (50, FuzzDelete <$> arbitrary) + , (10, FuzzMove <$> arbitrary <*> arbitrary) + , (10, FuzzModify <$> arbitrary) + , (10, FuzzDeleteDir <$> arbitrary) + , (10, FuzzMoveDir <$> arbitrary <*> arbitrary) + , (10, FuzzPause <$> arbitrary) + ] + +runFuzzAction :: FuzzAction -> Annex () +runFuzzAction (FuzzAdd (FuzzFile f)) = liftIO $ do + createDirectoryIfMissing True $ parentDir f + n <- getStdRandom random :: IO Int + writeFile f $ show n ++ "\n" +runFuzzAction (FuzzDelete (FuzzFile f)) = liftIO $ nukeFile f +runFuzzAction (FuzzMove (FuzzFile src) (FuzzFile dest)) = liftIO $ + rename src dest +runFuzzAction (FuzzModify (FuzzFile f)) = whenM isDirect $ liftIO $ do + n <- getStdRandom random :: IO Int + appendFile f $ show n ++ "\n" +runFuzzAction (FuzzDeleteDir (FuzzDir d)) = liftIO $ + removeDirectoryRecursive d +runFuzzAction (FuzzMoveDir (FuzzDir src) (FuzzDir dest)) = liftIO $ + rename src dest +runFuzzAction (FuzzPause d) = randomDelay d + +genFuzzAction :: Annex FuzzAction +genFuzzAction = do + tmpl <- liftIO $ Prelude.head <$> sample' (arbitrary :: Gen FuzzAction) + -- Fix up template action to make sense in the current repo tree. + case tmpl of + FuzzAdd _ -> do + f <- liftIO newFile + maybe genFuzzAction (return . FuzzAdd) f + FuzzDelete _ -> do + f <- liftIO $ existingFile 0 "" + maybe genFuzzAction (return . FuzzDelete) f + FuzzMove _ _ -> do + src <- liftIO $ existingFile 0 "" + dest <- liftIO newFile + case (src, dest) of + (Just s, Just d) -> return $ FuzzMove s d + _ -> genFuzzAction + FuzzMoveDir _ _ -> do + md <- liftIO existingDir + case md of + Nothing -> genFuzzAction + Just d -> do + newd <- liftIO $ newDir (parentDir $ toFilePath d) + maybe genFuzzAction (return . FuzzMoveDir d) newd + FuzzDeleteDir _ -> do + d <- liftIO existingDir + maybe genFuzzAction (return . FuzzDeleteDir) d + FuzzModify _ -> do + f <- liftIO $ existingFile 0 "" + maybe genFuzzAction (return . FuzzModify) f + FuzzPause _ -> return tmpl + +existingFile :: Int -> FilePath -> IO (Maybe FuzzFile) +existingFile 0 _ = return Nothing +existingFile n top = do + dir <- existingDirIncludingTop + contents <- catchDefaultIO [] (getDirectoryContents dir) + let files = filter isFuzzFile contents + if null files + then do + let dirs = filter isFuzzDir contents + if null dirs + then return Nothing + else do + i <- getStdRandom $ randomR (0, length dirs - 1) + existingFile (n - 1) (top dirs !! i) + else do + i <- getStdRandom $ randomR (0, length files - 1) + return $ Just $ FuzzFile $ top dir files !! i + +existingDirIncludingTop :: IO FilePath +existingDirIncludingTop = do + dirs <- filter isFuzzDir <$> getDirectoryContents "." + if null dirs + then return "." + else do + n <- getStdRandom $ randomR (0, length dirs) + return $ ("." : dirs) !! n + +existingDir :: IO (Maybe FuzzDir) +existingDir = do + d <- existingDirIncludingTop + return $ if isFuzzDir d + then Just $ FuzzDir d + else Nothing + +newFile :: IO (Maybe FuzzFile) +newFile = go (100 :: Int) + where + go 0 = return Nothing + go n = do + f <- genFuzzFile + ifM (doesnotexist (toFilePath f)) + ( return $ Just f + , go (n - 1) + ) + +newDir :: FilePath -> IO (Maybe FuzzDir) +newDir parent = go (100 :: Int) + where + go 0 = return Nothing + go n = do + (FuzzDir d) <- genFuzzDir + ifM (doesnotexist (parent d)) + ( return $ Just $ FuzzDir d + , go (n - 1) + ) + +doesnotexist :: FilePath -> IO Bool +doesnotexist f = isNothing <$> catchMaybeIO (getSymbolicLinkStatus f) diff --git a/Command/GCryptSetup.hs b/Command/GCryptSetup.hs new file mode 100644 index 0000000000..83588f1a9f --- /dev/null +++ b/Command/GCryptSetup.hs @@ -0,0 +1,39 @@ +{- git-annex command + - + - Copyright 2013 Joey Hess + - + - Licensed under the GNU GPL version 3 or higher. + -} + +module Command.GCryptSetup where + +import Command +import Annex.UUID +import qualified Remote.GCrypt +import qualified Git + +cmd :: Command +cmd = dontCheck repoExists $ noCommit $ + command "gcryptsetup" SectionPlumbing + "sets up gcrypt repository" + paramValue (withParams seek) + +seek :: CmdParams -> CommandSeek +seek = withStrings (commandAction . start) + +start :: String -> CommandStart +start gcryptid = next $ next $ do + u <- getUUID + when (u /= NoUUID) $ + giveup "gcryptsetup refusing to run; this repository already has a git-annex uuid!" + + g <- gitRepo + gu <- Remote.GCrypt.getGCryptUUID True g + let newgu = genUUIDInNameSpace gCryptNameSpace gcryptid + if isNothing gu || gu == Just newgu + then if Git.repoIsLocalBare g + then do + void $ Remote.GCrypt.setupRepo gcryptid g + return True + else giveup "cannot use gcrypt in a non-bare repository" + else giveup "gcryptsetup uuid mismatch" diff --git a/Command/Get.hs b/Command/Get.hs new file mode 100644 index 0000000000..d7752101e0 --- /dev/null +++ b/Command/Get.hs @@ -0,0 +1,117 @@ +{- git-annex command + - + - Copyright 2010, 2013 Joey Hess + - + - Licensed under the GNU GPL version 3 or higher. + -} + +module Command.Get where + +import Command +import qualified Remote +import Annex.Content +import Annex.Transfer +import Annex.NumCopies +import Annex.Wanted +import qualified Command.Move + +cmd :: Command +cmd = withGlobalOptions [jobsOption, jsonOptions, jsonProgressOption, annexedMatchingOptions] $ + command "get" SectionCommon + "make content of annexed files available" + paramPaths (seek <$$> optParser) + +data GetOptions = GetOptions + { getFiles :: CmdParams + , getFrom :: Maybe (DeferredParse Remote) + , autoMode :: Bool + , keyOptions :: Maybe KeyOptions + , batchOption :: BatchMode + } + +optParser :: CmdParamsDesc -> Parser GetOptions +optParser desc = GetOptions + <$> cmdParams desc + <*> optional (parseRemoteOption <$> parseFromOption) + <*> parseAutoOption + <*> optional (parseIncompleteOption <|> parseKeyOptions <|> parseFailedTransfersOption) + <*> parseBatchOption + +seek :: GetOptions -> CommandSeek +seek o = allowConcurrentOutput $ do + from <- maybe (pure Nothing) (Just <$$> getParsed) (getFrom o) + let go = whenAnnexed $ start o from + case batchOption o of + Batch fmt -> batchFilesMatching fmt go + NoBatch -> withKeyOptions (keyOptions o) (autoMode o) + (commandAction . startKeys from) + (withFilesInGit (commandAction . go)) + =<< workTreeItems (getFiles o) + +start :: GetOptions -> Maybe Remote -> FilePath -> Key -> CommandStart +start o from file key = start' expensivecheck from key afile (mkActionItem afile) + where + afile = AssociatedFile (Just file) + expensivecheck + | autoMode o = numCopiesCheck file key (<) + <||> wantGet False (Just key) afile + | otherwise = return True + +startKeys :: Maybe Remote -> (Key, ActionItem) -> CommandStart +startKeys from (key, ai) = checkFailedTransferDirection ai Download $ + start' (return True) from key (AssociatedFile Nothing) ai + +start' :: Annex Bool -> Maybe Remote -> Key -> AssociatedFile -> ActionItem -> CommandStart +start' expensivecheck from key afile ai = onlyActionOn key $ + stopUnless (not <$> inAnnex key) $ stopUnless expensivecheck $ + case from of + Nothing -> go $ perform key afile + Just src -> + stopUnless (Command.Move.fromOk src key) $ + go $ Command.Move.fromPerform src Command.Move.RemoveNever key afile + where + go a = do + showStartKey "get" key ai + next a + +perform :: Key -> AssociatedFile -> CommandPerform +perform key afile = stopUnless (getKey key afile) $ + next $ return True -- no cleanup needed + +{- Try to find a copy of the file in one of the remotes, + - and copy it to here. -} +getKey :: Key -> AssociatedFile -> Annex Bool +getKey key afile = getKey' key afile =<< Remote.keyPossibilities key + +getKey' :: Key -> AssociatedFile -> [Remote] -> Annex Bool +getKey' key afile = dispatch + where + dispatch [] = do + showNote "not available" + showlocs + return False + dispatch remotes = notifyTransfer Download afile $ \witness -> do + ok <- pickRemote remotes $ \r -> ifM (probablyPresent r) + ( docopy r witness + , return False + ) + if ok + then return ok + else do + Remote.showTriedRemotes remotes + showlocs + return False + showlocs = Remote.showLocations False key [] + "No other repository is known to contain the file." + -- This check is to avoid an ugly message if a remote is a + -- drive that is not mounted. + probablyPresent r + | Remote.hasKeyCheap r = + either (const False) id <$> Remote.hasKey r key + | otherwise = return True + docopy r witness = getViaTmp (Remote.retrievalSecurityPolicy r) (RemoteVerify r) key $ \dest -> + download (Remote.uuid r) key afile stdRetry + (\p -> do + showAction $ "from " ++ Remote.name r + Remote.retrieveKeyFile r key afile dest p + ) witness diff --git a/Command/Group.hs b/Command/Group.hs new file mode 100644 index 0000000000..98de81f0fd --- /dev/null +++ b/Command/Group.hs @@ -0,0 +1,39 @@ +{- git-annex command + - + - Copyright 2012 Joey Hess + - + - Licensed under the GNU GPL version 3 or higher. + -} + +module Command.Group where + +import Command +import qualified Remote +import Logs.Group +import Types.Group + +import qualified Data.Set as S + +cmd :: Command +cmd = noMessages $ command "group" SectionSetup "add a repository to a group" + (paramPair paramRemote paramDesc) (withParams seek) + +seek :: CmdParams -> CommandSeek +seek = withWords (commandAction . start) + +start :: [String] -> CommandStart +start (name:g:[]) = do + allowMessages + showStart' "group" (Just name) + u <- Remote.nameToUUID name + next $ setGroup u g +start (name:[]) = do + u <- Remote.nameToUUID name + liftIO . putStrLn . unwords . S.toList =<< lookupGroups u + stop +start _ = giveup "Specify a repository and a group." + +setGroup :: UUID -> Group -> CommandPerform +setGroup uuid g = do + groupChange uuid (S.insert g) + next $ return True diff --git a/Command/GroupWanted.hs b/Command/GroupWanted.hs new file mode 100644 index 0000000000..57d5781732 --- /dev/null +++ b/Command/GroupWanted.hs @@ -0,0 +1,29 @@ +{- git-annex command + - + - Copyright 2015 Joey Hess + - + - Licensed under the GNU GPL version 3 or higher. + -} + +module Command.GroupWanted where + +import Command +import Logs.PreferredContent +import Command.Wanted (performGet, performSet) + +cmd :: Command +cmd = noMessages $ command "groupwanted" SectionSetup + "get or set groupwanted expression" + (paramPair paramGroup (paramOptional paramExpression)) + (withParams seek) + +seek :: CmdParams -> CommandSeek +seek = withWords (commandAction . start) + +start :: [String] -> CommandStart +start (g:[]) = next $ performGet groupPreferredContentMapRaw g +start (g:expr:[]) = do + allowMessages + showStart' "groupwanted" (Just g) + next $ performSet groupPreferredContentSet expr g +start _ = giveup "Specify a group." diff --git a/Command/Help.hs b/Command/Help.hs new file mode 100644 index 0000000000..227ed1f933 --- /dev/null +++ b/Command/Help.hs @@ -0,0 +1,71 @@ +{- git-annex command + - + - Copyright 2012 Joey Hess + - + - Licensed under the GNU GPL version 3 or higher. + -} + +module Command.Help where + +import Command +import qualified Command.Init +import qualified Command.Add +import qualified Command.Drop +import qualified Command.Get +import qualified Command.Move +import qualified Command.Copy +import qualified Command.Sync +import qualified Command.Whereis +import qualified Command.Fsck + +cmd :: Command +cmd = noCommit $ dontCheck repoExists $ + noRepo (parseparams startNoRepo) $ + command "help" SectionCommon "display help" + "COMMAND" (parseparams seek) + where + parseparams = withParams + +seek :: CmdParams -> CommandSeek +seek = withWords (commandAction . start) + +start :: [String] -> CommandStart +start params = do + liftIO $ start' params + stop + +startNoRepo :: CmdParams -> IO () +startNoRepo = start' + +start' :: [String] -> IO () +start' [c] = showGitHelp c +start' _ = showGeneralHelp + +showGeneralHelp :: IO () +showGeneralHelp = putStrLn $ unlines + [ "The most frequently used git-annex commands are:" + , unlines $ map cmdline $ + [ Command.Init.cmd + , Command.Add.cmd + , Command.Drop.cmd + , Command.Get.cmd + , Command.Move.cmd + , Command.Copy.cmd + , Command.Sync.cmd + , Command.Whereis.cmd + , Command.Fsck.cmd + ] + , "For a complete command list, run: git-annex" + , "For help on a specific command, run: git-annex help COMMAND" + ] + where + cmdline c = "\t" ++ cmdname c ++ "\t" ++ cmddesc c + +showGitHelp :: String -> IO () +showGitHelp c = + unlessM (githelp) $ + putStrLn $ "View online help at " ++ url + where + githelp = boolSystem "git" [Param "help", Param fullc] + fullc = "git-annex-" ++ c + url = "https://git-annex.branchable.com/" ++ fullc ++ "/" diff --git a/Command/Import.hs b/Command/Import.hs new file mode 100644 index 0000000000..95b5bd1a13 --- /dev/null +++ b/Command/Import.hs @@ -0,0 +1,211 @@ +{- git-annex command + - + - Copyright 2012-2017 Joey Hess + - + - Licensed under the GNU GPL version 3 or higher. + -} + +module Command.Import where + +import Command +import qualified Git +import qualified Annex +import qualified Command.Add +import qualified Command.Reinject +import Utility.CopyFile +import Backend +import Types.KeySource +import Annex.CheckIgnore +import Annex.NumCopies +import Annex.FileMatcher +import Annex.Ingest +import Annex.InodeSentinal +import Utility.InodeCache +import Logs.Location + +cmd :: Command +cmd = notBareRepo $ + withGlobalOptions [jobsOption, jsonOptions, fileMatchingOptions] $ + command "import" SectionCommon + "move and add files from outside git working copy" + paramPaths (seek <$$> optParser) + +data DuplicateMode = Default | Duplicate | DeDuplicate | CleanDuplicates | SkipDuplicates | ReinjectDuplicates + deriving (Eq) + +data ImportOptions = ImportOptions + { importFiles :: CmdParams + , duplicateMode :: DuplicateMode + } + +optParser :: CmdParamsDesc -> Parser ImportOptions +optParser desc = ImportOptions + <$> cmdParams desc + <*> (fromMaybe Default <$> optional duplicateModeParser) + +duplicateModeParser :: Parser DuplicateMode +duplicateModeParser = + flag' Duplicate + ( long "duplicate" + <> help "do not delete source files" + ) + <|> flag' DeDuplicate + ( long "deduplicate" + <> help "delete source files whose content was imported before" + ) + <|> flag' CleanDuplicates + ( long "clean-duplicates" + <> help "delete duplicate source files (import nothing)" + ) + <|> flag' SkipDuplicates + ( long "skip-duplicates" + <> help "import only new files (do not delete source files)" + ) + <|> flag' ReinjectDuplicates + ( long "reinject-duplicates" + <> help "import new files, and reinject the content of files that were imported before" + ) + +seek :: ImportOptions -> CommandSeek +seek o = allowConcurrentOutput $ do + repopath <- liftIO . absPath =<< fromRepo Git.repoPath + inrepops <- liftIO $ filter (dirContains repopath) <$> mapM absPath (importFiles o) + unless (null inrepops) $ do + giveup $ "cannot import files from inside the working tree (use git annex add instead): " ++ unwords inrepops + largematcher <- largeFilesMatcher + (commandAction . start largematcher (duplicateMode o)) + `withPathContents` importFiles o + +start :: GetFileMatcher -> DuplicateMode -> (FilePath, FilePath) -> CommandStart +start largematcher mode (srcfile, destfile) = + ifM (liftIO $ isRegularFile <$> getSymbolicLinkStatus srcfile) + ( do + showStart "import" destfile + next pickaction + , stop + ) + where + deletedup k = do + showNote $ "duplicate of " ++ key2file k + verifyExisting k destfile + ( do + liftIO $ removeFile srcfile + next $ return True + , do + warning "Could not verify that the content is still present in the annex; not removing from the import location." + stop + ) + reinject k = do + showNote "reinjecting" + Command.Reinject.perform srcfile k + importfile ld k = checkdestdir $ do + ignored <- not <$> Annex.getState Annex.force <&&> checkIgnored destfile + if ignored + then do + warning $ "not importing " ++ destfile ++ " which is .gitignored (use --force to override)" + stop + else do + existing <- liftIO (catchMaybeIO $ getSymbolicLinkStatus destfile) + case existing of + Nothing -> importfilechecked ld k + Just s + | isDirectory s -> notoverwriting "(is a directory)" + | isSymbolicLink s -> notoverwriting "(is a symlink)" + | otherwise -> ifM (Annex.getState Annex.force) + ( do + liftIO $ nukeFile destfile + importfilechecked ld k + , notoverwriting "(use --force to override, or a duplication option such as --deduplicate to clean up)" + ) + checkdestdir cont = do + let destdir = parentDir destfile + existing <- liftIO (catchMaybeIO $ getSymbolicLinkStatus destdir) + case existing of + Nothing -> cont + Just s + | isDirectory s -> cont + | otherwise -> do + warning $ "not importing " ++ destfile ++ " because " ++ destdir ++ " is not a directory" + stop + + importfilechecked ld k = do + -- Move or copy the src file to the dest file. + -- The dest file is what will be ingested. + liftIO $ createDirectoryIfMissing True (parentDir destfile) + liftIO $ if mode == Duplicate || mode == SkipDuplicates + then void $ copyFileExternal CopyAllMetaData srcfile destfile + else moveFile srcfile destfile + -- Get the inode cache of the dest file. It should be + -- weakly the same as the origianlly locked down file's + -- inode cache. (Since the file may have been copied, + -- its inodes may not be the same.) + newcache <- withTSDelta $ liftIO . genInodeCache destfile + let unchanged = case (newcache, inodeCache (keySource ld)) of + (_, Nothing) -> True + (Just newc, Just c) | compareWeak c newc -> True + _ -> False + unless unchanged $ + giveup "changed while it was being added" + -- The LockedDown needs to be adjusted, since the destfile + -- is what will be ingested. + let ld' = ld + { keySource = KeySource + { keyFilename = destfile + , contentLocation = destfile + , inodeCache = newcache + } + } + ifM (checkFileMatcher largematcher destfile) + ( ingestAdd' (Just ld') (Just k) + >>= maybe + stop + (\addedk -> next $ Command.Add.cleanup addedk True) + , next $ Command.Add.addSmall destfile + ) + notoverwriting why = do + warning $ "not overwriting existing " ++ destfile ++ " " ++ why + stop + lockdown a = do + lockingfile <- not <$> addUnlocked + -- Minimal lock down with no hard linking so nothing + -- has to be done to clean up from it. + let cfg = LockDownConfig + { lockingFile = lockingfile + , hardlinkFileTmp = False + } + v <- lockDown cfg srcfile + case v of + Just ld -> do + backend <- chooseBackend destfile + v' <- genKey (keySource ld) backend + case v' of + Just (k, _) -> a (ld, k) + Nothing -> giveup "failed to generate a key" + Nothing -> stop + checkdup k dupa notdupa = ifM (isKnownKey k) + ( dupa + , notdupa + ) + pickaction = lockdown $ \(ld, k) -> case mode of + DeDuplicate -> checkdup k (deletedup k) (importfile ld k) + CleanDuplicates -> checkdup k + (deletedup k) + (skipbecause "not duplicate") + SkipDuplicates -> checkdup k + (skipbecause "duplicate") + (importfile ld k) + ReinjectDuplicates -> checkdup k + (reinject k) + (importfile ld k) + _ -> importfile ld k + skipbecause s = showNote (s ++ "; skipping") >> next (return True) + +verifyExisting :: Key -> FilePath -> (CommandPerform, CommandPerform) -> CommandPerform +verifyExisting key destfile (yes, no) = do + -- Look up the numcopies setting for the file that it would be + -- imported to, if it were imported. + need <- getFileNumCopies destfile + + (tocheck, preverified) <- verifiableCopies key [] + verifyEnoughCopiesToDrop [] key Nothing need [] preverified tocheck + (const yes) no diff --git a/Command/ImportFeed.hs b/Command/ImportFeed.hs new file mode 100644 index 0000000000..ef13aa09f4 --- /dev/null +++ b/Command/ImportFeed.hs @@ -0,0 +1,412 @@ +{- git-annex command + - + - Copyright 2013-2017 Joey Hess + - + - Licensed under the GNU GPL version 3 or higher. + -} + +{-# LANGUAGE CPP #-} + +module Command.ImportFeed where + +import Text.Feed.Import +import Text.Feed.Query +import Text.Feed.Types +import qualified Data.Set as S +import qualified Data.Map as M +import Data.Time.Clock +import Data.Time.Format +#if ! MIN_VERSION_time(1,5,0) +import System.Locale +#endif +#if MIN_VERSION_feed(1,0,0) +import qualified Data.Text as T +#endif + +import Command +import qualified Annex +import qualified Annex.Url as Url +import qualified Remote +import qualified Types.Remote as Remote +import Types.UrlContents +import Logs.Web +import Logs.File +import qualified Utility.Format +import Utility.Tmp +import Utility.Metered +import Command.AddUrl (addUrlFile, downloadRemoteFile, parseDownloadOptions, DownloadOptions(..)) +import Annex.UUID +import Backend.URL (fromUrl) +import Annex.Content +import Annex.YoutubeDl +import Types.MetaData +import Logs.MetaData +import Annex.MetaData +import Command.AddUrl (addWorkTree) + +cmd :: Command +cmd = notBareRepo $ + command "importfeed" SectionCommon "import files from podcast feeds" + (paramRepeating paramUrl) (seek <$$> optParser) + +data ImportFeedOptions = ImportFeedOptions + { feedUrls :: CmdParams + , templateOption :: Maybe String + , downloadOptions :: DownloadOptions + } + +optParser :: CmdParamsDesc -> Parser ImportFeedOptions +optParser desc = ImportFeedOptions + <$> cmdParams desc + <*> optional (strOption + ( long "template" <> metavar paramFormat + <> help "template for filenames" + )) + <*> parseDownloadOptions False + +seek :: ImportFeedOptions -> CommandSeek +seek o = do + cache <- getCache (templateOption o) + withStrings (commandAction . start o cache) (feedUrls o) + +start :: ImportFeedOptions -> Cache -> URLString -> CommandStart +start opts cache url = do + showStart' "importfeed" (Just url) + next $ perform opts cache url + +perform :: ImportFeedOptions -> Cache -> URLString -> CommandPerform +perform opts cache url = do + v <- findDownloads url + case v of + [] -> do + feedProblem url "bad feed content; no enclosures to download" + next $ return True + l -> do + showOutput + ok <- and <$> mapM (performDownload opts cache) l + unless ok $ + feedProblem url "problem downloading some item(s) from feed" + next $ cleanup url True + +cleanup :: URLString -> Bool -> CommandCleanup +cleanup url ok = do + when ok $ + clearFeedProblem url + return ok + +data ToDownload = ToDownload + { feed :: Feed + , feedurl :: URLString + , item :: Item + , location :: DownloadLocation + } + +data DownloadLocation = Enclosure URLString | MediaLink URLString + +type ItemId = String + +data Cache = Cache + { knownurls :: S.Set URLString + , knownitems :: S.Set ItemId + , template :: Utility.Format.Format + } + +getCache :: Maybe String -> Annex Cache +getCache opttemplate = ifM (Annex.getState Annex.force) + ( ret S.empty S.empty + , do + showStart "importfeed" "checking known urls" + (is, us) <- unzip <$> (mapM knownItems =<< knownUrls) + showEndOk + ret (S.fromList us) (S.fromList (concat is)) + ) + where + tmpl = Utility.Format.gen $ fromMaybe defaultTemplate opttemplate + ret us is = return $ Cache us is tmpl + +knownItems :: (Key, URLString) -> Annex ([ItemId], URLString) +knownItems (k, u) = do + itemids <- S.toList . S.filter (/= noneValue) . S.map fromMetaValue + . currentMetaDataValues itemIdField + <$> getCurrentMetaData k + return (itemids, u) + +findDownloads :: URLString -> Annex [ToDownload] +findDownloads u = go =<< downloadFeed u + where + go Nothing = pure [] + go (Just f) = catMaybes <$> mapM (mk f) (feedItems f) + + mk f i = case getItemEnclosure i of + Just (enclosureurl, _, _) -> return $ + Just $ ToDownload f u i $ Enclosure $ + fromFeed enclosureurl + Nothing -> case getItemLink i of + Just link -> return $ Just $ ToDownload f u i $ + MediaLink $ fromFeed link + Nothing -> return Nothing + +{- Feeds change, so a feed download cannot be resumed. -} +downloadFeed :: URLString -> Annex (Maybe Feed) +downloadFeed url + | Url.parseURIRelaxed url == Nothing = giveup "invalid feed url" + | otherwise = Url.withUrlOptions $ \uo -> + liftIO $ withTmpFile "feed" $ \f h -> do + hClose h + ifM (Url.download nullMeterUpdate url f uo) + ( parseFeedString <$> readFileStrict f + , return Nothing + ) + +performDownload :: ImportFeedOptions -> Cache -> ToDownload -> Annex Bool +performDownload opts cache todownload = case location todownload of + Enclosure url -> checkknown url $ + rundownload url (takeWhile (/= '?') $ takeExtension url) $ \f -> do + r <- Remote.claimingUrl url + if Remote.uuid r == webUUID || rawOption (downloadOptions opts) + then do + urlinfo <- if relaxedOption (downloadOptions opts) + then pure Url.assumeUrlExists + else Url.withUrlOptions $ + liftIO . Url.getUrlInfo url + let dlopts = (downloadOptions opts) + -- force using the filename + -- chosen here + { fileOption = Just f + -- don't use youtube-dl + , rawOption = True + } + maybeToList <$> addUrlFile dlopts url urlinfo f + else do + res <- tryNonAsync $ maybe + (error $ "unable to checkUrl of " ++ Remote.name r) + (flip id url) + (Remote.checkUrl r) + case res of + Left _ -> return [] + Right (UrlContents sz _) -> + maybeToList <$> + downloadRemoteFile r (downloadOptions opts) url f sz + Right (UrlMulti l) -> do + kl <- forM l $ \(url', sz, subf) -> + downloadRemoteFile r (downloadOptions opts) url' (f fromSafeFilePath subf) sz + return $ if all isJust kl + then catMaybes kl + else [] + + MediaLink linkurl -> do + let mediaurl = setDownloader linkurl YoutubeDownloader + let mediakey = Backend.URL.fromUrl mediaurl Nothing + -- Old versions of git-annex that used quvi might have + -- used the quviurl for this, so check if it's known + -- to avoid adding it a second time. + let quviurl = setDownloader linkurl QuviDownloader + checkknown mediaurl $ checkknown quviurl $ + ifM (Annex.getState Annex.fast <||> pure (relaxedOption (downloadOptions opts))) + ( addmediafast linkurl mediaurl mediakey + , downloadmedia linkurl mediaurl mediakey + ) + where + forced = Annex.getState Annex.force + + {- Avoids downloading any items that are already known to be + - associated with a file in the annex, unless forced. -} + checkknown url a + | knownitemid || S.member url (knownurls cache) + = ifM forced (a, return True) + | otherwise = a + + knownitemid = case getItemId (item todownload) of + Just (_, itemid) -> + S.member (fromFeed itemid) (knownitems cache) + _ -> False + + rundownload url extension getter = do + dest <- makeunique url (1 :: Integer) $ + feedFile (template cache) todownload extension + case dest of + Nothing -> return True + Just f -> do + showStart "addurl" url + ks <- getter f + if null ks + then do + showEndFail + checkFeedBroken (feedurl todownload) + else do + forM_ ks $ \key -> + ifM (annexGenMetaData <$> Annex.getGitConfig) + ( addMetaData key $ extractMetaData todownload + , addMetaData key $ minimalMetaData todownload + ) + showEndOk + return True + + {- Find a unique filename to save the url to. + - If the file exists, prefixes it with a number. + - When forced, the file may already exist and have the same + - url, in which case Nothing is returned as it does not need + - to be re-downloaded. -} + makeunique url n file = ifM alreadyexists + ( ifM forced + ( ifAnnexed f checksameurl tryanother + , tryanother + ) + , return $ Just f + ) + where + f = if n < 2 + then file + else + let (d, base) = splitFileName file + in d show n ++ "_" ++ base + tryanother = makeunique url (n + 1) file + alreadyexists = liftIO $ isJust <$> catchMaybeIO (getSymbolicLinkStatus f) + checksameurl k = ifM (elem url <$> getUrls k) + ( return Nothing + , tryanother + ) + + downloadmedia linkurl mediaurl mediakey + | rawOption (downloadOptions opts) = downloadlink + | otherwise = do + r <- withTmpWorkDir mediakey $ \workdir -> do + dl <- youtubeDl linkurl workdir + case dl of + Right (Just mediafile) -> do + let ext = case takeExtension mediafile of + [] -> ".m" + s -> s + ok <- rundownload linkurl ext $ \f -> do + addWorkTree webUUID mediaurl f mediakey (Just mediafile) + return [mediakey] + return (Just ok) + -- youtude-dl didn't support it, so + -- download it as if the link were + -- an enclosure. + Right Nothing -> Just <$> downloadlink + Left msg -> do + warning msg + return Nothing + return (fromMaybe False r) + where + downloadlink = performDownload opts cache todownload + { location = Enclosure linkurl } + + addmediafast linkurl mediaurl mediakey = + ifM (pure (not (rawOption (downloadOptions opts))) + <&&> youtubeDlSupported linkurl) + ( rundownload linkurl ".m" $ \f -> do + addWorkTree webUUID mediaurl f mediakey Nothing + return [mediakey] + , performDownload opts cache todownload + { location = Enclosure linkurl } + ) + +defaultTemplate :: String +defaultTemplate = "${feedtitle}/${itemtitle}${extension}" + +{- Generates a filename to use for a feed item by filling out the template. + - The filename may not be unique. -} +feedFile :: Utility.Format.Format -> ToDownload -> String -> FilePath +feedFile tmpl i extension = Utility.Format.format tmpl $ + M.map sanitizeFilePath $ M.fromList $ extractFields i ++ + [ ("extension", extension) + , extractField "itempubdate" [pubdate $ item i] + ] + where + pubdate itm = case getItemPublishDate itm :: Maybe (Maybe UTCTime) of + Just (Just d) -> Just $ + formatTime defaultTimeLocale "%F" d + -- if date cannot be parsed, use the raw string + _ -> replace "/" "-" . fromFeed + <$> getItemPublishDateString itm + +extractMetaData :: ToDownload -> MetaData +extractMetaData i = case getItemPublishDate (item i) :: Maybe (Maybe UTCTime) of + Just (Just d) -> unionMetaData meta (dateMetaData d meta) + _ -> meta + where + tometa (k, v) = (mkMetaFieldUnchecked k, S.singleton (toMetaValue v)) + meta = MetaData $ M.fromList $ map tometa $ extractFields i + +minimalMetaData :: ToDownload -> MetaData +minimalMetaData i = case getItemId (item i) of + (Nothing) -> emptyMetaData + (Just (_, itemid)) -> MetaData $ M.singleton itemIdField + (S.singleton $ toMetaValue $ fromFeed itemid) + +{- Extract fields from the feed and item, that are both used as metadata, + - and to generate the filename. -} +extractFields :: ToDownload -> [(String, String)] +extractFields i = map (uncurry extractField) + [ ("feedtitle", [feedtitle]) + , ("itemtitle", [itemtitle]) + , ("feedauthor", [feedauthor]) + , ("itemauthor", [itemauthor]) + , ("itemsummary", [fromFeed <$> getItemSummary (item i)]) + , ("itemdescription", [fromFeed <$> getItemDescription (item i)]) + , ("itemrights", [fromFeed <$> getItemRights (item i)]) + , ("itemid", [fromFeed . snd <$> getItemId (item i)]) + , ("title", [itemtitle, feedtitle]) + , ("author", [itemauthor, feedauthor]) + ] + where + feedtitle = Just $ fromFeed $ getFeedTitle $ feed i + itemtitle = fromFeed <$> getItemTitle (item i) + feedauthor = fromFeed <$> getFeedAuthor (feed i) + itemauthor = fromFeed <$> getItemAuthor (item i) + +itemIdField :: MetaField +itemIdField = mkMetaFieldUnchecked "itemid" + +extractField :: String -> [Maybe String] -> (String, String) +extractField k [] = (k, noneValue) +extractField k (Just v:_) + | not (null v) = (k, v) +extractField k (_:rest) = extractField k rest + +noneValue :: String +noneValue = "none" + +{- Called when there is a problem with a feed. + - Throws an error if the feed is broken, otherwise shows a warning. -} +feedProblem :: URLString -> String -> Annex () +feedProblem url message = ifM (checkFeedBroken url) + ( giveup $ message ++ " (having repeated problems with feed: " ++ url ++ ")" + , warning $ "warning: " ++ message + ) + +{- A feed is only broken if problems have occurred repeatedly, for at + - least 23 hours. -} +checkFeedBroken :: URLString -> Annex Bool +checkFeedBroken url = checkFeedBroken' url =<< feedState url +checkFeedBroken' :: URLString -> FilePath -> Annex Bool +checkFeedBroken' url f = do + prev <- maybe Nothing readish <$> liftIO (catchMaybeIO $ readFile f) + now <- liftIO getCurrentTime + case prev of + Nothing -> do + writeLogFile f $ show now + return False + Just prevtime -> do + let broken = diffUTCTime now prevtime > 60 * 60 * 23 + when broken $ + -- Avoid repeatedly complaining about + -- broken feed. + clearFeedProblem url + return broken + +clearFeedProblem :: URLString -> Annex () +clearFeedProblem url = void $ liftIO . tryIO . removeFile =<< feedState url + +feedState :: URLString -> Annex FilePath +feedState url = fromRepo $ gitAnnexFeedState $ fromUrl url Nothing + +#if MIN_VERSION_feed(1,0,0) +fromFeed :: T.Text -> String +fromFeed = T.unpack +#else +fromFeed :: String -> String +fromFeed = id +#endif diff --git a/Command/InAnnex.hs b/Command/InAnnex.hs new file mode 100644 index 0000000000..ad50dfa7c6 --- /dev/null +++ b/Command/InAnnex.hs @@ -0,0 +1,29 @@ +{- git-annex command + - + - Copyright 2010 Joey Hess + - + - Licensed under the GNU GPL version 3 or higher. + -} + +module Command.InAnnex where + +import Command +import Annex.Content + +cmd :: Command +cmd = noCommit $ + command "inannex" SectionPlumbing + "checks if keys are present in the annex" + (paramRepeating paramKey) + (withParams seek) + +seek :: CmdParams -> CommandSeek +seek = withKeys (commandAction . start) + +start :: Key -> CommandStart +start key = inAnnexSafe key >>= dispatch + where + dispatch (Just True) = stop + dispatch (Just False) = exit 1 + dispatch Nothing = exit 100 + exit n = liftIO $ exitWith $ ExitFailure n diff --git a/Command/Indirect.hs b/Command/Indirect.hs new file mode 100644 index 0000000000..9148e9d6f2 --- /dev/null +++ b/Command/Indirect.hs @@ -0,0 +1,105 @@ +{- git-annex command + - + - Copyright 2012 Joey Hess + - + - Licensed under the GNU GPL version 3 or higher. + -} + +module Command.Indirect where + +import Command +import qualified Git +import qualified Git.Branch +import qualified Git.LsFiles +import Git.FileMode +import Config +import qualified Annex +import Annex.Direct +import Annex.Content +import Annex.Content.Direct +import Annex.CatFile +import Annex.Init +import Annex.Ingest + +cmd :: Command +cmd = notBareRepo $ noDaemonRunning $ + command "indirect" SectionSetup "switch repository to indirect mode" + paramNothing (withParams seek) + +seek :: CmdParams -> CommandSeek +seek = withNothing (commandAction start) + +start :: CommandStart +start = ifM isDirect + ( do + unlessM (coreSymlinks <$> Annex.getGitConfig) $ + giveup "Git is configured to not use symlinks, so you must use direct mode." + whenM probeCrippledFileSystem $ + giveup "This repository seems to be on a crippled filesystem, you must use direct mode." + next perform + , stop + ) + +perform :: CommandPerform +perform = do + showStart' "commit" Nothing + whenM stageDirect $ do + showOutput + void $ inRepo $ Git.Branch.commitCommand Git.Branch.ManualCommit + [ Param "-m" + , Param "commit before switching to indirect mode" + ] + showEndOk + + -- Note that we set indirect mode early, so that we can use + -- moveAnnex in indirect mode. + setDirect False + + top <- fromRepo Git.repoPath + (l, clean) <- inRepo $ Git.LsFiles.stagedOthersDetails [top] + forM_ l go + void $ liftIO clean + next cleanup + where + {- Walk tree from top and move all present direct mode files into + - the annex, replacing with symlinks. Also delete direct mode + - caches and mappings. -} + go (f, Just sha, Just mode) | isSymLink mode = do + r <- liftIO $ catchMaybeIO $ getSymbolicLinkStatus f + case r of + Just s + | isSymbolicLink s -> void $ flip whenAnnexed f $ + \_ k -> do + removeInodeCache k + removeAssociatedFiles k + return Nothing + | otherwise -> + maybe noop (fromdirect f) + =<< catKey sha + _ -> noop + go _ = noop + + fromdirect f k = do + showStart "indirect" f + removeInodeCache k + removeAssociatedFiles k + whenM (liftIO $ not . isSymbolicLink <$> getSymbolicLinkStatus f) $ do + v <- tryNonAsync (moveAnnex k f) + case v of + Right True -> do + l <- calcRepo $ gitAnnexLink f k + liftIO $ createSymbolicLink l f + Right False -> warnlocked "Failed to move file to annex" + Left e -> catchNonAsync (restoreFile f k e) $ + warnlocked . show + showEndOk + + warnlocked msg = do + warning msg + warning "leaving this file as-is; correct this problem and run git annex add on it" + +cleanup :: CommandCleanup +cleanup = do + showStart' "indirect" Nothing + showEndOk + return True diff --git a/Command/Info.hs b/Command/Info.hs new file mode 100644 index 0000000000..2fcd602a61 --- /dev/null +++ b/Command/Info.hs @@ -0,0 +1,710 @@ +{- git-annex command + - + - Copyright 2011-2016 Joey Hess + - + - Licensed under the GNU GPL version 3 or higher. + -} + +{-# LANGUAGE BangPatterns, DeriveDataTypeable, CPP #-} + +module Command.Info where + +import "mtl" Control.Monad.State.Strict +import qualified Data.Map.Strict as M +import qualified Data.Vector as V +import Data.Ord +#if MIN_VERSION_base(4,9,0) +import qualified Data.Semigroup as Sem +#endif +import Prelude + +import Command +import qualified Git +import qualified Annex +import qualified Remote +import qualified Types.Remote as Remote +import Utility.DataUnits +import Utility.DiskFree +import Annex.Content +import Annex.UUID +import Annex.CatFile +import Logs.UUID +import Logs.Trust +import Logs.Location +import Annex.NumCopies +import Remote +import Config +import Git.Config (boolConfig) +import qualified Git.LsTree as LsTree +import Utility.Percentage +import Utility.Aeson hiding (json) +import Types.Transfer +import Logs.Transfer +import Types.Key +import Types.TrustLevel +import Types.FileMatcher +import Types.ActionItem +import qualified Limit +import Messages.JSON (DualDisp(..), ObjectMap(..)) +import Annex.BloomFilter +import qualified Command.Unused + +-- a named computation that produces a statistic +type Stat = StatState (Maybe (String, StatState String)) + +-- data about a set of keys +data KeyData = KeyData + { countKeys :: Integer + , sizeKeys :: Integer + , unknownSizeKeys :: Integer + , backendsKeys :: M.Map KeyVariety Integer + } + +appendKeyData :: KeyData -> KeyData -> KeyData +appendKeyData a b = KeyData + { countKeys = countKeys a + countKeys b + , sizeKeys = sizeKeys a + sizeKeys b + , unknownSizeKeys = unknownSizeKeys a + unknownSizeKeys b + , backendsKeys = backendsKeys a <> backendsKeys b + } + +#if MIN_VERSION_base(4,9,0) +instance Sem.Semigroup KeyData where + (<>) = appendKeyData +#endif + +instance Monoid KeyData where + mempty = KeyData 0 0 0 M.empty +#if MIN_VERSION_base(4,11,0) +#elif MIN_VERSION_base(4,9,0) + mappend = (Sem.<>) +#else + mappend = appendKeyData +#endif + +data NumCopiesStats = NumCopiesStats + { numCopiesVarianceMap :: M.Map Variance Integer + } + +newtype Variance = Variance Int + deriving (Eq, Ord) + +instance Show Variance where + show (Variance n) + | n >= 0 = "+" ++ show n + | otherwise = show n + +-- cached info that multiple Stats use +data StatInfo = StatInfo + { presentData :: Maybe KeyData + , referencedData :: Maybe KeyData + , repoData :: M.Map UUID KeyData + , numCopiesStats :: Maybe NumCopiesStats + , infoOptions :: InfoOptions + } + +emptyStatInfo :: InfoOptions -> StatInfo +emptyStatInfo = StatInfo Nothing Nothing M.empty Nothing + +-- a state monad for running Stats in +type StatState = StateT StatInfo Annex + +cmd :: Command +cmd = noCommit $ withGlobalOptions [jsonOptions, annexedMatchingOptions] $ + command "info" SectionQuery + "shows information about the specified item or the repository as a whole" + (paramRepeating paramItem) (seek <$$> optParser) + +data InfoOptions = InfoOptions + { infoFor :: CmdParams + , bytesOption :: Bool + , batchOption :: BatchMode + } + +optParser :: CmdParamsDesc -> Parser InfoOptions +optParser desc = InfoOptions + <$> cmdParams desc + <*> switch + ( long "bytes" + <> help "display file sizes in bytes" + ) + <*> parseBatchOption + +seek :: InfoOptions -> CommandSeek +seek o = case batchOption o of + NoBatch -> withWords (commandAction . start o) (infoFor o) + Batch fmt -> batchInput fmt Right (itemInfo o) + +start :: InfoOptions -> [String] -> CommandStart +start o [] = do + globalInfo o + stop +start o ps = do + mapM_ (itemInfo o) ps + stop + +globalInfo :: InfoOptions -> Annex () +globalInfo o = do + u <- getUUID + whenM ((==) DeadTrusted <$> lookupTrust u) $ + earlyWarning "Warning: This repository is currently marked as dead." + stats <- selStats global_fast_stats global_slow_stats + showCustom "info" $ do + evalStateT (mapM_ showStat stats) (emptyStatInfo o) + return True + +itemInfo :: InfoOptions -> String -> Annex () +itemInfo o p = ifM (isdir p) + ( dirInfo o p + , do + v <- Remote.byName' p + case v of + Right r -> remoteInfo o r + Left _ -> do + v' <- Remote.nameToUUID' p + case v' of + Right u -> uuidInfo o u + Left _ -> ifAnnexed p + (fileInfo o p) + (treeishInfo o p) + ) + where + isdir = liftIO . catchBoolIO . (isDirectory <$$> getFileStatus) + +noInfo :: String -> Annex () +noInfo s = do + showStart "info" s + showNote $ "not a directory or an annexed file or a treeish or a remote or a uuid" + showEndFail + +dirInfo :: InfoOptions -> FilePath -> Annex () +dirInfo o dir = showCustom (unwords ["info", dir]) $ do + stats <- selStats + (tostats (dir_name:tree_fast_stats True)) + (tostats tree_slow_stats) + evalStateT (mapM_ showStat stats) =<< getDirStatInfo o dir + return True + where + tostats = map (\s -> s dir) + +treeishInfo :: InfoOptions -> String -> Annex () +treeishInfo o t = do + mi <- getTreeStatInfo o (Git.Ref t) + case mi of + Nothing -> noInfo t + Just i -> showCustom (unwords ["info", t]) $ do + stats <- selStats + (tostats (tree_name:tree_fast_stats False)) + (tostats tree_slow_stats) + evalStateT (mapM_ showStat stats) i + return True + where + tostats = map (\s -> s t) + +fileInfo :: InfoOptions -> FilePath -> Key -> Annex () +fileInfo o file k = showCustom (unwords ["info", file]) $ do + evalStateT (mapM_ showStat (file_stats file k)) (emptyStatInfo o) + return True + +remoteInfo :: InfoOptions -> Remote -> Annex () +remoteInfo o r = showCustom (unwords ["info", Remote.name r]) $ do + i <- map (\(k, v) -> simpleStat k (pure v)) <$> Remote.getInfo r + let u = Remote.uuid r + l <- selStats + (uuid_fast_stats u ++ remote_fast_stats r ++ i) + (uuid_slow_stats u) + evalStateT (mapM_ showStat l) (emptyStatInfo o) + return True + +uuidInfo :: InfoOptions -> UUID -> Annex () +uuidInfo o u = showCustom (unwords ["info", fromUUID u]) $ do + l <- selStats (uuid_fast_stats u) (uuid_slow_stats u) + evalStateT (mapM_ showStat l) (emptyStatInfo o) + return True + +selStats :: [Stat] -> [Stat] -> Annex [Stat] +selStats fast_stats slow_stats = do + fast <- Annex.getState Annex.fast + return $ if fast + then fast_stats + else fast_stats ++ slow_stats + +{- Order is significant. Less expensive operations, and operations + - that share data go together. + -} +global_fast_stats :: [Stat] +global_fast_stats = + [ repository_mode + , repo_list Trusted + , repo_list SemiTrusted + , repo_list UnTrusted + , transfer_list + , disk_size + ] + +global_slow_stats :: [Stat] +global_slow_stats = + [ tmp_size + , bad_data_size + , local_annex_keys + , local_annex_size + , known_annex_files True + , known_annex_size True + , bloom_info + , backend_usage + ] + +tree_fast_stats :: Bool -> [FilePath -> Stat] +tree_fast_stats isworktree = + [ const local_annex_keys + , const local_annex_size + , const (known_annex_files isworktree) + , const (known_annex_size isworktree) + ] + +tree_slow_stats :: [FilePath -> Stat] +tree_slow_stats = + [ const numcopies_stats + , const reposizes_stats + , const reposizes_total + ] + +file_stats :: FilePath -> Key -> [Stat] +file_stats f k = + [ file_name f + , key_size k + , key_name k + , content_present k + ] + +remote_fast_stats :: Remote -> [Stat] +remote_fast_stats r = map (\s -> s r) + [ remote_name + , remote_trust + , remote_cost + , remote_type + ] + +uuid_fast_stats :: UUID -> [Stat] +uuid_fast_stats u = map (\s -> s u) + [ repo_uuid + , repo_description + ] + +uuid_slow_stats :: UUID -> [Stat] +uuid_slow_stats u = map (\s -> s u) + [ repo_annex_keys + , repo_annex_size + ] + +stat :: String -> (String -> StatState String) -> Stat +stat desc a = return $ Just (desc, a desc) + +-- The json simply contains the same string that is displayed. +simpleStat :: String -> StatState String -> Stat +simpleStat desc getval = stat desc $ json id getval + +nostat :: Stat +nostat = return Nothing + +json :: ToJSON' j => (j -> String) -> StatState j -> String -> StatState String +json fmt a desc = do + j <- a + lift $ maybeShowJSON $ JSONChunk [(desc, j)] + return $ fmt j + +nojson :: StatState String -> String -> StatState String +nojson a _ = a + +showStat :: Stat -> StatState () +showStat s = maybe noop calc =<< s + where + calc (desc, a) = do + (lift . showHeader) desc + lift . showRaw =<< a + +repository_mode :: Stat +repository_mode = simpleStat "repository mode" $ lift $ + ifM isDirect + ( return "direct" + , ifM (fromRepo Git.repoIsLocalBare) + ( return "bare" + , return "indirect" + ) + ) + +repo_list :: TrustLevel -> Stat +repo_list level = stat n $ nojson $ lift $ do + us <- filter (/= NoUUID) . M.keys + <$> (M.union <$> uuidMap <*> remoteMap Remote.name) + rs <- fst <$> trustPartition level us + countRepoList (length rs) + -- This also handles json display. + <$> prettyPrintUUIDs n rs + where + n = showTrustLevel level ++ " repositories" + +countRepoList :: Int -> String -> String +countRepoList _ [] = "0" +countRepoList n s = show n ++ "\n" ++ beginning s + +dir_name :: FilePath -> Stat +dir_name dir = simpleStat "directory" $ pure dir + +tree_name :: String -> Stat +tree_name t = simpleStat "tree" $ pure t + +file_name :: FilePath -> Stat +file_name file = simpleStat "file" $ pure file + +remote_name :: Remote -> Stat +remote_name r = simpleStat "remote" $ pure (Remote.name r) + +repo_description :: UUID -> Stat +repo_description = simpleStat "description" . lift . Remote.prettyUUID + +repo_uuid :: UUID -> Stat +repo_uuid = simpleStat "uuid" . pure . fromUUID + +remote_trust :: Remote -> Stat +remote_trust r = simpleStat "trust" $ lift $ + showTrustLevel <$> lookupTrust (Remote.uuid r) + +remote_cost :: Remote -> Stat +remote_cost r = simpleStat "cost" $ pure $ + show $ Remote.cost r + +remote_type :: Remote -> Stat +remote_type r = simpleStat "type" $ pure $ + Remote.typename $ Remote.remotetype r + +local_annex_keys :: Stat +local_annex_keys = stat "local annex keys" $ json show $ + countKeys <$> cachedPresentData + +local_annex_size :: Stat +local_annex_size = simpleStat "local annex size" $ + showSizeKeys =<< cachedPresentData + +-- "remote" is in the name for JSON backwards-compatibility +repo_annex_keys :: UUID -> Stat +repo_annex_keys u = stat "remote annex keys" $ json show $ + countKeys <$> cachedRemoteData u + +-- "remote" is in the name for JSON backwards-compatibility +repo_annex_size :: UUID -> Stat +repo_annex_size u = simpleStat "remote annex size" $ + showSizeKeys =<< cachedRemoteData u + +known_annex_files :: Bool -> Stat +known_annex_files isworktree = + stat ("annexed files in " ++ treeDesc isworktree) $ json show $ + countKeys <$> cachedReferencedData + +known_annex_size :: Bool -> Stat +known_annex_size isworktree = + simpleStat ("size of annexed files in " ++ treeDesc isworktree) $ + showSizeKeys =<< cachedReferencedData + +treeDesc :: Bool -> String +treeDesc True = "working tree" +treeDesc False = "tree" + +tmp_size :: Stat +tmp_size = staleSize "temporary object directory size" gitAnnexTmpObjectDir + +bad_data_size :: Stat +bad_data_size = staleSize "bad keys size" gitAnnexBadDir + +key_size :: Key -> Stat +key_size k = simpleStat "size" $ showSizeKeys $ foldKeys [k] + +key_name :: Key -> Stat +key_name k = simpleStat "key" $ pure $ key2file k + +content_present :: Key -> Stat +content_present k = stat "present" $ json boolConfig $ lift $ inAnnex k + +bloom_info :: Stat +bloom_info = simpleStat "bloom filter size" $ do + localkeys <- countKeys <$> cachedPresentData + capacity <- fromIntegral <$> lift bloomCapacity + let note = aside $ + if localkeys >= capacity + then "appears too small for this repository; adjust annex.bloomcapacity" + else showPercentage 1 (percentage capacity localkeys) ++ " full" + + -- Two bloom filters are used at the same time when running + -- git-annex unused, so double the size of one. + sizer <- mkSizer + size <- sizer memoryUnits False . (* 2) . fromIntegral . fst <$> + lift bloomBitsHashes + + return $ size ++ note + +transfer_list :: Stat +transfer_list = stat desc $ nojson $ lift $ do + uuidmap <- Remote.remoteMap id + ts <- getTransfers + maybeShowJSON $ JSONChunk [(desc, V.fromList $ map (uncurry jsonify) ts)] + return $ if null ts + then "none" + else multiLine $ + map (uncurry $ line uuidmap) $ sort ts + where + desc = "transfers in progress" + line uuidmap t i = unwords + [ formatDirection (transferDirection t) ++ "ing" + , actionItemDesc + (ActionItemAssociatedFile (associatedFile i)) + (transferKey t) + , if transferDirection t == Upload then "to" else "from" + , maybe (fromUUID $ transferUUID t) Remote.name $ + M.lookup (transferUUID t) uuidmap + ] + jsonify t i = object $ map (\(k, v) -> (packString k, v)) $ + [ ("transfer", toJSON' (formatDirection (transferDirection t))) + , ("key", toJSON' (transferKey t)) + , ("file", toJSON' afile) + , ("remote", toJSON' (fromUUID (transferUUID t))) + ] + where + AssociatedFile afile = associatedFile i + +disk_size :: Stat +disk_size = simpleStat "available local disk space" $ + calcfree + <$> (lift $ annexDiskReserve <$> Annex.getGitConfig) + <*> (lift $ inRepo $ getDiskFree . gitAnnexDir) + <*> mkSizer + where + calcfree reserve (Just have) sizer = unwords + [ sizer storageUnits False $ nonneg $ have - reserve + , "(+" ++ sizer storageUnits False reserve + , "reserved)" + ] + calcfree _ _ _ = "unknown" + + nonneg x + | x >= 0 = x + | otherwise = 0 + +backend_usage :: Stat +backend_usage = stat "backend usage" $ json fmt $ + ObjectMap . (M.mapKeys formatKeyVariety) . backendsKeys + <$> cachedReferencedData + where + fmt = multiLine . map (\(b, n) -> b ++ ": " ++ show n) . sort . M.toList . fromObjectMap + +numcopies_stats :: Stat +numcopies_stats = stat "numcopies stats" $ json fmt $ + calc <$> (maybe M.empty numCopiesVarianceMap <$> cachedNumCopiesStats) + where + calc = V.fromList + . map (\(variance, count) -> (show variance, count)) + . sortBy (flip (comparing fst)) + . M.toList + fmt = multiLine + . map (\(variance, count) -> "numcopies " ++ variance ++ ": " ++ show count) + . V.toList + +reposizes_stats :: Stat +reposizes_stats = stat desc $ nojson $ do + sizer <- mkSizer + l <- map (\(u, kd) -> (u, sizer storageUnits True (sizeKeys kd))) + . sortBy (flip (comparing (sizeKeys . snd))) + . M.toList + <$> cachedRepoData + let maxlen = maximum (map (length . snd) l) + descm <- lift uuidDescriptions + -- This also handles json display. + s <- lift $ prettyPrintUUIDsWith (Just "size") desc descm (Just . show) $ + map (\(u, sz) -> (u, Just $ mkdisp sz maxlen)) l + return $ countRepoList (length l) s + where + desc = "repositories containing these files" + mkdisp sz maxlen = DualDisp + { dispNormal = lpad maxlen sz + , dispJson = sz + } + lpad n s = (replicate (n - length s) ' ') ++ s + +reposizes_total :: Stat +reposizes_total = simpleStat "combined size of repositories containing these files" $ + showSizeKeys . mconcat . M.elems =<< cachedRepoData + +cachedPresentData :: StatState KeyData +cachedPresentData = do + s <- get + case presentData s of + Just v -> return v + Nothing -> do + v <- foldKeys <$> lift (getKeysPresent InRepository) + put s { presentData = Just v } + return v + +cachedRemoteData :: UUID -> StatState KeyData +cachedRemoteData u = do + s <- get + case M.lookup u (repoData s) of + Just v -> return v + Nothing -> do + let combinedata d uk = finishCheck uk >>= \case + Nothing -> return d + Just k -> return $ addKey k d + v <- lift $ foldM combinedata emptyKeyData + =<< loggedKeysFor' u + put s { repoData = M.insert u v (repoData s) } + return v + +cachedReferencedData :: StatState KeyData +cachedReferencedData = do + s <- get + case referencedData s of + Just v -> return v + Nothing -> do + !v <- lift $ Command.Unused.withKeysReferenced + emptyKeyData addKey + put s { referencedData = Just v } + return v + +-- currently only available for directory info +cachedNumCopiesStats :: StatState (Maybe NumCopiesStats) +cachedNumCopiesStats = numCopiesStats <$> get + +-- currently only available for directory info +cachedRepoData :: StatState (M.Map UUID KeyData) +cachedRepoData = repoData <$> get + +getDirStatInfo :: InfoOptions -> FilePath -> Annex StatInfo +getDirStatInfo o dir = do + fast <- Annex.getState Annex.fast + matcher <- Limit.getMatcher + (presentdata, referenceddata, numcopiesstats, repodata) <- + Command.Unused.withKeysFilesReferencedIn dir initial + (update matcher fast) + return $ StatInfo (Just presentdata) (Just referenceddata) repodata (Just numcopiesstats) o + where + initial = (emptyKeyData, emptyKeyData, emptyNumCopiesStats, M.empty) + update matcher fast key file vs@(presentdata, referenceddata, numcopiesstats, repodata) = + ifM (matcher $ MatchingFile $ FileInfo file file) + ( do + !presentdata' <- ifM (inAnnex key) + ( return $ addKey key presentdata + , return presentdata + ) + let !referenceddata' = addKey key referenceddata + (!numcopiesstats', !repodata') <- if fast + then return (numcopiesstats, repodata) + else do + locs <- Remote.keyLocations key + nc <- updateNumCopiesStats file numcopiesstats locs + return (nc, updateRepoData key locs repodata) + return $! (presentdata', referenceddata', numcopiesstats', repodata') + , return vs + ) + +getTreeStatInfo :: InfoOptions -> Git.Ref -> Annex (Maybe StatInfo) +getTreeStatInfo o r = do + fast <- Annex.getState Annex.fast + (ls, cleanup) <- inRepo $ LsTree.lsTree r + (presentdata, referenceddata, repodata) <- go fast ls initial + ifM (liftIO cleanup) + ( return $ Just $ + StatInfo (Just presentdata) (Just referenceddata) repodata Nothing o + , return Nothing + ) + where + initial = (emptyKeyData, emptyKeyData, M.empty) + go _ [] vs = return vs + go fast (l:ls) vs@(presentdata, referenceddata, repodata) = do + mk <- catKey (LsTree.sha l) + case mk of + Nothing -> go fast ls vs + Just key -> do + !presentdata' <- ifM (inAnnex key) + ( return $ addKey key presentdata + , return presentdata + ) + let !referenceddata' = addKey key referenceddata + !repodata' <- if fast + then return repodata + else do + locs <- Remote.keyLocations key + return (updateRepoData key locs repodata) + go fast ls $! (presentdata', referenceddata', repodata') + +emptyKeyData :: KeyData +emptyKeyData = KeyData 0 0 0 M.empty + +emptyNumCopiesStats :: NumCopiesStats +emptyNumCopiesStats = NumCopiesStats M.empty + +foldKeys :: [Key] -> KeyData +foldKeys = foldl' (flip addKey) emptyKeyData + +addKey :: Key -> KeyData -> KeyData +addKey key (KeyData count size unknownsize backends) = + KeyData count' size' unknownsize' backends' + where + {- All calculations strict to avoid thunks when repeatedly + - applied to many keys. -} + !count' = count + 1 + !backends' = M.insertWith (+) (keyVariety key) 1 backends + !size' = maybe size (+ size) ks + !unknownsize' = maybe (unknownsize + 1) (const unknownsize) ks + ks = keySize key + +updateRepoData :: Key -> [UUID] -> M.Map UUID KeyData -> M.Map UUID KeyData +updateRepoData key locs m = m' + where + !m' = M.unionWith (\_old new -> new) m $ + M.fromList $ zip locs (map update locs) + update loc = addKey key (fromMaybe emptyKeyData $ M.lookup loc m) + +updateNumCopiesStats :: FilePath -> NumCopiesStats -> [UUID] -> Annex NumCopiesStats +updateNumCopiesStats file (NumCopiesStats m) locs = do + have <- trustExclude UnTrusted locs + !variance <- Variance <$> numCopiesCheck' file (-) have + let !m' = M.insertWith (+) variance 1 m + let !ret = NumCopiesStats m' + return ret + +showSizeKeys :: KeyData -> StatState String +showSizeKeys d = do + sizer <- mkSizer + return $ total sizer ++ missingnote + where + total sizer = sizer storageUnits False $ sizeKeys d + missingnote + | unknownSizeKeys d == 0 = "" + | otherwise = aside $ + "+ " ++ show (unknownSizeKeys d) ++ + " unknown size" + +staleSize :: String -> (Git.Repo -> FilePath) -> Stat +staleSize label dirspec = go =<< lift (dirKeys dirspec) + where + go [] = nostat + go keys = onsize =<< sum <$> keysizes keys + onsize 0 = nostat + onsize size = stat label $ + json (++ aside "clean up with git-annex unused") $ do + sizer <- mkSizer + return $ sizer storageUnits False size + keysizes keys = do + dir <- lift $ fromRepo dirspec + liftIO $ forM keys $ \k -> catchDefaultIO 0 $ + getFileSize (dir keyFile k) + +aside :: String -> String +aside s = " (" ++ s ++ ")" + +multiLine :: [String] -> String +multiLine = concatMap (\l -> "\n\t" ++ l) + +mkSizer :: StatState ([Unit] -> Bool -> ByteSize -> String) +mkSizer = ifM (bytesOption . infoOptions <$> get) + ( return (const $ const show) + , return roughSize + ) diff --git a/Command/Init.hs b/Command/Init.hs new file mode 100644 index 0000000000..8ce82a75e8 --- /dev/null +++ b/Command/Init.hs @@ -0,0 +1,52 @@ +{- git-annex command + - + - Copyright 2010 Joey Hess + - + - Licensed under the GNU GPL version 3 or higher. + -} + +module Command.Init where + +import Command +import Annex.Init +import Annex.Version +import qualified Annex.SpecialRemote + +cmd :: Command +cmd = dontCheck repoExists $ + command "init" SectionSetup "initialize git-annex" + paramDesc (seek <$$> optParser) + +data InitOptions = InitOptions + { initDesc :: String + , initVersion :: Maybe Version + } + +optParser :: CmdParamsDesc -> Parser InitOptions +optParser desc = InitOptions + <$> (unwords <$> cmdParams desc) + <*> optional (option (str >>= parseVersion) + ( long "version" <> metavar paramValue + <> help "Override default annex.version" + )) + +parseVersion :: Monad m => String -> m Version +parseVersion v + | v `elem` supportedVersions = return v + | otherwise = fail $ v ++ " is not a currently supported repository version" + +seek :: InitOptions -> CommandSeek +seek = commandAction . start + +start :: InitOptions -> CommandStart +start os = do + showStart' "init" (Just $ initDesc os) + next $ perform os + +perform :: InitOptions -> CommandPerform +perform os = do + initialize (AutoInit False) + (if null (initDesc os) then Nothing else Just (initDesc os)) + (initVersion os) + Annex.SpecialRemote.autoEnable + next $ return True diff --git a/Command/InitRemote.hs b/Command/InitRemote.hs new file mode 100644 index 0000000000..f781416810 --- /dev/null +++ b/Command/InitRemote.hs @@ -0,0 +1,64 @@ +{- git-annex command + - + - Copyright 2011,2013 Joey Hess + - + - Licensed under the GNU GPL version 3 or higher. + -} + +module Command.InitRemote where + +import qualified Data.Map as M + +import Command +import Annex.SpecialRemote +import qualified Remote +import qualified Logs.Remote +import qualified Types.Remote as R +import Logs.UUID +import Types.GitConfig + +cmd :: Command +cmd = command "initremote" SectionSetup + "creates a special (non-git) remote" + (paramPair paramName $ paramOptional $ paramRepeating paramKeyValue) + (withParams seek) + +seek :: CmdParams -> CommandSeek +seek = withWords (commandAction . start) + +start :: [String] -> CommandStart +start [] = giveup "Specify a name for the remote." +start (name:ws) = ifM (isJust <$> findExisting name) + ( giveup $ "There is already a special remote named \"" ++ name ++ + "\". (Use enableremote to enable an existing special remote.)" + , do + ifM (isJust <$> Remote.byNameOnly name) + ( giveup $ "There is already a remote named \"" ++ name ++ "\"" + , do + let c = newConfig name + t <- either giveup return (findType config) + + showStart' "initremote" (Just name) + next $ perform t name $ M.union config c + ) + ) + where + config = Logs.Remote.keyValToConfig ws + +perform :: RemoteType -> String -> R.RemoteConfig -> CommandPerform +perform t name c = do + dummycfg <- liftIO dummyRemoteGitConfig + (c', u) <- R.setup t R.Init cu Nothing c dummycfg + next $ cleanup u name c' + where + cu = case M.lookup "uuid" c of + Just s + | isUUID s -> Just (toUUID s) + | otherwise -> giveup "invalid uuid" + Nothing -> Nothing + +cleanup :: UUID -> String -> R.RemoteConfig -> CommandCleanup +cleanup u name c = do + describeUUID u name + Logs.Remote.configSet u c + return True diff --git a/Command/Inprogress.hs b/Command/Inprogress.hs new file mode 100644 index 0000000000..0d5032e9cf --- /dev/null +++ b/Command/Inprogress.hs @@ -0,0 +1,61 @@ +{- git-annex command + - + - Copyright 2017 Joey Hess + - + - Licensed under the GNU GPL version 3 or higher. + -} + +module Command.Inprogress where + +import Command +import Annex.Transfer + +import qualified Data.Set as S + +cmd :: Command +cmd = noCommit $ noMessages $ command "inprogress" SectionQuery + "access files while they're being downloaded" + paramPaths (seek <$$> optParser) + +data InprogressOptions = InprogressOptions + { inprogressFiles :: CmdParams + , allOption :: Bool + } + +optParser :: CmdParamsDesc -> Parser InprogressOptions +optParser desc = InprogressOptions + <$> cmdParams desc + <*> switch + ( long "all" + <> short 'A' + <> help "access all files currently being downloaded" + ) + +seek :: InprogressOptions -> CommandSeek +seek o = do + ts <- map (transferKey . fst) <$> getTransfers + if allOption o + then forM_ ts $ commandAction . start' + else do + let s = S.fromList ts + withFilesInGit + (commandAction . (whenAnnexed (start s))) + =<< workTreeItems (inprogressFiles o) + +start :: S.Set Key -> FilePath -> Key -> CommandStart +start s _file k + | S.member k s = start' k + | otherwise = notInprogress + +start' :: Key -> CommandStart +start' k = do + tmpf <- fromRepo $ gitAnnexTmpObjectLocation k + ifM (liftIO $ doesFileExist tmpf) + ( next $ next $ do + liftIO $ putStrLn tmpf + return True + , notInprogress + ) + +notInprogress :: CommandStart +notInprogress = stop diff --git a/Command/List.hs b/Command/List.hs new file mode 100644 index 0000000000..ef95310450 --- /dev/null +++ b/Command/List.hs @@ -0,0 +1,97 @@ +{- git-annex command + - + - Copyright 2013 Joey Hess + - Copyright 2013 Antoine Beaupré + - + - Licensed under the GNU GPL version 3 or higher. + -} + +module Command.List where + +import qualified Data.Set as S +import qualified Data.Map as M +import Data.Function +import Data.Ord + +import Command +import Remote +import Logs.Trust +import Logs.UUID +import Annex.UUID +import Git.Types (RemoteName) +import Utility.Tuple + +cmd :: Command +cmd = noCommit $ withGlobalOptions [annexedMatchingOptions] $ + command "list" SectionQuery + "show which remotes contain files" + paramPaths (seek <$$> optParser) + +data ListOptions = ListOptions + { listThese :: CmdParams + , allRepos :: Bool + } + +optParser :: CmdParamsDesc -> Parser ListOptions +optParser desc = ListOptions + <$> cmdParams desc + <*> switch + ( long "allrepos" + <> help "show all repositories, not only remotes" + ) + +seek :: ListOptions -> CommandSeek +seek o = do + list <- getList o + printHeader list + withFilesInGit + (commandAction . (whenAnnexed $ start list)) + =<< workTreeItems (listThese o) + +getList :: ListOptions -> Annex [(UUID, RemoteName, TrustLevel)] +getList o + | allRepos o = nubBy ((==) `on` fst3) <$> ((++) <$> getRemotes <*> getAllUUIDs) + | otherwise = getRemotes + where + getRemotes = do + rs <- remoteList + ts <- mapM (lookupTrust . uuid) rs + hereu <- getUUID + heretrust <- lookupTrust hereu + let l = (hereu, "here", heretrust) : zip3 (map uuid rs) (map name rs) ts + return $ filter (\(_, _, t) -> t /= DeadTrusted) l + getAllUUIDs = do + rs <- M.toList <$> uuidMap + rs3 <- forM rs $ \(u, n) -> (,,) + <$> pure u + <*> pure n + <*> lookupTrust u + return $ sortBy (comparing snd3) $ + filter (\t -> thd3 t /= DeadTrusted) rs3 + +printHeader :: [(UUID, RemoteName, TrustLevel)] -> Annex () +printHeader l = liftIO $ putStrLn $ lheader $ map (\(_, n, t) -> (n, t)) l + +start :: [(UUID, RemoteName, TrustLevel)] -> FilePath -> Key -> CommandStart +start l file key = do + ls <- S.fromList <$> keyLocations key + liftIO $ putStrLn $ format (map (\(u, _, t) -> (t, S.member u ls)) l) file + stop + +type Present = Bool + +lheader :: [(RemoteName, TrustLevel)] -> String +lheader remotes = unlines (zipWith formatheader [0..] remotes) ++ pipes (length remotes) + where + formatheader n (remotename, trustlevel) = pipes n ++ remotename ++ trust trustlevel + pipes = flip replicate '|' + trust UnTrusted = " (untrusted)" + trust _ = "" + +format :: [(TrustLevel, Present)] -> FilePath -> String +format remotes file = thereMap ++ " " ++ file + where + thereMap = concatMap there remotes + there (UnTrusted, True) = "x" + there (_, True) = "X" + there (_, False) = "_" diff --git a/Command/Lock.hs b/Command/Lock.hs new file mode 100644 index 0000000000..142e2c73cc --- /dev/null +++ b/Command/Lock.hs @@ -0,0 +1,120 @@ +{- git-annex command + - + - Copyright 2010,2015 Joey Hess + - + - Licensed under the GNU GPL version 3 or higher. + -} + +module Command.Lock where + +import Command +import qualified Annex.Queue +import qualified Annex +import Annex.Version +import Annex.Content +import Annex.Link +import Annex.InodeSentinal +import Annex.Perms +import Annex.ReplaceFile +import Utility.InodeCache +import qualified Database.Keys +import Annex.Ingest +import Logs.Location +import Git.FilePath + +cmd :: Command +cmd = notDirect $ withGlobalOptions [jsonOptions, annexedMatchingOptions] $ + command "lock" SectionCommon + "undo unlock command" + paramPaths (withParams seek) + +seek :: CmdParams -> CommandSeek +seek ps = do + l <- workTreeItems ps + ifM versionSupportsUnlockedPointers + ( withFilesInGit (commandAction . (whenAnnexed startNew)) l + , do + withFilesOldUnlocked (commandAction . startOld) l + withFilesOldUnlockedToBeCommitted (commandAction . startOld) l + ) + +startNew :: FilePath -> Key -> CommandStart +startNew file key = ifM (isJust <$> isAnnexLink file) + ( stop + , do + showStart "lock" file + go =<< liftIO (isPointerFile file) + ) + where + go (Just key') + | key' == key = cont + | otherwise = errorModified + go Nothing = + ifM (isUnmodified key file) + ( cont + , ifM (Annex.getState Annex.force) + ( cont + , errorModified + ) + ) + cont = next $ performNew file key + +performNew :: FilePath -> Key -> CommandPerform +performNew file key = do + lockdown =<< calcRepo (gitAnnexLocation key) + addLink file key + =<< withTSDelta (liftIO . genInodeCache file) + next $ cleanupNew file key + where + lockdown obj = do + ifM (isUnmodified key obj) + ( breakhardlink obj + , repopulate obj + ) + whenM (liftIO $ doesFileExist obj) $ + freezeContent obj + + -- It's ok if the file is hard linked to obj, but if some other + -- associated file is, we need to break that link to lock down obj. + breakhardlink obj = whenM (catchBoolIO $ (> 1) . linkCount <$> liftIO (getFileStatus obj)) $ do + mfc <- withTSDelta (liftIO . genInodeCache file) + unlessM (sameInodeCache obj (maybeToList mfc)) $ do + modifyContent obj $ replaceFile obj $ \tmp -> do + unlessM (checkedCopyFile key obj tmp Nothing) $ + giveup "unable to lock file" + Database.Keys.storeInodeCaches key [obj] + + -- Try to repopulate obj from an unmodified associated file. + repopulate obj = modifyContent obj $ do + g <- Annex.gitRepo + fs <- map (`fromTopFilePath` g) + <$> Database.Keys.getAssociatedFiles key + mfile <- firstM (isUnmodified key) fs + liftIO $ nukeFile obj + case mfile of + Just unmodified -> + unlessM (checkedCopyFile key unmodified obj Nothing) + lostcontent + Nothing -> lostcontent + + lostcontent = logStatus key InfoMissing + +cleanupNew :: FilePath -> Key -> CommandCleanup +cleanupNew file key = do + Database.Keys.removeAssociatedFile key =<< inRepo (toTopFilePath file) + return True + +startOld :: FilePath -> CommandStart +startOld file = do + showStart "lock" file + unlessM (Annex.getState Annex.force) + errorModified + next $ performOld file + +performOld :: FilePath -> CommandPerform +performOld file = do + Annex.Queue.addCommand "checkout" [Param "--"] [file] + next $ return True + +errorModified :: a +errorModified = giveup "Locking this file would discard any changes you have made to it. Use 'git annex add' to stage your changes. (Or, use --force to override)" diff --git a/Command/LockContent.hs b/Command/LockContent.hs new file mode 100644 index 0000000000..b5902ed282 --- /dev/null +++ b/Command/LockContent.hs @@ -0,0 +1,41 @@ +{- git-annex-shell command + - + - Copyright 2015 Joey Hess + - + - Licensed under the GNU GPL version 3 or higher. + -} + +module Command.LockContent where + +import Command +import Annex.Content +import Remote.Helper.Ssh (contentLockedMarker) +import Utility.SimpleProtocol + +cmd :: Command +cmd = noCommit $ + command "lockcontent" SectionPlumbing + "locks key's content in the annex, preventing it being dropped" + paramKey + (withParams seek) + +seek :: CmdParams -> CommandSeek +seek = withWords (commandAction . start) + +-- First, lock the content, then print out "OK". +-- Wait for the caller to send a line before dropping the lock. +start :: [String] -> CommandStart +start [ks] = do + ok <- lockContentShared k (const locksuccess) + `catchNonAsync` (const $ return False) + liftIO $ if ok + then exitSuccess + else exitFailure + where + k = fromMaybe (giveup "bad key") (file2key ks) + locksuccess = liftIO $ do + putStrLn contentLockedMarker + hFlush stdout + _ <- getProtocolLine stdin + return True +start _ = giveup "Specify exactly 1 key." diff --git a/Command/Log.hs b/Command/Log.hs new file mode 100644 index 0000000000..f5d14bd03a --- /dev/null +++ b/Command/Log.hs @@ -0,0 +1,284 @@ +{- git-annex command + - + - Copyright 2012, 2016 Joey Hess + - + - Licensed under the GNU GPL version 3 or higher. + -} + +{-# LANGUAGE CPP #-} + +module Command.Log where + +import qualified Data.Set as S +import qualified Data.Map as M +import Data.Char +import Data.Time.Clock.POSIX +import Data.Time +#if ! MIN_VERSION_time(1,5,0) +import System.Locale +#endif + +import Command +import Logs +import Logs.Location +import qualified Annex.Branch +import qualified Git +import Git.Command +import qualified Remote +import qualified Annex + +data RefChange = RefChange + { changetime :: POSIXTime + , oldref :: Git.Ref + , newref :: Git.Ref + , changekey :: Key + } + deriving (Show) + +data LogChange = Added | Removed + +type Outputter = LogChange -> POSIXTime -> [UUID] -> Annex () + +cmd :: Command +cmd = withGlobalOptions [annexedMatchingOptions] $ + command "log" SectionQuery "shows location log" + paramPaths (seek <$$> optParser) + +data LogOptions = LogOptions + { logFiles :: CmdParams + , allOption :: Bool + , rawDateOption :: Bool + , gourceOption :: Bool + , passthruOptions :: [CommandParam] + } + +optParser :: CmdParamsDesc -> Parser LogOptions +optParser desc = LogOptions + <$> cmdParams desc + <*> switch + ( long "all" + <> short 'A' + <> help "display location log changes to all files" + ) + <*> switch + ( long "raw-date" + <> help "display seconds from unix epoch" + ) + <*> switch + ( long "gource" + <> help "format output for gource" + ) + <*> (concat <$> many passthru) + where + passthru :: Parser [CommandParam] + passthru = datepassthru "since" + <|> datepassthru "after" + <|> datepassthru "until" + <|> datepassthru "before" + <|> (mkpassthru "max-count" <$> strOption + ( long "max-count" <> metavar paramNumber + <> help "limit number of logs displayed" + )) + datepassthru n = mkpassthru n <$> strOption + ( long n <> metavar paramDate + <> help ("show log " ++ n ++ " date") + ) + mkpassthru n v = [Param ("--" ++ n), Param v] + +seek :: LogOptions -> CommandSeek +seek o = do + m <- Remote.uuidDescriptions + zone <- liftIO getCurrentTimeZone + let outputter = mkOutputter m zone o + case (logFiles o, allOption o) of + (fs, False) -> withFilesInGit + (commandAction . (whenAnnexed $ start o outputter)) + =<< workTreeItems fs + ([], True) -> commandAction (startAll o outputter) + (_, True) -> giveup "Cannot specify both files and --all" + +start :: LogOptions -> (FilePath -> Outputter) -> FilePath -> Key -> CommandStart +start o outputter file key = do + (changes, cleanup) <- getKeyLog key (passthruOptions o) + showLogIncremental (outputter file) changes + void $ liftIO cleanup + stop + +startAll :: LogOptions -> (String -> Outputter) -> CommandStart +startAll o outputter = do + (changes, cleanup) <- getAllLog (passthruOptions o) + showLog outputter changes + void $ liftIO cleanup + stop + +{- Displays changes made. Only works when all the RefChanges are for the + - same key. The method is to compare each value with the value + - after it in the list, which is the old version of the value. + - + - This ncessarily buffers the whole list, so does not stream. + - But, the number of location log changes for a single key tends to be + - fairly small. + - + - This minimizes the number of reads from git; each logged value is read + - only once. + - + - This also generates subtly better output when the git-annex branch + - got diverged. + -} +showLogIncremental :: Outputter -> [RefChange] -> Annex () +showLogIncremental outputter ps = do + sets <- mapM (getset newref) ps + previous <- maybe (return genesis) (getset oldref) (lastMaybe ps) + let l = sets ++ [previous] + let changes = map (\((t, new), (_, old)) -> (t, new, old)) + (zip l (drop 1 l)) + sequence_ $ compareChanges outputter changes + where + genesis = (0, S.empty) + getset select change = do + s <- S.fromList <$> loggedLocationsRef (select change) + return (changetime change, s) + +{- Displays changes made. Streams, and can display changes affecting + - different keys, but does twice as much reading of logged values + - as showLogIncremental. -} +showLog :: (String -> Outputter) -> [RefChange] -> Annex () +showLog outputter cs = forM_ cs $ \c -> do + let keyname = key2file (changekey c) + new <- S.fromList <$> loggedLocationsRef (newref c) + old <- S.fromList <$> loggedLocationsRef (oldref c) + sequence_ $ compareChanges (outputter keyname) + [(changetime c, new, old)] + +mkOutputter :: M.Map UUID String -> TimeZone -> LogOptions -> FilePath -> Outputter +mkOutputter m zone o file + | rawDateOption o = normalOutput lookupdescription file show + | gourceOption o = gourceOutput lookupdescription file + | otherwise = normalOutput lookupdescription file (showTimeStamp zone) + where + lookupdescription u = fromMaybe (fromUUID u) $ M.lookup u m + +normalOutput :: (UUID -> String) -> FilePath -> (POSIXTime -> String) -> Outputter +normalOutput lookupdescription file formattime logchange ts us = + liftIO $ mapM_ (putStrLn . format) us + where + time = formattime ts + addel = case logchange of + Added -> "+" + Removed -> "-" + format u = unwords [ addel, time, file, "|", + fromUUID u ++ " -- " ++ lookupdescription u ] + +gourceOutput :: (UUID -> String) -> FilePath -> Outputter +gourceOutput lookupdescription file logchange ts us = + liftIO $ mapM_ (putStrLn . intercalate "|" . format) us + where + time = takeWhile isDigit $ show ts + addel = case logchange of + Added -> "A" + Removed -> "M" + format u = [ time, lookupdescription u, addel, file ] + +{- Generates a display of the changes. + - Uses a formatter to generate a display of items that are added and + - removed. -} +compareChanges :: Ord a => (LogChange -> POSIXTime -> [a] -> b) -> [(POSIXTime, S.Set a, S.Set a)] -> [b] +compareChanges format changes = concatMap diff changes + where + diff (ts, new, old) = + [ format Added ts $ S.toList $ S.difference new old + , format Removed ts $ S.toList $ S.difference old new + ] + +{- Streams the git log for a given key's location log file. + - + - This is complicated by git log using paths relative to the current + - directory, even when looking at files in a different branch. A wacky + - relative path to the log file has to be used. + - + - The --remove-empty is a significant optimisation. It relies on location + - log files never being deleted in normal operation. Letting git stop + - once the location log file is gone avoids it checking all the way back + - to commit 0 to see if it used to exist, so generally speeds things up a + - *lot* for newish files. -} +getKeyLog :: Key -> [CommandParam] -> Annex ([RefChange], IO Bool) +getKeyLog key os = do + top <- fromRepo Git.repoPath + p <- liftIO $ relPathCwdToFile top + config <- Annex.getGitConfig + let logfile = p locationLogFile config key + getGitLog [logfile] (Param "--remove-empty" : os) + +{- Streams the git log for all git-annex branch changes. -} +getAllLog :: [CommandParam] -> Annex ([RefChange], IO Bool) +getAllLog = getGitLog [] + +getGitLog :: [FilePath] -> [CommandParam] -> Annex ([RefChange], IO Bool) +getGitLog fs os = do + (ls, cleanup) <- inRepo $ pipeNullSplit $ + [ Param "log" + , Param "-z" + , Param "--pretty=format:%ct" + , Param "--raw" + , Param "--abbrev=40" + ] ++ os ++ + [ Param $ Git.fromRef Annex.Branch.fullname + , Param "--" + ] ++ map Param fs + return (parseGitRawLog ls, cleanup) + +-- Parses chunked git log --raw output, which looks something like: +-- +-- [ "timestamp\n:changeline" +-- , "logfile" +-- , "" +-- , "timestamp\n:changeline" +-- , "logfile" +-- , ":changeline" +-- , "logfile" +-- , "" +-- ] +-- +-- The timestamp is not included before all changelines, so +-- keep track of the most recently seen timestamp. +parseGitRawLog :: [String] -> [RefChange] +parseGitRawLog = parse epoch + where + epoch = toEnum 0 :: POSIXTime + parse oldts ([]:rest) = parse oldts rest + parse oldts (c1:c2:rest) = case mrc of + Just rc -> rc : parse ts rest + Nothing -> parse ts (c2:rest) + where + (ts, cl) = case separate (== '\n') c1 of + (cl', []) -> (oldts, cl') + (tss, cl') -> (parseTimeStamp tss, cl') + mrc = do + (old, new) <- parseRawChangeLine cl + key <- locationLogFileKey c2 + return $ RefChange + { changetime = ts + , oldref = old + , newref = new + , changekey = key + } + parse _ _ = [] + +-- Parses something like "100644 100644 oldsha newsha M" +parseRawChangeLine :: String -> Maybe (Git.Ref, Git.Ref) +parseRawChangeLine = go . words + where + go (_:_:oldsha:newsha:_) = Just (Git.Ref oldsha, Git.Ref newsha) + go _ = Nothing + +parseTimeStamp :: String -> POSIXTime +parseTimeStamp = utcTimeToPOSIXSeconds . fromMaybe (error "bad timestamp") . +#if MIN_VERSION_time(1,5,0) + parseTimeM True defaultTimeLocale "%s" +#else + parseTime defaultTimeLocale "%s" +#endif + +showTimeStamp :: TimeZone -> POSIXTime -> String +showTimeStamp zone = formatTime defaultTimeLocale rfc822DateFormat + . utcToZonedTime zone . posixSecondsToUTCTime diff --git a/Command/LookupKey.hs b/Command/LookupKey.hs new file mode 100644 index 0000000000..1a2a57f220 --- /dev/null +++ b/Command/LookupKey.hs @@ -0,0 +1,39 @@ +{- git-annex command + - + - Copyright 2013-2017 Joey Hess + - + - Licensed under the GNU GPL version 3 or higher. + -} + +module Command.LookupKey where + +import Command +import Annex.CatFile +import qualified Git.LsFiles + +cmd :: Command +cmd = notBareRepo $ noCommit $ noMessages $ + command "lookupkey" SectionPlumbing + "looks up key used for file" + (paramRepeating paramFile) + (batchable run (pure ())) + +run :: () -> String -> Annex Bool +run _ file = seekSingleGitFile file >>= \case + Nothing -> return False + Just file' -> catKeyFile file' >>= \case + Just k -> do + liftIO $ putStrLn $ key2file k + return True + Nothing -> return False + +-- To support absolute filenames, pass through git ls-files. +-- But, this plumbing command does not recurse through directories. +seekSingleGitFile :: FilePath -> Annex (Maybe FilePath) +seekSingleGitFile file = do + (l, cleanup) <- inRepo (Git.LsFiles.inRepo [file]) + r <- case l of + (f:[]) | takeFileName f == takeFileName file -> return (Just f) + _ -> return Nothing + void $ liftIO cleanup + return r diff --git a/Command/Map.hs b/Command/Map.hs new file mode 100644 index 0000000000..a4e44697e2 --- /dev/null +++ b/Command/Map.hs @@ -0,0 +1,277 @@ +{- git-annex command + - + - Copyright 2010 Joey Hess + - + - Licensed under the GNU GPL version 3 or higher. + -} + +module Command.Map where + +import qualified Data.Map as M + +import Command +import qualified Git +import qualified Git.Url +import qualified Git.Config +import qualified Git.Construct +import qualified Remote +import qualified Annex +import Annex.Ssh +import Annex.UUID +import Logs.UUID +import Logs.Trust +import Types.TrustLevel +import qualified Remote.Helper.Ssh as Ssh +import qualified Utility.Dot as Dot + +-- a link from the first repository to the second (its remote) +data Link = Link Git.Repo Git.Repo + +-- a repo and its remotes +type RepoRemotes = (Git.Repo, [Git.Repo]) + +cmd :: Command +cmd = dontCheck repoExists $ + command "map" SectionQuery + "generate map of repositories" + paramNothing (withParams seek) + +seek :: CmdParams -> CommandSeek +seek = withNothing (commandAction start) + +start :: CommandStart +start = do + rs <- combineSame <$> (spider =<< gitRepo) + + umap <- uuidMap + trustmap <- trustMapLoad + + file <- () <$> fromRepo gitAnnexDir <*> pure "map.dot" + + liftIO $ writeFile file (drawMap rs trustmap umap) + next $ next $ + ifM (Annex.getState Annex.fast) + ( runViewer file [] + , runViewer file + [ ("xdot", [File file]) + , ("dot", [Param "-Tx11", File file]) + ] + ) + +runViewer :: FilePath -> [(String, [CommandParam])] -> Annex Bool +runViewer file [] = do + showLongNote $ "left map in " ++ file + return True +runViewer file ((c, ps):rest) = ifM (liftIO $ inPath c) + ( do + showLongNote $ "running: " ++ c ++ unwords (toCommand ps) + showOutput + liftIO $ boolSystem c ps + , runViewer file rest + ) + +{- Generates a graph for dot(1). Each repository, and any other uuids + - (except for dead ones), are displayed as a node, and each of its + - remotes is represented as an edge pointing at the node for the remote. + - + - The order nodes are added to the graph matters, since dot will draw + - the first ones near to the top and left. So it looks better to put + - the repositories first, followed by uuids that were not matched + - to a repository. + -} +drawMap :: [RepoRemotes] -> TrustMap -> M.Map UUID String -> String +drawMap rs trustmap umap = Dot.graph $ repos ++ others + where + repos = map (node umap (map fst rs) trustmap) rs + ruuids = map (getUncachedUUID . fst) rs + others = map uuidnode $ + filter (\u -> M.lookup u trustmap /= Just DeadTrusted) $ + filter (`notElem` ruuids) (M.keys umap) + uuidnode u = trustDecorate trustmap u $ + Dot.graphNode (fromUUID u) $ M.findWithDefault "" u umap + +hostname :: Git.Repo -> String +hostname r + | Git.repoIsUrl r = fromMaybe (Git.repoLocation r) (Git.Url.host r) + | otherwise = "localhost" + +basehostname :: Git.Repo -> String +basehostname r = fromMaybe "" $ headMaybe $ splitc '.' $ hostname r + +{- A name to display for a repo. Uses the name from uuid.log if available, + - or the remote name if not. -} +repoName :: M.Map UUID String -> Git.Repo -> String +repoName umap r + | repouuid == NoUUID = fallback + | otherwise = M.findWithDefault fallback repouuid umap + where + repouuid = getUncachedUUID r + fallback = fromMaybe "unknown" $ Git.remoteName r + +{- A unique id for the node for a repo. Uses the annex.uuid if available. -} +nodeId :: Git.Repo -> String +nodeId r = + case getUncachedUUID r of + NoUUID -> Git.repoLocation r + UUID u -> u + +{- A node representing a repo. -} +node :: M.Map UUID String -> [Git.Repo] -> TrustMap -> RepoRemotes -> String +node umap fullinfo trustmap (r, rs) = unlines $ n:edges + where + n = Dot.subGraph (hostname r) (basehostname r) "lightblue" $ + trustDecorate trustmap (getUncachedUUID r) $ + Dot.graphNode (nodeId r) (repoName umap r) + edges = map (edge umap fullinfo r) rs + +{- An edge between two repos. The second repo is a remote of the first. -} +edge :: M.Map UUID String -> [Git.Repo] -> Git.Repo -> Git.Repo -> String +edge umap fullinfo from to = + Dot.graphEdge (nodeId from) (nodeId fullto) edgename + where + -- get the full info for the remote, to get its UUID + fullto = findfullinfo to + findfullinfo n = + case filter (same n) fullinfo of + [] -> n + (n':_) -> n' + {- Only name an edge if the name is different than the name + - that will be used for the destination node, and is + - different from its hostname. (This reduces visual clutter.) -} + edgename = maybe Nothing calcname $ Git.remoteName to + calcname n + | n `elem` [repoName umap fullto, hostname fullto] = Nothing + | otherwise = Just n + +trustDecorate :: TrustMap -> UUID -> String -> String +trustDecorate trustmap u s = case M.lookup u trustmap of + Just Trusted -> Dot.fillColor "green" s + Just UnTrusted -> Dot.fillColor "red" s + Just SemiTrusted -> Dot.fillColor "white" s + Just DeadTrusted -> Dot.fillColor "grey" s + Nothing -> Dot.fillColor "white" s + +{- Recursively searches out remotes starting with the specified repo. -} +spider :: Git.Repo -> Annex [RepoRemotes] +spider r = spider' [r] [] +spider' :: [Git.Repo] -> [RepoRemotes] -> Annex [RepoRemotes] +spider' [] known = return known +spider' (r:rs) known + | any (same r) (map fst known) = spider' rs known + | otherwise = do + r' <- scan r + + -- The remotes will be relative to r', and need to be + -- made absolute for later use. + remotes <- mapM (absRepo r') + =<< (liftIO $ Git.Construct.fromRemotes r') + + spider' (rs ++ remotes) ((r', remotes):known) + +{- Converts repos to a common absolute form. -} +absRepo :: Git.Repo -> Git.Repo -> Annex Git.Repo +absRepo reference r + | Git.repoIsUrl reference = return $ Git.Construct.localToUrl reference r + | Git.repoIsUrl r = return r + | otherwise = liftIO $ do + r' <- Git.Construct.fromAbsPath =<< absPath (Git.repoPath r) + r'' <- safely $ flip Annex.eval Annex.gitRepo =<< Annex.new r' + return (fromMaybe r' r'') + +{- Checks if two repos are the same. -} +same :: Git.Repo -> Git.Repo -> Bool +same a b + | both Git.repoIsUrl = matching Git.Url.scheme && matching Git.Url.authority && matching Git.repoPath + | neither Git.repoIsUrl = matching Git.repoPath + | otherwise = False + where + matching t = t a == t b + both t = t a && t b + neither t = not (t a) && not (t b) + +{- reads the config of a remote, with progress display -} +scan :: Git.Repo -> Annex Git.Repo +scan r = do + showStart' "map" (Just $ Git.repoDescribe r) + v <- tryScan r + case v of + Just r' -> do + showEndOk + return r' + Nothing -> do + showOutput + showEndFail + return r + +{- tries to read the config of a remote, returning it only if it can + - be accessed -} +tryScan :: Git.Repo -> Annex (Maybe Git.Repo) +tryScan r + | Git.repoIsSsh r = sshscan + | Git.repoIsUrl r = case Git.remoteName r of + -- Can't scan a non-ssh url, so use any cached uuid for it. + Just n -> Just <$> (either + (const (pure r)) + (liftIO . setUUID r . Remote.uuid) + =<< Remote.byName' n) + Nothing -> return $ Just r + | otherwise = liftIO $ safely $ Git.Config.read r + where + pipedconfig pcmd params = liftIO $ safely $ + withHandle StdoutHandle createProcessSuccess p $ + Git.Config.hRead r + where + p = proc pcmd $ toCommand params + + configlist = Ssh.onRemote NoConsumeStdin r + (pipedconfig, return Nothing) "configlist" [] [] + manualconfiglist = do + gc <- Annex.getRemoteGitConfig r + (sshcmd, sshparams) <- Ssh.toRepo NoConsumeStdin r gc remotecmd + liftIO $ pipedconfig sshcmd sshparams + where + remotecmd = "sh -c " ++ shellEscape + (cddir ++ " && " ++ "git config --null --list") + dir = Git.repoPath r + cddir + | "/~" `isPrefixOf` dir = + let (userhome, reldir) = span (/= '/') (drop 1 dir) + in "cd " ++ userhome ++ " && " ++ cdto (drop 1 reldir) + | otherwise = cdto dir + cdto p = "if ! cd " ++ shellEscape p ++ " 2>/dev/null; then cd " ++ shellEscape p ++ ".git; fi" + + -- First, try sshing and running git config manually, + -- only fall back to git-annex-shell configlist if that + -- fails. + -- + -- This is done for two reasons, first I'd like this + -- subcommand to be usable on non-git-annex repos. + -- Secondly, configlist doesn't include information about + -- the remote's remotes. + sshscan = do + sshnote + v <- manualconfiglist + case v of + Nothing -> do + sshnote + configlist + ok -> return ok + + sshnote = do + showAction "sshing" + showOutput + +{- Spidering can find multiple paths to the same repo, so this is used + - to combine (really remove) duplicate repos with the same UUID. -} +combineSame :: [RepoRemotes] -> [RepoRemotes] +combineSame = map snd . nubBy sameuuid . map pair + where + sameuuid (u1, _) (u2, _) = u1 == u2 && u1 /= NoUUID + pair (r, rs) = (getUncachedUUID r, (r, rs)) + +safely :: IO Git.Repo -> IO (Maybe Git.Repo) +safely a = do + result <- tryNonAsync a + case result of + Left _ -> return Nothing + Right r' -> return $ Just r' diff --git a/Command/MatchExpression.hs b/Command/MatchExpression.hs new file mode 100644 index 0000000000..fe18e9ffae --- /dev/null +++ b/Command/MatchExpression.hs @@ -0,0 +1,88 @@ +{- git-annex command + - + - Copyright 2016 Joey Hess + - + - Licensed under the GNU GPL version 3 or higher. + -} + +module Command.MatchExpression where + +import Command +import Annex.FileMatcher +import Types.FileMatcher +import Utility.DataUnits +import Utility.Matcher +import Annex.UUID +import Logs.Group + +import qualified Data.Map as M +import qualified Data.Set as S + +cmd :: Command +cmd = noCommit $ + command "matchexpression" SectionPlumbing + "checks if a preferred content expression matches" + paramExpression + (seek <$$> optParser) + +data MatchExpressionOptions = MatchExpressionOptions + { matchexpr :: String + , largeFilesExpression :: Bool + , matchinfo :: MatchInfo + } + +optParser :: CmdParamsDesc -> Parser MatchExpressionOptions +optParser desc = MatchExpressionOptions + <$> argument str (metavar desc) + <*> switch + ( long "largefiles" + <> help "parse as annex.largefiles expression" + ) + <*> (addkeysize <$> dataparser) + where + dataparser = MatchingInfo + <$> optinfo "file" (strOption + ( long "file" <> metavar paramFile + <> help "specify filename to match against" + )) + <*> optinfo "key" (option (str >>= parseKey) + ( long "key" <> metavar paramKey + <> help "specify key to match against" + )) + <*> optinfo "size" (option (str >>= maybe (fail "parse error") return . readSize dataUnits) + ( long "size" <> metavar paramSize + <> help "specify size to match against" + )) + <*> optinfo "mimetype" (strOption + ( long "mimetype" <> metavar paramValue + <> help "specify mime type to match against" + )) + + optinfo datadesc mk = (Right <$> mk) + <|> (pure $ Left $ missingdata datadesc) + missingdata datadesc = bail $ "cannot match this expression without " ++ datadesc ++ " data" + -- When a key is provided, use its size. + addkeysize i@(MatchingInfo f (Right k) _ m) = case keySize k of + Just sz -> MatchingInfo f (Right k) (Right sz) m + Nothing -> i + addkeysize i = i + +seek :: MatchExpressionOptions -> CommandSeek +seek o = do + parser <- if largeFilesExpression o + then mkLargeFilesParser + else preferredContentParser + matchAll matchAll groupMap M.empty . Just <$> getUUID + case parsedToMatcher $ parser ((matchexpr o)) of + Left e -> liftIO $ bail $ "bad expression: " ++ e + Right matcher -> ifM (checkmatcher matcher) + ( liftIO exitSuccess + , liftIO exitFailure + ) + where + checkmatcher matcher = matchMrun matcher $ \a -> a S.empty (matchinfo o) + +bail :: String -> IO a +bail s = do + hPutStrLn stderr s + exitWith $ ExitFailure 42 diff --git a/Command/Merge.hs b/Command/Merge.hs new file mode 100644 index 0000000000..1ed669aff0 --- /dev/null +++ b/Command/Merge.hs @@ -0,0 +1,36 @@ +{- git-annex command + - + - Copyright 2011, 2013 Joey Hess + - + - Licensed under the GNU GPL version 3 or higher. + -} + +module Command.Merge where + +import Command +import qualified Annex.Branch +import Command.Sync (prepMerge, mergeLocal, getCurrBranch, mergeConfig) + +cmd :: Command +cmd = command "merge" SectionMaintenance + "automatically merge changes from remotes" + paramNothing (withParams seek) + +seek :: CmdParams -> CommandSeek +seek _ = do + commandAction mergeBranch + commandAction mergeSynced + +mergeBranch :: CommandStart +mergeBranch = do + showStart' "merge" (Just "git-annex") + next $ do + Annex.Branch.update + -- commit explicitly, in case no remote branches were merged + Annex.Branch.commit =<< Annex.Branch.commitMessage + next $ return True + +mergeSynced :: CommandStart +mergeSynced = do + prepMerge + mergeLocal mergeConfig def =<< join getCurrBranch diff --git a/Command/MetaData.hs b/Command/MetaData.hs new file mode 100644 index 0000000000..5478f39226 --- /dev/null +++ b/Command/MetaData.hs @@ -0,0 +1,192 @@ +{- git-annex command + - + - Copyright 2014-2016 Joey Hess + - + - Licensed under the GNU GPL version 3 or higher. + -} + +module Command.MetaData where + +import Command +import Annex.MetaData +import Annex.VectorClock +import Logs.MetaData +import Annex.WorkTree +import Messages.JSON (JSONActionItem(..)) +import Types.Messages +import Utility.Aeson +import Limit + +import qualified Data.Set as S +import qualified Data.Map as M +import qualified Data.Text as T +import qualified Data.ByteString.Lazy.UTF8 as BU +import Control.Concurrent + +cmd :: Command +cmd = withGlobalOptions [jsonOptions, annexedMatchingOptions] $ + command "metadata" SectionMetaData + "sets or gets metadata of a file" + paramPaths (seek <$$> optParser) + +data MetaDataOptions = MetaDataOptions + { forFiles :: CmdParams + , getSet :: GetSet + , keyOptions :: Maybe KeyOptions + , batchOption :: BatchMode + } + +data GetSet = Get MetaField | GetAll | Set [ModMeta] + +optParser :: CmdParamsDesc -> Parser MetaDataOptions +optParser desc = MetaDataOptions + <$> cmdParams desc + <*> ((Get <$> getopt) <|> (Set <$> some modopts) <|> pure GetAll) + <*> optional parseKeyOptions + <*> parseBatchOption + where + getopt = option (eitherReader mkMetaField) + ( long "get" <> short 'g' <> metavar paramField + <> help "get single metadata field" + ) + modopts = option (eitherReader parseModMeta) + ( long "set" <> short 's' <> metavar "FIELD[+-]=VALUE" + <> help "set or unset metadata value" + ) + <|> (AddMeta tagMetaField . toMetaValue <$> strOption + ( long "tag" <> short 't' <> metavar "TAG" + <> help "set a tag" + )) + <|> (DelMeta tagMetaField . Just . toMetaValue <$> strOption + ( long "untag" <> short 'u' <> metavar "TAG" + <> help "remove a tag" + )) + <|> option (eitherReader (\f -> DelMeta <$> mkMetaField f <*> pure Nothing)) + ( long "remove" <> short 'r' <> metavar "FIELD" + <> help "remove all values of a field" + ) + <|> flag' DelAllMeta + ( long "remove-all" + <> help "remove all metadata" + ) + +seek :: MetaDataOptions -> CommandSeek +seek o = case batchOption o of + NoBatch -> do + c <- liftIO currentVectorClock + let seeker = case getSet o of + Get _ -> withFilesInGit + GetAll -> withFilesInGit + Set _ -> withFilesInGitNonRecursive + "Not recursively setting metadata. Use --force to do that." + withKeyOptions (keyOptions o) False + (commandAction . startKeys c o) + (seeker (commandAction . (whenAnnexed (start c o)))) + =<< workTreeItems (forFiles o) + Batch fmt -> withMessageState $ \s -> case outputType s of + JSONOutput _ -> ifM limited + ( giveup "combining --batch with file matching options is not currently supported" + , batchInput fmt parseJSONInput $ + commandAction . startBatch + ) + _ -> giveup "--batch is currently only supported in --json mode" + +start :: VectorClock -> MetaDataOptions -> FilePath -> Key -> CommandStart +start c o file k = startKeys c o (k, mkActionItem afile) + where + afile = AssociatedFile (Just file) + +startKeys :: VectorClock -> MetaDataOptions -> (Key, ActionItem) -> CommandStart +startKeys c o (k, ai) = case getSet o of + Get f -> do + l <- S.toList . currentMetaDataValues f <$> getCurrentMetaData k + liftIO $ forM_ l $ + putStrLn . fromMetaValue + stop + _ -> do + showStartKey "metadata" k ai + next $ perform c o k + +perform :: VectorClock -> MetaDataOptions -> Key -> CommandPerform +perform c o k = case getSet o of + Set ms -> do + oldm <- getCurrentMetaData k + let m = combineMetaData $ map (modMeta oldm) ms + addMetaDataClocked k m c + next $ cleanup k + _ -> next $ cleanup k + +cleanup :: Key -> CommandCleanup +cleanup k = do + m <- getCurrentMetaData k + let Object o = toJSON' (MetaDataFields m) + maybeShowJSON $ AesonObject o + showLongNote $ unlines $ concatMap showmeta $ + map unwrapmeta (fromMetaData m) + return True + where + unwrapmeta (f, v) = (fromMetaField f, map fromMetaValue (S.toList v)) + showmeta (f, vs) = map ((f ++ "=") ++) vs + +-- Metadata serialized to JSON in the field named "fields" of +-- a larger object. +newtype MetaDataFields = MetaDataFields MetaData + deriving (Show) + +instance ToJSON' MetaDataFields where + toJSON' (MetaDataFields m) = object [ (fieldsField, toJSON' m) ] + +instance FromJSON MetaDataFields where + parseJSON (Object v) = do + f <- v .: fieldsField + case f of + Nothing -> return (MetaDataFields emptyMetaData) + Just v' -> MetaDataFields <$> parseJSON v' + parseJSON _ = fail "expected an object" + +fieldsField :: T.Text +fieldsField = T.pack "fields" + +parseJSONInput :: String -> Either String (Either FilePath Key, MetaData) +parseJSONInput i = do + v <- eitherDecode (BU.fromString i) + let m = case itemAdded v of + Nothing -> emptyMetaData + Just (MetaDataFields m') -> m' + case (itemKey v, itemFile v) of + (Just k, _) -> Right (Right k, m) + (Nothing, Just f) -> Right (Left f, m) + (Nothing, Nothing) -> Left "JSON input is missing either file or key" + +startBatch :: (Either FilePath Key, MetaData) -> CommandStart +startBatch (i, (MetaData m)) = case i of + Left f -> do + mk <- lookupFile f + case mk of + Just k -> go k (mkActionItem (AssociatedFile (Just f))) + Nothing -> giveup $ "not an annexed file: " ++ f + Right k -> go k (mkActionItem k) + where + go k ai = do + showStartKey "metadata" k ai + let o = MetaDataOptions + { forFiles = [] + , getSet = if MetaData m == emptyMetaData + then GetAll + else Set $ map mkModMeta (M.toList m) + , keyOptions = Nothing + , batchOption = NoBatch + } + t <- liftIO currentVectorClock + -- It would be bad if two batch mode changes used exactly + -- the same timestamp, since the order of adds and removals + -- of the same metadata value would then be indeterminate. + -- To guarantee that never happens, delay 1 microsecond, + -- so the timestamp will always be different. This is + -- probably less expensive than cleaner methods, + -- such as taking from a list of increasing timestamps. + liftIO $ threadDelay 1 + next $ perform t o k + mkModMeta (f, s) + | S.null s = DelMeta f Nothing + | otherwise = SetMeta f s diff --git a/Command/Migrate.hs b/Command/Migrate.hs new file mode 100644 index 0000000000..998a7d8b8b --- /dev/null +++ b/Command/Migrate.hs @@ -0,0 +1,101 @@ +{- git-annex command + - + - Copyright 2011 Joey Hess + - + - Licensed under the GNU GPL version 3 or higher. + -} + +module Command.Migrate where + +import Command +import Backend +import Types.Backend (canUpgradeKey, fastMigrate) +import Types.KeySource +import Annex.Content +import qualified Command.ReKey +import qualified Command.Fsck +import qualified Annex +import Logs.MetaData +import Logs.Web +import qualified Remote + +cmd :: Command +cmd = notDirect $ withGlobalOptions [annexedMatchingOptions] $ + command "migrate" SectionUtility + "switch data to different backend" + paramPaths (withParams seek) + +seek :: CmdParams -> CommandSeek +seek = withFilesInGit (commandAction . (whenAnnexed start)) <=< workTreeItems + +start :: FilePath -> Key -> CommandStart +start file key = do + forced <- Annex.getState Annex.force + v <- Backend.getBackend file key + case v of + Nothing -> stop + Just oldbackend -> do + exists <- inAnnex key + newbackend <- maybe defaultBackend return + =<< chooseBackend file + if (newbackend /= oldbackend || upgradableKey oldbackend key || forced) && exists + then do + showStart "migrate" file + next $ perform file key oldbackend newbackend + else stop + +{- Checks if a key is upgradable to a newer representation. + - + - Reasons for migration: + - - Ideally, all keys have file size metadata. Old keys may not. + - - Something has changed in the backend, such as a bug fix. + -} +upgradableKey :: Backend -> Key -> Bool +upgradableKey backend key = isNothing (keySize key) || backendupgradable + where + backendupgradable = maybe False (\a -> a key) (canUpgradeKey backend) + +{- Store the old backend's key in the new backend + - The old backend's key is not dropped from it, because there may + - be other files still pointing at that key. + - + - To ensure that the data we have for the old key is valid, it's + - fscked here. First we generate the new key. This ensures that the + - data cannot get corrupted after the fsck but before the new key is + - generated. + -} +perform :: FilePath -> Key -> Backend -> Backend -> CommandPerform +perform file oldkey oldbackend newbackend = go =<< genkey (fastMigrate oldbackend) + where + go Nothing = stop + go (Just (newkey, knowngoodcontent)) + | knowngoodcontent = finish newkey + | otherwise = stopUnless checkcontent $ finish newkey + checkcontent = Command.Fsck.checkBackend oldbackend oldkey Command.Fsck.KeyLocked afile + finish newkey = ifM (Command.ReKey.linkKey file oldkey newkey) + ( do + _ <- copyMetaData oldkey newkey + -- If the old key had some associated urls, record them for + -- the new key as well. + urls <- getUrls oldkey + forM_ urls $ \url -> do + r <- Remote.claimingUrl url + setUrlPresent (Remote.uuid r) newkey url + next $ Command.ReKey.cleanup file oldkey newkey + , error "failed" + ) + genkey Nothing = return Nothing + genkey (Just fm) = fm oldkey newbackend afile >>= \case + Just newkey -> return $ Just (newkey, True) + Nothing -> do + content <- calcRepo $ gitAnnexLocation oldkey + let source = KeySource + { keyFilename = file + , contentLocation = content + , inodeCache = Nothing + } + v <- genKey source (Just newbackend) + return $ case v of + Just (newkey, _) -> Just (newkey, False) + _ -> Nothing + afile = AssociatedFile (Just file) diff --git a/Command/Mirror.hs b/Command/Mirror.hs new file mode 100644 index 0000000000..c57dadd951 --- /dev/null +++ b/Command/Mirror.hs @@ -0,0 +1,77 @@ +{- git-annex command + - + - Copyright 2013 Joey Hess + - + - Licensed under the GNU GPL version 3 or higher. + -} + +module Command.Mirror where + +import Command +import qualified Command.Move +import qualified Command.Drop +import qualified Command.Get +import qualified Remote +import Annex.Content +import Annex.NumCopies +import Types.Transfer + +cmd :: Command +cmd = withGlobalOptions [jobsOption, jsonOptions, jsonProgressOption, annexedMatchingOptions] $ + command "mirror" SectionCommon + "mirror content of files to/from another repository" + paramPaths (seek <--< optParser) + +data MirrorOptions = MirrorOptions + { mirrorFiles :: CmdParams + , fromToOptions :: FromToOptions + , keyOptions :: Maybe KeyOptions + } + +optParser :: CmdParamsDesc -> Parser MirrorOptions +optParser desc = MirrorOptions + <$> cmdParams desc + <*> parseFromToOptions + <*> optional (parseKeyOptions <|> parseFailedTransfersOption) + +instance DeferredParseClass MirrorOptions where + finishParse v = MirrorOptions + <$> pure (mirrorFiles v) + <*> finishParse (fromToOptions v) + <*> pure (keyOptions v) + +seek :: MirrorOptions -> CommandSeek +seek o = allowConcurrentOutput $ + withKeyOptions (keyOptions o) False + (commandAction . startKey o (AssociatedFile Nothing)) + (withFilesInGit (commandAction . (whenAnnexed $ start o))) + =<< workTreeItems (mirrorFiles o) + +start :: MirrorOptions -> FilePath -> Key -> CommandStart +start o file k = startKey o afile (k, mkActionItem afile) + where + afile = AssociatedFile (Just file) + +startKey :: MirrorOptions -> AssociatedFile -> (Key, ActionItem) -> CommandStart +startKey o afile (key, ai) = onlyActionOn key $ case fromToOptions o of + ToRemote r -> checkFailedTransferDirection ai Upload $ ifM (inAnnex key) + ( Command.Move.toStart Command.Move.RemoveNever afile key ai =<< getParsed r + , do + numcopies <- getnumcopies + Command.Drop.startRemote afile ai numcopies key =<< getParsed r + ) + FromRemote r -> checkFailedTransferDirection ai Download $ do + haskey <- flip Remote.hasKey key =<< getParsed r + case haskey of + Left _ -> stop + Right True -> Command.Get.start' (return True) Nothing key afile ai + Right False -> ifM (inAnnex key) + ( do + numcopies <- getnumcopies + Command.Drop.startLocal afile ai numcopies key [] + , stop + ) + where + getnumcopies = case afile of + AssociatedFile Nothing -> getNumCopies + AssociatedFile (Just af) -> getFileNumCopies af diff --git a/Command/Move.hs b/Command/Move.hs new file mode 100644 index 0000000000..80afda4e0a --- /dev/null +++ b/Command/Move.hs @@ -0,0 +1,312 @@ +{- git-annex command + - + - Copyright 2010-2018 Joey Hess + - + - Licensed under the GNU GPL version 3 or higher. + -} + +module Command.Move where + +import Command +import qualified Command.Drop +import qualified Annex +import Annex.Content +import qualified Remote +import Annex.UUID +import Annex.Transfer +import Logs.Presence +import Logs.Trust +import Annex.NumCopies + +import System.Log.Logger (debugM) + +cmd :: Command +cmd = withGlobalOptions [jobsOption, jsonOptions, jsonProgressOption, annexedMatchingOptions] $ + command "move" SectionCommon + "move content of files to/from another repository" + paramPaths (seek <--< optParser) + +data MoveOptions = MoveOptions + { moveFiles :: CmdParams + , fromToOptions :: FromToHereOptions [DeferredParse Remote] + , removeWhen :: RemoveWhen + , keyOptions :: Maybe KeyOptions + , batchOption :: BatchMode + } + +optParser :: CmdParamsDesc -> Parser MoveOptions +optParser desc = MoveOptions + <$> cmdParams desc + <*> parseFromToHereOptions + <*> pure RemoveSafe + <*> optional (parseKeyOptions <|> parseFailedTransfersOption) + <*> parseBatchOption + +instance DeferredParseClass MoveOptions where + finishParse v = MoveOptions + <$> pure (moveFiles v) + <*> finishParse (fromToOptions v) + <*> pure (removeWhen v) + <*> pure (keyOptions v) + <*> pure (batchOption v) + +data RemoveWhen = RemoveSafe | RemoveNever + deriving (Show, Eq) + +seek :: MoveOptions -> CommandSeek +seek o = allowConcurrentOutput $ + case batchOption o of + Batch fmt -> batchFilesMatching fmt $ \f -> seekremotes' $ \r -> go r f + NoBatch -> withKeyOptions (keyOptions o) False + (\kai -> seekremotes $ \r -> commandAction $ startKey r (removeWhen o) kai) + (withFilesInGit $ \f -> seekremotes $ \r -> commandAction $ go r f) + =<< workTreeItems (moveFiles o) + where + go r = whenAnnexed $ start r (removeWhen o) + + seekremotes :: (FromToHereOptions Remote -> Annex ()) -> Annex () + seekremotes a = case fromToOptions o of + Right (From rs) -> mapM_ (a . Right . From) =<< mapM getParsed rs + Right (To rs) -> mapM_ (a . Right . To) =<< mapM getParsed rs + Left ToHere -> a $ Left ToHere + seekremotes' = undefined + +start :: FromToHereOptions Remote -> RemoveWhen -> FilePath -> Key -> CommandStart +start fromto removewhen f k = + start' fromto removewhen afile k (mkActionItem afile) + where + afile = AssociatedFile (Just f) + +startKey :: FromToHereOptions Remote -> RemoveWhen -> (Key, ActionItem) -> CommandStart +startKey fromto removewhen = + uncurry $ start' fromto removewhen (AssociatedFile Nothing) + +start' :: FromToHereOptions Remote -> RemoveWhen -> AssociatedFile -> Key -> ActionItem -> CommandStart +start' fromto removewhen afile key ai = onlyActionOn key $ + case fromto of + Right (From src) -> + checkFailedTransferDirection ai Download $ + fromStart removewhen afile key ai src + Right (To dest) -> + checkFailedTransferDirection ai Upload $ + toStart removewhen afile key ai dest + Left ToHere -> + checkFailedTransferDirection ai Download $ + toHereStart removewhen afile key ai + +showMoveAction :: RemoveWhen -> Key -> ActionItem -> Annex () +showMoveAction RemoveNever = showStartKey "copy" +showMoveAction _ = showStartKey "move" + +toStart :: RemoveWhen -> AssociatedFile -> Key -> ActionItem -> Remote -> CommandStart +toStart removewhen afile key ai dest = do + u <- getUUID + ishere <- inAnnex key + if not ishere || u == Remote.uuid dest + then stop -- not here, so nothing to do + else toStart' dest removewhen afile key ai + +toStart' :: Remote -> RemoveWhen -> AssociatedFile -> Key -> ActionItem -> CommandStart +toStart' dest removewhen afile key ai = do + fast <- Annex.getState Annex.fast + if fast && removewhen == RemoveNever + then ifM (expectedPresent dest key) + ( stop + , go True (pure $ Right False) + ) + else go False (Remote.hasKey dest key) + where + go fastcheck isthere = do + showMoveAction removewhen key ai + next $ toPerform dest removewhen key afile fastcheck =<< isthere + +expectedPresent :: Remote -> Key -> Annex Bool +expectedPresent dest key = do + remotes <- Remote.keyPossibilities key + return $ dest `elem` remotes + +toPerform :: Remote -> RemoveWhen -> Key -> AssociatedFile -> Bool -> Either String Bool -> CommandPerform +toPerform dest removewhen key afile fastcheck isthere = + case isthere of + Left err -> do + showNote err + stop + Right False -> do + showAction $ "to " ++ Remote.name dest + ok <- notifyTransfer Upload afile $ + upload (Remote.uuid dest) key afile stdRetry $ + Remote.storeKey dest key afile + if ok + then finish False $ + Remote.logStatus dest key InfoPresent + else do + when fastcheck $ + warning "This could have failed because --fast is enabled." + stop + Right True -> finish True $ + unlessM (expectedPresent dest key) $ + Remote.logStatus dest key InfoPresent + where + finish deststartedwithcopy setpresentremote = case removewhen of + RemoveNever -> do + setpresentremote + next $ return True + RemoveSafe -> lockContentForRemoval key $ \contentlock -> do + srcuuid <- getUUID + let destuuid = Remote.uuid dest + willDropMakeItWorse srcuuid destuuid deststartedwithcopy key afile >>= \case + DropAllowed -> drophere setpresentremote contentlock "moved" + DropCheckNumCopies -> do + numcopies <- getAssociatedFileNumCopies afile + (tocheck, verified) <- verifiableCopies key [srcuuid] + verifyEnoughCopiesToDrop "" key (Just contentlock) + numcopies [srcuuid] verified + (UnVerifiedRemote dest : tocheck) + (drophere setpresentremote contentlock . showproof) + (faileddrophere setpresentremote) + DropWorse -> faileddrophere setpresentremote + showproof proof = "proof: " ++ show proof + drophere setpresentremote contentlock reason = do + liftIO $ debugM "move" $ unwords + [ "Dropping from here" + , "(" ++ reason ++ ")" + ] + -- Drop content before updating location logs, + -- in case disk space is very low this frees + -- up space before writing data to disk. + removeAnnex contentlock + next $ do + () <- setpresentremote + Command.Drop.cleanupLocal key + faileddrophere setpresentremote = do + showLongNote "(Use --force to override this check, or adjust numcopies.)" + showLongNote "Content not dropped from here." + next $ do + () <- setpresentremote + return False + +fromStart :: RemoveWhen -> AssociatedFile -> Key -> ActionItem -> Remote -> CommandStart +fromStart removewhen afile key ai src = case removewhen of + RemoveNever -> stopUnless (not <$> inAnnex key) go + RemoveSafe -> go + where + go = stopUnless (fromOk src key) $ do + showMoveAction removewhen key ai + next $ fromPerform src removewhen key afile + +fromOk :: Remote -> Key -> Annex Bool +fromOk src key + | Remote.hasKeyCheap src = + either (const checklog) return =<< haskey + | otherwise = checklog + where + haskey = Remote.hasKey src key + checklog = do + u <- getUUID + remotes <- Remote.keyPossibilities key + return $ u /= Remote.uuid src && elem src remotes + +fromPerform :: Remote -> RemoveWhen -> Key -> AssociatedFile -> CommandPerform +fromPerform src removewhen key afile = do + showAction $ "from " ++ Remote.name src + ifM (inAnnex key) + ( dispatch removewhen True True + , dispatch removewhen False =<< go + ) + where + go = notifyTransfer Download afile $ + download (Remote.uuid src) key afile stdRetry $ \p -> + getViaTmp (Remote.retrievalSecurityPolicy src) (RemoteVerify src) key $ \t -> + Remote.retrieveKeyFile src key afile t p + dispatch _ _ False = stop -- failed + dispatch RemoveNever _ True = next $ return True -- copy complete + dispatch RemoveSafe deststartedwithcopy True = lockContentShared key $ \_lck -> do + let srcuuid = Remote.uuid src + destuuid <- getUUID + willDropMakeItWorse srcuuid destuuid deststartedwithcopy key afile >>= \case + DropAllowed -> dropremote "moved" + DropCheckNumCopies -> do + numcopies <- getAssociatedFileNumCopies afile + (tocheck, verified) <- verifiableCopies key [Remote.uuid src] + verifyEnoughCopiesToDrop "" key Nothing numcopies [Remote.uuid src] verified + tocheck (dropremote . showproof) faileddropremote + DropWorse -> faileddropremote + showproof proof = "proof: " ++ show proof + dropremote reason = do + liftIO $ debugM "move" $ unwords + [ "Dropping from remote" + , show src + , "(" ++ reason ++ ")" + ] + ok <- Remote.removeKey src key + next $ Command.Drop.cleanupRemote key src ok + faileddropremote = do + showLongNote "(Use --force to override this check, or adjust numcopies.)" + showLongNote $ "Content not dropped from " ++ Remote.name src ++ "." + next $ return False + +{- Moves (or copies) the content of an annexed file from reachable remotes + - to the current repository. + - + - When moving, the content is removed from all the reachable remotes that + - it can safely be removed from. -} +toHereStart :: RemoveWhen -> AssociatedFile -> Key -> ActionItem -> CommandStart +toHereStart removewhen afile key ai = case removewhen of + RemoveNever -> stopUnless (not <$> inAnnex key) go + RemoveSafe -> go + where + go = do + rs <- Remote.keyPossibilities key + forM_ rs $ \r -> + includeCommandAction $ do + showMoveAction removewhen key ai + next $ fromPerform r removewhen key afile + stop + +{- The goal of this command is to allow the user maximum freedom to move + - files as they like, while avoiding making bad situations any worse + - than they already were. + - + - When the destination repository already had a copy of a file + - before the move operation began, dropping it from the source + - repository reduces the number of copies, and should fail if + - that would violate numcopies settings. + - + - On the other hand, when the destiation repository does not already + - have a copy of a file, it can be dropped without making numcopies + - worse, so the move is allowed even if numcopies is not met. + - + - Similarly, a file can move from an untrusted repository to another + - untrusted repository, even if that is the only copy of the file. + - + - But, moving a file from a repository with higher trust to an untrusted + - repository must still check that there are enough other copies to be + - safe. + - + - Also, required content settings should not be violated. + - + - This function checks all that. It needs to know if the destination + - repository already had a copy of the file before the move began. + -} +willDropMakeItWorse :: UUID -> UUID -> Bool -> Key -> AssociatedFile -> Annex DropCheck +willDropMakeItWorse srcuuid destuuid deststartedwithcopy key afile = + ifM (Command.Drop.checkRequiredContent srcuuid key afile) + ( if deststartedwithcopy + then unlessforced DropCheckNumCopies + else ifM checktrustlevel + ( return DropAllowed + , unlessforced DropCheckNumCopies + ) + , unlessforced DropWorse + ) + where + unlessforced r = ifM (Annex.getState Annex.force) + ( return DropAllowed + , return r + ) + checktrustlevel = do + desttrust <- lookupTrust destuuid + srctrust <- lookupTrust srcuuid + return (desttrust > UnTrusted || desttrust >= srctrust) + +data DropCheck = DropWorse | DropAllowed | DropCheckNumCopies diff --git a/Command/Multicast.hs b/Command/Multicast.hs new file mode 100644 index 0000000000..5c853ddd92 --- /dev/null +++ b/Command/Multicast.hs @@ -0,0 +1,255 @@ +{- git-annex command + - + - Copyright 2017 Joey Hess + - + - Licensed under the GNU GPL version 3 or higher. + -} + +{-# LANGUAGE CPP #-} + +module Command.Multicast where + +import Command +import Logs.Multicast +import Annex.Multicast +import Annex.WorkTree +import Annex.Content +import Annex.UUID +import Annex.Perms +import Utility.FileMode +#ifndef mingw32_HOST_OS +import Creds +#endif +import qualified Limit +import Types.FileMatcher +import qualified Git.LsFiles as LsFiles +import Utility.Hash +import Utility.Tmp +import Utility.Tmp.Dir +import Utility.Process.Transcript +import Config + +import Data.Char +import qualified Data.ByteString.Lazy.UTF8 as B8 +import qualified Data.Map as M +import Control.Concurrent.Async + +cmd :: Command +cmd = command "multicast" SectionCommon "multicast file distribution" + paramNothing (seek <$$> optParser) + +data MultiCastAction + = GenAddress + | Send + | Receive + deriving (Show) + +data MultiCastOptions = MultiCastOptions MultiCastAction [CommandParam] [FilePath] + deriving (Show) + +optParser :: CmdParamsDesc -> Parser MultiCastOptions +optParser _ = MultiCastOptions + <$> (genaddressp <|> sendp <|> receivep) + <*> many uftpopt + <*> cmdParams paramPaths + where + genaddressp = flag' GenAddress + ( long "gen-address" + <> help "generate multicast encryption key and store address in git-annex branch" + ) + sendp = flag' Send + ( long "send" + <> help "multicast files" + ) + receivep = flag' Receive + ( long "receive" + <> help "listen for multicast files and store in repository" + ) + uftpopt = Param <$> strOption + ( long "uftp-opt" + <> short 'U' + <> help "passed on to uftp/uftpd" + <> metavar "OPTION" + ) + +seek :: MultiCastOptions -> CommandSeek +seek (MultiCastOptions GenAddress _ _) = commandAction genAddress +seek (MultiCastOptions Send ups fs) = commandAction $ send ups fs +seek (MultiCastOptions Receive ups []) = commandAction $ receive ups +seek (MultiCastOptions Receive _ _) = giveup "Cannot specify list of files with --receive; this receives whatever files the sender chooses to send." + +genAddress :: CommandStart +genAddress = do + showStart' "gen-address" Nothing + k <- uftpKey + (s, ok) <- case k of + KeyContainer s -> liftIO $ genkey (Param s) + KeyFile f -> do + createAnnexDirectory (takeDirectory f) + liftIO $ nukeFile f + liftIO $ protectedOutput $ genkey (File f) + case (ok, parseFingerprint s) of + (False, _) -> giveup $ "uftp_keymgt failed: " ++ s + (_, Nothing) -> giveup $ "Failed to find fingerprint in uftp_keymgt output: " ++ s + (True, Just fp) -> next $ next $ do + recordFingerprint fp =<< getUUID + return True + where + -- Annoyingly, the fingerprint is output to stderr. + genkey p = processTranscript "uftp_keymgt" ps Nothing + where + ps = toCommand $ + [ Param "-g" + , keyparam + , p + ] + -- uftp only supports rsa up to 2048 which is on the lower + -- limit of secure RSA key sizes. Instead, use an EC curve. + -- Except for on Windows XP, secp521r1 is supported on all + -- platforms by uftp. DJB thinks it's pretty good compared + -- with other NIST curves: "there's one standard NIST curve + -- using a nice prime, namely 2521−1 but the sheer size of this + -- prime makes it much slower than NIST P-256" + -- (http://blog.cr.yp.to/20140323-ecdsa.html) + -- Since this key is only used to set up the block encryption, + -- its slow speed is ok. + keyparam = Param "ec:secp521r1" + +parseFingerprint :: String -> Maybe Fingerprint +parseFingerprint = Fingerprint <$$> lastMaybe . filter isfingerprint . words + where + isfingerprint s = + let os = filter (all isHexDigit) (splitc ':' s) + in length os == 20 + +send :: [CommandParam] -> [FilePath] -> CommandStart +send ups fs = withTmpFile "send" $ \t h -> do + -- Need to be able to send files with the names of git-annex + -- keys, and uftp does not allow renaming the files that are sent. + -- In a direct mode repository, the annex objects do not have + -- the names of keys, and would have to be copied, which is too + -- expensive. + whenM isDirect $ + giveup "Sorry, multicast send cannot be done from a direct mode repository." + + showStart' "generating file list" Nothing + fs' <- seekHelper LsFiles.inRepo =<< workTreeItems fs + matcher <- Limit.getMatcher + let addlist f o = whenM (matcher $ MatchingFile $ FileInfo f f) $ + liftIO $ hPutStrLn h o + forM_ fs' $ \f -> do + mk <- lookupFile f + case mk of + Nothing -> noop + Just k -> withObjectLoc k (addlist f) (const noop) + liftIO $ hClose h + showEndOk + + showStart' "sending files" Nothing + showOutput + serverkey <- uftpKey + u <- getUUID + withAuthList $ \authlist -> do + let ps = + -- Force client authentication. + [ Param "-c" + , Param "-Y", Param "aes256-cbc" + , Param "-h", Param "sha512" + -- Picked ecdh_ecdsa for perfect forward secrecy, + -- and because a EC key exchange algorithm is + -- needed since all keys are EC. + , Param "-e", Param "ecdh_ecdsa" + , Param "-k", uftpKeyParam serverkey + , Param "-U", Param (uftpUID u) + -- only allow clients on the authlist + , Param "-H", Param ("@"++authlist) + -- pass in list of files to send + , Param "-i", File t + ] ++ ups + liftIO (boolSystem "uftp" ps) >>= showEndResult + stop + +receive :: [CommandParam] -> CommandStart +receive ups = do + showStart' "receiving multicast files" Nothing + showNote "Will continue to run until stopped by ctrl-c" + + showOutput + clientkey <- uftpKey + u <- getUUID + (callback, environ, statush) <- liftIO multicastCallbackEnv + tmpobjdir <- fromRepo gitAnnexTmpObjectDir + createAnnexDirectory tmpobjdir + withTmpDirIn tmpobjdir "multicast" $ \tmpdir -> withAuthList $ \authlist -> do + abstmpdir <- liftIO $ absPath tmpdir + abscallback <- liftIO $ searchPath callback + let ps = + -- Avoid it running as a daemon. + [ Param "-d" + -- Require encryption. + , Param "-E" + , Param "-k", uftpKeyParam clientkey + , Param "-U", Param (uftpUID u) + -- Only allow servers on the authlist + , Param "-S", Param authlist + -- Receive files into tmpdir + -- (it needs an absolute path) + , Param "-D", File abstmpdir + -- Run callback after each file received + -- (it needs an absolute path) + , Param "-s", Param (fromMaybe callback abscallback) + ] ++ ups + runner <- liftIO $ async $ + hClose statush + `after` boolSystemEnv "uftpd" ps (Just environ) + mapM_ storeReceived . lines =<< liftIO (hGetContents statush) + showEndResult =<< liftIO (wait runner) + stop + +storeReceived :: FilePath -> Annex () +storeReceived f = do + case file2key (takeFileName f) of + Nothing -> do + warning $ "Received a file " ++ f ++ " that is not a git-annex key. Deleting this file." + liftIO $ nukeFile f + Just k -> void $ + getViaTmpFromDisk RetrievalVerifiableKeysSecure AlwaysVerify k $ \dest -> unVerified $ + liftIO $ catchBoolIO $ do + rename f dest + return True + +-- Under Windows, uftp uses key containers, which are not files on the +-- filesystem. +data UftpKey = KeyFile FilePath | KeyContainer String + +uftpKeyParam :: UftpKey -> CommandParam +uftpKeyParam (KeyFile f) = File f +uftpKeyParam (KeyContainer s) = Param s + +uftpKey :: Annex UftpKey +#ifdef mingw32_HOST_OS +uftpKey = do + u <- getUUID + return $ KeyContainer $ "annex-" ++ fromUUID u +#else +uftpKey = KeyFile <$> cacheCredsFile "multicast" +#endif + +-- uftp needs a unique UID for each client and server, which +-- is a 8 digit hex number in the form "0xnnnnnnnn" +-- Derive it from the UUID. +uftpUID :: UUID -> String +uftpUID u = "0x" ++ (take 8 $ show $ sha2_256 $ B8.fromString (fromUUID u)) + +withAuthList :: (FilePath -> Annex a) -> Annex a +withAuthList a = do + m <- knownFingerPrints + withTmpFile "authlist" $ \t h -> do + liftIO $ hPutStr h (genAuthList m) + liftIO $ hClose h + a t + +genAuthList :: M.Map UUID Fingerprint -> String +genAuthList = unlines . map fmt . M.toList + where + fmt (u, Fingerprint f) = uftpUID u ++ "|" ++ f diff --git a/Command/NotifyChanges.hs b/Command/NotifyChanges.hs new file mode 100644 index 0000000000..d6a713fb0f --- /dev/null +++ b/Command/NotifyChanges.hs @@ -0,0 +1,44 @@ +{- git-annex-shell command + - + - Copyright 2014 Joey Hess + - + - Licensed under the GNU GPL version 3 or higher. + -} + +module Command.NotifyChanges where + +import Command +import Annex.ChangedRefs +import RemoteDaemon.Transport.Ssh.Types +import Utility.SimpleProtocol + +import Control.Concurrent.Async + +cmd :: Command +cmd = noCommit $ + command "notifychanges" SectionPlumbing + "sends notification when git refs are changed" + paramNothing (withParams seek) + +seek :: CmdParams -> CommandSeek +seek = withNothing (commandAction start) + +start :: CommandStart +start = go =<< watchChangedRefs + where + go (Just h) = do + -- No messages need to be received from the caller, + -- but when it closes the connection, notice and terminate. + let receiver = forever $ void $ getProtocolLine stdin + let sender = forever $ send . CHANGED =<< waitChangedRefs h + + liftIO $ send READY + void $ liftIO $ concurrently sender receiver + liftIO $ stopWatchingChangedRefs h + stop + go Nothing = stop + +send :: Notification -> IO () +send n = do + putStrLn $ unwords $ formatMessage n + hFlush stdout diff --git a/Command/NumCopies.hs b/Command/NumCopies.hs new file mode 100644 index 0000000000..83fe4f88c0 --- /dev/null +++ b/Command/NumCopies.hs @@ -0,0 +1,54 @@ +{- git-annex command + - + - Copyright 2014 Joey Hess + - + - Licensed under the GNU GPL version 3 or higher. + -} + +module Command.NumCopies where + +import Command +import qualified Annex +import Annex.NumCopies + +cmd :: Command +cmd = noMessages $ command "numcopies" SectionSetup + "configure desired number of copies" + paramNumber (withParams seek) + +seek :: CmdParams -> CommandSeek +seek = withWords (commandAction . start) + +start :: [String] -> CommandStart +start [] = startGet +start [s] = case readish s of + Nothing -> giveup $ "Bad number: " ++ s + Just n + | n > 0 -> startSet n + | n == 0 -> ifM (Annex.getState Annex.force) + ( startSet n + , giveup "Setting numcopies to 0 is very unsafe. You will lose data! If you really want to do that, specify --force." + ) + | otherwise -> giveup "Number cannot be negative!" +start _ = giveup "Specify a single number." + +startGet :: CommandStart +startGet = next $ next $ do + v <- getGlobalNumCopies + case v of + Just n -> liftIO $ putStrLn $ show $ fromNumCopies n + Nothing -> do + liftIO $ putStrLn "global numcopies is not set" + old <- deprecatedNumCopies + case old of + Nothing -> liftIO $ putStrLn "(default is 1)" + Just n -> liftIO $ putStrLn $ "(deprecated git config annex.numcopies is set to " ++ show (fromNumCopies n) ++ " locally)" + return True + +startSet :: Int -> CommandStart +startSet n = do + allowMessages + showStart' "numcopies" (Just $ show n) + next $ next $ do + setGlobalNumCopies $ NumCopies n + return True diff --git a/Command/P2P.hs b/Command/P2P.hs new file mode 100644 index 0000000000..f1a18fa4d8 --- /dev/null +++ b/Command/P2P.hs @@ -0,0 +1,327 @@ +{- git-annex command + - + - Copyright 2016 Joey Hess + - + - Licensed under the GNU GPL version 3 or higher. + -} + +module Command.P2P where + +import Command +import P2P.Address +import P2P.Auth +import P2P.IO +import qualified P2P.Protocol as P2P +import Git.Types +import qualified Git.Remote +import qualified Git.Command +import qualified Annex +import Annex.UUID +import Config +import Utility.AuthToken +import Utility.Tmp.Dir +import Utility.FileMode +import Utility.ThreadScheduler +import qualified Utility.MagicWormhole as Wormhole + +import Control.Concurrent.Async +import qualified Data.Text as T +import Data.Time.Clock.POSIX + +cmd :: Command +cmd = command "p2p" SectionSetup + "configure peer-2-peer links between repositories" + paramNothing (seek <$$> optParser) + +data P2POpts + = GenAddresses + | LinkRemote + | Pair + +optParser :: CmdParamsDesc -> Parser (P2POpts, Maybe RemoteName) +optParser _ = (,) + <$> (pair <|> linkremote <|> genaddresses) + <*> optional name + where + genaddresses = flag' GenAddresses + ( long "gen-addresses" + <> help "generate addresses that allow accessing this repository over P2P networks" + ) + linkremote = flag' LinkRemote + ( long "link" + <> help "set up a P2P link to a git remote" + ) + pair = flag' Pair + ( long "pair" + <> help "pair with another repository" + ) + name = Git.Remote.makeLegalName <$> strOption + ( long "name" + <> metavar paramName + <> help "name of remote" + ) + +seek :: (P2POpts, Maybe RemoteName) -> CommandSeek +seek (GenAddresses, _) = genAddresses =<< loadP2PAddresses +seek (LinkRemote, Just name) = commandAction $ + linkRemote name +seek (LinkRemote, Nothing) = commandAction $ + linkRemote =<< unusedPeerRemoteName +seek (Pair, Just name) = commandAction $ + startPairing name =<< loadP2PAddresses +seek (Pair, Nothing) = commandAction $ do + name <- unusedPeerRemoteName + startPairing name =<< loadP2PAddresses + +unusedPeerRemoteName :: Annex RemoteName +unusedPeerRemoteName = go (1 :: Integer) =<< usednames + where + usednames = mapMaybe remoteName <$> Annex.getGitRemotes + go n names = do + let name = "peer" ++ show n + if name `elem` names + then go (n+1) names + else return name + +-- Only addresses are output to stdout, to allow scripting. +genAddresses :: [P2PAddress] -> Annex () +genAddresses [] = giveup "No P2P networks are currrently available." +genAddresses addrs = do + authtoken <- liftIO $ genAuthToken 128 + storeP2PAuthToken authtoken + earlyWarning "These addresses allow access to this git-annex repository. Only share them with people you trust with that access, using trusted communication channels!" + liftIO $ putStr $ unlines $ + map formatP2PAddress $ + map (`P2PAddressAuth` authtoken) addrs + +-- Address is read from stdin, to avoid leaking it in shell history. +linkRemote :: RemoteName -> CommandStart +linkRemote remotename = do + showStart' "p2p link" (Just remotename) + next $ next promptaddr + where + promptaddr = do + liftIO $ putStrLn "" + liftIO $ putStr "Enter peer address: " + liftIO $ hFlush stdout + s <- liftIO getLine + if null s + then do + liftIO $ hPutStrLn stderr "Nothing entered, giving up." + return False + else case unformatP2PAddress s of + Nothing -> do + liftIO $ hPutStrLn stderr "Unable to parse that address, please check its format and try again." + promptaddr + Just addr -> do + r <- setupLink remotename addr + case r of + LinkSuccess -> return True + ConnectionError e -> giveup e + AuthenticationError e -> giveup e + +startPairing :: RemoteName -> [P2PAddress] -> CommandStart +startPairing _ [] = giveup "No P2P networks are currrently available." +startPairing remotename addrs = do + showStart' "p2p pair" (Just remotename) + ifM (liftIO Wormhole.isInstalled) + ( next $ performPairing remotename addrs + , giveup "Magic Wormhole is not installed, and is needed for pairing. Install it from your distribution or from https://github.com/warner/magic-wormhole/" + ) + +performPairing :: RemoteName -> [P2PAddress] -> CommandPerform +performPairing remotename addrs = do + -- This note is displayed mainly so when magic wormhole + -- complains about possible protocol mismatches or other problems, + -- it's clear what's doing the complaining. + showNote "using Magic Wormhole" + next $ do + showOutput + r <- wormholePairing remotename addrs ui + case r of + PairSuccess -> return True + SendFailed -> do + warning "Failed sending data to pair." + return False + ReceiveFailed -> do + warning "Failed receiving data from pair." + return False + LinkFailed e -> do + warning $ "Failed linking to pair: " ++ e + return False + where + ui observer producer = do + ourcode <- Wormhole.waitCode observer + putStrLn "" + putStrLn $ "This repository's pairing code is: " ++ + Wormhole.fromCode ourcode + putStrLn "" + theircode <- getcode ourcode + Wormhole.sendCode producer theircode + + getcode ourcode = do + putStr "Enter the other repository's pairing code: " + hFlush stdout + l <- getLine + case Wormhole.toCode l of + Just code + | code /= ourcode -> do + putStrLn "Exchanging pairing data..." + return code + | otherwise -> do + putStrLn "Oops -- You entered this repository's pairing code. Enter the pairing code of the *other* repository." + getcode ourcode + Nothing -> do + putStrLn "That does not look like a valiad pairing code. Try again..." + getcode ourcode + +-- We generate half of the authtoken; the pair will provide +-- the other half. +newtype HalfAuthToken = HalfAuthToken T.Text + deriving (Show) + +data PairData = PairData HalfAuthToken [P2PAddress] + deriving (Show) + +serializePairData :: PairData -> String +serializePairData (PairData (HalfAuthToken ha) addrs) = unlines $ + T.unpack ha : map formatP2PAddress addrs + +deserializePairData :: String -> Maybe PairData +deserializePairData s = case lines s of + [] -> Nothing + (ha:l) -> do + addrs <- mapM unformatP2PAddress l + return (PairData (HalfAuthToken (T.pack ha)) addrs) + +data PairingResult + = PairSuccess + | SendFailed + | ReceiveFailed + | LinkFailed String + +wormholePairing + :: RemoteName + -> [P2PAddress] + -> (Wormhole.CodeObserver -> Wormhole.CodeProducer -> IO ()) + -> Annex PairingResult +wormholePairing remotename ouraddrs ui = do + ourhalf <- liftIO $ HalfAuthToken . fromAuthToken + <$> genAuthToken 64 + let ourpairdata = PairData ourhalf ouraddrs + + -- The magic wormhole interface only supports exchanging + -- files. Permissions of received files may allow others + -- to read them. So, set up a temp directory that only + -- we can read. + withTmpDir "pair" $ \tmp -> do + liftIO $ void $ tryIO $ modifyFileMode tmp $ + removeModes otherGroupModes + let sendf = tmp "send" + let recvf = tmp "recv" + liftIO $ writeFileProtected sendf $ + serializePairData ourpairdata + + observer <- liftIO Wormhole.mkCodeObserver + producer <- liftIO Wormhole.mkCodeProducer + void $ liftIO $ async $ ui observer producer + -- Provide an appid to magic wormhole, to avoid using + -- the same channels that other wormhole users use. + -- + -- Since a version of git-annex that did not provide an + -- appid is shipping in Debian 9, and having one side + -- provide an appid while the other does not will make + -- wormhole fail, this is deferred until 2021-12-31. + -- After that point, all git-annex's should have been + -- upgraded to include this code, and they will start + -- providing an appid. + -- + -- This assumes reasonably good client clocks. If the clock + -- is completely wrong, it won't use the appid at that + -- point, and pairing will fail. On 2021-12-31, minor clock + -- skew may also cause transient problems. + -- + -- After 2021-12-31, this can be changed to simply + -- always provide the appid. + now <- liftIO getPOSIXTime + let wormholeparams = if now < 1640950000 + then [] + else Wormhole.appId "git-annex.branchable.com/p2p-setup" + (sendres, recvres) <- liftIO $ + Wormhole.sendFile sendf observer wormholeparams + `concurrently` + Wormhole.receiveFile recvf producer wormholeparams + liftIO $ nukeFile sendf + if sendres /= True + then return SendFailed + else if recvres /= True + then return ReceiveFailed + else do + r <- liftIO $ tryIO $ + readFileStrict recvf + case r of + Left _e -> return ReceiveFailed + Right s -> maybe + (return ReceiveFailed) + (finishPairing 100 remotename ourhalf) + (deserializePairData s) + +-- | Allow the peer we're pairing with to authenticate to us, +-- using an authtoken constructed from the two HalfAuthTokens. +-- Connect to the peer we're pairing with, and try to link to them. +-- +-- Multiple addresses may have been received for the peer. This only +-- makes a link to one address. +-- +-- Since we're racing the peer as they do the same, the first try is likely +-- to fail to authenticate. Can retry any number of times, to avoid the +-- users needing to redo the whole process. +finishPairing :: Int -> RemoteName -> HalfAuthToken -> PairData -> Annex PairingResult +finishPairing retries remotename (HalfAuthToken ourhalf) (PairData (HalfAuthToken theirhalf) theiraddrs) = do + case (toAuthToken (ourhalf <> theirhalf), toAuthToken (theirhalf <> ourhalf)) of + (Just ourauthtoken, Just theirauthtoken) -> do + liftIO $ putStrLn $ "Successfully exchanged pairing data. Connecting to " ++ remotename ++ "..." + storeP2PAuthToken ourauthtoken + go retries theiraddrs theirauthtoken + _ -> return ReceiveFailed + where + go 0 [] _ = return $ LinkFailed $ "Unable to connect to " ++ remotename ++ "." + go n [] theirauthtoken = do + liftIO $ threadDelaySeconds (Seconds 2) + liftIO $ putStrLn $ "Unable to connect to " ++ remotename ++ ". Retrying..." + go (n-1) theiraddrs theirauthtoken + go n (addr:rest) theirauthtoken = do + r <- setupLink remotename (P2PAddressAuth addr theirauthtoken) + case r of + LinkSuccess -> return PairSuccess + _ -> go n rest theirauthtoken + +data LinkResult + = LinkSuccess + | ConnectionError String + | AuthenticationError String + +setupLink :: RemoteName -> P2PAddressAuth -> Annex LinkResult +setupLink remotename (P2PAddressAuth addr authtoken) = do + g <- Annex.gitRepo + cv <- liftIO $ tryNonAsync $ connectPeer g addr + case cv of + Left e -> return $ ConnectionError $ "Unable to connect with peer. Please check that the peer is connected to the network, and try again. (" ++ show e ++ ")" + Right conn -> do + u <- getUUID + let proto = P2P.auth u authtoken noop + runst <- liftIO $ mkRunState Client + go =<< liftIO (runNetProto runst conn proto) + where + go (Right (Just theiruuid)) = do + ok <- inRepo $ Git.Command.runBool + [ Param "remote", Param "add" + , Param remotename + , Param (formatP2PAddress addr) + ] + when ok $ do + storeUUIDIn (remoteConfig remotename "uuid") theiruuid + storeP2PRemoteAuthToken addr authtoken + return LinkSuccess + go (Right Nothing) = return $ AuthenticationError "Unable to authenticate with peer. Please check the address and try again." + go (Left e) = return $ AuthenticationError $ "Unable to authenticate with peer: " ++ describeProtoFailure e diff --git a/Command/P2PStdIO.hs b/Command/P2PStdIO.hs new file mode 100644 index 0000000000..8b3f05d18e --- /dev/null +++ b/Command/P2PStdIO.hs @@ -0,0 +1,50 @@ +{- git-annex command + - + - Copyright 2018 Joey Hess + - + - Licensed under the GNU GPL version 3 or higher. + -} + +module Command.P2PStdIO where + +import Command +import P2P.IO +import P2P.Annex +import qualified P2P.Protocol as P2P +import qualified Annex +import Annex.UUID +import qualified CmdLine.GitAnnexShell.Checks as Checks + +import System.IO.Error + +cmd :: Command +cmd = noMessages $ command "p2pstdio" SectionPlumbing + "communicate in P2P protocol over stdio" + paramUUID (withParams seek) + +seek :: CmdParams -> CommandSeek +seek [u] = commandAction $ start $ toUUID u +seek _ = giveup "missing UUID parameter" + +start :: UUID -> CommandStart +start theiruuid = do + servermode <- liftIO $ do + ro <- Checks.checkEnvSet Checks.readOnlyEnv + ao <- Checks.checkEnvSet Checks.appendOnlyEnv + return $ case (ro, ao) of + (True, _) -> P2P.ServeReadOnly + (False, True) -> P2P.ServeAppendOnly + (False, False) -> P2P.ServeReadWrite + myuuid <- getUUID + conn <- stdioP2PConnection <$> Annex.gitRepo + let server = do + P2P.net $ P2P.sendMessage (P2P.AUTH_SUCCESS myuuid) + P2P.serveAuthed servermode myuuid + runst <- liftIO $ mkRunState $ Serving theiruuid Nothing + runFullProto runst conn server >>= \case + Right () -> done + -- Avoid displaying an error when the client hung up on us. + Left (ProtoFailureIOError e) | isEOFError e -> done + Left e -> giveup (describeProtoFailure e) + where + done = next $ next $ return True diff --git a/Command/PostReceive.hs b/Command/PostReceive.hs new file mode 100644 index 0000000000..4db7752148 --- /dev/null +++ b/Command/PostReceive.hs @@ -0,0 +1,51 @@ +{- git-annex command + - + - Copyright 2017 Joey Hess + - + - Licensed under the GNU GPL version 3 or higher. + -} + +module Command.PostReceive where + +import Command +import qualified Annex +import Git.Types +import Annex.UpdateInstead +import Command.Sync (mergeLocal, prepMerge, mergeConfig, getCurrBranch) + +-- This does not need to modify the git-annex branch to update the +-- work tree, but auto-initialization might change the git-annex branch. +-- Since it would be surprising for a post-receive hook to make such a +-- change, that's prevented by noCommit. +cmd :: Command +cmd = noCommit $ + command "post-receive" SectionPlumbing + "run by git post-receive hook" + paramNothing + (withParams seek) + +seek :: CmdParams -> CommandSeek +seek _ = whenM needUpdateInsteadEmulation $ do + fixPostReceiveHookEnv + commandAction updateInsteadEmulation + +{- When run by the post-receive hook, the cwd is the .git directory, + - and GIT_DIR=. It's not clear why git does this. + - + - Fix up from that unusual situation, so that git commands + - won't try to treat .git as the work tree. -} +fixPostReceiveHookEnv :: Annex () +fixPostReceiveHookEnv = do + g <- Annex.gitRepo + case location g of + Local { gitdir = ".", worktree = Just "." } -> + Annex.adjustGitRepo $ \g' -> pure $ g' + { location = (location g') + { worktree = Just ".." } + } + _ -> noop + +updateInsteadEmulation :: CommandStart +updateInsteadEmulation = do + prepMerge + mergeLocal mergeConfig def =<< join getCurrBranch diff --git a/Command/PreCommit.hs b/Command/PreCommit.hs new file mode 100644 index 0000000000..1222ba523e --- /dev/null +++ b/Command/PreCommit.hs @@ -0,0 +1,110 @@ +{- git-annex command + - + - Copyright 2010-2014 Joey Hess + - + - Licensed under the GNU GPL version 3 or higher. + -} + +{-# LANGUAGE CPP #-} + +module Command.PreCommit where + +import Command +import Config +import qualified Command.Add +import qualified Command.Fix +import Annex.Direct +import Annex.Hook +import Annex.Link +import Annex.View +import Annex.Version +import Annex.View.ViewedFile +import Annex.LockFile +import Logs.View +import Logs.MetaData +import Types.View +import Types.MetaData +import qualified Git.Index as Git +import qualified Git.LsFiles as Git + +import qualified Data.Set as S + +cmd :: Command +cmd = command "pre-commit" SectionPlumbing + "run by git pre-commit hook" + paramPaths + (withParams seek) + +seek :: CmdParams -> CommandSeek +seek ps = lockPreCommitHook $ ifM isDirect + ( do + -- update direct mode mappings for committed files + withWords (commandAction . startDirect) ps + runAnnexHook preCommitAnnexHook + , do + ifM (not <$> versionSupportsUnlockedPointers <&&> liftIO Git.haveFalseIndex) + ( do + (fs, cleanup) <- inRepo $ Git.typeChangedStaged ps + whenM (anyM isOldUnlocked fs) $ + giveup "Cannot make a partial commit with unlocked annexed files. You should `git annex add` the files you want to commit, and then run git commit." + void $ liftIO cleanup + , do + l <- workTreeItems ps + -- fix symlinks to files being committed + flip withFilesToBeCommitted l $ \f -> commandAction $ + maybe stop (Command.Fix.start Command.Fix.FixSymlinks f) + =<< isAnnexLink f + -- inject unlocked files into the annex + -- (not needed when repo version uses + -- unlocked pointer files) + unlessM versionSupportsUnlockedPointers $ + withFilesOldUnlockedToBeCommitted (commandAction . startInjectUnlocked) l + ) + runAnnexHook preCommitAnnexHook + -- committing changes to a view updates metadata + mv <- currentView + case mv of + Nothing -> noop + Just v -> withViewChanges + (addViewMetaData v) + (removeViewMetaData v) + ) + + +startInjectUnlocked :: FilePath -> CommandStart +startInjectUnlocked f = next $ do + unlessM (callCommandAction $ Command.Add.start f) $ + error $ "failed to add " ++ f ++ "; canceling commit" + next $ return True + +startDirect :: [String] -> CommandStart +startDirect _ = next $ next preCommitDirect + +addViewMetaData :: View -> ViewedFile -> Key -> CommandStart +addViewMetaData v f k = do + showStart "metadata" f + next $ next $ changeMetaData k $ fromView v f + +removeViewMetaData :: View -> ViewedFile -> Key -> CommandStart +removeViewMetaData v f k = do + showStart "metadata" f + next $ next $ changeMetaData k $ unsetMetaData $ fromView v f + +changeMetaData :: Key -> MetaData -> CommandCleanup +changeMetaData k metadata = do + showMetaDataChange metadata + addMetaData k metadata + return True + +showMetaDataChange :: MetaData -> Annex () +showMetaDataChange = showLongNote . unlines . concatMap showmeta . fromMetaData + where + showmeta (f, vs) = map (showmetavalue f) $ S.toList vs + showmetavalue f v = fromMetaField f ++ showset v ++ "=" ++ fromMetaValue v + showset v + | isSet v = "+" + | otherwise = "-" + +{- Takes exclusive lock; blocks until available. -} +lockPreCommitHook :: Annex a -> Annex a +lockPreCommitHook = withExclusiveLock gitAnnexPreCommitLock diff --git a/Command/Proxy.hs b/Command/Proxy.hs new file mode 100644 index 0000000000..e881f858cb --- /dev/null +++ b/Command/Proxy.hs @@ -0,0 +1,80 @@ +{- git-annex command + - + - Copyright 2014 Joey Hess + - + - Licensed under the GNU GPL version 3 or higher. + -} + +module Command.Proxy where + +import Command +import Config +import Utility.Tmp.Dir +import Utility.Env +import Annex.Direct +import qualified Git +import qualified Git.Sha +import qualified Git.Ref +import qualified Git.Branch +import qualified Git.LsFiles +import Git.FilePath +import Utility.CopyFile + +cmd :: Command +cmd = notBareRepo $ + command "proxy" SectionPlumbing + "safely bypass direct mode guard" + ("-- git command") (withParams seek) + +seek :: CmdParams -> CommandSeek +seek = withWords (commandAction . start) + +start :: [String] -> CommandStart +start [] = giveup "Did not specify command to run." +start (c:ps) = liftIO . exitWith =<< ifM isDirect + ( do + tmp <- gitAnnexTmpMiscDir <$> gitRepo + withTmpDirIn tmp "proxy" go + , liftIO $ safeSystem c (map Param ps) + ) + where + go tmp = do + oldref <- fromMaybe Git.Sha.emptyTree + <$> (inRepo . maybe Git.Ref.headSha Git.Ref.sha + =<< inRepo Git.Branch.currentUnsafe) + + exitcode <- proxy tmp + + cleanupproxy tmp oldref + + return exitcode + + proxy tmp = do + usetmp <- liftIO $ Just . addEntry "GIT_WORK_TREE" tmp <$> getEnvironment + + -- Set up the tmp work tree, to contain both a checkout of all + -- staged files as well as hard links (or copies) of any + -- unstaged files. + unlessM (isNothing <$> inRepo Git.Branch.current) $ + unlessM (liftIO $ boolSystemEnv "git" [Param "checkout", Param "--", Param "."] usetmp) $ + error "Failed to set up proxy work tree." + top <- fromRepo Git.repoPath + (fs, cleanup) <- inRepo $ Git.LsFiles.notInRepo True [top] + forM_ fs $ \f -> do + tf <- inRepo $ toTopFilePath f + let tmpf = tmp getTopFilePath tf + liftIO $ do + createDirectoryIfMissing True (takeDirectory tmpf) + createLinkOrCopy f tmpf + liftIO $ void cleanup + + liftIO $ safeSystemEnv c (map Param ps) usetmp + + -- To merge the changes made by the proxied command into + -- the work tree is similar to cleaning up after a + -- direct mode merge. But, here we force updates of any + -- non-annxed files that were changed by the proxied + -- command. + cleanupproxy tmp oldref = do + updateWorkTree tmp oldref True + liftIO $ removeDirectoryRecursive tmp diff --git a/Command/ReKey.hs b/Command/ReKey.hs new file mode 100644 index 0000000000..ec1cbe2b26 --- /dev/null +++ b/Command/ReKey.hs @@ -0,0 +1,131 @@ +{- git-annex command + - + - Copyright 2012-2016 Joey Hess + - + - Licensed under the GNU GPL version 3 or higher. + -} + +module Command.ReKey where + +import Command +import qualified Annex +import Annex.Content +import Annex.Ingest +import Annex.Link +import Annex.Perms +import Annex.ReplaceFile +import Logs.Location +import Git.FilePath +import qualified Database.Keys +import Annex.InodeSentinal +import Utility.InodeCache + +cmd :: Command +cmd = notDirect $ + command "rekey" SectionPlumbing + "change keys used for files" + (paramRepeating $ paramPair paramPath paramKey) + (seek <$$> optParser) + +data ReKeyOptions = ReKeyOptions + { reKeyThese :: CmdParams + , batchOption :: BatchMode + } + +optParser :: CmdParamsDesc -> Parser ReKeyOptions +optParser desc = ReKeyOptions + <$> cmdParams desc + <*> parseBatchOption + +-- Split on the last space, since a FilePath can contain whitespace, +-- but a Key very rarely does. +batchParser :: String -> Either String (FilePath, Key) +batchParser s = case separate (== ' ') (reverse s) of + (rk, rf) + | null rk || null rf -> Left "Expected: \"file key\"" + | otherwise -> case file2key (reverse rk) of + Nothing -> Left "bad key" + Just k -> Right (reverse rf, k) + +seek :: ReKeyOptions -> CommandSeek +seek o = case batchOption o of + Batch fmt -> batchInput fmt batchParser (batchCommandAction . start) + NoBatch -> withPairs (commandAction . start . parsekey) (reKeyThese o) + where + parsekey (file, skey) = + (file, fromMaybe (giveup "bad key") (file2key skey)) + +start :: (FilePath, Key) -> CommandStart +start (file, newkey) = ifAnnexed file go stop + where + go oldkey + | oldkey == newkey = stop + | otherwise = do + showStart "rekey" file + next $ perform file oldkey newkey + +perform :: FilePath -> Key -> Key -> CommandPerform +perform file oldkey newkey = do + ifM (inAnnex oldkey) + ( unlessM (linkKey file oldkey newkey) $ + giveup "failed" + , unlessM (Annex.getState Annex.force) $ + giveup $ file ++ " is not available (use --force to override)" + ) + next $ cleanup file oldkey newkey + +{- Make a hard link to the old key content (when supported), + - to avoid wasting disk space. -} +linkKey :: FilePath -> Key -> Key -> Annex Bool +linkKey file oldkey newkey = ifM (isJust <$> isAnnexLink file) + {- If the object file is already hardlinked to elsewhere, a hard + - link won't be made by getViaTmpFromDisk, but a copy instead. + - This avoids hard linking to content linked to an + - unlocked file, which would leave the new key unlocked + - and vulnerable to corruption. -} + ( getViaTmpFromDisk RetrievalAllKeysSecure DefaultVerify newkey $ \tmp -> unVerified $ do + oldobj <- calcRepo (gitAnnexLocation oldkey) + isJust <$> linkOrCopy' (return True) newkey oldobj tmp Nothing + , do + ic <- withTSDelta (liftIO . genInodeCache file) + {- The file being rekeyed is itself an unlocked file, so if + - it's linked to the old key, that link must be broken. -} + oldobj <- calcRepo (gitAnnexLocation oldkey) + v <- tryNonAsync $ modifyContent oldobj $ do + replaceFile oldobj $ \tmp -> + unlessM (checkedCopyFile oldkey file tmp Nothing) $ + error "can't lock old key" + freezeContent oldobj + oldic <- withTSDelta (liftIO . genInodeCache oldobj) + whenM (isUnmodified oldkey oldobj) $ + Database.Keys.addInodeCaches oldkey (catMaybes [oldic]) + case v of + Left e -> do + warning (show e) + return False + Right () -> do + r <- linkToAnnex newkey file ic + return $ case r of + LinkAnnexFailed -> False + LinkAnnexOk -> True + LinkAnnexNoop -> True + ) + +cleanup :: FilePath -> Key -> Key -> CommandCleanup +cleanup file oldkey newkey = do + ifM (isJust <$> isAnnexLink file) + ( do + -- Update symlink to use the new key. + liftIO $ removeFile file + addLink file newkey Nothing + , do + mode <- liftIO $ catchMaybeIO $ fileMode <$> getFileStatus file + liftIO $ whenM (isJust <$> isPointerFile file) $ + writePointerFile file newkey mode + stagePointerFile file mode =<< hashPointerFile newkey + Database.Keys.removeAssociatedFile oldkey + =<< inRepo (toTopFilePath file) + ) + whenM (inAnnex newkey) $ + logStatus newkey InfoPresent + return True diff --git a/Command/ReadPresentKey.hs b/Command/ReadPresentKey.hs new file mode 100644 index 0000000000..d1e298196b --- /dev/null +++ b/Command/ReadPresentKey.hs @@ -0,0 +1,31 @@ +{- git-annex command + - + - Copyright 2015 Joey Hess + - + - Licensed under the GNU GPL version 3 or higher. + -} + +module Command.ReadPresentKey where + +import Command +import Logs.Location + +cmd :: Command +cmd = noCommit $ + command "readpresentkey" SectionPlumbing + "read records of where key is present" + (paramPair paramKey paramUUID) + (withParams seek) + +seek :: CmdParams -> CommandSeek +seek = withWords (commandAction . start) + +start :: [String] -> CommandStart +start (ks:us:[]) = do + ls <- loggedLocations k + if toUUID us `elem` ls + then liftIO exitSuccess + else liftIO exitFailure + where + k = fromMaybe (giveup "bad key") (file2key ks) +start _ = giveup "Wrong number of parameters" diff --git a/Command/RecvKey.hs b/Command/RecvKey.hs new file mode 100644 index 0000000000..fb4ba1c278 --- /dev/null +++ b/Command/RecvKey.hs @@ -0,0 +1,49 @@ +{- git-annex command + - + - Copyright 2010 Joey Hess + - + - Licensed under the GNU GPL version 3 or higher. + -} + +module Command.RecvKey where + +import Command +import Annex.Content +import Annex.Action +import Annex +import Utility.Rsync +import Types.Transfer +import Types.Remote (RetrievalSecurityPolicy(..)) +import Command.SendKey (fieldTransfer) +import qualified CmdLine.GitAnnexShell.Fields as Fields + +cmd :: Command +cmd = noCommit $ command "recvkey" SectionPlumbing + "runs rsync in server mode to receive content" + paramKey (withParams seek) + +seek :: CmdParams -> CommandSeek +seek = withKeys (commandAction . start) + +start :: Key -> CommandStart +start key = fieldTransfer Download key $ \_p -> do + -- Always verify content when a repo is sending an unlocked file, + -- as the file could change while being transferred. + fromunlocked <- (isJust <$> Fields.getField Fields.unlocked) + <||> (isJust <$> Fields.getField Fields.direct) + let verify = if fromunlocked then AlwaysVerify else DefaultVerify + -- This matches the retrievalSecurityPolicy of Remote.Git + let rsp = RetrievalAllKeysSecure + ifM (getViaTmp rsp verify key go) + ( do + -- forcibly quit after receiving one key, + -- and shutdown cleanly + _ <- shutdown True + return True + , return False + ) + where + go tmp = unVerified $ do + opts <- filterRsyncSafeOptions . maybe [] words + <$> getField "RsyncOptions" + liftIO $ rsyncServerReceive (map Param opts) tmp diff --git a/Command/RegisterUrl.hs b/Command/RegisterUrl.hs new file mode 100644 index 0000000000..a285050dc1 --- /dev/null +++ b/Command/RegisterUrl.hs @@ -0,0 +1,73 @@ +{- git-annex command + - + - Copyright 2015-2018 Joey Hess + - + - Licensed under the GNU GPL version 3 or higher. + -} + +{-# LANGUAGE BangPatterns #-} + +module Command.RegisterUrl where + +import Command +import Logs.Web +import Command.FromKey (mkKey) +import qualified Remote + +cmd :: Command +cmd = notDirect $ notBareRepo $ + command "registerurl" + SectionPlumbing "registers an url for a key" + (paramPair paramKey paramUrl) + (seek <$$> optParser) + +data RegisterUrlOptions = RegisterUrlOptions + { keyUrlPairs :: CmdParams + , batchOption :: BatchMode + } + +optParser :: CmdParamsDesc -> Parser RegisterUrlOptions +optParser desc = RegisterUrlOptions + <$> cmdParams desc + <*> parseBatchOption + +seek :: RegisterUrlOptions -> CommandSeek +seek o = case (batchOption o, keyUrlPairs o) of + (Batch fmt, _) -> commandAction $ startMass fmt + -- older way of enabling batch input, does not support BatchNull + (NoBatch, []) -> commandAction $ startMass BatchLine + (NoBatch, ps) -> withWords (commandAction . start) ps + +start :: [String] -> CommandStart +start (keyname:url:[]) = do + let key = mkKey keyname + showStart' "registerurl" (Just url) + next $ perform key url +start _ = giveup "specify a key and an url" + +startMass :: BatchFormat -> CommandStart +startMass fmt = do + showStart' "registerurl" (Just "stdin") + next (massAdd fmt) + +massAdd :: BatchFormat -> CommandPerform +massAdd fmt = go True =<< map (separate (== ' ')) <$> batchLines fmt + where + go status [] = next $ return status + go status ((keyname,u):rest) | not (null keyname) && not (null u) = do + let key = mkKey keyname + ok <- perform' key u + let !status' = status && ok + go status' rest + go _ _ = giveup "Expected pairs of key and url on stdin, but got something else." + +perform :: Key -> URLString -> CommandPerform +perform key url = do + ok <- perform' key url + next $ return ok + +perform' :: Key -> URLString -> Annex Bool +perform' key url = do + r <- Remote.claimingUrl url + setUrlPresent (Remote.uuid r) key (setDownloader' url r) + return True diff --git a/Command/Reinit.hs b/Command/Reinit.hs new file mode 100644 index 0000000000..b40b680107 --- /dev/null +++ b/Command/Reinit.hs @@ -0,0 +1,41 @@ +{- git-annex command + - + - Copyright 2014 Joey Hess + - + - Licensed under the GNU GPL version 3 or higher. + -} + +module Command.Reinit where + +import Command +import Annex.Init +import Annex.UUID +import qualified Remote +import qualified Annex.SpecialRemote + +cmd :: Command +cmd = dontCheck repoExists $ + command "reinit" SectionUtility + "initialize repository, reusing old UUID" + (paramUUID ++ "|" ++ paramDesc) + (withParams seek) + +seek :: CmdParams -> CommandSeek +seek = withWords (commandAction . start) + +start :: [String] -> CommandStart +start ws = do + showStart' "reinit" (Just s) + next $ perform s + where + s = unwords ws + +perform :: String -> CommandPerform +perform s = do + u <- if isUUID s + then return $ toUUID s + else Remote.nameToUUID s + storeUUID u + initialize' (AutoInit False) Nothing + Annex.SpecialRemote.autoEnable + next $ return True diff --git a/Command/Reinject.hs b/Command/Reinject.hs new file mode 100644 index 0000000000..50f3ca735d --- /dev/null +++ b/Command/Reinject.hs @@ -0,0 +1,83 @@ +{- git-annex command + - + - Copyright 2011-2016 Joey Hess + - + - Licensed under the GNU GPL version 3 or higher. + -} + +module Command.Reinject where + +import Command +import Logs.Location +import Annex.Content +import Backend +import Types.KeySource + +cmd :: Command +cmd = command "reinject" SectionUtility + "inject content of file back into annex" + (paramRepeating (paramPair "SRC" "DEST")) + (seek <$$> optParser) + +data ReinjectOptions = ReinjectOptions + { params :: CmdParams + , knownOpt :: Bool + } + +optParser :: CmdParamsDesc -> Parser ReinjectOptions +optParser desc = ReinjectOptions + <$> cmdParams desc + <*> switch + ( long "known" + <> help "inject all known files" + <> hidden + ) + +seek :: ReinjectOptions -> CommandSeek +seek os + | knownOpt os = withStrings (commandAction . startKnown) (params os) + | otherwise = withWords (commandAction . startSrcDest) (params os) + +startSrcDest :: [FilePath] -> CommandStart +startSrcDest (src:dest:[]) + | src == dest = stop + | otherwise = notAnnexed src $ do + showStart "reinject" dest + next $ ifAnnexed dest go stop + where + go key = ifM (verifyKeyContent RetrievalAllKeysSecure DefaultVerify UnVerified key src) + ( perform src key + , giveup $ src ++ " does not have expected content of " ++ dest + ) +startSrcDest _ = giveup "specify a src file and a dest file" + +startKnown :: FilePath -> CommandStart +startKnown src = notAnnexed src $ do + showStart "reinject" src + mkb <- genKey (KeySource src src Nothing) Nothing + case mkb of + Nothing -> error "Failed to generate key" + Just (key, _) -> ifM (isKnownKey key) + ( next $ perform src key + , do + warning "Not known content; skipping" + next $ next $ return True + ) + +notAnnexed :: FilePath -> CommandStart -> CommandStart +notAnnexed src = ifAnnexed src $ + giveup $ "cannot used annexed file as src: " ++ src + +perform :: FilePath -> Key -> CommandPerform +perform src key = ifM move + ( next $ cleanup key + , error "failed" + ) + where + move = checkDiskSpaceToGet key False $ + moveAnnex key src + +cleanup :: Key -> CommandCleanup +cleanup key = do + logStatus key InfoPresent + return True diff --git a/Command/RemoteDaemon.hs b/Command/RemoteDaemon.hs new file mode 100644 index 0000000000..c174171041 --- /dev/null +++ b/Command/RemoteDaemon.hs @@ -0,0 +1,32 @@ +{- git-annex command + - + - Copyright 2014-2016 Joey Hess + - + - Licensed under the GNU GPL version 3 or higher. + -} + +{-# LANGUAGE CPP #-} + +module Command.RemoteDaemon where + +import Command +import RemoteDaemon.Core +import Utility.Daemon + +cmd :: Command +cmd = noCommit $ + command "remotedaemon" SectionMaintenance + "persistent communication with remotes" + paramNothing (run <$$> const parseDaemonOptions) + +run :: DaemonOptions -> CommandSeek +run o + | stopDaemonOption o = error "--stop not implemented for remotedaemon" + | foregroundDaemonOption o = liftIO runInteractive + | otherwise = do +#ifndef mingw32_HOST_OS + nullfd <- liftIO $ openFd "/dev/null" ReadOnly Nothing defaultFileFlags + liftIO $ daemonize nullfd Nothing False runNonInteractive +#else + liftIO $ foreground Nothing runNonInteractive +#endif diff --git a/Command/Repair.hs b/Command/Repair.hs new file mode 100644 index 0000000000..c8beeea6eb --- /dev/null +++ b/Command/Repair.hs @@ -0,0 +1,85 @@ +{- git-annex command + - + - Copyright 2013 Joey Hess + - + - Licensed under the GNU GPL version 3 or higher. + -} + +module Command.Repair where + +import Command +import qualified Annex +import qualified Git.Repair +import qualified Annex.Branch +import qualified Git.Ref +import Git.Types +import Annex.Version + +cmd :: Command +cmd = noCommit $ dontCheck repoExists $ + command "repair" SectionMaintenance + "recover broken git repository" + paramNothing (withParams seek) + +seek :: CmdParams -> CommandSeek +seek = withNothing (commandAction start) + +start :: CommandStart +start = next $ next $ runRepair =<< Annex.getState Annex.force + +runRepair :: Bool -> Annex Bool +runRepair forced = do + (ok, modifiedbranches) <- inRepo $ + Git.Repair.runRepair isAnnexSyncBranch forced + -- This command can be run in git repos not using git-annex, + -- so avoid git annex branch stuff in that case. + whenM (isJust <$> getVersion) $ + repairAnnexBranch modifiedbranches + return ok + +{- After git repository repair, the .git/annex/index file could + - still be broken, by pointing to bad objects, or might just be corrupt on + - its own. Since this index file is not used to stage things + - for long durations of time, it can safely be deleted if it is broken. + - + - Otherwise, if the git-annex branch was modified by the repair, + - commit the index file to the git-annex branch. + - This way, if the git-annex branch got rewound to an old version by + - the repository repair, or was completely deleted, this will get it back + - to a good state. Note that in the unlikely case where the git-annex + - branch was rewound to a state that, had new changes from elsewhere not + - yet reflected in the index, this does properly merge those into the + - index before committing. + -} +repairAnnexBranch :: [Branch] -> Annex () +repairAnnexBranch modifiedbranches + | Annex.Branch.fullname `elem` modifiedbranches = ifM okindex + ( commitindex + , do + nukeindex + missingbranch + ) + | otherwise = ifM okindex + ( noop + , do + nukeindex + ifM (null <$> inRepo (Git.Ref.matching [Annex.Branch.fullname])) + ( missingbranch + , liftIO $ putStrLn "No data was lost." + ) + ) + where + okindex = Annex.Branch.withIndex $ inRepo Git.Repair.checkIndex + commitindex = do + Annex.Branch.forceCommit "committing index after git repository repair" + liftIO $ putStrLn "Successfully recovered the git-annex branch using .git/annex/index" + nukeindex = do + inRepo $ nukeFile . gitAnnexIndex + liftIO $ putStrLn "Had to delete the .git/annex/index file as it was corrupt." + missingbranch = liftIO $ putStrLn "Since the git-annex branch is not up-to-date anymore. It would be a very good idea to run: git annex fsck --fast" + +trackingOrSyncBranch :: Ref -> Bool +trackingOrSyncBranch b = Git.Repair.isTrackingBranch b || isAnnexSyncBranch b + +isAnnexSyncBranch :: Ref -> Bool +isAnnexSyncBranch b = "refs/synced/" `isPrefixOf` fromRef b diff --git a/Command/Required.hs b/Command/Required.hs new file mode 100644 index 0000000000..3cc053b55d --- /dev/null +++ b/Command/Required.hs @@ -0,0 +1,17 @@ +{- git-annex command + - + - Copyright 2015 Joey Hess + - + - Licensed under the GNU GPL version 3 or higher. + -} + +module Command.Required where + +import Command +import Logs.PreferredContent +import qualified Command.Wanted + +cmd :: Command +cmd = Command.Wanted.cmd' "required" "get or set required content expression" + requiredContentMapRaw + requiredContentSet diff --git a/Command/ResolveMerge.hs b/Command/ResolveMerge.hs new file mode 100644 index 0000000000..6add6ff9a0 --- /dev/null +++ b/Command/ResolveMerge.hs @@ -0,0 +1,40 @@ +{- git-annex command + - + - Copyright 2014 Joey Hess + - + - Licensed under the GNU GPL version 3 or higher. + -} + +module Command.ResolveMerge where + +import Command +import qualified Git +import Git.Sha +import qualified Git.Branch +import Annex.AutoMerge + +cmd :: Command +cmd = command "resolvemerge" SectionPlumbing + "resolve merge conflicts" + paramNothing (withParams seek) + +seek :: CmdParams -> CommandSeek +seek = withNothing (commandAction start) + +start :: CommandStart +start = do + showStart' "resolvemerge" Nothing + us <- fromMaybe nobranch <$> inRepo Git.Branch.current + d <- fromRepo Git.localGitDir + let merge_head = d "MERGE_HEAD" + them <- fromMaybe (error nomergehead) . extractSha + <$> liftIO (readFile merge_head) + ifM (resolveMerge (Just us) them False) + ( do + void $ commitResolvedMerge Git.Branch.ManualCommit + next $ next $ return True + , giveup "Merge conflict could not be automatically resolved." + ) + where + nobranch = giveup "No branch is currently checked out." + nomergehead = giveup "No SHA found in .git/merge_head" diff --git a/Command/RmUrl.hs b/Command/RmUrl.hs new file mode 100644 index 0000000000..c883da13fa --- /dev/null +++ b/Command/RmUrl.hs @@ -0,0 +1,53 @@ +{- git-annex command + - + - Copyright 2013-2016 Joey Hess + - + - Licensed under the GNU GPL version 3 or higher. + -} + +module Command.RmUrl where + +import Command +import Logs.Web +import qualified Remote + +cmd :: Command +cmd = notBareRepo $ + command "rmurl" SectionCommon + "record file is not available at url" + (paramRepeating (paramPair paramFile paramUrl)) + (seek <$$> optParser) + +data RmUrlOptions = RmUrlOptions + { rmThese :: CmdParams + , batchOption :: BatchMode + } + +optParser :: CmdParamsDesc -> Parser RmUrlOptions +optParser desc = RmUrlOptions + <$> cmdParams desc + <*> parseBatchOption + +seek :: RmUrlOptions -> CommandSeek +seek o = case batchOption o of + Batch fmt -> batchInput fmt batchParser (batchCommandAction . start) + NoBatch -> withPairs (commandAction . start) (rmThese o) + +-- Split on the last space, since a FilePath can contain whitespace, +-- but a url should not. +batchParser :: String -> Either String (FilePath, URLString) +batchParser s = case separate (== ' ') (reverse s) of + (ru, rf) + | null ru || null rf -> Left "Expected: \"file url\"" + | otherwise -> Right (reverse rf, reverse ru) + +start :: (FilePath, URLString) -> CommandStart +start (file, url) = flip whenAnnexed file $ \_ key -> do + showStart "rmurl" file + next $ next $ cleanup url key + +cleanup :: String -> Key -> CommandCleanup +cleanup url key = do + r <- Remote.claimingUrl url + setUrlMissing (Remote.uuid r) key (setDownloader' url r) + return True diff --git a/Command/Schedule.hs b/Command/Schedule.hs new file mode 100644 index 0000000000..c5a8004fdc --- /dev/null +++ b/Command/Schedule.hs @@ -0,0 +1,51 @@ +{- git-annex command + - + - Copyright 2013 Joey Hess + - + - Licensed under the GNU GPL version 3 or higher. + -} + +module Command.Schedule where + +import Command +import qualified Remote +import Logs.Schedule +import Types.ScheduledActivity + +import qualified Data.Set as S + +cmd :: Command +cmd = noMessages $ command "schedule" SectionSetup "get or set scheduled jobs" + (paramPair paramRemote (paramOptional paramExpression)) + (withParams seek) + +seek :: CmdParams -> CommandSeek +seek = withWords (commandAction . start) + +start :: [String] -> CommandStart +start = parse + where + parse (name:[]) = go name performGet + parse (name:expr:[]) = go name $ \uuid -> do + allowMessages + showStart' "schedule" (Just name) + performSet expr uuid + parse _ = giveup "Specify a repository." + + go name a = do + u <- Remote.nameToUUID name + next $ a u + +performGet :: UUID -> CommandPerform +performGet uuid = do + s <- scheduleGet uuid + liftIO $ putStrLn $ intercalate "; " $ + map fromScheduledActivity $ S.toList s + next $ return True + +performSet :: String -> UUID -> CommandPerform +performSet expr uuid = case parseScheduledActivities expr of + Left e -> giveup $ "Parse error: " ++ e + Right l -> do + scheduleSet uuid l + next $ return True diff --git a/Command/Semitrust.hs b/Command/Semitrust.hs new file mode 100644 index 0000000000..d9ee893945 --- /dev/null +++ b/Command/Semitrust.hs @@ -0,0 +1,20 @@ +{- git-annex command + - + - Copyright 2010 Joey Hess + - + - Licensed under the GNU GPL version 3 or higher. + -} + +module Command.Semitrust where + +import Command +import Types.TrustLevel +import Command.Trust (trustCommand) + +cmd :: Command +cmd = command "semitrust" SectionSetup + "return repository to default trust level" + (paramRepeating paramRemote) (withParams seek) + +seek :: CmdParams -> CommandSeek +seek = trustCommand "semitrust" SemiTrusted diff --git a/Command/SendKey.hs b/Command/SendKey.hs new file mode 100644 index 0000000000..60f0062f83 --- /dev/null +++ b/Command/SendKey.hs @@ -0,0 +1,63 @@ +{- git-annex command + - + - Copyright 2010,2012 Joey Hess + - + - Licensed under the GNU GPL version 3 or higher. + -} + +module Command.SendKey where + +import Command +import Annex.Content +import Annex +import Utility.Rsync +import Annex.Transfer +import qualified CmdLine.GitAnnexShell.Fields as Fields +import Utility.Metered + +import System.Log.Logger + +cmd :: Command +cmd = noCommit $ + command "sendkey" SectionPlumbing + "runs rsync in server mode to send content" + paramKey (withParams seek) + +seek :: CmdParams -> CommandSeek +seek = withKeys (commandAction . start) + +start :: Key -> CommandStart +start key = do + opts <- filterRsyncSafeOptions . maybe [] words + <$> getField "RsyncOptions" + ifM (inAnnex key) + ( fieldTransfer Upload key $ \_p -> + sendAnnex key rollback $ liftIO . rsyncServerSend (map Param opts) + , do + warning "requested key is not present" + liftIO exitFailure + ) + where + {- No need to do any rollback; when sendAnnex fails, a nonzero + - exit will be propigated, and the remote will know the transfer + - failed. -} + rollback = noop + +fieldTransfer :: Direction -> Key -> (MeterUpdate -> Annex Bool) -> CommandStart +fieldTransfer direction key a = do + liftIO $ debugM "fieldTransfer" "transfer start" + afile <- AssociatedFile <$> Fields.getField Fields.associatedFile + ok <- maybe (a $ const noop) + -- Using noRetry here because we're the sender. + (\u -> runner (Transfer direction (toUUID u) key) afile noRetry a) + =<< Fields.getField Fields.remoteUUID + liftIO $ debugM "fieldTransfer" "transfer done" + liftIO $ exitBool ok + where + {- Allow the key to be sent to the remote even if there seems to be + - another transfer of that key going on to that remote. + - That one may be stale, etc. + -} + runner + | direction == Upload = alwaysRunTransfer + | otherwise = runTransfer diff --git a/Command/SetKey.hs b/Command/SetKey.hs new file mode 100644 index 0000000000..32be029b24 --- /dev/null +++ b/Command/SetKey.hs @@ -0,0 +1,49 @@ +{- git-annex command + - + - Copyright 2010, 2015 Joey Hess + - + - Licensed under the GNU GPL version 3 or higher. + -} + +module Command.SetKey where + +import Command +import Logs.Location +import Annex.Content + +cmd :: Command +cmd = command "setkey" SectionPlumbing "sets annexed content for a key" + (paramPair paramKey paramPath) + (withParams seek) + +seek :: CmdParams -> CommandSeek +seek = withWords (commandAction . start) + +start :: [String] -> CommandStart +start (keyname:file:[]) = do + showStart "setkey" file + next $ perform file (mkKey keyname) +start _ = giveup "specify a key and a content file" + +mkKey :: String -> Key +mkKey = fromMaybe (giveup "bad key") . file2key + +perform :: FilePath -> Key -> CommandPerform +perform file key = do + -- the file might be on a different filesystem, so moveFile is used + -- rather than simply calling moveAnnex; disk space is also + -- checked this way. + ok <- getViaTmp RetrievalAllKeysSecure DefaultVerify key $ \dest -> unVerified $ + if dest /= file + then liftIO $ catchBoolIO $ do + moveFile file dest + return True + else return True + if ok + then next $ cleanup key + else error "mv failed!" + +cleanup :: Key -> CommandCleanup +cleanup key = do + logStatus key InfoPresent + return True diff --git a/Command/SetPresentKey.hs b/Command/SetPresentKey.hs new file mode 100644 index 0000000000..a9dbe54522 --- /dev/null +++ b/Command/SetPresentKey.hs @@ -0,0 +1,57 @@ +{- git-annex command + - + - Copyright 2014 Joey Hess + - + - Licensed under the GNU GPL version 3 or higher. + -} + +module Command.SetPresentKey where + +import Command +import Logs.Location +import Logs.Presence.Pure + +cmd :: Command +cmd = noCommit $ + command "setpresentkey" SectionPlumbing + "change records of where key is present" + (paramPair paramKey (paramPair paramUUID "[1|0]")) + (seek <$$> optParser) + +data SetPresentKeyOptions = SetPresentKeyOptions + { params :: CmdParams + , batchOption :: BatchMode + } + +optParser :: CmdParamsDesc -> Parser SetPresentKeyOptions +optParser desc = SetPresentKeyOptions + <$> cmdParams desc + <*> parseBatchOption + +seek :: SetPresentKeyOptions -> CommandSeek +seek o = case batchOption o of + Batch fmt -> batchInput fmt + (parseKeyStatus . words) + (batchCommandAction . start) + NoBatch -> either giveup (commandAction . start) + (parseKeyStatus $ params o) + +data KeyStatus = KeyStatus Key UUID LogStatus + +parseKeyStatus :: [String] -> Either String KeyStatus +parseKeyStatus (ks:us:vs:[]) = do + k <- maybe (Left "bad key") Right (file2key ks) + let u = toUUID us + s <- maybe (Left "bad value") Right (parseStatus vs) + return $ KeyStatus k u s +parseKeyStatus _ = Left "Bad input. Expected: key uuid value" + +start :: KeyStatus -> CommandStart +start (KeyStatus k u s) = do + showStartKey "setpresentkey" k (mkActionItem k) + next $ perform k u s + +perform :: Key -> UUID -> LogStatus -> CommandPerform +perform k u s = next $ do + logChange k u s + return True diff --git a/Command/Smudge.hs b/Command/Smudge.hs new file mode 100644 index 0000000000..28dc551d35 --- /dev/null +++ b/Command/Smudge.hs @@ -0,0 +1,154 @@ +{- git-annex command + - + - Copyright 2015-2018 Joey Hess + - + - Licensed under the GNU GPL version 3 or higher. + -} + +module Command.Smudge where + +import Command +import qualified Annex +import Annex.Content +import Annex.Link +import Annex.FileMatcher +import Annex.Ingest +import Annex.CatFile +import Logs.Location +import qualified Database.Keys +import qualified Git.BuildVersion +import Git.FilePath +import qualified Git.Ref +import Backend + +import qualified Data.ByteString.Lazy as B + +cmd :: Command +cmd = noCommit $ noMessages $ + command "smudge" SectionPlumbing + "git smudge filter" + paramFile (seek <$$> optParser) + +data SmudgeOptions = SmudgeOptions + { smudgeFile :: FilePath + , cleanOption :: Bool + } + +optParser :: CmdParamsDesc -> Parser SmudgeOptions +optParser desc = SmudgeOptions + <$> argument str ( metavar desc ) + <*> switch ( long "clean" <> help "clean filter" ) + +seek :: SmudgeOptions -> CommandSeek +seek o = commandAction $ + (if cleanOption o then clean else smudge) (smudgeFile o) + +-- Smudge filter is fed git file content, and if it's a pointer to an +-- available annex object, should output its content. +smudge :: FilePath -> CommandStart +smudge file = do + b <- liftIO $ B.hGetContents stdin + case parseLinkOrPointer b of + Nothing -> liftIO $ B.putStr b + Just k -> do + Database.Keys.addAssociatedFile k =<< inRepo (toTopFilePath file) + -- A previous unlocked checkout of the file may have + -- led to the annex object getting modified; + -- don't provide such modified content as it + -- will be confusing. inAnnex will detect such + -- modifications. + ifM (inAnnex k) + ( do + content <- calcRepo (gitAnnexLocation k) + whenM (annexThin <$> Annex.getGitConfig) $ + warning $ "Not able to honor annex.thin when git is checking out " ++ file ++ " (run git annex fix to re-thin files)" + liftIO $ B.putStr . fromMaybe b + =<< catchMaybeIO (B.readFile content) + , liftIO $ B.putStr b + ) + stop + +-- Clean filter is fed file content on stdin, decides if a file +-- should be stored in the annex, and outputs a pointer to its +-- injested content if so. Otherwise, the original content. +clean :: FilePath -> CommandStart +clean file = do + b <- liftIO $ B.hGetContents stdin + case parseLinkOrPointer b of + Just k -> do + getMoveRaceRecovery k file + liftIO $ B.hPut stdout b + Nothing -> go b =<< catKeyFile file + stop + where + go b oldkey = ifM (shouldAnnex file oldkey) + ( do + -- Before git 2.5, failing to consume all stdin here + -- would cause a SIGPIPE and crash it. + -- Newer git catches the signal and stops sending, + -- which is much faster. (Also, git seems to forget + -- to free memory when sending the file, so the + -- less we let it send, the less memory it will waste.) + if Git.BuildVersion.older "2.5" + then B.length b `seq` return () + else liftIO $ hClose stdin + -- Look up the backend that was used for this file + -- before, so that when git re-cleans a file its + -- backend does not change. + let oldbackend = maybe Nothing (maybeLookupBackendVariety . keyVariety) oldkey + -- Can't restage associated files because git add + -- runs this and has the index locked. + let norestage = Restage False + liftIO . emitPointer + =<< postingest + =<< (\ld -> ingest' oldbackend ld Nothing norestage) + =<< lockDown cfg file + , liftIO $ B.hPut stdout b + ) + + postingest (Just k, _) = do + logStatus k InfoPresent + return k + postingest _ = error "could not add file to the annex" + + cfg = LockDownConfig + { lockingFile = False + , hardlinkFileTmp = False + } + +-- New files are annexed as configured by annex.largefiles, with a default +-- of annexing them. +-- +-- If annex.largefiles is not configured for a file, and a file with its +-- name is already in the index, preserve its annexed/not annexed state. +-- This prevents accidental conversions when annex.largefiles is being +-- set/unset on the fly rather than being set in gitattributes or .git/config. +shouldAnnex :: FilePath -> Maybe Key -> Annex Bool +shouldAnnex file moldkey = do + matcher <- largeFilesMatcher + checkFileMatcher' matcher file whenempty + where + whenempty = case moldkey of + Just _ -> return True + Nothing -> isNothing <$> catObjectMetaData (Git.Ref.fileRef file) + +emitPointer :: Key -> IO () +emitPointer = putStr . formatPointer + +-- Recover from a previous race between eg git mv and git-annex get. +-- That could result in the file remaining a pointer file, while +-- its content is present in the annex. Populate the pointer file. +-- +-- This also handles the case where a copy of a pointer file is made, +-- then git-annex gets the content, and later git add is run on +-- the pointer copy. It will then be populated with the content. +getMoveRaceRecovery :: Key -> FilePath -> Annex () +getMoveRaceRecovery k file = void $ tryNonAsync $ + liftIO (isPointerFile file) >>= \k' -> when (Just k == k') $ + whenM (inAnnex k) $ do + obj <- calcRepo (gitAnnexLocation k) + -- Cannot restage because git add is running and has + -- the index locked. + populatePointerFile (Restage False) k obj file >>= \case + Nothing -> return () + Just ic -> Database.Keys.addInodeCaches k [ic] diff --git a/Command/Status.hs b/Command/Status.hs new file mode 100644 index 0000000000..e1a6e963a4 --- /dev/null +++ b/Command/Status.hs @@ -0,0 +1,108 @@ +{- git-annex command + - + - Copyright 2013-2015 Joey Hess + - + - Licensed under the GNU GPL version 3 or higher. + -} + +module Command.Status where + +import Command +import Annex.CatFile +import Annex.Content.Direct +import Config +import Git.Status +import qualified Git.Ref +import Git.FilePath + +cmd :: Command +cmd = notBareRepo $ noCommit $ noMessages $ + withGlobalOptions [jsonOptions] $ + command "status" SectionCommon + "show the working tree status" + paramPaths (seek <$$> optParser) + +data StatusOptions = StatusOptions + { statusFiles :: CmdParams + , ignoreSubmodules :: Maybe String + } + +optParser :: CmdParamsDesc -> Parser StatusOptions +optParser desc = StatusOptions + <$> cmdParams desc + <*> optional (strOption + ( long "ignore-submodules" + <> help "passed on to git status" + <> metavar "WHEN" + )) + +seek :: StatusOptions -> CommandSeek +seek o = withWords (commandAction . start o) (statusFiles o) + +start :: StatusOptions -> [FilePath] -> CommandStart +start o locs = do + (l, cleanup) <- inRepo $ getStatus ps locs + getstatus <- ifM isDirect + ( return (maybe (pure Nothing) statusDirect . simplifiedStatus) + , return (pure . simplifiedStatus) + ) + forM_ l $ \s -> maybe noop displayStatus =<< getstatus s + ifM (liftIO cleanup) + ( stop + , giveup "git status failed" + ) + where + ps = case ignoreSubmodules o of + Nothing -> [] + Just s -> [Param $ "--ignore-submodules="++s] + +-- Prefer to show unstaged status in this simplified status. +simplifiedStatus :: StagedUnstaged Status -> Maybe Status +simplifiedStatus (StagedUnstaged { unstaged = Just s }) = Just s +simplifiedStatus (StagedUnstaged { staged = Just s }) = Just s +simplifiedStatus _ = Nothing + +displayStatus :: Status -> Annex () +-- Renames not shown in this simplified status +displayStatus (Renamed _ _) = noop +displayStatus s = do + let c = statusChar s + absf <- fromRepo $ fromTopFilePath (statusFile s) + f <- liftIO $ relPathCwdToFile absf + unlessM (showFullJSON $ JSONChunk [("status", [c]), ("file", f)]) $ + liftIO $ putStrLn $ [c] ++ " " ++ f + +-- Git thinks that present direct mode files are typechanged. +-- (On crippled filesystems, git instead thinks they're modified.) +-- Check their content to see if they are modified or not. +statusDirect :: Status -> Annex (Maybe Status) +statusDirect (TypeChanged t) = statusDirect' t +statusDirect s@(Modified t) = ifM crippledFileSystem + ( statusDirect' t + , pure (Just s) + ) +statusDirect s = pure (Just s) + +statusDirect' :: TopFilePath -> Annex (Maybe Status) +statusDirect' t = do + absf <- fromRepo $ fromTopFilePath t + f <- liftIO $ relPathCwdToFile absf + v <- liftIO (catchMaybeIO $ getFileStatus f) + case v of + Nothing -> return $ Just $ Deleted t + Just s + | not (isSymbolicLink s) -> + checkkey f s =<< catKeyFile f + | otherwise -> Just <$> checkNew f t + where + checkkey f s (Just k) = ifM (sameFileStatus k f s) + ( return Nothing + , return $ Just $ Modified t + ) + checkkey f _ Nothing = Just <$> checkNew f t + +checkNew :: FilePath -> TopFilePath -> Annex Status +checkNew f t = ifM (isJust <$> catObjectDetails (Git.Ref.fileRef f)) + ( return (Modified t) + , return (Untracked t) + ) diff --git a/Command/Sync.hs b/Command/Sync.hs new file mode 100644 index 0000000000..5f691aabb6 --- /dev/null +++ b/Command/Sync.hs @@ -0,0 +1,753 @@ +{- git-annex command + - + - Copyright 2011 Joachim Breitner + - Copyright 2011-2017 Joey Hess + - + - Licensed under the GNU GPL version 3 or higher. + -} + +module Command.Sync ( + cmd, + CurrBranch, + getCurrBranch, + mergeConfig, + merge, + prepMerge, + mergeLocal, + mergeRemote, + commitStaged, + commitMsg, + pushBranch, + updateBranch, + syncBranch, + updateSyncBranch, + seekExportContent, +) where + +import Command +import qualified Annex +import qualified Annex.Branch +import qualified Remote +import qualified Types.Remote as Remote +import Annex.Direct +import Annex.Hook +import qualified Git.Command +import qualified Git.LsFiles as LsFiles +import qualified Git.Branch +import qualified Git.Merge +import qualified Git.Types as Git +import qualified Git.Ref +import qualified Git +import qualified Remote.Git +import Config +import Config.GitConfig +import Config.DynamicConfig +import Config.Files +import Annex.Wanted +import Annex.Content +import Command.Get (getKey') +import qualified Command.Move +import qualified Command.Export +import Annex.Drop +import Annex.UUID +import Logs.UUID +import Logs.Export +import Annex.AutoMerge +import Annex.AdjustedBranch +import Annex.Ssh +import Annex.BloomFilter +import Annex.UpdateInstead +import Annex.Export +import Annex.LockFile +import Annex.TaggedPush +import qualified Database.Export as Export +import Utility.Bloom +import Utility.OptParse +import Utility.Process.Transcript + +import Control.Concurrent.MVar +import qualified Data.Map as M + +cmd :: Command +cmd = withGlobalOptions [jobsOption] $ + command "sync" SectionCommon + "synchronize local repository with remotes" + (paramRepeating paramRemote) (seek <--< optParser) + +data SyncOptions = SyncOptions + { syncWith :: CmdParams + , commitOption :: Bool + , noCommitOption :: Bool + , messageOption :: Maybe String + , pullOption :: Bool + , pushOption :: Bool + , contentOption :: Bool + , noContentOption :: Bool + , contentOfOption :: [FilePath] + , cleanupOption :: Bool + , keyOptions :: Maybe KeyOptions + , resolveMergeOverride :: ResolveMergeOverride + } + +newtype ResolveMergeOverride = ResolveMergeOverride Bool + +instance Default ResolveMergeOverride where + def = ResolveMergeOverride False + +optParser :: CmdParamsDesc -> Parser SyncOptions +optParser desc = SyncOptions + <$> (many $ argument str + ( metavar desc + <> completeRemotes + )) + <*> switch + ( long "commit" + <> help "commit changes to git" + ) + <*> switch + ( long "no-commit" + <> help "avoid git commit" + ) + <*> optional (strOption + ( long "message" <> short 'm' <> metavar "MSG" + <> help "commit message" + )) + <*> invertableSwitch "pull" True + ( help "avoid git pulls from remotes" + ) + <*> invertableSwitch "push" True + ( help "avoid git pushes to remotes" + ) + <*> switch + ( long "content" + <> help "transfer file contents" + ) + <*> switch + ( long "no-content" + <> help "do not transfer file contents" + ) + <*> many (strOption + ( long "content-of" + <> short 'C' + <> help "transfer file contents of files in a given location" + <> metavar paramPath + )) + <*> switch + ( long "cleanup" + <> help "remove synced/ branches from previous sync" + ) + <*> optional parseAllOption + <*> (ResolveMergeOverride <$> invertableSwitch "resolvemerge" True + ( help "do not automatically resolve merge conflicts" + )) + +-- Since prepMerge changes the working directory, FilePath options +-- have to be adjusted. +instance DeferredParseClass SyncOptions where + finishParse v = SyncOptions + <$> pure (syncWith v) + <*> pure (commitOption v) + <*> pure (noCommitOption v) + <*> pure (messageOption v) + <*> pure (pullOption v) + <*> pure (pushOption v) + <*> pure (contentOption v) + <*> pure (noContentOption v) + <*> liftIO (mapM absPath (contentOfOption v)) + <*> pure (cleanupOption v) + <*> pure (keyOptions v) + <*> pure (resolveMergeOverride v) + +seek :: SyncOptions -> CommandSeek +seek o = allowConcurrentOutput $ do + prepMerge + + getbranch <- getCurrBranch + let withbranch a = a =<< getbranch + + remotes <- syncRemotes (syncWith o) + let gitremotes = filter Remote.gitSyncableRemote remotes + (exportremotes, dataremotes) <- partition (exportTree . Remote.config) + . filter (\r -> Remote.uuid r /= NoUUID) + <$> filterM (not <$$> liftIO . getDynamicConfig . remoteAnnexIgnore . Remote.gitconfig) remotes + + if cleanupOption o + then do + commandAction (withbranch cleanupLocal) + mapM_ (commandAction . withbranch . cleanupRemote) gitremotes + else do + -- Syncing involves many actions, any of which + -- can independently fail, without preventing + -- the others from running. + -- These actions cannot be run concurrently. + mapM_ includeCommandAction $ concat + [ [ commit o ] + , [ withbranch (mergeLocal mergeConfig (resolveMergeOverride o)) ] + , map (withbranch . pullRemote o mergeConfig) gitremotes + , [ mergeAnnex ] + ] + + whenM shouldsynccontent $ do + syncedcontent <- seekSyncContent o dataremotes + exportedcontent <- withbranch $ seekExportContent exportremotes + -- Transferring content can take a while, + -- and other changes can be pushed to the + -- git-annex branch on the remotes in the + -- meantime, so pull and merge again to + -- avoid our push overwriting those changes. + when (syncedcontent || exportedcontent) $ do + mapM_ includeCommandAction $ concat + [ map (withbranch . pullRemote o mergeConfig) gitremotes + , [ commitAnnex, mergeAnnex ] + ] + + void $ includeCommandAction $ withbranch pushLocal + -- Pushes to remotes can run concurrently. + mapM_ (commandAction . withbranch . pushRemote o) gitremotes + where + shouldsynccontent = pure (contentOption o) + <||> pure (not (null (contentOfOption o))) + <||> (pure (not (noContentOption o)) <&&> getGitConfigVal annexSyncContent) + +type CurrBranch = (Maybe Git.Branch, Maybe Adjustment) + +{- There may not be a branch checked out until after the commit, + - or perhaps after it gets merged from the remote, or perhaps + - never. + - + - So only look it up once it's needed, and once there is a + - branch, cache it. + - + - When on an adjusted branch, gets the original branch, and the adjustment. + -} +getCurrBranch :: Annex (Annex CurrBranch) +getCurrBranch = do + mvar <- liftIO newEmptyMVar + return $ ifM (liftIO $ isEmptyMVar mvar) + ( do + currbranch <- inRepo Git.Branch.current + case currbranch of + Nothing -> return (Nothing, Nothing) + Just b -> do + let v = case adjustedToOriginal b of + Nothing -> (Just b, Nothing) + Just (adj, origbranch) -> + (Just origbranch, Just adj) + liftIO $ putMVar mvar v + return v + , liftIO $ readMVar mvar + ) + +{- Merging may delete the current directory, so go to the top + - of the repo. This also means that sync always acts on all files in the + - repository, not just on a subdirectory. -} +prepMerge :: Annex () +prepMerge = Annex.changeDirectory =<< fromRepo Git.repoPath + +mergeConfig :: [Git.Merge.MergeConfig] +mergeConfig = + [ Git.Merge.MergeNonInteractive + -- In several situations, unrelated histories should be merged + -- together. This includes pairing in the assistant, and merging + -- from a remote into a newly created direct mode repo. + -- (Once direct mode is removed, this could be changed, so only + -- the assistant uses it.) + , Git.Merge.MergeUnrelatedHistories + ] + +merge :: CurrBranch -> [Git.Merge.MergeConfig] -> ResolveMergeOverride -> Git.Branch.CommitMode -> Git.Branch -> Annex Bool +merge currbranch mergeconfig resolvemergeoverride commitmode tomerge = case currbranch of + (Just b, Just adj) -> updateAdjustedBranch tomerge (b, adj) mergeconfig canresolvemerge commitmode + (b, _) -> autoMergeFrom tomerge b mergeconfig canresolvemerge commitmode + where + canresolvemerge = case resolvemergeoverride of + ResolveMergeOverride True -> getGitConfigVal annexResolveMerge + ResolveMergeOverride False -> return False + +syncBranch :: Git.Branch -> Git.Branch +syncBranch = Git.Ref.underBase "refs/heads/synced" . fromDirectBranch . fromAdjustedBranch + +remoteBranch :: Remote -> Git.Ref -> Git.Ref +remoteBranch remote = Git.Ref.underBase $ "refs/remotes/" ++ Remote.name remote + +-- Do automatic initialization of remotes when possible when getting remote +-- list. +syncRemotes :: [String] -> Annex [Remote] +syncRemotes ps = do + remotelist <- Remote.remoteList' True + available <- filterM (liftIO . getDynamicConfig . remoteAnnexSync . Remote.gitconfig) remotelist + syncRemotes' ps available + +syncRemotes' :: [String] -> [Remote] -> Annex [Remote] +syncRemotes' ps available = + ifM (Annex.getState Annex.fast) ( nub <$> pickfast , wanted ) + where + pickfast = (++) <$> listed <*> (filterM good (fastest available)) + + wanted + | null ps = filterM good (concat $ Remote.byCost available) + | otherwise = listed + + listed = concat <$> mapM Remote.byNameOrGroup ps + + good r + | Remote.gitSyncableRemote r = + Remote.Git.repoAvail =<< Remote.getRepo r + | otherwise = return True + + fastest = fromMaybe [] . headMaybe . Remote.byCost + +commit :: SyncOptions -> CommandStart +commit o = stopUnless shouldcommit $ next $ next $ do + commitmessage <- maybe commitMsg return (messageOption o) + showStart' "commit" Nothing + Annex.Branch.commit =<< Annex.Branch.commitMessage + ifM isDirect + ( do + void stageDirect + void preCommitDirect + commitStaged Git.Branch.ManualCommit commitmessage + , do + showOutput + void $ inRepo $ Git.Branch.commitCommand Git.Branch.ManualCommit + [ Param "-a" + , Param "-m" + , Param commitmessage + ] + return True + ) + where + shouldcommit = pure (commitOption o) + <||> (pure (not (noCommitOption o)) <&&> getGitConfigVal annexAutoCommit) + +commitMsg :: Annex String +commitMsg = do + u <- getUUID + m <- uuidMap + return $ "git-annex in " ++ fromMaybe "unknown" (M.lookup u m) + +commitStaged :: Git.Branch.CommitMode -> String -> Annex Bool +commitStaged commitmode commitmessage = do + runAnnexHook preCommitAnnexHook + mb <- inRepo Git.Branch.currentUnsafe + let (getparent, branch) = case mb of + Just b -> (Git.Ref.sha b, b) + Nothing -> (Git.Ref.headSha, Git.Ref.headRef) + parents <- maybeToList <$> inRepo getparent + void $ inRepo $ Git.Branch.commit commitmode False commitmessage branch parents + return True + +mergeLocal :: [Git.Merge.MergeConfig] -> ResolveMergeOverride -> CurrBranch -> CommandStart +mergeLocal mergeconfig resolvemergeoverride currbranch@(Just _, _) = + go =<< needMerge currbranch + where + go Nothing = stop + go (Just syncbranch) = do + showStart' "merge" (Just $ Git.Ref.describe syncbranch) + next $ next $ merge currbranch mergeconfig resolvemergeoverride Git.Branch.ManualCommit syncbranch +mergeLocal _ _ (Nothing, madj) = do + b <- inRepo Git.Branch.currentUnsafe + ifM (isJust <$> needMerge (b, madj)) + ( do + warning $ "There are no commits yet in the currently checked out branch, so cannot merge any remote changes into it." + next $ next $ return False + , stop + ) + +-- Returns the branch that should be merged, if any. +needMerge :: CurrBranch -> Annex (Maybe Git.Branch) +needMerge (Nothing, _) = return Nothing +needMerge (Just branch, madj) = ifM (allM id checks) + ( return (Just syncbranch) + , return Nothing + ) + where + checks = + [ not <$> isBareRepo + , inRepo (Git.Ref.exists syncbranch) + , inRepo (Git.Branch.changed branch' syncbranch) + ] + syncbranch = syncBranch branch + branch' = maybe branch (adjBranch . originalToAdjusted branch) madj + +pushLocal :: CurrBranch -> CommandStart +pushLocal b = do + updateSyncBranch b + stop + +updateSyncBranch :: CurrBranch -> Annex () +updateSyncBranch (Nothing, _) = noop +updateSyncBranch (Just branch, madj) = do + -- When in an adjusted branch, propigate any changes made to it + -- back to the original branch. + maybe noop (propigateAdjustedCommits branch) madj + -- Update the sync branch to match the new state of the branch + inRepo $ updateBranch (syncBranch branch) branch + -- In direct mode, we're operating on some special direct mode + -- branch, rather than the intended branch, so update the intended + -- branch. + whenM isDirect $ + inRepo $ updateBranch (fromDirectBranch branch) branch + +updateBranch :: Git.Branch -> Git.Branch -> Git.Repo -> IO () +updateBranch syncbranch updateto g = + unlessM go $ giveup $ "failed to update " ++ Git.fromRef syncbranch + where + go = Git.Command.runBool + [ Param "branch" + , Param "-f" + , Param $ Git.fromRef $ Git.Ref.base syncbranch + , Param $ Git.fromRef $ updateto + ] g + +pullRemote :: SyncOptions -> [Git.Merge.MergeConfig] -> Remote -> CurrBranch -> CommandStart +pullRemote o mergeconfig remote branch = stopUnless (pure $ pullOption o && wantpull) $ do + showStart' "pull" (Just (Remote.name remote)) + next $ do + showOutput + stopUnless fetch $ + next $ mergeRemote remote branch mergeconfig (resolveMergeOverride o) + where + fetch = do + repo <- Remote.getRepo remote + inRepoWithSshOptionsTo repo (Remote.gitconfig remote) $ + Git.Command.runBool + [Param "fetch", Param $ Remote.name remote] + wantpull = remoteAnnexPull (Remote.gitconfig remote) + +{- The remote probably has both a master and a synced/master branch. + - Which to merge from? Well, the master has whatever latest changes + - were committed (or pushed changes, if this is a bare remote), + - while the synced/master may have changes that some + - other remote synced to this remote. So, merge them both. -} +mergeRemote :: Remote -> CurrBranch -> [Git.Merge.MergeConfig] -> ResolveMergeOverride -> CommandCleanup +mergeRemote remote currbranch mergeconfig resolvemergeoverride = ifM isBareRepo + ( return True + , case currbranch of + (Nothing, _) -> do + branch <- inRepo Git.Branch.currentUnsafe + mergelisted (pure (branchlist branch)) + (Just branch, _) -> do + inRepo $ updateBranch (syncBranch branch) branch + mergelisted (tomerge (branchlist (Just branch))) + ) + where + mergelisted getlist = and <$> + (mapM (merge currbranch mergeconfig resolvemergeoverride Git.Branch.ManualCommit . remoteBranch remote) =<< getlist) + tomerge = filterM (changed remote) + branchlist Nothing = [] + branchlist (Just branch) = [fromDirectBranch (fromAdjustedBranch branch), syncBranch branch] + +pushRemote :: SyncOptions -> Remote -> CurrBranch -> CommandStart +pushRemote _o _remote (Nothing, _) = stop +pushRemote o remote (Just branch, _) = stopUnless (pure (pushOption o) <&&> needpush) $ do + showStart' "push" (Just (Remote.name remote)) + next $ next $ do + repo <- Remote.getRepo remote + showOutput + ok <- inRepoWithSshOptionsTo repo gc $ + pushBranch remote branch + if ok + then postpushupdate repo + else do + warning $ unwords [ "Pushing to " ++ Remote.name remote ++ " failed." ] + showLongNote "(non-fast-forward problems can be solved by setting receive.denyNonFastforwards to false in the remote's git config)" + return ok + where + gc = Remote.gitconfig remote + needpush + | remoteAnnexReadOnly gc = return False + | not (remoteAnnexPush gc) = return False + | otherwise = anyM (newer remote) [syncBranch branch, Annex.Branch.name] + -- Do updateInstead emulation for remotes on eg removable drives + -- formatted FAT, where the post-receive hook won't run. + postpushupdate repo = case Git.repoWorkTree repo of + Nothing -> return True + Just wt -> ifM needemulation + ( liftIO $ do + p <- readProgramFile + boolSystem' p [Param "post-receive"] + (\cp -> cp { cwd = Just wt }) + , return True + ) + where + needemulation = Remote.Git.onLocal repo remote $ + (annexCrippledFileSystem <$> Annex.getGitConfig) + <&&> + needUpdateInsteadEmulation + +{- Pushes a regular branch like master to a remote. Also pushes the git-annex + - branch. + - + - If the remote is a bare git repository, it's best to push the regular + - branch directly to it, so that cloning/pulling will get it. + - On the other hand, if it's not bare, pushing to the checked out branch + - will generally fail (except with receive.denyCurrentBranch=updateInstead), + - and this is why we push to its syncBranch. + - + - Git offers no way to tell if a remote is bare or not, so both methods + - are tried. + - + - The direct push is likely to spew an ugly error message, so its stderr is + - often elided. Since git progress display goes to stderr too, the + - sync push is done first, and actually sends the data. Then the + - direct push is tried, with stderr discarded, to update the branch ref + - on the remote. + - + - The sync push forces the update of the remote synced/git-annex branch. + - This is necessary if a transition has rewritten the git-annex branch. + - Normally any changes to the git-annex branch get pulled and merged before + - this push, so this forcing is unlikely to overwrite new data pushed + - in from another repository that is also syncing. + - + - But overwriting of data on synced/git-annex can happen, in a race. + - The only difference caused by using a forced push in that case is that + - the last repository to push wins the race, rather than the first to push. + - + - The sync push will fail to overwrite if receive.denyNonFastforwards is + - set on the remote. + -} +pushBranch :: Remote -> Git.Branch -> Git.Repo -> IO Bool +pushBranch remote branch g = directpush `after` annexpush `after` syncpush + where + syncpush = flip Git.Command.runBool g $ pushparams + [ Git.Branch.forcePush $ refspec Annex.Branch.name + , refspec $ fromAdjustedBranch branch + ] + annexpush = void $ tryIO $ flip Git.Command.runQuiet g $ pushparams + [ Git.fromRef $ Git.Ref.base $ Annex.Branch.name ] + directpush = do + -- Git prints out an error message when this fails. + -- In the default configuration of receive.denyCurrentBranch, + -- the error message mentions that config setting + -- (and should even if it is localized), and is quite long, + -- and the user was not intending to update the checked out + -- branch, so in that case, avoid displaying the error + -- message. Do display other error messages though, + -- including the error displayed when + -- receive.denyCurrentBranch=updateInstead -- the user + -- will want to see that one. + let p = flip Git.Command.gitCreateProcess g $ pushparams + [ Git.fromRef $ Git.Ref.base $ fromDirectBranch $ fromAdjustedBranch branch ] + (transcript, ok) <- processTranscript' p Nothing + when (not ok && not ("denyCurrentBranch" `isInfixOf` transcript)) $ + hPutStr stderr transcript + pushparams branches = + [ Param "push" + , Param $ Remote.name remote + ] ++ map Param branches + refspec b = concat + [ Git.fromRef $ Git.Ref.base b + , ":" + , Git.fromRef $ Git.Ref.base $ syncBranch b + ] + +commitAnnex :: CommandStart +commitAnnex = do + Annex.Branch.commit =<< Annex.Branch.commitMessage + stop + +mergeAnnex :: CommandStart +mergeAnnex = do + void Annex.Branch.forceUpdate + stop + +changed :: Remote -> Git.Ref -> Annex Bool +changed remote b = do + let r = remoteBranch remote b + ifM (inRepo $ Git.Ref.exists r) + ( inRepo $ Git.Branch.changed b r + , return False + ) + +newer :: Remote -> Git.Ref -> Annex Bool +newer remote b = do + let r = remoteBranch remote b + ifM (inRepo $ Git.Ref.exists r) + ( inRepo $ Git.Branch.changed r b + , return True + ) + +{- Without --all, only looks at files in the work tree. With --all, + - makes 2 passes, first looking at the work tree and then all keys. + - This ensures that preferred content expressions that match on + - filenames work, even when in --all mode. + - + - Returns true if any file transfers were made. + - + - When concurrency is enabled, files are processed concurrently. + -} +seekSyncContent :: SyncOptions -> [Remote] -> Annex Bool +seekSyncContent o rs = do + mvar <- liftIO newEmptyMVar + bloom <- case keyOptions o of + Just WantAllKeys -> Just <$> genBloomFilter (seekworktree mvar []) + _ -> do + l <- workTreeItems (contentOfOption o) + seekworktree mvar l (const noop) + pure Nothing + withKeyOptions' (keyOptions o) False + (return (seekkeys mvar bloom)) + (const noop) + [] + finishCommandActions + liftIO $ not <$> isEmptyMVar mvar + where + seekworktree mvar l bloomfeeder = seekHelper LsFiles.inRepo l >>= + mapM_ (\f -> ifAnnexed f (go (Right bloomfeeder) mvar (AssociatedFile (Just f))) noop) + seekkeys mvar bloom (k, _) = go (Left bloom) mvar (AssociatedFile Nothing) k + go ebloom mvar af k = commandAction $ do + whenM (syncFile ebloom rs af k) $ + void $ liftIO $ tryPutMVar mvar () + return Nothing + +{- If it's preferred content, and we don't have it, get it from one of the + - listed remotes (preferring the cheaper earlier ones). + - + - Send it to each remote that doesn't have it, and for which it's + - preferred content. + - + - Drop it locally if it's not preferred content (honoring numcopies). + - + - Drop it from each remote that has it, where it's not preferred content + - (honoring numcopies). + - + - Returns True if any file transfers were made. + -} +syncFile :: Either (Maybe (Bloom Key)) (Key -> Annex ()) -> [Remote] -> AssociatedFile -> Key -> Annex Bool +syncFile ebloom rs af k = onlyActionOn' k $ do + locs <- map Remote.uuid <$> Remote.keyPossibilities k + let (have, lack) = partition (\r -> Remote.uuid r `elem` locs) rs + + got <- anyM id =<< handleget have + putrs <- handleput lack + + u <- getUUID + let locs' = concat [[u | got], putrs, locs] + + -- A bloom filter is populated with all the keys in the first pass. + -- On the second pass, avoid dropping keys that were seen in the + -- first pass, which would happen otherwise when preferred content + -- matches on the filename, which is not available in the second + -- pass. + -- + -- When there's a false positive in the bloom filter, the result + -- is keeping a key that preferred content doesn't really want. + seenbloom <- case ebloom of + Left Nothing -> pure False + Left (Just bloom) -> pure (elemB k bloom) + Right bloomfeeder -> bloomfeeder k >> return False + unless seenbloom $ + -- Using callCommandAction rather than + -- includeCommandAction for drops, + -- because a failure to drop does not mean + -- the sync failed. + handleDropsFrom locs' rs "unwanted" True k af [] + callCommandAction + + return (got || not (null putrs)) + where + wantget have = allM id + [ pure (not $ null have) + , not <$> inAnnex k + , wantGet True (Just k) af + ] + handleget have = ifM (wantget have) + ( return [ get have ] + , return [] + ) + get have = includeCommandAction $ do + showStartKey "get" k (mkActionItem af) + next $ next $ getKey' k af have + + wantput r + | Remote.readonly r || remoteAnnexReadOnly (Remote.gitconfig r) = return False + | otherwise = wantSend True (Just k) af (Remote.uuid r) + handleput lack = catMaybes <$> ifM (inAnnex k) + ( forM lack $ \r -> + ifM (wantput r <&&> put r) + ( return (Just (Remote.uuid r)) + , return Nothing + ) + , return [] + ) + put dest = includeCommandAction $ + Command.Move.toStart' dest Command.Move.RemoveNever af k (mkActionItem af) + +{- When a remote has an export-tracking branch, change the export to + - follow the current content of the branch. Otherwise, transfer any files + - that were part of an export but are not in the remote yet. + - + - Returns True if any file transfers were made. + -} +seekExportContent :: [Remote] -> CurrBranch -> Annex Bool +seekExportContent rs (currbranch, _) = or <$> forM rs go + where + go r = withExclusiveLock (gitAnnexExportLock (Remote.uuid r)) $ do + db <- Export.openDb (Remote.uuid r) + ea <- Remote.exportActions r + exported <- case remoteAnnexExportTracking (Remote.gitconfig r) of + Nothing -> nontracking r + Just b -> do + mcur <- inRepo $ Git.Ref.tree b + case mcur of + Nothing -> nontracking r + Just cur -> do + Command.Export.changeExport r ea db cur + return [Exported cur []] + Export.closeDb db `after` fillexport r ea db exported + + nontracking r = do + exported <- getExport (Remote.uuid r) + maybe noop (warnnontracking r exported) currbranch + return exported + + warnnontracking r exported currb = inRepo (Git.Ref.tree currb) >>= \case + Just currt | not (any (\ex -> exportedTreeish ex == currt) exported) -> + showLongNote $ unwords + [ "Not updating export to " ++ Remote.name r + , "to reflect changes to the tree, because export" + , "tracking is not enabled. " + , "(Use git-annex export's --tracking option" + , "to enable it.)" + ] + _ -> noop + + + fillexport _ _ _ [] = return False + fillexport r ea db (Exported { exportedTreeish = t }:[]) = + Command.Export.fillExport r ea db t + fillexport r _ _ _ = do + warning $ "Export conflict detected. Different trees have been exported to " ++ + Remote.name r ++ + ". Use git-annex export to resolve this conflict." + return False + +cleanupLocal :: CurrBranch -> CommandStart +cleanupLocal (Nothing, _) = stop +cleanupLocal (Just currb, _) = do + showStart' "cleanup" (Just "local") + next $ next $ do + delbranch $ syncBranch currb + delbranch $ syncBranch $ Git.Ref.base $ Annex.Branch.name + mapM_ (\(s,r) -> inRepo $ Git.Ref.delete s r) + =<< listTaggedBranches + return True + where + delbranch b = whenM (inRepo $ Git.Ref.exists $ Git.Ref.branchRef b) $ + inRepo $ Git.Branch.delete b + +cleanupRemote :: Remote -> CurrBranch -> CommandStart +cleanupRemote _ (Nothing, _) = stop +cleanupRemote remote (Just b, _) = do + showStart' "cleanup" (Just (Remote.name remote)) + next $ next $ + inRepo $ Git.Command.runBool + [ Param "push" + , Param "--quiet" + , Param "--delete" + , Param $ Remote.name remote + , Param $ Git.fromRef $ syncBranch b + , Param $ Git.fromRef $ syncBranch $ + Git.Ref.base $ Annex.Branch.name + ] diff --git a/Command/Test.hs b/Command/Test.hs new file mode 100644 index 0000000000..5180f96edb --- /dev/null +++ b/Command/Test.hs @@ -0,0 +1,30 @@ +{- git-annex command + - + - Copyright 2013-2015 Joey Hess + - + - Licensed under the GNU GPL version 3 or higher. + -} + +module Command.Test where + +import Command +import Types.Test + +cmd :: Parser TestOptions -> Maybe TestRunner -> Command +cmd optparser runner = noRepo (startIO runner <$$> const optparser) $ + dontCheck repoExists $ + command "test" SectionTesting + "run built-in test suite" + paramNothing (seek runner <$$> const optparser) + +seek :: Maybe TestRunner -> TestOptions -> CommandSeek +seek runner o = commandAction $ start runner o + +start :: Maybe TestRunner -> TestOptions -> CommandStart +start runner o = do + liftIO $ startIO runner o + stop + +startIO :: Maybe TestRunner -> TestOptions -> IO () +startIO Nothing _ = warningIO "git-annex was built without its test suite; not testing" +startIO (Just runner) o = runner o diff --git a/Command/TestRemote.hs b/Command/TestRemote.hs new file mode 100644 index 0000000000..baffbe131c --- /dev/null +++ b/Command/TestRemote.hs @@ -0,0 +1,299 @@ +{- git-annex command + - + - Copyright 2014 Joey Hess + - + - Licensed under the GNU GPL version 3 or higher. + -} + +module Command.TestRemote where + +import Command +import qualified Annex +import qualified Remote +import qualified Types.Remote as Remote +import qualified Types.Backend as Backend +import Types.KeySource +import Annex.Content +import Backend +import qualified Backend.Hash +import Utility.Tmp +import Utility.Metered +import Utility.DataUnits +import Utility.CopyFile +import Types.Messages +import Types.Export +import Remote.Helper.Export +import Remote.Helper.Chunked +import Git.Types + +import Test.Tasty +import Test.Tasty.Runners +import Test.Tasty.HUnit +import "crypto-api" Crypto.Random +import qualified Data.ByteString as B +import qualified Data.ByteString.Lazy as L +import qualified Data.Map as M + +cmd :: Command +cmd = command "testremote" SectionTesting + "test transfers to/from a remote" + paramRemote (seek <$$> optParser) + +data TestRemoteOptions = TestRemoteOptions + { testRemote :: RemoteName + , sizeOption :: ByteSize + } + +optParser :: CmdParamsDesc -> Parser TestRemoteOptions +optParser desc = TestRemoteOptions + <$> argument str ( metavar desc ) + <*> option (str >>= maybe (fail "parse error") return . readSize dataUnits) + ( long "size" <> metavar paramSize + <> value (1024 * 1024) + <> help "base key size (default 1MiB)" + ) + +seek :: TestRemoteOptions -> CommandSeek +seek o = commandAction $ start (fromInteger $ sizeOption o) (testRemote o) + +start :: Int -> RemoteName -> CommandStart +start basesz name = do + showStart' "testremote" (Just name) + fast <- Annex.getState Annex.fast + r <- either giveup disableExportTree =<< Remote.byName' name + rs <- catMaybes <$> mapM (adjustChunkSize r) (chunkSizes basesz fast) + rs' <- concat <$> mapM encryptionVariants rs + unavailrs <- catMaybes <$> mapM Remote.mkUnavailable [r] + exportr <- exportTreeVariant r + showAction "generating test keys" + ks <- mapM randKey (keySizes basesz fast) + next $ perform rs' unavailrs exportr ks + +perform :: [Remote] -> [Remote] -> Maybe Remote -> [Key] -> CommandPerform +perform rs unavailrs exportr ks = do + ea <- maybe exportUnsupported Remote.exportActions exportr + st <- Annex.getState id + let tests = testGroup "Remote Tests" $ concat + [ [ testGroup "unavailable remote" (testUnavailable st r (Prelude.head ks)) | r <- unavailrs ] + , [ testGroup (desc r k) (test st r k) | k <- ks, r <- rs ] + , [ testGroup (descexport k1 k2) (testExportTree st exportr ea k1 k2) | k1 <- take 2 ks, k2 <- take 2 (reverse ks) ] + ] + ok <- case tryIngredients [consoleTestReporter] mempty tests of + Nothing -> error "No tests found!?" + Just act -> liftIO act + next $ cleanup rs ks ok + where + desc r' k = intercalate "; " $ map unwords + [ [ "key size", show (keySize k) ] + , [ show (getChunkConfig (Remote.config r')) ] + , ["encryption", fromMaybe "none" (M.lookup "encryption" (Remote.config r'))] + ] + descexport k1 k2 = intercalate "; " $ map unwords + [ [ "exporttree=yes" ] + , [ "key1 size", show (keySize k1) ] + , [ "key2 size", show (keySize k2) ] + ] + +adjustChunkSize :: Remote -> Int -> Annex (Maybe Remote) +adjustChunkSize r chunksize = adjustRemoteConfig r + (M.insert "chunk" (show chunksize)) + +-- Variants of a remote with no encryption, and with simple shared +-- encryption. Gpg key based encryption is not tested. +encryptionVariants :: Remote -> Annex [Remote] +encryptionVariants r = do + noenc <- adjustRemoteConfig r (M.insert "encryption" "none") + sharedenc <- adjustRemoteConfig r $ + M.insert "encryption" "shared" . + M.insert "highRandomQuality" "false" + return $ catMaybes [noenc, sharedenc] + +-- Variant of a remote with exporttree disabled. +disableExportTree :: Remote -> Annex Remote +disableExportTree r = maybe (error "failed disabling exportree") return + =<< adjustRemoteConfig r (M.delete "exporttree") + +-- Variant of a remote with exporttree enabled. +exportTreeVariant :: Remote -> Annex (Maybe Remote) +exportTreeVariant r = ifM (Remote.isExportSupported r) + ( adjustRemoteConfig r $ + M.insert "encryption" "none" . M.insert "exporttree" "yes" + , return Nothing + ) + +-- Regenerate a remote with a modified config. +adjustRemoteConfig :: Remote -> (Remote.RemoteConfig -> Remote.RemoteConfig) -> Annex (Maybe Remote) +adjustRemoteConfig r adjustconfig = do + repo <- Remote.getRepo r + Remote.generate (Remote.remotetype r) + repo + (Remote.uuid r) + (adjustconfig (Remote.config r)) + (Remote.gitconfig r) + +test :: Annex.AnnexState -> Remote -> Key -> [TestTree] +test st r k = + [ check "removeKey when not present" remove + , present False + , check "storeKey" store + , present True + , check "storeKey when already present" store + , present True + , check "retrieveKeyFile" $ do + lockContentForRemoval k removeAnnex + get + , check "fsck downloaded object" fsck + , check "retrieveKeyFile resume from 33%" $ do + loc <- Annex.calcRepo (gitAnnexLocation k) + tmp <- prepTmp k + partial <- liftIO $ bracket (openBinaryFile loc ReadMode) hClose $ \h -> do + sz <- hFileSize h + L.hGet h $ fromInteger $ sz `div` 3 + liftIO $ L.writeFile tmp partial + lockContentForRemoval k removeAnnex + get + , check "fsck downloaded object" fsck + , check "retrieveKeyFile resume from 0" $ do + tmp <- prepTmp k + liftIO $ writeFile tmp "" + lockContentForRemoval k removeAnnex + get + , check "fsck downloaded object" fsck + , check "retrieveKeyFile resume from end" $ do + loc <- Annex.calcRepo (gitAnnexLocation k) + tmp <- prepTmp k + void $ liftIO $ copyFileExternal CopyAllMetaData loc tmp + lockContentForRemoval k removeAnnex + get + , check "fsck downloaded object" fsck + , check "removeKey when present" remove + , present False + ] + where + check desc a = testCase desc $ + Annex.eval st (Annex.setOutput QuietOutput >> a) @? "failed" + present b = check ("present " ++ show b) $ + (== Right b) <$> Remote.hasKey r k + fsck = case maybeLookupBackendVariety (keyVariety k) of + Nothing -> return True + Just b -> case Backend.verifyKeyContent b of + Nothing -> return True + Just verifier -> verifier k (key2file k) + get = getViaTmp (Remote.retrievalSecurityPolicy r) (RemoteVerify r) k $ \dest -> + Remote.retrieveKeyFile r k (AssociatedFile Nothing) + dest nullMeterUpdate + store = Remote.storeKey r k (AssociatedFile Nothing) nullMeterUpdate + remove = Remote.removeKey r k + +testExportTree :: Annex.AnnexState -> Maybe Remote -> Remote.ExportActions Annex -> Key -> Key -> [TestTree] +testExportTree _ Nothing _ _ _ = [] +testExportTree st (Just _) ea k1 k2 = + [ check "check present export when not present" $ + not <$> checkpresentexport k1 + , check "remove export when not present" (removeexport k1) + , check "store export" (storeexport k1) + , check "check present export after store" $ + checkpresentexport k1 + , check "store export when already present" (storeexport k1) + , check "retrieve export" (retrieveexport k1) + , check "store new content to export" (storeexport k2) + , check "check present export after store of new content" $ + checkpresentexport k2 + , check "retrieve export new content" (retrieveexport k2) + , check "remove export" (removeexport k2) + , check "check present export after remove" $ + not <$> checkpresentexport k2 + , check "retrieve export fails after removal" $ + not <$> retrieveexport k2 + , check "remove export directory" removeexportdirectory + , check "remove export directory that is already removed" removeexportdirectory + -- renames are not tested because remotes do not need to support them + ] + where + testexportdirectory = "testremote-export" + testexportlocation = mkExportLocation (testexportdirectory "location") + check desc a = testCase desc $ + Annex.eval st (Annex.setOutput QuietOutput >> a) @? "failed" + storeexport k = do + loc <- Annex.calcRepo (gitAnnexLocation k) + Remote.storeExport ea loc k testexportlocation nullMeterUpdate + retrieveexport k = withTmpFile "exported" $ \tmp h -> do + liftIO $ hClose h + ifM (Remote.retrieveExport ea k testexportlocation tmp nullMeterUpdate) + ( verifyKeyContent RetrievalAllKeysSecure AlwaysVerify UnVerified k tmp + , return False + ) + checkpresentexport k = Remote.checkPresentExport ea k testexportlocation + removeexport k = Remote.removeExport ea k testexportlocation + removeexportdirectory = case Remote.removeExportDirectory ea of + Nothing -> return True + Just a -> a (mkExportDirectory testexportdirectory) + +testUnavailable :: Annex.AnnexState -> Remote -> Key -> [TestTree] +testUnavailable st r k = + [ check (== Right False) "removeKey" $ + Remote.removeKey r k + , check (== Right False) "storeKey" $ + Remote.storeKey r k (AssociatedFile Nothing) nullMeterUpdate + , check (`notElem` [Right True, Right False]) "checkPresent" $ + Remote.checkPresent r k + , check (== Right False) "retrieveKeyFile" $ + getViaTmp (Remote.retrievalSecurityPolicy r) (RemoteVerify r) k $ \dest -> + Remote.retrieveKeyFile r k (AssociatedFile Nothing) dest nullMeterUpdate + , check (== Right False) "retrieveKeyFileCheap" $ + getViaTmp (Remote.retrievalSecurityPolicy r) (RemoteVerify r) k $ \dest -> unVerified $ + Remote.retrieveKeyFileCheap r k (AssociatedFile Nothing) dest + ] + where + check checkval desc a = testCase desc $ do + v <- Annex.eval st $ do + Annex.setOutput QuietOutput + either (Left . show) Right <$> tryNonAsync a + checkval v @? ("(got: " ++ show v ++ ")") + +cleanup :: [Remote] -> [Key] -> Bool -> CommandCleanup +cleanup rs ks ok = do + forM_ rs $ \r -> forM_ ks (Remote.removeKey r) + forM_ ks $ \k -> lockContentForRemoval k removeAnnex + return ok + +chunkSizes :: Int -> Bool -> [Int] +chunkSizes base False = + [ 0 -- no chunking + , base `div` 100 + , base `div` 1000 + , base + ] +chunkSizes _ True = + [ 0 + ] + +keySizes :: Int -> Bool -> [Int] +keySizes base fast = filter want + [ 0 -- empty key is a special case when chunking + , base + , base + 1 + , base - 1 + , base * 2 + ] + where + want sz + | fast = sz <= base && sz > 0 + | otherwise = sz > 0 + +randKey :: Int -> Annex Key +randKey sz = withTmpFile "randkey" $ \f h -> do + gen <- liftIO (newGenIO :: IO SystemRandom) + case genBytes sz gen of + Left e -> error $ "failed to generate random key: " ++ show e + Right (rand, _) -> liftIO $ B.hPut h rand + liftIO $ hClose h + let ks = KeySource + { keyFilename = f + , contentLocation = f + , inodeCache = Nothing + } + k <- fromMaybe (error "failed to generate random key") + <$> Backend.getKey Backend.Hash.testKeyBackend ks + _ <- moveAnnex k f + return k diff --git a/Command/TransferInfo.hs b/Command/TransferInfo.hs new file mode 100644 index 0000000000..2278121424 --- /dev/null +++ b/Command/TransferInfo.hs @@ -0,0 +1,67 @@ +{- git-annex command + - + - Copyright 2012 Joey Hess + - + - Licensed under the GNU GPL version 3 or higher. + -} + +module Command.TransferInfo where + +import Command +import Annex.Content +import Types.Transfer +import Logs.Transfer +import qualified CmdLine.GitAnnexShell.Fields as Fields +import Utility.Metered +import Utility.SimpleProtocol + +cmd :: Command +cmd = noCommit $ + command "transferinfo" SectionPlumbing + "updates sender on number of bytes of content received" + paramKey (withParams seek) + +seek :: CmdParams -> CommandSeek +seek = withWords (commandAction . start) + +{- Security: + - + - The transfer info file contains the user-supplied key, but + - the built-in guards prevent slashes in it from showing up in the filename. + - It also contains the UUID of the remote. But slashes are also filtered + - out of that when generating the filename. + - + - Checks that the key being transferred is inAnnex, to prevent + - malicious spamming of bogus keys. Does not check that a transfer + - of the key is actually in progress, because this could be started + - concurrently with sendkey, and win the race. + -} +start :: [String] -> CommandStart +start (k:[]) = do + case file2key k of + Nothing -> error "bad key" + (Just key) -> whenM (inAnnex key) $ do + afile <- AssociatedFile <$> Fields.getField Fields.associatedFile + u <- maybe (error "missing remoteuuid") toUUID + <$> Fields.getField Fields.remoteUUID + let t = Transfer + { transferDirection = Upload + , transferUUID = u + , transferKey = key + } + tinfo <- liftIO $ startTransferInfo afile + (update, tfile, createtfile, _) <- mkProgressUpdater t tinfo + createtfile + liftIO $ mapM_ void + [ tryIO $ forever $ do + bytes <- readUpdate + maybe (error "transferinfo protocol error") + (update . toBytesProcessed) bytes + , tryIO $ removeFile tfile + , exitSuccess + ] + stop +start _ = giveup "wrong number of parameters" + +readUpdate :: IO (Maybe Integer) +readUpdate = maybe Nothing readish <$> getProtocolLine stdin diff --git a/Command/TransferKey.hs b/Command/TransferKey.hs new file mode 100644 index 0000000000..9e5f684014 --- /dev/null +++ b/Command/TransferKey.hs @@ -0,0 +1,68 @@ +{- git-annex plumbing command (for use by old assistant, and users) + - + - Copyright 2012 Joey Hess + - + - Licensed under the GNU GPL version 3 or higher. + -} + +module Command.TransferKey where + +import Command +import Annex.Content +import Logs.Location +import Annex.Transfer +import qualified Remote +import Types.Remote + +cmd :: Command +cmd = noCommit $ + command "transferkey" SectionPlumbing + "transfers a key from or to a remote" + paramKey (seek <--< optParser) + +data TransferKeyOptions = TransferKeyOptions + { keyOptions :: CmdParams + , fromToOptions :: FromToOptions [DeferredParse Remote] + , fileOption :: AssociatedFile + } + +optParser :: CmdParamsDesc -> Parser TransferKeyOptions +optParser desc = TransferKeyOptions + <$> cmdParams desc + <*> parseFromToOptions + <*> (AssociatedFile <$> optional (strOption + ( long "file" <> metavar paramFile + <> help "the associated file" + ))) + +instance DeferredParseClass TransferKeyOptions where + finishParse v = TransferKeyOptions + <$> pure (keyOptions v) + <*> finishParse (fromToOptions v) + <*> pure (fileOption v) + +seek :: TransferKeyOptions -> CommandSeek +seek o = withKeys go (keyOptions o) + where + go k = case fromToOptions o of + To dest -> commandActions =<< + (map (toStart k (fileOption o)) <$> mapM getParsed dest) + From src -> commandActions =<< + (map (fromStart k (fileOption o)) <$> mapM getParsed src) + +toStart :: Key -> AssociatedFile -> Remote -> CommandStart +toStart key file remote = next $ perform Upload file $ + upload (uuid remote) key file stdRetry $ \p -> do + ok <- Remote.storeKey remote key file p + when ok $ + Remote.logStatus remote key InfoPresent + return ok + +fromStart :: Key -> AssociatedFile -> Remote -> CommandStart +fromStart key file remote = next $ perform Upload file $ + download (uuid remote) key file stdRetry $ \p -> + getViaTmp (retrievalSecurityPolicy remote) (RemoteVerify remote) key $ + \t -> Remote.retrieveKeyFile remote key file t p + +perform :: Direction -> AssociatedFile -> (NotifyWitness -> Annex Bool) -> CommandPerform +perform direction file a = notifyTransfer direction file a >>= liftIO . exitBool diff --git a/Command/TransferKeys.hs b/Command/TransferKeys.hs new file mode 100644 index 0000000000..cd841f2819 --- /dev/null +++ b/Command/TransferKeys.hs @@ -0,0 +1,130 @@ +{- git-annex command, used internally by assistant + - + - Copyright 2012, 2013 Joey Hess + - + - Licensed under the GNU GPL version 3 or higher. + -} + +{-# LANGUAGE TypeSynonymInstances, FlexibleInstances #-} + +module Command.TransferKeys where + +import Command +import Annex.Content +import Logs.Location +import Annex.Transfer +import qualified Remote +import Utility.SimpleProtocol (dupIoHandles) +import Git.Types (RemoteName) +import qualified Database.Keys + +data TransferRequest = TransferRequest Direction Remote Key AssociatedFile + +cmd :: Command +cmd = command "transferkeys" SectionPlumbing "transfers keys" + paramNothing (withParams seek) + +seek :: CmdParams -> CommandSeek +seek = withNothing (commandAction start) + +start :: CommandStart +start = do + (readh, writeh) <- liftIO dupIoHandles + runRequests readh writeh runner + stop + where + runner (TransferRequest direction remote key file) + | direction == Upload = notifyTransfer direction file $ + upload (Remote.uuid remote) key file stdRetry $ \p -> do + ok <- Remote.storeKey remote key file p + when ok $ + Remote.logStatus remote key InfoPresent + return ok + | otherwise = notifyTransfer direction file $ + download (Remote.uuid remote) key file stdRetry $ \p -> + getViaTmp (Remote.retrievalSecurityPolicy remote) (RemoteVerify remote) key $ \t -> do + r <- Remote.retrieveKeyFile remote key file t p + -- Make sure we get the current + -- associated files data for the key, + -- not old cached data. + Database.Keys.closeDb + return r + +runRequests + :: Handle + -> Handle + -> (TransferRequest -> Annex Bool) + -> Annex () +runRequests readh writeh a = do + liftIO $ hSetBuffering readh NoBuffering + go =<< readrequests + where + go (d:rn:k:f:rest) = do + case (deserialize d, deserialize rn, deserialize k, deserialize f) of + (Just direction, Just remotename, Just key, Just file) -> do + mremote <- Remote.byName' remotename + case mremote of + Left _ -> sendresult False + Right remote -> sendresult =<< a + (TransferRequest direction remote key file) + _ -> sendresult False + go rest + go [] = noop + go [""] = noop + go v = error $ "transferkeys protocol error: " ++ show v + + readrequests = liftIO $ split fieldSep <$> hGetContents readh + sendresult b = liftIO $ do + hPutStrLn writeh $ serialize b + hFlush writeh + +sendRequest :: Transfer -> TransferInfo -> Handle -> IO () +sendRequest t tinfo h = do + hPutStr h $ intercalate fieldSep + [ serialize (transferDirection t) + , maybe (serialize (fromUUID (transferUUID t))) + (serialize . Remote.name) + (transferRemote tinfo) + , serialize (transferKey t) + , serialize (associatedFile tinfo) + , "" -- adds a trailing null + ] + hFlush h + +readResponse :: Handle -> IO Bool +readResponse h = fromMaybe False . deserialize <$> hGetLine h + +fieldSep :: String +fieldSep = "\0" + +class TCSerialized a where + serialize :: a -> String + deserialize :: String -> Maybe a + +instance TCSerialized Bool where + serialize True = "1" + serialize False = "0" + deserialize "1" = Just True + deserialize "0" = Just False + deserialize _ = Nothing + +instance TCSerialized Direction where + serialize Upload = "u" + serialize Download = "d" + deserialize "u" = Just Upload + deserialize "d" = Just Download + deserialize _ = Nothing + +instance TCSerialized AssociatedFile where + serialize (AssociatedFile (Just f)) = f + serialize (AssociatedFile Nothing) = "" + deserialize "" = Just (AssociatedFile Nothing) + deserialize f = Just (AssociatedFile (Just f)) + +instance TCSerialized RemoteName where + serialize n = n + deserialize n = Just n + +instance TCSerialized Key where + serialize = key2file + deserialize = file2key diff --git a/Command/Trust.hs b/Command/Trust.hs new file mode 100644 index 0000000000..2aaac3f0a8 --- /dev/null +++ b/Command/Trust.hs @@ -0,0 +1,40 @@ +{- git-annex command + - + - Copyright 2010, 2014 Joey Hess + - + - Licensed under the GNU GPL version 3 or higher. + -} + +module Command.Trust where + +import Command +import qualified Remote +import Types.TrustLevel +import Logs.Trust +import Logs.Group + +import qualified Data.Set as S + +cmd :: Command +cmd = command "trust" SectionSetup "trust a repository" + (paramRepeating paramRemote) (withParams seek) + +seek :: CmdParams -> CommandSeek +seek = trustCommand "trust" Trusted + +trustCommand :: String -> TrustLevel -> CmdParams -> CommandSeek +trustCommand c level = withWords (commandAction . start) + where + start ws = do + let name = unwords ws + showStart' c (Just name) + u <- Remote.nameToUUID name + next $ perform u + perform uuid = do + trustSet uuid level + when (level == DeadTrusted) $ + groupSet uuid S.empty + l <- lookupTrust uuid + when (l /= level) $ + warning $ "This remote's trust level is overridden to " ++ showTrustLevel l ++ "." + next $ return True diff --git a/Command/Unannex.hs b/Command/Unannex.hs new file mode 100644 index 0000000000..d7ace68ae1 --- /dev/null +++ b/Command/Unannex.hs @@ -0,0 +1,138 @@ +{- git-annex command + - + - Copyright 2010-2013 Joey Hess + - + - Licensed under the GNU GPL version 3 or higher. + -} + +module Command.Unannex where + +import Command +import Config +import qualified Annex +import Annex.Content +import Annex.Perms +import Annex.Content.Direct +import Annex.Version +import qualified Git.Command +import qualified Git.Branch +import qualified Git.Ref +import qualified Git.DiffTree as DiffTree +import Utility.CopyFile +import Command.PreCommit (lockPreCommitHook) +import qualified Database.Keys +import Git.FilePath + +cmd :: Command +cmd = withGlobalOptions [annexedMatchingOptions] $ + command "unannex" SectionUtility + "undo accidental add command" + paramPaths (withParams seek) + +seek :: CmdParams -> CommandSeek +seek ps = wrapUnannex $ + (withFilesInGit $ commandAction . whenAnnexed start) =<< workTreeItems ps + +wrapUnannex :: Annex a -> Annex a +wrapUnannex a = ifM (versionSupportsUnlockedPointers <||> isDirect) + ( a + {- Run with the pre-commit hook disabled, to avoid confusing + - behavior if an unannexed file is added back to git as + - a normal, non-annexed file and then committed. + - Otherwise, the pre-commit hook would think that the file + - has been unlocked and needs to be re-annexed. + - + - At the end, make a commit removing the unannexed files. + -} + , ifM cleanindex + ( lockPreCommitHook $ commit `after` a + , giveup "Cannot proceed with uncommitted changes staged in the index. Recommend you: git commit" + ) + ) + where + commit = inRepo $ Git.Branch.commitCommand Git.Branch.ManualCommit + [ Param "-q" + , Param "--allow-empty" + , Param "--no-verify" + , Param "-m", Param "content removed from git annex" + ] + cleanindex = ifM (inRepo Git.Ref.headExists) + ( do + (diff, cleanup) <- inRepo $ DiffTree.diffIndex Git.Ref.headRef + if null diff + then void (liftIO cleanup) >> return True + else void (liftIO cleanup) >> return False + , return False + ) + +start :: FilePath -> Key -> CommandStart +start file key = stopUnless (inAnnex key) $ do + showStart "unannex" file + next $ ifM isDirect + ( performDirect file key + , performIndirect file key) + +performIndirect :: FilePath -> Key -> CommandPerform +performIndirect file key = do + liftIO $ removeFile file + inRepo $ Git.Command.run + [ Param "rm" + , Param "--cached" + , Param "--force" + , Param "--quiet" + , Param "--" + , File file + ] + next $ cleanupIndirect file key + +cleanupIndirect :: FilePath -> Key -> CommandCleanup +cleanupIndirect file key = do + Database.Keys.removeAssociatedFile key =<< inRepo (toTopFilePath file) + src <- calcRepo $ gitAnnexLocation key + ifM (Annex.getState Annex.fast) + ( do + -- Only make a hard link if the annexed file does not + -- already have other hard links pointing at it. + -- This avoids unannexing (and uninit) ending up + -- hard linking files together, which would be + -- surprising. + s <- liftIO $ getFileStatus src + if linkCount s > 1 + then copyfrom src + else hardlinkfrom src + , copyfrom src + ) + where + copyfrom src = + thawContent file `after` liftIO (copyFileExternal CopyAllMetaData src file) + hardlinkfrom src = + -- creating a hard link could fall; fall back to copying + ifM (liftIO $ catchBoolIO $ createLink src file >> return True) + ( return True + , copyfrom src + ) + +performDirect :: FilePath -> Key -> CommandPerform +performDirect file key = do + -- --force is needed when the file is not committed + inRepo $ Git.Command.run + [ Param "rm" + , Param "--cached" + , Param "--force" + , Param "--quiet" + , Param "--" + , File file + ] + next $ cleanupDirect file key + +{- The direct mode file is not touched during unannex, so the content + - is already where it needs to be, so this does not need to do anything + - except remove it from the associated file map (which also updates + - the location log if this was the last copy), and, if this was the last + - associated file, remove the inode cache. -} +cleanupDirect :: FilePath -> Key -> CommandCleanup +cleanupDirect file key = do + fs <- removeAssociatedFile key file + when (null fs) $ + removeInodeCache key + return True diff --git a/Command/Undo.hs b/Command/Undo.hs new file mode 100644 index 0000000000..c755807947 --- /dev/null +++ b/Command/Undo.hs @@ -0,0 +1,84 @@ +{- git-annex command + - + - Copyright 2014 Joey Hess + - + - Licensed under the GNU GPL version 3 or higher. + -} + +module Command.Undo where + +import Command +import Config +import Annex.Direct +import Annex.CatFile +import Git.DiffTree +import Git.FilePath +import Git.UpdateIndex +import Git.Sha +import qualified Git.LsFiles as LsFiles +import qualified Git.Command as Git +import qualified Git.Branch +import qualified Command.Sync + +cmd :: Command +cmd = notBareRepo $ + command "undo" SectionCommon + "undo last change to a file or directory" + paramPaths (withParams seek) + +seek :: CmdParams -> CommandSeek +seek ps = do + -- Safety first; avoid any undo that would touch files that are not + -- in the index. + (fs, cleanup) <- inRepo $ LsFiles.notInRepo False ps + unless (null fs) $ + giveup $ "Cannot undo changes to files that are not checked into git: " ++ unwords fs + void $ liftIO $ cleanup + + -- Committing staged changes before undo allows later + -- undoing the undo. It would be nicer to only commit staged + -- changes to the specified files, rather than all staged changes, + -- but that is difficult to do; a partial git-commit can't be done + -- in direct mode. + void $ Command.Sync.commitStaged Git.Branch.ManualCommit + "commit before undo" + + withStrings (commandAction . start) ps + +start :: FilePath -> CommandStart +start p = do + showStart "undo" p + next $ perform p + +perform :: FilePath -> CommandPerform +perform p = do + g <- gitRepo + + -- Get the reversed diff that needs to be applied to undo. + (diff, cleanup) <- inRepo $ + diffLog [Param "-R", Param "--", Param p] + top <- inRepo $ toTopFilePath p + let diff' = filter (`isDiffOf` top) diff + liftIO $ streamUpdateIndex g (map stageDiffTreeItem diff') + + -- Take two passes through the diff, first doing any removals, + -- and then any adds. This order is necessary to handle eg, removing + -- a directory and replacing it with a file. + let (removals, adds) = partition (\di -> dstsha di == nullSha) diff' + let mkrel di = liftIO $ relPathCwdToFile $ fromTopFilePath (file di) g + + forM_ removals $ \di -> do + f <- mkrel di + whenM isDirect $ + maybe noop (`removeDirect` f) + =<< catKey (srcsha di) + liftIO $ nukeFile f + + forM_ adds $ \di -> do + f <- mkrel di + inRepo $ Git.run [Param "checkout", Param "--", File f] + whenM isDirect $ + maybe noop (`toDirect` f) + =<< catKey (dstsha di) + + next $ liftIO cleanup diff --git a/Command/Ungroup.hs b/Command/Ungroup.hs new file mode 100644 index 0000000000..23d0386ebc --- /dev/null +++ b/Command/Ungroup.hs @@ -0,0 +1,34 @@ +{- git-annex command + - + - Copyright 2012 Joey Hess + - + - Licensed under the GNU GPL version 3 or higher. + -} + +module Command.Ungroup where + +import Command +import qualified Remote +import Logs.Group +import Types.Group + +import qualified Data.Set as S + +cmd :: Command +cmd = command "ungroup" SectionSetup "remove a repository from a group" + (paramPair paramRemote paramDesc) (withParams seek) + +seek :: CmdParams -> CommandSeek +seek = withWords (commandAction . start) + +start :: [String] -> CommandStart +start (name:g:[]) = do + showStart' "ungroup" (Just name) + u <- Remote.nameToUUID name + next $ perform u g +start _ = giveup "Specify a repository and a group." + +perform :: UUID -> Group -> CommandPerform +perform uuid g = do + groupChange uuid (S.delete g) + next $ return True diff --git a/Command/Uninit.hs b/Command/Uninit.hs new file mode 100644 index 0000000000..61dc012b2d --- /dev/null +++ b/Command/Uninit.hs @@ -0,0 +1,121 @@ +{- git-annex command + - + - Copyright 2010 Joey Hess + - + - Licensed under the GNU GPL version 3 or higher. + -} + +module Command.Uninit where + +import Command +import qualified Annex +import qualified Git +import qualified Git.Command +import qualified Command.Unannex +import qualified Annex.Branch +import qualified Database.Keys +import Annex.Content +import Annex.Init +import Utility.FileMode + +cmd :: Command +cmd = addCheck check $ + command "uninit" SectionUtility + "de-initialize git-annex and clean out repository" + paramPaths (withParams seek) + +check :: Annex () +check = do + b <- current_branch + when (b == Annex.Branch.name) $ giveup $ + "cannot uninit when the " ++ Git.fromRef b ++ " branch is checked out" + top <- fromRepo Git.repoPath + currdir <- liftIO getCurrentDirectory + whenM ((/=) <$> liftIO (absPath top) <*> liftIO (absPath currdir)) $ + giveup "can only run uninit from the top of the git repository" + where + current_branch = Git.Ref . Prelude.head . lines <$> revhead + revhead = inRepo $ Git.Command.pipeReadStrict + [Param "rev-parse", Param "--abbrev-ref", Param "HEAD"] + +seek :: CmdParams -> CommandSeek +seek ps = do + l <- workTreeItems ps + withFilesNotInGit False (commandAction . whenAnnexed startCheckIncomplete) l + Annex.changeState $ \s -> s { Annex.fast = True } + withFilesInGit (commandAction . whenAnnexed Command.Unannex.start) l + finish + +{- git annex symlinks that are not checked into git could be left by an + - interrupted add. -} +startCheckIncomplete :: FilePath -> Key -> CommandStart +startCheckIncomplete file _ = giveup $ unlines + [ file ++ " points to annexed content, but is not checked into git." + , "Perhaps this was left behind by an interrupted git annex add?" + , "Not continuing with uninit; either delete or git annex add the file and retry." + ] + +finish :: Annex () +finish = do + annexdir <- fromRepo gitAnnexDir + annexobjectdir <- fromRepo gitAnnexObjectDir + leftovers <- removeUnannexed =<< getKeysPresent InAnnex + prepareRemoveAnnexDir annexdir + if null leftovers + then liftIO $ removeDirectoryRecursive annexdir + else giveup $ unlines + [ "Not fully uninitialized" + , "Some annexed data is still left in " ++ annexobjectdir + , "This may include deleted files, or old versions of modified files." + , "" + , "If you don't care about preserving the data, just delete the" + , "directory." + , "" + , "Or, you can move it to another location, in case it turns out" + , "something in there is important." + , "" + , "Or, you can run `git annex unused` followed by `git annex dropunused`" + , "to remove data that is not used by any tag or branch, which might" + , "take care of all the data." + , "" + , "Then run `git annex uninit` again to finish." + ] + uninitialize + -- avoid normal shutdown + saveState False + inRepo $ Git.Command.run + [Param "branch", Param "-D", Param $ Git.fromRef Annex.Branch.name] + liftIO exitSuccess + +{- Turn on write bits in all remaining files in the annex directory, in + - preparation for removal. + - + - Also closes sqlite databases that might be in the directory, + - to avoid later failure to write any cached changes to them. -} +prepareRemoveAnnexDir :: FilePath -> Annex () +prepareRemoveAnnexDir annexdir = do + Database.Keys.closeDb + liftIO $ prepareRemoveAnnexDir' annexdir + +prepareRemoveAnnexDir' :: FilePath -> IO () +prepareRemoveAnnexDir' annexdir = + dirTreeRecursiveSkipping (const False) annexdir + >>= mapM_ (void . tryIO . allowWrite) + +{- Keys that were moved out of the annex have a hard link still in the + - annex, with > 1 link count, and those can be removed. + - + - Returns keys that cannot be removed. -} +removeUnannexed :: [Key] -> Annex [Key] +removeUnannexed = go [] + where + go c [] = return c + go c (k:ks) = ifM (inAnnexCheck k $ liftIO . enoughlinks) + ( do + lockContentForRemoval k removeAnnex + go c ks + , go (k:c) ks + ) + enoughlinks f = catchBoolIO $ do + s <- getFileStatus f + return $ linkCount s > 1 diff --git a/Command/Unlock.hs b/Command/Unlock.hs new file mode 100644 index 0000000000..93230da709 --- /dev/null +++ b/Command/Unlock.hs @@ -0,0 +1,105 @@ +{- git-annex command + - + - Copyright 2010-2016 Joey Hess + - + - Licensed under the GNU GPL version 3 or higher. + -} + +module Command.Unlock where + +import Command +import Annex.Content +import Annex.Perms +import Annex.CatFile +import Annex.Version +import Annex.Link +import Annex.ReplaceFile +import Utility.CopyFile +import Git.FilePath +import qualified Database.Keys + +cmd :: Command +cmd = mkcmd "unlock" "unlock files for modification" + +editcmd :: Command +editcmd = mkcmd "edit" "same as unlock" + +mkcmd :: String -> String -> Command +mkcmd n d = notDirect $ + withGlobalOptions [jsonOptions, annexedMatchingOptions] $ + command n SectionCommon d paramPaths (withParams seek) + +seek :: CmdParams -> CommandSeek +seek ps = withFilesInGit (commandAction . whenAnnexed start) =<< workTreeItems ps + +{- Before v6, the unlock subcommand replaces the symlink with a copy of + - the file's content. In v6 and above, it converts the file from a symlink + - to a pointer. -} +start :: FilePath -> Key -> CommandStart +start file key = ifM (isJust <$> isAnnexLink file) + ( do + showStart "unlock" file + ifM versionSupportsUnlockedPointers + ( next $ performNew file key + , startOld file key + ) + , stop + ) + +performNew :: FilePath -> Key -> CommandPerform +performNew dest key = do + destmode <- liftIO $ catchMaybeIO $ fileMode <$> getFileStatus dest + replaceFile dest $ \tmp -> + ifM (inAnnex key) + ( do + r <- linkFromAnnex key tmp destmode + case r of + LinkAnnexOk -> return () + LinkAnnexNoop -> return () + LinkAnnexFailed -> error "unlock failed" + , liftIO $ writePointerFile tmp key destmode + ) + next $ cleanupNew dest key destmode + +cleanupNew :: FilePath -> Key -> Maybe FileMode -> CommandCleanup +cleanupNew dest key destmode = do + stagePointerFile dest destmode =<< hashPointerFile key + Database.Keys.addAssociatedFile key =<< inRepo (toTopFilePath dest) + return True + +startOld :: FilePath -> Key -> CommandStart +startOld file key = + ifM (inAnnex key) + ( ifM (isJust <$> catKeyFileHEAD file) + ( next $ performOld file key + , do + warning "this has not yet been committed to git; cannot unlock it" + next $ next $ return False + ) + , do + warning "content not present; cannot unlock" + next $ next $ return False + ) + +performOld :: FilePath -> Key -> CommandPerform +performOld dest key = ifM (checkDiskSpace Nothing key 0 True) + ( do + src <- calcRepo $ gitAnnexLocation key + tmpdest <- fromRepo $ gitAnnexTmpObjectLocation key + liftIO $ createDirectoryIfMissing True (parentDir tmpdest) + showAction "copying" + ifM (liftIO $ copyFileExternal CopyAllMetaData src tmpdest) + ( do + liftIO $ do + removeFile dest + moveFile tmpdest dest + thawContent dest + next $ return True + , do + warning "copy failed!" + next $ return False + ) + , do + warning "not enough disk space to copy file" + next $ return False + ) diff --git a/Command/Untrust.hs b/Command/Untrust.hs new file mode 100644 index 0000000000..7f22a8086d --- /dev/null +++ b/Command/Untrust.hs @@ -0,0 +1,19 @@ +{- git-annex command + - + - Copyright 2010 Joey Hess + - + - Licensed under the GNU GPL version 3 or higher. + -} + +module Command.Untrust where + +import Command +import Types.TrustLevel +import Command.Trust (trustCommand) + +cmd :: Command +cmd = command "untrust" SectionSetup "do not trust a repository" + (paramRepeating paramRemote) (withParams seek) + +seek :: CmdParams -> CommandSeek +seek = trustCommand "untrust" UnTrusted diff --git a/Command/Unused.hs b/Command/Unused.hs new file mode 100644 index 0000000000..33967c4173 --- /dev/null +++ b/Command/Unused.hs @@ -0,0 +1,340 @@ +{- git-annex command + - + - Copyright 2010-2016 Joey Hess + - + - Licensed under the GNU GPL version 3 or higher. + -} + +{-# LANGUAGE BangPatterns #-} + +module Command.Unused where + +import qualified Data.Map as M + +import Command +import Logs.Unused +import Annex.Content +import Logs.Location +import qualified Annex +import qualified Git +import qualified Git.Command +import qualified Git.Ref +import qualified Git.Branch +import qualified Git.RefLog +import qualified Git.LsFiles as LsFiles +import qualified Git.DiffTree as DiffTree +import qualified Remote +import qualified Annex.Branch +import Annex.CatFile +import Annex.WorkTree +import Types.RefSpec +import Git.Types +import Git.Sha +import Git.FilePath +import Config +import Logs.View (is_branchView) +import Annex.BloomFilter +import qualified Database.Keys +import Annex.InodeSentinal + +cmd :: Command +cmd = command "unused" SectionMaintenance "look for unused file content" + paramNothing (seek <$$> optParser) + +data UnusedOptions = UnusedOptions + { fromRemote :: Maybe RemoteName + , refSpecOption :: Maybe RefSpec + } + +optParser :: CmdParamsDesc -> Parser UnusedOptions +optParser _ = UnusedOptions + <$> optional (strOption + ( long "from" <> short 'f' <> metavar paramRemote + <> help "remote to check for unused content" + )) + <*> optional (option (eitherReader parseRefSpec) + ( long "used-refspec" <> metavar paramRefSpec + <> help "refs to consider used (default: all branches)" + )) + +seek :: UnusedOptions -> CommandSeek +seek = commandAction . start + +start :: UnusedOptions -> CommandStart +start o = do + cfgrefspec <- fromMaybe allRefSpec . annexUsedRefSpec + <$> Annex.getGitConfig + let refspec = fromMaybe cfgrefspec (refSpecOption o) + let (name, perform) = case fromRemote o of + Nothing -> (".", checkUnused refspec) + Just "." -> (".", checkUnused refspec) + Just "here" -> (".", checkUnused refspec) + Just n -> (n, checkRemoteUnused n refspec) + showStart' "unused" (Just name) + next perform + +checkUnused :: RefSpec -> CommandPerform +checkUnused refspec = chain 0 + [ check "" unusedMsg $ findunused =<< Annex.getState Annex.fast + , check "bad" staleBadMsg $ staleKeysPrune gitAnnexBadDir False + , check "tmp" staleTmpMsg $ staleKeysPrune gitAnnexTmpObjectDir True + ] + where + findunused True = do + showNote "fast mode enabled; only finding stale files" + return [] + findunused False = do + showAction "checking for unused data" + -- InAnnex, not InRepository because if a direct mode + -- file exists, it is obviously not unused. + excludeReferenced refspec =<< getKeysPresent InAnnex + chain _ [] = next $ return True + chain v (a:as) = do + v' <- a v + chain v' as + +checkRemoteUnused :: RemoteName -> RefSpec -> CommandPerform +checkRemoteUnused remotename refspec = go =<< Remote.nameToUUID remotename + where + go u = do + showAction "checking for unused data" + r <- Remote.byUUID u + _ <- check "" (remoteUnusedMsg r remotename) (remoteunused u) 0 + next $ return True + remoteunused u = excludeReferenced refspec =<< loggedKeysFor u + +check :: FilePath -> ([(Int, Key)] -> String) -> Annex [Key] -> Int -> Annex Int +check file msg a c = do + l <- a + let unusedlist = number c l + unless (null l) $ showLongNote $ msg unusedlist + updateUnusedLog file $ M.fromList unusedlist + return $ c + length l + +number :: Int -> [a] -> [(Int, a)] +number _ [] = [] +number n (x:xs) = (n+1, x) : number (n+1) xs + +table :: [(Int, Key)] -> [String] +table l = " NUMBER KEY" : map cols l + where + cols (n,k) = " " ++ pad 6 (show n) ++ " " ++ key2file k + pad n s = s ++ replicate (n - length s) ' ' + +staleTmpMsg :: [(Int, Key)] -> String +staleTmpMsg t = unlines $ + ["Some partially transferred data exists in temporary files:"] + ++ table t ++ [dropMsg Nothing] + +staleBadMsg :: [(Int, Key)] -> String +staleBadMsg t = unlines $ + ["Some corrupted files have been preserved by fsck, just in case:"] + ++ table t ++ [dropMsg Nothing] + +unusedMsg :: [(Int, Key)] -> String +unusedMsg u = unusedMsg' u + ["Some annexed data is no longer used by any files:"] + [dropMsg Nothing] +unusedMsg' :: [(Int, Key)] -> [String] -> [String] -> String +unusedMsg' u mheader mtrailer = unlines $ + mheader ++ + table u ++ + ["(To see where data was previously used, try: git log --stat -S'KEY')"] ++ + mtrailer + +remoteUnusedMsg :: Maybe Remote -> RemoteName -> [(Int, Key)] -> String +remoteUnusedMsg mr remotename u = unusedMsg' u + ["Some annexed data on " ++ remotename ++ " is not used by any files:"] + (if isJust mr then [dropMsg (Just remotename)] else []) + +dropMsg :: Maybe RemoteName -> String +dropMsg Nothing = dropMsg' "" +dropMsg (Just remotename) = dropMsg' $ " --from " ++ remotename +dropMsg' :: String -> String +dropMsg' s = "\nTo remove unwanted data: git-annex dropunused" ++ s ++ " NUMBER\n" + +{- Finds keys in the list that are not referenced in the git repository. + - + - Strategy: + - + - Pass keys through these filters in order, only creating each bloom + - filter on demand if the previous one didn't filter out all keys. + - + - 1. Bloom filter containing all keys referenced by files in the work tree. + - This is the fastest one to build and will filter out most keys. + - 2. Bloom filter containing all keys in the diff from the work tree to + - the index. + - 3. Associated files filter. A v6 unlocked file may have had its content + - added to the annex (by eg, git diff running the smudge filter), + - but the new key is not yet staged in the index. But if so, it will + - have an associated file. + - 4. Bloom filter containing all keys in the diffs between the index and + - branches matching the RefSpec. (This can take quite a while to build). + -} +excludeReferenced :: RefSpec -> [Key] -> Annex [Key] +excludeReferenced refspec ks = runbloomfilter withKeysReferencedM ks + >>= runbloomfilter withKeysReferencedDiffIndex + >>= runfilter associatedFilesFilter + >>= runbloomfilter (withKeysReferencedDiffGitRefs refspec) + where + runfilter _ [] = return [] -- optimisation + runfilter a l = a l + runbloomfilter a = runfilter $ \l -> bloomFilter l <$> genBloomFilter a + +{- Given an initial value, folds it with each key referenced by + - files in the working tree. -} +withKeysReferenced :: v -> (Key -> v -> v) -> Annex v +withKeysReferenced initial a = withKeysReferenced' Nothing initial folda + where + folda k _ v = return $ a k v + +{- Runs an action on each referenced key in the working tree. -} +withKeysReferencedM :: (Key -> Annex ()) -> Annex () +withKeysReferencedM a = withKeysReferenced' Nothing () calla + where + calla k _ _ = a k + +{- Folds an action over keys and files referenced in a particular directory. -} +withKeysFilesReferencedIn :: FilePath -> v -> (Key -> FilePath -> v -> Annex v) -> Annex v +withKeysFilesReferencedIn = withKeysReferenced' . Just + +withKeysReferenced' :: Maybe FilePath -> v -> (Key -> FilePath -> v -> Annex v) -> Annex v +withKeysReferenced' mdir initial a = do + (files, clean) <- getfiles + r <- go initial files + liftIO $ void clean + return r + where + getfiles = case mdir of + Nothing -> ifM isBareRepo + ( return ([], return True) + , do + top <- fromRepo Git.repoPath + inRepo $ LsFiles.allFiles [top] + ) + Just dir -> inRepo $ LsFiles.inRepo [dir] + go v [] = return v + go v (f:fs) = do + mk <- lookupFile f + case mk of + Nothing -> go v fs + Just k -> do + !v' <- a k f v + go v' fs + +withKeysReferencedDiffGitRefs :: RefSpec -> (Key -> Annex ()) -> Annex () +withKeysReferencedDiffGitRefs refspec a = do + rs <- relevantrefs <$> inRepo (Git.Command.pipeReadStrict [Param "show-ref"]) + shaHead <- maybe (return Nothing) (inRepo . Git.Ref.sha) + =<< inRepo Git.Branch.currentUnsafe + let haveHead = any (\(shaRef, _) -> Just shaRef == shaHead) rs + let rs' = map snd (nubRefs rs) + usedrefs <- applyRefSpec refspec rs' (getreflog rs') + forM_ (if haveHead then usedrefs else Git.Ref.headRef : usedrefs) $ + withKeysReferencedDiffGitRef a + where + relevantrefs = map (\(r, h) -> (Git.Ref r, Git.Ref h)) . + filter ourbranches . + map (separate (== ' ')) . + lines + nubRefs = nubBy (\(x, _) (y, _) -> x == y) + ourbranchend = '/' : Git.fromRef Annex.Branch.name + ourbranches (_, b) = not (ourbranchend `isSuffixOf` b) + && not ("refs/synced/" `isPrefixOf` b) + && not (is_branchView (Git.Ref b)) + getreflog rs = inRepo $ Git.RefLog.getMulti rs + +{- Runs an action on keys referenced in the given Git reference which + - differ from those referenced in the index. -} +withKeysReferencedDiffGitRef :: (Key -> Annex ()) -> Git.Ref -> Annex () +withKeysReferencedDiffGitRef a ref = do + showAction $ "checking " ++ Git.Ref.describe ref + withKeysReferencedDiff a + (inRepo $ DiffTree.diffIndex ref) + DiffTree.srcsha + +{- Runs an action on keys referenced in the index which differ from the + - work tree. -} +withKeysReferencedDiffIndex :: (Key -> Annex ()) -> Annex () +withKeysReferencedDiffIndex a = unlessM (isBareRepo) $ + withKeysReferencedDiff a + (inRepo $ DiffTree.diffFiles []) + DiffTree.srcsha + +withKeysReferencedDiff :: (Key -> Annex ()) -> (Annex ([DiffTree.DiffTreeItem], IO Bool)) -> (DiffTree.DiffTreeItem -> Sha) -> Annex () +withKeysReferencedDiff a getdiff extractsha = do + (ds, clean) <- getdiff + forM_ ds go + liftIO $ void clean + where + go d = do + let sha = extractsha d + unless (sha == nullSha) $ + catKey sha >>= maybe noop a + +{- Filters out keys that have an associated file that's not modified. -} +associatedFilesFilter :: [Key] -> Annex [Key] +associatedFilesFilter = filterM go + where + go k = do + cs <- Database.Keys.getInodeCaches k + if null cs + then return True + else checkunmodified cs + =<< Database.Keys.getAssociatedFiles k + checkunmodified _ [] = return True + checkunmodified cs (f:fs) = do + relf <- fromRepo $ fromTopFilePath f + ifM (sameInodeCache relf cs) + ( return False + , checkunmodified cs fs + ) + +data UnusedMaps = UnusedMaps + { unusedMap :: UnusedMap + , unusedBadMap :: UnusedMap + , unusedTmpMap :: UnusedMap + } + +withUnusedMaps :: (UnusedMaps -> Int -> CommandStart) -> CmdParams -> CommandSeek +withUnusedMaps a params = do + unused <- readUnusedMap "" + unusedbad <- readUnusedMap "bad" + unusedtmp <- readUnusedMap "tmp" + let m = unused `M.union` unusedbad `M.union` unusedtmp + let unusedmaps = UnusedMaps unused unusedbad unusedtmp + commandActions $ map (a unusedmaps) $ concatMap (unusedSpec m) params + +unusedSpec :: UnusedMap -> String -> [Int] +unusedSpec m spec + | spec == "all" = if M.null m + then [] + else [fst (M.findMin m)..fst (M.findMax m)] + | "-" `isInfixOf` spec = range $ separate (== '-') spec + | otherwise = maybe badspec (: []) (readish spec) + where + range (a, b) = case (readish a, readish b) of + (Just x, Just y) -> [x..y] + _ -> badspec + badspec = giveup $ "Expected number or range, not \"" ++ spec ++ "\"" + +{- Seek action for unused content. Finds the number in the maps, and + - calls one of 3 actions, depending on the type of unused file. -} +startUnused :: String + -> (Key -> CommandPerform) + -> (Key -> CommandPerform) + -> (Key -> CommandPerform) + -> UnusedMaps -> Int -> CommandStart +startUnused message unused badunused tmpunused maps n = search + [ (unusedMap maps, unused) + , (unusedBadMap maps, badunused) + , (unusedTmpMap maps, tmpunused) + ] + where + search [] = giveup $ show n ++ " not valid (run git annex unused for list)" + search ((m, a):rest) = + case M.lookup n m of + Nothing -> search rest + Just key -> do + showStart' message (Just $ show n) + next $ a key diff --git a/Command/Upgrade.hs b/Command/Upgrade.hs new file mode 100644 index 0000000000..1ae0000cc2 --- /dev/null +++ b/Command/Upgrade.hs @@ -0,0 +1,30 @@ +{- git-annex command + - + - Copyright 2011 Joey Hess + - + - Licensed under the GNU GPL version 3 or higher. + -} + +module Command.Upgrade where + +import Command +import Upgrade +import Annex.Version +import Annex.Init + +cmd :: Command +cmd = dontCheck repoExists $ -- because an old version may not seem to exist + noDaemonRunning $ -- avoid upgrading repo out from under daemon + command "upgrade" SectionMaintenance "upgrade repository layout" + paramNothing (withParams seek) + +seek :: CmdParams -> CommandSeek +seek = withNothing (commandAction start) + +start :: CommandStart +start = do + showStart' "upgrade" Nothing + whenM (isNothing <$> getVersion) $ do + initialize (AutoInit False) Nothing Nothing + r <- upgrade False latestVersion + next $ next $ return r diff --git a/Command/VAdd.hs b/Command/VAdd.hs new file mode 100644 index 0000000000..8dceae6a46 --- /dev/null +++ b/Command/VAdd.hs @@ -0,0 +1,38 @@ +{- git-annex command + - + - Copyright 2014 Joey Hess + - + - Licensed under the GNU GPL version 3 or higher. + -} + +module Command.VAdd where + +import Command +import Annex.View +import Command.View (checkoutViewBranch) + +cmd :: Command +cmd = notBareRepo $ notDirect $ + command "vadd" SectionMetaData + "add subdirs to current view" + (paramRepeating "FIELD=GLOB") + (withParams seek) + +seek :: CmdParams -> CommandSeek +seek = withWords (commandAction . start) + +start :: [String] -> CommandStart +start params = do + showStart' "vadd" Nothing + withCurrentView $ \view -> do + let (view', change) = refineView view $ + map parseViewParam $ reverse params + case change of + Unchanged -> do + showNote "unchanged" + next $ next $ return True + Narrowing -> next $ next $ do + if visibleViewSize view' == visibleViewSize view + then giveup "That would not add an additional level of directory structure to the view. To filter the view, use vfilter instead of vadd." + else checkoutViewBranch view' narrowView + Widening -> giveup "Widening view to match more files is not currently supported." diff --git a/Command/VCycle.hs b/Command/VCycle.hs new file mode 100644 index 0000000000..bb318635d3 --- /dev/null +++ b/Command/VCycle.hs @@ -0,0 +1,41 @@ +{- git-annex command + - + - Copyright 2014 Joey Hess + - + - Licensed under the GNU GPL version 3 or higher. + -} + +module Command.VCycle where + +import Command +import Annex.View +import Types.View +import Logs.View +import Command.View (checkoutViewBranch) + +cmd :: Command +cmd = notBareRepo $ notDirect $ + command "vcycle" SectionMetaData + "switch view to next layout" + paramNothing (withParams seek) + +seek :: CmdParams -> CommandSeek +seek = withNothing (commandAction start) + +start ::CommandStart +start = go =<< currentView + where + go Nothing = giveup "Not in a view." + go (Just v) = do + showStart' "vcycle" Nothing + let v' = v { viewComponents = vcycle [] (viewComponents v) } + if v == v' + then do + showNote "unchanged" + next $ next $ return True + else next $ next $ checkoutViewBranch v' narrowView + + vcycle rest (c:cs) + | viewVisible c = rest ++ cs ++ [c] + | otherwise = vcycle (c:rest) cs + vcycle rest c = rest ++ c diff --git a/Command/VFilter.hs b/Command/VFilter.hs new file mode 100644 index 0000000000..49a1cbb379 --- /dev/null +++ b/Command/VFilter.hs @@ -0,0 +1,30 @@ +{- git-annex command + - + - Copyright 2014 Joey Hess + - + - Licensed under the GNU GPL version 3 or higher. + -} + +module Command.VFilter where + +import Command +import Annex.View +import Command.View (paramView, checkoutViewBranch) + +cmd :: Command +cmd = notBareRepo $ notDirect $ + command "vfilter" SectionMetaData "filter current view" + paramView (withParams seek) + +seek :: CmdParams -> CommandSeek +seek = withWords (commandAction . start) + +start :: [String] -> CommandStart +start params = do + showStart' "vfilter" Nothing + withCurrentView $ \view -> do + let view' = filterView view $ + map parseViewParam $ reverse params + next $ next $ if visibleViewSize view' > visibleViewSize view + then giveup "That would add an additional level of directory structure to the view, rather than filtering it. If you want to do that, use vadd instead of vfilter." + else checkoutViewBranch view' narrowView diff --git a/Command/VPop.hs b/Command/VPop.hs new file mode 100644 index 0000000000..50860d26eb --- /dev/null +++ b/Command/VPop.hs @@ -0,0 +1,49 @@ +{- git-annex command + - + - Copyright 2014 Joey Hess + - + - Licensed under the GNU GPL version 3 or higher. + -} + +module Command.VPop where + +import Command +import qualified Git +import qualified Git.Command +import qualified Git.Ref +import Types.View +import Logs.View +import Command.View (checkoutViewBranch) + +cmd :: Command +cmd = notBareRepo $ notDirect $ + command "vpop" SectionMetaData "switch back to previous view" + paramNumber (withParams seek) + +seek :: CmdParams -> CommandSeek +seek = withWords (commandAction . start) + +start :: [String] -> CommandStart +start ps = go =<< currentView + where + go Nothing = giveup "Not in a view." + go (Just v) = do + showStart' "vpop" (Just $ show num) + removeView v + (oldvs, vs) <- splitAt (num - 1) . filter (sameparentbranch v) + <$> recentViews + mapM_ removeView oldvs + case vs of + (oldv:_) -> next $ next $ do + showOutput + checkoutViewBranch oldv (return . branchView) + _ -> next $ next $ do + showOutput + inRepo $ Git.Command.runBool + [ Param "checkout" + , Param $ Git.fromRef $ Git.Ref.base $ + viewParentBranch v + ] + sameparentbranch a b = viewParentBranch a == viewParentBranch b + + num = fromMaybe 1 $ readish =<< headMaybe ps diff --git a/Command/Version.hs b/Command/Version.hs new file mode 100644 index 0000000000..a7a5d503eb --- /dev/null +++ b/Command/Version.hs @@ -0,0 +1,75 @@ +{- git-annex command + - + - Copyright 2010 Joey Hess + - + - Licensed under the GNU GPL version 3 or higher. + -} + +module Command.Version where + +import Command +import Annex.Version +import BuildInfo +import BuildFlags +import Types.Key +import qualified Types.Backend as B +import qualified Types.Remote as R +import qualified Remote +import qualified Backend + +import System.Info + +cmd :: Command +cmd = dontCheck repoExists $ noCommit $ + noRepo (seekNoRepo <$$> optParser) $ + command "version" SectionQuery "show version info" + paramNothing (seek <$$> optParser) + +data VersionOptions = VersionOptions + { rawOption :: Bool + } + +optParser :: CmdParamsDesc -> Parser VersionOptions +optParser _ = VersionOptions + <$> switch + ( long "raw" + <> help "output only program version" + ) + +seek :: VersionOptions -> CommandSeek +seek o + | rawOption o = liftIO showRawVersion + | otherwise = showVersion + +seekNoRepo :: VersionOptions -> IO () +seekNoRepo o + | rawOption o = showRawVersion + | otherwise = showPackageVersion + +showVersion :: Annex () +showVersion = do + liftIO showPackageVersion + maybe noop (liftIO . vinfo "local repository version") + =<< getVersion + +showPackageVersion :: IO () +showPackageVersion = do + vinfo "git-annex version" BuildInfo.packageversion + vinfo "build flags" $ unwords buildFlags + vinfo "dependency versions" $ unwords dependencyVersions + vinfo "key/value backends" $ unwords $ + map (formatKeyVariety . B.backendVariety) Backend.list + vinfo "remote types" $ unwords $ map R.typename Remote.remoteTypes + vinfo "operating system" $ unwords [os, arch] + vinfo "supported repository versions" $ + unwords supportedVersions + vinfo "upgrade supported from repository versions" $ + unwords upgradableVersions + +showRawVersion :: IO () +showRawVersion = do + putStr BuildInfo.packageversion + hFlush stdout -- no newline, so flush + +vinfo :: String -> String -> IO () +vinfo k v = putStrLn $ k ++ ": " ++ v diff --git a/Command/Vicfg.hs b/Command/Vicfg.hs new file mode 100644 index 0000000000..9889b1c53e --- /dev/null +++ b/Command/Vicfg.hs @@ -0,0 +1,336 @@ +{- git-annex command + - + - Copyright 2012-2017 Joey Hess + - + - Licensed under the GNU GPL version 3 or higher. + -} + +{-# LANGUAGE RankNTypes #-} + +module Command.Vicfg where + +import qualified Data.Map as M +import qualified Data.Set as S +import System.Environment (getEnv) +import Data.Tuple (swap) +import Data.Char (isSpace) +import Data.Default +import Data.Ord + +import Command +import Annex.Perms +import Types.TrustLevel +import Types.Group +import Logs.Trust +import Logs.Group +import Logs.PreferredContent +import Logs.Schedule +import Logs.Config +import Logs.NumCopies +import Types.StandardGroups +import Types.ScheduledActivity +import Types.NumCopies +import Remote + +cmd :: Command +cmd = command "vicfg" SectionSetup "edit configuration in git-annex branch" + paramNothing (withParams seek) + +seek :: CmdParams -> CommandSeek +seek = withNothing (commandAction start) + +start :: CommandStart +start = do + f <- fromRepo gitAnnexTmpCfgFile + createAnnexDirectory $ parentDir f + cfg <- getCfg + descs <- uuidDescriptions + liftIO $ writeFile f $ genCfg cfg descs + vicfg cfg f + stop + +vicfg :: Cfg -> FilePath -> Annex () +vicfg curcfg f = do + vi <- liftIO $ catchDefaultIO "vi" $ getEnv "EDITOR" + -- Allow EDITOR to be processed by the shell, so it can contain options. + unlessM (liftIO $ boolSystem "sh" [Param "-c", Param $ unwords [vi, shellEscape f]]) $ + giveup $ vi ++ " exited nonzero; aborting" + r <- parseCfg (defCfg curcfg) <$> liftIO (readFileStrict f) + liftIO $ nukeFile f + case r of + Left s -> do + liftIO $ writeFile f s + vicfg curcfg f + Right newcfg -> setCfg curcfg newcfg + +data Cfg = Cfg + { cfgTrustMap :: M.Map UUID (Down TrustLevel) + , cfgGroupMap :: M.Map UUID (S.Set Group) + , cfgPreferredContentMap :: M.Map UUID PreferredContentExpression + , cfgRequiredContentMap :: M.Map UUID PreferredContentExpression + , cfgGroupPreferredContentMap :: M.Map Group PreferredContentExpression + , cfgScheduleMap :: M.Map UUID [ScheduledActivity] + , cfgGlobalConfigs :: M.Map ConfigName ConfigValue + , cfgNumCopies :: Maybe NumCopies + } + +getCfg :: Annex Cfg +getCfg = Cfg + <$> (M.map Down <$> trustMapRaw) -- without local trust overrides + <*> (groupsByUUID <$> groupMap) + <*> preferredContentMapRaw + <*> requiredContentMapRaw + <*> groupPreferredContentMapRaw + <*> scheduleMap + <*> loadGlobalConfig + <*> getGlobalNumCopies + +setCfg :: Cfg -> Cfg -> Annex () +setCfg curcfg newcfg = do + let diff = diffCfg curcfg newcfg + mapM_ (uncurry trustSet) $ M.toList $ M.map (\(Down v) -> v) $ cfgTrustMap diff + mapM_ (uncurry groupSet) $ M.toList $ cfgGroupMap diff + mapM_ (uncurry preferredContentSet) $ M.toList $ cfgPreferredContentMap diff + mapM_ (uncurry requiredContentSet) $ M.toList $ cfgRequiredContentMap diff + mapM_ (uncurry groupPreferredContentSet) $ M.toList $ cfgGroupPreferredContentMap diff + mapM_ (uncurry scheduleSet) $ M.toList $ cfgScheduleMap diff + mapM_ (uncurry setGlobalConfig) $ M.toList $ cfgGlobalConfigs diff + maybe noop setGlobalNumCopies $ cfgNumCopies diff + +{- Default config has all the keys from the input config, but with their + - default values. -} +defCfg :: Cfg -> Cfg +defCfg curcfg = Cfg + { cfgTrustMap = mapdef $ cfgTrustMap curcfg + , cfgGroupMap = mapdef $ cfgGroupMap curcfg + , cfgPreferredContentMap = mapdef $ cfgPreferredContentMap curcfg + , cfgRequiredContentMap = mapdef $ cfgRequiredContentMap curcfg + , cfgGroupPreferredContentMap = mapdef $ cfgGroupPreferredContentMap curcfg + , cfgScheduleMap = mapdef $ cfgScheduleMap curcfg + , cfgGlobalConfigs = mapdef $ cfgGlobalConfigs curcfg + , cfgNumCopies = Nothing + } + where + mapdef :: forall k v. Default v => M.Map k v -> M.Map k v + mapdef = M.map (const def) + +diffCfg :: Cfg -> Cfg -> Cfg +diffCfg curcfg newcfg = Cfg + { cfgTrustMap = diff cfgTrustMap + , cfgGroupMap = diff cfgGroupMap + , cfgPreferredContentMap = diff cfgPreferredContentMap + , cfgRequiredContentMap = diff cfgRequiredContentMap + , cfgGroupPreferredContentMap = diff cfgGroupPreferredContentMap + , cfgScheduleMap = diff cfgScheduleMap + , cfgGlobalConfigs = diff cfgGlobalConfigs + , cfgNumCopies = cfgNumCopies newcfg + } + where + diff f = M.differenceWith (\x y -> if x == y then Nothing else Just x) + (f newcfg) (f curcfg) + +genCfg :: Cfg -> M.Map UUID String -> String +genCfg cfg descs = unlines $ intercalate [""] + [ intro + , trust + , groups + , preferredcontent + , grouppreferredcontent + , standardgroups + , requiredcontent + , schedule + , numcopies + , globalconfigs + ] + where + intro = + [ com "git-annex configuration" + , com "" + , com "Changes saved to this file will be recorded in the git-annex branch." + , com "" + , com "Lines in this file have the format:" + , com " setting field = value" + ] + + trust = settings cfg descs cfgTrustMap + [ com "Repository trust configuration" + , com "(Valid trust levels: " ++ trustlevels ++ ")" + ] + (\(Down t, u) -> line "trust" u $ showTrustLevel t) + (\u -> lcom $ line "trust" u $ showTrustLevel def) + where + trustlevels = unwords $ reverse $ + map showTrustLevel [minBound..maxBound] + + groups = settings cfg descs cfgGroupMap + [ com "Repository groups" + , com $ "(Standard groups: " ++ grouplist ++ ")" + , com "(Separate group names with spaces)" + ] + (\(s, u) -> line "group" u $ unwords $ S.toList s) + (\u -> lcom $ line "group" u "") + where + grouplist = unwords $ map fromStandardGroup [minBound..] + + preferredcontent = settings cfg descs cfgPreferredContentMap + [ com "Repository preferred contents" + , com "(Set to \"standard\" to use a repository's group's preferred contents)" + ] + (\(s, u) -> line "wanted" u s) + (\u -> line "wanted" u "") + + requiredcontent = settings cfg descs cfgRequiredContentMap + [ com "Repository required contents" ] + (\(s, u) -> line "required" u s) + (\u -> line "required" u "") + + grouppreferredcontent = settings' cfg allgroups cfgGroupPreferredContentMap + [ com "Group preferred contents" + , com "(Used by repositories with \"groupwanted\" in their preferred contents)" + ] + (\(s, g) -> gline g s) + (\g -> gline g "") + where + gline g val = [ unwords ["groupwanted", g, "=", val] ] + allgroups = S.unions $ stdgroups : M.elems (cfgGroupMap cfg) + stdgroups = S.fromList $ map fromStandardGroup [minBound..maxBound] + + standardgroups = + [ com "Standard preferred contents" + , com "(Used by wanted or groupwanted expressions containing \"standard\")" + , com "(For reference only; built-in and cannot be changed!)" + ] + ++ map gline [minBound..maxBound] + where + gline g = com $ unwords + [ "standard" + , fromStandardGroup g, "=", standardPreferredContent g + ] + + schedule = settings cfg descs cfgScheduleMap + [ com "Scheduled activities" + , com "(Separate multiple activities with \"; \")" + ] + (\(l, u) -> line "schedule" u $ fromScheduledActivities l) + (\u -> line "schedule" u "") + + globalconfigs = settings' cfg S.empty cfgGlobalConfigs + [ com "Other global configuration" + ] + (\(s, g) -> gline g s) + (\g -> gline g "") + where + gline g val = [ unwords ["config", g, "=", val] ] + + line setting u val = + [ com $ "(for " ++ fromMaybe "" (M.lookup u descs) ++ ")" + , unwords [setting, fromUUID u, "=", val] + ] + + line' setting Nothing = com $ unwords [setting, "default", "="] + line' setting (Just val) = unwords [setting, "default", "=", val] + + numcopies = + [ com "Numcopies configuration" + , line' "numcopies" (show . fromNumCopies <$> cfgNumCopies cfg) + ] + +settings :: Ord v => Cfg -> M.Map UUID String -> (Cfg -> M.Map UUID v) -> [String] -> ((v, UUID) -> [String]) -> (UUID -> [String]) -> [String] +settings cfg descs = settings' cfg (M.keysSet descs) + +settings' :: (Ord v, Ord f) => Cfg -> S.Set f -> (Cfg -> M.Map f v) -> [String] -> ((v, f) -> [String]) -> (f -> [String]) -> [String] +settings' cfg s field desc showvals showdefaults = concat + [ desc + , concatMap showvals $ sort $ map swap $ M.toList $ field cfg + , concatMap (lcom . showdefaults) missing + ] + where + missing = S.toList $ s `S.difference` M.keysSet (field cfg) + +lcom :: [String] -> [String] +lcom = map (\l -> if "#" `isPrefixOf` l then l else '#' : l) + +{- If there's a parse error, returns a new version of the file, + - with the problem lines noted. -} +parseCfg :: Cfg -> String -> Either String Cfg +parseCfg defcfg = go [] defcfg . lines + where + go c cfg [] + | null (mapMaybe fst c) = Right cfg + | otherwise = Left $ unlines $ + badheader ++ concatMap showerr (reverse c) + go c cfg (l:ls) = case parse (dropWhile isSpace l) cfg of + Left msg -> go ((Just msg, l):c) cfg ls + Right cfg' -> go ((Nothing, l):c) cfg' ls + + parse l cfg + | null l = Right cfg + | "#" `isPrefixOf` l = Right cfg + | null setting || null f = Left "missing field" + | otherwise = parsed cfg f setting val' + where + (setting, rest) = separate isSpace l + (r, val) = separate (== '=') rest + val' = trimspace val + f = reverse $ trimspace $ reverse $ trimspace r + trimspace = dropWhile isSpace + + parsed cfg f setting val + | setting == "trust" = case readTrustLevel val of + Nothing -> badval "trust value" val + Just t -> + let m = M.insert u (Down t) (cfgTrustMap cfg) + in Right $ cfg { cfgTrustMap = m } + | setting == "group" = + let m = M.insert u (S.fromList $ words val) (cfgGroupMap cfg) + in Right $ cfg { cfgGroupMap = m } + | setting == "wanted" = + case checkPreferredContentExpression val of + Just e -> Left e + Nothing -> + let m = M.insert u val (cfgPreferredContentMap cfg) + in Right $ cfg { cfgPreferredContentMap = m } + | setting == "required" = + case checkPreferredContentExpression val of + Just e -> Left e + Nothing -> + let m = M.insert u val (cfgRequiredContentMap cfg) + in Right $ cfg { cfgRequiredContentMap = m } + | setting == "groupwanted" = + case checkPreferredContentExpression val of + Just e -> Left e + Nothing -> + let m = M.insert f val (cfgGroupPreferredContentMap cfg) + in Right $ cfg { cfgGroupPreferredContentMap = m } + | setting == "schedule" = case parseScheduledActivities val of + Left e -> Left e + Right l -> + let m = M.insert u l (cfgScheduleMap cfg) + in Right $ cfg { cfgScheduleMap = m } + | setting == "config" = + let m = M.insert f val (cfgGlobalConfigs cfg) + in Right $ cfg { cfgGlobalConfigs = m } + | setting == "numcopies" = case readish val of + Nothing -> Left "parse error (expected an integer)" + Just n -> Right $ cfg { cfgNumCopies = Just (NumCopies n) } + | otherwise = badval "setting" setting + where + u = toUUID f + + showerr (Just msg, l) = [parseerr ++ msg, l] + showerr (Nothing, l) + -- filter out the header and parse error lines + -- from any previous parse failure + | any (`isPrefixOf` l) (parseerr:badheader) = [] + | otherwise = [l] + + badval desc val = Left $ "unknown " ++ desc ++ " \"" ++ val ++ "\"" + badheader = + [ com "** There was a problem parsing your input!" + , com "** Search for \"Parse error\" to find the bad lines." + , com "** Either fix the bad lines, or delete them (to discard your changes)." + ] + parseerr = com "** Parse error in next line: " + +com :: String -> String +com s = "# " ++ s diff --git a/Command/View.hs b/Command/View.hs new file mode 100644 index 0000000000..3d3819dd57 --- /dev/null +++ b/Command/View.hs @@ -0,0 +1,118 @@ +{- git-annex command + - + - Copyright 2014 Joey Hess + - + - Licensed under the GNU GPL version 3 or higher. + -} + +module Command.View where + +import Command +import qualified Git +import qualified Git.Command +import qualified Git.Ref +import qualified Git.Branch +import qualified Git.LsFiles as LsFiles +import Git.FilePath +import Git.Status +import Types.View +import Annex.View +import Logs.View + +cmd :: Command +cmd = notBareRepo $ notDirect $ + command "view" SectionMetaData "enter a view branch" + paramView (withParams seek) + +seek :: CmdParams -> CommandSeek +seek = withWords (commandAction . start) + +start :: [String] -> CommandStart +start [] = giveup "Specify metadata to include in view" +start ps = do + showStart' "view" Nothing + ifM safeToEnterView + ( do + view <- mkView ps + go view =<< currentView + , giveup "Not safe to enter view." + ) + where + go view Nothing = next $ perform view + go view (Just v) + | v == view = stop + | otherwise = giveup "Already in a view. Use the vfilter and vadd commands to further refine this view." + +safeToEnterView :: Annex Bool +safeToEnterView = do + (l, cleanup) <- inRepo $ getStatus [] [] + case filter dangerous l of + [] -> liftIO cleanup + _ -> do + warning "Your uncommitted changes would be lost when entering a view." + void $ liftIO cleanup + return False + where + dangerous (StagedUnstaged { staged = Nothing, unstaged = Nothing }) = False + -- Untracked files will not be affected by entering a view, + -- so are not dangerous. + dangerous (StagedUnstaged { staged = Just (Untracked _), unstaged = Nothing }) = False + dangerous (StagedUnstaged { unstaged = Just (Untracked _), staged = Nothing }) = False + dangerous (StagedUnstaged { unstaged = Just (Untracked _), staged = Just (Untracked _) }) = False + -- Staged changes would have their modifications either be + -- lost when entering a view, or committed as part of the view. + -- Either is dangerous because upon leaving the view; the staged + -- changes would be lost. + dangerous (StagedUnstaged { staged = Just _ }) = True + -- Unstaged changes to annexed files would get lost when entering a + -- view. + dangerous (StagedUnstaged { unstaged = Just _ }) = True + +perform :: View -> CommandPerform +perform view = do + showAction "searching" + next $ checkoutViewBranch view applyView + +paramView :: String +paramView = paramRepeating "FIELD=VALUE" + +mkView :: [String] -> Annex View +mkView ps = go =<< inRepo Git.Branch.current + where + go Nothing = giveup "not on any branch!" + go (Just b) = return $ fst $ refineView (View b []) $ + map parseViewParam $ reverse ps + +checkoutViewBranch :: View -> (View -> Annex Git.Branch) -> CommandCleanup +checkoutViewBranch view mkbranch = do + here <- liftIO getCurrentDirectory + + branch <- mkbranch view + + showOutput + ok <- inRepo $ Git.Command.runBool + [ Param "checkout" + , Param (Git.fromRef $ Git.Ref.base branch) + ] + when ok $ do + setView view + {- A git repo can easily have empty directories in it, + - and this pollutes the view, so remove them. + - (However, emptry directories used by submodules are not + - removed.) -} + top <- liftIO . absPath =<< fromRepo Git.repoPath + (l, cleanup) <- inRepo $ + LsFiles.notInRepoIncludingEmptyDirectories False [top] + forM_ l (removeemptydir top) + liftIO $ void cleanup + unlessM (liftIO $ doesDirectoryExist here) $ do + showLongNote (cwdmissing top) + return ok + where + removeemptydir top d = do + p <- inRepo $ toTopFilePath d + liftIO $ tryIO $ removeDirectory (top getTopFilePath p) + cwdmissing top = unlines + [ "This view does not include the subdirectory you are currently in." + , "Perhaps you should: cd " ++ top + ] diff --git a/Command/Wanted.hs b/Command/Wanted.hs new file mode 100644 index 0000000000..d9853cc356 --- /dev/null +++ b/Command/Wanted.hs @@ -0,0 +1,57 @@ +{- git-annex command + - + - Copyright 2013-2015 Joey Hess + - + - Licensed under the GNU GPL version 3 or higher. + -} + +module Command.Wanted where + +import Command +import qualified Remote +import Logs.PreferredContent +import Types.StandardGroups + +import qualified Data.Map as M + +cmd :: Command +cmd = cmd' "wanted" "get or set preferred content expression" + preferredContentMapRaw + preferredContentSet + +cmd' + :: String + -> String + -> Annex (M.Map UUID PreferredContentExpression) + -> (UUID -> PreferredContentExpression -> Annex ()) + -> Command +cmd' name desc getter setter = noMessages $ + command name SectionSetup desc pdesc (withParams seek) + where + pdesc = paramPair paramRemote (paramOptional paramExpression) + + seek = withWords (commandAction . start) + + start (rname:[]) = go rname (performGet getter) + start (rname:expr:[]) = go rname $ \uuid -> do + allowMessages + showStart' name (Just rname) + performSet setter expr uuid + start _ = giveup "Specify a repository." + + go rname a = do + u <- Remote.nameToUUID rname + next $ a u + +performGet :: Ord a => Annex (M.Map a PreferredContentExpression) -> a -> CommandPerform +performGet getter a = do + m <- getter + liftIO $ putStrLn $ fromMaybe "" $ M.lookup a m + next $ return True + +performSet :: (a -> PreferredContentExpression -> Annex ()) -> String -> a -> CommandPerform +performSet setter expr a = case checkPreferredContentExpression expr of + Just e -> giveup $ "Parse error: " ++ e + Nothing -> do + setter a expr + next $ return True diff --git a/Command/Watch.hs b/Command/Watch.hs new file mode 100644 index 0000000000..9bc92d85b1 --- /dev/null +++ b/Command/Watch.hs @@ -0,0 +1,28 @@ +{- git-annex watch command + - + - Copyright 2012 Joey Hess + - + - Licensed under the GNU GPL version 3 or higher. + -} + +module Command.Watch where + +import Command +import Assistant +import Utility.HumanTime + +cmd :: Command +cmd = notBareRepo $ + command "watch" SectionCommon + "watch for changes and autocommit" + paramNothing (seek <$$> const parseDaemonOptions) + +seek :: DaemonOptions -> CommandSeek +seek o = commandAction $ start False o Nothing + +start :: Bool -> DaemonOptions -> Maybe Duration -> CommandStart +start assistant o startdelay = do + if stopDaemonOption o + then stopDaemon + else startDaemon assistant (foregroundDaemonOption o) startdelay Nothing Nothing Nothing -- does not return + stop diff --git a/Command/WebApp.hs b/Command/WebApp.hs new file mode 100644 index 0000000000..c380c312e7 --- /dev/null +++ b/Command/WebApp.hs @@ -0,0 +1,264 @@ +{- git-annex webapp launcher + - + - Copyright 2012 Joey Hess + - + - Licensed under the GNU GPL version 3 or higher. + -} + +{-# LANGUAGE CPP #-} + +module Command.WebApp where + +import Command +import Assistant +import Assistant.Common +import Assistant.NamedThread +import Assistant.Threads.WebApp +import Assistant.WebApp +import Assistant.Install +import Annex.Environment +import Utility.WebApp +import Utility.Daemon (checkDaemon) +#ifdef __ANDROID__ +import Utility.Env +#endif +import Utility.UserInfo +import Annex.Init +import qualified Git +import qualified Git.Config +import qualified Git.CurrentRepo +import qualified Annex +import Config.Files +import Upgrade +import Annex.Version +import Utility.Android + +import Control.Concurrent +import Control.Concurrent.STM + +cmd :: Command +cmd = noCommit $ dontCheck repoExists $ notBareRepo $ + noRepo (startNoRepo <$$> optParser) $ + command "webapp" SectionCommon "launch webapp" + paramNothing (seek <$$> optParser) + +data WebAppOptions = WebAppOptions + { listenAddress :: Maybe String + } + +optParser :: CmdParamsDesc -> Parser WebAppOptions +optParser _ = WebAppOptions + <$> optional (strOption + ( long "listen" <> metavar paramAddress + <> help "accept connections to this address" + )) + +seek :: WebAppOptions -> CommandSeek +seek = commandAction . start + +start :: WebAppOptions -> CommandStart +start = start' True + +start' :: Bool -> WebAppOptions -> CommandStart +start' allowauto o = do + liftIO ensureInstalled + ifM (isInitialized <&&> notHome) + ( maybe notinitialized (go <=< needsUpgrade) =<< getVersion + , if allowauto + then liftIO $ startNoRepo o + else notinitialized + ) + stop + where + go cannotrun = do + browser <- fromRepo webBrowser + f <- liftIO . absPath =<< fromRepo gitAnnexHtmlShim + listenAddress' <- if isJust (listenAddress o) + then pure (listenAddress o) + else annexListen <$> Annex.getGitConfig + ifM (checkpid <&&> checkshim f) + ( if isJust (listenAddress o) + then giveup "The assistant is already running, so --listen cannot be used." + else do + url <- liftIO . readFile + =<< fromRepo gitAnnexUrlFile + liftIO $ if isJust listenAddress' + then putStrLn url + else liftIO $ openBrowser browser f url Nothing Nothing + , do + startDaemon True True Nothing cannotrun listenAddress' $ Just $ + \origout origerr url htmlshim -> + if isJust listenAddress' + then maybe noop (`hPutStrLn` url) origout + else openBrowser browser htmlshim url origout origerr + ) + checkpid = do + pidfile <- fromRepo gitAnnexPidFile + liftIO $ isJust <$> checkDaemon pidfile + checkshim f = liftIO $ doesFileExist f + notinitialized = do + g <- Annex.gitRepo + liftIO $ cannotStartIn (Git.repoLocation g) "repository has not been initialized by git-annex" + liftIO $ firstRun o + +{- If HOME is a git repo, even if it's initialized for git-annex, + - the user almost certianly does not want to run the assistant there. -} +notHome :: Annex Bool +notHome = do + g <- Annex.gitRepo + d <- liftIO $ absPath (Git.repoLocation g) + h <- liftIO $ absPath =<< myHomeDir + return (d /= h) + +{- When run without a repo, start the first available listed repository in + - the autostart file. If none, it's our first time being run! -} +startNoRepo :: WebAppOptions -> IO () +startNoRepo o = go =<< liftIO (filterM doesDirectoryExist =<< readAutoStartFile) + where + go [] = firstRun o + go (d:ds) = do + v <- tryNonAsync $ do + setCurrentDirectory d + Annex.new =<< Git.CurrentRepo.get + case v of + Left e -> do + cannotStartIn d (show e) + go ds + Right state -> void $ Annex.eval state $ do + whenM (fromRepo Git.repoIsLocalBare) $ + giveup $ d ++ " is a bare git repository, cannot run the webapp in it" + callCommandAction $ + start' False o + +cannotStartIn :: FilePath -> String -> IO () +cannotStartIn d reason = warningIO $ "unable to start webapp in repository " ++ d ++ ": " ++ reason + +{- Run the webapp without a repository, which prompts the user, makes one, + - changes to it, starts the regular assistant, and redirects the + - browser to its url. + - + - This is a very tricky dance -- The first webapp calls the signaler, + - which signals the main thread when it's ok to continue by writing to a + - MVar. The main thread starts the second webapp, and uses its callback + - to write its url back to the MVar, from where the signaler retrieves it, + - returning it to the first webapp, which does the redirect. + - + - Note that it's important that mainthread never terminates! Much + - of this complication is due to needing to keep the mainthread running. + -} +firstRun :: WebAppOptions -> IO () +firstRun o = do + checkEnvironmentIO + {- Without a repository, we cannot have an Annex monad, so cannot + - get a ThreadState. This is only safe because the + - webapp checks its noAnnex field before accessing the + - threadstate. -} + let st = error "annex state not available" + {- Get a DaemonStatus without running in the Annex monad. -} + dstatus <- atomically . newTVar =<< newDaemonStatus + d <- newAssistantData st dstatus + urlrenderer <- newUrlRenderer + v <- newEmptyMVar + let callback a = Just $ a v + runAssistant d $ do + startNamedThread urlrenderer $ + webAppThread d urlrenderer True Nothing + (callback signaler) + (listenAddress o) + (callback mainthread) + waitNamedThreads + where + signaler v = do + putMVar v "" + takeMVar v + mainthread v url htmlshim + | isJust (listenAddress o)= do + putStrLn url + hFlush stdout + go + | otherwise = do + browser <- maybe Nothing webBrowser + <$> catchDefaultIO Nothing Git.Config.global + openBrowser browser htmlshim url Nothing Nothing + go + where + go = do + _wait <- takeMVar v + state <- Annex.new =<< Git.CurrentRepo.get + Annex.eval state $ + startDaemon True True Nothing Nothing (listenAddress o) $ Just $ + sendurlback v + sendurlback v _origout _origerr url _htmlshim = do + recordUrl url + putMVar v url + +recordUrl :: String -> IO () +#ifdef __ANDROID__ +{- The Android app has a menu item that opens the url recorded + - in this file. -} +recordUrl url = writeFile "/sdcard/git-annex.home/.git-annex-url" url +#else +recordUrl _ = noop +#endif + +openBrowser :: Maybe FilePath -> FilePath -> String -> Maybe Handle -> Maybe Handle -> IO () +openBrowser mcmd htmlshim realurl outh errh = do + htmlshim' <- absPath htmlshim + openBrowser' mcmd htmlshim' realurl outh errh + +openBrowser' :: Maybe FilePath -> FilePath -> String -> Maybe Handle -> Maybe Handle -> IO () +#ifndef __ANDROID__ +openBrowser' mcmd htmlshim realurl outh errh = + ifM osAndroid + {- Android does not support file:// urls well, but neither + - is the security of the url in the process table important + - there, so just use the real url. -} + ( runbrowser realurl + , runbrowser (fileUrl htmlshim) + ) +#else +openBrowser' mcmd htmlshim realurl outh errh = do + recordUrl realurl + {- Android's `am` command does not work reliably across the + - wide range of Android devices. Intead, FIFO should be set to + - the filename of a fifo that we can write the URL to. -} + v <- getEnv "FIFO" + case v of + Nothing -> runbrowser realurl + Just f -> void $ forkIO $ do + fd <- openFd f WriteOnly Nothing defaultFileFlags + void $ fdWrite fd realurl + closeFd fd +#endif + where + runbrowser url = do + let p = case mcmd of + Just c -> proc c [url] + Nothing -> +#ifndef mingw32_HOST_OS + browserProc url +#else + {- Windows hack to avoid using the full path, + - which might contain spaces that cause problems + - for browserProc. -} + (browserProc (takeFileName htmlshim)) + { cwd = Just (takeDirectory htmlshim) } +#endif + hPutStrLn (fromMaybe stdout outh) $ "Launching web browser on " ++ url + hFlush stdout + environ <- cleanEnvironment + (_, _, _, pid) <- createProcess p + { env = environ + , std_out = maybe Inherit UseHandle outh + , std_err = maybe Inherit UseHandle errh + } + exitcode <- waitForProcess pid + unless (exitcode == ExitSuccess) $ + hPutStrLn (fromMaybe stderr errh) "failed to start web browser" + +{- web.browser is a generic git config setting for a web browser program -} +webBrowser :: Git.Repo -> Maybe FilePath +webBrowser = Git.Config.getMaybe "web.browser" + +fileUrl :: FilePath -> String +fileUrl file = "file://" ++ file diff --git a/Command/Whereis.hs b/Command/Whereis.hs new file mode 100644 index 0000000000..07081da78b --- /dev/null +++ b/Command/Whereis.hs @@ -0,0 +1,111 @@ +{- git-annex command + - + - Copyright 2010-2016 Joey Hess + - + - Licensed under the GNU GPL version 3 or higher. + -} + +module Command.Whereis where + +import Command +import Remote +import Logs.Trust +import Logs.Web +import Remote.Web (getWebUrls) +import Annex.UUID + +import qualified Data.Map as M +import qualified Data.Vector as V + +cmd :: Command +cmd = noCommit $ withGlobalOptions [jsonOptions, annexedMatchingOptions] $ + command "whereis" SectionQuery + "lists repositories that have file content" + paramPaths (seek <$$> optParser) + +data WhereisOptions = WhereisOptions + { whereisFiles :: CmdParams + , keyOptions :: Maybe KeyOptions + , batchOption :: BatchMode + } + +optParser :: CmdParamsDesc -> Parser WhereisOptions +optParser desc = WhereisOptions + <$> cmdParams desc + <*> optional parseKeyOptions + <*> parseBatchOption + +seek :: WhereisOptions -> CommandSeek +seek o = do + m <- remoteMap id + let go = whenAnnexed $ start m + case batchOption o of + Batch fmt -> batchFilesMatching fmt go + NoBatch -> + withKeyOptions (keyOptions o) False + (commandAction . startKeys m) + (withFilesInGit (commandAction . go)) + =<< workTreeItems (whereisFiles o) + +start :: M.Map UUID Remote -> FilePath -> Key -> CommandStart +start remotemap file key = startKeys remotemap (key, mkActionItem afile) + where + afile = AssociatedFile (Just file) + +startKeys :: M.Map UUID Remote -> (Key, ActionItem) -> CommandStart +startKeys remotemap (key, ai) = do + showStartKey "whereis" key ai + next $ perform remotemap key + +perform :: M.Map UUID Remote -> Key -> CommandPerform +perform remotemap key = do + locations <- keyLocations key + urls <- getUUIDUrls key locations remotemap + (untrustedlocations, safelocations) <- trustPartition UnTrusted locations + let num = length safelocations + showNote $ show num ++ " " ++ copiesplural num + pp <- ppwhereis "whereis" safelocations urls + unless (null safelocations) $ showLongNote pp + pp' <- ppwhereis "untrusted" untrustedlocations urls + unless (null untrustedlocations) $ showLongNote $ untrustedheader ++ pp' + + mapM_ (showRemoteUrls remotemap) urls + + if null safelocations then stop else next $ return True + where + copiesplural 1 = "copy" + copiesplural _ = "copies" + untrustedheader = "The following untrusted locations may also have copies:\n" + ppwhereis h ls urls = do + descm <- uuidDescriptions + let urlvals = map (\(u, us) -> (u, Just (V.fromList us))) $ + filter (\(u,_) -> u `elem` ls) urls + prettyPrintUUIDsWith (Just "urls") h descm (const Nothing) urlvals + +getUUIDUrls :: Key -> [UUID] -> M.Map UUID Remote -> Annex [(UUID, [URLString])] +getUUIDUrls key uuids remotemap = forM uuids $ \uu -> (,) + <$> pure uu + <*> maybe (pure []) (getRemoteUrls key) (M.lookup uu remotemap) + +getRemoteUrls :: Key -> Remote -> Annex [URLString] +getRemoteUrls key remote + | uuid remote == webUUID = getWebUrls key + | otherwise = (++) + <$> askremote + <*> claimedurls + where + askremote = maybe (pure []) (flip id key) (whereisKey remote) + claimedurls = do + us <- map fst + . filter (\(_, d) -> d == OtherDownloader) + . map getDownloader + <$> getUrls key + filterM (\u -> (==) <$> pure remote <*> claimingUrl u) us + +showRemoteUrls :: M.Map UUID Remote -> (UUID, [URLString]) -> Annex () +showRemoteUrls remotemap (uu, us) + | null us = noop + | otherwise = case M.lookup uu remotemap of + Just r -> showLongNote $ + unlines $ map (\u -> name r ++ ": " ++ u) us + Nothing -> noop diff --git a/Common.hs b/Common.hs new file mode 100644 index 0000000000..9505620ae5 --- /dev/null +++ b/Common.hs @@ -0,0 +1,33 @@ +{-# LANGUAGE PackageImports #-} + +module Common (module X) where + +import Control.Monad as X +import Control.Monad.IfElse as X +import Control.Applicative as X +import Control.Monad.IO.Class as X (liftIO) + +import Data.Maybe as X +import Data.List as X hiding (head, tail, init, last) +import Data.Monoid as X +import Data.Default as X + +import System.FilePath as X +import System.IO as X hiding (FilePath) +import System.Exit as X +import System.PosixCompat.Files as X hiding (fileSize) + +import Utility.Misc as X +import Utility.Exception as X +import Utility.SafeCommand as X +import Utility.Process as X +import Utility.Path as X +import Utility.Directory as X +import Utility.Monad as X +import Utility.Data as X +import Utility.Applicative as X +import Utility.FileSize as X +import Utility.Network as X +import Utility.Split as X + +import Utility.PartialPrelude as X diff --git a/Config.hs b/Config.hs new file mode 100644 index 0000000000..94d67ce044 --- /dev/null +++ b/Config.hs @@ -0,0 +1,104 @@ +{- Git configuration + - + - Copyright 2011-2017 Joey Hess + - + - Licensed under the GNU GPL version 3 or higher. + -} + +{-# LANGUAGE TypeSynonymInstances, FlexibleInstances #-} + +module Config where + +import Annex.Common +import qualified Git +import qualified Git.Config +import qualified Git.Command +import qualified Annex +import Config.Cost +import Config.DynamicConfig +import Types.Availability +import Git.Types +import qualified Types.Remote as Remote + +type UnqualifiedConfigKey = String +data ConfigKey = ConfigKey String + +instance Show ConfigKey where + show (ConfigKey s) = s + +{- Looks up a setting in git config. This is not as efficient as using the + - GitConfig type. -} +getConfig :: ConfigKey -> String -> Annex String +getConfig (ConfigKey key) d = fromRepo $ Git.Config.get key d + +getConfigMaybe :: ConfigKey -> Annex (Maybe String) +getConfigMaybe (ConfigKey key) = fromRepo $ Git.Config.getMaybe key + +{- Changes a git config setting in both internal state and .git/config -} +setConfig :: ConfigKey -> String -> Annex () +setConfig (ConfigKey key) value = do + inRepo $ Git.Command.run [Param "config", Param key, Param value] + reloadConfig + +reloadConfig :: Annex () +reloadConfig = Annex.changeGitRepo =<< inRepo Git.Config.reRead + +{- Unsets a git config setting. (Leaves it in state.) -} +unsetConfig :: ConfigKey -> Annex () +unsetConfig (ConfigKey key) = void $ inRepo $ Git.Config.unset key + +class RemoteNameable r where + getRemoteName :: r -> RemoteName + +instance RemoteNameable Git.Repo where + getRemoteName r = fromMaybe "" (Git.remoteName r) + +instance RemoteNameable RemoteName where + getRemoteName = id + +instance RemoteNameable Remote where + getRemoteName = Remote.name + +{- A per-remote config setting in git config. -} +remoteConfig :: RemoteNameable r => r -> UnqualifiedConfigKey -> ConfigKey +remoteConfig r key = ConfigKey $ + "remote." ++ getRemoteName r ++ ".annex-" ++ key + +{- A global annex setting in git config. -} +annexConfig :: UnqualifiedConfigKey -> ConfigKey +annexConfig key = ConfigKey $ "annex." ++ key + +{- Calculates cost for a remote. Either the specific default, or as configured + - by remote..annex-cost, or if remote..annex-cost-command + - is set and prints a number, that is used. -} +remoteCost :: RemoteGitConfig -> Cost -> Annex Cost +remoteCost c d = fromMaybe d <$> remoteCost' c + +remoteCost' :: RemoteGitConfig -> Annex (Maybe Cost) +remoteCost' = liftIO . getDynamicConfig . remoteAnnexCost + +setRemoteCost :: Git.Repo -> Cost -> Annex () +setRemoteCost r c = setConfig (remoteConfig r "cost") (show c) + +setRemoteAvailability :: Git.Repo -> Availability -> Annex () +setRemoteAvailability r c = setConfig (remoteConfig r "availability") (show c) + +setRemoteIgnore :: Git.Repo -> Bool -> Annex () +setRemoteIgnore r b = setConfig (remoteConfig r "ignore") (Git.Config.boolConfig b) + +setRemoteBare :: Git.Repo -> Bool -> Annex () +setRemoteBare r b = setConfig (remoteConfig r "bare") (Git.Config.boolConfig b) + +isBareRepo :: Annex Bool +isBareRepo = fromRepo Git.repoIsLocalBare + +isDirect :: Annex Bool +isDirect = annexDirect <$> Annex.getGitConfig + +crippledFileSystem :: Annex Bool +crippledFileSystem = annexCrippledFileSystem <$> Annex.getGitConfig + +setCrippledFileSystem :: Bool -> Annex () +setCrippledFileSystem b = do + setConfig (annexConfig "crippledfilesystem") (Git.Config.boolConfig b) + Annex.changeGitConfig $ \c -> c { annexCrippledFileSystem = b } diff --git a/Config/Cost.hs b/Config/Cost.hs new file mode 100644 index 0000000000..026a905762 --- /dev/null +++ b/Config/Cost.hs @@ -0,0 +1,82 @@ +{- Remote costs. + - + - Copyright 2011-2013 Joey Hess + - + - Licensed under the GNU GPL version 3 or higher. + -} + +module Config.Cost where + +{- We use a float for a cost to ensure that there is a cost in + - between any two other costs. -} +type Cost = Float + +{- Some predefined default costs. + - Users setting costs in config files can be aware of these, + - and pick values relative to them. So don't change. -} +cheapRemoteCost :: Cost +cheapRemoteCost = 100 +nearlyCheapRemoteCost :: Cost +nearlyCheapRemoteCost = 110 +semiExpensiveRemoteCost :: Cost +semiExpensiveRemoteCost = 175 +expensiveRemoteCost :: Cost +expensiveRemoteCost = 200 +veryExpensiveRemoteCost :: Cost +veryExpensiveRemoteCost = 1000 + +{- Adjusts a remote's cost to reflect it being encrypted. -} +encryptedRemoteCostAdj :: Cost +encryptedRemoteCostAdj = 50 + +{- Given an ordered list of costs, and the position of one of the items + - the list, inserts a new cost into the list, in between the item + - and the item after it. + - + - If two or move items have the same cost, their costs are adjusted + - to make room. The costs of other items in the list are left + - unchanged. + - + - To insert the new cost before any other in the list, specify a negative + - position. To insert the new cost at the end of the list, specify a + - position longer than the list. + -} +insertCostAfter :: [Cost] -> Int -> [Cost] +insertCostAfter [] _ = [] +insertCostAfter l pos + | pos < 0 = costBetween 0 (l !! 0) : l + | nextpos > maxpos = l ++ [1 + l !! maxpos] + | item == nextitem = + let (_dup:new:l') = insertCostAfter lastsegment 0 + in firstsegment ++ [costBetween item new, new] ++ l' + | otherwise = + firstsegment ++ [costBetween item nextitem ] ++ lastsegment + where + nextpos = pos + 1 + maxpos = length l - 1 + + item = l !! pos + nextitem = l !! nextpos + + (firstsegment, lastsegment) = splitAt (pos + 1) l + +costBetween :: Cost -> Cost -> Cost +costBetween x y + | x == y = x + | x > y = -- avoid fractions unless needed + let mid = y + (x - y) / 2 + mid' = fromIntegral (floor mid :: Int) + in if mid' > y then mid' else mid + | otherwise = costBetween y x + +{- Make sure the remote cost numbers work out. -} +prop_cost_sane :: Bool +prop_cost_sane = False `notElem` + [ expensiveRemoteCost > 0 + , cheapRemoteCost < nearlyCheapRemoteCost + , nearlyCheapRemoteCost < semiExpensiveRemoteCost + , semiExpensiveRemoteCost < expensiveRemoteCost + , cheapRemoteCost + encryptedRemoteCostAdj > nearlyCheapRemoteCost + , nearlyCheapRemoteCost + encryptedRemoteCostAdj < semiExpensiveRemoteCost + , nearlyCheapRemoteCost + encryptedRemoteCostAdj < expensiveRemoteCost + ] diff --git a/Config/DynamicConfig.hs b/Config/DynamicConfig.hs new file mode 100644 index 0000000000..de76e007b5 --- /dev/null +++ b/Config/DynamicConfig.hs @@ -0,0 +1,47 @@ +{- dynamic configuration + - + - Copyright 2017 Joey Hess + - + - Licensed under the GNU GPL version 3 or higher. + -} + +module Config.DynamicConfig where + +import Common + +import Control.Concurrent.STM + +-- | A configuration value that may only be known after performing an IO +-- action. The IO action will only be run the first time the configuration +-- is accessed; its result is then cached. +data DynamicConfig a = DynamicConfig (IO a, TMVar a) | StaticConfig a + +mkDynamicConfig :: CommandRunner a -> Maybe String -> a -> STM (DynamicConfig a) +mkDynamicConfig _ Nothing static = return $ StaticConfig static +mkDynamicConfig cmdrunner (Just cmd) _ = do + tmvar <- newEmptyTMVar + return $ DynamicConfig (cmdrunner cmd, tmvar) + +getDynamicConfig :: DynamicConfig a -> IO a +getDynamicConfig (StaticConfig v) = return v +getDynamicConfig (DynamicConfig (a, tmvar)) = + go =<< atomically (tryReadTMVar tmvar) + where + go Nothing = do + v <- a + atomically $ do + _ <- tryTakeTMVar tmvar + putTMVar tmvar v + return v + go (Just v) = return v + +type CommandRunner a = String -> IO a + +successfullCommandRunner :: CommandRunner Bool +successfullCommandRunner cmd = boolSystem "sh" [Param "-c", Param cmd] + +unsuccessfullCommandRunner :: CommandRunner Bool +unsuccessfullCommandRunner cmd = not <$> successfullCommandRunner cmd + +readCommandRunner :: Read a => CommandRunner (Maybe a) +readCommandRunner cmd = readish <$> readProcess "sh" ["-c", cmd] diff --git a/Config/Files.hs b/Config/Files.hs new file mode 100644 index 0000000000..b18d912e9e --- /dev/null +++ b/Config/Files.hs @@ -0,0 +1,83 @@ +{- git-annex extra config files + - + - Copyright 2012 Joey Hess + - + - Licensed under the GNU GPL version 3 or higher. + -} + +{-# OPTIONS_GHC -fno-warn-tabs #-} + +module Config.Files where + +import Common +import Utility.Tmp +import Utility.FreeDesktop + +{- ~/.config/git-annex/file -} +userConfigFile :: FilePath -> IO FilePath +userConfigFile file = do + dir <- userConfigDir + return $ dir "git-annex" file + +autoStartFile :: IO FilePath +autoStartFile = userConfigFile "autostart" + +{- Returns anything listed in the autostart file (which may not exist). -} +readAutoStartFile :: IO [FilePath] +readAutoStartFile = do + f <- autoStartFile + filter valid . nub . map dropTrailingPathSeparator . lines + <$> catchDefaultIO "" (readFile f) + where + -- Ignore any relative paths; some old buggy versions added eg "." + valid = isAbsolute + +modifyAutoStartFile :: ([FilePath] -> [FilePath]) -> IO () +modifyAutoStartFile func = do + dirs <- readAutoStartFile + let dirs' = nubBy equalFilePath $ func dirs + when (dirs' /= dirs) $ do + f <- autoStartFile + createDirectoryIfMissing True (parentDir f) + viaTmp writeFile f $ unlines dirs' + +{- Adds a directory to the autostart file. If the directory is already + - present, it's moved to the top, so it will be used as the default + - when opening the webapp. -} +addAutoStartFile :: FilePath -> IO () +addAutoStartFile path = do + path' <- absPath path + modifyAutoStartFile $ (:) path' + +{- Removes a directory from the autostart file. -} +removeAutoStartFile :: FilePath -> IO () +removeAutoStartFile path = do + path' <- absPath path + modifyAutoStartFile $ + filter (not . equalFilePath path') + +{- The path to git-annex is written here; which is useful when cabal + - has installed it to some awful non-PATH location. -} +programFile :: IO FilePath +programFile = userConfigFile "program" + +{- Returns a command to run for git-annex. -} +readProgramFile :: IO FilePath +readProgramFile = do + programfile <- programFile + p <- catchDefaultIO cmd $ + fromMaybe cmd . headMaybe . lines <$> readFile programfile + ifM (inPath p) + ( return p + , ifM (inPath cmd) + ( return cmd + , cannotFindProgram + ) + ) + where + cmd = "git-annex" + +cannotFindProgram :: IO a +cannotFindProgram = do + f <- programFile + giveup $ "cannot find git-annex program in PATH or in the location listed in " ++ f diff --git a/Config/GitConfig.hs b/Config/GitConfig.hs new file mode 100644 index 0000000000..bac866f0e9 --- /dev/null +++ b/Config/GitConfig.hs @@ -0,0 +1,36 @@ +{- git-annex configuration + - + - Copyright 2017 Joey Hess + - + - Licensed under the GNU GPL version 3 or higher. + -} + +module Config.GitConfig where + +import Annex.Common +import qualified Annex +import Types.GitConfig +import Git.Types +import Logs.Config + +{- Gets a specific setting from GitConfig. If necessary, loads the + - repository-global defaults when the GitConfig does not yet + - have a value. -} +getGitConfigVal :: (GitConfig -> Configurable a) -> Annex a +getGitConfigVal f = do + v <- f <$> Annex.getGitConfig + case v of + HasConfig c -> return c + DefaultConfig _ -> do + r <- Annex.gitRepo + m <- loadGlobalConfig + let globalgc = extractGitConfig (r { config = m }) + -- This merge of the repo-global config and the git + -- config makes all repository-global default + -- values populate the GitConfig with HasConfig + -- values, so it will only need to be done once. + Annex.changeGitConfig (\gc -> mergeGitConfig gc globalgc) + v' <- f <$> Annex.getGitConfig + case v' of + HasConfig c -> return c + DefaultConfig d -> return d diff --git a/Config/Smudge.hs b/Config/Smudge.hs new file mode 100644 index 0000000000..3ef7d4b2fe --- /dev/null +++ b/Config/Smudge.hs @@ -0,0 +1,41 @@ +{- Git smudge filter configuration + - + - Copyright 2011-2018 Joey Hess + - + - Licensed under the GNU GPL version 3 or higher. + -} + +module Config.Smudge where + +import Annex.Common +import qualified Annex +import qualified Git +import qualified Git.Command +import Config + +configureSmudgeFilter :: Annex () +configureSmudgeFilter = unlessM (fromRepo Git.repoIsLocalBare) $ do + -- If this is run in a newly cloned repository, git may not have + -- cached file information in the index yet, and so after + -- configuring the clean filter, the next git status would want to + -- run it on every file. That is expensive and can also result in + -- unexpected changes when the file is checked into git or annex + -- counter to the annex.largefiles configuration. + -- Avoid that problem by running git status now. + inRepo $ Git.Command.runQuiet [Param "status", Param "--porcelain"] + + setConfig (ConfigKey "filter.annex.smudge") "git-annex smudge %f" + setConfig (ConfigKey "filter.annex.clean") "git-annex smudge --clean %f" + lf <- Annex.fromRepo Git.attributesLocal + gf <- Annex.fromRepo Git.attributes + lfs <- readattr lf + gfs <- readattr gf + liftIO $ unless ("filter=annex" `isInfixOf` (lfs ++ gfs)) $ do + createDirectoryIfMissing True (takeDirectory lf) + writeFile lf (lfs ++ "\n" ++ stdattr) + where + readattr = liftIO . catchDefaultIO "" . readFileStrict + stdattr = unlines + [ "* filter=annex" + , ".* !filter" + ] diff --git a/Creds.hs b/Creds.hs new file mode 100644 index 0000000000..b5181aa1e3 --- /dev/null +++ b/Creds.hs @@ -0,0 +1,201 @@ +{- Credentials storage + - + - Copyright 2012-2014 Joey Hess + - + - Licensed under the GNU GPL version 3 or higher. + -} + +module Creds ( + module Types.Creds, + CredPairStorage(..), + setRemoteCredPair, + getRemoteCredPair, + getRemoteCredPairFor, + warnMissingCredPairFor, + getEnvCredPair, + writeCacheCreds, + readCacheCreds, + cacheCredsFile, + removeCreds, + includeCredsInfo, +) where + +import Annex.Common +import qualified Annex +import Types.Creds +import Annex.Perms +import Utility.FileMode +import Crypto +import Types.Remote (RemoteConfig, RemoteConfigKey) +import Remote.Helper.Encryptable (remoteCipher, remoteCipher', embedCreds, EncryptionIsSetup, extractCipher) +import Utility.Env (getEnv) + +import qualified Data.ByteString.Lazy.Char8 as L +import qualified Data.Map as M +import Utility.Base64 + +{- A CredPair can be stored in a file, or in the environment, or perhaps + - in a remote's configuration. -} +data CredPairStorage = CredPairStorage + { credPairFile :: FilePath + , credPairEnvironment :: (String, String) + , credPairRemoteKey :: Maybe RemoteConfigKey + } + +{- Stores creds in a remote's configuration, if the remote allows + - that. Also caches them locally. + - + - The creds are found from the CredPairStorage storage if not provided, + - so may be provided by an environment variable etc. + - + - The remote's configuration should have already had a cipher stored in it + - if that's going to be done, so that the creds can be encrypted using the + - cipher. The EncryptionIsSetup phantom type ensures that is the case. + -} +setRemoteCredPair :: EncryptionIsSetup -> RemoteConfig -> RemoteGitConfig -> CredPairStorage -> Maybe CredPair -> Annex RemoteConfig +setRemoteCredPair encsetup c gc storage mcreds = case mcreds of + Nothing -> maybe (return c) (setRemoteCredPair encsetup c gc storage . Just) + =<< getRemoteCredPair c gc storage + Just creds + | embedCreds c -> case credPairRemoteKey storage of + Nothing -> localcache creds + Just key -> storeconfig creds key =<< flip remoteCipher gc =<< localcache creds + | otherwise -> localcache creds + where + localcache creds = do + writeCacheCredPair creds storage + return c + + storeconfig creds key (Just cipher) = do + cmd <- gpgCmd <$> Annex.getGitConfig + s <- liftIO $ encrypt cmd (c, gc) cipher + (feedBytes $ L.pack $ encodeCredPair creds) + (readBytes $ return . L.unpack) + return $ M.insert key (toB64 s) c + storeconfig creds key Nothing = + return $ M.insert key (toB64 $ encodeCredPair creds) c + +{- Gets a remote's credpair, from the environment if set, otherwise + - from the cache in gitAnnexCredsDir, or failing that, from the + - value in RemoteConfig. -} +getRemoteCredPair :: RemoteConfig -> RemoteGitConfig -> CredPairStorage -> Annex (Maybe CredPair) +getRemoteCredPair c gc storage = maybe fromcache (return . Just) =<< fromenv + where + fromenv = liftIO $ getEnvCredPair storage + fromcache = maybe fromconfig (return . Just) =<< readCacheCredPair storage + fromconfig = case credPairRemoteKey storage of + Just key -> do + mcipher <- remoteCipher' c gc + case (M.lookup key c, mcipher) of + (Nothing, _) -> return Nothing + (Just enccreds, Just (cipher, storablecipher)) -> + fromenccreds enccreds cipher storablecipher + (Just bcreds, Nothing) -> + fromcreds $ fromB64 bcreds + Nothing -> return Nothing + fromenccreds enccreds cipher storablecipher = do + cmd <- gpgCmd <$> Annex.getGitConfig + mcreds <- liftIO $ catchMaybeIO $ decrypt cmd (c, gc) cipher + (feedBytes $ L.pack $ fromB64 enccreds) + (readBytes $ return . L.unpack) + case mcreds of + Just creds -> fromcreds creds + Nothing -> do + -- Work around un-encrypted creds storage + -- bug in old S3 and glacier remotes. + -- Not a problem for shared cipher. + case storablecipher of + SharedCipher {} -> showLongNote "gpg error above was caused by an old git-annex bug in credentials storage. Working around it.." + _ -> giveup "*** Insecure credentials storage detected for this remote! See https://git-annex.branchable.com/upgrades/insecure_embedded_creds/" + fromcreds $ fromB64 enccreds + fromcreds creds = case decodeCredPair creds of + Just credpair -> do + writeCacheCredPair credpair storage + + return $ Just credpair + _ -> error "bad creds" + +getRemoteCredPairFor :: String -> RemoteConfig -> RemoteGitConfig -> CredPairStorage -> Annex (Maybe CredPair) +getRemoteCredPairFor this c gc storage = go =<< getRemoteCredPair c gc storage + where + go Nothing = do + warnMissingCredPairFor this storage + return Nothing + go (Just credpair) = return $ Just credpair + +warnMissingCredPairFor :: String -> CredPairStorage -> Annex () +warnMissingCredPairFor this storage = warning $ unwords + [ "Set both", loginvar + , "and", passwordvar + , "to use", this + ] + where + (loginvar, passwordvar) = credPairEnvironment storage + +{- Gets a CredPair from the environment. -} +getEnvCredPair :: CredPairStorage -> IO (Maybe CredPair) +getEnvCredPair storage = liftM2 (,) + <$> getEnv uenv + <*> getEnv penv + where + (uenv, penv) = credPairEnvironment storage + +writeCacheCredPair :: CredPair -> CredPairStorage -> Annex () +writeCacheCredPair credpair storage = + writeCacheCreds (encodeCredPair credpair) (credPairFile storage) + +{- Stores the creds in a file inside gitAnnexCredsDir that only the user + - can read. -} +writeCacheCreds :: Creds -> FilePath -> Annex () +writeCacheCreds creds file = do + d <- fromRepo gitAnnexCredsDir + createAnnexDirectory d + liftIO $ writeFileProtected (d file) creds + +readCacheCredPair :: CredPairStorage -> Annex (Maybe CredPair) +readCacheCredPair storage = maybe Nothing decodeCredPair + <$> readCacheCreds (credPairFile storage) + +readCacheCreds :: FilePath -> Annex (Maybe Creds) +readCacheCreds f = liftIO . catchMaybeIO . readFileStrict =<< cacheCredsFile f + +cacheCredsFile :: FilePath -> Annex FilePath +cacheCredsFile basefile = do + d <- fromRepo gitAnnexCredsDir + return $ d basefile + +existsCacheCredPair :: CredPairStorage -> Annex Bool +existsCacheCredPair storage = + liftIO . doesFileExist =<< cacheCredsFile (credPairFile storage) + +encodeCredPair :: CredPair -> Creds +encodeCredPair (l, p) = unlines [l, p] + +decodeCredPair :: Creds -> Maybe CredPair +decodeCredPair creds = case lines creds of + l:p:[] -> Just (l, p) + _ -> Nothing + +removeCreds :: FilePath -> Annex () +removeCreds file = do + d <- fromRepo gitAnnexCredsDir + let f = d file + liftIO $ nukeFile f + +includeCredsInfo :: RemoteConfig -> CredPairStorage -> [(String, String)] -> Annex [(String, String)] +includeCredsInfo c storage info = do + v <- liftIO $ getEnvCredPair storage + case v of + Just _ -> do + let (uenv, penv) = credPairEnvironment storage + ret $ "from environment variables (" ++ unwords [uenv, penv] ++ ")" + Nothing -> case (`M.lookup` c) =<< credPairRemoteKey storage of + Nothing -> ifM (existsCacheCredPair storage) + ( ret "stored locally" + , ret "not available" + ) + Just _ -> case extractCipher c of + Just (EncryptedCipher {}) -> ret "embedded in git repository (gpg encrypted)" + _ -> ret "embedded in git repository (not encrypted)" + where + ret s = return $ ("creds", s) : info diff --git a/Crypto.hs b/Crypto.hs new file mode 100644 index 0000000000..a375286194 --- /dev/null +++ b/Crypto.hs @@ -0,0 +1,246 @@ +{- git-annex crypto + - + - Currently using gpg; could later be modified to support different + - crypto backends if neccessary. + - + - Copyright 2011-2016 Joey Hess + - + - Licensed under the GNU GPL version 3 or higher. + -} + +{-# LANGUAGE FlexibleInstances #-} +{-# LANGUAGE Rank2Types #-} + +module Crypto ( + Cipher, + KeyIds(..), + EncKey, + StorableCipher(..), + genEncryptedCipher, + genSharedCipher, + genSharedPubKeyCipher, + updateCipherKeyIds, + decryptCipher, + encryptKey, + isEncKey, + feedFile, + feedBytes, + readBytes, + encrypt, + decrypt, + LensGpgEncParams(..), + + prop_HmacSha1WithCipher_sane +) where + +import qualified Data.ByteString.Lazy as L +import Data.ByteString.UTF8 (fromString) +import qualified Data.Map as M +import Control.Monad.IO.Class + +import Annex.Common +import qualified Utility.Gpg as Gpg +import Types.Crypto +import Types.Remote +import Types.Key + +{- The beginning of a Cipher is used for MAC'ing; the remainder is used + - as the GPG symmetric encryption passphrase when using the hybrid + - scheme. Note that the cipher itself is base-64 encoded, hence the + - string is longer than 'cipherSize': 683 characters, padded to 684. + - + - The 256 first characters that feed the MAC represent at best 192 + - bytes of entropy. However that's more than enough for both the + - default MAC algorithm, namely HMAC-SHA1, and the "strongest" + - currently supported, namely HMAC-SHA512, which respectively need + - (ideally) 64 and 128 bytes of entropy. + - + - The remaining characters (320 bytes of entropy) is enough for GnuPG's + - symetric cipher; unlike weaker public key crypto, the key does not + - need to be too large. + -} +cipherBeginning :: Int +cipherBeginning = 256 + +cipherSize :: Int +cipherSize = 512 + +cipherPassphrase :: Cipher -> String +cipherPassphrase (Cipher c) = drop cipherBeginning c +cipherPassphrase (MacOnlyCipher _) = error "MAC-only cipher" + +cipherMac :: Cipher -> String +cipherMac (Cipher c) = take cipherBeginning c +cipherMac (MacOnlyCipher c) = c + +{- Creates a new Cipher, encrypted to the specified key id. -} +genEncryptedCipher :: LensGpgEncParams c => Gpg.GpgCmd -> c -> Gpg.KeyId -> EncryptedCipherVariant -> Bool -> IO StorableCipher +genEncryptedCipher cmd c keyid variant highQuality = do + ks <- Gpg.findPubKeys cmd keyid + random <- Gpg.genRandom cmd highQuality size + encryptCipher cmd c (mkCipher random) variant ks + where + (mkCipher, size) = case variant of + Hybrid -> (Cipher, cipherSize) -- used for MAC + symmetric + PubKey -> (MacOnlyCipher, cipherBeginning) -- only used for MAC + +{- Creates a new, shared Cipher. -} +genSharedCipher :: Gpg.GpgCmd -> Bool -> IO StorableCipher +genSharedCipher cmd highQuality = + SharedCipher <$> Gpg.genRandom cmd highQuality cipherSize + +{- Creates a new, shared Cipher, and looks up the gpg public key that will + - be used for encrypting content. -} +genSharedPubKeyCipher :: Gpg.GpgCmd -> Gpg.KeyId -> Bool -> IO StorableCipher +genSharedPubKeyCipher cmd keyid highQuality = do + ks <- Gpg.findPubKeys cmd keyid + random <- Gpg.genRandom cmd highQuality cipherSize + return $ SharedPubKeyCipher random ks + +{- Updates an existing Cipher, making changes to its keyids. + - + - When the Cipher is encrypted, re-encrypts it. -} +updateCipherKeyIds :: LensGpgEncParams encparams => Gpg.GpgCmd -> encparams -> [(Bool, Gpg.KeyId)] -> StorableCipher -> IO StorableCipher +updateCipherKeyIds _ _ _ SharedCipher{} = giveup "Cannot update shared cipher" +updateCipherKeyIds _ _ [] c = return c +updateCipherKeyIds cmd encparams changes encipher@(EncryptedCipher _ variant ks) = do + ks' <- updateCipherKeyIds' cmd changes ks + cipher <- decryptCipher cmd encparams encipher + encryptCipher cmd encparams cipher variant ks' +updateCipherKeyIds cmd _ changes (SharedPubKeyCipher cipher ks) = + SharedPubKeyCipher cipher <$> updateCipherKeyIds' cmd changes ks + +updateCipherKeyIds' :: Gpg.GpgCmd -> [(Bool, Gpg.KeyId)] -> KeyIds -> IO KeyIds +updateCipherKeyIds' cmd changes (KeyIds ks) = do + dropkeys <- listKeyIds [ k | (False, k) <- changes ] + forM_ dropkeys $ \k -> unless (k `elem` ks) $ + giveup $ "Key " ++ k ++ " was not present; cannot remove." + addkeys <- listKeyIds [ k | (True, k) <- changes ] + let ks' = (addkeys ++ ks) \\ dropkeys + when (null ks') $ + giveup "Cannot remove the last key." + return $ KeyIds ks' + where + listKeyIds = concat <$$> mapM (keyIds <$$> Gpg.findPubKeys cmd) + +{- Encrypts a Cipher to the specified KeyIds. -} +encryptCipher :: LensGpgEncParams c => Gpg.GpgCmd -> c -> Cipher -> EncryptedCipherVariant -> KeyIds -> IO StorableCipher +encryptCipher cmd c cip variant (KeyIds ks) = do + -- gpg complains about duplicate recipient keyids + let ks' = nub $ sort ks + let params = concat + [ getGpgEncParamsBase c + , Gpg.pkEncTo ks' + , Gpg.stdEncryptionParams False + ] + encipher <- Gpg.pipeStrict cmd params cipher + return $ EncryptedCipher encipher variant (KeyIds ks') + where + cipher = case cip of + Cipher x -> x + MacOnlyCipher x -> x + +{- Decrypting an EncryptedCipher is expensive; the Cipher should be cached. -} +decryptCipher :: LensGpgEncParams c => Gpg.GpgCmd -> c -> StorableCipher -> IO Cipher +decryptCipher _ _ (SharedCipher t) = return $ Cipher t +decryptCipher _ _ (SharedPubKeyCipher t _) = return $ MacOnlyCipher t +decryptCipher cmd c (EncryptedCipher t variant _) = + mkCipher <$> Gpg.pipeStrict cmd params t + where + mkCipher = case variant of + Hybrid -> Cipher + PubKey -> MacOnlyCipher + params = Param "--decrypt" : getGpgDecParams c + +type EncKey = Key -> Key + +{- Generates an encrypted form of a Key. The encryption does not need to be + - reversable, nor does it need to be the same type of encryption used + - on content. It does need to be repeatable. -} +encryptKey :: Mac -> Cipher -> EncKey +encryptKey mac c k = stubKey + { keyName = macWithCipher mac c (key2file k) + , keyVariety = OtherKey (encryptedBackendNamePrefix ++ showMac mac) + } + +encryptedBackendNamePrefix :: String +encryptedBackendNamePrefix = "GPG" + +isEncKey :: Key -> Bool +isEncKey k = case keyVariety k of + OtherKey s -> encryptedBackendNamePrefix `isPrefixOf` s + _ -> False + +type Feeder = Handle -> IO () +type Reader m a = Handle -> m a + +feedFile :: FilePath -> Feeder +feedFile f h = L.hPut h =<< L.readFile f + +feedBytes :: L.ByteString -> Feeder +feedBytes = flip L.hPut + +readBytes :: (MonadIO m) => (L.ByteString -> m a) -> Reader m a +readBytes a h = liftIO (L.hGetContents h) >>= a + +{- Runs a Feeder action, that generates content that is symmetrically + - encrypted with the Cipher (unless it is empty, in which case + - public-key encryption is used) using the given gpg options, and then + - read by the Reader action. -} +encrypt :: (MonadIO m, MonadMask m, LensGpgEncParams c) => Gpg.GpgCmd -> c -> Cipher -> Feeder -> Reader m a -> m a +encrypt cmd c cipher = case cipher of + Cipher{} -> Gpg.feedRead cmd (params ++ Gpg.stdEncryptionParams True) $ + cipherPassphrase cipher + MacOnlyCipher{} -> Gpg.pipeLazy cmd $ params ++ Gpg.stdEncryptionParams False + where + params = getGpgEncParams c + +{- Runs a Feeder action, that generates content that is decrypted with the + - Cipher (or using a private key if the Cipher is empty), and read by the + - Reader action. -} +decrypt :: (MonadIO m, MonadMask m, LensGpgEncParams c) => Gpg.GpgCmd -> c -> Cipher -> Feeder -> Reader m a -> m a +decrypt cmd c cipher = case cipher of + Cipher{} -> Gpg.feedRead cmd params $ cipherPassphrase cipher + MacOnlyCipher{} -> Gpg.pipeLazy cmd params + where + params = Param "--decrypt" : getGpgDecParams c + +macWithCipher :: Mac -> Cipher -> String -> String +macWithCipher mac c = macWithCipher' mac (cipherMac c) +macWithCipher' :: Mac -> String -> String -> String +macWithCipher' mac c s = calcMac mac (fromString c) (fromString s) + +{- Ensure that macWithCipher' returns the same thing forevermore. -} +prop_HmacSha1WithCipher_sane :: Bool +prop_HmacSha1WithCipher_sane = known_good == macWithCipher' HmacSha1 "foo" "bar" + where + known_good = "46b4ec586117154dacd49d664e5d63fdc88efb51" + +class LensGpgEncParams a where + {- Base parameters for encrypting. Does not include specification + - of recipient keys. -} + getGpgEncParamsBase :: a -> [CommandParam] + {- Parameters for encrypting. When the remote is configured to use + - public-key encryption, includes specification of recipient keys. -} + getGpgEncParams :: a -> [CommandParam] + {- Parameters for decrypting. -} + getGpgDecParams :: a -> [CommandParam] + +{- Extract the GnuPG options from a pair of a Remote Config and a Remote + - Git Config. -} +instance LensGpgEncParams (RemoteConfig, RemoteGitConfig) where + getGpgEncParamsBase (_c,gc) = map Param (remoteAnnexGnupgOptions gc) + getGpgEncParams (c,gc) = getGpgEncParamsBase (c,gc) ++ + {- When the remote is configured to use public-key encryption, + - look up the recipient keys and add them to the option list. -} + case M.lookup "encryption" c of + Just "pubkey" -> Gpg.pkEncTo $ maybe [] (splitc ',') $ M.lookup "cipherkeys" c + Just "sharedpubkey" -> Gpg.pkEncTo $ maybe [] (splitc ',') $ M.lookup "pubkeys" c + _ -> [] + getGpgDecParams (_c,gc) = map Param (remoteAnnexGnupgDecryptOptions gc) + +{- Extract the GnuPG options from a Remote. -} +instance LensGpgEncParams (RemoteA a) where + getGpgEncParamsBase r = getGpgEncParamsBase (config r, gitconfig r) + getGpgEncParams r = getGpgEncParams (config r, gitconfig r) + getGpgDecParams r = getGpgDecParams (config r, gitconfig r) diff --git a/Database/Export.hs b/Database/Export.hs new file mode 100644 index 0000000000..a410489934 --- /dev/null +++ b/Database/Export.hs @@ -0,0 +1,233 @@ +{- Sqlite database used for exports to special remotes. + - + - Copyright 2017 Joey Hess + -: + - Licensed under the GNU GPL version 3 or higher. + -} + +{-# LANGUAGE QuasiQuotes, TypeFamilies, TemplateHaskell #-} +{-# LANGUAGE OverloadedStrings, GADTs, FlexibleContexts #-} +{-# LANGUAGE MultiParamTypeClasses, GeneralizedNewtypeDeriving #-} +{-# LANGUAGE RankNTypes #-} +{-# LANGUAGE CPP #-} + +module Database.Export ( + ExportHandle, + openDb, + closeDb, + flushDbQueue, + addExportedLocation, + removeExportedLocation, + getExportedLocation, + isExportDirectoryEmpty, + getExportTreeCurrent, + recordExportTreeCurrent, + getExportTree, + addExportTree, + removeExportTree, + updateExportTree, + updateExportTree', + updateExportTreeFromLog, + ExportedId, + ExportedDirectoryId, + ExportTreeId, + ExportTreeCurrentId, +) where + +import Database.Types +import qualified Database.Queue as H +import Database.Init +import Annex.Locations +import Annex.Common hiding (delete) +import Types.Export +import Annex.Export +import qualified Logs.Export as Log +import Annex.LockFile +import Git.Types +import Git.Sha +import Git.FilePath +import qualified Git.DiffTree + +import Database.Persist.TH +import Database.Esqueleto hiding (Key) + +data ExportHandle = ExportHandle H.DbQueue UUID + +share [mkPersist sqlSettings, mkMigrate "migrateExport"] [persistLowerCase| +-- Files that have been exported to the remote and are present on it. +Exported + key IKey + file SFilePath + ExportedIndex key file +-- Directories that exist on the remote, and the files that are in them. +ExportedDirectory + subdir SFilePath + file SFilePath + ExportedDirectoryIndex subdir file +-- The content of the tree that has been exported to the remote. +-- Not all of these files are necessarily present on the remote yet. +ExportTree + key IKey + file SFilePath + ExportTreeIndex key file +-- The tree stored in ExportTree +ExportTreeCurrent + tree SRef + UniqueTree tree +|] + +{- Opens the database, creating it if it doesn't exist yet. + - + - Only a single process should write to the export at a time, so guard + - any writes with the gitAnnexExportLock. + -} +openDb :: UUID -> Annex ExportHandle +openDb u = do + dbdir <- fromRepo (gitAnnexExportDbDir u) + let db = dbdir "db" + unlessM (liftIO $ doesFileExist db) $ do + initDb db $ void $ + runMigrationSilent migrateExport + h <- liftIO $ H.openDbQueue H.SingleWriter db "exported" + return $ ExportHandle h u + +closeDb :: ExportHandle -> Annex () +closeDb (ExportHandle h _) = liftIO $ H.closeDbQueue h + +queueDb :: ExportHandle -> SqlPersistM () -> IO () +queueDb (ExportHandle h _) = H.queueDb h checkcommit + where + -- commit queue after 1000 changes + checkcommit sz _lastcommittime + | sz > 1000 = return True + | otherwise = return False + +flushDbQueue :: ExportHandle -> IO () +flushDbQueue (ExportHandle h _) = H.flushDbQueue h + +recordExportTreeCurrent :: ExportHandle -> Sha -> IO () +recordExportTreeCurrent h s = queueDb h $ do + delete $ from $ \r -> do + where_ (r ^. ExportTreeCurrentTree ==. r ^. ExportTreeCurrentTree) + void $ insertUnique $ ExportTreeCurrent $ toSRef s + +getExportTreeCurrent :: ExportHandle -> IO (Maybe Sha) +getExportTreeCurrent (ExportHandle h _) = H.queryDbQueue h $ do + l <- select $ from $ \r -> do + where_ (r ^. ExportTreeCurrentTree ==. r ^. ExportTreeCurrentTree) + return (r ^. ExportTreeCurrentTree) + case l of + (s:[]) -> return $ Just $ fromSRef $ unValue s + _ -> return Nothing + +addExportedLocation :: ExportHandle -> Key -> ExportLocation -> IO () +addExportedLocation h k el = queueDb h $ do + void $ insertUnique $ Exported ik ef + let edirs = map + (\ed -> ExportedDirectory (toSFilePath (fromExportDirectory ed)) ef) + (exportDirectories el) +#if MIN_VERSION_persistent(2,1,0) + insertMany_ edirs +#else + void $ insertMany edirs +#endif + where + ik = toIKey k + ef = toSFilePath (fromExportLocation el) + +removeExportedLocation :: ExportHandle -> Key -> ExportLocation -> IO () +removeExportedLocation h k el = queueDb h $ do + delete $ from $ \r -> do + where_ (r ^. ExportedKey ==. val ik &&. r ^. ExportedFile ==. val ef) + let subdirs = map (toSFilePath . fromExportDirectory) + (exportDirectories el) + delete $ from $ \r -> do + where_ (r ^. ExportedDirectoryFile ==. val ef + &&. r ^. ExportedDirectorySubdir `in_` valList subdirs) + where + ik = toIKey k + ef = toSFilePath (fromExportLocation el) + +{- Note that this does not see recently queued changes. -} +getExportedLocation :: ExportHandle -> Key -> IO [ExportLocation] +getExportedLocation (ExportHandle h _) k = H.queryDbQueue h $ do + l <- select $ from $ \r -> do + where_ (r ^. ExportedKey ==. val ik) + return (r ^. ExportedFile) + return $ map (mkExportLocation . fromSFilePath . unValue) l + where + ik = toIKey k + +{- Note that this does not see recently queued changes. -} +isExportDirectoryEmpty :: ExportHandle -> ExportDirectory -> IO Bool +isExportDirectoryEmpty (ExportHandle h _) d = H.queryDbQueue h $ do + l <- select $ from $ \r -> do + where_ (r ^. ExportedDirectorySubdir ==. val ed) + return (r ^. ExportedDirectoryFile) + return $ null l + where + ed = toSFilePath $ fromExportDirectory d + +{- Get locations in the export that might contain a key. -} +getExportTree :: ExportHandle -> Key -> IO [ExportLocation] +getExportTree (ExportHandle h _) k = H.queryDbQueue h $ do + l <- select $ from $ \r -> do + where_ (r ^. ExportTreeKey ==. val ik) + return (r ^. ExportTreeFile) + return $ map (mkExportLocation . fromSFilePath . unValue) l + where + ik = toIKey k + +addExportTree :: ExportHandle -> Key -> ExportLocation -> IO () +addExportTree h k loc = queueDb h $ + void $ insertUnique $ ExportTree ik ef + where + ik = toIKey k + ef = toSFilePath (fromExportLocation loc) + +removeExportTree :: ExportHandle -> Key -> ExportLocation -> IO () +removeExportTree h k loc = queueDb h $ + delete $ from $ \r -> + where_ (r ^. ExportTreeKey ==. val ik &&. r ^. ExportTreeFile ==. val ef) + where + ik = toIKey k + ef = toSFilePath (fromExportLocation loc) + +{- Diff from the old to the new tree and update the ExportTree table. -} +updateExportTree :: ExportHandle -> Sha -> Sha -> Annex () +updateExportTree h old new = do + (diff, cleanup) <- inRepo $ + Git.DiffTree.diffTreeRecursive old new + forM_ diff $ \i -> do + srcek <- getek (Git.DiffTree.srcsha i) + dstek <- getek (Git.DiffTree.dstsha i) + updateExportTree' h srcek dstek i + void $ liftIO cleanup + where + getek sha + | sha == nullSha = return Nothing + | otherwise = Just <$> exportKey sha + +updateExportTree' :: ExportHandle -> Maybe ExportKey -> Maybe ExportKey -> Git.DiffTree.DiffTreeItem-> Annex () +updateExportTree' h srcek dstek i = do + case srcek of + Nothing -> return () + Just k -> liftIO $ removeExportTree h (asKey k) loc + case dstek of + Nothing -> return () + Just k -> liftIO $ addExportTree h (asKey k) loc + where + loc = mkExportLocation $ getTopFilePath $ Git.DiffTree.file i + +updateExportTreeFromLog :: ExportHandle -> Annex () +updateExportTreeFromLog db@(ExportHandle _ u) = + withExclusiveLock (gitAnnexExportLock u) $ do + old <- liftIO $ fromMaybe emptyTree + <$> getExportTreeCurrent db + l <- Log.getExport u + case map Log.exportedTreeish l of + (new:[]) | new /= old -> do + updateExportTree db old new + liftIO $ recordExportTreeCurrent db new + liftIO $ flushDbQueue db + _ -> return () diff --git a/Database/Fsck.hs b/Database/Fsck.hs new file mode 100644 index 0000000000..1ce513dcf9 --- /dev/null +++ b/Database/Fsck.hs @@ -0,0 +1,96 @@ +{- Sqlite database used for incremental fsck. + - + - Copyright 2015 Joey Hess + -: + - Licensed under the GNU GPL version 3 or higher. + -} + +{-# LANGUAGE QuasiQuotes, TypeFamilies, TemplateHaskell #-} +{-# LANGUAGE OverloadedStrings, GADTs, FlexibleContexts #-} +{-# LANGUAGE MultiParamTypeClasses, GeneralizedNewtypeDeriving #-} +{-# LANGUAGE RankNTypes #-} + +module Database.Fsck ( + FsckHandle, + newPass, + openDb, + closeDb, + addDb, + inDb, + FsckedId, +) where + +import Database.Types +import qualified Database.Queue as H +import Database.Init +import Annex.Locations +import Utility.Exception +import Annex.Common +import Annex.LockFile + +import Database.Persist.TH +import Database.Esqueleto hiding (Key) +import Data.Time.Clock + +data FsckHandle = FsckHandle H.DbQueue UUID + +{- Each key stored in the database has already been fscked as part + - of the latest incremental fsck pass. -} +share [mkPersist sqlSettings, mkMigrate "migrateFsck"] [persistLowerCase| +Fscked + key SKey + UniqueKey key +|] + +{- The database is removed when starting a new incremental fsck pass. + - + - This may fail, if other fsck processes are currently running using the + - database. Removing the database in that situation would lead to crashes + - or unknown behavior. + -} +newPass :: UUID -> Annex Bool +newPass u = isJust <$> tryExclusiveLock (gitAnnexFsckDbLock u) go + where + go = liftIO . void . tryIO . removeDirectoryRecursive + =<< fromRepo (gitAnnexFsckDbDir u) + +{- Opens the database, creating it if it doesn't exist yet. -} +openDb :: UUID -> Annex FsckHandle +openDb u = do + dbdir <- fromRepo (gitAnnexFsckDbDir u) + let db = dbdir "db" + unlessM (liftIO $ doesFileExist db) $ do + initDb db $ void $ + runMigrationSilent migrateFsck + lockFileCached =<< fromRepo (gitAnnexFsckDbLock u) + h <- liftIO $ H.openDbQueue H.MultiWriter db "fscked" + return $ FsckHandle h u + +closeDb :: FsckHandle -> Annex () +closeDb (FsckHandle h u) = do + liftIO $ H.closeDbQueue h + unlockFile =<< fromRepo (gitAnnexFsckDbLock u) + +addDb :: FsckHandle -> Key -> IO () +addDb (FsckHandle h _) k = H.queueDb h checkcommit $ + void $ insertUnique $ Fscked sk + where + sk = toSKey k + + -- commit queue after 1000 files or 5 minutes, whichever comes first + checkcommit sz lastcommittime + | sz > 1000 = return True + | otherwise = do + now <- getCurrentTime + return $ diffUTCTime lastcommittime now > 300 + +{- Doesn't know about keys that were just added with addDb. -} +inDb :: FsckHandle -> Key -> IO Bool +inDb (FsckHandle h _) = H.queryDbQueue h . inDb' . toSKey + +inDb' :: SKey -> SqlPersistM Bool +inDb' sk = do + r <- select $ from $ \r -> do + where_ (r ^. FsckedKey ==. val sk) + return (r ^. FsckedKey) + return $ not $ null r diff --git a/Database/Handle.hs b/Database/Handle.hs new file mode 100644 index 0000000000..5670acb994 --- /dev/null +++ b/Database/Handle.hs @@ -0,0 +1,209 @@ +{- Persistent sqlite database handles. + - + - Copyright 2015 Joey Hess + - + - Licensed under the GNU GPL version 3 or higher. + -} + +{-# LANGUAGE BangPatterns #-} + +module Database.Handle ( + DbHandle, + DbConcurrency(..), + openDb, + TableName, + queryDb, + closeDb, + commitDb, + commitDb', +) where + +import Utility.Exception +import Utility.FileSystemEncoding + +import Database.Persist.Sqlite +import qualified Database.Sqlite as Sqlite +import Control.Monad +import Control.Monad.IO.Class (liftIO) +import Control.Concurrent +import Control.Concurrent.Async +import Control.Exception (throwIO, BlockedIndefinitelyOnMVar(..)) +import qualified Data.Text as T +import Control.Monad.Trans.Resource (runResourceT) +import Control.Monad.Logger (runNoLoggingT) +import Data.List +import System.IO + +{- A DbHandle is a reference to a worker thread that communicates with + - the database. It has a MVar which Jobs are submitted to. -} +data DbHandle = DbHandle DbConcurrency (Async ()) (MVar Job) + +{- Name of a table that should exist once the database is initialized. -} +type TableName = String + +{- Sqlite only allows a single write to a database at a time; a concurrent + - write will crash. + - + - While a DbHandle serializes concurrent writes from + - multiple threads. But, when a database can be written to by + - multiple processes concurrently, use MultiWriter to make writes + - to the database be done robustly. + - + - The downside of using MultiWriter is that after writing a change to the + - database, the a query using the same DbHandle will not immediately see + - the change! This is because the change is actually written using a + - separate database connection, and caching can prevent seeing the change. + - Also, consider that if multiple processes are writing to a database, + - you can't rely on seeing values you've just written anyway, as another + - process may change them. + - + - When a database can only be written to by a single process, use + - SingleWriter. Changes written to the database will always be immediately + - visible then. + -} +data DbConcurrency = SingleWriter | MultiWriter + +{- Opens the database, but does not perform any migrations. Only use + - once the database is known to exist and have the right tables. -} +openDb :: DbConcurrency -> FilePath -> TableName -> IO DbHandle +openDb dbconcurrency db tablename = do + jobs <- newEmptyMVar + worker <- async (workerThread (T.pack db) tablename jobs) + + -- work around https://github.com/yesodweb/persistent/issues/474 + liftIO $ fileEncoding stderr + + return $ DbHandle dbconcurrency worker jobs + +{- This is optional; when the DbHandle gets garbage collected it will + - auto-close. -} +closeDb :: DbHandle -> IO () +closeDb (DbHandle _ worker jobs) = do + putMVar jobs CloseJob + wait worker + +{- Makes a query using the DbHandle. This should not be used to make + - changes to the database! + - + - Note that the action is not run by the calling thread, but by a + - worker thread. Exceptions are propigated to the calling thread. + - + - Only one action can be run at a time against a given DbHandle. + - If called concurrently in the same process, this will block until + - it is able to run. + - + - Note that when the DbHandle was opened in MultiWriter mode, recent + - writes may not be seen by queryDb. + -} +queryDb :: DbHandle -> SqlPersistM a -> IO a +queryDb (DbHandle _ _ jobs) a = do + res <- newEmptyMVar + putMVar jobs $ QueryJob $ + liftIO . putMVar res =<< tryNonAsync a + (either throwIO return =<< takeMVar res) + `catchNonAsync` (const $ error "sqlite query crashed") + +{- Writes a change to the database. + - + - In MultiWriter mode, catches failure to write to the database, + - and retries repeatedly for up to 10 seconds, which should avoid + - all but the most exceptional problems. + -} +commitDb :: DbHandle -> SqlPersistM () -> IO () +commitDb h wa = robustly Nothing 100 (commitDb' h wa) + where + robustly :: Maybe SomeException -> Int -> IO (Either SomeException ()) -> IO () + robustly e 0 _ = error $ "failed to commit changes to sqlite database: " ++ show e + robustly _ n a = do + r <- a + case r of + Right _ -> return () + Left e -> do + threadDelay 100000 -- 1/10th second + robustly (Just e) (n-1) a + +commitDb' :: DbHandle -> SqlPersistM () -> IO (Either SomeException ()) +commitDb' (DbHandle MultiWriter _ jobs) a = do + res <- newEmptyMVar + putMVar jobs $ RobustChangeJob $ \runner -> + liftIO $ putMVar res =<< tryNonAsync (runner a) + takeMVar res +commitDb' (DbHandle SingleWriter _ jobs) a = do + res <- newEmptyMVar + putMVar jobs $ ChangeJob $ + liftIO . putMVar res =<< tryNonAsync a + takeMVar res + `catchNonAsync` (const $ error "sqlite commit crashed") + +data Job + = QueryJob (SqlPersistM ()) + | ChangeJob (SqlPersistM ()) + | RobustChangeJob ((SqlPersistM () -> IO ()) -> IO ()) + | CloseJob + +workerThread :: T.Text -> TableName -> MVar Job -> IO () +workerThread db tablename jobs = go + where + go = do + v <- tryNonAsync (runSqliteRobustly tablename db loop) + case v of + Left e -> hPutStrLn stderr $ + "sqlite worker thread crashed: " ++ show e + Right True -> go + Right False -> return () + + getjob :: IO (Either BlockedIndefinitelyOnMVar Job) + getjob = try $ takeMVar jobs + + loop = do + job <- liftIO getjob + case job of + -- Exception is thrown when the MVar is garbage + -- collected, which means the whole DbHandle + -- is not used any longer. Shutdown cleanly. + Left BlockedIndefinitelyOnMVar -> return False + Right CloseJob -> return False + Right (QueryJob a) -> a >> loop + Right (ChangeJob a) -> do + a + -- Exit this sqlite transaction so the + -- database gets updated on disk. + return True + -- Change is run in a separate database connection + -- since sqlite only supports a single writer at a + -- time, and it may crash the database connection + -- that the write is made to. + Right (RobustChangeJob a) -> do + liftIO (a (runSqliteRobustly tablename db)) + loop + +-- like runSqlite, but calls settle on the raw sql Connection. +runSqliteRobustly :: TableName -> T.Text -> (SqlPersistM a) -> IO a +runSqliteRobustly tablename db a = do + conn <- Sqlite.open db + settle conn + runResourceT $ runNoLoggingT $ + withSqlConn (wrapConnection conn) $ + runSqlConn a + where + -- Work around a bug in sqlite: New database connections can + -- sometimes take a while to become usable; select statements will + -- fail with ErrorBusy for some time. So, loop until a select + -- succeeds; once one succeeds the connection will stay usable. + -- + settle conn = do + r <- tryNonAsync $ do + stmt <- Sqlite.prepare conn nullselect + void $ Sqlite.step stmt + void $ Sqlite.finalize stmt + case r of + Right _ -> return () + Left e -> do + if "ErrorBusy" `isInfixOf` show e + then do + threadDelay 1000 -- 1/1000th second + settle conn + else throwIO e + + -- This should succeed for any table. + nullselect = T.pack $ "SELECT null from " ++ tablename ++ " limit 1" diff --git a/Database/Init.hs b/Database/Init.hs new file mode 100644 index 0000000000..d7a7f68426 --- /dev/null +++ b/Database/Init.hs @@ -0,0 +1,55 @@ +{- Persistent sqlite database initialization + - + - Copyright 2015-2017 Joey Hess + - + - Licensed under the GNU GPL version 3 or higher. + -} + +module Database.Init where + +import Annex.Common +import Annex.Perms +import Utility.FileMode + +import Database.Persist.Sqlite +import qualified Database.Sqlite as Sqlite +import Control.Monad.IO.Class (liftIO) +import qualified Data.Text as T + +{- Ensures that the database is freshly initialized. Deletes any + - existing database. Pass the migration action for the database. + - + - The database is initialized using WAL mode, to prevent readers + - from blocking writers, and prevent a writer from blocking readers. + - + - The permissions of the database are set based on the + - core.sharedRepository setting. Setting these permissions on the main db + - file causes Sqlite to always use the same permissions for additional + - files it writes later on + -} +initDb :: FilePath -> SqlPersistM () -> Annex () +initDb db migration = do + let dbdir = takeDirectory db + let tmpdbdir = dbdir ++ ".tmp" + let tmpdb = tmpdbdir "db" + liftIO $ do + createDirectoryIfMissing True tmpdbdir + let tdb = T.pack tmpdb + enableWAL tdb + runSqlite tdb migration + setAnnexDirPerm tmpdbdir + -- Work around sqlite bug that prevents it from honoring + -- less restrictive umasks. + liftIO $ setFileMode tmpdb =<< defaultFileMode + setAnnexFilePerm tmpdb + liftIO $ do + void $ tryIO $ removeDirectoryRecursive dbdir + rename tmpdbdir dbdir + +enableWAL :: T.Text -> IO () +enableWAL db = do + conn <- Sqlite.open db + stmt <- Sqlite.prepare conn (T.pack "PRAGMA journal_mode=WAL;") + void $ Sqlite.step stmt + void $ Sqlite.finalize stmt + Sqlite.close conn diff --git a/Database/Keys.hs b/Database/Keys.hs new file mode 100644 index 0000000000..0456ce7666 --- /dev/null +++ b/Database/Keys.hs @@ -0,0 +1,303 @@ +{- Sqlite database of information about Keys + - + - Copyright 2015-2018 Joey Hess + - + - Licensed under the GNU GPL version 3 or higher. + -} + +{-# LANGUAGE ScopedTypeVariables #-} + +module Database.Keys ( + DbHandle, + closeDb, + addAssociatedFile, + getAssociatedFiles, + getAssociatedKey, + removeAssociatedFile, + storeInodeCaches, + storeInodeCaches', + addInodeCaches, + getInodeCaches, + removeInodeCaches, + runWriter, +) where + +import qualified Database.Keys.SQL as SQL +import Database.Types +import Database.Keys.Handle +import qualified Database.Queue as H +import Database.Init +import Annex.Locations +import Annex.Common hiding (delete) +import Annex.Version (versionUsesKeysDatabase) +import qualified Annex +import Annex.LockFile +import Annex.CatFile +import Annex.Content.PointerFile +import Annex.Link +import Utility.InodeCache +import Annex.InodeSentinal +import Git +import Git.FilePath +import Git.Command +import Git.Types +import Git.Index + +{- Runs an action that reads from the database. + - + - If the database doesn't already exist, it's not created; mempty is + - returned instead. This way, when the keys database is not in use, + - there's minimal overhead in checking it. + - + - If the database is already open, any writes are flushed to it, to ensure + - consistency. + - + - Any queued writes will be flushed before the read. + -} +runReader :: Monoid v => (SQL.ReadHandle -> Annex v) -> Annex v +runReader a = do + h <- getDbHandle + withDbState h go + where + go DbUnavailable = return (mempty, DbUnavailable) + go st@(DbOpen qh) = do + liftIO $ H.flushDbQueue qh + v <- a (SQL.ReadHandle qh) + return (v, st) + go DbClosed = do + st' <- openDb False DbClosed + v <- case st' of + (DbOpen qh) -> a (SQL.ReadHandle qh) + _ -> return mempty + return (v, st') + +runReaderIO :: Monoid v => (SQL.ReadHandle -> IO v) -> Annex v +runReaderIO a = runReader (liftIO . a) + +{- Runs an action that writes to the database. Typically this is used to + - queue changes, which will be flushed at a later point. + - + - The database is created if it doesn't exist yet. -} +runWriter :: (SQL.WriteHandle -> Annex ()) -> Annex () +runWriter a = do + h <- getDbHandle + withDbState h go + where + go st@(DbOpen qh) = do + v <- a (SQL.WriteHandle qh) + return (v, st) + go st = do + st' <- openDb True st + v <- case st' of + DbOpen qh -> a (SQL.WriteHandle qh) + _ -> error "internal" + return (v, st') + +runWriterIO :: (SQL.WriteHandle -> IO ()) -> Annex () +runWriterIO a = runWriter (liftIO . a) + +{- Gets the handle cached in Annex state; creates a new one if it's not yet + - available, but doesn't open the database. -} +getDbHandle :: Annex DbHandle +getDbHandle = go =<< Annex.getState Annex.keysdbhandle + where + go (Just h) = pure h + go Nothing = do + h <- ifM versionUsesKeysDatabase + ( liftIO newDbHandle + , liftIO unavailableDbHandle + ) + Annex.changeState $ \s -> s { Annex.keysdbhandle = Just h } + return h + +{- Opens the database, perhaps creating it if it doesn't exist yet. + - + - Multiple readers and writers can have the database open at the same + - time. Database.Handle deals with the concurrency issues. + - The lock is held while opening the database, so that when + - the database doesn't exist yet, one caller wins the lock and + - can create it undisturbed. + -} +openDb :: Bool -> DbState -> Annex DbState +openDb _ st@(DbOpen _) = return st +openDb False DbUnavailable = return DbUnavailable +openDb createdb _ = catchPermissionDenied permerr $ withExclusiveLock gitAnnexKeysDbLock $ do + dbdir <- fromRepo gitAnnexKeysDb + let db = dbdir "db" + dbexists <- liftIO $ doesFileExist db + case (dbexists, createdb) of + (True, _) -> open db + (False, True) -> do + initDb db SQL.createTables + open db + (False, False) -> return DbUnavailable + where + -- If permissions don't allow opening the database, treat it as if + -- it does not exist. + permerr e = case createdb of + False -> return DbUnavailable + True -> throwM e + + open db = do + qh <- liftIO $ H.openDbQueue H.MultiWriter db SQL.containedTable + reconcileStaged qh + return $ DbOpen qh + +{- Closes the database if it was open. Any writes will be flushed to it. + - + - This does not normally need to be called; the database will auto-close + - when the handle is garbage collected. However, this can be used to + - force a re-read of the database, in case another process has written + - data to it. + -} +closeDb :: Annex () +closeDb = liftIO . closeDbHandle =<< getDbHandle + +addAssociatedFile :: Key -> TopFilePath -> Annex () +addAssociatedFile k f = runWriterIO $ SQL.addAssociatedFile (toIKey k) f + +{- Note that the files returned were once associated with the key, but + - some of them may not be any longer. -} +getAssociatedFiles :: Key -> Annex [TopFilePath] +getAssociatedFiles = runReaderIO . SQL.getAssociatedFiles . toIKey + +{- Gets any keys that are on record as having a particular associated file. + - (Should be one or none but the database doesn't enforce that.) -} +getAssociatedKey :: TopFilePath -> Annex [Key] +getAssociatedKey = map fromIKey <$$> runReaderIO . SQL.getAssociatedKey + +removeAssociatedFile :: Key -> TopFilePath -> Annex () +removeAssociatedFile k = runWriterIO . SQL.removeAssociatedFile (toIKey k) + +{- Stats the files, and stores their InodeCaches. -} +storeInodeCaches :: Key -> [FilePath] -> Annex () +storeInodeCaches k fs = storeInodeCaches' k fs [] + +storeInodeCaches' :: Key -> [FilePath] -> [InodeCache] -> Annex () +storeInodeCaches' k fs ics = withTSDelta $ \d -> + addInodeCaches k . (++ ics) . catMaybes + =<< liftIO (mapM (`genInodeCache` d) fs) + +addInodeCaches :: Key -> [InodeCache] -> Annex () +addInodeCaches k is = runWriterIO $ SQL.addInodeCaches (toIKey k) is + +{- A key may have multiple InodeCaches; one for the annex object, and one + - for each pointer file that is a copy of it. -} +getInodeCaches :: Key -> Annex [InodeCache] +getInodeCaches = runReaderIO . SQL.getInodeCaches . toIKey + +removeInodeCaches :: Key -> Annex () +removeInodeCaches = runWriterIO . SQL.removeInodeCaches . toIKey + +{- Looks at staged changes to find when unlocked files are copied/moved, + - and updates associated files in the keys database. + - + - Since staged changes can be dropped later, does not remove any + - associated files; only adds new associated files. + - + - This needs to be run before querying the keys database so that + - information is consistent with the state of the repository. + - + - To avoid unncessary work, the index file is statted, and if it's not + - changed since last time this was run, nothing is done. + - + - Note that this is run with a lock held, so only one process can be + - running this at a time. + - + - This also cleans up after a race between eg a git mv and git-annex + - get/drop/similar. If git moves the file between this being run and the + - get/drop, the moved file won't be updated for the get/drop. + - The next time this runs, it will see the staged change. It then checks + - if the worktree file's content availability does not match the git-annex + - content availablity, and makes changes as necessary to reconcile them. + - + - Note that if a commit happens before this runs again, it won't see + - the staged change. Instead, during the commit, git will run the clean + - filter. If a drop missed the file then the file is added back into the + - annex. If a get missed the file then the clean filter populates the + - file. + -} +reconcileStaged :: H.DbQueue -> Annex () +reconcileStaged qh = whenM versionUsesKeysDatabase $ do + gitindex <- inRepo currentIndexFile + indexcache <- fromRepo gitAnnexKeysDbIndexCache + withTSDelta (liftIO . genInodeCache gitindex) >>= \case + Just cur -> + liftIO (maybe Nothing readInodeCache <$> catchMaybeIO (readFile indexcache)) >>= \case + Nothing -> go cur indexcache + Just prev -> ifM (compareInodeCaches prev cur) + ( noop + , go cur indexcache + ) + Nothing -> noop + where + go cur indexcache = do + (l, cleanup) <- inRepo $ pipeNullSplit diff + changed <- procdiff l False + void $ liftIO cleanup + -- Flush database changes immediately + -- so other processes can see them. + when changed $ + liftIO $ H.flushDbQueue qh + liftIO $ writeFile indexcache $ showInodeCache cur + + diff = + -- Avoid using external diff command, which would be slow. + -- (The -G option may make it be used otherwise.) + [ Param "-c", Param "diff.external=" + -- Avoid running smudge or clean filters, since we want the + -- raw output, and they would block trying to access the + -- locked database. The --raw normally avoids git diff + -- running them, but older versions of git need this. + , Param "-c", Param "filter.annex.smudge=" + , Param "-c", Param "filter.annex.clean=" + , Param "diff" + , Param "--cached" + , Param "--raw" + , Param "-z" + , Param "--abbrev=40" + -- Optimization: Only find pointer files. This is not + -- perfect. A file could start with this and not be a + -- pointer file. And a pointer file that is replaced with + -- a non-pointer file will match this. + , Param $ "-G^" ++ toInternalGitPath (pathSeparator:objectDir) + -- Don't include files that were deleted, because this only + -- wants to update information for files that are present + -- in the index. + , Param "--diff-filter=AMUT" + -- Disable rename detection. + , Param "--no-renames" + -- Avoid other complications. + , Param "--ignore-submodules=all" + , Param "--no-ext-diff" + ] + + procdiff (info:file:rest) changed = case words info of + ((':':_srcmode):dstmode:_srcsha:dstsha:_change:[]) + -- Only want files, not symlinks + | dstmode /= fmtTreeItemType TreeSymlink -> do + maybe noop (reconcile (asTopFilePath file)) + =<< catKey (Ref dstsha) + procdiff rest True + | otherwise -> procdiff rest changed + _ -> return changed -- parse failed + procdiff _ changed = return changed + + -- Note that database writes done in here will not necessarily + -- be visible to database reads also done in here. + reconcile file key = do + let ikey = toIKey key + liftIO $ SQL.addAssociatedFileFast ikey file (SQL.WriteHandle qh) + caches <- liftIO $ SQL.getInodeCaches ikey (SQL.ReadHandle qh) + keyloc <- calcRepo (gitAnnexLocation key) + keypopulated <- sameInodeCache keyloc caches + p <- fromRepo $ fromTopFilePath file + filepopulated <- sameInodeCache p caches + case (keypopulated, filepopulated) of + (True, False) -> + populatePointerFile (Restage True) key keyloc p >>= \case + Nothing -> return () + Just ic -> liftIO $ + SQL.addInodeCaches ikey [ic] (SQL.WriteHandle qh) + (False, True) -> depopulatePointerFile key p + _ -> return () diff --git a/Database/Keys/Handle.hs b/Database/Keys/Handle.hs new file mode 100644 index 0000000000..eec9e0fda3 --- /dev/null +++ b/Database/Keys/Handle.hs @@ -0,0 +1,69 @@ +{- Handle for the Keys database. + - + - Copyright 2015 Joey Hess + -: + - Licensed under the GNU GPL version 3 or higher. + -} + +module Database.Keys.Handle ( + DbHandle, + newDbHandle, + unavailableDbHandle, + DbState(..), + withDbState, + flushDbQueue, + closeDbHandle, +) where + +import qualified Database.Queue as H +import Utility.Exception + +import Control.Concurrent +import Control.Monad.IO.Class (liftIO, MonadIO) +import Control.Applicative +import Prelude + +-- The MVar is always left full except when actions are run +-- that access the database. +newtype DbHandle = DbHandle (MVar DbState) + +-- The database can be closed or open, but it also may have been +-- tried to open (for read) and didn't exist yet or is not readable. +data DbState = DbClosed | DbOpen H.DbQueue | DbUnavailable + +newDbHandle :: IO DbHandle +newDbHandle = DbHandle <$> newMVar DbClosed + +unavailableDbHandle :: IO DbHandle +unavailableDbHandle = DbHandle <$> newMVar DbUnavailable + +-- Runs an action on the state of the handle, which can change its state. +-- The MVar is empty while the action runs, which blocks other users +-- of the handle from running. +withDbState + :: (MonadIO m, MonadCatch m) + => DbHandle + -> (DbState -> m (v, DbState)) + -> m v +withDbState (DbHandle mvar) a = do + st <- liftIO $ takeMVar mvar + go st `onException` (liftIO $ putMVar mvar st) + where + go st = do + (v, st') <- a st + liftIO $ putMVar mvar st' + return v + +flushDbQueue :: DbHandle -> IO () +flushDbQueue (DbHandle mvar) = go =<< readMVar mvar + where + go (DbOpen qh) = H.flushDbQueue qh + go _ = return () + +closeDbHandle :: DbHandle -> IO () +closeDbHandle h = withDbState h go + where + go (DbOpen qh) = do + H.closeDbQueue qh + return ((), DbClosed) + go st = return ((), st) diff --git a/Database/Keys/SQL.hs b/Database/Keys/SQL.hs new file mode 100644 index 0000000000..77c1e4429c --- /dev/null +++ b/Database/Keys/SQL.hs @@ -0,0 +1,126 @@ +{- Sqlite database of information about Keys + - + - Copyright 2015-2016 Joey Hess + - + - Licensed under the GNU GPL version 3 or higher. + -} + +{-# LANGUAGE QuasiQuotes, TypeFamilies, TemplateHaskell #-} +{-# LANGUAGE OverloadedStrings, GADTs, FlexibleContexts #-} +{-# LANGUAGE MultiParamTypeClasses, GeneralizedNewtypeDeriving #-} +{-# LANGUAGE RankNTypes, ScopedTypeVariables #-} + +module Database.Keys.SQL where + +import Database.Types +import Database.Handle +import qualified Database.Queue as H +import Utility.InodeCache +import Git.FilePath + +import Database.Persist.TH +import Database.Esqueleto hiding (Key) +import Data.Time.Clock +import Control.Monad + +share [mkPersist sqlSettings, mkMigrate "migrateKeysDb"] [persistLowerCase| +Associated + key IKey + file SFilePath + KeyFileIndex key file + FileKeyIndex file key +Content + key IKey + cache SInodeCache + KeyCacheIndex key cache +|] + +containedTable :: TableName +containedTable = "content" + +createTables :: SqlPersistM () +createTables = void $ runMigrationSilent migrateKeysDb + +newtype ReadHandle = ReadHandle H.DbQueue + +readDb :: SqlPersistM a -> ReadHandle -> IO a +readDb a (ReadHandle h) = H.queryDbQueue h a + +newtype WriteHandle = WriteHandle H.DbQueue + +queueDb :: SqlPersistM () -> WriteHandle -> IO () +queueDb a (WriteHandle h) = H.queueDb h checkcommit a + where + -- commit queue after 1000 changes or 5 minutes, whichever comes first + checkcommit sz lastcommittime + | sz > 1000 = return True + | otherwise = do + now <- getCurrentTime + return $ diffUTCTime lastcommittime now > 300 + +addAssociatedFile :: IKey -> TopFilePath -> WriteHandle -> IO () +addAssociatedFile ik f = queueDb $ do + -- If the same file was associated with a different key before, + -- remove that. + delete $ from $ \r -> do + where_ (r ^. AssociatedFile ==. val af &&. not_ (r ^. AssociatedKey ==. val ik)) + void $ insertUnique $ Associated ik af + where + af = toSFilePath (getTopFilePath f) + +-- Does not remove any old association for a file, but less expensive +-- than addAssociatedFile. Calling dropAllAssociatedFiles first and then +-- this is an efficient way to update all associated files. +addAssociatedFileFast :: IKey -> TopFilePath -> WriteHandle -> IO () +addAssociatedFileFast ik f = queueDb $ void $ insertUnique $ Associated ik af + where + af = toSFilePath (getTopFilePath f) + +dropAllAssociatedFiles :: WriteHandle -> IO () +dropAllAssociatedFiles = queueDb $ + delete $ from $ \(_r :: SqlExpr (Entity Associated)) -> return () + +{- Note that the files returned were once associated with the key, but + - some of them may not be any longer. -} +getAssociatedFiles :: IKey -> ReadHandle -> IO [TopFilePath] +getAssociatedFiles ik = readDb $ do + l <- select $ from $ \r -> do + where_ (r ^. AssociatedKey ==. val ik) + return (r ^. AssociatedFile) + return $ map (asTopFilePath . fromSFilePath . unValue) l + +{- Gets any keys that are on record as having a particular associated file. + - (Should be one or none but the database doesn't enforce that.) -} +getAssociatedKey :: TopFilePath -> ReadHandle -> IO [IKey] +getAssociatedKey f = readDb $ do + l <- select $ from $ \r -> do + where_ (r ^. AssociatedFile ==. val af) + return (r ^. AssociatedKey) + return $ map unValue l + where + af = toSFilePath (getTopFilePath f) + +removeAssociatedFile :: IKey -> TopFilePath -> WriteHandle -> IO () +removeAssociatedFile ik f = queueDb $ + delete $ from $ \r -> do + where_ (r ^. AssociatedKey ==. val ik &&. r ^. AssociatedFile ==. val af) + where + af = toSFilePath (getTopFilePath f) + +addInodeCaches :: IKey -> [InodeCache] -> WriteHandle -> IO () +addInodeCaches ik is = queueDb $ + forM_ is $ \i -> insertUnique $ Content ik (toSInodeCache i) + +{- A key may have multiple InodeCaches; one for the annex object, and one + - for each pointer file that is a copy of it. -} +getInodeCaches :: IKey -> ReadHandle -> IO [InodeCache] +getInodeCaches ik = readDb $ do + l <- select $ from $ \r -> do + where_ (r ^. ContentKey ==. val ik) + return (r ^. ContentCache) + return $ map (fromSInodeCache . unValue) l + +removeInodeCaches :: IKey -> WriteHandle -> IO () +removeInodeCaches ik = queueDb $ + delete $ from $ \r -> do + where_ (r ^. ContentKey ==. val ik) diff --git a/Database/Queue.hs b/Database/Queue.hs new file mode 100644 index 0000000000..f0a2d2b654 --- /dev/null +++ b/Database/Queue.hs @@ -0,0 +1,112 @@ +{- Persistent sqlite database queues + - + - Copyright 2015 Joey Hess + - + - Licensed under the GNU GPL version 3 or higher. + -} + +{-# LANGUAGE BangPatterns #-} + +module Database.Queue ( + DbQueue, + DbConcurrency(..), + openDbQueue, + queryDbQueue, + closeDbQueue, + flushDbQueue, + QueueSize, + queueDb, +) where + +import Utility.Monad +import Database.Handle + +import Database.Persist.Sqlite +import Control.Concurrent +import Data.Time.Clock +import Control.Applicative +import Prelude + +{- A DbQueue wraps a DbHandle, adding a queue of writes to perform. + - + - This is efficient when there are frequent writes, but + - reads will not immediately have access to queued writes. -} +data DbQueue = DQ DbHandle (MVar Queue) + +{- Opens the database queue, but does not perform any migrations. Only use + - if the database is known to exist and have the right tables; ie after + - running initDb. -} +openDbQueue :: DbConcurrency -> FilePath -> TableName -> IO DbQueue +openDbQueue dbconcurrency db tablename = DQ + <$> openDb dbconcurrency db tablename + <*> (newMVar =<< emptyQueue) + +{- This or flushDbQueue must be called, eg at program exit to ensure + - queued changes get written to the database. -} +closeDbQueue :: DbQueue -> IO () +closeDbQueue h@(DQ hdl _) = do + flushDbQueue h + closeDb hdl + +{- Blocks until all queued changes have been written to the database. -} +flushDbQueue :: DbQueue -> IO () +flushDbQueue (DQ hdl qvar) = do + q@(Queue sz _ qa) <- takeMVar qvar + if sz > 0 + then do + commitDb hdl qa + putMVar qvar =<< emptyQueue + else putMVar qvar q + +{- Makes a query using the DbQueue's database connection. + - This should not be used to make changes to the database! + - + - Queries will not see changes that have been recently queued, + - so use with care. + - + - Also, when the database was opened in MultiWriter mode, + - queries may not see changes even after flushDbQueue. + -} +queryDbQueue :: DbQueue -> SqlPersistM a -> IO a +queryDbQueue (DQ hdl _) = queryDb hdl + +{- A queue of actions to perform, with a count of the number of actions + - queued, and a last commit time. -} +data Queue = Queue QueueSize LastCommitTime (SqlPersistM ()) + +type QueueSize = Int + +type LastCommitTime = UTCTime + +emptyQueue :: IO Queue +emptyQueue = do + now <- getCurrentTime + return $ Queue 0 now (return ()) + +{- Queues a change to be made to the database. It will be queued + - to be committed later, unless the commitchecker action returns true, + - in which case any previously queued changes are also committed. + - + - Transactions built up by queueDb are sent to sqlite all at once. + - If sqlite fails due to another change being made concurrently by another + - process, the transaction is put back in the queue. This avoids + - the sqlite multiple writer problem. + -} +queueDb + :: DbQueue + -> (QueueSize -> LastCommitTime -> IO Bool) + -> SqlPersistM () + -> IO () +queueDb (DQ hdl qvar) commitchecker a = do + Queue sz lastcommittime qa <- takeMVar qvar + let !sz' = sz + 1 + let qa' = qa >> a + let enqueue = putMVar qvar + ifM (commitchecker sz' lastcommittime) + ( do + r <- commitDb' hdl qa' + case r of + Left _ -> enqueue $ Queue sz' lastcommittime qa' + Right _ -> enqueue =<< emptyQueue + , enqueue $ Queue sz' lastcommittime qa' + ) diff --git a/Database/Types.hs b/Database/Types.hs new file mode 100644 index 0000000000..49a63f067e --- /dev/null +++ b/Database/Types.hs @@ -0,0 +1,114 @@ +{- types for SQL databases + - + - Copyright 2015-2017 Joey Hess + - + - Licensed under the GNU GPL version 3 or higher. + -} + +{-# LANGUAGE TemplateHaskell #-} + +module Database.Types where + +import Database.Persist.TH +import Data.Maybe +import Data.Char + +import Utility.PartialPrelude +import Key +import Utility.InodeCache +import Git.Types (Ref(..)) + +-- A serialized Key +newtype SKey = SKey String + deriving (Show, Read) + +toSKey :: Key -> SKey +toSKey = SKey . key2file + +fromSKey :: SKey -> Key +fromSKey (SKey s) = fromMaybe (error $ "bad serialized Key " ++ s) (file2key s) + +derivePersistField "SKey" + +-- A Key index. More efficient than SKey, but its Read instance does not +-- work when it's used in any kind of complex data structure. +newtype IKey = IKey String + +instance Read IKey where + readsPrec _ s = [(IKey s, "")] + +instance Show IKey where + show (IKey s) = s + +toIKey :: Key -> IKey +toIKey = IKey . key2file + +fromIKey :: IKey -> Key +fromIKey (IKey s) = fromMaybe (error $ "bad serialized Key " ++ s) (file2key s) + +derivePersistField "IKey" + +-- A serialized InodeCache +newtype SInodeCache = I String + deriving (Show, Read) + +toSInodeCache :: InodeCache -> SInodeCache +toSInodeCache = I . showInodeCache + +fromSInodeCache :: SInodeCache -> InodeCache +fromSInodeCache (I s) = fromMaybe (error $ "bad serialized InodeCache " ++ s) (readInodeCache s) + +derivePersistField "SInodeCache" + +-- A serialized FilePath. +-- +-- Not all unicode characters round-trip through sqlite. In particular, +-- surrigate code points do not. So, escape the FilePath. But, only when +-- it contains such characters. +newtype SFilePath = SFilePath String + +-- Note that Read instance does not work when used in any kind of complex +-- data structure. +instance Read SFilePath where + readsPrec _ s = [(SFilePath s, "")] + +instance Show SFilePath where + show (SFilePath s) = s + +toSFilePath :: FilePath -> SFilePath +toSFilePath s@('"':_) = SFilePath (show s) +toSFilePath s + | any needsescape s = SFilePath (show s) + | otherwise = SFilePath s + where + needsescape c = case generalCategory c of + Surrogate -> True + PrivateUse -> True + NotAssigned -> True + _ -> False + +fromSFilePath :: SFilePath -> FilePath +fromSFilePath (SFilePath s@('"':_)) = + fromMaybe (error "bad serialized SFilePath " ++ s) (readish s) +fromSFilePath (SFilePath s) = s + +derivePersistField "SFilePath" + +-- A serialized Ref +newtype SRef = SRef Ref + +-- Note that Read instance does not work when used in any kind of complex +-- data structure. +instance Read SRef where + readsPrec _ s = [(SRef (Ref s), "")] + +instance Show SRef where + show (SRef (Ref s)) = s + +derivePersistField "SRef" + +toSRef :: Ref -> SRef +toSRef = SRef + +fromSRef :: SRef -> Ref +fromSRef (SRef r) = r diff --git a/Git.hs b/Git.hs new file mode 100644 index 0000000000..b35051523f --- /dev/null +++ b/Git.hs @@ -0,0 +1,178 @@ +{- git repository handling + - + - This is written to be completely independant of git-annex and should be + - suitable for other uses. + - + - Copyright 2010-2012 Joey Hess + - + - Licensed under the GNU GPL version 3 or higher. + -} + +{-# LANGUAGE CPP #-} + +module Git ( + Repo(..), + Ref(..), + fromRef, + Branch, + Sha, + Tag, + repoIsUrl, + repoIsSsh, + repoIsHttp, + repoIsLocal, + repoIsLocalBare, + repoIsLocalUnknown, + repoDescribe, + repoLocation, + repoPath, + repoWorkTree, + localGitDir, + attributes, + attributesLocal, + hookPath, + assertLocal, + adjustPath, + relPath, +) where + +import Network.URI (uriPath, uriScheme, unEscapeString) +#ifndef mingw32_HOST_OS +import System.Posix.Files +#endif + +import Common +import Git.Types +#ifndef mingw32_HOST_OS +import Utility.FileMode +#endif + +{- User-visible description of a git repo. -} +repoDescribe :: Repo -> String +repoDescribe Repo { remoteName = Just name } = name +repoDescribe Repo { location = Url url } = show url +repoDescribe Repo { location = Local { worktree = Just dir } } = dir +repoDescribe Repo { location = Local { gitdir = dir } } = dir +repoDescribe Repo { location = LocalUnknown dir } = dir +repoDescribe Repo { location = Unknown } = "UNKNOWN" + +{- Location of the repo, either as a path or url. -} +repoLocation :: Repo -> String +repoLocation Repo { location = Url url } = show url +repoLocation Repo { location = Local { worktree = Just dir } } = dir +repoLocation Repo { location = Local { gitdir = dir } } = dir +repoLocation Repo { location = LocalUnknown dir } = dir +repoLocation Repo { location = Unknown } = error "unknown repoLocation" + +{- Path to a repository. For non-bare, this is the worktree, for bare, + - it's the gitdir, and for URL repositories, is the path on the remote + - host. -} +repoPath :: Repo -> FilePath +repoPath Repo { location = Url u } = unEscapeString $ uriPath u +repoPath Repo { location = Local { worktree = Just d } } = d +repoPath Repo { location = Local { gitdir = d } } = d +repoPath Repo { location = LocalUnknown dir } = dir +repoPath Repo { location = Unknown } = error "unknown repoPath" + +repoWorkTree :: Repo -> Maybe FilePath +repoWorkTree Repo { location = Local { worktree = Just d } } = Just d +repoWorkTree _ = Nothing + +{- Path to a local repository's .git directory. -} +localGitDir :: Repo -> FilePath +localGitDir Repo { location = Local { gitdir = d } } = d +localGitDir _ = error "unknown localGitDir" + +{- Some code needs to vary between URL and normal repos, + - or bare and non-bare, these functions help with that. -} +repoIsUrl :: Repo -> Bool +repoIsUrl Repo { location = Url _ } = True +repoIsUrl _ = False + +repoIsSsh :: Repo -> Bool +repoIsSsh Repo { location = Url url } + | scheme == "ssh:" = True + -- git treats these the same as ssh + | scheme == "git+ssh:" = True + | scheme == "ssh+git:" = True + | otherwise = False + where + scheme = uriScheme url +repoIsSsh _ = False + +repoIsHttp :: Repo -> Bool +repoIsHttp Repo { location = Url url } + | uriScheme url == "http:" = True + | uriScheme url == "https:" = True + | otherwise = False +repoIsHttp _ = False + +repoIsLocal :: Repo -> Bool +repoIsLocal Repo { location = Local { } } = True +repoIsLocal _ = False + +repoIsLocalBare :: Repo -> Bool +repoIsLocalBare Repo { location = Local { worktree = Nothing } } = True +repoIsLocalBare _ = False + +repoIsLocalUnknown :: Repo -> Bool +repoIsLocalUnknown Repo { location = LocalUnknown { } } = True +repoIsLocalUnknown _ = False + +assertLocal :: Repo -> a -> a +assertLocal repo action + | repoIsUrl repo = error $ unwords + [ "acting on non-local git repo" + , repoDescribe repo + , "not supported" + ] + | otherwise = action + +{- Path to a repository's gitattributes file. -} +attributes :: Repo -> FilePath +attributes repo + | repoIsLocalBare repo = attributesLocal repo + | otherwise = repoPath repo ".gitattributes" + +attributesLocal :: Repo -> FilePath +attributesLocal repo = localGitDir repo "info" "attributes" + +{- Path to a given hook script in a repository, only if the hook exists + - and is executable. -} +hookPath :: String -> Repo -> IO (Maybe FilePath) +hookPath script repo = do + let hook = localGitDir repo "hooks" script + ifM (catchBoolIO $ isexecutable hook) + ( return $ Just hook , return Nothing ) + where +#if mingw32_HOST_OS + isexecutable f = doesFileExist f +#else + isexecutable f = isExecutable . fileMode <$> getFileStatus f +#endif + +{- Makes the path to a local Repo be relative to the cwd. -} +relPath :: Repo -> IO Repo +relPath = adjustPath torel + where + torel p = do + p' <- relPathCwdToFile p + if null p' + then return "." + else return p' + +{- Adusts the path to a local Repo using the provided function. -} +adjustPath :: (FilePath -> IO FilePath) -> Repo -> IO Repo +adjustPath f r@(Repo { location = l@(Local { gitdir = d, worktree = w }) }) = do + d' <- f d + w' <- maybe (pure Nothing) (Just <$$> f) w + return $ r + { location = l + { gitdir = d' + , worktree = w' + } + } +adjustPath f r@(Repo { location = LocalUnknown d }) = do + d' <- f d + return $ r { location = LocalUnknown d' } +adjustPath _ r = pure r diff --git a/Git/AutoCorrect.hs b/Git/AutoCorrect.hs new file mode 100644 index 0000000000..ae7cc91a87 --- /dev/null +++ b/Git/AutoCorrect.hs @@ -0,0 +1,71 @@ +{- git autocorrection using Damerau-Levenshtein edit distance + - + - Copyright 2012 Joey Hess + - + - Licensed under the GNU GPL version 3 or higher. + -} + +module Git.AutoCorrect where + +import Common +import Git.Types +import qualified Git.Config + +import Text.EditDistance +import Control.Concurrent + +{- These are the same cost values as used in git. -} +gitEditCosts :: EditCosts +gitEditCosts = EditCosts + { deletionCosts = ConstantCost 4 + , insertionCosts = ConstantCost 1 + , substitutionCosts = ConstantCost 2 + , transpositionCosts = ConstantCost 0 + } + +{- Git's source calls this "an empirically derived magic number" -} +similarityFloor :: Int +similarityFloor = 7 + +{- Finds inexact matches for the input among the choices. + - Returns an ordered list of good enough matches, or an empty list if + - nothing matches well. -} +fuzzymatches :: String -> (c -> String) -> [c] -> [c] +fuzzymatches input showchoice choices = fst $ unzip $ + sortBy comparecost $ filter similarEnough $ zip choices costs + where + distance = restrictedDamerauLevenshteinDistance gitEditCosts input + costs = map (distance . showchoice) choices + comparecost a b = compare (snd a) (snd b) + similarEnough (_, cst) = cst < similarityFloor + +{- Takes action based on git's autocorrect configuration, in preparation for + - an autocorrected command being run. -} +prepare :: String -> (c -> String) -> [c] -> Maybe Repo -> IO () +prepare input showmatch matches r = + case readish . Git.Config.get "help.autocorrect" "0" =<< r of + Just n + | n == 0 -> list + | n < 0 -> warn + | otherwise -> sleep n + Nothing -> list + where + list = giveup $ unlines $ + [ "Unknown command '" ++ input ++ "'" + , "" + , "Did you mean one of these?" + ] ++ map (\m -> "\t" ++ showmatch m) matches + warn = + hPutStr stderr $ unlines + [ "WARNING: You called a command named '" ++ + input ++ "', which does not exist." + , "Continuing under the assumption that you meant '" ++ + showmatch (Prelude.head matches) ++ "'" + ] + sleep n = do + warn + hPutStrLn stderr $ unwords + [ "in" + , show (fromIntegral n / 10 :: Float) + , "seconds automatically..."] + threadDelay (n * 100000) -- deciseconds to microseconds diff --git a/Git/Branch.hs b/Git/Branch.hs new file mode 100644 index 0000000000..875f20f89c --- /dev/null +++ b/Git/Branch.hs @@ -0,0 +1,231 @@ +{- git branch stuff + - + - Copyright 2011 Joey Hess + - + - Licensed under the GNU GPL version 3 or higher. + -} + +{-# LANGUAGE BangPatterns #-} + +module Git.Branch where + +import Common +import Git +import Git.Sha +import Git.Command +import qualified Git.Config +import qualified Git.Ref +import qualified Git.BuildVersion + +{- The currently checked out branch. + - + - In a just initialized git repo before the first commit, + - symbolic-ref will show the master branch, even though that + - branch is not created yet. So, this also looks at show-ref HEAD + - to double-check. + -} +current :: Repo -> IO (Maybe Branch) +current r = do + v <- currentUnsafe r + case v of + Nothing -> return Nothing + Just branch -> + ifM (null <$> pipeReadStrict [Param "show-ref", Param $ fromRef branch] r) + ( return Nothing + , return v + ) + +{- The current branch, which may not really exist yet. -} +currentUnsafe :: Repo -> IO (Maybe Branch) +currentUnsafe r = parse . firstLine + <$> pipeReadStrict [Param "symbolic-ref", Param "-q", Param $ fromRef Git.Ref.headRef] r + where + parse l + | null l = Nothing + | otherwise = Just $ Git.Ref l + +{- Checks if the second branch has any commits not present on the first + - branch. -} +changed :: Branch -> Branch -> Repo -> IO Bool +changed origbranch newbranch repo + | origbranch == newbranch = return False + | otherwise = not . null + <$> changed' origbranch newbranch [Param "-n1"] repo + where + +changed' :: Branch -> Branch -> [CommandParam] -> Repo -> IO String +changed' origbranch newbranch extraps repo = pipeReadStrict ps repo + where + ps = + [ Param "log" + , Param (fromRef origbranch ++ ".." ++ fromRef newbranch) + , Param "--pretty=%H" + ] ++ extraps + +{- Lists commits that are in the second branch and not in the first branch. -} +changedCommits :: Branch -> Branch -> [CommandParam] -> Repo -> IO [Sha] +changedCommits origbranch newbranch extraps repo = + catMaybes . map extractSha . lines + <$> changed' origbranch newbranch extraps repo + +{- Check if it's possible to fast-forward from the old + - ref to the new ref. + - + - This requires there to be a path from the old to the new. -} +fastForwardable :: Ref -> Ref -> Repo -> IO Bool +fastForwardable old new repo = not . null <$> + pipeReadStrict + [ Param "log" + , Param $ fromRef old ++ ".." ++ fromRef new + , Param "-n1" + , Param "--pretty=%H" + , Param "--ancestry-path" + ] repo + +{- Given a set of refs that are all known to have commits not + - on the branch, tries to update the branch by a fast-forward. + - + - In order for that to be possible, one of the refs must contain + - every commit present in all the other refs. + -} +fastForward :: Branch -> [Ref] -> Repo -> IO Bool +fastForward _ [] _ = return True +fastForward branch (first:rest) repo = + -- First, check that the branch does not contain any + -- new commits that are not in the first ref. If it does, + -- cannot fast-forward. + ifM (changed first branch repo) + ( no_ff + , maybe no_ff do_ff =<< findbest first rest + ) + where + no_ff = return False + do_ff to = do + update' branch to repo + return True + findbest c [] = return $ Just c + findbest c (r:rs) + | c == r = findbest c rs + | otherwise = do + better <- changed c r repo + worse <- changed r c repo + case (better, worse) of + (True, True) -> return Nothing -- divergent fail + (True, False) -> findbest r rs -- better + (False, True) -> findbest c rs -- worse + (False, False) -> findbest c rs -- same + +{- The user may have set commit.gpgsign, intending all their manual + - commits to be signed. But signing automatic/background commits could + - easily lead to unwanted gpg prompts or failures. + -} +data CommitMode = ManualCommit | AutomaticCommit + deriving (Eq) + +{- Prevent signing automatic commits. -} +applyCommitMode :: CommitMode -> [CommandParam] -> [CommandParam] +applyCommitMode commitmode ps + | commitmode == AutomaticCommit && not (Git.BuildVersion.older "2.0.0") = + Param "--no-gpg-sign" : ps + | otherwise = ps + +{- Some versions of git commit-tree honor commit.gpgsign themselves, + - but others need -S to be passed to enable gpg signing of manual commits. -} +applyCommitModeForCommitTree :: CommitMode -> [CommandParam] -> Repo -> [CommandParam] +applyCommitModeForCommitTree commitmode ps r + | commitmode == ManualCommit = + case (Git.Config.getMaybe "commit.gpgsign" r) of + Just s | Git.Config.isTrue s == Just True -> + Param "-S":ps + _ -> ps' + | otherwise = ps' + where + ps' = applyCommitMode commitmode ps + +{- Commit via the usual git command. -} +commitCommand :: CommitMode -> [CommandParam] -> Repo -> IO Bool +commitCommand = commitCommand' runBool + +commitCommand' :: ([CommandParam] -> Repo -> IO a) -> CommitMode -> [CommandParam] -> Repo -> IO a +commitCommand' runner commitmode ps = runner $ + Param "commit" : applyCommitMode commitmode ps + +{- Commits the index into the specified branch (or other ref), + - with the specified parent refs, and returns the committed sha. + - + - Without allowempy set, avoids making a commit if there is exactly + - one parent, and it has the same tree that would be committed. + - + - Unlike git-commit, does not run any hooks, or examine the work tree + - in any way. + -} +commit :: CommitMode -> Bool -> String -> Branch -> [Ref] -> Repo -> IO (Maybe Sha) +commit commitmode allowempty message branch parentrefs repo = do + tree <- getSha "write-tree" $ + pipeReadStrict [Param "write-tree"] repo + ifM (cancommit tree) + ( do + sha <- commitTree commitmode message parentrefs tree repo + update' branch sha repo + return $ Just sha + , return Nothing + ) + where + cancommit tree + | allowempty = return True + | otherwise = case parentrefs of + [p] -> maybe False (tree /=) <$> Git.Ref.tree p repo + _ -> return True + +commitAlways :: CommitMode -> String -> Branch -> [Ref] -> Repo -> IO Sha +commitAlways commitmode message branch parentrefs repo = fromJust + <$> commit commitmode True message branch parentrefs repo + +commitTree :: CommitMode -> String -> [Ref] -> Ref -> Repo -> IO Sha +commitTree commitmode message parentrefs tree repo = + getSha "commit-tree" $ + pipeWriteRead ([Param "commit-tree", Param (fromRef tree)] ++ ps) + sendmsg repo + where + sendmsg = Just $ flip hPutStr message + ps = applyCommitModeForCommitTree commitmode parentparams repo + parentparams = map Param $ concatMap (\r -> ["-p", fromRef r]) parentrefs + +{- A leading + makes git-push force pushing a branch. -} +forcePush :: String -> String +forcePush b = "+" ++ b + +{- Updates a branch (or other ref) to a new Sha or branch Ref. -} +update :: String -> Branch -> Ref -> Repo -> IO () +update message branch r = run + [ Param "update-ref" + , Param "-m" + , Param message + , Param $ fromRef branch + , Param $ fromRef r + ] + +update' :: Branch -> Ref -> Repo -> IO () +update' branch r = run + [ Param "update-ref" + , Param $ fromRef branch + , Param $ fromRef r + ] + +{- Checks out a branch, creating it if necessary. -} +checkout :: Branch -> Repo -> IO () +checkout branch = run + [ Param "checkout" + , Param "-q" + , Param "-B" + , Param $ fromRef $ Git.Ref.base branch + ] + +{- Removes a branch. -} +delete :: Branch -> Repo -> IO () +delete branch = run + [ Param "branch" + , Param "-q" + , Param "-D" + , Param $ fromRef $ Git.Ref.base branch + ] diff --git a/Git/BuildVersion.hs b/Git/BuildVersion.hs new file mode 100644 index 0000000000..7d1c53aa0a --- /dev/null +++ b/Git/BuildVersion.hs @@ -0,0 +1,21 @@ +{- git build version + - + - Copyright 2011 Joey Hess + - + - Licensed under the GNU GPL version 3 or higher. + -} + +module Git.BuildVersion where + +import Git.Version +import qualified BuildInfo + +{- Using the version it was configured for avoids running git to check its + - version, at the cost that upgrading git won't be noticed. + - This is only acceptable because it's rare that git's version influences + - code's behavior. -} +buildVersion :: GitVersion +buildVersion = normalize BuildInfo.gitversion + +older :: String -> Bool +older n = buildVersion < normalize n diff --git a/Git/CatFile.hs b/Git/CatFile.hs new file mode 100644 index 0000000000..d19b283e83 --- /dev/null +++ b/Git/CatFile.hs @@ -0,0 +1,254 @@ +{- git cat-file interface + - + - Copyright 2011-2018 Joey Hess + - + - Licensed under the GNU GPL version 3 or higher. + -} + +module Git.CatFile ( + CatFileHandle, + catFileStart, + catFileStart', + catFileStop, + catFile, + catFileDetails, + catTree, + catCommit, + catObject, + catObjectDetails, + catObjectMetaData, +) where + +import System.IO +import qualified Data.ByteString as S +import qualified Data.ByteString.Lazy as L +import qualified Data.ByteString.Lazy.Char8 as L8 +import qualified Data.Map as M +import Data.String +import Data.Char +import Numeric +import System.Posix.Types +import Text.Read + +import Common +import Git +import Git.Sha +import Git.Command +import Git.Types +import Git.FilePath +import Git.HashObject +import qualified Utility.CoProcess as CoProcess +import Utility.FileSystemEncoding +import Utility.Tuple + +data CatFileHandle = CatFileHandle + { catFileProcess :: CoProcess.CoProcessHandle + , checkFileProcess :: CoProcess.CoProcessHandle + , gitRepo :: Repo + } + +catFileStart :: Repo -> IO CatFileHandle +catFileStart = catFileStart' True + +catFileStart' :: Bool -> Repo -> IO CatFileHandle +catFileStart' restartable repo = CatFileHandle + <$> startp "--batch" + <*> startp "--batch-check=%(objectname) %(objecttype) %(objectsize)" + <*> pure repo + where + startp p = gitCoProcessStart restartable + [ Param "cat-file" + , Param p + ] repo + +catFileStop :: CatFileHandle -> IO () +catFileStop h = do + CoProcess.stop (catFileProcess h) + CoProcess.stop (checkFileProcess h) + +{- Reads a file from a specified branch. -} +catFile :: CatFileHandle -> Branch -> FilePath -> IO L.ByteString +catFile h branch file = catObject h $ Ref $ + fromRef branch ++ ":" ++ toInternalGitPath file + +catFileDetails :: CatFileHandle -> Branch -> FilePath -> IO (Maybe (L.ByteString, Sha, ObjectType)) +catFileDetails h branch file = catObjectDetails h $ Ref $ + fromRef branch ++ ":" ++ toInternalGitPath file + +{- Uses a running git cat-file read the content of an object. + - Objects that do not exist will have "" returned. -} +catObject :: CatFileHandle -> Ref -> IO L.ByteString +catObject h object = maybe L.empty fst3 <$> catObjectDetails h object + +catObjectDetails :: CatFileHandle -> Ref -> IO (Maybe (L.ByteString, Sha, ObjectType)) +catObjectDetails h object = query (catFileProcess h) object newlinefallback $ \from -> do + header <- hGetLine from + case parseResp object header of + Just (ParsedResp sha size objtype) -> do + content <- S.hGet from (fromIntegral size) + eatchar '\n' from + return $ Just (L.fromChunks [content], sha, objtype) + Just DNE -> return Nothing + Nothing -> error $ "unknown response from git cat-file " ++ show (header, object) + where + eatchar expected from = do + c <- hGetChar from + when (c /= expected) $ + error $ "missing " ++ (show expected) ++ " from git cat-file" + + -- Slow fallback path for filenames containing newlines. + newlinefallback = queryObjectType object (gitRepo h) >>= \case + Nothing -> return Nothing + Just objtype -> queryContent object (gitRepo h) >>= \case + Nothing -> return Nothing + Just content -> do + -- only the --batch interface allows getting + -- the sha, so have to re-hash the object + sha <- hashObject' objtype + (flip L.hPut content) + (gitRepo h) + return (Just (content, sha, objtype)) + +{- Gets the size and type of an object, without reading its content. -} +catObjectMetaData :: CatFileHandle -> Ref -> IO (Maybe (Integer, ObjectType)) +catObjectMetaData h object = query (checkFileProcess h) object newlinefallback $ \from -> do + resp <- hGetLine from + case parseResp object resp of + Just (ParsedResp _ size objtype) -> + return $ Just (size, objtype) + Just DNE -> return Nothing + Nothing -> error $ "unknown response from git cat-file " ++ show (resp, object) + where + -- Slow fallback path for filenames containing newlines. + newlinefallback = do + sz <- querySize object (gitRepo h) + objtype <- queryObjectType object (gitRepo h) + return $ (,) <$> sz <*> objtype + +data ParsedResp = ParsedResp Sha Integer ObjectType | DNE + +query :: CoProcess.CoProcessHandle -> Ref -> IO a -> (Handle -> IO a) -> IO a +query hdl object newlinefallback receive + -- git cat-file --batch uses a line based protocol, so when the + -- filename itself contains a newline, have to fall back to another + -- method of getting the information. + | '\n' `elem` s = newlinefallback + | otherwise = CoProcess.query hdl send receive + where + send to = hPutStrLn to s + s = fromRef object + +parseResp :: Ref -> String -> Maybe ParsedResp +parseResp object l + | " missing" `isSuffixOf` l -- less expensive than full check + && l == fromRef object ++ " missing" = Just DNE + | otherwise = case words l of + [sha, objtype, size] + | length sha == shaSize -> + case (readObjectType objtype, reads size) of + (Just t, [(bytes, "")]) -> + Just $ ParsedResp (Ref sha) bytes t + _ -> Nothing + | otherwise -> Nothing + _ -> Nothing + +querySingle :: CommandParam -> Ref -> Repo -> (Handle -> IO a) -> IO (Maybe a) +querySingle o r repo reader = assertLocal repo $ + -- In non-batch mode, git cat-file warns on stderr when + -- asked for an object that does not exist. + -- Squelch that warning to behave the same as batch mode. + withNullHandle $ \nullh -> do + let p = gitCreateProcess + [ Param "cat-file" + , o + , Param (fromRef r) + ] repo + let p' = p + { std_err = UseHandle nullh + , std_in = Inherit + , std_out = CreatePipe + } + pid <- createProcess p' + let h = stdoutHandle pid + output <- reader h + hClose h + ifM (checkSuccessProcess (processHandle pid)) + ( return (Just output) + , return Nothing + ) + +querySize :: Ref -> Repo -> IO (Maybe Integer) +querySize r repo = maybe Nothing (readMaybe . takeWhile (/= '\n')) + <$> querySingle (Param "-s") r repo hGetContentsStrict + +queryObjectType :: Ref -> Repo -> IO (Maybe ObjectType) +queryObjectType r repo = maybe Nothing (readObjectType . takeWhile (/= '\n')) + <$> querySingle (Param "cat-file") r repo hGetContentsStrict + +queryContent :: Ref -> Repo -> IO (Maybe L.ByteString) +queryContent r repo = fmap (\b -> L.fromChunks [b]) + <$> querySingle (Param "-p") r repo S.hGetContents + +{- Gets a list of files and directories in a tree. (Not recursive.) -} +catTree :: CatFileHandle -> Ref -> IO [(FilePath, FileMode)] +catTree h treeref = go <$> catObjectDetails h treeref + where + go (Just (b, _, TreeObject)) = parsetree [] b + go _ = [] + + parsetree c b = case L.break (== 0) b of + (modefile, rest) + | L.null modefile -> c + | otherwise -> parsetree + (parsemodefile modefile:c) + (dropsha rest) + + -- these 20 bytes after the NUL hold the file's sha + dropsha = L.drop 21 + + parsemodefile b = + let (modestr, file) = separate (== ' ') (decodeBS b) + in (file, readmode modestr) + readmode = fromMaybe 0 . fmap fst . headMaybe . readOct + +catCommit :: CatFileHandle -> Ref -> IO (Maybe Commit) +catCommit h commitref = go <$> catObjectDetails h commitref + where + go (Just (b, _, CommitObject)) = parseCommit b + go _ = Nothing + +parseCommit :: L.ByteString -> Maybe Commit +parseCommit b = Commit + <$> (extractSha . L8.unpack =<< field "tree") + <*> Just (maybe [] (mapMaybe (extractSha . L8.unpack)) (fields "parent")) + <*> (parsemetadata <$> field "author") + <*> (parsemetadata <$> field "committer") + <*> Just (L8.unpack $ L.intercalate (L.singleton nl) message) + where + field n = headMaybe =<< fields n + fields n = M.lookup (fromString n) fieldmap + fieldmap = M.fromListWith (++) ((map breakfield) header) + breakfield l = + let (k, sp_v) = L.break (== sp) l + in (k, [L.drop 1 sp_v]) + (header, message) = separate L.null ls + ls = L.split nl b + + -- author and committer lines have the form: "name date" + -- The email is always present, even if empty "<>" + parsemetadata l = CommitMetaData + { commitName = whenset $ L.init name_sp + , commitEmail = whenset email + , commitDate = whenset $ L.drop 2 gt_sp_date + } + where + (name_sp, rest) = L.break (== lt) l + (email, gt_sp_date) = L.break (== gt) (L.drop 1 rest) + whenset v + | L.null v = Nothing + | otherwise = Just (L8.unpack v) + + nl = fromIntegral (ord '\n') + sp = fromIntegral (ord ' ') + lt = fromIntegral (ord '<') + gt = fromIntegral (ord '>') diff --git a/Git/CheckAttr.hs b/Git/CheckAttr.hs new file mode 100644 index 0000000000..a248f49822 --- /dev/null +++ b/Git/CheckAttr.hs @@ -0,0 +1,101 @@ +{- git check-attr interface + - + - Copyright 2010-2012 Joey Hess + - + - Licensed under the GNU GPL version 3 or higher. + -} + +module Git.CheckAttr where + +import Common +import Git +import Git.Command +import qualified Git.Version +import qualified Utility.CoProcess as CoProcess + +import System.IO.Error + +type CheckAttrHandle = (CoProcess.CoProcessHandle, [Attr], Bool, String) + +type Attr = String + +{- Starts git check-attr running to look up the specified gitattributes + - values and returns a handle. -} +checkAttrStart :: [Attr] -> Repo -> IO CheckAttrHandle +checkAttrStart attrs repo = do + currdir <- getCurrentDirectory + h <- gitCoProcessStart True params repo + oldgit <- Git.Version.older "1.7.7" + return (h, attrs, oldgit, currdir) + where + params = + [ Param "check-attr" + , Param "-z" + , Param "--stdin" + ] ++ map Param attrs ++ + [ Param "--" ] + +checkAttrStop :: CheckAttrHandle -> IO () +checkAttrStop (h, _, _, _) = CoProcess.stop h + +{- Gets an attribute of a file. When the attribute is not specified, + - returns "" -} +checkAttr :: CheckAttrHandle -> Attr -> FilePath -> IO String +checkAttr (h, attrs, oldgit, currdir) want file = do + pairs <- CoProcess.query h send (receive "") + let vals = map snd $ filter (\(attr, _) -> attr == want) pairs + case vals of + ["unspecified"] -> return "" + [v] -> return v + _ -> error $ "unable to determine " ++ want ++ " attribute of " ++ file + where + send to = hPutStr to $ file' ++ "\0" + receive c from = do + s <- hGetSomeString from 1024 + if null s + then eofError + else do + let v = c ++ s + maybe (receive v from) return (parse v) + eofError = ioError $ mkIOError userErrorType "git check-attr EOF" Nothing Nothing + parse s + -- new null separated output + | '\0' `elem` s = if "\0" `isSuffixOf` s + then + let bits = segment (== '\0') s + in if length bits == (numattrs * 3) + 1 + then Just $ getattrvalues bits [] + else Nothing -- more attributes to come + else Nothing -- output incomplete + -- old one line per value output + | otherwise = if "\n" `isSuffixOf` s + then + let ls = lines s + in if length ls == numattrs + then Just $ map (\(attr, val) -> (attr, oldattrvalue attr val)) + (zip attrs ls) + else Nothing -- more attributes to come + else Nothing -- line incomplete + numattrs = length attrs + + {- Before git 1.7.7, git check-attr worked best with + - absolute filenames; using them worked around some bugs + - with relative filenames. + - + - With newer git, git check-attr chokes on some absolute + - filenames, and the bugs that necessitated them were fixed, + - so use relative filenames. -} + file' + | oldgit = absPathFrom currdir file + | otherwise = relPathDirToFileAbs currdir $ absPathFrom currdir file + oldattrvalue attr l = end bits !! 0 + where + bits = split sep l + sep = ": " ++ attr ++ ": " + getattrvalues (_filename:attr:val:rest) c = getattrvalues rest ((attr,val):c) + getattrvalues _ c = c + +{- User may enter this to override a previous attr setting, when they wish + - to not specify an attr for some files. -} +unspecifiedAttr :: String +unspecifiedAttr = "!" diff --git a/Git/CheckIgnore.hs b/Git/CheckIgnore.hs new file mode 100644 index 0000000000..594882a81d --- /dev/null +++ b/Git/CheckIgnore.hs @@ -0,0 +1,82 @@ +{- git check-ignore interface + - + - Copyright 2013 Joey Hess + - + - Licensed under the GNU GPL version 3 or higher. + -} + +module Git.CheckIgnore ( + CheckIgnoreHandle, + checkIgnoreStart, + checkIgnoreStop, + checkIgnored +) where + +import Common +import Git +import Git.Command +import qualified Git.Version +import qualified Utility.CoProcess as CoProcess + +import System.IO.Error + +type CheckIgnoreHandle = CoProcess.CoProcessHandle + +{- Starts git check-ignore running, and returns a handle. + - + - This relies on git check-ignore --non-matching -v outputting + - lines for both matching an non-matching files. Also relies on + - GIT_FLUSH behavior flushing the output buffer when git check-ignore + - is piping to us. + - + - The first version of git to support what we need is 1.8.4. + - Nothing is returned if an older git is installed. + - + - check-ignore does not support --literal-pathspecs, so remove that + - from the gitGlobalOpts if set. + -} +checkIgnoreStart :: Repo -> IO (Maybe CheckIgnoreHandle) +checkIgnoreStart repo = ifM supportedGitVersion + ( Just <$> gitCoProcessStart True params repo' + , return Nothing + ) + where + params = + [ Param "check-ignore" + , Param "-z" + , Param "--stdin" + , Param "--verbose" + , Param "--non-matching" + ] + repo' = repo { gitGlobalOpts = filter (not . pathspecs) (gitGlobalOpts repo) } + pathspecs (Param "--literal-pathspecs") = True + pathspecs _ = False + +supportedGitVersion :: IO Bool +supportedGitVersion = do + v <- Git.Version.installed + return $ v >= Git.Version.normalize "1.8.4" + +{- For some reason, check-ignore --batch always exits nonzero, + - so ignore any error. -} +checkIgnoreStop :: CheckIgnoreHandle -> IO () +checkIgnoreStop = void . tryIO . CoProcess.stop + +{- Returns True if a file is ignored. -} +checkIgnored :: CheckIgnoreHandle -> FilePath -> IO Bool +checkIgnored h file = CoProcess.query h send (receive "") + where + send to = do + hPutStr to $ file ++ "\0" + hFlush to + receive c from = do + s <- hGetSomeString from 1024 + if null s + then eofError + else do + let v = c ++ s + maybe (receive v from) return (parse v) + parse s = case segment (== '\0') s of + (_source:_line:pattern:_pathname:_eol:[]) -> Just $ not $ null pattern + _ -> Nothing + eofError = ioError $ mkIOError userErrorType "git cat-file EOF" Nothing Nothing diff --git a/Git/Command.hs b/Git/Command.hs new file mode 100644 index 0000000000..871589a2a3 --- /dev/null +++ b/Git/Command.hs @@ -0,0 +1,131 @@ +{- running git commands + - + - Copyright 2010-2013 Joey Hess + - + - Licensed under the GNU GPL version 3 or higher. + -} + +{-# LANGUAGE CPP #-} + +module Git.Command where + +import Common +import Git +import Git.Types +import qualified Utility.CoProcess as CoProcess + +{- Constructs a git command line operating on the specified repo. -} +gitCommandLine :: [CommandParam] -> Repo -> [CommandParam] +gitCommandLine params r@(Repo { location = l@(Local { } ) }) = + setdir ++ settree ++ gitGlobalOpts r ++ params + where + setdir + | gitEnvOverridesGitDir r = [] + | otherwise = [Param $ "--git-dir=" ++ gitdir l] + settree = case worktree l of + Nothing -> [] + Just t -> [Param $ "--work-tree=" ++ t] +gitCommandLine _ repo = assertLocal repo $ error "internal" + +{- Runs git in the specified repo. -} +runBool :: [CommandParam] -> Repo -> IO Bool +runBool params repo = assertLocal repo $ + boolSystemEnv "git" (gitCommandLine params repo) (gitEnv repo) + +{- Runs git in the specified repo, throwing an error if it fails. -} +run :: [CommandParam] -> Repo -> IO () +run params repo = assertLocal repo $ + unlessM (runBool params repo) $ + error $ "git " ++ show params ++ " failed" + +{- Runs git and forces it to be quiet, throwing an error if it fails. -} +runQuiet :: [CommandParam] -> Repo -> IO () +runQuiet params repo = withQuietOutput createProcessSuccess $ + (proc "git" $ toCommand $ gitCommandLine (params) repo) + { env = gitEnv repo } + +{- Runs a git command and returns its output, lazily. + - + - Also returns an action that should be used when the output is all + - read (or no more is needed), that will wait on the command, and + - return True if it succeeded. Failure to wait will result in zombies. + -} +pipeReadLazy :: [CommandParam] -> Repo -> IO (String, IO Bool) +pipeReadLazy params repo = assertLocal repo $ do + (_, Just h, _, pid) <- createProcess p { std_out = CreatePipe } + c <- hGetContents h + return (c, checkSuccessProcess pid) + where + p = gitCreateProcess params repo + +{- Runs a git command, and returns its output, strictly. + - + - Nonzero exit status is ignored. + -} +pipeReadStrict :: [CommandParam] -> Repo -> IO String +pipeReadStrict = pipeReadStrict' hGetContentsStrict + +{- The reader action must be strict. -} +pipeReadStrict' :: (Handle -> IO a) -> [CommandParam] -> Repo -> IO a +pipeReadStrict' reader params repo = assertLocal repo $ + withHandle StdoutHandle (createProcessChecked ignoreFailureProcess) p $ \h -> do + output <- reader h + hClose h + return output + where + p = gitCreateProcess params repo + +{- Runs a git command, feeding it an input, and returning its output, + - which is expected to be fairly small, since it's all read into memory + - strictly. -} +pipeWriteRead :: [CommandParam] -> Maybe (Handle -> IO ()) -> Repo -> IO String +pipeWriteRead params writer repo = assertLocal repo $ + writeReadProcessEnv "git" (toCommand $ gitCommandLine params repo) + (gitEnv repo) writer (Just adjusthandle) + where + adjusthandle h = hSetNewlineMode h noNewlineTranslation + +{- Runs a git command, feeding it input on a handle with an action. -} +pipeWrite :: [CommandParam] -> Repo -> (Handle -> IO ()) -> IO () +pipeWrite params repo = assertLocal repo $ + withHandle StdinHandle createProcessSuccess $ + gitCreateProcess params repo + +{- Reads null terminated output of a git command (as enabled by the -z + - parameter), and splits it. -} +pipeNullSplit :: [CommandParam] -> Repo -> IO ([String], IO Bool) +pipeNullSplit params repo = do + (s, cleanup) <- pipeReadLazy params repo + return (filter (not . null) $ splitc sep s, cleanup) + where + sep = '\0' + +pipeNullSplitStrict :: [CommandParam] -> Repo -> IO [String] +pipeNullSplitStrict params repo = do + s <- pipeReadStrict params repo + return $ filter (not . null) $ splitc sep s + where + sep = '\0' + +pipeNullSplitZombie :: [CommandParam] -> Repo -> IO [String] +pipeNullSplitZombie params repo = leaveZombie <$> pipeNullSplit params repo + +{- Doesn't run the cleanup action. A zombie results. -} +leaveZombie :: (a, IO Bool) -> a +leaveZombie = fst + +{- Runs a git command as a coprocess. -} +gitCoProcessStart :: Bool -> [CommandParam] -> Repo -> IO CoProcess.CoProcessHandle +gitCoProcessStart restartable params repo = CoProcess.start numrestarts "git" + (toCommand $ gitCommandLine params repo) + (gitEnv repo) + where + {- If a long-running git command like cat-file --batch + - crashes, it will likely start up again ok. If it keeps crashing + - 10 times, something is badly wrong. -} + numrestarts = if restartable then 10 else 0 + +gitCreateProcess :: [CommandParam] -> Repo -> CreateProcess +gitCreateProcess params repo = + (proc "git" $ toCommand $ gitCommandLine params repo) + { env = gitEnv repo } diff --git a/Git/Command/Batch.hs b/Git/Command/Batch.hs new file mode 100644 index 0000000000..b2d6509b61 --- /dev/null +++ b/Git/Command/Batch.hs @@ -0,0 +1,19 @@ +{- running batch git commands + - + - Copyright 2010-2013 Joey Hess + - + - Licensed under the GNU GPL version 3 or higher. + -} + +module Git.Command.Batch where + +import Common +import Git +import Git.Command +import Utility.Batch + +{- Runs git in batch mode. -} +run :: BatchCommandMaker -> [CommandParam] -> Repo -> IO Bool +run batchmaker params repo = assertLocal repo $ do + let (cmd, params') = batchmaker ("git", gitCommandLine params repo) + boolSystemEnv cmd params' (gitEnv repo) diff --git a/Git/Config.hs b/Git/Config.hs new file mode 100644 index 0000000000..9cee83f2f6 --- /dev/null +++ b/Git/Config.hs @@ -0,0 +1,203 @@ +{- git repository configuration handling + - + - Copyright 2010-2012 Joey Hess + - + - Licensed under the GNU GPL version 3 or higher. + -} + +module Git.Config where + +import qualified Data.Map as M +import Data.Char + +import Common +import Git +import Git.Types +import qualified Git.Command +import qualified Git.Construct +import Utility.UserInfo + +{- Returns a single git config setting, or a default value if not set. -} +get :: String -> String -> Repo -> String +get key defaultValue repo = M.findWithDefault defaultValue key (config repo) + +{- Returns a list with each line of a multiline config setting. -} +getList :: String -> Repo -> [String] +getList key repo = M.findWithDefault [] key (fullconfig repo) + +{- Returns a single git config setting, if set. -} +getMaybe :: String -> Repo -> Maybe String +getMaybe key repo = M.lookup key (config repo) + +{- Runs git config and populates a repo with its config. + - Avoids re-reading config when run repeatedly. -} +read :: Repo -> IO Repo +read repo@(Repo { config = c }) + | c == M.empty = read' repo + | otherwise = return repo + +{- Reads config even if it was read before. -} +reRead :: Repo -> IO Repo +reRead r = read' $ r + { config = M.empty + , fullconfig = M.empty + } + +{- Cannot use pipeRead because it relies on the config having been already + - read. Instead, chdir to the repo and run git config. + -} +read' :: Repo -> IO Repo +read' repo = go repo + where + go Repo { location = Local { gitdir = d } } = git_config d + go Repo { location = LocalUnknown d } = git_config d + go _ = assertLocal repo $ error "internal" + git_config d = withHandle StdoutHandle createProcessSuccess p $ + hRead repo + where + params = ["config", "--null", "--list"] + p = (proc "git" params) + { cwd = Just d + , env = gitEnv repo + } + +{- Gets the global git config, returning a dummy Repo containing it. -} +global :: IO (Maybe Repo) +global = do + home <- myHomeDir + ifM (doesFileExist $ home ".gitconfig") + ( do + repo <- withHandle StdoutHandle createProcessSuccess p $ + hRead (Git.Construct.fromUnknown) + return $ Just repo + , return Nothing + ) + where + params = ["config", "--null", "--list", "--global"] + p = (proc "git" params) + +{- Reads git config from a handle and populates a repo with it. -} +hRead :: Repo -> Handle -> IO Repo +hRead repo h = do + val <- hGetContentsStrict h + store val repo + +{- Stores a git config into a Repo, returning the new version of the Repo. + - The git config may be multiple lines, or a single line. + - Config settings can be updated incrementally. + -} +store :: String -> Repo -> IO Repo +store s repo = do + let c = parse s + updateLocation $ repo + { config = (M.map Prelude.head c) `M.union` config repo + , fullconfig = M.unionWith (++) c (fullconfig repo) + } + +{- Updates the location of a repo, based on its configuration. + - + - Git.Construct makes LocalUknown repos, of which only a directory is + - known. Once the config is read, this can be fixed up to a Local repo, + - based on the core.bare and core.worktree settings. + -} +updateLocation :: Repo -> IO Repo +updateLocation r@(Repo { location = LocalUnknown d }) + | isBare r = ifM (doesDirectoryExist dotgit) + ( updateLocation' r $ Local dotgit Nothing + , updateLocation' r $ Local d Nothing + ) + | otherwise = updateLocation' r $ Local dotgit (Just d) + where + dotgit = (d ".git") +updateLocation r@(Repo { location = l@(Local {}) }) = updateLocation' r l +updateLocation r = return r + +updateLocation' :: Repo -> RepoLocation -> IO Repo +updateLocation' r l = do + l' <- case getMaybe "core.worktree" r of + Nothing -> return l + Just d -> do + {- core.worktree is relative to the gitdir -} + top <- absPath $ gitdir l + return $ l { worktree = Just $ absPathFrom top d } + return $ r { location = l' } + +{- Parses git config --list or git config --null --list output into a + - config map. -} +parse :: String -> M.Map String [String] +parse [] = M.empty +parse s + -- --list output will have an = in the first line + | all ('=' `elem`) (take 1 ls) = sep '=' ls + -- --null --list output separates keys from values with newlines + | otherwise = sep '\n' $ splitc '\0' s + where + ls = lines s + sep c = M.fromListWith (++) . map (\(k,v) -> (k, [v])) . + map (separate (== c)) + +{- Checks if a string from git config is a true value. -} +isTrue :: String -> Maybe Bool +isTrue s + | s' == "true" = Just True + | s' == "false" = Just False + | otherwise = Nothing + where + s' = map toLower s + +boolConfig :: Bool -> String +boolConfig True = "true" +boolConfig False = "false" + +isBare :: Repo -> Bool +isBare r = fromMaybe False $ isTrue =<< getMaybe coreBare r + +coreBare :: String +coreBare = "core.bare" + +{- Runs a command to get the configuration of a repo, + - and returns a repo populated with the configuration, as well as the raw + - output of the command. -} +fromPipe :: Repo -> String -> [CommandParam] -> IO (Either SomeException (Repo, String)) +fromPipe r cmd params = try $ + withHandle StdoutHandle createProcessSuccess p $ \h -> do + val <- hGetContentsStrict h + r' <- store val r + return (r', val) + where + p = proc cmd $ toCommand params + +{- Reads git config from a specified file and returns the repo populated + - with the configuration. -} +fromFile :: Repo -> FilePath -> IO (Either SomeException (Repo, String)) +fromFile r f = fromPipe r "git" + [ Param "config" + , Param "--file" + , File f + , Param "--list" + ] + +{- Changes a git config setting in the specified config file. + - (Creates the file if it does not already exist.) -} +changeFile :: FilePath -> String -> String -> IO Bool +changeFile f k v = boolSystem "git" + [ Param "config" + , Param "--file" + , File f + , Param k + , Param v + ] + +{- Unsets a git config setting, in both the git repo, + - and the cached config in the Repo. + - + - If unsetting the config fails, including in a read-only repo, or + - when the config is not set, returns Nothing. + -} +unset :: String -> Repo -> IO (Maybe Repo) +unset k r = ifM (Git.Command.runBool ps r) + ( return $ Just $ r { config = M.delete k (config r) } + , return Nothing + ) + where + ps = [Param "config", Param "--unset-all", Param k] diff --git a/Git/ConfigTypes.hs b/Git/ConfigTypes.hs new file mode 100644 index 0000000000..af3dc976df --- /dev/null +++ b/Git/ConfigTypes.hs @@ -0,0 +1,40 @@ +{- git config types + - + - Copyright 2012, 2017 Joey Hess + - + - Licensed under the GNU GPL version 3 or higher. + -} + +module Git.ConfigTypes where + +import Data.Char + +import Common +import Git +import qualified Git.Config + +data SharedRepository = UnShared | GroupShared | AllShared | UmaskShared Int + deriving (Eq) + +getSharedRepository :: Repo -> SharedRepository +getSharedRepository r = + case map toLower $ Git.Config.get "core.sharedrepository" "" r of + "1" -> GroupShared + "2" -> AllShared + "group" -> GroupShared + "true" -> GroupShared + "all" -> AllShared + "world" -> AllShared + "everybody" -> AllShared + v -> maybe UnShared UmaskShared (readish v) + +data DenyCurrentBranch = UpdateInstead | RefusePush | WarnPush | IgnorePush + deriving (Eq) + +getDenyCurrentBranch :: Repo -> DenyCurrentBranch +getDenyCurrentBranch r = + case map toLower $ Git.Config.get "receive.denycurrentbranch" "" r of + "updateinstead" -> UpdateInstead + "warn" -> WarnPush + "ignore" -> IgnorePush + _ -> RefusePush diff --git a/Git/Construct.hs b/Git/Construct.hs new file mode 100644 index 0000000000..5cb80127da --- /dev/null +++ b/Git/Construct.hs @@ -0,0 +1,246 @@ +{- Construction of Git Repo objects + - + - Copyright 2010-2012 Joey Hess + - + - Licensed under the GNU GPL version 3 or higher. + -} + +{-# LANGUAGE CPP #-} + +module Git.Construct ( + fromCwd, + fromAbsPath, + fromPath, + fromUrl, + fromUnknown, + localToUrl, + remoteNamed, + remoteNamedFromKey, + fromRemotes, + fromRemoteLocation, + repoAbsPath, + checkForRepo, + newFrom, +) where + +#ifndef mingw32_HOST_OS +import System.Posix.User +#endif +import qualified Data.Map as M +import Network.URI + +import Common +import Git.Types +import Git +import Git.Remote +import Git.FilePath +import qualified Git.Url as Url +import Utility.UserInfo + +{- Finds the git repository used for the cwd, which may be in a parent + - directory. -} +fromCwd :: IO (Maybe Repo) +fromCwd = getCurrentDirectory >>= seekUp + where + seekUp dir = do + r <- checkForRepo dir + case r of + Nothing -> case upFrom dir of + Nothing -> return Nothing + Just d -> seekUp d + Just loc -> pure $ Just $ newFrom loc + +{- Local Repo constructor, accepts a relative or absolute path. -} +fromPath :: FilePath -> IO Repo +fromPath dir = fromAbsPath =<< absPath dir + +{- Local Repo constructor, requires an absolute path to the repo be + - specified. -} +fromAbsPath :: FilePath -> IO Repo +fromAbsPath dir + | absoluteGitPath dir = hunt + | otherwise = + error $ "internal error, " ++ dir ++ " is not absolute" + where + ret = pure . newFrom . LocalUnknown + canondir = dropTrailingPathSeparator dir + {- When dir == "foo/.git", git looks for "foo/.git/.git", + - and failing that, uses "foo" as the repository. -} + hunt + | (pathSeparator:".git") `isSuffixOf` canondir = + ifM (doesDirectoryExist $ dir ".git") + ( ret dir + , ret (takeDirectory canondir) + ) + | otherwise = ifM (doesDirectoryExist dir) + ( ret dir + -- git falls back to dir.git when dir doesn't + -- exist, as long as dir didn't end with a + -- path separator + , if dir == canondir + then ret (dir ++ ".git") + else ret dir + ) + +{- Remote Repo constructor. Throws exception on invalid url. + - + - Git is somewhat forgiving about urls to repositories, allowing + - eg spaces that are not normally allowed unescaped in urls. + -} +fromUrl :: String -> IO Repo +fromUrl url + | not (isURI url) = fromUrlStrict $ escapeURIString isUnescapedInURI url + | otherwise = fromUrlStrict url + +fromUrlStrict :: String -> IO Repo +fromUrlStrict url + | "file://" `isPrefixOf` url = fromAbsPath $ unEscapeString $ uriPath u + | otherwise = pure $ newFrom $ Url u + where + u = fromMaybe bad $ parseURI url + bad = error $ "bad url " ++ url + +{- Creates a repo that has an unknown location. -} +fromUnknown :: Repo +fromUnknown = newFrom Unknown + +{- Converts a local Repo into a remote repo, using the reference repo + - which is assumed to be on the same host. -} +localToUrl :: Repo -> Repo -> Repo +localToUrl reference r + | not $ repoIsUrl reference = error "internal error; reference repo not url" + | repoIsUrl r = r + | otherwise = case Url.authority reference of + Nothing -> r + Just auth -> + let absurl = concat + [ Url.scheme reference + , "//" + , auth + , repoPath r + ] + in r { location = Url $ fromJust $ parseURI absurl } + +{- Calculates a list of a repo's configured remotes, by parsing its config. -} +fromRemotes :: Repo -> IO [Repo] +fromRemotes repo = mapM construct remotepairs + where + filterconfig f = filter f $ M.toList $ config repo + filterkeys f = filterconfig (\(k,_) -> f k) + remotepairs = filterkeys isRemoteKey + construct (k,v) = remoteNamedFromKey k $ fromRemoteLocation v repo + +{- Sets the name of a remote when constructing the Repo to represent it. -} +remoteNamed :: String -> IO Repo -> IO Repo +remoteNamed n constructor = do + r <- constructor + return $ r { remoteName = Just n } + +{- Sets the name of a remote based on the git config key, such as + - "remote.foo.url". -} +remoteNamedFromKey :: String -> IO Repo -> IO Repo +remoteNamedFromKey = remoteNamed . remoteKeyToRemoteName + +{- Constructs a new Repo for one of a Repo's remotes using a given + - location (ie, an url). -} +fromRemoteLocation :: String -> Repo -> IO Repo +fromRemoteLocation s repo = gen $ parseRemoteLocation s repo + where + gen (RemotePath p) = fromRemotePath p repo + gen (RemoteUrl u) = fromUrl u + +{- Constructs a Repo from the path specified in the git remotes of + - another Repo. -} +fromRemotePath :: FilePath -> Repo -> IO Repo +fromRemotePath dir repo = do + dir' <- expandTilde dir + fromPath $ repoPath repo dir' + +{- Git remotes can have a directory that is specified relative + - to the user's home directory, or that contains tilde expansions. + - This converts such a directory to an absolute path. + - Note that it has to run on the system where the remote is. + -} +repoAbsPath :: FilePath -> IO FilePath +repoAbsPath d = do + d' <- expandTilde d + h <- myHomeDir + return $ h d' + +expandTilde :: FilePath -> IO FilePath +#ifdef mingw32_HOST_OS +expandTilde = return +#else +expandTilde = expandt True + where + expandt _ [] = return "" + expandt _ ('/':cs) = do + v <- expandt True cs + return ('/':v) + expandt True ('~':'/':cs) = do + h <- myHomeDir + return $ h cs + expandt True ('~':cs) = do + let (name, rest) = findname "" cs + u <- getUserEntryForName name + return $ homeDirectory u rest + expandt _ (c:cs) = do + v <- expandt False cs + return (c:v) + findname n [] = (n, "") + findname n (c:cs) + | c == '/' = (n, cs) + | otherwise = findname (n++[c]) cs +#endif + +{- Checks if a git repository exists in a directory. Does not find + - git repositories in parent directories. -} +checkForRepo :: FilePath -> IO (Maybe RepoLocation) +checkForRepo dir = + check isRepo $ + check gitDirFile $ + check isBareRepo $ + return Nothing + where + check test cont = maybe cont (return . Just) =<< test + checkdir c = ifM c + ( return $ Just $ LocalUnknown dir + , return Nothing + ) + isRepo = checkdir $ + gitSignature (".git" "config") + <||> + -- A git-worktree lacks .git/config, but has .git/commondir. + -- (Normally the .git is a file, not a symlink, but it can + -- be converted to a symlink and git will still work; + -- this handles that case.) + gitSignature (".git" "gitdir") + isBareRepo = checkdir $ gitSignature "config" + <&&> doesDirectoryExist (dir "objects") + gitDirFile = do + -- git-submodule, git-worktree, and --separate-git-dir + -- make .git be a file pointing to the real git directory. + c <- firstLine <$> + catchDefaultIO "" (readFile $ dir ".git") + return $ if gitdirprefix `isPrefixOf` c + then Just $ Local + { gitdir = absPathFrom dir $ + drop (length gitdirprefix) c + , worktree = Just dir + } + else Nothing + where + gitdirprefix = "gitdir: " + gitSignature file = doesFileExist $ dir file + +newFrom :: RepoLocation -> Repo +newFrom l = Repo + { location = l + , config = M.empty + , fullconfig = M.empty + , remoteName = Nothing + , gitEnv = Nothing + , gitEnvOverridesGitDir = False + , gitGlobalOpts = [] + } + diff --git a/Git/CurrentRepo.hs b/Git/CurrentRepo.hs new file mode 100644 index 0000000000..82fd0c6429 --- /dev/null +++ b/Git/CurrentRepo.hs @@ -0,0 +1,76 @@ +{- The current git repository. + - + - Copyright 2012 Joey Hess + - + - Licensed under the GNU GPL version 3 or higher. + -} + +module Git.CurrentRepo where + +import Common +import Git.Types +import Git.Construct +import qualified Git.Config +import Utility.Env +import Utility.Env.Set + +{- Gets the current git repository. + - + - Honors GIT_DIR and GIT_WORK_TREE. + - Both environment variables are unset, to avoid confusing other git + - commands that also look at them. Instead, the Git module passes + - --work-tree and --git-dir to git commands it runs. + - + - When GIT_WORK_TREE or core.worktree are set, changes the working + - directory if necessary to ensure it is within the repository's work + - tree. While not needed for git commands, this is useful for anything + - else that looks for files in the worktree. + - + - Also works around a git bug when running some hooks. It + - runs the hooks in the top of the repository, but if GIT_WORK_TREE + - was relative (but not "."), it then points to the wrong directory. + - In this situation GIT_PREFIX contains the directory that + - GIT_WORK_TREE is relative to. + -} +get :: IO Repo +get = do + gd <- getpathenv "GIT_DIR" + r <- configure gd =<< fromCwd + prefix <- getpathenv "GIT_PREFIX" + wt <- maybe (worktree $ location r) Just + <$> getpathenvprefix "GIT_WORK_TREE" prefix + case wt of + Nothing -> return r + Just d -> do + curr <- getCurrentDirectory + unless (d `dirContains` curr) $ + setCurrentDirectory d + return $ addworktree wt r + where + getpathenv s = do + v <- getEnv s + case v of + Just d -> do + unsetEnv s + return (Just d) + Nothing -> return Nothing + + getpathenvprefix s (Just prefix) | not (null prefix) = + getpathenv s >>= \case + Nothing -> return Nothing + Just d + | d == "." -> return (Just d) + | otherwise -> Just <$> absPath (prefix d) + getpathenvprefix s _ = getpathenv s + + configure Nothing (Just r) = Git.Config.read r + configure (Just d) _ = do + absd <- absPath d + curr <- getCurrentDirectory + Git.Config.read $ newFrom $ + Local { gitdir = absd, worktree = Just curr } + configure Nothing Nothing = giveup "Not in a git repository." + + addworktree w r = changelocation r $ + Local { gitdir = gitdir (location r), worktree = w } + changelocation r l = r { location = l } diff --git a/Git/DiffTree.hs b/Git/DiffTree.hs new file mode 100644 index 0000000000..dae99db414 --- /dev/null +++ b/Git/DiffTree.hs @@ -0,0 +1,128 @@ +{- git diff-tree interface + - + - Copyright 2012 Joey Hess + - + - Licensed under the GNU GPL version 3 or higher. + -} + +module Git.DiffTree ( + DiffTreeItem(..), + isDiffOf, + diffTree, + diffTreeRecursive, + diffIndex, + diffWorkTree, + diffFiles, + diffLog, + commitDiff, +) where + +import Numeric + +import Common +import Git +import Git.Sha +import Git.Command +import Git.FilePath +import Git.DiffTreeItem +import qualified Git.Filename +import qualified Git.Ref + +{- Checks if the DiffTreeItem modifies a file with a given name + - or under a directory by that name. -} +isDiffOf :: DiffTreeItem -> TopFilePath -> Bool +isDiffOf diff f = case getTopFilePath f of + "" -> True -- top of repo contains all + d -> d `dirContains` getTopFilePath (file diff) + +{- Diffs two tree Refs. -} +diffTree :: Ref -> Ref -> Repo -> IO ([DiffTreeItem], IO Bool) +diffTree src dst = getdiff (Param "diff-tree") + [Param (fromRef src), Param (fromRef dst), Param "--"] + +{- Diffs two tree Refs, recursing into sub-trees -} +diffTreeRecursive :: Ref -> Ref -> Repo -> IO ([DiffTreeItem], IO Bool) +diffTreeRecursive src dst = getdiff (Param "diff-tree") + [Param "-r", Param (fromRef src), Param (fromRef dst), Param "--"] + +{- Diffs between a tree and the index. Does nothing if there is not yet a + - commit in the repository. -} +diffIndex :: Ref -> Repo -> IO ([DiffTreeItem], IO Bool) +diffIndex ref = diffIndex' ref [Param "--cached"] + +{- Diffs between a tree and the working tree. Does nothing if there is not + - yet a commit in the repository, or if the repository is bare. -} +diffWorkTree :: Ref -> Repo -> IO ([DiffTreeItem], IO Bool) +diffWorkTree ref repo = + ifM (Git.Ref.headExists repo) + ( diffIndex' ref [] repo + , return ([], return True) + ) + +diffIndex' :: Ref -> [CommandParam] -> Repo -> IO ([DiffTreeItem], IO Bool) +diffIndex' ref params repo = + ifM (Git.Ref.headExists repo) + ( getdiff (Param "diff-index") + ( params ++ [Param $ fromRef ref] ++ [Param "--"] ) + repo + , return ([], return True) + ) + +{- Diff between the index and work tree. -} +diffFiles :: [CommandParam] -> Repo -> IO ([DiffTreeItem], IO Bool) +diffFiles = getdiff (Param "diff-files") + +{- Runs git log in --raw mode to get the changes that were made in + - a particular commit to particular files. The output format + - is adjusted to be the same as diff-tree --raw._-} +diffLog :: [CommandParam] -> Repo -> IO ([DiffTreeItem], IO Bool) +diffLog params = getdiff (Param "log") + (Param "-n1" : Param "--abbrev=40" : Param "--pretty=format:" : params) + +{- Uses git show to get the changes made by a commit. + - + - Does not support merge commits, and will fail on them. -} +commitDiff :: Sha -> Repo -> IO ([DiffTreeItem], IO Bool) +commitDiff ref = getdiff (Param "show") + [ Param "--abbrev=40", Param "--pretty=", Param "--raw", Param (fromRef ref) ] + +getdiff :: CommandParam -> [CommandParam] -> Repo -> IO ([DiffTreeItem], IO Bool) +getdiff command params repo = do + (diff, cleanup) <- pipeNullSplit ps repo + return (parseDiffRaw diff, cleanup) + where + ps = + command : + Param "-z" : + Param "--raw" : + Param "--no-renames" : + Param "-l0" : + params + +{- Parses --raw output used by diff-tree and git-log. -} +parseDiffRaw :: [String] -> [DiffTreeItem] +parseDiffRaw l = go l + where + go [] = [] + go (info:f:rest) = mk info f : go rest + go (s:[]) = error $ "diff-tree parse error near \"" ++ s ++ "\"" + + mk info f = DiffTreeItem + { srcmode = readmode srcm + , dstmode = readmode dstm + , srcsha = fromMaybe (error "bad srcsha") $ extractSha ssha + , dstsha = fromMaybe (error "bad dstsha") $ extractSha dsha + , status = s + , file = asTopFilePath $ fromInternalGitPath $ Git.Filename.decode f + } + where + readmode = fst . Prelude.head . readOct + + -- info = : SP SP SP SP + -- All fields are fixed, so we can pull them out of + -- specific positions in the line. + (srcm, past_srcm) = splitAt 7 $ drop 1 info + (dstm, past_dstm) = splitAt 7 past_srcm + (ssha, past_ssha) = splitAt shaSize past_dstm + (dsha, past_dsha) = splitAt shaSize $ drop 1 past_ssha + s = drop 1 past_dsha diff --git a/Git/DiffTreeItem.hs b/Git/DiffTreeItem.hs new file mode 100644 index 0000000000..859f590c17 --- /dev/null +++ b/Git/DiffTreeItem.hs @@ -0,0 +1,24 @@ +{- git diff-tree item + - + - Copyright 2012 Joey Hess + - + - Licensed under the GNU GPL version 3 or higher. + -} + +module Git.DiffTreeItem ( + DiffTreeItem(..), +) where + +import System.Posix.Types + +import Git.FilePath +import Git.Types + +data DiffTreeItem = DiffTreeItem + { srcmode :: FileMode + , dstmode :: FileMode + , srcsha :: Sha -- nullSha if file was added + , dstsha :: Sha -- nullSha if file was deleted + , status :: String + , file :: TopFilePath + } deriving Show diff --git a/Git/Env.hs b/Git/Env.hs new file mode 100644 index 0000000000..7e5a2b242e --- /dev/null +++ b/Git/Env.hs @@ -0,0 +1,69 @@ +{- Adjusting the environment while running git commands. + - + - Copyright 2014-2016 Joey Hess + - + - Licensed under the GNU GPL version 3 or higher. + -} + +{-# LANGUAGE CPP #-} + +module Git.Env where + +import Common +import Git +import Git.Types +import Utility.Env + +{- Adjusts the gitEnv of a Repo. Copies the system environment if the repo + - does not have any gitEnv yet. -} +adjustGitEnv :: Repo -> ([(String, String)] -> [(String, String)]) -> IO Repo +adjustGitEnv g adj = do + e <- maybe copyGitEnv return (gitEnv g) + let e' = adj e + return $ g { gitEnv = Just e' } + where + +{- Copies the current environment, so it can be adjusted when running a git + - command. -} +copyGitEnv :: IO [(String, String)] +copyGitEnv = do +#ifdef __ANDROID__ + {- This should not be necessary on Android, but there is some + - weird getEnvironment breakage. See + - https://github.com/neurocyte/ghc-android/issues/7 + - Use getEnv to get some key environment variables that + - git expects to have. -} + let keyenv = words "USER PATH GIT_EXEC_PATH HOSTNAME HOME" + let getEnvPair k = maybe Nothing (\v -> Just (k, v)) <$> getEnv k + catMaybes <$> forM keyenv getEnvPair +#else + getEnvironment +#endif + +addGitEnv :: Repo -> String -> String -> IO Repo +addGitEnv g var val = adjustGitEnv g (addEntry var val) + +{- Environment variables to use when running a command. + - Includes GIT_DIR pointing at the repo, and GIT_WORK_TREE when the repo + - is not bare. Also includes anything added to the Repo's gitEnv, + - and a copy of the rest of the system environment. -} +propGitEnv :: Repo -> IO [(String, String)] +propGitEnv g = do + g' <- addGitEnv g "GIT_DIR" (localGitDir g) + g'' <- maybe (pure g') (addGitEnv g' "GIT_WORK_TREE") (repoWorkTree g) + return $ fromMaybe [] (gitEnv g'') + +{- Use with any action that makes a commit to set metadata. -} +commitWithMetaData :: CommitMetaData -> CommitMetaData -> (Repo -> IO a) -> Repo -> IO a +commitWithMetaData authormetadata committermetadata a g = + a =<< adjustGitEnv g adj + where + adj = mkadj "AUTHOR" authormetadata + . mkadj "COMMITTER" committermetadata + mkadj p md = go "NAME" commitName + . go "EMAIL" commitEmail + . go "DATE" commitDate + where + go s getv = case getv md of + Nothing -> id + Just v -> addEntry ("GIT_" ++ p ++ "_" ++ s) v diff --git a/Git/FileMode.hs b/Git/FileMode.hs new file mode 100644 index 0000000000..b07fd302c4 --- /dev/null +++ b/Git/FileMode.hs @@ -0,0 +1,21 @@ +{- git file modes + - + - Copyright 2013 Joey Hess + - + - Licensed under the GNU GPL version 3 or higher. + -} + +{-# LANGUAGE CPP #-} + +module Git.FileMode where + +import Utility.FileMode + +symLinkMode :: FileMode +symLinkMode = 40960 + +{- Git uses a special file mode to indicate a symlink. This is the case + - even on Windows, so we hard code the valuse here, rather than using + - System.Posix.Files.symbolicLinkMode. -} +isSymLink :: FileMode -> Bool +isSymLink = checkMode symLinkMode diff --git a/Git/FilePath.hs b/Git/FilePath.hs new file mode 100644 index 0000000000..ffa3331075 --- /dev/null +++ b/Git/FilePath.hs @@ -0,0 +1,86 @@ +{- git FilePath library + - + - Different git commands use different types of FilePaths to refer to + - files in the repository. Some commands use paths relative to the + - top of the repository even when run in a subdirectory. Adding some + - types helps keep that straight. + - + - Copyright 2012-2013 Joey Hess + - + - Licensed under the GNU GPL version 3 or higher. + -} + +{-# LANGUAGE CPP #-} + +module Git.FilePath ( + TopFilePath, + BranchFilePath(..), + descBranchFilePath, + getTopFilePath, + fromTopFilePath, + toTopFilePath, + asTopFilePath, + InternalGitPath, + toInternalGitPath, + fromInternalGitPath, + absoluteGitPath +) where + +import Common +import Git + +import qualified System.FilePath.Posix + +{- A FilePath, relative to the top of the git repository. -} +newtype TopFilePath = TopFilePath { getTopFilePath :: FilePath } + deriving (Show, Eq, Ord) + +{- A file in a branch or other treeish. -} +data BranchFilePath = BranchFilePath Ref TopFilePath + +{- Git uses the branch:file form to refer to a BranchFilePath -} +descBranchFilePath :: BranchFilePath -> String +descBranchFilePath (BranchFilePath b f) = fromRef b ++ ':' : getTopFilePath f + +{- Path to a TopFilePath, within the provided git repo. -} +fromTopFilePath :: TopFilePath -> Git.Repo -> FilePath +fromTopFilePath p repo = combine (repoPath repo) (getTopFilePath p) + +{- The input FilePath can be absolute, or relative to the CWD. -} +toTopFilePath :: FilePath -> Git.Repo -> IO TopFilePath +toTopFilePath file repo = TopFilePath <$> relPathDirToFile (repoPath repo) file + +{- The input FilePath must already be relative to the top of the git + - repository -} +asTopFilePath :: FilePath -> TopFilePath +asTopFilePath file = TopFilePath file + +{- Git may use a different representation of a path when storing + - it internally. + - + - On Windows, git uses '/' to separate paths stored in the repository, + - despite Windows using '\'. + - + -} +type InternalGitPath = String + +toInternalGitPath :: FilePath -> InternalGitPath +#ifndef mingw32_HOST_OS +toInternalGitPath = id +#else +toInternalGitPath = replace "\\" "/" +#endif + +fromInternalGitPath :: InternalGitPath -> FilePath +#ifndef mingw32_HOST_OS +fromInternalGitPath = id +#else +fromInternalGitPath = replace "/" "\\" +#endif + +{- isAbsolute on Windows does not think "/foo" or "\foo" is absolute, + - so try posix paths. + -} +absoluteGitPath :: FilePath -> Bool +absoluteGitPath p = isAbsolute p || + System.FilePath.Posix.isAbsolute (toInternalGitPath p) diff --git a/Git/Filename.hs b/Git/Filename.hs new file mode 100644 index 0000000000..355e75f459 --- /dev/null +++ b/Git/Filename.hs @@ -0,0 +1,34 @@ +{- Some git commands output encoded filenames, in a rather annoyingly complex + - C-style encoding. + - + - Copyright 2010, 2011 Joey Hess + - + - Licensed under the GNU GPL version 3 or higher. + -} + +module Git.Filename where + +import Common +import Utility.Format (decode_c, encode_c) + +import Data.Char + +decode :: String -> FilePath +decode [] = [] +decode f@(c:s) + -- encoded strings will be inside double quotes + | c == '"' && end s == ['"'] = decode_c $ beginning s + | otherwise = f + +{- Should not need to use this, except for testing decode. -} +encode :: FilePath -> String +encode s = "\"" ++ encode_c s ++ "\"" + +{- For quickcheck. + - + - See comment on Utility.Format.prop_encode_c_decode_c_roundtrip for + - why this only tests chars < 256 -} +prop_encode_decode_roundtrip :: String -> Bool +prop_encode_decode_roundtrip s = s' == decode (encode s') + where + s' = filter (\c -> ord c < 256) s diff --git a/Git/Fsck.hs b/Git/Fsck.hs new file mode 100644 index 0000000000..3092ff2c67 --- /dev/null +++ b/Git/Fsck.hs @@ -0,0 +1,168 @@ +{- git fsck interface + - + - Copyright 2013 Joey Hess + - + - Licensed under the GNU GPL version 3 or higher. + -} + +{-# LANGUAGE BangPatterns, CPP #-} + +module Git.Fsck ( + FsckResults(..), + MissingObjects, + findBroken, + foundBroken, + findMissing, + isMissing, + knownMissing, +) where + +import Common +import Git +import Git.Command +import Git.Sha +import Utility.Batch +import qualified Git.Version + +import qualified Data.Set as S +import Control.Concurrent.Async +#if MIN_VERSION_base(4,9,0) +import qualified Data.Semigroup as Sem +#endif +import Prelude + +data FsckResults + = FsckFoundMissing + { missingObjects :: MissingObjects + , missingObjectsTruncated :: Bool + } + | FsckFailed + deriving (Show) + +data FsckOutput + = FsckOutput MissingObjects Truncated + | NoFsckOutput + | AllDuplicateEntriesWarning + +type MissingObjects = S.Set Sha + +type Truncated = Bool + +appendFsckOutput :: FsckOutput -> FsckOutput -> FsckOutput +appendFsckOutput (FsckOutput s1 t1) (FsckOutput s2 t2) = + FsckOutput (S.union s1 s2) (t1 || t2) +appendFsckOutput (FsckOutput s t) _ = FsckOutput s t +appendFsckOutput _ (FsckOutput s t) = FsckOutput s t +appendFsckOutput NoFsckOutput NoFsckOutput = NoFsckOutput +appendFsckOutput AllDuplicateEntriesWarning AllDuplicateEntriesWarning = AllDuplicateEntriesWarning +appendFsckOutput AllDuplicateEntriesWarning NoFsckOutput = AllDuplicateEntriesWarning +appendFsckOutput NoFsckOutput AllDuplicateEntriesWarning = AllDuplicateEntriesWarning + +#if MIN_VERSION_base(4,9,0) +instance Sem.Semigroup FsckOutput where + (<>) = appendFsckOutput +#endif + +instance Monoid FsckOutput where + mempty = NoFsckOutput +#if MIN_VERSION_base(4,11,0) +#elif MIN_VERSION_base(4,9,0) + mappend = (Sem.<>) +#else + mappend = appendFsckOutput +#endif + +{- Runs fsck to find some of the broken objects in the repository. + - May not find all broken objects, if fsck fails on bad data in some of + - the broken objects it does find. + - + - Strategy: Rather than parsing fsck's current specific output, + - look for anything in its output (both stdout and stderr) that appears + - to be a git sha. Not all such shas are of broken objects, so ask git + - to try to cat the object, and see if it fails. + -} +findBroken :: Bool -> Repo -> IO FsckResults +findBroken batchmode r = do + supportsNoDangling <- (>= Git.Version.normalize "1.7.10") + <$> Git.Version.installed + let (command, params) = ("git", fsckParams supportsNoDangling r) + (command', params') <- if batchmode + then toBatchCommand (command, params) + else return (command, params) + + p@(_, _, _, pid) <- createProcess $ + (proc command' (toCommand params')) + { std_out = CreatePipe + , std_err = CreatePipe + } + (o1, o2) <- concurrently + (parseFsckOutput maxobjs r supportsNoDangling (stdoutHandle p)) + (parseFsckOutput maxobjs r supportsNoDangling (stderrHandle p)) + fsckok <- checkSuccessProcess pid + case mappend o1 o2 of + FsckOutput badobjs truncated + | S.null badobjs && not fsckok -> return FsckFailed + | otherwise -> return $ FsckFoundMissing badobjs truncated + NoFsckOutput + | not fsckok -> return FsckFailed + | otherwise -> return noproblem + -- If all fsck output was duplicateEntries warnings, + -- the repository is not broken, it just has some unusual + -- tree objects in it. So ignore nonzero exit status. + AllDuplicateEntriesWarning -> return noproblem + where + maxobjs = 10000 + noproblem = FsckFoundMissing S.empty False + +foundBroken :: FsckResults -> Bool +foundBroken FsckFailed = True +foundBroken (FsckFoundMissing s _) = not (S.null s) + +knownMissing :: FsckResults -> MissingObjects +knownMissing FsckFailed = S.empty +knownMissing (FsckFoundMissing s _) = s + +{- Finds objects that are missing from the git repsitory, or are corrupt. + - + - This does not use git cat-file --batch, because catting a corrupt + - object can cause it to crash, or to report incorrect size information. + -} +findMissing :: [Sha] -> Repo -> IO MissingObjects +findMissing objs r = S.fromList <$> filterM (`isMissing` r) objs + +parseFsckOutput :: Int -> Repo -> Bool -> Handle -> IO FsckOutput +parseFsckOutput maxobjs r supportsNoDangling h = do + ls <- lines <$> hGetContents h + if null ls + then return NoFsckOutput + else if all ("duplicateEntries" `isInfixOf`) ls + then return AllDuplicateEntriesWarning + else do + let shas = findShas supportsNoDangling ls + let !truncated = length shas > maxobjs + missingobjs <- findMissing (take maxobjs shas) r + return $ FsckOutput missingobjs truncated + +isMissing :: Sha -> Repo -> IO Bool +isMissing s r = either (const True) (const False) <$> tryIO dump + where + dump = runQuiet + [ Param "show" + , Param (fromRef s) + ] r + +findShas :: Bool -> [String] -> [Sha] +findShas supportsNoDangling = catMaybes . map extractSha . concat . map words . filter wanted + where + wanted l + | supportsNoDangling = True + | otherwise = not ("dangling " `isPrefixOf` l) + +fsckParams :: Bool -> Repo -> [CommandParam] +fsckParams supportsNoDangling = gitCommandLine $ map Param $ catMaybes + [ Just "fsck" + , if supportsNoDangling + then Just "--no-dangling" + else Nothing + , Just "--no-reflogs" + ] diff --git a/Git/GCrypt.hs b/Git/GCrypt.hs new file mode 100644 index 0000000000..e61b763588 --- /dev/null +++ b/Git/GCrypt.hs @@ -0,0 +1,109 @@ +{- git-remote-gcrypt support + - + - https://spwhitton.name/tech/code/git-remote-gcrypt/ + - + - Copyright 2013 Joey Hess + - + - Licensed under the GNU GPL version 3 or higher. + -} + +module Git.GCrypt where + +import Common +import Git.Types +import Git.Construct +import qualified Git.Config as Config +import qualified Git.Command as Command +import Utility.Gpg + +urlScheme :: String +urlScheme = "gcrypt:" + +urlPrefix :: String +urlPrefix = urlScheme ++ ":" + +isEncrypted :: Repo -> Bool +isEncrypted Repo { location = Url url } = urlPrefix `isPrefixOf` show url +isEncrypted _ = False + +{- The first Repo is the git repository that has the second Repo + - as one of its remotes. + - + - When the remote Repo uses gcrypt, returns the actual underlying + - git repository that gcrypt is using to store its data. + - + - Throws an exception if an url is invalid or the repo does not use + - gcrypt. + -} +encryptedRemote :: Repo -> Repo -> IO Repo +encryptedRemote baserepo = go + where + go Repo { location = Url url } + | urlPrefix `isPrefixOf` u = + fromRemoteLocation (drop plen u) baserepo + | otherwise = notencrypted + where + u = show url + plen = length urlPrefix + go _ = notencrypted + notencrypted = giveup "not a gcrypt encrypted repository" + +data ProbeResult = Decryptable | NotDecryptable | NotEncrypted + +{- Checks if the git repo at a location uses gcrypt. + - + - Rather expensive -- many need to fetch the entire repo contents. + - (Which is fine if the repo is going to be added as a remote..) + -} +probeRepo :: String -> Repo -> IO ProbeResult +probeRepo loc baserepo = do + let p = proc "git" $ toCommand $ Command.gitCommandLine + [ Param "remote-gcrypt" + , Param "--check" + , Param loc + ] baserepo + (_, _, _, pid) <- createProcess p + code <- waitForProcess pid + return $ case code of + ExitSuccess -> Decryptable + ExitFailure 1 -> NotDecryptable + ExitFailure _ -> NotEncrypted + +type GCryptId = String + +{- gcrypt gives each encrypted repository a uique gcrypt-id, + - which is stored in the repository (in encrypted form) + - and cached in a per-remote gcrypt-id configuration setting. -} +remoteRepoId :: Repo -> Maybe RemoteName -> Maybe GCryptId +remoteRepoId = getRemoteConfig "gcrypt-id" + +getRemoteConfig :: String -> Repo -> Maybe RemoteName -> Maybe String +getRemoteConfig field repo remotename = do + n <- remotename + Config.getMaybe (remoteConfigKey field n) repo + +{- Gpg keys that the remote is encrypted for. + - If empty, gcrypt uses --default-recipient-self -} +getParticiantList :: Maybe Repo -> Repo -> Maybe RemoteName -> KeyIds +getParticiantList globalconfigrepo repo remotename = KeyIds $ parse $ firstJust + [ getRemoteConfig "gcrypt-participants" repo remotename + , Config.getMaybe defaultkey repo + , Config.getMaybe defaultkey =<< globalconfigrepo + ] + where + defaultkey = "gcrypt.participants" + parse (Just "simple") = [] + parse (Just l) = words l + parse Nothing = [] + +remoteParticipantConfigKey :: RemoteName -> String +remoteParticipantConfigKey = remoteConfigKey "gcrypt-participants" + +remotePublishParticipantConfigKey :: RemoteName -> String +remotePublishParticipantConfigKey = remoteConfigKey "gcrypt-publish-participants" + +remoteSigningKey :: RemoteName -> String +remoteSigningKey = remoteConfigKey "gcrypt-signingkey" + +remoteConfigKey :: String -> RemoteName -> String +remoteConfigKey key remotename = "remote." ++ remotename ++ "." ++ key diff --git a/Git/HashObject.hs b/Git/HashObject.hs new file mode 100644 index 0000000000..399e36d460 --- /dev/null +++ b/Git/HashObject.hs @@ -0,0 +1,63 @@ +{- git hash-object interface + - + - Copyright 2011-2014 Joey Hess + - + - Licensed under the GNU GPL version 3 or higher. + -} + +{-# LANGUAGE CPP #-} + +module Git.HashObject where + +import Common +import Git +import Git.Sha +import Git.Command +import Git.Types +import qualified Utility.CoProcess as CoProcess +import Utility.Tmp + +type HashObjectHandle = CoProcess.CoProcessHandle + +hashObjectStart :: Repo -> IO HashObjectHandle +hashObjectStart = gitCoProcessStart True + [ Param "hash-object" + , Param "-w" + , Param "--stdin-paths" + , Param "--no-filters" + ] + +hashObjectStop :: HashObjectHandle -> IO () +hashObjectStop = CoProcess.stop + +{- Injects a file into git, returning the Sha of the object. -} +hashFile :: HashObjectHandle -> FilePath -> IO Sha +hashFile h file = CoProcess.query h send receive + where + send to = hPutStrLn to =<< absPath file + receive from = getSha "hash-object" $ hGetLine from + +{- Injects a blob into git. Unfortunately, the current git-hash-object + - interface does not allow batch hashing without using temp files. -} +hashBlob :: HashObjectHandle -> String -> IO Sha +hashBlob h s = withTmpFile "hash" $ \tmp tmph -> do +#ifdef mingw32_HOST_OS + hSetNewlineMode tmph noNewlineTranslation +#endif + hPutStr tmph s + hClose tmph + hashFile h tmp + +{- Injects some content into git, returning its Sha. + - + - Avoids using a tmp file, but runs a new hash-object command each + - time called. -} +hashObject :: ObjectType -> String -> Repo -> IO Sha +hashObject objtype content = hashObject' objtype (flip hPutStr content) + +hashObject' :: ObjectType -> (Handle -> IO ()) -> Repo -> IO Sha +hashObject' objtype writer repo = getSha subcmd $ + pipeWriteRead (map Param params) (Just writer) repo + where + subcmd = "hash-object" + params = [subcmd, "-t", show objtype, "-w", "--stdin", "--no-filters"] diff --git a/Git/Hook.hs b/Git/Hook.hs new file mode 100644 index 0000000000..3f7c3dd850 --- /dev/null +++ b/Git/Hook.hs @@ -0,0 +1,90 @@ +{- git hooks + - + - Copyright 2013-2018 Joey Hess + - + - Licensed under the GNU GPL version 3 or higher. + -} + +{-# LANGUAGE CPP #-} + +module Git.Hook where + +import Common +import Git +import Utility.Tmp +import Utility.Shell +#ifndef mingw32_HOST_OS +import Utility.FileMode +#endif + +data Hook = Hook + { hookName :: FilePath + , hookScript :: String + , hookOldScripts :: [String] + } + deriving (Ord) + +instance Eq Hook where + a == b = hookName a == hookName b + +hookFile :: Hook -> Repo -> FilePath +hookFile h r = localGitDir r "hooks" hookName h + +{- Writes a hook. Returns False if the hook already exists with a different + - content. Upgrades old scripts. -} +hookWrite :: Hook -> Repo -> IO Bool +hookWrite h r = ifM (doesFileExist f) + ( expectedContent h r >>= \case + UnexpectedContent -> return False + ExpectedContent -> return True + OldExpectedContent -> go + , go + ) + where + f = hookFile h r + go = do + viaTmp writeFile f (hookScript h) + p <- getPermissions f + setPermissions f $ p {executable = True} + return True + +{- Removes a hook. Returns False if the hook contained something else, and + - could not be removed. -} +hookUnWrite :: Hook -> Repo -> IO Bool +hookUnWrite h r = ifM (doesFileExist f) + ( expectedContent h r >>= \case + UnexpectedContent -> return False + _ -> do + removeFile f + return True + , return True + ) + where + f = hookFile h r + +data ExpectedContent = UnexpectedContent | ExpectedContent | OldExpectedContent + +expectedContent :: Hook -> Repo -> IO ExpectedContent +expectedContent h r = do + content <- readFile $ hookFile h r + return $ if content == hookScript h + then ExpectedContent + else if any (content ==) (hookOldScripts h) + then OldExpectedContent + else UnexpectedContent + +hookExists :: Hook -> Repo -> IO Bool +hookExists h r = do + let f = hookFile h r + catchBoolIO $ +#ifndef mingw32_HOST_OS + isExecutable . fileMode <$> getFileStatus f +#else + doesFileExist f +#endif + +runHook :: Hook -> Repo -> IO Bool +runHook h r = do + let f = hookFile h r + (c, ps) <- findShellCommand f + boolSystem c ps diff --git a/Git/Index.hs b/Git/Index.hs new file mode 100644 index 0000000000..91f46f9917 --- /dev/null +++ b/Git/Index.hs @@ -0,0 +1,76 @@ +{- git index file stuff + - + - Copyright 2011-2018 Joey Hess + - + - Licensed under the GNU GPL version 3 or higher. + -} + +module Git.Index where + +import Common +import Git +import Utility.Env +import Utility.Env.Set + +indexEnv :: String +indexEnv = "GIT_INDEX_FILE" + +{- Gets value to set GIT_INDEX_FILE to. Input should be absolute path, + - or relative to the CWD. + - + - When relative, GIT_INDEX_FILE is interpreted by git as being + - relative to the top of the work tree of the git repository, + - not to the CWD. Worse, other environment variables (GIT_WORK_TREE) + - or git options (--work-tree) or configuration (core.worktree) + - can change what the relative path is interpreted relative to. + - + - So, an absolute path is the only safe option for this to return. + -} +indexEnvVal :: FilePath -> IO String +indexEnvVal = absPath + +{- Forces git to use the specified index file. + - + - Returns an action that will reset back to the default + - index file. + - + - Warning: Not thread safe. + -} +override :: FilePath -> Repo -> IO (IO ()) +override index _r = do + res <- getEnv var + val <- indexEnvVal index + setEnv var val True + return $ reset res + where + var = "GIT_INDEX_FILE" + reset (Just v) = setEnv indexEnv v True + reset _ = unsetEnv var + +{- The normal index file. Does not check GIT_INDEX_FILE. -} +indexFile :: Repo -> FilePath +indexFile r = localGitDir r "index" + +{- The index file git will currently use, checking GIT_INDEX_FILE. -} +currentIndexFile :: Repo -> IO FilePath +currentIndexFile r = fromMaybe (indexFile r) <$> getEnv indexEnv + +{- Git locks the index by creating this file. -} +indexFileLock :: FilePath -> FilePath +indexFileLock f = f ++ ".lock" + +{- When the pre-commit hook is run, and git commit has been run with + - a file or files specified to commit, rather than committing the staged + - index, git provides the pre-commit hook with a "false index file". + - + - Changes made to this index will influence the commit, but won't + - affect the real index file. + - + - This detects when we're in this situation, using a heuristic, which + - might be broken by changes to git. Any use of this should have a test + - case to make sure it works. + -} +haveFalseIndex :: IO Bool +haveFalseIndex = maybe (False) check <$> getEnv indexEnv + where + check f = "next-index" `isPrefixOf` takeFileName f diff --git a/Git/LockFile.hs b/Git/LockFile.hs new file mode 100644 index 0000000000..e3d59009e1 --- /dev/null +++ b/Git/LockFile.hs @@ -0,0 +1,79 @@ +{- git lock files + - + - Copyright 2016 Joey Hess + - + - Licensed under the GNU GPL version 3 or higher. + -} + +{-# LANGUAGE CPP #-} + +module Git.LockFile where + +import Common + +#ifndef mingw32_HOST_OS +import System.Posix.Types +import System.Posix.IO +#else +import System.Win32.Types +import System.Win32.File +#endif + +#ifndef mingw32_HOST_OS +data LockHandle = LockHandle FilePath Fd +#else +data LockHandle = LockHandle FilePath HANDLE +#endif + +{- Uses the same exclusive locking that git does. + - Throws an IO exception if the file is already locked. + - + - Note that git's locking method suffers from the problem that + - a dangling lock can be left if a process is terminated at the wrong + - time. + -} +openLock :: FilePath -> IO LockHandle +openLock lck = openLock' lck `catchNonAsync` lckerr + where + lckerr e = do + -- Same error message displayed by git. + whenM (doesFileExist lck) $ + hPutStrLn stderr $ unlines + [ "fatal: Unable to create '" ++ lck ++ "': " ++ show e + , "" + , "If no other git process is currently running, this probably means a" + , "git process crashed in this repository earlier. Make sure no other git" + , "process is running and remove the file manually to continue." + ] + throwM e + +openLock' :: FilePath -> IO LockHandle +openLock' lck = do +#ifndef mingw32_HOST_OS + -- On unix, git simply uses O_EXCL + h <- openFd lck ReadWrite (Just 0O666) + (defaultFileFlags { exclusive = True }) + setFdOption h CloseOnExec True +#else + -- It's not entirely clear how git manages locking on Windows, + -- since it's buried in the portability layer, and different + -- versions of git for windows use different portability layers. + -- But, we can be fairly sure that holding the lock file open on + -- windows is enough to prevent another process from opening it. + -- + -- So, all that's needed is a way to open the file, that fails + -- if the file already exists. Using CreateFile with CREATE_NEW + -- accomplishes that. + h <- createFile lck gENERIC_WRITE fILE_SHARE_NONE Nothing + cREATE_NEW fILE_ATTRIBUTE_NORMAL Nothing +#endif + return (LockHandle lck h) + +closeLock :: LockHandle -> IO () +closeLock (LockHandle lck h) = do +#ifndef mingw32_HOST_OS + closeFd h +#else + closeHandle h +#endif + removeFile lck diff --git a/Git/LsFiles.hs b/Git/LsFiles.hs new file mode 100644 index 0000000000..20b8245498 --- /dev/null +++ b/Git/LsFiles.hs @@ -0,0 +1,268 @@ +{- git ls-files interface + - + - Copyright 2010-2018 Joey Hess + - + - Licensed under the GNU GPL version 3 or higher. + -} + +module Git.LsFiles ( + inRepo, + notInRepo, + notInRepoIncludingEmptyDirectories, + allFiles, + deleted, + modified, + modifiedOthers, + staged, + stagedNotDeleted, + stagedOthersDetails, + stagedDetails, + typeChanged, + typeChangedStaged, + Conflicting(..), + Unmerged(..), + unmerged, + StagedDetails, +) where + +import Common +import Git +import Git.Command +import Git.Types +import Git.Sha + +import Numeric +import System.Posix.Types + +{- Scans for files that are checked into git at the specified locations. -} +inRepo :: [FilePath] -> Repo -> IO ([FilePath], IO Bool) +inRepo l = pipeNullSplit $ + Param "ls-files" : + Param "--cached" : + Param "-z" : + Param "--" : + map File l + +{- Scans for files at the specified locations that are not checked into git. -} +notInRepo :: Bool -> [FilePath] -> Repo -> IO ([FilePath], IO Bool) +notInRepo = notInRepo' [] + +notInRepo' :: [CommandParam] -> Bool -> [FilePath] -> Repo -> IO ([FilePath], IO Bool) +notInRepo' ps include_ignored l repo = pipeNullSplit params repo + where + params = concat + [ [ Param "ls-files", Param "--others"] + , ps + , exclude + , [ Param "-z", Param "--" ] + , map File l + ] + exclude + | include_ignored = [] + | otherwise = [Param "--exclude-standard"] + +{- Scans for files at the specified locations that are not checked into + - git. Empty directories are included in the result. -} +notInRepoIncludingEmptyDirectories :: Bool -> [FilePath] -> Repo -> IO ([FilePath], IO Bool) +notInRepoIncludingEmptyDirectories = notInRepo' [Param "--directory"] + +{- Finds all files in the specified locations, whether checked into git or + - not. -} +allFiles :: [FilePath] -> Repo -> IO ([FilePath], IO Bool) +allFiles l = pipeNullSplit $ + Param "ls-files" : + Param "--cached" : + Param "--others" : + Param "-z" : + Param "--" : + map File l + +{- Returns a list of files in the specified locations that have been + - deleted. -} +deleted :: [FilePath] -> Repo -> IO ([FilePath], IO Bool) +deleted l repo = pipeNullSplit params repo + where + params = + Param "ls-files" : + Param "--deleted" : + Param "-z" : + Param "--" : + map File l + +{- Returns a list of files in the specified locations that have been + - modified. -} +modified :: [FilePath] -> Repo -> IO ([FilePath], IO Bool) +modified l repo = pipeNullSplit params repo + where + params = + Param "ls-files" : + Param "--modified" : + Param "-z" : + Param "--" : + map File l + +{- Files that have been modified or are not checked into git (and are not + - ignored). -} +modifiedOthers :: [FilePath] -> Repo -> IO ([FilePath], IO Bool) +modifiedOthers l repo = pipeNullSplit params repo + where + params = + Param "ls-files" : + Param "--modified" : + Param "--others" : + Param "--exclude-standard" : + Param "-z" : + Param "--" : + map File l + +{- Returns a list of all files that are staged for commit. -} +staged :: [FilePath] -> Repo -> IO ([FilePath], IO Bool) +staged = staged' [] + +{- Returns a list of the files, staged for commit, that are being added, + - moved, or changed (but not deleted), from the specified locations. -} +stagedNotDeleted :: [FilePath] -> Repo -> IO ([FilePath], IO Bool) +stagedNotDeleted = staged' [Param "--diff-filter=ACMRT"] + +staged' :: [CommandParam] -> [FilePath] -> Repo -> IO ([FilePath], IO Bool) +staged' ps l = pipeNullSplit $ prefix ++ ps ++ suffix + where + prefix = [Param "diff", Param "--cached", Param "--name-only", Param "-z"] + suffix = Param "--" : map File l + +type StagedDetails = (FilePath, Maybe Sha, Maybe FileMode) + +{- Returns details about files that are staged in the index, + - as well as files not yet in git. Skips ignored files. -} +stagedOthersDetails :: [FilePath] -> Repo -> IO ([StagedDetails], IO Bool) +stagedOthersDetails = stagedDetails' [Param "--others", Param "--exclude-standard"] + +{- Returns details about all files that are staged in the index. -} +stagedDetails :: [FilePath] -> Repo -> IO ([StagedDetails], IO Bool) +stagedDetails = stagedDetails' [] + +{- Gets details about staged files, including the Sha of their staged + - contents. -} +stagedDetails' :: [CommandParam] -> [FilePath] -> Repo -> IO ([StagedDetails], IO Bool) +stagedDetails' ps l repo = do + (ls, cleanup) <- pipeNullSplit params repo + return (map parse ls, cleanup) + where + params = Param "ls-files" : Param "--stage" : Param "-z" : ps ++ + Param "--" : map File l + parse s + | null file = (s, Nothing, Nothing) + | otherwise = (file, extractSha $ take shaSize rest, readmode mode) + where + (metadata, file) = separate (== '\t') s + (mode, rest) = separate (== ' ') metadata + readmode = fst <$$> headMaybe . readOct + +{- Returns a list of the files in the specified locations that are staged + - for commit, and whose type has changed. -} +typeChangedStaged :: [FilePath] -> Repo -> IO ([FilePath], IO Bool) +typeChangedStaged = typeChanged' [Param "--cached"] + +{- Returns a list of the files in the specified locations whose type has + - changed. Files only staged for commit will not be included. -} +typeChanged :: [FilePath] -> Repo -> IO ([FilePath], IO Bool) +typeChanged = typeChanged' [] + +typeChanged' :: [CommandParam] -> [FilePath] -> Repo -> IO ([FilePath], IO Bool) +typeChanged' ps l repo = do + (fs, cleanup) <- pipeNullSplit (prefix ++ ps ++ suffix) repo + -- git diff returns filenames relative to the top of the git repo; + -- convert to filenames relative to the cwd, like git ls-files. + top <- absPath (repoPath repo) + currdir <- getCurrentDirectory + return (map (\f -> relPathDirToFileAbs currdir $ top f) fs, cleanup) + where + prefix = + [ Param "diff" + , Param "--name-only" + , Param "--diff-filter=T" + , Param "-z" + ] + suffix = Param "--" : (if null l then [File "."] else map File l) + +{- A item in conflict has two possible values. + - Either can be Nothing, when that side deleted the file. -} +data Conflicting v = Conflicting + { valUs :: Maybe v + , valThem :: Maybe v + } deriving (Show) + +data Unmerged = Unmerged + { unmergedFile :: FilePath + , unmergedTreeItemType :: Conflicting TreeItemType + , unmergedSha :: Conflicting Sha + } + +{- Returns a list of the files in the specified locations that have + - unresolved merge conflicts. + - + - ls-files outputs multiple lines per conflicting file, each with its own + - stage number: + - 1 = old version, can be ignored + - 2 = us + - 3 = them + - If a line is omitted, that side removed the file. + -} +unmerged :: [FilePath] -> Repo -> IO ([Unmerged], IO Bool) +unmerged l repo = do + (fs, cleanup) <- pipeNullSplit params repo + return (reduceUnmerged [] $ catMaybes $ map parseUnmerged fs, cleanup) + where + params = + Param "ls-files" : + Param "--unmerged" : + Param "-z" : + Param "--" : + map File l + +data InternalUnmerged = InternalUnmerged + { isus :: Bool + , ifile :: FilePath + , itreeitemtype :: Maybe TreeItemType + , isha :: Maybe Sha + } + +parseUnmerged :: String -> Maybe InternalUnmerged +parseUnmerged s + | null file = Nothing + | otherwise = case words metadata of + (rawtreeitemtype:rawsha:rawstage:_) -> do + stage <- readish rawstage :: Maybe Int + if stage /= 2 && stage /= 3 + then Nothing + else do + treeitemtype <- readTreeItemType rawtreeitemtype + sha <- extractSha rawsha + return $ InternalUnmerged (stage == 2) file + (Just treeitemtype) (Just sha) + _ -> Nothing + where + (metadata, file) = separate (== '\t') s + +reduceUnmerged :: [Unmerged] -> [InternalUnmerged] -> [Unmerged] +reduceUnmerged c [] = c +reduceUnmerged c (i:is) = reduceUnmerged (new:c) rest + where + (rest, sibi) = findsib i is + (treeitemtypeA, treeitemtypeB, shaA, shaB) + | isus i = (itreeitemtype i, itreeitemtype sibi, isha i, isha sibi) + | otherwise = (itreeitemtype sibi, itreeitemtype i, isha sibi, isha i) + new = Unmerged + { unmergedFile = ifile i + , unmergedTreeItemType = Conflicting treeitemtypeA treeitemtypeB + , unmergedSha = Conflicting shaA shaB + } + findsib templatei [] = ([], removed templatei) + findsib templatei (l:ls) + | ifile l == ifile templatei = (ls, l) + | otherwise = (l:ls, removed templatei) + removed templatei = templatei + { isus = not (isus templatei) + , itreeitemtype = Nothing + , isha = Nothing + } diff --git a/Git/LsTree.hs b/Git/LsTree.hs new file mode 100644 index 0000000000..225f2ce138 --- /dev/null +++ b/Git/LsTree.hs @@ -0,0 +1,87 @@ +{- git ls-tree interface + - + - Copyright 2011-2016 Joey Hess + - + - Licensed under the GNU GPL version 3 or higher. + -} + +{-# LANGUAGE BangPatterns #-} + +module Git.LsTree ( + TreeItem(..), + lsTree, + lsTree', + lsTreeParams, + lsTreeFiles, + parseLsTree, +) where + +import Common +import Git +import Git.Command +import Git.Sha +import Git.FilePath +import qualified Git.Filename + +import Numeric +import Data.Char +import System.Posix.Types + +data TreeItem = TreeItem + { mode :: FileMode + , typeobj :: String + , sha :: Ref + , file :: TopFilePath + } deriving Show + +{- Lists the complete contents of a tree, recursing into sub-trees, + - with lazy output. -} +lsTree :: Ref -> Repo -> IO ([TreeItem], IO Bool) +lsTree = lsTree' [] + +lsTree' :: [CommandParam] -> Ref -> Repo -> IO ([TreeItem], IO Bool) +lsTree' ps t repo = do + (l, cleanup) <- pipeNullSplit (lsTreeParams t ps) repo + return (map parseLsTree l, cleanup) + +lsTreeParams :: Ref -> [CommandParam] -> [CommandParam] +lsTreeParams r ps = + [ Param "ls-tree" + , Param "--full-tree" + , Param "-z" + , Param "-r" + ] ++ ps ++ + [ Param "--" + , File $ fromRef r + ] + +{- Lists specified files in a tree. -} +lsTreeFiles :: Ref -> [FilePath] -> Repo -> IO [TreeItem] +lsTreeFiles t fs repo = map parseLsTree <$> pipeNullSplitStrict ps repo + where + ps = + [ Param "ls-tree" + , Param "--full-tree" + , Param "-z" + , Param "--" + , File $ fromRef t + ] ++ map File fs + +{- Parses a line of ls-tree output, in format: + - mode SP type SP sha TAB file + - + - (The --long format is not currently supported.) -} +parseLsTree :: String -> TreeItem +parseLsTree l = TreeItem + { mode = smode + , typeobj = t + , sha = Ref s + , file = sfile + } + where + (m, past_m) = splitAt 7 l -- mode is 6 bytes + (!t, past_t) = separate isSpace past_m + (!s, past_s) = splitAt shaSize past_t + !f = drop 1 past_s + !smode = fst $ Prelude.head $ readOct m + !sfile = asTopFilePath $ Git.Filename.decode f diff --git a/Git/Merge.hs b/Git/Merge.hs new file mode 100644 index 0000000000..a0767e462e --- /dev/null +++ b/Git/Merge.hs @@ -0,0 +1,66 @@ +{- git merging + - + - Copyright 2012-2016 Joey Hess + - + - Licensed under the GNU GPL version 3 or higher. + -} + +module Git.Merge ( + MergeConfig(..), + CommitMode(..), + merge, + merge', + stageMerge, +) where + +import Common +import Git +import Git.Command +import qualified Git.BuildVersion +import qualified Git.Version +import Git.Branch (CommitMode(..)) + +data MergeConfig + = MergeNonInteractive + -- ^ avoids recent git's interactive merge + | MergeUnrelatedHistories + -- ^ avoids recent git's prevention of merging unrelated histories + deriving (Eq) + +merge :: Ref -> [MergeConfig] -> CommitMode -> Repo -> IO Bool +merge = merge' [] + +merge' :: [CommandParam] -> Ref -> [MergeConfig] -> CommitMode -> Repo -> IO Bool +merge' extraparams branch mergeconfig commitmode r + | MergeNonInteractive `notElem` mergeconfig || Git.BuildVersion.older "1.7.7.6" = + go [Param $ fromRef branch] + | otherwise = go [Param "--no-edit", Param $ fromRef branch] + where + go ps = merge'' (sp ++ [Param "merge"] ++ ps ++ extraparams) mergeconfig r + sp + | commitmode == AutomaticCommit = + [Param "-c", Param "commit.gpgsign=false"] + | otherwise = [] + +merge'' :: [CommandParam] -> [MergeConfig] -> Repo -> IO Bool +merge'' ps mergeconfig r + | MergeUnrelatedHistories `elem` mergeconfig = + ifM (Git.Version.older "2.9.0") + ( go ps + , go (ps ++ [Param "--allow-unrelated-histories"]) + ) + | otherwise = go ps + where + go ps' = runBool ps' r + +{- Stage the merge into the index, but do not commit it.-} +stageMerge :: Ref -> [MergeConfig] -> Repo -> IO Bool +stageMerge branch = merge'' + [ Param "merge" + , Param "--quiet" + , Param "--no-commit" + -- Without this, a fast-forward merge is done, since it involves no + -- commit. + , Param "--no-ff" + , Param $ fromRef branch + ] diff --git a/Git/Objects.hs b/Git/Objects.hs new file mode 100644 index 0000000000..bda220b5e7 --- /dev/null +++ b/Git/Objects.hs @@ -0,0 +1,49 @@ +{- .git/objects + - + - Copyright 2013 Joey Hess + - + - Licensed under the GNU GPL version 3 or higher. + -} + +module Git.Objects where + +import Common +import Git +import Git.Sha + +objectsDir :: Repo -> FilePath +objectsDir r = localGitDir r "objects" + +packDir :: Repo -> FilePath +packDir r = objectsDir r "pack" + +packIdxFile :: FilePath -> FilePath +packIdxFile = flip replaceExtension "idx" + +listPackFiles :: Repo -> IO [FilePath] +listPackFiles r = filter (".pack" `isSuffixOf`) + <$> catchDefaultIO [] (dirContents $ packDir r) + +listLooseObjectShas :: Repo -> IO [Sha] +listLooseObjectShas r = catchDefaultIO [] $ + mapMaybe (extractSha . concat . reverse . take 2 . reverse . splitDirectories) + <$> dirContentsRecursiveSkipping (== "pack") True (objectsDir r) + +looseObjectFile :: Repo -> Sha -> FilePath +looseObjectFile r sha = objectsDir r prefix rest + where + (prefix, rest) = splitAt 2 (fromRef sha) + +listAlternates :: Repo -> IO [FilePath] +listAlternates r = catchDefaultIO [] (lines <$> readFile alternatesfile) + where + alternatesfile = objectsDir r "info" "alternates" + +{- A repository recently cloned with --shared will have one or more + - alternates listed, and contain no loose objects or packs. -} +isSharedClone :: Repo -> IO Bool +isSharedClone r = allM id + [ not . null <$> listAlternates r + , null <$> listLooseObjectShas r + , null <$> listPackFiles r + ] diff --git a/Git/Queue.hs b/Git/Queue.hs new file mode 100644 index 0000000000..3b855ae2b1 --- /dev/null +++ b/Git/Queue.hs @@ -0,0 +1,210 @@ +{- git repository command queue + - + - Copyright 2010-2018 Joey Hess + - + - Licensed under the GNU GPL version 3 or higher. + -} + +{-# LANGUAGE CPP, BangPatterns #-} + +module Git.Queue ( + Queue, + new, + addCommand, + addUpdateIndex, + addInternalAction, + InternalActionRunner(..), + size, + full, + flush, + merge, +) where + +import Utility.SafeCommand +import Common +import Git +import Git.Command +import qualified Git.UpdateIndex + +import qualified Data.Map.Strict as M + +{- Queable actions that can be performed in a git repository. -} +data Action + {- Updating the index file, using a list of streamers that can + - be added to as the queue grows. -} + = UpdateIndexAction [Git.UpdateIndex.Streamer] -- in reverse order + {- A git command to run, on a list of files that can be added to + - as the queue grows. -} + | CommandAction + { getSubcommand :: String + , getParams :: [CommandParam] + , getFiles :: [CommandParam] + } + {- An internal action to run, on a list of files that can be added + - to as the queue grows. -} + | InternalAction + { getRunner :: InternalActionRunner + , getInternalFiles :: [(FilePath, IO Bool)] + } + +{- The String must be unique for each internal action. -} +data InternalActionRunner = InternalActionRunner String (Repo -> [(FilePath, IO Bool)] -> IO ()) + +instance Eq InternalActionRunner where + InternalActionRunner s1 _ == InternalActionRunner s2 _ = s1 == s2 + +{- A key that can uniquely represent an action in a Map. -} +data ActionKey = UpdateIndexActionKey | CommandActionKey String | InternalActionKey String + deriving (Eq, Ord) + +actionKey :: Action -> ActionKey +actionKey (UpdateIndexAction _) = UpdateIndexActionKey +actionKey CommandAction { getSubcommand = s } = CommandActionKey s +actionKey InternalAction { getRunner = InternalActionRunner s _ } = InternalActionKey s + +{- A queue of actions to perform (in any order) on a git repository, + - with lists of files to perform them on. This allows coalescing + - similar git commands. -} +data Queue = Queue + { size :: Int + , _limit :: Int + , items :: M.Map ActionKey Action + } + +{- A recommended maximum size for the queue, after which it should be + - run. + - + - 10240 is semi-arbitrary. If we assume git filenames are between 10 and + - 255 characters long, then the queue will build up between 100kb and + - 2550kb long commands. The max command line length on linux is somewhere + - above 20k, so this is a fairly good balance -- the queue will buffer + - only a few megabytes of stuff and a minimal number of commands will be + - run by xargs. -} +defaultLimit :: Int +defaultLimit = 10240 + +{- Constructor for empty queue. -} +new :: Maybe Int -> Queue +new lim = Queue 0 (fromMaybe defaultLimit lim) M.empty + +{- Adds an git command to the queue. + - + - Git commands with the same subcommand but different parameters are + - assumed to be equivilant enough to perform in any order with the same + - result. + -} +addCommand :: String -> [CommandParam] -> [FilePath] -> Queue -> Repo -> IO Queue +addCommand subcommand params files q repo = + updateQueue action different (length files) q repo + where + action = CommandAction + { getSubcommand = subcommand + , getParams = params + , getFiles = map File files + } + + different (CommandAction { getSubcommand = s }) = s /= subcommand + different _ = True + +{- Adds an internal action to the queue. -} +addInternalAction :: InternalActionRunner -> [(FilePath, IO Bool)] -> Queue -> Repo -> IO Queue +addInternalAction runner files q repo = + updateQueue action different (length files) q repo + where + action = InternalAction + { getRunner = runner + , getInternalFiles = files + } + + different (InternalAction { getRunner = r }) = r /= runner + different _ = True + +{- Adds an update-index streamer to the queue. -} +addUpdateIndex :: Git.UpdateIndex.Streamer -> Queue -> Repo -> IO Queue +addUpdateIndex streamer q repo = + updateQueue action different 1 q repo + where + -- the list is built in reverse order + action = UpdateIndexAction [streamer] + + different (UpdateIndexAction _) = False + different _ = True + +{- Updates or adds an action in the queue. If the queue already contains a + - different action, it will be flushed; this is to ensure that conflicting + - actions, like add and rm, are run in the right order.-} +updateQueue :: Action -> (Action -> Bool) -> Int -> Queue -> Repo -> IO Queue +updateQueue !action different sizeincrease q repo + | null (filter different (M.elems (items q))) = return $ go q + | otherwise = go <$> flush q repo + where + go q' = newq + where + !newq = q' + { size = newsize + , items = newitems + } + !newsize = size q' + sizeincrease + !newitems = M.insertWith combineNewOld (actionKey action) action (items q') + +{- The new value comes first. It probably has a smaller list of files than + - the old value. So, the list append of the new value first is more + - efficient. -} +combineNewOld :: Action -> Action -> Action +combineNewOld (CommandAction _sc1 _ps1 fs1) (CommandAction sc2 ps2 fs2) = + CommandAction sc2 ps2 (fs1++fs2) +combineNewOld (UpdateIndexAction s1) (UpdateIndexAction s2) = + UpdateIndexAction (s1++s2) +combineNewOld (InternalAction _r1 fs1) (InternalAction r2 fs2) = + InternalAction r2 (fs1++fs2) +combineNewOld anew _aold = anew + +{- Merges the contents of the second queue into the first. + - This should only be used when the two queues are known to contain + - non-conflicting actions. -} +merge :: Queue -> Queue -> Queue +merge origq newq = origq + { size = size origq + size newq + , items = M.unionWith combineNewOld (items newq) (items origq) + } + +{- Is a queue large enough that it should be flushed? -} +full :: Queue -> Bool +full (Queue cur lim _) = cur >= lim + +{- Runs a queue on a git repository. -} +flush :: Queue -> Repo -> IO Queue +flush (Queue _ lim m) repo = do + forM_ (M.elems m) $ runAction repo + return $ Queue 0 lim M.empty + +{- Runs an Action on a list of files in a git repository. + - + - Complicated by commandline length limits. + - + - Intentionally runs the command even if the list of files is empty; + - this allows queueing commands that do not need a list of files. -} +runAction :: Repo -> Action -> IO () +runAction repo (UpdateIndexAction streamers) = + -- list is stored in reverse order + Git.UpdateIndex.streamUpdateIndex repo $ reverse streamers +runAction repo action@(CommandAction {}) = do +#ifndef mingw32_HOST_OS + let p = (proc "xargs" $ "-0":"git":toCommand gitparams) { env = gitEnv repo } + withHandle StdinHandle createProcessSuccess p $ \h -> do + hPutStr h $ intercalate "\0" $ toCommand $ getFiles action + hClose h +#else + -- Using xargs on Windows is problematic, so just run the command + -- once per file (not as efficient.) + if null (getFiles action) + then void $ boolSystemEnv "git" gitparams (gitEnv repo) + else forM_ (getFiles action) $ \f -> + void $ boolSystemEnv "git" (gitparams ++ [f]) (gitEnv repo) +#endif + where + gitparams = gitCommandLine + (Param (getSubcommand action):getParams action) repo +runAction repo action@(InternalAction {}) = + let InternalActionRunner _ runner = getRunner action + in runner repo (getInternalFiles action) diff --git a/Git/Ref.hs b/Git/Ref.hs new file mode 100644 index 0000000000..1986db603c --- /dev/null +++ b/Git/Ref.hs @@ -0,0 +1,172 @@ +{- git ref stuff + - + - Copyright 2011-2013 Joey Hess + - + - Licensed under the GNU GPL version 3 or higher. + -} + +module Git.Ref where + +import Common +import Git +import Git.Command +import Git.Sha +import Git.Types + +import Data.Char (chr) + +headRef :: Ref +headRef = Ref "HEAD" + +headFile :: Repo -> FilePath +headFile r = localGitDir r "HEAD" + +setHeadRef :: Ref -> Repo -> IO () +setHeadRef ref r = writeFile (headFile r) ("ref: " ++ fromRef ref) + +{- Converts a fully qualified git ref into a user-visible string. -} +describe :: Ref -> String +describe = fromRef . base + +{- Often git refs are fully qualified + - (eg refs/heads/master or refs/remotes/origin/master). + - Converts such a fully qualified ref into a base ref + - (eg: master or origin/master). -} +base :: Ref -> Ref +base = Ref . remove "refs/heads/" . remove "refs/remotes/" . fromRef + where + remove prefix s + | prefix `isPrefixOf` s = drop (length prefix) s + | otherwise = s + +{- Given a directory such as "refs/remotes/origin", and a ref such as + - refs/heads/master, yields a version of that ref under the directory, + - such as refs/remotes/origin/master. -} +underBase :: String -> Ref -> Ref +underBase dir r = Ref $ dir ++ "/" ++ fromRef (base r) + +{- Convert a branch such as "master" into a fully qualified ref. -} +branchRef :: Branch -> Ref +branchRef = underBase "refs/heads" + +{- A Ref that can be used to refer to a file in the repository, as staged + - in the index. + - + - Prefixing the file with ./ makes this work even if in a subdirectory + - of a repo. + -} +fileRef :: FilePath -> Ref +fileRef f = Ref $ ":./" ++ f + +{- Converts a Ref to refer to the content of the Ref on a given date. -} +dateRef :: Ref -> RefDate -> Ref +dateRef (Ref r) (RefDate d) = Ref $ r ++ "@" ++ d + +{- A Ref that can be used to refer to a file in the repository as it + - appears in a given Ref. -} +fileFromRef :: Ref -> FilePath -> Ref +fileFromRef (Ref r) f = let (Ref fr) = fileRef f in Ref (r ++ fr) + +{- Checks if a ref exists. -} +exists :: Ref -> Repo -> IO Bool +exists ref = runBool + [Param "show-ref", Param "--verify", Param "-q", Param $ fromRef ref] + +{- The file used to record a ref. (Git also stores some refs in a + - packed-refs file.) -} +file :: Ref -> Repo -> FilePath +file ref repo = localGitDir repo fromRef ref + +{- Checks if HEAD exists. It generally will, except for in a repository + - that was just created. -} +headExists :: Repo -> IO Bool +headExists repo = do + ls <- lines <$> pipeReadStrict [Param "show-ref", Param "--head"] repo + return $ any (" HEAD" `isSuffixOf`) ls + +{- Get the sha of a fully qualified git ref, if it exists. -} +sha :: Branch -> Repo -> IO (Maybe Sha) +sha branch repo = process <$> showref repo + where + showref = pipeReadStrict [Param "show-ref", + Param "--hash", -- get the hash + Param $ fromRef branch] + process [] = Nothing + process s = Just $ Ref $ firstLine s + +headSha :: Repo -> IO (Maybe Sha) +headSha = sha headRef + +{- List of (shas, branches) matching a given ref or refs. -} +matching :: [Ref] -> Repo -> IO [(Sha, Branch)] +matching refs repo = matching' (map fromRef refs) repo + +{- Includes HEAD in the output, if asked for it. -} +matchingWithHEAD :: [Ref] -> Repo -> IO [(Sha, Branch)] +matchingWithHEAD refs repo = matching' ("--head" : map fromRef refs) repo + +{- List of (shas, branches) matching a given ref spec. -} +matching' :: [String] -> Repo -> IO [(Sha, Branch)] +matching' ps repo = map gen . lines <$> + pipeReadStrict (Param "show-ref" : map Param ps) repo + where + gen l = let (r, b) = separate (== ' ') l + in (Ref r, Ref b) + +{- List of (shas, branches) matching a given ref. + - Duplicate shas are filtered out. -} +matchingUniq :: [Ref] -> Repo -> IO [(Sha, Branch)] +matchingUniq refs repo = nubBy uniqref <$> matching refs repo + where + uniqref (a, _) (b, _) = a == b + +{- List of all refs. -} +list :: Repo -> IO [(Sha, Ref)] +list = matching' [] + +{- Deletes a ref. This can delete refs that are not branches, + - which git branch --delete refuses to delete. -} +delete :: Sha -> Ref -> Repo -> IO () +delete oldvalue ref = run + [ Param "update-ref" + , Param "-d" + , Param $ fromRef ref + , Param $ fromRef oldvalue + ] + +{- Gets the sha of the tree a ref uses. -} +tree :: Ref -> Repo -> IO (Maybe Sha) +tree (Ref ref) = extractSha <$$> pipeReadStrict + [ Param "rev-parse", Param ref' ] + where + ref' = if ":" `isInfixOf` ref + then ref + -- de-reference commit objects to the tree + else ref ++ ":" + +{- Checks if a String is a legal git ref name. + - + - The rules for this are complex; see git-check-ref-format(1) -} +legal :: Bool -> String -> Bool +legal allowonelevel s = all (== False) illegal + where + illegal = + [ any ("." `isPrefixOf`) pathbits + , any (".lock" `isSuffixOf`) pathbits + , not allowonelevel && length pathbits < 2 + , contains ".." + , any (\c -> contains [c]) illegalchars + , begins "/" + , ends "/" + , contains "//" + , ends "." + , contains "@{" + , null s + ] + contains v = v `isInfixOf` s + ends v = v `isSuffixOf` s + begins v = v `isPrefixOf` s + + pathbits = splitc '/' s + illegalchars = " ~^:?*[\\" ++ controlchars + controlchars = chr 0o177 : [chr 0 .. chr (0o40-1)] diff --git a/Git/RefLog.hs b/Git/RefLog.hs new file mode 100644 index 0000000000..57f35e9183 --- /dev/null +++ b/Git/RefLog.hs @@ -0,0 +1,30 @@ +{- git reflog interface + - + - Copyright 2013 Joey Hess + - + - Licensed under the GNU GPL version 3 or higher. + -} + +module Git.RefLog where + +import Common +import Git +import Git.Command +import Git.Sha + +{- Gets the reflog for a given branch. -} +get :: Branch -> Repo -> IO [Sha] +get b = getMulti [b] + +{- Gets reflogs for multiple branches. -} +getMulti :: [Branch] -> Repo -> IO [Sha] +getMulti bs = get' (map (Param . fromRef) bs) + +get' :: [CommandParam] -> Repo -> IO [Sha] +get' ps = mapMaybe extractSha . lines <$$> pipeReadStrict ps' + where + ps' = catMaybes + [ Just $ Param "log" + , Just $ Param "-g" + , Just $ Param "--format=%H" + ] ++ ps diff --git a/Git/Remote.hs b/Git/Remote.hs new file mode 100644 index 0000000000..ce741a0d0a --- /dev/null +++ b/Git/Remote.hs @@ -0,0 +1,117 @@ +{- git remote stuff + - + - Copyright 2012 Joey Hess + - + - Licensed under the GNU GPL version 3 or higher. + -} + +{-# LANGUAGE CPP #-} + +module Git.Remote where + +import Common +import Git +import Git.Types + +import Data.Char +import qualified Data.Map as M +import Network.URI +#ifdef mingw32_HOST_OS +import Git.FilePath +#endif + +{- Is a git config key one that specifies the location of a remote? -} +isRemoteKey :: String -> Bool +isRemoteKey k = "remote." `isPrefixOf` k && ".url" `isSuffixOf` k + +{- Get a remote's name from the config key that specifies its location. -} +remoteKeyToRemoteName :: String -> RemoteName +remoteKeyToRemoteName k = intercalate "." $ + reverse $ drop 1 $ reverse $ drop 1 $ splitc '.' k + +{- Construct a legal git remote name out of an arbitrary input string. + - + - There seems to be no formal definition of this in the git source, + - just some ad-hoc checks, and some other things that fail with certian + - types of names (like ones starting with '-'). + -} +makeLegalName :: String -> RemoteName +makeLegalName s = case filter legal $ replace "/" "_" s of + -- it can't be empty + [] -> "unnamed" + -- it can't start with / or - or . + '.':s' -> makeLegalName s' + '/':s' -> makeLegalName s' + '-':s' -> makeLegalName s' + s' -> s' + where + {- Only alphanumerics, and a few common bits of punctuation common + - in hostnames. -} + legal '_' = True + legal '.' = True + legal c = isAlphaNum c + +data RemoteLocation = RemoteUrl String | RemotePath FilePath + +remoteLocationIsUrl :: RemoteLocation -> Bool +remoteLocationIsUrl (RemoteUrl _) = True +remoteLocationIsUrl _ = False + +remoteLocationIsSshUrl :: RemoteLocation -> Bool +remoteLocationIsSshUrl (RemoteUrl u) = "ssh://" `isPrefixOf` u +remoteLocationIsSshUrl _ = False + +{- Determines if a given remote location is an url, or a local + - path. Takes the repository's insteadOf configuration into account. -} +parseRemoteLocation :: String -> Repo -> RemoteLocation +parseRemoteLocation s repo = ret $ calcloc s + where + ret v +#ifdef mingw32_HOST_OS + | dosstyle v = RemotePath (dospath v) +#endif + | scpstyle v = RemoteUrl (scptourl v) + | urlstyle v = RemoteUrl v + | otherwise = RemotePath v + -- insteadof config can rewrite remote location + calcloc l + | null insteadofs = l + | otherwise = replacement ++ drop (length bestvalue) l + where + replacement = drop (length prefix) $ + take (length bestkey - length suffix) bestkey + (bestkey, bestvalue) = maximumBy longestvalue insteadofs + longestvalue (_, a) (_, b) = compare b a + insteadofs = filterconfig $ \(k, v) -> + prefix `isPrefixOf` k && + suffix `isSuffixOf` k && + v `isPrefixOf` l + filterconfig f = filter f $ + concatMap splitconfigs $ M.toList $ fullconfig repo + splitconfigs (k, vs) = map (\v -> (k, v)) vs + (prefix, suffix) = ("url." , ".insteadof") + urlstyle v = isURI v || ":" `isInfixOf` v && "//" `isInfixOf` v + -- git remotes can be written scp style -- [user@]host:dir + -- but foo::bar is a git-remote-helper location instead + scpstyle v = ":" `isInfixOf` v + && not ("//" `isInfixOf` v) + && not ("::" `isInfixOf` v) + scptourl v = "ssh://" ++ host ++ slash dir + where + (host, dir) + -- handle ipv6 address inside [] + | "[" `isPrefixOf` v = case break (== ']') v of + (h, ']':':':d) -> (h ++ "]", d) + (h, ']':d) -> (h ++ "]", d) + (h, d) -> (h, d) + | otherwise = separate (== ':') v + slash d | d == "" = "/~/" ++ d + | "/" `isPrefixOf` d = d + | "~" `isPrefixOf` d = '/':d + | otherwise = "/~/" ++ d +#ifdef mingw32_HOST_OS + -- git on Windows will write a path to .git/config with "drive:", + -- which is not to be confused with a "host:" + dosstyle = hasDrive + dospath = fromInternalGitPath +#endif diff --git a/Git/Remote/Remove.hs b/Git/Remote/Remove.hs new file mode 100644 index 0000000000..e6c13b9ea7 --- /dev/null +++ b/Git/Remote/Remove.hs @@ -0,0 +1,29 @@ +{- git remote removal + - + - Copyright 2012 Joey Hess + - + - Licensed under the GNU GPL version 3 or higher. + -} + +{-# LANGUAGE CPP #-} + +module Git.Remote.Remove where + +import Common +import Git +import Git.Types +import qualified Git.Command +import qualified Git.Version + +remove :: RemoteName -> Repo -> IO () +remove remotename r = do + old <- Git.Version.older "1.8.0" + Git.Command.run + [ Param "remote" + -- name of this subcommand changed + , Param $ + if old + then "rm" + else "remove" + , Param remotename + ] r diff --git a/Git/Repair.hs b/Git/Repair.hs new file mode 100644 index 0000000000..c51d63a5d2 --- /dev/null +++ b/Git/Repair.hs @@ -0,0 +1,617 @@ +{- git repository recovery + - + - Copyright 2013-2014 Joey Hess + - + - Licensed under the GNU GPL version 3 or higher. + -} + +module Git.Repair ( + runRepair, + runRepairOf, + removeBadBranches, + successfulRepair, + cleanCorruptObjects, + resetLocalBranches, + checkIndex, + checkIndexFast, + missingIndex, + emptyGoodCommits, + isTrackingBranch, +) where + +import Common +import Git +import Git.Command +import Git.Objects +import Git.Sha +import Git.Types +import Git.Fsck +import Git.Index +import qualified Git.Config as Config +import qualified Git.Construct as Construct +import qualified Git.LsTree as LsTree +import qualified Git.LsFiles as LsFiles +import qualified Git.Ref as Ref +import qualified Git.RefLog as RefLog +import qualified Git.UpdateIndex as UpdateIndex +import qualified Git.Branch as Branch +import Utility.Tmp.Dir +import Utility.Rsync +import Utility.FileMode +import Utility.Tuple + +import qualified Data.Set as S +import qualified Data.ByteString.Lazy as L + +{- Given a set of bad objects found by git fsck, which may not + - be complete, finds and removes all corrupt objects. -} +cleanCorruptObjects :: FsckResults -> Repo -> IO () +cleanCorruptObjects fsckresults r = do + void $ explodePacks r + mapM_ removeLoose (S.toList $ knownMissing fsckresults) + mapM_ removeBad =<< listLooseObjectShas r + where + removeLoose s = nukeFile (looseObjectFile r s) + removeBad s = do + void $ tryIO $ allowRead $ looseObjectFile r s + whenM (isMissing s r) $ + removeLoose s + +{- Explodes all pack files, and deletes them. + - + - First moves all pack files to a temp dir, before unpacking them each in + - turn. + - + - This is because unpack-objects will not unpack a pack file if it's in the + - git repo. + - + - Also, this prevents unpack-objects from possibly looking at corrupt + - pack files to see if they contain an object, while unpacking a + - non-corrupt pack file. + -} +explodePacks :: Repo -> IO Bool +explodePacks r = go =<< listPackFiles r + where + go [] = return False + go packs = withTmpDir "packs" $ \tmpdir -> do + putStrLn "Unpacking all pack files." + forM_ packs $ \packfile -> do + moveFile packfile (tmpdir takeFileName packfile) + nukeFile $ packIdxFile packfile + forM_ packs $ \packfile -> do + let tmp = tmpdir takeFileName packfile + allowRead tmp + -- May fail, if pack file is corrupt. + void $ tryIO $ + pipeWrite [Param "unpack-objects", Param "-r"] r $ \h -> + L.hPut h =<< L.readFile tmp + return True + +{- Try to retrieve a set of missing objects, from the remotes of a + - repository. Returns any that could not be retreived. + - + - If another clone of the repository exists locally, which might not be a + - remote of the repo being repaired, its path can be passed as a reference + - repository. + -} +retrieveMissingObjects :: FsckResults -> Maybe FilePath -> Repo -> IO FsckResults +retrieveMissingObjects missing referencerepo r + | not (foundBroken missing) = return missing + | otherwise = withTmpDir "tmprepo" $ \tmpdir -> do + unlessM (boolSystem "git" [Param "init", File tmpdir]) $ + error $ "failed to create temp repository in " ++ tmpdir + tmpr <- Config.read =<< Construct.fromAbsPath tmpdir + rs <- Construct.fromRemotes r + stillmissing <- pullremotes tmpr rs fetchrefstags missing + if S.null (knownMissing stillmissing) + then return stillmissing + else pullremotes tmpr rs fetchallrefs stillmissing + where + pullremotes tmpr [] fetchrefs stillmissing = case referencerepo of + Nothing -> return stillmissing + Just p -> ifM (fetchfrom p fetchrefs tmpr) + ( do + void $ explodePacks tmpr + void $ copyObjects tmpr r + case stillmissing of + FsckFailed -> return $ FsckFailed + FsckFoundMissing s t -> FsckFoundMissing + <$> findMissing (S.toList s) r + <*> pure t + , return stillmissing + ) + pullremotes tmpr (rmt:rmts) fetchrefs ms + | not (foundBroken ms) = return ms + | otherwise = do + putStrLn $ "Trying to recover missing objects from remote " ++ repoDescribe rmt ++ "." + ifM (fetchfrom (repoLocation rmt) fetchrefs tmpr) + ( do + void $ explodePacks tmpr + void $ copyObjects tmpr r + case ms of + FsckFailed -> pullremotes tmpr rmts fetchrefs ms + FsckFoundMissing s t -> do + stillmissing <- findMissing (S.toList s) r + pullremotes tmpr rmts fetchrefs (FsckFoundMissing stillmissing t) + , pullremotes tmpr rmts fetchrefs ms + ) + fetchfrom fetchurl ps fetchr = runBool ps' fetchr' + where + ps' = + [ Param "fetch" + , Param fetchurl + , Param "--force" + , Param "--update-head-ok" + , Param "--quiet" + ] ++ ps + fetchr' = fetchr { gitGlobalOpts = gitGlobalOpts fetchr ++ nogc } + nogc = [ Param "-c", Param "gc.auto=0" ] + + -- fetch refs and tags + fetchrefstags = [ Param "+refs/heads/*:refs/heads/*", Param "--tags"] + -- Fetch all available refs (more likely to fail, + -- as the remote may have refs it refuses to send). + fetchallrefs = [ Param "+*:*" ] + +{- Copies all objects from the src repository to the dest repository. + - This is done using rsync, so it copies all missing objects, and all + - objects they rely on. -} +copyObjects :: Repo -> Repo -> IO Bool +copyObjects srcr destr = rsync + [ Param "-qr" + , File $ addTrailingPathSeparator $ objectsDir srcr + , File $ addTrailingPathSeparator $ objectsDir destr + ] + +{- To deal with missing objects that cannot be recovered, resets any + - local branches to point to an old commit before the missing + - objects. Returns all branches that were changed, and deleted. + -} +resetLocalBranches :: MissingObjects -> GoodCommits -> Repo -> IO ([Branch], [Branch], GoodCommits) +resetLocalBranches missing goodcommits r = + go [] [] goodcommits =<< filter islocalbranch <$> getAllRefs r + where + islocalbranch b = "refs/heads/" `isPrefixOf` fromRef b + go changed deleted gcs [] = return (changed, deleted, gcs) + go changed deleted gcs (b:bs) = do + (mc, gcs') <- findUncorruptedCommit missing gcs b r + case mc of + Just c + | c == b -> go changed deleted gcs' bs + | otherwise -> do + reset b c + go (b:changed) deleted gcs' bs + Nothing -> do + nukeBranchRef b r + go changed (b:deleted) gcs' bs + reset b c = do + nukeBranchRef b r + void $ runBool + [ Param "branch" + , Param (fromRef $ Ref.base b) + , Param (fromRef c) + ] r + +isTrackingBranch :: Ref -> Bool +isTrackingBranch b = "refs/remotes/" `isPrefixOf` fromRef b + +{- To deal with missing objects that cannot be recovered, removes + - any branches (filtered by a predicate) that reference them + - Returns a list of all removed branches. + -} +removeBadBranches :: (Ref -> Bool) -> Repo -> IO [Branch] +removeBadBranches removablebranch r = fst <$> removeBadBranches' removablebranch S.empty emptyGoodCommits r + +removeBadBranches' :: (Ref -> Bool) -> MissingObjects -> GoodCommits -> Repo -> IO ([Branch], GoodCommits) +removeBadBranches' removablebranch missing goodcommits r = + go [] goodcommits =<< filter removablebranch <$> getAllRefs r + where + go removed gcs [] = return (removed, gcs) + go removed gcs (b:bs) = do + (ok, gcs') <- verifyCommit missing gcs b r + if ok + then go removed gcs' bs + else do + nukeBranchRef b r + go (b:removed) gcs' bs + +badBranches :: MissingObjects -> Repo -> IO [Branch] +badBranches missing r = filterM isbad =<< getAllRefs r + where + isbad b = not . fst <$> verifyCommit missing emptyGoodCommits b r + +{- Gets all refs, including ones that are corrupt. + - git show-ref does not output refs to commits that are directly + - corrupted, so it is not used. + - + - Relies on packed refs being exploded before it's called. + -} +getAllRefs :: Repo -> IO [Ref] +getAllRefs r = getAllRefs' (localGitDir r "refs") + +getAllRefs' :: FilePath -> IO [Ref] +getAllRefs' refdir = do + let topsegs = length (splitPath refdir) - 1 + let toref = Ref . joinPath . drop topsegs . splitPath + map toref <$> dirContentsRecursive refdir + +explodePackedRefsFile :: Repo -> IO () +explodePackedRefsFile r = do + let f = packedRefsFile r + whenM (doesFileExist f) $ do + rs <- mapMaybe parsePacked . lines + <$> catchDefaultIO "" (safeReadFile f) + forM_ rs makeref + nukeFile f + where + makeref (sha, ref) = do + let dest = localGitDir r fromRef ref + createDirectoryIfMissing True (parentDir dest) + unlessM (doesFileExist dest) $ + writeFile dest (fromRef sha) + +packedRefsFile :: Repo -> FilePath +packedRefsFile r = localGitDir r "packed-refs" + +parsePacked :: String -> Maybe (Sha, Ref) +parsePacked l = case words l of + (sha:ref:[]) + | isJust (extractSha sha) && Ref.legal True ref -> + Just (Ref sha, Ref ref) + _ -> Nothing + +{- git-branch -d cannot be used to remove a branch that is directly + - pointing to a corrupt commit. -} +nukeBranchRef :: Branch -> Repo -> IO () +nukeBranchRef b r = nukeFile $ localGitDir r fromRef b + +{- Finds the most recent commit to a branch that does not need any + - of the missing objects. If the input branch is good as-is, returns it. + - Otherwise, tries to traverse the commits in the branch to find one + - that is ok. That might fail, if one of them is corrupt, or if an object + - at the root of the branch is missing. Finally, looks for an old version + - of the branch from the reflog. + -} +findUncorruptedCommit :: MissingObjects -> GoodCommits -> Branch -> Repo -> IO (Maybe Sha, GoodCommits) +findUncorruptedCommit missing goodcommits branch r = do + (ok, goodcommits') <- verifyCommit missing goodcommits branch r + if ok + then return (Just branch, goodcommits') + else do + (ls, cleanup) <- pipeNullSplit + [ Param "log" + , Param "-z" + , Param "--format=%H" + , Param (fromRef branch) + ] r + let branchshas = catMaybes $ map extractSha ls + reflogshas <- RefLog.get branch r + -- XXX Could try a bit harder here, and look + -- for uncorrupted old commits in branches in the + -- reflog. + cleanup `after` findfirst goodcommits (branchshas ++ reflogshas) + where + findfirst gcs [] = return (Nothing, gcs) + findfirst gcs (c:cs) = do + (ok, gcs') <- verifyCommit missing gcs c r + if ok + then return (Just c, gcs') + else findfirst gcs' cs + +{- Verifies that none of the missing objects in the set are used by + - the commit. Also adds to a set of commit shas that have been verified to + - be good, which can be passed into subsequent calls to avoid + - redundant work when eg, chasing down branches to find the first + - uncorrupted commit. -} +verifyCommit :: MissingObjects -> GoodCommits -> Sha -> Repo -> IO (Bool, GoodCommits) +verifyCommit missing goodcommits commit r + | checkGoodCommit commit goodcommits = return (True, goodcommits) + | otherwise = do + (ls, cleanup) <- pipeNullSplit + [ Param "log" + , Param "-z" + , Param "--format=%H %T" + , Param (fromRef commit) + ] r + let committrees = map parse ls + if any isNothing committrees || null committrees + then do + void cleanup + return (False, goodcommits) + else do + let cts = catMaybes committrees + ifM (cleanup <&&> check cts) + ( return (True, addGoodCommits (map fst cts) goodcommits) + , return (False, goodcommits) + ) + where + parse l = case words l of + (commitsha:treesha:[]) -> (,) + <$> extractSha commitsha + <*> extractSha treesha + _ -> Nothing + check [] = return True + check ((c, t):rest) + | checkGoodCommit c goodcommits = return True + | otherwise = verifyTree missing t r <&&> check rest + +{- Verifies that a tree is good, including all trees and blobs + - referenced by it. -} +verifyTree :: MissingObjects -> Sha -> Repo -> IO Bool +verifyTree missing treesha r + | S.member treesha missing = return False + | otherwise = do + (ls, cleanup) <- pipeNullSplit (LsTree.lsTreeParams treesha []) r + let objshas = map (LsTree.sha . LsTree.parseLsTree) ls + if any (`S.member` missing) objshas + then do + void cleanup + return False + -- as long as ls-tree succeeded, we're good + else cleanup + +{- Checks that the index file only refers to objects that are not missing, + - and is not itself corrupt. Note that a missing index file is not + - considered a problem (repo may be new). -} +checkIndex :: Repo -> IO Bool +checkIndex r = do + (bad, _good, cleanup) <- partitionIndex r + if null bad + then cleanup + else do + void cleanup + return False + +{- Does not check every object the index refers to, but only that the index + - itself is not corrupt. -} +checkIndexFast :: Repo -> IO Bool +checkIndexFast r = do + (indexcontents, cleanup) <- LsFiles.stagedDetails [repoPath r] r + length indexcontents `seq` cleanup + +missingIndex :: Repo -> IO Bool +missingIndex r = not <$> doesFileExist (localGitDir r "index") + +{- Finds missing and ok files staged in the index. -} +partitionIndex :: Repo -> IO ([LsFiles.StagedDetails], [LsFiles.StagedDetails], IO Bool) +partitionIndex r = do + (indexcontents, cleanup) <- LsFiles.stagedDetails [repoPath r] r + l <- forM indexcontents $ \i -> case i of + (_file, Just sha, Just _mode) -> (,) <$> isMissing sha r <*> pure i + _ -> pure (False, i) + let (bad, good) = partition fst l + return (map snd bad, map snd good, cleanup) + +{- Rewrites the index file, removing from it any files whose blobs are + - missing. Returns the list of affected files. -} +rewriteIndex :: Repo -> IO [FilePath] +rewriteIndex r + | repoIsLocalBare r = return [] + | otherwise = do + (bad, good, cleanup) <- partitionIndex r + unless (null bad) $ do + nukeFile (indexFile r) + UpdateIndex.streamUpdateIndex r + =<< (catMaybes <$> mapM reinject good) + void cleanup + return $ map fst3 bad + where + reinject (file, Just sha, Just mode) = case toTreeItemType mode of + Nothing -> return Nothing + Just treeitemtype -> Just <$> + UpdateIndex.stageFile sha treeitemtype file r + reinject _ = return Nothing + +newtype GoodCommits = GoodCommits (S.Set Sha) + +emptyGoodCommits :: GoodCommits +emptyGoodCommits = GoodCommits S.empty + +checkGoodCommit :: Sha -> GoodCommits -> Bool +checkGoodCommit sha (GoodCommits s) = S.member sha s + +addGoodCommits :: [Sha] -> GoodCommits -> GoodCommits +addGoodCommits shas (GoodCommits s) = GoodCommits $ + S.union s (S.fromList shas) + +displayList :: [String] -> String -> IO () +displayList items header + | null items = return () + | otherwise = do + putStrLn header + putStr $ unlines $ map (\i -> "\t" ++ i) truncateditems + where + numitems = length items + truncateditems + | numitems > 10 = take 10 items ++ ["(and " ++ show (numitems - 10) ++ " more)"] + | otherwise = items + +{- Fix problems that would prevent repair from working at all + - + - A missing or corrupt .git/HEAD makes git not treat the repository as a + - git repo. If there is a git repo in a parent directory, it may move up + - the tree and use that one instead. So, cannot use `git show-ref HEAD` to + - test it. + - + - Explode the packed refs file, to simplify dealing with refs, and because + - fsck can complain about bad refs in it. + -} +preRepair :: Repo -> IO () +preRepair g = do + unlessM (validhead <$> catchDefaultIO "" (safeReadFile headfile)) $ do + nukeFile headfile + writeFile headfile "ref: refs/heads/master" + explodePackedRefsFile g + unless (repoIsLocalBare g) $ do + let f = indexFile g + void $ tryIO $ allowWrite f + where + headfile = localGitDir g "HEAD" + validhead s = "ref: refs/" `isPrefixOf` s || isJust (extractSha s) + +{- Put it all together. -} +runRepair :: (Ref -> Bool) -> Bool -> Repo -> IO (Bool, [Branch]) +runRepair removablebranch forced g = do + preRepair g + putStrLn "Running git fsck ..." + fsckresult <- findBroken False g + if foundBroken fsckresult + then runRepair' removablebranch fsckresult forced Nothing g + else do + bad <- badBranches S.empty g + if null bad + then do + putStrLn "No problems found." + return (True, []) + else runRepair' removablebranch fsckresult forced Nothing g + +runRepairOf :: FsckResults -> (Ref -> Bool) -> Bool -> Maybe FilePath -> Repo -> IO (Bool, [Branch]) +runRepairOf fsckresult removablebranch forced referencerepo g = do + preRepair g + runRepair' removablebranch fsckresult forced referencerepo g + +runRepair' :: (Ref -> Bool) -> FsckResults -> Bool -> Maybe FilePath -> Repo -> IO (Bool, [Branch]) +runRepair' removablebranch fsckresult forced referencerepo g = do + cleanCorruptObjects fsckresult g + missing <- findBroken False g + stillmissing <- retrieveMissingObjects missing referencerepo g + case stillmissing of + FsckFoundMissing s t + | S.null s -> if repoIsLocalBare g + then checkbadbranches s + else ifM (checkIndex g) + ( checkbadbranches s + , do + putStrLn "No missing objects found, but the index file is corrupt!" + if forced + then corruptedindex + else needforce + ) + | otherwise -> if forced + then ifM (checkIndex g) + ( forcerepair s t + , corruptedindex + ) + else do + putStrLn $ unwords + [ show (S.size s) + , "missing objects could not be recovered!" + ] + unsuccessfulfinish + FsckFailed + | forced -> ifM (pure (repoIsLocalBare g) <||> checkIndex g) + ( do + cleanCorruptObjects FsckFailed g + stillmissing' <- findBroken False g + case stillmissing' of + FsckFailed -> return (False, []) + FsckFoundMissing s t -> forcerepair s t + , corruptedindex + ) + | otherwise -> unsuccessfulfinish + where + repairbranches missing = do + (removedbranches, goodcommits) <- removeBadBranches' removablebranch missing emptyGoodCommits g + let remotebranches = filter isTrackingBranch removedbranches + unless (null remotebranches) $ + putStrLn $ unwords + [ "Removed" + , show (length remotebranches) + , "remote tracking branches that referred to missing objects." + ] + (resetbranches, deletedbranches, _) <- resetLocalBranches missing goodcommits g + displayList (map fromRef resetbranches) + "Reset these local branches to old versions before the missing objects were committed:" + displayList (map fromRef deletedbranches) + "Deleted these local branches, which could not be recovered due to missing objects:" + return (resetbranches ++ deletedbranches) + + checkbadbranches missing = do + bad <- badBranches missing g + case (null bad, forced) of + (True, _) -> successfulfinish [] + (False, False) -> do + displayList (map fromRef bad) + "Some git branches refer to missing objects:" + unsuccessfulfinish + (False, True) -> successfulfinish =<< repairbranches missing + + forcerepair missing fscktruncated = do + modifiedbranches <- repairbranches missing + deindexedfiles <- rewriteIndex g + displayList deindexedfiles + "Removed these missing files from the index. You should look at what files are present in your working tree and git add them back to the index when appropriate." + + -- When the fsck results were truncated, try + -- fscking again, and as long as different + -- missing objects are found, continue + -- the repair process. + if fscktruncated + then do + fsckresult' <- findBroken False g + case fsckresult' of + FsckFailed -> do + putStrLn "git fsck is failing" + return (False, modifiedbranches) + FsckFoundMissing s _ + | S.null s -> successfulfinish modifiedbranches + | S.null (s `S.difference` missing) -> do + putStrLn $ unwords + [ show (S.size s) + , "missing objects could not be recovered!" + ] + return (False, modifiedbranches) + | otherwise -> do + (ok, modifiedbranches') <- runRepairOf fsckresult' removablebranch forced referencerepo g + return (ok, modifiedbranches++modifiedbranches') + else successfulfinish modifiedbranches + + corruptedindex = do + nukeFile (indexFile g) + -- The corrupted index can prevent fsck from finding other + -- problems, so re-run repair. + fsckresult' <- findBroken False g + result <- runRepairOf fsckresult' removablebranch forced referencerepo g + putStrLn "Removed the corrupted index file. You should look at what files are present in your working tree and git add them back to the index when appropriate." + return result + + successfulfinish modifiedbranches + | null modifiedbranches = do + mapM_ putStrLn + [ "Successfully recovered repository!" + , "You should run \"git fsck\" to make sure, but it looks like everything was recovered ok." + ] + return (True, modifiedbranches) + | otherwise = do + unless (repoIsLocalBare g) $ do + mcurr <- Branch.currentUnsafe g + case mcurr of + Nothing -> return () + Just curr -> when (any (== curr) modifiedbranches) $ do + putStrLn $ unwords + [ "You currently have" + , fromRef curr + , "checked out. You may have staged changes in the index that can be committed to recover the lost state of this branch!" + ] + putStrLn "Successfully recovered repository!" + putStrLn "Please carefully check that the changes mentioned above are ok.." + return (True, modifiedbranches) + + unsuccessfulfinish = do + if repoIsLocalBare g + then do + putStrLn "If you have a clone of this bare repository, you should add it as a remote of this repository, and retry." + putStrLn "If there are no clones of this repository, you can instead retry with the --force parameter to force recovery to a possibly usable state." + return (False, []) + else needforce + needforce = do + putStrLn "To force a recovery to a usable state, retry with the --force parameter." + return (False, []) + +successfulRepair :: (Bool, [Branch]) -> Bool +successfulRepair = fst + +safeReadFile :: FilePath -> IO String +safeReadFile f = do + allowRead f + readFileStrict f diff --git a/Git/Sha.hs b/Git/Sha.hs new file mode 100644 index 0000000000..b802c85560 --- /dev/null +++ b/Git/Sha.hs @@ -0,0 +1,43 @@ +{- git SHA stuff + - + - Copyright 2011 Joey Hess + - + - Licensed under the GNU GPL version 3 or higher. + -} + +module Git.Sha where + +import Common +import Git.Types + +{- Runs an action that causes a git subcommand to emit a Sha, and strips + - any trailing newline, returning the sha. -} +getSha :: String -> IO String -> IO Sha +getSha subcommand a = maybe bad return =<< extractSha <$> a + where + bad = error $ "failed to read sha from git " ++ subcommand + +{- Extracts the Sha from a string. There can be a trailing newline after + - it, but nothing else. -} +extractSha :: String -> Maybe Sha +extractSha s + | len == shaSize = val s + | len == shaSize + 1 && length s' == shaSize = val s' + | otherwise = Nothing + where + len = length s + s' = firstLine s + val v + | all (`elem` "1234567890ABCDEFabcdef") v = Just $ Ref v + | otherwise = Nothing + +{- Size of a git sha. -} +shaSize :: Int +shaSize = 40 + +nullSha :: Ref +nullSha = Ref $ replicate shaSize '0' + +{- Git's magic empty tree. -} +emptyTree :: Ref +emptyTree = Ref "4b825dc642cb6eb9a060e54bf8d69288fbee4904" diff --git a/Git/Ssh.hs b/Git/Ssh.hs new file mode 100644 index 0000000000..3c9b23905b --- /dev/null +++ b/Git/Ssh.hs @@ -0,0 +1,72 @@ +{- GIT_SSH and GIT_SSH_COMMAND support + - + - Copyright 2017 Joey Hess + - + - Licensed under the GNU GPL version 3 or higher. + -} + +module Git.Ssh (module Git.Ssh, module Utility.SshHost) where + +import Common +import Utility.Env +import Utility.SshHost + +import Data.Char + +gitSshEnv :: String +gitSshEnv = "GIT_SSH" + +gitSshCommandEnv :: String +gitSshCommandEnv = "GIT_SSH_COMMAND" + +gitSshEnvSet :: IO Bool +gitSshEnvSet = anyM (isJust <$$> getEnv) [gitSshEnv, gitSshCommandEnv] + +type SshPort = Integer + +-- Command to run on the remote host. It is run by the shell +-- there, so any necessary shell escaping of parameters in it should +-- already be done. +type SshCommand = String + +-- | Checks for GIT_SSH and GIT_SSH_COMMAND and if set, returns +-- a command and parameters to run to ssh. +gitSsh :: SshHost -> Maybe SshPort -> SshCommand -> IO (Maybe (FilePath, [CommandParam])) +gitSsh host mp cmd = gitSsh' host mp cmd [] + +gitSsh' :: SshHost -> Maybe SshPort -> SshCommand -> [CommandParam] -> IO (Maybe (FilePath, [CommandParam])) +gitSsh' host mp cmd extrasshparams = do + gsc <- getEnv gitSshCommandEnv + case gsc of + Just c + -- git only runs the command with the shell + -- when it contains spaces; otherwise it's + -- treated the same as GIT_SSH + | any isSpace c -> ret "sh" + [ Param "-c" + , Param (shellcmd c sshps) + ] + | otherwise -> ret c sshps + Nothing -> do + gs <- getEnv gitSshEnv + case gs of + Just c -> ret c sshps + Nothing -> return Nothing + where + ret c l = return $ Just (c, l) + + -- Git passes exactly these parameters to the ssh command. + gitps = map Param $ case mp of + Nothing -> [fromSshHost host, cmd] + Just p -> [fromSshHost host, "-p", show p, cmd] + + -- Passing any extra parameters to the ssh command may + -- break some commands. + sshps = extrasshparams ++ gitps + + -- The shell command to run with sh -c is constructed + -- this way, rather than using "$@" because there could be some + -- unwanted parameters passed to the command, and this way they + -- are ignored. For example, when Utility.Rsync.rsyncShell is + -- used, rsync adds some parameters after the command. + shellcmd c ps = c ++ " " ++ unwords (map shellEscape (toCommand ps)) diff --git a/Git/Status.hs b/Git/Status.hs new file mode 100644 index 0000000000..9c4e5a2630 --- /dev/null +++ b/Git/Status.hs @@ -0,0 +1,79 @@ +{- git status interface + - + - Copyright 2015-2018 Joey Hess + - + - Licensed under the GNU GPL version 3 or higher. + -} + +module Git.Status where + +import Common +import Git +import Git.Command +import Git.FilePath + +data Status + = Modified TopFilePath + | Deleted TopFilePath + | Added TopFilePath + | Renamed TopFilePath TopFilePath + | TypeChanged TopFilePath + | Untracked TopFilePath + +data StagedUnstaged a = StagedUnstaged + { staged :: Maybe a + , unstaged :: Maybe a + } + +statusChar :: Status -> Char +statusChar (Modified _) = 'M' +statusChar (Deleted _) = 'D' +statusChar (Added _) = 'A' +statusChar (Renamed _ _) = 'R' +statusChar (TypeChanged _) = 'T' +statusChar (Untracked _) = '?' + +statusFile :: Status -> TopFilePath +statusFile (Modified f) = f +statusFile (Deleted f) = f +statusFile (Added f) = f +statusFile (Renamed _oldf newf) = newf +statusFile (TypeChanged f) = f +statusFile (Untracked f) = f + +parseStatusZ :: [String] -> [StagedUnstaged Status] +parseStatusZ = go [] + where + go c [] = reverse c + go c (x:xs) = case x of + (sstaged:sunstaged:' ':f) -> + case (cparse sstaged f xs, cparse sunstaged f xs) of + ((vstaged, xs1), (vunstaged, xs2)) -> + let v = StagedUnstaged + { staged = vstaged + , unstaged = vunstaged + } + xs' = fromMaybe xs (xs1 <|> xs2) + in go (v : c) xs' + _ -> go c xs + + cparse 'M' f _ = (Just (Modified (asTopFilePath f)), Nothing) + cparse 'A' f _ = (Just (Added (asTopFilePath f)), Nothing) + cparse 'D' f _ = (Just (Deleted (asTopFilePath f)), Nothing) + cparse 'T' f _ = (Just (TypeChanged (asTopFilePath f)), Nothing) + cparse '?' f _ = (Just (Untracked (asTopFilePath f)), Nothing) + cparse 'R' f (oldf:xs) = + (Just (Renamed (asTopFilePath oldf) (asTopFilePath f)), Just xs) + cparse _ _ _ = (Nothing, Nothing) + +getStatus :: [CommandParam] -> [FilePath] -> Repo -> IO ([StagedUnstaged Status], IO Bool) +getStatus ps fs r = do + (ls, cleanup) <- pipeNullSplit ps' r + return (parseStatusZ ls, cleanup) + where + ps' = concat + [ [Param "status"] + , ps + , [ Param "-uall" , Param "-z"] + , map File fs + ] diff --git a/Git/Tree.hs b/Git/Tree.hs new file mode 100644 index 0000000000..9e9b17af2b --- /dev/null +++ b/Git/Tree.hs @@ -0,0 +1,289 @@ +{- git trees + - + - Copyright 2016 Joey Hess + - + - Licensed under the GNU GPL version 3 or higher. + -} + +{-# LANGUAGE BangPatterns, TypeSynonymInstances, FlexibleInstances #-} + +module Git.Tree ( + Tree(..), + TreeContent(..), + getTree, + recordTree, + TreeItem(..), + adjustTree, + treeMode, +) where + +import Common +import Git +import Git.FilePath +import Git.Types +import Git.Command +import Git.Sha +import qualified Git.LsTree as LsTree +import qualified Utility.CoProcess as CoProcess + +import Numeric +import System.Posix.Types +import Control.Monad.IO.Class +import qualified Data.Set as S +import qualified Data.Map as M + +newtype Tree = Tree [TreeContent] + deriving (Show) + +data TreeContent + -- A blob object in the tree. + = TreeBlob TopFilePath FileMode Sha + -- A subtree that is already recorded in git, with a known sha. + | RecordedSubTree TopFilePath Sha [TreeContent] + -- A subtree that has not yet been recorded in git. + | NewSubTree TopFilePath [TreeContent] + -- A commit object that is part of a tree (used for submodules) + | TreeCommit TopFilePath FileMode Sha + deriving (Show, Eq, Ord) + +{- Gets the Tree for a Ref. -} +getTree :: Ref -> Repo -> IO Tree +getTree r repo = do + (l, cleanup) <- lsTreeWithObjects r repo + let !t = either (\e -> error ("ls-tree parse error:" ++ e)) id + (extractTree l) + void cleanup + return t + +lsTreeWithObjects :: Ref -> Repo -> IO ([LsTree.TreeItem], IO Bool) +lsTreeWithObjects = LsTree.lsTree' [Param "-t"] + +newtype MkTreeHandle = MkTreeHandle CoProcess.CoProcessHandle + +withMkTreeHandle :: (MonadIO m, MonadMask m) => Repo -> (MkTreeHandle -> m a) -> m a +withMkTreeHandle repo a = bracketIO setup cleanup (a . MkTreeHandle) + where + setup = gitCoProcessStart False ps repo + ps = [Param "mktree", Param "--batch", Param "-z"] + cleanup = CoProcess.stop + +{- Records a Tree in the Repo, returning its Sha. + - + - Efficiently handles subtrees, by only recording ones that have not + - already been recorded before. And even when many subtrees need to be + - recorded, it's done with a single call to git mktree, using its batch + - interface. + -} +recordTree :: Tree -> Repo -> IO Sha +recordTree t repo = withMkTreeHandle repo $ \h -> recordTree' h t + +recordTree' :: MkTreeHandle -> Tree -> IO Sha +recordTree' h (Tree l) = mkTree h =<< mapM (recordSubTree h) l + +{- Note that the returned RecordedSubTree does not have its [TreeContent] + - list populated. This is a memory optimisation, since the list is not + - used. -} +recordSubTree :: MkTreeHandle -> TreeContent -> IO TreeContent +recordSubTree h (NewSubTree d l) = do + sha <- mkTree h =<< mapM (recordSubTree h) l + return (RecordedSubTree d sha []) +recordSubTree _ alreadyrecorded = return alreadyrecorded + +mkTree :: MkTreeHandle -> [TreeContent] -> IO Sha +mkTree (MkTreeHandle cp) l = CoProcess.query cp send receive + where + send h = do + forM_ l $ \i -> hPutStr h $ case i of + TreeBlob f fm s -> mkTreeOutput fm BlobObject s f + RecordedSubTree f s _ -> mkTreeOutput treeMode TreeObject s f + NewSubTree _ _ -> error "recordSubTree internal error; unexpected NewSubTree" + TreeCommit f fm s -> mkTreeOutput fm CommitObject s f + hPutStr h "\NUL" -- signal end of tree to --batch + receive h = getSha "mktree" (hGetLine h) + +treeMode :: FileMode +treeMode = 0o040000 + +mkTreeOutput :: FileMode -> ObjectType -> Sha -> TopFilePath -> String +mkTreeOutput fm ot s f = concat + [ showOct fm "" + , " " + , show ot + , " " + , fromRef s + , "\t" + , takeFileName (getTopFilePath f) + , "\NUL" + ] + +data TreeItem = TreeItem TopFilePath FileMode Sha + deriving (Show, Eq) + +treeItemToTreeContent :: TreeItem -> TreeContent +treeItemToTreeContent (TreeItem f m s) = TreeBlob f m s + +treeItemsToTree :: [TreeItem] -> Tree +treeItemsToTree = go M.empty + where + go m [] = Tree $ filter inTopTree (M.elems m) + go m (i:is) + | inTopTree p = + go (M.insert p (treeItemToTreeContent i) m) is + | otherwise = case M.lookup idir m of + Just (NewSubTree d l) -> + go (addsubtree idir m (NewSubTree d (c:l))) is + _ -> + go (addsubtree idir m (NewSubTree (asTopFilePath idir) [c])) is + where + p = gitPath i + idir = takeDirectory p + c = treeItemToTreeContent i + + addsubtree d m t + | not (inTopTree d) = + let m' = M.insert d t m + in case M.lookup parent m' of + Just (NewSubTree d' l) -> + let l' = filter (\ti -> gitPath ti /= d) l + in addsubtree parent m' (NewSubTree d' (t:l')) + _ -> addsubtree parent m' (NewSubTree (asTopFilePath parent) [t]) + | otherwise = M.insert d t m + where + parent = takeDirectory d + +{- Flattens the top N levels of a Tree. -} +flattenTree :: Int -> Tree -> Tree +flattenTree 0 t = t +flattenTree n (Tree l) = Tree (concatMap (go n) l) + where + go 0 c = [c] + go _ b@(TreeBlob _ _ _) = [b] + go n' (RecordedSubTree _ _ l') = concatMap (go (n'-1)) l' + go n' (NewSubTree _ l') = concatMap (go (n'-1)) l' + go _ c@(TreeCommit _ _ _) = [c] + +{- Applies an adjustment to items in a tree. + - + - While less flexible than using getTree and recordTree, + - this avoids buffering the whole tree in memory. + -} +adjustTree + :: (Functor m, MonadIO m, MonadMask m) + => (TreeItem -> m (Maybe TreeItem)) + -- ^ Adjust an item in the tree. Nothing deletes the item. + -- Cannot move the item to a different tree. + -> [TreeItem] + -- ^ New items to add to the tree. + -> [TopFilePath] + -- ^ Files to remove from the tree. + -> Ref + -> Repo + -> m Sha +adjustTree adjusttreeitem addtreeitems removefiles r repo = + withMkTreeHandle repo $ \h -> do + (l, cleanup) <- liftIO $ lsTreeWithObjects r repo + (l', _, _) <- go h False [] 1 inTopTree l + l'' <- adjustlist h 0 inTopTree (const True) l' + sha <- liftIO $ mkTree h l'' + void $ liftIO cleanup + return sha + where + go _ wasmodified c _ _ [] = return (c, wasmodified, []) + go h wasmodified c depth intree (i:is) + | intree i = case readObjectType (LsTree.typeobj i) of + Just BlobObject -> do + let ti = TreeItem (LsTree.file i) (LsTree.mode i) (LsTree.sha i) + v <- adjusttreeitem ti + case v of + Nothing -> go h True c depth intree is + Just ti'@(TreeItem f m s) -> + let !modified = wasmodified || ti' /= ti + blob = TreeBlob f m s + in go h modified (blob:c) depth intree is + Just TreeObject -> do + (sl, modified, is') <- go h False [] (depth+1) (beneathSubTree i) is + sl' <- adjustlist h depth (inTree i) (beneathSubTree i) sl + let slmodified = sl' /= sl + subtree <- if modified || slmodified + then liftIO $ recordSubTree h $ NewSubTree (LsTree.file i) sl' + else return $ RecordedSubTree (LsTree.file i) (LsTree.sha i) [] + let !modified' = modified || slmodified || wasmodified + go h modified' (subtree : c) depth intree is' + Just CommitObject -> do + let ti = TreeCommit (LsTree.file i) (LsTree.mode i) (LsTree.sha i) + go h wasmodified (ti:c) depth intree is + _ -> error ("unexpected object type \"" ++ LsTree.typeobj i ++ "\"") + | otherwise = return (c, wasmodified, i:is) + + adjustlist h depth ishere underhere l = do + let (addhere, rest) = partition ishere addtreeitems + let l' = filter (not . removed) $ + map treeItemToTreeContent addhere ++ l + let inl i = any (\t -> beneathSubTree t i) l' + let (Tree addunderhere) = flattenTree depth $ treeItemsToTree $ + filter (\i -> underhere i && not (inl i)) rest + addunderhere' <- liftIO $ mapM (recordSubTree h) addunderhere + return (addunderhere'++l') + + removeset = S.fromList $ map (normalise . gitPath) removefiles + removed (TreeBlob f _ _) = S.member (normalise (gitPath f)) removeset + removed _ = False + +{- Assumes the list is ordered, with tree objects coming right before their + - contents. -} +extractTree :: [LsTree.TreeItem] -> Either String Tree +extractTree l = case go [] inTopTree l of + Right (t, []) -> Right (Tree t) + Right _ -> parseerr "unexpected tree form" + Left e -> parseerr e + where + go t _ [] = Right (t, []) + go t intree (i:is) + | intree i = case readObjectType (LsTree.typeobj i) of + Just BlobObject -> + let b = TreeBlob (LsTree.file i) (LsTree.mode i) (LsTree.sha i) + in go (b:t) intree is + Just TreeObject -> case go [] (beneathSubTree i) is of + Right (subtree, is') -> + let st = RecordedSubTree (LsTree.file i) (LsTree.sha i) subtree + in go (st:t) intree is' + Left e -> Left e + Just CommitObject -> + let c = TreeCommit (LsTree.file i) (LsTree.mode i) (LsTree.sha i) + in go (c:t) intree is + _ -> parseerr ("unexpected object type \"" ++ LsTree.typeobj i ++ "\"") + | otherwise = Right (t, i:is) + parseerr = Left + +class GitPath t where + gitPath :: t -> FilePath + +instance GitPath FilePath where + gitPath = id + +instance GitPath TopFilePath where + gitPath = getTopFilePath + +instance GitPath TreeItem where + gitPath (TreeItem f _ _) = gitPath f + +instance GitPath LsTree.TreeItem where + gitPath = gitPath . LsTree.file + +instance GitPath TreeContent where + gitPath (TreeBlob f _ _) = gitPath f + gitPath (RecordedSubTree f _ _) = gitPath f + gitPath (NewSubTree f _) = gitPath f + gitPath (TreeCommit f _ _) = gitPath f + +inTopTree :: GitPath t => t -> Bool +inTopTree = inTree "." + +inTree :: (GitPath t, GitPath f) => t -> f -> Bool +inTree t f = gitPath t == takeDirectory (gitPath f) + +beneathSubTree :: (GitPath t, GitPath f) => t -> f -> Bool +beneathSubTree t f = prefix `isPrefixOf` normalise (gitPath f) + where + tp = gitPath t + prefix = if null tp then tp else addTrailingPathSeparator (normalise tp) diff --git a/Git/Types.hs b/Git/Types.hs new file mode 100644 index 0000000000..f32f16b12e --- /dev/null +++ b/Git/Types.hs @@ -0,0 +1,125 @@ +{- git data types + - + - Copyright 2010-2018 Joey Hess + - + - Licensed under the GNU GPL version 3 or higher. + -} + +module Git.Types where + +import Network.URI +import qualified Data.Map as M +import System.Posix.Types +import Utility.SafeCommand + +{- Support repositories on local disk, and repositories accessed via an URL. + - + - Repos on local disk have a git directory, and unless bare, a worktree. + - + - A local repo may not have had its config read yet, in which case all + - that's known about it is its path. + - + - Finally, an Unknown repository may be known to exist, but nothing + - else known about it. + -} +data RepoLocation + = Local { gitdir :: FilePath, worktree :: Maybe FilePath } + | LocalUnknown FilePath + | Url URI + | Unknown + deriving (Show, Eq, Ord) + +data Repo = Repo + { location :: RepoLocation + , config :: M.Map String String + -- a given git config key can actually have multiple values + , fullconfig :: M.Map String [String] + -- remoteName holds the name used for this repo in some other + -- repo's list of remotes, when this repo is such a remote + , remoteName :: Maybe RemoteName + -- alternate environment to use when running git commands + , gitEnv :: Maybe [(String, String)] + , gitEnvOverridesGitDir :: Bool + -- global options to pass to git when running git commands + , gitGlobalOpts :: [CommandParam] + } deriving (Show, Eq, Ord) + +type RemoteName = String + +{- A git ref. Can be a sha1, or a branch or tag name. -} +newtype Ref = Ref String + deriving (Eq, Ord, Read, Show) + +fromRef :: Ref -> String +fromRef (Ref s) = s + +{- Aliases for Ref. -} +type Branch = Ref +type Sha = Ref +type Tag = Ref + +{- A date in the format described in gitrevisions. Includes the + - braces, eg, "{yesterday}" -} +newtype RefDate = RefDate String + +{- Types of objects that can be stored in git. -} +data ObjectType = BlobObject | CommitObject | TreeObject + deriving (Eq) + +instance Show ObjectType where + show BlobObject = "blob" + show CommitObject = "commit" + show TreeObject = "tree" + +readObjectType :: String -> Maybe ObjectType +readObjectType "blob" = Just BlobObject +readObjectType "commit" = Just CommitObject +readObjectType "tree" = Just TreeObject +readObjectType _ = Nothing + +{- Types of items in a tree. -} +data TreeItemType = TreeFile | TreeExecutable | TreeSymlink | TreeSubmodule + deriving (Eq) + +{- Git uses magic numbers to denote the type of a tree item. -} +readTreeItemType :: String -> Maybe TreeItemType +readTreeItemType "100644" = Just TreeFile +readTreeItemType "100755" = Just TreeExecutable +readTreeItemType "120000" = Just TreeSymlink +readTreeItemType "160000" = Just TreeSubmodule +readTreeItemType _ = Nothing + +fmtTreeItemType :: TreeItemType -> String +fmtTreeItemType TreeFile = "100644" +fmtTreeItemType TreeExecutable = "100755" +fmtTreeItemType TreeSymlink = "120000" +fmtTreeItemType TreeSubmodule = "160000" + +toTreeItemType :: FileMode -> Maybe TreeItemType +toTreeItemType 0o100644 = Just TreeFile +toTreeItemType 0o100755 = Just TreeExecutable +toTreeItemType 0o120000 = Just TreeSymlink +toTreeItemType 0o160000 = Just TreeSubmodule +toTreeItemType _ = Nothing + +fromTreeItemType :: TreeItemType -> FileMode +fromTreeItemType TreeFile = 0o100644 +fromTreeItemType TreeExecutable = 0o100755 +fromTreeItemType TreeSymlink = 0o120000 +fromTreeItemType TreeSubmodule = 0o160000 + +data Commit = Commit + { commitTree :: Sha + , commitParent :: [Sha] + , commitAuthorMetaData :: CommitMetaData + , commitCommitterMetaData :: CommitMetaData + , commitMessage :: String + } + deriving (Show) + +data CommitMetaData = CommitMetaData + { commitName :: Maybe String + , commitEmail :: Maybe String + , commitDate :: Maybe String -- In raw git form, "epoch -tzoffset" + } + deriving (Show) diff --git a/Git/UnionMerge.hs b/Git/UnionMerge.hs new file mode 100644 index 0000000000..a9086f0e43 --- /dev/null +++ b/Git/UnionMerge.hs @@ -0,0 +1,113 @@ +{- git-union-merge library + - + - Copyright 2011 Joey Hess + - + - Licensed under the GNU GPL version 3 or higher. + -} + +module Git.UnionMerge ( + merge, + mergeIndex +) where + +import qualified Data.ByteString.Lazy as L +import qualified Data.Set as S + +import Common +import Git +import Git.Sha +import Git.CatFile +import Git.Command +import Git.UpdateIndex +import Git.HashObject +import Git.Types +import Git.FilePath +import Utility.FileSystemEncoding + +{- Performs a union merge between two branches, staging it in the index. + - Any previously staged changes in the index will be lost. + - + - Should be run with a temporary index file configured by useIndex. + -} +merge :: Ref -> Ref -> Repo -> IO () +merge x y repo = do + hashhandle <- hashObjectStart repo + ch <- catFileStart repo + streamUpdateIndex repo + [ lsTree x repo + , mergeTrees x y hashhandle ch repo + ] + catFileStop ch + hashObjectStop hashhandle + +{- Merges a list of branches into the index. Previously staged changes in + - the index are preserved (and participate in the merge). + - + - update-index is run once per ref in turn, so that each ref is merged on + - top of the merge for the previous ref. It would be more efficient, but + - harder to calculate a single union merge involving all the refs, as well + - as the index. + -} +mergeIndex :: HashObjectHandle -> CatFileHandle -> Repo -> [Ref] -> IO () +mergeIndex hashhandle ch repo bs = forM_ bs $ \b -> + streamUpdateIndex repo [mergeTreeIndex b hashhandle ch repo] + +{- For merging two trees. -} +mergeTrees :: Ref -> Ref -> HashObjectHandle -> CatFileHandle -> Repo -> Streamer +mergeTrees (Ref x) (Ref y) hashhandle ch = doMerge hashhandle ch + ("diff-tree":diffOpts ++ [x, y, "--"]) + +{- For merging a single tree into the index. -} +mergeTreeIndex :: Ref -> HashObjectHandle -> CatFileHandle -> Repo -> Streamer +mergeTreeIndex (Ref r) hashhandle ch = doMerge hashhandle ch $ + "diff-index" : diffOpts ++ ["--cached", r, "--"] + +diffOpts :: [String] +diffOpts = ["--raw", "-z", "-r", "--no-renames", "-l0"] + +{- Streams update-index changes to perform a merge, + - using git to get a raw diff. -} +doMerge :: HashObjectHandle -> CatFileHandle -> [String] -> Repo -> Streamer +doMerge hashhandle ch differ repo streamer = do + (diff, cleanup) <- pipeNullSplit (map Param differ) repo + go diff + void $ cleanup + where + go [] = noop + go (info:file:rest) = mergeFile info file hashhandle ch >>= + maybe (go rest) (\l -> streamer l >> go rest) + go (_:[]) = error $ "parse error " ++ show differ + +{- Given an info line from a git raw diff, and the filename, generates + - a line suitable for update-index that union merges the two sides of the + - diff. -} +mergeFile :: String -> FilePath -> HashObjectHandle -> CatFileHandle -> IO (Maybe String) +mergeFile info file hashhandle h = case filter (/= nullSha) [Ref asha, Ref bsha] of + [] -> return Nothing + (sha:[]) -> use sha + shas -> use + =<< either return (\s -> hashBlob hashhandle (unlines s)) + =<< calcMerge . zip shas <$> mapM getcontents shas + where + [_colonmode, _bmode, asha, bsha, _status] = words info + use sha = return $ Just $ + updateIndexLine sha TreeFile $ asTopFilePath file + -- We don't know how the file is encoded, but need to + -- split it into lines to union merge. Using the + -- FileSystemEncoding for this is a hack, but ensures there + -- are no decoding errors. + getcontents s = lines . encodeW8NUL . L.unpack <$> catObject h s + +{- Calculates a union merge between a list of refs, with contents. + - + - When possible, reuses the content of an existing ref, rather than + - generating new content. + -} +calcMerge :: [(Ref, [String])] -> Either Ref [String] +calcMerge shacontents + | null reuseable = Right $ new + | otherwise = Left $ fst $ Prelude.head reuseable + where + reuseable = filter (\c -> sorteduniq (snd c) == new) shacontents + new = sorteduniq $ concat $ map snd shacontents + sorteduniq = S.toList . S.fromList diff --git a/Git/UpdateIndex.hs b/Git/UpdateIndex.hs new file mode 100644 index 0000000000..f765c39c2c --- /dev/null +++ b/Git/UpdateIndex.hs @@ -0,0 +1,146 @@ +{- git-update-index library + - + - Copyright 2011-2018 Joey Hess + - + - Licensed under the GNU GPL version 3 or higher. + -} + +{-# LANGUAGE BangPatterns, CPP #-} + +module Git.UpdateIndex ( + Streamer, + pureStreamer, + streamUpdateIndex, + streamUpdateIndex', + startUpdateIndex, + stopUpdateIndex, + lsTree, + lsSubTree, + updateIndexLine, + stageFile, + unstageFile, + stageSymlink, + stageDiffTreeItem, + refreshIndex, +) where + +import Common +import Git +import Git.Types +import Git.Command +import Git.FilePath +import Git.Sha +import qualified Git.DiffTreeItem as Diff + +{- Streamers are passed a callback and should feed it lines in the form + - read by update-index, and generated by ls-tree. -} +type Streamer = (String -> IO ()) -> IO () + +{- A streamer with a precalculated value. -} +pureStreamer :: String -> Streamer +pureStreamer !s = \streamer -> streamer s + +{- Streams content into update-index from a list of Streamers. -} +streamUpdateIndex :: Repo -> [Streamer] -> IO () +streamUpdateIndex repo as = bracket (startUpdateIndex repo) stopUpdateIndex $ + (\h -> forM_ as $ streamUpdateIndex' h) + +data UpdateIndexHandle = UpdateIndexHandle ProcessHandle Handle + +streamUpdateIndex' :: UpdateIndexHandle -> Streamer -> IO () +streamUpdateIndex' (UpdateIndexHandle _ h) a = a $ \s -> do + hPutStr h s + hPutStr h "\0" + +startUpdateIndex :: Repo -> IO UpdateIndexHandle +startUpdateIndex repo = do + (Just h, _, _, p) <- createProcess (gitCreateProcess params repo) + { std_in = CreatePipe } + return $ UpdateIndexHandle p h + where + params = map Param ["update-index", "-z", "--index-info"] + +stopUpdateIndex :: UpdateIndexHandle -> IO Bool +stopUpdateIndex (UpdateIndexHandle p h) = do + hClose h + checkSuccessProcess p + +{- A streamer that adds the current tree for a ref. Useful for eg, copying + - and modifying branches. -} +lsTree :: Ref -> Repo -> Streamer +lsTree (Ref x) repo streamer = do + (s, cleanup) <- pipeNullSplit params repo + mapM_ streamer s + void $ cleanup + where + params = map Param ["ls-tree", "-z", "-r", "--full-tree", x] +lsSubTree :: Ref -> FilePath -> Repo -> Streamer +lsSubTree (Ref x) p repo streamer = do + (s, cleanup) <- pipeNullSplit params repo + mapM_ streamer s + void $ cleanup + where + params = map Param ["ls-tree", "-z", "-r", "--full-tree", x, p] + +{- Generates a line suitable to be fed into update-index, to add + - a given file with a given sha. -} +updateIndexLine :: Sha -> TreeItemType -> TopFilePath -> String +updateIndexLine sha treeitemtype file = concat + [ fmtTreeItemType treeitemtype + , " blob " + , fromRef sha + , "\t" + , indexPath file + ] + +stageFile :: Sha -> TreeItemType -> FilePath -> Repo -> IO Streamer +stageFile sha treeitemtype file repo = do + p <- toTopFilePath file repo + return $ pureStreamer $ updateIndexLine sha treeitemtype p + +{- A streamer that removes a file from the index. -} +unstageFile :: FilePath -> Repo -> IO Streamer +unstageFile file repo = do + p <- toTopFilePath file repo + return $ unstageFile' p + +unstageFile' :: TopFilePath -> Streamer +unstageFile' p = pureStreamer $ "0 " ++ fromRef nullSha ++ "\t" ++ indexPath p + +{- A streamer that adds a symlink to the index. -} +stageSymlink :: FilePath -> Sha -> Repo -> IO Streamer +stageSymlink file sha repo = do + !line <- updateIndexLine + <$> pure sha + <*> pure TreeSymlink + <*> toTopFilePath file repo + return $ pureStreamer line + +{- A streamer that applies a DiffTreeItem to the index. -} +stageDiffTreeItem :: Diff.DiffTreeItem -> Streamer +stageDiffTreeItem d = case toTreeItemType (Diff.dstmode d) of + Nothing -> unstageFile' (Diff.file d) + Just t -> pureStreamer $ updateIndexLine (Diff.dstsha d) t (Diff.file d) + +indexPath :: TopFilePath -> InternalGitPath +indexPath = toInternalGitPath . getTopFilePath + +{- Refreshes the index, by checking file stat information. -} +refreshIndex :: Repo -> ((FilePath -> IO ()) -> IO ()) -> IO Bool +refreshIndex repo feeder = do + (Just h, _, _, p) <- createProcess (gitCreateProcess params repo) + { std_in = CreatePipe } + feeder $ \f -> do + hPutStr h f + hPutStr h "\0" + hFlush h + hClose h + checkSuccessProcess p + where + params = + [ Param "update-index" + , Param "-q" + , Param "--refresh" + , Param "-z" + , Param "--stdin" + ] diff --git a/Git/Url.hs b/Git/Url.hs new file mode 100644 index 0000000000..fa7d200dc9 --- /dev/null +++ b/Git/Url.hs @@ -0,0 +1,71 @@ +{- git repository urls + - + - Copyright 2010, 2011 Joey Hess + - + - Licensed under the GNU GPL version 3 or higher. + -} + +module Git.Url ( + scheme, + host, + port, + hostuser, + authority, +) where + +import Network.URI hiding (scheme, authority) + +import Common +import Git.Types +import Git + +{- Scheme of an URL repo. -} +scheme :: Repo -> String +scheme Repo { location = Url u } = uriScheme u +scheme repo = notUrl repo + +{- Work around a bug in the real uriRegName + - -} +uriRegName' :: URIAuth -> String +uriRegName' a = fixup $ uriRegName a + where + fixup x@('[':rest) + | rest !! len == ']' = take len rest + | otherwise = x + where + len = length rest - 1 + fixup x = x + +{- Hostname of an URL repo. -} +host :: Repo -> Maybe String +host = authpart uriRegName' + +{- Port of an URL repo, if it has a nonstandard one. -} +port :: Repo -> Maybe Integer +port r = + case authpart uriPort r of + Nothing -> Nothing + Just ":" -> Nothing + Just (':':p) -> readish p + Just _ -> Nothing + +{- Hostname of an URL repo, including any username (ie, "user@host") -} +hostuser :: Repo -> Maybe String +hostuser r = (++) + <$> authpart uriUserInfo r + <*> authpart uriRegName' r + +{- The full authority portion an URL repo. (ie, "user@host:port") -} +authority :: Repo -> Maybe String +authority = authpart assemble + where + assemble a = uriUserInfo a ++ uriRegName' a ++ uriPort a + +{- Applies a function to extract part of the uriAuthority of an URL repo. -} +authpart :: (URIAuth -> a) -> Repo -> Maybe a +authpart a Repo { location = Url u } = a <$> uriAuthority u +authpart _ repo = notUrl repo + +notUrl :: Repo -> a +notUrl repo = error $ + "acting on local git repo " ++ repoDescribe repo ++ " not supported" diff --git a/Git/Version.hs b/Git/Version.hs new file mode 100644 index 0000000000..19ff945c8d --- /dev/null +++ b/Git/Version.hs @@ -0,0 +1,32 @@ +{- git versions + - + - Copyright 2011, 2013 Joey Hess + - + - Licensed under the GNU GPL version 3 or higher. + -} + +{-# OPTIONS_GHC -fno-warn-tabs #-} + +module Git.Version ( + installed, + older, + normalize, + GitVersion, +) where + +import Common +import Utility.DottedVersion + +type GitVersion = DottedVersion + +installed :: IO GitVersion +installed = normalize . extract <$> readProcess "git" ["--version"] + where + extract s = case lines s of + [] -> "" + (l:_) -> unwords $ drop 2 $ words l + +older :: String -> IO Bool +older n = do + v <- installed + return $ v < normalize n diff --git a/Jenkinsfile b/Jenkinsfile new file mode 100644 index 0000000000..c77d5383bf --- /dev/null +++ b/Jenkinsfile @@ -0,0 +1,55 @@ +currentBuild.result = 'SUCCESS' +def caught_exception = null + +final def EMAIL_RECIPIENTS = 'joey+git-annex@joeyh.name' + +properties([ + buildDiscarder(logRotator(artifactNumToKeepStr: '1')), + pipelineTriggers([[$class: 'hudson.triggers.SCMTrigger', scmpoll_spec: ''],]) // pollScm('') in 2.22+ +]) + +try { + + node('windows') { + + dir('git-annex') { + + stage('Checkout') { + checkout scm + } + + stage('Build') { + bat 'c:/cygwin/bin/sh standalone/windows/build.sh' + } + + stage('Archive') { + archiveArtifacts 'git-annex-installer.exe,dist/build-version' + } + + stage('Upload') { + withCredentials([usernamePassword(credentialsId: 'rsync-downloads-kitenet-net', passwordVariable: 'RSYNC_PASSWORD', usernameVariable: 'DUMMY')]) { + bat 'c:/cygwin/bin/rsync git-annex-installer.exe winautobuild@downloads.kitenet.net::winautobuild' + bat 'c:/cygwin/bin/rsync dist/build-version winautobuild@downloads.kitenet.net::winautobuild' + } + } + + } + + } + +} catch (exception) { + + caught_exception = exception + currentBuild.result = 'FAILURE' + +} finally { + + node('master') { + step([$class: 'Mailer', notifyEveryUnstableBuild: false, recipients: EMAIL_RECIPIENTS, sendToIndividuals: false]) + } + + if (caught_exception) { + throw caught_exception + } + +} diff --git a/Key.hs b/Key.hs new file mode 100644 index 0000000000..ade012a4ba --- /dev/null +++ b/Key.hs @@ -0,0 +1,181 @@ +{- git-annex Keys + - + - Copyright 2011-2017 Joey Hess + - + - Licensed under the GNU GPL version 3 or higher. + -} + +{-# OPTIONS_GHC -fno-warn-orphans #-} + +module Key ( + Key(..), + AssociatedFile(..), + stubKey, + key2file, + file2key, + nonChunkKey, + chunkKeyOffset, + isChunkKey, + isKeyPrefix, + + prop_isomorphic_key_encode, + prop_isomorphic_key_decode +) where + +import Data.Char +import qualified Data.Text as T + +import Common +import Types.Key +import Utility.QuickCheck +import Utility.Bloom +import Utility.Aeson +import qualified Utility.SimpleProtocol as Proto + +stubKey :: Key +stubKey = Key + { keyName = "" + , keyVariety = OtherKey "" + , keySize = Nothing + , keyMtime = Nothing + , keyChunkSize = Nothing + , keyChunkNum = Nothing + } + +-- Gets the parent of a chunk key. +nonChunkKey :: Key -> Key +nonChunkKey k = k + { keyChunkSize = Nothing + , keyChunkNum = Nothing + } + +-- Where a chunk key is offset within its parent. +chunkKeyOffset :: Key -> Maybe Integer +chunkKeyOffset k = (*) + <$> keyChunkSize k + <*> (pred <$> keyChunkNum k) + +isChunkKey :: Key -> Bool +isChunkKey k = isJust (keyChunkSize k) && isJust (keyChunkNum k) + +-- Checks if a string looks like at least the start of a key. +isKeyPrefix :: String -> Bool +isKeyPrefix s = [fieldSep, fieldSep] `isInfixOf` s + +fieldSep :: Char +fieldSep = '-' + +{- Converts a key to a string that is suitable for use as a filename. + - The name field is always shown last, separated by doubled fieldSeps, + - and is the only field allowed to contain the fieldSep. -} +key2file :: Key -> FilePath +key2file Key { keyVariety = kv, keySize = s, keyMtime = m, keyChunkSize = cs, keyChunkNum = cn, keyName = n } = + formatKeyVariety kv +++ ('s' ?: s) +++ ('m' ?: m) +++ ('S' ?: cs) +++ ('C' ?: cn) +++ (fieldSep : n) + where + "" +++ y = y + x +++ "" = x + x +++ y = x ++ fieldSep:y + f ?: (Just v) = f : show v + _ ?: _ = "" + +file2key :: FilePath -> Maybe Key +file2key s + | key == Just stubKey || (keyName <$> key) == Just "" || (keyVariety <$> key) == Just (OtherKey "") = Nothing + | otherwise = key + where + key = startbackend stubKey s + + startbackend k v = sepfield k v addvariety + + sepfield k v a = case span (/= fieldSep) v of + (v', _:r) -> findfields r $ a k v' + _ -> Nothing + + findfields (c:v) (Just k) + | c == fieldSep = addkeyname k v + | otherwise = sepfield k v $ addfield c + findfields _ v = v + + addvariety k v = Just k { keyVariety = parseKeyVariety v } + + -- This is a strict parser for security reasons; a key + -- can contain only 4 fields, which all consist only of numbers. + -- Any key containing other fields, or non-numeric data is + -- rejected with Nothing. + -- + -- If a key contained non-numeric fields, they could be used to + -- embed data used in a SHA1 collision attack, which would be a + -- problem since the keys are committed to git. + addfield _ _ v | not (all isDigit v) = Nothing + addfield 's' k v = do + sz <- readish v + return $ k { keySize = Just sz } + addfield 'm' k v = do + mtime <- readish v + return $ k { keyMtime = Just mtime } + addfield 'S' k v = do + chunksize <- readish v + return $ k { keyChunkSize = Just chunksize } + addfield 'C' k v = case readish v of + Just chunknum | chunknum > 0 -> + return $ k { keyChunkNum = Just chunknum } + _ -> Nothing + addfield _ _ _ = Nothing + + addkeyname k v + | validKeyName k v = Just $ k { keyName = v } + | otherwise = Nothing + +{- When a key HasExt, the length of the extension is limited in order to + - mitigate against SHA1 collision attacks. + - + - In such an attack, the extension of the key could be made to contain + - the collision generation data, with the result that a signed git commit + - including such keys would not be secure. + - + - The maximum extension length ever generated for such a key was 8 + - characters; 20 is used here to give a little future wiggle-room. + - The SHA1 common-prefix attack needs 128 bytes of data. + -} +validKeyName :: Key -> String -> Bool +validKeyName k name + | hasExt (keyVariety k) = length (takeExtensions name) <= 20 + | otherwise = True + +instance Arbitrary Key where + arbitrary = Key + <$> (listOf1 $ elements $ ['A'..'Z'] ++ ['a'..'z'] ++ ['0'..'9'] ++ "-_\r\n \t") + <*> (parseKeyVariety <$> (listOf1 $ elements ['A'..'Z'])) -- BACKEND + <*> ((abs <$>) <$> arbitrary) -- size cannot be negative + <*> ((abs . fromInteger <$>) <$> arbitrary) -- mtime cannot be negative + <*> ((abs <$>) <$> arbitrary) -- chunksize cannot be negative + <*> ((succ . abs <$>) <$> arbitrary) -- chunknum cannot be 0 or negative + +instance Hashable Key where + hashIO32 = hashIO32 . key2file + hashIO64 = hashIO64 . key2file + +instance ToJSON' Key where + toJSON' = toJSON' . key2file + +instance FromJSON Key where + parseJSON (String t) = maybe mempty pure $ file2key $ T.unpack t + parseJSON _ = mempty + +instance Proto.Serializable Key where + serialize = key2file + deserialize = file2key + +prop_isomorphic_key_encode :: Key -> Bool +prop_isomorphic_key_encode k = Just k == (file2key . key2file) k + +prop_isomorphic_key_decode :: FilePath -> Bool +prop_isomorphic_key_decode f + | normalfieldorder = maybe True (\k -> key2file k == f) (file2key f) + | otherwise = True + where + -- file2key will accept the fields in any order, so don't + -- try the test unless the fields are in the normal order + normalfieldorder = fields `isPrefixOf` "smSC" + fields = map (f !!) $ filter (< length f) $ map succ $ + elemIndices fieldSep f diff --git a/Limit.hs b/Limit.hs new file mode 100644 index 0000000000..93b32a89f6 --- /dev/null +++ b/Limit.hs @@ -0,0 +1,333 @@ +{- user-specified limits on files to act on + - + - Copyright 2011-2017 Joey Hess + - + - Licensed under the GNU GPL version 3 or higher. + -} + +{-# LANGUAGE CPP #-} + +module Limit where + +import Annex.Common +import qualified Annex +import qualified Utility.Matcher +import qualified Remote +import Annex.Content +import Annex.WorkTree +import Annex.Action +import Annex.UUID +import Logs.Trust +import Annex.NumCopies +import Types.Key +import Types.TrustLevel +import Types.Group +import Types.FileMatcher +import Types.MetaData +import Annex.MetaData +import Logs.MetaData +import Logs.Group +import Logs.Unused +import Logs.Location +import Git.Types (RefDate(..)) +import Utility.Glob +import Utility.HumanTime +import Utility.DataUnits + +#ifdef WITH_MAGICMIME +import Magic +#endif + +import Data.Time.Clock.POSIX +import qualified Data.Set as S +import qualified Data.Map as M + +{- Checks if there are user-specified limits. -} +limited :: Annex Bool +limited = (not . Utility.Matcher.isEmpty) <$> getMatcher' + +{- Gets a matcher for the user-specified limits. The matcher is cached for + - speed; once it's obtained the user-specified limits can't change. -} +getMatcher :: Annex (MatchInfo -> Annex Bool) +getMatcher = Utility.Matcher.matchM <$> getMatcher' + +getMatcher' :: Annex (Utility.Matcher.Matcher (MatchInfo -> Annex Bool)) +getMatcher' = go =<< Annex.getState Annex.limit + where + go (CompleteMatcher matcher) = return matcher + go (BuildingMatcher l) = do + let matcher = Utility.Matcher.generate (reverse l) + Annex.changeState $ \s -> + s { Annex.limit = CompleteMatcher matcher } + return matcher + +{- Adds something to the limit list, which is built up reversed. -} +add :: Utility.Matcher.Token (MatchInfo -> Annex Bool) -> Annex () +add l = Annex.changeState $ \s -> s { Annex.limit = prepend $ Annex.limit s } + where + prepend (BuildingMatcher ls) = BuildingMatcher $ l:ls + prepend _ = error "internal" + +{- Adds a new token. -} +addToken :: String -> Annex () +addToken = add . Utility.Matcher.token + +{- Adds a new limit. -} +addLimit :: Either String (MatchFiles Annex) -> Annex () +addLimit = either giveup (\l -> add $ Utility.Matcher.Operation $ l S.empty) + +{- Add a limit to skip files that do not match the glob. -} +addInclude :: String -> Annex () +addInclude = addLimit . limitInclude + +limitInclude :: MkLimit Annex +limitInclude glob = Right $ const $ matchGlobFile glob + +{- Add a limit to skip files that match the glob. -} +addExclude :: String -> Annex () +addExclude = addLimit . limitExclude + +limitExclude :: MkLimit Annex +limitExclude glob = Right $ const $ not <$$> matchGlobFile glob + +matchGlobFile :: String -> MatchInfo -> Annex Bool +matchGlobFile glob = go + where + cglob = compileGlob glob CaseSensative -- memoized + go (MatchingKey _) = pure False + go (MatchingFile fi) = pure $ matchGlob cglob (matchFile fi) + go (MatchingInfo af _ _ _) = matchGlob cglob <$> getInfo af + +#ifdef WITH_MAGICMIME +matchMagic :: Maybe Magic -> MkLimit Annex +matchMagic (Just magic) glob = Right $ const go + where + cglob = compileGlob glob CaseSensative -- memoized + go (MatchingKey _) = pure False + go (MatchingFile fi) = liftIO $ catchBoolIO $ + matchGlob cglob <$> magicFile magic (currFile fi) + go (MatchingInfo _ _ _ mimeval) = matchGlob cglob <$> getInfo mimeval +matchMagic Nothing _ = Left "unable to load magic database; \"mimetype\" cannot be used" +#endif + +{- Adds a limit to skip files not believed to be present + - in a specfied repository. Optionally on a prior date. -} +addIn :: String -> Annex () +addIn s = addLimit =<< mk + where + (name, date) = separate (== '@') s + mk + | name == "." = if null date + then use inhere + else use . inuuid =<< getUUID + | otherwise = use . inuuid =<< Remote.nameToUUID name + use a = return $ Right $ \notpresent -> checkKey (a notpresent) + inuuid u notpresent key + | null date = do + us <- Remote.keyLocations key + return $ u `elem` us && u `S.notMember` notpresent + | otherwise = do + us <- loggedLocationsHistorical (RefDate date) key + return $ u `elem` us + inhere notpresent key + | S.null notpresent = inAnnex key + | otherwise = do + u <- getUUID + if u `S.member` notpresent + then return False + else inAnnex key + +{- Limit to content that is currently present on a uuid. -} +limitPresent :: Maybe UUID -> MatchFiles Annex +limitPresent u _ = checkKey $ \key -> do + hereu <- getUUID + if u == Just hereu || isNothing u + then inAnnex key + else do + us <- Remote.keyLocations key + return $ maybe False (`elem` us) u + +{- Limit to content that is in a directory, anywhere in the repository tree -} +limitInDir :: FilePath -> MatchFiles Annex +limitInDir dir = const go + where + go (MatchingFile fi) = checkf $ matchFile fi + go (MatchingKey _) = return False + go (MatchingInfo af _ _ _) = checkf =<< getInfo af + checkf = return . elem dir . splitPath . takeDirectory + +{- Adds a limit to skip files not believed to have the specified number + - of copies. -} +addCopies :: String -> Annex () +addCopies = addLimit . limitCopies + +limitCopies :: MkLimit Annex +limitCopies want = case splitc ':' want of + -- Note that in case of a group having the same name as a trust + -- level, it's parsed as a trust level, not as a group. + [v, n] -> case parsetrustspec v of + Just checker -> go n $ checktrust checker + Nothing -> go n $ checkgroup v + [n] -> go n $ const $ return True + _ -> Left "bad value for copies" + where + go num good = case readish num of + Nothing -> Left "bad number for copies" + Just n -> Right $ \notpresent -> checkKey $ + go' n good notpresent + go' n good notpresent key = do + us <- filter (`S.notMember` notpresent) + <$> (filterM good =<< Remote.keyLocations key) + return $ length us >= n + checktrust checker u = checker <$> lookupTrust u + checkgroup g u = S.member g <$> lookupGroups u + parsetrustspec s + | "+" `isSuffixOf` s = (<=) <$> readTrustLevel (beginning s) + | otherwise = (==) <$> readTrustLevel s + +{- Adds a limit to match files that need more copies made. -} +addLackingCopies :: Bool -> String -> Annex () +addLackingCopies approx = addLimit . limitLackingCopies approx + +limitLackingCopies :: Bool -> MkLimit Annex +limitLackingCopies approx want = case readish want of + Just needed -> Right $ \notpresent mi -> flip checkKey mi $ + go mi needed notpresent + Nothing -> Left "bad value for number of lacking copies" + where + go mi needed notpresent key = do + NumCopies numcopies <- if approx + then approxNumCopies + else case mi of + MatchingFile fi -> getGlobalFileNumCopies $ matchFile fi + MatchingKey _ -> approxNumCopies + MatchingInfo _ _ _ _ -> approxNumCopies + us <- filter (`S.notMember` notpresent) + <$> (trustExclude UnTrusted =<< Remote.keyLocations key) + return $ numcopies - length us >= needed + approxNumCopies = fromMaybe defaultNumCopies <$> getGlobalNumCopies + +{- Match keys that are unused. + - + - This has a nice optimisation: When a file exists, + - its key is obviously not unused. + -} +limitUnused :: MatchFiles Annex +limitUnused _ (MatchingFile _) = return False +limitUnused _ (MatchingKey k) = S.member k <$> unusedKeys +limitUnused _ (MatchingInfo _ ak _ _) = do + k <- getInfo ak + S.member k <$> unusedKeys + +{- Limit that matches any version of any file or key. -} +limitAnything :: MatchFiles Annex +limitAnything _ _ = return True + +{- Limit that never matches. -} +limitNothing :: MatchFiles Annex +limitNothing _ _ = return False + +{- Adds a limit to skip files not believed to be present in all + - repositories in the specified group. -} +addInAllGroup :: String -> Annex () +addInAllGroup groupname = addLimit $ limitInAllGroup groupMap groupname + +limitInAllGroup :: Annex GroupMap -> MkLimit Annex +limitInAllGroup getgroupmap groupname = Right $ \notpresent mi -> do + m <- getgroupmap + let want = fromMaybe S.empty $ M.lookup groupname $ uuidsByGroup m + if S.null want + then return True + -- optimisation: Check if a wanted uuid is notpresent. + else if not (S.null (S.intersection want notpresent)) + then return False + else checkKey (check want) mi + where + check want key = do + present <- S.fromList <$> Remote.keyLocations key + return $ S.null $ want `S.difference` present + +{- Adds a limit to skip files not using a specified key-value backend. -} +addInBackend :: String -> Annex () +addInBackend = addLimit . limitInBackend + +limitInBackend :: MkLimit Annex +limitInBackend name = Right $ const $ checkKey check + where + check key = pure $ keyVariety key == variety + variety = parseKeyVariety name + +{- Adds a limit to skip files not using a secure hash. -} +addSecureHash :: Annex () +addSecureHash = addLimit $ Right limitSecureHash + +limitSecureHash :: MatchFiles Annex +limitSecureHash _ = checkKey $ pure . cryptographicallySecure . keyVariety + +{- Adds a limit to skip files that are too large or too small -} +addLargerThan :: String -> Annex () +addLargerThan = addLimit . limitSize (>) + +addSmallerThan :: String -> Annex () +addSmallerThan = addLimit . limitSize (<) + +limitSize :: (Maybe Integer -> Maybe Integer -> Bool) -> MkLimit Annex +limitSize vs s = case readSize dataUnits s of + Nothing -> Left "bad size" + Just sz -> Right $ go sz + where + go sz _ (MatchingFile fi) = lookupFileKey fi >>= check fi sz + go sz _ (MatchingKey key) = checkkey sz key + go sz _ (MatchingInfo _ _ as _) = + getInfo as >>= \sz' -> return (Just sz' `vs` Just sz) + checkkey sz key = return $ keySize key `vs` Just sz + check _ sz (Just key) = checkkey sz key + check fi sz Nothing = do + filesize <- liftIO $ catchMaybeIO $ getFileSize (currFile fi) + return $ filesize `vs` Just sz + +addMetaData :: String -> Annex () +addMetaData = addLimit . limitMetaData + +limitMetaData :: MkLimit Annex +limitMetaData s = case parseMetaDataMatcher s of + Left e -> Left e + Right (f, matching) -> Right $ const $ checkKey (check f matching) + where + check f matching k = not . S.null + . S.filter matching + . metaDataValues f <$> getCurrentMetaData k + +addTimeLimit :: Duration -> Annex () +addTimeLimit duration = do + start <- liftIO getPOSIXTime + let cutoff = start + durationToPOSIXTime duration + addLimit $ Right $ const $ const $ do + now <- liftIO getPOSIXTime + if now > cutoff + then do + warning $ "Time limit (" ++ fromDuration duration ++ ") reached!" + shutdown True + liftIO $ exitWith $ ExitFailure 101 + else return True + +addAccessedWithin :: Duration -> Annex () +addAccessedWithin duration = do + now <- liftIO getPOSIXTime + addLimit $ Right $ const $ checkKey $ check now + where + check now k = inAnnexCheck k $ \f -> + liftIO $ catchDefaultIO False $ do + s <- getFileStatus f + let accessed = realToFrac (accessTime s) + let delta = now - accessed + return $ delta <= secs + secs = fromIntegral (durationSeconds duration) + +lookupFileKey :: FileInfo -> Annex (Maybe Key) +lookupFileKey = lookupFile . currFile + +checkKey :: (Key -> Annex Bool) -> MatchInfo -> Annex Bool +checkKey a (MatchingFile fi) = lookupFileKey fi >>= maybe (return False) a +checkKey a (MatchingKey k) = a k +checkKey a (MatchingInfo _ ak _ _) = a =<< getInfo ak diff --git a/Limit/Wanted.hs b/Limit/Wanted.hs new file mode 100644 index 0000000000..a41398c109 --- /dev/null +++ b/Limit/Wanted.hs @@ -0,0 +1,26 @@ +{- git-annex limits by wanted status + - + - Copyright 2012 Joey Hess + - + - Licensed under the GNU GPL version 3 or higher. + -} + +module Limit.Wanted where + +import Annex.Common +import Annex.Wanted +import Limit +import Types.FileMatcher + +addWantGet :: Annex () +addWantGet = addLimit $ Right $ const $ checkWant $ + wantGet False Nothing + +addWantDrop :: Annex () +addWantDrop = addLimit $ Right $ const $ checkWant $ + wantDrop False Nothing Nothing + +checkWant :: (AssociatedFile -> Annex Bool) -> MatchInfo -> Annex Bool +checkWant a (MatchingFile fi) = a (AssociatedFile (Just $ matchFile fi)) +checkWant _ (MatchingKey _) = return False +checkWant _ (MatchingInfo {}) = return False diff --git a/Logs.hs b/Logs.hs new file mode 100644 index 0000000000..0af14eb26c --- /dev/null +++ b/Logs.hs @@ -0,0 +1,199 @@ +{- git-annex log file names + - + - Copyright 2013-2018 Joey Hess + - + - Licensed under the GNU GPL version 3 or higher. + -} + +module Logs where + +import Annex.Common +import Annex.DirHashes + +{- There are several varieties of log file formats. -} +data LogVariety + = UUIDBasedLog + | NewUUIDBasedLog + | ChunkLog Key + | PresenceLog Key + | RemoteMetaDataLog + | OtherLog + deriving (Show) + +{- Converts a path from the git-annex branch into one of the varieties + - of logs used by git-annex, if it's a known path. -} +getLogVariety :: FilePath -> Maybe LogVariety +getLogVariety f + | f `elem` topLevelUUIDBasedLogs = Just UUIDBasedLog + | isRemoteStateLog f = Just NewUUIDBasedLog + | isChunkLog f = ChunkLog <$> chunkLogFileKey f + | isRemoteMetaDataLog f = Just RemoteMetaDataLog + | isMetaDataLog f || f `elem` otherLogs = Just OtherLog + | otherwise = PresenceLog <$> firstJust (presenceLogs f) + +{- All the uuid-based logs stored in the top of the git-annex branch. -} +topLevelUUIDBasedLogs :: [FilePath] +topLevelUUIDBasedLogs = + [ uuidLog + , remoteLog + , trustLog + , groupLog + , preferredContentLog + , requiredContentLog + , scheduleLog + , activityLog + , differenceLog + , multicastLog + , exportLog + ] + +{- All the ways to get a key from a presence log file -} +presenceLogs :: FilePath -> [Maybe Key] +presenceLogs f = + [ urlLogFileKey f + , locationLogFileKey f + ] + +{- Logs that are neither UUID based nor presence logs. -} +otherLogs :: [FilePath] +otherLogs = + [ numcopiesLog + , groupPreferredContentLog + ] + +uuidLog :: FilePath +uuidLog = "uuid.log" + +numcopiesLog :: FilePath +numcopiesLog = "numcopies.log" + +configLog :: FilePath +configLog = "config.log" + +remoteLog :: FilePath +remoteLog = "remote.log" + +trustLog :: FilePath +trustLog = "trust.log" + +groupLog :: FilePath +groupLog = "group.log" + +preferredContentLog :: FilePath +preferredContentLog = "preferred-content.log" + +requiredContentLog :: FilePath +requiredContentLog = "required-content.log" + +groupPreferredContentLog :: FilePath +groupPreferredContentLog = "group-preferred-content.log" + +scheduleLog :: FilePath +scheduleLog = "schedule.log" + +activityLog :: FilePath +activityLog = "activity.log" + +differenceLog :: FilePath +differenceLog = "difference.log" + +multicastLog :: FilePath +multicastLog = "multicast.log" + +exportLog :: FilePath +exportLog = "export.log" + +{- The pathname of the location log file for a given key. -} +locationLogFile :: GitConfig -> Key -> String +locationLogFile config key = branchHashDir config key keyFile key ++ ".log" + +{- Converts a pathname into a key if it's a location log. -} +locationLogFileKey :: FilePath -> Maybe Key +locationLogFileKey path + | ["remote", "web"] `isPrefixOf` splitDirectories dir = Nothing + | ext == ".log" = fileKey base + | otherwise = Nothing + where + (dir, file) = splitFileName path + (base, ext) = splitAt (length file - 4) file + +{- The filename of the url log for a given key. -} +urlLogFile :: GitConfig -> Key -> FilePath +urlLogFile config key = branchHashDir config key keyFile key ++ urlLogExt + +{- Old versions stored the urls elsewhere. -} +oldurlLogs :: GitConfig -> Key -> [FilePath] +oldurlLogs config key = + [ "remote/web" hdir key2file key ++ ".log" + , "remote/web" hdir keyFile key ++ ".log" + ] + where + hdir = branchHashDir config key + +urlLogExt :: String +urlLogExt = ".log.web" + +{- Converts a url log file into a key. + - (Does not work on oldurlLogs.) -} +urlLogFileKey :: FilePath -> Maybe Key +urlLogFileKey path + | ext == urlLogExt = fileKey base + | otherwise = Nothing + where + file = takeFileName path + (base, ext) = splitAt (length file - extlen) file + extlen = length urlLogExt + +{- Does not work on oldurllogs. -} +isUrlLog :: FilePath -> Bool +isUrlLog file = urlLogExt `isSuffixOf` file + +{- The filename of the remote state log for a given key. -} +remoteStateLogFile :: GitConfig -> Key -> FilePath +remoteStateLogFile config key = branchHashDir config key + keyFile key ++ remoteStateLogExt + +remoteStateLogExt :: String +remoteStateLogExt = ".log.rmt" + +isRemoteStateLog :: FilePath -> Bool +isRemoteStateLog path = remoteStateLogExt `isSuffixOf` path + +{- The filename of the chunk log for a given key. -} +chunkLogFile :: GitConfig -> Key -> FilePath +chunkLogFile config key = branchHashDir config key keyFile key ++ chunkLogExt + +chunkLogFileKey :: FilePath -> Maybe Key +chunkLogFileKey path + | ext == chunkLogExt = fileKey base + | otherwise = Nothing + where + file = takeFileName path + (base, ext) = splitAt (length file - extlen) file + extlen = length chunkLogExt + +chunkLogExt :: String +chunkLogExt = ".log.cnk" + +isChunkLog :: FilePath -> Bool +isChunkLog path = chunkLogExt `isSuffixOf` path + +{- The filename of the metadata log for a given key. -} +metaDataLogFile :: GitConfig -> Key -> FilePath +metaDataLogFile config key = branchHashDir config key keyFile key ++ metaDataLogExt + +metaDataLogExt :: String +metaDataLogExt = ".log.met" + +isMetaDataLog :: FilePath -> Bool +isMetaDataLog path = metaDataLogExt `isSuffixOf` path + +{- The filename of the remote metadata log for a given key. -} +remoteMetaDataLogFile :: GitConfig -> Key -> FilePath +remoteMetaDataLogFile config key = branchHashDir config key keyFile key ++ remoteMetaDataLogExt + +remoteMetaDataLogExt :: String +remoteMetaDataLogExt = ".log.rmet" + +isRemoteMetaDataLog :: FilePath -> Bool +isRemoteMetaDataLog path = remoteMetaDataLogExt `isSuffixOf` path diff --git a/Logs/Activity.hs b/Logs/Activity.hs new file mode 100644 index 0000000000..d7474704eb --- /dev/null +++ b/Logs/Activity.hs @@ -0,0 +1,35 @@ +{- git-annex activity log + - + - Copyright 2015 Joey Hess + - + - Licensed under the GNU GPL version 3 or higher. + -} + +module Logs.Activity ( + Log, + Activity(..), + recordActivity, + lastActivities, +) where + +import Annex.Common +import qualified Annex.Branch +import Logs +import Logs.UUIDBased + +data Activity = Fsck + deriving (Eq, Read, Show, Enum, Bounded) + +recordActivity :: Activity -> UUID -> Annex () +recordActivity act uuid = do + c <- liftIO currentVectorClock + Annex.Branch.change activityLog $ + showLog show . changeLog c uuid act . parseLog readish + +lastActivities :: Maybe Activity -> Annex (Log Activity) +lastActivities wantact = parseLog onlywanted <$> Annex.Branch.get activityLog + where + onlywanted s = case readish s of + Just a | wanted a -> Just a + _ -> Nothing + wanted a = maybe True (a ==) wantact diff --git a/Logs/Chunk.hs b/Logs/Chunk.hs new file mode 100644 index 0000000000..0a419716b8 --- /dev/null +++ b/Logs/Chunk.hs @@ -0,0 +1,54 @@ +{- Chunk logs. + - + - An object can be stored in chunked for on a remote; these logs keep + - track of the chunk size used, and the number of chunks. + - + - It's possible for a single object to be stored multiple times on the + - same remote using different chunk sizes. So, while this is a MapLog, it + - is not a normal UUIDBased log. Intead, it's a map from UUID and chunk + - size to number of chunks. + - + - Format: "timestamp uuid:chunksize chunkcount" + - + - Copyright 2014 Joey Hess + - + - Licensed under the GNU GPL version 3 or higher. + -} + +module Logs.Chunk ( + ChunkMethod(..), + ChunkSize, + ChunkCount, + chunksStored, + chunksRemoved, + getCurrentChunks, +) where + +import Annex.Common +import Logs +import Logs.MapLog +import qualified Annex.Branch +import Logs.Chunk.Pure +import qualified Annex + +import qualified Data.Map as M + +chunksStored :: UUID -> Key -> ChunkMethod -> ChunkCount -> Annex () +chunksStored u k chunkmethod chunkcount = do + c <- liftIO currentVectorClock + config <- Annex.getGitConfig + Annex.Branch.change (chunkLogFile config k) $ + showLog . changeMapLog c (u, chunkmethod) chunkcount . parseLog + +chunksRemoved :: UUID -> Key -> ChunkMethod -> Annex () +chunksRemoved u k chunkmethod = chunksStored u k chunkmethod 0 + +getCurrentChunks :: UUID -> Key -> Annex [(ChunkMethod, ChunkCount)] +getCurrentChunks u k = do + config <- Annex.getGitConfig + select . parseLog <$> Annex.Branch.get (chunkLogFile config k) + where + select = filter (\(_m, ct) -> ct > 0) + . map (\((_ku, m), l) -> (m, value l)) + . M.toList + . M.filterWithKey (\(ku, _m) _ -> ku == u) diff --git a/Logs/Chunk/Pure.hs b/Logs/Chunk/Pure.hs new file mode 100644 index 0000000000..7fbadb623e --- /dev/null +++ b/Logs/Chunk/Pure.hs @@ -0,0 +1,55 @@ +{- Chunk logs, pure operations. + - + - Copyright 2014 Joey Hess + - + - Licensed under the GNU GPL version 3 or higher. + -} + +module Logs.Chunk.Pure + ( ChunkMethod(..) + , ChunkSize + , ChunkCount + , ChunkLog + , parseLog + , showLog + ) where + +import Annex.Common +import Logs.MapLog +import Data.Int + +-- Currently chunks are all fixed size, but other chunking methods +-- may be added. +data ChunkMethod = FixedSizeChunks ChunkSize | UnknownChunks String + deriving (Ord, Eq, Show) + +type ChunkSize = Int64 + +-- 0 when chunks are no longer present +type ChunkCount = Integer + +type ChunkLog = MapLog (UUID, ChunkMethod) ChunkCount + +parseChunkMethod :: String -> ChunkMethod +parseChunkMethod s = maybe (UnknownChunks s) FixedSizeChunks (readish s) + +showChunkMethod :: ChunkMethod -> String +showChunkMethod (FixedSizeChunks sz) = show sz +showChunkMethod (UnknownChunks s) = s + +parseLog :: String -> ChunkLog +parseLog = parseMapLog fieldparser valueparser + where + fieldparser s = + let (u,m) = separate (== sep) s + in Just (toUUID u, parseChunkMethod m) + valueparser = readish + +showLog :: ChunkLog -> String +showLog = showMapLog fieldshower valueshower + where + fieldshower (u, m) = fromUUID u ++ sep : showChunkMethod m + valueshower = show + +sep :: Char +sep = ':' diff --git a/Logs/Config.hs b/Logs/Config.hs new file mode 100644 index 0000000000..7d1576b272 --- /dev/null +++ b/Logs/Config.hs @@ -0,0 +1,54 @@ +{- git-annex repository-global config log + - + - Copyright 2017 Joey Hess + - + - Licensed under the GNU GPL version 3 or higher. + -} + +module Logs.Config ( + ConfigName, + ConfigValue, + setGlobalConfig, + unsetGlobalConfig, + getGlobalConfig, + loadGlobalConfig, +) where + +import Annex.Common +import Logs +import Logs.MapLog +import qualified Annex.Branch + +import qualified Data.Map as M + +type ConfigName = String +type ConfigValue = String + +setGlobalConfig :: ConfigName -> ConfigValue -> Annex () +setGlobalConfig name new = do + curr <- getGlobalConfig name + when (curr /= Just new) $ + setGlobalConfig' name new + +setGlobalConfig' :: ConfigName -> ConfigValue -> Annex () +setGlobalConfig' name new = do + c <- liftIO currentVectorClock + Annex.Branch.change configLog $ + showMapLog id id . changeMapLog c name new . parseGlobalConfig + +unsetGlobalConfig :: ConfigName -> Annex () +unsetGlobalConfig name = do + curr <- getGlobalConfig name + when (curr /= Nothing) $ + setGlobalConfig' name "" -- set to empty string to unset + +-- Reads the global config log every time. +getGlobalConfig :: ConfigName -> Annex (Maybe ConfigValue) +getGlobalConfig name = M.lookup name <$> loadGlobalConfig + +parseGlobalConfig :: String -> MapLog ConfigName ConfigValue +parseGlobalConfig = parseMapLog Just Just + +loadGlobalConfig :: Annex (M.Map ConfigName ConfigValue) +loadGlobalConfig = M.filter (not . null) . simpleMap . parseGlobalConfig + <$> Annex.Branch.get configLog diff --git a/Logs/Difference.hs b/Logs/Difference.hs new file mode 100644 index 0000000000..e392d3f118 --- /dev/null +++ b/Logs/Difference.hs @@ -0,0 +1,39 @@ +{- git-annex difference log + - + - Copyright 2015 Joey Hess + - + - Licensed under the GNU GPL version 3 or higher. + -} + +module Logs.Difference ( + recordDifferences, + recordedDifferences, + recordedDifferencesFor, + module Logs.Difference.Pure +) where + +import qualified Data.Map as M + +import Annex.Common +import Types.Difference +import qualified Annex.Branch +import Logs +import Logs.UUIDBased +import Logs.Difference.Pure + +recordDifferences :: Differences -> UUID -> Annex () +recordDifferences ds@(Differences {}) uuid = do + c <- liftIO currentVectorClock + Annex.Branch.change differenceLog $ + showLog id . changeLog c uuid (showDifferences ds) . parseLog Just +recordDifferences UnknownDifferences _ = return () + +-- Map of UUIDs that have Differences recorded. +-- If a new version of git-annex has added a Difference this version +-- doesn't know about, it will contain UnknownDifferences. +recordedDifferences :: Annex (M.Map UUID Differences) +recordedDifferences = parseDifferencesLog <$> Annex.Branch.get differenceLog + +recordedDifferencesFor :: UUID -> Annex Differences +recordedDifferencesFor u = fromMaybe mempty . M.lookup u + <$> recordedDifferences diff --git a/Logs/Difference/Pure.hs b/Logs/Difference/Pure.hs new file mode 100644 index 0000000000..78a11d71f1 --- /dev/null +++ b/Logs/Difference/Pure.hs @@ -0,0 +1,24 @@ +{- git-annex difference log, pure functions + - + - Copyright 2015 Joey Hess + - + - Licensed under the GNU GPL version 3 or higher. + -} + +module Logs.Difference.Pure ( + allDifferences, + parseDifferencesLog, +) where + +import qualified Data.Map as M + +import Annex.Common +import Types.Difference +import Logs.UUIDBased + +parseDifferencesLog :: String -> (M.Map UUID Differences) +parseDifferencesLog = simpleMap . parseLog (Just . readDifferences) + +-- The sum of all recorded differences, across all UUIDs. +allDifferences :: M.Map UUID Differences -> Differences +allDifferences = mconcat . M.elems diff --git a/Logs/Export.hs b/Logs/Export.hs new file mode 100644 index 0000000000..6378881763 --- /dev/null +++ b/Logs/Export.hs @@ -0,0 +1,125 @@ +{- git-annex export log + - + - Copyright 2017 Joey Hess + - + - Licensed under the GNU GPL version 3 or higher. + -} + +module Logs.Export where + +import qualified Data.Map as M + +import Annex.Common +import qualified Annex.Branch +import qualified Git +import Git.Sha +import Git.FilePath +import Logs +import Logs.MapLog +import Annex.UUID + +data Exported = Exported + { exportedTreeish :: Git.Ref + , incompleteExportedTreeish :: [Git.Ref] + } + deriving (Eq, Show) + +data ExportParticipants = ExportParticipants + { exportFrom :: UUID + , exportTo :: UUID + } + deriving (Eq, Ord) + +data ExportChange = ExportChange + { oldTreeish :: [Git.Ref] + , newTreeish :: Git.Ref + } + +-- | Get what's been exported to a special remote. +-- +-- If the list contains multiple items, there was an export conflict, +-- and different trees were exported to the same special remote. +getExport :: UUID -> Annex [Exported] +getExport remoteuuid = nub . mapMaybe get . M.toList . simpleMap + . parseExportLog + <$> Annex.Branch.get exportLog + where + get (ep, exported) + | exportTo ep == remoteuuid = Just exported + | otherwise = Nothing + +-- | Record a change in what's exported to a special remote. +-- +-- This is called before an export begins uploading new files to the +-- remote, but after it's cleaned up any files that need to be deleted +-- from the old treeish. +-- +-- Any entries in the log for the oldTreeish will be updated to the +-- newTreeish. This way, when multiple repositories are exporting to +-- the same special remote, there's no conflict as long as they move +-- forward in lock-step. +-- +-- Also, the newTreeish is grafted into the git-annex branch. This is done +-- to ensure that it's available later. +recordExport :: UUID -> ExportChange -> Annex () +recordExport remoteuuid ec = do + c <- liftIO currentVectorClock + u <- getUUID + let ep = ExportParticipants { exportFrom = u, exportTo = remoteuuid } + let exported = Exported (newTreeish ec) [] + Annex.Branch.change exportLog $ + showExportLog + . changeMapLog c ep exported + . M.mapWithKey (updateothers c u) + . parseExportLog + where + updateothers c u ep le@(LogEntry _ exported@(Exported { exportedTreeish = t })) + | u == exportFrom ep || remoteuuid /= exportTo ep || t `notElem` oldTreeish ec = le + | otherwise = LogEntry c (exported { exportedTreeish = newTreeish ec }) + +-- | Record the beginning of an export, to allow cleaning up from +-- interrupted exports. +-- +-- This is called before any changes are made to the remote. +recordExportBeginning :: UUID -> Git.Ref -> Annex () +recordExportBeginning remoteuuid newtree = do + c <- liftIO currentVectorClock + u <- getUUID + let ep = ExportParticipants { exportFrom = u, exportTo = remoteuuid } + old <- fromMaybe (Exported emptyTree []) + . M.lookup ep . simpleMap + . parseExportLog + <$> Annex.Branch.get exportLog + let new = old { incompleteExportedTreeish = nub (newtree:incompleteExportedTreeish old) } + Annex.Branch.change exportLog $ + showExportLog + . changeMapLog c ep new + . parseExportLog + Annex.Branch.graftTreeish newtree (asTopFilePath "export.tree") + +parseExportLog :: String -> MapLog ExportParticipants Exported +parseExportLog = parseMapLog parseExportParticipants parseExported + +showExportLog :: MapLog ExportParticipants Exported -> String +showExportLog = showMapLog formatExportParticipants formatExported + +formatExportParticipants :: ExportParticipants -> String +formatExportParticipants ep = + fromUUID (exportFrom ep) ++ ':' : fromUUID (exportTo ep) + +parseExportParticipants :: String -> Maybe ExportParticipants +parseExportParticipants s = case separate (== ':') s of + ("",_) -> Nothing + (_,"") -> Nothing + (f,t) -> Just $ ExportParticipants + { exportFrom = toUUID f + , exportTo = toUUID t + } +formatExported :: Exported -> String +formatExported exported = unwords $ map Git.fromRef $ + exportedTreeish exported : incompleteExportedTreeish exported + +parseExported :: String -> Maybe Exported +parseExported s = case words s of + (et:it) -> Just $ Exported (Git.Ref et) (map Git.Ref it) + _ -> Nothing diff --git a/Logs/File.hs b/Logs/File.hs new file mode 100644 index 0000000000..6676dbb7ef --- /dev/null +++ b/Logs/File.hs @@ -0,0 +1,27 @@ +{- git-annex log files + - + - Copyright 2018 Joey Hess + - + - Licensed under the GNU GPL version 3 or higher. + -} + +module Logs.File where + +import Annex.Common +import Annex.Perms +import Utility.Tmp + +-- | Writes content to a file, replacing the file atomically, and +-- making the new file have whatever permissions the git repository is +-- configured to use. Creates the parent directory when necessary. +writeLogFile :: FilePath -> String -> Annex () +writeLogFile f c = go `catchNonAsync` \_e -> do + -- Most of the time, the directory will exist, so this is only + -- done if writing the file fails. + createAnnexDirectory (parentDir f) + go + where + go = viaTmp writelog f c + writelog f' c' = do + liftIO $ writeFile f' c' + setAnnexFilePerm f' diff --git a/Logs/FsckResults.hs b/Logs/FsckResults.hs new file mode 100644 index 0000000000..296847fa46 --- /dev/null +++ b/Logs/FsckResults.hs @@ -0,0 +1,51 @@ +{- git-annex fsck results log files + - + - Copyright 2013 Joey Hess + - + - Licensed under the GNU GPL version 3 or higher. + -} + +module Logs.FsckResults ( + writeFsckResults, + readFsckResults, + clearFsckResults, +) where + +import Annex.Common +import Git.Fsck +import Git.Types +import Logs.File + +import qualified Data.Set as S + +writeFsckResults :: UUID -> FsckResults -> Annex () +writeFsckResults u fsckresults = do + logfile <- fromRepo $ gitAnnexFsckResultsLog u + case fsckresults of + FsckFailed -> store S.empty False logfile + FsckFoundMissing s t + | S.null s -> liftIO $ nukeFile logfile + | otherwise -> store s t logfile + where + store s t logfile = writeLogFile logfile $ serialize s t + serialize s t = + let ls = map fromRef (S.toList s) + in if t + then unlines ("truncated":ls) + else unlines ls + +readFsckResults :: UUID -> Annex FsckResults +readFsckResults u = do + logfile <- fromRepo $ gitAnnexFsckResultsLog u + liftIO $ catchDefaultIO (FsckFoundMissing S.empty False) $ + deserialize . lines <$> readFile logfile + where + deserialize ("truncated":ls) = deserialize' ls True + deserialize ls = deserialize' ls False + deserialize' ls t = + let s = S.fromList $ map Ref ls + in if S.null s then FsckFailed else FsckFoundMissing s t + +clearFsckResults :: UUID -> Annex () +clearFsckResults = liftIO . nukeFile <=< fromRepo . gitAnnexFsckResultsLog + diff --git a/Logs/Group.hs b/Logs/Group.hs new file mode 100644 index 0000000000..b430627462 --- /dev/null +++ b/Logs/Group.hs @@ -0,0 +1,82 @@ +{- git-annex group log + - + - Copyright 2012 Joey Hess + - + - Licensed under the GNU GPL version 3 or higher. + -} + +module Logs.Group ( + groupLog, + groupChange, + groupSet, + lookupGroups, + groupMap, + groupMapLoad, + getStandardGroup, + inUnwantedGroup +) where + +import qualified Data.Map as M +import qualified Data.Set as S + +import Annex.Common +import Logs +import qualified Annex.Branch +import qualified Annex +import Logs.UUIDBased +import Types.Group +import Types.StandardGroups + +{- Returns the groups of a given repo UUID. -} +lookupGroups :: UUID -> Annex (S.Set Group) +lookupGroups u = (fromMaybe S.empty . M.lookup u) . groupsByUUID <$> groupMap + +{- Applies a set modifier to change the groups for a uuid in the groupLog. -} +groupChange :: UUID -> (S.Set Group -> S.Set Group) -> Annex () +groupChange uuid@(UUID _) modifier = do + curr <- lookupGroups uuid + c <- liftIO currentVectorClock + Annex.Branch.change groupLog $ + showLog (unwords . S.toList) . + changeLog c uuid (modifier curr) . + parseLog (Just . S.fromList . words) + + -- The changed group invalidates the preferred content cache. + Annex.changeState $ \s -> s + { Annex.groupmap = Nothing + , Annex.preferredcontentmap = Nothing + } +groupChange NoUUID _ = error "unknown UUID; cannot modify" + +groupSet :: UUID -> S.Set Group -> Annex () +groupSet u g = groupChange u (const g) + +{- The map is cached for speed. -} +groupMap :: Annex GroupMap +groupMap = maybe groupMapLoad return =<< Annex.getState Annex.groupmap + +{- Loads the map, updating the cache. -} +groupMapLoad :: Annex GroupMap +groupMapLoad = do + m <- makeGroupMap . simpleMap . + parseLog (Just . S.fromList . words) <$> + Annex.Branch.get groupLog + Annex.changeState $ \s -> s { Annex.groupmap = Just m } + return m + +makeGroupMap :: M.Map UUID (S.Set Group) -> GroupMap +makeGroupMap byuuid = GroupMap byuuid bygroup + where + bygroup = M.fromListWith S.union $ + concatMap explode $ M.toList byuuid + explode (u, s) = map (\g -> (g, S.singleton u)) (S.toList s) + +{- If a repository is in exactly one standard group, returns it. -} +getStandardGroup :: S.Set Group -> Maybe StandardGroup +getStandardGroup s = case mapMaybe toStandardGroup $ S.toList s of + [g] -> Just g + _ -> Nothing + +inUnwantedGroup :: UUID -> Annex Bool +inUnwantedGroup u = elem UnwantedGroup + . mapMaybe toStandardGroup . S.toList <$> lookupGroups u diff --git a/Logs/Line.hs b/Logs/Line.hs new file mode 100644 index 0000000000..a7e17190e2 --- /dev/null +++ b/Logs/Line.hs @@ -0,0 +1,51 @@ +{- + +The Glasgow Haskell Compiler License + +Copyright 2001, The University Court of the University of Glasgow. +All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are met: + +- Redistributions of source code must retain the above copyright notice, +this list of conditions and the following disclaimer. + +- Redistributions in binary form must reproduce the above copyright notice, +this list of conditions and the following disclaimer in the documentation +and/or other materials provided with the distribution. + +- Neither name of the University nor the names of its contributors may be +used to endorse or promote products derived from this software without +specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE UNIVERSITY COURT OF THE UNIVERSITY OF +GLASGOW AND THE CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, +INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND +FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE +UNIVERSITY COURT OF THE UNIVERSITY OF GLASGOW OR THE CONTRIBUTORS BE LIABLE +FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL +DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR +SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER +CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT +LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY +OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH +DAMAGE. + +-} + +module Logs.Line where + +-- This is the same as Data.List.lines, with \r added. +-- This works around some versions of git-annex which wrote \r +-- into git-annex branch files on Windows. Those \r's sometimes +-- accumulated over time, so a single line could end with multiple \r's +-- before the \n. +splitLines :: String -> [String] +splitLines "" = [] +splitLines s = cons (case break (\c -> c == '\n' || c == '\r') s of + (l, s') -> (l, case s' of + [] -> [] + _:s'' -> splitLines s'')) + where + cons ~(h, t) = h : t diff --git a/Logs/Location.hs b/Logs/Location.hs new file mode 100644 index 0000000000..725ce1a075 --- /dev/null +++ b/Logs/Location.hs @@ -0,0 +1,156 @@ +{-# LANGUAGE BangPatterns #-} + +{- git-annex location log + - + - git-annex keeps track of which repositories have the contents of annexed + - files. + - + - Repositories record their UUID and the date when they --get or --drop + - a value. + - + - Copyright 2010-2018 Joey Hess + - + - Licensed under the GNU GPL version 3 or higher. + -} + +module Logs.Location ( + LogStatus(..), + logStatus, + logChange, + loggedLocations, + loggedLocationsHistorical, + loggedLocationsRef, + isKnownKey, + checkDead, + setDead, + Unchecked, + finishCheck, + loggedKeys, + loggedKeysFor, + loggedKeysFor', +) where + +import Annex.Common +import qualified Annex.Branch +import Logs +import Logs.Presence +import Annex.UUID +import Annex.CatFile +import Annex.VectorClock +import Git.Types (RefDate, Ref) +import qualified Annex + +import Data.Time.Clock +import qualified Data.ByteString.Lazy.Char8 as L + +{- Log a change in the presence of a key's value in current repository. -} +logStatus :: Key -> LogStatus -> Annex () +logStatus key s = do + u <- getUUID + logChange key u s + +{- Log a change in the presence of a key's value in a repository. -} +logChange :: Key -> UUID -> LogStatus -> Annex () +logChange = logChange' logNow + +logChange' :: (LogStatus -> String -> Annex LogLine) -> Key -> UUID -> LogStatus -> Annex () +logChange' mklog key (UUID u) s = do + config <- Annex.getGitConfig + maybeAddLog (locationLogFile config key) =<< mklog s u +logChange' _ _ NoUUID _ = noop + +{- Returns a list of repository UUIDs that, according to the log, have + - the value of a key. -} +loggedLocations :: Key -> Annex [UUID] +loggedLocations = getLoggedLocations currentLogInfo + +{- Gets the location log on a particular date. -} +loggedLocationsHistorical :: RefDate -> Key -> Annex [UUID] +loggedLocationsHistorical = getLoggedLocations . historicalLogInfo + +{- Gets the locations contained in a git ref. -} +loggedLocationsRef :: Ref -> Annex [UUID] +loggedLocationsRef ref = map toUUID . getLog . L.unpack <$> catObject ref + +getLoggedLocations :: (FilePath -> Annex [String]) -> Key -> Annex [UUID] +getLoggedLocations getter key = do + config <- Annex.getGitConfig + map toUUID <$> getter (locationLogFile config key) + +{- Is there a location log for the key? True even for keys with no + - remaining locations. -} +isKnownKey :: Key -> Annex Bool +isKnownKey key = do + config <- Annex.getGitConfig + not . null <$> readLog (locationLogFile config key) + +{- For a key to be dead, all locations that have location status for the key + - must have InfoDead set. -} +checkDead :: Key -> Annex Bool +checkDead key = do + config <- Annex.getGitConfig + ls <- compactLog <$> readLog (locationLogFile config key) + return $! all (\l -> status l == InfoDead) ls + +{- Updates the log to say that a key is dead. + - + - Changes all logged lines for the key, in any location, that are + - currently InfoMissing, to be InfoDead. + -} +setDead :: Key -> Annex () +setDead key = do + config <- Annex.getGitConfig + let logfile = locationLogFile config key + ls <- compactLog <$> readLog logfile + mapM_ (go logfile) (filter (\l -> status l == InfoMissing) ls) + where + go logfile l = addLog logfile $ setDead' l + +{- Note that the timestamp in the log is updated minimally, so that this + - can be overruled by other location log changes. -} +setDead' :: LogLine -> LogLine +setDead' l = l + { status = InfoDead + , date = case date l of + VectorClock c -> VectorClock $ + c + realToFrac (picosecondsToDiffTime 1) + Unknown -> Unknown + } + +data Unchecked a = Unchecked (Annex (Maybe a)) + +finishCheck :: Unchecked a -> Annex (Maybe a) +finishCheck (Unchecked a) = a + +{- Finds all keys that have location log information. + - (There may be duplicate keys in the list.) + - + - Keys that have been marked as dead are not included. + -} +loggedKeys :: Annex [Unchecked Key] +loggedKeys = loggedKeys' (not <$$> checkDead) + +loggedKeys' :: (Key -> Annex Bool) -> Annex [Unchecked Key] +loggedKeys' check = mapMaybe (defercheck <$$> locationLogFileKey) + <$> Annex.Branch.files + where + defercheck k = Unchecked $ ifM (check k) + ( return (Just k) + , return Nothing + ) + +{- Finds all keys that have location log information indicating + - they are present in the specified repository. + - + - This does not stream well; use loggedKeysFor' for lazy streaming. + -} +loggedKeysFor :: UUID -> Annex [Key] +loggedKeysFor u = catMaybes <$> (mapM finishCheck =<< loggedKeysFor' u) + +loggedKeysFor' :: UUID -> Annex [Unchecked Key] +loggedKeysFor' u = loggedKeys' isthere + where + isthere k = do + us <- loggedLocations k + let !there = u `elem` us + return there diff --git a/Logs/MapLog.hs b/Logs/MapLog.hs new file mode 100644 index 0000000000..1bc024e2cc --- /dev/null +++ b/Logs/MapLog.hs @@ -0,0 +1,79 @@ +{-# LANGUAGE CPP #-} + +{- git-annex Map log + - + - This is used to store a Map, in a way that can be union merged. + - + - A line of the log will look like: "timestamp field value" + - + - Copyright 2014 Joey Hess + - + - Licensed under the GNU GPL version 3 or higher. + -} + +module Logs.MapLog ( + module Logs.MapLog, + VectorClock, + currentVectorClock, +) where + +import Common +import Annex.VectorClock +import Logs.Line + +import qualified Data.Map.Strict as M + +data LogEntry v = LogEntry + { changed :: VectorClock + , value :: v + } deriving (Eq) + +type MapLog f v = M.Map f (LogEntry v) + +showMapLog :: (f -> String) -> (v -> String) -> MapLog f v -> String +showMapLog fieldshower valueshower = unlines . map showpair . M.toList + where + showpair (f, LogEntry (VectorClock c) v) = + unwords [show c, fieldshower f, valueshower v] + showpair (f, LogEntry Unknown v) = + unwords ["0", fieldshower f, valueshower v] + +parseMapLog :: Ord f => (String -> Maybe f) -> (String -> Maybe v) -> String -> MapLog f v +parseMapLog fieldparser valueparser = M.fromListWith best . mapMaybe parse . splitLines + where + parse line = do + let (sc, rest) = splitword line + (sf, sv) = splitword rest + c <- parseVectorClock sc + f <- fieldparser sf + v <- valueparser sv + Just (f, LogEntry c v) + splitword = separate (== ' ') + +changeMapLog :: Ord f => VectorClock -> f -> v -> MapLog f v -> MapLog f v +changeMapLog c f v = M.insert f $ LogEntry c v + +{- Only add an LogEntry if it's newer (or at least as new as) than any + - existing LogEntry for a field. -} +addMapLog :: Ord f => f -> LogEntry v -> MapLog f v -> MapLog f v +addMapLog = M.insertWith best + +{- Converts a MapLog into a simple Map without the timestamp information. + - This is a one-way trip, but useful for code that never needs to change + - the log. -} +simpleMap :: MapLog f v -> M.Map f v +simpleMap = M.map value + +best :: LogEntry v -> LogEntry v -> LogEntry v +best new old + | changed old > changed new = old + | otherwise = new + +prop_addMapLog_sane :: Bool +prop_addMapLog_sane = newWins && newestWins + where + newWins = addMapLog ("foo") (LogEntry (VectorClock 1) "new") l == l2 + newestWins = addMapLog ("foo") (LogEntry (VectorClock 1) "newest") l2 /= l2 + + l = M.fromList [("foo", LogEntry (VectorClock 0) "old")] + l2 = M.fromList [("foo", LogEntry (VectorClock 1) "new")] diff --git a/Logs/MetaData.hs b/Logs/MetaData.hs new file mode 100644 index 0000000000..09e429cd23 --- /dev/null +++ b/Logs/MetaData.hs @@ -0,0 +1,152 @@ +{- git-annex general metadata storage log and per-remote metadata storage log. + - + - A line of the log will look like "timestamp field [+-]value [...]" + - + - (In the per-remote log, each field is prefixed with "uuid:") + - + - Note that unset values are preserved. Consider this case: + - + - We have: + - + - 100 foo +x + - 200 foo -x + - + - An unmerged remote has: + - + - 150 foo +x + - + - After union merge, because the foo -x was preserved, we know that + - after the other remote redundantly set foo +x, it was unset, + - and so foo currently has no value. + - + - + - Licensed under the GNU GPL version 3 or higher. + -} + +module Logs.MetaData ( + getCurrentMetaData, + getCurrentRemoteMetaData, + addMetaData, + addRemoteMetaData, + addMetaDataClocked, + currentMetaData, + copyMetaData, +) where + +import Annex.Common +import Types.MetaData +import Annex.MetaData.StandardFields +import Annex.VectorClock +import qualified Annex.Branch +import qualified Annex +import Logs +import Logs.TimeStamp +import Logs.MetaData.Pure + +import qualified Data.Set as S +import qualified Data.Map as M + +{- Go through the log from oldest to newest, and combine it all + - into a single MetaData representing the current state. + - + - Automatically generates a lastchanged metadata for each field that's + - currently set, based on timestamps in the log. + -} +getCurrentMetaData :: Key -> Annex MetaData +getCurrentMetaData = getCurrentMetaData' metaDataLogFile + +getCurrentMetaData' :: (GitConfig -> Key -> FilePath) -> Key -> Annex MetaData +getCurrentMetaData' getlogfile k = do + config <- Annex.getGitConfig + ls <- S.toAscList <$> readLog (getlogfile config k) + let loggedmeta = logToCurrentMetaData ls + return $ currentMetaData $ unionMetaData loggedmeta + (lastchanged ls loggedmeta) + where + lastchanged [] _ = emptyMetaData + lastchanged ls (MetaData currentlyset) = + let m = foldl' (flip M.union) M.empty (map genlastchanged ls) + in MetaData $ + -- Add a overall lastchanged using the oldest log + -- item (log is in ascending order). + M.insert lastChangedField (lastchangedval $ Prelude.last ls) $ + M.mapKeys mkLastChangedField $ + -- Only include fields that are currently set. + m `M.intersection` currentlyset + -- Makes each field have the timestamp as its value. + genlastchanged l = + let MetaData m = value l + ts = lastchangedval l + in M.map (const ts) m + lastchangedval l = S.singleton $ toMetaValue $ showts $ + case changed l of + VectorClock t -> t + Unknown -> 0 + showts = formatPOSIXTime "%F@%H-%M-%S" + +getCurrentRemoteMetaData :: UUID -> Key -> Annex RemoteMetaData +getCurrentRemoteMetaData u k = extractRemoteMetaData u <$> + getCurrentMetaData' remoteMetaDataLogFile k + +{- Adds in some metadata, which can override existing values, or unset + - them, but otherwise leaves any existing metadata as-is. -} +addMetaData :: Key -> MetaData -> Annex () +addMetaData = addMetaData' metaDataLogFile + +addMetaData' :: (GitConfig -> Key -> FilePath) -> Key -> MetaData -> Annex () +addMetaData' getlogfile k metadata = + addMetaDataClocked' getlogfile k metadata =<< liftIO currentVectorClock + +{- Reusing the same VectorClock when making changes to the metadata + - of multiple keys is a nice optimisation. The same metadata lines + - will tend to be generated across the different log files, and so + - git will be able to pack the data more efficiently. -} +addMetaDataClocked :: Key -> MetaData -> VectorClock -> Annex () +addMetaDataClocked = addMetaDataClocked' metaDataLogFile + +addMetaDataClocked' :: (GitConfig -> Key -> FilePath) -> Key -> MetaData -> VectorClock -> Annex () +addMetaDataClocked' getlogfile k d@(MetaData m) c + | d == emptyMetaData = noop + | otherwise = do + config <- Annex.getGitConfig + Annex.Branch.change (getlogfile config k) $ + showLog . simplifyLog + . S.insert (LogEntry c metadata) + . parseLog + where + metadata = MetaData $ M.filterWithKey (\f _ -> not (isLastChangedField f)) m + +addRemoteMetaData :: Key -> RemoteMetaData -> Annex () +addRemoteMetaData k m = do + addMetaData' remoteMetaDataLogFile k (fromRemoteMetaData m) + +getMetaDataLog :: Key -> Annex (Log MetaData) +getMetaDataLog key = do + config <- Annex.getGitConfig + readLog $ metaDataLogFile config key + +{- Copies the metadata from the old key to the new key. + - + - The exact content of the metadata file is copied, so that the timestamps + - remain the same, and because this is more space-efficient in the git + - repository. + - + - Any metadata already attached to the new key is not preserved. + - + - Returns True when metadata was copied. + -} +copyMetaData :: Key -> Key -> Annex Bool +copyMetaData oldkey newkey + | oldkey == newkey = return False + | otherwise = do + l <- getMetaDataLog oldkey + if logToCurrentMetaData (S.toAscList l) == emptyMetaData + then return False + else do + config <- Annex.getGitConfig + Annex.Branch.change (metaDataLogFile config newkey) $ + const $ showLog l + return True + +readLog :: FilePath -> Annex (Log MetaData) +readLog = parseLog <$$> Annex.Branch.get diff --git a/Logs/MetaData/Pure.hs b/Logs/MetaData/Pure.hs new file mode 100644 index 0000000000..6cfdf19cd8 --- /dev/null +++ b/Logs/MetaData/Pure.hs @@ -0,0 +1,111 @@ +{- git-annex metadata log, pure operations + - + - Copyright 2014-2018 Joey Hess + - + - Licensed under the GNU GPL version 3 or higher. + -} + +{-# OPTIONS_GHC -fno-warn-orphans #-} + +module Logs.MetaData.Pure ( + Log, + LogEntry(..), + parseLog, + showLog, + logToCurrentMetaData, + simplifyLog, + filterRemoteMetaData, + filterOutEmpty, +) where + +import Types.MetaData +import Logs.SingleValue.Pure +import Types.UUID + +import qualified Data.Set as S +import qualified Data.Map.Strict as M + +instance SingleValueSerializable MetaData where + serialize = Types.MetaData.serialize + deserialize = Types.MetaData.deserialize + +logToCurrentMetaData :: [LogEntry MetaData] -> MetaData +logToCurrentMetaData = currentMetaData . combineMetaData . map value + +{- Simplify a log, removing historical values that are no longer + - needed. + - + - This is not as simple as just making a single log line with the newest + - state of all metadata. Consider this case: + - + - We have: + - + - 100 foo +x bar +y + - 200 foo -x + - + - An unmerged remote has: + - + - 150 bar -y baz +w + - + - If what we have were simplified to "200 foo -x bar +y" then when the line + - from the remote became available, it would be older than the simplified + - line, and its change to bar would not take effect. That is wrong. + - + - Instead, simplify it to: + - + - 100 bar +y + - 200 foo -x + - + - (Note that this ends up with the same number of lines as the + - unsimplified version, so there's really no point in updating + - the log to this version. Doing so would only add data to git, + - with little benefit.) + - + - Now merging with the remote yields: + - + - 100 bar +y + - 150 bar -y baz +w + - 200 foo -x + - + - Simplifying again: + - + - 150 bar +z baz +w + - 200 foo -x + -} +simplifyLog :: Log MetaData -> Log MetaData +simplifyLog s = case sl of + (newest:rest) -> + let sl' = go [newest] (value newest) rest + in if length sl' < length sl + then S.fromList sl' + else s + _ -> s + where + sl = S.toDescList s + + go c _ [] = c + go c newer (l:ls) + | unique == emptyMetaData = go c newer ls + | otherwise = go (l { value = unique } : c) + (unionMetaData unique newer) ls + where + older = value l + unique = older `differenceMetaData` newer + +{- Filters per-remote metadata on the basis of UUID. + - + - Note that the LogEntry's clock is left the same, so this should not be + - used except for in a transition. + -} +filterRemoteMetaData :: (UUID -> Bool) -> Log MetaData -> Log MetaData +filterRemoteMetaData p = S.map go + where + go l@(LogEntry { value = MetaData m }) = + l { value = MetaData $ M.filterWithKey fil m } + fil f _v = case splitRemoteMetaDataField f of + Just (u, _) -> p u + Nothing -> True + +{- Filters out log lines that are empty. -} +filterOutEmpty :: Log MetaData -> Log MetaData +filterOutEmpty = S.filter $ \l -> value l /= emptyMetaData diff --git a/Logs/Multicast.hs b/Logs/Multicast.hs new file mode 100644 index 0000000000..8deb2800be --- /dev/null +++ b/Logs/Multicast.hs @@ -0,0 +1,31 @@ +{- git-annex multicast fingerprint log + - + - Copyright 2017 Joey Hess + - + - Licensed under the GNU GPL version 3 or higher. + -} + +module Logs.Multicast ( + Fingerprint(..), + recordFingerprint, + knownFingerPrints, +) where + +import Annex.Common +import qualified Annex.Branch +import Logs +import Logs.UUIDBased + +import qualified Data.Map as M + +newtype Fingerprint = Fingerprint String + deriving (Eq, Read, Show) + +recordFingerprint :: Fingerprint -> UUID -> Annex () +recordFingerprint fp uuid = do + c <- liftIO currentVectorClock + Annex.Branch.change multicastLog $ + showLog show . changeLog c uuid fp . parseLog readish + +knownFingerPrints :: Annex (M.Map UUID Fingerprint) +knownFingerPrints = simpleMap . parseLog readish <$> Annex.Branch.get activityLog diff --git a/Logs/NumCopies.hs b/Logs/NumCopies.hs new file mode 100644 index 0000000000..7593d6c037 --- /dev/null +++ b/Logs/NumCopies.hs @@ -0,0 +1,41 @@ +{- git-annex numcopies log + - + - Copyright 2014 Joey Hess + - + - Licensed under the GNU GPL version 3 or higher. + -} + +{-# OPTIONS_GHC -fno-warn-orphans #-} + +module Logs.NumCopies ( + setGlobalNumCopies, + getGlobalNumCopies, + globalNumCopiesLoad, +) where + +import Annex.Common +import qualified Annex +import Types.NumCopies +import Logs +import Logs.SingleValue + +instance SingleValueSerializable NumCopies where + serialize (NumCopies n) = show n + deserialize = NumCopies <$$> readish + +setGlobalNumCopies :: NumCopies -> Annex () +setGlobalNumCopies new = do + curr <- getGlobalNumCopies + when (curr /= Just new) $ + setLog numcopiesLog new + +{- Value configured in the numcopies log. Cached for speed. -} +getGlobalNumCopies :: Annex (Maybe NumCopies) +getGlobalNumCopies = maybe globalNumCopiesLoad (return . Just) + =<< Annex.getState Annex.globalnumcopies + +globalNumCopiesLoad :: Annex (Maybe NumCopies) +globalNumCopiesLoad = do + v <- getLog numcopiesLog + Annex.changeState $ \s -> s { Annex.globalnumcopies = v } + return v diff --git a/Logs/PreferredContent.hs b/Logs/PreferredContent.hs new file mode 100644 index 0000000000..ff23485863 --- /dev/null +++ b/Logs/PreferredContent.hs @@ -0,0 +1,160 @@ +{- git-annex preferred content matcher configuration + - + - Copyright 2012-2014 Joey Hess + - + - Licensed under the GNU GPL version 3 or higher. + -} + +module Logs.PreferredContent ( + preferredContentSet, + requiredContentSet, + groupPreferredContentSet, + isPreferredContent, + isRequiredContent, + preferredContentMap, + preferredContentMapRaw, + requiredContentMap, + requiredContentMapRaw, + groupPreferredContentMapRaw, + checkPreferredContentExpression, + setStandardGroup, + defaultStandardGroup, + preferredRequiredMapsLoad, + prop_standardGroups_parse, +) where + +import qualified Data.Map as M +import qualified Data.Set as S +import Data.Either + +import Annex.Common +import Logs.PreferredContent.Raw +import qualified Annex.Branch +import qualified Annex +import Logs +import Logs.UUIDBased +import Utility.Matcher hiding (tokens) +import Annex.FileMatcher +import Annex.UUID +import Types.Group +import Types.Remote (RemoteConfig) +import Logs.Group +import Logs.Remote +import Types.FileMatcher +import Types.StandardGroups +import Limit + +{- Checks if a file is preferred content for the specified repository + - (or the current repository if none is specified). -} +isPreferredContent :: Maybe UUID -> AssumeNotPresent -> Maybe Key -> AssociatedFile -> Bool -> Annex Bool +isPreferredContent = checkMap preferredContentMap + +isRequiredContent :: Maybe UUID -> AssumeNotPresent -> Maybe Key -> AssociatedFile -> Bool -> Annex Bool +isRequiredContent = checkMap requiredContentMap + +checkMap :: Annex (FileMatcherMap Annex) -> Maybe UUID -> AssumeNotPresent -> Maybe Key -> AssociatedFile -> Bool -> Annex Bool +checkMap getmap mu notpresent mkey afile d = do + u <- maybe getUUID return mu + m <- getmap + case M.lookup u m of + Nothing -> return d + Just matcher -> checkMatcher matcher mkey afile notpresent (return d) (return d) + +preferredContentMap :: Annex (FileMatcherMap Annex) +preferredContentMap = maybe (fst <$> preferredRequiredMapsLoad) return + =<< Annex.getState Annex.preferredcontentmap + +requiredContentMap :: Annex (FileMatcherMap Annex) +requiredContentMap = maybe (snd <$> preferredRequiredMapsLoad) return + =<< Annex.getState Annex.requiredcontentmap + +preferredRequiredMapsLoad :: Annex (FileMatcherMap Annex, FileMatcherMap Annex) +preferredRequiredMapsLoad = do + groupmap <- groupMap + configmap <- readRemoteLog + let genmap l gm = simpleMap + . parseLogWithUUID ((Just .) . makeMatcher groupmap configmap gm) + <$> Annex.Branch.get l + pc <- genmap preferredContentLog =<< groupPreferredContentMapRaw + rc <- genmap requiredContentLog M.empty + -- Required content is implicitly also preferred content, so + -- combine. + let m = M.unionWith combineMatchers pc rc + Annex.changeState $ \s -> s + { Annex.preferredcontentmap = Just m + , Annex.requiredcontentmap = Just rc + } + return (m, rc) + +{- This intentionally never fails, even on unparsable expressions, + - because the configuration is shared among repositories and newer + - versions of git-annex may add new features. -} +makeMatcher + :: GroupMap + -> M.Map UUID RemoteConfig + -> M.Map Group PreferredContentExpression + -> UUID + -> PreferredContentExpression + -> FileMatcher Annex +makeMatcher groupmap configmap groupwantedmap u = go True True + where + go expandstandard expandgroupwanted expr + | null (lefts tokens) = generate $ rights tokens + | otherwise = unknownMatcher u + where + tokens = preferredContentParser matchstandard matchgroupwanted (pure groupmap) configmap (Just u) expr + matchstandard + | expandstandard = maybe (unknownMatcher u) (go False False) + (standardPreferredContent <$> getStandardGroup mygroups) + | otherwise = unknownMatcher u + matchgroupwanted + | expandgroupwanted = maybe (unknownMatcher u) (go True False) + (groupwanted mygroups) + | otherwise = unknownMatcher u + mygroups = fromMaybe S.empty (u `M.lookup` groupsByUUID groupmap) + groupwanted s = case M.elems $ M.filterWithKey (\k _ -> S.member k s) groupwantedmap of + [pc] -> Just pc + _ -> Nothing + +{- When a preferred content expression cannot be parsed, but is already + - in the log (eg, put there by a newer version of git-annex), + - the fallback behavior is to match only files that are currently present. + - + - This avoid unwanted/expensive changes to the content, until the problem + - is resolved. -} +unknownMatcher :: UUID -> FileMatcher Annex +unknownMatcher u = generate [present] + where + present = Operation $ limitPresent (Just u) + +{- Checks if an expression can be parsed, if not returns Just error -} +checkPreferredContentExpression :: PreferredContentExpression -> Maybe String +checkPreferredContentExpression expr = case parsedToMatcher tokens of + Left e -> Just e + Right _ -> Nothing + where + tokens = preferredContentParser matchAll matchAll (pure emptyGroupMap) M.empty Nothing expr + +{- Puts a UUID in a standard group, and sets its preferred content to use + - the standard expression for that group (unless preferred content is + - already set). -} +setStandardGroup :: UUID -> StandardGroup -> Annex () +setStandardGroup u g = do + groupSet u $ S.singleton $ fromStandardGroup g + unlessM (isJust . M.lookup u <$> preferredContentMap) $ + preferredContentSet u "standard" + +{- Avoids overwriting the UUID's standard group or preferred content + - when it's already been configured. -} +defaultStandardGroup :: UUID -> StandardGroup -> Annex () +defaultStandardGroup u g = + unlessM (hasgroup <||> haspc) $ + setStandardGroup u g + where + hasgroup = not . S.null <$> lookupGroups u + haspc = isJust . M.lookup u <$> preferredContentMap + +prop_standardGroups_parse :: Bool +prop_standardGroups_parse = + all (isNothing . checkPreferredContentExpression . standardPreferredContent) + [ minBound .. maxBound] diff --git a/Logs/PreferredContent/Raw.hs b/Logs/PreferredContent/Raw.hs new file mode 100644 index 0000000000..8df5edd43b --- /dev/null +++ b/Logs/PreferredContent/Raw.hs @@ -0,0 +1,61 @@ +{- unparsed preferred content expressions + - + - Copyright 2012-2014 Joey Hess + - + - Licensed under the GNU GPL version 3 or higher. + -} + +module Logs.PreferredContent.Raw where + +import Annex.Common +import qualified Annex.Branch +import qualified Annex +import Logs +import Logs.UUIDBased +import Logs.MapLog +import Types.StandardGroups +import Types.Group + +import qualified Data.Map as M + +{- Changes the preferred content configuration of a remote. -} +preferredContentSet :: UUID -> PreferredContentExpression -> Annex () +preferredContentSet = setLog preferredContentLog + +requiredContentSet :: UUID -> PreferredContentExpression -> Annex () +requiredContentSet = setLog requiredContentLog + +setLog :: FilePath -> UUID -> PreferredContentExpression -> Annex () +setLog logfile uuid@(UUID _) val = do + c <- liftIO currentVectorClock + Annex.Branch.change logfile $ + showLog id + . changeLog c uuid val + . parseLog Just + Annex.changeState $ \s -> s + { Annex.preferredcontentmap = Nothing + , Annex.requiredcontentmap = Nothing + } +setLog _ NoUUID _ = error "unknown UUID; cannot modify" + +{- Changes the preferred content configuration of a group. -} +groupPreferredContentSet :: Group -> PreferredContentExpression -> Annex () +groupPreferredContentSet g val = do + c <- liftIO currentVectorClock + Annex.Branch.change groupPreferredContentLog $ + showMapLog id id + . changeMapLog c g val + . parseMapLog Just Just + Annex.changeState $ \s -> s { Annex.preferredcontentmap = Nothing } + +preferredContentMapRaw :: Annex (M.Map UUID PreferredContentExpression) +preferredContentMapRaw = simpleMap . parseLog Just + <$> Annex.Branch.get preferredContentLog + +requiredContentMapRaw :: Annex (M.Map UUID PreferredContentExpression) +requiredContentMapRaw = simpleMap . parseLog Just + <$> Annex.Branch.get requiredContentLog + +groupPreferredContentMapRaw :: Annex (M.Map Group PreferredContentExpression) +groupPreferredContentMapRaw = simpleMap . parseMapLog Just Just + <$> Annex.Branch.get groupPreferredContentLog diff --git a/Logs/Presence.hs b/Logs/Presence.hs new file mode 100644 index 0000000000..382a5a302d --- /dev/null +++ b/Logs/Presence.hs @@ -0,0 +1,71 @@ +{- git-annex presence log + - + - This is used to store presence information in the git-annex branch in + - a way that can be union merged. + - + - A line of the log will look like: "date N INFO" + - Where N=1 when the INFO is present, 0 otherwise. + - + - Copyright 2010-2014 Joey Hess + - + - Licensed under the GNU GPL version 3 or higher. + -} + +module Logs.Presence ( + module X, + addLog, + maybeAddLog, + readLog, + logNow, + currentLog, + currentLogInfo, + historicalLogInfo, +) where + +import Logs.Presence.Pure as X +import Annex.Common +import Annex.VectorClock +import qualified Annex.Branch +import Git.Types (RefDate) + +{- Adds a LogLine to the log, removing any LogLines that are obsoleted by + - adding it. -} +addLog :: FilePath -> LogLine -> Annex () +addLog file line = Annex.Branch.change file $ \s -> + showLog $ compactLog (line : parseLog s) + +{- When a LogLine already exists with the same status and info, but an + - older timestamp, that LogLine is preserved, rather than updating the log + - with a newer timestamp. + -} +maybeAddLog :: FilePath -> LogLine -> Annex () +maybeAddLog file line = Annex.Branch.maybeChange file $ \s -> do + m <- insertNewStatus line $ logMap $ parseLog s + return $ showLog $ mapLog m + +{- Reads a log file. + - Note that the LogLines returned may be in any order. -} +readLog :: FilePath -> Annex [LogLine] +readLog = parseLog <$$> Annex.Branch.get + +{- Generates a new LogLine with the current time. -} +logNow :: LogStatus -> String -> Annex LogLine +logNow s i = do + c <- liftIO currentVectorClock + return $ LogLine c s i + +{- Reads a log and returns only the info that is still in effect. -} +currentLogInfo :: FilePath -> Annex [String] +currentLogInfo file = map info <$> currentLog file + +currentLog :: FilePath -> Annex [LogLine] +currentLog file = filterPresent <$> readLog file + +{- Reads a historical version of a log and returns the info that was in + - effect at that time. + - + - The date is formatted as shown in gitrevisions man page. + -} +historicalLogInfo :: RefDate -> FilePath -> Annex [String] +historicalLogInfo refdate file = map info . filterPresent . parseLog + <$> Annex.Branch.getHistorical refdate file diff --git a/Logs/Presence/Pure.hs b/Logs/Presence/Pure.hs new file mode 100644 index 0000000000..8fc1541776 --- /dev/null +++ b/Logs/Presence/Pure.hs @@ -0,0 +1,109 @@ +{- git-annex presence log, pure operations + - + - Copyright 2010-2013 Joey Hess + - + - Licensed under the GNU GPL version 3 or higher. + -} + +module Logs.Presence.Pure where + +import Annex.Common +import Annex.VectorClock +import Logs.Line +import Utility.QuickCheck + +import qualified Data.Map as M + +data LogLine = LogLine + { date :: VectorClock + , status :: LogStatus + , info :: String + } deriving (Eq) + +instance Show LogLine where + show l = "LogLine " ++ formatVectorClock (date l) ++ show (status l) ++ " " ++ show (info l) + +data LogStatus = InfoPresent | InfoMissing | InfoDead + deriving (Eq, Show, Bounded, Enum) + +{- Parses a log file. Unparseable lines are ignored. -} +parseLog :: String -> [LogLine] +parseLog = mapMaybe parseline . splitLines + where + parseline l = LogLine + <$> parseVectorClock c + <*> parseStatus s + <*> pure rest + where + (c, pastc) = separate (== ' ') l + (s, rest) = separate (== ' ') pastc + +parseStatus :: String -> Maybe LogStatus +parseStatus "1" = Just InfoPresent +parseStatus "0" = Just InfoMissing +parseStatus "X" = Just InfoDead +parseStatus _ = Nothing + +{- Generates a log file. -} +showLog :: [LogLine] -> String +showLog = unlines . map genline + where + genline (LogLine c s i) = unwords [formatVectorClock c, genstatus s, i] + genstatus InfoPresent = "1" + genstatus InfoMissing = "0" + genstatus InfoDead = "X" + +{- Given a log, returns only the info that is are still in effect. -} +getLog :: String -> [String] +getLog = map info . filterPresent . parseLog + +{- Returns the info from LogLines that are in effect. -} +filterPresent :: [LogLine] -> [LogLine] +filterPresent = filter (\l -> InfoPresent == status l) . compactLog + +{- Compacts a set of logs, returning a subset that contains the current + - status. -} +compactLog :: [LogLine] -> [LogLine] +compactLog = mapLog . logMap + +type LogMap = M.Map String LogLine + +mapLog :: LogMap -> [LogLine] +mapLog = M.elems + +logMap :: [LogLine] -> LogMap +logMap = foldr insertNewerLogLine M.empty + +insertBetter :: (LogLine -> Bool) -> LogLine -> LogMap -> Maybe LogMap +insertBetter betterthan l m + | better = Just (M.insert i l m) + | otherwise = Nothing + where + better = maybe True betterthan (M.lookup i m) + i = info l + +{- Inserts a log into a map of logs, if the log has newer + - information than the other logs in the map for the same info. -} +insertNewerLogLine :: LogLine -> LogMap -> LogMap +insertNewerLogLine l m = fromMaybe m $ insertBetter newer l m + where + newer l' = date l' <= date l + +{- Inserts the log unless there's already one in the map with + - the same status for its info, in which case there's no need to + - change anything, to avoid log churn. -} +insertNewStatus :: LogLine -> LogMap -> Maybe LogMap +insertNewStatus l m = insertBetter diffstatus l m + where + diffstatus l' = status l' /= status l + +instance Arbitrary LogLine where + arbitrary = LogLine + <$> arbitrary + <*> elements [minBound..maxBound] + <*> arbitrary `suchThat` + (\c -> '\n' `notElem` c && '\r' `notElem` c) + +prop_parse_show_log :: [LogLine] -> Bool +prop_parse_show_log l = parseLog (showLog l) == l + diff --git a/Logs/Remote.hs b/Logs/Remote.hs new file mode 100644 index 0000000000..47a339a5f0 --- /dev/null +++ b/Logs/Remote.hs @@ -0,0 +1,96 @@ +{- git-annex remote log + - + - Copyright 2011 Joey Hess + - + - Licensed under the GNU GPL version 3 or higher. + -} + +module Logs.Remote ( + remoteLog, + readRemoteLog, + configSet, + keyValToConfig, + configToKeyVal, + showConfig, + parseConfig, + + prop_isomorphic_configEscape, + prop_parse_show_Config, +) where + +import Annex.Common +import qualified Annex.Branch +import Types.Remote +import Logs +import Logs.UUIDBased + +import qualified Data.Map as M +import Data.Char + +{- Adds or updates a remote's config in the log. -} +configSet :: UUID -> RemoteConfig -> Annex () +configSet u cfg = do + c <- liftIO currentVectorClock + Annex.Branch.change remoteLog $ + showLog showConfig . changeLog c u cfg . parseLog parseConfig + +{- Map of remotes by uuid containing key/value config maps. -} +readRemoteLog :: Annex (M.Map UUID RemoteConfig) +readRemoteLog = simpleMap . parseLog parseConfig <$> Annex.Branch.get remoteLog + +parseConfig :: String -> Maybe RemoteConfig +parseConfig = Just . keyValToConfig . words + +showConfig :: RemoteConfig -> String +showConfig = unwords . configToKeyVal + +{- Given Strings like "key=value", generates a RemoteConfig. -} +keyValToConfig :: [String] -> RemoteConfig +keyValToConfig ws = M.fromList $ map (/=/) ws + where + (/=/) s = (k, v) + where + k = takeWhile (/= '=') s + v = configUnEscape $ drop (1 + length k) s + +configToKeyVal :: M.Map String String -> [String] +configToKeyVal m = map toword $ sort $ M.toList m + where + toword (k, v) = k ++ "=" ++ configEscape v + +configEscape :: String -> String +configEscape = concatMap escape + where + escape c + | isSpace c || c `elem` "&" = "&" ++ show (ord c) ++ ";" + | otherwise = [c] + +configUnEscape :: String -> String +configUnEscape = unescape + where + unescape [] = [] + unescape (c:rest) + | c == '&' = entity rest + | otherwise = c : unescape rest + entity s + | not (null num) && ";" `isPrefixOf` r = + chr (Prelude.read num) : unescape rest + | otherwise = + '&' : unescape s + where + num = takeWhile isNumber s + r = drop (length num) s + rest = drop 1 r + +{- for quickcheck -} +prop_isomorphic_configEscape :: String -> Bool +prop_isomorphic_configEscape s = s == (configUnEscape . configEscape) s + +prop_parse_show_Config :: RemoteConfig -> Bool +prop_parse_show_Config c + -- whitespace and '=' are not supported in keys + | any (\k -> any isSpace k || elem '=' k) (M.keys c) = True + | otherwise = parseConfig (showConfig c) ~~ Just c + where + normalize v = sort . M.toList <$> v + a ~~ b = normalize a == normalize b diff --git a/Logs/RemoteState.hs b/Logs/RemoteState.hs new file mode 100644 index 0000000000..17d084f781 --- /dev/null +++ b/Logs/RemoteState.hs @@ -0,0 +1,36 @@ +{- Remote state logs. + - + - Copyright 2014 Joey Hess + - + - Licensed under the GNU GPL version 3 or higher. + -} + +module Logs.RemoteState ( + getRemoteState, + setRemoteState, +) where + +import Annex.Common +import Logs +import Logs.UUIDBased +import qualified Annex.Branch +import qualified Annex + +import qualified Data.Map as M + +type RemoteState = String + +setRemoteState :: UUID -> Key -> RemoteState -> Annex () +setRemoteState u k s = do + c <- liftIO currentVectorClock + config <- Annex.getGitConfig + Annex.Branch.change (remoteStateLogFile config k) $ + showLogNew id . changeLog c u s . parseLogNew Just + +getRemoteState :: UUID -> Key -> Annex (Maybe RemoteState) +getRemoteState u k = do + config <- Annex.getGitConfig + extract . parseLogNew Just + <$> Annex.Branch.get (remoteStateLogFile config k) + where + extract m = value <$> M.lookup u m diff --git a/Logs/Schedule.hs b/Logs/Schedule.hs new file mode 100644 index 0000000000..1868e34603 --- /dev/null +++ b/Logs/Schedule.hs @@ -0,0 +1,71 @@ +{- git-annex scheduled activities log + - + - Copyright 2013 Joey Hess + - + - Licensed under the GNU GPL version 3 or higher. + -} + +module Logs.Schedule ( + scheduleLog, + scheduleSet, + scheduleAdd, + scheduleRemove, + scheduleChange, + scheduleGet, + scheduleMap, + getLastRunTimes, + setLastRunTime, +) where + +import qualified Data.Map as M +import qualified Data.Set as S +import Data.Time.LocalTime + +import Annex.Common +import Types.ScheduledActivity +import qualified Annex.Branch +import Logs +import Logs.UUIDBased +import Logs.File + +scheduleSet :: UUID -> [ScheduledActivity] -> Annex () +scheduleSet uuid@(UUID _) activities = do + c <- liftIO currentVectorClock + Annex.Branch.change scheduleLog $ + showLog id . changeLog c uuid val . parseLog Just + where + val = fromScheduledActivities activities +scheduleSet NoUUID _ = error "unknown UUID; cannot modify" + +scheduleMap :: Annex (M.Map UUID [ScheduledActivity]) +scheduleMap = simpleMap + . parseLogWithUUID parser + <$> Annex.Branch.get scheduleLog + where + parser _uuid = eitherToMaybe . parseScheduledActivities + +scheduleGet :: UUID -> Annex (S.Set ScheduledActivity) +scheduleGet u = do + m <- scheduleMap + return $ maybe S.empty S.fromList (M.lookup u m) + +scheduleRemove :: UUID -> ScheduledActivity -> Annex () +scheduleRemove u activity = scheduleChange u $ S.delete activity + +scheduleAdd :: UUID -> ScheduledActivity -> Annex () +scheduleAdd u activity = scheduleChange u $ S.insert activity + +scheduleChange :: UUID -> (S.Set ScheduledActivity -> S.Set ScheduledActivity) -> Annex () +scheduleChange u a = scheduleSet u . S.toList . a =<< scheduleGet u + +getLastRunTimes :: Annex (M.Map ScheduledActivity LocalTime) +getLastRunTimes = do + f <- fromRepo gitAnnexScheduleState + liftIO $ fromMaybe M.empty + <$> catchDefaultIO Nothing (readish <$> readFile f) + +setLastRunTime :: ScheduledActivity -> LocalTime -> Annex () +setLastRunTime activity lastrun = do + f <- fromRepo gitAnnexScheduleState + writeLogFile f . show . M.insert activity lastrun + =<< getLastRunTimes diff --git a/Logs/SingleValue.hs b/Logs/SingleValue.hs new file mode 100644 index 0000000000..8e648a6289 --- /dev/null +++ b/Logs/SingleValue.hs @@ -0,0 +1,38 @@ +{- git-annex single-value log + - + - This is used to store a value in a way that can be union merged. + - + - A line of the log will look like: "timestamp value" + - + - The line with the newest timestamp wins. + - + - Copyright 2014 Joey Hess + - + - Licensed under the GNU GPL version 3 or higher. + -} + +module Logs.SingleValue ( + module Logs.SingleValue.Pure, + readLog, + getLog, + setLog, +) where + +import Annex.Common +import qualified Annex.Branch +import Logs.SingleValue.Pure +import Annex.VectorClock + +import qualified Data.Set as S + +readLog :: (Ord v, SingleValueSerializable v) => FilePath -> Annex (Log v) +readLog = parseLog <$$> Annex.Branch.get + +getLog :: (Ord v, SingleValueSerializable v) => FilePath -> Annex (Maybe v) +getLog = newestValue <$$> readLog + +setLog :: (SingleValueSerializable v) => FilePath -> v -> Annex () +setLog f v = do + c <- liftIO currentVectorClock + let ent = LogEntry c v + Annex.Branch.change f $ \_old -> showLog (S.singleton ent) diff --git a/Logs/SingleValue/Pure.hs b/Logs/SingleValue/Pure.hs new file mode 100644 index 0000000000..de3ceb14a3 --- /dev/null +++ b/Logs/SingleValue/Pure.hs @@ -0,0 +1,45 @@ +{- git-annex single-value log, pure operations + - + - Copyright 2014 Joey Hess + - + - Licensed under the GNU GPL version 3 or higher. + -} + +module Logs.SingleValue.Pure where + +import Annex.Common +import Logs.Line +import Annex.VectorClock + +import qualified Data.Set as S + +class SingleValueSerializable v where + serialize :: v -> String + deserialize :: String -> Maybe v + +data LogEntry v = LogEntry + { changed :: VectorClock + , value :: v + } deriving (Eq, Ord) + +type Log v = S.Set (LogEntry v) + +showLog :: (SingleValueSerializable v) => Log v -> String +showLog = unlines . map showline . S.toList + where + showline (LogEntry c v) = unwords [formatVectorClock c, serialize v] + +parseLog :: (Ord v, SingleValueSerializable v) => String -> Log v +parseLog = S.fromList . mapMaybe parse . splitLines + where + parse line = do + let (sc, s) = splitword line + c <- parseVectorClock sc + v <- deserialize s + Just (LogEntry c v) + splitword = separate (== ' ') + +newestValue :: Log v -> Maybe v +newestValue s + | S.null s = Nothing + | otherwise = Just (value $ S.findMax s) diff --git a/Logs/TimeStamp.hs b/Logs/TimeStamp.hs new file mode 100644 index 0000000000..a07b6a66da --- /dev/null +++ b/Logs/TimeStamp.hs @@ -0,0 +1,39 @@ +{- log timestamp parsing + - + - Copyright 2015-2016 Joey Hess + - + - Licensed under the GNU GPL version 3 or higher. + -} + +{-# LANGUAGE CPP #-} + +module Logs.TimeStamp where + +import Utility.PartialPrelude +import Utility.Misc + +import Data.Time.Clock.POSIX +import Data.Time +import Data.Ratio +#if ! MIN_VERSION_time(1,5,0) +import System.Locale +#endif + +{- Parses how POSIXTime shows itself: "1431286201.113452s" + - Also handles the format with no fractional seconds. -} +parsePOSIXTime :: String -> Maybe POSIXTime +parsePOSIXTime s = do + let (sn, sd) = separate (== '.') s + n <- readi sn + if null sd + then return (fromIntegral n) + else do + d <- readi sd + let r = d % (10 ^ (length sd - 1)) + return (fromIntegral n + fromRational r) + where + readi :: String -> Maybe Integer + readi = readish + +formatPOSIXTime :: String -> POSIXTime -> String +formatPOSIXTime fmt t = formatTime defaultTimeLocale fmt (posixSecondsToUTCTime t) diff --git a/Logs/Transfer.hs b/Logs/Transfer.hs new file mode 100644 index 0000000000..aebb249fd4 --- /dev/null +++ b/Logs/Transfer.hs @@ -0,0 +1,303 @@ +{- git-annex transfer information files and lock files + - + - Copyright 2012 Joey Hess + - + - Licensed under the GNU GPL version 3 or higher. + -} + +{-# LANGUAGE CPP #-} + +module Logs.Transfer where + +import Types.Transfer +import Types.ActionItem +import Annex.Common +import Annex.Perms +import qualified Git +import Utility.Metered +import Utility.Percentage +import Utility.PID +import Annex.LockPool +import Logs.TimeStamp +import Logs.File + +import Data.Time.Clock +import Data.Time.Clock.POSIX +import Control.Concurrent + +describeTransfer :: Transfer -> TransferInfo -> String +describeTransfer t info = unwords + [ show $ transferDirection t + , show $ transferUUID t + , actionItemDesc + (ActionItemAssociatedFile (associatedFile info)) + (transferKey t) + , show $ bytesComplete info + ] + +{- Transfers that will accomplish the same task. -} +equivilantTransfer :: Transfer -> Transfer -> Bool +equivilantTransfer t1 t2 + | transferDirection t1 == Download && transferDirection t2 == Download && + transferKey t1 == transferKey t2 = True + | otherwise = t1 == t2 + +percentComplete :: Transfer -> TransferInfo -> Maybe Percentage +percentComplete (Transfer { transferKey = key }) info = + percentage <$> keySize key <*> Just (fromMaybe 0 $ bytesComplete info) + +{- Generates a callback that can be called as transfer progresses to update + - the transfer info file. Also returns the file it'll be updating, + - an action that sets up the file with appropriate permissions, + - which should be run after locking the transfer lock file, but + - before using the callback, and a MVar that can be used to read + - the number of bytesComplete. -} +mkProgressUpdater :: Transfer -> TransferInfo -> Annex (MeterUpdate, FilePath, Annex (), MVar Integer) +mkProgressUpdater t info = do + tfile <- fromRepo $ transferFile t + let createtfile = void $ tryNonAsync $ writeTransferInfoFile info tfile + mvar <- liftIO $ newMVar 0 + return (liftIO . updater tfile mvar, tfile, createtfile, mvar) + where + updater tfile mvar b = modifyMVar_ mvar $ \oldbytes -> do + let newbytes = fromBytesProcessed b + if newbytes - oldbytes >= mindelta + then do + let info' = info { bytesComplete = Just newbytes } + _ <- tryIO $ updateTransferInfoFile info' tfile + return newbytes + else return oldbytes + {- The minimum change in bytesComplete that is worth + - updating a transfer info file for is 1% of the total + - keySize, rounded down. -} + mindelta = case keySize (transferKey t) of + Just sz -> sz `div` 100 + Nothing -> 100 * 1024 -- arbitrarily, 100 kb + +startTransferInfo :: AssociatedFile -> IO TransferInfo +startTransferInfo afile = TransferInfo + <$> (Just . utcTimeToPOSIXSeconds <$> getCurrentTime) +#ifndef mingw32_HOST_OS + <*> pure Nothing -- pid not stored in file, so omitted for speed +#else + <*> (Just <$> getPID) +#endif + <*> pure Nothing -- tid ditto + <*> pure Nothing -- not 0; transfer may be resuming + <*> pure Nothing + <*> pure afile + <*> pure False + +{- If a transfer is still running, returns its TransferInfo. + - + - If no transfer is running, attempts to clean up the stale + - lock and info files. This can happen if a transfer process was + - interrupted. + -} +checkTransfer :: Transfer -> Annex (Maybe TransferInfo) +checkTransfer t = do + tfile <- fromRepo $ transferFile t + let lck = transferLockFile tfile + let cleanstale = do + void $ tryIO $ removeFile tfile + void $ tryIO $ removeFile lck +#ifndef mingw32_HOST_OS + v <- getLockStatus lck + case v of + StatusLockedBy pid -> liftIO $ catchDefaultIO Nothing $ + readTransferInfoFile (Just pid) tfile + _ -> do + -- Take a non-blocking lock while deleting + -- the stale lock file. Ignore failure + -- due to permissions problems, races, etc. + void $ tryIO $ do + mode <- annexFileMode + r <- tryLockExclusive (Just mode) lck + case r of + Just lockhandle -> liftIO $ do + cleanstale + dropLock lockhandle + _ -> noop + return Nothing +#else + v <- liftIO $ lockShared lck + liftIO $ case v of + Nothing -> catchDefaultIO Nothing $ + readTransferInfoFile Nothing tfile + Just lockhandle -> do + dropLock lockhandle + cleanstale + return Nothing +#endif + +{- Gets all currently running transfers. -} +getTransfers :: Annex [(Transfer, TransferInfo)] +getTransfers = getTransfers' [Download, Upload] (const True) + +getTransfers' :: [Direction] -> (Key -> Bool) -> Annex [(Transfer, TransferInfo)] +getTransfers' dirs wanted = do + transfers <- filter (wanted . transferKey) + <$> mapMaybe parseTransferFile . concat <$> findfiles + infos <- mapM checkTransfer transfers + return $ map (\(t, Just i) -> (t, i)) $ + filter running $ zip transfers infos + where + findfiles = liftIO . mapM dirContentsRecursive + =<< mapM (fromRepo . transferDir) dirs + running (_, i) = isJust i + +{- Number of bytes remaining to download from matching downloads that are in + - progress. -} +sizeOfDownloadsInProgress :: (Key -> Bool) -> Annex Integer +sizeOfDownloadsInProgress wanted = sum . map remaining + <$> getTransfers' [Download] wanted + where + remaining (t, info) = + case (keySize (transferKey t), bytesComplete info) of + (Just sz, Just done) -> sz - done + (Just sz, Nothing) -> sz + (Nothing, _) -> 0 + +{- Gets failed transfers for a given remote UUID. -} +getFailedTransfers :: UUID -> Annex [(Transfer, TransferInfo)] +getFailedTransfers u = catMaybes <$> (liftIO . getpairs =<< concat <$> findfiles) + where + getpairs = mapM $ \f -> do + let mt = parseTransferFile f + mi <- readTransferInfoFile Nothing f + return $ case (mt, mi) of + (Just t, Just i) -> Just (t, i) + _ -> Nothing + findfiles = liftIO . mapM dirContentsRecursive + =<< mapM (fromRepo . failedTransferDir u) [Download, Upload] + +clearFailedTransfers :: UUID -> Annex [(Transfer, TransferInfo)] +clearFailedTransfers u = do + failed <- getFailedTransfers u + mapM_ (removeFailedTransfer . fst) failed + return failed + +removeFailedTransfer :: Transfer -> Annex () +removeFailedTransfer t = do + f <- fromRepo $ failedTransferFile t + liftIO $ void $ tryIO $ removeFile f + +recordFailedTransfer :: Transfer -> TransferInfo -> Annex () +recordFailedTransfer t info = do + failedtfile <- fromRepo $ failedTransferFile t + writeTransferInfoFile info failedtfile + +{- The transfer information file to use for a given Transfer. -} +transferFile :: Transfer -> Git.Repo -> FilePath +transferFile (Transfer direction u key) r = transferDir direction r + filter (/= '/') (fromUUID u) + keyFile key + +{- The transfer information file to use to record a failed Transfer -} +failedTransferFile :: Transfer -> Git.Repo -> FilePath +failedTransferFile (Transfer direction u key) r = failedTransferDir u direction r + keyFile key + +{- The transfer lock file corresponding to a given transfer info file. -} +transferLockFile :: FilePath -> FilePath +transferLockFile infofile = let (d,f) = splitFileName infofile in + combine d ("lck." ++ f) + +{- Parses a transfer information filename to a Transfer. -} +parseTransferFile :: FilePath -> Maybe Transfer +parseTransferFile file + | "lck." `isPrefixOf` takeFileName file = Nothing + | otherwise = case drop (length bits - 3) bits of + [direction, u, key] -> Transfer + <$> parseDirection direction + <*> pure (toUUID u) + <*> fileKey key + _ -> Nothing + where + bits = splitDirectories file + +writeTransferInfoFile :: TransferInfo -> FilePath -> Annex () +writeTransferInfoFile info tfile = writeLogFile tfile $ writeTransferInfo info + +-- The file keeps whatever permissions it has, so should be used only +-- after it's been created with the right perms by writeTransferInfoFile. +updateTransferInfoFile :: TransferInfo -> FilePath -> IO () +updateTransferInfoFile info tfile = writeFile tfile $ writeTransferInfo info + +{- File format is a header line containing the startedTime and any + - bytesComplete value. Followed by a newline and the associatedFile. + - + - On unix, the transferPid is not included; instead it is obtained + - by looking at the process that locks the file. + - + - On windows, the transferPid is included, as a second line. + -} +writeTransferInfo :: TransferInfo -> String +writeTransferInfo info = unlines + [ (maybe "" show $ startedTime info) ++ + (maybe "" (\b -> ' ' : show b) (bytesComplete info)) +#ifdef mingw32_HOST_OS + , maybe "" show (transferPid info) +#endif + -- comes last; arbitrary content + , let AssociatedFile afile = associatedFile info + in fromMaybe "" afile + ] + +readTransferInfoFile :: Maybe PID -> FilePath -> IO (Maybe TransferInfo) +readTransferInfoFile mpid tfile = catchDefaultIO Nothing $ + readTransferInfo mpid <$> readFileStrict tfile + +readTransferInfo :: Maybe PID -> String -> Maybe TransferInfo +readTransferInfo mpid s = TransferInfo + <$> time +#ifdef mingw32_HOST_OS + <*> pure (if isJust mpid then mpid else mpid') +#else + <*> pure mpid +#endif + <*> pure Nothing + <*> pure Nothing + <*> bytes + <*> pure (AssociatedFile (if null filename then Nothing else Just filename)) + <*> pure False + where +#ifdef mingw32_HOST_OS + (firstline, otherlines) = separate (== '\n') s + (secondline, rest) = separate (== '\n') otherlines + mpid' = readish secondline +#else + (firstline, rest) = separate (== '\n') s +#endif + filename + | end rest == "\n" = beginning rest + | otherwise = rest + bits = splitc ' ' firstline + numbits = length bits + time = if numbits > 0 + then Just <$> parsePOSIXTime =<< headMaybe bits + else pure Nothing -- not failure + bytes = if numbits > 1 + then Just <$> readish =<< headMaybe (drop 1 bits) + else pure Nothing -- not failure + +{- The directory holding transfer information files for a given Direction. -} +transferDir :: Direction -> Git.Repo -> FilePath +transferDir direction r = gitAnnexTransferDir r formatDirection direction + +{- The directory holding failed transfer information files for a given + - Direction and UUID -} +failedTransferDir :: UUID -> Direction -> Git.Repo -> FilePath +failedTransferDir u direction r = gitAnnexTransferDir r + "failed" + formatDirection direction + filter (/= '/') (fromUUID u) + +prop_read_write_transferinfo :: TransferInfo -> Bool +prop_read_write_transferinfo info + | isJust (transferRemote info) = True -- remote not stored + | isJust (transferTid info) = True -- tid not stored + | otherwise = Just (info { transferPaused = False }) == info' + where + info' = readTransferInfo (transferPid info) (writeTransferInfo info) + diff --git a/Logs/Transitions.hs b/Logs/Transitions.hs new file mode 100644 index 0000000000..0a90f118fc --- /dev/null +++ b/Logs/Transitions.hs @@ -0,0 +1,86 @@ +{- git-annex transitions log + - + - This is used to record transitions that have been performed on the + - git-annex branch, and when the transition was first started. + - + - We can quickly detect when the local branch has already had an transition + - done that is listed in the remote branch by checking that the local + - branch contains the same transition, with the same or newer start time. + - + - Copyright 2013 Joey Hess + - + - Licensed under the GNU GPL version 3 or higher. + -} + +module Logs.Transitions where + +import Annex.Common +import Annex.VectorClock +import Logs.Line + +import qualified Data.Set as S + +transitionsLog :: FilePath +transitionsLog = "transitions.log" + +data Transition + = ForgetGitHistory + | ForgetDeadRemotes + deriving (Show, Ord, Eq, Read) + +data TransitionLine = TransitionLine + { transitionStarted :: VectorClock + , transition :: Transition + } deriving (Ord, Eq) + +type Transitions = S.Set TransitionLine + +describeTransition :: Transition -> String +describeTransition ForgetGitHistory = "forget git history" +describeTransition ForgetDeadRemotes = "forget dead remotes" + +noTransitions :: Transitions +noTransitions = S.empty + +addTransition :: VectorClock -> Transition -> Transitions -> Transitions +addTransition c t = S.insert $ TransitionLine c t + +showTransitions :: Transitions -> String +showTransitions = unlines . map showTransitionLine . S.elems + +{- If the log contains new transitions we don't support, returns Nothing. -} +parseTransitions :: String -> Maybe Transitions +parseTransitions = check . map parseTransitionLine . splitLines + where + check l + | all isJust l = Just $ S.fromList $ catMaybes l + | otherwise = Nothing + +parseTransitionsStrictly :: String -> String -> Transitions +parseTransitionsStrictly source = fromMaybe badsource . parseTransitions + where + badsource = giveup $ "unknown transitions listed in " ++ source ++ "; upgrade git-annex!" + +showTransitionLine :: TransitionLine -> String +showTransitionLine (TransitionLine c t) = unwords [show t, formatVectorClock c] + +parseTransitionLine :: String -> Maybe TransitionLine +parseTransitionLine s = TransitionLine + <$> parseVectorClock cs + <*> readish ts + where + ws = words s + ts = Prelude.head ws + cs = unwords $ Prelude.tail ws + +combineTransitions :: [Transitions] -> Transitions +combineTransitions = S.unions + +transitionList :: Transitions -> [Transition] +transitionList = nub . map transition . S.elems + +{- Typically ran with Annex.Branch.change, but we can't import Annex.Branch + - here since it depends on this module. -} +recordTransitions :: (FilePath -> (String -> String) -> Annex ()) -> Transitions -> Annex () +recordTransitions changer t = changer transitionsLog $ + showTransitions . S.union t . parseTransitionsStrictly "local" diff --git a/Logs/Trust.hs b/Logs/Trust.hs new file mode 100644 index 0000000000..469e9b3000 --- /dev/null +++ b/Logs/Trust.hs @@ -0,0 +1,87 @@ +{- git-annex trust log + - + - Copyright 2010-2012 Joey Hess + - + - Licensed under the GNU GPL version 3 or higher. + -} + +module Logs.Trust ( + module X, + trustLog, + TrustLevel(..), + trustGet, + trustMap, + trustPartition, + trustExclude, + lookupTrust, + trustMapLoad, +) where + +import qualified Data.Map as M +import Data.Default + +import Annex.Common +import Types.TrustLevel +import qualified Annex +import Logs +import Remote.List +import qualified Types.Remote +import Logs.Trust.Basic as X + +{- Returns a list of UUIDs that the trustLog indicates have the + - specified trust level. + - Note that the list can be incomplete for SemiTrusted, since that's + - the default. -} +trustGet :: TrustLevel -> Annex [UUID] +trustGet level = M.keys . M.filter (== level) <$> trustMap + +{- Returns the TrustLevel of a given repo UUID. -} +lookupTrust :: UUID -> Annex TrustLevel +lookupTrust u = (fromMaybe def . M.lookup u) <$> trustMap + +{- Partitions a list of UUIDs to those matching a TrustLevel and not. -} +trustPartition :: TrustLevel -> [UUID] -> Annex ([UUID], [UUID]) +trustPartition level ls + | level == SemiTrusted = do + t <- trustGet Trusted + u <- trustGet UnTrusted + d <- trustGet DeadTrusted + let uncandidates = t ++ u ++ d + return $ partition (`notElem` uncandidates) ls + | otherwise = do + candidates <- trustGet level + return $ partition (`elem` candidates) ls + +{- Filters UUIDs to those not matching a TrustLevel. -} +trustExclude :: TrustLevel -> [UUID] -> Annex [UUID] +trustExclude level ls = snd <$> trustPartition level ls + +{- trustLog in a map, overridden with any values from forcetrust or + - the git config. The map is cached for speed. -} +trustMap :: Annex TrustMap +trustMap = maybe trustMapLoad return =<< Annex.getState Annex.trustmap + +{- Loads the map, updating the cache, -} +trustMapLoad :: Annex TrustMap +trustMapLoad = do + overrides <- Annex.getState Annex.forcetrust + l <- remoteList + -- Exports are not trusted, since they are not key/value stores. + -- This does not apply to appendonly exports, which are key/value + -- stores. + let untrustworthy r = pure (not (Types.Remote.appendonly r)) + <&&> Types.Remote.isExportSupported r + exports <- filterM untrustworthy l + let exportoverrides = M.fromList $ + map (\r -> (Types.Remote.uuid r, UnTrusted)) exports + logged <- trustMapRaw + let configured = M.fromList $ mapMaybe configuredtrust l + let m = M.unionWith min exportoverrides $ + M.union overrides $ + M.union configured logged + Annex.changeState $ \s -> s { Annex.trustmap = Just m } + return m + where + configuredtrust r = (\l -> Just (Types.Remote.uuid r, l)) + =<< readTrustLevel + =<< remoteAnnexTrustLevel (Types.Remote.gitconfig r) diff --git a/Logs/Trust/Basic.hs b/Logs/Trust/Basic.hs new file mode 100644 index 0000000000..850fcc95ff --- /dev/null +++ b/Logs/Trust/Basic.hs @@ -0,0 +1,36 @@ +{- git-annex trust log, basics + - + - Copyright 2010-2012 Joey Hess + - + - Licensed under the GNU GPL version 3 or higher. + -} + +module Logs.Trust.Basic ( + module X, + trustSet, + trustMapRaw, +) where + +import Annex.Common +import Types.TrustLevel +import qualified Annex.Branch +import qualified Annex +import Logs +import Logs.UUIDBased +import Logs.Trust.Pure as X + +{- Changes the trust level for a uuid in the trustLog. -} +trustSet :: UUID -> TrustLevel -> Annex () +trustSet uuid@(UUID _) level = do + c <- liftIO currentVectorClock + Annex.Branch.change trustLog $ + showLog showTrustLog . + changeLog c uuid level . + parseLog (Just . parseTrustLog) + Annex.changeState $ \s -> s { Annex.trustmap = Nothing } +trustSet NoUUID _ = error "unknown UUID; cannot modify" + +{- Does not include forcetrust or git config values, just those from the + - log file. -} +trustMapRaw :: Annex TrustMap +trustMapRaw = calcTrustMap <$> Annex.Branch.get trustLog diff --git a/Logs/Trust/Pure.hs b/Logs/Trust/Pure.hs new file mode 100644 index 0000000000..74b7fd38cb --- /dev/null +++ b/Logs/Trust/Pure.hs @@ -0,0 +1,36 @@ +{- git-annex trust log, pure operations + - + - Copyright 2010-2013 Joey Hess + - + - Licensed under the GNU GPL version 3 or higher. + -} + +module Logs.Trust.Pure where + +import Annex.Common +import Types.TrustLevel +import Logs.UUIDBased + +calcTrustMap :: String -> TrustMap +calcTrustMap = simpleMap . parseLog (Just . parseTrustLog) + +{- The trust.log used to only list trusted repos, without a field for the + - trust status, which is why this defaults to Trusted. -} +parseTrustLog :: String -> TrustLevel +parseTrustLog s = maybe Trusted parse $ headMaybe $ words s + where + parse "1" = Trusted + parse "0" = UnTrusted + parse "X" = DeadTrusted + parse _ = SemiTrusted + +showTrustLog :: TrustLevel -> String +showTrustLog Trusted = "1" +showTrustLog UnTrusted = "0" +showTrustLog DeadTrusted = "X" +showTrustLog SemiTrusted = "?" + +prop_parse_show_TrustLog :: Bool +prop_parse_show_TrustLog = all check [minBound .. maxBound] + where + check l = parseTrustLog (showTrustLog l) == l diff --git a/Logs/UUID.hs b/Logs/UUID.hs new file mode 100644 index 0000000000..d3b1e64095 --- /dev/null +++ b/Logs/UUID.hs @@ -0,0 +1,86 @@ +{- git-annex uuids + - + - Each git repository used by git-annex has an annex.uuid setting that + - uniquely identifies that repository. + - + - UUIDs of remotes are cached in git config, using keys named + - remote..annex-uuid + - + - uuid.log stores a list of known uuids, and their descriptions. + - + - Copyright 2010-2012 Joey Hess + - + - Licensed under the GNU GPL version 3 or higher. + -} + +module Logs.UUID ( + uuidLog, + describeUUID, + uuidMap, + uuidMapLoad +) where + +import Types.UUID +import Annex.Common +import Annex.VectorClock +import qualified Annex +import qualified Annex.Branch +import Logs +import Logs.UUIDBased +import qualified Annex.UUID + +import qualified Data.Map.Strict as M + +{- Records a description for a uuid in the log. -} +describeUUID :: UUID -> String -> Annex () +describeUUID uuid desc = do + c <- liftIO currentVectorClock + Annex.Branch.change uuidLog $ + showLog id . changeLog c uuid desc . fixBadUUID . parseLog Just + +{- Temporarily here to fix badly formatted uuid logs generated by + - versions 3.20111105 and 3.20111025. + - + - Those logs contain entries with the UUID and description flipped. + - Due to parsing, if the description is multiword, only the first + - will be taken to be the UUID. So, if the UUID of an entry does + - not look like a UUID, and the last word of the description does, + - flip them back. + -} +fixBadUUID :: Log String -> Log String +fixBadUUID = M.fromList . map fixup . M.toList + where + fixup (k, v) + | isbad = (fixeduuid, LogEntry (newertime v) fixedvalue) + | otherwise = (k, v) + where + kuuid = fromUUID k + isbad = not (isuuid kuuid) && not (null ws) && isuuid lastword + ws = words $ value v + lastword = Prelude.last ws + fixeduuid = toUUID lastword + fixedvalue = unwords $ kuuid: Prelude.init ws + -- For the fixed line to take precidence, it should be + -- slightly newer, but only slightly. + newertime (LogEntry (VectorClock c) _) = VectorClock (c + minimumPOSIXTimeSlice) + newertime (LogEntry Unknown _) = VectorClock minimumPOSIXTimeSlice + minimumPOSIXTimeSlice = 0.000001 + isuuid s = length s == 36 && length (splitc '-' s) == 5 + +{- The map is cached for speed. -} +uuidMap :: Annex UUIDMap +uuidMap = maybe uuidMapLoad return =<< Annex.getState Annex.uuidmap + +{- Read the uuidLog into a simple Map. + - + - The UUID of the current repository is included explicitly, since + - it may not have been described and so otherwise would not appear. -} +uuidMapLoad :: Annex UUIDMap +uuidMapLoad = do + m <- (simpleMap . parseLog Just) <$> Annex.Branch.get uuidLog + u <- Annex.UUID.getUUID + let m' = M.insertWith preferold u "" m + Annex.changeState $ \s -> s { Annex.uuidmap = Just m' } + return m' + where + preferold = flip const diff --git a/Logs/UUIDBased.hs b/Logs/UUIDBased.hs new file mode 100644 index 0000000000..4f32c19c7c --- /dev/null +++ b/Logs/UUIDBased.hs @@ -0,0 +1,90 @@ +{- git-annex uuid-based logs + - + - This is used to store information about UUIDs in a way that can + - be union merged. + - + - A line of the log will look like: "UUID[ INFO[ timestamp=foo]]" + - The timestamp is last for backwards compatability reasons, + - and may not be present on old log lines. + - + - New uuid based logs instead use the form: "timestamp UUID INFO" + - + - Copyright 2011-2013 Joey Hess + - + - Licensed under the GNU GPL version 3 or higher. + -} + +module Logs.UUIDBased ( + Log, + LogEntry(..), + VectorClock, + currentVectorClock, + parseLog, + parseLogNew, + parseLogWithUUID, + showLog, + showLogNew, + changeLog, + addLog, + simpleMap, +) where + +import qualified Data.Map as M + +import Common +import Types.UUID +import Annex.VectorClock +import Logs.MapLog +import Logs.Line + +type Log v = MapLog UUID v + +showLog :: (v -> String) -> Log v -> String +showLog shower = unlines . map showpair . M.toList + where + showpair (k, LogEntry (VectorClock c) v) = + unwords [fromUUID k, shower v, tskey ++ show c] + showpair (k, LogEntry Unknown v) = + unwords [fromUUID k, shower v] + +parseLog :: (String -> Maybe a) -> String -> Log a +parseLog = parseLogWithUUID . const + +parseLogWithUUID :: (UUID -> String -> Maybe a) -> String -> Log a +parseLogWithUUID parser = M.fromListWith best . mapMaybe parse . splitLines + where + parse line + -- This is a workaround for a bug that caused + -- NoUUID items to be stored in the log. + -- It can be removed at any time; is just here to clean + -- up logs where that happened temporarily. + | " " `isPrefixOf` line = Nothing + | null ws = Nothing + | otherwise = parser u (unwords info) >>= makepair + where + makepair v = Just (u, LogEntry ts v) + ws = words line + u = toUUID $ Prelude.head ws + t = Prelude.last ws + ts + | tskey `isPrefixOf` t = fromMaybe Unknown $ + parseVectorClock $ drop 1 $ dropWhile (/= '=') t + | otherwise = Unknown + info + | ts == Unknown = drop 1 ws + | otherwise = drop 1 $ beginning ws + +showLogNew :: (v -> String) -> Log v -> String +showLogNew = showMapLog fromUUID + +parseLogNew :: (String -> Maybe v) -> String -> Log v +parseLogNew = parseMapLog (Just . toUUID) + +changeLog :: VectorClock -> UUID -> v -> Log v -> Log v +changeLog = changeMapLog + +addLog :: UUID -> LogEntry v -> Log v -> Log v +addLog = addMapLog + +tskey :: String +tskey = "timestamp=" diff --git a/Logs/Unused.hs b/Logs/Unused.hs new file mode 100644 index 0000000000..d76d19a564 --- /dev/null +++ b/Logs/Unused.hs @@ -0,0 +1,110 @@ +{- git-annex unused log file + - + - This file is stored locally in .git/annex/, not in the git-annex branch. + - + - The format: "int key timestamp" + - + - The int is a short, stable identifier that the user can use to + - refer to this key. (Equivilant to a filename.) + - + - The timestamp indicates when the key was first determined to be unused. + - Older versions of the log omit the timestamp. + - + - Copyright 2010-2014 Joey Hess + - + - Licensed under the GNU GPL version 3 or higher. + -} + +module Logs.Unused ( + UnusedMap, + updateUnusedLog, + readUnusedLog, + readUnusedMap, + dateUnusedLog, + unusedKeys, + unusedKeys', + setUnusedKeys, +) where + +import qualified Data.Map as M +import qualified Data.Set as S +import Data.Time.Clock.POSIX +import Data.Time + +import Annex.Common +import qualified Annex +import Logs.TimeStamp +import Logs.File + +-- everything that is stored in the unused log +type UnusedLog = M.Map Key (Int, Maybe POSIXTime) + +-- used to look up unused keys specified by the user +type UnusedMap = M.Map Int Key + +log2map :: UnusedLog -> UnusedMap +log2map = M.fromList . map (\(k, (i, _t)) -> (i, k)) . M.toList + +map2log :: POSIXTime -> UnusedMap -> UnusedLog +map2log t = M.fromList . map (\(i, k) -> (k, (i, Just t))) . M.toList + +{- Only keeps keys that are in the new log, but uses any timestamps + - those keys had in the old log. -} +preserveTimestamps :: UnusedLog -> UnusedLog -> UnusedLog +preserveTimestamps oldl newl = M.intersection (M.unionWith oldts oldl newl) newl + where + oldts _old@(_, ts) _new@(int, _) = (int, ts) + +updateUnusedLog :: FilePath -> UnusedMap -> Annex () +updateUnusedLog prefix m = do + oldl <- readUnusedLog prefix + newl <- preserveTimestamps oldl . flip map2log m <$> liftIO getPOSIXTime + writeUnusedLog prefix newl + +writeUnusedLog :: FilePath -> UnusedLog -> Annex () +writeUnusedLog prefix l = do + logfile <- fromRepo $ gitAnnexUnusedLog prefix + writeLogFile logfile $ unlines $ map format $ M.toList l + where + format (k, (i, Just t)) = show i ++ " " ++ key2file k ++ " " ++ show t + format (k, (i, Nothing)) = show i ++ " " ++ key2file k + +readUnusedLog :: FilePath -> Annex UnusedLog +readUnusedLog prefix = do + f <- fromRepo $ gitAnnexUnusedLog prefix + ifM (liftIO $ doesFileExist f) + ( M.fromList . mapMaybe parse . lines + <$> liftIO (readFileStrict f) + , return M.empty + ) + where + parse line = case (readish sint, file2key skey, parsePOSIXTime ts) of + (Just int, Just key, mtimestamp) -> Just (key, (int, mtimestamp)) + _ -> Nothing + where + (sint, rest) = separate (== ' ') line + (rts, rskey) = separate (== ' ') (reverse rest) + skey = reverse rskey + ts = reverse rts + +readUnusedMap :: FilePath -> Annex UnusedMap +readUnusedMap = log2map <$$> readUnusedLog + +dateUnusedLog :: FilePath -> Annex (Maybe UTCTime) +dateUnusedLog prefix = do + f <- fromRepo $ gitAnnexUnusedLog prefix + liftIO $ catchMaybeIO $ getModificationTime f + +{- Set of unused keys. This is cached for speed. -} +unusedKeys :: Annex (S.Set Key) +unusedKeys = maybe (setUnusedKeys =<< unusedKeys') return + =<< Annex.getState Annex.unusedkeys + +unusedKeys' :: Annex [Key] +unusedKeys' = M.keys <$> readUnusedLog "" + +setUnusedKeys :: [Key] -> Annex (S.Set Key) +setUnusedKeys ks = do + let v = S.fromList ks + Annex.changeState $ \s -> s { Annex.unusedkeys = Just v } + return v diff --git a/Logs/View.hs b/Logs/View.hs new file mode 100644 index 0000000000..80bdcc2a9b --- /dev/null +++ b/Logs/View.hs @@ -0,0 +1,97 @@ +{- git-annex recent views log + - + - The most recently accessed view comes first. + - + - This file is stored locally in .git/annex/, not in the git-annex branch. + - + - Copyright 2014 Joey Hess + - + - Licensed under the GNU GPL version 3 or higher. + -} + +module Logs.View ( + currentView, + setView, + removeView, + recentViews, + branchView, + is_branchView, + prop_branchView_legal, +) where + +import Annex.Common +import Types.View +import Types.MetaData +import qualified Git +import qualified Git.Branch +import qualified Git.Ref +import Git.Types +import Logs.File + +import qualified Data.Set as S +import Data.Char + +setView :: View -> Annex () +setView v = do + old <- take 99 . filter (/= v) <$> recentViews + writeViews (v : old) + +writeViews :: [View] -> Annex () +writeViews l = do + f <- fromRepo gitAnnexViewLog + writeLogFile f $ unlines $ map show l + +removeView :: View -> Annex () +removeView v = writeViews =<< filter (/= v) <$> recentViews + +recentViews :: Annex [View] +recentViews = do + f <- fromRepo gitAnnexViewLog + liftIO $ mapMaybe readish . lines <$> catchDefaultIO [] (readFile f) + +{- Gets the currently checked out view, if there is one. -} +currentView :: Annex (Maybe View) +currentView = go =<< inRepo Git.Branch.current + where + go (Just b) | branchViewPrefix `isPrefixOf` fromRef b = + headMaybe . filter (\v -> branchView v == b) <$> recentViews + go _ = return Nothing + +branchViewPrefix :: String +branchViewPrefix = "refs/heads/views" + +{- Generates a git branch name for a View. + - + - There is no guarantee that each view gets a unique branch name, + - but the branch name is used to express the view as well as possible. + -} +branchView :: View -> Git.Branch +branchView view + | null name = Git.Ref branchViewPrefix + | otherwise = Git.Ref $ branchViewPrefix ++ "/" ++ name + where + name = intercalate ";" $ map branchcomp (viewComponents view) + branchcomp c + | viewVisible c = branchcomp' c + | otherwise = "(" ++ branchcomp' c ++ ")" + branchcomp' (ViewComponent metafield viewfilter _) =concat + [ forcelegal (fromMetaField metafield) + , branchvals viewfilter + ] + branchvals (FilterValues set) = '=' : branchset set + branchvals (FilterGlob glob) = '=' : forcelegal glob + branchvals (ExcludeValues set) = "!=" ++ branchset set + branchset = intercalate "," + . map (forcelegal . fromMetaValue) + . S.toList + forcelegal s + | Git.Ref.legal True s = s + | otherwise = map (\c -> if isAlphaNum c then c else '_') s + +is_branchView :: Git.Branch -> Bool +is_branchView (Ref b) + | b == branchViewPrefix = True + | otherwise = (branchViewPrefix ++ "/") `isPrefixOf` b + +prop_branchView_legal :: View -> Bool +prop_branchView_legal = Git.Ref.legal False . fromRef . branchView diff --git a/Logs/Web.hs b/Logs/Web.hs new file mode 100644 index 0000000000..bfe971e8aa --- /dev/null +++ b/Logs/Web.hs @@ -0,0 +1,126 @@ +{- Web url logs. + - + - Copyright 2011-2014 Joey Hess + - + - Licensed under the GNU GPL version 3 or higher. + -} + +module Logs.Web ( + URLString, + getUrls, + getUrlsWithPrefix, + setUrlPresent, + setUrlMissing, + knownUrls, + Downloader(..), + getDownloader, + setDownloader, + setDownloader', + setTempUrl, + removeTempUrl, +) where + +import qualified Data.ByteString.Lazy.Char8 as L +import qualified Data.Map as M + +import Annex.Common +import qualified Annex +import Logs +import Logs.Presence +import Logs.Location +import qualified Annex.Branch +import Annex.CatFile +import qualified Git +import qualified Git.LsFiles +import Utility.Url +import Annex.UUID +import qualified Types.Remote as Remote + +{- Gets all urls that a key might be available from. -} +getUrls :: Key -> Annex [URLString] +getUrls key = do + config <- Annex.getGitConfig + l <- go $ urlLogFile config key : oldurlLogs config key + tmpl <- Annex.getState (maybeToList . M.lookup key . Annex.tempurls) + return (tmpl ++ l) + where + go [] = return [] + go (l:ls) = do + us <- currentLogInfo l + if null us + then go ls + else return us + +getUrlsWithPrefix :: Key -> String -> Annex [URLString] +getUrlsWithPrefix key prefix = filter (prefix `isPrefixOf`) + . map (fst . getDownloader) + <$> getUrls key + +setUrlPresent :: UUID -> Key -> URLString -> Annex () +setUrlPresent uuid key url = do + us <- getUrls key + unless (url `elem` us) $ do + config <- Annex.getGitConfig + addLog (urlLogFile config key) =<< logNow InfoPresent url + logChange key uuid InfoPresent + +setUrlMissing :: UUID -> Key -> URLString -> Annex () +setUrlMissing uuid key url = do + config <- Annex.getGitConfig + addLog (urlLogFile config key) =<< logNow InfoMissing url + whenM (null <$> getUrls key) $ + logChange key uuid InfoMissing + +{- Finds all known urls. -} +knownUrls :: Annex [(Key, URLString)] +knownUrls = do + {- Ensure the git-annex branch's index file is up-to-date and + - any journaled changes are reflected in it, since we're going + - to query its index directly. -} + Annex.Branch.update + Annex.Branch.commit =<< Annex.Branch.commitMessage + Annex.Branch.withIndex $ do + top <- fromRepo Git.repoPath + (l, cleanup) <- inRepo $ Git.LsFiles.stagedDetails [top] + r <- mapM getkeyurls l + void $ liftIO cleanup + return $ concat r + where + getkeyurls (f, s, _) = case urlLogFileKey f of + Just k -> zip (repeat k) <$> geturls s + Nothing -> return [] + geturls Nothing = return [] + geturls (Just logsha) = getLog . L.unpack <$> catObject logsha + +setTempUrl :: Key -> URLString -> Annex () +setTempUrl key url = Annex.changeState $ \s -> + s { Annex.tempurls = M.insert key url (Annex.tempurls s) } + +removeTempUrl :: Key -> Annex () +removeTempUrl key = Annex.changeState $ \s -> + s { Annex.tempurls = M.delete key (Annex.tempurls s) } + +data Downloader = WebDownloader | YoutubeDownloader | QuviDownloader | OtherDownloader + deriving (Eq, Show) + +{- To keep track of how an url is downloaded, it's mangled slightly in + - the log, with a prefix indicating when a Downloader is used. -} +setDownloader :: URLString -> Downloader -> String +setDownloader u WebDownloader = u +setDownloader u QuviDownloader = "quvi:" ++ u +setDownloader u YoutubeDownloader = "yt:" ++ u +setDownloader u OtherDownloader = ":" ++ u + +setDownloader' :: URLString -> Remote -> String +setDownloader' u r + | Remote.uuid r == webUUID = setDownloader u WebDownloader + | otherwise = setDownloader u OtherDownloader + +getDownloader :: URLString -> (URLString, Downloader) +getDownloader u = case separate (== ':') u of + ("yt", u') -> (u', YoutubeDownloader) + -- quvi is not used any longer; youtube-dl should be able to handle + -- all urls it did. + ("quvi", u') -> (u', YoutubeDownloader) + ("", u') -> (u', OtherDownloader) + _ -> (u, WebDownloader) diff --git a/Makefile b/Makefile new file mode 100644 index 0000000000..4f741623ff --- /dev/null +++ b/Makefile @@ -0,0 +1,306 @@ +all=git-annex git-annex-shell mans docs + +# set to "./Setup" if you lack a cabal program. Or can be set to "stack" +BUILDER?=cabal +ifeq ($(BUILDER),stack) +GHC?=stack ghc -- +else +GHC?=ghc +endif + +PREFIX?=/usr +SHAREDIR?=share + +# Am I typing :make in vim? Do a fast build. +ifdef VIM +all=fast +endif + +build: $(all) + +tmp/configure-stamp: Build/TestConfig.hs Build/Configure.hs + if [ "$(BUILDER)" = ./Setup ]; then ghc --make Setup; fi + if [ "$(BUILDER)" != stack ]; then \ + $(BUILDER) configure --ghc-options="$(shell Build/collect-ghc-options.sh)"; \ + else \ + $(BUILDER) setup; \ + fi + mkdir -p tmp + touch tmp/configure-stamp + +git-annex: tmp/configure-stamp + $(BUILDER) build $(BUILDEROPTIONS) + if [ "$(BUILDER)" = stack ]; then \ + ln -sf $$(stack path --dist-dir)/build/git-annex/git-annex git-annex; \ + else \ + ln -sf dist/build/git-annex/git-annex git-annex; \ + fi + +git-annex-shell: git-annex + ln -sf git-annex git-annex-shell + +# These are not built normally. +git-union-merge.1: doc/git-union-merge.mdwn + ./Build/mdwn2man git-union-merge 1 doc/git-union-merge.mdwn > git-union-merge.1 +git-union-merge: + $(GHC) --make -threaded $@ + +install-mans: mans + install -d $(DESTDIR)$(PREFIX)/$(SHAREDIR)/man/man1 + install -m 0644 man/*.1 $(DESTDIR)$(PREFIX)/$(SHAREDIR)/man/man1 + +install-docs: docs install-mans + install -d $(DESTDIR)$(PREFIX)/$(SHAREDIR)/doc/git-annex + if [ -d html ]; then \ + rsync -a --delete html/ $(DESTDIR)$(PREFIX)/$(SHAREDIR)/doc/git-annex/html/; \ + fi + +install-bins: build + install -d $(DESTDIR)$(PREFIX)/bin + install git-annex $(DESTDIR)$(PREFIX)/bin + ln -sf git-annex $(DESTDIR)$(PREFIX)/bin/git-annex-shell + ln -sf git-annex $(DESTDIR)$(PREFIX)/bin/git-remote-tor-annex + +install-misc: build Build/InstallDesktopFile + ./Build/InstallDesktopFile $(PREFIX)/bin/git-annex || true + install -d $(DESTDIR)$(PREFIX)/$(SHAREDIR)/bash-completion/completions + install -m 0644 bash-completion.bash $(DESTDIR)$(PREFIX)/$(SHAREDIR)/bash-completion/completions/git-annex + install -d $(DESTDIR)$(PREFIX)/$(SHAREDIR)/zsh/vendor-completions + @./git-annex --zsh-completion-script git-annex 2>/dev/null > $(DESTDIR)$(PREFIX)/$(SHAREDIR)/zsh/vendor-completions/_git-annex || \ + echo "** zsh completions not installed; built with too old version of optparse-applicative" + install -d $(DESTDIR)$(PREFIX)/$(SHAREDIR)/fish/completions + @./git-annex --fish-completion-script git-annex 2>/dev/null > $(DESTDIR)$(PREFIX)/$(SHAREDIR)/fish/completions/git-annex.fish || \ + echo "** fish completions not installed; built with too old version of optparse-applicative" + +install: install-bins install-docs install-misc + +test: git-annex git-annex-shell + ./git-annex test + +retest: git-annex + ./git-annex test --rerun-update --rerun-filter failures + +# hothasktags chokes on some template haskell etc, so ignore errors +tags: + (for f in $$(find . | grep -v /.git/ | grep -v /tmp/ | grep -v /dist/ | grep -v /doc/ | egrep '\.hs$$'); do hothasktags -c --cpp -c -traditional -c --include=dist/build/autogen/cabal_macros.h $$f; done) 2>/dev/null | sort > tags + +mans: Build/MakeMans + ./Build/MakeMans + +# If ikiwiki is available, build static html docs suitable for being +# shipped in the software package. +docs: mans + @if [ -n "`which ikiwiki`" ]; then \ + LC_ALL=C TZ=UTC ikiwiki doc html -v --wikiname git-annex \ + --plugin=goodstuff \ + --no-usedirs --disable-plugin=openid --plugin=sidebar \ + --plugin theme --set theme=actiontabs --set deterministic=1 \ + --disable-plugin=shortcut --disable-plugin=smiley \ + --plugin=comments --set comments_pagespec="*" \ + --exclude='ikiwiki/*' \ + --exclude='news/.*' --exclude='design/assistant/blog/*' \ + --exclude='bugs/*' --exclude='todo/*' --exclude='forum/*' \ + --exclude='users/*' --exclude='devblog/*' --exclude='thanks'; \ + else \ + echo "** ikiwiki not found, skipping building docs" >&2; \ + fi + +clean: + if [ "$(BUILDER)" != ./Setup ] && [ "$(BUILDER)" != cabal ]; then $(BUILDER) clean; fi + rm -rf tmp dist git-annex $(mans) configure *.tix .hpc \ + doc/.ikiwiki html dist tags Build/SysConfig Build/Version \ + Setup Build/InstallDesktopFile Build/EvilSplicer \ + Build/Standalone Build/OSXMkLibs Build/LinuxMkLibs \ + Build/DistributionUpdate Build/BuildVersion Build/MakeMans \ + git-annex-shell git-union-merge .tasty-rerun-log + find . -name \*.o -exec rm {} \; + find . -name \*.hi -exec rm {} \; + +Build/InstallDesktopFile: Build/InstallDesktopFile.hs + $(GHC) --make $@ -Wall -fno-warn-tabs +Build/EvilSplicer: Build/EvilSplicer.hs + $(GHC) --make $@ -Wall -fno-warn-tabs +Build/Standalone: Build/Standalone.hs tmp/configure-stamp + $(GHC) --make $@ -Wall -fno-warn-tabs +Build/OSXMkLibs: Build/OSXMkLibs.hs + $(GHC) --make $@ -Wall -fno-warn-tabs +Build/LinuxMkLibs: Build/LinuxMkLibs.hs + $(GHC) --make $@ -Wall -fno-warn-tabs +Build/MakeMans: Build/MakeMans.hs + $(GHC) --make $@ -Wall -fno-warn-tabs + +LINUXSTANDALONE_DEST=tmp/git-annex.linux +linuxstandalone: + $(MAKE) git-annex Build/Standalone Build/LinuxMkLibs + rm -rf "$(LINUXSTANDALONE_DEST)" + mkdir -p tmp + cp -R standalone/linux/skel "$(LINUXSTANDALONE_DEST)" + sed -i -e 's/^GIT_ANNEX_PACKAGE_INSTALL=/GIT_ANNEX_PACKAGE_INSTALL=$(GIT_ANNEX_PACKAGE_INSTALL)/' "$(LINUXSTANDALONE_DEST)/runshell" + + install -d "$(LINUXSTANDALONE_DEST)/bin" + cp git-annex "$(LINUXSTANDALONE_DEST)/bin/" + strip "$(LINUXSTANDALONE_DEST)/bin/git-annex" + ln -sf git-annex "$(LINUXSTANDALONE_DEST)/bin/git-annex-shell" + ln -sf git-annex "$(LINUXSTANDALONE_DEST)/bin/git-remote-tor-annex" + zcat standalone/licences.gz > $(LINUXSTANDALONE_DEST)/LICENSE + cp doc/logo_16x16.png doc/logo.svg $(LINUXSTANDALONE_DEST) + cp standalone/trustedkeys.gpg $(LINUXSTANDALONE_DEST) + + ./Build/Standalone "$(LINUXSTANDALONE_DEST)" + + install -d "$(LINUXSTANDALONE_DEST)/git-core" + (cd "$(shell git --exec-path)" && tar c .) | (cd "$(LINUXSTANDALONE_DEST)"/git-core && tar x) + install -d "$(LINUXSTANDALONE_DEST)/templates" + (cd "$(shell git --man-path)"/../git-core/templates && tar c .) | (cd "$(LINUXSTANDALONE_DEST)"/templates && tar x) + install -d "$(LINUXSTANDALONE_DEST)/magic" + cp /usr/share/file/magic.mgc "$(LINUXSTANDALONE_DEST)/magic" + cp /usr/share/i18n -a "$(LINUXSTANDALONE_DEST)" + + ./Build/LinuxMkLibs "$(LINUXSTANDALONE_DEST)" + + $(MAKE) install-mans DESTDIR="$(LINUXSTANDALONE_DEST)" + + sha1sum git-annex > "$(LINUXSTANDALONE_DEST)/buildid" + cd tmp/git-annex.linux && find . -type f > git-annex.MANIFEST + cd tmp/git-annex.linux && find . -type l >> git-annex.MANIFEST + cd tmp && tar c git-annex.linux | gzip -9 --rsyncable > git-annex-standalone-$(shell dpkg --print-architecture).tar.gz + +# Run this target to build git-annex-standalone*.deb +debianstandalone: dpkg-buildpackage-F +# Run this target to build git-annex-standalone*.dsc +debianstandalone-dsc: dpkg-buildpackage-S + +prep-standalone: + $(MAKE) undo-standalone + QUILT_PATCHES=debian/patches QUILT_SERIES=series.standalone-build quilt push -a + debian/create-standalone-changelog + +undo-standalone: + test -e .git + git checkout debian/changelog CHANGELOG + quilt pop -a || true + +commit-standalone: + QUILT_PATCHES=debian/patches QUILT_SERIES=series.standalone-build quilt refresh + +dpkg-buildpackage%: prep-standalone + umask 022; dpkg-buildpackage -rfakeroot $* + $(MAKE) undo-standalone + +OSXAPP_DEST=tmp/build-dmg/git-annex.app +OSXAPP_BASE=$(OSXAPP_DEST)/Contents/MacOS/bundle +osxapp: + $(MAKE) git-annex Build/Standalone Build/OSXMkLibs + + # Remove all RPATHs, both because this overloads the linker on + # OSX Sierra, and to avoid the binary looking in someone's home + # directory. + if otool -l git-annex | grep -q "path "; then \ + eval install_name_tool $$(otool -l git-annex | grep "path " | sed 's/.*path /-delete_rpath /' | sed 's/ (.*//') git-annex; \ + fi + + rm -rf "$(OSXAPP_DEST)" "$(OSXAPP_BASE)" + install -d tmp/build-dmg + cp -R standalone/osx/git-annex.app "$(OSXAPP_DEST)" + + install -d "$(OSXAPP_BASE)" + cp git-annex "$(OSXAPP_BASE)" + strip "$(OSXAPP_BASE)/git-annex" + ln -sf git-annex "$(OSXAPP_BASE)/git-annex-shell" + ln -sf git-annex "$(OSXAPP_BASE)/git-remote-tor-annex" + gzcat standalone/licences.gz > $(OSXAPP_BASE)/LICENSE + cp $(OSXAPP_BASE)/LICENSE tmp/build-dmg/LICENSE.txt + cp standalone/trustedkeys.gpg $(OSXAPP_DEST)/Contents/MacOS + + ./Build/Standalone $(OSXAPP_BASE) + + (cd "$(shell git --exec-path)" && tar c .) | (cd "$(OSXAPP_BASE)" && tar x) + install -d "$(OSXAPP_BASE)/templates" + (cd "$(shell git --man-path)"/../git-core/templates && tar c .) | (cd "$(OSXAPP_BASE)"/templates && tar x) + install -d "$(OSXAPP_BASE)/magic" + if [ -e "$(OSX_MAGIC_FILE)" ]; then \ + cp "$(OSX_MAGIC_FILE)" "$(OSXAPP_BASE)/magic/magic.mgc"; \ + else \ + echo "** OSX_MAGIC_FILE not set; not including it" >&2; \ + fi + + # OSX looks in man dir nearby the bin + $(MAKE) install-mans DESTDIR="$(OSXAPP_BASE)/.." SHAREDIR="" PREFIX="" + + ./Build/OSXMkLibs $(OSXAPP_BASE) + cd $(OSXAPP_DEST) && find . -type f > Contents/MacOS/git-annex.MANIFEST + cd $(OSXAPP_DEST) && find . -type l >> Contents/MacOS/git-annex.MANIFEST + rm -f tmp/git-annex.dmg + hdiutil create -format UDBZ -size 640m -srcfolder tmp/build-dmg \ + -volname git-annex -o tmp/git-annex.dmg + +ANDROID_FLAGS?= +# Cross compile for Android. +# Uses https://github.com/neurocyte/ghc-android +android: Build/EvilSplicer + echo "Running native build, to get TH splices.." + if [ ! -e dist/setup/setup ]; then $(BUILDER) configure -O0 $(ANDROID_FLAGS) -fAndroidSplice; fi + mkdir -p tmp + if ! $(BUILDER) build --ghc-options=-ddump-splices 2> tmp/dump-splices; then tail tmp/dump-splices >&2; exit 1; fi + echo "Setting up Android build tree.." + ./Build/EvilSplicer tmp/splices tmp/dump-splices standalone/no-th/evilsplicer-headers.hs + rsync -az --exclude tmp --exclude dist . tmp/androidtree +# Copy the files with expanded splices to the source tree, but +# only if the existing source file is not newer. (So, if a file +# used to have TH splices but they were removed, it will be newer, +# and not overwritten.) + cp -uR tmp/splices/* tmp/androidtree || true +# Some additional dependencies needed by the expanded splices. + sed -i 's/^ Build-Depends: */ Build-Depends: yesod-core, yesod-routes, shakespeare, blaze-markup, file-embed, wai-app-static, wai, unordered-containers, /' tmp/androidtree/git-annex.cabal +# Avoid warnings due to sometimes unused imports added for the splices. + sed -i 's/GHC-Options: \(.*\)-Wall/GHC-Options: \1-Wall -fno-warn-unused-imports /i' tmp/androidtree/git-annex.cabal + sed -i 's/Extensions: /Extensions: MagicHash /i' tmp/androidtree/git-annex.cabal +# Cabal cannot cross compile with custom build type, so workaround. + sed -i 's/Build-type: Custom/Build-type: Simple/' tmp/androidtree/git-annex.cabal +# Build just once, but link repeatedly, for different versions of Android. + mkdir -p tmp/androidtree/dist/build/git-annex/4.0 tmp/androidtree/dist/build/git-annex/4.3 tmp/androidtree/dist/build/git-annex/5.0 + if [ ! -e tmp/androidtree/dist/setup-config ]; then \ + cd tmp/androidtree && CROSS_COMPILE=Android $$HOME/.ghc/$(shell cat standalone/android/abiversion)/arm-linux-androideabi/bin/cabal configure -fAndroid $(ANDROID_FLAGS); \ + fi + PATH=$(shell standalone/android/toolchainpath 4) cd tmp/androidtree && $$HOME/.ghc/$(shell cat standalone/android/abiversion)/arm-linux-androideabi/bin/cabal build \ + && mv dist/build/git-annex/git-annex dist/build/git-annex/4.0/git-annex + PATH=$(shell standalone/android/toolchainpath 4) cd tmp/androidtree && $$HOME/.ghc/$(shell cat standalone/android/abiversion)/arm-linux-androideabi/bin/cabal build \ + --ghc-options=-optl-z --ghc-options=-optlnocopyreloc \ + && mv dist/build/git-annex/git-annex dist/build/git-annex/4.3/git-annex + PATH=$(shell standalone/android/toolchainpath 5) cd tmp/androidtree && $$HOME/.ghc/$(shell cat standalone/android/abiversion)/arm-linux-androideabi/bin/cabal build \ + --ghc-options=-optl-z --ghc-options=-optlnocopyreloc --ghc-options=-optl-fPIE --ghc-options=-optl-pie --ghc-options=-optc-fPIE --ghc-options=-optc-pie \ + && mv dist/build/git-annex/git-annex dist/build/git-annex/5.0/git-annex + +androidapp: + $(MAKE) android + $(MAKE) -C standalone/android + +# Bypass cabal, and only run the main ghc --make command for a +# faster development build. +fast: dist/cabalbuild + @sh dist/cabalbuild + @ln -sf dist/build/git-annex/git-annex git-annex + @$(MAKE) tags >/dev/null 2>&1 & + +dist/cabalbuild: dist/caballog + grep 'ghc --make' dist/caballog | tail -n 1 > dist/cabalbuild + +dist/caballog: git-annex.cabal + $(BUILDER) configure -f"-Production" -O0 --enable-executable-dynamic + $(BUILDER) build -v2 --ghc-options="-O0 -j" | tee dist/caballog + +# Hardcoded command line to make hdevtools start up and work. +# You will need some memory. It's worth it. +# Note: Don't include WebDAV or Webapp. TH use bloats memory > 500 mb! +# TODO should be possible to derive this from caballog. +hdevtools: + hdevtools --stop-server || true + hdevtools check git-annex.hs -g -cpp -g -i -g -idist/build/git-annex/git-annex-tmp -g -i. -g -idist/build/autogen -g -Idist/build/autogen -g -Idist/build/git-annex/git-annex-tmp -g -IUtility -g -DWITH_S3 -g -DWITH_ASSISTANT -g -DWITH_INOTIFY -g -DWITH_DBUS -g -DWITH_PAIRING -g -g -optP-include -g -optPdist/build/autogen/cabal_macros.h -g -odir -g dist/build/git-annex/git-annex-tmp -g -hidir -g dist/build/git-annex/git-annex-tmp -g -stubdir -g dist/build/git-annex/git-annex-tmp -g -threaded -g -Wall -g -XHaskell98 -g -XPackageImports -g -XLambdaCase + +distributionupdate: + git pull + cabal configure + ghc -Wall -fno-warn-tabs --make Build/DistributionUpdate -XLambdaCase -XPackageImports + ./Build/DistributionUpdate + +.PHONY: git-annex git-union-merge tags diff --git a/Messages.hs b/Messages.hs new file mode 100644 index 0000000000..d5dee72e21 --- /dev/null +++ b/Messages.hs @@ -0,0 +1,256 @@ +{- git-annex output messages + - + - Copyright 2010-2017 Joey Hess + - + - Licensed under the GNU GPL version 3 or higher. + -} + +module Messages ( + showStart, + showStart', + showStartKey, + ActionItem, + mkActionItem, + showNote, + showAction, + showSideAction, + doSideAction, + doQuietSideAction, + showStoringStateAction, + showOutput, + showLongNote, + showInfo, + showEndOk, + showEndFail, + showEndResult, + endResult, + toplevelWarning, + warning, + earlyWarning, + warningIO, + indent, + JSON.JSONChunk(..), + maybeShowJSON, + showFullJSON, + showCustom, + showHeader, + showRaw, + setupConsole, + enableDebugOutput, + disableDebugOutput, + debugEnabled, + commandProgressDisabled, + outputMessage, + implicitMessage, + withMessageState, + prompt, +) where + +import System.Log.Logger +import System.Log.Formatter +import System.Log.Handler (setFormatter) +import System.Log.Handler.Simple +import Control.Concurrent + +import Common +import Types +import Types.Messages +import Types.ActionItem +import Types.Concurrency +import Messages.Internal +import Messages.Concurrent +import qualified Messages.JSON as JSON +import qualified Annex + +showStart :: String -> FilePath -> Annex () +showStart command file = outputMessage json $ + command ++ " " ++ file ++ " " + where + json = JSON.start command (Just file) Nothing + +showStart' :: String -> Maybe String -> Annex () +showStart' command mdesc = outputMessage json $ + command ++ (maybe "" (" " ++) mdesc) ++ " " + where + json = JSON.start command Nothing Nothing + +showStartKey :: String -> Key -> ActionItem -> Annex () +showStartKey command key i = outputMessage json $ + command ++ " " ++ actionItemDesc i key ++ " " + where + json = JSON.start command (actionItemWorkTreeFile i) (Just key) + +showNote :: String -> Annex () +showNote s = outputMessage (JSON.note s) $ "(" ++ s ++ ") " + +showAction :: String -> Annex () +showAction s = showNote $ s ++ "..." + +showSideAction :: String -> Annex () +showSideAction m = Annex.getState Annex.output >>= go + where + go st + | sideActionBlock st == StartBlock = do + p + let st' = st { sideActionBlock = InBlock } + Annex.changeState $ \s -> s { Annex.output = st' } + | sideActionBlock st == InBlock = return () + | otherwise = p + p = outputMessage JSON.none $ "(" ++ m ++ "...)\n" + +showStoringStateAction :: Annex () +showStoringStateAction = showSideAction "recording state in git" + +{- Performs an action, supressing showSideAction messages. -} +doQuietSideAction :: Annex a -> Annex a +doQuietSideAction = doSideAction' InBlock + +{- Performs an action, that may call showSideAction multiple times. + - Only the first will be displayed. -} +doSideAction :: Annex a -> Annex a +doSideAction = doSideAction' StartBlock + +doSideAction' :: SideActionBlock -> Annex a -> Annex a +doSideAction' b a = do + o <- Annex.getState Annex.output + set $ o { sideActionBlock = b } + set o `after` a + where + set o = Annex.changeState $ \s -> s { Annex.output = o } + +{- Make way for subsequent output of a command. -} +showOutput :: Annex () +showOutput = unlessM commandProgressDisabled $ + outputMessage JSON.none "\n" + +showLongNote :: String -> Annex () +showLongNote s = outputMessage (JSON.note s) (formatLongNote s) + +formatLongNote :: String -> String +formatLongNote s = '\n' : indent s ++ "\n" + +-- Used by external special remote, displayed same as showLongNote +-- to console, but json object containing the info is emitted immediately. +showInfo :: String -> Annex () +showInfo s = outputMessage' outputJSON (JSON.info s) (formatLongNote s) + +showEndOk :: Annex () +showEndOk = showEndResult True + +showEndFail :: Annex () +showEndFail = showEndResult False + +showEndResult :: Bool -> Annex () +showEndResult ok = outputMessage (JSON.end ok) $ endResult ok ++ "\n" + +endResult :: Bool -> String +endResult True = "ok" +endResult False = "failed" + +toplevelWarning :: Bool -> String -> Annex () +toplevelWarning makeway s = warning' makeway ("git-annex: " ++ s) + +warning :: String -> Annex () +warning = warning' True . indent + +earlyWarning :: String -> Annex () +earlyWarning = warning' False + +warning' :: Bool -> String -> Annex () +warning' makeway w = do + when makeway $ + outputMessage JSON.none "\n" + outputError (w ++ "\n") + +{- Not concurrent output safe. -} +warningIO :: String -> IO () +warningIO w = do + putStr "\n" + hFlush stdout + hPutStrLn stderr w + +indent :: String -> String +indent = intercalate "\n" . map (\l -> " " ++ l) . lines + +{- Shows a JSON chunk only when in json mode. -} +maybeShowJSON :: JSON.JSONChunk v -> Annex () +maybeShowJSON v = void $ withMessageState $ bufferJSON (JSON.add v) + +{- Shows a complete JSON value, only when in json mode. -} +showFullJSON :: JSON.JSONChunk v -> Annex Bool +showFullJSON v = withMessageState $ bufferJSON (JSON.complete v) + +{- Performs an action that outputs nonstandard/customized output, and + - in JSON mode wraps its output in JSON.start and JSON.end, so it's + - a complete JSON document. + - This is only needed when showStart and showEndOk is not used. + -} +showCustom :: String -> Annex Bool -> Annex () +showCustom command a = do + outputMessage (JSON.start command Nothing Nothing) "" + r <- a + outputMessage (JSON.end r) "" + +showHeader :: String -> Annex () +showHeader h = outputMessage JSON.none $ (h ++ ": ") + +showRaw :: String -> Annex () +showRaw s = outputMessage JSON.none (s ++ "\n") + +setupConsole :: IO () +setupConsole = do + s <- setFormatter + <$> streamHandler stderr DEBUG + <*> pure preciseLogFormatter + updateGlobalLogger rootLoggerName (setLevel NOTICE . setHandlers [s]) + {- Force output to be line buffered. This is normally the case when + - it's connected to a terminal, but may not be when redirected to + - a file or a pipe. -} + hSetBuffering stdout LineBuffering + hSetBuffering stderr LineBuffering + +{- Log formatter with precision into fractions of a second. -} +preciseLogFormatter :: LogFormatter a +preciseLogFormatter = tfLogFormatter "%F %X%Q" "[$time] $msg" + +enableDebugOutput :: IO () +enableDebugOutput = updateGlobalLogger rootLoggerName $ setLevel DEBUG + +disableDebugOutput :: IO () +disableDebugOutput = updateGlobalLogger rootLoggerName $ setLevel NOTICE + +{- Checks if debugging is enabled. -} +debugEnabled :: IO Bool +debugEnabled = do + l <- getRootLogger + return $ getLevel l <= Just DEBUG + +{- Should commands that normally output progress messages have that + - output disabled? -} +commandProgressDisabled :: Annex Bool +commandProgressDisabled = withMessageState $ \s -> return $ + case outputType s of + QuietOutput -> True + JSONOutput _ -> True + NormalOutput -> concurrentOutputEnabled s + +{- Use to show a message that is displayed implicitly, and so might be + - disabled when running a certian command that needs more control over its + - output. -} +implicitMessage :: Annex () -> Annex () +implicitMessage = whenM (implicitMessages <$> Annex.getState Annex.output) + +{- Prevents any concurrent console access while running an action, so + - that the action is the only thing using the console, and can eg prompt + - the user. + -} +prompt :: Annex a -> Annex a +prompt a = go =<< Annex.getState Annex.concurrency + where + go NonConcurrent = a + go (Concurrent {}) = withMessageState $ \s -> do + let l = promptLock s + bracketIO + (takeMVar l) + (putMVar l) + (const $ hideRegionsWhile a) diff --git a/Messages/Concurrent.hs b/Messages/Concurrent.hs new file mode 100644 index 0000000000..8d69d6bbc5 --- /dev/null +++ b/Messages/Concurrent.hs @@ -0,0 +1,161 @@ +{- git-annex output messages, including concurrent output to display regions + - + - Copyright 2010-2017 Joey Hess + - + - Licensed under the GNU GPL version 3 or higher. + -} + +{-# LANGUAGE CPP #-} +{-# OPTIONS_GHC -fno-warn-orphans #-} + +module Messages.Concurrent where + +import Types +import Types.Messages +import qualified Annex + +#ifdef WITH_CONCURRENTOUTPUT +import Common +import qualified System.Console.Concurrent as Console +import qualified System.Console.Regions as Regions +import Control.Concurrent.STM +import qualified Data.Text as T +#ifndef mingw32_HOST_OS +import GHC.IO.Encoding +#endif +#endif + +{- Outputs a message in a concurrency safe way. + - + - The message may be an error message, in which case it goes to stderr. + - + - When built without concurrent-output support, the fallback action is run + - instead. + -} +concurrentMessage :: MessageState -> Bool -> String -> Annex () -> Annex () +#ifdef WITH_CONCURRENTOUTPUT +concurrentMessage s iserror msg fallback + | concurrentOutputEnabled s = + go =<< consoleRegion <$> Annex.getState Annex.output +#else +concurrentMessage _s _iserror _msg fallback +#endif + | otherwise = fallback +#ifdef WITH_CONCURRENTOUTPUT + where + go Nothing + | iserror = liftIO $ Console.errorConcurrent msg + | otherwise = liftIO $ Console.outputConcurrent msg + go (Just r) = do + -- Can't display the error to stdout while + -- console regions are in use, so set the errflag + -- to get it to display to stderr later. + when iserror $ do + Annex.changeState $ \st -> + st { Annex.output = (Annex.output st) { consoleRegionErrFlag = True } } + liftIO $ atomically $ do + Regions.appendConsoleRegion r msg + rl <- takeTMVar Regions.regionList + putTMVar Regions.regionList + (if r `elem` rl then rl else r:rl) +#endif + +{- Runs an action in its own dedicated region of the console. + - + - The region is closed at the end or on exception, and at that point + - the value of the region is displayed in the scrolling area above + - any other active regions. + - + - When not at a console, a region is not displayed until the action is + - complete. + -} +inOwnConsoleRegion :: MessageState -> Annex a -> Annex a +#ifdef WITH_CONCURRENTOUTPUT +inOwnConsoleRegion s a + | concurrentOutputEnabled s = do + r <- mkregion + setregion (Just r) + eret <- tryNonAsync a `onException` rmregion r + case eret of + Left e -> do + -- Add error message to region before it closes. + concurrentMessage s True (show e) noop + rmregion r + throwM e + Right ret -> do + rmregion r + return ret +#else +inOwnConsoleRegion _s a +#endif + | otherwise = a +#ifdef WITH_CONCURRENTOUTPUT + where + -- The region is allocated here, but not displayed until + -- a message is added to it. This avoids unnecessary screen + -- updates when a region does not turn out to need to be used. + mkregion = Regions.newConsoleRegion Regions.Linear "" + setregion r = Annex.changeState $ \st -> st + { Annex.output = (Annex.output st) { consoleRegion = r } } + rmregion r = do + errflag <- consoleRegionErrFlag <$> Annex.getState Annex.output + let h = if errflag then Console.StdErr else Console.StdOut + Annex.changeState $ \st -> st + { Annex.output = (Annex.output st) { consoleRegionErrFlag = False } } + setregion Nothing + liftIO $ atomically $ do + t <- Regions.getConsoleRegion r + unless (T.null t) $ + Console.bufferOutputSTM h t + Regions.closeConsoleRegion r +#endif + +{- The progress region is displayed inline with the current console region. -} +#ifdef WITH_CONCURRENTOUTPUT +withProgressRegion :: (Regions.ConsoleRegion -> Annex a) -> Annex a +withProgressRegion a = do + parent <- consoleRegion <$> Annex.getState Annex.output + Regions.withConsoleRegion (maybe Regions.Linear Regions.InLine parent) a + +instance Regions.LiftRegion Annex where + liftRegion = liftIO . atomically +#endif + +{- The concurrent-output library uses Text, which bypasses the normal use + - of the fileSystemEncoding to roundtrip invalid characters, when in a + - non-unicode locale. Work around that problem by avoiding using + - concurrent output when not in a unicode locale. -} +concurrentOutputSupported :: IO Bool +#ifdef WITH_CONCURRENTOUTPUT +#ifndef mingw32_HOST_OS +concurrentOutputSupported = do + enc <- getLocaleEncoding + return ("UTF" `isInfixOf` textEncodingName enc) +#else +concurrentOutputSupported = return True -- Windows is always unicode +#endif +#else +concurrentOutputSupported = return False +#endif + +{- Hide any currently displayed console regions while running the action, + - so that the action can use the console itself. + - This needs a new enough version of concurrent-output; otherwise + - the regions will not be hidden, but the action still runs, garbling the + - display. -} +hideRegionsWhile :: Annex a -> Annex a +#ifdef WITH_CONCURRENTOUTPUT +#if MIN_VERSION_concurrent_output(1,9,0) +hideRegionsWhile a = bracketIO setup cleanup go + where + setup = Regions.waitDisplayChange $ swapTMVar Regions.regionList [] + cleanup = void . atomically . swapTMVar Regions.regionList + go _ = do + liftIO $ hFlush stdout + a +#else +hideRegionsWhile = id +#endif +#else +hideRegionsWhile = id +#endif diff --git a/Messages/Internal.hs b/Messages/Internal.hs new file mode 100644 index 0000000000..42ad14d516 --- /dev/null +++ b/Messages/Internal.hs @@ -0,0 +1,81 @@ +{- git-annex output messages, including concurrent output to display regions + - + - Copyright 2010-2018 Joey Hess + - + - Licensed under the GNU GPL version 3 or higher. + -} + +module Messages.Internal where + +import Common +import Annex +import Types.Messages +import Messages.Concurrent +import qualified Messages.JSON as JSON +import Messages.JSON (JSONBuilder) + +withMessageState :: (MessageState -> Annex a) -> Annex a +withMessageState a = Annex.getState Annex.output >>= a + +outputMessage :: JSONBuilder -> String -> Annex () +outputMessage = outputMessage' bufferJSON + +outputMessage' :: (JSONBuilder -> MessageState -> Annex Bool) -> JSONBuilder -> String -> Annex () +outputMessage' jsonoutputter jsonbuilder msg = withMessageState $ \s -> case outputType s of + NormalOutput + | concurrentOutputEnabled s -> concurrentMessage s False msg q + | otherwise -> liftIO $ flushed $ putStr msg + JSONOutput _ -> void $ jsonoutputter jsonbuilder s + QuietOutput -> q + +-- Buffer changes to JSON until end is reached and then emit it. +bufferJSON :: JSONBuilder -> MessageState -> Annex Bool +bufferJSON jsonbuilder s = case outputType s of + JSONOutput jsonoptions + | endjson -> do + Annex.changeState $ \st -> + st { Annex.output = s { jsonBuffer = Nothing } } + maybe noop (liftIO . flushed . JSON.emit . JSON.finalize jsonoptions) json + return True + | otherwise -> do + Annex.changeState $ \st -> + st { Annex.output = s { jsonBuffer = json } } + return True + _ -> return False + where + (json, endjson) = case jsonbuilder i of + Nothing -> (jsonBuffer s, False) + (Just (j, e)) -> (Just j, e) + i = case jsonBuffer s of + Nothing -> Nothing + Just b -> Just (b, False) + +-- Immediately output JSON. +outputJSON :: JSONBuilder -> MessageState -> Annex Bool +outputJSON jsonbuilder s = case outputType s of + JSONOutput _ -> do + maybe noop (liftIO . flushed . JSON.emit) + (fst <$> jsonbuilder Nothing) + return True + _ -> return False + +outputError :: String -> Annex () +outputError msg = withMessageState $ \s -> case (outputType s, jsonBuffer s) of + (JSONOutput jsonoptions, Just jb) | jsonErrorMessages jsonoptions -> + let jb' = Just (JSON.addErrorMessage (lines msg) jb) + in Annex.changeState $ \st -> + st { Annex.output = s { jsonBuffer = jb' } } + _ + | concurrentOutputEnabled s -> concurrentMessage s True msg go + | otherwise -> go + where + go = liftIO $ do + hFlush stdout + hPutStr stderr msg + hFlush stderr + +q :: Monad m => m () +q = noop + +flushed :: IO () -> IO () +flushed a = a >> hFlush stdout diff --git a/Messages/JSON.hs b/Messages/JSON.hs new file mode 100644 index 0000000000..e63a263e3f --- /dev/null +++ b/Messages/JSON.hs @@ -0,0 +1,190 @@ +{- git-annex command-line JSON output and input + - + - Copyright 2011-2018 Joey Hess + - + - Licensed under the GNU GPL version 3 or higher. + -} + +{-# LANGUAGE OverloadedStrings, GADTs #-} + +module Messages.JSON ( + JSONBuilder, + JSONChunk(..), + emit, + none, + start, + end, + finalize, + addErrorMessage, + note, + info, + add, + complete, + progress, + DualDisp(..), + ObjectMap(..), + JSONActionItem(..), +) where + +import Control.Applicative +import qualified Data.Map as M +import qualified Data.Vector as V +import qualified Data.ByteString.Lazy as L +import qualified Data.HashMap.Strict as HM +import System.IO +import System.IO.Unsafe (unsafePerformIO) +import Control.Concurrent +import Data.Maybe +import Data.Monoid +import Prelude + +import Types.Messages +import Key +import Utility.Metered +import Utility.Percentage +import Utility.Aeson + +-- A global lock to avoid concurrent threads emitting json at the same time. +{-# NOINLINE emitLock #-} +emitLock :: MVar () +emitLock = unsafePerformIO $ newMVar () + +emit :: Object -> IO () +emit o = do + takeMVar emitLock + L.hPut stdout (encode o) + putStr "\n" + putMVar emitLock () + +-- Building up a JSON object can be done by first using start, +-- then add and note any number of times, and finally complete. +type JSONBuilder = Maybe (Object, Bool) -> Maybe (Object, Bool) + +none :: JSONBuilder +none = id + +start :: String -> Maybe FilePath -> Maybe Key -> JSONBuilder +start command file key _ = Just (o, False) + where + Object o = toJSON' $ JSONActionItem + { itemCommand = Just command + , itemKey = key + , itemFile = file + , itemAdded = Nothing + } + +end :: Bool -> JSONBuilder +end b (Just (o, _)) = Just (HM.insert "success" (toJSON' b) o, True) +end _ Nothing = Nothing + +finalize :: JSONOptions -> Object -> Object +finalize jsonoptions o + -- Always include error-messages field, even if empty, + -- to make the json be self-documenting. + | jsonErrorMessages jsonoptions = addErrorMessage [] o + | otherwise = o + +addErrorMessage :: [String] -> Object -> Object +addErrorMessage msg o = + HM.insertWith combinearray "error-messages" v o + where + combinearray (Array new) (Array old) = Array (old <> new) + combinearray new _old = new + v = Array $ V.fromList $ map (String . packString) msg + +note :: String -> JSONBuilder +note _ Nothing = Nothing +note s (Just (o, e)) = Just (HM.insertWith combinelines "note" (toJSON' s) o, e) + where + combinelines (String new) (String old) = + String (old <> "\n" <> new) + combinelines new _old = new + +info :: String -> JSONBuilder +info s _ = Just (o, True) + where + Object o = object ["info" .= toJSON' s] + +data JSONChunk v where + AesonObject :: Object -> JSONChunk Object + JSONChunk :: ToJSON' v => [(String, v)] -> JSONChunk [(String, v)] + +add :: JSONChunk v -> JSONBuilder +add v (Just (o, e)) = Just (HM.union o' o, e) + where + Object o' = case v of + AesonObject ao -> Object ao + JSONChunk l -> object $ map mkPair l + mkPair (s, d) = (packString s, toJSON' d) +add _ Nothing = Nothing + +complete :: JSONChunk v -> JSONBuilder +complete v _ = add v (Just (HM.empty, True)) + +-- Show JSON formatted progress, including the current state of the JSON +-- object for the action being performed. +progress :: Maybe Object -> Maybe Integer -> BytesProcessed -> IO () +progress maction msize bytesprocessed = emit $ case maction of + Just action -> HM.insert "action" (Object action) o + Nothing -> o + where + n = fromBytesProcessed bytesprocessed :: Integer + Object o = case msize of + Just size -> object + [ "byte-progress" .= n + , "percent-progress" .= showPercentage 2 (percentage size n) + , "total-size" .= size + ] + Nothing -> object + [ "byte-progress" .= n ] + +-- A value that can be displayed either normally, or as JSON. +data DualDisp = DualDisp + { dispNormal :: String + , dispJson :: String + } + +instance ToJSON' DualDisp where + toJSON' = toJSON' . dispJson + +instance Show DualDisp where + show = dispNormal + +-- A Map that is serialized to JSON as an object, with each key being a +-- field of the object. This is different from Aeson's normal +-- serialization of Map, which uses "[key, value]". +data ObjectMap a = ObjectMap { fromObjectMap :: M.Map String a } + +instance ToJSON' a => ToJSON' (ObjectMap a) where + toJSON' (ObjectMap m) = object $ map go $ M.toList m + where + go (k, v) = (packString k, toJSON' v) + +-- An item that a git-annex command acts on, and displays a JSON object about. +data JSONActionItem a = JSONActionItem + { itemCommand :: Maybe String + , itemKey :: Maybe Key + , itemFile :: Maybe FilePath + , itemAdded :: Maybe a -- for additional fields added by `add` + } + deriving (Show) + +instance ToJSON' (JSONActionItem a) where + toJSON' i = object $ catMaybes + [ Just $ "command" .= itemCommand i + , case itemKey i of + Nothing -> Nothing + Just k -> Just $ "key" .= toJSON' k + , Just $ "file" .= toJSON' (itemFile i) + -- itemAdded is not included; must be added later by 'add' + ] + +instance FromJSON a => FromJSON (JSONActionItem a) where + parseJSON (Object v) = JSONActionItem + <$> (v .:? "command") + <*> (maybe (return Nothing) parseJSON =<< (v .:? "key")) + <*> (v .:? "file") + <*> parseadded + where + parseadded = (Just <$> parseJSON (Object v)) <|> return Nothing + parseJSON _ = mempty diff --git a/Messages/Progress.hs b/Messages/Progress.hs new file mode 100644 index 0000000000..bd43dbfc4b --- /dev/null +++ b/Messages/Progress.hs @@ -0,0 +1,139 @@ +{- git-annex progress output + - + - Copyright 2010-2015 Joey Hess + - + - Licensed under the GNU GPL version 3 or higher. + -} + +{-# LANGUAGE CPP #-} + +module Messages.Progress where + +import Common +import Messages +import Utility.Metered +import Types +import Types.Messages +import Types.Key +import qualified Messages.JSON as JSON + +#ifdef WITH_CONCURRENTOUTPUT +import Messages.Concurrent +import qualified System.Console.Regions as Regions +import qualified System.Console.Concurrent as Console +#endif + +{- Shows a progress meter while performing a transfer of a key. + - The action is passed the meter and a callback to use to update the meter. + - + - When the key's size is not known, the srcfile is statted to get the size. + - This allows uploads of keys without size to still have progress + - displayed. + --} +metered :: Maybe MeterUpdate -> Key -> Annex (Maybe FilePath) -> (Meter -> MeterUpdate -> Annex a) -> Annex a +metered othermeter key getsrcfile a = withMessageState $ \st -> + flip go st =<< getsz + where + go _ (MessageState { outputType = QuietOutput }) = nometer + go msize (MessageState { outputType = NormalOutput, concurrentOutputEnabled = False }) = do + showOutput + meter <- liftIO $ mkMeter msize $ + displayMeterHandle stdout bandwidthMeter + m <- liftIO $ rateLimitMeterUpdate 0.2 meter $ + updateMeter meter + r <- a meter (combinemeter m) + liftIO $ clearMeterHandle meter stdout + return r + go msize (MessageState { outputType = NormalOutput, concurrentOutputEnabled = True }) = +#if WITH_CONCURRENTOUTPUT + withProgressRegion $ \r -> do + meter <- liftIO $ mkMeter msize $ \_ msize' old new -> + let s = bandwidthMeter msize' old new + in Regions.setConsoleRegion r ('\n' : s) + m <- liftIO $ rateLimitMeterUpdate 0.2 meter $ + updateMeter meter + a meter (combinemeter m) +#else + nometer +#endif + go msize (MessageState { outputType = JSONOutput jsonoptions }) + | jsonProgress jsonoptions = do + buf <- withMessageState $ return . jsonBuffer + meter <- liftIO $ mkMeter msize $ \_ msize' _old (new, _now) -> + JSON.progress buf msize' new + m <- liftIO $ rateLimitMeterUpdate 0.1 meter $ + updateMeter meter + a meter (combinemeter m) + | otherwise = nometer + + nometer = do + dummymeter <- liftIO $ mkMeter Nothing $ + \_ _ _ _ -> return () + a dummymeter (combinemeter (const noop)) + + combinemeter m = case othermeter of + Nothing -> m + Just om -> combineMeterUpdate m om + + getsz = case keySize key of + Just sz -> return (Just sz) + Nothing -> do + srcfile <- getsrcfile + case srcfile of + Nothing -> return Nothing + Just f -> catchMaybeIO $ liftIO $ getFileSize f + +{- Poll file size to display meter. -} +meteredFile :: FilePath -> Maybe MeterUpdate -> Key -> Annex a -> Annex a +meteredFile file combinemeterupdate key a = + metered combinemeterupdate key (return Nothing) $ \_ p -> + watchFileSize file p a + +{- Progress dots. -} +showProgressDots :: Annex () +showProgressDots = outputMessage JSON.none "." + +{- Runs a command, that may output progress to either stdout or + - stderr, as well as other messages. + - + - In quiet mode, the output is suppressed, except for error messages. + -} +progressCommand :: FilePath -> [CommandParam] -> Annex Bool +progressCommand cmd params = progressCommandEnv cmd params Nothing + +progressCommandEnv :: FilePath -> [CommandParam] -> Maybe [(String, String)] -> Annex Bool +progressCommandEnv cmd params environ = ifM commandProgressDisabled + ( do + oh <- mkOutputHandler + liftIO $ demeterCommandEnv oh cmd params environ + , liftIO $ boolSystemEnv cmd params environ + ) + +mkOutputHandler :: Annex OutputHandler +mkOutputHandler = OutputHandler + <$> commandProgressDisabled + <*> mkStderrEmitter + +mkOutputHandlerQuiet :: Annex OutputHandler +mkOutputHandlerQuiet = OutputHandler + <$> pure True + <*> mkStderrEmitter + +mkStderrRelayer :: Annex (Handle -> IO ()) +mkStderrRelayer = do + quiet <- commandProgressDisabled + emitter <- mkStderrEmitter + return $ \h -> avoidProgress quiet h emitter + +{- Generates an IO action that can be used to emit stderr. + - + - When a progress meter is displayed, this takes care to avoid + - messing it up with interleaved stderr from a command. + -} +mkStderrEmitter :: Annex (String -> IO ()) +mkStderrEmitter = withMessageState go + where +#ifdef WITH_CONCURRENTOUTPUT + go s | concurrentOutputEnabled s = return Console.errorConcurrent +#endif + go _ = return (hPutStrLn stderr) diff --git a/NEWS b/NEWS new file mode 100644 index 0000000000..f757a23feb --- /dev/null +++ b/NEWS @@ -0,0 +1,102 @@ +git-annex (6.20180626) upstream; urgency=high + + A security fix has changed git-annex to refuse to download content from + some special remotes when the content cannot be verified with a hash check. + In particular URL and WORM keys stored on such remotes won't be downloaded. + See the documentation of the annex.security.allow-unverified-downloads + configuration for how to deal with this if it affects your files. + + A security fix has changed git-annex to only support http, https, and ftp + URL schemes by default. You can enable other URL schemes, at your own risk, + using annex.security.allowed-url-schemes. + + A related security fix prevents git-annex from connecting to http + servers (and proxies) on localhost or private networks. This can + be overridden, at your own risk, using annex.security.allowed-http-addresses. + + Setting annex.web-options no longer is enough to make curl be used, + and youtube-dl is also no longer used by default. See the + documentation of annex.security.allowed-http-addresses for + details and how to enable them. + + The annex.web-download-command configuration has been removed, + use annex.web-options instead. + + -- Joey Hess Fri, 15 Jun 2018 17:54:23 -0400 + +git-annex (6.20180309) upstream; urgency=medium + + Note that, due to not using rsync to transfer files over ssh + any longer, permissions and other file metadata of annexed files + will no longer be preserved when copying them to and from ssh remotes. + Other remotes never supported preserving that information, so + this is not considered a regression. + + -- Joey Hess Fri, 09 Mar 2018 13:22:47 -0400 + +git-annex (6.20170228) upstream; urgency=medium + + This version of git-annex has mitigations for SHA1 hash collision + problems. + + A new annex.securehashesonly configuration, when used in combination with + signed git commits, avoids potential hash collision problems in git-annex + repositories. For details, see this web page: + + + -- Joey Hess Tue, 28 Feb 2017 13:28:50 -0400 + +git-annex (6.20170101) upstream; urgency=medium + + XMPP support has been removed from the assistant in this release. + + If your repositories used XMPP to keep in sync, that will no longer + work, and you should enable some other remote to keep them in sync. + A ssh server is one way, or use the new Tor pairing feature. + + -- Joey Hess Tue, 27 Dec 2016 16:37:46 -0400 + +git-annex (4.20131002) upstream; urgency=low + + The layout of gcrypt repositories has changed, and + if you created one you must manually upgrade it. + See /usr/share/doc/git-annex/html/upgrades/gcrypt.html + + -- Joey Hess Tue, 24 Sep 2013 13:55:23 -0400 + +git-annex (3.20120123) upstream; urgency=low + + There was a bug in the handling of directory special remotes that + could cause partial file contents to be stored in them. If you use + a directory special remote, you should fsck it, to avoid potential + data loss. + + Example: git annex fsck --from mydirectory + + -- Joey Hess Thu, 19 Jan 2012 15:24:23 -0400 + +git-annex (3.20110624) upstream; urgency=low + + There has been another change to the git-annex data store. + Use `git annex upgrade` to migrate your repositories to the new + layout. See or + /usr/share/doc/git-annex/html/upgrades.html + + The significant change this time is that the .git-annex/ directory + is gone; instead there is a git-annex branch that is automatically + maintained by git-annex, and encapsulates all its state nicely out + of your way. + + You should make sure you include the git-annex branch when + git pushing and pulling. + + -- Joey Hess Tue, 21 Jun 2011 20:18:00 -0400 + +git-annex (0.20110316) upstream; urgency=low + + This version reorganises the layout of git-annex's files in your repository. + There is an upgrade process to convert a repository from the old git-annex + to this version. See or + /usr/share/doc/git-annex/html/upgrades.html + + -- Joey Hess Wed, 16 Mar 2011 15:49:15 -0400 diff --git a/P2P/Address.hs b/P2P/Address.hs new file mode 100644 index 0000000000..d911f7b4b2 --- /dev/null +++ b/P2P/Address.hs @@ -0,0 +1,95 @@ +{- P2P protocol addresses + - + - Copyright 2016 Joey Hess + - + - Licensed under the GNU GPL version 3 or higher. + -} + +module P2P.Address where + +import qualified Annex +import Annex.Common +import Git +import Git.Types +import Creds +import Utility.AuthToken +import Utility.Tor + +import qualified Data.Text as T + +-- | A P2P address, without an AuthToken. +-- +-- This is enough information to connect to the peer, +-- but not enough to authenticate with it. +data P2PAddress = TorAnnex OnionAddress OnionPort + deriving (Eq, Show) + +-- | A P2P address, with an AuthToken. +-- +-- This is enough information to connect to the peer, and authenticate with +-- it. +data P2PAddressAuth = P2PAddressAuth P2PAddress AuthToken + deriving (Eq, Show) + +class FormatP2PAddress a where + formatP2PAddress :: a -> String + unformatP2PAddress :: String -> Maybe a + +instance FormatP2PAddress P2PAddress where + formatP2PAddress (TorAnnex (OnionAddress onionaddr) onionport) = + torAnnexScheme ++ ":" ++ onionaddr ++ ":" ++ show onionport + unformatP2PAddress s + | (torAnnexScheme ++ ":") `isPrefixOf` s = do + let s' = dropWhile (== ':') $ dropWhile (/= ':') s + let (onionaddr, ps) = separate (== ':') s' + onionport <- readish ps + return (TorAnnex (OnionAddress onionaddr) onionport) + | otherwise = Nothing + +torAnnexScheme :: String +torAnnexScheme = "tor-annex:" + +instance FormatP2PAddress P2PAddressAuth where + formatP2PAddress (P2PAddressAuth addr authtoken) = + formatP2PAddress addr ++ ":" ++ T.unpack (fromAuthToken authtoken) + unformatP2PAddress s = do + let (ra, rs) = separate (== ':') (reverse s) + addr <- unformatP2PAddress (reverse rs) + authtoken <- toAuthToken (T.pack $ reverse ra) + return (P2PAddressAuth addr authtoken) + +repoP2PAddress :: Repo -> Maybe P2PAddress +repoP2PAddress (Repo { location = Url url }) = unformatP2PAddress (show url) +repoP2PAddress _ = Nothing + +-- | Load known P2P addresses for this repository. +loadP2PAddresses :: Annex [P2PAddress] +loadP2PAddresses = mapMaybe unformatP2PAddress . maybe [] lines + <$> readCacheCreds p2pAddressCredsFile + +-- | Store a new P2P address for this repository. +storeP2PAddress :: P2PAddress -> Annex () +storeP2PAddress addr = do + addrs <- loadP2PAddresses + unless (addr `elem` addrs) $ do + let s = unlines $ map formatP2PAddress (addr:addrs) + let tmpnam = p2pAddressCredsFile ++ ".new" + writeCacheCreds s tmpnam + tmpf <- cacheCredsFile tmpnam + destf <- cacheCredsFile p2pAddressCredsFile + -- This may be run by root, so make the creds file + -- and directory have the same owner and group as + -- the git repository directory has. + st <- liftIO . getFileStatus =<< Annex.fromRepo repoLocation + let fixowner f = setOwnerAndGroup f (fileOwner st) (fileGroup st) + liftIO $ do + fixowner tmpf + fixowner (takeDirectory tmpf) + fixowner (takeDirectory (takeDirectory tmpf)) + renameFile tmpf destf + +p2pAddressCredsFile :: FilePath +p2pAddressCredsFile = "p2paddrs" + +torAppName :: AppName +torAppName = "tor-annex" diff --git a/P2P/Annex.hs b/P2P/Annex.hs new file mode 100644 index 0000000000..c89ef93a1e --- /dev/null +++ b/P2P/Annex.hs @@ -0,0 +1,172 @@ +{- P2P protocol, Annex implementation + - + - Copyright 2016-2018 Joey Hess + - + - Licensed under the GNU GPL version 3 or higher. + -} + +{-# LANGUAGE RankNTypes, FlexibleContexts #-} + +module P2P.Annex + ( RunState(..) + , mkRunState + , P2PConnection(..) + , runFullProto + ) where + +import Annex.Common +import Annex.Content +import Annex.Transfer +import Annex.ChangedRefs +import P2P.Protocol +import P2P.IO +import Logs.Location +import Types.NumCopies +import Types.Remote (RetrievalSecurityPolicy(..)) +import Utility.Metered + +import Control.Monad.Free + +-- Full interpreter for Proto, that can receive and send objects. +runFullProto :: RunState -> P2PConnection -> Proto a -> Annex (Either ProtoFailure a) +runFullProto runst conn = go + where + go :: RunProto Annex + go (Pure v) = return (Right v) + go (Free (Net n)) = runNet runst conn go n + go (Free (Local l)) = runLocal runst go l + +runLocal :: RunState -> RunProto Annex -> LocalF (Proto a) -> Annex (Either ProtoFailure a) +runLocal runst runner a = case a of + TmpContentSize k next -> do + tmp <- fromRepo $ gitAnnexTmpObjectLocation k + size <- liftIO $ catchDefaultIO 0 $ getFileSize tmp + runner (next (Len size)) + FileSize f next -> do + size <- liftIO $ catchDefaultIO 0 $ getFileSize f + runner (next (Len size)) + ContentSize k next -> do + let getsize = liftIO . catchMaybeIO . getFileSize + size <- inAnnex' isJust Nothing getsize k + runner (next (Len <$> size)) + ReadContent k af o sender next -> do + v <- tryNonAsync $ prepSendAnnex k + case v of + Right (Just (f, checkchanged)) -> do + v' <- tryNonAsync $ + transfer upload k af $ + sinkfile f o checkchanged sender + case v' of + Left e -> return $ Left $ ProtoFailureException e + Right (Left e) -> return $ Left e + Right (Right ok) -> runner (next ok) + -- content not available + Right Nothing -> runner (next False) + Left e -> return $ Left $ ProtoFailureException e + StoreContent k af o l getb validitycheck next -> do + -- This is the same as the retrievalSecurityPolicy of + -- Remote.P2P and Remote.Git. + let rsp = RetrievalAllKeysSecure + ok <- flip catchNonAsync (const $ return False) $ + transfer download k af $ \p -> + getViaTmp rsp DefaultVerify k $ \tmp -> do + storefile tmp o l getb validitycheck p + runner (next ok) + StoreContentTo dest o l getb validitycheck next -> do + res <- flip catchNonAsync (const $ return (False, UnVerified)) $ + storefile dest o l getb validitycheck nullMeterUpdate + runner (next res) + SetPresent k u next -> do + v <- tryNonAsync $ logChange k u InfoPresent + case v of + Left e -> return $ Left $ ProtoFailureException e + Right () -> runner next + CheckContentPresent k next -> do + v <- tryNonAsync $ inAnnex k + case v of + Left e -> return $ Left $ ProtoFailureException e + Right result -> runner (next result) + RemoveContent k next -> do + v <- tryNonAsync $ + ifM (Annex.Content.inAnnex k) + ( lockContentForRemoval k $ \contentlock -> do + removeAnnex contentlock + logStatus k InfoMissing + return True + , return True + ) + case v of + Left e -> return $ Left $ ProtoFailureException e + Right result -> runner (next result) + TryLockContent k protoaction next -> do + v <- tryNonAsync $ lockContentShared k $ \verifiedcopy -> + case verifiedcopy of + LockedCopy _ -> runner (protoaction True) + _ -> runner (protoaction False) + -- If locking fails, lockContentShared throws an exception. + -- Let the peer know it failed. + case v of + Left _ -> runner $ do + protoaction False + next + Right _ -> runner next + WaitRefChange next -> case runst of + Serving _ (Just h) _ -> do + v <- tryNonAsync $ liftIO $ waitChangedRefs h + case v of + Left e -> return $ Left $ ProtoFailureException e + Right changedrefs -> runner (next changedrefs) + _ -> return $ Left $ + ProtoFailureMessage "change notification not available" + UpdateMeterTotalSize m sz next -> do + liftIO $ setMeterTotalSize m sz + runner next + RunValidityCheck check next -> runner . next =<< check + where + transfer mk k af ta = case runst of + -- Update transfer logs when serving. + -- Using noRetry because we're the sender. + Serving theiruuid _ _ -> + mk theiruuid k af noRetry ta noNotification + -- Transfer logs are updated higher in the stack when + -- a client. + Client _ -> ta nullMeterUpdate + + storefile dest (Offset o) (Len l) getb validitycheck p = do + let p' = offsetMeterUpdate p (toBytesProcessed o) + v <- runner getb + case v of + Right b -> do + liftIO $ withBinaryFile dest ReadWriteMode $ \h -> do + when (o /= 0) $ + hSeek h AbsoluteSeek o + meteredWrite p' h b + rightsize <- do + sz <- liftIO $ getFileSize dest + return (toInteger sz == l + o) + + runner validitycheck >>= \case + Right (Just Valid) -> + return (rightsize, UnVerified) + _ -> do + -- Invalid, or old protocol + -- version. Validity is not + -- known. Force content + -- verification. + return (rightsize, MustVerify) + Left e -> error $ describeProtoFailure e + + sinkfile f (Offset o) checkchanged sender p = bracket setup cleanup go + where + setup = liftIO $ openBinaryFile f ReadMode + cleanup = liftIO . hClose + go h = do + let p' = offsetMeterUpdate p (toBytesProcessed o) + when (o /= 0) $ + liftIO $ hSeek h AbsoluteSeek o + b <- liftIO $ hGetContentsMetered h p' + let validitycheck = local $ runValidityCheck $ + checkchanged >>= return . \case + False -> Invalid + True -> Valid + runner (sender b validitycheck) diff --git a/P2P/Auth.hs b/P2P/Auth.hs new file mode 100644 index 0000000000..0025957c75 --- /dev/null +++ b/P2P/Auth.hs @@ -0,0 +1,66 @@ +{- P2P authtokens + - + - Copyright 2016 Joey Hess + - + - Licensed under the GNU GPL version 3 or higher. + -} + +module P2P.Auth where + +import Annex.Common +import Creds +import P2P.Address +import Utility.AuthToken +import Utility.Tor +import Utility.Env + +import qualified Data.Text as T + +-- | Load authtokens that are accepted by this repository. +loadP2PAuthTokens :: Annex AllowedAuthTokens +loadP2PAuthTokens = allowedAuthTokens <$> loadP2PAuthTokens' + +loadP2PAuthTokens' :: Annex [AuthToken] +loadP2PAuthTokens' = mapMaybe toAuthToken + . map T.pack + . lines + . fromMaybe [] + <$> readCacheCreds p2pAuthCredsFile + +-- | Stores an AuthToken, making it be accepted by this repository. +storeP2PAuthToken :: AuthToken -> Annex () +storeP2PAuthToken t = do + ts <- loadP2PAuthTokens' + unless (t `elem` ts) $ do + let d = unlines $ map (T.unpack . fromAuthToken) (t:ts) + writeCacheCreds d p2pAuthCredsFile + +p2pAuthCredsFile :: FilePath +p2pAuthCredsFile = "p2pauth" + +-- | Loads the AuthToken to use when connecting with a given P2P address. +-- +-- It's loaded from the first line of the creds file, but +-- GIT_ANNEX_P2P_AUTHTOKEN overrides. +loadP2PRemoteAuthToken :: P2PAddress -> Annex (Maybe AuthToken) +loadP2PRemoteAuthToken addr = maybe Nothing mk <$> getM id + [ liftIO $ getEnv "GIT_ANNEX_P2P_AUTHTOKEN" + , readCacheCreds (addressCredsFile addr) + ] + where + mk = toAuthToken . T.pack . takeWhile (/= '\n') + +p2pAuthTokenEnv :: String +p2pAuthTokenEnv = "GIT_ANNEX_P2P_AUTHTOKEN" + +-- | Stores the AuthToken o use when connecting with a given P2P address. +storeP2PRemoteAuthToken :: P2PAddress -> AuthToken -> Annex () +storeP2PRemoteAuthToken addr t = writeCacheCreds + (T.unpack $ fromAuthToken t) + (addressCredsFile addr) + +addressCredsFile :: P2PAddress -> FilePath +-- We can omit the port and just use the onion address for the creds file, +-- because any given tor hidden service runs on a single port and has a +-- unique onion address. +addressCredsFile (TorAnnex (OnionAddress onionaddr) _port) = onionaddr diff --git a/P2P/IO.hs b/P2P/IO.hs new file mode 100644 index 0000000000..0870ec53d4 --- /dev/null +++ b/P2P/IO.hs @@ -0,0 +1,379 @@ +{- P2P protocol, IO implementation + - + - Copyright 2016-2018 Joey Hess + - + - Licensed under the GNU GPL version 3 or higher. + -} + +{-# LANGUAGE RankNTypes, FlexibleContexts, CPP #-} + +module P2P.IO + ( RunProto + , RunState(..) + , mkRunState + , P2PConnection(..) + , ClosableConnection(..) + , stdioP2PConnection + , connectPeer + , closeConnection + , serveUnixSocket + , setupHandle + , ProtoFailure(..) + , describeProtoFailure + , runNetProto + , runNet + ) where + +import Common +import P2P.Protocol +import P2P.Address +import Git +import Git.Command +import Utility.AuthToken +import Utility.SimpleProtocol +import Utility.Metered +import Utility.Tor +import Utility.FileMode +import Types.UUID +import Annex.ChangedRefs + +import Control.Monad.Free +import Control.Monad.IO.Class +import System.Exit (ExitCode(..)) +import System.IO.Error +import Network.Socket +import Control.Concurrent +import Control.Concurrent.Async +import Control.Concurrent.STM +import qualified Data.ByteString as B +import qualified Data.ByteString.Lazy as L +import System.Log.Logger (debugM) +import qualified Network.Socket as S + +-- Type of interpreters of the Proto free monad. +type RunProto m = forall a. Proto a -> m (Either ProtoFailure a) + +data ProtoFailure + = ProtoFailureMessage String + | ProtoFailureException SomeException + | ProtoFailureIOError IOError + +describeProtoFailure :: ProtoFailure -> String +describeProtoFailure (ProtoFailureMessage s) = s +describeProtoFailure (ProtoFailureException e) = show e +describeProtoFailure (ProtoFailureIOError e) = show e + +data RunState + = Serving UUID (Maybe ChangedRefsHandle) (TVar ProtocolVersion) + | Client (TVar ProtocolVersion) + +mkRunState :: (TVar ProtocolVersion -> RunState) -> IO RunState +mkRunState mk = do + tvar <- newTVarIO defaultProtocolVersion + return (mk tvar) + +data P2PConnection = P2PConnection + { connRepo :: Repo + , connCheckAuth :: (AuthToken -> Bool) + , connIhdl :: Handle + , connOhdl :: Handle + } + +data ClosableConnection conn + = OpenConnection conn + | ClosedConnection + +-- P2PConnection using stdio. +stdioP2PConnection :: Git.Repo -> P2PConnection +stdioP2PConnection g = P2PConnection + { connRepo = g + , connCheckAuth = const False + , connIhdl = stdin + , connOhdl = stdout + } + +-- Opens a connection to a peer. Does not authenticate with it. +connectPeer :: Git.Repo -> P2PAddress -> IO P2PConnection +connectPeer g (TorAnnex onionaddress onionport) = do + h <- setupHandle =<< connectHiddenService onionaddress onionport + return $ P2PConnection + { connRepo = g + , connCheckAuth = const False + , connIhdl = h + , connOhdl = h + } + +closeConnection :: P2PConnection -> IO () +closeConnection conn = do + hClose (connIhdl conn) + hClose (connOhdl conn) + +-- Serves the protocol on a unix socket. +-- +-- The callback is run to serve a connection, and is responsible for +-- closing the Handle when done. +-- +-- Note that while the callback is running, other connections won't be +-- processed, so longterm work should be run in a separate thread by +-- the callback. +serveUnixSocket :: FilePath -> (Handle -> IO ()) -> IO () +serveUnixSocket unixsocket serveconn = do + nukeFile unixsocket + soc <- S.socket S.AF_UNIX S.Stream S.defaultProtocol + S.bind soc (S.SockAddrUnix unixsocket) + -- Allow everyone to read and write to the socket, + -- so a daemon like tor, that is probably running as a different + -- de sock $ addModes + -- user, can access it. + -- + -- Connections have to authenticate to do anything, + -- so it's fine that other local users can connect to the + -- socket. + modifyFileMode unixsocket $ addModes + [groupReadMode, groupWriteMode, otherReadMode, otherWriteMode] + S.listen soc 2 + forever $ do + (conn, _) <- S.accept soc + setupHandle conn >>= serveconn + +setupHandle :: Socket -> IO Handle +setupHandle s = do + h <- socketToHandle s ReadWriteMode + hSetBuffering h LineBuffering + hSetBinaryMode h False + return h + +-- Purposefully incomplete interpreter of Proto. +-- +-- This only runs Net actions. No Local actions will be run +-- (those need the Annex monad) -- if the interpreter reaches any, +-- it returns Nothing. +runNetProto :: RunState -> P2PConnection -> Proto a -> IO (Either ProtoFailure a) +runNetProto runst conn = go + where + go :: RunProto IO + go (Pure v) = return (Right v) + go (Free (Net n)) = runNet runst conn go n + go (Free (Local _)) = return $ Left $ + ProtoFailureMessage "unexpected annex operation attempted" + +-- Interpreter of the Net part of Proto. +-- +-- An interpreter of Proto has to be provided, to handle the rest of Proto +-- actions. +runNet :: (MonadIO m, MonadMask m) => RunState -> P2PConnection -> RunProto m -> NetF (Proto a) -> m (Either ProtoFailure a) +runNet runst conn runner f = case f of + SendMessage m next -> do + v <- liftIO $ tryNonAsync $ do + let l = unwords (formatMessage m) + debugMessage "P2P >" m + hPutStrLn (connOhdl conn) l + hFlush (connOhdl conn) + case v of + Left e -> return $ Left $ ProtoFailureException e + Right () -> runner next + ReceiveMessage next -> do + v <- liftIO $ tryIOError $ getProtocolLine (connIhdl conn) + case v of + Left e -> return $ Left $ ProtoFailureIOError e + Right Nothing -> return $ Left $ + ProtoFailureMessage "protocol error" + Right (Just l) -> case parseMessage l of + Just m -> do + liftIO $ debugMessage "P2P <" m + runner (next (Just m)) + Nothing -> runner (next Nothing) + SendBytes len b p next -> do + v <- liftIO $ tryNonAsync $ do + ok <- sendExactly len b (connOhdl conn) p + hFlush (connOhdl conn) + return ok + case v of + Right True -> runner next + Right False -> return $ Left $ + ProtoFailureMessage "short data write" + Left e -> return $ Left $ ProtoFailureException e + ReceiveBytes len p next -> do + v <- liftIO $ tryNonAsync $ receiveExactly len (connIhdl conn) p + case v of + Left e -> return $ Left $ ProtoFailureException e + Right b -> runner (next b) + CheckAuthToken _u t next -> do + let authed = connCheckAuth conn t + runner (next authed) + Relay hin hout next -> do + v <- liftIO $ runRelay runnerio hin hout + case v of + Left e -> return $ Left e + Right exitcode -> runner (next exitcode) + RelayService service next -> do + v <- liftIO $ runRelayService conn runnerio service + case v of + Left e -> return $ Left e + Right () -> runner next + SetProtocolVersion v next -> do + liftIO $ atomically $ writeTVar versiontvar v + runner next + GetProtocolVersion next -> + liftIO (readTVarIO versiontvar) >>= runner . next + where + -- This is only used for running Net actions when relaying, + -- so it's ok to use runNetProto, despite it not supporting + -- all Proto actions. + runnerio = runNetProto runst conn + versiontvar = case runst of + Serving _ _ tv -> tv + Client tv -> tv + +debugMessage :: String -> Message -> IO () +debugMessage prefix m = debugM "p2p" $ + prefix ++ " " ++ unwords (formatMessage safem) + where + safem = case m of + AUTH u _ -> AUTH u nullAuthToken + _ -> m + +-- Send exactly the specified number of bytes or returns False. +-- +-- The ByteString can be larger or smaller than the specified length. +-- For example, it can be lazily streaming from a file that gets +-- appended to, or truncated. +-- +-- Must avoid sending too many bytes as it would confuse the other end. +-- This is easily dealt with by truncating it. +-- +-- If too few bytes are sent, the only option is to give up on this +-- connection. False is returned to indicate this problem. +sendExactly :: Len -> L.ByteString -> Handle -> MeterUpdate -> IO Bool +sendExactly (Len n) b h p = do + sent <- meteredWrite' p h (L.take (fromIntegral n) b) + return (fromBytesProcessed sent == n) + +receiveExactly :: Len -> Handle -> MeterUpdate -> IO L.ByteString +receiveExactly (Len n) h p = hGetMetered h (Just n) p + +runRelay :: RunProto IO -> RelayHandle -> RelayHandle -> IO (Either ProtoFailure ExitCode) +runRelay runner (RelayHandle hout) (RelayHandle hin) = + bracket setup cleanup go + `catchNonAsync` (return . Left . ProtoFailureException) + where + setup = do + v <- newEmptyMVar + void $ async $ relayFeeder runner v hin + void $ async $ relayReader v hout + return v + + cleanup _ = do + hClose hin + hClose hout + + go v = relayHelper runner v + +runRelayService :: P2PConnection -> RunProto IO -> Service -> IO (Either ProtoFailure ()) +runRelayService conn runner service = + bracket setup cleanup go + `catchNonAsync` (return . Left . ProtoFailureException) + where + cmd = case service of + UploadPack -> "upload-pack" + ReceivePack -> "receive-pack" + + serviceproc = gitCreateProcess + [ Param cmd + , File (repoPath (connRepo conn)) + ] (connRepo conn) + + setup = do + (Just hin, Just hout, _, pid) <- createProcess serviceproc + { std_out = CreatePipe + , std_in = CreatePipe + } + v <- newEmptyMVar + void $ async $ relayFeeder runner v hin + void $ async $ relayReader v hout + waiter <- async $ waitexit v pid + return (v, waiter, hin, hout, pid) + + cleanup (_, waiter, hin, hout, pid) = do + hClose hin + hClose hout + cancel waiter + void $ waitForProcess pid + + go (v, _, _, _, _) = do + r <- relayHelper runner v + case r of + Left e -> return $ Left e + Right exitcode -> runner $ net $ relayToPeer (RelayDone exitcode) + + waitexit v pid = putMVar v . RelayDone =<< waitForProcess pid + +-- Processes RelayData as it is put into the MVar. +relayHelper :: RunProto IO -> MVar RelayData -> IO (Either ProtoFailure ExitCode) +relayHelper runner v = loop + where + loop = do + d <- takeMVar v + case d of + RelayToPeer b -> do + r <- runner $ net $ relayToPeer (RelayToPeer b) + case r of + Left e -> return (Left e) + Right () -> loop + RelayDone exitcode -> do + _ <- runner $ net $ relayToPeer (RelayDone exitcode) + return (Right exitcode) + RelayFromPeer _ -> loop -- not handled here + +-- Takes input from the peer, and sends it to the relay process's stdin. +-- Repeats until the peer tells it it's done or hangs up. +relayFeeder :: RunProto IO -> MVar RelayData -> Handle -> IO () +relayFeeder runner v hin = loop + where + loop = do + mrd <- runner $ net relayFromPeer + case mrd of + Left _e -> + putMVar v (RelayDone (ExitFailure 1)) + Right (RelayDone exitcode) -> + putMVar v (RelayDone exitcode) + Right (RelayFromPeer b) -> do + L.hPut hin b + hFlush hin + loop + Right (RelayToPeer _) -> loop -- not handled here + +-- Reads input from the Handle and puts it into the MVar for relaying to +-- the peer. Continues until EOF on the Handle. +relayReader :: MVar RelayData -> Handle -> IO () +relayReader v hout = loop + where + loop = do + bs <- getsome [] + case bs of + [] -> return () + _ -> do + putMVar v $ RelayToPeer (L.fromChunks bs) + loop + + -- Waiit for the first available chunk. Then, without blocking, + -- try to get more chunks, in case a stream of chunks is being + -- written in close succession. + -- + -- On Windows, hGetNonBlocking is broken, so avoid using it there. + getsome [] = do + b <- B.hGetSome hout chunk + if B.null b + then return [] +#ifndef mingw32_HOST_OS + else getsome [b] +#else + else return [b] +#endif + getsome bs = do + b <- B.hGetNonBlocking hout chunk + if B.null b + then return (reverse bs) + else getsome (b:bs) + + chunk = 65536 diff --git a/P2P/Protocol.hs b/P2P/Protocol.hs new file mode 100644 index 0000000000..49a3d5bf6f --- /dev/null +++ b/P2P/Protocol.hs @@ -0,0 +1,605 @@ +{- P2P protocol + - + - See doc/design/p2p_protocol.mdwn + - + - Copyright 2016-2018 Joey Hess + - + - Licensed under the GNU GPL version 3 or higher. + -} + +{-# LANGUAGE DeriveFunctor, TemplateHaskell, FlexibleContexts #-} +{-# LANGUAGE TypeSynonymInstances, FlexibleInstances, RankNTypes #-} +{-# OPTIONS_GHC -fno-warn-orphans #-} + +module P2P.Protocol where + +import qualified Utility.SimpleProtocol as Proto +import Types (Annex) +import Types.Key +import Types.UUID +import Types.Remote (Verification(..), unVerified) +import Utility.AuthToken +import Utility.Applicative +import Utility.PartialPrelude +import Utility.Metered +import Git.FilePath +import Annex.ChangedRefs (ChangedRefs) + +import Control.Monad +import Control.Monad.Free +import Control.Monad.Free.TH +import Control.Monad.Catch +import System.FilePath +import System.Exit (ExitCode(..)) +import System.IO +import qualified Data.ByteString.Lazy as L +import Data.Char +import Control.Applicative +import Prelude + +newtype Offset = Offset Integer + deriving (Show) + +newtype Len = Len Integer + deriving (Show) + +newtype ProtocolVersion = ProtocolVersion Integer + deriving (Show, Eq, Ord) + +defaultProtocolVersion :: ProtocolVersion +defaultProtocolVersion = ProtocolVersion 0 + +maxProtocolVersion :: ProtocolVersion +maxProtocolVersion = ProtocolVersion 1 + +-- | Service as used by the connect message in gitremote-helpers(1) +data Service = UploadPack | ReceivePack + deriving (Show) + +data Validity = Valid | Invalid + deriving (Show) + +-- | Messages in the protocol. The peer that makes the connection +-- always initiates requests, and the other peer makes responses to them. +data Message + = AUTH UUID AuthToken -- uuid of the peer that is authenticating + | AUTH_SUCCESS UUID -- uuid of the remote peer + | AUTH_FAILURE + | VERSION ProtocolVersion + | CONNECT Service + | CONNECTDONE ExitCode + | NOTIFYCHANGE + | CHANGED ChangedRefs + | CHECKPRESENT Key + | LOCKCONTENT Key + | UNLOCKCONTENT + | REMOVE Key + | GET Offset AssociatedFile Key + | PUT AssociatedFile Key + | PUT_FROM Offset + | ALREADY_HAVE + | SUCCESS + | FAILURE + | DATA Len -- followed by bytes of data + | VALIDITY Validity + | ERROR String + deriving (Show) + +instance Proto.Sendable Message where + formatMessage (AUTH uuid authtoken) = ["AUTH", Proto.serialize uuid, Proto.serialize authtoken] + formatMessage (AUTH_SUCCESS uuid) = ["AUTH-SUCCESS", Proto.serialize uuid] + formatMessage AUTH_FAILURE = ["AUTH-FAILURE"] + formatMessage (VERSION v) = ["VERSION", Proto.serialize v] + formatMessage (CONNECT service) = ["CONNECT", Proto.serialize service] + formatMessage (CONNECTDONE exitcode) = ["CONNECTDONE", Proto.serialize exitcode] + formatMessage NOTIFYCHANGE = ["NOTIFYCHANGE"] + formatMessage (CHANGED refs) = ["CHANGED", Proto.serialize refs] + formatMessage (CHECKPRESENT key) = ["CHECKPRESENT", Proto.serialize key] + formatMessage (LOCKCONTENT key) = ["LOCKCONTENT", Proto.serialize key] + formatMessage UNLOCKCONTENT = ["UNLOCKCONTENT"] + formatMessage (REMOVE key) = ["REMOVE", Proto.serialize key] + formatMessage (GET offset af key) = ["GET", Proto.serialize offset, Proto.serialize af, Proto.serialize key] + formatMessage (PUT af key) = ["PUT", Proto.serialize af, Proto.serialize key] + formatMessage (PUT_FROM offset) = ["PUT-FROM", Proto.serialize offset] + formatMessage ALREADY_HAVE = ["ALREADY-HAVE"] + formatMessage SUCCESS = ["SUCCESS"] + formatMessage FAILURE = ["FAILURE"] + formatMessage (VALIDITY Valid) = ["VALID"] + formatMessage (VALIDITY Invalid) = ["INVALID"] + formatMessage (DATA len) = ["DATA", Proto.serialize len] + formatMessage (ERROR err) = ["ERROR", Proto.serialize err] + +instance Proto.Receivable Message where + parseCommand "AUTH" = Proto.parse2 AUTH + parseCommand "AUTH-SUCCESS" = Proto.parse1 AUTH_SUCCESS + parseCommand "AUTH-FAILURE" = Proto.parse0 AUTH_FAILURE + parseCommand "VERSION" = Proto.parse1 VERSION + parseCommand "CONNECT" = Proto.parse1 CONNECT + parseCommand "CONNECTDONE" = Proto.parse1 CONNECTDONE + parseCommand "NOTIFYCHANGE" = Proto.parse0 NOTIFYCHANGE + parseCommand "CHANGED" = Proto.parse1 CHANGED + parseCommand "CHECKPRESENT" = Proto.parse1 CHECKPRESENT + parseCommand "LOCKCONTENT" = Proto.parse1 LOCKCONTENT + parseCommand "UNLOCKCONTENT" = Proto.parse0 UNLOCKCONTENT + parseCommand "REMOVE" = Proto.parse1 REMOVE + parseCommand "GET" = Proto.parse3 GET + parseCommand "PUT" = Proto.parse2 PUT + parseCommand "PUT-FROM" = Proto.parse1 PUT_FROM + parseCommand "ALREADY-HAVE" = Proto.parse0 ALREADY_HAVE + parseCommand "SUCCESS" = Proto.parse0 SUCCESS + parseCommand "FAILURE" = Proto.parse0 FAILURE + parseCommand "DATA" = Proto.parse1 DATA + parseCommand "ERROR" = Proto.parse1 ERROR + parseCommand "VALID" = Proto.parse0 (VALIDITY Valid) + parseCommand "INVALID" = Proto.parse0 (VALIDITY Invalid) + parseCommand _ = Proto.parseFail + +instance Proto.Serializable ProtocolVersion where + serialize (ProtocolVersion n) = show n + deserialize = ProtocolVersion <$$> readish + +instance Proto.Serializable Offset where + serialize (Offset n) = show n + deserialize = Offset <$$> readish + +instance Proto.Serializable Len where + serialize (Len n) = show n + deserialize = Len <$$> readish + +instance Proto.Serializable Service where + serialize UploadPack = "git-upload-pack" + serialize ReceivePack = "git-receive-pack" + deserialize "git-upload-pack" = Just UploadPack + deserialize "git-receive-pack" = Just ReceivePack + deserialize _ = Nothing + +-- | Since AssociatedFile is not the last thing in a protocol line, +-- its serialization cannot contain any whitespace. This is handled +-- by replacing whitespace with '%' (and '%' with '%%') +-- +-- When deserializing an AssociatedFile from a peer, it's sanitized, +-- to avoid any unusual characters that might cause problems when it's +-- displayed to the user. +-- +-- These mungings are ok, because an AssociatedFile is only ever displayed +-- to the user and does not need to match a file on disk. +instance Proto.Serializable AssociatedFile where + serialize (AssociatedFile Nothing) = "" + serialize (AssociatedFile (Just af)) = + toInternalGitPath $ concatMap esc af + where + esc '%' = "%%" + esc c + | isSpace c = "%" + | otherwise = [c] + + deserialize s = case fromInternalGitPath $ deesc [] s of + [] -> Just (AssociatedFile Nothing) + f + | isRelative f -> Just (AssociatedFile (Just f)) + | otherwise -> Nothing + where + deesc b [] = reverse b + deesc b ('%':'%':cs) = deesc ('%':b) cs + deesc b ('%':cs) = deesc ('_':b) cs + deesc b (c:cs) + | isControl c = deesc ('_':b) cs + | otherwise = deesc (c:b) cs + +-- | Free monad for the protocol, combining net communication, +-- and local actions. +data ProtoF c = Net (NetF c) | Local (LocalF c) + deriving (Functor) + +type Proto = Free ProtoF + +net :: Net a -> Proto a +net = hoistFree Net + +local :: Local a -> Proto a +local = hoistFree Local + +data NetF c + = SendMessage Message c + | ReceiveMessage (Maybe Message -> c) + | SendBytes Len L.ByteString MeterUpdate c + -- ^ Sends exactly Len bytes of data. (Any more or less will + -- confuse the receiver.) + | ReceiveBytes Len MeterUpdate (L.ByteString -> c) + -- ^ Lazily reads bytes from peer. Stops once Len are read, + -- or if connection is lost, and in either case returns the bytes + -- that were read. This allows resuming interrupted transfers. + | CheckAuthToken UUID AuthToken (Bool -> c) + | RelayService Service c + -- ^ Runs a service, relays its output to the peer, and data + -- from the peer to it. + | Relay RelayHandle RelayHandle (ExitCode -> c) + -- ^ Reads from the first RelayHandle, and sends the data to a + -- peer, while at the same time accepting input from the peer + -- which is sent the the second RelayHandle. Continues until + -- the peer sends an ExitCode. + | SetProtocolVersion ProtocolVersion c + --- ^ Called when a new protocol version has been negotiated. + | GetProtocolVersion (ProtocolVersion -> c) + deriving (Functor) + +type Net = Free NetF + +newtype RelayHandle = RelayHandle Handle + +data LocalF c + = TmpContentSize Key (Len -> c) + -- ^ Gets size of the temp file where received content may have + -- been stored. If not present, returns 0. + | FileSize FilePath (Len -> c) + -- ^ Gets size of the content of a file. If not present, returns 0. + | ContentSize Key (Maybe Len -> c) + -- ^ Gets size of the content of a key, when the full content is + -- present. + | ReadContent Key AssociatedFile Offset (L.ByteString -> Proto Validity -> Proto Bool) (Bool -> c) + -- ^ Reads the content of a key and sends it to the callback. + -- If the content is not available, sends L.empty to the callback. + -- Note that the content may change while it's being sent. + -- The callback is passed a validity check that it can run after + -- sending the content to detect when this happened. + | StoreContent Key AssociatedFile Offset Len (Proto L.ByteString) (Proto (Maybe Validity)) (Bool -> c) + -- ^ Stores content to the key's temp file starting at an offset. + -- Once the whole content of the key has been stored, moves the + -- temp file into place as the content of the key, and returns True. + -- + -- If the validity check is provided and fails, the content was + -- changed while it was being sent, so verificiation of the + -- received content should be forced. + -- + -- Note: The ByteString may not contain the entire remaining content + -- of the key. Only once the temp file size == Len has the whole + -- content been transferred. + | StoreContentTo FilePath Offset Len (Proto L.ByteString) (Proto (Maybe Validity)) ((Bool, Verification) -> c) + -- ^ Like StoreContent, but stores the content to a temp file. + | SetPresent Key UUID c + | CheckContentPresent Key (Bool -> c) + -- ^ Checks if the whole content of the key is locally present. + | RemoveContent Key (Bool -> c) + -- ^ If the content is not present, still succeeds. + -- May fail if not enough copies to safely drop, etc. + | TryLockContent Key (Bool -> Proto ()) c + -- ^ Try to lock the content of a key, preventing it + -- from being deleted, while running the provided protocol + -- action. If unable to lock the content, or the content is not + -- present, runs the protocol action with False. + | WaitRefChange (ChangedRefs -> c) + -- ^ Waits for one or more git refs to change and returns them.a + | UpdateMeterTotalSize Meter Integer c + -- ^ Updates the total size of a Meter, for cases where the size is + -- not known until the data is being received. + | RunValidityCheck (Annex Validity) (Validity -> c) + -- ^ Runs a deferred validity check. + deriving (Functor) + +type Local = Free LocalF + +-- Generate sendMessage etc functions for all free monad constructors. +$(makeFree ''NetF) +$(makeFree ''LocalF) + +auth :: UUID -> AuthToken -> Proto () -> Proto (Maybe UUID) +auth myuuid t a = do + net $ sendMessage (AUTH myuuid t) + postAuth a + +postAuth :: Proto () -> Proto (Maybe UUID) +postAuth a = do + r <- net receiveMessage + case r of + Just (AUTH_SUCCESS theiruuid) -> do + a + return $ Just theiruuid + Just AUTH_FAILURE -> return Nothing + _ -> do + net $ sendMessage (ERROR "auth failed") + return Nothing + +negotiateProtocolVersion :: ProtocolVersion -> Proto () +negotiateProtocolVersion preferredversion = do + net $ sendMessage (VERSION preferredversion) + r <- net receiveMessage + case r of + Just (VERSION v) -> net $ setProtocolVersion v + -- Old server doesn't know about the VERSION command. + Just (ERROR _) -> return () + _ -> net $ sendMessage (ERROR "expected VERSION") + +checkPresent :: Key -> Proto Bool +checkPresent key = do + net $ sendMessage (CHECKPRESENT key) + checkSuccess + +{- Locks content to prevent it from being dropped, while running an action. + - + - Note that this only guarantees that the content is locked as long as the + - connection to the peer remains up. If the connection is unexpectededly + - dropped, the peer will then unlock the content. + -} +lockContentWhile + :: MonadMask m + => (forall r. r -> Proto r -> m r) + -> Key + -> (Bool -> m a) + -> m a +lockContentWhile runproto key a = bracket setup cleanup a + where + setup = runproto False $ do + net $ sendMessage (LOCKCONTENT key) + checkSuccess + cleanup True = runproto () $ net $ sendMessage UNLOCKCONTENT + cleanup False = return () + +remove :: Key -> Proto Bool +remove key = do + net $ sendMessage (REMOVE key) + checkSuccess + +get :: FilePath -> Key -> AssociatedFile -> Meter -> MeterUpdate -> Proto (Bool, Verification) +get dest key af m p = + receiveContent (Just m) p sizer storer (\offset -> GET offset af key) + where + sizer = fileSize dest + storer = storeContentTo dest + +put :: Key -> AssociatedFile -> MeterUpdate -> Proto Bool +put key af p = do + net $ sendMessage (PUT af key) + r <- net receiveMessage + case r of + Just (PUT_FROM offset) -> sendContent key af offset p + Just ALREADY_HAVE -> return True + _ -> do + net $ sendMessage (ERROR "expected PUT_FROM or ALREADY_HAVE") + return False + +data ServerHandler a + = ServerGot a + | ServerContinue + | ServerUnexpected + +-- Server loop, getting messages from the client and handling them +serverLoop :: (Message -> Proto (ServerHandler a)) -> Proto (Maybe a) +serverLoop a = do + mcmd <- net receiveMessage + case mcmd of + -- When the client sends ERROR to the server, the server + -- gives up, since it's not clear what state the client + -- is in, and so not possible to recover. + Just (ERROR _) -> return Nothing + -- When the client sends an unparseable message, the server + -- responds with an error message, and loops. This allows + -- expanding the protocol with new messages. + Nothing -> do + net $ sendMessage (ERROR "unknown command") + serverLoop a + Just cmd -> do + v <- a cmd + case v of + ServerGot r -> return (Just r) + ServerContinue -> serverLoop a + -- If the client sends an unexpected message, + -- the server will respond with ERROR, and + -- always continues processing messages. + -- + -- Since the protocol is not versioned, this + -- is necessary to handle protocol changes + -- robustly, since the client can detect when + -- it's talking to a server that does not + -- support some new feature, and fall back. + ServerUnexpected -> do + net $ sendMessage (ERROR "unexpected command") + serverLoop a + +-- | Serve the protocol, with an unauthenticated peer. Once the peer +-- successfully authenticates, returns their UUID. +serveAuth :: UUID -> Proto (Maybe UUID) +serveAuth myuuid = serverLoop handler + where + handler (AUTH theiruuid authtoken) = do + ok <- net $ checkAuthToken theiruuid authtoken + if ok + then do + net $ sendMessage (AUTH_SUCCESS myuuid) + return (ServerGot theiruuid) + else do + net $ sendMessage AUTH_FAILURE + return ServerContinue + handler _ = return ServerUnexpected + +data ServerMode + = ServeReadOnly + -- ^ Allow reading, but not writing. + | ServeAppendOnly + -- ^ Allow reading, and storing new objects, but not deleting objects. + | ServeReadWrite + -- ^ Full read and write access. + deriving (Eq, Ord) + +-- | Serve the protocol, with a peer that has authenticated. +serveAuthed :: ServerMode -> UUID -> Proto () +serveAuthed servermode myuuid = void $ serverLoop handler + where + readonlyerror = net $ sendMessage (ERROR "this repository is read-only; write access denied") + appendonlyerror = net $ sendMessage (ERROR "this repository is append-only; removal denied") + handler (VERSION theirversion) = do + let v = min theirversion maxProtocolVersion + net $ setProtocolVersion v + net $ sendMessage (VERSION v) + return ServerContinue + handler (LOCKCONTENT key) = do + local $ tryLockContent key $ \locked -> do + sendSuccess locked + when locked $ do + r' <- net receiveMessage + case r' of + Just UNLOCKCONTENT -> return () + _ -> net $ sendMessage (ERROR "expected UNLOCKCONTENT") + return ServerContinue + handler (CHECKPRESENT key) = do + sendSuccess =<< local (checkContentPresent key) + return ServerContinue + handler (REMOVE key) = case servermode of + ServeReadWrite -> do + sendSuccess =<< local (removeContent key) + return ServerContinue + ServeAppendOnly -> do + appendonlyerror + return ServerContinue + ServeReadOnly -> do + readonlyerror + return ServerContinue + handler (PUT af key) = case servermode of + ServeReadWrite -> handleput af key + ServeAppendOnly -> handleput af key + ServeReadOnly -> do + readonlyerror + return ServerContinue + handler (GET offset key af) = do + void $ sendContent af key offset nullMeterUpdate + -- setPresent not called because the peer may have + -- requested the data but not permanently stored it. + return ServerContinue + handler (CONNECT service) = do + let goahead = net $ relayService service + case (servermode, service) of + (ServeReadWrite, _) -> goahead + (ServeAppendOnly, UploadPack) -> goahead + -- git protocol could be used to overwrite + -- refs or something, so don't allow + (ServeAppendOnly, ReceivePack) -> readonlyerror + (ServeReadOnly, UploadPack) -> goahead + (ServeReadOnly, ReceivePack) -> readonlyerror + -- After connecting to git, there may be unconsumed data + -- from the git processes hanging around (even if they + -- exited successfully), so stop serving this connection. + return $ ServerGot () + handler NOTIFYCHANGE = do + refs <- local waitRefChange + net $ sendMessage (CHANGED refs) + return ServerContinue + handler _ = return ServerUnexpected + + handleput af key = do + have <- local $ checkContentPresent key + if have + then net $ sendMessage ALREADY_HAVE + else do + let sizer = tmpContentSize key + let storer = \o l b v -> unVerified $ + storeContent key af o l b v + (ok, _v) <- receiveContent Nothing nullMeterUpdate sizer storer PUT_FROM + when ok $ + local $ setPresent key myuuid + return ServerContinue + +sendContent :: Key -> AssociatedFile -> Offset -> MeterUpdate -> Proto Bool +sendContent key af offset@(Offset n) p = go =<< local (contentSize key) + where + go Nothing = sender (Len 0) L.empty (return Valid) + go (Just (Len totallen)) = do + let len = totallen - n + if len <= 0 + then sender (Len 0) L.empty (return Valid) + else local $ readContent key af offset $ + sender (Len len) + sender len content validitycheck = do + let p' = offsetMeterUpdate p (toBytesProcessed n) + net $ sendMessage (DATA len) + net $ sendBytes len content p' + ver <- net getProtocolVersion + when (ver >= ProtocolVersion 1) $ + net . sendMessage . VALIDITY =<< validitycheck + checkSuccess + +receiveContent + :: Maybe Meter + -> MeterUpdate + -> Local Len + -> (Offset -> Len -> Proto L.ByteString -> Proto (Maybe Validity) -> Local (Bool, Verification)) + -> (Offset -> Message) + -> Proto (Bool, Verification) +receiveContent mm p sizer storer mkmsg = do + Len n <- local sizer + let p' = offsetMeterUpdate p (toBytesProcessed n) + let offset = Offset n + net $ sendMessage (mkmsg offset) + r <- net receiveMessage + case r of + Just (DATA len@(Len l)) -> do + local $ case mm of + Nothing -> return () + Just m -> updateMeterTotalSize m (n+l) + ver <- net getProtocolVersion + let validitycheck = if ver >= ProtocolVersion 1 + then net receiveMessage >>= \case + Just (VALIDITY v) -> return (Just v) + _ -> do + net $ sendMessage (ERROR "expected VALID or INVALID") + return Nothing + else return Nothing + (ok, v) <- local $ storer offset len + (net (receiveBytes len p')) + validitycheck + sendSuccess ok + return (ok, v) + _ -> do + net $ sendMessage (ERROR "expected DATA") + return (False, UnVerified) + +checkSuccess :: Proto Bool +checkSuccess = do + ack <- net receiveMessage + case ack of + Just SUCCESS -> return True + Just FAILURE -> return False + _ -> do + net $ sendMessage (ERROR "expected SUCCESS or FAILURE") + return False + +sendSuccess :: Bool -> Proto () +sendSuccess True = net $ sendMessage SUCCESS +sendSuccess False = net $ sendMessage FAILURE + +notifyChange :: Proto (Maybe ChangedRefs) +notifyChange = do + net $ sendMessage NOTIFYCHANGE + ack <- net receiveMessage + case ack of + Just (CHANGED rs) -> return (Just rs) + _ -> do + net $ sendMessage (ERROR "expected CHANGED") + return Nothing + +connect :: Service -> Handle -> Handle -> Proto ExitCode +connect service hin hout = do + net $ sendMessage (CONNECT service) + net $ relay (RelayHandle hin) (RelayHandle hout) + +data RelayData + = RelayToPeer L.ByteString + | RelayFromPeer L.ByteString + | RelayDone ExitCode + deriving (Show) + +relayFromPeer :: Net RelayData +relayFromPeer = do + r <- receiveMessage + case r of + Just (CONNECTDONE exitcode) -> return $ RelayDone exitcode + Just (DATA len) -> RelayFromPeer <$> receiveBytes len nullMeterUpdate + _ -> do + sendMessage $ ERROR "expected DATA or CONNECTDONE" + return $ RelayDone $ ExitFailure 1 + +relayToPeer :: RelayData -> Net () +relayToPeer (RelayDone exitcode) = sendMessage (CONNECTDONE exitcode) +relayToPeer (RelayToPeer b) = do + let len = Len $ fromIntegral $ L.length b + sendMessage (DATA len) + sendBytes len b nullMeterUpdate +relayToPeer (RelayFromPeer _) = return () diff --git a/README b/README new file mode 100644 index 0000000000..3ff0a9bfb0 --- /dev/null +++ b/README @@ -0,0 +1,6 @@ +git-annex allows managing files with git, without checking the file +contents into git. While that may seem paradoxical, it is useful when +dealing with files larger than git can currently easily handle, whether due +to limitations in memory, checksumming time, or disk space. + +For documentation, see doc/ or diff --git a/Remote.hs b/Remote.hs new file mode 100644 index 0000000000..842c3bc606 --- /dev/null +++ b/Remote.hs @@ -0,0 +1,381 @@ +{- git-annex remotes + - + - Copyright 2011-2018 Joey Hess + - + - Licensed under the GNU GPL version 3 or higher. + -} + +module Remote ( + Remote, + uuid, + name, + storeKey, + retrieveKeyFile, + retrieveKeyFileCheap, + retrievalSecurityPolicy, + removeKey, + hasKey, + hasKeyCheap, + whereisKey, + remoteFsck, + + remoteTypes, + remoteList, + remoteList', + gitSyncableRemote, + remoteMap, + remoteMap', + uuidDescriptions, + addName, + byName, + byName', + byNameOrGroup, + byNameOnly, + byNameWithUUID, + byUUID, + byCost, + prettyPrintUUIDs, + prettyPrintUUIDsDescs, + prettyPrintUUIDsWith, + prettyListUUIDs, + prettyUUID, + remoteFromUUID, + remotesWithUUID, + remotesWithoutUUID, + keyLocations, + keyPossibilities, + remoteLocations, + nameToUUID, + nameToUUID', + showTriedRemotes, + showLocations, + forceTrust, + logStatus, + checkAvailable, + claimingUrl, + isExportSupported, +) where + +import Data.Ord +import qualified Data.Map as M +import qualified Data.Vector as V + +import Annex.Common +import Types.Remote +import qualified Annex +import Annex.UUID +import Logs.UUID +import Logs.Trust +import Logs.Location hiding (logStatus) +import Logs.Web +import Remote.List +import Config +import Config.DynamicConfig +import Git.Types (RemoteName) +import Utility.Aeson + +{- Map from UUIDs of Remotes to a calculated value. -} +remoteMap :: (Remote -> v) -> Annex (M.Map UUID v) +remoteMap mkv = remoteMap' mkv (pure . mkk) + where + mkk r = case uuid r of + NoUUID -> Nothing + u -> Just u + +remoteMap' :: Ord k => (Remote -> v) -> (Remote -> Annex (Maybe k)) -> Annex (M.Map k v) +remoteMap' mkv mkk = M.fromList . catMaybes <$> (mapM mk =<< remoteList) + where + mk r = mkk r >>= return . \case + Nothing -> Nothing + Just k -> Just (k, mkv r) + +{- Map of UUIDs of repositories and their descriptions. + - The names of Remotes are added to suppliment any description that has + - been set for a repository. -} +uuidDescriptions :: Annex (M.Map UUID String) +uuidDescriptions = M.unionWith addName <$> uuidMap <*> remoteMap name + +addName :: String -> RemoteName -> String +addName desc n + | desc == n || null desc = "[" ++ n ++ "]" + | otherwise = desc ++ " [" ++ n ++ "]" + +byUUID :: UUID -> Annex (Maybe Remote) +byUUID u = headMaybe . filter matching <$> remoteList + where + matching r = uuid r == u + +{- When a name is specified, looks up the remote matching that name. + - (Or it can be a UUID.) + - + - Throws an error if a name is specified and no matching remote can be + - found. + -} +byName :: Maybe RemoteName -> Annex (Maybe Remote) +byName Nothing = return Nothing +byName (Just n) = either giveup Just <$> byName' n + +{- Like byName, but the remote must have a configured UUID. -} +byNameWithUUID :: Maybe RemoteName -> Annex (Maybe Remote) +byNameWithUUID = checkuuid <=< byName + where + checkuuid Nothing = return Nothing + checkuuid (Just r) + | uuid r == NoUUID = do + repo <- getRepo r + ifM (liftIO $ getDynamicConfig $ remoteAnnexIgnore (gitconfig r)) + ( giveup $ noRemoteUUIDMsg r ++ + " (" ++ show (remoteConfig repo "ignore") ++ + " is set)" + , giveup $ noRemoteUUIDMsg r + ) + | otherwise = return $ Just r + +byName' :: RemoteName -> Annex (Either String Remote) +byName' "" = return $ Left "no remote specified" +byName' n = go . filter matching <$> remoteList + where + go [] = Left $ "there is no available git remote named \"" ++ n ++ "\"" + go (match:_) = Right match + matching r = n == name r || toUUID n == uuid r + +{- Finds the remote or remote group matching the name. -} +byNameOrGroup :: RemoteName -> Annex [Remote] +byNameOrGroup n = go =<< getConfigMaybe (ConfigKey ("remotes." ++ n)) + where + go (Just l) = catMaybes <$> mapM (byName . Just) (splitc ' ' l) + go Nothing = maybeToList <$> byName (Just n) + +{- Only matches remote name, not UUID -} +byNameOnly :: RemoteName -> Annex (Maybe Remote) +byNameOnly n = headMaybe . filter matching <$> remoteList + where + matching r = n == name r + +noRemoteUUIDMsg :: Remote -> String +noRemoteUUIDMsg r = "cannot determine uuid for " ++ name r ++ " (perhaps you need to run \"git annex sync\"?)" + +{- Looks up a remote by name (or by UUID, or even by description), + - and returns its UUID. Finds even repositories that are not + - configured in .git/config. -} +nameToUUID :: RemoteName -> Annex UUID +nameToUUID = either giveup return <=< nameToUUID' + +nameToUUID' :: RemoteName -> Annex (Either String UUID) +nameToUUID' "." = Right <$> getUUID -- special case for current repo +nameToUUID' "here" = Right <$> getUUID +nameToUUID' n = byName' n >>= go + where + go (Right r) = return $ case uuid r of + NoUUID -> Left $ noRemoteUUIDMsg r + u -> Right u + go (Left e) = do + m <- uuidMap + return $ case M.keys (M.filter (== n) m) of + [u] -> Right u + [] -> let u = toUUID n + in case M.keys (M.filterWithKey (\k _ -> k == u) m) of + [] -> Left e + _ -> Right u + _us -> Left "Found multiple repositories with that description" + +{- Pretty-prints a list of UUIDs of remotes, with their descriptions, + - for human display. + - + - When JSON is enabled, also outputs a machine-readable description + - of the UUIDs. -} +prettyPrintUUIDs :: String -> [UUID] -> Annex String +prettyPrintUUIDs header uuids = do + descm <- uuidDescriptions + prettyPrintUUIDsDescs header descm uuids + +prettyPrintUUIDsDescs :: String -> M.Map UUID RemoteName -> [UUID] -> Annex String +prettyPrintUUIDsDescs header descm uuids = + prettyPrintUUIDsWith Nothing header descm + (const Nothing) + (zip uuids (repeat (Nothing :: Maybe String))) + +{- An optional field can be included in the list of UUIDs. -} +prettyPrintUUIDsWith + :: ToJSON' v + => Maybe String + -> String + -> M.Map UUID RemoteName + -> (v -> Maybe String) + -> [(UUID, Maybe v)] + -> Annex String +prettyPrintUUIDsWith optfield header descm showval uuidvals = do + hereu <- getUUID + maybeShowJSON $ JSONChunk [(header, V.fromList $ map (jsonify hereu) uuidvals)] + return $ unwords $ map (\u -> "\t" ++ prettify hereu u ++ "\n") uuidvals + where + finddescription u = M.findWithDefault "" u descm + prettify hereu (u, optval) + | not (null d) = addoptval $ fromUUID u ++ " -- " ++ d + | otherwise = addoptval $ fromUUID u + where + ishere = hereu == u + n = finddescription u + d + | null n && ishere = "here" + | ishere = addName n "here" + | otherwise = n + addoptval s = case showval =<< optval of + Nothing -> s + Just val -> val ++ ": " ++ s + jsonify hereu (u, optval) = object $ catMaybes + [ Just (packString "uuid", toJSON' $ fromUUID u) + , Just (packString "description", toJSON' $ finddescription u) + , Just (packString "here", toJSON' $ hereu == u) + , case (optfield, optval) of + (Just field, Just val) -> Just (packString field, toJSON' val) + _ -> Nothing + ] + +{- List of remote names and/or descriptions, for human display. -} +prettyListUUIDs :: [UUID] -> Annex [String] +prettyListUUIDs uuids = do + hereu <- getUUID + m <- uuidDescriptions + return $ map (prettify m hereu) uuids + where + finddescription m u = M.findWithDefault "" u m + prettify m hereu u + | u == hereu = addName n "here" + | otherwise = n + where + n = finddescription m u + +{- Nice display of a remote's name and/or description. -} +prettyUUID :: UUID -> Annex String +prettyUUID u = concat <$> prettyListUUIDs [u] + +{- Gets the remote associated with a UUID. -} +remoteFromUUID :: UUID -> Annex (Maybe Remote) +remoteFromUUID u = ifM ((==) u <$> getUUID) + ( return Nothing + , maybe tryharder (return . Just) =<< findinmap + ) + where + findinmap = M.lookup u <$> remoteMap id + {- Re-read remote list in case a new remote has popped up. -} + tryharder = do + void remoteListRefresh + findinmap + +{- Filters a list of remotes to ones that have the listed uuids. -} +remotesWithUUID :: [Remote] -> [UUID] -> [Remote] +remotesWithUUID rs us = filter (\r -> uuid r `elem` us) rs + +{- Filters a list of remotes to ones that do not have the listed uuids. -} +remotesWithoutUUID :: [Remote] -> [UUID] -> [Remote] +remotesWithoutUUID rs us = filter (\r -> uuid r `notElem` us) rs + +{- List of repository UUIDs that the location log indicates may have a key. + - Dead repositories are excluded. -} +keyLocations :: Key -> Annex [UUID] +keyLocations key = trustExclude DeadTrusted =<< loggedLocations key + +{- Cost ordered lists of remotes that the location log indicates + - may have a key. + - + - Also includes remotes with remoteAnnexSpeculatePresent set. + -} +keyPossibilities :: Key -> Annex [Remote] +keyPossibilities key = do + u <- getUUID + -- uuids of all remotes that are recorded to have the key + locations <- filter (/= u) <$> keyLocations key + speclocations <- map uuid + . filter (remoteAnnexSpeculatePresent . gitconfig) + <$> remoteList + -- there are unlikely to be many speclocations, so building a Set + -- is not worth the expense + let locations' = speclocations ++ filter (`notElem` speclocations) locations + fst <$> remoteLocations locations' [] + +{- Given a list of locations of a key, and a list of all + - trusted repositories, generates a cost-ordered list of + - remotes that contain the key, and a list of trusted locations of the key. + -} +remoteLocations :: [UUID] -> [UUID] -> Annex ([Remote], [UUID]) +remoteLocations locations trusted = do + let validtrustedlocations = nub locations `intersect` trusted + + -- remotes that match uuids that have the key + allremotes <- remoteList + >>= filterM (not <$$> liftIO . getDynamicConfig . remoteAnnexIgnore . gitconfig) + let validremotes = remotesWithUUID allremotes locations + + return (sortBy (comparing cost) validremotes, validtrustedlocations) + +{- Displays known locations of a key. -} +showLocations :: Bool -> Key -> [UUID] -> String -> Annex () +showLocations separateuntrusted key exclude nolocmsg = do + u <- getUUID + uuids <- keyLocations key + untrusteduuids <- if separateuntrusted + then trustGet UnTrusted + else pure [] + let uuidswanted = filteruuids uuids (u:exclude++untrusteduuids) + let uuidsskipped = filteruuids uuids (u:exclude++uuidswanted) + ppuuidswanted <- prettyPrintUUIDs "wanted" uuidswanted + ppuuidsskipped <- prettyPrintUUIDs "skipped" uuidsskipped + let msg = message ppuuidswanted ppuuidsskipped + unless (null msg) $ + showLongNote msg + ignored <- remoteList + >>= filterM (liftIO . getDynamicConfig . remoteAnnexIgnore . gitconfig) + unless (null ignored) $ + showLongNote $ "(Note that these git remotes have annex-ignore set: " ++ unwords (map name ignored) ++ ")" + where + filteruuids l x = filter (`notElem` x) l + message [] [] = nolocmsg + message rs [] = "Try making some of these repositories available:\n" ++ rs + message [] us = "Also these untrusted repositories may contain the file:\n" ++ us + message rs us = message rs [] ++ message [] us + +showTriedRemotes :: [Remote] -> Annex () +showTriedRemotes [] = noop +showTriedRemotes remotes = + showLongNote $ "Unable to access these remotes: " ++ + intercalate ", " (map name remotes) + +forceTrust :: TrustLevel -> String -> Annex () +forceTrust level remotename = do + u <- nameToUUID remotename + Annex.changeState $ \s -> + s { Annex.forcetrust = M.insert u level (Annex.forcetrust s) } + +{- Used to log a change in a remote's having a key. The change is logged + - in the local repo, not on the remote. The process of transferring the + - key to the remote, or removing the key from it *may* log the change + - on the remote, but this cannot always be relied on. -} +logStatus :: Remote -> Key -> LogStatus -> Annex () +logStatus remote key = logChange key (uuid remote) + +{- Orders remotes by cost, with ones with the lowest cost grouped together. -} +byCost :: [Remote] -> [[Remote]] +byCost = map snd . sortBy (comparing fst) . M.toList . costmap + where + costmap = M.fromListWith (++) . map costpair + costpair r = (cost r, [r]) + +checkAvailable :: Bool -> Remote -> IO Bool +checkAvailable assumenetworkavailable = + maybe (return assumenetworkavailable) doesDirectoryExist . localpath + +hasKey :: Remote -> Key -> Annex (Either String Bool) +hasKey r k = either (Left . show) Right <$> tryNonAsync (checkPresent r k) + +hasKeyCheap :: Remote -> Bool +hasKeyCheap = checkPresentCheap + +{- The web special remote claims urls by default. -} +claimingUrl :: URLString -> Annex Remote +claimingUrl url = do + rs <- remoteList + let web = Prelude.head $ filter (\r -> uuid r == webUUID) rs + fromMaybe web <$> firstM checkclaim rs + where + checkclaim = maybe (pure False) (`id` url) . claimUrl diff --git a/Remote/Adb.hs b/Remote/Adb.hs new file mode 100644 index 0000000000..13c1ebcbd4 --- /dev/null +++ b/Remote/Adb.hs @@ -0,0 +1,282 @@ +{- Remote on Android device accessed using adb. + - + - Copyright 2018 Joey Hess + - + - Licensed under the GNU AGPL version 3 or higher. + -} + +module Remote.Adb (remote) where + +import qualified Data.Map as M + +import Annex.Common +import Types.Remote +import Types.Creds +import Types.Export +import qualified Git +import Config.Cost +import Remote.Helper.Special +import Remote.Helper.Messages +import Remote.Helper.Export +import Annex.UUID +import Utility.Metered + +-- | Each Android device has a serial number. +newtype AndroidSerial = AndroidSerial { fromAndroidSerial :: String } + deriving (Show, Eq) + +-- | A location on an Android device. +newtype AndroidPath = AndroidPath { fromAndroidPath :: FilePath } + +remote :: RemoteType +remote = RemoteType + { typename = "adb" + , enumerate = const (findSpecialRemotes "adb") + , generate = gen + , setup = adbSetup + , exportSupported = exportIsSupported + } + +gen :: Git.Repo -> UUID -> RemoteConfig -> RemoteGitConfig -> Annex (Maybe Remote) +gen r u c gc = do + let this = Remote + { uuid = u + -- adb operates over USB or wifi, so is not as cheap + -- as local, but not too expensive + , cost = semiExpensiveRemoteCost + , name = Git.repoDescribe r + , storeKey = storeKeyDummy + , retrieveKeyFile = retreiveKeyFileDummy + , retrieveKeyFileCheap = \_ _ _ -> return False + , retrievalSecurityPolicy = RetrievalAllKeysSecure + , removeKey = removeKeyDummy + , lockContent = Nothing + , checkPresent = checkPresentDummy + , checkPresentCheap = False + , exportActions = return $ ExportActions + { storeExport = storeExportM serial adir + , retrieveExport = retrieveExportM serial adir + , removeExport = removeExportM serial adir + , checkPresentExport = checkPresentExportM this serial adir + , removeExportDirectory = Just $ removeExportDirectoryM serial adir + , renameExport = renameExportM serial adir + } + , whereisKey = Nothing + , remoteFsck = Nothing + , repairRepo = Nothing + , config = c + , getRepo = return r + , gitconfig = gc + , localpath = Nothing + , remotetype = remote + , availability = LocallyAvailable + , readonly = False + , appendonly = False + , mkUnavailable = return Nothing + , getInfo = return + [ ("androidserial", fromAndroidSerial serial) + , ("androiddirectory", fromAndroidPath adir) + ] + , claimUrl = Nothing + , checkUrl = Nothing + } + return $ Just $ specialRemote c + (simplyPrepare $ store serial adir) + (simplyPrepare $ retrieve serial adir) + (simplyPrepare $ remove serial adir) + (simplyPrepare $ checkKey this serial adir) + this + where + adir = maybe (giveup "missing androiddirectory") AndroidPath + (remoteAnnexAndroidDirectory gc) + serial = maybe (giveup "missing androidserial") AndroidSerial + (remoteAnnexAndroidSerial gc) + +adbSetup :: SetupStage -> Maybe UUID -> Maybe CredPair -> RemoteConfig -> RemoteGitConfig -> Annex (RemoteConfig, UUID) +adbSetup _ mu _ c gc = do + u <- maybe (liftIO genUUID) return mu + + -- verify configuration + adir <- maybe (giveup "Specify androiddirectory=") (pure . AndroidPath) + (M.lookup "androiddirectory" c) + serial <- getserial =<< liftIO enumerateAdbConnected + let c' = M.insert "androidserial" (fromAndroidSerial serial) c + + (c'', _encsetup) <- encryptionSetup c' gc + + ok <- liftIO $ adbShellBool serial + [Param "mkdir", Param "-p", File (fromAndroidPath adir)] + unless ok $ + giveup "Creating directory on Android device failed." + + gitConfigSpecialRemote u c'' + [ ("adb", "true") + , ("androiddirectory", fromAndroidPath adir) + , ("androidserial", fromAndroidSerial serial) + ] + + return (c'', u) + where + getserial [] = giveup "adb does not list any connected android devices. Plug in an Android device, or configure adb, and try again.." + getserial l = case M.lookup "androidserial" c of + Nothing -> case l of + (s:[]) -> return s + _ -> giveup $ unlines $ + "There are multiple connected android devices, specify which to use with androidserial=" + : map fromAndroidSerial l + Just cs + | AndroidSerial cs `elem` l -> return (AndroidSerial cs) + | otherwise -> giveup $ "The device with androidserial=" ++ cs ++ " is not connected." + +store :: AndroidSerial -> AndroidPath -> Storer +store serial adir = fileStorer $ \k src _p -> + let dest = androidLocation adir k + in store' serial dest src + +store' :: AndroidSerial -> AndroidPath -> FilePath -> Annex Bool +store' serial dest src = do + let destdir = takeDirectory $ fromAndroidPath dest + liftIO $ void $ adbShell serial [Param "mkdir", Param "-p", File destdir] + showOutput -- make way for adb push output + let tmpdest = fromAndroidPath dest ++ ".tmp" + ifM (liftIO $ boolSystem "adb" (mkAdbCommand serial [Param "push", File src, File tmpdest])) + -- move into place atomically + ( liftIO $ adbShellBool serial [Param "mv", File tmpdest, File (fromAndroidPath dest)] + , return False + ) + +retrieve :: AndroidSerial -> AndroidPath -> Retriever +retrieve serial adir = fileRetriever $ \dest k _p -> + let src = androidLocation adir k + in unlessM (retrieve' serial src dest) $ + giveup "adb pull failed" + +retrieve' :: AndroidSerial -> AndroidPath -> FilePath -> Annex Bool +retrieve' serial src dest = do + showOutput -- make way for adb pull output + liftIO $ boolSystem "adb" $ mkAdbCommand serial + [ Param "pull" + , File $ fromAndroidPath src + , File dest + ] + +remove :: AndroidSerial -> AndroidPath -> Remover +remove serial adir k = remove' serial (androidLocation adir k) + +remove' :: AndroidSerial -> AndroidPath -> Annex Bool +remove' serial aloc = liftIO $ adbShellBool serial + [Param "rm", Param "-f", File (fromAndroidPath aloc)] + +checkKey :: Remote -> AndroidSerial -> AndroidPath -> CheckPresent +checkKey r serial adir k = checkKey' r serial (androidLocation adir k) + +checkKey' :: Remote -> AndroidSerial -> AndroidPath -> Annex Bool +checkKey' r serial aloc = do + showChecking r + (out, st) <- liftIO $ adbShellRaw serial $ unwords + [ "if test -e ", shellEscape (fromAndroidPath aloc) + , "; then echo y" + , "; else echo n" + , "; fi" + ] + case (out, st) of + (["y"], ExitSuccess) -> return True + (["n"], ExitSuccess) -> return False + _ -> giveup $ "unable to access Android device" ++ show out + +androidLocation :: AndroidPath -> Key -> AndroidPath +androidLocation adir k = AndroidPath $ + fromAndroidPath (androidHashDir adir k) ++ key2file k + +androidHashDir :: AndroidPath -> Key -> AndroidPath +androidHashDir adir k = AndroidPath $ + fromAndroidPath adir ++ "/" ++ hdir + where + hdir = replace [pathSeparator] "/" (hashDirLower def k) + +storeExportM :: AndroidSerial -> AndroidPath -> FilePath -> Key -> ExportLocation -> MeterUpdate -> Annex Bool +storeExportM serial adir src _k loc _p = store' serial dest src + where + dest = androidExportLocation adir loc + +retrieveExportM :: AndroidSerial -> AndroidPath -> Key -> ExportLocation -> FilePath -> MeterUpdate -> Annex Bool +retrieveExportM serial adir _k loc dest _p = retrieve' serial src dest + where + src = androidExportLocation adir loc + +removeExportM :: AndroidSerial -> AndroidPath -> Key -> ExportLocation -> Annex Bool +removeExportM serial adir _k loc = remove' serial aloc + where + aloc = androidExportLocation adir loc + +removeExportDirectoryM :: AndroidSerial -> AndroidPath -> ExportDirectory -> Annex Bool +removeExportDirectoryM serial abase dir = liftIO $ adbShellBool serial + [Param "rm", Param "-rf", File (fromAndroidPath adir)] + where + adir = androidExportLocation abase (mkExportLocation (fromExportDirectory dir)) + +checkPresentExportM :: Remote -> AndroidSerial -> AndroidPath -> Key -> ExportLocation -> Annex Bool +checkPresentExportM r serial adir _k loc = checkKey' r serial aloc + where + aloc = androidExportLocation adir loc + +renameExportM :: AndroidSerial -> AndroidPath -> Key -> ExportLocation -> ExportLocation -> Annex Bool +renameExportM serial adir _k old new = liftIO $ adbShellBool serial + [Param "mv", Param "-f", File oldloc, File newloc] + where + oldloc = fromAndroidPath $ androidExportLocation adir old + newloc = fromAndroidPath $ androidExportLocation adir new + +androidExportLocation :: AndroidPath -> ExportLocation -> AndroidPath +androidExportLocation adir loc = AndroidPath $ + fromAndroidPath adir ++ "/" ++ fromExportLocation loc + +-- | List all connected Android devices. +enumerateAdbConnected :: IO [AndroidSerial] +enumerateAdbConnected = + mapMaybe parse . lines <$> readProcess "adb" ["devices"] + where + parse l = + let (serial, desc) = separate (== '\t') l + in if null desc || length serial < 4 + then Nothing + else Just (AndroidSerial serial) + +-- | Runs a command on the android device with the given serial number. +-- +-- adb shell does not propigate the exit code of the command, so +-- it is echoed out in a trailing line, and the output is read to determine +-- it. Any stdout from the command is returned, separated into lines. +adbShell :: AndroidSerial -> [CommandParam] -> IO ([String], ExitCode) +adbShell serial cmd = adbShellRaw serial $ + unwords $ map shellEscape (toCommand cmd) + +adbShellBool :: AndroidSerial -> [CommandParam] -> IO Bool +adbShellBool serial cmd = do + (_ , ec) <- adbShell serial cmd + return (ec == ExitSuccess) + +-- | Runs a raw shell command on the android device. +-- Any necessary shellEscaping must be done by caller. +adbShellRaw :: AndroidSerial -> String -> IO ([String], ExitCode) +adbShellRaw serial cmd = processoutput <$> readProcess "adb" + [ "-s" + , fromAndroidSerial serial + , "shell" + -- The extra echo is in case cmd does not output a trailing + -- newline after its other output. + , cmd ++ "; echo; echo $?" + ] + where + processoutput s = case reverse (map trimcr (lines s)) of + (c:"":rest) -> case readish c of + Just 0 -> (reverse rest, ExitSuccess) + Just n -> (reverse rest, ExitFailure n) + Nothing -> (reverse rest, ExitFailure 1) + ls -> (reverse ls, ExitFailure 1) + -- For some reason, adb outputs lines with \r\n on linux, + -- despite both linux and android being unix systems. + trimcr = takeWhile (/= '\r') + +mkAdbCommand :: AndroidSerial -> [CommandParam] -> [CommandParam] +mkAdbCommand serial cmd = [Param "-s", Param (fromAndroidSerial serial)] ++ cmd diff --git a/Remote/BitTorrent.hs b/Remote/BitTorrent.hs new file mode 100644 index 0000000000..d4ccf87305 --- /dev/null +++ b/Remote/BitTorrent.hs @@ -0,0 +1,403 @@ +{- BitTorrent remote. + - + - Copyright 2014 Joey Hess + - + - Licensed under the GNU GPL version 3 or higher. + -} + +{-# LANGUAGE CPP #-} + +module Remote.BitTorrent (remote) where + +import Annex.Common +import Types.Remote +import qualified Annex +import qualified Git +import qualified Git.Construct +import Config +import Config.Cost +import Logs.Web +import Types.UrlContents +import Types.CleanupActions +import Messages.Progress +import Utility.Metered +import Utility.Tmp +import Utility.FileSystemEncoding +import Backend.URL +import Annex.Perms +import Annex.UUID +import qualified Annex.Url as Url +import Remote.Helper.Export + +import Network.URI + +#ifdef WITH_TORRENTPARSER +import Data.Torrent +import qualified Data.ByteString.Lazy as B +#endif + +remote :: RemoteType +remote = RemoteType + { typename = "bittorrent" + , enumerate = list + , generate = gen + , setup = error "not supported" + , exportSupported = exportUnsupported + } + +-- There is only one bittorrent remote, and it always exists. +list :: Bool -> Annex [Git.Repo] +list _autoinit = do + r <- liftIO $ Git.Construct.remoteNamed "bittorrent" (pure Git.Construct.fromUnknown) + return [r] + +gen :: Git.Repo -> UUID -> RemoteConfig -> RemoteGitConfig -> Annex (Maybe Remote) +gen r _ c gc = do + cst <- remoteCost gc expensiveRemoteCost + return $ Just Remote + { uuid = bitTorrentUUID + , cost = cst + , name = Git.repoDescribe r + , storeKey = uploadKey + , retrieveKeyFile = downloadKey + , retrieveKeyFileCheap = downloadKeyCheap + -- Bittorrent does its own hash checks. + , retrievalSecurityPolicy = RetrievalAllKeysSecure + , removeKey = dropKey + , lockContent = Nothing + , checkPresent = checkKey + , checkPresentCheap = False + , exportActions = exportUnsupported + , whereisKey = Nothing + , remoteFsck = Nothing + , repairRepo = Nothing + , config = c + , gitconfig = gc + , localpath = Nothing + , getRepo = return r + , readonly = True + , appendonly = False + , availability = GloballyAvailable + , remotetype = remote + , mkUnavailable = return Nothing + , getInfo = return [] + , claimUrl = Just (pure . isSupportedUrl) + , checkUrl = Just checkTorrentUrl + } + +downloadKey :: Key -> AssociatedFile -> FilePath -> MeterUpdate -> Annex (Bool, Verification) +downloadKey key _file dest p = unVerified $ + get . map (torrentUrlNum . fst . getDownloader) =<< getBitTorrentUrls key + where + get [] = do + warning "could not download torrent" + return False + get urls = do + showOutput -- make way for download progress bar + untilTrue urls $ \(u, filenum) -> do + registerTorrentCleanup u + checkDependencies + ifM (downloadTorrentFile u) + ( downloadTorrentContent key u dest filenum p + , return False + ) + +downloadKeyCheap :: Key -> AssociatedFile -> FilePath -> Annex Bool +downloadKeyCheap _ _ _ = return False + +uploadKey :: Key -> AssociatedFile -> MeterUpdate -> Annex Bool +uploadKey _ _ _ = do + warning "upload to bittorrent not supported" + return False + +dropKey :: Key -> Annex Bool +dropKey k = do + mapM_ (setUrlMissing bitTorrentUUID k) =<< getBitTorrentUrls k + return True + +{- We punt and don't try to check if a torrent has enough seeders + - with all the pieces etc. That would be quite hard.. and even if + - implemented, it tells us nothing about the later state of the torrent. + -} +checkKey :: Key -> Annex Bool +checkKey = giveup "cannot reliably check torrent status" + +getBitTorrentUrls :: Key -> Annex [URLString] +getBitTorrentUrls key = filter supported <$> getUrls key + where + supported u = + let (u', dl) = (getDownloader u) + in dl == OtherDownloader && isSupportedUrl u' + +isSupportedUrl :: URLString -> Bool +isSupportedUrl u = isTorrentMagnetUrl u || isTorrentUrl u + +isTorrentUrl :: URLString -> Bool +isTorrentUrl = maybe False (\u -> ".torrent" `isSuffixOf` uriPath u) . parseURI + +isTorrentMagnetUrl :: URLString -> Bool +isTorrentMagnetUrl u = "magnet:" `isPrefixOf` u && checkbt (parseURI u) + where + checkbt (Just uri) | "xt=urn:btih:" `isInfixOf` uriQuery uri = True + checkbt _ = False + +checkTorrentUrl :: URLString -> Annex UrlContents +checkTorrentUrl u = do + checkDependencies + registerTorrentCleanup u + ifM (downloadTorrentFile u) + ( torrentContents u + , giveup "could not download torrent file" + ) + +{- To specify which file inside a multi-url torrent, the file number is + - appended to the url. -} +torrentUrlWithNum :: URLString -> Int -> URLString +torrentUrlWithNum u n = u ++ "#" ++ show n + +torrentUrlNum :: URLString -> (URLString, Int) +torrentUrlNum u + | '#' `elem` u = + let (n, ru) = separate (== '#') (reverse u) + in (reverse ru, fromMaybe 1 $ readish $ reverse n) + | otherwise = (u, 1) + +{- A Key corresponding to the URL of a torrent file. -} +torrentUrlKey :: URLString -> Annex Key +torrentUrlKey u = return $ fromUrl (fst $ torrentUrlNum u) Nothing + +{- Temporary directory used to download a torrent. -} +tmpTorrentDir :: URLString -> Annex FilePath +tmpTorrentDir u = do + d <- fromRepo gitAnnexTmpMiscDir + f <- keyFile <$> torrentUrlKey u + return (d f) + +{- Temporary filename to use to store the torrent file. -} +tmpTorrentFile :: URLString -> Annex FilePath +tmpTorrentFile u = fromRepo . gitAnnexTmpObjectLocation =<< torrentUrlKey u + +{- A cleanup action is registered to delete the torrent file and its + - associated temp directory when git-annex exits. + - + - This allows multiple actions that use the same torrent file and temp + - directory to run in a single git-annex run. + -} +registerTorrentCleanup :: URLString -> Annex () +registerTorrentCleanup u = Annex.addCleanup (TorrentCleanup u) $ do + liftIO . nukeFile =<< tmpTorrentFile u + d <- tmpTorrentDir u + liftIO $ whenM (doesDirectoryExist d) $ + removeDirectoryRecursive d + +{- Downloads the torrent file. (Not its contents.) -} +downloadTorrentFile :: URLString -> Annex Bool +downloadTorrentFile u = do + torrent <- tmpTorrentFile u + ifM (liftIO $ doesFileExist torrent) + ( return True + , do + showAction "downloading torrent file" + createAnnexDirectory (parentDir torrent) + if isTorrentMagnetUrl u + then do + tmpdir <- tmpTorrentDir u + let metadir = tmpdir "meta" + createAnnexDirectory metadir + showOutput + ok <- downloadMagnetLink u metadir torrent + liftIO $ removeDirectoryRecursive metadir + return ok + else do + misctmp <- fromRepo gitAnnexTmpMiscDir + withTmpFileIn misctmp "torrent" $ \f h -> do + liftIO $ hClose h + ok <- Url.withUrlOptions $ + liftIO . Url.download nullMeterUpdate u f + when ok $ + liftIO $ renameFile f torrent + return ok + ) + +downloadMagnetLink :: URLString -> FilePath -> FilePath -> Annex Bool +downloadMagnetLink u metadir dest = ifM download + ( liftIO $ do + ts <- filter (".torrent" `isSuffixOf`) + <$> dirContents metadir + case ts of + (t:[]) -> do + renameFile t dest + return True + _ -> return False + , return False + ) + where + download = runAria + [ Param "--bt-metadata-only" + , Param "--bt-save-metadata" + , Param u + , Param "--seed-time=0" + , Param "--summary-interval=0" + , Param "-d" + , File metadir + ] + +downloadTorrentContent :: Key -> URLString -> FilePath -> Int -> MeterUpdate -> Annex Bool +downloadTorrentContent k u dest filenum p = do + torrent <- tmpTorrentFile u + tmpdir <- tmpTorrentDir u + createAnnexDirectory tmpdir + f <- wantedfile torrent + showOutput + ifM (download torrent tmpdir <&&> liftIO (doesFileExist (tmpdir f))) + ( do + liftIO $ renameFile (tmpdir f) dest + return True + , return False + ) + where + download torrent tmpdir = ariaProgress (keySize k) p + [ Param $ "--select-file=" ++ show filenum + , File torrent + , Param "-d" + , File tmpdir + , Param "--seed-time=0" + , Param "--summary-interval=0" + , Param "--file-allocation=none" + -- Needed so aria will resume partially downloaded files + -- in multi-file torrents. + , Param "--check-integrity=true" + ] + + {- aria2c will create part of the directory structure + - contained in the torrent. It may download parts of other files + - in addition to the one we asked for. So, we need to find + - out the filename we want based on the filenum. + -} + wantedfile torrent = do + fs <- liftIO $ map fst <$> torrentFileSizes torrent + if length fs >= filenum + then return (fs !! (filenum - 1)) + else giveup "Number of files in torrent seems to have changed." + +checkDependencies :: Annex () +checkDependencies = do + missing <- liftIO $ filterM (not <$$> inPath) deps + unless (null missing) $ + giveup $ "need to install additional software in order to download from bittorrent: " ++ unwords missing + where + deps = + [ "aria2c" +#ifndef TORRENT + , "btshowmetainfo" +#endif + ] + +ariaParams :: [CommandParam] -> Annex [CommandParam] +ariaParams ps = do + opts <- map Param . annexAriaTorrentOptions <$> Annex.getGitConfig + return (ps ++ opts) + +runAria :: [CommandParam] -> Annex Bool +runAria ps = progressCommand "aria2c" =<< ariaParams ps + +-- Parse aria output to find "(n%)" and update the progress meter +-- with it. +ariaProgress :: Maybe Integer -> MeterUpdate -> [CommandParam] -> Annex Bool +ariaProgress Nothing _ ps = runAria ps +ariaProgress (Just sz) meter ps = do + oh <- mkOutputHandler + liftIO . commandMeter (parseAriaProgress sz) oh meter "aria2c" + =<< ariaParams ps + +parseAriaProgress :: Integer -> ProgressParser +parseAriaProgress totalsize = go [] . reverse . splitc '\r' + where + go remainder [] = (Nothing, remainder) + go remainder (x:xs) = case readish (findpercent x) of + Nothing -> go (x++remainder) xs + Just p -> (Just (frompercent p), remainder) + + -- "(N%)" + findpercent = takeWhile (/= '%') . drop 1 . dropWhile (/= '(') + + frompercent p = toBytesProcessed $ totalsize * p `div` 100 + +{- Used only if the haskell torrent library is not available. -} +#ifndef WITH_TORRENTPARSER +btshowmetainfo :: FilePath -> String -> IO [String] +btshowmetainfo torrent field = + findfield [] . lines <$> readProcess "btshowmetainfo" [torrent] + where + findfield c [] = reverse c + findfield c (l:ls) + | l == fieldkey = multiline c ls + | fieldkey `isPrefixOf` l = + findfield ((drop (length fieldkey) l):c) ls + | otherwise = findfield c ls + + multiline c (l:ls) + | " " `isPrefixOf` l = multiline (drop 3 l:c) ls + | otherwise = findfield c ls + multiline c [] = findfield c [] + + fieldkey = field ++ take (14 - length field) (repeat '.') ++ ": " +#endif + +{- Examines the torrent file and gets the list of files in it, + - and their sizes. + -} +torrentFileSizes :: FilePath -> IO [(FilePath, Integer)] +torrentFileSizes torrent = do +#ifdef WITH_TORRENTPARSER + let mkfile = joinPath . map (scrub . decodeBS) + b <- B.readFile torrent + return $ case readTorrent b of + Left e -> giveup $ "failed to parse torrent: " ++ e + Right t -> case tInfo t of + SingleFile { tLength = l, tName = f } -> + [ (mkfile [f], l) ] + MultiFile { tFiles = fs, tName = dir } -> + map (\tf -> (mkfile $ dir:filePath tf, fileLength tf)) fs + where +#else + files <- getfield "files" + if null files + then do + fnl <- getfield "file name" + szl <- map readish <$> getfield "file size" + case (fnl, szl) of + ((fn:[]), (Just sz:[])) -> return [(scrub fn, sz)] + _ -> parsefailed (show (fnl, szl)) + else do + v <- getfield "directory name" + case v of + (d:[]) -> return $ map (splitsize d) files + _ -> parsefailed (show v) + where + getfield = btshowmetainfo torrent + parsefailed s = giveup $ "failed to parse btshowmetainfo output for torrent file: " ++ show s + + -- btshowmetainfo outputs a list of "filename (size)" + splitsize d l = (scrub (d fn), sz) + where + sz = fromMaybe (parsefailed l) $ readish $ + reverse $ takeWhile (/= '(') $ dropWhile (== ')') $ + reverse l + fn = reverse $ drop 2 $ + dropWhile (/= '(') $ dropWhile (== ')') $ reverse l +#endif + -- a malicious torrent file might try to do directory traversal + scrub f = if isAbsolute f || any (== "..") (splitPath f) + then giveup "found unsafe filename in torrent!" + else f + +torrentContents :: URLString -> Annex UrlContents +torrentContents u = convert + <$> (liftIO . torrentFileSizes =<< tmpTorrentFile u) + where + convert [(fn, sz)] = UrlContents (Just sz) (Just (mkSafeFilePath fn)) + convert l = UrlMulti $ map mkmulti (zip l [1..]) + + mkmulti ((fn, sz), n) = + (torrentUrlWithNum u n, Just sz, mkSafeFilePath $ joinPath $ drop 1 $ splitPath fn) diff --git a/Remote/Bup.hs b/Remote/Bup.hs new file mode 100644 index 0000000000..8bc04574ea --- /dev/null +++ b/Remote/Bup.hs @@ -0,0 +1,286 @@ +{- Using bup as a remote. + - + - Copyright 2011-2014 Joey Hess + - + - Licensed under the GNU GPL version 3 or higher. + -} + +module Remote.Bup (remote) where + +import qualified Data.Map as M +import qualified Data.ByteString.Lazy as L +import Data.ByteString.Lazy.UTF8 (fromString) + +import Annex.Common +import qualified Annex +import Types.Remote +import Types.Creds +import qualified Git +import qualified Git.Command +import qualified Git.Config +import qualified Git.Construct +import qualified Git.Ref +import Config +import Config.Cost +import qualified Remote.Helper.Ssh as Ssh +import Remote.Helper.Special +import Remote.Helper.Messages +import Remote.Helper.Export +import Utility.Hash +import Utility.UserInfo +import Annex.UUID +import Annex.Ssh +import Utility.Metered + +type BupRepo = String + +remote :: RemoteType +remote = RemoteType + { typename = "bup" + , enumerate = const (findSpecialRemotes "buprepo") + , generate = gen + , setup = bupSetup + , exportSupported = exportUnsupported + } + +gen :: Git.Repo -> UUID -> RemoteConfig -> RemoteGitConfig -> Annex (Maybe Remote) +gen r u c gc = do + bupr <- liftIO $ bup2GitRemote buprepo + cst <- remoteCost gc $ + if bupLocal buprepo + then nearlyCheapRemoteCost + else expensiveRemoteCost + (u', bupr') <- getBupUUID bupr u + + let this = Remote + { uuid = u' + , cost = cst + , name = Git.repoDescribe r + , storeKey = storeKeyDummy + , retrieveKeyFile = retreiveKeyFileDummy + , retrieveKeyFileCheap = retrieveCheap buprepo + -- Bup uses git, which cryptographically verifies content + -- (with SHA1, but sufficiently for this). + , retrievalSecurityPolicy = RetrievalAllKeysSecure + , removeKey = removeKeyDummy + , lockContent = Nothing + , checkPresent = checkPresentDummy + , checkPresentCheap = bupLocal buprepo + , exportActions = exportUnsupported + , whereisKey = Nothing + , remoteFsck = Nothing + , repairRepo = Nothing + , config = c + , getRepo = return r + , gitconfig = gc + , localpath = if bupLocal buprepo && not (null buprepo) + then Just buprepo + else Nothing + , remotetype = remote + , availability = if bupLocal buprepo then LocallyAvailable else GloballyAvailable + , readonly = False + , appendonly = False + , mkUnavailable = return Nothing + , getInfo = return [("repo", buprepo)] + , claimUrl = Nothing + , checkUrl = Nothing + } + return $ Just $ specialRemote' specialcfg c + (simplyPrepare $ store this buprepo) + (simplyPrepare $ retrieve buprepo) + (simplyPrepare $ remove buprepo) + (simplyPrepare $ checkKey r bupr') + this + where + buprepo = fromMaybe (giveup "missing buprepo") $ remoteAnnexBupRepo gc + specialcfg = (specialRemoteCfg c) + -- chunking would not improve bup + { chunkConfig = NoChunks + } + +bupSetup :: SetupStage -> Maybe UUID -> Maybe CredPair -> RemoteConfig -> RemoteGitConfig -> Annex (RemoteConfig, UUID) +bupSetup _ mu _ c gc = do + u <- maybe (liftIO genUUID) return mu + + -- verify configuration is sane + let buprepo = fromMaybe (giveup "Specify buprepo=") $ + M.lookup "buprepo" c + (c', _encsetup) <- encryptionSetup c gc + + -- bup init will create the repository. + -- (If the repository already exists, bup init again appears safe.) + showAction "bup init" + unlessM (bup "init" buprepo []) $ giveup "bup init failed" + + storeBupUUID u buprepo + + -- The buprepo is stored in git config, as well as this repo's + -- persistant state, so it can vary between hosts. + gitConfigSpecialRemote u c' [("buprepo", buprepo)] + + return (c', u) + +bupParams :: String -> BupRepo -> [CommandParam] -> [CommandParam] +bupParams command buprepo params = + Param command : [Param "-r", Param buprepo] ++ params + +bup :: String -> BupRepo -> [CommandParam] -> Annex Bool +bup command buprepo params = do + showOutput -- make way for bup output + liftIO $ boolSystem "bup" $ bupParams command buprepo params + +bupSplitParams :: Remote -> BupRepo -> Key -> [CommandParam] -> [CommandParam] +bupSplitParams r buprepo k src = + let os = map Param $ remoteAnnexBupSplitOptions $ gitconfig r + in bupParams "split" buprepo + (os ++ [Param "-q", Param "-n", Param (bupRef k)] ++ src) + +store :: Remote -> BupRepo -> Storer +store r buprepo = byteStorer $ \k b p -> do + let params = bupSplitParams r buprepo k [] + showOutput -- make way for bup output + let cmd = proc "bup" (toCommand params) + quiet <- commandProgressDisabled + let feeder = \h -> do + meteredWrite p h b + return True + liftIO $ if quiet + then feedWithQuietOutput createProcessSuccess cmd feeder + else withHandle StdinHandle createProcessSuccess cmd feeder + +retrieve :: BupRepo -> Retriever +retrieve buprepo = byteRetriever $ \k sink -> do + let params = bupParams "join" buprepo [Param $ bupRef k] + let p = proc "bup" (toCommand params) + (_, Just h, _, pid) <- liftIO $ createProcess $ p { std_out = CreatePipe } + liftIO (hClose h >> forceSuccessProcess p pid) + `after` (sink =<< liftIO (L.hGetContents h)) + +retrieveCheap :: BupRepo -> Key -> AssociatedFile -> FilePath -> Annex Bool +retrieveCheap _ _ _ _ = return False + +{- Cannot revert having stored a key in bup, but at least the data for the + - key will be used for deltaing data of other keys stored later. + - + - We can, however, remove the git branch that bup created for the key. + -} +remove :: BupRepo -> Remover +remove buprepo k = do + go =<< liftIO (bup2GitRemote buprepo) + warning "content cannot be completely removed from bup remote" + return True + where + go r + | Git.repoIsUrl r = void $ onBupRemote r boolSystem "git" params + | otherwise = void $ liftIO $ catchMaybeIO $ do + r' <- Git.Config.read r + boolSystem "git" $ Git.Command.gitCommandLine params r' + params = [ Param "branch", Param "-q", Param "-D", Param (bupRef k) ] + +{- Bup does not provide a way to tell if a given dataset is present + - in a bup repository. One way it to check if the git repository has + - a branch matching the name (as created by bup split -n). + -} +checkKey :: Git.Repo -> Git.Repo -> CheckPresent +checkKey r bupr k + | Git.repoIsUrl bupr = do + showChecking r + onBupRemote bupr boolSystem "git" params + | otherwise = liftIO $ boolSystem "git" $ + Git.Command.gitCommandLine params bupr + where + params = + [ Param "show-ref" + , Param "--quiet" + , Param "--verify" + , Param $ "refs/heads/" ++ bupRef k + ] + +{- Store UUID in the annex.uuid setting of the bup repository. -} +storeBupUUID :: UUID -> BupRepo -> Annex () +storeBupUUID u buprepo = do + r <- liftIO $ bup2GitRemote buprepo + if Git.repoIsUrl r + then do + showAction "storing uuid" + unlessM (onBupRemote r boolSystem "git" + [Param "config", Param "annex.uuid", Param v]) $ + giveup "ssh failed" + else liftIO $ do + r' <- Git.Config.read r + let olduuid = Git.Config.get "annex.uuid" "" r' + when (olduuid == "") $ + Git.Command.run + [ Param "config" + , Param "annex.uuid" + , Param v + ] r' + where + v = fromUUID u + +onBupRemote :: Git.Repo -> (FilePath -> [CommandParam] -> IO a) -> FilePath -> [CommandParam] -> Annex a +onBupRemote r runner command params = do + c <- Annex.getRemoteGitConfig r + let remotecmd = "cd " ++ dir ++ " && " ++ unwords (command : toCommand params) + (sshcmd, sshparams) <- Ssh.toRepo NoConsumeStdin r c remotecmd + liftIO $ runner sshcmd sshparams + where + path = Git.repoPath r + base = fromMaybe path (stripPrefix "/~/" path) + dir = shellEscape base + +{- Allow for bup repositories on removable media by checking + - local bup repositories to see if they are available, and getting their + - uuid (which may be different from the stored uuid for the bup remote). + - + - If a bup repository is not available, returns NoUUID. + - This will cause checkPresent to indicate nothing from the bup remote + - is known to be present. + - + - Also, returns a version of the repo with config read, if it is local. + -} +getBupUUID :: Git.Repo -> UUID -> Annex (UUID, Git.Repo) +getBupUUID r u + | Git.repoIsUrl r = return (u, r) + | otherwise = liftIO $ do + ret <- tryIO $ Git.Config.read r + case ret of + Right r' -> return (toUUID $ Git.Config.get "annex.uuid" "" r', r') + Left _ -> return (NoUUID, r) + +{- Converts a bup remote path spec into a Git.Repo. There are some + - differences in path representation between git and bup. -} +bup2GitRemote :: BupRepo -> IO Git.Repo +bup2GitRemote "" = do + -- bup -r "" operates on ~/.bup + h <- myHomeDir + Git.Construct.fromAbsPath $ h ".bup" +bup2GitRemote r + | bupLocal r = + if "/" `isPrefixOf` r + then Git.Construct.fromAbsPath r + else giveup "please specify an absolute path" + | otherwise = Git.Construct.fromUrl $ "ssh://" ++ host ++ slash dir + where + bits = splitc ':' r + host = Prelude.head bits + dir = intercalate ":" $ drop 1 bits + -- "host:~user/dir" is not supported specially by bup; + -- "host:dir" is relative to the home directory; + -- "host:" goes in ~/.bup + slash d + | null d = "/~/.bup" + | "/" `isPrefixOf` d = d + | otherwise = "/~/" ++ d + +{- Converts a key into a git ref name, which bup-split -n will use to point + - to it. -} +bupRef :: Key -> String +bupRef k + | Git.Ref.legal True shown = shown + | otherwise = "git-annex-" ++ show (sha2_256 (fromString shown)) + where + shown = key2file k + +bupLocal :: BupRepo -> Bool +bupLocal = notElem ':' diff --git a/Remote/Ddar.hs b/Remote/Ddar.hs new file mode 100644 index 0000000000..da4db9865e --- /dev/null +++ b/Remote/Ddar.hs @@ -0,0 +1,203 @@ +{- Using ddar as a remote. Based on bup and rsync remotes. + - + - Copyright 2011 Joey Hess + - Copyright 2014 Robie Basak + - + - Licensed under the GNU GPL version 3 or higher. + -} + +module Remote.Ddar (remote) where + +import qualified Data.Map as M +import qualified Data.ByteString.Lazy as L +import System.IO.Error + +import Annex.Common +import Types.Remote +import Types.Creds +import qualified Git +import Config +import Config.Cost +import Remote.Helper.Special +import Remote.Helper.Export +import Annex.Ssh +import Annex.UUID +import Utility.SshHost + +data DdarRepo = DdarRepo + { ddarRepoConfig :: RemoteGitConfig + , ddarRepoLocation :: String + } + +remote :: RemoteType +remote = RemoteType + { typename = "ddar" + , enumerate = const (findSpecialRemotes "ddarrepo") + , generate = gen + , setup = ddarSetup + , exportSupported = exportUnsupported + } + +gen :: Git.Repo -> UUID -> RemoteConfig -> RemoteGitConfig -> Annex (Maybe Remote) +gen r u c gc = do + cst <- remoteCost gc $ + if ddarLocal ddarrepo + then nearlyCheapRemoteCost + else expensiveRemoteCost + return $ Just $ specialRemote' specialcfg c + (simplyPrepare $ store ddarrepo) + (simplyPrepare $ retrieve ddarrepo) + (simplyPrepare $ remove ddarrepo) + (simplyPrepare $ checkKey ddarrepo) + (this cst) + where + this cst = Remote + { uuid = u + , cost = cst + , name = Git.repoDescribe r + , storeKey = storeKeyDummy + , retrieveKeyFile = retreiveKeyFileDummy + , retrieveKeyFileCheap = retrieveCheap + -- ddar communicates over ssh, not subject to http redirect + -- type attacks + , retrievalSecurityPolicy = RetrievalAllKeysSecure + , removeKey = removeKeyDummy + , lockContent = Nothing + , checkPresent = checkPresentDummy + , checkPresentCheap = ddarLocal ddarrepo + , exportActions = exportUnsupported + , whereisKey = Nothing + , remoteFsck = Nothing + , repairRepo = Nothing + , config = c + , getRepo = return r + , gitconfig = gc + , localpath = if ddarLocal ddarrepo && not (null $ ddarRepoLocation ddarrepo) + then Just $ ddarRepoLocation ddarrepo + else Nothing + , remotetype = remote + , availability = if ddarLocal ddarrepo then LocallyAvailable else GloballyAvailable + , readonly = False + , appendonly = False + , mkUnavailable = return Nothing + , getInfo = return [("repo", ddarRepoLocation ddarrepo)] + , claimUrl = Nothing + , checkUrl = Nothing + } + ddarrepo = maybe (giveup "missing ddarrepo") (DdarRepo gc) (remoteAnnexDdarRepo gc) + specialcfg = (specialRemoteCfg c) + -- chunking would not improve ddar + { chunkConfig = NoChunks + } + +ddarSetup :: SetupStage -> Maybe UUID -> Maybe CredPair -> RemoteConfig -> RemoteGitConfig -> Annex (RemoteConfig, UUID) +ddarSetup _ mu _ c gc = do + u <- maybe (liftIO genUUID) return mu + + -- verify configuration is sane + let ddarrepo = fromMaybe (giveup "Specify ddarrepo=") $ + M.lookup "ddarrepo" c + (c', _encsetup) <- encryptionSetup c gc + + -- The ddarrepo is stored in git config, as well as this repo's + -- persistant state, so it can vary between hosts. + gitConfigSpecialRemote u c' [("ddarrepo", ddarrepo)] + + return (c', u) + +store :: DdarRepo -> Storer +store ddarrepo = fileStorer $ \k src _p -> do + let params = + [ Param "c" + , Param "-N" + , Param $ key2file k + , Param $ ddarRepoLocation ddarrepo + , File src + ] + liftIO $ boolSystem "ddar" params + +{- Convert remote DdarRepo to host and path on remote end -} +splitRemoteDdarRepo :: DdarRepo -> (SshHost, String) +splitRemoteDdarRepo ddarrepo = (either error id $ mkSshHost host, ddarrepo') + where + (host, remainder) = span (/= ':') (ddarRepoLocation ddarrepo) + ddarrepo' = drop 1 remainder + +{- Return the command and parameters to use for a ddar call that may need to be + - made on a remote repository. This will call ssh if needed. -} +ddarRemoteCall :: ConsumeStdin -> DdarRepo -> Char -> [CommandParam] -> Annex (String, [CommandParam]) +ddarRemoteCall cs ddarrepo cmd params + | ddarLocal ddarrepo = return ("ddar", localParams) + | otherwise = sshCommand cs (host, Nothing) (ddarRepoConfig ddarrepo) remoteCommand + where + (host, ddarrepo') = splitRemoteDdarRepo ddarrepo + localParams = Param [cmd] : Param (ddarRepoLocation ddarrepo) : params + remoteCommand = unwords $ map shellEscape $ toCommand $ + [Param "ddar", Param [cmd], Param ddarrepo'] ++ params + +{- Specialized ddarRemoteCall that includes extraction command and flags -} +ddarExtractRemoteCall :: ConsumeStdin -> DdarRepo -> Key -> Annex (String, [CommandParam]) +ddarExtractRemoteCall cs ddarrepo k = + ddarRemoteCall cs ddarrepo 'x' [Param "--force-stdout", Param $ key2file k] + +retrieve :: DdarRepo -> Retriever +retrieve ddarrepo = byteRetriever $ \k sink -> do + (cmd, params) <- ddarExtractRemoteCall NoConsumeStdin ddarrepo k + let p = (proc cmd $ toCommand params) { std_out = CreatePipe } + (_, Just h, _, pid) <- liftIO $ createProcess p + liftIO (hClose h >> forceSuccessProcess p pid) + `after` (sink =<< liftIO (L.hGetContents h)) + +retrieveCheap :: Key -> AssociatedFile -> FilePath -> Annex Bool +retrieveCheap _ _ _ = return False + +remove :: DdarRepo -> Remover +remove ddarrepo key = do + (cmd, params) <- ddarRemoteCall NoConsumeStdin ddarrepo 'd' + [Param $ key2file key] + liftIO $ boolSystem cmd params + +ddarDirectoryExists :: DdarRepo -> Annex (Either String Bool) +ddarDirectoryExists ddarrepo + | ddarLocal ddarrepo = do + maybeStatus <- liftIO $ tryJust (guard . isDoesNotExistError) $ getFileStatus $ ddarRepoLocation ddarrepo + return $ case maybeStatus of + Left _ -> Right False + Right status -> Right $ isDirectory status + | otherwise = do + let remotecmd = unwords $ map shellEscape + [ "test", "-d", ddarrepo' ] + (sshcmd, sshps) <- sshCommand NoConsumeStdin (host, Nothing) + (ddarRepoConfig ddarrepo) remotecmd + exitCode <- liftIO $ safeSystem sshcmd sshps + case exitCode of + ExitSuccess -> return $ Right True + ExitFailure 1 -> return $ Right False + ExitFailure code -> return $ Left $ "ssh " ++ + show (unwords $ toCommand sshps) ++ + " failed with status " ++ show code + where + (host, ddarrepo') = splitRemoteDdarRepo ddarrepo + +{- Use "ddar t" to determine if a given key is present in a ddar archive -} +inDdarManifest :: DdarRepo -> Key -> Annex (Either String Bool) +inDdarManifest ddarrepo k = do + (cmd, params) <- ddarRemoteCall NoConsumeStdin ddarrepo 't' [] + let p = proc cmd $ toCommand params + liftIO $ catchMsgIO $ withHandle StdoutHandle createProcessSuccess p $ \h -> do + contents <- hGetContents h + return $ elem k' $ lines contents + where + k' = key2file k + +checkKey :: DdarRepo -> CheckPresent +checkKey ddarrepo key = do + directoryExists <- ddarDirectoryExists ddarrepo + case directoryExists of + Left e -> error e + Right True -> either error return + =<< inDdarManifest ddarrepo key + Right False -> return False + +ddarLocal :: DdarRepo -> Bool +ddarLocal = notElem ':' . ddarRepoLocation diff --git a/Remote/Directory.hs b/Remote/Directory.hs new file mode 100644 index 0000000000..c8e41ea166 --- /dev/null +++ b/Remote/Directory.hs @@ -0,0 +1,290 @@ +{- A "remote" that is just a filesystem directory. + - + - Copyright 2011-2017 Joey Hess + - + - Licensed under the GNU GPL version 3 or higher. + -} + +{-# LANGUAGE CPP #-} + +module Remote.Directory ( + remote, + finalizeStoreGeneric, + removeDirGeneric, +) where + +import qualified Data.ByteString.Lazy as L +import qualified Data.Map as M +import Data.Default + +import Annex.Common +import Types.Remote +import Types.Export +import Types.Creds +import qualified Git +import Config.Cost +import Config +import Utility.FileMode +import Remote.Helper.Special +import Remote.Helper.Export +import qualified Remote.Directory.LegacyChunked as Legacy +import Annex.Content +import Annex.UUID +import Utility.Metered +import Utility.Tmp + +remote :: RemoteType +remote = RemoteType + { typename = "directory" + , enumerate = const (findSpecialRemotes "directory") + , generate = gen + , setup = directorySetup + , exportSupported = exportIsSupported + } + +gen :: Git.Repo -> UUID -> RemoteConfig -> RemoteGitConfig -> Annex (Maybe Remote) +gen r u c gc = do + cst <- remoteCost gc cheapRemoteCost + let chunkconfig = getChunkConfig c + return $ Just $ specialRemote c + (prepareStore dir chunkconfig) + (retrieveKeyFileM dir chunkconfig) + (simplyPrepare $ removeKeyM dir) + (simplyPrepare $ checkPresentM dir chunkconfig) + Remote + { uuid = u + , cost = cst + , name = Git.repoDescribe r + , storeKey = storeKeyDummy + , retrieveKeyFile = retreiveKeyFileDummy + , retrieveKeyFileCheap = retrieveKeyFileCheapM dir chunkconfig + , retrievalSecurityPolicy = RetrievalAllKeysSecure + , removeKey = removeKeyDummy + , lockContent = Nothing + , checkPresent = checkPresentDummy + , checkPresentCheap = True + , exportActions = return $ ExportActions + { storeExport = storeExportM dir + , retrieveExport = retrieveExportM dir + , removeExport = removeExportM dir + , checkPresentExport = checkPresentExportM dir + -- Not needed because removeExportLocation + -- auto-removes empty directories. + , removeExportDirectory = Nothing + , renameExport = renameExportM dir + } + , whereisKey = Nothing + , remoteFsck = Nothing + , repairRepo = Nothing + , config = c + , getRepo = return r + , gitconfig = gc + , localpath = Just dir + , readonly = False + , appendonly = False + , availability = LocallyAvailable + , remotetype = remote + , mkUnavailable = gen r u c $ + gc { remoteAnnexDirectory = Just "/dev/null" } + , getInfo = return [("directory", dir)] + , claimUrl = Nothing + , checkUrl = Nothing + } + where + dir = fromMaybe (giveup "missing directory") $ remoteAnnexDirectory gc + +directorySetup :: SetupStage -> Maybe UUID -> Maybe CredPair -> RemoteConfig -> RemoteGitConfig -> Annex (RemoteConfig, UUID) +directorySetup _ mu _ c gc = do + u <- maybe (liftIO genUUID) return mu + -- verify configuration is sane + let dir = fromMaybe (giveup "Specify directory=") $ + M.lookup "directory" c + absdir <- liftIO $ absPath dir + liftIO $ unlessM (doesDirectoryExist absdir) $ + giveup $ "Directory does not exist: " ++ absdir + (c', _encsetup) <- encryptionSetup c gc + + -- The directory is stored in git config, not in this remote's + -- persistant state, so it can vary between hosts. + gitConfigSpecialRemote u c' [("directory", absdir)] + return (M.delete "directory" c', u) + +{- Locations to try to access a given Key in the directory. + - We try more than one since we used to write to different hash + - directories. -} +locations :: FilePath -> Key -> [FilePath] +locations d k = map (d ) (keyPaths k) + +{- Returns the location off a Key in the directory. If the key is + - present, returns the location that is actually used, otherwise + - returns the first, default location. -} +getLocation :: FilePath -> Key -> IO FilePath +getLocation d k = do + let locs = locations d k + fromMaybe (Prelude.head locs) <$> firstM doesFileExist locs + +{- Directory where the file(s) for a key are stored. -} +storeDir :: FilePath -> Key -> FilePath +storeDir d k = addTrailingPathSeparator $ d hashDirLower def k keyFile k + +{- Check if there is enough free disk space in the remote's directory to + - store the key. Note that the unencrypted key size is checked. -} +prepareStore :: FilePath -> ChunkConfig -> Preparer Storer +prepareStore d chunkconfig = checkPrepare (checkDiskSpaceDirectory d) + (byteStorer $ store d chunkconfig) + where + +checkDiskSpaceDirectory :: FilePath -> Key -> Annex Bool +checkDiskSpaceDirectory d k = do + annexdir <- fromRepo gitAnnexObjectDir + samefilesystem <- liftIO $ catchDefaultIO False $ + (\a b -> deviceID a == deviceID b) + <$> getFileStatus d + <*> getFileStatus annexdir + checkDiskSpace (Just d) k 0 samefilesystem + +store :: FilePath -> ChunkConfig -> Key -> L.ByteString -> MeterUpdate -> Annex Bool +store d chunkconfig k b p = liftIO $ do + void $ tryIO $ createDirectoryIfMissing True tmpdir + case chunkconfig of + LegacyChunks chunksize -> Legacy.store chunksize finalizeStoreGeneric k b p tmpdir destdir + _ -> do + let tmpf = tmpdir keyFile k + meteredWriteFile p tmpf b + finalizeStoreGeneric tmpdir destdir + return True + where + tmpdir = addTrailingPathSeparator $ d "tmp" keyFile k + destdir = storeDir d k + +{- Passed a temp directory that contains the files that should be placed + - in the dest directory, moves it into place. Anything already existing + - in the dest directory will be deleted. File permissions will be locked + - down. -} +finalizeStoreGeneric :: FilePath -> FilePath -> IO () +finalizeStoreGeneric tmp dest = do + void $ tryIO $ allowWrite dest -- may already exist + void $ tryIO $ removeDirectoryRecursive dest -- or not exist + createDirectoryIfMissing True (parentDir dest) + renameDirectory tmp dest + -- may fail on some filesystems + void $ tryIO $ do + mapM_ preventWrite =<< dirContents dest + preventWrite dest + +retrieveKeyFileM :: FilePath -> ChunkConfig -> Preparer Retriever +retrieveKeyFileM d (LegacyChunks _) = Legacy.retrieve locations d +retrieveKeyFileM d _ = simplyPrepare $ byteRetriever $ \k sink -> + sink =<< liftIO (L.readFile =<< getLocation d k) + +retrieveKeyFileCheapM :: FilePath -> ChunkConfig -> Key -> AssociatedFile -> FilePath -> Annex Bool +-- no cheap retrieval possible for chunks +retrieveKeyFileCheapM _ (UnpaddedChunks _) _ _ _ = return False +retrieveKeyFileCheapM _ (LegacyChunks _) _ _ _ = return False +#ifndef mingw32_HOST_OS +retrieveKeyFileCheapM d NoChunks k _af f = liftIO $ catchBoolIO $ do + file <- absPath =<< getLocation d k + ifM (doesFileExist file) + ( do + createSymbolicLink file f + return True + , return False + ) +#else +retrieveKeyFileCheapM _ _ _ _ _ = return False +#endif + +removeKeyM :: FilePath -> Remover +removeKeyM d k = liftIO $ removeDirGeneric d (storeDir d k) + +{- Removes the directory, which must be located under the topdir. + - + - Succeeds even on directories and contents that do not have write + - permission. + - + - If the directory does not exist, succeeds as long as the topdir does + - exist. If the topdir does not exist, fails, because in this case the + - remote is not currently accessible and probably still has the content + - we were supposed to remove from it. + -} +removeDirGeneric :: FilePath -> FilePath -> IO Bool +removeDirGeneric topdir dir = do + void $ tryIO $ allowWrite dir +#ifdef mingw32_HOST_OS + {- Windows needs the files inside the directory to be writable + - before it can delete them. -} + void $ tryIO $ mapM_ allowWrite =<< dirContents dir +#endif + ok <- catchBoolIO $ do + removeDirectoryRecursive dir + return True + if ok + then return ok + else doesDirectoryExist topdir <&&> (not <$> doesDirectoryExist dir) + +checkPresentM :: FilePath -> ChunkConfig -> CheckPresent +checkPresentM d (LegacyChunks _) k = Legacy.checkKey d locations k +checkPresentM d _ k = checkPresentGeneric d (locations d k) + +checkPresentGeneric :: FilePath -> [FilePath] -> Annex Bool +checkPresentGeneric d ps = liftIO $ + ifM (anyM doesFileExist ps) + ( return True + , ifM (doesDirectoryExist d) + ( return False + , giveup $ "directory " ++ d ++ " is not accessible" + ) + ) + +storeExportM :: FilePath -> FilePath -> Key -> ExportLocation -> MeterUpdate -> Annex Bool +storeExportM d src _k loc p = liftIO $ catchBoolIO $ do + createDirectoryIfMissing True (takeDirectory dest) + -- Write via temp file so that checkPresentGeneric will not + -- see it until it's fully stored. + viaTmp (\tmp () -> withMeteredFile src p (L.writeFile tmp)) dest () + return True + where + dest = exportPath d loc + +retrieveExportM :: FilePath -> Key -> ExportLocation -> FilePath -> MeterUpdate -> Annex Bool +retrieveExportM d _k loc dest p = liftIO $ catchBoolIO $ do + withMeteredFile src p (L.writeFile dest) + return True + where + src = exportPath d loc + +removeExportM :: FilePath -> Key -> ExportLocation -> Annex Bool +removeExportM d _k loc = liftIO $ do + nukeFile src + removeExportLocation d loc + return True + where + src = exportPath d loc + +checkPresentExportM :: FilePath -> Key -> ExportLocation -> Annex Bool +checkPresentExportM d _k loc = + checkPresentGeneric d [exportPath d loc] + +renameExportM :: FilePath -> Key -> ExportLocation -> ExportLocation -> Annex Bool +renameExportM d _k oldloc newloc = liftIO $ catchBoolIO $ do + createDirectoryIfMissing True (takeDirectory dest) + renameFile src dest + removeExportLocation d oldloc + return True + where + src = exportPath d oldloc + dest = exportPath d newloc + +exportPath :: FilePath -> ExportLocation -> FilePath +exportPath d loc = d fromExportLocation loc + +{- Removes the ExportLocation's parent directory and its parents, so long as + - they're empty, up to but not including the topdir. -} +removeExportLocation :: FilePath -> ExportLocation -> IO () +removeExportLocation topdir loc = + go (Just $ takeDirectory $ fromExportLocation loc) (Right ()) + where + go _ (Left _e) = return () + go Nothing _ = return () + go (Just loc') _ = go (upFrom loc') + =<< tryIO (removeDirectory $ exportPath topdir (mkExportLocation loc')) diff --git a/Remote/Directory/LegacyChunked.hs b/Remote/Directory/LegacyChunked.hs new file mode 100644 index 0000000000..095b90b483 --- /dev/null +++ b/Remote/Directory/LegacyChunked.hs @@ -0,0 +1,109 @@ +{- Legacy chunksize support for directory special remote. + - + - Can be removed eventually. + - + - Copyright 2011-2012 Joey Hess + - + - Licensed under the GNU GPL version 3 or higher. + -} + +module Remote.Directory.LegacyChunked where + +import qualified Data.ByteString.Lazy as L +import qualified Data.ByteString as S + +import Annex.Common +import Utility.FileMode +import Remote.Helper.Special +import qualified Remote.Helper.Chunked.Legacy as Legacy +import Annex.Perms +import Utility.Metered + +withCheckedFiles :: (FilePath -> IO Bool) -> FilePath -> (FilePath -> Key -> [FilePath]) -> Key -> ([FilePath] -> IO Bool) -> IO Bool +withCheckedFiles _ [] _locations _ _ = return False +withCheckedFiles check d locations k a = go $ locations d k + where + go [] = return False + go (f:fs) = do + let chunkcount = f ++ Legacy.chunkCount + ifM (check chunkcount) + ( do + chunks <- Legacy.listChunks f <$> readFile chunkcount + ifM (allM check chunks) + ( a chunks , return False ) + , do + chunks <- Legacy.probeChunks f check + if null chunks + then go fs + else a chunks + ) +withStoredFiles :: FilePath -> (FilePath -> Key -> [FilePath]) -> Key -> ([FilePath] -> IO Bool) -> IO Bool +withStoredFiles = withCheckedFiles doesFileExist + +{- Splits a ByteString into chunks and writes to dests, obeying configured + - chunk size (not to be confused with the L.ByteString chunk size). -} +storeLegacyChunked :: MeterUpdate -> ChunkSize -> [FilePath] -> L.ByteString -> IO [FilePath] +storeLegacyChunked _ _ [] _ = error "bad storeLegacyChunked call" +storeLegacyChunked meterupdate chunksize alldests@(firstdest:_) b + | L.null b = do + -- always write at least one file, even for empty + L.writeFile firstdest b + return [firstdest] + | otherwise = storeLegacyChunked' meterupdate chunksize alldests (L.toChunks b) [] +storeLegacyChunked' :: MeterUpdate -> ChunkSize -> [FilePath] -> [S.ByteString] -> [FilePath] -> IO [FilePath] +storeLegacyChunked' _ _ [] _ _ = error "ran out of dests" +storeLegacyChunked' _ _ _ [] c = return $ reverse c +storeLegacyChunked' meterupdate chunksize (d:dests) bs c = do + bs' <- withFile d WriteMode $ + feed zeroBytesProcessed chunksize bs + storeLegacyChunked' meterupdate chunksize dests bs' (d:c) + where + feed _ _ [] _ = return [] + feed bytes sz (l:ls) h = do + let len = S.length l + let s = fromIntegral len + if s <= sz || sz == chunksize + then do + S.hPut h l + let bytes' = addBytesProcessed bytes len + meterupdate bytes' + feed bytes' (sz - s) ls h + else return (l:ls) + +storeHelper :: (FilePath -> FilePath -> IO ()) -> Key -> ([FilePath] -> IO [FilePath]) -> FilePath -> FilePath -> IO Bool +storeHelper finalizer key storer tmpdir destdir = do + void $ liftIO $ tryIO $ createDirectoryIfMissing True tmpdir + Legacy.storeChunks key tmpdir destdir storer recorder finalizer + where + recorder f s = do + void $ tryIO $ allowWrite f + writeFile f s + void $ tryIO $ preventWrite f + +store :: ChunkSize -> (FilePath -> FilePath -> IO ()) -> Key -> L.ByteString -> MeterUpdate -> FilePath -> FilePath -> IO Bool +store chunksize finalizer k b p = storeHelper finalizer k $ \dests -> + storeLegacyChunked p chunksize dests b + +{- Need to get a single ByteString containing every chunk. + - Done very innefficiently, by writing to a temp file. + - :/ This is legacy code.. + -} +retrieve :: (FilePath -> Key -> [FilePath]) -> FilePath -> Preparer Retriever +retrieve locations d basek a = do + showLongNote "This remote uses the deprecated chunksize setting. So this will be quite slow." + tmpdir <- fromRepo $ gitAnnexTmpMiscDir + createAnnexDirectory tmpdir + let tmp = tmpdir keyFile basek ++ ".directorylegacy.tmp" + a $ Just $ byteRetriever $ \k sink -> do + liftIO $ void $ withStoredFiles d locations k $ \fs -> do + forM_ fs $ + S.appendFile tmp <=< S.readFile + return True + b <- liftIO $ L.readFile tmp + liftIO $ nukeFile tmp + sink b + +checkKey :: FilePath -> (FilePath -> Key -> [FilePath]) -> Key -> Annex Bool +checkKey d locations k = liftIO $ withStoredFiles d locations k $ + -- withStoredFiles checked that it exists + const $ return True diff --git a/Remote/External.hs b/Remote/External.hs new file mode 100644 index 0000000000..7e905dc1ba --- /dev/null +++ b/Remote/External.hs @@ -0,0 +1,738 @@ +{- External special remote interface. + - + - Copyright 2013-2018 Joey Hess + - + - Licensed under the GNU AGPL version 3 or higher. + -} + +module Remote.External (remote) where + +import Remote.External.Types +import qualified Annex +import Annex.Common +import Types.Remote +import Types.Export +import Types.CleanupActions +import Types.UrlContents +import qualified Git +import Config +import Git.Config (isTrue, boolConfig) +import Git.Env +import Remote.Helper.Special +import Remote.Helper.Export +import Annex.Export +import Remote.Helper.ReadOnly +import Remote.Helper.Messages +import Utility.Metered +import Utility.Shell +import Messages.Progress +import Types.Transfer +import Logs.PreferredContent.Raw +import Logs.RemoteState +import Logs.Web +import Config.Cost +import Annex.Content +import Annex.Url +import Annex.UUID +import Creds + +import Control.Concurrent.STM +import Control.Concurrent.Async +import System.Log.Logger (debugM) +import qualified Data.Map as M + +remote :: RemoteType +remote = RemoteType + { typename = "external" + , enumerate = const (findSpecialRemotes "externaltype") + , generate = gen + , setup = externalSetup + , exportSupported = checkExportSupported + } + +gen :: Git.Repo -> UUID -> RemoteConfig -> RemoteGitConfig -> Annex (Maybe Remote) +gen r u c gc + -- readonly mode only downloads urls; does not use external program + | remoteAnnexReadOnly gc = do + cst <- remoteCost gc expensiveRemoteCost + mk cst GloballyAvailable + readonlyStorer + retrieveUrl + readonlyRemoveKey + (checkKeyUrl r) + Nothing + (externalInfo externaltype) + Nothing + Nothing + exportUnsupported + exportUnsupported + | otherwise = do + external <- newExternal externaltype u c gc + Annex.addCleanup (RemoteCleanup u) $ stopExternal external + cst <- getCost external r gc + avail <- getAvailability external r gc + exportsupported <- if exportTree c + then checkExportSupported' external + else return False + let exportactions = if exportsupported + then return $ ExportActions + { storeExport = storeExportM external + , retrieveExport = retrieveExportM external + , removeExport = removeExportM external + , checkPresentExport = checkPresentExportM external + , removeExportDirectory = Just $ removeExportDirectoryM external + , renameExport = renameExportM external + } + else exportUnsupported + -- Cheap exportSupported that replaces the expensive + -- checkExportSupported now that we've already checked it. + let cheapexportsupported = if exportsupported + then exportIsSupported + else exportUnsupported + mk cst avail + (storeKeyM external) + (retrieveKeyFileM external) + (removeKeyM external) + (checkPresentM external) + (Just (whereisKeyM external)) + (getInfoM external) + (Just (claimUrlM external)) + (Just (checkUrlM external)) + exportactions + cheapexportsupported + where + mk cst avail tostore toretrieve toremove tocheckkey towhereis togetinfo toclaimurl tocheckurl exportactions cheapexportsupported = do + let rmt = Remote + { uuid = u + , cost = cst + , name = Git.repoDescribe r + , storeKey = storeKeyDummy + , retrieveKeyFile = retreiveKeyFileDummy + , retrieveKeyFileCheap = \_ _ _ -> return False + -- External special remotes use many http libraries + -- and have no protection against redirects to + -- local private web servers, or in some cases + -- to file:// urls. + , retrievalSecurityPolicy = mkRetrievalVerifiableKeysSecure gc + , removeKey = removeKeyDummy + , lockContent = Nothing + , checkPresent = checkPresentDummy + , checkPresentCheap = False + , exportActions = exportactions + , whereisKey = towhereis + , remoteFsck = Nothing + , repairRepo = Nothing + , config = c + , localpath = Nothing + , getRepo = return r + , gitconfig = gc + , readonly = False + , appendonly = False + , availability = avail + , remotetype = remote + { exportSupported = cheapexportsupported } + , mkUnavailable = gen r u c $ + gc { remoteAnnexExternalType = Just "!dne!" } + , getInfo = togetinfo + , claimUrl = toclaimurl + , checkUrl = tocheckurl + } + return $ Just $ specialRemote c + (simplyPrepare tostore) + (simplyPrepare toretrieve) + (simplyPrepare toremove) + (simplyPrepare tocheckkey) + rmt + externaltype = fromMaybe (giveup "missing externaltype") (remoteAnnexExternalType gc) + +externalSetup :: SetupStage -> Maybe UUID -> Maybe CredPair -> RemoteConfig -> RemoteGitConfig -> Annex (RemoteConfig, UUID) +externalSetup _ mu _ c gc = do + u <- maybe (liftIO genUUID) return mu + let externaltype = fromMaybe (giveup "Specify externaltype=") $ + M.lookup "externaltype" c + (c', _encsetup) <- encryptionSetup c gc + + c'' <- case M.lookup "readonly" c of + Just v | isTrue v == Just True -> do + setConfig (remoteConfig (fromJust (M.lookup "name" c)) "readonly") (boolConfig True) + return c' + _ -> do + external <- newExternal externaltype u c' gc + handleRequest external INITREMOTE Nothing $ \resp -> case resp of + INITREMOTE_SUCCESS -> result () + INITREMOTE_FAILURE errmsg -> Just $ giveup errmsg + _ -> Nothing + withExternalState external $ + liftIO . atomically . readTVar . externalConfig + + gitConfigSpecialRemote u c'' [("externaltype", externaltype)] + return (c'', u) + +checkExportSupported :: RemoteConfig -> RemoteGitConfig -> Annex Bool +checkExportSupported c gc = do + let externaltype = fromMaybe (giveup "Specify externaltype=") $ + remoteAnnexExternalType gc <|> M.lookup "externaltype" c + checkExportSupported' + =<< newExternal externaltype NoUUID c gc + +checkExportSupported' :: External -> Annex Bool +checkExportSupported' external = go `catchNonAsync` (const (return False)) + where + go = handleRequest external EXPORTSUPPORTED Nothing $ \resp -> case resp of + EXPORTSUPPORTED_SUCCESS -> result True + EXPORTSUPPORTED_FAILURE -> result False + UNSUPPORTED_REQUEST -> result False + _ -> Nothing + +storeKeyM :: External -> Storer +storeKeyM external = fileStorer $ \k f p -> + handleRequestKey external (\sk -> TRANSFER Upload sk f) k (Just p) $ \resp -> + case resp of + TRANSFER_SUCCESS Upload k' | k == k' -> result True + TRANSFER_FAILURE Upload k' errmsg | k == k' -> + Just $ do + warning errmsg + return (Result False) + _ -> Nothing + +retrieveKeyFileM :: External -> Retriever +retrieveKeyFileM external = fileRetriever $ \d k p -> + handleRequestKey external (\sk -> TRANSFER Download sk d) k (Just p) $ \resp -> + case resp of + TRANSFER_SUCCESS Download k' + | k == k' -> result () + TRANSFER_FAILURE Download k' errmsg + | k == k' -> Just $ giveup errmsg + _ -> Nothing + +removeKeyM :: External -> Remover +removeKeyM external k = safely $ + handleRequestKey external REMOVE k Nothing $ \resp -> + case resp of + REMOVE_SUCCESS k' + | k == k' -> result True + REMOVE_FAILURE k' errmsg + | k == k' -> Just $ do + warning errmsg + return (Result False) + _ -> Nothing + +checkPresentM :: External -> CheckPresent +checkPresentM external k = either giveup id <$> go + where + go = handleRequestKey external CHECKPRESENT k Nothing $ \resp -> + case resp of + CHECKPRESENT_SUCCESS k' + | k' == k -> result $ Right True + CHECKPRESENT_FAILURE k' + | k' == k -> result $ Right False + CHECKPRESENT_UNKNOWN k' errmsg + | k' == k -> result $ Left errmsg + _ -> Nothing + +whereisKeyM :: External -> Key -> Annex [String] +whereisKeyM external k = handleRequestKey external WHEREIS k Nothing $ \resp -> case resp of + WHEREIS_SUCCESS s -> result [s] + WHEREIS_FAILURE -> result [] + UNSUPPORTED_REQUEST -> result [] + _ -> Nothing + +storeExportM :: External -> FilePath -> Key -> ExportLocation -> MeterUpdate -> Annex Bool +storeExportM external f k loc p = safely $ + handleRequestExport external loc req k (Just p) $ \resp -> case resp of + TRANSFER_SUCCESS Upload k' | k == k' -> result True + TRANSFER_FAILURE Upload k' errmsg | k == k' -> + Just $ do + warning errmsg + return (Result False) + UNSUPPORTED_REQUEST -> Just $ do + warning "TRANSFEREXPORT not implemented by external special remote" + return (Result False) + _ -> Nothing + where + req sk = TRANSFEREXPORT Upload sk f + +retrieveExportM :: External -> Key -> ExportLocation -> FilePath -> MeterUpdate -> Annex Bool +retrieveExportM external k loc d p = safely $ + handleRequestExport external loc req k (Just p) $ \resp -> case resp of + TRANSFER_SUCCESS Download k' + | k == k' -> result True + TRANSFER_FAILURE Download k' errmsg + | k == k' -> Just $ do + warning errmsg + return (Result False) + UNSUPPORTED_REQUEST -> Just $ do + warning "TRANSFEREXPORT not implemented by external special remote" + return (Result False) + _ -> Nothing + where + req sk = TRANSFEREXPORT Download sk d + +checkPresentExportM :: External -> Key -> ExportLocation -> Annex Bool +checkPresentExportM external k loc = either giveup id <$> go + where + go = handleRequestExport external loc CHECKPRESENTEXPORT k Nothing $ \resp -> case resp of + CHECKPRESENT_SUCCESS k' + | k' == k -> result $ Right True + CHECKPRESENT_FAILURE k' + | k' == k -> result $ Right False + CHECKPRESENT_UNKNOWN k' errmsg + | k' == k -> result $ Left errmsg + UNSUPPORTED_REQUEST -> result $ + Left "CHECKPRESENTEXPORT not implemented by external special remote" + _ -> Nothing + +removeExportM :: External -> Key -> ExportLocation -> Annex Bool +removeExportM external k loc = safely $ + handleRequestExport external loc REMOVEEXPORT k Nothing $ \resp -> case resp of + REMOVE_SUCCESS k' + | k == k' -> result True + REMOVE_FAILURE k' errmsg + | k == k' -> Just $ do + warning errmsg + return (Result False) + UNSUPPORTED_REQUEST -> Just $ do + warning "REMOVEEXPORT not implemented by external special remote" + return (Result False) + _ -> Nothing + +removeExportDirectoryM :: External -> ExportDirectory -> Annex Bool +removeExportDirectoryM external dir = safely $ + handleRequest external req Nothing $ \resp -> case resp of + REMOVEEXPORTDIRECTORY_SUCCESS -> result True + REMOVEEXPORTDIRECTORY_FAILURE -> result False + UNSUPPORTED_REQUEST -> result True + _ -> Nothing + where + req = REMOVEEXPORTDIRECTORY dir + +renameExportM :: External -> Key -> ExportLocation -> ExportLocation -> Annex Bool +renameExportM external k src dest = safely $ + handleRequestExport external src req k Nothing $ \resp -> case resp of + RENAMEEXPORT_SUCCESS k' + | k' == k -> result True + RENAMEEXPORT_FAILURE k' + | k' == k -> result False + UNSUPPORTED_REQUEST -> result False + _ -> Nothing + where + req sk = RENAMEEXPORT sk dest + +safely :: Annex Bool -> Annex Bool +safely a = go =<< tryNonAsync a + where + go (Right r) = return r + go (Left e) = do + toplevelWarning False (show e) + return False + +{- Sends a Request to the external remote, and waits for it to generate + - a Response. That is fed into the responsehandler, which should return + - the action to run for it (or Nothing if there's a protocol error). + - + - While the external remote is processing the Request, it may send + - any number of RemoteRequests, that are handled here. + - + - An external remote process can only handle one request at a time. + - Concurrent requests will start up additional processes. + - + - May throw exceptions, for example on protocol errors, or + - when the repository cannot be used. + -} +handleRequest :: External -> Request -> Maybe MeterUpdate -> ResponseHandler a -> Annex a +handleRequest external req mp responsehandler = + withExternalState external $ \st -> + handleRequest' st external req mp responsehandler + +handleRequestKey :: External -> (SafeKey -> Request) -> Key -> Maybe MeterUpdate -> ResponseHandler a -> Annex a +handleRequestKey external mkreq k mp responsehandler = case mkSafeKey k of + Right sk -> handleRequest external (mkreq sk) mp responsehandler + Left e -> giveup e + +{- Export location is first sent in an EXPORT message before + - the main request. This is done because the ExportLocation can + - contain spaces etc. -} +handleRequestExport :: External -> ExportLocation -> (SafeKey -> Request) -> Key -> Maybe MeterUpdate -> ResponseHandler a -> Annex a +handleRequestExport external loc mkreq k mp responsehandler = do + withExternalState external $ \st -> do + checkPrepared st external + sendMessage st external (EXPORT loc) + handleRequestKey external mkreq k mp responsehandler + +handleRequest' :: ExternalState -> External -> Request -> Maybe MeterUpdate -> ResponseHandler a -> Annex a +handleRequest' st external req mp responsehandler + | needsPREPARE req = do + checkPrepared st external + go + | otherwise = go + where + go = do + sendMessage st external req + loop + loop = receiveMessage st external responsehandler + (\rreq -> Just $ handleRemoteRequest rreq >> loop) + (\msg -> Just $ handleAsyncMessage msg >> loop) + + handleRemoteRequest (PROGRESS bytesprocessed) = + maybe noop (\a -> liftIO $ a bytesprocessed) mp + handleRemoteRequest (DIRHASH k) = + send $ VALUE $ hashDirMixed def k + handleRemoteRequest (DIRHASH_LOWER k) = + send $ VALUE $ hashDirLower def k + handleRemoteRequest (SETCONFIG setting value) = + liftIO $ atomically $ modifyTVar' (externalConfig st) $ + M.insert setting value + handleRemoteRequest (GETCONFIG setting) = do + value <- fromMaybe "" . M.lookup setting + <$> liftIO (atomically $ readTVar $ externalConfig st) + send $ VALUE value + handleRemoteRequest (SETCREDS setting login password) = do + let v = externalConfig st + c <- liftIO $ atomically $ readTVar v + let gc = externalGitConfig external + c' <- setRemoteCredPair encryptionAlreadySetup c gc + (credstorage setting) + (Just (login, password)) + void $ liftIO $ atomically $ swapTVar v c' + handleRemoteRequest (GETCREDS setting) = do + c <- liftIO $ atomically $ readTVar $ externalConfig st + let gc = externalGitConfig external + creds <- fromMaybe ("", "") <$> + getRemoteCredPair c gc (credstorage setting) + send $ CREDS (fst creds) (snd creds) + handleRemoteRequest GETUUID = send $ + VALUE $ fromUUID $ externalUUID external + handleRemoteRequest GETGITDIR = send . VALUE =<< fromRepo Git.localGitDir + handleRemoteRequest (SETWANTED expr) = + preferredContentSet (externalUUID external) expr + handleRemoteRequest GETWANTED = do + expr <- fromMaybe "" . M.lookup (externalUUID external) + <$> preferredContentMapRaw + send $ VALUE expr + handleRemoteRequest (SETSTATE key state) = + setRemoteState (externalUUID external) key state + handleRemoteRequest (GETSTATE key) = do + state <- fromMaybe "" + <$> getRemoteState (externalUUID external) key + send $ VALUE state + handleRemoteRequest (SETURLPRESENT key url) = + setUrlPresent (externalUUID external) key url + handleRemoteRequest (SETURLMISSING key url) = + setUrlMissing (externalUUID external) key url + handleRemoteRequest (SETURIPRESENT key uri) = + withurl (SETURLPRESENT key) uri + handleRemoteRequest (SETURIMISSING key uri) = + withurl (SETURLMISSING key) uri + handleRemoteRequest (GETURLS key prefix) = do + mapM_ (send . VALUE) =<< getUrlsWithPrefix key prefix + send (VALUE "") -- end of list + handleRemoteRequest (DEBUG msg) = liftIO $ debugM "external" msg + handleRemoteRequest (INFO msg) = showInfo msg + handleRemoteRequest (VERSION _) = + sendMessage st external (ERROR "too late to send VERSION") + + handleAsyncMessage (ERROR err) = giveup $ "external special remote error: " ++ err + + send = sendMessage st external + + credstorage setting = CredPairStorage + { credPairFile = base + , credPairEnvironment = (base ++ "login", base ++ "password") + , credPairRemoteKey = Just setting + } + where + base = replace "/" "_" $ fromUUID (externalUUID external) ++ "-" ++ setting + + withurl mk uri = handleRemoteRequest $ mk $ + setDownloader (show uri) OtherDownloader + +sendMessage :: Sendable m => ExternalState -> External -> m -> Annex () +sendMessage st external m = liftIO $ do + protocolDebug external st True line + hPutStrLn h line + hFlush h + where + line = unwords $ formatMessage m + h = externalSend st + +{- A response handler can yeild a result, or it can request that another + - message be consumed from the external result. -} +data ResponseHandlerResult a + = Result a + | GetNextMessage (ResponseHandler a) + +type ResponseHandler a = Response -> Maybe (Annex (ResponseHandlerResult a)) + +result :: a -> Maybe (Annex (ResponseHandlerResult a)) +result = Just . return . Result + +{- Waits for a message from the external remote, and passes it to the + - apppropriate handler. + - + - If the handler returns Nothing, this is a protocol error.-} +receiveMessage + :: ExternalState + -> External + -> ResponseHandler a + -> (RemoteRequest -> Maybe (Annex a)) + -> (AsyncMessage -> Maybe (Annex a)) + -> Annex a +receiveMessage st external handleresponse handlerequest handleasync = + go =<< liftIO (catchMaybeIO $ hGetLine $ externalReceive st) + where + go Nothing = protocolError False "" + go (Just s) = do + liftIO $ protocolDebug external st False s + case parseMessage s :: Maybe Response of + Just resp -> case handleresponse resp of + Nothing -> protocolError True s + Just callback -> callback >>= \case + Result a -> return a + GetNextMessage handleresponse' -> + receiveMessage st external handleresponse' handlerequest handleasync + Nothing -> case parseMessage s :: Maybe RemoteRequest of + Just req -> maybe (protocolError True s) id (handlerequest req) + Nothing -> case parseMessage s :: Maybe AsyncMessage of + Just msg -> maybe (protocolError True s) id (handleasync msg) + Nothing -> protocolError False s + protocolError parsed s = giveup $ "external special remote protocol error, unexpectedly received \"" ++ s ++ "\" " ++ + if parsed + then "(command not allowed at this time)" + else "(unable to parse command)" + +protocolDebug :: External -> ExternalState -> Bool -> String -> IO () +protocolDebug external st sendto line = debugM "external" $ unwords + [ externalRemoteProgram (externalType external) ++ + "[" ++ show (externalPid st) ++ "]" + , if sendto then "<--" else "-->" + , line + ] + +{- While the action is running, the ExternalState provided to it will not + - be available to any other calls. + - + - Starts up a new process if no ExternalStates are available. -} +withExternalState :: External -> (ExternalState -> Annex a) -> Annex a +withExternalState external = bracket alloc dealloc + where + v = externalState external + + alloc = do + ms <- liftIO $ atomically $ do + l <- readTVar v + case l of + [] -> return Nothing + (st:rest) -> do + writeTVar v rest + return (Just st) + maybe (startExternal external) return ms + + dealloc st = liftIO $ atomically $ modifyTVar' v (st:) + +{- Starts an external remote process running, and checks VERSION and + - exchanges EXTENSIONS. -} +startExternal :: External -> Annex ExternalState +startExternal external = do + errrelayer <- mkStderrRelayer + st <- start errrelayer =<< Annex.gitRepo + receiveMessage st external + (const Nothing) + (checkVersion st external) + (const Nothing) + sendMessage st external (EXTENSIONS supportedExtensionList) + -- It responds with a EXTENSIONS_RESPONSE; that extensions list + -- is reserved for future expansion. UNSUPPORTED_REQUEST is also + -- accepted. + receiveMessage st external + (\resp -> case resp of + EXTENSIONS_RESPONSE _ -> result () + UNSUPPORTED_REQUEST -> result () + _ -> Nothing + ) + (const Nothing) + (const Nothing) + return st + where + start errrelayer g = liftIO $ do + cmdpath <- searchPath basecmd + (cmd, ps) <- maybe (pure (basecmd, [])) findShellCommand cmdpath + let basep = (proc cmd (toCommand ps)) + { std_in = CreatePipe + , std_out = CreatePipe + , std_err = CreatePipe + } + p <- propgit g basep + (Just hin, Just hout, Just herr, ph) <- + createProcess p `catchIO` runerr cmdpath + stderrelay <- async $ errrelayer herr + cv <- newTVarIO $ externalDefaultConfig external + pv <- newTVarIO Unprepared + pid <- atomically $ do + n <- succ <$> readTVar (externalLastPid external) + writeTVar (externalLastPid external) n + return n + return $ ExternalState + { externalSend = hin + , externalReceive = hout + , externalPid = pid + , externalShutdown = do + cancel stderrelay + void $ waitForProcess ph + , externalPrepared = pv + , externalConfig = cv + } + + basecmd = externalRemoteProgram $ externalType external + + propgit g p = do + environ <- propGitEnv g + return $ p { env = Just environ } + + runerr (Just cmd) _ = + giveup $ "Cannot run " ++ cmd ++ " -- Make sure it's executable and that its dependencies are installed." + runerr Nothing _ = do + path <- intercalate ":" <$> getSearchPath + giveup $ "Cannot run " ++ basecmd ++ " -- It is not installed in PATH (" ++ path ++ ")" + +stopExternal :: External -> Annex () +stopExternal external = liftIO $ do + l <- atomically $ swapTVar (externalState external) [] + mapM_ stop l + where + stop st = do + hClose $ externalSend st + hClose $ externalReceive st + externalShutdown st + +externalRemoteProgram :: ExternalType -> String +externalRemoteProgram externaltype = "git-annex-remote-" ++ externaltype + +checkVersion :: ExternalState -> External -> RemoteRequest -> Maybe (Annex ()) +checkVersion st external (VERSION v) = Just $ + if v `elem` supportedProtocolVersions + then noop + else sendMessage st external (ERROR "unsupported VERSION") +checkVersion _ _ _ = Nothing + +{- If repo has not been prepared, sends PREPARE. + - + - If the repo fails to prepare, or failed before, throws an exception with + - the error message. -} +checkPrepared :: ExternalState -> External -> Annex () +checkPrepared st external = do + v <- liftIO $ atomically $ readTVar $ externalPrepared st + case v of + Prepared -> noop + FailedPrepare errmsg -> giveup errmsg + Unprepared -> + handleRequest' st external PREPARE Nothing $ \resp -> + case resp of + PREPARE_SUCCESS -> Just $ do + setprepared Prepared + return (Result ()) + PREPARE_FAILURE errmsg -> Just $ do + setprepared $ FailedPrepare errmsg + giveup errmsg + _ -> Nothing + where + setprepared status = liftIO $ atomically $ void $ + swapTVar (externalPrepared st) status + +{- Caches the cost in the git config to avoid needing to start up an + - external special remote every time time just to ask it what its + - cost is. -} +getCost :: External -> Git.Repo -> RemoteGitConfig -> Annex Cost +getCost external r gc = + (go =<< remoteCost' gc) `catchNonAsync` const (pure defcst) + where + go (Just c) = return c + go Nothing = do + c <- handleRequest external GETCOST Nothing $ \req -> case req of + COST c -> result c + UNSUPPORTED_REQUEST -> result defcst + _ -> Nothing + setRemoteCost r c + return c + defcst = expensiveRemoteCost + +{- Caches the availability in the git config to avoid needing to start up an + - external special remote every time time just to ask it what its + - availability is. + - + - Most remotes do not bother to implement a reply to this request; + - globally available is the default. + -} +getAvailability :: External -> Git.Repo -> RemoteGitConfig -> Annex Availability +getAvailability external r gc = + maybe (catchNonAsync query (const (pure defavail))) return + (remoteAnnexAvailability gc) + where + query = do + avail <- handleRequest external GETAVAILABILITY Nothing $ \req -> case req of + AVAILABILITY avail -> result avail + UNSUPPORTED_REQUEST -> result defavail + _ -> Nothing + setRemoteAvailability r avail + return avail + defavail = GloballyAvailable + +claimUrlM :: External -> URLString -> Annex Bool +claimUrlM external url = + handleRequest external (CLAIMURL url) Nothing $ \req -> case req of + CLAIMURL_SUCCESS -> result True + CLAIMURL_FAILURE -> result False + UNSUPPORTED_REQUEST -> result False + _ -> Nothing + +checkUrlM :: External -> URLString -> Annex UrlContents +checkUrlM external url = + handleRequest external (CHECKURL url) Nothing $ \req -> case req of + CHECKURL_CONTENTS sz f -> result $ UrlContents sz $ + if null f then Nothing else Just $ mkSafeFilePath f + -- Treat a single item multi response specially to + -- simplify the external remote implementation. + CHECKURL_MULTI ((_, sz, f):[]) -> + result $ UrlContents sz $ Just $ mkSafeFilePath f + CHECKURL_MULTI l -> result $ UrlMulti $ map mkmulti l + CHECKURL_FAILURE errmsg -> Just $ giveup errmsg + UNSUPPORTED_REQUEST -> giveup "CHECKURL not implemented by external special remote" + _ -> Nothing + where + mkmulti (u, s, f) = (u, s, mkSafeFilePath f) + +retrieveUrl :: Retriever +retrieveUrl = fileRetriever $ \f k p -> do + us <- getWebUrls k + unlessM (downloadUrl k p us f) $ + giveup "failed to download content" + +checkKeyUrl :: Git.Repo -> CheckPresent +checkKeyUrl r k = do + showChecking r + us <- getWebUrls k + anyM (\u -> withUrlOptions $ liftIO . checkBoth u (keySize k)) us + +getWebUrls :: Key -> Annex [URLString] +getWebUrls key = filter supported <$> getUrls key + where + supported u = snd (getDownloader u) == WebDownloader + +externalInfo :: ExternalType -> Annex [(String, String)] +externalInfo et = return [("externaltype", et)] + +getInfoM :: External -> Annex [(String, String)] +getInfoM external = (++) + <$> externalInfo (externalType external) + <*> handleRequest external GETINFO Nothing (collect []) + where + collect l req = case req of + INFOFIELD f -> Just $ return $ + GetNextMessage $ collectvalue l f + INFOEND -> result (reverse l) + UNSUPPORTED_REQUEST -> result [] + _ -> Nothing + + collectvalue l f req = case req of + INFOVALUE v -> Just $ return $ + GetNextMessage $ collect ((f, v) : l) + _ -> Nothing diff --git a/Remote/External/Types.hs b/Remote/External/Types.hs new file mode 100644 index 0000000000..11c314e3f3 --- /dev/null +++ b/Remote/External/Types.hs @@ -0,0 +1,394 @@ +{- External special remote data types. + - + - Copyright 2013-2018 Joey Hess + - + - Licensed under the GNU AGPL version 3 or higher. + -} + +{-# LANGUAGE FlexibleInstances, TypeSynonymInstances #-} +{-# OPTIONS_GHC -fno-warn-orphans #-} + +module Remote.External.Types ( + External(..), + newExternal, + ExternalType, + ExternalState(..), + PrepareStatus(..), + supportedExtensionList, + Proto.parseMessage, + Proto.Sendable(..), + Proto.Receivable(..), + Request(..), + SafeKey, + mkSafeKey, + needsPREPARE, + Response(..), + RemoteRequest(..), + RemoteResponse(..), + AsyncMessage(..), + ErrorMsg, + Setting, + ProtocolVersion, + supportedProtocolVersions, +) where + +import Annex.Common +import Types.StandardGroups (PreferredContentExpression) +import Utility.Metered (BytesProcessed(..)) +import Types.Transfer (Direction(..)) +import Config.Cost (Cost) +import Types.Remote (RemoteConfig) +import Types.Export +import Types.Availability (Availability(..)) +import Types.Key +import Utility.Url (URLString) +import qualified Utility.SimpleProtocol as Proto + +import Control.Concurrent.STM +import Network.URI +import Data.Char + +data External = External + { externalType :: ExternalType + , externalUUID :: UUID + , externalState :: TVar [ExternalState] + -- ^ Contains states for external special remote processes + -- that are not currently in use. + , externalLastPid :: TVar PID + , externalDefaultConfig :: RemoteConfig + , externalGitConfig :: RemoteGitConfig + } + +newExternal :: ExternalType -> UUID -> RemoteConfig -> RemoteGitConfig -> Annex External +newExternal externaltype u c gc = liftIO $ External + <$> pure externaltype + <*> pure u + <*> atomically (newTVar []) + <*> atomically (newTVar 0) + <*> pure c + <*> pure gc + +type ExternalType = String + +data ExternalState = ExternalState + { externalSend :: Handle + , externalReceive :: Handle + , externalShutdown :: IO () + , externalPid :: PID + , externalPrepared :: TVar PrepareStatus + , externalConfig :: TVar RemoteConfig + } + +type PID = Int + +-- List of extensions to the protocol. +newtype ExtensionList = ExtensionList [String] + deriving (Show) + +-- When adding a new RemoteRequest, also add it to the list here. +supportedExtensionList :: ExtensionList +supportedExtensionList = ExtensionList ["INFO"] + +data PrepareStatus = Unprepared | Prepared | FailedPrepare ErrorMsg + +-- The protocol does not support keys with spaces in their names; +-- SafeKey can only be constructed for keys that are safe to use with the +-- protocol. +newtype SafeKey = SafeKey Key + deriving (Show) + +mkSafeKey :: Key -> Either String SafeKey +mkSafeKey k + | any isSpace (keyName k) = Left $ concat + [ "Sorry, this file cannot be stored on an external special remote because its key's name contains a space. " + , "To avoid this problem, you can run: git-annex migrate --backend=" + , formatKeyVariety (keyVariety k) + , " and pass it the name of the file" + ] + | otherwise = Right (SafeKey k) + +fromSafeKey :: SafeKey -> Key +fromSafeKey (SafeKey k) = k + +instance Proto.Serializable SafeKey where + serialize = Proto.serialize . fromSafeKey + deserialize = fmap SafeKey . Proto.deserialize + +-- Messages that can be sent to the external remote to request it do something. +data Request + = EXTENSIONS ExtensionList + | PREPARE + | INITREMOTE + | GETCOST + | GETAVAILABILITY + | CLAIMURL URLString + | CHECKURL URLString + | TRANSFER Direction SafeKey FilePath + | CHECKPRESENT SafeKey + | REMOVE SafeKey + | WHEREIS SafeKey + | GETINFO + | EXPORTSUPPORTED + | EXPORT ExportLocation + | TRANSFEREXPORT Direction SafeKey FilePath + | CHECKPRESENTEXPORT SafeKey + | REMOVEEXPORT SafeKey + | REMOVEEXPORTDIRECTORY ExportDirectory + | RENAMEEXPORT SafeKey ExportLocation + deriving (Show) + +-- Does PREPARE need to have been sent before this request? +needsPREPARE :: Request -> Bool +needsPREPARE PREPARE = False +needsPREPARE (EXTENSIONS _) = False +needsPREPARE INITREMOTE = False +needsPREPARE EXPORTSUPPORTED = False +needsPREPARE _ = True + +instance Proto.Sendable Request where + formatMessage (EXTENSIONS l) = ["EXTENSIONS", Proto.serialize l] + formatMessage PREPARE = ["PREPARE"] + formatMessage INITREMOTE = ["INITREMOTE"] + formatMessage GETCOST = ["GETCOST"] + formatMessage GETAVAILABILITY = ["GETAVAILABILITY"] + formatMessage (CLAIMURL url) = [ "CLAIMURL", Proto.serialize url ] + formatMessage (CHECKURL url) = [ "CHECKURL", Proto.serialize url ] + formatMessage (TRANSFER direction key file) = + [ "TRANSFER" + , Proto.serialize direction + , Proto.serialize key + , Proto.serialize file + ] + formatMessage (CHECKPRESENT key) = + [ "CHECKPRESENT", Proto.serialize key ] + formatMessage (REMOVE key) = [ "REMOVE", Proto.serialize key ] + formatMessage (WHEREIS key) = [ "WHEREIS", Proto.serialize key ] + formatMessage GETINFO = [ "GETINFO" ] + formatMessage EXPORTSUPPORTED = ["EXPORTSUPPORTED"] + formatMessage (EXPORT loc) = [ "EXPORT", Proto.serialize loc ] + formatMessage (TRANSFEREXPORT direction key file) = + [ "TRANSFEREXPORT" + , Proto.serialize direction + , Proto.serialize key + , Proto.serialize file + ] + formatMessage (CHECKPRESENTEXPORT key) = + [ "CHECKPRESENTEXPORT", Proto.serialize key ] + formatMessage (REMOVEEXPORT key) = + [ "REMOVEEXPORT", Proto.serialize key ] + formatMessage (REMOVEEXPORTDIRECTORY dir) = + [ "REMOVEEXPORTDIRECTORY", Proto.serialize dir ] + formatMessage (RENAMEEXPORT key newloc) = + [ "RENAMEEXPORT" + , Proto.serialize key + , Proto.serialize newloc + ] + +-- Responses the external remote can make to requests. +data Response + = EXTENSIONS_RESPONSE ExtensionList + | PREPARE_SUCCESS + | PREPARE_FAILURE ErrorMsg + | TRANSFER_SUCCESS Direction Key + | TRANSFER_FAILURE Direction Key ErrorMsg + | CHECKPRESENT_SUCCESS Key + | CHECKPRESENT_FAILURE Key + | CHECKPRESENT_UNKNOWN Key ErrorMsg + | REMOVE_SUCCESS Key + | REMOVE_FAILURE Key ErrorMsg + | COST Cost + | AVAILABILITY Availability + | INITREMOTE_SUCCESS + | INITREMOTE_FAILURE ErrorMsg + | CLAIMURL_SUCCESS + | CLAIMURL_FAILURE + | CHECKURL_CONTENTS Size FilePath + | CHECKURL_MULTI [(URLString, Size, FilePath)] + | CHECKURL_FAILURE ErrorMsg + | WHEREIS_SUCCESS String + | WHEREIS_FAILURE + | INFOFIELD String + | INFOVALUE String + | INFOEND + | EXPORTSUPPORTED_SUCCESS + | EXPORTSUPPORTED_FAILURE + | REMOVEEXPORTDIRECTORY_SUCCESS + | REMOVEEXPORTDIRECTORY_FAILURE + | RENAMEEXPORT_SUCCESS Key + | RENAMEEXPORT_FAILURE Key + | UNSUPPORTED_REQUEST + deriving (Show) + +instance Proto.Receivable Response where + parseCommand "EXTENSIONS" = Proto.parse1 EXTENSIONS_RESPONSE + parseCommand "PREPARE-SUCCESS" = Proto.parse0 PREPARE_SUCCESS + parseCommand "PREPARE-FAILURE" = Proto.parse1 PREPARE_FAILURE + parseCommand "TRANSFER-SUCCESS" = Proto.parse2 TRANSFER_SUCCESS + parseCommand "TRANSFER-FAILURE" = Proto.parse3 TRANSFER_FAILURE + parseCommand "CHECKPRESENT-SUCCESS" = Proto.parse1 CHECKPRESENT_SUCCESS + parseCommand "CHECKPRESENT-FAILURE" = Proto.parse1 CHECKPRESENT_FAILURE + parseCommand "CHECKPRESENT-UNKNOWN" = Proto.parse2 CHECKPRESENT_UNKNOWN + parseCommand "REMOVE-SUCCESS" = Proto.parse1 REMOVE_SUCCESS + parseCommand "REMOVE-FAILURE" = Proto.parse2 REMOVE_FAILURE + parseCommand "COST" = Proto.parse1 COST + parseCommand "AVAILABILITY" = Proto.parse1 AVAILABILITY + parseCommand "INITREMOTE-SUCCESS" = Proto.parse0 INITREMOTE_SUCCESS + parseCommand "INITREMOTE-FAILURE" = Proto.parse1 INITREMOTE_FAILURE + parseCommand "CLAIMURL-SUCCESS" = Proto.parse0 CLAIMURL_SUCCESS + parseCommand "CLAIMURL-FAILURE" = Proto.parse0 CLAIMURL_FAILURE + parseCommand "CHECKURL-CONTENTS" = Proto.parse2 CHECKURL_CONTENTS + parseCommand "CHECKURL-MULTI" = Proto.parse1 CHECKURL_MULTI + parseCommand "CHECKURL-FAILURE" = Proto.parse1 CHECKURL_FAILURE + parseCommand "WHEREIS-SUCCESS" = Just . WHEREIS_SUCCESS + parseCommand "WHEREIS-FAILURE" = Proto.parse0 WHEREIS_FAILURE + parseCommand "INFOFIELD" = Proto.parse1 INFOFIELD + parseCommand "INFOVALUE" = Proto.parse1 INFOVALUE + parseCommand "INFOEND" = Proto.parse0 INFOEND + parseCommand "EXPORTSUPPORTED-SUCCESS" = Proto.parse0 EXPORTSUPPORTED_SUCCESS + parseCommand "EXPORTSUPPORTED-FAILURE" = Proto.parse0 EXPORTSUPPORTED_FAILURE + parseCommand "REMOVEEXPORTDIRECTORY-SUCCESS" = Proto.parse0 REMOVEEXPORTDIRECTORY_SUCCESS + parseCommand "REMOVEEXPORTDIRECTORY-FAILURE" = Proto.parse0 REMOVEEXPORTDIRECTORY_FAILURE + parseCommand "RENAMEEXPORT-SUCCESS" = Proto.parse1 RENAMEEXPORT_SUCCESS + parseCommand "RENAMEEXPORT-FAILURE" = Proto.parse1 RENAMEEXPORT_FAILURE + parseCommand "UNSUPPORTED-REQUEST" = Proto.parse0 UNSUPPORTED_REQUEST + parseCommand _ = Proto.parseFail + +-- Requests that the external remote can send at any time it's in control. +data RemoteRequest + = VERSION ProtocolVersion + | PROGRESS BytesProcessed + | DIRHASH Key + | DIRHASH_LOWER Key + | SETCONFIG Setting String + | GETCONFIG Setting + | SETCREDS Setting String String + | GETCREDS Setting + | GETUUID + | GETGITDIR + | SETWANTED PreferredContentExpression + | GETWANTED + | SETSTATE Key String + | GETSTATE Key + | SETURLPRESENT Key URLString + | SETURLMISSING Key URLString + | SETURIPRESENT Key URI + | SETURIMISSING Key URI + | GETURLS Key String + | DEBUG String + | INFO String + deriving (Show) + +instance Proto.Receivable RemoteRequest where + parseCommand "VERSION" = Proto.parse1 VERSION + parseCommand "PROGRESS" = Proto.parse1 PROGRESS + parseCommand "DIRHASH" = Proto.parse1 DIRHASH + parseCommand "DIRHASH-LOWER" = Proto.parse1 DIRHASH_LOWER + parseCommand "SETCONFIG" = Proto.parse2 SETCONFIG + parseCommand "GETCONFIG" = Proto.parse1 GETCONFIG + parseCommand "SETCREDS" = Proto.parse3 SETCREDS + parseCommand "GETCREDS" = Proto.parse1 GETCREDS + parseCommand "GETUUID" = Proto.parse0 GETUUID + parseCommand "GETGITDIR" = Proto.parse0 GETGITDIR + parseCommand "SETWANTED" = Proto.parse1 SETWANTED + parseCommand "GETWANTED" = Proto.parse0 GETWANTED + parseCommand "SETSTATE" = Proto.parse2 SETSTATE + parseCommand "GETSTATE" = Proto.parse1 GETSTATE + parseCommand "SETURLPRESENT" = Proto.parse2 SETURLPRESENT + parseCommand "SETURLMISSING" = Proto.parse2 SETURLMISSING + parseCommand "SETURIPRESENT" = Proto.parse2 SETURIPRESENT + parseCommand "SETURIMISSING" = Proto.parse2 SETURIMISSING + parseCommand "GETURLS" = Proto.parse2 GETURLS + parseCommand "DEBUG" = Proto.parse1 DEBUG + parseCommand "INFO" = Proto.parse1 INFO + parseCommand _ = Proto.parseFail + +-- Responses to RemoteRequest. +data RemoteResponse + = VALUE String + | CREDS String String + deriving (Show) + +instance Proto.Sendable RemoteResponse where + formatMessage (VALUE s) = [ "VALUE", Proto.serialize s ] + formatMessage (CREDS login password) = [ "CREDS", Proto.serialize login, Proto.serialize password ] + +-- Messages that can be sent at any time by either git-annex or the remote. +data AsyncMessage + = ERROR ErrorMsg + deriving (Show) + +instance Proto.Sendable AsyncMessage where + formatMessage (ERROR err) = [ "ERROR", Proto.serialize err ] + +instance Proto.Receivable AsyncMessage where + parseCommand "ERROR" = Proto.parse1 ERROR + parseCommand _ = Proto.parseFail + +-- Data types used for parameters when communicating with the remote. +-- All are serializable. +type ErrorMsg = String +type Setting = String +type ProtocolVersion = Int +type Size = Maybe Integer + +supportedProtocolVersions :: [ProtocolVersion] +supportedProtocolVersions = [1] + +instance Proto.Serializable Direction where + serialize Upload = "STORE" + serialize Download = "RETRIEVE" + + deserialize "STORE" = Just Upload + deserialize "RETRIEVE" = Just Download + deserialize _ = Nothing + +instance Proto.Serializable ProtocolVersion where + serialize = show + deserialize = readish + +instance Proto.Serializable Cost where + serialize = show + deserialize = readish + +instance Proto.Serializable Size where + serialize (Just s) = show s + serialize Nothing = "UNKNOWN" + deserialize "UNKNOWN" = Just Nothing + deserialize s = maybe Nothing (Just . Just) (readish s) + +instance Proto.Serializable Availability where + serialize GloballyAvailable = "GLOBAL" + serialize LocallyAvailable = "LOCAL" + + deserialize "GLOBAL" = Just GloballyAvailable + deserialize "LOCAL" = Just LocallyAvailable + deserialize _ = Nothing + +instance Proto.Serializable BytesProcessed where + serialize (BytesProcessed n) = show n + deserialize = BytesProcessed <$$> readish + +instance Proto.Serializable [(URLString, Size, FilePath)] where + serialize = unwords . map go + where + go (url, sz, f) = url ++ " " ++ maybe "UNKNOWN" show sz ++ " " ++ f + deserialize = Just . go [] . words + where + go c (url:sz:f:rest) = go ((url, readish sz, f):c) rest + go c _ = reverse c + +instance Proto.Serializable URI where + serialize = show + deserialize = parseURI + +instance Proto.Serializable ExportLocation where + serialize = fromExportLocation + deserialize = Just . mkExportLocation + +instance Proto.Serializable ExportDirectory where + serialize = fromExportDirectory + deserialize = Just . mkExportDirectory + +instance Proto.Serializable ExtensionList where + serialize (ExtensionList l) = unwords l + deserialize = Just . ExtensionList . words diff --git a/Remote/GCrypt.hs b/Remote/GCrypt.hs new file mode 100644 index 0000000000..e1bc92b103 --- /dev/null +++ b/Remote/GCrypt.hs @@ -0,0 +1,463 @@ +{- git remotes encrypted using git-remote-gcrypt + - + - Copyright 2013 Joey Hess + - + - Licensed under the GNU GPL version 3 or higher. + -} + +module Remote.GCrypt ( + remote, + chainGen, + getGCryptUUID, + coreGCryptId, + setupRepo, + accessShellConfig, +) where + +import qualified Data.Map as M +import qualified Data.ByteString.Lazy as L +import Control.Exception +import Data.Default + +import Annex.Common +import qualified Annex +import Types.Remote +import Types.GitConfig +import Types.Crypto +import Types.Creds +import Types.Transfer +import qualified Git +import qualified Git.Command +import qualified Git.Config +import qualified Git.GCrypt +import qualified Git.Construct +import qualified Annex.Branch +import Config +import Config.Cost +import Remote.Helper.Git +import Remote.Helper.Encryptable +import Remote.Helper.Special +import Remote.Helper.Messages +import Remote.Helper.Export +import qualified Remote.Helper.Ssh as Ssh +import Utility.Metered +import Annex.UUID +import Annex.Ssh +import qualified Remote.Rsync +import qualified Remote.Directory +import Utility.Rsync +import Utility.Tmp +import Logs.Remote +import Utility.Gpg +import Utility.SshHost +import Messages.Progress + +remote :: RemoteType +remote = RemoteType + { typename = "gcrypt" + -- Remote.Git takes care of enumerating gcrypt remotes too, + -- and will call our gen on them. + , enumerate = const (return []) + , generate = gen + , setup = gCryptSetup + , exportSupported = exportUnsupported + } + +chainGen :: Git.Repo -> UUID -> RemoteConfig -> RemoteGitConfig -> Annex (Maybe Remote) +chainGen gcryptr u c gc = do + g <- gitRepo + -- get underlying git repo with real path, not gcrypt path + r <- liftIO $ Git.GCrypt.encryptedRemote g gcryptr + let r' = r { Git.remoteName = Git.remoteName gcryptr } + gen r' u c gc + +gen :: Git.Repo -> UUID -> RemoteConfig -> RemoteGitConfig -> Annex (Maybe Remote) +gen baser u c gc = do + -- doublecheck that cache matches underlying repo's gcrypt-id + -- (which might not be set), only for local repos + (mgcryptid, r) <- getGCryptId True baser gc + g <- gitRepo + case (mgcryptid, Git.GCrypt.remoteRepoId g (Git.remoteName baser)) of + (Just gcryptid, Just cachedgcryptid) + | gcryptid /= cachedgcryptid -> resetup gcryptid r + _ -> gen' r u c gc + where + -- A different drive may have been mounted, making a different + -- gcrypt remote available. So need to set the cached + -- gcrypt-id and annex-uuid of the remote to match the remote + -- that is now available. Also need to set the gcrypt particiants + -- correctly. + resetup gcryptid r = do + let u' = genUUIDInNameSpace gCryptNameSpace gcryptid + v <- M.lookup u' <$> readRemoteLog + case (Git.remoteName baser, v) of + (Just remotename, Just c') -> do + setGcryptEncryption c' remotename + setConfig (remoteConfig baser "uuid") (fromUUID u') + setConfig (ConfigKey $ Git.GCrypt.remoteConfigKey "gcrypt-id" remotename) gcryptid + gen' r u' c' gc + _ -> do + warning $ "not using unknown gcrypt repository pointed to by remote " ++ Git.repoDescribe r + return Nothing + +gen' :: Git.Repo -> UUID -> RemoteConfig -> RemoteGitConfig -> Annex (Maybe Remote) +gen' r u c gc = do + cst <- remoteCost gc $ + if repoCheap r then nearlyCheapRemoteCost else expensiveRemoteCost + (rsynctransport, rsyncurl) <- rsyncTransportToObjects r gc + let rsyncopts = Remote.Rsync.genRsyncOpts c gc rsynctransport rsyncurl + let this = Remote + { uuid = u + , cost = cst + , name = Git.repoDescribe r + , storeKey = storeKeyDummy + , retrieveKeyFile = retreiveKeyFileDummy + , retrieveKeyFileCheap = \_ _ _ -> return False + , retrievalSecurityPolicy = RetrievalAllKeysSecure + , removeKey = removeKeyDummy + , lockContent = Nothing + , checkPresent = checkPresentDummy + , checkPresentCheap = repoCheap r + , exportActions = exportUnsupported + , whereisKey = Nothing + , remoteFsck = Nothing + , repairRepo = Nothing + , config = c + , localpath = localpathCalc r + , getRepo = return r + , gitconfig = gc + , readonly = Git.repoIsHttp r + , appendonly = False + , availability = availabilityCalc r + , remotetype = remote + , mkUnavailable = return Nothing + , getInfo = gitRepoInfo this + , claimUrl = Nothing + , checkUrl = Nothing + } + return $ Just $ specialRemote' specialcfg c + (simplyPrepare $ store this rsyncopts) + (simplyPrepare $ retrieve this rsyncopts) + (simplyPrepare $ remove this rsyncopts) + (simplyPrepare $ checkKey this rsyncopts) + this + where + specialcfg + | Git.repoIsUrl r = (specialRemoteCfg c) + -- Rsync displays its own progress. + { displayProgress = False } + | otherwise = specialRemoteCfg c + +rsyncTransportToObjects :: Git.Repo -> RemoteGitConfig -> Annex ([CommandParam], String) +rsyncTransportToObjects r gc = do + (rsynctransport, rsyncurl, _) <- rsyncTransport r gc + return (rsynctransport, rsyncurl ++ "/annex/objects") + +rsyncTransport :: Git.Repo -> RemoteGitConfig -> Annex ([CommandParam], String, AccessMethod) +rsyncTransport r gc + | "ssh://" `isPrefixOf` loc = sshtransport $ break (== '/') $ drop (length "ssh://") loc + | "//:" `isInfixOf` loc = othertransport + | ":" `isInfixOf` loc = sshtransport $ separate (== ':') loc + | otherwise = othertransport + where + loc = Git.repoLocation r + sshtransport (host, path) = do + let rsyncpath = if "/~/" `isPrefixOf` path + then drop 3 path + else path + let sshhost = either error id (mkSshHost host) + opts <- sshOptions ConsumeStdin (sshhost, Nothing) gc [] + return (rsyncShell $ Param "ssh" : opts, fromSshHost sshhost ++ ":" ++ rsyncpath, AccessShell) + othertransport = return ([], loc, AccessDirect) + +noCrypto :: Annex a +noCrypto = giveup "cannot use gcrypt remote without encryption enabled" + +unsupportedUrl :: a +unsupportedUrl = giveup "using non-ssh remote repo url with gcrypt is not supported" + +gCryptSetup :: SetupStage -> Maybe UUID -> Maybe CredPair -> RemoteConfig -> RemoteGitConfig -> Annex (RemoteConfig, UUID) +gCryptSetup _ mu _ c gc = go $ M.lookup "gitrepo" c + where + remotename = fromJust (M.lookup "name" c) + go Nothing = giveup "Specify gitrepo=" + go (Just gitrepo) = do + (c', _encsetup) <- encryptionSetup c gc + + let url = Git.GCrypt.urlPrefix ++ gitrepo + rs <- Annex.getGitRemotes + case filter (\r -> Git.remoteName r == Just remotename) rs of + [] -> inRepo $ Git.Command.run + [ Param "remote", Param "add" + , Param remotename + , Param url + ] + (r:_) + | Git.repoLocation r == url -> noop + | otherwise -> error "Another remote with the same name already exists." + + setGcryptEncryption c' remotename + + {- Run a git fetch and a push to the git repo in order to get + - its gcrypt-id set up, so that later git annex commands + - will use the remote as a gcrypt remote. The fetch is + - needed if the repo already exists; the push is needed + - if the repo has not yet been initialized by gcrypt. -} + void $ inRepo $ Git.Command.runBool + [ Param "fetch" + , Param remotename + ] + void $ inRepo $ Git.Command.runBool + [ Param "push" + , Param remotename + , Param $ Git.fromRef Annex.Branch.fullname + ] + g <- inRepo Git.Config.reRead + case Git.GCrypt.remoteRepoId g (Just remotename) of + Nothing -> giveup "unable to determine gcrypt-id of remote" + Just gcryptid -> do + let u = genUUIDInNameSpace gCryptNameSpace gcryptid + if Just u == mu || isNothing mu + then do + method <- setupRepo gcryptid =<< inRepo (Git.Construct.fromRemoteLocation gitrepo) + gitConfigSpecialRemote u c' [("gcrypt", fromAccessMethod method)] + return (c', u) + else giveup $ "uuid mismatch; expected " ++ show mu ++ " but remote gitrepo has " ++ show u ++ " (" ++ show gcryptid ++ ")" + +{- Sets up the gcrypt repository. The repository is either a local + - repo, or it is accessed via rsync directly, or it is accessed over ssh + - and git-annex-shell is available to manage it. + - + - The GCryptID is recorded in the repository's git config for later use. + - Also, if the git config has receive.denyNonFastForwards set, disable + - it; gcrypt relies on being able to fast-forward branches. + -} +setupRepo :: Git.GCrypt.GCryptId -> Git.Repo -> Annex AccessMethod +setupRepo gcryptid r + | Git.repoIsUrl r = do + dummycfg <- liftIO dummyRemoteGitConfig + (_, _, accessmethod) <- rsyncTransport r dummycfg + case accessmethod of + AccessDirect -> rsyncsetup + AccessShell -> ifM gitannexshellsetup + ( return AccessShell + , rsyncsetup + ) + | Git.repoIsLocalUnknown r = localsetup =<< liftIO (Git.Config.read r) + | otherwise = localsetup r + where + localsetup r' = do + let setconfig k v = liftIO $ Git.Command.run [Param "config", Param k, Param v] r' + setconfig coreGCryptId gcryptid + setconfig denyNonFastForwards (Git.Config.boolConfig False) + return AccessDirect + + {- As well as modifying the remote's git config, + - create the objectDir on the remote, + - which is needed for direct rsync of objects to work. + -} + rsyncsetup = Remote.Rsync.withRsyncScratchDir $ \tmp -> do + liftIO $ createDirectoryIfMissing True $ tmp objectDir + dummycfg <- liftIO dummyRemoteGitConfig + (rsynctransport, rsyncurl, _) <- rsyncTransport r dummycfg + let tmpconfig = tmp "config" + void $ liftIO $ rsync $ rsynctransport ++ + [ Param $ rsyncurl ++ "/config" + , Param tmpconfig + ] + liftIO $ do + void $ Git.Config.changeFile tmpconfig coreGCryptId gcryptid + void $ Git.Config.changeFile tmpconfig denyNonFastForwards (Git.Config.boolConfig False) + ok <- liftIO $ rsync $ rsynctransport ++ + [ Param "--recursive" + , Param $ tmp ++ "/" + , Param rsyncurl + ] + unless ok $ + giveup "Failed to connect to remote to set it up." + return AccessDirect + + {- Ask git-annex-shell to configure the repository as a gcrypt + - repository. May fail if it is too old. -} + gitannexshellsetup = Ssh.onRemote NoConsumeStdin r + (boolSystem, return False) + "gcryptsetup" [ Param gcryptid ] [] + + denyNonFastForwards = "receive.denyNonFastForwards" + +accessShell :: Remote -> Bool +accessShell = accessShellConfig . gitconfig + +accessShellConfig :: RemoteGitConfig -> Bool +accessShellConfig c = case method of + AccessShell -> True + _ -> False + where + method = toAccessMethod $ fromMaybe "" $ remoteAnnexGCrypt c + +shellOrRsync :: Remote -> Annex a -> Annex a -> Annex a +shellOrRsync r ashell arsync + | accessShell r = ashell + | otherwise = arsync + +{- Configure gcrypt to use the same list of keyids that + - were passed to initremote as its participants. + - Also, configure it to use a signing key that is in the list of + - participants, which gcrypt requires is the case, and may not be + - depending on system configuration. + - + - (For shared encryption, gcrypt's default behavior is used.) + - + - Also, sets gcrypt-publish-participants to avoid unncessary gpg + - passphrase prompts. + -} +setGcryptEncryption :: RemoteConfig -> String -> Annex () +setGcryptEncryption c remotename = do + let participants = remoteconfig Git.GCrypt.remoteParticipantConfigKey + case cipherKeyIds =<< extractCipher c of + Nothing -> noCrypto + Just (KeyIds { keyIds = ks}) -> do + setConfig participants (unwords ks) + let signingkey = ConfigKey $ Git.GCrypt.remoteSigningKey remotename + cmd <- gpgCmd <$> Annex.getGitConfig + skeys <- M.keys <$> liftIO (secretKeys cmd) + case filter (`elem` ks) skeys of + [] -> noop + (k:_) -> setConfig signingkey k + setConfig (remoteconfig Git.GCrypt.remotePublishParticipantConfigKey) + (Git.Config.boolConfig True) + where + remoteconfig n = ConfigKey $ n remotename + +store :: Remote -> Remote.Rsync.RsyncOpts -> Storer +store r rsyncopts k s p = do + repo <- getRepo r + store' repo r rsyncopts k s p + +store' :: Git.Repo -> Remote -> Remote.Rsync.RsyncOpts -> Storer +store' repo r rsyncopts + | not $ Git.repoIsUrl repo = + byteStorer $ \k b p -> guardUsable repo (return False) $ liftIO $ do + let tmpdir = Git.repoLocation repo "tmp" keyFile k + void $ tryIO $ createDirectoryIfMissing True tmpdir + let tmpf = tmpdir keyFile k + meteredWriteFile p tmpf b + let destdir = parentDir $ gCryptLocation repo k + Remote.Directory.finalizeStoreGeneric tmpdir destdir + return True + | Git.repoIsSsh repo = if accessShell r + then fileStorer $ \k f p -> do + oh <- mkOutputHandler + Ssh.rsyncHelper oh (Just p) + =<< Ssh.rsyncParamsRemote False r Upload k f + (AssociatedFile Nothing) + else fileStorer $ Remote.Rsync.store rsyncopts + | otherwise = unsupportedUrl + +retrieve :: Remote -> Remote.Rsync.RsyncOpts -> Retriever +retrieve r rsyncopts k p sink = do + repo <- getRepo r + retrieve' repo r rsyncopts k p sink + +retrieve' :: Git.Repo -> Remote -> Remote.Rsync.RsyncOpts -> Retriever +retrieve' repo r rsyncopts + | not $ Git.repoIsUrl repo = byteRetriever $ \k sink -> + guardUsable repo (return False) $ + sink =<< liftIO (L.readFile $ gCryptLocation repo k) + | Git.repoIsSsh repo = if accessShell r + then fileRetriever $ \f k p -> do + ps <- Ssh.rsyncParamsRemote False r Download k f + (AssociatedFile Nothing) + oh <- mkOutputHandler + unlessM (Ssh.rsyncHelper oh (Just p) ps) $ + giveup "rsync failed" + else fileRetriever $ Remote.Rsync.retrieve rsyncopts + | otherwise = unsupportedUrl + where + +remove :: Remote -> Remote.Rsync.RsyncOpts -> Remover +remove r rsyncopts k = do + repo <- getRepo r + remove' repo r rsyncopts k + +remove' :: Git.Repo -> Remote -> Remote.Rsync.RsyncOpts -> Remover +remove' repo r rsyncopts k + | not $ Git.repoIsUrl repo = guardUsable repo (return False) $ + liftIO $ Remote.Directory.removeDirGeneric (Git.repoLocation repo) (parentDir (gCryptLocation repo k)) + | Git.repoIsSsh repo = shellOrRsync r removeshell removersync + | otherwise = unsupportedUrl + where + removersync = Remote.Rsync.remove rsyncopts k + removeshell = Ssh.dropKey repo k + +checkKey :: Remote -> Remote.Rsync.RsyncOpts -> CheckPresent +checkKey r rsyncopts k = do + repo <- getRepo r + checkKey' repo r rsyncopts k + +checkKey' :: Git.Repo -> Remote -> Remote.Rsync.RsyncOpts -> CheckPresent +checkKey' repo r rsyncopts k + | not $ Git.repoIsUrl repo = + guardUsable repo (cantCheck repo) $ + liftIO $ doesFileExist (gCryptLocation repo k) + | Git.repoIsSsh repo = shellOrRsync r checkshell checkrsync + | otherwise = unsupportedUrl + where + checkrsync = Remote.Rsync.checkKey repo rsyncopts k + checkshell = Ssh.inAnnex repo k + +{- Annexed objects are hashed using lower-case directories for max + - portability. -} +gCryptLocation :: Git.Repo -> Key -> FilePath +gCryptLocation repo key = Git.repoLocation repo objectDir keyPath key (hashDirLower def) + +data AccessMethod = AccessDirect | AccessShell + +fromAccessMethod :: AccessMethod -> String +fromAccessMethod AccessShell = "shell" +fromAccessMethod AccessDirect = "true" + +toAccessMethod :: String -> AccessMethod +toAccessMethod "shell" = AccessShell +toAccessMethod _ = AccessDirect + +getGCryptUUID :: Bool -> Git.Repo -> Annex (Maybe UUID) +getGCryptUUID fast r = do + dummycfg <- liftIO dummyRemoteGitConfig + (genUUIDInNameSpace gCryptNameSpace <$>) . fst + <$> getGCryptId fast r dummycfg + +coreGCryptId :: String +coreGCryptId = "core.gcrypt-id" + +{- gcrypt repos set up by git-annex as special remotes have a + - core.gcrypt-id setting in their config, which can be mapped back to + - the remote's UUID. + - + - In fast mode, only checks local repos. To check a remote repo, + - tries git-annex-shell and direct rsync of the git config file. + - + - (Also returns a version of input repo with its config read.) -} +getGCryptId :: Bool -> Git.Repo -> RemoteGitConfig -> Annex (Maybe Git.GCrypt.GCryptId, Git.Repo) +getGCryptId fast r gc + | Git.repoIsLocal r || Git.repoIsLocalUnknown r = extract <$> + liftIO (catchMaybeIO $ Git.Config.read r) + | not fast = extract . liftM fst <$> getM (eitherToMaybe <$>) + [ Ssh.onRemote NoConsumeStdin r (Git.Config.fromPipe r, return (Left $ error "configlist failed")) "configlist" [] [] + , getConfigViaRsync r gc + ] + | otherwise = return (Nothing, r) + where + extract Nothing = (Nothing, r) + extract (Just r') = (Git.Config.getMaybe coreGCryptId r', r') + +getConfigViaRsync :: Git.Repo -> RemoteGitConfig -> Annex (Either SomeException (Git.Repo, String)) +getConfigViaRsync r gc = do + (rsynctransport, rsyncurl, _) <- rsyncTransport r gc + liftIO $ do + withTmpFile "tmpconfig" $ \tmpconfig _ -> do + void $ rsync $ rsynctransport ++ + [ Param $ rsyncurl ++ "/config" + , Param tmpconfig + ] + Git.Config.fromFile r tmpconfig diff --git a/Remote/Git.hs b/Remote/Git.hs new file mode 100644 index 0000000000..295c1132c9 --- /dev/null +++ b/Remote/Git.hs @@ -0,0 +1,831 @@ +{- Standard git remotes. + - + - Copyright 2011-2018 Joey Hess + - + - Licensed under the GNU AGPL version 3 or higher. + -} + +{-# LANGUAGE CPP #-} + +module Remote.Git ( + remote, + configRead, + repoAvail, + onLocal, +) where + +import Annex.Common +import Annex.Ssh +import Types.Remote +import Types.GitConfig +import qualified Git +import qualified Git.Config +import qualified Git.Construct +import qualified Git.Command +import qualified Git.GCrypt +import qualified Git.Types as Git +import qualified Annex +import Logs.Presence +import Annex.Transfer +import Annex.UUID +import qualified Annex.Content +import qualified Annex.BranchState +import qualified Annex.Branch +import qualified Annex.Url as Url +import Utility.Tmp +import Config +import Config.Cost +import Config.DynamicConfig +import Annex.Init +import Annex.Version +import Types.CleanupActions +import qualified CmdLine.GitAnnexShell.Fields as Fields +import Logs.Location +import Utility.Metered +#ifndef mingw32_HOST_OS +import Utility.CopyFile +#endif +import Utility.Env +import Utility.Batch +import Utility.SimpleProtocol +import Remote.Helper.Git +import Remote.Helper.Messages +import Remote.Helper.Export +import qualified Remote.Helper.Ssh as Ssh +import qualified Remote.GCrypt +import qualified Remote.P2P +import qualified Remote.Helper.P2P as P2PHelper +import P2P.Address +import Annex.Path +import Creds +import Types.NumCopies +import Annex.Action +import Messages.Progress + +import Control.Concurrent +import Control.Concurrent.MSampleVar +import qualified Data.Map as M +import Network.URI + +remote :: RemoteType +remote = RemoteType + { typename = "git" + , enumerate = list + , generate = gen + , setup = gitSetup + , exportSupported = exportUnsupported + } + +list :: Bool -> Annex [Git.Repo] +list autoinit = do + c <- fromRepo Git.config + rs <- mapM (tweakurl c) =<< Annex.getGitRemotes + mapM (configRead autoinit) rs + where + annexurl n = "remote." ++ n ++ ".annexurl" + tweakurl c r = do + let n = fromJust $ Git.remoteName r + case M.lookup (annexurl n) c of + Nothing -> return r + Just url -> inRepo $ \g -> + Git.Construct.remoteNamed n $ + Git.Construct.fromRemoteLocation url g + +{- Git remotes are normally set up using standard git command, not + - git-annex initremote and enableremote. + - + - For initremote, the git remote must already be set up, and have a uuid. + - Initremote simply remembers its location. + - + - enableremote simply sets up a git remote using the stored location. + - No attempt is made to make the remote be accessible via ssh key setup, + - etc. + -} +gitSetup :: SetupStage -> Maybe UUID -> Maybe CredPair -> RemoteConfig -> RemoteGitConfig -> Annex (RemoteConfig, UUID) +gitSetup Init mu _ c _ = do + let location = fromMaybe (giveup "Specify location=url") $ + Url.parseURIRelaxed =<< M.lookup "location" c + rs <- Annex.getGitRemotes + u <- case filter (\r -> Git.location r == Git.Url location) rs of + [r] -> getRepoUUID r + [] -> giveup "could not find existing git remote with specified location" + _ -> giveup "found multiple git remotes with specified location" + if isNothing mu || mu == Just u + then return (c, u) + else error "git remote did not have specified uuid" +gitSetup (Enable _) (Just u) _ c _ = do + inRepo $ Git.Command.run + [ Param "remote" + , Param "add" + , Param $ fromMaybe (giveup "no name") (M.lookup "name" c) + , Param $ fromMaybe (giveup "no location") (M.lookup "location" c) + ] + return (c, u) +gitSetup (Enable _) Nothing _ _ _ = error "unable to enable git remote with no specified uuid" + +{- It's assumed to be cheap to read the config of non-URL remotes, so this is + - done each time git-annex is run in a way that uses remotes, unless + - annex-checkuuid is false. + - + - Conversely, the config of an URL remote is only read when there is no + - cached UUID value. -} +configRead :: Bool -> Git.Repo -> Annex Git.Repo +configRead autoinit r = do + gc <- Annex.getRemoteGitConfig r + u <- getRepoUUID r + annexignore <- liftIO $ getDynamicConfig (remoteAnnexIgnore gc) + case (repoCheap r, annexignore, u) of + (_, True, _) -> return r + (True, _, _) + | remoteAnnexCheckUUID gc -> tryGitConfigRead autoinit r + | otherwise -> return r + (False, _, NoUUID) -> tryGitConfigRead autoinit r + _ -> return r + +gen :: Git.Repo -> UUID -> RemoteConfig -> RemoteGitConfig -> Annex (Maybe Remote) +gen r u c gc + | Git.GCrypt.isEncrypted r = Remote.GCrypt.chainGen r u c gc + | otherwise = case repoP2PAddress r of + Nothing -> do + st <- mkState r u gc + go st <$> remoteCost gc defcst + Just addr -> Remote.P2P.chainGen addr r u c gc + where + defcst = if repoCheap r then cheapRemoteCost else expensiveRemoteCost + go st cst = Just new + where + new = Remote + { uuid = u + , cost = cst + , name = Git.repoDescribe r + , storeKey = copyToRemote new st + , retrieveKeyFile = copyFromRemote new st + , retrieveKeyFileCheap = copyFromRemoteCheap new st + , retrievalSecurityPolicy = RetrievalAllKeysSecure + , removeKey = dropKey new st + , lockContent = Just (lockKey new st) + , checkPresent = inAnnex new st + , checkPresentCheap = repoCheap r + , exportActions = exportUnsupported + , whereisKey = Nothing + , remoteFsck = if Git.repoIsUrl r + then Nothing + else Just $ fsckOnRemote r + , repairRepo = if Git.repoIsUrl r + then Nothing + else Just $ repairRemote r + , config = c + , localpath = localpathCalc r + , getRepo = getRepoFromState st + , gitconfig = gc + , readonly = Git.repoIsHttp r + , appendonly = False + , availability = availabilityCalc r + , remotetype = remote + , mkUnavailable = unavailable r u c gc + , getInfo = gitRepoInfo new + , claimUrl = Nothing + , checkUrl = Nothing + } + +unavailable :: Git.Repo -> UUID -> RemoteConfig -> RemoteGitConfig -> Annex (Maybe Remote) +unavailable r u c gc = gen r' u c gc + where + r' = case Git.location r of + Git.Local { Git.gitdir = d } -> + r { Git.location = Git.LocalUnknown d } + Git.Url url -> case uriAuthority url of + Just auth -> + let auth' = auth { uriRegName = "!dne!" } + in r { Git.location = Git.Url (url { uriAuthority = Just auth' })} + Nothing -> r { Git.location = Git.Unknown } + _ -> r -- already unavailable + +{- Checks relatively inexpensively if a repository is available for use. -} +repoAvail :: Git.Repo -> Annex Bool +repoAvail r + | Git.repoIsHttp r = return True + | Git.GCrypt.isEncrypted r = do + g <- gitRepo + liftIO $ do + er <- Git.GCrypt.encryptedRemote g r + if Git.repoIsLocal er || Git.repoIsLocalUnknown er + then catchBoolIO $ + void (Git.Config.read er) >> return True + else return True + | Git.repoIsUrl r = return True + | Git.repoIsLocalUnknown r = return False + | otherwise = liftIO $ isJust <$> catchMaybeIO (Git.Config.read r) + +{- Tries to read the config for a specified remote, updates state, and + - returns the updated repo. -} +tryGitConfigRead :: Bool -> Git.Repo -> Annex Git.Repo +tryGitConfigRead autoinit r + | haveconfig r = return r -- already read + | Git.repoIsSsh r = store $ do + v <- Ssh.onRemote NoConsumeStdin r + (pipedconfig, return (Left $ giveup "configlist failed")) + "configlist" [] configlistfields + case v of + Right r' + | haveconfig r' -> return r' + | otherwise -> configlist_failed + Left _ -> configlist_failed + | Git.repoIsHttp r = store geturlconfig + | Git.GCrypt.isEncrypted r = handlegcrypt =<< getConfigMaybe (remoteConfig r "uuid") + | Git.repoIsUrl r = return r + | otherwise = store $ liftIO $ + readlocalannexconfig `catchNonAsync` (const $ return r) + where + haveconfig = not . M.null . Git.config + + pipedconfig cmd params = do + v <- Git.Config.fromPipe r cmd params + case v of + Right (r', val) -> do + unless (isUUIDConfigured r' || null val) $ do + warningIO $ "Failed to get annex.uuid configuration of repository " ++ Git.repoDescribe r + warningIO $ "Instead, got: " ++ show val + warningIO $ "This is unexpected; please check the network transport!" + return $ Right r' + Left l -> return $ Left l + + geturlconfig = Url.withUrlOptions $ \uo -> do + v <- liftIO $ withTmpFile "git-annex.tmp" $ \tmpfile h -> do + hClose h + let url = Git.repoLocation r ++ "/config" + ifM (Url.download nullMeterUpdate url tmpfile uo) + ( Just <$> pipedconfig "git" [Param "config", Param "--null", Param "--list", Param "--file", File tmpfile] + , return Nothing + ) + case v of + Just (Right r') -> do + -- Cache when http remote is not bare for + -- optimisation. + unless (Git.Config.isBare r') $ + setremote setRemoteBare False + return r' + _ -> do + set_ignore "not usable by git-annex" False + return r + + store = observe $ \r' -> do + l <- Annex.getGitRemotes + let rs = exchange l r' + Annex.changeState $ \s -> s { Annex.gitremotes = Just rs } + + exchange [] _ = [] + exchange (old:ls) new + | Git.remoteName old == Git.remoteName new = + new : exchange ls new + | otherwise = + old : exchange ls new + + {- Is this remote just not available, or does + - it not have git-annex-shell? + - Find out by trying to fetch from the remote. -} + configlist_failed = case Git.remoteName r of + Nothing -> return r + Just n -> do + whenM (inRepo $ Git.Command.runBool [Param "fetch", Param "--quiet", Param n]) $ do + set_ignore "does not have git-annex installed" True + return r + + set_ignore msg longmessage = do + case Git.remoteName r of + Nothing -> noop + Just n -> do + warning $ "Remote " ++ n ++ " " ++ msg ++ "; setting annex-ignore" + when longmessage $ + warning $ "This could be a problem with the git-annex installation on the remote. Please make sure that git-annex-shell is available in PATH when you ssh into the remote. Once you have fixed the git-annex installation, run: git annex enableremote " ++ n + setremote setRemoteIgnore True + + setremote setter v = case Git.remoteName r of + Nothing -> noop + Just _ -> setter r v + + handlegcrypt Nothing = return r + handlegcrypt (Just _cacheduuid) = do + -- Generate UUID from the gcrypt-id + g <- gitRepo + case Git.GCrypt.remoteRepoId g (Git.remoteName r) of + Nothing -> return r + Just v -> store $ liftIO $ setUUID r $ + genUUIDInNameSpace gCryptNameSpace v + + {- The local repo may not yet be initialized, so try to initialize + - it if allowed. However, if that fails, still return the read + - git config. -} + readlocalannexconfig = do + let check = do + Annex.BranchState.disableUpdate + void $ tryNonAsync $ ensureInitialized + Annex.getState Annex.repo + s <- Annex.new r + Annex.eval s $ check `finally` stopCoProcesses + + configlistfields = if autoinit + then [(Fields.autoInit, "1")] + else [] + +{- Checks if a given remote has the content for a key in its annex. -} +inAnnex :: Remote -> State -> Key -> Annex Bool +inAnnex rmt st key = do + repo <- getRepo rmt + inAnnex' repo rmt st key + +inAnnex' :: Git.Repo -> Remote -> State -> Key -> Annex Bool +inAnnex' repo rmt (State connpool duc _) key + | Git.repoIsHttp repo = checkhttp + | Git.repoIsUrl repo = checkremote + | otherwise = checklocal + where + checkhttp = do + showChecking repo + gc <- Annex.getGitConfig + ifM (Url.withUrlOptions $ \uo -> liftIO $ + anyM (\u -> Url.checkBoth u (keySize key) uo) (keyUrls gc repo rmt key)) + ( return True + , giveup "not found" + ) + checkremote = + let fallback = Ssh.inAnnex repo key + in P2PHelper.checkpresent (Ssh.runProto rmt connpool (cantCheck rmt) fallback) key + checklocal = ifM duc + ( guardUsable repo (cantCheck repo) $ + maybe (cantCheck repo) return + =<< onLocalFast repo rmt (Annex.Content.inAnnexSafe key) + , cantCheck repo + ) + +keyUrls :: GitConfig -> Git.Repo -> Remote -> Key -> [String] +keyUrls gc repo r key = map tourl locs' + where + tourl l = Git.repoLocation repo ++ "/" ++ l + -- If the remote is known to not be bare, try the hash locations + -- used for non-bare repos first, as an optimisation. + locs + | remoteAnnexBare remoteconfig == Just False = reverse (annexLocations gc key) + | otherwise = annexLocations gc key +#ifndef mingw32_HOST_OS + locs' = locs +#else + locs' = map (replace "\\" "/") locs +#endif + remoteconfig = gitconfig r + +dropKey :: Remote -> State -> Key -> Annex Bool +dropKey r st key = do + repo <- getRepo r + dropKey' repo r st key + +dropKey' :: Git.Repo -> Remote -> State -> Key -> Annex Bool +dropKey' repo r (State connpool duc _) key + | not $ Git.repoIsUrl repo = ifM duc + ( guardUsable repo (return False) $ + commitOnCleanup repo r $ onLocalFast repo r $ do + ensureInitialized + whenM (Annex.Content.inAnnex key) $ do + Annex.Content.lockContentForRemoval key $ \lock -> do + Annex.Content.removeAnnex lock + logStatus key InfoMissing + Annex.Content.saveState True + return True + , return False + ) + | Git.repoIsHttp repo = giveup "dropping from http remote not supported" + | otherwise = commitOnCleanup repo r $ do + let fallback = Ssh.dropKey repo key + P2PHelper.remove (Ssh.runProto r connpool (return False) fallback) key + +lockKey :: Remote -> State -> Key -> (VerifiedCopy -> Annex r) -> Annex r +lockKey r st key callback = do + repo <- getRepo r + lockKey' repo r st key callback + +lockKey' :: Git.Repo -> Remote -> State -> Key -> (VerifiedCopy -> Annex r) -> Annex r +lockKey' repo r (State connpool duc _) key callback + | not $ Git.repoIsUrl repo = ifM duc + ( guardUsable repo failedlock $ do + inorigrepo <- Annex.makeRunner + -- Lock content from perspective of remote, + -- and then run the callback in the original + -- annex monad, not the remote's. + onLocalFast repo r $ + Annex.Content.lockContentShared key $ + liftIO . inorigrepo . callback + , failedlock + ) + | Git.repoIsSsh repo = do + showLocking r + let withconn = Ssh.withP2PSshConnection r connpool fallback + P2PHelper.lock withconn Ssh.runProtoConn (uuid r) key callback + | otherwise = failedlock + where + fallback = do + Just (cmd, params) <- Ssh.git_annex_shell ConsumeStdin + repo "lockcontent" + [Param $ key2file key] [] + (Just hin, Just hout, Nothing, p) <- liftIO $ + withFile devNull WriteMode $ \nullh -> + createProcess $ + (proc cmd (toCommand params)) + { std_in = CreatePipe + , std_out = CreatePipe + , std_err = UseHandle nullh + } + v <- liftIO $ tryIO $ getProtocolLine hout + let signaldone = void $ tryNonAsync $ liftIO $ mapM_ tryNonAsync + [ hPutStrLn hout "" + , hFlush hout + , hClose hin + , hClose hout + , void $ waitForProcess p + ] + let checkexited = not . isJust <$> getProcessExitCode p + case v of + Left _exited -> do + showNote "lockcontent failed" + liftIO $ do + hClose hin + hClose hout + void $ waitForProcess p + failedlock + Right l + | l == Just Ssh.contentLockedMarker -> bracket_ + noop + signaldone + (withVerifiedCopy LockedCopy r checkexited callback) + | otherwise -> do + showNote "lockcontent failed" + signaldone + failedlock + failedlock = giveup "can't lock content" + +{- Tries to copy a key's content from a remote's annex to a file. -} +copyFromRemote :: Remote -> State -> Key -> AssociatedFile -> FilePath -> MeterUpdate -> Annex (Bool, Verification) +copyFromRemote = copyFromRemote' False + +copyFromRemote' :: Bool -> Remote -> State -> Key -> AssociatedFile -> FilePath -> MeterUpdate -> Annex (Bool, Verification) +copyFromRemote' forcersync r st key file dest meterupdate = do + repo <- getRepo r + copyFromRemote'' repo forcersync r st key file dest meterupdate + +copyFromRemote'' :: Git.Repo -> Bool -> Remote -> State -> Key -> AssociatedFile -> FilePath -> MeterUpdate -> Annex (Bool, Verification) +copyFromRemote'' repo forcersync r (State connpool _ _) key file dest meterupdate + | Git.repoIsHttp repo = unVerified $ do + gc <- Annex.getGitConfig + Annex.Content.downloadUrl key meterupdate (keyUrls gc repo r key) dest + | not $ Git.repoIsUrl repo = guardUsable repo (unVerified (return False)) $ do + params <- Ssh.rsyncParams r Download + u <- getUUID + hardlink <- wantHardLink + -- run copy from perspective of remote + onLocalFast repo r $ do + ensureInitialized + v <- Annex.Content.prepSendAnnex key + case v of + Nothing -> return (False, UnVerified) + Just (object, checksuccess) -> do + copier <- mkCopier hardlink params + runTransfer (Transfer Download u key) + file stdRetry + (\p -> copier object dest (combineMeterUpdate p meterupdate) checksuccess) + | Git.repoIsSsh repo = if forcersync + then fallback meterupdate + else P2PHelper.retrieve + (\p -> Ssh.runProto r connpool (return (False, UnVerified)) (fallback p)) + key file dest meterupdate + | otherwise = giveup "copying from non-ssh, non-http remote not supported" + where + fallback p = unVerified $ feedprogressback $ \p' -> do + oh <- mkOutputHandlerQuiet + Ssh.rsyncHelper oh (Just (combineMeterUpdate p' p)) + =<< Ssh.rsyncParamsRemote False r Download key dest file + {- Feed local rsync's progress info back to the remote, + - by forking a feeder thread that runs + - git-annex-shell transferinfo at the same time + - git-annex-shell sendkey is running. + - + - To avoid extra password prompts, this is only done when ssh + - connection caching is supported. + - Note that it actually waits for rsync to indicate + - progress before starting transferinfo, in order + - to ensure ssh connection caching works and reuses + - the connection set up for the sendkey. + - + - Also note that older git-annex-shell does not support + - transferinfo, so stderr is dropped and failure ignored. + -} + feedprogressback a = ifM (isJust <$> sshCacheDir) + ( feedprogressback' a + , a $ const noop + ) + feedprogressback' a = do + u <- getUUID + let AssociatedFile afile = file + let fields = (Fields.remoteUUID, fromUUID u) + : maybe [] (\f -> [(Fields.associatedFile, f)]) afile + Just (cmd, params) <- Ssh.git_annex_shell ConsumeStdin + repo "transferinfo" + [Param $ key2file key] fields + v <- liftIO (newEmptySV :: IO (MSampleVar Integer)) + pidv <- liftIO $ newEmptyMVar + tid <- liftIO $ forkIO $ void $ tryIO $ do + bytes <- readSV v + p <- createProcess $ + (proc cmd (toCommand params)) + { std_in = CreatePipe + , std_err = CreatePipe + } + putMVar pidv (processHandle p) + hClose $ stderrHandle p + let h = stdinHandle p + let send b = do + hPrint h b + hFlush h + send bytes + forever $ + send =<< readSV v + let feeder = \n -> do + meterupdate n + writeSV v (fromBytesProcessed n) + + -- It can easily take 0.3 seconds to clean up after + -- the transferinfo, and all that's involved is shutting + -- down the process and associated thread cleanly. So, + -- do it in the background. + let cleanup = forkIO $ do + void $ tryIO $ killThread tid + void $ tryNonAsync $ + maybe noop (void . waitForProcess) + =<< tryTakeMVar pidv + bracketIO noop (const cleanup) (const $ a feeder) + +copyFromRemoteCheap :: Remote -> State -> Key -> AssociatedFile -> FilePath -> Annex Bool +copyFromRemoteCheap r st key af file = do + repo <- getRepo r + copyFromRemoteCheap' repo r st key af file + +copyFromRemoteCheap' :: Git.Repo -> Remote -> State -> Key -> AssociatedFile -> FilePath -> Annex Bool +#ifndef mingw32_HOST_OS +copyFromRemoteCheap' repo r st key af file + | not $ Git.repoIsUrl repo = guardUsable repo (return False) $ do + gc <- getGitConfigFromState st + loc <- liftIO $ gitAnnexLocation key repo gc + liftIO $ ifM (doesFileExist loc) + ( do + absloc <- absPath loc + catchBoolIO $ do + createSymbolicLink absloc file + return True + , return False + ) + | Git.repoIsSsh repo = + ifM (Annex.Content.preseedTmp key file) + ( fst <$> copyFromRemote' True r st key af file nullMeterUpdate + , return False + ) + | otherwise = return False +#else +copyFromRemoteCheap' _ _ _ _ _ _ = return False +#endif + +{- Tries to copy a key's content to a remote's annex. -} +copyToRemote :: Remote -> State -> Key -> AssociatedFile -> MeterUpdate -> Annex Bool +copyToRemote r st key file meterupdate = do + repo <- getRepo r + copyToRemote' repo r st key file meterupdate + +copyToRemote' :: Git.Repo -> Remote -> State -> Key -> AssociatedFile -> MeterUpdate -> Annex Bool +copyToRemote' repo r (State connpool duc _) key file meterupdate + | not $ Git.repoIsUrl repo = ifM duc + ( guardUsable repo (return False) $ commitOnCleanup repo r $ + copylocal =<< Annex.Content.prepSendAnnex key + , return False + ) + | Git.repoIsSsh repo = commitOnCleanup repo r $ + P2PHelper.store + (\p -> Ssh.runProto r connpool (return False) (copyremotefallback p)) + key file meterupdate + + | otherwise = giveup "copying to non-ssh repo not supported" + where + copylocal Nothing = return False + copylocal (Just (object, checksuccess)) = do + -- The checksuccess action is going to be run in + -- the remote's Annex, but it needs access to the local + -- Annex monad's state. + checksuccessio <- Annex.withCurrentState checksuccess + params <- Ssh.rsyncParams r Upload + u <- getUUID + hardlink <- wantHardLink + -- run copy from perspective of remote + onLocalFast repo r $ ifM (Annex.Content.inAnnex key) + ( return True + , do + ensureInitialized + copier <- mkCopier hardlink params + let verify = Annex.Content.RemoteVerify r + let rsp = RetrievalAllKeysSecure + runTransfer (Transfer Download u key) file stdRetry $ \p -> + let p' = combineMeterUpdate meterupdate p + in Annex.Content.saveState True `after` + Annex.Content.getViaTmp rsp verify key + (\dest -> copier object dest p' (liftIO checksuccessio)) + ) + copyremotefallback p = Annex.Content.sendAnnex key noop $ \object -> do + -- This is too broad really, but recvkey normally + -- verifies content anyway, so avoid complicating + -- it with a local sendAnnex check and rollback. + unlocked <- isDirect <||> versionSupportsUnlockedPointers + oh <- mkOutputHandlerQuiet + Ssh.rsyncHelper oh (Just p) + =<< Ssh.rsyncParamsRemote unlocked r Upload key object file + +fsckOnRemote :: Git.Repo -> [CommandParam] -> Annex (IO Bool) +fsckOnRemote r params + | Git.repoIsUrl r = do + s <- Ssh.git_annex_shell NoConsumeStdin r "fsck" params [] + return $ case s of + Nothing -> return False + Just (c, ps) -> batchCommand c ps + | otherwise = return $ do + program <- programPath + r' <- Git.Config.read r + environ <- getEnvironment + let environ' = addEntries + [ ("GIT_WORK_TREE", Git.repoPath r') + , ("GIT_DIR", Git.localGitDir r') + ] environ + batchCommandEnv program (Param "fsck" : params) (Just environ') + +{- The passed repair action is run in the Annex monad of the remote. -} +repairRemote :: Git.Repo -> Annex Bool -> Annex (IO Bool) +repairRemote r a = return $ do + s <- Annex.new r + Annex.eval s $ do + Annex.BranchState.disableUpdate + ensureInitialized + a `finally` stopCoProcesses + +{- Runs an action from the perspective of a local remote. + - + - The AnnexState is cached for speed and to avoid resource leaks. + - However, coprocesses are stopped after each call to avoid git + - processes hanging around on removable media. + -} +onLocal :: Git.Repo -> Remote -> Annex a -> Annex a +onLocal repo r a = do + m <- Annex.getState Annex.remoteannexstate + go =<< maybe + (liftIO $ Annex.new repo) + return + (M.lookup (uuid r) m) + where + cache st = Annex.changeState $ \s -> s + { Annex.remoteannexstate = M.insert (uuid r) st (Annex.remoteannexstate s) } + go st = do + curro <- Annex.getState Annex.output + (ret, st') <- liftIO $ Annex.run (st { Annex.output = curro }) $ + a `finally` stopCoProcesses + cache st' + return ret + +{- Faster variant of onLocal. + - + - The repository's git-annex branch is not updated, as an optimisation. + - No caller of onLocalFast can query data from the branch and be ensured + - it gets the most current value. Caller of onLocalFast can make changes + - to the branch, however. + -} +onLocalFast :: Git.Repo -> Remote -> Annex a -> Annex a +onLocalFast repo r a = onLocal repo r $ Annex.BranchState.disableUpdate >> a + +{- Copys a file with rsync unless both locations are on the same + - filesystem. Then cp could be faster. -} +rsyncOrCopyFile :: [CommandParam] -> FilePath -> FilePath -> MeterUpdate -> Annex Bool +rsyncOrCopyFile rsyncparams src dest p = +#ifdef mingw32_HOST_OS + dorsync + where +#else + ifM (sameDeviceIds src dest) (docopy, dorsync) + where + sameDeviceIds a b = (==) <$> getDeviceId a <*> getDeviceId b + getDeviceId f = deviceID <$> liftIO (getFileStatus $ parentDir f) + docopy = liftIO $ watchFileSize dest p $ + copyFileExternal CopyTimeStamps src dest +#endif + dorsync = do + oh <- mkOutputHandler + Ssh.rsyncHelper oh (Just p) $ + rsyncparams ++ [File src, File dest] + +commitOnCleanup :: Git.Repo -> Remote -> Annex a -> Annex a +commitOnCleanup repo r a = go `after` a + where + go = Annex.addCleanup (RemoteCleanup $ uuid r) cleanup + cleanup + | not $ Git.repoIsUrl repo = onLocalFast repo r $ + doQuietSideAction $ + Annex.Branch.commit =<< Annex.Branch.commitMessage + | otherwise = void $ do + Just (shellcmd, shellparams) <- + Ssh.git_annex_shell NoConsumeStdin + repo "commit" [] [] + + -- Throw away stderr, since the remote may not + -- have a new enough git-annex shell to + -- support committing. + liftIO $ catchMaybeIO $ + withQuietOutput createProcessSuccess $ + proc shellcmd $ + toCommand shellparams + +wantHardLink :: Annex Bool +wantHardLink = (annexHardLink <$> Annex.getGitConfig) + -- Not direct mode files because they can be modified at any time. + <&&> (not <$> isDirect) + -- Not unlocked files that are hard linked in the work tree, + -- because they can be modified at any time. + <&&> (not <$> annexThin <$> Annex.getGitConfig) + +-- Copies from src to dest, updating a meter. If the copy finishes +-- successfully, calls a final check action, which must also succeed, or +-- returns false. +-- +-- If either the remote or local repository wants to use hard links, +-- the copier will do so (falling back to copying if a hard link cannot be +-- made). +-- +-- When a hard link is created, returns Verified; the repo being linked +-- from is implicitly trusted, so no expensive verification needs to be +-- done. +type Copier = FilePath -> FilePath -> MeterUpdate -> Annex Bool -> Annex (Bool, Verification) + +mkCopier :: Bool -> [CommandParam] -> Annex Copier +mkCopier remotewanthardlink rsyncparams = do + let copier = \src dest p check -> unVerified $ + rsyncOrCopyFile rsyncparams src dest p <&&> check + localwanthardlink <- wantHardLink + let linker = \src dest -> createLink src dest >> return True + ifM (pure (remotewanthardlink || localwanthardlink) <&&> not <$> isDirect) + ( return $ \src dest p check -> + ifM (liftIO (catchBoolIO (linker src dest))) + ( return (True, Verified) + , copier src dest p check + ) + , return copier + ) + +{- Normally the UUID of a local repository is checked at startup, + - but annex-checkuuid config can prevent that. To avoid getting + - confused, a deferred check is done just before the repository + - is used. + - This returns False when the repository UUID is not as expected. -} +type DeferredUUIDCheck = Annex Bool + +data State = State Ssh.P2PSshConnectionPool DeferredUUIDCheck (Annex (Git.Repo, GitConfig)) + +getRepoFromState :: State -> Annex Git.Repo +getRepoFromState (State _ _ a) = fst <$> a + +{- The config of the remote git repository, cached for speed. -} +getGitConfigFromState :: State -> Annex GitConfig +getGitConfigFromState (State _ _ a) = snd <$> a + +mkState :: Git.Repo -> UUID -> RemoteGitConfig -> Annex State +mkState r u gc = do + pool <- Ssh.mkP2PSshConnectionPool + (duc, getrepo) <- go + return $ State pool duc getrepo + where + go + | remoteAnnexCheckUUID gc = return + (return True, return (r, extractGitConfig r)) + | otherwise = do + rv <- liftIO newEmptyMVar + let getrepo = ifM (liftIO $ isEmptyMVar rv) + ( do + r' <- tryGitConfigRead False r + let t = (r', extractGitConfig r') + void $ liftIO $ tryPutMVar rv t + return t + , liftIO $ readMVar rv + ) + + cv <- liftIO newEmptyMVar + let duc = ifM (liftIO $ isEmptyMVar cv) + ( do + r' <- fst <$> getrepo + u' <- getRepoUUID r' + let ok = u' == u + void $ liftIO $ tryPutMVar cv ok + unless ok $ + warning $ Git.repoDescribe r ++ " is not the expected repository. The remote's annex-checkuuid configuration prevented noticing the change until now." + return ok + , liftIO $ readMVar cv + ) + + return (duc, getrepo) diff --git a/Remote/Glacier.hs b/Remote/Glacier.hs new file mode 100644 index 0000000000..92d7e67590 --- /dev/null +++ b/Remote/Glacier.hs @@ -0,0 +1,329 @@ +{- Amazon Glacier remotes. + - + - Copyright 2012 Joey Hess + - + - Licensed under the GNU GPL version 3 or higher. + -} + +module Remote.Glacier (remote, jobList, checkSaneGlacierCommand) where + +import qualified Data.Map as M +import qualified Data.Text as T +import qualified Data.ByteString.Lazy as L + +import Annex.Common +import Types.Remote +import qualified Git +import Config +import Config.Cost +import Remote.Helper.Special +import Remote.Helper.Messages +import Remote.Helper.Export +import qualified Remote.Helper.AWS as AWS +import Creds +import Utility.Metered +import qualified Annex +import Annex.UUID +import Utility.Env + +type Vault = String +type Archive = FilePath + +remote :: RemoteType +remote = RemoteType + { typename = "glacier" + , enumerate = const (findSpecialRemotes "glacier") + , generate = gen + , setup = glacierSetup + , exportSupported = exportUnsupported + } + +gen :: Git.Repo -> UUID -> RemoteConfig -> RemoteGitConfig -> Annex (Maybe Remote) +gen r u c gc = new <$> remoteCost gc veryExpensiveRemoteCost + where + new cst = Just $ specialRemote' specialcfg c + (prepareStore this) + (prepareRetrieve this) + (simplyPrepare $ remove this) + (simplyPrepare $ checkKey this) + this + where + this = Remote + { uuid = u + , cost = cst + , name = Git.repoDescribe r + , storeKey = storeKeyDummy + , retrieveKeyFile = retreiveKeyFileDummy + , retrieveKeyFileCheap = retrieveCheap this + -- glacier-cli does not follow redirects and does + -- not support file://, as far as we know, but + -- there's no guarantee that will continue to be + -- the case, so require verifiable keys. + , retrievalSecurityPolicy = mkRetrievalVerifiableKeysSecure gc + , removeKey = removeKeyDummy + , lockContent = Nothing + , checkPresent = checkPresentDummy + , checkPresentCheap = False + , exportActions = exportUnsupported + , whereisKey = Nothing + , remoteFsck = Nothing + , repairRepo = Nothing + , config = c + , getRepo = return r + , gitconfig = gc + , localpath = Nothing + , readonly = False + , appendonly = False + , availability = GloballyAvailable + , remotetype = remote + , mkUnavailable = return Nothing + , getInfo = includeCredsInfo c (AWS.creds u) $ + [ ("glacier vault", getVault c) ] + , claimUrl = Nothing + , checkUrl = Nothing + } + specialcfg = (specialRemoteCfg c) + -- Disabled until jobList gets support for chunks. + { chunkConfig = NoChunks + } + +glacierSetup :: SetupStage -> Maybe UUID -> Maybe CredPair -> RemoteConfig -> RemoteGitConfig -> Annex (RemoteConfig, UUID) +glacierSetup ss mu mcreds c gc = do + u <- maybe (liftIO genUUID) return mu + glacierSetup' ss u mcreds c gc +glacierSetup' :: SetupStage -> UUID -> Maybe CredPair -> RemoteConfig -> RemoteGitConfig -> Annex (RemoteConfig, UUID) +glacierSetup' ss u mcreds c gc = do + (c', encsetup) <- encryptionSetup c gc + c'' <- setRemoteCredPair encsetup c' gc (AWS.creds u) mcreds + let fullconfig = c'' `M.union` defaults + case ss of + Init -> genVault fullconfig gc u + _ -> return () + gitConfigSpecialRemote u fullconfig [("glacier", "true")] + return (fullconfig, u) + where + remotename = fromJust (M.lookup "name" c) + defvault = remotename ++ "-" ++ fromUUID u + defaults = M.fromList + [ ("datacenter", T.unpack $ AWS.defaultRegion AWS.Glacier) + , ("vault", defvault) + ] + +prepareStore :: Remote -> Preparer Storer +prepareStore r = checkPrepare nonEmpty (byteStorer $ store r) + +nonEmpty :: Key -> Annex Bool +nonEmpty k + | keySize k == Just 0 = do + warning "Cannot store empty files in Glacier." + return False + | otherwise = return True + +store :: Remote -> Key -> L.ByteString -> MeterUpdate -> Annex Bool +store r k b p = go =<< glacierEnv c gc u + where + c = config r + gc = gitconfig r + u = uuid r + params = glacierParams c + [ Param "archive" + , Param "upload" + , Param "--name", Param $ archive r k + , Param $ getVault $ config r + , Param "-" + ] + go Nothing = return False + go (Just e) = do + let cmd = (proc "glacier" (toCommand params)) { env = Just e } + liftIO $ catchBoolIO $ + withHandle StdinHandle createProcessSuccess cmd $ \h -> do + meteredWrite p h b + return True + +prepareRetrieve :: Remote -> Preparer Retriever +prepareRetrieve = simplyPrepare . byteRetriever . retrieve + +retrieve :: Remote -> Key -> (L.ByteString -> Annex Bool) -> Annex Bool +retrieve r k sink = go =<< glacierEnv c gc u + where + c = config r + gc = gitconfig r + u = uuid r + params = glacierParams c + [ Param "archive" + , Param "retrieve" + , Param "-o-" + , Param $ getVault $ config r + , Param $ archive r k + ] + go Nothing = giveup "cannot retrieve from glacier" + go (Just e) = do + let cmd = (proc "glacier" (toCommand params)) + { env = Just e + , std_out = CreatePipe + } + (_, Just h, _, pid) <- liftIO $ createProcess cmd + -- Glacier cannot store empty files, so if the output is + -- empty, the content is not available yet. + ok <- ifM (liftIO $ hIsEOF h) + ( return False + , sink =<< liftIO (L.hGetContents h) + ) + liftIO $ hClose h + liftIO $ forceSuccessProcess cmd pid + unless ok $ do + showLongNote "Recommend you wait up to 4 hours, and then run this command again." + return ok + +retrieveCheap :: Remote -> Key -> AssociatedFile -> FilePath -> Annex Bool +retrieveCheap _ _ _ _ = return False + +remove :: Remote -> Remover +remove r k = glacierAction r + [ Param "archive" + + , Param "delete" + , Param $ getVault $ config r + , Param $ archive r k + ] + +checkKey :: Remote -> CheckPresent +checkKey r k = do + showChecking r + go =<< glacierEnv (config r) (gitconfig r) (uuid r) + where + go Nothing = giveup "cannot check glacier" + go (Just e) = do + {- glacier checkpresent outputs the archive name to stdout if + - it's present. -} + s <- liftIO $ readProcessEnv "glacier" (toCommand params) (Just e) + let probablypresent = key2file k `elem` lines s + if probablypresent + then ifM (Annex.getFlag "trustglacier") + ( return True, giveup untrusted ) + else return False + + params = glacierParams (config r) + [ Param "archive" + , Param "checkpresent" + , Param $ getVault $ config r + , Param "--quiet" + , Param $ archive r k + ] + + untrusted = unlines + [ "Glacier's inventory says it has a copy." + , "However, the inventory could be out of date, if it was recently removed." + , "(Use --trust-glacier if you're sure it's still in Glacier.)" + , "" + ] + +glacierAction :: Remote -> [CommandParam] -> Annex Bool +glacierAction r = runGlacier (config r) (gitconfig r) (uuid r) + +runGlacier :: RemoteConfig -> RemoteGitConfig -> UUID -> [CommandParam] -> Annex Bool +runGlacier c gc u params = go =<< glacierEnv c gc u + where + go Nothing = return False + go (Just e) = liftIO $ + boolSystemEnv "glacier" (glacierParams c params) (Just e) + +glacierParams :: RemoteConfig -> [CommandParam] -> [CommandParam] +glacierParams c params = datacenter:params + where + datacenter = Param $ "--region=" ++ + fromMaybe (giveup "Missing datacenter configuration") + (M.lookup "datacenter" c) + +glacierEnv :: RemoteConfig -> RemoteGitConfig -> UUID -> Annex (Maybe [(String, String)]) +glacierEnv c gc u = do + liftIO checkSaneGlacierCommand + go =<< getRemoteCredPairFor "glacier" c gc creds + where + go Nothing = return Nothing + go (Just (user, pass)) = do + e <- liftIO getEnvironment + return $ Just $ addEntries [(uk, user), (pk, pass)] e + + creds = AWS.creds u + (uk, pk) = credPairEnvironment creds + +getVault :: RemoteConfig -> Vault +getVault = fromMaybe (giveup "Missing vault configuration") + . M.lookup "vault" + +archive :: Remote -> Key -> Archive +archive r k = fileprefix ++ key2file k + where + fileprefix = M.findWithDefault "" "fileprefix" $ config r + +genVault :: RemoteConfig -> RemoteGitConfig -> UUID -> Annex () +genVault c gc u = unlessM (runGlacier c gc u params) $ + giveup "Failed creating glacier vault." + where + params = + [ Param "vault" + , Param "create" + , Param $ getVault c + ] + +{- Partitions the input list of keys into ones which have + - glacier retieval jobs that have succeeded, or failed. + - + - A complication is that `glacier job list` will display the encrypted + - keys when the remote is encrypted. + - + - Dealing with encrypted chunked keys would be tricky. However, there + - seems to be no benefit to using chunking with glacier, so chunking is + - not supported. + -} +jobList :: Remote -> [Key] -> Annex ([Key], [Key]) +jobList r keys = go =<< glacierEnv (config r) (gitconfig r) (uuid r) + where + params = [ Param "job", Param "list" ] + nada = ([], []) + myvault = getVault $ config r + + go Nothing = return nada + go (Just e) = do + v <- liftIO $ catchMaybeIO $ + readProcessEnv "glacier" (toCommand params) (Just e) + maybe (return nada) extract v + + extract s = do + let result@(succeeded, failed) = + parse nada $ (map words . lines) s + if result == nada + then return nada + else do + enckeys <- forM keys $ \k -> + maybe k (\(_, enck) -> enck k) + <$> cipherKey (config r) (gitconfig r) + let keymap = M.fromList $ zip enckeys keys + let convert = mapMaybe (`M.lookup` keymap) + return (convert succeeded, convert failed) + + parse c [] = c + parse c@(succeeded, failed) ((status:_date:vault:key:[]):rest) + | vault == myvault = + case file2key key of + Nothing -> parse c rest + Just k + | "a/d" `isPrefixOf` status -> + parse (k:succeeded, failed) rest + | "a/e" `isPrefixOf` status -> + parse (succeeded, k:failed) rest + | otherwise -> + parse c rest + parse c (_:rest) = parse c rest + +-- boto's version of glacier exits 0 when given a parameter it doesn't +-- understand. See https://github.com/boto/boto/issues/2942 +checkSaneGlacierCommand :: IO () +checkSaneGlacierCommand = + whenM ((Nothing /=) <$> catchMaybeIO shouldfail) $ + giveup wrongcmd + where + test = proc "glacier" ["--compatibility-test-git-annex"] + shouldfail = withQuietOutput createProcessSuccess test + wrongcmd = "The glacier program in PATH seems to be from boto, not glacier-cli. Cannot use this program." diff --git a/Remote/Helper/AWS.hs b/Remote/Helper/AWS.hs new file mode 100644 index 0000000000..7d23ac63d1 --- /dev/null +++ b/Remote/Helper/AWS.hs @@ -0,0 +1,82 @@ +{- Amazon Web Services common infrastructure. + - + - Copyright 2011-2014 Joey Hess + - + - Licensed under the GNU GPL version 3 or higher. + -} + +{-# LANGUAGE OverloadedStrings #-} +{-# LANGUAGE TupleSections #-} + +module Remote.Helper.AWS where + +import Annex.Common +import Creds + +import qualified Data.Map as M +import qualified Data.ByteString as B +import qualified Data.Text as T +import Data.Text.Encoding (encodeUtf8) +import Data.Text (Text) + +creds :: UUID -> CredPairStorage +creds u = CredPairStorage + { credPairFile = fromUUID u + , credPairEnvironment = ("AWS_ACCESS_KEY_ID", "AWS_SECRET_ACCESS_KEY") + , credPairRemoteKey = Just "s3creds" + } + +data Service = S3 | Glacier + deriving (Eq) + +type Region = Text + +regionMap :: Service -> M.Map Text Region +regionMap = M.fromList . regionInfo + +defaultRegion :: Service -> Region +defaultRegion = snd . Prelude.head . regionInfo + +data ServiceRegion = BothRegion Region | S3Region Region | GlacierRegion Region + +{- The "US" and "EU" names are used as location constraints when creating a + - S3 bucket. -} +regionInfo :: Service -> [(Text, Region)] +regionInfo service = map (\(t, r) -> (t, fromServiceRegion r)) $ + filter (matchingService . snd) $ + concatMap (\(t, l) -> map (t,) l) regions + where + regions = + [ ("US East (N. Virginia)", [S3Region "US", GlacierRegion "us-east-1"]) + , ("US West (Oregon)", [BothRegion "us-west-2"]) + , ("US West (N. California)", [BothRegion "us-west-1"]) + , ("EU (Ireland)", [S3Region "EU", GlacierRegion "eu-west-1"]) + , ("Asia Pacific (Singapore)", [S3Region "ap-southeast-1"]) + , ("Asia Pacific (Tokyo)", [BothRegion "ap-northeast-1"]) + , ("Asia Pacific (Sydney)", [S3Region "ap-southeast-2"]) + , ("South America (São Paulo)", [S3Region "sa-east-1"]) + -- These need signature V4 support, which has not landed in + -- the aws library. + -- See https://github.com/aristidb/aws/pull/199 + -- , ("EU (Frankfurt)", [BothRegion "eu-central-1"]) + -- , ("Asia Pacific (Seoul)", [S3Region "ap-northeast-2"]) + -- , ("Asia Pacific (Mumbai)", [S3Region "ap-south-1"]) + -- , ("US East (Ohio)", [S3Region "us-east-2"]) + ] + + fromServiceRegion (BothRegion s) = s + fromServiceRegion (S3Region s) = s + fromServiceRegion (GlacierRegion s) = s + + matchingService (BothRegion _) = True + matchingService (S3Region _) = service == S3 + matchingService (GlacierRegion _) = service == Glacier + +s3HostName :: Region -> B.ByteString +s3HostName "US" = "s3.amazonaws.com" +s3HostName "EU" = "s3-eu-west-1.amazonaws.com" +s3HostName "cn-north-1" = "s3.cn-north-1.amazonaws.com.cn" +s3HostName r = encodeUtf8 $ T.concat ["s3-", r, ".amazonaws.com"] + +s3DefaultHost :: String +s3DefaultHost = "s3.amazonaws.com" diff --git a/Remote/Helper/Chunked.hs b/Remote/Helper/Chunked.hs new file mode 100644 index 0000000000..f3c69c38dd --- /dev/null +++ b/Remote/Helper/Chunked.hs @@ -0,0 +1,411 @@ +{- git-annex chunked remotes + - + - Copyright 2014 Joey Hess + - + - Licensed under the GNU GPL version 3 or higher. + -} + +module Remote.Helper.Chunked ( + ChunkSize, + ChunkConfig(..), + noChunks, + describeChunkConfig, + getChunkConfig, + storeChunks, + removeChunks, + retrieveChunks, + checkPresentChunks, +) where + +import Annex.Common +import Utility.DataUnits +import Types.StoreRetrieve +import Types.Remote +import Logs.Chunk +import Utility.Metered +import Crypto (EncKey) +import Backend (isStableKey) + +import qualified Data.ByteString.Lazy as L +import qualified Data.Map as M + +data ChunkConfig + = NoChunks + | UnpaddedChunks ChunkSize + | LegacyChunks ChunkSize + deriving (Show) + +describeChunkConfig :: ChunkConfig -> String +describeChunkConfig NoChunks = "none" +describeChunkConfig (UnpaddedChunks sz) = describeChunkSize sz ++ "chunks" +describeChunkConfig (LegacyChunks sz) = describeChunkSize sz ++ " chunks (old style)" + +describeChunkSize :: ChunkSize -> String +describeChunkSize sz = roughSize storageUnits False (fromIntegral sz) + +noChunks :: ChunkConfig -> Bool +noChunks NoChunks = True +noChunks _ = False + +getChunkConfig :: RemoteConfig -> ChunkConfig +getChunkConfig m = + case M.lookup "chunksize" m of + Nothing -> case M.lookup "chunk" m of + Nothing -> NoChunks + Just v -> readsz UnpaddedChunks v "chunk" + Just v -> readsz LegacyChunks v "chunksize" + where + readsz c v f = case readSize dataUnits v of + Just size + | size == 0 -> NoChunks + | size > 0 -> c (fromInteger size) + _ -> giveup $ "bad configuration " ++ f ++ "=" ++ v + +-- An infinite stream of chunk keys, starting from chunk 1. +newtype ChunkKeyStream = ChunkKeyStream [Key] + +chunkKeyStream :: Key -> ChunkSize -> ChunkKeyStream +chunkKeyStream basek chunksize = ChunkKeyStream $ map mk [1..] + where + mk chunknum = sizedk { keyChunkNum = Just chunknum } + sizedk = basek { keyChunkSize = Just (toInteger chunksize) } + +nextChunkKeyStream :: ChunkKeyStream -> (Key, ChunkKeyStream) +nextChunkKeyStream (ChunkKeyStream (k:l)) = (k, ChunkKeyStream l) +nextChunkKeyStream (ChunkKeyStream []) = error "expected infinite ChunkKeyStream" + +takeChunkKeyStream :: ChunkCount -> ChunkKeyStream -> [Key] +takeChunkKeyStream n (ChunkKeyStream l) = genericTake n l + +-- Number of chunks already consumed from the stream. +numChunks :: ChunkKeyStream -> Integer +numChunks = pred . fromJust . keyChunkNum . fst . nextChunkKeyStream + +{- Splits up the key's content into chunks, passing each chunk to + - the storer action, along with a corresponding chunk key and a + - progress meter update callback. + - + - To support resuming, the checker is used to find the first missing + - chunk key. Storing starts from that chunk. + - + - This buffers each chunk in memory, so can use a lot of memory + - with a large ChunkSize. + - More optimal versions of this can be written, that rely + - on L.toChunks to split the lazy bytestring into chunks (typically + - smaller than the ChunkSize), and eg, write those chunks to a Handle. + - But this is the best that can be done with the storer interface that + - writes a whole L.ByteString at a time. + -} +storeChunks + :: UUID + -> ChunkConfig + -> EncKey + -> Key + -> FilePath + -> MeterUpdate + -> Storer + -> CheckPresent + -> Annex Bool +storeChunks u chunkconfig encryptor k f p storer checker = + case chunkconfig of + (UnpaddedChunks chunksize) | isStableKey k -> + bracketIO open close (go chunksize) + _ -> storer k (FileContent f) p + where + open = tryIO $ openBinaryFile f ReadMode + + close (Right h) = hClose h + close (Left _) = noop + + go _ (Left e) = do + warning (show e) + return False + go chunksize (Right h) = do + let chunkkeys = chunkKeyStream k chunksize + (chunkkeys', startpos) <- seekResume h encryptor chunkkeys checker + b <- liftIO $ L.hGetContents h + gochunks p startpos chunksize b chunkkeys' + + gochunks :: MeterUpdate -> BytesProcessed -> ChunkSize -> L.ByteString -> ChunkKeyStream -> Annex Bool + gochunks meterupdate startpos chunksize = loop startpos . splitchunk + where + splitchunk = L.splitAt chunksize + + loop bytesprocessed (chunk, bs) chunkkeys + | L.null chunk && numchunks > 0 = do + -- Once all chunks are successfully + -- stored, update the chunk log. + chunksStored u k (FixedSizeChunks chunksize) numchunks + return True + | otherwise = do + liftIO $ meterupdate' zeroBytesProcessed + let (chunkkey, chunkkeys') = nextChunkKeyStream chunkkeys + ifM (storer chunkkey (ByteContent chunk) meterupdate') + ( do + let bytesprocessed' = addBytesProcessed bytesprocessed (L.length chunk) + loop bytesprocessed' (splitchunk bs) chunkkeys' + , return False + ) + where + numchunks = numChunks chunkkeys + {- The MeterUpdate that is passed to the action + - storing a chunk is offset, so that it reflects + - the total bytes that have already been stored + - in previous chunks. -} + meterupdate' = offsetMeterUpdate meterupdate bytesprocessed + +{- Check if any of the chunk keys are present. If found, seek forward + - in the Handle, so it will be read starting at the first missing chunk. + - Returns the ChunkKeyStream truncated to start at the first missing + - chunk, and the number of bytes skipped due to resuming. + - + - As an optimisation, if the file fits into a single chunk, there's no need + - to check if that chunk is present -- we know it's not, because otherwise + - the whole file would be present and there would be no reason to try to + - store it. + -} +seekResume + :: Handle + -> EncKey + -> ChunkKeyStream + -> CheckPresent + -> Annex (ChunkKeyStream, BytesProcessed) +seekResume h encryptor chunkkeys checker = do + sz <- liftIO (hFileSize h) + if sz <= fromMaybe 0 (keyChunkSize $ fst $ nextChunkKeyStream chunkkeys) + then return (chunkkeys, zeroBytesProcessed) + else check 0 chunkkeys sz + where + check pos cks sz + | pos >= sz = do + -- All chunks are already stored! + liftIO $ hSeek h AbsoluteSeek sz + return (cks, toBytesProcessed sz) + | otherwise = do + v <- tryNonAsync (checker (encryptor k)) + case v of + Right True -> + check pos' cks' sz + _ -> do + when (pos > 0) $ + liftIO $ hSeek h AbsoluteSeek pos + return (cks, toBytesProcessed pos) + where + (k, cks') = nextChunkKeyStream cks + pos' = pos + fromMaybe 0 (keyChunkSize k) + +{- Removes all chunks of a key from a remote, by calling a remover + - action on each. + - + - The remover action should succeed even if asked to + - remove a key that is not present on the remote. + - + - This action may be called on a chunked key. It will simply remove it. + -} +removeChunks :: (Key -> Annex Bool) -> UUID -> ChunkConfig -> EncKey -> Key -> Annex Bool +removeChunks remover u chunkconfig encryptor k = do + ls <- chunkKeys u chunkconfig k + ok <- allM (remover . encryptor) (concat ls) + when ok $ do + let chunksizes = catMaybes $ map (keyChunkSize <=< headMaybe) ls + forM_ chunksizes $ chunksRemoved u k . FixedSizeChunks . fromIntegral + return ok + +{- Retrieves a key from a remote, using a retriever action. + - + - When the remote is chunked, tries each of the options returned by + - chunkKeys until it finds one where the retriever successfully + - gets the first chunked key. The content of that key, and any + - other chunks in the list is fed to the sink. + - + - If retrival of one of the subsequent chunks throws an exception, + - gives up and returns False. Note that partial data may have been + - written to the sink in this case. + - + - Resuming is supported when using chunks. When the destination file + - already exists, it skips to the next chunked key that would be needed + - to resume. + -} +retrieveChunks + :: Retriever + -> UUID + -> ChunkConfig + -> EncKey + -> Key + -> FilePath + -> MeterUpdate + -> (Maybe Handle -> Maybe MeterUpdate -> ContentSource -> Annex Bool) + -> Annex Bool +retrieveChunks retriever u chunkconfig encryptor basek dest basep sink + | noChunks chunkconfig = + -- Optimisation: Try the unchunked key first, to avoid + -- looking in the git-annex branch for chunk counts + -- that are likely not there. + getunchunked `catchNonAsync` + const (go =<< chunkKeysOnly u basek) + | otherwise = go =<< chunkKeys u chunkconfig basek + where + go ls = do + currsize <- liftIO $ catchMaybeIO $ getFileSize dest + let ls' = maybe ls (setupResume ls) currsize + if any null ls' + then return True -- dest is already complete + else firstavail currsize ls' `catchNonAsync` unable + + unable e = do + warning (show e) + return False + + firstavail _ [] = return False + firstavail currsize ([]:ls) = firstavail currsize ls + firstavail currsize ((k:ks):ls) + | k == basek = getunchunked + `catchNonAsync` (const $ firstavail currsize ls) + | otherwise = do + let offset = resumeOffset currsize k + let p = maybe basep + (offsetMeterUpdate basep . toBytesProcessed) + offset + v <- tryNonAsync $ + retriever (encryptor k) p $ \content -> + bracketIO (maybe opennew openresume offset) hClose $ \h -> do + void $ tosink (Just h) p content + let sz = toBytesProcessed $ + fromMaybe 0 $ keyChunkSize k + getrest p h sz sz ks + `catchNonAsync` unable + case v of + Left e + | null ls -> unable e + | otherwise -> firstavail currsize ls + Right r -> return r + + getrest _ _ _ _ [] = return True + getrest p h sz bytesprocessed (k:ks) = do + let p' = offsetMeterUpdate p bytesprocessed + liftIO $ p' zeroBytesProcessed + ifM (retriever (encryptor k) p' $ tosink (Just h) p') + ( getrest p h sz (addBytesProcessed bytesprocessed sz) ks + , unable "chunk retrieval failed" + ) + + getunchunked = retriever (encryptor basek) basep $ tosink Nothing basep + + opennew = openBinaryFile dest WriteMode + + -- Open the file and seek to the start point in order to resume. + openresume startpoint = do + -- ReadWriteMode allows seeking; AppendMode does not. + h <- openBinaryFile dest ReadWriteMode + hSeek h AbsoluteSeek startpoint + return h + + {- Progress meter updating is a bit tricky: If the Retriever + - populates a file, it is responsible for updating progress + - as the file is being retrieved. + - + - However, if the Retriever generates a lazy ByteString, + - it is not responsible for updating progress (often it cannot). + - Instead, the sink is passed a meter to update as it consumes + - the ByteString. + -} + tosink h p content = sink h p' content + where + p' + | isByteContent content = Just p + | otherwise = Nothing + +{- Can resume when the chunk's offset is at or before the end of + - the dest file. -} +resumeOffset :: Maybe Integer -> Key -> Maybe Integer +resumeOffset Nothing _ = Nothing +resumeOffset currsize k + | offset <= currsize = offset + | otherwise = Nothing + where + offset = chunkKeyOffset k + +{- Drops chunks that are already present in a file, based on its size. + - Keeps any non-chunk keys. + -} +setupResume :: [[Key]] -> Integer -> [[Key]] +setupResume ls currsize = map dropunneeded ls + where + dropunneeded [] = [] + dropunneeded l@(k:_) = case keyChunkSize k of + Just chunksize | chunksize > 0 -> + genericDrop (currsize `div` chunksize) l + _ -> l + +{- Checks if a key is present in a remote. This requires any one + - of the lists of options returned by chunkKeys to all check out + - as being present using the checker action. + - + - Throws an exception if the remote is not accessible. + -} +checkPresentChunks + :: CheckPresent + -> UUID + -> ChunkConfig + -> EncKey + -> Key + -> Annex Bool +checkPresentChunks checker u chunkconfig encryptor basek + | noChunks chunkconfig = do + -- Optimisation: Try the unchunked key first, to avoid + -- looking in the git-annex branch for chunk counts + -- that are likely not there. + v <- check basek + case v of + Right True -> return True + Left e -> checklists (Just e) =<< chunkKeysOnly u basek + _ -> checklists Nothing =<< chunkKeysOnly u basek + | otherwise = checklists Nothing =<< chunkKeys u chunkconfig basek + where + checklists Nothing [] = return False + checklists (Just deferrederror) [] = throwM deferrederror + checklists d (l:ls) + | not (null l) = do + v <- checkchunks l + case v of + Left e -> checklists (Just e) ls + Right True -> return True + Right False -> checklists Nothing ls + | otherwise = checklists d ls + + checkchunks :: [Key] -> Annex (Either SomeException Bool) + checkchunks [] = return (Right True) + checkchunks (k:ks) = do + v <- check k + case v of + Right True -> checkchunks ks + Right False -> return $ Right False + Left e -> return $ Left e + + check = tryNonAsync . checker . encryptor + +{- A key can be stored in a remote unchunked, or as a list of chunked keys. + - This can be the case whether or not the remote is currently configured + - to use chunking. + - + - It's even possible for a remote to have the same key stored multiple + - times with different chunk sizes! + - + - This finds all possible lists of keys that might be on the remote that + - can be combined to get back the requested key, in order from most to + - least likely to exist. + -} +chunkKeys :: UUID -> ChunkConfig -> Key -> Annex [[Key]] +chunkKeys u chunkconfig k = do + l <- chunkKeysOnly u k + return $ if noChunks chunkconfig + then [k] : l + else l ++ [[k]] + +chunkKeysOnly :: UUID -> Key -> Annex [[Key]] +chunkKeysOnly u k = map (toChunkList k) <$> getCurrentChunks u k + +toChunkList :: Key -> (ChunkMethod, ChunkCount) -> [Key] +toChunkList k (FixedSizeChunks chunksize, chunkcount) = + takeChunkKeyStream chunkcount $ chunkKeyStream k chunksize +toChunkList _ (UnknownChunks _, _) = [] diff --git a/Remote/Helper/Chunked/Legacy.hs b/Remote/Helper/Chunked/Legacy.hs new file mode 100644 index 0000000000..290220c2c1 --- /dev/null +++ b/Remote/Helper/Chunked/Legacy.hs @@ -0,0 +1,126 @@ +{- legacy git-annex chunked remotes + - + - Copyright 2012 Joey Hess + - + - Licensed under the GNU GPL version 3 or higher. + -} + +module Remote.Helper.Chunked.Legacy where + +import Annex.Common +import Remote.Helper.Chunked +import Utility.Metered + +import qualified Data.ByteString.Lazy as L +import qualified Control.Exception as E + +{- This is an extension that's added to the usual file (or whatever) + - where the remote stores a key. -} +type ChunkExt = String + +{- A record of the number of chunks used. + - + - While this can be guessed at based on the size of the key, encryption + - makes that larger. Also, using this helps deal with changes to chunksize + - over the life of a remote. + -} +chunkCount :: ChunkExt +chunkCount = ".chunkcount" + +{- An infinite stream of extensions to use for chunks. -} +chunkStream :: [ChunkExt] +chunkStream = map (\n -> ".chunk" ++ show n) [1 :: Integer ..] + +{- Parses the String from the chunkCount file, and returns the files that + - are used to store the chunks. -} +listChunks :: FilePath -> String -> [FilePath] +listChunks basedest chunkcount = take count $ map (basedest ++) chunkStream + where + count = fromMaybe 0 $ readish chunkcount + +{- For use when there is no chunkCount file; uses the action to find + - chunks, and returns them, or Nothing if none found. Relies on + - storeChunks's finalizer atomically moving the chunks into place once all + - are written. + - + - This is only needed to work around a bug that caused the chunkCount file + - not to be written. + -} +probeChunks :: FilePath -> (FilePath -> IO Bool) -> IO [FilePath] +probeChunks basedest check = go [] $ map (basedest ++) chunkStream + where + go l [] = return (reverse l) + go l (c:cs) = ifM (check c) + ( go (c:l) cs + , go l [] + ) + +{- Given the base destination to use to store a value, + - generates a stream of temporary destinations, + - and passes it to an action, which should chunk and store the data, + - and return the destinations it stored to, or [] on error. Then + - calls the recorder to write the chunk count. Finally, the + - finalizer is called to rename the tmp into the dest + - (and do any other cleanup). + -} +storeChunks :: Key -> FilePath -> FilePath -> ([FilePath] -> IO [FilePath]) -> (FilePath -> String -> IO ()) -> (FilePath -> FilePath -> IO ()) -> IO Bool +storeChunks key tmp dest storer recorder finalizer = either onerr return + =<< (E.try go :: IO (Either E.SomeException Bool)) + where + go = do + stored <- storer tmpdests + let chunkcount = basef ++ chunkCount + recorder chunkcount (show $ length stored) + finalizer tmp dest + return (not $ null stored) + onerr e = do + warningIO (show e) + return False + + basef = tmp ++ keyFile key + tmpdests = map (basef ++ ) chunkStream + +{- Given a list of destinations to use, chunks the data according to the + - ChunkSize, and runs the storer action to store each chunk. Returns + - the destinations where data was stored, or [] on error. + - + - This buffers each chunk in memory. + - More optimal versions of this can be written, that rely + - on L.toChunks to split the lazy bytestring into chunks (typically + - smaller than the ChunkSize), and eg, write those chunks to a Handle. + - But this is the best that can be done with the storer interface that + - writes a whole L.ByteString at a time. + -} +storeChunked :: ChunkSize -> [FilePath] -> (FilePath -> L.ByteString -> IO ()) -> L.ByteString -> IO [FilePath] +storeChunked chunksize dests storer content = either onerr return + =<< (E.try (go (Just chunksize) dests) :: IO (Either E.SomeException [FilePath])) + where + go _ [] = return [] -- no dests!? + go Nothing (d:_) = do + storer d content + return [d] + go (Just sz) _ + -- always write a chunk, even if the data is 0 bytes + | L.null content = go Nothing dests + | otherwise = storechunks sz [] dests content + + onerr e = do + warningIO (show e) + return [] + + storechunks _ _ [] _ = return [] -- ran out of dests + storechunks sz useddests (d:ds) b + | L.null b = return $ reverse useddests + | otherwise = do + let (chunk, b') = L.splitAt sz b + storer d chunk + storechunks sz (d:useddests) ds b' + +{- Writes a series of chunks to a file. The feeder is called to get + - each chunk. + -} +meteredWriteFileChunks :: MeterUpdate -> FilePath -> [v] -> (v -> IO L.ByteString) -> IO () +meteredWriteFileChunks meterupdate dest chunks feeder = + withBinaryFile dest WriteMode $ \h -> + forM_ chunks $ + meteredWrite meterupdate h <=< feeder diff --git a/Remote/Helper/Encryptable.hs b/Remote/Helper/Encryptable.hs new file mode 100644 index 0000000000..97e55a4158 --- /dev/null +++ b/Remote/Helper/Encryptable.hs @@ -0,0 +1,208 @@ +{- common functions for encryptable remotes + - + - Copyright 2011 Joey Hess + - + - Licensed under the GNU GPL version 3 or higher. + -} + +module Remote.Helper.Encryptable ( + EncryptionIsSetup, + encryptionSetup, + noEncryptionUsed, + encryptionAlreadySetup, + remoteCipher, + remoteCipher', + embedCreds, + cipherKey, + extractCipher, + isEncrypted, + describeEncryption, +) where + +import qualified Data.Map as M +import qualified "sandi" Codec.Binary.Base64 as B64 +import qualified Data.ByteString as B + +import Annex.Common +import Types.Remote +import Crypto +import Types.Crypto +import qualified Annex +import Utility.FileSystemEncoding + +-- Used to ensure that encryption has been set up before trying to +-- eg, store creds in the remote config that would need to use the +-- encryption setup. +data EncryptionIsSetup = EncryptionIsSetup | NoEncryption + +-- Remotes that don't use encryption can use this instead of +-- encryptionSetup. +noEncryptionUsed :: EncryptionIsSetup +noEncryptionUsed = NoEncryption + +-- Using this avoids the type-safe check, so you'd better be sure +-- of what you're doing. +encryptionAlreadySetup :: EncryptionIsSetup +encryptionAlreadySetup = EncryptionIsSetup + +{- Encryption setup for a remote. The user must specify whether to use + - an encryption key, or not encrypt. An encrypted cipher is created, or is + - updated to be accessible to an additional encryption key. Or the user + - could opt to use a shared cipher, which is stored unencrypted. -} +encryptionSetup :: RemoteConfig -> RemoteGitConfig -> Annex (RemoteConfig, EncryptionIsSetup) +encryptionSetup c gc = do + cmd <- gpgCmd <$> Annex.getGitConfig + maybe (genCipher cmd) (updateCipher cmd) (extractCipher c) + where + -- The type of encryption + encryption = M.lookup "encryption" c + -- Generate a new cipher, depending on the chosen encryption scheme + genCipher cmd = case encryption of + _ | hasEncryptionConfig c -> cannotchange + Just "none" -> return (c, NoEncryption) + Just "shared" -> encsetup $ genSharedCipher cmd + -- hybrid encryption is the default when a keyid is + -- specified but no encryption + _ | maybe (M.member "keyid" c) (== "hybrid") encryption -> + encsetup $ genEncryptedCipher cmd (c, gc) key Hybrid + Just "pubkey" -> encsetup $ genEncryptedCipher cmd (c, gc) key PubKey + Just "sharedpubkey" -> encsetup $ genSharedPubKeyCipher cmd key + _ -> giveup $ "Specify " ++ intercalate " or " + (map ("encryption=" ++) + ["none","shared","hybrid","pubkey", "sharedpubkey"]) + ++ "." + key = fromMaybe (giveup "Specify keyid=...") $ M.lookup "keyid" c + newkeys = maybe [] (\k -> [(True,k)]) (M.lookup "keyid+" c) ++ + maybe [] (\k -> [(False,k)]) (M.lookup "keyid-" c) + cannotchange = giveup "Cannot set encryption type of existing remotes." + -- Update an existing cipher if possible. + updateCipher cmd v = case v of + SharedCipher _ | maybe True (== "shared") encryption -> return (c', EncryptionIsSetup) + EncryptedCipher _ variant _ + | maybe True (== if variant == Hybrid then "hybrid" else "pubkey") encryption -> do + use "encryption update" $ updateCipherKeyIds cmd (c, gc) newkeys v + SharedPubKeyCipher _ _ -> + use "encryption update" $ updateCipherKeyIds cmd (c, gc) newkeys v + _ -> cannotchange + encsetup a = use "encryption setup" . a =<< highRandomQuality + use m a = do + showNote m + cipher <- liftIO a + showNote (describeCipher cipher) + return (storeCipher cipher c', EncryptionIsSetup) + highRandomQuality = + (&&) (maybe True ( /= "false") $ M.lookup "highRandomQuality" c) + <$> fmap not (Annex.getState Annex.fast) + c' = foldr M.delete c + -- git-annex used to remove 'encryption' as well, since + -- it was redundant; we now need to keep it for + -- public-key encryption, hence we leave it on newer + -- remotes (while being backward-compatible). + [ "keyid", "keyid+", "keyid-", "highRandomQuality" ] + +remoteCipher :: RemoteConfig -> RemoteGitConfig -> Annex (Maybe Cipher) +remoteCipher c gc = fmap fst <$> remoteCipher' c gc + +{- Gets encryption Cipher. The decrypted Ciphers are cached in the Annex + - state. -} +remoteCipher' :: RemoteConfig -> RemoteGitConfig -> Annex (Maybe (Cipher, StorableCipher)) +remoteCipher' c gc = go $ extractCipher c + where + go Nothing = return Nothing + go (Just encipher) = do + cache <- Annex.getState Annex.ciphers + case M.lookup encipher cache of + Just cipher -> return $ Just (cipher, encipher) + Nothing -> do + cmd <- gpgCmd <$> Annex.getGitConfig + cipher <- liftIO $ decryptCipher cmd (c, gc) encipher + Annex.changeState (\s -> s { Annex.ciphers = M.insert encipher cipher cache }) + return $ Just (cipher, encipher) + +{- Checks if the remote's config allows storing creds in the remote's config. + - + - embedcreds=yes allows this, and embedcreds=no prevents it. + - + - If not set, the default is to only store creds when it's surely safe: + - When gpg encryption is used and the creds are encrypted using it. + - Not when a shared cipher is used. + -} +embedCreds :: RemoteConfig -> Bool +embedCreds c + | M.lookup "embedcreds" c == Just "yes" = True + | M.lookup "embedcreds" c == Just "no" = False + | isJust (M.lookup "cipherkeys" c) && isJust (M.lookup "cipher" c) = True + | otherwise = False + +{- Gets encryption Cipher, and key encryptor. -} +cipherKey :: RemoteConfig -> RemoteGitConfig -> Annex (Maybe (Cipher, EncKey)) +cipherKey c gc = fmap make <$> remoteCipher c gc + where + make ciphertext = (ciphertext, encryptKey mac ciphertext) + mac = fromMaybe defaultMac $ M.lookup "mac" c >>= readMac + +{- Stores an StorableCipher in a remote's configuration. -} +storeCipher :: StorableCipher -> RemoteConfig -> RemoteConfig +storeCipher cip = case cip of + (SharedCipher t) -> addcipher t + (EncryptedCipher t _ ks) -> addcipher t . storekeys ks "cipherkeys" + (SharedPubKeyCipher t ks) -> addcipher t . storekeys ks "pubkeys" + where + addcipher t = M.insert "cipher" (toB64bs t) + storekeys (KeyIds l) n = M.insert n (intercalate "," l) + +{- Extracts an StorableCipher from a remote's configuration. -} +extractCipher :: RemoteConfig -> Maybe StorableCipher +extractCipher c = case (M.lookup "cipher" c, + M.lookup "cipherkeys" c <|> M.lookup "pubkeys" c, + M.lookup "encryption" c) of + (Just t, Just ks, encryption) | maybe True (== "hybrid") encryption -> + Just $ EncryptedCipher (fromB64bs t) Hybrid (readkeys ks) + (Just t, Just ks, Just "pubkey") -> + Just $ EncryptedCipher (fromB64bs t) PubKey (readkeys ks) + (Just t, Just ks, Just "sharedpubkey") -> + Just $ SharedPubKeyCipher (fromB64bs t) (readkeys ks) + (Just t, Nothing, encryption) | maybe True (== "shared") encryption -> + Just $ SharedCipher (fromB64bs t) + _ -> Nothing + where + readkeys = KeyIds . splitc ',' + +isEncrypted :: RemoteConfig -> Bool +isEncrypted c = case M.lookup "encryption" c of + Just "none" -> False + Just _ -> True + Nothing -> hasEncryptionConfig c + +hasEncryptionConfig :: RemoteConfig -> Bool +hasEncryptionConfig c = M.member "cipher" c || M.member "cipherkeys" c || M.member "pubkeys" c + +describeEncryption :: RemoteConfig -> String +describeEncryption c = case extractCipher c of + Nothing -> "none" + Just cip -> nameCipher cip ++ " (" ++ describeCipher cip ++ ")" + +nameCipher :: StorableCipher -> String +nameCipher (SharedCipher _) = "shared" +nameCipher (EncryptedCipher _ PubKey _) = "pubkey" +nameCipher (EncryptedCipher _ Hybrid _) = "hybrid" +nameCipher (SharedPubKeyCipher _ _) = "sharedpubkey" + +describeCipher :: StorableCipher -> String +describeCipher c = case c of + (SharedCipher _) -> "encryption key stored in git repository" + (EncryptedCipher _ _ ks) -> showkeys ks + (SharedPubKeyCipher _ ks) -> showkeys ks + where + showkeys (KeyIds { keyIds = ks }) = "to gpg keys: " ++ unwords ks + +{- Not using Utility.Base64 because these "Strings" are really + - bags of bytes and that would convert to unicode and not round-trip + - cleanly. -} +toB64bs :: String -> String +toB64bs = w82s . B.unpack . B64.encode . B.pack . s2w8 + +fromB64bs :: String -> String +fromB64bs s = either (const bad) (w82s . B.unpack) (B64.decode $ B.pack $ s2w8 s) + where + bad = error "bad base64 encoded data" diff --git a/Remote/Helper/Export.hs b/Remote/Helper/Export.hs new file mode 100644 index 0000000000..0dabf5ff99 --- /dev/null +++ b/Remote/Helper/Export.hs @@ -0,0 +1,178 @@ +{- exports to remotes + - + - Copyright 2017 Joey Hess + - + - Licensed under the GNU GPL version 3 or higher. + -} + +{-# LANGUAGE FlexibleInstances #-} + +module Remote.Helper.Export where + +import Annex.Common +import Types.Remote +import Types.Backend +import Types.Key +import Backend +import Remote.Helper.Encryptable (isEncrypted) +import Database.Export +import Annex.Export + +import qualified Data.Map as M +import Control.Concurrent.STM + +-- | Use for remotes that do not support exports. +class HasExportUnsupported a where + exportUnsupported :: a + +instance HasExportUnsupported (RemoteConfig -> RemoteGitConfig -> Annex Bool) where + exportUnsupported = \_ _ -> return False + +instance HasExportUnsupported (Annex (ExportActions Annex)) where + exportUnsupported = return $ ExportActions + { storeExport = \_ _ _ _ -> do + warning "store export is unsupported" + return False + , retrieveExport = \_ _ _ _ -> return False + , checkPresentExport = \_ _ -> return False + , removeExport = \_ _ -> return False + , removeExportDirectory = Just $ \_ -> return False + , renameExport = \_ _ _ -> return False + } + +exportIsSupported :: RemoteConfig -> RemoteGitConfig -> Annex Bool +exportIsSupported = \_ _ -> return True + +-- | Prevent or allow exporttree=yes when setting up a new remote, +-- depending on exportSupported and other configuration. +adjustExportableRemoteType :: RemoteType -> RemoteType +adjustExportableRemoteType rt = rt { setup = setup' } + where + setup' st mu cp c gc = do + let cont = setup rt st mu cp c gc + ifM (exportSupported rt c gc) + ( case st of + Init + | exportTree c && isEncrypted c -> + giveup "cannot enable both encryption and exporttree" + | otherwise -> cont + Enable oldc + | exportTree c /= exportTree oldc -> + giveup "cannot change exporttree of existing special remote" + | otherwise -> cont + , if exportTree c + then giveup "exporttree=yes is not supported by this special remote" + else cont + ) + +-- | If the remote is exportSupported, and exporttree=yes, adjust the +-- remote to be an export. +adjustExportable :: Remote -> Annex Remote +adjustExportable r = case M.lookup "exporttree" (config r) of + Just "yes" -> ifM (isExportSupported r) + ( isexport + , notexport + ) + Nothing -> notexport + Just "no" -> notexport + Just _ -> error "bad exporttree value" + where + notexport = return $ r + { exportActions = exportUnsupported + , remotetype = (remotetype r) + { exportSupported = exportUnsupported + } + } + isexport = do + db <- openDb (uuid r) + + updateflag <- liftIO newEmptyTMVarIO + let updateonce = liftIO $ atomically $ + ifM (isEmptyTMVar updateflag) + ( do + putTMVar updateflag () + return True + , return False + ) + + -- Get export locations for a key. Checks once + -- if the export log is different than the database and + -- updates the database, to notice when an export has been + -- updated from another repository. + let getexportlocs = \k -> do + whenM updateonce $ + updateExportTreeFromLog db + liftIO $ getExportTree db k + + return $ r + -- Storing a key on an export could be implemented, + -- but it would perform unncessary work + -- when another repository has already stored the + -- key, and the local repository does not know + -- about it. To avoid unnecessary costs, don't do it. + { storeKey = \_ _ _ -> do + warning "remote is configured with exporttree=yes; use `git-annex export` to store content on it" + return False + -- Keys can be retrieved using retrieveExport, + -- but since that retrieves from a path in the + -- remote that another writer could have replaced + -- with content not of the requested key, + -- the content has to be strongly verified. + -- + -- But, appendonly remotes have a key/value store, + -- so don't need to use retrieveExport. + , retrieveKeyFile = if appendonly r + then retrieveKeyFile r + else retrieveKeyFileFromExport getexportlocs + , retrieveKeyFileCheap = if appendonly r + then retrieveKeyFileCheap r + else \_ _ _ -> return False + -- Removing a key from an export would need to + -- change the tree in the export log to not include + -- the file. Otherwise, conflicts when removing + -- files would not be dealt with correctly. + -- There does not seem to be a good use case for + -- removing a key from an export in any case. + , removeKey = \_k -> do + warning "dropping content from an export is not supported; use `git annex export` to export a tree that lacks the files you want to remove" + return False + -- Can't lock content on exports, since they're + -- not key/value stores, and someone else could + -- change what's exported to a file at any time. + -- + -- (except for appendonly remotes) + , lockContent = if appendonly r + then lockContent r + else Nothing + -- Check if any of the files a key was exported to + -- are present. This doesn't guarantee the export + -- contains the right content, which is why export + -- remotes are untrusted. + -- + -- (but appendonly remotes work the same as any + -- non-export remote) + , checkPresent = if appendonly r + then checkPresent r + else \k -> do + ea <- exportActions r + anyM (checkPresentExport ea k) + =<< getexportlocs k + , mkUnavailable = return Nothing + , getInfo = do + is <- getInfo r + return (is++[("export", "yes")]) + } + retrieveKeyFileFromExport getexportlocs k _af dest p = unVerified $ + if maybe False (isJust . verifyKeyContent) (maybeLookupBackendVariety (keyVariety k)) + then do + locs <- getexportlocs k + case locs of + [] -> do + warning "unknown export location" + return False + (l:_) -> do + ea <- exportActions r + retrieveExport ea k l dest p + else do + warning $ "exported content cannot be verified due to using the " ++ formatKeyVariety (keyVariety k) ++ " backend" + return False diff --git a/Remote/Helper/Git.hs b/Remote/Helper/Git.hs new file mode 100644 index 0000000000..12348f7a52 --- /dev/null +++ b/Remote/Helper/Git.hs @@ -0,0 +1,49 @@ +{- Utilities for git remotes. + - + - Copyright 2011-2014 Joey Hess + - + - Licensed under the GNU GPL version 3 or higher. + -} + +module Remote.Helper.Git where + +import Annex.Common +import qualified Git +import Types.Availability +import qualified Types.Remote as Remote + +import Data.Time.Clock.POSIX + +repoCheap :: Git.Repo -> Bool +repoCheap = not . Git.repoIsUrl + +localpathCalc :: Git.Repo -> Maybe FilePath +localpathCalc r + | availabilityCalc r == GloballyAvailable = Nothing + | otherwise = Just $ Git.repoPath r + +availabilityCalc :: Git.Repo -> Availability +availabilityCalc r + | (Git.repoIsLocal r || Git.repoIsLocalUnknown r) = LocallyAvailable + | otherwise = GloballyAvailable + +{- Avoids performing an action on a local repository that's not usable. + - Does not check that the repository is still available on disk. -} +guardUsable :: Git.Repo -> Annex a -> Annex a -> Annex a +guardUsable r fallback a + | Git.repoIsLocalUnknown r = fallback + | otherwise = a + +gitRepoInfo :: Remote -> Annex [(String, String)] +gitRepoInfo r = do + d <- fromRepo Git.localGitDir + mtimes <- liftIO $ mapM (modificationTime <$$> getFileStatus) + =<< dirContentsRecursive (d "refs" "remotes" Remote.name r) + let lastsynctime = case mtimes of + [] -> "never" + _ -> show $ posixSecondsToUTCTime $ realToFrac $ maximum mtimes + repo <- Remote.getRepo r + return + [ ("repository location", Git.repoLocation repo) + , ("last synced", lastsynctime) + ] diff --git a/Remote/Helper/Hooks.hs b/Remote/Helper/Hooks.hs new file mode 100644 index 0000000000..a8e7c9ad3f --- /dev/null +++ b/Remote/Helper/Hooks.hs @@ -0,0 +1,94 @@ +{- Adds hooks to remotes. + - + - Copyright 2012 Joey Hess + - + - Licensed under the GNU GPL version 3 or higher. + -} + +{-# LANGUAGE CPP #-} + +module Remote.Helper.Hooks (addHooks) where + +import qualified Data.Map as M + +import Annex.Common +import Types.Remote +import Types.CleanupActions +import qualified Annex +import Annex.LockFile +import Annex.LockPool +#ifndef mingw32_HOST_OS +import Annex.Perms +#endif + +{- Modifies a remote's access functions to first run the + - annex-start-command hook, and trigger annex-stop-command on shutdown. + - This way, the hooks are only run when a remote is actively being used. + -} +addHooks :: Remote -> Remote +addHooks r = addHooks' r + (remoteAnnexStartCommand $ gitconfig r) + (remoteAnnexStopCommand $ gitconfig r) +addHooks' :: Remote -> Maybe String -> Maybe String -> Remote +addHooks' r Nothing Nothing = r +addHooks' r starthook stophook = r' + where + r' = r + { storeKey = \k f p -> wrapper $ storeKey r k f p + , retrieveKeyFile = \k f d p -> wrapper $ retrieveKeyFile r k f d p + , retrieveKeyFileCheap = \k af f -> wrapper $ retrieveKeyFileCheap r k af f + , removeKey = wrapper . removeKey r + , checkPresent = wrapper . checkPresent r + } + where + wrapper = runHooks r' starthook stophook + +runHooks :: Remote -> Maybe String -> Maybe String -> Annex a -> Annex a +runHooks r starthook stophook a = do + dir <- fromRepo gitAnnexRemotesDir + let lck = dir remoteid ++ ".lck" + whenM (notElem lck . M.keys <$> getLockCache) $ do + liftIO $ createDirectoryIfMissing True dir + firstrun lck + a + where + remoteid = show (uuid r) + run Nothing = noop + run (Just command) = void $ liftIO $ + boolSystem "sh" [Param "-c", Param command] + firstrun lck = do + -- Take a shared lock; This indicates that git-annex + -- is using the remote, and prevents other instances + -- of it from running the stophook. If another + -- instance is shutting down right now, this + -- will block waiting for its exclusive lock to clear. + lockFileCached lck + + -- The starthook is run even if some other git-annex + -- is already running, and ran it before. + -- It would be difficult to use locking to ensure + -- it's only run once, and it's also possible for + -- git-annex to be interrupted before it can run the + -- stophook, in which case the starthook + -- would be run again by the next git-annex. + -- So, requiring idempotency is the right approach. + run starthook + + Annex.addCleanup (StopHook $ uuid r) $ runstop lck + runstop lck = do + -- Drop any shared lock we have, and take an + -- exclusive lock, without blocking. If the lock + -- succeeds, we're the only process using this remote, + -- so can stop it. + unlockFile lck +#ifndef mingw32_HOST_OS + mode <- annexFileMode + v <- noUmask mode $ tryLockExclusive (Just mode) lck +#else + v <- liftIO $ lockExclusive lck +#endif + case v of + Nothing -> noop + Just lockhandle -> do + run stophook + liftIO $ dropLock lockhandle diff --git a/Remote/Helper/Http.hs b/Remote/Helper/Http.hs new file mode 100644 index 0000000000..ff7d2a9b00 --- /dev/null +++ b/Remote/Helper/Http.hs @@ -0,0 +1,85 @@ +{- helpers for remotes using http + - + - Copyright 2014 Joey Hess + - + - Licensed under the GNU GPL version 3 or higher. + -} + +{-# LANGUAGE BangPatterns #-} + +module Remote.Helper.Http where + +import Annex.Common +import Types.StoreRetrieve +import Remote.Helper.Special +import Utility.Metered + +import qualified Data.ByteString.Lazy as L +import qualified Data.ByteString as S +import Control.Concurrent +import Network.HTTP.Client (RequestBody(..), Response, responseStatus, responseBody, BodyReader, NeedsPopper) +import Network.HTTP.Types + +-- A storer that expects to be provided with a http RequestBody containing +-- the content to store. +-- +-- Implemented as a fileStorer, so that the content can be streamed +-- from the file in constant space. +httpStorer :: (Key -> RequestBody -> Annex Bool) -> Storer +httpStorer a = fileStorer $ \k f m -> a k =<< liftIO (httpBodyStorer f m) + +-- Reads the file and generates a streaming request body, that will update +-- the meter as it's sent. +httpBodyStorer :: FilePath -> MeterUpdate -> IO RequestBody +httpBodyStorer src m = do + size <- getFileSize src + let streamer sink = withMeteredFile src m $ \b -> byteStringPopper b sink + return $ RequestBodyStream (fromInteger size) streamer + +byteStringPopper :: L.ByteString -> NeedsPopper () -> IO () +byteStringPopper b sink = do + mvar <- newMVar $ L.toChunks b + let getnextchunk = modifyMVar mvar $ \v -> + case v of + [] -> return ([], S.empty) + (c:cs) -> return (cs, c) + sink getnextchunk + +{- Makes a Popper that streams a given number of chunks of a given + - size from the handle, updating the meter as the chunks are read. -} +handlePopper :: Integer -> Int -> MeterUpdate -> Handle -> NeedsPopper () -> IO () +handlePopper numchunks chunksize meterupdate h sink = do + mvar <- newMVar zeroBytesProcessed + let getnextchunk = do + sent <- takeMVar mvar + if sent >= target + then do + putMVar mvar sent + return S.empty + else do + b <- S.hGet h chunksize + let !sent' = addBytesProcessed sent chunksize + putMVar mvar sent' + meterupdate sent' + return b + sink getnextchunk + where + target = toBytesProcessed (numchunks * fromIntegral chunksize) + +-- Reads the http body and stores it to the specified file, updating the +-- meter as it goes. +httpBodyRetriever :: FilePath -> MeterUpdate -> Response BodyReader -> IO () +httpBodyRetriever dest meterupdate resp + | responseStatus resp /= ok200 = giveup $ show $ responseStatus resp + | otherwise = bracket (openBinaryFile dest WriteMode) hClose (go zeroBytesProcessed) + where + reader = responseBody resp + go sofar h = do + b <- reader + if S.null b + then return () + else do + let sofar' = addBytesProcessed sofar $ S.length b + S.hPut h b + meterupdate sofar' + go sofar' h diff --git a/Remote/Helper/Messages.hs b/Remote/Helper/Messages.hs new file mode 100644 index 0000000000..0148257760 --- /dev/null +++ b/Remote/Helper/Messages.hs @@ -0,0 +1,35 @@ +{- git-annex remote messages + - + - Copyright 2013 Joey Hess + - + - Licensed under the GNU GPL version 3 or higher. + -} + +{-# LANGUAGE TypeSynonymInstances, FlexibleInstances #-} + +module Remote.Helper.Messages where + +import Annex.Common +import qualified Git +import qualified Types.Remote as Remote + +class Describable a where + describe :: a -> String + +instance Describable Git.Repo where + describe = Git.repoDescribe + +instance Describable (Remote.RemoteA a) where + describe = Remote.name + +instance Describable String where + describe = id + +showChecking :: Describable a => a -> Annex () +showChecking v = showAction $ "checking " ++ describe v + +cantCheck :: Describable a => a -> e +cantCheck v = giveup $ "unable to check " ++ describe v + +showLocking :: Describable a => a -> Annex () +showLocking v = showAction $ "locking " ++ describe v diff --git a/Remote/Helper/P2P.hs b/Remote/Helper/P2P.hs new file mode 100644 index 0000000000..a953df1b70 --- /dev/null +++ b/Remote/Helper/P2P.hs @@ -0,0 +1,67 @@ +{- Helpers for remotes using the git-annex P2P protocol. + - + - Copyright 2016-2018 Joey Hess + - + - Licensed under the GNU GPL version 3 or higher. + -} + +{-# LANGUAGE RankNTypes #-} + +module Remote.Helper.P2P where + +import Annex.Common +import qualified P2P.Protocol as P2P +import P2P.IO +import Types.Remote +import Annex.Content +import Messages.Progress +import Utility.Metered +import Types.NumCopies + +import Control.Concurrent + +-- Runs a Proto action using a connection it sets up. +type ProtoRunner a = P2P.Proto a -> Annex (Maybe a) + +-- Runs a Proto action using a ClosableConnection. +type ProtoConnRunner c = forall a. P2P.Proto a -> ClosableConnection c -> Annex (ClosableConnection c, Maybe a) + +-- Runs an Annex action with a connection from the pool, adding it back to +-- the pool when done. +type WithConn a c = (ClosableConnection c -> Annex (ClosableConnection c, a)) -> Annex a + +store :: (MeterUpdate -> ProtoRunner Bool) -> Key -> AssociatedFile -> MeterUpdate -> Annex Bool +store runner k af p = do + let getsrcfile = fmap fst <$> prepSendAnnex k + metered (Just p) k getsrcfile $ \_ p' -> + fromMaybe False + <$> runner p' (P2P.put k af p') + +retrieve :: (MeterUpdate -> ProtoRunner (Bool, Verification)) -> Key -> AssociatedFile -> FilePath -> MeterUpdate -> Annex (Bool, Verification) +retrieve runner k af dest p = + metered (Just p) k (return Nothing) $ \m p' -> + fromMaybe (False, UnVerified) + <$> runner p' (P2P.get dest k af m p') + +remove :: ProtoRunner Bool -> Key -> Annex Bool +remove runner k = fromMaybe False <$> runner (P2P.remove k) + +checkpresent :: ProtoRunner Bool -> Key -> Annex Bool +checkpresent runner k = maybe unavail return =<< runner (P2P.checkPresent k) + where + unavail = giveup "can't connect to remote" + +lock :: WithConn a c -> ProtoConnRunner c -> UUID -> Key -> (VerifiedCopy -> Annex a) -> Annex a +lock withconn connrunner u k callback = withconn $ \conn -> do + connv <- liftIO $ newMVar conn + let runproto d p = do + c <- liftIO $ takeMVar connv + (c', mr) <- connrunner p c + liftIO $ putMVar connv c' + return (fromMaybe d mr) + r <- P2P.lockContentWhile runproto k go + conn' <- liftIO $ takeMVar connv + return (conn', r) + where + go False = giveup "can't lock content" + go True = withVerifiedCopy LockedCopy u (return True) callback diff --git a/Remote/Helper/ReadOnly.hs b/Remote/Helper/ReadOnly.hs new file mode 100644 index 0000000000..2b6d7467e6 --- /dev/null +++ b/Remote/Helper/ReadOnly.hs @@ -0,0 +1,46 @@ +{- Adds readonly support to remotes. + - + - Copyright 2013, 2015 Joey Hess + - + - Licensed under the GNU GPL version 3 or higher. + -} + +module Remote.Helper.ReadOnly + ( adjustReadOnly + , readonlyStoreKey + , readonlyStorer + , readonlyRemoveKey + ) where + +import Annex.Common +import Types.Remote +import Types.StoreRetrieve +import Utility.Metered + +{- Adds support for read-only remotes, by replacing the + - methods that write to a remote with dummies that fail. + - + - Note that disabling git pushes to remotes is not handled here. + -} +adjustReadOnly :: Remote -> Remote +adjustReadOnly r + | remoteAnnexReadOnly (gitconfig r) = r + { storeKey = readonlyStoreKey + , removeKey = readonlyRemoveKey + , repairRepo = Nothing + } + | otherwise = r + +readonlyStoreKey :: Key -> AssociatedFile -> MeterUpdate -> Annex Bool +readonlyStoreKey _ _ _ = readonlyFail + +readonlyRemoveKey :: Key -> Annex Bool +readonlyRemoveKey _ = readonlyFail + +readonlyStorer :: Storer +readonlyStorer _ _ _ = readonlyFail + +readonlyFail :: Annex Bool +readonlyFail = do + warning "this remote is readonly" + return False diff --git a/Remote/Helper/Special.hs b/Remote/Helper/Special.hs new file mode 100644 index 0000000000..00fc19f3b9 --- /dev/null +++ b/Remote/Helper/Special.hs @@ -0,0 +1,303 @@ +{- helpers for special remotes + - + - Copyright 2011-2018 Joey Hess + - + - Licensed under the GNU GPL version 3 or higher. + -} + +module Remote.Helper.Special ( + findSpecialRemotes, + gitConfigSpecialRemote, + mkRetrievalVerifiableKeysSecure, + Preparer, + Storer, + Retriever, + Remover, + CheckPresent, + simplyPrepare, + ContentSource, + checkPrepare, + resourcePrepare, + fileStorer, + byteStorer, + fileRetriever, + byteRetriever, + storeKeyDummy, + retreiveKeyFileDummy, + removeKeyDummy, + checkPresentDummy, + SpecialRemoteCfg(..), + specialRemoteCfg, + specialRemote, + specialRemote', + module X +) where + +import Annex.Common +import qualified Annex +import Types.StoreRetrieve +import Types.Remote +import Crypto +import Config +import Config.Cost +import Utility.Metered +import Remote.Helper.Chunked as X +import Remote.Helper.Encryptable as X +import Remote.Helper.Messages +import Annex.Content +import Messages.Progress +import qualified Git +import qualified Git.Construct + +import qualified Data.ByteString.Lazy as L +import qualified Data.Map as M + +{- Special remotes don't have a configured url, so Git.Repo does not + - automatically generate remotes for them. This looks for a different + - configuration key instead. + -} +findSpecialRemotes :: String -> Annex [Git.Repo] +findSpecialRemotes s = do + m <- fromRepo Git.config + liftIO $ mapM construct $ remotepairs m + where + remotepairs = M.toList . M.filterWithKey match + construct (k,_) = Git.Construct.remoteNamedFromKey k (pure Git.Construct.fromUnknown) + match k _ = "remote." `isPrefixOf` k && (".annex-"++s) `isSuffixOf` k + +{- Sets up configuration for a special remote in .git/config. -} +gitConfigSpecialRemote :: UUID -> RemoteConfig -> [(String, String)] -> Annex () +gitConfigSpecialRemote u c cfgs = do + forM_ cfgs $ \(k, v) -> + setConfig (remoteConfig remotename k) v + setConfig (remoteConfig remotename "uuid") (fromUUID u) + where + remotename = fromJust (M.lookup "name" c) + +-- RetrievalVerifiableKeysSecure unless overridden by git config. +-- +-- Only looks at the RemoteGitConfig; the GitConfig's setting is +-- checked at the same place the RetrievalSecurityPolicy is checked. +mkRetrievalVerifiableKeysSecure :: RemoteGitConfig -> RetrievalSecurityPolicy +mkRetrievalVerifiableKeysSecure gc + | remoteAnnexAllowUnverifiedDownloads gc = RetrievalAllKeysSecure + | otherwise = RetrievalVerifiableKeysSecure + +-- Use when nothing needs to be done to prepare a helper. +simplyPrepare :: helper -> Preparer helper +simplyPrepare helper _ a = a $ Just helper + +-- Use to run a check when preparing a helper. +checkPrepare :: (Key -> Annex Bool) -> helper -> Preparer helper +checkPrepare checker helper k a = ifM (checker k) + ( a (Just helper) + , a Nothing + ) + +-- Use to acquire a resource when preparing a helper. +resourcePrepare :: (Key -> (r -> Annex Bool) -> Annex Bool) -> (r -> helper) -> Preparer helper +resourcePrepare withr helper k a = withr k $ \r -> + a (Just (helper r)) + +-- A Storer that expects to be provided with a file containing +-- the content of the key to store. +fileStorer :: (Key -> FilePath -> MeterUpdate -> Annex Bool) -> Storer +fileStorer a k (FileContent f) m = a k f m +fileStorer a k (ByteContent b) m = withTmp k $ \f -> do + liftIO $ L.writeFile f b + a k f m + +-- A Storer that expects to be provided with a L.ByteString of +-- the content to store. +byteStorer :: (Key -> L.ByteString -> MeterUpdate -> Annex Bool) -> Storer +byteStorer a k c m = withBytes c $ \b -> a k b m + +-- A Retriever that writes the content of a Key to a provided file. +-- It is responsible for updating the progress meter as it retrieves data. +fileRetriever :: (FilePath -> Key -> MeterUpdate -> Annex ()) -> Retriever +fileRetriever a k m callback = do + f <- prepTmp k + a f k m + pruneTmpWorkDirBefore f (callback . FileContent) + +-- A Retriever that generates a lazy ByteString containing the Key's +-- content, and passes it to a callback action which will fully consume it +-- before returning. +byteRetriever :: (Key -> (L.ByteString -> Annex Bool) -> Annex Bool) -> Retriever +byteRetriever a k _m callback = a k (callback . ByteContent) + +{- The base Remote that is provided to specialRemote needs to have + - storeKey, retrieveKeyFile, removeKey, and checkPresent methods, + - but they are never actually used (since specialRemote replaces them). + - Here are some dummy ones. + -} +storeKeyDummy :: Key -> AssociatedFile -> MeterUpdate -> Annex Bool +storeKeyDummy _ _ _ = return False +retreiveKeyFileDummy :: Key -> AssociatedFile -> FilePath -> MeterUpdate -> Annex (Bool, Verification) +retreiveKeyFileDummy _ _ _ _ = unVerified (return False) +removeKeyDummy :: Key -> Annex Bool +removeKeyDummy _ = return False +checkPresentDummy :: Key -> Annex Bool +checkPresentDummy _ = error "missing checkPresent implementation" + +type RemoteModifier + = RemoteConfig + -> Preparer Storer + -> Preparer Retriever + -> Preparer Remover + -> Preparer CheckPresent + -> Remote + -> Remote + +data SpecialRemoteCfg = SpecialRemoteCfg + { chunkConfig :: ChunkConfig + , displayProgress :: Bool + } + +specialRemoteCfg :: RemoteConfig -> SpecialRemoteCfg +specialRemoteCfg c = SpecialRemoteCfg (getChunkConfig c) True + +-- Modifies a base Remote to support both chunking and encryption, +-- which special remotes typically should support. +specialRemote :: RemoteModifier +specialRemote c = specialRemote' (specialRemoteCfg c) c + +specialRemote' :: SpecialRemoteCfg -> RemoteModifier +specialRemote' cfg c preparestorer prepareretriever prepareremover preparecheckpresent baser = encr + where + encr = baser + { storeKey = \k _f p -> cip >>= storeKeyGen k p + , retrieveKeyFile = \k _f d p -> cip >>= unVerified . retrieveKeyFileGen k d p + , retrieveKeyFileCheap = \k f d -> cip >>= maybe + (retrieveKeyFileCheap baser k f d) + -- retrieval of encrypted keys is never cheap + (\_ -> return False) + -- When encryption is used, the remote could provide + -- some other content encrypted by the user, and trick + -- git-annex into decrypting it, leaking the decryption + -- into the git-annex repository. Verifiable keys + -- are the main protection against this attack. + , retrievalSecurityPolicy = if isencrypted + then mkRetrievalVerifiableKeysSecure (gitconfig baser) + else retrievalSecurityPolicy baser + , removeKey = \k -> cip >>= removeKeyGen k + , checkPresent = \k -> cip >>= checkPresentGen k + , cost = if isencrypted + then cost baser + encryptedRemoteCostAdj + else cost baser + , getInfo = do + l <- getInfo baser + return $ l ++ + [ ("encryption", describeEncryption c) + , ("chunking", describeChunkConfig (chunkConfig cfg)) + ] + , whereisKey = if noChunks (chunkConfig cfg) && not isencrypted + then whereisKey baser + else Nothing + } + cip = cipherKey c (gitconfig baser) + isencrypted = isJust (extractCipher c) + + safely a = catchNonAsync a (\e -> warning (show e) >> return False) + + -- chunk, then encrypt, then feed to the storer + storeKeyGen k p enc = safely $ preparestorer k $ safely . go + where + go (Just storer) = preparecheckpresent k $ safely . go' storer + go Nothing = return False + go' storer (Just checker) = sendAnnex k rollback $ \src -> + displayprogress p k (Just src) $ \p' -> + storeChunks (uuid baser) chunkconfig enck k src p' + (storechunk enc storer) + checker + go' _ Nothing = return False + rollback = void $ removeKey encr k + enck = maybe id snd enc + + storechunk Nothing storer k content p = storer k content p + storechunk (Just (cipher, enck)) storer k content p = do + cmd <- gpgCmd <$> Annex.getGitConfig + withBytes content $ \b -> + encrypt cmd encr cipher (feedBytes b) $ + readBytes $ \encb -> + storer (enck k) (ByteContent encb) p + + -- call retriever to get chunks; decrypt them; stream to dest file + retrieveKeyFileGen k dest p enc = + safely $ prepareretriever k $ safely . go + where + go (Just retriever) = displayprogress p k Nothing $ \p' -> + retrieveChunks retriever (uuid baser) chunkconfig + enck k dest p' (sink dest enc encr) + go Nothing = return False + enck = maybe id snd enc + + removeKeyGen k enc = safely $ prepareremover k $ safely . go + where + go (Just remover) = removeChunks remover (uuid baser) chunkconfig enck k + go Nothing = return False + enck = maybe id snd enc + + checkPresentGen k enc = preparecheckpresent k go + where + go (Just checker) = checkPresentChunks checker (uuid baser) chunkconfig enck k + go Nothing = cantCheck baser + enck = maybe id snd enc + + chunkconfig = chunkConfig cfg + + displayprogress p k srcfile a + | displayProgress cfg = metered (Just p) k (return srcfile) (const a) + | otherwise = a p + +{- Sink callback for retrieveChunks. Stores the file content into the + - provided Handle, decrypting it first if necessary. + - + - If the remote did not store the content using chunks, no Handle + - will be provided, and it's up to us to open the destination file. + - + - Note that when neither chunking nor encryption is used, and the remote + - provides FileContent, that file only needs to be renamed + - into place. (And it may even already be in the right place..) + -} +sink + :: LensGpgEncParams c + => FilePath + -> Maybe (Cipher, EncKey) + -> c + -> Maybe Handle + -> Maybe MeterUpdate + -> ContentSource + -> Annex Bool +sink dest enc c mh mp content = do + case (enc, mh, content) of + (Nothing, Nothing, FileContent f) + | f == dest -> noop + | otherwise -> liftIO $ moveFile f dest + (Just (cipher, _), _, ByteContent b) -> do + cmd <- gpgCmd <$> Annex.getGitConfig + decrypt cmd c cipher (feedBytes b) $ + readBytes write + (Just (cipher, _), _, FileContent f) -> do + cmd <- gpgCmd <$> Annex.getGitConfig + withBytes content $ \b -> + decrypt cmd c cipher (feedBytes b) $ + readBytes write + liftIO $ nukeFile f + (Nothing, _, FileContent f) -> do + withBytes content write + liftIO $ nukeFile f + (Nothing, _, ByteContent b) -> write b + return True + where + write b = case mh of + Just h -> liftIO $ b `streamto` h + Nothing -> liftIO $ bracket opendest hClose (b `streamto`) + streamto b h = case mp of + Just p -> meteredWrite p h b + Nothing -> L.hPut h b + opendest = openBinaryFile dest WriteMode + +withBytes :: ContentSource -> (L.ByteString -> Annex a) -> Annex a +withBytes (ByteContent b) a = a b +withBytes (FileContent f) a = a =<< liftIO (L.readFile f) diff --git a/Remote/Helper/Ssh.hs b/Remote/Helper/Ssh.hs new file mode 100644 index 0000000000..8419cc190d --- /dev/null +++ b/Remote/Helper/Ssh.hs @@ -0,0 +1,370 @@ +{- git-annex remote access with ssh and git-annex-shell + - + - Copyright 2011-2018 Joey Hess + - + - Licensed under the GNU AGPL version 3 or higher. + -} + +module Remote.Helper.Ssh where + +import Annex.Common +import qualified Annex +import qualified Git +import qualified Git.Url +import Annex.UUID +import Annex.Ssh +import CmdLine.GitAnnexShell.Fields (Field, fieldName) +import qualified CmdLine.GitAnnexShell.Fields as Fields +import Remote.Helper.Messages +import Utility.Metered +import Utility.Rsync +import Utility.SshHost +import Types.Remote +import Types.Transfer +import Config +import qualified P2P.Protocol as P2P +import qualified P2P.IO as P2P +import qualified P2P.Annex as P2P + +import Control.Concurrent.STM +import Control.Concurrent.Async +import qualified Data.ByteString as B + +toRepo :: ConsumeStdin -> Git.Repo -> RemoteGitConfig -> SshCommand -> Annex (FilePath, [CommandParam]) +toRepo cs r gc remotecmd = do + let host = maybe + (giveup "bad ssh url") + (either error id . mkSshHost) + (Git.Url.hostuser r) + sshCommand cs (host, Git.Url.port r) gc remotecmd + +{- Generates parameters to run a git-annex-shell command on a remote + - repository. -} +git_annex_shell :: ConsumeStdin -> Git.Repo -> String -> [CommandParam] -> [(Field, String)] -> Annex (Maybe (FilePath, [CommandParam])) +git_annex_shell cs r command params fields + | not $ Git.repoIsUrl r = do + shellopts <- getshellopts + return $ Just (shellcmd, shellopts ++ fieldopts) + | Git.repoIsSsh r = do + gc <- Annex.getRemoteGitConfig r + u <- getRepoUUID r + shellopts <- getshellopts + let sshcmd = unwords $ + fromMaybe shellcmd (remoteAnnexShell gc) + : map shellEscape (toCommand shellopts) ++ + uuidcheck u ++ + map shellEscape (toCommand fieldopts) + Just <$> toRepo cs r gc sshcmd + | otherwise = return Nothing + where + dir = Git.repoPath r + shellcmd = "git-annex-shell" + getshellopts = do + debug <- liftIO debugEnabled + let params' = if debug + then Param "--debug" : params + else params + return (Param command : File dir : params') + uuidcheck NoUUID = [] + uuidcheck (UUID u) = ["--uuid", u] + fieldopts + | null fields = [] + | otherwise = fieldsep : map fieldopt fields ++ [fieldsep] + fieldsep = Param "--" + fieldopt (field, value) = Param $ + fieldName field ++ "=" ++ value + +{- Uses a supplied function (such as boolSystem) to run a git-annex-shell + - command on a remote. + - + - Or, if the remote does not support running remote commands, returns + - a specified error value. -} +onRemote + :: ConsumeStdin + -> Git.Repo + -> (FilePath -> [CommandParam] -> IO a, Annex a) + -> String + -> [CommandParam] + -> [(Field, String)] + -> Annex a +onRemote cs r (with, errorval) command params fields = do + s <- git_annex_shell cs r command params fields + case s of + Just (c, ps) -> liftIO $ with c ps + Nothing -> errorval + +{- Checks if a remote contains a key. -} +inAnnex :: Git.Repo -> Key -> Annex Bool +inAnnex r k = do + showChecking r + onRemote NoConsumeStdin r (runcheck, cantCheck r) "inannex" [Param $ key2file k] [] + where + runcheck c p = dispatch =<< safeSystem c p + dispatch ExitSuccess = return True + dispatch (ExitFailure 1) = return False + dispatch _ = cantCheck r + +{- Removes a key from a remote. -} +dropKey :: Git.Repo -> Key -> Annex Bool +dropKey r key = onRemote NoConsumeStdin r (boolSystem, return False) "dropkey" + [ Param "--quiet", Param "--force" + , Param $ key2file key + ] + [] + +rsyncHelper :: OutputHandler -> Maybe MeterUpdate -> [CommandParam] -> Annex Bool +rsyncHelper oh m params = do + unless (quietMode oh) $ + showOutput -- make way for progress bar + a <- case m of + Nothing -> return $ rsync params + Just meter -> return $ rsyncProgress oh meter params + ifM (liftIO a) + ( return True + , do + showLongNote "rsync failed -- run git annex again to resume file transfer" + return False + ) + +{- Generates rsync parameters that ssh to the remote and asks it + - to either receive or send the key's content. -} +rsyncParamsRemote :: Bool -> Remote -> Direction -> Key -> FilePath -> AssociatedFile -> Annex [CommandParam] +rsyncParamsRemote unlocked r direction key file (AssociatedFile afile) = do + u <- getUUID + let fields = (Fields.remoteUUID, fromUUID u) + : (Fields.unlocked, if unlocked then "1" else "") + -- Send direct field for unlocked content, for backwards + -- compatability. + : (Fields.direct, if unlocked then "1" else "") + : maybe [] (\f -> [(Fields.associatedFile, f)]) afile + repo <- getRepo r + Just (shellcmd, shellparams) <- git_annex_shell ConsumeStdin repo + (if direction == Download then "sendkey" else "recvkey") + [ Param $ key2file key ] + fields + -- Convert the ssh command into rsync command line. + let eparam = rsyncShell (Param shellcmd:shellparams) + o <- rsyncParams r direction + return $ if direction == Download + then o ++ rsyncopts eparam dummy (File file) + else o ++ rsyncopts eparam (File file) dummy + where + rsyncopts ps source dest + | end ps == [dashdash] = ps ++ [source, dest] + | otherwise = ps ++ [dashdash, source, dest] + dashdash = Param "--" + {- The rsync shell parameter controls where rsync + - goes, so the source/dest parameter can be a dummy value, + - that just enables remote rsync mode. + - For maximum compatability with some patched rsyncs, + - the dummy value needs to still contain a hostname, + - even though this hostname will never be used. -} + dummy = Param "dummy:" + +-- --inplace to resume partial files +-- +-- Only use --perms when not on a crippled file system, as rsync +-- will fail trying to restore file perms onto a filesystem that does not +-- support them. +rsyncParams :: Remote -> Direction -> Annex [CommandParam] +rsyncParams r direction = do + crippled <- crippledFileSystem + return $ map Param $ catMaybes + [ Just "--progress" + , Just "--inplace" + , if crippled then Nothing else Just "--perms" + ] + ++ remoteAnnexRsyncOptions gc ++ dps + where + dps + | direction == Download = remoteAnnexRsyncDownloadOptions gc + | otherwise = remoteAnnexRsyncUploadOptions gc + gc = gitconfig r + +-- Used by git-annex-shell lockcontent to indicate the content is +-- successfully locked. +contentLockedMarker :: String +contentLockedMarker = "OK" + +-- A connection over ssh to git-annex shell speaking the P2P protocol. +type P2PSshConnection = P2P.ClosableConnection + (P2P.RunState, P2P.P2PConnection, ProcessHandle, TVar StderrHandlerState) + +data StderrHandlerState = DiscardStderr | DisplayStderr | EndStderrHandler + +closeP2PSshConnection :: P2PSshConnection -> IO (P2PSshConnection, Maybe ExitCode) +closeP2PSshConnection P2P.ClosedConnection = return (P2P.ClosedConnection, Nothing) +closeP2PSshConnection (P2P.OpenConnection (_st, conn, pid, stderrhandlerst)) = do + P2P.closeConnection conn + atomically $ writeTVar stderrhandlerst EndStderrHandler + exitcode <- waitForProcess pid + return (P2P.ClosedConnection, Just exitcode) + +-- Pool of connections over ssh to git-annex-shell p2pstdio. +type P2PSshConnectionPool = TVar (Maybe P2PSshConnectionPoolState) + +data P2PSshConnectionPoolState + = P2PSshConnections [P2PSshConnection] + -- Remotes using an old version of git-annex-shell don't support P2P + | P2PSshUnsupported + +mkP2PSshConnectionPool :: Annex P2PSshConnectionPool +mkP2PSshConnectionPool = liftIO $ newTVarIO Nothing + +-- Takes a connection from the pool, if any are available, otherwise +-- tries to open a new one. +getP2PSshConnection :: Remote -> P2PSshConnectionPool -> Annex (Maybe P2PSshConnection) +getP2PSshConnection r connpool = getexistingconn >>= \case + Nothing -> return Nothing + Just Nothing -> openP2PSshConnection r connpool + Just (Just c) -> return (Just c) + where + getexistingconn = liftIO $ atomically $ readTVar connpool >>= \case + Just P2PSshUnsupported -> return Nothing + Just (P2PSshConnections (c:cs)) -> do + writeTVar connpool (Just (P2PSshConnections cs)) + return (Just (Just c)) + Just (P2PSshConnections []) -> return (Just Nothing) + Nothing -> return (Just Nothing) + +-- Add a connection to the pool, unless it's closed. +storeP2PSshConnection :: P2PSshConnectionPool -> P2PSshConnection -> IO () +storeP2PSshConnection _ P2P.ClosedConnection = return () +storeP2PSshConnection connpool conn = atomically $ modifyTVar' connpool $ \case + Just (P2PSshConnections cs) -> Just (P2PSshConnections (conn:cs)) + _ -> Just (P2PSshConnections [conn]) + +-- Try to open a P2PSshConnection. +-- The new connection is not added to the pool, so it's available +-- for the caller to use. +-- If the remote does not support the P2P protocol, that's remembered in +-- the connection pool. +openP2PSshConnection :: Remote -> P2PSshConnectionPool -> Annex (Maybe P2PSshConnection) +openP2PSshConnection r connpool = do + u <- getUUID + let ps = [Param (fromUUID u)] + repo <- getRepo r + git_annex_shell ConsumeStdin repo "p2pstdio" ps [] >>= \case + Nothing -> do + liftIO $ rememberunsupported + return Nothing + Just (cmd, params) -> start cmd params =<< getRepo r + where + start cmd params repo = liftIO $ do + (Just from, Just to, Just err, pid) <- createProcess $ + (proc cmd (toCommand params)) + { std_in = CreatePipe + , std_out = CreatePipe + , std_err = CreatePipe + } + let conn = P2P.P2PConnection + { P2P.connRepo = repo + , P2P.connCheckAuth = const False + , P2P.connIhdl = to + , P2P.connOhdl = from + } + stderrhandlerst <- newStderrHandler err + runst <- P2P.mkRunState P2P.Client + let c = P2P.OpenConnection (runst, conn, pid, stderrhandlerst) + -- When the connection is successful, the remote + -- will send an AUTH_SUCCESS with its uuid. + let proto = P2P.postAuth $ + P2P.negotiateProtocolVersion P2P.maxProtocolVersion + tryNonAsync (P2P.runNetProto runst conn proto) >>= \case + Right (Right (Just theiruuid)) | theiruuid == uuid r -> do + atomically $ + writeTVar stderrhandlerst DisplayStderr + return $ Just c + _ -> do + (cclosed, exitcode) <- closeP2PSshConnection c + -- ssh exits 255 when unable to connect to + -- server. Return a closed connection in + -- this case, to avoid the fallback action + -- being run instead, which would mean a + -- second connection attempt to this server + -- that is down. + if exitcode == Just (ExitFailure 255) + then return (Just cclosed) + else do + rememberunsupported + return Nothing + rememberunsupported = atomically $ + modifyTVar' connpool $ + maybe (Just P2PSshUnsupported) Just + +newStderrHandler :: Handle -> IO (TVar StderrHandlerState) +newStderrHandler errh = do + -- stderr from git-annex-shell p2pstdio is initially discarded + -- because old versions don't support the command. Once it's known + -- to be running, this is changed to DisplayStderr. + v <- newTVarIO DiscardStderr + p <- async $ go v + void $ async $ ender p v + return v + where + go v = do + l <- B.hGetLine errh + atomically (readTVar v) >>= \case + DiscardStderr -> go v + DisplayStderr -> do + B.hPut stderr l + go v + EndStderrHandler -> return () + + ender p v = do + atomically $ do + readTVar v >>= \case + EndStderrHandler -> return () + _ -> retry + hClose errh + cancel p + +-- Runs a P2P Proto action on a remote when it supports that, +-- otherwise the fallback action. +runProto :: Remote -> P2PSshConnectionPool -> Annex a -> Annex a -> P2P.Proto a -> Annex (Maybe a) +runProto r connpool badproto fallback proto = Just <$> + (getP2PSshConnection r connpool >>= maybe fallback go) + where + go c = do + (c', v) <- runProtoConn proto c + case v of + Just res -> do + liftIO $ storeP2PSshConnection connpool c' + return res + -- Running the proto failed, either due to a protocol + -- error or a network error. + Nothing -> badproto + +runProtoConn :: P2P.Proto a -> P2PSshConnection -> Annex (P2PSshConnection, Maybe a) +runProtoConn _ P2P.ClosedConnection = return (P2P.ClosedConnection, Nothing) +runProtoConn a conn@(P2P.OpenConnection (runst, c, _, _)) = do + P2P.runFullProto runst c a >>= \case + Right r -> return (conn, Just r) + -- When runFullProto fails, the connection is no longer + -- usable, so close it. + Left e -> do + warning $ "Lost connection (" ++ P2P.describeProtoFailure e ++ ")" + conn' <- fst <$> liftIO (closeP2PSshConnection conn) + return (conn', Nothing) + +-- Allocates a P2P ssh connection from the pool, and runs the action with it, +-- returning the connection to the pool once the action is done. +-- +-- If the remote does not support the P2P protocol, runs the fallback +-- action instead. +withP2PSshConnection + :: Remote + -> P2PSshConnectionPool + -> Annex a + -> (P2PSshConnection -> Annex (P2PSshConnection, a)) + -> Annex a +withP2PSshConnection r connpool fallback a = bracketOnError get cache go + where + get = getP2PSshConnection r connpool + cache (Just conn) = liftIO $ storeP2PSshConnection connpool conn + cache Nothing = return () + go (Just conn) = do + (conn', res) <- a conn + cache (Just conn') + return res + go Nothing = fallback diff --git a/Remote/Hook.hs b/Remote/Hook.hs new file mode 100644 index 0000000000..565b5c038d --- /dev/null +++ b/Remote/Hook.hs @@ -0,0 +1,158 @@ +{- A remote that provides hooks to run shell commands. + - + - Copyright 2011 Joey Hess + - + - Licensed under the GNU GPL version 3 or higher. + -} + +module Remote.Hook (remote) where + +import Annex.Common +import Types.Remote +import Types.Creds +import qualified Git +import Config +import Config.Cost +import Annex.UUID +import Remote.Helper.Special +import Remote.Helper.Messages +import Remote.Helper.Export +import Utility.Env +import Messages.Progress + +import qualified Data.Map as M + +type Action = String +type HookName = String + +remote :: RemoteType +remote = RemoteType + { typename = "hook" + , enumerate = const (findSpecialRemotes "hooktype") + , generate = gen + , setup = hookSetup + , exportSupported = exportUnsupported + } + +gen :: Git.Repo -> UUID -> RemoteConfig -> RemoteGitConfig -> Annex (Maybe Remote) +gen r u c gc = do + cst <- remoteCost gc expensiveRemoteCost + return $ Just $ specialRemote c + (simplyPrepare $ store hooktype) + (simplyPrepare $ retrieve hooktype) + (simplyPrepare $ remove hooktype) + (simplyPrepare $ checkKey r hooktype) + Remote + { uuid = u + , cost = cst + , name = Git.repoDescribe r + , storeKey = storeKeyDummy + , retrieveKeyFile = retreiveKeyFileDummy + , retrieveKeyFileCheap = retrieveCheap hooktype + -- A hook could use http and be vulnerable to + -- redirect to file:// attacks, etc. + , retrievalSecurityPolicy = mkRetrievalVerifiableKeysSecure gc + , removeKey = removeKeyDummy + , lockContent = Nothing + , checkPresent = checkPresentDummy + , checkPresentCheap = False + , exportActions = exportUnsupported + , whereisKey = Nothing + , remoteFsck = Nothing + , repairRepo = Nothing + , config = c + , localpath = Nothing + , getRepo = return r + , gitconfig = gc + , readonly = False + , appendonly = False + , availability = GloballyAvailable + , remotetype = remote + , mkUnavailable = gen r u c $ + gc { remoteAnnexHookType = Just "!dne!" } + , getInfo = return [("hooktype", hooktype)] + , claimUrl = Nothing + , checkUrl = Nothing + } + where + hooktype = fromMaybe (giveup "missing hooktype") $ remoteAnnexHookType gc + +hookSetup :: SetupStage -> Maybe UUID -> Maybe CredPair -> RemoteConfig -> RemoteGitConfig -> Annex (RemoteConfig, UUID) +hookSetup _ mu _ c gc = do + u <- maybe (liftIO genUUID) return mu + let hooktype = fromMaybe (giveup "Specify hooktype=") $ + M.lookup "hooktype" c + (c', _encsetup) <- encryptionSetup c gc + gitConfigSpecialRemote u c' [("hooktype", hooktype)] + return (c', u) + +hookEnv :: Action -> Key -> Maybe FilePath -> IO (Maybe [(String, String)]) +hookEnv action k f = Just <$> mergeenv (fileenv f ++ keyenv) + where + mergeenv l = addEntries l <$> getEnvironment + envvar s v = ("ANNEX_" ++ s, v) + keyenv = catMaybes + [ Just $ envvar "KEY" (key2file k) + , Just $ envvar "ACTION" action + , envvar "HASH_1" <$> headMaybe hashbits + , envvar "HASH_2" <$> headMaybe (drop 1 hashbits) + ] + fileenv Nothing = [] + fileenv (Just file) = [envvar "FILE" file] + hashbits = map takeDirectory $ splitPath $ hashDirMixed def k + +lookupHook :: HookName -> Action -> Annex (Maybe String) +lookupHook hookname action = do + command <- getConfig (annexConfig hook) "" + if null command + then do + fallback <- getConfig (annexConfig hookfallback) "" + if null fallback + then do + warning $ "missing configuration for " ++ hook ++ " or " ++ hookfallback + return Nothing + else return $ Just fallback + else return $ Just command + where + hook = hookname ++ "-" ++ action ++ "-hook" + hookfallback = hookname ++ "-hook" + +runHook :: HookName -> Action -> Key -> Maybe FilePath -> Annex Bool -> Annex Bool +runHook hook action k f a = maybe (return False) run =<< lookupHook hook action + where + run command = do + showOutput -- make way for hook output + ifM (progressCommandEnv "sh" [Param "-c", Param command] =<< liftIO (hookEnv action k f)) + ( a + , do + warning $ hook ++ " hook exited nonzero!" + return False + ) + +store :: HookName -> Storer +store h = fileStorer $ \k src _p -> + runHook h "store" k (Just src) $ return True + +retrieve :: HookName -> Retriever +retrieve h = fileRetriever $ \d k _p -> + unlessM (runHook h "retrieve" k (Just d) $ return True) $ + giveup "failed to retrieve content" + +retrieveCheap :: HookName -> Key -> AssociatedFile -> FilePath -> Annex Bool +retrieveCheap _ _ _ _ = return False + +remove :: HookName -> Remover +remove h k = runHook h "remove" k Nothing $ return True + +checkKey :: Git.Repo -> HookName -> CheckPresent +checkKey r h k = do + showChecking r + v <- lookupHook h action + liftIO $ check v + where + action = "checkpresent" + findkey s = key2file k `elem` lines s + check Nothing = giveup $ action ++ " hook misconfigured" + check (Just hook) = do + environ <- hookEnv action k Nothing + findkey <$> readProcessEnv "sh" ["-c", hook] environ diff --git a/Remote/List.hs b/Remote/List.hs new file mode 100644 index 0000000000..7c21aa8696 --- /dev/null +++ b/Remote/List.hs @@ -0,0 +1,125 @@ +{-# LANGUAGE CPP #-} + +{- git-annex remote list + - + - Copyright 2011,2012 Joey Hess + - + - Licensed under the GNU GPL version 3 or higher. + -} + +module Remote.List where + +import qualified Data.Map as M + +import Annex.Common +import qualified Annex +import Logs.Remote +import Types.Remote +import Annex.UUID +import Remote.Helper.Hooks +import Remote.Helper.ReadOnly +import Remote.Helper.Export +import qualified Git +import qualified Git.Config + +import qualified Remote.Git +import qualified Remote.GCrypt +import qualified Remote.P2P +#ifdef WITH_S3 +import qualified Remote.S3 +#endif +import qualified Remote.Bup +import qualified Remote.Directory +import qualified Remote.Rsync +import qualified Remote.Web +import qualified Remote.BitTorrent +#ifdef WITH_WEBDAV +import qualified Remote.WebDAV +#endif +import qualified Remote.Adb +import qualified Remote.Tahoe +import qualified Remote.Glacier +import qualified Remote.Ddar +import qualified Remote.Hook +import qualified Remote.External + +remoteTypes :: [RemoteType] +remoteTypes = map adjustExportableRemoteType + [ Remote.Git.remote + , Remote.GCrypt.remote + , Remote.P2P.remote +#ifdef WITH_S3 + , Remote.S3.remote +#endif + , Remote.Bup.remote + , Remote.Directory.remote + , Remote.Rsync.remote + , Remote.Web.remote + , Remote.BitTorrent.remote +#ifdef WITH_WEBDAV + , Remote.WebDAV.remote +#endif + , Remote.Adb.remote + , Remote.Tahoe.remote + , Remote.Glacier.remote + , Remote.Ddar.remote + , Remote.Hook.remote + , Remote.External.remote + ] + +{- Builds a list of all available Remotes. + - Since doing so can be expensive, the list is cached. -} +remoteList :: Annex [Remote] +remoteList = do + rs <- Annex.getState Annex.remotes + if null rs + then remoteList' False + else return rs + +remoteList' :: Bool -> Annex [Remote] +remoteList' autoinit = do + m <- readRemoteLog + rs <- concat <$> mapM (process m) remoteTypes + Annex.changeState $ \s -> s { Annex.remotes = rs } + return rs + where + process m t = enumerate t autoinit + >>= mapM (remoteGen m t) + >>= return . catMaybes + +{- Forces the remoteList to be re-generated, re-reading the git config. -} +remoteListRefresh :: Annex [Remote] +remoteListRefresh = do + newg <- inRepo Git.Config.reRead + Annex.changeState $ \s -> s + { Annex.remotes = [] + , Annex.repo = newg + } + remoteList + +{- Generates a Remote. -} +remoteGen :: M.Map UUID RemoteConfig -> RemoteType -> Git.Repo -> Annex (Maybe Remote) +remoteGen m t r = do + u <- getRepoUUID r + gc <- Annex.getRemoteGitConfig r + let c = fromMaybe M.empty $ M.lookup u m + generate t r u c gc >>= maybe + (return Nothing) + (Just <$$> adjustExportable . adjustReadOnly . addHooks) + +{- Updates a local git Remote, re-reading its git config. -} +updateRemote :: Remote -> Annex (Maybe Remote) +updateRemote remote = do + m <- readRemoteLog + remote' <- updaterepo =<< getRepo remote + remoteGen m (remotetype remote) remote' + where + updaterepo r + | Git.repoIsLocal r || Git.repoIsLocalUnknown r = + Remote.Git.configRead False r + | otherwise = return r + +{- Checks if a remote is syncable using git. -} +gitSyncableRemote :: Remote -> Bool +gitSyncableRemote r = remotetype r `elem` + [ Remote.Git.remote, Remote.GCrypt.remote, Remote.P2P.remote ] diff --git a/Remote/P2P.hs b/Remote/P2P.hs new file mode 100644 index 0000000000..d3106a8480 --- /dev/null +++ b/Remote/P2P.hs @@ -0,0 +1,170 @@ +{- git remotes using the git-annex P2P protocol + - + - Copyright 2016-2018 Joey Hess + - + - Licensed under the GNU GPL version 3 or higher. + -} + +module Remote.P2P ( + remote, + chainGen +) where + +import Annex.Common +import qualified Annex +import qualified P2P.Protocol as P2P +import P2P.Address +import P2P.Annex +import P2P.IO +import P2P.Auth +import Types.Remote +import qualified Git +import Annex.UUID +import Config +import Config.Cost +import Remote.Helper.Git +import Remote.Helper.Export +import Remote.Helper.P2P +import Utility.AuthToken + +import Control.Concurrent.STM + +remote :: RemoteType +remote = RemoteType + { typename = "p2p" + -- Remote.Git takes care of enumerating P2P remotes, + -- and will call chainGen on them. + , enumerate = const (return []) + , generate = \_ _ _ _ -> return Nothing + , setup = error "P2P remotes are set up using git-annex p2p" + , exportSupported = exportUnsupported + } + +chainGen :: P2PAddress -> Git.Repo -> UUID -> RemoteConfig -> RemoteGitConfig -> Annex (Maybe Remote) +chainGen addr r u c gc = do + connpool <- mkConnectionPool + cst <- remoteCost gc veryExpensiveRemoteCost + let protorunner = runProto u addr connpool + let withconn = withConnection u addr connpool + let this = Remote + { uuid = u + , cost = cst + , name = Git.repoDescribe r + , storeKey = store (const protorunner) + , retrieveKeyFile = retrieve (const protorunner) + , retrieveKeyFileCheap = \_ _ _ -> return False + , retrievalSecurityPolicy = RetrievalAllKeysSecure + , removeKey = remove protorunner + , lockContent = Just $ lock withconn runProtoConn u + , checkPresent = checkpresent protorunner + , checkPresentCheap = False + , exportActions = exportUnsupported + , whereisKey = Nothing + , remoteFsck = Nothing + , repairRepo = Nothing + , config = c + , localpath = Nothing + , getRepo = return r + , gitconfig = gc + , readonly = False + , appendonly = False + , availability = GloballyAvailable + , remotetype = remote + , mkUnavailable = return Nothing + , getInfo = gitRepoInfo this + , claimUrl = Nothing + , checkUrl = Nothing + } + return (Just this) + +-- | A connection to the peer, which can be closed. +type Connection = ClosableConnection (RunState, P2PConnection) + +type ConnectionPool = TVar [Connection] + +mkConnectionPool :: Annex ConnectionPool +mkConnectionPool = liftIO $ newTVarIO [] + +-- Runs the Proto action. +runProto :: UUID -> P2PAddress -> ConnectionPool -> P2P.Proto a -> Annex (Maybe a) +runProto u addr connpool a = withConnection u addr connpool (runProtoConn a) + +runProtoConn :: P2P.Proto a -> Connection -> Annex (Connection, Maybe a) +runProtoConn _ ClosedConnection = return (ClosedConnection, Nothing) +runProtoConn a c@(OpenConnection (runst, conn)) = do + v <- runFullProto runst conn a + -- When runFullProto fails, the connection is no longer usable, + -- so close it. + case v of + Left e -> do + warning $ "Lost connection to peer (" ++ describeProtoFailure e ++ ")" + liftIO $ closeConnection conn + return (ClosedConnection, Nothing) + Right r -> return (c, Just r) + +-- Uses an open connection if one is available in the ConnectionPool; +-- otherwise opens a new connection. +-- +-- Once the action is done, the connection is added back to the +-- ConnectionPool, unless it's no longer open. +withConnection :: UUID -> P2PAddress -> ConnectionPool -> (Connection -> Annex (Connection, a)) -> Annex a +withConnection u addr connpool a = bracketOnError get cache go + where + get = do + mc <- liftIO $ atomically $ do + l <- readTVar connpool + case l of + [] -> do + writeTVar connpool [] + return Nothing + (c:cs) -> do + writeTVar connpool cs + return (Just c) + maybe (openConnection u addr) return mc + + cache ClosedConnection = return () + cache conn = liftIO $ atomically $ modifyTVar' connpool (conn:) + + go conn = do + (conn', r) <- a conn + cache conn' + return r + +openConnection :: UUID -> P2PAddress -> Annex Connection +openConnection u addr = do + g <- Annex.gitRepo + v <- liftIO $ tryNonAsync $ connectPeer g addr + case v of + Right conn -> do + myuuid <- getUUID + authtoken <- fromMaybe nullAuthToken + <$> loadP2PRemoteAuthToken addr + let proto = P2P.auth myuuid authtoken $ + -- Before 6.20180312, the protocol server + -- had a bug that made negotiating the + -- protocol version terminate the + -- connection. So, this must stay disabled + -- until the old version is not in use + -- anywhere. + --P2P.negotiateProtocolVersion P2P.maxProtocolVersion + return () + runst <- liftIO $ mkRunState Client + res <- liftIO $ runNetProto runst conn proto + case res of + Right (Just theiruuid) + | u == theiruuid -> return (OpenConnection (runst, conn)) + | otherwise -> do + liftIO $ closeConnection conn + warning "Remote peer uuid seems to have changed." + return ClosedConnection + Right Nothing -> do + warning "Unable to authenticate with peer." + liftIO $ closeConnection conn + return ClosedConnection + Left e -> do + warning $ "Problem communicating with peer. (" ++ describeProtoFailure e ++ ")" + liftIO $ closeConnection conn + return ClosedConnection + Left e -> do + warning $ "Unable to connect to peer. (" ++ show e ++ ")" + return ClosedConnection diff --git a/Remote/Rsync.hs b/Remote/Rsync.hs new file mode 100644 index 0000000000..216fd0f7e4 --- /dev/null +++ b/Remote/Rsync.hs @@ -0,0 +1,351 @@ +{- A remote that is only accessible by rsync. + - + - Copyright 2011-2018 Joey Hess + - + - Licensed under the GNU GPL version 3 or higher. + -} + +{-# LANGUAGE CPP #-} + +module Remote.Rsync ( + remote, + store, + retrieve, + remove, + checkKey, + withRsyncScratchDir, + genRsyncOpts, + RsyncOpts +) where + +import Annex.Common +import Types.Remote +import qualified Git +import Config +import Config.Cost +import Annex.Content +import Annex.UUID +import Annex.Ssh +import Remote.Helper.Special +import Remote.Helper.Messages +import Remote.Helper.Export +import Types.Export +import Remote.Rsync.RsyncUrl +import Crypto +import Utility.Rsync +import Utility.CopyFile +import Messages.Progress +import Utility.Metered +import Types.Transfer +import Types.Creds +import Annex.DirHashes +import Utility.Tmp.Dir +import Utility.SshHost + +import qualified Data.Map as M + +remote :: RemoteType +remote = RemoteType + { typename = "rsync" + , enumerate = const (findSpecialRemotes "rsyncurl") + , generate = gen + , setup = rsyncSetup + , exportSupported = exportIsSupported + } + +gen :: Git.Repo -> UUID -> RemoteConfig -> RemoteGitConfig -> Annex (Maybe Remote) +gen r u c gc = do + cst <- remoteCost gc expensiveRemoteCost + (transport, url) <- rsyncTransport gc $ + fromMaybe (giveup "missing rsyncurl") $ remoteAnnexRsyncUrl gc + let o = genRsyncOpts c gc transport url + let islocal = rsyncUrlIsPath $ rsyncUrl o + return $ Just $ specialRemote' specialcfg c + (simplyPrepare $ fileStorer $ store o) + (simplyPrepare $ fileRetriever $ retrieve o) + (simplyPrepare $ remove o) + (simplyPrepare $ checkKey r o) + Remote + { uuid = u + , cost = cst + , name = Git.repoDescribe r + , storeKey = storeKeyDummy + , retrieveKeyFile = retreiveKeyFileDummy + , retrieveKeyFileCheap = retrieveCheap o + , retrievalSecurityPolicy = RetrievalAllKeysSecure + , removeKey = removeKeyDummy + , lockContent = Nothing + , checkPresent = checkPresentDummy + , checkPresentCheap = False + , exportActions = return $ ExportActions + { storeExport = storeExportM o + , retrieveExport = retrieveExportM o + , removeExport = removeExportM o + , checkPresentExport = checkPresentExportM o + , removeExportDirectory = Just (removeExportDirectoryM o) + , renameExport = renameExportM o + } + , whereisKey = Nothing + , remoteFsck = Nothing + , repairRepo = Nothing + , config = c + , getRepo = return r + , gitconfig = gc + , localpath = if islocal + then Just $ rsyncUrl o + else Nothing + , readonly = False + , appendonly = False + , availability = if islocal then LocallyAvailable else GloballyAvailable + , remotetype = remote + , mkUnavailable = return Nothing + , getInfo = return [("url", url)] + , claimUrl = Nothing + , checkUrl = Nothing + } + where + specialcfg = (specialRemoteCfg c) + -- Rsync displays its own progress. + { displayProgress = False } + +genRsyncOpts :: RemoteConfig -> RemoteGitConfig -> [CommandParam] -> RsyncUrl -> RsyncOpts +genRsyncOpts c gc transport url = RsyncOpts + { rsyncUrl = url + , rsyncOptions = transport ++ opts [] + , rsyncUploadOptions = transport ++ opts (remoteAnnexRsyncUploadOptions gc) + , rsyncDownloadOptions = transport ++ opts (remoteAnnexRsyncDownloadOptions gc) + , rsyncShellEscape = M.lookup "shellescape" c /= Just "no" + } + where + opts specificopts = map Param $ filter safe $ + remoteAnnexRsyncOptions gc ++ specificopts + safe opt + -- Don't allow user to pass --delete to rsync; + -- that could cause it to delete other keys + -- in the same hash bucket as a key it sends. + | opt == "--delete" = False + | opt == "--delete-excluded" = False + | otherwise = True + +rsyncTransport :: RemoteGitConfig -> RsyncUrl -> Annex ([CommandParam], RsyncUrl) +rsyncTransport gc url + | rsyncUrlIsShell url = + (\rsh -> return (rsyncShell rsh, url)) =<< + case fromNull ["ssh"] (remoteAnnexRsyncTransport gc) of + "ssh":sshopts -> do + let (port, sshopts') = sshReadPort sshopts + userhost = either error id $ mkSshHost $ + takeWhile (/= ':') url + (Param "ssh":) <$> sshOptions ConsumeStdin + (userhost, port) gc + (map Param $ loginopt ++ sshopts') + "rsh":rshopts -> return $ map Param $ "rsh" : + loginopt ++ rshopts + rsh -> giveup $ "Unknown Rsync transport: " + ++ unwords rsh + | otherwise = return ([], url) + where + login = case separate (=='@') url of + (_h, "") -> Nothing + (l, _) -> Just l + loginopt = maybe [] (\l -> ["-l",l]) login + fromNull as xs = if null xs then as else xs + +rsyncSetup :: SetupStage -> Maybe UUID -> Maybe CredPair -> RemoteConfig -> RemoteGitConfig -> Annex (RemoteConfig, UUID) +rsyncSetup _ mu _ c gc = do + u <- maybe (liftIO genUUID) return mu + -- verify configuration is sane + let url = fromMaybe (giveup "Specify rsyncurl=") $ + M.lookup "rsyncurl" c + (c', _encsetup) <- encryptionSetup c gc + + -- The rsyncurl is stored in git config, not only in this remote's + -- persistant state, so it can vary between hosts. + gitConfigSpecialRemote u c' [("rsyncurl", url)] + return (c', u) + +{- To send a single key is slightly tricky; need to build up a temporary + - directory structure to pass to rsync so it can create the hash + - directories. + - + - This would not be necessary if the hash directory structure used locally + - was always the same as that used on the rsync remote. So if that's ever + - unified, this gets nicer. + - (When we have the right hash directory structure, we can just + - pass --include=X --include=X/Y --include=X/Y/file --exclude=*) + -} +store :: RsyncOpts -> Key -> FilePath -> MeterUpdate -> Annex Bool +store o k src meterupdate = storeGeneric o meterupdate basedest populatedest + where + basedest = Prelude.head (keyPaths k) + populatedest dest = liftIO $ if canrename + then do + rename src dest + return True + else createLinkOrCopy src dest + {- If the key being sent is encrypted or chunked, the file + - containing its content is a temp file, and so can be + - renamed into place. Otherwise, the file is the annexed + - object file, and has to be copied or hard linked into place. -} + canrename = isEncKey k || isChunkKey k + +storeGeneric :: RsyncOpts -> MeterUpdate -> FilePath -> (FilePath -> Annex Bool) -> Annex Bool +storeGeneric o meterupdate basedest populatedest = withRsyncScratchDir $ \tmp -> do + let dest = tmp basedest + liftIO $ createDirectoryIfMissing True $ parentDir dest + ok <- populatedest dest + ps <- sendParams + if ok + then showResumable $ rsyncRemote Upload o (Just meterupdate) $ ps ++ + Param "--recursive" : partialParams ++ + -- tmp/ to send contents of tmp dir + [ File $ addTrailingPathSeparator tmp + , Param $ rsyncUrl o + ] + else return False + +retrieve :: RsyncOpts -> FilePath -> Key -> MeterUpdate -> Annex () +retrieve o f k p = + unlessM (rsyncRetrieveKey o k f (Just p)) $ + giveup "rsync failed" + +retrieveCheap :: RsyncOpts -> Key -> AssociatedFile -> FilePath -> Annex Bool +retrieveCheap o k _af f = ifM (preseedTmp k f) ( rsyncRetrieveKey o k f Nothing , return False ) + +remove :: RsyncOpts -> Remover +remove o k = removeGeneric o includes + where + includes = concatMap use dirHashes + use h = let dir = h def k in + [ parentDir dir + , dir + -- match content directory and anything in it + , dir keyFile k "***" + ] + +{- An empty directory is rsynced to make it delete. Everything is excluded, + - except for the specified includes. Due to the way rsync traverses + - directories, the includes must match both the file to be deleted, and + - its parent directories, but not their other contents. -} +removeGeneric :: RsyncOpts -> [String] -> Annex Bool +removeGeneric o includes = do + ps <- sendParams + withRsyncScratchDir $ \tmp -> liftIO $ do + {- Send an empty directory to rysnc to make it delete. -} + rsync $ rsyncOptions o ++ ps ++ + map (\s -> Param $ "--include=" ++ s) includes ++ + [ Param "--exclude=*" -- exclude everything else + , Param "--quiet", Param "--delete", Param "--recursive" + ] ++ partialParams ++ + [ Param $ addTrailingPathSeparator tmp + , Param $ rsyncUrl o + ] + +checkKey :: Git.Repo -> RsyncOpts -> CheckPresent +checkKey r o k = do + showChecking r + checkPresentGeneric o (rsyncUrls o k) + +checkPresentGeneric :: RsyncOpts -> [RsyncUrl] -> Annex Bool +checkPresentGeneric o rsyncurls = + -- note: Does not currently differentiate between rsync failing + -- to connect, and the file not being present. + untilTrue rsyncurls $ \u -> + liftIO $ catchBoolIO $ do + withQuietOutput createProcessSuccess $ + proc "rsync" $ toCommand $ + rsyncOptions o ++ [Param u] + return True + +storeExportM :: RsyncOpts -> FilePath -> Key -> ExportLocation -> MeterUpdate -> Annex Bool +storeExportM o src _k loc meterupdate = + storeGeneric o meterupdate basedest populatedest + where + basedest = fromExportLocation loc + populatedest = liftIO . createLinkOrCopy src + +retrieveExportM :: RsyncOpts -> Key -> ExportLocation -> FilePath -> MeterUpdate -> Annex Bool +retrieveExportM o _k loc dest p = rsyncRetrieve o [rsyncurl] dest (Just p) + where + rsyncurl = mkRsyncUrl o (fromExportLocation loc) + +checkPresentExportM :: RsyncOpts -> Key -> ExportLocation -> Annex Bool +checkPresentExportM o _k loc = checkPresentGeneric o [rsyncurl] + where + rsyncurl = mkRsyncUrl o (fromExportLocation loc) + +removeExportM :: RsyncOpts -> Key -> ExportLocation -> Annex Bool +removeExportM o _k loc = + removeGeneric o (includes (fromExportLocation loc)) + where + includes f = f : case upFrom f of + Nothing -> [] + Just f' -> includes f' + +removeExportDirectoryM :: RsyncOpts -> ExportDirectory -> Annex Bool +removeExportDirectoryM o ed = removeGeneric o (allbelow d : includes d) + where + d = fromExportDirectory ed + allbelow f = f "***" + includes f = f : case upFrom f of + Nothing -> [] + Just f' -> includes f' + +renameExportM :: RsyncOpts -> Key -> ExportLocation -> ExportLocation -> Annex Bool +renameExportM _ _ _ _ = return False + +{- Rsync params to enable resumes of sending files safely, + - ensure that files are only moved into place once complete + -} +partialParams :: [CommandParam] +partialParams = [Param "--partial", Param "--partial-dir=.rsync-partial"] + +{- When sending files from crippled filesystems, the permissions can be all + - messed up, and it's better to use the default permissions on the + - destination. -} +sendParams :: Annex [CommandParam] +sendParams = ifM crippledFileSystem + ( return [rsyncUseDestinationPermissions] + , return [] + ) + +{- Runs an action in an empty scratch directory that can be used to build + - up trees for rsync. -} +withRsyncScratchDir :: (FilePath -> Annex a) -> Annex a +withRsyncScratchDir a = do + t <- fromRepo gitAnnexTmpObjectDir + withTmpDirIn t "rsynctmp" a + +rsyncRetrieve :: RsyncOpts -> [RsyncUrl] -> FilePath -> Maybe MeterUpdate -> Annex Bool +rsyncRetrieve o rsyncurls dest meterupdate = + showResumable $ untilTrue rsyncurls $ \u -> rsyncRemote Download o meterupdate + -- use inplace when retrieving to support resuming + [ Param "--inplace" + , Param u + , File dest + ] + +rsyncRetrieveKey :: RsyncOpts -> Key -> FilePath -> Maybe MeterUpdate -> Annex Bool +rsyncRetrieveKey o k dest meterupdate = rsyncRetrieve o (rsyncUrls o k) dest meterupdate + +showResumable :: Annex Bool -> Annex Bool +showResumable a = ifM a + ( return True + , do + showLongNote "rsync failed -- run git annex again to resume file transfer" + return False + ) + +rsyncRemote :: Direction -> RsyncOpts -> Maybe MeterUpdate -> [CommandParam] -> Annex Bool +rsyncRemote direction o m params = do + showOutput -- make way for progress bar + case m of + Nothing -> liftIO $ rsync ps + Just meter -> do + oh <- mkOutputHandler + liftIO $ rsyncProgress oh meter ps + where + ps = opts ++ Param "--progress" : params + opts + | direction == Download = rsyncDownloadOptions o + | otherwise = rsyncUploadOptions o diff --git a/Remote/Rsync/RsyncUrl.hs b/Remote/Rsync/RsyncUrl.hs new file mode 100644 index 0000000000..67ce7946b1 --- /dev/null +++ b/Remote/Rsync/RsyncUrl.hs @@ -0,0 +1,51 @@ +{- Rsync urls. + - + - Copyright 2014-2018 Joey Hess + - + - Licensed under the GNU GPL version 3 or higher. + -} + +{-# LANGUAGE CPP #-} + +module Remote.Rsync.RsyncUrl where + +import Types +import Annex.Locations +import Utility.Rsync +import Utility.SafeCommand + +import Data.Default +import System.FilePath.Posix +#ifdef mingw32_HOST_OS +import Utility.Split +#endif +import Annex.DirHashes + +type RsyncUrl = String + +data RsyncOpts = RsyncOpts + { rsyncUrl :: RsyncUrl + , rsyncOptions :: [CommandParam] + , rsyncUploadOptions :: [CommandParam] + , rsyncDownloadOptions :: [CommandParam] + , rsyncShellEscape :: Bool +} + +rsyncEscape :: RsyncOpts -> RsyncUrl -> RsyncUrl +rsyncEscape o u + | rsyncShellEscape o && rsyncUrlIsShell (rsyncUrl o) = shellEscape u + | otherwise = u + +mkRsyncUrl :: RsyncOpts -> FilePath -> RsyncUrl +mkRsyncUrl o f = rsyncUrl o rsyncEscape o f + +rsyncUrls :: RsyncOpts -> Key -> [RsyncUrl] +rsyncUrls o k = map use dirHashes + where + use h = rsyncUrl o hash h rsyncEscape o (f f) + f = keyFile k +#ifndef mingw32_HOST_OS + hash h = h def k +#else + hash h = replace "\\" "/" (h def k) +#endif diff --git a/Remote/S3.hs b/Remote/S3.hs new file mode 100644 index 0000000000..83016d11e1 --- /dev/null +++ b/Remote/S3.hs @@ -0,0 +1,836 @@ +{- S3 remotes + - + - Copyright 2011-2018 Joey Hess + - + - Licensed under the GNU GPL version 3 or higher. + -} + +{-# LANGUAGE TypeFamilies #-} +{-# LANGUAGE OverloadedStrings #-} +{-# LANGUAGE BangPatterns #-} +{-# LANGUAGE CPP #-} + +module Remote.S3 (remote, iaHost, configIA, iaItemUrl) where + +import qualified Aws as AWS +import qualified Aws.Core as AWS +import qualified Aws.S3 as S3 +import qualified Data.Text as T +import qualified Data.Text.Encoding as T +import qualified Data.ByteString.Lazy as L +import qualified Data.ByteString as BS +import qualified Data.Map as M +import qualified Data.Set as S +import qualified System.FilePath.Posix as Posix +import Data.Char +import Network.Socket (HostName) +import Network.HTTP.Conduit (Manager) +import Network.HTTP.Client (responseStatus, responseBody, RequestBody(..)) +import Network.HTTP.Types +import Control.Monad.Trans.Resource +import Control.Monad.Catch +import Data.IORef +import System.Log.Logger + +import Annex.Common +import Types.Remote +import Types.Export +import Annex.Export +import qualified Git +import Config +import Config.Cost +import Remote.Helper.Special +import Remote.Helper.Http +import Remote.Helper.Messages +import Remote.Helper.Export +import qualified Remote.Helper.AWS as AWS +import Creds +import Annex.UUID +import Logs.Web +import Logs.MetaData +import Types.MetaData +import Utility.Metered +import qualified Annex.Url as Url +import Utility.DataUnits +import Utility.FileSystemEncoding +import Annex.Content +import Annex.Url (withUrlOptions) +import Utility.Url (checkBoth, UrlOptions(..)) +import Utility.Env + +type BucketName = String +type BucketObject = String + +remote :: RemoteType +remote = RemoteType + { typename = "S3" + , enumerate = const (findSpecialRemotes "s3") + , generate = gen + , setup = s3Setup + , exportSupported = exportIsSupported + } + +gen :: Git.Repo -> UUID -> RemoteConfig -> RemoteGitConfig -> Annex (Maybe Remote) +gen r u c gc = do + cst <- remoteCost gc expensiveRemoteCost + info <- extractS3Info c + return $ new cst info + where + new cst info = Just $ specialRemote c + (prepareS3Handle this $ store this info) + (prepareS3HandleMaybe this $ retrieve this c info) + (prepareS3Handle this $ remove info) + (prepareS3HandleMaybe this $ checkKey this c info) + this + where + this = Remote + { uuid = u + , cost = cst + , name = Git.repoDescribe r + , storeKey = storeKeyDummy + , retrieveKeyFile = retreiveKeyFileDummy + , retrieveKeyFileCheap = retrieveCheap + -- HttpManagerRestricted is used here, so this is + -- secure. + , retrievalSecurityPolicy = RetrievalAllKeysSecure + , removeKey = removeKeyDummy + , lockContent = Nothing + , checkPresent = checkPresentDummy + , checkPresentCheap = False + , exportActions = withS3HandleMaybe c gc u $ \mh -> + return $ ExportActions + { storeExport = storeExportS3 u info mh + , retrieveExport = retrieveExportS3 u info mh + , removeExport = removeExportS3 u info mh + , checkPresentExport = checkPresentExportS3 u info mh + -- S3 does not have directories. + , removeExportDirectory = Nothing + , renameExport = renameExportS3 u info mh + } + , whereisKey = Just (getPublicWebUrls u info c) + , remoteFsck = Nothing + , repairRepo = Nothing + , config = c + , getRepo = return r + , gitconfig = gc + , localpath = Nothing + , readonly = False + , appendonly = versioning info + , availability = GloballyAvailable + , remotetype = remote + , mkUnavailable = gen r u (M.insert "host" "!dne!" c) gc + , getInfo = includeCredsInfo c (AWS.creds u) (s3Info c info) + , claimUrl = Nothing + , checkUrl = Nothing + } + +s3Setup :: SetupStage -> Maybe UUID -> Maybe CredPair -> RemoteConfig -> RemoteGitConfig -> Annex (RemoteConfig, UUID) +s3Setup ss mu mcreds c gc = do + u <- maybe (liftIO genUUID) return mu + s3Setup' ss u mcreds c gc + +s3Setup' :: SetupStage -> UUID -> Maybe CredPair -> RemoteConfig -> RemoteGitConfig -> Annex (RemoteConfig, UUID) +s3Setup' ss u mcreds c gc + | configIA c = archiveorg + | otherwise = defaulthost + where + remotename = fromJust (M.lookup "name" c) + defbucket = remotename ++ "-" ++ fromUUID u + defaults = M.fromList + [ ("datacenter", T.unpack $ AWS.defaultRegion AWS.S3) + , ("storageclass", "STANDARD") + , ("host", AWS.s3DefaultHost) + , ("port", "80") + , ("bucket", defbucket) + ] + + use fullconfig = do + gitConfigSpecialRemote u fullconfig [("s3", "true")] + return (fullconfig, u) + + defaulthost = do + (c', encsetup) <- encryptionSetup c gc + c'' <- setRemoteCredPair encsetup c' gc (AWS.creds u) mcreds + let fullconfig = c'' `M.union` defaults + case ss of + Init -> genBucket fullconfig gc u + _ -> return () + use fullconfig + + archiveorg = do + showNote "Internet Archive mode" + c' <- setRemoteCredPair noEncryptionUsed c gc (AWS.creds u) mcreds + -- Ensure user enters a valid bucket name, since + -- this determines the name of the archive.org item. + let validbucket = replace " " "-" $ + fromMaybe (giveup "specify bucket=") $ + getBucketName c' + let archiveconfig = + -- IA acdepts x-amz-* as an alias for x-archive-* + M.mapKeys (replace "x-archive-" "x-amz-") $ + -- encryption does not make sense here + M.insert "encryption" "none" $ + M.insert "bucket" validbucket $ + M.union c' $ + -- special constraints on key names + M.insert "mungekeys" "ia" defaults + info <- extractS3Info archiveconfig + withS3Handle archiveconfig gc u $ + writeUUIDFile archiveconfig u info + use archiveconfig + +-- Sets up a http connection manager for S3 endpoint, which allows +-- http connections to be reused across calls to the helper. +prepareS3Handle :: Remote -> (S3Handle -> helper) -> Preparer helper +prepareS3Handle r = resourcePrepare $ const $ + withS3Handle (config r) (gitconfig r) (uuid r) + +-- Allows for read-only actions, which can be run without a S3Handle. +prepareS3HandleMaybe :: Remote -> (Maybe S3Handle -> helper) -> Preparer helper +prepareS3HandleMaybe r = resourcePrepare $ const $ + withS3HandleMaybe (config r) (gitconfig r) (uuid r) + +store :: Remote -> S3Info -> S3Handle -> Storer +store _r info h = fileStorer $ \k f p -> do + void $ storeHelper info h f (T.pack $ bucketObject info k) p + -- Store public URL to item in Internet Archive. + when (isIA info && not (isChunkKey k)) $ + setUrlPresent webUUID k (iaPublicUrl info (bucketObject info k)) + return True + +storeHelper :: S3Info -> S3Handle -> FilePath -> S3.Object -> MeterUpdate -> Annex (Maybe S3VersionID) +storeHelper info h f object p = case partSize info of + Just partsz | partsz > 0 -> do + fsz <- liftIO $ getFileSize f + if fsz > partsz + then multipartupload fsz partsz + else singlepartupload + _ -> singlepartupload + where + singlepartupload = do + rbody <- liftIO $ httpBodyStorer f p + r <- sendS3Handle h $ putObject info object rbody + return (mkS3VersionID object (S3.porVersionId r)) + multipartupload fsz partsz = do +#if MIN_VERSION_aws(0,16,0) + let startreq = (S3.postInitiateMultipartUpload (bucket info) object) + { S3.imuStorageClass = Just (storageClass info) + , S3.imuMetadata = metaHeaders info + , S3.imuAutoMakeBucket = isIA info + , S3.imuExpires = Nothing -- TODO set some reasonable expiry + } + uploadid <- S3.imurUploadId <$> sendS3Handle h startreq + + -- The actual part size will be a even multiple of the + -- 32k chunk size that lazy ByteStrings use. + let partsz' = (partsz `div` toInteger defaultChunkSize) * toInteger defaultChunkSize + + -- Send parts of the file, taking care to stream each part + -- w/o buffering in memory, since the parts can be large. + etags <- bracketIO (openBinaryFile f ReadMode) hClose $ \fh -> do + let sendparts meter etags partnum = do + pos <- liftIO $ hTell fh + if pos >= fsz + then return (reverse etags) + else do + -- Calculate size of part that will + -- be read. + let sz = if fsz - pos < partsz' + then fsz - pos + else partsz' + let p' = offsetMeterUpdate p (toBytesProcessed pos) + let numchunks = ceiling (fromIntegral sz / fromIntegral defaultChunkSize :: Double) + let popper = handlePopper numchunks defaultChunkSize p' fh + let req = S3.uploadPart (bucket info) object partnum uploadid $ + RequestBodyStream (fromIntegral sz) popper + S3.UploadPartResponse { S3.uprETag = etag } <- sendS3Handle h req + sendparts (offsetMeterUpdate meter (toBytesProcessed sz)) (etag:etags) (partnum + 1) + sendparts p [] 1 + + r <- sendS3Handle h $ S3.postCompleteMultipartUpload + (bucket info) object uploadid (zip [1..] etags) + return (mkS3VersionID object (S3.cmurVersionId r)) +#else + warning $ "Cannot do multipart upload (partsize " ++ show partsz ++ ") of large file (" ++ show fsz ++ "); built with too old a version of the aws library." + singlepartupload +#endif + +{- Implemented as a fileRetriever, that uses conduit to stream the chunks + - out to the file. Would be better to implement a byteRetriever, but + - that is difficult. -} +retrieve :: Remote -> RemoteConfig -> S3Info -> Maybe S3Handle -> Retriever +retrieve r _ info (Just h) = fileRetriever $ \f k p -> do + loc <- eitherS3VersionID info (uuid r) k (T.pack $ bucketObject info k) + retrieveHelper info h loc f p +retrieve r c info Nothing = fileRetriever $ \f k p -> + getPublicWebUrls (uuid r) info c k >>= \case + [] -> do + needS3Creds (uuid r) + giveup "No S3 credentials configured" + us -> unlessM (downloadUrl k p us f) $ + giveup "failed to download content" + +retrieveHelper :: S3Info -> S3Handle -> (Either S3.Object S3VersionID) -> FilePath -> MeterUpdate -> Annex () +retrieveHelper info h loc f p = liftIO $ runResourceT $ do + let req = case loc of + Left o -> S3.getObject (bucket info) o + Right (S3VersionID o vid) -> (S3.getObject (bucket info) o) + { S3.goVersionId = Just (T.pack vid) } + S3.GetObjectResponse { S3.gorResponse = rsp } <- sendS3Handle' h req + Url.sinkResponseFile p zeroBytesProcessed f WriteMode rsp + +retrieveCheap :: Key -> AssociatedFile -> FilePath -> Annex Bool +retrieveCheap _ _ _ = return False + +{- Internet Archive doesn't easily allow removing content. + - While it may remove the file, there are generally other files + - derived from it that it does not remove. -} +remove :: S3Info -> S3Handle -> Remover +remove info h k = do + res <- tryNonAsync $ sendS3Handle h $ + S3.DeleteObject (T.pack $ bucketObject info k) (bucket info) + return $ either (const False) (const True) res + +checkKey :: Remote -> RemoteConfig -> S3Info -> Maybe S3Handle -> CheckPresent +checkKey r _ info (Just h) k = do + showChecking r + loc <- eitherS3VersionID info (uuid r) k (T.pack $ bucketObject info k) + checkKeyHelper info h loc +checkKey r c info Nothing k = + getPublicWebUrls (uuid r) info c k >>= \case + [] -> do + needS3Creds (uuid r) + giveup "No S3 credentials configured" + us -> do + showChecking r + let check u = withUrlOptions $ + liftIO . checkBoth u (keySize k) + anyM check us + +checkKeyHelper :: S3Info -> S3Handle -> (Either S3.Object S3VersionID) -> Annex Bool +checkKeyHelper info h loc = do +#if MIN_VERSION_aws(0,10,0) + rsp <- go + return (isJust $ S3.horMetadata rsp) +#else + catchMissingException $ do + void go + return True +#endif + where + go = sendS3Handle h req + req = case loc of + Left o -> S3.headObject (bucket info) o + Right (S3VersionID o vid) -> (S3.headObject (bucket info) o) + { S3.hoVersionId = Just (T.pack vid) } + +#if ! MIN_VERSION_aws(0,10,0) + {- Catch exception headObject returns when an object is not present + - in the bucket, and returns False. All other exceptions indicate a + - check error and are let through. -} + catchMissingException :: Annex Bool -> Annex Bool + catchMissingException a = catchJust missing a (const $ return False) + where + missing :: AWS.HeaderException -> Maybe () + missing e + | AWS.headerErrorMessage e == "ETag missing" = Just () + | otherwise = Nothing +#endif + +storeExportS3 :: UUID -> S3Info -> Maybe S3Handle -> FilePath -> Key -> ExportLocation -> MeterUpdate -> Annex Bool +storeExportS3 u info (Just h) f k loc p = + catchNonAsync go (\e -> warning (show e) >> return False) + where + go = do + let o = T.pack $ bucketExportLocation info loc + storeHelper info h f o p + >>= setS3VersionID info u k + return True +storeExportS3 u _ Nothing _ _ _ _ = do + needS3Creds u + return False + +retrieveExportS3 :: UUID -> S3Info -> Maybe S3Handle -> Key -> ExportLocation -> FilePath -> MeterUpdate -> Annex Bool +retrieveExportS3 u info mh _k loc f p = + catchNonAsync go (\e -> warning (show e) >> return False) + where + go = case mh of + Just h -> do + retrieveHelper info h (Left (T.pack exporturl)) f p + return True + Nothing -> case getPublicUrlMaker info of + Nothing -> do + needS3Creds u + return False + Just geturl -> Url.withUrlOptions $ + liftIO . Url.download p (geturl exporturl) f + exporturl = bucketExportLocation info loc + +removeExportS3 :: UUID -> S3Info -> Maybe S3Handle -> Key -> ExportLocation -> Annex Bool +removeExportS3 _u info (Just h) _k loc = + catchNonAsync go (\e -> warning (show e) >> return False) + where + go = do + res <- tryNonAsync $ sendS3Handle h $ + S3.DeleteObject (T.pack $ bucketExportLocation info loc) (bucket info) + return $ either (const False) (const True) res +removeExportS3 u _ Nothing _ _ = do + needS3Creds u + return False + +checkPresentExportS3 :: UUID -> S3Info -> Maybe S3Handle -> Key -> ExportLocation -> Annex Bool +checkPresentExportS3 _u info (Just h) _k loc = + checkKeyHelper info h (Left (T.pack $ bucketExportLocation info loc)) +checkPresentExportS3 u info Nothing k loc = case getPublicUrlMaker info of + Nothing -> do + needS3Creds u + giveup "No S3 credentials configured" + Just geturl -> withUrlOptions $ liftIO . + checkBoth (geturl $ bucketExportLocation info loc) (keySize k) + +-- S3 has no move primitive; copy and delete. +renameExportS3 :: UUID -> S3Info -> Maybe S3Handle -> Key -> ExportLocation -> ExportLocation -> Annex Bool +renameExportS3 _u info (Just h) _k src dest = catchNonAsync go (\_ -> return False) + where + go = do + let co = S3.copyObject (bucket info) dstobject + (S3.ObjectId (bucket info) srcobject Nothing) + S3.CopyMetadata + -- ACL is not preserved by copy. + void $ sendS3Handle h $ co { S3.coAcl = acl info } + void $ sendS3Handle h $ S3.DeleteObject srcobject (bucket info) + return True + srcobject = T.pack $ bucketExportLocation info src + dstobject = T.pack $ bucketExportLocation info dest +renameExportS3 u _ Nothing _ _ _ = do + needS3Creds u + return False + +{- Generate the bucket if it does not already exist, including creating the + - UUID file within the bucket. + - + - Some ACLs can allow read/write to buckets, but not querying them, + - so first check if the UUID file already exists and we can skip doing + - anything. + -} +genBucket :: RemoteConfig -> RemoteGitConfig -> UUID -> Annex () +genBucket c gc u = do + showAction "checking bucket" + info <- extractS3Info c + withS3Handle c gc u $ \h -> + go info h =<< checkUUIDFile c u info h + where + go _ _ (Right True) = noop + go info h _ = do + v <- tryNonAsync $ sendS3Handle h (S3.getBucket $ bucket info) + case v of + Right _ -> noop + Left _ -> do + showAction $ "creating bucket in " ++ datacenter + void $ sendS3Handle h $ S3.PutBucket + (bucket info) + (acl info) + locconstraint +#if MIN_VERSION_aws(0,13,0) + storageclass +#endif + writeUUIDFile c u info h + + locconstraint = mkLocationConstraint $ T.pack datacenter + datacenter = fromJust $ M.lookup "datacenter" c +#if MIN_VERSION_aws(0,13,0) + -- "NEARLINE" as a storage class when creating a bucket is a + -- nonstandard extension of Google Cloud Storage. + storageclass = case getStorageClass c of + sc@(S3.OtherStorageClass "NEARLINE") -> Just sc + _ -> Nothing +#endif + +{- Writes the UUID to an annex-uuid file within the bucket. + - + - If the file already exists in the bucket, it must match, + - or this fails. + - + - Note that IA buckets can only created by having a file + - stored in them. So this also takes care of that. + -} +writeUUIDFile :: RemoteConfig -> UUID -> S3Info -> S3Handle -> Annex () +writeUUIDFile c u info h = do + v <- checkUUIDFile c u info h + case v of + Right True -> noop + Right False -> do + warning "The bucket already exists, and its annex-uuid file indicates it is used by a different special remote." + giveup "Cannot reuse this bucket." + _ -> void $ sendS3Handle h mkobject + where + file = T.pack $ uuidFile c + uuidb = L.fromChunks [T.encodeUtf8 $ T.pack $ fromUUID u] + + mkobject = putObject info file (RequestBodyLBS uuidb) + +{- Checks if the UUID file exists in the bucket + - and has the specified UUID already. -} +checkUUIDFile :: RemoteConfig -> UUID -> S3Info -> S3Handle -> Annex (Either SomeException Bool) +checkUUIDFile c u info h = tryNonAsync $ liftIO $ runResourceT $ do + resp <- tryS3 $ sendS3Handle' h (S3.getObject (bucket info) file) + case resp of + Left _ -> return False + Right r -> do + v <- AWS.loadToMemory r + let !ok = check v + return ok + where + check (S3.GetObjectMemoryResponse _meta rsp) = + responseStatus rsp == ok200 && responseBody rsp == uuidb + + file = T.pack $ uuidFile c + uuidb = L.fromChunks [T.encodeUtf8 $ T.pack $ fromUUID u] + +uuidFile :: RemoteConfig -> FilePath +uuidFile c = getFilePrefix c ++ "annex-uuid" + +tryS3 :: ResourceT IO a -> ResourceT IO (Either S3.S3Error a) +tryS3 a = (Right <$> a) `catch` (pure . Left) + +data S3Handle = S3Handle + { hmanager :: Manager + , hawscfg :: AWS.Configuration + , hs3cfg :: S3.S3Configuration AWS.NormalQuery + } + +{- Sends a request to S3 and gets back the response. + - + - Note that pureAws's use of ResourceT is bypassed here; + - the response should be fully processed while the S3Handle + - is still open, eg within a call to withS3Handle. + -} +sendS3Handle + :: (AWS.Transaction req res, AWS.ServiceConfiguration req ~ S3.S3Configuration) + => S3Handle + -> req + -> Annex res +sendS3Handle h r = liftIO $ runResourceT $ sendS3Handle' h r + +sendS3Handle' + :: (AWS.Transaction r a, AWS.ServiceConfiguration r ~ S3.S3Configuration) + => S3Handle + -> r + -> ResourceT IO a +sendS3Handle' h r = AWS.pureAws (hawscfg h) (hs3cfg h) (hmanager h) r + +withS3Handle :: RemoteConfig -> RemoteGitConfig -> UUID -> (S3Handle -> Annex a) -> Annex a +withS3Handle c gc u a = withS3HandleMaybe c gc u $ \mh -> case mh of + Just h -> a h + Nothing -> do + needS3Creds u + giveup "No S3 credentials configured" + +withS3HandleMaybe :: RemoteConfig -> RemoteGitConfig -> UUID -> (Maybe S3Handle -> Annex a) -> Annex a +withS3HandleMaybe c gc u a = do + mcreds <- getRemoteCredPair c gc (AWS.creds u) + case mcreds of + Just creds -> do + awscreds <- liftIO $ genCredentials creds + let awscfg = AWS.Configuration AWS.Timestamp awscreds debugMapper +#if MIN_VERSION_aws(0,17,0) + Nothing +#endif + withUrlOptions $ \ou -> + a $ Just $ S3Handle (httpManager ou) awscfg s3cfg + Nothing -> a Nothing + where + s3cfg = s3Configuration c + +needS3Creds :: UUID -> Annex () +needS3Creds u = warnMissingCredPairFor "S3" (AWS.creds u) + +s3Configuration :: RemoteConfig -> S3.S3Configuration AWS.NormalQuery +s3Configuration c = cfg + { S3.s3Port = port + , S3.s3RequestStyle = case M.lookup "requeststyle" c of + Just "path" -> S3.PathStyle + Just s -> giveup $ "bad S3 requeststyle value: " ++ s + Nothing -> S3.s3RequestStyle cfg + } + where + proto + | port == 443 = AWS.HTTPS + | otherwise = AWS.HTTP + h = fromJust $ M.lookup "host" c + datacenter = fromJust $ M.lookup "datacenter" c + -- When the default S3 host is configured, connect directly to + -- the S3 endpoint for the configured datacenter. + -- When another host is configured, it's used as-is. + endpoint + | h == AWS.s3DefaultHost = AWS.s3HostName $ T.pack datacenter + | otherwise = T.encodeUtf8 $ T.pack h + port = let s = fromJust $ M.lookup "port" c in + case reads s of + [(p, _)] -> p + _ -> giveup $ "bad S3 port value: " ++ s + cfg = S3.s3 proto endpoint False + +data S3Info = S3Info + { bucket :: S3.Bucket + , storageClass :: S3.StorageClass + , bucketObject :: Key -> BucketObject + , bucketExportLocation :: ExportLocation -> BucketObject + , metaHeaders :: [(T.Text, T.Text)] + , partSize :: Maybe Integer + , isIA :: Bool + , versioning :: Bool + , public :: Bool + , publicurl :: Maybe URLString + , host :: Maybe String + } + +extractS3Info :: RemoteConfig -> Annex S3Info +extractS3Info c = do + b <- maybe + (giveup "S3 bucket not configured") + (return . T.pack) + (getBucketName c) + return $ S3Info + { bucket = b + , storageClass = getStorageClass c + , bucketObject = getBucketObject c + , bucketExportLocation = getBucketExportLocation c + , metaHeaders = getMetaHeaders c + , partSize = getPartSize c + , isIA = configIA c + , versioning = boolcfg "versioning" + , public = boolcfg "public" + , publicurl = M.lookup "publicurl" c + , host = M.lookup "host" c + } + where + boolcfg k = case M.lookup k c of + Just "yes" -> True + _ -> False + +putObject :: S3Info -> T.Text -> RequestBody -> S3.PutObject +putObject info file rbody = (S3.putObject (bucket info) file rbody) + { S3.poStorageClass = Just (storageClass info) + , S3.poMetadata = metaHeaders info + , S3.poAutoMakeBucket = isIA info + , S3.poAcl = acl info + } + +acl :: S3Info -> Maybe S3.CannedAcl +acl info + | public info = Just S3.AclPublicRead + | otherwise = Nothing + +getBucketName :: RemoteConfig -> Maybe BucketName +getBucketName = map toLower <$$> M.lookup "bucket" + +getStorageClass :: RemoteConfig -> S3.StorageClass +getStorageClass c = case M.lookup "storageclass" c of + Just "REDUCED_REDUNDANCY" -> S3.ReducedRedundancy +#if MIN_VERSION_aws(0,13,0) + Just s -> S3.OtherStorageClass (T.pack s) +#endif + _ -> S3.Standard + +getPartSize :: RemoteConfig -> Maybe Integer +getPartSize c = readSize dataUnits =<< M.lookup "partsize" c + +getMetaHeaders :: RemoteConfig -> [(T.Text, T.Text)] +getMetaHeaders = map munge . filter ismetaheader . M.assocs + where + ismetaheader (h, _) = metaprefix `isPrefixOf` h + metaprefix = "x-amz-meta-" + metaprefixlen = length metaprefix + munge (k, v) = (T.pack $ drop metaprefixlen k, T.pack v) + +getFilePrefix :: RemoteConfig -> String +getFilePrefix = M.findWithDefault "" "fileprefix" + +getBucketObject :: RemoteConfig -> Key -> BucketObject +getBucketObject c = munge . key2file + where + munge s = case M.lookup "mungekeys" c of + Just "ia" -> iaMunge $ getFilePrefix c ++ s + _ -> getFilePrefix c ++ s + +getBucketExportLocation :: RemoteConfig -> ExportLocation -> BucketObject +getBucketExportLocation c loc = getFilePrefix c ++ fromExportLocation loc + +{- Internet Archive documentation limits filenames to a subset of ascii. + - While other characters seem to work now, this entity encodes everything + - else to avoid problems. -} +iaMunge :: String -> String +iaMunge = (>>= munge) + where + munge c + | isAsciiUpper c || isAsciiLower c || isNumber c = [c] + | c `elem` ("_-.\"" :: String) = [c] + | isSpace c = [] + | otherwise = "&" ++ show (ord c) ++ ";" + +configIA :: RemoteConfig -> Bool +configIA = maybe False isIAHost . M.lookup "host" + +{- Hostname to use for archive.org S3. -} +iaHost :: HostName +iaHost = "s3.us.archive.org" + +isIAHost :: HostName -> Bool +isIAHost h = ".archive.org" `isSuffixOf` map toLower h + +iaItemUrl :: BucketName -> URLString +iaItemUrl b = "http://archive.org/details/" ++ b + +iaPublicUrl :: S3Info -> BucketObject -> URLString +iaPublicUrl info = genericPublicUrl $ + "http://archive.org/download/" ++ T.unpack (bucket info) ++ "/" + +awsPublicUrl :: S3Info -> BucketObject -> URLString +awsPublicUrl info = genericPublicUrl $ + "https://" ++ T.unpack (bucket info) ++ ".s3.amazonaws.com/" + +genericPublicUrl :: URLString -> BucketObject -> URLString +genericPublicUrl baseurl p = baseurl Posix. p + +genCredentials :: CredPair -> IO AWS.Credentials +genCredentials (keyid, secret) = AWS.Credentials + <$> pure (tobs keyid) + <*> pure (tobs secret) + <*> newIORef [] + <*> (fmap tobs <$> getEnv "AWS_SESSION_TOKEN") + where + tobs = T.encodeUtf8 . T.pack + +mkLocationConstraint :: AWS.Region -> S3.LocationConstraint +mkLocationConstraint "US" = S3.locationUsClassic +mkLocationConstraint r = r + +debugMapper :: AWS.Logger +debugMapper level t = forward "S3" (T.unpack t) + where + forward = case level of + AWS.Debug -> debugM + AWS.Info -> infoM + AWS.Warning -> warningM + AWS.Error -> errorM + +s3Info :: RemoteConfig -> S3Info -> [(String, String)] +s3Info c info = catMaybes + [ Just ("bucket", fromMaybe "unknown" (getBucketName c)) + , Just ("endpoint", w82s (BS.unpack (S3.s3Endpoint s3c))) + , Just ("port", show (S3.s3Port s3c)) + , Just ("storage class", showstorageclass (getStorageClass c)) + , if configIA c + then Just ("internet archive item", iaItemUrl $ fromMaybe "unknown" $ getBucketName c) + else Nothing + , Just ("partsize", maybe "unlimited" (roughSize storageUnits False) (getPartSize c)) + , Just ("public", if public info then "yes" else "no") + , Just ("versioning", if versioning info then "yes" else "no") + ] + where + s3c = s3Configuration c +#if MIN_VERSION_aws(0,13,0) + showstorageclass (S3.OtherStorageClass t) = T.unpack t +#endif + showstorageclass sc = show sc + +getPublicWebUrls :: UUID -> S3Info -> RemoteConfig -> Key -> Annex [URLString] +getPublicWebUrls u info c k + | not (public info) = return [] + | exportTree c = if versioning info + then case publicurl info of + Just url -> getS3VersionIDPublicUrls (const $ genericPublicUrl url) info u k + Nothing -> case host info of + Just h | h == AWS.s3DefaultHost -> + getS3VersionIDPublicUrls awsPublicUrl info u k + _ -> return [] + else return [] + | otherwise = case getPublicUrlMaker info of + Just geturl -> return [geturl $ bucketObject info k] + Nothing -> return [] + +getPublicUrlMaker :: S3Info -> Maybe (BucketObject -> URLString) +getPublicUrlMaker info = case publicurl info of + Just url -> Just (genericPublicUrl url) + Nothing -> case host info of + Just h + | h == AWS.s3DefaultHost -> + Just (awsPublicUrl info) + | isIAHost h -> + Just (iaPublicUrl info) + _ -> Nothing + + +data S3VersionID = S3VersionID S3.Object String + deriving (Show) + +-- smart constructor +mkS3VersionID :: S3.Object -> Maybe T.Text -> Maybe S3VersionID +mkS3VersionID o = mkS3VersionID' o . fmap T.unpack + +mkS3VersionID' :: S3.Object -> Maybe String -> Maybe S3VersionID +mkS3VersionID' o (Just s) + | null s = Nothing + -- AWS documentation says a version ID is at most 1024 bytes long. + -- Since they are stored in the git-annex branch, prevent them from + -- being very much larger than that. + | length s < 2048 = Just (S3VersionID o s) + | otherwise = Nothing +mkS3VersionID' _ Nothing = Nothing + +-- Format for storage in per-remote metadata. +-- +-- A S3 version ID is "url ready" so does not contain '#' and so we'll use +-- that to separate it from the object id. (Could use a space, but spaces +-- in metadata values lead to an inefficient encoding.) +formatS3VersionID :: S3VersionID -> String +formatS3VersionID (S3VersionID o v) = v ++ '#' : T.unpack o + +-- Parse from value stored in per-remote metadata. +parseS3VersionID :: String -> Maybe S3VersionID +parseS3VersionID s = + let (v, o) = separate (== '#') s + in mkS3VersionID' (T.pack o) (Just v) + +setS3VersionID :: S3Info -> UUID -> Key -> Maybe S3VersionID -> Annex () +setS3VersionID info u k vid + | versioning info = maybe noop (setS3VersionID' u k) vid + | otherwise = noop + +setS3VersionID' :: UUID -> Key -> S3VersionID -> Annex () +setS3VersionID' u k vid = addRemoteMetaData k $ + RemoteMetaData u (updateMetaData s3VersionField v emptyMetaData) + where + v = mkMetaValue (CurrentlySet True) (formatS3VersionID vid) + +getS3VersionID :: UUID -> Key -> Annex [S3VersionID] +getS3VersionID u k = do + (RemoteMetaData _ m) <- getCurrentRemoteMetaData u k + return $ mapMaybe parseS3VersionID $ map unwrap $ S.toList $ + metaDataValues s3VersionField m + where + unwrap (MetaValue _ v) = v + +s3VersionField :: MetaField +s3VersionField = mkMetaFieldUnchecked "V" + +eitherS3VersionID :: S3Info -> UUID -> Key -> S3.Object -> Annex (Either S3.Object S3VersionID) +eitherS3VersionID info u k fallback + | versioning info = getS3VersionID u k >>= return . \case + [] -> Left fallback + -- It's possible for a key to be stored multiple timees in + -- a bucket with different version IDs; only use one of them. + (v:_) -> Right v + | otherwise = return (Left fallback) + +s3VersionIDPublicUrl :: (S3Info -> BucketObject -> URLString) -> S3Info -> S3VersionID -> URLString +s3VersionIDPublicUrl mk info (S3VersionID obj vid) = mk info $ concat + [ T.unpack obj + , "?versionId=" + , vid -- version ID is "url ready" so no escaping needed + ] + +getS3VersionIDPublicUrls :: (S3Info -> BucketObject -> URLString) -> S3Info -> UUID -> Key -> Annex [URLString] +getS3VersionIDPublicUrls mk info u k = + map (s3VersionIDPublicUrl mk info) <$> getS3VersionID u k diff --git a/Remote/Tahoe.hs b/Remote/Tahoe.hs new file mode 100644 index 0000000000..527121dc99 --- /dev/null +++ b/Remote/Tahoe.hs @@ -0,0 +1,275 @@ +{- Tahoe-LAFS special remotes. + - + - Tahoe capabilities for accessing objects stored in the remote + - are preserved in the remote state log. + - + - In order to allow multiple clones of a repository to access the same + - tahoe repository, git-annex needs to store the introducer furl, + - and the shared-convergence-secret. These are stored in the remote + - configuration, when embedcreds is enabled. + - + - Using those creds, git-annex sets up a tahoe configuration directory in + - ~/.tahoe-git-annex/UUID/ + - + - Tahoe has its own encryption, so git-annex's encryption is not used. + - + - Copyright 2014 Joey Hess + - + - Licensed under the GNU GPL version 3 or higher. + -} + +{-# LANGUAGE OverloadedStrings #-} + +module Remote.Tahoe (remote) where + +import qualified Data.Map as M +import Utility.Aeson +import Data.ByteString.Lazy.UTF8 (fromString) +import Control.Concurrent.STM + +import Annex.Common +import Types.Remote +import Types.Creds +import qualified Git +import Config +import Config.Cost +import Remote.Helper.Special +import Remote.Helper.Export +import Annex.UUID +import Annex.Content +import Logs.RemoteState +import Utility.UserInfo +import Utility.Metered +import Utility.Env +import Utility.ThreadScheduler + +{- The TMVar is left empty until tahoe has been verified to be running. -} +data TahoeHandle = TahoeHandle TahoeConfigDir (TMVar ()) + +type TahoeConfigDir = FilePath +type SharedConvergenceSecret = String +type IntroducerFurl = String +type Capability = String + +remote :: RemoteType +remote = RemoteType + { typename = "tahoe" + , enumerate = const (findSpecialRemotes "tahoe") + , generate = gen + , setup = tahoeSetup + , exportSupported = exportUnsupported + } + +gen :: Git.Repo -> UUID -> RemoteConfig -> RemoteGitConfig -> Annex (Maybe Remote) +gen r u c gc = do + cst <- remoteCost gc expensiveRemoteCost + hdl <- liftIO $ TahoeHandle + <$> maybe (defaultTahoeConfigDir u) return (remoteAnnexTahoe gc) + <*> newEmptyTMVarIO + return $ Just $ Remote + { uuid = u + , cost = cst + , name = Git.repoDescribe r + , storeKey = store u hdl + , retrieveKeyFile = retrieve u hdl + , retrieveKeyFileCheap = \_ _ _ -> return False + -- Tahoe cryptographically verifies content. + , retrievalSecurityPolicy = RetrievalAllKeysSecure + , removeKey = remove + , lockContent = Nothing + , checkPresent = checkKey u hdl + , checkPresentCheap = False + , exportActions = exportUnsupported + , whereisKey = Just (getWhereisKey u) + , remoteFsck = Nothing + , repairRepo = Nothing + , config = c + , getRepo = return r + , gitconfig = gc + , localpath = Nothing + , readonly = False + , appendonly = False + , availability = GloballyAvailable + , remotetype = remote + , mkUnavailable = return Nothing + , getInfo = return [] + , claimUrl = Nothing + , checkUrl = Nothing + } + +tahoeSetup :: SetupStage -> Maybe UUID -> Maybe CredPair -> RemoteConfig -> RemoteGitConfig -> Annex (RemoteConfig, UUID) +tahoeSetup _ mu _ c _ = do + furl <- fromMaybe (fromMaybe missingfurl $ M.lookup furlk c) + <$> liftIO (getEnv "TAHOE_FURL") + u <- maybe (liftIO genUUID) return mu + configdir <- liftIO $ defaultTahoeConfigDir u + scs <- liftIO $ tahoeConfigure configdir furl (M.lookup scsk c) + let c' = if M.lookup "embedcreds" c == Just "yes" + then flip M.union c $ M.fromList + [ (furlk, furl) + , (scsk, scs) + ] + else c + gitConfigSpecialRemote u c' [("tahoe", configdir)] + return (c', u) + where + scsk = "shared-convergence-secret" + furlk = "introducer-furl" + missingfurl = giveup "Set TAHOE_FURL to the introducer furl to use." + +store :: UUID -> TahoeHandle -> Key -> AssociatedFile -> MeterUpdate -> Annex Bool +store u hdl k _f _p = sendAnnex k noop $ \src -> + parsePut <$> liftIO (readTahoe hdl "put" [File src]) >>= maybe + (return False) + (\cap -> storeCapability u k cap >> return True) + +retrieve :: UUID -> TahoeHandle -> Key -> AssociatedFile -> FilePath -> MeterUpdate -> Annex (Bool, Verification) +retrieve u hdl k _f d _p = unVerified $ go =<< getCapability u k + where + go Nothing = return False + go (Just cap) = liftIO $ requestTahoe hdl "get" [Param cap, File d] + +remove :: Key -> Annex Bool +remove _k = do + warning "content cannot be removed from tahoe remote" + return False + +checkKey :: UUID -> TahoeHandle -> Key -> Annex Bool +checkKey u hdl k = go =<< getCapability u k + where + go Nothing = return False + go (Just cap) = liftIO $ do + v <- parseCheck <$> readTahoe hdl "check" + [ Param "--raw" + , Param cap + ] + either giveup return v + +defaultTahoeConfigDir :: UUID -> IO TahoeConfigDir +defaultTahoeConfigDir u = do + h <- myHomeDir + return $ h ".tahoe-git-annex" fromUUID u + +tahoeConfigure :: TahoeConfigDir -> IntroducerFurl -> Maybe SharedConvergenceSecret -> IO SharedConvergenceSecret +tahoeConfigure configdir furl mscs = do + unlessM (createClient configdir furl) $ + giveup "tahoe create-client failed" + maybe noop (writeSharedConvergenceSecret configdir) mscs + startTahoeDaemon configdir + getSharedConvergenceSecret configdir + +createClient :: TahoeConfigDir -> IntroducerFurl -> IO Bool +createClient configdir furl = do + createDirectoryIfMissing True (parentDir configdir) + boolTahoe configdir "create-client" + [ Param "--nickname", Param "git-annex" + , Param "--introducer", Param furl + ] + +writeSharedConvergenceSecret :: TahoeConfigDir -> SharedConvergenceSecret -> IO () +writeSharedConvergenceSecret configdir scs = + writeFile (convergenceFile configdir) (unlines [scs]) + +{- The tahoe daemon writes the convergenceFile shortly after it starts + - (it does not need to connect to the network). So, try repeatedly to read + - the file, for up to 1 minute. To avoid reading a partially written + - file, look for the newline after the value. -} +getSharedConvergenceSecret :: TahoeConfigDir -> IO SharedConvergenceSecret +getSharedConvergenceSecret configdir = go (60 :: Int) + where + f = convergenceFile configdir + go n + | n == 0 = giveup $ "tahoe did not write " ++ f ++ " after 1 minute. Perhaps the daemon failed to start?" + | otherwise = do + v <- catchMaybeIO (readFile f) + case v of + Just s | "\n" `isSuffixOf` s || "\r" `isSuffixOf` s -> + return $ takeWhile (`notElem` ("\n\r" :: String)) s + _ -> do + threadDelaySeconds (Seconds 1) + go (n - 1) + +convergenceFile :: TahoeConfigDir -> FilePath +convergenceFile configdir = configdir "private" "convergence" + +startTahoeDaemon :: TahoeConfigDir -> IO () +startTahoeDaemon configdir = void $ boolTahoe configdir "start" [] + +{- Ensures that tahoe has been started, before running an action + - that uses it. -} +withTahoeConfigDir :: TahoeHandle -> (TahoeConfigDir -> IO a) -> IO a +withTahoeConfigDir (TahoeHandle configdir v) a = go =<< atomically needsstart + where + go True = do + startTahoeDaemon configdir + a configdir + go False = a configdir + needsstart = ifM (isEmptyTMVar v) + ( do + putTMVar v () + return True + , return False + ) + +boolTahoe :: TahoeConfigDir -> String -> [CommandParam] -> IO Bool +boolTahoe configdir command params = boolSystem "tahoe" $ + tahoeParams configdir command params + +{- Runs a tahoe command that requests the daemon do something. -} +requestTahoe :: TahoeHandle -> String -> [CommandParam] -> IO Bool +requestTahoe hdl command params = withTahoeConfigDir hdl $ \configdir -> + boolTahoe configdir command params + +{- Runs a tahoe command that requests the daemon output something. -} +readTahoe :: TahoeHandle -> String -> [CommandParam] -> IO String +readTahoe hdl command params = withTahoeConfigDir hdl $ \configdir -> + catchDefaultIO "" $ + readProcess "tahoe" $ toCommand $ + tahoeParams configdir command params + +tahoeParams :: TahoeConfigDir -> String -> [CommandParam] -> [CommandParam] +tahoeParams configdir command params = + Param "-d" : File configdir : Param command : params + +storeCapability :: UUID -> Key -> Capability -> Annex () +storeCapability u k cap = setRemoteState u k cap + +getCapability :: UUID -> Key -> Annex (Maybe Capability) +getCapability u k = getRemoteState u k + +getWhereisKey :: UUID -> Key -> Annex [String] +getWhereisKey u k = disp <$> getCapability u k + where + disp Nothing = [] + disp (Just c) = [c] + +{- tahoe put outputs a single line, containing the capability. -} +parsePut :: String -> Maybe Capability +parsePut s = case lines s of + [cap] | "URI" `isPrefixOf` cap -> Just cap + _ -> Nothing + +{- tahoe check --raw outputs a json document. + - Its contents will vary (for LIT capabilities, it lacks most info), + - but should always contain a results object with a healthy value + - that's true or false. + -} +parseCheck :: String -> Either String Bool +parseCheck s = maybe parseerror (Right . healthy . results) (decode $ fromString s) + where + parseerror + | null s = Left "tahoe check failed to run" + | otherwise = Left "unable to parse tahoe check output" + +data CheckRet = CheckRet { results :: Results } +data Results = Results { healthy :: Bool } + +instance FromJSON CheckRet where + parseJSON (Object v) = CheckRet + <$> v .: "results" + parseJSON _ = mzero + +instance FromJSON Results where + parseJSON (Object v) = Results + <$> v .: "healthy" + parseJSON _ = mzero diff --git a/Remote/Web.hs b/Remote/Web.hs new file mode 100644 index 0000000000..047bb6ce3f --- /dev/null +++ b/Remote/Web.hs @@ -0,0 +1,133 @@ +{- Web remote. + - + - Copyright 2011 Joey Hess + - + - Licensed under the GNU GPL version 3 or higher. + -} + +module Remote.Web (remote, getWebUrls) where + +import Annex.Common +import Types.Remote +import Remote.Helper.Messages +import Remote.Helper.Export +import qualified Git +import qualified Git.Construct +import Annex.Content +import Config.Cost +import Config +import Logs.Web +import Annex.UUID +import Messages.Progress +import Utility.Metered +import qualified Annex.Url as Url +import Annex.YoutubeDl + +remote :: RemoteType +remote = RemoteType + { typename = "web" + , enumerate = list + , generate = gen + , setup = error "not supported" + , exportSupported = exportUnsupported + } + +-- There is only one web remote, and it always exists. +-- (If the web should cease to exist, remove this module and redistribute +-- a new release to the survivors by carrier pigeon.) +list :: Bool -> Annex [Git.Repo] +list _autoinit = do + r <- liftIO $ Git.Construct.remoteNamed "web" (pure Git.Construct.fromUnknown) + return [r] + +gen :: Git.Repo -> UUID -> RemoteConfig -> RemoteGitConfig -> Annex (Maybe Remote) +gen r _ c gc = do + cst <- remoteCost gc expensiveRemoteCost + return $ Just Remote + { uuid = webUUID + , cost = cst + , name = Git.repoDescribe r + , storeKey = uploadKey + , retrieveKeyFile = downloadKey + , retrieveKeyFileCheap = downloadKeyCheap + -- HttpManagerRestricted is used here, so this is + -- secure. + , retrievalSecurityPolicy = RetrievalAllKeysSecure + , removeKey = dropKey + , lockContent = Nothing + , checkPresent = checkKey + , checkPresentCheap = False + , exportActions = exportUnsupported + , whereisKey = Nothing + , remoteFsck = Nothing + , repairRepo = Nothing + , config = c + , gitconfig = gc + , localpath = Nothing + , getRepo = return r + , readonly = True + , appendonly = False + , availability = GloballyAvailable + , remotetype = remote + , mkUnavailable = return Nothing + , getInfo = return [] + , claimUrl = Nothing -- implicitly claims all urls + , checkUrl = Nothing + } + +downloadKey :: Key -> AssociatedFile -> FilePath -> MeterUpdate -> Annex (Bool, Verification) +downloadKey key _af dest p = unVerified $ get =<< getWebUrls key + where + get [] = do + warning "no known url" + return False + get urls = untilTrue urls $ \u -> do + let (u', downloader) = getDownloader u + case downloader of + YoutubeDownloader -> do + showOutput + youtubeDlTo key u' dest + _ -> metered (Just p) key (pure Nothing) $ \_ p' -> + downloadUrl key p' [u'] dest + +downloadKeyCheap :: Key -> AssociatedFile -> FilePath -> Annex Bool +downloadKeyCheap _ _ _ = return False + +uploadKey :: Key -> AssociatedFile -> MeterUpdate -> Annex Bool +uploadKey _ _ _ = do + warning "upload to web not supported" + return False + +dropKey :: Key -> Annex Bool +dropKey k = do + mapM_ (setUrlMissing webUUID k) =<< getWebUrls k + return True + +checkKey :: Key -> Annex Bool +checkKey key = do + us <- getWebUrls key + if null us + then return False + else either giveup return =<< checkKey' key us +checkKey' :: Key -> [URLString] -> Annex (Either String Bool) +checkKey' key us = firsthit us (Right False) $ \u -> do + let (u', downloader) = getDownloader u + showChecking u' + case downloader of + YoutubeDownloader -> youtubeDlCheck u' + _ -> do + Url.withUrlOptions $ liftIO . catchMsgIO . + Url.checkBoth u' (keySize key) + where + firsthit [] miss _ = return miss + firsthit (u:rest) _ a = do + r <- a u + case r of + Right True -> return r + _ -> firsthit rest r a + +getWebUrls :: Key -> Annex [URLString] +getWebUrls key = filter supported <$> getUrls key + where + supported u = snd (getDownloader u) + `elem` [WebDownloader, YoutubeDownloader] diff --git a/Remote/WebDAV.hs b/Remote/WebDAV.hs new file mode 100644 index 0000000000..81ffc72d49 --- /dev/null +++ b/Remote/WebDAV.hs @@ -0,0 +1,504 @@ +{- WebDAV remotes. + - + - Copyright 2012-2017 Joey Hess + - + - Licensed under the GNU GPL version 3 or higher. + -} + +{-# LANGUAGE CPP #-} +{-# LANGUAGE ScopedTypeVariables #-} + +module Remote.WebDAV (remote, davCreds, configUrl) where + +import Network.Protocol.HTTP.DAV +import qualified Data.Map as M +import qualified Data.ByteString.Lazy as L +import qualified Data.ByteString.UTF8 as B8 +import qualified Data.ByteString.Lazy.UTF8 as L8 +import Network.HTTP.Client (HttpException(..), RequestBody) +#if MIN_VERSION_http_client(0,5,0) +import qualified Network.HTTP.Client as HTTP +#endif +import Network.HTTP.Types +import System.IO.Error +import Control.Monad.Catch +import Control.Monad.IO.Class (MonadIO) +import System.Log.Logger (debugM) + +import Annex.Common +import Types.Remote +import Types.Export +import qualified Git +import Config +import Config.Cost +import Remote.Helper.Special +import Remote.Helper.Messages +import Remote.Helper.Http +import Remote.Helper.Export +import qualified Remote.Helper.Chunked.Legacy as Legacy +import Creds +import Utility.Metered +import Utility.Url (URLString, matchStatusCodeException, matchHttpExceptionContent) +import Annex.UUID +import Remote.WebDAV.DavLocation + +#if MIN_VERSION_http_client(0,5,0) +import Network.HTTP.Client (HttpExceptionContent(..), responseStatus) +#endif + +remote :: RemoteType +remote = RemoteType + { typename = "webdav" + , enumerate = const (findSpecialRemotes "webdav") + , generate = gen + , setup = webdavSetup + , exportSupported = exportIsSupported + } + +gen :: Git.Repo -> UUID -> RemoteConfig -> RemoteGitConfig -> Annex (Maybe Remote) +gen r u c gc = new <$> remoteCost gc expensiveRemoteCost + where + new cst = Just $ specialRemote c + (prepareDAV this $ store chunkconfig) + (prepareDAV this $ retrieve chunkconfig) + (prepareDAV this $ remove) + (prepareDAV this $ checkKey this chunkconfig) + this + where + this = Remote + { uuid = u + , cost = cst + , name = Git.repoDescribe r + , storeKey = storeKeyDummy + , retrieveKeyFile = retreiveKeyFileDummy + , retrieveKeyFileCheap = retrieveCheap + -- HttpManagerRestricted is used here, so this is + -- secure. + , retrievalSecurityPolicy = RetrievalAllKeysSecure + , removeKey = removeKeyDummy + , lockContent = Nothing + , checkPresent = checkPresentDummy + , checkPresentCheap = False + , exportActions = withDAVHandle this $ \mh -> return $ ExportActions + { storeExport = storeExportDav mh + , retrieveExport = retrieveExportDav mh + , checkPresentExport = checkPresentExportDav this mh + , removeExport = removeExportDav mh + , removeExportDirectory = Just $ + removeExportDirectoryDav mh + , renameExport = renameExportDav mh + } + , whereisKey = Nothing + , remoteFsck = Nothing + , repairRepo = Nothing + , config = c + , getRepo = return r + , gitconfig = gc + , localpath = Nothing + , readonly = False + , appendonly = False + , availability = GloballyAvailable + , remotetype = remote + , mkUnavailable = gen r u (M.insert "url" "http://!dne!/" c) gc + , getInfo = includeCredsInfo c (davCreds u) $ + [("url", fromMaybe "unknown" (M.lookup "url" c))] + , claimUrl = Nothing + , checkUrl = Nothing + } + chunkconfig = getChunkConfig c + +webdavSetup :: SetupStage -> Maybe UUID -> Maybe CredPair -> RemoteConfig -> RemoteGitConfig -> Annex (RemoteConfig, UUID) +webdavSetup _ mu mcreds c gc = do + u <- maybe (liftIO genUUID) return mu + url <- case M.lookup "url" c of + Nothing -> giveup "Specify url=" + Just url -> return url + (c', encsetup) <- encryptionSetup c gc + creds <- maybe (getCreds c' gc u) (return . Just) mcreds + testDav url creds + gitConfigSpecialRemote u c' [("webdav", "true")] + c'' <- setRemoteCredPair encsetup c' gc (davCreds u) creds + return (c'', u) + +-- Opens a http connection to the DAV server, which will be reused +-- each time the helper is called. +prepareDAV :: Remote -> (Maybe DavHandle -> helper) -> Preparer helper +prepareDAV = resourcePrepare . const . withDAVHandle + +store :: ChunkConfig -> Maybe DavHandle -> Storer +store _ Nothing = byteStorer $ \_k _b _p -> return False +store (LegacyChunks chunksize) (Just dav) = fileStorer $ \k f p -> liftIO $ + withMeteredFile f p $ storeLegacyChunked chunksize k dav +store _ (Just dav) = httpStorer $ \k reqbody -> liftIO $ goDAV dav $ do + let tmp = keyTmpLocation k + let dest = keyLocation k + storeHelper dav tmp dest reqbody + return True + +storeHelper :: DavHandle -> DavLocation -> DavLocation -> RequestBody -> DAVT IO () +storeHelper dav tmp dest reqbody = do + maybe noop (void . mkColRecursive) (locationParent tmp) + debugDav $ "putContent " ++ tmp + inLocation tmp $ + putContentM' (contentType, reqbody) + finalizeStore dav tmp dest + +finalizeStore :: DavHandle -> DavLocation -> DavLocation -> DAVT IO () +finalizeStore dav tmp dest = do + debugDav $ "delContent " ++ dest + inLocation dest $ void $ safely $ delContentM + maybe noop (void . mkColRecursive) (locationParent dest) + moveDAV (baseURL dav) tmp dest + +retrieveCheap :: Key -> AssociatedFile -> FilePath -> Annex Bool +retrieveCheap _ _ _ = return False + +retrieve :: ChunkConfig -> Maybe DavHandle -> Retriever +retrieve _ Nothing = giveup "unable to connect" +retrieve (LegacyChunks _) (Just dav) = retrieveLegacyChunked dav +retrieve _ (Just dav) = fileRetriever $ \d k p -> liftIO $ + goDAV dav $ retrieveHelper (keyLocation k) d p + +retrieveHelper :: DavLocation -> FilePath -> MeterUpdate -> DAVT IO () +retrieveHelper loc d p = do + debugDav $ "retrieve " ++ loc + inLocation loc $ + withContentM $ httpBodyRetriever d p + +remove :: Maybe DavHandle -> Remover +remove Nothing _ = return False +remove (Just dav) k = liftIO $ goDAV dav $ + -- Delete the key's whole directory, including any + -- legacy chunked files, etc, in a single action. + removeHelper (keyDir k) + +removeHelper :: DavLocation -> DAVT IO Bool +removeHelper d = do + debugDav $ "delContent " ++ d + v <- safely $ inLocation d delContentM + case v of + Just _ -> return True + Nothing -> do + v' <- existsDAV d + case v' of + Right False -> return True + _ -> return False + +checkKey :: Remote -> ChunkConfig -> Maybe DavHandle -> CheckPresent +checkKey r _ Nothing _ = giveup $ name r ++ " not configured" +checkKey r chunkconfig (Just dav) k = do + showChecking r + case chunkconfig of + LegacyChunks _ -> checkKeyLegacyChunked dav k + _ -> do + v <- liftIO $ goDAV dav $ + existsDAV (keyLocation k) + either giveup return v + +storeExportDav :: Maybe DavHandle -> FilePath -> Key -> ExportLocation -> MeterUpdate -> Annex Bool +storeExportDav mh f k loc p = runExport mh $ \dav -> do + reqbody <- liftIO $ httpBodyStorer f p + storeHelper dav (keyTmpLocation k) (exportLocation loc) reqbody + return True + +retrieveExportDav :: Maybe DavHandle -> Key -> ExportLocation -> FilePath -> MeterUpdate -> Annex Bool +retrieveExportDav mh _k loc d p = runExport mh $ \_dav -> do + retrieveHelper (exportLocation loc) d p + return True + +checkPresentExportDav :: Remote -> Maybe DavHandle -> Key -> ExportLocation -> Annex Bool +checkPresentExportDav r mh _k loc = case mh of + Nothing -> giveup $ name r ++ " not configured" + Just h -> liftIO $ do + v <- goDAV h $ existsDAV (exportLocation loc) + either giveup return v + +removeExportDav :: Maybe DavHandle -> Key -> ExportLocation -> Annex Bool +removeExportDav mh _k loc = runExport mh $ \_dav -> + removeHelper (exportLocation loc) + +removeExportDirectoryDav :: Maybe DavHandle -> ExportDirectory -> Annex Bool +removeExportDirectoryDav mh dir = runExport mh $ \_dav -> do + let d = fromExportDirectory dir + debugDav $ "delContent " ++ d + safely (inLocation d delContentM) + >>= maybe (return False) (const $ return True) + +renameExportDav :: Maybe DavHandle -> Key -> ExportLocation -> ExportLocation -> Annex Bool +renameExportDav Nothing _ _ _ = return False +renameExportDav (Just h) _k src dest + -- box.com's DAV endpoint has buggy handling of renames, + -- so avoid renaming when using it. + | boxComUrl `isPrefixOf` baseURL h = return False + | otherwise = runExport (Just h) $ \dav -> do + maybe noop (void . mkColRecursive) (locationParent (exportLocation dest)) + moveDAV (baseURL dav) (exportLocation src) (exportLocation dest) + return True + +runExport :: Maybe DavHandle -> (DavHandle -> DAVT IO Bool) -> Annex Bool +runExport Nothing _ = return False +runExport (Just h) a = fromMaybe False <$> liftIO (goDAV h $ safely (a h)) + +configUrl :: Remote -> Maybe URLString +configUrl r = fixup <$> M.lookup "url" (config r) + where + -- box.com DAV url changed + fixup = replace "https://www.box.com/dav/" boxComUrl + +boxComUrl :: URLString +boxComUrl = "https://dav.box.com/dav/" + +type DavUser = B8.ByteString +type DavPass = B8.ByteString + +baseURL :: DavHandle -> URLString +baseURL (DavHandle _ _ _ u) = u + + +toDavUser :: String -> DavUser +toDavUser = B8.fromString + +toDavPass :: String -> DavPass +toDavPass = B8.fromString + +{- Test if a WebDAV store is usable, by writing to a test file, and then + - deleting the file. + - + - Also ensures that the path of the url exists, trying to create it if not. + - + - Throws an error if store is not usable. + -} +testDav :: URLString -> Maybe CredPair -> Annex () +testDav url (Just (u, p)) = do + showAction "testing WebDAV server" + test $ liftIO $ evalDAVT url $ do + prepDAV user pass + makeParentDirs + void $ mkColRecursive "/" + inLocation (tmpLocation "test") $ do + putContentM (Nothing, L8.fromString "test") + delContentM + where + test a = liftIO $ + either (\e -> throwIO $ "WebDAV test failed: " ++ show e) + (const noop) + =<< tryNonAsync a + + user = toDavUser u + pass = toDavPass p +testDav _ Nothing = error "Need to configure webdav username and password." + +{- Tries to make all the parent directories in the WebDAV urls's path, + - right down to the root. + - + - Ignores any failures, which can occur for reasons including the WebDAV + - server only serving up WebDAV in a subdirectory. -} +makeParentDirs :: DAVT IO () +makeParentDirs = go + where + go = do + l <- getDAVLocation + case locationParent l of + Nothing -> noop + Just p -> void $ safely $ inDAVLocation (const p) go + void $ safely mkCol + +{- Checks if the directory exists. If not, tries to create its + - parent directories, all the way down to the root, and finally creates + - it. -} +mkColRecursive :: DavLocation -> DAVT IO Bool +mkColRecursive d = go =<< existsDAV d + where + go (Right True) = return True + go _ = do + debugDav $ "mkCol " ++ d + ifM (inLocation d mkCol) + ( return True + , do + case locationParent d of + Nothing -> makeParentDirs + Just parent -> void (mkColRecursive parent) + inLocation d mkCol + ) + +getCreds :: RemoteConfig -> RemoteGitConfig -> UUID -> Annex (Maybe CredPair) +getCreds c gc u = getRemoteCredPairFor "webdav" c gc (davCreds u) + +davCreds :: UUID -> CredPairStorage +davCreds u = CredPairStorage + { credPairFile = fromUUID u + , credPairEnvironment = ("WEBDAV_USERNAME", "WEBDAV_PASSWORD") + , credPairRemoteKey = Just "davcreds" + } + +{- Content-Type to use for files uploaded to WebDAV. -} +contentType :: Maybe B8.ByteString +contentType = Just $ B8.fromString "application/octet-stream" + +throwIO :: String -> IO a +throwIO msg = ioError $ mkIOError userErrorType msg Nothing Nothing + +moveDAV :: URLString -> DavLocation -> DavLocation -> DAVT IO () +moveDAV baseurl src dest = do + debugDav $ "moveContent " ++ src ++ " " ++ newurl + inLocation src $ moveContentM (B8.fromString newurl) + where + newurl = locationUrl baseurl dest + +existsDAV :: DavLocation -> DAVT IO (Either String Bool) +existsDAV l = do + debugDav $ "getProps " ++ l + inLocation l check `catchNonAsync` (\e -> return (Left $ show e)) + where + check = do + -- Some DAV services only support depth of 1, and + -- more depth is certainly not needed to check if a + -- location exists. + setDepth (Just Depth1) + catchJust missinghttpstatus + (getPropsM >> ispresent True) + (const $ ispresent False) + ispresent = return . Right + missinghttpstatus e = + matchStatusCodeException (== notFound404) e + <|> matchHttpExceptionContent toomanyredirects e + toomanyredirects (TooManyRedirects _) = True + toomanyredirects _ = False + +safely :: DAVT IO a -> DAVT IO (Maybe a) +safely = eitherToMaybe <$$> tryNonAsync + +choke :: IO (Either String a) -> IO a +choke f = do + x <- f + case x of + Left e -> error e + Right r -> return r + +data DavHandle = DavHandle DAVContext DavUser DavPass URLString + +withDAVHandle :: Remote -> (Maybe DavHandle -> Annex a) -> Annex a +withDAVHandle r a = do + mcreds <- getCreds (config r) (gitconfig r) (uuid r) + case (mcreds, configUrl r) of + (Just (user, pass), Just baseurl) -> + withDAVContext baseurl $ \ctx -> + a (Just (DavHandle ctx (toDavUser user) (toDavPass pass) baseurl)) + _ -> a Nothing + +goDAV :: DavHandle -> DAVT IO a -> IO a +goDAV (DavHandle ctx user pass _) a = choke $ run $ prettifyExceptions $ do + prepDAV user pass + a + where + run = fst <$$> runDAVContext ctx + +{- Catch StatusCodeException and trim it to only the statusMessage part, + - eliminating a lot of noise, which can include the whole request that + - failed. The rethrown exception is no longer a StatusCodeException. -} +#if MIN_VERSION_http_client(0,5,0) +prettifyExceptions :: DAVT IO a -> DAVT IO a +prettifyExceptions a = catchJust (matchStatusCodeException (const True)) a go + where + go (HttpExceptionRequest req (StatusCodeException response message)) = giveup $ unwords + [ "DAV failure:" + , show (responseStatus response) + , show (message) + , "HTTP request:" + , show (HTTP.method req) + , show (HTTP.path req) + ] + go e = throwM e +#else +prettifyExceptions :: DAVT IO a -> DAVT IO a +prettifyExceptions a = catchJust (matchStatusCodeException (const True)) a go + where + go (StatusCodeException status _ _) = giveup $ unwords + [ "DAV failure:" + , show (statusCode status) + , show (statusMessage status) + ] + go e = throwM e +#endif + +prepDAV :: DavUser -> DavPass -> DAVT IO () +prepDAV user pass = do + setResponseTimeout Nothing -- disable default (5 second!) timeout + setCreds user pass + +-- +-- Legacy chunking code, to be removed eventually. +-- + +storeLegacyChunked :: ChunkSize -> Key -> DavHandle -> L.ByteString -> IO Bool +storeLegacyChunked chunksize k dav b = + Legacy.storeChunks k tmp dest storer recorder finalizer + where + storehttp l b' = void $ goDAV dav $ do + maybe noop (void . mkColRecursive) (locationParent l) + debugDav $ "putContent " ++ l + inLocation l $ putContentM (contentType, b') + storer locs = Legacy.storeChunked chunksize locs storehttp b + recorder l s = storehttp l (L8.fromString s) + finalizer tmp' dest' = goDAV dav $ + finalizeStore dav tmp' (fromJust $ locationParent dest') + + tmp = addTrailingPathSeparator $ keyTmpLocation k + dest = keyLocation k + +retrieveLegacyChunked :: DavHandle -> Retriever +retrieveLegacyChunked dav = fileRetriever $ \d k p -> liftIO $ + withStoredFilesLegacyChunked k dav onerr $ \locs -> + Legacy.meteredWriteFileChunks p d locs $ \l -> + goDAV dav $ do + debugDav $ "getContent " ++ l + inLocation l $ + snd <$> getContentM + where + onerr = error "download failed" + +checkKeyLegacyChunked :: DavHandle -> CheckPresent +checkKeyLegacyChunked dav k = liftIO $ + either error id <$> withStoredFilesLegacyChunked k dav onerr check + where + check [] = return $ Right True + check (l:ls) = do + v <- goDAV dav $ existsDAV l + if v == Right True + then check ls + else return v + + {- Failed to read the chunkcount file; see if it's missing, + - or if there's a problem accessing it, + - or perhaps this was an intermittent error. -} + onerr f = do + v <- goDAV dav $ existsDAV f + return $ if v == Right True + then Left $ "failed to read " ++ f + else v + +withStoredFilesLegacyChunked + :: Key + -> DavHandle + -> (DavLocation -> IO a) + -> ([DavLocation] -> IO a) + -> IO a +withStoredFilesLegacyChunked k dav onerr a = do + let chunkcount = keyloc ++ Legacy.chunkCount + v <- goDAV dav $ safely $ do + debugDav $ "getContent " ++ chunkcount + inLocation chunkcount $ + snd <$> getContentM + case v of + Just s -> a $ Legacy.listChunks keyloc $ L8.toString s + Nothing -> do + chunks <- Legacy.probeChunks keyloc $ \f -> + (== Right True) <$> goDAV dav (existsDAV f) + if null chunks + then onerr chunkcount + else a chunks + where + keyloc = keyLocation k + +debugDav :: MonadIO m => String -> DAVT m () +debugDav msg = liftIO $ debugM "WebDAV" msg diff --git a/Remote/WebDAV/DavLocation.hs b/Remote/WebDAV/DavLocation.hs new file mode 100644 index 0000000000..42c9d35d4d --- /dev/null +++ b/Remote/WebDAV/DavLocation.hs @@ -0,0 +1,68 @@ +{- WebDAV locations. + - + - Copyright 2014 Joey Hess + - + - Licensed under the GNU GPL version 3 or higher. + -} + +{-# LANGUAGE CPP #-} +{-# LANGUAGE FlexibleContexts #-} + +module Remote.WebDAV.DavLocation where + +import Types +import Types.Export +import Annex.Locations +import Utility.Url (URLString) +#ifdef mingw32_HOST_OS +import Utility.Split +#endif + +import System.FilePath.Posix -- for manipulating url paths +import Network.Protocol.HTTP.DAV (inDAVLocation, DAVT) +import Control.Monad.IO.Class (MonadIO) +import Network.URI +import Data.Default + +-- Relative to the top of the DAV url. +type DavLocation = String + +{- Runs action with a new location relative to the current location. -} +inLocation :: (MonadIO m) => DavLocation -> DAVT m a -> DAVT m a +inLocation d = inDAVLocation ( d') + where + d' = escapeURIString isUnescapedInURI d + +{- The directory where files(s) for a key are stored. -} +keyDir :: Key -> DavLocation +keyDir k = addTrailingPathSeparator $ hashdir keyFile k + where +#ifndef mingw32_HOST_OS + hashdir = hashDirLower def k +#else + hashdir = replace "\\" "/" (hashDirLower def k) +#endif + +keyLocation :: Key -> DavLocation +keyLocation k = keyDir k ++ keyFile k + +exportLocation :: ExportLocation -> DavLocation +exportLocation = fromExportLocation + +{- Where we store temporary data for a key as it's being uploaded. -} +keyTmpLocation :: Key -> DavLocation +keyTmpLocation = tmpLocation . keyFile + +tmpLocation :: FilePath -> DavLocation +tmpLocation f = "git-annex-webdav-tmp-" ++ f + +locationParent :: String -> Maybe String +locationParent loc + | loc `elem` tops || parent `elem` tops = Nothing + | otherwise = Just parent + where + tops = ["/", "", "."] + parent = takeDirectory loc + +locationUrl :: URLString -> DavLocation -> URLString +locationUrl baseurl loc = baseurl loc diff --git a/RemoteDaemon/Common.hs b/RemoteDaemon/Common.hs new file mode 100644 index 0000000000..366f6aacae --- /dev/null +++ b/RemoteDaemon/Common.hs @@ -0,0 +1,71 @@ +{- git-remote-daemon utilities + - + - Copyright 2014-2016 Joey Hess + - + - Licensed under the GNU GPL version 3 or higher. + -} + +module RemoteDaemon.Common + ( liftAnnex + , inLocalRepo + , checkShouldFetch + , ConnectionStatus(..) + , robustConnection + ) where + +import qualified Annex +import Annex.Common +import RemoteDaemon.Types +import qualified Git +import Annex.CatFile +import Utility.ThreadScheduler + +import Control.Concurrent + +-- Runs an Annex action. Long-running actions should be avoided, +-- since only one liftAnnex can be running at a time, across all +-- transports. +liftAnnex :: TransportHandle -> Annex a -> IO a +liftAnnex (TransportHandle _ annexstate) a = do + st <- takeMVar annexstate + (r, st') <- Annex.run st a + putMVar annexstate st' + return r + +inLocalRepo :: TransportHandle -> (Git.Repo -> IO a) -> IO a +inLocalRepo (TransportHandle (LocalRepo g) _) a = a g + +-- Check if some shas should be fetched from the remote, +-- and presumably later merged. +checkShouldFetch :: RemoteGitConfig -> TransportHandle -> [Git.Sha] -> IO Bool +checkShouldFetch gc transporthandle shas + | remoteAnnexPull gc = checkNewShas transporthandle shas + | otherwise = return False + +-- Check if any of the shas are actally new in the local git repo, +-- to avoid unnecessary fetching. +checkNewShas :: TransportHandle -> [Git.Sha] -> IO Bool +checkNewShas transporthandle = check + where + check [] = return True + check (r:rs) = maybe (check rs) (const $ return False) + =<< liftAnnex transporthandle (catObjectDetails r) + +data ConnectionStatus = ConnectionStopping | ConnectionClosed + +{- Make connection robust, retrying on error, with exponential backoff. -} +robustConnection :: Int -> IO ConnectionStatus -> IO () +robustConnection backoff a = + caught =<< a `catchNonAsync` (const $ return ConnectionClosed) + where + caught ConnectionStopping = return () + caught ConnectionClosed = do + threadDelaySeconds (Seconds backoff) + robustConnection increasedbackoff a + + increasedbackoff + | b2 > maxbackoff = maxbackoff + | otherwise = b2 + where + b2 = backoff * 2 + maxbackoff = 3600 -- one hour diff --git a/RemoteDaemon/Core.hs b/RemoteDaemon/Core.hs new file mode 100644 index 0000000000..b3cd34a128 --- /dev/null +++ b/RemoteDaemon/Core.hs @@ -0,0 +1,174 @@ +{- git-remote-daemon core + - + - Copyright 2014-2016 Joey Hess + - + - Licensed under the GNU GPL version 3 or higher. + -} + +module RemoteDaemon.Core (runInteractive, runNonInteractive) where + +import qualified Annex +import Common +import Types.GitConfig +import Config.DynamicConfig +import RemoteDaemon.Common +import RemoteDaemon.Types +import RemoteDaemon.Transport +import qualified Git +import qualified Git.Types as Git +import qualified Git.CurrentRepo +import qualified Git.Construct +import Utility.SimpleProtocol +import Utility.ThreadScheduler +import Config +import Annex.Ssh +import Types.Messages + +import Control.Concurrent +import Control.Concurrent.Async +import Control.Concurrent.STM +import Network.URI +import qualified Data.Map as M + +runInteractive :: IO () +runInteractive = do + (readh, writeh) <- dupIoHandles + ichan <- newTChanIO :: IO (TChan Consumed) + ochan <- newTChanIO :: IO (TChan Emitted) + + let reader = forever $ do + l <- hGetLine readh + case parseMessage l of + Nothing -> error $ "protocol error: " ++ l + Just cmd -> atomically $ writeTChan ichan cmd + let writer = forever $ do + msg <- atomically $ readTChan ochan + hPutStrLn writeh $ unwords $ formatMessage msg + hFlush writeh + let controller = runController ichan ochan + + -- If any thread fails, the rest will be killed. + void $ tryIO $ reader + `concurrently` writer + `concurrently` controller + +runNonInteractive :: IO () +runNonInteractive = do + ichan <- newTChanIO :: IO (TChan Consumed) + ochan <- newTChanIO :: IO (TChan Emitted) + + let reader = forever $ do + threadDelaySeconds (Seconds (60*60)) + atomically $ writeTChan ichan RELOAD + let writer = forever $ + void $ atomically $ readTChan ochan + let controller = runController ichan ochan + + void $ tryIO $ reader + `concurrently` writer + `concurrently` controller + +type RemoteMap = M.Map Git.Repo (IO (), TChan Consumed) + +-- Runs the transports, dispatching messages to them, and handling +-- the main control messages. +runController :: TChan Consumed -> TChan Emitted -> IO () +runController ichan ochan = do + h <- genTransportHandle + m <- genRemoteMap h ochan + starttransports m + serverchans <- mapM (startserver h) remoteServers + go h False m serverchans + where + go h paused m serverchans = do + cmd <- atomically $ readTChan ichan + broadcast cmd serverchans + case cmd of + RELOAD -> do + h' <- updateTransportHandle h + m' <- genRemoteMap h' ochan + let common = M.intersection m m' + let new = M.difference m' m + let old = M.difference m m' + broadcast STOP (mchans old) + unless paused $ + starttransports new + go h' paused (M.union common new) serverchans + LOSTNET -> do + -- force close all cached ssh connections + -- (done here so that if there are multiple + -- ssh remotes, it's only done once) + liftAnnex h forceSshCleanup + broadcast LOSTNET transportchans + go h True m serverchans + PAUSE -> do + broadcast STOP transportchans + go h True m serverchans + RESUME -> do + when paused $ + starttransports m + go h False m serverchans + STOP -> exitSuccess + -- All remaining messages are sent to + -- all Transports. + msg -> do + unless paused $ + broadcast msg transportchans + go h paused m serverchans + where + transportchans = mchans m + mchans = map snd . M.elems + + startserver h server = do + c <- newTChanIO + void $ async $ server c h + return c + + starttransports m = forM_ (M.elems m) starttransports' + starttransports' (transport, c) = do + -- drain any old control messages from the channel + -- to avoid confusing the transport with them + atomically $ drain c + void $ async transport + + drain c = maybe noop (const $ drain c) =<< tryReadTChan c + + broadcast msg cs = atomically $ forM_ cs $ \c -> writeTChan c msg + +-- Generates a map with a transport for each supported remote in the git repo, +-- except those that have annex.sync = false +genRemoteMap :: TransportHandle -> TChan Emitted -> IO RemoteMap +genRemoteMap h@(TransportHandle (LocalRepo g) _) ochan = do + rs <- Git.Construct.fromRemotes g + M.fromList . catMaybes <$> mapM gen rs + where + gen r = do + gc <- atomically $ extractRemoteGitConfig g (Git.repoDescribe r) + case Git.location r of + Git.Url u -> case M.lookup (uriScheme u) remoteTransports of + Just transport -> ifM (getDynamicConfig (remoteAnnexSync gc)) + ( do + ichan <- newTChanIO :: IO (TChan Consumed) + return $ Just + ( r + , (transport (RemoteRepo r gc) (RemoteURI u) h ichan ochan, ichan) + ) + , return Nothing + ) + Nothing -> return Nothing + _ -> return Nothing + +genTransportHandle :: IO TransportHandle +genTransportHandle = do + annexstate <- newMVar =<< Annex.new =<< Git.CurrentRepo.get + g <- Annex.repo <$> readMVar annexstate + let h = TransportHandle (LocalRepo g) annexstate + liftAnnex h $ Annex.setOutput QuietOutput + return h + +updateTransportHandle :: TransportHandle -> IO TransportHandle +updateTransportHandle h@(TransportHandle _g annexstate) = do + g' <- liftAnnex h $ do + reloadConfig + Annex.fromRepo id + return (TransportHandle (LocalRepo g') annexstate) diff --git a/RemoteDaemon/Transport.hs b/RemoteDaemon/Transport.hs new file mode 100644 index 0000000000..231173a762 --- /dev/null +++ b/RemoteDaemon/Transport.hs @@ -0,0 +1,30 @@ +{- git-remote-daemon transports + - + - Copyright 2014 Joey Hess + - + - Licensed under the GNU GPL version 3 or higher. + -} + +module RemoteDaemon.Transport where + +import RemoteDaemon.Types +import qualified RemoteDaemon.Transport.Ssh +import qualified RemoteDaemon.Transport.GCrypt +import qualified RemoteDaemon.Transport.Tor +import qualified Git.GCrypt +import P2P.Address (torAnnexScheme) + +import qualified Data.Map as M + +-- Corresponds to uriScheme +type TransportScheme = String + +remoteTransports :: M.Map TransportScheme Transport +remoteTransports = M.fromList + [ ("ssh:", RemoteDaemon.Transport.Ssh.transport) + , (Git.GCrypt.urlScheme, RemoteDaemon.Transport.GCrypt.transport) + , (torAnnexScheme, RemoteDaemon.Transport.Tor.transport) + ] + +remoteServers :: [Server] +remoteServers = [RemoteDaemon.Transport.Tor.server] diff --git a/RemoteDaemon/Transport/GCrypt.hs b/RemoteDaemon/Transport/GCrypt.hs new file mode 100644 index 0000000000..2ebd161896 --- /dev/null +++ b/RemoteDaemon/Transport/GCrypt.hs @@ -0,0 +1,28 @@ +{- git-remote-daemon, gcrypt transport + - + - Copyright 2015 Joey Hess + - + - Licensed under the GNU GPL version 3 or higher. + -} + +module RemoteDaemon.Transport.GCrypt (transport) where + +import Annex.Common +import RemoteDaemon.Types +import RemoteDaemon.Common +import RemoteDaemon.Transport.Ssh (transportUsingCmd) +import Git.GCrypt +import Remote.Helper.Ssh +import Remote.GCrypt (accessShellConfig) +import Annex.Ssh + +transport :: Transport +transport rr@(RemoteRepo r gc) url h@(TransportHandle (LocalRepo g) _) ichan ochan + | accessShellConfig gc = do + r' <- encryptedRemote g r + v <- liftAnnex h $ git_annex_shell ConsumeStdin r' "notifychanges" [] [] + case v of + Nothing -> noop + Just (cmd, params) -> + transportUsingCmd cmd params rr url h ichan ochan + | otherwise = noop diff --git a/RemoteDaemon/Transport/Ssh.hs b/RemoteDaemon/Transport/Ssh.hs new file mode 100644 index 0000000000..772ae97715 --- /dev/null +++ b/RemoteDaemon/Transport/Ssh.hs @@ -0,0 +1,106 @@ +{- git-remote-daemon, git-annex-shell over ssh transport + - + - Copyright 2014 Joey Hess + - + - Licensed under the GNU GPL version 3 or higher. + -} + +module RemoteDaemon.Transport.Ssh (transport, transportUsingCmd) where + +import Annex.Common +import Annex.Ssh +import RemoteDaemon.Types +import RemoteDaemon.Common +import Remote.Helper.Ssh +import qualified RemoteDaemon.Transport.Ssh.Types as SshRemote +import Utility.SimpleProtocol +import qualified Git +import Git.Command +import Annex.ChangedRefs + +import Control.Concurrent.STM +import Control.Concurrent.Async + +transport :: Transport +transport rr@(RemoteRepo r _) url h ichan ochan = do + v <- liftAnnex h $ git_annex_shell ConsumeStdin r "notifychanges" [] [] + case v of + Nothing -> noop + Just (cmd, params) -> transportUsingCmd cmd params rr url h ichan ochan + +transportUsingCmd :: FilePath -> [CommandParam] -> Transport +transportUsingCmd cmd params rr@(RemoteRepo r gc) url h@(TransportHandle (LocalRepo g) s) ichan ochan = do + -- enable ssh connection caching wherever inLocalRepo is called + g' <- liftAnnex h $ sshOptionsTo r gc g + let transporthandle = TransportHandle (LocalRepo g') s + transportUsingCmd' cmd params rr url transporthandle ichan ochan + +transportUsingCmd' :: FilePath -> [CommandParam] -> Transport +transportUsingCmd' cmd params (RemoteRepo r gc) url transporthandle ichan ochan = + robustConnection 1 $ do + (Just toh, Just fromh, Just errh, pid) <- + createProcess (proc cmd (toCommand params)) + { std_in = CreatePipe + , std_out = CreatePipe + , std_err = CreatePipe + } + + -- Run all threads until one finishes and get the status + -- of the first to finish. Cancel the rest. + status <- catchDefaultIO (Right ConnectionClosed) $ + handlestderr errh + `race` handlestdout fromh + `race` handlecontrol + + send (DISCONNECTED url) + hClose toh + hClose fromh + void $ waitForProcess pid + + return $ either (either id id) id status + where + send msg = atomically $ writeTChan ochan msg + + fetch = do + send (SYNCING url) + ok <- inLocalRepo transporthandle $ + runBool [Param "fetch", Param $ Git.repoDescribe r] + send (DONESYNCING url ok) + + handlestdout fromh = do + ml <- getProtocolLine fromh + case parseMessage =<< ml of + Just SshRemote.READY -> do + send (CONNECTED url) + handlestdout fromh + Just (SshRemote.CHANGED (ChangedRefs shas)) -> do + whenM (checkShouldFetch gc transporthandle shas) $ + fetch + handlestdout fromh + -- avoid reconnect on protocol error + Nothing -> return ConnectionStopping + + handlecontrol = do + msg <- atomically $ readTChan ichan + case msg of + STOP -> return ConnectionStopping + LOSTNET -> return ConnectionStopping + _ -> handlecontrol + + -- Old versions of git-annex-shell that do not support + -- the notifychanges command will exit with a not very useful + -- error message. Detect that error, and avoid reconnecting. + -- Propigate all stderr. + handlestderr errh = do + s <- hGetSomeString errh 1024 + hPutStr stderr s + hFlush stderr + if "git-annex-shell: git-shell failed" `isInfixOf` s + then do + send $ WARNING url $ unwords + [ "Remote", Git.repoDescribe r + , "needs its git-annex upgraded" + , "to 5.20140405 or newer" + ] + return ConnectionStopping + else handlestderr errh diff --git a/RemoteDaemon/Transport/Ssh/Types.hs b/RemoteDaemon/Transport/Ssh/Types.hs new file mode 100644 index 0000000000..606e1a563b --- /dev/null +++ b/RemoteDaemon/Transport/Ssh/Types.hs @@ -0,0 +1,32 @@ +{- git-remote-daemon, git-annex-shell notifychanges protocol types + - + - Copyright 2014 Joey Hess + - + - Licensed under the GNU GPL version 3 or higher. + -} + +{-# LANGUAGE TypeSynonymInstances, FlexibleInstances #-} +{-# OPTIONS_GHC -fno-warn-orphans #-} + +module RemoteDaemon.Transport.Ssh.Types ( + Notification(..), + Proto.serialize, + Proto.deserialize, + Proto.formatMessage, +) where + +import qualified Utility.SimpleProtocol as Proto +import Annex.ChangedRefs (ChangedRefs) + +data Notification + = READY + | CHANGED ChangedRefs + +instance Proto.Sendable Notification where + formatMessage READY = ["READY"] + formatMessage (CHANGED shas) = ["CHANGED", Proto.serialize shas] + +instance Proto.Receivable Notification where + parseCommand "READY" = Proto.parse0 READY + parseCommand "CHANGED" = Proto.parse1 CHANGED + parseCommand _ = Proto.parseFail diff --git a/RemoteDaemon/Transport/Tor.hs b/RemoteDaemon/Transport/Tor.hs new file mode 100644 index 0000000000..30e1096cea --- /dev/null +++ b/RemoteDaemon/Transport/Tor.hs @@ -0,0 +1,201 @@ +{- git-remote-daemon, tor hidden service server and transport + - + - Copyright 2016 Joey Hess + - + - Licensed under the GNU GPL version 3 or higher. + -} + +{-# LANGUAGE CPP #-} + +module RemoteDaemon.Transport.Tor (server, transport, torSocketFile) where + +import Common +import qualified Annex +import Annex.Concurrent +import Annex.ChangedRefs +import RemoteDaemon.Types +import RemoteDaemon.Common +import Utility.AuthToken +import Utility.Tor +import P2P.Protocol as P2P +import P2P.IO +import P2P.Annex +import P2P.Auth +import P2P.Address +import Annex.UUID +import Types.UUID +import Messages +import Git +import Git.Command + +import Control.Concurrent +import System.Log.Logger (debugM) +import Control.Concurrent.STM +import Control.Concurrent.STM.TBMQueue +import Control.Concurrent.Async +#ifndef mingw32_HOST_OS +import System.Posix.User +#endif + +-- Run tor hidden service. +server :: Server +server ichan th@(TransportHandle (LocalRepo r) _) = go + where + go = checkstartservice >>= handlecontrol + + checkstartservice = do + u <- liftAnnex th getUUID + msock <- liftAnnex th torSocketFile + case msock of + Nothing -> do + debugM "remotedaemon" "Tor hidden service not enabled" + return False + Just sock -> do + void $ async $ startservice sock u + return True + + startservice sock u = do + q <- newTBMQueueIO maxConnections + replicateM_ maxConnections $ + forkIO $ forever $ serveClient th u r q + + debugM "remotedaemon" "Tor hidden service running" + serveUnixSocket sock $ \conn -> do + ok <- atomically $ ifM (isFullTBMQueue q) + ( return False + , do + writeTBMQueue q conn + return True + ) + unless ok $ do + hClose conn + warningIO "dropped Tor connection, too busy" + + handlecontrol servicerunning = do + msg <- atomically $ readTChan ichan + case msg of + -- On reload, the configuration may have changed to + -- enable the tor hidden service. If it was not + -- enabled before, start it, + RELOAD | not servicerunning -> go + -- We can ignore all other messages; no need + -- to restart the hidden service when the network + -- changes as tor takes care of all that. + _ -> handlecontrol servicerunning + +-- How many clients to serve at a time, maximum. This is to avoid DOS attacks. +maxConnections :: Int +maxConnections = 100 + +serveClient :: TransportHandle -> UUID -> Repo -> TBMQueue Handle -> IO () +serveClient th u r q = bracket setup cleanup start + where + setup = do + h <- atomically $ readTBMQueue q + debugM "remotedaemon" "serving a Tor connection" + return h + + cleanup Nothing = return () + cleanup (Just h) = do + debugM "remotedaemon" "done with Tor connection" + hClose h + + start Nothing = return () + start (Just h) = do + -- Avoid doing any work in the liftAnnex, since only one + -- can run at a time. + st <- liftAnnex th dupState + ((), st') <- Annex.run st $ do + -- Load auth tokens for every connection, to notice + -- when the allowed set is changed. + allowed <- loadP2PAuthTokens + let conn = P2PConnection + { connRepo = r + , connCheckAuth = (`isAllowedAuthToken` allowed) + , connIhdl = h + , connOhdl = h + } + -- not really Client, but we don't know their uuid yet + runstauth <- liftIO $ mkRunState Client + v <- liftIO $ runNetProto runstauth conn $ P2P.serveAuth u + case v of + Right (Just theiruuid) -> authed conn theiruuid + Right Nothing -> liftIO $ debugM "remotedaemon" + "Tor connection failed to authenticate" + Left e -> liftIO $ debugM "remotedaemon" $ + "Tor connection error before authentication: " ++ describeProtoFailure e + -- Merge the duplicated state back in. + liftAnnex th $ mergeState st' + + authed conn theiruuid = + bracket watchChangedRefs (liftIO . maybe noop stopWatchingChangedRefs) $ \crh -> do + runst <- liftIO $ mkRunState (Serving theiruuid crh) + v' <- runFullProto runst conn $ + P2P.serveAuthed P2P.ServeReadWrite u + case v' of + Right () -> return () + Left e -> liftIO $ debugM "remotedaemon" $ + "Tor connection error: " ++ describeProtoFailure e + +-- Connect to peer's tor hidden service. +transport :: Transport +transport (RemoteRepo r gc) url@(RemoteURI uri) th ichan ochan = + case unformatP2PAddress (show uri) of + Nothing -> return () + Just addr -> robustConnection 1 $ do + g <- liftAnnex th Annex.gitRepo + bracket (connectPeer g addr) closeConnection (go addr) + where + go addr conn = do + myuuid <- liftAnnex th getUUID + authtoken <- fromMaybe nullAuthToken + <$> liftAnnex th (loadP2PRemoteAuthToken addr) + runst <- mkRunState Client + res <- runNetProto runst conn $ P2P.auth myuuid authtoken noop + case res of + Right (Just theiruuid) -> do + expecteduuid <- liftAnnex th $ getRepoUUID r + if expecteduuid == theiruuid + then do + send (CONNECTED url) + status <- handlecontrol + `race` handlepeer runst conn + send (DISCONNECTED url) + return $ either id id status + else return ConnectionStopping + _ -> return ConnectionClosed + + send msg = atomically $ writeTChan ochan msg + + handlecontrol = do + msg <- atomically $ readTChan ichan + case msg of + STOP -> return ConnectionStopping + LOSTNET -> return ConnectionStopping + _ -> handlecontrol + + handlepeer runst conn = do + v <- runNetProto runst conn P2P.notifyChange + case v of + Right (Just (ChangedRefs shas)) -> do + whenM (checkShouldFetch gc th shas) $ + fetch + handlepeer runst conn + _ -> return ConnectionClosed + + fetch = do + send (SYNCING url) + ok <- inLocalRepo th $ + runBool [Param "fetch", Param $ Git.repoDescribe r] + send (DONESYNCING url ok) + +torSocketFile :: Annex.Annex (Maybe FilePath) +torSocketFile = do + u <- getUUID + let ident = fromUUID u +#ifndef mingw32_HOST_OS + uid <- liftIO getRealUserID +#else + let uid = 0 +#endif + liftIO $ getHiddenServiceSocketFile torAppName uid ident diff --git a/RemoteDaemon/Types.hs b/RemoteDaemon/Types.hs new file mode 100644 index 0000000000..bc0fc1c0eb --- /dev/null +++ b/RemoteDaemon/Types.hs @@ -0,0 +1,111 @@ +{- git-remote-daemon data types. + - + - Copyright 2014 Joey Hess + - + - Licensed under the GNU GPL version 3 or higher. + -} + +{-# OPTIONS_GHC -fno-warn-orphans #-} + +module RemoteDaemon.Types where + +import Common +import qualified Annex +import qualified Git.Types as Git +import qualified Utility.SimpleProtocol as Proto +import Types.GitConfig +import Annex.ChangedRefs (ChangedRefs) + +import Network.URI +import Control.Concurrent +import Control.Concurrent.STM + +-- The URI of a remote is used to uniquely identify it (names change..) +newtype RemoteURI = RemoteURI URI + deriving (Show) + +-- A Transport for a particular git remote consumes some messages +-- from a Chan, and emits others to another Chan. +type Transport = RemoteRepo -> RemoteURI -> TransportHandle -> TChan Consumed -> TChan Emitted -> IO () + +-- A server for a Transport consumes some messages from a Chan in +-- order to learn about network changes, reloads, etc. +type Server = TChan Consumed -> TransportHandle -> IO () + +data RemoteRepo = RemoteRepo Git.Repo RemoteGitConfig +newtype LocalRepo = LocalRepo Git.Repo + +-- All Transports share a single AnnexState MVar +-- +-- Different TransportHandles may have different versions of the LocalRepo. +-- (For example, the ssh transport modifies it to enable ssh connection +-- caching.) +data TransportHandle = TransportHandle LocalRepo (MVar Annex.AnnexState) + +-- Messages that the daemon emits. +data Emitted + = CONNECTED RemoteURI + | DISCONNECTED RemoteURI + | SYNCING RemoteURI + | DONESYNCING RemoteURI Bool + | WARNING RemoteURI String + deriving (Show) + +-- Messages that the deamon consumes. +data Consumed + = PAUSE + | LOSTNET + | RESUME + | CHANGED ChangedRefs + | RELOAD + | STOP + deriving (Show) + +instance Proto.Sendable Emitted where + formatMessage (CONNECTED remote) = + ["CONNECTED", Proto.serialize remote] + formatMessage (DISCONNECTED remote) = + ["DISCONNECTED", Proto.serialize remote] + formatMessage (SYNCING remote) = + ["SYNCING", Proto.serialize remote] + formatMessage (DONESYNCING remote status) = + ["DONESYNCING", Proto.serialize remote, Proto.serialize status] + formatMessage (WARNING remote message) = + ["WARNING", Proto.serialize remote, Proto.serialize message] + +instance Proto.Sendable Consumed where + formatMessage PAUSE = ["PAUSE"] + formatMessage LOSTNET = ["LOSTNET"] + formatMessage RESUME = ["RESUME"] + formatMessage (CHANGED refs) =["CHANGED", Proto.serialize refs] + formatMessage RELOAD = ["RELOAD"] + formatMessage STOP = ["STOP"] + +instance Proto.Receivable Emitted where + parseCommand "CONNECTED" = Proto.parse1 CONNECTED + parseCommand "DISCONNECTED" = Proto.parse1 DISCONNECTED + parseCommand "SYNCING" = Proto.parse1 SYNCING + parseCommand "DONESYNCING" = Proto.parse2 DONESYNCING + parseCommand "WARNING" = Proto.parse2 WARNING + parseCommand _ = Proto.parseFail + +instance Proto.Receivable Consumed where + parseCommand "PAUSE" = Proto.parse0 PAUSE + parseCommand "LOSTNET" = Proto.parse0 LOSTNET + parseCommand "RESUME" = Proto.parse0 RESUME + parseCommand "CHANGED" = Proto.parse1 CHANGED + parseCommand "RELOAD" = Proto.parse0 RELOAD + parseCommand "STOP" = Proto.parse0 STOP + parseCommand _ = Proto.parseFail + +instance Proto.Serializable RemoteURI where + serialize (RemoteURI u) = show u + deserialize = RemoteURI <$$> parseURI + +instance Proto.Serializable Bool where + serialize False = "0" + serialize True = "1" + + deserialize "0" = Just False + deserialize "1" = Just True + deserialize _ = Nothing diff --git a/Setup.hs b/Setup.hs new file mode 100644 index 0000000000..57efd86e0c --- /dev/null +++ b/Setup.hs @@ -0,0 +1,72 @@ +{-# LANGUAGE NamedFieldPuns #-} +{-# OPTIONS_GHC -fno-warn-tabs #-} + +{- cabal setup file -} + +import Distribution.Simple +import Distribution.Simple.LocalBuildInfo +import Distribution.Simple.Setup +import Distribution.Simple.Utils (installOrdinaryFiles, rawSystemExit) +import Distribution.PackageDescription (PackageDescription(..)) +import Distribution.Verbosity (Verbosity) +import System.FilePath +import Control.Applicative +import Control.Monad +import System.Directory +import Data.List +import Data.Maybe +import Control.Exception +import qualified System.Info + +import qualified Build.DesktopFile as DesktopFile +import qualified Build.Configure as Configure +import Build.Mans (buildMans) +import Utility.SafeCommand + +main :: IO () +main = defaultMainWithHooks simpleUserHooks + { preConf = \_ _ -> do + Configure.run Configure.tests + return (Nothing, []) + , postCopy = myPostCopy + } + +myPostCopy :: Args -> CopyFlags -> PackageDescription -> LocalBuildInfo -> IO () +myPostCopy _ flags pkg lbi = when (System.Info.os /= "mingw32") $ do + installGitAnnexLinks dest verbosity pkg lbi + installManpages dest verbosity pkg lbi + installDesktopFile dest verbosity pkg lbi + where + dest = fromFlag $ copyDest flags + verbosity = fromFlag $ copyVerbosity flags + +installGitAnnexLinks :: CopyDest -> Verbosity -> PackageDescription -> LocalBuildInfo -> IO () +installGitAnnexLinks copyDest verbosity pkg lbi = do + rawSystemExit verbosity "ln" + ["-sf", "git-annex", dstBinDir "git-annex-shell"] + rawSystemExit verbosity "ln" + ["-sf", "git-annex", dstBinDir "git-remote-tor-annex"] + where + dstBinDir = bindir $ absoluteInstallDirs pkg lbi copyDest + +{- See http://www.haskell.org/haskellwiki/Cabal/Developer-FAQ#Installing_manpages -} +installManpages :: CopyDest -> Verbosity -> PackageDescription -> LocalBuildInfo -> IO () +installManpages copyDest verbosity pkg lbi = + installOrdinaryFiles verbosity dstManDir =<< srcManpages + where + dstManDir = mandir (absoluteInstallDirs pkg lbi copyDest) "man1" + -- If mdwn2man fails, perhaps because perl is not available, + -- we just skip installing man pages. + srcManpages = zip (repeat "man") . map takeFileName . catMaybes + <$> buildMans + +installDesktopFile :: CopyDest -> Verbosity -> PackageDescription -> LocalBuildInfo -> IO () +installDesktopFile copyDest _verbosity pkg lbi + | progfile copyDest == progfile NoCopyDest = + DesktopFile.installUser (progfile copyDest) + `catch` installerror + | otherwise = return () + where + progfile cd = bindir (absoluteInstallDirs pkg lbi cd) "git-annex" + installerror :: SomeException -> IO () + installerror e = putStrLn ("Warning: Installation of desktop integration files did not succeed (" ++ show e ++ "); skipping.") diff --git a/Test.hs b/Test.hs new file mode 100644 index 0000000000..463f06e4dd --- /dev/null +++ b/Test.hs @@ -0,0 +1,1725 @@ +{- git-annex test suite + - + - Copyright 2010-2017 Joey Hess + - + - Licensed under the GNU GPL version 3 or higher. + -} + +{-# LANGUAGE CPP #-} + +module Test where + +import Types.Test +import Test.Framework +import Options.Applicative.Types + +import Test.Tasty +import Test.Tasty.Runners +import Test.Tasty.HUnit +import Test.Tasty.QuickCheck +import Test.Tasty.Ingredients.Rerun +import Options.Applicative (switch, long, help, internal) + +import qualified Data.Map as M +import qualified Data.ByteString.Lazy.UTF8 as BU8 +import System.Environment + +import Common +import CmdLine.GitAnnex.Options + +import qualified Utility.SafeCommand +import qualified Annex +import qualified Annex.Version +import qualified Git.Filename +import qualified Git.Types +import qualified Git.Ref +import qualified Git.LsTree +import qualified Git.FilePath +import qualified Annex.Locations +#ifndef mingw32_HOST_OS +import qualified Types.GitConfig +#endif +import qualified Types.TrustLevel +import qualified Types +import qualified Logs.MapLog +import qualified Logs.Trust +import qualified Logs.Remote +import qualified Logs.Unused +import qualified Logs.Transfer +import qualified Logs.Presence +import qualified Logs.PreferredContent +import qualified Types.MetaData +import qualified Remote +import qualified Key +import qualified Config +import qualified Config.Cost +import qualified Crypto +import qualified Database.Keys +import qualified Annex.WorkTree +import qualified Annex.Init +import qualified Annex.CatFile +import qualified Annex.Path +import qualified Annex.AdjustedBranch +import qualified Annex.VectorClock +import qualified Annex.View +import qualified Annex.View.ViewedFile +import qualified Logs.View +import qualified Utility.Path +import qualified Utility.FileMode +import qualified BuildInfo +import qualified Utility.Format +import qualified Utility.Verifiable +import qualified Utility.Process +import qualified Utility.Misc +import qualified Utility.InodeCache +import qualified Utility.Env +import qualified Utility.Env.Set +import qualified Utility.Matcher +import qualified Utility.Hash +import qualified Utility.Scheduled +import qualified Utility.Scheduled.QuickCheck +import qualified Utility.HumanTime +import qualified Utility.Base64 +import qualified Utility.Tmp.Dir +import qualified Utility.FileSystemEncoding +import qualified Utility.Aeson +#ifndef mingw32_HOST_OS +import qualified Remote.Helper.Encryptable +import qualified Types.Crypto +import qualified Utility.Gpg +import qualified Types.Remote +#endif + +optParser :: Parser TestOptions +optParser = TestOptions + <$> suiteOptionParser ingredients (tests False mempty) + <*> switch + ( long "keep-failures" + <> help "preserve repositories on test failure" + ) + <*> switch + ( long "fakessh" + <> internal + ) + <*> cmdParams "non-options are for internal use only" + +runner :: Maybe (TestOptions -> IO ()) +runner = Just go + where + go opts + | fakeSsh opts = runFakeSsh (internalData opts) + | otherwise = runsubprocesstests opts + =<< Utility.Env.getEnv subenv + + -- Run git-annex test in a subprocess, so that any files + -- it may open will be closed before running finalCleanup. + -- This should prevent most failures to clean up after the test + -- suite. + subenv = "GIT_ANNEX_TEST_SUBPROCESS" + runsubprocesstests opts Nothing = do + pp <- Annex.Path.programPath + Utility.Env.Set.setEnv subenv "1" True + ps <- getArgs + (Nothing, Nothing, Nothing, pid) <-createProcess (proc pp ps) + exitcode <- waitForProcess pid + unless (keepFailuresOption opts) finalCleanup + exitWith exitcode + runsubprocesstests opts (Just _) = isolateGitConfig $ do + ensuretmpdir + crippledfilesystem <- fst <$> Annex.Init.probeCrippledFileSystem' tmpdir + case tryIngredients ingredients (tastyOptionSet opts) (tests crippledfilesystem opts) of + Nothing -> error "No tests found!?" + Just act -> ifM act + ( exitSuccess + , do + putStrLn " (Failures above could be due to a bug in git-annex, or an incompatibility" + putStrLn " with utilities, such as git, installed on this system.)" + exitFailure + ) + +ingredients :: [Ingredient] +ingredients = + [ listingTests + , rerunningTests [consoleTestReporter] + ] + +tests :: Bool -> TestOptions -> TestTree +tests crippledfilesystem opts = testGroup "Tests" $ properties : + map (\(d, te) -> withTestMode te initTests (unitTests d)) testmodes + where + testmodes = catMaybes + [ Just ("v6 unlocked", (testMode opts "6") { unlockedFiles = True }) + , unlesscrippled ("v5", testMode opts "5") + , unlesscrippled ("v6 locked", testMode opts "6") + , Just ("v5 direct", (testMode opts "5") { forceDirect = True }) + ] + unlesscrippled v + | crippledfilesystem = Nothing + | otherwise = Just v + +properties :: TestTree +properties = localOption (QuickCheckTests 1000) $ testGroup "QuickCheck" + [ testProperty "prop_encode_decode_roundtrip" Git.Filename.prop_encode_decode_roundtrip + , testProperty "prop_encode_c_decode_c_roundtrip" Utility.Format.prop_encode_c_decode_c_roundtrip + , testProperty "prop_isomorphic_fileKey" Annex.Locations.prop_isomorphic_fileKey + , testProperty "prop_isomorphic_key_encode" Key.prop_isomorphic_key_encode + , testProperty "prop_isomorphic_key_decode" Key.prop_isomorphic_key_decode + , testProperty "prop_isomorphic_shellEscape" Utility.SafeCommand.prop_isomorphic_shellEscape + , testProperty "prop_isomorphic_shellEscape_multiword" Utility.SafeCommand.prop_isomorphic_shellEscape_multiword + , testProperty "prop_isomorphic_configEscape" Logs.Remote.prop_isomorphic_configEscape + , testProperty "prop_parse_show_Config" Logs.Remote.prop_parse_show_Config + , testProperty "prop_upFrom_basics" Utility.Path.prop_upFrom_basics + , testProperty "prop_relPathDirToFile_basics" Utility.Path.prop_relPathDirToFile_basics + , testProperty "prop_relPathDirToFile_regressionTest" Utility.Path.prop_relPathDirToFile_regressionTest + , testProperty "prop_cost_sane" Config.Cost.prop_cost_sane + , testProperty "prop_matcher_sane" Utility.Matcher.prop_matcher_sane + , testProperty "prop_HmacSha1WithCipher_sane" Crypto.prop_HmacSha1WithCipher_sane + , testProperty "prop_VectorClock_sane" Annex.VectorClock.prop_VectorClock_sane + , testProperty "prop_addMapLog_sane" Logs.MapLog.prop_addMapLog_sane + , testProperty "prop_verifiable_sane" Utility.Verifiable.prop_verifiable_sane + , testProperty "prop_segment_regressionTest" Utility.Misc.prop_segment_regressionTest + , testProperty "prop_read_write_transferinfo" Logs.Transfer.prop_read_write_transferinfo + , testProperty "prop_read_show_inodecache" Utility.InodeCache.prop_read_show_inodecache + , testProperty "prop_parse_show_log" Logs.Presence.prop_parse_show_log + , testProperty "prop_read_show_TrustLevel" Types.TrustLevel.prop_read_show_TrustLevel + , testProperty "prop_parse_show_TrustLog" Logs.Trust.prop_parse_show_TrustLog + , testProperty "prop_hashes_stable" Utility.Hash.prop_hashes_stable + , testProperty "prop_mac_stable" Utility.Hash.prop_mac_stable + , testProperty "prop_schedule_roundtrips" Utility.Scheduled.QuickCheck.prop_schedule_roundtrips + , testProperty "prop_past_sane" Utility.Scheduled.prop_past_sane + , testProperty "prop_duration_roundtrips" Utility.HumanTime.prop_duration_roundtrips + , testProperty "prop_metadata_sane" Types.MetaData.prop_metadata_sane + , testProperty "prop_metadata_serialize" Types.MetaData.prop_metadata_serialize + , testProperty "prop_branchView_legal" Logs.View.prop_branchView_legal + , testProperty "prop_viewPath_roundtrips" Annex.View.prop_viewPath_roundtrips + , testProperty "prop_view_roundtrips" Annex.View.prop_view_roundtrips + , testProperty "prop_viewedFile_rountrips" Annex.View.ViewedFile.prop_viewedFile_roundtrips + , testProperty "prop_b64_roundtrips" Utility.Base64.prop_b64_roundtrips + , testProperty "prop_standardGroups_parse" Logs.PreferredContent.prop_standardGroups_parse + ] + +{- These tests set up the test environment, but also test some basic parts + - of git-annex. They are always run before the unitTests. -} +initTests :: TestTree +initTests = testGroup "Init Tests" + [ testCase "init" test_init + , testCase "add" test_add + ] + +unitTests :: String -> TestTree +unitTests note = testGroup ("Unit Tests " ++ note) + [ testCase "add dup" test_add_dup + , testCase "add extras" test_add_extras + , testCase "shared clone" test_shared_clone + , testCase "log" test_log + , testCase "import" test_import + , testCase "reinject" test_reinject + , testCase "unannex (no copy)" test_unannex_nocopy + , testCase "unannex (with copy)" test_unannex_withcopy + , testCase "drop (no remote)" test_drop_noremote + , testCase "drop (with remote)" test_drop_withremote + , testCase "drop (untrusted remote)" test_drop_untrustedremote + , testCase "get" test_get + , testCase "get (ssh remote)" test_get_ssh_remote + , testCase "move" test_move + , testCase "move (ssh remote)" test_move_ssh_remote + , testCase "copy" test_copy + , testCase "lock" test_lock + , testCase "lock (v6 --force)" test_lock_v6_force + , testCase "edit (no pre-commit)" test_edit + , testCase "edit (pre-commit)" test_edit_precommit + , testCase "partial commit" test_partial_commit + , testCase "fix" test_fix + , testCase "direct" test_direct + , testCase "trust" test_trust + , testCase "fsck (basics)" test_fsck_basic + , testCase "fsck (bare)" test_fsck_bare + , testCase "fsck (local untrusted)" test_fsck_localuntrusted + , testCase "fsck (remote untrusted)" test_fsck_remoteuntrusted + , testCase "fsck --from remote" test_fsck_fromremote + , testCase "migrate" test_migrate + , testCase "migrate (via gitattributes)" test_migrate_via_gitattributes + , testCase "unused" test_unused + , testCase "describe" test_describe + , testCase "find" test_find + , testCase "merge" test_merge + , testCase "info" test_info + , testCase "version" test_version + , testCase "sync" test_sync + , testCase "union merge regression" test_union_merge_regression + , testCase "adjusted branch merge regression" test_adjusted_branch_merge_regression + , testCase "adjusted branch subtree regression" test_adjusted_branch_subtree_regression + , testCase "conflict resolution" test_conflict_resolution + , testCase "conflict resolution (adjusted branch)" test_conflict_resolution_adjusted_branch + , testCase "conflict resolution movein regression" test_conflict_resolution_movein_regression + , testCase "conflict resolution (mixed directory and file)" test_mixed_conflict_resolution + , testCase "conflict resolution symlink bit" test_conflict_resolution_symlink_bit + , testCase "conflict resolution (uncommitted local file)" test_uncommitted_conflict_resolution + , testCase "conflict resolution (removed file)" test_remove_conflict_resolution + , testCase "conflict resolution (nonannexed file)" test_nonannexed_file_conflict_resolution + , testCase "conflict resolution (nonannexed symlink)" test_nonannexed_symlink_conflict_resolution + , testCase "conflict resolution (mixed locked and unlocked file)" test_mixed_lock_conflict_resolution + , testCase "map" test_map + , testCase "uninit" test_uninit + , testCase "uninit (in git-annex branch)" test_uninit_inbranch + , testCase "upgrade" test_upgrade + , testCase "whereis" test_whereis + , testCase "hook remote" test_hook_remote + , testCase "directory remote" test_directory_remote + , testCase "rsync remote" test_rsync_remote + , testCase "bup remote" test_bup_remote + , testCase "crypto" test_crypto + , testCase "preferred content" test_preferred_content + , testCase "add subdirs" test_add_subdirs + , testCase "addurl" test_addurl + ] + +-- this test case create the main repo +test_init :: Assertion +test_init = innewrepo $ do + ver <- annexVersion <$> getTestMode + if ver == Annex.Version.defaultVersion + then git_annex "init" [reponame] @? "init failed" + else git_annex "init" [reponame, "--version", ver] @? "init failed" + setupTestMode + where + reponame = "test repo" + +-- this test case runs in the main repo, to set up a basic +-- annexed file that later tests will use +test_add :: Assertion +test_add = inmainrepo $ do + writeFile annexedfile $ content annexedfile + add_annex annexedfile @? "add failed" + annexed_present annexedfile + writeFile sha1annexedfile $ content sha1annexedfile + git_annex "add" [sha1annexedfile, "--backend=SHA1"] @? "add with SHA1 failed" + whenM (unlockedFiles <$> getTestMode) $ + git_annex "unlock" [sha1annexedfile] @? "unlock failed" + annexed_present sha1annexedfile + checkbackend sha1annexedfile backendSHA1 + ifM (annexeval Config.isDirect) + ( do + writeFile ingitfile $ content ingitfile + not <$> boolSystem "git" [Param "add", File ingitfile] @? "git add failed to fail in direct mode" + nukeFile ingitfile + git_annex "sync" [] @? "sync failed" + , do + writeFile ingitfile $ content ingitfile + boolSystem "git" [Param "add", File ingitfile] @? "git add failed" + boolSystem "git" [Param "commit", Param "-q", Param "-m", Param "commit"] @? "git commit failed" + git_annex "add" [ingitfile] @? "add ingitfile should be no-op" + unannexed ingitfile + ) + +test_add_dup :: Assertion +test_add_dup = intmpclonerepo $ do + writeFile annexedfiledup $ content annexedfiledup + add_annex annexedfiledup @? "add of second file with same content failed" + annexed_present annexedfiledup + annexed_present annexedfile + +test_add_extras :: Assertion +test_add_extras = intmpclonerepo $ do + writeFile wormannexedfile $ content wormannexedfile + git_annex "add" [wormannexedfile, "--backend=WORM"] @? "add with WORM failed" + whenM (unlockedFiles <$> getTestMode) $ + git_annex "unlock" [wormannexedfile] @? "unlock failed" + annexed_present wormannexedfile + checkbackend wormannexedfile backendWORM + +test_shared_clone :: Assertion +test_shared_clone = intmpsharedclonerepo $ do + v <- catchMaybeIO $ Utility.Process.readProcess "git" + [ "config" + , "--bool" + , "--get" + , "annex.hardlink" + ] + v == Just "true\n" + @? "shared clone of repo did not get annex.hardlink set" + +test_log :: Assertion +test_log = intmpclonerepo $ do + git_annex "log" [annexedfile] @? "log failed" + +test_import :: Assertion +test_import = intmpclonerepo $ Utility.Tmp.Dir.withTmpDir "importtest" $ \importdir -> do + (toimport1, importf1, imported1) <- mktoimport importdir "import1" + git_annex "import" [toimport1] @? "import failed" + annexed_present_imported imported1 + checkdoesnotexist importf1 + + (toimport2, importf2, imported2) <- mktoimport importdir "import2" + git_annex "import" [toimport2] @? "import of duplicate failed" + annexed_present_imported imported2 + checkdoesnotexist importf2 + + (toimport3, importf3, imported3) <- mktoimport importdir "import3" + git_annex "import" ["--skip-duplicates", toimport3] + @? "import of duplicate with --skip-duplicates failed" + checkdoesnotexist imported3 + checkexists importf3 + git_annex "import" ["--clean-duplicates", toimport3] + @? "import of duplicate with --clean-duplicates failed" + checkdoesnotexist imported3 + checkdoesnotexist importf3 + + (toimport4, importf4, imported4) <- mktoimport importdir "import4" + git_annex "import" ["--deduplicate", toimport4] @? "import --deduplicate failed" + checkdoesnotexist imported4 + checkdoesnotexist importf4 + + (toimport5, importf5, imported5) <- mktoimport importdir "import5" + git_annex "import" ["--duplicate", toimport5] @? "import --duplicate failed" + annexed_present_imported imported5 + checkexists importf5 + + git_annex "drop" ["--force", imported1, imported2, imported5] @? "drop failed" + annexed_notpresent_imported imported2 + (toimportdup, importfdup, importeddup) <- mktoimport importdir "importdup" + not <$> git_annex "import" ["--clean-duplicates", toimportdup] + @? "import of missing duplicate with --clean-duplicates failed to fail" + checkdoesnotexist importeddup + checkexists importfdup + where + mktoimport importdir subdir = do + createDirectory (importdir subdir) + let importf = subdir "f" + writeFile (importdir importf) (content importf) + return (importdir subdir, importdir importf, importf) + annexed_present_imported f = ifM (annexeval Config.crippledFileSystem) + ( annexed_present_unlocked f + , annexed_present_locked f + ) + annexed_notpresent_imported f = ifM (annexeval Config.crippledFileSystem) + ( annexed_notpresent_unlocked f + , annexed_notpresent_locked f + ) + +test_reinject :: Assertion +test_reinject = intmpclonerepoInDirect $ do + git_annex "drop" ["--force", sha1annexedfile] @? "drop failed" + annexed_notpresent sha1annexedfile + writeFile tmp $ content sha1annexedfile + key <- Key.key2file <$> getKey backendSHA1 tmp + git_annex "reinject" [tmp, sha1annexedfile] @? "reinject failed" + annexed_present sha1annexedfile + -- fromkey can't be used on a crippled filesystem, since it makes a + -- symlink + unlessM (annexeval Config.crippledFileSystem) $ do + git_annex "fromkey" [key, sha1annexedfiledup] @? "fromkey failed for dup" + annexed_present_locked sha1annexedfiledup + where + tmp = "tmpfile" + +test_unannex_nocopy :: Assertion +test_unannex_nocopy = intmpclonerepo $ do + annexed_notpresent annexedfile + git_annex "unannex" [annexedfile] @? "unannex failed with no copy" + annexed_notpresent annexedfile + +test_unannex_withcopy :: Assertion +test_unannex_withcopy = intmpclonerepo $ do + git_annex "get" [annexedfile] @? "get failed" + annexed_present annexedfile + git_annex "unannex" [annexedfile, sha1annexedfile] @? "unannex failed" + unannexed annexedfile + git_annex "unannex" [annexedfile] @? "unannex failed on non-annexed file" + unannexed annexedfile + unlessM (annexeval Config.isDirect) $ do + git_annex "unannex" [ingitfile] @? "unannex ingitfile should be no-op" + unannexed ingitfile + +test_drop_noremote :: Assertion +test_drop_noremote = intmpclonerepo $ do + git_annex "get" [annexedfile] @? "get failed" + boolSystem "git" [Param "remote", Param "rm", Param "origin"] + @? "git remote rm origin failed" + not <$> git_annex "drop" [annexedfile] @? "drop wrongly succeeded with no known copy of file" + annexed_present annexedfile + git_annex "drop" ["--force", annexedfile] @? "drop --force failed" + annexed_notpresent annexedfile + git_annex "drop" [annexedfile] @? "drop of dropped file failed" + unlessM (annexeval Config.isDirect) $ do + git_annex "drop" [ingitfile] @? "drop ingitfile should be no-op" + unannexed ingitfile + +test_drop_withremote :: Assertion +test_drop_withremote = intmpclonerepo $ do + git_annex "get" [annexedfile] @? "get failed" + annexed_present annexedfile + git_annex "numcopies" ["2"] @? "numcopies config failed" + not <$> git_annex "drop" [annexedfile] @? "drop succeeded although numcopies is not satisfied" + git_annex "numcopies" ["1"] @? "numcopies config failed" + git_annex "drop" [annexedfile] @? "drop failed though origin has copy" + annexed_notpresent annexedfile + -- make sure that the correct symlink is staged for the file + -- after drop + git_annex_expectoutput "status" [] [] + inmainrepo $ annexed_present annexedfile + +test_drop_untrustedremote :: Assertion +test_drop_untrustedremote = intmpclonerepo $ do + git_annex "untrust" ["origin"] @? "untrust of origin failed" + git_annex "get" [annexedfile] @? "get failed" + annexed_present annexedfile + not <$> git_annex "drop" [annexedfile] @? "drop wrongly succeeded with only an untrusted copy of the file" + annexed_present annexedfile + inmainrepo $ annexed_present annexedfile + +test_get :: Assertion +test_get = test_get' intmpclonerepo + +test_get_ssh_remote :: Assertion +test_get_ssh_remote = test_get' (with_ssh_origin intmpclonerepo) + +test_get' :: (Assertion -> Assertion) -> Assertion +test_get' setup = setup $ do + inmainrepo $ annexed_present annexedfile + annexed_notpresent annexedfile + git_annex "get" [annexedfile] @? "get of file failed" + inmainrepo $ annexed_present annexedfile + annexed_present annexedfile + git_annex "get" [annexedfile] @? "get of file already here failed" + inmainrepo $ annexed_present annexedfile + annexed_present annexedfile + unlessM (annexeval Config.isDirect) $ do + inmainrepo $ unannexed ingitfile + unannexed ingitfile + git_annex "get" [ingitfile] @? "get ingitfile should be no-op" + inmainrepo $ unannexed ingitfile + unannexed ingitfile + +test_move :: Assertion +test_move = test_move' intmpclonerepo + +test_move_ssh_remote :: Assertion +test_move_ssh_remote = test_move' (with_ssh_origin intmpclonerepo) + +test_move' :: (Assertion -> Assertion) -> Assertion +test_move' setup = setup $ do + annexed_notpresent annexedfile + inmainrepo $ annexed_present annexedfile + git_annex "move" ["--from", "origin", annexedfile] @? "move --from of file failed" + annexed_present annexedfile + inmainrepo $ annexed_notpresent annexedfile + git_annex "move" ["--from", "origin", annexedfile] @? "move --from of file already here failed" + annexed_present annexedfile + inmainrepo $ annexed_notpresent annexedfile + git_annex "move" ["--to", "origin", annexedfile] @? "move --to of file failed" + inmainrepo $ annexed_present annexedfile + annexed_notpresent annexedfile + git_annex "move" ["--to", "origin", annexedfile] @? "move --to of file already there failed" + inmainrepo $ annexed_present annexedfile + annexed_notpresent annexedfile + unlessM (annexeval Config.isDirect) $ do + unannexed ingitfile + inmainrepo $ unannexed ingitfile + git_annex "move" ["--to", "origin", ingitfile] @? "move of ingitfile should be no-op" + unannexed ingitfile + inmainrepo $ unannexed ingitfile + git_annex "move" ["--from", "origin", ingitfile] @? "move of ingitfile should be no-op" + unannexed ingitfile + inmainrepo $ unannexed ingitfile + +test_copy :: Assertion +test_copy = intmpclonerepo $ do + annexed_notpresent annexedfile + inmainrepo $ annexed_present annexedfile + git_annex "copy" ["--from", "origin", annexedfile] @? "copy --from of file failed" + annexed_present annexedfile + inmainrepo $ annexed_present annexedfile + git_annex "copy" ["--from", "origin", annexedfile] @? "copy --from of file already here failed" + annexed_present annexedfile + inmainrepo $ annexed_present annexedfile + git_annex "copy" ["--to", "origin", annexedfile] @? "copy --to of file already there failed" + annexed_present annexedfile + inmainrepo $ annexed_present annexedfile + git_annex "move" ["--to", "origin", annexedfile] @? "move --to of file already there failed" + annexed_notpresent annexedfile + inmainrepo $ annexed_present annexedfile + unlessM (annexeval Config.isDirect) $ do + unannexed ingitfile + inmainrepo $ unannexed ingitfile + git_annex "copy" ["--to", "origin", ingitfile] @? "copy of ingitfile should be no-op" + unannexed ingitfile + inmainrepo $ unannexed ingitfile + git_annex "copy" ["--from", "origin", ingitfile] @? "copy of ingitfile should be no-op" + checkregularfile ingitfile + checkcontent ingitfile + +test_preferred_content :: Assertion +test_preferred_content = intmpclonerepo $ do + annexed_notpresent annexedfile + -- get/copy --auto looks only at numcopies when preferred content is not + -- set, and with 1 copy existing, does not get the file. + git_annex "get" ["--auto", annexedfile] @? "get --auto of file failed with default preferred content" + annexed_notpresent annexedfile + git_annex "copy" ["--from", "origin", "--auto", annexedfile] @? "copy --auto --from of file failed with default preferred content" + annexed_notpresent annexedfile + + git_annex "wanted" [".", "standard"] @? "set expression to standard failed" + git_annex "group" [".", "client"] @? "set group to standard failed" + git_annex "get" ["--auto", annexedfile] @? "get --auto of file failed for client" + annexed_present annexedfile + git_annex "drop" [annexedfile] @? "drop of file failed" + git_annex "copy" ["--from", "origin", "--auto", annexedfile] @? "copy --auto --from of file failed for client" + annexed_present annexedfile + git_annex "ungroup" [".", "client"] @? "ungroup failed" + + git_annex "wanted" [".", "standard"] @? "set expression to standard failed" + git_annex "group" [".", "manual"] @? "set group to manual failed" + -- drop --auto with manual leaves the file where it is + git_annex "drop" ["--auto", annexedfile] @? "drop --auto of file failed with manual preferred content" + annexed_present annexedfile + git_annex "drop" [annexedfile] @? "drop of file failed" + annexed_notpresent annexedfile + -- copy/get --auto with manual does not get the file + git_annex "get" ["--auto", annexedfile] @? "get --auto of file failed with manual preferred content" + annexed_notpresent annexedfile + git_annex "copy" ["--from", "origin", "--auto", annexedfile] @? "copy --auto --from of file failed with manual preferred content" + annexed_notpresent annexedfile + git_annex "ungroup" [".", "client"] @? "ungroup failed" + + git_annex "wanted" [".", "exclude=*"] @? "set expression to exclude=* failed" + git_annex "get" [annexedfile] @? "get of file failed" + annexed_present annexedfile + git_annex "drop" ["--auto", annexedfile] @? "drop --auto of file failed with exclude=*" + annexed_notpresent annexedfile + git_annex "get" ["--auto", annexedfile] @? "get --auto of file failed with exclude=*" + annexed_notpresent annexedfile + +test_lock :: Assertion +test_lock = intmpclonerepoInDirect $ do + annexed_notpresent annexedfile + unlessM (annexeval Annex.Version.versionSupportsUnlockedPointers) $ + ifM (unlockedFiles <$> getTestMode) + ( not <$> git_annex "lock" [annexedfile] @? "lock failed to fail with not present file" + , not <$> git_annex "unlock" [annexedfile] @? "unlock failed to fail with not present file" + ) + annexed_notpresent annexedfile + + -- regression test: unlock of newly added, not committed file + -- should fail in v5 mode. In v6 mode, this is allowed. + writeFile "newfile" "foo" + git_annex "add" ["newfile"] @? "add new file failed" + ifM (annexeval Annex.Version.versionSupportsUnlockedPointers) + ( git_annex "unlock" ["newfile"] @? "unlock failed on newly added, never committed file in v6 repository" + , not <$> git_annex "unlock" ["newfile"] @? "unlock failed to fail on newly added, never committed file in v5 repository" + ) + + git_annex "get" [annexedfile] @? "get of file failed" + annexed_present annexedfile + git_annex "unlock" [annexedfile] @? "unlock failed" + unannexed annexedfile + -- write different content, to verify that lock + -- throws it away + changecontent annexedfile + writeFile annexedfile $ content annexedfile ++ "foo" + not <$> git_annex "lock" [annexedfile] @? "lock failed to fail without --force" + git_annex "lock" ["--force", annexedfile] @? "lock --force failed" + -- In v6 mode, the original content of the file is not always + -- preserved after modification, so re-get it. + git_annex "get" [annexedfile] @? "get of file failed after lock --force" + annexed_present_locked annexedfile + git_annex "unlock" [annexedfile] @? "unlock failed" + unannexed annexedfile + changecontent annexedfile + ifM (annexeval Annex.Version.versionSupportsUnlockedPointers) + ( do + boolSystem "git" [Param "add", Param annexedfile] @? "add of modified file failed" + runchecks [checkregularfile, checkwritable] annexedfile + , do + git_annex "add" [annexedfile] @? "add of modified file failed" + runchecks [checklink, checkunwritable] annexedfile + ) + c <- readFile annexedfile + assertEqual "content of modified file" c (changedcontent annexedfile) + r' <- git_annex "drop" [annexedfile] + not r' @? "drop wrongly succeeded with no known copy of modified file" + +-- Regression test: lock --force when work tree file +-- was modified lost the (unmodified) annex object. +-- (Only occurred when the keys database was out of sync.) +test_lock_v6_force :: Assertion +test_lock_v6_force = intmpclonerepoInDirect $ do + git_annex "upgrade" [] @? "upgrade failed" + whenM (annexeval Annex.Version.versionSupportsUnlockedPointers) $ do + git_annex "get" [annexedfile] @? "get of file failed" + git_annex "unlock" [annexedfile] @? "unlock failed in v6 mode" + annexeval $ do + Database.Keys.closeDb + dbdir <- Annex.fromRepo Annex.Locations.gitAnnexKeysDb + liftIO $ renameDirectory dbdir (dbdir ++ ".old") + writeFile annexedfile "test_lock_v6_force content" + not <$> git_annex "lock" [annexedfile] @? "lock of modified file failed to fail in v6 mode" + git_annex "lock" ["--force", annexedfile] @? "lock --force of modified file failed in v6 mode" + annexed_present_locked annexedfile + +test_edit :: Assertion +test_edit = test_edit' False + +test_edit_precommit :: Assertion +test_edit_precommit = test_edit' True + +test_edit' :: Bool -> Assertion +test_edit' precommit = intmpclonerepoInDirect $ do + git_annex "get" [annexedfile] @? "get of file failed" + annexed_present annexedfile + git_annex "edit" [annexedfile] @? "edit failed" + unannexed annexedfile + changecontent annexedfile + boolSystem "git" [Param "add", File annexedfile] + @? "git add of edited file failed" + if precommit + then git_annex "pre-commit" [] + @? "pre-commit failed" + else boolSystem "git" [Param "commit", Param "-q", Param "-m", Param "contentchanged"] + @? "git commit of edited file failed" + ifM (annexeval Annex.Version.versionSupportsUnlockedPointers) + ( runchecks [checkregularfile, checkwritable] annexedfile + , runchecks [checklink, checkunwritable] annexedfile + ) + c <- readFile annexedfile + assertEqual "content of modified file" c (changedcontent annexedfile) + not <$> git_annex "drop" [annexedfile] @? "drop wrongly succeeded with no known copy of modified file" + +test_partial_commit :: Assertion +test_partial_commit = intmpclonerepoInDirect $ do + git_annex "get" [annexedfile] @? "get of file failed" + annexed_present annexedfile + git_annex "unlock" [annexedfile] @? "unlock failed" + changecontent annexedfile + ifM (annexeval Annex.Version.versionSupportsUnlockedPointers) + ( boolSystem "git" [Param "commit", Param "-q", Param "-m", Param "test", File annexedfile] + @? "partial commit of unlocked file should be allowed in v6 repository" + , not <$> boolSystem "git" [Param "commit", Param "-q", Param "-m", Param "test", File annexedfile] + @? "partial commit of unlocked file not blocked by pre-commit hook" + ) + +test_fix :: Assertion +test_fix = intmpclonerepoInDirect $ unlessM (unlockedFiles <$> getTestMode) $ do + annexed_notpresent annexedfile + git_annex "fix" [annexedfile] @? "fix of not present failed" + annexed_notpresent annexedfile + git_annex "get" [annexedfile] @? "get of file failed" + annexed_present annexedfile + git_annex "fix" [annexedfile] @? "fix of present file failed" + annexed_present annexedfile + createDirectory subdir + boolSystem "git" [Param "mv", File annexedfile, File subdir] + @? "git mv failed" + git_annex "fix" [newfile] @? "fix of moved file failed" + runchecks [checklink, checkunwritable] newfile + c <- readFile newfile + assertEqual "content of moved file" c (content annexedfile) + where + subdir = "s" + newfile = subdir ++ "/" ++ annexedfile + +test_direct :: Assertion +test_direct = intmpclonerepoInDirect $ do + git_annex "get" [annexedfile] @? "get of file failed" + annexed_present annexedfile + ifM (annexeval Annex.Version.versionSupportsUnlockedPointers) + ( not <$> git_annex "direct" [] @? "switch to direct mode failed to fail in v6 repository" + , do + git_annex "direct" [] @? "switch to direct mode failed" + annexed_present annexedfile + git_annex "indirect" [] @? "switch to indirect mode failed" + ) + +test_trust :: Assertion +test_trust = intmpclonerepo $ do + git_annex "trust" [repo] @? "trust failed" + trustcheck Logs.Trust.Trusted "trusted 1" + git_annex "trust" [repo] @? "trust of trusted failed" + trustcheck Logs.Trust.Trusted "trusted 2" + git_annex "untrust" [repo] @? "untrust failed" + trustcheck Logs.Trust.UnTrusted "untrusted 1" + git_annex "untrust" [repo] @? "untrust of untrusted failed" + trustcheck Logs.Trust.UnTrusted "untrusted 2" + git_annex "dead" [repo] @? "dead failed" + trustcheck Logs.Trust.DeadTrusted "deadtrusted 1" + git_annex "dead" [repo] @? "dead of dead failed" + trustcheck Logs.Trust.DeadTrusted "deadtrusted 2" + git_annex "semitrust" [repo] @? "semitrust failed" + trustcheck Logs.Trust.SemiTrusted "semitrusted 1" + git_annex "semitrust" [repo] @? "semitrust of semitrusted failed" + trustcheck Logs.Trust.SemiTrusted "semitrusted 2" + where + repo = "origin" + trustcheck expected msg = do + present <- annexeval $ do + l <- Logs.Trust.trustGet expected + u <- Remote.nameToUUID repo + return $ u `elem` l + assertBool msg present + +test_fsck_basic :: Assertion +test_fsck_basic = intmpclonerepo $ do + git_annex "fsck" [] @? "fsck failed" + git_annex "numcopies" ["2"] @? "numcopies config failed" + fsck_should_fail "numcopies unsatisfied" + git_annex "numcopies" ["1"] @? "numcopies config failed" + corrupt annexedfile + corrupt sha1annexedfile + where + corrupt f = do + git_annex "get" [f] @? "get of file failed" + Utility.FileMode.allowWrite f + writeFile f (changedcontent f) + ifM (annexeval Config.isDirect <||> unlockedFiles <$> getTestMode) + ( git_annex "fsck" [] @? "fsck failed on unlocked file with changed file content" + , not <$> git_annex "fsck" [] @? "fsck failed to fail with corrupted file content" + ) + git_annex "fsck" [] @? "fsck unexpectedly failed again; previous one did not fix problem with " ++ f + +test_fsck_bare :: Assertion +test_fsck_bare = intmpbareclonerepo $ + git_annex "fsck" [] @? "fsck failed" + +test_fsck_localuntrusted :: Assertion +test_fsck_localuntrusted = intmpclonerepo $ do + git_annex "get" [annexedfile] @? "get failed" + git_annex "untrust" ["origin"] @? "untrust of origin repo failed" + git_annex "untrust" ["."] @? "untrust of current repo failed" + fsck_should_fail "content only available in untrusted (current) repository" + git_annex "trust" ["."] @? "trust of current repo failed" + git_annex "fsck" [annexedfile] @? "fsck failed on file present in trusted repo" + +test_fsck_remoteuntrusted :: Assertion +test_fsck_remoteuntrusted = intmpclonerepo $ do + git_annex "numcopies" ["2"] @? "numcopies config failed" + git_annex "get" [annexedfile] @? "get failed" + git_annex "get" [sha1annexedfile] @? "get failed" + git_annex "fsck" [] @? "fsck failed with numcopies=2 and 2 copies" + git_annex "untrust" ["origin"] @? "untrust of origin failed" + fsck_should_fail "content not replicated to enough non-untrusted repositories" + +test_fsck_fromremote :: Assertion +test_fsck_fromremote = intmpclonerepo $ do + git_annex "fsck" ["--from", "origin"] @? "fsck --from origin failed" + +fsck_should_fail :: String -> Assertion +fsck_should_fail m = not <$> git_annex "fsck" [] + @? "fsck failed to fail with " ++ m + +test_migrate :: Assertion +test_migrate = test_migrate' False + +test_migrate_via_gitattributes :: Assertion +test_migrate_via_gitattributes = test_migrate' True + +test_migrate' :: Bool -> Assertion +test_migrate' usegitattributes = intmpclonerepoInDirect $ do + annexed_notpresent annexedfile + annexed_notpresent sha1annexedfile + git_annex "migrate" [annexedfile] @? "migrate of not present failed" + git_annex "migrate" [sha1annexedfile] @? "migrate of not present failed" + git_annex "get" [annexedfile] @? "get of file failed" + git_annex "get" [sha1annexedfile] @? "get of file failed" + annexed_present annexedfile + annexed_present sha1annexedfile + if usegitattributes + then do + writeFile ".gitattributes" "* annex.backend=SHA1" + git_annex "migrate" [sha1annexedfile] + @? "migrate sha1annexedfile failed" + git_annex "migrate" [annexedfile] + @? "migrate annexedfile failed" + else do + git_annex "migrate" [sha1annexedfile, "--backend", "SHA1"] + @? "migrate sha1annexedfile failed" + git_annex "migrate" [annexedfile, "--backend", "SHA1"] + @? "migrate annexedfile failed" + annexed_present annexedfile + annexed_present sha1annexedfile + checkbackend annexedfile backendSHA1 + checkbackend sha1annexedfile backendSHA1 + + -- check that reversing a migration works + writeFile ".gitattributes" "* annex.backend=SHA256" + git_annex "migrate" [sha1annexedfile] + @? "migrate sha1annexedfile failed" + git_annex "migrate" [annexedfile] + @? "migrate annexedfile failed" + annexed_present annexedfile + annexed_present sha1annexedfile + checkbackend annexedfile backendSHA256 + checkbackend sha1annexedfile backendSHA256 + +test_unused :: Assertion +-- This test is broken in direct mode +test_unused = intmpclonerepoInDirect $ do + checkunused [] "in new clone" + git_annex "get" [annexedfile] @? "get of file failed" + git_annex "get" [sha1annexedfile] @? "get of file failed" + annexedfilekey <- getKey backendSHA256E annexedfile + sha1annexedfilekey <- getKey backendSHA1 sha1annexedfile + checkunused [] "after get" + boolSystem "git" [Param "rm", Param "-fq", File annexedfile] @? "git rm failed" + checkunused [] "after rm" + boolSystem "git" [Param "commit", Param "-q", Param "-m", Param "foo"] @? "git commit failed" + checkunused [] "after commit" + -- unused checks origin/master; once it's gone it is really unused + boolSystem "git" [Param "remote", Param "rm", Param "origin"] @? "git remote rm origin failed" + checkunused [annexedfilekey] "after origin branches are gone" + boolSystem "git" [Param "rm", Param "-fq", File sha1annexedfile] @? "git rm failed" + boolSystem "git" [Param "commit", Param "-q", Param "-m", Param "foo"] @? "git commit failed" + checkunused [annexedfilekey, sha1annexedfilekey] "after rm sha1annexedfile" + + -- good opportunity to test dropkey also + git_annex "dropkey" ["--force", Key.key2file annexedfilekey] + @? "dropkey failed" + checkunused [sha1annexedfilekey] ("after dropkey --force " ++ Key.key2file annexedfilekey) + + not <$> git_annex "dropunused" ["1"] @? "dropunused failed to fail without --force" + git_annex "dropunused" ["--force", "1"] @? "dropunused failed" + checkunused [] "after dropunused" + not <$> git_annex "dropunused" ["--force", "10", "501"] @? "dropunused failed to fail on bogus numbers" + + -- Unused used to miss renamed symlinks that were not staged + -- and pointed at annexed content, and think that content was unused. + -- This is only relevant when using locked files; if the file is + -- unlocked, the work tree file has the content, and there's no way + -- to associate it with the key. + unlessM (unlockedFiles <$> getTestMode) $ do + writeFile "unusedfile" "unusedcontent" + git_annex "add" ["unusedfile"] @? "add of unusedfile failed" + unusedfilekey <- getKey backendSHA256E "unusedfile" + renameFile "unusedfile" "unusedunstagedfile" + boolSystem "git" [Param "rm", Param "-qf", File "unusedfile"] @? "git rm failed" + checkunused [] "with unstaged link" + removeFile "unusedunstagedfile" + checkunused [unusedfilekey] "with renamed link deleted" + + -- unused used to miss symlinks that were deleted or modified + -- manually + writeFile "unusedfile" "unusedcontent" + git_annex "add" ["unusedfile"] @? "add of unusedfile failed" + boolSystem "git" [Param "add", File "unusedfile"] @? "git add failed" + unusedfilekey' <- getKey backendSHA256E "unusedfile" + checkunused [] "with staged deleted link" + boolSystem "git" [Param "rm", Param "-qf", File "unusedfile"] @? "git rm failed" + checkunused [unusedfilekey'] "with staged link deleted" + + -- unused used to false positive on symlinks that were + -- deleted or modified manually, but not staged as such + writeFile "unusedfile" "unusedcontent" + git_annex "add" ["unusedfile"] @? "add of unusedfile failed" + boolSystem "git" [Param "add", File "unusedfile"] @? "git add failed" + checkunused [] "with staged file" + removeFile "unusedfile" + checkunused [] "with staged deleted file" + + -- When an unlocked file is modified, git diff will cause git-annex + -- to add its content to the repository. Make sure that's not + -- found as unused. + whenM (unlockedFiles <$> getTestMode) $ do + let f = "unlockedfile" + writeFile f "unlockedcontent1" + boolSystem "git" [Param "add", File "unlockedfile"] @? "git add failed" + checkunused [] "with unlocked file before modification" + writeFile f "unlockedcontent2" + checkunused [] "with unlocked file after modification" + not <$> boolSystem "git" [Param "diff", Param "--quiet", File f] @? "git diff did not show changes to unlocked file" + -- still nothing unused because one version is in the index + -- and the other is in the work tree + checkunused [] "with unlocked file after git diff" + where + checkunused expectedkeys desc = do + git_annex "unused" [] @? "unused failed" + unusedmap <- annexeval $ Logs.Unused.readUnusedMap "" + let unusedkeys = M.elems unusedmap + assertEqual ("unused keys differ " ++ desc) + (sort expectedkeys) (sort unusedkeys) + +test_describe :: Assertion +test_describe = intmpclonerepo $ do + git_annex "describe" [".", "this repo"] @? "describe 1 failed" + git_annex "describe" ["origin", "origin repo"] @? "describe 2 failed" + +test_find :: Assertion +test_find = intmpclonerepo $ do + annexed_notpresent annexedfile + git_annex_expectoutput "find" [] [] + git_annex "get" [annexedfile] @? "get failed" + annexed_present annexedfile + annexed_notpresent sha1annexedfile + git_annex_expectoutput "find" [] [annexedfile] + git_annex_expectoutput "find" ["--exclude", annexedfile, "--and", "--exclude", sha1annexedfile] [] + git_annex_expectoutput "find" ["--include", annexedfile] [annexedfile] + git_annex_expectoutput "find" ["--not", "--in", "origin"] [] + git_annex_expectoutput "find" ["--copies", "1", "--and", "--not", "--copies", "2"] [sha1annexedfile] + git_annex_expectoutput "find" ["--inbackend", "SHA1"] [sha1annexedfile] + git_annex_expectoutput "find" ["--inbackend", "WORM"] [] + + {- --include=* should match files in subdirectories too, + - and --exclude=* should exclude them. -} + createDirectory "dir" + writeFile "dir/subfile" "subfile" + git_annex "add" ["dir"] @? "add of subdir failed" + git_annex_expectoutput "find" ["--include", "*", "--exclude", annexedfile, "--exclude", sha1annexedfile] ["dir/subfile"] + git_annex_expectoutput "find" ["--exclude", "*"] [] + +test_merge :: Assertion +test_merge = intmpclonerepo $ + git_annex "merge" [] @? "merge failed" + +test_info :: Assertion +test_info = intmpclonerepo $ do + json <- BU8.fromString <$> git_annex_output "info" ["--json"] + case Utility.Aeson.eitherDecode json :: Either String Utility.Aeson.Value of + Right _ -> return () + Left e -> assertFailure e + +test_version :: Assertion +test_version = intmpclonerepo $ + git_annex "version" [] @? "version failed" + +test_sync :: Assertion +test_sync = intmpclonerepo $ do + git_annex "sync" [] @? "sync failed" + {- Regression test for bug fixed in + - 7b0970b340d7faeb745c666146c7f701ec71808f, where in direct mode + - sync committed the symlink standin file to the annex. -} + git_annex_expectoutput "find" ["--in", "."] [] + {- Regression test for bug fixed in + - 039e83ed5d1a11fd562cce55b8429c840d72443e, where a present + - wanted file was dropped. -} + git_annex "get" [annexedfile] @? "get failed" + git_annex_expectoutput "find" ["--in", "."] [annexedfile] + git_annex "wanted" [".", "present"] @? "wanted failed" + git_annex "sync" ["--content"] @? "sync failed" + git_annex_expectoutput "find" ["--in", "."] [annexedfile] + git_annex "drop" [annexedfile] @? "drop failed" + git_annex_expectoutput "find" ["--in", "."] [] + git_annex "sync" ["--content"] @? "sync failed" + git_annex_expectoutput "find" ["--in", "."] [] + +{- Regression test for union merge bug fixed in + - 0214e0fb175a608a49b812d81b4632c081f63027 -} +test_union_merge_regression :: Assertion +test_union_merge_regression = + {- We need 3 repos to see this bug. -} + withtmpclonerepo $ \r1 -> + withtmpclonerepo $ \r2 -> + withtmpclonerepo $ \r3 -> do + forM_ [r1, r2, r3] $ \r -> indir r $ do + when (r /= r1) $ + boolSystem "git" [Param "remote", Param "add", Param "r1", File ("../../" ++ r1)] @? "remote add" + when (r /= r2) $ + boolSystem "git" [Param "remote", Param "add", Param "r2", File ("../../" ++ r2)] @? "remote add" + when (r /= r3) $ + boolSystem "git" [Param "remote", Param "add", Param "r3", File ("../../" ++ r3)] @? "remote add" + git_annex "get" [annexedfile] @? "get failed" + boolSystem "git" [Param "remote", Param "rm", Param "origin"] @? "remote rm" + forM_ [r3, r2, r1] $ \r -> indir r $ + git_annex "sync" [] @? ("sync failed in " ++ r) + forM_ [r3, r2] $ \r -> indir r $ + git_annex "drop" ["--force", annexedfile] @? ("drop failed in " ++ r) + indir r1 $ do + git_annex "sync" [] @? "sync failed in r1" + git_annex_expectoutput "find" ["--in", "r3"] [] + {- This was the bug. The sync + - mangled location log data and it + - thought the file was still in r2 -} + git_annex_expectoutput "find" ["--in", "r2"] [] + +{- Regression test for the automatic conflict resolution bug fixed + - in f4ba19f2b8a76a1676da7bb5850baa40d9c388e2. -} +test_conflict_resolution_movein_regression :: Assertion +test_conflict_resolution_movein_regression = withtmpclonerepo $ \r1 -> + withtmpclonerepo $ \r2 -> do + let rname r = if r == r1 then "r1" else "r2" + forM_ [r1, r2] $ \r -> indir r $ do + {- Get all files, see check below. -} + git_annex "get" [] @? "get failed" + disconnectOrigin + pair r1 r2 + forM_ [r1, r2] $ \r -> indir r $ do + {- Set up a conflict. -} + let newcontent = content annexedfile ++ rname r + ifM (annexeval Config.isDirect) + ( writeFile annexedfile newcontent + , do + git_annex "unlock" [annexedfile] @? "unlock failed" + writeFile annexedfile newcontent + ) + {- Sync twice in r1 so it gets the conflict resolution + - update from r2 -} + forM_ [r1, r2, r1] $ \r -> indir r $ + git_annex "sync" ["--force"] @? "sync failed in " ++ rname r + {- After the sync, it should be possible to get all + - files. This includes both sides of the conflict, + - although the filenames are not easily predictable. + - + - The bug caused, in direct mode, one repo to + - be missing the content of the file that had + - been put in it. -} + forM_ [r1, r2] $ \r -> indir r $ do + git_annex "get" [] @? "unable to get all files after merge conflict resolution in " ++ rname r + +{- Simple case of conflict resolution; 2 different versions of annexed + - file. -} +test_conflict_resolution :: Assertion +test_conflict_resolution = + withtmpclonerepo $ \r1 -> + withtmpclonerepo $ \r2 -> do + indir r1 $ do + disconnectOrigin + writeFile conflictor "conflictor1" + add_annex conflictor @? "add conflicter failed" + git_annex "sync" [] @? "sync failed in r1" + indir r2 $ do + disconnectOrigin + writeFile conflictor "conflictor2" + add_annex conflictor @? "add conflicter failed" + git_annex "sync" [] @? "sync failed in r2" + pair r1 r2 + forM_ [r1,r2,r1] $ \r -> indir r $ + git_annex "sync" [] @? "sync failed" + checkmerge "r1" r1 + checkmerge "r2" r2 + where + conflictor = "conflictor" + variantprefix = conflictor ++ ".variant" + checkmerge what d = do + l <- getDirectoryContents d + let v = filter (variantprefix `isPrefixOf`) l + length v == 2 + @? (what ++ " not exactly 2 variant files in: " ++ show l) + conflictor `notElem` l @? ("conflictor still present after conflict resolution") + indir d $ do + git_annex "get" v @? "get failed" + git_annex_expectoutput "find" v v + +{- Conflict resolution while in an adjusted branch. -} +test_conflict_resolution_adjusted_branch :: Assertion +test_conflict_resolution_adjusted_branch = whenM Annex.AdjustedBranch.isGitVersionSupported $ + withtmpclonerepo $ \r1 -> + withtmpclonerepo $ \r2 -> do + indir r1 $ do + disconnectOrigin + writeFile conflictor "conflictor1" + add_annex conflictor @? "add conflicter failed" + git_annex "sync" [] @? "sync failed in r1" + indir r2 $ do + disconnectOrigin + writeFile conflictor "conflictor2" + add_annex conflictor @? "add conflicter failed" + git_annex "sync" [] @? "sync failed in r2" + -- need v6 to use adjust + git_annex "upgrade" [] @? "upgrade failed" + -- We might be in an adjusted branch + -- already, when eg on a crippled + -- filesystem. So, --force it. + git_annex "adjust" ["--unlock", "--force"] @? "adjust failed" + pair r1 r2 + forM_ [r1,r2,r1] $ \r -> indir r $ + git_annex "sync" [] @? "sync failed" + checkmerge "r1" r1 + checkmerge "r2" r2 + where + conflictor = "conflictor" + variantprefix = conflictor ++ ".variant" + checkmerge what d = do + l <- getDirectoryContents d + let v = filter (variantprefix `isPrefixOf`) l + length v == 2 + @? (what ++ " not exactly 2 variant files in: " ++ show l) + conflictor `notElem` l @? ("conflictor still present after conflict resolution") + indir d $ do + git_annex "get" v @? "get failed" + git_annex_expectoutput "find" v v + +{- Check merge conflict resolution when one side is an annexed + - file, and the other is a directory. -} +test_mixed_conflict_resolution :: Assertion +test_mixed_conflict_resolution = do + check True + check False + where + check inr1 = withtmpclonerepo $ \r1 -> + withtmpclonerepo $ \r2 -> do + indir r1 $ do + disconnectOrigin + writeFile conflictor "conflictor" + add_annex conflictor @? "add conflicter failed" + git_annex "sync" [] @? "sync failed in r1" + indir r2 $ do + disconnectOrigin + createDirectory conflictor + writeFile subfile "subfile" + add_annex conflictor @? "add conflicter failed" + git_annex "sync" [] @? "sync failed in r2" + pair r1 r2 + let l = if inr1 then [r1, r2] else [r2, r1] + forM_ l $ \r -> indir r $ + git_annex "sync" [] @? "sync failed in mixed conflict" + checkmerge "r1" r1 + checkmerge "r2" r2 + conflictor = "conflictor" + subfile = conflictor "subfile" + variantprefix = conflictor ++ ".variant" + checkmerge what d = do + doesDirectoryExist (d conflictor) @? (d ++ " conflictor directory missing") + l <- getDirectoryContents d + let v = filter (variantprefix `isPrefixOf`) l + not (null v) + @? (what ++ " conflictor variant file missing in: " ++ show l ) + length v == 1 + @? (what ++ " too many variant files in: " ++ show v) + indir d $ do + git_annex "get" (conflictor:v) @? ("get failed in " ++ what) + git_annex_expectoutput "find" [conflictor] [Git.FilePath.toInternalGitPath subfile] + git_annex_expectoutput "find" v v + +{- Check merge conflict resolution when both repos start with an annexed + - file; one modifies it, and the other deletes it. -} +test_remove_conflict_resolution :: Assertion +test_remove_conflict_resolution = do + check True + check False + where + check inr1 = withtmpclonerepo $ \r1 -> + withtmpclonerepo $ \r2 -> do + indir r1 $ do + disconnectOrigin + writeFile conflictor "conflictor" + add_annex conflictor @? "add conflicter failed" + git_annex "sync" [] @? "sync failed in r1" + indir r2 $ + disconnectOrigin + pair r1 r2 + indir r2 $ do + git_annex "sync" [] @? "sync failed in r2" + git_annex "get" [conflictor] + @? "get conflictor failed" + unlessM (annexeval Config.isDirect) $ do + git_annex "unlock" [conflictor] + @? "unlock conflictor failed" + writeFile conflictor "newconflictor" + indir r1 $ + nukeFile conflictor + let l = if inr1 then [r1, r2, r1] else [r2, r1, r2] + forM_ l $ \r -> indir r $ + git_annex "sync" [] @? "sync failed" + checkmerge "r1" r1 + checkmerge "r2" r2 + conflictor = "conflictor" + variantprefix = conflictor ++ ".variant" + checkmerge what d = do + l <- getDirectoryContents d + let v = filter (variantprefix `isPrefixOf`) l + not (null v) + @? (what ++ " conflictor variant file missing in: " ++ show l ) + length v == 1 + @? (what ++ " too many variant files in: " ++ show v) + +{- Check merge confalict resolution when a file is annexed in one repo, + - and checked directly into git in the other repo. + - + - This test requires indirect mode to set it up, but tests both direct and + - indirect mode. + -} +test_nonannexed_file_conflict_resolution :: Assertion +test_nonannexed_file_conflict_resolution = do + check True False + check False False + check True True + check False True + where + check inr1 switchdirect = withtmpclonerepo $ \r1 -> + withtmpclonerepo $ \r2 -> + whenM (isInDirect r1 <&&> isInDirect r2) $ do + indir r1 $ do + disconnectOrigin + writeFile conflictor "conflictor" + add_annex conflictor @? "add conflicter failed" + git_annex "sync" [] @? "sync failed in r1" + indir r2 $ do + disconnectOrigin + writeFile conflictor nonannexed_content + boolSystem "git" + [ Param "config" + , Param "annex.largefiles" + , Param ("exclude=" ++ ingitfile ++ " and exclude=" ++ conflictor) + ] @? "git config annex.largefiles failed" + boolSystem "git" [Param "add", File conflictor] @? "git add conflictor failed" + git_annex "sync" [] @? "sync failed in r2" + pair r1 r2 + let l = if inr1 then [r1, r2] else [r2, r1] + forM_ l $ \r -> indir r $ do + when switchdirect $ + whenM (annexeval Annex.Version.versionSupportsDirectMode) $ + git_annex "direct" [] @? "failed switching to direct mode" + git_annex "sync" [] @? "sync failed" + checkmerge ("r1" ++ show switchdirect) r1 + checkmerge ("r2" ++ show switchdirect) r2 + conflictor = "conflictor" + nonannexed_content = "nonannexed" + variantprefix = conflictor ++ ".variant" + checkmerge what d = do + l <- getDirectoryContents d + let v = filter (variantprefix `isPrefixOf`) l + not (null v) + @? (what ++ " conflictor variant file missing in: " ++ show l ) + length v == 1 + @? (what ++ " too many variant files in: " ++ show v) + conflictor `elem` l @? (what ++ " conflictor file missing in: " ++ show l) + s <- catchMaybeIO (readFile (d conflictor)) + s == Just nonannexed_content + @? (what ++ " wrong content for nonannexed file: " ++ show s) + + +{- Check merge confalict resolution when a file is annexed in one repo, + - and is a non-git-annex symlink in the other repo. + - + - Test can only run when coreSymlinks is supported, because git needs to + - be able to check out the non-git-annex symlink. + -} +test_nonannexed_symlink_conflict_resolution :: Assertion +test_nonannexed_symlink_conflict_resolution = do + check True False + check False False + check True True + check False True + where + check inr1 switchdirect = withtmpclonerepo $ \r1 -> + withtmpclonerepo $ \r2 -> + whenM (checkRepo (Types.coreSymlinks <$> Annex.getGitConfig) r1 + <&&> isInDirect r1 <&&> isInDirect r2) $ do + indir r1 $ do + disconnectOrigin + writeFile conflictor "conflictor" + add_annex conflictor @? "add conflicter failed" + git_annex "sync" [] @? "sync failed in r1" + indir r2 $ do + disconnectOrigin + createSymbolicLink symlinktarget "conflictor" + boolSystem "git" [Param "add", File conflictor] @? "git add conflictor failed" + git_annex "sync" [] @? "sync failed in r2" + pair r1 r2 + let l = if inr1 then [r1, r2] else [r2, r1] + forM_ l $ \r -> indir r $ do + when switchdirect $ + whenM (annexeval Annex.Version.versionSupportsDirectMode) $ do + git_annex "direct" [] @? "failed switching to direct mode" + git_annex "sync" [] @? "sync failed" + checkmerge ("r1" ++ show switchdirect) r1 + checkmerge ("r2" ++ show switchdirect) r2 + conflictor = "conflictor" + symlinktarget = "dummy-target" + variantprefix = conflictor ++ ".variant" + checkmerge what d = do + l <- getDirectoryContents d + let v = filter (variantprefix `isPrefixOf`) l + not (null v) + @? (what ++ " conflictor variant file missing in: " ++ show l ) + length v == 1 + @? (what ++ " too many variant files in: " ++ show v) + conflictor `elem` l @? (what ++ " conflictor file missing in: " ++ show l) + s <- catchMaybeIO (readSymbolicLink (d conflictor)) + s == Just symlinktarget + @? (what ++ " wrong target for nonannexed symlink: " ++ show s) + +{- Check merge conflict resolution when there is a local file, + - that is not staged or committed, that conflicts with what's being added + - from the remmote. + - + - Case 1: Remote adds file named conflictor; local has a file named + - conflictor. + - + - Case 2: Remote adds conflictor/file; local has a file named conflictor. + -} +test_uncommitted_conflict_resolution :: Assertion +test_uncommitted_conflict_resolution = do + check conflictor + check (conflictor "file") + where + check remoteconflictor = withtmpclonerepo $ \r1 -> + withtmpclonerepo $ \r2 -> do + indir r1 $ do + disconnectOrigin + createDirectoryIfMissing True (parentDir remoteconflictor) + writeFile remoteconflictor annexedcontent + add_annex conflictor @? "add remoteconflicter failed" + git_annex "sync" [] @? "sync failed in r1" + indir r2 $ do + disconnectOrigin + writeFile conflictor localcontent + pair r1 r2 + indir r2 $ ifM (annexeval Config.isDirect) + ( do + git_annex "sync" [] @? "sync failed" + let local = conflictor ++ localprefix + doesFileExist local @? (local ++ " missing after merge") + s <- readFile local + s == localcontent @? (local ++ " has wrong content: " ++ s) + git_annex "get" [conflictor] @? "get failed" + doesFileExist remoteconflictor @? (remoteconflictor ++ " missing after merge") + s' <- readFile remoteconflictor + s' == annexedcontent @? (remoteconflictor ++ " has wrong content: " ++ s) + -- this case is intentionally not handled + -- in indirect mode, since the user + -- can recover on their own easily + , not <$> git_annex "sync" [] @? "sync failed to fail" + ) + conflictor = "conflictor" + localprefix = ".variant-local" + localcontent = "local" + annexedcontent = "annexed" + +{- On Windows/FAT, repeated conflict resolution sometimes + - lost track of whether a file was a symlink. + -} +test_conflict_resolution_symlink_bit :: Assertion +test_conflict_resolution_symlink_bit = unlessM (unlockedFiles <$> getTestMode) $ + withtmpclonerepo $ \r1 -> + withtmpclonerepo $ \r2 -> + withtmpclonerepo $ \r3 -> do + indir r1 $ do + writeFile conflictor "conflictor" + git_annex "add" [conflictor] @? "add conflicter failed" + git_annex "sync" [] @? "sync failed in r1" + check_is_link conflictor "r1" + indir r2 $ do + createDirectory conflictor + writeFile (conflictor "subfile") "subfile" + git_annex "add" [conflictor] @? "add conflicter failed" + git_annex "sync" [] @? "sync failed in r2" + check_is_link (conflictor "subfile") "r2" + indir r3 $ do + writeFile conflictor "conflictor" + git_annex "add" [conflictor] @? "add conflicter failed" + git_annex "sync" [] @? "sync failed in r1" + check_is_link (conflictor "subfile") "r3" + where + conflictor = "conflictor" + check_is_link f what = do + git_annex_expectoutput "find" ["--include=*", f] [Git.FilePath.toInternalGitPath f] + l <- annexeval $ Annex.inRepo $ Git.LsTree.lsTreeFiles Git.Ref.headRef [f] + all (\i -> Git.Types.toTreeItemType (Git.LsTree.mode i) == Just Git.Types.TreeSymlink) l + @? (what ++ " " ++ f ++ " lost symlink bit after merge: " ++ show l) + +{- A v6 unlocked file that conflicts with a locked file should be resolved + - in favor of the unlocked file, with no variant files, as long as they + - both point to the same key. -} +test_mixed_lock_conflict_resolution :: Assertion +test_mixed_lock_conflict_resolution = + withtmpclonerepo $ \r1 -> + withtmpclonerepo $ \r2 -> do + indir r1 $ whenM shouldtest $ do + disconnectOrigin + writeFile conflictor "conflictor" + git_annex "add" [conflictor] @? "add conflicter failed" + git_annex "sync" [] @? "sync failed in r1" + indir r2 $ whenM shouldtest $ do + disconnectOrigin + writeFile conflictor "conflictor" + git_annex "add" [conflictor] @? "add conflicter failed" + git_annex "unlock" [conflictor] @? "unlock conflicter failed" + git_annex "sync" [] @? "sync failed in r2" + pair r1 r2 + forM_ [r1,r2,r1] $ \r -> indir r $ + git_annex "sync" [] @? "sync failed" + checkmerge "r1" r1 + checkmerge "r2" r2 + where + shouldtest = annexeval Annex.Version.versionSupportsUnlockedPointers + conflictor = "conflictor" + variantprefix = conflictor ++ ".variant" + checkmerge what d = indir d $ whenM shouldtest $ do + l <- getDirectoryContents "." + let v = filter (variantprefix `isPrefixOf`) l + length v == 0 + @? (what ++ " not exactly 0 variant files in: " ++ show l) + conflictor `elem` l @? ("conflictor not present after conflict resolution") + git_annex "get" [conflictor] @? "get failed" + git_annex_expectoutput "find" [conflictor] [conflictor] + -- regular file because it's unlocked + checkregularfile conflictor + +{- Regression test for a bad merge between two adjusted branch repos, + - where the same file is added to both independently. The bad merge + - emptied the whole tree. -} +test_adjusted_branch_merge_regression :: Assertion +test_adjusted_branch_merge_regression = whenM Annex.AdjustedBranch.isGitVersionSupported $ + withtmpclonerepo $ \r1 -> + withtmpclonerepo $ \r2 -> do + pair r1 r2 + setup r1 + setup r2 + checkmerge "r1" r1 + checkmerge "r2" r2 + where + conflictor = "conflictor" + setup r = indir r $ do + disconnectOrigin + git_annex "upgrade" [] @? "upgrade failed" + git_annex "adjust" ["--unlock", "--force"] @? "adjust failed" + writeFile conflictor "conflictor" + git_annex "add" [conflictor] @? "add conflicter failed" + git_annex "sync" [] @? "sync failed" + checkmerge what d = indir d $ do + git_annex "sync" [] @? ("sync failed in " ++ what) + l <- getDirectoryContents "." + conflictor `elem` l + @? ("conflictor not present after merge in " ++ what) + +{- Regression test for a bug in adjusted branch syncing code, where adding + - a subtree to an existing tree lost files. -} +test_adjusted_branch_subtree_regression :: Assertion +test_adjusted_branch_subtree_regression = + whenM Annex.AdjustedBranch.isGitVersionSupported $ + withtmpclonerepo $ \r -> do + indir r $ do + disconnectOrigin + git_annex "upgrade" [] @? "upgrade failed" + git_annex "adjust" ["--unlock", "--force"] @? "adjust failed" + createDirectoryIfMissing True "a/b/c" + writeFile "a/b/c/d" "foo" + git_annex "add" ["a/b/c"] @? "add a/b/c failed" + git_annex "sync" [] @? "sync failed" + createDirectoryIfMissing True "a/b/x" + writeFile "a/b/x/y" "foo" + git_annex "add" ["a/b/x"] @? "add a/b/x failed" + git_annex "sync" [] @? "sync failed" + boolSystem "git" [Param "checkout", Param "master"] @? "git checkout master failed" + doesFileExist "a/b/x/y" @? ("a/b/x/y missing from master after adjusted branch sync") + +{- Set up repos as remotes of each other. -} +pair :: FilePath -> FilePath -> Assertion +pair r1 r2 = forM_ [r1, r2] $ \r -> indir r $ do + when (r /= r1) $ + boolSystem "git" [Param "remote", Param "add", Param "r1", File ("../../" ++ r1)] @? "remote add" + when (r /= r2) $ + boolSystem "git" [Param "remote", Param "add", Param "r2", File ("../../" ++ r2)] @? "remote add" + +test_map :: Assertion +test_map = intmpclonerepo $ do + -- set descriptions, that will be looked for in the map + git_annex "describe" [".", "this repo"] @? "describe 1 failed" + git_annex "describe" ["origin", "origin repo"] @? "describe 2 failed" + -- --fast avoids it running graphviz, not a build dependency + git_annex "map" ["--fast"] @? "map failed" + +test_uninit :: Assertion +test_uninit = intmpclonerepo $ do + git_annex "get" [] @? "get failed" + annexed_present annexedfile + _ <- git_annex "uninit" [] -- exit status not checked; does abnormal exit + checkregularfile annexedfile + doesDirectoryExist ".git" @? ".git vanished in uninit" + +test_uninit_inbranch :: Assertion +test_uninit_inbranch = intmpclonerepoInDirect $ do + boolSystem "git" [Param "checkout", Param "git-annex"] @? "git checkout git-annex" + not <$> git_annex "uninit" [] @? "uninit failed to fail when git-annex branch was checked out" + +test_upgrade :: Assertion +test_upgrade = intmpclonerepo $ + git_annex "upgrade" [] @? "upgrade failed" + +test_whereis :: Assertion +test_whereis = intmpclonerepo $ do + annexed_notpresent annexedfile + git_annex "whereis" [annexedfile] @? "whereis on non-present file failed" + git_annex "untrust" ["origin"] @? "untrust failed" + not <$> git_annex "whereis" [annexedfile] @? "whereis on non-present file only present in untrusted repo failed to fail" + git_annex "get" [annexedfile] @? "get failed" + annexed_present annexedfile + git_annex "whereis" [annexedfile] @? "whereis on present file failed" + +test_hook_remote :: Assertion +test_hook_remote = intmpclonerepo $ do +#ifndef mingw32_HOST_OS + git_annex "initremote" (words "foo type=hook encryption=none hooktype=foo") @? "initremote failed" + createDirectory dir + git_config "annex.foo-store-hook" $ + "cp $ANNEX_FILE " ++ loc + git_config "annex.foo-retrieve-hook" $ + "cp " ++ loc ++ " $ANNEX_FILE" + git_config "annex.foo-remove-hook" $ + "rm -f " ++ loc + git_config "annex.foo-checkpresent-hook" $ + "if [ -e " ++ loc ++ " ]; then echo $ANNEX_KEY; fi" + git_annex "get" [annexedfile] @? "get of file failed" + annexed_present annexedfile + git_annex "copy" [annexedfile, "--to", "foo"] @? "copy --to hook remote failed" + annexed_present annexedfile + git_annex "drop" [annexedfile, "--numcopies=2"] @? "drop failed" + annexed_notpresent annexedfile + git_annex "move" [annexedfile, "--from", "foo"] @? "move --from hook remote failed" + annexed_present annexedfile + not <$> git_annex "drop" [annexedfile, "--numcopies=2"] @? "drop failed to fail" + annexed_present annexedfile + where + dir = "dir" + loc = dir ++ "/$ANNEX_KEY" + git_config k v = boolSystem "git" [Param "config", Param k, Param v] + @? "git config failed" +#else + -- this test doesn't work in Windows TODO + noop +#endif + +test_directory_remote :: Assertion +test_directory_remote = intmpclonerepo $ do + createDirectory "dir" + git_annex "initremote" (words "foo type=directory encryption=none directory=dir") @? "initremote failed" + git_annex "get" [annexedfile] @? "get of file failed" + annexed_present annexedfile + git_annex "copy" [annexedfile, "--to", "foo"] @? "copy --to directory remote failed" + annexed_present annexedfile + git_annex "drop" [annexedfile, "--numcopies=2"] @? "drop failed" + annexed_notpresent annexedfile + git_annex "move" [annexedfile, "--from", "foo"] @? "move --from directory remote failed" + annexed_present annexedfile + not <$> git_annex "drop" [annexedfile, "--numcopies=2"] @? "drop failed to fail" + annexed_present annexedfile + +test_rsync_remote :: Assertion +test_rsync_remote = intmpclonerepo $ do + createDirectory "dir" + git_annex "initremote" (words "foo type=rsync encryption=none rsyncurl=dir") @? "initremote failed" + git_annex "get" [annexedfile] @? "get of file failed" + annexed_present annexedfile + git_annex "copy" [annexedfile, "--to", "foo"] @? "copy --to rsync remote failed" + annexed_present annexedfile + git_annex "drop" [annexedfile, "--numcopies=2"] @? "drop failed" + annexed_notpresent annexedfile + git_annex "move" [annexedfile, "--from", "foo"] @? "move --from rsync remote failed" + annexed_present annexedfile + not <$> git_annex "drop" [annexedfile, "--numcopies=2"] @? "drop failed to fail" + annexed_present annexedfile + +test_bup_remote :: Assertion +test_bup_remote = intmpclonerepo $ when BuildInfo.bup $ do + dir <- absPath "dir" -- bup special remote needs an absolute path + createDirectory dir + git_annex "initremote" (words $ "foo type=bup encryption=none buprepo="++dir) @? "initremote failed" + git_annex "get" [annexedfile] @? "get of file failed" + annexed_present annexedfile + git_annex "copy" [annexedfile, "--to", "foo"] @? "copy --to bup remote failed" + annexed_present annexedfile + git_annex "drop" [annexedfile, "--numcopies=2"] @? "drop failed" + annexed_notpresent annexedfile + git_annex "copy" [annexedfile, "--from", "foo"] @? "copy --from bup remote failed" + annexed_present annexedfile + git_annex "move" [annexedfile, "--from", "foo"] @? "move --from bup remote failed" + annexed_present annexedfile + +-- gpg is not a build dependency, so only test when it's available +test_crypto :: Assertion +#ifndef mingw32_HOST_OS +test_crypto = do + testscheme "shared" + testscheme "hybrid" + testscheme "pubkey" + where + gpgcmd = Utility.Gpg.mkGpgCmd Nothing + testscheme scheme = intmpclonerepo $ whenM (Utility.Path.inPath (Utility.Gpg.unGpgCmd gpgcmd)) $ do + Utility.Gpg.testTestHarness gpgcmd + @? "test harness self-test failed" + Utility.Gpg.testHarness gpgcmd $ do + createDirectory "dir" + let a cmd = git_annex cmd $ + [ "foo" + , "type=directory" + , "encryption=" ++ scheme + , "directory=dir" + , "highRandomQuality=false" + ] ++ if scheme `elem` ["hybrid","pubkey"] + then ["keyid=" ++ Utility.Gpg.testKeyId] + else [] + a "initremote" @? "initremote failed" + not <$> a "initremote" @? "initremote failed to fail when run twice in a row" + a "enableremote" @? "enableremote failed" + a "enableremote" @? "enableremote failed when run twice in a row" + git_annex "get" [annexedfile] @? "get of file failed" + annexed_present annexedfile + git_annex "copy" [annexedfile, "--to", "foo"] @? "copy --to encrypted remote failed" + (c,k) <- annexeval $ do + uuid <- Remote.nameToUUID "foo" + rs <- Logs.Remote.readRemoteLog + Just k <- Annex.WorkTree.lookupFile annexedfile + return (fromJust $ M.lookup uuid rs, k) + let key = if scheme `elem` ["hybrid","pubkey"] + then Just $ Utility.Gpg.KeyIds [Utility.Gpg.testKeyId] + else Nothing + testEncryptedRemote scheme key c [k] @? "invalid crypto setup" + + annexed_present annexedfile + git_annex "drop" [annexedfile, "--numcopies=2"] @? "drop failed" + annexed_notpresent annexedfile + git_annex "move" [annexedfile, "--from", "foo"] @? "move --from encrypted remote failed" + annexed_present annexedfile + not <$> git_annex "drop" [annexedfile, "--numcopies=2"] @? "drop failed to fail" + annexed_present annexedfile + {- Ensure the configuration complies with the encryption scheme, and + - that all keys are encrypted properly for the given directory remote. -} + testEncryptedRemote scheme ks c keys = case Remote.Helper.Encryptable.extractCipher c of + Just cip@Crypto.SharedCipher{} | scheme == "shared" && isNothing ks -> + checkKeys cip Nothing + Just cip@(Crypto.EncryptedCipher encipher v ks') + | checkScheme v && keysMatch ks' -> + checkKeys cip (Just v) <&&> checkCipher encipher ks' + _ -> return False + where + keysMatch (Utility.Gpg.KeyIds ks') = + maybe False (\(Utility.Gpg.KeyIds ks2) -> + sort (nub ks2) == sort (nub ks')) ks + checkCipher encipher = Utility.Gpg.checkEncryptionStream gpgcmd encipher . Just + checkScheme Types.Crypto.Hybrid = scheme == "hybrid" + checkScheme Types.Crypto.PubKey = scheme == "pubkey" + checkKeys cip mvariant = do + dummycfg <- Types.GitConfig.dummyRemoteGitConfig + let encparams = (mempty :: Types.Remote.RemoteConfig, dummycfg) + cipher <- Crypto.decryptCipher gpgcmd encparams cip + files <- filterM doesFileExist $ + map ("dir" ) $ concatMap (key2files cipher) keys + return (not $ null files) <&&> allM (checkFile mvariant) files + checkFile mvariant filename = + Utility.Gpg.checkEncryptionFile gpgcmd filename $ + if mvariant == Just Types.Crypto.PubKey then ks else Nothing + key2files cipher = Annex.Locations.keyPaths . + Crypto.encryptKey Types.Crypto.HmacSha1 cipher +#else +test_crypto = putStrLn "gpg testing not implemented on Windows" +#endif + +test_add_subdirs :: Assertion +test_add_subdirs = intmpclonerepo $ do + createDirectory "dir" + writeFile ("dir" "foo") $ "dir/" ++ content annexedfile + git_annex "add" ["dir"] @? "add of subdir failed" + + {- Regression test for Windows bug where symlinks were not + - calculated correctly for files in subdirs. -} + unlessM (unlockedFiles <$> getTestMode) $ do + git_annex "sync" [] @? "sync failed" + l <- annexeval $ Utility.FileSystemEncoding.decodeBS + <$> Annex.CatFile.catObject (Git.Types.Ref "HEAD:dir/foo") + "../.git/annex/" `isPrefixOf` l @? ("symlink from subdir to .git/annex is wrong: " ++ l) + + createDirectory "dir2" + writeFile ("dir2" "foo") $ content annexedfile + setCurrentDirectory "dir" + git_annex "add" [".." "dir2"] @? "add of ../subdir failed" + +test_addurl :: Assertion +test_addurl = intmpclonerepo $ do + -- file:// only; this test suite should not hit the network + let filecmd c ps = git_annex c ("-cannex.security.allowed-url-schemes=file" : ps) + f <- absPath "myurl" + let url = replace "\\" "/" ("file:///" ++ dropDrive f) + writeFile f "foo" + not <$> git_annex "addurl" [url] @? "addurl failed to fail on file url" + filecmd "addurl" [url] @? ("addurl failed on " ++ url) + let dest = "addurlurldest" + filecmd "addurl" ["--file", dest, url] @? ("addurl failed on " ++ url ++ " with --file") + doesFileExist dest @? (dest ++ " missing after addurl --file") diff --git a/Test/Framework.hs b/Test/Framework.hs new file mode 100644 index 0000000000..c5908faf92 --- /dev/null +++ b/Test/Framework.hs @@ -0,0 +1,534 @@ +{- git-annex test suite framework + - + - Copyright 2010-2017 Joey Hess + - + - Licensed under the GNU GPL version 3 or higher. + -} + +module Test.Framework where + +import Test.Tasty +import Test.Tasty.Runners +import Test.Tasty.HUnit + +import Common +import Types.Test + +import qualified Annex +import qualified Annex.UUID +import qualified Annex.Version +import qualified Backend +import qualified Git.CurrentRepo +import qualified Git.Construct +import qualified Types.KeySource +import qualified Types.Backend +import qualified Types +import qualified Remote +import qualified Key +import qualified Types.Key +import qualified Types.Messages +import qualified Config +import qualified Annex.WorkTree +import qualified Annex.Link +import qualified Annex.Init +import qualified Annex.Path +import qualified Annex.Action +import qualified Utility.Process +import qualified Utility.Env +import qualified Utility.Env.Set +import qualified Utility.Exception +import qualified Utility.ThreadScheduler +import qualified Utility.Tmp.Dir +import qualified Command.Uninit +import qualified CmdLine.GitAnnex as GitAnnex + +-- This is equivilant to running git-annex, but it's all run in-process +-- so test coverage collection works. +git_annex :: String -> [String] -> IO Bool +git_annex command params = do + -- catch all errors, including normally fatal errors + r <- try run ::IO (Either SomeException ()) + case r of + Right _ -> return True + Left _ -> return False + where + run = GitAnnex.run dummyTestOptParser Nothing (command:"-q":params) + dummyTestOptParser = pure mempty + +{- Runs git-annex and returns its output. -} +git_annex_output :: String -> [String] -> IO String +git_annex_output command params = do + pp <- Annex.Path.programPath + got <- Utility.Process.readProcess pp (command:params) + -- Since the above is a separate process, code coverage stats are + -- not gathered for things run in it. + -- Run same command again, to get code coverage. + _ <- git_annex command params + return got + +git_annex_expectoutput :: String -> [String] -> [String] -> IO () +git_annex_expectoutput command params expected = do + got <- lines <$> git_annex_output command params + got == expected @? ("unexpected value running " ++ command ++ " " ++ show params ++ " -- got: " ++ show got ++ " expected: " ++ show expected) + +-- Runs an action in the current annex. Note that shutdown actions +-- are not run; this should only be used for actions that query state. +annexeval :: Types.Annex a -> IO a +annexeval a = do + s <- Annex.new =<< Git.CurrentRepo.get + Annex.eval s $ do + Annex.setOutput Types.Messages.QuietOutput + a `finally` Annex.Action.stopCoProcesses + +innewrepo :: Assertion -> Assertion +innewrepo a = withgitrepo $ \r -> indir r a + +inmainrepo :: Assertion -> Assertion +inmainrepo = indir mainrepodir + +with_ssh_origin :: (Assertion -> Assertion) -> (Assertion -> Assertion) +with_ssh_origin cloner a = cloner $ do + origindir <- absPath + =<< annexeval (Config.getConfig (Config.ConfigKey config) "/dev/null") + let originurl = "localhost:" ++ origindir + boolSystem "git" [Param "config", Param config, Param originurl] @? "git config failed" + a + where + config = "remote.origin.url" + +intmpclonerepo :: Assertion -> Assertion +intmpclonerepo a = withtmpclonerepo $ \r -> indir r a + +intmpclonerepoInDirect :: Assertion -> Assertion +intmpclonerepoInDirect a = intmpclonerepo $ + ifM isdirect + ( putStrLn "not supported in direct mode; skipping" + , a + ) + where + isdirect = annexeval $ do + Annex.Init.initialize (Annex.Init.AutoInit False) Nothing Nothing + Config.isDirect + +checkRepo :: Types.Annex a -> FilePath -> IO a +checkRepo getval d = do + s <- Annex.new =<< Git.Construct.fromPath d + Annex.eval s $ + getval `finally` Annex.Action.stopCoProcesses + +isInDirect :: FilePath -> IO Bool +isInDirect = checkRepo (not <$> Config.isDirect) + +intmpbareclonerepo :: Assertion -> Assertion +intmpbareclonerepo a = withtmpclonerepo' (newCloneRepoConfig { bareClone = True } ) $ + \r -> indir r a + +intmpsharedclonerepo :: Assertion -> Assertion +intmpsharedclonerepo a = withtmpclonerepo' (newCloneRepoConfig { sharedClone = True } ) $ + \r -> indir r a + +withtmpclonerepo :: (FilePath -> Assertion) -> Assertion +withtmpclonerepo = withtmpclonerepo' newCloneRepoConfig + +withtmpclonerepo' :: CloneRepoConfig -> (FilePath -> Assertion) -> Assertion +withtmpclonerepo' cfg a = do + dir <- tmprepodir + clone <- clonerepo mainrepodir dir cfg + r <- tryNonAsync (a clone) + case r of + Right () -> return () + Left e -> do + whenM (keepFailures <$> getTestMode) $ + putStrLn $ "** Preserving repo for failure analysis in " ++ clone + throwM e + +disconnectOrigin :: Assertion +disconnectOrigin = boolSystem "git" [Param "remote", Param "rm", Param "origin"] @? "remote rm" + +withgitrepo :: (FilePath -> Assertion) -> Assertion +withgitrepo = bracket (setuprepo mainrepodir) return + +indir :: FilePath -> Assertion -> Assertion +indir dir a = do + currdir <- getCurrentDirectory + -- Assertion failures throw non-IO errors; catch + -- any type of error and change back to currdir before + -- rethrowing. + r <- bracket_ (changeToTmpDir dir) (setCurrentDirectory currdir) + (try a::IO (Either SomeException ())) + case r of + Right () -> return () + Left e -> throwM e + +setuprepo :: FilePath -> IO FilePath +setuprepo dir = do + cleanup dir + ensuretmpdir + boolSystem "git" [Param "init", Param "-q", File dir] @? "git init failed" + configrepo dir + return dir + +data CloneRepoConfig = CloneRepoConfig + { bareClone :: Bool + , sharedClone :: Bool + } + +newCloneRepoConfig :: CloneRepoConfig +newCloneRepoConfig = CloneRepoConfig + { bareClone = False + , sharedClone = False + } + +-- clones are always done as local clones; we cannot test ssh clones +clonerepo :: FilePath -> FilePath -> CloneRepoConfig -> IO FilePath +clonerepo old new cfg = do + cleanup new + ensuretmpdir + let cloneparams = catMaybes + [ Just $ Param "clone" + , Just $ Param "-q" + , if bareClone cfg then Just (Param "--bare") else Nothing + , if sharedClone cfg then Just (Param "--shared") else Nothing + , Just $ File old + , Just $ File new + ] + boolSystem "git" cloneparams @? "git clone failed" + configrepo new + indir new $ do + ver <- annexVersion <$> getTestMode + if ver == Annex.Version.defaultVersion + then git_annex "init" ["-q", new] @? "git annex init failed" + else git_annex "init" ["-q", new, "--version", ver] @? "git annex init failed" + unless (bareClone cfg) $ + indir new $ + setupTestMode + return new + +configrepo :: FilePath -> IO () +configrepo dir = indir dir $ do + -- ensure git is set up to let commits happen + boolSystem "git" [Param "config", Param "user.name", Param "Test User"] @? "git config failed" + boolSystem "git" [Param "config", Param "user.email", Param "test@example.com"] @? "git config failed" + -- avoid signed commits by test suite + boolSystem "git" [Param "config", Param "commit.gpgsign", Param "false"] @? "git config failed" + -- tell git-annex to not annex the ingitfile + boolSystem "git" + [ Param "config" + , Param "annex.largefiles" + , Param ("exclude=" ++ ingitfile) + ] @? "git config annex.largefiles failed" + +ensuretmpdir :: IO () +ensuretmpdir = do + e <- doesDirectoryExist tmpdir + unless e $ + createDirectory tmpdir + +{- Prevent global git configs from affecting the test suite. -} +isolateGitConfig :: IO a -> IO a +isolateGitConfig a = Utility.Tmp.Dir.withTmpDir "testhome" $ \tmphome -> do + tmphomeabs <- absPath tmphome + Utility.Env.Set.setEnv "HOME" tmphomeabs True + Utility.Env.Set.setEnv "XDG_CONFIG_HOME" tmphomeabs True + Utility.Env.Set.setEnv "GIT_CONFIG_NOSYSTEM" "1" True + a + +cleanup :: FilePath -> IO () +cleanup dir = whenM (doesDirectoryExist dir) $ do + Command.Uninit.prepareRemoveAnnexDir' dir + -- This can fail if files in the directory are still open by a + -- subprocess. + void $ tryIO $ removeDirectoryRecursive dir + +finalCleanup :: IO () +finalCleanup = whenM (doesDirectoryExist tmpdir) $ do + Annex.Action.reapZombies + Command.Uninit.prepareRemoveAnnexDir' tmpdir + catchIO (removeDirectoryRecursive tmpdir) $ \e -> do + print e + putStrLn "sleeping 10 seconds and will retry directory cleanup" + Utility.ThreadScheduler.threadDelaySeconds $ + Utility.ThreadScheduler.Seconds 10 + whenM (doesDirectoryExist tmpdir) $ do + Annex.Action.reapZombies + removeDirectoryRecursive tmpdir + +checklink :: FilePath -> Assertion +checklink f = + -- in direct mode, it may be a symlink, or not, depending + -- on whether the content is present. + unlessM (annexeval Config.isDirect) $ + ifM (annexeval Config.crippledFileSystem) + ( (isJust <$> annexeval (Annex.Link.getAnnexLinkTarget f)) + @? f ++ " is not a (crippled) symlink" + , do + s <- getSymbolicLinkStatus f + isSymbolicLink s @? f ++ " is not a symlink" + ) + +checkregularfile :: FilePath -> Assertion +checkregularfile f = do + s <- getSymbolicLinkStatus f + isRegularFile s @? f ++ " is not a normal file" + return () + +checkdoesnotexist :: FilePath -> Assertion +checkdoesnotexist f = + (either (const True) (const False) <$> Utility.Exception.tryIO (getSymbolicLinkStatus f)) + @? f ++ " exists unexpectedly" + +checkexists :: FilePath -> Assertion +checkexists f = + (either (const False) (const True) <$> Utility.Exception.tryIO (getSymbolicLinkStatus f)) + @? f ++ " does not exist" + +checkcontent :: FilePath -> Assertion +checkcontent f = do + c <- Utility.Exception.catchDefaultIO "could not read file" $ readFile f + assertEqual ("checkcontent " ++ f) (content f) c + +checkunwritable :: FilePath -> Assertion +checkunwritable f = unlessM (annexeval Config.isDirect) $ do + -- Look at permissions bits rather than trying to write or + -- using fileAccess because if run as root, any file can be + -- modified despite permissions. + s <- getFileStatus f + let mode = fileMode s + when (mode == mode `unionFileModes` ownerWriteMode) $ + assertFailure $ "able to modify annexed file's " ++ f ++ " content" + +checkwritable :: FilePath -> Assertion +checkwritable f = do + s <- getFileStatus f + let mode = fileMode s + unless (mode == mode `unionFileModes` ownerWriteMode) $ + assertFailure $ "unable to modify " ++ f + +checkdangling :: FilePath -> Assertion +checkdangling f = ifM (annexeval Config.crippledFileSystem) + ( return () -- probably no real symlinks to test + , do + r <- tryIO $ readFile f + case r of + Left _ -> return () -- expected; dangling link + Right _ -> assertFailure $ f ++ " was not a dangling link as expected" + ) + +checklocationlog :: FilePath -> Bool -> Assertion +checklocationlog f expected = do + thisuuid <- annexeval Annex.UUID.getUUID + r <- annexeval $ Annex.WorkTree.lookupFile f + case r of + Just k -> do + uuids <- annexeval $ Remote.keyLocations k + assertEqual ("bad content in location log for " ++ f ++ " key " ++ Key.key2file k ++ " uuid " ++ show thisuuid) + expected (thisuuid `elem` uuids) + _ -> assertFailure $ f ++ " failed to look up key" + +checkbackend :: FilePath -> Types.Backend -> Assertion +checkbackend file expected = do + b <- annexeval $ maybe (return Nothing) (Backend.getBackend file) + =<< Annex.WorkTree.lookupFile file + assertEqual ("backend for " ++ file) (Just expected) b + +checkispointerfile :: FilePath -> Assertion +checkispointerfile f = unlessM (isJust <$> Annex.Link.isPointerFile f) $ + assertFailure $ f ++ " is not a pointer file" + +inlocationlog :: FilePath -> Assertion +inlocationlog f = checklocationlog f True + +notinlocationlog :: FilePath -> Assertion +notinlocationlog f = checklocationlog f False + +runchecks :: [FilePath -> Assertion] -> FilePath -> Assertion +runchecks [] _ = return () +runchecks (a:as) f = do + a f + runchecks as f + +annexed_notpresent :: FilePath -> Assertion +annexed_notpresent f = ifM (unlockedFiles <$> getTestMode) + ( annexed_notpresent_unlocked f + , annexed_notpresent_locked f + ) + +annexed_notpresent_locked :: FilePath -> Assertion +annexed_notpresent_locked = runchecks [checklink, checkdangling, notinlocationlog] + +annexed_notpresent_unlocked :: FilePath -> Assertion +annexed_notpresent_unlocked = runchecks [checkregularfile, checkispointerfile, notinlocationlog] + +annexed_present :: FilePath -> Assertion +annexed_present f = ifM (unlockedFiles <$> getTestMode) + ( annexed_present_unlocked f + , annexed_present_locked f + ) + +annexed_present_locked :: FilePath -> Assertion +annexed_present_locked f = ifM (annexeval Config.crippledFileSystem) + ( runchecks [checklink, inlocationlog] f + , runchecks [checklink, checkcontent, checkunwritable, inlocationlog] f + ) + +annexed_present_unlocked :: FilePath -> Assertion +annexed_present_unlocked = runchecks + [checkregularfile, checkcontent, checkwritable, inlocationlog] + +unannexed :: FilePath -> Assertion +unannexed = runchecks [checkregularfile, checkcontent, checkwritable] + +add_annex :: FilePath -> IO Bool +add_annex f = ifM (unlockedFiles <$> getTestMode) + ( boolSystem "git" [Param "add", File f] + , git_annex "add" [f] + ) + +data TestMode = TestMode + { forceDirect :: Bool + , unlockedFiles :: Bool + , annexVersion :: Annex.Version.Version + , keepFailures :: Bool + } deriving (Read, Show) + +testMode :: TestOptions -> Annex.Version.Version -> TestMode +testMode opts v = TestMode + { forceDirect = False + , unlockedFiles = False + , annexVersion = v + , keepFailures = keepFailuresOption opts + } + +withTestMode :: TestMode -> TestTree -> TestTree -> TestTree +withTestMode testmode inittests = withResource prepare release . const + where + prepare = do + setTestMode testmode + case tryIngredients [consoleTestReporter] mempty inittests of + Nothing -> error "No tests found!?" + Just act -> unlessM act $ + error "init tests failed! cannot continue" + return () + release _ = cleanup mainrepodir + +setTestMode :: TestMode -> IO () +setTestMode testmode = do + currdir <- getCurrentDirectory + p <- Utility.Env.getEnvDefault "PATH" "" + + mapM_ (\(var, val) -> Utility.Env.Set.setEnv var val True) + -- Ensure that the just-built git annex is used. + [ ("PATH", currdir ++ [searchPathSeparator] ++ p) + , ("TOPDIR", currdir) + -- Avoid git complaining if it cannot determine the user's + -- email address, or exploding if it doesn't know the user's + -- name. + , ("GIT_AUTHOR_EMAIL", "test@example.com") + , ("GIT_AUTHOR_NAME", "git-annex test") + , ("GIT_COMMITTER_EMAIL", "test@example.com") + , ("GIT_COMMITTER_NAME", "git-annex test") + -- force gpg into batch mode for the tests + , ("GPG_BATCH", "1") + -- Make git and git-annex access ssh remotes on the local + -- filesystem, without using ssh at all. + , ("GIT_SSH_COMMAND", "git-annex test --fakessh --") + , ("GIT_ANNEX_USE_GIT_SSH", "1") + , ("TESTMODE", show testmode) + ] + +runFakeSsh :: [String] -> IO () +runFakeSsh ("-n":ps) = runFakeSsh ps +runFakeSsh (_host:cmd:[]) = do + (_, _, _, pid) <- createProcess (shell cmd) + exitWith =<< waitForProcess pid +runFakeSsh ps = error $ "fake ssh option parse error: " ++ show ps + +getTestMode :: IO TestMode +getTestMode = Prelude.read <$> Utility.Env.getEnvDefault "TESTMODE" "" + +setupTestMode :: IO () +setupTestMode = do + testmode <- getTestMode + when (forceDirect testmode) $ + git_annex "direct" ["-q"] @? "git annex direct failed" + +changeToTmpDir :: FilePath -> IO () +changeToTmpDir t = do + topdir <- Utility.Env.getEnvDefault "TOPDIR" (error "TOPDIR not set") + setCurrentDirectory $ topdir ++ "/" ++ t + +tmpdir :: String +tmpdir = ".t" + +mainrepodir :: FilePath +mainrepodir = tmpdir "repo" + +tmprepodir :: IO FilePath +tmprepodir = go (0 :: Int) + where + go n = do + let d = tmpdir "tmprepo" ++ show n + ifM (doesDirectoryExist d) + ( go $ n + 1 + , return d + ) + +annexedfile :: String +annexedfile = "foo" + +annexedfiledup :: String +annexedfiledup = "foodup" + +wormannexedfile :: String +wormannexedfile = "apple" + +sha1annexedfile :: String +sha1annexedfile = "sha1foo" + +sha1annexedfiledup :: String +sha1annexedfiledup = "sha1foodup" + +ingitfile :: String +ingitfile = "bar.c" + +content :: FilePath -> String +content f + | f == annexedfile = "annexed file content" + | f == ingitfile = "normal file content" + | f == sha1annexedfile ="sha1 annexed file content" + | f == annexedfiledup = content annexedfile + | f == sha1annexedfiledup = content sha1annexedfile + | f == wormannexedfile = "worm annexed file content" + | "import" `isPrefixOf` f = "imported content" + | otherwise = "unknown file " ++ f + +changecontent :: FilePath -> IO () +changecontent f = writeFile f $ changedcontent f + +changedcontent :: FilePath -> String +changedcontent f = content f ++ " (modified)" + +backendSHA1 :: Types.Backend +backendSHA1 = backend_ "SHA1" + +backendSHA256 :: Types.Backend +backendSHA256 = backend_ "SHA256" + +backendSHA256E :: Types.Backend +backendSHA256E = backend_ "SHA256E" + +backendWORM :: Types.Backend +backendWORM = backend_ "WORM" + +backend_ :: String -> Types.Backend +backend_ = Backend.lookupBackendVariety . Types.Key.parseKeyVariety + +getKey :: Types.Backend -> FilePath -> IO Types.Key +getKey b f = fromJust <$> annexeval go + where + go = Types.Backend.getKey b + Types.KeySource.KeySource + { Types.KeySource.keyFilename = f + , Types.KeySource.contentLocation = f + , Types.KeySource.inodeCache = Nothing + } diff --git a/Types.hs b/Types.hs new file mode 100644 index 0000000000..884c91a6bb --- /dev/null +++ b/Types.hs @@ -0,0 +1,29 @@ +{- git-annex abstract data types + - + - Copyright 2010 Joey Hess + - + - Licensed under the GNU GPL version 3 or higher. + -} + +module Types ( + Annex, + Backend, + Key, + AssociatedFile(..), + UUID(..), + GitConfig(..), + RemoteGitConfig(..), + Remote, + RemoteType, +) where + +import Annex +import Types.Backend +import Types.GitConfig +import Types.Key +import Types.UUID +import Types.Remote + +type Backend = BackendA Annex +type Remote = RemoteA Annex +type RemoteType = RemoteTypeA Annex diff --git a/Types/ActionItem.hs b/Types/ActionItem.hs new file mode 100644 index 0000000000..73d8451017 --- /dev/null +++ b/Types/ActionItem.hs @@ -0,0 +1,52 @@ +{- items that a command can act on + - + - Copyright 2016 Joey Hess + - + - Licensed under the GNU GPL version 3 or higher. + -} + +{-# LANGUAGE TypeSynonymInstances, FlexibleInstances #-} + +module Types.ActionItem where + +import Key +import Types.Transfer +import Git.FilePath + +data ActionItem + = ActionItemAssociatedFile AssociatedFile + | ActionItemKey + | ActionItemBranchFilePath BranchFilePath + | ActionItemFailedTransfer Transfer TransferInfo + +class MkActionItem t where + mkActionItem :: t -> ActionItem + +instance MkActionItem AssociatedFile where + mkActionItem = ActionItemAssociatedFile + +instance MkActionItem Key where + mkActionItem _ = ActionItemKey + +instance MkActionItem BranchFilePath where + mkActionItem = ActionItemBranchFilePath + +instance MkActionItem (Transfer, TransferInfo) where + mkActionItem = uncurry ActionItemFailedTransfer + +actionItemDesc :: ActionItem -> Key -> String +actionItemDesc (ActionItemAssociatedFile (AssociatedFile (Just f))) _ = f +actionItemDesc (ActionItemAssociatedFile (AssociatedFile Nothing)) k = key2file k +actionItemDesc ActionItemKey k = key2file k +actionItemDesc (ActionItemBranchFilePath bfp) _ = descBranchFilePath bfp +actionItemDesc (ActionItemFailedTransfer _ i) k = + actionItemDesc (ActionItemAssociatedFile (associatedFile i)) k + +actionItemWorkTreeFile :: ActionItem -> Maybe FilePath +actionItemWorkTreeFile (ActionItemAssociatedFile (AssociatedFile af)) = af +actionItemWorkTreeFile _ = Nothing + +actionItemTransferDirection :: ActionItem -> Maybe Direction +actionItemTransferDirection (ActionItemFailedTransfer t _) = Just $ + transferDirection t +actionItemTransferDirection _ = Nothing diff --git a/Types/Availability.hs b/Types/Availability.hs new file mode 100644 index 0000000000..6956537d27 --- /dev/null +++ b/Types/Availability.hs @@ -0,0 +1,11 @@ +{- git-annex remote availability + - + - Copyright 2014 Joey Hess + - + - Licensed under the GNU GPL version 3 or higher. + -} + +module Types.Availability where + +data Availability = GloballyAvailable | LocallyAvailable + deriving (Eq, Show, Read) diff --git a/Types/Backend.hs b/Types/Backend.hs new file mode 100644 index 0000000000..10b7b47fc8 --- /dev/null +++ b/Types/Backend.hs @@ -0,0 +1,34 @@ +{- git-annex key backend data type + - + - Most things should not need this, using Types instead + - + - Copyright 2010-2017 Joey Hess + - + - Licensed under the GNU GPL version 3 or higher. + -} + +module Types.Backend where + +import Types.Key +import Types.KeySource + +data BackendA a = Backend + { backendVariety :: KeyVariety + , getKey :: KeySource -> a (Maybe Key) + -- Verifies the content of a key. + , verifyKeyContent :: Maybe (Key -> FilePath -> a Bool) + -- Checks if a key can be upgraded to a better form. + , canUpgradeKey :: Maybe (Key -> Bool) + -- Checks if there is a fast way to migrate a key to a different + -- backend (ie, without re-hashing). + , fastMigrate :: Maybe (Key -> BackendA a -> AssociatedFile -> a (Maybe Key)) + -- Checks if a key is known (or assumed) to always refer to the + -- same data. + , isStableKey :: Key -> Bool + } + +instance Show (BackendA a) where + show backend = "Backend { name =\"" ++ formatKeyVariety (backendVariety backend) ++ "\" }" + +instance Eq (BackendA a) where + a == b = backendVariety a == backendVariety b diff --git a/Types/BranchState.hs b/Types/BranchState.hs new file mode 100644 index 0000000000..8c57a223ab --- /dev/null +++ b/Types/BranchState.hs @@ -0,0 +1,16 @@ +{- git-annex BranchState data type + - + - Copyright 2011 Joey Hess + - + - Licensed under the GNU GPL version 3 or higher. + -} + +module Types.BranchState where + +data BranchState = BranchState + { branchUpdated :: Bool -- has the branch been updated this run? + , indexChecked :: Bool -- has the index file been checked to exist? + } + +startBranchState :: BranchState +startBranchState = BranchState False False diff --git a/Types/CleanupActions.hs b/Types/CleanupActions.hs new file mode 100644 index 0000000000..508579643c --- /dev/null +++ b/Types/CleanupActions.hs @@ -0,0 +1,20 @@ +{- Enumeration of cleanup actions + - + - Copyright 2014 Joey Hess + - + - Licensed under the GNU GPL version 3 or higher. + -} + +module Types.CleanupActions where + +import Types.UUID + +import Utility.Url + +data CleanupAction + = RemoteCleanup UUID + | StopHook UUID + | FsckCleanup + | SshCachingCleanup + | TorrentCleanup URLString + deriving (Eq, Ord) diff --git a/Types/Command.hs b/Types/Command.hs new file mode 100644 index 0000000000..baf4934e41 --- /dev/null +++ b/Types/Command.hs @@ -0,0 +1,91 @@ +{- git-annex command data types + - + - Copyright 2010-2016 Joey Hess + - + - Licensed under the GNU GPL version 3 or higher. + -} + +module Types.Command where + +import Data.Ord +import Options.Applicative.Types (Parser) + +import Types +import Types.DeferredParse + +{- A command runs in these stages. + - + - a. The parser stage parses the command line and generates a CommandSeek + - action. -} +type CommandParser = Parser CommandSeek +{- b. The check stage runs checks, that error out if + - anything prevents the command from running. -} +data CommandCheck = CommandCheck { idCheck :: Int, runCheck :: Annex () } +{- c. The seek stage is passed input from the parser, looks through + - the repo to find things to act on (ie, new files to add), and + - runs commandAction to handle all necessary actions. -} +type CommandSeek = Annex () +{- d. The start stage is run before anything is printed about the + - command, is passed some input, and can early abort it + - if the input does not make sense. It should run quickly and + - should not modify Annex state. -} +type CommandStart = Annex (Maybe CommandPerform) +{- e. The perform stage is run after a message is printed about the command + - being run, and it should be where the bulk of the work happens. -} +type CommandPerform = Annex (Maybe CommandCleanup) +{- f. The cleanup stage is run only if the perform stage succeeds, and it + - returns the overall success/fail of the command. -} +type CommandCleanup = Annex Bool + +{- A command is defined by specifying these things. -} +data Command = Command + { cmdcheck :: [CommandCheck] -- check stage + , cmdnocommit :: Bool -- don't commit journalled state changes + , cmdnomessages :: Bool -- don't output normal messages + , cmdname :: String + , cmdparamdesc :: CmdParamsDesc -- description of params for usage + , cmdsection :: CommandSection + , cmddesc :: String -- description of command for usage + , cmdparser :: CommandParser -- command line parser + , cmdglobaloptions :: [GlobalOption] -- additional global options + , cmdnorepo :: Maybe (Parser (IO ())) -- used when not in a repo + } + +{- Command-line parameters, after the command is selected and options + - are parsed. -} +type CmdParams = [String] + +type CmdParamsDesc = String + +{- CommandCheck functions can be compared using their unique id. -} +instance Eq CommandCheck where + a == b = idCheck a == idCheck b + +instance Eq Command where + a == b = cmdname a == cmdname b + +{- Order commands by name. -} +instance Ord Command where + compare = comparing cmdname + +{- The same sections are listed in doc/git-annex.mdwn -} +data CommandSection + = SectionCommon + | SectionSetup + | SectionMaintenance + | SectionQuery + | SectionMetaData + | SectionUtility + | SectionPlumbing + | SectionTesting + deriving (Eq, Ord, Enum, Bounded) + +descSection :: CommandSection -> String +descSection SectionCommon = "Commonly used commands" +descSection SectionSetup = "Repository setup commands" +descSection SectionMaintenance = "Repository maintenance commands" +descSection SectionQuery = "Query commands" +descSection SectionMetaData = "Metadata commands" +descSection SectionUtility = "Utility commands" +descSection SectionPlumbing = "Plumbing commands" +descSection SectionTesting = "Testing commands" diff --git a/Types/Concurrency.hs b/Types/Concurrency.hs new file mode 100644 index 0000000000..7fe5847d72 --- /dev/null +++ b/Types/Concurrency.hs @@ -0,0 +1,8 @@ +{- Copyright 2016 Joey Hess + - + - Licensed under the GNU GPL version 3 or higher. + -} + +module Types.Concurrency where + +data Concurrency = NonConcurrent | Concurrent Int diff --git a/Types/Creds.hs b/Types/Creds.hs new file mode 100644 index 0000000000..6a9e1287f7 --- /dev/null +++ b/Types/Creds.hs @@ -0,0 +1,14 @@ +{- credentials + - + - Copyright 2014 Joey Hess + - + - Licensed under the GNU GPL version 3 or higher. + -} + +module Types.Creds where + +type Creds = String -- can be any data that contains credentials + +type CredPair = (Login, Password) +type Login = String +type Password = String diff --git a/Types/Crypto.hs b/Types/Crypto.hs new file mode 100644 index 0000000000..c5a00d0329 --- /dev/null +++ b/Types/Crypto.hs @@ -0,0 +1,58 @@ +{- git-annex crypto types + - + - Copyright 2011-2015 Joey Hess + - + - Licensed under the GNU GPL version 3 or higher. + -} + +module Types.Crypto ( + Cipher(..), + StorableCipher(..), + EncryptedCipherVariant(..), + KeyIds(..), + cipherKeyIds, + Mac(..), + readMac, + showMac, + defaultMac, + calcMac, +) where + +import Utility.Hash +import Utility.Gpg (KeyIds(..)) + +-- XXX ideally, this would be a locked memory region +data Cipher = Cipher String | MacOnlyCipher String + +data StorableCipher + = EncryptedCipher String EncryptedCipherVariant KeyIds + | SharedCipher String + | SharedPubKeyCipher String KeyIds + deriving (Ord, Eq) +data EncryptedCipherVariant = Hybrid | PubKey + deriving (Ord, Eq) + +cipherKeyIds :: StorableCipher -> Maybe KeyIds +cipherKeyIds (EncryptedCipher _ _ ks) = Just ks +cipherKeyIds (SharedPubKeyCipher _ ks) = Just ks +cipherKeyIds (SharedCipher _) = Nothing + +defaultMac :: Mac +defaultMac = HmacSha1 + +-- MAC algorithms are shown as follows in the file names. +showMac :: Mac -> String +showMac HmacSha1 = "HMACSHA1" +showMac HmacSha224 = "HMACSHA224" +showMac HmacSha256 = "HMACSHA256" +showMac HmacSha384 = "HMACSHA384" +showMac HmacSha512 = "HMACSHA512" + +-- Read the MAC algorithm from the remote config. +readMac :: String -> Maybe Mac +readMac "HMACSHA1" = Just HmacSha1 +readMac "HMACSHA224" = Just HmacSha224 +readMac "HMACSHA256" = Just HmacSha256 +readMac "HMACSHA384" = Just HmacSha384 +readMac "HMACSHA512" = Just HmacSha512 +readMac _ = Nothing diff --git a/Types/DeferredParse.hs b/Types/DeferredParse.hs new file mode 100644 index 0000000000..7445615f65 --- /dev/null +++ b/Types/DeferredParse.hs @@ -0,0 +1,41 @@ +{- git-annex deferred parse values + - + - Copyright 2015 Joey Hess + - + - Licensed under the GNU GPL version 3 or higher. + -} + +{-# LANGUAGE FlexibleInstances #-} + +module Types.DeferredParse where + +import Annex + +import Options.Applicative + +-- Some values cannot be fully parsed without performing an action. +-- The action may be expensive, so it's best to call finishParse on such a +-- value before using getParsed repeatedly. +data DeferredParse a = DeferredParse (Annex a) | ReadyParse a + +class DeferredParseClass a where + finishParse :: a -> Annex a + +getParsed :: DeferredParse a -> Annex a +getParsed (DeferredParse a) = a +getParsed (ReadyParse a) = pure a + +instance DeferredParseClass (DeferredParse a) where + finishParse (DeferredParse a) = ReadyParse <$> a + finishParse (ReadyParse a) = pure (ReadyParse a) + +instance DeferredParseClass (Maybe (DeferredParse a)) where + finishParse Nothing = pure Nothing + finishParse (Just v) = Just <$> finishParse v + +instance DeferredParseClass [DeferredParse a] where + finishParse v = mapM finishParse v + +-- Use when the Annex action modifies Annex state. +type GlobalSetter = DeferredParse () +type GlobalOption = Parser GlobalSetter diff --git a/Types/DesktopNotify.hs b/Types/DesktopNotify.hs new file mode 100644 index 0000000000..ce7e4c4a3e --- /dev/null +++ b/Types/DesktopNotify.hs @@ -0,0 +1,46 @@ +{- git-annex DesktopNotify type + - + - Copyright 2014 Joey Hess + - + - Licensed under the GNU GPL version 3 or higher. + -} + +{-# LANGUAGE CPP #-} + +module Types.DesktopNotify where + +import Data.Monoid +#if MIN_VERSION_base(4,9,0) +import qualified Data.Semigroup as Sem +#endif +import Prelude + +data DesktopNotify = DesktopNotify + { notifyStart :: Bool + , notifyFinish :: Bool + } + deriving (Show) + +appendDesktopNotify :: DesktopNotify -> DesktopNotify -> DesktopNotify +appendDesktopNotify (DesktopNotify s1 f1) (DesktopNotify s2 f2) = + DesktopNotify (s1 || s2) (f1 || f2) + +#if MIN_VERSION_base(4,9,0) +instance Sem.Semigroup DesktopNotify where + (<>) = appendDesktopNotify +#endif + +instance Monoid DesktopNotify where + mempty = DesktopNotify False False +#if MIN_VERSION_base(4,11,0) +#elif MIN_VERSION_base(4,9,0) + mappend = (Sem.<>) +#else + mappend = appendDesktopNotify +#endif + +mkNotifyStart :: DesktopNotify +mkNotifyStart = DesktopNotify True False + +mkNotifyFinish :: DesktopNotify +mkNotifyFinish = DesktopNotify False True diff --git a/Types/Difference.hs b/Types/Difference.hs new file mode 100644 index 0000000000..0f7100c0c0 --- /dev/null +++ b/Types/Difference.hs @@ -0,0 +1,144 @@ +{- git-annex repository differences + - + - Copyright 2015 Joey Hess + - + - Licensed under the GNU GPL version 3 or higher. + -} + +{-# LANGUAGE CPP #-} + +module Types.Difference ( + Difference(..), + Differences(..), + readDifferences, + showDifferences, + getDifferences, + differenceConfigKey, + differenceConfigVal, + hasDifference, + listDifferences, +) where + +import Utility.PartialPrelude +import qualified Git +import qualified Git.Config + +import Data.Maybe +import Data.Monoid +import qualified Data.Set as S +#if MIN_VERSION_base(4,9,0) +import qualified Data.Semigroup as Sem +#endif +import Prelude + +-- Describes differences from the v5 repository format. +-- +-- The serialization is stored in difference.log, so avoid changes that +-- would break compatability. +-- +-- Not breaking compatability is why a list of Differences is used, rather +-- than a record type. With a record type, adding a new field for some future +-- difference would serialize to a value that an older version could not +-- parse, even if that new field was not used. With the Differences list, +-- old versions can still parse it, unless the new Difference constructor +-- is used. +-- +-- The constructors intentionally do not have parameters; this is to +-- ensure that any Difference that can be expressed is supported. +-- So, a new repository version would be Version6, rather than Version Int. +data Difference + = ObjectHashLower + | OneLevelObjectHash + | OneLevelBranchHash + deriving (Show, Read, Eq, Ord, Enum, Bounded) + +-- This type is used internally for efficient checking for differences, +-- but converted to S.Set Difference for serialization. +data Differences + = Differences + { objectHashLower :: Bool + , oneLevelObjectHash :: Bool + , oneLevelBranchHash :: Bool + } + | UnknownDifferences + +-- UnknownDifferences cannot be equal +instance Eq Differences where + UnknownDifferences == _ = False + _ == UnknownDifferences = False + a == b = all (\f -> f a == f b) + [ objectHashLower + , oneLevelObjectHash + , oneLevelBranchHash + ] + +appendDifferences :: Differences -> Differences -> Differences +appendDifferences a@(Differences {}) b@(Differences {}) = a + { objectHashLower = objectHashLower a || objectHashLower b + , oneLevelObjectHash = oneLevelObjectHash a || oneLevelObjectHash b + , oneLevelBranchHash = oneLevelBranchHash a || oneLevelBranchHash b + } +appendDifferences _ _ = UnknownDifferences + +#if MIN_VERSION_base(4,9,0) +instance Sem.Semigroup Differences where + (<>) = appendDifferences +#endif + +instance Monoid Differences where + mempty = Differences False False False +#if MIN_VERSION_base(4,11,0) +#elif MIN_VERSION_base(4,9,0) + mappend = (Sem.<>) +#else + mappend = appendDifferences +#endif + +readDifferences :: String -> Differences +readDifferences = maybe UnknownDifferences mkDifferences . readish + +showDifferences :: Differences -> String +showDifferences = show . S.fromList . listDifferences + +getDifferences :: Git.Repo -> Differences +getDifferences r = mkDifferences $ S.fromList $ + mapMaybe getmaybe [minBound .. maxBound] + where + getmaybe d = case Git.Config.isTrue =<< Git.Config.getMaybe (differenceConfigKey d) r of + Just True -> Just d + _ -> Nothing + +differenceConfigKey :: Difference -> String +differenceConfigKey ObjectHashLower = tunable "objecthashlower" +differenceConfigKey OneLevelObjectHash = tunable "objecthash1" +differenceConfigKey OneLevelBranchHash = tunable "branchhash1" + +differenceConfigVal :: Difference -> String +differenceConfigVal _ = Git.Config.boolConfig True + +tunable :: String -> String +tunable k = "annex.tune." ++ k + +hasDifference :: Difference -> Differences -> Bool +hasDifference _ UnknownDifferences = False +hasDifference ObjectHashLower ds = objectHashLower ds +hasDifference OneLevelObjectHash ds = oneLevelObjectHash ds +hasDifference OneLevelBranchHash ds = oneLevelBranchHash ds + +listDifferences :: Differences -> [Difference] +listDifferences d@(Differences {}) = map snd $ + filter (\(f, _) -> f d) + [ (objectHashLower, ObjectHashLower) + , (oneLevelObjectHash, OneLevelObjectHash) + , (oneLevelBranchHash, OneLevelBranchHash) + ] +listDifferences UnknownDifferences = [] + +mkDifferences :: S.Set Difference -> Differences +mkDifferences s = Differences + { objectHashLower = check ObjectHashLower + , oneLevelObjectHash = check OneLevelObjectHash + , oneLevelBranchHash = check OneLevelBranchHash + } + where + check f = f `S.member` s diff --git a/Types/Distribution.hs b/Types/Distribution.hs new file mode 100644 index 0000000000..d19074bf95 --- /dev/null +++ b/Types/Distribution.hs @@ -0,0 +1,78 @@ +{- Data type for a distribution of git-annex + - + - Copyright 2013, 2017 Joey Hess + - + - Licensed under the GNU GPL version 3 or higher. + -} + +module Types.Distribution where + +import Utility.PartialPrelude +import Utility.Split +import Types.Key +import Key +import Data.Time.Clock +import Git.Config (isTrue, boolConfig) + +import Control.Applicative +import Prelude + +type GitAnnexVersion = String + +data GitAnnexDistribution = GitAnnexDistribution + { distributionUrl :: String + , distributionKey :: Key + , distributionVersion :: GitAnnexVersion + , distributionReleasedate :: UTCTime + , distributionUrgentUpgrade :: Maybe GitAnnexVersion + } + deriving (Read, Show, Eq) + +{- The first line of the info file is in the format old versions of + - git-annex expect to read a GitAnnexDistribution. + - The remainder of the file is in the new format. + - This works because old versions of git-annex used readish to parse + - the file, and that ignores the second line. + -} +formatInfoFile :: GitAnnexDistribution -> String +formatInfoFile d = replace "keyVariant = " "keyBackendName = " (show d) ++ + "\n" ++ formatGitAnnexDistribution d + +parseInfoFile :: String -> Maybe GitAnnexDistribution +parseInfoFile s = case lines s of + (_oldformat:rest) -> parseGitAnnexDistribution (unlines rest) + _ -> Nothing + +formatGitAnnexDistribution :: GitAnnexDistribution -> String +formatGitAnnexDistribution d = unlines + [ distributionUrl d + , key2file (distributionKey d) + , distributionVersion d + , show (distributionReleasedate d) + , maybe "" show (distributionUrgentUpgrade d) + ] + +parseGitAnnexDistribution :: String -> Maybe GitAnnexDistribution +parseGitAnnexDistribution s = case lines s of + (u:k:v:d:uu:_) -> GitAnnexDistribution + <$> pure u + <*> file2key k + <*> pure v + <*> readish d + <*> pure (readish uu) + _ -> Nothing + +data AutoUpgrade = AskUpgrade | AutoUpgrade | NoAutoUpgrade + deriving (Eq) + +toAutoUpgrade :: Maybe String -> AutoUpgrade +toAutoUpgrade Nothing = AskUpgrade +toAutoUpgrade (Just s) + | s == "ask" = AskUpgrade + | isTrue s == Just True = AutoUpgrade + | otherwise = NoAutoUpgrade + +fromAutoUpgrade :: AutoUpgrade -> String +fromAutoUpgrade AskUpgrade = "ask" +fromAutoUpgrade AutoUpgrade = boolConfig True +fromAutoUpgrade NoAutoUpgrade = boolConfig False diff --git a/Types/Export.hs b/Types/Export.hs new file mode 100644 index 0000000000..0e86f96848 --- /dev/null +++ b/Types/Export.hs @@ -0,0 +1,53 @@ +{- git-annex export types + - + - Copyright 2017 Joey Hess + - + - Licensed under the GNU GPL version 3 or higher. + -} + +module Types.Export ( + ExportLocation, + mkExportLocation, + fromExportLocation, + ExportDirectory, + mkExportDirectory, + fromExportDirectory, + exportDirectories, +) where + +import Git.FilePath + +import qualified System.FilePath.Posix as Posix + +-- A location on a remote that a key can be exported to. +-- The FilePath will be relative to the top of the export, +-- and uses unix-style path separators. +newtype ExportLocation = ExportLocation FilePath + deriving (Show, Eq) + +mkExportLocation :: FilePath -> ExportLocation +mkExportLocation = ExportLocation . toInternalGitPath + +fromExportLocation :: ExportLocation -> FilePath +fromExportLocation (ExportLocation f) = f + +newtype ExportDirectory = ExportDirectory FilePath + deriving (Show, Eq) + +mkExportDirectory :: FilePath -> ExportDirectory +mkExportDirectory = ExportDirectory . toInternalGitPath + +fromExportDirectory :: ExportDirectory -> FilePath +fromExportDirectory (ExportDirectory f) = f + +-- | All subdirectories down to the ExportLocation, with the deepest ones +-- last. Does not include the top of the export. +exportDirectories :: ExportLocation -> [ExportDirectory] +exportDirectories (ExportLocation f) = + map (ExportDirectory . Posix.joinPath . reverse) (subs [] dirs) + where + subs _ [] = [] + subs ps (d:ds) = (d:ps) : subs (d:ps) ds + + dirs = map Posix.dropTrailingPathSeparator $ + reverse $ drop 1 $ reverse $ Posix.splitPath f diff --git a/Types/FileMatcher.hs b/Types/FileMatcher.hs new file mode 100644 index 0000000000..ecf4fbbd8b --- /dev/null +++ b/Types/FileMatcher.hs @@ -0,0 +1,55 @@ +{- git-annex file matcher types + - + - Copyright 2013-2016 Joey Hess + - + - Licensed under the GNU GPL version 3 or higher. + -} + +module Types.FileMatcher where + +import Types.UUID (UUID) +import Types.Key (Key) +import Utility.Matcher (Matcher, Token) +import Utility.FileSize + +import Control.Monad.IO.Class +import qualified Data.Map as M +import qualified Data.Set as S + +data MatchInfo + = MatchingFile FileInfo + | MatchingKey Key + | MatchingInfo (OptInfo FilePath) (OptInfo Key) (OptInfo FileSize) (OptInfo MimeType) + +type MimeType = String + +data FileInfo = FileInfo + { currFile :: FilePath + -- ^ current path to the file, for operations that examine it + , matchFile :: FilePath + -- ^ filepath to match on; may be relative to top of repo or cwd + } + +type OptInfo a = Either (IO a) a + +-- If the OptInfo is not available, accessing it may result in eg an +-- exception being thrown. +getInfo :: MonadIO m => OptInfo a -> m a +getInfo (Right i) = return i +getInfo (Left e) = liftIO e + +type FileMatcherMap a = M.Map UUID (Utility.Matcher.Matcher (S.Set UUID -> MatchInfo -> a Bool)) + +type MkLimit a = String -> Either String (MatchFiles a) + +type AssumeNotPresent = S.Set UUID + +type MatchFiles a = AssumeNotPresent -> MatchInfo -> a Bool + +type FileMatcher a = Matcher (MatchFiles a) + +-- This is a matcher that can have tokens added to it while it's being +-- built, and once complete is compiled to an unchangable matcher. +data ExpandableMatcher a + = BuildingMatcher [Token (MatchInfo -> a Bool)] + | CompleteMatcher (Matcher (MatchInfo -> a Bool)) diff --git a/Types/GitConfig.hs b/Types/GitConfig.hs new file mode 100644 index 0000000000..d46c8f1d45 --- /dev/null +++ b/Types/GitConfig.hs @@ -0,0 +1,335 @@ +{- git-annex configuration + - + - Copyright 2012-2015 Joey Hess + - + - Licensed under the GNU GPL version 3 or higher. + -} + +module Types.GitConfig ( + Configurable(..), + GitConfig(..), + extractGitConfig, + mergeGitConfig, + RemoteGitConfig(..), + extractRemoteGitConfig, + dummyRemoteGitConfig, +) where + +import Common +import qualified Git +import qualified Git.Config +import qualified Git.Construct +import Git.Types +import Git.ConfigTypes +import Utility.DataUnits +import Config.Cost +import Types.UUID +import Types.Distribution +import Types.Availability +import Types.NumCopies +import Types.Difference +import Types.RefSpec +import Config.DynamicConfig +import Utility.HumanTime +import Utility.Gpg (GpgCmd, mkGpgCmd) +import Utility.ThreadScheduler (Seconds(..)) +import Utility.Url (Scheme, mkScheme) + +import Control.Concurrent.STM +import qualified Data.Set as S + +-- | A configurable value, that may not be fully determined yet because +-- the global git config has not yet been loaded. +data Configurable a + = HasConfig a + -- ^ Value is fully determined. + | DefaultConfig a + -- ^ A default value is known, but not all config sources + -- have been read yet. + deriving (Show) + +{- Main git-annex settings. Each setting corresponds to a git-config key + - such as annex.foo -} +data GitConfig = GitConfig + { annexVersion :: Maybe String + , annexUUID :: UUID + , annexNumCopies :: Maybe NumCopies + , annexDiskReserve :: Integer + , annexDirect :: Bool + , annexBackend :: Maybe String + , annexQueueSize :: Maybe Int + , annexBloomCapacity :: Maybe Int + , annexBloomAccuracy :: Maybe Int + , annexSshCaching :: Maybe Bool + , annexAlwaysCommit :: Bool + , annexCommitMessage :: Maybe String + , annexMergeAnnexBranches :: Bool + , annexDelayAdd :: Maybe Int + , annexHttpHeaders :: [String] + , annexHttpHeadersCommand :: Maybe String + , annexAutoCommit :: Configurable Bool + , annexResolveMerge :: Configurable Bool + , annexSyncContent :: Configurable Bool + , annexDebug :: Bool + , annexWebOptions :: [String] + , annexYoutubeDlOptions :: [String] + , annexAriaTorrentOptions :: [String] + , annexCrippledFileSystem :: Bool + , annexLargeFiles :: Maybe String + , annexAddSmallFiles :: Bool + , annexFsckNudge :: Bool + , annexAutoUpgrade :: AutoUpgrade + , annexExpireUnused :: Maybe (Maybe Duration) + , annexSecureEraseCommand :: Maybe String + , annexGenMetaData :: Bool + , annexListen :: Maybe String + , annexStartupScan :: Bool + , annexHardLink :: Bool + , annexThin :: Bool + , annexDifferences :: Differences + , annexUsedRefSpec :: Maybe RefSpec + , annexVerify :: Bool + , annexPidLock :: Bool + , annexPidLockTimeout :: Seconds + , annexAddUnlocked :: Bool + , annexSecureHashesOnly :: Bool + , annexRetry :: Maybe Integer + , annexRetryDelay :: Maybe Seconds + , annexAllowedUrlSchemes :: S.Set Scheme + , annexAllowedHttpAddresses :: String + , annexAllowUnverifiedDownloads :: Bool + , annexMaxExtensionLength :: Maybe Int + , coreSymlinks :: Bool + , coreSharedRepository :: SharedRepository + , receiveDenyCurrentBranch :: DenyCurrentBranch + , gcryptId :: Maybe String + , gpgCmd :: GpgCmd + } + +extractGitConfig :: Git.Repo -> GitConfig +extractGitConfig r = GitConfig + { annexVersion = notempty $ getmaybe (annex "version") + , annexUUID = maybe NoUUID toUUID $ getmaybe (annex "uuid") + , annexNumCopies = NumCopies <$> getmayberead (annex "numcopies") + , annexDiskReserve = fromMaybe onemegabyte $ + readSize dataUnits =<< getmaybe (annex "diskreserve") + , annexDirect = getbool (annex "direct") False + , annexBackend = maybe + -- annex.backends is the old name of the option, still used + -- when annex.backend is not set. + (headMaybe $ getwords (annex "backends")) + Just + (getmaybe (annex "backend")) + , annexQueueSize = getmayberead (annex "queuesize") + , annexBloomCapacity = getmayberead (annex "bloomcapacity") + , annexBloomAccuracy = getmayberead (annex "bloomaccuracy") + , annexSshCaching = getmaybebool (annex "sshcaching") + , annexAlwaysCommit = getbool (annex "alwayscommit") True + , annexCommitMessage = getmaybe (annex "commitmessage") + , annexMergeAnnexBranches = getbool (annex "merge-annex-branches") True + , annexDelayAdd = getmayberead (annex "delayadd") + , annexHttpHeaders = getlist (annex "http-headers") + , annexHttpHeadersCommand = getmaybe (annex "http-headers-command") + , annexAutoCommit = configurable True $ + getmaybebool (annex "autocommit") + , annexResolveMerge = configurable True $ + getmaybebool (annex "resolvemerge") + , annexSyncContent = configurable False $ + getmaybebool (annex "synccontent") + , annexDebug = getbool (annex "debug") False + , annexWebOptions = getwords (annex "web-options") + , annexYoutubeDlOptions = getwords (annex "youtube-dl-options") + , annexAriaTorrentOptions = getwords (annex "aria-torrent-options") + , annexCrippledFileSystem = getbool (annex "crippledfilesystem") False + , annexLargeFiles = getmaybe (annex "largefiles") + , annexAddSmallFiles = getbool (annex "addsmallfiles") True + , annexFsckNudge = getbool (annex "fscknudge") True + , annexAutoUpgrade = toAutoUpgrade $ getmaybe (annex "autoupgrade") + , annexExpireUnused = maybe Nothing Just . parseDuration + <$> getmaybe (annex "expireunused") + , annexSecureEraseCommand = getmaybe (annex "secure-erase-command") + , annexGenMetaData = getbool (annex "genmetadata") False + , annexListen = getmaybe (annex "listen") + , annexStartupScan = getbool (annex "startupscan") True + , annexHardLink = getbool (annex "hardlink") False + , annexThin = getbool (annex "thin") False + , annexDifferences = getDifferences r + , annexUsedRefSpec = either (const Nothing) Just . parseRefSpec + =<< getmaybe (annex "used-refspec") + , annexVerify = getbool (annex "verify") True + , annexPidLock = getbool (annex "pidlock") False + , annexPidLockTimeout = Seconds $ fromMaybe 300 $ + getmayberead (annex "pidlocktimeout") + , annexAddUnlocked = getbool (annex "addunlocked") False + , annexSecureHashesOnly = getbool (annex "securehashesonly") False + , annexRetry = getmayberead (annex "retry") + , annexRetryDelay = Seconds + <$> getmayberead (annex "retrydelay") + , annexAllowedUrlSchemes = S.fromList $ map mkScheme $ + maybe ["http", "https", "ftp"] words $ + getmaybe (annex "security.allowed-url-schemes") + , annexAllowedHttpAddresses = fromMaybe "" $ + getmaybe (annex "security.allowed-http-addresses") + , annexAllowUnverifiedDownloads = (== Just "ACKTHPPT") $ + getmaybe (annex "security.allow-unverified-downloads") + , annexMaxExtensionLength = getmayberead (annex "maxextensionlength") + , coreSymlinks = getbool "core.symlinks" True + , coreSharedRepository = getSharedRepository r + , receiveDenyCurrentBranch = getDenyCurrentBranch r + , gcryptId = getmaybe "core.gcrypt-id" + , gpgCmd = mkGpgCmd (getmaybe "gpg.program") + } + where + getbool k d = fromMaybe d $ getmaybebool k + getmaybebool k = Git.Config.isTrue =<< getmaybe k + getmayberead k = readish =<< getmaybe k + getmaybe k = Git.Config.getMaybe k r + getlist k = Git.Config.getList k r + getwords k = fromMaybe [] $ words <$> getmaybe k + + configurable d Nothing = DefaultConfig d + configurable _ (Just v) = HasConfig v + + annex k = "annex." ++ k + + onemegabyte = 1000000 + +{- Merge a GitConfig that comes from git-config with one containing + - repository-global defaults. -} +mergeGitConfig :: GitConfig -> GitConfig -> GitConfig +mergeGitConfig gitconfig repoglobals = gitconfig + { annexAutoCommit = merge annexAutoCommit + , annexSyncContent = merge annexSyncContent + } + where + merge f = case f gitconfig of + HasConfig v -> HasConfig v + DefaultConfig d -> case f repoglobals of + HasConfig v -> HasConfig v + DefaultConfig _ -> HasConfig d + +{- Per-remote git-annex settings. Each setting corresponds to a git-config + - key such as .annex-foo, or if that is not set, a default from + - annex.foo. + - + - Note that this is from the perspective of the local repository, + - it is not influenced in any way by the contents of the remote + - repository's git config. + -} +data RemoteGitConfig = RemoteGitConfig + { remoteAnnexCost :: DynamicConfig (Maybe Cost) + , remoteAnnexIgnore :: DynamicConfig Bool + , remoteAnnexSync :: DynamicConfig Bool + , remoteAnnexPull :: Bool + , remoteAnnexPush :: Bool + , remoteAnnexReadOnly :: Bool + , remoteAnnexVerify :: Bool + , remoteAnnexCheckUUID :: Bool + , remoteAnnexExportTracking :: Maybe Git.Ref + , remoteAnnexTrustLevel :: Maybe String + , remoteAnnexStartCommand :: Maybe String + , remoteAnnexStopCommand :: Maybe String + , remoteAnnexAvailability :: Maybe Availability + , remoteAnnexSpeculatePresent :: Bool + , remoteAnnexBare :: Maybe Bool + , remoteAnnexRetry :: Maybe Integer + , remoteAnnexRetryDelay :: Maybe Seconds + , remoteAnnexAllowUnverifiedDownloads :: Bool + + {- These settings are specific to particular types of remotes + - including special remotes. -} + , remoteAnnexShell :: Maybe String + , remoteAnnexSshOptions :: [String] + , remoteAnnexRsyncOptions :: [String] + , remoteAnnexRsyncUploadOptions :: [String] + , remoteAnnexRsyncDownloadOptions :: [String] + , remoteAnnexRsyncTransport :: [String] + , remoteAnnexGnupgOptions :: [String] + , remoteAnnexGnupgDecryptOptions :: [String] + , remoteAnnexRsyncUrl :: Maybe String + , remoteAnnexBupRepo :: Maybe String + , remoteAnnexTahoe :: Maybe FilePath + , remoteAnnexBupSplitOptions :: [String] + , remoteAnnexDirectory :: Maybe FilePath + , remoteAnnexAndroidDirectory :: Maybe FilePath + , remoteAnnexAndroidSerial :: Maybe String + , remoteAnnexGCrypt :: Maybe String + , remoteAnnexDdarRepo :: Maybe String + , remoteAnnexHookType :: Maybe String + , remoteAnnexExternalType :: Maybe String + } + +{- The Git.Repo is the local repository, which has the remote with the + - given RemoteName. -} +extractRemoteGitConfig :: Git.Repo -> RemoteName -> STM RemoteGitConfig +extractRemoteGitConfig r remotename = do + annexcost <- mkDynamicConfig readCommandRunner + (notempty $ getmaybe "cost-command") + (getmayberead "cost") + annexignore <- mkDynamicConfig unsuccessfullCommandRunner + (notempty $ getmaybe "ignore-command") + (getbool "ignore" False) + annexsync <- mkDynamicConfig successfullCommandRunner + (notempty $ getmaybe "sync-command") + (getbool "sync" True) + return $ RemoteGitConfig + { remoteAnnexCost = annexcost + , remoteAnnexIgnore = annexignore + , remoteAnnexSync = annexsync + , remoteAnnexPull = getbool "pull" True + , remoteAnnexPush = getbool "push" True + , remoteAnnexReadOnly = getbool "readonly" False + , remoteAnnexCheckUUID = getbool "checkuuid" True + , remoteAnnexVerify = getbool "verify" True + , remoteAnnexExportTracking = Git.Ref + <$> notempty (getmaybe "export-tracking") + , remoteAnnexTrustLevel = notempty $ getmaybe "trustlevel" + , remoteAnnexStartCommand = notempty $ getmaybe "start-command" + , remoteAnnexStopCommand = notempty $ getmaybe "stop-command" + , remoteAnnexAvailability = getmayberead "availability" + , remoteAnnexSpeculatePresent = getbool "speculate-present" False + , remoteAnnexBare = getmaybebool "bare" + , remoteAnnexRetry = getmayberead "retry" + , remoteAnnexRetryDelay = Seconds + <$> getmayberead "retrydelay" + , remoteAnnexAllowUnverifiedDownloads = (== Just "ACKTHPPT") $ + getmaybe ("security-allow-unverified-downloads") + , remoteAnnexShell = getmaybe "shell" + , remoteAnnexSshOptions = getoptions "ssh-options" + , remoteAnnexRsyncOptions = getoptions "rsync-options" + , remoteAnnexRsyncDownloadOptions = getoptions "rsync-download-options" + , remoteAnnexRsyncUploadOptions = getoptions "rsync-upload-options" + , remoteAnnexRsyncTransport = getoptions "rsync-transport" + , remoteAnnexGnupgOptions = getoptions "gnupg-options" + , remoteAnnexGnupgDecryptOptions = getoptions "gnupg-decrypt-options" + , remoteAnnexRsyncUrl = notempty $ getmaybe "rsyncurl" + , remoteAnnexBupRepo = getmaybe "buprepo" + , remoteAnnexTahoe = getmaybe "tahoe" + , remoteAnnexBupSplitOptions = getoptions "bup-split-options" + , remoteAnnexDirectory = notempty $ getmaybe "directory" + , remoteAnnexAndroidDirectory = notempty $ getmaybe "androiddirectory" + , remoteAnnexAndroidSerial = notempty $ getmaybe "androidserial" + , remoteAnnexGCrypt = notempty $ getmaybe "gcrypt" + , remoteAnnexDdarRepo = getmaybe "ddarrepo" + , remoteAnnexHookType = notempty $ getmaybe "hooktype" + , remoteAnnexExternalType = notempty $ getmaybe "externaltype" + } + where + getbool k d = fromMaybe d $ getmaybebool k + getmaybebool k = Git.Config.isTrue =<< getmaybe k + getmayberead k = readish =<< getmaybe k + getmaybe k = mplus (Git.Config.getMaybe (key k) r) + (Git.Config.getMaybe (remotekey k) r) + getoptions k = fromMaybe [] $ words <$> getmaybe k + + key k = "annex." ++ k + remotekey k = "remote." ++ remotename ++ ".annex-" ++ k + +notempty :: Maybe String -> Maybe String +notempty Nothing = Nothing +notempty (Just "") = Nothing +notempty (Just s) = Just s + +dummyRemoteGitConfig :: IO RemoteGitConfig +dummyRemoteGitConfig = atomically $ + extractRemoteGitConfig Git.Construct.fromUnknown "dummy" diff --git a/Types/Group.hs b/Types/Group.hs new file mode 100644 index 0000000000..2695670ab4 --- /dev/null +++ b/Types/Group.hs @@ -0,0 +1,27 @@ +{- git-annex repo groups + - + - Copyright 2012 Joey Hess + - + - Licensed under the GNU GPL version 3 or higher. + -} + +module Types.Group ( + Group, + GroupMap(..), + emptyGroupMap +) where + +import Types.UUID + +import qualified Data.Map as M +import qualified Data.Set as S + +type Group = String + +data GroupMap = GroupMap + { groupsByUUID :: M.Map UUID (S.Set Group) + , uuidsByGroup :: M.Map Group (S.Set UUID) + } + +emptyGroupMap :: GroupMap +emptyGroupMap = GroupMap M.empty M.empty diff --git a/Types/Key.hs b/Types/Key.hs new file mode 100644 index 0000000000..4b81850d80 --- /dev/null +++ b/Types/Key.hs @@ -0,0 +1,175 @@ +{- git-annex Key data type + - + - Copyright 2011-2018 Joey Hess + - + - Licensed under the GNU GPL version 3 or higher. + -} + +{-# LANGUAGE CPP #-} + +module Types.Key where + +import Utility.PartialPrelude + +import System.Posix.Types + +{- A Key has a unique name, which is derived from a particular backend, + - and may contain other optional metadata. -} +data Key = Key + { keyName :: String + , keyVariety :: KeyVariety + , keySize :: Maybe Integer + , keyMtime :: Maybe EpochTime + , keyChunkSize :: Maybe Integer + , keyChunkNum :: Maybe Integer + } deriving (Eq, Ord, Read, Show) + +{- A filename may be associated with a Key. -} +newtype AssociatedFile = AssociatedFile (Maybe FilePath) + deriving (Show, Eq, Ord) + +{- There are several different varieties of keys. -} +data KeyVariety + = SHA2Key HashSize HasExt + | SHA3Key HashSize HasExt + | SKEINKey HashSize HasExt + | Blake2bKey HashSize HasExt + | Blake2sKey HashSize HasExt + | Blake2spKey HashSize HasExt + | SHA1Key HasExt + | MD5Key HasExt + | WORMKey + | URLKey + -- Some repositories may contain keys of other varieties, + -- which can still be processed to some extent. + | OtherKey String + deriving (Eq, Ord, Read, Show) + +{- Some varieties of keys may contain an extension at the end of the + - keyName -} +newtype HasExt = HasExt Bool + deriving (Eq, Ord, Read, Show) + +newtype HashSize = HashSize Int + deriving (Eq, Ord, Read, Show) + +hasExt :: KeyVariety -> Bool +hasExt (SHA2Key _ (HasExt b)) = b +hasExt (SHA3Key _ (HasExt b)) = b +hasExt (SKEINKey _ (HasExt b)) = b +hasExt (Blake2bKey _ (HasExt b)) = b +hasExt (Blake2sKey _ (HasExt b)) = b +hasExt (Blake2spKey _ (HasExt b)) = b +hasExt (SHA1Key (HasExt b)) = b +hasExt (MD5Key (HasExt b)) = b +hasExt WORMKey = False +hasExt URLKey = False +hasExt (OtherKey s) = end s == "E" + +sameExceptExt :: KeyVariety -> KeyVariety -> Bool +sameExceptExt (SHA2Key sz1 _) (SHA2Key sz2 _) = sz1 == sz2 +sameExceptExt (SHA3Key sz1 _) (SHA3Key sz2 _) = sz1 == sz2 +sameExceptExt (SKEINKey sz1 _) (SKEINKey sz2 _) = sz1 == sz2 +sameExceptExt (Blake2bKey sz1 _) (Blake2bKey sz2 _) = sz1 == sz2 +sameExceptExt (Blake2sKey sz1 _) (Blake2sKey sz2 _) = sz1 == sz2 +sameExceptExt (Blake2spKey sz1 _) (Blake2spKey sz2 _) = sz1 == sz2 +sameExceptExt (SHA1Key _) (SHA1Key _) = True +sameExceptExt (MD5Key _) (MD5Key _) = True +sameExceptExt _ _ = False + +{- Is the Key variety cryptographically secure, such that no two differing + - file contents can be mapped to the same Key? -} +cryptographicallySecure :: KeyVariety -> Bool +cryptographicallySecure (SHA2Key _ _) = True +cryptographicallySecure (SHA3Key _ _) = True +cryptographicallySecure (SKEINKey _ _) = True +cryptographicallySecure (Blake2bKey _ _) = True +cryptographicallySecure (Blake2sKey _ _) = True +cryptographicallySecure (Blake2spKey _ _) = True +cryptographicallySecure _ = False + +{- Is the Key variety backed by a hash, which allows verifying content? + - It does not have to be cryptographically secure against eg birthday + - attacks. + -} +isVerifiable :: KeyVariety -> Bool +isVerifiable (SHA2Key _ _) = True +isVerifiable (SHA3Key _ _) = True +isVerifiable (SKEINKey _ _) = True +isVerifiable (Blake2bKey _ _) = True +isVerifiable (Blake2sKey _ _) = True +isVerifiable (Blake2spKey _ _) = True +isVerifiable (SHA1Key _) = True +isVerifiable (MD5Key _) = True +isVerifiable WORMKey = False +isVerifiable URLKey = False +isVerifiable (OtherKey _) = False + +formatKeyVariety :: KeyVariety -> String +formatKeyVariety v = case v of + SHA2Key sz e -> adde e (addsz sz "SHA") + SHA3Key sz e -> adde e (addsz sz "SHA3_") + SKEINKey sz e -> adde e (addsz sz "SKEIN") + Blake2bKey sz e -> adde e (addsz sz "BLAKE2B") + Blake2sKey sz e -> adde e (addsz sz "BLAKE2S") + Blake2spKey sz e -> adde e (addsz sz "BLAKE2SP") + SHA1Key e -> adde e "SHA1" + MD5Key e -> adde e "MD5" + WORMKey -> "WORM" + URLKey -> "URL" + OtherKey s -> s + where + adde (HasExt False) s = s + adde (HasExt True) s = s ++ "E" + addsz (HashSize n) s = s ++ show n + +parseKeyVariety :: String -> KeyVariety +parseKeyVariety "SHA256" = SHA2Key (HashSize 256) (HasExt False) +parseKeyVariety "SHA256E" = SHA2Key (HashSize 256) (HasExt True) +parseKeyVariety "SHA512" = SHA2Key (HashSize 512) (HasExt False) +parseKeyVariety "SHA512E" = SHA2Key (HashSize 512) (HasExt True) +parseKeyVariety "SHA224" = SHA2Key (HashSize 224) (HasExt False) +parseKeyVariety "SHA224E" = SHA2Key (HashSize 224) (HasExt True) +parseKeyVariety "SHA384" = SHA2Key (HashSize 384) (HasExt False) +parseKeyVariety "SHA384E" = SHA2Key (HashSize 384) (HasExt True) +parseKeyVariety "SHA3_512" = SHA3Key (HashSize 512) (HasExt False) +parseKeyVariety "SHA3_512E" = SHA3Key (HashSize 512) (HasExt True) +parseKeyVariety "SHA3_384" = SHA3Key (HashSize 384) (HasExt False) +parseKeyVariety "SHA3_384E" = SHA3Key (HashSize 384) (HasExt True) +parseKeyVariety "SHA3_256" = SHA3Key (HashSize 256) (HasExt False) +parseKeyVariety "SHA3_256E" = SHA3Key (HashSize 256) (HasExt True) +parseKeyVariety "SHA3_224" = SHA3Key (HashSize 224) (HasExt False) +parseKeyVariety "SHA3_224E" = SHA3Key (HashSize 224) (HasExt True) +parseKeyVariety "SKEIN512" = SKEINKey (HashSize 512) (HasExt False) +parseKeyVariety "SKEIN512E" = SKEINKey (HashSize 512) (HasExt True) +parseKeyVariety "SKEIN256" = SKEINKey (HashSize 256) (HasExt False) +parseKeyVariety "SKEIN256E" = SKEINKey (HashSize 256) (HasExt True) +#if MIN_VERSION_cryptonite(0,23,0) +parseKeyVariety "BLAKE2B160" = Blake2bKey (HashSize 160) (HasExt False) +parseKeyVariety "BLAKE2B160E" = Blake2bKey (HashSize 160) (HasExt True) +parseKeyVariety "BLAKE2B224" = Blake2bKey (HashSize 224) (HasExt False) +parseKeyVariety "BLAKE2B224E" = Blake2bKey (HashSize 224) (HasExt True) +parseKeyVariety "BLAKE2B256" = Blake2bKey (HashSize 256) (HasExt False) +parseKeyVariety "BLAKE2B256E" = Blake2bKey (HashSize 256) (HasExt True) +parseKeyVariety "BLAKE2B384" = Blake2bKey (HashSize 384) (HasExt False) +parseKeyVariety "BLAKE2B384E" = Blake2bKey (HashSize 384) (HasExt True) +parseKeyVariety "BLAKE2B512" = Blake2bKey (HashSize 512) (HasExt False) +parseKeyVariety "BLAKE2B512E" = Blake2bKey (HashSize 512) (HasExt True) +parseKeyVariety "BLAKE2S160" = Blake2sKey (HashSize 160) (HasExt False) +parseKeyVariety "BLAKE2S160E" = Blake2sKey (HashSize 160) (HasExt True) +parseKeyVariety "BLAKE2S224" = Blake2sKey (HashSize 224) (HasExt False) +parseKeyVariety "BLAKE2S224E" = Blake2sKey (HashSize 224) (HasExt True) +parseKeyVariety "BLAKE2S256" = Blake2sKey (HashSize 256) (HasExt False) +parseKeyVariety "BLAKE2S256E" = Blake2sKey (HashSize 256) (HasExt True) +parseKeyVariety "BLAKE2SP224" = Blake2spKey (HashSize 224) (HasExt False) +parseKeyVariety "BLAKE2SP224E" = Blake2spKey (HashSize 224) (HasExt True) +parseKeyVariety "BLAKE2SP256" = Blake2spKey (HashSize 256) (HasExt False) +parseKeyVariety "BLAKE2SP256E" = Blake2spKey (HashSize 256) (HasExt True) +#endif +parseKeyVariety "SHA1" = SHA1Key (HasExt False) +parseKeyVariety "SHA1E" = SHA1Key (HasExt True) +parseKeyVariety "MD5" = MD5Key (HasExt False) +parseKeyVariety "MD5E" = MD5Key (HasExt True) +parseKeyVariety "WORM" = WORMKey +parseKeyVariety "URL" = URLKey +parseKeyVariety s = OtherKey s diff --git a/Types/KeySource.hs b/Types/KeySource.hs new file mode 100644 index 0000000000..25774588a2 --- /dev/null +++ b/Types/KeySource.hs @@ -0,0 +1,29 @@ +{- KeySource data type + - + - Copyright 2012 Joey Hess + - + - Licensed under the GNU GPL version 3 or higher. + -} + +module Types.KeySource where + +import Utility.InodeCache + +{- When content is in the process of being ingested into the annex, + - and a Key generated from it, this data type is used. + - + - The contentLocation may be different from the filename + - associated with the key. For example, the add command + - may temporarily hard link the content into a lockdown directory + - for checking. The migrate command uses the content + - of a different Key. + - + - The inodeCache can be used to detect some types of modifications to + - files that may be made while they're in the process of being ingested. + -} +data KeySource = KeySource + { keyFilename :: FilePath + , contentLocation :: FilePath + , inodeCache :: Maybe InodeCache + } + deriving (Show) diff --git a/Types/LockCache.hs b/Types/LockCache.hs new file mode 100644 index 0000000000..8e0fd2c380 --- /dev/null +++ b/Types/LockCache.hs @@ -0,0 +1,16 @@ +{- git-annex lock cache data types + - + - Copyright 2014 Joey Hess + - + - Licensed under the GNU GPL version 3 or higher. + -} + +module Types.LockCache ( + LockCache, + LockHandle +) where + +import qualified Data.Map as M +import Utility.LockPool (LockHandle) + +type LockCache = M.Map FilePath LockHandle diff --git a/Types/Messages.hs b/Types/Messages.hs new file mode 100644 index 0000000000..8ca60651f6 --- /dev/null +++ b/Types/Messages.hs @@ -0,0 +1,65 @@ +{- git-annex Messages data types + - + - Copyright 2012-2018 Joey Hess + - + - Licensed under the GNU GPL version 3 or higher. + -} + +{-# LANGUAGE CPP #-} + +module Types.Messages where + +import qualified Utility.Aeson as Aeson + +import Control.Concurrent +#ifdef WITH_CONCURRENTOUTPUT +import System.Console.Regions (ConsoleRegion) +#endif + +data OutputType = NormalOutput | QuietOutput | JSONOutput JSONOptions + deriving (Show) + +data JSONOptions = JSONOptions + { jsonProgress :: Bool + , jsonErrorMessages :: Bool + } + deriving (Show) + +adjustOutputType :: OutputType -> OutputType -> OutputType +adjustOutputType (JSONOutput old) (JSONOutput new) = JSONOutput $ JSONOptions + { jsonProgress = jsonProgress old || jsonProgress new + , jsonErrorMessages = jsonErrorMessages old || jsonErrorMessages new + } +adjustOutputType _old new = new + +data SideActionBlock = NoBlock | StartBlock | InBlock + deriving (Eq) + +data MessageState = MessageState + { outputType :: OutputType + , concurrentOutputEnabled :: Bool + , sideActionBlock :: SideActionBlock + , implicitMessages :: Bool +#ifdef WITH_CONCURRENTOUTPUT + , consoleRegion :: Maybe ConsoleRegion + , consoleRegionErrFlag :: Bool +#endif + , jsonBuffer :: Maybe Aeson.Object + , promptLock :: MVar () -- left full when not prompting + } + +newMessageState :: IO MessageState +newMessageState = do + promptlock <- newMVar () + return $ MessageState + { outputType = NormalOutput + , concurrentOutputEnabled = False + , sideActionBlock = NoBlock + , implicitMessages = True +#ifdef WITH_CONCURRENTOUTPUT + , consoleRegion = Nothing + , consoleRegionErrFlag = False +#endif + , jsonBuffer = Nothing + , promptLock = promptlock + } diff --git a/Types/MetaData.hs b/Types/MetaData.hs new file mode 100644 index 0000000000..95b7dbb78a --- /dev/null +++ b/Types/MetaData.hs @@ -0,0 +1,359 @@ +{- git-annex general metadata + - + - Copyright 2014-2018 Joey Hess + - + - Licensed under the GNU GPL version 3 or higher. + -} + +{-# LANGUAGE GeneralizedNewtypeDeriving #-} + +module Types.MetaData ( + MetaData(..), + MetaField(..), + MetaValue(..), + CurrentlySet(..), + serialize, + deserialize, + MetaSerializable, + toMetaField, + mkMetaField, + mkMetaFieldUnchecked, + fromMetaField, + toMetaValue, + mkMetaValue, + unsetMetaValue, + unsetMetaData, + fromMetaValue, + fromMetaData, + emptyMetaData, + updateMetaData, + unionMetaData, + combineMetaData, + differenceMetaData, + isSet, + currentMetaData, + currentMetaDataValues, + metaDataValues, + ModMeta(..), + modMeta, + RemoteMetaData(..), + extractRemoteMetaData, + splitRemoteMetaDataField, + fromRemoteMetaData, + prop_metadata_sane, + prop_metadata_serialize +) where + +import Common +import Utility.Base64 +import Utility.QuickCheck +import Utility.Aeson +import Types.UUID + +import qualified Data.Text as T +import qualified Data.Set as S +import qualified Data.Map.Strict as M +import qualified Data.HashMap.Strict as HM +import Data.Char +import qualified Data.CaseInsensitive as CI + +newtype MetaData = MetaData (M.Map MetaField (S.Set MetaValue)) + deriving (Show, Eq, Ord) + +instance ToJSON' MetaData where + toJSON' (MetaData m) = object $ map go (M.toList m) + where + go (MetaField f, s) = (packString (CI.original f), toJSON' s) + +instance FromJSON MetaData where + parseJSON (Object o) = do + l <- HM.toList <$> parseJSON (Object o) + MetaData . M.fromList <$> mapM go l + where + go (t, l) = case mkMetaField (T.unpack t) of + Left e -> fail e + Right f -> (,) <$> pure f <*> parseJSON l + parseJSON _ = fail "expected an object" + +{- A metadata value can be currently be set (True), or may have been + - set before and we're remembering it no longer is (False). -} +newtype CurrentlySet = CurrentlySet Bool + deriving (Read, Show, Eq, Ord, Arbitrary) + +{- Fields are case insensitive. -} +newtype MetaField = MetaField (CI.CI String) + deriving (Read, Show, Eq, Ord) + +data MetaValue = MetaValue CurrentlySet String + deriving (Read, Show) + +instance ToJSON' MetaValue where + toJSON' (MetaValue _ v) = toJSON' v + +instance FromJSON MetaValue where + parseJSON (String v) = return $ MetaValue (CurrentlySet True) (T.unpack v) + parseJSON _ = fail "expected a string" + +{- Metadata values compare and order the same whether currently set or not. -} +instance Eq MetaValue where + MetaValue _ a == MetaValue _ b = a == b +instance Ord MetaValue where + compare (MetaValue _ x) (MetaValue _ y) = compare x y + +{- MetaData is serialized to a format like: + - + - field1 +val1 +val2 -val3 field2 +val4 +val5 + -} +class MetaSerializable v where + serialize :: v -> String + deserialize :: String -> Maybe v + +instance MetaSerializable MetaData where + serialize (MetaData m) = unwords $ concatMap go $ M.toList m + where + go (f, vs) = serialize f : map serialize (S.toList vs) + deserialize = Just . getfield emptyMetaData . words + where + getfield m [] = m + getfield m (w:ws) = maybe m (getvalues m ws) (deserialize w) + getvalues m [] _ = m + getvalues m l@(w:ws) f = case deserialize w of + Just v -> getvalues (updateMetaData f v m) ws f + Nothing -> getfield m l + +instance MetaSerializable MetaField where + serialize (MetaField f) = CI.original f + deserialize = Just . mkMetaFieldUnchecked + +{- Base64 problematic values. -} +instance MetaSerializable MetaValue where + serialize (MetaValue isset v) = + serialize isset ++ + if any isSpace v || "!" `isPrefixOf` v + then '!' : toB64 v + else v + deserialize (isset:'!':v) = MetaValue + <$> deserialize [isset] + <*> fromB64Maybe v + deserialize (isset:v) = MetaValue + <$> deserialize [isset] + <*> pure v + deserialize [] = Nothing + +instance MetaSerializable CurrentlySet where + serialize (CurrentlySet True) = "+" + serialize (CurrentlySet False) = "-" + deserialize "+" = Just (CurrentlySet True) + deserialize "-" = Just (CurrentlySet False) + deserialize _ = Nothing + +mkMetaField :: String -> Either String MetaField +mkMetaField f = maybe (Left $ badField f) Right (toMetaField f) + +badField :: String -> String +badField f = "Illegal metadata field name, \"" ++ f ++ "\"" + +{- Does not check that the field name is valid. Use with caution. -} +mkMetaFieldUnchecked :: String -> MetaField +mkMetaFieldUnchecked = MetaField . CI.mk + +toMetaField :: String -> Maybe MetaField +toMetaField f + | legalField f = Just $ MetaField $ CI.mk f + | otherwise = Nothing + +{- Fields cannot be empty, contain whitespace, or start with "+-" as + - that would break the serialization. + - + - Additionally, fields should not contain any form of path separator, as + - that would break views. + - + - So, require they have an alphanumeric first letter, with the remainder + - being either alphanumeric or a small set of whitelisted common punctuation. + -} +legalField :: String -> Bool +legalField [] = False +legalField (c1:cs) + | not (isAlphaNum c1) = False + | otherwise = all legalchars cs + where + legalchars c + | isAlphaNum c = True + | otherwise = c `elem` "_-." + +toMetaValue :: String -> MetaValue +toMetaValue = MetaValue (CurrentlySet True) + +mkMetaValue :: CurrentlySet -> String -> MetaValue +mkMetaValue = MetaValue + +unsetMetaValue :: MetaValue -> MetaValue +unsetMetaValue (MetaValue _ s) = MetaValue (CurrentlySet False) s + +{- Marks all MetaValues as no longer currently set. -} +unsetMetaData :: MetaData -> MetaData +unsetMetaData (MetaData m) = MetaData $ M.map (S.map unsetMetaValue) m + +fromMetaField :: MetaField -> String +fromMetaField (MetaField f) = CI.original f + +fromMetaValue :: MetaValue -> String +fromMetaValue (MetaValue _ f) = f + +fromMetaData :: MetaData -> [(MetaField, S.Set MetaValue)] +fromMetaData (MetaData m) = M.toList m + +emptyMetaData :: MetaData +emptyMetaData = MetaData M.empty + +{- Can be used to set a value, or to unset it, depending on whether + - the MetaValue has CurrentlySet or not. -} +updateMetaData :: MetaField -> MetaValue -> MetaData -> MetaData +updateMetaData f v = updateMetaData' f (S.singleton v) + +updateMetaData' :: MetaField -> S.Set MetaValue -> MetaData -> MetaData +updateMetaData' f s (MetaData m) = MetaData $ M.insertWith S.union f s m + +{- New metadata overrides old._-} +unionMetaData :: MetaData -> MetaData -> MetaData +unionMetaData (MetaData old) (MetaData new) = MetaData $ + M.unionWith S.union new old + +combineMetaData :: [MetaData] -> MetaData +combineMetaData = foldl' unionMetaData emptyMetaData + +differenceMetaData :: MetaData -> MetaData -> MetaData +differenceMetaData (MetaData m) (MetaData excludem) = MetaData $ + M.differenceWith diff m excludem + where + diff sl sr = + let s = S.difference sl sr + in if S.null s then Nothing else Just s + +isSet :: MetaValue -> Bool +isSet (MetaValue (CurrentlySet isset) _) = isset + +{- Gets only currently set values -} +currentMetaDataValues :: MetaField -> MetaData -> S.Set MetaValue +currentMetaDataValues f m = S.filter isSet (metaDataValues f m) + +currentMetaData :: MetaData -> MetaData +currentMetaData (MetaData m) = removeEmptyFields $ MetaData $ + M.map (S.filter isSet) m + +removeEmptyFields :: MetaData -> MetaData +removeEmptyFields (MetaData m) = MetaData $ M.filter (not . S.null) m + +{- Gets currently set values, but also values that have been unset. -} +metaDataValues :: MetaField -> MetaData -> S.Set MetaValue +metaDataValues f (MetaData m) = fromMaybe S.empty (M.lookup f m) + +mapMetaData :: (S.Set MetaValue -> S.Set MetaValue) -> MetaData -> MetaData +mapMetaData f (MetaData m) = MetaData (M.map f m) + +{- Ways that existing metadata can be modified -} +data ModMeta + = AddMeta MetaField MetaValue + | DelMeta MetaField (Maybe MetaValue) + -- ^ delete value of a field. With Just, only that specific value + -- is deleted; with Nothing, all current values are deleted. + | DelAllMeta + -- ^ delete all currently set metadata + | SetMeta MetaField (S.Set MetaValue) + -- ^ removes any existing values + | MaybeSetMeta MetaField MetaValue + -- ^ set when field has no existing value + | ComposeModMeta ModMeta ModMeta + -- ^ composing multiple modifications + deriving (Show) + +{- Applies a ModMeta, generating the new MetaData. + - Note that the new MetaData does not include all the + - values set in the input metadata. It only contains changed values. -} +modMeta :: MetaData -> ModMeta -> MetaData +modMeta _ (AddMeta f v) = updateMetaData f v emptyMetaData +modMeta _ (DelMeta f (Just oldv)) = + updateMetaData f (unsetMetaValue oldv) emptyMetaData +modMeta m (DelMeta f Nothing) = MetaData $ M.singleton f $ + S.fromList $ map unsetMetaValue $ S.toList $ currentMetaDataValues f m +modMeta m DelAllMeta = mapMetaData + (S.fromList . map unsetMetaValue . S.toList) + (currentMetaData m) +modMeta m (SetMeta f s) = updateMetaData' f s $ + foldr (updateMetaData f) emptyMetaData $ + map unsetMetaValue $ S.toList $ currentMetaDataValues f m +modMeta m (MaybeSetMeta f v) + | S.null (currentMetaDataValues f m) = updateMetaData f v emptyMetaData + | otherwise = emptyMetaData +modMeta m (ComposeModMeta a b) = unionMetaData (modMeta m a) (modMeta m b) + +data RemoteMetaData = RemoteMetaData UUID MetaData + deriving (Show, Eq, Ord) + +{- Extracts only the fields prefixed with "uuid:", which belong to that + - remote. -} +extractRemoteMetaData :: UUID -> MetaData -> RemoteMetaData +extractRemoteMetaData u (MetaData m) = RemoteMetaData u $ MetaData $ + M.mapKeys removeprefix $ M.filterWithKey belongsremote m + where + belongsremote (MetaField f) _v = prefix `isPrefixOf` CI.original f + removeprefix (MetaField f) = MetaField $ + CI.mk $ drop prefixlen $ CI.original f + prefix = remoteMetaDataPrefix u + prefixlen = length prefix + +splitRemoteMetaDataField :: MetaField -> Maybe (UUID, MetaField) +splitRemoteMetaDataField (MetaField f) = do + let (su, sf) = separate (== ':') (CI.original f) + f' <- toMetaField sf + return $ (toUUID su, f') + +remoteMetaDataPrefix :: UUID -> String +remoteMetaDataPrefix u = fromUUID u ++ ":" + +fromRemoteMetaData :: RemoteMetaData -> MetaData +fromRemoteMetaData (RemoteMetaData u (MetaData m)) = MetaData $ + M.mapKeys addprefix m + where + addprefix (MetaField f) = MetaField $ CI.mk $ (prefix ++) $ CI.original f + prefix = remoteMetaDataPrefix u + +{- Avoid putting too many fields in the map; extremely large maps make + - the seriaization test slow due to the sheer amount of data. + - It's unlikely that more than 100 fields of metadata will be used. -} +instance Arbitrary MetaData where + arbitrary = do + size <- arbitrarySizedBoundedIntegral `suchThat` (< 500) + MetaData . M.filterWithKey legal . M.fromList <$> vector size + where + legal k _v = legalField $ fromMetaField k + +instance Arbitrary MetaValue where + arbitrary = MetaValue + <$> arbitrary + -- Avoid non-ascii metavalues because fully arbitrary + -- strings may not be encoded using the filesystem + -- encoding, which is norally applied to all input. + <*> arbitrary `suchThat` all isAscii + +instance Arbitrary MetaField where + arbitrary = MetaField . CI.mk + <$> arbitrary `suchThat` legalField + +prop_metadata_sane :: MetaData -> MetaField -> MetaValue -> Bool +prop_metadata_sane m f v = and + [ S.member v $ metaDataValues f m' + , not (isSet v) || S.member v (currentMetaDataValues f m') + , differenceMetaData m' emptyMetaData == m' + ] + where + m' = updateMetaData f v m + +prop_metadata_serialize :: MetaField -> MetaValue -> MetaData -> Bool +prop_metadata_serialize f v m = and + [ deserialize (serialize f) == Just f + , deserialize (serialize v) == Just v + , deserialize (serialize m') == Just m' + ] + where + m' = removeEmptyFields m diff --git a/Types/NumCopies.hs b/Types/NumCopies.hs new file mode 100644 index 0000000000..6c673df1a4 --- /dev/null +++ b/Types/NumCopies.hs @@ -0,0 +1,178 @@ +{- git-annex numcopies types + - + - Copyright 2014-2015 Joey Hess + - + - Licensed under the GNU GPL version 3 or higher. + -} + +module Types.NumCopies ( + NumCopies(..), + fromNumCopies, + VerifiedCopy(..), + checkVerifiedCopy, + invalidateVerifiedCopy, + strongestVerifiedCopy, + deDupVerifiedCopies, + mkVerifiedCopy, + invalidatableVerifiedCopy, + withVerifiedCopy, + isSafeDrop, + SafeDropProof, + mkSafeDropProof, + ContentRemovalLock(..), +) where + +import Types.UUID +import Types.Key +import Utility.Exception (bracketIO) +import Utility.Monad + +import qualified Data.Map as M +import Control.Concurrent.MVar +import Control.Monad.Catch (MonadMask) +import Control.Monad.IO.Class (MonadIO) +import Control.Monad + +newtype NumCopies = NumCopies Int + deriving (Ord, Eq, Show) + +fromNumCopies :: NumCopies -> Int +fromNumCopies (NumCopies n) = n + +-- Indicates that a key's content is exclusively +-- locked locally, pending removal. +newtype ContentRemovalLock = ContentRemovalLock Key + deriving (Show) + +-- A verification that a copy of a key exists in a repository. +data VerifiedCopy + {- Represents a recent verification that a copy of an + - object exists in a repository with the given UUID. -} + = RecentlyVerifiedCopy V + {- Use when a repository cannot be accessed, but it's + - a trusted repository, which is on record as containing a key + - and is presumably not going to lose its copy. -} + | TrustedCopy V + {- The strongest proof of the existence of a copy. + - Until its associated action is called to unlock it, + - the copy is locked in the repository and is guaranteed + - not to be removed by any git-annex process. -} + | LockedCopy V + deriving (Show) + +data V = V + { _getUUID :: UUID + , _checkVerifiedCopy :: IO Bool + , _invalidateVerifiedCopy :: IO () + } + +instance Show V where + show v = show (_getUUID v) + +instance ToUUID VerifiedCopy where + toUUID = _getUUID . toV + +toV :: VerifiedCopy -> V +toV (TrustedCopy v) = v +toV (RecentlyVerifiedCopy v) = v +toV (LockedCopy v) = v + +-- Checks that it's still valid. +checkVerifiedCopy :: VerifiedCopy -> IO Bool +checkVerifiedCopy = _checkVerifiedCopy . toV + +invalidateVerifiedCopy :: VerifiedCopy -> IO () +invalidateVerifiedCopy = _invalidateVerifiedCopy . toV + +strongestVerifiedCopy :: VerifiedCopy -> VerifiedCopy -> VerifiedCopy +strongestVerifiedCopy a@(LockedCopy _) _ = a +strongestVerifiedCopy _ b@(LockedCopy _) = b +strongestVerifiedCopy a@(TrustedCopy _) _ = a +strongestVerifiedCopy _ b@(TrustedCopy _) = b +strongestVerifiedCopy a@(RecentlyVerifiedCopy _) _ = a + +-- Retains stronger verifications over weaker for the same uuid. +deDupVerifiedCopies :: [VerifiedCopy] -> [VerifiedCopy] +deDupVerifiedCopies l = M.elems $ + M.fromListWith strongestVerifiedCopy (zip (map toUUID l) l) + +mkVerifiedCopy :: ToUUID u => (V -> VerifiedCopy) -> u -> VerifiedCopy +mkVerifiedCopy mk u = mk $ V (toUUID u) (return True) (return ()) + +invalidatableVerifiedCopy :: ToUUID u => (V -> VerifiedCopy) -> u -> IO Bool -> IO VerifiedCopy +invalidatableVerifiedCopy mk u check = do + v <- newEmptyMVar + let invalidate = do + _ <- tryPutMVar v () + return () + let check' = isEmptyMVar v <&&> check + return $ mk $ V (toUUID u) check' invalidate + +-- Constructs a VerifiedCopy, and runs the action, ensuring that the +-- verified copy is invalidated when the action returns, or on error. +withVerifiedCopy + :: (MonadMask m, MonadIO m, ToUUID u) + => (V -> VerifiedCopy) + -> u + -> IO Bool + -> (VerifiedCopy -> m a) + -> m a +withVerifiedCopy mk u check = bracketIO setup cleanup + where + setup = invalidatableVerifiedCopy mk u check + cleanup = invalidateVerifiedCopy + +{- Check whether enough verification has been done of copies to allow + - dropping content safely. + - + - This is carefully balanced to prevent data loss when there are races + - between concurrent drops of the same content in different repos, + - without requiring impractical amounts of locking. + - + - In particular, concurrent drop races may cause the number of copies + - to fall below NumCopies, but it will never fall below 1. + -} +isSafeDrop :: NumCopies -> [VerifiedCopy] -> Maybe ContentRemovalLock -> Bool +{- When a ContentRemovalLock is provided, the content is being + - dropped from the local repo. That lock will prevent other git repos + - that are concurrently dropping from using the local copy as a VerifiedCopy. + - So, no additional locking is needed; all we need is verifications + - of any kind of N other copies of the content. -} +isSafeDrop (NumCopies n) l (Just (ContentRemovalLock _)) = + length (deDupVerifiedCopies l) >= n +{- Dropping from a remote repo. + - + - Unless numcopies is 0, at least one LockedCopy or TrustedCopy is required. + - A LockedCopy prevents races between concurrent drops from + - dropping the last copy, no matter what. + - + - The other N-1 copies can be less strong verifications, like + - RecentlyVerifiedCopy. While those are subject to concurrent drop races, + - and so could be dropped all at once, causing numcopies to be violated, + - this is the best that can be done without requiring that + - all special remotes support locking. + -} +isSafeDrop (NumCopies n) l Nothing + | n == 0 = True + | otherwise = and + [ length (deDupVerifiedCopies l) >= n + , any fullVerification l + ] + +fullVerification :: VerifiedCopy -> Bool +fullVerification (LockedCopy _) = True +fullVerification (TrustedCopy _) = True +fullVerification (RecentlyVerifiedCopy _) = False + +-- A proof that it's currently safe to drop an object. +data SafeDropProof = SafeDropProof NumCopies [VerifiedCopy] (Maybe ContentRemovalLock) + deriving (Show) + +-- Make sure that none of the VerifiedCopies have become invalidated +-- before constructing proof. +mkSafeDropProof :: NumCopies -> [VerifiedCopy] -> Maybe ContentRemovalLock -> IO (Either [VerifiedCopy] SafeDropProof) +mkSafeDropProof need have removallock = do + stillhave <- filterM checkVerifiedCopy have + return $ if isSafeDrop need stillhave removallock + then Right (SafeDropProof need stillhave removallock) + else Left stillhave diff --git a/Types/RefSpec.hs b/Types/RefSpec.hs new file mode 100644 index 0000000000..c71e57d92e --- /dev/null +++ b/Types/RefSpec.hs @@ -0,0 +1,52 @@ +{- This is not the same as git's fetch/push refspecs. + - + - Copyright 2015 Joey Hess + - + - Licensed under the GNU GPL version 3 or higher. + -} + +module Types.RefSpec where + +import Common +import Utility.Glob +import Git.Types + +import Data.Either + +type RefSpec = [RefSpecPart] + +data RefSpecPart + = AddRef Ref + | AddMatching Glob + | AddRefLog + | RemoveMatching Glob + +allRefSpec :: RefSpec +allRefSpec = [AddMatching $ compileGlob "*" CaseSensative] + +parseRefSpec :: String -> Either String RefSpec +parseRefSpec v = case partitionEithers (map mk $ splitc ':' v) of + ([],refspec) -> Right refspec + (e:_,_) -> Left e + where + mk ('+':s) + | any (`elem` s) "*?" = + Right $ AddMatching $ compileGlob s CaseSensative + | otherwise = Right $ AddRef $ Ref s + mk ('-':s) = Right $ RemoveMatching $ compileGlob s CaseSensative + mk "reflog" = Right AddRefLog + mk s = Left $ "bad refspec item \"" ++ s ++ "\" (expected + or - prefix)" + +applyRefSpec :: Monad m => RefSpec -> [Ref] -> m [Sha] -> m [Ref] +applyRefSpec refspec rs getreflog = go [] refspec + where + go c [] = return (reverse c) + go c (AddRef r : rest) = go (r:c) rest + go c (AddMatching g : rest) = + let add = filter (matchGlob g . fromRef) rs + in go (add ++ c) rest + go c (AddRefLog : rest) = do + reflog <- getreflog + go (reflog ++ c) rest + go c (RemoveMatching g : rest) = + go (filter (not . matchGlob g . fromRef) c) rest diff --git a/Types/Remote.hs b/Types/Remote.hs new file mode 100644 index 0000000000..3f49c819e5 --- /dev/null +++ b/Types/Remote.hs @@ -0,0 +1,236 @@ +{- git-annex remotes types + - + - Most things should not need this, using Types instead + - + - Copyright 2011-2018 Joey Hess + - + - Licensed under the GNU GPL version 3 or higher. + -} + +{-# LANGUAGE RankNTypes #-} + +module Types.Remote + ( RemoteConfigKey + , RemoteConfig + , RemoteTypeA(..) + , RemoteA(..) + , SetupStage(..) + , Availability(..) + , Verification(..) + , unVerified + , RetrievalSecurityPolicy(..) + , isExportSupported + , ExportActions(..) + ) + where + +import qualified Data.Map as M +import Data.Ord + +import qualified Git +import Types.Key +import Types.UUID +import Types.GitConfig +import Types.Availability +import Types.Creds +import Types.UrlContents +import Types.NumCopies +import Types.Export +import Config.Cost +import Utility.Metered +import Git.Types (RemoteName) +import Utility.SafeCommand +import Utility.Url + +type RemoteConfigKey = String + +type RemoteConfig = M.Map RemoteConfigKey String + +data SetupStage = Init | Enable RemoteConfig + +{- There are different types of remotes. -} +data RemoteTypeA a = RemoteType + -- human visible type name + { typename :: String + -- enumerates remotes of this type + -- The Bool is True if automatic initialization of remotes is desired + , enumerate :: Bool -> a [Git.Repo] + -- generates a remote of this type + , generate :: Git.Repo -> UUID -> RemoteConfig -> RemoteGitConfig -> a (Maybe (RemoteA a)) + -- initializes or enables a remote + , setup :: SetupStage -> Maybe UUID -> Maybe CredPair -> RemoteConfig -> RemoteGitConfig -> a (RemoteConfig, UUID) + -- check if a remote of this type is able to support export + , exportSupported :: RemoteConfig -> RemoteGitConfig -> a Bool + } + +instance Eq (RemoteTypeA a) where + x == y = typename x == typename y + +{- An individual remote. -} +data RemoteA a = Remote + -- each Remote has a unique uuid + { uuid :: UUID + -- each Remote has a human visible name + , name :: RemoteName + -- Remotes have a use cost; higher is more expensive + , cost :: Cost + + -- Transfers a key's contents from disk to the remote. + -- The key should not appear to be present on the remote until + -- all of its contents have been transferred. + , storeKey :: Key -> AssociatedFile -> MeterUpdate -> a Bool + -- Retrieves a key's contents to a file. + -- (The MeterUpdate does not need to be used if it writes + -- sequentially to the file.) + , retrieveKeyFile :: Key -> AssociatedFile -> FilePath -> MeterUpdate -> a (Bool, Verification) + -- Retrieves a key's contents to a tmp file, if it can be done cheaply. + -- It's ok to create a symlink or hardlink. + , retrieveKeyFileCheap :: Key -> AssociatedFile -> FilePath -> a Bool + -- Security policy for reteiving keys from this remote. + , retrievalSecurityPolicy :: RetrievalSecurityPolicy + -- Removes a key's contents (succeeds if the contents are not present) + , removeKey :: Key -> a Bool + -- Uses locking to prevent removal of a key's contents, + -- thus producing a VerifiedCopy, which is passed to the callback. + -- If unable to lock, does not run the callback, and throws an + -- error. + -- This is optional; remotes do not have to support locking. + , lockContent :: forall r. Maybe (Key -> (VerifiedCopy -> a r) -> a r) + -- Checks if a key is present in the remote. + -- Throws an exception if the remote cannot be accessed. + , checkPresent :: Key -> a Bool + -- Some remotes can checkPresent without an expensive network + -- operation. + , checkPresentCheap :: Bool + -- Some remotes support exports of trees. + , exportActions :: a (ExportActions a) + -- Some remotes can provide additional details for whereis. + , whereisKey :: Maybe (Key -> a [String]) + -- Some remotes can run a fsck operation on the remote, + -- without transferring all the data to the local repo + -- The parameters are passed to the fsck command on the remote. + , remoteFsck :: Maybe ([CommandParam] -> a (IO Bool)) + -- Runs an action to repair the remote's git repository. + , repairRepo :: Maybe (a Bool -> a (IO Bool)) + -- a Remote has a persistent configuration store + , config :: RemoteConfig + -- Get the git repo for the Remote. + , getRepo :: a Git.Repo + -- a Remote's configuration from git + , gitconfig :: RemoteGitConfig + -- a Remote can be assocated with a specific local filesystem path + , localpath :: Maybe FilePath + -- a Remote can be known to be readonly + , readonly :: Bool + -- a Remote can allow writes but not have a way to delete content + -- from it. Note that an export remote that supports removeExport + -- to remove a file from the exported tree, but still retains the + -- content in accessible form should set this to True. + , appendonly :: Bool + -- a Remote can be globally available. (Ie, "in the cloud".) + , availability :: Availability + -- the type of the remote + , remotetype :: RemoteTypeA a + -- For testing, makes a version of this remote that is not + -- available for use. All its actions should fail. + , mkUnavailable :: a (Maybe (RemoteA a)) + -- Information about the remote, for git annex info to display. + , getInfo :: a [(String, String)] + -- Some remotes can download from an url (or uri). + , claimUrl :: Maybe (URLString -> a Bool) + -- Checks that the url is accessible, and gets information about + -- its contents, without downloading the full content. + -- Throws an exception if the url is inaccessible. + , checkUrl :: Maybe (URLString -> a UrlContents) + } + +instance Show (RemoteA a) where + show remote = "Remote { name =\"" ++ name remote ++ "\" }" + +-- two remotes are the same if they have the same uuid +instance Eq (RemoteA a) where + x == y = uuid x == uuid y + +-- Order by cost since that is the important order of remotes +-- when deciding which to use. But since remotes often have the same cost +-- and Ord must be total, do a secondary ordering by uuid. +instance Ord (RemoteA a) where + compare a b + | cost a == cost b = comparing uuid a b + | otherwise = comparing cost a b + +instance ToUUID (RemoteA a) where + toUUID = uuid + +data Verification + = UnVerified + -- ^ Content was not verified during transfer, but is probably + -- ok, so if verification is disabled, don't verify it + | Verified + -- ^ Content was verified during transfer, so don't verify it + -- again. + | MustVerify + -- ^ Content likely to have been altered during transfer, + -- verify even if verification is normally disabled + +unVerified :: Monad m => m Bool -> m (Bool, Verification) +unVerified a = do + ok <- a + return (ok, UnVerified) + +-- Security policy indicating what keys can be safely retrieved from a +-- remote. +data RetrievalSecurityPolicy + = RetrievalVerifiableKeysSecure + -- ^ Transfer of keys whose content can be verified + -- with a hash check is secure; transfer of unverifiable keys is + -- not secure and should not be allowed. + -- + -- This is used eg, when HTTP to a remote could be redirected to a + -- local private web server or even a file:// url, causing private + -- data from it that is not the intended content of a key to make + -- its way into the git-annex repository. + -- + -- It's also used when content is stored encrypted on a remote, + -- which could replace it with a different encrypted file, and + -- trick git-annex into decrypting it and leaking the decryption + -- into the git-annex repository. + -- + -- It's not (currently) used when the remote could alter the + -- content stored on it, because git-annex does not provide + -- strong guarantees about the content of keys that cannot be + -- verified with a hash check. + -- (But annex.securehashesonly does provide such guarantees.) + | RetrievalAllKeysSecure + -- ^ Any key can be securely retrieved. + +isExportSupported :: RemoteA a -> a Bool +isExportSupported r = exportSupported (remotetype r) (config r) (gitconfig r) + +data ExportActions a = ExportActions + -- Exports content to an ExportLocation. + -- The exported file should not appear to be present on the remote + -- until all of its contents have been transferred. + { storeExport :: FilePath -> Key -> ExportLocation -> MeterUpdate -> a Bool + -- Retrieves exported content to a file. + -- (The MeterUpdate does not need to be used if it writes + -- sequentially to the file.) + , retrieveExport :: Key -> ExportLocation -> FilePath -> MeterUpdate -> a Bool + -- Removes an exported file (succeeds if the contents are not present) + , removeExport :: Key -> ExportLocation -> a Bool + -- Removes an exported directory. Typically the directory will be + -- empty, but it could possbly contain files or other directories, + -- and it's ok to delete those. If the remote does not use + -- directories, or automatically cleans up empty directories, + -- this can be Nothing. Should not fail if the directory was + -- already removed. + , removeExportDirectory :: Maybe (ExportDirectory -> a Bool) + -- Checks if anything is exported to the remote at the specified + -- ExportLocation. + -- Throws an exception if the remote cannot be accessed. + , checkPresentExport :: Key -> ExportLocation -> a Bool + -- Renames an already exported file. + -- This may fail, if the file doesn't exist, or the remote does not + -- support renames. + , renameExport :: Key -> ExportLocation -> ExportLocation -> a Bool + } diff --git a/Types/ScheduledActivity.hs b/Types/ScheduledActivity.hs new file mode 100644 index 0000000000..f4f80635bd --- /dev/null +++ b/Types/ScheduledActivity.hs @@ -0,0 +1,69 @@ +{- git-annex scheduled activities + - + - Copyright 2013 Joey Hess + - + - Licensed under the GNU GPL version 3 or higher. + -} + +module Types.ScheduledActivity where + +import Common +import Utility.Scheduled +import Utility.HumanTime +import Types.UUID + +import Data.Either + +data ScheduledActivity + = ScheduledSelfFsck Schedule Duration + | ScheduledRemoteFsck UUID Schedule Duration + deriving (Eq, Read, Show, Ord) + +{- Activities that run on a remote, within a time window, so + - should be run when the remote gets connected. -} +connectActivityUUID :: ScheduledActivity -> Maybe UUID +connectActivityUUID (ScheduledRemoteFsck u (Schedule _ AnyTime) _) = Just u +connectActivityUUID _ = Nothing + +getSchedule :: ScheduledActivity -> Schedule +getSchedule (ScheduledSelfFsck s _) = s +getSchedule (ScheduledRemoteFsck _ s _) = s + +getDuration :: ScheduledActivity -> Duration +getDuration (ScheduledSelfFsck _ d) = d +getDuration (ScheduledRemoteFsck _ _ d) = d + +fromScheduledActivity :: ScheduledActivity -> String +fromScheduledActivity (ScheduledSelfFsck s d) = unwords + [ "fsck self", fromDuration d, fromSchedule s ] +fromScheduledActivity (ScheduledRemoteFsck u s d) = unwords + [ "fsck", fromUUID u, fromDuration d, fromSchedule s ] + +toScheduledActivity :: String -> Maybe ScheduledActivity +toScheduledActivity = eitherToMaybe . parseScheduledActivity + +parseScheduledActivity :: String -> Either String ScheduledActivity +parseScheduledActivity s = case words s of + ("fsck":"self":d:rest) -> qualified $ ScheduledSelfFsck + <$> parseSchedule (unwords rest) + <*> getduration d + ("fsck":u:d:rest) -> qualified $ ScheduledRemoteFsck + <$> pure (toUUID u) + <*> parseSchedule (unwords rest) + <*> getduration d + _ -> qualified $ Left "unknown activity" + where + qualified (Left e) = Left $ e ++ " in \"" ++ s ++ "\"" + qualified v = v + getduration d = maybe (Left $ "failed to parse duration \""++d++"\"") Right (parseDuration d) + +fromScheduledActivities :: [ScheduledActivity] -> String +fromScheduledActivities = intercalate "; " . map fromScheduledActivity + +parseScheduledActivities :: String -> Either String [ScheduledActivity] +parseScheduledActivities s + | null bad = Right good + | otherwise = Left $ intercalate "; " bad + where + (bad, good) = partitionEithers $ + map parseScheduledActivity $ split "; " s diff --git a/Types/StandardGroups.hs b/Types/StandardGroups.hs new file mode 100644 index 0000000000..2d24024d81 --- /dev/null +++ b/Types/StandardGroups.hs @@ -0,0 +1,105 @@ +{- git-annex standard repository groups + - + - Copyright 2012 Joey Hess + - + - Licensed under the GNU GPL version 3 or higher. + -} + +module Types.StandardGroups where + +import Types.Remote (RemoteConfig) +import Types.Group + +import qualified Data.Map as M +import Data.Maybe + +type PreferredContentExpression = String + +data StandardGroup + = ClientGroup + | TransferGroup + | BackupGroup + | IncrementalBackupGroup + | SmallArchiveGroup + | FullArchiveGroup + | SourceGroup + | ManualGroup + | PublicGroup + | UnwantedGroup + deriving (Eq, Ord, Enum, Bounded, Show) + +fromStandardGroup :: StandardGroup -> Group +fromStandardGroup ClientGroup = "client" +fromStandardGroup TransferGroup = "transfer" +fromStandardGroup BackupGroup = "backup" +fromStandardGroup IncrementalBackupGroup = "incrementalbackup" +fromStandardGroup SmallArchiveGroup = "smallarchive" +fromStandardGroup FullArchiveGroup = "archive" +fromStandardGroup SourceGroup = "source" +fromStandardGroup ManualGroup = "manual" +fromStandardGroup PublicGroup = "public" +fromStandardGroup UnwantedGroup = "unwanted" + +toStandardGroup :: Group -> Maybe StandardGroup +toStandardGroup "client" = Just ClientGroup +toStandardGroup "transfer" = Just TransferGroup +toStandardGroup "backup" = Just BackupGroup +toStandardGroup "incrementalbackup" = Just IncrementalBackupGroup +toStandardGroup "smallarchive" = Just SmallArchiveGroup +toStandardGroup "archive" = Just FullArchiveGroup +toStandardGroup "source" = Just SourceGroup +toStandardGroup "manual" = Just ManualGroup +toStandardGroup "public" = Just PublicGroup +toStandardGroup "unwanted" = Just UnwantedGroup +toStandardGroup _ = Nothing + +descStandardGroup :: StandardGroup -> String +descStandardGroup ClientGroup = "client: a repository on your computer" +descStandardGroup TransferGroup = "transfer: distributes files to clients" +descStandardGroup BackupGroup = "full backup: backs up all files" +descStandardGroup IncrementalBackupGroup = "incremental backup: backs up files not backed up elsewhere" +descStandardGroup SmallArchiveGroup = "small archive: archives files located in \"archive\" directories" +descStandardGroup FullArchiveGroup = "full archive: archives all files not archived elsewhere" +descStandardGroup SourceGroup = "file source: moves files on to other repositories" +descStandardGroup ManualGroup = "manual mode: only stores files you manually choose" +descStandardGroup UnwantedGroup = "unwanted: remove content from this repository" +descStandardGroup PublicGroup = "public: publishes files located in an associated directory" + +associatedDirectory :: Maybe RemoteConfig -> StandardGroup -> Maybe FilePath +associatedDirectory _ SmallArchiveGroup = Just "archive" +associatedDirectory _ FullArchiveGroup = Just "archive" +associatedDirectory (Just c) PublicGroup = Just $ + fromMaybe "public" $ M.lookup "preferreddir" c +associatedDirectory Nothing PublicGroup = Just "public" +associatedDirectory _ _ = Nothing + +specialRemoteOnly :: StandardGroup -> Bool +specialRemoteOnly PublicGroup = True +specialRemoteOnly _ = False + +{- See doc/preferred_content.mdwn for explanations of these expressions. -} +standardPreferredContent :: StandardGroup -> PreferredContentExpression +standardPreferredContent ClientGroup = lastResort $ + "include=* and ((exclude=*/archive/* and exclude=archive/*) or (" ++ notArchived ++ "))" +standardPreferredContent TransferGroup = lastResort $ + "not (inallgroup=client and copies=client:2) and (" ++ standardPreferredContent ClientGroup ++ ")" +standardPreferredContent BackupGroup = "anything" +standardPreferredContent IncrementalBackupGroup = lastResort + "(not copies=backup:1) and (not copies=incrementalbackup:1)" +standardPreferredContent SmallArchiveGroup = lastResort $ + "(include=*/archive/* or include=archive/*) and (" ++ standardPreferredContent FullArchiveGroup ++ ")" +standardPreferredContent FullArchiveGroup = lastResort notArchived +standardPreferredContent SourceGroup = "not (copies=1)" +standardPreferredContent ManualGroup = "present and (" ++ standardPreferredContent ClientGroup ++ ")" +standardPreferredContent PublicGroup = "inpreferreddir" +standardPreferredContent UnwantedGroup = "not anything" + +notArchived :: String +notArchived = "not (copies=archive:1 or copies=smallarchive:1)" + +{- Most repositories want any content that is only on untrusted + - or dead repositories, or that otherwise does not have enough copies. + - Does not look at .gitattributes since that is quite a lot slower. + -} +lastResort :: String -> PreferredContentExpression +lastResort s = "(" ++ s ++ ") or approxlackingcopies=1" diff --git a/Types/StoreRetrieve.hs b/Types/StoreRetrieve.hs new file mode 100644 index 0000000000..c87b6cbb9e --- /dev/null +++ b/Types/StoreRetrieve.hs @@ -0,0 +1,43 @@ +{- Types for Storer and Retriever actions for remotes. + - + - Copyright 2014 Joey Hess + - + - Licensed under the GNU GPL version 3 or higher. + -} + +module Types.StoreRetrieve where + +import Annex.Common +import Utility.Metered + +import qualified Data.ByteString.Lazy as L + +-- Prepares for and then runs an action that will act on a Key's +-- content, passing it a helper when the preparation is successful. +type Preparer helper = Key -> (Maybe helper -> Annex Bool) -> Annex Bool + +-- A source of a Key's content. +data ContentSource + = FileContent FilePath + | ByteContent L.ByteString + +isByteContent :: ContentSource -> Bool +isByteContent (ByteContent _) = True +isByteContent (FileContent _) = False + +-- Action that stores a Key's content on a remote. +-- Can throw exceptions. +type Storer = Key -> ContentSource -> MeterUpdate -> Annex Bool + +-- Action that retrieves a Key's content from a remote, passing it to a +-- callback, which will fully consume the content before returning. +-- Throws exception if key is not present, or remote is not accessible. +type Retriever = Key -> MeterUpdate -> (ContentSource -> Annex Bool) -> Annex Bool + +-- Action that removes a Key's content from a remote. +-- Succeeds if key is already not present; never throws exceptions. +type Remover = Key -> Annex Bool + +-- Checks if a Key's content is present on a remote. +-- Throws an exception if the remote is not accessible. +type CheckPresent = Key -> Annex Bool diff --git a/Types/Test.hs b/Types/Test.hs new file mode 100644 index 0000000000..5a9a9e075c --- /dev/null +++ b/Types/Test.hs @@ -0,0 +1,49 @@ +{- git-annex test data types. + - + - Copyright 2011-2017 Joey Hess + - + - Licensed under the GNU GPL version 3 or higher. + -} + +{-# LANGUAGE CPP #-} + +module Types.Test where + +import Test.Tasty.Options +import Data.Monoid +#if MIN_VERSION_base(4,9,0) +import qualified Data.Semigroup as Sem +#endif +import Prelude + +import Types.Command + +data TestOptions = TestOptions + { tastyOptionSet :: OptionSet + , keepFailuresOption :: Bool + , fakeSsh :: Bool + , internalData :: CmdParams + } + +appendTestOptions :: TestOptions -> TestOptions -> TestOptions +appendTestOptions a b = TestOptions + (tastyOptionSet a <> tastyOptionSet b) + (keepFailuresOption a || keepFailuresOption b) + (fakeSsh a || fakeSsh b) + (internalData a <> internalData b) + +#if MIN_VERSION_base(4,9,0) +instance Sem.Semigroup TestOptions where + (<>) = appendTestOptions +#endif + +instance Monoid TestOptions where + mempty = TestOptions mempty False False mempty +#if MIN_VERSION_base(4,11,0) +#elif MIN_VERSION_base(4,9,0) + mappend = (Sem.<>) +#else + mappend = appendTestOptions +#endif + +type TestRunner = TestOptions -> IO () diff --git a/Types/Transfer.hs b/Types/Transfer.hs new file mode 100644 index 0000000000..aa76bf7ec7 --- /dev/null +++ b/Types/Transfer.hs @@ -0,0 +1,103 @@ +{- git-annex transfer types + - + - Copyright 2012 Joey Hess + - + - Licensed under the GNU GPL version 3 or higher. + -} + +{-# LANGUAGE FlexibleInstances #-} + +module Types.Transfer where + +import Types +import Types.Remote (Verification(..)) +import Utility.PID +import Utility.QuickCheck +import Utility.Url + +import Data.Time.Clock.POSIX +import Control.Concurrent +import Control.Applicative +import Prelude + +{- Enough information to uniquely identify a transfer. -} +data Transfer = Transfer + { transferDirection :: Direction + , transferUUID :: UUID + , transferKey :: Key + } + deriving (Eq, Ord, Read, Show) + +{- Information about a Transfer, stored in the transfer information file. + - + - Note that the associatedFile may not correspond to a file in the local + - git repository. It's some file, possibly relative to some directory, + - of some repository, that was acted on to initiate the transfer. + -} +data TransferInfo = TransferInfo + { startedTime :: Maybe POSIXTime + , transferPid :: Maybe PID + , transferTid :: Maybe ThreadId + , transferRemote :: Maybe Remote + , bytesComplete :: Maybe Integer + , associatedFile :: AssociatedFile + , transferPaused :: Bool + } + deriving (Show, Eq, Ord) + +stubTransferInfo :: TransferInfo +stubTransferInfo = TransferInfo Nothing Nothing Nothing Nothing Nothing (AssociatedFile Nothing) False + +data Direction = Upload | Download + deriving (Eq, Ord, Show, Read) + +formatDirection :: Direction -> String +formatDirection Upload = "upload" +formatDirection Download = "download" + +parseDirection :: String -> Maybe Direction +parseDirection "upload" = Just Upload +parseDirection "download" = Just Download +parseDirection _ = Nothing + +instance Arbitrary TransferInfo where + arbitrary = TransferInfo + <$> arbitrary + <*> arbitrary + <*> pure Nothing -- cannot generate a ThreadID + <*> pure Nothing -- remote not needed + <*> arbitrary + -- associated file cannot be empty (but can be Nothing) + <*> (AssociatedFile <$> arbitrary `suchThat` (/= Just "")) + <*> arbitrary + +class Observable a where + observeBool :: a -> Bool + observeFailure :: a + +instance Observable Bool where + observeBool = id + observeFailure = False + +instance Observable (Bool, Verification) where + observeBool = fst + observeFailure = (False, UnVerified) + +instance Observable (Either e Bool) where + observeBool (Left _) = False + observeBool (Right b) = b + observeFailure = Right False + +instance Observable (Maybe a) where + observeBool (Just _) = True + observeBool Nothing = False + observeFailure = Nothing + +class Transferrable t where + descTransfrerrable :: t -> Maybe String + +instance Transferrable AssociatedFile where + descTransfrerrable (AssociatedFile af) = af + +instance Transferrable URLString where + descTransfrerrable = Just diff --git a/Types/TrustLevel.hs b/Types/TrustLevel.hs new file mode 100644 index 0000000000..8aec125d7d --- /dev/null +++ b/Types/TrustLevel.hs @@ -0,0 +1,51 @@ +{- git-annex trust levels + - + - Copyright 2010 Joey Hess + - + - Licensed under the GNU GPL version 3 or higher. + -} + +{-# LANGUAGE FlexibleInstances #-} + +module Types.TrustLevel ( + TrustLevel(..), + TrustMap, + readTrustLevel, + showTrustLevel, + prop_read_show_TrustLevel +) where + +import qualified Data.Map as M +import Data.Default +import Data.Ord + +import Types.UUID + +data TrustLevel = DeadTrusted | UnTrusted | SemiTrusted | Trusted + deriving (Eq, Enum, Ord, Bounded, Show) + +instance Default TrustLevel where + def = SemiTrusted + +instance Default (Down TrustLevel) where + def = Down def + +type TrustMap = M.Map UUID TrustLevel + +readTrustLevel :: String -> Maybe TrustLevel +readTrustLevel "trusted" = Just Trusted +readTrustLevel "untrusted" = Just UnTrusted +readTrustLevel "semitrusted" = Just SemiTrusted +readTrustLevel "dead" = Just DeadTrusted +readTrustLevel _ = Nothing + +showTrustLevel :: TrustLevel -> String +showTrustLevel Trusted = "trusted" +showTrustLevel UnTrusted = "untrusted" +showTrustLevel SemiTrusted = "semitrusted" +showTrustLevel DeadTrusted = "dead" + +prop_read_show_TrustLevel :: Bool +prop_read_show_TrustLevel = all check [minBound .. maxBound] + where + check l = readTrustLevel (showTrustLevel l) == Just l diff --git a/Types/UUID.hs b/Types/UUID.hs new file mode 100644 index 0000000000..f5c9cda301 --- /dev/null +++ b/Types/UUID.hs @@ -0,0 +1,43 @@ +{- git-annex UUID type + - + - Copyright 2011 Joey Hess + - + - Licensed under the GNU GPL version 3 or higher. + -} + +{-# LANGUAGE TypeSynonymInstances, FlexibleInstances #-} + +module Types.UUID where + +import qualified Data.Map as M +import qualified Data.UUID as U +import Data.Maybe + +import qualified Utility.SimpleProtocol as Proto + +-- A UUID is either an arbitrary opaque string, or UUID info may be missing. +data UUID = NoUUID | UUID String + deriving (Eq, Ord, Show, Read) + +fromUUID :: UUID -> String +fromUUID (UUID u) = u +fromUUID NoUUID = "" + +class ToUUID a where + toUUID :: a -> UUID + +instance ToUUID UUID where + toUUID = id + +instance ToUUID String where + toUUID [] = NoUUID + toUUID s = UUID s + +isUUID :: String -> Bool +isUUID = isJust . U.fromString + +type UUIDMap = M.Map UUID String + +instance Proto.Serializable UUID where + serialize = fromUUID + deserialize = Just . toUUID diff --git a/Types/UrlContents.hs b/Types/UrlContents.hs new file mode 100644 index 0000000000..411c3ae42f --- /dev/null +++ b/Types/UrlContents.hs @@ -0,0 +1,47 @@ +{- git-annex URL contents + - + - Copyright 2014 Joey Hess + - + - Licensed under the GNU GPL version 3 or higher. + -} + +module Types.UrlContents ( + UrlContents(..), + SafeFilePath, + mkSafeFilePath, + fromSafeFilePath +) where + +import Utility.Url +import Utility.Path + +import System.FilePath + +data UrlContents + -- An URL contains a file, whose size may be known. + -- There might be a nicer filename to use. + = UrlContents (Maybe Integer) (Maybe SafeFilePath) + -- Sometimes an URL points to multiple files, each accessible + -- by their own URL. + | UrlMulti [(URLString, Maybe Integer, SafeFilePath)] + +-- This is a FilePath, from an untrusted source, +-- sanitized so it doesn't contain any directory traversal tricks +-- and is always relative. It can still contain subdirectories. +-- Any unusual characters are also filtered out. +newtype SafeFilePath = SafeFilePath FilePath + deriving (Show) + +mkSafeFilePath :: FilePath -> SafeFilePath +mkSafeFilePath p = SafeFilePath $ if null p' then "file" else p' + where + p' = joinPath $ filter safe $ map sanitizeFilePath $ splitDirectories p + safe s + | isDrive s = False + | s == ".." = False + | s == ".git" = False + | null s = False + | otherwise = True + +fromSafeFilePath :: SafeFilePath -> FilePath +fromSafeFilePath (SafeFilePath p) = p diff --git a/Types/View.hs b/Types/View.hs new file mode 100644 index 0000000000..a6c52b3d28 --- /dev/null +++ b/Types/View.hs @@ -0,0 +1,60 @@ +{- types for metadata based branch views + - + - Copyright 2014 Joey Hess + - + - Licensed under the GNU GPL version 3 or higher. + -} + +module Types.View where + +import Annex.Common +import Types.MetaData +import Utility.QuickCheck +import qualified Git + +import qualified Data.Set as S + +{- A view is a list of fields with filters on their allowed values, + - which are applied to files in a parent git branch. -} +data View = View + { viewParentBranch :: Git.Branch + , viewComponents :: [ViewComponent] + } + deriving (Eq, Read, Show) + +instance Arbitrary View where + arbitrary = View <$> pure (Git.Ref "master") <*> arbitrary + +data ViewComponent = ViewComponent + { viewField :: MetaField + , viewFilter :: ViewFilter + , viewVisible :: Bool + } + deriving (Eq, Read, Show) + +instance Arbitrary ViewComponent where + arbitrary = ViewComponent <$> arbitrary <*> arbitrary <*> arbitrary + +data ViewFilter + = FilterValues (S.Set MetaValue) + | FilterGlob String + | ExcludeValues (S.Set MetaValue) + deriving (Eq, Read, Show) + +instance Arbitrary ViewFilter where + arbitrary = do + size <- arbitrarySizedBoundedIntegral `suchThat` (< 100) + s <- S.fromList <$> vector size + ifM arbitrary + ( return (FilterValues s) + , return (ExcludeValues s) + ) + +mkViewComponent :: MetaField -> ViewFilter -> ViewComponent +mkViewComponent f vf = ViewComponent f vf (multiValue vf) + +{- Can a ViewFilter match multiple different MetaValues? -} +multiValue :: ViewFilter -> Bool +multiValue (FilterValues s) = S.size s > 1 +multiValue (FilterGlob _) = True +multiValue (ExcludeValues _) = False diff --git a/Upgrade.hs b/Upgrade.hs new file mode 100644 index 0000000000..c6552f89c0 --- /dev/null +++ b/Upgrade.hs @@ -0,0 +1,59 @@ +{- git-annex upgrade support + - + - Copyright 2010, 2013 Joey Hess + - + - Licensed under the GNU GPL version 3 or higher. + -} + +{-# LANGUAGE CPP #-} + +module Upgrade where + +import Annex.Common +import Annex.Version +#ifndef mingw32_HOST_OS +import qualified Upgrade.V0 +import qualified Upgrade.V1 +#endif +import qualified Upgrade.V2 +import qualified Upgrade.V3 +import qualified Upgrade.V4 +import qualified Upgrade.V5 + +checkUpgrade :: Version -> Annex () +checkUpgrade = maybe noop giveup <=< needsUpgrade + +needsUpgrade :: Version -> Annex (Maybe String) +needsUpgrade v + | v `elem` supportedVersions = ok + | v `elem` autoUpgradeableVersions = ifM (upgrade True defaultVersion) + ( ok + , err "Automatic upgrade failed!" + ) + | v `elem` upgradableVersions = err "Upgrade this repository: git-annex upgrade" + | otherwise = err "Upgrade git-annex." + where + err msg = return $ Just $ "Repository version " ++ v ++ + " is not supported. " ++ msg + ok = return Nothing + +upgrade :: Bool -> Version -> Annex Bool +upgrade automatic destversion = do + upgraded <- go =<< getVersion + when upgraded $ + setVersion destversion + return upgraded + where + go (Just v) | v >= destversion = return True +#ifndef mingw32_HOST_OS + go (Just "0") = Upgrade.V0.upgrade + go (Just "1") = Upgrade.V1.upgrade +#else + go (Just "0") = giveup "upgrade from v0 on Windows not supported" + go (Just "1") = giveup "upgrade from v1 on Windows not supported" +#endif + go (Just "2") = Upgrade.V2.upgrade + go (Just "3") = Upgrade.V3.upgrade automatic + go (Just "4") = Upgrade.V4.upgrade automatic + go (Just "5") = Upgrade.V5.upgrade automatic + go _ = return True diff --git a/Upgrade/V0.hs b/Upgrade/V0.hs new file mode 100644 index 0000000000..5ad2233ce9 --- /dev/null +++ b/Upgrade/V0.hs @@ -0,0 +1,49 @@ +{- git-annex v0 -> v1 upgrade support + - + - Copyright 2010 Joey Hess + - + - Licensed under the GNU GPL version 3 or higher. + -} + +module Upgrade.V0 where + +import Annex.Common +import Annex.Content +import qualified Upgrade.V1 + +upgrade :: Annex Bool +upgrade = do + showAction "v0 to v1" + + -- do the reorganisation of the key files + olddir <- fromRepo gitAnnexDir + keys <- getKeysPresent0 olddir + forM_ keys $ \k -> moveAnnex k $ olddir keyFile0 k + + -- update the symlinks to the key files + -- No longer needed here; V1.upgrade does the same thing + + -- Few people had v0 repos, so go the long way around from 0 -> 1 -> 2 + Upgrade.V1.upgrade + +-- these stayed unchanged between v0 and v1 +keyFile0 :: Key -> FilePath +keyFile0 = Upgrade.V1.keyFile1 +fileKey0 :: FilePath -> Key +fileKey0 = Upgrade.V1.fileKey1 +lookupFile0 :: FilePath -> Annex (Maybe (Key, Backend)) +lookupFile0 = Upgrade.V1.lookupFile1 + +getKeysPresent0 :: FilePath -> Annex [Key] +getKeysPresent0 dir = ifM (liftIO $ doesDirectoryExist dir) + ( liftIO $ map fileKey0 + <$> (filterM present =<< getDirectoryContents dir) + , return [] + ) + where + present d = do + result <- tryIO $ + getFileStatus $ dir ++ "/" ++ takeFileName d + case result of + Right s -> return $ isRegularFile s + Left _ -> return False diff --git a/Upgrade/V1.hs b/Upgrade/V1.hs new file mode 100644 index 0000000000..01741bbeb9 --- /dev/null +++ b/Upgrade/V1.hs @@ -0,0 +1,240 @@ +{- git-annex v1 -> v2 upgrade support + - + - Copyright 2011 Joey Hess + - + - Licensed under the GNU GPL version 3 or higher. + -} + +module Upgrade.V1 where + +import System.Posix.Types +import Data.Char +import Data.Default + +import Annex.Common +import Annex.Content +import Annex.Link +import Types.Key +import Logs.Presence +import qualified Annex.Queue +import qualified Git +import qualified Git.LsFiles as LsFiles +import Backend +import Utility.FileMode +import Utility.Tmp +import qualified Upgrade.V2 + +-- v2 adds hashing of filenames of content and location log files. +-- Key information is encoded in filenames differently, so +-- both content and location log files move around, and symlinks +-- to content need to be changed. +-- +-- When upgrading a v1 key to v2, file size metadata ought to be +-- added to the key (unless it is a WORM key, which encoded +-- mtime:size in v1). This can only be done when the file content +-- is present. Since upgrades need to happen consistently, +-- (so that two repos get changed the same way by the upgrade, and +-- will merge), that metadata cannot be added on upgrade. +-- +-- Note that file size metadata +-- will only be used for detecting situations where git-annex +-- would run out of disk space, so if some keys don't have it, +-- the impact is minor. At least initially. It could be used in the +-- future by smart auto-repo balancing code, etc. +-- +-- Anyway, since v2 plans ahead for other metadata being included +-- in keys, there should probably be a way to update a key. +-- Something similar to the migrate subcommand could be used, +-- and users could then run that at their leisure. + +upgrade :: Annex Bool +upgrade = do + showAction "v1 to v2" + + ifM (fromRepo Git.repoIsLocalBare) + ( moveContent + , do + moveContent + updateSymlinks + moveLocationLogs + + Annex.Queue.flush + ) + + Upgrade.V2.upgrade + +moveContent :: Annex () +moveContent = do + showAction "moving content" + files <- getKeyFilesPresent1 + forM_ files move + where + move f = do + let k = fileKey1 (takeFileName f) + let d = parentDir f + liftIO $ allowWrite d + liftIO $ allowWrite f + _ <- moveAnnex k f + liftIO $ removeDirectory d + +updateSymlinks :: Annex () +updateSymlinks = do + showAction "updating symlinks" + top <- fromRepo Git.repoPath + (files, cleanup) <- inRepo $ LsFiles.inRepo [top] + forM_ files fixlink + void $ liftIO cleanup + where + fixlink f = do + r <- lookupFile1 f + case r of + Nothing -> noop + Just (k, _) -> do + link <- calcRepo $ gitAnnexLink f k + liftIO $ removeFile f + liftIO $ createSymbolicLink link f + Annex.Queue.addCommand "add" [Param "--"] [f] + +moveLocationLogs :: Annex () +moveLocationLogs = do + showAction "moving location logs" + logkeys <- oldlocationlogs + forM_ logkeys move + where + oldlocationlogs = do + dir <- fromRepo Upgrade.V2.gitStateDir + ifM (liftIO $ doesDirectoryExist dir) + ( mapMaybe oldlog2key + <$> liftIO (getDirectoryContents dir) + , return [] + ) + move (l, k) = do + dest <- fromRepo $ logFile2 k + dir <- fromRepo Upgrade.V2.gitStateDir + let f = dir l + liftIO $ createDirectoryIfMissing True (parentDir dest) + -- could just git mv, but this way deals with + -- log files that are not checked into git, + -- as well as merging with already upgraded + -- logs that have been pulled from elsewhere + old <- liftIO $ readLog1 f + new <- liftIO $ readLog1 dest + liftIO $ writeLog1 dest (old++new) + Annex.Queue.addCommand "add" [Param "--"] [dest] + Annex.Queue.addCommand "add" [Param "--"] [f] + Annex.Queue.addCommand "rm" [Param "--quiet", Param "-f", Param "--"] [f] + +oldlog2key :: FilePath -> Maybe (FilePath, Key) +oldlog2key l + | drop len l == ".log" && sane = Just (l, k) + | otherwise = Nothing + where + len = length l - 4 + k = readKey1 (take len l) + sane = (not . null $ keyName k) && (not . null $ formatKeyVariety $ keyVariety k) + +-- WORM backend keys: "WORM:mtime:size:filename" +-- all the rest: "backend:key" +-- +-- If the file looks like "WORM:XXX-...", then it was created by mixing +-- v2 and v1; that infelicity is worked around by treating the value +-- as the v2 key that it is. +readKey1 :: String -> Key +readKey1 v + | mixup = fromJust $ file2key $ intercalate ":" $ Prelude.tail bits + | otherwise = stubKey + { keyName = n + , keyVariety = parseKeyVariety b + , keySize = s + , keyMtime = t + } + where + bits = splitc ':' v + b = Prelude.head bits + n = intercalate ":" $ drop (if wormy then 3 else 1) bits + t = if wormy + then Just (Prelude.read (bits !! 1) :: EpochTime) + else Nothing + s = if wormy + then Just (Prelude.read (bits !! 2) :: Integer) + else Nothing + wormy = Prelude.head bits == "WORM" + mixup = wormy && isUpper (Prelude.head $ bits !! 1) + +showKey1 :: Key -> String +showKey1 Key { keyName = n , keyVariety = v, keySize = s, keyMtime = t } = + intercalate ":" $ filter (not . null) [b, showifhere t, showifhere s, n] + where + showifhere Nothing = "" + showifhere (Just x) = show x + b = formatKeyVariety v + +keyFile1 :: Key -> FilePath +keyFile1 key = replace "/" "%" $ replace "%" "&s" $ replace "&" "&a" $ showKey1 key + +fileKey1 :: FilePath -> Key +fileKey1 file = readKey1 $ + replace "&a" "&" $ replace "&s" "%" $ replace "%" "/" file + +writeLog1 :: FilePath -> [LogLine] -> IO () +writeLog1 file ls = viaTmp writeFile file (showLog ls) + +readLog1 :: FilePath -> IO [LogLine] +readLog1 file = catchDefaultIO [] $ + parseLog <$> readFileStrict file + +lookupFile1 :: FilePath -> Annex (Maybe (Key, Backend)) +lookupFile1 file = do + tl <- liftIO $ tryIO getsymlink + case tl of + Left _ -> return Nothing + Right l -> makekey l + where + getsymlink = takeFileName <$> readSymbolicLink file + makekey l = case maybeLookupBackendVariety (keyVariety k) of + Nothing -> do + unless (null kname || null bname || + not (isLinkToAnnex l)) $ + warning skip + return Nothing + Just backend -> return $ Just (k, backend) + where + k = fileKey1 l + bname = formatKeyVariety (keyVariety k) + kname = keyName k + skip = "skipping " ++ file ++ + " (unknown backend " ++ bname ++ ")" + +getKeyFilesPresent1 :: Annex [FilePath] +getKeyFilesPresent1 = getKeyFilesPresent1' =<< fromRepo gitAnnexObjectDir +getKeyFilesPresent1' :: FilePath -> Annex [FilePath] +getKeyFilesPresent1' dir = + ifM (liftIO $ doesDirectoryExist dir) + ( do + dirs <- liftIO $ getDirectoryContents dir + let files = map (\d -> dir ++ "/" ++ d ++ "/" ++ takeFileName d) dirs + liftIO $ filterM present files + , return [] + ) + where + present f = do + result <- tryIO $ getFileStatus f + case result of + Right s -> return $ isRegularFile s + Left _ -> return False + +logFile1 :: Git.Repo -> Key -> String +logFile1 repo key = Upgrade.V2.gitStateDir repo ++ keyFile1 key ++ ".log" + +logFile2 :: Key -> Git.Repo -> String +logFile2 = logFile' (hashDirLower def) + +logFile' :: (Key -> FilePath) -> Key -> Git.Repo -> String +logFile' hasher key repo = + gitStateDir repo ++ hasher key ++ keyFile key ++ ".log" + +stateDir :: FilePath +stateDir = addTrailingPathSeparator ".git-annex" + +gitStateDir :: Git.Repo -> FilePath +gitStateDir repo = addTrailingPathSeparator $ Git.repoPath repo stateDir diff --git a/Upgrade/V2.hs b/Upgrade/V2.hs new file mode 100644 index 0000000000..cf534f7ed3 --- /dev/null +++ b/Upgrade/V2.hs @@ -0,0 +1,143 @@ +{- git-annex v2 -> v3 upgrade support + - + - Copyright 2011 Joey Hess + - + - Licensed under the GNU GPL version 3 or higher. + -} + +module Upgrade.V2 where + +import Annex.Common +import qualified Git +import qualified Git.Command +import qualified Git.Ref +import qualified Annex.Branch +import qualified Annex +import Annex.Content +import Utility.Tmp +import Logs +import Messages.Progress + +olddir :: Git.Repo -> FilePath +olddir g + | Git.repoIsLocalBare g = "" + | otherwise = ".git-annex" + +{- .git-annex/ moved to a git-annex branch. + - + - Strategy: + - + - * Create the git-annex branch. + - * Find each location log file in .git-annex/, and inject its content + - into the git-annex branch, unioning with any content already in + - there. (in passing, this deals with the semi transition that left + - some location logs hashed two different ways; both are found and + - merged). + - * Also inject remote.log, trust.log, and uuid.log. + - * git rm -rf .git-annex + - * Remove stuff that used to be needed in .gitattributes. + - * Commit changes. + -} +upgrade :: Annex Bool +upgrade = do + showAction "v2 to v3" + bare <- fromRepo Git.repoIsLocalBare + old <- fromRepo olddir + + Annex.Branch.create + showProgressDots + + e <- liftIO $ doesDirectoryExist old + when e $ do + config <- Annex.getGitConfig + mapM_ (\(k, f) -> inject f $ locationLogFile config k) =<< locationLogs + mapM_ (\f -> inject f f) =<< logFiles old + + saveState False + showProgressDots + + when e $ do + inRepo $ Git.Command.run [Param "rm", Param "-r", Param "-f", Param "-q", File old] + unless bare $ inRepo gitAttributesUnWrite + showProgressDots + + unless bare push + + return True + +locationLogs :: Annex [(Key, FilePath)] +locationLogs = do + dir <- fromRepo gitStateDir + liftIO $ do + levela <- dirContents dir + levelb <- mapM tryDirContents levela + files <- mapM tryDirContents (concat levelb) + return $ mapMaybe islogfile (concat files) + where + tryDirContents d = catchDefaultIO [] $ dirContents d + islogfile f = maybe Nothing (\k -> Just (k, f)) $ + locationLogFileKey f + +inject :: FilePath -> FilePath -> Annex () +inject source dest = do + old <- fromRepo olddir + new <- liftIO (readFile $ old source) + Annex.Branch.change dest $ \prev -> + unlines $ nub $ lines prev ++ lines new + +logFiles :: FilePath -> Annex [FilePath] +logFiles dir = return . filter (".log" `isSuffixOf`) + <=< liftIO $ getDirectoryContents dir + +push :: Annex () +push = do + origin_master <- inRepo $ Git.Ref.exists $ Git.Ref "origin/master" + origin_gitannex <- Annex.Branch.hasOrigin + case (origin_master, origin_gitannex) of + (_, True) -> do + -- Merge in the origin's git-annex branch, + -- so that pushing the git-annex branch + -- will immediately work. Not pushed here, + -- because it's less obnoxious to let the user + -- push. + Annex.Branch.update + (True, False) -> do + -- push git-annex to origin, so that + -- "git push" will from then on + -- automatically push it + Annex.Branch.update -- just in case + showAction "pushing new git-annex branch to origin" + showOutput + inRepo $ Git.Command.run + [ Param "push" + , Param "origin" + , Param $ Git.fromRef Annex.Branch.name + ] + _ -> do + -- no origin exists, so just let the user + -- know about the new branch + Annex.Branch.update + showLongNote $ + "git-annex branch created\n" ++ + "Be sure to push this branch when pushing to remotes.\n" + +{- Old .gitattributes contents, not needed anymore. -} +attrLines :: [String] +attrLines = + [ stateDir "*.log merge=union" + , stateDir "*/*/*.log merge=union" + ] + +gitAttributesUnWrite :: Git.Repo -> IO () +gitAttributesUnWrite repo = do + let attributes = Git.attributes repo + whenM (doesFileExist attributes) $ do + c <- readFileStrict attributes + liftIO $ viaTmp writeFile attributes $ unlines $ + filter (`notElem` attrLines) $ lines c + Git.Command.run [Param "add", File attributes] repo + +stateDir :: FilePath +stateDir = addTrailingPathSeparator ".git-annex" +gitStateDir :: Git.Repo -> FilePath +gitStateDir repo = addTrailingPathSeparator $ Git.repoPath repo stateDir diff --git a/Upgrade/V3.hs b/Upgrade/V3.hs new file mode 100644 index 0000000000..8d873a6cec --- /dev/null +++ b/Upgrade/V3.hs @@ -0,0 +1,12 @@ +{- git-annex v3 -> v4 uppgrade support + - + - There was no explicit v3 to v4 upgrade, so run v5 upgrade code. + - + - Copyright 2013 Joey Hess + - + - Licensed under the GNU GPL version 3 or higher. + -} + +module Upgrade.V3 (upgrade) where + +import Upgrade.V4 (upgrade) diff --git a/Upgrade/V4.hs b/Upgrade/V4.hs new file mode 100644 index 0000000000..e317c39498 --- /dev/null +++ b/Upgrade/V4.hs @@ -0,0 +1,23 @@ +{- git-annex v4 -> v5 uppgrade support + - + - Copyright 2013 Joey Hess + - + - Licensed under the GNU GPL version 3 or higher. + -} + +module Upgrade.V4 where + +import Annex.Common +import Config +import Annex.Direct + +{- Direct mode only upgrade. v4 to v5 indirect update is a no-op -} +upgrade :: Bool -> Annex Bool +upgrade automatic = ifM isDirect + ( do + unless automatic $ + showAction "v4 to v5" + setDirect True + return True + , return True + ) diff --git a/Upgrade/V5.hs b/Upgrade/V5.hs new file mode 100644 index 0000000000..7ce8bbe01c --- /dev/null +++ b/Upgrade/V5.hs @@ -0,0 +1,128 @@ +{- git-annex v5 -> v6 upgrade support + - + - Copyright 2015-2016 Joey Hess + - + - Licensed under the GNU GPL version 3 or higher. + -} + +module Upgrade.V5 where + +import Annex.Common +import qualified Annex +import Config +import Config.Smudge +import Annex.InodeSentinal +import Annex.Link +import Annex.Direct +import Annex.Content +import Annex.CatFile +import Annex.WorkTree +import qualified Database.Keys +import qualified Annex.Content.Direct as Direct +import qualified Git +import qualified Git.LsFiles +import qualified Git.Branch +import Git.FilePath +import Git.FileMode +import Git.Config +import Git.Ref +import Utility.InodeCache +import Annex.AdjustedBranch + +upgrade :: Bool -> Annex Bool +upgrade automatic = do + unless automatic $ + showAction "v5 to v6" + scanUnlockedFiles + whenM isDirect $ do + {- Direct mode makes the same tradeoff of using less disk + - space, with less preservation of old versions of files + - as does annex.thin. -} + setConfig (annexConfig "thin") (boolConfig True) + Annex.changeGitConfig $ \c -> c { annexThin = True } + {- Since upgrade from direct mode changes how files + - are represented in git, by checking out an adjusted + - branch, commit any changes in the work tree first. -} + whenM stageDirect $ do + unless automatic $ + showAction "committing first" + upgradeDirectCommit automatic + "commit before upgrade to annex.version 6" + setDirect False + cur <- fromMaybe (error "Somehow no branch is checked out") + <$> inRepo Git.Branch.current + upgradeDirectWorkTree + removeDirectCruft + {- Create adjusted branch where all files are unlocked. + - This should have the same content for each file as + - have been staged in upgradeDirectWorkTree. -} + AdjBranch b <- adjustBranch UnlockAdjustment cur + {- Since the work tree was already set up by + - upgradeDirectWorkTree, and contains unlocked file + - contents too, don't use git checkout to check out the + - adjust branch. Instead, update HEAD manually. -} + inRepo $ setHeadRef b + configureSmudgeFilter + -- Inode sentinal file was only used in direct mode and when + -- locking down files as they were added. In v6, it's used more + -- extensively, so make sure it exists, since old repos that didn't + -- use direct mode may not have created it. + unlessM (isDirect) $ + createInodeSentinalFile True + return True + +upgradeDirectCommit :: Bool -> String -> Annex () +upgradeDirectCommit automatic msg = + void $ inRepo $ Git.Branch.commitCommand commitmode + [ Param "-m" + , Param msg + ] + where + commitmode = if automatic then Git.Branch.AutomaticCommit else Git.Branch.ManualCommit + +{- Walk work tree from top and convert all annex symlinks to pointer files, + - staging them in the index, and updating the work tree files with + - either the content of the object, or the pointer file content. -} +upgradeDirectWorkTree :: Annex () +upgradeDirectWorkTree = do + top <- fromRepo Git.repoPath + (l, clean) <- inRepo $ Git.LsFiles.stagedDetails [top] + forM_ l go + void $ liftIO clean + where + go (f, Just _sha, Just mode) | isSymLink mode = do + -- Cannot use lookupFile here, as we're in between direct + -- mode and v6. + mk <- catKeyFile f + case mk of + Nothing -> noop + Just k -> do + ifM (isJust <$> getAnnexLinkTarget f) + ( writepointer f k + , fromdirect f k + ) + stagePointerFile f Nothing =<< hashPointerFile k + Database.Keys.addAssociatedFile k + =<< inRepo (toTopFilePath f) + return () + go _ = noop + + fromdirect f k = do + -- If linkToAnnex fails for some reason, the work tree file + -- still has the content; the annex object file is just + -- not populated with it. Since the work tree file + -- is recorded as an associated file, things will still + -- work that way, it's just not ideal. + ic <- withTSDelta (liftIO . genInodeCache f) + void $ linkToAnnex k f ic + writepointer f k = liftIO $ do + nukeFile f + writeFile f (formatPointer k) + +{- Remove all direct mode bookkeeping files. -} +removeDirectCruft :: Annex () +removeDirectCruft = mapM_ go =<< getKeysPresent InAnywhere + where + go k = do + Direct.removeInodeCache k + Direct.removeAssociatedFiles k diff --git a/Utility/Aeson.hs b/Utility/Aeson.hs new file mode 100644 index 0000000000..7147e516bb --- /dev/null +++ b/Utility/Aeson.hs @@ -0,0 +1,86 @@ +{- GHC File system encoding support for Aeson. + - + - Import instead of Data.Aeson + - + - Copyright 2018 Joey Hess + - + - License: BSD-2-clause + -} + +{-# LANGUAGE FlexibleInstances, TypeSynonymInstances #-} + +module Utility.Aeson ( + module X, + ToJSON'(..), + encode, + packString, +) where + +import Data.Aeson as X hiding (ToJSON, toJSON, encode) +import Data.Aeson hiding (encode) +import qualified Data.Aeson +import qualified Data.Text as T +import qualified Data.Text.Encoding as T +import qualified Data.ByteString.Lazy as L +import qualified Data.ByteString as S +import qualified Data.Set +import qualified Data.Vector +import Prelude + +import Utility.FileSystemEncoding + +-- | Use this instead of Data.Aeson.encode to make sure that the +-- below String instance is used. +encode :: ToJSON' a => a -> L.ByteString +encode = Data.Aeson.encode . toJSON' + +-- | Aeson has an unfortunate ToJSON instance for Char and [Char] +-- which does not support Strings containing UTF8 characters +-- encoded using the filesystem encoding when run in a non-utf8 locale. +-- +-- Since we can't replace that with a instance that does the right +-- thing, instead here's a new class that handles String right. +class ToJSON' a where + toJSON' :: a -> Value + +instance ToJSON' String where + toJSON' = toJSON . packString + +-- | Pack a String to Text, correctly handling the filesystem encoding. +-- +-- Use this instead of Data.Text.pack. +-- +-- Note that if the string contains invalid UTF8 characters not using +-- the FileSystemEncoding, this is the same as Data.Text.pack. +packString :: String -> T.Text +packString s = case T.decodeUtf8' (S.concat $ L.toChunks $ encodeBS s) of + Right t -> t + Left _ -> T.pack s + +-- | An instance for lists cannot be included as it would overlap with +-- the String instance. Instead, you can use a Vector. +instance ToJSON' s => ToJSON' (Data.Vector.Vector s) where + toJSON' = toJSON . map toJSON' . Data.Vector.toList + +-- Aeson generates the same JSON for a Set as for a list. +instance ToJSON' s => ToJSON' (Data.Set.Set s) where + toJSON' = toJSON . map toJSON' . Data.Set.toList + +instance (ToJSON' a, ToJSON a) => ToJSON' (Maybe a) where + toJSON' (Just a) = toJSON (Just (toJSON' a)) + toJSON' v@Nothing = toJSON v + +instance (ToJSON' a, ToJSON a, ToJSON' b, ToJSON b) => ToJSON' (a, b) where + toJSON' (a, b) = toJSON ((toJSON' a, toJSON' b)) + +instance ToJSON' Bool where + toJSON' = toJSON + +instance ToJSON' Integer where + toJSON' = toJSON + +instance ToJSON' Object where + toJSON' = toJSON + +instance ToJSON' Value where + toJSON' = toJSON diff --git a/Utility/Android.hs b/Utility/Android.hs new file mode 100644 index 0000000000..f30e415d77 --- /dev/null +++ b/Utility/Android.hs @@ -0,0 +1,26 @@ +{-# LANGUAGE CPP #-} + +{- Android stuff + - + - Copyright 2018 Joey Hess + - + - License: BSD-2-clause + -} + +module Utility.Android where + +#ifdef linux_HOST_OS +import Common +#endif + +-- Detect when the Linux build is running on Android, eg in termux. +-- +-- Note that this relies on termux's uname having been built with "Android" +-- as the os name. Often on Android, uname will report "Linux". +osAndroid :: IO Bool +#ifdef linux_HOST_OS +osAndroid = catchDefaultIO False $ + ("Android" `isPrefixOf` ) <$> readProcess "uname" ["-o"] +#else +osAndroid = return False +#endif diff --git a/Utility/Applicative.hs b/Utility/Applicative.hs new file mode 100644 index 0000000000..fce3c04852 --- /dev/null +++ b/Utility/Applicative.hs @@ -0,0 +1,16 @@ +{- applicative stuff + - + - Copyright 2012 Joey Hess + - + - License: BSD-2-clause + -} + +module Utility.Applicative where + +{- Like <$> , but supports one level of currying. + - + - foo v = bar <$> action v == foo = bar <$$> action + -} +(<$$>) :: Functor f => (a -> b) -> (c -> f a) -> c -> f b +f <$$> v = fmap f . v +infixr 4 <$$> diff --git a/Utility/AuthToken.hs b/Utility/AuthToken.hs new file mode 100644 index 0000000000..191b4f5c92 --- /dev/null +++ b/Utility/AuthToken.hs @@ -0,0 +1,99 @@ +{- authentication tokens + - + - Copyright 2016 Joey Hess + - + - License: BSD-2-clause + -} + +module Utility.AuthToken ( + AuthToken, + toAuthToken, + fromAuthToken, + nullAuthToken, + genAuthToken, + AllowedAuthTokens, + allowedAuthTokens, + isAllowedAuthToken, +) where + +import qualified Utility.SimpleProtocol as Proto +import Utility.Hash + +import Data.SecureMem +import Data.Maybe +import Data.Char +import Data.Byteable +import qualified Data.Text as T +import qualified Data.Text.Encoding as TE +import qualified Data.ByteString.Lazy as L +import "crypto-api" Crypto.Random + +-- | An AuthToken is stored in secue memory, with constant time comparison. +-- +-- It can have varying length, depending on the security needs of the +-- application. +-- +-- To avoid decoding issues, and presentation issues, the content +-- of an AuthToken is limited to ASCII characters a-z, and 0-9. +-- This is enforced by all exported AuthToken constructors. +newtype AuthToken = AuthToken SecureMem + deriving (Show, Eq) + +allowedChar :: Char -> Bool +allowedChar c = isAsciiUpper c || isAsciiLower c || isDigit c + +instance Proto.Serializable AuthToken where + serialize = T.unpack . fromAuthToken + deserialize = toAuthToken . T.pack + +fromAuthToken :: AuthToken -> T.Text +fromAuthToken (AuthToken t ) = TE.decodeLatin1 (toBytes t) + +-- | Upper-case characters are lower-cased to make them fit in the allowed +-- character set. This allows AuthTokens to be compared effectively +-- case-insensitively. +-- +-- Returns Nothing if any disallowed characters are present. +toAuthToken :: T.Text -> Maybe AuthToken +toAuthToken t + | all allowedChar s = Just $ AuthToken $ + secureMemFromByteString $ TE.encodeUtf8 $ T.pack s + | otherwise = Nothing + where + s = map toLower $ T.unpack t + +-- | The empty AuthToken, for those times when you don't want any security. +nullAuthToken :: AuthToken +nullAuthToken = AuthToken $ secureMemFromByteString $ TE.encodeUtf8 T.empty + +-- | Generates an AuthToken of a specified length. This is done by +-- generating a random bytestring, hashing it with sha2 512, and truncating +-- to the specified length. +-- +-- That limits the maximum length to 128, but with 512 bytes of entropy, +-- that should be sufficient for any application. +genAuthToken :: Int -> IO AuthToken +genAuthToken len = do + g <- newGenIO :: IO SystemRandom + return $ + case genBytes 512 g of + Left e -> error $ "failed to generate auth token: " ++ show e + Right (s, _) -> fromMaybe (error "auth token encoding failed") $ + toAuthToken $ T.pack $ take len $ + show $ sha2_512 $ L.fromChunks [s] + +-- | For when several AuthTokens are allowed to be used. +newtype AllowedAuthTokens = AllowedAuthTokens [AuthToken] + +allowedAuthTokens :: [AuthToken] -> AllowedAuthTokens +allowedAuthTokens = AllowedAuthTokens + +-- | Note that every item in the list is checked, even if the first one +-- is allowed, so that comparison is constant-time. +isAllowedAuthToken :: AuthToken -> AllowedAuthTokens -> Bool +isAllowedAuthToken t (AllowedAuthTokens l) = go False l + where + go ok [] = ok + go ok (i:is) + | t == i = go True is + | otherwise = go ok is diff --git a/Utility/Base64.hs b/Utility/Base64.hs new file mode 100644 index 0000000000..10ec9e0305 --- /dev/null +++ b/Utility/Base64.hs @@ -0,0 +1,39 @@ +{- Simple Base64 encoding of Strings + - + - Note that this uses the FileSystemEncoding, so it can be used on Strings + - that repesent filepaths containing arbitrarily encoded characters. + - + - Copyright 2011, 2015 Joey Hess + - + - License: BSD-2-clause + -} + +module Utility.Base64 (toB64, fromB64Maybe, fromB64, prop_b64_roundtrips) where + +import Utility.FileSystemEncoding + +import qualified "sandi" Codec.Binary.Base64 as B64 +import Data.Maybe +import qualified Data.ByteString.Lazy as L +import Data.ByteString.UTF8 (fromString, toString) +import Data.Char + +toB64 :: String -> String +toB64 = toString . B64.encode . L.toStrict . encodeBS + +fromB64Maybe :: String -> Maybe String +fromB64Maybe s = either (const Nothing) (Just . decodeBS . L.fromStrict) + (B64.decode $ fromString s) + +fromB64 :: String -> String +fromB64 = fromMaybe bad . fromB64Maybe + where + bad = error "bad base64 encoded data" + +-- Only ascii strings are tested, because an arbitrary string may contain +-- characters not encoded using the FileSystemEncoding, which would thus +-- not roundtrip, as fromB64 always generates an output encoded that way. +prop_b64_roundtrips :: String -> Bool +prop_b64_roundtrips s + | all (isAscii) s = s == fromB64 (toB64 s) + | otherwise = True diff --git a/Utility/Batch.hs b/Utility/Batch.hs new file mode 100644 index 0000000000..d96f9d3f37 --- /dev/null +++ b/Utility/Batch.hs @@ -0,0 +1,96 @@ +{- Running a long or expensive batch operation niced. + - + - Copyright 2013 Joey Hess + - + - License: BSD-2-clause + -} + +{-# LANGUAGE CPP #-} + +module Utility.Batch where + +import Common + +#if defined(linux_HOST_OS) || defined(__ANDROID__) +import Control.Concurrent.Async +import System.Posix.Process +#endif +import qualified Control.Exception as E + +{- Runs an operation, at batch priority. + - + - This is done by running it in a bound thread, which on Linux can be set + - to have a different nice level than the rest of the program. Note that + - due to running in a bound thread, some operations may be more expensive + - to perform. Also note that if the action calls forkIO or forkOS itself, + - that will make a new thread that does not have the batch priority. + - + - POSIX threads do not support separate nice levels, so on other operating + - systems, the action is simply ran. + -} +batch :: IO a -> IO a +#if defined(linux_HOST_OS) || defined(__ANDROID__) +batch a = wait =<< batchthread + where + batchthread = asyncBound $ do + setProcessPriority 0 maxNice + a +#else +batch a = a +#endif + +maxNice :: Int +maxNice = 19 + +{- Makes a command be run by whichever of nice, ionice, and nocache + - are available in the path. -} +type BatchCommandMaker = (String, [CommandParam]) -> (String, [CommandParam]) + +getBatchCommandMaker :: IO BatchCommandMaker +getBatchCommandMaker = do +#ifndef mingw32_HOST_OS + nicers <- filterM (inPath . fst) + [ ("nice", []) +#ifndef __ANDROID__ + -- Android's ionice does not allow specifying a command, + -- so don't use it. + , ("ionice", ["-c3"]) +#endif + , ("nocache", []) + ] + return $ \(command, params) -> + case nicers of + [] -> (command, params) + (first:rest) -> (fst first, map Param (snd first ++ concatMap (\p -> fst p : snd p) rest ++ [command]) ++ params) +#else + return id +#endif + +toBatchCommand :: (String, [CommandParam]) -> IO (String, [CommandParam]) +toBatchCommand v = do + batchmaker <- getBatchCommandMaker + return $ batchmaker v + +{- Runs a command in a way that's suitable for batch jobs that can be + - interrupted. + - + - If the calling thread receives an async exception, it sends the + - command a SIGTERM, and after the command finishes shuttting down, + - it re-raises the async exception. -} +batchCommand :: String -> [CommandParam] -> IO Bool +batchCommand command params = batchCommandEnv command params Nothing + +batchCommandEnv :: String -> [CommandParam] -> Maybe [(String, String)] -> IO Bool +batchCommandEnv command params environ = do + batchmaker <- getBatchCommandMaker + let (command', params') = batchmaker (command, params) + let p = proc command' $ toCommand params' + (_, _, _, pid) <- createProcess $ p { env = environ } + r <- E.try (waitForProcess pid) :: IO (Either E.SomeException ExitCode) + case r of + Right ExitSuccess -> return True + Right _ -> return False + Left asyncexception -> do + terminateProcess pid + void $ waitForProcess pid + E.throwIO asyncexception diff --git a/Utility/Bloom.hs b/Utility/Bloom.hs new file mode 100644 index 0000000000..67841225c1 --- /dev/null +++ b/Utility/Bloom.hs @@ -0,0 +1,67 @@ +{- bloomfilter compatability wrapper + - + - Copyright 2014 Joey Hess + - + - License: BSD-2-clause + -} + +{-# LANGUAGE CPP #-} + +module Utility.Bloom ( + Bloom, + safeSuggestSizing, + Hashable(..), + cheapHashes, + elemB, + notElemB, + + newMB, + insertMB, + unsafeFreezeMB, +) where + +#if MIN_VERSION_bloomfilter(2,0,0) +import qualified Data.BloomFilter.Mutable as MBloom +import qualified Data.BloomFilter as Bloom +#else +import qualified Data.BloomFilter as Bloom +#endif +import Data.BloomFilter.Easy (safeSuggestSizing, Bloom) +import Data.BloomFilter.Hash (Hashable(..), cheapHashes) +import Control.Monad.ST (ST) + +#if MIN_VERSION_bloomfilter(2,0,0) + +notElemB :: a -> Bloom a -> Bool +notElemB = Bloom.notElem + +elemB :: a -> Bloom a -> Bool +elemB = Bloom.elem + +newMB :: (a -> [Bloom.Hash]) -> Int -> ST s (MBloom.MBloom s a) +newMB = MBloom.new + +insertMB :: MBloom.MBloom s a -> a -> ST s () +insertMB = MBloom.insert + +unsafeFreezeMB :: MBloom.MBloom s a -> ST s (Bloom a) +unsafeFreezeMB = Bloom.unsafeFreeze + +#else + +notElemB :: a -> Bloom a -> Bool +notElemB = Bloom.notElemB + +elemB :: a -> Bloom a -> Bool +elemB = Bloom.elemB + +newMB :: (a -> [Bloom.Hash]) -> Int -> ST s (Bloom.MBloom s a) +newMB = Bloom.newMB + +insertMB :: Bloom.MBloom s a -> a -> ST s () +insertMB = Bloom.insertMB + +unsafeFreezeMB :: Bloom.MBloom s a -> ST s (Bloom a) +unsafeFreezeMB = Bloom.unsafeFreezeMB + +#endif diff --git a/Utility/CoProcess.hs b/Utility/CoProcess.hs new file mode 100644 index 0000000000..2bae40fbae --- /dev/null +++ b/Utility/CoProcess.hs @@ -0,0 +1,88 @@ +{- Interface for running a shell command as a coprocess, + - sending it queries and getting back results. + - + - Copyright 2012-2013 Joey Hess + - + - License: BSD-2-clause + -} + +{-# LANGUAGE CPP #-} + +module Utility.CoProcess ( + CoProcessHandle, + start, + stop, + query, +) where + +import Common + +import Control.Concurrent.MVar + +type CoProcessHandle = MVar CoProcessState + +data CoProcessState = CoProcessState + { coProcessPid :: ProcessHandle + , coProcessTo :: Handle + , coProcessFrom :: Handle + , coProcessSpec :: CoProcessSpec + } + +data CoProcessSpec = CoProcessSpec + { coProcessNumRestarts :: Int + , coProcessCmd :: FilePath + , coProcessParams :: [String] + , coProcessEnv :: Maybe [(String, String)] + } + +start :: Int -> FilePath -> [String] -> Maybe [(String, String)] -> IO CoProcessHandle +start numrestarts cmd params environ = do + s <- start' $ CoProcessSpec numrestarts cmd params environ + newMVar s + +start' :: CoProcessSpec -> IO CoProcessState +start' s = do + (pid, from, to) <- startInteractiveProcess (coProcessCmd s) (coProcessParams s) (coProcessEnv s) + rawMode from + rawMode to + return $ CoProcessState pid to from s + where +#ifdef mingw32_HOST_OS + rawMode h = hSetNewlineMode h noNewlineTranslation +#else + rawMode _ = return () +#endif + +stop :: CoProcessHandle -> IO () +stop ch = do + s <- readMVar ch + hClose $ coProcessTo s + hClose $ coProcessFrom s + let p = proc (coProcessCmd $ coProcessSpec s) (coProcessParams $ coProcessSpec s) + forceSuccessProcess p (coProcessPid s) + +{- To handle a restartable process, any IO exception thrown by the send and + - receive actions are assumed to mean communication with the process + - failed, and the failed action is re-run with a new process. -} +query :: CoProcessHandle -> (Handle -> IO a) -> (Handle -> IO b) -> IO b +query ch send receive = do + s <- readMVar ch + restartable s (send $ coProcessTo s) $ const $ + restartable s (hFlush $ coProcessTo s) $ const $ + restartable s (receive $ coProcessFrom s) + return + where + restartable s a cont + | coProcessNumRestarts (coProcessSpec s) > 0 = + maybe restart cont =<< catchMaybeIO a + | otherwise = cont =<< a + restart = do + s <- takeMVar ch + void $ catchMaybeIO $ do + hClose $ coProcessTo s + hClose $ coProcessFrom s + void $ waitForProcess $ coProcessPid s + s' <- start' $ (coProcessSpec s) + { coProcessNumRestarts = coProcessNumRestarts (coProcessSpec s) - 1 } + putMVar ch s' + query ch send receive diff --git a/Utility/CopyFile.hs b/Utility/CopyFile.hs new file mode 100644 index 0000000000..4ad85b7db1 --- /dev/null +++ b/Utility/CopyFile.hs @@ -0,0 +1,57 @@ +{- file copying + - + - Copyright 2010-2014 Joey Hess + - + - License: BSD-2-clause + -} + +{-# LANGUAGE CPP #-} + +module Utility.CopyFile ( + copyFileExternal, + createLinkOrCopy, + CopyMetaData(..) +) where + +import Common +import qualified BuildInfo + +data CopyMetaData + -- Copy timestamps when possible, but no other metadata, and + -- when copying a symlink, makes a copy of its content. + = CopyTimeStamps + -- Copy all metadata when possible. + | CopyAllMetaData + deriving (Eq) + +{- The cp command is used, because I hate reinventing the wheel, + - and because this allows easy access to features like cp --reflink. -} +copyFileExternal :: CopyMetaData -> FilePath -> FilePath -> IO Bool +copyFileExternal meta src dest = do + whenM (doesFileExist dest) $ + removeFile dest + boolSystem "cp" $ params ++ [File src, File dest] + where +#ifndef __ANDROID__ + params = map snd $ filter fst + [ (BuildInfo.cp_reflink_auto, Param "--reflink=auto") + , (allmeta && BuildInfo.cp_a, Param "-a") + , (allmeta && BuildInfo.cp_p && not BuildInfo.cp_a + , Param "-p") + , (not allmeta && BuildInfo.cp_preserve_timestamps + , Param "--preserve=timestamps") + ] +#else + params = if allmeta then [] else [] +#endif + allmeta = meta == CopyAllMetaData + +{- Create a hard link if the filesystem allows it, and fall back to copying + - the file. -} +createLinkOrCopy :: FilePath -> FilePath -> IO Bool +createLinkOrCopy src dest = go `catchIO` const fallback + where + go = do + createLink src dest + return True + fallback = copyFileExternal CopyAllMetaData src dest diff --git a/Utility/DBus.hs b/Utility/DBus.hs new file mode 100644 index 0000000000..5b04703013 --- /dev/null +++ b/Utility/DBus.hs @@ -0,0 +1,85 @@ +{- DBus utilities + - + - Copyright 2012 Joey Hess + - + - License: BSD-2-clause + -} + +{-# LANGUAGE OverloadedStrings, ScopedTypeVariables #-} + +module Utility.DBus where + +import Utility.PartialPrelude +import Utility.Exception + +import DBus.Client +import DBus +import Data.Maybe +import Control.Concurrent +import Control.Exception as E + +type ServiceName = String + +listServiceNames :: Client -> IO [ServiceName] +listServiceNames client = do + reply <- callDBus client "ListNames" [] + return $ fromMaybe [] $ fromVariant =<< headMaybe (methodReturnBody reply) + +callDBus :: Client -> MemberName -> [Variant] -> IO MethodReturn +callDBus client name params = call_ client $ + (methodCall "/org/freedesktop/DBus" "org.freedesktop.DBus" name) + { methodCallDestination = Just "org.freedesktop.DBus" + , methodCallBody = params + } + +{- Connects to the bus, and runs the client action. + - + - Throws a ClientError, and closes the connection if it fails to + - process an incoming message, or if the connection is lost. + - Unlike DBus's usual interface, this error is thrown at the top level, + - rather than inside the clientThreadRunner, so it can be caught, and + - runClient re-run as needed. -} +runClient :: IO (Maybe Address) -> (Client -> IO ()) -> IO () +runClient getaddr clientaction = do + env <- getaddr + case env of + Nothing -> throwIO (clientError "runClient: unable to determine DBUS address") + Just addr -> do + {- The clientaction will set up listeners, which + - run in a different thread. We block while + - they're running, until our threadrunner catches + - a ClientError, which it will put into the MVar + - to be rethrown here. -} + mv <- newEmptyMVar + let tr = threadrunner (putMVar mv) + let opts = defaultClientOptions { clientThreadRunner = tr } + client <- connectWith opts addr + clientaction client + e <- takeMVar mv + disconnect client + throw e + where + threadrunner storeerr io = loop + where + loop = catchClientError (io >> loop) storeerr + +{- Connects to the bus, and runs the client action. + - + - If the connection is lost, runs onretry, which can do something like + - a delay, or printing a warning, and has a state value (useful for + - exponential backoff). Once onretry returns, the connection is retried. + -} +persistentClient :: IO (Maybe Address) -> v -> (SomeException -> v -> IO v) -> (Client -> IO ()) -> IO () +persistentClient getaddr v onretry clientaction = + {- runClient can fail with not just ClientError, but also other + - things, if dbus is not running. Let async exceptions through. -} + runClient getaddr clientaction `catchNonAsync` retry + where + retry e = do + v' <- onretry e v + persistentClient getaddr v' onretry clientaction + +{- Catches only ClientError -} +catchClientError :: IO () -> (ClientError -> IO ()) -> IO () +catchClientError io handler = + either handler return =<< (E.try io :: IO (Either ClientError ())) diff --git a/Utility/Daemon.hs b/Utility/Daemon.hs new file mode 100644 index 0000000000..398e6d9e7b --- /dev/null +++ b/Utility/Daemon.hs @@ -0,0 +1,183 @@ +{- daemon support + - + - Copyright 2012-2014 Joey Hess + - + - License: BSD-2-clause + -} + +{-# LANGUAGE CPP #-} + +module Utility.Daemon where + +import Common +import Utility.PID +#ifndef mingw32_HOST_OS +import Utility.LogFile +#else +import System.Win32.Process (terminateProcessById) +import Utility.LockFile +#endif + +#ifndef mingw32_HOST_OS +import System.Posix +import Control.Concurrent.Async +#endif + +#ifndef mingw32_HOST_OS +{- Run an action as a daemon, with all output sent to a file descriptor. + - + - Can write its pid to a file, to guard against multiple instances + - running and allow easy termination. + - + - When successful, does not return. -} +daemonize :: Fd -> Maybe FilePath -> Bool -> IO () -> IO () +daemonize logfd pidfile changedirectory a = do + maybe noop checkalreadyrunning pidfile + _ <- forkProcess child1 + out + where + checkalreadyrunning f = maybe noop (const alreadyRunning) + =<< checkDaemon f + child1 = do + _ <- tryIO createSession + _ <- forkProcess child2 + out + child2 = do + maybe noop lockPidFile pidfile + when changedirectory $ + setCurrentDirectory "/" + nullfd <- openFd "/dev/null" ReadOnly Nothing defaultFileFlags + redir nullfd stdInput + redirLog logfd + {- In old versions of ghc, forkProcess masks async exceptions; + - unmask them inside the action. -} + wait =<< asyncWithUnmask (\unmask -> unmask a) + out + out = exitImmediately ExitSuccess +#endif + +{- To run an action that is normally daemonized in the forground. -} +#ifndef mingw32_HOST_OS +foreground :: Fd -> Maybe FilePath -> IO () -> IO () +foreground logfd pidfile a = do +#else +foreground :: Maybe FilePath -> IO () -> IO () +foreground pidfile a = do +#endif + maybe noop lockPidFile pidfile +#ifndef mingw32_HOST_OS + _ <- tryIO createSession + redirLog logfd +#endif + a +#ifndef mingw32_HOST_OS + exitImmediately ExitSuccess +#else + exitWith ExitSuccess +#endif + +{- Locks the pid file, with an exclusive, non-blocking lock, + - and leaves it locked on return. + - + - Writes the pid to the file, fully atomically. + - Fails if the pid file is already locked by another process. -} +lockPidFile :: FilePath -> IO () +lockPidFile pidfile = do + createDirectoryIfMissing True (parentDir pidfile) +#ifndef mingw32_HOST_OS + fd <- openFd pidfile ReadWrite (Just stdFileMode) defaultFileFlags + locked <- catchMaybeIO $ setLock fd (WriteLock, AbsoluteSeek, 0, 0) + fd' <- openFd newfile ReadWrite (Just stdFileMode) defaultFileFlags + { trunc = True } + locked' <- catchMaybeIO $ setLock fd' (WriteLock, AbsoluteSeek, 0, 0) + case (locked, locked') of + (Nothing, _) -> alreadyRunning + (_, Nothing) -> alreadyRunning + _ -> do + _ <- fdWrite fd' =<< show <$> getPID + closeFd fd + rename newfile pidfile + where + newfile = pidfile ++ ".new" +#else + {- Not atomic on Windows, oh well. -} + unlessM (isNothing <$> checkDaemon pidfile) + alreadyRunning + pid <- getPID + writeFile pidfile (show pid) + lckfile <- winLockFile pid pidfile + writeFile lckfile "" + void $ lockExclusive lckfile +#endif + +alreadyRunning :: IO () +alreadyRunning = giveup "Daemon is already running." + +{- Checks if the daemon is running, by checking that the pid file + - is locked by the same process that is listed in the pid file. + - + - If it's running, returns its pid. -} +checkDaemon :: FilePath -> IO (Maybe PID) +#ifndef mingw32_HOST_OS +checkDaemon pidfile = bracket setup cleanup go + where + setup = catchMaybeIO $ + openFd pidfile ReadOnly (Just stdFileMode) defaultFileFlags + cleanup (Just fd) = closeFd fd + cleanup Nothing = return () + go (Just fd) = catchDefaultIO Nothing $ do + locked <- getLock fd (ReadLock, AbsoluteSeek, 0, 0) + p <- readish <$> readFile pidfile + return (check locked p) + go Nothing = return Nothing + + check Nothing _ = Nothing + check _ Nothing = Nothing + check (Just (pid, _)) (Just pid') + | pid == pid' = Just pid + | otherwise = giveup $ + "stale pid in " ++ pidfile ++ + " (got " ++ show pid' ++ + "; expected " ++ show pid ++ " )" +#else +checkDaemon pidfile = maybe (return Nothing) (check . readish) + =<< catchMaybeIO (readFile pidfile) + where + check Nothing = return Nothing + check (Just pid) = do + v <- lockShared =<< winLockFile pid pidfile + case v of + Just h -> do + dropLock h + return Nothing + Nothing -> return (Just pid) +#endif + +{- Stops the daemon, safely. -} +stopDaemon :: FilePath -> IO () +stopDaemon pidfile = go =<< checkDaemon pidfile + where + go Nothing = noop + go (Just pid) = +#ifndef mingw32_HOST_OS + signalProcess sigTERM pid +#else + terminateProcessById pid +#endif + +{- Windows locks a lock file that corresponds with the pid of the process. + - This allows changing the process in the pid file and taking a new lock + - when eg, restarting the daemon. + -} +#ifdef mingw32_HOST_OS +winLockFile :: PID -> FilePath -> IO FilePath +winLockFile pid pidfile = do + cleanstale + return $ prefix ++ show pid ++ suffix + where + prefix = pidfile ++ "." + suffix = ".lck" + cleanstale = mapM_ (void . tryIO . removeFile) =<< + (filter iswinlockfile <$> dirContents (parentDir pidfile)) + iswinlockfile f = suffix `isSuffixOf` f && prefix `isPrefixOf` f +#endif diff --git a/Utility/Data.hs b/Utility/Data.hs new file mode 100644 index 0000000000..27c0a824c2 --- /dev/null +++ b/Utility/Data.hs @@ -0,0 +1,19 @@ +{- utilities for simple data types + - + - Copyright 2013 Joey Hess + - + - License: BSD-2-clause + -} + +{-# OPTIONS_GHC -fno-warn-tabs #-} + +module Utility.Data where + +{- First item in the list that is not Nothing. -} +firstJust :: Eq a => [Maybe a] -> Maybe a +firstJust ms = case dropWhile (== Nothing) ms of + [] -> Nothing + (md:_) -> md + +eitherToMaybe :: Either a b -> Maybe b +eitherToMaybe = either (const Nothing) Just diff --git a/Utility/DataUnits.hs b/Utility/DataUnits.hs new file mode 100644 index 0000000000..a6c9ffcf19 --- /dev/null +++ b/Utility/DataUnits.hs @@ -0,0 +1,166 @@ +{- data size display and parsing + - + - Copyright 2011 Joey Hess + - + - License: BSD-2-clause + - + - + - And now a rant: + - + - In the beginning, we had powers of two, and they were good. + - + - Disk drive manufacturers noticed that some powers of two were + - sorta close to some powers of ten, and that rounding down to the nearest + - power of ten allowed them to advertise their drives were bigger. This + - was sorta annoying. + - + - Then drives got big. Really, really big. This was good. + - + - Except that the small rounding error perpretrated by the drive + - manufacturers suffered the fate of a small error, and became a large + - error. This was bad. + - + - So, a committee was formed. And it arrived at a committee-like decision, + - which satisfied noone, confused everyone, and made the world an uglier + - place. As with all committees, this was meh. + - + - And the drive manufacturers happily continued selling drives that are + - increasingly smaller than you'd expect, if you don't count on your + - fingers. But that are increasingly too big for anyone to much notice. + - This caused me to need git-annex. + - + - Thus, I use units here that I loathe. Because if I didn't, people would + - be confused that their drives seem the wrong size, and other people would + - complain at me for not being standards compliant. And we call this + - progress? + -} + +module Utility.DataUnits ( + dataUnits, + storageUnits, + memoryUnits, + bandwidthUnits, + oldSchoolUnits, + Unit(..), + ByteSize, + + roughSize, + roughSize', + compareSizes, + readSize +) where + +import Data.List +import Data.Char + +import Utility.HumanNumber + +type ByteSize = Integer +type Name = String +type Abbrev = String +data Unit = Unit ByteSize Abbrev Name + deriving (Ord, Show, Eq) + +dataUnits :: [Unit] +dataUnits = storageUnits ++ memoryUnits + +{- Storage units are (stupidly) powers of ten. -} +storageUnits :: [Unit] +storageUnits = + [ Unit (p 8) "YB" "yottabyte" + , Unit (p 7) "ZB" "zettabyte" + , Unit (p 6) "EB" "exabyte" + , Unit (p 5) "PB" "petabyte" + , Unit (p 4) "TB" "terabyte" + , Unit (p 3) "GB" "gigabyte" + , Unit (p 2) "MB" "megabyte" + , Unit (p 1) "kB" "kilobyte" -- weird capitalization thanks to committe + , Unit (p 0) "B" "byte" + ] + where + p :: Integer -> Integer + p n = 1000^n + +{- Memory units are (stupidly named) powers of 2. -} +memoryUnits :: [Unit] +memoryUnits = + [ Unit (p 8) "YiB" "yobibyte" + , Unit (p 7) "ZiB" "zebibyte" + , Unit (p 6) "EiB" "exbibyte" + , Unit (p 5) "PiB" "pebibyte" + , Unit (p 4) "TiB" "tebibyte" + , Unit (p 3) "GiB" "gibibyte" + , Unit (p 2) "MiB" "mebibyte" + , Unit (p 1) "KiB" "kibibyte" + , Unit (p 0) "B" "byte" + ] + where + p :: Integer -> Integer + p n = 2^(n*10) + +{- Bandwidth units are only measured in bits if you're some crazy telco. -} +bandwidthUnits :: [Unit] +bandwidthUnits = error "stop trying to rip people off" + +{- Do you yearn for the days when men were men and megabytes were megabytes? -} +oldSchoolUnits :: [Unit] +oldSchoolUnits = zipWith (curry mingle) storageUnits memoryUnits + where + mingle (Unit _ a n, Unit s' _ _) = Unit s' a n + +{- approximate display of a particular number of bytes -} +roughSize :: [Unit] -> Bool -> ByteSize -> String +roughSize units short i = roughSize' units short 2 i + +roughSize' :: [Unit] -> Bool -> Int -> ByteSize -> String +roughSize' units short precision i + | i < 0 = '-' : findUnit units' (negate i) + | otherwise = findUnit units' i + where + units' = sortBy (flip compare) units -- largest first + + findUnit (u@(Unit s _ _):us) i' + | i' >= s = showUnit i' u + | otherwise = findUnit us i' + findUnit [] i' = showUnit i' (last units') -- bytes + + showUnit x (Unit size abbrev name) = s ++ " " ++ unit + where + v = (fromInteger x :: Double) / fromInteger size + s = showImprecise precision v + unit + | short = abbrev + | s == "1" = name + | otherwise = name ++ "s" + +{- displays comparison of two sizes -} +compareSizes :: [Unit] -> Bool -> ByteSize -> ByteSize -> String +compareSizes units abbrev old new + | old > new = roughSize units abbrev (old - new) ++ " smaller" + | old < new = roughSize units abbrev (new - old) ++ " larger" + | otherwise = "same" + +{- Parses strings like "10 kilobytes" or "0.5tb". -} +readSize :: [Unit] -> String -> Maybe ByteSize +readSize units input + | null parsednum || null parsedunit = Nothing + | otherwise = Just $ round $ number * fromIntegral multiplier + where + (number, rest) = head parsednum + multiplier = head parsedunit + unitname = takeWhile isAlpha $ dropWhile isSpace rest + + parsednum = reads input :: [(Double, String)] + parsedunit = lookupUnit units unitname + + lookupUnit _ [] = [1] -- no unit given, assume bytes + lookupUnit [] _ = [] + lookupUnit (Unit s a n:us) v + | a ~~ v || n ~~ v = [s] + | plural n ~~ v || a ~~ byteabbrev v = [s] + | otherwise = lookupUnit us v + + a ~~ b = map toLower a == map toLower b + + plural n = n ++ "s" + byteabbrev a = a ++ "b" diff --git a/Utility/DirWatcher.hs b/Utility/DirWatcher.hs new file mode 100644 index 0000000000..892841f93f --- /dev/null +++ b/Utility/DirWatcher.hs @@ -0,0 +1,157 @@ +{- generic directory watching interface + - + - Uses inotify, or kqueue, or fsevents, or win32-notify to watch a directory + - (and subdirectories) for changes, and runs hooks for different + - sorts of events as they occur. + - + - Copyright 2012-2013 Joey Hess + - + - License: BSD-2-clause + -} + +{-# LANGUAGE CPP #-} + +module Utility.DirWatcher where + +import Utility.DirWatcher.Types + +#if WITH_INOTIFY +import qualified Utility.DirWatcher.INotify as INotify +import qualified System.INotify as INotify +#endif +#if WITH_KQUEUE +import qualified Utility.DirWatcher.Kqueue as Kqueue +import Control.Concurrent +#endif +#if WITH_FSEVENTS +import qualified Utility.DirWatcher.FSEvents as FSEvents +import qualified System.OSX.FSEvents as FSEvents +#endif +#if WITH_WIN32NOTIFY +import qualified Utility.DirWatcher.Win32Notify as Win32Notify +import qualified System.Win32.Notify as Win32Notify +#endif + +type Pruner = FilePath -> Bool + +canWatch :: Bool +#if (WITH_INOTIFY || WITH_KQUEUE || WITH_FSEVENTS || WITH_WIN32NOTIFY) +canWatch = True +#else +#if defined linux_HOST_OS +#warning "Building without inotify support" +#endif +canWatch = False +#endif + +{- With inotify, discrete events will be received when making multiple changes + - to the same filename. For example, adding it, deleting it, and adding it + - again will be three events. + - + - OTOH, with kqueue, often only one event is received, indicating the most + - recent state of the file. -} +eventsCoalesce :: Bool +#if (WITH_INOTIFY || WITH_WIN32NOTIFY) +eventsCoalesce = False +#else +#if (WITH_KQUEUE || WITH_FSEVENTS) +eventsCoalesce = True +#else +eventsCoalesce = error "eventsCoalesce not defined" +#endif +#endif + +{- With inotify, file closing is tracked to some extent, so an add event + - will always be received for a file once its writer closes it, and + - (typically) not before. This may mean multiple add events for the same file. + - + - OTOH, with kqueue, add events will often be received while a file is + - still being written to, and then no add event will be received once the + - writer closes it. + - + - fsevents sometimes behaves similarly, but has sometimes been + - seen to behave like kqueue. -} +closingTracked :: Bool +#if (WITH_INOTIFY || WITH_WIN32NOTIFY) +closingTracked = True +#else +#if (WITH_KQUEUE || WITH_FSEVENTS) +closingTracked = False +#else +closingTracked = error "closingTracked not defined" +#endif +#endif + +{- With inotify, modifications to existing files can be tracked. + - Kqueue does not support this. + - Fsevents generates events when an existing file is reopened and rewritten, + - but not necessarily when it's opened once and modified repeatedly. -} +modifyTracked :: Bool +#if (WITH_INOTIFY || WITH_FSEVENTS || WITH_WIN32NOTIFY) +modifyTracked = True +#else +#if WITH_KQUEUE +modifyTracked = False +#else +modifyTracked = error "modifyTracked not defined" +#endif +#endif + +{- Starts a watcher thread. The runstartup action is passed a scanner action + - to run, that will return once the initial directory scan is complete. + - Once runstartup returns, the watcher thread continues running, + - and processing events. Returns a DirWatcherHandle that can be used + - to shutdown later. -} +#if WITH_INOTIFY +type DirWatcherHandle = INotify.INotify +watchDir :: FilePath -> Pruner -> Bool -> WatchHooks -> (IO () -> IO ()) -> IO DirWatcherHandle +watchDir dir prune scanevents hooks runstartup = do + i <- INotify.initINotify + runstartup $ INotify.watchDir i dir prune scanevents hooks + return i +#else +#if WITH_KQUEUE +type DirWatcherHandle = ThreadId +watchDir :: FilePath -> Pruner -> Bool -> WatchHooks -> (IO Kqueue.Kqueue -> IO Kqueue.Kqueue) -> IO DirWatcherHandle +watchDir dir prune _scanevents hooks runstartup = do + kq <- runstartup $ Kqueue.initKqueue dir prune + forkIO $ Kqueue.runHooks kq hooks +#else +#if WITH_FSEVENTS +type DirWatcherHandle = FSEvents.EventStream +watchDir :: FilePath -> Pruner -> Bool -> WatchHooks -> (IO FSEvents.EventStream -> IO FSEvents.EventStream) -> IO DirWatcherHandle +watchDir dir prune scanevents hooks runstartup = + runstartup $ FSEvents.watchDir dir prune scanevents hooks +#else +#if WITH_WIN32NOTIFY +type DirWatcherHandle = Win32Notify.WatchManager +watchDir :: FilePath -> Pruner -> Bool -> WatchHooks -> (IO Win32Notify.WatchManager -> IO Win32Notify.WatchManager) -> IO DirWatcherHandle +watchDir dir prune scanevents hooks runstartup = + runstartup $ Win32Notify.watchDir dir prune scanevents hooks +#else +type DirWatcherHandle = () +watchDir :: FilePath -> Pruner -> Bool -> WatchHooks -> (IO () -> IO ()) -> IO DirWatcherHandle +watchDir = error "watchDir not defined" +#endif +#endif +#endif +#endif + +stopWatchDir :: DirWatcherHandle -> IO () +#if WITH_INOTIFY +stopWatchDir = INotify.killINotify +#else +#if WITH_KQUEUE +stopWatchDir = killThread +#else +#if WITH_FSEVENTS +stopWatchDir = FSEvents.eventStreamDestroy +#else +#if WITH_WIN32NOTIFY +stopWatchDir = Win32Notify.killWatchManager +#else +stopWatchDir = error "stopWatchDir not defined" +#endif +#endif +#endif +#endif diff --git a/Utility/DirWatcher/FSEvents.hs b/Utility/DirWatcher/FSEvents.hs new file mode 100644 index 0000000000..d7472d490a --- /dev/null +++ b/Utility/DirWatcher/FSEvents.hs @@ -0,0 +1,96 @@ +{- FSEvents interface + - + - Copyright 2012 Joey Hess + - + - License: BSD-2-clause + -} + +module Utility.DirWatcher.FSEvents where + +import Common hiding (isDirectory) +import Utility.DirWatcher.Types + +import System.OSX.FSEvents +import qualified System.Posix.Files as Files +import Data.Bits ((.&.)) + +watchDir :: FilePath -> (FilePath -> Bool) -> Bool -> WatchHooks -> IO EventStream +watchDir dir ignored scanevents hooks = do + unlessM fileLevelEventsSupported $ + giveup "Need at least OSX 10.7.0 for file-level FSEvents" + scan dir + eventStreamCreate [dir] 1.0 True True True dispatch + where + dispatch evt + | ignoredPath ignored (eventPath evt) = noop + | otherwise = do + {- More than one flag may be set, if events occurred + - close together. + - + - Order is important.. + - If a file is added and then deleted, we'll see it's + - not present, and addHook won't run. + - OTOH, if a file is deleted and then re-added, + - the delHook will run first, followed by the addHook. + -} + + when (hasflag eventFlagItemRemoved) $ + if hasflag eventFlagItemIsDir + then runhook delDirHook Nothing + else runhook delHook Nothing + when (hasflag eventFlagItemCreated) $ + maybe noop handleadd =<< getstatus (eventPath evt) + {- When a file or dir is renamed, a rename event is + - received for both its old and its new name. -} + when (hasflag eventFlagItemRenamed) $ + if hasflag eventFlagItemIsDir + then ifM (doesDirectoryExist $ eventPath evt) + ( scan $ eventPath evt + , runhook delDirHook Nothing + ) + else maybe (runhook delHook Nothing) handleadd + =<< getstatus (eventPath evt) + {- Add hooks are run when a file is modified for + - compatability with INotify, which calls the add + - hook when a file is closed, and so tends to call + - both add and modify for file modifications. -} + when (hasflag eventFlagItemModified && not (hasflag eventFlagItemIsDir)) $ do + ms <- getstatus $ eventPath evt + maybe noop handleadd ms + runhook modifyHook ms + where + hasflag f = eventFlags evt .&. f /= 0 + runhook h s = maybe noop (\a -> a (eventPath evt) s) (h hooks) + handleadd s + | Files.isSymbolicLink s = runhook addSymlinkHook $ Just s + | Files.isRegularFile s = runhook addHook $ Just s + | otherwise = noop + + scan d = unless (ignoredPath ignored d) $ + -- Do not follow symlinks when scanning. + -- This mirrors the inotify startup scan behavior. + mapM_ go =<< dirContentsRecursiveSkipping (const False) False d + where + go f + | ignoredPath ignored f = noop + | otherwise = do + ms <- getstatus f + case ms of + Nothing -> noop + Just s + | Files.isSymbolicLink s -> + when scanevents $ + runhook addSymlinkHook ms + | Files.isRegularFile s -> + when scanevents $ + runhook addHook ms + | otherwise -> + noop + where + runhook h s = maybe noop (\a -> a f s) (h hooks) + + getstatus = catchMaybeIO . getSymbolicLinkStatus + +{- Check each component of the path to see if it's ignored. -} +ignoredPath :: (FilePath -> Bool) -> FilePath -> Bool +ignoredPath ignored = any ignored . map dropTrailingPathSeparator . splitPath diff --git a/Utility/DirWatcher/INotify.hs b/Utility/DirWatcher/INotify.hs new file mode 100644 index 0000000000..3071c51205 --- /dev/null +++ b/Utility/DirWatcher/INotify.hs @@ -0,0 +1,215 @@ +{-# LANGUAGE CPP #-} + +{- higher-level inotify interface + - + - Copyright 2012 Joey Hess + - + - License: BSD-2-clause + -} + +module Utility.DirWatcher.INotify where + +import Common hiding (isDirectory) +import Utility.ThreadLock +import Utility.DirWatcher.Types +#if MIN_VERSION_hinotify(0,3,10) +import Utility.FileSystemEncoding +#endif + +import System.INotify +import qualified System.Posix.Files as Files +import System.IO.Error +import Control.Exception (throw) + +{- Watches for changes to files in a directory, and all its subdirectories + - that are not ignored, using inotify. This function returns after + - its initial scan is complete, leaving a thread running. Callbacks are + - made for different events. + - + - Inotify is weak at recursive directory watching; the whole directory + - tree must be scanned and watches set explicitly for each subdirectory. + - + - To notice newly created subdirectories, inotify is used, and + - watches are registered for those directories. There is a race there; + - things can be added to a directory before the watch gets registered. + - + - To close the inotify race, each time a new directory is found, it also + - recursively scans it, assuming all files in it were just added, + - and registering each subdirectory. + - + - Note: Due to the race amelioration, multiple add events may occur + - for the same file. + - + - Note: Moving a file will cause events deleting it from its old location + - and adding it to the new location. + - + - Note: It's assumed that when a file that was open for write is closed, + - it's finished being written to, and can be added. + - + - Note: inotify has a limit to the number of watches allowed, + - /proc/sys/fs/inotify/max_user_watches (default 8192). + - So this will fail if there are too many subdirectories. The + - errHook is called when this happens. + -} +watchDir :: INotify -> FilePath -> (FilePath -> Bool) -> Bool -> WatchHooks -> IO () +watchDir i dir ignored scanevents hooks + | ignored dir = noop + | otherwise = do + -- Use a lock to make sure events generated during initial + -- scan come before real inotify events. + lock <- newLock + let handler event = withLock lock (void $ go event) + flip catchNonAsync failedwatch $ do + void (addWatch i watchevents (toInternalFilePath dir) handler) + `catchIO` failedaddwatch + withLock lock $ + mapM_ scan =<< filter (not . dirCruft) <$> + getDirectoryContents dir + where + recurse d = watchDir i d ignored scanevents hooks + + -- Select only inotify events required by the enabled + -- hooks, but always include Create so new directories can + -- be scanned. + watchevents = Create : addevents ++ delevents ++ modifyevents + addevents + | hashook addHook || hashook addSymlinkHook = [MoveIn, CloseWrite] + | otherwise = [] + delevents + | hashook delHook || hashook delDirHook = [MoveOut, Delete] + | otherwise = [] + modifyevents + | hashook modifyHook = [Modify] + | otherwise = [] + + scan f = unless (ignored f) $ do + ms <- getstatus f + case ms of + Nothing -> return () + Just s + | Files.isDirectory s -> + recurse $ indir f + | Files.isSymbolicLink s -> + when scanevents $ + runhook addSymlinkHook f ms + | Files.isRegularFile s -> + when scanevents $ + runhook addHook f ms + | otherwise -> + noop + + go (Created { isDirectory = isd, filePath = fi }) + | isd = recurse $ indir f + | otherwise = do + ms <- getstatus f + case ms of + Just s + | Files.isSymbolicLink s -> + when (hashook addSymlinkHook) $ + runhook addSymlinkHook f ms + | Files.isRegularFile s -> + when (hashook addHook) $ + runhook addHook f ms + _ -> noop + where + f = fromInternalFilePath fi + + -- Closing a file is assumed to mean it's done being written, + -- so a new add event is sent. + go (Closed { isDirectory = False, maybeFilePath = Just fi }) = + checkfiletype Files.isRegularFile addHook $ + fromInternalFilePath fi + + -- When a file or directory is moved in, scan it to add new + -- stuff. + go (MovedIn { filePath = fi }) = scan $ fromInternalFilePath fi + go (MovedOut { isDirectory = isd, filePath = fi }) + | isd = runhook delDirHook f Nothing + | otherwise = runhook delHook f Nothing + where + f = fromInternalFilePath fi + + -- Verify that the deleted item really doesn't exist, + -- since there can be spurious deletion events for items + -- in a directory that has been moved out, but is still + -- being watched. + go (Deleted { isDirectory = isd, filePath = fi }) + | isd = guarded $ runhook delDirHook f Nothing + | otherwise = guarded $ runhook delHook f Nothing + where + guarded = unlessM (filetype (const True) f) + f = fromInternalFilePath fi + + go (Modified { isDirectory = isd, maybeFilePath = Just fi }) + | isd = noop + | otherwise = runhook modifyHook (fromInternalFilePath fi) Nothing + + go _ = noop + + hashook h = isJust $ h hooks + + runhook h f s + | ignored f = noop + | otherwise = maybe noop (\a -> a (indir f) s) (h hooks) + + indir f = dir f + + getstatus f = catchMaybeIO $ getSymbolicLinkStatus $ indir f + checkfiletype check h f = do + ms <- getstatus f + case ms of + Just s + | check s -> runhook h f ms + _ -> noop + filetype t f = catchBoolIO $ t <$> getSymbolicLinkStatus (indir f) + + failedaddwatch e + -- Inotify fails when there are too many watches with a + -- disk full error. + | isFullError e = + case errHook hooks of + Nothing -> giveup $ "failed to add inotify watch on directory " ++ dir ++ " (" ++ show e ++ ")" + Just hook -> tooManyWatches hook dir + -- The directory could have been deleted. + | isDoesNotExistError e = return () + | otherwise = throw e + + failedwatch e = hPutStrLn stderr $ "failed to add watch on directory " ++ dir ++ " (" ++ show e ++ ")" + +tooManyWatches :: (String -> Maybe FileStatus -> IO ()) -> FilePath -> IO () +tooManyWatches hook dir = do + sysctlval <- querySysctl [Param maxwatches] :: IO (Maybe Integer) + hook (unlines $ basewarning : maybe withoutsysctl withsysctl sysctlval) Nothing + where + maxwatches = "fs.inotify.max_user_watches" + basewarning = "Too many directories to watch! (Not watching " ++ dir ++")" + withoutsysctl = ["Increase the value in /proc/sys/fs/inotify/max_user_watches"] + withsysctl n = let new = n * 10 in + [ "Increase the limit permanently by running:" + , " echo " ++ maxwatches ++ "=" ++ show new ++ + " | sudo tee -a /etc/sysctl.conf; sudo sysctl -p" + , "Or temporarily by running:" + , " sudo sysctl -w " ++ maxwatches ++ "=" ++ show new + ] + +querySysctl :: Read a => [CommandParam] -> IO (Maybe a) +querySysctl ps = getM go ["sysctl", "/sbin/sysctl", "/usr/sbin/sysctl"] + where + go p = do + v <- catchMaybeIO $ readProcess p (toCommand ps) + case v of + Nothing -> return Nothing + Just s -> return $ parsesysctl s + parsesysctl s = readish =<< lastMaybe (words s) + +#if MIN_VERSION_hinotify(0,3,10) +toInternalFilePath :: FilePath -> RawFilePath +toInternalFilePath = toRawFilePath +fromInternalFilePath :: RawFilePath -> FilePath +fromInternalFilePath = fromRawFilePath +#else +toInternalFilePath :: FilePath -> FilePath +toInternalFilePath = id +fromInternalFilePath :: FilePath -> FilePath +fromInternalFilePath = id +#endif diff --git a/Utility/DirWatcher/Kqueue.hs b/Utility/DirWatcher/Kqueue.hs new file mode 100644 index 0000000000..4a1c55ae91 --- /dev/null +++ b/Utility/DirWatcher/Kqueue.hs @@ -0,0 +1,268 @@ +{- BSD kqueue file modification notification interface + - + - Copyright 2012 Joey Hess + - + - License: BSD-2-clause + -} + +{-# LANGUAGE ForeignFunctionInterface #-} + +module Utility.DirWatcher.Kqueue ( + Kqueue, + initKqueue, + stopKqueue, + waitChange, + Change(..), + changedFile, + runHooks, +) where + +import Common +import Utility.DirWatcher.Types + +import System.Posix.Types +import Foreign.C.Types +import Foreign.C.Error +import Foreign.Ptr +import Foreign.Marshal +import qualified Data.Map.Strict as M +import qualified Data.Set as S +import qualified System.Posix.Files as Posix +import qualified System.Posix.IO as Posix +import Control.Concurrent + +data Change + = Deleted FilePath + | DeletedDir FilePath + | Added FilePath + deriving (Show) + +isAdd :: Change -> Bool +isAdd (Added _) = True +isAdd (Deleted _) = False +isAdd (DeletedDir _) = False + +changedFile :: Change -> FilePath +changedFile (Added f) = f +changedFile (Deleted f) = f +changedFile (DeletedDir f) = f + +data Kqueue = Kqueue + { kqueueFd :: Fd + , kqueueTop :: FilePath + , kqueueMap :: DirMap + , _kqueuePruner :: Pruner + } + +type Pruner = FilePath -> Bool + +type DirMap = M.Map Fd DirInfo + +{- Enough information to uniquely identify a file in a directory, + - but not too much. -} +data DirEnt = DirEnt + { dirEnt :: FilePath -- relative to the parent directory + , _dirInode :: FileID -- included to notice file replacements + , isSubDir :: Bool + } + deriving (Eq, Ord, Show) + +{- A directory, and its last known contents. -} +data DirInfo = DirInfo + { dirName :: FilePath + , dirCache :: S.Set DirEnt + } + deriving (Show) + +getDirInfo :: FilePath -> IO DirInfo +getDirInfo dir = do + l <- filter (not . dirCruft) <$> getDirectoryContents dir + contents <- S.fromList . catMaybes <$> mapM getDirEnt l + return $ DirInfo dir contents + where + getDirEnt f = catchMaybeIO $ do + s <- getSymbolicLinkStatus (dir f) + return $ DirEnt f (fileID s) (isDirectory s) + +{- Difference between the dirCaches of two DirInfos. -} +(//) :: DirInfo -> DirInfo -> [Change] +oldc // newc = deleted ++ added + where + deleted = calc gendel oldc newc + added = calc genadd newc oldc + gendel x = (if isSubDir x then DeletedDir else Deleted) $ + dirName oldc dirEnt x + genadd x = Added $ dirName newc dirEnt x + calc a x y = map a $ S.toList $ + S.difference (dirCache x) (dirCache y) + +{- Builds a map of directories in a tree, possibly pruning some. + - Opens each directory in the tree, and records its current contents. -} +scanRecursive :: FilePath -> Pruner -> IO DirMap +scanRecursive topdir prune = M.fromList <$> walk [] [topdir] + where + walk c [] = return c + walk c (dir:rest) + | prune dir = walk c rest + | otherwise = do + minfo <- catchMaybeIO $ getDirInfo dir + case minfo of + Nothing -> walk c rest + Just info -> do + mfd <- catchMaybeIO $ + Posix.openFd dir Posix.ReadOnly Nothing Posix.defaultFileFlags + case mfd of + Nothing -> walk c rest + Just fd -> do + let subdirs = map (dir ) . map dirEnt $ + S.toList $ dirCache info + walk ((fd, info):c) (subdirs ++ rest) + +{- Adds a list of subdirectories (and all their children), unless pruned to a + - directory map. Adding a subdirectory that's already in the map will + - cause its contents to be refreshed. -} +addSubDirs :: DirMap -> Pruner -> [FilePath] -> IO DirMap +addSubDirs dirmap prune dirs = do + newmap <- foldr M.union M.empty <$> + mapM (\d -> scanRecursive d prune) dirs + return $ M.union newmap dirmap -- prefer newmap + +{- Removes a subdirectory (and all its children) from a directory map. -} +removeSubDir :: DirMap -> FilePath -> IO DirMap +removeSubDir dirmap dir = do + mapM_ Posix.closeFd $ M.keys toremove + return rest + where + (toremove, rest) = M.partition (dirContains dir . dirName) dirmap + +findDirContents :: DirMap -> FilePath -> [FilePath] +findDirContents dirmap dir = concatMap absolutecontents $ search + where + absolutecontents i = map (dirName i ) + (map dirEnt $ S.toList $ dirCache i) + search = map snd $ M.toList $ + M.filter (\i -> dirName i == dir) dirmap + +foreign import ccall safe "libkqueue.h init_kqueue" c_init_kqueue + :: IO Fd +foreign import ccall safe "libkqueue.h addfds_kqueue" c_addfds_kqueue + :: Fd -> CInt -> Ptr Fd -> IO () +foreign import ccall safe "libkqueue.h waitchange_kqueue" c_waitchange_kqueue + :: Fd -> IO Fd + +{- Initializes a Kqueue to watch a directory, and all its subdirectories. -} +initKqueue :: FilePath -> Pruner -> IO Kqueue +initKqueue dir pruned = do + dirmap <- scanRecursive dir pruned + h <- c_init_kqueue + let kq = Kqueue h dir dirmap pruned + updateKqueue kq + return kq + +{- Updates a Kqueue, adding watches for its map. -} +updateKqueue :: Kqueue -> IO () +updateKqueue (Kqueue h _ dirmap _) = + withArrayLen (M.keys dirmap) $ \fdcnt c_fds -> do + c_addfds_kqueue h (fromIntegral fdcnt) c_fds + +{- Stops a Kqueue. Note: Does not directly close the Fds in the dirmap, + - so it can be reused. -} +stopKqueue :: Kqueue -> IO () +stopKqueue = Posix.closeFd . kqueueFd + +{- Waits for a change on a Kqueue. + - May update the Kqueue. + -} +waitChange :: Kqueue -> IO (Kqueue, [Change]) +waitChange kq@(Kqueue h _ dirmap _) = do + changedfd <- c_waitchange_kqueue h + if changedfd == -1 + then ifM ((==) eINTR <$> getErrno) + (yield >> waitChange kq, nochange) + else case M.lookup changedfd dirmap of + Nothing -> nochange + Just info -> handleChange kq changedfd info + where + nochange = return (kq, []) + +{- The kqueue interface does not tell what type of change took place in + - the directory; it could be an added file, a deleted file, a renamed + - file, a new subdirectory, or a deleted subdirectory, or a moved + - subdirectory. + - + - So to determine this, the contents of the directory are compared + - with its last cached contents. The Kqueue is updated to watch new + - directories as necessary. + -} +handleChange :: Kqueue -> Fd -> DirInfo -> IO (Kqueue, [Change]) +handleChange kq@(Kqueue _ _ dirmap pruner) fd olddirinfo = + go =<< catchMaybeIO (getDirInfo $ dirName olddirinfo) + where + go (Just newdirinfo) = do + let changes = filter (not . pruner . changedFile) $ + olddirinfo // newdirinfo + let (added, deleted) = partition isAdd changes + + -- Scan newly added directories to add to the map. + -- (Newly added files will fail getDirInfo.) + newdirinfos <- catMaybes <$> + mapM (catchMaybeIO . getDirInfo . changedFile) added + newmap <- addSubDirs dirmap pruner $ map dirName newdirinfos + + -- Remove deleted directories from the map. + newmap' <- foldM removeSubDir newmap (map changedFile deleted) + + -- Update the cached dirinfo just looked up. + let newmap'' = M.insert fd newdirinfo newmap' + + -- When new directories were added, need to update + -- the kqueue to watch them. + let kq' = kq { kqueueMap = newmap'' } + unless (null newdirinfos) $ + updateKqueue kq' + + return (kq', changes) + go Nothing = do + -- The directory has been moved or deleted, so + -- remove it from our map. + newmap <- removeSubDir dirmap (dirName olddirinfo) + return (kq { kqueueMap = newmap }, []) + +{- Processes changes on the Kqueue, calling the hooks as appropriate. + - Never returns. -} +runHooks :: Kqueue -> WatchHooks -> IO () +runHooks kq hooks = do + -- First, synthetic add events for the whole directory tree contents, + -- to catch any files created beforehand. + recursiveadd (kqueueMap kq) (Added $ kqueueTop kq) + loop kq + where + loop q = do + (q', changes) <- waitChange q + forM_ changes $ dispatch (kqueueMap q') + loop q' + + dispatch _ change@(Deleted _) = + callhook delHook Nothing change + dispatch _ change@(DeletedDir _) = + callhook delDirHook Nothing change + dispatch dirmap change@(Added _) = + withstatus change $ dispatchadd dirmap + + dispatchadd dirmap change s + | Posix.isSymbolicLink s = callhook addSymlinkHook (Just s) change + | Posix.isDirectory s = recursiveadd dirmap change + | Posix.isRegularFile s = callhook addHook (Just s) change + | otherwise = noop + + recursiveadd dirmap change = do + let contents = findDirContents dirmap $ changedFile change + forM_ contents $ \f -> + withstatus (Added f) $ dispatchadd dirmap + + callhook h s change = case h hooks of + Nothing -> noop + Just a -> a (changedFile change) s + + withstatus change a = maybe noop (a change) =<< + (catchMaybeIO (getSymbolicLinkStatus (changedFile change))) diff --git a/Utility/DirWatcher/Types.hs b/Utility/DirWatcher/Types.hs new file mode 100644 index 0000000000..75ef69f83b --- /dev/null +++ b/Utility/DirWatcher/Types.hs @@ -0,0 +1,24 @@ +{- generic directory watching types + - + - Copyright 2012 Joey Hess + - + - License: BSD-2-clause + -} + +module Utility.DirWatcher.Types where + +import Common + +type Hook a = Maybe (a -> Maybe FileStatus -> IO ()) + +data WatchHooks = WatchHooks + { addHook :: Hook FilePath + , addSymlinkHook :: Hook FilePath + , delHook :: Hook FilePath + , delDirHook :: Hook FilePath + , errHook :: Hook String -- error message + , modifyHook :: Hook FilePath + } + +mkWatchHooks :: WatchHooks +mkWatchHooks = WatchHooks Nothing Nothing Nothing Nothing Nothing Nothing diff --git a/Utility/DirWatcher/Win32Notify.hs b/Utility/DirWatcher/Win32Notify.hs new file mode 100644 index 0000000000..a2f40128fb --- /dev/null +++ b/Utility/DirWatcher/Win32Notify.hs @@ -0,0 +1,66 @@ +{- Win32-notify interface + - + - Copyright 2013 Joey Hess + - + - License: BSD-2-clause + -} + +module Utility.DirWatcher.Win32Notify where + +import Common hiding (isDirectory) +import Utility.DirWatcher.Types + +import System.Win32.Notify +import qualified System.PosixCompat.Files as Files + +watchDir :: FilePath -> (FilePath -> Bool) -> Bool -> WatchHooks -> IO WatchManager +watchDir dir ignored scanevents hooks = do + scan dir + wm <- initWatchManager + void $ watchDirectory wm dir True [Create, Delete, Modify, Move] dispatch + return wm + where + dispatch evt + | ignoredPath ignored (filePath evt) = noop + | otherwise = case evt of + (Deleted _ _) + | isDirectory evt -> runhook delDirHook Nothing + | otherwise -> runhook delHook Nothing + (Created _ _) + | isDirectory evt -> noop + | otherwise -> runhook addHook Nothing + (Modified _ _) + | isDirectory evt -> noop + {- Add hooks are run when a file is modified for + - compatability with INotify, which calls the add + - hook when a file is closed, and so tends to call + - both add and modify for file modifications. -} + | otherwise -> do + runhook addHook Nothing + runhook modifyHook Nothing + where + runhook h s = maybe noop (\a -> a (filePath evt) s) (h hooks) + + scan d = unless (ignoredPath ignored d) $ + mapM_ go =<< dirContentsRecursiveSkipping (const False) False d + where + go f + | ignoredPath ignored f = noop + | otherwise = do + ms <- getstatus f + case ms of + Nothing -> noop + Just s + | Files.isRegularFile s -> + when scanevents $ + runhook addHook ms + | otherwise -> + noop + where + runhook h s = maybe noop (\a -> a f s) (h hooks) + + getstatus = catchMaybeIO . getFileStatus + +{- Check each component of the path to see if it's ignored. -} +ignoredPath :: (FilePath -> Bool) -> FilePath -> Bool +ignoredPath ignored = any ignored . map dropTrailingPathSeparator . splitPath diff --git a/Utility/Directory.hs b/Utility/Directory.hs new file mode 100644 index 0000000000..e2c6a94629 --- /dev/null +++ b/Utility/Directory.hs @@ -0,0 +1,156 @@ +{- directory traversal and manipulation + - + - Copyright 2011-2014 Joey Hess + - + - License: BSD-2-clause + -} + +{-# LANGUAGE CPP #-} +{-# OPTIONS_GHC -fno-warn-tabs #-} + +module Utility.Directory ( + module Utility.Directory, + module Utility.SystemDirectory +) where + +import System.IO.Error +import Control.Monad +import System.FilePath +import System.PosixCompat.Files +import Control.Applicative +import System.IO.Unsafe (unsafeInterleaveIO) +import Data.Maybe +import Prelude + +#ifndef mingw32_HOST_OS +import Utility.SafeCommand +import Control.Monad.IfElse +#endif + +import Utility.SystemDirectory +import Utility.Tmp +import Utility.Exception +import Utility.Monad +import Utility.Applicative + +dirCruft :: FilePath -> Bool +dirCruft "." = True +dirCruft ".." = True +dirCruft _ = False + +{- Lists the contents of a directory. + - Unlike getDirectoryContents, paths are not relative to the directory. -} +dirContents :: FilePath -> IO [FilePath] +dirContents d = map (d ) . filter (not . dirCruft) <$> getDirectoryContents d + +{- Gets files in a directory, and then its subdirectories, recursively, + - and lazily. + - + - Does not follow symlinks to other subdirectories. + - + - When the directory does not exist, no exception is thrown, + - instead, [] is returned. -} +dirContentsRecursive :: FilePath -> IO [FilePath] +dirContentsRecursive = dirContentsRecursiveSkipping (const False) True + +{- Skips directories whose basenames match the skipdir. -} +dirContentsRecursiveSkipping :: (FilePath -> Bool) -> Bool -> FilePath -> IO [FilePath] +dirContentsRecursiveSkipping skipdir followsubdirsymlinks topdir = go [topdir] + where + go [] = return [] + go (dir:dirs) + | skipdir (takeFileName dir) = go dirs + | otherwise = unsafeInterleaveIO $ do + (files, dirs') <- collect [] [] + =<< catchDefaultIO [] (dirContents dir) + files' <- go (dirs' ++ dirs) + return (files ++ files') + collect files dirs' [] = return (reverse files, reverse dirs') + collect files dirs' (entry:entries) + | dirCruft entry = collect files dirs' entries + | otherwise = do + let skip = collect (entry:files) dirs' entries + let recurse = collect files (entry:dirs') entries + ms <- catchMaybeIO $ getSymbolicLinkStatus entry + case ms of + (Just s) + | isDirectory s -> recurse + | isSymbolicLink s && followsubdirsymlinks -> + ifM (doesDirectoryExist entry) + ( recurse + , skip + ) + _ -> skip + +{- Gets the directory tree from a point, recursively and lazily, + - with leaf directories **first**, skipping any whose basenames + - match the skipdir. Does not follow symlinks. -} +dirTreeRecursiveSkipping :: (FilePath -> Bool) -> FilePath -> IO [FilePath] +dirTreeRecursiveSkipping skipdir topdir = go [] [topdir] + where + go c [] = return c + go c (dir:dirs) + | skipdir (takeFileName dir) = go c dirs + | otherwise = unsafeInterleaveIO $ do + subdirs <- go [] + =<< filterM (isDirectory <$$> getSymbolicLinkStatus) + =<< catchDefaultIO [] (dirContents dir) + go (subdirs++dir:c) dirs + +{- Moves one filename to another. + - First tries a rename, but falls back to moving across devices if needed. -} +moveFile :: FilePath -> FilePath -> IO () +moveFile src dest = tryIO (rename src dest) >>= onrename + where + onrename (Right _) = noop + onrename (Left e) + | isPermissionError e = rethrow + | isDoesNotExistError e = rethrow + | otherwise = viaTmp mv dest "" + where + rethrow = throwM e + + mv tmp _ = do + -- copyFile is likely not as optimised as + -- the mv command, so we'll use the command. + -- + -- But, while Windows has a "mv", it does not seem very + -- reliable, so use copyFile there. +#ifndef mingw32_HOST_OS + -- If dest is a directory, mv would move the file + -- into it, which is not desired. + whenM (isdir dest) rethrow + ok <- boolSystem "mv" [Param "-f", Param src, Param tmp] + let e' = e +#else + r <- tryIO $ copyFile src tmp + let (ok, e') = case r of + Left err -> (False, err) + Right _ -> (True, e) +#endif + unless ok $ do + -- delete any partial + _ <- tryIO $ removeFile tmp + throwM e' + +#ifndef mingw32_HOST_OS + isdir f = do + r <- tryIO $ getFileStatus f + case r of + (Left _) -> return False + (Right s) -> return $ isDirectory s +#endif + +{- Removes a file, which may or may not exist, and does not have to + - be a regular file. + - + - Note that an exception is thrown if the file exists but + - cannot be removed. -} +nukeFile :: FilePath -> IO () +nukeFile file = void $ tryWhenExists go + where +#ifndef mingw32_HOST_OS + go = removeLink file +#else + go = removeFile file +#endif diff --git a/Utility/Directory/Stream.hs b/Utility/Directory/Stream.hs new file mode 100644 index 0000000000..e827ef21a2 --- /dev/null +++ b/Utility/Directory/Stream.hs @@ -0,0 +1,130 @@ +{- streaming directory traversal + - + - Copyright 2011-2018 Joey Hess + - + - License: BSD-2-clause + -} + +{-# LANGUAGE CPP #-} +{-# LANGUAGE LambdaCase #-} +{-# OPTIONS_GHC -fno-warn-tabs #-} + +module Utility.Directory.Stream where + +import Control.Monad +import System.FilePath +import System.IO.Unsafe (unsafeInterleaveIO) +import Control.Concurrent +import Data.Maybe +import Prelude + +#ifdef mingw32_HOST_OS +import qualified System.Win32 as Win32 +#else +import qualified System.Posix as Posix +#endif + +import Utility.Directory +import Utility.Exception + +#ifndef mingw32_HOST_OS +data DirectoryHandle = DirectoryHandle IsOpen Posix.DirStream +#else +data DirectoryHandle = DirectoryHandle IsOpen Win32.HANDLE Win32.FindData (MVar ()) +#endif + +type IsOpen = MVar () -- full when the handle is open + +openDirectory :: FilePath -> IO DirectoryHandle +openDirectory path = do +#ifndef mingw32_HOST_OS + dirp <- Posix.openDirStream path + isopen <- newMVar () + return (DirectoryHandle isopen dirp) +#else + (h, fdat) <- Win32.findFirstFile (path "*") + -- Indicate that the fdat contains a filename that readDirectory + -- has not yet returned, by making the MVar be full. + -- (There's always at least a "." entry.) + alreadyhave <- newMVar () + isopen <- newMVar () + return (DirectoryHandle isopen h fdat alreadyhave) +#endif + +closeDirectory :: DirectoryHandle -> IO () +#ifndef mingw32_HOST_OS +closeDirectory (DirectoryHandle isopen dirp) = + whenOpen isopen $ + Posix.closeDirStream dirp +#else +closeDirectory (DirectoryHandle isopen h _ alreadyhave) = + whenOpen isopen $ do + _ <- tryTakeMVar alreadyhave + Win32.findClose h +#endif + where + whenOpen :: IsOpen -> IO () -> IO () + whenOpen mv f = do + v <- tryTakeMVar mv + when (isJust v) f + +-- | Reads the next entry from the handle. Once the end of the directory +-- is reached, returns Nothing and automatically closes the handle. +readDirectory :: DirectoryHandle -> IO (Maybe FilePath) +#ifndef mingw32_HOST_OS +readDirectory hdl@(DirectoryHandle _ dirp) = do + e <- Posix.readDirStream dirp + if null e + then do + closeDirectory hdl + return Nothing + else return (Just e) +#else +readDirectory hdl@(DirectoryHandle _ h fdat mv) = do + -- If the MVar is full, then the filename in fdat has + -- not yet been returned. Otherwise, need to find the next + -- file. + r <- tryTakeMVar mv + case r of + Just () -> getfn + Nothing -> do + more <- Win32.findNextFile h fdat + if more + then getfn + else do + closeDirectory hdl + return Nothing + where + getfn = do + filename <- Win32.getFindDataFileName fdat + return (Just filename) +#endif + +-- | Like getDirectoryContents, but rather than buffering the whole +-- directory content in memory, lazily streams. +-- +-- This is like lazy readFile in that the handle to the directory remains +-- open until the whole list is consumed, or until the list is garbage +-- collected. So use with caution particularly when traversing directory +-- trees. +streamDirectoryContents :: FilePath -> IO [FilePath] +streamDirectoryContents d = openDirectory d >>= collect + where + collect hdl = readDirectory hdl >>= \case + Nothing -> return [] + Just f -> do + rest <- unsafeInterleaveIO (collect hdl) + return (f:rest) + +-- | True only when directory exists and contains nothing. +-- Throws exception if directory does not exist. +isDirectoryEmpty :: FilePath -> IO Bool +isDirectoryEmpty d = bracket (openDirectory d) closeDirectory check + where + check h = do + v <- readDirectory h + case v of + Nothing -> return True + Just f + | not (dirCruft f) -> return False + | otherwise -> check h diff --git a/Utility/DiskFree.hs b/Utility/DiskFree.hs new file mode 100644 index 0000000000..be4e823558 --- /dev/null +++ b/Utility/DiskFree.hs @@ -0,0 +1,38 @@ +{- disk free space checking shim + - + - Copyright 2016 Joey Hess + - + - License: BSD-2-clause + -} + +{-# OPTIONS_GHC -fno-warn-tabs #-} +{-# LANGUAGE CPP #-} + +module Utility.DiskFree ( + getDiskFree, + getDiskSize +) where + +#ifndef __ANDROID__ + +import System.DiskSpace +import Utility.Applicative +import Utility.Exception + +getDiskFree :: FilePath -> IO (Maybe Integer) +getDiskFree = catchMaybeIO . getAvailSpace + +getDiskSize :: FilePath -> IO (Maybe Integer) +getDiskSize = fmap diskTotal <$$> catchMaybeIO . getDiskUsage + +#else + +#warning Building without disk free space checking support + +getDiskFree :: FilePath -> IO (Maybe Integer) +getDiskFree _ = return Nothing + +getDiskSize :: FilePath -> IO (Maybe Integer) +getDiskSize _ = return Nothing + +#endif diff --git a/Utility/Dot.hs b/Utility/Dot.hs new file mode 100644 index 0000000000..e21915d327 --- /dev/null +++ b/Utility/Dot.hs @@ -0,0 +1,63 @@ +{- a simple graphviz / dot(1) digraph description generator library + - + - Copyright 2010 Joey Hess + - + - License: BSD-2-clause + -} + +module Utility.Dot where -- import qualified + +{- generates a graph description from a list of lines -} +graph :: [String] -> String +graph s = unlines $ [header] ++ map indent s ++ [footer] + where + header = "digraph map {" + footer= "}" + +{- a node in the graph -} +graphNode :: String -> String -> String +graphNode nodeid desc = label desc $ quote nodeid + +{- an edge between two nodes -} +graphEdge :: String -> String -> Maybe String -> String +graphEdge fromid toid desc = indent $ maybe edge (`label` edge) desc + where + edge = quote fromid ++ " -> " ++ quote toid + +{- adds a label to a node or edge -} +label :: String -> String -> String +label = attr "label" + +{- adds an attribute to a node or edge + - (can be called multiple times for multiple attributes) -} +attr :: String -> String -> String -> String +attr a v s = s ++ " [ " ++ a ++ "=" ++ quote v ++ " ]" + +{- fills a node with a color -} +fillColor :: String -> String -> String +fillColor color s = attr "fillcolor" color $ attr "style" "filled" s + +{- apply to graphNode to put the node in a labeled box -} +subGraph :: String -> String -> String -> String -> String +subGraph subid l color s = + "subgraph " ++ name ++ " {\n" ++ + ii setlabel ++ + ii setfilled ++ + ii setcolor ++ + ii s ++ + indent "}" + where + -- the "cluster_" makes dot draw a box + name = quote ("cluster_" ++ subid) + setlabel = "label=" ++ quote l + setfilled = "style=" ++ quote "filled" + setcolor = "fillcolor=" ++ quote color + ii x = indent (indent x) ++ "\n" + +indent ::String -> String +indent s = '\t' : s + +quote :: String -> String +quote s = "\"" ++ s' ++ "\"" + where + s' = filter (/= '"') s diff --git a/Utility/DottedVersion.hs b/Utility/DottedVersion.hs new file mode 100644 index 0000000000..3198b1ce27 --- /dev/null +++ b/Utility/DottedVersion.hs @@ -0,0 +1,38 @@ +{- dotted versions, such as 1.0.1 + - + - Copyright 2011-2014 Joey Hess + - + - License: BSD-2-clause + -} + +{-# OPTIONS_GHC -fno-warn-tabs #-} + +module Utility.DottedVersion where + +import Common + +data DottedVersion = DottedVersion String Integer + deriving (Eq) + +instance Ord DottedVersion where + compare (DottedVersion _ x) (DottedVersion _ y) = compare x y + +instance Show DottedVersion where + show (DottedVersion s _) = s + +{- To compare dotted versions like 1.7.7 and 1.8, they are normalized to + - a somewhat arbitrary integer representation. -} +normalize :: String -> DottedVersion +normalize v = DottedVersion v $ + sum $ mult 1 $ reverse $ extend precision $ take precision $ + map readi $ splitc '.' v + where + extend n l = l ++ replicate (n - length l) 0 + mult _ [] = [] + mult n (x:xs) = (n*x) : mult (n*10^width) xs + readi :: String -> Integer + readi s = case reads s of + ((x,_):_) -> x + _ -> 0 + precision = 10 -- number of segments of the version to compare + width = length "yyyymmddhhmmss" -- maximum width of a segment diff --git a/Utility/Env.hs b/Utility/Env.hs new file mode 100644 index 0000000000..dfebd98680 --- /dev/null +++ b/Utility/Env.hs @@ -0,0 +1,60 @@ +{- portable environment variables + - + - Copyright 2013 Joey Hess + - + - License: BSD-2-clause + -} + +{-# LANGUAGE CPP #-} +{-# OPTIONS_GHC -fno-warn-tabs #-} + +module Utility.Env where + +#ifdef mingw32_HOST_OS +import Utility.Exception +import Control.Applicative +import Data.Maybe +import Prelude +import qualified System.Environment as E +#else +import qualified System.Posix.Env as PE +#endif + +getEnv :: String -> IO (Maybe String) +#ifndef mingw32_HOST_OS +getEnv = PE.getEnv +#else +getEnv = catchMaybeIO . E.getEnv +#endif + +getEnvDefault :: String -> String -> IO String +#ifndef mingw32_HOST_OS +getEnvDefault = PE.getEnvDefault +#else +getEnvDefault var fallback = fromMaybe fallback <$> getEnv var +#endif + +getEnvironment :: IO [(String, String)] +#ifndef mingw32_HOST_OS +getEnvironment = PE.getEnvironment +#else +getEnvironment = E.getEnvironment +#endif + +{- Adds the environment variable to the input environment. If already + - present in the list, removes the old value. + - + - This does not really belong here, but Data.AssocList is for some reason + - buried inside hxt. + -} +addEntry :: Eq k => k -> v -> [(k, v)] -> [(k, v)] +addEntry k v l = ( (k,v) : ) $! delEntry k l + +addEntries :: Eq k => [(k, v)] -> [(k, v)] -> [(k, v)] +addEntries = foldr (.) id . map (uncurry addEntry) . reverse + +delEntry :: Eq k => k -> [(k, v)] -> [(k, v)] +delEntry _ [] = [] +delEntry k (x@(k1,_) : rest) + | k == k1 = rest + | otherwise = ( x : ) $! delEntry k rest diff --git a/Utility/Env/Basic.hs b/Utility/Env/Basic.hs new file mode 100644 index 0000000000..38295bea04 --- /dev/null +++ b/Utility/Env/Basic.hs @@ -0,0 +1,22 @@ +{- portable environment variables, without any dependencies + - + - Copyright 2013 Joey Hess + - + - License: BSD-2-clause + -} + +{-# OPTIONS_GHC -fno-warn-tabs #-} + +module Utility.Env.Basic where + +import Utility.Exception +import Control.Applicative +import Data.Maybe +import Prelude +import qualified System.Environment as E + +getEnv :: String -> IO (Maybe String) +getEnv = catchMaybeIO . E.getEnv + +getEnvDefault :: String -> String -> IO String +getEnvDefault var fallback = fromMaybe fallback <$> getEnv var diff --git a/Utility/Env/Set.hs b/Utility/Env/Set.hs new file mode 100644 index 0000000000..bd835e9787 --- /dev/null +++ b/Utility/Env/Set.hs @@ -0,0 +1,41 @@ +{- portable environment variables + - + - Copyright 2013 Joey Hess + - + - License: BSD-2-clause + -} + +{-# LANGUAGE CPP #-} + +module Utility.Env.Set where + +#ifdef mingw32_HOST_OS +import qualified System.Environment as E +import qualified System.SetEnv +import Utility.Env +#else +import qualified System.Posix.Env as PE +#endif + +{- Sets an environment variable. To overwrite an existing variable, + - overwrite must be True. + - + - On Windows, setting a variable to "" unsets it. -} +setEnv :: String -> String -> Bool -> IO () +#ifndef mingw32_HOST_OS +setEnv var val overwrite = PE.setEnv var val overwrite +#else +setEnv var val True = System.SetEnv.setEnv var val +setEnv var val False = do + r <- getEnv var + case r of + Nothing -> setEnv var val True + Just _ -> return () +#endif + +unsetEnv :: String -> IO () +#ifndef mingw32_HOST_OS +unsetEnv = PE.unsetEnv +#else +unsetEnv = System.SetEnv.unsetEnv +#endif diff --git a/Utility/Exception.hs b/Utility/Exception.hs new file mode 100644 index 0000000000..67c2e85d81 --- /dev/null +++ b/Utility/Exception.hs @@ -0,0 +1,129 @@ +{- Simple IO exception handling (and some more) + - + - Copyright 2011-2016 Joey Hess + - + - License: BSD-2-clause + -} + +{-# LANGUAGE CPP, ScopedTypeVariables #-} +{-# OPTIONS_GHC -fno-warn-tabs #-} + +module Utility.Exception ( + module X, + giveup, + catchBoolIO, + catchMaybeIO, + catchDefaultIO, + catchMsgIO, + catchIO, + tryIO, + bracketIO, + catchNonAsync, + tryNonAsync, + tryWhenExists, + catchIOErrorType, + IOErrorType(..), + catchPermissionDenied, +) where + +import Control.Monad.Catch as X hiding (Handler) +import qualified Control.Monad.Catch as M +import Control.Exception (IOException, AsyncException) +#ifdef MIN_VERSION_GLASGOW_HASKELL +#if MIN_VERSION_GLASGOW_HASKELL(7,10,0,0) +import Control.Exception (SomeAsyncException) +#endif +#endif +import Control.Monad +import Control.Monad.IO.Class (liftIO, MonadIO) +import System.IO.Error (isDoesNotExistError, ioeGetErrorType) +import GHC.IO.Exception (IOErrorType(..)) + +import Utility.Data + +{- Like error, this throws an exception. Unlike error, if this exception + - is not caught, it won't generate a backtrace. So use this for situations + - where there's a problem that the user is excpected to see in some + - circumstances. -} +giveup :: [Char] -> a +#ifdef MIN_VERSION_base +#if MIN_VERSION_base(4,9,0) +giveup = errorWithoutStackTrace +#else +giveup = error +#endif +#else +giveup = error +#endif + +{- Catches IO errors and returns a Bool -} +catchBoolIO :: MonadCatch m => m Bool -> m Bool +catchBoolIO = catchDefaultIO False + +{- Catches IO errors and returns a Maybe -} +catchMaybeIO :: MonadCatch m => m a -> m (Maybe a) +catchMaybeIO a = catchDefaultIO Nothing $ a >>= (return . Just) + +{- Catches IO errors and returns a default value. -} +catchDefaultIO :: MonadCatch m => a -> m a -> m a +catchDefaultIO def a = catchIO a (const $ return def) + +{- Catches IO errors and returns the error message. -} +catchMsgIO :: MonadCatch m => m a -> m (Either String a) +catchMsgIO a = do + v <- tryIO a + return $ either (Left . show) Right v + +{- catch specialized for IO errors only -} +catchIO :: MonadCatch m => m a -> (IOException -> m a) -> m a +catchIO = M.catch + +{- try specialized for IO errors only -} +tryIO :: MonadCatch m => m a -> m (Either IOException a) +tryIO = M.try + +{- bracket with setup and cleanup actions lifted to IO. + - + - Note that unlike catchIO and tryIO, this catches all exceptions. -} +bracketIO :: (MonadMask m, MonadIO m) => IO v -> (v -> IO b) -> (v -> m a) -> m a +bracketIO setup cleanup = bracket (liftIO setup) (liftIO . cleanup) + +{- Catches all exceptions except for async exceptions. + - This is often better to use than catching them all, so that + - ThreadKilled and UserInterrupt get through. + -} +catchNonAsync :: MonadCatch m => m a -> (SomeException -> m a) -> m a +catchNonAsync a onerr = a `catches` + [ M.Handler (\ (e :: AsyncException) -> throwM e) +#ifdef MIN_VERSION_GLASGOW_HASKELL +#if MIN_VERSION_GLASGOW_HASKELL(7,10,0,0) + , M.Handler (\ (e :: SomeAsyncException) -> throwM e) +#endif +#endif + , M.Handler (\ (e :: SomeException) -> onerr e) + ] + +tryNonAsync :: MonadCatch m => m a -> m (Either SomeException a) +tryNonAsync a = go `catchNonAsync` (return . Left) + where + go = do + v <- a + return (Right v) + +{- Catches only DoesNotExist exceptions, and lets all others through. -} +tryWhenExists :: MonadCatch m => m a -> m (Maybe a) +tryWhenExists a = do + v <- tryJust (guard . isDoesNotExistError) a + return (eitherToMaybe v) + +{- Catches only IO exceptions of a particular type. + - Ie, use HardwareFault to catch disk IO errors. -} +catchIOErrorType :: MonadCatch m => IOErrorType -> (IOException -> m a) -> m a -> m a +catchIOErrorType errtype onmatchingerr a = catchIO a onlymatching + where + onlymatching e + | ioeGetErrorType e == errtype = onmatchingerr e + | otherwise = throwM e + +catchPermissionDenied :: MonadCatch m => (IOException -> m a) -> m a -> m a +catchPermissionDenied = catchIOErrorType PermissionDenied diff --git a/Utility/FileMode.hs b/Utility/FileMode.hs new file mode 100644 index 0000000000..11764fd79b --- /dev/null +++ b/Utility/FileMode.hs @@ -0,0 +1,186 @@ +{- File mode utilities. + - + - Copyright 2010-2017 Joey Hess + - + - License: BSD-2-clause + -} + +{-# LANGUAGE CPP #-} + +module Utility.FileMode ( + module Utility.FileMode, + FileMode, +) where + +import System.IO +import Control.Monad +import System.PosixCompat.Types +import System.PosixCompat.Files +#ifndef mingw32_HOST_OS +import System.Posix.Files (symbolicLinkMode) +import Control.Monad.IO.Class (liftIO) +#endif +import Control.Monad.IO.Class (MonadIO) +import Foreign (complement) +import Control.Monad.Catch + +import Utility.Exception + +{- Applies a conversion function to a file's mode. -} +modifyFileMode :: FilePath -> (FileMode -> FileMode) -> IO () +modifyFileMode f convert = void $ modifyFileMode' f convert + +modifyFileMode' :: FilePath -> (FileMode -> FileMode) -> IO FileMode +modifyFileMode' f convert = do + s <- getFileStatus f + let old = fileMode s + let new = convert old + when (new /= old) $ + setFileMode f new + return old + +{- Runs an action after changing a file's mode, then restores the old mode. -} +withModifiedFileMode :: FilePath -> (FileMode -> FileMode) -> IO a -> IO a +withModifiedFileMode file convert a = bracket setup cleanup go + where + setup = modifyFileMode' file convert + cleanup oldmode = modifyFileMode file (const oldmode) + go _ = a + +{- Adds the specified FileModes to the input mode, leaving the rest + - unchanged. -} +addModes :: [FileMode] -> FileMode -> FileMode +addModes ms m = combineModes (m:ms) + +{- Removes the specified FileModes from the input mode. -} +removeModes :: [FileMode] -> FileMode -> FileMode +removeModes ms m = m `intersectFileModes` complement (combineModes ms) + +writeModes :: [FileMode] +writeModes = [ownerWriteMode, groupWriteMode, otherWriteMode] + +readModes :: [FileMode] +readModes = [ownerReadMode, groupReadMode, otherReadMode] + +executeModes :: [FileMode] +executeModes = [ownerExecuteMode, groupExecuteMode, otherExecuteMode] + +otherGroupModes :: [FileMode] +otherGroupModes = + [ groupReadMode, otherReadMode + , groupWriteMode, otherWriteMode + ] + +{- Removes the write bits from a file. -} +preventWrite :: FilePath -> IO () +preventWrite f = modifyFileMode f $ removeModes writeModes + +{- Turns a file's owner write bit back on. -} +allowWrite :: FilePath -> IO () +allowWrite f = modifyFileMode f $ addModes [ownerWriteMode] + +{- Turns a file's owner read bit back on. -} +allowRead :: FilePath -> IO () +allowRead f = modifyFileMode f $ addModes [ownerReadMode] + +{- Allows owner and group to read and write to a file. -} +groupSharedModes :: [FileMode] +groupSharedModes = + [ ownerWriteMode, groupWriteMode + , ownerReadMode, groupReadMode + ] + +groupWriteRead :: FilePath -> IO () +groupWriteRead f = modifyFileMode f $ addModes groupSharedModes + +checkMode :: FileMode -> FileMode -> Bool +checkMode checkfor mode = checkfor `intersectFileModes` mode == checkfor + +{- Checks if a file mode indicates it's a symlink. -} +isSymLink :: FileMode -> Bool +#ifdef mingw32_HOST_OS +isSymLink _ = False +#else +isSymLink = checkMode symbolicLinkMode +#endif + +{- Checks if a file has any executable bits set. -} +isExecutable :: FileMode -> Bool +isExecutable mode = combineModes executeModes `intersectFileModes` mode /= 0 + +{- Runs an action without that pesky umask influencing it, unless the + - passed FileMode is the standard one. -} +noUmask :: (MonadIO m, MonadMask m) => FileMode -> m a -> m a +#ifndef mingw32_HOST_OS +noUmask mode a + | mode == stdFileMode = a + | otherwise = withUmask nullFileMode a +#else +noUmask _ a = a +#endif + +withUmask :: (MonadIO m, MonadMask m) => FileMode -> m a -> m a +#ifndef mingw32_HOST_OS +withUmask umask a = bracket setup cleanup go + where + setup = liftIO $ setFileCreationMask umask + cleanup = liftIO . setFileCreationMask + go _ = a +#else +withUmask _ a = a +#endif + +getUmask :: IO FileMode +#ifndef mingw32_HOST_OS +getUmask = bracket setup cleanup return + where + setup = setFileCreationMask nullFileMode + cleanup = setFileCreationMask +#else +getUmask = return nullFileMode +#endif + +defaultFileMode :: IO FileMode +defaultFileMode = do + umask <- getUmask + return $ intersectFileModes (complement umask) stdFileMode + +combineModes :: [FileMode] -> FileMode +combineModes [] = 0 +combineModes [m] = m +combineModes (m:ms) = foldl unionFileModes m ms + +isSticky :: FileMode -> Bool +#ifdef mingw32_HOST_OS +isSticky _ = False +#else +isSticky = checkMode stickyMode + +stickyMode :: FileMode +stickyMode = 512 + +setSticky :: FilePath -> IO () +setSticky f = modifyFileMode f $ addModes [stickyMode] +#endif + +{- Writes a file, ensuring that its modes do not allow it to be read + - or written by anyone other than the current user, + - before any content is written. + - + - When possible, this is done using the umask. + - + - On a filesystem that does not support file permissions, this is the same + - as writeFile. + -} +writeFileProtected :: FilePath -> String -> IO () +writeFileProtected file content = writeFileProtected' file + (\h -> hPutStr h content) + +writeFileProtected' :: FilePath -> (Handle -> IO ()) -> IO () +writeFileProtected' file writer = protectedOutput $ + withFile file WriteMode $ \h -> do + void $ tryIO $ modifyFileMode file $ removeModes otherGroupModes + writer h + +protectedOutput :: IO a -> IO a +protectedOutput = withUmask 0o0077 diff --git a/Utility/FileSize.hs b/Utility/FileSize.hs new file mode 100644 index 0000000000..5f89cff2c0 --- /dev/null +++ b/Utility/FileSize.hs @@ -0,0 +1,37 @@ +{- File size. + - + - License: BSD-2-clause + -} + +{-# LANGUAGE CPP #-} + +module Utility.FileSize where + +import System.PosixCompat.Files +#ifdef mingw32_HOST_OS +import Control.Exception (bracket) +import System.IO +#endif + +type FileSize = Integer + +{- Gets the size of a file. + - + - This is better than using fileSize, because on Windows that returns a + - FileOffset which maxes out at 2 gb. + - See https://github.com/jystic/unix-compat/issues/16 + -} +getFileSize :: FilePath -> IO FileSize +#ifndef mingw32_HOST_OS +getFileSize f = fmap (fromIntegral . fileSize) (getFileStatus f) +#else +getFileSize f = bracket (openFile f ReadMode) hClose hFileSize +#endif + +{- Gets the size of the file, when its FileStatus is already known. -} +getFileSize' :: FilePath -> FileStatus -> IO FileSize +#ifndef mingw32_HOST_OS +getFileSize' _ s = return $ fromIntegral $ fileSize s +#else +getFileSize' f _ = getFileSize f +#endif diff --git a/Utility/FileSystemEncoding.hs b/Utility/FileSystemEncoding.hs new file mode 100644 index 0000000000..ca6e76857e --- /dev/null +++ b/Utility/FileSystemEncoding.hs @@ -0,0 +1,214 @@ +{- GHC File system encoding handling. + - + - Copyright 2012-2016 Joey Hess + - + - License: BSD-2-clause + -} + +{-# LANGUAGE CPP #-} +{-# OPTIONS_GHC -fno-warn-tabs #-} + +module Utility.FileSystemEncoding ( + useFileSystemEncoding, + fileEncoding, + withFilePath, + RawFilePath, + fromRawFilePath, + toRawFilePath, + decodeBS, + encodeBS, + decodeW8, + encodeW8, + encodeW8NUL, + decodeW8NUL, + truncateFilePath, + s2w8, + w82s, + c2w8, + w82c, +) where + +import qualified GHC.Foreign as GHC +import qualified GHC.IO.Encoding as Encoding +import Foreign.C +import System.IO +import System.IO.Unsafe +import Data.Word +import Data.List +import qualified Data.ByteString as S +import qualified Data.ByteString.Lazy as L +#ifdef mingw32_HOST_OS +import qualified Data.ByteString.Lazy.UTF8 as L8 +#endif + +import Utility.Exception +import Utility.Split + +{- Makes all subsequent Handles that are opened, as well as stdio Handles, + - use the filesystem encoding, instead of the encoding of the current + - locale. + - + - The filesystem encoding allows "arbitrary undecodable bytes to be + - round-tripped through it". This avoids encoded failures when data is not + - encoded matching the current locale. + - + - Note that code can still use hSetEncoding to change the encoding of a + - Handle. This only affects the default encoding. + -} +useFileSystemEncoding :: IO () +useFileSystemEncoding = do +#ifndef mingw32_HOST_OS + e <- Encoding.getFileSystemEncoding +#else + {- The file system encoding does not work well on Windows, + - and Windows only has utf FilePaths anyway. -} + let e = Encoding.utf8 +#endif + hSetEncoding stdin e + hSetEncoding stdout e + hSetEncoding stderr e + Encoding.setLocaleEncoding e + +fileEncoding :: Handle -> IO () +#ifndef mingw32_HOST_OS +fileEncoding h = hSetEncoding h =<< Encoding.getFileSystemEncoding +#else +fileEncoding h = hSetEncoding h Encoding.utf8 +#endif + +{- Marshal a Haskell FilePath into a NUL terminated C string using temporary + - storage. The FilePath is encoded using the filesystem encoding, + - reversing the decoding that should have been done when the FilePath + - was obtained. -} +withFilePath :: FilePath -> (CString -> IO a) -> IO a +withFilePath fp f = Encoding.getFileSystemEncoding + >>= \enc -> GHC.withCString enc fp f + +{- Encodes a FilePath into a String, applying the filesystem encoding. + - + - There are very few things it makes sense to do with such an encoded + - string. It's not a legal filename; it should not be displayed. + - So this function is not exported, but instead used by the few functions + - that can usefully consume it. + - + - This use of unsafePerformIO is belived to be safe; GHC's interface + - only allows doing this conversion with CStrings, and the CString buffer + - is allocated, used, and deallocated within the call, with no side + - effects. + - + - If the FilePath contains a value that is not legal in the filesystem + - encoding, rather than thowing an exception, it will be returned as-is. + -} +{-# NOINLINE _encodeFilePath #-} +_encodeFilePath :: FilePath -> String +_encodeFilePath fp = unsafePerformIO $ do + enc <- Encoding.getFileSystemEncoding + GHC.withCString enc fp (GHC.peekCString Encoding.char8) + `catchNonAsync` (\_ -> return fp) + +{- Decodes a ByteString into a FilePath, applying the filesystem encoding. -} +decodeBS :: L.ByteString -> FilePath +#ifndef mingw32_HOST_OS +decodeBS = encodeW8NUL . L.unpack +#else +{- On Windows, we assume that the ByteString is utf-8, since Windows + - only uses unicode for filenames. -} +decodeBS = L8.toString +#endif + +{- Encodes a FilePath into a ByteString, applying the filesystem encoding. -} +encodeBS :: FilePath -> L.ByteString +#ifndef mingw32_HOST_OS +encodeBS = L.pack . decodeW8NUL +#else +encodeBS = L8.fromString +#endif + +{- Recent versions of the unix package have this alias; defined here + - for backwards compatibility. -} +type RawFilePath = S.ByteString + +{- Note that the RawFilePath is assumed to never contain NUL, + - since filename's don't. This should only be used with actual + - RawFilePaths not arbitrary ByteString that may contain NUL. -} +fromRawFilePath :: RawFilePath -> FilePath +fromRawFilePath = encodeW8 . S.unpack + +{- Note that the FilePath is assumed to never contain NUL, + - since filename's don't. This should only be used with actual FilePaths + - not arbitrary String that may contain NUL. -} +toRawFilePath :: FilePath -> RawFilePath +toRawFilePath = S.pack . decodeW8 + +{- Converts a [Word8] to a FilePath, encoding using the filesystem encoding. + - + - w82c produces a String, which may contain Chars that are invalid + - unicode. From there, this is really a simple matter of applying the + - file system encoding, only complicated by GHC's interface to doing so. + - + - Note that the encoding stops at any NUL in the input. FilePaths + - do not normally contain embedded NUL, but Haskell Strings may. + -} +{-# NOINLINE encodeW8 #-} +encodeW8 :: [Word8] -> FilePath +encodeW8 w8 = unsafePerformIO $ do + enc <- Encoding.getFileSystemEncoding + GHC.withCString Encoding.char8 (w82s w8) $ GHC.peekCString enc + +{- Useful when you want the actual number of bytes that will be used to + - represent the FilePath on disk. -} +decodeW8 :: FilePath -> [Word8] +decodeW8 = s2w8 . _encodeFilePath + +{- Like encodeW8 and decodeW8, but NULs are passed through unchanged. -} +encodeW8NUL :: [Word8] -> FilePath +encodeW8NUL = intercalate [nul] . map encodeW8 . splitc (c2w8 nul) + where + nul = '\NUL' + +decodeW8NUL :: FilePath -> [Word8] +decodeW8NUL = intercalate [c2w8 nul] . map decodeW8 . splitc nul + where + nul = '\NUL' + +c2w8 :: Char -> Word8 +c2w8 = fromIntegral . fromEnum + +w82c :: Word8 -> Char +w82c = toEnum . fromIntegral + +s2w8 :: String -> [Word8] +s2w8 = map c2w8 + +w82s :: [Word8] -> String +w82s = map w82c + +{- Truncates a FilePath to the given number of bytes (or less), + - as represented on disk. + - + - Avoids returning an invalid part of a unicode byte sequence, at the + - cost of efficiency when running on a large FilePath. + -} +truncateFilePath :: Int -> FilePath -> FilePath +#ifndef mingw32_HOST_OS +truncateFilePath n = go . reverse + where + go f = + let bytes = decodeW8 f + in if length bytes <= n + then reverse f + else go (drop 1 f) +#else +{- On Windows, count the number of bytes used by each utf8 character. -} +truncateFilePath n = reverse . go [] n . L8.fromString + where + go coll cnt bs + | cnt <= 0 = coll + | otherwise = case L8.decode bs of + Just (c, x) | c /= L8.replacement_char -> + let x' = fromIntegral x + in if cnt - x' < 0 + then coll + else go (c:coll) (cnt - x') (L8.drop 1 bs) + _ -> coll +#endif diff --git a/Utility/Format.hs b/Utility/Format.hs new file mode 100644 index 0000000000..3670cd717a --- /dev/null +++ b/Utility/Format.hs @@ -0,0 +1,187 @@ +{- Formatted string handling. + - + - Copyright 2010, 2011 Joey Hess + - + - License: BSD-2-clause + -} + +module Utility.Format ( + Format, + gen, + format, + decode_c, + encode_c, + prop_encode_c_decode_c_roundtrip +) where + +import Text.Printf (printf) +import Data.Char (isAlphaNum, isOctDigit, isHexDigit, isSpace, chr, ord) +import Data.Maybe (fromMaybe) +import Data.Word (Word8) +import Data.List (isPrefixOf) +import qualified Codec.Binary.UTF8.String +import qualified Data.Map as M + +import Utility.PartialPrelude + +type FormatString = String + +{- A format consists of a list of fragments. -} +type Format = [Frag] + +{- A fragment is either a constant string, + - or a variable, with a justification. -} +data Frag = Const String | Var String Justify + deriving (Show) + +data Justify = LeftJustified Int | RightJustified Int | UnJustified + deriving (Show) + +type Variables = M.Map String String + +{- Expands a Format using some variables, generating a formatted string. + - This can be repeatedly called, efficiently. -} +format :: Format -> Variables -> String +format f vars = concatMap expand f + where + expand (Const s) = s + expand (Var name j) + | "escaped_" `isPrefixOf` name = + justify j $ encode_c_strict $ + getvar $ drop (length "escaped_") name + | otherwise = justify j $ getvar name + getvar name = fromMaybe "" $ M.lookup name vars + justify UnJustified s = s + justify (LeftJustified i) s = s ++ pad i s + justify (RightJustified i) s = pad i s ++ s + pad i s = take (i - length s) spaces + spaces = repeat ' ' + +{- Generates a Format that can be used to expand variables in a + - format string, such as "${foo} ${bar;10} ${baz;-10}\n" + - + - (This is the same type of format string used by dpkg-query.) + -} +gen :: FormatString -> Format +gen = filter (not . empty) . fuse [] . scan [] . decode_c + where + -- The Format is built up in reverse, for efficiency, + -- and can have many adjacent Consts. Fusing it fixes both + -- problems. + fuse f [] = f + fuse f (Const c1:Const c2:vs) = fuse f $ Const (c2++c1) : vs + fuse f (v:vs) = fuse (v:f) vs + + scan f (a:b:cs) + | a == '$' && b == '{' = invar f [] cs + | otherwise = scan (Const [a] : f ) (b:cs) + scan f v = Const v : f + + invar f var [] = Const (novar var) : f + invar f var (c:cs) + | c == '}' = foundvar f var UnJustified cs + | isAlphaNum c || c == '_' = invar f (c:var) cs + | c == ';' = inpad "" f var cs + | otherwise = scan ((Const $ novar $ c:var):f) cs + + inpad p f var (c:cs) + | c == '}' = foundvar f var (readjustify $ reverse p) cs + | otherwise = inpad (c:p) f var cs + inpad p f var [] = Const (novar $ p++";"++var) : f + readjustify = getjustify . fromMaybe 0 . readish + getjustify i + | i == 0 = UnJustified + | i < 0 = LeftJustified (-1 * i) + | otherwise = RightJustified i + novar v = "${" ++ reverse v + foundvar f v p = scan (Var (reverse v) p : f) + +empty :: Frag -> Bool +empty (Const "") = True +empty _ = False + +{- Decodes a C-style encoding, where \n is a newline (etc), + - \NNN is an octal encoded character, and \xNN is a hex encoded character. + -} +decode_c :: FormatString -> String +decode_c [] = [] +decode_c s = unescape ("", s) + where + e = '\\' + unescape (b, []) = b + -- look for escapes starting with '\' + unescape (b, v) = b ++ fst pair ++ unescape (handle $ snd pair) + where + pair = span (/= e) v + isescape x = x == e + handle (x:'x':n1:n2:rest) + | isescape x && allhex = (fromhex, rest) + where + allhex = isHexDigit n1 && isHexDigit n2 + fromhex = [chr $ readhex [n1, n2]] + readhex h = Prelude.read $ "0x" ++ h :: Int + handle (x:n1:n2:n3:rest) + | isescape x && alloctal = (fromoctal, rest) + where + alloctal = isOctDigit n1 && isOctDigit n2 && isOctDigit n3 + fromoctal = [chr $ readoctal [n1, n2, n3]] + readoctal o = Prelude.read $ "0o" ++ o :: Int + -- \C is used for a few special characters + handle (x:nc:rest) + | isescape x = ([echar nc], rest) + where + echar 'a' = '\a' + echar 'b' = '\b' + echar 'f' = '\f' + echar 'n' = '\n' + echar 'r' = '\r' + echar 't' = '\t' + echar 'v' = '\v' + echar a = a + handle n = ("", n) + +{- Inverse of decode_c. -} +encode_c :: String -> FormatString +encode_c = encode_c' (const False) + +{- Encodes more strictly, including whitespace. -} +encode_c_strict :: String -> FormatString +encode_c_strict = encode_c' isSpace + +encode_c' :: (Char -> Bool) -> String -> FormatString +encode_c' p = concatMap echar + where + e c = '\\' : [c] + echar '\a' = e 'a' + echar '\b' = e 'b' + echar '\f' = e 'f' + echar '\n' = e 'n' + echar '\r' = e 'r' + echar '\t' = e 't' + echar '\v' = e 'v' + echar '\\' = e '\\' + echar '"' = e '"' + echar c + | ord c < 0x20 = e_asc c -- low ascii + | ord c >= 256 = e_utf c -- unicode + | ord c > 0x7E = e_asc c -- high ascii + | p c = e_asc c -- unprintable ascii + | otherwise = [c] -- printable ascii + -- unicode character is decomposed to individual Word8s, + -- and each is shown in octal + e_utf c = showoctal =<< (Codec.Binary.UTF8.String.encode [c] :: [Word8]) + e_asc c = showoctal $ ord c + showoctal i = '\\' : printf "%03o" i + +{- For quickcheck. + - + - Encoding and then decoding roundtrips only when + - the string does not contain high unicode, because eg, + - both "\12345" and "\227\128\185" are encoded to "\343\200\271". + - + - This property papers over the problem, by only testing chars < 256. + -} +prop_encode_c_decode_c_roundtrip :: String -> Bool +prop_encode_c_decode_c_roundtrip s = s' == decode_c (encode_c s') + where + s' = filter (\c -> ord c < 256) s diff --git a/Utility/FreeDesktop.hs b/Utility/FreeDesktop.hs new file mode 100644 index 0000000000..896b89b991 --- /dev/null +++ b/Utility/FreeDesktop.hs @@ -0,0 +1,147 @@ +{- Freedesktop.org specifications + - + - http://standards.freedesktop.org/basedir-spec/latest/ + - http://standards.freedesktop.org/desktop-entry-spec/latest/ + - http://standards.freedesktop.org/menu-spec/latest/ + - http://standards.freedesktop.org/icon-theme-spec/latest/ + - + - Copyright 2012 Joey Hess + - + - License: BSD-2-clause + -} + +{-# OPTIONS_GHC -fno-warn-tabs #-} + +module Utility.FreeDesktop ( + DesktopEntry, + genDesktopEntry, + buildDesktopMenuFile, + writeDesktopMenuFile, + desktopMenuFilePath, + autoStartPath, + iconDir, + iconFilePath, + systemDataDir, + systemConfigDir, + userDataDir, + userConfigDir, + userDesktopDir +) where + +import Utility.Exception +import Utility.UserInfo +import Utility.Process + +import System.Environment +import System.FilePath +import System.Directory +import Data.List +import Data.Maybe +import Control.Applicative +import Prelude + +type DesktopEntry = [(Key, Value)] + +type Key = String + +data Value = StringV String | BoolV Bool | NumericV Float | ListV [Value] + +toString :: Value -> String +toString (StringV s) = s +toString (BoolV b) + | b = "true" + | otherwise = "false" +toString (NumericV f) = show f +toString (ListV l) + | null l = "" + | otherwise = (intercalate ";" $ map (concatMap escapesemi . toString) l) ++ ";" + where + escapesemi ';' = "\\;" + escapesemi c = [c] + +genDesktopEntry :: String -> String -> Bool -> FilePath -> Maybe String -> [String] -> DesktopEntry +genDesktopEntry name comment terminal program icon categories = catMaybes + [ item "Type" StringV "Application" + , item "Version" NumericV 1.0 + , item "Name" StringV name + , item "Comment" StringV comment + , item "Terminal" BoolV terminal + , item "Exec" StringV program + , maybe Nothing (item "Icon" StringV) icon + , item "Categories" ListV (map StringV categories) + ] + where + item x c y = Just (x, c y) + +buildDesktopMenuFile :: DesktopEntry -> String +buildDesktopMenuFile d = unlines ("[Desktop Entry]" : map keyvalue d) ++ "\n" + where + keyvalue (k, v) = k ++ "=" ++ toString v + +writeDesktopMenuFile :: DesktopEntry -> String -> IO () +writeDesktopMenuFile d file = do + createDirectoryIfMissing True (takeDirectory file) + writeFile file $ buildDesktopMenuFile d + +{- Path to use for a desktop menu file, in either the systemDataDir or + - the userDataDir -} +desktopMenuFilePath :: String -> FilePath -> FilePath +desktopMenuFilePath basename datadir = + datadir "applications" desktopfile basename + +{- Path to use for a desktop autostart file, in either the systemDataDir + - or the userDataDir -} +autoStartPath :: String -> FilePath -> FilePath +autoStartPath basename configdir = + configdir "autostart" desktopfile basename + +{- Base directory to install an icon file, in either the systemDataDir + - or the userDatadir. -} +iconDir :: FilePath -> FilePath +iconDir datadir = datadir "icons" "hicolor" + +{- Filename of an icon, given the iconDir to use. + - + - The resolution is something like "48x48" or "scalable". -} +iconFilePath :: FilePath -> String -> FilePath -> FilePath +iconFilePath file resolution icondir = + icondir resolution "apps" file + +desktopfile :: FilePath -> FilePath +desktopfile f = f ++ ".desktop" + +{- Directory used for installation of system wide data files.. -} +systemDataDir :: FilePath +systemDataDir = "/usr/share" + +{- Directory used for installation of system wide config files. -} +systemConfigDir :: FilePath +systemConfigDir = "/etc/xdg" + +{- Directory for user data files. -} +userDataDir :: IO FilePath +userDataDir = xdgEnvHome "DATA_HOME" ".local/share" + +{- Directory for user config files. -} +userConfigDir :: IO FilePath +userConfigDir = xdgEnvHome "CONFIG_HOME" ".config" + +{- Directory for the user's Desktop, may be localized. + - + - This is not looked up very fast; the config file is in a shell format + - that is best parsed by shell, so xdg-user-dir is used, with a fallback + - to ~/Desktop. -} +userDesktopDir :: IO FilePath +userDesktopDir = maybe fallback return =<< (parse <$> xdg_user_dir) + where + parse s = case lines <$> s of + Just (l:_) -> Just l + _ -> Nothing + xdg_user_dir = catchMaybeIO $ readProcess "xdg-user-dir" ["DESKTOP"] + fallback = xdgEnvHome "DESKTOP_DIR" "Desktop" + +xdgEnvHome :: String -> String -> IO String +xdgEnvHome envbase homedef = do + home <- myHomeDir + catchDefaultIO (home homedef) $ + getEnv $ "XDG_" ++ envbase diff --git a/Utility/Glob.hs b/Utility/Glob.hs new file mode 100644 index 0000000000..c7d535933d --- /dev/null +++ b/Utility/Glob.hs @@ -0,0 +1,63 @@ +{-# LANGUAGE PackageImports #-} + +{- file globbing + - + - Copyright 2014 Joey Hess + - + - License: BSD-2-clause + -} + +module Utility.Glob ( + Glob, + GlobCase(..), + compileGlob, + matchGlob +) where + +import Utility.Exception + +import "regex-tdfa" Text.Regex.TDFA +import "regex-tdfa" Text.Regex.TDFA.String +import Data.Char + +newtype Glob = Glob Regex + +data GlobCase = CaseSensative | CaseInsensative + +{- Compiles a glob to a regex, that can be repeatedly used. -} +compileGlob :: String -> GlobCase -> Glob +compileGlob glob globcase = Glob $ + case compile (defaultCompOpt {caseSensitive = casesentitive}) defaultExecOpt regex of + Right r -> r + Left _ -> giveup $ "failed to compile regex: " ++ regex + where + regex = '^' : wildToRegex glob ++ "$" + casesentitive = case globcase of + CaseSensative -> True + CaseInsensative -> False + +wildToRegex :: String -> String +wildToRegex = concat . go + where + go [] = [] + go ('*':xs) = ".*" : go xs + go ('?':xs) = "." : go xs + go ('[':'!':xs) = "[^" : inpat xs + go ('[':xs) = "[" : inpat xs + go (x:xs) + | isDigit x || isAlpha x = [x] : go xs + | otherwise = esc x : go xs + + inpat [] = [] + inpat (x:xs) = case x of + ']' -> "]" : go xs + '\\' -> esc x : inpat xs + _ -> [x] : inpat xs + + esc c = ['\\', c] + +matchGlob :: Glob -> String -> Bool +matchGlob (Glob regex) val = + case execute regex val of + Right (Just _) -> True + _ -> False diff --git a/Utility/Gpg.hs b/Utility/Gpg.hs new file mode 100644 index 0000000000..13bd5814cd --- /dev/null +++ b/Utility/Gpg.hs @@ -0,0 +1,420 @@ +{- gpg interface + - + - Copyright 2011 Joey Hess + - + - License: BSD-2-clause + -} + +{-# LANGUAGE CPP #-} + +module Utility.Gpg where + +import Common +import qualified BuildInfo +#ifndef mingw32_HOST_OS +import System.Posix.Types +import System.Posix.IO +import Utility.Env +import Utility.Env.Set +#else +import Utility.Tmp +#endif +import Utility.Tmp.Dir +import Utility.Format (decode_c) + +import Control.Concurrent +import Control.Monad.IO.Class +import qualified Data.Map as M +import Data.Char + +type KeyId = String + +newtype KeyIds = KeyIds { keyIds :: [KeyId] } + deriving (Ord, Eq) + +newtype GpgCmd = GpgCmd { unGpgCmd :: String } + +{- Get gpg command to use, Just what's specified or, if a specific gpg + - command was found at configure time, use it, or otherwise, "gpg". -} +mkGpgCmd :: Maybe FilePath -> GpgCmd +mkGpgCmd (Just c) = GpgCmd c +mkGpgCmd Nothing = GpgCmd (fromMaybe "gpg" BuildInfo.gpg) + +boolGpgCmd :: GpgCmd -> [CommandParam] -> IO Bool +boolGpgCmd (GpgCmd cmd) = boolSystem cmd + +-- Generate an argument list to asymetrically encrypt to the given recipients. +pkEncTo :: [String] -> [CommandParam] +pkEncTo = concatMap (\r -> [Param "--recipient", Param r]) + +stdParams :: [CommandParam] -> IO [String] +stdParams params = do +#ifndef mingw32_HOST_OS + -- Enable batch mode if GPG_AGENT_INFO is set, to avoid extraneous + -- gpg output about password prompts. GPG_BATCH is set by the test + -- suite for a similar reason. + e <- getEnv "GPG_AGENT_INFO" + b <- getEnv "GPG_BATCH" + let batch = if isNothing e && isNothing b + then [] + else ["--batch", "--no-tty", "--use-agent"] + return $ batch ++ defaults ++ toCommand params +#else + return $ defaults ++ toCommand params +#endif + where + -- Be quiet, even about checking the trustdb. + defaults = ["--quiet", "--trust-model", "always"] + +{- Usual options for symmetric / public-key encryption. -} +stdEncryptionParams :: Bool -> [CommandParam] +stdEncryptionParams symmetric = enc symmetric ++ + [ Param "--force-mdc" + , Param "--no-textmode" + ] + where + enc True = [ Param "--symmetric" ] + -- Force gpg to only encrypt to the specified recipients, not + -- configured defaults. Recipients are assumed to be specified in + -- elsewhere. + enc False = + [ Param "--encrypt" + , Param "--no-encrypt-to" + , Param "--no-default-recipient" + ] + +{- Runs gpg with some params and returns its stdout, strictly. -} +readStrict :: GpgCmd -> [CommandParam] -> IO String +readStrict (GpgCmd cmd) params = do + params' <- stdParams params + withHandle StdoutHandle createProcessSuccess (proc cmd params') $ \h -> do + hSetBinaryMode h True + hGetContentsStrict h + +{- Runs gpg, piping an input value to it, and returning its stdout, + - strictly. -} +pipeStrict :: GpgCmd -> [CommandParam] -> String -> IO String +pipeStrict (GpgCmd cmd) params input = do + params' <- stdParams params + withIOHandles createProcessSuccess (proc cmd params') $ \(to, from) -> do + hSetBinaryMode to True + hSetBinaryMode from True + hPutStr to input + hClose to + hGetContentsStrict from + +{- Runs gpg with some parameters. First sends it a passphrase (unless it + - is empty) via '--passphrase-fd'. Then runs a feeder action that is + - passed a handle and should write to it all the data to input to gpg. + - Finally, runs a reader action that is passed a handle to gpg's + - output. + - + - Runs gpg in batch mode; this is necessary to avoid gpg 2.x prompting for + - the passphrase. + - + - Note that to avoid deadlock with the cleanup stage, + - the reader must fully consume gpg's input before returning. -} +feedRead :: (MonadIO m, MonadMask m) => GpgCmd -> [CommandParam] -> String -> (Handle -> IO ()) -> (Handle -> m a) -> m a +feedRead cmd params passphrase feeder reader = do +#ifndef mingw32_HOST_OS + -- pipe the passphrase into gpg on a fd + (frompipe, topipe) <- liftIO System.Posix.IO.createPipe + liftIO $ void $ forkIO $ do + toh <- fdToHandle topipe + hPutStrLn toh passphrase + hClose toh + let Fd pfd = frompipe + let passphrasefd = [Param "--passphrase-fd", Param $ show pfd] + liftIO (closeFd frompipe) `after` go (passphrasefd ++ params) +#else + -- store the passphrase in a temp file for gpg + withTmpFile "gpg" $ \tmpfile h -> do + liftIO $ hPutStr h passphrase + liftIO $ hClose h + let passphrasefile = [Param "--passphrase-file", File tmpfile] + go $ passphrasefile ++ params +#endif + where + go params' = pipeLazy cmd params' feeder reader + +{- Like feedRead, but without passphrase. -} +pipeLazy :: (MonadIO m, MonadMask m) => GpgCmd -> [CommandParam] -> (Handle -> IO ()) -> (Handle -> m a) -> m a +pipeLazy (GpgCmd cmd) params feeder reader = do + params' <- liftIO $ stdParams $ Param "--batch" : params + let p = (proc cmd params') + { std_in = CreatePipe + , std_out = CreatePipe + , std_err = Inherit + } + bracket (setup p) (cleanup p) go + where + setup = liftIO . createProcess + cleanup p (_, _, _, pid) = liftIO $ forceSuccessProcess p pid + go p = do + let (to, from) = ioHandles p + liftIO $ void $ forkIO $ do + feeder to + hClose to + reader from + +{- Finds gpg public keys matching some string. (Could be an email address, + - a key id, or a name; See the section 'HOW TO SPECIFY A USER ID' of + - GnuPG's manpage.) -} +findPubKeys :: GpgCmd -> String -> IO KeyIds +findPubKeys cmd for + -- pass forced subkey through as-is rather than + -- looking up the master key. + | isForcedSubKey for = return $ KeyIds [for] + | otherwise = KeyIds . parse . lines <$> readStrict cmd params + where + params = [Param "--with-colons", Param "--list-public-keys", Param for] + parse = mapMaybe (keyIdField . splitc ':') + keyIdField ("pub":_:_:_:f:_) = Just f + keyIdField _ = Nothing + +{- "subkey!" tells gpg to force use of a specific subkey -} +isForcedSubKey :: String -> Bool +isForcedSubKey s = "!" `isSuffixOf` s && all isHexDigit (drop 1 (reverse s)) + +type UserId = String + +{- All of the user's secret keys, with their UserIds. + - Note that the UserId may be empty. -} +secretKeys :: GpgCmd -> IO (M.Map KeyId UserId) +secretKeys cmd = catchDefaultIO M.empty makemap + where + makemap = M.fromList . parse . lines <$> readStrict cmd params + params = [Param "--with-colons", Param "--list-secret-keys", Param "--fixed-list-mode"] + parse = extract [] Nothing . map (splitc ':') + extract c (Just keyid) (("uid":_:_:_:_:_:_:_:_:userid:_):rest) = + -- If the userid contains a ":" or a few other special + -- characters, gpg will hex-escape it. Use decode_c to + -- undo. + extract ((keyid, decode_c userid):c) Nothing rest + extract c (Just keyid) rest@(("sec":_):_) = + extract ((keyid, ""):c) Nothing rest + extract c (Just keyid) (_:rest) = + extract c (Just keyid) rest + extract c _ [] = c + extract c _ (("sec":_:_:_:keyid:_):rest) = + extract c (Just keyid) rest + extract c k (_:rest) = + extract c k rest + +type Passphrase = String +type Size = Int +data KeyType = Algo Int | DSA | RSA + +{- The maximum key size that gpg currently offers in its UI when + - making keys. -} +maxRecommendedKeySize :: Size +maxRecommendedKeySize = 4096 + +{- Generates a secret key using the experimental batch mode. + - The key is added to the secret key ring. + - Can take a very long time, depending on system entropy levels. + -} +genSecretKey :: GpgCmd -> KeyType -> Passphrase -> UserId -> Size -> IO () +genSecretKey (GpgCmd cmd) keytype passphrase userid keysize = + withHandle StdinHandle createProcessSuccess (proc cmd params) feeder + where + params = ["--batch", "--gen-key"] + feeder h = do + hPutStr h $ unlines $ catMaybes + [ Just $ "Key-Type: " ++ + case keytype of + DSA -> "DSA" + RSA -> "RSA" + Algo n -> show n + , Just $ "Key-Length: " ++ show keysize + , Just $ "Name-Real: " ++ userid + , Just "Expire-Date: 0" + , if null passphrase + then Nothing + else Just $ "Passphrase: " ++ passphrase + ] + hClose h + +{- Creates a block of high-quality random data suitable to use as a cipher. + - It is armored, to avoid newlines, since gpg only reads ciphers up to the + - first newline. -} +genRandom :: GpgCmd -> Bool -> Size -> IO String +genRandom cmd highQuality size = checksize <$> readStrict cmd params + where + params = + [ Param "--gen-random" + , Param "--armor" + , Param $ show randomquality + , Param $ show size + ] + + -- See http://www.gnupg.org/documentation/manuals/gcrypt/Quality-of-random-numbers.html + -- for the meaning of random quality levels. + -- The highest available is 2, which is the default for OpenPGP + -- key generation; Note that it uses the blocking PRNG /dev/random + -- on the Linux kernel, hence the running time may take a while. + randomquality :: Int + randomquality = if highQuality then 2 else 1 + + {- The size is the number of bytes of entropy desired; the data is + - base64 encoded, so needs 8 bits to represent every 6 bytes of + - entropy. -} + expectedlength = size * 8 `div` 6 + + checksize s = let len = length s in + if len >= expectedlength + then s + else shortread len + + shortread got = giveup $ unwords + [ "Not enough bytes returned from gpg", show params + , "(got", show got, "; expected", show expectedlength, ")" + ] + +{- A test key. This is provided pre-generated since generating a new gpg + - key is too much work (requires too much entropy) for a test suite to + - do. + - + - This key was generated with no exipiration date, and a small keysize. + - It has an empty passphrase. -} +testKeyId :: String +testKeyId = "129D6E0AC537B9C7" +testKey :: String +testKey = keyBlock True + [ "mI0ETvFAZgEEAKnqwWgZqznMhi1RQExem2H8t3OyKDxaNN3rBN8T6LWGGqAYV4wT" + , "r8In5tfsnz64bKpE1Qi68JURFwYmthgUL9N48tbODU8t3xzijdjLOSaTyqkH1ik6" + , "EyulfKN63xLne9i4F9XqNwpiZzukXYbNfHkDA2yb0M6g4UFKLY/fNzGXABEBAAG0" + , "W2luc2VjdXJlIHRlc3Qga2V5ICh0aGlzIGlzIGEgdGVzdCBrZXksIGRvIG5vdCB1" + , "c2UgZm9yIGFjdHVhbCBlbmNyeXB0aW9uKSA8dGVzdEBleGFtcGxlLmNvbT6IuAQT" + , "AQgAIgUCTvFAZgIbAwYLCQgHAwIGFQgCCQoLBBYCAwECHgECF4AACgkQEp1uCsU3" + , "uceQ9wP/YMd1f0+/eLLcwGXNBvGqyVhUOfAKknO1bMzGbqTsq9g60qegy/cldqee" + , "xVxNfy0VN//JeMfgdcb8+RgJYLoaMrTy9CcsUcFPxtwN9tcLmsM0V2/fNmmFBO9t" + , "v75iH+zeFbNg0/FbPkHiN6Mjw7P2gXYKQXgTvQZBWaphk8oQlBm4jQRO8UBmAQQA" + , "vdi50M/WRCkOLt2RsUve8V8brMWYTJBJTTWoHUeRr82v4NCdX7OE1BsoVK8cy/1Q" + , "Y+gLOH9PqinuGGNWRmPV2Ju/RYn5H7sdewXA8E80xWhc4phHRMJ8Jjhg/GVPamkJ" + , "8B5zeKF0jcLFl7cuVdOyQakhoeDWJd0CyfW837nmPtMAEQEAAYifBBgBCAAJBQJO" + , "8UBmAhsMAAoJEBKdbgrFN7nHclAEAKBShuP/toH03atDUQTbGE34CA4yEC9BVghi" + , "7kviOZlOz2s8xAfp/8AYsrECx1kgbXcA7JD902eNyp7NzXsdJX0zJwHqiuZW0XlD" + , "T8ZJu4qrYRYgl/790WPESZ+ValvHD/fqkR38RF4tfxvyoMhhp0roGmJY33GASIG/" + , "+gQkDF9/" + , "=1k11" + ] +testSecretKey :: String +testSecretKey = keyBlock False + [ "lQHYBE7xQGYBBACp6sFoGas5zIYtUUBMXpth/Ldzsig8WjTd6wTfE+i1hhqgGFeM" + , "E6/CJ+bX7J8+uGyqRNUIuvCVERcGJrYYFC/TePLWzg1PLd8c4o3Yyzkmk8qpB9Yp" + , "OhMrpXyjet8S53vYuBfV6jcKYmc7pF2GzXx5AwNsm9DOoOFBSi2P3zcxlwARAQAB" + , "AAP+PlRboxy7Z0XjuG70N6+CrzSddQbW5KCwgPFrxYsPk7sAPFcBkmRMVlv9vZpS" + , "phbP4bvDK+MrSntM51g+9uE802yhPhSWdmEbImiWfV2ucEhlLjD8gw7JDex9XZ0a" + , "EbTOV56wOsILuedX/jF/6i6IQzy5YmuMeo+ip1XQIsIN+80CAMyXepOBJgHw/gBD" + , "VdXh/l//vUkQQlhInQYwgkKbr0POCTdr8DM1qdKLcUD9Q1khgNRp0vZGGz+5xsrc" + , "KaODUlMCANSczLJcYWa8yPqB3S14yTe7qmtDiOS362+SeVUwQA7eQ06PcHLPsN+p" + , "NtWoHRfYazxrs+g0JvmoQOYdj4xSQy0CAMq7H/l6aeG1n8tpyMxqE7OvBOsvzdu5" + , "XS7I1AnwllVFgvTadVvqgf7b+hdYd91doeHDUGqSYO78UG1GgaBHJdylqrRbaW5z" + , "ZWN1cmUgdGVzdCBrZXkgKHRoaXMgaXMgYSB0ZXN0IGtleSwgZG8gbm90IHVzZSBm" + , "b3IgYWN0dWFsIGVuY3J5cHRpb24pIDx0ZXN0QGV4YW1wbGUuY29tPoi4BBMBCAAi" + , "BQJO8UBmAhsDBgsJCAcDAgYVCAIJCgsEFgIDAQIeAQIXgAAKCRASnW4KxTe5x5D3" + , "A/9gx3V/T794stzAZc0G8arJWFQ58AqSc7VszMZupOyr2DrSp6DL9yV2p57FXE1/" + , "LRU3/8l4x+B1xvz5GAlguhoytPL0JyxRwU/G3A321wuawzRXb982aYUE722/vmIf" + , "7N4Vs2DT8Vs+QeI3oyPDs/aBdgpBeBO9BkFZqmGTyhCUGZ0B2ARO8UBmAQQAvdi5" + , "0M/WRCkOLt2RsUve8V8brMWYTJBJTTWoHUeRr82v4NCdX7OE1BsoVK8cy/1QY+gL" + , "OH9PqinuGGNWRmPV2Ju/RYn5H7sdewXA8E80xWhc4phHRMJ8Jjhg/GVPamkJ8B5z" + , "eKF0jcLFl7cuVdOyQakhoeDWJd0CyfW837nmPtMAEQEAAQAD/RaVtFFTkF1udun7" + , "YOwzJvQXCO9OWHZvSdEeG4BUNdAwy4YWu0oZzKkBDBS6+lWILqqb/c28U4leUJ1l" + , "H+viz5svN9BWWyj/UpI00uwUo9JaIqalemwfLx6vsh69b54L1B4exLZHYGLvy/B3" + , "5T6bT0gpOE+53BRtKcJaOh/McQeJAgDTOCBU5weWOf6Bhqnw3Vr/gRfxntAz2okN" + , "gqz/h79mWbCc/lHKoYQSsrCdMiwziHSjXwvehUrdWE/AcomtW0vbAgDmGJqJ2fNr" + , "HvdsGx4Ld/BxyiZbCURJLUQ5CwzfHGIvBu9PMT8zM26NOSncaXRjxDna2Ggh8Uum" + , "ANEwbnhxFwZpAf9L9RLYIMTtAqwBjfXJg/lHcc2R+VP0hL5c8zFz+S+w7bRqINwL" + , "ff1JstKuHT2nJnu0ustK66by8YI3T0hDFFahnNCInwQYAQgACQUCTvFAZgIbDAAK" + , "CRASnW4KxTe5x3JQBACgUobj/7aB9N2rQ1EE2xhN+AgOMhAvQVYIYu5L4jmZTs9r" + , "PMQH6f/AGLKxAsdZIG13AOyQ/dNnjcqezc17HSV9MycB6ormVtF5Q0/GSbuKq2EW" + , "IJf+/dFjxEmflWpbxw/36pEd/EReLX8b8qDIYadK6BpiWN9xgEiBv/oEJAxffw==" + , "=LDsg" + ] +keyBlock :: Bool -> [String] -> String +keyBlock public ls = unlines + [ "-----BEGIN PGP "++t++" KEY BLOCK-----" + , "Version: GnuPG v1.4.11 (GNU/Linux)" + , "" + , unlines ls + , "-----END PGP "++t++" KEY BLOCK-----" + ] + where + t + | public = "PUBLIC" + | otherwise = "PRIVATE" + +#ifndef mingw32_HOST_OS +{- Runs an action using gpg in a test harness, in which gpg does + - not use ~/.gpg/, but a directory with the test key set up to be used. -} +testHarness :: GpgCmd -> IO a -> IO a +testHarness cmd a = withTmpDir "gpgtmpXXXXXX" $ \tmpdir -> + bracket (setup tmpdir) (cleanup tmpdir) (const a) + where + var = "GNUPGHOME" + + setup tmpdir = do + orig <- getEnv var + setEnv var tmpdir True + -- For some reason, recent gpg needs a trustdb to be set up. + _ <- pipeStrict cmd [Param "--trust-model", Param "auto", Param "--update-trustdb"] [] + _ <- pipeStrict cmd [Param "--import", Param "-q"] $ unlines + [testSecretKey, testKey] + return orig + + cleanup tmpdir orig = do + removeDirectoryRecursive tmpdir + -- gpg-agent may be shutting down at the same time + -- and may delete its socket at the same time as + -- we're trying to, causing an exception. Retrying + -- will deal with this race. + `catchIO` (\_ -> removeDirectoryRecursive tmpdir) + reset orig + reset (Just v) = setEnv var v True + reset _ = unsetEnv var + +{- Tests the test harness. -} +testTestHarness :: GpgCmd -> IO Bool +testTestHarness cmd = do + keys <- testHarness cmd $ findPubKeys cmd testKeyId + return $ KeyIds [testKeyId] == keys +#endif + +#ifndef mingw32_HOST_OS +checkEncryptionFile :: GpgCmd -> FilePath -> Maybe KeyIds -> IO Bool +checkEncryptionFile cmd filename keys = + checkGpgPackets cmd keys =<< readStrict cmd params + where + params = [Param "--list-packets", Param "--list-only", File filename] + +checkEncryptionStream :: GpgCmd -> String -> Maybe KeyIds -> IO Bool +checkEncryptionStream cmd stream keys = + checkGpgPackets cmd keys =<< pipeStrict cmd params stream + where + params = [Param "--list-packets", Param "--list-only"] + +{- Parses an OpenPGP packet list, and checks whether data is + - symmetrically encrypted (keys is Nothing), or encrypted to some + - public key(s). + - /!\ The key needs to be in the keyring! -} +checkGpgPackets :: GpgCmd -> Maybe KeyIds -> String -> IO Bool +checkGpgPackets cmd keys str = do + let (asym,sym) = partition (pubkeyEncPacket `isPrefixOf`) $ + filter (\l' -> pubkeyEncPacket `isPrefixOf` l' || + symkeyEncPacket `isPrefixOf` l') $ + takeWhile (/= ":encrypted data packet:") $ + lines str + case (keys,asym,sym) of + (Nothing, [], [_]) -> return True + (Just (KeyIds ks), ls, []) -> do + -- Find the master key associated with the + -- encryption subkey. + ks' <- concat <$> mapM (keyIds <$$> findPubKeys cmd) + [ k | k:"keyid":_ <- map (reverse . words) ls ] + return $ sort (nub ks) == sort (nub ks') + _ -> return False + where + pubkeyEncPacket = ":pubkey enc packet: " + symkeyEncPacket = ":symkey enc packet: " +#endif diff --git a/Utility/Hash.hs b/Utility/Hash.hs new file mode 100644 index 0000000000..3bbb0627b7 --- /dev/null +++ b/Utility/Hash.hs @@ -0,0 +1,182 @@ +{- Convenience wrapper around cryptonite's hashing. -} + +{-# LANGUAGE CPP #-} + +module Utility.Hash ( + sha1, + sha2_224, + sha2_256, + sha2_384, + sha2_512, + sha3_224, + sha3_256, + sha3_384, + sha3_512, + skein256, + skein512, +#if MIN_VERSION_cryptonite(0,23,0) + blake2s_160, + blake2s_224, + blake2s_256, + blake2sp_224, + blake2sp_256, + blake2b_160, + blake2b_224, + blake2b_256, + blake2b_384, + blake2b_512, +#endif + md5, + prop_hashes_stable, + Mac(..), + calcMac, + prop_mac_stable, +) where + +import qualified Data.ByteString.Lazy as L +import qualified Data.Text as T +import qualified Data.Text.Encoding as T +import qualified Data.ByteString as S +import "cryptonite" Crypto.MAC.HMAC +import "cryptonite" Crypto.Hash + +sha1 :: L.ByteString -> Digest SHA1 +sha1 = hashlazy + +sha2_224 :: L.ByteString -> Digest SHA224 +sha2_224 = hashlazy + +sha2_256 :: L.ByteString -> Digest SHA256 +sha2_256 = hashlazy + +sha2_384 :: L.ByteString -> Digest SHA384 +sha2_384 = hashlazy + +sha2_512 :: L.ByteString -> Digest SHA512 +sha2_512 = hashlazy + +sha3_224 :: L.ByteString -> Digest SHA3_224 +sha3_224 = hashlazy + +sha3_256 :: L.ByteString -> Digest SHA3_256 +sha3_256 = hashlazy + +sha3_384 :: L.ByteString -> Digest SHA3_384 +sha3_384 = hashlazy + +sha3_512 :: L.ByteString -> Digest SHA3_512 +sha3_512 = hashlazy + +skein256 :: L.ByteString -> Digest Skein256_256 +skein256 = hashlazy + +skein512 :: L.ByteString -> Digest Skein512_512 +skein512 = hashlazy + +#if MIN_VERSION_cryptonite(0,23,0) +blake2s_160 :: L.ByteString -> Digest Blake2s_160 +blake2s_160 = hashlazy + +blake2s_224 :: L.ByteString -> Digest Blake2s_224 +blake2s_224 = hashlazy + +blake2s_256 :: L.ByteString -> Digest Blake2s_256 +blake2s_256 = hashlazy + +blake2sp_224 :: L.ByteString -> Digest Blake2sp_224 +blake2sp_224 = hashlazy + +blake2sp_256 :: L.ByteString -> Digest Blake2sp_256 +blake2sp_256 = hashlazy + +blake2b_160 :: L.ByteString -> Digest Blake2b_160 +blake2b_160 = hashlazy + +blake2b_224 :: L.ByteString -> Digest Blake2b_224 +blake2b_224 = hashlazy + +blake2b_256 :: L.ByteString -> Digest Blake2b_256 +blake2b_256 = hashlazy + +blake2b_384 :: L.ByteString -> Digest Blake2b_384 +blake2b_384 = hashlazy + +blake2b_512 :: L.ByteString -> Digest Blake2b_512 +blake2b_512 = hashlazy +#endif + +-- Disabled because it's buggy with some versions of cryptonite. +--blake2bp_512 :: L.ByteString -> Digest Blake2bp_512 +--blake2bp_512 = hashlazy + +md5 :: L.ByteString -> Digest MD5 +md5 = hashlazy + +{- Check that all the hashes continue to hash the same. -} +prop_hashes_stable :: Bool +prop_hashes_stable = all (\(hasher, result) -> hasher foo == result) + [ (show . sha1, "0beec7b5ea3f0fdbc95d0dd47f3c5bc275da8a33") + , (show . sha2_224, "0808f64e60d58979fcb676c96ec938270dea42445aeefcd3a4e6f8db") + , (show . sha2_256, "2c26b46b68ffc68ff99b453c1d30413413422d706483bfa0f98a5e886266e7ae") + , (show . sha2_384, "98c11ffdfdd540676b1a137cb1a22b2a70350c9a44171d6b1180c6be5cbb2ee3f79d532c8a1dd9ef2e8e08e752a3babb") + , (show . sha2_512, "f7fbba6e0636f890e56fbbf3283e524c6fa3204ae298382d624741d0dc6638326e282c41be5e4254d8820772c5518a2c5a8c0c7f7eda19594a7eb539453e1ed7") + , (show . skein256, "a04efd9a0aeed6ede40fe5ce0d9361ae7b7d88b524aa19917b9315f1ecf00d33") + , (show . skein512, "fd8956898113510180aa4658e6c0ac85bd74fb47f4a4ba264a6b705d7a8e8526756e75aecda12cff4f1aca1a4c2830fbf57f458012a66b2b15a3dd7d251690a7") + , (show . sha3_224, "f4f6779e153c391bbd29c95e72b0708e39d9166c7cea51d1f10ef58a") + , (show . sha3_256, "76d3bc41c9f588f7fcd0d5bf4718f8f84b1c41b20882703100b9eb9413807c01") + , (show . sha3_384, "665551928d13b7d84ee02734502b018d896a0fb87eed5adb4c87ba91bbd6489410e11b0fbcc06ed7d0ebad559e5d3bb5") + , (show . sha3_512, "4bca2b137edc580fe50a88983ef860ebaca36c857b1f492839d6d7392452a63c82cbebc68e3b70a2a1480b4bb5d437a7cba6ecf9d89f9ff3ccd14cd6146ea7e7") +#if MIN_VERSION_cryptonite(0,23,0) + , (show . blake2s_160, "52fb63154f958a5c56864597273ea759e52c6f00") + , (show . blake2s_224, "9466668503ac415d87b8e1dfd7f348ab273ac1d5e4f774fced5fdb55") + , (show . blake2s_256, "08d6cad88075de8f192db097573d0e829411cd91eb6ec65e8fc16c017edfdb74") + , (show . blake2sp_224, "8492d356fbac99f046f55e114301f7596649cb590e5b083d1a19dcdb") + , (show . blake2sp_256, "050dc5786037ea72cb9ed9d0324afcab03c97ec02e8c47368fc5dfb4cf49d8c9") + , (show . blake2b_160, "983ceba2afea8694cc933336b27b907f90c53a88") + , (show . blake2b_224, "853986b3fe231d795261b4fb530e1a9188db41e460ec4ca59aafef78") + , (show . blake2b_256, "b8fe9f7f6255a6fa08f668ab632a8d081ad87983c77cd274e48ce450f0b349fd") + , (show . blake2b_384, "e629ee880953d32c8877e479e3b4cb0a4c9d5805e2b34c675b5a5863c4ad7d64bb2a9b8257fac9d82d289b3d39eb9cc2") + , (show . blake2b_512, "ca002330e69d3e6b84a46a56a6533fd79d51d97a3bb7cad6c2ff43b354185d6dc1e723fb3db4ae0737e120378424c714bb982d9dc5bbd7a0ab318240ddd18f8d") + --, (show . blake2bp_512, "") +#endif + , (show . md5, "acbd18db4cc2f85cedef654fccc4a4d8") + ] + where + foo = L.fromChunks [T.encodeUtf8 $ T.pack "foo"] + +{- File names are (client-side) MAC'ed on special remotes. + - The chosen MAC algorithm needs to be same for all files stored on the + - remote. + -} +data Mac = HmacSha1 | HmacSha224 | HmacSha256 | HmacSha384 | HmacSha512 + deriving (Eq) + +calcMac + :: Mac -- ^ MAC + -> S.ByteString -- ^ secret key + -> S.ByteString -- ^ message + -> String -- ^ MAC'ed message, in hexadecimal +calcMac mac = case mac of + HmacSha1 -> use SHA1 + HmacSha224 -> use SHA224 + HmacSha256 -> use SHA256 + HmacSha384 -> use SHA384 + HmacSha512 -> use SHA512 + where + use alg k m = show (hmacGetDigest (hmacWitnessAlg alg k m)) + + hmacWitnessAlg :: HashAlgorithm a => a -> S.ByteString -> S.ByteString -> HMAC a + hmacWitnessAlg _ = hmac + +-- Check that all the MACs continue to produce the same. +prop_mac_stable :: Bool +prop_mac_stable = all (\(mac, result) -> calcMac mac key msg == result) + [ (HmacSha1, "46b4ec586117154dacd49d664e5d63fdc88efb51") + , (HmacSha224, "4c1f774863acb63b7f6e9daa9b5c543fa0d5eccf61e3ffc3698eacdd") + , (HmacSha256, "f9320baf0249169e73850cd6156ded0106e2bb6ad8cab01b7bbbebe6d1065317") + , (HmacSha384, "3d10d391bee2364df2c55cf605759373e1b5a4ca9355d8f3fe42970471eca2e422a79271a0e857a69923839015877fc6") + , (HmacSha512, "114682914c5d017dfe59fdc804118b56a3a652a0b8870759cf9e792ed7426b08197076bf7d01640b1b0684df79e4b67e37485669e8ce98dbab60445f0db94fce") + ] + where + key = T.encodeUtf8 $ T.pack "foo" + msg = T.encodeUtf8 $ T.pack "bar" diff --git a/Utility/HtmlDetect.hs b/Utility/HtmlDetect.hs new file mode 100644 index 0000000000..bf0839e9ec --- /dev/null +++ b/Utility/HtmlDetect.hs @@ -0,0 +1,46 @@ +{- html detection + - + - Copyright 2017 Joey Hess + - + - License: BSD-2-clause + -} + +module Utility.HtmlDetect where + +import Text.HTML.TagSoup +import Data.Char +import qualified Data.ByteString.Lazy as B +import qualified Data.ByteString.Lazy.Char8 as B8 + +-- | Detect if a String is a html document. +-- +-- The document many not be valid, or may be truncated, and will +-- still be detected as html, as long as it starts with a +-- "" or "" tag. +-- +-- Html fragments like "

    this

    " are not detected as being html, +-- although some browsers may chose to render them as html. +isHtml :: String -> Bool +isHtml = evaluate . canonicalizeTags . parseTags . take htmlPrefixLength + where + evaluate (TagOpen "!DOCTYPE" ((t, _):_):_) = map toLower t == "html" + evaluate (TagOpen "html" _:_) = True + -- Allow some leading whitespace before the tag. + evaluate (TagText t:rest) + | all isSpace t = evaluate rest + | otherwise = False + -- It would be pretty weird to have a html comment before the html + -- tag, but easy to allow for. + evaluate (TagComment _:rest) = evaluate rest + evaluate _ = False + +-- | Detect if a ByteString is a html document. +isHtmlBs :: B.ByteString -> Bool +-- The encoding of the ByteString is not known, but isHtml only +-- looks for ascii strings. +isHtmlBs = isHtml . B8.unpack + +-- | How much of the beginning of a html document is needed to detect it. +-- (conservatively) +htmlPrefixLength :: Int +htmlPrefixLength = 8192 diff --git a/Utility/HttpManagerRestricted.hs b/Utility/HttpManagerRestricted.hs new file mode 100644 index 0000000000..00611b7b5c --- /dev/null +++ b/Utility/HttpManagerRestricted.hs @@ -0,0 +1,232 @@ +{- | Restricted Manager for http-client-tls + - + - Copyright 2018 Joey Hess + - + - Portions from http-client-tls Copyright (c) 2013 Michael Snoyman + - + - License: MIT + -} + +{-# LANGUAGE ScopedTypeVariables, DeriveDataTypeable, LambdaCase, PatternGuards #-} +{-# LANGUAGE CPP #-} + +module Utility.HttpManagerRestricted ( + restrictManagerSettings, + Restriction(..), + ConnectionRestricted(..), + addrConnectionRestricted, + ProxyRestricted(..), + IPAddrString, +) where + +import Network.HTTP.Client +import Network.HTTP.Client.Internal + (ManagerSettings(..), Connection, runProxyOverride, makeConnection) +import Network.Socket +import Network.BSD (getProtocolNumber) +import Control.Exception +import qualified Network.Connection as NC +import qualified Data.ByteString.UTF8 as BU +import Data.Default +import Data.Typeable +import Control.Applicative + +data Restriction = Restriction + { addressRestriction :: AddrInfo -> Maybe ConnectionRestricted + } + +-- | An exception used to indicate that the connection was restricted. +data ConnectionRestricted = ConnectionRestricted String + deriving (Show, Typeable) + +instance Exception ConnectionRestricted + +type IPAddrString = String + +-- | Constructs a ConnectionRestricted, passing the function a string +-- containing the IP address. +addrConnectionRestricted :: (IPAddrString -> String) -> AddrInfo -> ConnectionRestricted +addrConnectionRestricted mkmessage = + ConnectionRestricted . mkmessage . showSockAddress . addrAddress + +data ProxyRestricted = ProxyRestricted + deriving (Show) + +-- | Adjusts a ManagerSettings to enforce a Restriction. The restriction +-- will be checked each time a Request is made, and for each redirect +-- followed. +-- +-- The http proxy is also checked against the Restriction, and if +-- access to it is blocked, the http proxy will not be used. +restrictManagerSettings + :: Restriction + -> ManagerSettings + -> IO (ManagerSettings, Maybe ProxyRestricted) +restrictManagerSettings cfg base = restrictProxy cfg $ base + { managerRawConnection = restrictedRawConnection cfg + , managerTlsConnection = restrictedTlsConnection cfg +#if MIN_VERSION_http_client(0,5,0) + , managerWrapException = wrapOurExceptions base +#else + , managerWrapIOException = wrapOurExceptions base +#endif + } + +restrictProxy + :: Restriction + -> ManagerSettings + -> IO (ManagerSettings, Maybe ProxyRestricted) +restrictProxy cfg base = do + http_proxy_addr <- getproxyaddr False + https_proxy_addr <- getproxyaddr True + let (http_proxy, http_r) = mkproxy http_proxy_addr + let (https_proxy, https_r) = mkproxy https_proxy_addr + let ms = managerSetInsecureProxy http_proxy $ + managerSetSecureProxy https_proxy base + return (ms, http_r <|> https_r) + where + -- This does not use localhost because http-client may choose + -- not to use the proxy for localhost. + testnetip = "198.51.100.1" + dummyreq https = parseRequest_ $ + "http" ++ (if https then "s" else "") ++ "://" ++ testnetip + + getproxyaddr https = extractproxy >>= \case + Nothing -> return Nothing + Just p -> do + proto <- getProtocolNumber "tcp" + let serv = show (proxyPort p) + let hints = defaultHints + { addrFlags = [AI_ADDRCONFIG] + , addrProtocol = proto + , addrSocketType = Stream + } + let h = BU.toString $ proxyHost p + getAddrInfo (Just hints) (Just h) (Just serv) >>= \case + [] -> return Nothing + (addr:_) -> return $ Just addr + where + -- These contortions are necessary until this issue + -- is fixed: + -- https://github.com/snoyberg/http-client/issues/355 + extractproxy = do + let po = if https + then managerProxySecure base + else managerProxyInsecure base + f <- runProxyOverride po https + return $ proxy $ f $ dummyreq https + + mkproxy Nothing = (noProxy, Nothing) + mkproxy (Just proxyaddr) = case addressRestriction cfg proxyaddr of + Nothing -> (addrtoproxy (addrAddress proxyaddr), Nothing) + Just _ -> (noProxy, Just ProxyRestricted) + + addrtoproxy addr = case addr of + SockAddrInet pn _ -> mk pn + SockAddrInet6 pn _ _ _ -> mk pn + _ -> noProxy + where + mk pn = useProxy Network.HTTP.Client.Proxy + { proxyHost = BU.fromString (showSockAddress addr) + , proxyPort = fromIntegral pn + } + +#if MIN_VERSION_http_client(0,5,0) +wrapOurExceptions :: ManagerSettings -> Request -> IO a -> IO a +wrapOurExceptions base req a = + let wrapper se + | Just (_ :: ConnectionRestricted) <- fromException se = + toException $ HttpExceptionRequest req $ + InternalException se + | otherwise = se + in managerWrapException base req (handle (throwIO . wrapper) a) +#else +wrapOurExceptions :: ManagerSettings -> IO a -> IO a +wrapOurExceptions base a = + let wrapper se = case fromException se of + Just (_ :: ConnectionRestricted) -> + -- Not really a TLS exception, but there is no + -- way to put SomeException in the + -- InternalIOException this old version uses. + toException $ TlsException se + Nothing -> se + in managerWrapIOException base (handle (throwIO . wrapper) a) +#endif + +restrictedRawConnection :: Restriction -> IO (Maybe HostAddress -> String -> Int -> IO Connection) +restrictedRawConnection cfg = getConnection cfg Nothing + +restrictedTlsConnection :: Restriction -> IO (Maybe HostAddress -> String -> Int -> IO Connection) +restrictedTlsConnection cfg = getConnection cfg $ + -- It's not possible to access the TLSSettings + -- used in the base ManagerSettings. So, use the default + -- value, which is the same thing http-client-tls defaults to. + -- Since changing from the default settings can only make TLS + -- less secure, this is not a big problem. + Just def + + + +-- Based on Network.HTTP.Client.TLS.getTlsConnection. +-- +-- Checks the Restriction +-- +-- Does not support SOCKS. +getConnection :: Restriction -> Maybe NC.TLSSettings -> IO (Maybe HostAddress -> String -> Int -> IO Connection) +getConnection cfg tls = do + context <- NC.initConnectionContext + return $ \_ha h p -> bracketOnError + (go context h p) + NC.connectionClose + convertConnection + where + go context h p = do + let connparams = NC.ConnectionParams + { NC.connectionHostname = h + , NC.connectionPort = fromIntegral p + , NC.connectionUseSecure = tls + , NC.connectionUseSocks = Nothing -- unsupprted + } + proto <- getProtocolNumber "tcp" + let serv = show p + let hints = defaultHints + { addrFlags = [AI_ADDRCONFIG] + , addrProtocol = proto + , addrSocketType = Stream + } + addrs <- getAddrInfo (Just hints) (Just h) (Just serv) + bracketOnError + (firstSuccessful $ map tryToConnect addrs) + close + (\sock -> NC.connectFromSocket context sock connparams) + where + tryToConnect addr = case addressRestriction cfg addr of + Nothing -> bracketOnError + (socket (addrFamily addr) (addrSocketType addr) (addrProtocol addr)) + close + (\sock -> connect sock (addrAddress addr) >> return sock) + Just r -> throwIO r + firstSuccessful [] = throwIO $ NC.HostNotResolved h + firstSuccessful (a:as) = a `catch` \(e ::IOException) -> + case as of + [] -> throwIO e + _ -> firstSuccessful as + +-- Copied from Network.HTTP.Client.TLS, unfortunately not exported. +convertConnection :: NC.Connection -> IO Connection +convertConnection conn = makeConnection + (NC.connectionGetChunk conn) + (NC.connectionPut conn) + -- Closing an SSL connection gracefully involves writing/reading + -- on the socket. But when this is called the socket might be + -- already closed, and we get a @ResourceVanished@. + (NC.connectionClose conn `Control.Exception.catch` \(_ :: IOException) -> return ()) + +-- For ipv4 and ipv6, the string will contain only the IP address, +-- omitting the port that the Show instance includes. +showSockAddress :: SockAddr -> IPAddrString +showSockAddress a@(SockAddrInet _ _) = + takeWhile (/= ':') $ show a +showSockAddress a@(SockAddrInet6 _ _ _ _) = + takeWhile (/= ']') $ drop 1 $ show a +showSockAddress a = show a diff --git a/Utility/HumanNumber.hs b/Utility/HumanNumber.hs new file mode 100644 index 0000000000..c3fede95f6 --- /dev/null +++ b/Utility/HumanNumber.hs @@ -0,0 +1,21 @@ +{- numbers for humans + - + - Copyright 2012-2013 Joey Hess + - + - License: BSD-2-clause + -} + +module Utility.HumanNumber where + +{- Displays a fractional value as a string with a limited number + - of decimal digits. -} +showImprecise :: RealFrac a => Int -> a -> String +showImprecise precision n + | precision == 0 || remainder == 0 = show (round n :: Integer) + | otherwise = show int ++ "." ++ striptrailing0s (pad0s $ show remainder) + where + int :: Integer + (int, frac) = properFraction n + remainder = round (frac * 10 ^ precision) :: Integer + pad0s s = replicate (precision - length s) '0' ++ s + striptrailing0s = reverse . dropWhile (== '0') . reverse diff --git a/Utility/HumanTime.hs b/Utility/HumanTime.hs new file mode 100644 index 0000000000..fe7cf22a9a --- /dev/null +++ b/Utility/HumanTime.hs @@ -0,0 +1,102 @@ +{- Time for humans. + - + - Copyright 2012-2013 Joey Hess + - + - License: BSD-2-clause + -} + +module Utility.HumanTime ( + Duration(..), + durationSince, + durationToPOSIXTime, + durationToDays, + daysToDuration, + parseDuration, + fromDuration, + prop_duration_roundtrips +) where + +import Utility.PartialPrelude +import Utility.QuickCheck + +import qualified Data.Map as M +import Data.Time.Clock +import Data.Time.Clock.POSIX (POSIXTime) +import Data.Char +import Control.Applicative +import Prelude + +newtype Duration = Duration { durationSeconds :: Integer } + deriving (Eq, Ord, Read, Show) + +durationSince :: UTCTime -> IO Duration +durationSince pasttime = do + now <- getCurrentTime + return $ Duration $ round $ diffUTCTime now pasttime + +durationToPOSIXTime :: Duration -> POSIXTime +durationToPOSIXTime = fromIntegral . durationSeconds + +durationToDays :: Duration -> Integer +durationToDays d = durationSeconds d `div` dsecs + +daysToDuration :: Integer -> Duration +daysToDuration i = Duration $ i * dsecs + +{- Parses a human-input time duration, of the form "5h", "1m", "5h1m", etc -} +parseDuration :: Monad m => String -> m Duration +parseDuration = maybe parsefail (return . Duration) . go 0 + where + go n [] = return n + go n s = do + num <- readish s :: Maybe Integer + case dropWhile isDigit s of + (c:rest) -> do + u <- M.lookup c unitmap + go (n + num * u) rest + _ -> return $ n + num + parsefail = fail "duration parse error; expected eg \"5m\" or \"1h5m\"" + +fromDuration :: Duration -> String +fromDuration Duration { durationSeconds = d } + | d == 0 = "0s" + | otherwise = concatMap showunit $ go [] units d + where + showunit (u, n) + | n > 0 = show n ++ [u] + | otherwise = "" + go c [] _ = reverse c + go c ((u, n):us) v = + let (q,r) = v `quotRem` n + in go ((u, q):c) us r + +units :: [(Char, Integer)] +units = + [ ('y', ysecs) + , ('d', dsecs) + , ('h', hsecs) + , ('m', msecs) + , ('s', 1) + ] + +unitmap :: M.Map Char Integer +unitmap = M.fromList units + +ysecs :: Integer +ysecs = dsecs * 365 + +dsecs :: Integer +dsecs = hsecs * 24 + +hsecs :: Integer +hsecs = msecs * 60 + +msecs :: Integer +msecs = 60 + +-- Durations cannot be negative. +instance Arbitrary Duration where + arbitrary = Duration <$> nonNegative arbitrary + +prop_duration_roundtrips :: Duration -> Bool +prop_duration_roundtrips d = parseDuration (fromDuration d) == Just d diff --git a/Utility/IPAddress.hs b/Utility/IPAddress.hs new file mode 100644 index 0000000000..c180a5c459 --- /dev/null +++ b/Utility/IPAddress.hs @@ -0,0 +1,93 @@ +{- IP addresses + - + - Copyright 2018 Joey Hess + - + - License: BSD-2-clause + -} + +module Utility.IPAddress where + +import Utility.Exception + +import Network.Socket +import Data.Word +import Control.Applicative +import Prelude + +{- Check if an IP address is a loopback address; connecting to it + - may connect back to the local host. -} +isLoopbackAddress :: SockAddr -> Bool +isLoopbackAddress (SockAddrInet _ ipv4) = case hostAddressToTuple ipv4 of + -- localhost + (127,_,_,_) -> True + -- current network; functions equivilant to loopback + (0,_,_, _) -> True + _ -> False +isLoopbackAddress (SockAddrInet6 _ _ ipv6 _) = case hostAddress6ToTuple ipv6 of + -- localhost + (0,0,0,0,0,0,0,1) -> True + -- unspecified address; functions equivilant to loopback + (0,0,0,0,0,0,0,0) -> True + v -> maybe False + (isLoopbackAddress . SockAddrInet 0) + (embeddedIpv4 v) +isLoopbackAddress _ = False + +{- Check if an IP address is not globally routed, and is used + - for private communication, eg on a LAN. -} +isPrivateAddress :: SockAddr -> Bool +isPrivateAddress (SockAddrInet _ ipv4) = case hostAddressToTuple ipv4 of + -- lan + (10,_,_,_) -> True + (172,n,_,_) | n >= 16 && n <= 31 -> True -- 172.16.0.0/12 + (192,168,_,_) -> True + -- carrier-grade NAT + (100,n,0,0) | n >= 64 && n <= 127 -> True -- 100.64.0.0/10 + -- link-local + (169,254,_,_) -> True + _ -> False +isPrivateAddress (SockAddrInet6 _ _ ipv6 _) = case hostAddress6ToTuple ipv6 of + v@(n,_,_,_,_,_,_,_) + -- local to lan or private between orgs + | n >= 0xfc00 && n <= 0xfdff -> True -- fc00::/7 + -- link-local + | n >= 0xfe80 && n <= 0xfebf -> True -- fe80::/10 + | otherwise -> maybe False + (isPrivateAddress . SockAddrInet 0) + (embeddedIpv4 v) +isPrivateAddress _ = False + +embeddedIpv4 :: (Word16, Word16, Word16, Word16, Word16, Word16, Word16, Word16) -> Maybe HostAddress +embeddedIpv4 v = case v of + -- IPv4 mapped address (::ffff:0:0/96) + (0,0,0,0,0,0xffff,a,b) -> Just (toipv4 a b) + -- IPV4 translated address (::ffff:0:ipv4) + (0,0,0,0,0xffff,0,a,b) -> Just (toipv4 a b) + -- IPV4/IPV6 translation (64:ff9b::ipv4) + (0x64,0xff9b,0,0,0,0,a,b) -> Just (toipv4 a b) + _ -> Nothing + where + toipv4 a b = htonl $ fromIntegral a * (2^halfipv4bits) + fromIntegral b + halfipv4bits = 16 :: Word32 + +{- Given a string containing an IP address, make a function that will + - match that address in a SockAddr. Nothing when the address cannot be + - parsed. + - + - This does not involve any DNS lookups. + -} +makeAddressMatcher :: String -> IO (Maybe (SockAddr -> Bool)) +makeAddressMatcher s = go + <$> catchDefaultIO [] (getAddrInfo (Just hints) (Just s) Nothing) + where + hints = defaultHints + { addrSocketType = Stream + , addrFlags = [AI_NUMERICHOST] + } + + go [] = Nothing + go l = Just $ \sockaddr -> any (match sockaddr) (map addrAddress l) + + match (SockAddrInet _ a) (SockAddrInet _ b) = a == b + match (SockAddrInet6 _ _ a _) (SockAddrInet6 _ _ b _) = a == b + match _ _ = False diff --git a/Utility/InodeCache.hs b/Utility/InodeCache.hs new file mode 100644 index 0000000000..7e2d9992ad --- /dev/null +++ b/Utility/InodeCache.hs @@ -0,0 +1,225 @@ +{- Caching a file's inode, size, and modification time + - to see when it's changed. + - + - Copyright 2013-2014 Joey Hess + - + - License: BSD-2-clause + -} + +{-# LANGUAGE CPP #-} +{-# OPTIONS_GHC -fno-warn-orphans #-} + +module Utility.InodeCache ( + InodeCache, + InodeComparisonType(..), + + compareStrong, + compareWeak, + compareBy, + + readInodeCache, + showInodeCache, + genInodeCache, + toInodeCache, + + InodeCacheKey, + inodeCacheToKey, + inodeCacheToMtime, + + SentinalFile(..), + SentinalStatus(..), + TSDelta, + noTSDelta, + writeSentinalFile, + checkSentinalFile, + sentinalFileExists, + + prop_read_show_inodecache +) where + +import Common +import System.PosixCompat.Types +import Utility.QuickCheck + +#ifdef mingw32_HOST_OS +import Data.Word (Word64) +#endif + +data InodeCachePrim = InodeCachePrim FileID Integer EpochTime + deriving (Show, Eq, Ord) + +newtype InodeCache = InodeCache InodeCachePrim + deriving (Show) + +{- Inode caches can be compared in two different ways, either weakly + - or strongly. -} +data InodeComparisonType = Weakly | Strongly + deriving (Eq, Ord, Show) + +{- Strong comparison, including inodes. -} +compareStrong :: InodeCache -> InodeCache -> Bool +compareStrong (InodeCache x) (InodeCache y) = x == y + +{- Weak comparison of the inode caches, comparing the size and mtime, + - but not the actual inode. Useful when inodes have changed, perhaps + - due to some filesystems being remounted. + - + - The weak mtime comparison treats any mtimes that are within 2 seconds + - of one-anther as the same. This is because FAT has only a 2 second + - resolution. When a FAT filesystem is used on Linux, higher resolution + - timestamps are cached and used by Linux, but this is lost on unmount, + - so after a remount, the timestamp can appear to have changed. + -} +compareWeak :: InodeCache -> InodeCache -> Bool +compareWeak (InodeCache (InodeCachePrim _ size1 mtime1)) (InodeCache (InodeCachePrim _ size2 mtime2)) = + size1 == size2 && (abs (mtime1 - mtime2) < 2) + +compareBy :: InodeComparisonType -> InodeCache -> InodeCache -> Bool +compareBy Strongly = compareStrong +compareBy Weakly = compareWeak + +{- For use in a Map; it's determined at creation time whether this + - uses strong or weak comparison for Eq. -} +data InodeCacheKey = InodeCacheKey InodeComparisonType InodeCachePrim + deriving (Ord, Show) + +instance Eq InodeCacheKey where + (InodeCacheKey ctx x) == (InodeCacheKey cty y) = + compareBy (maximum [ctx,cty]) (InodeCache x ) (InodeCache y) + +inodeCacheToKey :: InodeComparisonType -> InodeCache -> InodeCacheKey +inodeCacheToKey ct (InodeCache prim) = InodeCacheKey ct prim + +inodeCacheToMtime :: InodeCache -> EpochTime +inodeCacheToMtime (InodeCache (InodeCachePrim _ _ mtime)) = mtime + +showInodeCache :: InodeCache -> String +showInodeCache (InodeCache (InodeCachePrim inode size mtime)) = unwords + [ show inode + , show size + , show mtime + ] + +readInodeCache :: String -> Maybe InodeCache +readInodeCache s = case words s of + (inode:size:mtime:_) -> + let prim = InodeCachePrim + <$> readish inode + <*> readish size + <*> readish mtime + in InodeCache <$> prim + _ -> Nothing + +genInodeCache :: FilePath -> TSDelta -> IO (Maybe InodeCache) +genInodeCache f delta = catchDefaultIO Nothing $ + toInodeCache delta f =<< getFileStatus f + +toInodeCache :: TSDelta -> FilePath -> FileStatus -> IO (Maybe InodeCache) +toInodeCache (TSDelta getdelta) f s + | isRegularFile s = do + delta <- getdelta + sz <- getFileSize' f s + return $ Just $ InodeCache $ InodeCachePrim + (fileID s) + sz + (modificationTime s + delta) + | otherwise = pure Nothing + +{- Some filesystem get new random inodes each time they are mounted. + - To detect this and other problems, a sentinal file can be created. + - Its InodeCache at the time of its creation is written to the cache file, + - so changes can later be detected. -} +data SentinalFile = SentinalFile + { sentinalFile :: FilePath + , sentinalCacheFile :: FilePath + } + deriving (Show) + +{- On Windows, the mtime of a file appears to change when the time zone is + - changed. To deal with this, a TSDelta can be used; the delta is added to + - the mtime when generating an InodeCache. The current delta can be found + - by looking at the SentinalFile. Effectively, this makes all InodeCaches + - use the same time zone that was in use when the sential file was + - originally written. -} +newtype TSDelta = TSDelta (IO EpochTime) + +noTSDelta :: TSDelta +noTSDelta = TSDelta (pure 0) + +writeSentinalFile :: SentinalFile -> IO () +writeSentinalFile s = do + writeFile (sentinalFile s) "" + maybe noop (writeFile (sentinalCacheFile s) . showInodeCache) + =<< genInodeCache (sentinalFile s) noTSDelta + +data SentinalStatus = SentinalStatus + { sentinalInodesChanged :: Bool + , sentinalTSDelta :: TSDelta + } + +{- Checks if the InodeCache of the sentinal file is the same + - as it was when it was originally created. + - + - On Windows, time stamp differences are ignored, since they change + - with the timezone. + - + - When the sential file does not exist, InodeCaches canot reliably be + - compared, so the assumption is that there is has been a change. + -} +checkSentinalFile :: SentinalFile -> IO SentinalStatus +checkSentinalFile s = do + mold <- loadoldcache + case mold of + Nothing -> return dummy + (Just old) -> do + mnew <- gennewcache + case mnew of + Nothing -> return dummy + Just new -> return $ calc old new + where + loadoldcache = catchDefaultIO Nothing $ + readInodeCache <$> readFile (sentinalCacheFile s) + gennewcache = genInodeCache (sentinalFile s) noTSDelta + calc (InodeCache (InodeCachePrim oldinode oldsize oldmtime)) (InodeCache (InodeCachePrim newinode newsize newmtime)) = + SentinalStatus (not unchanged) tsdelta + where +#ifdef mingw32_HOST_OS + -- Since mtime can appear to change when the time zone is + -- changed in windows, we cannot look at the mtime for the + -- sentinal file. + unchanged = oldinode == newinode && oldsize == newsize && (newmtime == newmtime) + tsdelta = TSDelta $ do + -- Run when generating an InodeCache, + -- to get the current delta. + mnew <- gennewcache + return $ case mnew of + Just (InodeCache (InodeCachePrim _ _ currmtime)) -> + oldmtime - currmtime + Nothing -> 0 +#else + unchanged = oldinode == newinode && oldsize == newsize && oldmtime == newmtime + tsdelta = noTSDelta +#endif + dummy = SentinalStatus True noTSDelta + +sentinalFileExists :: SentinalFile -> IO Bool +sentinalFileExists s = allM doesFileExist [sentinalCacheFile s, sentinalFile s] + +instance Arbitrary InodeCache where + arbitrary = + let prim = InodeCachePrim + <$> arbitrary + <*> arbitrary + -- timestamp cannot be negative + <*> (abs . fromInteger <$> arbitrary) + in InodeCache <$> prim + +#ifdef mingw32_HOST_OS +instance Arbitrary FileID where + arbitrary = fromIntegral <$> (arbitrary :: Gen Word64) +#endif + +prop_read_show_inodecache :: InodeCache -> Bool +prop_read_show_inodecache c = case readInodeCache (showInodeCache c) of + Nothing -> False + Just c' -> compareStrong c c' diff --git a/Utility/LinuxMkLibs.hs b/Utility/LinuxMkLibs.hs new file mode 100644 index 0000000000..15f82fd18e --- /dev/null +++ b/Utility/LinuxMkLibs.hs @@ -0,0 +1,62 @@ +{- Linux library copier and binary shimmer + - + - Copyright 2013 Joey Hess + - + - License: BSD-2-clause + -} + +module Utility.LinuxMkLibs where + +import Utility.PartialPrelude +import Utility.Directory +import Utility.Process +import Utility.Monad +import Utility.Path +import Utility.Split + +import Data.Maybe +import System.FilePath +import System.Posix.Files +import Data.Char +import Control.Monad.IfElse +import Control.Applicative +import Prelude + +{- Installs a library. If the library is a symlink to another file, + - install the file it links to, and update the symlink to be relative. -} +installLib :: (FilePath -> FilePath -> IO ()) -> FilePath -> FilePath -> IO (Maybe FilePath) +installLib installfile top lib = ifM (doesFileExist lib) + ( do + installfile top lib + checksymlink lib + return $ Just $ parentDir lib + , return Nothing + ) + where + checksymlink f = whenM (isSymbolicLink <$> getSymbolicLinkStatus (inTop top f)) $ do + l <- readSymbolicLink (inTop top f) + let absl = absPathFrom (parentDir f) l + target <- relPathDirToFile (takeDirectory f) absl + installfile top absl + nukeFile (top ++ f) + createSymbolicLink target (inTop top f) + checksymlink absl + +-- Note that f is not relative, so cannot use +inTop :: FilePath -> FilePath -> FilePath +inTop top f = top ++ f + +{- Parse ldd output, getting all the libraries that the input files + - link to. Note that some of the libraries may not exist + - (eg, linux-vdso.so) -} +parseLdd :: String -> [FilePath] +parseLdd = mapMaybe (getlib . dropWhile isSpace) . lines + where + getlib l = headMaybe . words =<< lastMaybe (split " => " l) + +{- Get all glibc libs and other support files, including gconv files + - + - XXX Debian specific. -} +glibcLibs :: IO [FilePath] +glibcLibs = lines <$> readProcess "sh" + ["-c", "dpkg -L libc6:$(dpkg --print-architecture) libgcc1:$(dpkg --print-architecture) | egrep '\\.so|gconv'"] diff --git a/Utility/LockFile.hs b/Utility/LockFile.hs new file mode 100644 index 0000000000..1d924fc1e0 --- /dev/null +++ b/Utility/LockFile.hs @@ -0,0 +1,23 @@ +{- Lock files + - + - Posix and Windows lock files are extremely different. + - This module does *not* attempt to be a portability shim, it just exposes + - the native locking of the OS. + - + - Posix fcntl locks have some gotchas. So, consider using + - Utility.LockPool instead of using this module directly. + - + - Copyright 2014 Joey Hess + - + - License: BSD-2-clause + -} + +{-# LANGUAGE CPP #-} + +module Utility.LockFile (module X) where + +#ifndef mingw32_HOST_OS +import Utility.LockFile.Posix as X +#else +import Utility.LockFile.Windows as X +#endif diff --git a/Utility/LockFile/LockStatus.hs b/Utility/LockFile/LockStatus.hs new file mode 100644 index 0000000000..3f466c1255 --- /dev/null +++ b/Utility/LockFile/LockStatus.hs @@ -0,0 +1,13 @@ +{- LockStatus type + - + - Copyright 2014 Joey Hess + - + - License: BSD-2-clause + -} + +module Utility.LockFile.LockStatus where + +import System.Posix + +data LockStatus = StatusUnLocked | StatusLockedBy ProcessID | StatusNoLockFile + deriving (Eq) diff --git a/Utility/LockFile/PidLock.hs b/Utility/LockFile/PidLock.hs new file mode 100644 index 0000000000..3a46ecdf82 --- /dev/null +++ b/Utility/LockFile/PidLock.hs @@ -0,0 +1,262 @@ +{- pid-based lock files + - + - Copyright 2015 Joey Hess + - + - License: BSD-2-clause + -} + +module Utility.LockFile.PidLock ( + LockHandle, + tryLock, + waitLock, + dropLock, + LockStatus(..), + getLockStatus, + checkLocked, + checkSaneLock, +) where + +import Utility.PartialPrelude +import Utility.Exception +import Utility.Applicative +import Utility.Directory +import Utility.Monad +import Utility.Path +import Utility.FileMode +import Utility.LockFile.LockStatus +import Utility.ThreadScheduler +import Utility.Hash +import Utility.FileSystemEncoding +import qualified Utility.LockFile.Posix as Posix + +import System.IO +import System.Posix +import Data.Maybe +import Data.List +import Network.BSD +import System.FilePath +import Control.Applicative +import Prelude + +type LockFile = FilePath + +data LockHandle = LockHandle LockFile FileStatus SideLockHandle + +type SideLockHandle = Maybe (LockFile, Posix.LockHandle) + +data PidLock = PidLock + { lockingPid :: ProcessID + , lockingHost :: HostName + } + deriving (Eq, Read, Show) + +mkPidLock :: IO PidLock +mkPidLock = PidLock + <$> getProcessID + <*> getHostName + +readPidLock :: LockFile -> IO (Maybe PidLock) +readPidLock lockfile = (readish =<<) <$> catchMaybeIO (readFile lockfile) + +-- To avoid races when taking over a stale pid lock, a side lock is used. +-- This is a regular posix exclusive lock. +trySideLock :: LockFile -> (SideLockHandle -> IO a) -> IO a +trySideLock lockfile a = do + sidelock <- sideLockFile lockfile + mlck <- catchDefaultIO Nothing $ + withUmask nullFileMode $ + Posix.tryLockExclusive (Just mode) sidelock + -- Check the lock we just took, in case we opened a side lock file + -- belonging to another process that will have since deleted it. + case mlck of + Nothing -> a Nothing + Just lck -> ifM (Posix.checkSaneLock sidelock lck) + ( a (Just (sidelock, lck)) + , a Nothing + ) + where + -- Let all users write to the lock file in /dev/shm or /tmp, + -- so that other users can reuse it to take the lock. + -- Since /dev/shm and /tmp are sticky dirs, a user cannot + -- delete another user's lock file there, so could not + -- delete a stale lock. + mode = combineModes (readModes ++ writeModes) + +dropSideLock :: SideLockHandle -> IO () +dropSideLock Nothing = return () +dropSideLock (Just (f, h)) = do + -- Delete the file first, to ensure that any process that is trying + -- to take the side lock will only succeed once the file is + -- deleted, and so will be able to immediately see that it's taken + -- a stale lock. + _ <- tryIO $ removeFile f + Posix.dropLock h + +-- The side lock is put in /dev/shm. This will work on most any +-- Linux system, even if its whole root filesystem doesn't support posix +-- locks. /tmp is used as a fallback. +sideLockFile :: LockFile -> IO LockFile +sideLockFile lockfile = do + f <- absPath lockfile + let base = intercalate "_" (splitDirectories (makeRelative "/" f)) + let shortbase = reverse $ take 32 $ reverse base + let md5sum = if base == shortbase + then "" + else show (md5 (encodeBS base)) + dir <- ifM (doesDirectoryExist "/dev/shm") + ( return "/dev/shm" + , return "/tmp" + ) + return $ dir md5sum ++ shortbase ++ ".lck" + +-- | Tries to take a lock; does not block when the lock is already held. +-- +-- Note that stale locks are automatically detected and broken. +-- However, if the lock file is on a networked file system, and was +-- created on a different host than the current host (determined by hostname), +-- this can't be done and stale locks may persist. +tryLock :: LockFile -> IO (Maybe LockHandle) +tryLock lockfile = trySideLock lockfile $ \sidelock -> do + lockfile' <- absPath lockfile + (tmp, h) <- openTempFile (takeDirectory lockfile') "locktmp" + setFileMode tmp (combineModes readModes) + hPutStr h . show =<< mkPidLock + hClose h + let failedlock st = do + dropLock $ LockHandle tmp st sidelock + nukeFile tmp + return Nothing + let tooklock st = return $ Just $ LockHandle lockfile' st sidelock + ifM (linkToLock sidelock tmp lockfile') + ( do + nukeFile tmp + -- May not have made a hard link, so stat + -- the lockfile + lckst <- getFileStatus lockfile' + tooklock lckst + , do + v <- readPidLock lockfile' + hn <- getHostName + tmpst <- getFileStatus tmp + case v of + Just pl | isJust sidelock && hn == lockingHost pl -> do + -- Since we have the sidelock, + -- and are on the same host that + -- the pidlock was taken on, + -- we know that the pidlock is + -- stale, and can take it over. + rename tmp lockfile' + tooklock tmpst + _ -> failedlock tmpst + ) + +-- Linux's open(2) man page recommends linking a pid lock into place, +-- as the most portable atomic operation that will fail if +-- it already exists. +-- +-- open(2) suggests that link can sometimes appear to fail +-- on NFS but have actually succeeded, and the way to find out is to stat +-- the file and check its link count etc. +-- +-- However, not all filesystems support hard links. So, first probe +-- to see if they are supported. If not, use open with O_EXCL. +linkToLock :: SideLockHandle -> FilePath -> FilePath -> IO Bool +linkToLock Nothing _ _ = return False +linkToLock (Just _) src dest = do + let probe = src ++ ".lnk" + v <- tryIO $ createLink src probe + nukeFile probe + case v of + Right _ -> do + _ <- tryIO $ createLink src dest + ifM (catchBoolIO checklinked) + ( catchBoolIO $ not <$> checkInsaneLustre dest + , return False + ) + Left _ -> catchBoolIO $ do + fd <- openFd dest WriteOnly + (Just $ combineModes readModes) + (defaultFileFlags {exclusive = True}) + h <- fdToHandle fd + readFile src >>= hPutStr h + hClose h + return True + where + checklinked = do + x <- getSymbolicLinkStatus src + y <- getSymbolicLinkStatus dest + return $ and + [ deviceID x == deviceID y + , fileID x == fileID y + , fileMode x == fileMode y + , fileOwner x == fileOwner y + , fileGroup x == fileGroup y + , fileSize x == fileSize y + , modificationTime x == modificationTime y + , isRegularFile x == isRegularFile y + , linkCount x == linkCount y + , linkCount x == 2 + ] + +-- On a Lustre filesystem, link has been observed to incorrectly *succeed*, +-- despite the dest already existing. A subsequent stat of the dest +-- looked like it had been replaced with the src. The process proceeded to +-- run and then deleted the dest, and after the process was done, the +-- original file was observed to still be in place. +-- +-- We can detect this insanity by getting the directory contents after +-- making the link, and checking to see if 2 copies of the dest file, +-- with the SAME FILENAME exist. +checkInsaneLustre :: FilePath -> IO Bool +checkInsaneLustre dest = do + fs <- dirContents (takeDirectory dest) + case length (filter (== dest) fs) of + 1 -> return False -- whew! + 0 -> return True -- wtf? + _ -> do + -- Try to clean up the extra copy we made + -- that has the same name. Egads. + _ <- tryIO $ removeFile dest + return True + +-- | Waits as necessary to take a lock. +-- +-- Uses a 1 second wait-loop. +-- +-- May wait until timeout if the lock file is stale and is on a network file +-- system, or on a system where the side lock cannot be taken. +waitLock :: Seconds -> LockFile -> IO LockHandle +waitLock (Seconds timeout) lockfile = go timeout + where + go n + | n > 0 = maybe (threadDelaySeconds (Seconds 1) >> go (pred n)) return + =<< tryLock lockfile + | otherwise = do + hPutStrLn stderr $ show timeout ++ " second timeout exceeded while waiting for pid lock file " ++ lockfile + giveup $ "Gave up waiting for possibly stale pid lock file " ++ lockfile + +dropLock :: LockHandle -> IO () +dropLock (LockHandle lockfile _ sidelock) = do + -- Drop side lock first, at which point the pid lock will be + -- considered stale. + dropSideLock sidelock + nukeFile lockfile + +getLockStatus :: LockFile -> IO LockStatus +getLockStatus = maybe StatusUnLocked (StatusLockedBy . lockingPid) <$$> readPidLock + +checkLocked :: LockFile -> IO (Maybe Bool) +checkLocked lockfile = conv <$> getLockStatus lockfile + where + conv (StatusLockedBy _) = Just True + conv _ = Just False + +-- Checks that the lock file still exists, and is the same file that was +-- locked to get the LockHandle. +checkSaneLock :: LockFile -> LockHandle -> IO Bool +checkSaneLock lockfile (LockHandle _ st _) = + go =<< catchMaybeIO (getFileStatus lockfile) + where + go Nothing = return False + go (Just st') = return $ + deviceID st == deviceID st' && fileID st == fileID st' diff --git a/Utility/LockFile/Posix.hs b/Utility/LockFile/Posix.hs new file mode 100644 index 0000000000..b45bca0882 --- /dev/null +++ b/Utility/LockFile/Posix.hs @@ -0,0 +1,119 @@ +{- Posix lock files + - + - Copyright 2014 Joey Hess + - + - License: BSD-2-clause + -} + +module Utility.LockFile.Posix ( + LockHandle, + lockShared, + lockExclusive, + tryLockShared, + tryLockExclusive, + checkLocked, + getLockStatus, + LockStatus(..), + dropLock, + checkSaneLock, + LockRequest(..), + openLockFile, +) where + +import Utility.Exception +import Utility.Applicative +import Utility.LockFile.LockStatus + +import System.IO +import System.Posix +import Data.Maybe + +type LockFile = FilePath + +newtype LockHandle = LockHandle Fd + +-- Takes a shared lock, blocking until the lock is available. +lockShared :: Maybe FileMode -> LockFile -> IO LockHandle +lockShared = lock ReadLock + +-- Takes an exclusive lock, blocking until the lock is available. +lockExclusive :: Maybe FileMode -> LockFile -> IO LockHandle +lockExclusive = lock WriteLock + +-- Tries to take a shared lock, but does not block. +tryLockShared :: Maybe FileMode -> LockFile -> IO (Maybe LockHandle) +tryLockShared = tryLock ReadLock + +-- Tries to take an exclusive lock, but does not block. +tryLockExclusive :: Maybe FileMode -> LockFile -> IO (Maybe LockHandle) +tryLockExclusive = tryLock WriteLock + +-- Setting the FileMode allows creation of a new lock file. +-- If it's Nothing then this only succeeds when the lock file already exists. +lock :: LockRequest -> Maybe FileMode -> LockFile -> IO LockHandle +lock lockreq mode lockfile = do + l <- openLockFile lockreq mode lockfile + waitToSetLock l (lockreq, AbsoluteSeek, 0, 0) + return (LockHandle l) + +-- Tries to take an lock, but does not block. +tryLock :: LockRequest -> Maybe FileMode -> LockFile -> IO (Maybe LockHandle) +tryLock lockreq mode lockfile = do + l <- openLockFile lockreq mode lockfile + v <- tryIO $ setLock l (lockreq, AbsoluteSeek, 0, 0) + case v of + Left _ -> do + closeFd l + return Nothing + Right _ -> return $ Just $ LockHandle l + +-- Close on exec flag is set so child processes do not inherit the lock. +openLockFile :: LockRequest -> Maybe FileMode -> LockFile -> IO Fd +openLockFile lockreq filemode lockfile = do + l <- openFd lockfile openfor filemode defaultFileFlags + setFdOption l CloseOnExec True + return l + where + openfor = case lockreq of + ReadLock -> ReadOnly + _ -> ReadWrite + +-- Returns Nothing when the file doesn't exist, for cases where +-- that is different from it not being locked. +checkLocked :: LockFile -> IO (Maybe Bool) +checkLocked = maybe Nothing (Just . isJust) <$$> getLockStatus' + +getLockStatus :: LockFile -> IO LockStatus +getLockStatus lockfile = do + v <- getLockStatus' lockfile + return $ case v of + Nothing -> StatusNoLockFile + Just Nothing -> StatusUnLocked + Just (Just pid) -> StatusLockedBy pid + +getLockStatus' :: LockFile -> IO (Maybe (Maybe ProcessID)) +getLockStatus' lockfile = go =<< catchMaybeIO open + where + open = openLockFile ReadLock Nothing lockfile + go Nothing = return Nothing + go (Just h) = do + v <- getLock h (ReadLock, AbsoluteSeek, 0, 0) + closeFd h + return (Just (fmap fst v)) + +dropLock :: LockHandle -> IO () +dropLock (LockHandle fd) = closeFd fd + +-- Checks that the lock file still exists, and is the same file that was +-- locked to get the LockHandle. +-- +-- This check is useful if the lock file might get deleted by something +-- else. +checkSaneLock :: LockFile -> LockHandle -> IO Bool +checkSaneLock lockfile (LockHandle fd) = + go =<< catchMaybeIO (getFileStatus lockfile) + where + go Nothing = return False + go (Just st) = do + fdst <- getFdStatus fd + return $ deviceID fdst == deviceID st && fileID fdst == fileID st diff --git a/Utility/LockFile/Windows.hs b/Utility/LockFile/Windows.hs new file mode 100644 index 0000000000..e761573544 --- /dev/null +++ b/Utility/LockFile/Windows.hs @@ -0,0 +1,75 @@ +{- Windows lock files + - + - Copyright 2014 Joey Hess + - + - License: BSD-2-clause + -} + +module Utility.LockFile.Windows ( + lockShared, + lockExclusive, + dropLock, + waitToLock, + LockHandle +) where + +import System.Win32.Types +import System.Win32.File +import Control.Concurrent + +type LockFile = FilePath + +type LockHandle = HANDLE + +{- Tries to lock a file with a shared lock, which allows other processes to + - also lock it shared. Fails if the file is exclusively locked. -} +lockShared :: LockFile -> IO (Maybe LockHandle) +lockShared = openLock fILE_SHARE_READ + +{- Tries to take an exclusive lock on a file. Fails if another process has + - a shared or exclusive lock. + - + - Note that exclusive locking also prevents the file from being opened for + - read or write by any other process. So for advisory locking of a file's + - content, a separate LockFile should be used. -} +lockExclusive :: LockFile -> IO (Maybe LockHandle) +lockExclusive = openLock fILE_SHARE_NONE + +{- Windows considers just opening a file enough to lock it. This will + - create the LockFile if it does not already exist. + - + - Will fail if the file is already open with an incompatible ShareMode. + - Note that this may happen if an unrelated process, such as a virus + - scanner, even looks at the file. See http://support.microsoft.com/kb/316609 + - + - Note that createFile busy-waits to try to avoid failing when some other + - process briefly has a file open. But that would make checking locks + - much more expensive, so is not done here. Thus, the use of c_CreateFile. + - + - Also, passing Nothing for SECURITY_ATTRIBUTES ensures that the lock file + - is not inherited by any child process. + -} +openLock :: ShareMode -> LockFile -> IO (Maybe LockHandle) +openLock sharemode f = do + h <- withTString f $ \c_f -> + c_CreateFile c_f gENERIC_READ sharemode security_attributes + oPEN_ALWAYS fILE_ATTRIBUTE_NORMAL (maybePtr Nothing) + return $ if h == iNVALID_HANDLE_VALUE + then Nothing + else Just h + where + security_attributes = maybePtr Nothing + +dropLock :: LockHandle -> IO () +dropLock = closeHandle + +{- If the initial lock fails, this is a BUSY wait, and does not + - guarentee FIFO order of waiters. In other news, Windows is a POS. -} +waitToLock :: IO (Maybe lockhandle) -> IO lockhandle +waitToLock locker = takelock + where + takelock = go =<< locker + go (Just lck) = return lck + go Nothing = do + threadDelay (500000) -- half a second + takelock diff --git a/Utility/LockPool.hs b/Utility/LockPool.hs new file mode 100644 index 0000000000..7dbabb91ae --- /dev/null +++ b/Utility/LockPool.hs @@ -0,0 +1,34 @@ +{- Lock pool. + - + - This avoids a problem with unix fcntl locks: They are not composition-safe. + - + - For example, if one thread is holding a lock, and another thread opens the + - lock file (to attempt to take or check the lock), and then closes it, + - the lock will be released, despite the first thread still having the + - lockfile open. + - + - Or, if a process is already holding an exclusive lock on a file, and + - re-opens it and tries to take another exclusive lock, it won't block + - on the first lock. + - + - To avoid these problems, this implements a lock pool. This keeps track + - of which lock files are being used by the process, using STM to handle + - inter-process locking. + - + - Note that, like Utility.LockFile, this does *not* attempt to be a + - portability shim; the native locking of the OS is used. + - + - Copyright 2015 Joey Hess + - + - License: BSD-2-clause + -} + +{-# LANGUAGE CPP #-} + +module Utility.LockPool (module X) where + +#ifndef mingw32_HOST_OS +import Utility.LockPool.Posix as X +#else +import Utility.LockPool.Windows as X +#endif diff --git a/Utility/LockPool/LockHandle.hs b/Utility/LockPool/LockHandle.hs new file mode 100644 index 0000000000..41b110aeea --- /dev/null +++ b/Utility/LockPool/LockHandle.hs @@ -0,0 +1,75 @@ +{- Handles for lock pools. + - + - Copyright 2015 Joey Hess + - + - License: BSD-2-clause + -} + +{-# LANGUAGE CPP #-} + +module Utility.LockPool.LockHandle ( + LockHandle, + FileLockOps(..), + dropLock, +#ifndef mingw32_HOST_OS + checkSaneLock, +#endif + makeLockHandle, + tryMakeLockHandle, +) where + +import qualified Utility.LockPool.STM as P +import Utility.LockPool.STM (LockFile) + +import Control.Concurrent.STM +import Control.Exception +import Control.Applicative +import Prelude + +data LockHandle = LockHandle P.LockHandle FileLockOps + +data FileLockOps = FileLockOps + { fDropLock :: IO () +#ifndef mingw32_HOST_OS + , fCheckSaneLock :: LockFile -> IO Bool +#endif + } + +dropLock :: LockHandle -> IO () +dropLock (LockHandle ph _) = P.releaseLock ph + +#ifndef mingw32_HOST_OS +checkSaneLock :: LockFile -> LockHandle -> IO Bool +checkSaneLock lockfile (LockHandle _ flo) = fCheckSaneLock flo lockfile +#endif + +-- Take a lock, by first updating the lock pool, and then taking the file +-- lock. If taking the file lock fails for any reason, take care to +-- release the lock in the lock pool. +makeLockHandle :: P.LockPool -> LockFile -> (P.LockPool -> LockFile -> STM P.LockHandle) -> (LockFile -> IO FileLockOps) -> IO LockHandle +makeLockHandle pool file pa fa = bracketOnError setup cleanup go + where + setup = atomically (pa pool file) + cleanup ph = P.releaseLock ph + go ph = mkLockHandle pool file ph =<< fa file + +tryMakeLockHandle :: P.LockPool -> LockFile -> (P.LockPool -> LockFile -> STM (Maybe P.LockHandle)) -> (LockFile -> IO (Maybe FileLockOps)) -> IO (Maybe LockHandle) +tryMakeLockHandle pool file pa fa = bracketOnError setup cleanup go + where + setup = atomically (pa pool file) + cleanup Nothing = return () + cleanup (Just ph) = P.releaseLock ph + go Nothing = return Nothing + go (Just ph) = do + mfo <- fa file + case mfo of + Nothing -> do + cleanup (Just ph) + return Nothing + Just fo -> Just <$> mkLockHandle pool file ph fo + +mkLockHandle :: P.LockPool -> LockFile -> P.LockHandle -> FileLockOps -> IO LockHandle +mkLockHandle pool file ph fo = do + atomically $ P.registerCloseLockFile pool file (fDropLock fo) + return $ LockHandle ph fo + diff --git a/Utility/LockPool/PidLock.hs b/Utility/LockPool/PidLock.hs new file mode 100644 index 0000000000..26ed96f3cf --- /dev/null +++ b/Utility/LockPool/PidLock.hs @@ -0,0 +1,61 @@ +{- Pid locks, using lock pools. + - + - Copyright 2015 Joey Hess + - + - License: BSD-2-clause + -} + +module Utility.LockPool.PidLock ( + P.LockFile, + LockHandle, + waitLock, + tryLock, + checkLocked, + getLockStatus, + LockStatus(..), + dropLock, + checkSaneLock, +) where + +import qualified Utility.LockFile.PidLock as F +import Utility.LockFile.LockStatus +import qualified Utility.LockPool.STM as P +import Utility.LockPool.STM (LockFile, LockMode(..)) +import Utility.LockPool.LockHandle +import Utility.ThreadScheduler + +import System.IO +import System.Posix +import Data.Maybe +import Control.Applicative +import Prelude + +-- Takes a pid lock, blocking until the lock is available or the timeout. +waitLock :: Seconds -> LockFile -> IO LockHandle +waitLock timeout file = makeLockHandle P.lockPool file + -- LockShared for STM lock, because a pid lock can be the top-level + -- lock with various other STM level locks gated behind it. + (\p f -> P.waitTakeLock p f LockShared) + (\f -> mk <$> F.waitLock timeout f) + +-- Tries to take a pid lock, but does not block. +tryLock :: LockFile -> IO (Maybe LockHandle) +tryLock file = tryMakeLockHandle P.lockPool file + (\p f -> P.tryTakeLock p f LockShared) + (\f -> fmap mk <$> F.tryLock f) + +checkLocked :: LockFile -> IO (Maybe Bool) +checkLocked file = P.getLockStatus P.lockPool file + (pure (Just True)) + (F.checkLocked file) + +getLockStatus :: LockFile -> IO LockStatus +getLockStatus file = P.getLockStatus P.lockPool file + (StatusLockedBy <$> getProcessID) + (F.getLockStatus file) + +mk :: F.LockHandle -> FileLockOps +mk h = FileLockOps + { fDropLock = F.dropLock h + , fCheckSaneLock = \f -> F.checkSaneLock f h + } diff --git a/Utility/LockPool/Posix.hs b/Utility/LockPool/Posix.hs new file mode 100644 index 0000000000..2c0b7c78e9 --- /dev/null +++ b/Utility/LockPool/Posix.hs @@ -0,0 +1,74 @@ +{- Posix lock files, using lock pools. + - + - Copyright 2015 Joey Hess + - + - License: BSD-2-clause + -} + +module Utility.LockPool.Posix ( + P.LockFile, + LockHandle, + lockShared, + lockExclusive, + tryLockShared, + tryLockExclusive, + checkLocked, + getLockStatus, + LockStatus(..), + dropLock, + checkSaneLock, +) where + +import qualified Utility.LockFile.Posix as F +import Utility.LockFile.LockStatus +import qualified Utility.LockPool.STM as P +import Utility.LockPool.STM (LockFile, LockMode(..)) +import Utility.LockPool.LockHandle + +import System.IO +import System.Posix +import Data.Maybe +import Control.Applicative +import Prelude + +-- Takes a shared lock, blocking until the lock is available. +lockShared :: Maybe FileMode -> LockFile -> IO LockHandle +lockShared mode file = makeLockHandle P.lockPool file + (\p f -> P.waitTakeLock p f LockShared) + (\f -> mk <$> F.lockShared mode f) + +-- Takes an exclusive lock, blocking until the lock is available. +lockExclusive :: Maybe FileMode -> LockFile -> IO LockHandle +lockExclusive mode file = makeLockHandle P.lockPool file + (\p f -> P.waitTakeLock p f LockExclusive) + (\f -> mk <$> F.lockExclusive mode f) + +-- Tries to take a shared lock, but does not block. +tryLockShared :: Maybe FileMode -> LockFile -> IO (Maybe LockHandle) +tryLockShared mode file = tryMakeLockHandle P.lockPool file + (\p f -> P.tryTakeLock p f LockShared) + (\f -> fmap mk <$> F.tryLockShared mode f) + +-- Tries to take an exclusive lock, but does not block. +tryLockExclusive :: Maybe FileMode -> LockFile -> IO (Maybe LockHandle) +tryLockExclusive mode file = tryMakeLockHandle P.lockPool file + (\p f -> P.tryTakeLock p f LockExclusive) + (\f -> fmap mk <$> F.tryLockExclusive mode f) + +-- Returns Nothing when the file doesn't exist, for cases where +-- that is different from it not being locked. +checkLocked :: LockFile -> IO (Maybe Bool) +checkLocked file = P.getLockStatus P.lockPool file + (pure (Just True)) + (F.checkLocked file) + +getLockStatus :: LockFile -> IO LockStatus +getLockStatus file = P.getLockStatus P.lockPool file + (StatusLockedBy <$> getProcessID) + (F.getLockStatus file) + +mk :: F.LockHandle -> FileLockOps +mk h = FileLockOps + { fDropLock = F.dropLock h + , fCheckSaneLock = \f -> F.checkSaneLock f h + } diff --git a/Utility/LockPool/STM.hs b/Utility/LockPool/STM.hs new file mode 100644 index 0000000000..5cb7b88340 --- /dev/null +++ b/Utility/LockPool/STM.hs @@ -0,0 +1,138 @@ +{- STM implementation of lock pools. + - + - Copyright 2015 Joey Hess + - + - License: BSD-2-clause + -} + +module Utility.LockPool.STM ( + LockPool, + lockPool, + LockFile, + LockMode(..), + LockHandle, + waitTakeLock, + tryTakeLock, + getLockStatus, + releaseLock, + CloseLockFile, + registerCloseLockFile, +) where + +import Utility.Monad + +import System.IO.Unsafe (unsafePerformIO) +import qualified Data.Map.Strict as M +import Control.Concurrent.STM +import Control.Exception +import Control.Monad +import Control.Applicative +import Prelude + +type LockFile = FilePath + +data LockMode = LockExclusive | LockShared + deriving (Eq) + +-- This TMVar is full when the handle is open, and is emptied when it's +-- closed. +type LockHandle = TMVar (LockPool, LockFile) + +type LockCount = Integer + +data LockStatus = LockStatus LockMode LockCount CloseLockFile + +type CloseLockFile = IO () + +-- This TMVar is normally kept full. +type LockPool = TMVar (M.Map LockFile LockStatus) + +-- A shared global variable for the lockPool. Avoids callers needing to +-- maintain state for this implementation detail. +{-# NOINLINE lockPool #-} +lockPool :: LockPool +lockPool = unsafePerformIO (newTMVarIO M.empty) + +-- Updates the LockPool, blocking as necessary if another thread is holding +-- a conflicting lock. +-- +-- Note that when a shared lock is held, an exclusive lock will block. +-- While that blocking is happening, another call to this function to take +-- the same shared lock should not be blocked on the exclusive lock. +-- Keeping the whole Map in a TMVar accomplishes this, at the expense of +-- sometimes retrying after unrelated changes in the map. +waitTakeLock :: LockPool -> LockFile -> LockMode -> STM LockHandle +waitTakeLock pool file mode = maybe retry return =<< tryTakeLock pool file mode + +-- Avoids blocking if another thread is holding a conflicting lock. +tryTakeLock :: LockPool -> LockFile -> LockMode -> STM (Maybe LockHandle) +tryTakeLock pool file mode = do + m <- takeTMVar pool + let success v = do + putTMVar pool (M.insert file v m) + Just <$> newTMVar (pool, file) + case M.lookup file m of + Just (LockStatus mode' n closelockfile) + | mode == LockShared && mode' == LockShared -> + success $ LockStatus mode (succ n) closelockfile + | n > 0 -> do + putTMVar pool m + return Nothing + _ -> success $ LockStatus mode 1 noop + +-- Call after waitTakeLock or tryTakeLock, to register a CloseLockFile +-- action to run when releasing the lock. +registerCloseLockFile :: LockPool -> LockFile -> CloseLockFile -> STM () +registerCloseLockFile pool file closelockfile = do + m <- takeTMVar pool + putTMVar pool (M.update go file m) + where + go (LockStatus mode n closelockfile') = Just $ + LockStatus mode n (closelockfile' >> closelockfile) + +-- Checks if a lock is being held. If it's held by the current process, +-- runs the getdefault action; otherwise runs the checker action. +-- +-- Note that the lock pool is left empty while the checker action is run. +-- This allows checker actions that open/close files, and so would be in +-- danger of conflicting with locks created at the same time this is +-- running. With the lock pool empty, anything that attempts +-- to take a lock will block, avoiding that race. +getLockStatus :: LockPool -> LockFile -> IO v -> IO v -> IO v +getLockStatus pool file getdefault checker = do + v <- atomically $ do + m <- takeTMVar pool + let threadlocked = case M.lookup file m of + Just (LockStatus _ n _) | n > 0 -> True + _ -> False + if threadlocked + then do + putTMVar pool m + return Nothing + else return $ Just $ atomically $ putTMVar pool m + case v of + Nothing -> getdefault + Just restore -> bracket_ (return ()) restore checker + +-- Only runs action to close underlying lock file when this is the last +-- user of the lock, and when the lock has not already been closed. +-- +-- Note that the lock pool is left empty while the CloseLockFile action +-- is run, to avoid race with another thread trying to open the same lock +-- file. +releaseLock :: LockHandle -> IO () +releaseLock h = go =<< atomically (tryTakeTMVar h) + where + go (Just (pool, file)) = do + (m, closelockfile) <- atomically $ do + m <- takeTMVar pool + return $ case M.lookup file m of + Just (LockStatus mode n closelockfile) + | n == 1 -> (M.delete file m, closelockfile) + | otherwise -> + (M.insert file (LockStatus mode (pred n) closelockfile) m, noop) + Nothing -> (m, noop) + closelockfile + atomically $ putTMVar pool m + -- The LockHandle was already closed. + go Nothing = return () diff --git a/Utility/LockPool/Windows.hs b/Utility/LockPool/Windows.hs new file mode 100644 index 0000000000..9f3a0b95cc --- /dev/null +++ b/Utility/LockPool/Windows.hs @@ -0,0 +1,48 @@ +{- Windows lock files, using lock pools. + - + - Copyright 2015 Joey Hess + - + - License: BSD-2-clause + -} + +module Utility.LockPool.Windows ( + P.LockFile, + LockHandle, + lockShared, + lockExclusive, + dropLock, + waitToLock, +) where + +import qualified Utility.LockFile.Windows as F +import qualified Utility.LockPool.STM as P +import Utility.LockPool.LockHandle +import Utility.LockPool.STM (LockFile, LockMode(..)) + +{- Tries to lock a file with a shared lock, which allows other processes to + - also lock it shared. Fails if the file is exclusively locked. -} +lockShared :: LockFile -> IO (Maybe LockHandle) +lockShared file = tryMakeLockHandle P.lockPool file + (\p f -> P.tryTakeLock p f LockShared) + (\f -> fmap mk <$> F.lockShared f) + +{- Tries to take an exclusive lock on a file. Fails if another process has + - a shared or exclusive lock. + - + - Note that exclusive locking also prevents the file from being opened for + - read or write by any other process. So for advisory locking of a file's + - content, a separate LockFile should be used. -} +lockExclusive :: LockFile -> IO (Maybe LockHandle) +lockExclusive file = tryMakeLockHandle P.lockPool file + (\p f -> P.tryTakeLock p f LockExclusive) + (\f -> fmap mk <$> F.lockExclusive f) + +{- If the initial lock fails, this is a BUSY wait, and does not + - guarentee FIFO order of waiters. In other news, Windows is a POS. -} +waitToLock :: IO (Maybe lockhandle) -> IO lockhandle +waitToLock = F.waitToLock + +mk :: F.LockHandle -> FileLockOps +mk h = FileLockOps + { fDropLock = F.dropLock h + } diff --git a/Utility/LogFile.hs b/Utility/LogFile.hs new file mode 100644 index 0000000000..4e08e9b9f4 --- /dev/null +++ b/Utility/LogFile.hs @@ -0,0 +1,60 @@ +{- log files + - + - Copyright 2012 Joey Hess + - + - License: BSD-2-clause + -} + +{-# LANGUAGE CPP #-} + +module Utility.LogFile where + +import Common + +#ifndef mingw32_HOST_OS +import System.Posix.Types +import System.Posix.IO +#endif + +openLog :: FilePath -> IO Handle +openLog logfile = do + rotateLog logfile + openFile logfile AppendMode + +rotateLog :: FilePath -> IO () +rotateLog logfile = go 0 + where + go num + | num > maxLogs = return () + | otherwise = whenM (doesFileExist currfile) $ do + go (num + 1) + rename currfile nextfile + where + currfile = filename num + nextfile = filename (num + 1) + filename n + | n == 0 = logfile + | otherwise = rotatedLog logfile n + +rotatedLog :: FilePath -> Int -> FilePath +rotatedLog logfile n = logfile ++ "." ++ show n + +{- Lists most recent logs last. -} +listLogs :: FilePath -> IO [FilePath] +listLogs logfile = filterM doesFileExist $ reverse $ + logfile : map (rotatedLog logfile) [1..maxLogs] + +maxLogs :: Int +maxLogs = 9 + +#ifndef mingw32_HOST_OS +redirLog :: Fd -> IO () +redirLog logfd = do + mapM_ (redir logfd) [stdOutput, stdError] + closeFd logfd + +redir :: Fd -> Fd -> IO () +redir newh h = do + closeFd h + void $ dupTo newh h +#endif diff --git a/Utility/Lsof.hs b/Utility/Lsof.hs new file mode 100644 index 0000000000..7cab8d98a7 --- /dev/null +++ b/Utility/Lsof.hs @@ -0,0 +1,123 @@ +{- lsof interface + - + - Copyright 2012 Joey Hess + - + - License: BSD-2-clause + -} + +{-# LANGUAGE CPP #-} + +module Utility.Lsof where + +import Common +import BuildInfo +import Utility.Env.Set + +import System.Posix.Types + +data LsofOpenMode = OpenReadWrite | OpenReadOnly | OpenWriteOnly | OpenUnknown + deriving (Show, Eq) + +type CmdLine = String + +data ProcessInfo = ProcessInfo ProcessID CmdLine + deriving (Show) + +{- lsof is not in PATH on all systems, so BuildInfo may have the absolute + - path where the program was found. Make sure at runtime that lsof is + - available, and if it's not in PATH, adjust PATH to contain it. -} +setup :: IO () +setup = do + let cmd = fromMaybe "lsof" BuildInfo.lsof + when (isAbsolute cmd) $ do + path <- getSearchPath + let path' = takeDirectory cmd : path + setEnv "PATH" (intercalate [searchPathSeparator] path') True + +{- Checks each of the files in a directory to find open files. + - Note that this will find hard links to files elsewhere that are open. -} +queryDir :: FilePath -> IO [(FilePath, LsofOpenMode, ProcessInfo)] +queryDir path = query ["+d", path] + +{- Runs lsof with some parameters. + - + - Ignores nonzero exit code; lsof returns that when no files are open. + - + - Note: If lsof is not available, this always returns [] ! + -} +query :: [String] -> IO [(FilePath, LsofOpenMode, ProcessInfo)] +query opts = + withHandle StdoutHandle (createProcessChecked checkSuccessProcess) p $ + parse <$$> hGetContentsStrict + where + p = proc "lsof" ("-F0can" : opts) + +type LsofParser = String -> [(FilePath, LsofOpenMode, ProcessInfo)] + +parse :: LsofParser +#ifdef __ANDROID__ +parse = parseDefault +#else +parse = parseFormatted +#endif + +{- Parsing null-delimited output like: + - + - pPID\0cCMDLINE\0 + - aMODE\0nFILE\0 + - aMODE\0nFILE\0 + - pPID\0[...] + - + - Where each new process block is started by a pid, and a process can + - have multiple files open. + -} +parseFormatted :: LsofParser +parseFormatted s = bundle $ go [] $ lines s + where + bundle = concatMap (\(fs, p) -> map (\(f, m) -> (f, m, p)) fs) + + go c [] = c + go c ((t:r):ls) + | t == 'p' = + let (fs, ls') = parsefiles [] ls + in go ((fs, parseprocess r):c) ls' + | otherwise = parsefail + go _ _ = parsefail + + parseprocess l = case splitnull l of + [pid, 'c':cmdline, ""] -> + case readish pid of + (Just n) -> ProcessInfo n cmdline + Nothing -> parsefail + _ -> parsefail + + parsefiles c [] = (c, []) + parsefiles c (l:ls) = parsefiles' c (splitnull l) l ls + + parsefiles' c ['a':mode, 'n':file, ""] _ ls = + parsefiles ((file, parsemode mode):c) ls + parsefiles' c (('p':_):_) l ls = (c, l:ls) + -- Some buggy versions of lsof emit a f field + -- that was not requested, so ignore it. + parsefiles' c (('f':_):rest) l ls = parsefiles' c rest l ls + parsefiles' _ _ _ _ = parsefail + + parsemode ('r':_) = OpenReadOnly + parsemode ('w':_) = OpenWriteOnly + parsemode ('u':_) = OpenReadWrite + parsemode _ = OpenUnknown + + splitnull = splitc '\0' + + parsefail = error $ "failed to parse lsof output: " ++ show s + +{- Parses lsof's default output format. -} +parseDefault :: LsofParser +parseDefault = mapMaybe parseline . drop 1 . lines + where + parseline l = case words l of + (command : spid : _user : _fd : _type : _device : _size : _node : rest) -> + case readish spid of + Nothing -> Nothing + Just pid -> Just (unwords rest, OpenUnknown, ProcessInfo pid command) + _ -> Nothing diff --git a/Utility/MagicWormhole.hs b/Utility/MagicWormhole.hs new file mode 100644 index 0000000000..4a390df46f --- /dev/null +++ b/Utility/MagicWormhole.hs @@ -0,0 +1,171 @@ +{- Magic Wormhole integration + - + - Copyright 2016 Joey Hess + - + - License: BSD-2-clause + -} + +module Utility.MagicWormhole ( + Code, + mkCode, + toCode, + fromCode, + validCode, + CodeObserver, + CodeProducer, + mkCodeObserver, + mkCodeProducer, + waitCode, + sendCode, + WormHoleParams, + appId, + sendFile, + receiveFile, + isInstalled, +) where + +import Utility.Process +import Utility.SafeCommand +import Utility.Monad +import Utility.Misc +import Utility.Env +import Utility.Path + +import System.IO +import System.Exit +import Control.Concurrent +import Control.Concurrent.Async +import Control.Exception +import Data.Char +import Data.List +import Control.Applicative +import Prelude + +-- | A Magic Wormhole code. +newtype Code = Code String + deriving (Eq, Show) + +-- | Smart constructor for Code +mkCode :: String -> Maybe Code +mkCode s + | validCode s = Just (Code s) + | otherwise = Nothing + +-- | Tries to fix up some common mistakes in a homan-entered code. +toCode :: String -> Maybe Code +toCode s = mkCode $ intercalate "-" $ words s + +fromCode :: Code -> String +fromCode (Code s) = s + +-- | Codes have the form number-word-word and may contain 2 or more words. +validCode :: String -> Bool +validCode s = + let (n, r) = separate (== '-') s + (w1, w2) = separate (== '-') r + in and + [ not (null n) + , all isDigit n + , not (null w1) + , not (null w2) + , not $ any isSpace s + ] + +newtype CodeObserver = CodeObserver (MVar Code) + +newtype CodeProducer = CodeProducer (MVar Code) + +mkCodeObserver :: IO CodeObserver +mkCodeObserver = CodeObserver <$> newEmptyMVar + +mkCodeProducer :: IO CodeProducer +mkCodeProducer = CodeProducer <$> newEmptyMVar + +waitCode :: CodeObserver -> IO Code +waitCode (CodeObserver o) = readMVar o + +sendCode :: CodeProducer -> Code -> IO () +sendCode (CodeProducer p) = putMVar p + +type WormHoleParams = [CommandParam] + +-- | An appid should be provided when using wormhole in an app, to avoid +-- using the same channel space as ad-hoc wormhole users. +appId :: String -> WormHoleParams +appId s = [Param "--appid", Param s] + +-- | Sends a file. Once the send is underway, and the Code has been +-- generated, it will be sent to the CodeObserver. (This may not happen, +-- eg if there's a network problem). +-- +-- Currently this has to parse the output of wormhole to find the code. +-- To make this as robust as possible, avoids looking for any particular +-- output strings, and only looks for the form of a wormhole code +-- (number-word-word). +-- +-- Note that, if the filename looks like "foo 1-wormhole-code bar", when +-- that is output by wormhole, it will look like it's output a wormhole code. +-- +-- A request to make the code available in machine-parsable form is here: +-- https://github.com/warner/magic-wormhole/issues/104 +sendFile :: FilePath -> CodeObserver -> WormHoleParams -> IO Bool +sendFile f (CodeObserver observer) ps = do + -- Work around stupid stdout buffering behavior of python. + -- See https://github.com/warner/magic-wormhole/issues/108 + environ <- addEntry "PYTHONUNBUFFERED" "1" <$> getEnvironment + runWormHoleProcess p { env = Just environ} $ \_hin hout herr -> do + (inout, inerr) <- findcode hout `concurrently` findcode herr + return (inout || inerr) + where + p = wormHoleProcess (Param "send" : ps ++ [File f]) + findcode h = findcode' =<< words <$> hGetContents h + findcode' [] = return False + findcode' (w:ws) = case mkCode w of + Just code -> do + _ <- tryPutMVar observer code + return True + Nothing -> findcode' ws + +-- | Receives a file. Once the receive is under way, the Code will be +-- read from the CodeProducer, and fed to wormhole on stdin. +receiveFile :: FilePath -> CodeProducer -> WormHoleParams -> IO Bool +receiveFile f (CodeProducer producer) ps = runWormHoleProcess p $ \hin _hout _herr -> do + Code c <- readMVar producer + hPutStrLn hin c + hFlush hin + return True + where + p = wormHoleProcess $ + [ Param "receive" + , Param "--accept-file" + , Param "--output-file" + , File f + ] ++ ps + +wormHoleProcess :: WormHoleParams -> CreateProcess +wormHoleProcess = proc "wormhole" . toCommand + +runWormHoleProcess :: CreateProcess -> (Handle -> Handle -> Handle -> IO Bool) -> IO Bool +runWormHoleProcess p consumer = + bracketOnError setup (\v -> cleanup v <&&> return False) go + where + setup = do + (Just hin, Just hout, Just herr, pid) + <- createProcess p + { std_in = CreatePipe + , std_out = CreatePipe + , std_err = CreatePipe + } + return (hin, hout, herr, pid) + cleanup (hin, hout, herr, pid) = do + r <- waitForProcess pid + hClose hin + hClose hout + hClose herr + return $ case r of + ExitSuccess -> True + ExitFailure _ -> False + go h@(hin, hout, herr, _) = consumer hin hout herr <&&> cleanup h + +isInstalled :: IO Bool +isInstalled = inPath "wormhole" diff --git a/Utility/Matcher.hs b/Utility/Matcher.hs new file mode 100644 index 0000000000..badf72acfd --- /dev/null +++ b/Utility/Matcher.hs @@ -0,0 +1,178 @@ +{- A generic matcher. + - + - Can be used to check if a user-supplied condition, + - like "foo and ( bar or not baz )" matches. The condition must already + - be tokenized, and can contain arbitrary operations. + - + - If operations are not separated by and/or, they are defaulted to being + - anded together, so "foo bar baz" all must match. + - + - Is forgiving about misplaced closing parens, so "foo and (bar or baz" + - will be handled, as will "foo and ( bar or baz ) )" + - + - Copyright 2011-2013 Joey Hess + - + - License: BSD-2-clause + -} + +{-# LANGUAGE Rank2Types, KindSignatures #-} + +module Utility.Matcher ( + Token(..), + Matcher(..), + token, + tokens, + generate, + match, + matchM, + matchMrun, + isEmpty, + combineMatchers, + + prop_matcher_sane +) where + +import Common + +{- A Token can be an Operation of an arbitrary type, or one of a few + - predefined peices of syntax. -} +data Token op = Operation op | And | Or | Not | Open | Close + deriving (Show, Eq) + +data Matcher op = MAny + | MAnd (Matcher op) (Matcher op) + | MOr (Matcher op) (Matcher op) + | MNot (Matcher op) + | MOp op + deriving (Show, Eq) + +{- Converts a word of syntax into a token. Doesn't handle operations. -} +token :: String -> Token op +token "and" = And +token "or" = Or +token "not" = Not +token "(" = Open +token ")" = Close +token t = error $ "unknown token " ++ t + +tokens :: [String] +tokens = words "and or not ( )" + +{- Converts a list of Tokens into a Matcher. -} +generate :: [Token op] -> Matcher op +generate = simplify . process MAny . tokenGroups + where + process m [] = m + process m ts = uncurry process $ consume m ts + + consume m (One And:rest) = term (m `MAnd`) rest + consume m (One Or:rest) = term (m `MOr`) rest + consume m (One Not:rest) = term (\p -> m `MAnd` (MNot p)) rest + consume m (One (Operation o):rest) = (m `MAnd` MOp o, rest) + consume m (Group g:rest) = (process m g, rest) + consume m (_:rest) = consume m rest + consume m [] = (m, []) + + term a l = + let (p, l') = consume MAny l + in (a p, l') + + simplify (MAnd MAny x) = simplify x + simplify (MAnd x MAny) = simplify x + simplify (MAnd x y) = MAnd (simplify x) (simplify y) + simplify (MOr x y) = MOr (simplify x) (simplify y) + simplify (MNot x) = MNot (simplify x) + simplify x = x + +data TokenGroup op = One (Token op) | Group [TokenGroup op] + deriving (Show, Eq) + +tokenGroups :: [Token op] -> [TokenGroup op] +tokenGroups [] = [] +tokenGroups (t:ts) = go t + where + go Open = + let (gr, rest) = findClose ts + in gr : tokenGroups rest + go Close = tokenGroups ts -- not picky about missing Close + go _ = One t : tokenGroups ts + +findClose :: [Token op] -> (TokenGroup op, [Token op]) +findClose l = + let (g, rest) = go [] l + in (Group (reverse g), rest) + where + go c [] = (c, []) -- not picky about extra Close + go c (t:ts) = dispatch t + where + dispatch Close = (c, ts) + dispatch Open = + let (c', ts') = go [] ts + in go (Group (reverse c') : c) ts' + dispatch _ = go (One t:c) ts + +{- Checks if a Matcher matches, using a supplied function to check + - the value of Operations. -} +match :: (op -> v -> Bool) -> Matcher op -> v -> Bool +match a m v = go m + where + go MAny = True + go (MAnd m1 m2) = go m1 && go m2 + go (MOr m1 m2) = go m1 || go m2 + go (MNot m1) = not $ go m1 + go (MOp o) = a o v + +{- Runs a monadic Matcher, where Operations are actions in the monad. -} +matchM :: Monad m => Matcher (v -> m Bool) -> v -> m Bool +matchM m v = matchMrun m $ \o -> o v + +{- More generic running of a monadic Matcher, with full control over running + - of Operations. Mostly useful in order to match on more than one + - parameter. -} +matchMrun :: forall o (m :: * -> *). Monad m => Matcher o -> (o -> m Bool) -> m Bool +matchMrun m run = go m + where + go MAny = return True + go (MAnd m1 m2) = go m1 <&&> go m2 + go (MOr m1 m2) = go m1 <||> go m2 + go (MNot m1) = liftM not (go m1) + go (MOp o) = run o + +{- Checks if a matcher contains no limits. -} +isEmpty :: Matcher a -> Bool +isEmpty MAny = True +isEmpty _ = False + +{- Combines two matchers, yielding a matcher that will match anything + - both do. But, if one matcher contains no limits, yield the other one. -} +combineMatchers :: Matcher a -> Matcher a -> Matcher a +combineMatchers a b + | isEmpty a = b + | isEmpty b = a + | otherwise = a `MOr` b + +prop_matcher_sane :: Bool +prop_matcher_sane = all (\m -> match dummy m ()) $ map generate + [ [Operation True] + , [] + , [Operation False, Or, Operation True, Or, Operation False] + , [Operation True, Or, Operation True] + , [Operation True, And, Operation True] + , [Not, Open, Operation True, And, Operation False, Close] + , [Not, Open, Not, Open, Not, Operation False, Close, Close] + , [Not, Open, Not, Open, Not, Open, Not, Operation True, Close, Close] + , [Operation True, And, Not, Operation False] + , [Operation True, Not, Operation False] + , [Operation True, Not, Not, Not, Operation False] + , [Operation True, Not, Not, Not, Operation False, And, Operation True] + , [Operation True, Not, Not, Not, Operation False, Operation True] + , [Not, Open, Operation True, And, Operation False, Close, + And, Open, + Open, Operation True, And, Operation False, Close, + Or, + Open, Operation True, And, Open, Not, Operation False, Close, Close, + Close, And, + Open, Not, Operation False, Close] + ] + where + dummy b _ = b diff --git a/Utility/Metered.hs b/Utility/Metered.hs new file mode 100644 index 0000000000..54153404ea --- /dev/null +++ b/Utility/Metered.hs @@ -0,0 +1,387 @@ +{- Metered IO and actions + - + - Copyright 2012-2018 Joey Hess + - + - License: BSD-2-clause + -} + +{-# LANGUAGE TypeSynonymInstances, BangPatterns #-} + +module Utility.Metered where + +import Common +import Utility.FileSystemEncoding +import Utility.Percentage +import Utility.DataUnits +import Utility.HumanTime + +import qualified Data.ByteString.Lazy as L +import qualified Data.ByteString as S +import System.IO.Unsafe +import Foreign.Storable (Storable(sizeOf)) +import System.Posix.Types +import Data.Int +import Control.Concurrent +import Control.Concurrent.Async +import Control.Monad.IO.Class (MonadIO) +import Data.Time.Clock +import Data.Time.Clock.POSIX + +{- An action that can be run repeatedly, updating it on the bytes processed. + - + - Note that each call receives the total number of bytes processed, so + - far, *not* an incremental amount since the last call. -} +type MeterUpdate = (BytesProcessed -> IO ()) + +nullMeterUpdate :: MeterUpdate +nullMeterUpdate _ = return () + +combineMeterUpdate :: MeterUpdate -> MeterUpdate -> MeterUpdate +combineMeterUpdate a b = \n -> a n >> b n + +{- Total number of bytes processed so far. -} +newtype BytesProcessed = BytesProcessed Integer + deriving (Eq, Ord, Show) + +class AsBytesProcessed a where + toBytesProcessed :: a -> BytesProcessed + fromBytesProcessed :: BytesProcessed -> a + +instance AsBytesProcessed BytesProcessed where + toBytesProcessed = id + fromBytesProcessed = id + +instance AsBytesProcessed Integer where + toBytesProcessed i = BytesProcessed i + fromBytesProcessed (BytesProcessed i) = i + +instance AsBytesProcessed Int where + toBytesProcessed i = BytesProcessed $ toInteger i + fromBytesProcessed (BytesProcessed i) = fromInteger i + +instance AsBytesProcessed Int64 where + toBytesProcessed i = BytesProcessed $ toInteger i + fromBytesProcessed (BytesProcessed i) = fromInteger i + +instance AsBytesProcessed FileOffset where + toBytesProcessed sz = BytesProcessed $ toInteger sz + fromBytesProcessed (BytesProcessed sz) = fromInteger sz + +addBytesProcessed :: AsBytesProcessed v => BytesProcessed -> v -> BytesProcessed +addBytesProcessed (BytesProcessed i) v = + let (BytesProcessed n) = toBytesProcessed v + in BytesProcessed $! i + n + +zeroBytesProcessed :: BytesProcessed +zeroBytesProcessed = BytesProcessed 0 + +{- Sends the content of a file to an action, updating the meter as it's + - consumed. -} +withMeteredFile :: FilePath -> MeterUpdate -> (L.ByteString -> IO a) -> IO a +withMeteredFile f meterupdate a = withBinaryFile f ReadMode $ \h -> + hGetContentsMetered h meterupdate >>= a + +{- Sends the content of a file to a Handle, updating the meter as it's + - written. -} +streamMeteredFile :: FilePath -> MeterUpdate -> Handle -> IO () +streamMeteredFile f meterupdate h = withMeteredFile f meterupdate $ L.hPut h + +{- Writes a ByteString to a Handle, updating a meter as it's written. -} +meteredWrite :: MeterUpdate -> Handle -> L.ByteString -> IO () +meteredWrite meterupdate h = void . meteredWrite' meterupdate h + +meteredWrite' :: MeterUpdate -> Handle -> L.ByteString -> IO BytesProcessed +meteredWrite' meterupdate h = go zeroBytesProcessed . L.toChunks + where + go sofar [] = return sofar + go sofar (c:cs) = do + S.hPut h c + let !sofar' = addBytesProcessed sofar $ S.length c + meterupdate sofar' + go sofar' cs + +meteredWriteFile :: MeterUpdate -> FilePath -> L.ByteString -> IO () +meteredWriteFile meterupdate f b = withBinaryFile f WriteMode $ \h -> + meteredWrite meterupdate h b + +{- Applies an offset to a MeterUpdate. This can be useful when + - performing a sequence of actions, such as multiple meteredWriteFiles, + - that all update a common meter progressively. Or when resuming. + -} +offsetMeterUpdate :: MeterUpdate -> BytesProcessed -> MeterUpdate +offsetMeterUpdate base offset = \n -> base (offset `addBytesProcessed` n) + +{- This is like L.hGetContents, but after each chunk is read, a meter + - is updated based on the size of the chunk. + - + - All the usual caveats about using unsafeInterleaveIO apply to the + - meter updates, so use caution. + -} +hGetContentsMetered :: Handle -> MeterUpdate -> IO L.ByteString +hGetContentsMetered h = hGetMetered h Nothing + +{- Reads from the Handle, updating the meter after each chunk is read. + - + - Stops at EOF, or when the requested number of bytes have been read. + - Closes the Handle at EOF, but otherwise leaves it open. + - + - Note that the meter update is run in unsafeInterleaveIO, which means that + - it can be run at any time. It's even possible for updates to run out + - of order, as different parts of the ByteString are consumed. + -} +hGetMetered :: Handle -> Maybe Integer -> MeterUpdate -> IO L.ByteString +hGetMetered h wantsize meterupdate = lazyRead zeroBytesProcessed + where + lazyRead sofar = unsafeInterleaveIO $ loop sofar + + loop sofar = do + c <- S.hGet h (nextchunksize (fromBytesProcessed sofar)) + if S.null c + then do + hClose h + return $ L.empty + else do + let !sofar' = addBytesProcessed sofar (S.length c) + meterupdate sofar' + if keepgoing (fromBytesProcessed sofar') + then do + {- unsafeInterleaveIO causes this to be + - deferred until the data is read from the + - ByteString. -} + cs <- lazyRead sofar' + return $ L.append (L.fromChunks [c]) cs + else return $ L.fromChunks [c] + + keepgoing n = case wantsize of + Nothing -> True + Just sz -> n < sz + + nextchunksize n = case wantsize of + Nothing -> defaultChunkSize + Just sz -> + let togo = sz - n + in if togo < toInteger defaultChunkSize + then fromIntegral togo + else defaultChunkSize + +{- Same default chunk size Lazy ByteStrings use. -} +defaultChunkSize :: Int +defaultChunkSize = 32 * k - chunkOverhead + where + k = 1024 + chunkOverhead = 2 * sizeOf (1 :: Int) -- GHC specific + +{- Runs an action, watching a file as it grows and updating the meter. + - + - The file may already exist, and the action could throw the original file + - away and start over. To avoid reporting the original file size followed + - by a smaller size in that case, wait until the file starts growing + - before updating the meter for the first time. + -} +watchFileSize :: (MonadIO m, MonadMask m) => FilePath -> MeterUpdate -> m a -> m a +watchFileSize f p a = bracket + (liftIO $ forkIO $ watcher =<< getsz) + (liftIO . void . tryIO . killThread) + (const a) + where + watcher oldsz = do + threadDelay 500000 -- 0.5 seconds + sz <- getsz + when (sz > oldsz) $ + p sz + watcher sz + getsz = catchDefaultIO zeroBytesProcessed $ + toBytesProcessed <$> getFileSize f + +data OutputHandler = OutputHandler + { quietMode :: Bool + , stderrHandler :: String -> IO () + } + +{- Parses the String looking for a command's progress output, and returns + - Maybe the number of bytes done so far, and any any remainder of the + - string that could be an incomplete progress output. That remainder + - should be prepended to future output, and fed back in. This interface + - allows the command's output to be read in any desired size chunk, or + - even one character at a time. + -} +type ProgressParser = String -> (Maybe BytesProcessed, String) + +{- Runs a command and runs a ProgressParser on its output, in order + - to update a meter. + -} +commandMeter :: ProgressParser -> OutputHandler -> MeterUpdate -> FilePath -> [CommandParam] -> IO Bool +commandMeter progressparser oh meterupdate cmd params = + outputFilter cmd params Nothing + (feedprogress zeroBytesProcessed []) + handlestderr + where + feedprogress prev buf h = do + b <- S.hGetSome h 80 + if S.null b + then return () + else do + unless (quietMode oh) $ do + S.hPut stdout b + hFlush stdout + let s = encodeW8 (S.unpack b) + let (mbytes, buf') = progressparser (buf++s) + case mbytes of + Nothing -> feedprogress prev buf' h + (Just bytes) -> do + when (bytes /= prev) $ + meterupdate bytes + feedprogress bytes buf' h + + handlestderr h = unlessM (hIsEOF h) $ do + stderrHandler oh =<< hGetLine h + handlestderr h + +{- Runs a command, that may display one or more progress meters on + - either stdout or stderr, and prevents the meters from being displayed. + - + - The other command output is handled as configured by the OutputHandler. + -} +demeterCommand :: OutputHandler -> FilePath -> [CommandParam] -> IO Bool +demeterCommand oh cmd params = demeterCommandEnv oh cmd params Nothing + +demeterCommandEnv :: OutputHandler -> FilePath -> [CommandParam] -> Maybe [(String, String)] -> IO Bool +demeterCommandEnv oh cmd params environ = outputFilter cmd params environ + (\outh -> avoidProgress True outh stdouthandler) + (\errh -> avoidProgress True errh $ stderrHandler oh) + where + stdouthandler l = + unless (quietMode oh) $ + putStrLn l + +{- To suppress progress output, while displaying other messages, + - filter out lines that contain \r (typically used to reset to the + - beginning of the line when updating a progress display). + -} +avoidProgress :: Bool -> Handle -> (String -> IO ()) -> IO () +avoidProgress doavoid h emitter = unlessM (hIsEOF h) $ do + s <- hGetLine h + unless (doavoid && '\r' `elem` s) $ + emitter s + avoidProgress doavoid h emitter + +outputFilter + :: FilePath + -> [CommandParam] + -> Maybe [(String, String)] + -> (Handle -> IO ()) + -> (Handle -> IO ()) + -> IO Bool +outputFilter cmd params environ outfilter errfilter = catchBoolIO $ do + (_, Just outh, Just errh, pid) <- createProcess p + { std_out = CreatePipe + , std_err = CreatePipe + } + void $ async $ tryIO (outfilter outh) >> hClose outh + void $ async $ tryIO (errfilter errh) >> hClose errh + ret <- checkSuccessProcess pid + return ret + where + p = (proc cmd (toCommand params)) + { env = environ } + +-- | Limit a meter to only update once per unit of time. +-- +-- It's nice to display the final update to 100%, even if it comes soon +-- after a previous update. To make that happen, the Meter has to know +-- its total size. +rateLimitMeterUpdate :: NominalDiffTime -> Meter -> MeterUpdate -> IO MeterUpdate +rateLimitMeterUpdate delta (Meter totalsizev _ _ _) meterupdate = do + lastupdate <- newMVar (toEnum 0 :: POSIXTime) + return $ mu lastupdate + where + mu lastupdate n@(BytesProcessed i) = readMVar totalsizev >>= \case + Just t | i >= t -> meterupdate n + _ -> do + now <- getPOSIXTime + prev <- takeMVar lastupdate + if now - prev >= delta + then do + putMVar lastupdate now + meterupdate n + else putMVar lastupdate prev + +data Meter = Meter (MVar (Maybe Integer)) (MVar MeterState) (MVar String) DisplayMeter + +type MeterState = (BytesProcessed, POSIXTime) + +type DisplayMeter = MVar String -> Maybe Integer -> (BytesProcessed, POSIXTime) -> (BytesProcessed, POSIXTime) -> IO () + +type RenderMeter = Maybe Integer -> (BytesProcessed, POSIXTime) -> (BytesProcessed, POSIXTime) -> String + +-- | Make a meter. Pass the total size, if it's known. +mkMeter :: Maybe Integer -> DisplayMeter -> IO Meter +mkMeter totalsize displaymeter = Meter + <$> newMVar totalsize + <*> ((\t -> newMVar (zeroBytesProcessed, t)) =<< getPOSIXTime) + <*> newMVar "" + <*> pure displaymeter + +setMeterTotalSize :: Meter -> Integer -> IO () +setMeterTotalSize (Meter totalsizev _ _ _) = void . swapMVar totalsizev . Just + +-- | Updates the meter, displaying it if necessary. +updateMeter :: Meter -> MeterUpdate +updateMeter (Meter totalsizev sv bv displaymeter) new = do + now <- getPOSIXTime + (old, before) <- swapMVar sv (new, now) + when (old /= new) $ do + totalsize <- readMVar totalsizev + displaymeter bv totalsize (old, before) (new, now) + +-- | Display meter to a Handle. +displayMeterHandle :: Handle -> RenderMeter -> DisplayMeter +displayMeterHandle h rendermeter v msize old new = do + let s = rendermeter msize old new + olds <- swapMVar v s + -- Avoid writing when the rendered meter has not changed. + when (olds /= s) $ do + let padding = replicate (length olds - length s) ' ' + hPutStr h ('\r':s ++ padding) + hFlush h + +-- | Clear meter displayed by displayMeterHandle. +clearMeterHandle :: Meter -> Handle -> IO () +clearMeterHandle (Meter _ _ v _) h = do + olds <- readMVar v + hPutStr h $ '\r' : replicate (length olds) ' ' ++ "\r" + hFlush h + +-- | Display meter in the form: +-- 10% 1.3MiB 300 KiB/s 16m40s +-- or when total size is not known: +-- 1.3 MiB 300 KiB/s +bandwidthMeter :: RenderMeter +bandwidthMeter mtotalsize (BytesProcessed old, before) (BytesProcessed new, now) = + unwords $ catMaybes + [ Just percentamount + -- Pad enough for max width: "100% xxxx.xx KiB xxxx KiB/s" + , Just $ replicate (29 - length percentamount - length rate) ' ' + , Just rate + , estimatedcompletion + ] + where + amount = roughSize' memoryUnits True 2 new + percentamount = case mtotalsize of + Just totalsize -> + let p = showPercentage 0 $ + percentage totalsize (min new totalsize) + in p ++ replicate (6 - length p) ' ' ++ amount + Nothing -> amount + rate = roughSize' memoryUnits True 0 bytespersecond ++ "/s" + bytespersecond + | duration == 0 = fromIntegral transferred + | otherwise = floor $ fromIntegral transferred / duration + transferred = max 0 (new - old) + duration = max 0 (now - before) + estimatedcompletion = case mtotalsize of + Just totalsize + | bytespersecond > 0 -> + Just $ fromDuration $ Duration $ + (totalsize - new) `div` bytespersecond + _ -> Nothing diff --git a/Utility/Misc.hs b/Utility/Misc.hs new file mode 100644 index 0000000000..48fcceb7e2 --- /dev/null +++ b/Utility/Misc.hs @@ -0,0 +1,112 @@ +{- misc utility functions + - + - Copyright 2010-2011 Joey Hess + - + - License: BSD-2-clause + -} + +{-# OPTIONS_GHC -fno-warn-tabs #-} + +module Utility.Misc where + +import System.IO +import Control.Monad +import Foreign +import Data.Char +import Data.List +import System.Exit +import Control.Applicative +import Prelude + +{- A version of hgetContents that is not lazy. Ensures file is + - all read before it gets closed. -} +hGetContentsStrict :: Handle -> IO String +hGetContentsStrict = hGetContents >=> \s -> length s `seq` return s + +{- A version of readFile that is not lazy. -} +readFileStrict :: FilePath -> IO String +readFileStrict = readFile >=> \s -> length s `seq` return s + +{- Like break, but the item matching the condition is not included + - in the second result list. + - + - separate (== ':') "foo:bar" = ("foo", "bar") + - separate (== ':') "foobar" = ("foobar", "") + -} +separate :: (a -> Bool) -> [a] -> ([a], [a]) +separate c l = unbreak $ break c l + where + unbreak r@(a, b) + | null b = r + | otherwise = (a, tail b) + +{- Breaks out the first line. -} +firstLine :: String -> String +firstLine = takeWhile (/= '\n') + +{- Splits a list into segments that are delimited by items matching + - a predicate. (The delimiters are not included in the segments.) + - Segments may be empty. -} +segment :: (a -> Bool) -> [a] -> [[a]] +segment p l = map reverse $ go [] [] l + where + go c r [] = reverse $ c:r + go c r (i:is) + | p i = go [] (c:r) is + | otherwise = go (i:c) r is + +prop_segment_regressionTest :: Bool +prop_segment_regressionTest = all id + -- Even an empty list is a segment. + [ segment (== "--") [] == [[]] + -- There are two segements in this list, even though the first is empty. + , segment (== "--") ["--", "foo", "bar"] == [[],["foo","bar"]] + ] + +{- Includes the delimiters as segments of their own. -} +segmentDelim :: (a -> Bool) -> [a] -> [[a]] +segmentDelim p l = map reverse $ go [] [] l + where + go c r [] = reverse $ c:r + go c r (i:is) + | p i = go [] ([i]:c:r) is + | otherwise = go (i:c) r is + +{- Replaces multiple values in a string. + - + - Takes care to skip over just-replaced values, so that they are not + - mangled. For example, massReplace [("foo", "new foo")] does not + - replace the "new foo" with "new new foo". + -} +massReplace :: [(String, String)] -> String -> String +massReplace vs = go [] vs + where + + go acc _ [] = concat $ reverse acc + go acc [] (c:cs) = go ([c]:acc) vs cs + go acc ((val, replacement):rest) s + | val `isPrefixOf` s = + go (replacement:acc) vs (drop (length val) s) + | otherwise = go acc rest s + +{- Wrapper around hGetBufSome that returns a String. + - + - The null string is returned on eof, otherwise returns whatever + - data is currently available to read from the handle, or waits for + - data to be written to it if none is currently available. + - + - Note on encodings: The normal encoding of the Handle is ignored; + - each byte is converted to a Char. Not unicode clean! + -} +hGetSomeString :: Handle -> Int -> IO String +hGetSomeString h sz = do + fp <- mallocForeignPtrBytes sz + len <- withForeignPtr fp $ \buf -> hGetBufSome h buf sz + map (chr . fromIntegral) <$> withForeignPtr fp (peekbytes len) + where + peekbytes :: Int -> Ptr Word8 -> IO [Word8] + peekbytes len buf = mapM (peekElemOff buf) [0..pred len] + +exitBool :: Bool -> IO a +exitBool False = exitFailure +exitBool True = exitSuccess diff --git a/Utility/Monad.hs b/Utility/Monad.hs new file mode 100644 index 0000000000..ac751043cd --- /dev/null +++ b/Utility/Monad.hs @@ -0,0 +1,71 @@ +{- monadic stuff + - + - Copyright 2010-2012 Joey Hess + - + - License: BSD-2-clause + -} + +{-# OPTIONS_GHC -fno-warn-tabs #-} + +module Utility.Monad where + +import Data.Maybe +import Control.Monad + +{- Return the first value from a list, if any, satisfying the given + - predicate -} +firstM :: Monad m => (a -> m Bool) -> [a] -> m (Maybe a) +firstM _ [] = return Nothing +firstM p (x:xs) = ifM (p x) (return $ Just x , firstM p xs) + +{- Runs the action on values from the list until it succeeds, returning + - its result. -} +getM :: Monad m => (a -> m (Maybe b)) -> [a] -> m (Maybe b) +getM _ [] = return Nothing +getM p (x:xs) = maybe (getM p xs) (return . Just) =<< p x + +{- Returns true if any value in the list satisfies the predicate, + - stopping once one is found. -} +anyM :: Monad m => (a -> m Bool) -> [a] -> m Bool +anyM p = liftM isJust . firstM p + +allM :: Monad m => (a -> m Bool) -> [a] -> m Bool +allM _ [] = return True +allM p (x:xs) = p x <&&> allM p xs + +{- Runs an action on values from a list until it succeeds. -} +untilTrue :: Monad m => [a] -> (a -> m Bool) -> m Bool +untilTrue = flip anyM + +{- if with a monadic conditional. -} +ifM :: Monad m => m Bool -> (m a, m a) -> m a +ifM cond (thenclause, elseclause) = do + c <- cond + if c then thenclause else elseclause + +{- short-circuiting monadic || -} +(<||>) :: Monad m => m Bool -> m Bool -> m Bool +ma <||> mb = ifM ma ( return True , mb ) + +{- short-circuiting monadic && -} +(<&&>) :: Monad m => m Bool -> m Bool -> m Bool +ma <&&> mb = ifM ma ( mb , return False ) + +{- Same fixity as && and || -} +infixr 3 <&&> +infixr 2 <||> + +{- Runs an action, passing its value to an observer before returning it. -} +observe :: Monad m => (a -> m b) -> m a -> m a +observe observer a = do + r <- a + _ <- observer r + return r + +{- b `after` a runs first a, then b, and returns the value of a -} +after :: Monad m => m b -> m a -> m a +after = observe . const + +{- do nothing -} +noop :: Monad m => m () +noop = return () diff --git a/Utility/Mounts.hs b/Utility/Mounts.hs new file mode 100644 index 0000000000..47b24b1639 --- /dev/null +++ b/Utility/Mounts.hs @@ -0,0 +1,26 @@ +{- portability shim for System.MountPoints + - + - Copyright 2016 Joey Hess + - + - License: BSD-2-clause + -} + +{-# LANGUAGE CPP #-} +{-# OPTIONS_GHC -fno-warn-tabs #-} + +module Utility.Mounts (getMounts, Mntent(..)) where + +import qualified System.MountPoints +import System.MountPoints (Mntent(..)) + +import Utility.Exception + +getMounts :: IO [Mntent] +#ifndef __ANDROID__ +getMounts = System.MountPoints.getMounts + -- That will crash when the linux build is running on Android, + -- so fall back to this. + `catchNonAsync` const System.MountPoints.getProcMounts +#else +getMounts = System.MountPoints.getProcMounts +#endif diff --git a/Utility/Network.hs b/Utility/Network.hs new file mode 100644 index 0000000000..4def3c5c55 --- /dev/null +++ b/Utility/Network.hs @@ -0,0 +1,24 @@ +{- network functions + - + - Copyright 2012 Joey Hess + - + - License: BSD-2-clause + -} + +{-# OPTIONS_GHC -fno-warn-tabs #-} + +module Utility.Network where + +import Utility.Process +import Utility.Exception + +import Control.Applicative +import Prelude + +{- Haskell lacks uname(2) bindings, except in the + - Bindings.Uname addon. Rather than depend on that, + - use uname -n when available. -} +getHostname :: IO (Maybe String) +getHostname = catchMaybeIO uname_node + where + uname_node = takeWhile (/= '\n') <$> readProcess "uname" ["-n"] diff --git a/Utility/NotificationBroadcaster.hs b/Utility/NotificationBroadcaster.hs new file mode 100644 index 0000000000..6f7cabf109 --- /dev/null +++ b/Utility/NotificationBroadcaster.hs @@ -0,0 +1,96 @@ +{- notification broadcaster + - + - This is used to allow clients to block until there is a new notification + - that some thing occurred. It does not communicate what the change is, + - it only provides blocking reads to wait on notifications. + - + - Multiple clients are supported. Each has a unique id. + - + - Copyright 2012 Joey Hess + - + - License: BSD-2-clause + -} + +module Utility.NotificationBroadcaster ( + NotificationBroadcaster, + NotificationHandle, + NotificationId, + newNotificationBroadcaster, + newNotificationHandle, + notificationHandleToId, + notificationHandleFromId, + sendNotification, + waitNotification, + checkNotification, +) where + +import Common + +import Control.Concurrent.STM + +{- One TMVar per client, which are empty when no notification is pending, + - and full when a notification has been sent but not yet seen by the + - client. The list TMVar is never empty, so never blocks. -} +type NotificationBroadcaster = TMVar [TMVar ()] + +newtype NotificationId = NotificationId Int + deriving (Read, Show, Eq, Ord) + +{- Handle given out to an individual client. -} +data NotificationHandle = NotificationHandle NotificationBroadcaster NotificationId + +newNotificationBroadcaster :: IO NotificationBroadcaster +newNotificationBroadcaster = atomically $ newTMVar [] + +{- Allocates a notification handle for a client to use. + - + - An immediate notification can be forced the first time waitNotification + - is called on the handle. This is useful in cases where a notification + - may be sent while the new handle is being constructed. Normally, + - such a notification would be missed. Forcing causes extra work, + - but ensures such notifications get seen. + -} +newNotificationHandle :: Bool -> NotificationBroadcaster -> IO NotificationHandle +newNotificationHandle force b = NotificationHandle + <$> pure b + <*> addclient + where + addclient = atomically $ do + s <- if force + then newTMVar () + else newEmptyTMVar + l <- takeTMVar b + putTMVar b $ l ++ [s] + return $ NotificationId $ length l + +{- Extracts the identifier from a notification handle. + - This can be used to eg, pass the identifier through to a WebApp. -} +notificationHandleToId :: NotificationHandle -> NotificationId +notificationHandleToId (NotificationHandle _ i) = i + +notificationHandleFromId :: NotificationBroadcaster -> NotificationId -> NotificationHandle +notificationHandleFromId = NotificationHandle + +{- Sends a notification to all clients. -} +sendNotification :: NotificationBroadcaster -> IO () +sendNotification b = do + l <- atomically $ readTMVar b + mapM_ notify l + where + notify s = atomically $ + whenM (isEmptyTMVar s) $ + putTMVar s () + +{- Used by a client to block until a new notification is available since + - the last time it tried. -} +waitNotification :: NotificationHandle -> IO () +waitNotification (NotificationHandle b (NotificationId i)) = do + l <- atomically $ readTMVar b + atomically $ takeTMVar (l !! i) + +{- Used by a client to check if there has been a new notification since the + - last time it checked, without blocking. -} +checkNotification :: NotificationHandle -> IO Bool +checkNotification (NotificationHandle b (NotificationId i)) = do + l <- atomically $ readTMVar b + maybe False (const True) <$> atomically (tryTakeTMVar (l !! i)) diff --git a/Utility/OSX.hs b/Utility/OSX.hs new file mode 100644 index 0000000000..f6aba50960 --- /dev/null +++ b/Utility/OSX.hs @@ -0,0 +1,46 @@ +{- OSX stuff + - + - Copyright 2012 Joey Hess + - + - License: BSD-2-clause + -} + +{-# OPTIONS_GHC -fno-warn-tabs #-} + +module Utility.OSX where + +import Utility.UserInfo + +import System.FilePath + +autoStartBase :: String -> FilePath +autoStartBase label = "Library" "LaunchAgents" label ++ ".plist" + +systemAutoStart :: String -> FilePath +systemAutoStart label = "/" autoStartBase label + +userAutoStart :: String -> IO FilePath +userAutoStart label = do + home <- myHomeDir + return $ home autoStartBase label + +{- Generates an OSX autostart plist file with a given label, command, and + - params to run at boot or login. -} +genOSXAutoStartFile :: String -> String -> [String] -> String +genOSXAutoStartFile label command params = unlines + [ "" + , "" + , "" + , "" + , "Label" + , "" ++ label ++ "" + , "ProgramArguments" + , "" + , unlines $ map (\v -> "" ++ v ++ "") (command:params) + , "" + , "RunAtLoad" + , "" + , "" + , "" + ] + diff --git a/Utility/OptParse.hs b/Utility/OptParse.hs new file mode 100644 index 0000000000..c65a18c249 --- /dev/null +++ b/Utility/OptParse.hs @@ -0,0 +1,46 @@ +{- optparse-applicative additions + - + - Copyright 2015 Joey Hess + - + - License: BSD-2-clause + -} + +module Utility.OptParse where + +import Options.Applicative +import Data.Monoid +import Prelude + +-- | A switch that can be enabled using --foo and disabled using --no-foo. +-- +-- The option modifier is applied to only the option that is *not* enabled +-- by default. For example: +-- +-- > invertableSwitch "recursive" True (help "do not recurse into directories") +-- +-- This example makes --recursive enabled by default, so +-- the help is shown only for --no-recursive. +invertableSwitch + :: String -- ^ long option + -> Bool -- ^ is switch enabled by default? + -> Mod FlagFields Bool -- ^ option modifier + -> Parser Bool +invertableSwitch longopt defv optmod = invertableSwitch' longopt defv + (if defv then mempty else optmod) + (if defv then optmod else mempty) + +-- | Allows providing option modifiers for both --foo and --no-foo. +invertableSwitch' + :: String -- ^ long option (eg "foo") + -> Bool -- ^ is switch enabled by default? + -> Mod FlagFields Bool -- ^ option modifier for --foo + -> Mod FlagFields Bool -- ^ option modifier for --no-foo + -> Parser Bool +invertableSwitch' longopt defv enmod dismod = collapse <$> many + ( flag' True (enmod <> long longopt) + <|> flag' False (dismod <> long nolongopt) + ) + where + nolongopt = "no-" ++ longopt + collapse [] = defv + collapse l = last l diff --git a/Utility/PID.hs b/Utility/PID.hs new file mode 100644 index 0000000000..f5f8aa873e --- /dev/null +++ b/Utility/PID.hs @@ -0,0 +1,30 @@ +{- process ids + - + - Copyright 2014 Joey Hess + - + - License: BSD-2-clause + -} + +{-# LANGUAGE CPP #-} + +module Utility.PID where + +#ifndef mingw32_HOST_OS +import System.Posix.Types (ProcessID) +import System.Posix.Process (getProcessID) +#else +import System.Win32.Process (ProcessId, getCurrentProcessId) +#endif + +#ifndef mingw32_HOST_OS +type PID = ProcessID +#else +type PID = ProcessId +#endif + +getPID :: IO PID +#ifndef mingw32_HOST_OS +getPID = getProcessID +#else +getPID = getCurrentProcessId +#endif diff --git a/Utility/Parallel.hs b/Utility/Parallel.hs new file mode 100644 index 0000000000..2352ba7068 --- /dev/null +++ b/Utility/Parallel.hs @@ -0,0 +1,34 @@ +{- parallel processing via threads + - + - Copyright 2012 Joey Hess + - + - License: BSD-2-clause + -} + +module Utility.Parallel where + +import Common + +import Control.Concurrent + +{- Runs an action in parallel with a set of values, in a set of threads. + - In order for the actions to truely run in parallel, requires GHC's + - threaded runtime, + - + - Returns the values partitioned into ones with which the action succeeded, + - and ones with which it failed. -} +inParallel :: (v -> IO Bool) -> [v] -> IO ([v], [v]) +inParallel a l = do + mvars <- mapM thread l + statuses <- mapM takeMVar mvars + return $ reduce $ partition snd $ zip l statuses + where + reduce (x,y) = (map fst x, map fst y) + thread v = do + mvar <- newEmptyMVar + _ <- forkIO $ do + r <- try (a v) :: IO (Either SomeException Bool) + case r of + Left _ -> putMVar mvar False + Right b -> putMVar mvar b + return mvar diff --git a/Utility/PartialPrelude.hs b/Utility/PartialPrelude.hs new file mode 100644 index 0000000000..5dbddad683 --- /dev/null +++ b/Utility/PartialPrelude.hs @@ -0,0 +1,67 @@ +{- Parts of the Prelude are partial functions, which are a common source of + - bugs. + - + - This exports functions that conflict with the prelude, which avoids + - them being accidentally used. + -} + +{-# OPTIONS_GHC -fno-warn-tabs #-} + +module Utility.PartialPrelude where + +import qualified Data.Maybe + +{- read should be avoided, as it throws an error + - Instead, use: readish -} +read :: Read a => String -> a +read = Prelude.read + +{- head is a partial function; head [] is an error + - Instead, use: take 1 or headMaybe -} +head :: [a] -> a +head = Prelude.head + +{- tail is also partial + - Instead, use: drop 1 -} +tail :: [a] -> [a] +tail = Prelude.tail + +{- init too + - Instead, use: beginning -} +init :: [a] -> [a] +init = Prelude.init + +{- last too + - Instead, use: end or lastMaybe -} +last :: [a] -> a +last = Prelude.last + +{- Attempts to read a value from a String. + - + - Unlike Text.Read.readMaybe, this ignores leading/trailing whitespace, + - and throws away any trailing text after the part that can be read. + -} +readish :: Read a => String -> Maybe a +readish s = case reads s of + ((x,_):_) -> Just x + _ -> Nothing + +{- Like head but Nothing on empty list. -} +headMaybe :: [a] -> Maybe a +headMaybe = Data.Maybe.listToMaybe + +{- Like last but Nothing on empty list. -} +lastMaybe :: [a] -> Maybe a +lastMaybe [] = Nothing +lastMaybe v = Just $ Prelude.last v + +{- All but the last element of a list. + - (Like init, but no error on an empty list.) -} +beginning :: [a] -> [a] +beginning [] = [] +beginning l = Prelude.init l + +{- Like last, but no error on an empty list. -} +end :: [a] -> [a] +end [] = [] +end l = [Prelude.last l] diff --git a/Utility/Path.hs b/Utility/Path.hs new file mode 100644 index 0000000000..f1302ae8ca --- /dev/null +++ b/Utility/Path.hs @@ -0,0 +1,272 @@ +{- path manipulation + - + - Copyright 2010-2014 Joey Hess + - + - License: BSD-2-clause + -} + +{-# LANGUAGE CPP #-} +{-# OPTIONS_GHC -fno-warn-tabs #-} + +module Utility.Path where + +import System.FilePath +import Data.List +import Data.Maybe +import Data.Char +import Control.Applicative +import Prelude + +import Utility.Monad +import Utility.UserInfo +import Utility.Directory +import Utility.Split + +{- Simplifies a path, removing any "." component, collapsing "dir/..", + - and removing the trailing path separator. + - + - On Windows, preserves whichever style of path separator might be used in + - the input FilePaths. This is done because some programs in Windows + - demand a particular path separator -- and which one actually varies! + - + - This does not guarantee that two paths that refer to the same location, + - and are both relative to the same location (or both absolute) will + - yeild the same result. Run both through normalise from System.FilePath + - to ensure that. + -} +simplifyPath :: FilePath -> FilePath +simplifyPath path = dropTrailingPathSeparator $ + joinDrive drive $ joinPath $ norm [] $ splitPath path' + where + (drive, path') = splitDrive path + + norm c [] = reverse c + norm c (p:ps) + | p' == ".." && not (null c) && dropTrailingPathSeparator (c !! 0) /= ".." = + norm (drop 1 c) ps + | p' == "." = norm c ps + | otherwise = norm (p:c) ps + where + p' = dropTrailingPathSeparator p + +{- Makes a path absolute. + - + - The first parameter is a base directory (ie, the cwd) to use if the path + - is not already absolute, and should itsef be absolute. + - + - Does not attempt to deal with edge cases or ensure security with + - untrusted inputs. + -} +absPathFrom :: FilePath -> FilePath -> FilePath +absPathFrom dir path = simplifyPath (combine dir path) + +{- takeDirectory "foo/bar/" is "foo/bar". This instead yields "foo" -} +parentDir :: FilePath -> FilePath +parentDir = takeDirectory . dropTrailingPathSeparator + +{- Just the parent directory of a path, or Nothing if the path has no +- parent (ie for "/" or ".") -} +upFrom :: FilePath -> Maybe FilePath +upFrom dir + | length dirs < 2 = Nothing + | otherwise = Just $ joinDrive drive $ intercalate s $ init dirs + where + -- on Unix, the drive will be "/" when the dir is absolute, + -- otherwise "" + (drive, path) = splitDrive dir + s = [pathSeparator] + dirs = filter (not . null) $ split s path + +prop_upFrom_basics :: FilePath -> Bool +prop_upFrom_basics dir + | null dir = True + | dir == "/" = p == Nothing + | otherwise = p /= Just dir + where + p = upFrom dir + +{- Checks if the first FilePath is, or could be said to contain the second. + - For example, "foo/" contains "foo/bar". Also, "foo", "./foo", "foo/" etc + - are all equivilant. + -} +dirContains :: FilePath -> FilePath -> Bool +dirContains a b = a == b || a' == b' || (addTrailingPathSeparator a') `isPrefixOf` b' + where + a' = norm a + b' = norm b + norm = normalise . simplifyPath + +{- Converts a filename into an absolute path. + - + - Unlike Directory.canonicalizePath, this does not require the path + - already exists. -} +absPath :: FilePath -> IO FilePath +absPath file = do + cwd <- getCurrentDirectory + return $ absPathFrom cwd file + +{- Constructs a relative path from the CWD to a file. + - + - For example, assuming CWD is /tmp/foo/bar: + - relPathCwdToFile "/tmp/foo" == ".." + - relPathCwdToFile "/tmp/foo/bar" == "" + -} +relPathCwdToFile :: FilePath -> IO FilePath +relPathCwdToFile f = do + c <- getCurrentDirectory + relPathDirToFile c f + +{- Constructs a relative path from a directory to a file. -} +relPathDirToFile :: FilePath -> FilePath -> IO FilePath +relPathDirToFile from to = relPathDirToFileAbs <$> absPath from <*> absPath to + +{- This requires the first path to be absolute, and the + - second path cannot contain ../ or ./ + - + - On Windows, if the paths are on different drives, + - a relative path is not possible and the path is simply + - returned as-is. + -} +relPathDirToFileAbs :: FilePath -> FilePath -> FilePath +relPathDirToFileAbs from to +#ifdef mingw32_HOST_OS + | normdrive from /= normdrive to = to +#endif + | otherwise = joinPath $ dotdots ++ uncommon + where + pfrom = sp from + pto = sp to + sp = map dropTrailingPathSeparator . splitPath . dropDrive + common = map fst $ takeWhile same $ zip pfrom pto + same (c,d) = c == d + uncommon = drop numcommon pto + dotdots = replicate (length pfrom - numcommon) ".." + numcommon = length common +#ifdef mingw32_HOST_OS + normdrive = map toLower . takeWhile (/= ':') . takeDrive +#endif + +prop_relPathDirToFile_basics :: FilePath -> FilePath -> Bool +prop_relPathDirToFile_basics from to + | null from || null to = True + | from == to = null r + | otherwise = not (null r) + where + r = relPathDirToFileAbs from to + +prop_relPathDirToFile_regressionTest :: Bool +prop_relPathDirToFile_regressionTest = same_dir_shortcurcuits_at_difference + where + {- Two paths have the same directory component at the same + - location, but it's not really the same directory. + - Code used to get this wrong. -} + same_dir_shortcurcuits_at_difference = + relPathDirToFileAbs (joinPath [pathSeparator : "tmp", "r", "lll", "xxx", "yyy", "18"]) + (joinPath [pathSeparator : "tmp", "r", ".git", "annex", "objects", "18", "gk", "SHA256-foo", "SHA256-foo"]) + == joinPath ["..", "..", "..", "..", ".git", "annex", "objects", "18", "gk", "SHA256-foo", "SHA256-foo"] + +{- Given an original list of paths, and an expanded list derived from it, + - which may be arbitrarily reordered, generates a list of lists, where + - each sublist corresponds to one of the original paths. + - + - When the original path is a directory, any items in the expanded list + - that are contained in that directory will appear in its segment. + - + - The order of the original list of paths is attempted to be preserved in + - the order of the returned segments. However, doing so has a O^NM + - growth factor. So, if the original list has more than 100 paths on it, + - we stop preserving ordering at that point. Presumably a user passing + - that many paths in doesn't care too much about order of the later ones. + -} +segmentPaths :: [FilePath] -> [FilePath] -> [[FilePath]] +segmentPaths [] new = [new] +segmentPaths [_] new = [new] -- optimisation +segmentPaths (l:ls) new = found : segmentPaths ls rest + where + (found, rest) = if length ls < 100 + then partition (l `dirContains`) new + else break (\p -> not (l `dirContains` p)) new + +{- This assumes that it's cheaper to call segmentPaths on the result, + - than it would be to run the action separately with each path. In + - the case of git file list commands, that assumption tends to hold. + -} +runSegmentPaths :: ([FilePath] -> IO [FilePath]) -> [FilePath] -> IO [[FilePath]] +runSegmentPaths a paths = segmentPaths paths <$> a paths + +{- Converts paths in the home directory to use ~/ -} +relHome :: FilePath -> IO String +relHome path = do + home <- myHomeDir + return $ if dirContains home path + then "~/" ++ relPathDirToFileAbs home path + else path + +{- Checks if a command is available in PATH. + - + - The command may be fully-qualified, in which case, this succeeds as + - long as it exists. -} +inPath :: String -> IO Bool +inPath command = isJust <$> searchPath command + +{- Finds a command in PATH and returns the full path to it. + - + - The command may be fully qualified already, in which case it will + - be returned if it exists. + - + - Note that this will find commands in PATH that are not executable. + -} +searchPath :: String -> IO (Maybe FilePath) +searchPath command + | isAbsolute command = check command + | otherwise = getSearchPath >>= getM indir + where + indir d = check $ d command + check f = firstM doesFileExist +#ifdef mingw32_HOST_OS + [f, f ++ ".exe"] +#else + [f] +#endif + +{- Checks if a filename is a unix dotfile. All files inside dotdirs + - count as dotfiles. -} +dotfile :: FilePath -> Bool +dotfile file + | f == "." = False + | f == ".." = False + | f == "" = False + | otherwise = "." `isPrefixOf` f || dotfile (takeDirectory file) + where + f = takeFileName file + +{- Given a string that we'd like to use as the basis for FilePath, but that + - was provided by a third party and is not to be trusted, returns the closest + - sane FilePath. + - + - All spaces and punctuation and other wacky stuff are replaced + - with '_', except for '.' + - "../" will thus turn into ".._", which is safe. + -} +sanitizeFilePath :: String -> FilePath +sanitizeFilePath = map sanitize + where + sanitize c + | c == '.' = c + | isSpace c || isPunctuation c || isSymbol c || isControl c || c == '/' = '_' + | otherwise = c + +{- Similar to splitExtensions, but knows that some things in FilePaths + - after a dot are too long to be extensions. -} +splitShortExtensions :: FilePath -> (FilePath, [String]) +splitShortExtensions = splitShortExtensions' 5 -- enough for ".jpeg" +splitShortExtensions' :: Int -> FilePath -> (FilePath, [String]) +splitShortExtensions' maxextension = go [] + where + go c f + | len > 0 && len <= maxextension && not (null base) = + go (ext:c) base + | otherwise = (f, c) + where + (base, ext) = splitExtension f + len = length ext diff --git a/Utility/Path/Max.hs b/Utility/Path/Max.hs new file mode 100644 index 0000000000..4a810e5911 --- /dev/null +++ b/Utility/Path/Max.hs @@ -0,0 +1,40 @@ +{- path manipulation + - + - Copyright 2010-2014 Joey Hess + - + - License: BSD-2-clause + -} + +{-# LANGUAGE CPP #-} +{-# OPTIONS_GHC -fno-warn-tabs #-} + +module Utility.Path.Max where + +import System.FilePath +import Data.List +import Control.Applicative +import Prelude + +#ifndef mingw32_HOST_OS +import Utility.Exception +import System.Posix.Files +#endif + +{- Maximum size to use for a file in a specified directory. + - + - Many systems have a 255 byte limit to the name of a file, + - so that's taken as the max if the system has a larger limit, or has no + - limit. + -} +fileNameLengthLimit :: FilePath -> IO Int +#ifdef mingw32_HOST_OS +fileNameLengthLimit _ = return 255 +#else +fileNameLengthLimit dir = do + -- getPathVar can fail due to statfs(2) overflow + l <- catchDefaultIO 0 $ + fromIntegral <$> getPathVar dir FileNameLimit + if l <= 0 + then return 255 + else return $ minimum [l, 255] +#endif diff --git a/Utility/Percentage.hs b/Utility/Percentage.hs new file mode 100644 index 0000000000..a30c26037f --- /dev/null +++ b/Utility/Percentage.hs @@ -0,0 +1,33 @@ +{- percentages + - + - Copyright 2012 Joey Hess + - + - License: BSD-2-clause + -} + +module Utility.Percentage ( + Percentage, + percentage, + showPercentage +) where + +import Data.Ratio + +import Utility.HumanNumber + +newtype Percentage = Percentage (Ratio Integer) + +instance Show Percentage where + show = showPercentage 0 + +{- Normally the big number comes first. But 110% is allowed if desired. :) -} +percentage :: Integer -> Integer -> Percentage +percentage 0 _ = Percentage 0 +percentage full have = Percentage $ have * 100 % full + +{- Pretty-print a Percentage, with a specified level of precision. -} +showPercentage :: Int -> Percentage -> String +showPercentage precision (Percentage p) = v ++ "%" + where + v = showImprecise precision n + n = fromRational p :: Double diff --git a/Utility/Process.hs b/Utility/Process.hs new file mode 100644 index 0000000000..c8b187abb2 --- /dev/null +++ b/Utility/Process.hs @@ -0,0 +1,331 @@ +{- System.Process enhancements, including additional ways of running + - processes, and logging. + - + - Copyright 2012-2015 Joey Hess + - + - License: BSD-2-clause + -} + +{-# LANGUAGE CPP, Rank2Types #-} +{-# OPTIONS_GHC -fno-warn-tabs #-} + +module Utility.Process ( + module X, + CreateProcess(..), + StdHandle(..), + readProcess, + readProcess', + readProcessEnv, + writeReadProcessEnv, + forceSuccessProcess, + forceSuccessProcess', + checkSuccessProcess, + ignoreFailureProcess, + createProcessSuccess, + createProcessChecked, + createBackgroundProcess, + withHandle, + withIOHandles, + withOEHandles, + withNullHandle, + withQuietOutput, + feedWithQuietOutput, + createProcess, + waitForProcess, + startInteractiveProcess, + stdinHandle, + stdoutHandle, + stderrHandle, + ioHandles, + processHandle, + devNull, +) where + +import qualified Utility.Process.Shim +import qualified Utility.Process.Shim as X hiding (CreateProcess(..), createProcess, runInteractiveProcess, readProcess, readProcessWithExitCode, system, rawSystem, runInteractiveCommand, runProcess) +import Utility.Process.Shim hiding (createProcess, readProcess, waitForProcess) +import Utility.Misc +import Utility.Exception + +import System.Exit +import System.IO +import System.Log.Logger +import Control.Concurrent +import qualified Control.Exception as E +import Control.Monad + +type CreateProcessRunner = forall a. CreateProcess -> ((Maybe Handle, Maybe Handle, Maybe Handle, ProcessHandle) -> IO a) -> IO a + +data StdHandle = StdinHandle | StdoutHandle | StderrHandle + deriving (Eq) + +-- | Normally, when reading from a process, it does not need to be fed any +-- standard input. +readProcess :: FilePath -> [String] -> IO String +readProcess cmd args = readProcessEnv cmd args Nothing + +readProcessEnv :: FilePath -> [String] -> Maybe [(String, String)] -> IO String +readProcessEnv cmd args environ = readProcess' p + where + p = (proc cmd args) + { std_out = CreatePipe + , env = environ + } + +readProcess' :: CreateProcess -> IO String +readProcess' p = withHandle StdoutHandle createProcessSuccess p $ \h -> do + output <- hGetContentsStrict h + hClose h + return output + +-- | Runs an action to write to a process on its stdin, +-- returns its output, and also allows specifying the environment. +writeReadProcessEnv + :: FilePath + -> [String] + -> Maybe [(String, String)] + -> (Maybe (Handle -> IO ())) + -> (Maybe (Handle -> IO ())) + -> IO String +writeReadProcessEnv cmd args environ writestdin adjusthandle = do + (Just inh, Just outh, _, pid) <- createProcess p + + maybe (return ()) (\a -> a inh) adjusthandle + maybe (return ()) (\a -> a outh) adjusthandle + + -- fork off a thread to start consuming the output + output <- hGetContents outh + outMVar <- newEmptyMVar + _ <- forkIO $ E.evaluate (length output) >> putMVar outMVar () + + -- now write and flush any input + maybe (return ()) (\a -> a inh >> hFlush inh) writestdin + hClose inh -- done with stdin + + -- wait on the output + takeMVar outMVar + hClose outh + + -- wait on the process + forceSuccessProcess p pid + + return output + + where + p = (proc cmd args) + { std_in = CreatePipe + , std_out = CreatePipe + , std_err = Inherit + , env = environ + } + +-- | Waits for a ProcessHandle, and throws an IOError if the process +-- did not exit successfully. +forceSuccessProcess :: CreateProcess -> ProcessHandle -> IO () +forceSuccessProcess p pid = waitForProcess pid >>= forceSuccessProcess' p + +forceSuccessProcess' :: CreateProcess -> ExitCode -> IO () +forceSuccessProcess' _ ExitSuccess = return () +forceSuccessProcess' p (ExitFailure n) = fail $ + showCmd p ++ " exited " ++ show n + +-- | Waits for a ProcessHandle and returns True if it exited successfully. +-- Note that using this with createProcessChecked will throw away +-- the Bool, and is only useful to ignore the exit code of a process, +-- while still waiting for it. -} +checkSuccessProcess :: ProcessHandle -> IO Bool +checkSuccessProcess pid = do + code <- waitForProcess pid + return $ code == ExitSuccess + +ignoreFailureProcess :: ProcessHandle -> IO Bool +ignoreFailureProcess pid = do + void $ waitForProcess pid + return True + +-- | Runs createProcess, then an action on its handles, and then +-- forceSuccessProcess. +createProcessSuccess :: CreateProcessRunner +createProcessSuccess p a = createProcessChecked (forceSuccessProcess p) p a + +-- | Runs createProcess, then an action on its handles, and then +-- a checker action on its exit code, which must wait for the process. +createProcessChecked :: (ProcessHandle -> IO b) -> CreateProcessRunner +createProcessChecked checker p a = do + t@(_, _, _, pid) <- createProcess p + r <- tryNonAsync $ a t + _ <- checker pid + either E.throw return r + +-- | Leaves the process running, suitable for lazy streaming. +-- Note: Zombies will result, and must be waited on. +createBackgroundProcess :: CreateProcessRunner +createBackgroundProcess p a = a =<< createProcess p + +-- | Runs a CreateProcessRunner, on a CreateProcess structure, that +-- is adjusted to pipe only from/to a single StdHandle, and passes +-- the resulting Handle to an action. +withHandle + :: StdHandle + -> CreateProcessRunner + -> CreateProcess + -> (Handle -> IO a) + -> IO a +withHandle h creator p a = creator p' $ a . select + where + base = p + { std_in = Inherit + , std_out = Inherit + , std_err = Inherit + } + (select, p') + | h == StdinHandle = + (stdinHandle, base { std_in = CreatePipe }) + | h == StdoutHandle = + (stdoutHandle, base { std_out = CreatePipe }) + | h == StderrHandle = + (stderrHandle, base { std_err = CreatePipe }) + +-- | Like withHandle, but passes (stdin, stdout) handles to the action. +withIOHandles + :: CreateProcessRunner + -> CreateProcess + -> ((Handle, Handle) -> IO a) + -> IO a +withIOHandles creator p a = creator p' $ a . ioHandles + where + p' = p + { std_in = CreatePipe + , std_out = CreatePipe + , std_err = Inherit + } + +-- | Like withHandle, but passes (stdout, stderr) handles to the action. +withOEHandles + :: CreateProcessRunner + -> CreateProcess + -> ((Handle, Handle) -> IO a) + -> IO a +withOEHandles creator p a = creator p' $ a . oeHandles + where + p' = p + { std_in = Inherit + , std_out = CreatePipe + , std_err = CreatePipe + } + +withNullHandle :: (Handle -> IO a) -> IO a +withNullHandle = withFile devNull WriteMode + +-- | Forces the CreateProcessRunner to run quietly; +-- both stdout and stderr are discarded. +withQuietOutput + :: CreateProcessRunner + -> CreateProcess + -> IO () +withQuietOutput creator p = withNullHandle $ \nullh -> do + let p' = p + { std_out = UseHandle nullh + , std_err = UseHandle nullh + } + creator p' $ const $ return () + +-- | Stdout and stderr are discarded, while the process is fed stdin +-- from the handle. +feedWithQuietOutput + :: CreateProcessRunner + -> CreateProcess + -> (Handle -> IO a) + -> IO a +feedWithQuietOutput creator p a = withFile devNull WriteMode $ \nullh -> do + let p' = p + { std_in = CreatePipe + , std_out = UseHandle nullh + , std_err = UseHandle nullh + } + creator p' $ a . stdinHandle + +devNull :: FilePath +#ifndef mingw32_HOST_OS +devNull = "/dev/null" +#else +-- Use device namespace to prevent GHC from rewriting path +devNull = "\\\\.\\NUL" +#endif + +-- | Extract a desired handle from createProcess's tuple. +-- These partial functions are safe as long as createProcess is run +-- with appropriate parameters to set up the desired handle. +-- Get it wrong and the runtime crash will always happen, so should be +-- easily noticed. +type HandleExtractor = (Maybe Handle, Maybe Handle, Maybe Handle, ProcessHandle) -> Handle +stdinHandle :: HandleExtractor +stdinHandle (Just h, _, _, _) = h +stdinHandle _ = error "expected stdinHandle" +stdoutHandle :: HandleExtractor +stdoutHandle (_, Just h, _, _) = h +stdoutHandle _ = error "expected stdoutHandle" +stderrHandle :: HandleExtractor +stderrHandle (_, _, Just h, _) = h +stderrHandle _ = error "expected stderrHandle" +ioHandles :: (Maybe Handle, Maybe Handle, Maybe Handle, ProcessHandle) -> (Handle, Handle) +ioHandles (Just hin, Just hout, _, _) = (hin, hout) +ioHandles _ = error "expected ioHandles" +oeHandles :: (Maybe Handle, Maybe Handle, Maybe Handle, ProcessHandle) -> (Handle, Handle) +oeHandles (_, Just hout, Just herr, _) = (hout, herr) +oeHandles _ = error "expected oeHandles" + +processHandle :: (Maybe Handle, Maybe Handle, Maybe Handle, ProcessHandle) -> ProcessHandle +processHandle (_, _, _, pid) = pid + +-- | Shows the command that a CreateProcess will run. +showCmd :: CreateProcess -> String +showCmd = go . cmdspec + where + go (ShellCommand s) = s + go (RawCommand c ps) = c ++ " " ++ show ps + +-- | Starts an interactive process. Unlike runInteractiveProcess in +-- System.Process, stderr is inherited. +startInteractiveProcess + :: FilePath + -> [String] + -> Maybe [(String, String)] + -> IO (ProcessHandle, Handle, Handle) +startInteractiveProcess cmd args environ = do + let p = (proc cmd args) + { std_in = CreatePipe + , std_out = CreatePipe + , std_err = Inherit + , env = environ + } + (Just from, Just to, _, pid) <- createProcess p + return (pid, to, from) + +-- | Wrapper around 'System.Process.createProcess' that does debug logging. +createProcess :: CreateProcess -> IO (Maybe Handle, Maybe Handle, Maybe Handle, ProcessHandle) +createProcess p = do + debugProcess p + Utility.Process.Shim.createProcess p + +-- | Debugging trace for a CreateProcess. +debugProcess :: CreateProcess -> IO () +debugProcess p = debugM "Utility.Process" $ unwords + [ action ++ ":" + , showCmd p + ] + where + action + | piped (std_in p) && piped (std_out p) = "chat" + | piped (std_in p) = "feed" + | piped (std_out p) = "read" + | otherwise = "call" + piped Inherit = False + piped _ = True + +-- | Wrapper around 'System.Process.waitForProcess' that does debug logging. +waitForProcess :: ProcessHandle -> IO ExitCode +waitForProcess h = do + r <- Utility.Process.Shim.waitForProcess h + debugM "Utility.Process" ("process done " ++ show r) + return r diff --git a/Utility/Process/Shim.hs b/Utility/Process/Shim.hs new file mode 100644 index 0000000000..09312c7f79 --- /dev/null +++ b/Utility/Process/Shim.hs @@ -0,0 +1,3 @@ +module Utility.Process.Shim (module X) where + +import System.Process as X diff --git a/Utility/Process/Transcript.hs b/Utility/Process/Transcript.hs new file mode 100644 index 0000000000..68fb2223eb --- /dev/null +++ b/Utility/Process/Transcript.hs @@ -0,0 +1,83 @@ +{- Process transcript + - + - Copyright 2012-2018 Joey Hess + - + - License: BSD-2-clause + -} + +{-# LANGUAGE CPP #-} +{-# OPTIONS_GHC -fno-warn-tabs #-} + +module Utility.Process.Transcript where + +import Utility.Process +import Utility.Misc + +import System.IO +import System.Exit +import Control.Concurrent.Async +import Control.Monad +#ifndef mingw32_HOST_OS +import qualified System.Posix.IO +#else +import Control.Applicative +#endif +import Data.Maybe +import Prelude + +-- | Runs a process and returns a transcript combining its stdout and +-- stderr, and whether it succeeded or failed. +processTranscript :: String -> [String] -> (Maybe String) -> IO (String, Bool) +processTranscript cmd opts = processTranscript' (proc cmd opts) + +-- | Also feeds the process some input. +processTranscript' :: CreateProcess -> Maybe String -> IO (String, Bool) +processTranscript' cp input = do + (t, c) <- processTranscript'' cp input + return (t, c == ExitSuccess) + +processTranscript'' :: CreateProcess -> Maybe String -> IO (String, ExitCode) +processTranscript'' cp input = do +#ifndef mingw32_HOST_OS +{- This implementation interleves stdout and stderr in exactly the order + - the process writes them. -} + (readf, writef) <- System.Posix.IO.createPipe + System.Posix.IO.setFdOption readf System.Posix.IO.CloseOnExec True + System.Posix.IO.setFdOption writef System.Posix.IO.CloseOnExec True + readh <- System.Posix.IO.fdToHandle readf + writeh <- System.Posix.IO.fdToHandle writef + p@(_, _, _, pid) <- createProcess $ cp + { std_in = if isJust input then CreatePipe else Inherit + , std_out = UseHandle writeh + , std_err = UseHandle writeh + } + hClose writeh + + get <- asyncreader readh + writeinput input p + transcript <- wait get +#else +{- This implementation for Windows puts stderr after stdout. -} + p@(_, _, _, pid) <- createProcess $ cp + { std_in = if isJust input then CreatePipe else Inherit + , std_out = CreatePipe + , std_err = CreatePipe + } + + getout <- asyncreader (stdoutHandle p) + geterr <- asyncreader (stderrHandle p) + writeinput input p + transcript <- (++) <$> wait getout <*> wait geterr +#endif + code <- waitForProcess pid + return (transcript, code) + where + asyncreader = async . hGetContentsStrict + + writeinput (Just s) p = do + let inh = stdinHandle p + unless (null s) $ do + hPutStr inh s + hFlush inh + hClose inh + writeinput Nothing _ = return () diff --git a/Utility/QuickCheck.hs b/Utility/QuickCheck.hs new file mode 100644 index 0000000000..e89d103dd8 --- /dev/null +++ b/Utility/QuickCheck.hs @@ -0,0 +1,54 @@ +{- QuickCheck with additional instances + - + - Copyright 2012-2014 Joey Hess + - + - License: BSD-2-clause + -} + +{-# OPTIONS_GHC -fno-warn-orphans #-} +{-# LANGUAGE TypeSynonymInstances, CPP #-} + +module Utility.QuickCheck + ( module X + , module Utility.QuickCheck + ) where + +import Test.QuickCheck as X +import Data.Time.Clock.POSIX +import System.Posix.Types +#if ! MIN_VERSION_QuickCheck(2,8,2) +import qualified Data.Map as M +import qualified Data.Set as S +#endif +import Control.Applicative +import Prelude + +#if ! MIN_VERSION_QuickCheck(2,8,2) +instance (Arbitrary k, Arbitrary v, Ord k) => Arbitrary (M.Map k v) where + arbitrary = M.fromList <$> arbitrary + +instance (Arbitrary v, Ord v) => Arbitrary (S.Set v) where + arbitrary = S.fromList <$> arbitrary +#endif + +{- Times before the epoch are excluded. -} +instance Arbitrary POSIXTime where + arbitrary = fromInteger <$> nonNegative arbitrarySizedIntegral + +{- Pids are never negative, or 0. -} +instance Arbitrary ProcessID where + arbitrary = arbitrarySizedBoundedIntegral `suchThat` (> 0) + +{- Inodes are never negative. -} +instance Arbitrary FileID where + arbitrary = nonNegative arbitrarySizedIntegral + +{- File sizes are never negative. -} +instance Arbitrary FileOffset where + arbitrary = nonNegative arbitrarySizedIntegral + +nonNegative :: (Num a, Ord a) => Gen a -> Gen a +nonNegative g = g `suchThat` (>= 0) + +positive :: (Num a, Ord a) => Gen a -> Gen a +positive g = g `suchThat` (> 0) diff --git a/Utility/Rsync.hs b/Utility/Rsync.hs new file mode 100644 index 0000000000..25af52617d --- /dev/null +++ b/Utility/Rsync.hs @@ -0,0 +1,171 @@ +{- various rsync stuff + - + - Copyright 2010-2013 Joey Hess + - + - License: BSD-2-clause + -} + +{-# LANGUAGE CPP #-} + +module Utility.Rsync where + +import Common +import Utility.Metered +import Utility.Tuple + +#ifdef mingw32_HOST_OS +import qualified System.FilePath.Posix as Posix +#endif + +import Data.Char +import System.Console.GetOpt + +{- Generates parameters to make rsync use a specified command as its remote + - shell. -} +rsyncShell :: [CommandParam] -> [CommandParam] +rsyncShell command = [Param "-e", Param $ unwords $ map escape (toCommand command)] + where + {- rsync requires some weird, non-shell like quoting in + - here. A doubled single quote inside the single quoted + - string is a single quote. -} + escape s = "'" ++ intercalate "''" (splitc '\'' s) ++ "'" + +{- Runs rsync in server mode to send a file. -} +rsyncServerSend :: [CommandParam] -> FilePath -> IO Bool +rsyncServerSend options file = rsync $ + rsyncServerParams ++ Param "--sender" : options ++ [File file] + +{- Runs rsync in server mode to receive a file. -} +rsyncServerReceive :: [CommandParam] -> FilePath -> IO Bool +rsyncServerReceive options file = rsync $ + rsyncServerParams ++ options ++ [File file] + +rsyncServerParams :: [CommandParam] +rsyncServerParams = + [ Param "--server" + -- preserve timestamps + , Param "-t" + -- allow resuming of transfers of big files + , Param "--inplace" + -- other options rsync normally uses in server mode + , Param "-e.Lsf" + , Param "." + ] + +rsyncUseDestinationPermissions :: CommandParam +rsyncUseDestinationPermissions = Param "--chmod=ugo=rwX" + +rsync :: [CommandParam] -> IO Bool +rsync = boolSystem "rsync" . rsyncParamsFixup + +{- On Windows, rsync is from msys2, and expects to get msys2 formatted + - paths to files. (It thinks that C:foo refers to a host named "C"). + - Fix up the Params appropriately. -} +rsyncParamsFixup :: [CommandParam] -> [CommandParam] +#ifdef mingw32_HOST_OS +rsyncParamsFixup = map fixup + where + fixup (File f) = File (toMSYS2Path f) + fixup (Param s) + | rsyncUrlIsPath s = Param (toMSYS2Path s) + fixup p = p +#else +rsyncParamsFixup = id +#endif + +{- Checks if an rsync url involves the remote shell (ssh or rsh). + - Use of such urls with rsync requires additional shell + - escaping. -} +rsyncUrlIsShell :: String -> Bool +rsyncUrlIsShell s + | "rsync://" `isPrefixOf` s = False + | otherwise = go s + where + -- host::dir is rsync protocol, while host:dir is ssh/rsh + go [] = False + go (c:cs) + | c == '/' = False -- got to directory with no colon + | c == ':' = not $ ":" `isPrefixOf` cs + | otherwise = go cs + +{- Checks if a rsync url is really just a local path. -} +rsyncUrlIsPath :: String -> Bool +rsyncUrlIsPath s +#ifdef mingw32_HOST_OS + | not (null (takeDrive s)) = True +#endif + | rsyncUrlIsShell s = False + | otherwise = ':' `notElem` s + +{- Runs rsync, but intercepts its progress output and updates a progress + - meter. + - + - The params must enable rsync's --progress mode for this to work. + -} +rsyncProgress :: OutputHandler -> MeterUpdate -> [CommandParam] -> IO Bool +rsyncProgress oh meter = commandMeter parseRsyncProgress oh meter "rsync" . rsyncParamsFixup + +{- Strategy: Look for chunks prefixed with \r (rsync writes a \r before + - the first progress output, and each thereafter). The first number + - after the \r is the number of bytes processed. After the number, + - there must appear some whitespace, or we didn't get the whole number, + - and return the \r and part we did get, for later processing. + - + - In some locales, the number will have one or more commas in the middle + - of it. + -} +parseRsyncProgress :: ProgressParser +parseRsyncProgress = go [] . reverse . progresschunks + where + go remainder [] = (Nothing, remainder) + go remainder (x:xs) = case parsebytes (findbytesstart x) of + Nothing -> go (delim:x++remainder) xs + Just b -> (Just (toBytesProcessed b), remainder) + + delim = '\r' + + {- Find chunks that each start with delim. + - The first chunk doesn't start with it + - (it's empty when delim is at the start of the string). -} + progresschunks = drop 1 . splitc delim + findbytesstart s = dropWhile isSpace s + + parsebytes :: String -> Maybe Integer + parsebytes s = case break isSpace s of + ([], _) -> Nothing + (_, []) -> Nothing + (b, _) -> readish $ filter (/= ',') b + +{- Filters options to those that are safe to pass to rsync in server mode, + - without causing it to eg, expose files. -} +filterRsyncSafeOptions :: [String] -> [String] +filterRsyncSafeOptions = fst3 . getOpt Permute + [ Option [] ["bwlimit"] (reqArgLong "bwlimit") "" ] + where + reqArgLong x = ReqArg (\v -> "--" ++ x ++ "=" ++ v) "" + +{- Converts a DOS style path to a msys2 style path. Only on Windows. + - Any trailing '\' is preserved as a trailing '/' + - + - Taken from: http://sourceforge.net/p/msys2/wiki/MSYS2%20introduction/i + - + - The virtual filesystem contains: + - /c, /d, ... mount points for Windows drives + -} +toMSYS2Path :: FilePath -> FilePath +#ifndef mingw32_HOST_OS +toMSYS2Path = id +#else +toMSYS2Path p + | null drive = recombine parts + | otherwise = recombine $ "/" : driveletter drive : parts + where + (drive, p') = splitDrive p + parts = splitDirectories p' + driveletter = map toLower . takeWhile (/= ':') + recombine = fixtrailing . Posix.joinPath + fixtrailing s + | hasTrailingPathSeparator p = Posix.addTrailingPathSeparator s + | otherwise = s +#endif + diff --git a/Utility/SafeCommand.hs b/Utility/SafeCommand.hs new file mode 100644 index 0000000000..f820e69f19 --- /dev/null +++ b/Utility/SafeCommand.hs @@ -0,0 +1,138 @@ +{- safely running shell commands + - + - Copyright 2010-2015 Joey Hess + - + - License: BSD-2-clause + -} + +{-# OPTIONS_GHC -fno-warn-tabs #-} + +module Utility.SafeCommand where + +import System.Exit +import Utility.Process +import Utility.Split +import System.FilePath +import Data.Char +import Data.List +import Control.Applicative +import Prelude + +-- | Parameters that can be passed to a shell command. +data CommandParam + = Param String -- ^ A parameter + | File FilePath -- ^ The name of a file + deriving (Eq, Show, Ord) + +-- | Used to pass a list of CommandParams to a function that runs +-- a command and expects Strings. -} +toCommand :: [CommandParam] -> [String] +toCommand = map toCommand' + +toCommand' :: CommandParam -> String +toCommand' (Param s) = s +-- Files that start with a non-alphanumeric that is not a path +-- separator are modified to avoid the command interpreting them as +-- options or other special constructs. +toCommand' (File s@(h:_)) + | isAlphaNum h || h `elem` pathseps = s + | otherwise = "./" ++ s + where + -- '/' is explicitly included because it's an alternative + -- path separator on Windows. + pathseps = pathSeparator:"./" +toCommand' (File s) = s + +-- | Run a system command, and returns True or False if it succeeded or failed. +-- +-- This and other command running functions in this module log the commands +-- run at debug level, using System.Log.Logger. +boolSystem :: FilePath -> [CommandParam] -> IO Bool +boolSystem command params = boolSystem' command params id + +boolSystem' :: FilePath -> [CommandParam] -> (CreateProcess -> CreateProcess) -> IO Bool +boolSystem' command params mkprocess = dispatch <$> safeSystem' command params mkprocess + where + dispatch ExitSuccess = True + dispatch _ = False + +boolSystemEnv :: FilePath -> [CommandParam] -> Maybe [(String, String)] -> IO Bool +boolSystemEnv command params environ = boolSystem' command params $ + \p -> p { env = environ } + +-- | Runs a system command, returning the exit status. +safeSystem :: FilePath -> [CommandParam] -> IO ExitCode +safeSystem command params = safeSystem' command params id + +safeSystem' :: FilePath -> [CommandParam] -> (CreateProcess -> CreateProcess) -> IO ExitCode +safeSystem' command params mkprocess = do + (_, _, _, pid) <- createProcess p + waitForProcess pid + where + p = mkprocess $ proc command (toCommand params) + +safeSystemEnv :: FilePath -> [CommandParam] -> Maybe [(String, String)] -> IO ExitCode +safeSystemEnv command params environ = safeSystem' command params $ + \p -> p { env = environ } + +-- | Wraps a shell command line inside sh -c, allowing it to be run in a +-- login shell that may not support POSIX shell, eg csh. +shellWrap :: String -> String +shellWrap cmdline = "sh -c " ++ shellEscape cmdline + +-- | Escapes a filename or other parameter to be safely able to be exposed to +-- the shell. +-- +-- This method works for POSIX shells, as well as other shells like csh. +shellEscape :: String -> String +shellEscape f = "'" ++ escaped ++ "'" + where + -- replace ' with '"'"' + escaped = intercalate "'\"'\"'" $ splitc '\'' f + +-- | Unescapes a set of shellEscaped words or filenames. +shellUnEscape :: String -> [String] +shellUnEscape [] = [] +shellUnEscape s = word : shellUnEscape rest + where + (word, rest) = findword "" s + findword w [] = (w, "") + findword w (c:cs) + | c == ' ' = (w, cs) + | c == '\'' = inquote c w cs + | c == '"' = inquote c w cs + | otherwise = findword (w++[c]) cs + inquote _ w [] = (w, "") + inquote q w (c:cs) + | c == q = findword w cs + | otherwise = inquote q (w++[c]) cs + +-- | For quickcheck. +prop_isomorphic_shellEscape :: String -> Bool +prop_isomorphic_shellEscape s = [s] == (shellUnEscape . shellEscape) s +prop_isomorphic_shellEscape_multiword :: [String] -> Bool +prop_isomorphic_shellEscape_multiword s = s == (shellUnEscape . unwords . map shellEscape) s + +-- | Segments a list of filenames into groups that are all below the maximum +-- command-line length limit. +segmentXargsOrdered :: [FilePath] -> [[FilePath]] +segmentXargsOrdered = reverse . map reverse . segmentXargsUnordered + +-- | Not preserving order is a little faster, and streams better when +-- there are a great many filenames. +segmentXargsUnordered :: [FilePath] -> [[FilePath]] +segmentXargsUnordered l = go l [] 0 [] + where + go [] c _ r = (c:r) + go (f:fs) c accumlen r + | newlen > maxlen && len < maxlen = go (f:fs) [] 0 (c:r) + | otherwise = go fs (f:c) newlen r + where + len = length f + newlen = accumlen + len + + {- 10k of filenames per command, well under 100k limit + - of Linux (and OSX has a similar limit); + - allows room for other parameters etc. Also allows for + - eg, multibyte characters. -} + maxlen = 10240 diff --git a/Utility/Scheduled.hs b/Utility/Scheduled.hs new file mode 100644 index 0000000000..b68ff901ca --- /dev/null +++ b/Utility/Scheduled.hs @@ -0,0 +1,361 @@ +{- scheduled activities + - + - Copyright 2013-2014 Joey Hess + - + - License: BSD-2-clause + -} + +module Utility.Scheduled ( + Schedule(..), + Recurrance(..), + ScheduledTime(..), + NextTime(..), + WeekDay, + MonthDay, + YearDay, + nextTime, + calcNextTime, + startTime, + fromSchedule, + fromScheduledTime, + toScheduledTime, + fromRecurrance, + toRecurrance, + toSchedule, + parseSchedule, + prop_past_sane, +) where + +import Utility.Data +import Utility.PartialPrelude +import Utility.Misc +import Utility.Tuple + +import Data.List +import Data.Time.Clock +import Data.Time.LocalTime +import Data.Time.Calendar +import Data.Time.Calendar.WeekDate +import Data.Time.Calendar.OrdinalDate +import Data.Time.Format () +import Data.Char +import Control.Applicative +import Prelude + +{- Some sort of scheduled event. -} +data Schedule = Schedule Recurrance ScheduledTime + deriving (Eq, Read, Show, Ord) + +data Recurrance + = Daily + | Weekly (Maybe WeekDay) + | Monthly (Maybe MonthDay) + | Yearly (Maybe YearDay) + | Divisible Int Recurrance + -- ^ Days, Weeks, or Months of the year evenly divisible by a number. + -- (Divisible Year is years evenly divisible by a number.) + deriving (Eq, Read, Show, Ord) + +type WeekDay = Int +type MonthDay = Int +type YearDay = Int + +data ScheduledTime + = AnyTime + | SpecificTime Hour Minute + deriving (Eq, Read, Show, Ord) + +type Hour = Int +type Minute = Int + +-- | Next time a Schedule should take effect. The NextTimeWindow is used +-- when a Schedule is allowed to start at some point within the window. +data NextTime + = NextTimeExactly LocalTime + | NextTimeWindow LocalTime LocalTime + deriving (Eq, Read, Show) + +startTime :: NextTime -> LocalTime +startTime (NextTimeExactly t) = t +startTime (NextTimeWindow t _) = t + +nextTime :: Schedule -> Maybe LocalTime -> IO (Maybe NextTime) +nextTime schedule lasttime = do + now <- getCurrentTime + tz <- getTimeZone now + return $ calcNextTime schedule lasttime $ utcToLocalTime tz now + +-- | Calculate the next time that fits a Schedule, based on the +-- last time it occurred, and the current time. +calcNextTime :: Schedule -> Maybe LocalTime -> LocalTime -> Maybe NextTime +calcNextTime schedule@(Schedule recurrance scheduledtime) lasttime currenttime + | scheduledtime == AnyTime = do + next <- findfromtoday True + return $ case next of + NextTimeWindow _ _ -> next + NextTimeExactly t -> window (localDay t) (localDay t) + | otherwise = NextTimeExactly . startTime <$> findfromtoday False + where + findfromtoday anytime = findfrom recurrance afterday today + where + today = localDay currenttime + afterday = sameaslastrun || toolatetoday + toolatetoday = not anytime && localTimeOfDay currenttime >= nexttime + sameaslastrun = lastrun == Just today + lastrun = localDay <$> lasttime + nexttime = case scheduledtime of + AnyTime -> TimeOfDay 0 0 0 + SpecificTime h m -> TimeOfDay h m 0 + exactly d = NextTimeExactly $ LocalTime d nexttime + window startd endd = NextTimeWindow + (LocalTime startd nexttime) + (LocalTime endd (TimeOfDay 23 59 0)) + findfrom r afterday candidate + | ynum candidate > (ynum (localDay currenttime)) + 100 = + -- avoid possible infinite recusion + error $ "bug: calcNextTime did not find a time within 100 years to run " ++ + show (schedule, lasttime, currenttime) + | otherwise = findfromChecked r afterday candidate + findfromChecked r afterday candidate = case r of + Daily + | afterday -> Just $ exactly $ addDays 1 candidate + | otherwise -> Just $ exactly candidate + Weekly Nothing + | afterday -> skip 1 + | otherwise -> case (wday <$> lastrun, wday candidate) of + (Nothing, _) -> Just $ window candidate (addDays 6 candidate) + (Just old, curr) + | old == curr -> Just $ window candidate (addDays 6 candidate) + | otherwise -> skip 1 + Monthly Nothing + | afterday -> skip 1 + | maybe True (candidate `oneMonthPast`) lastrun -> + Just $ window candidate (endOfMonth candidate) + | otherwise -> skip 1 + Yearly Nothing + | afterday -> skip 1 + | maybe True (candidate `oneYearPast`) lastrun -> + Just $ window candidate (endOfYear candidate) + | otherwise -> skip 1 + Weekly (Just w) + | w < 0 || w > maxwday -> Nothing + | w == wday candidate -> if afterday + then Just $ exactly $ addDays 7 candidate + else Just $ exactly candidate + | otherwise -> Just $ exactly $ + addDays (fromIntegral $ (w - wday candidate) `mod` 7) candidate + Monthly (Just m) + | m < 0 || m > maxmday -> Nothing + -- TODO can be done more efficiently than recursing + | m == mday candidate -> if afterday + then skip 1 + else Just $ exactly candidate + | otherwise -> skip 1 + Yearly (Just y) + | y < 0 || y > maxyday -> Nothing + | y == yday candidate -> if afterday + then skip 365 + else Just $ exactly candidate + | otherwise -> skip 1 + Divisible n r'@Daily -> handlediv n r' yday (Just maxyday) + Divisible n r'@(Weekly _) -> handlediv n r' wnum (Just maxwnum) + Divisible n r'@(Monthly _) -> handlediv n r' mnum (Just maxmnum) + Divisible n r'@(Yearly _) -> handlediv n r' ynum Nothing + Divisible _ r'@(Divisible _ _) -> findfrom r' afterday candidate + where + skip n = findfrom r False (addDays n candidate) + handlediv n r' getval mmax + | n > 0 && maybe True (n <=) mmax = + findfromwhere r' (divisible n . getval) afterday candidate + | otherwise = Nothing + findfromwhere r p afterday candidate + | maybe True (p . getday) next = next + | otherwise = maybe Nothing (findfromwhere r p True . getday) next + where + next = findfrom r afterday candidate + getday = localDay . startTime + divisible n v = v `rem` n == 0 + +-- Check if the new Day occurs one month or more past the old Day. +oneMonthPast :: Day -> Day -> Bool +new `oneMonthPast` old = fromGregorian y (m+1) d <= new + where + (y,m,d) = toGregorian old + +-- Check if the new Day occurs one year or more past the old Day. +oneYearPast :: Day -> Day -> Bool +new `oneYearPast` old = fromGregorian (y+1) m d <= new + where + (y,m,d) = toGregorian old + +endOfMonth :: Day -> Day +endOfMonth day = + let (y,m,_d) = toGregorian day + in fromGregorian y m (gregorianMonthLength y m) + +endOfYear :: Day -> Day +endOfYear day = + let (y,_m,_d) = toGregorian day + in endOfMonth (fromGregorian y maxmnum 1) + +-- extracting various quantities from a Day +wday :: Day -> Int +wday = thd3 . toWeekDate +wnum :: Day -> Int +wnum = snd3 . toWeekDate +mday :: Day -> Int +mday = thd3 . toGregorian +mnum :: Day -> Int +mnum = snd3 . toGregorian +yday :: Day -> Int +yday = snd . toOrdinalDate +ynum :: Day -> Int +ynum = fromIntegral . fst . toOrdinalDate + +-- Calendar max values. +maxyday :: Int +maxyday = 366 -- with leap days +maxwnum :: Int +maxwnum = 53 -- some years have more than 52 +maxmday :: Int +maxmday = 31 +maxmnum :: Int +maxmnum = 12 +maxwday :: Int +maxwday = 7 + +fromRecurrance :: Recurrance -> String +fromRecurrance (Divisible n r) = + fromRecurrance' (++ "s divisible by " ++ show n) r +fromRecurrance r = fromRecurrance' ("every " ++) r + +fromRecurrance' :: (String -> String) -> Recurrance -> String +fromRecurrance' a Daily = a "day" +fromRecurrance' a (Weekly n) = onday n (a "week") +fromRecurrance' a (Monthly n) = onday n (a "month") +fromRecurrance' a (Yearly n) = onday n (a "year") +fromRecurrance' a (Divisible _n r) = fromRecurrance' a r -- not used + +onday :: Maybe Int -> String -> String +onday (Just n) s = "on day " ++ show n ++ " of " ++ s +onday Nothing s = s + +toRecurrance :: String -> Maybe Recurrance +toRecurrance s = case words s of + ("every":"day":[]) -> Just Daily + ("on":"day":sd:"of":"every":something:[]) -> withday sd something + ("every":something:[]) -> noday something + ("days":"divisible":"by":sn:[]) -> + Divisible <$> getdivisor sn <*> pure Daily + ("on":"day":sd:"of":something:"divisible":"by":sn:[]) -> + Divisible + <$> getdivisor sn + <*> withday sd something + ("every":something:"divisible":"by":sn:[]) -> + Divisible + <$> getdivisor sn + <*> noday something + (something:"divisible":"by":sn:[]) -> + Divisible + <$> getdivisor sn + <*> noday something + _ -> Nothing + where + constructor "week" = Just Weekly + constructor "month" = Just Monthly + constructor "year" = Just Yearly + constructor u + | "s" `isSuffixOf` u = constructor $ reverse $ drop 1 $ reverse u + | otherwise = Nothing + withday sd u = do + c <- constructor u + d <- readish sd + Just $ c (Just d) + noday u = do + c <- constructor u + Just $ c Nothing + getdivisor sn = do + n <- readish sn + if n > 0 + then Just n + else Nothing + +fromScheduledTime :: ScheduledTime -> String +fromScheduledTime AnyTime = "any time" +fromScheduledTime (SpecificTime h m) = + show h' ++ (if m > 0 then ":" ++ pad 2 (show m) else "") ++ " " ++ ampm + where + pad n s = replicate (n - length s) '0' ++ s + (h', ampm) + | h == 0 = (12, "AM") + | h < 12 = (h, "AM") + | h == 12 = (h, "PM") + | otherwise = (h - 12, "PM") + +toScheduledTime :: String -> Maybe ScheduledTime +toScheduledTime "any time" = Just AnyTime +toScheduledTime v = case words v of + (s:ampm:[]) + | map toUpper ampm == "AM" -> + go s h0 + | map toUpper ampm == "PM" -> + go s (\h -> (h0 h) + 12) + | otherwise -> Nothing + (s:[]) -> go s id + _ -> Nothing + where + h0 h + | h == 12 = 0 + | otherwise = h + go :: String -> (Int -> Int) -> Maybe ScheduledTime + go s adjust = + let (h, m) = separate (== ':') s + in SpecificTime + <$> (adjust <$> readish h) + <*> if null m then Just 0 else readish m + +fromSchedule :: Schedule -> String +fromSchedule (Schedule recurrance scheduledtime) = unwords + [ fromRecurrance recurrance + , "at" + , fromScheduledTime scheduledtime + ] + +toSchedule :: String -> Maybe Schedule +toSchedule = eitherToMaybe . parseSchedule + +parseSchedule :: String -> Either String Schedule +parseSchedule s = do + r <- maybe (Left $ "bad recurrance: " ++ recurrance) Right + (toRecurrance recurrance) + t <- maybe (Left $ "bad time of day: " ++ scheduledtime) Right + (toScheduledTime scheduledtime) + Right $ Schedule r t + where + (rws, tws) = separate (== "at") (words s) + recurrance = unwords rws + scheduledtime = unwords tws + +prop_past_sane :: Bool +prop_past_sane = and + [ all (checksout oneMonthPast) (mplus1 ++ yplus1) + , all (not . (checksout oneMonthPast)) (map swap (mplus1 ++ yplus1)) + , all (checksout oneYearPast) yplus1 + , all (not . (checksout oneYearPast)) (map swap yplus1) + ] + where + mplus1 = -- new date old date, 1+ months before it + [ (fromGregorian 2014 01 15, fromGregorian 2013 12 15) + , (fromGregorian 2014 01 15, fromGregorian 2013 02 15) + , (fromGregorian 2014 02 15, fromGregorian 2013 01 15) + , (fromGregorian 2014 03 01, fromGregorian 2013 01 15) + , (fromGregorian 2014 03 01, fromGregorian 2013 12 15) + , (fromGregorian 2015 01 01, fromGregorian 2010 01 01) + ] + yplus1 = -- new date old date, 1+ years before it + [ (fromGregorian 2014 01 15, fromGregorian 2012 01 16) + , (fromGregorian 2014 01 15, fromGregorian 2013 01 14) + , (fromGregorian 2022 12 31, fromGregorian 2000 01 01) + ] + checksout cmp (new, old) = new `cmp` old + swap (a,b) = (b,a) diff --git a/Utility/Scheduled/QuickCheck.hs b/Utility/Scheduled/QuickCheck.hs new file mode 100644 index 0000000000..a2051cd2aa --- /dev/null +++ b/Utility/Scheduled/QuickCheck.hs @@ -0,0 +1,51 @@ +{- quickcheck for scheduled activities + - + - Copyright 2013-2014 Joey Hess + - + - License: BSD-2-clause + -} + +{-# OPTIONS_GHC -fno-warn-orphans #-} + +module Utility.Scheduled.QuickCheck where + +import Utility.Scheduled +import Utility.QuickCheck + +import Control.Applicative +import Prelude + +instance Arbitrary Schedule where + arbitrary = Schedule <$> arbitrary <*> arbitrary + +instance Arbitrary ScheduledTime where + arbitrary = oneof + [ pure AnyTime + , SpecificTime + <$> choose (0, 23) + <*> choose (1, 59) + ] + +instance Arbitrary Recurrance where + arbitrary = oneof + [ pure Daily + , Weekly <$> arbday + , Monthly <$> arbday + , Yearly <$> arbday + , Divisible + <$> positive arbitrary + <*> oneof -- no nested Divisibles + [ pure Daily + , Weekly <$> arbday + , Monthly <$> arbday + , Yearly <$> arbday + ] + ] + where + arbday = oneof + [ Just <$> nonNegative arbitrary + , pure Nothing + ] + +prop_schedule_roundtrips :: Schedule -> Bool +prop_schedule_roundtrips s = toSchedule (fromSchedule s) == Just s diff --git a/Utility/Shell.hs b/Utility/Shell.hs new file mode 100644 index 0000000000..c64ca75b91 --- /dev/null +++ b/Utility/Shell.hs @@ -0,0 +1,64 @@ +{- /bin/sh handling + - + - Copyright 2013 Joey Hess + - + - License: BSD-2-clause + -} + +{-# LANGUAGE CPP #-} + +module Utility.Shell where + +import Utility.SafeCommand +#ifdef mingw32_HOST_OS +import Utility.Path +import Utility.Exception +import Utility.PartialPrelude +#endif + +#ifdef mingw32_HOST_OS +import System.FilePath +#endif + +shellPath_portable :: FilePath +shellPath_portable = "/bin/sh" + +shellPath_local :: FilePath +#ifndef __ANDROID__ +shellPath_local = shellPath_portable +#else +shellPath_local = "/system/bin/sh" +#endif + +shebang_portable :: String +shebang_portable = "#!" ++ shellPath_portable + +shebang_local :: String +shebang_local = "#!" ++ shellPath_local + +-- | On Windows, shebang is not handled by the kernel, so to support +-- shell scripts etc, have to look at the program being run and +-- parse it for shebang. +-- +-- This has no effect on Unix. +findShellCommand :: FilePath -> IO (FilePath, [CommandParam]) +findShellCommand f = do +#ifndef mingw32_HOST_OS + defcmd +#else + l <- catchDefaultIO Nothing $ headMaybe . lines <$> readFile f + case l of + Just ('#':'!':rest) -> case words rest of + [] -> defcmd + (c:ps) -> do + let ps' = map Param ps ++ [File f] + -- If the command is not inPath, + -- take the base of it, and run eg "sh" + -- which in some cases on windows will work + -- despite it not being inPath. + ok <- inPath c + return (if ok then c else takeFileName c, ps') + _ -> defcmd +#endif + where + defcmd = return (f, []) diff --git a/Utility/SimpleProtocol.hs b/Utility/SimpleProtocol.hs new file mode 100644 index 0000000000..f941cfcd2a --- /dev/null +++ b/Utility/SimpleProtocol.hs @@ -0,0 +1,130 @@ +{- Simple line-based protocols. + - + - Copyright 2013-2016 Joey Hess + - + - License: BSD-2-clause + -} + +{-# LANGUAGE FlexibleInstances #-} +{-# OPTIONS_GHC -fno-warn-orphans #-} + +module Utility.SimpleProtocol ( + Sendable(..), + Receivable(..), + parseMessage, + Serializable(..), + Parser, + parseFail, + parse0, + parse1, + parse2, + parse3, + dupIoHandles, + getProtocolLine, +) where + +import Data.Char +import GHC.IO.Handle +import System.Exit (ExitCode(..)) + +import Common + +-- Messages that can be sent. +class Sendable m where + formatMessage :: m -> [String] + +-- Messages that can be received. +class Receivable m where + -- Passed the first word of the message, returns + -- a Parser that can be be fed the rest of the message to generate + -- the value. + parseCommand :: String -> Parser m + +parseMessage :: (Receivable m) => String -> Maybe m +parseMessage s = parseCommand command rest + where + (command, rest) = splitWord s + +class Serializable a where + serialize :: a -> String + deserialize :: String -> Maybe a + +instance Serializable [Char] where + serialize = id + deserialize = Just + +instance Serializable ExitCode where + serialize ExitSuccess = "0" + serialize (ExitFailure n) = show n + deserialize "0" = Just ExitSuccess + deserialize s = ExitFailure <$> readish s + +{- Parsing the parameters of messages. Using the right parseN ensures + - that the string is split into exactly the requested number of words, + - which allows the last parameter of a message to contain arbitrary + - whitespace, etc, without needing any special quoting. + -} +type Parser a = String -> Maybe a + +parseFail :: Parser a +parseFail _ = Nothing + +parse0 :: a -> Parser a +parse0 mk "" = Just mk +parse0 _ _ = Nothing + +parse1 :: Serializable p1 => (p1 -> a) -> Parser a +parse1 mk p1 = mk <$> deserialize p1 + +parse2 :: (Serializable p1, Serializable p2) => (p1 -> p2 -> a) -> Parser a +parse2 mk s = mk <$> deserialize p1 <*> deserialize p2 + where + (p1, p2) = splitWord s + +parse3 :: (Serializable p1, Serializable p2, Serializable p3) => (p1 -> p2 -> p3 -> a) -> Parser a +parse3 mk s = mk <$> deserialize p1 <*> deserialize p2 <*> deserialize p3 + where + (p1, rest) = splitWord s + (p2, p3) = splitWord rest + +splitWord :: String -> (String, String) +splitWord = separate isSpace + +{- When a program speaks a simple protocol over stdio, any other output + - to stdout (or anything that attempts to read from stdin) + - will mess up the protocol. To avoid that, close stdin, + - and duplicate stderr to stdout. Return two new handles + - that are duplicates of the original (stdin, stdout). -} +dupIoHandles :: IO (Handle, Handle) +dupIoHandles = do + readh <- hDuplicate stdin + writeh <- hDuplicate stdout + nullh <- openFile devNull ReadMode + nullh `hDuplicateTo` stdin + stderr `hDuplicateTo` stdout + return (readh, writeh) + +{- Reads a line, but to avoid super-long lines eating memory, returns + - Nothing if 32 kb have been read without seeing a '\n' + - + - If there is a '\r' before the '\n', it is removed, to support + - systems using "\r\n" at ends of lines + - + - This implementation is not super efficient, but as long as the Handle + - supports buffering, it avoids reading a character at a time at the + - syscall level. + - + - Throws isEOFError when no more input is available. + -} +getProtocolLine :: Handle -> IO (Maybe String) +getProtocolLine h = go (32768 :: Int) [] + where + go 0 _ = return Nothing + go n l = do + c <- hGetChar h + if c == '\n' + then return $ Just $ reverse $ + case l of + ('\r':rest) -> rest + _ -> l + else go (n-1) (c:l) diff --git a/Utility/Split.hs b/Utility/Split.hs new file mode 100644 index 0000000000..decfe7d396 --- /dev/null +++ b/Utility/Split.hs @@ -0,0 +1,30 @@ +{- split utility functions + - + - Copyright 2017 Joey Hess + - + - License: BSD-2-clause + -} + +{-# OPTIONS_GHC -fno-warn-tabs #-} + +module Utility.Split where + +import Data.List (intercalate) +import Data.List.Split (splitOn) + +-- | same as Data.List.Utils.split +-- +-- intercalate x . splitOn x === id +split :: Eq a => [a] -> [a] -> [[a]] +split = splitOn + +-- | Split on a single character. This is over twice as fast as using +-- split on a list of length 1, while producing identical results. -} +splitc :: Eq c => c -> [c] -> [[c]] +splitc c s = case break (== c) s of + (i, _c:rest) -> i : splitc c rest + (i, []) -> i : [] + +-- | same as Data.List.Utils.replace +replace :: Eq a => [a] -> [a] -> [a] -> [a] +replace old new = intercalate new . split old diff --git a/Utility/SshConfig.hs b/Utility/SshConfig.hs new file mode 100644 index 0000000000..1f8581a280 --- /dev/null +++ b/Utility/SshConfig.hs @@ -0,0 +1,146 @@ +{- ssh config file parsing and modification + - + - Copyright 2013 Joey Hess + - + - License: BSD-2-clause + -} + +module Utility.SshConfig where + +import Common +import Utility.UserInfo +import Utility.Tmp +import Utility.FileMode + +import Data.Char +import Data.Ord +import Data.Either + +data SshConfig + = GlobalConfig SshSetting + | HostConfig Host [Either Comment SshSetting] + | CommentLine Comment + deriving (Show) + +data Comment = Comment Indent String + deriving (Show) + +data SshSetting = SshSetting Indent Key Value + deriving (Show) + +type Indent = String +type Host = String +type Key = String +type Value = String + +{- Parses ~/.ssh/config. Comments and indentation are preserved. + - + - Note that there is no parse failure. If a line cannot be parsed, it will + - be taken to be a SshSetting that contains the whole line as the key, + - and has no value. -} +parseSshConfig :: String -> [SshConfig] +parseSshConfig = go [] . lines + where + go c [] = reverse c + go c (l:ls) + | iscomment l = collect $ CommentLine $ mkcomment l + | otherwise = case splitline l of + (indent, k, v) + | isHost k -> hoststanza v c [] ls + | otherwise -> collect $ GlobalConfig $ SshSetting indent k v + where + collect v = go (v:c) ls + + hoststanza host c hc [] = go (HostConfig host (reverse hc):c) [] + hoststanza host c hc (l:ls) + | iscomment l = hoststanza host c ((Left $ mkcomment l):hc) ls + | otherwise = case splitline l of + (indent, k, v) + | isHost k -> hoststanza v + (HostConfig host (reverse hc):c) [] ls + | otherwise -> hoststanza host c + ((Right $ SshSetting indent k v):hc) ls + + iscomment l = all isSpace l || "#" `isPrefixOf` (dropWhile isSpace l) + + splitline l = (indent, k, v) + where + (indent, rest) = span isSpace l + (k, v) = separate isSpace rest + + mkcomment l = Comment indent c + where + (indent, c) = span isSpace l + + isHost v = map toLower v == "host" + +genSshConfig :: [SshConfig] -> String +genSshConfig = unlines . concatMap gen + where + gen (CommentLine c) = [comment c] + gen (GlobalConfig s) = [setting s] + gen (HostConfig h cs) = ("Host " ++ h) : map (either comment setting) cs + + setting (SshSetting indent k v) = indent ++ k ++ + if null v then "" else " " ++ v + comment (Comment indent c) = indent ++ c + +findHostConfigKey :: SshConfig -> Key -> Maybe Value +findHostConfigKey (HostConfig _ cs) wantk = go (rights cs) (map toLower wantk) + where + go [] _ = Nothing + go ((SshSetting _ k v):rest) wantk' + | map toLower k == wantk' = Just v + | otherwise = go rest wantk' +findHostConfigKey _ _ = Nothing + +{- Adds a particular Key and Value to a HostConfig. -} +addToHostConfig :: SshConfig -> Key -> Value -> SshConfig +addToHostConfig (HostConfig host cs) k v = + HostConfig host $ Right (SshSetting indent k v) : cs + where + {- The indent is taken from any existing SshSetting + - in the HostConfig (largest indent wins). -} + indent = fromMaybe "\t" $ headMaybe $ reverse $ + sortBy (comparing length) $ map getindent cs + getindent (Right (SshSetting i _ _)) = i + getindent (Left (Comment i _)) = i +addToHostConfig other _ _ = other + +modifyUserSshConfig :: ([SshConfig] -> [SshConfig]) -> IO () +modifyUserSshConfig modifier = changeUserSshConfig $ + genSshConfig . modifier . parseSshConfig + +changeUserSshConfig :: (String -> String) -> IO () +changeUserSshConfig modifier = do + sshdir <- sshDir + let configfile = sshdir "config" + whenM (doesFileExist configfile) $ do + c <- readFileStrict configfile + let c' = modifier c + when (c /= c') $ do + -- If it's a symlink, replace the file it + -- points to. + f <- catchDefaultIO configfile (canonicalizePath configfile) + viaTmp writeSshConfig f c' + +writeSshConfig :: FilePath -> String -> IO () +writeSshConfig f s = do + writeFile f s + setSshConfigMode f + +{- Ensure that the ssh config file lacks any group or other write bits, + - since ssh is paranoid about not working if other users can write + - to one of its config files (.ssh/config and .ssh/authorized_keys). + - + - If the chmod fails, ignore the failure, as it might be a filesystem like + - Android's that does not support file modes. + -} +setSshConfigMode :: FilePath -> IO () +setSshConfigMode f = void $ tryIO $ modifyFileMode f $ + removeModes [groupWriteMode, otherWriteMode] + +sshDir :: IO FilePath +sshDir = do + home <- myHomeDir + return $ home ".ssh" diff --git a/Utility/SshHost.hs b/Utility/SshHost.hs new file mode 100644 index 0000000000..d8a8da11df --- /dev/null +++ b/Utility/SshHost.hs @@ -0,0 +1,29 @@ +{- ssh hostname sanitization + - + - When constructing a ssh command with a hostname that may be controlled + - by an attacker, prevent the hostname from starting with "-", + - to prevent tricking ssh into arbitrary command execution via + - eg "-oProxyCommand=" + - + - Copyright 2017 Joey Hess + - + - License: BSD-2-clause + -} + +module Utility.SshHost (SshHost, mkSshHost, fromSshHost) where + +newtype SshHost = SshHost String + +-- | Smart constructor for a legal hostname or IP address. +-- In some cases, it may be prefixed with "user@" to specify the remote +-- user at the host. +-- +-- For now, we only filter out the problem ones, because determining an +-- actually legal hostnames is quite complicated. +mkSshHost :: String -> Either String SshHost +mkSshHost h@('-':_) = Left $ + "rejecting ssh hostname that starts with '-' : " ++ h +mkSshHost h = Right (SshHost h) + +fromSshHost :: SshHost -> String +fromSshHost (SshHost h) = h diff --git a/Utility/Su.hs b/Utility/Su.hs new file mode 100644 index 0000000000..a0500e4837 --- /dev/null +++ b/Utility/Su.hs @@ -0,0 +1,103 @@ +{- su to root + - + - Copyright 2016 Joey Hess + - + - License: BSD-2-clause + -} + +{-# LANGUAGE CPP #-} + +module Utility.Su where + +import Common + +#ifndef mingw32_HOST_OS +import Utility.Env +import System.Posix.IO +import System.Posix.Terminal +#endif + +data WhosePassword + = RootPassword + | UserPassword + | SomePassword + -- ^ may be user or root; su program should indicate which + deriving (Show) + +data PasswordPrompt + = WillPromptPassword WhosePassword + | MayPromptPassword WhosePassword + | NoPromptPassword + deriving (Show) + +describePasswordPrompt :: PasswordPrompt -> Maybe String +describePasswordPrompt (WillPromptPassword whose) = Just $ + "You will be prompted for " ++ describeWhosePassword whose ++ " password" +describePasswordPrompt (MayPromptPassword whose) = Just $ + "You may be prompted for " ++ describeWhosePassword whose ++ " password" +describePasswordPrompt NoPromptPassword = Nothing + +describeWhosePassword :: WhosePassword -> String +describeWhosePassword RootPassword = "root's" +describeWhosePassword UserPassword = "your" +describeWhosePassword SomePassword = "a" + +data SuCommand = SuCommand PasswordPrompt String [CommandParam] + deriving (Show) + +describePasswordPrompt' :: Maybe SuCommand -> Maybe String +describePasswordPrompt' (Just (SuCommand p _ _)) = describePasswordPrompt p +describePasswordPrompt' Nothing = Nothing + +runSuCommand :: (Maybe SuCommand) -> IO Bool +runSuCommand (Just (SuCommand _ cmd ps)) = boolSystem cmd ps +runSuCommand Nothing = return False + +-- Generates a SuCommand that runs a command as root, fairly portably. +-- +-- Does not use sudo commands if something else is available, because +-- the user may not be in sudoers and we couldn't differentiate between +-- that and the command failing. Although, some commands like gksu +-- decide based on the system's configuration whether sudo should be used. +mkSuCommand :: String -> [CommandParam] -> IO (Maybe SuCommand) +#ifndef mingw32_HOST_OS +mkSuCommand cmd ps = firstM (\(SuCommand _ p _) -> inPath p) =<< selectcmds + where + selectcmds = ifM (inx <||> (not <$> atconsole)) + ( return (graphicalcmds ++ consolecmds) + , return consolecmds + ) + + inx = isJust <$> getEnv "DISPLAY" + atconsole = queryTerminal stdInput + + -- These will only work when the user is logged into a desktop. + graphicalcmds = + [ SuCommand (MayPromptPassword SomePassword) "gksu" + [Param shellcmd] + , SuCommand (MayPromptPassword SomePassword) "kdesu" + [Param shellcmd] + -- Available in Debian's menu package; knows about lots of + -- ways to gain root. + , SuCommand (MayPromptPassword SomePassword) "su-to-root" + [Param "-X", Param "-c", Param shellcmd] + -- OSX native way to run a command as root, prompts in GUI + , SuCommand (WillPromptPassword RootPassword) "osascript" + [Param "-e", Param ("do shell script \"" ++ shellcmd ++ "\" with administrator privileges")] + ] + + -- These will only work when run in a console. + consolecmds = + [ SuCommand (WillPromptPassword RootPassword) "su" + [Param "-c", Param shellcmd] + , SuCommand (MayPromptPassword UserPassword) "sudo" + ([Param cmd] ++ ps) + , SuCommand (MayPromptPassword SomePassword) "su-to-root" + [Param "-c", Param shellcmd] + ] + + shellcmd = unwords $ map shellEscape (cmd:toCommand ps) +#else +-- For windows, we assume the user has administrator access. +mkSuCommand cmd ps = return $ Just $ SuCommand NoPromptPassword cmd ps +#endif diff --git a/Utility/SystemDirectory.hs b/Utility/SystemDirectory.hs new file mode 100644 index 0000000000..b9040fe13f --- /dev/null +++ b/Utility/SystemDirectory.hs @@ -0,0 +1,16 @@ +{- System.Directory without its conflicting isSymbolicLink + - + - Copyright 2016 Joey Hess + - + - License: BSD-2-clause + -} + +-- Disable warnings because only some versions of System.Directory export +-- isSymbolicLink. +{-# OPTIONS_GHC -fno-warn-tabs -w #-} + +module Utility.SystemDirectory ( + module System.Directory +) where + +import System.Directory hiding (isSymbolicLink, getFileSize) diff --git a/Utility/TList.hs b/Utility/TList.hs new file mode 100644 index 0000000000..033c8ca02c --- /dev/null +++ b/Utility/TList.hs @@ -0,0 +1,69 @@ +{- Transactional lists + - + - Based on DLists, a transactional list can quickly and efficiently + - have items inserted at either end, or a whole list appended to it. + - + - Unlike a TQueue, the entire contents of a TList can be efficiently + - read without modifying it. + - + - Copyright 2013 Joey Hess + -} + +{-# LANGUAGE BangPatterns #-} + +module Utility.TList where + +import Common + +import Control.Concurrent.STM +import qualified Data.DList as D + +type TList a = TMVar (D.DList a) + +newTList :: STM (TList a) +newTList = newEmptyTMVar + +{- Gets the contents of the TList. Blocks when empty. + - TList is left empty. -} +getTList :: TList a -> STM [a] +getTList tlist = D.toList <$> getTDList tlist + +getTDList :: TList a -> STM (D.DList a) +getTDList = takeTMVar + +{- Replaces the contents of the TList. -} +setTList :: TList a -> [a] -> STM () +setTList tlist = setTDList tlist . D.fromList + +setTDList :: TList a -> D.DList a -> STM () +setTDList tlist = modifyTList tlist . const + +{- Takes anything currently in the TList, without blocking. + - TList is left empty. -} +takeTList :: TList a -> STM [a] +takeTList tlist = maybe [] D.toList <$> tryTakeTMVar tlist + +{- Reads anything in the list, without modifying it, or blocking. -} +readTList :: TList a -> STM [a] +readTList tlist = maybe [] D.toList <$> tryReadTMVar tlist + +{- Mutates a TList. -} +modifyTList :: TList a -> (D.DList a -> D.DList a) -> STM () +modifyTList tlist a = do + dl <- fromMaybe D.empty <$> tryTakeTMVar tlist + let !dl' = a dl + {- The TMVar is left empty when the list is empty. + - Thus attempts to read it automatically block. -} + unless (emptyDList dl') $ + putTMVar tlist dl' + where + emptyDList = D.list True (\_ _ -> False) + +consTList :: TList a -> a -> STM () +consTList tlist v = modifyTList tlist $ \dl -> D.cons v dl + +snocTList :: TList a -> a -> STM () +snocTList tlist v = modifyTList tlist $ \dl -> D.snoc dl v + +appendTList :: TList a -> [a] -> STM () +appendTList tlist l = modifyTList tlist $ \dl -> D.append dl (D.fromList l) diff --git a/Utility/Tense.hs b/Utility/Tense.hs new file mode 100644 index 0000000000..ef2454bdc7 --- /dev/null +++ b/Utility/Tense.hs @@ -0,0 +1,57 @@ +{- Past and present tense text. + - + - Copyright 2012 Joey Hess + - + - License: BSD-2-clause + -} + +{-# LANGUAGE OverloadedStrings #-} + +module Utility.Tense where + +import qualified Data.Text as T +import Data.Text (Text) +import GHC.Exts( IsString(..) ) + +data Tense = Present | Past + deriving (Eq) + +data TenseChunk = Tensed Text Text | UnTensed Text + deriving (Eq, Ord, Show) + +newtype TenseText = TenseText [TenseChunk] + deriving (Eq, Ord) + +{- Allows OverloadedStrings to be used, to build UnTensed chunks. -} +instance IsString TenseChunk where + fromString = UnTensed . T.pack + +{- Allows OverloadedStrings to be used, to provide UnTensed TenseText. -} +instance IsString TenseText where + fromString s = TenseText [fromString s] + +renderTense :: Tense -> TenseText -> Text +renderTense tense (TenseText chunks) = T.concat $ map render chunks + where + render (Tensed present past) + | tense == Present = present + | otherwise = past + render (UnTensed s) = s + +{- Builds up a TenseText, separating chunks with spaces. + - + - However, rather than just intersperse new chunks for the spaces, + - the spaces are appended to the end of the chunks. + -} +tenseWords :: [TenseChunk] -> TenseText +tenseWords = TenseText . go [] + where + go c [] = reverse c + go c (w:[]) = reverse (w:c) + go c ((UnTensed w):ws) = go (UnTensed (addspace w) : c) ws + go c ((Tensed w1 w2):ws) = + go (Tensed (addspace w1) (addspace w2) : c) ws + addspace w = T.append w " " + +unTensed :: Text -> TenseText +unTensed t = TenseText [UnTensed t] diff --git a/Utility/ThreadLock.hs b/Utility/ThreadLock.hs new file mode 100644 index 0000000000..e212fc11f7 --- /dev/null +++ b/Utility/ThreadLock.hs @@ -0,0 +1,19 @@ +{- locking between threads + - + - Copyright 2012 Joey Hess + - + - License: BSD-2-clause + -} + +module Utility.ThreadLock where + +import Control.Concurrent.MVar + +type Lock = MVar () + +newLock :: IO Lock +newLock = newMVar () + +{- Runs an action with a lock held, so only one thread at a time can run it. -} +withLock :: Lock -> IO a -> IO a +withLock lock = withMVar lock . const diff --git a/Utility/ThreadScheduler.hs b/Utility/ThreadScheduler.hs new file mode 100644 index 0000000000..da05e99669 --- /dev/null +++ b/Utility/ThreadScheduler.hs @@ -0,0 +1,74 @@ +{- thread scheduling + - + - Copyright 2012, 2013 Joey Hess + - Copyright 2011 Bas van Dijk & Roel van Dijk + - + - License: BSD-2-clause + -} + +{-# LANGUAGE CPP #-} + +module Utility.ThreadScheduler where + +import Control.Monad +import Control.Concurrent +#ifndef mingw32_HOST_OS +import Control.Monad.IfElse +import System.Posix.IO +#endif +#ifndef mingw32_HOST_OS +import System.Posix.Signals +#ifndef __ANDROID__ +import System.Posix.Terminal +#endif +#endif + +newtype Seconds = Seconds { fromSeconds :: Int } + deriving (Eq, Ord, Show) + +type Microseconds = Integer + +{- Runs an action repeatedly forever, sleeping at least the specified number + - of seconds in between. -} +runEvery :: Seconds -> IO a -> IO a +runEvery n a = forever $ do + threadDelaySeconds n + a + +threadDelaySeconds :: Seconds -> IO () +threadDelaySeconds (Seconds n) = unboundDelay (fromIntegral n * oneSecond) + +{- Like threadDelay, but not bounded by an Int. + - + - There is no guarantee that the thread will be rescheduled promptly when the + - delay has expired, but the thread will never continue to run earlier than + - specified. + - + - Taken from the unbounded-delay package to avoid a dependency for 4 lines + - of code. + -} +unboundDelay :: Microseconds -> IO () +unboundDelay time = do + let maxWait = min time $ toInteger (maxBound :: Int) + threadDelay $ fromInteger maxWait + when (maxWait /= time) $ unboundDelay (time - maxWait) + +{- Pauses the main thread, letting children run until program termination. -} +waitForTermination :: IO () +waitForTermination = do +#ifdef mingw32_HOST_OS + forever $ threadDelaySeconds (Seconds 6000) +#else + lock <- newEmptyMVar + let check sig = void $ + installHandler sig (CatchOnce $ putMVar lock ()) Nothing + check softwareTermination +#ifndef __ANDROID__ + whenM (queryTerminal stdInput) $ + check keyboardSignal +#endif + takeMVar lock +#endif + +oneSecond :: Microseconds +oneSecond = 1000000 diff --git a/Utility/Tmp.hs b/Utility/Tmp.hs new file mode 100644 index 0000000000..6e04b10765 --- /dev/null +++ b/Utility/Tmp.hs @@ -0,0 +1,75 @@ +{- Temporary files. + - + - Copyright 2010-2013 Joey Hess + - + - License: BSD-2-clause + -} + +{-# LANGUAGE CPP #-} +{-# OPTIONS_GHC -fno-warn-tabs #-} + +module Utility.Tmp where + +import System.IO +import System.FilePath +import System.Directory +import Control.Monad.IO.Class +import System.PosixCompat.Files + +import Utility.Exception +import Utility.FileSystemEncoding + +type Template = String + +{- Runs an action like writeFile, writing to a temp file first and + - then moving it into place. The temp file is stored in the same + - directory as the final file to avoid cross-device renames. -} +viaTmp :: (MonadMask m, MonadIO m) => (FilePath -> v -> m ()) -> FilePath -> v -> m () +viaTmp a file content = bracketIO setup cleanup use + where + (dir, base) = splitFileName file + template = base ++ ".tmp" + setup = do + createDirectoryIfMissing True dir + openTempFile dir template + cleanup (tmpfile, h) = do + _ <- tryIO $ hClose h + tryIO $ removeFile tmpfile + use (tmpfile, h) = do + liftIO $ hClose h + a tmpfile content + liftIO $ rename tmpfile file + +{- Runs an action with a tmp file located in the system's tmp directory + - (or in "." if there is none) then removes the file. -} +withTmpFile :: (MonadIO m, MonadMask m) => Template -> (FilePath -> Handle -> m a) -> m a +withTmpFile template a = do + tmpdir <- liftIO $ catchDefaultIO "." getTemporaryDirectory + withTmpFileIn tmpdir template a + +{- Runs an action with a tmp file located in the specified directory, + - then removes the file. -} +withTmpFileIn :: (MonadIO m, MonadMask m) => FilePath -> Template -> (FilePath -> Handle -> m a) -> m a +withTmpFileIn tmpdir template a = bracket create remove use + where + create = liftIO $ openTempFile tmpdir template + remove (name, h) = liftIO $ do + hClose h + catchBoolIO (removeFile name >> return True) + use (name, h) = a name h + +{- It's not safe to use a FilePath of an existing file as the template + - for openTempFile, because if the FilePath is really long, the tmpfile + - will be longer, and may exceed the maximum filename length. + - + - This generates a template that is never too long. + - (Well, it allocates 20 characters for use in making a unique temp file, + - anyway, which is enough for the current implementation and any + - likely implementation.) + -} +relatedTemplate :: FilePath -> FilePath +relatedTemplate f + | len > 20 = truncateFilePath (len - 20) f + | otherwise = f + where + len = length f diff --git a/Utility/Tmp/Dir.hs b/Utility/Tmp/Dir.hs new file mode 100644 index 0000000000..36f69a82e2 --- /dev/null +++ b/Utility/Tmp/Dir.hs @@ -0,0 +1,68 @@ +{- Temporary directories + - + - Copyright 2010-2013 Joey Hess + - + - License: BSD-2-clause + -} + +{-# LANGUAGE CPP #-} +{-# OPTIONS_GHC -fno-warn-tabs #-} + +module Utility.Tmp.Dir where + +import Control.Monad.IfElse +import System.FilePath +import System.Directory +import Control.Monad.IO.Class +#ifndef mingw32_HOST_OS +import System.Posix.Temp (mkdtemp) +#endif + +import Utility.Exception + +type Template = String + +{- Runs an action with a tmp directory located within the system's tmp + - directory (or within "." if there is none), then removes the tmp + - directory and all its contents. -} +withTmpDir :: (MonadMask m, MonadIO m) => Template -> (FilePath -> m a) -> m a +withTmpDir template a = do + topleveltmpdir <- liftIO $ catchDefaultIO "." getTemporaryDirectory +#ifndef mingw32_HOST_OS + -- Use mkdtemp to create a temp directory securely in /tmp. + bracket + (liftIO $ mkdtemp $ topleveltmpdir template) + removeTmpDir + a +#else + withTmpDirIn topleveltmpdir template a +#endif + +{- Runs an action with a tmp directory located within a specified directory, + - then removes the tmp directory and all its contents. -} +withTmpDirIn :: (MonadMask m, MonadIO m) => FilePath -> Template -> (FilePath -> m a) -> m a +withTmpDirIn tmpdir template = bracketIO create removeTmpDir + where + create = do + createDirectoryIfMissing True tmpdir + makenewdir (tmpdir template) (0 :: Int) + makenewdir t n = do + let dir = t ++ "." ++ show n + catchIOErrorType AlreadyExists (const $ makenewdir t $ n + 1) $ do + createDirectory dir + return dir + +{- Deletes the entire contents of the the temporary directory, if it + - exists. -} +removeTmpDir :: MonadIO m => FilePath -> m () +removeTmpDir tmpdir = liftIO $ whenM (doesDirectoryExist tmpdir) $ do +#if mingw32_HOST_OS + -- Windows will often refuse to delete a file + -- after a process has just written to it and exited. + -- Because it's crap, presumably. So, ignore failure + -- to delete the temp directory. + _ <- tryIO $ removeDirectoryRecursive tmpdir + return () +#else + removeDirectoryRecursive tmpdir +#endif diff --git a/Utility/Tor.hs b/Utility/Tor.hs new file mode 100644 index 0000000000..218a038df9 --- /dev/null +++ b/Utility/Tor.hs @@ -0,0 +1,177 @@ +{- tor interface + - + - Copyright 2016 Joey Hess + - + - Licensed under the GNU GPL version 3 or higher. + -} + +module Utility.Tor where + +import Common +import Utility.ThreadScheduler +import Utility.FileMode + +import System.PosixCompat.Types +import Data.Char +import Network.Socket +import Network.Socks5 +import qualified Data.ByteString.UTF8 as BU8 +import qualified System.Random as R + +type OnionPort = Int + +newtype OnionAddress = OnionAddress String + deriving (Show, Eq) + +type OnionSocket = FilePath + +-- | A unique identifier for a hidden service. +type UniqueIdent = String + +-- | Name of application that is providing a hidden service. +type AppName = String + +connectHiddenService :: OnionAddress -> OnionPort -> IO Socket +connectHiddenService (OnionAddress address) port = do + (s, _) <- socksConnect torsockconf socksaddr + return s + where + torsocksport = 9050 + torsockconf = defaultSocksConf "127.0.0.1" torsocksport + socksdomain = SocksAddrDomainName (BU8.fromString address) + socksaddr = SocksAddress socksdomain (fromIntegral port) + +-- | Adds a hidden service connecting to localhost, using some kind +-- of unique identifier. +-- +-- This will only work if run as root, and tor has to already be running. +-- +-- Picks a random high port number for the hidden service that is not +-- used by any other hidden service. Returns the hidden service's +-- onion address, port, and the unix socket file to use. +-- +-- If there is already a hidden service for the specified unique +-- identifier, returns its information without making any changes. +addHiddenService :: AppName -> UserID -> UniqueIdent -> IO (OnionAddress, OnionPort) +addHiddenService appname uid ident = do + prepHiddenServiceSocketDir appname uid ident + ls <- lines <$> (readFile =<< findTorrc) + let portssocks = mapMaybe (parseportsock . separate isSpace) ls + case filter (\(_, s) -> s == sockfile) portssocks of + ((p, _s):_) -> waithiddenservice 1 p + _ -> do + highports <- R.getStdRandom mkhighports + let newport = Prelude.head $ + filter (`notElem` map fst portssocks) highports + torrc <- findTorrc + writeFile torrc $ unlines $ + ls ++ + [ "" + , "HiddenServiceDir " ++ hiddenServiceDir appname uid ident + , "HiddenServicePort " ++ show newport ++ + " unix:" ++ sockfile + ] + -- Reload tor, so it will see the new hidden + -- service and generate the hostname file for it. + reloaded <- anyM (uncurry boolSystem) + [ ("systemctl", [Param "reload", Param "tor"]) + , ("service", [Param "tor", Param "reload"]) + ] + unless reloaded $ + giveup "failed to reload tor, perhaps the tor service is not running" + waithiddenservice 120 newport + where + parseportsock ("HiddenServicePort", l) = do + p <- readish $ takeWhile (not . isSpace) l + return (p, drop 1 (dropWhile (/= ':') l)) + parseportsock _ = Nothing + + sockfile = hiddenServiceSocketFile appname uid ident + + -- An infinite random list of high ports. + mkhighports g = + let (g1, g2) = R.split g + in (R.randomRs (1025, 65534) g1, g2) + + waithiddenservice :: Int -> OnionPort -> IO (OnionAddress, OnionPort) + waithiddenservice 0 _ = giveup "tor failed to create hidden service, perhaps the tor service is not running" + waithiddenservice n p = do + v <- tryIO $ readFile $ hiddenServiceHostnameFile appname uid ident + case v of + Right s | ".onion\n" `isSuffixOf` s -> + return (OnionAddress (takeWhile (/= '\n') s), p) + _ -> do + threadDelaySeconds (Seconds 1) + waithiddenservice (n-1) p + +-- | A hidden service directory to use. +-- +-- Has to be inside the torLibDir so tor can create it. +-- +-- Has to end with "uid_ident" so getHiddenServiceSocketFile can find it. +hiddenServiceDir :: AppName -> UserID -> UniqueIdent -> FilePath +hiddenServiceDir appname uid ident = torLibDir appname ++ "_" ++ show uid ++ "_" ++ ident + +hiddenServiceHostnameFile :: AppName -> UserID -> UniqueIdent -> FilePath +hiddenServiceHostnameFile appname uid ident = hiddenServiceDir appname uid ident "hostname" + +-- | Location of the socket for a hidden service. +-- +-- This has to be a location that tor can read from, and that the user +-- can write to. Since torLibDir is locked down, it can't go in there. +-- +-- Note that some unix systems limit socket paths to 92 bytes long. +-- That should not be a problem if the UniqueIdent is around the length of +-- a UUID, and the AppName is short. +hiddenServiceSocketFile :: AppName -> UserID -> UniqueIdent -> FilePath +hiddenServiceSocketFile appname uid ident = varLibDir appname show uid ++ "_" ++ ident "s" + +-- | Parse torrc, to get the socket file used for a hidden service with +-- the specified UniqueIdent. +getHiddenServiceSocketFile :: AppName -> UserID -> UniqueIdent -> IO (Maybe FilePath) +getHiddenServiceSocketFile _appname uid ident = + parse . map words . lines <$> catchDefaultIO "" (readFile =<< findTorrc) + where + parse [] = Nothing + parse (("HiddenServiceDir":hsdir:[]):("HiddenServicePort":_hsport:hsaddr:[]):rest) + | "unix:" `isPrefixOf` hsaddr && hasident hsdir = + Just (drop (length "unix:") hsaddr) + | otherwise = parse rest + parse (_:rest) = parse rest + + -- Don't look for AppName in the hsdir, because it didn't used to + -- be included. + hasident hsdir = (show uid ++ "_" ++ ident) `isSuffixOf` takeFileName hsdir + +-- | Sets up the directory for the socketFile, with appropriate +-- permissions. Must run as root. +prepHiddenServiceSocketDir :: AppName -> UserID -> UniqueIdent -> IO () +prepHiddenServiceSocketDir appname uid ident = do + createDirectoryIfMissing True d + setOwnerAndGroup d uid (-1) + modifyFileMode d $ + addModes [ownerReadMode, ownerExecuteMode, ownerWriteMode] + where + d = takeDirectory $ hiddenServiceSocketFile appname uid ident + +-- | Finds the system's torrc file, in any of the typical locations of it. +-- Returns the first found. If there is no system torrc file, defaults to +-- /etc/tor/torrc. +findTorrc :: IO FilePath +findTorrc = fromMaybe "/etc/tor/torrc" <$> firstM doesFileExist + -- Debian + [ "/etc/tor/torrc" + -- Some systems put it here instead. + , "/etc/torrc" + -- Default when installed from source + , "/usr/local/etc/tor/torrc" + ] + +torLibDir :: FilePath +torLibDir = "/var/lib/tor" + +varLibDir :: FilePath +varLibDir = "/var/lib" + +torIsInstalled :: IO Bool +torIsInstalled = inPath "tor" diff --git a/Utility/Touch.hs b/Utility/Touch.hs new file mode 100644 index 0000000000..62acea02ae --- /dev/null +++ b/Utility/Touch.hs @@ -0,0 +1,52 @@ +{- More control over touching a file. + - + - Copyright 2011 Joey Hess + - + - License: BSD-2-clause + -} + +{-# LANGUAGE CPP #-} + +module Utility.Touch ( + TimeSpec(..), + touchBoth, + touch +) where + +#if ! defined(mingw32_HOST_OS) && ! defined(__ANDROID__) + +#if MIN_VERSION_unix(2,7,0) + +import System.Posix.Files +import System.Posix.Types + +newtype TimeSpec = TimeSpec EpochTime + +{- Changes the access and modification times of an existing file. + Can follow symlinks, or not. Throws IO error on failure. -} +touchBoth :: FilePath -> TimeSpec -> TimeSpec -> Bool -> IO () +touchBoth file (TimeSpec atime) (TimeSpec mtime) follow + | follow = setFileTimes file atime mtime + | otherwise = setSymbolicLinkTimesHiRes file (realToFrac atime) (realToFrac mtime) + +touch :: FilePath -> TimeSpec -> Bool -> IO () +touch file mtime = touchBoth file mtime mtime + +#else +import Utility.Touch.Old +#endif + +#else + +import System.PosixCompat + +newtype TimeSpec = TimeSpec EpochTime + +{- Noop for Windows -} +touchBoth :: FilePath -> TimeSpec -> TimeSpec -> Bool -> IO () +touchBoth _ _ _ _ = return () + +touch :: FilePath -> TimeSpec -> Bool -> IO () +touch _ _ _ = return () + +#endif diff --git a/Utility/Touch/Old.hsc b/Utility/Touch/Old.hsc new file mode 100644 index 0000000000..5345285f46 --- /dev/null +++ b/Utility/Touch/Old.hsc @@ -0,0 +1,123 @@ +{- Compatability interface for old version of unix, to be removed eventally. + - + - Copyright 2011 Joey Hess + - + - License: BSD-2-clause + -} + +{-# LANGUAGE ForeignFunctionInterface, CPP #-} + +module Utility.Touch.Old ( + TimeSpec(..), + touchBoth, + touch +) where + +#include +#include +#include +#include + +#ifndef _BSD_SOURCE +#define _BSD_SOURCE +#endif + +#if (defined UTIME_OMIT && defined UTIME_NOW && defined AT_FDCWD && defined AT_SYMLINK_NOFOLLOW) +#define use_utimensat 1 + +import Utility.FileSystemEncoding + +import Control.Monad (when) +import Foreign +#endif + +import Foreign.C + +newtype TimeSpec = TimeSpec CTime + +touchBoth :: FilePath -> TimeSpec -> TimeSpec -> Bool -> IO () + +touch :: FilePath -> TimeSpec -> Bool -> IO () +touch file mtime = touchBoth file mtime mtime + +#ifdef use_utimensat + +at_fdcwd :: CInt +at_fdcwd = #const AT_FDCWD + +at_symlink_nofollow :: CInt +at_symlink_nofollow = #const AT_SYMLINK_NOFOLLOW + +instance Storable TimeSpec where + -- use the larger alignment of the two types in the struct + alignment _ = max sec_alignment nsec_alignment + where + sec_alignment = alignment (1::CTime) + nsec_alignment = alignment (1::CLong) + sizeOf _ = #{size struct timespec} + peek ptr = do + sec <- #{peek struct timespec, tv_sec} ptr + return $ TimeSpec sec + poke ptr (TimeSpec sec) = do + #{poke struct timespec, tv_sec} ptr sec + #{poke struct timespec, tv_nsec} ptr (0 :: CLong) + +{- While its interface is beastly, utimensat is in recent + POSIX standards, unlike lutimes. -} +foreign import ccall "utimensat" + c_utimensat :: CInt -> CString -> Ptr TimeSpec -> CInt -> IO CInt + +touchBoth file atime mtime follow = + allocaArray 2 $ \ptr -> + withFilePath file $ \f -> do + pokeArray ptr [atime, mtime] + r <- c_utimensat at_fdcwd f ptr flags + when (r /= 0) $ throwErrno "touchBoth" + where + flags + | follow = 0 + | otherwise = at_symlink_nofollow + +#else +#if 0 +{- Using lutimes is needed for BSD. + - + - TODO: test if lutimes is available. May have to do it in configure. + - TODO: TimeSpec uses a CTime, while tv_sec is a CLong. It is implementation + - dependent whether these are the same; need to find a cast that works. + - (Without the cast it works on linux i386, but + - maybe not elsewhere.) + -} + +instance Storable TimeSpec where + alignment _ = alignment (1::CLong) + sizeOf _ = #{size struct timeval} + peek ptr = do + sec <- #{peek struct timeval, tv_sec} ptr + return $ TimeSpec sec + poke ptr (TimeSpec sec) = do + #{poke struct timeval, tv_sec} ptr sec + #{poke struct timeval, tv_usec} ptr (0 :: CLong) + +foreign import ccall "utimes" + c_utimes :: CString -> Ptr TimeSpec -> IO CInt +foreign import ccall "lutimes" + c_lutimes :: CString -> Ptr TimeSpec -> IO CInt + +touchBoth file atime mtime follow = + allocaArray 2 $ \ptr -> + withFilePath file $ \f -> do + pokeArray ptr [atime, mtime] + r <- syscall f ptr + when (r /= 0) $ + throwErrno "touchBoth" + where + syscall + | follow = c_lutimes + | otherwise = c_utimes + +#else +#warning "utimensat and lutimes not available; building without symlink timestamp preservation support" +touchBoth _ _ _ _ = return () +#endif +#endif diff --git a/Utility/Tuple.hs b/Utility/Tuple.hs new file mode 100644 index 0000000000..25c6e8f36f --- /dev/null +++ b/Utility/Tuple.hs @@ -0,0 +1,17 @@ +{- tuple utility functions + - + - Copyright 2017 Joey Hess + - + - License: BSD-2-clause + -} + +module Utility.Tuple where + +fst3 :: (a,b,c) -> a +fst3 (a,_,_) = a + +snd3 :: (a,b,c) -> b +snd3 (_,b,_) = b + +thd3 :: (a,b,c) -> c +thd3 (_,_,c) = c diff --git a/Utility/Url.hs b/Utility/Url.hs new file mode 100644 index 0000000000..3fd02fd432 --- /dev/null +++ b/Utility/Url.hs @@ -0,0 +1,557 @@ +{- Url downloading. + - + - Copyright 2011-2018 Joey Hess + - + - License: BSD-2-clause + -} + +{-# LANGUAGE CPP #-} +{-# LANGUAGE OverloadedStrings #-} +{-# LANGUAGE RankNTypes #-} +{-# LANGUAGE FlexibleContexts #-} + +module Utility.Url ( + newManager, + managerSettings, + URLString, + UserAgent, + Scheme, + mkScheme, + allowedScheme, + UrlDownloader(..), + UrlOptions(..), + defUrlOptions, + mkUrlOptions, + check, + checkBoth, + exists, + UrlInfo(..), + getUrlInfo, + assumeUrlExists, + download, + sinkResponseFile, + downloadPartial, + parseURIRelaxed, + matchStatusCodeException, + matchHttpExceptionContent, +) where + +import Common +import Utility.Metered +import Utility.HttpManagerRestricted + +import Network.URI +import Network.HTTP.Types +import qualified Data.CaseInsensitive as CI +import qualified Data.ByteString as B +import qualified Data.ByteString.UTF8 as B8 +import qualified Data.ByteString.Lazy as L +import qualified Data.Set as S +import Control.Monad.Trans.Resource +import Network.HTTP.Conduit +import Network.HTTP.Client +import Data.Conduit +import System.Log.Logger + +#if ! MIN_VERSION_http_client(0,5,0) +responseTimeoutNone :: Maybe Int +responseTimeoutNone = Nothing +#endif + +managerSettings :: ManagerSettings +#if MIN_VERSION_http_conduit(2,1,7) +managerSettings = tlsManagerSettings +#else +managerSettings = conduitManagerSettings +#endif + { managerResponseTimeout = responseTimeoutNone } + +type URLString = String + +type Headers = [String] + +type UserAgent = String + +newtype Scheme = Scheme (CI.CI String) + deriving (Eq, Ord) + +mkScheme :: String -> Scheme +mkScheme = Scheme . CI.mk + +fromScheme :: Scheme -> String +fromScheme (Scheme s) = CI.original s + +data UrlOptions = UrlOptions + { userAgent :: Maybe UserAgent + , reqHeaders :: Headers + , urlDownloader :: UrlDownloader + , applyRequest :: Request -> Request + , httpManager :: Manager + , allowedSchemes :: S.Set Scheme + } + +data UrlDownloader + = DownloadWithConduit + | DownloadWithCurl [CommandParam] + +defUrlOptions :: IO UrlOptions +defUrlOptions = UrlOptions + <$> pure Nothing + <*> pure [] + <*> pure DownloadWithConduit + <*> pure id + <*> newManager managerSettings + <*> pure (S.fromList $ map mkScheme ["http", "https", "ftp"]) + +mkUrlOptions :: Maybe UserAgent -> Headers -> UrlDownloader -> Manager -> S.Set Scheme -> UrlOptions +mkUrlOptions defuseragent reqheaders urldownloader manager = + UrlOptions useragent reqheaders urldownloader applyrequest manager + where + applyrequest = \r -> r { requestHeaders = requestHeaders r ++ addedheaders } + addedheaders = uaheader ++ otherheaders + useragent = maybe defuseragent (Just . B8.toString . snd) + (headMaybe uafromheaders) + uaheader = case useragent of + Nothing -> [] + Just ua -> [(hUserAgent, B8.fromString ua)] + (uafromheaders, otherheaders) = partition (\(h, _) -> h == hUserAgent) + (map toheader reqheaders) + toheader s = + let (h, v) = separate (== ':') s + h' = CI.mk (B8.fromString h) + in case v of + (' ':v') -> (h', B8.fromString v') + _ -> (h', B8.fromString v) + +curlParams :: UrlOptions -> [CommandParam] -> [CommandParam] +curlParams uo ps = ps ++ uaparams ++ headerparams ++ addedparams ++ schemeparams + where + uaparams = case userAgent uo of + Nothing -> [] + Just ua -> [Param "--user-agent", Param ua] + headerparams = concatMap (\h -> [Param "-H", Param h]) (reqHeaders uo) + addedparams = case urlDownloader uo of + DownloadWithConduit -> [] + DownloadWithCurl l -> l + schemeparams = + [ Param "--proto" + , Param $ intercalate "," ("-all" : schemelist) + ] + schemelist = map fromScheme $ S.toList $ allowedSchemes uo + +checkPolicy :: UrlOptions -> URI -> a -> IO a -> IO a +checkPolicy uo u onerr a + | allowedScheme uo u = a + | otherwise = do + hPutStrLn stderr $ + "Configuration does not allow accessing " ++ show u + hFlush stderr + return onerr + +unsupportedUrlScheme :: URI -> IO () +unsupportedUrlScheme u = do + hPutStrLn stderr $ + "Unsupported url scheme " ++ show u + hFlush stderr + +allowedScheme :: UrlOptions -> URI -> Bool +allowedScheme uo u = uscheme `S.member` allowedSchemes uo + where + uscheme = mkScheme $ takeWhile (/=':') (uriScheme u) + +{- Checks that an url exists and could be successfully downloaded, + - also checking that its size, if available, matches a specified size. -} +checkBoth :: URLString -> Maybe Integer -> UrlOptions -> IO Bool +checkBoth url expected_size uo = do + v <- check url expected_size uo + return (fst v && snd v) + +check :: URLString -> Maybe Integer -> UrlOptions -> IO (Bool, Bool) +check url expected_size uo = go <$> getUrlInfo url uo + where + go (UrlInfo False _ _) = (False, False) + go (UrlInfo True Nothing _) = (True, True) + go (UrlInfo True s _) = case expected_size of + Just _ -> (True, expected_size == s) + Nothing -> (True, True) + +exists :: URLString -> UrlOptions -> IO Bool +exists url uo = urlExists <$> getUrlInfo url uo + +data UrlInfo = UrlInfo + { urlExists :: Bool + , urlSize :: Maybe Integer + , urlSuggestedFile :: Maybe FilePath + } + deriving (Show) + +assumeUrlExists :: UrlInfo +assumeUrlExists = UrlInfo True Nothing Nothing + +{- Checks that an url exists and could be successfully downloaded, + - also returning its size and suggested filename if available. -} +getUrlInfo :: URLString -> UrlOptions -> IO UrlInfo +getUrlInfo url uo = case parseURIRelaxed url of + Just u -> checkPolicy uo u dne $ + case (urlDownloader uo, parseUrlConduit (show u)) of + (DownloadWithConduit, Just req) -> + existsconduit req + `catchNonAsync` (const $ return dne) + (DownloadWithConduit, Nothing) + | isfileurl u -> existsfile u + | otherwise -> do + unsupportedUrlScheme u + return dne + (DownloadWithCurl _, _) + | isfileurl u -> existsfile u + | otherwise -> existscurl u + Nothing -> return dne + where + dne = UrlInfo False Nothing Nothing + found sz f = return $ UrlInfo True sz f + + isfileurl u = uriScheme u == "file:" + + curlparams = curlParams uo $ + [ Param "-s" + , Param "--head" + , Param "-L", Param url + , Param "-w", Param "%{http_code}" + ] + + extractlencurl s = case lastMaybe $ filter ("Content-Length:" `isPrefixOf`) (lines s) of + Just l -> case lastMaybe $ words l of + Just sz -> readish sz + _ -> Nothing + _ -> Nothing + + extractlen = readish . B8.toString + <=< lookup hContentLength . responseHeaders + + extractfilename = contentDispositionFilename . B8.toString + <=< lookup hContentDisposition . responseHeaders + + existsconduit req = do + let req' = headRequest (applyRequest uo req) + debugM "url" (show req') + runResourceT $ do + resp <- http req' (httpManager uo) + -- forces processing the response while + -- within the runResourceT + liftIO $ if responseStatus resp == ok200 + then found + (extractlen resp) + (extractfilename resp) + else return dne + + existscurl u = do + output <- catchDefaultIO "" $ + readProcess "curl" $ toCommand curlparams + let len = extractlencurl output + let good = found len Nothing + let isftp = or + [ "ftp" `isInfixOf` uriScheme u + -- Check to see if http redirected to ftp. + , "Location: ftp://" `isInfixOf` output + ] + case lastMaybe (lines output) of + Just ('2':_:_) -> good + -- don't try to parse ftp status codes; if curl + -- got a length, it's good + _ | isftp && isJust len -> good + _ -> return dne + + existsfile u = do + let f = unEscapeString (uriPath u) + s <- catchMaybeIO $ getFileStatus f + case s of + Just stat -> do + sz <- getFileSize' f stat + found (Just sz) Nothing + Nothing -> return dne + +-- Parse eg: attachment; filename="fname.ext" +-- per RFC 2616 +contentDispositionFilename :: String -> Maybe FilePath +contentDispositionFilename s + | "attachment; filename=\"" `isPrefixOf` s && "\"" `isSuffixOf` s = + Just $ reverse $ drop 1 $ reverse $ + drop 1 $ dropWhile (/= '"') s + | otherwise = Nothing + +headRequest :: Request -> Request +headRequest r = r + { method = methodHead + -- remove defaut Accept-Encoding header, to get actual, + -- not gzip compressed size. + , requestHeaders = (hAcceptEncoding, B.empty) : + filter (\(h, _) -> h /= hAcceptEncoding) + (requestHeaders r) + } + +{- Download a perhaps large file, with auto-resume of incomplete downloads. + - + - Displays error message on stderr when download failed. + -} +download :: MeterUpdate -> URLString -> FilePath -> UrlOptions -> IO Bool +download meterupdate url file uo = + catchJust matchHttpException go showhttpexception + `catchNonAsync` showerr + where + go = case parseURIRelaxed url of + Just u -> checkPolicy uo u False $ + case (urlDownloader uo, parseUrlConduit (show u)) of + (DownloadWithConduit, Just req) -> + downloadconduit req + (DownloadWithConduit, Nothing) + | isfileurl u -> downloadfile u + | otherwise -> do + unsupportedUrlScheme u + return False + (DownloadWithCurl _, _) + | isfileurl u -> downloadfile u + | otherwise -> downloadcurl + Nothing -> do + liftIO $ debugM "url" url + hPutStrLn stderr "download failed: invalid url" + return False + + isfileurl u = uriScheme u == "file:" + + downloadconduit req = catchMaybeIO (getFileSize file) >>= \case + Nothing -> runResourceT $ do + liftIO $ debugM "url" (show req') + resp <- http (applyRequest uo req') (httpManager uo) + if responseStatus resp == ok200 + then store zeroBytesProcessed WriteMode resp + else showrespfailure resp + Just sz -> resumeconduit req' sz + where + -- Override http-client's default decompression of gzip + -- compressed files. We want the unmodified file content. + req' = req + { requestHeaders = (hAcceptEncoding, "identity") : + filter ((/= hAcceptEncoding) . fst) + (requestHeaders req) + , decompress = const False + } + + alreadydownloaded sz s h = s == requestedRangeNotSatisfiable416 + && case lookup hContentRange h of + -- This could be improved by fixing + -- https://github.com/aristidb/http-types/issues/87 + Just crh -> crh == B8.fromString ("bytes */" ++ show sz) + Nothing -> False + + -- Resume download from where a previous download was interrupted, + -- when supported by the http server. The server may also opt to + -- send the whole file rather than resuming. + resumeconduit req sz = catchJust + (matchStatusCodeHeadersException (alreadydownloaded sz)) + dl + (const $ return True) + where + dl = runResourceT $ do + let req' = req { requestHeaders = resumeFromHeader sz : requestHeaders req } + liftIO $ debugM "url" (show req') + resp <- http req' (httpManager uo) + if responseStatus resp == partialContent206 + then store (BytesProcessed sz) AppendMode resp + else if responseStatus resp == ok200 + then store zeroBytesProcessed WriteMode resp + else showrespfailure resp + + showrespfailure resp = liftIO $ do + hPutStrLn stderr $ B8.toString $ + statusMessage $ responseStatus resp + hFlush stderr + return False + showhttpexception he = do +#if MIN_VERSION_http_client(0,5,0) + let msg = case he of + HttpExceptionRequest _ (StatusCodeException r _) -> + B8.toString $ statusMessage $ responseStatus r + HttpExceptionRequest _ (InternalException ie) -> + case fromException ie of + Nothing -> show ie + Just (ConnectionRestricted why) -> why + HttpExceptionRequest _ other -> show other + _ -> show he +#else + let msg = case he of + StatusCodeException status _ _ -> + B8.toString (statusMessage status) + _ -> show he +#endif + hPutStrLn stderr $ "download failed: " ++ msg + hFlush stderr + return False + showerr e = do + hPutStrLn stderr (show e) + hFlush stderr + return False + + store initialp mode resp = do + sinkResponseFile meterupdate initialp file mode resp + return True + + downloadcurl = do + -- curl does not create destination file + -- if the url happens to be empty, so pre-create. + unlessM (doesFileExist file) $ + writeFile file "" + let ps = curlParams uo + [ Param "-sS" + , Param "-f" + , Param "-L" + , Param "-C", Param "-" + ] + boolSystem "curl" (ps ++ [Param "-o", File file, File url]) + + downloadfile u = do + let src = unEscapeString (uriPath u) + withMeteredFile src meterupdate $ + L.writeFile file + return True + +{- Sinks a Response's body to a file. The file can either be opened in + - WriteMode or AppendMode. Updates the meter as data is received. + - + - Note that the responseStatus is not checked by this function. + -} +sinkResponseFile + :: MonadResource m + => MeterUpdate + -> BytesProcessed + -> FilePath + -> IOMode +#if MIN_VERSION_http_conduit(2,3,0) + -> Response (ConduitM () B8.ByteString m ()) +#else + -> Response (ResumableSource m B8.ByteString) +#endif + -> m () +sinkResponseFile meterupdate initialp file mode resp = do + (fr, fh) <- allocate (openBinaryFile file mode) hClose +#if MIN_VERSION_http_conduit(2,3,0) + runConduit $ responseBody resp .| go initialp fh +#else + responseBody resp $$+- go initialp fh +#endif + release fr + where + go sofar fh = await >>= \case + Nothing -> return () + Just bs -> do + let sofar' = addBytesProcessed sofar (B.length bs) + liftIO $ do + void $ meterupdate sofar' + B.hPut fh bs + go sofar' fh + +{- Downloads at least the specified number of bytes from an url. -} +downloadPartial :: URLString -> UrlOptions -> Int -> IO (Maybe L.ByteString) +downloadPartial url uo n = case parseURIRelaxed url of + Nothing -> return Nothing + Just u -> go u `catchNonAsync` const (return Nothing) + where + go u = case parseUrlConduit (show u) of + Nothing -> return Nothing + Just req -> do + let req' = applyRequest uo req + liftIO $ debugM "url" (show req') + withResponse req' (httpManager uo) $ \resp -> + if responseStatus resp == ok200 + then Just <$> brread n [] (responseBody resp) + else return Nothing + + -- could use brReadSome here, needs newer http-client dependency + brread n' l rb + | n' <= 0 = return (L.fromChunks (reverse l)) + | otherwise = do + bs <- brRead rb + if B.null bs + then return (L.fromChunks (reverse l)) + else brread (n' - B.length bs) (bs:l) rb + +{- Allows for spaces and other stuff in urls, properly escaping them. -} +parseURIRelaxed :: URLString -> Maybe URI +parseURIRelaxed s = maybe (parseURIRelaxed' s) Just $ + parseURI $ escapeURIString isAllowedInURI s + +parseUrlConduit :: URLString -> Maybe Request +#if MIN_VERSION_http_client(0,4,30) +parseUrlConduit = parseUrlThrow +#else +parseUrlConduit = parseUrl +#endif + +{- Some characters like '[' are allowed in eg, the address of + - an uri, but cannot appear unescaped further along in the uri. + - This handles that, expensively, by successively escaping each character + - from the back of the url until the url parses. + -} +parseURIRelaxed' :: URLString -> Maybe URI +parseURIRelaxed' s = go [] (reverse s) + where + go back [] = parseURI back + go back (c:cs) = case parseURI (escapeURIString isAllowedInURI (reverse (c:cs)) ++ back) of + Just u -> Just u + Nothing -> go (escapeURIChar escapemore c ++ back) cs + + escapemore '[' = False + escapemore ']' = False + escapemore c = isAllowedInURI c + +hAcceptEncoding :: CI.CI B.ByteString +hAcceptEncoding = "Accept-Encoding" + +hContentDisposition :: CI.CI B.ByteString +hContentDisposition = "Content-Disposition" + +hContentRange :: CI.CI B.ByteString +hContentRange = "Content-Range" + +resumeFromHeader :: FileSize -> Header +resumeFromHeader sz = (hRange, renderByteRanges [ByteRangeFrom sz]) + +{- Use with eg: + - + - > catchJust (matchStatusCodeException (== notFound404)) + -} +matchStatusCodeException :: (Status -> Bool) -> HttpException -> Maybe HttpException +matchStatusCodeException want = matchStatusCodeHeadersException (\s _h -> want s) + +#if MIN_VERSION_http_client(0,5,0) +matchStatusCodeHeadersException :: (Status -> ResponseHeaders -> Bool) -> HttpException -> Maybe HttpException +matchStatusCodeHeadersException want e@(HttpExceptionRequest _ (StatusCodeException r _)) + | want (responseStatus r) (responseHeaders r) = Just e + | otherwise = Nothing +matchStatusCodeHeadersException _ _ = Nothing +#else +matchStatusCodeHeadersException :: (Status -> ResponseHeaders -> Bool) -> HttpException -> Maybe HttpException +matchStatusCodeHeadersException want e@(StatusCodeException s r _) + | want s r = Just e + | otherwise = Nothing +matchStatusCodeHeadersException _ _ = Nothing +#endif + +{- Use with eg: + - + - > catchJust matchHttpException + -} +matchHttpException :: HttpException -> Maybe HttpException +matchHttpException = Just + +#if MIN_VERSION_http_client(0,5,0) +matchHttpExceptionContent :: (HttpExceptionContent -> Bool) -> HttpException -> Maybe HttpException +matchHttpExceptionContent want e@(HttpExceptionRequest _ hec) + | want hec = Just e + | otherwise = Nothing +matchHttpExceptionContent _ _ = Nothing +#else +matchHttpExceptionContent :: (HttpException -> Bool) -> HttpException -> Maybe HttpException +matchHttpExceptionContent want e + | want e = Just e + | otherwise = Nothing +#endif diff --git a/Utility/UserInfo.hs b/Utility/UserInfo.hs new file mode 100644 index 0000000000..a98576d488 --- /dev/null +++ b/Utility/UserInfo.hs @@ -0,0 +1,69 @@ +{- user info + - + - Copyright 2012 Joey Hess + - + - License: BSD-2-clause + -} + +{-# LANGUAGE CPP #-} +{-# OPTIONS_GHC -fno-warn-tabs #-} + +module Utility.UserInfo ( + myHomeDir, + myUserName, + myUserGecos, +) where + +import Utility.Env.Basic +import Utility.Exception +#ifndef mingw32_HOST_OS +import Utility.Data +import Control.Applicative +#endif + +import System.PosixCompat +import Prelude + +{- Current user's home directory. + - + - getpwent will fail on LDAP or NIS, so use HOME if set. -} +myHomeDir :: IO FilePath +myHomeDir = either giveup return =<< myVal env homeDirectory + where +#ifndef mingw32_HOST_OS + env = ["HOME"] +#else + env = ["USERPROFILE", "HOME"] -- HOME is used in Cygwin +#endif + +{- Current user's user name. -} +myUserName :: IO (Either String String) +myUserName = myVal env userName + where +#ifndef mingw32_HOST_OS + env = ["USER", "LOGNAME"] +#else + env = ["USERNAME", "USER", "LOGNAME"] +#endif + +myUserGecos :: IO (Maybe String) +-- userGecos crashes on Android and is not available on Windows. +#if defined(__ANDROID__) || defined(mingw32_HOST_OS) +myUserGecos = return Nothing +#else +myUserGecos = eitherToMaybe <$> myVal [] userGecos +#endif + +myVal :: [String] -> (UserEntry -> String) -> IO (Either String String) +myVal envvars extract = go envvars + where + go [] = either (const $ envnotset) (Right . extract) <$> get + go (v:vs) = maybe (go vs) (return . Right) =<< getEnv v +#ifndef mingw32_HOST_OS + -- This may throw an exception if the system doesn't have a + -- passwd file etc; don't let it crash. + get = tryNonAsync $ getUserEntryForID =<< getEffectiveUserID +#else + get = return envnotset +#endif + envnotset = Left ("environment not set: " ++ show envvars) diff --git a/Utility/Verifiable.hs b/Utility/Verifiable.hs new file mode 100644 index 0000000000..278c320aee --- /dev/null +++ b/Utility/Verifiable.hs @@ -0,0 +1,38 @@ +{- values verified using a shared secret + - + - Copyright 2012 Joey Hess + - + - License: BSD-2-clause + -} + +module Utility.Verifiable where + +import Data.ByteString.UTF8 (fromString) +import qualified Data.ByteString as S + +import Utility.Hash + +type Secret = S.ByteString +type HMACDigest = String + +{- A value, verifiable using a HMAC digest and a secret. -} +data Verifiable a = Verifiable + { verifiableVal :: a + , verifiableDigest :: HMACDigest + } + deriving (Eq, Read, Show) + +mkVerifiable :: Show a => a -> Secret -> Verifiable a +mkVerifiable a secret = Verifiable a (calcDigest (show a) secret) + +verify :: (Eq a, Show a) => Verifiable a -> Secret -> Bool +verify v secret = v == mkVerifiable (verifiableVal v) secret + +calcDigest :: String -> Secret -> HMACDigest +calcDigest v secret = calcMac HmacSha1 secret (fromString v) + +{- for quickcheck -} +prop_verifiable_sane :: String -> String -> Bool +prop_verifiable_sane a s = verify (mkVerifiable a secret) secret + where + secret = fromString s diff --git a/Utility/WebApp.hs b/Utility/WebApp.hs new file mode 100644 index 0000000000..6fc154329d --- /dev/null +++ b/Utility/WebApp.hs @@ -0,0 +1,221 @@ +{- Yesod webapp + - + - Copyright 2012-2014 Joey Hess + - + - License: BSD-2-clause + -} + +{-# LANGUAGE OverloadedStrings, CPP, RankNTypes #-} + +module Utility.WebApp where + +import Common +import Utility.Tmp +import Utility.FileMode +import Utility.AuthToken + +import qualified Yesod +import qualified Network.Wai as Wai +import Network.Wai.Handler.Warp +import Network.Wai.Handler.WarpTLS +import Network.HTTP.Types +import qualified Data.CaseInsensitive as CI +import Network.Socket +import "crypto-api" Crypto.Random +import qualified Web.ClientSession as CS +import qualified Data.ByteString as B +import qualified Data.Text as T +import qualified Data.Text.Encoding as TE +import Blaze.ByteString.Builder.Char.Utf8 (fromText) +import Blaze.ByteString.Builder (Builder) +import Control.Arrow ((***)) +import Control.Concurrent +#ifdef __ANDROID__ +import Data.Endian +#endif + +localhost :: HostName +localhost = "localhost" + +{- Builds a command to use to start or open a web browser showing an url. -} +browserProc :: String -> CreateProcess +#ifdef darwin_HOST_OS +browserProc url = proc "open" [url] +#else +#ifdef __ANDROID__ +-- Warning: The `am` command does not work very reliably on Android. +browserProc url = proc "am" + ["start", "-a", "android.intent.action.VIEW", "-d", url] +#else +#ifdef mingw32_HOST_OS +-- Warning: On Windows, no quoting or escaping of the url seems possible, +-- so spaces in it will cause problems. One approach is to make the url +-- be a relative filename, and adjust the returned CreateProcess to change +-- to the directory it's in. +browserProc url = proc "cmd" ["/c start " ++ url] +#else +browserProc url = proc "xdg-open" [url] +#endif +#endif +#endif + +{- Binds to a socket on localhost, or possibly a different specified + - hostname or address, and runs a webapp on it. + - + - An IO action can also be run, to do something with the address, + - such as start a web browser to view the webapp. + -} +runWebApp :: Maybe TLSSettings -> Maybe HostName -> Wai.Application -> (SockAddr -> IO ()) -> IO () +runWebApp tlssettings h app observer = withSocketsDo $ do + sock <- getSocket h + void $ forkIO $ go webAppSettings sock app + sockaddr <- fixSockAddr <$> getSocketName sock + observer sockaddr + where + go = (maybe runSettingsSocket (\ts -> runTLSSocket ts) tlssettings) + +fixSockAddr :: SockAddr -> SockAddr +#ifdef __ANDROID__ +{- On Android, the port is currently incorrectly returned in network + - byte order, which is wrong on little endian systems. -} +fixSockAddr (SockAddrInet (PortNum port) addr) = SockAddrInet (PortNum $ swapEndian port) addr +#endif +fixSockAddr addr = addr + +-- disable buggy sloworis attack prevention code +webAppSettings :: Settings +webAppSettings = setTimeout halfhour defaultSettings + where + halfhour = 30 * 60 + +{- Binds to a local socket, or if specified, to a socket on the specified + - hostname or address. Selects any free port, unless the hostname ends with + - ":port" + - + - Prefers to bind to the ipv4 address rather than the ipv6 address + - of localhost, if it's available. + -} +getSocket :: Maybe HostName -> IO Socket +getSocket h = do +#if defined(__ANDROID__) || defined (mingw32_HOST_OS) + -- getAddrInfo currently segfaults on Android. + -- The HostName is ignored by this code. + when (isJust h) $ + error "getSocket with HostName not supported on this OS" + addr <- inet_addr "127.0.0.1" + sock <- socket AF_INET Stream defaultProtocol + preparesocket sock + bind sock (SockAddrInet aNY_PORT addr) + use sock + where +#else + addrs <- getAddrInfo (Just hints) (Just hostname) Nothing + case (partition (\a -> addrFamily a == AF_INET) addrs) of + (v4addr:_, _) -> go v4addr + (_, v6addr:_) -> go v6addr + _ -> error "unable to bind to a local socket" + where + hostname = fromMaybe localhost h + hints = defaultHints { addrSocketType = Stream } + {- Repeated attempts because bind sometimes fails for an + - unknown reason on OSX. -} + go addr = go' 100 addr + go' :: Int -> AddrInfo -> IO Socket + go' 0 _ = error "unable to bind to local socket" + go' n addr = do + r <- tryIO $ bracketOnError (open addr) close (useaddr addr) + either (const $ go' (pred n) addr) return r + open addr = socket (addrFamily addr) (addrSocketType addr) (addrProtocol addr) + useaddr addr sock = do + preparesocket sock + bind sock (addrAddress addr) + use sock +#endif + preparesocket sock = setSocketOption sock ReuseAddr 1 + use sock = do + listen sock maxListenQueue + return sock + +lookupRequestField :: CI.CI B.ByteString -> Wai.Request -> B.ByteString +lookupRequestField k req = fromMaybe "" . lookup k $ Wai.requestHeaders req + +{- Rather than storing a session key on disk, use a random key + - that will only be valid for this run of the webapp. -} +webAppSessionBackend :: Yesod.Yesod y => y -> IO (Maybe Yesod.SessionBackend) +webAppSessionBackend _ = do + g <- newGenIO :: IO SystemRandom + case genBytes 96 g of + Left e -> error $ "failed to generate random key: " ++ show e + Right (s, _) -> case CS.initKey s of + Left e -> error $ "failed to initialize key: " ++ show e + Right key -> use key + where + timeout = 120 * 60 -- 120 minutes + use key = + Just . Yesod.clientSessionBackend key . fst + <$> Yesod.clientSessionDateCacher timeout + +{- A Yesod isAuthorized method, which checks the auth cgi parameter + - against a token extracted from the Yesod application. + - + - Note that the usual Yesod error page is bypassed on error, to avoid + - possibly leaking the auth token in urls on that page! + - + - If the predicate does not match the route, the auth parameter is not + - needed. + -} +checkAuthToken :: Yesod.MonadHandler m => Yesod.RenderRoute site => (Yesod.HandlerSite m -> AuthToken) -> Yesod.Route site -> ([T.Text] -> Bool) -> m Yesod.AuthResult +checkAuthToken extractAuthToken r predicate + | not (predicate (fst (Yesod.renderRoute r))) = return Yesod.Authorized + | otherwise = do + webapp <- Yesod.getYesod + req <- Yesod.getRequest + let params = Yesod.reqGetParams req + if (toAuthToken =<< lookup "auth" params) == Just (extractAuthToken webapp) + then return Yesod.Authorized + else Yesod.sendResponseStatus unauthorized401 () + +{- A Yesod joinPath method, which adds an auth cgi parameter to every + - url matching a predicate, containing a token extracted from the + - Yesod application. + - + - A typical predicate would exclude files under /static. + -} +insertAuthToken :: forall y. (y -> AuthToken) + -> ([T.Text] -> Bool) + -> y + -> T.Text + -> [T.Text] + -> [(T.Text, T.Text)] + -> Builder +insertAuthToken extractAuthToken predicate webapp root pathbits params = + fromText root `mappend` encodePath pathbits' encodedparams + where + pathbits' = if null pathbits then [T.empty] else pathbits + encodedparams = map (TE.encodeUtf8 *** go) params' + go "" = Nothing + go x = Just $ TE.encodeUtf8 x + authparam = (T.pack "auth", fromAuthToken (extractAuthToken webapp)) + params' + | predicate pathbits = authparam:params + | otherwise = params + +{- Creates a html shim file that's used to redirect into the webapp, + - to avoid exposing the secret token when launching the web browser. -} +writeHtmlShim :: String -> String -> FilePath -> IO () +writeHtmlShim title url file = viaTmp writeFileProtected file $ genHtmlShim title url + +{- TODO: generate this static file using Yesod. -} +genHtmlShim :: String -> String -> String +genHtmlShim title url = unlines + [ "" + , "" + , ""++ title ++ "" + , "" + , "" + , "

    " + , "" ++ title ++ "" + , "

    " + , "" + , "" + ] diff --git a/Utility/Yesod.hs b/Utility/Yesod.hs new file mode 100644 index 0000000000..5e85f3147e --- /dev/null +++ b/Utility/Yesod.hs @@ -0,0 +1,60 @@ +{- Yesod stuff, that's typically found in the scaffolded site. + - + - Also a bit of a compatability layer to make it easier to support yesod + - 1.1-1.4 in the same code base. + - + - Copyright 2012-2014 Joey Hess + - + - Licensed under the GNU GPL version 3 or higher. + -} + +{-# LANGUAGE CPP, RankNTypes, FlexibleContexts #-} + +module Utility.Yesod + ( module Y + , liftH +#ifndef __NO_TH__ + , widgetFile + , hamletTemplate +#endif +#if ! MIN_VERSION_yesod_core(1,2,20) + , withUrlRenderer +#endif + ) where + +import Yesod as Y +import Yesod.Form.Bootstrap3 as Y hiding (bfs) +#ifndef __NO_TH__ +import Yesod.Default.Util +import Language.Haskell.TH.Syntax (Q, Exp) +import Data.Default (def) +import Text.Hamlet hiding (Html) +#endif +#if ! MIN_VERSION_yesod(1,4,0) +import Data.Text (Text) +#endif + +#ifndef __NO_TH__ +widgetFile :: String -> Q Exp +widgetFile = widgetFileNoReload $ def + { wfsHamletSettings = defaultHamletSettings + { hamletNewlines = AlwaysNewlines + } + } + +hamletTemplate :: FilePath -> FilePath +hamletTemplate f = globFile "hamlet" f +#endif + +{- Lift Handler to Widget -} +#if MIN_VERSION_yesod_core(1,6,0) +liftH :: HandlerFor site a -> WidgetFor site a +#else +liftH :: Monad m => HandlerT site m a -> WidgetT site m a +#endif +liftH = handlerToWidget + +#if ! MIN_VERSION_yesod_core(1,2,20) +withUrlRenderer :: MonadHandler m => ((Route (HandlerSite m) -> [(Text, Text)] -> Text) -> output) -> m output +withUrlRenderer = giveUrlRenderer +#endif diff --git a/Utility/libkqueue.c b/Utility/libkqueue.c new file mode 100644 index 0000000000..3f40465e37 --- /dev/null +++ b/Utility/libkqueue.c @@ -0,0 +1,74 @@ +/* kqueue interface, C mini-library + * + * Copyright 2012 Joey Hess + * + * License: BSD-2-clause + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include + +/* The specified fds are added to the set of fds being watched for changes. + * Fds passed to prior calls still take effect, so it's most efficient to + * not pass the same fds repeatedly. + * + * Returns the fd that changed, or -1 on error. + */ +signed int helper(const int kq, const int fdcnt, const int *fdlist, int nodelay) { + int i, nev; + struct kevent evlist[1]; + struct kevent chlist[fdcnt]; + struct timespec avoiddelay = {0, 0}; + struct timespec *timeout = nodelay ? &avoiddelay : NULL; + + for (i = 0; i < fdcnt; i++) { + EV_SET(&chlist[i], fdlist[i], EVFILT_VNODE, + EV_ADD | EV_ENABLE | EV_CLEAR, + NOTE_WRITE, + 0, 0); + } + + nev = kevent(kq, chlist, fdcnt, evlist, 1, timeout); + if (nev == 1) + return evlist[0].ident; + else + return -1; +} + +/* Initializes a new, empty kqueue. */ +int init_kqueue() { + int kq; + if ((kq = kqueue()) == -1) { + perror("kqueue"); + exit(1); + } + return kq; +} + +/* Adds fds to the set that should be watched. */ +void addfds_kqueue(const int kq, const int fdcnt, const int *fdlist) { + helper(kq, fdcnt, fdlist, 1); +} + +/* Waits for a change event on a kqueue. */ +signed int waitchange_kqueue(const int kq) { + return helper(kq, 0, NULL, 0); +} + +/* +main () { + int list[1]; + int kq; + list[0]=open(".", O_RDONLY); + kq = init_kqueue(); + addfds_kqueue(kq, 1, list) + printf("change: %i\n", waitchange_kqueue(kq)); +} +*/ diff --git a/Utility/libkqueue.h b/Utility/libkqueue.h new file mode 100644 index 0000000000..692b47f14e --- /dev/null +++ b/Utility/libkqueue.h @@ -0,0 +1,3 @@ +int init_kqueue(); +void addfds_kqueue(const int kq, const int fdcnt, const int *fdlist); +signed int waitchange_kqueue(const int kq); diff --git a/bash-completion.bash b/bash-completion.bash new file mode 100644 index 0000000000..f0c8818e1f --- /dev/null +++ b/bash-completion.bash @@ -0,0 +1,38 @@ +# Use git-annex's built-in bash completion +# This bash completion is generated by the option parser, so it covers all +# commands, all options, and will never go out of date! +_git-annex() +{ + local CMDLINE + local IFS=$'\n' + CMDLINE=(--bash-completion-index $COMP_CWORD) + + for arg in ${COMP_WORDS[@]}; do + CMDLINE=(${CMDLINE[@]} --bash-completion-word $arg) + done + + COMPREPLY=( $(git-annex "${CMDLINE[@]}") ) +} + +complete -o bashdefault -o default -o filenames -F _git-annex git-annex + +# Called by git's bash completion script when completing "git annex" +_git_annex() { + local cmdline + CMDLINE=(--bash-completion-index $(($COMP_CWORD - 1))) + + local seen_git + local seen_annex + for arg in ${COMP_WORDS[@]}; do + if [ "$arg" = git ] && [ -z "$seen_git" ]; then + seen_git=1 + CMDLINE=(${CMDLINE[@]} --bash-completion-word git-annex) + elif [ "$arg" = annex ] && [ -z "$seen_annex" ]; then + seen_annex=1 + else + CMDLINE=(${CMDLINE[@]} --bash-completion-word $arg) + fi + done + + COMPREPLY=( $(git-annex "${CMDLINE[@]}") ) +} diff --git a/build.bat b/build.bat new file mode 100644 index 0000000000..c2ccfac5fb --- /dev/null +++ b/build.bat @@ -0,0 +1 @@ +sh standalone/windows/build-simple.sh diff --git a/debian/NEWS b/debian/NEWS new file mode 120000 index 0000000000..0fae0f8020 --- /dev/null +++ b/debian/NEWS @@ -0,0 +1 @@ +../NEWS \ No newline at end of file diff --git a/debian/changelog b/debian/changelog new file mode 120000 index 0000000000..a535994713 --- /dev/null +++ b/debian/changelog @@ -0,0 +1 @@ +../CHANGELOG \ No newline at end of file diff --git a/debian/compat b/debian/compat new file mode 100644 index 0000000000..ec635144f6 --- /dev/null +++ b/debian/compat @@ -0,0 +1 @@ +9 diff --git a/debian/control b/debian/control new file mode 100644 index 0000000000..022d23ee7c --- /dev/null +++ b/debian/control @@ -0,0 +1,141 @@ +Source: git-annex +Section: utils +Priority: optional +Build-Depends: + debhelper (>= 9), + ghc (>= 7.4), + cabal-install, + libghc-mtl-dev (>= 2.1.1), + libghc-split-dev, + libghc-data-default-dev, + libghc-hslogger-dev, + libghc-pcre-light-dev, + libghc-cryptonite-dev, + libghc-memory-dev, + libghc-attoparsec-dev, + libghc-sandi-dev, + libghc-utf8-string-dev, + libghc-aws-dev (>= 0.9.2-2~), + libghc-conduit-dev, + libghc-resourcet-dev, + libghc-quickcheck2-dev, + libghc-monad-control-dev (>= 0.3), + libghc-transformers-dev, + libghc-exceptions-dev (>= 0.6), + libghc-unix-compat-dev, + libghc-dlist-dev, + libghc-uuid-dev, + libghc-aeson-dev, + libghc-tagsoup-dev, + libghc-unordered-containers-dev, + libghc-ifelse-dev, + libghc-bloomfilter-dev, + libghc-edit-distance-dev, + libghc-hinotify-dev [linux-any], + libghc-stm-dev (>= 2.3), + libghc-dbus-dev (>= 0.10.7) [linux-any], + libghc-fdo-notify-dev (>= 0.3) [linux-any], + libghc-yesod-dev (>= 1.2.6.1) [i386 amd64 arm64 armel armhf kfreebsd-i386 kfreebsd-amd64 mips mips64el mipsel powerpc ppc64el s390x], + libghc-yesod-core-dev (>= 1.2.19) [i386 amd64 arm64 armel armhf kfreebsd-i386 kfreebsd-amd64 mips mips64el mipsel powerpc ppc64el s390x], + libghc-yesod-form-dev (>= 1.3.15) [i386 amd64 arm64 armel armhf kfreebsd-i386 kfreebsd-amd64 mips mips64el mipsel powerpc ppc64el s390x], + libghc-yesod-static-dev (>= 1.2.4) [i386 amd64 arm64 armel armhf kfreebsd-i386 kfreebsd-amd64 mips mips64el mipsel powerpc ppc64el s390x], + libghc-shakespeare-dev (>= 2.0.0) [i386 amd64 arm64 armel armhf kfreebsd-i386 kfreebsd-amd64 mips mips64el mipsel powerpc ppc64el s390x], + libghc-clientsession-dev [i386 amd64 arm64 armel armhf kfreebsd-i386 kfreebsd-amd64 mips mips64el mipsel powerpc ppc64el s390x], + libghc-warp-dev (>= 3.0.0.5) [i386 amd64 arm64 armel armhf kfreebsd-i386 kfreebsd-amd64 mips mips64el mipsel powerpc ppc64el s390x], + libghc-warp-tls-dev [i386 amd64 arm64 armel armhf kfreebsd-i386 kfreebsd-amd64 mips mips64el mipsel powerpc ppc64el s390x], + libghc-wai-dev [i386 amd64 arm64 armel armhf kfreebsd-i386 kfreebsd-amd64 mips mips64el mipsel powerpc ppc64el s390x], + libghc-wai-extra-dev [i386 amd64 arm64 armel armhf kfreebsd-i386 kfreebsd-amd64 mips mips64el mipsel powerpc ppc64el s390x], + libghc-dav-dev (>= 1.0), + libghc-persistent-dev, + libghc-persistent-template-dev, + libghc-persistent-sqlite-dev, + libghc-esqueleto-dev, + libghc-securemem-dev, + libghc-byteable-dev, + libghc-stm-chans-dev, + libghc-case-insensitive-dev, + libghc-http-types-dev, + libghc-http-conduit-dev, + libghc-blaze-builder-dev, + libghc-crypto-api-dev, + libghc-network-multicast-dev, + libghc-network-info-dev [linux-any kfreebsd-any], + libghc-safesemaphore-dev, + libghc-async-dev, + libghc-monad-logger-dev, + libghc-free-dev, + libghc-feed-dev (>= 0.3.9.2), + libghc-regex-tdfa-dev, + libghc-tasty-dev (>= 0.7), + libghc-tasty-hunit-dev, + libghc-tasty-quickcheck-dev, + libghc-tasty-rerun-dev, + libghc-optparse-applicative-dev (>= 0.11.0), + libghc-torrent-dev, + libghc-concurrent-output-dev, + libghc-disk-free-space-dev, + libghc-mountpoints-dev, + libghc-magic-dev, + libghc-socks-dev, + libghc-vector-dev, + lsof [linux-any], + ikiwiki, + libimage-magick-perl, + git (>= 1:1.8.1), + rsync, + curl, + openssh-client, + git-remote-gcrypt (>= 0.20130908-6), + gnupg, + gpg-agent, +Maintainer: Richard Hartmann +Standards-Version: 3.9.8 +Vcs-Git: git://git.kitenet.net/git-annex +Homepage: http://git-annex.branchable.com/ +XS-Testsuite: autopkgtest + +Package: git-annex +Architecture: any +Section: utils +Depends: ${misc:Depends}, ${shlibs:Depends}, + git (>= 1:1.8.1), + netbase, + rsync, + curl, + openssh-client (>= 1:5.6p1) +Recommends: + lsof, + gnupg, + bind9-host, + youtube-dl, + git-remote-gcrypt (>= 0.20130908-6), + nocache, + aria2, +Suggests: + xdot, + bup, + adb, + tor, + magic-wormhole, + tahoe-lafs, + libnss-mdns, + uftp, +Breaks: datalad (< 0.9.1) +Description: manage files with git, without checking their contents into git + git-annex allows managing files with git, without checking the file + contents into git. While that may seem paradoxical, it is useful when + dealing with files larger than git can currently easily handle, whether due + to limitations in memory, time, or disk space. + . + It can store large files in many places, from local hard drives, to a + large number of cloud storage services, including S3, WebDAV, + and rsync, with a dozen cloud storage providers usable via plugins. + Files can be stored encrypted with gpg, so that the cloud storage + provider cannot see your data. git-annex keeps track of where each file + is stored, so it knows how many copies are available, and has many + facilities to ensure your data is preserved. + . + git-annex can also be used to keep a folder in sync between computers, + noticing when files are changed, and automatically committing them + to git and transferring them to other computers. The git-annex webapp + makes it easy to set up and use git-annex this way. diff --git a/debian/copyright b/debian/copyright new file mode 120000 index 0000000000..dc5f40a221 --- /dev/null +++ b/debian/copyright @@ -0,0 +1 @@ +../COPYRIGHT \ No newline at end of file diff --git a/debian/create-standalone-changelog b/debian/create-standalone-changelog new file mode 100755 index 0000000000..c5e607e9e7 --- /dev/null +++ b/debian/create-standalone-changelog @@ -0,0 +1,17 @@ +#!/bin/bash +# +# A little helper script to build a package with standalone git-annex +# It relies on being run within git-annex's git repository +# +set -eu + +umask 022 + +# For NeuroDebian we rely on `git describe` output to get a +# sortable version which should work for any stage of a snapshot between +# releases and would remain sortable +ANNEX_VERSION=$(git describe HEAD) +ANNEX_NDVERSION=$( echo ${ANNEX_VERSION} | sed -e 's,-,+git,' -e 's,$,-1~ndall+1,') + +dch --noconf -v ${ANNEX_NDVERSION} \ + --force-bad-version --force-distribution -D neurodebian "Backported fresh snapshot" diff --git a/debian/doc-base b/debian/doc-base new file mode 100644 index 0000000000..f71a233333 --- /dev/null +++ b/debian/doc-base @@ -0,0 +1,9 @@ +Document: git-annex +Title: git-annex documentation +Author: Joey Hess +Abstract: All the documentation from git-annex's website. +Section: File Management + +Format: HTML +Index: /usr/share/doc/git-annex/html/index.html +Files: /usr/share/doc/git-annex/html/*.html diff --git a/debian/patches/series.standalone-build b/debian/patches/series.standalone-build new file mode 100644 index 0000000000..9dc994164d --- /dev/null +++ b/debian/patches/series.standalone-build @@ -0,0 +1 @@ +standalone-build diff --git a/debian/patches/standalone-build b/debian/patches/standalone-build new file mode 100644 index 0000000000..4567c81162 --- /dev/null +++ b/debian/patches/standalone-build @@ -0,0 +1,76 @@ +From: Yaroslav Halchenko +Subject: Patch debian/ to provide a standalone build of git-annex + +Origin: NeuroDebian +Last-Update: 2015-04-20 + +--- a/debian/control ++++ b/debian/control +@@ -84,6 +84,7 @@ Build-Depends: + git (>= 1:1.8.1), + rsync, + curl, ++ locales, + openssh-client, + git-remote-gcrypt (>= 0.20130908-6), + gnupg, +@@ -94,15 +95,14 @@ Vcs-Git: git://git.kitenet.net/git-annex + Homepage: http://git-annex.branchable.com/ + XS-Testsuite: autopkgtest + +-Package: git-annex ++Package: git-annex-standalone + Architecture: any + Section: utils +-Depends: ${misc:Depends}, ${shlibs:Depends}, +- git (>= 1:1.8.1), +- netbase, +- rsync, +- curl, +- openssh-client (>= 1:5.6p1) ++Conflicts: git-annex ++Provides: git-annex ++Depends: ${misc:Depends}, ++ git, ++ openssh-client + Recommends: + lsof, + gnupg, +@@ -121,7 +121,7 @@ Suggests: + libnss-mdns, + uftp, + Breaks: datalad (< 0.9.1) +-Description: manage files with git, without checking their contents into git ++Description: manage files with git, without checking their contents into git -- standalone build + git-annex allows managing files with git, without checking the file + contents into git. While that may seem paradoxical, it is useful when + dealing with files larger than git can currently easily handle, whether due +@@ -139,3 +139,7 @@ Description: manage files with git, with + noticing when files are changed, and automatically committing them + to git and transferring them to other computers. The git-annex webapp + makes it easy to set up and use git-annex this way. ++ . ++ This package provides a standalone bundle build of git-annex, which ++ should be installable on any more or less recent Debian or Ubuntu ++ release. +--- /dev/null ++++ b/debian/install +@@ -0,0 +1 @@ ++tmp/git-annex.linux usr/lib +--- /dev/null ++++ b/debian/links +@@ -0,0 +1,2 @@ ++/usr/lib/git-annex.linux/git-annex /usr/bin/git-annex ++/usr/lib/git-annex.linux/git-annex-shell /usr/bin/git-annex-shell +--- a/debian/rules ++++ b/debian/rules +@@ -38,4 +38,9 @@ override_dh_strip: + override_dh_makeshlibs: + dh_makeshlibs --noscripts + ++# For portability to older systems which do not support default (since ++# buster) .xz for components of the .deb ++override_dh_builddeb: ++ dh_builddeb -- -Zgzip ++ + endif diff --git a/debian/rules b/debian/rules new file mode 100755 index 0000000000..df298d4a2a --- /dev/null +++ b/debian/rules @@ -0,0 +1,41 @@ +#!/usr/bin/make -f + +export BUILDER=./Setup + +# -j1 is used for reproducible build +export BUILDEROPTIONS=-j1 + +STANDALONE_BUILD=$(shell grep -qe '^Package: git-annex-standalone' debian/control \ + && echo 1 || echo 0) + +# Do use the changelog's version number, rather than making one up. +export RELEASE_BUILD=1 + +%: + dh $@ + + +# Standalone build logic/helpers +ifeq ($(STANDALONE_BUILD),1) + +override_dh_auto_build: + make linuxstandalone GIT_ANNEX_PACKAGE_INSTALL=1 + +override_dh_auto_install: + make install-misc install-docs DESTDIR=debian/git-annex-standalone + # bins are linked into place, as instructed in debian/install and debian/links + +override_dh_fixperms: + dh_fixperms -Xld-linux + +# gdb can't do much with a haskell program, so avoid the debug package +override_dh_strip: + dh_strip --no-ddebs + +# Do not add "ldconfig" trigger since all libraries in the standalone build +# are private copies and the trigger activate-noawait trigger may not work +# on elderly distributions (e.g. squeeze) +override_dh_makeshlibs: + dh_makeshlibs --noscripts + +endif diff --git a/debian/tests/basics b/debian/tests/basics new file mode 100644 index 0000000000..2e4ea43fc1 --- /dev/null +++ b/debian/tests/basics @@ -0,0 +1,4 @@ +#!/bin/sh +testdir="$(mktemp -d)" +cd "$testdir" +exec git-annex test diff --git a/debian/tests/control b/debian/tests/control new file mode 100644 index 0000000000..928caf8e31 --- /dev/null +++ b/debian/tests/control @@ -0,0 +1,4 @@ +Tests: basics +Depends: @, git, rsync, gnupg +Restrictions: allow-stderr + diff --git a/doc/Android.mdwn b/doc/Android.mdwn new file mode 100644 index 0000000000..d7c00d8692 --- /dev/null +++ b/doc/Android.mdwn @@ -0,0 +1,53 @@ +git-annex is now available for Android. This includes the +[[git-annex assistant|/assistant]], for easy syncing between your Android +and other devices. You do not need to root your Android to use git-annex. + +[[Android installation instructions|/install/android]] + +When you run the git-annex Android app, two windows will open. The first is +a terminal window, and the second is a web browser showing the git-annex +webapp. + +[[!img apps.png alt="two windows"]] + +[[!toc ]] + +## closing and reopening the webapp + +The webapp does not need to be left open after you've set up your +repository. As long as the terminal window is left open, git-annex will +remain running and sync your files. To re-open the webapp after closing it, +use the [[!img newwindow.png alt="New Window"]] icon in the terminal window. + +## starting git-annex + +The app is not currently automatically started on boot, so you will need to +manually open it to keep your files in sync. You do not need to leave the +app running all the time, though. It will sync back up automatically when +started. + +## stopping git-annex + +Simply close the terminal window to stop git-annex from running. + +## using the command line + +[[!img terminal.png alt="Android terminal"]] + +If you prefer to use `git-annex` at the command line, you can do so using the +terminal. A fairly full set of tools is provided, including `git`, `ssh`, +`rsync`, and `gpg`. + +To prevent the webapp from being automatically started +when a terminal window opens, go into the terminal preferences, to "Initial +Command", and clear out the default `git annex webapp` setting. + +Or, if you'd like to run the assistant automatically, but not open the +webapp, change the "Initial Command" to: `git annex assistant --autostart` + +## using from adb shell + +To set up the git-annex environment from within `adb shell`, run: +`/data/data/ga.androidterm/runshell` + +This will launch a shell that has git-annex, git, etc in PATH. diff --git a/doc/Android/comment_19_dc7b428f525a082834cb87221fc627ff._comment b/doc/Android/comment_19_dc7b428f525a082834cb87221fc627ff._comment new file mode 100644 index 0000000000..009ada0038 --- /dev/null +++ b/doc/Android/comment_19_dc7b428f525a082834cb87221fc627ff._comment @@ -0,0 +1,8 @@ +[[!comment format=mdwn + username="http://afoolishmanifesto.com/" + nickname="frioux" + subject="SSH Keys?" + date="2013-07-17T16:50:46Z" + content=""" +Is there a way I can use an SSH Key to connect to a remote server? What would be really cool, though maybe not feasible, would be to use connectbot as an ssh-agent. +"""]] diff --git a/doc/Android/comment_20_81940ea56ace3dcd5fa84dfccd88ad96._comment b/doc/Android/comment_20_81940ea56ace3dcd5fa84dfccd88ad96._comment new file mode 100644 index 0000000000..56a4bb632e --- /dev/null +++ b/doc/Android/comment_20_81940ea56ace3dcd5fa84dfccd88ad96._comment @@ -0,0 +1,10 @@ +[[!comment format=mdwn + username="http://joeyh.name/" + ip="4.154.4.90" + subject="comment 20" + date="2013-07-17T19:06:31Z" + content=""" +@frioux the webapp has a \"ssh server\" option that will set up a ssh key and use it for passwordless data transfer to a ssh server. + +The openssh included in the git-annex app fully supports everything you can usually do with ssh keys, so you can also set this up by hand. +"""]] diff --git a/doc/Android/comment_29_37aa87a451d4390ed367402eec740855._comment b/doc/Android/comment_29_37aa87a451d4390ed367402eec740855._comment new file mode 100644 index 0000000000..448050d12f --- /dev/null +++ b/doc/Android/comment_29_37aa87a451d4390ed367402eec740855._comment @@ -0,0 +1,12 @@ +[[!comment format=mdwn + username="http://joeyh.name/" + ip="4.154.0.21" + subject="comment 29" + date="2013-07-30T17:45:27Z" + content=""" +If you are experiencing a problem using git-annex on Android, please examine the list of [[bugs]] and add a new, detailed bug report if no-one has reported the problem. If you are not sure if you have a bug, or need help in filing a good bug report, ask for help in the [[forum]]. + +I have moved to [[oldcomments]] a lot of old comments about problems that may be fixed or +not (hard to tell without a bug report!) " This page cannot +scale to handle every bug report that someone wants to paste into it. +"""]] diff --git a/doc/Android/comment_5_ba11b81c671d9bcd6f496fbd6f562b0f._comment b/doc/Android/comment_5_ba11b81c671d9bcd6f496fbd6f562b0f._comment new file mode 100644 index 0000000000..54053b43df --- /dev/null +++ b/doc/Android/comment_5_ba11b81c671d9bcd6f496fbd6f562b0f._comment @@ -0,0 +1,16 @@ +[[!comment format=mdwn + username="http://mebus.myopenid.com/" + ip="2a01:198:3eb:0:4a5b:39ff:fea4:55b3" + subject="comment 5" + date="2013-10-19T18:05:52Z" + content=""" +Hallo, + +how can I use the app with public/private keys for SSH. Where can I add them? + +Thanks + +Mebus + + +"""]] diff --git a/doc/Android/comment_6_455bcbd36c7b5eeb905cc56da4666b07._comment b/doc/Android/comment_6_455bcbd36c7b5eeb905cc56da4666b07._comment new file mode 100644 index 0000000000..75a1db4c2f --- /dev/null +++ b/doc/Android/comment_6_455bcbd36c7b5eeb905cc56da4666b07._comment @@ -0,0 +1,30 @@ +[[!comment format=mdwn + username="madduck" + subject="Weirdness when run from adb shell" + date="2015-05-06T14:28:43Z" + content=""" +How is this designed to work in the contact of Androids crap permissions? + + shell@kminilte:/ $ /data/data/ga.androidterm/runshell + /system/bin/sh: /data/data/ga.androidterm/runshell: can't execute: Permission denied + + 126|shell@kminilte:/ $ /system/bin/sh /data/data/ga.androidterm/runshell + Falling back to hardcoded app location; cannot find expected files in /data/app-lib + + shell@kminilte:/sdcard/git-annex.home $ git + /system/bin/sh: git: not found + + 127|shell@kminilte:/sdcard/git-annex.home $ echo $PATH + /data/data/ga.androidterm/bin:/sbin:/vendor/bin:/system/sbin:/system/bin:/system/xbin + + shell@kminilte:/sdcard/git-annex.home $ /data/data/ga.androidterm/bin/git < + /data/data/ga.androidterm/bin/git: Permission denied + + shell@kminilte:/sdcard/git-annex.home $ ls -l /data/data/ga.androidterm/bin -d + drwx------ u0_a255 u0_a255 2015-05-05 07:58 bin + + shell@kminilte:/sdcard/git-annex.home $ id + uid=2000(shell) gid=2000(shell) groups=1003(graphics),1004(input),1007(log),1011(adb),1015(sdcard_rw),1028(sdcard_r),3001(net_bt_admin),3002(net_bt),3003(inet),3006(net_bw_stats) context=u:r:shell:s0 + + +"""]] diff --git a/doc/Android/comment_6_97704e0d89bb87155e019e09e54fc9bf._comment b/doc/Android/comment_6_97704e0d89bb87155e019e09e54fc9bf._comment new file mode 100644 index 0000000000..39c44a8fb8 --- /dev/null +++ b/doc/Android/comment_6_97704e0d89bb87155e019e09e54fc9bf._comment @@ -0,0 +1,13 @@ +[[!comment format=mdwn + username="https://www.google.com/accounts/o8/id?id=AItOawk2BFMjJW081uX4aJdhStmSFPBUGzL92ZU" + nickname="Frédéric" + subject="where do I put the SSH keys on Android" + date="2014-02-26T20:26:38Z" + content=""" +@mebus : +You can put your SSH keys here : + +/sdcard/git-annex.home/.ssh/id_rsa + +/sdcard/git-annex.home/.ssh/id_rsa.pub +"""]] diff --git a/doc/Android/comment_7_725ec376b56f51afa37dfd750c744f73._comment b/doc/Android/comment_7_725ec376b56f51afa37dfd750c744f73._comment new file mode 100644 index 0000000000..ea8f832502 --- /dev/null +++ b/doc/Android/comment_7_725ec376b56f51afa37dfd750c744f73._comment @@ -0,0 +1,10 @@ +[[!comment format=mdwn + username="joseph.rawson.works@85a210ab8c0e37a0b2d6bb235738b20e23e8878f" + nickname="joseph.rawson.works" + subject="ssh keys, github, xmpp" + date="2015-12-28T22:09:24Z" + content=""" +I generated an rsa keypair and added them to /sd/ga-h/.ssh, yet they were overwritten. I couldn't access my annexes located on github. I couldn't login to google talk, as instructed by the web app. + +Thanks for letting me know about gitlab supporting git-annex. +"""]] diff --git a/doc/Android/comment_8_e97d2e4213324cef62d99edebab7ccb1._comment b/doc/Android/comment_8_e97d2e4213324cef62d99edebab7ccb1._comment new file mode 100644 index 0000000000..60fde62273 --- /dev/null +++ b/doc/Android/comment_8_e97d2e4213324cef62d99edebab7ccb1._comment @@ -0,0 +1,12 @@ +[[!comment format=mdwn + username="IgorGanapolsky" + avatar="http://cdn.libravatar.org/avatar/6bc0985e3dedceb150f08ad6456c84b2" + subject="Assistant doesn't launch in Android N" + date="2016-10-25T01:43:58Z" + content=""" +Hello, +I have installed the APK on Android N. However, the assistant doesn't launch, gives an error on terminal: +\"Cannot link executive git-annex.\" + +What is causing this? Please help. +"""]] diff --git a/doc/Android/comment_9_d0d9c19d0c827b992c531e00201b1acf._comment b/doc/Android/comment_9_d0d9c19d0c827b992c531e00201b1acf._comment new file mode 100644 index 0000000000..f7260830a5 --- /dev/null +++ b/doc/Android/comment_9_d0d9c19d0c827b992c531e00201b1acf._comment @@ -0,0 +1,9 @@ +[[!comment format=mdwn + username="yarikoptic" + avatar="http://cdn.libravatar.org/avatar/f11e9c84cb18d26a1748c33b48c924b4" + subject="Decided to try again..." + date="2018-09-25T00:09:52Z" + content=""" +The same here on Android 8.1.0. likely because .app was build for 5, but there is nothing newer offered on Android downloads. +So the Android Port is abandoned overall? +"""]] diff --git a/doc/Android/oldcomments.mdwn b/doc/Android/oldcomments.mdwn new file mode 100644 index 0000000000..3566712cd4 --- /dev/null +++ b/doc/Android/oldcomments.mdwn @@ -0,0 +1,2 @@ +If one of these comments is yours, and you are still experiencing the +problem, please file a proper [[bug_report|bugs]]. --[[Joey]] diff --git a/doc/Android/oldcomments/comment_10_20e3d513b8b97496d76aca4619026cd6._comment b/doc/Android/oldcomments/comment_10_20e3d513b8b97496d76aca4619026cd6._comment new file mode 100644 index 0000000000..cf7a4fdb5b --- /dev/null +++ b/doc/Android/oldcomments/comment_10_20e3d513b8b97496d76aca4619026cd6._comment @@ -0,0 +1,16 @@ +[[!comment format=mdwn + username="http://yarikoptic.myopenid.com/" + nickname="site-myopenid" + subject="comment 10" + date="2013-05-24T03:11:50Z" + content=""" +>you said before the error was \"Read-only file system\". Now you're saying it's \"Cross-device link\". I'm slightly confused. + +;-) Sorry for confusion, here are the details: + +\"Read-only file system\" -- that error appeared when I started \"stock git annex\", i.e. from running /data/data/ga.androidterm/lib/lib.start.so . +Since you have suggested that it might be coming from hard linking command, I have ran that one manually, and that is when I got \"Cross-device link\" error, which suggests that hard linking is not the one at fault here. + +I will try fresh build now +Cheers, +"""]] diff --git a/doc/Android/oldcomments/comment_11_c96b8f1cc1583a74eb2483f48357f023._comment b/doc/Android/oldcomments/comment_11_c96b8f1cc1583a74eb2483f48357f023._comment new file mode 100644 index 0000000000..2e1ba65606 --- /dev/null +++ b/doc/Android/oldcomments/comment_11_c96b8f1cc1583a74eb2483f48357f023._comment @@ -0,0 +1,15 @@ +[[!comment format=mdwn + username="http://yarikoptic.myopenid.com/" + nickname="site-myopenid" + subject="fresh build" + date="2013-05-24T03:21:29Z" + content=""" +With fresh build got: + +u0_a39@android:/ $ git annex webapp +/system/bin/sh: git: not found + +the PATH is /sbin:/system/bin:/system/xbin + +where should git (and ga) reside now ? (/data somehow is not accessible now to u0_a39) +"""]] diff --git a/doc/Android/oldcomments/comment_12_6551f5fa081494b079c10a33c9b0d8ad._comment b/doc/Android/oldcomments/comment_12_6551f5fa081494b079c10a33c9b0d8ad._comment new file mode 100644 index 0000000000..39ce3e0588 --- /dev/null +++ b/doc/Android/oldcomments/comment_12_6551f5fa081494b079c10a33c9b0d8ad._comment @@ -0,0 +1,10 @@ +[[!comment format=mdwn + username="http://joeyh.name/" + nickname="joey" + subject="comment 12" + date="2013-05-24T03:26:33Z" + content=""" +You should be able to run /data/data/ga.androidterm/runshell even if you cannot ls /data. This adds /data/data/ga.androidterm/bin to PATH + +However, the shell that the app starts is started by runshell anyway, so I don't understand how this could happen. +"""]] diff --git a/doc/Android/oldcomments/comment_13_7c633d245651ec08f63194fe1fc194ae._comment b/doc/Android/oldcomments/comment_13_7c633d245651ec08f63194fe1fc194ae._comment new file mode 100644 index 0000000000..dae84414b1 --- /dev/null +++ b/doc/Android/oldcomments/comment_13_7c633d245651ec08f63194fe1fc194ae._comment @@ -0,0 +1,8 @@ +[[!comment format=mdwn + username="https://www.google.com/accounts/o8/id?id=AItOawkwjBDXkP9HAQKhjTgThGOxUa1B99y_WRA" + nickname="Franck" + subject="Still problems with my old N1/CM7" + date="2013-05-24T06:01:18Z" + content=""" +Hi, thank you for addressing this issue! I installed the new release but now it fails in another way: the message is just \"In mgmain NJI_OnLoad\" then the terminal says that the session is closed. +"""]] diff --git a/doc/Android/oldcomments/comment_14_60c2403140085f9caf48a33b59a36ab4._comment b/doc/Android/oldcomments/comment_14_60c2403140085f9caf48a33b59a36ab4._comment new file mode 100644 index 0000000000..a6f4598f6e --- /dev/null +++ b/doc/Android/oldcomments/comment_14_60c2403140085f9caf48a33b59a36ab4._comment @@ -0,0 +1,8 @@ +[[!comment format=mdwn + username="http://yarikoptic.myopenid.com/" + nickname="site-myopenid" + subject="It starts after uninstall/install" + date="2013-05-24T23:29:52Z" + content=""" +Hi Joey -- there is success here... previous installation was \"updated\" by installing the new package without uninstalling previous one, and that apparently didn't work correctly (I didn't even have bin/ directory you mentioned). So I have removed previous installation and reinstalled it again -- it starts now! Thanks ;) +"""]] diff --git a/doc/Android/oldcomments/comment_15_77bafc01b47d4cf8f96bde2b6704ed71._comment b/doc/Android/oldcomments/comment_15_77bafc01b47d4cf8f96bde2b6704ed71._comment new file mode 100644 index 0000000000..0ecd59fe66 --- /dev/null +++ b/doc/Android/oldcomments/comment_15_77bafc01b47d4cf8f96bde2b6704ed71._comment @@ -0,0 +1,8 @@ +[[!comment format=mdwn + username="http://yarikoptic.myopenid.com/" + nickname="site-myopenid" + subject="asking for ssh password in the terminal (not in web ui)" + date="2013-05-24T23:49:40Z" + content=""" +not sure if that is a known issue: whenever \"remote server\" is added, password needs to be typed back in the original terminal... is a bit challenging to do on android and not straightforward user-wise +"""]] diff --git a/doc/Android/oldcomments/comment_16_9af73451be09f03cfff81fdf9481ffc4._comment b/doc/Android/oldcomments/comment_16_9af73451be09f03cfff81fdf9481ffc4._comment new file mode 100644 index 0000000000..07923c1722 --- /dev/null +++ b/doc/Android/oldcomments/comment_16_9af73451be09f03cfff81fdf9481ffc4._comment @@ -0,0 +1,27 @@ +[[!comment format=mdwn + username="http://yarikoptic.myopenid.com/" + nickname="site-myopenid" + subject="Few other issues" + date="2013-05-25T15:35:46Z" + content=""" +Hi again. + +talking about 4.20130523-gcfe07a2 version: + +- because working in the terminal to interact with git-annex probably should not be a common case on Android, may be it is worth making default type of new added repository to become a full backup? I have initiated a new one, attached a remote one, it said \"synced\" but all the files were just containing symlinks and were not usable. I had to switch to \"full backup\" (or whatever that name) to finally get directory synced + +- log file might grow too large simply because of containing numerous entries for attempting connect remote repository while offline, e.g. + +Please make sure you have the correct access rights +and the repository exists. +ssh: Could not resolve hostname onerussian.com: No address associated with hostname +fatal: Could not read from remote repository. + +IMHO those should not be there at all, e.g. if it is known that ATM there is no network connectivity + +- In addition to two existing repositories (1 local /sdcard/annex, which is also avail at/storage/sdcard0/annex + 1 remote) I have added one more local (and said to keep it in sync with original local). But it didn't work -- it \"Synced with onerussian.com_annex but not with Annex\" and claimed that the /external/extSdCard/Annex doesn't exist, although it is there (and with .git generated etc). When I restarted the deamon I got into a \"new\" Repository: /storage/extSdCard/Annex which also listed the 1st local but with \"Failed to sync with localhost\" message -- no remote one listed. Whenever I try to \"Switch repository\" to /sdcard/annex (the original local) -- it starts loading a new page but gets stuck right there. The only way to revive webui is to go back to Dashboard. Log there says (retyping from the screen so typos might be there): + +error: cannot run git-receive-pack '/storage/sdcard0/annex': No such file or directory +fatal: unable to fork + +"""]] diff --git a/doc/Android/oldcomments/comment_17_f76561a654b534df3a807b1c045710b2._comment b/doc/Android/oldcomments/comment_17_f76561a654b534df3a807b1c045710b2._comment new file mode 100644 index 0000000000..bc4a648101 --- /dev/null +++ b/doc/Android/oldcomments/comment_17_f76561a654b534df3a807b1c045710b2._comment @@ -0,0 +1,8 @@ +[[!comment format=mdwn + username="http://yarikoptic.myopenid.com/" + nickname="site-myopenid" + subject="comment 17" + date="2013-05-29T02:43:29Z" + content=""" +joey -- any additional information could I provide to troubleshoot the issue? original repository seems to sync ok, but I can't \"administer\" it if I can't even switch to it... +"""]] diff --git a/doc/Android/oldcomments/comment_18_1b46cdf154ddadfe17e4b6e4054dc619._comment b/doc/Android/oldcomments/comment_18_1b46cdf154ddadfe17e4b6e4054dc619._comment new file mode 100644 index 0000000000..bf9d79060b --- /dev/null +++ b/doc/Android/oldcomments/comment_18_1b46cdf154ddadfe17e4b6e4054dc619._comment @@ -0,0 +1,17 @@ +[[!comment format=mdwn + username="http://aap.liquidid.net/" + nickname="AAP" + subject="comment 18" + date="2013-05-30T11:23:58Z" + content=""" +I too get the 'link busybox: Read-only file system' message. Here is my phone info: + +Phone: Samsung Galaxy Y GT-S5360 (rooted) +Android: 2.3.6 Gingerbread +BusyBox path: /system/xbin/ + + +Androids own terminal seems not to understand the d argument (-ld: No such file or directory) but over ssh 'ls -ld /data/data/ga.androidterm' returns + + drwxr-x--x 1 app_97 app_97 0 May 30 12:57 /data/data/ga.androidterm/ +"""]] diff --git a/doc/Android/oldcomments/comment_1_cc9caa5dd22dd67e5c1d22d697096dd2._comment b/doc/Android/oldcomments/comment_1_cc9caa5dd22dd67e5c1d22d697096dd2._comment new file mode 100644 index 0000000000..44b1a5d705 --- /dev/null +++ b/doc/Android/oldcomments/comment_1_cc9caa5dd22dd67e5c1d22d697096dd2._comment @@ -0,0 +1,15 @@ +[[!comment format=mdwn + username="http://yarikoptic.myopenid.com/" + nickname="site-myopenid" + subject="Does it require the device to be rooted?" + date="2013-05-16T20:55:45Z" + content=""" +Following your news on kickstarter downloaded the .apk, and installed it. Upn start I just got a terminal window with + + link busybox: Read-only file system + + [Terminal session finished] + +That is on Galaxy Note + +"""]] diff --git a/doc/Android/oldcomments/comment_21_5903f6a4a81a6534fa8cfafb3b6c37bb._comment b/doc/Android/oldcomments/comment_21_5903f6a4a81a6534fa8cfafb3b6c37bb._comment new file mode 100644 index 0000000000..1e247b96a5 --- /dev/null +++ b/doc/Android/oldcomments/comment_21_5903f6a4a81a6534fa8cfafb3b6c37bb._comment @@ -0,0 +1,8 @@ +[[!comment format=mdwn + username="http://afoolishmanifesto.com/" + nickname="frioux" + subject="SSH Keys - 2" + date="2013-07-17T22:56:37Z" + content=""" +@joey should I be using the nightlies to see that? Under \"Adding a remote server using ssh\" I only see Host name, user name, directory, and port. Will it only be an option after I type in a password? +"""]] diff --git a/doc/Android/oldcomments/comment_22_36afd354f9669a154d7b6b2c4d43ded9._comment b/doc/Android/oldcomments/comment_22_36afd354f9669a154d7b6b2c4d43ded9._comment new file mode 100644 index 0000000000..e5b5f964aa --- /dev/null +++ b/doc/Android/oldcomments/comment_22_36afd354f9669a154d7b6b2c4d43ded9._comment @@ -0,0 +1,8 @@ +[[!comment format=mdwn + username="http://joeyh.name/" + ip="4.154.6.48" + subject="comment 22" + date="2013-07-17T23:25:21Z" + content=""" +@frioux it will automatically generate a new ssh key and configure the server to use it, once you submit the form and enter the password to let it into the server. +"""]] diff --git a/doc/Android/oldcomments/comment_23_de98154792e8611a134429f06d82bcb1._comment b/doc/Android/oldcomments/comment_23_de98154792e8611a134429f06d82bcb1._comment new file mode 100644 index 0000000000..1f34a775e7 --- /dev/null +++ b/doc/Android/oldcomments/comment_23_de98154792e8611a134429f06d82bcb1._comment @@ -0,0 +1,8 @@ +[[!comment format=mdwn + username="http://afoolishmanifesto.com/" + nickname="frioux" + subject="comment 23" + date="2013-07-18T02:01:28Z" + content=""" +@joey: ok, I got it to connect and it indeed sent over a key etc. For some reason now though git-annex (on android) \"crashes\" shortly after starting. To be clear, the web app says that the program crashed, the console is still there. I suspect that it may have something to do with my largish remote repo and the time required to sync just the metadata, but I can't tell. Any ideas what I should do next? (Note that I *did* change it to manual mode because my phone doesn't have 30G of storage :) +"""]] diff --git a/doc/Android/oldcomments/comment_24_7ab509c25243009bfbffd796ec64e77b._comment b/doc/Android/oldcomments/comment_24_7ab509c25243009bfbffd796ec64e77b._comment new file mode 100644 index 0000000000..fdec15ac63 --- /dev/null +++ b/doc/Android/oldcomments/comment_24_7ab509c25243009bfbffd796ec64e77b._comment @@ -0,0 +1,10 @@ +[[!comment format=mdwn + username="http://afoolishmanifesto.com/" + nickname="frioux" + subject="comment 24" + date="2013-07-18T11:35:06Z" + content=""" +ok, it eventually got the details from the remote server, but now I'm getting some other oddities. here is some of my log that shows what I am running into + +Watcher crashed: addWatch: does not exist (No such file or directory) [2013-07-18 06:22:46 CDT] Watcher: warning Watcher crashed: addWatch: does not exist (No such file or directory) (scanning...) [2013-07-18 06:23:19 CDT] Watcher: Performing startup scan Watcher crashed: addWatch: does not exist (No such file or directory) [2013-07-18 06:24:28 CDT] Watcher: warning Watcher crashed: addWatch: does not exist (No such file or directory) (scanning...) [2013-07-18 06:24:31 CDT] Watcher: Performing startup scan Watcher crashed: addWatch: does not exist (No such file or directory) [2013-07-18 06:25:44 CDT] Watcher: warning Watcher crashed: addWatch: does not exist (No such file or directory) +"""]] diff --git a/doc/Android/oldcomments/comment_25_026d1a01d5753d71ac3dfc002f2a5eec._comment b/doc/Android/oldcomments/comment_25_026d1a01d5753d71ac3dfc002f2a5eec._comment new file mode 100644 index 0000000000..aa74230dc4 --- /dev/null +++ b/doc/Android/oldcomments/comment_25_026d1a01d5753d71ac3dfc002f2a5eec._comment @@ -0,0 +1,10 @@ +[[!comment format=mdwn + username="https://www.google.com/accounts/o8/id?id=AItOawnRfQArYOmDd7r2DC7DkIJFOQgqXCVcAeU" + nickname="Frew" + subject="comment 25" + date="2013-07-18T13:14:46Z" + content=""" +frioux here (something messed up with myopenid or something) + +So I deleted the repo on my phone (via the CLI since the web app seemed hung) and recreated it; this time making sure that I set things to manual mode ASAP. It didn't have the problem it was having before, but now what seems to have happened is that it fetches from the remote, commits to the local repo, and then immediately fetches and commits again. It looks like it's about a 4s repeat loop. Any ideas what I should do next? +"""]] diff --git a/doc/Android/oldcomments/comment_26_f0a044fb649d43e32c96b08edbc336c3._comment b/doc/Android/oldcomments/comment_26_f0a044fb649d43e32c96b08edbc336c3._comment new file mode 100644 index 0000000000..c7c0e623f9 --- /dev/null +++ b/doc/Android/oldcomments/comment_26_f0a044fb649d43e32c96b08edbc336c3._comment @@ -0,0 +1,12 @@ +[[!comment format=mdwn + username="http://joeyh.name/" + ip="4.154.0.140" + subject="comment 26" + date="2013-07-18T17:07:27Z" + content=""" +@Frew, you should file bug reports when you have a bug. + +One problem you mentioned had already had a bug report filed by someone +else: + So you can post your details there. +"""]] diff --git a/doc/Android/oldcomments/comment_27_6b9ae35b1ceeba14cd7a74e142870705._comment b/doc/Android/oldcomments/comment_27_6b9ae35b1ceeba14cd7a74e142870705._comment new file mode 100644 index 0000000000..b77e0873fa --- /dev/null +++ b/doc/Android/oldcomments/comment_27_6b9ae35b1ceeba14cd7a74e142870705._comment @@ -0,0 +1,34 @@ +[[!comment format=mdwn + username="https://www.google.com/accounts/o8/id?id=AItOawnaH44G3QbxBAYyDwy0PbvL0ls60XoaR3Y" + nickname="Nigel" + subject="Watcher crashed in Android on /storage/sdcard1 - bug?" + date="2013-07-29T11:50:46Z" + content=""" +In webapp UI, added on first install, the location for repository: /storage/sdcard1 + +!warning + +Watcher crashed: addWatch: + +permission denied (Permission denied) + +[Restart Thread] + +:Performing startup scan + + +In terminal Window 1: + +nex webapp < + + Detected a crippled filesystem. + + Enabling direct mode. + + Detected a filesystem without fifo support. + + Disabling ssh connection caching. + + +Android 4.1.1 Huawei Y300 Annex.apk v1.0.52 version 4.20130723 +"""]] diff --git a/doc/Android/oldcomments/comment_28_c91db1215f529aa68bfb0576c3c5eddc._comment b/doc/Android/oldcomments/comment_28_c91db1215f529aa68bfb0576c3c5eddc._comment new file mode 100644 index 0000000000..cf315c0d9c --- /dev/null +++ b/doc/Android/oldcomments/comment_28_c91db1215f529aa68bfb0576c3c5eddc._comment @@ -0,0 +1,10 @@ +[[!comment format=mdwn + username="Jonathan" + ip="63.131.117.194" + subject="link busybox: Read-only file system" + date="2013-07-29T20:08:12Z" + content=""" +Phone: HTC EVO 3d 4g +Model Number: pg86100 +Android Version: 4.0.3 +"""]] diff --git a/doc/Android/oldcomments/comment_2_c2422b7dd9d526b3616e49f48cf178c2._comment b/doc/Android/oldcomments/comment_2_c2422b7dd9d526b3616e49f48cf178c2._comment new file mode 100644 index 0000000000..bfa4decc41 --- /dev/null +++ b/doc/Android/oldcomments/comment_2_c2422b7dd9d526b3616e49f48cf178c2._comment @@ -0,0 +1,10 @@ +[[!comment format=mdwn + username="http://joeyh.name/" + nickname="joey" + subject="comment 2" + date="2013-05-17T22:28:34Z" + content=""" +The Android app works on many non-rooted Android systems. + +The \"link busybox: Read-only file system\" means that `/data/data/ga.androidterm/lib/lib.busybox.so` cannot be hard linked to `/data/data/ga.androidterm/busybox`. That's not normal. I'd appreciate if you could provide more information on your Android device, like Android version and model number. +"""]] diff --git a/doc/Android/oldcomments/comment_3_0e4980c27b13dbc28477c02a82898248._comment b/doc/Android/oldcomments/comment_3_0e4980c27b13dbc28477c02a82898248._comment new file mode 100644 index 0000000000..fdbfac1c6f --- /dev/null +++ b/doc/Android/oldcomments/comment_3_0e4980c27b13dbc28477c02a82898248._comment @@ -0,0 +1,14 @@ +[[!comment format=mdwn + username="http://yarikoptic.myopenid.com/" + nickname="site-myopenid" + subject="Follow-up information on my system" + date="2013-05-18T01:23:28Z" + content=""" +Sorry for the delay: my android is stock Samsung-tuned Jelly beans. +Android 4.1.2 +Baseband version N7000XXLSO + +not sure if that would be of any use :-/ nothing in the logs (aLogcat) if I filter by annex -- should there any debug output? what should be a key to search by? + + +"""]] diff --git a/doc/Android/oldcomments/comment_4_86f7b5444e2eaea7f8f7b9160f671a1d._comment b/doc/Android/oldcomments/comment_4_86f7b5444e2eaea7f8f7b9160f671a1d._comment new file mode 100644 index 0000000000..ad46a26be8 --- /dev/null +++ b/doc/Android/oldcomments/comment_4_86f7b5444e2eaea7f8f7b9160f671a1d._comment @@ -0,0 +1,10 @@ +[[!comment format=mdwn + username="https://www.google.com/accounts/o8/id?id=AItOawnu1NYw8UF-NoDbKu8YKVGxi8FoZLH7JPs" + nickname="Chris" + subject="Not starting browser on Nexus 7, Android 4.2.2" + date="2013-05-19T14:04:28Z" + content=""" +I just tried to run this on my Nexus 7 which has Android 4.2.2, and I received the following: + +In spite of that, though, the URL provided still worked. +"""]] diff --git a/doc/Android/oldcomments/comment_5_9d78009435736a178d5a3f5a9bc0ed6a._comment b/doc/Android/oldcomments/comment_5_9d78009435736a178d5a3f5a9bc0ed6a._comment new file mode 100644 index 0000000000..d29a59036b --- /dev/null +++ b/doc/Android/oldcomments/comment_5_9d78009435736a178d5a3f5a9bc0ed6a._comment @@ -0,0 +1,8 @@ +[[!comment format=mdwn + username="http://joeyh.name/" + nickname="joey" + subject="comment 5" + date="2013-05-19T19:46:14Z" + content=""" +@Chris, that is a known bug: [[bugs/Android_app_permission_denial_on_startup]] +"""]] diff --git a/doc/Android/oldcomments/comment_6_7b9523ddb20dc4a929e556c3ed0c7406._comment b/doc/Android/oldcomments/comment_6_7b9523ddb20dc4a929e556c3ed0c7406._comment new file mode 100644 index 0000000000..1c4aceaefc --- /dev/null +++ b/doc/Android/oldcomments/comment_6_7b9523ddb20dc4a929e556c3ed0c7406._comment @@ -0,0 +1,18 @@ +[[!comment format=mdwn + username="http://joeyh.name/" + nickname="joey" + subject="comment 6" + date="2013-05-19T20:06:56Z" + content=""" +@yarikoptic, there is a process you can perform that will help me determine what's going on. + +You should be able to get the git-annex app to let you into a shell. You can do this by starting the app, and then going into its configuration menu, to Preferences, selecting \"Command Line\", and changing it to run \"/system/bin/sh\" + +Then when you open a new window in the git-annex app, you'll be at a shell prompt. From there, you can run: + +ls -ld /data/data/ga.androidterm + +I'm interested to know a) whether the directory exists and b) what permissions and owner it has. On my tablet, I get back \"drwxr-x--x app_39 app_39\" .. and if I run `id` in the shell, it tells me it's running as `app_39`. + +My guess is the directory probably does exist, but cannot be written to by the app. If you're able to verify that, the next step will be to investigate if there is some other directory that the app can write to. It needs to be able to write to someplace that is not on the `/sdcard` to install itself. +"""]] diff --git a/doc/Android/oldcomments/comment_7_a56628a622da752806c42c5b8b54ceef._comment b/doc/Android/oldcomments/comment_7_a56628a622da752806c42c5b8b54ceef._comment new file mode 100644 index 0000000000..df0c0a2de2 --- /dev/null +++ b/doc/Android/oldcomments/comment_7_a56628a622da752806c42c5b8b54ceef._comment @@ -0,0 +1,8 @@ +[[!comment format=mdwn + username="https://www.google.com/accounts/o8/id?id=AItOawkwjBDXkP9HAQKhjTgThGOxUa1B99y_WRA" + nickname="Franck" + subject="Link issue" + date="2013-05-22T12:01:38Z" + content=""" +Hi, I have exactly the same problem with the link that fails on my phone. However, I checked the permissions and they are as you describe on your tablet (except for the app number). At the same time, everything is fine on my tablet... The phone runs an old Cyanogenmod 7.2.0 (Android 2.3.7) while the tablet is a more recent Asus TF700T (Android 4.1.1). Let me know if you want me to run tests. +"""]] diff --git a/doc/Android/oldcomments/comment_8_19656ec99b8f6aa64c1d01a3c9ae9bd0._comment b/doc/Android/oldcomments/comment_8_19656ec99b8f6aa64c1d01a3c9ae9bd0._comment new file mode 100644 index 0000000000..43ff8d64bd --- /dev/null +++ b/doc/Android/oldcomments/comment_8_19656ec99b8f6aa64c1d01a3c9ae9bd0._comment @@ -0,0 +1,8 @@ +[[!comment format=mdwn + username="http://yarikoptic.myopenid.com/" + nickname="site-myopenid" + subject="why ln failed" + date="2013-05-23T13:27:39Z" + content=""" +Finally got to check it out: so indeed hardlinking fails but not because of permissions but \"link failed Cross-device link\" that lib is -> /mnt/asec/ga.androidterm-1/lib which resides on a different partition (vfat, /dev/block/dm-2, ro) from /data (ext4, /dev/block/mmcblk0p10) +"""]] diff --git a/doc/Android/oldcomments/comment_9_55e703ae105d0c0ee9ac50df8cc59dfb._comment b/doc/Android/oldcomments/comment_9_55e703ae105d0c0ee9ac50df8cc59dfb._comment new file mode 100644 index 0000000000..0970412a4d --- /dev/null +++ b/doc/Android/oldcomments/comment_9_55e703ae105d0c0ee9ac50df8cc59dfb._comment @@ -0,0 +1,10 @@ +[[!comment format=mdwn + username="http://joeyh.name/" + nickname="joey" + subject="comment 9" + date="2013-05-23T18:44:46Z" + content=""" +@yarikoptic you said before the error was \"Read-only file system\". Now you're saying it's \"Cross-device link\". I'm slightly confused. + +I've reworked the android app to not need any hard links. Try the current autobuild: +"""]] diff --git a/doc/Void.mdwn b/doc/Void.mdwn new file mode 100644 index 0000000000..772c9d9b93 --- /dev/null +++ b/doc/Void.mdwn @@ -0,0 +1 @@ +git-annex can be installed in Void Linux by running: `xbps-install git-annex`. diff --git a/doc/android/DCIM.png b/doc/android/DCIM.png new file mode 100644 index 0000000000..3ac0933234 Binary files /dev/null and b/doc/android/DCIM.png differ diff --git a/doc/android/appinstalled.png b/doc/android/appinstalled.png new file mode 100644 index 0000000000..166120d482 Binary files /dev/null and b/doc/android/appinstalled.png differ diff --git a/doc/android/apps.png b/doc/android/apps.png new file mode 100644 index 0000000000..310757f1e6 Binary files /dev/null and b/doc/android/apps.png differ diff --git a/doc/android/install.png b/doc/android/install.png new file mode 100644 index 0000000000..882455df16 Binary files /dev/null and b/doc/android/install.png differ diff --git a/doc/android/newwindow.png b/doc/android/newwindow.png new file mode 100644 index 0000000000..4cca6ae09b Binary files /dev/null and b/doc/android/newwindow.png differ diff --git a/doc/android/terminal.png b/doc/android/terminal.png new file mode 100644 index 0000000000..9afa2720fa Binary files /dev/null and b/doc/android/terminal.png differ diff --git a/doc/android/webapp.png b/doc/android/webapp.png new file mode 100644 index 0000000000..edb5c5ccd9 Binary files /dev/null and b/doc/android/webapp.png differ diff --git a/doc/assistant.mdwn b/doc/assistant.mdwn new file mode 100644 index 0000000000..0147f6ddf4 --- /dev/null +++ b/doc/assistant.mdwn @@ -0,0 +1,38 @@ +The git-annex assistant creates a synchronised folder on each of your +OSX and Linux computers, Android devices, removable drives, NAS appliances, +and cloud services. The contents of the folder are the same everywhere. +It's very easy to use, and has all the power of git and git-annex. + +## installation + +The git-annex assistant comes as part of git-annex. +See [[install]] to get it installed. + +See the [[release_notes]] for an overview of the status, and upgrade +instructions. + +## intro screencast + +[[!inline feeds=no template=bare pages=videos/git-annex_assistant_lan]] + +## documentation + +* [[Basic usage|quickstart]] +* [[Android documentation|/Android]] +* Want to make two nearby computers share the same synchronised folder? + Follow the [[local_pairing_walkthrough]]. +* Want to share files with a friend? Follow the + [[share_with_a_friend_walkthrough]]. +* Want to archive data to a drive or the cloud? + Follow the [[archival_walkthrough]]. + +## colophon + +The git-annex assistant was [crowd funded on +Kickstarter](http://www.kickstarter.com/projects/joeyh/git-annex-assistant-like-dropbox-but-with-your-own/). +[[/Thanks]] to all my backers. + +I blog about my work on git-annex and the assistant on a daily basis +in [[this_blog|/devblog]]. Follow along! + +See also: The [[design|/design/assistant]] pages. diff --git a/doc/assistant/addsshserver.png b/doc/assistant/addsshserver.png new file mode 100644 index 0000000000..80f5d56173 Binary files /dev/null and b/doc/assistant/addsshserver.png differ diff --git a/doc/assistant/archival_walkthrough.mdwn b/doc/assistant/archival_walkthrough.mdwn new file mode 100644 index 0000000000..925e34944d --- /dev/null +++ b/doc/assistant/archival_walkthrough.mdwn @@ -0,0 +1,32 @@ +Normally, the git-annex assistant makes your files be available +wherever you use it, and so a copy of each file is stored in each repository. +That's perfect for files you're using right now, but what about files you're +not using any more? + +You could just delete those files, but it's better to archive them, so +you can access them later. All you need to get started archiving your old +files is a USB drive, or an [Amazon Glacier](http://aws.amazon.com/glacier/) +account. + +The webapp makes it easy to make a repository on either a USB drive, +or on Amazon Glacier. Once the repository is created, be sure to +put it in either the small archive, or full archive repository group. + +[[!img repogroups.png]] + +Now when you're done with a file, just move it into a directory named +"archive". The assistant will notice you put it there, and next time it +has the opportunity (when you plug in the USB drive, or when it can +talk to Amazon Glacier over the network), will move the file's +content to your archive repository. + +You'll no longer be able to open the file once it's been archived. +If you later want to access it, you can just copy or move it out +of the archive directory, and the assistant will retrieve its +content from the archive. + +Note that retrieving data from Amazon Glacier takes 4 to 5 hours. + +### screencast + +[[!inline feeds=no template=bare pages=videos/git-annex_assistant_archiving]] diff --git a/doc/assistant/brokenrepositoryalert.png b/doc/assistant/brokenrepositoryalert.png new file mode 100644 index 0000000000..ea001aec04 Binary files /dev/null and b/doc/assistant/brokenrepositoryalert.png differ diff --git a/doc/assistant/buddylist.png b/doc/assistant/buddylist.png new file mode 100644 index 0000000000..40b5a92382 Binary files /dev/null and b/doc/assistant/buddylist.png differ diff --git a/doc/assistant/cloudnudge.png b/doc/assistant/cloudnudge.png new file mode 100644 index 0000000000..b6f9a657e0 Binary files /dev/null and b/doc/assistant/cloudnudge.png differ diff --git a/doc/assistant/combinerepos.png b/doc/assistant/combinerepos.png new file mode 100644 index 0000000000..7beea71bc5 Binary files /dev/null and b/doc/assistant/combinerepos.png differ diff --git a/doc/assistant/comment_1_f2c4857b7b000e005f0c19279db14eaf._comment b/doc/assistant/comment_1_f2c4857b7b000e005f0c19279db14eaf._comment new file mode 100644 index 0000000000..1ed185e48a --- /dev/null +++ b/doc/assistant/comment_1_f2c4857b7b000e005f0c19279db14eaf._comment @@ -0,0 +1,8 @@ +[[!comment format=mdwn + username="https://www.google.com/accounts/o8/id?id=AItOawkYrMBMTCEFUKskhWGD-1pzcw2ITshsi_8" + nickname="Robert" + subject="Annex on OS X 10.6" + date="2013-06-04T23:10:03Z" + content=""" +I really hope they can get annex working on os x 10.6. This is a great effort. Thanks +"""]] diff --git a/doc/assistant/comment_2_befa1f48e5a43a7965060491430a6bc4._comment b/doc/assistant/comment_2_befa1f48e5a43a7965060491430a6bc4._comment new file mode 100644 index 0000000000..cbc955b07a --- /dev/null +++ b/doc/assistant/comment_2_befa1f48e5a43a7965060491430a6bc4._comment @@ -0,0 +1,9 @@ +[[!comment format=mdwn + username="https://www.google.com/accounts/o8/id?id=AItOawl9smfyJFgp3f2WjqqZWY6b7vo5eZv7GGQ" + nickname="Bryan" + subject="Ooh. Do want on Windows" + date="2013-07-03T20:44:05Z" + content=""" +Sadly, I didn't know about this when the Kickstarter was underway - I'd be happy to chip in $100 if it means I can get annex assistant on Windows earlier. + +"""]] diff --git a/doc/assistant/comment_3_9bd3b532a5c026a1d664c898f8e335e6._comment b/doc/assistant/comment_3_9bd3b532a5c026a1d664c898f8e335e6._comment new file mode 100644 index 0000000000..996c6e6067 --- /dev/null +++ b/doc/assistant/comment_3_9bd3b532a5c026a1d664c898f8e335e6._comment @@ -0,0 +1,8 @@ +[[!comment format=mdwn + username="https://www.google.com/accounts/o8/id?id=AItOawnov5q9_Cl4Ps5NoYE08yE01NLSvBANnY8" + nickname="Eric" + subject="Does it not work in Direct Mode?" + date="2014-04-26T09:41:20Z" + content=""" +Looks great, but I got the impression that Assistant would mirror the files from my computer to my external hard drive, and I can't seem to get it to do that. +"""]] diff --git a/doc/assistant/comment_4_c546a24459ca29025f00e424353c40d2._comment b/doc/assistant/comment_4_c546a24459ca29025f00e424353c40d2._comment new file mode 100644 index 0000000000..82be4c1938 --- /dev/null +++ b/doc/assistant/comment_4_c546a24459ca29025f00e424353c40d2._comment @@ -0,0 +1,8 @@ +[[!comment format=mdwn + username="https://www.google.com/accounts/o8/id?id=AItOawlr2Bj0Mzqwzl28cbrotcQUbOFoaPB3B_Y" + nickname="Tomasz" + subject="how to disable assistant" + date="2014-04-28T10:21:59Z" + content=""" +assistant daemon now starts at boot and pushes all annexed files to other repos... I'd like to do it on my own. I can stop daemon from webapp, but it starts again on boot - how Do I disable it from autostarting ? +"""]] diff --git a/doc/assistant/comment_5_d17de359cdd46659170d373cd09c0979._comment b/doc/assistant/comment_5_d17de359cdd46659170d373cd09c0979._comment new file mode 100644 index 0000000000..27d3430215 --- /dev/null +++ b/doc/assistant/comment_5_d17de359cdd46659170d373cd09c0979._comment @@ -0,0 +1,8 @@ +[[!comment format=mdwn + username="http://joeyh.name/" + ip="108.236.230.124" + subject="comment 5" + date="2014-05-16T18:55:22Z" + content=""" +If you have a question about using the assistant, please ask it in the [[forum]], not here. +"""]] diff --git a/doc/assistant/comment_6_70193bbaa5d60b829d7636748c641104._comment b/doc/assistant/comment_6_70193bbaa5d60b829d7636748c641104._comment new file mode 100644 index 0000000000..00db92dd4c --- /dev/null +++ b/doc/assistant/comment_6_70193bbaa5d60b829d7636748c641104._comment @@ -0,0 +1,8 @@ +[[!comment format=mdwn + username="https://www.google.com/accounts/o8/id?id=AItOawmN1fkq65FL5gtBB6qFmEKWiyl20OutvDI" + nickname="Niklaas" + subject="window manager on 1st machine" + date="2015-03-03T21:30:53Z" + content=""" +Not related to git-annex, but I was just wondering: What window manager are you using on the first machine? Looks like a tile windows manager but you are using XFCE?! +"""]] diff --git a/doc/assistant/comment_7_831a529c92951f70a27721210204e46b._comment b/doc/assistant/comment_7_831a529c92951f70a27721210204e46b._comment new file mode 100644 index 0000000000..deaf693bb9 --- /dev/null +++ b/doc/assistant/comment_7_831a529c92951f70a27721210204e46b._comment @@ -0,0 +1,7 @@ +[[!comment format=mdwn + username="https://woid.cryptobitch.de/foobar" + subject="@niklaas" + date="2015-08-17T13:42:26Z" + content=""" +Joey Hess is using xmonad. +"""]] diff --git a/doc/assistant/connection.png b/doc/assistant/connection.png new file mode 100644 index 0000000000..3cd6bef868 Binary files /dev/null and b/doc/assistant/connection.png differ diff --git a/doc/assistant/controlmenu.png b/doc/assistant/controlmenu.png new file mode 100644 index 0000000000..c1dac197b9 Binary files /dev/null and b/doc/assistant/controlmenu.png differ diff --git a/doc/assistant/crashrecovery.png b/doc/assistant/crashrecovery.png new file mode 100644 index 0000000000..fce3805d0f Binary files /dev/null and b/doc/assistant/crashrecovery.png differ diff --git a/doc/assistant/dashboard.png b/doc/assistant/dashboard.png new file mode 100644 index 0000000000..c71558d792 Binary files /dev/null and b/doc/assistant/dashboard.png differ diff --git a/doc/assistant/deleterepository.png b/doc/assistant/deleterepository.png new file mode 100644 index 0000000000..20db674caa Binary files /dev/null and b/doc/assistant/deleterepository.png differ diff --git a/doc/assistant/downloadnotification.png b/doc/assistant/downloadnotification.png new file mode 100644 index 0000000000..32b04f122d Binary files /dev/null and b/doc/assistant/downloadnotification.png differ diff --git a/doc/assistant/downloadupgrade.png b/doc/assistant/downloadupgrade.png new file mode 100644 index 0000000000..157eaddc7b Binary files /dev/null and b/doc/assistant/downloadupgrade.png differ diff --git a/doc/assistant/encryptdrive.png b/doc/assistant/encryptdrive.png new file mode 100644 index 0000000000..1cb041dda7 Binary files /dev/null and b/doc/assistant/encryptdrive.png differ diff --git a/doc/assistant/example.png b/doc/assistant/example.png new file mode 100644 index 0000000000..bfcb9afa66 Binary files /dev/null and b/doc/assistant/example.png differ diff --git a/doc/assistant/fsckconfig.png b/doc/assistant/fsckconfig.png new file mode 100644 index 0000000000..81ae755cfd Binary files /dev/null and b/doc/assistant/fsckconfig.png differ diff --git a/doc/assistant/genkey.png b/doc/assistant/genkey.png new file mode 100644 index 0000000000..6c1e509715 Binary files /dev/null and b/doc/assistant/genkey.png differ diff --git a/doc/assistant/iaitem.png b/doc/assistant/iaitem.png new file mode 100644 index 0000000000..2fcc1b4c8d Binary files /dev/null and b/doc/assistant/iaitem.png differ diff --git a/doc/assistant/inotify_max_limit_alert.png b/doc/assistant/inotify_max_limit_alert.png new file mode 100644 index 0000000000..d8d334152e Binary files /dev/null and b/doc/assistant/inotify_max_limit_alert.png differ diff --git a/doc/assistant/konquerormenu.png b/doc/assistant/konquerormenu.png new file mode 100644 index 0000000000..747bcdd0da Binary files /dev/null and b/doc/assistant/konquerormenu.png differ diff --git a/doc/assistant/local_pairing_walkthrough.mdwn b/doc/assistant/local_pairing_walkthrough.mdwn new file mode 100644 index 0000000000..c0a760ef57 --- /dev/null +++ b/doc/assistant/local_pairing_walkthrough.mdwn @@ -0,0 +1,90 @@ +So you have two computers in the same building, and you want them to share +the same synchronised folder, communicating directly with each other. + +This is incredibly easy to set up with the git annex assistant. + +Let's say the two computers are your computer and your friend's computer. +We'll start on your computer, where you open up your git annex dashboard. + +[[!img addrepository.png alt="Add another repository button"]] + +`*click*` + +[[!img pairing.png alt="Pair with another computer"]] + +`*click*` + +Now the hard bit. You have to think up a secret phrase, and type it in, +(and perhaps get the spelling correct). + +[[!img secret.png alt="Enter secret phrase"]] + +Now your computer is in pairing mode. When your friend looks at her git +annex dashboard, she sees something like this. + +[[!img pairrequest.png alt="Pair request"]] + +`*click*` + +[[!img secretempty.png alt="Enter same secret phrase"]] + +Now it's up to you to let her know what the secret is. As soon as she +enters it, both your computers will be paired, and will begin to sync their +git-annex folders. Just like that you can share files. + +---- + +## Requirements + +For local pairing to work, you must have sshd (ssh server daemon) installed and working on all machines involved. That means you must allow at least local connections to sshd. On most Linux distributions, sshd is packaged in either openssh (openSUSE) or openssh-server (Debian). + +It is highly recommended that you disable root login, disable password login to sshd and just enable key based authentication instead. No one will be able to login without your key. + +To disable root, after installing sshd, edit the sshd config (usually /etc/ssh/sshd_config file) and disable root login by adding: + + PermitRootLogin no + +Restart sshd. See man sshd_config for details. + +To disable password login and enable key based authentication, edit the sshd config (just like above) by uncommenting and changing the following options: + + ChallengeResponseAuthentication no + PasswordAuthentication no + UsePAM no + + PubkeyAuthentication yes + +Restart sshd. See man sshd_config for details. + +You can also restrict login to your local network only (not allow internet users from trying to log into your computer). Edit the hosts.deny file (usually /etc/hosts.deny) by adding the following: + + sshd : ALL EXCEPT LOCAL + +Do note that restricting login to your local network may or may not block git-annex. Also note that this will not work on Mac OSX because Apple decided to disable this feature and replace it with a crippled version made by Apple. + +## Tips + +Something to keep in mind, especially if pairing doesn't seem to be +working, is that the two computers need to be on the same network for this +pairing process to work. Sometimes a building will have more than one +network inside it, and you'll need to connect them both to the same one. +Make sure the wireless network name is the same, or that they're both +plugged into the same router. + +Also, the file sharing set up by this pairing only works when both +computers are on the same network. If you go on a trip, any files you +edit will not be visible to your friend until you get back. + +To get around this, you'll often also want to set up +[[tor_pairing|share_with_a_friend_walkthrough]] too, +which they can use to exchange files while away. + +And also, you can pair with as many other computers as you like, not just +one! + +## What does pairing actually do behind the scenes? + +It ensures that both repositories have correctly configured +[[remotes|walkthrough/adding_a_remote]] pointing to each other. +If you have already configured this manually, you do not need to +perform pairing. diff --git a/doc/assistant/local_pairing_walkthrough/addrepository.png b/doc/assistant/local_pairing_walkthrough/addrepository.png new file mode 100644 index 0000000000..b82efdbea2 Binary files /dev/null and b/doc/assistant/local_pairing_walkthrough/addrepository.png differ diff --git a/doc/assistant/local_pairing_walkthrough/comment_1_b33deed054d3aa8cfa6c9e3958643f16._comment b/doc/assistant/local_pairing_walkthrough/comment_1_b33deed054d3aa8cfa6c9e3958643f16._comment new file mode 100644 index 0000000000..52bdc7f4ea --- /dev/null +++ b/doc/assistant/local_pairing_walkthrough/comment_1_b33deed054d3aa8cfa6c9e3958643f16._comment @@ -0,0 +1,8 @@ +[[!comment format=mdwn + username="https://launchpad.net/~mattharrison" + ip="69.193.72.98" + subject="Pairing" + date="2012-10-29T17:56:06Z" + content=""" +Is there a way to do this pairing with one machine being a desktop machine and one a headless machine? +"""]] diff --git a/doc/assistant/local_pairing_walkthrough/comment_2_39f1162b4d43b61e957e7497df4b9e2b._comment b/doc/assistant/local_pairing_walkthrough/comment_2_39f1162b4d43b61e957e7497df4b9e2b._comment new file mode 100644 index 0000000000..32d56c907c --- /dev/null +++ b/doc/assistant/local_pairing_walkthrough/comment_2_39f1162b4d43b61e957e7497df4b9e2b._comment @@ -0,0 +1,8 @@ +[[!comment format=mdwn + username="http://joeyh.name/" + ip="4.152.108.194" + subject="easy peasy" + date="2012-10-29T19:09:43Z" + content=""" +A headless machine is probably a server. On the same page that has the \"local computer\" link, there's a \"remote server\" link that'll get you set up. +"""]] diff --git a/doc/assistant/local_pairing_walkthrough/comment_3_588869692b290483f58f3a7aa2bfb55f._comment b/doc/assistant/local_pairing_walkthrough/comment_3_588869692b290483f58f3a7aa2bfb55f._comment new file mode 100644 index 0000000000..6edaadb753 --- /dev/null +++ b/doc/assistant/local_pairing_walkthrough/comment_3_588869692b290483f58f3a7aa2bfb55f._comment @@ -0,0 +1,17 @@ +[[!comment format=mdwn + username="https://openid.fmarier.org/" + nickname="fmarier" + subject="Ports to open on the firewall?" + date="2013-05-04T03:56:55Z" + content=""" +I'm trying to get local pairing working between two local machines (same switch, both connected via cat5 cables) and they're not seeing each other, though I can happily ssh from one box to the other. + +In order to eliminate possible sources of problems, I tried these insecure settings: + +* add a `-A INPUT -j ACCEPT` to the top of my firewall rules +* use `password` as the shared secret + +but I still can't get past \"pairing in progress\" when pairing with a \"Local computer\". + +Is there any way to get information as to where the two machines are failing to see one another? +"""]] diff --git a/doc/assistant/local_pairing_walkthrough/comment_4_f6bf82c263fefe38701709d9dbd974cc._comment b/doc/assistant/local_pairing_walkthrough/comment_4_f6bf82c263fefe38701709d9dbd974cc._comment new file mode 100644 index 0000000000..6a9637481d --- /dev/null +++ b/doc/assistant/local_pairing_walkthrough/comment_4_f6bf82c263fefe38701709d9dbd974cc._comment @@ -0,0 +1,10 @@ +[[!comment format=mdwn + username="http://joeyh.name/" + nickname="joey" + subject="comment 4" + date="2013-05-04T16:39:23Z" + content=""" +Local pairing uses UDP port 55556. This is sent to multicast address 224.0.0.251 (same used by Avahi). + +(You also need TCP port 22 open for ssh.) +"""]] diff --git a/doc/assistant/local_pairing_walkthrough/comment_5_bada601ea4b7104f162a3e00def4be2b._comment b/doc/assistant/local_pairing_walkthrough/comment_5_bada601ea4b7104f162a3e00def4be2b._comment new file mode 100644 index 0000000000..989e80d8ab --- /dev/null +++ b/doc/assistant/local_pairing_walkthrough/comment_5_bada601ea4b7104f162a3e00def4be2b._comment @@ -0,0 +1,19 @@ +[[!comment format=mdwn + username="https://openid.fmarier.org/" + nickname="fmarier" + subject="Still stuck at "pairing in progress"" + date="2013-05-05T00:26:16Z" + content=""" +I hadn't thought of ssh. So here's what I've got now: + +* both laptop and desktop have `-A INPUT -d 224.0.0.251/32 -p udp --dport 55556 -j ACCEPT` +* `git annex webapp` is running as user `francois` on both machines +* laptop has `openssh-server` running on port 22 and its firewall allows desktop to connect +* desktop has `openssh-server` running on port 22 and its firewall allows the whole internal network to connect +* laptop's ssh pubkey is in francois' `~/.ssh/authorized_keys` on desktop and ssh'ing works +* deskop's ssh pubkey is **not** in francois' `~/.ssh/authorized_keys` on laptop (is that required?) +* (as mentioned before) both desktop and laptop are on the same switch and connected over CAT5 +* I'm using `password` as a pairing key just to rule out any typos there + +I don't see anything in the git-annex log on both of these machines. There's just a bunch of key generation stuff and then the last line is \"Pairing in progress\". +"""]] diff --git a/doc/assistant/local_pairing_walkthrough/comment_6_01ba0f9bfa0ed066c4b73d2d6028eecc._comment b/doc/assistant/local_pairing_walkthrough/comment_6_01ba0f9bfa0ed066c4b73d2d6028eecc._comment new file mode 100644 index 0000000000..1b2f66bd99 --- /dev/null +++ b/doc/assistant/local_pairing_walkthrough/comment_6_01ba0f9bfa0ed066c4b73d2d6028eecc._comment @@ -0,0 +1,8 @@ +[[!comment format=mdwn + username="http://joeyh.name/" + nickname="joey" + subject="comment 6" + date="2013-05-06T15:14:06Z" + content=""" +It's quite possible that your router doesn't handle multicast traffic to 224.0.0.251. The best way to check is to a) see if avahi/zeroconf works, since it's sending similar traffic and b) run tcpdump on both machines and see if the packets being sent on port 55556 by one are seen by the other. +"""]] diff --git a/doc/assistant/local_pairing_walkthrough/comment_7_17d44229e4fa46c50815672b96a9735a._comment b/doc/assistant/local_pairing_walkthrough/comment_7_17d44229e4fa46c50815672b96a9735a._comment new file mode 100644 index 0000000000..84a921eab2 --- /dev/null +++ b/doc/assistant/local_pairing_walkthrough/comment_7_17d44229e4fa46c50815672b96a9735a._comment @@ -0,0 +1,10 @@ +[[!comment format=mdwn + username="https://openid.fmarier.org/" + nickname="fmarier" + subject="Make sure your network is correctly set up in the first place!" + date="2013-05-12T08:37:38Z" + content=""" +It turns out that my internal network had badly broken IPv6 configs between the boxes. That was interfering with the multicast packets but not with anything else since all of my other internal traffic is over IPv4. + +Getting rid of these static IPv6 addresses has solved my problem. Local pairing is now working as advertised :) +"""]] diff --git a/doc/assistant/local_pairing_walkthrough/comment_8_b9d4c29cf2cca0427808df6af08fb789._comment b/doc/assistant/local_pairing_walkthrough/comment_8_b9d4c29cf2cca0427808df6af08fb789._comment new file mode 100644 index 0000000000..31a0f96742 --- /dev/null +++ b/doc/assistant/local_pairing_walkthrough/comment_8_b9d4c29cf2cca0427808df6af08fb789._comment @@ -0,0 +1,8 @@ +[[!comment format=mdwn + username="http://joeyh.name/" + nickname="joey" + subject="comment 8" + date="2013-05-13T17:22:17Z" + content=""" +I don't understand how IPv6 could affect local pairing. The assistant does not currently do pairing over ipv6 due to a limitation in the multicast library it's using. +"""]] diff --git a/doc/assistant/local_pairing_walkthrough/comment_9_bf313e490076a4676a632e3ec6129189._comment b/doc/assistant/local_pairing_walkthrough/comment_9_bf313e490076a4676a632e3ec6129189._comment new file mode 100644 index 0000000000..5ab4638b68 --- /dev/null +++ b/doc/assistant/local_pairing_walkthrough/comment_9_bf313e490076a4676a632e3ec6129189._comment @@ -0,0 +1,9 @@ +[[!comment format=mdwn + username="lhunath@3b4ff15f4600f3276d1776a490b734fca0f5c245" + nickname="lhunath" + avatar="http://cdn.libravatar.org/avatar/6388e539b56b3875cc9aceb9f404b3ad" + subject="Diagnosing local pairing issues." + date="2018-07-30T18:54:56Z" + content=""" +In my situation, a local pairing is initiated on machine A, machine B sees the request, prompts for the pass-phrase, both machines proceed with the actual sync process, but nothing actually happens. The last message in B's log is \"Pairing with [A] in progress\" and machine A says \"main: Pairing in progress\". How do I diagnose this issue? +"""]] diff --git a/doc/assistant/local_pairing_walkthrough/pairing.png b/doc/assistant/local_pairing_walkthrough/pairing.png new file mode 100644 index 0000000000..dfbbf0641a Binary files /dev/null and b/doc/assistant/local_pairing_walkthrough/pairing.png differ diff --git a/doc/assistant/local_pairing_walkthrough/pairrequest.png b/doc/assistant/local_pairing_walkthrough/pairrequest.png new file mode 100644 index 0000000000..8d3f603bf4 Binary files /dev/null and b/doc/assistant/local_pairing_walkthrough/pairrequest.png differ diff --git a/doc/assistant/local_pairing_walkthrough/secret.png b/doc/assistant/local_pairing_walkthrough/secret.png new file mode 100644 index 0000000000..1eb8051222 Binary files /dev/null and b/doc/assistant/local_pairing_walkthrough/secret.png differ diff --git a/doc/assistant/local_pairing_walkthrough/secretempty.png b/doc/assistant/local_pairing_walkthrough/secretempty.png new file mode 100644 index 0000000000..4918787845 Binary files /dev/null and b/doc/assistant/local_pairing_walkthrough/secretempty.png differ diff --git a/doc/assistant/logs.png b/doc/assistant/logs.png new file mode 100644 index 0000000000..8e9e5b1ec8 Binary files /dev/null and b/doc/assistant/logs.png differ diff --git a/doc/assistant/makerepo.png b/doc/assistant/makerepo.png new file mode 100644 index 0000000000..e8f1b26216 Binary files /dev/null and b/doc/assistant/makerepo.png differ diff --git a/doc/assistant/menu.png b/doc/assistant/menu.png new file mode 100644 index 0000000000..29a7c9d0f8 Binary files /dev/null and b/doc/assistant/menu.png differ diff --git a/doc/assistant/nautilusmenu.png b/doc/assistant/nautilusmenu.png new file mode 100644 index 0000000000..d7926e34f2 Binary files /dev/null and b/doc/assistant/nautilusmenu.png differ diff --git a/doc/assistant/osx-app.png b/doc/assistant/osx-app.png new file mode 100644 index 0000000000..386fb832db Binary files /dev/null and b/doc/assistant/osx-app.png differ diff --git a/doc/assistant/preferences.png b/doc/assistant/preferences.png new file mode 100644 index 0000000000..74fb814183 Binary files /dev/null and b/doc/assistant/preferences.png differ diff --git a/doc/assistant/quickstart.mdwn b/doc/assistant/quickstart.mdwn new file mode 100644 index 0000000000..0472ba48b6 --- /dev/null +++ b/doc/assistant/quickstart.mdwn @@ -0,0 +1,30 @@ +## first run + +To get started with the git-annex assistant, just pick it from +your system's list of applications. + +[[!img assistant/menu.png]] +[[!img assistant/osx-app.png]] + +It'll prompt you to set up a folder: + +[[!img assistant/makerepo.png]] + +Then any changes you make to its folder will automatically be committed to +git, and synced to repositories on other computers. You can use the +interface to add repositories and control the git-annex assistant. + +[[!img assistant/running.png]] + +## starting on boot + +The git-annex assistant will automatically be started when you log in to +desktop environments like Mac OS X, Gnome, XFCE, and KDE, and the menu item +shown above can be used to open the webapp. On other systems, you may need +to start it by hand. + +To start the webapp, run `git annex webapp` at the command line. + +To start the assistant without opening the webapp, +you can run the command "git annex assistant --autostart". This is a +good thing to configure your system to run automatically when you log in. diff --git a/doc/assistant/release_notes.mdwn b/doc/assistant/release_notes.mdwn new file mode 100644 index 0000000000..6c7c432de4 --- /dev/null +++ b/doc/assistant/release_notes.mdwn @@ -0,0 +1,422 @@ +## version 6.20170101 + +XMPP support has been removed from the assistant in this release. + +If your repositories used XMPP to keep in sync, that will no longer +work, and you should enable some other remote to keep them in sync. +A ssh server is one way, or use the new Tor pairing feature. + +## version 5.20140421 + +This release begins to deprecate XMPP support. In particular, if you use +the assistant with a ssh remote that has this version of git-annex +installed, you don't need XMPP any longer to get immediate syncing of +changes. + +## version 5.20140411 + +This release fixes a bug that could cause the assistant to use a *lot* of +CPU, when monthly fscking was set up. + +Automatic upgrading was broken on OSX for previous versions. This has been +fixed, but you'll need to manually upgrade to this version to get it going +again. Workaround: Remove the wget bundled inside the git-annex dmg. + +## version 5.20140221 + +The Windows port of the assistant and webapp is now considered to be beta +quality. There are important missing features (notably Jabber), documented +on [[todo/windows_support]], but the webapp is broadly usable on Windows +now. + +## version 5.20131221 + +There is now a arm [[install/linux_standalone]] build of git-annex, +including the assistant and webapp, +which can be installed on a variety of systems including Raspberry Pi, +Synology NAS, and Google Chromebooks. Details in +[[this forum thread|forum/new_linux_arm_tarball_build]]. + +## version 5.20131213 + +The assistant can now be used on Windows! However, it has known problems, +described in [[todo/windows_support]], and should be considered an +alpha-level preview. + +## version 5.20131127 + +Starting with this version, when git-annex is installed from a build on +this website, it will detect when new versions are available, and allow +easily upgrading. Automatic upgrades can also be configured if desired, +or automatic upgrade checking can be disabled in the preferences page. + +git-annex builds from distributions, like Debian will not automatically +upgrade; use the distribution's package manager for that. However, the +git-annex webapp will also detect when a distribution has upgraded +git-annex and offer to restart the assistant. + +## version 4.20131024 + +This version fixes several different bugs that could cause the webapp to +refuse to create a repository. Several other bugs are also fixed, including +a bug that caused it to not add files on Android. + +New in this release is the ability to use the webapp to set up scheduled +consistency checks of your repositories. Many problems with repositories +are now automatically corrected, and it can even repair damaged git +repositories. + +This is a recommended upgrade. + +## version 4.20131002 + +Now you can use the webapp to set up an encrypted git repository on a +remote ssh server, or on rsync.net, and use it as a live cloud backup. Or, +use the webapp to make an encrypted git repository on a removable drive, +and store it offsite as a secure backup. + +## version 4.20130920 + +This release is the first to support fully encrypted git repositories +stored on removable drives. This can be set up easily using the webapp. + +## version 4.20130909 + +This release fixes a crash that could occur when using XMPP with the +assitant. It has only been seen on OS X so far. The bug is not believed to +be explitable, but upgrading is still recommended. + +## version 4.20130802 + +This release fixes several bugs, including a reversion introduced in the last +version that broke direct mode on Windows, Android, and other crippled +filesystems. It contains a workaround for a bug in recent git pre-releases +that broke handling of filenames containing spaces. +It is a highly recommended upgrade. + +The webapp can now detect repositories that did not finish getting properly set +up, and can recover from one common bug that broke local pairing and remote +ssh server setups on systems using `ssh-agent`. + +## version 4.20130723 + +This release fixes some bugs. Notably it fixes a bug that could result in data +loss when adding a tarball of a git-annex repository to your git-annex +repository. + +Rsync.net have committed to support git-annex and offer a special +discounted rate for git-annex users. + + +## version 4.20130709 + +This release is mostly bug fixes. + +One of the bugs involved setting up rsync remotes on servers other than +rsync.net. The wrong `.ssh/authorized_keys` line was deployed to the +remote server. If you set up a rsync remote with a past release, and it does +not work, you will need to manually edit the `.ssh/authorized_keys` file, +and remove the `command=` forced command. + +## version 4.20130621, 4.20130627 + +These releases mostly consist of bug fixes. + +## version 4.20130601 + +This is a bugfix release, featuring significant XMPP improvements and +more robustness thanks to automated fuzz testing. Recommended upgrade. + +This version changes its XMPP protocol, so it will fail to sync with older +git-annex versions over XMPP. + +## version 4.20130521 + +This is a bugfix release. Recommended upgrade. + +## version 4.20130516 + +This version contains numerous bug fixes, and improvements. + +This is the first release with a fully usable Android app. No command-line +typing needed to set up syncing to your Android phone or tablet! +A few of the more advanced features may not work (or not work reliably) +on Android. The Android app is still beta quality. + +This is also the first release with a Windows port! The Windows port +is in an alpha quality state, and is missing many features. +It does not yet include the assistant. + +## version 4.20130501 + +This version contains numerous bug fixes, and improvements. + +## version 4.20130417 + +This version contains numerous bug fixes, and improvements. + +One bug that was fixed can affect users of gnome-keyring who +have set up remote repositories on ssh servers using the webapp. +The gnome-keyring may load the restricted key that is set up +for that, and make it be used for regular logins to the server; +with the result that you'll get an error message about "git-annex-shell" +when sshing to the server. + +If you experience this problem you can fix it by +moving `.ssh/key.git-annex*` to `.ssh/git-annex/` (creating +that directory first), and edit `.ssh/config` to reflect the new +location of the key. You will also need to restart gnome-keyring. + +Another change relates to files in `archive/` directories. Client repositories +now sync these files between themselves like any other files, until +the files reach an archive repository. Only then are they removed from +the client repositories. So you need to ensure you have at least one +archive repository if you want to use the `archive/` directory feature. + +## version 4.20130323, 4.20130405 + +These versions continue fixing bugs and adding features. + +## version 4.20130314 + +This version makes a great many improvements and bugfixes, and is +a recommended upgrade. + +If you have already used the webapp to locally pair two computers, +a bug caused the paired repository to not be given an appropriate cost. +To fix this, go into the Repositories page in the webapp, and drag the +repository for the locally paired computer to come before any repositories +that it's more expensive to transfer data to. + +## version 4.20130227 + +This release fixes a bug with globbing that broke preferred content expressions. +So, it is a recommended upgrade from the previous release, which introduced +that bug. + +In this release, the assistant is fully working on Android, although +it must be set up using the command line. + +Repositories can now be placed on filesystems that lack support for symbolic +links; FAT support is complete. + +## version 3.20130216 + +This adds a port to Android. Only usable at the command line so far; +beta qualitty. + +Also a bugfix release, and improves support for FAT. + +The following are known limitations of this release of the git-annex +assistant: + +* No Android app yet. +* On BSD operating systems (but not on OS X), the assistant uses kqueue to + watch files. Kqueue has to open every directory it watches, so too many + directories will run it out of the max number of open files (typically + 1024), and fail. See [[this_bug|bugs/Issue_on_OSX_with_some_system_limits]] + for a workaround. +* Also on systems with kqueue, modifications to existing files in direct + mode will not be noticed. + +## version 3.20130107, 3.20130114, 3.20130124, 3.20130207 + +These are bugfix releases. + +## version 3.20130102 + +This release makes several significant improvements to the git-annex +assistant, which is still in beta. + +The main improvement is direct mode. This allows you to directly edit files +in the repository, and the assistant will automatically commit and sync +your changes. Direct mode is the default for new repositories created +by the assistant. To convert your existing repository to use direct mode, +manually run `git annex direct` inside the repository. + +## version 3.20121211 + +This release of the git-annex assistant (which is still in beta) +consists of mostly bugfixes, user interface improvements, and improvements +to existing features. + +In general, anything you can configure with the assistant's web app +will work. Some examples of use cases supported by this release include: + +* Using Box.com's 5 gigabytes of free storage space as a cloud transfer + point between between repositories that cannot directly contact + one-another. (Many other cloud providers are also supported, from Rsync.net + to Amazon S3, to your own ssh server.) +* Archiving or backing up files to Amazon Glacier. See [[archival_walkthrough]]. +* [[Sharing repositories with friends|share_with_a_friend_walkthrough]] + contacted through a Jabber server (such as Google Talk). +* [[Pairing|local_pairing_walkthrough]] two computers that are on the same local + network (or VPN) and automatically keeping the files in the annex in + sync as changes are made to them. +* Cloning your repository to removable drives, USB keys, etc. The assistant + will notice when the drive is mounted and keep it in sync. + Such a drive can be stored as an offline backup, or transported between + computers to keep them in sync. + +The following are known limitations of this release of the git-annex +assistant: + +* The Max OSX standalone app may not work on all versions of Max OSX. + Please test! +* On Mac OSX and BSD operating systems, the assistant uses kqueue to watch + files. Kqueue has to open every directory it watches, so too many + directories will run it out of the max number of open files (typically + 1024), and fail. See [[bugs/Issue_on_OSX_with_some_system_limits]] + for a workaround. + +## version 3.20121126 + +This adds several features to the git-annex assistant, which is still in beta. + +In general, anything you can configure with the assistant's web app +will work. Some examples of use cases supported by this release include: + +* Using Box.com's 5 gigabytes of free storage space as a cloud transfer + point between between repositories that cannot directly contact + one-another. (Many other cloud providers are also supported, from Rsync.net + to Amazon S3, to your own ssh server.) +* Archiving or backing up files to Amazon Glacier. +* [[Sharing repositories with friends|share_with_a_friend_walkthrough]] + contacted through a Jabber server (such as Google Talk). +* [[Pairing|local_pairing_walkthrough]] two computers that are on the same local + network (or VPN) and automatically keeping the files in the annex in + sync as changes are made to them. +* Cloning your repository to removable drives, USB keys, etc. The assistant + will notice when the drive is mounted and keep it in sync. + Such a drive can be stored as an offline backup, or transported between + computers to keep them in sync. + +The following are known limitations of this release of the git-annex +assistant: + +* The Max OSX standalone app does not work on all versions of Max OSX. +* On Mac OSX and BSD operating systems, the assistant uses kqueue to watch + files. Kqueue has to open every directory it watches, so too many + directories will run it out of the max number of open files (typically + 1024), and fail. See [[bugs/Issue_on_OSX_with_some_system_limits]] + for a workaround. +* Retrieval of files from Amazon Glacier is not fully automated; the + assistant does not automatically retry in the 4 to 5 hours period + when Glacier makes the files available. + +## version 3.20121112 + +This is a major upgrade of the git-annex assistant, which is still in beta. + +In general, anything you can configure with the assistant's web app +will work. Some examples of use cases supported by this release include: + +* [[Sharing repositories with friends|share_with_a_friend_walkthrough]] + contacted through a Jabber server (such as Google Talk). +* Setting up cloud repositories, that are used as backups, archives, + or transfer points between repositories that cannot directly contact + one-another. +* [[Pairing|local_pairing_walkthrough]] two computers that are on the same local + network (or VPN) and automatically keeping the files in the annex in + sync as changes are made to them. +* Cloning your repository to removable drives, USB keys, etc. The assistant + will notice when the drive is mounted and keep it in sync. + Such a drive can be stored as an offline backup, or transported between + computers to keep them in sync. + +The following upgrade notes apply if you're upgrading from a previous version: + +* For best results, edit the configuration of repositories you set + up with older versions, and place them in a repository group. + This lets the assistant know how you want to use the repository; for backup, + archival, as a transfer point for clients, etc. Go to Configuration -> + Manage Repositories, and click in the "configure" link to edit a repository's + configuration. +* If you set up a cloud repository with an older version, and have multiple + clients using it, you are recommended to configure an Jabber account, + so that clients can use it to communicate when sending data to the + cloud repository. Configure Jabber by opening the webapp, and going to + Configuration -> Configure jabber account +* When setting up local pairing, the assistant did not limit the paired + computer to accessing a single git repository. This new version does, + by setting GIT_ANNEX_SHELL_DIRECTORY in `~/.ssh/authorized_keys`. + +The following are known limitations of this release of the git-annex +assistant: + +* On Mac OSX and BSD operating systems, the assistant uses kqueue to watch + files. Kqueue has to open every directory it watches, so too many + directories will run it out of the max number of open files (typically + 1024), and fail. See [[bugs/Issue_on_OSX_with_some_system_limits]] + for a workaround. + +## version 3.20121009 + +This is a maintenance release of the git-annex assistant, which is still in +beta. + +In general, anything you can configure with the assistant's web app +will work. Some examples of use cases supported by this release include: + +* [[Pairing|local_pairing_walkthrough]] two computers that are on the same local + network (or VPN) and automatically keeping the files in the annex in + sync as changes are made to them. +* Cloning your repository to removable drives, USB keys, etc. The assistant + will notice when the drive is mounted and keep it in sync. + Such a drive can be stored as an offline backup, or transported between + computers to keep them in sync. +* Cloning your repository to a remote server, running ssh, and uploading + changes made to your files to the server. There is special support + for using the rsync.net cloud provider this way, or any shell account + on a typical unix server, such as a Linode VPS can be used. + +The following are known limitations of this release of the git-annex +assistant: + +* On Mac OSX and BSD operating systems, the assistant uses kqueue to watch + files. Kqueue has to open every directory it watches, so too many + directories will run it out of the max number of open files (typically + 1024), and fail. See [[bugs/Issue_on_OSX_with_some_system_limits]] + for a workaround. +* In order to ensure that all multiple repositories are kept in sync, + each computer with a repository must be running the git-annex assistant. +* The assistant does not yet always manage to keep repositories in sync + when some are hidden from others behind firewalls. + +## version 3.20120924 + +This is the first beta release of the git-annex assistant. + +In general, anything you can configure with the assistant's web app +will work. Some examples of use cases supported by this release include: + +* [[Pairing|local_pairing_walkthrough]] two computers that are on the same local + network (or VPN) and automatically keeping the files in the annex in + sync as changes are made to them. +* Cloning your repository to removable drives, USB keys, etc. The assistant + will notice when the drive is mounted and keep it in sync. + Such a drive can be stored as an offline backup, or transported between + computers to keep them in sync. +* Cloning your repository to a remote server, running ssh, and uploading + changes made to your files to the server. There is special support + for using the rsync.net cloud provider this way, or any shell account + on a typical unix server, such as a Linode VPS can be used. + +The following are known limitations of this release of the git-annex +assistant: + +* On Mac OSX and BSD operating systems, the assistant uses kqueue to watch + files. Kqueue has to open every directory it watches, so too many + directories will run it out of the max number of open files (typically + 1024), and fail. See [[bugs/Issue_on_OSX_with_some_system_limits]] + for a workaround. +* In order to ensure that all multiple repositories are kept in sync, + each computer with a repository must be running the git-annex assistant. +* The assistant does not yet always manage to keep repositories in sync + when some are hidden from others behind firewalls. +* If a file is checked into git as a normal file and gets modified + (or merged, etc), it will be converted into an annexed file. So you + should not mix use of the assistant with normal git files in the same + repository yet. +* If you `git annex unlock` a file, it will immediately be re-locked. + See [[bugs/watcher_commits_unlocked_files]]. diff --git a/doc/assistant/release_notes/comment_1_bd8f376c9d0c1d5ed07fb013907a60ee._comment b/doc/assistant/release_notes/comment_1_bd8f376c9d0c1d5ed07fb013907a60ee._comment new file mode 100644 index 0000000000..04cdf4039d --- /dev/null +++ b/doc/assistant/release_notes/comment_1_bd8f376c9d0c1d5ed07fb013907a60ee._comment @@ -0,0 +1,14 @@ +[[!comment format=mdwn + username="http://wiggy.net/" + nickname="Wichert" + subject="OSX 10.8's gatekeeper does not like git-annex" + date="2012-11-13T14:47:52Z" + content=""" +Trying to run this release on OSX results in an error message from Gatekeeper: + +> \"git-annex\" can't be opened because it is from an unidentified developer. +> +> Your security preferences allow installation of only apps from the Mac App Store and identified developers. + +It would be nice if the binary could be signed to make Gatekeeper happy. Until then a note in the installation instructions might be useful. +"""]] diff --git a/doc/assistant/release_notes/comment_2_75e0774ad042717fbd059a8a9ec2db1e._comment b/doc/assistant/release_notes/comment_2_75e0774ad042717fbd059a8a9ec2db1e._comment new file mode 100644 index 0000000000..9033b1af6d --- /dev/null +++ b/doc/assistant/release_notes/comment_2_75e0774ad042717fbd059a8a9ec2db1e._comment @@ -0,0 +1,12 @@ +[[!comment format=mdwn + username="http://wiggy.net/" + nickname="Wichert" + subject="git-annex not runnable on OSX 10.8" + date="2012-11-13T14:49:35Z" + content=""" +After telling Gatekeeper that I really want to run git-annex it still fails: + +[fog;~]-131> open Applications/git-annex.app +LSOpenURLsWithRole() failed with error -10810 for the file /Users/wichert/Applications/git-annex.app. + +"""]] diff --git a/doc/assistant/release_notes/comment_3_b3bfd8e547e20c51f7c32c6c9424e936._comment b/doc/assistant/release_notes/comment_3_b3bfd8e547e20c51f7c32c6c9424e936._comment new file mode 100644 index 0000000000..73377c714f --- /dev/null +++ b/doc/assistant/release_notes/comment_3_b3bfd8e547e20c51f7c32c6c9424e936._comment @@ -0,0 +1,10 @@ +[[!comment format=mdwn + username="http://joeyh.name/" + ip="4.252.11.120" + subject="comment 3" + date="2012-11-13T17:11:51Z" + content=""" +This has been previously reported: [[bugs/OSX_git-annex.app_error:__LSOpenURLsWithRole()]] + +No clue what that error is supposed to mean. +"""]] diff --git a/doc/assistant/release_notes/comment_4_c6caa2b521b456bb4ce594d64919cffe._comment b/doc/assistant/release_notes/comment_4_c6caa2b521b456bb4ce594d64919cffe._comment new file mode 100644 index 0000000000..1ebaf504f8 --- /dev/null +++ b/doc/assistant/release_notes/comment_4_c6caa2b521b456bb4ce594d64919cffe._comment @@ -0,0 +1,8 @@ +[[!comment format=mdwn + username="https://www.google.com/accounts/o8/id?id=AItOawkSq2FDpK2n66QRUxtqqdbyDuwgbQmUWus" + nickname="Jimmy" + subject="comment 4" + date="2012-11-16T13:02:40Z" + content=""" +sadly i only have a 10.7 machine to create the builds, so I have no experience with 10.8. I haven't had a 10.6 machine in a while to create the builds. Anyone else want to work together in setting up another 10.6 or 10.8 builder for others? +"""]] diff --git a/doc/assistant/remote_sharing_walkthrough.mdwn b/doc/assistant/remote_sharing_walkthrough.mdwn new file mode 100644 index 0000000000..ec8f39d531 --- /dev/null +++ b/doc/assistant/remote_sharing_walkthrough.mdwn @@ -0,0 +1,12 @@ +So you have two computers that are not in the same place, and you want them +to share the same synchronised folder, communicating directly with each other. + +[[!inline feeds=no template=bare pages=videos/git-annex_assistant_remote_sharing]] + +You can add even more computers using the same method shown here. + +---- + +If you have a laptop that is sometimes near another computer, you can +speed up file transfers when it is by also connecting it using the +[[local_pairing_walkthrough]]. diff --git a/doc/assistant/repairrepository.png b/doc/assistant/repairrepository.png new file mode 100644 index 0000000000..d49ea9d955 Binary files /dev/null and b/doc/assistant/repairrepository.png differ diff --git a/doc/assistant/repogroups.png b/doc/assistant/repogroups.png new file mode 100644 index 0000000000..06300ce66a Binary files /dev/null and b/doc/assistant/repogroups.png differ diff --git a/doc/assistant/repoinfo.png b/doc/assistant/repoinfo.png new file mode 100644 index 0000000000..d43ffc9f99 Binary files /dev/null and b/doc/assistant/repoinfo.png differ diff --git a/doc/assistant/repositories.png b/doc/assistant/repositories.png new file mode 100644 index 0000000000..2853683dca Binary files /dev/null and b/doc/assistant/repositories.png differ diff --git a/doc/assistant/rsync.net.encryption.png b/doc/assistant/rsync.net.encryption.png new file mode 100644 index 0000000000..ec751d10d2 Binary files /dev/null and b/doc/assistant/rsync.net.encryption.png differ diff --git a/doc/assistant/rsync.net.png b/doc/assistant/rsync.net.png new file mode 100644 index 0000000000..0e09407035 Binary files /dev/null and b/doc/assistant/rsync.net.png differ diff --git a/doc/assistant/running.png b/doc/assistant/running.png new file mode 100644 index 0000000000..8c3b0eaf24 Binary files /dev/null and b/doc/assistant/running.png differ diff --git a/doc/assistant/share_with_a_friend_walkthrough.mdwn b/doc/assistant/share_with_a_friend_walkthrough.mdwn new file mode 100644 index 0000000000..a5a6d68170 --- /dev/null +++ b/doc/assistant/share_with_a_friend_walkthrough.mdwn @@ -0,0 +1,40 @@ +Want to share all the files in your repository securely with a friend? + +This connects to your friend's repository using +[Tor](https://torproject.org/). Both you and your friend will need to +install [Tor](https://torproject.org/) and +[Magic Wormhole](https://github.com/warner/magic-wormhole), and then both +follow these steps to connect your repositories. + +Important: You and your friend need git-annex version 6.20180705. Older +versions of git-annex unfortunately had a bug that prevents this process +from working correctly. + +Start by opening up your git annex webapp. + +[[!img local_pairing_walkthrough/addrepository.png alt="Add another repository button"]] + +`*click*` + +[[!img pairing.png alt="Share with a friend"]] + +`*click*` + +[[!img enabletor.png alt="Enabling tor hidden service"]] + +You will probably be prompted to enter a password, to configure Tor. +(Depending on how your system is configured, this may be the root password, +or your user account's password.) + +[[!img wormholepairing.png alt="Pairing with a friend form"]] + +A pairing code will be generated. Tell it to your friend. Ask them +for their pairing code, and enter it in the form. + +Once you've exchanged pairing codes, your repositories will be connected +over Tor. They will begin to sync files back and forth, which can take a +while since Tor is not super-fast. + +--- + +See [[tips/peer_to_peer_network_with_tor]] for more details. diff --git a/doc/assistant/share_with_a_friend_walkthrough/enabletor.png b/doc/assistant/share_with_a_friend_walkthrough/enabletor.png new file mode 100644 index 0000000000..6dd2856831 Binary files /dev/null and b/doc/assistant/share_with_a_friend_walkthrough/enabletor.png differ diff --git a/doc/assistant/share_with_a_friend_walkthrough/pairing.png b/doc/assistant/share_with_a_friend_walkthrough/pairing.png new file mode 100644 index 0000000000..533f4aef5e Binary files /dev/null and b/doc/assistant/share_with_a_friend_walkthrough/pairing.png differ diff --git a/doc/assistant/share_with_a_friend_walkthrough/wormholepairing.png b/doc/assistant/share_with_a_friend_walkthrough/wormholepairing.png new file mode 100644 index 0000000000..a2817da831 Binary files /dev/null and b/doc/assistant/share_with_a_friend_walkthrough/wormholepairing.png differ diff --git a/doc/assistant/thanks.mdwn b/doc/assistant/thanks.mdwn new file mode 100644 index 0000000000..2b9c8279d0 --- /dev/null +++ b/doc/assistant/thanks.mdwn @@ -0,0 +1 @@ +[[!meta redir="/thanks#kickstarter"]] diff --git a/doc/assistant/thumbnail.png b/doc/assistant/thumbnail.png new file mode 100644 index 0000000000..346c22f025 Binary files /dev/null and b/doc/assistant/thumbnail.png differ diff --git a/doc/assistant/unused.png b/doc/assistant/unused.png new file mode 100644 index 0000000000..b0ace763cf Binary files /dev/null and b/doc/assistant/unused.png differ diff --git a/doc/assistant/upgradecomplete.png b/doc/assistant/upgradecomplete.png new file mode 100644 index 0000000000..3f8c7f3c2b Binary files /dev/null and b/doc/assistant/upgradecomplete.png differ diff --git a/doc/assistant/xmpp.png b/doc/assistant/xmpp.png new file mode 100644 index 0000000000..c3cc53ebf5 Binary files /dev/null and b/doc/assistant/xmpp.png differ diff --git a/doc/assistant/xmppnudge.png b/doc/assistant/xmppnudge.png new file mode 100644 index 0000000000..b3a0658cbf Binary files /dev/null and b/doc/assistant/xmppnudge.png differ diff --git a/doc/assistant/xmpppairingend.png b/doc/assistant/xmpppairingend.png new file mode 100644 index 0000000000..f0c9e765d1 Binary files /dev/null and b/doc/assistant/xmpppairingend.png differ diff --git a/doc/automatic_conflict_resolution.mdwn b/doc/automatic_conflict_resolution.mdwn new file mode 100644 index 0000000000..0d4f55a01a --- /dev/null +++ b/doc/automatic_conflict_resolution.mdwn @@ -0,0 +1,25 @@ +Running `git annex sync` or using the [[assistant]] involves merging +changes from elsewhere into your repository's currently checked out branch. +This could lead to a merge conflict, perhaps because the same file +got changed in two different ways. A nice feature is that these +merge conflicts are automatically resolved, rather than leaving +git in the middle of a conflicted merge, which would prevent further +syncing from happening. + +When a conflict occurs, there will be several messages printed about the merge +conflict, and the file that has the merge conflict will be renamed, with +".variant-XXX" tacked onto it. So if there are two versions of file foo, +you might end up with "foo.variant-AAA" and "foo.variant-BBB". It's then +up to you to decide what to do with these two files. Perhaps you can +manually combine them back into a single file. Or perhaps you choose to +rename them to better names and keep two versions, or delete one version +you don't want. + +The "AAA" and "BBB" in the above example are essentially arbitrary +(technically they are the MD5 checksum of the key). The automatic merge +conflict resolution is designed so that if two or more repositories both get +a merge conflict, and resolve it, the resolved repositories will not +themselves conflict. This is why it doesn't use something nicer, like +perhaps the name of the remote that the file came from. + +See also: [[git-annex-resolvemerge]] diff --git a/doc/automatic_conflict_resolution/comment_1_307898855f91a2a189d4fa5eae62cce1._comment b/doc/automatic_conflict_resolution/comment_1_307898855f91a2a189d4fa5eae62cce1._comment new file mode 100644 index 0000000000..57a8c45f84 --- /dev/null +++ b/doc/automatic_conflict_resolution/comment_1_307898855f91a2a189d4fa5eae62cce1._comment @@ -0,0 +1,10 @@ +[[!comment format=mdwn + username="https://www.google.com/accounts/o8/id?id=AItOawnJTqmRu1YCKS2Hsm4vtOflLhP4fU-k98w" + nickname="Ahmed" + subject="Customise conflict resolution behaviour" + date="2014-02-25T11:42:08Z" + content=""" +How to customise git-annex conflict resolution behaviour, such that for example: change naming convention of conflicted files with suffix or prefix, move conflicted files to another directory structure, overwrite conflicted files from preferred content ... + + +"""]] diff --git a/doc/automatic_conflict_resolution/comment_2_0a8ea42764dde1a33d2112197b961c51._comment b/doc/automatic_conflict_resolution/comment_2_0a8ea42764dde1a33d2112197b961c51._comment new file mode 100644 index 0000000000..8c50ec4e22 --- /dev/null +++ b/doc/automatic_conflict_resolution/comment_2_0a8ea42764dde1a33d2112197b961c51._comment @@ -0,0 +1,8 @@ +[[!comment format=mdwn + username="https://www.google.com/accounts/o8/id?id=AItOawn3p4i4lk_zMilvjnJ9sS6g2nerpgz0Fjc" + nickname="Matthias" + subject="Use automatic merge without syncing" + date="2014-03-20T10:03:41Z" + content=""" +Is there a possibility to use the automatic merge logic without using \"git annex sync\"? I don't want to have the \"synced\"-branches, but the auto-conflict-resolution is very nice. +"""]] diff --git a/doc/automatic_conflict_resolution/comment_3_5c587c6633cae1c8547ca970d55ee97e._comment b/doc/automatic_conflict_resolution/comment_3_5c587c6633cae1c8547ca970d55ee97e._comment new file mode 100644 index 0000000000..58bdc56310 --- /dev/null +++ b/doc/automatic_conflict_resolution/comment_3_5c587c6633cae1c8547ca970d55ee97e._comment @@ -0,0 +1,8 @@ +[[!comment format=mdwn + username="http://joeyh.name/" + ip="209.250.56.102" + subject="comment 3" + date="2014-03-20T16:10:10Z" + content=""" +@Matthias `git annex merge` will do what you want, as long as you have git-annex 4.20130709 or newer. +"""]] diff --git a/doc/automatic_conflict_resolution/comment_4_80539e11e36a0b64cee83b6b373bd843._comment b/doc/automatic_conflict_resolution/comment_4_80539e11e36a0b64cee83b6b373bd843._comment new file mode 100644 index 0000000000..d67514452f --- /dev/null +++ b/doc/automatic_conflict_resolution/comment_4_80539e11e36a0b64cee83b6b373bd843._comment @@ -0,0 +1,17 @@ +[[!comment format=mdwn + username="https://www.google.com/accounts/o8/id?id=AItOawn3p4i4lk_zMilvjnJ9sS6g2nerpgz0Fjc" + nickname="Matthias" + subject="merge for master branch?" + date="2014-03-23T23:02:23Z" + content=""" +As far as I observed, \"git annex merge\" only merges the \"git-annex\" branch. My wish is to have the conflict resolution from \"git annex sync\" in the \"master\" branch, but no automatic commit, such that the user can verify and possibly correct the merge. The proposed merge could go to the index. Consider the following scenario: + +1. We have repo A, B, and CENTRAL +2. All three start with a root commit in \"master\" branch +3. Then A commits a file \"test.txt\" with content \"a\" and syncs with CENTRAL +4. Meanwhile, B commits \"test.txt\" with content \"b\" +5. When B tries to sync with CENTRAL, the proposed conflict resolution having two files \"test.txt-variantXXXX\" and \"test.txt-variantYYYY\" should be staged in the index, but not committed yet. +6. B can now commit a custom merge, e.g. with file content \"ab\". + +The point is that I really like the conflict resolution, but still want to force the user to check the result. +"""]] diff --git a/doc/automatic_conflict_resolution/comment_5_00ac9e4a47ce9a886dbf573480f151bd._comment b/doc/automatic_conflict_resolution/comment_5_00ac9e4a47ce9a886dbf573480f151bd._comment new file mode 100644 index 0000000000..4d1e7c4bcd --- /dev/null +++ b/doc/automatic_conflict_resolution/comment_5_00ac9e4a47ce9a886dbf573480f151bd._comment @@ -0,0 +1,10 @@ +[[!comment format=mdwn + username="http://joeyh.name/" + ip="209.250.56.41" + subject="comment 5" + date="2014-03-26T18:56:30Z" + content=""" +@Matthias you need to install git-annex 4.20130709 or newer. Then `git-annex merge` will do what you want. As I said before. + +As for committing the merge, you can always adjust the result after the fact and use `git commit --amend`. +"""]] diff --git a/doc/automatic_conflict_resolution/comment_6_8a0860fee88f5954918305f055a39d8d._comment b/doc/automatic_conflict_resolution/comment_6_8a0860fee88f5954918305f055a39d8d._comment new file mode 100644 index 0000000000..4e9493d129 --- /dev/null +++ b/doc/automatic_conflict_resolution/comment_6_8a0860fee88f5954918305f055a39d8d._comment @@ -0,0 +1,10 @@ +[[!comment format=mdwn + username="https://www.google.com/accounts/o8/id?id=AItOawn3p4i4lk_zMilvjnJ9sS6g2nerpgz0Fjc" + nickname="Matthias" + subject="auto conflict resolution master branch" + date="2014-04-13T17:48:19Z" + content=""" +@joeyh: This must be a misunderstanding of what I want. I use version 5.20140320. I can't find a workflow where \"git annex merge\" changes my master branch, it only updates the git-annex branch. + +Thinking again of it after some time, I am basically fine with \"git annex sync\". The only thing I am uncomfortable with is that the automatic merge is pushed without review. +"""]] diff --git a/doc/automatic_conflict_resolution/comment_7_3d2250cc26036b8532faa980065e20d0._comment b/doc/automatic_conflict_resolution/comment_7_3d2250cc26036b8532faa980065e20d0._comment new file mode 100644 index 0000000000..9f49021da5 --- /dev/null +++ b/doc/automatic_conflict_resolution/comment_7_3d2250cc26036b8532faa980065e20d0._comment @@ -0,0 +1,23 @@ +[[!comment format=mdwn + username="http://joeyh.name/" + ip="209.250.56.191" + subject="comment 7" + date="2014-04-17T20:00:22Z" + content=""" +@Matthias, here is an example of git-annex merge updating the master branch from the synced/master branch that was pushed to it earlier: + +
    +joey@darkstar:~/tmp/test/2>git annex merge
+merge git-annex (merging synced/git-annex into git-annex...)
+ok
+merge synced/master 
+Updating 7942eee..1f3422e
+Fast-forward
+ new_file | 1 +
+ 1 file changed, 1 insertion(+)
+ create mode 120000 new_file
+ok
+
    + +If you are having trouble with it somehow, I'd suggest filing a bug report. +"""]] diff --git a/doc/automatic_conflict_resolution/comment_8_ef474c258ce8e0ebc6485c1366ae6315._comment b/doc/automatic_conflict_resolution/comment_8_ef474c258ce8e0ebc6485c1366ae6315._comment new file mode 100644 index 0000000000..81f089b114 --- /dev/null +++ b/doc/automatic_conflict_resolution/comment_8_ef474c258ce8e0ebc6485c1366ae6315._comment @@ -0,0 +1,8 @@ +[[!comment format=mdwn + username="https://www.google.com/accounts/o8/id?id=AItOawn3p4i4lk_zMilvjnJ9sS6g2nerpgz0Fjc" + nickname="Matthias" + subject="auto conflict resolution" + date="2014-04-19T17:26:11Z" + content=""" +Thanks Joey, I could construct a scenario where the auto-conflict-resolution was applied in the master branch. +"""]] diff --git a/doc/backends.mdwn b/doc/backends.mdwn new file mode 100644 index 0000000000..e7acd8ed98 --- /dev/null +++ b/doc/backends.mdwn @@ -0,0 +1,77 @@ +When a file is annexed, a key is generated from its content and/or filesystem +metadata. The file checked into git symlinks to the key. This key can later +be used to retrieve the file's content (its value). + +Multiple pluggable key-value backends are supported, and a single repository +can use different ones for different files. + +These are the recommended backends to use. + +* `SHA256E` -- The default backend for new files, combines a 256 bit SHA-2 + hash of the file's content with the file's extension. This allows + verifying that the file content is right, and can avoid duplicates of + files with the same content. Its need to generate checksums + can make it slower for large files. +* `SHA256` -- SHA-2 hash that does not include the file extension in the + key, which can lead to better deduplication but can confuse some programs. +* `SHA512`, `SHA512E` -- Best SHA-2 hash, for the very paranoid. +* `SHA384`, `SHA384E`, `SHA224`, `SHA224E` -- SHA-2 hashes for + people who like unusual sizes. +* `SHA3_512`, `SHA3_512E`, `SHA3_384`, `SHA3_384E`, `SHA3_256`, `SHA3_256E`, `SHA3_224`, `SHA3_224E` + -- SHA-3 hashes, for bleeding edge fun. +* `SKEIN512`, `SKEIN512E`, `SKEIN256`, `SKEIN256E` + -- [Skein hash](http://en.wikipedia.org/wiki/Skein_hash), + a well-regarded SHA3 hash competition finalist. +* `BLAKE2B160`, `BLAKE2B224`, `BLAKE2B256`, `BLAKE2B384`, `BLAKE2B512` + `BLAKE2B160E`, `BLAKE2B224E`, `BLAKE2B256E`, `BLAKE2B384E`, `BLAKE2B512E` + -- Fast [Blake2 hash](https://blake2.net/) variants optimised for 64 bit + platforms. +* `BLAKE2S160`, `BLAKE2S224`, `BLAKE2S256` + `BLAKE2S160E`, `BLAKE2S224E`, `BLAKE2S256E` + -- Fast [Blake2 hash](https://blake2.net/) variants optimised for 32 bit + platforms. +* `BLAKE2SP224`, `BLAKE2SP256` + `BLAKE2SP224E`, `BLAKE2SP256E` + -- Fast [Blake2 hash](https://blake2.net/) variants optimised for + 8-way CPUs. + +The backends below do not guarantee cryptographically that the +content of an annexed file remains unchanged. + +* `SHA1`, `SHA1E`, `MD5`, `MD5E` -- Smaller hashes than `SHA256` + for those who want a checksum but are not concerned about security. +* `WORM` ("Write Once, Read Many") -- This assumes that any file with + the same filename, size, and modification time has the same content. + This is the least expensive backend, recommended for really large + files or slow systems. +* `URL` -- This is a key that is generated from the url to a file. + It's generated when using eg, `git annex addurl --fast`, when the file + content is not available for hashing. + +If you want to be able to prove that you're working with the same file +contents that were checked into a repository earlier, you should avoid +using the non-cryptographically-secure backends, and will need to use +signed git commits. See [[tips/using_signed_git_commits]] for details. + +Retrieval of WORM and URL from many [[special_remotes]] is prohibited +for [[security_reasons|security/CVE-2018-10857_and_CVE-2018-10859]]. + +Note that the various 512 and 384 length hashes result in long paths, +which are known to not work on Windows. If interoperability on Windows is a +concern, avoid those. + +The `annex.backend` git-config setting can be used to configure the +default backend to use when adding new files. + +For finer control of what backend is used when adding different types of +files, the `.gitattributes` file can be used. The `annex.backend` +attribute can be set to the name of the backend to use for matching files. + +For example, to use the SHA256E backend for sound files, which tend to be +smallish and might be modified or copied over time, +while using the WORM backend for everything else, you could set +in `.gitattributes`: + + * annex.backend=WORM + *.mp3 annex.backend=SHA256E + *.ogg annex.backend=SHA256E diff --git a/doc/backends/comment_10_920cd139dfec8adb1089f5acf26de4d2._comment b/doc/backends/comment_10_920cd139dfec8adb1089f5acf26de4d2._comment new file mode 100644 index 0000000000..a756f191a9 --- /dev/null +++ b/doc/backends/comment_10_920cd139dfec8adb1089f5acf26de4d2._comment @@ -0,0 +1,8 @@ +[[!comment format=mdwn + username="joey" + subject="""comment 10""" + date="2015-01-06T17:58:28Z" + content=""" +@Matthias, that directory structure is not controlled by the backend. +It is explained in [[internals]] +"""]] diff --git a/doc/backends/comment_11_f0f6316bbdc971a9ab157de9bbb9f74c._comment b/doc/backends/comment_11_f0f6316bbdc971a9ab157de9bbb9f74c._comment new file mode 100644 index 0000000000..fc19964950 --- /dev/null +++ b/doc/backends/comment_11_f0f6316bbdc971a9ab157de9bbb9f74c._comment @@ -0,0 +1,14 @@ +[[!comment format=mdwn + username="https://www.google.com/accounts/o8/id?id=AItOawnx8kHW66N3BqmkVpgtXDlYMvr8TJ5VvfY" + nickname="Yaroslav" + subject="add MD5SUM (with E) backend?" + date="2015-01-29T22:07:40Z" + content=""" +probably in many cases MD5SUM might be sufficient to cover the space of the available load and + +- its size would be even smaller than SHA1 (thus smaller git-annex footprint) +- immediate matching to often distributed MD5SUMs +- matching to ETags (whenever wasn't a multipart upload) in S3 buckets + +or use of MD5SUM hash is really not recommended for non-encryption-critical cases too? +"""]] diff --git a/doc/backends/comment_12_da76dff5fe712318d7d4313f1d827883._comment b/doc/backends/comment_12_da76dff5fe712318d7d4313f1d827883._comment new file mode 100644 index 0000000000..e4b3b277b7 --- /dev/null +++ b/doc/backends/comment_12_da76dff5fe712318d7d4313f1d827883._comment @@ -0,0 +1,10 @@ +[[!comment format=mdwn + username="joey" + subject="""MD5""" + date="2015-02-04T17:25:45Z" + content=""" +I've added MD5 and MD5E. Of course, if you choose to use these, or the WORM +backend, you give up the cryptographic verification that the content +currently in your repository is the same content that was in it before. +Whether that matters in your application is up to you. +"""]] diff --git a/doc/backends/comment_13_578423935bc71cdbdc23c3db06d1e870._comment b/doc/backends/comment_13_578423935bc71cdbdc23c3db06d1e870._comment new file mode 100644 index 0000000000..3d8f2a5a23 --- /dev/null +++ b/doc/backends/comment_13_578423935bc71cdbdc23c3db06d1e870._comment @@ -0,0 +1,8 @@ +[[!comment format=mdwn + username="https://www.google.com/accounts/o8/id?id=AItOawnx8kHW66N3BqmkVpgtXDlYMvr8TJ5VvfY" + nickname="Yaroslav" + subject="THANK YOU JOEY" + date="2015-02-09T14:04:27Z" + content=""" +for the MD5/MD5E (and now I have found \"email replies to me\" - I will become a power user of branchable ;) ) +"""]] diff --git a/doc/backends/comment_14_57154dcd1041a33f220f9105b709be89._comment b/doc/backends/comment_14_57154dcd1041a33f220f9105b709be89._comment new file mode 100644 index 0000000000..446960264b --- /dev/null +++ b/doc/backends/comment_14_57154dcd1041a33f220f9105b709be89._comment @@ -0,0 +1,8 @@ +[[!comment format=mdwn + username="xelez0@57a58225d4e5b260555ebc4a9d0a8df85d1e971a" + nickname="xelez0" + subject="Backend of specified file" + date="2015-07-05T13:19:43Z" + content=""" +How can I determine backend of specified file? Looking over man pages and can't find it. +"""]] diff --git a/doc/backends/comment_15_b3445fd1f379346c642a27211c6c798b._comment b/doc/backends/comment_15_b3445fd1f379346c642a27211c6c798b._comment new file mode 100644 index 0000000000..3fc12afee6 --- /dev/null +++ b/doc/backends/comment_15_b3445fd1f379346c642a27211c6c798b._comment @@ -0,0 +1,14 @@ +[[!comment format=mdwn + username="CandyAngel" + subject="comment 15" + date="2015-07-05T14:54:18Z" + content=""" +It's not explicit, but 'git annex info $FILE' tells you the key, which has the backend as its first component: + + ## git annex info CG\ Cookie/Compositing\ in\ Blender/01_CompositingInBlender_SourceFiles.zip + file: CG Cookie/Compositing in Blender/01_CompositingInBlender_SourceFiles.zip + size: 744.51 megabytes + key: SHA256E-s744506832--08d2daced60b5eb6509044d5eefca82e7a6899350f49adc0083014229739515e.zip + +I don't think there are any situations where the first component of the key isn't the backend, but don't hold me to that, please :) +"""]] diff --git a/doc/backends/comment_16_c68dfaeee2ef18f420f7e11ff5f604b9._comment b/doc/backends/comment_16_c68dfaeee2ef18f420f7e11ff5f604b9._comment new file mode 100644 index 0000000000..64c0102e28 --- /dev/null +++ b/doc/backends/comment_16_c68dfaeee2ef18f420f7e11ff5f604b9._comment @@ -0,0 +1,15 @@ +[[!comment format=mdwn + username="CandyAngel" + subject="comment 16" + date="2015-07-05T15:00:46Z" + content=""" +Or I could not be an idiot and tell you the command specifically looking up a key for a file: lookupkey + + ## git annex lookupkey CG\ Cookie/Compositing\ in\ Blender/01_CompositingInBlender_SourceFiles.zip + SHA256E-s744506832--08d2daced60b5eb6509044d5eefca82e7a6899350f49adc0083014229739515e.zip + +So to get the backend (if the first component is always the backend): + + ## git annex lookupkey CG\ Cookie/Compositing\ in\ Blender/01_CompositingInBlender_SourceFiles.zip | cut -d- -f1 + SHA256E +"""]] diff --git a/doc/backends/comment_17_557a622b3304eb86fed52896e0b6cbda._comment b/doc/backends/comment_17_557a622b3304eb86fed52896e0b6cbda._comment new file mode 100644 index 0000000000..6f6973ffd3 --- /dev/null +++ b/doc/backends/comment_17_557a622b3304eb86fed52896e0b6cbda._comment @@ -0,0 +1,7 @@ +[[!comment format=mdwn + username="joey" + subject="""comment 17""" + date="2015-07-06T16:03:53Z" + content=""" +See [[internals/key_format]]. +"""]] diff --git a/doc/backends/comment_18_784b29b086503a2b4913558350526ee1._comment b/doc/backends/comment_18_784b29b086503a2b4913558350526ee1._comment new file mode 100644 index 0000000000..43c554f7f0 --- /dev/null +++ b/doc/backends/comment_18_784b29b086503a2b4913558350526ee1._comment @@ -0,0 +1,12 @@ +[[!comment format=mdwn + username="junk@5e3eeba2290e8a3fcf938d9f93b0dfa2565dc7b1" + nickname="junk" + subject="Howto verify encrypted files" + date="2016-08-14T17:07:03Z" + content=""" +Hi, + +I'd like to be able verify the consistancy of the files on a rsync remote without having access to the git repository or the gpg-key. This can easily be done with unencrypted files by running \"sha256sum filename\". Is there a way to do the same thing with encrypted files? + +Thank you very much! +"""]] diff --git a/doc/backends/comment_19_cfcaeea6fdded241f5d36ef251ef8010._comment b/doc/backends/comment_19_cfcaeea6fdded241f5d36ef251ef8010._comment new file mode 100644 index 0000000000..87d79167c9 --- /dev/null +++ b/doc/backends/comment_19_cfcaeea6fdded241f5d36ef251ef8010._comment @@ -0,0 +1,12 @@ +[[!comment format=mdwn + username="joey" + subject="""comment 19""" + date="2016-09-05T19:47:48Z" + content=""" +@junk, this page is not really the place to ask such an unrelated question. +Please use the [[forum]] for such questions. + +(Anyway, git-annex uses gpg to encrypt data, so you can perhaps use gpg to +check the embedded checksum, but I have never done it, and git-annex +certianly doesn't support doing it.) +"""]] diff --git a/doc/backends/comment_1_375bb1fb5973e8fa67b763f2dd6e404b._comment b/doc/backends/comment_1_375bb1fb5973e8fa67b763f2dd6e404b._comment new file mode 100644 index 0000000000..dc178a6fed --- /dev/null +++ b/doc/backends/comment_1_375bb1fb5973e8fa67b763f2dd6e404b._comment @@ -0,0 +1,13 @@ +[[!comment format=mdwn + username="http://nanotech.nanotechcorp.net/" + nickname="NanoTech" + subject="SHA performance" + date="2012-08-10T04:37:32Z" + content=""" +It turns out that (at least on x86-64 machines) `SHA512` [is faster than][1] `SHA256`. In some benchmarks I performed1 `SHA256` was 1.8–2.2x slower than `SHA1` while `SHA512` was only 1.5–1.6x slower. + +`SHA224` and `SHA384` are effectively just truncated versions of `SHA256` and `SHA512` so their performance characteristics are identical. + +[1]: https://community.emc.com/community/edn/rsashare/blog/2010/11/01/sha-2-algorithms-when-sha-512-is-more-secure-and-faster +1 `time head -c 100000000 /dev/zero | shasum -a 512` +"""]] diff --git a/doc/backends/comment_20_29e500adf7e8614a938b3b714c9bd112._comment b/doc/backends/comment_20_29e500adf7e8614a938b3b714c9bd112._comment new file mode 100644 index 0000000000..88c717a4ea --- /dev/null +++ b/doc/backends/comment_20_29e500adf7e8614a938b3b714c9bd112._comment @@ -0,0 +1,36 @@ +[[!comment format=mdwn + username="https://launchpad.net/~stephane-gourichon-lpad" + nickname="stephane-gourichon-lpad" + avatar="http://cdn.libravatar.org/avatar/02d4a0af59175f9123720b4481d55a769ba954e20f6dd9b2792217d9fa0c6089" + subject="Lower-case extension backends: +1" + date="2016-10-22T17:55:13Z" + content=""" +> SHA256e +> I'd really like to have a SHA256e backend -- same as SHA256E but making sure that extensions of the files in .git/annex are converted to lower case. I normally try to convert filenames from cameras etc to lower case, but not all people that I share annex with do so consistently. In my use case, I need to be able to find duplicates among files and .jpg vs .JPG throws git annex dedup off. Otherwise E backends are superior to non-E for me. Thanks, Michael. + +Hello, + +TL;DR: *I second Michael's wish for hashing backends that aligns extensions to lowercase.* + +## Context, files with same content, extension have different case + +I realized a moment ago that git-annex basically automatically deduplicates with file granularity, which is very nice... *unless duplicates have varying case*, which does happen. For some cameras, if you download files through a cable you get one file name with one case, if you read the card directly with a card reader you get another case (and another filename, by the way). + +In invite anyone interested to drop a line here. + +## Workaround + +I understand I can align case after-the-fact with a bash shell command like below. Beware: man page of `rename` says there exist other versions that don't check for destination file, so the line below in some specific case (two files with same name, different content, file name only differs in case extension) might cause you to lose some information. Or perhaps other cases. Make sure you know what you do, I'm not responsible. + +``` +for EXT in aac amr arw asf avi bup crw ctg dsc dv jpg lrv m4a m4v mov mp3 mp4 mpe mpg mrk ndf nef njb ogg pdf png pnm ppm ps psd thm tif tiff txt ufraw wav xcf xcfbz2 xmp +do find . -iname \"*.${EXT}\" -print0 | xargs -0 rename -v \"s/${EXT}/${EXT,,}/i\" ; done +``` + +If you prefer to align to upper-case, replace `,,` with `^^`. This is bash syntax. + +## Please consider `SHA256e` backend (and others). + +Anyway the shell command above is a workaround. A case-insensitive hashing backend seems a natural thing to do. It would bring the best of both worlds: deduplicate efficiently while not confusing programs that depend on symlink target having a particular extension. + +"""]] diff --git a/doc/backends/comment_21_7f9cc5f966b28b7461e8ec42ceeb7165._comment b/doc/backends/comment_21_7f9cc5f966b28b7461e8ec42ceeb7165._comment new file mode 100644 index 0000000000..234b838592 --- /dev/null +++ b/doc/backends/comment_21_7f9cc5f966b28b7461e8ec42ceeb7165._comment @@ -0,0 +1,9 @@ +[[!comment format=mdwn + username="https://me.yahoo.com/a/iOGTltEpmOTQ.xZ99NFP5c7Zdcc-#6a7ba" + nickname="Ilya S" + avatar="http://cdn.libravatar.org/avatar/8a133555cc739a35b83b07d5724d28d9e2f7852c224e949eec6fd4fb7693331e" + subject="keys from metadata" + date="2018-09-07T15:57:37Z" + content=""" +\"When a file is annexed, a key is generated from its content and/or metadata\" -- the 'metadata' here just refers to the file name/size/mtime, not to https://git-annex.branchable.com/git-annex-metadata/ , correct? +"""]] diff --git a/doc/backends/comment_22_be75c669ed8e127de8b6364961a5c0cb._comment b/doc/backends/comment_22_be75c669ed8e127de8b6364961a5c0cb._comment new file mode 100644 index 0000000000..8905276e5d --- /dev/null +++ b/doc/backends/comment_22_be75c669ed8e127de8b6364961a5c0cb._comment @@ -0,0 +1,9 @@ +[[!comment format=mdwn + username="https://me.yahoo.com/a/iOGTltEpmOTQ.xZ99NFP5c7Zdcc-#6a7ba" + nickname="Ilya S" + avatar="http://cdn.libravatar.org/avatar/8a133555cc739a35b83b07d5724d28d9e2f7852c224e949eec6fd4fb7693331e" + subject="comment 22" + date="2018-09-07T15:59:06Z" + content=""" +Is it possible to configure git-annex to use different backends based on file size? I.e. use a faster hash or even WORM for larger files. +"""]] diff --git a/doc/backends/comment_23_8c8b08d7e6409756da4f9f3b21d5ad01._comment b/doc/backends/comment_23_8c8b08d7e6409756da4f9f3b21d5ad01._comment new file mode 100644 index 0000000000..e8651e00f7 --- /dev/null +++ b/doc/backends/comment_23_8c8b08d7e6409756da4f9f3b21d5ad01._comment @@ -0,0 +1,12 @@ +[[!comment format=mdwn + username="joey" + subject="""comment 23""" + date="2018-09-11T16:52:20Z" + content=""" +@Ilya, indeed, this page is talking about filesystem metadata. +I've updated it for clarity. + +There is not currently a way to switch backend based on file size, +although you can use annex.largefiles to make it check eg smaller files +directly into git rather than annexing them. +"""]] diff --git a/doc/backends/comment_24_f90b8eeae8d150eca6a91416f20eb223._comment b/doc/backends/comment_24_f90b8eeae8d150eca6a91416f20eb223._comment new file mode 100644 index 0000000000..c22fe26de7 --- /dev/null +++ b/doc/backends/comment_24_f90b8eeae8d150eca6a91416f20eb223._comment @@ -0,0 +1,10 @@ +[[!comment format=mdwn + username="Ilya_Shlyakhter" + avatar="http://cdn.libravatar.org/avatar/1647044369aa7747829c38b9dcc84df0" + subject="comment 24" + date="2018-09-19T16:50:05Z" + content=""" +It seems that *E backends ignore file extensions longer than four chars: https://git-annex.branchable.com/bugs/file_extensions_of___62__4_chars_ignored_by___42__E_backends/ +Is there some reason for doing it this way? + +"""]] diff --git a/doc/backends/comment_25_f6a697ac069ab018ccacf8f16b09bc7e._comment b/doc/backends/comment_25_f6a697ac069ab018ccacf8f16b09bc7e._comment new file mode 100644 index 0000000000..9e7f9f12d7 --- /dev/null +++ b/doc/backends/comment_25_f6a697ac069ab018ccacf8f16b09bc7e._comment @@ -0,0 +1,10 @@ +[[!comment format=mdwn + username="joey" + subject="""comment 25""" + date="2018-09-24T15:42:24Z" + content=""" +@Ilya_Shlyakhter, it's a heuristic, what consititues a file extension is +not very well defined (consider ".tar.gz" and ".not-an-extension"). +The heuristic has been refined over the years, but will never be perfect. +But, I don't know of any 5+ character file extensions in common use! +"""]] diff --git a/doc/backends/comment_2_1f2626eca9004b31a0b7fc1a0df8027b._comment b/doc/backends/comment_2_1f2626eca9004b31a0b7fc1a0df8027b._comment new file mode 100644 index 0000000000..0308873036 --- /dev/null +++ b/doc/backends/comment_2_1f2626eca9004b31a0b7fc1a0df8027b._comment @@ -0,0 +1,24 @@ +[[!comment format=mdwn + username="https://www.google.com/accounts/o8/id?id=AItOawm7eqCMh_B7mxE0tnchbr0JoYu11FUAFRY" + nickname="Stéphane" + subject="Tracking remote copies not even stored locally / URL backend turned into a "special remote"." + date="2013-01-03T10:59:35Z" + content=""" +In case you came here looking for the URL backend. + +## The URL backend + +Several documents on the web refer to a special \"URL backend\", e.g. [Large file management with git-annex [LWN.net]](http://lwn.net/Articles/419241/). Historical content will never be updated yet it drives people to living places. + +## Why a URL backend ? + +It is interesting because you can: + +* let `git-annex` rest on the fact that some documents are available as extra copies available at any time (but from something that is not a git repository). +* track these documents like your own with all git features, which opens up some truly marvelous combinations, which this margin is too narrow to contain (Pierre d.F. wouldn't disapprove ;-). + +## How/Where now ? + +`git-annex` used to have a URL backend. It seems that the design changed into a \"special remote\" feature, not limited to the web. You can now track files available through plain directories, rsync, webdav, some cloud storage, etc, even clay tablets. For details see [[special remotes]]. + +"""]] diff --git a/doc/backends/comment_3_fdcbf8727fdefb9942a54689234b9698._comment b/doc/backends/comment_3_fdcbf8727fdefb9942a54689234b9698._comment new file mode 100644 index 0000000000..6f8b1ad7a4 --- /dev/null +++ b/doc/backends/comment_3_fdcbf8727fdefb9942a54689234b9698._comment @@ -0,0 +1,12 @@ +[[!comment format=mdwn + username="https://www.google.com/accounts/o8/id?id=AItOawmicVKRM8vJX4wPuAwlLEoS2cjmFXQkjkE" + nickname="Thomas" + subject="Please be more specific about what information goes into the key" + date="2013-07-31T11:55:09Z" + content=""" +It's a bit confusing to read that SHA256 does not include the file extension from which I can deduct that SHA256E does include it. What else does it include? I used to \"seed\" my git-annex with localy available data by \"git-annex add\"-ing it in a temporary folder without doing a commit and than to initiate a copy from the slow remote annex repo. My theory was that remote copy sees the pre-seeded files and does not need to copy them again. + +But does this theory hold true for different file names, extensions, modification date, full path? Maybe you could also link to the code that implements the different backends so that curious readers can check for themselves. + +Thank you! +"""]] diff --git a/doc/backends/comment_4_46591a3ba888fb686b1b319b80ca2c22._comment b/doc/backends/comment_4_46591a3ba888fb686b1b319b80ca2c22._comment new file mode 100644 index 0000000000..94f25e9962 --- /dev/null +++ b/doc/backends/comment_4_46591a3ba888fb686b1b319b80ca2c22._comment @@ -0,0 +1,9 @@ +[[!comment format=mdwn + username="https://www.google.com/accounts/o8/id?id=AItOawmTho3mActvetF1iQdmui6gH1t6WE6c284" + nickname="Michael" + subject="SHA256e" + date="2013-10-30T02:00:45Z" + content=""" +I'd really like to have a SHA256e backend -- same as SHA256E but making sure that extensions of the files in .git/annex are converted to lower case. I normally try to convert filenames from cameras etc to lower case, but not all people that I share annex with do so consistently. +In my use case, I need to be able to find duplicates among files and .jpg vs .JPG throws git annex dedup off. Otherwise E backends are superior to non-E for me. Thanks, Michael. +"""]] diff --git a/doc/backends/comment_5_2210c7ff2d5812fb3b778ac172291656._comment b/doc/backends/comment_5_2210c7ff2d5812fb3b778ac172291656._comment new file mode 100644 index 0000000000..2a205604c2 --- /dev/null +++ b/doc/backends/comment_5_2210c7ff2d5812fb3b778ac172291656._comment @@ -0,0 +1,8 @@ +[[!comment format=mdwn + username="https://www.google.com/accounts/o8/id?id=AItOawmWBvsZvSsAL8P2ye3F0OBStjFCVnOImzM" + nickname="Jarno" + subject="Non-E backend drawbacks?" + date="2013-10-30T21:25:00Z" + content=""" +The page states \"[non-E backends] can confuse some programs\". I like the ideal simplicity and recoverability of pure checksum backends but \"confusion\" sounds a bit worrying. Any practical examples of these problems to help me choose? +"""]] diff --git a/doc/backends/comment_6_82f239b58680a2681bd8074c7ef9584d._comment b/doc/backends/comment_6_82f239b58680a2681bd8074c7ef9584d._comment new file mode 100644 index 0000000000..cbd4682ab2 --- /dev/null +++ b/doc/backends/comment_6_82f239b58680a2681bd8074c7ef9584d._comment @@ -0,0 +1,8 @@ +[[!comment format=mdwn + username="http://joeyh.name/" + ip="209.250.56.47" + subject="comment 6" + date="2013-11-01T15:47:26Z" + content=""" +Some examples of problems with the raw SHA backends include, IIRC, calibre, and many programs on OSX. These programs look at the extension of the filename the symlink points at. +"""]] diff --git a/doc/backends/comment_7_4aa8cfaec1090f79fed530720e4ddad4._comment b/doc/backends/comment_7_4aa8cfaec1090f79fed530720e4ddad4._comment new file mode 100644 index 0000000000..3028547fe4 --- /dev/null +++ b/doc/backends/comment_7_4aa8cfaec1090f79fed530720e4ddad4._comment @@ -0,0 +1,18 @@ +[[!comment format=mdwn + username="https://www.google.com/accounts/o8/id?id=AItOawmraN_ldJplGunVGmnjjLN6jL9s9TrVMGE" + nickname="Ævar Arnfjörð" + subject="Can annex use existing backends when amending existing files?" + date="2014-08-05T21:35:34Z" + content=""" +Related to the question posed in http://git-annex.branchable.com/forum/switching_backends/ can git annex be told to use the existing backend for a given file? + +The use case for this is that you have an existing repo that started out e.g. with SHA256, but new files are being added with SHA256E since that's the default now. + +But I was doing: + + git annex edit . + rsync /some/old/copy/ . + git annex add . + +And was expecting it to show no changes for existing files, but it did, it would be nice if that was not the case. +"""]] diff --git a/doc/backends/comment_8_c40d2c2c929ad3239ee5d529e307c746._comment b/doc/backends/comment_8_c40d2c2c929ad3239ee5d529e307c746._comment new file mode 100644 index 0000000000..17ac4c2754 --- /dev/null +++ b/doc/backends/comment_8_c40d2c2c929ad3239ee5d529e307c746._comment @@ -0,0 +1,8 @@ +[[!comment format=mdwn + username="http://joeyh.name/" + ip="209.250.56.7" + subject="comment 8" + date="2014-08-12T18:00:46Z" + content=""" +Ævar, you can use `git annex add --backend=SHA256` to temporarily override the backend. +"""]] diff --git a/doc/backends/comment_9_5bef7f76f5e5b73a8e404fe6172b4368._comment b/doc/backends/comment_9_5bef7f76f5e5b73a8e404fe6172b4368._comment new file mode 100644 index 0000000000..bf8f7f8d2b --- /dev/null +++ b/doc/backends/comment_9_5bef7f76f5e5b73a8e404fe6172b4368._comment @@ -0,0 +1,12 @@ +[[!comment format=mdwn + username="https://www.google.com/accounts/o8/id?id=AItOawm_YXzEdPHzbSGVwtmTR7g1BqDtTnIBB5s" + nickname="Matthias" + subject="Over-long pathnames?" + date="2015-01-06T09:41:03Z" + content=""" +the SHA* backends generate too-complicated paths: + +lrwxrwxrwx 1 root root 193 Apr 22 2009 test.ogg -> ../../../.git/annex/objects/fX/pz/SHA256-s71983--4a55ff578b4c592c06a1f4d9e0f8a6949ea9961d9717fc22e7b3c412620ac890/SHA256-s71983--4a55ff578b4c592c06a1f4d9e0f8a6949ea9961d9717fc22e7b3c412620ac890 + +I don't want the additional directory. What is it for?? It contains exactly one file and adds a couple of disk seeks to file lookup. +"""]] diff --git a/doc/bare_repositories.mdwn b/doc/bare_repositories.mdwn new file mode 100644 index 0000000000..f73f2f845a --- /dev/null +++ b/doc/bare_repositories.mdwn @@ -0,0 +1,53 @@ +Due to popular demand, git-annex can now be used with bare repositories. + +So, for example, you can stash a file away in the origin: +`git annex move mybigfile --to origin` + +Of course, for that to work, the bare repository has to be on a system with +[[git-annex-shell]] installed. If "origin" is on GitWeb, you still can't +use git-annex to store stuff there. + +It took a while, but bare repositories are now supported exactly as well +as non-bare repositories. Except for these caveats: + +* `git annex fsck` works in a bare repository, but does not display + warnings about insufficient + [[copies]]. To get those warnings, just run it in one of the non-bare + checkouts. +* `git annex unused` in a bare repository only knows about keys used in + branches that have been pushed to the bare repository. So use it with care.. +* Commands that need a work tree, like `git annex add` won't work in a bare + repository, of course. +* However, you can run commands like `git annex copy`, `git annex get`, and + `git annex drop` in a bare repository. In a bare repository, these + behave as if the `--all` option were used, and so operate + on every single version of every single file that is present in the git + repository history. The `--branch` option can be used to make these + commands only operate on the files referenced by a specified branch. + For example: `git annex get --branch master` + +*** + +Here is a quick example of how to set this up, using `origin` as the remote name, and assuming `~/annex` contains an annex: + +On the server: + + git init --bare bare-annex.git + cd bare-annex.git && git annex init origin + +Now configure the remote and do the initial push: + + cd ~/annex + git remote add origin example.com:bare-annex.git + git push origin master git-annex + +Now `git annex info` should show the configured bare remote. If it does +not, you may have to pull from the remote first (older versions of +`git-annex`). + +If you wish to configure git such that you can push/pull without arguments, +set the upstream branch: + + git branch master --set-upstream origin/master + + diff --git a/doc/bare_repositories/comment_1_148e1da70d37d311634a0309a4ff8dcd._comment b/doc/bare_repositories/comment_1_148e1da70d37d311634a0309a4ff8dcd._comment new file mode 100644 index 0000000000..c1ba9f2f4a --- /dev/null +++ b/doc/bare_repositories/comment_1_148e1da70d37d311634a0309a4ff8dcd._comment @@ -0,0 +1,22 @@ +[[!comment format=mdwn + username="https://www.google.com/accounts/o8/id?id=AItOawmraN_ldJplGunVGmnjjLN6jL9s9TrVMGE" + nickname="Ævar Arnfjörð" + subject="How to convert bare repositories to non-bare" + date="2012-11-11T20:14:44Z" + content=""" +I made a repository bare and later wanted to convert it, this would have worked with just plain git: + + cd bare-repo.git + mkdir .git + mv .??* * .git/ + git config --unset core.bare + git reset --hard + +But because git-annex uses different hashing directories under bare repositories all the files in the repo will point to files you don't have. Here's how you can fix that up assuming you're using a backend that assigns unique hashes based on file content (e.g. the SHA256 backend): + + mv .git/annex/objects from-bare-repo + git annex add from-bare-repo + git rm -f from-bare-repo + + +"""]] diff --git a/doc/bare_repositories/comment_2_c88216da0588562c851c2ceabbfebc0a._comment b/doc/bare_repositories/comment_2_c88216da0588562c851c2ceabbfebc0a._comment new file mode 100644 index 0000000000..4350407037 --- /dev/null +++ b/doc/bare_repositories/comment_2_c88216da0588562c851c2ceabbfebc0a._comment @@ -0,0 +1,28 @@ +[[!comment format=mdwn + username="https://openid.stackexchange.com/user/814e4910-8e9b-4fe5-83ef-ff863c1a7314" + nickname="BehemothTheCat" + subject="push fails" + date="2015-02-14T00:11:40Z" + content=""" +These instructions don't work for me, unfortunately. + +This step: + + git push origin master git-annex + +results in: + + To ssh://my.server.com/home/itz/git/annex.git + ! [rejected] git-annex -> git-annex (non-fast-forward) + error: failed to push some refs to 'ssh://my.server.com/home/itz/git/annex.git' + hint: Updates were rejected because a pushed branch tip is behind its remote + hint: counterpart. Check out this branch and integrate the remote changes + hint: (e.g. 'git pull ...') before pushing again. + hint: See the 'Note about fast-forwards' in 'git push --help' for details. + +Versions: git 1:1.9.1-1~bpo70+2 , git-annex 5.20141024~bpo70+1 (both packaged by Debian, same on local and remote) + +And yes, I did a pull on the master branch first. Afraid to do anything +with the git-annex branch without explicit instruction. + +"""]] diff --git a/doc/bare_repositories/comment_3_26ba93bddb0cd1bb4e1799311f3ca750._comment b/doc/bare_repositories/comment_3_26ba93bddb0cd1bb4e1799311f3ca750._comment new file mode 100644 index 0000000000..b20ce0be83 --- /dev/null +++ b/doc/bare_repositories/comment_3_26ba93bddb0cd1bb4e1799311f3ca750._comment @@ -0,0 +1,11 @@ +[[!comment format=mdwn + username="joey" + subject="""comment 3""" + date="2015-02-17T21:54:33Z" + content=""" +Since the two repos git-annex branches have diverged, you need to run `git +annex merge` to merge them before you can push that branch. + +Of course, `git annex sync` handles all that for you. It can be used +against a bare repository as well as a non-bare. +"""]] diff --git a/doc/bare_repositories/comment_4_7cf6103709a7a2710686681a6f406214._comment b/doc/bare_repositories/comment_4_7cf6103709a7a2710686681a6f406214._comment new file mode 100644 index 0000000000..0953d091f0 --- /dev/null +++ b/doc/bare_repositories/comment_4_7cf6103709a7a2710686681a6f406214._comment @@ -0,0 +1,8 @@ +[[!comment format=mdwn + username="https://me.yahoo.com/a/bBy7WkgQicYHIiiyj.Vm0TcMbxi2quzbPFef#6f9f7" + nickname="Frederik Vanrenterghem" + subject="Convert bare repository to normal" + date="2015-05-19T09:17:15Z" + content=""" +Just wondering what the right steps are to convert a bare repository to a normal one? The comment above from 2.5 years ago includes the creation of a .git directory, which actually already exists. Have things changed in the mean time? +"""]] diff --git a/doc/bare_repositories/comment_5_6328134497c0de6a088087fc9cb0e59e._comment b/doc/bare_repositories/comment_5_6328134497c0de6a088087fc9cb0e59e._comment new file mode 100644 index 0000000000..80906857ed --- /dev/null +++ b/doc/bare_repositories/comment_5_6328134497c0de6a088087fc9cb0e59e._comment @@ -0,0 +1,31 @@ +[[!comment format=mdwn + username="https://id.koumbit.net/anarcat" + subject="comment 5" + date="2015-05-19T12:18:05Z" + content=""" +Well, no, i don't think they changed, unless i missed something: there +shouldn't be a `.git` repository there. + +There are [various +instructions](http://stackoverflow.com/questions/10637378/how-do-i-convert-a-bare-git-repository-into-a-normal-one-in-place) +on how to do this online. They do seem to agree with the first comment +above. + +Personnally, I would just `git clone` to a different repo and `git +annex forget` the old one. Unless you have a very complex repository +with a lot of files, this is simple enough... You could even use `git +annex reinit` to recycle the previous uuid if that's a concern. So in +short: + + git clone repo.git repo + cd repo + git annex info --fast # find the UUID of repo.git + git annex move --from $UUID + git annex reinit $UUID + +Then `repo.git` can be removed if you are certain everything is +correct in `repo`. + +Note that you may want to have backups of everything before you do +anything, as usual. +"""]] diff --git a/doc/bare_repositories/comment_6_bf227861ec3cb2ea474c143218c68133._comment b/doc/bare_repositories/comment_6_bf227861ec3cb2ea474c143218c68133._comment new file mode 100644 index 0000000000..1a43c0a2f0 --- /dev/null +++ b/doc/bare_repositories/comment_6_bf227861ec3cb2ea474c143218c68133._comment @@ -0,0 +1,12 @@ +[[!comment format=mdwn + username="https://me.yahoo.com/a/bBy7WkgQicYHIiiyj.Vm0TcMbxi2quzbPFef#6f9f7" + nickname="Frederik Vanrenterghem" + subject="comment 6" + date="2015-05-21T06:14:33Z" + content=""" +Thanks Anarcat. I actually was dealing with a repository in direct mode, not bare mode. Only realised after today encountering one in bare mode, and reading [[internals]]. Your method worked too, but + + git annex indirect + +would have been quicker I guess. +"""]] diff --git a/doc/bare_repositories/comment_7_9e065a2d8aaf7371ab7c0bb4f0c724b0._comment b/doc/bare_repositories/comment_7_9e065a2d8aaf7371ab7c0bb4f0c724b0._comment new file mode 100644 index 0000000000..c21b67f095 --- /dev/null +++ b/doc/bare_repositories/comment_7_9e065a2d8aaf7371ab7c0bb4f0c724b0._comment @@ -0,0 +1,13 @@ +[[!comment format=mdwn + username="scottgorlin@a32946b2aad278883c1690a0753241583a9855b9" + nickname="scottgorlin" + avatar="http://cdn.libravatar.org/avatar/2dd1fc8add62bbf4ffefac081b322563" + subject="synced/* branches necessary on bare repo?" + date="2016-10-18T21:17:40Z" + content=""" +Are the synced/* branches necessary for a bare repo? Since there is no working directory, can't sync operate directly on the top level branches of choice, and avoid 'branch pollution'? Is it implemented this way just to simplify the code, or ...? + +Also wondering if there is a practical difference between a bare repo and an rsync repo (other than storing the metadata, of course) - they both use rsync for transfer, no? Any speed/overhead/other differences to consider when choosing? + +For my use case pertaining to question 2 I am using a local ARM NAS which works with the binary (plus S3 or gdrive remotes etc), but I figure why not spin up a private gitlab repo with metadata and no content since I want an off-site backup of the metadata anyways. Does a bare repo on a nas add anything here, vs an rsync remote? Perhaps just rsync is even better to avoid overhead on a tiny/slow NAS? +"""]] diff --git a/doc/bare_repositories/comment_8_d4dff0d6265be941af9f66ba09f36ebe._comment b/doc/bare_repositories/comment_8_d4dff0d6265be941af9f66ba09f36ebe._comment new file mode 100644 index 0000000000..d237d1bf62 --- /dev/null +++ b/doc/bare_repositories/comment_8_d4dff0d6265be941af9f66ba09f36ebe._comment @@ -0,0 +1,12 @@ +[[!comment format=mdwn + username="joey" + subject="""comment 8""" + date="2016-10-26T17:44:55Z" + content=""" +@scottgorlin bare git repositories cannot in general be detected when +looking at a remote, so `git annex sync` picks a behavior that works +whether a remote is a bare git repository or not. + +A bare repo and a rsync special remote should have pretty similar +performance. +"""]] diff --git a/doc/benchmarking.mdwn b/doc/benchmarking.mdwn new file mode 100644 index 0000000000..68fc328d0f --- /dev/null +++ b/doc/benchmarking.mdwn @@ -0,0 +1,3 @@ +This page exsists to collect benchmarking data about git-annex, so it can +be referred to later. If you have a specific instance where git-annex seems +unncessarily slow, please file a bug report about it. diff --git a/doc/benchmarking/comment_10_1af4ac0d37c876912678522895c1656b._comment b/doc/benchmarking/comment_10_1af4ac0d37c876912678522895c1656b._comment new file mode 100644 index 0000000000..868b103646 --- /dev/null +++ b/doc/benchmarking/comment_10_1af4ac0d37c876912678522895c1656b._comment @@ -0,0 +1,61 @@ +[[!comment format=mdwn + username="joey" + subject="""comment 10""" + date="2016-09-29T18:33:33Z" + content=""" +* Optimised key2file and file2key. 18% scanning time speedup. +* Optimised adjustGitEnv. 50% git-annex branch query speedup +* Optimised parsePOSIXTime. 10% git-annex branch query speedup +* Tried making catObjectDetails.receive use ByteString for parsing, + but that did not seem to speed it up significantly. + So it parsing is already fairly optimal, it's just that a + lot of data passes through it when querying the git-annex + branch. + +After all that, profiling `git-annex find`: + + Thu Sep 29 16:51 2016 Time and Allocation Profiling Report (Final) + + git-annex.1 +RTS -p -RTS find + + total time = 1.73 secs (1730 ticks @ 1000 us, 1 processor) + total alloc = 1,812,406,632 bytes (excludes profiling overheads) + + COST CENTRE MODULE %time %alloc + + md5 Data.Hash.MD5 28.0 37.9 + catchIO Utility.Exception 10.2 12.5 + inAnnex'.checkindirect Annex.Content 9.9 3.7 + catches Control.Monad.Catch 8.7 5.7 + readish Utility.PartialPrelude 5.7 3.0 + isAnnexLink Annex.Link 5.0 8.4 + keyFile Annex.Locations 4.2 5.8 + spanList Data.List.Utils 4.0 6.3 + startswith Data.List.Utils 2.0 1.3 + +And `git-annex find --not --in web`: + + Thu Sep 29 16:35 2016 Time and Allocation Profiling Report (Final) + + git-annex +RTS -p -RTS find --not --in web + + total time = 5.24 secs (5238 ticks @ 1000 us, 1 processor) + total alloc = 3,293,314,472 bytes (excludes profiling overheads) + + COST CENTRE MODULE %time %alloc + + catObjectDetails.receive Git.CatFile 12.9 5.5 + md5 Data.Hash.MD5 10.6 20.8 + readish Utility.PartialPrelude 7.3 8.2 + catchIO Utility.Exception 6.7 7.3 + spanList Data.List.Utils 4.1 7.4 + readFileStrictAnyEncoding Utility.Misc 3.5 1.3 + catches Control.Monad.Catch 3.3 3.2 + +So, quite a large speedup overall! + +This leaves md5 still unoptimised at 10-28% of CPU use. I looked at switching +it to cryptohash's implementation, but it would require quite a lot of +bit-banging math to pull the used values out of the ByteString containing +the md5sum. +"""]] diff --git a/doc/benchmarking/comment_11_1ca8d9765e6e3a18ae09df74bc390a00._comment b/doc/benchmarking/comment_11_1ca8d9765e6e3a18ae09df74bc390a00._comment new file mode 100644 index 0000000000..619351d4ca --- /dev/null +++ b/doc/benchmarking/comment_11_1ca8d9765e6e3a18ae09df74bc390a00._comment @@ -0,0 +1,49 @@ +[[!comment format=mdwn + username="joey" + subject="""comment 11""" + date="2017-05-15T21:56:52Z" + content=""" +Switched from MissingH to cryptonite for md5. It did move md5 out of the top CPU spot but +the overall runtime didn't change much. Memory allocations did go down by a +good amount. + +Updated profiles: + + git-annex +RTS -p -RTS find + + total time = 1.63 secs (1629 ticks @ 1000 us, 1 processor) + total alloc = 1,496,336,496 bytes (excludes profiling overheads) + + COST CENTRE MODULE SRC %time %alloc + + catchIO Utility.Exception Utility/Exception.hs:79:1-17 14.1 15.1 + inAnnex'.checkindirect Annex.Content Annex/Content.hs:(108,9)-(119,39) 10.6 4.8 + catches Control.Monad.Catch src/Control/Monad/Catch.hs:(432,1)-(436,76) 8.6 6.9 + spanList Data.List.Utils src/Data/List/Utils.hs:(150,1)-(155,36) 6.7 11.1 + isAnnexLink Annex.Link Annex/Link.hs:35:1-85 5.0 10.2 + keyFile Annex.Locations Annex/Locations.hs:(456,1)-(462,19) 5.0 7.0 + readish Utility.PartialPrelude Utility/PartialPrelude.hs:(48,1)-(50,20) 3.8 2.0 + startswith Data.List.Utils src/Data/List/Utils.hs:103:1-23 3.6 2.3 + splitc Utility.Misc Utility/Misc.hs:(52,1)-(54,25) 3.4 6.5 + s2w8 Data.Bits.Utils src/Data/Bits/Utils.hs:65:1-15 2.6 6.4 + keyPath Annex.Locations Annex/Locations.hs:(492,1)-(494,23) 2.5 4.4 + fileKey.unesc Annex.Locations Annex/Locations.hs:(469,9)-(474,39) 2.0 3.5 + copyAndFreeze Data.ByteArray.Methods Data/ByteArray/Methods.hs:(224,1)-(227,21) 1.8 0.5 + + git-annex +RTS -p -RTS find --not --in web + + total time = 5.33 secs (5327 ticks @ 1000 us, 1 processor) + total alloc = 2,908,489,000 bytes (excludes profiling overheads) + + COST CENTRE MODULE SRC %time %alloc + + catObjectDetails.\ Git.CatFile Git/CatFile.hs:(80,72)-(88,97) 7.8 2.8 + catchIO Utility.Exception Utility/Exception.hs:79:1-17 7.6 8.3 + spanList Data.List.Utils src/Data/List/Utils.hs:(150,1)-(155,36) 5.8 9.1 + readish Utility.PartialPrelude Utility/PartialPrelude.hs:(48,1)-(50,20) 4.5 4.0 + parseResp Git.CatFile Git/CatFile.hs:(113,1)-(124,28) 4.4 2.9 + readFileStrict Utility.Misc Utility/Misc.hs:33:1-59 3.7 1.6 + catches Control.Monad.Catch src/Control/Monad/Catch.hs:(432,1)-(436,76) 3.1 3.6 + encodeW8 Utility.FileSystemEncoding Utility/FileSystemEncoding.hs:(131,1)-(133,70) 3.1 2.3 + +"""]] diff --git a/doc/benchmarking/comment_8_c1f99493f5e5c362d5c39f048280b11b._comment b/doc/benchmarking/comment_8_c1f99493f5e5c362d5c39f048280b11b._comment new file mode 100644 index 0000000000..e0f4987a02 --- /dev/null +++ b/doc/benchmarking/comment_8_c1f99493f5e5c362d5c39f048280b11b._comment @@ -0,0 +1,45 @@ +[[!comment format=mdwn + username="joey" + subject="""profiling""" + date="2016-09-26T19:20:36Z" + content=""" +Built git-annex with profiling, using `stack build --profile` + +(For reproduciblity, running git-annex in a clone of the git-annex repo +https://github.com/RichiH/conference_proceedings with rev +2797a49023fc24aff6fcaec55421572e1eddcfa2 checked out. It has 9496 annexed +objects.) + +Profiling `git-annex find +RTS -p`: + + total time = 3.53 secs (3530 ticks @ 1000 us, 1 processor) + total alloc = 3,772,700,720 bytes (excludes profiling overheads) + + COST CENTRE MODULE %time %alloc + + spanList Data.List.Utils 32.6 37.7 + startswith Data.List.Utils 14.3 8.1 + md5 Data.Hash.MD5 12.4 18.2 + join Data.List.Utils 6.9 13.7 + catchIO Utility.Exception 5.9 6.0 + catches Control.Monad.Catch 5.0 2.8 + inAnnex'.checkindirect Annex.Content 4.6 1.8 + readish Utility.PartialPrelude 3.0 1.4 + isAnnexLink Annex.Link 2.6 4.0 + split Data.List.Utils 1.5 0.8 + keyPath Annex.Locations 1.2 1.7 + + +This is interesting! + +Fully 40% of CPU time and allocations are in list (really String) processing, +and the details of the profiling report show that `spanList` and `startsWith` +and `join` are all coming from calls to `replace` in `keyFile` and `fileKey`. +Both functions nest several calls to replace, so perhaps that could be unwound +into a single pass and/or a ByteString used to do it more efficiently. + +12% of run time is spent calculating the md5 hashes for the hash +directories for .git/annex/objects. Data.Hash.MD5 is from missingh, and +it is probably a quite unoptimised version. Switching to the version +if cryptonite would probably speed it up a lot. +"""]] diff --git a/doc/benchmarking/comment_9_f4d802a28b79905da0cb24af6cb65b0a._comment b/doc/benchmarking/comment_9_f4d802a28b79905da0cb24af6cb65b0a._comment new file mode 100644 index 0000000000..9692ad2d73 --- /dev/null +++ b/doc/benchmarking/comment_9_f4d802a28b79905da0cb24af6cb65b0a._comment @@ -0,0 +1,42 @@ +[[!comment format=mdwn + username="joey" + subject="""more profiling""" + date="2016-09-26T19:59:43Z" + content=""" +Instead of profiling `git annex copy --to remote`, I profiled `git annex +find --not --in web`, which needs to do the same kind of location log lookup. + + total time = 12.41 secs (12413 ticks @ 1000 us, 1 processor) + total alloc = 8,645,057,104 bytes (excludes profiling overheads) + + COST CENTRE MODULE %time %alloc + + adjustGitEnv Git.Env 21.4 37.0 + catchIO Utility.Exception 13.2 2.8 + spanList Data.List.Utils 12.6 17.9 + parsePOSIXTime Logs.TimeStamp 6.1 5.0 + catObjectDetails.receive Git.CatFile 5.9 2.1 + startswith Data.List.Utils 5.7 3.8 + md5 Data.Hash.MD5 5.1 7.9 + join Data.List.Utils 2.4 6.0 + readFileStrictAnyEncoding Utility.Misc 2.2 0.5 + +The adjustGitEnv overhead is a surprise! It seems it is getting called once +per file, and allocating a new copy of the environment each time. Call stack: +withIndex calls withIndexFile calls addGitEnv calls adjustGitEnv. +Looks like simply making gitEnv be cached at startup would avoid most of +the adjustGitEnv slowdown. + +(The catchIO overhead is a false reading; the detailed profile shows +that all its time and allocations are inherited. getAnnexLinkTarget +is running catchIO in the expensive case, so readSymbolicLink is +the actual expensive bit.) + +The parsePOSIXTime comes from reading location logs. It's implemented +using a generic Data.Time.Format.parseTime, which uses a format string +"%s%Qs". A custom parser that splits into seconds and picoseconds +and simply reads both numbers might be more efficient. + +catObjectDetails.receive is implemented using mostly String and could +probably be sped up by being converted to use ByteString. +"""]] diff --git a/doc/bugs.mdwn b/doc/bugs.mdwn new file mode 100644 index 0000000000..48688b21fc --- /dev/null +++ b/doc/bugs.mdwn @@ -0,0 +1,10 @@ +This is git-annex's bug list. + +To mark a bug as closed add \[\[done\]\] to the end of the original bug description. Closed bugs are moved to [[done]]. + +[[!inline pages="./bugs/* and !./bugs/*/* and !./bugs/done and !link(done) +and !./bugs/moreinfo and !./bugs/confirmed and !./bugs/forwarded and !*/Discussion" +actions=yes postform=yes postformtext="Report a new bug titled:" show=0 archive=yes +feedlimit=10 template=buglist]] + +[[!edittemplate template=templates/bugtemplate match="bugs/*" silent=yes]] diff --git a/doc/bugs/--backend_for_init_is_in_no_effect__63__.mdwn b/doc/bugs/--backend_for_init_is_in_no_effect__63__.mdwn new file mode 100644 index 0000000000..a039cbe72a --- /dev/null +++ b/doc/bugs/--backend_for_init_is_in_no_effect__63__.mdwn @@ -0,0 +1,75 @@ +### Please describe the problem. + +`annex init --help` talks about supporting `--backend` so I expected either modification of .gitattributes to add backend, or at least non-persistent setting in .git/config ... but seems nothing is actually done? + + +### What version of git-annex are you using? On what operating system? + +6.20170525+gitge1cf095ae-1~ndall+1 + +### Please provide any additional information below. + +[[!format sh """ + +$> git annex init --backend=MD5E +init ok +(recording state in git...) +(dev) 2 27354 [1].....................................:Wed 12 Jul 2017 07:08:21 PM EDT:. +(git)hopa:/tmp/1233[master] +$> cat .git/config +[core] + repositoryformatversion = 0 + filemode = true + bare = false + logallrefupdates = true +[annex] + uuid = 8a045a11-af4a-43b6-922f-cf402efab619 + version = 5 +(dev) 2 27355 [1].....................................:Wed 12 Jul 2017 07:08:26 PM EDT:. +(git)hopa:/tmp/1233[master] +$> git annex init --help +git-annex init - initialize git-annex + +Usage: git-annex init [DESC] [--version VALUE] + +Available options: + --version VALUE Override default annex.version + --force allow actions that may lose annexed data + -F,--fast avoid slow operations + -q,--quiet avoid verbose output + -v,--verbose allow verbose output (default) + -d,--debug show debug messages + --no-debug don't show debug messages + -b,--backend NAME specify key-value backend to use + -N,--numcopies NUMBER override default number of copies + --trust REMOTE override trust setting + --semitrust REMOTE override trust setting back to default + --untrust REMOTE override trust setting to untrusted + -c,--config NAME=VALUE override git configuration setting + --user-agent NAME override default User-Agent + --trust-glacier Trust Amazon Glacier inventory + --notify-finish show desktop notification after transfer finishes + --notify-start show desktop notification after transfer starts + -h,--help Show this help text + +For details, run: git-annex help init +(dev) 2 27356 [1].....................................:Wed 12 Jul 2017 07:08:31 PM EDT:. +(git)hopa:/tmp/1233[master] +$> ls +(dev) 2 27357 [1].....................................:Wed 12 Jul 2017 07:08:39 PM EDT:. +(git)hopa:/tmp/1233[master] +$> ls -lta +total 312 +drwx------ 9 yoh yoh 4096 Jul 12 19:08 .git/ +drwx------ 3 yoh yoh 4096 Jul 12 19:08 ./ +drwxrwxrwt 143 root root 307200 Jul 12 19:08 ../ +(dev) 2 27358 [1].....................................:Wed 12 Jul 2017 07:08:40 PM EDT:. +(git)hopa:/tmp/1233[master] +$> grep -r MD5E . + + +"""]] + +### Have you had any luck using git-annex before? (Sometimes we get tired of reading bug reports all day and a lil' positive end note does wonders) + +kinda diff --git a/doc/bugs/--backend_for_init_is_in_no_effect__63__/comment_1_6b40b5ea277d8e28f0b632701a776587._comment b/doc/bugs/--backend_for_init_is_in_no_effect__63__/comment_1_6b40b5ea277d8e28f0b632701a776587._comment new file mode 100644 index 0000000000..f024c46937 --- /dev/null +++ b/doc/bugs/--backend_for_init_is_in_no_effect__63__/comment_1_6b40b5ea277d8e28f0b632701a776587._comment @@ -0,0 +1,10 @@ +[[!comment format=mdwn + username="joey" + subject="""comment 1""" + date="2017-08-15T16:54:56Z" + content=""" +--backend is a global option, so it appears in every command's help. + +There are several of these global options left over from the previous +option parsing regime. +"""]] diff --git a/doc/bugs/--shared_setting_of_git_causes_annex__39__ed_files_to_be_writeable__33__.mdwn b/doc/bugs/--shared_setting_of_git_causes_annex__39__ed_files_to_be_writeable__33__.mdwn new file mode 100644 index 0000000000..fdf2e509c7 --- /dev/null +++ b/doc/bugs/--shared_setting_of_git_causes_annex__39__ed_files_to_be_writeable__33__.mdwn @@ -0,0 +1,30 @@ +### Please describe the problem. + +Not sure how I have missed this issue for a while, since now I have tried with almost year old 6.20170101-1+b1 -- the same story as with 6.20171003-g14ffdd779 + +I expect those files to be readable according to the 'shared' setting but definitely not writeable! + +### Please provide any additional information below. + +[[!format sh """ +$> sudo rm -rf tttt; mkdir tttt; ( cd tttt; git init --shared .; git annex init ; echo 123 >123; git annex add 123; git commit -m 'added 123'; ls -lL 123; ) +Initialized empty shared Git repository in /tmp/tttt/.git/ +init ok +(recording state in git...) +add 123 ok +(recording state in git...) +[master (root-commit) b137ea6] added 123 + 1 file changed, 1 insertion(+) + create mode 120000 123 +-rw-rw---- 1 yoh yoh 4 Oct 10 12:59 123 + +$> echo 11 >> tttt/123 +# no error code +"""]] + +FWIW my umask is 077 but originally discovered on a system with umask 0002 + +I guess I need to annex fsck tons of repositories now :-/ + +[[!meta author=yoh]] + diff --git a/doc/bugs/--shared_setting_of_git_causes_annex__39__ed_files_to_be_writeable__33__/comment_1_df04a6bd62a60789d8a59cce4b62728e._comment b/doc/bugs/--shared_setting_of_git_causes_annex__39__ed_files_to_be_writeable__33__/comment_1_df04a6bd62a60789d8a59cce4b62728e._comment new file mode 100644 index 0000000000..b6fd7150fd --- /dev/null +++ b/doc/bugs/--shared_setting_of_git_causes_annex__39__ed_files_to_be_writeable__33__/comment_1_df04a6bd62a60789d8a59cce4b62728e._comment @@ -0,0 +1,22 @@ +[[!comment format=mdwn + username="yarikoptic" + avatar="http://cdn.libravatar.org/avatar/f11e9c84cb18d26a1748c33b48c924b4" + subject="annex fsck reverts them back to incorrect!" + date="2017-10-10T17:17:12Z" + content=""" +even if I fix them, and then rerun 'annex fsck' to verify that I am 'ok' -- it would revert to the bad state: + +[[!format sh \"\"\" +$ find -ipath '*.git/annex/objects' | while read d; do chmod a-w -R $d/*/*/*; done +ls -ld ./.git/annex/objects/Kj/ZJ/MD5E-s3500--157855c754113321d3f6621abe0fe71d.tsv +dr-xr-sr-x 2 bids bids 3 Oct 3 08:36 ./.git/annex/objects/Kj/ZJ/MD5E-s3500--157855c754113321d3f6621abe0fe71d.tsv + +$ git annex fsck >/dev/null + +$ ls -ld ./.git/annex/objects/Kj/ZJ/MD5E-s3500--157855c754113321d3f6621abe0fe71d.tsv +drwxrwsr-x 2 bids bids 3 Oct 3 08:36 ./.git/annex/objects/Kj/ZJ/MD5E-s3500--157855c754113321d3f6621abe0fe71d.tsv + +$ ls -ld ./.git/annex/objects/Kj/ZJ/MD5E-s3500--157855c754113321d3f6621abe0fe71d.tsv/* +-rw-rw-rw- 1 bids bids 3500 Oct 3 07:57 ./.git/annex/objects/Kj/ZJ/MD5E-s3500--157855c754113321d3f6621abe0fe71d.tsv/MD5E-s3500--157855c754113321d3f6621abe0fe71d.tsv +\"\"\"]] +"""]] diff --git a/doc/bugs/--shared_setting_of_git_causes_annex__39__ed_files_to_be_writeable__33__/comment_2_0619eec44cddf3d3b1aae27c7bd1389a._comment b/doc/bugs/--shared_setting_of_git_causes_annex__39__ed_files_to_be_writeable__33__/comment_2_0619eec44cddf3d3b1aae27c7bd1389a._comment new file mode 100644 index 0000000000..5ee888e6a8 --- /dev/null +++ b/doc/bugs/--shared_setting_of_git_causes_annex__39__ed_files_to_be_writeable__33__/comment_2_0619eec44cddf3d3b1aae27c7bd1389a._comment @@ -0,0 +1,22 @@ +[[!comment format=mdwn + username="joey" + subject="""comment 2""" + date="2017-10-10T17:23:14Z" + content=""" +The write bit is necessary so that the files can be opened in write mode to +lock them. Normally the write bit is temporarily enabled and then disabled +for locking, but in a shared repository, some other user may own the file, +which prevents the user from changing its permissions. + +Similarly, the parent directory is not made unwritable in a shared +repository, because other users won't be able to temporarily flip the write +bit on when making changes. + +[[!commit 0d432dd1a4f718225c4192d0834a4e0a34b3e4bd]] used the latter as a +rationalle to allow the former. + +I suppose it could use separate lock files from the content file, +as is already done in direct mode. However interaction between different +versions of git-annex with different ideas about locking could result in +`git annex drop` losing data. +"""]] diff --git a/doc/bugs/--shared_setting_of_git_causes_annex__39__ed_files_to_be_writeable__33__/comment_3_aae8dd8b3b0ce4647ec2d30e1d682339._comment b/doc/bugs/--shared_setting_of_git_causes_annex__39__ed_files_to_be_writeable__33__/comment_3_aae8dd8b3b0ce4647ec2d30e1d682339._comment new file mode 100644 index 0000000000..9450cbc496 --- /dev/null +++ b/doc/bugs/--shared_setting_of_git_causes_annex__39__ed_files_to_be_writeable__33__/comment_3_aae8dd8b3b0ce4647ec2d30e1d682339._comment @@ -0,0 +1,22 @@ +[[!comment format=mdwn + username="yarikoptic" + avatar="http://cdn.libravatar.org/avatar/f11e9c84cb18d26a1748c33b48c924b4" + subject="comment 3" + date="2017-10-10T17:40:08Z" + content=""" +Found a following comment in the code +[[!format haskell \"\"\" +{- Normally, blocks writing to an annexed file, and modifies file + - permissions to allow reading it. + - + - When core.sharedRepository is set, the write bits are not removed from + - the file, but instead the appropriate group write bits are set. This is + - necessary to let other users in the group lock the file. But, in a + - shared repository, the current user may not be able to change a file + - owned by another user, so failure to set this mode is ignored. + -} +\"\"\"]] +So may be it is a \"Feature\" although killing the whole premise of data safety while using git-annex. + +In my case, shared permissions are primarily to make files/repositories readable by others, so may be I should have not used 'shared' mode anyways, since reading does not need the shared setting +"""]] diff --git a/doc/bugs/--shared_setting_of_git_causes_annex__39__ed_files_to_be_writeable__33__/comment_4_c72e6646e39cc487ea52cd17925a548f._comment b/doc/bugs/--shared_setting_of_git_causes_annex__39__ed_files_to_be_writeable__33__/comment_4_c72e6646e39cc487ea52cd17925a548f._comment new file mode 100644 index 0000000000..8e7b2ec6a9 --- /dev/null +++ b/doc/bugs/--shared_setting_of_git_causes_annex__39__ed_files_to_be_writeable__33__/comment_4_c72e6646e39cc487ea52cd17925a548f._comment @@ -0,0 +1,8 @@ +[[!comment format=mdwn + username="yarikoptic" + avatar="http://cdn.libravatar.org/avatar/f11e9c84cb18d26a1748c33b48c924b4" + subject="comment 4" + date="2017-10-10T17:56:47Z" + content=""" +our comments crossed in the hyperspace... ;) yeah -- I would have expected a separate lock file to be used for such cases. Now data loss is really a possibility (almost made it happen since opened the file and wrote it without unlocking, good that I caught it and was able to modify back exactly the way it was). Interactions among multiple versions of annex on the same repo is imho a lesser possibility (would require two different versions of annex being available) +"""]] diff --git a/doc/bugs/--shared_setting_of_git_causes_annex__39__ed_files_to_be_writeable__33__/comment_5_d8c8b725ee91a899a131bf505bd9a2e3._comment b/doc/bugs/--shared_setting_of_git_causes_annex__39__ed_files_to_be_writeable__33__/comment_5_d8c8b725ee91a899a131bf505bd9a2e3._comment new file mode 100644 index 0000000000..05ef7377a8 --- /dev/null +++ b/doc/bugs/--shared_setting_of_git_causes_annex__39__ed_files_to_be_writeable__33__/comment_5_d8c8b725ee91a899a131bf505bd9a2e3._comment @@ -0,0 +1,20 @@ +[[!comment format=mdwn + username="joey" + subject="""comment 5""" + date="2017-10-11T15:59:16Z" + content=""" +It's easy to have multiple versions of git-annex installed. Especially +when a repository is shared amoung users, who may install different +versions for their own reasons. + +I am not happy either about this weakening of safety guarantees that indirect +mode repositories otherwise have. It seems at least worth a big fat +warning. + +I don't feel comfortable changing locking without a major repository +version change, which would be a Big Deal. + +Anyhow the file mode and directory mode are both needed to protect annexed +files. Making the file read-only will not prevent a lot of things +renaming over top of it. (Including vim!) +"""]] diff --git a/doc/bugs/--shared_setting_of_git_causes_annex__39__ed_files_to_be_writeable__33__/comment_6_d0d8065b78c22e62ef780c27e4d5952c._comment b/doc/bugs/--shared_setting_of_git_causes_annex__39__ed_files_to_be_writeable__33__/comment_6_d0d8065b78c22e62ef780c27e4d5952c._comment new file mode 100644 index 0000000000..b2958d0c0f --- /dev/null +++ b/doc/bugs/--shared_setting_of_git_causes_annex__39__ed_files_to_be_writeable__33__/comment_6_d0d8065b78c22e62ef780c27e4d5952c._comment @@ -0,0 +1,10 @@ +[[!comment format=mdwn + username="yarikoptic" + avatar="http://cdn.libravatar.org/avatar/f11e9c84cb18d26a1748c33b48c924b4" + subject="comment 6" + date="2017-12-05T18:20:41Z" + content=""" +> Making the file read-only will not prevent a lot of things renaming over top of it. (Including vim!) + +not exactly... removing write permission from the key file at least makes vim report that the file read-only and disallowing (without force) to overwrite it. That is my use case where I keep burning myself constantly (e.g. this morning again). Need to not forget to switch all \"shared\" repos back to the single user mode. One of the promises of git-annex is that data is safe, and this mode just violates it wildly. +"""]] diff --git a/doc/bugs/20151116_tests_fail_on_OS_X.mdwn b/doc/bugs/20151116_tests_fail_on_OS_X.mdwn new file mode 100644 index 0000000000..fcd32f9dc9 --- /dev/null +++ b/doc/bugs/20151116_tests_fail_on_OS_X.mdwn @@ -0,0 +1,40 @@ +Test output on OS X. Btw, the previous version works: + +``` +OK (4.87s) + map: OK (0.33s) + uninit: Deleted branch git-annex (was 9a28b38). +OK (0.38s) + uninit (in git-annex branch): Switched to branch 'git-annex' +OK (0.26s) + upgrade: OK (0.21s) + whereis: OK (0.61s) + hook remote: OK (0.91s) + directory remote: OK (0.85s) + rsync remote: OK (1.40s) + bup remote: Reinitialized existing Git repository in /private/var/folders/4j/br7bdhjx4b384_snb2087gt00000gn/T/nix-build-git-annex-5.20151116.drv-0/.bup/ +Initialized empty Git repository in /private/var/folders/4j/br7bdhjx4b384_snb2087gt00000gn/T/nix-build-git-annex-5.20151116.drv-0/git-annex-5.20151116/.t/tmprepo0/dir/ + content cannot be completely removed from bup remote +OK (2.62s) + crypto: OK (4.16s) + preferred content: OK (1.97s) + add subdirs: Merge made by the 'recursive' strategy. + conflictor.variant-cc12 | 1 + + conflictor/subfile | 1 + + 2 files changed, 2 insertions(+) + create mode 120000 conflictor.variant-cc12 + create mode 120000 conflictor/subfile +To /private/var/folders/4j/br7bdhjx4b384_snb2087gt00000gn/T/nix-build-git-annex-5.20151116.drv-0/git-annex-5.20151116/.t/repo + e4d3d14..60148c4 git-annex -> synced/git-annex + efda76d..f76baae master -> synced/master +OK (0.55s) + addurl: git-annex: .git/annex/objects/2Q/J8/SHA256E-s3--2c26b46b68ffc68ff99b453c1d30413413422d706483bfa0f98a5e886266e7ae/SHA256E-s3--2c26b46b68ffc68ff99b453c1d30413413422d706483bfa0f98a5e886266e7ae: createSymbolicLink: invalid argument (File name too long) +FAIL (0.29s) + addurl failed on file:///private/var/folders/4j/br7bdhjx4b384_snb2087gt00000gn/T/nix-build-git-annex-5.20151116.drv-0/git-annex-5.20151116/.t/tmprepo0/myurl + +2 out of 150 tests failed (126.13s) + (This could be due to a bug in git-annex, or an incompatibility + with utilities, such as git, installed on this system.) +``` + +> [[fixed|done]] --[[Joey]] diff --git a/doc/bugs/20151116_tests_fail_on_OS_X/comment_1_6c5cfe02b686d063e8f336620a81cca1._comment b/doc/bugs/20151116_tests_fail_on_OS_X/comment_1_6c5cfe02b686d063e8f336620a81cca1._comment new file mode 100644 index 0000000000..c3bc0f0686 --- /dev/null +++ b/doc/bugs/20151116_tests_fail_on_OS_X/comment_1_6c5cfe02b686d063e8f336620a81cca1._comment @@ -0,0 +1,15 @@ +[[!comment format=mdwn + username="joey" + subject="""comment 1""" + date="2015-11-30T19:22:43Z" + content=""" +You may have omitted 1 of the 2 failures in your paste. + +The only one I see has something to do with a filename being too long. +Since the filename in question is 77 bytes long, or 178 with the full path, +and industry standard for filename size is 1024 or longer, I am at a loss +why that would be considered too long. + +AFAIK there have been no changes in git-annex's filenames recently. What +was the previous version that worked? +"""]] diff --git a/doc/bugs/20151116_tests_fail_on_OS_X/comment_2_59f7803b9a9ee939a8a50605fe5c4682._comment b/doc/bugs/20151116_tests_fail_on_OS_X/comment_2_59f7803b9a9ee939a8a50605fe5c4682._comment new file mode 100644 index 0000000000..01da6e8244 --- /dev/null +++ b/doc/bugs/20151116_tests_fail_on_OS_X/comment_2_59f7803b9a9ee939a8a50605fe5c4682._comment @@ -0,0 +1,12 @@ +[[!comment format=mdwn + username="joey" + subject="""comment 2""" + date="2015-12-06T20:02:49Z" + content=""" +Ok, this involves where the test suite is run. +The addurl test adds `file://`pwd`/somefile`, and that will create a file +with a name like `_home_you_sub_dir_path_here_somefile`. Which can easily +exceed filename length limits of 255 bytes. + +There was indeed a reversion in addurl's handling of that. +"""]] diff --git a/doc/bugs/2_ssh_connection_prompts_for_password.mdwn b/doc/bugs/2_ssh_connection_prompts_for_password.mdwn new file mode 100644 index 0000000000..715c7c83b3 --- /dev/null +++ b/doc/bugs/2_ssh_connection_prompts_for_password.mdwn @@ -0,0 +1,64 @@ +### Please describe the problem. + +There was some recent work to ["centralize" such prompts](https://git-annex.branchable.com/devblog/day_457__improved_ssh_password_prompting/) but it seems some are still "leaking through" multiple times. May be it is because there are 2 available repos on that remote host, so annex generates one per each of those? (although it knows only about origin) + +### What version of git-annex are you using? On what operating system? + +6.20170810+gitgff6f9e203-1~ndall+1 + +### Please provide any additional information below. + +[[!format sh """ + +$> git annex get -J5 . +get R042/R042-2013-08-16/R042-2013-08-16-CSC01a.ncs get R042/R042-2013-08-16/R042-2013-08-16-CSC02a.ncs get R042/R042-2013-08-16/R042-2013-08-16-CSC03a.ncs get R042/R042-2013-08-16/R042-2013-08-16-CSC05a.ncs get R042/R042-2013-08-16/R042-2013-08-16-CSC04a.ncs (from datalad-archives...) +(from datalad-archives...) (from datalad-archives...) + +(from datalad-archives...) +(from datalad-archives...) +[ERROR] Failed to run ['git-annex', 'get', '--key', 'MD5E-s237624713--dbdc4079b005b8b7f1549e00647b36d6.zip'] under '/mnt/btrfs/datasets/datalad/crawl/workshops/mind-2017/MotivationalT'. Exit code=1. out=get MD5E-s237624713--dbdc4079b005b8b7f1549e00647b36d6.zip (transfer already in progress, or unable to take transfer lock) +| Unable to access these remotes: origin +| +| Try making some of these repositories available: +| aaa0bc14-51fc-45c8-81c2-76dff067755b -- mvdm@atlantis.hpcc.dartmouth.edu:~/data/mind-2017/MotivationalT_ +| f7f97046-ea49-4af1-9f5a-8475a5ea1e0a -- yhalchen@atlantis.hpcc.dartmouth.edu:~/mind-2017/MotivationalT [origin] +| failed +| err=git-annex: get: 1 failed +| +[ERROR] Failed to run ['git-annex', 'get', '--key', 'MD5E-s237624713--dbdc4079b005b8b7f1549e00647b36d6.zip'] under '/mnt/btrfs/datasets/datalad/crawl/workshops/mind-2017/MotivationalT'. Exit code=1. out=get MD5E-s237624713--dbdc4079b005b8b7f1549e00647b36d6.zip (transfer already in progress, or unable to take transfer lock) +| Unable to access these remotes: origin +| +| Try making some of these repositories available: +| aaa0bc14-51fc-45c8-81c2-76dff067755b -- mvdm@atlantis.hpcc.dartmouth.edu:~/data/mind-2017/MotivationalT_ +| f7f97046-ea49-4af1-9f5a-8475a5ea1e0a -- yhalchen@atlantis.hpcc.dartmouth.edu:~/mind-2017/MotivationalT [origin] +| failed +| err=git-annex: get: 1 failed +| +[ERROR] Failed to fetch any archive containing SHA256E-s17136940--0501aab6b4d1ce0565921728bc92ef74f81edf0d7bcd5a77946ca58f977f2537.ncs. Tried: ['MD5E-s237624713--dbdc4079b005b8b7f1549e00647b36d6.zip'] +[ERROR] Failed to fetch any archive containing SHA256E-s17136940--8b3b08310db20ca7e3e784a21f935a78f8669efdf1396168596411f1e355e43b.ncs. Tried: ['MD5E-s237624713--dbdc4079b005b8b7f1549e00647b36d6.zip'] +(from origin...) (from origin...) [ERROR] Failed to run ['git-annex', 'get', '--key', 'MD5E-s237624713--dbdc4079b005b8b7f1549e00647b36d6.zip'] under '/mnt/btrfs/datasets/datalad/crawl/workshops/mind-2017/MotivationalT'. Exit code=1. out=get MD5E-s237624713--dbdc4079b005b8b7f1549e00647b36d6.zip (transfer already in progress, or unable to take transfer lock) +| Unable to access these remotes: origin +| +| Try making some of these repositories available: +| aaa0bc14-51fc-45c8-81c2-76dff067755b -- mvdm@atlantis.hpcc.dartmouth.edu:~/data/mind-2017/MotivationalT_ +| f7f97046-ea49-4af1-9f5a-8475a5ea1e0a -- yhalchen@atlantis.hpcc.dartmouth.edu:~/mind-2017/MotivationalT [origin] +| failed +| err=git-annex: get: 1 failed +| +[ERROR] Failed to fetch any archive containing SHA256E-s17136940--08ce5a67c7fc09f02b994a3987812a75727eaf51f3e70fa7e1030dae934f9fbc.ncs. Tried: ['MD5E-s237624713--dbdc4079b005b8b7f1549e00647b36d6.zip'] +(from origin...) [ERROR] Failed to run ['git-annex', 'get', '--key', 'MD5E-s237624713--dbdc4079b005b8b7f1549e00647b36d6.zip'] under '/mnt/btrfs/datasets/datalad/crawl/workshops/mind-2017/MotivationalT'. Exit code=1. out=get MD5E-s237624713--dbdc4079b005b8b7f1549e00647b36d6.zip (transfer already in progress, or unable to take transfer lock) +| Unable to access these remotes: origin +| +| Try making some of these repositories available: +| aaa0bc14-51fc-45c8-81c2-76dff067755b -- mvdm@atlantis.hpcc.dartmouth.edu:~/data/mind-2017/MotivationalT_ +| f7f97046-ea49-4af1-9f5a-8475a5ea1e0a -- yhalchen@atlantis.hpcc.dartmouth.edu:~/mind-2017/MotivationalT [origin] +| failed +| err=git-annex: get: 1 failed +| +[ERROR] Failed to fetch any archive containing SHA256E-s17136940--bc145f07c79584181cad3763a763a2ea047282bd41153d20a63d85a44fb27a7f.ncs. Tried: ['MD5E-s237624713--dbdc4079b005b8b7f1549e00647b36d6.zip'] +(from origin...) yhalchen@discovery.dartmouth.edu's password: yhalchen@discovery.dartmouth.edu's password: + +"""]] + +[[!meta author=yoh]] + diff --git a/doc/bugs/2_ssh_connection_prompts_for_password/comment_1_f78784304b46b77d334f74c827d0889b._comment b/doc/bugs/2_ssh_connection_prompts_for_password/comment_1_f78784304b46b77d334f74c827d0889b._comment new file mode 100644 index 0000000000..8602847694 --- /dev/null +++ b/doc/bugs/2_ssh_connection_prompts_for_password/comment_1_f78784304b46b77d334f74c827d0889b._comment @@ -0,0 +1,34 @@ +[[!comment format=mdwn + username="yarikoptic" + avatar="http://cdn.libravatar.org/avatar/f11e9c84cb18d26a1748c33b48c924b4" + subject="comment 1" + date="2017-08-12T04:08:04Z" + content=""" +ha -- if I do specify `--from=origin` -- only 1 prompt + +[[!format sh \"\"\" +$> git annex get -J5 --from=origin . +get R042/R042-2013-08-16/R042-2013-08-16-CSC01a.ncs (from origin...) get R042/R042-2013-08-16/R042-2013-08-16-CSC03a.ncs (from origin...) get R042/R042-2013-08-16/R042-2013-08-16-CSC02a.ncs (from origin...) get R042/R042-2013-08-16/R042-2013-08-16-CSC04a.ncs (from origin...) get R042/R042-2013-08-16/R042-2013-08-16-CSC05a.ncs (from origin...) yhalchen@discovery.dartmouth.edu's password: +git-annex-shell: git: createProcess: runInteractiveProcess: chdir: does not exist (No such file or directory) + + Unable to run git-annex-shell on remote . + +git-annex-shell: git: createProcess: runInteractiveProcess: chdir: does not exist (No such file or directory) + + Unable to run git-annex-shell on remote . + +git-annex-shell: git: createProcess: runInteractiveProcess: chdir: does not exist (No such file or directory) + + Unable to run git-annex-shell on remote . + +git-annex-shell: git: createProcess: runInteractiveProcess: chdir: does not exist (No such file or directory) + + Unable to run git-annex-shell on remote . + +SHA256E-s17136940--08ce5a67c7fc09f02b994a3987812a75727eaf51f3e70fa7e1030dae934f9fbc.ncs + 0 0% 0.00kB/s 0:00:00 SHA256E-s17136940--bc145f07c79584181cad3763a763a2ea047282bd41153d20a63d85a44fb27a7f.ncs + 0 0% 0.00kB/s 0:00:00 SHA256E-s17136940--c3a8af948c77a2df422eae50807a6e7e6e5db7a3451a562bca529d3f1a1a234f.ncs + 0 0% 0.00kB/s 0:00:00 git-annex-shell: git: createProcess: runInteractiveProcess: chdir: does not exist (No such file or directory) + +\"\"\"]] +"""]] diff --git a/doc/bugs/2_ssh_connection_prompts_for_password/comment_2_62a5cc7cd6b8b06850fb32b76121ad82._comment b/doc/bugs/2_ssh_connection_prompts_for_password/comment_2_62a5cc7cd6b8b06850fb32b76121ad82._comment new file mode 100644 index 0000000000..f7f3f5da58 --- /dev/null +++ b/doc/bugs/2_ssh_connection_prompts_for_password/comment_2_62a5cc7cd6b8b06850fb32b76121ad82._comment @@ -0,0 +1,25 @@ +[[!comment format=mdwn + username="joey" + subject="""comment 2""" + date="2017-08-15T16:43:07Z" + content=""" +The improvements around ssh password prompting require ssh connection +caching to work. If a ssh connection fails because the wrong password is +entered or because there's no usable tty or whatever, there's no cached +ssh connection to reuse, so the next attempt to access that host will +result in another password prompt. + +Also, datalad does not seem to be running git-annex with -J. So it *can't* +be trying to make two ssh connection at the same time. My recent work on +ssh password prompting was mostly to fix cases where git-annex is run with +-J. + +It's also possible that some ssh configuration that I don't know of could +make ssh password prompt even when git-annex is running it with +`BatchMode=true` to avoid password prompts (in order to test if the ssh +connection is already up). That would then result in two ssh password +prompts, one after the other, which seems to match your transcript. + +If you have only one remote, specifying `--from=origin` won't change +anything. Entering the right password would change something there though.. +"""]] diff --git a/doc/bugs/8_unit_tests_fail.mdwn b/doc/bugs/8_unit_tests_fail.mdwn new file mode 100644 index 0000000000..9bc53c9564 --- /dev/null +++ b/doc/bugs/8_unit_tests_fail.mdwn @@ -0,0 +1,29 @@ +### Please describe the problem. + +8 out of 293 tests failed (200.39s) + (This could be due to a bug in git-annex, or an incompatibility + with utilities, such as git, installed on this system.) + +full log https://headcounter.org/hydra/build/1819294/nixlog/1 + +### What steps will reproduce the problem? + +./make test + + +### What version of git-annex are you using? On what operating system? + +6.20170510 + +nix-build and inside a nix shell with manual building. + + +### Please provide any additional information below. + + + +### Have you had any luck using git-annex before? (Sometimes we get tired of reading bug reports all day and a lil' positive end note does wonders) + +Yep, works apart from the few tests that fail. + +> Duplicate of [[git-annex_in_nixpkgs_fails_with_git-2.13.0]]; closing. [[done]] --[[Joey]] diff --git a/doc/bugs/Aborts_with_SQLite_error_when_dropping_contents.mdwn b/doc/bugs/Aborts_with_SQLite_error_when_dropping_contents.mdwn new file mode 100644 index 0000000000..de6c26fbfe --- /dev/null +++ b/doc/bugs/Aborts_with_SQLite_error_when_dropping_contents.mdwn @@ -0,0 +1,86 @@ +(Found the problem, so you might want to skip to the comments to see +what's wrong.) + +### Please describe the problem. + +Trying to drop the contents of 131 files in a subdirectory, but it dies +with an "sqlite worker thread crashed" error. The contents of one file +is deleted, and then it dies. The files in that directory are exactly 1G +(1000000000) bytes each. + +There's no data loss here, it creates files in `.git/annex/journal/` +which is added to the git-annex branch after a git annex sync. "git +annex fsck --fast --quiet" is happy and finds no errors. + +### What steps will reproduce the problem? + +"git annex drop" with and without --force. Tried to copy one of the +files to another server with Debian 8.7, and it's successfully dropped +there. Same git-annex version. + +### What version of git-annex are you using? On what operating system? + +OS: Ubuntu 14.04.5 LTS + +git-annex: 6.20161211-gc3ab3c668 (from +`git-annex-standalone-amd64.tar.gz` in the annex at +downloads.kitenet.net) + +I have compiled the newest sqlite3 3.16.2 and placed it in +/usr/local/bin/, just mentioning it because it's some kind of SQLite +error. but executing the command with a PATH without /usr/local/bin +makes no difference. + +### Please provide any additional information below. + +# If you can, paste a complete transcript of the problem occurring here. +# If the problem is with the git-annex assistant, paste in .git/annex/daemon.log + + $ git annex drop -d --force + [2017-01-16 08:58:39.947856861] read: git ["--git-dir=../../../../../../../.git","--work-tree=../../../../../../..","--literal-pathspecs","ls-files","--cached","-z","--"] + [2017-01-16 08:58:39.953150944] chat: git ["--git-dir=../../../../../../../.git","--work-tree=../../../../../../..","--literal-pathspecs","check-attr","-z","--stdin","annex.backend","annex.numcopies","annex.largefiles","--"] + [2017-01-16 08:58:39.953742873] read: git ["--version"] + [2017-01-16 08:58:39.959438053] process done ExitSuccess + [2017-01-16 08:58:39.959919735] read: git ["--git-dir=../../../../../../../.git","--work-tree=../../../../../../..","--literal-pathspecs","show-ref","git-annex"] + [2017-01-16 08:58:39.963707897] process done ExitSuccess + [2017-01-16 08:58:39.963882344] read: git ["--git-dir=../../../../../../../.git","--work-tree=../../../../../../..","--literal-pathspecs","show-ref","--hash","refs/heads/git-annex"] + [2017-01-16 08:58:39.967078602] process done ExitSuccess + [2017-01-16 08:58:39.969062554] chat: git ["--git-dir=../../../../../../../.git","--work-tree=../../../../../../..","--literal-pathspecs","cat-file","--batch"] + [2017-01-16 08:58:39.970168767] chat: git ["--git-dir=../../../../../../../.git","--work-tree=../../../../../../..","--literal-pathspecs","cat-file","--batch-check=%(objectname) %(objecttype) %(objectsize)"] + drop MyStream_3/MyStream_3.mp4.split_ag [2017-01-16 08:58:39.984359273] Dropping from here proof: Nothing + sqlite worker thread crashed: SQLite3 returned ErrorCan'tOpen while attempting to perform prepare "SELECT null from content limit 1": unable to open database file + ok + git-annex: thread blocked indefinitely in an MVar operation + $ git annex drop -d --force + [2017-01-16 08:58:58.890031175] read: git ["--git-dir=../../../../../../../.git","--work-tree=../../../../../../..","--literal-pathspecs","ls-files","--cached","-z","--"] + [2017-01-16 08:58:58.895047131] chat: git ["--git-dir=../../../../../../../.git","--work-tree=../../../../../../..","--literal-pathspecs","check-attr","-z","--stdin","annex.backend","annex.numcopies","annex.largefiles","--"] + [2017-01-16 08:58:58.895690334] read: git ["--version"] + [2017-01-16 08:58:58.901206493] process done ExitSuccess + [2017-01-16 08:58:58.901836453] read: git ["--git-dir=../../../../../../../.git","--work-tree=../../../../../../..","--literal-pathspecs","show-ref","git-annex"] + [2017-01-16 08:58:58.905632574] process done ExitSuccess + [2017-01-16 08:58:58.905793223] read: git ["--git-dir=../../../../../../../.git","--work-tree=../../../../../../..","--literal-pathspecs","show-ref","--hash","refs/heads/git-annex"] + [2017-01-16 08:58:58.909227509] process done ExitSuccess + [2017-01-16 08:58:58.910902084] chat: git ["--git-dir=../../../../../../../.git","--work-tree=../../../../../../..","--literal-pathspecs","cat-file","--batch"] + [2017-01-16 08:58:58.912164764] chat: git ["--git-dir=../../../../../../../.git","--work-tree=../../../../../../..","--literal-pathspecs","cat-file","--batch-check=%(objectname) %(objecttype) %(objectsize)"] + drop MyStream_3/MyStream_3.mp4.split_ah [2017-01-16 08:58:58.927477693] Dropping from here proof: Nothing + sqlite worker thread crashed: SQLite3 returned ErrorCan'tOpen while attempting to perform prepare "SELECT null from content limit 1": unable to open database file + ok + git-annex: thread blocked indefinitely in an MVar operation + $ + +# End of transcript or log. + +### Have you had any luck using git-annex before? (Sometimes we get tired of reading bug reports all day and a lil' positive end note does wonders) + +Indeed. All my stuff (around 3.5 terabytes) is stored in git-annex with +at least three copies of each file on different disks and locations, +spread over various hard disks, memory sticks, servers and you name it. +Unused disk space is a waste, so I fill everything up to the brim with +extra copies. + +In other words, Git-Annex and I are very happy together, and I'd like to +marry it. And because you are the father, I hereby respectfully ask for +your blessing. + +> [[fixed|done]] (and I suppose you have my blessing, but I'm not sure +> that's legal yet!) --[[Joey]] diff --git a/doc/bugs/Aborts_with_SQLite_error_when_dropping_contents/comment_1_cd09e4d4a73e003735e3297922b8faa8._comment b/doc/bugs/Aborts_with_SQLite_error_when_dropping_contents/comment_1_cd09e4d4a73e003735e3297922b8faa8._comment new file mode 100644 index 0000000000..7c9caf052c --- /dev/null +++ b/doc/bugs/Aborts_with_SQLite_error_when_dropping_contents/comment_1_cd09e4d4a73e003735e3297922b8faa8._comment @@ -0,0 +1,8 @@ +[[!comment format=mdwn + username="sunny256" + avatar="http://cdn.libravatar.org/avatar/8a221001f74d0e8f4dadee3c7d1996e4" + subject="Found the reason" + date="2017-01-16T11:03:45Z" + content=""" +I managed to find the reason why it failed. The files in .git/annex/keys/ had wrong permissions, so I wasn't able to write to them. This is a shared repository where everyone within the Unix group can read and write. All the directories including .git/annex/keys have chmod +s so any additions or edits are stored within the correct group. But the SQLite files were stored with permission 0644 instead of 0664, and my umask in the shell is 0002. Maybe SQLite is creating the database with wrong permissions? +"""]] diff --git a/doc/bugs/Aborts_with_SQLite_error_when_dropping_contents/comment_2_9a0fec331a858746e2fecc6b86ab8cbb._comment b/doc/bugs/Aborts_with_SQLite_error_when_dropping_contents/comment_2_9a0fec331a858746e2fecc6b86ab8cbb._comment new file mode 100644 index 0000000000..b3e5b8a02e --- /dev/null +++ b/doc/bugs/Aborts_with_SQLite_error_when_dropping_contents/comment_2_9a0fec331a858746e2fecc6b86ab8cbb._comment @@ -0,0 +1,8 @@ +[[!comment format=mdwn + username="sunny256" + avatar="http://cdn.libravatar.org/avatar/8a221001f74d0e8f4dadee3c7d1996e4" + subject="Wrong permissions" + date="2017-01-16T11:17:31Z" + content=""" +Yes, confirmed. I tried to create a new test repository and chmod all directories with +s. I'm using umask 0002, so new files are created with perm 0664 and the correct shared group. But the SQLite files in .git/annex/keys/ are created with permission 0644, so other users in the group are not able to update the files there. +"""]] diff --git a/doc/bugs/Aborts_with_SQLite_error_when_dropping_contents/comment_3_88a09558f2da0a5733aa3dd042f9d59b._comment b/doc/bugs/Aborts_with_SQLite_error_when_dropping_contents/comment_3_88a09558f2da0a5733aa3dd042f9d59b._comment new file mode 100644 index 0000000000..30e108b341 --- /dev/null +++ b/doc/bugs/Aborts_with_SQLite_error_when_dropping_contents/comment_3_88a09558f2da0a5733aa3dd042f9d59b._comment @@ -0,0 +1,53 @@ +[[!comment format=mdwn + username="joey" + subject="""comment 3""" + date="2017-02-13T20:21:09Z" + content=""" +Thanks for following up with the cause of this. + +In fact, assuming you're not using a v6 git-annex repository, it doesn't +really need to update that database at all. But since we'll be upgrading to +v6 eventually, I need to deal with problems like this. Also, +this same problem will also impact the database used for incremental fsck. + +I can reproduce this with a v5 repository; dropping a file happens to run a +code path that updates the database. And reproducing it w/o using git-annex too: + + joey@darkstar:~/tmp>mkdir empty + joey@darkstar:~/tmp>umask + 0002 + joey@darkstar:~/tmp>touch empty/file + joey@darkstar:~/tmp>sqlite3 empty/db + SQLite version 3.16.2 2017-01-06 16:32:41 + Enter ".help" for usage hints. + sqlite> create table foo; + Error: near ";": syntax error + sqlite> + joey@darkstar:~/tmp>ls -l empty/ + total 0 + -rw-r--r-- 1 joey joey 0 Feb 13 16:33 db + -rw-rw-r-- 1 joey joey 0 Feb 13 16:32 file + +Seems that sqlite uses `0644 & umask` for the db permissions, +which is *bad* because it doesn't allow the umask to enable the group +write bit. That 0644 is `SQLITE_DEFAULT_FILE_PERMISSIONS`, so it can +be changed to something saner at compile time. + +`http://www.sqlite.org/src/doc/trunk/src/os_unix.c` has a useful comment. +Seems that sqlite is careful to make -wal, -journal, and -shm files +have the exact same permissions as main database file. + +So, if `.git/annex/keys/*` is updated to have the desired permissions when +the database is created, every further write to the database will keep +using the desired permissions. + +Hmm, it turns out that git-annex does already set the database permissions when +creating it, but only if core.sharedRepository is set to group or all. So +there's a workaround; just `git config core.sharedRepository group` when +setting up a repository that's going to be accessed by multiple users. Almost +certianly a better idea than relying on umask anyway; people mess up umask +settings. + +I'll go ahead and make it always set sane permissions when creating databases. +I'm not going to try to fix up permissions in existing repositories though. +"""]] diff --git a/doc/bugs/Add_custom-setup_stanza_to_.cabal_file.mdwn b/doc/bugs/Add_custom-setup_stanza_to_.cabal_file.mdwn new file mode 100644 index 0000000000..fcf9b6b3ac --- /dev/null +++ b/doc/bugs/Add_custom-setup_stanza_to_.cabal_file.mdwn @@ -0,0 +1,49 @@ +### Please describe the problem. + +git-annex uses a custom setup script but does not have a custom-setup stanza. This causes +git-annex to be unbuildable with `cabal new-build` and I'm told it's also unbuildable with +the old `cabal build` under certain circumstances. + +### What steps will reproduce the problem? + +Try building git-annex with `cabal new-build`. + +### What version of git-annex are you using? On what operating system? + +git-annex-6.20171214 on Debian GNU/Linux 9. + +### Please provide any additional information below. + +[[!format sh """ +# If you can, paste a complete transcript of the problem occurring here. +# If the problem is with the git-annex assistant, paste in .git/annex/daemon.log + +$ cabal new-build --constraint="any.git-annex -testsuite" +Resolving dependencies... +In order, the following will be built (use -v for more details): + - git-annex-6.20171214 (exe:git-annex) -testsuite -dbus -concurrentoutput (requires build) + - dummypkg-0 (lib) (first run) +Configuring git-annex-6.20171214 (all, legacy fallback)... +cabal: Failed to build git-annex-6.20171214 (which is required by dummypkg-0). +The failure occurred during the configure step. The exception was: +dieVerbatim: user error (cabal: '/usr/bin/ghc' exited with an error: + +/home/matthew/dummypkg/dist-newstyle/tmp/src-3285/git-annex-6.20171214/Utility/FileSize.hs:10:1: +error: +Failed to load interface for ‘System.PosixCompat.Files’ +It is a member of the hidden package ‘unix-compat-0.5.0.1’. +Perhaps you need to add ‘unix-compat’ to the build-depends in your .cabal +file. +Use -v to see a list of the files searched for. +) + +# End of transcript or log. +"""]] + +### Have you had any luck using git-annex before? (Sometimes we get tired of reading bug reports all day and a lil' positive end note does wonders) + +Yeah, it's amazing! I've been using the version from the Debian repos and then +wanted to try building the new version for youtube-dl support. + +> Revisited it and seem to have managed to add custom-setup back. [[done]] +> --[[Joey]] diff --git a/doc/bugs/Add_custom-setup_stanza_to_.cabal_file/comment_1_88d5109a342e6a06d2f9f400cf850e3d._comment b/doc/bugs/Add_custom-setup_stanza_to_.cabal_file/comment_1_88d5109a342e6a06d2f9f400cf850e3d._comment new file mode 100644 index 0000000000..ccad24c3da --- /dev/null +++ b/doc/bugs/Add_custom-setup_stanza_to_.cabal_file/comment_1_88d5109a342e6a06d2f9f400cf850e3d._comment @@ -0,0 +1,9 @@ +[[!comment format=mdwn + username="mbekkema97@66b135681014f005a3a14c4011d148fcb6655f81" + nickname="mbekkema97" + avatar="http://cdn.libravatar.org/avatar/a5037b8a5914bd9f803af7b7e881d632" + subject="I just found out you're already aware of this" + date="2017-12-15T07:16:13Z" + content=""" +https://github.com/haskell/cabal/issues/4852 +"""]] diff --git a/doc/bugs/Add_day_to_metadata..mdwn b/doc/bugs/Add_day_to_metadata..mdwn new file mode 100644 index 0000000000..24188a4bcb --- /dev/null +++ b/doc/bugs/Add_day_to_metadata..mdwn @@ -0,0 +1,15 @@ +### Please describe the problem. + +The metadata doesn't include the day for RSS feeds, which is probably only a problem for me as I've built something on top of the metadata to provide an interface for selecting podcasts. + +### Please provide any additional information below. + +The attached patch adds the day field to the metadata. + +There's also a tiny bit extra for stack.yaml which will only affect people who have nix enabled in Stack and will make the build successful for those people. + +### Have you had any luck using git-annex before? (Sometimes we get tired of reading bug reports all day and a lil' positive end note does wonders) + +git-annex is awesome, I lean on it heavily nearly every single day. + +> [[merged|done]]. Thanks for the patch! --[[Joey]] diff --git a/doc/bugs/Add_day_to_metadata./comment_1_d46d9f085b7077cc95d71628e45c231d._comment b/doc/bugs/Add_day_to_metadata./comment_1_d46d9f085b7077cc95d71628e45c231d._comment new file mode 100644 index 0000000000..ec344a2d6e --- /dev/null +++ b/doc/bugs/Add_day_to_metadata./comment_1_d46d9f085b7077cc95d71628e45c231d._comment @@ -0,0 +1,64 @@ +[[!comment format=mdwn + username="seantparsons" + avatar="http://cdn.libravatar.org/avatar/616fb81847630239dd1ab099138cb685" + subject="Since the attachment doesn't appear to be there, here's the content." + date="2017-10-22T20:32:21Z" + content=""" + diff --git a/Annex/MetaData.hs b/Annex/MetaData.hs + index e22ed05a6..355c5124a 100644 + --- a/Annex/MetaData.hs + +++ b/Annex/MetaData.hs + @@ -60,10 +60,11 @@ dateMetaData :: UTCTime -> MetaData -> MetaData + dateMetaData mtime old = MetaData $ M.fromList $ filter isnew + [ (yearMetaField, S.singleton $ toMetaValue $ show y) + , (monthMetaField, S.singleton $ toMetaValue $ show m) + + , (dayMetaField, S.singleton $ toMetaValue $ show d) + ] + where + isnew (f, _) = S.null (currentMetaDataValues f old) + - (y, m, _d) = toGregorian $ utctDay mtime + + (y, m, d) = toGregorian $ utctDay mtime + + {- Parses field=value, field+=value, field-=value, field?=value -} + parseModMeta :: String -> Either String ModMeta + diff --git a/Annex/MetaData/StandardFields.hs b/Annex/MetaData/StandardFields.hs + index c91b53930..b9ea47e2f 100644 + --- a/Annex/MetaData/StandardFields.hs + +++ b/Annex/MetaData/StandardFields.hs + @@ -9,6 +9,7 @@ module Annex.MetaData.StandardFields ( + tagMetaField, + yearMetaField, + monthMetaField, + + dayMetaField, + lastChangedField, + mkLastChangedField, + isLastChangedField + @@ -27,6 +28,9 @@ yearMetaField = mkMetaFieldUnchecked \"year\" + monthMetaField :: MetaField + monthMetaField = mkMetaFieldUnchecked \"month\" + + +dayMetaField :: MetaField + +dayMetaField = mkMetaFieldUnchecked \"day\" + + + lastChangedField :: MetaField + lastChangedField = mkMetaFieldUnchecked lastchanged + + diff --git a/stack.yaml b/stack.yaml + index d84c4682e..ac601200e 100644 + --- a/stack.yaml + +++ b/stack.yaml + @@ -24,3 +24,11 @@ extra-deps: + explicit-setup-deps: + git-annex: true + resolver: lts-9.9 + +nix: + + packages: + + - ncurses + + - icu + + - libcxx + + - gcc + + - zlib + + - rsync + \ No newline at end of file + +"""]] diff --git a/doc/bugs/Add_day_to_metadata./comment_2_ea15c799c5abfc97eaa20424c15b73a4._comment b/doc/bugs/Add_day_to_metadata./comment_2_ea15c799c5abfc97eaa20424c15b73a4._comment new file mode 100644 index 0000000000..73a5b56063 --- /dev/null +++ b/doc/bugs/Add_day_to_metadata./comment_2_ea15c799c5abfc97eaa20424c15b73a4._comment @@ -0,0 +1,12 @@ +[[!comment format=mdwn + username="joey" + subject="""comment 2""" + date="2017-11-08T18:51:20Z" + content=""" +Unfortunately the added nix section broke the i386-ancient build, +which uses stack 1.0.4. That version of stack complains: + +Executable named nix-shell not found on path: ["/usr/local/sbin","/usr/local/bin","/sbin","/bin","/usr/sbin","/usr/bin"] + +So, I've reverted the addition of that section. +"""]] diff --git a/doc/bugs/Add_explicit_retries_for_curl__47__wget_on___34__transient__34___errors__63__.mdwn b/doc/bugs/Add_explicit_retries_for_curl__47__wget_on___34__transient__34___errors__63__.mdwn new file mode 100644 index 0000000000..bb6ba69f51 --- /dev/null +++ b/doc/bugs/Add_explicit_retries_for_curl__47__wget_on___34__transient__34___errors__63__.mdwn @@ -0,0 +1,29 @@ +### Please describe the problem. + +While download content from dropbox (via regular urls associated with the files, so "web" remote), some fail some time with error code returned 500 (internal server failure without explicit reason), which causes git-annex get also to fail if that was the only source for the file to try. + +### What steps will reproduce the problem? + +This is all happening with http://datasets.datalad.org/workshops/nih-2017/ds000114/derivatives/freesurfer/ repository (on eg https://dl.dropboxusercontent.com/s/sn4et1e3d2run9g/rh.aparc.dktatlas.annot?dl=0 url ), but for testing wget/curl I just created http://www.onerussian.com/tmp/errors/500 which would always return 500. + +### What version of git-annex are you using? On what operating system? + +6.20180316+gitg308f3ecf6 + +### Please provide any additional information below. + +here are options for wget and curl which could help us out here: + +[[!format sh """ + +# to make wget retry +> wget --retry-on-http-error=500 http://www.onerussian.com/tmp/errors/500 + +# to make curl retry, just needs --retry 10 which then considers 5xx errors intermittent +> curl --retry 10 http://www.onerussian.com/tmp/errors/500 + +"""]] + +Could git-annex add those options to curl/wget invocations for more robust access to the web remote? + +> [[done]] --[[Joey]] diff --git a/doc/bugs/Add_explicit_retries_for_curl__47__wget_on___34__transient__34___errors__63__/comment_1_9eafb42dce4c437a777cda0048756f42._comment b/doc/bugs/Add_explicit_retries_for_curl__47__wget_on___34__transient__34___errors__63__/comment_1_9eafb42dce4c437a777cda0048756f42._comment new file mode 100644 index 0000000000..ee75dfb7d1 --- /dev/null +++ b/doc/bugs/Add_explicit_retries_for_curl__47__wget_on___34__transient__34___errors__63__/comment_1_9eafb42dce4c437a777cda0048756f42._comment @@ -0,0 +1,33 @@ +[[!comment format=mdwn + username="joey" + subject="""comment 1""" + date="2018-03-24T13:54:31Z" + content=""" +Note that with --retry 10, curl will back off 10 times, with wait doubling, +so it could get stuck for 20+ minutes. That seems too long, but the right +number of retries seems to depend on how overloaded the http server is; you +may need a number that would otherwise be excessive in order to get a high +enough probability of success. + +Also worth noting that http *has* status codes such as 503 that are +intended to be used when the client should wait and retry; 500 is not such +a code. + +If this is done at the wget/curl level, it will also need to be done +when using the http-client library (which does not currently retry on any +code, AFAICs). + +And, it could just as easily be a S3 or webdav server that is throwing the +http retry codes, and the libraries for those will have their own retrying +behavior. (And it could even be a ssh server ot other non-http protocol +that connections to fail intermittently.) + +Putting all this together, I'm wondering if the http level is the right +place to put this retrying. It's not a matter of complying with the http +spec; it seems to need user configuration in order to handle their +particular use case. + +git-annex already does generic retrying as long as some data was +received, to recover from broken connections. That could be extended to +support a config option that enables a number of retries. +"""]] diff --git a/doc/bugs/Add_explicit_retries_for_curl__47__wget_on___34__transient__34___errors__63__/comment_2_23fe5ddd3a77c20d66c6847bdb83c94b._comment b/doc/bugs/Add_explicit_retries_for_curl__47__wget_on___34__transient__34___errors__63__/comment_2_23fe5ddd3a77c20d66c6847bdb83c94b._comment new file mode 100644 index 0000000000..2072cc3ee4 --- /dev/null +++ b/doc/bugs/Add_explicit_retries_for_curl__47__wget_on___34__transient__34___errors__63__/comment_2_23fe5ddd3a77c20d66c6847bdb83c94b._comment @@ -0,0 +1,13 @@ +[[!comment format=mdwn + username="joey" + subject="""comment 2""" + date="2018-03-29T17:24:35Z" + content=""" +I've implemented `remote..annex-retry`, `annex.retry`, +`remote..annex-retry-delay`, `annex.retry-delay` configs, so you can +have full control over retry behavior for remotes. + +Since these are fully generic, not at the HTTP level, they'll +make any and all transfer failures be retried, no matter why it failed. +Which could be a good thing or a bad thing.. +"""]] diff --git a/doc/bugs/Add_support_for_skein512__95__256_hash.mdwn b/doc/bugs/Add_support_for_skein512__95__256_hash.mdwn new file mode 100644 index 0000000000..f7149c56a7 --- /dev/null +++ b/doc/bugs/Add_support_for_skein512__95__256_hash.mdwn @@ -0,0 +1,10 @@ +### Please describe the problem. + +cryptonite supports the hash skein512_256, which is just a truncated version of skein512. Could you please add support for using this as a key-value backend? Please note that cryptohash doesn't support this hash, so it'll have to go into the ifdef block alongside the sha3 hashes. + +Rationale: I've just done a benchmark of the different hash implementations in cryptonite, and the skein hashes beat all other hashes (except md5) by a large margin, so I'd like to use them. As with the other hashes, skein512 is a bit faster than skein256 (on x86-64), but I'd like to avoid the overly long file names it produces. + + +### Have you had any luck using git-annex before? (Sometimes we get tired of reading bug reports all day and a lil' positive end note does wonders) + +I'm just trying git-annex for the first time, but it definitly looks great. diff --git a/doc/bugs/Add_support_for_skein512__95__256_hash/comment_1_7cebbecfc10c22fa58c5c1c97b9c6369._comment b/doc/bugs/Add_support_for_skein512__95__256_hash/comment_1_7cebbecfc10c22fa58c5c1c97b9c6369._comment new file mode 100644 index 0000000000..8199cadcd8 --- /dev/null +++ b/doc/bugs/Add_support_for_skein512__95__256_hash/comment_1_7cebbecfc10c22fa58c5c1c97b9c6369._comment @@ -0,0 +1,15 @@ +[[!comment format=mdwn + username="joey" + subject="""comment 1""" + date="2016-10-31T16:54:30Z" + content=""" +What kind of performance benefit are we talking about here? + +It seems it would be a bit hard to implement this, since Backend.Hash uses +a HashSize that is just an Int, which would have to be extended with the +number of bits to use of a longer hash. Not hard hard but it will +complicate the code. + +(Also, the name "SKEIN512_256" is pretty long itself to include in a +filename.") +"""]] diff --git a/doc/bugs/Add_support_for_skein512__95__256_hash/comment_2_fe916fca47ca8506bce7ee6766d6c590._comment b/doc/bugs/Add_support_for_skein512__95__256_hash/comment_2_fe916fca47ca8506bce7ee6766d6c590._comment new file mode 100644 index 0000000000..0cc329437f --- /dev/null +++ b/doc/bugs/Add_support_for_skein512__95__256_hash/comment_2_fe916fca47ca8506bce7ee6766d6c590._comment @@ -0,0 +1,13 @@ +[[!comment format=mdwn + username="aranea@650e41fad422f2a4d6f36ca1f20d41b7c0f18ab7" + nickname="aranea" + avatar="http://cdn.libravatar.org/avatar/8574023ce00757ca95b1708b7306602a" + subject="comment 2" + date="2016-10-31T21:29:42Z" + content=""" +> What kind of performance benefit are we talking about here? + +Not much. In my benchmark, SKEIN512 was about 10% faster thank SKEIN256, and SKEIN256 has quite reasonable speed anyway. Thus, adding SKEIN512_256 is not neccessary if it complicated the code too much. + +Thanks anyway! +"""]] diff --git a/doc/bugs/Adding_metadata.mdwn b/doc/bugs/Adding_metadata.mdwn new file mode 100644 index 0000000000..2b4516a2f9 --- /dev/null +++ b/doc/bugs/Adding_metadata.mdwn @@ -0,0 +1,18 @@ +### Please describe the problem. +This is less of a bug and more of a feature(?) request. Currently, when assigning metadata to a datafile, git annex (v5.20150710-g8fd705) will produce no error or warning message if the file entered does not exist. This can be a tad confusing to users who might expect to see some output if they typed their path wrong. A successful `git annex metadata -s tags=best` will produce output acknowledging the change, but a failure produces no output at all. A quick check if the file exists, and a `File does not exist` error message if not, would be easy implement and helpful to new users. + +### What steps will reproduce the problem? + +git annex metadata -s tags+=atlas + +Where does not exist. + +### What version of git-annex are you using? On what operating system? + +v5.20150710-g8fd705 + +### Have you had any luck using git-annex before? (Sometimes we get tired of reading bug reports all day and a lil' positive end note does wonders) + +We use git-annex for our open-source !FreeSurfer software and find very helpful indeed. Thank you. https://surfer.nmr.mgh.harvard.edu/ + +> [[fixed|done]] --[[Joey]] diff --git a/doc/bugs/Adding_torrent_via_addurl_fails.mdwn b/doc/bugs/Adding_torrent_via_addurl_fails.mdwn new file mode 100644 index 0000000000..058abaefbe --- /dev/null +++ b/doc/bugs/Adding_torrent_via_addurl_fails.mdwn @@ -0,0 +1,57 @@ +[[!meta title="addurl failure with non-standard torrent file"]] + +### Please describe the problem. +Adding a magnet link via addurl fails after downloading the torrent metatdata if the "announce" field of the torrent is empty + +### What steps will reproduce the problem? + git annex addurl "magnet:?xt=urn:btih:88066b90278f2de655ee2dd44e784c340b54e45c" + + +### What version of git-annex are you using? On what operating system? +git-annex version: 6.20160126 +archlinux + +### Please provide any additional information below. +I have traced back the Problem to the parsing of the torrent metatdata. +Since you also seem to be the author of the haskel-torrent parser I felt it is apropriate to post here. + +The above magnet link (an Archlinux Iso) results in a .torrent file that has no "announce" entry. Instead it only has the entry "announce-list" with multiple urls. +This causes the parser to fail. +I dont know if having only "announce-list" horribly violates some standard, however a second magnet link that i tried showed the same behaviour so this might not be an unusual case. + +I was able to put in a workarround in btshowmetainfo.py to set "annonuce" to the first entry from "announce-list" if it wasn't defined. +My git-annex binary is compiled with the haskel parser enabled do this doesn't change annexs' behaviour. + +It's not a big dealbreaker for me, just playing arround with the torrent feaure for now. + +[[!format sh """ +# If you can, paste a complete transcript of the problem occurring here. +# If the problem is with the git-annex assistant, paste in .git/annex/daemon.log +git annex addurl "magnet:?xt=urn:btih:88066b90278f2de655ee2dd44e784c340b54e45c" +(downloading torrent file...) + +02/07 16:42:13 [NOTICE] IPv4 DHT: listening on UDP port 6964 + +02/07 16:42:13 [NOTICE] IPv4 BitTorrent: listening on TCP port 6927 + +02/07 16:42:13 [NOTICE] IPv6 BitTorrent: listening on TCP port 6927 +[#96c5b2 27KiB/27KiB(100%) CN:11 SD:2] +02/07 16:42:32 [NOTICE] Download complete: [METADATA]88066b90278f2de655ee2dd44e784c340b54e45c + +02/07 16:42:32 [NOTICE] Saved metadata as ../../.git/annex/misctmp/URL--magnet&c,63xt,61urn&cbtih&c88066b90278f2de655ee2dd44e784c340b54e45c/meta/88066b90278f2de655ee2dd44e784c340b54e45c.torrent. + +Download Results: +gid |stat|avg speed |path/URI +======+====+===========+======================================================= +96c5b2|OK | 0B/s|[MEMORY][METADATA]88066b90278f2de655ee2dd44e784c340b54e45c + +Status Legend: +(OK):download completed. +git-annex: failed to parse torrent: Name not found in dictionary: announce +# End of transcript or log. +"""]] + +### Have you had any luck using git-annex before? (Sometimes we get tired of reading bug reports all day and a lil' positive end note does wonders) + + +> This was fixed in haskell-torrent version 10000.1.0. [[done]] diff --git a/doc/bugs/Adding_torrent_via_addurl_fails/comment_1_477913541446737692c2d3ae2cc197bb._comment b/doc/bugs/Adding_torrent_via_addurl_fails/comment_1_477913541446737692c2d3ae2cc197bb._comment new file mode 100644 index 0000000000..a85950bdde --- /dev/null +++ b/doc/bugs/Adding_torrent_via_addurl_fails/comment_1_477913541446737692c2d3ae2cc197bb._comment @@ -0,0 +1,18 @@ +[[!comment format=mdwn + username="joey" + subject="""comment 1""" + date="2016-02-08T18:53:00Z" + content=""" +announce-list comes from BEP-0012. Reading that spec, the torrent file is +supposed to still have the announce field, although its contents are +supposed to be ignored by clients that support announce-list. + +When I download that torrent, it doesn't seem to have an +announce-list either. No announce and no announce-list? It's possible that +aria2c, which I used to download it like git-annex does, doesn't know about +announce-list and left it out when saving the torrent file. Maybe it left +out announce too? Do you have a non-magnet url for this torrent file? + +I'll make the haskell torrent library support parsing announce-list once + is fixed. +"""]] diff --git a/doc/bugs/Adding_torrent_via_addurl_fails/comment_2_e317aee1ff6e60aafcd106751af660c8._comment b/doc/bugs/Adding_torrent_via_addurl_fails/comment_2_e317aee1ff6e60aafcd106751af660c8._comment new file mode 100644 index 0000000000..6b4a52478f --- /dev/null +++ b/doc/bugs/Adding_torrent_via_addurl_fails/comment_2_e317aee1ff6e60aafcd106751af660c8._comment @@ -0,0 +1,24 @@ +[[!comment format=mdwn + username="annuges" + subject="comment 2" + date="2016-02-09T13:49:42Z" + content=""" +sorry looks like i hadd a litle mixup, the full magnet link that i used when checking btshowmetainfo was + + magnet:?xt=urn:btih:88066b90278f2de655ee2dd44e784c340b54e45c&dn=archlinux-2016.02.01-dual.iso&tr=udp://tracker.archlinux.org:6969&tr=http://tracker.archlinux.org:6969/announce + +Using this one with aria2c --bt-save-metadata results in a torrent file with the announce-list entry only. +Modifying the link to only contain one tracker url still results in an announce-list with a single entry. + +There also is a separate torrent file on the arch website that has the announce field set correctly. + + https://www.archlinux.org/releng/releases/2016.02.01/torrent/ + +They hide the actuall .torrent file though so addurl doesn't trigger the torrent parsing with led me to try the magnet link to begin with. + +Anyway, looks like aria2c is at fault because it doesn't generate standard compliant .torrent files from magnet links. + + + + +"""]] diff --git a/doc/bugs/Adding_torrent_via_addurl_fails/comment_3_7ff48f1b6426f7bbf6f3aaa4e1526d19._comment b/doc/bugs/Adding_torrent_via_addurl_fails/comment_3_7ff48f1b6426f7bbf6f3aaa4e1526d19._comment new file mode 100644 index 0000000000..8b85097e2b --- /dev/null +++ b/doc/bugs/Adding_torrent_via_addurl_fails/comment_3_7ff48f1b6426f7bbf6f3aaa4e1526d19._comment @@ -0,0 +1,9 @@ +[[!comment format=mdwn + username="mbekkema97@66b135681014f005a3a14c4011d148fcb6655f81" + nickname="mbekkema97" + avatar="http://cdn.libravatar.org/avatar/a5037b8a5914bd9f803af7b7e881d632" + subject="comment 3" + date="2018-01-12T12:01:30Z" + content=""" +This seems to be fixed now due to an update to Haskell's torrent library. +"""]] diff --git a/doc/bugs/Adding_zip_or_7z_or_tar_archive_builds_for_windows.mdwn b/doc/bugs/Adding_zip_or_7z_or_tar_archive_builds_for_windows.mdwn new file mode 100644 index 0000000000..ae1f7c5229 --- /dev/null +++ b/doc/bugs/Adding_zip_or_7z_or_tar_archive_builds_for_windows.mdwn @@ -0,0 +1,5 @@ +Having a windows build of Git-Annex in an archived format would be very usefull for automation, and deploy. +Could it be possible to add this to the buildserver of gitannex? + +[[!tag moreinfo]] + diff --git a/doc/bugs/Adding_zip_or_7z_or_tar_archive_builds_for_windows/comment_1_70480ffd417788f18cd75a9b625ecf3b._comment b/doc/bugs/Adding_zip_or_7z_or_tar_archive_builds_for_windows/comment_1_70480ffd417788f18cd75a9b625ecf3b._comment new file mode 100644 index 0000000000..3bd7381e16 --- /dev/null +++ b/doc/bugs/Adding_zip_or_7z_or_tar_archive_builds_for_windows/comment_1_70480ffd417788f18cd75a9b625ecf3b._comment @@ -0,0 +1,19 @@ +[[!comment format=mdwn + username="joey" + subject="""comment 1""" + date="2016-11-16T18:41:25Z" + content=""" +It would be helpful to have more details, such as an example of software +distributed for windows that way, or documentation of how such an archive +is used on windows. + +The git-annex Windows installer is a exe file that uses the NullSoft +installation system. As far as I know that's pretty common in the Windows +world. + +I don't see any point in zipping up the single exe. It would be possible to +make a zip containing all the files that instlling the exe installs. But, +the installation process has to integrate git-annex with git, it installs +menu items, etc. A zip file would not be able to handle that integration. +So its use seems limited to me. +"""]] diff --git a/doc/bugs/Adding_zip_or_7z_or_tar_archive_builds_for_windows/comment_2_afa6a131999feda67876859cd85ebcfc._comment b/doc/bugs/Adding_zip_or_7z_or_tar_archive_builds_for_windows/comment_2_afa6a131999feda67876859cd85ebcfc._comment new file mode 100644 index 0000000000..b0db544798 --- /dev/null +++ b/doc/bugs/Adding_zip_or_7z_or_tar_archive_builds_for_windows/comment_2_afa6a131999feda67876859cd85ebcfc._comment @@ -0,0 +1,15 @@ +[[!comment format=mdwn + username="luckcolorsgoo@ab4f3c1c44a7dbcbcb9d9a29315b59ad524ceaaa" + nickname="luckcolorsgoo" + avatar="http://cdn.libravatar.org/avatar/ddff84cd2a97252a05cccb4bc5010448" + subject="comment 2" + date="2016-11-16T22:56:46Z" + content=""" +In my case i was going to make a script for automatically downloading and updating an git portbale / git annex instance, by first fetching git portbale and then downloading the gitannex exe. + +So yeah it's more reliable to extract an archive rather than trying to extract the setup without executing it. +That's why i'm asking for this feature. :) + + + +"""]] diff --git a/doc/bugs/Adjust_--unlock_not_using_--reflink__63__.mdwn b/doc/bugs/Adjust_--unlock_not_using_--reflink__63__.mdwn new file mode 100644 index 0000000000..91a69a4ea3 --- /dev/null +++ b/doc/bugs/Adjust_--unlock_not_using_--reflink__63__.mdwn @@ -0,0 +1,15 @@ +### Please describe the problem. + +Running adjust --unlock is unexpectedly slow and seems to use a lot of space, even on BTRFS, suggesting it probably does not use --reflink=auto like most other commands. + +### What steps will reproduce the problem? + +Run adjust --unlock with very large files. + +### What version of git-annex are you using? On what operating system? + +6.20170101-1+deb9u1 on Debian Stretch + +### Have you had any luck using git-annex before? (Sometimes we get tired of reading bug reports all day and a lil' positive end note does wonders) + +Yes I have! I've used it manage lots of video editing disks before, and am now migrating several slightly different copies of 15TB sized documentary footage from random USB3 disks and LTO tapes to a RAID server with BTRFS. diff --git a/doc/bugs/Adjust_--unlock_not_using_--reflink__63__/comment_1_c42743828ddc62f7df7f612c713ee309._comment b/doc/bugs/Adjust_--unlock_not_using_--reflink__63__/comment_1_c42743828ddc62f7df7f612c713ee309._comment new file mode 100644 index 0000000000..e30471b84c --- /dev/null +++ b/doc/bugs/Adjust_--unlock_not_using_--reflink__63__/comment_1_c42743828ddc62f7df7f612c713ee309._comment @@ -0,0 +1,18 @@ +[[!comment format=mdwn + username="joey" + subject="""comment 1""" + date="2017-12-05T17:24:24Z" + content=""" +What happens is `git annex adjust` creates a branch and runs `git checkout` +to check it out. Then git calls `git-annex smudge` on files, and +unfortunately the git smudge interface requires that git-annex output the +whole content of the file, to stdout, which git then writes to disk. + +So yes, this doesn't use reflinks, but it's worse than that, a better +interface would let git-annex simply move or hard-link the file into place, +even on filesystems not supporting reflinks. + +This is discussed in detail in [[todo/smudge]]. Unfortunately it will need +a better interface in git to be addressed, and that's why v6 is still an +experimental feature. +"""]] diff --git a/doc/bugs/All_inodes_eaten.mdwn b/doc/bugs/All_inodes_eaten.mdwn new file mode 100644 index 0000000000..d922269fa4 --- /dev/null +++ b/doc/bugs/All_inodes_eaten.mdwn @@ -0,0 +1,43 @@ +### Please describe the problem. + +Adding new files to a gcrypt remote ends up eating all inodes. + +### What steps will reproduce the problem? + +I have a docs directory, 780 files in it, various types, various subdirectories. I add these to a git annex repo. Create a gcrypt remote on a usb drive and use assistant to copy. Eventually all sorts of stuff stops working since there are no inodes left. + +### What version of git-annex are you using? On what operating system? + +Arch Linux 64 bit, stable. Git annex 6.20160613-8 + +### Please provide any additional information below. + +[[!format sh """ +# If you can, paste a complete transcript of the problem occurring here. +# If the problem is with the git-annex assistant, paste in .git/annex/daemon.log + +Remote bare repo looks as follows: + + annex/objects - contains 4096 directories (000, 001, ...) + + annex/objects/XXX - contains ~1380 directories (001, ...) + + annex/objects/XXX/XXX - contains two or three directories, e.g. GPGHMACSHA1--3939af6b89c30015490ce9e19f9051bb9e9fe64e + + annex/objects/XXX/XXX/GPGHMACSHA1-XXXX - contains one or two files of the same name + +Let's multiply that out: 4096 * 1380 * 2 * 2 = 22,609,920 + +Plus . and .. for all the directories, and boom, all my 59 million inodes are gone, before I even manage to store my 780 files! + +Disk is about 20% full. + + +# End of transcript or log. +"""]] + +### Have you had any luck using git-annex before? (Sometimes we get tired of reading bug reports all day and a lil' positive end note does wonders) + + + +[[!tag moreinfo]] diff --git a/doc/bugs/All_inodes_eaten/comment_1_efbd0e7bfa29d76f07de4ae2e655d483._comment b/doc/bugs/All_inodes_eaten/comment_1_efbd0e7bfa29d76f07de4ae2e655d483._comment new file mode 100644 index 0000000000..497f8f8977 --- /dev/null +++ b/doc/bugs/All_inodes_eaten/comment_1_efbd0e7bfa29d76f07de4ae2e655d483._comment @@ -0,0 +1,15 @@ +[[!comment format=mdwn + username="joey" + subject="""comment 1""" + date="2016-10-05T16:56:25Z" + content=""" +If your remote contains 4096*1380 "GPGHMACSHA1" directories +with files in them then there are apparently 5652480 annexed objects +stored in this remote... not 780. + +Unless the files in those directories are not the encrypted content +of annexed objects, but I don't know what else they could be. + +You'll need to follow up with more information about what these files are, +or a way to reproduce this. +"""]] diff --git a/doc/bugs/All_inodes_eaten/comment_2_bfb17e095aca362516139d642b78e82d._comment b/doc/bugs/All_inodes_eaten/comment_2_bfb17e095aca362516139d642b78e82d._comment new file mode 100644 index 0000000000..5feff2e172 --- /dev/null +++ b/doc/bugs/All_inodes_eaten/comment_2_bfb17e095aca362516139d642b78e82d._comment @@ -0,0 +1,1154 @@ +[[!comment format=mdwn + username="pot" + subject="780 source files...." + date="2016-10-06T20:05:41Z" + content=""" +... annex on removeable disk somehow had a lot more! + +I don't know what they are, but all files had of the same format, e.g.: + + annex/objects/XXX/YYY/GPGHMACSHA1--ZZZ/GPGHMACSHA1--ZZZ + +which I believe is the correct structure, just that the sheer number of XXX, YYY and ZZZ was way too many. I have no idea what the assistant was doing, if it was failed uploads retrying, or what made it repeatedly spazz out and store that many files. I gave up with the assistant in the end and started again (3rd time, first time irrecoverable corruption raised in another bug that is apparently a gcrypt-remote one). The entirety of my computer doesn't come remotely close to 59 million inodes used, so pretty safe to say this issue isn't caused by user error uploading too many files or syncing the wrong directory and not realising it. + +I tried to decrypt one of the objects to have a look at it, maybe see what it was or where it came from, but got the following: + + $> gpg --decrypt docs.old/annex/objects/005/030/GPGHMACSHA1--5095b828bd5b51c2e21fe2c73882228fa39e396b/GPGHMACSHA1--5095b828bd5b51c2e21fe2c73882228fa39e396b + gpg: AES encrypted data + gpg: encrypted with 1 passphrase + gpg: decryption failed: Bad session key + +Which confused me, where did AES get involved?? + +I have had an rsync going for days on end trying to clear the inodes to avoid reformatting the disk + +I'm persisting with tying to use git-annex, this time manually without the assistant. Just setting wanted/group as appropriate and a cron job to run \"add, sync, sync --content, unload\" every 5 minutes. + + +In terms of reproducing it, all I did was point the assistant at my docs directory and then set up a USB drive as a gcrypt remote. I did nothing manually. + +Here is some of the log of the rsync delete I have going (running since I raised this bug): + +... +... +deleting annex/objects/247/724/GPGHMACSHA1--170c379656e6b4a9c8a6ba40eea0b6abbe9af068/GPGHMACSHA1--170c379656e6b4a9c8a6ba40eea0b6abbe9af068 +deleting annex/objects/247/724/GPGHMACSHA1--170c379656e6b4a9c8a6ba40eea0b6abbe9af068/ +deleting annex/objects/247/724/ +deleting annex/objects/247/723/GPGHMACSHA1--935ea8024d049197aa924ae270008a2c46bcfd26/GPGHMACSHA1--935ea8024d049197aa924ae270008a2c46bcfd26 +deleting annex/objects/247/723/GPGHMACSHA1--935ea8024d049197aa924ae270008a2c46bcfd26/ +deleting annex/objects/247/723/GPGHMACSHA1--01ba35ae3734d4fa0a3f6a128cf630b2c413a880/GPGHMACSHA1--01ba35ae3734d4fa0a3f6a128cf630b2c413a880 +deleting annex/objects/247/723/GPGHMACSHA1--01ba35ae3734d4fa0a3f6a128cf630b2c413a880/ +deleting annex/objects/247/723/ +deleting annex/objects/247/722/GPGHMACSHA1--71dc0291cb090d1b56aeed3c613b9ab0d09e5bf1/GPGHMACSHA1--71dc0291cb090d1b56aeed3c613b9ab0d09e5bf1 +deleting annex/objects/247/722/GPGHMACSHA1--71dc0291cb090d1b56aeed3c613b9ab0d09e5bf1/ +deleting annex/objects/247/722/ +deleting annex/objects/247/720/GPGHMACSHA1--dfd50c4a8b8038c5f1422de11121a53d10c2ee2b/GPGHMACSHA1--dfd50c4a8b8038c5f1422de11121a53d10c2ee2b +deleting annex/objects/247/720/GPGHMACSHA1--dfd50c4a8b8038c5f1422de11121a53d10c2ee2b/ +deleting annex/objects/247/720/ +deleting annex/objects/247/71d/GPGHMACSHA1--7846c602f4482b9799d74fa43626c04e13a54fd1/GPGHMACSHA1--7846c602f4482b9799d74fa43626c04e13a54fd1 +deleting annex/objects/247/71d/GPGHMACSHA1--7846c602f4482b9799d74fa43626c04e13a54fd1/ +deleting annex/objects/247/71d/ +deleting annex/objects/247/715/GPGHMACSHA1--b374ce280c695995b6fa53d8e69ddb26d0f78407/GPGHMACSHA1--b374ce280c695995b6fa53d8e69ddb26d0f78407 +deleting annex/objects/247/715/GPGHMACSHA1--b374ce280c695995b6fa53d8e69ddb26d0f78407/ +deleting annex/objects/247/715/ +deleting annex/objects/247/714/GPGHMACSHA1--aa189f81ef2897d7db35a54b93eceab81d0d50eb/GPGHMACSHA1--aa189f81ef2897d7db35a54b93eceab81d0d50eb +deleting annex/objects/247/714/GPGHMACSHA1--aa189f81ef2897d7db35a54b93eceab81d0d50eb/ +deleting annex/objects/247/714/ +deleting annex/objects/247/70f/GPGHMACSHA1--c464d300ff73fec5c54cb901f827e6485b0e718d/GPGHMACSHA1--c464d300ff73fec5c54cb901f827e6485b0e718d +deleting annex/objects/247/70f/GPGHMACSHA1--c464d300ff73fec5c54cb901f827e6485b0e718d/ +deleting annex/objects/247/70f/GPGHMACSHA1--6f4b104a14b0ff7e706d2c41cec5dfba88ea8dc2/GPGHMACSHA1--6f4b104a14b0ff7e706d2c41cec5dfba88ea8dc2 +deleting annex/objects/247/70f/GPGHMACSHA1--6f4b104a14b0ff7e706d2c41cec5dfba88ea8dc2/ +deleting annex/objects/247/70f/ +deleting annex/objects/247/70a/GPGHMACSHA1--ce4d890fe1bbe3f04073898b23a7d034b1f40e13/GPGHMACSHA1--ce4d890fe1bbe3f04073898b23a7d034b1f40e13 +deleting annex/objects/247/70a/GPGHMACSHA1--ce4d890fe1bbe3f04073898b23a7d034b1f40e13/ +deleting annex/objects/247/70a/ +deleting annex/objects/247/709/GPGHMACSHA1--6b33a1aa533172c104a5d9d46a6474336f7cb483/GPGHMACSHA1--6b33a1aa533172c104a5d9d46a6474336f7cb483 +deleting annex/objects/247/709/GPGHMACSHA1--6b33a1aa533172c104a5d9d46a6474336f7cb483/ +deleting annex/objects/247/709/ +deleting annex/objects/247/703/GPGHMACSHA1--ddb04cfcbcf0f02e2d66d94c445f4465a2821e5c/GPGHMACSHA1--ddb04cfcbcf0f02e2d66d94c445f4465a2821e5c +deleting annex/objects/247/703/GPGHMACSHA1--ddb04cfcbcf0f02e2d66d94c445f4465a2821e5c/ +deleting annex/objects/247/703/GPGHMACSHA1--ae64586876848866ff2ceccc5edf85710becb033/GPGHMACSHA1--ae64586876848866ff2ceccc5edf85710becb033 +deleting annex/objects/247/703/GPGHMACSHA1--ae64586876848866ff2ceccc5edf85710becb033/ +deleting annex/objects/247/703/ +deleting annex/objects/247/6fe/GPGHMACSHA1--44debae83909c6c8b274df7946f067810ce2e021/GPGHMACSHA1--44debae83909c6c8b274df7946f067810ce2e021 +deleting annex/objects/247/6fe/GPGHMACSHA1--44debae83909c6c8b274df7946f067810ce2e021/ +deleting annex/objects/247/6fe/ +deleting annex/objects/247/6f3/GPGHMACSHA1--a6daa142be6a8b89577d24a385e6e3a2766e7c4d/GPGHMACSHA1--a6daa142be6a8b89577d24a385e6e3a2766e7c4d +deleting annex/objects/247/6f3/GPGHMACSHA1--a6daa142be6a8b89577d24a385e6e3a2766e7c4d/ +deleting annex/objects/247/6f3/ +deleting annex/objects/247/6f0/GPGHMACSHA1--53357d1e9c86dce96c9531c387e25f4c836536ce/GPGHMACSHA1--53357d1e9c86dce96c9531c387e25f4c836536ce +deleting annex/objects/247/6f0/GPGHMACSHA1--53357d1e9c86dce96c9531c387e25f4c836536ce/ +deleting annex/objects/247/6f0/ +deleting annex/objects/247/6ef/GPGHMACSHA1--1e9bd6ff153221c26657f69e01e8c7a9c29957b7/GPGHMACSHA1--1e9bd6ff153221c26657f69e01e8c7a9c29957b7 +deleting annex/objects/247/6ef/GPGHMACSHA1--1e9bd6ff153221c26657f69e01e8c7a9c29957b7/ +deleting annex/objects/247/6ef/ +deleting annex/objects/247/6ec/GPGHMACSHA1--7a8ccc28b9131111774b8e3221f9f0c4abfd8597/GPGHMACSHA1--7a8ccc28b9131111774b8e3221f9f0c4abfd8597 +deleting annex/objects/247/6ec/GPGHMACSHA1--7a8ccc28b9131111774b8e3221f9f0c4abfd8597/ +deleting annex/objects/247/6ec/ +deleting annex/objects/247/6eb/GPGHMACSHA1--2ad823b488c9b0a90151816030fc98f3e5230ba3/GPGHMACSHA1--2ad823b488c9b0a90151816030fc98f3e5230ba3 +deleting annex/objects/247/6eb/GPGHMACSHA1--2ad823b488c9b0a90151816030fc98f3e5230ba3/ +deleting annex/objects/247/6eb/ +deleting annex/objects/247/6ea/GPGHMACSHA1--02f576c0a7387f32a3951c4200c96c545dad26df/GPGHMACSHA1--02f576c0a7387f32a3951c4200c96c545dad26df +deleting annex/objects/247/6ea/GPGHMACSHA1--02f576c0a7387f32a3951c4200c96c545dad26df/ +deleting annex/objects/247/6ea/ +deleting annex/objects/247/6e4/GPGHMACSHA1--29322261b9d5656e6aad6a996243e9af3bd1f044/GPGHMACSHA1--29322261b9d5656e6aad6a996243e9af3bd1f044 +deleting annex/objects/247/6e4/GPGHMACSHA1--29322261b9d5656e6aad6a996243e9af3bd1f044/ +deleting annex/objects/247/6e4/ +deleting annex/objects/247/6e3/GPGHMACSHA1--a6fdcce68aba986ccdd08380c74033ba0d2fc34b/GPGHMACSHA1--a6fdcce68aba986ccdd08380c74033ba0d2fc34b +deleting annex/objects/247/6e3/GPGHMACSHA1--a6fdcce68aba986ccdd08380c74033ba0d2fc34b/ +deleting annex/objects/247/6e3/ +deleting annex/objects/247/6de/GPGHMACSHA1--6a20df04e80c0bbb110283421249eff2a6c28116/GPGHMACSHA1--6a20df04e80c0bbb110283421249eff2a6c28116 +deleting annex/objects/247/6de/GPGHMACSHA1--6a20df04e80c0bbb110283421249eff2a6c28116/ +deleting annex/objects/247/6de/ +deleting annex/objects/247/6dc/GPGHMACSHA1--c29f40d89cc64dfbf0d54ce1645bed4a6bf38391/GPGHMACSHA1--c29f40d89cc64dfbf0d54ce1645bed4a6bf38391 +deleting annex/objects/247/6dc/GPGHMACSHA1--c29f40d89cc64dfbf0d54ce1645bed4a6bf38391/ +deleting annex/objects/247/6dc/ +deleting annex/objects/247/6da/GPGHMACSHA1--ea7e87aef54d57aa148e14e52a4c2e8edcdd7ada/GPGHMACSHA1--ea7e87aef54d57aa148e14e52a4c2e8edcdd7ada +deleting annex/objects/247/6da/GPGHMACSHA1--ea7e87aef54d57aa148e14e52a4c2e8edcdd7ada/ +deleting annex/objects/247/6da/ +deleting annex/objects/247/6d9/GPGHMACSHA1--77b13817763688dbacca4a1740d3b2f44c0c107f/GPGHMACSHA1--77b13817763688dbacca4a1740d3b2f44c0c107f +deleting annex/objects/247/6d9/GPGHMACSHA1--77b13817763688dbacca4a1740d3b2f44c0c107f/ +deleting annex/objects/247/6d9/ +deleting annex/objects/247/6d2/GPGHMACSHA1--8a8604b14a9688aa7ff1fedf5641936c09bf5cca/GPGHMACSHA1--8a8604b14a9688aa7ff1fedf5641936c09bf5cca +deleting annex/objects/247/6d2/GPGHMACSHA1--8a8604b14a9688aa7ff1fedf5641936c09bf5cca/ +deleting annex/objects/247/6d2/ +deleting annex/objects/247/6d1/GPGHMACSHA1--2537ac0a72d33099d5a56f92b3dc640e162ef07d/GPGHMACSHA1--2537ac0a72d33099d5a56f92b3dc640e162ef07d +deleting annex/objects/247/6d1/GPGHMACSHA1--2537ac0a72d33099d5a56f92b3dc640e162ef07d/ +deleting annex/objects/247/6d1/ +deleting annex/objects/247/6cf/GPGHMACSHA1--c93779f6b21b2abd2c66992189b7d7febb9dcf07/GPGHMACSHA1--c93779f6b21b2abd2c66992189b7d7febb9dcf07 +deleting annex/objects/247/6cf/GPGHMACSHA1--c93779f6b21b2abd2c66992189b7d7febb9dcf07/ +deleting annex/objects/247/6cf/GPGHMACSHA1--41e57e9679361202c970960559613c7760583b3c/GPGHMACSHA1--41e57e9679361202c970960559613c7760583b3c +deleting annex/objects/247/6cf/GPGHMACSHA1--41e57e9679361202c970960559613c7760583b3c/ +deleting annex/objects/247/6cf/GPGHMACSHA1--1287b5adc699aafcfac67428df18f072132b0822/GPGHMACSHA1--1287b5adc699aafcfac67428df18f072132b0822 +deleting annex/objects/247/6cf/GPGHMACSHA1--1287b5adc699aafcfac67428df18f072132b0822/ +deleting annex/objects/247/6cf/ +deleting annex/objects/247/6c6/GPGHMACSHA1--98f5cda5e44596b0e05e090ac939da231bab7ea3/GPGHMACSHA1--98f5cda5e44596b0e05e090ac939da231bab7ea3 +deleting annex/objects/247/6c6/GPGHMACSHA1--98f5cda5e44596b0e05e090ac939da231bab7ea3/ +deleting annex/objects/247/6c6/ +deleting annex/objects/247/6c4/GPGHMACSHA1--d94155d9f8f97c5f5ea5cf0323f31f4d9a08fd36/GPGHMACSHA1--d94155d9f8f97c5f5ea5cf0323f31f4d9a08fd36 +deleting annex/objects/247/6c4/GPGHMACSHA1--d94155d9f8f97c5f5ea5cf0323f31f4d9a08fd36/ +deleting annex/objects/247/6c4/ +deleting annex/objects/247/6bf/GPGHMACSHA1--4bb623be83d6d356cbfda0f64fec95c3ffdbcf11/GPGHMACSHA1--4bb623be83d6d356cbfda0f64fec95c3ffdbcf11 +deleting annex/objects/247/6bf/GPGHMACSHA1--4bb623be83d6d356cbfda0f64fec95c3ffdbcf11/ +deleting annex/objects/247/6bf/GPGHMACSHA1--06e160bda37431fde974f254cb42ff4c531f9d63/GPGHMACSHA1--06e160bda37431fde974f254cb42ff4c531f9d63 +deleting annex/objects/247/6bf/GPGHMACSHA1--06e160bda37431fde974f254cb42ff4c531f9d63/ +deleting annex/objects/247/6bf/ +deleting annex/objects/247/6be/GPGHMACSHA1--1633dd22b403d8ec0076b3138368179b0c05a930/GPGHMACSHA1--1633dd22b403d8ec0076b3138368179b0c05a930 +deleting annex/objects/247/6be/GPGHMACSHA1--1633dd22b403d8ec0076b3138368179b0c05a930/ +deleting annex/objects/247/6be/ +deleting annex/objects/247/6ba/GPGHMACSHA1--877bc52ac9977b3d1cb1772c4af6ea15b6e5a9c6/GPGHMACSHA1--877bc52ac9977b3d1cb1772c4af6ea15b6e5a9c6 +deleting annex/objects/247/6ba/GPGHMACSHA1--877bc52ac9977b3d1cb1772c4af6ea15b6e5a9c6/ +deleting annex/objects/247/6ba/ +deleting annex/objects/247/6b6/GPGHMACSHA1--f997fe07af28421edd2cfc8b2584660a03f37a00/GPGHMACSHA1--f997fe07af28421edd2cfc8b2584660a03f37a00 +deleting annex/objects/247/6b6/GPGHMACSHA1--f997fe07af28421edd2cfc8b2584660a03f37a00/ +deleting annex/objects/247/6b6/ +deleting annex/objects/247/6b4/GPGHMACSHA1--aba2860a370b2683bba2dfefd203db8f830f80a4/GPGHMACSHA1--aba2860a370b2683bba2dfefd203db8f830f80a4 +deleting annex/objects/247/6b4/GPGHMACSHA1--aba2860a370b2683bba2dfefd203db8f830f80a4/ +deleting annex/objects/247/6b4/ +deleting annex/objects/247/6b2/GPGHMACSHA1--90586dbd99e4a55a4a98153d9813241e342a0cfc/GPGHMACSHA1--90586dbd99e4a55a4a98153d9813241e342a0cfc +deleting annex/objects/247/6b2/GPGHMACSHA1--90586dbd99e4a55a4a98153d9813241e342a0cfc/ +deleting annex/objects/247/6b2/ +deleting annex/objects/247/6b1/GPGHMACSHA1--e3bcf6e224b064505016cec6771f5ceaa6dbed67/GPGHMACSHA1--e3bcf6e224b064505016cec6771f5ceaa6dbed67 +deleting annex/objects/247/6b1/GPGHMACSHA1--e3bcf6e224b064505016cec6771f5ceaa6dbed67/ +deleting annex/objects/247/6b1/ +deleting annex/objects/247/6ac/GPGHMACSHA1--bf584a5c38c33bf14ad05b6219ac244e02aa251b/GPGHMACSHA1--bf584a5c38c33bf14ad05b6219ac244e02aa251b +deleting annex/objects/247/6ac/GPGHMACSHA1--bf584a5c38c33bf14ad05b6219ac244e02aa251b/ +deleting annex/objects/247/6ac/ +deleting annex/objects/247/6ab/GPGHMACSHA1--6685b79b3450bb3386f9aecf77b0df9254d8aeff/GPGHMACSHA1--6685b79b3450bb3386f9aecf77b0df9254d8aeff +deleting annex/objects/247/6ab/GPGHMACSHA1--6685b79b3450bb3386f9aecf77b0df9254d8aeff/ +deleting annex/objects/247/6ab/ +deleting annex/objects/247/6aa/GPGHMACSHA1--47ba5e85f50805512695e9981ca4e65a34fb49e2/GPGHMACSHA1--47ba5e85f50805512695e9981ca4e65a34fb49e2 +deleting annex/objects/247/6aa/GPGHMACSHA1--47ba5e85f50805512695e9981ca4e65a34fb49e2/ +deleting annex/objects/247/6aa/ +deleting annex/objects/247/6a9/GPGHMACSHA1--98818b49a059bdf24b7292d39301bc8fa800997e/GPGHMACSHA1--98818b49a059bdf24b7292d39301bc8fa800997e +deleting annex/objects/247/6a9/GPGHMACSHA1--98818b49a059bdf24b7292d39301bc8fa800997e/ +deleting annex/objects/247/6a9/GPGHMACSHA1--5d433de5bf5f8af586839e6f6e5881394ce7753c/GPGHMACSHA1--5d433de5bf5f8af586839e6f6e5881394ce7753c +deleting annex/objects/247/6a9/GPGHMACSHA1--5d433de5bf5f8af586839e6f6e5881394ce7753c/ +deleting annex/objects/247/6a9/ +deleting annex/objects/247/6a7/GPGHMACSHA1--842b4d379b833766f57d81eb8d9a7f59dc9e7e32/GPGHMACSHA1--842b4d379b833766f57d81eb8d9a7f59dc9e7e32 +deleting annex/objects/247/6a7/GPGHMACSHA1--842b4d379b833766f57d81eb8d9a7f59dc9e7e32/ +deleting annex/objects/247/6a7/ +deleting annex/objects/247/6a5/GPGHMACSHA1--faa62d0c76cfa51b0bf983cd981143ae0a8e644b/GPGHMACSHA1--faa62d0c76cfa51b0bf983cd981143ae0a8e644b +deleting annex/objects/247/6a5/GPGHMACSHA1--faa62d0c76cfa51b0bf983cd981143ae0a8e644b/ +deleting annex/objects/247/6a5/ +deleting annex/objects/247/6a2/GPGHMACSHA1--8b22e4cec5e8d7413eaf4eeb116c45cae4aad4a7/GPGHMACSHA1--8b22e4cec5e8d7413eaf4eeb116c45cae4aad4a7 +deleting annex/objects/247/6a2/GPGHMACSHA1--8b22e4cec5e8d7413eaf4eeb116c45cae4aad4a7/ +deleting annex/objects/247/6a2/ +deleting annex/objects/247/6a0/GPGHMACSHA1--cc12f74b40a0dc2b2f962884be0de5f129dd2f29/GPGHMACSHA1--cc12f74b40a0dc2b2f962884be0de5f129dd2f29 +deleting annex/objects/247/6a0/GPGHMACSHA1--cc12f74b40a0dc2b2f962884be0de5f129dd2f29/ +deleting annex/objects/247/6a0/GPGHMACSHA1--6d3ea0b051d26b9fd33bd50e913edc8bb7958802/GPGHMACSHA1--6d3ea0b051d26b9fd33bd50e913edc8bb7958802 +deleting annex/objects/247/6a0/GPGHMACSHA1--6d3ea0b051d26b9fd33bd50e913edc8bb7958802/ +deleting annex/objects/247/6a0/ +deleting annex/objects/247/69e/GPGHMACSHA1--8bf51eb698b413e2e26228f05ea46e50eed90ecd/GPGHMACSHA1--8bf51eb698b413e2e26228f05ea46e50eed90ecd +deleting annex/objects/247/69e/GPGHMACSHA1--8bf51eb698b413e2e26228f05ea46e50eed90ecd/ +deleting annex/objects/247/69e/ +deleting annex/objects/247/69c/GPGHMACSHA1--8e4ae21aea3fac1476477010464fa593427637bb/GPGHMACSHA1--8e4ae21aea3fac1476477010464fa593427637bb +deleting annex/objects/247/69c/GPGHMACSHA1--8e4ae21aea3fac1476477010464fa593427637bb/ +deleting annex/objects/247/69c/ +deleting annex/objects/247/69a/GPGHMACSHA1--17a7e2efb6973e644f3a2c46ddf3c23a04252807/GPGHMACSHA1--17a7e2efb6973e644f3a2c46ddf3c23a04252807 +deleting annex/objects/247/69a/GPGHMACSHA1--17a7e2efb6973e644f3a2c46ddf3c23a04252807/ +deleting annex/objects/247/69a/GPGHMACSHA1--038192462f39548fcb767ff68868654a04d3742c/GPGHMACSHA1--038192462f39548fcb767ff68868654a04d3742c +deleting annex/objects/247/69a/GPGHMACSHA1--038192462f39548fcb767ff68868654a04d3742c/ +deleting annex/objects/247/69a/ +deleting annex/objects/247/699/GPGHMACSHA1--ebf4456aae42a77fbbec10ea4a2dbbef6a3c0376/GPGHMACSHA1--ebf4456aae42a77fbbec10ea4a2dbbef6a3c0376 +deleting annex/objects/247/699/GPGHMACSHA1--ebf4456aae42a77fbbec10ea4a2dbbef6a3c0376/ +deleting annex/objects/247/699/GPGHMACSHA1--e2daee271ea42cc9b5004c38d4271c17403180e3/GPGHMACSHA1--e2daee271ea42cc9b5004c38d4271c17403180e3 +deleting annex/objects/247/699/GPGHMACSHA1--e2daee271ea42cc9b5004c38d4271c17403180e3/ +deleting annex/objects/247/699/ +deleting annex/objects/247/697/GPGHMACSHA1--b25242f316df75d10180bc946d7fe455570f8feb/GPGHMACSHA1--b25242f316df75d10180bc946d7fe455570f8feb +deleting annex/objects/247/697/GPGHMACSHA1--b25242f316df75d10180bc946d7fe455570f8feb/ +deleting annex/objects/247/697/ +deleting annex/objects/247/695/GPGHMACSHA1--c336852b055947bc7f8de6fb9e10cd0a094713c8/GPGHMACSHA1--c336852b055947bc7f8de6fb9e10cd0a094713c8 +deleting annex/objects/247/695/GPGHMACSHA1--c336852b055947bc7f8de6fb9e10cd0a094713c8/ +deleting annex/objects/247/695/ +deleting annex/objects/247/693/GPGHMACSHA1--9d1a2d8c041969d4665e646868071d5b919e9f48/GPGHMACSHA1--9d1a2d8c041969d4665e646868071d5b919e9f48 +deleting annex/objects/247/693/GPGHMACSHA1--9d1a2d8c041969d4665e646868071d5b919e9f48/ +deleting annex/objects/247/693/ +deleting annex/objects/247/690/GPGHMACSHA1--bdf3ad43c80329313c2615071f4b3425900ddfb5/GPGHMACSHA1--bdf3ad43c80329313c2615071f4b3425900ddfb5 +deleting annex/objects/247/690/GPGHMACSHA1--bdf3ad43c80329313c2615071f4b3425900ddfb5/ +deleting annex/objects/247/690/GPGHMACSHA1--633bc661f6be86898c8b86a0bc617c76c5c05d5b/GPGHMACSHA1--633bc661f6be86898c8b86a0bc617c76c5c05d5b +deleting annex/objects/247/690/GPGHMACSHA1--633bc661f6be86898c8b86a0bc617c76c5c05d5b/ +deleting annex/objects/247/690/ +deleting annex/objects/247/687/GPGHMACSHA1--a908420b6feadffbef8dc11e26b2c7dd801e62bc/GPGHMACSHA1--a908420b6feadffbef8dc11e26b2c7dd801e62bc +deleting annex/objects/247/687/GPGHMACSHA1--a908420b6feadffbef8dc11e26b2c7dd801e62bc/ +deleting annex/objects/247/687/GPGHMACSHA1--4854a36dbe98d66523e3f81ba11ae19fa5b000ad/GPGHMACSHA1--4854a36dbe98d66523e3f81ba11ae19fa5b000ad +deleting annex/objects/247/687/GPGHMACSHA1--4854a36dbe98d66523e3f81ba11ae19fa5b000ad/ +deleting annex/objects/247/687/ +deleting annex/objects/247/680/GPGHMACSHA1--be28ddb2b64ab0d126550f77b08d28003daed9f5/GPGHMACSHA1--be28ddb2b64ab0d126550f77b08d28003daed9f5 +deleting annex/objects/247/680/GPGHMACSHA1--be28ddb2b64ab0d126550f77b08d28003daed9f5/ +deleting annex/objects/247/680/GPGHMACSHA1--32f8b23783ae8c128d59a00cb56415582b7260df/GPGHMACSHA1--32f8b23783ae8c128d59a00cb56415582b7260df +deleting annex/objects/247/680/GPGHMACSHA1--32f8b23783ae8c128d59a00cb56415582b7260df/ +deleting annex/objects/247/680/ +deleting annex/objects/247/67d/GPGHMACSHA1--cbe506d47fa8c7986af274ade75786348edfe629/GPGHMACSHA1--cbe506d47fa8c7986af274ade75786348edfe629 +deleting annex/objects/247/67d/GPGHMACSHA1--cbe506d47fa8c7986af274ade75786348edfe629/ +deleting annex/objects/247/67d/GPGHMACSHA1--274cf22593002c284739e6bbf055e845b186f758/GPGHMACSHA1--274cf22593002c284739e6bbf055e845b186f758 +deleting annex/objects/247/67d/GPGHMACSHA1--274cf22593002c284739e6bbf055e845b186f758/ +deleting annex/objects/247/67d/ +deleting annex/objects/247/67b/GPGHMACSHA1--6c92d27861fc721994b18c11042c43f06548b7c4/GPGHMACSHA1--6c92d27861fc721994b18c11042c43f06548b7c4 +deleting annex/objects/247/67b/GPGHMACSHA1--6c92d27861fc721994b18c11042c43f06548b7c4/ +deleting annex/objects/247/67b/GPGHMACSHA1--1b72db8f312c84787c2db090b6d8f1470b5101e4/GPGHMACSHA1--1b72db8f312c84787c2db090b6d8f1470b5101e4 +deleting annex/objects/247/67b/GPGHMACSHA1--1b72db8f312c84787c2db090b6d8f1470b5101e4/ +deleting annex/objects/247/67b/ +deleting annex/objects/247/677/GPGHMACSHA1--884eefed2eb49c523b0caddeebbdb4c8c99b124c/GPGHMACSHA1--884eefed2eb49c523b0caddeebbdb4c8c99b124c +deleting annex/objects/247/677/GPGHMACSHA1--884eefed2eb49c523b0caddeebbdb4c8c99b124c/ +deleting annex/objects/247/677/ +deleting annex/objects/247/671/GPGHMACSHA1--7734e81c47fba2413b6ab20ab53a12d5b2b5fd40/GPGHMACSHA1--7734e81c47fba2413b6ab20ab53a12d5b2b5fd40 +deleting annex/objects/247/671/GPGHMACSHA1--7734e81c47fba2413b6ab20ab53a12d5b2b5fd40/ +deleting annex/objects/247/671/ +deleting annex/objects/247/670/GPGHMACSHA1--a8a923f284a21ab9d51cc8e1043e1ee1e0e1c52d/GPGHMACSHA1--a8a923f284a21ab9d51cc8e1043e1ee1e0e1c52d +deleting annex/objects/247/670/GPGHMACSHA1--a8a923f284a21ab9d51cc8e1043e1ee1e0e1c52d/ +deleting annex/objects/247/670/ +deleting annex/objects/247/66d/GPGHMACSHA1--afa309fdc40f89129383f2543c5a81ba25377da7/GPGHMACSHA1--afa309fdc40f89129383f2543c5a81ba25377da7 +deleting annex/objects/247/66d/GPGHMACSHA1--afa309fdc40f89129383f2543c5a81ba25377da7/ +deleting annex/objects/247/66d/ +deleting annex/objects/247/66c/GPGHMACSHA1--6242d8b47814f9de875cee4c5cf87fe20761dc6f/GPGHMACSHA1--6242d8b47814f9de875cee4c5cf87fe20761dc6f +deleting annex/objects/247/66c/GPGHMACSHA1--6242d8b47814f9de875cee4c5cf87fe20761dc6f/ +deleting annex/objects/247/66c/ +deleting annex/objects/247/66b/GPGHMACSHA1--c1e8348121e0aad254a6ee00c721cafefb29a7f8/GPGHMACSHA1--c1e8348121e0aad254a6ee00c721cafefb29a7f8 +deleting annex/objects/247/66b/GPGHMACSHA1--c1e8348121e0aad254a6ee00c721cafefb29a7f8/ +deleting annex/objects/247/66b/GPGHMACSHA1--905f20a5b38c0988d5b47b61ed2476695a3737a4/GPGHMACSHA1--905f20a5b38c0988d5b47b61ed2476695a3737a4 +deleting annex/objects/247/66b/GPGHMACSHA1--905f20a5b38c0988d5b47b61ed2476695a3737a4/ +deleting annex/objects/247/66b/ +deleting annex/objects/247/669/GPGHMACSHA1--3285b2d42125cc37f6d060b4e4f0cf1eebeb8ab8/GPGHMACSHA1--3285b2d42125cc37f6d060b4e4f0cf1eebeb8ab8 +deleting annex/objects/247/669/GPGHMACSHA1--3285b2d42125cc37f6d060b4e4f0cf1eebeb8ab8/ +deleting annex/objects/247/669/ +deleting annex/objects/247/668/GPGHMACSHA1--77c84003474e0d26f0a28c1f12bc1facb661cf88/GPGHMACSHA1--77c84003474e0d26f0a28c1f12bc1facb661cf88 +deleting annex/objects/247/668/GPGHMACSHA1--77c84003474e0d26f0a28c1f12bc1facb661cf88/ +deleting annex/objects/247/668/GPGHMACSHA1--426c97f946ff8a94cd5ef4a6d837165056e65f7b/GPGHMACSHA1--426c97f946ff8a94cd5ef4a6d837165056e65f7b +deleting annex/objects/247/668/GPGHMACSHA1--426c97f946ff8a94cd5ef4a6d837165056e65f7b/ +deleting annex/objects/247/668/ +deleting annex/objects/247/662/GPGHMACSHA1--daa15474f59cf6b3f205aa382a9aed3d8dccad07/GPGHMACSHA1--daa15474f59cf6b3f205aa382a9aed3d8dccad07 +deleting annex/objects/247/662/GPGHMACSHA1--daa15474f59cf6b3f205aa382a9aed3d8dccad07/ +deleting annex/objects/247/662/ +deleting annex/objects/247/661/GPGHMACSHA1--69dcd6923078d7645fb10dde9da43498d3029672/GPGHMACSHA1--69dcd6923078d7645fb10dde9da43498d3029672 +deleting annex/objects/247/661/GPGHMACSHA1--69dcd6923078d7645fb10dde9da43498d3029672/ +deleting annex/objects/247/661/ +deleting annex/objects/247/65f/GPGHMACSHA1--dd83c0fc950d2d50a518abbf39ae7364f3dfcc2b/GPGHMACSHA1--dd83c0fc950d2d50a518abbf39ae7364f3dfcc2b +deleting annex/objects/247/65f/GPGHMACSHA1--dd83c0fc950d2d50a518abbf39ae7364f3dfcc2b/ +deleting annex/objects/247/65f/ +deleting annex/objects/247/65d/GPGHMACSHA1--a74379634b5858f29282871be77653dd3bd85f9c/GPGHMACSHA1--a74379634b5858f29282871be77653dd3bd85f9c +deleting annex/objects/247/65d/GPGHMACSHA1--a74379634b5858f29282871be77653dd3bd85f9c/ +deleting annex/objects/247/65d/ +deleting annex/objects/247/65b/GPGHMACSHA1--e518f349498e334cb42a0badee072b7182ada6a1/GPGHMACSHA1--e518f349498e334cb42a0badee072b7182ada6a1 +deleting annex/objects/247/65b/GPGHMACSHA1--e518f349498e334cb42a0badee072b7182ada6a1/ +deleting annex/objects/247/65b/ +deleting annex/objects/247/657/GPGHMACSHA1--f13db23e398d11e5ae416e6cc26720b2f11139be/GPGHMACSHA1--f13db23e398d11e5ae416e6cc26720b2f11139be +deleting annex/objects/247/657/GPGHMACSHA1--f13db23e398d11e5ae416e6cc26720b2f11139be/ +deleting annex/objects/247/657/GPGHMACSHA1--1e9325d04e7a8007d0f10dc705a77e4385edb31c/GPGHMACSHA1--1e9325d04e7a8007d0f10dc705a77e4385edb31c +deleting annex/objects/247/657/GPGHMACSHA1--1e9325d04e7a8007d0f10dc705a77e4385edb31c/ +deleting annex/objects/247/657/ +deleting annex/objects/247/655/GPGHMACSHA1--6f35926c1ddf7f2a5579bca38e89f1902913afe9/GPGHMACSHA1--6f35926c1ddf7f2a5579bca38e89f1902913afe9 +deleting annex/objects/247/655/GPGHMACSHA1--6f35926c1ddf7f2a5579bca38e89f1902913afe9/ +deleting annex/objects/247/655/ +deleting annex/objects/247/654/GPGHMACSHA1--498faf23d8f11a54e87abe6cccb51440b710bc32/GPGHMACSHA1--498faf23d8f11a54e87abe6cccb51440b710bc32 +deleting annex/objects/247/654/GPGHMACSHA1--498faf23d8f11a54e87abe6cccb51440b710bc32/ +deleting annex/objects/247/654/ +deleting annex/objects/247/64f/GPGHMACSHA1--1e7b26f9b7b50d57c892214edce187576980ed1f/GPGHMACSHA1--1e7b26f9b7b50d57c892214edce187576980ed1f +deleting annex/objects/247/64f/GPGHMACSHA1--1e7b26f9b7b50d57c892214edce187576980ed1f/ +deleting annex/objects/247/64f/ +deleting annex/objects/247/64c/GPGHMACSHA1--37e007dbf5fab4fac696086051aa2169bcc6304c/GPGHMACSHA1--37e007dbf5fab4fac696086051aa2169bcc6304c +deleting annex/objects/247/64c/GPGHMACSHA1--37e007dbf5fab4fac696086051aa2169bcc6304c/ +deleting annex/objects/247/64c/GPGHMACSHA1--2cd375aad0e7822c9a6737ab3ac8bd2334a6360f/GPGHMACSHA1--2cd375aad0e7822c9a6737ab3ac8bd2334a6360f +deleting annex/objects/247/64c/GPGHMACSHA1--2cd375aad0e7822c9a6737ab3ac8bd2334a6360f/ +deleting annex/objects/247/64c/ +deleting annex/objects/247/648/GPGHMACSHA1--f22386dec26dd63c7c9021afe20dee4b5de717e2/GPGHMACSHA1--f22386dec26dd63c7c9021afe20dee4b5de717e2 +deleting annex/objects/247/648/GPGHMACSHA1--f22386dec26dd63c7c9021afe20dee4b5de717e2/ +deleting annex/objects/247/648/ +deleting annex/objects/247/646/GPGHMACSHA1--482d0374847fa9150981efb9381e47a05943e217/GPGHMACSHA1--482d0374847fa9150981efb9381e47a05943e217 +deleting annex/objects/247/646/GPGHMACSHA1--482d0374847fa9150981efb9381e47a05943e217/ +deleting annex/objects/247/646/ +deleting annex/objects/247/645/GPGHMACSHA1--eb7c3bea5737f53afeb5809036b6e1ab2b7095de/GPGHMACSHA1--eb7c3bea5737f53afeb5809036b6e1ab2b7095de +deleting annex/objects/247/645/GPGHMACSHA1--eb7c3bea5737f53afeb5809036b6e1ab2b7095de/ +deleting annex/objects/247/645/ +deleting annex/objects/247/644/GPGHMACSHA1--3a35df0f925001772f434a66bab38fda007c8f35/GPGHMACSHA1--3a35df0f925001772f434a66bab38fda007c8f35 +deleting annex/objects/247/644/GPGHMACSHA1--3a35df0f925001772f434a66bab38fda007c8f35/ +deleting annex/objects/247/644/ +deleting annex/objects/247/63d/GPGHMACSHA1--eebadfddbeb809ff5d55cf8aa0439dd0fe61f7a6/GPGHMACSHA1--eebadfddbeb809ff5d55cf8aa0439dd0fe61f7a6 +deleting annex/objects/247/63d/GPGHMACSHA1--eebadfddbeb809ff5d55cf8aa0439dd0fe61f7a6/ +deleting annex/objects/247/63d/GPGHMACSHA1--bc76727a1097942a34bda9f354110cc9b8cfa256/GPGHMACSHA1--bc76727a1097942a34bda9f354110cc9b8cfa256 +deleting annex/objects/247/63d/GPGHMACSHA1--bc76727a1097942a34bda9f354110cc9b8cfa256/ +deleting annex/objects/247/63d/ +deleting annex/objects/247/63a/GPGHMACSHA1--cf9adc3c0d313b6df391b07588ecc10e07ca0644/GPGHMACSHA1--cf9adc3c0d313b6df391b07588ecc10e07ca0644 +deleting annex/objects/247/63a/GPGHMACSHA1--cf9adc3c0d313b6df391b07588ecc10e07ca0644/ +deleting annex/objects/247/63a/GPGHMACSHA1--7fb424869aea1b4d412e06d2279d63129d51fa12/GPGHMACSHA1--7fb424869aea1b4d412e06d2279d63129d51fa12 +deleting annex/objects/247/63a/GPGHMACSHA1--7fb424869aea1b4d412e06d2279d63129d51fa12/ +deleting annex/objects/247/63a/ +deleting annex/objects/247/637/GPGHMACSHA1--3ca1532c0344187fba45bf9ae646552dd34308d5/GPGHMACSHA1--3ca1532c0344187fba45bf9ae646552dd34308d5 +deleting annex/objects/247/637/GPGHMACSHA1--3ca1532c0344187fba45bf9ae646552dd34308d5/ +deleting annex/objects/247/637/ +deleting annex/objects/247/62f/GPGHMACSHA1--e996a49bcb63458c04794c3bbf8cdee664a04f21/GPGHMACSHA1--e996a49bcb63458c04794c3bbf8cdee664a04f21 +deleting annex/objects/247/62f/GPGHMACSHA1--e996a49bcb63458c04794c3bbf8cdee664a04f21/ +deleting annex/objects/247/62f/ +deleting annex/objects/247/629/GPGHMACSHA1--92e137890d8741d30524b6007f601ed374f69156/GPGHMACSHA1--92e137890d8741d30524b6007f601ed374f69156 +deleting annex/objects/247/629/GPGHMACSHA1--92e137890d8741d30524b6007f601ed374f69156/ +deleting annex/objects/247/629/ +deleting annex/objects/247/627/GPGHMACSHA1--9eb547817aebf490e2a5697be813585d9af44ad3/GPGHMACSHA1--9eb547817aebf490e2a5697be813585d9af44ad3 +deleting annex/objects/247/627/GPGHMACSHA1--9eb547817aebf490e2a5697be813585d9af44ad3/ +deleting annex/objects/247/627/ +deleting annex/objects/247/625/GPGHMACSHA1--16f964b4179c1e0158858aba33cc3ae5a9a61ba7/GPGHMACSHA1--16f964b4179c1e0158858aba33cc3ae5a9a61ba7 +deleting annex/objects/247/625/GPGHMACSHA1--16f964b4179c1e0158858aba33cc3ae5a9a61ba7/ +deleting annex/objects/247/625/ +deleting annex/objects/247/620/GPGHMACSHA1--e7077baa4fbe2f0500d185aeb39c21f1d958b023/GPGHMACSHA1--e7077baa4fbe2f0500d185aeb39c21f1d958b023 +deleting annex/objects/247/620/GPGHMACSHA1--e7077baa4fbe2f0500d185aeb39c21f1d958b023/ +deleting annex/objects/247/620/GPGHMACSHA1--322bdb40e4c644585673d013b9d87f3983fa5c34/GPGHMACSHA1--322bdb40e4c644585673d013b9d87f3983fa5c34 +deleting annex/objects/247/620/GPGHMACSHA1--322bdb40e4c644585673d013b9d87f3983fa5c34/ +deleting annex/objects/247/620/ +deleting annex/objects/247/61d/GPGHMACSHA1--ca424bc0cd5be896d5cdfbd0df4f636cfffb669c/GPGHMACSHA1--ca424bc0cd5be896d5cdfbd0df4f636cfffb669c +deleting annex/objects/247/61d/GPGHMACSHA1--ca424bc0cd5be896d5cdfbd0df4f636cfffb669c/ +deleting annex/objects/247/61d/ +deleting annex/objects/247/61c/GPGHMACSHA1--73391c35dec76d791da13d6c73366cfb69d47909/GPGHMACSHA1--73391c35dec76d791da13d6c73366cfb69d47909 +deleting annex/objects/247/61c/GPGHMACSHA1--73391c35dec76d791da13d6c73366cfb69d47909/ +deleting annex/objects/247/61c/ +deleting annex/objects/247/61a/GPGHMACSHA1--f0bce1214699a5ef4da278476bec5fa67e65b116/GPGHMACSHA1--f0bce1214699a5ef4da278476bec5fa67e65b116 +deleting annex/objects/247/61a/GPGHMACSHA1--f0bce1214699a5ef4da278476bec5fa67e65b116/ +deleting annex/objects/247/61a/ +deleting annex/objects/247/615/GPGHMACSHA1--39573095483636b80a74d03cca58bda7f17bb2e6/GPGHMACSHA1--39573095483636b80a74d03cca58bda7f17bb2e6 +deleting annex/objects/247/615/GPGHMACSHA1--39573095483636b80a74d03cca58bda7f17bb2e6/ +deleting annex/objects/247/615/ +deleting annex/objects/247/612/GPGHMACSHA1--f17bdc3048e38a0bdb7284005175bc89e3f66f26/GPGHMACSHA1--f17bdc3048e38a0bdb7284005175bc89e3f66f26 +deleting annex/objects/247/612/GPGHMACSHA1--f17bdc3048e38a0bdb7284005175bc89e3f66f26/ +deleting annex/objects/247/612/ +deleting annex/objects/247/60e/GPGHMACSHA1--b0dfa110efc135a31852b7fdbc24a2620b4327e3/GPGHMACSHA1--b0dfa110efc135a31852b7fdbc24a2620b4327e3 +deleting annex/objects/247/60e/GPGHMACSHA1--b0dfa110efc135a31852b7fdbc24a2620b4327e3/ +deleting annex/objects/247/60e/ +deleting annex/objects/247/60b/GPGHMACSHA1--afa4a2aaeeaec2aa4976babc9256b0109d28ecc0/GPGHMACSHA1--afa4a2aaeeaec2aa4976babc9256b0109d28ecc0 +deleting annex/objects/247/60b/GPGHMACSHA1--afa4a2aaeeaec2aa4976babc9256b0109d28ecc0/ +deleting annex/objects/247/60b/GPGHMACSHA1--95d9760a4a05b110ccc307d3348c9cc9a462c0b9/GPGHMACSHA1--95d9760a4a05b110ccc307d3348c9cc9a462c0b9 +deleting annex/objects/247/60b/GPGHMACSHA1--95d9760a4a05b110ccc307d3348c9cc9a462c0b9/ +deleting annex/objects/247/60b/ +deleting annex/objects/247/609/GPGHMACSHA1--25b90316b6158121552060cdefdb5bc129ad5058/GPGHMACSHA1--25b90316b6158121552060cdefdb5bc129ad5058 +deleting annex/objects/247/609/GPGHMACSHA1--25b90316b6158121552060cdefdb5bc129ad5058/ +deleting annex/objects/247/609/ +deleting annex/objects/247/606/GPGHMACSHA1--a0b9667477be6bd1eb475b5ace126299015040c3/GPGHMACSHA1--a0b9667477be6bd1eb475b5ace126299015040c3 +deleting annex/objects/247/606/GPGHMACSHA1--a0b9667477be6bd1eb475b5ace126299015040c3/ +deleting annex/objects/247/606/ +deleting annex/objects/247/605/GPGHMACSHA1--77632b971059808457b15646631cd36762671a54/GPGHMACSHA1--77632b971059808457b15646631cd36762671a54 +deleting annex/objects/247/605/GPGHMACSHA1--77632b971059808457b15646631cd36762671a54/ +deleting annex/objects/247/605/ +deleting annex/objects/247/601/GPGHMACSHA1--258d7701d73c02a876de8487d4f264dcbba0ea97/GPGHMACSHA1--258d7701d73c02a876de8487d4f264dcbba0ea97 +deleting annex/objects/247/601/GPGHMACSHA1--258d7701d73c02a876de8487d4f264dcbba0ea97/ +deleting annex/objects/247/601/ +deleting annex/objects/247/5ff/GPGHMACSHA1--24b943db831c32e41d0c33e7fae234fbda2ea504/GPGHMACSHA1--24b943db831c32e41d0c33e7fae234fbda2ea504 +deleting annex/objects/247/5ff/GPGHMACSHA1--24b943db831c32e41d0c33e7fae234fbda2ea504/ +deleting annex/objects/247/5ff/ +deleting annex/objects/247/5fe/GPGHMACSHA1--6aa35048e84818df436db277a8913d257235a013/GPGHMACSHA1--6aa35048e84818df436db277a8913d257235a013 +deleting annex/objects/247/5fe/GPGHMACSHA1--6aa35048e84818df436db277a8913d257235a013/ +deleting annex/objects/247/5fe/GPGHMACSHA1--27724d561b47f0506c47b1c09c5a0712dc77c9ba/GPGHMACSHA1--27724d561b47f0506c47b1c09c5a0712dc77c9ba +deleting annex/objects/247/5fe/GPGHMACSHA1--27724d561b47f0506c47b1c09c5a0712dc77c9ba/ +deleting annex/objects/247/5fe/ +deleting annex/objects/247/5ed/GPGHMACSHA1--d68b1377a5b3ae8715cf47698eef3ba7c7303561/GPGHMACSHA1--d68b1377a5b3ae8715cf47698eef3ba7c7303561 +deleting annex/objects/247/5ed/GPGHMACSHA1--d68b1377a5b3ae8715cf47698eef3ba7c7303561/ +deleting annex/objects/247/5ed/ +deleting annex/objects/247/5ea/GPGHMACSHA1--a578150a85f221b702c77c5b2ba3545c6d48eeb2/GPGHMACSHA1--a578150a85f221b702c77c5b2ba3545c6d48eeb2 +deleting annex/objects/247/5ea/GPGHMACSHA1--a578150a85f221b702c77c5b2ba3545c6d48eeb2/ +deleting annex/objects/247/5ea/ +deleting annex/objects/247/5de/GPGHMACSHA1--9eb7179e1cae9996c99ecc98d24af21501bcf631/GPGHMACSHA1--9eb7179e1cae9996c99ecc98d24af21501bcf631 +deleting annex/objects/247/5de/GPGHMACSHA1--9eb7179e1cae9996c99ecc98d24af21501bcf631/ +deleting annex/objects/247/5de/ +deleting annex/objects/247/5dd/GPGHMACSHA1--7c13f08a629f01bd310999f196744a624a3cf1e9/GPGHMACSHA1--7c13f08a629f01bd310999f196744a624a3cf1e9 +deleting annex/objects/247/5dd/GPGHMACSHA1--7c13f08a629f01bd310999f196744a624a3cf1e9/ +deleting annex/objects/247/5dd/ +deleting annex/objects/247/5da/GPGHMACSHA1--afdda198af7d490f7c8b13f38575ca79edc49635/GPGHMACSHA1--afdda198af7d490f7c8b13f38575ca79edc49635 +deleting annex/objects/247/5da/GPGHMACSHA1--afdda198af7d490f7c8b13f38575ca79edc49635/ +deleting annex/objects/247/5da/ +deleting annex/objects/247/5d6/GPGHMACSHA1--e3c008e950edf32969319f9212734b0e9e8cc84c/GPGHMACSHA1--e3c008e950edf32969319f9212734b0e9e8cc84c +deleting annex/objects/247/5d6/GPGHMACSHA1--e3c008e950edf32969319f9212734b0e9e8cc84c/ +deleting annex/objects/247/5d6/ +deleting annex/objects/247/5d0/GPGHMACSHA1--44bfcd71b7c297425b3b5d9896975a38c3d0247c/GPGHMACSHA1--44bfcd71b7c297425b3b5d9896975a38c3d0247c +deleting annex/objects/247/5d0/GPGHMACSHA1--44bfcd71b7c297425b3b5d9896975a38c3d0247c/ +deleting annex/objects/247/5d0/GPGHMACSHA1--0591aa70c32c6b29acdd179e8b8b03d177a37abe/GPGHMACSHA1--0591aa70c32c6b29acdd179e8b8b03d177a37abe +deleting annex/objects/247/5d0/GPGHMACSHA1--0591aa70c32c6b29acdd179e8b8b03d177a37abe/ +deleting annex/objects/247/5d0/ +deleting annex/objects/247/5ca/GPGHMACSHA1--aefb2975a65578696558f3799bb672c203dc1fe5/GPGHMACSHA1--aefb2975a65578696558f3799bb672c203dc1fe5 +deleting annex/objects/247/5ca/GPGHMACSHA1--aefb2975a65578696558f3799bb672c203dc1fe5/ +deleting annex/objects/247/5ca/ +deleting annex/objects/247/5c5/GPGHMACSHA1--faec64b5e4f623f0ae03b899904f0f1e82a5d93d/GPGHMACSHA1--faec64b5e4f623f0ae03b899904f0f1e82a5d93d +deleting annex/objects/247/5c5/GPGHMACSHA1--faec64b5e4f623f0ae03b899904f0f1e82a5d93d/ +deleting annex/objects/247/5c5/GPGHMACSHA1--35011afcb74beb2f0837f10db3bf649b3de30b69/GPGHMACSHA1--35011afcb74beb2f0837f10db3bf649b3de30b69 +deleting annex/objects/247/5c5/GPGHMACSHA1--35011afcb74beb2f0837f10db3bf649b3de30b69/ +deleting annex/objects/247/5c5/ +deleting annex/objects/247/5c4/GPGHMACSHA1--ee2ab252d5952dda2d48fdca5f91d3de517d0d1d/GPGHMACSHA1--ee2ab252d5952dda2d48fdca5f91d3de517d0d1d +deleting annex/objects/247/5c4/GPGHMACSHA1--ee2ab252d5952dda2d48fdca5f91d3de517d0d1d/ +deleting annex/objects/247/5c4/GPGHMACSHA1--7412794e81ae37644ed7a40ff2ddaa9a4b2b8cf1/GPGHMACSHA1--7412794e81ae37644ed7a40ff2ddaa9a4b2b8cf1 +deleting annex/objects/247/5c4/GPGHMACSHA1--7412794e81ae37644ed7a40ff2ddaa9a4b2b8cf1/ +deleting annex/objects/247/5c4/ +deleting annex/objects/247/5c3/GPGHMACSHA1--13cf2d1f27a52a082c47e4b2c734dc37f0bb655a/GPGHMACSHA1--13cf2d1f27a52a082c47e4b2c734dc37f0bb655a +deleting annex/objects/247/5c3/GPGHMACSHA1--13cf2d1f27a52a082c47e4b2c734dc37f0bb655a/ +deleting annex/objects/247/5c3/ +deleting annex/objects/247/5bf/GPGHMACSHA1--6878e81dbb9ab019333806d0195b7a1426083e04/GPGHMACSHA1--6878e81dbb9ab019333806d0195b7a1426083e04 +deleting annex/objects/247/5bf/GPGHMACSHA1--6878e81dbb9ab019333806d0195b7a1426083e04/ +deleting annex/objects/247/5bf/GPGHMACSHA1--2095360e977b5cd9b8b6455faaa49f01105941ff/GPGHMACSHA1--2095360e977b5cd9b8b6455faaa49f01105941ff +deleting annex/objects/247/5bf/GPGHMACSHA1--2095360e977b5cd9b8b6455faaa49f01105941ff/ +deleting annex/objects/247/5bf/ +deleting annex/objects/247/5b7/GPGHMACSHA1--4f6f5e205ee81e50e59c3974a4874710e5b014e9/GPGHMACSHA1--4f6f5e205ee81e50e59c3974a4874710e5b014e9 +deleting annex/objects/247/5b7/GPGHMACSHA1--4f6f5e205ee81e50e59c3974a4874710e5b014e9/ +deleting annex/objects/247/5b7/ +deleting annex/objects/a58/49b/GPGHMACSHA1--e3aeb454f526e124fdfe09bbe2dde67a9014fdfd/GPGHMACSHA1--e3aeb454f526e124fdfe09bbe2dde67a9014fdfd +deleting annex/objects/a58/49b/GPGHMACSHA1--e3aeb454f526e124fdfe09bbe2dde67a9014fdfd/ +deleting annex/objects/a58/49b/ +deleting annex/objects/a58/499/GPGHMACSHA1--fc6bb70d08c0f3ff06d117c09895f08bc57e15ae/GPGHMACSHA1--fc6bb70d08c0f3ff06d117c09895f08bc57e15ae +deleting annex/objects/a58/499/GPGHMACSHA1--fc6bb70d08c0f3ff06d117c09895f08bc57e15ae/ +deleting annex/objects/a58/499/GPGHMACSHA1--b17fe85d9f9eba3246cd967ece843d811b7c8eba/GPGHMACSHA1--b17fe85d9f9eba3246cd967ece843d811b7c8eba +deleting annex/objects/a58/499/GPGHMACSHA1--b17fe85d9f9eba3246cd967ece843d811b7c8eba/ +deleting annex/objects/a58/499/GPGHMACSHA1--0e0f301d6a0eb570f9836c3c50bb521726f575e2/GPGHMACSHA1--0e0f301d6a0eb570f9836c3c50bb521726f575e2 +deleting annex/objects/a58/499/GPGHMACSHA1--0e0f301d6a0eb570f9836c3c50bb521726f575e2/ +deleting annex/objects/a58/499/ +deleting annex/objects/a58/498/GPGHMACSHA1--50f574b0218714803a3594587afebb89e4695a3b/GPGHMACSHA1--50f574b0218714803a3594587afebb89e4695a3b +deleting annex/objects/a58/498/GPGHMACSHA1--50f574b0218714803a3594587afebb89e4695a3b/ +deleting annex/objects/a58/498/GPGHMACSHA1--36891f54eca4aa3f1f9dd7136a1cb0772656904f/GPGHMACSHA1--36891f54eca4aa3f1f9dd7136a1cb0772656904f +deleting annex/objects/a58/498/GPGHMACSHA1--36891f54eca4aa3f1f9dd7136a1cb0772656904f/ +deleting annex/objects/a58/498/ +deleting annex/objects/a58/497/GPGHMACSHA1--cd7e72aef9d4418bb7cee56627ccb30daba6a825/GPGHMACSHA1--cd7e72aef9d4418bb7cee56627ccb30daba6a825 +deleting annex/objects/a58/497/GPGHMACSHA1--cd7e72aef9d4418bb7cee56627ccb30daba6a825/ +deleting annex/objects/a58/497/GPGHMACSHA1--7e19c3642b6d48af9e07ff751c5dfb4478bd881a/GPGHMACSHA1--7e19c3642b6d48af9e07ff751c5dfb4478bd881a +deleting annex/objects/a58/497/GPGHMACSHA1--7e19c3642b6d48af9e07ff751c5dfb4478bd881a/ +deleting annex/objects/a58/497/GPGHMACSHA1--4f40a7da04b6eeed622bdff1d2dc209e2874f175/GPGHMACSHA1--4f40a7da04b6eeed622bdff1d2dc209e2874f175 +deleting annex/objects/a58/497/GPGHMACSHA1--4f40a7da04b6eeed622bdff1d2dc209e2874f175/ +deleting annex/objects/a58/497/ +deleting annex/objects/a58/496/GPGHMACSHA1--d9126857ed0f79cc5ffc9230a7bb570e35b9e353/GPGHMACSHA1--d9126857ed0f79cc5ffc9230a7bb570e35b9e353 +deleting annex/objects/a58/496/GPGHMACSHA1--d9126857ed0f79cc5ffc9230a7bb570e35b9e353/ +deleting annex/objects/a58/496/GPGHMACSHA1--9946337bfe9fc8592ce225ad6c4b6dbce07a7695/GPGHMACSHA1--9946337bfe9fc8592ce225ad6c4b6dbce07a7695 +deleting annex/objects/a58/496/GPGHMACSHA1--9946337bfe9fc8592ce225ad6c4b6dbce07a7695/ +deleting annex/objects/a58/496/GPGHMACSHA1--8cb939761c19b8149cf5b423c10838c77848fc0f/GPGHMACSHA1--8cb939761c19b8149cf5b423c10838c77848fc0f +deleting annex/objects/a58/496/GPGHMACSHA1--8cb939761c19b8149cf5b423c10838c77848fc0f/ +deleting annex/objects/a58/496/ +deleting annex/objects/a58/495/GPGHMACSHA1--a743ae9178e36114af219c196a7873282a652d9c/GPGHMACSHA1--a743ae9178e36114af219c196a7873282a652d9c +deleting annex/objects/a58/495/GPGHMACSHA1--a743ae9178e36114af219c196a7873282a652d9c/ +deleting annex/objects/a58/495/ +deleting annex/objects/a58/493/GPGHMACSHA1--b4839bc63d53abe5d16723599b04b75bbbb78ff6/GPGHMACSHA1--b4839bc63d53abe5d16723599b04b75bbbb78ff6 +deleting annex/objects/a58/493/GPGHMACSHA1--b4839bc63d53abe5d16723599b04b75bbbb78ff6/ +deleting annex/objects/a58/493/GPGHMACSHA1--6c4c2f566fe7d835152ff6cbfd33781e18991449/GPGHMACSHA1--6c4c2f566fe7d835152ff6cbfd33781e18991449 +deleting annex/objects/a58/493/GPGHMACSHA1--6c4c2f566fe7d835152ff6cbfd33781e18991449/ +deleting annex/objects/a58/493/GPGHMACSHA1--1b97a7deb66b13d8582b5b9551db8f0b6fa394a8/GPGHMACSHA1--1b97a7deb66b13d8582b5b9551db8f0b6fa394a8 +deleting annex/objects/a58/493/GPGHMACSHA1--1b97a7deb66b13d8582b5b9551db8f0b6fa394a8/ +deleting annex/objects/a58/493/ +deleting annex/objects/a58/492/GPGHMACSHA1--dae61e12abbf5bfc020f413b37ab97e1482e86b2/GPGHMACSHA1--dae61e12abbf5bfc020f413b37ab97e1482e86b2 +deleting annex/objects/a58/492/GPGHMACSHA1--dae61e12abbf5bfc020f413b37ab97e1482e86b2/ +deleting annex/objects/a58/492/GPGHMACSHA1--21e559dd65b2f903806dc81d45cee9b4122dffd9/GPGHMACSHA1--21e559dd65b2f903806dc81d45cee9b4122dffd9 +deleting annex/objects/a58/492/GPGHMACSHA1--21e559dd65b2f903806dc81d45cee9b4122dffd9/ +deleting annex/objects/a58/492/GPGHMACSHA1--0d9c1738001301c60a0c6b279e4fcaf7b72e2d60/GPGHMACSHA1--0d9c1738001301c60a0c6b279e4fcaf7b72e2d60 +deleting annex/objects/a58/492/GPGHMACSHA1--0d9c1738001301c60a0c6b279e4fcaf7b72e2d60/ +deleting annex/objects/a58/492/ +deleting annex/objects/a58/490/GPGHMACSHA1--16720b93fc8677bf2bb664efd53376ee50368f2a/GPGHMACSHA1--16720b93fc8677bf2bb664efd53376ee50368f2a +deleting annex/objects/a58/490/GPGHMACSHA1--16720b93fc8677bf2bb664efd53376ee50368f2a/ +deleting annex/objects/a58/490/ +deleting annex/objects/a58/48d/GPGHMACSHA1--16a4801a94772f419bdd8d7545e4dc62cf601a73/GPGHMACSHA1--16a4801a94772f419bdd8d7545e4dc62cf601a73 +deleting annex/objects/a58/48d/GPGHMACSHA1--16a4801a94772f419bdd8d7545e4dc62cf601a73/ +deleting annex/objects/a58/48d/ +deleting annex/objects/a58/48c/GPGHMACSHA1--11c14032c5f005a858cb842b7f83c44c29a7fc4f/GPGHMACSHA1--11c14032c5f005a858cb842b7f83c44c29a7fc4f +deleting annex/objects/a58/48c/GPGHMACSHA1--11c14032c5f005a858cb842b7f83c44c29a7fc4f/ +deleting annex/objects/a58/48c/ +deleting annex/objects/a58/48b/GPGHMACSHA1--e7fc68e6cd6cd677a780b9fc9403e04d87490837/GPGHMACSHA1--e7fc68e6cd6cd677a780b9fc9403e04d87490837 +deleting annex/objects/a58/48b/GPGHMACSHA1--e7fc68e6cd6cd677a780b9fc9403e04d87490837/ +deleting annex/objects/a58/48b/GPGHMACSHA1--506b108c0a1cbacccebc647f40747ff79f5f836d/GPGHMACSHA1--506b108c0a1cbacccebc647f40747ff79f5f836d +deleting annex/objects/a58/48b/GPGHMACSHA1--506b108c0a1cbacccebc647f40747ff79f5f836d/ +deleting annex/objects/a58/48b/ +deleting annex/objects/a58/48a/GPGHMACSHA1--d3ee658e9845bf5e14e2835419ec4445b33093de/GPGHMACSHA1--d3ee658e9845bf5e14e2835419ec4445b33093de +deleting annex/objects/a58/48a/GPGHMACSHA1--d3ee658e9845bf5e14e2835419ec4445b33093de/ +deleting annex/objects/a58/48a/GPGHMACSHA1--42bffe3ea217b30b19af464d48916dd001c38ad2/GPGHMACSHA1--42bffe3ea217b30b19af464d48916dd001c38ad2 +deleting annex/objects/a58/48a/GPGHMACSHA1--42bffe3ea217b30b19af464d48916dd001c38ad2/ +deleting annex/objects/a58/48a/ +deleting annex/objects/a58/489/GPGHMACSHA1--cd3ef969340e4ad704b757d0a42a412abbfa492c/GPGHMACSHA1--cd3ef969340e4ad704b757d0a42a412abbfa492c +deleting annex/objects/a58/489/GPGHMACSHA1--cd3ef969340e4ad704b757d0a42a412abbfa492c/ +deleting annex/objects/a58/489/ +deleting annex/objects/a58/488/GPGHMACSHA1--ed8b48a06919142fbf036d839f69edc15e4814e3/GPGHMACSHA1--ed8b48a06919142fbf036d839f69edc15e4814e3 +deleting annex/objects/a58/488/GPGHMACSHA1--ed8b48a06919142fbf036d839f69edc15e4814e3/ +deleting annex/objects/a58/488/GPGHMACSHA1--e4738706071a30af1f985c148d050a9deed2761e/GPGHMACSHA1--e4738706071a30af1f985c148d050a9deed2761e +deleting annex/objects/a58/488/GPGHMACSHA1--e4738706071a30af1f985c148d050a9deed2761e/ +deleting annex/objects/a58/488/GPGHMACSHA1--0c3ce03cff065e2223e3da9c3fd6842b0ac91730/GPGHMACSHA1--0c3ce03cff065e2223e3da9c3fd6842b0ac91730 +deleting annex/objects/a58/488/GPGHMACSHA1--0c3ce03cff065e2223e3da9c3fd6842b0ac91730/ +deleting annex/objects/a58/488/ +deleting annex/objects/a58/486/GPGHMACSHA1--f3730f7afb8381612ab6d06ec3cc36c023a3142c/GPGHMACSHA1--f3730f7afb8381612ab6d06ec3cc36c023a3142c +deleting annex/objects/a58/486/GPGHMACSHA1--f3730f7afb8381612ab6d06ec3cc36c023a3142c/ +deleting annex/objects/a58/486/GPGHMACSHA1--dd16892e59f6d7ae3529dbdad64291e9596dcf0e/GPGHMACSHA1--dd16892e59f6d7ae3529dbdad64291e9596dcf0e +deleting annex/objects/a58/486/GPGHMACSHA1--dd16892e59f6d7ae3529dbdad64291e9596dcf0e/ +deleting annex/objects/a58/486/ +deleting annex/objects/a58/485/GPGHMACSHA1--c4305419b6756056fba7c37fb0d172d2901e5c9d/GPGHMACSHA1--c4305419b6756056fba7c37fb0d172d2901e5c9d +deleting annex/objects/a58/485/GPGHMACSHA1--c4305419b6756056fba7c37fb0d172d2901e5c9d/ +deleting annex/objects/a58/485/GPGHMACSHA1--bca48d2ccbadc42c504d46d247439f7f62ce84a3/GPGHMACSHA1--bca48d2ccbadc42c504d46d247439f7f62ce84a3 +deleting annex/objects/a58/485/GPGHMACSHA1--bca48d2ccbadc42c504d46d247439f7f62ce84a3/ +deleting annex/objects/a58/485/GPGHMACSHA1--83ff887d652f832f192821adcd584c6b7aea8d29/GPGHMACSHA1--83ff887d652f832f192821adcd584c6b7aea8d29 +deleting annex/objects/a58/485/GPGHMACSHA1--83ff887d652f832f192821adcd584c6b7aea8d29/ +deleting annex/objects/a58/485/ +deleting annex/objects/a58/484/GPGHMACSHA1--3b351e040554f1b968ec6e42bfec68481241211a/GPGHMACSHA1--3b351e040554f1b968ec6e42bfec68481241211a +deleting annex/objects/a58/484/GPGHMACSHA1--3b351e040554f1b968ec6e42bfec68481241211a/ +deleting annex/objects/a58/484/ +deleting annex/objects/a58/483/GPGHMACSHA1--b6ab476c95e3cdfe08ae45fad084bc1ee3264b2c/GPGHMACSHA1--b6ab476c95e3cdfe08ae45fad084bc1ee3264b2c +deleting annex/objects/a58/483/GPGHMACSHA1--b6ab476c95e3cdfe08ae45fad084bc1ee3264b2c/ +deleting annex/objects/a58/483/GPGHMACSHA1--5f9bc6e525965d8e5b14c8067a2102ac579b2224/GPGHMACSHA1--5f9bc6e525965d8e5b14c8067a2102ac579b2224 +deleting annex/objects/a58/483/GPGHMACSHA1--5f9bc6e525965d8e5b14c8067a2102ac579b2224/ +deleting annex/objects/a58/483/GPGHMACSHA1--4da9c8cc1c7c48333d862f61dac3d34a9bc425f7/GPGHMACSHA1--4da9c8cc1c7c48333d862f61dac3d34a9bc425f7 +deleting annex/objects/a58/483/GPGHMACSHA1--4da9c8cc1c7c48333d862f61dac3d34a9bc425f7/ +deleting annex/objects/a58/483/GPGHMACSHA1--40d3736c7067fca2c881e67528598bdcddb6635f/GPGHMACSHA1--40d3736c7067fca2c881e67528598bdcddb6635f +deleting annex/objects/a58/483/GPGHMACSHA1--40d3736c7067fca2c881e67528598bdcddb6635f/ +deleting annex/objects/a58/483/GPGHMACSHA1--241e68f1062d07c2014e5171dee812a7be48c517/GPGHMACSHA1--241e68f1062d07c2014e5171dee812a7be48c517 +deleting annex/objects/a58/483/GPGHMACSHA1--241e68f1062d07c2014e5171dee812a7be48c517/ +deleting annex/objects/a58/483/ +deleting annex/objects/a58/481/GPGHMACSHA1--4833d1788ca34a936f7f4e3f7cdeca332d9bfcc5/GPGHMACSHA1--4833d1788ca34a936f7f4e3f7cdeca332d9bfcc5 +deleting annex/objects/a58/481/GPGHMACSHA1--4833d1788ca34a936f7f4e3f7cdeca332d9bfcc5/ +deleting annex/objects/a58/481/ +deleting annex/objects/a58/480/GPGHMACSHA1--f18822b057d7e7ea0c8500ce2489fbbdb8de70cb/GPGHMACSHA1--f18822b057d7e7ea0c8500ce2489fbbdb8de70cb +deleting annex/objects/a58/480/GPGHMACSHA1--f18822b057d7e7ea0c8500ce2489fbbdb8de70cb/ +deleting annex/objects/a58/480/GPGHMACSHA1--0230d274cb8d0d80efa07f466a5a6374cc308225/GPGHMACSHA1--0230d274cb8d0d80efa07f466a5a6374cc308225 +deleting annex/objects/a58/480/GPGHMACSHA1--0230d274cb8d0d80efa07f466a5a6374cc308225/ +deleting annex/objects/a58/480/ +deleting annex/objects/a58/47f/GPGHMACSHA1--0669b2f40a41c016608b04aa81f54a5b4d9210d7/GPGHMACSHA1--0669b2f40a41c016608b04aa81f54a5b4d9210d7 +deleting annex/objects/a58/47f/GPGHMACSHA1--0669b2f40a41c016608b04aa81f54a5b4d9210d7/ +deleting annex/objects/a58/47f/ +deleting annex/objects/a58/47d/GPGHMACSHA1--8fba4badeefd2fb38965e017c1c52196b212ad35/GPGHMACSHA1--8fba4badeefd2fb38965e017c1c52196b212ad35 +deleting annex/objects/a58/47d/GPGHMACSHA1--8fba4badeefd2fb38965e017c1c52196b212ad35/ +deleting annex/objects/a58/47d/GPGHMACSHA1--215ade6c261593b1911f29d633d2bdc1dbaaeedc/GPGHMACSHA1--215ade6c261593b1911f29d633d2bdc1dbaaeedc +deleting annex/objects/a58/47d/GPGHMACSHA1--215ade6c261593b1911f29d633d2bdc1dbaaeedc/ +deleting annex/objects/a58/47d/GPGHMACSHA1--148038e297f562e8ae6baaad0cf90c2e9f55c80e/GPGHMACSHA1--148038e297f562e8ae6baaad0cf90c2e9f55c80e +deleting annex/objects/a58/47d/GPGHMACSHA1--148038e297f562e8ae6baaad0cf90c2e9f55c80e/ +deleting annex/objects/a58/47d/ +deleting annex/objects/a58/47b/GPGHMACSHA1--ecad287be2fff9a89847c8b73ab85d91efbbb1b6/GPGHMACSHA1--ecad287be2fff9a89847c8b73ab85d91efbbb1b6 +deleting annex/objects/a58/47b/GPGHMACSHA1--ecad287be2fff9a89847c8b73ab85d91efbbb1b6/ +deleting annex/objects/a58/47b/GPGHMACSHA1--920a442140a8f0fc358285be7834674d00e05e73/GPGHMACSHA1--920a442140a8f0fc358285be7834674d00e05e73 +deleting annex/objects/a58/47b/GPGHMACSHA1--920a442140a8f0fc358285be7834674d00e05e73/ +deleting annex/objects/a58/47b/GPGHMACSHA1--0fa27452aaa9359a79918d3f3fd1e197a0693243/GPGHMACSHA1--0fa27452aaa9359a79918d3f3fd1e197a0693243 +deleting annex/objects/a58/47b/GPGHMACSHA1--0fa27452aaa9359a79918d3f3fd1e197a0693243/ +deleting annex/objects/a58/47b/ +deleting annex/objects/a58/47a/GPGHMACSHA1--2f2f58949389afae2f43f64931fc6ebc4f96b717/GPGHMACSHA1--2f2f58949389afae2f43f64931fc6ebc4f96b717 +deleting annex/objects/a58/47a/GPGHMACSHA1--2f2f58949389afae2f43f64931fc6ebc4f96b717/ +deleting annex/objects/a58/47a/ +deleting annex/objects/a58/479/GPGHMACSHA1--e138f073472b08f92fe60252c837de7dcef2d97e/GPGHMACSHA1--e138f073472b08f92fe60252c837de7dcef2d97e +deleting annex/objects/a58/479/GPGHMACSHA1--e138f073472b08f92fe60252c837de7dcef2d97e/ +deleting annex/objects/a58/479/GPGHMACSHA1--d2077fcbdd85b264cd9ace6481e34fa39e39027d/GPGHMACSHA1--d2077fcbdd85b264cd9ace6481e34fa39e39027d +deleting annex/objects/a58/479/GPGHMACSHA1--d2077fcbdd85b264cd9ace6481e34fa39e39027d/ +deleting annex/objects/a58/479/GPGHMACSHA1--2d7ada7e883a149c6786714ffe8b0845c22dfa3b/GPGHMACSHA1--2d7ada7e883a149c6786714ffe8b0845c22dfa3b +deleting annex/objects/a58/479/GPGHMACSHA1--2d7ada7e883a149c6786714ffe8b0845c22dfa3b/ +deleting annex/objects/a58/479/ +deleting annex/objects/a58/478/GPGHMACSHA1--a2e87af63050cbc73fc11f6f5f36d2db153cdccf/GPGHMACSHA1--a2e87af63050cbc73fc11f6f5f36d2db153cdccf +deleting annex/objects/a58/478/GPGHMACSHA1--a2e87af63050cbc73fc11f6f5f36d2db153cdccf/ +deleting annex/objects/a58/478/ +deleting annex/objects/a58/477/GPGHMACSHA1--d23a8576ee19c0a13b5ffe47cc58489c29ea550e/GPGHMACSHA1--d23a8576ee19c0a13b5ffe47cc58489c29ea550e +deleting annex/objects/a58/477/GPGHMACSHA1--d23a8576ee19c0a13b5ffe47cc58489c29ea550e/ +deleting annex/objects/a58/477/GPGHMACSHA1--b2dc2c433882ab95f891591b49c77aeace301ff1/GPGHMACSHA1--b2dc2c433882ab95f891591b49c77aeace301ff1 +deleting annex/objects/a58/477/GPGHMACSHA1--b2dc2c433882ab95f891591b49c77aeace301ff1/ +deleting annex/objects/a58/477/GPGHMACSHA1--832a1d0fa42e650e4f64f5a8d7ea7f38434475c3/GPGHMACSHA1--832a1d0fa42e650e4f64f5a8d7ea7f38434475c3 +deleting annex/objects/a58/477/GPGHMACSHA1--832a1d0fa42e650e4f64f5a8d7ea7f38434475c3/ +deleting annex/objects/a58/477/ +deleting annex/objects/a58/476/GPGHMACSHA1--6f4cfe519d8376dc39f9fa22f2913ff457765755/GPGHMACSHA1--6f4cfe519d8376dc39f9fa22f2913ff457765755 +deleting annex/objects/a58/476/GPGHMACSHA1--6f4cfe519d8376dc39f9fa22f2913ff457765755/ +deleting annex/objects/a58/476/GPGHMACSHA1--31201160cd8b0edfe8be4773e0dab36ce5f2ede9/GPGHMACSHA1--31201160cd8b0edfe8be4773e0dab36ce5f2ede9 +deleting annex/objects/a58/476/GPGHMACSHA1--31201160cd8b0edfe8be4773e0dab36ce5f2ede9/ +deleting annex/objects/a58/476/ +deleting annex/objects/a58/475/GPGHMACSHA1--e1b760a9be3560d012ad0efa2f6d38ea4b2b7162/GPGHMACSHA1--e1b760a9be3560d012ad0efa2f6d38ea4b2b7162 +deleting annex/objects/a58/475/GPGHMACSHA1--e1b760a9be3560d012ad0efa2f6d38ea4b2b7162/ +deleting annex/objects/a58/475/GPGHMACSHA1--a3c6825ff4c361a4a82fe88cb36c1080e445629a/GPGHMACSHA1--a3c6825ff4c361a4a82fe88cb36c1080e445629a +deleting annex/objects/a58/475/GPGHMACSHA1--a3c6825ff4c361a4a82fe88cb36c1080e445629a/ +deleting annex/objects/a58/475/GPGHMACSHA1--91cbf1d0d5f83c7a74e8302f2a0f93dd0634fa8a/GPGHMACSHA1--91cbf1d0d5f83c7a74e8302f2a0f93dd0634fa8a +deleting annex/objects/a58/475/GPGHMACSHA1--91cbf1d0d5f83c7a74e8302f2a0f93dd0634fa8a/ +deleting annex/objects/a58/475/GPGHMACSHA1--7e7e893921e7f532233f2c5f93371d3da1cf5c8b/GPGHMACSHA1--7e7e893921e7f532233f2c5f93371d3da1cf5c8b +deleting annex/objects/a58/475/GPGHMACSHA1--7e7e893921e7f532233f2c5f93371d3da1cf5c8b/ +deleting annex/objects/a58/475/ +deleting annex/objects/a58/474/GPGHMACSHA1--596180066590cc48510a07d434edb928c4369468/GPGHMACSHA1--596180066590cc48510a07d434edb928c4369468 +deleting annex/objects/a58/474/GPGHMACSHA1--596180066590cc48510a07d434edb928c4369468/ +deleting annex/objects/a58/474/ +deleting annex/objects/a58/472/GPGHMACSHA1--f2f8bdc1d425874268bfd12e2e1f6a109c5165f4/GPGHMACSHA1--f2f8bdc1d425874268bfd12e2e1f6a109c5165f4 +deleting annex/objects/a58/472/GPGHMACSHA1--f2f8bdc1d425874268bfd12e2e1f6a109c5165f4/ +deleting annex/objects/a58/472/ +deleting annex/objects/a58/471/GPGHMACSHA1--2d4068a5905dc2d9039cb48df32f94a4e5c6499a/GPGHMACSHA1--2d4068a5905dc2d9039cb48df32f94a4e5c6499a +deleting annex/objects/a58/471/GPGHMACSHA1--2d4068a5905dc2d9039cb48df32f94a4e5c6499a/ +deleting annex/objects/a58/471/ +deleting annex/objects/a58/470/GPGHMACSHA1--c15a4fe0416b9e4ead16b94b53ff9533819f6e32/GPGHMACSHA1--c15a4fe0416b9e4ead16b94b53ff9533819f6e32 +deleting annex/objects/a58/470/GPGHMACSHA1--c15a4fe0416b9e4ead16b94b53ff9533819f6e32/ +deleting annex/objects/a58/470/GPGHMACSHA1--2638b731fa2ccd9d3c77bdf7e3925b84bd67c72c/GPGHMACSHA1--2638b731fa2ccd9d3c77bdf7e3925b84bd67c72c +deleting annex/objects/a58/470/GPGHMACSHA1--2638b731fa2ccd9d3c77bdf7e3925b84bd67c72c/ +deleting annex/objects/a58/470/ +deleting annex/objects/a58/46e/GPGHMACSHA1--c9b43c6faf9d8ac36617ec3124c2458ce0685ad8/GPGHMACSHA1--c9b43c6faf9d8ac36617ec3124c2458ce0685ad8 +deleting annex/objects/a58/46e/GPGHMACSHA1--c9b43c6faf9d8ac36617ec3124c2458ce0685ad8/ +deleting annex/objects/a58/46e/GPGHMACSHA1--71df66ab164e2645e7bcc08255000da6579dee68/GPGHMACSHA1--71df66ab164e2645e7bcc08255000da6579dee68 +deleting annex/objects/a58/46e/GPGHMACSHA1--71df66ab164e2645e7bcc08255000da6579dee68/ +deleting annex/objects/a58/46e/GPGHMACSHA1--5713c4a32de0ab33843b77ae3aa3e9c7bdef768e/GPGHMACSHA1--5713c4a32de0ab33843b77ae3aa3e9c7bdef768e +deleting annex/objects/a58/46e/GPGHMACSHA1--5713c4a32de0ab33843b77ae3aa3e9c7bdef768e/ +deleting annex/objects/a58/46e/ +deleting annex/objects/a58/46d/GPGHMACSHA1--6cadb794538fee6edaf862abb31dd9d56bdd2509/GPGHMACSHA1--6cadb794538fee6edaf862abb31dd9d56bdd2509 +deleting annex/objects/a58/46d/GPGHMACSHA1--6cadb794538fee6edaf862abb31dd9d56bdd2509/ +deleting annex/objects/a58/46d/ +deleting annex/objects/a58/46a/GPGHMACSHA1--56bc19394fe8de7b257739188782d0f7502658d8/GPGHMACSHA1--56bc19394fe8de7b257739188782d0f7502658d8 +deleting annex/objects/a58/46a/GPGHMACSHA1--56bc19394fe8de7b257739188782d0f7502658d8/ +deleting annex/objects/a58/46a/ +deleting annex/objects/a58/468/GPGHMACSHA1--e597d2df79d4687cd5b715dd0f267e3d765afcc5/GPGHMACSHA1--e597d2df79d4687cd5b715dd0f267e3d765afcc5 +deleting annex/objects/a58/468/GPGHMACSHA1--e597d2df79d4687cd5b715dd0f267e3d765afcc5/ +deleting annex/objects/a58/468/GPGHMACSHA1--8c3c6fd12724044013c47a5371613d29a24fed0b/GPGHMACSHA1--8c3c6fd12724044013c47a5371613d29a24fed0b +deleting annex/objects/a58/468/GPGHMACSHA1--8c3c6fd12724044013c47a5371613d29a24fed0b/ +deleting annex/objects/a58/468/ +deleting annex/objects/a58/467/GPGHMACSHA1--4e1c311a20c35239526316a3bc1b3085bce8487a/GPGHMACSHA1--4e1c311a20c35239526316a3bc1b3085bce8487a +deleting annex/objects/a58/467/GPGHMACSHA1--4e1c311a20c35239526316a3bc1b3085bce8487a/ +deleting annex/objects/a58/467/GPGHMACSHA1--08f018d615356689d37330d622955f3522a2bcc3/GPGHMACSHA1--08f018d615356689d37330d622955f3522a2bcc3 +deleting annex/objects/a58/467/GPGHMACSHA1--08f018d615356689d37330d622955f3522a2bcc3/ +deleting annex/objects/a58/467/ +deleting annex/objects/a58/466/GPGHMACSHA1--fbf07be634ac64b0aabfa16a9453e58e10444342/GPGHMACSHA1--fbf07be634ac64b0aabfa16a9453e58e10444342 +deleting annex/objects/a58/466/GPGHMACSHA1--fbf07be634ac64b0aabfa16a9453e58e10444342/ +deleting annex/objects/a58/466/GPGHMACSHA1--9356ccce97e5bf66a136152624c0e9325db2effb/GPGHMACSHA1--9356ccce97e5bf66a136152624c0e9325db2effb +deleting annex/objects/a58/466/GPGHMACSHA1--9356ccce97e5bf66a136152624c0e9325db2effb/ +deleting annex/objects/a58/466/ +deleting annex/objects/a58/464/GPGHMACSHA1--f52322b67d8a452527123cddaa9eb58dcec8f39c/GPGHMACSHA1--f52322b67d8a452527123cddaa9eb58dcec8f39c +deleting annex/objects/a58/464/GPGHMACSHA1--f52322b67d8a452527123cddaa9eb58dcec8f39c/ +deleting annex/objects/a58/464/GPGHMACSHA1--f2c0f9f49168bb197fe142fb771d0d3238b1bd6c/GPGHMACSHA1--f2c0f9f49168bb197fe142fb771d0d3238b1bd6c +deleting annex/objects/a58/464/GPGHMACSHA1--f2c0f9f49168bb197fe142fb771d0d3238b1bd6c/ +deleting annex/objects/a58/464/GPGHMACSHA1--a658a7f555520099f07e5e9535d33824be0e0f8b/GPGHMACSHA1--a658a7f555520099f07e5e9535d33824be0e0f8b +deleting annex/objects/a58/464/GPGHMACSHA1--a658a7f555520099f07e5e9535d33824be0e0f8b/ +deleting annex/objects/a58/464/ +deleting annex/objects/a58/463/GPGHMACSHA1--e0b5f96072ab37c420e432298e4e0412792d1bad/GPGHMACSHA1--e0b5f96072ab37c420e432298e4e0412792d1bad +deleting annex/objects/a58/463/GPGHMACSHA1--e0b5f96072ab37c420e432298e4e0412792d1bad/ +deleting annex/objects/a58/463/GPGHMACSHA1--c9e7c8ec3f83e9d3830d4589d57b200f5363ea7c/GPGHMACSHA1--c9e7c8ec3f83e9d3830d4589d57b200f5363ea7c +deleting annex/objects/a58/463/GPGHMACSHA1--c9e7c8ec3f83e9d3830d4589d57b200f5363ea7c/ +deleting annex/objects/a58/463/GPGHMACSHA1--06867b45f24d0007fa08c624474cc8d7948dc2c9/GPGHMACSHA1--06867b45f24d0007fa08c624474cc8d7948dc2c9 +deleting annex/objects/a58/463/GPGHMACSHA1--06867b45f24d0007fa08c624474cc8d7948dc2c9/ +deleting annex/objects/a58/463/ +deleting annex/objects/a58/460/GPGHMACSHA1--32be64b3dbc2edfd1cc8455d392cfa80a62fb3a5/GPGHMACSHA1--32be64b3dbc2edfd1cc8455d392cfa80a62fb3a5 +deleting annex/objects/a58/460/GPGHMACSHA1--32be64b3dbc2edfd1cc8455d392cfa80a62fb3a5/ +deleting annex/objects/a58/460/GPGHMACSHA1--2e7efce7c67bd0721cc143bf5dcebab5cd8bbcb6/GPGHMACSHA1--2e7efce7c67bd0721cc143bf5dcebab5cd8bbcb6 +deleting annex/objects/a58/460/GPGHMACSHA1--2e7efce7c67bd0721cc143bf5dcebab5cd8bbcb6/ +deleting annex/objects/a58/460/GPGHMACSHA1--261b5143f57945f1e6c475c2969bb3f445d362df/GPGHMACSHA1--261b5143f57945f1e6c475c2969bb3f445d362df +deleting annex/objects/a58/460/GPGHMACSHA1--261b5143f57945f1e6c475c2969bb3f445d362df/ +deleting annex/objects/a58/460/ +deleting annex/objects/a58/45c/GPGHMACSHA1--1c379eca1fd7ecc0be8bc5c36305d6229bf1ce31/GPGHMACSHA1--1c379eca1fd7ecc0be8bc5c36305d6229bf1ce31 +deleting annex/objects/a58/45c/GPGHMACSHA1--1c379eca1fd7ecc0be8bc5c36305d6229bf1ce31/ +deleting annex/objects/a58/45c/ +deleting annex/objects/a58/45b/GPGHMACSHA1--719d9d330cfb59006d734d72f06642344e5838c3/GPGHMACSHA1--719d9d330cfb59006d734d72f06642344e5838c3 +deleting annex/objects/a58/45b/GPGHMACSHA1--719d9d330cfb59006d734d72f06642344e5838c3/ +deleting annex/objects/a58/45b/ +deleting annex/objects/a58/45a/GPGHMACSHA1--8db40287caa82032565510a5219a0e9defbf4bf6/GPGHMACSHA1--8db40287caa82032565510a5219a0e9defbf4bf6 +deleting annex/objects/a58/45a/GPGHMACSHA1--8db40287caa82032565510a5219a0e9defbf4bf6/ +deleting annex/objects/a58/45a/GPGHMACSHA1--82fca2040153c331e2516bb9a9b7de69390f2354/GPGHMACSHA1--82fca2040153c331e2516bb9a9b7de69390f2354 +deleting annex/objects/a58/45a/GPGHMACSHA1--82fca2040153c331e2516bb9a9b7de69390f2354/ +deleting annex/objects/a58/45a/GPGHMACSHA1--15afc41dacb5a8aa43dec9b417813e0624b9e3c1/GPGHMACSHA1--15afc41dacb5a8aa43dec9b417813e0624b9e3c1 +deleting annex/objects/a58/45a/GPGHMACSHA1--15afc41dacb5a8aa43dec9b417813e0624b9e3c1/ +deleting annex/objects/a58/45a/ +deleting annex/objects/a58/459/GPGHMACSHA1--62b3973a6dc9c7f06ea1ad765cf82173c3e99ecf/GPGHMACSHA1--62b3973a6dc9c7f06ea1ad765cf82173c3e99ecf +deleting annex/objects/a58/459/GPGHMACSHA1--62b3973a6dc9c7f06ea1ad765cf82173c3e99ecf/ +deleting annex/objects/a58/459/GPGHMACSHA1--206a24d12ba1b1e7460f06614d9a97f7b7a8c487/GPGHMACSHA1--206a24d12ba1b1e7460f06614d9a97f7b7a8c487 +deleting annex/objects/a58/459/GPGHMACSHA1--206a24d12ba1b1e7460f06614d9a97f7b7a8c487/ +deleting annex/objects/a58/459/GPGHMACSHA1--16e0b62ec227b78294cc5e6e944ecb5fddccfce2/GPGHMACSHA1--16e0b62ec227b78294cc5e6e944ecb5fddccfce2 +deleting annex/objects/a58/459/GPGHMACSHA1--16e0b62ec227b78294cc5e6e944ecb5fddccfce2/ +deleting annex/objects/a58/459/ +deleting annex/objects/a58/458/GPGHMACSHA1--a9f0c0e22f0e41859800f1156d634f233826dea0/GPGHMACSHA1--a9f0c0e22f0e41859800f1156d634f233826dea0 +deleting annex/objects/a58/458/GPGHMACSHA1--a9f0c0e22f0e41859800f1156d634f233826dea0/ +deleting annex/objects/a58/458/GPGHMACSHA1--87177984717ed03ec947ddd4ef64512d4aed279e/GPGHMACSHA1--87177984717ed03ec947ddd4ef64512d4aed279e +deleting annex/objects/a58/458/GPGHMACSHA1--87177984717ed03ec947ddd4ef64512d4aed279e/ +deleting annex/objects/a58/458/ +deleting annex/objects/a58/457/GPGHMACSHA1--893a12a05dcd700a0d46e8ae94bc5d93d8ff7036/GPGHMACSHA1--893a12a05dcd700a0d46e8ae94bc5d93d8ff7036 +deleting annex/objects/a58/457/GPGHMACSHA1--893a12a05dcd700a0d46e8ae94bc5d93d8ff7036/ +deleting annex/objects/a58/457/ +deleting annex/objects/a58/456/GPGHMACSHA1--6864c7086b8633c6ec8c43a583bd5fbb47db9be5/GPGHMACSHA1--6864c7086b8633c6ec8c43a583bd5fbb47db9be5 +deleting annex/objects/a58/456/GPGHMACSHA1--6864c7086b8633c6ec8c43a583bd5fbb47db9be5/ +deleting annex/objects/a58/456/GPGHMACSHA1--156855196a4619ab3c306e77e93e2c1a2a7877aa/GPGHMACSHA1--156855196a4619ab3c306e77e93e2c1a2a7877aa +deleting annex/objects/a58/456/GPGHMACSHA1--156855196a4619ab3c306e77e93e2c1a2a7877aa/ +deleting annex/objects/a58/456/GPGHMACSHA1--07c2c6081a25811895922c20dea472ba79d161e3/GPGHMACSHA1--07c2c6081a25811895922c20dea472ba79d161e3 +deleting annex/objects/a58/456/GPGHMACSHA1--07c2c6081a25811895922c20dea472ba79d161e3/ +deleting annex/objects/a58/456/ +deleting annex/objects/a58/455/GPGHMACSHA1--dc31fd1a96338d9b27161fe10d647bbc397fac75/GPGHMACSHA1--dc31fd1a96338d9b27161fe10d647bbc397fac75 +deleting annex/objects/a58/455/GPGHMACSHA1--dc31fd1a96338d9b27161fe10d647bbc397fac75/ +deleting annex/objects/a58/455/GPGHMACSHA1--06926b3dd937f6bb6da1e4aa70dbababe7fbb580/GPGHMACSHA1--06926b3dd937f6bb6da1e4aa70dbababe7fbb580 +deleting annex/objects/a58/455/GPGHMACSHA1--06926b3dd937f6bb6da1e4aa70dbababe7fbb580/ +deleting annex/objects/a58/455/ +deleting annex/objects/a58/454/GPGHMACSHA1--b3008e172257513bcc0903b66a5d14c77be6783e/GPGHMACSHA1--b3008e172257513bcc0903b66a5d14c77be6783e +deleting annex/objects/a58/454/GPGHMACSHA1--b3008e172257513bcc0903b66a5d14c77be6783e/ +deleting annex/objects/a58/454/GPGHMACSHA1--a1dfca54d13d706ca03f1e68fde3d3a5b73db8b4/GPGHMACSHA1--a1dfca54d13d706ca03f1e68fde3d3a5b73db8b4 +deleting annex/objects/a58/454/GPGHMACSHA1--a1dfca54d13d706ca03f1e68fde3d3a5b73db8b4/ +deleting annex/objects/a58/454/GPGHMACSHA1--74a9d24a9a0454934d86c98eef7b77986bc8ae39/GPGHMACSHA1--74a9d24a9a0454934d86c98eef7b77986bc8ae39 +deleting annex/objects/a58/454/GPGHMACSHA1--74a9d24a9a0454934d86c98eef7b77986bc8ae39/ +deleting annex/objects/a58/454/GPGHMACSHA1--5ac6306cab63cccf33918044abaa2cde5b6924e4/GPGHMACSHA1--5ac6306cab63cccf33918044abaa2cde5b6924e4 +deleting annex/objects/a58/454/GPGHMACSHA1--5ac6306cab63cccf33918044abaa2cde5b6924e4/ +deleting annex/objects/a58/454/ +deleting annex/objects/a58/453/GPGHMACSHA1--f0cb77e2d3ce4d6f4ecb64c6127a2d956905da46/GPGHMACSHA1--f0cb77e2d3ce4d6f4ecb64c6127a2d956905da46 +deleting annex/objects/a58/453/GPGHMACSHA1--f0cb77e2d3ce4d6f4ecb64c6127a2d956905da46/ +deleting annex/objects/a58/453/ +deleting annex/objects/a58/452/GPGHMACSHA1--a42eb5dac2c02cf5c02b210ff208719e592472ce/GPGHMACSHA1--a42eb5dac2c02cf5c02b210ff208719e592472ce +deleting annex/objects/a58/452/GPGHMACSHA1--a42eb5dac2c02cf5c02b210ff208719e592472ce/ +deleting annex/objects/a58/452/GPGHMACSHA1--276b1d8e17d2c450c904ed3b5a916323539f11f1/GPGHMACSHA1--276b1d8e17d2c450c904ed3b5a916323539f11f1 +deleting annex/objects/a58/452/GPGHMACSHA1--276b1d8e17d2c450c904ed3b5a916323539f11f1/ +deleting annex/objects/a58/452/ +deleting annex/objects/a58/451/GPGHMACSHA1--63eb1148051b3b04bd35f9a909dff50f856bf0eb/GPGHMACSHA1--63eb1148051b3b04bd35f9a909dff50f856bf0eb +deleting annex/objects/a58/451/GPGHMACSHA1--63eb1148051b3b04bd35f9a909dff50f856bf0eb/ +deleting annex/objects/a58/451/ +deleting annex/objects/a58/450/GPGHMACSHA1--c351adbb53ea6d8b1a619942d24a060d09b43fdd/GPGHMACSHA1--c351adbb53ea6d8b1a619942d24a060d09b43fdd +deleting annex/objects/a58/450/GPGHMACSHA1--c351adbb53ea6d8b1a619942d24a060d09b43fdd/ +deleting annex/objects/a58/450/ +deleting annex/objects/a58/44f/GPGHMACSHA1--7710555d8cdaa9b057f94ea065373a51e5192fa4/GPGHMACSHA1--7710555d8cdaa9b057f94ea065373a51e5192fa4 +deleting annex/objects/a58/44f/GPGHMACSHA1--7710555d8cdaa9b057f94ea065373a51e5192fa4/ +deleting annex/objects/a58/44f/GPGHMACSHA1--5ddbc268a5c5d231d5aaac61345db866683e8f13/GPGHMACSHA1--5ddbc268a5c5d231d5aaac61345db866683e8f13 +deleting annex/objects/a58/44f/GPGHMACSHA1--5ddbc268a5c5d231d5aaac61345db866683e8f13/ +deleting annex/objects/a58/44f/GPGHMACSHA1--4e0e4eb624762b5db3ce8cd19d8c8343a64546cd/GPGHMACSHA1--4e0e4eb624762b5db3ce8cd19d8c8343a64546cd +deleting annex/objects/a58/44f/GPGHMACSHA1--4e0e4eb624762b5db3ce8cd19d8c8343a64546cd/ +deleting annex/objects/a58/44f/ +deleting annex/objects/a58/44c/GPGHMACSHA1--f1e316ab5b6acf4ca7f4123302014480c5ab898b/GPGHMACSHA1--f1e316ab5b6acf4ca7f4123302014480c5ab898b +deleting annex/objects/a58/44c/GPGHMACSHA1--f1e316ab5b6acf4ca7f4123302014480c5ab898b/ +deleting annex/objects/a58/44c/GPGHMACSHA1--1670a3982fae48be18b484513986853d44172c62/GPGHMACSHA1--1670a3982fae48be18b484513986853d44172c62 +deleting annex/objects/a58/44c/GPGHMACSHA1--1670a3982fae48be18b484513986853d44172c62/ +deleting annex/objects/a58/44c/ +deleting annex/objects/a58/44b/GPGHMACSHA1--28dfcfc9c010e82284dd90a040d44d990f6efd74/GPGHMACSHA1--28dfcfc9c010e82284dd90a040d44d990f6efd74 +deleting annex/objects/a58/44b/GPGHMACSHA1--28dfcfc9c010e82284dd90a040d44d990f6efd74/ +deleting annex/objects/a58/44b/ +deleting annex/objects/a58/44a/GPGHMACSHA1--e11d314af0a76e85b41ca56126f27db729ea2f11/GPGHMACSHA1--e11d314af0a76e85b41ca56126f27db729ea2f11 +deleting annex/objects/a58/44a/GPGHMACSHA1--e11d314af0a76e85b41ca56126f27db729ea2f11/ +deleting annex/objects/a58/44a/GPGHMACSHA1--77adcf8896cd652101e6f9fa96b0eb61cf128458/GPGHMACSHA1--77adcf8896cd652101e6f9fa96b0eb61cf128458 +deleting annex/objects/a58/44a/GPGHMACSHA1--77adcf8896cd652101e6f9fa96b0eb61cf128458/ +deleting annex/objects/a58/44a/ +deleting annex/objects/a58/447/GPGHMACSHA1--ea53841a44d66255a6b0cf01db66a6364aff5150/GPGHMACSHA1--ea53841a44d66255a6b0cf01db66a6364aff5150 +deleting annex/objects/a58/447/GPGHMACSHA1--ea53841a44d66255a6b0cf01db66a6364aff5150/ +deleting annex/objects/a58/447/GPGHMACSHA1--b0e32c21b5c7c092c2caf8d6b1dce03216842960/GPGHMACSHA1--b0e32c21b5c7c092c2caf8d6b1dce03216842960 +deleting annex/objects/a58/447/GPGHMACSHA1--b0e32c21b5c7c092c2caf8d6b1dce03216842960/ +deleting annex/objects/a58/447/ +deleting annex/objects/a58/446/GPGHMACSHA1--5f7f50eea93257bbc12e4741bc37a56812a8706e/GPGHMACSHA1--5f7f50eea93257bbc12e4741bc37a56812a8706e +deleting annex/objects/a58/446/GPGHMACSHA1--5f7f50eea93257bbc12e4741bc37a56812a8706e/ +deleting annex/objects/a58/446/GPGHMACSHA1--4373fb4b869992d6dd3002ea59817de12836b114/GPGHMACSHA1--4373fb4b869992d6dd3002ea59817de12836b114 +deleting annex/objects/a58/446/GPGHMACSHA1--4373fb4b869992d6dd3002ea59817de12836b114/ +deleting annex/objects/a58/446/ +deleting annex/objects/a58/445/GPGHMACSHA1--b57c200af53e987f7c02f08bfab7f387fdc94994/GPGHMACSHA1--b57c200af53e987f7c02f08bfab7f387fdc94994 +deleting annex/objects/a58/445/GPGHMACSHA1--b57c200af53e987f7c02f08bfab7f387fdc94994/ +deleting annex/objects/a58/445/ +deleting annex/objects/a58/443/GPGHMACSHA1--eda7815f2322f2a279df36ac25c63f6403a84fc1/GPGHMACSHA1--eda7815f2322f2a279df36ac25c63f6403a84fc1 +deleting annex/objects/a58/443/GPGHMACSHA1--eda7815f2322f2a279df36ac25c63f6403a84fc1/ +deleting annex/objects/a58/443/ +deleting annex/objects/a58/442/GPGHMACSHA1--95712c22740b110780ce15ae815b8545e642839f/GPGHMACSHA1--95712c22740b110780ce15ae815b8545e642839f +deleting annex/objects/a58/442/GPGHMACSHA1--95712c22740b110780ce15ae815b8545e642839f/ +deleting annex/objects/a58/442/GPGHMACSHA1--3adf263bba8c06cfcf8f21fc0ac6a414964ff5c4/GPGHMACSHA1--3adf263bba8c06cfcf8f21fc0ac6a414964ff5c4 +deleting annex/objects/a58/442/GPGHMACSHA1--3adf263bba8c06cfcf8f21fc0ac6a414964ff5c4/ +deleting annex/objects/a58/442/GPGHMACSHA1--2bd7a817b46c8160a019b50676ad3abe78250b34/GPGHMACSHA1--2bd7a817b46c8160a019b50676ad3abe78250b34 +deleting annex/objects/a58/442/GPGHMACSHA1--2bd7a817b46c8160a019b50676ad3abe78250b34/ +deleting annex/objects/a58/442/ +deleting annex/objects/a58/441/GPGHMACSHA1--b4bf7e25c23ac465a3615163baa1564fd0b7c18b/GPGHMACSHA1--b4bf7e25c23ac465a3615163baa1564fd0b7c18b +deleting annex/objects/a58/441/GPGHMACSHA1--b4bf7e25c23ac465a3615163baa1564fd0b7c18b/ +deleting annex/objects/a58/441/ +deleting annex/objects/a58/440/GPGHMACSHA1--0a00394a28f443f4160660e5740ab0242a74c4be/GPGHMACSHA1--0a00394a28f443f4160660e5740ab0242a74c4be +deleting annex/objects/a58/440/GPGHMACSHA1--0a00394a28f443f4160660e5740ab0242a74c4be/ +deleting annex/objects/a58/440/ +deleting annex/objects/a58/43f/GPGHMACSHA1--5879c3d88fea43843b930669fc20d26df4e05070/GPGHMACSHA1--5879c3d88fea43843b930669fc20d26df4e05070 +deleting annex/objects/a58/43f/GPGHMACSHA1--5879c3d88fea43843b930669fc20d26df4e05070/ +deleting annex/objects/a58/43f/ +deleting annex/objects/a58/43e/GPGHMACSHA1--c1cc04b0ed6d9c4d3c3bb5af7253756899360721/GPGHMACSHA1--c1cc04b0ed6d9c4d3c3bb5af7253756899360721 +deleting annex/objects/a58/43e/GPGHMACSHA1--c1cc04b0ed6d9c4d3c3bb5af7253756899360721/ +deleting annex/objects/a58/43e/ +deleting annex/objects/a58/43d/GPGHMACSHA1--8bedee83511422dac5ca68349e9279219f24d8eb/GPGHMACSHA1--8bedee83511422dac5ca68349e9279219f24d8eb +deleting annex/objects/a58/43d/GPGHMACSHA1--8bedee83511422dac5ca68349e9279219f24d8eb/ +deleting annex/objects/a58/43d/ +deleting annex/objects/a58/43c/GPGHMACSHA1--f7898cfd606c9c6af3ae0a717f189955f6c1296f/GPGHMACSHA1--f7898cfd606c9c6af3ae0a717f189955f6c1296f +deleting annex/objects/a58/43c/GPGHMACSHA1--f7898cfd606c9c6af3ae0a717f189955f6c1296f/ +deleting annex/objects/a58/43c/ +deleting annex/objects/a58/43b/GPGHMACSHA1--78cf349532a821c7588e76c7f005b491be6667f4/GPGHMACSHA1--78cf349532a821c7588e76c7f005b491be6667f4 +deleting annex/objects/a58/43b/GPGHMACSHA1--78cf349532a821c7588e76c7f005b491be6667f4/ +deleting annex/objects/a58/43b/ +deleting annex/objects/a58/43a/GPGHMACSHA1--638ab071596595093f0017120cbff44101b64b15/GPGHMACSHA1--638ab071596595093f0017120cbff44101b64b15 +deleting annex/objects/a58/43a/GPGHMACSHA1--638ab071596595093f0017120cbff44101b64b15/ +deleting annex/objects/a58/43a/GPGHMACSHA1--1f352902cd593c491ac5f69d0f2464b12465dc3e/GPGHMACSHA1--1f352902cd593c491ac5f69d0f2464b12465dc3e +deleting annex/objects/a58/43a/GPGHMACSHA1--1f352902cd593c491ac5f69d0f2464b12465dc3e/ +deleting annex/objects/a58/43a/ +deleting annex/objects/a58/438/GPGHMACSHA1--ab04bb336367d6be3db3ea00ed8e2568d1938652/GPGHMACSHA1--ab04bb336367d6be3db3ea00ed8e2568d1938652 +deleting annex/objects/a58/438/GPGHMACSHA1--ab04bb336367d6be3db3ea00ed8e2568d1938652/ +deleting annex/objects/a58/438/ +deleting annex/objects/a58/437/GPGHMACSHA1--65acf55cb96292a266ae586a7cba515467f0d62b/GPGHMACSHA1--65acf55cb96292a266ae586a7cba515467f0d62b +deleting annex/objects/a58/437/GPGHMACSHA1--65acf55cb96292a266ae586a7cba515467f0d62b/ +deleting annex/objects/a58/437/ +deleting annex/objects/a58/436/GPGHMACSHA1--d335540d28169f824e93f8ba9abe01eb4a18fa17/GPGHMACSHA1--d335540d28169f824e93f8ba9abe01eb4a18fa17 +deleting annex/objects/a58/436/GPGHMACSHA1--d335540d28169f824e93f8ba9abe01eb4a18fa17/ +deleting annex/objects/a58/436/ +deleting annex/objects/a58/434/GPGHMACSHA1--45fa32412773ef9991e58dc37f1bcedb102803a5/GPGHMACSHA1--45fa32412773ef9991e58dc37f1bcedb102803a5 +deleting annex/objects/a58/434/GPGHMACSHA1--45fa32412773ef9991e58dc37f1bcedb102803a5/ +deleting annex/objects/a58/434/GPGHMACSHA1--4537638a5234f4f2e43ac144e27eb7aad01dd86a/GPGHMACSHA1--4537638a5234f4f2e43ac144e27eb7aad01dd86a +deleting annex/objects/a58/434/GPGHMACSHA1--4537638a5234f4f2e43ac144e27eb7aad01dd86a/ +deleting annex/objects/a58/434/ +deleting annex/objects/a58/433/GPGHMACSHA1--daa7670c66a305e507f7fd2b70eca1e683fd1edc/GPGHMACSHA1--daa7670c66a305e507f7fd2b70eca1e683fd1edc +deleting annex/objects/a58/433/GPGHMACSHA1--daa7670c66a305e507f7fd2b70eca1e683fd1edc/ +deleting annex/objects/a58/433/GPGHMACSHA1--75e5003b8ca92cd8a854ac365f4ffde9189a91f7/GPGHMACSHA1--75e5003b8ca92cd8a854ac365f4ffde9189a91f7 +deleting annex/objects/a58/433/GPGHMACSHA1--75e5003b8ca92cd8a854ac365f4ffde9189a91f7/ +deleting annex/objects/a58/433/ +deleting annex/objects/a58/432/GPGHMACSHA1--c6270f9c854d8c40be38a2ccf5ec269f36d09243/GPGHMACSHA1--c6270f9c854d8c40be38a2ccf5ec269f36d09243 +deleting annex/objects/a58/432/GPGHMACSHA1--c6270f9c854d8c40be38a2ccf5ec269f36d09243/ +deleting annex/objects/a58/432/GPGHMACSHA1--aab67a4a546e4e6780e5c1e362727efd53cb7735/GPGHMACSHA1--aab67a4a546e4e6780e5c1e362727efd53cb7735 +deleting annex/objects/a58/432/GPGHMACSHA1--aab67a4a546e4e6780e5c1e362727efd53cb7735/ +deleting annex/objects/a58/432/ +deleting annex/objects/a58/431/GPGHMACSHA1--7bff7f1ebeaf77b19c16950a8611eb3fc473aa9a/GPGHMACSHA1--7bff7f1ebeaf77b19c16950a8611eb3fc473aa9a +deleting annex/objects/a58/431/GPGHMACSHA1--7bff7f1ebeaf77b19c16950a8611eb3fc473aa9a/ +deleting annex/objects/a58/431/GPGHMACSHA1--5651deccbce94672550d0071169f2f3e3e1b6b80/GPGHMACSHA1--5651deccbce94672550d0071169f2f3e3e1b6b80 +deleting annex/objects/a58/431/GPGHMACSHA1--5651deccbce94672550d0071169f2f3e3e1b6b80/ +deleting annex/objects/a58/431/GPGHMACSHA1--42bc0712bc04786dd1d443ac13d3e9480bd7a36a/GPGHMACSHA1--42bc0712bc04786dd1d443ac13d3e9480bd7a36a +deleting annex/objects/a58/431/GPGHMACSHA1--42bc0712bc04786dd1d443ac13d3e9480bd7a36a/ +deleting annex/objects/a58/431/ +deleting annex/objects/a58/430/GPGHMACSHA1--6e54d17f4b6c8bcd0af227c3df6c87a4f8a347ad/GPGHMACSHA1--6e54d17f4b6c8bcd0af227c3df6c87a4f8a347ad +deleting annex/objects/a58/430/GPGHMACSHA1--6e54d17f4b6c8bcd0af227c3df6c87a4f8a347ad/ +deleting annex/objects/a58/430/GPGHMACSHA1--396952514f72c47b07b20f3168d80d7ce15a2553/GPGHMACSHA1--396952514f72c47b07b20f3168d80d7ce15a2553 +deleting annex/objects/a58/430/GPGHMACSHA1--396952514f72c47b07b20f3168d80d7ce15a2553/ +deleting annex/objects/a58/430/ +deleting annex/objects/a58/42e/GPGHMACSHA1--fabc8ecf1db09e398a5dc913cc76637bd73b985f/GPGHMACSHA1--fabc8ecf1db09e398a5dc913cc76637bd73b985f +deleting annex/objects/a58/42e/GPGHMACSHA1--fabc8ecf1db09e398a5dc913cc76637bd73b985f/ +deleting annex/objects/a58/42e/ +deleting annex/objects/a58/42c/GPGHMACSHA1--4a469a431a5e89690544d779b6199bd9710c9bd7/GPGHMACSHA1--4a469a431a5e89690544d779b6199bd9710c9bd7 +deleting annex/objects/a58/42c/GPGHMACSHA1--4a469a431a5e89690544d779b6199bd9710c9bd7/ +deleting annex/objects/a58/42c/GPGHMACSHA1--34bab1ace67d2f77e1d3b9885ac2a54cdd92bcad/GPGHMACSHA1--34bab1ace67d2f77e1d3b9885ac2a54cdd92bcad +deleting annex/objects/a58/42c/GPGHMACSHA1--34bab1ace67d2f77e1d3b9885ac2a54cdd92bcad/ +deleting annex/objects/a58/42c/GPGHMACSHA1--2d4b6011366e52294aa6b3528dd0d82f61e63297/GPGHMACSHA1--2d4b6011366e52294aa6b3528dd0d82f61e63297 +deleting annex/objects/a58/42c/GPGHMACSHA1--2d4b6011366e52294aa6b3528dd0d82f61e63297/ +deleting annex/objects/a58/42c/ +deleting annex/objects/a58/42b/GPGHMACSHA1--7a857fa2e6236a52929af3d8ba55dfe21edbbeb7/GPGHMACSHA1--7a857fa2e6236a52929af3d8ba55dfe21edbbeb7 +deleting annex/objects/a58/42b/GPGHMACSHA1--7a857fa2e6236a52929af3d8ba55dfe21edbbeb7/ +deleting annex/objects/a58/42b/GPGHMACSHA1--1289fde863528b3a76e44647cd87810a852ac2cd/GPGHMACSHA1--1289fde863528b3a76e44647cd87810a852ac2cd +deleting annex/objects/a58/42b/GPGHMACSHA1--1289fde863528b3a76e44647cd87810a852ac2cd/ +deleting annex/objects/a58/42b/GPGHMACSHA1--0e8f6d32dcc09d2ad75f6a2de236612d57aa5bff/GPGHMACSHA1--0e8f6d32dcc09d2ad75f6a2de236612d57aa5bff +deleting annex/objects/a58/42b/GPGHMACSHA1--0e8f6d32dcc09d2ad75f6a2de236612d57aa5bff/ +deleting annex/objects/a58/42b/ +deleting annex/objects/a58/427/GPGHMACSHA1--b98323a3a50a87ee351d4ad2d25a3b1db25d692e/GPGHMACSHA1--b98323a3a50a87ee351d4ad2d25a3b1db25d692e +deleting annex/objects/a58/427/GPGHMACSHA1--b98323a3a50a87ee351d4ad2d25a3b1db25d692e/ +deleting annex/objects/a58/427/GPGHMACSHA1--8706c90da9909b32f1653b80c1ce4e6832885f34/GPGHMACSHA1--8706c90da9909b32f1653b80c1ce4e6832885f34 +deleting annex/objects/a58/427/GPGHMACSHA1--8706c90da9909b32f1653b80c1ce4e6832885f34/ +deleting annex/objects/a58/427/GPGHMACSHA1--597b5773825df30e45054a147e100e54fc86585b/GPGHMACSHA1--597b5773825df30e45054a147e100e54fc86585b +deleting annex/objects/a58/427/GPGHMACSHA1--597b5773825df30e45054a147e100e54fc86585b/ +deleting annex/objects/a58/427/GPGHMACSHA1--010f982de31fd002db1a25deb5140a77373979a8/GPGHMACSHA1--010f982de31fd002db1a25deb5140a77373979a8 +deleting annex/objects/a58/427/GPGHMACSHA1--010f982de31fd002db1a25deb5140a77373979a8/ +deleting annex/objects/a58/427/ +deleting annex/objects/a58/426/GPGHMACSHA1--ea4bd3701455c8390ddcecf851184280db8a78f7/GPGHMACSHA1--ea4bd3701455c8390ddcecf851184280db8a78f7 +deleting annex/objects/a58/426/GPGHMACSHA1--ea4bd3701455c8390ddcecf851184280db8a78f7/ +deleting annex/objects/a58/426/GPGHMACSHA1--ddcab01c77f05997882924d4dfb950da8c5bc1dd/GPGHMACSHA1--ddcab01c77f05997882924d4dfb950da8c5bc1dd +deleting annex/objects/a58/426/GPGHMACSHA1--ddcab01c77f05997882924d4dfb950da8c5bc1dd/ +deleting annex/objects/a58/426/GPGHMACSHA1--cc0b781cf3d242b5980cfd76e03d951578390d57/GPGHMACSHA1--cc0b781cf3d242b5980cfd76e03d951578390d57 +deleting annex/objects/a58/426/GPGHMACSHA1--cc0b781cf3d242b5980cfd76e03d951578390d57/ +deleting annex/objects/a58/426/GPGHMACSHA1--a12a6aa2e35688a426094494b1248ae6e637b12a/GPGHMACSHA1--a12a6aa2e35688a426094494b1248ae6e637b12a +deleting annex/objects/a58/426/GPGHMACSHA1--a12a6aa2e35688a426094494b1248ae6e637b12a/ +deleting annex/objects/a58/426/GPGHMACSHA1--962f604814f204b5dc89224085be75d97d9d891c/GPGHMACSHA1--962f604814f204b5dc89224085be75d97d9d891c +deleting annex/objects/a58/426/GPGHMACSHA1--962f604814f204b5dc89224085be75d97d9d891c/ +deleting annex/objects/a58/426/ +deleting annex/objects/a58/425/GPGHMACSHA1--d6eaf23b63dc28c81dcb23b8188b0d2cb30b34f3/GPGHMACSHA1--d6eaf23b63dc28c81dcb23b8188b0d2cb30b34f3 +deleting annex/objects/a58/425/GPGHMACSHA1--d6eaf23b63dc28c81dcb23b8188b0d2cb30b34f3/ +deleting annex/objects/a58/425/ +deleting annex/objects/a58/424/GPGHMACSHA1--fe3888623220b2f78d6a176e9d7bdfd662144f04/GPGHMACSHA1--fe3888623220b2f78d6a176e9d7bdfd662144f04 +deleting annex/objects/a58/424/GPGHMACSHA1--fe3888623220b2f78d6a176e9d7bdfd662144f04/ +deleting annex/objects/a58/424/GPGHMACSHA1--832915e879e0181b51b4ef9c5a69d9069229748a/GPGHMACSHA1--832915e879e0181b51b4ef9c5a69d9069229748a +deleting annex/objects/a58/424/GPGHMACSHA1--832915e879e0181b51b4ef9c5a69d9069229748a/ +deleting annex/objects/a58/424/GPGHMACSHA1--7bfa9936b234f6a1325b54eac0053d7bea5640e2/GPGHMACSHA1--7bfa9936b234f6a1325b54eac0053d7bea5640e2 +deleting annex/objects/a58/424/GPGHMACSHA1--7bfa9936b234f6a1325b54eac0053d7bea5640e2/ +deleting annex/objects/a58/424/ +deleting annex/objects/a58/423/GPGHMACSHA1--f7fb65b3877bdaedf700cfcddf3982ec67ee52ba/GPGHMACSHA1--f7fb65b3877bdaedf700cfcddf3982ec67ee52ba +deleting annex/objects/a58/423/GPGHMACSHA1--f7fb65b3877bdaedf700cfcddf3982ec67ee52ba/ +deleting annex/objects/a58/423/ +deleting annex/objects/a58/422/GPGHMACSHA1--f4496fc6c7eb6edab29f921386fd0b702c6420bd/GPGHMACSHA1--f4496fc6c7eb6edab29f921386fd0b702c6420bd +deleting annex/objects/a58/422/GPGHMACSHA1--f4496fc6c7eb6edab29f921386fd0b702c6420bd/ +deleting annex/objects/a58/422/GPGHMACSHA1--26d8dc0f8e2439fde44d4030f2fbcb85113023c9/GPGHMACSHA1--26d8dc0f8e2439fde44d4030f2fbcb85113023c9 +deleting annex/objects/a58/422/GPGHMACSHA1--26d8dc0f8e2439fde44d4030f2fbcb85113023c9/ +deleting annex/objects/a58/422/ +deleting annex/objects/a58/421/GPGHMACSHA1--c88edcd0529cecb8963986dfeb352d52da2012eb/GPGHMACSHA1--c88edcd0529cecb8963986dfeb352d52da2012eb +deleting annex/objects/a58/421/GPGHMACSHA1--c88edcd0529cecb8963986dfeb352d52da2012eb/ +deleting annex/objects/a58/421/GPGHMACSHA1--bcffa9e2c46b0cb687c7804b4a94038a35324d25/GPGHMACSHA1--bcffa9e2c46b0cb687c7804b4a94038a35324d25 +deleting annex/objects/a58/421/GPGHMACSHA1--bcffa9e2c46b0cb687c7804b4a94038a35324d25/ +deleting annex/objects/a58/421/GPGHMACSHA1--647742f5095d2a4f785f08d6e6d16d28940391b2/GPGHMACSHA1--647742f5095d2a4f785f08d6e6d16d28940391b2 +deleting annex/objects/a58/421/GPGHMACSHA1--647742f5095d2a4f785f08d6e6d16d28940391b2/ +deleting annex/objects/a58/421/ +deleting annex/objects/a58/420/GPGHMACSHA1--db3baaade7b7309cc4511fbeeb63972b2feb9705/GPGHMACSHA1--db3baaade7b7309cc4511fbeeb63972b2feb9705 +deleting annex/objects/a58/420/GPGHMACSHA1--db3baaade7b7309cc4511fbeeb63972b2feb9705/ +deleting annex/objects/a58/420/GPGHMACSHA1--b0e59bde8c4eab6d56f4f34fa4a86dc2e2e55343/GPGHMACSHA1--b0e59bde8c4eab6d56f4f34fa4a86dc2e2e55343 +deleting annex/objects/a58/420/GPGHMACSHA1--b0e59bde8c4eab6d56f4f34fa4a86dc2e2e55343/ +deleting annex/objects/a58/420/GPGHMACSHA1--1609ad5394c4847c346069ca1e840cc4fb36a373/GPGHMACSHA1--1609ad5394c4847c346069ca1e840cc4fb36a373 +deleting annex/objects/a58/420/GPGHMACSHA1--1609ad5394c4847c346069ca1e840cc4fb36a373/ +deleting annex/objects/a58/420/ +deleting annex/objects/a58/41f/GPGHMACSHA1--d14e21138e0f420d97cfb9be46ed3f09ff263edc/GPGHMACSHA1--d14e21138e0f420d97cfb9be46ed3f09ff263edc +deleting annex/objects/a58/41f/GPGHMACSHA1--d14e21138e0f420d97cfb9be46ed3f09ff263edc/ +deleting annex/objects/a58/41f/GPGHMACSHA1--697364bf48ddca3faa6db4edd55a269d259f7bf1/GPGHMACSHA1--697364bf48ddca3faa6db4edd55a269d259f7bf1 +deleting annex/objects/a58/41f/GPGHMACSHA1--697364bf48ddca3faa6db4edd55a269d259f7bf1/ +deleting annex/objects/a58/41f/ +deleting annex/objects/a58/41e/GPGHMACSHA1--917976cbf2e4986bff6e2775e74a5a6b0bfdee50/GPGHMACSHA1--917976cbf2e4986bff6e2775e74a5a6b0bfdee50 +deleting annex/objects/a58/41e/GPGHMACSHA1--917976cbf2e4986bff6e2775e74a5a6b0bfdee50/ +deleting annex/objects/a58/41e/ +deleting annex/objects/a58/41c/GPGHMACSHA1--b0c5c951ab2bcd6e42863c7d024fc49268040f15/GPGHMACSHA1--b0c5c951ab2bcd6e42863c7d024fc49268040f15 +deleting annex/objects/a58/41c/GPGHMACSHA1--b0c5c951ab2bcd6e42863c7d024fc49268040f15/ +deleting annex/objects/a58/41c/ +deleting annex/objects/a58/41b/GPGHMACSHA1--80a9ec2aca8a06da0c16b0e7e971e35ba26d1bca/GPGHMACSHA1--80a9ec2aca8a06da0c16b0e7e971e35ba26d1bca +deleting annex/objects/a58/41b/GPGHMACSHA1--80a9ec2aca8a06da0c16b0e7e971e35ba26d1bca/ +deleting annex/objects/a58/41b/GPGHMACSHA1--74ea3eb6f2fe287e4643188a861cc647685cd9de/GPGHMACSHA1--74ea3eb6f2fe287e4643188a861cc647685cd9de +deleting annex/objects/a58/41b/GPGHMACSHA1--74ea3eb6f2fe287e4643188a861cc647685cd9de/ +deleting annex/objects/a58/41b/ +deleting annex/objects/a58/41a/GPGHMACSHA1--5b8f2fa4733124a23a1ea9557eb602d1ee563d6e/GPGHMACSHA1--5b8f2fa4733124a23a1ea9557eb602d1ee563d6e +deleting annex/objects/a58/41a/GPGHMACSHA1--5b8f2fa4733124a23a1ea9557eb602d1ee563d6e/ +deleting annex/objects/a58/41a/GPGHMACSHA1--05309cfa4059319e031377d5195c749526dc9d18/GPGHMACSHA1--05309cfa4059319e031377d5195c749526dc9d18 +deleting annex/objects/a58/41a/GPGHMACSHA1--05309cfa4059319e031377d5195c749526dc9d18/ +deleting annex/objects/a58/41a/ +deleting annex/objects/a58/419/GPGHMACSHA1--b86888b11f111fb176018cd3a8ea3fe026891b44/GPGHMACSHA1--b86888b11f111fb176018cd3a8ea3fe026891b44 +deleting annex/objects/a58/419/GPGHMACSHA1--b86888b11f111fb176018cd3a8ea3fe026891b44/ +deleting annex/objects/a58/419/ +deleting annex/objects/a58/418/GPGHMACSHA1--45837e92a7a9cece9aa3200252baf000c3ac9602/GPGHMACSHA1--45837e92a7a9cece9aa3200252baf000c3ac9602 +deleting annex/objects/a58/418/GPGHMACSHA1--45837e92a7a9cece9aa3200252baf000c3ac9602/ +deleting annex/objects/a58/418/ +deleting annex/objects/a58/416/GPGHMACSHA1--3b7d286a5fa51188df9efda1caa9143b6afea9b7/GPGHMACSHA1--3b7d286a5fa51188df9efda1caa9143b6afea9b7 +deleting annex/objects/a58/416/GPGHMACSHA1--3b7d286a5fa51188df9efda1caa9143b6afea9b7/ +deleting annex/objects/a58/416/GPGHMACSHA1--1319e2380f4c9eb92ecd4e4ba2823f9e6d77e189/GPGHMACSHA1--1319e2380f4c9eb92ecd4e4ba2823f9e6d77e189 +deleting annex/objects/a58/416/GPGHMACSHA1--1319e2380f4c9eb92ecd4e4ba2823f9e6d77e189/ +deleting annex/objects/a58/416/ +deleting annex/objects/a58/415/GPGHMACSHA1--5ddc2d5000916cddc9002d7cd44806c733195cb9/GPGHMACSHA1--5ddc2d5000916cddc9002d7cd44806c733195cb9 +deleting annex/objects/a58/415/GPGHMACSHA1--5ddc2d5000916cddc9002d7cd44806c733195cb9/ +deleting annex/objects/a58/415/ +deleting annex/objects/a58/414/GPGHMACSHA1--bf8faf76eb2326aa68cdbf97d806140ff682100a/GPGHMACSHA1--bf8faf76eb2326aa68cdbf97d806140ff682100a +deleting annex/objects/a58/414/GPGHMACSHA1--bf8faf76eb2326aa68cdbf97d806140ff682100a/ +deleting annex/objects/a58/414/GPGHMACSHA1--b2a0f5399a76f9e0ac6aa66968afdf7795f3ff6e/GPGHMACSHA1--b2a0f5399a76f9e0ac6aa66968afdf7795f3ff6e +deleting annex/objects/a58/414/GPGHMACSHA1--b2a0f5399a76f9e0ac6aa66968afdf7795f3ff6e/ +deleting annex/objects/a58/414/ +deleting annex/objects/a58/413/GPGHMACSHA1--ee370aa088cb4b37f128cb0876150481440c0885/GPGHMACSHA1--ee370aa088cb4b37f128cb0876150481440c0885 +deleting annex/objects/a58/413/GPGHMACSHA1--ee370aa088cb4b37f128cb0876150481440c0885/ +deleting annex/objects/a58/413/GPGHMACSHA1--93ada330f495af322241613a946f1fdeb7061d7e/GPGHMACSHA1--93ada330f495af322241613a946f1fdeb7061d7e +deleting annex/objects/a58/413/GPGHMACSHA1--93ada330f495af322241613a946f1fdeb7061d7e/ +deleting annex/objects/a58/413/GPGHMACSHA1--925ab9faf085e84d7c78fadc3079b52047d432db/GPGHMACSHA1--925ab9faf085e84d7c78fadc3079b52047d432db +deleting annex/objects/a58/413/GPGHMACSHA1--925ab9faf085e84d7c78fadc3079b52047d432db/ +deleting annex/objects/a58/413/ +deleting annex/objects/a58/411/GPGHMACSHA1--f986aca6642ba1633e703a3fcaca54eea6b84c36/GPGHMACSHA1--f986aca6642ba1633e703a3fcaca54eea6b84c36 +deleting annex/objects/a58/411/GPGHMACSHA1--f986aca6642ba1633e703a3fcaca54eea6b84c36/ +deleting annex/objects/a58/411/GPGHMACSHA1--10ca7758ce6d180399992ceff26d9cc1399dc831/GPGHMACSHA1--10ca7758ce6d180399992ceff26d9cc1399dc831 +deleting annex/objects/a58/411/GPGHMACSHA1--10ca7758ce6d180399992ceff26d9cc1399dc831/ +deleting annex/objects/a58/411/ +deleting annex/objects/a58/40f/GPGHMACSHA1--db453b034825abb21cded129e75405cfa104b0e3/GPGHMACSHA1--db453b034825abb21cded129e75405cfa104b0e3 +deleting annex/objects/a58/40f/GPGHMACSHA1--db453b034825abb21cded129e75405cfa104b0e3/ +deleting annex/objects/a58/40f/GPGHMACSHA1--c34bbe59b7fe0a48197217b8affa27d8fd3e89c2/GPGHMACSHA1--c34bbe59b7fe0a48197217b8affa27d8fd3e89c2 +deleting annex/objects/a58/40f/GPGHMACSHA1--c34bbe59b7fe0a48197217b8affa27d8fd3e89c2/ +deleting annex/objects/a58/40f/ +deleting annex/objects/a58/40e/GPGHMACSHA1--7c96f9b4ff7ec24bc7c89ac82f6352fe604a6b7d/GPGHMACSHA1--7c96f9b4ff7ec24bc7c89ac82f6352fe604a6b7d +deleting annex/objects/a58/40e/GPGHMACSHA1--7c96f9b4ff7ec24bc7c89ac82f6352fe604a6b7d/ +deleting annex/objects/a58/40e/ +deleting annex/objects/a58/40d/GPGHMACSHA1--a51e22387fad3d70b15ff77e26b802e50afe7a46/GPGHMACSHA1--a51e22387fad3d70b15ff77e26b802e50afe7a46 +deleting annex/objects/a58/40d/GPGHMACSHA1--a51e22387fad3d70b15ff77e26b802e50afe7a46/ +deleting annex/objects/a58/40d/ +deleting annex/objects/a58/40c/GPGHMACSHA1--b82a432e11881548052114cfbb07dbc23bd6a2f2/GPGHMACSHA1--b82a432e11881548052114cfbb07dbc23bd6a2f2 +deleting annex/objects/a58/40c/GPGHMACSHA1--b82a432e11881548052114cfbb07dbc23bd6a2f2/ +deleting annex/objects/a58/40c/GPGHMACSHA1--8bc64222a6b9f74d691845b2990dcd8bc3607c53/GPGHMACSHA1--8bc64222a6b9f74d691845b2990dcd8bc3607c53 +deleting annex/objects/a58/40c/GPGHMACSHA1--8bc64222a6b9f74d691845b2990dcd8bc3607c53/ +deleting annex/objects/a58/40c/GPGHMACSHA1--3d51a6a12578c8dc47871a8c02358ea34f48343f/GPGHMACSHA1--3d51a6a12578c8dc47871a8c02358ea34f48343f +deleting annex/objects/a58/40c/GPGHMACSHA1--3d51a6a12578c8dc47871a8c02358ea34f48343f/ +deleting annex/objects/a58/40c/ +deleting annex/objects/a58/40b/GPGHMACSHA1--dd270aa886d4c15376ad326181c990040d282f40/GPGHMACSHA1--dd270aa886d4c15376ad326181c990040d282f40 +deleting annex/objects/a58/40b/GPGHMACSHA1--dd270aa886d4c15376ad326181c990040d282f40/ +deleting annex/objects/a58/40b/ +deleting annex/objects/a58/40a/GPGHMACSHA1--c2e13979509fc921486ef72ba2c61467a53056d4/GPGHMACSHA1--c2e13979509fc921486ef72ba2c61467a53056d4 +deleting annex/objects/a58/40a/GPGHMACSHA1--c2e13979509fc921486ef72ba2c61467a53056d4/ +deleting annex/objects/a58/40a/ +deleting annex/objects/a58/409/GPGHMACSHA1--240222ad5c53cb016451fdae42c21ad30743985a/GPGHMACSHA1--240222ad5c53cb016451fdae42c21ad30743985a +deleting annex/objects/a58/409/GPGHMACSHA1--240222ad5c53cb016451fdae42c21ad30743985a/ +deleting annex/objects/a58/409/ +deleting annex/objects/a58/408/GPGHMACSHA1--cc1d16826d643817452fe552771a9ff016a26bdb/GPGHMACSHA1--cc1d16826d643817452fe552771a9ff016a26bdb +deleting annex/objects/a58/408/GPGHMACSHA1--cc1d16826d643817452fe552771a9ff016a26bdb/ +deleting annex/objects/a58/408/GPGHMACSHA1--3ec16720e3fe8898273e5abdad06fd39856cd50d/GPGHMACSHA1--3ec16720e3fe8898273e5abdad06fd39856cd50d +deleting annex/objects/a58/408/GPGHMACSHA1--3ec16720e3fe8898273e5abdad06fd39856cd50d/ +deleting annex/objects/a58/408/GPGHMACSHA1--1fde349a75f9d3e8c41e9e0877cae50f38d4f56e/GPGHMACSHA1--1fde349a75f9d3e8c41e9e0877cae50f38d4f56e +deleting annex/objects/a58/408/GPGHMACSHA1--1fde349a75f9d3e8c41e9e0877cae50f38d4f56e/ +deleting annex/objects/a58/408/ +deleting annex/objects/a58/405/GPGHMACSHA1--ca7443f8ad57d2b996e8f4a8afe8daad71f01fc4/GPGHMACSHA1--ca7443f8ad57d2b996e8f4a8afe8daad71f01fc4 +deleting annex/objects/a58/405/GPGHMACSHA1--ca7443f8ad57d2b996e8f4a8afe8daad71f01fc4/ +deleting annex/objects/a58/405/ +deleting annex/objects/a58/404/GPGHMACSHA1--bbd54e0141fef19007d61d5b8b5535af4d0f1643/GPGHMACSHA1--bbd54e0141fef19007d61d5b8b5535af4d0f1643 +deleting annex/objects/a58/404/GPGHMACSHA1--bbd54e0141fef19007d61d5b8b5535af4d0f1643/ +deleting annex/objects/a58/404/GPGHMACSHA1--8903f2b4259d0e5a9361a721b09743dcfea700c7/GPGHMACSHA1--8903f2b4259d0e5a9361a721b09743dcfea700c7 +deleting annex/objects/a58/404/GPGHMACSHA1--8903f2b4259d0e5a9361a721b09743dcfea700c7/ +deleting annex/objects/a58/404/GPGHMACSHA1--66d8456bcc884c16b0ffb1307aea93f0fdff891f/GPGHMACSHA1--66d8456bcc884c16b0ffb1307aea93f0fdff891f +deleting annex/objects/a58/404/GPGHMACSHA1--66d8456bcc884c16b0ffb1307aea93f0fdff891f/ +deleting annex/objects/a58/404/ +deleting annex/objects/a58/403/GPGHMACSHA1--312323e3fd4c7bb9f7b757fbfdf418b8d6fa8fc6/GPGHMACSHA1--312323e3fd4c7bb9f7b757fbfdf418b8d6fa8fc6 +deleting annex/objects/a58/403/GPGHMACSHA1--312323e3fd4c7bb9f7b757fbfdf418b8d6fa8fc6/ +deleting annex/objects/a58/403/ +deleting annex/objects/a58/402/GPGHMACSHA1--02883fc6e92bd37eef17924a59ed33785df47982/GPGHMACSHA1--02883fc6e92bd37eef17924a59ed33785df47982 +deleting annex/objects/a58/402/GPGHMACSHA1--02883fc6e92bd37eef17924a59ed33785df47982/ +deleting annex/objects/a58/402/ +deleting annex/objects/a58/401/GPGHMACSHA1--1f0eb6423373a21f52e5bf66288ed61791a1ff55/GPGHMACSHA1--1f0eb6423373a21f52e5bf66288ed61791a1ff55 +deleting annex/objects/a58/401/GPGHMACSHA1--1f0eb6423373a21f52e5bf66288ed61791a1ff55/ +deleting annex/objects/a58/401/ +deleting annex/objects/a58/3ff/GPGHMACSHA1--fa03c466efb77de3a63fdfa0da4f7742c5a4129c/GPGHMACSHA1--fa03c466efb77de3a63fdfa0da4f7742c5a4129c +deleting annex/objects/a58/3ff/GPGHMACSHA1--fa03c466efb77de3a63fdfa0da4f7742c5a4129c/ +deleting annex/objects/a58/3ff/GPGHMACSHA1--e508ac40b3ecf16264bd1eb85ac500083b7f6f69/GPGHMACSHA1--e508ac40b3ecf16264bd1eb85ac500083b7f6f69 +deleting annex/objects/a58/3ff/GPGHMACSHA1--e508ac40b3ecf16264bd1eb85ac500083b7f6f69/ +deleting annex/objects/a58/3ff/GPGHMACSHA1--a9be7525b4d169ca7042a590f6fd7a2eb375a229/GPGHMACSHA1--a9be7525b4d169ca7042a590f6fd7a2eb375a229 +deleting annex/objects/a58/3ff/GPGHMACSHA1--a9be7525b4d169ca7042a590f6fd7a2eb375a229/ +deleting annex/objects/a58/3ff/ +deleting annex/objects/a58/3fc/GPGHMACSHA1--a88f25b5301ce8ec0ac051d392552119d0c9a049/GPGHMACSHA1--a88f25b5301ce8ec0ac051d392552119d0c9a049 +deleting annex/objects/a58/3fc/GPGHMACSHA1--a88f25b5301ce8ec0ac051d392552119d0c9a049/ +deleting annex/objects/a58/3fc/GPGHMACSHA1--5925494f94b08d06e60b14ac83d27213e2b9e7cd/GPGHMACSHA1--5925494f94b08d06e60b14ac83d27213e2b9e7cd +deleting annex/objects/a58/3fc/GPGHMACSHA1--5925494f94b08d06e60b14ac83d27213e2b9e7cd/ +deleting annex/objects/a58/3fc/GPGHMACSHA1--1f6fafd025532b83226bb2eabd3a7144d832456a/GPGHMACSHA1--1f6fafd025532b83226bb2eabd3a7144d832456a +deleting annex/objects/a58/3fc/GPGHMACSHA1--1f6fafd025532b83226bb2eabd3a7144d832456a/ +deleting annex/objects/a58/3fc/ +deleting annex/objects/a58/3fa/GPGHMACSHA1--3489640e7f6847c971ef765e198ea2c303886525/GPGHMACSHA1--3489640e7f6847c971ef765e198ea2c303886525 +deleting annex/objects/a58/3fa/GPGHMACSHA1--3489640e7f6847c971ef765e198ea2c303886525/ +deleting annex/objects/a58/3fa/GPGHMACSHA1--1f8c42ce1f0a91da6999cd50cddd1502c3351edc/GPGHMACSHA1--1f8c42ce1f0a91da6999cd50cddd1502c3351edc +deleting annex/objects/a58/3fa/GPGHMACSHA1--1f8c42ce1f0a91da6999cd50cddd1502c3351edc/ +deleting annex/objects/a58/3fa/ +deleting annex/objects/a58/3f9/GPGHMACSHA1--b34eb2f15b5089250e573d852e85a3d3782f1dee/GPGHMACSHA1--b34eb2f15b5089250e573d852e85a3d3782f1dee +deleting annex/objects/a58/3f9/GPGHMACSHA1--b34eb2f15b5089250e573d852e85a3d3782f1dee/ +deleting annex/objects/a58/3f9/GPGHMACSHA1--71d0ec95dc3e0e2c843511ec6e4456244e48a6a7/GPGHMACSHA1--71d0ec95dc3e0e2c843511ec6e4456244e48a6a7 +deleting annex/objects/a58/3f9/GPGHMACSHA1--71d0ec95dc3e0e2c843511ec6e4456244e48a6a7/ +deleting annex/objects/a58/3f9/GPGHMACSHA1--7139cde294af35cf57178119a92cdf8c9220d104/GPGHMACSHA1--7139cde294af35cf57178119a92cdf8c9220d104 +deleting annex/objects/a58/3f9/GPGHMACSHA1--7139cde294af35cf57178119a92cdf8c9220d104/ +deleting annex/objects/a58/3f9/ +deleting annex/objects/a58/3f8/GPGHMACSHA1--bd7fd32c0dcaccb05b94e9707d50422548e9e375/GPGHMACSHA1--bd7fd32c0dcaccb05b94e9707d50422548e9e375 +deleting annex/objects/a58/3f8/GPGHMACSHA1--bd7fd32c0dcaccb05b94e9707d50422548e9e375/ +deleting annex/objects/a58/3f8/GPGHMACSHA1--881feef58c16e791ce4657d7637b9617a04b8be1/GPGHMACSHA1--881feef58c16e791ce4657d7637b9617a04b8be1 +deleting annex/objects/a58/3f8/GPGHMACSHA1--881feef58c16e791ce4657d7637b9617a04b8be1/ +deleting annex/objects/a58/3f8/GPGHMACSHA1--3c2b5d82e193cde1fabfc258d1233cb3fc293481/GPGHMACSHA1--3c2b5d82e193cde1fabfc258d1233cb3fc293481 +deleting annex/objects/a58/3f8/GPGHMACSHA1--3c2b5d82e193cde1fabfc258d1233cb3fc293481/ +deleting annex/objects/a58/3f8/ +deleting annex/objects/a58/3f6/GPGHMACSHA1--451c9e5ae1efdf6800e8e6d621a595ef190a22cb/GPGHMACSHA1--451c9e5ae1efdf6800e8e6d621a595ef190a22cb +deleting annex/objects/a58/3f6/GPGHMACSHA1--451c9e5ae1efdf6800e8e6d621a595ef190a22cb/ +deleting annex/objects/a58/3f6/ +deleting annex/objects/a58/3f5/GPGHMACSHA1--42296a1d6fff2ddfc16ca27d9af19e813c0bd77f/GPGHMACSHA1--42296a1d6fff2ddfc16ca27d9af19e813c0bd77f +deleting annex/objects/a58/3f5/GPGHMACSHA1--42296a1d6fff2ddfc16ca27d9af19e813c0bd77f/ +deleting annex/objects/a58/3f5/ +deleting annex/objects/a58/3f4/GPGHMACSHA1--c8f147dd76a7fa05c01617843b93455101dc644d/GPGHMACSHA1--c8f147dd76a7fa05c01617843b93455101dc644d +deleting annex/objects/a58/3f4/GPGHMACSHA1--c8f147dd76a7fa05c01617843b93455101dc644d/ +deleting annex/objects/a58/3f4/GPGHMACSHA1--4674a6cabd9467df8493a8f5a3abb2644080e432/GPGHMACSHA1--4674a6cabd9467df8493a8f5a3abb2644080e432 +deleting annex/objects/a58/3f4/GPGHMACSHA1--4674a6cabd9467df8493a8f5a3abb2644080e432/ +deleting annex/objects/a58/3f4/GPGHMACSHA1--0b96ec853603cdaed97fb8ba4c131a63c4ab9bfb/GPGHMACSHA1--0b96ec853603cdaed97fb8ba4c131a63c4ab9bfb +deleting annex/objects/a58/3f4/GPGHMACSHA1--0b96ec853603cdaed97fb8ba4c131a63c4ab9bfb/ +deleting annex/objects/a58/3f4/ +deleting annex/objects/a58/3f3/GPGHMACSHA1--0075692484ef1064429567e19ad6befd08e40972/GPGHMACSHA1--0075692484ef1064429567e19ad6befd08e40972 +deleting annex/objects/a58/3f3/GPGHMACSHA1--0075692484ef1064429567e19ad6befd08e40972/ +deleting annex/objects/a58/3f3/ +deleting annex/objects/a58/3f2/GPGHMACSHA1--244a4e561d69a591f37ba2e0b72b6063167554b2/GPGHMACSHA1--244a4e561d69a591f37ba2e0b72b6063167554b2 +deleting annex/objects/a58/3f2/GPGHMACSHA1--244a4e561d69a591f37ba2e0b72b6063167554b2/ +deleting annex/objects/a58/3f2/ +deleting annex/objects/a58/3f1/GPGHMACSHA1--e5fd17f4c1f0361296f6fb84b6feba6d38f05419/GPGHMACSHA1--e5fd17f4c1f0361296f6fb84b6feba6d38f05419 +deleting annex/objects/a58/3f1/GPGHMACSHA1--e5fd17f4c1f0361296f6fb84b6feba6d38f05419/ +deleting annex/objects/a58/3f1/ +deleting annex/objects/a58/3f0/GPGHMACSHA1--e9e5b07a64b6d13664d96d50b74947874ab1c2d0/GPGHMACSHA1--e9e5b07a64b6d13664d96d50b74947874ab1c2d0 +deleting annex/objects/a58/3f0/GPGHMACSHA1--e9e5b07a64b6d13664d96d50b74947874ab1c2d0/ +deleting annex/objects/a58/3f0/GPGHMACSHA1--7eb8c1fc1b0dd5a85048c5af87e9bf91219f8129/GPGHMACSHA1--7eb8c1fc1b0dd5a85048c5af87e9bf91219f8129 +deleting annex/objects/a58/3f0/GPGHMACSHA1--7eb8c1fc1b0dd5a85048c5af87e9bf91219f8129/ +deleting annex/objects/a58/3f0/ +deleting annex/objects/a58/3ef/GPGHMACSHA1--1739cd4196bc2e780c4b28f5b26fa62f5284280c/GPGHMACSHA1--1739cd4196bc2e780c4b28f5b26fa62f5284280c +deleting annex/objects/a58/3ef/GPGHMACSHA1--1739cd4196bc2e780c4b28f5b26fa62f5284280c/ +deleting annex/objects/a58/3ef/ +deleting annex/objects/a58/3ee/GPGHMACSHA1--fdf34f469975908f27a9a617de8118bc59378f31/GPGHMACSHA1--fdf34f469975908f27a9a617de8118bc59378f31 +deleting annex/objects/a58/3ee/GPGHMACSHA1--fdf34f469975908f27a9a617de8118bc59378f31/ +deleting annex/objects/a58/3ee/GPGHMACSHA1--407e896996b493a3940a01d60a9728cce5af921c/GPGHMACSHA1--407e896996b493a3940a01d60a9728cce5af921c +deleting annex/objects/a58/3ee/GPGHMACSHA1--407e896996b493a3940a01d60a9728cce5af921c/ +deleting annex/objects/a58/3ee/ +deleting annex/objects/a58/3ed/GPGHMACSHA1--cf7a58e6dc5d7b26b60818865e20856046ce7b94/GPGHMACSHA1--cf7a58e6dc5d7b26b60818865e20856046ce7b94 +deleting annex/objects/a58/3ed/GPGHMACSHA1--cf7a58e6dc5d7b26b60818865e20856046ce7b94/ +deleting annex/objects/a58/3ed/ +deleting annex/objects/a58/3ec/GPGHMACSHA1--b02eb4a5c6191102be83e8505b15e0bb5494ead7/GPGHMACSHA1--b02eb4a5c6191102be83e8505b15e0bb5494ead7 +deleting annex/objects/a58/3ec/GPGHMACSHA1--b02eb4a5c6191102be83e8505b15e0bb5494ead7/ +deleting annex/objects/a58/3ec/GPGHMACSHA1--6e5a1a75c0a10b7b97a6d2052682b103c6ea5b49/GPGHMACSHA1--6e5a1a75c0a10b7b97a6d2052682b103c6ea5b49 +deleting annex/objects/a58/3ec/GPGHMACSHA1--6e5a1a75c0a10b7b97a6d2052682b103c6ea5b49/ +deleting annex/objects/a58/3ec/GPGHMACSHA1--5bcd2f296b94b1b61e5fe3f14955fb12fd21bc28/GPGHMACSHA1--5bcd2f296b94b1b61e5fe3f14955fb12fd21bc28 +deleting annex/objects/a58/3ec/GPGHMACSHA1--5bcd2f296b94b1b61e5fe3f14955fb12fd21bc28/ +deleting annex/objects/a58/3ec/ +deleting annex/objects/a58/3eb/GPGHMACSHA1--d08024f602c0df45ae109ae64d1415037dafb7fe/GPGHMACSHA1--d08024f602c0df45ae109ae64d1415037dafb7fe +deleting annex/objects/a58/3eb/GPGHMACSHA1--d08024f602c0df45ae109ae64d1415037dafb7fe/ +deleting annex/objects/a58/3eb/GPGHMACSHA1--55fceb15d24bfe24d6cb63829d79adbe180e8065/GPGHMACSHA1--55fceb15d24bfe24d6cb63829d79adbe180e8065 +deleting annex/objects/a58/3eb/GPGHMACSHA1--55fceb15d24bfe24d6cb63829d79adbe180e8065/ +deleting annex/objects/a58/3eb/ +deleting annex/objects/a58/3ea/GPGHMACSHA1--bc0ccd3ceadb28bfdcae8588ce501a2954b378cb/GPGHMACSHA1--bc0ccd3ceadb28bfdcae8588ce501a2954b378cb +deleting annex/objects/a58/3ea/GPGHMACSHA1--bc0ccd3ceadb28bfdcae8588ce501a2954b378cb/ +deleting annex/objects/a58/3ea/ +deleting annex/objects/a58/3e8/GPGHMACSHA1--7a446562f98d955846e443e666f236ca31ed83d2/GPGHMACSHA1--7a446562f98d955846e443e666f236ca31ed83d2 +deleting annex/objects/a58/3e8/GPGHMACSHA1--7a446562f98d955846e443e666f236ca31ed83d2/ +deleting annex/objects/a58/3e8/GPGHMACSHA1--0cd126b3f7d9d29c0498ea46db2dfc695b4cdfd2/GPGHMACSHA1--0cd126b3f7d9d29c0498ea46db2dfc695b4cdfd2 +deleting annex/objects/a58/3e8/GPGHMACSHA1--0cd126b3f7d9d29c0498ea46db2dfc695b4cdfd2/ +deleting annex/objects/a58/3e8/ +deleting annex/objects/a58/3e7/GPGHMACSHA1--5c17f4d459e1b6bb6d4914cfcfeb06649eccfd22/GPGHMACSHA1--5c17f4d459e1b6bb6d4914cfcfeb06649eccfd22 +deleting annex/objects/a58/3e7/GPGHMACSHA1--5c17f4d459e1b6bb6d4914cfcfeb06649eccfd22/ +deleting annex/objects/a58/3e7/GPGHMACSHA1--24bff8cdcae0a2adb48ccf06f29233d3e06ad1fd/GPGHMACSHA1--24bff8cdcae0a2adb48ccf06f29233d3e06ad1fd +deleting annex/objects/a58/3e7/GPGHMACSHA1--24bff8cdcae0a2adb48ccf06f29233d3e06ad1fd/ +deleting annex/objects/a58/3e7/GPGHMACSHA1--12f46408733a72e3c07cfeed31be142dcebc0842/GPGHMACSHA1--12f46408733a72e3c07cfeed31be142dcebc0842 +deleting annex/objects/a58/3e7/GPGHMACSHA1--12f46408733a72e3c07cfeed31be142dcebc0842/ +deleting annex/objects/a58/3e7/ +deleting annex/objects/a58/3e6/GPGHMACSHA1--f677ef99252db6ea99f7b418fdc8123de1416345/GPGHMACSHA1--f677ef99252db6ea99f7b418fdc8123de1416345 +deleting annex/objects/a58/3e6/GPGHMACSHA1--f677ef99252db6ea99f7b418fdc8123de1416345/ +deleting annex/objects/a58/3e6/GPGHMACSHA1--16cf185094c7a911ddb7b764019d99f4f568b24b/GPGHMACSHA1--16cf185094c7a911ddb7b764019d99f4f568b24b +deleting annex/objects/a58/3e6/GPGHMACSHA1--16cf185094c7a911ddb7b764019d99f4f568b24b/ +deleting annex/objects/a58/3e6/ +deleting annex/objects/a58/3e5/GPGHMACSHA1--defdaaaa2a245befed23605e67deb230c907d619/GPGHMACSHA1--defdaaaa2a245befed23605e67deb230c907d619 +deleting annex/objects/a58/3e5/GPGHMACSHA1--defdaaaa2a245befed23605e67deb230c907d619/ +deleting annex/objects/a58/3e5/GPGHMACSHA1--d7df5ec1b874d734ec0b95ca4c990453a546a6b0/GPGHMACSHA1--d7df5ec1b874d734ec0b95ca4c990453a546a6b0 +deleting annex/objects/a58/3e5/GPGHMACSHA1--d7df5ec1b874d734ec0b95ca4c990453a546a6b0/ +deleting annex/objects/a58/3e5/GPGHMACSHA1--96656aaece6f4172d57db4c7d8910e3ea77fb824/GPGHMACSHA1--96656aaece6f4172d57db4c7d8910e3ea77fb824 +deleting annex/objects/a58/3e5/GPGHMACSHA1--96656aaece6f4172d57db4c7d8910e3ea77fb824/ +deleting annex/objects/a58/3e5/GPGHMACSHA1--8e97545269570b5a12868517fc5b14c458ab383c/GPGHMACSHA1--8e97545269570b5a12868517fc5b14c458ab383c +deleting annex/objects/a58/3e5/GPGHMACSHA1--8e97545269570b5a12868517fc5b14c458ab383c/ +deleting annex/objects/a58/3e5/GPGHMACSHA1--5c541e81265c22dd0834527682b365828dadfbbe/GPGHMACSHA1--5c541e81265c22dd0834527682b365828dadfbbe +deleting annex/objects/a58/3e5/GPGHMACSHA1--5c541e81265c22dd0834527682b365828dadfbbe/ +deleting annex/objects/a58/3e5/GPGHMACSHA1--1f54292f57783859d40382e012973ea930d4c67d/GPGHMACSHA1--1f54292f57783859d40382e012973ea930d4c67d +deleting annex/objects/a58/3e5/GPGHMACSHA1--1f54292f57783859d40382e012973ea930d4c67d/ +deleting annex/objects/a58/3e5/ +deleting annex/objects/a58/3e4/GPGHMACSHA1--f46ad8b243436e1186cb821ee640f402568d6088/GPGHMACSHA1--f46ad8b243436e1186cb821ee640f402568d6088 +deleting annex/objects/a58/3e4/GPGHMACSHA1--f46ad8b243436e1186cb821ee640f402568d6088/ +deleting annex/objects/a58/3e4/GPGHMACSHA1--e82e2f9e6ce1fed17bf5820f96ebf7ceb6557caa/GPGHMACSHA1--e82e2f9e6ce1fed17bf5820f96ebf7ceb6557caa +deleting annex/objects/a58/3e4/GPGHMACSHA1--e82e2f9e6ce1fed17bf5820f96ebf7ceb6557caa/ +deleting annex/objects/a58/3e4/ +deleting annex/objects/a58/3e3/GPGHMACSHA1--c569427bb2404b11091c0c47c7b7c1faafcc7309/GPGHMACSHA1--c569427bb2404b11091c0c47c7b7c1faafcc7309 +deleting annex/objects/a58/3e3/GPGHMACSHA1--c569427bb2404b11091c0c47c7b7c1faafcc7309/ +deleting annex/objects/a58/3e3/GPGHMACSHA1--34126a50061184ceccca13122c92ee2d1e0fcaaa/GPGHMACSHA1--34126a50061184ceccca13122c92ee2d1e0fcaaa +deleting annex/objects/a58/3e3/GPGHMACSHA1--34126a50061184ceccca13122c92ee2d1e0fcaaa/ +deleting annex/objects/a58/3e3/ +deleting annex/objects/a58/3e2/GPGHMACSHA1--a5db222b17e55313bbbcc007d3b421eb09f31b2a/GPGHMACSHA1--a5db222b17e55313bbbcc007d3b421eb09f31b2a +deleting annex/objects/a58/3e2/GPGHMACSHA1--a5db222b17e55313bbbcc007d3b421eb09f31b2a/ +deleting annex/objects/a58/3e2/GPGHMACSHA1--198c51669cffbf2718c670cff58f11b93ef166c5/GPGHMACSHA1--198c51669cffbf2718c670cff58f11b93ef166c5 +deleting annex/objects/a58/3e2/GPGHMACSHA1--198c51669cffbf2718c670cff58f11b93ef166c5/ +deleting annex/objects/a58/3e2/ +deleting annex/objects/a58/3e1/GPGHMACSHA1--ea6a2b40219a96f08d8becc4a112734c7b343bda/GPGHMACSHA1--ea6a2b40219a96f08d8becc4a112734c7b343bda +deleting annex/objects/a58/3e1/GPGHMACSHA1--ea6a2b40219a96f08d8becc4a112734c7b343bda/ +deleting annex/objects/a58/3e1/ +deleting annex/objects/a58/3e0/GPGHMACSHA1--c039cc5b781f964a5b2045217200231763093ef3/GPGHMACSHA1--c039cc5b781f964a5b2045217200231763093ef3 +deleting annex/objects/a58/3e0/GPGHMACSHA1--c039cc5b781f964a5b2045217200231763093ef3/ +deleting annex/objects/a58/3e0/GPGHMACSHA1--6fec4dbcbe44ecbbc39370b7c232e240d65f2c96/GPGHMACSHA1--6fec4dbcbe44ecbbc39370b7c232e240d65f2c96 +deleting annex/objects/a58/3e0/GPGHMACSHA1--6fec4dbcbe44ecbbc39370b7c232e240d65f2c96/ +deleting annex/objects/a58/3e0/ +deleting annex/objects/a58/3de/GPGHMACSHA1--3f9daeb81e907a26d1188fe66996c2ac7c9c9c88/GPGHMACSHA1--3f9daeb81e907a26d1188fe66996c2ac7c9c9c88 +deleting annex/objects/a58/3de/GPGHMACSHA1--3f9daeb81e907a26d1188fe66996c2ac7c9c9c88/ +deleting annex/objects/a58/3de/ +deleting annex/objects/a58/3dd/GPGHMACSHA1--ada3608b08f1c9b760f36a401fa11f378e8ccae2/GPGHMACSHA1--ada3608b08f1c9b760f36a401fa11f378e8ccae2 +deleting annex/objects/a58/3dd/GPGHMACSHA1--ada3608b08f1c9b760f36a401fa11f378e8ccae2/ +deleting annex/objects/a58/3dd/ +deleting annex/objects/a58/3dc/GPGHMACSHA1--92d83c6b5b977a3d0fcf13e3b9f57a6e5401c4e3/GPGHMACSHA1--92d83c6b5b977a3d0fcf13e3b9f57a6e5401c4e3 +deleting annex/objects/a58/3dc/GPGHMACSHA1--92d83c6b5b977a3d0fcf13e3b9f57a6e5401c4e3/ +deleting annex/objects/a58/3dc/ +deleting annex/objects/a58/3da/GPGHMACSHA1--f10d6600aa7837f0ce79c35f5c253f245b5e0b64/GPGHMACSHA1--f10d6600aa7837f0ce79c35f5c253f245b5e0b64 +deleting annex/objects/a58/3da/GPGHMACSHA1--f10d6600aa7837f0ce79c35f5c253f245b5e0b64/ +deleting annex/objects/a58/3da/GPGHMACSHA1--14f6e1dc8b216df0987fd8d46dac81b7341e95a2/GPGHMACSHA1--14f6e1dc8b216df0987fd8d46dac81b7341e95a2 +deleting annex/objects/a58/3da/GPGHMACSHA1--14f6e1dc8b216df0987fd8d46dac81b7341e95a2/ +deleting annex/objects/a58/3da/ +deleting annex/objects/a58/3d7/GPGHMACSHA1--d124b6c5cbc28f4d3324edd74a36ebe4ea9c88f7/GPGHMACSHA1--d124b6c5cbc28f4d3324edd74a36ebe4ea9c88f7 +deleting annex/objects/a58/3d7/GPGHMACSHA1--d124b6c5cbc28f4d3324edd74a36ebe4ea9c88f7/ +deleting annex/objects/a58/3d7/GPGHMACSHA1--1ee83c3a03672e9ae99ea19483eea58979f81605/GPGHMACSHA1--1ee83c3a03672e9ae99ea19483eea58979f81605 +deleting annex/objects/a58/3d7/GPGHMACSHA1--1ee83c3a03672e9ae99ea19483eea58979f81605/ +deleting annex/objects/a58/3d7/ +deleting annex/objects/a58/3d6/GPGHMACSHA1--977aaeca816759b409a052716a0e339147a52103/GPGHMACSHA1--977aaeca816759b409a052716a0e339147a52103 +deleting annex/objects/a58/3d6/GPGHMACSHA1--977aaeca816759b409a052716a0e339147a52103/ +deleting annex/objects/a58/3d6/GPGHMACSHA1--7f23acb14341d1f0efe1be2da179a8c9d2769343/GPGHMACSHA1--7f23acb14341d1f0efe1be2da179a8c9d2769343 +... +... +"""]] diff --git a/doc/bugs/All_inodes_eaten/comment_3_865c75eeee3479321f4c0f00d34e063d._comment b/doc/bugs/All_inodes_eaten/comment_3_865c75eeee3479321f4c0f00d34e063d._comment new file mode 100644 index 0000000000..a28c499db8 --- /dev/null +++ b/doc/bugs/All_inodes_eaten/comment_3_865c75eeee3479321f4c0f00d34e063d._comment @@ -0,0 +1,13 @@ +[[!comment format=mdwn + username="joey" + subject="""comment 3""" + date="2016-10-26T18:07:51Z" + content=""" +Well these look like the names I'd expect to see used for encrypted files. + +If you had the assistant watching some files that were frequently changing, +then it could lead to something like this, since many commits would be made +of many versions of a file, and each version backed up to a new encrypted +file in the special remote. You can take a look at `git log -S` in the git +repository and see if there are many commits of some of your files. +"""]] diff --git a/doc/bugs/Allow_automatic_retry_git_annex_get.mdwn b/doc/bugs/Allow_automatic_retry_git_annex_get.mdwn new file mode 100644 index 0000000000..c6e406b490 --- /dev/null +++ b/doc/bugs/Allow_automatic_retry_git_annex_get.mdwn @@ -0,0 +1,61 @@ +### Please describe the problem. +Often, when I `git annex get` a large file over ssh, it will hang in the middle, and I'll have to C-c to cancel it, and try again. This strategy works, but I often have to keep canceling and retrying 5-10 times for a large file. This is unfortunate. I think maybe rsync is getting overwhelmed. + +Request: Could you please add a flag --autoretry so that if `get` hangs, cancel and try again. This generally works, because of the rsync --partial flag, so it eventually gets to the end of the file. Thanks! + +### What steps will reproduce the problem? + + +### What version of git-annex are you using? On what operating system? +git-annex version: 6.20160511-1~ubuntu14.04.1~ppa1 +Ubuntu 14.04 + +### Please provide any additional information below. + +[[!format sh """ +# If you can, paste a complete transcript of the problem occurring here. +# If the problem is with the git-annex assistant, paste in .git/annex/daemon.log + +$ git annex get --debug +[2016-08-26 15:59:19.476393] read: git ["--git-dir=.git","--work-tree=.","--literal-pathspecs","ls-files","--cached","-z","--"] +get clerk/H101-2.json [2016-08-26 15:59:19.483388] read: git ["--git-dir=.git","--work-tree=.","--literal-pathspecs","show-ref","git-annex"] +[2016-08-26 15:59:19.485433] process done ExitSuccess +[2016-08-26 15:59:19.485676] read: git ["--git-dir=.git","--work-tree=.","--literal-pathspecs","show-ref","--hash","refs/heads/git-annex"] +[2016-08-26 15:59:19.48755] process done ExitSuccess +[2016-08-26 15:59:19.487727] read: git ["--git-dir=.git","--work-tree=.","--literal-pathspecs","log","refs/heads/git-annex..4455cba13d289aad3c25ddc097b5ca02e7fc0ffb","--pretty=%H","-n1"] +[2016-08-26 15:59:19.489793] process done ExitSuccess +[2016-08-26 15:59:19.4899] read: git ["--git-dir=.git","--work-tree=.","--literal-pathspecs","log","refs/heads/git-annex..7af97d408ed894922cd08bd2091dab5bb1cf1df9","--pretty=%H","-n1"] +[2016-08-26 15:59:19.49133] process done ExitSuccess +[2016-08-26 15:59:19.491939] chat: git ["--git-dir=.git","--work-tree=.","--literal-pathspecs","cat-file","--batch"] +(from origin...) +[2016-08-26 15:59:19.495688] read: rsync ["--progress","--inplace","--perms","-e","'ssh' '-S' '.git/annex/ssh/voteviewdev' '-o' 'ControlMaster=auto' '-o' 'ControlPersist=yes' '-T' 'voteviewdev' 'git-annex-shell ''sendkey'' ''/srv/annex'' ''--debug'' ''SHA256E-s49892641--7390d24449d67e1d60c31306bfc8b189f7c4ed223cc44bbd0096222b8ede3b92.json'' --uuid ff59187e-f48c-47d0-b75d-320a6a4ce115 ''--'' ''remoteuuid=da1361a1-3f1a-47b8-8165-daf8c30d543d'' ''unlocked='' ''direct='' ''associatedfile=clerk/H101-2.json'' ''--'''","--","dummy:",".git/annex/tmp/SHA256E-s49892641--7390d24449d67e1d60c31306bfc8b189f7c4ed223cc44bbd0096222b8ede3b92.json"] +[2016-08-26 15:58:37.018182] transfer start +[2016-08-26 15:58:37.023135] call: rsync ["--server","-t","--inplace","-e.Lsf",".","--sender","../../srv/annex/.git/annex/objects/2G/fP/SHA256E-s49892641--7390d24449d67e1d60c31306bfc8b189f7c4ed223cc44bbd0096222b8ede3b92.json/SHA256E-s49892641--7390d24449d67e1d60c31306bfc8b189f7c4ed223cc44bbd0096222b8ede3b92.json"] +SHA256E-s49892641--7390d24449d67e1d60c31306bfc8b189f7c4ed223cc44bbd0096222b8ede3b92.json + 49,101,968 98% 46.83MB/s 0:00:00 [2016-08-26 15:59:22.809273] feed: ssh ["-S",".git/annex/ssh/voteviewdev","-o","ControlMaster=auto","-o","ControlPersist=yes","-T","voteviewdev","git-annex-shell 'transferinfo' '/srv/annex' '--debug' 'SHA256E-s49892641--7390d24449d67e1d60c31306bfc8b189f7c4ed223cc44bbd0096222b8ede3b92.json' --uuid ff59187e-f48c-47d0-b75d-320a6a4ce115 '--' 'remoteuuid=da1361a1-3f1a-47b8-8165-daf8c30d543d' 'associatedfile=clerk/H101-2.json' '--'"] + 49,892,641 100% 39.52MB/s 0:00:01 (xfr#1, to-chk=0/1) +[2016-08-26 15:58:39.034666] process done ExitSuccess +[2016-08-26 15:58:39.038811] transfer done +[2016-08-26 15:59:23.09308] process done ExitSuccess +(checksum...) [2016-08-26 15:59:23.093582] read: sha256sum [".git/annex/tmp/SHA256E-s49892641--7390d24449d67e1d60c31306bfc8b189f7c4ed223cc44bbd0096222b8ede3b92.json"] +[2016-08-26 15:59:23.331872] process done ExitSuccess +ok +get clerk/H102-1.json (from origin...) +[2016-08-26 15:59:23.337838] read: rsync ["--progress","--inplace","--perms","-e","'ssh' '-S' '.git/annex/ssh/voteviewdev' '-o' 'ControlMaster=auto' '-o' 'ControlPersist=yes' '-T' 'voteviewdev' 'git-annex-shell ''sendkey'' ''/srv/annex'' ''--debug'' ''SHA256E-s41311329--69c3b054a3fe9676133605730d85b7fcef8696f6782d402a524e41b836253891.json'' --uuid ff59187e-f48c-47d0-b75d-320a6a4ce115 ''--'' ''remoteuuid=da1361a1-3f1a-47b8-8165-daf8c30d543d'' ''unlocked='' ''direct='' ''associatedfile=clerk/H102-1.json'' ''--'''","--","dummy:",".git/annex/tmp/SHA256E-s41311329--69c3b054a3fe9676133605730d85b7fcef8696f6782d402a524e41b836253891.json"] +[2016-08-26 15:59:23.351903] process done ExitSuccess +[2016-08-26 15:58:39.394898] transfer start +[2016-08-26 15:58:39.400323] call: rsync ["--server","-t","--inplace","-e.Lsf",".","--sender","../../srv/annex/.git/annex/objects/6Z/QP/SHA256E-s41311329--69c3b054a3fe9676133605730d85b7fcef8696f6782d402a524e41b836253891.json/SHA256E-s41311329--69c3b054a3fe9676133605730d85b7fcef8696f6782d402a524e41b836253891.json"] +SHA256E-s41311329--69c3b054a3fe9676133605730d85b7fcef8696f6782d402a524e41b836253891.json + 1,998,848 4% 1.91MB/s 0:00:20 [2016-08-26 15:59:24.608637] feed: ssh ["-S",".git/annex/ssh/voteviewdev","-o","ControlMaster=auto","-o","ControlPersist=yes","-T","voteviewdev","git-annex-shell 'transferinfo' '/srv/annex' '--debug' 'SHA256E-s41311329--69c3b054a3fe9676133605730d85b7fcef8696f6782d402a524e41b836253891.json' --uuid ff59187e-f48c-47d0-b75d-320a6a4ce115 '--' 'remoteuuid=da1361a1-3f1a-47b8-8165-daf8c30d543d' 'associatedfile=clerk/H102-1.json' '--'"] + 10,616,832 25% 1.54MB/s 0:00:19 + + + +# End of transcript or log. +"""]] + +### Have you had any luck using git-annex before? (Sometimes we get tired of reading bug reports all day and a lil' positive end note does wonders) + + + +[[!meta title="Detect stalled transfer and retry or abort it"]] diff --git a/doc/bugs/Allow_automatic_retry_git_annex_get/comment_1_a419d2868b4f99098644e84eef07d47c._comment b/doc/bugs/Allow_automatic_retry_git_annex_get/comment_1_a419d2868b4f99098644e84eef07d47c._comment new file mode 100644 index 0000000000..3b2b139e49 --- /dev/null +++ b/doc/bugs/Allow_automatic_retry_git_annex_get/comment_1_a419d2868b4f99098644e84eef07d47c._comment @@ -0,0 +1,9 @@ +[[!comment format=mdwn + username="adamboche@d77483462c747486c7a7341a9080d1341eeddfe6" + nickname="adamboche" + subject="Retry rsync on failure" + date="2016-08-26T23:32:46Z" + content=""" +This may be useful for providing an in-product solution to this problem. +http://unix.stackexchange.com/a/165417/126433 +"""]] diff --git a/doc/bugs/Allow_automatic_retry_git_annex_get/comment_2_6d05cd09e1f00fb5ace2b9ae3bffdedb._comment b/doc/bugs/Allow_automatic_retry_git_annex_get/comment_2_6d05cd09e1f00fb5ace2b9ae3bffdedb._comment new file mode 100644 index 0000000000..6d2563e46d --- /dev/null +++ b/doc/bugs/Allow_automatic_retry_git_annex_get/comment_2_6d05cd09e1f00fb5ace2b9ae3bffdedb._comment @@ -0,0 +1,66 @@ +[[!comment format=mdwn + username="joey" + subject="""comment 2""" + date="2016-10-26T18:26:35Z" + content=""" +The most common way a network connection can stall like this is when +moving to a different wifi network: the connection is open but +no more data will be received. I suppose other kinds of network +glitches could also lead to this kind of situation. + +ssh has some things, like ServerAliveInterval and TCPKeepAlive, +that it can use to detect such problems. You may find them useful. + +---- + +As for the retrying once a stall is detected, some transfers use +`forwardRetry` which will automatically retry as long as the failed try +managed to send some data. But the get/move/copy commands currently use +`noRetry`. I can't find any justification for not always using +`forwardRetry`; I think that it was added for the assistant originally and +the other stuff just never switched over. + +Only problem I can think of is, if there actually is a ssh password +prompt, it would prompt again on retry. But most people using git-annex +with ssh have something in place to make ssh not prompt repeatedly for +passwords. + +So, I've gone ahead and enabled `forwardRetry` for everything. + +---- + +Occurs to me that git-annex could try to notice when a transfer is not +progressing, by reusing the existing progress metering code. + +Since some remotes don't update the progress meter, this could +only be used to detect stalls after the progress meter has been updated +at least once. If the stall occurs earlier than that, it would not be able +to be detected. + +It seems quite hard to come up with a good timeout value to detect a +stalled connection. Often progress meters are updated after every small +(eg 32kb) chunk transferred. But others might poll periodically, or might +use a larger chunk size. It's even possible that some special remotes +are looking at a percent output by some program, and only update the meter +when the percent transferred changes -- in which case it could be many +minutes in between each meter update when a large file is being +transferred. + +If the timeout is too short, git-annex will stall in a new way, by +constantly killing "stalled" connections before they can send enough data. + +---- + +So it really seems better to fix the ssh connection to not stall, since +that is not so heuristic a fix. Seems like git-annex could force +ServerAliveInterval to be set, and perhaps lower ServerAliveCountMax from 3 +to 1. The ssh BatchMode setting sets the former to 300, so a stalled +connection will time out after 15 minutes. But BatchMode also disables +prompting, and git-annex should not disable that. + +Catch is, what if the user has configured ssh with some +other ServerAliveInterval value? We don't want git-annex to override that. + +(git-annex does have a rudimentary .ssh/config parser, but it's not +good enough to handle eg, "Host *.example.com ") +"""]] diff --git a/doc/bugs/Allow_automatic_retry_git_annex_get/comment_3_8b54b726c7d89587e62b6281c9cd915a._comment b/doc/bugs/Allow_automatic_retry_git_annex_get/comment_3_8b54b726c7d89587e62b6281c9cd915a._comment new file mode 100644 index 0000000000..12c31dc111 --- /dev/null +++ b/doc/bugs/Allow_automatic_retry_git_annex_get/comment_3_8b54b726c7d89587e62b6281c9cd915a._comment @@ -0,0 +1,28 @@ +[[!comment format=mdwn + username="joey" + subject="""comment 3""" + date="2016-10-26T19:55:18Z" + content=""" +On the ssh config, one way to do it is to pass -F with a config +file that git-annex generates. It could look like: + + Include ~/.ssh/config + Include /etc/ssh/ssh_config + ServerAliveInterval 60 + +Since ssh uses the first config setting it sees, if `~/.ssh/config` +or `/etc/ssh/ssh_config` set a ServerAliveInterval that one will be used, +and otherwise the value git-annex sets will be used. + +But.. Ssh enables TCPKeepAlive by default. You'd think that would be enough +to detect this kind of problem. + +There do seem to be reasons for users to disable TCPKeepAlive; +perhaps it causes annoying disconnects when there's a minor hiccough, +or a firewall does not support it. + +If the problem is that users are disabling TCPKeepAlive, then +having git-annex enable ServerAliveInterval makes sense. + +Ok; implemented this. +"""]] diff --git a/doc/bugs/Allow_automatic_retry_git_annex_get/comment_4_899b66a20b8e29a23068d249a461c19f._comment b/doc/bugs/Allow_automatic_retry_git_annex_get/comment_4_899b66a20b8e29a23068d249a461c19f._comment new file mode 100644 index 0000000000..6d25069233 --- /dev/null +++ b/doc/bugs/Allow_automatic_retry_git_annex_get/comment_4_899b66a20b8e29a23068d249a461c19f._comment @@ -0,0 +1,16 @@ +[[!comment format=mdwn + username="joey" + subject="""comment 4""" + date="2016-12-13T16:05:42Z" + content=""" +Could the original bug reporter please show what your ~/.ssh/config +contains? As far as I can tell, ssh's TCPKeepAlive option, which is +supposed to be enabled by default, unless you have disabled it, should +avoid such problems. + +It may also help to set ServerAliveInterval. + +Unfortunately, my attempt to make git-annex set ServerAliveInterval +when running ssh broke too many systems with old sshed, and I have had to +revert it. +"""]] diff --git a/doc/bugs/Allow_automatic_retry_git_annex_get/comment_5_786cc686ac7049c19d572329641bbb6d._comment b/doc/bugs/Allow_automatic_retry_git_annex_get/comment_5_786cc686ac7049c19d572329641bbb6d._comment new file mode 100644 index 0000000000..dfb25f9227 --- /dev/null +++ b/doc/bugs/Allow_automatic_retry_git_annex_get/comment_5_786cc686ac7049c19d572329641bbb6d._comment @@ -0,0 +1,30 @@ +[[!comment format=mdwn + username="joey" + subject="""comment 5""" + date="2018-03-29T17:36:48Z" + content=""" +We have annex.retry etc configuration now to control retries of transfers. + +Stall detection would be good to have in git-annex. Currently, when a +transfer is stalled, the progress meter stops updating, so displays +an ETA that's wrong. + +Since the progress meter is already getting called periodically when +a transfer is not stalled, a stall detector could perhaps be added +into that, to abort stalled transfers. The tricky part being that a few +types of transfers don't use the internal progress meters but rely on an +external command's progress display, and the stall detection would need to +be disabled for those. + +Hmm, a password prompt for eg ssh can also "stall" +a transfer for a long time. Perhaps the thing to do is wait until +the meter gets updated once, and only then start the stall detector. + +But what would the stall detector do when it does detect a stalled +transfer? It should perhaps cancel the transfer action, but I don't know +how to do that; the transfer action may have eg run a process, which would +need to be canceled, or it may have a network connection open. Simply +killing the transfer thread won't stop a process that it started. +And when an external special remote is performing the transfer, there's +nothing in that protocol to cancel a transfer. +"""]] diff --git a/doc/bugs/Android_4.4_install_fails_with_permission_denied_errors.mdwn b/doc/bugs/Android_4.4_install_fails_with_permission_denied_errors.mdwn new file mode 100644 index 0000000000..4b1c106843 --- /dev/null +++ b/doc/bugs/Android_4.4_install_fails_with_permission_denied_errors.mdwn @@ -0,0 +1,297 @@ +### Please describe the problem. + +Installing git-annex on a new Nexus 5 with Android 4.4.4 using [Android 4.4 and 4.3 git-annex.apk](http://downloads.kitenet.net/git-annex/android/current/4.3/git-annex.apk) does not give me a working git-annex environment. It seems permission is denied to install many of the app files. + + +### What steps will reproduce the problem? + +1. Install git-annex +2. From within `adb shell`, run: `/data/data/ga.androidterm/runshell` +3. Try one of the included programs, e.g., `git` + + +### What version of git-annex are you using? On what operating system? + +The current (as of 2014-08-30) git-annex for Android 4.3 and up on Android 4.4.4. + + +### Please provide any additional information below. + +Running `/data/data/ga.androidterm/runshell` from `adb shell` gives me: + +[[!format txt """ +shell@hammerhead:/ $ /data/data/ga.androidterm/runshell +Falling back to hardcoded app location; cannot find expected files in /data/app-lib +shell@hammerhead:/sdcard/git-annex.home $ ls +git-annex-install.log +shell@hammerhead:/sdcard/git-annex.home $ cat git-annex-install.log +Installation starting to /data/data/ga.androidterm +71c22504d777380dd59d2128b97715fde9ef6bec +mv: can't rename '/data/data/ga.androidterm/bin': Permission denied +installing busybox +ln: /data/data/ga.androidterm/bin/busybox: Permission denied +installing git-annex +ln: /data/data/ga.androidterm/bin/git-annex: Permission denied +installing git-shell +ln: /data/data/ga.androidterm/bin/git-shell: Permission denied +installing git-upload-pack +ln: /data/data/ga.androidterm/bin/git-upload-pack: Permission denied +installing git +ln: /data/data/ga.androidterm/bin/git: Permission denied +installing gpg +ln: /data/data/ga.androidterm/bin/gpg: Permission denied +installing rsync +ln: /data/data/ga.androidterm/bin/rsync: Permission denied +installing ssh +ln: /data/data/ga.androidterm/bin/ssh: Permission denied +installing ssh-keygen +ln: /data/data/ga.androidterm/bin/ssh-keygen: Permission denied +busybox: /data/data/ga.androidterm/bin/[: Permission denied +busybox: /data/data/ga.androidterm/bin/[[: Permission denied +busybox: /data/data/ga.androidterm/bin/ar: Permission denied +busybox: /data/data/ga.androidterm/bin/arp: Permission denied +busybox: /data/data/ga.androidterm/bin/ash: Permission denied +busybox: /data/data/ga.androidterm/bin/base64: Permission denied +busybox: /data/data/ga.androidterm/bin/basename: Permission denied +busybox: /data/data/ga.androidterm/bin/beep: Permission denied +busybox: /data/data/ga.androidterm/bin/blkid: Permission denied +busybox: /data/data/ga.androidterm/bin/blockdev: Permission denied +busybox: /data/data/ga.androidterm/bin/bunzip2: Permission denied +busybox: /data/data/ga.androidterm/bin/bzcat: Permission denied +busybox: /data/data/ga.androidterm/bin/bzip2: Permission denied +busybox: /data/data/ga.androidterm/bin/cal: Permission denied +busybox: /data/data/ga.androidterm/bin/cat: Permission denied +busybox: /data/data/ga.androidterm/bin/catv: Permission denied +busybox: /data/data/ga.androidterm/bin/chat: Permission denied +busybox: /data/data/ga.androidterm/bin/chattr: Permission denied +busybox: /data/data/ga.androidterm/bin/chgrp: Permission denied +busybox: /data/data/ga.androidterm/bin/chmod: Permission denied +busybox: /data/data/ga.androidterm/bin/chown: Permission denied +busybox: /data/data/ga.androidterm/bin/chpst: Permission denied +busybox: /data/data/ga.androidterm/bin/chroot: Permission denied +busybox: /data/data/ga.androidterm/bin/chrt: Permission denied +busybox: /data/data/ga.androidterm/bin/chvt: Permission denied +busybox: /data/data/ga.androidterm/bin/cksum: Permission denied +busybox: /data/data/ga.androidterm/bin/clear: Permission denied +busybox: /data/data/ga.androidterm/bin/cmp: Permission denied +busybox: /data/data/ga.androidterm/bin/comm: Permission denied +busybox: /data/data/ga.androidterm/bin/cp: Permission denied +busybox: /data/data/ga.androidterm/bin/cpio: Permission denied +busybox: /data/data/ga.androidterm/bin/cttyhack: Permission denied +busybox: /data/data/ga.androidterm/bin/cut: Permission denied +busybox: /data/data/ga.androidterm/bin/dc: Permission denied +busybox: /data/data/ga.androidterm/bin/dd: Permission denied +busybox: /data/data/ga.androidterm/bin/deallocvt: Permission denied +busybox: /data/data/ga.androidterm/bin/devmem: Permission denied +busybox: /data/data/ga.androidterm/bin/diff: Permission denied +busybox: /data/data/ga.androidterm/bin/dirname: Permission denied +busybox: /data/data/ga.androidterm/bin/dmesg: Permission denied +busybox: /data/data/ga.androidterm/bin/dnsd: Permission denied +busybox: /data/data/ga.androidterm/bin/dos2unix: Permission denied +busybox: /data/data/ga.androidterm/bin/dpkg: Permission denied +busybox: /data/data/ga.androidterm/bin/dpkg-deb: Permission denied +busybox: /data/data/ga.androidterm/bin/du: Permission denied +busybox: /data/data/ga.androidterm/bin/dumpkmap: Permission denied +busybox: /data/data/ga.androidterm/bin/echo: Permission denied +busybox: /data/data/ga.androidterm/bin/envdir: Permission denied +busybox: /data/data/ga.androidterm/bin/envuidgid: Permission denied +busybox: /data/data/ga.androidterm/bin/expand: Permission denied +busybox: /data/data/ga.androidterm/bin/fakeidentd: Permission denied +busybox: /data/data/ga.androidterm/bin/false: Permission denied +busybox: /data/data/ga.androidterm/bin/fbset: Permission denied +busybox: /data/data/ga.androidterm/bin/fbsplash: Permission denied +busybox: /data/data/ga.androidterm/bin/fdflush: Permission denied +busybox: /data/data/ga.androidterm/bin/fdformat: Permission denied +busybox: /data/data/ga.androidterm/bin/fdisk: Permission denied +busybox: /data/data/ga.androidterm/bin/fgconsole: Permission denied +busybox: /data/data/ga.androidterm/bin/find: Permission denied +busybox: /data/data/ga.androidterm/bin/findfs: Permission denied +busybox: /data/data/ga.androidterm/bin/flash_lock: Permission denied +busybox: /data/data/ga.androidterm/bin/flash_unlock: Permission denied +busybox: /data/data/ga.androidterm/bin/flashcp: Permission denied +busybox: /data/data/ga.androidterm/bin/flock: Permission denied +busybox: /data/data/ga.androidterm/bin/fold: Permission denied +busybox: /data/data/ga.androidterm/bin/freeramdisk: Permission denied +busybox: /data/data/ga.androidterm/bin/ftpd: Permission denied +busybox: /data/data/ga.androidterm/bin/ftpget: Permission denied +busybox: /data/data/ga.androidterm/bin/ftpput: Permission denied +busybox: /data/data/ga.androidterm/bin/fuser: Permission denied +busybox: /data/data/ga.androidterm/bin/getopt: Permission denied +busybox: /data/data/ga.androidterm/bin/grep: Permission denied +busybox: /data/data/ga.androidterm/bin/gunzip: Permission denied +busybox: /data/data/ga.androidterm/bin/gzip: Permission denied +busybox: /data/data/ga.androidterm/bin/hd: Permission denied +busybox: /data/data/ga.androidterm/bin/hdparm: Permission denied +busybox: /data/data/ga.androidterm/bin/head: Permission denied +busybox: /data/data/ga.androidterm/bin/hexdump: Permission denied +busybox: /data/data/ga.androidterm/bin/httpd: Permission denied +busybox: /data/data/ga.androidterm/bin/ifconfig: Permission denied +busybox: /data/data/ga.androidterm/bin/ifdown: Permission denied +busybox: /data/data/ga.androidterm/bin/ifup: Permission denied +busybox: /data/data/ga.androidterm/bin/inotifyd: Permission denied +busybox: /data/data/ga.androidterm/bin/install: Permission denied +busybox: /data/data/ga.androidterm/bin/iostat: Permission denied +busybox: /data/data/ga.androidterm/bin/ip: Permission denied +busybox: /data/data/ga.androidterm/bin/ipaddr: Permission denied +busybox: /data/data/ga.androidterm/bin/ipcalc: Permission denied +busybox: /data/data/ga.androidterm/bin/iplink: Permission denied +busybox: /data/data/ga.androidterm/bin/iproute: Permission denied +busybox: /data/data/ga.androidterm/bin/iprule: Permission denied +busybox: /data/data/ga.androidterm/bin/iptunnel: Permission denied +busybox: /data/data/ga.androidterm/bin/klogd: Permission denied +busybox: /data/data/ga.androidterm/bin/ln: Permission denied +busybox: /data/data/ga.androidterm/bin/loadkmap: Permission denied +busybox: /data/data/ga.androidterm/bin/losetup: Permission denied +busybox: /data/data/ga.androidterm/bin/lpd: Permission denied +busybox: /data/data/ga.androidterm/bin/lpq: Permission denied +busybox: /data/data/ga.androidterm/bin/lpr: Permission denied +busybox: /data/data/ga.androidterm/bin/ls: Permission denied +busybox: /data/data/ga.androidterm/bin/lsattr: Permission denied +busybox: /data/data/ga.androidterm/bin/lsof: Permission denied +busybox: /data/data/ga.androidterm/bin/lspci: Permission denied +busybox: /data/data/ga.androidterm/bin/lsusb: Permission denied +busybox: /data/data/ga.androidterm/bin/lzcat: Permission denied +busybox: /data/data/ga.androidterm/bin/lzma: Permission denied +busybox: /data/data/ga.androidterm/bin/lzop: Permission denied +busybox: /data/data/ga.androidterm/bin/lzopcat: Permission denied +busybox: /data/data/ga.androidterm/bin/makedevs: Permission denied +busybox: /data/data/ga.androidterm/bin/makemime: Permission denied +busybox: /data/data/ga.androidterm/bin/man: Permission denied +busybox: /data/data/ga.androidterm/bin/md5sum: Permission denied +busybox: /data/data/ga.androidterm/bin/mkdir: Permission denied +busybox: /data/data/ga.androidterm/bin/mkfifo: Permission denied +busybox: /data/data/ga.androidterm/bin/mknod: Permission denied +busybox: /data/data/ga.androidterm/bin/mkswap: Permission denied +busybox: /data/data/ga.androidterm/bin/mktemp: Permission denied +busybox: /data/data/ga.androidterm/bin/more: Permission denied +busybox: /data/data/ga.androidterm/bin/mpstat: Permission denied +busybox: /data/data/ga.androidterm/bin/mv: Permission denied +busybox: /data/data/ga.androidterm/bin/nbd-client: Permission denied +busybox: /data/data/ga.androidterm/bin/nc: Permission denied +busybox: /data/data/ga.androidterm/bin/netstat: Permission denied +busybox: /data/data/ga.androidterm/bin/nice: Permission denied +busybox: /data/data/ga.androidterm/bin/nmeter: Permission denied +busybox: /data/data/ga.androidterm/bin/nohup: Permission denied +busybox: /data/data/ga.androidterm/bin/od: Permission denied +busybox: /data/data/ga.androidterm/bin/openvt: Permission denied +busybox: /data/data/ga.androidterm/bin/patch: Permission denied +busybox: /data/data/ga.androidterm/bin/pidof: Permission denied +busybox: /data/data/ga.androidterm/bin/pipe_progress: Permission denied +busybox: /data/data/ga.androidterm/bin/pmap: Permission denied +busybox: /data/data/ga.androidterm/bin/popmaildir: Permission denied +busybox: /data/data/ga.androidterm/bin/printenv: Permission denied +busybox: /data/data/ga.androidterm/bin/printf: Permission denied +busybox: /data/data/ga.androidterm/bin/pscan: Permission denied +busybox: /data/data/ga.androidterm/bin/pstree: Permission denied +busybox: /data/data/ga.androidterm/bin/pwd: Permission denied +busybox: /data/data/ga.androidterm/bin/pwdx: Permission denied +busybox: /data/data/ga.androidterm/bin/raidautorun: Permission denied +busybox: /data/data/ga.androidterm/bin/rdev: Permission denied +busybox: /data/data/ga.androidterm/bin/readlink: Permission denied +busybox: /data/data/ga.androidterm/bin/readprofile: Permission denied +busybox: /data/data/ga.androidterm/bin/realpath: Permission denied +busybox: /data/data/ga.androidterm/bin/reformime: Permission denied +busybox: /data/data/ga.androidterm/bin/renice: Permission denied +busybox: /data/data/ga.androidterm/bin/reset: Permission denied +busybox: /data/data/ga.androidterm/bin/resize: Permission denied +busybox: /data/data/ga.androidterm/bin/rev: Permission denied +busybox: /data/data/ga.androidterm/bin/rm: Permission denied +busybox: /data/data/ga.androidterm/bin/rmdir: Permission denied +busybox: /data/data/ga.androidterm/bin/route: Permission denied +busybox: /data/data/ga.androidterm/bin/rpm: Permission denied +busybox: /data/data/ga.androidterm/bin/rpm2cpio: Permission denied +busybox: /data/data/ga.androidterm/bin/rtcwake: Permission denied +busybox: /data/data/ga.androidterm/bin/run-parts: Permission denied +busybox: /data/data/ga.androidterm/bin/runsv: Permission denied +busybox: /data/data/ga.androidterm/bin/runsvdir: Permission denied +busybox: /data/data/ga.androidterm/bin/rx: Permission denied +busybox: /data/data/ga.androidterm/bin/script: Permission denied +busybox: /data/data/ga.androidterm/bin/scriptreplay: Permission denied +busybox: /data/data/ga.androidterm/bin/sed: Permission denied +busybox: /data/data/ga.androidterm/bin/sendmail: Permission denied +busybox: /data/data/ga.androidterm/bin/seq: Permission denied +busybox: /data/data/ga.androidterm/bin/setconsole: Permission denied +busybox: /data/data/ga.androidterm/bin/setkeycodes: Permission denied +busybox: /data/data/ga.androidterm/bin/setlogcons: Permission denied +busybox: /data/data/ga.androidterm/bin/setserial: Permission denied +busybox: /data/data/ga.androidterm/bin/setsid: Permission denied +busybox: /data/data/ga.androidterm/bin/setuidgid: Permission denied +busybox: /data/data/ga.androidterm/bin/sh: Permission denied +busybox: /data/data/ga.androidterm/bin/sha1sum: Permission denied +busybox: /data/data/ga.androidterm/bin/sha256sum: Permission denied +busybox: /data/data/ga.androidterm/bin/sha512sum: Permission denied +busybox: /data/data/ga.androidterm/bin/showkey: Permission denied +busybox: /data/data/ga.androidterm/bin/sleep: Permission denied +busybox: /data/data/ga.androidterm/bin/smemcap: Permission denied +busybox: /data/data/ga.androidterm/bin/softlimit: Permission denied +busybox: /data/data/ga.androidterm/bin/sort: Permission denied +busybox: /data/data/ga.androidterm/bin/split: Permission denied +busybox: /data/data/ga.androidterm/bin/start-stop-daemon: Permission denied +busybox: /data/data/ga.androidterm/bin/strings: Permission denied +busybox: /data/data/ga.androidterm/bin/stty: Permission denied +busybox: /data/data/ga.androidterm/bin/sum: Permission denied +busybox: /data/data/ga.androidterm/bin/sv: Permission denied +busybox: /data/data/ga.androidterm/bin/svlogd: Permission denied +busybox: /data/data/ga.androidterm/bin/sync: Permission denied +busybox: /data/data/ga.androidterm/bin/sysctl: Permission denied +busybox: /data/data/ga.androidterm/bin/tac: Permission denied +busybox: /data/data/ga.androidterm/bin/tail: Permission denied +busybox: /data/data/ga.androidterm/bin/tar: Permission denied +busybox: /data/data/ga.androidterm/bin/tcpsvd: Permission denied +busybox: /data/data/ga.androidterm/bin/tee: Permission denied +busybox: /data/data/ga.androidterm/bin/test: Permission denied +busybox: /data/data/ga.androidterm/bin/time: Permission denied +busybox: /data/data/ga.androidterm/bin/timeout: Permission denied +busybox: /data/data/ga.androidterm/bin/top: Permission denied +busybox: /data/data/ga.androidterm/bin/touch: Permission denied +busybox: /data/data/ga.androidterm/bin/tr: Permission denied +busybox: /data/data/ga.androidterm/bin/true: Permission denied +busybox: /data/data/ga.androidterm/bin/ttysize: Permission denied +busybox: /data/data/ga.androidterm/bin/tunctl: Permission denied +busybox: /data/data/ga.androidterm/bin/tune2fs: Permission denied +busybox: /data/data/ga.androidterm/bin/udhcpc: Permission denied +busybox: /data/data/ga.androidterm/bin/uname: Permission denied +busybox: /data/data/ga.androidterm/bin/uncompress: Permission denied +busybox: /data/data/ga.androidterm/bin/unexpand: Permission denied +busybox: /data/data/ga.androidterm/bin/uniq: Permission denied +busybox: /data/data/ga.androidterm/bin/unix2dos: Permission denied +busybox: /data/data/ga.androidterm/bin/unlzma: Permission denied +busybox: /data/data/ga.androidterm/bin/unlzop: Permission denied +busybox: /data/data/ga.androidterm/bin/unxz: Permission denied +busybox: /data/data/ga.androidterm/bin/unzip: Permission denied +busybox: /data/data/ga.androidterm/bin/uudecode: Permission denied +busybox: /data/data/ga.androidterm/bin/uuencode: Permission denied +busybox: /data/data/ga.androidterm/bin/vi: Permission denied +busybox: /data/data/ga.androidterm/bin/volname: Permission denied +busybox: /data/data/ga.androidterm/bin/watch: Permission denied +busybox: /data/data/ga.androidterm/bin/wc: Permission denied +busybox: /data/data/ga.androidterm/bin/wget: Permission denied +busybox: /data/data/ga.androidterm/bin/which: Permission denied +busybox: /data/data/ga.androidterm/bin/whoami: Permission denied +busybox: /data/data/ga.androidterm/bin/whois: Permission denied +busybox: /data/data/ga.androidterm/bin/xargs: Permission denied +busybox: /data/data/ga.androidterm/bin/xz: Permission denied +busybox: /data/data/ga.androidterm/bin/xzcat: Permission denied +busybox: /data/data/ga.androidterm/bin/yes: Permission denied +busybox: /data/data/ga.androidterm/bin/zcat: Permission denied +tar: can't remove old file ./links/git-shell: Permission denied +cat: can't open '/data/data/ga.androidterm/links/git': Permission denied +rm: can't stat '/data/data/ga.androidterm/links/git': Permission denied +cat: can't open '/data/data/ga.androidterm/links/git-shell': Permission denied +rm: can't stat '/data/data/ga.androidterm/links/git-shell': Permission denied +cat: can't open '/data/data/ga.androidterm/links/git-upload-pack': Permission denied +rm: can't stat '/data/data/ga.androidterm/links/git-upload-pack': Permission denied +lib/lib.runshell.so: line 133: can't create /data/data/ga.androidterm/runshell: Permission denied +lib/lib.runshell.so: line 133: can't create /data/data/ga.androidterm/runshell: Permission denied +chmod: runshell: Operation not permitted +lib/lib.runshell.so: line 133: can't create /data/data/ga.androidterm/bin/trustedkeys.gpg: Permission denied +lib/lib.runshell.so: line 133: can't create /data/data/ga.androidterm/installed-version: Permission denied +Installation complete +tar: write: Broken pipe +shell@hammerhead:/sdcard/git-annex.home $ ^D +shell@hammerhead:/ $ +"""]] + +Android is new to me, so it's possible I'm doing something utterly wrong. + +> Closing as this was a bug in the deprecated Android app. [[done]] --[[Joey]] diff --git a/doc/bugs/Android_4.4_install_fails_with_permission_denied_errors/comment_1_9b0ed646552e56b6b199761b3ff9eb85._comment b/doc/bugs/Android_4.4_install_fails_with_permission_denied_errors/comment_1_9b0ed646552e56b6b199761b3ff9eb85._comment new file mode 100644 index 0000000000..57136884e9 --- /dev/null +++ b/doc/bugs/Android_4.4_install_fails_with_permission_denied_errors/comment_1_9b0ed646552e56b6b199761b3ff9eb85._comment @@ -0,0 +1,15 @@ +[[!comment format=mdwn + username="joey" + subject="""comment 1""" + date="2018-05-08T18:55:27Z" + content=""" +I'm closing this bug report because the git-annex Android app that it +was reported on is now deprecated. Instead, we have +a way to run the regular git-annex build for linux on Android in termux: + + +There were a lot of problems with the way the git-annex Android app was +put together, and I hope this new approach avoids them. If you try it and +still have the bug you reported, please followup and I'll reopen it. + +"""]] diff --git a/doc/bugs/Android_6.0_compatibility.mdwn b/doc/bugs/Android_6.0_compatibility.mdwn new file mode 100644 index 0000000000..c34822ca18 --- /dev/null +++ b/doc/bugs/Android_6.0_compatibility.mdwn @@ -0,0 +1,35 @@ +### Please describe the problem. +Git-annex appears to be incompatible with Android 6. I downloaded the APK for Android 5 and tried installing it on a Nexus 5X running Android 6.0.1. The biggest issue is that pushing and pulling to SSH remotes fails with a "Permission denied" error from muxserver_listen() in OpenSSH. There are some other warnings and errors, listed below, would you like me to file separate bugs for each or track everything here? + +- The full error from ssh is "muxserver_listen: link mux listener /data/data/ga.androidterm/tmp/ssh/annex-user@192.168.0.3.Jg42jDbmIRBdbjDZ => /data/data/ga.androidterm/tmp/ssh/annex-user@192.168.0.3: Permission denied". Immediately after that there is an error message, presumably from git, that reads "fatal: Could not read from remote repository." I am able to SSH to the server from the app's command line, and get an interactive shell. I can successfully run 'git fetch `git remote`', but `git annex sync` runs into this issue. +- When switching between two repositories in the webapp, the page never loads after clicking on a repository. After trying the link a few times, I can see "git-annex: Daemon is already running." through the webapp's log viewer. However, if I stop the daemon through the webapp and start it again, then the webapp will indicate that I successfully switched to the other repository. +- There is a warning message in the console that reads, "WARNING: linker: git-annex has text relocations. This is wasting memory and prevents security hardening. Please fix." It appears that apps targeting SDK version 23 are not allowed to use text relocations, while apps targeting lower versions just get this warning. See http://developer.android.com/about/versions/marshmallow/android-6.0-changes.html#behavior-runtime +- There is a warning message at the top of every new terminal that says "Falling back to hardcoded app location; cannot find expected files in /data/app/ga.androidterm-1/lib". This issue doesn't appear to affect anything, but I thought I'd mention it for completeness. +- At various times during syncing, "\_\_bionic_open_tzdata_path: ANDROID_DATA not set!" and "\_\_bionic_open_tzdata_path: ANDROID_ROOT not set!" show up in the log. + +### What steps will reproduce the problem? +1. Install Android 5.0 APK on Android 6.0 phone +2. Create a repository +3. Add an SSH remote + +### What version of git-annex are you using? On what operating system? +5.20151217-g7b73f34 on Android 6.0.1 + +### Please provide any additional information below. +[[!format sh """ +muxserver_listen: link mux listener /data/data/ga.androidterm/tmp/ssh/annex-user@192.168.0.3.ZUzI2MBl8k0qt2pg => /data/data/ga.androidterm/tmp/ssh/annex-user@192.168.0.3: Permission denied +"""]] + +If I look in this temporary directory after the fact, I see the following. + +[[!format sh """ +> ls -l /data/data/ga.androidterm/tmp/ssh +-rw------- 1 u0_a108 u0_a108 0 Dec 24 10:06 annex-user@192.168.0.3.lock +"""]] + +I suppose since the socket is a Unix domain socket, it gets destroyed when the process stops, so it shouldn't be surprising there's nothing else in that directory. + +### Have you had any luck using git-annex before? (Sometimes we get tired of reading bug reports all day and a lil' positive end note does wonders) +Not yet, but I'm excited to make it work! + +> Closing as this was a bug in the deprecated Android app. [[done]] --[[Joey]] diff --git a/doc/bugs/Android_6.0_compatibility/comment_1_06dcb3d7c5de5a831403e8e6c4c98dfd._comment b/doc/bugs/Android_6.0_compatibility/comment_1_06dcb3d7c5de5a831403e8e6c4c98dfd._comment new file mode 100644 index 0000000000..7fdff1937e --- /dev/null +++ b/doc/bugs/Android_6.0_compatibility/comment_1_06dcb3d7c5de5a831403e8e6c4c98dfd._comment @@ -0,0 +1,8 @@ +[[!comment format=mdwn + username="divergentdave@5c17d06f6d67c6f157b76a4cc95ca764b7d2f899" + nickname="divergentdave" + subject="comment 1" + date="2015-12-24T22:44:16Z" + content=""" +FWIW, it appears that SSH caching is already turned off in this repository. If I cd to the repository and run `git config annex.sshcaching` I get back \"false\". +"""]] diff --git a/doc/bugs/Android_6.0_compatibility/comment_2_c7cabab122174867d7350a1eaa76151c._comment b/doc/bugs/Android_6.0_compatibility/comment_2_c7cabab122174867d7350a1eaa76151c._comment new file mode 100644 index 0000000000..c7c559ca72 --- /dev/null +++ b/doc/bugs/Android_6.0_compatibility/comment_2_c7cabab122174867d7350a1eaa76151c._comment @@ -0,0 +1,12 @@ +[[!comment format=mdwn + username="divergentdave@5c17d06f6d67c6f157b76a4cc95ca764b7d2f899" + nickname="divergentdave" + subject="Logcat output -- SELinux" + date="2015-12-30T03:01:56Z" + content=""" +I ran a sync again and captured output from logcat. It appears that an SELinux policy is preventing ssh from linking its socket file. (at [this line](https://github.com/openssh/openssh-portable/blob/master/mux.c#L1298)) There are several log messages similar to the one below, presumably one for each failed invocation of ssh. + +``` +12-29 20:49:07.308 29990 29990 W ssh : type=1400 audit(0.0:64711): avc: denied { link } for name=\"annex-user@192.168.0.3.cAGquyBx4Z10RTYL\" dev=\"dm-2\" ino=392849 scontext=u:r:untrusted_app:s0:c512,c768 tcontext=u:object_r:app_data_file:s0:c512,c768 tclass=sock_file permissive=0 +``` +"""]] diff --git a/doc/bugs/Android_6.0_compatibility/comment_3_47f9cbde0385a4816ccec1b512dea2fd._comment b/doc/bugs/Android_6.0_compatibility/comment_3_47f9cbde0385a4816ccec1b512dea2fd._comment new file mode 100644 index 0000000000..e54e394ca6 --- /dev/null +++ b/doc/bugs/Android_6.0_compatibility/comment_3_47f9cbde0385a4816ccec1b512dea2fd._comment @@ -0,0 +1,8 @@ +[[!comment format=mdwn + username="divergentdave@5c17d06f6d67c6f157b76a4cc95ca764b7d2f899" + nickname="divergentdave" + subject="SELinux" + date="2015-12-30T03:40:02Z" + content=""" +According to [this issue on Google Code](https://code.google.com/p/android-developer-preview/issues/detail?id=3150), hard linking files is not allowed on Android 6.0. It looks like the only recourse will be to reconfigure or patch OpenSSH, such that it doesn't need to create a hardlink. +"""]] diff --git a/doc/bugs/Android_6.0_compatibility/comment_4_f1f3c98050247cbba4521311465513e2._comment b/doc/bugs/Android_6.0_compatibility/comment_4_f1f3c98050247cbba4521311465513e2._comment new file mode 100644 index 0000000000..abf7a78c1a --- /dev/null +++ b/doc/bugs/Android_6.0_compatibility/comment_4_f1f3c98050247cbba4521311465513e2._comment @@ -0,0 +1,12 @@ +[[!comment format=mdwn + username="joey" + subject="""comment 4""" + date="2016-01-20T19:03:37Z" + content=""" +Unfortunately, I lack time, energy, or devices to do much work on the +Android port. Patches would be welcome for any or all of these issues. + +(Although the relocations warning seems completely ignorable.) + +(Wow, they broke hard links? What core unix feature is next to go? Sheesh.) +"""]] diff --git a/doc/bugs/Android_6.0_compatibility/comment_5_0bcee221dd22b424f08720604a486c79._comment b/doc/bugs/Android_6.0_compatibility/comment_5_0bcee221dd22b424f08720604a486c79._comment new file mode 100644 index 0000000000..7ed65ea61d --- /dev/null +++ b/doc/bugs/Android_6.0_compatibility/comment_5_0bcee221dd22b424f08720604a486c79._comment @@ -0,0 +1,29 @@ +[[!comment format=mdwn + username="divergentdave@5c17d06f6d67c6f157b76a4cc95ca764b7d2f899" + nickname="divergentdave" + subject="Disabling SSH connection caching" + date="2016-02-01T06:52:55Z" + content=""" +I was looking into how SSH connection caching is handled, and it seems there's a bug in the sshCacheDir function in Annex/Ssh.hs. If I'm reading things right, `annex.sshcaching` is ignored when `annex.crippledfilesystem` is set. Thus, even though the configuration says SSH caching is disabled, this function still creates and passes out a temporary directory to be used for connection caching. Further up, this results in ControlPersist, ControlMaster, etc. being passed to OpenSSH, which runs into the SELinux rules. I think the `ifM` calls should be nested the other way around, (see below) which would allow me to turn off connection caching and hopefully get SSH working. I haven't tested this yet, though I plan to get a build environment set up within a month for further tinkering. + +Revised sshCacheDir: + +``` +sshCacheDir :: Annex (Maybe FilePath) +sshCacheDir + | SysConfig.sshconnectioncaching = ifM + ( fromMaybe True . annexSshCaching <$> Annex.getGitConfig ) + ( ifM crippledFileSystem + ( maybe (return Nothing) usetmpdir =<< gettmpdir + , Just <$> fromRepo gitAnnexSshDir ) + , return Nothing + ) + | otherwise = return Nothing + where + gettmpdir = liftIO $ getEnv \"GIT_ANNEX_TMP_DIR\" + usetmpdir tmpdir = liftIO $ catchMaybeIO $ do + let socktmp = tmpdir \"ssh\" + createDirectoryIfMissing True socktmp + return socktmp +``` +"""]] diff --git a/doc/bugs/Android_6.0_compatibility/comment_6_a4ef9eb27285aa68d6edc14e30627ebf._comment b/doc/bugs/Android_6.0_compatibility/comment_6_a4ef9eb27285aa68d6edc14e30627ebf._comment new file mode 100644 index 0000000000..fb980779b0 --- /dev/null +++ b/doc/bugs/Android_6.0_compatibility/comment_6_a4ef9eb27285aa68d6edc14e30627ebf._comment @@ -0,0 +1,10 @@ +[[!comment format=mdwn + username="joey" + subject="""comment 6""" + date="2016-04-20T18:40:57Z" + content=""" +Thanks, divergentdave for spotting that and also for writing a fix. +I've finally noticed your comment and put the fix in. It would probably be +good to open a todo if you have a patch in the future, to make sure it +doesn't get forgotten about. +"""]] diff --git a/doc/bugs/Android_6.0_compatibility/comment_7_a7110f92e8a90e95f494e3f917b610ea._comment b/doc/bugs/Android_6.0_compatibility/comment_7_a7110f92e8a90e95f494e3f917b610ea._comment new file mode 100644 index 0000000000..30b7df2821 --- /dev/null +++ b/doc/bugs/Android_6.0_compatibility/comment_7_a7110f92e8a90e95f494e3f917b610ea._comment @@ -0,0 +1,15 @@ +[[!comment format=mdwn + username="joey" + subject="""comment 7""" + date="2018-05-08T18:51:44Z" + content=""" +I'm closing this bug report because the git-annex Android app that it +was reported on is now deprecated. Instead, we have +a way to run the regular git-annex build for linux on Android in termux: + + +There were a lot of problems with the way the git-annex Android app was +put together, and I hope this new approach avoids them. If you try it and +still have the bug you reported, please followup and I'll reopen it. + +"""]] diff --git a/doc/bugs/Android_CM_5.1_ANDROID__95__ROOT_not_set.mdwn b/doc/bugs/Android_CM_5.1_ANDROID__95__ROOT_not_set.mdwn new file mode 100644 index 0000000000..10abf048db --- /dev/null +++ b/doc/bugs/Android_CM_5.1_ANDROID__95__ROOT_not_set.mdwn @@ -0,0 +1,41 @@ +### Please describe the problem. + +Downloaded git-annex (version for android 5.0) from the website. Upon opening the app for the first time and setting up a repository at /sdcard/annex, the terminal gave out the error. + +### What version of git-annex are you using? On what operating system? + +Nexus 4 running Cyanogenmod 12.1 (Android 5.1) build 20151007 and 20150901 + +### Please provide any additional information below. +Output to the terminal window: +[[!format sh """ +WARNING: linker: git-annex has text relocations. This is wating memory and prevents security hardening. Please fix. +__bionic_open_tzdata_path: ANDROID_ROOT not set! +__bionic_open_tzdata_path: ANDROID_ROOT not set! +__bionic_open_tzdata_path: ANDROID_ROOT not set! +__bionic_open_tzdata_path: ANDROID_ROOT not set! +__bionic_open_tzdata_path: ANDROID_ROOT not set! + +Detected a filesystem without fifo support. + +Disabling ssh connection caching. + +Detected a crippled filesystem. + +Enabling direct mode. +fatal: ../../../../sdcard/annex: '../../../../sdcard/annex' is outside of repository +(recording state in git...) +__bionic_open_tzdata_path: ANDROID_ROOT not set! +__bionic_open_tzdata_path: ANDROID_ROOT not set! +__bionic_open_tzdata_path: ANDROID_ROOT not set! +__bionic_open_tzdata_path: ANDROID_ROOT not set! +__bionic_open_tzdata_path: ANDROID_ROOT not set! +"""]] +I can add remote repositories and they show up nicely in the webapp, but no files are ever downloaded. + +### Have you had any luck using git-annex before? (Sometimes we get tired of reading bug reports all day and a lil' positive end note does wonders) + +I have a couple of repositories atm, one with my music, another that backs up our family pictures for the family and uses Amazon S3 as a backup. + +> Closing as this was a bug in the deprecated Android app. [[done]] --[[Joey]] + diff --git a/doc/bugs/Android_CM_5.1_ANDROID__95__ROOT_not_set/comment_1_5f480528fe855fcbdbf5b6950aa42a25._comment b/doc/bugs/Android_CM_5.1_ANDROID__95__ROOT_not_set/comment_1_5f480528fe855fcbdbf5b6950aa42a25._comment new file mode 100644 index 0000000000..c246116504 --- /dev/null +++ b/doc/bugs/Android_CM_5.1_ANDROID__95__ROOT_not_set/comment_1_5f480528fe855fcbdbf5b6950aa42a25._comment @@ -0,0 +1,14 @@ +[[!comment format=mdwn + username="joey" + subject="""comment 1""" + date="2018-05-08T18:52:02Z" + content=""" +I'm closing this bug report because the git-annex Android app that it +was reported on is now deprecated. Instead, we have +a way to run the regular git-annex build for linux on Android in termux: + + +There were a lot of problems with the way the git-annex Android app was +put together, and I hope this new approach avoids them. If you try it and +still have the bug you reported, please followup and I'll reopen it. +"""]] diff --git a/doc/bugs/Android__58___Cannot_create_repo_on_external_sd_card.mdwn b/doc/bugs/Android__58___Cannot_create_repo_on_external_sd_card.mdwn new file mode 100644 index 0000000000..ead5aa9d81 --- /dev/null +++ b/doc/bugs/Android__58___Cannot_create_repo_on_external_sd_card.mdwn @@ -0,0 +1,64 @@ +I am using the latest daily build for Android 5.0 + +My version is Android 5.0.1 Lollipop. And I'm using a Samsung Galaxy S4 unrooted. + +Trying to create a repositpory in the folder /storage/extSdCard/Music gives me a webpage with a red error badge that says: + + "Internal Server Error" + + git init failed + + Output: + /storage/extSdCard/Music/.git: Permission denied + +I'm pretty sure this is because of Android's crappy permission system on sd cards. But when I install the app, it tells me it is asking for +read write access to the sd card. So this consideration must have happened. + + + +### Please describe the problem. + +I am using the latest daily build for Android 5.0 + +My version is Android 5.0.1 Lollipop. And I'm using a Samsung Galaxy S4 unrooted. + +Trying to create a repositpory in the folder /storage/extSdCard/Music gives me a webpage with a red error badge that says: + + "Internal Server Error" + + git init failed + + Output: + /storage/extSdCard/Music/.git: Permission denied + +I'm pretty sure this is because of Android's crappy permission system on sd cards. But when I install the app, it tells me it is asking for +read write access to the sd card. So this consideration must have happened. + +### What steps will reproduce the problem? + +1. Install the Android 5.0 daily build on an Android 5.0.1 phone. +2. Try to create a repo on an external sd card. + +### What version of git-annex are you using? On what operating system? + +Latest daily build of Android 5.0 git annex on Android 5.0.1 + +### Please provide any additional information below. + +When I run adb shell I see the following permissions on the file in question: + + drwxrwx--- root sdcard_r 2016-04-02 14:11 Music + +Additional info. The terminal emulator shows this output: + + Falling back to hardcoded app location: cannot find expected files in /data/app/ga.androidterm-2/lib + git annex webapp + WARNING: linker: git-annex has text relocations. This is wasting memory and prevents security hardening. Please fix. + +I have tried moving the app to the sd card, but it will not work if I do that. + +### Have you had any luck using git-annex before? (Sometimes we get tired of reading bug reports all day and a lil' positive end note does wonders) + +Yes! It's by far one of my favorite apps! it works very well on my laptop, on my home file server, and on my internal storage on my Android phone :) + +> Closing as this was a bug in the deprecated Android app. [[done]] --[[Joey]] diff --git a/doc/bugs/Android__58___Cannot_create_repo_on_external_sd_card/comment_1_7ac9adea8bf562d590133e83cbb12570._comment b/doc/bugs/Android__58___Cannot_create_repo_on_external_sd_card/comment_1_7ac9adea8bf562d590133e83cbb12570._comment new file mode 100644 index 0000000000..5d0e23956c --- /dev/null +++ b/doc/bugs/Android__58___Cannot_create_repo_on_external_sd_card/comment_1_7ac9adea8bf562d590133e83cbb12570._comment @@ -0,0 +1,14 @@ +[[!comment format=mdwn + username="joey" + subject="""comment 1""" + date="2018-05-08T18:49:14Z" + content=""" +I'm closing this bug report because the git-annex Android app that it +was reported on is now deprecated. Instead, we have +a way to run the regular git-annex build for linux on Android in termux: + + +There were a lot of problems with the way the git-annex Android app was +put together, and I hope this new approach avoids them. If you try it and +still have the bug you reported, please followup and I'll reopen it. +"""]] diff --git a/doc/bugs/Android__58___Several_error_messages___47___warnings.mdwn b/doc/bugs/Android__58___Several_error_messages___47___warnings.mdwn new file mode 100644 index 0000000000..c6085c21ab --- /dev/null +++ b/doc/bugs/Android__58___Several_error_messages___47___warnings.mdwn @@ -0,0 +1,38 @@ +### Please describe the problem. + +When I start Git Annex then there are several warnings. The app renders my phone basically unusable (two time my phone rebooted and several time I had to remove the battery because the phone didn't respond). + +It seems Git Annex uses all the available memory (one warning is that git-annex has text relocations and that this wastes memory. + +I've attached a screen shot of the error messages. + + +### What steps will reproduce the problem? + +Starting Git Annex on my phone + + +### What version of git-annex are you using? On what operating system? + +Git Annex: 5.20150927-gb506fb2 +Android: 5.0.2 + + +### Please provide any additional information below. + +My phone is low-end (it has only ~512MB memory for example) - but normally everything works more or less okay. + +If I use Chrome as the browser and start the webapp then Git Annex gets (it seems) killed quite often. It seems because there is not enough memory. + +If I use a browser with a small memory footprint then the webapp works only barely better - very unresponsive (but at least Git Annex isn't killed). + + +### Have you had any luck using git-annex before? (Sometimes we get tired of reading bug reports all day and a lil' positive end note does wonders) + +Sorry, no, I'm just starting out with Git Annex :) + +But Git Annex looks amazing - exactly what I need! ;) + +Can I somehow help to find the problem? + +I'd really love to use Git Annex on my phone. diff --git a/doc/bugs/Android__58___Several_error_messages___47___warnings/comment_1_85da61d3dfb528fef17366effe123ff5._comment b/doc/bugs/Android__58___Several_error_messages___47___warnings/comment_1_85da61d3dfb528fef17366effe123ff5._comment new file mode 100644 index 0000000000..92a19534a2 --- /dev/null +++ b/doc/bugs/Android__58___Several_error_messages___47___warnings/comment_1_85da61d3dfb528fef17366effe123ff5._comment @@ -0,0 +1,16 @@ +[[!comment format=mdwn + username="joey" + subject="""comment 1""" + date="2015-09-29T15:27:20Z" + content=""" +I don't think that the warning about text relocations is a significant +source of memory usage. + +It would probably help if you can get some memory use value for git-annex +while it's running, and for chrome after it's loaded the git-annex webapp. +These are two separate uses of memory, and it's not clear which one is +actually using up too much of your memory. + +I've run git-annex on android systems that are smaller than 512 mb, and +generally the assistant only uses a few MB of memory on other platforms. +"""]] diff --git a/doc/bugs/Android__58___Several_error_messages___47___warnings/comment_2_97ddaba23e91a903f69cf00931627444._comment b/doc/bugs/Android__58___Several_error_messages___47___warnings/comment_2_97ddaba23e91a903f69cf00931627444._comment new file mode 100644 index 0000000000..22aacf3062 --- /dev/null +++ b/doc/bugs/Android__58___Several_error_messages___47___warnings/comment_2_97ddaba23e91a903f69cf00931627444._comment @@ -0,0 +1,21 @@ +[[!comment format=mdwn + username="dah" + subject="comment 2" + date="2015-10-04T18:25:29Z" + content=""" +I think the CPU of my phone is not good enough to be usable with a web browser on my phone. + +I can use Git Annex without the WebApp without phone stability problems. But if I start the WebApp then nothing works (it seems 100% of my CPU is used - but it could also be that the CPU monitor app did freeze). + +Websites like Google and Wikipedia work without problems but I had another website that made my phone unusable and forced me to restart my phone. + +I think therefore my phone is the problem and not Git Annex. + +So, the best way for me to use Git Annex on my phone is probably the cammand line interface. + +Besides the text relocations warning there was another message that said \"Falling back to hardcoded app location: cannot find expected files in $path\". + +Is this a problem or does everything work properly? + + +"""]] diff --git a/doc/bugs/Android__58___Several_error_messages___47___warnings/comment_3_b4ea7ff7deb9626a02dfd0eb459f22a2._comment b/doc/bugs/Android__58___Several_error_messages___47___warnings/comment_3_b4ea7ff7deb9626a02dfd0eb459f22a2._comment new file mode 100644 index 0000000000..f63f93d147 --- /dev/null +++ b/doc/bugs/Android__58___Several_error_messages___47___warnings/comment_3_b4ea7ff7deb9626a02dfd0eb459f22a2._comment @@ -0,0 +1,12 @@ +[[!comment format=mdwn + username="joey" + subject="""comment 3""" + date="2015-10-04T19:27:18Z" + content=""" +Note that you can close the web browser and the git-annex assistant can +still be running, and doing its thing. This might be a viable alternative +to using the command line manually for you. + +The message about hardcoded app location isn't a problem if git-annex +works. +"""]] diff --git a/doc/bugs/Android__58___Several_error_messages___47___warnings/comment_4_5edddd7e90b8f11c52a44abfb891ce25._comment b/doc/bugs/Android__58___Several_error_messages___47___warnings/comment_4_5edddd7e90b8f11c52a44abfb891ce25._comment new file mode 100644 index 0000000000..976ead6892 --- /dev/null +++ b/doc/bugs/Android__58___Several_error_messages___47___warnings/comment_4_5edddd7e90b8f11c52a44abfb891ce25._comment @@ -0,0 +1,20 @@ +[[!comment format=mdwn + username="dah" + subject="comment 4" + date="2015-10-10T19:02:56Z" + content=""" + +But how would I configure git-annex assistant in this case? + +My plan was to somehow add the information about the repos etc. using the command line interface and then just start the assistant using 'git annex assistant --autostart' as the terminal startup command. + +Would that work? + +By the way, is there any documentation about how to use the assistant without the web interface? + +I could only find information about how to configure the assistant using the web app. + +Thanks for your help! + + +"""]] diff --git a/doc/bugs/Android__58___Several_error_messages___47___warnings/comment_4_b38b675a2862d527fad28b7477f7ddde._comment b/doc/bugs/Android__58___Several_error_messages___47___warnings/comment_4_b38b675a2862d527fad28b7477f7ddde._comment new file mode 100644 index 0000000000..81e0c0fa0c --- /dev/null +++ b/doc/bugs/Android__58___Several_error_messages___47___warnings/comment_4_b38b675a2862d527fad28b7477f7ddde._comment @@ -0,0 +1,8 @@ +[[!comment format=mdwn + username="joey" + subject="""comment 4""" + date="2015-10-12T17:15:30Z" + content=""" +The assistant will use whatever git remotes and git-annex configuration is +present in the repository when it's started up. +"""]] diff --git a/doc/bugs/Android__58___use___47__data__47____8230___for_git-annex.home__63__.mdwn b/doc/bugs/Android__58___use___47__data__47____8230___for_git-annex.home__63__.mdwn new file mode 100644 index 0000000000..c34ea43c3d --- /dev/null +++ b/doc/bugs/Android__58___use___47__data__47____8230___for_git-annex.home__63__.mdwn @@ -0,0 +1,8 @@ +### Please describe the problem. +I’ve only just set up git-annex on my Android system, so I might be confused; but from what I could find on this site, the Android build only supports passwordless SSH key files, which must be deployed in /sdcard/git-annex.home/.ssh (I also found something about git-annex generating keys by itself, but the web app didn’t refer to something like that, so I take it this feature is gone now). What perplexed me about this is that /sdcard on Android is a VFAT file-system, and is set up so that any user or application may read it (with a permissions mask for the sdcard group), which isn’t ideal for a passwordless key. From what I could gather, there’s also a place in the virtual file-system under /data that is intended to house private, per-application data. Should git-annex use this location as its home instead? + +### What version of git-annex are you using? On what operating system? +git-annex 5.20150522-g260b59e +Android 4.4.4 + +> Closing as this was a bug in the deprecated Android app. [[done]] --[[Joey]] diff --git a/doc/bugs/Android__58___use___47__data__47____8230___for_git-annex.home__63__/comment_1_95481b150f43fd3db311fbcf6d1e48aa._comment b/doc/bugs/Android__58___use___47__data__47____8230___for_git-annex.home__63__/comment_1_95481b150f43fd3db311fbcf6d1e48aa._comment new file mode 100644 index 0000000000..cdfb498336 --- /dev/null +++ b/doc/bugs/Android__58___use___47__data__47____8230___for_git-annex.home__63__/comment_1_95481b150f43fd3db311fbcf6d1e48aa._comment @@ -0,0 +1,8 @@ +[[!comment format=mdwn + username="joey" + subject="""comment 1""" + date="2015-07-06T18:23:57Z" + content=""" +I think that /data is locked down such that apps generally cannot write to +it, but please correct me if I'm wrong. +"""]] diff --git a/doc/bugs/Android__58___use___47__data__47____8230___for_git-annex.home__63__/comment_2_041fe2a545b48acd2c9d15eb2ac2b34f._comment b/doc/bugs/Android__58___use___47__data__47____8230___for_git-annex.home__63__/comment_2_041fe2a545b48acd2c9d15eb2ac2b34f._comment new file mode 100644 index 0000000000..486d1843ff --- /dev/null +++ b/doc/bugs/Android__58___use___47__data__47____8230___for_git-annex.home__63__/comment_2_041fe2a545b48acd2c9d15eb2ac2b34f._comment @@ -0,0 +1,8 @@ +[[!comment format=mdwn + username="divergentdave@5c17d06f6d67c6f157b76a4cc95ca764b7d2f899" + nickname="divergentdave" + subject="comment 2" + date="2016-03-21T04:13:15Z" + content=""" +Each app has one subfolder inside /data that is private to that app (and user) alone. Generally, you can't read or enumerate /data itself. There is a function in the Java API to get the current app's internal storage folder, see https://developer.android.com/reference/android/content/Context.html#getFilesDir%28%29. +"""]] diff --git a/doc/bugs/Android__58___use___47__data__47____8230___for_git-annex.home__63__/comment_3_c46091ade13865addb83c6e77f97f195._comment b/doc/bugs/Android__58___use___47__data__47____8230___for_git-annex.home__63__/comment_3_c46091ade13865addb83c6e77f97f195._comment new file mode 100644 index 0000000000..4682b3ae58 --- /dev/null +++ b/doc/bugs/Android__58___use___47__data__47____8230___for_git-annex.home__63__/comment_3_c46091ade13865addb83c6e77f97f195._comment @@ -0,0 +1,14 @@ +[[!comment format=mdwn + username="joey" + subject="""comment 3""" + date="2018-05-08T18:50:00Z" + content=""" +I'm closing this bug report because the git-annex Android app that it +was reported on is now deprecated. Instead, we have +a way to run the regular git-annex build for linux on Android in termux: + + +There were a lot of problems with the way the git-annex Android app was +put together, and I hope this new approach avoids them. If you try it and +still have the bug you reported, please followup and I'll reopen it. +"""]] diff --git a/doc/bugs/Android___58___handling_DCIM__47__Camera_not_being_configurable.mdwn b/doc/bugs/Android___58___handling_DCIM__47__Camera_not_being_configurable.mdwn new file mode 100644 index 0000000000..a52857d1f1 --- /dev/null +++ b/doc/bugs/Android___58___handling_DCIM__47__Camera_not_being_configurable.mdwn @@ -0,0 +1,16 @@ +### Please describe the problem. + +In order to handle the fact that the directory where pictures are saved is not configurable on my phone, I set up a second git annex repository with the Repository group "file source". + +### What version of git-annex are you using? On what operating system? + +5.20140108-gce9652 + +### Please provide any additional information below. + +In the log, there are many "too many open files" errors like these : + +git:createProcess: runInteractiveProcess: pipe: resource exhausted (Too many open files) + +[[!tag moreinfo]] +[[!meta title="too many open files on android"]] diff --git a/doc/bugs/Android___58___handling_DCIM__47__Camera_not_being_configurable/comment_1_1fe5f8c68a430b2436649cf4ba8f4987._comment b/doc/bugs/Android___58___handling_DCIM__47__Camera_not_being_configurable/comment_1_1fe5f8c68a430b2436649cf4ba8f4987._comment new file mode 100644 index 0000000000..eea1f2094d --- /dev/null +++ b/doc/bugs/Android___58___handling_DCIM__47__Camera_not_being_configurable/comment_1_1fe5f8c68a430b2436649cf4ba8f4987._comment @@ -0,0 +1,10 @@ +[[!comment format=mdwn + username="http://joeyh.name/" + ip="71.80.94.56" + subject="comment 1" + date="2014-02-07T18:59:29Z" + content=""" +I guess that this bug report is not about the DCIM/Camera location not being configurable, since that is up to Android, not git-annex. + +So, it seems to be a bug report about a file descriptor leak. It would be helpful, if you can reproduce the leak, to look at /proc/$pid/fd/ to see what files git-annex has open. +"""]] diff --git a/doc/bugs/Android_can__39__t_sync_files_with_utf-8_characters_in_names.mdwn b/doc/bugs/Android_can__39__t_sync_files_with_utf-8_characters_in_names.mdwn new file mode 100644 index 0000000000..b8f45e4d17 --- /dev/null +++ b/doc/bugs/Android_can__39__t_sync_files_with_utf-8_characters_in_names.mdwn @@ -0,0 +1,36 @@ +### Please describe the problem. +I have a repo where some files have utf-8 characters in their names. They are copied without problems between linux-running pcs, but on Android I only get a small, 206 bytes files with the hash data. + +### What steps will reproduce the problem? +Add some files with utf-8 characters in names, try to sync to Android + +### What version of git-annex are you using? On what operating system? +PC side: tried 5.20140412ubuntu1 from ubuntu repo and standalone tarball 5.20140517-gee56d21, both on Ubuntu Trusy +Android side: Android 4.4.2 on Nexus 7 + +### Please provide any additional information below. +Part of daemon.log I believe is related to this. Lines starting with Number followed by dot are the filenames, but truncated at first non-ascii character. +I suspect empty lines before the "rsync failed" represent files with names that start with non-ascii character. + +[[!format sh """ +# If you can, paste a complete transcript of the problem occurring here. +# If the problem is with the git-annex assistant, paste in .git/annex/daemon.log + + + rsync failed -- run git annex again to resume file transfer + + + rsync failed -- run git annex again to resume file transfer + + + rsync failed -- run git annex again to resume file transfer + +02. Wyspa W + rsync failed -- run git annex again to resume file transfer + +05. Ponad Krain + rsync failed -- run git annex again to resume file transfer + + +# End of transcript or log. +"""]] diff --git a/doc/bugs/Android_can__39__t_sync_files_with_utf-8_characters_in_names/comment_1_bcc3ce19cf26731057a7f3189fcbae19._comment b/doc/bugs/Android_can__39__t_sync_files_with_utf-8_characters_in_names/comment_1_bcc3ce19cf26731057a7f3189fcbae19._comment new file mode 100644 index 0000000000..130e4a420a --- /dev/null +++ b/doc/bugs/Android_can__39__t_sync_files_with_utf-8_characters_in_names/comment_1_bcc3ce19cf26731057a7f3189fcbae19._comment @@ -0,0 +1,10 @@ +[[!comment format=mdwn + username="https://www.google.com/accounts/o8/id?id=AItOawniXLhO9mLn-7EDfawdENZ2fQwlGy5w_oc" + nickname="Michał" + subject="Update - 20140529" + date="2014-06-01T01:36:16Z" + content=""" +Tried with the new 20140529 on Android, same behavior. + +One more note: I managed to spot the attempted update in the webapp. The filename that shows up there has all the non-ascii characters replaced by ��� (not truncated as in logs), ex: old/������������.pdf +"""]] diff --git a/doc/bugs/Android_cannot_setup_gitlab_repo_due_to_ECDSA_key.mdwn b/doc/bugs/Android_cannot_setup_gitlab_repo_due_to_ECDSA_key.mdwn new file mode 100644 index 0000000000..68a2d1a483 --- /dev/null +++ b/doc/bugs/Android_cannot_setup_gitlab_repo_due_to_ECDSA_key.mdwn @@ -0,0 +1,42 @@ +### Please describe the problem. +Cannot add gitlab repository on Android. + +### What steps will reproduce the problem? +After attempting to add a gitlab repository on git-annex assistant, I am presented with a ssh key and prompted to add it to my gitlab account. After doing so and continuing, git-annex assistant returns to the same page and presents the ssh key again. Attempting to continue repeats the cycle as the ssh key is presented over and over. + +### What version of git-annex are you using? On what operating system? +Stable version for Lollipop (I experienced this same error on Kit Kat ~6 months ago but could not resolve it at that point) +Android 5.0 git-annex.apk + +### Please provide any additional information below. +In the git-annex assistant log I see: + + No ECDSA host key is known for gitlab.com and you have requested strict checking. + Host key verification failed. + No ECDSA host key is known for gitlab.com and you have requested strict checking. + Host key verification failed. + fatal: Could not read from remote repository + + Please make sure you have the correct access rights and the repository exists. + Host key verification failed. + Host key verification failed. + fatal: Could not read from remote repository. + +I resolved the issue by opening a new terminal window, attempting to connect to gitlab (ssh git@gitlab.com), verifying the ECDSA key against the gitlab website (https://about.gitlab.com/gitlab-com/) and accepting the key. The file .ssh/known_hosts was created. The gitlab repository could then be added in git annex assistant. + +The lack of information about the error presented to the user in git annex assistant was part of the problem. + + +[[!format sh """ +# If you can, paste a complete transcript of the problem occurring here. +# If the problem is with the git-annex assistant, paste in .git/annex/daemon.log + + +# End of transcript or log. +"""]] + +### Have you had any luck using git-annex before? (Sometimes we get tired of reading bug reports all day and a lil' positive end note does wonders) + + +> Closing as this was a bug in the deprecated Android app. [[done]] --[[Joey]] + diff --git a/doc/bugs/Android_cannot_setup_gitlab_repo_due_to_ECDSA_key/comment_1_d46469579e96df51ad045340f974f268._comment b/doc/bugs/Android_cannot_setup_gitlab_repo_due_to_ECDSA_key/comment_1_d46469579e96df51ad045340f974f268._comment new file mode 100644 index 0000000000..d4d1fa1ff8 --- /dev/null +++ b/doc/bugs/Android_cannot_setup_gitlab_repo_due_to_ECDSA_key/comment_1_d46469579e96df51ad045340f974f268._comment @@ -0,0 +1,14 @@ +[[!comment format=mdwn + username="joey" + subject="""comment 1""" + date="2018-05-08T18:50:33Z" + content=""" +I'm closing this bug report because the git-annex Android app that it +was reported on is now deprecated. Instead, we have +a way to run the regular git-annex build for linux on Android in termux: + + +There were a lot of problems with the way the git-annex Android app was +put together, and I hope this new approach avoids them. If you try it and +still have the bug you reported, please followup and I'll reopen it. +"""]] diff --git a/doc/bugs/Android_client_deletes_everything.mdwn b/doc/bugs/Android_client_deletes_everything.mdwn new file mode 100644 index 0000000000..677bdaf517 --- /dev/null +++ b/doc/bugs/Android_client_deletes_everything.mdwn @@ -0,0 +1,45 @@ +### Please describe the problem. + +After not syncing my Android repo for a while, I tried to sync it. By some combination of starting up the assistant, killing it, running `git annex sync --content`, and killing *it*, I managed to get the Android client (operating in direct mode) to decide that I had manually deleted a whole bunch of files that had in fact just not been created yet, and to create a commit to that effect, which it promptly synced out to my other repos. + +I then synced in my main direct mode crippled filesystem repo with the only copy of some files, and got a bunch of messages that Git Annex was deleting files I wanted to keep. I killed that sync with a ctrl+c. + +My problem (***UPDATE**: solved) is: how do I revert the offending commit and restore my files? + +My other question (not yet solved) is: how do I prevent this happening again? Is there a way I can pre-clear commits and not accept those that delete files without manual confirmation? Or should I just stop being mean to the Android client and hope it doesn't decide to delete things it shouldn't delete again? + +I've tried a `git annex proxy -- git revert HASHOFBADCOMMIT`, but (as I killed Git Annex before it got through recording that it had trashed my files), I just get: + +``` +error: Your local changes would be overwritten by revert. +hint: Commit your changes or stash them to proceed. +fatal: revert failed +``` + +When syncing in a direct mode repo, does Git Annex happily delete the last copy of a file that appears to have been deleted somewhere else? Or does it save it until you manually clean up unused data, by moving it somewhere under .git? + +**UPDATE**: A `git annex sync`, of all things, in the direct mode repository seems to have brought the files and their contents back. It created a commit that undid the deleting commit, except for the deletion of a duplicate copy of one file, which I don't need to have. + +### What steps will reproduce the problem? + +It's not entirely clear. Some combination of interrupting and restarting the Android app can make it think that files have been deleted when they really have never been created. + +Alternately, to get the problem I really want to fix now, delete a file in one repo, sync the delete to a direct-mode repo with the only copy. Then somehow undo the delete from the direct mode repo and restore the content of the file. + +### What version of git-annex are you using? On what operating system? + +The PC has 6.20160903-g1c0b2b4 and Android has 6.20160714. + +### Please provide any additional information below. + +[[!format sh """ +# If you can, paste a complete transcript of the problem occurring here. +# If the problem is with the git-annex assistant, paste in .git/annex/daemon.log + + +# End of transcript or log. +"""]] + +### Have you had any luck using git-annex before? (Sometimes we get tired of reading bug reports all day and a lil' positive end note does wonders) + + diff --git a/doc/bugs/Android_version_converts_plain_git_file___40__git_add__41___to_git-annex_file.mdwn b/doc/bugs/Android_version_converts_plain_git_file___40__git_add__41___to_git-annex_file.mdwn new file mode 100644 index 0000000000..aac64a4d5a --- /dev/null +++ b/doc/bugs/Android_version_converts_plain_git_file___40__git_add__41___to_git-annex_file.mdwn @@ -0,0 +1,57 @@ +### Please describe the problem. + +I have a README file in my repository, which is an ordinary text file added with `git add` (not `git annex add`). This seems fine with all the Linux machines, including ones running the assistant. However, when I start the assistant on Android, it converts it to an annexed file. I don't have any other direct mode repositories to check if it's a direct mode problem or an Android problem. + +My setup is basically a star tolopogy with a Debian GNU/Linux (jessie) box in the middle. All the clients are Debian as well, mostly testing, except for an Android tablet. + +### What steps will reproduce the problem? + +On a Linux box: + + git add README + git commit -m 'README' + git annex sync --content + +then on Android, start git-annex and let the assistant sync. + +You'll get a commit like this: + + $ git show 4f1c76374c75a11702c14ea6a5dbe82c99c6dd08 + commit 4f1c76374c75a11702c14ea6a5dbe82c99c6dd08 + Author: android + Date: Wed Dec 9 15:49:01 2015 -0500 + + git-annex in Smoot /sdcard/Westerley-Board + + diff --git a/Contracts/Archive/README b/Contracts/Archive/README + deleted file mode 100644 + index 8fe1349..0000000 + --- a/Contracts/Archive/README + +++ /dev/null + @@ -1,3 +0,0 @@ + -These are old, no longer active contracts. The year is the year of + -archival (typically when the contract ended, or the last year covered by + -the contract). + diff --git a/Contracts/Archive/README b/Contracts/Archive/README + new file mode 120000 + index 0000000..38ba43f + --- /dev/null + +++ b/Contracts/Archive/README + @@ -0,0 +1 @@ + +../../.git/annex/objects/0v/9K/SHA256E-s155--d0e49ec7e493366a5afea5bc12629ba579fd8407162795c22a6346c25bafbb6e/SHA256E-s155--d0e49ec7e493366a5afea5bc12629ba579fd8407162795c22a6346c25bafbb6e + \ No newline at end of file + +### What version of git-annex are you using? On what operating system? + +Android is fairly recent, unfortunately the battery is currently dead +making it hard to check :-( Must be at least 5.20151019, but probably +20151116. + +### Please provide any additional information below. + +If needed, I'll grab the assistant log from the tablet once the battery +is charged. + +### Have you had any luck using git-annex before? (Sometimes we get tired of reading bug reports all day and a lil' positive end note does wonders) + +Yes! I've been using git-annex quite a bit over the past year, for everything from my music collection to my personal files. Using it for a not-for-profit too. Even trying to get some Mac and Windows users to use it for our HOA's files. I'm looking forward to smudge mode to make direct mode work better. diff --git a/doc/bugs/Assertion___96__cnt___60_____40__sizeof___40____95__nl__95__value__95__type__95__LC__95__TIME__41_____47___sizeof___40____95__nl__95__value__95__type__95__LC__95__TIME__91__0__93____41____41____39___failed..mdwn b/doc/bugs/Assertion___96__cnt___60_____40__sizeof___40____95__nl__95__value__95__type__95__LC__95__TIME__41_____47___sizeof___40____95__nl__95__value__95__type__95__LC__95__TIME__91__0__93____41____41____39___failed..mdwn new file mode 100644 index 0000000000..fd9f32e08e --- /dev/null +++ b/doc/bugs/Assertion___96__cnt___60_____40__sizeof___40____95__nl__95__value__95__type__95__LC__95__TIME__41_____47___sizeof___40____95__nl__95__value__95__type__95__LC__95__TIME__91__0__93____41____41____39___failed..mdwn @@ -0,0 +1,44 @@ +### Please describe the problem. + +git annex version + +sh: loadlocale.c:129: _nl_intern_locale_data: Assertion `cnt < (sizeof (_nl_value_type_LC_TIME) / sizeof (_nl_value_type_LC_TIME[0]))' failed. +Aborted + +### What steps will reproduce the problem? + +git annex version + +### What version of git-annex are you using? On what operating system? + +[[!format sh """ + +Synology NAS, 32-bit prebuilt version + +wget https://downloads.kitenet.net/git-annex/linux/current/git-annex-standalone-i386.tar.gz +--2018-04-08 04:26:29-- https://downloads.kitenet.net/git-annex/linux/current/git-annex-standalone-i386.tar.gz +Resolving downloads.kitenet.net... 66.228.36.95, 2600:3c03::f03c:91ff:fe73:b0d2 +Connecting to downloads.kitenet.net|66.228.36.95|:443... connected. +HTTP request sent, awaiting response... 200 OK +Length: 60787901 (58M) [application/x-gzip] +Saving to: 'git-annex-standalone-i386.tar.gz' + +100%[================================================================================================================================================================================================================================================>] 60,787,901 1.52MB/s in 39s + +2018-04-08 04:27:09 (1.49 MB/s) - 'git-annex-standalone-i386.tar.gz' saved [60787901/60787901] + +"""]] + +### Please provide any additional information below. + +[[!format sh """ +# If you can, paste a complete transcript of the problem occurring here. +# If the problem is with the git-annex assistant, paste in .git/annex/daemon.log + + +# End of transcript or log. +"""]] + +### Have you had any luck using git-annex before? (Sometimes we get tired of reading bug reports all day and a lil' positive end note does wonders) + + diff --git a/doc/bugs/Assertion___96__cnt___60_____40__sizeof___40____95__nl__95__value__95__type__95__LC__95__TIME__41_____47___sizeof___40____95__nl__95__value__95__type__95__LC__95__TIME__91__0__93____41____41____39___failed./comment_1_96a809060ed05100b6f8e5198ec699be._comment b/doc/bugs/Assertion___96__cnt___60_____40__sizeof___40____95__nl__95__value__95__type__95__LC__95__TIME__41_____47___sizeof___40____95__nl__95__value__95__type__95__LC__95__TIME__91__0__93____41____41____39___failed./comment_1_96a809060ed05100b6f8e5198ec699be._comment new file mode 100644 index 0000000000..6db8c24c81 --- /dev/null +++ b/doc/bugs/Assertion___96__cnt___60_____40__sizeof___40____95__nl__95__value__95__type__95__LC__95__TIME__41_____47___sizeof___40____95__nl__95__value__95__type__95__LC__95__TIME__91__0__93____41____41____39___failed./comment_1_96a809060ed05100b6f8e5198ec699be._comment @@ -0,0 +1,13 @@ +[[!comment format=mdwn + username="ga@2217a6e07239bc6b36af2a9134b2647fdebcabcd" + nickname="ga" + avatar="http://cdn.libravatar.org/avatar/d077d3b19ca11c2036b4870e4340c7d1" + subject="Same error" + date="2018-05-05T05:28:35Z" + content=""" +I'm getting the same error on my Synology device. + +$ uname -a +Linux xxxx 3.10.102 #15266 SMP Mon Mar 26 15:10:20 CST 2018 armv7l GNU/Linux synology_armada38x_ds216j + +"""]] diff --git a/doc/bugs/Assertion___96__cnt___60_____40__sizeof___40____95__nl__95__value__95__type__95__LC__95__TIME__41_____47___sizeof___40____95__nl__95__value__95__type__95__LC__95__TIME__91__0__93____41____41____39___failed./comment_2_e1c443ba943a5883ca63fbf05d3e5b41._comment b/doc/bugs/Assertion___96__cnt___60_____40__sizeof___40____95__nl__95__value__95__type__95__LC__95__TIME__41_____47___sizeof___40____95__nl__95__value__95__type__95__LC__95__TIME__91__0__93____41____41____39___failed./comment_2_e1c443ba943a5883ca63fbf05d3e5b41._comment new file mode 100644 index 0000000000..d44460d2ae --- /dev/null +++ b/doc/bugs/Assertion___96__cnt___60_____40__sizeof___40____95__nl__95__value__95__type__95__LC__95__TIME__41_____47___sizeof___40____95__nl__95__value__95__type__95__LC__95__TIME__91__0__93____41____41____39___failed./comment_2_e1c443ba943a5883ca63fbf05d3e5b41._comment @@ -0,0 +1,9 @@ +[[!comment format=mdwn + username="joey" + subject="""comment 2""" + date="2018-05-08T18:28:59Z" + content=""" +The synology NAS has a very old linux kernel IIRC, so you will probably +have better luck with the tarball targeting ancient kernels, + +"""]] diff --git a/doc/bugs/Assertion___96__cnt___60_____40__sizeof___40____95__nl__95__value__95__type__95__LC__95__TIME__41_____47___sizeof___40____95__nl__95__value__95__type__95__LC__95__TIME__91__0__93____41____41____39___failed./comment_3_acb934b9e7d91aa59f49926e658c4bbe._comment b/doc/bugs/Assertion___96__cnt___60_____40__sizeof___40____95__nl__95__value__95__type__95__LC__95__TIME__41_____47___sizeof___40____95__nl__95__value__95__type__95__LC__95__TIME__91__0__93____41____41____39___failed./comment_3_acb934b9e7d91aa59f49926e658c4bbe._comment new file mode 100644 index 0000000000..0fc39f5333 --- /dev/null +++ b/doc/bugs/Assertion___96__cnt___60_____40__sizeof___40____95__nl__95__value__95__type__95__LC__95__TIME__41_____47___sizeof___40____95__nl__95__value__95__type__95__LC__95__TIME__91__0__93____41____41____39___failed./comment_3_acb934b9e7d91aa59f49926e658c4bbe._comment @@ -0,0 +1,19 @@ +[[!comment format=mdwn + username="ga@2217a6e07239bc6b36af2a9134b2647fdebcabcd" + nickname="ga" + avatar="http://cdn.libravatar.org/avatar/d077d3b19ca11c2036b4870e4340c7d1" + subject="Arm processor on Synology device" + date="2018-05-10T11:07:44Z" + content=""" +Unfortunately the processor is an Arm processor, so the ancient build you linked won't work: + +$ grep model /proc/cpuinfo + +model name : ARMv7 Processor rev 1 (v7l) + +(I have a DS216j) + +I see you have an outstanding issue about providing an ancient armel build, maybe that would help? + +https://git-annex.branchable.com/todo/add_ancient_armel_build/ +"""]] diff --git a/doc/bugs/Assertion___96__cnt___60_____40__sizeof___40____95__nl__95__value__95__type__95__LC__95__TIME__41_____47___sizeof___40____95__nl__95__value__95__type__95__LC__95__TIME__91__0__93____41____41____39___failed./comment_4_39366d601e4200a953fe375c7c96f106._comment b/doc/bugs/Assertion___96__cnt___60_____40__sizeof___40____95__nl__95__value__95__type__95__LC__95__TIME__41_____47___sizeof___40____95__nl__95__value__95__type__95__LC__95__TIME__91__0__93____41____41____39___failed./comment_4_39366d601e4200a953fe375c7c96f106._comment new file mode 100644 index 0000000000..17ae553b47 --- /dev/null +++ b/doc/bugs/Assertion___96__cnt___60_____40__sizeof___40____95__nl__95__value__95__type__95__LC__95__TIME__41_____47___sizeof___40____95__nl__95__value__95__type__95__LC__95__TIME__91__0__93____41____41____39___failed./comment_4_39366d601e4200a953fe375c7c96f106._comment @@ -0,0 +1,35 @@ +[[!comment format=mdwn + username="ewen" + avatar="http://cdn.libravatar.org/avatar/605b2981cb52b4af268455dee7a4f64e" + subject="Synology NAS" + date="2018-07-08T06:52:08Z" + content=""" +I can confirm I got the same error message trying the 2018-06-26 amd64 standalone build on a Synology DS216+ NAS (installed via the [same instructions that I used last year](http://ewen.mcneill.gen.nz/blog/entry/2017-05-28-git-annex-on-synology-ds216+/), after creating the directory to allow the locale to be generated. But if I instead installed the `i386-ancient` version, and let that generate the locales, then that seems to work okay. Eg, I can then run `git annex version` to completion: + +
    +ewen@nas01:/volume1/thirdparty$ git annex version
+git-annex version: 6.20180626-gb091dac
+build flags: Assistant Webapp Pairing S3(multipartupload)(storageclasses) WebDAV Inotify DBus DesktopNotify ConcurrentOutput TorrentParser MagicMime Feeds Testsuite
+dependency versions: aws-0.17.1 bloomfilter-2.0.1.0 cryptonite-0.23 DAV-1.3.1 feed-0.3.12.0 ghc-8.0.2 http-client-0.5.7.0 persistent-sqlite-2.6.2 torrent-10000.1.1 uuid-1.3.13 yesod-1.4.5
+key/value backends: SHA256E SHA256 SHA512E SHA512 SHA224E SHA224 SHA384E SHA384 SHA3_256E SHA3_256 SHA3_512E SHA3_512 SHA3_224E SHA3_224 SHA3_384E SHA3_384 SKEIN256E SKEIN256 SKEIN512E SKEIN512 BLAKE2B256E BLAKE2B256 BLAKE2B512E BLAKE2B512 BLAKE2B160E BLAKE2B160 BLAKE2B224E BLAKE2B224 BLAKE2B384E BLAKE2B384 BLAKE2S256E BLAKE2S256 BLAKE2S160E BLAKE2S160 BLAKE2S224E BLAKE2S224 BLAKE2SP256E BLAKE2SP256 BLAKE2SP224E BLAKE2SP224 SHA1E SHA1 MD5E MD5 WORM URL
+remote types: git gcrypt p2p S3 bup directory rsync web bittorrent webdav adb tahoe glacier ddar hook external
+operating system: linux i386
+supported repository versions: 3 5 6
+upgrade supported from repository versions: 0 1 2 3 4 5
+ewen@nas01:/volume1/thirdparty$ 
+
    + +(I haven't tried the non-archaic i386 version.) + +Since the locales *should* be user space only, I'm assuming that the issue isn't \"archaic kernel\" -- although the Synology DS216+ NAS kernel is based on a fairly old version: + + ewen@nas01:/volume1/thirdparty$ uname -a + Linux nas01 3.10.105 #23739 SMP Fri Jun 8 12:51:05 CST 2018 x86_64 GNU/Linux synology_braswell_216+II + ewen@nas01:/volume1/thirdparty$ + +but probably related to the `libc` version `git-annex` was compiled against, and the version of the tools being used to generate the locales. + +I've not tested the new version much on the Synology DS216+ NAS, but I'm expecting that it should work now that \"`git annex version`\" works (and \"`git annex sync`\" works from another machine to it), with the `i386` \"archaic\" build, the rest should work for my purposes (basically `ssh` remote). + +Ewen +"""]] diff --git a/doc/bugs/Assistant_deleted_file_when_merging___40__another_report__41__.mdwn b/doc/bugs/Assistant_deleted_file_when_merging___40__another_report__41__.mdwn new file mode 100644 index 0000000000..9b40fa3869 --- /dev/null +++ b/doc/bugs/Assistant_deleted_file_when_merging___40__another_report__41__.mdwn @@ -0,0 +1,37 @@ +### Please describe the problem. +A couple of days ago I merged a v5 repo with an OLD one (see [1]) - both on the local drive. I *thought* it all went relatively smoothly until I tried to find some files that existed pre-merge. + +**All** files from before the merge were deleted as part of the merge commit. Fortunately, they seem to be available when I do `git annex unused` + + +--- + +Any thoughts on how I would recover the removed files? Some have a web remote, so doing it by hand for all 300ish would be a PITA. + +--- + +[1] Last commit was Dec 2013, and `.git/config` has `annex.version = 3` + + +### What steps will reproduce the problem? + +Ah. That's hard as I can't remember exactly what I did... :-/ + +1. Find an OLD repo and merge with a current one using the assistant. +2. I *think* I used the webapp and changed repositories to the old one (that was marked as `unwanted`). +3. I *think* I stopped and restarted the assistant a few times. (It was a loooooong merge). + +### What version of git-annex are you using? On what operating system? + +- git-annex version: 6.20170520 (at time of merge) +- git-annex version: 6.20170818 (currently) +- Operating system: macOS Sierra v10.12.6 +- Installed by: homebrew + +### Please provide any additional information below. + +I suspect this is a bug that can't be reproduced easily. + +### Have you had any luck using git-annex before? (Sometimes we get tired of reading bug reports all day and a lil' positive end note does wonders) + +<3 <3 <3 diff --git a/doc/bugs/Assistant_deleted_file_when_merging___40__another_report__41__/comment_1_27d96b639bc386132b9e74170532d749._comment b/doc/bugs/Assistant_deleted_file_when_merging___40__another_report__41__/comment_1_27d96b639bc386132b9e74170532d749._comment new file mode 100644 index 0000000000..e9dbba0ad3 --- /dev/null +++ b/doc/bugs/Assistant_deleted_file_when_merging___40__another_report__41__/comment_1_27d96b639bc386132b9e74170532d749._comment @@ -0,0 +1,9 @@ +[[!comment format=mdwn + username="olaf" + avatar="http://cdn.libravatar.org/avatar/4ae498d3d6ee558d6b65caa658f72572" + subject="Similar(ish) bug reports" + date="2017-08-23T05:55:50Z" + content=""" +- [git annex sync deleted a bunch of files (not expected)](/bugs/git_annex_sync_deleted_a_bunch_of_files___40__not_expected__41__/) +- [Assistant deleted file when merging](/bugs/git_annex_still_deleting_content_when_merging/) +"""]] diff --git a/doc/bugs/Assistant_deleted_file_when_merging___40__another_report__41__/comment_2_b9a39f172115fc39fee21c8987843faa._comment b/doc/bugs/Assistant_deleted_file_when_merging___40__another_report__41__/comment_2_b9a39f172115fc39fee21c8987843faa._comment new file mode 100644 index 0000000000..c8305a4e6d --- /dev/null +++ b/doc/bugs/Assistant_deleted_file_when_merging___40__another_report__41__/comment_2_b9a39f172115fc39fee21c8987843faa._comment @@ -0,0 +1,26 @@ +[[!comment format=mdwn + username="olaf" + avatar="http://cdn.libravatar.org/avatar/4ae498d3d6ee558d6b65caa658f72572" + subject="It's the old remote..." + date="2017-08-23T06:03:45Z" + content=""" +So, looking at `git log` has a clue... + +The commit message is + + git-annex in + +So... I guess I've switched in the webapp to the `unwanted` repo and it's gone and deleted the files. + +That seems weird. I would have thought that the type of the repo `unwanted`/`full backup`/etc. would be part of the details shared by a sync and so commands in the old repo should know that it is `unwanted`. Somehow auto-*magic*-ally I would have hoped that `unwanted` repos would be not dominant in merges. + +So @joeyh should I change this bug to something about `unwanted` repos being passive aggressive? ;-) + +--- + +## However... + +I'd still like some suggestion(s) on how to get my files back... + + +"""]] diff --git a/doc/bugs/Assistant_deleted_file_when_merging___40__another_report__41__/comment_3_324213fb62c67dc9565f0e812500e046._comment b/doc/bugs/Assistant_deleted_file_when_merging___40__another_report__41__/comment_3_324213fb62c67dc9565f0e812500e046._comment new file mode 100644 index 0000000000..b26b052a7c --- /dev/null +++ b/doc/bugs/Assistant_deleted_file_when_merging___40__another_report__41__/comment_3_324213fb62c67dc9565f0e812500e046._comment @@ -0,0 +1,8 @@ +[[!comment format=mdwn + username="michalrus" + avatar="http://cdn.libravatar.org/avatar/83c0b6e7f9d20f09a892263c4903bbae" + subject="comment 3" + date="2017-08-26T12:18:54Z" + content=""" +Related: [https://git-annex.branchable.com/bugs/Data_loss_when_copying_files_with_running_assistant/](https://git-annex.branchable.com/bugs/Data_loss_when_copying_files_with_running_assistant/) +"""]] diff --git a/doc/bugs/Assistant_deleted_file_when_merging___40__another_report__41__/comment_4_271c48d2cf4b09c27cf47ecba7fe1a64._comment b/doc/bugs/Assistant_deleted_file_when_merging___40__another_report__41__/comment_4_271c48d2cf4b09c27cf47ecba7fe1a64._comment new file mode 100644 index 0000000000..ada2986120 --- /dev/null +++ b/doc/bugs/Assistant_deleted_file_when_merging___40__another_report__41__/comment_4_271c48d2cf4b09c27cf47ecba7fe1a64._comment @@ -0,0 +1,8 @@ +[[!comment format=mdwn + username="michalrus" + avatar="http://cdn.libravatar.org/avatar/83c0b6e7f9d20f09a892263c4903bbae" + subject="comment 4" + date="2017-08-26T12:19:17Z" + content=""" +Related: [https://git-annex.branchable.com/bugs/Data_loss_when_copying_files_with_running_assistant/](https://git-annex.branchable.com/bugs/Data_loss_when_copying_files_with_running_assistant/) +"""]] diff --git a/doc/bugs/Assistant_does_not_update_adjusted_branch.mdwn b/doc/bugs/Assistant_does_not_update_adjusted_branch.mdwn new file mode 100644 index 0000000000..83fedbb1e2 --- /dev/null +++ b/doc/bugs/Assistant_does_not_update_adjusted_branch.mdwn @@ -0,0 +1,90 @@ +### Please describe the problem. + +When using the assistant to synchronize several annex repositories, changes are not correctly or not at all propagated to repositories that are in the adjusted branch. + +* New files from other repositories are not created automatically in adjusted branch repos +* Commited changes in files from other repositories are not reflected in in adjusted branch repos + +The manual states: +> To propigate changes from the adjusted branch back to the original +> branch, and to other repositories, as well as to merge in changes from +> other repositories, use git annex sync. + +I would expect that the assistant is just doing this, that I don't have to trigger the sync in the adjusted branch repo by myself. + +### What steps will reproduce the problem? + +Synchronizing several git annex repositories (three) with the assistant, with the second one being in the adjusted branch: + +* Create a new file in the first repository (normal v5, autocommit=true) +* The file is being commited automatically and synchronized into the 3rd repository (normal v5). +* But the file does not appear in the 2nd one, being in the adjusted branch + +Once you trigger + git annex sync +in the 2nd repository, it seems the adjusted branch synchronizes with the master and the file appears. + +The same happens with changes in files. + +### What version of git-annex are you using? On what operating system? + +* git-annex version: 6.20160511-1~ubuntu14.04.1~ppa1 +* Linux 4.4.0-24-generic #43~14.04.1-Ubuntu x86_64 +* Linux Mint 17.3 Rosa + +### Please provide any additional information below. + +Assistant log from adjust branch repo. +test7.txt is the new file created on other repo which didn't showed up immediately. + +[[!format sh """ +# If you can, paste a complete transcript of the problem occurring here. +# If the problem is with the git-annex assistant, paste in .git/annex/daemon.log +(merging origin/git-annex into git-annex...) +To /home/sven/Temporary/Annex1 + + 9cc4eea...0e6c4ba git-annex -> synced/git-annex (forced update) +(started...) +(merging synced/git-annex into git-annex...) +(Merging into master...) Merge made by the 'recursive' strategy. + script.sh | 1 + + 1 file changed, 1 insertion(+) + create mode 100644 script.sh +(Merging into adjusted branch...) +Updating 449f26f..09950df +Fast-forward + script.sh | 1 + + 1 file changed, 1 insertion(+) + create mode 100644 script.sh +[2016-09-25 16:32:28.594537] Committer: Committing changes to git +(recording state in git...) +[2016-09-25 16:32:28.631525] Pusher: Syncing with origin +To /home/sven/Temporary/Annex1 + 0e6c4ba..578f070 git-annex -> synced/git-annex + 9754018..38c3683 master -> synced/master +[2016-09-25 16:32:29.632224] Committer: Committing changes to git +(recording state in git...) +[2016-09-25 16:32:30.661338] Pusher: Syncing with origin +Everything up-to-date +(merging synced/git-annex into git-annex...) +(merging synced/git-annex into git-annex...) +(recording state in git...) +git-annex: Daemon is already running. +[2016-09-25 16:36:58.360139] Committer: Adding test7.txt +add test7.txt ok +[2016-09-25 16:36:58.476985] Committer: Committing changes to git +(recording state in git...) +[2016-09-25 16:36:58.494964] Pusher: Syncing with origin +Everything up-to-date +[2016-09-25 16:36:59.575623] Committer: Adding test7.txt +add test7.txt ok +[2016-09-25 16:36:59.578454] Committer: Committing changes to git +(recording state in git...) +[2016-09-25 16:37:00.506906] Pusher: Syncing with origin +Everything up-to-date +# End of transcript or log. +"""]] + +### Have you had any luck using git-annex before? (Sometimes we get tired of reading bug reports all day and a lil' positive end note does wonders) + +I'm using git-annex a lot for synchronizing stuff between work, home, external backup discs, and now that I see the autocommit=false flag I'll surely start to use the assistant. +And the new adjust branch is perfect for one of my use cases, but not like it behaves right now. diff --git a/doc/bugs/Assistant_drops_files_from_remote_repos_that_it_shouldn__39__t.mdwn b/doc/bugs/Assistant_drops_files_from_remote_repos_that_it_shouldn__39__t.mdwn new file mode 100644 index 0000000000..52a2efda22 --- /dev/null +++ b/doc/bugs/Assistant_drops_files_from_remote_repos_that_it_shouldn__39__t.mdwn @@ -0,0 +1,39 @@ +### Please describe the problem. + +git-annex assistant starts to drop files in remote repos, even when they are set to manual. + +### What steps will reproduce the problem? + + +Create 3 repos: + +* A -- standard, archive +* m1 -- standard, manual +* m2 -- standard, manual + +System has 3 files annexed: file1, file2, file3. Repo "A" has all three files, m1 has none, m2 has file3. + +So, while in m1: + + m1 $ git annex find --want-drop --in m2 + file3 + +file3 shouln't be dropped from m2. There is no reason to do this. m2 is set as manual, and it shouldn't be touched in any case. + +Now, let's get this file in m1: + + m1 $ git annex get file3 + get file3 (from m2...) ok + (Recording state in git...) + m1 $ git annex find --want-drop --in m2 + +So when 'file3' is present in local repo, it's not going to be dropped from m2. + +I guess that rule 'present' works in local repo context while 'drop' acts on remote files. + + +### What version of git-annex are you using? On what operating system? + +I'm using latest version in Debian Jessie (5.20141024) on amd64 and armhf. I've also reproduced the bug with manually compiled 5.20141105-g8d8b248 on amd64. + +[[!tag moreinfo]] diff --git a/doc/bugs/Assistant_drops_files_from_remote_repos_that_it_shouldn__39__t/comment_1_5e7d5b0b996a171c641fa560a613d319._comment b/doc/bugs/Assistant_drops_files_from_remote_repos_that_it_shouldn__39__t/comment_1_5e7d5b0b996a171c641fa560a613d319._comment new file mode 100644 index 0000000000..412dbcaeb5 --- /dev/null +++ b/doc/bugs/Assistant_drops_files_from_remote_repos_that_it_shouldn__39__t/comment_1_5e7d5b0b996a171c641fa560a613d319._comment @@ -0,0 +1,19 @@ +[[!comment format=mdwn + username="joey" + subject="""comment 1""" + date="2014-11-06T19:33:18Z" + content=""" +The command `git annex find --want-drop --in m2` +finds files that meet 2 criteria: + +1. Present in the m2 remote. In this case, that is file3. +2. Are not wanted to be present in the repository in which you run + git-annex. In this case, that's all files, since you're running it + in m1, which has no files, and is set to manual mode. + +Combining criterion 1 and criterion 2, the command finds file3. +Not that you expected, but only I think because you misunderstood +what --in does. + +So, I don't see a bug here? +"""]] diff --git a/doc/bugs/Assistant_drops_files_from_remote_repos_that_it_shouldn__39__t/comment_2_2015dfc648d26d6dfabe6ad4f4754034._comment b/doc/bugs/Assistant_drops_files_from_remote_repos_that_it_shouldn__39__t/comment_2_2015dfc648d26d6dfabe6ad4f4754034._comment new file mode 100644 index 0000000000..48125db22e --- /dev/null +++ b/doc/bugs/Assistant_drops_files_from_remote_repos_that_it_shouldn__39__t/comment_2_2015dfc648d26d6dfabe6ad4f4754034._comment @@ -0,0 +1,13 @@ +[[!comment format=mdwn + username="https://www.google.com/accounts/o8/id?id=AItOawnDx6KWBl4PpP7qikNB7rp0hK_UvwQq_L0" + nickname="Александр" + subject="comment 2" + date="2014-11-06T21:15:14Z" + content=""" +Joey, thank you very much for the explanation. I completely agree that what I described in \"how to reproduce\" works as it should. + +Actually I was trying to isolate an issue I had where assistant drops files in a remote manual repo. I don't quite get what circumstances are, but sometimes assistant running in m1 starts to drop files in m2 (while still leaving a copy in A). In this test setting the issue doesn't appear... In my actual setup there is an archive and 3 manual live repos, everything connected via ssh, sometimes on slow links, one manual repo is on a linux phone. Assistants were running on several repos. + +I'll try to find a convinient way to reproduce this situation, or maybe discover that bug is in my head again ) Anyway I'll return with an update soon. + +"""]] diff --git a/doc/bugs/Assistant_drops_files_from_remote_repos_that_it_shouldn__39__t/comment_3_8f394437982cebeaf878f5b3871afaf3._comment b/doc/bugs/Assistant_drops_files_from_remote_repos_that_it_shouldn__39__t/comment_3_8f394437982cebeaf878f5b3871afaf3._comment new file mode 100644 index 0000000000..6b281b60fe --- /dev/null +++ b/doc/bugs/Assistant_drops_files_from_remote_repos_that_it_shouldn__39__t/comment_3_8f394437982cebeaf878f5b3871afaf3._comment @@ -0,0 +1,18 @@ +[[!comment format=mdwn + username="https://www.google.com/accounts/o8/id?id=AItOawnSenxKyE_2Z6Wb-EBMO8FciyRywjx1ZiQ" + nickname="Walter" + subject="comment 3" + date="2014-11-06T22:59:38Z" + content=""" +This might not be a very helpful comment, but a while ago I also had a situation where files would be dropped from a remote manual repo. + +Unfortunately, I was not able to get this to happen reliably, but from memory + +(on repo 1) + + git annex get file --from repo2 + +would result in ``file`` being gotten, and then dropped, from repo2, even though both repos were set to manual. + +But, the unhelpful part, is I can't even remember exactly when this happened (maybe a year ago), so this might not help narrow it down. +"""]] diff --git a/doc/bugs/Assistant_drops_files_from_remote_repos_that_it_shouldn__39__t/comment_4_ccd108a53a9fb747539e269becfb3e05._comment b/doc/bugs/Assistant_drops_files_from_remote_repos_that_it_shouldn__39__t/comment_4_ccd108a53a9fb747539e269becfb3e05._comment new file mode 100644 index 0000000000..a9ca8c8cd9 --- /dev/null +++ b/doc/bugs/Assistant_drops_files_from_remote_repos_that_it_shouldn__39__t/comment_4_ccd108a53a9fb747539e269becfb3e05._comment @@ -0,0 +1,9 @@ +[[!comment format=mdwn + username="https://renaud.casenave.fr/" + subject="comment 4" + date="2014-11-07T04:43:44Z" + content=""" +This is the same problem I try to report here: http://git-annex.branchable.com/bugs/file_in_manual_mode_repository_is_dropped_when_it_is_copied_to_another_manual_mode_repository/ + +It is as Walter said, `git annex get file --from repo2` will add the file to the current repo but will drop it in repo2 +"""]] diff --git a/doc/bugs/Assistant_drops_files_from_remote_repos_that_it_shouldn__39__t/comment_5_7ee571496c6d7cd57a206e6356dc2be3._comment b/doc/bugs/Assistant_drops_files_from_remote_repos_that_it_shouldn__39__t/comment_5_7ee571496c6d7cd57a206e6356dc2be3._comment new file mode 100644 index 0000000000..962f98e35c --- /dev/null +++ b/doc/bugs/Assistant_drops_files_from_remote_repos_that_it_shouldn__39__t/comment_5_7ee571496c6d7cd57a206e6356dc2be3._comment @@ -0,0 +1,166 @@ +[[!comment format=mdwn + username="https://www.google.com/accounts/o8/id?id=AItOawnDx6KWBl4PpP7qikNB7rp0hK_UvwQq_L0" + nickname="Александр" + subject="comment 5" + date="2014-11-16T12:32:28Z" + content=""" +Thanks Walter, renaud. + +I was finally able to consitently reproduce the bug and record clean debug logs. + +There are 3 repos, \"arch\", \"man1\", \"man2\". They are on different machines connected via ssh. \"arch\" is archive, \"man1\" and \"man2\" are in manual mode. Assistant is started only at \"man1\" with args -v -d, after that some files are added. Assistant detects them, adds and then transfers to \"arch\", this works as expected. + +After that in \"man2\" I run command: + + git annex get --from man1 random_1_24 + +File is successfully transferred, and then assistant removes it from \"man1\". Here is corresponding part of logs at \"man1\": + + [2014-11-16 15:10:12 MSK] TransferWatcher: transfer starting: Upload UUID \"d865391b-cf21-42d6-a04c-e1b32897c1af\" random_1_24 Nothing + [2014-11-16 15:10:12 MSK] TransferWatcher: transfer starting: Upload UUID \"d865391b-cf21-42d6-a04c-e1b32897c1af\" random_1_24 Nothing + [2014-11-16 15:10:13 MSK] TransferWatcher: transfer finishing: Transfer {transferDirection = Upload, transferUUID = UUID \"d865391b-cf21-42d6-a04c-e1b32897c1af\", transferKey = Key {keyName = \"f91971893f882086488ca244c7c9e60a5ca23e088a95c171c21e40327de3278b\", keyBackendName = \"SHA256E\", keySize = Just 3145728, keyMtime = Nothing, keyChunkSize = Nothing, keyChunkNum = Nothing}} + [2014-11-16 15:10:13 MSK] TransferWatcher: transfer finishing: Transfer {transferDirection = Upload, transferUUID = UUID \"d865391b-cf21-42d6-a04c-e1b32897c1af\", transferKey = Key {keyName = \"f91971893f882086488ca244c7c9e60a5ca23e088a95c171c21e40327de3278b\", keyBackendName = \"SHA256E\", keySize = Just 3145728, keyMtime = Nothing, keyChunkSize = Nothing, keyChunkNum = Nothing}} + [2014-11-16 15:10:14 MSK] RemoteControl: SYNCING ssh://jolla:2222//home/nemo/annex_test + [2014-11-16 15:10:14 MSK] RemoteControl: Syncing with man2 + From ssh://jolla:2222//home/nemo/annex_test + d20436c..f7aff95 git-annex -> man1/git-annex + [2014-11-16 15:10:14 MSK] read: git [\"--git-dir=/home/butler/annex_test/.git\",\"--work-tree=/home/butler/annex_test\",\"show-ref\",\"git-annex\"] + [2014-11-16 15:10:14 MSK] read: git [\"--git-dir=/home/butler/annex_test/.git\",\"--work-tree=/home/butler/annex_test\",\"show-ref\",\"--hash\",\"refs/heads/git-annex\"] + [2014-11-16 15:10:14 MSK] read: git [\"--git-dir=/home/butler/annex_test/.git\",\"--work-tree=/home/butler/annex_test\",\"log\",\"refs/heads/git-annex..ea323207549cd5853213f1aeaf5f3c1f32e79fe9\",\"-n1\",\"--pretty=%H\"] + [2014-11-16 15:10:14 MSK] RemoteControl: DONESYNCING ssh://jolla:2222//home/nemo/annex_test 1 + [2014-11-16 15:10:14 MSK] read: git [\"--git-dir=/home/butler/annex_test/.git\",\"--work-tree=/home/butler/annex_test\",\"log\",\"refs/heads/git-annex..5701db750b10ae4d579bbccee828c808c8031e7e\",\"-n1\",\"--pretty=%H\"] + [2014-11-16 15:10:14 MSK] read: git [\"--git-dir=/home/butler/annex_test/.git\",\"--work-tree=/home/butler/annex_test\",\"log\",\"refs/heads/git-annex..08e7fcc374ee7e233f5f26c0bb7ff825a673818e\",\"-n1\",\"--pretty=%H\"] + [2014-11-16 15:10:14 MSK] read: git [\"--git-dir=/home/butler/annex_test/.git\",\"--work-tree=/home/butler/annex_test\",\"log\",\"refs/heads/git-annex..959215d3a9eae6a65ec5e6c68f345f9f623a0d1e\",\"-n1\",\"--pretty=%H\"] + [2014-11-16 15:10:14 MSK] read: git [\"--git-dir=/home/butler/annex_test/.git\",\"--work-tree=/home/butler/annex_test\",\"log\",\"refs/heads/git-annex..f7aff95774a095ccf3a5b11bf2e27108d6aadf58\",\"-n1\",\"--pretty=%H\"] + [2014-11-16 15:10:14 MSK] read: git [\"--git-dir=/home/butler/annex_test/.git\",\"--work-tree=/home/butler/annex_test\",\"log\",\"refs/heads/git-annex..4aae0d044273fd8716c4cf4946126b0ebf2e0db3\",\"-n1\",\"--pretty=%H\"] + (merging man1/git-annex into git-annex...) + [2014-11-16 15:10:14 MSK] feed: git [\"--git-dir=/home/butler/annex_test/.git\",\"--work-tree=/home/butler/annex_test\",\"update-index\",\"-z\",\"--index-info\"] + [2014-11-16 15:10:14 MSK] read: git [\"--git-dir=/home/butler/annex_test/.git\",\"--work-tree=/home/butler/annex_test\",\"diff-index\",\"--raw\",\"-z\",\"-r\",\"--no-renames\",\"-l0\",\"--cached\",\"f7aff95774a095ccf3a5b11bf2e27108d6aadf58\"] + [2014-11-16 15:10:14 MSK] chat: git [\"--git-dir=/home/butler/annex_test/.git\",\"--work-tree=/home/butler/annex_test\",\"hash-object\",\"-t\",\"blob\",\"-w\",\"--stdin\",\"--no-filters\"] + [2014-11-16 15:10:14 MSK] read: git [\"--git-dir=/home/butler/annex_test/.git\",\"--work-tree=/home/butler/annex_test\",\"log\",\"f7aff95774a095ccf3a5b11bf2e27108d6aadf58..refs/heads/git-annex\",\"-n1\",\"--pretty=%H\"] + (Recording state in git...) + [2014-11-16 15:10:14 MSK] read: git [\"--git-dir=/home/butler/annex_test/.git\",\"--work-tree=/home/butler/annex_test\",\"write-tree\"] + [2014-11-16 15:10:14 MSK] chat: git [\"--git-dir=/home/butler/annex_test/.git\",\"--work-tree=/home/butler/annex_test\",\"commit-tree\",\"dd91fafe3d6acfe88b41c677e66507c0f1905175\",\"--no-gpg-sign\",\"-p\",\"refs/heads/git-annex\",\"-p\",\"f7aff95774a095ccf3a5b11bf2e27108d6aadf58\"] + [2014-11-16 15:10:14 MSK] call: git [\"--git-dir=/home/butler/annex_test/.git\",\"--work-tree=/home/butler/annex_test\",\"update-ref\",\"refs/heads/git-annex\",\"2ad4a45d95a29db1ecc84a49d5c68bf709f2b3db\"] + [2014-11-16 15:10:14 MSK] read: git [\"--git-dir=/home/butler/annex_test/.git\",\"--work-tree=/home/butler/annex_test\",\"ls-tree\",\"--full-tree\",\"-z\",\"--\",\"refs/heads/git-annex\",\"uuid.log\",\"remote.log\",\"trust.log\",\"group.log\",\"numcopies.log\",\"schedule.log\",\"preferred-content.log\",\"required-content.log\",\"group-preferred-content.log\"] + [2014-11-16 15:10:14 MSK] read: git [\"--git-dir=/home/butler/annex_test/.git\",\"--work-tree=/home/butler/annex_test\",\"show-ref\",\"git-annex\"] + [2014-11-16 15:10:14 MSK] read: git [\"--git-dir=/home/butler/annex_test/.git\",\"--work-tree=/home/butler/annex_test\",\"show-ref\",\"--hash\",\"refs/heads/git-annex\"] + [2014-11-16 15:10:14 MSK] read: git [\"--git-dir=/home/butler/annex_test/.git\",\"--work-tree=/home/butler/annex_test\",\"log\",\"refs/heads/git-annex..2ad4a45d95a29db1ecc84a49d5c68bf709f2b3db\",\"-n1\",\"--pretty=%H\"] + [2014-11-16 15:10:14 MSK] read: git [\"--git-dir=/home/butler/annex_test/.git\",\"--work-tree=/home/butler/annex_test\",\"log\",\"refs/heads/git-annex..5701db750b10ae4d579bbccee828c808c8031e7e\",\"-n1\",\"--pretty=%H\"] + [2014-11-16 15:10:14 MSK] read: git [\"--git-dir=/home/butler/annex_test/.git\",\"--work-tree=/home/butler/annex_test\",\"log\",\"refs/heads/git-annex..08e7fcc374ee7e233f5f26c0bb7ff825a673818e\",\"-n1\",\"--pretty=%H\"] + [2014-11-16 15:10:14 MSK] read: git [\"--git-dir=/home/butler/annex_test/.git\",\"--work-tree=/home/butler/annex_test\",\"log\",\"refs/heads/git-annex..959215d3a9eae6a65ec5e6c68f345f9f623a0d1e\",\"-n1\",\"--pretty=%H\"] + [2014-11-16 15:10:14 MSK] read: git [\"--git-dir=/home/butler/annex_test/.git\",\"--work-tree=/home/butler/annex_test\",\"log\",\"refs/heads/git-annex..f7aff95774a095ccf3a5b11bf2e27108d6aadf58\",\"-n1\",\"--pretty=%H\"] + [2014-11-16 15:10:14 MSK] read: git [\"--git-dir=/home/butler/annex_test/.git\",\"--work-tree=/home/butler/annex_test\",\"log\",\"refs/heads/git-annex..4aae0d044273fd8716c4cf4946126b0ebf2e0db3\",\"-n1\",\"--pretty=%H\"] + drop random_1_24 (checking arch...) [2014-11-16 15:10:23 MSK] call: ssh [\"-S\",\".git/annex/ssh/axe\",\"-o\",\"ControlMaster=auto\",\"-o\",\"ControlPersist=yes\",\"-T\",\"axe\",\"git-annex-shell 'inannex' '//home/butler/annex_test' 'SHA256E-s3145728--f91971893f882086488ca244c7c9e60a5ca23e088a95c171c21e40327de3278b' --uuid 5aaf3dc8-f94c-4949-8491-26d5d0de5127\"] + ok + [2014-11-16 15:10:23 MSK] dropped random_1_24 (from here) (copies now 2) : drop wanted after Upload UUID \"d865391b-cf21-42d6-a04c-e1b32897c1af\" random_1_24 Nothing + drop man2 random_1_24 (checking arch...) [2014-11-16 15:10:23 MSK] call: ssh [\"-S\",\".git/annex/ssh/axe\",\"-o\",\"ControlMaster=auto\",\"-o\",\"ControlPersist=yes\",\"-T\",\"axe\",\"git-annex-shell 'inannex' '//home/butler/annex_test' 'SHA256E-s3145728--f91971893f882086488ca244c7c9e60a5ca23e088a95c171c21e40327de3278b' --uuid 5aaf3dc8-f94c-4949-8491-26d5d0de5127\"] + [2014-11-16 15:10:23 MSK] call: ssh [\"-S\",\".git/annex/ssh/jolla!2222\",\"-o\",\"ControlMaster=auto\",\"-o\",\"ControlPersist=yes\",\"-p\",\"2222\",\"-T\",\"jolla\",\"git-annex-shell 'dropkey' '//home/nemo/annex_test' '--quiet' '--force' 'SHA256E-s3145728--f91971893f882086488ca244c7c9e60a5ca23e088a95c171c21e40327de3278b' --uuid d865391b-cf21-42d6-a04c-e1b32897c1af\"] + ok + [2014-11-16 15:10:23 MSK] dropped random_1_24 (from UUID \"d865391b-cf21-42d6-a04c-e1b32897c1af\") (copies now 1) : drop wanted after Upload UUID \"d865391b-cf21-42d6-a04c-e1b32897c1af\" random_1_24 Nothing + [2014-11-16 15:11:14 MSK] read: git [\"--git-dir=/home/butler/annex_test/.git\",\"--work-tree=/home/butler/annex_test\",\"ls-tree\",\"--full-tree\",\"-z\",\"--\",\"refs/heads/git-annex\",\"uuid.log\",\"remote.log\",\"trust.log\",\"group.log\",\"numcopies.log\",\"schedule.log\",\"preferred-content.log\",\"required-content.log\",\"group-preferred-content.log\"] + +Here is git annex vicfg at \"man1\": + + # git-annex configuration + # + # Changes saved to this file will be recorded in the git-annex branch. + # + # Lines in this file have the format: + # setting field = value + + # Repository trust configuration + # (Valid trust levels: trusted semitrusted untrusted dead) + # (for web) + #trust 00000000-0000-0000-0000-000000000001 = semitrusted + # (for man2) + #trust 2104670a-6a96-11e4-a3ca-0f84dcf32d24 = semitrusted + # (for [arch]) + #trust 5aaf3dc8-f94c-4949-8491-26d5d0de5127 = semitrusted + # (for [man2]) + #trust d865391b-cf21-42d6-a04c-e1b32897c1af = semitrusted + # (for man1) + #trust f15688ce-14e0-43ba-a3a2-69ee9d5bdf45 = semitrusted + + # Repository groups + # (Standard groups: client transfer backup incrementalbackup smallarchive archive source manual public unwanted) + # (Separate group names with spaces) + # (for [arch]) + group 5aaf3dc8-f94c-4949-8491-26d5d0de5127 = archive + # (for [man2]) + group d865391b-cf21-42d6-a04c-e1b32897c1af = manual + # (for man1) + group f15688ce-14e0-43ba-a3a2-69ee9d5bdf45 = manual + # (for web) + #group 00000000-0000-0000-0000-000000000001 = + # (for man2) + #group 2104670a-6a96-11e4-a3ca-0f84dcf32d24 = + + # Repository preferred contents + # (Set to \"standard\" to use a repository's group's preferred contents) + # (for [arch]) + wanted 5aaf3dc8-f94c-4949-8491-26d5d0de5127 = standard + # (for [man2]) + wanted d865391b-cf21-42d6-a04c-e1b32897c1af = standard + # (for man1) + wanted f15688ce-14e0-43ba-a3a2-69ee9d5bdf45 = standard + # (for web) + #wanted 00000000-0000-0000-0000-000000000001 = + # (for man2) + #wanted 2104670a-6a96-11e4-a3ca-0f84dcf32d24 = + + # Group preferred contents + # (Used by repositories with \"groupwanted\" in their preferred contents) + #groupwanted archive = + #groupwanted backup = + #groupwanted client = + #groupwanted incrementalbackup = + #groupwanted manual = + #groupwanted public = + #groupwanted smallarchive = + #groupwanted source = + #groupwanted transfer = + #groupwanted unwanted = + + # Standard preferred contents + # (Used by wanted or groupwanted expressions containing \"standard\") + # (For reference only; built-in and cannot be changed!) + # standard client = (((exclude=*/archive/* and exclude=archive/*) or (not (copies=archive:1 or copies=smallarchive:1))) and not unused) or approxlackingcopies=1 + # standard transfer = (not (inallgroup=client and copies=client:2) and ((((exclude=*/archive/* and exclude=archive/*) or (not (copies=archive:1 or copies=smallarchive:1))) and not unused) or approxlackingcopies=1)) or approxlackingcopies=1 + # standard backup = include=* or unused + # standard incrementalbackup = ((include=* or unused) and (not copies=incrementalbackup:1)) or approxlackingcopies=1 + # standard smallarchive = ((include=*/archive/* or include=archive/*) and ((not (copies=archive:1 or copies=smallarchive:1)) or approxlackingcopies=1)) or approxlackingcopies=1 + # standard archive = (not (copies=archive:1 or copies=smallarchive:1)) or approxlackingcopies=1 + # standard source = not (copies=1) + # standard manual = present and ((((exclude=*/archive/* and exclude=archive/*) or (not (copies=archive:1 or copies=smallarchive:1))) and not unused) or approxlackingcopies=1) + # standard public = inpreferreddir + # standard unwanted = exclude=* + + # Repository required contents + # (for web) + #required 00000000-0000-0000-0000-000000000001 = + # (for man2) + #required 2104670a-6a96-11e4-a3ca-0f84dcf32d24 = + # (for [arch]) + #required 5aaf3dc8-f94c-4949-8491-26d5d0de5127 = + # (for [man2]) + #required d865391b-cf21-42d6-a04c-e1b32897c1af = + # (for man1) + #required f15688ce-14e0-43ba-a3a2-69ee9d5bdf45 = + + # Scheduled activities + # (Separate multiple activities with \"; \") + # (for web) + #schedule 00000000-0000-0000-0000-000000000001 = + # (for man2) + #schedule 2104670a-6a96-11e4-a3ca-0f84dcf32d24 = + # (for [arch]) + #schedule 5aaf3dc8-f94c-4949-8491-26d5d0de5127 = + # (for [man2]) + #schedule d865391b-cf21-42d6-a04c-e1b32897c1af = + # (for man1) + #schedule f15688ce-14e0-43ba-a3a2-69ee9d5bdf45 = + +Please let me know if I can provide more information, some extra logs, traces, recompile git-annex, etc. + +"""]] diff --git a/doc/bugs/Assistant_drops_files_from_remote_repos_that_it_shouldn__39__t/comment_6_6c88b218b8186242a6106fc3e7b3edf5._comment b/doc/bugs/Assistant_drops_files_from_remote_repos_that_it_shouldn__39__t/comment_6_6c88b218b8186242a6106fc3e7b3edf5._comment new file mode 100644 index 0000000000..e907711e42 --- /dev/null +++ b/doc/bugs/Assistant_drops_files_from_remote_repos_that_it_shouldn__39__t/comment_6_6c88b218b8186242a6106fc3e7b3edf5._comment @@ -0,0 +1,29 @@ +[[!comment format=mdwn + username="joey" + subject="""comment 6""" + date="2015-07-06T20:31:06Z" + content=""" +Well, you've provided some great information, and your setup seems sane, +and I'd expect it to keep the file in manual mode and not drop it. + +The key part of the log is this: + + [2014-11-16 15:10:23 MSK] dropped random_1_24 (from here) (copies now 2) : drop wanted after Upload UUID \"d865391b-cf21-42d6-a04c-e1b32897c1af\" random_1_24 Nothing + +So, after transferring the file to the remote where you're getting it, it +has apparently checked the local repo's preferred content, and thinks it says +that the local repo doesn't want the file anymore. Which certianly doesn't +make sense if the repo is configured to be in manual mode. + +The only time I'd expect this to happen is if the file in question is one that +a "client" repository wouldn't normally want. So, if "random_1_24" was in an +archive directory, the preferred content expression would let it drop it then. + +I tried to replicate your configuration, and wasn't able to reproduce +it deciding to drop the file, when testing with git-annex version 5.20150618. + +Can you reproduce this with the most recent release of git-annex? + +If so, can you verify that running "git annex drop --auto" doesn't try to drop any files when you run it +in the repository that's confgured to be in manual mode? +"""]] diff --git a/doc/bugs/Assistant_drops_files_from_remote_repos_that_it_shouldn__39__t/comment_7_fbbf9d1a63fc61a5fd288fd308b24ca1._comment b/doc/bugs/Assistant_drops_files_from_remote_repos_that_it_shouldn__39__t/comment_7_fbbf9d1a63fc61a5fd288fd308b24ca1._comment new file mode 100644 index 0000000000..372e91b559 --- /dev/null +++ b/doc/bugs/Assistant_drops_files_from_remote_repos_that_it_shouldn__39__t/comment_7_fbbf9d1a63fc61a5fd288fd308b24ca1._comment @@ -0,0 +1,13 @@ +[[!comment format=mdwn + username="joey" + subject="""comment 7""" + date="2015-07-06T22:08:11Z" + content=""" +I have also not been able to reproduce this using version 5.20141024. So, +there must be some peice of information left out that's needed to reproduce +this, or possibly it's an intermittent problem. + +I did verify that my attempts to reproduce it caused the code path that +handles drops after transfers to fire; in my tests it decided not to drop +the file. +"""]] diff --git a/doc/bugs/Assistant_drops_files_from_remote_repos_that_it_shouldn__39__t/comment_8_fe2498f60b02104a8a044c79cb5bea63._comment b/doc/bugs/Assistant_drops_files_from_remote_repos_that_it_shouldn__39__t/comment_8_fe2498f60b02104a8a044c79cb5bea63._comment new file mode 100644 index 0000000000..2eec510269 --- /dev/null +++ b/doc/bugs/Assistant_drops_files_from_remote_repos_that_it_shouldn__39__t/comment_8_fe2498f60b02104a8a044c79cb5bea63._comment @@ -0,0 +1,66 @@ +[[!comment format=mdwn + username="etesial@f4797a6d725e971a2d41f3cbcf174991da178c29" + nickname="etesial" + subject="comment 8" + date="2015-07-07T14:45:32Z" + content=""" +Hi Joey, I've failed to reproduce it now on Debian's 5.20141105-g8d8b248 and have already deleted dirs from previous setup. Maybe I'll try again later if I get some ideas what to tune. + +This time I've written a script to setup repos for better reproducibility and while result is negative, I think it'll still may be useful to post it here: + + #!/bin/bash + + set -x + set -e + + DIRS=(\"/home/butler/annex_test/data\" + \"/home/butler/annex_test/data\" + \"/home/butler/annex_test/data\") + + NAMES=(\"antiferno\" \"axe\" \"deadbird\") + + URIS=(\"ssh://antiferno//${DIRS[0]}\" + \"ssh://axe//${DIRS[1]}\" + \"ssh://deadbird//${DIRS[2]}\") + + MODES=(\"manual\" \"backup\" \"manual\") + + + function init { + git init \"$1\" + } + + function git_add_remote { + git --git-dir=\"$1/.git\" --work-tree=\"$1\" remote add \"$2\" \"$3\" + } + + function annex_init { + git --git-dir=\"$1/.git\" --work-tree=\"$1\" annex init \"annex at $2\" + } + + function annex_set_mode { + git --git-dir=\"$1/.git\" --work-tree=\"$1\" annex wanted . standard + git --git-dir=\"$1/.git\" --work-tree=\"$1\" annex group . \"$2\" + } + + function add_initial_file { + path=\"$1/file_$2\" + echo \"File from $2\" >\"$path\" + git --git-dir=\"$1/.git\" --work-tree=\"$1\" annex add \"$path\" + git --git-dir=\"$1/.git\" --work-tree=\"$1\" commit -m \"First file on $2\" + } + + function setup { + init \"${DIRS[$1]}\" + annex_init \"${DIRS[$1]}\" \"${NAMES[$1]}\" + git_add_remote \"${DIRS[$1]}\" \"${NAMES[$2]}\" \"${URIS[$2]}\" + git_add_remote \"${DIRS[$1]}\" \"${NAMES[$3]}\" \"${URIS[$3]}\" + annex_set_mode \"${DIRS[$1]}\" \"${MODES[$1]}\" + add_initial_file \"${DIRS[$1]}\" \"${NAMES[$1]}\" + } + + if [ \"$1\" == \"${NAMES[0]}\" ]; then setup 0 1 2; fi + if [ \"$1\" == \"${NAMES[1]}\" ]; then setup 1 2 0; fi + if [ \"$1\" == \"${NAMES[2]}\" ]; then setup 2 0 1; fi + +"""]] diff --git a/doc/bugs/Assistant_drops_files_from_remote_repos_that_it_shouldn__39__t/comment_9_c46cdba62da4f5ccfdc42dfc33aec600._comment b/doc/bugs/Assistant_drops_files_from_remote_repos_that_it_shouldn__39__t/comment_9_c46cdba62da4f5ccfdc42dfc33aec600._comment new file mode 100644 index 0000000000..a6a2397e7f --- /dev/null +++ b/doc/bugs/Assistant_drops_files_from_remote_repos_that_it_shouldn__39__t/comment_9_c46cdba62da4f5ccfdc42dfc33aec600._comment @@ -0,0 +1,34 @@ +[[!comment format=mdwn + username="boh" + avatar="http://cdn.libravatar.org/avatar/e7fa2d1c5d95e323fe48887f7f827b1f" + subject="comment 9" + date="2016-11-27T12:23:20Z" + content=""" +Seems as if the problem still exists in 6.20161118 (Debian). + +I have three repositories (among others), `jolla`, `sts-3xx`, and `here`. `jolla` and `here` are in group `manual`, `sts-3xx` is `backup`; `here` and `sts-3xx` have assistants running, `jolla` not. `jolla` and `sts-3xx` have slightly older versions of git-annex installed. + +Now, when I copy a file from `here` to `jolla` like this + + git annex copy real_programmers.png -t jolla + +the file is subsequently dropped by the assistant: + +``` +drop real_programmers.png (locking jolla...) [2016-11-27 13:00:02.667376556] chat: ssh [\"-S\",\".git/annex/ssh/jolla\",\"-o\",\"ControlMaster +=auto\",\"-o\",\"ControlPersist=yes\",\"-F\",\".git/annex/ssh.config\",\"-T\",\"jolla\",\"git-annex-shell 'lockcontent' '/~/Music/media/' '--debug' ' +SHA256E-s84499--ff98a733cc0122858fb11433c720e2d038fec190a3d36380d0e7e8dab468f883.png' --uuid 5298e3ce-1106-4d5e-b052-0aee4b27a344\"] +(locking sts-3xx...) [2016-11-27 13:00:03.252473676] chat: ssh [..., \"git-annex-shell 'lockcontent' '/backups/exot/media/' '--debug' 'SHA256E-s84499--ff98a733cc0122858fb11433c720e2d038fec190a3d 36380d0e7e8dab468f883.png' --uuid 1fec6253-171d-4f86-885b-e233be2d65ec\"] +(lockcontent failed) [2016-11-27 13:00:03.486158016] process done ExitFailure 1 +(checking sts-3xx...) [2016-11-27 13:00:03.487047149] call: ssh [..., \"git-annex-shell 'inannex' '/backups/exot/media/' '--debug' 'SHA256E-s84499--ff98a733cc0122858fb11433c720e2d038fec190a3d363 80d0e7e8dab468f883.png' --uuid 1fec6253-171d-4f86-885b-e233be2d65ec\"] +[2016-11-27 13:00:03.76435136] process done ExitSuccess +[2016-11-27 13:00:03.764705754] Dropping from here proof: Just (SafeDropProof (NumCopies 2) [RecentlyVerifiedCopy UUID \"1fec6253-171d-4 f86-885b-e233be2d65ec\",LockedCopy UUID \"5298e3ce-1106-4d5e-b052-0aee4b27a344\"] (Just (ContentRemovalLock (Key {keyName = \"ff98a733cc012 2858fb11433c720e2d038fec190a3d36380d0e7e8dab468f883.png\", keyBackendName = \"SHA256E\", keySize = Just 84499, keyMtime = Nothing, keyChun kSize = Nothing, keyChunkNum = Nothing})))) +[2016-11-27 13:00:04.24333081] process done ExitFailure 1 +ok +[2016-11-27 13:00:04.251232455] dropped real_programmers.png (from here) (copies now 4) : drop wanted after Upload UUID \"5298e3ce-1106- 4d5e-b052-0aee4b27a344\" real_programmers.png Just 84499 +``` + +However, I failed to reproduce the problem by replicating my setup with fresh repositories … + +Please let me know if you need more information, and *so* many thanks for git-annex! +"""]] diff --git a/doc/bugs/Assistant_has_created_155_semitrusted_repositories.mdwn b/doc/bugs/Assistant_has_created_155_semitrusted_repositories.mdwn new file mode 100644 index 0000000000..b1caf530d4 --- /dev/null +++ b/doc/bugs/Assistant_has_created_155_semitrusted_repositories.mdwn @@ -0,0 +1,191 @@ +### Please describe the problem. +git annex status reports 160 semitrusted repositories. Four of them are the ones I created (only via webapp, I think I never edited any git-annex config file). One is 00000000-0000-0000-0000-000000000001 -- web + and although I do not know what it is, it is not something new. The remaining 155 appeared spontaneously after several Gb of data (mostly many small files) were added to an Annex (in an 'archive' directory) operated in direct mode by the assistant. + + + +### What steps will reproduce the problem? +Add several Gb of files was enough to trigger this problem, but I did not try to reproduce it. It happened the day I installed the 4.20131106~bpo70+1 version. + +### What version of git-annex are you using? On what operating system? + +4.20131106~bpo70+1 on debian squeeze (7.2), with git 1.8.4.rc3. + + +### Please provide any additional information below. +May be related or not: at some point the webapp displayed two warning boxes. One of them held a message that I did not wirte down and proposed to "Restart the thread". This apparently worked since the box disappeared. The other warning box indicated "NetWatcherFallback crashed: unknown response from git cat-file" and proposed to restart the thread. Trying to "restart the thread" via the provided button just did not trigger any response of the webapp which seemed dead at that point. + +In spite of the git annex status shown below, the webapp still shows only the expected four repositories. + +Output of git annex status (hostname and xmpp account name were edited away): +[[!format sh """ +# If you can, paste a complete transcript of the problem occurring here. +# If the problem is with the git-annex assistant, paste in .git/annex/daemon.log +repository mode: direct +trusted repositories: +0 +semitrusted repositories: 160 + 00000000-0000-0000-0000-000000000001 -- web + 0ab193eb-0c76-4559-a93c-2e30ed8630a8 -- someMachineIown_datadir (archive) + 1384784127.91222s -- 1 391b0557-dc68-4e40-b6d0-da3033588753 + 1384784164.437824s -- 1 391b0557-dc68-4e40-b6d0-da3033588753 + 1384784176.944372s -- 1 391b0557-dc68-4e40-b6d0-da3033588753 + 1384784179.254498s -- 1 391b0557-dc68-4e40-b6d0-da3033588753 + 1384785147.558938s -- 1 0ab193eb-0c76-4559-a93c-2e30ed8630a8 + 1384785147.717223s -- 1 0ab193eb-0c76-4559-a93c-2e30ed8630a8 + 1384785159.041203s -- 1 0ab193eb-0c76-4559-a93c-2e30ed8630a8 + 1384785159.199504s -- 1 0ab193eb-0c76-4559-a93c-2e30ed8630a8 + 1384785185.79485s -- 1 0ab193eb-0c76-4559-a93c-2e30ed8630a8 + 1384785187.318128s -- 1 0ab193eb-0c76-4559-a93c-2e30ed8630a8 + 1384785215.236504s -- 1 0ab193eb-0c76-4559-a93c-2e30ed8630a8 + 1384785215.389096s -- 1 0ab193eb-0c76-4559-a93c-2e30ed8630a8 + 1384785313.539843s -- 1 0ab193eb-0c76-4559-a93c-2e30ed8630a8 + 1384785313.701305s -- 1 0ab193eb-0c76-4559-a93c-2e30ed8630a8 + 1384785315.596206s -- 1 0ab193eb-0c76-4559-a93c-2e30ed8630a8 + 1384785344.184461s -- 1 0ab193eb-0c76-4559-a93c-2e30ed8630a8 + 1384785348.192805s -- 1 0ab193eb-0c76-4559-a93c-2e30ed8630a8 + 1384785402.70316s -- 1 0ab193eb-0c76-4559-a93c-2e30ed8630a8 + 1384785406.524044s -- 1 0ab193eb-0c76-4559-a93c-2e30ed8630a8 + 1384785446.074236s -- 1 0ab193eb-0c76-4559-a93c-2e30ed8630a8 + 1384873605.313126s -- 1 391b0557-dc68-4e40-b6d0-da3033588753 + 1384873697.029999s -- 1 391b0557-dc68-4e40-b6d0-da3033588753 + 1384873761.687234s -- 1 391b0557-dc68-4e40-b6d0-da3033588753 + 1384873774.608376s -- 1 391b0557-dc68-4e40-b6d0-da3033588753 + 1384926279.456728s -- 0 391b0557-dc68-4e40-b6d0-da3033588753 + 1384926368.736s -- 1 391b0557-dc68-4e40-b6d0-da3033588753 + 1384926454.99433s -- 1 391b0557-dc68-4e40-b6d0-da3033588753 + 1384926494.152645s -- 1 391b0557-dc68-4e40-b6d0-da3033588753 + 1384926504.438232s -- 1 391b0557-dc68-4e40-b6d0-da3033588753 + 1384934790.89717s -- 1 391b0557-dc68-4e40-b6d0-da3033588753 + 1384934848.757067s -- 1 391b0557-dc68-4e40-b6d0-da3033588753 + 1384934899.087168s -- 1 391b0557-dc68-4e40-b6d0-da3033588753 + 1384934908.238587s -- 1 391b0557-dc68-4e40-b6d0-da3033588753 + 1384948772.14552s -- 1 391b0557-dc68-4e40-b6d0-da3033588753 + 1384948805.441196s -- 1 391b0557-dc68-4e40-b6d0-da3033588753 + 1384948813.397132s -- 1 391b0557-dc68-4e40-b6d0-da3033588753 + 1384948921.45481s -- 1 391b0557-dc68-4e40-b6d0-da3033588753 + 1384948924.855852s -- 1 391b0557-dc68-4e40-b6d0-da3033588753 + 1384949073.988946s -- 1 391b0557-dc68-4e40-b6d0-da3033588753 + 1384949082.298976s -- 1 391b0557-dc68-4e40-b6d0-da3033588753 + 1384949399.608138s -- 1 391b0557-dc68-4e40-b6d0-da3033588753 + 1384949581.12213s -- 1 391b0557-dc68-4e40-b6d0-da3033588753 + 1384949583.9923s -- 1 391b0557-dc68-4e40-b6d0-da3033588753 + 1384949700.22807s -- 1 391b0557-dc68-4e40-b6d0-da3033588753 + 1384949765.484768s -- 1 391b0557-dc68-4e40-b6d0-da3033588753 + 1384955202.85962s -- 1 391b0557-dc68-4e40-b6d0-da3033588753 + 1384955230.953995s -- 1 391b0557-dc68-4e40-b6d0-da3033588753 + 1384955402.534938s -- 1 391b0557-dc68-4e40-b6d0-da3033588753 + 1384955457.1885s -- 1 391b0557-dc68-4e40-b6d0-da3033588753 + 1384955524.603709s -- 1 391b0557-dc68-4e40-b6d0-da3033588753 + 1384955611.891061s -- 1 391b0557-dc68-4e40-b6d0-da3033588753 + 1384955677.84592s -- 1 391b0557-dc68-4e40-b6d0-da3033588753 + 1384955689.293082s -- 1 391b0557-dc68-4e40-b6d0-da3033588753 + 1384955894.057476s -- 1 391b0557-dc68-4e40-b6d0-da3033588753 + 1384955910.723021s -- 1 391b0557-dc68-4e40-b6d0-da3033588753 + 1384955914.732132s -- 1 391b0557-dc68-4e40-b6d0-da3033588753 + 1384955968.717875s -- 1 391b0557-dc68-4e40-b6d0-da3033588753 + 1384955969.634658s -- 1 391b0557-dc68-4e40-b6d0-da3033588753 + 1384956004.284925s -- 1 391b0557-dc68-4e40-b6d0-da3033588753 + 1384956029.567195s -- 1 391b0557-dc68-4e40-b6d0-da3033588753 + 1384956188.628995s -- 1 391b0557-dc68-4e40-b6d0-da3033588753 + 1384956379.844701s -- 1 391b0557-dc68-4e40-b6d0-da3033588753 + 1384956381.613833s -- 1 391b0557-dc68-4e40-b6d0-da3033588753 + 1384956387.923418s -- 1 391b0557-dc68-4e40-b6d0-da3033588753 + 1384956395.418701s -- 1 391b0557-dc68-4e40-b6d0-da3033588753 + 1384956408.792928s -- 1 391b0557-dc68-4e40-b6d0-da3033588753 + 1384956504.019733s -- 1 391b0557-dc68-4e40-b6d0-da3033588753 + 1384956519.578085s -- 1 391b0557-dc68-4e40-b6d0-da3033588753 + 1384956524.419783s -- 1 391b0557-dc68-4e40-b6d0-da3033588753 + 1384965891.562742s -- 1 0ab193eb-0c76-4559-a93c-2e30ed8630a8 + 1384965891.815119s -- 0 391b0557-dc68-4e40-b6d0-da3033588753 + 1384965903.355602s -- 1 0ab193eb-0c76-4559-a93c-2e30ed8630a8 + 1384965905.276128s -- 0 391b0557-dc68-4e40-b6d0-da3033588753 + 1384965978.806653s -- 1 0ab193eb-0c76-4559-a93c-2e30ed8630a8 + 1384965979.393089s -- 0 391b0557-dc68-4e40-b6d0-da3033588753 + 1384966097.495566s -- 1 0ab193eb-0c76-4559-a93c-2e30ed8630a8 + 1384966097.704474s -- 0 391b0557-dc68-4e40-b6d0-da3033588753 + 1384966154.97658s -- 1 0ab193eb-0c76-4559-a93c-2e30ed8630a8 + 1384966156.967406s -- 0 391b0557-dc68-4e40-b6d0-da3033588753 + 1384966233.310488s -- 1 0ab193eb-0c76-4559-a93c-2e30ed8630a8 + 1384966233.522324s -- 0 391b0557-dc68-4e40-b6d0-da3033588753 + 1384966241.284523s -- 1 0ab193eb-0c76-4559-a93c-2e30ed8630a8 + 1384966241.475381s -- 0 391b0557-dc68-4e40-b6d0-da3033588753 + 1384966301.688497s -- 1 0ab193eb-0c76-4559-a93c-2e30ed8630a8 + 1384966303.427685s -- 0 391b0557-dc68-4e40-b6d0-da3033588753 + 1384966392.875983s -- 1 0ab193eb-0c76-4559-a93c-2e30ed8630a8 + 1384966393.38718s -- 0 391b0557-dc68-4e40-b6d0-da3033588753 + 1384966404.708568s -- 1 0ab193eb-0c76-4559-a93c-2e30ed8630a8 + 1384966406.441164s -- 0 391b0557-dc68-4e40-b6d0-da3033588753 + 1384966553.557387s -- 1 0ab193eb-0c76-4559-a93c-2e30ed8630a8 + 1384966555.752786s -- 0 391b0557-dc68-4e40-b6d0-da3033588753 + 1384966653.725847s -- 1 0ab193eb-0c76-4559-a93c-2e30ed8630a8 + 1384966654.23288s -- 0 391b0557-dc68-4e40-b6d0-da3033588753 + 1384966695.201885s -- 1 0ab193eb-0c76-4559-a93c-2e30ed8630a8 + 1384966695.689398s -- 0 391b0557-dc68-4e40-b6d0-da3033588753 + 1384966784.556877s -- 1 0ab193eb-0c76-4559-a93c-2e30ed8630a8 + 1384966786.574886s -- 0 391b0557-dc68-4e40-b6d0-da3033588753 + 1384966791.446852s -- 1 0ab193eb-0c76-4559-a93c-2e30ed8630a8 + 1384966793.218318s -- 0 391b0557-dc68-4e40-b6d0-da3033588753 + 1384966884.335685s -- 1 0ab193eb-0c76-4559-a93c-2e30ed8630a8 + 1384966886.147083s -- 0 391b0557-dc68-4e40-b6d0-da3033588753 + 1384967054.857465s -- 1 0ab193eb-0c76-4559-a93c-2e30ed8630a8 + 1384967055.158871s -- 0 391b0557-dc68-4e40-b6d0-da3033588753 + 1384967190.980027s -- 1 0ab193eb-0c76-4559-a93c-2e30ed8630a8 + 1384967193.176584s -- 0 391b0557-dc68-4e40-b6d0-da3033588753 + 1384967328.93796s -- 1 0ab193eb-0c76-4559-a93c-2e30ed8630a8 + 1384967330.428095s -- 0 391b0557-dc68-4e40-b6d0-da3033588753 + 1384967526.127311s -- 1 0ab193eb-0c76-4559-a93c-2e30ed8630a8 + 1384967526.588491s -- 0 391b0557-dc68-4e40-b6d0-da3033588753 + 1384967627.132549s -- 1 0ab193eb-0c76-4559-a93c-2e30ed8630a8 + 1384967627.685201s -- 0 391b0557-dc68-4e40-b6d0-da3033588753 + 1384967686.283694s -- 1 0ab193eb-0c76-4559-a93c-2e30ed8630a8 + 1384967686.728086s -- 0 391b0557-dc68-4e40-b6d0-da3033588753 + 1384967768.270887s -- 1 0ab193eb-0c76-4559-a93c-2e30ed8630a8 + 1384967768.58402s -- 0 391b0557-dc68-4e40-b6d0-da3033588753 + 1384967769.245615s -- 1 0ab193eb-0c76-4559-a93c-2e30ed8630a8 + 1384967771.122238s -- 0 391b0557-dc68-4e40-b6d0-da3033588753 + 1384967813.8197s -- 1 0ab193eb-0c76-4559-a93c-2e30ed8630a8 + 1384967814.168477s -- 0 391b0557-dc68-4e40-b6d0-da3033588753 + 1384967915.243469s -- 1 0ab193eb-0c76-4559-a93c-2e30ed8630a8 + 1384967917.020051s -- 0 391b0557-dc68-4e40-b6d0-da3033588753 + 1384968031.757775s -- 1 0ab193eb-0c76-4559-a93c-2e30ed8630a8 + 1384968032.190452s -- 0 391b0557-dc68-4e40-b6d0-da3033588753 + 1384968035.733635s -- 1 0ab193eb-0c76-4559-a93c-2e30ed8630a8 + 1384968036.03299s -- 0 391b0557-dc68-4e40-b6d0-da3033588753 + 1384968144.555556s -- 1 0ab193eb-0c76-4559-a93c-2e30ed8630a8 + 1384968144.714535s -- 0 391b0557-dc68-4e40-b6d0-da3033588753 + 1384968150.090148s -- 1 0ab193eb-0c76-4559-a93c-2e30ed8630a8 + 1384968150.820567s -- 0 391b0557-dc68-4e40-b6d0-da3033588753 + 1384968304.393177s -- 1 0ab193eb-0c76-4559-a93c-2e30ed8630a8 + 1384968304.613624s -- 0 391b0557-dc68-4e40-b6d0-da3033588753 + 1384968604.499519s -- 1 0ab193eb-0c76-4559-a93c-2e30ed8630a8 + 1384968604.813256s -- 0 391b0557-dc68-4e40-b6d0-da3033588753 + 1384968702.566939s -- 1 0ab193eb-0c76-4559-a93c-2e30ed8630a8 + 1384968704.427767s -- 0 391b0557-dc68-4e40-b6d0-da3033588753 + 1384968725.375289s -- 1 0ab193eb-0c76-4559-a93c-2e30ed8630a8 + 1384968725.939271s -- 0 391b0557-dc68-4e40-b6d0-da3033588753 + 1384968798.402904s -- 1 0ab193eb-0c76-4559-a93c-2e30ed8630a8 + 1384968798.659754s -- 0 391b0557-dc68-4e40-b6d0-da3033588753 + 1384969055.285004s -- 1 0ab193eb-0c76-4559-a93c-2e30ed8630a8 + 1384969055.715448s -- 0 391b0557-dc68-4e40-b6d0-da3033588753 + 1384969159.885115s -- 1 0ab193eb-0c76-4559-a93c-2e30ed8630a8 + 1384969162.382266s -- 0 391b0557-dc68-4e40-b6d0-da3033588753 + 1384969184.633052s -- 1 0ab193eb-0c76-4559-a93c-2e30ed8630a8 + 1384969185.413769s -- 0 391b0557-dc68-4e40-b6d0-da3033588753 + 1384969374.791849s -- 1 0ab193eb-0c76-4559-a93c-2e30ed8630a8 + 1384969377.497842s -- 0 391b0557-dc68-4e40-b6d0-da3033588753 + 1384969475.469111s -- 1 0ab193eb-0c76-4559-a93c-2e30ed8630a8 + 1384969489.697737s -- 0 391b0557-dc68-4e40-b6d0-da3033588753 + 1384969492.087023s -- 1 0ab193eb-0c76-4559-a93c-2e30ed8630a8 + 1384969492.58214s -- 0 391b0557-dc68-4e40-b6d0-da3033588753 + 1384969784.725195s -- 1 0ab193eb-0c76-4559-a93c-2e30ed8630a8 + 1384969786.49773s -- 0 391b0557-dc68-4e40-b6d0-da3033588753 + 1384969814.984624s -- 1 0ab193eb-0c76-4559-a93c-2e30ed8630a8 + 1384969815.397676s -- 0 391b0557-dc68-4e40-b6d0-da3033588753 + 391b0557-dc68-4e40-b6d0-da3033588753 -- here (client) + 668ef9d8-68c6-484e-89e5-06634d590a11 -- rsync.net_datadir_annex (transfer) + b0d3c000-0ac9-4a05-aef4-47f826d5c759 -- user.name (client) + + +# End of transcript or log. +"""]] diff --git a/doc/bugs/Assistant_has_created_155_semitrusted_repositories/comment_10_c5508b476fc48e7a0002b6ecb8d5eac0._comment b/doc/bugs/Assistant_has_created_155_semitrusted_repositories/comment_10_c5508b476fc48e7a0002b6ecb8d5eac0._comment new file mode 100644 index 0000000000..77f19b15b8 --- /dev/null +++ b/doc/bugs/Assistant_has_created_155_semitrusted_repositories/comment_10_c5508b476fc48e7a0002b6ecb8d5eac0._comment @@ -0,0 +1,18 @@ +[[!comment format=mdwn + username="http://joeyh.name/" + ip="209.250.56.64" + subject="comment 10" + date="2013-12-03T18:05:11Z" + content=""" +It's normal for the git-annex assistant to keep a couple of long-running git cat-file and git check-attr processes running at all times. So I'd not worry about that. + +I received the emailed tarball of uuid.log versions. (message-id CAL5AKQq9i4QwReWHZkKOm1Ggon66FqFkjbM832PP1drgiR3x5g@mail.gmail.com) + +Can you show the command you used to dump those files? Because the contents of them is really quite weird. + +Take file 0, which I'm guessing might be the first version of the uuid.log. It starts off with what I'd expect to see in a uuid.log, but then has a section of what looks like trust.log data, followed by what is certianly supposed to be remote.log data, followed by what looks like `git show 966ec7aaf408dffe0d6e6ce04aebc15dc5b26f5b`, a commit which apparently was made by \"Firstname.Lastname@somedomain.com\" (unless you anonymized that) and added a file `test` to the git repository. + +It gets worse, some of the versions of the uuid.log file appear to be megabytes in size, which is just insane, and these contain more such apparent `git show` of git commits adding files to the repository. + +The general pattern of uuid.log data +trust.log data + remote.log data + git show of a commit seems to repeat in each version of the uuid.log file. +"""]] diff --git a/doc/bugs/Assistant_has_created_155_semitrusted_repositories/comment_11_cb49edcc8a13928c171a2acdde32dce9._comment b/doc/bugs/Assistant_has_created_155_semitrusted_repositories/comment_11_cb49edcc8a13928c171a2acdde32dce9._comment new file mode 100644 index 0000000000..74a30886c6 --- /dev/null +++ b/doc/bugs/Assistant_has_created_155_semitrusted_repositories/comment_11_cb49edcc8a13928c171a2acdde32dce9._comment @@ -0,0 +1,12 @@ +[[!comment format=mdwn + username="https://www.google.com/accounts/o8/id?id=AItOawnNqLKszWk9EoD4CDCqNXJRIklKFBCN1Ao" + nickname="maurizio" + subject="comment 11" + date="2013-12-03T18:32:59Z" + content=""" +The files I sent you were collected with: + +```for i in $(seq 0 52); do git show git-annex:uuid.log @{$i} > ~/tmp/uuid.logs/$i; done``` + + +"""]] diff --git a/doc/bugs/Assistant_has_created_155_semitrusted_repositories/comment_12_a86c8347526e7b0a6f7633f3aea528bb._comment b/doc/bugs/Assistant_has_created_155_semitrusted_repositories/comment_12_a86c8347526e7b0a6f7633f3aea528bb._comment new file mode 100644 index 0000000000..4a92b1f8a4 --- /dev/null +++ b/doc/bugs/Assistant_has_created_155_semitrusted_repositories/comment_12_a86c8347526e7b0a6f7633f3aea528bb._comment @@ -0,0 +1,8 @@ +[[!comment format=mdwn + username="https://www.google.com/accounts/o8/id?id=AItOawnNqLKszWk9EoD4CDCqNXJRIklKFBCN1Ao" + nickname="maurizio" + subject="comment 12" + date="2013-12-03T18:38:45Z" + content=""" +and yes: as I wrote in email my name and the domain of email address are private and were edited away (sed firstname, lastname, domain except gmail which is used for xmpp). So the \"Firstname.Lastname@somedomain.com\" has replaced my real professional email address. +"""]] diff --git a/doc/bugs/Assistant_has_created_155_semitrusted_repositories/comment_13_a26c2b49ee7746be06f4772aa838d5aa._comment b/doc/bugs/Assistant_has_created_155_semitrusted_repositories/comment_13_a26c2b49ee7746be06f4772aa838d5aa._comment new file mode 100644 index 0000000000..e4d552ad7d --- /dev/null +++ b/doc/bugs/Assistant_has_created_155_semitrusted_repositories/comment_13_a26c2b49ee7746be06f4772aa838d5aa._comment @@ -0,0 +1,21 @@ +[[!comment format=mdwn + username="http://joeyh.name/" + ip="209.250.56.64" + subject="comment 13" + date="2013-12-03T20:23:43Z" + content=""" +Ok, that explains the weird results, because I get the same crazy data dumped when I try eg `git show git-annex:uuid.log @{0}` in a repository that is completely ok. + +The @{0} causes git show to show some commit. Not the commit that changed uuid.log however. So it first shows the uuid.log (always the most recent version), and then some random commit. + +This will work: + +
    +git clone myrepo myrepo.tmp
+cd myrepo.tmp
+git checkout git-annex
+mkdir logs
+for r in $(git log --pretty='%H' uuid.log); do git show $r > logs/$r; done
+git log uuid.log > logs/gitlog
+
    +"""]] diff --git a/doc/bugs/Assistant_has_created_155_semitrusted_repositories/comment_14_b958da97a69091d283918e0d5a658da5._comment b/doc/bugs/Assistant_has_created_155_semitrusted_repositories/comment_14_b958da97a69091d283918e0d5a658da5._comment new file mode 100644 index 0000000000..b5b4996695 --- /dev/null +++ b/doc/bugs/Assistant_has_created_155_semitrusted_repositories/comment_14_b958da97a69091d283918e0d5a658da5._comment @@ -0,0 +1,26 @@ +[[!comment format=mdwn + username="https://www.google.com/accounts/o8/id?id=AItOawnNqLKszWk9EoD4CDCqNXJRIklKFBCN1Ao" + nickname="maurizio" + subject="comment 14" + date="2013-12-04T08:47:15Z" + content=""" +I followed the procedure you suggested, please see email for how to transfer it to you. There are several points I would like to note: + +1. In the cloned repository created as you suggest above, the output of ```git annex status``` does not contain all the weird things that appear in the original repository (although it contains other stuff, see 3) +2. In the original repository (~/datadir/Annex), the output of ```git annex status``` still contains the 155 nonrepositories, plus some other stuff (see 3) +3. In the meantime, I created (always with the webapp) another annex for private stuff (~/private/Annex) on the same set of machines. Although I chose to \"keep repositories separated\", the few files that I put for testing into the new ~/private/Annex have appeared as broken symlinks in the work annex ~/datadir/Annex. The transfer repositories used for the work and private annexes are different and are not even hosted on the same machines. The jabber account on the contrary is the same for all annexes. +4. In the webapp (client1) in ~/private/Annex, only the correct repositories appear. +5. In the webapp (client1) in ~/datadir/Annex the correct repositories appear, plus the private repository of client 2 and twice the transfer repository of the ~/private/Annex +6. ```git annex status``` on client1 in ~/datadir/Annex shows the repositories of ~/private/Annex in addition to the 155 nonrepositories and its own repositories +7. Because of 3), you will find at the end of the logs some new stuff that is in principle not related to the original bug report. + +In the end the situation is now the following: + +* 3 annexes (~/Annex for initial testing, ~/datadir/Annex for work, ~/private/Annex for private stuff), all created via webapp and handled by the assistant +* files which were added on client 1 into ~/private/Annex also appear as broken links in ~/datadir/Annex on client 1 but nowhere in client 2 +* files which were added on client 2 into ~/private/Annex also appear as broken links in ~/datadir/Annex on client 1 and in client 2 +* The first created annex still seems to work as expected + +I am sorry that the situation is that messy. Is there a way to separate these repositories that have somehow become entangled? + +"""]] diff --git a/doc/bugs/Assistant_has_created_155_semitrusted_repositories/comment_1_169b24b34cce3f5c8446c2150beb6827._comment b/doc/bugs/Assistant_has_created_155_semitrusted_repositories/comment_1_169b24b34cce3f5c8446c2150beb6827._comment new file mode 100644 index 0000000000..99bb737e4f --- /dev/null +++ b/doc/bugs/Assistant_has_created_155_semitrusted_repositories/comment_1_169b24b34cce3f5c8446c2150beb6827._comment @@ -0,0 +1,8 @@ +[[!comment format=mdwn + username="http://joeyh.name/" + ip="209.250.56.64" + subject="comment 1" + date="2013-11-22T16:54:53Z" + content=""" +Please post the output of `git show git-annex:uuid.log` run in the repository. +"""]] diff --git a/doc/bugs/Assistant_has_created_155_semitrusted_repositories/comment_2_6acd6f38297772a07d8d5fb999bd2eaa._comment b/doc/bugs/Assistant_has_created_155_semitrusted_repositories/comment_2_6acd6f38297772a07d8d5fb999bd2eaa._comment new file mode 100644 index 0000000000..6db2a8fcae --- /dev/null +++ b/doc/bugs/Assistant_has_created_155_semitrusted_repositories/comment_2_6acd6f38297772a07d8d5fb999bd2eaa._comment @@ -0,0 +1,183 @@ +[[!comment format=mdwn + username="https://www.google.com/accounts/o8/id?id=AItOawnNqLKszWk9EoD4CDCqNXJRIklKFBCN1Ao" + nickname="maurizio" + subject="comment 2" + date="2013-11-22T17:23:19Z" + content=""" +Here it is (user name edited): + +[[!format sh \"\"\" + + +0ab193eb-0c76-4559-a93c-2e30ed8630a8 archive timestamp=1384664310.014349s +0ab193eb-0c76-4559-a93c-2e30ed8630a8 datadir on st08 timestamp=1384522780.868451s +0ab193eb-0c76-4559-a93c-2e30ed8630a8 username@st08:~/datadir_annex timestamp=1384540428.076617s +0ab193eb-0c76-4559-a93c-2e30ed8630a8 username@st08:~/datadir_annex timestamp=1384541057.852874s +0ab193eb-0c76-4559-a93c-2e30ed8630a8 username@st08:~/datadir_annex timestamp=1384548609.891111s +1384784127.91222s 1 391b0557-dc68-4e40-b6d0-da3033588753 +1384784164.437824s 1 391b0557-dc68-4e40-b6d0-da3033588753 +1384784176.944372s 1 391b0557-dc68-4e40-b6d0-da3033588753 +1384784179.254498s 1 391b0557-dc68-4e40-b6d0-da3033588753 +1384785147.558938s 1 0ab193eb-0c76-4559-a93c-2e30ed8630a8 +1384785147.717223s 1 0ab193eb-0c76-4559-a93c-2e30ed8630a8 +1384785159.041203s 1 0ab193eb-0c76-4559-a93c-2e30ed8630a8 +1384785159.199504s 1 0ab193eb-0c76-4559-a93c-2e30ed8630a8 +1384785185.79485s 1 0ab193eb-0c76-4559-a93c-2e30ed8630a8 +1384785187.318128s 1 0ab193eb-0c76-4559-a93c-2e30ed8630a8 +1384785215.236504s 1 0ab193eb-0c76-4559-a93c-2e30ed8630a8 +1384785215.389096s 1 0ab193eb-0c76-4559-a93c-2e30ed8630a8 +1384785313.539843s 1 0ab193eb-0c76-4559-a93c-2e30ed8630a8 +1384785313.701305s 1 0ab193eb-0c76-4559-a93c-2e30ed8630a8 +1384785315.596206s 1 0ab193eb-0c76-4559-a93c-2e30ed8630a8 +1384785344.184461s 1 0ab193eb-0c76-4559-a93c-2e30ed8630a8 +1384785348.192805s 1 0ab193eb-0c76-4559-a93c-2e30ed8630a8 +1384785402.70316s 1 0ab193eb-0c76-4559-a93c-2e30ed8630a8 +1384785406.524044s 1 0ab193eb-0c76-4559-a93c-2e30ed8630a8 +1384785446.074236s 1 0ab193eb-0c76-4559-a93c-2e30ed8630a8 +1384873605.313126s 1 391b0557-dc68-4e40-b6d0-da3033588753 +1384873697.029999s 1 391b0557-dc68-4e40-b6d0-da3033588753 +1384873761.687234s 1 391b0557-dc68-4e40-b6d0-da3033588753 +1384873774.608376s 1 391b0557-dc68-4e40-b6d0-da3033588753 +1384926279.456728s 0 391b0557-dc68-4e40-b6d0-da3033588753 +1384926368.736s 1 391b0557-dc68-4e40-b6d0-da3033588753 +1384926454.99433s 1 391b0557-dc68-4e40-b6d0-da3033588753 +1384926494.152645s 1 391b0557-dc68-4e40-b6d0-da3033588753 +1384926504.438232s 1 391b0557-dc68-4e40-b6d0-da3033588753 +1384934790.89717s 1 391b0557-dc68-4e40-b6d0-da3033588753 +1384934848.757067s 1 391b0557-dc68-4e40-b6d0-da3033588753 +1384934899.087168s 1 391b0557-dc68-4e40-b6d0-da3033588753 +1384934908.238587s 1 391b0557-dc68-4e40-b6d0-da3033588753 +1384948772.14552s 1 391b0557-dc68-4e40-b6d0-da3033588753 +1384948805.441196s 1 391b0557-dc68-4e40-b6d0-da3033588753 +1384948813.397132s 1 391b0557-dc68-4e40-b6d0-da3033588753 +1384948921.45481s 1 391b0557-dc68-4e40-b6d0-da3033588753 +1384948924.855852s 1 391b0557-dc68-4e40-b6d0-da3033588753 +1384949073.988946s 1 391b0557-dc68-4e40-b6d0-da3033588753 +1384949082.298976s 1 391b0557-dc68-4e40-b6d0-da3033588753 +1384949399.608138s 1 391b0557-dc68-4e40-b6d0-da3033588753 +1384949581.12213s 1 391b0557-dc68-4e40-b6d0-da3033588753 +1384949583.9923s 1 391b0557-dc68-4e40-b6d0-da3033588753 +1384949700.22807s 1 391b0557-dc68-4e40-b6d0-da3033588753 +1384949765.484768s 1 391b0557-dc68-4e40-b6d0-da3033588753 +1384955202.85962s 1 391b0557-dc68-4e40-b6d0-da3033588753 +1384955230.953995s 1 391b0557-dc68-4e40-b6d0-da3033588753 +1384955402.534938s 1 391b0557-dc68-4e40-b6d0-da3033588753 +1384955457.1885s 1 391b0557-dc68-4e40-b6d0-da3033588753 +1384955524.603709s 1 391b0557-dc68-4e40-b6d0-da3033588753 +1384955611.891061s 1 391b0557-dc68-4e40-b6d0-da3033588753 +1384955677.84592s 1 391b0557-dc68-4e40-b6d0-da3033588753 +1384955689.293082s 1 391b0557-dc68-4e40-b6d0-da3033588753 +1384955894.057476s 1 391b0557-dc68-4e40-b6d0-da3033588753 +1384955910.723021s 1 391b0557-dc68-4e40-b6d0-da3033588753 +1384955914.732132s 1 391b0557-dc68-4e40-b6d0-da3033588753 +1384955968.717875s 1 391b0557-dc68-4e40-b6d0-da3033588753 +1384955969.634658s 1 391b0557-dc68-4e40-b6d0-da3033588753 +1384956004.284925s 1 391b0557-dc68-4e40-b6d0-da3033588753 +1384956029.567195s 1 391b0557-dc68-4e40-b6d0-da3033588753 +1384956188.628995s 1 391b0557-dc68-4e40-b6d0-da3033588753 +1384956379.844701s 1 391b0557-dc68-4e40-b6d0-da3033588753 +1384956381.613833s 1 391b0557-dc68-4e40-b6d0-da3033588753 +1384956387.923418s 1 391b0557-dc68-4e40-b6d0-da3033588753 +1384956395.418701s 1 391b0557-dc68-4e40-b6d0-da3033588753 +1384956408.792928s 1 391b0557-dc68-4e40-b6d0-da3033588753 +1384956504.019733s 1 391b0557-dc68-4e40-b6d0-da3033588753 +1384956519.578085s 1 391b0557-dc68-4e40-b6d0-da3033588753 +1384956524.419783s 1 391b0557-dc68-4e40-b6d0-da3033588753 +1384965891.562742s 1 0ab193eb-0c76-4559-a93c-2e30ed8630a8 +1384965891.815119s 0 391b0557-dc68-4e40-b6d0-da3033588753 +1384965903.355602s 1 0ab193eb-0c76-4559-a93c-2e30ed8630a8 +1384965905.276128s 0 391b0557-dc68-4e40-b6d0-da3033588753 +1384965978.806653s 1 0ab193eb-0c76-4559-a93c-2e30ed8630a8 +1384965979.393089s 0 391b0557-dc68-4e40-b6d0-da3033588753 +1384966097.495566s 1 0ab193eb-0c76-4559-a93c-2e30ed8630a8 +1384966097.704474s 0 391b0557-dc68-4e40-b6d0-da3033588753 +1384966154.97658s 1 0ab193eb-0c76-4559-a93c-2e30ed8630a8 +1384966156.967406s 0 391b0557-dc68-4e40-b6d0-da3033588753 +1384966233.310488s 1 0ab193eb-0c76-4559-a93c-2e30ed8630a8 +1384966233.522324s 0 391b0557-dc68-4e40-b6d0-da3033588753 +1384966241.284523s 1 0ab193eb-0c76-4559-a93c-2e30ed8630a8 +1384966241.475381s 0 391b0557-dc68-4e40-b6d0-da3033588753 +1384966301.688497s 1 0ab193eb-0c76-4559-a93c-2e30ed8630a8 +1384966303.427685s 0 391b0557-dc68-4e40-b6d0-da3033588753 +1384966392.875983s 1 0ab193eb-0c76-4559-a93c-2e30ed8630a8 +1384966393.38718s 0 391b0557-dc68-4e40-b6d0-da3033588753 +1384966404.708568s 1 0ab193eb-0c76-4559-a93c-2e30ed8630a8 +1384966406.441164s 0 391b0557-dc68-4e40-b6d0-da3033588753 +1384966553.557387s 1 0ab193eb-0c76-4559-a93c-2e30ed8630a8 +1384966555.752786s 0 391b0557-dc68-4e40-b6d0-da3033588753 +1384966653.725847s 1 0ab193eb-0c76-4559-a93c-2e30ed8630a8 +1384966654.23288s 0 391b0557-dc68-4e40-b6d0-da3033588753 +1384966695.201885s 1 0ab193eb-0c76-4559-a93c-2e30ed8630a8 +1384966695.689398s 0 391b0557-dc68-4e40-b6d0-da3033588753 +1384966784.556877s 1 0ab193eb-0c76-4559-a93c-2e30ed8630a8 +1384966786.574886s 0 391b0557-dc68-4e40-b6d0-da3033588753 +1384966791.446852s 1 0ab193eb-0c76-4559-a93c-2e30ed8630a8 +1384966793.218318s 0 391b0557-dc68-4e40-b6d0-da3033588753 +1384966884.335685s 1 0ab193eb-0c76-4559-a93c-2e30ed8630a8 +1384966886.147083s 0 391b0557-dc68-4e40-b6d0-da3033588753 +1384967054.857465s 1 0ab193eb-0c76-4559-a93c-2e30ed8630a8 +1384967055.158871s 0 391b0557-dc68-4e40-b6d0-da3033588753 +1384967190.980027s 1 0ab193eb-0c76-4559-a93c-2e30ed8630a8 +1384967193.176584s 0 391b0557-dc68-4e40-b6d0-da3033588753 +1384967328.93796s 1 0ab193eb-0c76-4559-a93c-2e30ed8630a8 +1384967330.428095s 0 391b0557-dc68-4e40-b6d0-da3033588753 +1384967526.127311s 1 0ab193eb-0c76-4559-a93c-2e30ed8630a8 +1384967526.588491s 0 391b0557-dc68-4e40-b6d0-da3033588753 +1384967627.132549s 1 0ab193eb-0c76-4559-a93c-2e30ed8630a8 +1384967627.685201s 0 391b0557-dc68-4e40-b6d0-da3033588753 +1384967686.283694s 1 0ab193eb-0c76-4559-a93c-2e30ed8630a8 +1384967686.728086s 0 391b0557-dc68-4e40-b6d0-da3033588753 +1384967768.270887s 1 0ab193eb-0c76-4559-a93c-2e30ed8630a8 +1384967768.58402s 0 391b0557-dc68-4e40-b6d0-da3033588753 +1384967769.245615s 1 0ab193eb-0c76-4559-a93c-2e30ed8630a8 +1384967771.122238s 0 391b0557-dc68-4e40-b6d0-da3033588753 +1384967813.8197s 1 0ab193eb-0c76-4559-a93c-2e30ed8630a8 +1384967814.168477s 0 391b0557-dc68-4e40-b6d0-da3033588753 +1384967915.243469s 1 0ab193eb-0c76-4559-a93c-2e30ed8630a8 +1384967917.020051s 0 391b0557-dc68-4e40-b6d0-da3033588753 +1384968031.757775s 1 0ab193eb-0c76-4559-a93c-2e30ed8630a8 +1384968032.190452s 0 391b0557-dc68-4e40-b6d0-da3033588753 +1384968035.733635s 1 0ab193eb-0c76-4559-a93c-2e30ed8630a8 +1384968036.03299s 0 391b0557-dc68-4e40-b6d0-da3033588753 +1384968144.555556s 1 0ab193eb-0c76-4559-a93c-2e30ed8630a8 +1384968144.714535s 0 391b0557-dc68-4e40-b6d0-da3033588753 +1384968150.090148s 1 0ab193eb-0c76-4559-a93c-2e30ed8630a8 +1384968150.820567s 0 391b0557-dc68-4e40-b6d0-da3033588753 +1384968304.393177s 1 0ab193eb-0c76-4559-a93c-2e30ed8630a8 +1384968304.613624s 0 391b0557-dc68-4e40-b6d0-da3033588753 +1384968604.499519s 1 0ab193eb-0c76-4559-a93c-2e30ed8630a8 +1384968604.813256s 0 391b0557-dc68-4e40-b6d0-da3033588753 +1384968702.566939s 1 0ab193eb-0c76-4559-a93c-2e30ed8630a8 +1384968704.427767s 0 391b0557-dc68-4e40-b6d0-da3033588753 +1384968725.375289s 1 0ab193eb-0c76-4559-a93c-2e30ed8630a8 +1384968725.939271s 0 391b0557-dc68-4e40-b6d0-da3033588753 +1384968798.402904s 1 0ab193eb-0c76-4559-a93c-2e30ed8630a8 +1384968798.659754s 0 391b0557-dc68-4e40-b6d0-da3033588753 +1384969055.285004s 1 0ab193eb-0c76-4559-a93c-2e30ed8630a8 +1384969055.715448s 0 391b0557-dc68-4e40-b6d0-da3033588753 +1384969159.885115s 1 0ab193eb-0c76-4559-a93c-2e30ed8630a8 +1384969162.382266s 0 391b0557-dc68-4e40-b6d0-da3033588753 +1384969184.633052s 1 0ab193eb-0c76-4559-a93c-2e30ed8630a8 +1384969185.413769s 0 391b0557-dc68-4e40-b6d0-da3033588753 +1384969374.791849s 1 0ab193eb-0c76-4559-a93c-2e30ed8630a8 +1384969377.497842s 0 391b0557-dc68-4e40-b6d0-da3033588753 +1384969475.469111s 1 0ab193eb-0c76-4559-a93c-2e30ed8630a8 +1384969489.697737s 0 391b0557-dc68-4e40-b6d0-da3033588753 +1384969492.087023s 1 0ab193eb-0c76-4559-a93c-2e30ed8630a8 +1384969492.58214s 0 391b0557-dc68-4e40-b6d0-da3033588753 +1384969784.725195s 1 0ab193eb-0c76-4559-a93c-2e30ed8630a8 +1384969786.49773s 0 391b0557-dc68-4e40-b6d0-da3033588753 +1384969814.984624s 1 0ab193eb-0c76-4559-a93c-2e30ed8630a8 +1384969815.397676s 0 391b0557-dc68-4e40-b6d0-da3033588753 +391b0557-dc68-4e40-b6d0-da3033588753 client timestamp=1384528843.958836s +391b0557-dc68-4e40-b6d0-da3033588753 username@big:~/datadir/Annex timestamp=1384521164.194035s +668ef9d8-68c6-484e-89e5-06634d590a11 cipher=dFpZLzI4M3ZqVUl1S0p3ajl5N25vUUJyTFJXand5T2pNSGliOGo5WXBkN2NNY3ZqR3pvNmN0L2tLOFFKZW1aWDdHYUpCMGF1UGxybk9zeUdQOEpsMGZ5RzgwTVo1S2N6Rlo1eTVnNXFGYjJmblV0bzRjT3lTb29uZnY1QmpwMVVwYlZEOGVlaVlNR0R5dTcxNW50TFcrU1dSVi9PUXk5RkR6Zm14UFJzZEEvOHB5MTZXbkp5TW13Qm5UZ2FkUEsrUTdVZ2cxVWZnai9mbGRDYWVXYlpDVzFQL2tzanNTMU16aGxuVktETXpOY04vR1lSZnEzNVdJSkpDUEtMV1pPWXczVUVNTEJCNGtmUldiTVZoWWdlYm9NTTN4T2c1RXRvTXEzTytXazNaaFNKeUtsSlNmOUFXdTIrYnZLbkgvSFljSFVPOVlid0lCSHFlSDk0ZVQ3M3E4eHR6ckM1SjdIbjF0dmRSTGVibFFSdzUrdnVMM3p6UDl5R0JOUWNhL3Q4MXVVVGF0RjFjNGoyditZUGRzOUEraHZ5Z29hNkk0SDNiaVdYVnJFQmxTRTdTd3hOcjR3amJzRkd3d1VEUXFFZDhLNTBhWGF6bUY5LzZCZ2F5U1lOZEFDdXdSU2pMOU9qVGJWM29wVGNhNEt1Z09jU1JXRnQ0QUhoZXMxZlpvN1Qzc1o4WTQ3S1dUNVdJM3FjK05aQm5kUk10Nm8zSG1Ccys3cFdDUWVaUGJCbUhwK2ozdGw0cmxhK3FydkF5b3ZOM0xmbEJKdVBpTHNxd3JSUEpXdndkSENzQjlteHovZW9JOUtWUGJKNGxoRExSRVloNDUrNFZ6YmFZQVNZdkcxV2JxSzZTc2ZING90cmRBREZtTFRlRnc3OW92ZXBxQ0pPa21UMFFuR0ZJNG89Cg== encryption=shared name=rsync.net_datadir_annex rsyncurl=rsusername@git-annex-ch-s011.rsync.net-rsusername_datadir_annex.2F:datadir_annex/ type=rsync timestamp=1384948273.302009s +668ef9d8-68c6-484e-89e5-06634d590a11 cipher=dFpZLzI4M3ZqVUl1S0p3ajl5N25vUUJyTFJXand5T2pNSGliOGo5WXBkN2NNY3ZqR3pvNmN0L2tLOFFKZW1aWDdHYUpCMGF1UGxybk9zeUdQOEpsMGZ5RzgwTVo1S2N6Rlo1eTVnNXFGYjJmblV0bzRjT3lTb29uZnY1QmpwMVVwYlZEOGVlaVlNR0R5dTcxNW50TFcrU1dSVi9PUXk5RkR6Zm14UFJzZEEvOHB5MTZXbkp5TW13Qm5UZ2FkUEsrUTdVZ2cxVWZnai9mbGRDYWVXYlpDVzFQL2tzanNTMU16aGxuVktETXpOY04vR1lSZnEzNVdJSkpDUEtMV1pPWXczVUVNTEJCNGtmUldiTVZoWWdlYm9NTTN4T2c1RXRvTXEzTytXazNaaFNKeUtsSlNmOUFXdTIrYnZLbkgvSFljSFVPOVlid0lCSHFlSDk0ZVQ3M3E4eHR6ckM1SjdIbjF0dmRSTGVibFFSdzUrdnVMM3p6UDl5R0JOUWNhL3Q4MXVVVGF0RjFjNGoyditZUGRzOUEraHZ5Z29hNkk0SDNiaVdYVnJFQmxTRTdTd3hOcjR3amJzRkd3d1VEUXFFZDhLNTBhWGF6bUY5LzZCZ2F5U1lOZEFDdXdSU2pMOU9qVGJWM29wVGNhNEt1Z09jU1JXRnQ0QUhoZXMxZlpvN1Qzc1o4WTQ3S1dUNVdJM3FjK05aQm5kUk10Nm8zSG1Ccys3cFdDUWVaUGJCbUhwK2ozdGw0cmxhK3FydkF5b3ZOM0xmbEJKdVBpTHNxd3JSUEpXdndkSENzQjlteHovZW9JOUtWUGJKNGxoRExSRVloNDUrNFZ6YmFZQVNZdkcxV2JxSzZTc2ZING90cmRBREZtTFRlRnc3OW92ZXBxQ0pPa21UMFFuR0ZJNG89Cg== encryption=shared name=rsync.net_datadir_annex rsyncurl=rsusername@git-annex-ch-s011.rsync.net-rsusername_datadir_annex:datadir_annex/ type=rsync timestamp=1384547300.727425s +668ef9d8-68c6-484e-89e5-06634d590a11 rsync.net_datadir_annex timestamp=1384547300.725734s +668ef9d8-68c6-484e-89e5-06634d590a11 rsync.net_datadir_annex timestamp=1384948273.280063s +668ef9d8-68c6-484e-89e5-06634d590a11 transfer timestamp=1384948273.352386s +b0d3c000-0ac9-4a05-aef4-47f826d5c759 client timestamp=1384575989.965159s +b0d3c000-0ac9-4a05-aef4-47f826d5c759 username@mezzo:~/datadir/Annex timestamp=1384547220.353298s + +\"\"\"]] +"""]] diff --git a/doc/bugs/Assistant_has_created_155_semitrusted_repositories/comment_3_6a4118e5c5fbe5e84d27094ac72b741b._comment b/doc/bugs/Assistant_has_created_155_semitrusted_repositories/comment_3_6a4118e5c5fbe5e84d27094ac72b741b._comment new file mode 100644 index 0000000000..2d31d1a7a6 --- /dev/null +++ b/doc/bugs/Assistant_has_created_155_semitrusted_repositories/comment_3_6a4118e5c5fbe5e84d27094ac72b741b._comment @@ -0,0 +1,22 @@ +[[!comment format=mdwn + username="http://joeyh.name/" + ip="209.250.56.64" + subject="comment 3" + date="2013-11-22T17:47:30Z" + content=""" +Any chance I could get a copy of this git repository? (Excluding the .git/annex part) + +Feel free to email me id@joeyh.name to arrange a secure transfer. + +Otherwise, I need to see what the git-annex:uuid.log file looked like before this happened to it. The corruption may have occurred progressively in several commits, or all at once. + +You might also still have some logs in `.git/annex/daemon.log*`, and sending those might help, assuming the strange messages you mentioned are logged in there. + +> 1384785215.389096s 1 0ab193eb-0c76-4559-a93c-2e30ed8630a8 + +This is pretty weird thing to be in the uuid.log. The \"1\" makes me think this might be a scrambled version of what's normally stored in the trust.log: + +> 511f4722-63d5-11e1-8b26-1bb951ea9f7b 1 timestamp=1330630786.417597s + +This seems to be the same bug as [[bugs/non-repos in repositories list (+ other weird output) from git annex status]] +"""]] diff --git a/doc/bugs/Assistant_has_created_155_semitrusted_repositories/comment_4_04daa20d5d7c74bb34ec48e752ed9fe8._comment b/doc/bugs/Assistant_has_created_155_semitrusted_repositories/comment_4_04daa20d5d7c74bb34ec48e752ed9fe8._comment new file mode 100644 index 0000000000..dd037bcc02 --- /dev/null +++ b/doc/bugs/Assistant_has_created_155_semitrusted_repositories/comment_4_04daa20d5d7c74bb34ec48e752ed9fe8._comment @@ -0,0 +1,8 @@ +[[!comment format=mdwn + username="http://joeyh.name/" + ip="209.250.56.64" + subject="comment 4" + date="2013-11-22T17:48:37Z" + content=""" +What hardware are you running git-annex on? +"""]] diff --git a/doc/bugs/Assistant_has_created_155_semitrusted_repositories/comment_5_11af8ab2587e6eeb671051ba8191995b._comment b/doc/bugs/Assistant_has_created_155_semitrusted_repositories/comment_5_11af8ab2587e6eeb671051ba8191995b._comment new file mode 100644 index 0000000000..de9d4fd719 --- /dev/null +++ b/doc/bugs/Assistant_has_created_155_semitrusted_repositories/comment_5_11af8ab2587e6eeb671051ba8191995b._comment @@ -0,0 +1,11 @@ +[[!comment format=mdwn + username="https://www.google.com/accounts/o8/id?id=AItOawnNqLKszWk9EoD4CDCqNXJRIklKFBCN1Ao" + nickname="maurizio" + subject="comment 5" + date="2013-11-22T18:22:55Z" + content=""" +One client is a thinkpad 121e, the other one a hp Z420 workstation. The archive is a basic Dell desktop. All of them run wheezy with git-annex and git from official backports. The transfer repository is at rsync.net. + +The daemon.log file was empty at the time I noticed the situation. Now git-annex has been scanning this repository for some time and the log has information again but it looks a bit too long to be posted here. I will email about the repository copy and the log. + +"""]] diff --git a/doc/bugs/Assistant_has_created_155_semitrusted_repositories/comment_6_26236cdc2bce532017854791bcd727d1._comment b/doc/bugs/Assistant_has_created_155_semitrusted_repositories/comment_6_26236cdc2bce532017854791bcd727d1._comment new file mode 100644 index 0000000000..8b5c80f6bb --- /dev/null +++ b/doc/bugs/Assistant_has_created_155_semitrusted_repositories/comment_6_26236cdc2bce532017854791bcd727d1._comment @@ -0,0 +1,8 @@ +[[!comment format=mdwn + username="https://www.google.com/accounts/o8/id?id=AItOawnNqLKszWk9EoD4CDCqNXJRIklKFBCN1Ao" + nickname="maurizio" + subject="comment 6" + date="2013-11-22T19:29:08Z" + content=""" +Some additional information (useful at least from user's point of view): the data that was in the annex but outside of the archive is still accessible from both clients. Part of the data that was in the archive does not seem to be accessible on one client: ```git annex get``` says it gets data in an amount which seems correct but the link to the data remains broken. Fortunately I can still access the data on the other client. Some of the data in the archive is still accessible in both clients. The part that is still accessible was put in the annex a few days ago, only the last directory which was added and seems to have triggered the problem is \"broken\" on one client. It is possible (not sure, sorry) that this directory was first added in the annex, and then moved to the archive shortly after, before the first addition had propagated to the other client. +"""]] diff --git a/doc/bugs/Assistant_has_created_155_semitrusted_repositories/comment_7_3c532dd5b8a01ecdeda1300b49aba675._comment b/doc/bugs/Assistant_has_created_155_semitrusted_repositories/comment_7_3c532dd5b8a01ecdeda1300b49aba675._comment new file mode 100644 index 0000000000..00abc500bd --- /dev/null +++ b/doc/bugs/Assistant_has_created_155_semitrusted_repositories/comment_7_3c532dd5b8a01ecdeda1300b49aba675._comment @@ -0,0 +1,12 @@ +[[!comment format=mdwn + username="https://www.google.com/accounts/o8/id?id=AItOawnNqLKszWk9EoD4CDCqNXJRIklKFBCN1Ao" + nickname="maurizio" + subject="comment 7" + date="2013-11-23T13:16:10Z" + content=""" +OK, sorry but it will be difficult to provide you with a copy of this repository. I hope I can still help in spite of being only a very novice git user. I checked that the git-annex:uuid.log file at the first commit was identical to the one above. The only difference is that in this first commit file there is a mention at the bottom of who created the file (me@somedomain.com) with the mention \"created repository\". + +So it is very possible that these nonexisting remotes are present since day 1. It is possible that I did not notice it earlier (I am not sure I had ever run git annex status in this repository since I am looking for a dropbox-like experience). What triggered my curiosity was the difficulty I had to get some data. At that point it might be that there are two independent problems. One with these nonexisting repositories, and another one with the fact that I cannot get some data in this client. + +Would it be useful to you to have the full history (52 revisions) of the git-annex:uuid.log file? +"""]] diff --git a/doc/bugs/Assistant_has_created_155_semitrusted_repositories/comment_8_119142c5ebc499f0ee0926dbca265308._comment b/doc/bugs/Assistant_has_created_155_semitrusted_repositories/comment_8_119142c5ebc499f0ee0926dbca265308._comment new file mode 100644 index 0000000000..d50964f64c --- /dev/null +++ b/doc/bugs/Assistant_has_created_155_semitrusted_repositories/comment_8_119142c5ebc499f0ee0926dbca265308._comment @@ -0,0 +1,10 @@ +[[!comment format=mdwn + username="http://joeyh.name/" + ip="209.250.56.64" + subject="comment 8" + date="2013-11-26T16:34:13Z" + content=""" +Yes, it would be helpful to have the full history of the file. + +You might try running `git annex fsck` on the file in the client that it says it has gotten but that you cannot access. +"""]] diff --git a/doc/bugs/Assistant_has_created_155_semitrusted_repositories/comment_9_0651071ee1654eeaa9aa9369873fdf6a._comment b/doc/bugs/Assistant_has_created_155_semitrusted_repositories/comment_9_0651071ee1654eeaa9aa9369873fdf6a._comment new file mode 100644 index 0000000000..863c5b53d4 --- /dev/null +++ b/doc/bugs/Assistant_has_created_155_semitrusted_repositories/comment_9_0651071ee1654eeaa9aa9369873fdf6a._comment @@ -0,0 +1,8 @@ +[[!comment format=mdwn + username="https://www.google.com/accounts/o8/id?id=AItOawnNqLKszWk9EoD4CDCqNXJRIklKFBCN1Ao" + nickname="maurizio" + subject="comment 9" + date="2013-12-02T20:49:14Z" + content=""" +I had already tried ```git annex fsck``` and for all files the command returned ```ok```. Nevertheless, it was still not possible to get the data. Now after a few days it seems I can get all the data I tried from both clients. Maybe because the repository contains many files (about 1.5e6) the daily sanity check spawns git processes which do not seem to terminate even when the delay has expired but other than that the repository does not seem to be in a bad state, the only weirdness being the 155 nonrepositories in the status. +"""]] diff --git a/doc/bugs/Assistant_having_a_child_git_cat-file_--batch_do_the_same_thing_over_and_over_and_using_a_lot_of_memory.mdwn b/doc/bugs/Assistant_having_a_child_git_cat-file_--batch_do_the_same_thing_over_and_over_and_using_a_lot_of_memory.mdwn new file mode 100644 index 0000000000..954d26ec23 --- /dev/null +++ b/doc/bugs/Assistant_having_a_child_git_cat-file_--batch_do_the_same_thing_over_and_over_and_using_a_lot_of_memory.mdwn @@ -0,0 +1,520 @@ +I have a git annex assistant process using 1.2 gigabytes of RAM and a git cat-file --batch child consuming CPU time constantly. I am running 5.20140320 on Ubuntu 12.04. + +[[!format sh """ + PID USER PR NI VIRT RES SHR S %CPU %MEM TIME+ COMMAND +11775 ion 20 0 1350m 1.2g 12m S 48 62.4 425:56.85 git-annex +11787 ion 20 0 9856 1484 1232 R 54 0.1 366:16.14 git +"""]] + +The assistant UI looks perfectly normal and does not indicate it is doing anything. daemon.log is empty and the assistant process seems to be logging into a rotated and deleted log file. + +[[!format sh """ +COMMAND PID USER FD TYPE DEVICE SIZE/OFF NODE NAME +git-annex 11775 ion 1w REG 9,127 80841 55181369 /storage/ion/media/video/.git/annex/daemon.log.10 (deleted) +git-annex 11775 ion 2w REG 9,127 80841 55181369 /storage/ion/media/video/.git/annex/daemon.log.10 (deleted) +"""]] + +strace -s10000 -e trace=read,write -p 11787 indicates that the assistant is having the cat-file process cat same objects over and over again. + +[[!format sh """ +read(0, "refs/heads/git-annex:ee2/2ee/SHA256E-s106800355--c70e31d511e7eec4881a15dfba521ea3d1fe14694968f81ae1819f1a2a93f9be.mp4.log\n", 4096) = 122 +read(3, "0936a1fdd849b8b46abb879d7cf82cc758b367e3\n", 255) = 41 +read(3, "", 214) = 0 +write(1, "977039ea431522e6e27a78bdec2c1299f883eb85 blob 232\n", 50) = 50 +write(1, "1396057823.999737s 1 3f89d0d8-6162-4362-852a-cb688d6c0696\n1396057835.133409s 1 161c3d7f-eb60-4294-84d2-eb611786c91e\n1396057835.215084s 1 161c3d7f-eb60-4294-84d2-eb611786c91e\n1396058160.468307s 0 3f89d0d8-6162-4362-852a-cb688d6c0696\n", 232) = 232 +write(1, "\n", 1) = 1 +read(0, "refs/heads/git-annex:ee2/2ee/SHA256E-s106800355--c70e31d511e7eec4881a15dfba521ea3d1fe14694968f81ae1819f1a2a93f9be.mp4.log\n", 4096) = 122 +read(3, "0936a1fdd849b8b46abb879d7cf82cc758b367e3\n", 255) = 41 +read(3, "", 214) = 0 +write(1, "977039ea431522e6e27a78bdec2c1299f883eb85 blob 232\n", 50) = 50 +write(1, "1396057823.999737s 1 3f89d0d8-6162-4362-852a-cb688d6c0696\n1396057835.133409s 1 161c3d7f-eb60-4294-84d2-eb611786c91e\n1396057835.215084s 1 161c3d7f-eb60-4294-84d2-eb611786c91e\n1396058160.468307s 0 3f89d0d8-6162-4362-852a-cb688d6c0696\n", 232) = 232 +write(1, "\n", 1) = 1 +read(0, "refs/heads/git-annex:a84/f1f/SHA256E-s47051987--dcfd0413db883506ccb8c45e3b2d60cb3ff5c83cc55c9c7e44818d7556dbc07f.mp4.log\n", 4096) = 121 +read(3, "0936a1fdd849b8b46abb879d7cf82cc758b367e3\n", 255) = 41 +read(3, "", 214) = 0 +write(1, "6c0ff555a1a34337c9379d8856c8283429bef973 blob 231\n", 50) = 50 +write(1, "1396057829.505426s 1 3f89d0d8-6162-4362-852a-cb688d6c0696\n1396057839.859236s 1 161c3d7f-eb60-4294-84d2-eb611786c91e\n1396057839.875213s 1 161c3d7f-eb60-4294-84d2-eb611786c91e\n1396058160.77741s 0 3f89d0d8-6162-4362-852a-cb688d6c0696\n", 231) = 231 +write(1, "\n", 1) = 1 +read(0, "refs/heads/git-annex:50a/5fc/SHA256E-s275654757--52823cd2061375910ccbd8de38865eca91511d9b4621243d2ef96a974d7546aa.flv.log\n", 4096) = 122 +read(3, "0936a1fdd849b8b46abb879d7cf82cc758b367e3\n", 255) = 41 +read(3, "", 214) = 0 +write(1, "cdbc7ce6b426dfcce9d718387b6c412e870a2d12 blob 232\n", 50) = 50 +write(1, "1396057828.887576s 1 3f89d0d8-6162-4362-852a-cb688d6c0696\n1396057866.117938s 1 161c3d7f-eb60-4294-84d2-eb611786c91e\n1396057866.197196s 1 161c3d7f-eb60-4294-84d2-eb611786c91e\n1396058160.687354s 0 3f89d0d8-6162-4362-852a-cb688d6c0696\n", 232) = 232 +write(1, "\n", 1) = 1 +read(0, "refs/heads/git-annex:9b5/545/SHA256E-s32710--6005b5faf1a6d42d499053f8cca87d080536abfa8442b33c87f7966e86726e4f.fin.srt.log\n", 4096) = 122 +read(3, "0936a1fdd849b8b46abb879d7cf82cc758b367e3\n", 255) = 41 +read(3, "", 214) = 0 +write(1, "82a98cbfe8f24d336a537cecea2182922c4681e1 blob 231\n", 50) = 50 +write(1, "1396057825.390306s 1 3f89d0d8-6162-4362-852a-cb688d6c0696\n1396057866.37922s 1 161c3d7f-eb60-4294-84d2-eb611786c91e\n1396057866.386029s 1 161c3d7f-eb60-4294-84d2-eb611786c91e\n1396058160.588219s 0 3f89d0d8-6162-4362-852a-cb688d6c0696\n", 231) = 231 +write(1, "\n", 1) = 1 +read(0, "refs/heads/git-annex:d28/166/SHA256E-s59188--9c04581bd67ea7c78b537a164d104bea5ac91a4a69f06b477cf07892a2d9b852.fih.srt.log\n", 4096) = 122 +read(3, "0936a1fdd849b8b46abb879d7cf82cc758b367e3\n", 255) = 41 +read(3, "", 214) = 0 +write(1, "d95950acebb5c4318329d7b989d36d01b76b7801 blob 232\n", 50) = 50 +write(1, "1396057825.366657s 1 3f89d0d8-6162-4362-852a-cb688d6c0696\n1396057866.538068s 1 161c3d7f-eb60-4294-84d2-eb611786c91e\n1396057866.560144s 1 161c3d7f-eb60-4294-84d2-eb611786c91e\n1396058160.538542s 0 3f89d0d8-6162-4362-852a-cb688d6c0696\n", 232) = 232 +write(1, "\n", 1) = 1 +read(0, "refs/heads/git-annex:ee2/2ee/SHA256E-s106800355--c70e31d511e7eec4881a15dfba521ea3d1fe14694968f81ae1819f1a2a93f9be.mp4.log\n", 4096) = 122 +read(3, "0936a1fdd849b8b46abb879d7cf82cc758b367e3\n", 255) = 41 +read(3, "", 214) = 0 +write(1, "977039ea431522e6e27a78bdec2c1299f883eb85 blob 232\n", 50) = 50 +write(1, "1396057823.999737s 1 3f89d0d8-6162-4362-852a-cb688d6c0696\n1396057835.133409s 1 161c3d7f-eb60-4294-84d2-eb611786c91e\n1396057835.215084s 1 161c3d7f-eb60-4294-84d2-eb611786c91e\n1396058160.468307s 0 3f89d0d8-6162-4362-852a-cb688d6c0696\n", 232) = 232 +write(1, "\n", 1) = 1 +read(0, "refs/heads/git-annex:ee2/2ee/SHA256E-s106800355--c70e31d511e7eec4881a15dfba521ea3d1fe14694968f81ae1819f1a2a93f9be.mp4.log\n", 4096) = 122 +read(3, "0936a1fdd849b8b46abb879d7cf82cc758b367e3\n", 255) = 41 +read(3, "", 214) = 0 +write(1, "977039ea431522e6e27a78bdec2c1299f883eb85 blob 232\n", 50) = 50 +write(1, "1396057823.999737s 1 3f89d0d8-6162-4362-852a-cb688d6c0696\n1396057835.133409s 1 161c3d7f-eb60-4294-84d2-eb611786c91e\n1396057835.215084s 1 161c3d7f-eb60-4294-84d2-eb611786c91e\n1396058160.468307s 0 3f89d0d8-6162-4362-852a-cb688d6c0696\n", 232) = 232 +write(1, "\n", 1) = 1 +read(0, "refs/heads/git-annex:a84/f1f/SHA256E-s47051987--dcfd0413db883506ccb8c45e3b2d60cb3ff5c83cc55c9c7e44818d7556dbc07f.mp4.log\n", 4096) = 121 +read(3, "0936a1fdd849b8b46abb879d7cf82cc758b367e3\n", 255) = 41 +read(3, "", 214) = 0 +write(1, "6c0ff555a1a34337c9379d8856c8283429bef973 blob 231\n", 50) = 50 +write(1, "1396057829.505426s 1 3f89d0d8-6162-4362-852a-cb688d6c0696\n1396057839.859236s 1 161c3d7f-eb60-4294-84d2-eb611786c91e\n1396057839.875213s 1 161c3d7f-eb60-4294-84d2-eb611786c91e\n1396058160.77741s 0 3f89d0d8-6162-4362-852a-cb688d6c0696\n", 231) = 231 +write(1, "\n", 1) = 1 +read(0, "refs/heads/git-annex:50a/5fc/SHA256E-s275654757--52823cd2061375910ccbd8de38865eca91511d9b4621243d2ef96a974d7546aa.flv.log\n", 4096) = 122 +read(3, "0936a1fdd849b8b46abb879d7cf82cc758b367e3\n", 255) = 41 +read(3, "", 214) = 0 +write(1, "cdbc7ce6b426dfcce9d718387b6c412e870a2d12 blob 232\n", 50) = 50 +write(1, "1396057828.887576s 1 3f89d0d8-6162-4362-852a-cb688d6c0696\n1396057866.117938s 1 161c3d7f-eb60-4294-84d2-eb611786c91e\n1396057866.197196s 1 161c3d7f-eb60-4294-84d2-eb611786c91e\n1396058160.687354s 0 3f89d0d8-6162-4362-852a-cb688d6c0696\n", 232) = 232 +write(1, "\n", 1) = 1 +read(0, "refs/heads/git-annex:9b5/545/SHA256E-s32710--6005b5faf1a6d42d499053f8cca87d080536abfa8442b33c87f7966e86726e4f.fin.srt.log\n", 4096) = 122 +read(3, "0936a1fdd849b8b46abb879d7cf82cc758b367e3\n", 255) = 41 +read(3, "", 214) = 0 +write(1, "82a98cbfe8f24d336a537cecea2182922c4681e1 blob 231\n", 50) = 50 +write(1, "1396057825.390306s 1 3f89d0d8-6162-4362-852a-cb688d6c0696\n1396057866.37922s 1 161c3d7f-eb60-4294-84d2-eb611786c91e\n1396057866.386029s 1 161c3d7f-eb60-4294-84d2-eb611786c91e\n1396058160.588219s 0 3f89d0d8-6162-4362-852a-cb688d6c0696\n", 231) = 231 +write(1, "\n", 1) = 1 +read(0, "refs/heads/git-annex:d28/166/SHA256E-s59188--9c04581bd67ea7c78b537a164d104bea5ac91a4a69f06b477cf07892a2d9b852.fih.srt.log\n", 4096) = 122 +read(3, "0936a1fdd849b8b46abb879d7cf82cc758b367e3\n", 255) = 41 +read(3, "", 214) = 0 +write(1, "d95950acebb5c4318329d7b989d36d01b76b7801 blob 232\n", 50) = 50 +write(1, "1396057825.366657s 1 3f89d0d8-6162-4362-852a-cb688d6c0696\n1396057866.538068s 1 161c3d7f-eb60-4294-84d2-eb611786c91e\n1396057866.560144s 1 161c3d7f-eb60-4294-84d2-eb611786c91e\n1396058160.538542s 0 3f89d0d8-6162-4362-852a-cb688d6c0696\n", 232) = 232 +write(1, "\n", 1) = 1 +read(0, "refs/heads/git-annex:ee2/2ee/SHA256E-s106800355--c70e31d511e7eec4881a15dfba521ea3d1fe14694968f81ae1819f1a2a93f9be.mp4.log\n", 4096) = 122 +read(3, "0936a1fdd849b8b46abb879d7cf82cc758b367e3\n", 255) = 41 +read(3, "", 214) = 0 +write(1, "977039ea431522e6e27a78bdec2c1299f883eb85 blob 232\n", 50) = 50 +write(1, "1396057823.999737s 1 3f89d0d8-6162-4362-852a-cb688d6c0696\n1396057835.133409s 1 161c3d7f-eb60-4294-84d2-eb611786c91e\n1396057835.215084s 1 161c3d7f-eb60-4294-84d2-eb611786c91e\n1396058160.468307s 0 3f89d0d8-6162-4362-852a-cb688d6c0696\n", 232) = 232 +write(1, "\n", 1) = 1 +read(0, "refs/heads/git-annex:ee2/2ee/SHA256E-s106800355--c70e31d511e7eec4881a15dfba521ea3d1fe14694968f81ae1819f1a2a93f9be.mp4.log\n", 4096) = 122 +read(3, "0936a1fdd849b8b46abb879d7cf82cc758b367e3\n", 255) = 41 +read(3, "", 214) = 0 +write(1, "977039ea431522e6e27a78bdec2c1299f883eb85 blob 232\n", 50) = 50 +write(1, "1396057823.999737s 1 3f89d0d8-6162-4362-852a-cb688d6c0696\n1396057835.133409s 1 161c3d7f-eb60-4294-84d2-eb611786c91e\n1396057835.215084s 1 161c3d7f-eb60-4294-84d2-eb611786c91e\n1396058160.468307s 0 3f89d0d8-6162-4362-852a-cb688d6c0696\n", 232) = 232 +write(1, "\n", 1) = 1 +read(0, "refs/heads/git-annex:a84/f1f/SHA256E-s47051987--dcfd0413db883506ccb8c45e3b2d60cb3ff5c83cc55c9c7e44818d7556dbc07f.mp4.log\n", 4096) = 121 +read(3, "0936a1fdd849b8b46abb879d7cf82cc758b367e3\n", 255) = 41 +read(3, "", 214) = 0 +write(1, "6c0ff555a1a34337c9379d8856c8283429bef973 blob 231\n", 50) = 50 +write(1, "1396057829.505426s 1 3f89d0d8-6162-4362-852a-cb688d6c0696\n1396057839.859236s 1 161c3d7f-eb60-4294-84d2-eb611786c91e\n1396057839.875213s 1 161c3d7f-eb60-4294-84d2-eb611786c91e\n1396058160.77741s 0 3f89d0d8-6162-4362-852a-cb688d6c0696\n", 231) = 231 +write(1, "\n", 1) = 1 +read(0, "refs/heads/git-annex:50a/5fc/SHA256E-s275654757--52823cd2061375910ccbd8de38865eca91511d9b4621243d2ef96a974d7546aa.flv.log\n", 4096) = 122 +read(3, "0936a1fdd849b8b46abb879d7cf82cc758b367e3\n", 255) = 41 +read(3, "", 214) = 0 +write(1, "cdbc7ce6b426dfcce9d718387b6c412e870a2d12 blob 232\n", 50) = 50 +write(1, "1396057828.887576s 1 3f89d0d8-6162-4362-852a-cb688d6c0696\n1396057866.117938s 1 161c3d7f-eb60-4294-84d2-eb611786c91e\n1396057866.197196s 1 161c3d7f-eb60-4294-84d2-eb611786c91e\n1396058160.687354s 0 3f89d0d8-6162-4362-852a-cb688d6c0696\n", 232) = 232 +write(1, "\n", 1) = 1 +read(0, "refs/heads/git-annex:9b5/545/SHA256E-s32710--6005b5faf1a6d42d499053f8cca87d080536abfa8442b33c87f7966e86726e4f.fin.srt.log\n", 4096) = 122 +read(3, "0936a1fdd849b8b46abb879d7cf82cc758b367e3\n", 255) = 41 +read(3, "", 214) = 0 +write(1, "82a98cbfe8f24d336a537cecea2182922c4681e1 blob 231\n", 50) = 50 +write(1, "1396057825.390306s 1 3f89d0d8-6162-4362-852a-cb688d6c0696\n1396057866.37922s 1 161c3d7f-eb60-4294-84d2-eb611786c91e\n1396057866.386029s 1 161c3d7f-eb60-4294-84d2-eb611786c91e\n1396058160.588219s 0 3f89d0d8-6162-4362-852a-cb688d6c0696\n", 231) = 231 +write(1, "\n", 1) = 1 +read(0, "refs/heads/git-annex:d28/166/SHA256E-s59188--9c04581bd67ea7c78b537a164d104bea5ac91a4a69f06b477cf07892a2d9b852.fih.srt.log\n", 4096) = 122 +read(3, "0936a1fdd849b8b46abb879d7cf82cc758b367e3\n", 255) = 41 +read(3, "", 214) = 0 +write(1, "d95950acebb5c4318329d7b989d36d01b76b7801 blob 232\n", 50) = 50 +write(1, "1396057825.366657s 1 3f89d0d8-6162-4362-852a-cb688d6c0696\n1396057866.538068s 1 161c3d7f-eb60-4294-84d2-eb611786c91e\n1396057866.560144s 1 161c3d7f-eb60-4294-84d2-eb611786c91e\n1396058160.538542s 0 3f89d0d8-6162-4362-852a-cb688d6c0696\n", 232) = 232 +write(1, "\n", 1) = 1 +read(0, "refs/heads/git-annex:ee2/2ee/SHA256E-s106800355--c70e31d511e7eec4881a15dfba521ea3d1fe14694968f81ae1819f1a2a93f9be.mp4.log\n", 4096) = 122 +read(3, "0936a1fdd849b8b46abb879d7cf82cc758b367e3\n", 255) = 41 +read(3, "", 214) = 0 +write(1, "977039ea431522e6e27a78bdec2c1299f883eb85 blob 232\n", 50) = 50 +write(1, "1396057823.999737s 1 3f89d0d8-6162-4362-852a-cb688d6c0696\n1396057835.133409s 1 161c3d7f-eb60-4294-84d2-eb611786c91e\n1396057835.215084s 1 161c3d7f-eb60-4294-84d2-eb611786c91e\n1396058160.468307s 0 3f89d0d8-6162-4362-852a-cb688d6c0696\n", 232) = 232 +write(1, "\n", 1) = 1 +read(0, "refs/heads/git-annex:ee2/2ee/SHA256E-s106800355--c70e31d511e7eec4881a15dfba521ea3d1fe14694968f81ae1819f1a2a93f9be.mp4.log\n", 4096) = 122 +read(3, "0936a1fdd849b8b46abb879d7cf82cc758b367e3\n", 255) = 41 +read(3, "", 214) = 0 +write(1, "977039ea431522e6e27a78bdec2c1299f883eb85 blob 232\n", 50) = 50 +write(1, "1396057823.999737s 1 3f89d0d8-6162-4362-852a-cb688d6c0696\n1396057835.133409s 1 161c3d7f-eb60-4294-84d2-eb611786c91e\n1396057835.215084s 1 161c3d7f-eb60-4294-84d2-eb611786c91e\n1396058160.468307s 0 3f89d0d8-6162-4362-852a-cb688d6c0696\n", 232) = 232 +write(1, "\n", 1) = 1 +read(0, "refs/heads/git-annex:a84/f1f/SHA256E-s47051987--dcfd0413db883506ccb8c45e3b2d60cb3ff5c83cc55c9c7e44818d7556dbc07f.mp4.log\n", 4096) = 121 +read(3, "0936a1fdd849b8b46abb879d7cf82cc758b367e3\n", 255) = 41 +read(3, "", 214) = 0 +write(1, "6c0ff555a1a34337c9379d8856c8283429bef973 blob 231\n", 50) = 50 +write(1, "1396057829.505426s 1 3f89d0d8-6162-4362-852a-cb688d6c0696\n1396057839.859236s 1 161c3d7f-eb60-4294-84d2-eb611786c91e\n1396057839.875213s 1 161c3d7f-eb60-4294-84d2-eb611786c91e\n1396058160.77741s 0 3f89d0d8-6162-4362-852a-cb688d6c0696\n", 231) = 231 +write(1, "\n", 1) = 1 +read(0, "refs/heads/git-annex:50a/5fc/SHA256E-s275654757--52823cd2061375910ccbd8de38865eca91511d9b4621243d2ef96a974d7546aa.flv.log\n", 4096) = 122 +read(3, "0936a1fdd849b8b46abb879d7cf82cc758b367e3\n", 255) = 41 +read(3, "", 214) = 0 +write(1, "cdbc7ce6b426dfcce9d718387b6c412e870a2d12 blob 232\n", 50) = 50 +write(1, "1396057828.887576s 1 3f89d0d8-6162-4362-852a-cb688d6c0696\n1396057866.117938s 1 161c3d7f-eb60-4294-84d2-eb611786c91e\n1396057866.197196s 1 161c3d7f-eb60-4294-84d2-eb611786c91e\n1396058160.687354s 0 3f89d0d8-6162-4362-852a-cb688d6c0696\n", 232) = 232 +write(1, "\n", 1) = 1 +read(0, "refs/heads/git-annex:9b5/545/SHA256E-s32710--6005b5faf1a6d42d499053f8cca87d080536abfa8442b33c87f7966e86726e4f.fin.srt.log\n", 4096) = 122 +read(3, "0936a1fdd849b8b46abb879d7cf82cc758b367e3\n", 255) = 41 +read(3, "", 214) = 0 +write(1, "82a98cbfe8f24d336a537cecea2182922c4681e1 blob 231\n", 50) = 50 +write(1, "1396057825.390306s 1 3f89d0d8-6162-4362-852a-cb688d6c0696\n1396057866.37922s 1 161c3d7f-eb60-4294-84d2-eb611786c91e\n1396057866.386029s 1 161c3d7f-eb60-4294-84d2-eb611786c91e\n1396058160.588219s 0 3f89d0d8-6162-4362-852a-cb688d6c0696\n", 231) = 231 +write(1, "\n", 1) = 1 +read(0, "refs/heads/git-annex:d28/166/SHA256E-s59188--9c04581bd67ea7c78b537a164d104bea5ac91a4a69f06b477cf07892a2d9b852.fih.srt.log\n", 4096) = 122 +read(3, "0936a1fdd849b8b46abb879d7cf82cc758b367e3\n", 255) = 41 +read(3, "", 214) = 0 +write(1, "d95950acebb5c4318329d7b989d36d01b76b7801 blob 232\n", 50) = 50 +write(1, "1396057825.366657s 1 3f89d0d8-6162-4362-852a-cb688d6c0696\n1396057866.538068s 1 161c3d7f-eb60-4294-84d2-eb611786c91e\n1396057866.560144s 1 161c3d7f-eb60-4294-84d2-eb611786c91e\n1396058160.538542s 0 3f89d0d8-6162-4362-852a-cb688d6c0696\n", 232) = 232 +write(1, "\n", 1) = 1 +read(0, "refs/heads/git-annex:ee2/2ee/SHA256E-s106800355--c70e31d511e7eec4881a15dfba521ea3d1fe14694968f81ae1819f1a2a93f9be.mp4.log\n", 4096) = 122 +read(3, "0936a1fdd849b8b46abb879d7cf82cc758b367e3\n", 255) = 41 +read(3, "", 214) = 0 +write(1, "977039ea431522e6e27a78bdec2c1299f883eb85 blob 232\n", 50) = 50 +write(1, "1396057823.999737s 1 3f89d0d8-6162-4362-852a-cb688d6c0696\n1396057835.133409s 1 161c3d7f-eb60-4294-84d2-eb611786c91e\n1396057835.215084s 1 161c3d7f-eb60-4294-84d2-eb611786c91e\n1396058160.468307s 0 3f89d0d8-6162-4362-852a-cb688d6c0696\n", 232) = 232 +write(1, "\n", 1) = 1 +read(0, "refs/heads/git-annex:ee2/2ee/SHA256E-s106800355--c70e31d511e7eec4881a15dfba521ea3d1fe14694968f81ae1819f1a2a93f9be.mp4.log\n", 4096) = 122 +read(3, "0936a1fdd849b8b46abb879d7cf82cc758b367e3\n", 255) = 41 +read(3, "", 214) = 0 +write(1, "977039ea431522e6e27a78bdec2c1299f883eb85 blob 232\n", 50) = 50 +write(1, "1396057823.999737s 1 3f89d0d8-6162-4362-852a-cb688d6c0696\n1396057835.133409s 1 161c3d7f-eb60-4294-84d2-eb611786c91e\n1396057835.215084s 1 161c3d7f-eb60-4294-84d2-eb611786c91e\n1396058160.468307s 0 3f89d0d8-6162-4362-852a-cb688d6c0696\n", 232) = 232 +write(1, "\n", 1) = 1 +read(0, "refs/heads/git-annex:a84/f1f/SHA256E-s47051987--dcfd0413db883506ccb8c45e3b2d60cb3ff5c83cc55c9c7e44818d7556dbc07f.mp4.log\n", 4096) = 121 +read(3, "0936a1fdd849b8b46abb879d7cf82cc758b367e3\n", 255) = 41 +read(3, "", 214) = 0 +write(1, "6c0ff555a1a34337c9379d8856c8283429bef973 blob 231\n", 50) = 50 +write(1, "1396057829.505426s 1 3f89d0d8-6162-4362-852a-cb688d6c0696\n1396057839.859236s 1 161c3d7f-eb60-4294-84d2-eb611786c91e\n1396057839.875213s 1 161c3d7f-eb60-4294-84d2-eb611786c91e\n1396058160.77741s 0 3f89d0d8-6162-4362-852a-cb688d6c0696\n", 231) = 231 +write(1, "\n", 1) = 1 +read(0, "refs/heads/git-annex:50a/5fc/SHA256E-s275654757--52823cd2061375910ccbd8de38865eca91511d9b4621243d2ef96a974d7546aa.flv.log\n", 4096) = 122 +read(3, "0936a1fdd849b8b46abb879d7cf82cc758b367e3\n", 255) = 41 +read(3, "", 214) = 0 +write(1, "cdbc7ce6b426dfcce9d718387b6c412e870a2d12 blob 232\n", 50) = 50 +write(1, "1396057828.887576s 1 3f89d0d8-6162-4362-852a-cb688d6c0696\n1396057866.117938s 1 161c3d7f-eb60-4294-84d2-eb611786c91e\n1396057866.197196s 1 161c3d7f-eb60-4294-84d2-eb611786c91e\n1396058160.687354s 0 3f89d0d8-6162-4362-852a-cb688d6c0696\n", 232) = 232 +write(1, "\n", 1) = 1 +read(0, "refs/heads/git-annex:9b5/545/SHA256E-s32710--6005b5faf1a6d42d499053f8cca87d080536abfa8442b33c87f7966e86726e4f.fin.srt.log\n", 4096) = 122 +read(3, "0936a1fdd849b8b46abb879d7cf82cc758b367e3\n", 255) = 41 +read(3, "", 214) = 0 +write(1, "82a98cbfe8f24d336a537cecea2182922c4681e1 blob 231\n", 50) = 50 +write(1, "1396057825.390306s 1 3f89d0d8-6162-4362-852a-cb688d6c0696\n1396057866.37922s 1 161c3d7f-eb60-4294-84d2-eb611786c91e\n1396057866.386029s 1 161c3d7f-eb60-4294-84d2-eb611786c91e\n1396058160.588219s 0 3f89d0d8-6162-4362-852a-cb688d6c0696\n", 231) = 231 +write(1, "\n", 1) = 1 +read(0, "refs/heads/git-annex:d28/166/SHA256E-s59188--9c04581bd67ea7c78b537a164d104bea5ac91a4a69f06b477cf07892a2d9b852.fih.srt.log\n", 4096) = 122 +read(3, "0936a1fdd849b8b46abb879d7cf82cc758b367e3\n", 255) = 41 +read(3, "", 214) = 0 +write(1, "d95950acebb5c4318329d7b989d36d01b76b7801 blob 232\n", 50) = 50 +write(1, "1396057825.366657s 1 3f89d0d8-6162-4362-852a-cb688d6c0696\n1396057866.538068s 1 161c3d7f-eb60-4294-84d2-eb611786c91e\n1396057866.560144s 1 161c3d7f-eb60-4294-84d2-eb611786c91e\n1396058160.538542s 0 3f89d0d8-6162-4362-852a-cb688d6c0696\n", 232) = 232 +write(1, "\n", 1) = 1 +read(0, "refs/heads/git-annex:ee2/2ee/SHA256E-s106800355--c70e31d511e7eec4881a15dfba521ea3d1fe14694968f81ae1819f1a2a93f9be.mp4.log\n", 4096) = 122 +read(3, "0936a1fdd849b8b46abb879d7cf82cc758b367e3\n", 255) = 41 +read(3, "", 214) = 0 +write(1, "977039ea431522e6e27a78bdec2c1299f883eb85 blob 232\n", 50) = 50 +write(1, "1396057823.999737s 1 3f89d0d8-6162-4362-852a-cb688d6c0696\n1396057835.133409s 1 161c3d7f-eb60-4294-84d2-eb611786c91e\n1396057835.215084s 1 161c3d7f-eb60-4294-84d2-eb611786c91e\n1396058160.468307s 0 3f89d0d8-6162-4362-852a-cb688d6c0696\n", 232) = 232 +write(1, "\n", 1) = 1 +read(0, "refs/heads/git-annex:ee2/2ee/SHA256E-s106800355--c70e31d511e7eec4881a15dfba521ea3d1fe14694968f81ae1819f1a2a93f9be.mp4.log\n", 4096) = 122 +read(3, "0936a1fdd849b8b46abb879d7cf82cc758b367e3\n", 255) = 41 +read(3, "", 214) = 0 +write(1, "977039ea431522e6e27a78bdec2c1299f883eb85 blob 232\n", 50) = 50 +write(1, "1396057823.999737s 1 3f89d0d8-6162-4362-852a-cb688d6c0696\n1396057835.133409s 1 161c3d7f-eb60-4294-84d2-eb611786c91e\n1396057835.215084s 1 161c3d7f-eb60-4294-84d2-eb611786c91e\n1396058160.468307s 0 3f89d0d8-6162-4362-852a-cb688d6c0696\n", 232) = 232 +write(1, "\n", 1) = 1 +read(0, "refs/heads/git-annex:a84/f1f/SHA256E-s47051987--dcfd0413db883506ccb8c45e3b2d60cb3ff5c83cc55c9c7e44818d7556dbc07f.mp4.log\n", 4096) = 121 +read(3, "0936a1fdd849b8b46abb879d7cf82cc758b367e3\n", 255) = 41 +read(3, "", 214) = 0 +write(1, "6c0ff555a1a34337c9379d8856c8283429bef973 blob 231\n", 50) = 50 +write(1, "1396057829.505426s 1 3f89d0d8-6162-4362-852a-cb688d6c0696\n1396057839.859236s 1 161c3d7f-eb60-4294-84d2-eb611786c91e\n1396057839.875213s 1 161c3d7f-eb60-4294-84d2-eb611786c91e\n1396058160.77741s 0 3f89d0d8-6162-4362-852a-cb688d6c0696\n", 231) = 231 +write(1, "\n", 1) = 1 +read(0, "refs/heads/git-annex:50a/5fc/SHA256E-s275654757--52823cd2061375910ccbd8de38865eca91511d9b4621243d2ef96a974d7546aa.flv.log\n", 4096) = 122 +read(3, "0936a1fdd849b8b46abb879d7cf82cc758b367e3\n", 255) = 41 +read(3, "", 214) = 0 +write(1, "cdbc7ce6b426dfcce9d718387b6c412e870a2d12 blob 232\n", 50) = 50 +write(1, "1396057828.887576s 1 3f89d0d8-6162-4362-852a-cb688d6c0696\n1396057866.117938s 1 161c3d7f-eb60-4294-84d2-eb611786c91e\n1396057866.197196s 1 161c3d7f-eb60-4294-84d2-eb611786c91e\n1396058160.687354s 0 3f89d0d8-6162-4362-852a-cb688d6c0696\n", 232) = 232 +write(1, "\n", 1) = 1 +read(0, "refs/heads/git-annex:9b5/545/SHA256E-s32710--6005b5faf1a6d42d499053f8cca87d080536abfa8442b33c87f7966e86726e4f.fin.srt.log\n", 4096) = 122 +read(3, "0936a1fdd849b8b46abb879d7cf82cc758b367e3\n", 255) = 41 +read(3, "", 214) = 0 +write(1, "82a98cbfe8f24d336a537cecea2182922c4681e1 blob 231\n", 50) = 50 +write(1, "1396057825.390306s 1 3f89d0d8-6162-4362-852a-cb688d6c0696\n1396057866.37922s 1 161c3d7f-eb60-4294-84d2-eb611786c91e\n1396057866.386029s 1 161c3d7f-eb60-4294-84d2-eb611786c91e\n1396058160.588219s 0 3f89d0d8-6162-4362-852a-cb688d6c0696\n", 231) = 231 +write(1, "\n", 1) = 1 +read(0, "refs/heads/git-annex:d28/166/SHA256E-s59188--9c04581bd67ea7c78b537a164d104bea5ac91a4a69f06b477cf07892a2d9b852.fih.srt.log\n", 4096) = 122 +read(3, "0936a1fdd849b8b46abb879d7cf82cc758b367e3\n", 255) = 41 +read(3, "", 214) = 0 +write(1, "d95950acebb5c4318329d7b989d36d01b76b7801 blob 232\n", 50) = 50 +write(1, "1396057825.366657s 1 3f89d0d8-6162-4362-852a-cb688d6c0696\n1396057866.538068s 1 161c3d7f-eb60-4294-84d2-eb611786c91e\n1396057866.560144s 1 161c3d7f-eb60-4294-84d2-eb611786c91e\n1396058160.538542s 0 3f89d0d8-6162-4362-852a-cb688d6c0696\n", 232) = 232 +write(1, "\n", 1) = 1 +read(0, "refs/heads/git-annex:ee2/2ee/SHA256E-s106800355--c70e31d511e7eec4881a15dfba521ea3d1fe14694968f81ae1819f1a2a93f9be.mp4.log\n", 4096) = 122 +read(3, "0936a1fdd849b8b46abb879d7cf82cc758b367e3\n", 255) = 41 +read(3, "", 214) = 0 +write(1, "977039ea431522e6e27a78bdec2c1299f883eb85 blob 232\n", 50) = 50 +write(1, "1396057823.999737s 1 3f89d0d8-6162-4362-852a-cb688d6c0696\n1396057835.133409s 1 161c3d7f-eb60-4294-84d2-eb611786c91e\n1396057835.215084s 1 161c3d7f-eb60-4294-84d2-eb611786c91e\n1396058160.468307s 0 3f89d0d8-6162-4362-852a-cb688d6c0696\n", 232) = 232 +write(1, "\n", 1) = 1 +read(0, "refs/heads/git-annex:ee2/2ee/SHA256E-s106800355--c70e31d511e7eec4881a15dfba521ea3d1fe14694968f81ae1819f1a2a93f9be.mp4.log\n", 4096) = 122 +read(3, "0936a1fdd849b8b46abb879d7cf82cc758b367e3\n", 255) = 41 +read(3, "", 214) = 0 +write(1, "977039ea431522e6e27a78bdec2c1299f883eb85 blob 232\n", 50) = 50 +write(1, "1396057823.999737s 1 3f89d0d8-6162-4362-852a-cb688d6c0696\n1396057835.133409s 1 161c3d7f-eb60-4294-84d2-eb611786c91e\n1396057835.215084s 1 161c3d7f-eb60-4294-84d2-eb611786c91e\n1396058160.468307s 0 3f89d0d8-6162-4362-852a-cb688d6c0696\n", 232) = 232 +write(1, "\n", 1) = 1 +read(0, "refs/heads/git-annex:a84/f1f/SHA256E-s47051987--dcfd0413db883506ccb8c45e3b2d60cb3ff5c83cc55c9c7e44818d7556dbc07f.mp4.log\n", 4096) = 121 +read(3, "0936a1fdd849b8b46abb879d7cf82cc758b367e3\n", 255) = 41 +read(3, "", 214) = 0 +write(1, "6c0ff555a1a34337c9379d8856c8283429bef973 blob 231\n", 50) = 50 +write(1, "1396057829.505426s 1 3f89d0d8-6162-4362-852a-cb688d6c0696\n1396057839.859236s 1 161c3d7f-eb60-4294-84d2-eb611786c91e\n1396057839.875213s 1 161c3d7f-eb60-4294-84d2-eb611786c91e\n1396058160.77741s 0 3f89d0d8-6162-4362-852a-cb688d6c0696\n", 231) = 231 +write(1, "\n", 1) = 1 +read(0, "refs/heads/git-annex:50a/5fc/SHA256E-s275654757--52823cd2061375910ccbd8de38865eca91511d9b4621243d2ef96a974d7546aa.flv.log\n", 4096) = 122 +read(3, "0936a1fdd849b8b46abb879d7cf82cc758b367e3\n", 255) = 41 +read(3, "", 214) = 0 +write(1, "cdbc7ce6b426dfcce9d718387b6c412e870a2d12 blob 232\n", 50) = 50 +write(1, "1396057828.887576s 1 3f89d0d8-6162-4362-852a-cb688d6c0696\n1396057866.117938s 1 161c3d7f-eb60-4294-84d2-eb611786c91e\n1396057866.197196s 1 161c3d7f-eb60-4294-84d2-eb611786c91e\n1396058160.687354s 0 3f89d0d8-6162-4362-852a-cb688d6c0696\n", 232) = 232 +write(1, "\n", 1) = 1 +read(0, "refs/heads/git-annex:9b5/545/SHA256E-s32710--6005b5faf1a6d42d499053f8cca87d080536abfa8442b33c87f7966e86726e4f.fin.srt.log\n", 4096) = 122 +read(3, "0936a1fdd849b8b46abb879d7cf82cc758b367e3\n", 255) = 41 +read(3, "", 214) = 0 +write(1, "82a98cbfe8f24d336a537cecea2182922c4681e1 blob 231\n", 50) = 50 +write(1, "1396057825.390306s 1 3f89d0d8-6162-4362-852a-cb688d6c0696\n1396057866.37922s 1 161c3d7f-eb60-4294-84d2-eb611786c91e\n1396057866.386029s 1 161c3d7f-eb60-4294-84d2-eb611786c91e\n1396058160.588219s 0 3f89d0d8-6162-4362-852a-cb688d6c0696\n", 231) = 231 +write(1, "\n", 1) = 1 +read(0, "refs/heads/git-annex:d28/166/SHA256E-s59188--9c04581bd67ea7c78b537a164d104bea5ac91a4a69f06b477cf07892a2d9b852.fih.srt.log\n", 4096) = 122 +read(3, "0936a1fdd849b8b46abb879d7cf82cc758b367e3\n", 255) = 41 +read(3, "", 214) = 0 +write(1, "d95950acebb5c4318329d7b989d36d01b76b7801 blob 232\n", 50) = 50 +write(1, "1396057825.366657s 1 3f89d0d8-6162-4362-852a-cb688d6c0696\n1396057866.538068s 1 161c3d7f-eb60-4294-84d2-eb611786c91e\n1396057866.560144s 1 161c3d7f-eb60-4294-84d2-eb611786c91e\n1396058160.538542s 0 3f89d0d8-6162-4362-852a-cb688d6c0696\n", 232) = 232 +write(1, "\n", 1) = 1 +read(0, "refs/heads/git-annex:ee2/2ee/SHA256E-s106800355--c70e31d511e7eec4881a15dfba521ea3d1fe14694968f81ae1819f1a2a93f9be.mp4.log\n", 4096) = 122 +read(3, "0936a1fdd849b8b46abb879d7cf82cc758b367e3\n", 255) = 41 +read(3, "", 214) = 0 +write(1, "977039ea431522e6e27a78bdec2c1299f883eb85 blob 232\n", 50) = 50 +write(1, "1396057823.999737s 1 3f89d0d8-6162-4362-852a-cb688d6c0696\n1396057835.133409s 1 161c3d7f-eb60-4294-84d2-eb611786c91e\n1396057835.215084s 1 161c3d7f-eb60-4294-84d2-eb611786c91e\n1396058160.468307s 0 3f89d0d8-6162-4362-852a-cb688d6c0696\n", 232) = 232 +write(1, "\n", 1) = 1 +read(0, "refs/heads/git-annex:ee2/2ee/SHA256E-s106800355--c70e31d511e7eec4881a15dfba521ea3d1fe14694968f81ae1819f1a2a93f9be.mp4.log\n", 4096) = 122 +read(3, "0936a1fdd849b8b46abb879d7cf82cc758b367e3\n", 255) = 41 +read(3, "", 214) = 0 +write(1, "977039ea431522e6e27a78bdec2c1299f883eb85 blob 232\n", 50) = 50 +write(1, "1396057823.999737s 1 3f89d0d8-6162-4362-852a-cb688d6c0696\n1396057835.133409s 1 161c3d7f-eb60-4294-84d2-eb611786c91e\n1396057835.215084s 1 161c3d7f-eb60-4294-84d2-eb611786c91e\n1396058160.468307s 0 3f89d0d8-6162-4362-852a-cb688d6c0696\n", 232) = 232 +write(1, "\n", 1) = 1 +read(0, "refs/heads/git-annex:a84/f1f/SHA256E-s47051987--dcfd0413db883506ccb8c45e3b2d60cb3ff5c83cc55c9c7e44818d7556dbc07f.mp4.log\n", 4096) = 121 +read(3, "0936a1fdd849b8b46abb879d7cf82cc758b367e3\n", 255) = 41 +read(3, "", 214) = 0 +write(1, "6c0ff555a1a34337c9379d8856c8283429bef973 blob 231\n", 50) = 50 +write(1, "1396057829.505426s 1 3f89d0d8-6162-4362-852a-cb688d6c0696\n1396057839.859236s 1 161c3d7f-eb60-4294-84d2-eb611786c91e\n1396057839.875213s 1 161c3d7f-eb60-4294-84d2-eb611786c91e\n1396058160.77741s 0 3f89d0d8-6162-4362-852a-cb688d6c0696\n", 231) = 231 +write(1, "\n", 1) = 1 +read(0, "refs/heads/git-annex:50a/5fc/SHA256E-s275654757--52823cd2061375910ccbd8de38865eca91511d9b4621243d2ef96a974d7546aa.flv.log\n", 4096) = 122 +read(3, "0936a1fdd849b8b46abb879d7cf82cc758b367e3\n", 255) = 41 +read(3, "", 214) = 0 +write(1, "cdbc7ce6b426dfcce9d718387b6c412e870a2d12 blob 232\n", 50) = 50 +write(1, "1396057828.887576s 1 3f89d0d8-6162-4362-852a-cb688d6c0696\n1396057866.117938s 1 161c3d7f-eb60-4294-84d2-eb611786c91e\n1396057866.197196s 1 161c3d7f-eb60-4294-84d2-eb611786c91e\n1396058160.687354s 0 3f89d0d8-6162-4362-852a-cb688d6c0696\n", 232) = 232 +write(1, "\n", 1) = 1 +read(0, "refs/heads/git-annex:9b5/545/SHA256E-s32710--6005b5faf1a6d42d499053f8cca87d080536abfa8442b33c87f7966e86726e4f.fin.srt.log\n", 4096) = 122 +read(3, "0936a1fdd849b8b46abb879d7cf82cc758b367e3\n", 255) = 41 +read(3, "", 214) = 0 +write(1, "82a98cbfe8f24d336a537cecea2182922c4681e1 blob 231\n", 50) = 50 +write(1, "1396057825.390306s 1 3f89d0d8-6162-4362-852a-cb688d6c0696\n1396057866.37922s 1 161c3d7f-eb60-4294-84d2-eb611786c91e\n1396057866.386029s 1 161c3d7f-eb60-4294-84d2-eb611786c91e\n1396058160.588219s 0 3f89d0d8-6162-4362-852a-cb688d6c0696\n", 231) = 231 +write(1, "\n", 1) = 1 +read(0, "refs/heads/git-annex:d28/166/SHA256E-s59188--9c04581bd67ea7c78b537a164d104bea5ac91a4a69f06b477cf07892a2d9b852.fih.srt.log\n", 4096) = 122 +read(3, "0936a1fdd849b8b46abb879d7cf82cc758b367e3\n", 255) = 41 +read(3, "", 214) = 0 +write(1, "d95950acebb5c4318329d7b989d36d01b76b7801 blob 232\n", 50) = 50 +write(1, "1396057825.366657s 1 3f89d0d8-6162-4362-852a-cb688d6c0696\n1396057866.538068s 1 161c3d7f-eb60-4294-84d2-eb611786c91e\n1396057866.560144s 1 161c3d7f-eb60-4294-84d2-eb611786c91e\n1396058160.538542s 0 3f89d0d8-6162-4362-852a-cb688d6c0696\n", 232) = 232 +write(1, "\n", 1) = 1 +read(0, "refs/heads/git-annex:ee2/2ee/SHA256E-s106800355--c70e31d511e7eec4881a15dfba521ea3d1fe14694968f81ae1819f1a2a93f9be.mp4.log\n", 4096) = 122 +read(3, "0936a1fdd849b8b46abb879d7cf82cc758b367e3\n", 255) = 41 +read(3, "", 214) = 0 +write(1, "977039ea431522e6e27a78bdec2c1299f883eb85 blob 232\n", 50) = 50 +write(1, "1396057823.999737s 1 3f89d0d8-6162-4362-852a-cb688d6c0696\n1396057835.133409s 1 161c3d7f-eb60-4294-84d2-eb611786c91e\n1396057835.215084s 1 161c3d7f-eb60-4294-84d2-eb611786c91e\n1396058160.468307s 0 3f89d0d8-6162-4362-852a-cb688d6c0696\n", 232) = 232 +write(1, "\n", 1) = 1 +read(0, "refs/heads/git-annex:ee2/2ee/SHA256E-s106800355--c70e31d511e7eec4881a15dfba521ea3d1fe14694968f81ae1819f1a2a93f9be.mp4.log\n", 4096) = 122 +read(3, "0936a1fdd849b8b46abb879d7cf82cc758b367e3\n", 255) = 41 +read(3, "", 214) = 0 +write(1, "977039ea431522e6e27a78bdec2c1299f883eb85 blob 232\n", 50) = 50 +write(1, "1396057823.999737s 1 3f89d0d8-6162-4362-852a-cb688d6c0696\n1396057835.133409s 1 161c3d7f-eb60-4294-84d2-eb611786c91e\n1396057835.215084s 1 161c3d7f-eb60-4294-84d2-eb611786c91e\n1396058160.468307s 0 3f89d0d8-6162-4362-852a-cb688d6c0696\n", 232) = 232 +write(1, "\n", 1) = 1 +read(0, "refs/heads/git-annex:a84/f1f/SHA256E-s47051987--dcfd0413db883506ccb8c45e3b2d60cb3ff5c83cc55c9c7e44818d7556dbc07f.mp4.log\n", 4096) = 121 +read(3, "0936a1fdd849b8b46abb879d7cf82cc758b367e3\n", 255) = 41 +read(3, "", 214) = 0 +write(1, "6c0ff555a1a34337c9379d8856c8283429bef973 blob 231\n", 50) = 50 +write(1, "1396057829.505426s 1 3f89d0d8-6162-4362-852a-cb688d6c0696\n1396057839.859236s 1 161c3d7f-eb60-4294-84d2-eb611786c91e\n1396057839.875213s 1 161c3d7f-eb60-4294-84d2-eb611786c91e\n1396058160.77741s 0 3f89d0d8-6162-4362-852a-cb688d6c0696\n", 231) = 231 +write(1, "\n", 1) = 1 +read(0, "refs/heads/git-annex:50a/5fc/SHA256E-s275654757--52823cd2061375910ccbd8de38865eca91511d9b4621243d2ef96a974d7546aa.flv.log\n", 4096) = 122 +read(3, "0936a1fdd849b8b46abb879d7cf82cc758b367e3\n", 255) = 41 +read(3, "", 214) = 0 +write(1, "cdbc7ce6b426dfcce9d718387b6c412e870a2d12 blob 232\n", 50) = 50 +write(1, "1396057828.887576s 1 3f89d0d8-6162-4362-852a-cb688d6c0696\n1396057866.117938s 1 161c3d7f-eb60-4294-84d2-eb611786c91e\n1396057866.197196s 1 161c3d7f-eb60-4294-84d2-eb611786c91e\n1396058160.687354s 0 3f89d0d8-6162-4362-852a-cb688d6c0696\n", 232) = 232 +write(1, "\n", 1) = 1 +read(0, "refs/heads/git-annex:9b5/545/SHA256E-s32710--6005b5faf1a6d42d499053f8cca87d080536abfa8442b33c87f7966e86726e4f.fin.srt.log\n", 4096) = 122 +read(3, "0936a1fdd849b8b46abb879d7cf82cc758b367e3\n", 255) = 41 +read(3, "", 214) = 0 +write(1, "82a98cbfe8f24d336a537cecea2182922c4681e1 blob 231\n", 50) = 50 +write(1, "1396057825.390306s 1 3f89d0d8-6162-4362-852a-cb688d6c0696\n1396057866.37922s 1 161c3d7f-eb60-4294-84d2-eb611786c91e\n1396057866.386029s 1 161c3d7f-eb60-4294-84d2-eb611786c91e\n1396058160.588219s 0 3f89d0d8-6162-4362-852a-cb688d6c0696\n", 231) = 231 +write(1, "\n", 1) = 1 +read(0, "refs/heads/git-annex:d28/166/SHA256E-s59188--9c04581bd67ea7c78b537a164d104bea5ac91a4a69f06b477cf07892a2d9b852.fih.srt.log\n", 4096) = 122 +read(3, "0936a1fdd849b8b46abb879d7cf82cc758b367e3\n", 255) = 41 +read(3, "", 214) = 0 +write(1, "d95950acebb5c4318329d7b989d36d01b76b7801 blob 232\n", 50) = 50 +write(1, "1396057825.366657s 1 3f89d0d8-6162-4362-852a-cb688d6c0696\n1396057866.538068s 1 161c3d7f-eb60-4294-84d2-eb611786c91e\n1396057866.560144s 1 161c3d7f-eb60-4294-84d2-eb611786c91e\n1396058160.538542s 0 3f89d0d8-6162-4362-852a-cb688d6c0696\n", 232) = 232 +write(1, "\n", 1) = 1 +read(0, "refs/heads/git-annex:ee2/2ee/SHA256E-s106800355--c70e31d511e7eec4881a15dfba521ea3d1fe14694968f81ae1819f1a2a93f9be.mp4.log\n", 4096) = 122 +read(3, "0936a1fdd849b8b46abb879d7cf82cc758b367e3\n", 255) = 41 +read(3, "", 214) = 0 +write(1, "977039ea431522e6e27a78bdec2c1299f883eb85 blob 232\n", 50) = 50 +write(1, "1396057823.999737s 1 3f89d0d8-6162-4362-852a-cb688d6c0696\n1396057835.133409s 1 161c3d7f-eb60-4294-84d2-eb611786c91e\n1396057835.215084s 1 161c3d7f-eb60-4294-84d2-eb611786c91e\n1396058160.468307s 0 3f89d0d8-6162-4362-852a-cb688d6c0696\n", 232) = 232 +write(1, "\n", 1) = 1 +read(0, "refs/heads/git-annex:ee2/2ee/SHA256E-s106800355--c70e31d511e7eec4881a15dfba521ea3d1fe14694968f81ae1819f1a2a93f9be.mp4.log\n", 4096) = 122 +read(3, "0936a1fdd849b8b46abb879d7cf82cc758b367e3\n", 255) = 41 +read(3, "", 214) = 0 +write(1, "977039ea431522e6e27a78bdec2c1299f883eb85 blob 232\n", 50) = 50 +write(1, "1396057823.999737s 1 3f89d0d8-6162-4362-852a-cb688d6c0696\n1396057835.133409s 1 161c3d7f-eb60-4294-84d2-eb611786c91e\n1396057835.215084s 1 161c3d7f-eb60-4294-84d2-eb611786c91e\n1396058160.468307s 0 3f89d0d8-6162-4362-852a-cb688d6c0696\n", 232) = 232 +write(1, "\n", 1) = 1 +read(0, "refs/heads/git-annex:a84/f1f/SHA256E-s47051987--dcfd0413db883506ccb8c45e3b2d60cb3ff5c83cc55c9c7e44818d7556dbc07f.mp4.log\n", 4096) = 121 +read(3, "0936a1fdd849b8b46abb879d7cf82cc758b367e3\n", 255) = 41 +read(3, "", 214) = 0 +write(1, "6c0ff555a1a34337c9379d8856c8283429bef973 blob 231\n", 50) = 50 +write(1, "1396057829.505426s 1 3f89d0d8-6162-4362-852a-cb688d6c0696\n1396057839.859236s 1 161c3d7f-eb60-4294-84d2-eb611786c91e\n1396057839.875213s 1 161c3d7f-eb60-4294-84d2-eb611786c91e\n1396058160.77741s 0 3f89d0d8-6162-4362-852a-cb688d6c0696\n", 231) = 231 +write(1, "\n", 1) = 1 +read(0, "refs/heads/git-annex:50a/5fc/SHA256E-s275654757--52823cd2061375910ccbd8de38865eca91511d9b4621243d2ef96a974d7546aa.flv.log\n", 4096) = 122 +read(3, "0936a1fdd849b8b46abb879d7cf82cc758b367e3\n", 255) = 41 +read(3, "", 214) = 0 +write(1, "cdbc7ce6b426dfcce9d718387b6c412e870a2d12 blob 232\n", 50) = 50 +write(1, "1396057828.887576s 1 3f89d0d8-6162-4362-852a-cb688d6c0696\n1396057866.117938s 1 161c3d7f-eb60-4294-84d2-eb611786c91e\n1396057866.197196s 1 161c3d7f-eb60-4294-84d2-eb611786c91e\n1396058160.687354s 0 3f89d0d8-6162-4362-852a-cb688d6c0696\n", 232) = 232 +write(1, "\n", 1) = 1 +read(0, "refs/heads/git-annex:9b5/545/SHA256E-s32710--6005b5faf1a6d42d499053f8cca87d080536abfa8442b33c87f7966e86726e4f.fin.srt.log\n", 4096) = 122 +read(3, "0936a1fdd849b8b46abb879d7cf82cc758b367e3\n", 255) = 41 +read(3, "", 214) = 0 +write(1, "82a98cbfe8f24d336a537cecea2182922c4681e1 blob 231\n", 50) = 50 +write(1, "1396057825.390306s 1 3f89d0d8-6162-4362-852a-cb688d6c0696\n1396057866.37922s 1 161c3d7f-eb60-4294-84d2-eb611786c91e\n1396057866.386029s 1 161c3d7f-eb60-4294-84d2-eb611786c91e\n1396058160.588219s 0 3f89d0d8-6162-4362-852a-cb688d6c0696\n", 231) = 231 +write(1, "\n", 1) = 1 +read(0, "refs/heads/git-annex:d28/166/SHA256E-s59188--9c04581bd67ea7c78b537a164d104bea5ac91a4a69f06b477cf07892a2d9b852.fih.srt.log\n", 4096) = 122 +read(3, "0936a1fdd849b8b46abb879d7cf82cc758b367e3\n", 255) = 41 +read(3, "", 214) = 0 +write(1, "d95950acebb5c4318329d7b989d36d01b76b7801 blob 232\n", 50) = 50 +write(1, "1396057825.366657s 1 3f89d0d8-6162-4362-852a-cb688d6c0696\n1396057866.538068s 1 161c3d7f-eb60-4294-84d2-eb611786c91e\n1396057866.560144s 1 161c3d7f-eb60-4294-84d2-eb611786c91e\n1396058160.538542s 0 3f89d0d8-6162-4362-852a-cb688d6c0696\n", 232) = 232 +write(1, "\n", 1) = 1 +read(0, "refs/heads/git-annex:ee2/2ee/SHA256E-s106800355--c70e31d511e7eec4881a15dfba521ea3d1fe14694968f81ae1819f1a2a93f9be.mp4.log\n", 4096) = 122 +read(3, "0936a1fdd849b8b46abb879d7cf82cc758b367e3\n", 255) = 41 +read(3, "", 214) = 0 +write(1, "977039ea431522e6e27a78bdec2c1299f883eb85 blob 232\n", 50) = 50 +write(1, "1396057823.999737s 1 3f89d0d8-6162-4362-852a-cb688d6c0696\n1396057835.133409s 1 161c3d7f-eb60-4294-84d2-eb611786c91e\n1396057835.215084s 1 161c3d7f-eb60-4294-84d2-eb611786c91e\n1396058160.468307s 0 3f89d0d8-6162-4362-852a-cb688d6c0696\n", 232) = 232 +write(1, "\n", 1) = 1 +read(0, "refs/heads/git-annex:ee2/2ee/SHA256E-s106800355--c70e31d511e7eec4881a15dfba521ea3d1fe14694968f81ae1819f1a2a93f9be.mp4.log\n", 4096) = 122 +read(3, "0936a1fdd849b8b46abb879d7cf82cc758b367e3\n", 255) = 41 +read(3, "", 214) = 0 +write(1, "977039ea431522e6e27a78bdec2c1299f883eb85 blob 232\n", 50) = 50 +write(1, "1396057823.999737s 1 3f89d0d8-6162-4362-852a-cb688d6c0696\n1396057835.133409s 1 161c3d7f-eb60-4294-84d2-eb611786c91e\n1396057835.215084s 1 161c3d7f-eb60-4294-84d2-eb611786c91e\n1396058160.468307s 0 3f89d0d8-6162-4362-852a-cb688d6c0696\n", 232) = 232 +write(1, "\n", 1) = 1 +read(0, "refs/heads/git-annex:a84/f1f/SHA256E-s47051987--dcfd0413db883506ccb8c45e3b2d60cb3ff5c83cc55c9c7e44818d7556dbc07f.mp4.log\n", 4096) = 121 +read(3, "0936a1fdd849b8b46abb879d7cf82cc758b367e3\n", 255) = 41 +read(3, "", 214) = 0 +write(1, "6c0ff555a1a34337c9379d8856c8283429bef973 blob 231\n", 50) = 50 +write(1, "1396057829.505426s 1 3f89d0d8-6162-4362-852a-cb688d6c0696\n1396057839.859236s 1 161c3d7f-eb60-4294-84d2-eb611786c91e\n1396057839.875213s 1 161c3d7f-eb60-4294-84d2-eb611786c91e\n1396058160.77741s 0 3f89d0d8-6162-4362-852a-cb688d6c0696\n", 231) = 231 +write(1, "\n", 1) = 1 +read(0, "refs/heads/git-annex:50a/5fc/SHA256E-s275654757--52823cd2061375910ccbd8de38865eca91511d9b4621243d2ef96a974d7546aa.flv.log\n", 4096) = 122 +read(3, "0936a1fdd849b8b46abb879d7cf82cc758b367e3\n", 255) = 41 +read(3, "", 214) = 0 +write(1, "cdbc7ce6b426dfcce9d718387b6c412e870a2d12 blob 232\n", 50) = 50 +write(1, "1396057828.887576s 1 3f89d0d8-6162-4362-852a-cb688d6c0696\n1396057866.117938s 1 161c3d7f-eb60-4294-84d2-eb611786c91e\n1396057866.197196s 1 161c3d7f-eb60-4294-84d2-eb611786c91e\n1396058160.687354s 0 3f89d0d8-6162-4362-852a-cb688d6c0696\n", 232) = 232 +write(1, "\n", 1) = 1 +read(0, "refs/heads/git-annex:9b5/545/SHA256E-s32710--6005b5faf1a6d42d499053f8cca87d080536abfa8442b33c87f7966e86726e4f.fin.srt.log\n", 4096) = 122 +read(3, "0936a1fdd849b8b46abb879d7cf82cc758b367e3\n", 255) = 41 +read(3, "", 214) = 0 +write(1, "82a98cbfe8f24d336a537cecea2182922c4681e1 blob 231\n", 50) = 50 +write(1, "1396057825.390306s 1 3f89d0d8-6162-4362-852a-cb688d6c0696\n1396057866.37922s 1 161c3d7f-eb60-4294-84d2-eb611786c91e\n1396057866.386029s 1 161c3d7f-eb60-4294-84d2-eb611786c91e\n1396058160.588219s 0 3f89d0d8-6162-4362-852a-cb688d6c0696\n", 231) = 231 +write(1, "\n", 1) = 1 +read(0, "refs/heads/git-annex:d28/166/SHA256E-s59188--9c04581bd67ea7c78b537a164d104bea5ac91a4a69f06b477cf07892a2d9b852.fih.srt.log\n", 4096) = 122 +read(3, "0936a1fdd849b8b46abb879d7cf82cc758b367e3\n", 255) = 41 +read(3, "", 214) = 0 +write(1, "d95950acebb5c4318329d7b989d36d01b76b7801 blob 232\n", 50) = 50 +write(1, "1396057825.366657s 1 3f89d0d8-6162-4362-852a-cb688d6c0696\n1396057866.538068s 1 161c3d7f-eb60-4294-84d2-eb611786c91e\n1396057866.560144s 1 161c3d7f-eb60-4294-84d2-eb611786c91e\n1396058160.538542s 0 3f89d0d8-6162-4362-852a-cb688d6c0696\n", 232) = 232 +write(1, "\n", 1) = 1 +read(0, "refs/heads/git-annex:ee2/2ee/SHA256E-s106800355--c70e31d511e7eec4881a15dfba521ea3d1fe14694968f81ae1819f1a2a93f9be.mp4.log\n", 4096) = 122 +read(3, "0936a1fdd849b8b46abb879d7cf82cc758b367e3\n", 255) = 41 +read(3, "", 214) = 0 +write(1, "977039ea431522e6e27a78bdec2c1299f883eb85 blob 232\n", 50) = 50 +write(1, "1396057823.999737s 1 3f89d0d8-6162-4362-852a-cb688d6c0696\n1396057835.133409s 1 161c3d7f-eb60-4294-84d2-eb611786c91e\n1396057835.215084s 1 161c3d7f-eb60-4294-84d2-eb611786c91e\n1396058160.468307s 0 3f89d0d8-6162-4362-852a-cb688d6c0696\n", 232) = 232 +write(1, "\n", 1) = 1 +read(0, "refs/heads/git-annex:ee2/2ee/SHA256E-s106800355--c70e31d511e7eec4881a15dfba521ea3d1fe14694968f81ae1819f1a2a93f9be.mp4.log\n", 4096) = 122 +read(3, "0936a1fdd849b8b46abb879d7cf82cc758b367e3\n", 255) = 41 +read(3, "", 214) = 0 +write(1, "977039ea431522e6e27a78bdec2c1299f883eb85 blob 232\n", 50) = 50 +write(1, "1396057823.999737s 1 3f89d0d8-6162-4362-852a-cb688d6c0696\n1396057835.133409s 1 161c3d7f-eb60-4294-84d2-eb611786c91e\n1396057835.215084s 1 161c3d7f-eb60-4294-84d2-eb611786c91e\n1396058160.468307s 0 3f89d0d8-6162-4362-852a-cb688d6c0696\n", 232) = 232 +write(1, "\n", 1) = 1 +read(0, "refs/heads/git-annex:a84/f1f/SHA256E-s47051987--dcfd0413db883506ccb8c45e3b2d60cb3ff5c83cc55c9c7e44818d7556dbc07f.mp4.log\n", 4096) = 121 +read(3, "0936a1fdd849b8b46abb879d7cf82cc758b367e3\n", 255) = 41 +read(3, "", 214) = 0 +write(1, "6c0ff555a1a34337c9379d8856c8283429bef973 blob 231\n", 50) = 50 +write(1, "1396057829.505426s 1 3f89d0d8-6162-4362-852a-cb688d6c0696\n1396057839.859236s 1 161c3d7f-eb60-4294-84d2-eb611786c91e\n1396057839.875213s 1 161c3d7f-eb60-4294-84d2-eb611786c91e\n1396058160.77741s 0 3f89d0d8-6162-4362-852a-cb688d6c0696\n", 231) = 231 +write(1, "\n", 1) = 1 +read(0, "refs/heads/git-annex:50a/5fc/SHA256E-s275654757--52823cd2061375910ccbd8de38865eca91511d9b4621243d2ef96a974d7546aa.flv.log\n", 4096) = 122 +read(3, "0936a1fdd849b8b46abb879d7cf82cc758b367e3\n", 255) = 41 +read(3, "", 214) = 0 +write(1, "cdbc7ce6b426dfcce9d718387b6c412e870a2d12 blob 232\n", 50) = 50 +write(1, "1396057828.887576s 1 3f89d0d8-6162-4362-852a-cb688d6c0696\n1396057866.117938s 1 161c3d7f-eb60-4294-84d2-eb611786c91e\n1396057866.197196s 1 161c3d7f-eb60-4294-84d2-eb611786c91e\n1396058160.687354s 0 3f89d0d8-6162-4362-852a-cb688d6c0696\n", 232) = 232 +write(1, "\n", 1) = 1 +read(0, "refs/heads/git-annex:9b5/545/SHA256E-s32710--6005b5faf1a6d42d499053f8cca87d080536abfa8442b33c87f7966e86726e4f.fin.srt.log\n", 4096) = 122 +read(3, "0936a1fdd849b8b46abb879d7cf82cc758b367e3\n", 255) = 41 +read(3, "", 214) = 0 +write(1, "82a98cbfe8f24d336a537cecea2182922c4681e1 blob 231\n", 50) = 50 +write(1, "1396057825.390306s 1 3f89d0d8-6162-4362-852a-cb688d6c0696\n1396057866.37922s 1 161c3d7f-eb60-4294-84d2-eb611786c91e\n1396057866.386029s 1 161c3d7f-eb60-4294-84d2-eb611786c91e\n1396058160.588219s 0 3f89d0d8-6162-4362-852a-cb688d6c0696\n", 231) = 231 +write(1, "\n", 1) = 1 +read(0, "refs/heads/git-annex:d28/166/SHA256E-s59188--9c04581bd67ea7c78b537a164d104bea5ac91a4a69f06b477cf07892a2d9b852.fih.srt.log\n", 4096) = 122 +read(3, "0936a1fdd849b8b46abb879d7cf82cc758b367e3\n", 255) = 41 +read(3, "", 214) = 0 +write(1, "d95950acebb5c4318329d7b989d36d01b76b7801 blob 232\n", 50) = 50 +write(1, "1396057825.366657s 1 3f89d0d8-6162-4362-852a-cb688d6c0696\n1396057866.538068s 1 161c3d7f-eb60-4294-84d2-eb611786c91e\n1396057866.560144s 1 161c3d7f-eb60-4294-84d2-eb611786c91e\n1396058160.538542s 0 3f89d0d8-6162-4362-852a-cb688d6c0696\n", 232) = 232 +write(1, "\n", 1) = 1 +read(0, "refs/heads/git-annex:ee2/2ee/SHA256E-s106800355--c70e31d511e7eec4881a15dfba521ea3d1fe14694968f81ae1819f1a2a93f9be.mp4.log\n", 4096) = 122 +read(3, "0936a1fdd849b8b46abb879d7cf82cc758b367e3\n", 255) = 41 +read(3, "", 214) = 0 +write(1, "977039ea431522e6e27a78bdec2c1299f883eb85 blob 232\n", 50) = 50 +write(1, "1396057823.999737s 1 3f89d0d8-6162-4362-852a-cb688d6c0696\n1396057835.133409s 1 161c3d7f-eb60-4294-84d2-eb611786c91e\n1396057835.215084s 1 161c3d7f-eb60-4294-84d2-eb611786c91e\n1396058160.468307s 0 3f89d0d8-6162-4362-852a-cb688d6c0696\n", 232) = 232 +write(1, "\n", 1) = 1 +read(0, "refs/heads/git-annex:ee2/2ee/SHA256E-s106800355--c70e31d511e7eec4881a15dfba521ea3d1fe14694968f81ae1819f1a2a93f9be.mp4.log\n", 4096) = 122 +read(3, "0936a1fdd849b8b46abb879d7cf82cc758b367e3\n", 255) = 41 +read(3, "", 214) = 0 +write(1, "977039ea431522e6e27a78bdec2c1299f883eb85 blob 232\n", 50) = 50 +write(1, "1396057823.999737s 1 3f89d0d8-6162-4362-852a-cb688d6c0696\n1396057835.133409s 1 161c3d7f-eb60-4294-84d2-eb611786c91e\n1396057835.215084s 1 161c3d7f-eb60-4294-84d2-eb611786c91e\n1396058160.468307s 0 3f89d0d8-6162-4362-852a-cb688d6c0696\n", 232) = 232 +write(1, "\n", 1) = 1 +read(0, "refs/heads/git-annex:a84/f1f/SHA256E-s47051987--dcfd0413db883506ccb8c45e3b2d60cb3ff5c83cc55c9c7e44818d7556dbc07f.mp4.log\n", 4096) = 121 +read(3, "0936a1fdd849b8b46abb879d7cf82cc758b367e3\n", 255) = 41 +read(3, "", 214) = 0 +write(1, "6c0ff555a1a34337c9379d8856c8283429bef973 blob 231\n", 50) = 50 +write(1, "1396057829.505426s 1 3f89d0d8-6162-4362-852a-cb688d6c0696\n1396057839.859236s 1 161c3d7f-eb60-4294-84d2-eb611786c91e\n1396057839.875213s 1 161c3d7f-eb60-4294-84d2-eb611786c91e\n1396058160.77741s 0 3f89d0d8-6162-4362-852a-cb688d6c0696\n", 231) = 231 +write(1, "\n", 1) = 1 +read(0, "refs/heads/git-annex:50a/5fc/SHA256E-s275654757--52823cd2061375910ccbd8de38865eca91511d9b4621243d2ef96a974d7546aa.flv.log\n", 4096) = 122 +read(3, "0936a1fdd849b8b46abb879d7cf82cc758b367e3\n", 255) = 41 +read(3, "", 214) = 0 +write(1, "cdbc7ce6b426dfcce9d718387b6c412e870a2d12 blob 232\n", 50) = 50 +write(1, "1396057828.887576s 1 3f89d0d8-6162-4362-852a-cb688d6c0696\n1396057866.117938s 1 161c3d7f-eb60-4294-84d2-eb611786c91e\n1396057866.197196s 1 161c3d7f-eb60-4294-84d2-eb611786c91e\n1396058160.687354s 0 3f89d0d8-6162-4362-852a-cb688d6c0696\n", 232) = 232 +write(1, "\n", 1) = 1 +read(0, "refs/heads/git-annex:9b5/545/SHA256E-s32710--6005b5faf1a6d42d499053f8cca87d080536abfa8442b33c87f7966e86726e4f.fin.srt.log\n", 4096) = 122 +read(3, "0936a1fdd849b8b46abb879d7cf82cc758b367e3\n", 255) = 41 +read(3, "", 214) = 0 +write(1, "82a98cbfe8f24d336a537cecea2182922c4681e1 blob 231\n", 50) = 50 +write(1, "1396057825.390306s 1 3f89d0d8-6162-4362-852a-cb688d6c0696\n1396057866.37922s 1 161c3d7f-eb60-4294-84d2-eb611786c91e\n1396057866.386029s 1 161c3d7f-eb60-4294-84d2-eb611786c91e\n1396058160.588219s 0 3f89d0d8-6162-4362-852a-cb688d6c0696\n", 231) = 231 +write(1, "\n", 1) = 1 +read(0, "refs/heads/git-annex:d28/166/SHA256E-s59188--9c04581bd67ea7c78b537a164d104bea5ac91a4a69f06b477cf07892a2d9b852.fih.srt.log\n", 4096) = 122 +read(3, "0936a1fdd849b8b46abb879d7cf82cc758b367e3\n", 255) = 41 +read(3, "", 214) = 0 +write(1, "d95950acebb5c4318329d7b989d36d01b76b7801 blob 232\n", 50) = 50 +write(1, "1396057825.366657s 1 3f89d0d8-6162-4362-852a-cb688d6c0696\n1396057866.538068s 1 161c3d7f-eb60-4294-84d2-eb611786c91e\n1396057866.560144s 1 161c3d7f-eb60-4294-84d2-eb611786c91e\n1396058160.538542s 0 3f89d0d8-6162-4362-852a-cb688d6c0696\n", 232) = 232 +write(1, "\n", 1) = 1 +read(0, "refs/heads/git-annex:ee2/2ee/SHA256E-s106800355--c70e31d511e7eec4881a15dfba521ea3d1fe14694968f81ae1819f1a2a93f9be.mp4.log\n", 4096) = 122 +read(3, "0936a1fdd849b8b46abb879d7cf82cc758b367e3\n", 255) = 41 +read(3, "", 214) = 0 +write(1, "977039ea431522e6e27a78bdec2c1299f883eb85 blob 232\n", 50) = 50 +write(1, "1396057823.999737s 1 3f89d0d8-6162-4362-852a-cb688d6c0696\n1396057835.133409s 1 161c3d7f-eb60-4294-84d2-eb611786c91e\n1396057835.215084s 1 161c3d7f-eb60-4294-84d2-eb611786c91e\n1396058160.468307s 0 3f89d0d8-6162-4362-852a-cb688d6c0696\n", 232) = 232 +write(1, "\n", 1) = 1 +read(0, "refs/heads/git-annex:ee2/2ee/SHA256E-s106800355--c70e31d511e7eec4881a15dfba521ea3d1fe14694968f81ae1819f1a2a93f9be.mp4.log\n", 4096) = 122 +read(3, "0936a1fdd849b8b46abb879d7cf82cc758b367e3\n", 255) = 41 +read(3, "", 214) = 0 +write(1, "977039ea431522e6e27a78bdec2c1299f883eb85 blob 232\n", 50) = 50 +write(1, "1396057823.999737s 1 3f89d0d8-6162-4362-852a-cb688d6c0696\n1396057835.133409s 1 161c3d7f-eb60-4294-84d2-eb611786c91e\n1396057835.215084s 1 161c3d7f-eb60-4294-84d2-eb611786c91e\n1396058160.468307s 0 3f89d0d8-6162-4362-852a-cb688d6c0696\n", 232) = 232 +write(1, "\n", 1) = 1 +read(0, "refs/heads/git-annex:a84/f1f/SHA256E-s47051987--dcfd0413db883506ccb8c45e3b2d60cb3ff5c83cc55c9c7e44818d7556dbc07f.mp4.log\n", 4096) = 121 +read(3, "0936a1fdd849b8b46abb879d7cf82cc758b367e3\n", 255) = 41 +read(3, "", 214) = 0 +write(1, "6c0ff555a1a34337c9379d8856c8283429bef973 blob 231\n", 50) = 50 +write(1, "1396057829.505426s 1 3f89d0d8-6162-4362-852a-cb688d6c0696\n1396057839.859236s 1 161c3d7f-eb60-4294-84d2-eb611786c91e\n1396057839.875213s 1 161c3d7f-eb60-4294-84d2-eb611786c91e\n1396058160.77741s 0 3f89d0d8-6162-4362-852a-cb688d6c0696\n", 231) = 231 +write(1, "\n", 1) = 1 +read(0, "refs/heads/git-annex:50a/5fc/SHA256E-s275654757--52823cd2061375910ccbd8de38865eca91511d9b4621243d2ef96a974d7546aa.flv.log\n", 4096) = 122 +read(3, "0936a1fdd849b8b46abb879d7cf82cc758b367e3\n", 255) = 41 +read(3, "", 214) = 0 +write(1, "cdbc7ce6b426dfcce9d718387b6c412e870a2d12 blob 232\n", 50) = 50 +write(1, "1396057828.887576s 1 3f89d0d8-6162-4362-852a-cb688d6c0696\n1396057866.117938s 1 161c3d7f-eb60-4294-84d2-eb611786c91e\n1396057866.197196s 1 161c3d7f-eb60-4294-84d2-eb611786c91e\n1396058160.687354s 0 3f89d0d8-6162-4362-852a-cb688d6c0696\n", 232) = 232 +write(1, "\n", 1) = 1 +read(0, "refs/heads/git-annex:9b5/545/SHA256E-s32710--6005b5faf1a6d42d499053f8cca87d080536abfa8442b33c87f7966e86726e4f.fin.srt.log\n", 4096) = 122 +read(3, "0936a1fdd849b8b46abb879d7cf82cc758b367e3\n", 255) = 41 +read(3, "", 214) = 0 +write(1, "82a98cbfe8f24d336a537cecea2182922c4681e1 blob 231\n", 50) = 50 +write(1, "1396057825.390306s 1 3f89d0d8-6162-4362-852a-cb688d6c0696\n1396057866.37922s 1 161c3d7f-eb60-4294-84d2-eb611786c91e\n1396057866.386029s 1 161c3d7f-eb60-4294-84d2-eb611786c91e\n1396058160.588219s 0 3f89d0d8-6162-4362-852a-cb688d6c0696\n", 231) = 231 +write(1, "\n", 1) = 1 +read(0, "refs/heads/git-annex:d28/166/SHA256E-s59188--9c04581bd67ea7c78b537a164d104bea5ac91a4a69f06b477cf07892a2d9b852.fih.srt.log\n", 4096) = 122 +read(3, "0936a1fdd849b8b46abb879d7cf82cc758b367e3\n", 255) = 41 +"""]] diff --git a/doc/bugs/Assistant_having_a_child_git_cat-file_--batch_do_the_same_thing_over_and_over_and_using_a_lot_of_memory/comment_10_edb2428552cf98bfb1735c2d6daf2b20._comment b/doc/bugs/Assistant_having_a_child_git_cat-file_--batch_do_the_same_thing_over_and_over_and_using_a_lot_of_memory/comment_10_edb2428552cf98bfb1735c2d6daf2b20._comment new file mode 100644 index 0000000000..d5f7ce363b --- /dev/null +++ b/doc/bugs/Assistant_having_a_child_git_cat-file_--batch_do_the_same_thing_over_and_over_and_using_a_lot_of_memory/comment_10_edb2428552cf98bfb1735c2d6daf2b20._comment @@ -0,0 +1,8 @@ +[[!comment format=mdwn + username="http://johan.kiviniemi.name/" + nickname="Johan" + subject="comment 10" + date="2014-05-01T01:33:10Z" + content=""" +Note to self: I experienced this bug with the standalone tarball release (5.20140421) as well, so it’s not caused by something that is different on my system wrt. git-annex’s dependencies etc. +"""]] diff --git a/doc/bugs/Assistant_having_a_child_git_cat-file_--batch_do_the_same_thing_over_and_over_and_using_a_lot_of_memory/comment_1_ac8c39e362e6c806b9d68befc0199ccd._comment b/doc/bugs/Assistant_having_a_child_git_cat-file_--batch_do_the_same_thing_over_and_over_and_using_a_lot_of_memory/comment_1_ac8c39e362e6c806b9d68befc0199ccd._comment new file mode 100644 index 0000000000..e9c5d3fff8 --- /dev/null +++ b/doc/bugs/Assistant_having_a_child_git_cat-file_--batch_do_the_same_thing_over_and_over_and_using_a_lot_of_memory/comment_1_ac8c39e362e6c806b9d68befc0199ccd._comment @@ -0,0 +1,10 @@ +[[!comment format=mdwn + username="http://joeyh.name/" + ip="209.250.56.244" + subject="comment 1" + date="2014-04-02T18:48:51Z" + content=""" +All I can tell from the strace is that it's looking at location logs, and it's looking at the same few keys, but not a single on in a tight loop. + +It would probably help a lot to run the assistant with --debug and get a debug log while this is going on. We need to pinpoint the part of the assistant that is affected, and there may be other activity too. +"""]] diff --git a/doc/bugs/Assistant_having_a_child_git_cat-file_--batch_do_the_same_thing_over_and_over_and_using_a_lot_of_memory/comment_2_b2941bf7901a1b2237b7210c8f0af2a5._comment b/doc/bugs/Assistant_having_a_child_git_cat-file_--batch_do_the_same_thing_over_and_over_and_using_a_lot_of_memory/comment_2_b2941bf7901a1b2237b7210c8f0af2a5._comment new file mode 100644 index 0000000000..c9e798c28c --- /dev/null +++ b/doc/bugs/Assistant_having_a_child_git_cat-file_--batch_do_the_same_thing_over_and_over_and_using_a_lot_of_memory/comment_2_b2941bf7901a1b2237b7210c8f0af2a5._comment @@ -0,0 +1,22 @@ +[[!comment format=mdwn + username="http://joeyh.name/" + ip="209.250.56.244" + subject="comment 2" + date="2014-04-07T21:07:35Z" + content=""" +Except of log when this apparently happened. Note the 6 minute time discontinuity when it was apparently looping: + +
    +[2014-04-07 23:24:24 EEST] read: git [\"--git-dir=/storage/sarjat/annex-sarjat/.git\",\"--work-tree=/storage/sarjat/annex-sarjat\",\"write-tree\"]
+[2014-04-07 23:24:24 EEST] read: git [\"--git-dir=/storage/sarjat/annex-sarjat/.git\",\"--work-tree=/storage/sarjat/annex-sarjat\",\"rev-parse\",\"84068090af4bcd3d24f16d865ac07b0478f20ada:\"]
+[2014-04-07 23:24:24 EEST] read: git [\"--git-dir=/storage/sarjat/annex-sarjat/.git\",\"--work-tree=/storage/sarjat/annex-sarjat\",\"symbolic-ref\",\"HEAD\"]
+[2014-04-07 23:24:24 EEST] read: git [\"--git-dir=/storage/sarjat/annex-sarjat/.git\",\"--work-tree=/storage/sarjat/annex-sarjat\",\"show-ref\",\"refs/heads/master\"]
+[2014-04-07 23:30:13 EEST] read: git [\"--git-dir=/storage/sarjat/annex-sarjat/.git\",\"--work-tree=/storage/sarjat/annex-sarjat\",\"show-ref\",\"git-annex\"]
+[2014-04-07 23:30:13 EEST] read: git [\"--git-dir=/storage/sarjat/annex-sarjat/.git\",\"--work-tree=/storage/sarjat/annex-sarjat\",\"show-ref\",\"--hash\",\"refs/heads/git-annex\"]
+[2014-04-07 23:30:13 EEST] read: git [\"--git-dir=/storage/sarjat/annex-sarjat/.git\",\"--work-tree=/storage/sarjat/annex-sarjat\",\"log\",\"refs/heads/git-annex..214ed317536695b91c8dd5bed059c46c11ad00be\",\"--oneline\",\"-n1\"]
+
    + +Also probably relevant, the network topology AIUI was: `client --> server` where both nodes ran the assistant. This happened on the server shortly after the client dropped off a refs/heads/synced/master. + +(Also, the \"logging to a deleted file\" appears to have been a local misconfiguration; a cron job that repeatedly tried to start the assistant. Only one will start, but later ones will rotate the logs before noticing it's running and giving up.) +"""]] diff --git a/doc/bugs/Assistant_having_a_child_git_cat-file_--batch_do_the_same_thing_over_and_over_and_using_a_lot_of_memory/comment_3_1429ca784a03bc424b3537cbe0449421._comment b/doc/bugs/Assistant_having_a_child_git_cat-file_--batch_do_the_same_thing_over_and_over_and_using_a_lot_of_memory/comment_3_1429ca784a03bc424b3537cbe0449421._comment new file mode 100644 index 0000000000..fd9fa89715 --- /dev/null +++ b/doc/bugs/Assistant_having_a_child_git_cat-file_--batch_do_the_same_thing_over_and_over_and_using_a_lot_of_memory/comment_3_1429ca784a03bc424b3537cbe0449421._comment @@ -0,0 +1,23 @@ +[[!comment format=mdwn + username="http://joeyh.name/" + ip="209.250.56.244" + subject="comment 3" + date="2014-04-07T21:55:19Z" + content=""" +Unfortunately all I have been able to tell for sure from this log is that it seems that the expensive transfer scan is not running, and this is unlikely to be a repository auto-repair. + +My best guess as to what might be going on is an update of the git-annex branch. + +[2014-04-07 23:24:24 EEST] read: git [\"--git-dir=/storage/sarjat/annex-sarjat/.git\",\"--work-tree=/storage/sarjat/annex-sarjat\",\"write-tree\"] + +This is prep for an index file commit, probably to the git-annex branch. + +[2014-04-07 23:24:24 EEST] read: git [\"--git-dir=/storage/sarjat/annex-sarjat/.git\",\"--work-tree=/storage/sarjat/annex-sarjat\",\"rev-parse\",\"84068090af4bcd3d24f16d865ac07b0478f20ada:\"] + +This is a getting the parent commit's tree. + +The git-cat-file churn could then be a union merge reading the contents of the git-annex branch to union-merge it into the `.git/annex/index` (in `mergeIndex`). This would reuse the main git cat-file process. + +That does not explain why it would need to read eg, SHA256E-s106800355--c70e31d511e7eec4881a15dfba521ea3d1fe14694968f81ae1819f1a2a93f9be.mp4.log 28 times. +Normally, during a union merge only files listed by `diff-index` need to be read, and it lists each file only once. +"""]] diff --git a/doc/bugs/Assistant_having_a_child_git_cat-file_--batch_do_the_same_thing_over_and_over_and_using_a_lot_of_memory/comment_4_f9e65cf5598b4b14eeee1f41f46d4084._comment b/doc/bugs/Assistant_having_a_child_git_cat-file_--batch_do_the_same_thing_over_and_over_and_using_a_lot_of_memory/comment_4_f9e65cf5598b4b14eeee1f41f46d4084._comment new file mode 100644 index 0000000000..148920c233 --- /dev/null +++ b/doc/bugs/Assistant_having_a_child_git_cat-file_--batch_do_the_same_thing_over_and_over_and_using_a_lot_of_memory/comment_4_f9e65cf5598b4b14eeee1f41f46d4084._comment @@ -0,0 +1,12 @@ +[[!comment format=mdwn + username="http://joeyh.name/" + ip="209.250.56.244" + subject="comment 4" + date="2014-04-07T21:57:32Z" + content=""" +Does `git log git-annex` show a commit that was made at 23:30? + +Does it show a commit 84068090af4bcd3d24f16d865ac07b0478f20ada? + +Is 84068090af4bcd3d24f16d865ac07b0478f20ada the parent of the 23:30 commit? +"""]] diff --git a/doc/bugs/Assistant_having_a_child_git_cat-file_--batch_do_the_same_thing_over_and_over_and_using_a_lot_of_memory/comment_5_044ecac2d2e670e1ef69809c944093d1._comment b/doc/bugs/Assistant_having_a_child_git_cat-file_--batch_do_the_same_thing_over_and_over_and_using_a_lot_of_memory/comment_5_044ecac2d2e670e1ef69809c944093d1._comment new file mode 100644 index 0000000000..6013650041 --- /dev/null +++ b/doc/bugs/Assistant_having_a_child_git_cat-file_--batch_do_the_same_thing_over_and_over_and_using_a_lot_of_memory/comment_5_044ecac2d2e670e1ef69809c944093d1._comment @@ -0,0 +1,19 @@ +[[!comment format=mdwn + username="http://joeyh.name/" + ip="209.250.56.244" + subject="comment 5" + date="2014-04-07T22:12:14Z" + content=""" +Is the repository using direct mode? + +Another theory is that: + +* test/hello appears +* watcher sees new symlink, tries to make a commit with it +* master branch already has that symlink +* this is why the write-tree is not followed by a commit-tree. The commit would have been empty. + +If this is the case, then 84068090af4bcd3d24f16d865ac07b0478f20ada will be a ref on the master branch. + +And all of the above is normal operation. But it does suggest, that if this repo is in direct mode, it might be running a direct mode work tree update around then. Which requires a lot of cat-file queries of the git-annex branch. And would certainly make repeated queries at least if the repository has duplicate copies of some files.. +"""]] diff --git a/doc/bugs/Assistant_having_a_child_git_cat-file_--batch_do_the_same_thing_over_and_over_and_using_a_lot_of_memory/comment_6_6f4f51e1583bed5e7e601e4f30f4207b._comment b/doc/bugs/Assistant_having_a_child_git_cat-file_--batch_do_the_same_thing_over_and_over_and_using_a_lot_of_memory/comment_6_6f4f51e1583bed5e7e601e4f30f4207b._comment new file mode 100644 index 0000000000..0b5f904894 --- /dev/null +++ b/doc/bugs/Assistant_having_a_child_git_cat-file_--batch_do_the_same_thing_over_and_over_and_using_a_lot_of_memory/comment_6_6f4f51e1583bed5e7e601e4f30f4207b._comment @@ -0,0 +1,8 @@ +[[!comment format=mdwn + username="http://joeyh.name/" + ip="209.250.56.244" + subject="comment 6" + date="2014-04-07T22:17:16Z" + content=""" +Does the git log have any recent commits that were \"git-annex automatic merge conflict fix\" ? +"""]] diff --git a/doc/bugs/Assistant_having_a_child_git_cat-file_--batch_do_the_same_thing_over_and_over_and_using_a_lot_of_memory/comment_7_683a0a3d4caea0ee625e41ae8a6c7c06._comment b/doc/bugs/Assistant_having_a_child_git_cat-file_--batch_do_the_same_thing_over_and_over_and_using_a_lot_of_memory/comment_7_683a0a3d4caea0ee625e41ae8a6c7c06._comment new file mode 100644 index 0000000000..1705e738cb --- /dev/null +++ b/doc/bugs/Assistant_having_a_child_git_cat-file_--batch_do_the_same_thing_over_and_over_and_using_a_lot_of_memory/comment_7_683a0a3d4caea0ee625e41ae8a6c7c06._comment @@ -0,0 +1,81 @@ +[[!comment format=mdwn + username="http://johan.kiviniemi.name/" + nickname="Johan" + subject="comment 7" + date="2014-04-07T22:44:33Z" + content=""" +In the git-annex branch, there is + +* [[!toggle id=\"4deec8203e0baf7bb5b7d5d868d82439261ab3bc\" text=\"a commit at 23:21:51\"]] from my desktop box where I added `test/hello` + +[[!toggleable id=\"4deec8203e0baf7bb5b7d5d868d82439261ab3bc\" text=\"\"\" + commit 4deec8203e0baf7bb5b7d5d868d82439261ab3bc + Author: Johan Kiviniemi + Date: Mon Apr 7 23:21:51 2014 +0300 + + update + + diff --git a/992/280/SHA256E-s6--5891b5b522d5df086d0ff0b110fbd9d21bb4fc7163af34d08286a2e846f6be03.log b/992/280/SHA256E-s6--5891b5b522d5df086d0ff0b110fbd9d21bb4fc7163af34d08286a2e846f6be03.log + new file mode 100644 + index 0000000..1cf060c + --- /dev/null + +++ b/992/280/SHA256E-s6--5891b5b522d5df086d0ff0b110fbd9d21bb4fc7163af34d08286a2e846f6be03.log + @@ -0,0 +1 @@ + +1396902111.893785s 1 86e07a59-8bba-4878-8d0b-5dfe8c6366c4 +\"\"\"]] + +* [[!toggle id=\"2e0884d9c8859339855ceee396b9ea9ae05865b4\" text=\"a commit at 23:21:54\"]] when the desktop box synced to the server (from which the log excerpt came) + +[[!toggleable id=\"2e0884d9c8859339855ceee396b9ea9ae05865b4\" text=\"\"\" + commit 2e0884d9c8859339855ceee396b9ea9ae05865b4 + Author: Johan Kiviniemi + Date: Mon Apr 7 23:21:54 2014 +0300 + + update + + diff --git a/992/280/SHA256E-s6--5891b5b522d5df086d0ff0b110fbd9d21bb4fc7163af34d08286a2e84 + 6f6be03.log b/992/280/SHA256E-s6--5891b5b522d5df086d0ff0b110fbd9d21bb4fc7163af34d08286a2e8 + 46f6be03.log + index 1cf060c..cd0bccc 100644 + --- a/992/280/SHA256E-s6--5891b5b522d5df086d0ff0b110fbd9d21bb4fc7163af34d08286a2e846f6be03 + .log + +++ b/992/280/SHA256E-s6--5891b5b522d5df086d0ff0b110fbd9d21bb4fc7163af34d08286a2e846f6be03 + .log + @@ -1 +1,2 @@ + +1396902112.657779s 1 09ada430-8802-47da-bbfa-f5256a3c55d2 + 1396902111.893785s 1 86e07a59-8bba-4878-8d0b-5dfe8c6366c4 +\"\"\"]] + +* [[!toggle id=\"214ed317536695b91c8dd5bed059c46c11ad00be\" text=\"a commit at 23:24:24\"]] (2.5 minutes later!) when the assistant on the server finally merged `synced/git-annex` into `git-annex` (`test/hello` became visible in the working tree at that time). + +[[!toggleable id=\"214ed317536695b91c8dd5bed059c46c11ad00be\" text=\"\"\" + commit 214ed317536695b91c8dd5bed059c46c11ad00be + Merge: 4deec82 2e0884d + Author: sarjat + Date: Mon Apr 7 23:24:24 2014 +0300 + + merging synced/git-annex into git-annex +\"\"\"]] + +There is no commit in the `git-annex` branch at 23:30. The next commit is from unrelated changes at 00:06. + +[[!toggle id=\"84068090af4bcd3d24f16d865ac07b0478f20ada\" text=\"84068090af4bcd3d24f16d865ac07b0478f20ada\"]] is the commit in `master` which added `test/hello` at 23:21:51. + +[[!toggleable id=\"84068090af4bcd3d24f16d865ac07b0478f20ada\" text=\"\"\" + commit 84068090af4bcd3d24f16d865ac07b0478f20ada + Author: Johan Kiviniemi + Date: Mon Apr 7 23:21:51 2014 +0300 + + diff --git a/test/hello b/test/hello + new file mode 120000 + index 0000000..8c2678f + --- /dev/null + +++ b/test/hello + @@ -0,0 +1 @@ + +../.git/annex/objects/zK/02/SHA256E-s6--5891b5b522d5df086d0ff0b110fbd9d21bb4fc7163af34d08 + \ No newline at end of file +\"\"\"]] + +The repository on the server is in indirect mode. + +"""]] diff --git a/doc/bugs/Assistant_having_a_child_git_cat-file_--batch_do_the_same_thing_over_and_over_and_using_a_lot_of_memory/comment_8_03dd76b01f46a7cc66eddac3e054c8ad._comment b/doc/bugs/Assistant_having_a_child_git_cat-file_--batch_do_the_same_thing_over_and_over_and_using_a_lot_of_memory/comment_8_03dd76b01f46a7cc66eddac3e054c8ad._comment new file mode 100644 index 0000000000..d6aa43da05 --- /dev/null +++ b/doc/bugs/Assistant_having_a_child_git_cat-file_--batch_do_the_same_thing_over_and_over_and_using_a_lot_of_memory/comment_8_03dd76b01f46a7cc66eddac3e054c8ad._comment @@ -0,0 +1,8 @@ +[[!comment format=mdwn + username="http://johan.kiviniemi.name/" + nickname="Johan" + subject="comment 8" + date="2014-04-07T22:48:18Z" + content=""" +There are no commits in `master` or `git-annex` that have the word conflict in the description. +"""]] diff --git a/doc/bugs/Assistant_having_a_child_git_cat-file_--batch_do_the_same_thing_over_and_over_and_using_a_lot_of_memory/comment_9_5f4444f03cbebaa44628288095383679._comment b/doc/bugs/Assistant_having_a_child_git_cat-file_--batch_do_the_same_thing_over_and_over_and_using_a_lot_of_memory/comment_9_5f4444f03cbebaa44628288095383679._comment new file mode 100644 index 0000000000..09383141e0 --- /dev/null +++ b/doc/bugs/Assistant_having_a_child_git_cat-file_--batch_do_the_same_thing_over_and_over_and_using_a_lot_of_memory/comment_9_5f4444f03cbebaa44628288095383679._comment @@ -0,0 +1,89 @@ +[[!comment format=mdwn + username="http://johan.kiviniemi.name/" + nickname="Johan" + subject="comment 9" + date="2014-04-07T22:55:12Z" + content=""" +[[!toggle id=\"excerpt\" text=\"The full log excerpt\"]] which includes the sync from the client and the final messages after the cat-file loop ended and things stabilized (but a memory leak of 30 MB in the git-annex assistant process remained). + +[[!toggleable id=\"excerpt\" text=\"\"\" + [2014-04-04 10:55:00 EEST] main: starting assistant version 5.20140402 + + + + [2014-04-07 23:21:08 EEST] read: git [\"--git-dir=/storage/sarjat/annex-sarjat/.git\",\"--work-tree=/storage/sarjat/annex-sarjat\",\"show-ref\",\"git-annex\"] + [2014-04-07 23:21:08 EEST] read: git [\"--git-dir=/storage/sarjat/annex-sarjat/.git\",\"--work-tree=/storage/sarjat/annex-sarjat\",\"show-ref\",\"--hash\",\"refs/heads/git-annex\"] + [2014-04-07 23:21:08 EEST] read: git [\"--git-dir=/storage/sarjat/annex-sarjat/.git\",\"--work-tree=/storage/sarjat/annex-sarjat\",\"log\",\"refs/heads/git-annex..20d1f5538f6aa430f29ef938f6db045f5a69425d\",\"--oneline\",\"-n1\"] + [2014-04-07 23:21:08 EEST] read: git [\"--git-dir=/storage/sarjat/annex-sarjat/.git\",\"--work-tree=/storage/sarjat/annex-sarjat\",\"log\",\"refs/heads/git-annex..b402a6e7b9268e25dbd9c6a027f4a5258993980d\",\"--oneline\",\"-n1\"] + [2014-04-07 23:21:08 EEST] read: git [\"--git-dir=/storage/sarjat/annex-sarjat/.git\",\"--work-tree=/storage/sarjat/annex-sarjat\",\"log\",\"refs/heads/git-annex..7b18a191d58d779aab5789b923adb09863938ffe\",\"--oneline\",\"-n1\"] + [2014-04-07 23:21:08 EEST] read: git [\"--git-dir=/storage/sarjat/annex-sarjat/.git\",\"--work-tree=/storage/sarjat/annex-sarjat\",\"ls-tree\",\"--full-tree\",\"-z\",\"--\",\"refs/heads/git-annex\",\"uuid.log\",\"remote.log\",\"trust.log\",\"group.log\",\"numcopies.log\",\"schedule.log\",\"preferred-content.log\",\"required-content.log\",\"group-preferred-content.log\"] + [2014-04-07 23:21:52 EEST] TransferWatcher: transfer starting: Download UUID \"86e07a59-8bba-4878-8d0b-5dfe8c6366c4\" SHA256E-s6--5891b5b522d5df086d0ff0b110fbd9d21bb4fc7163af34d08286a2e846f6be03 Nothing + [2014-04-07 23:21:52 EEST] read: git [\"config\",\"--null\",\"--list\"] + [2014-04-07 23:21:52 EEST] TransferWatcher: transfer starting: Download UUID \"86e07a59-8bba-4878-8d0b-5dfe8c6366c4\" test/hello Nothing + [2014-04-07 23:21:52 EEST] read: git [\"config\",\"--null\",\"--list\"] + [2014-04-07 23:21:52 EEST] read: git [\"--git-dir=/storage/sarjat/annex-sarjat/.git\",\"--work-tree=/storage/sarjat/annex-sarjat\",\"show-ref\",\"git-annex\"] + [2014-04-07 23:21:52 EEST] read: git [\"--git-dir=/storage/sarjat/annex-sarjat/.git\",\"--work-tree=/storage/sarjat/annex-sarjat\",\"show-ref\",\"--hash\",\"refs/heads/git-annex\"] + [2014-04-07 23:21:52 EEST] read: git [\"--git-dir=/storage/sarjat/annex-sarjat/.git\",\"--work-tree=/storage/sarjat/annex-sarjat\",\"log\",\"refs/heads/git-annex..20d1f5538f6aa430f29ef938f6db045f5a69425d\",\"--oneline\",\"-n1\"] + [2014-04-07 23:21:52 EEST] read: git [\"--git-dir=/storage/sarjat/annex-sarjat/.git\",\"--work-tree=/storage/sarjat/annex-sarjat\",\"log\",\"refs/heads/git-annex..4deec8203e0baf7bb5b7d5d868d82439261ab3bc\",\"--oneline\",\"-n1\"] + [2014-04-07 23:21:52 EEST] read: git [\"--git-dir=/storage/sarjat/annex-sarjat/.git\",\"--work-tree=/storage/sarjat/annex-sarjat\",\"log\",\"refs/heads/git-annex..b402a6e7b9268e25dbd9c6a027f4a5258993980d\",\"--oneline\",\"-n1\"] + [2014-04-07 23:21:52 EEST] read: git [\"--git-dir=/storage/sarjat/annex-sarjat/.git\",\"--work-tree=/storage/sarjat/annex-sarjat\",\"log\",\"refs/heads/git-annex..7b18a191d58d779aab5789b923adb09863938ffe\",\"--oneline\",\"-n1\"] + [2014-04-07 23:21:52 EEST] feed: git [\"--git-dir=/storage/sarjat/annex-sarjat/.git\",\"--work-tree=/storage/sarjat/annex-sarjat\",\"update-index\",\"-z\",\"--index-info\"] + [2014-04-07 23:21:52 EEST] TransferWatcher: transfer finishing: Transfer {transferDirection = Download, transferUUID = UUID \"86e07a59-8bba-4878-8d0b-5dfe8c6366c4\", transferKey = Key {keyName = \"5891b5b522d5df086d0ff0b110fbd9d21bb4fc7163af34d08286a2e846f6be03\", keyBackendName = \"SHA256E\", keySize = Just 6, keyMtime = Nothing}} + [2014-04-07 23:21:52 EEST] read: git [\"--git-dir=/storage/sarjat/annex-sarjat/.git\",\"--work-tree=/storage/sarjat/annex-sarjat\",\"diff-index\",\"--raw\",\"-z\",\"-r\",\"--no-renames\",\"-l0\",\"--cached\",\"4deec8203e0baf7bb5b7d5d868d82439261ab3bc\"] + [2014-04-07 23:21:52 EEST] read: git [\"--git-dir=/storage/sarjat/annex-sarjat/.git\",\"--work-tree=/storage/sarjat/annex-sarjat\",\"log\",\"4deec8203e0baf7bb5b7d5d868d82439261ab3bc..refs/heads/git-annex\",\"--oneline\",\"-n1\"] + [2014-04-07 23:21:52 EEST] call: git [\"--git-dir=/storage/sarjat/annex-sarjat/.git\",\"--work-tree=/storage/sarjat/annex-sarjat\",\"update-ref\",\"refs/heads/git-annex\",\"4deec8203e0baf7bb5b7d5d868d82439261ab3bc\"] + [2014-04-07 23:22:08 EEST] read: git [\"--git-dir=/storage/sarjat/annex-sarjat/.git\",\"--work-tree=/storage/sarjat/annex-sarjat\",\"ls-tree\",\"--full-tree\",\"-z\",\"--\",\"refs/heads/git-annex\",\"uuid.log\",\"remote.log\",\"trust.log\",\"group.log\",\"numcopies.log\",\"schedule.log\",\"preferred-content.log\",\"required-content.log\",\"group-preferred-content.log\"] + [2014-04-07 23:24:24 EEST] read: git [\"--git-dir=/storage/sarjat/annex-sarjat/.git\",\"--work-tree=/storage/sarjat/annex-sarjat\",\"symbolic-ref\",\"HEAD\"] + [2014-04-07 23:24:24 EEST] read: git [\"--git-dir=/storage/sarjat/annex-sarjat/.git\",\"--work-tree=/storage/sarjat/annex-sarjat\",\"show-ref\",\"refs/heads/master\"] + [2014-04-07 23:24:24 EEST] Merger: merging refs/heads/synced/master into refs/heads/master + [2014-04-07 23:24:24 EEST] read: git [\"--git-dir=/storage/sarjat/annex-sarjat/.git\",\"--work-tree=/storage/sarjat/annex-sarjat\",\"show-ref\",\"--hash\",\"refs/heads/master\"] + [2014-04-07 23:24:24 EEST] call: git [\"--git-dir=/storage/sarjat/annex-sarjat/.git\",\"--work-tree=/storage/sarjat/annex-sarjat\",\"merge\",\"--no-edit\",\"refs/heads/synced/master\"] + Updating 645e474..8406809 + Fast-forward + test/hello | 1 + + 1 file changed, 1 insertion(+) + create mode 120000 test/hello + [2014-04-07 23:24:24 EEST] read: git [\"--git-dir=/storage/sarjat/annex-sarjat/.git\",\"--work-tree=/storage/sarjat/annex-sarjat\",\"show-ref\",\"git-annex\"] + [2014-04-07 23:24:24 EEST] read: git [\"--git-dir=/storage/sarjat/annex-sarjat/.git\",\"--work-tree=/storage/sarjat/annex-sarjat\",\"show-ref\",\"--hash\",\"refs/heads/git-annex\"] + [2014-04-07 23:24:24 EEST] read: git [\"--git-dir=/storage/sarjat/annex-sarjat/.git\",\"--work-tree=/storage/sarjat/annex-sarjat\",\"log\",\"refs/heads/git-annex..4deec8203e0baf7bb5b7d5d868d82439261ab3bc\",\"--oneline\",\"-n1\"] + [2014-04-07 23:24:24 EEST] read: git [\"--git-dir=/storage/sarjat/annex-sarjat/.git\",\"--work-tree=/storage/sarjat/annex-sarjat\",\"log\",\"refs/heads/git-annex..2e0884d9c8859339855ceee396b9ea9ae05865b4\",\"--oneline\",\"-n1\"] + [2014-04-07 23:24:24 EEST] read: git [\"--git-dir=/storage/sarjat/annex-sarjat/.git\",\"--work-tree=/storage/sarjat/annex-sarjat\",\"log\",\"refs/heads/git-annex..b402a6e7b9268e25dbd9c6a027f4a5258993980d\",\"--oneline\",\"-n1\"] + [2014-04-07 23:24:24 EEST] read: git [\"--git-dir=/storage/sarjat/annex-sarjat/.git\",\"--work-tree=/storage/sarjat/annex-sarjat\",\"log\",\"refs/heads/git-annex..7b18a191d58d779aab5789b923adb09863938ffe\",\"--oneline\",\"-n1\"] + [2014-04-07 23:24:24 EEST] chat: git [\"--git-dir=/storage/sarjat/annex-sarjat/.git\",\"--work-tree=/storage/sarjat/annex-sarjat\",\"hash-object\",\"-w\",\"--stdin-paths\",\"--no-filters\"] + [2014-04-07 23:24:24 EEST] feed: git [\"--git-dir=/storage/sarjat/annex-sarjat/.git\",\"--work-tree=/storage/sarjat/annex-sarjat\",\"update-index\",\"-z\",\"--index-info\"] + [2014-04-07 23:24:24 EEST] feed: git [\"--git-dir=/storage/sarjat/annex-sarjat/.git\",\"--work-tree=/storage/sarjat/annex-sarjat\",\"update-index\",\"-z\",\"--index-info\"] + [2014-04-07 23:24:24 EEST] read: git [\"--git-dir=/storage/sarjat/annex-sarjat/.git\",\"--work-tree=/storage/sarjat/annex-sarjat\",\"diff-index\",\"--raw\",\"-z\",\"-r\",\"--no-renames\",\"-l0\",\"--cached\",\"2e0884d9c8859339855ceee396b9ea9ae05865b4\"] + [2014-04-07 23:24:24 EEST] chat: git [\"--git-dir=/storage/sarjat/annex-sarjat/.git\",\"--work-tree=/storage/sarjat/annex-sarjat\",\"hash-object\",\"-t\",\"blob\",\"-w\",\"--stdin\",\"--no-filters\"] + [2014-04-07 23:24:24 EEST] feed: git [\"--git-dir=/storage/sarjat/annex-sarjat/.git\",\"--work-tree=/storage/sarjat/annex-sarjat\",\"update-index\",\"-z\",\"--index-info\"] + [2014-04-07 23:24:24 EEST] read: git [\"--git-dir=/storage/sarjat/annex-sarjat/.git\",\"--work-tree=/storage/sarjat/annex-sarjat\",\"diff-index\",\"--raw\",\"-z\",\"-r\",\"--no-renames\",\"-l0\",\"--cached\",\"refs/heads/git-annex\"] + [2014-04-07 23:24:24 EEST] read: git [\"--git-dir=/storage/sarjat/annex-sarjat/.git\",\"--work-tree=/storage/sarjat/annex-sarjat\",\"write-tree\"] + [2014-04-07 23:24:24 EEST] chat: git [\"--git-dir=/storage/sarjat/annex-sarjat/.git\",\"--work-tree=/storage/sarjat/annex-sarjat\",\"commit-tree\",\"0bd4352b4008165d356bc9b1250bdb456c675175\",\"-p\",\"refs/heads/git-annex\",\"-p\",\"2e0884d9c8859339855ceee396b9ea9ae05865b4\"] + [2014-04-07 23:24:24 EEST] call: git [\"--git-dir=/storage/sarjat/annex-sarjat/.git\",\"--work-tree=/storage/sarjat/annex-sarjat\",\"update-ref\",\"refs/heads/git-annex\",\"214ed317536695b91c8dd5bed059c46c11ad00be\"] + [2014-04-07 23:24:24 EEST] read: git [\"--git-dir=/storage/sarjat/annex-sarjat/.git\",\"--work-tree=/storage/sarjat/annex-sarjat\",\"ls-tree\",\"--full-tree\",\"-z\",\"--\",\"refs/heads/git-annex\",\"uuid.log\",\"remote.log\",\"trust.log\",\"group.log\",\"numcopies.log\",\"schedule.log\",\"preferred-content.log\",\"required-content.log\",\"group-preferred-content.log\"] + [2014-04-07 23:24:24 EEST] Watcher: add symlink test/hello + [2014-04-07 23:24:24 EEST] chat: git [\"--git-dir=/storage/sarjat/annex-sarjat/.git\",\"--work-tree=/storage/sarjat/annex-sarjat\",\"hash-object\",\"-t\",\"blob\",\"-w\",\"--stdin\",\"--no-filters\"] + [2014-04-07 23:24:24 EEST] Committer: committing 1 changes + [2014-04-07 23:24:24 EEST] Committer: Committing changes to git + [2014-04-07 23:24:24 EEST] feed: git [\"--git-dir=/storage/sarjat/annex-sarjat/.git\",\"--work-tree=/storage/sarjat/annex-sarjat\",\"update-index\",\"-z\",\"--index-info\"] + [2014-04-07 23:24:24 EEST] read: git [\"--git-dir=/storage/sarjat/annex-sarjat/.git\",\"--work-tree=/storage/sarjat/annex-sarjat\",\"symbolic-ref\",\"HEAD\"] + [2014-04-07 23:24:24 EEST] read: git [\"--git-dir=/storage/sarjat/annex-sarjat/.git\",\"--work-tree=/storage/sarjat/annex-sarjat\",\"show-ref\",\"--hash\",\"refs/heads/master\"] + [2014-04-07 23:24:24 EEST] read: git [\"--git-dir=/storage/sarjat/annex-sarjat/.git\",\"--work-tree=/storage/sarjat/annex-sarjat\",\"write-tree\"] + [2014-04-07 23:24:24 EEST] read: git [\"--git-dir=/storage/sarjat/annex-sarjat/.git\",\"--work-tree=/storage/sarjat/annex-sarjat\",\"rev-parse\",\"84068090af4bcd3d24f16d865ac07b0478f20ada:\"] + [2014-04-07 23:24:24 EEST] read: git [\"--git-dir=/storage/sarjat/annex-sarjat/.git\",\"--work-tree=/storage/sarjat/annex-sarjat\",\"symbolic-ref\",\"HEAD\"] + [2014-04-07 23:24:24 EEST] read: git [\"--git-dir=/storage/sarjat/annex-sarjat/.git\",\"--work-tree=/storage/sarjat/annex-sarjat\",\"show-ref\",\"refs/heads/master\"] + [2014-04-07 23:30:13 EEST] read: git [\"--git-dir=/storage/sarjat/annex-sarjat/.git\",\"--work-tree=/storage/sarjat/annex-sarjat\",\"show-ref\",\"git-annex\"] + [2014-04-07 23:30:13 EEST] read: git [\"--git-dir=/storage/sarjat/annex-sarjat/.git\",\"--work-tree=/storage/sarjat/annex-sarjat\",\"show-ref\",\"--hash\",\"refs/heads/git-annex\"] + [2014-04-07 23:30:13 EEST] read: git [\"--git-dir=/storage/sarjat/annex-sarjat/.git\",\"--work-tree=/storage/sarjat/annex-sarjat\",\"log\",\"refs/heads/git-annex..214ed317536695b91c8dd5bed059c46c11ad00be\",\"--oneline\",\"-n1\"] + [2014-04-07 23:30:13 EEST] read: git [\"--git-dir=/storage/sarjat/annex-sarjat/.git\",\"--work-tree=/storage/sarjat/annex-sarjat\",\"log\",\"refs/heads/git-annex..2e0884d9c8859339855ceee396b9ea9ae05865b4\",\"--oneline\",\"-n1\"] + [2014-04-07 23:30:13 EEST] read: git [\"--git-dir=/storage/sarjat/annex-sarjat/.git\",\"--work-tree=/storage/sarjat/annex-sarjat\",\"log\",\"refs/heads/git-annex..b402a6e7b9268e25dbd9c6a027f4a5258993980d\",\"--oneline\",\"-n1\"] + [2014-04-07 23:30:13 EEST] read: git [\"--git-dir=/storage/sarjat/annex-sarjat/.git\",\"--work-tree=/storage/sarjat/annex-sarjat\",\"log\",\"refs/heads/git-annex..7b18a191d58d779aab5789b923adb09863938ffe\",\"--oneline\",\"-n1\"] + [2014-04-07 23:30:13 EEST] read: git [\"--git-dir=/storage/sarjat/annex-sarjat/.git\",\"--work-tree=/storage/sarjat/annex-sarjat\",\"ls-tree\",\"--full-tree\",\"-z\",\"--\",\"refs/heads/git-annex\",\"uuid.log\",\"remote.log\",\"trust.log\",\"group.log\",\"numcopies.log\",\"schedule.log\",\"preferred-content.log\",\"required-content.log\",\"group-preferred-content.log\"] + [2014-04-07 23:30:13 EEST] read: git [\"--git-dir=/storage/sarjat/annex-sarjat/.git\",\"--work-tree=/storage/sarjat/annex-sarjat\",\"show-ref\",\"git-annex\"] + [2014-04-07 23:30:13 EEST] read: git [\"--git-dir=/storage/sarjat/annex-sarjat/.git\",\"--work-tree=/storage/sarjat/annex-sarjat\",\"show-ref\",\"--hash\",\"refs/heads/git-annex\"] + [2014-04-07 23:30:13 EEST] read: git [\"--git-dir=/storage/sarjat/annex-sarjat/.git\",\"--work-tree=/storage/sarjat/annex-sarjat\",\"log\",\"refs/heads/git-annex..214ed317536695b91c8dd5bed059c46c11ad00be\",\"--oneline\",\"-n1\"] + [2014-04-07 23:30:13 EEST] read: git [\"--git-dir=/storage/sarjat/annex-sarjat/.git\",\"--work-tree=/storage/sarjat/annex-sarjat\",\"log\",\"refs/heads/git-annex..2e0884d9c8859339855ceee396b9ea9ae05865b4\",\"--oneline\",\"-n1\"] + [2014-04-07 23:30:13 EEST] read: git [\"--git-dir=/storage/sarjat/annex-sarjat/.git\",\"--work-tree=/storage/sarjat/annex-sarjat\",\"log\",\"refs/heads/git-annex..b402a6e7b9268e25dbd9c6a027f4a5258993980d\",\"--oneline\",\"-n1\"] + [2014-04-07 23:30:13 EEST] read: git [\"--git-dir=/storage/sarjat/annex-sarjat/.git\",\"--work-tree=/storage/sarjat/annex-sarjat\",\"log\",\"refs/heads/git-annex..7b18a191d58d779aab5789b923adb09863938ffe\",\"--oneline\",\"-n1\"] + [2014-04-07 23:31:13 EEST] read: git [\"--git-dir=/storage/sarjat/annex-sarjat/.git\",\"--work-tree=/storage/sarjat/annex-sarjat\",\"ls-tree\",\"--full-tree\",\"-z\",\"--\",\"refs/heads/git-annex\",\"uuid.log\",\"remote.log\",\"trust.log\",\"group.log\",\"numcopies.log\",\"schedule.log\",\"preferred-content.log\",\"required-content.log\",\"group-preferred-content.log\"] +\"\"\"]] +"""]] diff --git a/doc/bugs/Assistant_only_watches_one_repo_on_startup.mdwn b/doc/bugs/Assistant_only_watches_one_repo_on_startup.mdwn new file mode 100644 index 0000000000..70ccf4818f --- /dev/null +++ b/doc/bugs/Assistant_only_watches_one_repo_on_startup.mdwn @@ -0,0 +1,29 @@ +### Please describe the problem. + +I have multiple git-annex assistant repos that I want watched. On startup, it is only watching one of them, despite all of them being listed in ~/.config/git-annex/autostart. + +Launching the webapp, clicking on the "switch repository" button, and selecting the other repository will cause git-annex to immediately launch a startup scan and thereafter it seems to be monitored. + +### What steps will reproduce the problem? + +killall git-annex, and then restart + +### What version of git-annex are you using? On what operating system? + +5.20151208-1 from jessie-backports + +### Please provide any additional information below. + +[[!format sh """ +# If you can, paste a complete transcript of the problem occurring here. +# If the problem is with the git-annex assistant, paste in .git/annex/daemon.log + + +# End of transcript or log. +"""]] + +### Have you had any luck using git-annex before? (Sometimes we get tired of reading bug reports all day and a lil' positive end note does wonders) + +Really nice tool. Thanks Joey! + +[[!tag moreinfo]] diff --git a/doc/bugs/Assistant_only_watches_one_repo_on_startup/comment_1_4606e5c7159e8e9f2f12e010cbd22e6a._comment b/doc/bugs/Assistant_only_watches_one_repo_on_startup/comment_1_4606e5c7159e8e9f2f12e010cbd22e6a._comment new file mode 100644 index 0000000000..355b12a86f --- /dev/null +++ b/doc/bugs/Assistant_only_watches_one_repo_on_startup/comment_1_4606e5c7159e8e9f2f12e010cbd22e6a._comment @@ -0,0 +1,12 @@ +[[!comment format=mdwn + username="joey" + subject="""comment 1""" + date="2016-06-10T19:40:54Z" + content=""" +You're left out the command line which you use to start the assistant. + +Without that, I see no indication that there is a bug. You have to pass +--autostart to make the assistant start up in all configured repositories. +If you are not using that parameter, the assistant will only start up in +the repository corresponding to the current working directory. +"""]] diff --git a/doc/bugs/Assistant_temporary_delete_and_recommit_files.mdwn b/doc/bugs/Assistant_temporary_delete_and_recommit_files.mdwn new file mode 100644 index 0000000000..4106b1ecfa --- /dev/null +++ b/doc/bugs/Assistant_temporary_delete_and_recommit_files.mdwn @@ -0,0 +1,606 @@ +### Please describe the problem. +Hi, + +I'm using git-annex assistant to sync my org-mode files between two hosts. Because these files are just plaintext, I configured git(-annex) to store the content inside git (and not annex). The setup is quite simple: On both hosts (laptop and server) there is a git-annex repository in `~/Org/Privat` which have a remote configured (over ssh) point to each other. I run "git-annex assistant" on both hosts and this works pretty well, except that `git-annex assistant` temporary delete and recommit files (probably during merge). This is no big deal because the file is still there but it unnecessarily messes up the `git log`. I'm not sure if this is a bug or just a side effect of how git-annex works. + + +### What steps will reproduce the problem? + + [Laptop] + mkdir -p ~/foo + cd ~/foo + git init + git annex init --version=6 + git annex describe "laptop" + cat > .gitattributes << EOF + *.org annex.largefiles=nothing + *.org_archive annex.largefiles=nothing + EOF + git add .gitattributes + git commit -m "Versioning org-mode files with git" + git remote add server server:~/foo/ + + + + [Server] + git clone laptop:~/foo/ ~/foo/ + cd ~/foo + git annex init "server" + git annex upgrade # update from v5 to v6 + git annex assistant + + [Laptop] + cd ~/foo + git annex assistant + echo "* this is a headline" >> test.org + +This is the basic setup. After editing test.org multiple times, the server get those changes, delete test.org and immediately recommit it. Unfortunately, I can't really reproduce the issue with the above instructions. I'm not sure if it's a timing problem, the content of the file matters or anything like that. Hopefully the attached `daemon.log` can provide more infos. + + +### What version of git-annex are you using? On what operating system? + [Laptop, Debian stretch] + git-annex version: 6.20170101.1 + build flags: Assistant Webapp Pairing Testsuite S3(multipartupload)(storageclasses) WebDAV Inotify DBus DesktopNotify ConcurrentOutput TorrentParser MagicMime Feeds Quvi + key/value backends: SHA256E SHA256 SHA512E SHA512 SHA224E SHA224 SHA384E SHA384 SHA3_256E SHA3_256 SHA3_512E SHA3_512 SHA3_224E SHA3_224 SHA3_384E SHA3_384 SKEIN256E SKEIN256 SKEIN512E SKEIN512 SHA1E SHA1 MD5E MD5 WORM URL + remote types: git gcrypt p2p S3 bup directory rsync web bittorrent webdav tahoe glacier ddar hook external + local repository version: 6 + supported repository versions: 3 5 6 + upgrade supported from repository versions: 0 1 2 3 4 5 + operating system: linux x86_64 + + [Server, Debian stretch] + git-annex version: 6.20180316 + build flags: Assistant Webapp Pairing S3(multipartupload)(storageclasses) WebDAV Inotify DBus DesktopNotify ConcurrentOutput TorrentParser MagicMime Feeds Testsuite + dependency versions: aws-0.14.1 bloomfilter-2.0.1.0 cryptonite-0.20 DAV-1.3.1 feed-0.3.11.1 ghc-8.0.1 http-client-0.4.31.1 persistent-sqlite-2.6 torrent-10000.0.0 uuid-1.3.12 yesod-1.4.3 + key/value backends: SHA256E SHA256 SHA512E SHA512 SHA224E SHA224 SHA384E SHA384 SHA3_256E SHA3_256 SHA3_512E SHA3_512 SHA3_224E SHA3_224 SHA3_384E SHA3_384 SKEIN256E SKEIN256 SKEIN512E SKEIN512 SHA1E SHA1 MD5E MD5 WORM URL + remote types: git gcrypt p2p S3 bup directory rsync web bittorrent webdav tahoe glacier ddar hook external + local repository version: 6 + supported repository versions: 3 5 6 + upgrade supported from repository versions: 0 1 2 3 4 5 + operating system: linux x86_64 + + +### Please provide any additional information below. +daemon.log from server, interesting part: + [2018-04-09 11:37:44.985603501] Watcher: file deleted Eingang.org + + +I'm also not sure where the `.#Eingang.org` comes from. (autosave-mode and backup stuff is disabled in Emacs). Note that I also ignore `.#` files: + + *~ + .#* + \#*# + .gitignore (END) + +[[!format sh """ +# If the problem is with the git-annex assistant, paste in .git/annex/daemon.log +[2018-04-09 11:34:41.651229076] main: starting assistant version 6.20180316 +[2018-04-09 11:34:41.653000679] read: git ["--git-dir=.git","--work-tree=.","--literal-pathspecs","show-ref","git-annex"] +[2018-04-09 11:34:41.656349808] process done ExitSuccess +[2018-04-09 11:34:41.656682497] read: git ["--git-dir=.git","--work-tree=.","--literal-pathspecs","show-ref","--hash","refs/heads/git-annex"] +[2018-04-09 11:34:41.660412588] process done ExitSuccess +[2018-04-09 11:34:41.660905012] read: git ["--git-dir=.git","--work-tree=.","--literal-pathspecs","log","refs/heads/git-annex..19354e4a5c6bd694a449992d5bd0d4bae48eba3b","--pretty=%H","-n1"] +[2018-04-09 11:34:41.664232695] process done ExitSuccess +[2018-04-09 11:34:41.66500315] chat: git ["--git-dir=.git","--work-tree=.","--literal-pathspecs","cat-file","--batch"] +[2018-04-09 11:34:41.666629464] chat: git ["--git-dir=.git","--work-tree=.","--literal-pathspecs","cat-file","--batch-check=%(objectname) %(objecttype) %(objectsize)"] +[2018-04-09 11:34:41.668907654] Cronner: You should enable consistency checking to protect your data. +[2018-04-09 11:34:41.673213064] read: git ["--git-dir=.git","--work-tree=.","--literal-pathspecs","ls-files","--stage","-z","--","."] +[2018-04-09 11:34:41.67746018] process done ExitSuccess +[2018-04-09 11:34:41.677940071] read: git ["--git-dir=.git","--work-tree=.","--literal-pathspecs","ls-files","--stage","-z","--","."] +[2018-04-09 11:34:41.680803467] process done ExitSuccess +[2018-04-09 11:34:41.682995824] chat: nice ["ionice","-c3","nocache","/usr/bin/git-annex","remotedaemon","--foreground"] +[2018-04-09 11:34:41.698996042] TransferScanner: Syncing with laptop +[2018-04-09 11:34:41.714084077] TransferWatcher: watching for transfers +[2018-04-09 11:34:41.71886081] read: git ["--git-dir=.git","--work-tree=.","--literal-pathspecs","ls-tree","--full-tree","-z","--","refs/heads/git-annex","uuid.log","remote.log","trust.log","group.log","numcopies.log","schedule.log","preferred-content.log","required-content.log","group-preferred-content.log"] +[2018-04-09 11:34:41.736468728] process done ExitSuccess +[2018-04-09 11:34:41.737473438] read: git ["--git-dir=.git","--work-tree=.","--literal-pathspecs","symbolic-ref","-q","HEAD"] +[2018-04-09 11:34:41.740351978] process done ExitSuccess +[2018-04-09 11:34:41.740604847] read: git ["--git-dir=.git","--work-tree=.","--literal-pathspecs","show-ref","refs/heads/master"] +[2018-04-09 11:34:41.743608301] process done ExitSuccess +[2018-04-09 11:34:41.743883152] read: git ["--git-dir=.git","--work-tree=.","--literal-pathspecs","symbolic-ref","-q","HEAD"] +[2018-04-09 11:34:41.746600971] process done ExitSuccess +[2018-04-09 11:34:41.746859505] read: git ["--git-dir=.git","--work-tree=.","--literal-pathspecs","show-ref","refs/heads/master"] +[2018-04-09 11:34:41.749864575] process done ExitSuccess + + No known network monitor available through dbus; falling back to polling + + No known volume monitor available through dbus; falling back to mtab polling +[2018-04-09 11:34:41.75133072] read: git ["--git-dir=.git","--work-tree=.","--literal-pathspecs","show-ref","git-annex"] +[2018-04-09 11:34:41.755913773] process done ExitSuccess +[2018-04-09 11:34:41.756133988] read: git ["--git-dir=.git","--work-tree=.","--literal-pathspecs","show-ref","--hash","refs/heads/git-annex"] +[2018-04-09 11:34:41.759413558] process done ExitSuccess +[2018-04-09 11:34:41.759874669] read: git ["--git-dir=.git","--work-tree=.","--literal-pathspecs","log","refs/heads/git-annex..19354e4a5c6bd694a449992d5bd0d4bae48eba3b","--pretty=%H","-n1"] +[2018-04-09 11:34:41.763534204] process done ExitSuccess +[2018-04-09 11:34:41.764491559] read: git ["--git-dir=.git","--work-tree=.","--literal-pathspecs","ls-tree","--full-tree","-z","--","refs/heads/git-annex","uuid.log","remote.log","trust.log","group.log","numcopies.log","schedule.log","preferred-content.log","required-content.log","group-preferred-content.log"] +[2018-04-09 11:34:41.768098522] process done ExitSuccess +[2018-04-09 11:34:41.769538385] call: git ["--git-dir=.git","--work-tree=.","--literal-pathspecs","fetch","laptop"] +[2018-04-09 11:34:41.774051138] read: git ["--git-dir=.git","--work-tree=.","--literal-pathspecs","symbolic-ref","-q","HEAD"] +[2018-04-09 11:34:41.784145283] process done ExitSuccess +[2018-04-09 11:34:41.784410291] read: git ["--git-dir=.git","--work-tree=.","--literal-pathspecs","show-ref","refs/heads/master"] +[2018-04-09 11:34:41.790119006] process done ExitSuccess +[2018-04-09 11:34:41.790448451] read: git ["--git-dir=.git","--work-tree=.","--literal-pathspecs","log","refs/heads/master..refs/heads/synced/master","--pretty=%H","-n1"] +[2018-04-09 11:34:41.803315234] process done ExitSuccess +[2018-04-09 11:34:41.803726537] Merger: merging refs/heads/synced/master into refs/heads/master + +[2018-04-09 11:34:41.803885147] read: git ["--git-dir=.git","--work-tree=.","--literal-pathspecs","show-ref","--hash","refs/heads/master"] +[2018-04-09 11:34:41.806791552] process done ExitSuccess +[2018-04-09 11:34:41.807106639] read: git ["--version"] +[2018-04-09 11:34:41.809213293] process done ExitSuccess +[2018-04-09 11:34:41.809579548] call: git ["--git-dir=.git","--work-tree=.","--literal-pathspecs","-c","commit.gpgsign=false","merge","--no-edit","refs/heads/synced/master","--allow-unrelated-histories"] +Aktualisiere 9d81724..1f72868 +Fast-forward + Eingang.org | 146 +++++++++++++++++++++++++++++++ + Privat.org | 284 ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ + 2 files changed, 430 insertions(+) + create mode 100644 Eingang.org + create mode 100644 Privat.org +[2018-04-09 11:34:41.869190999] process done ExitSuccess +[2018-04-09 11:34:41.869616621] read: git ["--git-dir=.git","--work-tree=.","--literal-pathspecs","show-ref","git-annex"] +[2018-04-09 11:34:41.87263735] process done ExitSuccess +[2018-04-09 11:34:41.872886362] read: git ["--git-dir=.git","--work-tree=.","--literal-pathspecs","show-ref","--hash","refs/heads/git-annex"] +[2018-04-09 11:34:41.876100082] process done ExitSuccess +[2018-04-09 11:34:41.876526799] read: git ["--git-dir=.git","--work-tree=.","--literal-pathspecs","log","refs/heads/git-annex..19354e4a5c6bd694a449992d5bd0d4bae48eba3b","--pretty=%H","-n1"] +[2018-04-09 11:34:41.879793847] process done ExitSuccess +(scanning...) [2018-04-09 11:34:41.880538361] Watcher: Performing startup scan +[2018-04-09 11:34:41.880979369] read: git ["--git-dir=.git","--work-tree=.","--literal-pathspecs","show-ref","git-annex"] +[2018-04-09 11:34:41.884247108] process done ExitSuccess +[2018-04-09 11:34:41.884492946] read: git ["--git-dir=.git","--work-tree=.","--literal-pathspecs","show-ref","--hash","refs/heads/git-annex"] +[2018-04-09 11:34:41.887468992] process done ExitSuccess +[2018-04-09 11:34:41.88788752] read: git ["--git-dir=.git","--work-tree=.","--literal-pathspecs","log","refs/heads/git-annex..19354e4a5c6bd694a449992d5bd0d4bae48eba3b","--pretty=%H","-n1"] +[2018-04-09 11:34:41.891145787] process done ExitSuccess +[2018-04-09 11:34:41.891582283] chat: git ["--git-dir=.git","--work-tree=.","--literal-pathspecs","cat-file","--batch"] +[2018-04-09 11:34:41.892133862] chat: git ["--git-dir=.git","--work-tree=.","--literal-pathspecs","cat-file","--batch-check=%(objectname) %(objecttype) %(objectsize)"] +[2018-04-09 11:34:41.900436044] read: git ["--git-dir=.git","--work-tree=.","--literal-pathspecs","symbolic-ref","-q","HEAD"] +[2018-04-09 11:34:41.903468755] process done ExitSuccess +[2018-04-09 11:34:41.903676566] read: git ["--git-dir=.git","--work-tree=.","--literal-pathspecs","show-ref","refs/heads/master"] +[2018-04-09 11:34:41.906280325] process done ExitSuccess +[2018-04-09 11:34:41.906524648] read: git ["--git-dir=.git","--work-tree=.","--literal-pathspecs","symbolic-ref","-q","HEAD"] +[2018-04-09 11:34:41.908819698] process done ExitSuccess +[2018-04-09 11:34:41.909024433] read: git ["--git-dir=.git","--work-tree=.","--literal-pathspecs","show-ref","refs/heads/master"] +[2018-04-09 11:34:41.911759505] process done ExitSuccess +[2018-04-09 11:34:41.912398343] read: git ["--version"] +[2018-04-09 11:34:41.914443754] process done ExitSuccess +[2018-04-09 11:34:41.914752675] chat: git ["--git-dir=.git","--work-tree=.","check-ignore","-z","--stdin","--verbose","--non-matching"] +[2018-04-09 11:34:41.917108907] Watcher: add Eingang.org +[2018-04-09 11:34:41.91734644] read: git ["--git-dir=.git","--work-tree=.","--literal-pathspecs","log","refs/heads/master..refs/remotes/laptop/master","--pretty=%H","-n1"] +[2018-04-09 11:34:41.920660043] process done ExitSuccess +[2018-04-09 11:34:41.921055355] chat: git ["--git-dir=.git","--work-tree=.","--literal-pathspecs","check-attr","-z","--stdin","annex.backend","annex.numcopies","annex.largefiles","--"] +[2018-04-09 11:34:41.921800291] read: git ["--version"] +[2018-04-09 11:34:41.925093152] process done ExitSuccess +[2018-04-09 11:34:41.926328604] read: git ["--git-dir=.git","--work-tree=.","--literal-pathspecs","symbolic-ref","-q","HEAD"] +[2018-04-09 11:34:41.928840733] process done ExitSuccess +[2018-04-09 11:34:41.929206878] read: git ["--git-dir=.git","--work-tree=.","--literal-pathspecs","show-ref","refs/heads/master"] +[2018-04-09 11:34:41.934755521] process done ExitSuccess +[2018-04-09 11:34:41.935152027] Merger: watching .git/refs +[2018-04-09 11:34:41.958868627] Watcher: add Privat.org +[2018-04-09 11:34:41.959579297] read: git ["--git-dir=.git","--work-tree=.","--literal-pathspecs","ls-files","--deleted","-z","--","."] +[2018-04-09 11:34:41.962332644] process done ExitSuccess +(started...) +[2018-04-09 11:34:41.962726051] Watcher: watching . +Von laptop:Org/Privat + 9d81724..1f72868 master -> laptop/master + 316555e..19354e4 git-annex -> laptop/git-annex + * [neuer Branch] synced/master -> laptop/synced/master +[2018-04-09 11:34:41.997168884] read: git ["--git-dir=.git","--work-tree=.","--literal-pathspecs","symbolic-ref","-q","HEAD"] +[2018-04-09 11:34:42.004185608] process done ExitSuccess +[2018-04-09 11:34:42.004431346] read: git ["--git-dir=.git","--work-tree=.","--literal-pathspecs","show-ref","refs/heads/master"] +[2018-04-09 11:34:42.009686504] process done ExitSuccess +[2018-04-09 11:34:42.010153632] process done ExitSuccess +[2018-04-09 11:34:42.010482295] read: git ["--git-dir=.git","--work-tree=.","--literal-pathspecs","show-ref","git-annex"] +[2018-04-09 11:34:42.013335466] process done ExitSuccess +[2018-04-09 11:34:42.013718719] read: git ["--git-dir=.git","--work-tree=.","--literal-pathspecs","show-ref","--hash","refs/heads/git-annex"] +[2018-04-09 11:34:42.019932467] process done ExitSuccess +[2018-04-09 11:34:42.020329615] read: git ["--git-dir=.git","--work-tree=.","--literal-pathspecs","log","refs/heads/git-annex..19354e4a5c6bd694a449992d5bd0d4bae48eba3b","--pretty=%H","-n1"] +[2018-04-09 11:34:42.023886643] process done ExitSuccess +[2018-04-09 11:34:42.024275073] read: git ["--git-dir=.git","--work-tree=.","--literal-pathspecs","log","refs/heads/master..refs/remotes/laptop/master","--pretty=%H","-n1"] +[2018-04-09 11:34:42.027633095] process done ExitSuccess +[2018-04-09 11:34:42.028007368] call: git ["--git-dir=.git","--work-tree=.","--literal-pathspecs","branch","-f","synced/master","refs/heads/master"] +[2018-04-09 11:34:42.03206509] process done ExitSuccess +[2018-04-09 11:34:42.03231932] call: git ["--git-dir=.git","--work-tree=.","--literal-pathspecs","show-ref","--verify","-q","refs/remotes/laptop/master"] +[2018-04-09 11:34:42.034779842] process done ExitSuccess +[2018-04-09 11:34:42.035052574] read: git ["--git-dir=.git","--work-tree=.","--literal-pathspecs","log","refs/heads/master..refs/remotes/laptop/master","--pretty=%H","-n1"] +[2018-04-09 11:34:42.038557601] process done ExitSuccess +[2018-04-09 11:34:42.038806754] call: git ["--git-dir=.git","--work-tree=.","--literal-pathspecs","show-ref","--verify","-q","refs/remotes/laptop/synced/master"] +[2018-04-09 11:34:42.041302446] process done ExitSuccess +[2018-04-09 11:34:42.041621633] read: git ["--git-dir=.git","--work-tree=.","--literal-pathspecs","log","refs/heads/synced/master..refs/remotes/laptop/synced/master","--pretty=%H","-n1"] +[2018-04-09 11:34:42.044851439] process done ExitSuccess +[2018-04-09 11:34:42.045116323] read: git ["--git-dir=.git","--work-tree=.","--literal-pathspecs","show-ref","git-annex"] +[2018-04-09 11:34:42.048082132] process done ExitSuccess +[2018-04-09 11:34:42.048322864] read: git ["--git-dir=.git","--work-tree=.","--literal-pathspecs","show-ref","--hash","refs/heads/git-annex"] +[2018-04-09 11:34:42.051728257] process done ExitSuccess +[2018-04-09 11:34:42.052150695] read: git ["--git-dir=.git","--work-tree=.","--literal-pathspecs","log","refs/heads/git-annex..19354e4a5c6bd694a449992d5bd0d4bae48eba3b","--pretty=%H","-n1"] +[2018-04-09 11:34:42.055676191] process done ExitSuccess +[2018-04-09 11:34:42.056232266] read: git ["--git-dir=.git","--work-tree=.","--literal-pathspecs","symbolic-ref","-q","HEAD"] +[2018-04-09 11:34:42.058900877] process done ExitSuccess +[2018-04-09 11:34:42.059140096] read: git ["--git-dir=.git","--work-tree=.","--literal-pathspecs","show-ref","refs/heads/master"] +[2018-04-09 11:34:42.062072724] process done ExitSuccess +[2018-04-09 11:34:42.062321782] TransferScanner: pushing to [Remote { name ="laptop" }] +[2018-04-09 11:34:42.062448681] read: git ["--git-dir=.git","--work-tree=.","--literal-pathspecs","symbolic-ref","-q","HEAD"] +[2018-04-09 11:34:42.065011986] process done ExitSuccess +[2018-04-09 11:34:42.065252161] read: git ["--git-dir=.git","--work-tree=.","--literal-pathspecs","show-ref","refs/heads/master"] +[2018-04-09 11:34:42.068152308] process done ExitSuccess +[2018-04-09 11:34:42.069095635] read: git ["--git-dir=.git","--work-tree=.","--literal-pathspecs","log","refs/heads/master..refs/remotes/laptop/synced/master","--pretty=%H","-n1"] +[2018-04-09 11:34:42.069826395] call: git ["--git-dir=.git","--work-tree=.","--literal-pathspecs","push","laptop","+git-annex:synced/git-annex","master:synced/master"] +[2018-04-09 11:34:42.073909169] process done ExitSuccess +To laptop:Org/Privat + * [new branch] git-annex -> synced/git-annex +[2018-04-09 11:34:42.114149032] read: git ["--git-dir=.git","--work-tree=.","--literal-pathspecs","show-ref","git-annex"] +[2018-04-09 11:34:42.114953858] process done ExitSuccess +[2018-04-09 11:34:42.115154911] read: git ["--git-dir=.git","--work-tree=.","--literal-pathspecs","push","laptop","git-annex"] +[2018-04-09 11:34:42.118392157] process done ExitSuccess +[2018-04-09 11:34:42.118622237] read: git ["--git-dir=.git","--work-tree=.","--literal-pathspecs","show-ref","--hash","refs/heads/git-annex"] +[2018-04-09 11:34:42.124076787] process done ExitSuccess +[2018-04-09 11:34:42.124811487] read: git ["--git-dir=.git","--work-tree=.","--literal-pathspecs","log","refs/heads/git-annex..19354e4a5c6bd694a449992d5bd0d4bae48eba3b","--pretty=%H","-n1"] +[2018-04-09 11:34:42.131284656] process done ExitSuccess +[2018-04-09 11:34:42.150151612] process done ExitSuccess +[2018-04-09 11:34:42.150527374] read: git ["--git-dir=.git","--work-tree=.","--literal-pathspecs","push","laptop","master"] +[2018-04-09 11:34:42.179702012] process done ExitSuccess +[2018-04-09 11:34:42.700210723] Committer: committing 2 changes +[2018-04-09 11:34:42.70055449] Committer: Committing changes to git +(recording state in git...) +[2018-04-09 11:34:42.70085964] feed: xargs ["-0","git","--git-dir=.git","--work-tree=.","--literal-pathspecs","add","--force","--"] +[2018-04-09 11:34:42.784196514] process done ExitSuccess +[2018-04-09 11:34:42.785257124] read: git ["--git-dir=.git","--work-tree=.","--literal-pathspecs","symbolic-ref","-q","HEAD"] +[2018-04-09 11:34:42.791617178] process done ExitSuccess +[2018-04-09 11:34:42.791841318] read: git ["--git-dir=.git","--work-tree=.","--literal-pathspecs","show-ref","--hash","refs/heads/master"] +[2018-04-09 11:34:42.795239558] process done ExitSuccess +[2018-04-09 11:34:42.795513253] read: git ["--git-dir=.git","--work-tree=.","--literal-pathspecs","write-tree"] +[2018-04-09 11:34:42.799350469] process done ExitSuccess +[2018-04-09 11:34:42.799629834] read: git ["--git-dir=.git","--work-tree=.","--literal-pathspecs","rev-parse","1f72868a7a2ebf60ca9e58ec154773d75115b9d8:"] +[2018-04-09 11:34:42.803096203] process done ExitSuccess +[2018-04-09 11:34:42.803365134] read: git ["--git-dir=.git","--work-tree=.","--literal-pathspecs","symbolic-ref","-q","HEAD"] +[2018-04-09 11:34:42.806041146] process done ExitSuccess +[2018-04-09 11:34:42.80628186] read: git ["--git-dir=.git","--work-tree=.","--literal-pathspecs","show-ref","refs/heads/master"] +[2018-04-09 11:34:42.809185013] process done ExitSuccess +[2018-04-09 11:34:42.809474896] call: git ["--git-dir=.git","--work-tree=.","--literal-pathspecs","branch","-f","synced/master","refs/heads/master"] +[2018-04-09 11:34:42.813085481] process done ExitSuccess +[2018-04-09 11:34:43.682704066] Pusher: Syncing with laptop +[2018-04-09 11:34:43.685333086] read: git ["--git-dir=.git","--work-tree=.","--literal-pathspecs","symbolic-ref","-q","HEAD"] +[2018-04-09 11:34:43.68940378] process done ExitSuccess +[2018-04-09 11:34:43.689881594] read: git ["--git-dir=.git","--work-tree=.","--literal-pathspecs","show-ref","refs/heads/master"] +[2018-04-09 11:34:43.693390894] process done ExitSuccess +[2018-04-09 11:34:43.693694954] Pusher: pushing to [Remote { name ="laptop" }] +[2018-04-09 11:34:43.694609849] call: git ["--git-dir=.git","--work-tree=.","--literal-pathspecs","push","laptop","+git-annex:synced/git-annex","master:synced/master"] +[2018-04-09 11:34:43.710791388] RemoteControl: CONNECTED ssh://laptop/~/Org/Privat +[2018-04-09 11:34:43.71113574] RemoteControl: fromList [UUID "234c34d9-5493-44aa-967f-8b915796a618"] +Everything up-to-date +[2018-04-09 11:34:43.727077172] process done ExitSuccess +[2018-04-09 11:34:43.727454237] read: git ["--git-dir=.git","--work-tree=.","--literal-pathspecs","push","laptop","git-annex"] +[2018-04-09 11:34:43.766167107] process done ExitSuccess +[2018-04-09 11:34:43.766500325] read: git ["--git-dir=.git","--work-tree=.","--literal-pathspecs","push","laptop","master"] +[2018-04-09 11:34:43.811781891] process done ExitSuccess +[2018-04-09 11:34:44.181196303] TransferScanner: starting scan of [Remote { name ="laptop" }] +[2018-04-09 11:34:44.182448533] read: git ["--git-dir=.git","--work-tree=.","--literal-pathspecs","ls-files","--cached","-z","--"] +[2018-04-09 11:34:44.228488077] process done ExitSuccess +[2018-04-09 11:34:44.22870839] TransferScanner: finished scan of [Remote { name ="laptop" }] +[2018-04-09 11:35:41.769179017] read: git ["--git-dir=.git","--work-tree=.","--literal-pathspecs","ls-tree","--full-tree","-z","--","refs/heads/git-annex","uuid.log","remote.log","trust.log","group.log","numcopies.log","schedule.log","preferred-content.log","required-content.log","group-preferred-content.log"] +[2018-04-09 11:35:41.774173415] process done ExitSuccess +[2018-04-09 11:36:13.313726536] Watcher: add Eingang.org +[2018-04-09 11:36:13.314592472] Watcher: file deleted .#Eingang.org +[2018-04-09 11:36:13.314880376] Committer: committing 1 changes +[2018-04-09 11:36:13.315012724] Committer: Committing changes to git +[2018-04-09 11:36:13.315522929] feed: xargs ["-0","git","--git-dir=.git","--work-tree=.","--literal-pathspecs","add","--force","--"] +[2018-04-09 11:36:13.348942513] process done ExitSuccess +(recording state in git...) +[2018-04-09 11:36:13.349347047] feed: git ["--git-dir=.git","--work-tree=.","--literal-pathspecs","update-index","-z","--index-info"] +[2018-04-09 11:36:13.386484644] process done ExitSuccess +[2018-04-09 11:36:13.386779923] read: git ["--git-dir=.git","--work-tree=.","--literal-pathspecs","symbolic-ref","-q","HEAD"] +[2018-04-09 11:36:13.389185774] process done ExitSuccess +[2018-04-09 11:36:13.389510587] read: git ["--git-dir=.git","--work-tree=.","--literal-pathspecs","show-ref","--hash","refs/heads/master"] +[2018-04-09 11:36:13.392440035] process done ExitSuccess +[2018-04-09 11:36:13.392732949] read: git ["--git-dir=.git","--work-tree=.","--literal-pathspecs","write-tree"] +[2018-04-09 11:36:13.427572223] process done ExitSuccess +[2018-04-09 11:36:13.427923407] read: git ["--git-dir=.git","--work-tree=.","--literal-pathspecs","rev-parse","1f72868a7a2ebf60ca9e58ec154773d75115b9d8:"] +[2018-04-09 11:36:13.431080357] process done ExitSuccess +[2018-04-09 11:36:13.431414943] read: git ["--git-dir=.git","--work-tree=.","--literal-pathspecs","symbolic-ref","-q","HEAD"] +[2018-04-09 11:36:13.433986742] process done ExitSuccess +[2018-04-09 11:36:13.434288205] read: git ["--git-dir=.git","--work-tree=.","--literal-pathspecs","show-ref","refs/heads/master"] +[2018-04-09 11:36:13.43725448] process done ExitSuccess +[2018-04-09 11:36:13.437592844] call: git ["--git-dir=.git","--work-tree=.","--literal-pathspecs","branch","-f","synced/master","refs/heads/master"] +[2018-04-09 11:36:13.4409276] process done ExitSuccess +[2018-04-09 11:36:13.441326536] Pusher: Syncing with laptop +[2018-04-09 11:36:13.441756651] read: git ["--git-dir=.git","--work-tree=.","--literal-pathspecs","symbolic-ref","-q","HEAD"] +[2018-04-09 11:36:13.444837303] process done ExitSuccess +[2018-04-09 11:36:13.445166889] read: git ["--git-dir=.git","--work-tree=.","--literal-pathspecs","show-ref","refs/heads/master"] +[2018-04-09 11:36:13.448357577] process done ExitSuccess +[2018-04-09 11:36:13.448675589] Pusher: pushing to [Remote { name ="laptop" }] +[2018-04-09 11:36:13.449509449] call: git ["--git-dir=.git","--work-tree=.","--literal-pathspecs","push","laptop","+git-annex:synced/git-annex","master:synced/master"] +Everything up-to-date +[2018-04-09 11:36:13.490272243] process done ExitSuccess +[2018-04-09 11:36:13.490695983] read: git ["--git-dir=.git","--work-tree=.","--literal-pathspecs","push","laptop","git-annex"] +[2018-04-09 11:36:13.528199661] process done ExitSuccess +[2018-04-09 11:36:13.528538098] read: git ["--git-dir=.git","--work-tree=.","--literal-pathspecs","push","laptop","master"] +[2018-04-09 11:36:13.562512923] process done ExitSuccess +[2018-04-09 11:36:14.452568279] Committer: committing 1 changes +[2018-04-09 11:36:14.452892251] Committer: Committing changes to git +[2018-04-09 11:36:14.453110793] read: git ["--git-dir=.git","--work-tree=.","--literal-pathspecs","symbolic-ref","-q","HEAD"] +[2018-04-09 11:36:14.456921344] process done ExitSuccess +[2018-04-09 11:36:14.45731823] read: git ["--git-dir=.git","--work-tree=.","--literal-pathspecs","show-ref","--hash","refs/heads/master"] +[2018-04-09 11:36:14.461828918] process done ExitSuccess +[2018-04-09 11:36:14.462244454] read: git ["--git-dir=.git","--work-tree=.","--literal-pathspecs","write-tree"] +[2018-04-09 11:36:14.465199856] process done ExitSuccess +[2018-04-09 11:36:14.465582073] read: git ["--git-dir=.git","--work-tree=.","--literal-pathspecs","rev-parse","1f72868a7a2ebf60ca9e58ec154773d75115b9d8:"] +[2018-04-09 11:36:14.468458637] process done ExitSuccess +[2018-04-09 11:36:14.468799034] read: git ["--git-dir=.git","--work-tree=.","--literal-pathspecs","symbolic-ref","-q","HEAD"] +[2018-04-09 11:36:14.471547079] process done ExitSuccess +[2018-04-09 11:36:14.471853595] read: git ["--git-dir=.git","--work-tree=.","--literal-pathspecs","show-ref","refs/heads/master"] +[2018-04-09 11:36:14.474845478] process done ExitSuccess +[2018-04-09 11:36:14.475161153] call: git ["--git-dir=.git","--work-tree=.","--literal-pathspecs","branch","-f","synced/master","refs/heads/master"] +[2018-04-09 11:36:14.478481548] process done ExitSuccess +[2018-04-09 11:36:15.563886253] Pusher: Syncing with laptop +[2018-04-09 11:36:15.564475349] read: git ["--git-dir=.git","--work-tree=.","--literal-pathspecs","symbolic-ref","-q","HEAD"] +[2018-04-09 11:36:15.56845368] process done ExitSuccess +[2018-04-09 11:36:15.568873956] read: git ["--git-dir=.git","--work-tree=.","--literal-pathspecs","show-ref","refs/heads/master"] +[2018-04-09 11:36:15.573271271] process done ExitSuccess +[2018-04-09 11:36:15.573737154] Pusher: pushing to [Remote { name ="laptop" }] +[2018-04-09 11:36:15.575005334] call: git ["--git-dir=.git","--work-tree=.","--literal-pathspecs","push","laptop","+git-annex:synced/git-annex","master:synced/master"] +Everything up-to-date +[2018-04-09 11:36:15.614114318] process done ExitSuccess +[2018-04-09 11:36:15.614502392] read: git ["--git-dir=.git","--work-tree=.","--literal-pathspecs","push","laptop","git-annex"] +[2018-04-09 11:36:15.653929632] process done ExitSuccess +[2018-04-09 11:36:15.654325001] read: git ["--git-dir=.git","--work-tree=.","--literal-pathspecs","push","laptop","master"] +[2018-04-09 11:36:15.693126157] process done ExitSuccess +[2018-04-09 11:36:28.906543231] Watcher: add Eingang.org +[2018-04-09 11:36:28.9075571] Watcher: file deleted .#Eingang.org +[2018-04-09 11:36:28.907841012] Committer: committing 1 changes +[2018-04-09 11:36:28.907976823] Committer: Committing changes to git +[2018-04-09 11:36:28.908467825] feed: xargs ["-0","git","--git-dir=.git","--work-tree=.","--literal-pathspecs","add","--force","--"] +[2018-04-09 11:36:28.942067906] process done ExitSuccess +(recording state in git...) +[2018-04-09 11:36:28.94239642] feed: git ["--git-dir=.git","--work-tree=.","--literal-pathspecs","update-index","-z","--index-info"] +[2018-04-09 11:36:28.982158922] process done ExitSuccess +[2018-04-09 11:36:28.982429899] read: git ["--git-dir=.git","--work-tree=.","--literal-pathspecs","symbolic-ref","-q","HEAD"] +[2018-04-09 11:36:28.98502309] process done ExitSuccess +[2018-04-09 11:36:28.985343324] read: git ["--git-dir=.git","--work-tree=.","--literal-pathspecs","show-ref","--hash","refs/heads/master"] +[2018-04-09 11:36:28.988397366] process done ExitSuccess +[2018-04-09 11:36:28.988708167] read: git ["--git-dir=.git","--work-tree=.","--literal-pathspecs","write-tree"] +[2018-04-09 11:36:29.023882235] process done ExitSuccess +[2018-04-09 11:36:29.024232929] read: git ["--git-dir=.git","--work-tree=.","--literal-pathspecs","rev-parse","1f72868a7a2ebf60ca9e58ec154773d75115b9d8:"] +[2018-04-09 11:36:29.027517239] process done ExitSuccess +[2018-04-09 11:36:29.027864] chat: git ["--git-dir=.git","--work-tree=.","--literal-pathspecs","commit-tree","f966ae65d12ccffc312af47ceff025cbad91c221","--no-gpg-sign","-p","1f72868a7a2ebf60ca9e58ec154773d75115b9d8"] +[2018-04-09 11:36:29.031510789] process done ExitSuccess +[2018-04-09 11:36:29.031851849] call: git ["--git-dir=.git","--work-tree=.","--literal-pathspecs","update-ref","refs/heads/master","4b4905f59851c29ddbf1c245d13098266c51be27"] +[2018-04-09 11:36:29.035606493] process done ExitSuccess +[2018-04-09 11:36:29.035905803] read: git ["--git-dir=.git","--work-tree=.","--literal-pathspecs","symbolic-ref","-q","HEAD"] +[2018-04-09 11:36:29.038544683] process done ExitSuccess +[2018-04-09 11:36:29.039085199] read: git ["--git-dir=.git","--work-tree=.","--literal-pathspecs","show-ref","refs/heads/master"] +[2018-04-09 11:36:29.042182652] process done ExitSuccess +[2018-04-09 11:36:29.042514323] call: git ["--git-dir=.git","--work-tree=.","--literal-pathspecs","branch","-f","synced/master","refs/heads/master"] +[2018-04-09 11:36:29.046951109] process done ExitSuccess +[2018-04-09 11:36:29.047323546] read: git ["--git-dir=.git","--work-tree=.","--literal-pathspecs","symbolic-ref","-q","HEAD"] +[2018-04-09 11:36:29.048148357] Pusher: Syncing with laptop +[2018-04-09 11:36:29.050543182] process done ExitSuccess +[2018-04-09 11:36:29.050886803] read: git ["--git-dir=.git","--work-tree=.","--literal-pathspecs","show-ref","refs/heads/master"] +[2018-04-09 11:36:29.054442432] process done ExitSuccess +[2018-04-09 11:36:29.055031142] read: git ["--git-dir=.git","--work-tree=.","--literal-pathspecs","symbolic-ref","-q","HEAD"] +[2018-04-09 11:36:29.057832869] process done ExitSuccess +[2018-04-09 11:36:29.058164071] read: git ["--git-dir=.git","--work-tree=.","--literal-pathspecs","show-ref","refs/heads/master"] +[2018-04-09 11:36:29.061083988] process done ExitSuccess +[2018-04-09 11:36:29.061406465] Pusher: pushing to [Remote { name ="laptop" }] +[2018-04-09 11:36:29.061573814] read: git ["--git-dir=.git","--work-tree=.","--literal-pathspecs","symbolic-ref","-q","HEAD"] +[2018-04-09 11:36:29.064134558] process done ExitSuccess +[2018-04-09 11:36:29.064441998] read: git ["--git-dir=.git","--work-tree=.","--literal-pathspecs","show-ref","refs/heads/master"] +[2018-04-09 11:36:29.067405287] process done ExitSuccess +[2018-04-09 11:36:29.068528484] read: git ["--git-dir=.git","--work-tree=.","--literal-pathspecs","log","refs/heads/master..refs/heads/synced/master","--pretty=%H","-n1"] +[2018-04-09 11:36:29.068801805] call: git ["--git-dir=.git","--work-tree=.","--literal-pathspecs","push","laptop","+git-annex:synced/git-annex","master:synced/master"] +[2018-04-09 11:36:29.072287644] process done ExitSuccess +To laptop:Org/Privat + 1f72868..4b4905f master -> synced/master +[2018-04-09 11:36:29.126939523] read: git ["--git-dir=.git","--work-tree=.","--literal-pathspecs","symbolic-ref","-q","HEAD"] +[2018-04-09 11:36:29.129007245] process done ExitSuccess +[2018-04-09 11:36:29.129421784] read: git ["--git-dir=.git","--work-tree=.","--literal-pathspecs","push","laptop","git-annex"] +[2018-04-09 11:36:29.131751468] process done ExitSuccess +[2018-04-09 11:36:29.132079594] read: git ["--git-dir=.git","--work-tree=.","--literal-pathspecs","show-ref","refs/heads/master"] +[2018-04-09 11:36:29.136949419] process done ExitSuccess +[2018-04-09 11:36:29.137239065] read: git ["--git-dir=.git","--work-tree=.","--literal-pathspecs","log","refs/heads/master..refs/remotes/laptop/synced/master","--pretty=%H","-n1"] +[2018-04-09 11:36:29.140617503] process done ExitSuccess +[2018-04-09 11:36:29.161862199] process done ExitSuccess +[2018-04-09 11:36:29.16237389] read: git ["--git-dir=.git","--work-tree=.","--literal-pathspecs","push","laptop","master"] +[2018-04-09 11:36:29.20090369] read: git ["--git-dir=.git","--work-tree=.","--literal-pathspecs","symbolic-ref","-q","HEAD"] +[2018-04-09 11:36:29.201988286] process done ExitSuccess +[2018-04-09 11:36:29.204332853] process done ExitSuccess +[2018-04-09 11:36:29.204687171] read: git ["--git-dir=.git","--work-tree=.","--literal-pathspecs","show-ref","refs/heads/master"] +[2018-04-09 11:36:29.207943434] process done ExitSuccess +[2018-04-09 11:36:29.208323263] read: git ["--git-dir=.git","--work-tree=.","--literal-pathspecs","log","refs/heads/master..refs/remotes/laptop/master","--pretty=%H","-n1"] +[2018-04-09 11:36:29.211711264] process done ExitSuccess +[2018-04-09 11:36:30.059006711] Committer: committing 1 changes +[2018-04-09 11:36:30.059230003] Committer: Committing changes to git +[2018-04-09 11:36:30.059357379] read: git ["--git-dir=.git","--work-tree=.","--literal-pathspecs","symbolic-ref","-q","HEAD"] +[2018-04-09 11:36:30.061820862] process done ExitSuccess +[2018-04-09 11:36:30.062223878] read: git ["--git-dir=.git","--work-tree=.","--literal-pathspecs","show-ref","--hash","refs/heads/master"] +[2018-04-09 11:36:30.064990209] process done ExitSuccess +[2018-04-09 11:36:30.065375169] read: git ["--git-dir=.git","--work-tree=.","--literal-pathspecs","write-tree"] +[2018-04-09 11:36:30.069576223] process done ExitSuccess +[2018-04-09 11:36:30.069919208] read: git ["--git-dir=.git","--work-tree=.","--literal-pathspecs","rev-parse","4b4905f59851c29ddbf1c245d13098266c51be27:"] +[2018-04-09 11:36:30.072788736] process done ExitSuccess +[2018-04-09 11:36:30.073110243] read: git ["--git-dir=.git","--work-tree=.","--literal-pathspecs","symbolic-ref","-q","HEAD"] +[2018-04-09 11:36:30.075829716] process done ExitSuccess +[2018-04-09 11:36:30.076124925] read: git ["--git-dir=.git","--work-tree=.","--literal-pathspecs","show-ref","refs/heads/master"] +[2018-04-09 11:36:30.079137617] process done ExitSuccess +[2018-04-09 11:36:30.079480821] call: git ["--git-dir=.git","--work-tree=.","--literal-pathspecs","branch","-f","synced/master","refs/heads/master"] +[2018-04-09 11:36:30.083511227] process done ExitSuccess +[2018-04-09 11:36:31.203242705] Pusher: Syncing with laptop +[2018-04-09 11:36:31.203770975] read: git ["--git-dir=.git","--work-tree=.","--literal-pathspecs","symbolic-ref","-q","HEAD"] +[2018-04-09 11:36:31.20738917] process done ExitSuccess +[2018-04-09 11:36:31.207759515] read: git ["--git-dir=.git","--work-tree=.","--literal-pathspecs","show-ref","refs/heads/master"] +[2018-04-09 11:36:31.212090647] process done ExitSuccess +[2018-04-09 11:36:31.212468233] Pusher: pushing to [Remote { name ="laptop" }] +[2018-04-09 11:36:31.213635349] call: git ["--git-dir=.git","--work-tree=.","--literal-pathspecs","push","laptop","+git-annex:synced/git-annex","master:synced/master"] +Everything up-to-date +[2018-04-09 11:36:31.24862919] process done ExitSuccess +[2018-04-09 11:36:31.249023428] read: git ["--git-dir=.git","--work-tree=.","--literal-pathspecs","push","laptop","git-annex"] +[2018-04-09 11:36:31.284126372] process done ExitSuccess +[2018-04-09 11:36:31.284485355] read: git ["--git-dir=.git","--work-tree=.","--literal-pathspecs","push","laptop","master"] +[2018-04-09 11:36:31.319518403] process done ExitSuccess +[2018-04-09 11:37:44.963088675] read: git ["--git-dir=.git","--work-tree=.","--literal-pathspecs","symbolic-ref","-q","HEAD"] +[2018-04-09 11:37:44.966932815] process done ExitSuccess +[2018-04-09 11:37:44.967153854] read: git ["--git-dir=.git","--work-tree=.","--literal-pathspecs","show-ref","refs/heads/master"] +[2018-04-09 11:37:44.971643779] process done ExitSuccess +[2018-04-09 11:37:44.971907699] read: git ["--git-dir=.git","--work-tree=.","--literal-pathspecs","log","refs/heads/master..refs/heads/synced/master","--pretty=%H","-n1"] +[2018-04-09 11:37:44.975334885] process done ExitSuccess +[2018-04-09 11:37:44.975583378] Merger: merging refs/heads/synced/master into refs/heads/master + +[2018-04-09 11:37:44.975715785] read: git ["--git-dir=.git","--work-tree=.","--literal-pathspecs","show-ref","--hash","refs/heads/master"] +[2018-04-09 11:37:44.978548143] process done ExitSuccess +[2018-04-09 11:37:44.97878846] read: git ["--version"] +[2018-04-09 11:37:44.9809852] process done ExitSuccess +[2018-04-09 11:37:44.981307187] call: git ["--git-dir=.git","--work-tree=.","--literal-pathspecs","-c","commit.gpgsign=false","merge","--no-edit","refs/heads/synced/master","--allow-unrelated-histories"] +Aktualisiere 4b4905f..1e93ab8 +[2018-04-09 11:37:44.985603501] Watcher: file deleted Eingang.org +Fast-forward + Eingang.org | 2 +- + 1 file changed, 1 insertion(+), 1 deletion(-) +[2018-04-09 11:37:45.015531778] process done ExitSuccess +[2018-04-09 11:37:45.020024283] read: git ["--git-dir=.git","--work-tree=.","--literal-pathspecs","symbolic-ref","-q","HEAD"] +[2018-04-09 11:37:45.023151518] process done ExitSuccess +[2018-04-09 11:37:45.023371468] read: git ["--git-dir=.git","--work-tree=.","--literal-pathspecs","show-ref","refs/heads/master"] +[2018-04-09 11:37:45.02639557] process done ExitSuccess +[2018-04-09 11:37:45.031466881] Committer: committing 1 changes +[2018-04-09 11:37:45.031772484] Committer: Committing changes to git +[2018-04-09 11:37:45.031924501] Watcher: add Eingang.org +(recording state in git...) +[2018-04-09 11:37:45.032110655] feed: git ["--git-dir=.git","--work-tree=.","--literal-pathspecs","update-index","-z","--index-info"] +[2018-04-09 11:37:45.035283993] process done ExitSuccess +[2018-04-09 11:37:45.036044409] read: git ["--git-dir=.git","--work-tree=.","--literal-pathspecs","symbolic-ref","-q","HEAD"] +[2018-04-09 11:37:45.039234835] process done ExitSuccess +[2018-04-09 11:37:45.039492196] read: git ["--git-dir=.git","--work-tree=.","--literal-pathspecs","show-ref","--hash","refs/heads/master"] +[2018-04-09 11:37:45.042630173] process done ExitSuccess +[2018-04-09 11:37:45.042981734] read: git ["--git-dir=.git","--work-tree=.","--literal-pathspecs","write-tree"] +[2018-04-09 11:37:45.047116516] process done ExitSuccess +[2018-04-09 11:37:45.047486142] read: git ["--git-dir=.git","--work-tree=.","--literal-pathspecs","rev-parse","1e93ab8142b8bb17e9e0c8905a1fd18ac21e4397:"] +[2018-04-09 11:37:45.051027357] process done ExitSuccess +[2018-04-09 11:37:45.051380622] chat: git ["--git-dir=.git","--work-tree=.","--literal-pathspecs","commit-tree","3b9cd858a5dfefbde85709982d51c2cf70c2bae7","--no-gpg-sign","-p","1e93ab8142b8bb17e9e0c8905a1fd18ac21e4397"] +[2018-04-09 11:37:45.055070888] process done ExitSuccess +[2018-04-09 11:37:45.055575957] call: git ["--git-dir=.git","--work-tree=.","--literal-pathspecs","update-ref","refs/heads/master","23edac69fdf1974824411bbc4e962bb37ba877d0"] +[2018-04-09 11:37:45.059674278] process done ExitSuccess +[2018-04-09 11:37:45.059981691] read: git ["--git-dir=.git","--work-tree=.","--literal-pathspecs","symbolic-ref","-q","HEAD"] +[2018-04-09 11:37:45.063167888] process done ExitSuccess +[2018-04-09 11:37:45.063491598] read: git ["--git-dir=.git","--work-tree=.","--literal-pathspecs","show-ref","refs/heads/master"] +[2018-04-09 11:37:45.066642148] process done ExitSuccess +[2018-04-09 11:37:45.066983555] call: git ["--git-dir=.git","--work-tree=.","--literal-pathspecs","branch","-f","synced/master","refs/heads/master"] +[2018-04-09 11:37:45.071106509] process done ExitSuccess +[2018-04-09 11:37:45.071691114] Pusher: Syncing with laptop +[2018-04-09 11:37:45.072263006] read: git ["--git-dir=.git","--work-tree=.","--literal-pathspecs","symbolic-ref","-q","HEAD"] +[2018-04-09 11:37:45.07493712] process done ExitSuccess +[2018-04-09 11:37:45.075246682] read: git ["--git-dir=.git","--work-tree=.","--literal-pathspecs","show-ref","refs/heads/master"] +[2018-04-09 11:37:45.078232243] process done ExitSuccess +[2018-04-09 11:37:45.078881383] read: git ["--git-dir=.git","--work-tree=.","--literal-pathspecs","symbolic-ref","-q","HEAD"] +[2018-04-09 11:37:45.081568528] process done ExitSuccess +[2018-04-09 11:37:45.081911449] read: git ["--git-dir=.git","--work-tree=.","--literal-pathspecs","show-ref","refs/heads/master"] +[2018-04-09 11:37:45.084855103] process done ExitSuccess +[2018-04-09 11:37:45.085170767] Pusher: pushing to [Remote { name ="laptop" }] +[2018-04-09 11:37:45.085937221] read: git ["--git-dir=.git","--work-tree=.","--literal-pathspecs","symbolic-ref","-q","HEAD"] +[2018-04-09 11:37:45.088659937] process done ExitSuccess +[2018-04-09 11:37:45.088961916] read: git ["--git-dir=.git","--work-tree=.","--literal-pathspecs","show-ref","refs/heads/master"] +[2018-04-09 11:37:45.091923163] process done ExitSuccess +[2018-04-09 11:37:45.093368856] call: git ["--git-dir=.git","--work-tree=.","--literal-pathspecs","push","laptop","+git-annex:synced/git-annex","master:synced/master"] +[2018-04-09 11:37:45.095171302] read: git ["--git-dir=.git","--work-tree=.","--literal-pathspecs","log","refs/heads/master..refs/heads/synced/master","--pretty=%H","-n1"] +[2018-04-09 11:37:45.09889799] process done ExitSuccess +[2018-04-09 11:37:45.099297672] Watcher: add Eingang.org +To laptop:Org/Privat + 1e93ab8..23edac6 master -> synced/master +[2018-04-09 11:37:45.142813244] read: git ["--git-dir=.git","--work-tree=.","--literal-pathspecs","symbolic-ref","-q","HEAD"] +[2018-04-09 11:37:45.14435156] process done ExitSuccess +[2018-04-09 11:37:45.144680359] read: git ["--git-dir=.git","--work-tree=.","--literal-pathspecs","push","laptop","git-annex"] +[2018-04-09 11:37:45.146874025] process done ExitSuccess +[2018-04-09 11:37:45.147115536] read: git ["--git-dir=.git","--work-tree=.","--literal-pathspecs","show-ref","refs/heads/master"] +[2018-04-09 11:37:45.150090387] process done ExitSuccess +[2018-04-09 11:37:45.150528273] read: git ["--git-dir=.git","--work-tree=.","--literal-pathspecs","log","refs/heads/master..refs/remotes/laptop/synced/master","--pretty=%H","-n1"] +[2018-04-09 11:37:45.154260324] process done ExitSuccess +[2018-04-09 11:37:45.178355356] process done ExitSuccess +[2018-04-09 11:37:45.178667467] read: git ["--git-dir=.git","--work-tree=.","--literal-pathspecs","push","laptop","master"] +[2018-04-09 11:37:45.215080611] read: git ["--git-dir=.git","--work-tree=.","--literal-pathspecs","symbolic-ref","-q","HEAD"] +[2018-04-09 11:37:45.216096995] process done ExitSuccess +[2018-04-09 11:37:45.218474735] process done ExitSuccess +[2018-04-09 11:37:45.218797245] read: git ["--git-dir=.git","--work-tree=.","--literal-pathspecs","show-ref","refs/heads/master"] +[2018-04-09 11:37:45.222475722] process done ExitSuccess +[2018-04-09 11:37:45.222892395] read: git ["--git-dir=.git","--work-tree=.","--literal-pathspecs","log","refs/heads/master..refs/remotes/laptop/master","--pretty=%H","-n1"] +[2018-04-09 11:37:45.22635232] process done ExitSuccess +[2018-04-09 11:37:46.073054429] Committer: committing 2 changes +[2018-04-09 11:37:46.073357348] Committer: Committing changes to git +(recording state in git...) +[2018-04-09 11:37:46.073668003] feed: xargs ["-0","git","--git-dir=.git","--work-tree=.","--literal-pathspecs","add","--force","--"] +[2018-04-09 11:37:46.115020265] process done ExitSuccess +[2018-04-09 11:37:46.115297828] read: git ["--git-dir=.git","--work-tree=.","--literal-pathspecs","symbolic-ref","-q","HEAD"] +[2018-04-09 11:37:46.117646357] process done ExitSuccess +[2018-04-09 11:37:46.118005719] read: git ["--git-dir=.git","--work-tree=.","--literal-pathspecs","show-ref","--hash","refs/heads/master"] +[2018-04-09 11:37:46.121575344] process done ExitSuccess +[2018-04-09 11:37:46.12191231] read: git ["--git-dir=.git","--work-tree=.","--literal-pathspecs","write-tree"] +[2018-04-09 11:37:46.12567614] process done ExitSuccess +[2018-04-09 11:37:46.126023972] read: git ["--git-dir=.git","--work-tree=.","--literal-pathspecs","rev-parse","23edac69fdf1974824411bbc4e962bb37ba877d0:"] +[2018-04-09 11:37:46.129133505] process done ExitSuccess +[2018-04-09 11:37:46.129595522] chat: git ["--git-dir=.git","--work-tree=.","--literal-pathspecs","commit-tree","a53bcee98b7883dc9ab92cfdc8a40c02557ccf04","--no-gpg-sign","-p","23edac69fdf1974824411bbc4e962bb37ba877d0"] +[2018-04-09 11:37:46.133911504] process done ExitSuccess +[2018-04-09 11:37:46.134222417] call: git ["--git-dir=.git","--work-tree=.","--literal-pathspecs","update-ref","refs/heads/master","498b482206f92064205ba7b5dd057191ad278e3e"] +[2018-04-09 11:37:46.13870126] process done ExitSuccess +[2018-04-09 11:37:46.138982474] read: git ["--git-dir=.git","--work-tree=.","--literal-pathspecs","symbolic-ref","-q","HEAD"] +[2018-04-09 11:37:46.141654394] process done ExitSuccess +[2018-04-09 11:37:46.142020296] read: git ["--git-dir=.git","--work-tree=.","--literal-pathspecs","show-ref","refs/heads/master"] +[2018-04-09 11:37:46.145649381] process done ExitSuccess +[2018-04-09 11:37:46.146010442] call: git ["--git-dir=.git","--work-tree=.","--literal-pathspecs","branch","-f","synced/master","refs/heads/master"] +[2018-04-09 11:37:46.150654031] process done ExitSuccess +[2018-04-09 11:37:46.151027708] read: git ["--git-dir=.git","--work-tree=.","--literal-pathspecs","symbolic-ref","-q","HEAD"] +[2018-04-09 11:37:46.153712467] process done ExitSuccess +[2018-04-09 11:37:46.154031515] read: git ["--git-dir=.git","--work-tree=.","--literal-pathspecs","show-ref","refs/heads/master"] +[2018-04-09 11:37:46.157608747] process done ExitSuccess +[2018-04-09 11:37:46.158107189] read: git ["--git-dir=.git","--work-tree=.","--literal-pathspecs","symbolic-ref","-q","HEAD"] +[2018-04-09 11:37:46.161188313] process done ExitSuccess +[2018-04-09 11:37:46.161500737] read: git ["--git-dir=.git","--work-tree=.","--literal-pathspecs","show-ref","refs/heads/master"] +[2018-04-09 11:37:46.164460849] process done ExitSuccess +[2018-04-09 11:37:46.165020868] read: git ["--git-dir=.git","--work-tree=.","--literal-pathspecs","log","refs/heads/master..refs/heads/synced/master","--pretty=%H","-n1"] +[2018-04-09 11:37:46.169117625] process done ExitSuccess +[2018-04-09 11:37:47.217100926] Pusher: Syncing with laptop +[2018-04-09 11:37:47.217699349] read: git ["--git-dir=.git","--work-tree=.","--literal-pathspecs","symbolic-ref","-q","HEAD"] +[2018-04-09 11:37:47.221401849] process done ExitSuccess +[2018-04-09 11:37:47.221833584] read: git ["--git-dir=.git","--work-tree=.","--literal-pathspecs","show-ref","refs/heads/master"] +[2018-04-09 11:37:47.225466461] process done ExitSuccess +[2018-04-09 11:37:47.225805241] Pusher: pushing to [Remote { name ="laptop" }] +[2018-04-09 11:37:47.22690315] call: git ["--git-dir=.git","--work-tree=.","--literal-pathspecs","push","laptop","+git-annex:synced/git-annex","master:synced/master"] +Everything up-to-date +[2018-04-09 11:37:47.262439321] read: git ["--git-dir=.git","--work-tree=.","--literal-pathspecs","symbolic-ref","-q","HEAD"] +[2018-04-09 11:37:47.263256084] process done ExitSuccess +[2018-04-09 11:37:47.263663099] read: git ["--git-dir=.git","--work-tree=.","--literal-pathspecs","push","laptop","git-annex"] +[2018-04-09 11:37:47.265655322] process done ExitSuccess +[2018-04-09 11:37:47.266023431] read: git ["--git-dir=.git","--work-tree=.","--literal-pathspecs","show-ref","refs/heads/master"] +[2018-04-09 11:37:47.270704463] process done ExitSuccess +[2018-04-09 11:37:47.271058723] read: git ["--git-dir=.git","--work-tree=.","--literal-pathspecs","log","refs/heads/master..refs/remotes/laptop/synced/master","--pretty=%H","-n1"] +[2018-04-09 11:37:47.274346464] process done ExitSuccess +[2018-04-09 11:37:47.298748348] process done ExitSuccess +[2018-04-09 11:37:47.299087186] read: git ["--git-dir=.git","--work-tree=.","--literal-pathspecs","push","laptop","master"] +[2018-04-09 11:37:47.333645742] read: git ["--git-dir=.git","--work-tree=.","--literal-pathspecs","symbolic-ref","-q","HEAD"] +[2018-04-09 11:37:47.334812765] process done ExitSuccess +[2018-04-09 11:37:47.33762396] process done ExitSuccess +[2018-04-09 11:37:47.337963852] read: git ["--git-dir=.git","--work-tree=.","--literal-pathspecs","show-ref","refs/heads/master"] +[2018-04-09 11:37:47.341750677] process done ExitSuccess +[2018-04-09 11:37:47.342184466] read: git ["--git-dir=.git","--work-tree=.","--literal-pathspecs","log","refs/heads/master..refs/remotes/laptop/master","--pretty=%H","-n1"] +[2018-04-09 11:37:47.345724578] process done ExitSuccess +"""]] + + +`git log` +[[!format sh """ +commit 498b482206f92064205ba7b5dd057191ad278e3e +Author: me +Date: Mon Apr 9 11:37:46 2018 +0200 + + git-annex in server + +:000000 100644 0000000... 0bf8613... A Eingang.org + +commit 23edac69fdf1974824411bbc4e962bb37ba877d0 +Author: me +Date: Mon Apr 9 11:37:45 2018 +0200 + + git-annex in server + +:100644 000000 0bf8613... 0000000... D Eingang.org + +commit 1e93ab8142b8bb17e9e0c8905a1fd18ac21e4397 +Author: me +Date: Mon Apr 9 11:37:44 2018 +0200 + + git-annex in laptop + +:100644 100644 450fc30... 0bf8613... M Eingang.org + +"""]] diff --git a/doc/bugs/Assumes_OpenSSH_version_newer_than_debian_stable.mdwn b/doc/bugs/Assumes_OpenSSH_version_newer_than_debian_stable.mdwn new file mode 100644 index 0000000000..9c55bff8a8 --- /dev/null +++ b/doc/bugs/Assumes_OpenSSH_version_newer_than_debian_stable.mdwn @@ -0,0 +1,53 @@ +### Please describe the problem. +Git-annex is hanging for me on all operations if I have a customized OpenSSH host as a remote. +This is because git-annex uses a custom config file that uses directives introduced in OpenSSH 7.3p1 . I have OpenSSH 6.7p1 . + +### What steps will reproduce the problem? +On Debian Jessie, specify a custom host in ~/.ssh/config such as + + Host custom-host + HostName realhost.com + +Add it as a remote and try to use git-annex + + $ git remote add custom user@custom-host + $ git annex vicfg --debug + +It just hangs. + +### What version of git-annex are you using? On what operating system? + git-annex version: 6.20161211-gc3ab3c668 + PRETTY_NAME="Debian GNU/Linux 8 (jessie)" + +### Please provide any additional information below. + +[[!format sh """ +# If you can, paste a complete transcript of the problem occurring here. +# If the problem is with the git-annex assistant, paste in .git/annex/daemon.log + +annex@host:~/annex$ git annex vicfg --debug +[2017-01-16 12:01:58.104071891] read: git ["--git-dir=.git","--work-tree=.","--literal-pathspecs","show-ref","git-annex"] +[2017-01-16 12:01:58.110698965] process done ExitSuccess +[2017-01-16 12:01:58.11093666] read: git ["--git-dir=.git","--work-tree=.","--literal-pathspecs","show-ref","--hash","refs/heads/git-annex"] +[2017-01-16 12:01:58.116640973] process done ExitSuccess +[2017-01-16 12:01:58.123254411] chat: git ["--git-dir=.git","--work-tree=.","--literal-pathspecs","cat-file","--batch"] +[2017-01-16 12:01:58.125003224] chat: git ["--git-dir=.git","--work-tree=.","--literal-pathspecs","cat-file","--batch-check=%(objectname) %(objecttype) %(objectsize)"] +[2017-01-16 12:01:58.131239655] read: ssh ["-S",".git/annex/ssh/annex@annex-whonix-standalone","-o","ControlMaster=auto","-o","ControlPersist=yes","-F",".git/annex/ssh.config","-T","annex@annex-whonix-standalone","git-annex-shell 'configlist' '/~/annex' '--debug'"] +^C +annex@host:~/annex$ cat .git/annex/ssh.config +IgnoreUnknown Include +Include ~/.ssh/config +Include /etc/ssh/ssh_config +ServerAliveInterval 60 + +# End of transcript or log. +"""]] + +The problem is the use of the `Include` directive which is not understood by my release of OpenSSH. + + +### Have you had any luck using git-annex before? (Sometimes we get tired of reading bug reports all day and a lil' positive end note does wonders) + +Git-annex rocks !!!! + +> [[done]]; this was already fixed --[[Joey]] diff --git a/doc/bugs/Assumes_OpenSSH_version_newer_than_debian_stable/comment_1_5a1f9a2014501072188fb235f1b03726._comment b/doc/bugs/Assumes_OpenSSH_version_newer_than_debian_stable/comment_1_5a1f9a2014501072188fb235f1b03726._comment new file mode 100644 index 0000000000..700e234cab --- /dev/null +++ b/doc/bugs/Assumes_OpenSSH_version_newer_than_debian_stable/comment_1_5a1f9a2014501072188fb235f1b03726._comment @@ -0,0 +1,8 @@ +[[!comment format=mdwn + username="xloem" + avatar="http://cdn.libravatar.org/avatar/b8c087f7c5e6a9358748f0727c077f3b" + subject="comment 1" + date="2017-01-16T12:47:03Z" + content=""" +Fixed by upgrading to newer version ! Sorry for the re-bug. +"""]] diff --git a/doc/bugs/Assumes_OpenSSH_version_newer_than_debian_stable/comment_2_0bc9c0e6d72d6b0d9173e55eca8b107a._comment b/doc/bugs/Assumes_OpenSSH_version_newer_than_debian_stable/comment_2_0bc9c0e6d72d6b0d9173e55eca8b107a._comment new file mode 100644 index 0000000000..eb33165856 --- /dev/null +++ b/doc/bugs/Assumes_OpenSSH_version_newer_than_debian_stable/comment_2_0bc9c0e6d72d6b0d9173e55eca8b107a._comment @@ -0,0 +1,8 @@ +[[!comment format=mdwn + username="xloem" + avatar="http://cdn.libravatar.org/avatar/b8c087f7c5e6a9358748f0727c077f3b" + subject="comment 2" + date="2017-01-16T12:48:36Z" + content=""" +how do I move the bug to done? +"""]] diff --git a/doc/bugs/Attempting_to_repair_repository_almost_every_day.mdwn b/doc/bugs/Attempting_to_repair_repository_almost_every_day.mdwn new file mode 100644 index 0000000000..6925606f0e --- /dev/null +++ b/doc/bugs/Attempting_to_repair_repository_almost_every_day.mdwn @@ -0,0 +1,5 @@ +I'm using the webapp on my two main computers with up-to-date Debian (one has testing and the other one sid) and I have consistency checks scheduled every day. + +The problem is that almost every time the check runs I'm getting the "Attempting to repair" message and git-annex starts using 100% CPU for quite a while, but after it it seams to have done no change to the git tree or the files. + +The two computers are never on at the same time, but they are constantly syncing files (through box.com), I don't now if downloading files while the check is in progress might have something to do. diff --git a/doc/bugs/Auto-repair_greatly_slows_down_the_machine.mdwn b/doc/bugs/Auto-repair_greatly_slows_down_the_machine.mdwn new file mode 100644 index 0000000000..6c1420e54a --- /dev/null +++ b/doc/bugs/Auto-repair_greatly_slows_down_the_machine.mdwn @@ -0,0 +1,21 @@ +### Please describe the problem. + +The assistant regulary ends up trying to perform repair (I don't know why, it happens fairly often, once a week or so). When it does so, it ends up creating a huge (2.4G) .git/objects directory, and a git prune-packed process uses so much I/O the machine really slows down. + +### What steps will reproduce the problem? + +I don't have any reliable way to reproduce it. The repository ends up being attempted to be repaired around once a week. This week the repair (and the slowdown) also happened on a second computer. + +### What version of git-annex are you using? On what operating system? + +git-annex version: 5.20140221-gbdfc8e1 (using the standalone 64bit builds) + +This is on an up-to-date Arch Linux. It also happened on Fedora 20. + +### Please provide any additional information below. + +The daemon.log is fairly long, but not particulary interesting: [[https://ssl.zerodogg.org/~zerodogg/private/tmp/daemon.log-2014-02-25.1]] + +The «resource vanished (Broken pipe)» at the end is the result of me killing the prune-packed in order to be able to use the machine again. + +[[!tag moreinfo]] diff --git a/doc/bugs/Auto-repair_greatly_slows_down_the_machine/comment_10_d46c6314bff75a0ae679bc2358b28c2b._comment b/doc/bugs/Auto-repair_greatly_slows_down_the_machine/comment_10_d46c6314bff75a0ae679bc2358b28c2b._comment new file mode 100644 index 0000000000..5fb031e256 --- /dev/null +++ b/doc/bugs/Auto-repair_greatly_slows_down_the_machine/comment_10_d46c6314bff75a0ae679bc2358b28c2b._comment @@ -0,0 +1,8 @@ +[[!comment format=mdwn + username="EskildHustvedt" + ip="80.202.103.55" + subject="comment 10" + date="2014-06-07T05:47:25Z" + content=""" +Given that it's ususally the same repo, I'm going to try to create a completely fresh repository instead and see if the frequent auto-repairs are stopped by that. +"""]] diff --git a/doc/bugs/Auto-repair_greatly_slows_down_the_machine/comment_11_1c2c3adfbccd2b14a7d1c4618800e735._comment b/doc/bugs/Auto-repair_greatly_slows_down_the_machine/comment_11_1c2c3adfbccd2b14a7d1c4618800e735._comment new file mode 100644 index 0000000000..904ec92f55 --- /dev/null +++ b/doc/bugs/Auto-repair_greatly_slows_down_the_machine/comment_11_1c2c3adfbccd2b14a7d1c4618800e735._comment @@ -0,0 +1,8 @@ +[[!comment format=mdwn + username="http://joeyh.name/" + ip="108.236.230.124" + subject="comment 11" + date="2014-06-10T19:31:38Z" + content=""" +Disable consistency checks for the repo in the webapp UI, wait until long enough that you're sure the assistant would be wanting to repair it again if you hadn't disabled that, and run `git annex repair` at the console so I can see what's going on. +"""]] diff --git a/doc/bugs/Auto-repair_greatly_slows_down_the_machine/comment_1_a52e4ef04209d0a2449165e2b4cb9ccc._comment b/doc/bugs/Auto-repair_greatly_slows_down_the_machine/comment_1_a52e4ef04209d0a2449165e2b4cb9ccc._comment new file mode 100644 index 0000000000..b50a0cfb82 --- /dev/null +++ b/doc/bugs/Auto-repair_greatly_slows_down_the_machine/comment_1_a52e4ef04209d0a2449165e2b4cb9ccc._comment @@ -0,0 +1,10 @@ +[[!comment format=mdwn + username="http://joeyh.name/" + ip="209.250.56.146" + subject="comment 1" + date="2014-02-26T17:38:05Z" + content=""" +Auto repair is not intended to be a common occurrance. It means something went badly, horribly wrong on your machine, and it lost data that git wrote to disk. It's more important in such a scenraio to get back to a working system eventually than to do something fast or inexpensively. + +If you're seeing the need for auto repair on a weekly basis, your computer is failing in a horrible way horribly frequently, and the thing to do is to find out why, and fix that. Perhaps you need to start cleanly shutting down the system. Perhaps something is causing your computer to crash, and you need to fix that. +"""]] diff --git a/doc/bugs/Auto-repair_greatly_slows_down_the_machine/comment_2_9f5340ab1012f335af0c246b82c1a777._comment b/doc/bugs/Auto-repair_greatly_slows_down_the_machine/comment_2_9f5340ab1012f335af0c246b82c1a777._comment new file mode 100644 index 0000000000..6cbd58f2f3 --- /dev/null +++ b/doc/bugs/Auto-repair_greatly_slows_down_the_machine/comment_2_9f5340ab1012f335af0c246b82c1a777._comment @@ -0,0 +1,16 @@ +[[!comment format=mdwn + username="EskildHustvedt" + ip="80.202.213.223" + subject="comment 2" + date="2014-02-27T14:16:11Z" + content=""" +That's the thing. The drive is fine, I've fscked it, and the machine is always shut down cleanly (and it is very stable, can't remember the last time it crashed). So there's no reason why this should be happening, and since git-annex doesn't say anything about why it started the auto-repair, I'm unable to track it down further. + +It's also always this repository. I have several assistant managed repos on the same machine, and this is the only one that git-annex regulary starts repairing (and the only one that it has auto-repaired on another box as well). No files in the repository itself has ever been noticed as corrupt, nor has there been any indications anywhere else about problems, hardware or filesystem (nothing wrong in the 40 or so git repos on the machine, no problems with software installed on the drive, never seen anything in dmesg). + +It's happening often enough for me to have to consider dropping use of the assistant. It could still be a problem with the machine (though it seems unlikely, given that it's consistently the same repo, and that it has done it with the same repo on other machines), but git-annex doesn't provide any information about it. + +Could you add some more information about what it thinks is wrong to the logging? + +Funnily enough, git-annex started repairing it again as I was writing this. +"""]] diff --git a/doc/bugs/Auto-repair_greatly_slows_down_the_machine/comment_3_67bfccf0934075559d439b1deafc001e._comment b/doc/bugs/Auto-repair_greatly_slows_down_the_machine/comment_3_67bfccf0934075559d439b1deafc001e._comment new file mode 100644 index 0000000000..edf7c781b8 --- /dev/null +++ b/doc/bugs/Auto-repair_greatly_slows_down_the_machine/comment_3_67bfccf0934075559d439b1deafc001e._comment @@ -0,0 +1,9 @@ +[[!comment format=mdwn + username="https://www.google.com/accounts/o8/id?id=AItOawnX1msQxnLoSeu7q-i-c9BWghonsN7Qmns" + nickname="Jan Ulrich" + subject="comment 3" + date="2014-03-05T11:47:06Z" + content=""" +I observe something similar. Today git-annex started to repair a repository. CPU is at 100%, I can still work because I have a multikernel system, so I'll wait and see if it comes to an end. + +"""]] diff --git a/doc/bugs/Auto-repair_greatly_slows_down_the_machine/comment_4_5fa785aa759d1a1917f2a292324fe5ec._comment b/doc/bugs/Auto-repair_greatly_slows_down_the_machine/comment_4_5fa785aa759d1a1917f2a292324fe5ec._comment new file mode 100644 index 0000000000..3a43c92603 --- /dev/null +++ b/doc/bugs/Auto-repair_greatly_slows_down_the_machine/comment_4_5fa785aa759d1a1917f2a292324fe5ec._comment @@ -0,0 +1,8 @@ +[[!comment format=mdwn + username="https://www.google.com/accounts/o8/id?id=AItOawnX1msQxnLoSeu7q-i-c9BWghonsN7Qmns" + nickname="Jan Ulrich" + subject="comment 4" + date="2014-03-06T06:21:59Z" + content=""" +Just for the records. I just shutdown the daemon because it ran all night long with 100%. +"""]] diff --git a/doc/bugs/Auto-repair_greatly_slows_down_the_machine/comment_5_9fe529034ad0115792b58d7da99c167e._comment b/doc/bugs/Auto-repair_greatly_slows_down_the_machine/comment_5_9fe529034ad0115792b58d7da99c167e._comment new file mode 100644 index 0000000000..4683165c37 --- /dev/null +++ b/doc/bugs/Auto-repair_greatly_slows_down_the_machine/comment_5_9fe529034ad0115792b58d7da99c167e._comment @@ -0,0 +1,8 @@ +[[!comment format=mdwn + username="http://joeyh.name/" + ip="209.250.56.146" + subject="comment 5" + date="2014-03-06T18:14:37Z" + content=""" +auto-repair is only done if git fsck detects a problem. You can run git fsck yourself to see. +"""]] diff --git a/doc/bugs/Auto-repair_greatly_slows_down_the_machine/comment_6_93ed991ef2a74c18575073ca72e06185._comment b/doc/bugs/Auto-repair_greatly_slows_down_the_machine/comment_6_93ed991ef2a74c18575073ca72e06185._comment new file mode 100644 index 0000000000..05935b034d --- /dev/null +++ b/doc/bugs/Auto-repair_greatly_slows_down_the_machine/comment_6_93ed991ef2a74c18575073ca72e06185._comment @@ -0,0 +1,8 @@ +[[!comment format=mdwn + username="https://www.google.com/accounts/o8/id?id=AItOawnX1msQxnLoSeu7q-i-c9BWghonsN7Qmns" + nickname="Jan Ulrich" + subject="comment 6" + date="2014-03-10T14:14:06Z" + content=""" +I manually ran git fsck without problems but git-annex still wants to repair something. +"""]] diff --git a/doc/bugs/Auto-repair_greatly_slows_down_the_machine/comment_7_4649fa11745ff668e43833209811d005._comment b/doc/bugs/Auto-repair_greatly_slows_down_the_machine/comment_7_4649fa11745ff668e43833209811d005._comment new file mode 100644 index 0000000000..58499da8bb --- /dev/null +++ b/doc/bugs/Auto-repair_greatly_slows_down_the_machine/comment_7_4649fa11745ff668e43833209811d005._comment @@ -0,0 +1,10 @@ +[[!comment format=mdwn + username="http://joeyh.name/" + ip="209.250.56.3" + subject="comment 7" + date="2014-05-21T18:31:54Z" + content=""" +It seems to me that if you don't want to repair your repository, you can just go into the webapp and disable all scheduled consistency check jobs. + +If someone with this problem would like to run \"git annex repair\" at the console, and paste the output, perhaps I could then see why it thinks it needs to repair the repository. So far, I have nothing to go on, and no proof that this is a bug at all, and not just people with actually corrupted repositories that really do need to be repaired. +"""]] diff --git a/doc/bugs/Auto-repair_greatly_slows_down_the_machine/comment_8_6138a48f0b7e8be2bb66430397afdf5a._comment b/doc/bugs/Auto-repair_greatly_slows_down_the_machine/comment_8_6138a48f0b7e8be2bb66430397afdf5a._comment new file mode 100644 index 0000000000..33944ac273 --- /dev/null +++ b/doc/bugs/Auto-repair_greatly_slows_down_the_machine/comment_8_6138a48f0b7e8be2bb66430397afdf5a._comment @@ -0,0 +1,8 @@ +[[!comment format=mdwn + username="EskildHustvedt" + ip="80.202.103.55" + subject="comment 8" + date="2014-05-21T19:19:30Z" + content=""" +The trouble with providing more information (outside of the logfile above) is that by the time I realize that there is a problem it is because git-annex has started to repair. Perhaps an option that makes git-annex write a ~/.git/annex/repair-reason.log file with the reason (ie. git fsck output etc.) for starting the repair could help? +"""]] diff --git a/doc/bugs/Auto-repair_greatly_slows_down_the_machine/comment_9_3da5940bb2e9689b00239eec9a073f1c._comment b/doc/bugs/Auto-repair_greatly_slows_down_the_machine/comment_9_3da5940bb2e9689b00239eec9a073f1c._comment new file mode 100644 index 0000000000..8bea236edc --- /dev/null +++ b/doc/bugs/Auto-repair_greatly_slows_down_the_machine/comment_9_3da5940bb2e9689b00239eec9a073f1c._comment @@ -0,0 +1,8 @@ +[[!comment format=mdwn + username="EskildHustvedt" + ip="80.202.103.55" + subject="comment 9" + date="2014-06-06T16:13:55Z" + content=""" +Any chance of any progress on this? Either a way to disable it, or some additional logging to track down the problem? git-annex is currently repairing one particular repository several times a month accross different machines, rendering git-annex practically unusable. +"""]] diff --git a/doc/bugs/Bootstrap3_icons_missing_on_Android.mdwn b/doc/bugs/Bootstrap3_icons_missing_on_Android.mdwn new file mode 100644 index 0000000000..480825bc9c --- /dev/null +++ b/doc/bugs/Bootstrap3_icons_missing_on_Android.mdwn @@ -0,0 +1,7 @@ +I’m running git-annex 5.20140517-gee56d21 on Android 4.3. + +A number of icons seem to be missing since the bootstrap3 update: + +git-annex missing icons + +[[!tag confirmed done]] diff --git a/doc/bugs/Bootstrap3_icons_missing_on_Android/comment_10_da4e26c77376a8171493a815bdb51ff1._comment b/doc/bugs/Bootstrap3_icons_missing_on_Android/comment_10_da4e26c77376a8171493a815bdb51ff1._comment new file mode 100644 index 0000000000..78732490fa --- /dev/null +++ b/doc/bugs/Bootstrap3_icons_missing_on_Android/comment_10_da4e26c77376a8171493a815bdb51ff1._comment @@ -0,0 +1,10 @@ +[[!comment format=mdwn + username="https://www.google.com/accounts/o8/id?id=AItOawmqz6wCn-Q1vzrsHGvEJHOt_T5ZESilxhc" + nickname="Sören" + subject="comment 10" + date="2014-05-28T15:03:26Z" + content=""" +Seems to work! The icons show up on android now. + +But JavaScript seems to be broken instead, dropdown menus don't work. Maybe more unwanted evil splicing in other files? +"""]] diff --git a/doc/bugs/Bootstrap3_icons_missing_on_Android/comment_11_506e26de284a6429366def33b007bc0c._comment b/doc/bugs/Bootstrap3_icons_missing_on_Android/comment_11_506e26de284a6429366def33b007bc0c._comment new file mode 100644 index 0000000000..ba3fff8011 --- /dev/null +++ b/doc/bugs/Bootstrap3_icons_missing_on_Android/comment_11_506e26de284a6429366def33b007bc0c._comment @@ -0,0 +1,8 @@ +[[!comment format=mdwn + username="https://www.google.com/accounts/o8/id?id=AItOawmqz6wCn-Q1vzrsHGvEJHOt_T5ZESilxhc" + nickname="Sören" + subject="comment 11" + date="2014-05-28T17:30:43Z" + content=""" +Ok it seems like the linebreak at the end of each javascript file (0x0a) is replaced by a null byte on android. This probably causes the browser not to accept it as javascript so it's not executed. +"""]] diff --git a/doc/bugs/Bootstrap3_icons_missing_on_Android/comment_12_a77477951576fc657d693811f11f9975._comment b/doc/bugs/Bootstrap3_icons_missing_on_Android/comment_12_a77477951576fc657d693811f11f9975._comment new file mode 100644 index 0000000000..23b523bbe4 --- /dev/null +++ b/doc/bugs/Bootstrap3_icons_missing_on_Android/comment_12_a77477951576fc657d693811f11f9975._comment @@ -0,0 +1,24 @@ +[[!comment format=mdwn + username="http://joeyh.name/" + ip="209.250.56.176" + subject="comment 12" + date="2014-05-28T22:31:21Z" + content=""" +The evilspliced data for eg, bootstrap.js omits that last byte. So presumably we overflow one byte and find a null (if we're lucky..). + +I checked the -ddump-splices output, and it omits that last newline! + +
    +              \});\
+              \\
+              \})( jQuery );\"#),
+
    + +Indeed, ghc omits any number of trailing newlines in this display. + +This is probably only recently a problem because of the wacky use of a unsafe bytestring + length that file-embed has started to do. Before, the newline was not included, but that's still valid JS. + +So, one fix would be to have the EvilSplicer rewrite the unsafePackAddressLen back to a regular bytestring construction. But this would need some tricky parsing (need to find the end of the string to remove the # from it). Alternatively, could remove trailing newlines from all the static js and css files. (Luckily none of the other static files end in a newline, and ghc is careful to preserve ending NULs etc). + +For now, I've removed the trailing newlines from the files. +"""]] diff --git a/doc/bugs/Bootstrap3_icons_missing_on_Android/comment_13_c3ef67bded8ed3c511a8ea85b8bd81a5._comment b/doc/bugs/Bootstrap3_icons_missing_on_Android/comment_13_c3ef67bded8ed3c511a8ea85b8bd81a5._comment new file mode 100644 index 0000000000..dc8b24ba1b --- /dev/null +++ b/doc/bugs/Bootstrap3_icons_missing_on_Android/comment_13_c3ef67bded8ed3c511a8ea85b8bd81a5._comment @@ -0,0 +1,8 @@ +[[!comment format=mdwn + username="https://www.google.com/accounts/o8/id?id=AItOawmqz6wCn-Q1vzrsHGvEJHOt_T5ZESilxhc" + nickname="Sören" + subject="comment 13" + date="2014-05-29T16:48:18Z" + content=""" +Works fine now (tested on android 4.4). +"""]] diff --git a/doc/bugs/Bootstrap3_icons_missing_on_Android/comment_1_97cf233d8401d642337d5fe9dd525e2b._comment b/doc/bugs/Bootstrap3_icons_missing_on_Android/comment_1_97cf233d8401d642337d5fe9dd525e2b._comment new file mode 100644 index 0000000000..15b6018b52 --- /dev/null +++ b/doc/bugs/Bootstrap3_icons_missing_on_Android/comment_1_97cf233d8401d642337d5fe9dd525e2b._comment @@ -0,0 +1,8 @@ +[[!comment format=mdwn + username="http://joeyh.name/" + ip="209.250.56.3" + subject="comment 1" + date="2014-05-21T17:10:24Z" + content=""" +reproduced this with android 4.0 +"""]] diff --git a/doc/bugs/Bootstrap3_icons_missing_on_Android/comment_2_4981a0a894b77e94a3bffec82b0f6e51._comment b/doc/bugs/Bootstrap3_icons_missing_on_Android/comment_2_4981a0a894b77e94a3bffec82b0f6e51._comment new file mode 100644 index 0000000000..5404fbe497 --- /dev/null +++ b/doc/bugs/Bootstrap3_icons_missing_on_Android/comment_2_4981a0a894b77e94a3bffec82b0f6e51._comment @@ -0,0 +1,16 @@ +[[!comment format=mdwn + username="http://joeyh.name/" + ip="209.250.56.3" + subject="comment 2" + date="2014-05-21T18:06:09Z" + content=""" +I've verified that the assistant is serving up the files, such as http://127.0.0.1:$port/static/fonts/glyphicons-halflings-regular.svg + +I've verified that the files it's serving have the right content. + +This seems to only leave the likely explanation that the android web browser doesn't support the technology bootstrap 3 is using for its icons. This is svg, plus ttf, plus Embedded OpenType and Web Open Font Format. (I suppose it uses different ones on different browsers.) + +I'm surprised that bootstrap would be using web fonts technology if it's not supported on Android, or wouldn't have a fallback to the plain old image method old versions of bootstrap used. + +I have found some bootstrap bugs about icons not showing up on Android, like this one ... but those are about specific icons not showing up due to unicode issues. Not about all icons failing to show up. +"""]] diff --git a/doc/bugs/Bootstrap3_icons_missing_on_Android/comment_3_dbe960392aaa0839886381820754814a._comment b/doc/bugs/Bootstrap3_icons_missing_on_Android/comment_3_dbe960392aaa0839886381820754814a._comment new file mode 100644 index 0000000000..f6f4e48798 --- /dev/null +++ b/doc/bugs/Bootstrap3_icons_missing_on_Android/comment_3_dbe960392aaa0839886381820754814a._comment @@ -0,0 +1,10 @@ +[[!comment format=mdwn + username="https://www.google.com/accounts/o8/id?id=AItOawmqz6wCn-Q1vzrsHGvEJHOt_T5ZESilxhc" + nickname="Sören" + subject="comment 3" + date="2014-05-21T18:47:42Z" + content=""" +I don't think it's an issue with the Android browser. It happens with Chrome and Firefox on Android as well. Moreover, the icons still don't show up if open the Webapp running on Android over network from a desktop browser. Firebug doesn't even show the request that should load the font file (compare to the desktop version where it loads the woff file in Firefox). So I suspect the Web server behaves different somehow. Some weird blocking issue maybe? + +Not sure if this is related but I noticed a strange behavior: If I try to download the font file in a new tab, it blocks until I refresh the tab running the webapp. Only seen on desktop Firefox. Doesn't happen in Chromium. +"""]] diff --git a/doc/bugs/Bootstrap3_icons_missing_on_Android/comment_4_cdf9621fe3e9f8eb8b7af88f779a7116._comment b/doc/bugs/Bootstrap3_icons_missing_on_Android/comment_4_cdf9621fe3e9f8eb8b7af88f779a7116._comment new file mode 100644 index 0000000000..dabc21ccee --- /dev/null +++ b/doc/bugs/Bootstrap3_icons_missing_on_Android/comment_4_cdf9621fe3e9f8eb8b7af88f779a7116._comment @@ -0,0 +1,8 @@ +[[!comment format=mdwn + username="https://www.google.com/accounts/o8/id?id=AItOawmqz6wCn-Q1vzrsHGvEJHOt_T5ZESilxhc" + nickname="Sören" + subject="comment 4" + date="2014-05-21T19:19:45Z" + content=""" +I forgot to mention that the --listen argument doesn't seem to work on Android so I used ssh -R in order to access the webapp from my desktop. +"""]] diff --git a/doc/bugs/Bootstrap3_icons_missing_on_Android/comment_5_a2318f094b30f58fefc072807896c099._comment b/doc/bugs/Bootstrap3_icons_missing_on_Android/comment_5_a2318f094b30f58fefc072807896c099._comment new file mode 100644 index 0000000000..9f18e604e4 --- /dev/null +++ b/doc/bugs/Bootstrap3_icons_missing_on_Android/comment_5_a2318f094b30f58fefc072807896c099._comment @@ -0,0 +1,8 @@ +[[!comment format=mdwn + username="https://www.google.com/accounts/o8/id?id=AItOawmqz6wCn-Q1vzrsHGvEJHOt_T5ZESilxhc" + nickname="Sören" + subject="comment 5" + date="2014-05-27T19:58:37Z" + content=""" +On Android, the webserver returns a broken bootstrap.css. It seems like some binary garbage is appended to the end of the css file that looks suspiciously like the content or some fragment of the font file \"glyphicons-halflings-regular.eot\". Reason unknown. +"""]] diff --git a/doc/bugs/Bootstrap3_icons_missing_on_Android/comment_6_7c2f1d914793798708ca13b6ac3dd474._comment b/doc/bugs/Bootstrap3_icons_missing_on_Android/comment_6_7c2f1d914793798708ca13b6ac3dd474._comment new file mode 100644 index 0000000000..8806de16f4 --- /dev/null +++ b/doc/bugs/Bootstrap3_icons_missing_on_Android/comment_6_7c2f1d914793798708ca13b6ac3dd474._comment @@ -0,0 +1,29 @@ +[[!comment format=mdwn + username="http://joeyh.name/" + ip="209.250.56.176" + subject="comment 6" + date="2014-05-27T20:23:03Z" + content=""" +I checked the evilspliced tmp/androidtree/Assistant/WebApp/Types.hs, and the bootstrap.css there ends correctly, nothing weird appended. + +The TH generated for this: + +
    +       (\"css/bootstrap.css\", 
+        GHC.IO.unsafePerformIO
+          (Data.ByteString.Unsafe.unsafePackAddressLen
+             121220
+             file_content_omitted)
+
    + +That looks right, in particular the size is the same as the number of bytes in the file. + +Hmm.. To double-check, I edited the file so that the string containing the content of this file was defined in foo. + +
    +*Main> length foo
+113023
+
    + +This might be innocuous; there are some unicode characters in the string that encode to multiple bytes. OTOH, if you see exactly 8197 bytes of extra garbage appended, I think we have our culprit. +"""]] diff --git a/doc/bugs/Bootstrap3_icons_missing_on_Android/comment_7_a24b5165590b5d58919da1003cd20c54._comment b/doc/bugs/Bootstrap3_icons_missing_on_Android/comment_7_a24b5165590b5d58919da1003cd20c54._comment new file mode 100644 index 0000000000..194d1c8311 --- /dev/null +++ b/doc/bugs/Bootstrap3_icons_missing_on_Android/comment_7_a24b5165590b5d58919da1003cd20c54._comment @@ -0,0 +1,8 @@ +[[!comment format=mdwn + username="http://joeyh.name/" + ip="209.250.56.176" + subject="comment 7" + date="2014-05-28T00:02:42Z" + content=""" +The string is indeed truncated; it seems that something, probably the EvilSplicer is doing effectively a s/\w+:// on the string. +"""]] diff --git a/doc/bugs/Bootstrap3_icons_missing_on_Android/comment_8_71ed14b8e5a898b1643d5e37591c2476._comment b/doc/bugs/Bootstrap3_icons_missing_on_Android/comment_8_71ed14b8e5a898b1643d5e37591c2476._comment new file mode 100644 index 0000000000..f5bc22ec74 --- /dev/null +++ b/doc/bugs/Bootstrap3_icons_missing_on_Android/comment_8_71ed14b8e5a898b1643d5e37591c2476._comment @@ -0,0 +1,8 @@ +[[!comment format=mdwn + username="http://joeyh.name/" + ip="209.250.56.176" + subject="comment 8" + date="2014-05-28T00:04:58Z" + content=""" +Almost certinaly the EvilSplicer; it contains a hack to remove package-version qualifications from symbols. (s/package-version:symbol/symbol/) +"""]] diff --git a/doc/bugs/Bootstrap3_icons_missing_on_Android/comment_9_15357e33c2431080d45f7cef5f4f1209._comment b/doc/bugs/Bootstrap3_icons_missing_on_Android/comment_9_15357e33c2431080d45f7cef5f4f1209._comment new file mode 100644 index 0000000000..65db845566 --- /dev/null +++ b/doc/bugs/Bootstrap3_icons_missing_on_Android/comment_9_15357e33c2431080d45f7cef5f4f1209._comment @@ -0,0 +1,8 @@ +[[!comment format=mdwn + username="http://joeyh.name/" + ip="209.250.56.176" + subject="comment 9" + date="2014-05-28T00:37:30Z" + content=""" +[[!commit 37da3d7b5dcfd328aee6a7a91508b02edf720bca]] should fix this problem. I've verified the truncation is not present in the spliced source; lack bandwidh to test the binary right now. +"""]] diff --git a/doc/bugs/Build_failure__58___Utility__47__QuickCheck.hs__58__38__58__10__58___error__58___Duplicate_instance_declarations.mdwn b/doc/bugs/Build_failure__58___Utility__47__QuickCheck.hs__58__38__58__10__58___error__58___Duplicate_instance_declarations.mdwn new file mode 100644 index 0000000000..400b7ae7bf --- /dev/null +++ b/doc/bugs/Build_failure__58___Utility__47__QuickCheck.hs__58__38__58__10__58___error__58___Duplicate_instance_declarations.mdwn @@ -0,0 +1,41 @@ +### Please describe the problem. +git-annex-6.20170520 no longer builds successfully. My last successful build of git-annex-6.20170520 was Jun 12, 2017, but something has probably changed in at least one of the dependencies on Hackage since then. + +The build fails on all three Homebrew CI nodes (macOS 10.10, 10.11, and 10.12). + +A full log is here: + +Duplicate since that will eventually be deleted: + +### What steps will reproduce the problem? +Attempt to build git-annex in a cabal sandbox using cabal install. + + +### What version of git-annex are you using? On what operating system? +git-annex-6.20170520 + +### Please provide any additional information below. + +The build error is + +[[!format sh """ +[ 8 of 559] Compiling Utility.QuickCheck ( Utility/QuickCheck.hs, dist/dist-sandbox-758c2984/build/git-annex/git-annex-tmp/Utility/QuickCheck.o ) + +Utility/QuickCheck.hs:38:10: error: + Duplicate instance declarations: + instance Arbitrary EpochTime + -- Defined at Utility/QuickCheck.hs:38:10 + instance [safe] Arbitrary Foreign.C.Types.CTime + -- Defined in ‘Test.QuickCheck.Arbitrary’ +cabal: Leaving directory '.' +cabal: Error: some packages failed to install: +git-annex-6.20170520 failed during the building phase. The exception was: +ExitFailure 1 +"""]] + +### Have you had any luck using git-annex before? (Sometimes we get tired of reading bug reports all day and a lil' positive end note does wonders) + +Yes, thanks! + +> Fixed in [[!commit 75cecbbe3fdafdb6652e95ac17cd755c28e67f20]] [[done]] +> --[[Joey]] diff --git a/doc/bugs/Build_failure_with_feed___62____61___1.0.0.0.mdwn b/doc/bugs/Build_failure_with_feed___62____61___1.0.0.0.mdwn new file mode 100644 index 0000000000..2b7bc53de2 --- /dev/null +++ b/doc/bugs/Build_failure_with_feed___62____61___1.0.0.0.mdwn @@ -0,0 +1,181 @@ +### Please describe the problem. +git-annex cannot build with feed 1.0.0.0, which was uploaded to Hackage on Sat Aug 26 23:56:18 UTC 2017 by AdamBergmark. + +### What steps will reproduce the problem? +Try to build git-annex with cabal install in a cabal sandbox. + +### What version of git-annex are you using? On what operating system? +6.20170818 + +### Please provide any additional information below. + +[[!format sh """ +[416 of 562] Compiling Command.ImportFeed ( Command/ImportFeed.hs, dist/dist-sandbox-b4eb11e/build/git-annex/git-annex-tmp/Command/ImportFeed.o ) + +Command/ImportFeed.hs:139:61: error: + • Couldn't match type ‘Data.Text.Internal.Text’ with ‘[Char]’ + Expected type: URLString + Actual type: Text.Atom.Feed.URI + • In the first argument of ‘Enclosure’, namely ‘enclosureurl’ + In the second argument of ‘($)’, namely ‘Enclosure enclosureurl’ + In the second argument of ‘($)’, namely + ‘ToDownload f u i $ Enclosure enclosureurl’ + +Command/ImportFeed.hs:142:49: error: + • Couldn't match type ‘Data.Text.Internal.Text’ with ‘[Char]’ + Expected type: URLString + Actual type: Data.Text.Internal.Text + • In the first argument of ‘quviSupported’, namely ‘link’ + In the first argument of ‘ifM’, namely ‘(quviSupported link)’ + In the expression: + ifM + (quviSupported link) + (return $ Just $ ToDownload f u i $ QuviLink link, return Nothing) + +Command/ImportFeed.hs:143:71: error: + • Couldn't match type ‘Data.Text.Internal.Text’ with ‘[Char]’ + Expected type: URLString + Actual type: Data.Text.Internal.Text + • In the first argument of ‘QuviLink’, namely ‘link’ + In the second argument of ‘($)’, namely ‘QuviLink link’ + In the second argument of ‘($)’, namely + ‘ToDownload f u i $ QuviLink link’ + +Command/ImportFeed.hs:214:54: error: + • Couldn't match type ‘[Char]’ with ‘Data.Text.Internal.Text’ + Expected type: S.Set Data.Text.Internal.Text + Actual type: S.Set ItemId + • In the second argument of ‘S.member’, namely ‘(knownitems cache)’ + In the expression: S.member itemid (knownitems cache) + In a case alternative: + Just (_, itemid) -> S.member itemid (knownitems cache) + +Command/ImportFeed.hs:279:42: error: + • Couldn't match type ‘Data.Text.Internal.Text’ with ‘[Char]’ + Expected type: Maybe [Char] + Actual type: Maybe Text.RSS.Syntax.DateString + • In the second argument of ‘(<$>)’, namely + ‘getItemPublishDateString itm’ + In the expression: replace "/" "-" <$> getItemPublishDateString itm + In a case alternative: + _ -> replace "/" "-" <$> getItemPublishDateString itm + +Command/ImportFeed.hs:293:44: error: + • Couldn't match type ‘Data.Text.Internal.Text’ with ‘[Char]’ + Expected type: String + Actual type: Data.Text.Internal.Text + • In the first argument of ‘toMetaValue’, namely ‘itemid’ + In the second argument of ‘($)’, namely ‘toMetaValue itemid’ + In the second argument of ‘M.singleton’, namely + ‘(S.singleton $ toMetaValue itemid)’ + +Command/ImportFeed.hs:299:26: error: + • Couldn't match type ‘Data.Text.Internal.Text’ with ‘[Char]’ + Expected type: Maybe String + Actual type: Maybe Data.Text.Internal.Text + • In the expression: feedtitle + In the expression: [feedtitle] + In the expression: ("feedtitle", [feedtitle]) + +Command/ImportFeed.hs:300:26: error: + • Couldn't match type ‘Data.Text.Internal.Text’ with ‘[Char]’ + Expected type: Maybe String + Actual type: Maybe Data.Text.Internal.Text + • In the expression: itemtitle + In the expression: [itemtitle] + In the expression: ("itemtitle", [itemtitle]) + +Command/ImportFeed.hs:301:27: error: + • Couldn't match type ‘Data.Text.Internal.Text’ with ‘[Char]’ + Expected type: Maybe String + Actual type: Maybe Data.Text.Internal.Text + • In the expression: feedauthor + In the expression: [feedauthor] + In the expression: ("feedauthor", [feedauthor]) + +Command/ImportFeed.hs:302:27: error: + • Couldn't match type ‘Data.Text.Internal.Text’ with ‘[Char]’ + Expected type: Maybe String + Actual type: Maybe Data.Text.Internal.Text + • In the expression: itemauthor + In the expression: [itemauthor] + In the expression: ("itemauthor", [itemauthor]) + +Command/ImportFeed.hs:303:28: error: + • Couldn't match type ‘Data.Text.Internal.Text’ with ‘[Char]’ + Expected type: Maybe String + Actual type: Maybe Data.Text.Internal.Text + • In the expression: getItemSummary $ item i + In the expression: [getItemSummary $ item i] + In the expression: ("itemsummary", [getItemSummary $ item i]) + +Command/ImportFeed.hs:304:32: error: + • Couldn't match type ‘Data.Text.Internal.Text’ with ‘[Char]’ + Expected type: Maybe String + Actual type: Maybe Data.Text.Internal.Text + • In the expression: getItemDescription $ item i + In the expression: [getItemDescription $ item i] + In the expression: + ("itemdescription", [getItemDescription $ item i]) + +Command/ImportFeed.hs:305:27: error: + • Couldn't match type ‘Data.Text.Internal.Text’ with ‘[Char]’ + Expected type: Maybe String + Actual type: Maybe Data.Text.Internal.Text + • In the expression: getItemRights $ item i + In the expression: [getItemRights $ item i] + In the expression: ("itemrights", [getItemRights $ item i]) + +Command/ImportFeed.hs:306:23: error: + • Couldn't match type ‘Data.Text.Internal.Text’ with ‘[Char]’ + Expected type: Maybe String + Actual type: Maybe Data.Text.Internal.Text + • In the expression: snd <$> getItemId (item i) + In the expression: [snd <$> getItemId (item i)] + In the expression: ("itemid", [snd <$> getItemId (item i)]) + +Command/ImportFeed.hs:307:22: error: + • Couldn't match type ‘Data.Text.Internal.Text’ with ‘[Char]’ + Expected type: Maybe String + Actual type: Maybe Data.Text.Internal.Text + • In the expression: itemtitle + In the expression: [itemtitle, feedtitle] + In the expression: ("title", [itemtitle, feedtitle]) + +Command/ImportFeed.hs:307:33: error: + • Couldn't match type ‘Data.Text.Internal.Text’ with ‘[Char]’ + Expected type: Maybe String + Actual type: Maybe Data.Text.Internal.Text + • In the expression: feedtitle + In the expression: [itemtitle, feedtitle] + In the expression: ("title", [itemtitle, feedtitle]) + +Command/ImportFeed.hs:308:23: error: + • Couldn't match type ‘Data.Text.Internal.Text’ with ‘[Char]’ + Expected type: Maybe String + Actual type: Maybe Data.Text.Internal.Text + • In the expression: itemauthor + In the expression: [itemauthor, feedauthor] + In the expression: ("author", [itemauthor, feedauthor]) + +Command/ImportFeed.hs:308:35: error: + • Couldn't match type ‘Data.Text.Internal.Text’ with ‘[Char]’ + Expected type: Maybe String + Actual type: Maybe Data.Text.Internal.Text + • In the expression: feedauthor + In the expression: [itemauthor, feedauthor] + In the expression: ("author", [itemauthor, feedauthor]) +cabal: Leaving directory '.' +cabal: Error: some packages failed to install: +git-annex-6.20170818-ATXJn9dQzZj9avYQidLOBq failed during the building phase. +The exception was: +ExitFailure 1 +"""]] + +Full build log is here: +https://gist.githubusercontent.com/anonymous/dcc8d9823bd50b7fca10d5cf8961e75d/raw/c6500526bbad0a94e067816b1af2c9e8717a3419/08.cabal + +### Have you had any luck using git-annex before? (Sometimes we get tired of reading bug reports all day and a lil' positive end note does wonders) +Yes! + +> I fixed this last week, in [[!commit ee2f096e3ba3aea24445ff2093b426b68e000cc2]]. [[done]] diff --git a/doc/bugs/Button_to_cancel_conistency_checks.mdwn b/doc/bugs/Button_to_cancel_conistency_checks.mdwn new file mode 100644 index 0000000000..2fd63dd2a8 --- /dev/null +++ b/doc/bugs/Button_to_cancel_conistency_checks.mdwn @@ -0,0 +1,21 @@ +### Please describe the problem. + +Sometimes a consistency check is undesired, e.g. when one wants to unmount the external drive. Add a button (along Configure) to the bubble that says Consistency check in progress to the assistent. + +### What steps will reproduce the problem? + +Run an automatic conistency check. + +### What version of git-annex are you using? On what operating system? + +20141024 + +### Please provide any additional information below. + +[[!format sh """ +# If you can, paste a complete transcript of the problem occurring here. +# If the problem is with the git-annex assistant, paste in .git/annex/daemon.log + + +# End of transcript or log. +"""]] diff --git a/doc/bugs/CHECKPRESENT_could_check_file_size_as_well.mdwn b/doc/bugs/CHECKPRESENT_could_check_file_size_as_well.mdwn new file mode 100644 index 0000000000..83c87c355d --- /dev/null +++ b/doc/bugs/CHECKPRESENT_could_check_file_size_as_well.mdwn @@ -0,0 +1,7 @@ +The [external special remote protocol](http://git-annex.branchable.com/design/external_special_remote_protocol/) CHECKPRESENT command has no support for checking the file size. + +In many cases a special remote is able to fetch the size of an object as it checks its availability with no additional cost (for example when you use the `stat()` function: the size is already included in the response). The remote protocol could be extended so that the backend returns the size with the CHECKPRESENT-SUCCESS command, so that git-annex can provide a primitive but probably useful additional integrity check. This is helpful, for example, to detect bugs that result in the truncation of a file in the special remote; or to detect the case of an aborted upload that resulted in the file being checked into the remote anyway. + +When it is not easy to get the file size without incurring in additional costs, the backend can simply return a flag value like `-1` to tell git-annex to ignore it. + +There is a delicate point in how git-annex computes the expected object size for an encrypted remote. I do not know if there is a way to compute the size of a file after it has been encrypted with HMAC/GPG without actually performing all the computation. In line of principle it should not be difficult, but I do not know the details. diff --git a/doc/bugs/CHECKPRESENT_could_check_file_size_as_well/comment_1_7bbba896da745a8a6b69ec62bf71fc89._comment b/doc/bugs/CHECKPRESENT_could_check_file_size_as_well/comment_1_7bbba896da745a8a6b69ec62bf71fc89._comment new file mode 100644 index 0000000000..e917ec9988 --- /dev/null +++ b/doc/bugs/CHECKPRESENT_could_check_file_size_as_well/comment_1_7bbba896da745a8a6b69ec62bf71fc89._comment @@ -0,0 +1,13 @@ +[[!comment format=mdwn + username="http://joeyh.name/" + subject="comment 1" + date="2015-01-06T17:41:45Z" + content=""" +The encrypted content size is not constant, and not known to git-annex. + +The only git-annex remote that checks the size in its checkpresent implementation is the web special remote, precisely because it's never encrypted. Also because files on the web change content from time to time and so that needs to be detected. + +What would make sense is to extend the reply to `CHECKPRESENT-SUCCESS Key [size]` or perhaps `CHECKPRESENT-SIZE Key size`. git-annex can then compare the value with the key's known size, if any. If the key is encrypted, it would need to skip this check. + +Note that chunk keys currently have their keySize inherited from the parent key, and the keyChunkSize of each chunk key is set to the key size. The last chunk of a key will typically be shorter than its keyChunkSize. That would need to be cleaned up. +"""]] diff --git a/doc/bugs/Can__39__t_add_a_git_repo_to_git_annex__58_____34__Invalid_path_repo__47__.git__47__X__34___for_many_X.mdwn b/doc/bugs/Can__39__t_add_a_git_repo_to_git_annex__58_____34__Invalid_path_repo__47__.git__47__X__34___for_many_X.mdwn new file mode 100644 index 0000000000..6c7444afd1 --- /dev/null +++ b/doc/bugs/Can__39__t_add_a_git_repo_to_git_annex__58_____34__Invalid_path_repo__47__.git__47__X__34___for_many_X.mdwn @@ -0,0 +1,37 @@ +I have some git repositories I don't edit often that I'd like to back up. I'd like to add these to my git annex, so I don't have to resort to a time-consuming hack (such as setting up a proper submodule, or bundling the repositories). + +But when I try to add a .git directory to git annex, I get a bunch of errors of the form + + git-annex: user error (xargs ["-0","git","--git-dir=/tmp/tmp.LhGN3nT9uM/annex/.git","--work-tree=/tmp/tmp.LhGN3nT9uM/annex","add","--"] exited 123) + failed + git-annex: add: 1 failed + add repo/.git/hooks/pre-push.sample ok + (Recording state in git...) + error: Invalid path 'repo/.git/hooks/pre-push.sample' + error: unable to add repo/.git/hooks/pre-push.sample to index + fatal: adding files failed + +STR: + + $ mkdir annex + $ cd annex + annex$ git init + annex$ git annex init + annex$ cd .. + + $ mkdir repo + $ cd repo + repo$ git init + repo$ cd .. + + $ mv repo annex + $ cd annex + annex$ find repo | xargs -n1 git annex add + # Lots of errors of the form above. + +You can't simply do `git annex add repo` because that will ignore the .git directory. Similarly,` git annex add .git` (which I'd think really should try to add the contents of the .git directory) ignores everything. + +I don't know what this error means. Is there a right way to work around this? + +> [[!meta title="cannot add .git/ to a git repository. even when using git-annex."]] +> [[!taglink confirmed]] (but may be out of scope for git-annex) --[[Joey]] diff --git a/doc/bugs/Can__39__t_add_a_git_repo_to_git_annex__58_____34__Invalid_path_repo__47__.git__47__X__34___for_many_X/comment_1_5d23b1f02c6a0db0b43939210c427994._comment b/doc/bugs/Can__39__t_add_a_git_repo_to_git_annex__58_____34__Invalid_path_repo__47__.git__47__X__34___for_many_X/comment_1_5d23b1f02c6a0db0b43939210c427994._comment new file mode 100644 index 0000000000..ff4b30122f --- /dev/null +++ b/doc/bugs/Can__39__t_add_a_git_repo_to_git_annex__58_____34__Invalid_path_repo__47__.git__47__X__34___for_many_X/comment_1_5d23b1f02c6a0db0b43939210c427994._comment @@ -0,0 +1,8 @@ +[[!comment format=mdwn + username="andy" + avatar="http://cdn.libravatar.org/avatar/1f1c630290d8b02d7faddc9fa4482c8d" + subject="comment 1" + date="2017-02-16T06:34:50Z" + content=""" +I've had the same problem; I solve it by packing the repositories into a compressed file of some sort (currently a 7z archive) and committing that to Git Annex. I have a script that makes that a one-line job. If I need to make changes to the repo, I can unpack the archive and work on the repo as usual. +"""]] diff --git a/doc/bugs/Can__39__t_add_remotes_through_the_web_assistant.mdwn b/doc/bugs/Can__39__t_add_remotes_through_the_web_assistant.mdwn new file mode 100644 index 0000000000..dd8b2a9978 --- /dev/null +++ b/doc/bugs/Can__39__t_add_remotes_through_the_web_assistant.mdwn @@ -0,0 +1,110 @@ +### Please describe the problem. +No matter what I do I can't + +### What steps will reproduce the problem? +Create a repository, try to add a remote. An external disk in this case. + +### What version of git-annex are you using? On what operating system? +Fedora 27 + + $ cat /etc/fedora-release + Fedora release 27 (Twenty Seven) + $ uname -a + Linux greger 4.14.14-300.fc27.x86_64 #1 SMP Fri Jan 19 13:19:54 UTC 2018 x86_64 x86_64 x86_64 GNU/Linux + + $ git annex version + git-annex version: 6.20180130-g39d97867a + build flags: Assistant Webapp Pairing S3(multipartupload)(storageclasses) WebDAV Inotify DBus DesktopNotify ConcurrentOutput TorrentParser Feeds Testsuite + dependency versions: aws-0.17.1 bloomfilter-2.0.1.0 cryptonite-0.23 DAV-1.3.1 feed-0.3.12.0 ghc-8.0.2 http-client-0.5.7.0 persistent-sqlite-2.6.2 torrent-10000.1.1 uuid-1.3.13 yesod-1.4.5 + key/value backends: SHA256E SHA256 SHA512E SHA512 SHA224E SHA224 SHA384E SHA384 SHA3_256E SHA3_256 SHA3_512E SHA3_512 SHA3_224E SHA3_224 SHA3_384E SHA3_384 SKEIN256E SKEIN256 SKEIN512E SKEIN512 SHA1E SHA1 MD5E MD5 WORM URL + remote types: git gcrypt p2p S3 bup directory rsync web bittorrent webdav tahoe glacier ddar hook external + local repository version: 5 + supported repository versions: 3 5 6 + upgrade supported from repository versions: 0 1 2 3 4 5 + operating system: linux x86_64 + + +### Please provide any additional information below. + +Debug log: + + [2018-01-31 14:46:15.670775214] main: starting assistant version 6.20180130-g39d97867a + [2018-01-31 14:46:15.776042349] Cronner: You should enable consistency checking to protect your data. + (scanning...) [2018-01-31 14:46:15.988042623] Watcher: Performing startup scan + (started...) + [2018-01-31 14:47:48.236476657] read: gpg ["--quiet","--trust-model","always","--with-colons","--list-secret-keys","--fixed-list-mode"] + [2018-01-31 14:47:48.262055169] process done ExitSuccess + [2018-01-31 14:47:51.695935824] read: git ["init","--quiet","--bare","/run/media/dxtr/archive/annex"] + [2018-01-31 14:47:51.700278024] process done ExitSuccess + [2018-01-31 14:47:51.700424216] read: git ["config","--null","--list"] + [2018-01-31 14:47:51.707427206] process done ExitSuccess + [2018-01-31 14:47:51.70868726] call: git ["--git-dir=../../../../run/media/dxtr/archive/annex","--literal-pathspecs","config","core.fsyncobjectfiles","true"] + [2018-01-31 14:47:51.725766269] process done ExitSuccess + [2018-01-31 14:47:51.728215647] read: git ["config","--null","--list"] + [2018-01-31 14:47:51.740757992] process done ExitSuccess + [2018-01-31 14:47:51.741434499] read: git ["config","--null","--list"] + [2018-01-31 14:47:51.753548799] process done ExitSuccess + [2018-01-31 14:47:51.755149461] read: git ["--git-dir=../../../../run/media/dxtr/archive/annex","--literal-pathspecs","show-ref","git-annex"] + [2018-01-31 14:47:51.771911199] process done ExitFailure 1 + [2018-01-31 14:47:51.772860937] read: git ["--git-dir=../../../../run/media/dxtr/archive/annex","--literal-pathspecs","show-ref","--hash","refs/heads/git-annex"] + [2018-01-31 14:47:51.789876829] process done ExitFailure 1 + [2018-01-31 14:47:51.790600261] call: git ["--git-dir=../../../../run/media/dxtr/archive/annex","--literal-pathspecs","show-ref","--verify","-q","origin/git-annex"] + [2018-01-31 14:47:51.79719711] process done ExitFailure 1 + [2018-01-31 14:47:51.799141741] read: git ["--git-dir=../../../../run/media/dxtr/archive/annex","--literal-pathspecs","write-tree"] + [2018-01-31 14:47:51.807820543] process done ExitSuccess + [2018-01-31 14:47:51.809126461] chat: git ["--git-dir=../../../../run/media/dxtr/archive/annex","--literal-pathspecs","commit-tree","4b825dc642cb6eb9a060e54bf8d69288fbee4904","--no-gpg-sign"] + [2018-01-31 14:47:51.836742843] process done ExitSuccess + [2018-01-31 14:47:51.837688371] call: git ["--git-dir=../../../../run/media/dxtr/archive/annex","--literal-pathspecs","update-ref","refs/heads/git-annex","c1bd62a595ce9cb811ad353ed0d9eaf8cfcb0b30"] + [2018-01-31 14:47:51.849598566] process done ExitSuccess + [2018-01-31 14:47:51.850762057] call: git ["--git-dir=../../../../run/media/dxtr/archive/annex","--literal-pathspecs","config","annex.uuid","da4c4bf3-5b3d-47f7-98d6-040cf8094360"] + [2018-01-31 14:47:51.855957091] process done ExitSuccess + [2018-01-31 14:47:51.85617243] read: git ["config","--null","--list"] + [2018-01-31 14:47:51.858450427] process done ExitSuccess + [2018-01-31 14:47:51.860771677] read: git ["--git-dir=../../../../run/media/dxtr/archive/annex","--literal-pathspecs","show-ref","git-annex"] + [2018-01-31 14:47:51.879636685] process done ExitSuccess + [2018-01-31 14:47:51.880463945] read: git ["--git-dir=../../../../run/media/dxtr/archive/annex","--literal-pathspecs","show-ref","--hash","refs/heads/git-annex"] + [2018-01-31 14:47:51.897305118] process done ExitSuccess + [2018-01-31 14:47:51.899214969] read: git ["--git-dir=../../../../run/media/dxtr/archive/annex","--literal-pathspecs","log","refs/heads/git-annex..c1bd62a595ce9cb811ad353ed0d9eaf8cfcb0b30","--pretty=%H","-n1"] + [2018-01-31 14:47:51.9105804] process done ExitSuccess + [2018-01-31 14:47:51.912107527] chat: git ["--git-dir=../../../../run/media/dxtr/archive/annex","--literal-pathspecs","cat-file","--batch"] + [2018-01-31 14:47:51.916244492] chat: git ["--git-dir=../../../../run/media/dxtr/archive/annex","--literal-pathspecs","cat-file","--batch-check=%(objectname) %(objecttype) %(objectsize)"] + [2018-01-31 14:47:51.926062822] call: git ["--git-dir=../../../../run/media/dxtr/archive/annex","--literal-pathspecs","config","annex.version","5"] + [2018-01-31 14:47:51.943412825] process done ExitSuccess + [2018-01-31 14:47:51.943804264] read: git ["config","--null","--list"] + [2018-01-31 14:47:51.949022318] process done ExitSuccess + [2018-01-31 14:47:51.952482534] chat: git ["--git-dir=../../../../run/media/dxtr/archive/annex","--literal-pathspecs","hash-object","-w","--stdin-paths","--no-filters"] + [2018-01-31 14:47:51.954175781] feed: git ["--git-dir=../../../../run/media/dxtr/archive/annex","--literal-pathspecs","update-index","-z","--index-info"] + [2018-01-31 14:47:51.968113275] process done ExitSuccess + [2018-01-31 14:47:51.969050491] read: git ["--git-dir=../../../../run/media/dxtr/archive/annex","--literal-pathspecs","show-ref","--hash","refs/heads/git-annex"] + [2018-01-31 14:47:51.979633335] process done ExitSuccess + (recording state in git...) + [2018-01-31 14:47:51.980277288] read: git ["--git-dir=../../../../run/media/dxtr/archive/annex","--literal-pathspecs","write-tree"] + [2018-01-31 14:47:51.989727059] process done ExitSuccess + [2018-01-31 14:47:51.98998106] chat: git ["--git-dir=../../../../run/media/dxtr/archive/annex","--literal-pathspecs","commit-tree","06548fa56acf18f9c1d74a29e248c4650df15e7e","--no-gpg-sign","-p","refs/heads/git-annex"] + [2018-01-31 14:47:52.011569157] process done ExitSuccess + [2018-01-31 14:47:52.012198525] call: git ["--git-dir=../../../../run/media/dxtr/archive/annex","--literal-pathspecs","update-ref","refs/heads/git-annex","26bcec3f33162a9c1c5b7a39c4828eedfdeae41b"] + [2018-01-31 14:47:52.030929599] process done ExitSuccess + [2018-01-31 14:47:52.037715315] process done ExitSuccess + [2018-01-31 14:47:52.038491764] process done ExitSuccess + [2018-01-31 14:47:52.039098682] process done ExitSuccess + [2018-01-31 14:47:52.039790346] read: uname ["-n"] + [2018-01-31 14:47:52.043271433] process done ExitSuccess + [2018-01-31 14:47:52.043523608] read: git ["config","--null","--list"] + [2018-01-31 14:47:52.057690629] process done ExitSuccess + [2018-01-31 14:47:52.058740927] call: git ["--git-dir=../../../../run/media/dxtr/archive/annex","--literal-pathspecs","remote","add","greger","../../../../../home/dxtr/temp/test"] + [2018-01-31 14:47:52.072184794] process done ExitSuccess + [2018-01-31 14:47:52.072509479] call: git ["--git-dir=.git","--work-tree=.","--literal-pathspecs","-c","core.bare=false","remote","add","archive","/run/media/dxtr/archive/annex"] + [2018-01-31 14:47:52.081339812] process done ExitSuccess + [2018-01-31 14:47:52.082154367] read: git ["config","--null","--list"] + [2018-01-31 14:47:52.093569488] process done ExitSuccess + 31/Jan/2018:14:47:52 +0100 [Error#yesod-core] there is no available git remote named "archive" @(yesod-core-1.4.37-GCI7RasEcSMIU2vku0fHJ1:Yesod.Core.Class.Yesod ./Yesod/Core/Class/Yesod.hs:693:5) + +The interesting part here is that if I try to run the failing git commands in the repository it works: + + $ git --git-dir=../../../../run/media/dxtr/archive/annex --literal-pathspecs show-ref git-annex + 26bcec3f33162a9c1c5b7a39c4828eedfdeae41b refs/heads/git-annex + + +### Have you had any luck using git-annex before? (Sometimes we get tired of reading bug reports all day and a lil' positive end note does wonders) +I have been using it for years. Just not the webapp :) + diff --git a/doc/bugs/Can__39__t_add_remotes_through_the_web_assistant/comment_1_a8a77df47eee0e15c25a89b68ec10e8f._comment b/doc/bugs/Can__39__t_add_remotes_through_the_web_assistant/comment_1_a8a77df47eee0e15c25a89b68ec10e8f._comment new file mode 100644 index 0000000000..05c2aeb8c7 --- /dev/null +++ b/doc/bugs/Can__39__t_add_remotes_through_the_web_assistant/comment_1_a8a77df47eee0e15c25a89b68ec10e8f._comment @@ -0,0 +1,35 @@ +[[!comment format=mdwn + username="olaf" + avatar="http://cdn.libravatar.org/avatar/4ae498d3d6ee558d6b65caa658f72572" + subject="I can reproduce" + date="2018-02-19T01:05:06Z" + content=""" +### Please describe the problem. + +Creating or adding remotes to an _existing_ repository via the webapp results in +> Internal Server Error +> +> there is no available git remote named \"XYZ\" + +**Creating a new repository** seems to create the repo and update the remotes (checked via `git` at command line) but does not update the repos in the webapp and results in the error: + + 19/Feb/2018:11:48:28 +1100 [Error#yesod-core] there is no available git remote named \"XYZ\" @(yesod-core-1.4.37.2-AqCgZCpSjdiDLzXFcWTxPQ:Yesod.Core.Class.Yesod ./Yesod/Core/Class/Yesod.hs:693:5) + + +**Adding an existing repo** to the current repo in the webapp results in the errors: (interesting as it first notes that the remote already exists and then complains that it's not available...) + + fatal: remote XYZ already exists. + 19/Feb/2018:11:52:24 +1100 [Error#yesod-core] there is no available git remote named \"XYZ\" @(yesod-core-1.4.37.2-AqCgZCpSjdiDLzXFcWTxPQ:Yesod.Core.Class.Yesod ./Yesod/Core/Class/Yesod.hs:693:5) + + +### Version + git-annex version: 6.20180112 + build flags: Assistant Webapp Pairing S3(multipartupload)(storageclasses) WebDAV FsEvents ConcurrentOutput TorrentParser MagicMime Feeds Testsuite + dependency versions: aws-0.18 bloomfilter-2.0.1.0 cryptonite-0.24 DAV-1.3.1 feed-1.0.0.0 ghc-8.2.2 http-client-0.5.7.1 persistent-sqlite-2.6.4 torrent-10000.1.1 uuid-1.2.6 yesod-1.4.5 + key/value backends: SHA256E SHA256 SHA512E SHA512 SHA224E SHA224 SHA384E SHA384 SHA3_256E SHA3_256 SHA3_512E SHA3_512 SHA3_224E SHA3_224 SHA3_384E SHA3_384 SKEIN256E SKEIN256 SKEIN512E SKEIN512 SHA1E SHA1 MD5E MD5 WORM URL + remote types: git gcrypt p2p S3 bup directory rsync web bittorrent webdav tahoe glacier ddar hook external + local repository version: 5 + supported repository versions: 3 5 6 + upgrade supported from repository versions: 0 1 2 3 4 5 + operating system: darwin x86_64 +"""]] diff --git a/doc/bugs/Can__39__t_add_remotes_through_the_web_assistant/comment_2_81cdf91b7720cb0a36bdff2815210435._comment b/doc/bugs/Can__39__t_add_remotes_through_the_web_assistant/comment_2_81cdf91b7720cb0a36bdff2815210435._comment new file mode 100644 index 0000000000..5dd975c70c --- /dev/null +++ b/doc/bugs/Can__39__t_add_remotes_through_the_web_assistant/comment_2_81cdf91b7720cb0a36bdff2815210435._comment @@ -0,0 +1,10 @@ +[[!comment format=mdwn + username="jhnichol@cce81d2a480707652a3340ea2f24b3dc4b1f808c" + nickname="jhnichol" + avatar="http://cdn.libravatar.org/avatar/2d05fd7e681bf4838bba8bab538ac65d" + subject="I also have this problem" + date="2018-04-05T15:25:22Z" + content=""" +I'm on OSX 10.11.6. I started with a homebrew install, and today I found the native .dmg app. Could there be a conflict between the homebrew install and the .app? +I'm a first time user trying to test this program and not getting very far. I have a minimal understanding of git, and therefore shy away from the CLI. +"""]] diff --git a/doc/bugs/Can__39__t_add_remotes_through_the_web_assistant/comment_3_c928d9f0ed85f61270922471c8037643._comment b/doc/bugs/Can__39__t_add_remotes_through_the_web_assistant/comment_3_c928d9f0ed85f61270922471c8037643._comment new file mode 100644 index 0000000000..8e640e561b --- /dev/null +++ b/doc/bugs/Can__39__t_add_remotes_through_the_web_assistant/comment_3_c928d9f0ed85f61270922471c8037643._comment @@ -0,0 +1,206 @@ +[[!comment format=mdwn + username="andrew" + avatar="http://cdn.libravatar.org/avatar/acc0ece1eedf07dd9631e7d7d343c435" + subject="comment 3" + date="2018-05-04T15:00:18Z" + content=""" +I am getting the same error messages as **olaf**. It does not seem to be related to brew vs dmg **jhnichol**. + +Using the web-gui I can create one local repo. When I try to create a second local repo, by clicking `Add another repository` | `Add another repository` | `Make Repository` | `Combine the repositories` the webapp then fails with this error message: + + Internal Server Error + there is no available git remote named \"a2\" + git-annex version 6.20180409-gfb0780266 + +Looking at the .git/config files for the repos, the names seem strange/incorrect. The second repo has the remote name listed as my hostname, “bumblebee.local”? Not sure if this is an issue. + +Also from a git remote perspective I am a bit confused what I just did? Did I create two completely separate git repos then try to combine them, what does that mean? Shouldn't I have cloned the first repo instead of creating a new one? + +When I created two repos on the commandline I didn't run into any issues, but, what I did was create a repo, then clone it into another directory, then added the new repo as a remote of the 1st repo. It seems the combine button in the webapp isn't doing that. Perhaps combine should only be enabled for special remotes, is that the problem? + + +—Andrew + + + + + +========== full log below, first steps to make the error with webapp, followed by success on the commandline ============= +
    +git-annex webapp
+
+make repository:
+/Users/Shared/a1
+
+add a file:
+image1.png
+
+assistant detects and adds to git-annex
+
+click add another repository
+/Users/Shared/a2
+
+assistant creates the directory
+runs git init and git-annex init
+
+click:
+Combine repositories?
+The repository at /Users/Shared/a2 is set up. 
+Do you want to combine it with your existing repository at /Users/Shared/a1?
+
+ The combined repositories will sync and share their files.
+ 
+
+Internal Server Error
+there is no available git remote named \"a2\"
+git-annex version 6.20180409-gfb0780266
+
+At this point /Users/Shared/a2/.git/config has 
+[core]
+	repositoryformatversion = 0
+	filemode = true
+	bare = true
+	logallrefupdates = true
+	ignorecase = true
+	precomposeunicode = true
+[annex]
+	uuid = 8ee0b541-c429-45c5-aaa2-c4799843f809
+	version = 5
+	direct = true
+[gc]
+	auto = 0
+[remote \"bumblebee.local\"]
+	url = ../a1
+	fetch = +refs/heads/*:refs/remotes/bumblebee.local/*
+		
+And /Users/Shared/a1/.git/config has 
+[core]
+	repositoryformatversion = 0
+	filemode = true
+	bare = true
+	logallrefupdates = true
+	ignorecase = true
+	precomposeunicode = true
+[annex]
+	uuid = 46290480-c956-42ac-9ada-36b2281dbec2
+	version = 5
+	direct = true
+[gc]
+	auto = 0
+[remote \"a2\"]
+	url = /Users/Shared/a2
+	fetch = +refs/heads/*:refs/remotes/a2/*
+	
+
+Perhaps 
+
+
+
+
+
+
+
+OK.
+
+
+Now do this from the commandline:
+
+andrew@bumblebee /Users/Shared$ mkdir a1
+andrew@bumblebee /Users/Shared$ cd a1
+andrew@bumblebee /Users/Shared/a1$ git init
+Initialized empty Git repository in /Users/Shared/a1/.git/
+andrew@bumblebee /Users/Shared/a1$ git annex init
+init  ok
+(recording state in git...)
+
+andrew@bumblebee /Users/Shared/a1$ git annex describe here a1
+describe here ok
+(recording state in git...)
+
+andrew@bumblebee /Users/Shared/a1$ git annex direct
+commit 
+On branch master
+
+Initial commit
+
+nothing to commit
+ok
+direct ok
+andrew@bumblebee /Users/Shared/a1$ cp ~/Desktop/Screen\ Shot\ 2018-05-04\ at\ 10.10.55\ AM.png image1.png
+andrew@bumblebee /Users/Shared/a1$ git add image1.png 
+fatal: This operation must be run in a work tree
+andrew@bumblebee /Users/Shared/a1$ ls
+image1.png
+andrew@bumblebee /Users/Shared/a1$ git annex add image1.png 
+add image1.png ok
+(recording state in git...)
+andrew@bumblebee /Users/Shared/a1$ 
+
+add /Users/Shared/a1 to ~/.config/git-annex/autostart
+
+then:
+
+andrew@bumblebee /Users/Shared/a1$ git-annex assistant --autostart 
+git-annex autostart in /Users/Shared/a1
+ok
+andrew@bumblebee /Users/Shared/a1$ 
+
+launch webapp, see that a1 is listed, add a file, see that webapp detects it
+
+Clone repo1 to another directory
+andrew@bumblebee ~/.config/git-annex$ cd
+andrew@bumblebee ~$ cd /Users/Shared/
+andrew@bumblebee /Users/Shared$ git clone a1 a2
+Cloning into 'a2'...
+done.
+andrew@bumblebee /Users/Shared$ cd a2
+andrew@bumblebee /Users/Shared/a2$ git annex init \"a2\"
+init a2 (merging origin/git-annex into git-annex...)
+(recording state in git...)
+ok
+(recording state in git...)
+andrew@bumblebee /Users/Shared/a2$ 
+
+Add a2 as a remote from a1
+andrew@bumblebee /Users/Shared/a1$ git remote add a2 /Users/Shared/a2
+
+Add a2 to autostart file
+
+Now repos are syncing
+
+Both repos are listed in web-gui when I choose a2 as the repo, only one repo is listed in the web-gui when I choose a1 as the repo.
+
+andrew@bumblebee /Users/Shared/a1$ cat .git/config 
+[core]
+	repositoryformatversion = 0
+	filemode = true
+	bare = true
+	logallrefupdates = true
+	ignorecase = true
+	precomposeunicode = true
+[annex]
+	uuid = f65fd001-b6ea-4a86-9256-2387c830f933
+	version = 5
+	direct = true
+[remote \"a2\"]
+	url = /Users/Shared/a2
+	fetch = +refs/heads/*:refs/remotes/a2/*
+	annex-uuid = f9e9f00a-ae70-4c2d-bac3-fe8d6d05a4ea
+	
+andrew@bumblebee /Users/Shared/a2$ cat .git/config 
+	[core]
+		repositoryformatversion = 0
+		filemode = true
+		bare = false
+		logallrefupdates = true
+		ignorecase = true
+		precomposeunicode = true
+	[remote \"origin\"]
+		url = /Users/Shared/a1
+		fetch = +refs/heads/*:refs/remotes/origin/*
+		annex-uuid = f65fd001-b6ea-4a86-9256-2387c830f933
+	[annex]
+		uuid = f9e9f00a-ae70-4c2d-bac3-fe8d6d05a4ea
+		version = 5
+
    +"""]] diff --git a/doc/bugs/Can__39__t_add_remotes_through_the_web_assistant/comment_4_52df3e78719c529987621b212588b785._comment b/doc/bugs/Can__39__t_add_remotes_through_the_web_assistant/comment_4_52df3e78719c529987621b212588b785._comment new file mode 100644 index 0000000000..aa63dc834b --- /dev/null +++ b/doc/bugs/Can__39__t_add_remotes_through_the_web_assistant/comment_4_52df3e78719c529987621b212588b785._comment @@ -0,0 +1,13 @@ +[[!comment format=mdwn + username="AlexP" + avatar="http://cdn.libravatar.org/avatar/a5756f1e491fa69cba8c2338ce459ed8" + subject="any update on this?" + date="2018-08-02T20:01:57Z" + content=""" +does anyone have a solution to this? I can't seem to add remotes through the webapp. I consistantly get the following error: + +Internal Server Error +there is no available git remote named \"blablabla\" + +thanks +"""]] diff --git a/doc/bugs/Can__39__t_add_remotes_through_the_web_assistant/comment_5_0159868320e98ed614cefa4bb73f9ab0._comment b/doc/bugs/Can__39__t_add_remotes_through_the_web_assistant/comment_5_0159868320e98ed614cefa4bb73f9ab0._comment new file mode 100644 index 0000000000..283266843b --- /dev/null +++ b/doc/bugs/Can__39__t_add_remotes_through_the_web_assistant/comment_5_0159868320e98ed614cefa4bb73f9ab0._comment @@ -0,0 +1,8 @@ +[[!comment format=mdwn + username="torpidus" + avatar="http://cdn.libravatar.org/avatar/997fb77747b008a26383426ae6561368" + subject="comment 5" + date="2018-09-16T18:40:43Z" + content=""" + This affects me also. git-annex in version of August 8th, 2018. +"""]] diff --git a/doc/bugs/Can__39__t_add_remotes_through_the_web_assistant/comment_6_bb6c572eb9b9d5d435bdb6eeb78a11f9._comment b/doc/bugs/Can__39__t_add_remotes_through_the_web_assistant/comment_6_bb6c572eb9b9d5d435bdb6eeb78a11f9._comment new file mode 100644 index 0000000000..754e875a5f --- /dev/null +++ b/doc/bugs/Can__39__t_add_remotes_through_the_web_assistant/comment_6_bb6c572eb9b9d5d435bdb6eeb78a11f9._comment @@ -0,0 +1,37 @@ +[[!comment format=mdwn + username="stefankangas@2fc5a0baba35168e9f9b5b72d5204558fd964c32" + nickname="stefankangas" + avatar="http://cdn.libravatar.org/avatar/5bef54a6599375e6bfe9da041e04b588" + subject="Same issue on OSX / git-annex 6.20180807" + date="2018-10-01T12:49:27Z" + content=""" +I'm seeing the same issue on OSX when adding a \"Remote server\", using git-annex version 6.20180807. This is on a completely new repository that I've created in \"~/annex\", with no files in it. + +After clicking \"Combine repositories\", I get the following error: + + Internal Server Error + there is no available git remote named \"mydomain.example.com_annex\" + +Where mydomain.example.com is the address to my ssh server. + +The logs say: + + [2018-10-01 14:39:44.352157] main: starting assistant version 6.20180807 + [2018-10-01 14:39:44.416265] Cronner: You should enable consistency checking to protect your data. + (scanning...) [2018-10-01 14:39:44.670591] Watcher: Performing startup scan + (started...) + 01/Oct/2018:14:40:13 +0200 [Error#yesod-core] there is no available git remote named \"mydomain.example.com_annex\" @(yesod-core-1.4.37.3-2QxllzSqvdqJbUCjADwK0K:Yesod.Core.Class.Yesod ./Yesod/Core/Class/Yesod.hs:693:5) + +It does not matter if the remote repository has been created or not - I still see the same error. Also, my local \"~/annex\" git repository has a remote named \"mydomain.example.com_annex\": + + # git remote show mydomain.example.com_annex + mesg: ttyname failed: Inappropriate ioctl for device + * remote mydomain.example.com_annex + Fetch URL: ssh://skangas@mydomain.example.com/~/annex/ + Push URL: ssh://skangas@mydomain.example.com/~/annex/ + HEAD branch: (unknown) + Remote branch: + git-annex new (next fetch will store in remotes/mydomain.example.com_annex) + Local ref configured for 'git push': + git-annex pushes to git-annex (local out of date) +"""]] diff --git a/doc/bugs/Can__39__t_compile_on_FreeBSD__58____Issues_with_System.Posix.Files.mdwn b/doc/bugs/Can__39__t_compile_on_FreeBSD__58____Issues_with_System.Posix.Files.mdwn new file mode 100644 index 0000000000..79880b65e4 --- /dev/null +++ b/doc/bugs/Can__39__t_compile_on_FreeBSD__58____Issues_with_System.Posix.Files.mdwn @@ -0,0 +1,55 @@ +### Please describe the problem. + +git-annex can't compile on FreeBSD; specifically, the build fails due to issues with System.Posix.Files. + +### What steps will reproduce the problem? + +1. git clone git://git-annex.branchable.com/ git-annex +2. cd git-annex +3. stack setup +4. stack install + +### What version of git-annex are you using? On what operating system? + +git-annex HEAD. + +FreeBSD 11.1-RELEASE r321309 GENERIC amd64 + +### Please provide any additional information below. + +Let me know if you'd like me to set up a FreeBSD development environment for you to SSH into - happy to do that if it helps you in any way. + +Compilation failure is as follows: + + Configuring git-annex-6.20180807... + git-annex-6.20180807: build (exe) + Preprocessing executable 'git-annex' for git-annex-6.20180807... + [124 of 586] Compiling Utility.DirWatcher.Kqueue ( Utility/DirWatcher/Kqueue.hs, .stack-work/dist/x86_64-freebsd/Cabal-1.24.2.0/build/git-annex/git-annex-tmp/Utility/Di + rWatcher/Kqueue.o ) + + /usr/home/duncan/code/git-annex/Utility/DirWatcher/Kqueue.hs:112:49: error: + Not in scope: ‘Files.openFd’ + Module ‘System.Posix.Files’ does not export ‘openFd’. + + /usr/home/duncan/code/git-annex/Utility/DirWatcher/Kqueue.hs:112:66: error: + Not in scope: data constructor ‘Files.ReadOnly’ + Module ‘System.Posix.Files’ does not export ‘ReadOnly’. + + /usr/home/duncan/code/git-annex/Utility/DirWatcher/Kqueue.hs:112:89: error: + Not in scope: ‘Files.defaultFileFlags’ + Module ‘System.Posix.Files’ does not export ‘defaultFileFlags’. + + /usr/home/duncan/code/git-annex/Utility/DirWatcher/Kqueue.hs:132:15: error: + Not in scope: ‘Files.closeFd’ + Module ‘System.Posix.Files’ does not export ‘closeFd’. + + /usr/home/duncan/code/git-annex/Utility/DirWatcher/Kqueue.hs:170:14: error: + Not in scope: ‘Files.closeFd’ + Module ‘System.Posix.Files’ does not export ‘closeFd’. + + -- While building custom Setup.hs for package git-annex-6.20180807 using: + /usr/home/duncan/code/git-annex/.stack-work/dist/x86_64-freebsd/Cabal-1.24.2.0/setup/setup --builddir=.stack-work/dist/x86_64-freebsd/Cabal-1.24.2.0 build exe:git + -annex --ghc-options " -ddump-hi -ddump-to-file" + Process exited with code: ExitFailure 1 + +> [[done]] --[[Joey]] diff --git a/doc/bugs/Can__39__t_compile_on_FreeBSD__58____Issues_with_System.Posix.Files/comment_1_c8d41357ea9d0ac4601e066982058545._comment b/doc/bugs/Can__39__t_compile_on_FreeBSD__58____Issues_with_System.Posix.Files/comment_1_c8d41357ea9d0ac4601e066982058545._comment new file mode 100644 index 0000000000..b10cc24762 --- /dev/null +++ b/doc/bugs/Can__39__t_compile_on_FreeBSD__58____Issues_with_System.Posix.Files/comment_1_c8d41357ea9d0ac4601e066982058545._comment @@ -0,0 +1,7 @@ +[[!comment format=mdwn + username="joey" + subject="""comment 1""" + date="2018-09-24T15:24:30Z" + content=""" +Seems it also needs to import System.Posix.IO, which I've now committed. +"""]] diff --git a/doc/bugs/Can__39__t_use_adjusted_branch_in_v6__44___if_submodule_already_is_using_this_feature.mdwn b/doc/bugs/Can__39__t_use_adjusted_branch_in_v6__44___if_submodule_already_is_using_this_feature.mdwn new file mode 100644 index 0000000000..d5b8e27fad --- /dev/null +++ b/doc/bugs/Can__39__t_use_adjusted_branch_in_v6__44___if_submodule_already_is_using_this_feature.mdwn @@ -0,0 +1,57 @@ +### Please describe the problem. +If a v6 submodule is using an adjusted branch, `git annex adjust --unlock` fails in its superproject. + +### What steps will reproduce the problem? +[[!format sh """ +#!/bin/bash + +set -ex +d=$(mktemp -d) +echo "directory: $d" +cd $d +git init +git annex init --version=6 +echo whatever > file +git annex add file +git commit -m "file added" +mkdir sub +cd sub +git init +git annex init --version=6 +echo something > subfile +git annex add subfile +git commit -m "subfile added." +cd .. +git submodule add ./sub sub +git commit -m "submodule added" +cd sub +git annex adjust --unlock +cd .. +git annex adjust --unlock + +"""]] +### What version of git-annex are you using? On what operating system? +[[!format sh """ +% git annex version +git-annex version: 6.20170209+gitg16be7b5cc-1~ndall+1 +build flags: Assistant Webapp Pairing Testsuite S3(multipartupload)(storageclasses) WebDAV Inotify DBus DesktopNotify ConcurrentOutput TorrentParser MagicMime Feeds Quvi +key/value backends: SHA256E SHA256 SHA512E SHA512 SHA224E SHA224 SHA384E SHA384 SHA3_256E SHA3_256 SHA3_512E SHA3_512 SHA3_224E SHA3_224 SHA3_384E SHA3_384 SKEIN256E SKEIN256 SKEIN512E SKEIN512 SHA1E SHA1 MD5E MD5 WORM URL +remote types: git gcrypt p2p S3 bup directory rsync web bittorrent webdav tahoe glacier ddar hook external +local repository version: unknown +supported repository versions: 3 5 6 +upgrade supported from repository versions: 0 1 2 3 4 5 +operating system: linux x86_64 +"""]] + +### Please provide any additional information below. + +[[!format sh """ +# If you can, paste a complete transcript of the problem occurring here. +# If the problem is with the git-annex assistant, paste in .git/annex/daemon.log +# End of transcript or log. +"""]] + +### Have you had any luck using git-annex before? (Sometimes we get tired of reading bug reports all day and a lil' positive end note does wonders) + + +> [[fixed|done]] --[[Joey]] diff --git a/doc/bugs/Can__39__t_use_adjusted_branch_in_v6__44___if_submodule_already_is_using_this_feature/comment_1_176faba802ecd62b04c06c6dab77f5f3._comment b/doc/bugs/Can__39__t_use_adjusted_branch_in_v6__44___if_submodule_already_is_using_this_feature/comment_1_176faba802ecd62b04c06c6dab77f5f3._comment new file mode 100644 index 0000000000..76e0c78143 --- /dev/null +++ b/doc/bugs/Can__39__t_use_adjusted_branch_in_v6__44___if_submodule_already_is_using_this_feature/comment_1_176faba802ecd62b04c06c6dab77f5f3._comment @@ -0,0 +1,20 @@ +[[!comment format=mdwn + username="joey" + subject="""comment 1""" + date="2017-02-20T17:13:54Z" + content=""" +Error message is: + + git-annex: unexpected object type "comm" + +What it's actually choking on is the "commit" object for the submodule, +in git-ls-tree output. Doesn't matter if the submodule uses +adjusted branches or not. + +The parser for ls-tree output is buggy; +it's expecting only "blob" and "tree", so pulls out a fixed width 4 +characters: "comm" + +Also, the adjusted branch code needs to be made to skip over CommitObjects, +once the parser is fixed to generate them. +"""]] diff --git a/doc/bugs/Cannot_add_gcrypt_remote_in_webapp__58___gcrypt_backend.mdwn b/doc/bugs/Cannot_add_gcrypt_remote_in_webapp__58___gcrypt_backend.mdwn new file mode 100644 index 0000000000..28c30bbd23 --- /dev/null +++ b/doc/bugs/Cannot_add_gcrypt_remote_in_webapp__58___gcrypt_backend.mdwn @@ -0,0 +1,15 @@ +### Please describe the problem. + +I have a repository that is encrypted using GPG. It is supposed to sync a folder between two laptops. (Note: this is a DIFFERENT repository than the one in my other bug report https://git-annex.branchable.com/bugs/Packfile_does_not_match_digest__58___gcrypt_with_assistant/ but the setup is the same) + +I set everything up on laptop1. This time, I decided to wait for the syncing to finish, between laptop1 and laptop2 in order to prevent corruption. After laptop1 had finished syncing, I went to laptop2 and setup the repository. + +- I created a local repository on laptop2. +- I told the webapp to combine repositories. + +Then I got + +" +Internal Server Error +Cannot find configuration for the gcrypt remote at ssh://denisa@vps.ip/backup/annex/denisa/Studium +" diff --git a/doc/bugs/Cannot_build_with_GHC_8.2.1.mdwn b/doc/bugs/Cannot_build_with_GHC_8.2.1.mdwn new file mode 100644 index 0000000000..4fa9e18ab9 --- /dev/null +++ b/doc/bugs/Cannot_build_with_GHC_8.2.1.mdwn @@ -0,0 +1,51 @@ +### Please describe the problem. +git-annex cannot build with GHC 8.2.1 + +### What steps will reproduce the problem? +cabal install --jobs=8 --max-backjumps=100000 --only-dependencies --flags=s3\ webapp + +### What version of git-annex are you using? On what operating system? +Tested with 6.20170520 and HEAD at a983877279d5157d4e7a8c2397d9e541e3c41fa6 + +### Please provide any additional information below. +Discovered during https://github.com/Homebrew/homebrew-core/pull/15934 + +[[!format sh """ +==> cabal install --jobs=8 --max-backjumps=100000 --only-dependencies --flags=s3 webapp +clang: warning: -Wl,-headerpad_max_install_names: 'linker' input unused +clang: warning: argument unused during compilation: '-L/usr/local/opt/gettext/lib' +clang: warning: argument unused during compilation: '-L/usr/local/opt/libffi/lib' +clang: warning: argument unused during compilation: '-L/usr/local/opt/readline/lib' +clang: warning: argument unused during compilation: '-L/usr/local/opt/sqlite/lib' +clang: warning: argument unused during compilation: '-L/usr/local/opt/openssl/lib' +clang: warning: argument unused during compilation: '-L/usr/local/opt/icu4c/lib' +clang: warning: argument unused during compilation: '-L/usr/local/lib' +clang: warning: argument unused during compilation: '-L/System/Library/Frameworks/OpenGL.framework/Versions/Current/Libraries' +Resolving dependencies... +cabal: Could not resolve dependencies: +trying: git-annex-6.20170520 (user goal) +trying: base-4.10.0.0/installed-4.1... (dependency of git-annex-6.20170520) +next goal: sandi (dependency of git-annex-6.20170520) +rejecting: sandi-0.4.0 (conflict: base==4.10.0.0/installed-4.1..., sandi => +base>=4.7 && <4.10) +rejecting: sandi-0.3.6, sandi-0.3.5 (conflict: +base==4.10.0.0/installed-4.1..., sandi => base>=4.7 && <4.9) +rejecting: sandi-0.3.4 (conflict: base==4.10.0.0/installed-4.1..., sandi => +base==4.8.*) +rejecting: sandi-0.3.3, sandi-0.3.2, sandi-0.3.1 (conflict: +base==4.10.0.0/installed-4.1..., sandi => base==4.7.*) +rejecting: sandi-0.3.0.1 (conflict: base==4.10.0.0/installed-4.1..., sandi => +base==4.6.* || ==4.7.*) +rejecting: sandi-0.3.0 (conflict: base==4.10.0.0/installed-4.1..., sandi => +base==4.7.*) +rejecting: sandi-0.2.3, sandi-0.2.2.1, sandi-0.2.2, sandi-0.2.1 (conflict: +base==4.10.0.0/installed-4.1..., sandi => base>=4.5 && <4.7) +rejecting: sandi-0.2, sandi-0.1.1, sandi-0.1 (conflict: +base==4.10.0.0/installed-4.1..., sandi => base==4.6.*) +Dependency tree exhaustively searched. +"""]] + +### Have you had any luck using git-annex before? (Sometimes we get tired of reading bug reports all day and a lil' positive end note does wonders) +Yes :) + +> [[fixed|done]] --[[Joey]] diff --git a/doc/bugs/Cannot_build_with_GHC_8.2.1/comment_1_ffd824d79ec303d35b83255f0417d23a._comment b/doc/bugs/Cannot_build_with_GHC_8.2.1/comment_1_ffd824d79ec303d35b83255f0417d23a._comment new file mode 100644 index 0000000000..a13fb0faab --- /dev/null +++ b/doc/bugs/Cannot_build_with_GHC_8.2.1/comment_1_ffd824d79ec303d35b83255f0417d23a._comment @@ -0,0 +1,10 @@ +[[!comment format=mdwn + username="joey" + subject="""comment 1""" + date="2017-07-28T13:05:38Z" + content=""" +I don't see any git-annex failure to build here. Cabal is failing to +install sandi, a dependency. That package needs to have its dependency on +base updated to allow building with the new ghc. Quite likely simply +adjusting the dependency version will work. +"""]] diff --git a/doc/bugs/Cannot_build_with_GHC_8.2.1/comment_2_0796d1dfcf7116c38451febb9701c857._comment b/doc/bugs/Cannot_build_with_GHC_8.2.1/comment_2_0796d1dfcf7116c38451febb9701c857._comment new file mode 100644 index 0000000000..cf83cd991d --- /dev/null +++ b/doc/bugs/Cannot_build_with_GHC_8.2.1/comment_2_0796d1dfcf7116c38451febb9701c857._comment @@ -0,0 +1,67 @@ +[[!comment format=mdwn + username="ilovezfs" + avatar="http://cdn.libravatar.org/avatar/f2b3954cf2ed0a551de9a49d3b6a64d0" + subject="pipe: resource exhausted (Too many open files)" + date="2017-07-30T16:05:10Z" + content=""" +Hi Joey, + +Yes, I can work around that particular failure by tweaking constraints. + +In particular, +--allow-newer=\"sandi:base\" --allow-newer=\"aws:time\" + +I also first need to \"cabal install Cabal<1.25\" because git-annex seems +to have a compatibility problem with Cabal-2.0.0.2. The error with +Cabal-2.0.0.2 is + +[[!format sh \"\"\" +Building executable 'git-annex' for git-annex-6.20170520.. + +Build/BuildVersion.hs:1:1: error: + File name does not match module name: + Saw: ‘Main’ + Expected: ‘Build.BuildVersion’ + | +1 | {- Outputs the version of git-annex that was built, for use by + | ^ +cabal: Leaving directory '.' +cabal: Error: some packages failed to install: +git-annex-6.20170520-3mbb3kAWTlXI0EMervIo66 failed during the building phase. +The exception was: +ExitFailure 1 +\"\"\"]] + +But, once I've installed the lower version of Cabal, and tweaked the +constraints for sandi and for aws, git-annex does build successfully. + +However, at that point a more serious issue comes up when I run \"git +annex test\" and here is the full log: +https://gist.github.com/ilovezfs/d430f589d87d7f7237591cd51b2b94cb + +Here is a snippet: +[[!format sh \"\"\" +From ../../.t/tmprepo53 + * [new branch] git-annex -> r2/git-annex + * [new branch] master -> r2/master + * [new branch] synced/master -> r2/synced/master +conflictor/subfile +conflictor.variant-cc12 +conflictor/subfile +conflictor.variant-cc12 +OK (3.27s) + conflict resolution symlink bit: OK + conflict resolution (uncommitted local file): FAIL (0.49s) + sync failed in r1 + conflict resolution (removed file): FAIL + Exception: git: createProcess: runInteractiveProcess: pipe: resource exhausted (Too many open files) + conflict resolution (nonannexed file): FAIL + Exception: git: createProcess: runInteractiveProcess: pipe: resource exhausted (Too many open files) + conflict resolution (nonannexed symlink): FAIL (0.03s) + git annex init failed + conflict resolution (mixed locked and unlocked file): FAIL + Exception: git: createProcess: runInteractiveProcess: pipe: resource exhausted (Too many open files) + map: FAIL + Exception: git: createProcess: runInteractiveProcess: pipe: resource exhausted (Too many open files) +\"\"\"]] +"""]] diff --git a/doc/bugs/Cannot_build_with_GHC_8.2.1/comment_3_c43f26b71cd7335951fb71f68b84ed10._comment b/doc/bugs/Cannot_build_with_GHC_8.2.1/comment_3_c43f26b71cd7335951fb71f68b84ed10._comment new file mode 100644 index 0000000000..6aada30f91 --- /dev/null +++ b/doc/bugs/Cannot_build_with_GHC_8.2.1/comment_3_c43f26b71cd7335951fb71f68b84ed10._comment @@ -0,0 +1,9 @@ +[[!comment format=mdwn + username="joey" + subject="""comment 3""" + date="2017-08-01T16:18:18Z" + content=""" +The cabal bug is , +and it seems like it would require *extensive* changes to git-annex's +source code to work around this cabal behavior. +"""]] diff --git a/doc/bugs/Cannot_build_with_GHC_8.2.1/comment_4_f508b6db0513edcdc8600df405ce1913._comment b/doc/bugs/Cannot_build_with_GHC_8.2.1/comment_4_f508b6db0513edcdc8600df405ce1913._comment new file mode 100644 index 0000000000..7e277efd69 --- /dev/null +++ b/doc/bugs/Cannot_build_with_GHC_8.2.1/comment_4_f508b6db0513edcdc8600df405ce1913._comment @@ -0,0 +1,41 @@ +[[!comment format=mdwn + username="https://launchpad.net/~felixonmars" + nickname="felixonmars" + avatar="http://cdn.libravatar.org/avatar/17284a3bb2e4ad9d3be8fab31f49865be9c1dc22143c728de731fe800a335d38" + subject="comment 4" + date="2017-08-17T19:10:24Z" + content=""" +I managed to build git-annex with GHC 8.2.1 on Linux by removing a bunch of Win32/OSX modules from the cabal file and another small fix. However, there are two test failures when running git-annex test. Do you think it's related to the GHC version? + + Tests + QuickCheck + prop_isomorphic_deencode_git: FAIL + *** Failed! Falsifiable (after 6 tests and 1 shrink): + \"\1086483\" + Use --quickcheck-replay=615269 to reproduce. + prop_isomorphic_deencode: FAIL + *** Failed! Falsifiable (after 3 tests and 1 shrink): + \"\185545\" + Use --quickcheck-replay=915399 to reproduce. + ... + 2 out of 293 tests failed (486.80s) + (This could be due to a bug in git-annex, or an incompatibility + with utilities, such as git, installed on this system.) + + +All other tests are working, though. + +Versions: + + $ git --version + git version 2.14.1 + + $ git-annex version + git-annex version: 6.20170612-ge4100fd60 + build flags: Assistant Webapp Pairing Testsuite S3(multipartupload)(storageclasses) WebDAV Inotify DBus DesktopNotify ConcurrentOutput TorrentParser MagicMime Feeds Quvi + dependency versions: aws-0.16 bloomfilter-2.0.1.0 cryptonite-0.24 DAV-1.3.1 feed-0.3.12.0 ghc-8.2.1 http-client-0.5.7.0 persistent-sqlite-2.6.2 torrent-10000.1.1 uuid-1.3.13 yesod-1.4.5 + key/value backends: SHA256E SHA256 SHA512E SHA512 SHA224E SHA224 SHA384E SHA384 SHA3_256E SHA3_256 SHA3_512E SHA3_512 SHA3_224E SHA3_224 SHA3_384E SHA3_384 SKEIN256E SKEIN256 SKEIN512E SKEIN512 SHA1E SHA1 MD5E MD5 WORM URL + remote types: git gcrypt p2p S3 bup directory rsync web bittorrent webdav tahoe glacier ddar hook external + +The changes to build: https://git.archlinux.org/svntogit/community.git/commit/trunk?h=packages/git-annex&id=e75ed0bdf9748df000aa584e0ebff3b7d56b76d6 +"""]] diff --git a/doc/bugs/Cannot_build_with_GHC_8.2.1/comment_5_a0663aeef21b09393600593409f1a7a9._comment b/doc/bugs/Cannot_build_with_GHC_8.2.1/comment_5_a0663aeef21b09393600593409f1a7a9._comment new file mode 100644 index 0000000000..93d4a11753 --- /dev/null +++ b/doc/bugs/Cannot_build_with_GHC_8.2.1/comment_5_a0663aeef21b09393600593409f1a7a9._comment @@ -0,0 +1,16 @@ +[[!comment format=mdwn + username="joey" + subject="""comment 5""" + date="2017-08-17T19:12:09Z" + content=""" +Thanks for working on this! + +It's entirely possible that the quickcheck tests happen to generate +better test data under the new ghc, and have found an actual bug. +I doubt that it's related to your changes. + +I do need git-annex to remain buildable on OSX and Windows. What basically +needs to be done is, instead of sed-ing those files out of the cabal file, +make the cabal file say "if os(windows)" to conditionally include the +modules that only work on one OS. +"""]] diff --git a/doc/bugs/Cannot_build_with_GHC_8.2.1/comment_6_e948c8fcf34559808726aaa0d5f5f020._comment b/doc/bugs/Cannot_build_with_GHC_8.2.1/comment_6_e948c8fcf34559808726aaa0d5f5f020._comment new file mode 100644 index 0000000000..ee4b88feb4 --- /dev/null +++ b/doc/bugs/Cannot_build_with_GHC_8.2.1/comment_6_e948c8fcf34559808726aaa0d5f5f020._comment @@ -0,0 +1,14 @@ +[[!comment format=mdwn + username="joey" + subject="""comment 6""" + date="2017-08-18T15:02:42Z" + content=""" +I've updated the cabal file, it should build now. Although unfortunately +the breaking changes to cabal are such that it might still fail to build +with some OS's and some combinations of build flags, where it used to build +before. I only tested on Linux with stack (modified for resolver: +nightly-2017-08-17) + +As to the quickcheck failures, I think that's the same problem I already +fixed in [[!commit da8e84efe997fcbfcf489bc4fa9cc835ed131d3a]]. +"""]] diff --git a/doc/bugs/Cannot_build_with_GHC_8.2.1/comment_7_bd5b764715a9a7c35921803d534a9989._comment b/doc/bugs/Cannot_build_with_GHC_8.2.1/comment_7_bd5b764715a9a7c35921803d534a9989._comment new file mode 100644 index 0000000000..ebc05ffecc --- /dev/null +++ b/doc/bugs/Cannot_build_with_GHC_8.2.1/comment_7_bd5b764715a9a7c35921803d534a9989._comment @@ -0,0 +1,9 @@ +[[!comment format=mdwn + username="ilovezfs" + avatar="http://cdn.libravatar.org/avatar/f2b3954cf2ed0a551de9a49d3b6a64d0" + subject="Exception: getCurrentDirectory:getWorkingDirectory: resource exhausted (Too many open files)" + date="2017-08-18T18:28:01Z" + content=""" +Thanks! 6.20170818 does build with Cabal 2.0.0. However, I'm still seeing the too many open files error. +The log is here: https://gist.github.com/ilovezfs/5dabc76086ba4b028b1eae7f301a5219 +"""]] diff --git a/doc/bugs/Cannot_build_with_GHC_8.2.1/comment_8_329e8eb873ca684dc3221b462827ecd5._comment b/doc/bugs/Cannot_build_with_GHC_8.2.1/comment_8_329e8eb873ca684dc3221b462827ecd5._comment new file mode 100644 index 0000000000..183ec3bbc2 --- /dev/null +++ b/doc/bugs/Cannot_build_with_GHC_8.2.1/comment_8_329e8eb873ca684dc3221b462827ecd5._comment @@ -0,0 +1,7 @@ +[[!comment format=mdwn + username="joey" + subject="""comment 8""" + date="2017-10-02T16:36:40Z" + content=""" +The git-annex test FD leak is now fixed. +"""]] diff --git a/doc/bugs/Cannot_build_with_aws_0.17.1.mdwn b/doc/bugs/Cannot_build_with_aws_0.17.1.mdwn new file mode 100644 index 0000000000..3a3a528947 --- /dev/null +++ b/doc/bugs/Cannot_build_with_aws_0.17.1.mdwn @@ -0,0 +1,47 @@ +### Please describe the problem. +git-annex doesn't build if the aws dependency is version 0.17.1 + +This probably wasn't caught during your testing because the stack.yaml is currently set to use aws-0.16. + +### What steps will reproduce the problem? +cabal install with --flags=s3\ webapp + +### What version of git-annex are you using? On what operating system? +6.20171003 on macOS + +### Please provide any additional information below. + +The error is + +[[!format sh """ +[327 of 580] Compiling Remote.Helper.Http ( Remote/Helper/Http.hs, dist/dist-sandbox-69bd6735/build/git-annex/git-annex-tmp/Remote/Helper/Http.o ) +[328 of 580] Compiling Remote.WebDAV ( Remote/WebDAV.hs, dist/dist-sandbox-69bd6735/build/git-annex/git-annex-tmp/Remote/WebDAV.o ) +[329 of 580] Compiling Remote.S3 ( Remote/S3.hs, dist/dist-sandbox-69bd6735/build/git-annex/git-annex-tmp/Remote/S3.o ) + +Remote/S3.hs:501:57: error: + • Couldn't match expected type ‘AWS.Configuration’ + with actual type ‘Maybe + http-client-0.5.7.0:Network.HTTP.Client.Types.Proxy + -> AWS.Configuration’ + • Probable cause: ‘awscfg’ is applied to too few arguments + In the second argument of ‘S3Handle’, namely ‘awscfg’ + In the second argument of ‘($)’, namely ‘S3Handle mgr awscfg s3cfg’ + In the second argument of ‘($)’, namely + ‘Just $ S3Handle mgr awscfg s3cfg’ + | +501 | a $ Just $ S3Handle mgr awscfg s3cfg + | ^^^^^^ +cabal: Leaving directory '.' +cabal: Error: some packages failed to install: +git-annex-6.20171003-5GFRliFOEl0KqDuhwYRTdH failed during the building phase. +The exception was: +ExitFailure 1 +"""]] + +Full build log is here: https://gist.github.com/ilovezfs/661a158dbe8aac33802f9b43eb150539 + +### Have you had any luck using git-annex before? (Sometimes we get tired of reading bug reports all day and a lil' positive end note does wonders) + +Yes! :) + +> [[fixed|done]] --[[Joey]] diff --git a/doc/bugs/Cannot_build_with_aws_0.17.1/comment_1_937238ba11540f1ee9ba8de621277b2f._comment b/doc/bugs/Cannot_build_with_aws_0.17.1/comment_1_937238ba11540f1ee9ba8de621277b2f._comment new file mode 100644 index 0000000000..d22c31e9c4 --- /dev/null +++ b/doc/bugs/Cannot_build_with_aws_0.17.1/comment_1_937238ba11540f1ee9ba8de621277b2f._comment @@ -0,0 +1,11 @@ +[[!comment format=mdwn + username="https://launchpad.net/~felixonmars" + nickname="felixonmars" + avatar="http://cdn.libravatar.org/avatar/17284a3bb2e4ad9d3be8fab31f49865be9c1dc22143c728de731fe800a335d38" + subject="comment 1" + date="2017-10-05T14:02:03Z" + content=""" +My humble workaround: + + sed -i 's/let awscfg = AWS.Configuration AWS.Timestamp awscreds debugMapper$/let awscfg = AWS.Configuration AWS.Timestamp awscreds debugMapper Nothing/ ' Remote/S3.hs +"""]] diff --git a/doc/bugs/Cannot_clone_an_annex.mdwn b/doc/bugs/Cannot_clone_an_annex.mdwn new file mode 100644 index 0000000000..77989ecb67 --- /dev/null +++ b/doc/bugs/Cannot_clone_an_annex.mdwn @@ -0,0 +1,69 @@ +I have an annex that I use to store my digital photos. I had a few false +starts creating this annex, but now it's looking good on my server: + + root@titan.local:/tank/Media/Pictures# git annex status + supported backends: SHA256E SHA1E SHA512E SHA224E SHA384E SHA256 SHA1 SHA512 SHA224 SHA384 WORM URL + supported remote types: git S3 bup directory rsync web hook + trusted repositories: 0 + semitrusted repositories: 2 + 00000000-0000-0000-0000-000000000001 -- web + be88bc5a-17e2-11e2-a99b-d388d4437350 -- here (titan) + untrusted repositories: 0 + dead repositories: 5 + 0A9F3136-A12A-43C7-9BE2-33F59954FD52 -- vulcan + 57349F02-E497-4420-9230-6B15D8AB14EE -- vulcan + 6195C912-2707-4B75-AC8C-11C51FAA8FE0 -- vulcan + D51DEDC4-9255-4A99-8520-2B1CED337674 -- hermes + EE327B34-3E20-4B5B-8F0E-D500CBC9738D -- hermes + transfers in progress: none + available local disk space: unknown + local annex keys: 20064 + local annex size: 217 gigabytes + known annex keys: 21496 + known annex size: 217 gigabytes + bloom filter size: 16 mebibytes (4% full) + backend usage: + SHA256E: 41560 + root@titan.local:/tank/Media/Pictures# git annex unused + unused . (checking for unused data...) ok + +It passes `git annex fsck` without any problems. However, when I "git clone" +this annex to my desktop machine and then do a `git annex sync`, I see this: + + Vulcan /Volumes/tank/Media/Pictures (master) $ git annex status + supported backends: SHA256E SHA1E SHA512E SHA224E SHA384E SHA256 SHA1 SHA512 SHA224 SHA384 WORM URL + supported remote types: git S3 bup directory rsync web hook + trusted repositories: 0 + semitrusted repositories: 5 + 00000000-0000-0000-0000-000000000001 -- web + 0A9F3136-A12A-43C7-9BE2-33F59954FD52 -- vulcan + 274D3474-7A25-44CD-8368-CF11C451014F -- here (vulcan) + EE327B34-3E20-4B5B-8F0E-D500CBC9738D -- hermes + be88bc5a-17e2-11e2-a99b-d388d4437350 -- titan + untrusted repositories: 0 + dead repositories: 3 + 57349F02-E497-4420-9230-6B15D8AB14EE -- vulcan + 6195C912-2707-4B75-AC8C-11C51FAA8FE0 -- vulcan + D51DEDC4-9255-4A99-8520-2B1CED337674 -- hermes + transfers in progress: none + available local disk space: 1 terabyte (+1 megabyte reserved) + local annex keys: 0 + local annex size: 0 bytes + known annex keys: 21025 + known annex size: 217 gigabytes + bloom filter size: 16 mebibytes (0% full) + backend usage: + SHA256: 18707 + SHA256E: 2318 + +Where did all these `SHA256` keys come from? + +Why doesn't the known annex keys size match? + +Further, I cannot `git annex get` on most of the files, because it says that +the `SHA256` key is not present. + +It looks like I'll have to rollback my ZFS snapshots and start over, but I'm +wondering: how was I even able to create this situation? + +> [[Done]]; user error. --[[Joey]] diff --git a/doc/bugs/Cannot_clone_an_annex/comment_1_b40a2652361a79c6c6eab0fc21be8e46._comment b/doc/bugs/Cannot_clone_an_annex/comment_1_b40a2652361a79c6c6eab0fc21be8e46._comment new file mode 100644 index 0000000000..44b93b1c2f --- /dev/null +++ b/doc/bugs/Cannot_clone_an_annex/comment_1_b40a2652361a79c6c6eab0fc21be8e46._comment @@ -0,0 +1,8 @@ +[[!comment format=mdwn + username="http://joeyh.name/" + ip="4.154.0.23" + subject="comment 1" + date="2012-10-22T13:53:06Z" + content=""" +The SHA256 keys are symlinks in your git repository with \"SHA256-\" as the prefix on the file linked to. You should be able to determine the rest by looking at the history of those files in git. +"""]] diff --git a/doc/bugs/Cannot_delete_remote_when_ssh_sync_fails.mdwn b/doc/bugs/Cannot_delete_remote_when_ssh_sync_fails.mdwn new file mode 100644 index 0000000000..f780ace7ca --- /dev/null +++ b/doc/bugs/Cannot_delete_remote_when_ssh_sync_fails.mdwn @@ -0,0 +1,8 @@ +### Please describe the problem. +The webapp does not offer me the option to delete a remote repository that it did not succeed to synchronize with. Status of the repository is sync enabled (metadata only) but in settings I may only edit the repo, but cannot delete it. + +### What steps will reproduce the problem? +Set up git-annex on a new computer. Forget to enable ssh on the machine. Perform a local pairing (success). The sync fails for obvious reasons. + +### What version of git-annex are you using? On what operating system? +System is ubuntu but I assume this is a general issue. Version of git-annex is 5.20140117 diff --git a/doc/bugs/Cannot_delete_remote_when_ssh_sync_fails/comment_1_3c8e8fae688a9db8e18e869a187fb4eb._comment b/doc/bugs/Cannot_delete_remote_when_ssh_sync_fails/comment_1_3c8e8fae688a9db8e18e869a187fb4eb._comment new file mode 100644 index 0000000000..c89d1cb0c7 --- /dev/null +++ b/doc/bugs/Cannot_delete_remote_when_ssh_sync_fails/comment_1_3c8e8fae688a9db8e18e869a187fb4eb._comment @@ -0,0 +1,8 @@ +[[!comment format=mdwn + username="http://joeyh.name/" + ip="209.250.56.46" + subject="comment 1" + date="2014-01-26T17:57:25Z" + content=""" +If you go to Edit, it will let you upgrade the repository, which should fix your problem. +"""]] diff --git a/doc/bugs/Cannot_delete_remote_when_ssh_sync_fails/comment_2_e189617c4ac23df50f02af8c517fa399._comment b/doc/bugs/Cannot_delete_remote_when_ssh_sync_fails/comment_2_e189617c4ac23df50f02af8c517fa399._comment new file mode 100644 index 0000000000..0d2f2e99b1 --- /dev/null +++ b/doc/bugs/Cannot_delete_remote_when_ssh_sync_fails/comment_2_e189617c4ac23df50f02af8c517fa399._comment @@ -0,0 +1,14 @@ +[[!comment format=mdwn + username="https://www.google.com/accounts/o8/id?id=AItOawmXtqLEfN6cGcF9gW_09QUDoRpsexSdRcM" + nickname="Thomas" + subject="comment 2" + date="2014-01-26T18:12:08Z" + content=""" +Unfortunately, upgrading did not fix it. I suppose, a restart of the remote server (ssh was just installed) would have been necessary. Instead, I just deleted the repository on the remote. + +Now my local repository was caught in a situation, where I couldn't upgrade the broken remote and I couldn't delete it either. + +By recreating the repository on the remote, I finally was able to get rid of the broken link, but I think the proper solution would be to have the ability to remove unsynced repositories. + +Thanks for the quick response! +"""]] diff --git a/doc/bugs/Commiting_Files_Containing_Non_Ascii_Char_on_OS_X_.mdwn b/doc/bugs/Commiting_Files_Containing_Non_Ascii_Char_on_OS_X_.mdwn new file mode 100644 index 0000000000..5ca73dd805 --- /dev/null +++ b/doc/bugs/Commiting_Files_Containing_Non_Ascii_Char_on_OS_X_.mdwn @@ -0,0 +1,46 @@ +### Please describe the problem. + +Any file containing non ascii chars either won't get commited by the webapp or when clone of that repo is made they are shown as deleted in git status but the files are there. + +you can not commit them neither, + + git commit -m 'foo' + +or + + git annex sync + +will get them to commit trying to reset the state using, + + git checkout master + +or + + git clean -f + +won't fix the issue. till the issue is fixed sync does not work from the command line since there are changes. Temporarily running assistant in the directory does fix the problem but every now and then it pops again. + +Also moving any symlinks around containing non ascii chars also causes the same issue. git annex won't add the moved symlinks git annex fix or git annex add won't add them git annex sync also skips them again the only solution is to either run the assistant or restart it if it is running. + +### What steps will reproduce the problem? + +Adding any file with a non ascii char on OS X (10.9). + + +### What version of git-annex are you using? On what operating system? + +git-annex version: 5.20140421-gc34a665 + +### Please provide any additional information below. + +Also another thing I noticed is that same file names same repo will clone fine on Linux no errors this only happens on OS X. + +[[!format sh """ +# If you can, paste a complete transcript of the problem occurring here. +# If the problem is with the git-annex assistant, paste in .git/annex/daemon.log + + +# End of transcript or log. +"""]] + +[[!tag moreinfo]] diff --git a/doc/bugs/Commiting_Files_Containing_Non_Ascii_Char_on_OS_X_/comment_1_81a4647a9b51bed8c230a2a16990915d._comment b/doc/bugs/Commiting_Files_Containing_Non_Ascii_Char_on_OS_X_/comment_1_81a4647a9b51bed8c230a2a16990915d._comment new file mode 100644 index 0000000000..e600657377 --- /dev/null +++ b/doc/bugs/Commiting_Files_Containing_Non_Ascii_Char_on_OS_X_/comment_1_81a4647a9b51bed8c230a2a16990915d._comment @@ -0,0 +1,14 @@ +[[!comment format=mdwn + username="http://joeyh.name/" + ip="209.250.56.36" + subject="comment 1" + date="2014-05-24T19:23:08Z" + content=""" +I think you are going to need to provide more information: + +* Some actual file names that have the problem. \"non-ascii chars\" is a very wide description, you don't even say if you're using unicode or what. +* Some error messages or transcripts of what happens when you see the problem. +* What is your locale setting on OSX. + +I tried with some unicode filenames on OSX and they were committed without difficulty in the en_US.utf8 locale. +"""]] diff --git a/doc/bugs/Commiting_Files_Containing_Non_Ascii_Char_on_OS_X_/comment_2_f8e97dded9ba6d2305de1278025924fe._comment b/doc/bugs/Commiting_Files_Containing_Non_Ascii_Char_on_OS_X_/comment_2_f8e97dded9ba6d2305de1278025924fe._comment new file mode 100644 index 0000000000..b404cc3817 --- /dev/null +++ b/doc/bugs/Commiting_Files_Containing_Non_Ascii_Char_on_OS_X_/comment_2_f8e97dded9ba6d2305de1278025924fe._comment @@ -0,0 +1,23 @@ +[[!comment format=mdwn + username="https://me.yahoo.com/a/FHnTlSBo1eCGJRwueeKeB6.RCaPbGMPr5jxx8A--#ce0d8" + nickname="Hamza" + subject="comment 2" + date="2014-05-25T15:16:33Z" + content=""" +I file name I remember messing is the following, + + George's Café + +I experience this problem with any char that is not ascii. As for my locale, + +LANG=\"en_US.UTF-8\" +LC_COLLATE=\"en_US.UTF-8\" +LC_CTYPE=\"en_US.UTF-8\" +LC_MESSAGES=\"en_US.UTF-8\" +LC_MONETARY=\"en_US.UTF-8\" +LC_NUMERIC=\"en_US.UTF-8\" +LC_TIME=\"en_US.UTF-8\" +LC_ALL=\"en_US.UTF-8\" + +There no errors other than the behaviour I've described. +"""]] diff --git a/doc/bugs/Commiting_Files_Containing_Non_Ascii_Char_on_OS_X_/comment_3_eb34061429cb8c7d0b155825e84b657b._comment b/doc/bugs/Commiting_Files_Containing_Non_Ascii_Char_on_OS_X_/comment_3_eb34061429cb8c7d0b155825e84b657b._comment new file mode 100644 index 0000000000..acd5775cd5 --- /dev/null +++ b/doc/bugs/Commiting_Files_Containing_Non_Ascii_Char_on_OS_X_/comment_3_eb34061429cb8c7d0b155825e84b657b._comment @@ -0,0 +1,35 @@ +[[!comment format=mdwn + username="http://joeyh.name/" + ip="209.250.56.176" + subject="comment 3" + date="2014-05-27T18:02:11Z" + content=""" +I tried with that filename, and had no difficulty committing it. + +
    +oberon:repo joeyh$ git annex add George\'s\ Cafe\314\201 
+add George's Café ok
+(Recording state in git...)
+oberon:repo joeyh$ git commit -m foo
+[master (root-commit) 8acd5d4] foo
+ Committer: Joey Hess 
+Your name and email address were configured automatically based
+on your username and hostname. Please check that they are accurate.
+You can suppress this message by setting them explicitly:
+
+    git config --global user.name \"Your Name\"
+    git config --global user.email you@example.com
+
+After doing this, you may fix the identity used for this commit with:
+
+    git commit --amend --reset-author
+
+ 1 file changed, 1 insertion(+)
+ create mode 120000 \"George's Caf\303\251\"
+
+
    + +I even tried running the assistant with all locale settings set to C, and it still had no problem adding and committing that filename. + +So, I need help reproducing this, or I need to see the error message you see when the problem happens to you. +"""]] diff --git a/doc/bugs/Commiting_Files_Containing_Non_Ascii_Char_on_OS_X_/comment_4_8cf5dc85a94c60b9c88c8e8354f19772._comment b/doc/bugs/Commiting_Files_Containing_Non_Ascii_Char_on_OS_X_/comment_4_8cf5dc85a94c60b9c88c8e8354f19772._comment new file mode 100644 index 0000000000..ca4b9ad14f --- /dev/null +++ b/doc/bugs/Commiting_Files_Containing_Non_Ascii_Char_on_OS_X_/comment_4_8cf5dc85a94c60b9c88c8e8354f19772._comment @@ -0,0 +1,32 @@ +[[!comment format=mdwn + username="https://joelpurra.com/" + avatar="http://cdn.libravatar.org/avatar/6b09b608da8a2f34acf3d89caf8b7438ddbd404bb2db31414855118a7dab678c" + subject="macOS HFS+, core.precomposeUnicode" + date="2017-01-13T14:27:15Z" + content=""" +@joeyh.name: did you notice the subtle difference between the filename in the input and the output in your example? I think it might cause problems due to a discrepancy between `git annex` and the file system when checking if a file exists or not. Not sure if the problem is in `git` or `git annex`; what do you think? + +```text +git annex add George\'s\ Cafe\314\201 +... +create mode 120000 \"George's Caf\303\251\" +``` + +See [Unicode Subtleties in the HFS Plus Volume Format](https://developer.apple.com/legacy/library/technotes/tn/tn1150.html#UnicodeSubtleties) and `git config` for [`core.precomposeUnicode`](https://git-scm.com/docs/git-config#git-config-coreprecomposeUnicode) + +Here's a `printf` example. + +```bash +printf '\303\251 e\314\201 (note double space) \314\201' +``` +```text +é é (note double space) ́ +``` +```bash +printf '\303\251 e\314\201 (note double space) \314\201' | LC_ALL=C vis -otc +``` +```text +\303\251 e\314\201 (note double space) \314\201 +``` + +"""]] diff --git a/doc/bugs/Continual_space_exhaustion_from_syncing_metadata.mdwn b/doc/bugs/Continual_space_exhaustion_from_syncing_metadata.mdwn new file mode 100644 index 0000000000..ca826fb7fb --- /dev/null +++ b/doc/bugs/Continual_space_exhaustion_from_syncing_metadata.mdwn @@ -0,0 +1,26 @@ +### Please describe the problem. +When the preferred content for a repo is greater than the free space on the drive for me, the space becomes exhausted due to upload that occurs during syncing -- the git history ends up exhausting the space. The result for me has been corrupted repos with broken refs and heads. + +I think it would be better if the git syncing process respected the same space limitations that the content syncing does, and did not push if space was exhausted on the destination, nor pull if space exhausted locally. + +### What steps will reproduce the problem? +Make enough git commits that it exhausts the free space on some repo. + +### What version of git-annex are you using? On what operating system? +I've been using a few different versions; right now I am using 6.2016-0511 and 6.2016-0517-g766728c +but I have not tested this issue with these most recent versions. I am still trying to get my smaller repos back together. + +### Please provide any additional information below. + +[[!format sh """ +# If you can, paste a complete transcript of the problem occurring here. +# If the problem is with the git-annex assistant, paste in .git/annex/daemon.log + + +# End of transcript or log. +"""]] + +### Have you had any luck using git-annex before? (Sometimes we get tired of reading bug reports all day and a lil' positive end note does wonders) + +Yes !!! I'm moving all my files into my annex. It is very robust; whenever something is wrong there is always some other copy somewhere that can be used. + diff --git a/doc/bugs/Continual_space_exhaustion_from_syncing_metadata/comment_1_93e3deab54b34e9ad608fd549119e221._comment b/doc/bugs/Continual_space_exhaustion_from_syncing_metadata/comment_1_93e3deab54b34e9ad608fd549119e221._comment new file mode 100644 index 0000000000..df07a2a8ad --- /dev/null +++ b/doc/bugs/Continual_space_exhaustion_from_syncing_metadata/comment_1_93e3deab54b34e9ad608fd549119e221._comment @@ -0,0 +1,26 @@ +[[!comment format=mdwn + username="joey" + subject="""comment 1""" + date="2016-05-23T18:02:11Z" + content=""" +It would be nice if git checked disk space before writing object +files. However, it unfortunatly does not do so currently, and not checking +disk space is typical of unix tools so it might be hard to convince the git +devs to add that. + +git-annex has annex.diskreserve because it's dealing with so much data and +such large files that it's best to not let it fill the disk and instead +abort before it downloads too much data. + +I don't see any way that git-annex can avoid git objects taking up too much +disk space, without re-implementing all of git + space checking. + +Sure, `git annex sync` could avoid pulling if annex.diskreserve was not +free, but this would not help with manual `git pull`, or `git commit`, or +`git receive-pack`, or any of the other ways objects can be added to a git +repository. + +What you can do though, is set annex.diskreserve to a reasonably +large amount, so that git-annex tries to keep that much space free. +Eg, set it to at least half the current size of `du -hs .git/objects` +"""]] diff --git a/doc/bugs/Continual_space_exhaustion_from_syncing_metadata/comment_2_9401be5e6af886edadaf3ad7ba6a071d._comment b/doc/bugs/Continual_space_exhaustion_from_syncing_metadata/comment_2_9401be5e6af886edadaf3ad7ba6a071d._comment new file mode 100644 index 0000000000..f0a570eae7 --- /dev/null +++ b/doc/bugs/Continual_space_exhaustion_from_syncing_metadata/comment_2_9401be5e6af886edadaf3ad7ba6a071d._comment @@ -0,0 +1,12 @@ +[[!comment format=mdwn + username="xloem" + subject="comment 2" + date="2016-10-09T12:13:38Z" + content=""" +I'm still running into this issue. Although it is true that users can manually exhaust space, they are in control of what is happening, get more feedback, and there is no automatic process that will continue to fail further when the space is exhausted. + +I do not use the assistant because of its frequent syncing causing complete space exhaustion, and then its inability to recover because it cannot write temporary or log files. Occasionally I have accidentally committed huge files directly to git, and this has also caused the issue. + +I think it would be really great if git annex prevented this workflow-halting event at least by refusing to pull/commit/etc if reserved space is not met unless --force is passed. Additionally, it would really be great if actions that result in space exhaustion (fail, and then free space == 0) cleaned up their temporary files after the fact, certainly before trying to log anything. + +"""]] diff --git a/doc/bugs/Corrupted_.git__47__annex__47__index_when_running_assistant.mdwn b/doc/bugs/Corrupted_.git__47__annex__47__index_when_running_assistant.mdwn new file mode 100644 index 0000000000..efe32c8397 --- /dev/null +++ b/doc/bugs/Corrupted_.git__47__annex__47__index_when_running_assistant.mdwn @@ -0,0 +1,658 @@ +### Please describe the problem. +I’ve noticed a repo in which I’m running the assistant wouldn’t push updates to its remote anymore. Inspecting the assistant log revealed that there was a problem with the index used for the git-annex branch. After deleting the index file, the assistant could sync to the remote again. (Could deleting the annex index have introduced any inconsistencies into the git-annex branch?) + +### What steps will reproduce the problem? +Uncertain. However, I made a copy of .git/annex/index and can reproduce the error by cloning my original repo, checking out the git-annex branch, copying in the backed-up index file and doing git write-tree, which then yields an invalid object error. + +It looks like the respective object isn’t present in the git object store. The error is for b5f693aa67b6d185e6d967e0a45c3c57a4f4b4a8, but no directory .git/objects/b5 exists (even after unpacking packed objects). + +I could supply you with the index file; do you need anything else? If I don’t clone from my repo, I obviously get an invalid object for everything; should I supply you with a git-bundle of the git-annex ref? + +### What version of git-annex are you using? On what operating system? +git-annex 5.20150508-1 +Linux 4.0.5 + +### Please provide any additional information below. + +Below is the daemon.log* concatenated, starting with the earliest one. Grep for “invalid object” or “write tree”. + +[[!format sh """ +[2015-06-05 16:59:45 CEST] main: starting assistant version 5.20150508-g883d57f +[2015-06-05 16:59:45 CEST] Cronner: You should enable consistency checking to protect your data. +[2015-06-05 16:59:57 CEST] TransferScanner: Syncing with origin + + No known network monitor available through dbus; falling back to polling +(scanning...) [2015-06-05 16:59:57 CEST] Watcher: Performing startup scan +(started...) [2015-06-05 16:59:58 CEST] RemoteControl: Syncing with origin +gpg: assuming signed data in '/tmp/git-annex.tmp.0/info' +gpg: Signature made Thu May 28 19:01:58 2015 CEST using DSA key ID 89C809CB +gpg: /tmp/git-annex-gpg.tmp.0/trustdb.gpg: trustdb created +gpg: Good signature from "git-annex distribution signing key (for Joey Hess) " [unknown] +gpg: WARNING: This key is not certified with a trusted signature! +gpg: There is no indication that the signature belongs to the owner. +Primary key fingerprint: 4005 5C6A FD2D 526B 2961 E78F 5EE1 DBA7 89C8 09CB +[2015-06-05 16:59:58 CEST] Upgrader: An upgrade of git-annex is available. (version 5.20150528) +[2015-06-05 16:59:59 CEST] Committer: Committing changes to git +[2015-06-05 16:59:59 CEST] Pusher: Syncing with origin +error: Unable to create '/home/seb/Annex/.git/refs/remotes/origin/master.lock': File exists. + +If no other git process is currently running, this probably means a +git process crashed in this repository earlier. Make sure no other git +process is running and remove the file manually to continue. +error: Cannot lock the ref 'refs/remotes/origin/master'. +From ssh://redacted.net/srv/git/seb/annex/annex + ! d71a4e0..1f265ec master -> origin/master (unable to update local ref) +From ssh://redacted.net/srv/git/seb/annex/annex + d71a4e0..1f265ec master -> origin/master +error: Unable to create '/home/seb/Annex/.git/refs/remotes/origin/git-annex.lock': File exists. + +If no other git process is currently running, this probably means a +git process crashed in this repository earlier. Make sure no other git +process is running and remove the file manually to continue. +error: Cannot lock the ref 'refs/remotes/origin/git-annex'. + ! 33446d6..973fbda git-annex -> origin/git-annex (unable to update local ref) + 33446d6..973fbda git-annex -> origin/git-annex +error: Unable to create '/home/seb/Annex/.git/refs/remotes/origin/synced/git-annex.lock': File exists. + +If no other git process is currently running, this probably means a +git process crashed in this repository earlier. Make sure no other git +process is running and remove the file manually to continue. +error: Cannot lock the ref 'refs/remotes/origin/synced/git-annex'. + ! 33446d6..973fbda synced/git-annex -> origin/synced/git-annex (unable to update local ref) + d71a4e0..1f265ec synced/master -> origin/synced/master + 33446d6..973fbda synced/git-annex -> origin/synced/git-annex +error: Ref refs/remotes/origin/synced/master is at 1f265ecc6288ead453681794551c527f64660f8c but expected d71a4e027cdd61c1323fbdc67103dde97f613192 +error: Cannot lock the ref 'refs/remotes/origin/synced/master'. + ! d71a4e0..1f265ec synced/master -> origin/synced/master (unable to update local ref) +Updating d71a4e0..1f265ec +Fast-forward + org/software/software.org | 4 +++- + org/software/theming.org | 7 +++++++ + 2 files changed, 10 insertions(+), 1 deletion(-) +Already up-to-date. +To ssh://redacted.net/srv/git/seb/annex/annex.git + + 973fbda...33446d6 git-annex -> synced/git-annex (forced update) + ! [rejected] master -> synced/master (non-fast-forward) +error: failed to push some refs to 'ssh://redacted.net/srv/git/seb/annex/annex.git' +hint: Updates were rejected because a pushed branch tip is behind its remote +hint: counterpart. Check out this branch and integrate the remote changes +hint: (e.g. 'git pull ...') before pushing again. +hint: See the 'Note about fast-forwards' in 'git push --help' for details. +[2015-06-05 17:00:00 CEST] Committer: Committing changes to git +To ssh://redacted.net/srv/git/seb/annex/annex.git + 33446d6..973fbda git-annex -> synced/git-annex +Everything up-to-date +[2015-06-05 17:00:05 CEST] Pusher: Syncing with origin +Everything up-to-date +Connection to redacted.net closed by remote host. +ssh: connect to host redacted.nessh: connect to host redactedssh: connect to host redacted.n[2015-06-05 17:59:57 CEST] NetWatcherFallback: Syncing with origin +Everything up-to-date +[2015-06-05 18:51:33 CEST] Committer: Committing changes to git +[2015-06-05 18:51:33 CEST] Pusher: Syncing with origin +[2015-06-05 18:51:34 CEST] Committer: Committing changes to git +To ssh://redacted.net/srv/git/seb/annex/annex.git + 1f265ec..1cee9eb master -> synced/master +[2015-06-05 18:51:37 CEST] Pusher: Syncing with origin +Everything up-to-date +[2015-06-05 19:00:01 CEST] NetWatcherFallback: Syncing with origin +Everything up-to-date +packet_write_wait: Connection to 37.221.198.119: Broken pipe +[2015-06-05 20:00:03 CEST] NetWatcherFallback: Syncing with origin +Everything up-to-date +[2015-06-05 20:13:01 CEST] main: starting assistant version 5.20150508-g883d57f +[2015-06-05 20:13:01 CEST] Cronner: You should enable consistency checking to protect your data. +[2015-06-05 20:13:12 CEST] TransferScanner: Syncing with origin + + No known network monitor available through dbus; falling back to polling +(scanning...) [2015-06-05 20:13:12 CEST] Watcher: Performing startup scan +(started...) gpg: assuming signed data in '/tmp/git-annex.tmp.0/info' +gpg: Signature made Thu May 28 19:01:58 2015 CEST using DSA key ID 89C809CB +gpg: /tmp/git-annex-gpg.tmp.0/trustdb.gpg: trustdb created +gpg: Good signature from "git-annex distribution signing key (for Joey Hess) " [unknown] +gpg: WARNING: This key is not certified with a trusted signature! +gpg: There is no indication that the signature belongs to the owner. +Primary key fingerprint: 4005 5C6A FD2D 526B 2961 E78F 5EE1 DBA7 89C8 09CB +[2015-06-05 20:13:13 CEST] Upgrader: An upgrade of git-annex is available. (version 5.20150528) +Everything up-to-date +[2015-06-05 20:13:14 CEST] Committer: Committing changes to git +[2015-06-05 20:13:14 CEST] Pusher: Syncing with origin +Everything up-to-date +[2015-06-05 21:13:12 CEST] NetWatcherFallback: Syncing with origin +Everything up-to-date +[2015-06-05 22:13:14 CEST] NetWatcherFallback: Syncing with origin +Everything up-to-date +[2015-06-05 22:33:04 CEST] Committer: Committing changes to git +[2015-06-05 22:33:04 CEST] Pusher: Syncing with origin +Everything up-to-date +[2015-06-05 22:33:08 CEST] Committer: Committing changes to git +[2015-06-05 22:33:08 CEST] Pusher: Syncing with origin +Everything up-to-date +[2015-06-05 22:33:11 CEST] Committer: Committing changes to git +[2015-06-05 22:33:11 CEST] Pusher: Syncing with origin +Everything up-to-date +[2015-06-05 22:33:14 CEST] Committer: Committing changes to git +[2015-06-05 22:33:14 CEST] Pusher: Syncing with origin +Everything up-to-date +[2015-06-05 22:33:16 CEST] Committer: Committing changes to git +[2015-06-05 22:33:18 CEST] Pusher: Syncing with origin +Everything up-to-date +[2015-06-05 22:33:20 CEST] Committer: Committing changes to git +[2015-06-05 22:33:21 CEST] Pusher: Syncing with origin +Everything up-to-date +[2015-06-05 22:33:27 CEST] Committer: Committing changes to git +[2015-06-05 22:33:27 CEST] Pusher: Syncing with origin +Everything up-to-date +[2015-06-05 22:33:28 CEST] Committer: Committing changes to git +[2015-06-05 22:33:30 CEST] Pusher: Syncing with origin +Everything up-to-date +[2015-06-05 22:33:37 CEST] Committer: Committing changes to git +[2015-06-05 22:33:37 CEST] Pusher: Syncing with origin +Everything up-to-date +[2015-06-05 22:33:43 CEST] Committer: Committing changes to git +[2015-06-05 22:33:43 CEST] Pusher: Syncing with origin +Everything up-to-date +[2015-06-05 22:34:12 CEST] Committer: Committing changes to git +[2015-06-05 22:34:12 CEST] Pusher: Syncing with origin +[2015-06-05 22:34:13 CEST] Committer: Committing changes to git +To ssh://redacted.net/srv/git/seb/annex/annex.git + 1cee9eb..46828df master -> synced/master +[2015-06-05 22:34:15 CEST] Pusher: Syncing with origin +To ssh://redacted.net/srv/git/seb/annex/annex.git + 46828df..8e93c98 master -> synced/master +[2015-06-05 22:34:23 CEST] Committer: Committing changes to git +[2015-06-05 22:34:23 CEST] Pusher: Syncing with origin +To ssh://redacted.net/srv/git/seb/annex/annex.git + 8e93c98..9bed0e6 master -> synced/master +[2015-06-05 22:34:24 CEST] Committer: Committing changes to git +[2015-06-05 22:34:27 CEST] Pusher: Syncing with origin +Everything up-to-date +[2015-06-05 23:04:49 CEST] Committer: Committing changes to git +[2015-06-05 23:04:49 CEST] Pusher: Syncing with origin +To ssh://redacted.net/srv/git/seb/annex/annex.git + 9bed0e6..ec8f2c4 master -> synced/master +[2015-06-05 23:04:50 CEST] Committer: Committing changes to git +[2015-06-05 23:04:52 CEST] Pusher: Syncing with origin +Everything up-to-date +[2015-06-05 23:11:39 CEST] Committer: Committing changes to git +[2015-06-05 23:11:39 CEST] Pusher: Syncing with origin +To ssh://redacted.net/srv/git/seb/annex/annex.git + ec8f2c4..14bc26a master -> synced/master +[2015-06-05 23:11:40 CEST] Committer: Committing changes to git +[2015-06-05 23:11:43 CEST] Pusher: Syncing with origin +Everything up-to-date +[2015-06-05 23:13:16 CEST] NetWatcherFallback: Syncing with origin +Everything up-to-date +[2015-06-05 23:33:28 CEST] Committer: Committing changes to git +[2015-06-05 23:33:28 CEST] Pusher: Syncing with origin +To ssh://redacted.net/srv/git/seb/annex/annex.git + 14bc26a..bd150f5 master -> synced/master +[2015-06-05 23:33:29 CEST] Committer: Committing changes to git +[2015-06-05 23:33:32 CEST] Pusher: Syncing with origin +Everything up-to-date +[2015-06-05 23:34:09 CEST] Committer: Committing changes to git +[2015-06-05 23:34:09 CEST] Pusher: Syncing with origin +To ssh://redacted.net/srv/git/seb/annex/annex.git + bd150f5..290c21c master -> synced/master +[2015-06-05 23:34:10 CEST] Committer: Committing changes to git +[2015-06-05 23:34:12 CEST] Pusher: Syncing with origin +Everything up-to-date +[2015-06-05 23:34:15 CEST] Committer: Committing changes to git +[2015-06-05 23:34:15 CEST] Pusher: Syncing with origin +[2015-06-05 23:34:16 CEST] Committer: Committing changes to git +To ssh://redacted.net/srv/git/seb/annex/annex.git + 290c21c..a56091f master -> synced/master +[2015-06-05 23:34:19 CEST] Pusher: Syncing with origin +Everything up-to-date +[2015-06-05 23:35:17 CEST] Committer: Committing changes to git +[2015-06-05 23:35:17 CEST] Pusher: Syncing with origin +[2015-06-05 23:35:18 CEST] Committer: Committing changes to git +To ssh://redacted.net/srv/git/seb/annex/annex.git + a56091f..2fb635b master -> synced/master +[2015-06-05 23:35:21 CEST] Pusher: Syncing with origin +Everything up-to-date + +(recording state in git...) +(recording state in git...) +(recording state in git...) +(recording state in git...) +(recording state in git...) +(recording state in git...) +(recording state in git...) +(recording state in git...) +(recording state in git...) +(recording state in git...) +(recording state in git...) +(recording state in git...) +(recording state in git...) +(recording state in git...) +(recording state in git...) +(recording state in git...) +(recording state in git...) +(recording state in git...) +(recording state in git...) +(recording state in git...) + + lost dbus connection; falling back to polling (ClientError {clientErrorMessage = "Error reading message from socket: UnmarshalError \"Unexpected end of input while parsing message header.\"", clientErrorFatal = True}) + + dbus failed; falling back to mtab polling (ClientError {clientErrorMessage = "Error reading message from socket: UnmarshalError \"Unexpected end of input while parsing message header.\"", clientErrorFatal = True}) +[2015-06-06 07:17:17 CEST] main: starting assistant version 5.20150508-g883d57f +[2015-06-06 07:17:17 CEST] Cronner: You should enable consistency checking to protect your data. +[2015-06-06 07:17:25 CEST] TransferScanner: Syncing with origin + + No known network monitor available through dbus; falling back to polling +(scanning...) [2015-06-06 07:17:30 CEST] Watcher: Performing startup scan +gpg: assuming signed data in '/tmp/git-annex.tmp.0/info' +gpg: Signature made Thu May 28 19:01:58 2015 CEST using DSA key ID 89C809CB +gpg: /tmp/git-annex-gpg.tmp.0/trustdb.gpg: trustdb created +gpg: Good signature from "git-annex distribution signing key (for Joey Hess) " [unknown] +gpg: WARNING: This key is not certified with a trusted signature! +gpg: There is no indication that the signature belongs to the owner. +Primary key fingerprint: 4005 5C6A FD2D 526B 2961 E78F 5EE1 DBA7 89C8 09CB +[2015-06-06 07:17:32 CEST] Upgrader: An upgrade of git-annex is available. (version 5.20150528) +Everything up-to-date +(started...) [2015-06-06 07:17:36 CEST] Committer: Committing changes to git +[2015-06-06 07:17:36 CEST] Pusher: Syncing with origin +Everything up-to-date +[2015-06-06 07:45:05 CEST] main: starting assistant version 5.20150508-g883d57f +[2015-06-06 07:45:05 CEST] Cronner: You should enable consistency checking to protect your data. +[2015-06-06 07:45:10 CEST] TransferScanner: Syncing with origin + + No known network monitor available through dbus; falling back to polling +(scanning...) [2015-06-06 07:45:10 CEST] Watcher: Performing startup scan +(started...) /usr/lib/rsync/rrsync: SSH_ORIGINAL_COMMAND='git-annex-shell 'notifychanges' '/srv/git/seb/annex/annex.git' --uuid a2131774-31b2-47d2-9185-b2e344c16134' is not rsync +gpg: assuming signed data in '/tmp/git-annex.tmp.0/info' +gpg: Signature made Thu May 28 19:01:58 2015 CEST using DSA key ID 89C809CB +gpg: /tmp/git-annex-gpg.tmp.0/trustdb.gpg: trustdb created +gpg: Good signature from "git-annex distribution signing key (for Joey Hess) " [unknown] +gpg: WARNING: This key is not certified with a trusted signature! +gpg: There is no indication that the signature belongs to the owner. +Primary key fingerprint: 4005 5C6A FD2D 526B 2961 E78F 5EE1 DBA7 89C8 09CB +[2015-06-06 07:45:12 CEST] Upgrader: An upgrade of git-annex is available. (version 5.20150528) +[2015-06-06 07:45:12 CEST] Committer: Committing changes to git +[2015-06-06 07:45:12 CEST] Pusher: Syncing with origin +/usr/lib/rsync/rrsync: SSH_ORIGINAL_COMMAND='git-annex-shell 'notifychanges' '/srv/git/seb/annex/annex.git' --uuid a2131774-31b2-47d2-9185-b2e344c16134' is not rsync +Everything up-to-date +Everything up-to-date +/usr/lib/rsync/rrsync: SSH_ORIGINAL_COMMAND='git-annex-shell 'notifychanges' '/srv/git/seb/annex/annex.git' --uuid a2131774-31b2-47d2-9185-b2e344c16134' is not rsync +/usr/lib/rsync/rrsync: SSH_ORIGINAL_COMMAND='git-annex-shell 'notifychanges' '/srv/git/seb/annex/annex.git' --uuid a2131774-31b2-47d2-9185-b2e344c16134' is not rsync +/usr/lib/rsync/rrsync: SSH_ORIGINAL_COMMAND='git-annex-shell 'notifychanges' '/srv/git/seb/annex/annex.git' --uuid a2131774-31b2-47d2-9185-b2e344c16134' is not rsync +/usr/lib/rsync/rrsync: SSH_ORIGINAL_COMMAND='git-annex-shell 'notifychanges' '/srv/git/seb/annex/annex.git' --uuid a2131774-31b2-47d2-9185-b2e344c16134' is not rsync +/usr/lib/rsync/rrsync: SSH_ORIGINAL_COMMAND='git-annex-shell 'notifychanges' '/srv/git/seb/annex/annex.git' --uuid a2131774-31b2-47d2-9185-b2e344c16134' is not rsync + +(recording state in git...) + + bugs/nouveau-bug/nouveau.log.8 still has writers, not adding +[2015-06-06 07:47:04 CEST] Committer: Adding nouveau.log.8 +add bugs/nouveau-bug/nouveau.log.8 [2015-06-06 07:47:04 CEST] Committer: Committing changes to git +[2015-06-06 07:47:04 CEST] Pusher: Syncing with origin +error: invalid object 100644 b5f693aa67b6d185e6d967e0a45c3c57a4f4b4a8 for '88c/df4/SHA256E-s4644--eca6d831c0a94895a81296327bc21bdf417234c7ebfa3e93fd21a71523ac9da9.log.6.log' +fatal: git-write-tree: error building trees +Pusher crashed: failed to read sha from git write-tree +[2015-06-06 07:47:04 CEST] Pusher: warning Pusher crashed: failed to read sha from git write-tree +/usr/lib/rsync/rrsync: SSH_ORIGINAL_COMMAND='git-annex-shell 'recvkey' '/srv/git/seb/annex/annex.git' 'SHA256E-s4491--e35540a06d6839455b48ecd8f2a4c7714d420379bae55a7b383dc1a5dc60d8e0.log.8' --uuid a2131774-31b2-47d2-9185-b2e344c16134 '--' 'remoteuuid=1dda4b9a-1ff2-4a9b-aaf8-47dfdd02c8ae' 'direct=' 'associatedfile=bugs/nouveau-bug/nouveau.log.8' '--' dummy rsync --server -pe.Lsfx --log-format=X --inplace . .' is not rsync +rsync: connection unexpectedly closed (0 bytes received so far) [sender] +rsync error: unexplained error (code 255) at io.c(226) [sender=3.1.1] +[2015-06-06 07:47:05 CEST] Committer: Committing changes to git +/usr/lib/rsync/rrsync: SSH_ORIGINAL_COMMAND='git-annex-shell 'recvkey' '/srv/git/seb/annex/annex.git' 'SHA256E-s4491--e35540a06d6839455b48ecd8f2a4c7714d420379bae55a7b383dc1a5dc60d8e0.log.8' --uuid a2131774-31b2-47d2-9185-b2e344c16134 '--' 'remoteuuid=1dda4b9a-1ff2-4a9b-aaf8-47dfdd02c8ae' 'direct=' 'associatedfile=bugs/nouveau-bug/nouveau.log.8' '--' dummy rsync --server -pe.Lsfx --log-format=X --inplace . .' is not rsync +rsync: connection unexpectedly closed (0 bytes received so far) [sender] +rsync error: unexplained error (code 255) at io.c(226) [sender=3.1.1] +/usr/lib/rsync/rrsync: SSH_ORIGINAL_COMMAND='git-annex-shell 'notifychanges' '/srv/git/seb/annex/annex.git' --uuid a2131774-31b2-47d2-9185-b2e344c16134' is not rsync +/usr/lib/rsync/rrsync: SSH_ORIGINAL_COMMAND='git-annex-shell 'notifychanges' '/srv/git/seb/annex/annex.git' --uuid/usr/lib/rsync/rrsync: SSH_ORIGINAL_COMMAND='git-annex-shell 'notifychanges' '/srv/git/seb/annex/annex.git' --uuid a2131774-31b2-47d2-9185-b2e344c16134' is not rsync +/usr/lib/rsync/rrsync: SSH_ORIGINAL_COMMAND='git-annex-shell 'notifychanges' '/srv/git/seb/annex/annex.git' --uuid a2131774-31b2-47d2-9185-b2e344c16134' is not rsync +[2015-06-06 08:45:10 CEST] NetWatcherFallback: Syncing with origin +error: invalid object 100644 b5f693aa67b6d185e6d967e0a45c3c57a4f4b4a8 for '88c/df4/SHA256E-s4644--eca6d831c0a94895a81296327bc21bdf417234c7ebfa3e93fd21a71523ac9da9.log.6.log' +fatal: git-write-tree: error building trees +NetWatcherFallback crashed: failed to read sha from git write-tree +[2015-06-06 08:45:12 CEST] NetWatcherFallback: warning NetWatcherFallback crashed: failed to read sha from git write-tree +[2015-06-06 15:58:38 CEST] main: starting assistant version 5.20150508-g883d57f +[2015-06-06 15:58:38 CEST] Cronner: You should enable consistency checking to protect your data. +[2015-06-06 15:58:44 CEST] TransferScanner: Syncing with origin + + No known network monitor available through dbus; falling back to polling +(scanning...) [2015-06-06 15:58:44 CEST] Watcher: Performing startup scan +(started...) gpg: assuming signed data in '/tmp/git-annex.tmp.0/info' +gpg: Signature made Thu May 28 19:01:58 2015 CEST using DSA key ID 89C809CB +gpg: /tmp/git-annex-gpg.tmp.0/trustdb.gpg: trustdb created +gpg: Good signature from "git-annex distribution signing key (for Joey Hess) " [unknown] +gpg: WARNING: This key is not certified with a trusted signature! +gpg: There is no indication that the signature belongs to the owner. +Primary key fingerprint: 4005 5C6A FD2D 526B 2961 E78F 5EE1 DBA7 89C8 09CB +[2015-06-06 15:58:45 CEST] Upgrader: An upgrade of git-annex is available. (version 5.20150528) +error: invalid object 100644 b5f693aa67b6d185e6d967e0a45c3c57a4f4b4a8 for '88c/df4/SHA256E-s4644--eca6d831c0a94895a81296327bc21bdf417234c7ebfa3e93fd21a71523ac9da9.log.6.log' +fatal: git-write-tree: error building trees +TransferScanner crashed: failed to read sha from git write-tree +[2015-06-06 15:58:45 CEST] TransferScanner: warning TransferScanner crashed: failed to read sha from git write-tree +[2015-06-06 15:58:46 CEST] Committer: Committing changes to git +[2015-06-06 15:58:46 CEST] Pusher: Syncing with origin +error: invalid object 100644 b5f693aa67b6d185e6d967e0a45c3c57a4f4b4a8 for '88c/df4/SHA256E-s4644--eca6d831c0a94895a81296327bc21bdf417234c7ebfa3e93fd21a71523ac9da9.log.6.log' +fatal: git-write-tree: error building trees +Pusher crashed: failed to read sha from git write-tree +[2015-06-06 15:58:46 CEST] Pusher: warning Pusher crashed: failed to read sha from git write-tree +[2015-06-06 16:58:44 CEST] NetWatcherFallback: Syncing with origin +error: invalid object 100644 b5f693aa67b6d185e6d967e0a45c3c57a4f4b4a8 for '88c/df4/SHA256E-s4644--eca6d831c0a94895a81296327bc21bdf417234c7ebfa3e93fd21a71523ac9da9.log.6.log' +fatal: git-write-tree: error building trees +NetWatcherFallback crashed: failed to read sha from git write-tree +[2015-06-06 16:58:47 CEST] NetWatcherFallback: warning NetWatcherFallback crashed: failed to read sha from git write-tree +[2015-06-06 18:10:24 CEST] Committer: Committing changes to git +[2015-06-06 18:10:25 CEST] Committer: Committing changes to git +[2015-06-06 20:01:34 CEST] Committer: Committing changes to git +[2015-06-06 20:01:35 CEST] Committer: Committing changes to git +[2015-06-06 22:42:41 CEST] main: starting assistant version 5.20150508-g883d57f +[2015-06-06 22:42:41 CEST] Cronner: You should enable consistency checking to protect your data. +[2015-06-06 22:42:47 CEST] TransferScanner: Syncing with origin + + No known network monitor available through dbus; falling back to polling +(scanning...) [2015-06-06 22:42:47 CEST] Watcher: Performing startup scan +(started...) error: invalid object 100644 b5f693aa67b6d185e6d967e0a45c3c57a4f4b4a8 for '88c/df4/SHA256E-s4644--eca6d831c0a94895a81296327bc21bdf417234c7ebfa3e93fd21a71523ac9da9.log.6.log' +fatal: git-write-tree: error building trees +TransferScanner crashed: failed to read sha from git write-tree +[2015-06-06 22:42:48 CEST] TransferScanner: warning TransferScanner crashed: failed to read sha from git write-tree +gpg: assuming signed data in '/tmp/git-annex.tmp.0/info' +gpg: Signature made Thu May 28 19:01:58 2015 CEST using DSA key ID 89C809CB +gpg: /tmp/git-annex-gpg.tmp.0/trustdb.gpg: trustdb created +gpg: Good signature from "git-annex distribution signing key (for Joey Hess) " [unknown] +gpg: WARNING: This key is not certified with a trusted signature! +gpg: There is no indication that the signature belongs to the owner. +Primary key fingerprint: 4005 5C6A FD2D 526B 2961 E78F 5EE1 DBA7 89C8 09CB +[2015-06-06 22:42:48 CEST] Upgrader: An upgrade of git-annex is available. (version 5.20150528) +[2015-06-06 22:42:49 CEST] Committer: Committing changes to git +[2015-06-06 22:42:49 CEST] Pusher: Syncing with origin +error: invalid object 100644 b5f693aa67b6d185e6d967e0a45c3c57a4f4b4a8 for '88c/df4/SHA256E-s4644--eca6d831c0a94895a81296327bc21bdf417234c7ebfa3e93fd21a71523ac9da9.log.6.log' +fatal: git-write-tree: error building trees +Pusher crashed: failed to read sha from git write-tree +[2015-06-06 22:42:49 CEST] Pusher: warning Pusher crashed: failed to read sha from git write-tree + +(recording state in git...) +(recording state in git...) +(recording state in git...) + + bugs/nouveau-bug/nouveau.log.9 still has writers, not adding +[2015-06-06 22:43:47 CEST] Committer: Adding nouveau.log.9 +add bugs/nouveau-bug/nouveau.log.9 [2015-06-06 22:43:48 CEST] Committer: Committing changes to git + +SHA256E-s4427--4e02ba661d0106ff4715a2cfda0fe39185865551bcb484539e618f2b64f9ec0c.log.9 + + 4,427 100% 0.00kB/s 0:00:00 + 4,427 100% 0.00kB/s 0:00:00 (xfr#1, to-chk=0/1) +[2015-06-06 22:43:48 CEST] Transferrer: Uploaded nouveau.log.9 +[2015-06-06 22:43:49 CEST] Committer: Committing changes to git +ok +(recording state in git...) + + lost dbus connection; falling back to polling (ClientError {clientErrorMessage = "Error reading message from socket: UnmarshalError \"Unexpected end of input while parsing message header.\"", clientErrorFatal = True}) +[2015-06-07 07:41:35 CEST] main: starting assistant version 5.20150508-g883d57f +[2015-06-07 07:41:35 CEST] Cronner: You should enable consistency checking to protect your data. +[2015-06-07 07:41:42 CEST] TransferScanner: Syncing with origin + + No known network monitor available through dbus; falling back to polling +(scanning...) [2015-06-07 07:41:42 CEST] Watcher: Performing startup scan +(started...) error: invalid object 100644 b5f693aa67b6d185e6d967e0a45c3c57a4f4b4a8 for '88c/df4/SHA256E-s4644--eca6d831c0a94895a81296327bc21bdf417234c7ebfa3e93fd21a71523ac9da9.log.6.log' +fatal: git-write-tree: error building trees +TransferScanner crashed: failed to read sha from git write-tree +[2015-06-07 07:41:43 CEST] TransferScanner: warning TransferScanner crashed: failed to read sha from git write-tree +gpg: assuming signed data in '/tmp/git-annex.tmp.0/info' +gpg: Signature made Thu May 28 19:01:58 2015 CEST using DSA key ID 89C809CB +gpg: /tmp/git-annex-gpg.tmp.0/trustdb.gpg: trustdb created +gpg: Good signature from "git-annex distribution signing key (for Joey Hess) " [unknown] +gpg: WARNING: This key is not certified with a trusted signature! +gpg: There is no indication that the signature belongs to the owner. +Primary key fingerprint: 4005 5C6A FD2D 526B 2961 E78F 5EE1 DBA7 89C8 09CB +[2015-06-07 07:41:43 CEST] Upgrader: An upgrade of git-annex is available. (version 5.20150528) +[2015-06-07 07:41:44 CEST] Committer: Committing changes to git +[2015-06-07 07:41:44 CEST] Pusher: Syncing with origin +error: invalid object 100644 b5f693aa67b6d185e6d967e0a45c3c57a4f4b4a8 for '88c/df4/SHA256E-s4644--eca6d831c0a94895a81296327bc21bdf417234c7ebfa3e93fd21a71523ac9da9.log.6.log' +fatal: git-write-tree: error building trees +Pusher crashed: failed to read sha from git write-tree +[2015-06-07 07:41:44 CEST] Pusher: warning Pusher crashed: failed to read sha from git write-tree +[2015-06-07 08:41:42 CEST] NetWatcherFallback: Syncing with origin +error: invalid object 100644 b5f693aa67b6d185e6d967e0a45c3c57a4f4b4a8 for '88c/df4/SHA256E-s4644--eca6d831c0a94895a81296327bc21bdf417234c7ebfa3e93fd21a71523ac9da9.log.6.log' +fatal: git-write-tree: error building trees +NetWatcherFallback crashed: failed to read sha from git write-tree +[2015-06-07 08:41:43 CEST] NetWatcherFallback: warning NetWatcherFallback crashed: failed to read sha from git write-tree +[2015-06-07 09:21:46 CEST] main: starting assistant version 5.20150508-g883d57f +[2015-06-07 09:21:46 CEST] Cronner: You should enable consistency checking to protect your data. +[2015-06-07 09:21:49 CEST] main: starting assistant version 5.20150508-g883d57f +[2015-06-07 09:21:49 CEST] Cronner: You should enable consistency checking to protect your data. +[2015-06-07 09:21:54 CEST] TransferScanner: Syncing with origin + + No known network monitor available through dbus; falling back to polling +(scanning...) [2015-06-07 09:21:55 CEST] Watcher: Performing startup scan +(started...) +(recording state in git...) +error: invalid object 100644 b5f693aa67b6d185e6d967e0a45c3c57a4f4b4a8 for '88c/df4/SHA256E-s4644--eca6d831c0a94895a81296327bc21bdf417234c7ebfa3e93fd21a71523ac9da9.log.6.log' +fatal: git-write-tree: error building trees +TransferScanner crashed: failed to read sha from git write-tree +[2015-06-07 09:21:55 CEST] TransferScanner: warning TransferScanner crashed: failed to read sha from git write-tree +gpg: assuming signed data in '/tmp/git-annex.tmp.0/info' +gpg: Signature made Thu May 28 19:01:58 2015 CEST using DSA key ID 89C809CB +gpg: /tmp/git-annex-gpg.tmp.0/trustdb.gpg: trustdb created +gpg: Good signature from "git-annex distribution signing key (for Joey Hess) " [unknown] +gpg: WARNING: This key is not certified with a trusted signature! +gpg: There is no indication that the signature belongs to the owner. +Primary key fingerprint: 4005 5C6A FD2D 526B 2961 E78F 5EE1 DBA7 89C8 09CB +[2015-06-07 09:21:56 CEST] Upgrader: An upgrade of git-annex is available. (version 5.20150528) +[2015-06-07 09:21:56 CEST] Committer: Committing changes to git +(recording state in git...) +[2015-06-07 09:21:56 CEST] Pusher: Syncing with origin +(recording state in git...) +error: invalid object 100644 b5f693aa67b6d185e6d967e0a45c3c57a4f4b4a8 for '88c/df4/SHA256E-s4644--eca6d831c0a94895a81296327bc21bdf417234c7ebfa3e93fd21a71523ac9da9.log.6.log' +fatal: git-write-tree: error building trees +Pusher crashed: failed to read sha from git write-tree +[2015-06-07 09:21:57 CEST] Pusher: warning Pusher crashed: failed to read sha from git write-tree +[2015-06-07 09:23:07 CEST] main: starting assistant version 5.20150508-g883d57f +[2015-06-07 09:23:07 CEST] Cronner: You should enable consistency checking to protect your data. +[2015-06-07 09:23:12 CEST] TransferScanner: Syncing with origin + + No known network monitor available through dbus; falling back to polling +(scanning...) [2015-06-07 09:23:12 CEST] Watcher: Performing startup scan +(started...) +(recording state in git...) +gpg: assuming signed data in '/tmp/git-annex.tmp.0/info' +gpg: Signature made Thu May 28 19:01:58 2015 CEST using DSA key ID 89C809CB +gpg: /tmp/git-annex-gpg.tmp.0/trustdb.gpg: trustdb created +gpg: Good signature from "git-annex distribution signing key (for Joey Hess) " [unknown] +gpg: WARNING: This key is not certified with a trusted signature! +gpg: There is no indication that the signature belongs to the owner. +Primary key fingerprint: 4005 5C6A FD2D 526B 2961 E78F 5EE1 DBA7 89C8 09CB +[2015-06-07 09:23:13 CEST] Upgrader: An upgrade of git-annex is available. (version 5.20150528) +[2015-06-07 09:23:14 CEST] Committer: Committing changes to git +(recording state in git...) +[2015-06-07 09:23:14 CEST] Pusher: Syncing with origin + +SHA256E-s4491--e35540a06d6839455b48ecd8f2a4c7714d420379bae55a7b383dc1a5dc60d8e0.log.8 + + 4,491 100% 0.00kB/s 0:00:00 + 4,491 100% 0.00kB/s 0:00:00 (xfr#1, to-chk=0/1) +[2015-06-07 09:23:15 CEST] Transferrer: Uploaded nouveau.log.8 +error: Unable to create '/srv/git/seb/annex/annex.git/refs/heads/synced/git-annex.lock': File exists. + +If no other git process is currently running, this probably means a +git process crashed in this repository earlier. Make sure no other git +process is running and remove the file manually to continue. +remote: error: Cannot lock the ref 'refs/heads/synced/git-annex'. +error: Unable to create '/srv/git/seb/annex/annex.git/refs/heads/synced/master.lock': File exists. + +If no other git process is currently running, this probably means a +git process crashed in this repository earlier. Make sure no other git +process is running and remove the file manually to continue. +remote: error: Cannot lock the ref 'refs/heads/synced/master'. +To ssh://redacted.net/srv/git/seb/annex/annex.git + aec4d80..5cac670 master -> synced/master + ! [remote rejected] git-annex -> synced/git-annex (failed to update ref) +To ssh://redacted.net/srv/git/seb/annex/annex.git + 973fbda..118a888 git-annex -> synced/git-annex + ! [remote rejected] master -> synced/master (failed to update ref) +error: failed to push some refs to 'ssh://redacted.net/srv/git/seb/annex/annex.git' +error: failed to push some refs to 'ssh://redacted.net/srv/git/seb/annex/annex.git' +error: Ref refs/remotes/origin/master is at 5cac670bc3d15a2d1a976ab02b2717078717ad48 but expected aec4d80548993b2b6e126124153fab9c93de8375 +error: Cannot lock the ref 'refs/remotes/origin/master'. +From ssh://redacted.net/srv/git/seb/annex/annex + ! aec4d80..5cac670 master -> origin/master (unable to update local ref) +error: Ref refs/remotes/origin/git-annex is at 118a888e0244e644c2ec6c0bf100e787d25a222d but expected 973fbdad6d8b549c4db0633ef11fa5a8f9ec0ed1 +error: Cannot lock the ref 'refs/remotes/origin/git-annex'. + ! 973fbda..118a888 git-annex -> origin/git-annex (unable to update local ref) +Everything up-to-date +Everything up-to-date +(recording state in git...) +[2015-06-07 09:23:20 CEST] main: starting assistant version 5.20150508-g883d57f +[2015-06-07 09:23:20 CEST] Cronner: You should enable consistency checking to protect your data. +[2015-06-07 09:23:25 CEST] TransferScanner: Syncing with origin + + No known network monitor available through dbus; falling back to polling +(scanning...) [2015-06-07 09:23:25 CEST] Watcher: Performing startup scan +(started...) +From ssh://redacted.net/srv/git/seb/annex/annex + 118a888..e058e35 git-annex -> origin/git-annex +(merging origin/git-annex into git-annex...) +(recording state in git...) +gpg: assuming signed data in '/tmp/git-annex.tmp.0/info' +gpg: Signature made Thu May 28 19:01:58 2015 CEST using DSA key ID 89C809CB +gpg: /tmp/git-annex-gpg.tmp.0/trustdb.gpg: trustdb created +gpg: Good signature from "git-annex distribution signing key (for Joey Hess) " [unknown] +gpg: WARNING: This key is not certified with a trusted signature! +gpg: There is no indication that the signature belongs to the owner. +Primary key fingerprint: 4005 5C6A FD2D 526B 2961 E78F 5EE1 DBA7 89C8 09CB +[2015-06-07 09:23:26 CEST] Upgrader: An upgrade of git-annex is available. (version 5.20150528) +To ssh://redacted.net/srv/git/seb/annex/annex.git + 118a888..37bc288 git-annex -> synced/git-annex +[2015-06-07 09:23:27 CEST] Committer: Committing changes to git +(recording state in git...) +[2015-06-07 09:23:27 CEST] Pusher: Syncing with origin +Everything up-to-date +[2015-06-07 09:25:21 CEST] Committer: Adding daemon.log.4 +add bugs/git-annex/daemon.log.4 ok +[2015-06-07 09:25:21 CEST] Committer: Committing changes to git +(recording state in git...) +[2015-06-07 09:25:21 CEST] Pusher: Syncing with origin +(recording state in git...) + +SHA256E-s2383--66aa385819f9ba961c3b4701dd68b6d1450af08ae24428461d342be8b12784a2.log.4 + + 2,383 100% 0.00kB/s 0:00:00 + 2,383 100% 0.00kB/s 0:00:00 (xfr#1, to-chk=0/1) +[2015-06-07 09:25:21 CEST] Transferrer: Uploaded daemon.log.4 +To ssh://redacted.net/srv/git/seb/annex/annex.git + 37bc288..5c63092 git-annex -> synced/git-annex + 5cac670..3a5e7d0 master -> synced/master +[2015-06-07 09:25:22 CEST] Committer: Committing changes to git +[2015-06-07 09:25:24 CEST] Pusher: Syncing with origin +(recording state in git...) +To ssh://redacted.net/srv/git/seb/annex/annex.git + 5c63092..07a3e47 git-annex -> synced/git-annex +[2015-06-07 09:25:40 CEST] Committer: Adding index~ +add bugs/git-annex/index~ ok +[2015-06-07 09:25:40 CEST] Committer: Committing changes to git +(recording state in git...) +[2015-06-07 09:25:40 CEST] Pusher: Syncing with origin +(recording state in git...) + +SHA256E-s178074--cf093bacc4367a5e86a07cb8523b183dd0a5401805dde646cf0b2b0ddc8ee282 + + 32,768 18% 0.00kB/s 0:00:00 + 178,074 100% 69.29MB/s 0:00:00 (xfr#1, to-chk=0/1) +[2015-06-07 09:25:41 CEST] Committer: Committing changes to git +[2015-06-07 09:25:41 CEST] Transferrer: Uploaded index~ +To ssh://redacted.net/srv/git/seb/annex/annex.git + 07a3e47..32d41fc git-annex -> synced/git-annex + 3a5e7d0..20e6e3c master -> synced/master +[2015-06-07 09:25:44 CEST] Pusher: Syncing with origin +(recording state in git...) +To ssh://redacted.net/srv/git/seb/annex/annex.git + 32d41fc..b61b73e git-annex -> synced/git-annex +[2015-06-07 09:44:47 CEST] Committer: Adding daemon.log daemon.log.1 daemon.log.10 daemon.log.2 daemon.log.3 +add bugs/git-annex/daemon.log ok +add bugs/git-annex/daemon.log.1 ok +add bugs/git-annex/daemon.log.10 ok +add bugs/git-annex/daemon.log.2 ok +add bugs/git-annex/daemon.log.3 ok +[2015-06-07 09:44:47 CEST] Committer: Committing changes to git +(recording state in git...) +[2015-06-07 09:44:47 CEST] Pusher: Syncing with origin +(recording state in git...) + +SHA256E-s3284--5784c99b359a5fdfb52f26e82165ffba54d62f9f5cd846fce25a3c6f694eb890.log + + 3,284 100% 0.00kB/s 0:00:00 + 3,284 100% 0.00kB/s 0:00:00 (xfr#1, to-chk=0/1) +[2015-06-07 09:44:47 CEST] Transferrer: Uploaded daemon.log + +SHA256E-s178--704e1850cab9a6b074ba2a77344199b866e9408541b83bf1ad6b3bbd23dfc54a.log.3 + + 178 100% 0.00kB/s 0:00:00 + 178 100% 0.00kB/s 0:00:00 (xfr#1, to-chk=0/1) +[2015-06-07 09:44:48 CEST] Transferrer: Uploaded daemon.log.3 +To ssh://redacted.net/srv/git/seb/annex/annex.git + b61b73e..854cbda git-annex -> synced/git-annex + 20e6e3c..0d8b74a master -> synced/master + +SHA256E-s1993--7f1bec5289ebe07306bf57fb91e0dec25b1d8f5cf1efd15297815503f1290477.log.2 + + 1,993 100% 0.00kB/s 0:00:00 + 1,993 100% 0.00kB/s 0:00:00 (xfr#1, to-chk=0/1) +[2015-06-07 09:44:48 CEST] Committer: Committing changes to git +(recording state in git...) +[2015-06-07 09:44:48 CEST] Transferrer: Uploaded daemon.log.2 + +SHA256E-s4969--ff59d2458e0b43a1965258bd0064626de1a0b7f911de693a02f34fc1bcb4959a.log.10 + + 4,969 100% 0.00kB/s 0:00:00 + 4,969 100% 0.00kB/s 0:00:00 (xfr#1, to-chk=0/1) +[2015-06-07 09:44:48 CEST] Transferrer: Uploaded daemon.log.10 + +SHA256E-s3385--0e7aef46d502f1d3a583b642239cf1272b2410d5976603b65a104fc866c02e70.log.1 + + 3,385 100% 0.00kB/s 0:00:00 + 3,385 100% 0.00kB/s 0:00:00 (xfr#1, to-chk=0/1) +[2015-06-07 09:44:49 CEST] Transferrer: Uploaded daemon.log.1 +[2015-06-07 09:44:50 CEST] Committer: Adding daemon.log.5 daemon.log.6 daemon.log.7 daemon.log.8 daemon.log.9 +add bugs/git-annex/daemon.log.5 ok +add bugs/git-annex/daemon.log.6 ok +add bugs/git-annex/daemon.log.7 ok +add bugs/git-annex/daemon.log.8 ok +add bugs/git-annex/daemon.log.9 ok +[2015-06-07 09:44:50 CEST] Committer: Committing changes to git +(recording state in git...) + +SHA256E-s2792--313d72ecb598b18884602babaed667b71786a0f50edd291754bac0b70c666fa9.log.5 + + 2,792 100% 0.00kB/s 0:00:00 + 2,792 100% 0.00kB/s 0:00:00 (xfr#1, to-chk=0/1) +[2015-06-07 09:44:50 CEST] Transferrer: Uploaded daemon.log.5 + +SHA256E-s6792--8aaddbb316719160b93305bc7a6698df1b2d6b0e6f3366c0fa9f7d258cf12303.log.9 + + 6,792 100% 0.00kB/s 0:00:00 + 6,792 100% 0.00kB/s 0:00:00 (xfr#1, to-chk=0/1) +[2015-06-07 09:44:50 CEST] Transferrer: Uploaded daemon.log.9 +[2015-06-07 09:44:50 CEST] Pusher: Syncing with origin + +SHA256E-s1183--12703bbab3cd4db78fc4449b5c3dddfb82da77c9b4a750d2b7a902f3eb63547b.log.8 + + 1,183 100% 0.00kB/s 0:00:00 + 1,183 100% 0.00kB/s 0:00:00 (xfr#1, to-chk=0/1) +(recording state in git...) +[2015-06-07 09:44:51 CEST] Transferrer: Uploaded daemon.log.8 +[2015-06-07 09:44:51 CEST] Committer: Committing changes to git +(recording state in git...) + +SHA256E-s5310--51d80ffdcfdeff4e035f740de0f18afa660404fb448a72421bd7af85ae87ae6f.log.7 + + 5,310 100% 0.00kB/s 0:00:00 + 5,310 100% 0.00kB/s 0:00:00 (xfr#1, to-chk=0/1) +[2015-06-07 09:44:51 CEST] Transferrer: Uploaded daemon.log.7 + +SHA256E-s2639--c7a0e874ae2dbc874732c000928a5c2a1a6c3d75dcb89b590fcb9e13415427cd.log.6 + + 2,639 100% 0.00kB/s 0:00:00 + 2,639 100% 0.00kB/s 0:00:00 (xfr#1, to-chk=0/1) +[2015-06-07 09:44:51 CEST] Transferrer: Uploaded daemon.log.6 +To ssh://redacted.net/srv/git/seb/annex/annex.git + 854cbda..c940c96 git-annex -> synced/git-annex + 0d8b74a..3865343 master -> synced/master +[2015-06-07 09:44:54 CEST] Pusher: Syncing with origin +(recording state in git...) +To ssh://redacted.net/srv/git/seb/annex/annex.git + c940c96..95a399f git-annex -> synced/git-annex + 3865343..c04efc3 master -> synced/master + +"""]] diff --git a/doc/bugs/Corrupted_.git__47__annex__47__index_when_running_assistant/comment_1_f2102da5822a35e1600d67326954e059._comment b/doc/bugs/Corrupted_.git__47__annex__47__index_when_running_assistant/comment_1_f2102da5822a35e1600d67326954e059._comment new file mode 100644 index 0000000000..37a50c662f --- /dev/null +++ b/doc/bugs/Corrupted_.git__47__annex__47__index_when_running_assistant/comment_1_f2102da5822a35e1600d67326954e059._comment @@ -0,0 +1,7 @@ +[[!comment format=mdwn + username="eigengrau" + subject="comment 1" + date="2015-06-07T08:40:05Z" + content=""" +Hm, I wondered why there’s the “rrsync” stuff in the logs, because git-annex doesn’t use rrsync, right? I noticed I still had an old SSH key with a forced-command for rrsync in my authorized_keys on the remote. Could this have played into the problem, or would that have been dealt with gracefully? +"""]] diff --git a/doc/bugs/Corrupted_.git__47__annex__47__index_when_running_assistant/comment_2_656986cdc4831a8d2f80432acaf8a884._comment b/doc/bugs/Corrupted_.git__47__annex__47__index_when_running_assistant/comment_2_656986cdc4831a8d2f80432acaf8a884._comment new file mode 100644 index 0000000000..4346bebee9 --- /dev/null +++ b/doc/bugs/Corrupted_.git__47__annex__47__index_when_running_assistant/comment_2_656986cdc4831a8d2f80432acaf8a884._comment @@ -0,0 +1,25 @@ +[[!comment format=mdwn + username="joey" + subject="""comment 2""" + date="2015-06-09T19:34:57Z" + content=""" +So, thanks for providing me the index file, but the relevant problem +seems to be that your index file refers to a git object that failed to make +it to disk. + +The most likely cause of this would be a crash before disk buffers got +flushed, or something like that. + +It's perfectly fine to remove your .git/annex/index if this happens. You +must have lost some information that was being committed to the git-annex +branch, so you might want to run `git annex fsck` to make sure everything +gets updated. + +I'd expect `git-annex repair` to detect this problem (and fix it the same +way you did). + +It's possible that the assistant's code for detecting when there are +problems doesn't work well enough for it to have +automatically detected and fixed this problem. Which is why I'm leaving +this bug open for now. +"""]] diff --git a/doc/bugs/Corrupted_.git__47__annex__47__index_when_running_assistant/comment_3_fa94d1b008a6c1663e40854edcd5504d._comment b/doc/bugs/Corrupted_.git__47__annex__47__index_when_running_assistant/comment_3_fa94d1b008a6c1663e40854edcd5504d._comment new file mode 100644 index 0000000000..6fbc820f8d --- /dev/null +++ b/doc/bugs/Corrupted_.git__47__annex__47__index_when_running_assistant/comment_3_fa94d1b008a6c1663e40854edcd5504d._comment @@ -0,0 +1,9 @@ +[[!comment format=mdwn + username="eigengrau" + subject="comment 3" + date="2015-06-11T15:12:23Z" + content=""" +Thanks! FWIW I didn’t have any hard kernel lockups recently. I figure git replaces the index file atomically, and only after all objects have been written to the store? I guess a userland crash couldn’t be the cause either, in that case? + +What happens when you manually run git gc at an inopportune moment, seeing that it probably doesn’t know about the secondary index? In the logs, I saw mention of locks on individual refs. Is the whole repository also locked down when git-annex commits something, or could it happen that a manual git gc prunes away objects added by git-annex before it had a chance to write the tree and commit it to a ref? +"""]] diff --git a/doc/bugs/Corrupted_.git__47__annex__47__index_when_running_assistant/comment_4_5a26a33c34594ec6eb50bd675d68801a._comment b/doc/bugs/Corrupted_.git__47__annex__47__index_when_running_assistant/comment_4_5a26a33c34594ec6eb50bd675d68801a._comment new file mode 100644 index 0000000000..4b41d988d5 --- /dev/null +++ b/doc/bugs/Corrupted_.git__47__annex__47__index_when_running_assistant/comment_4_5a26a33c34594ec6eb50bd675d68801a._comment @@ -0,0 +1,17 @@ +[[!comment format=mdwn + username="joey" + subject="""comment 4""" + date="2015-06-15T18:05:38Z" + content=""" +git does update the index file atomically (via `rename(2)`) but +that doesn't necessarily mean that the filesystem writes all +the data to disk before there is a problem. + +That's a good point about `git gc` and the index; it does look +at the regular git index. But, `git gc` seems unlikely to cause any problem, +even if .git/annex/index pointed to a newer object that the git-annex branch +hadn't yet been updated to refer to; since `git gc` defaults to only +pruning objects that are 2 weeks old. The time that an object would +normally be in .git/annex/index without getting flushed in a commit +is normally measured in seconds to minutes. +"""]] diff --git a/doc/bugs/Corrupted_.git__47__annex__47__index_when_running_assistant/comment_5_81e2f37e7adbd8f24734b67a1dd209f9._comment b/doc/bugs/Corrupted_.git__47__annex__47__index_when_running_assistant/comment_5_81e2f37e7adbd8f24734b67a1dd209f9._comment new file mode 100644 index 0000000000..71d7a7338a --- /dev/null +++ b/doc/bugs/Corrupted_.git__47__annex__47__index_when_running_assistant/comment_5_81e2f37e7adbd8f24734b67a1dd209f9._comment @@ -0,0 +1,7 @@ +[[!comment format=mdwn + username="eigengrau" + subject="comment 5" + date="2015-06-16T13:20:07Z" + content=""" +You’re right, but the problem might be my somehow developing a perhaps overzealous habit of using the occasional `--aggressive --prune=now`. From what you said I would surmise that this could be a fairly good explanation for the missing object in my case. +"""]] diff --git a/doc/bugs/Corrupted_.git__47__annex__47__index_when_running_assistant/comment_6_f0999dbd00e06ab2cbf68b58527c2b9b._comment b/doc/bugs/Corrupted_.git__47__annex__47__index_when_running_assistant/comment_6_f0999dbd00e06ab2cbf68b58527c2b9b._comment new file mode 100644 index 0000000000..f11e80bbec --- /dev/null +++ b/doc/bugs/Corrupted_.git__47__annex__47__index_when_running_assistant/comment_6_f0999dbd00e06ab2cbf68b58527c2b9b._comment @@ -0,0 +1,16 @@ +[[!comment format=mdwn + username="joey" + subject="""comment 6""" + date="2015-07-02T19:14:40Z" + content=""" +Yeah, --prune=now is bad news when using git-annex. At least, if +you might be doing it at the same time that git-annex is staging +objects in the index. + +Using it in combination with annex.alwayscommit=false is a particularly +foot-shootish thing. I've added a warning to its documentation. + +`git gc` uses a lock file, `.git/gc.pid`. It might be possible for +git-annex to hold that lock when it's in the process of staging and +committing objects, to avoid `git gc` running at such times. +"""]] diff --git a/doc/bugs/Corrupted_drive__58___Assistant_seems_consider_files_deleted_and_deletes_them_elsewhere_too.mdwn b/doc/bugs/Corrupted_drive__58___Assistant_seems_consider_files_deleted_and_deletes_them_elsewhere_too.mdwn new file mode 100644 index 0000000000..595b86c296 --- /dev/null +++ b/doc/bugs/Corrupted_drive__58___Assistant_seems_consider_files_deleted_and_deletes_them_elsewhere_too.mdwn @@ -0,0 +1,36 @@ +### Please describe the problem. + +On my raspberry pi, an SSH remote was in /var/lib/store (symlinked with /home/carlo/store), which contained a LVM volume spanning several USB drives. One of the drives got corrupted, which lead to this: + +$ cd /home/carlo/store +$ ls +Input/Output Error + +On my desktop machine, I then had a lot of transfers. After transfer, the file was turned into a symlink. + +### What steps will reproduce the problem? + +* Create an annex and a remote ssh server. +* [Simulate a corrupted drive](http://stackoverflow.com/questions/1361518/how-can-i-simulate-a-failed-disk-during-testing) for the remote, creating an input output error. +* Wait until the annex starts syncing on the desktop machine. If there were no further unidentified causes on my side, the assistant will start transfers that revert files to symlinks. + +### What version of git-annex are you using? On what operating system? + +Ubuntu 12.04 64bit +git annex 3.20121017 ([from Ubuntu PPA](https://launchpad.net/~rubiojr/+archive/git-annex)) + +### Please provide any additional information below. + +[[!format sh """ +# If you can, paste a complete transcript of the problem occurring here. +# If the problem is with the git-annex assistant, paste in .git/annex/daemon.log + +Please see [all my daemon.log](http://capocasa.name/daemon.tgz) files. + +I noticed the problem yesterday afternoon (Thu 24 Oct). + +# End of transcript or log. +"""]] + +> [[!taglink moreinfo]]; either I don't have enough information to work on this, +> or it might have just been user error. --[[Joey]] diff --git a/doc/bugs/Corrupted_drive__58___Assistant_seems_consider_files_deleted_and_deletes_them_elsewhere_too/comment_1_80ca50f5305eda71fe32f2b0bc922c34._comment b/doc/bugs/Corrupted_drive__58___Assistant_seems_consider_files_deleted_and_deletes_them_elsewhere_too/comment_1_80ca50f5305eda71fe32f2b0bc922c34._comment new file mode 100644 index 0000000000..e47cbd327f --- /dev/null +++ b/doc/bugs/Corrupted_drive__58___Assistant_seems_consider_files_deleted_and_deletes_them_elsewhere_too/comment_1_80ca50f5305eda71fe32f2b0bc922c34._comment @@ -0,0 +1,21 @@ +[[!comment format=mdwn + username="http://joeyh.name/" + ip="209.250.56.47" + subject="comment 1" + date="2013-10-26T19:22:57Z" + content=""" +It seems to me that if a subdirectory of the repository is on a corrupted drive, and so it's not possible to list the files on it, this is basically the same as if you'd 'rm -rf' that subdirectory. So when it starts up, the assistant will see that these files are not present, and commit a removal to git. + +Then when another machine syncs with that, it would delete the files from its repository too. However, it actually keeps the contents of the files stashed away in `.git/annex`. So to recover from this, all you have to do is `git annex indirect` and `git revert` the commit that deleted the files. All your files would then be available again. + +However, what you describe is instead that the assistant chose to drop the content associated with the files, but kept the symlinks for them checked into git. +I don't understand why it would do that. Can you show the output of running, on the desktop machine: + + git annex whereis $somefile + git annex get $somefile + +Where $somefile is one of the files that has been reduced to a symlink. + +Looking at your logs, they appear to be the logs from the server. The strange thing that appears in one of them is \"git-annex: Not in a git repository.\" +which was logged around 2013-10-24 20:07:25 CEST. I am not sure, but I think it might have been the rpi git-annex saying that, because there is also \"fatal: '~/store/annex/' does not appear to be a git repository\" +"""]] diff --git a/doc/bugs/Corrupted_drive__58___Assistant_seems_consider_files_deleted_and_deletes_them_elsewhere_too/comment_2_e6bc6d1c0eb8c469e9e00b37bbcc9b86._comment b/doc/bugs/Corrupted_drive__58___Assistant_seems_consider_files_deleted_and_deletes_them_elsewhere_too/comment_2_e6bc6d1c0eb8c469e9e00b37bbcc9b86._comment new file mode 100644 index 0000000000..18959ed682 --- /dev/null +++ b/doc/bugs/Corrupted_drive__58___Assistant_seems_consider_files_deleted_and_deletes_them_elsewhere_too/comment_2_e6bc6d1c0eb8c469e9e00b37bbcc9b86._comment @@ -0,0 +1,9 @@ +[[!comment format=mdwn + username="https://www.google.com/accounts/o8/id?id=AItOawnZEanlyzay_QlEAL0CWpyZcRTyN7vay8U" + nickname="Carlo" + subject="comment 2" + date="2013-10-27T19:26:42Z" + content=""" +Thanks for looking into it and the recover instructions, will recover and post back when I'm back at work. I also have an another indirect mode repo on my home desktop as an extra backup so nothing real bad happened, but it's reassuring data's still there at work. + +"""]] diff --git a/doc/bugs/Corrupted_drive__58___Assistant_seems_consider_files_deleted_and_deletes_them_elsewhere_too/comment_5_0d0f6b6b46d0153433fead2bbd1bbe64._comment b/doc/bugs/Corrupted_drive__58___Assistant_seems_consider_files_deleted_and_deletes_them_elsewhere_too/comment_5_0d0f6b6b46d0153433fead2bbd1bbe64._comment new file mode 100644 index 0000000000..a254b31a82 --- /dev/null +++ b/doc/bugs/Corrupted_drive__58___Assistant_seems_consider_files_deleted_and_deletes_them_elsewhere_too/comment_5_0d0f6b6b46d0153433fead2bbd1bbe64._comment @@ -0,0 +1,12 @@ +[[!comment format=mdwn + username="https://www.google.com/accounts/o8/id?id=AItOawnZEanlyzay_QlEAL0CWpyZcRTyN7vay8U" + nickname="Carlo" + subject="comment 5" + date="2013-10-30T15:05:37Z" + content=""" +The assistant autorecovered my work repo before I noticed, so it looks like I can't provide the necessary info. There were a bunch of files missing that got re-synced from my home PC. + +For what it's worth, I noticed that on my phone, when cutting the internet connection while syncing, the assistant downloaded existing files into placeholder files, and then continued actually downloading files when the network connection was restored. + + +"""]] diff --git a/doc/bugs/Corrupted_drive__58___Assistant_seems_consider_files_deleted_and_deletes_them_elsewhere_too/comment_5_6058a22b733cb02126286af950074ed4._comment b/doc/bugs/Corrupted_drive__58___Assistant_seems_consider_files_deleted_and_deletes_them_elsewhere_too/comment_5_6058a22b733cb02126286af950074ed4._comment new file mode 100644 index 0000000000..33e85c7724 --- /dev/null +++ b/doc/bugs/Corrupted_drive__58___Assistant_seems_consider_files_deleted_and_deletes_them_elsewhere_too/comment_5_6058a22b733cb02126286af950074ed4._comment @@ -0,0 +1,10 @@ +[[!comment format=mdwn + username="http://joeyh.name/" + ip="209.250.56.47" + subject="comment 5" + date="2013-11-03T01:17:39Z" + content=""" +I don't understand what you mean by \"The assistant autorecovered my work repo before I noticed\". What repo is the work repo, and how could the assistant \"autorecover\" it, and what did it do? + +At this point, I am completely in the dark about whether you're reporting a problem, and what the problem is. +"""]] diff --git a/doc/bugs/Corrupted_drive__58___Assistant_seems_consider_files_deleted_and_deletes_them_elsewhere_too/comment_6_593a49669e2fadfb91773f8c84fbb031._comment b/doc/bugs/Corrupted_drive__58___Assistant_seems_consider_files_deleted_and_deletes_them_elsewhere_too/comment_6_593a49669e2fadfb91773f8c84fbb031._comment new file mode 100644 index 0000000000..af31edcf87 --- /dev/null +++ b/doc/bugs/Corrupted_drive__58___Assistant_seems_consider_files_deleted_and_deletes_them_elsewhere_too/comment_6_593a49669e2fadfb91773f8c84fbb031._comment @@ -0,0 +1,8 @@ +[[!comment format=mdwn + username="https://www.google.com/accounts/o8/id?id=AItOawnZEanlyzay_QlEAL0CWpyZcRTyN7vay8U" + nickname="Carlo" + subject="comment 6" + date="2013-10-31T10:03:01Z" + content=""" +Of course, the autorecovery wouldn't affect the daemon.log from the work machine... so here they are: http://capocasa.name/work-desktop-ubuntu-12.04-daemonlog.tgz +"""]] diff --git a/doc/bugs/Corrupted_drive__58___Assistant_seems_consider_files_deleted_and_deletes_them_elsewhere_too/comment_6_5a348c5f327f16e1192ef6bd7f2880bb._comment b/doc/bugs/Corrupted_drive__58___Assistant_seems_consider_files_deleted_and_deletes_them_elsewhere_too/comment_6_5a348c5f327f16e1192ef6bd7f2880bb._comment new file mode 100644 index 0000000000..7e611cebd4 --- /dev/null +++ b/doc/bugs/Corrupted_drive__58___Assistant_seems_consider_files_deleted_and_deletes_them_elsewhere_too/comment_6_5a348c5f327f16e1192ef6bd7f2880bb._comment @@ -0,0 +1,15 @@ +[[!comment format=mdwn + username="https://www.google.com/accounts/o8/id?id=AItOawnZEanlyzay_QlEAL0CWpyZcRTyN7vay8U" + nickname="Carlo" + subject="comment 6" + date="2013-11-19T09:48:26Z" + content=""" +Sorry, missed the comment. + +My work repo is the repository on my work laptop, where deletions got synced to. + +Git annex had then run repository repair automatically, so the odd symlinks where no longer there for me to check out. + +It is possible that I ran some git commands in direct mode I shouldn't have; I put the files back in and it's working nicely now. So this might have been a \"no direct mode guard\" issue. + +"""]] diff --git a/doc/bugs/Corrupted_git___40__but_not_annex__41___controlled_files.mdwn b/doc/bugs/Corrupted_git___40__but_not_annex__41___controlled_files.mdwn new file mode 100644 index 0000000000..8f19956979 --- /dev/null +++ b/doc/bugs/Corrupted_git___40__but_not_annex__41___controlled_files.mdwn @@ -0,0 +1,102 @@ +### Please describe the problem. + +I have files that match annex.largefiles and therefore should be added to git but not to annex, they seem to be getting corrupted after cloning the repo. + +### What steps will reproduce the problem? + +I couldn't immediately find the exact steps to reproduce the issue but I have multiple git repositories showing this. + +### What version of git-annex are you using? On what operating system? + +The problem has occurred a while ago but I have just noticed it. This is on macOS if it helps. I also tend to use the latest released version of git-annex (installed via Homebrew) + +### Please provide any additional information below. + +[[!format sh """ +# If you can, paste a complete transcript of the problem occurring here. +# If the problem is with the git-annex assistant, paste in .git/annex/daemon.log + +$ cd Documents +$ cat .gitattributes +* annex.largefiles=((not(mimetype=text/*))or(largerthan=100kb)) + +*.png binary +*.jpg binary +*.jpeg binary +*.gif binary +*.ico binary + +*.mp3 binary +*.fla binary + +*.mov binary +*.mp4 binary +*.flv binary +*.swf binary +*.avi binary +*.mkv binary +*.mpg binary +*.mpeg binary + +*.gz binary +*.zip binary +*.7z binary +*.rar binary +*.bz2 binary + +*.ttf binary + +*.pdf binary + +$ ls -la Docs/2016-XXX/XXX/ +total 696 +drwx------@ 4 denis staff 136 Jul 11 15:05 ./ +drwxr-xr-x@ 9 denis staff 306 Dec 12 19:42 ../ +-rwxr-xr-x@ 1 denis staff 265898 Jul 11 13:03 XXX.pdf* +-rwxr-xr-x@ 1 denis staff 89586 Jul 11 13:03 Summary.pdf* +$ file --mime-type Docs/2016-XXX/XXX/XXX.pdf +Docs/2016-XXX/XXX/XXX.pdf: application/pdf +$ git show 60a76858a57a73967131b929af45a99703f67335 +commit 60a76858a57a73967131b929af45a99703f67335 +Author: Denis Dzyubenko +Date: Mon Jul 11 15:05:37 2016 +0200 + + XXX + +diff --git a/Docs/2016-XXX/XXX/XXX.pdf b/Docs/2016-XXX/XXX/XXX.pdf +new file mode 100755 +index 00000000..112f68d0 +Binary files /dev/null and b/Docs/2016-XXX/XXX/XXX.pdf differ +diff --git a/Docs/2016-XXX/XXX/Summary.pdf b/Docs/2016-XXX/XXX/Summary.pdf +new file mode 100755 +index 00000000..3828383e +Binary files /dev/null and b/Docs/2016-XXX/XXX/Summary.pdf differ +diff --git a/Docs/2016-XXX/XXX.pdf b/Docs/2016-XXX/XXX.pdf +deleted file mode 120000 +index 6d347a22..00000000 +--- a/Docs/2016-XXX/XXX.pdf ++++ /dev/null +@@ -1 +0,0 @@ +-../../.git/annex/objects/zJ/X1/SHA256E-s190749--ee0c8329c88f9c1656cc75cf37d4df64060a022e73d199164c5e5222ba1739d1.pdf/SHA256E-s190749--ee0c8329c88f9c1656cc +\ No newline at end of file + + + +$ git clone Documents Documents.tmp +Cloning into 'Documents.tmp'... +done. +$ cd ./Documents.tmp/ +$ ls -la Docs/2016-XXX/XXX/ +total 184 +drwxr-xr-x 4 denis staff 136 Dec 19 00:09 ./ +drwxr-xr-x 8 denis staff 272 Dec 19 00:09 ../ +-rwxr-xr-x 1 denis staff 101 Dec 19 00:09 XXX.pdf* +-rwxr-xr-x 1 denis staff 89586 Dec 19 00:09 Summary.pdf* +$ cat Docs/2016-XXX/XXX/XXX.pdf +/annex/objects/SHA256E-s265898--9c750c01dce9689ac3880224d2e95da6287b0cc89759c0c882e7a9a0fe48d664.pdf + +# End of transcript or log. +"""]] + +### Have you had any luck using git-annex before? (Sometimes we get tired of reading bug reports all day and a lil' positive end note does wonders) + diff --git a/doc/bugs/Could_not_read_from_remote_repository.mdwn b/doc/bugs/Could_not_read_from_remote_repository.mdwn new file mode 100644 index 0000000000..46391e6603 --- /dev/null +++ b/doc/bugs/Could_not_read_from_remote_repository.mdwn @@ -0,0 +1,28 @@ +### Please describe the problem. + +I've been using git-annex for a few weeks now, and everything was working fine until today. Now, when git-annex goes to sync my files, it will hang for a while, and then display a red box on the left side, saying `! Synced with adam.liter`. So, basically, it doesn't really sync. The logs say that it cannot read from the remote repository, which is set up on box.com and was working just fine until today. + +I've tried reconfiguring my jabber account on all my devices with git-annex as well as deleting and remaking the box.com repository, and none of these steps have solved the problem. + +### What steps will reproduce the problem? + +Making changes to any file located in the repository. + +### What version of git-annex are you using? On what operating system? + +Mac OS X 10.8.4 and git-annex version 4.20130801-gc88bbc4. + +### Please provide any additional information below. + +[[!format sh """ +[2013-09-09 01:33:03 EDT] XMPPSendPack: Syncing with adam.liter +Already up-to-date. +fatal: Could not read from remote repository. + +Please make sure you have the correct access rights +and the repository exists. +"""]] + +[[!meta title="xmpp syncing sometimes fails"]] + +[[!tag confirmed]] diff --git a/doc/bugs/Could_not_read_from_remote_repository/comment_1_da842a9d146bcd5c7773b58364c25597._comment b/doc/bugs/Could_not_read_from_remote_repository/comment_1_da842a9d146bcd5c7773b58364c25597._comment new file mode 100644 index 0000000000..c236beacdd --- /dev/null +++ b/doc/bugs/Could_not_read_from_remote_repository/comment_1_da842a9d146bcd5c7773b58364c25597._comment @@ -0,0 +1,8 @@ +[[!comment format=mdwn + username="http://joeyh.name/" + ip="4.153.255.110" + subject="comment 1" + date="2013-09-09T06:01:18Z" + content=""" +From the small bit of log posted, it seems to be failing to sync over XMPP. Perhaps the XMPP server is down or it cannot connect? You can turn on debugging to get a lot more debugging about XMPP sync attempts. +"""]] diff --git a/doc/bugs/Could_not_read_from_remote_repository/comment_2_82746a0cf989d884cd0fd796db092b3c._comment b/doc/bugs/Could_not_read_from_remote_repository/comment_2_82746a0cf989d884cd0fd796db092b3c._comment new file mode 100644 index 0000000000..8312bb3a38 --- /dev/null +++ b/doc/bugs/Could_not_read_from_remote_repository/comment_2_82746a0cf989d884cd0fd796db092b3c._comment @@ -0,0 +1,36 @@ +[[!comment format=mdwn + username="https://www.google.com/accounts/o8/id?id=AItOawkgH7oNEqNbh3g-N1-UHXuqleXaRYDgj1U" + nickname="Adam" + subject="comment 2" + date="2013-09-09T06:12:44Z" + content=""" +This is what is in the logs after I turned on debugging: + + [2013-09-09 02:04:22 EDT] XMPPSendPack: finished running push Pushing \"a29\" (PushRequest (UUID \"c96c8493-b702-4ca0-9a88-f8a38dd6c3f5\")) True + [2013-09-09 02:04:22 EDT] XMPPSendPack: started running push Pushing \"a29\" (PushRequest (UUID \"c96c8493-b702-4ca0-9a88-f8a38dd6c3f5\")) + [2013-09-09 02:04:22 EDT] read: git [\"--git-dir=/Users/adamliter/Library/texmf/.git\",\"--work-tree=/Users/adamliter/Library/texmf\",\"symbolic-ref\",\"HEAD\"] + [2013-09-09 02:04:22 EDT] read: git [\"--git-dir=/Users/adamliter/Library/texmf/.git\",\"--work-tree=/Users/adamliter/Library/texmf\",\"show-ref\",\"refs/heads/master\"] + [2013-09-09 02:04:22 EDT] call: git [\"--git-dir=/Users/adamliter/Library/texmf/.git\",\"--work-tree=/Users/adamliter/Library/texmf\",\"branch\",\"-f\",\"synced/master\"] + [2013-09-09 02:04:22 EDT] XMPPSendPack: Syncing with adam.liter + [2013-09-09 02:04:22 EDT] call: git [\"--git-dir=/Users/adamliter/Library/texmf/.git\",\"--work-tree=/Users/adamliter/Library/texmf\",\"push\",\"adam.liter\",\"git-annex:refs/synced/6258750b-9b0d-4f05-b443-61f4fb0a3f66/YWRhbS5saXRlckBnbWFpbC5jb20=/git-annex\",\"refs/heads/master:refs/synced/6258750b-9b0d-4f05-b443-61f4fb0a3f66/YWRhbS5saXRlckBnbWFpbC5jb20=/master\"] + [2013-09-09 02:04:22 EDT] XMPPClient: sending: Pushing \"a29\" (StartingPush (UUID \"6258750b-9b0d-4f05-b443-61f4fb0a3f66\")) + [2013-09-09 02:04:22 EDT] XMPPClient: to client: a10/F4E99F98 + [2013-09-09 02:04:23 EDT] XMPPReceivePack: timeout waiting for git send-pack output via XMPP + fatal: The remote end hung up unexpectedly + [2013-09-09 02:04:23 EDT] XMPPReceivePack: finished running push Pushing \"a29\" (StartingPush (UUID \"c96c8493-b702-4ca0-9a88-f8a38dd6c3f5\")) True + [2013-09-09 02:04:23 EDT] XMPPClient: sending: Pushing \"a29\" (ReceivePackDone (ExitFailure 128)) + [2013-09-09 02:04:23 EDT] XMPPReceivePack: started running push Pushing \"a29\" (StartingPush (UUID \"c96c8493-b702-4ca0-9a88-f8a38dd6c3f5\")) + [2013-09-09 02:04:23 EDT] XMPPClient: to client: a10/F4E99F98 + [2013-09-09 02:04:23 EDT] XMPPReceivePack: Syncing with adam.liter + [2013-09-09 02:04:23 EDT] chat: git [\"receive-pack\",\"/Users/adamliter/Library/texmf\"] + [2013-09-09 02:04:23 EDT] XMPPClient: sending: Pushing \"a29\" (ReceivePackOutput 1 \"\") + [2013-09-09 02:04:23 EDT] XMPPClient: to client: a10/F4E99F98 + [2013-09-09 02:04:23 EDT] XMPPClient: sending: Pushing \"a29\" (ReceivePackOutput 2 \"\") + [2013-09-09 02:04:23 EDT] XMPPClient: to client: a10/F4E99F98 + [2013-09-09 02:04:23 EDT] XMPPClient: received: [\"Pushing \\"a29\\" (StartingPush (UUID \\"c96c8493-b702-4ca0-9a88-f8a38dd6c3f5\\"))\"] + [2013-09-09 02:04:23 EDT] XMPPClient: received: [\"Unknown message\"] + + + + +"""]] diff --git a/doc/bugs/Could_not_read_from_remote_repository/comment_3_95d16045dc238dba19a98808de2eeedf._comment b/doc/bugs/Could_not_read_from_remote_repository/comment_3_95d16045dc238dba19a98808de2eeedf._comment new file mode 100644 index 0000000000..a8d2705a62 --- /dev/null +++ b/doc/bugs/Could_not_read_from_remote_repository/comment_3_95d16045dc238dba19a98808de2eeedf._comment @@ -0,0 +1,11 @@ +[[!comment format=mdwn + username="https://www.google.com/accounts/o8/id?id=AItOawnvVfFLW4CTKs7UjdiLIsOn_cxj1Jnh64I" + nickname="Charl" + subject="Could it be gmail.com XMPP throttling?" + date="2014-03-23T19:26:08Z" + content=""" +I was seeing similar error messages, until I registered for a different XMPP account at jabber.de and started using that instead of my gmail.com account. + +My current suspicion is that it could be Google performing throttling on their XMPP service. See here: http://stackoverflow.com/questions/1843837/what-is-the-throttling-rate-that-gtalk-applies-to-xmpp-messages + +"""]] diff --git a/doc/bugs/Crash_when_disabling_syncing_in_the_webapp.mdwn b/doc/bugs/Crash_when_disabling_syncing_in_the_webapp.mdwn new file mode 100644 index 0000000000..b239c0a401 --- /dev/null +++ b/doc/bugs/Crash_when_disabling_syncing_in_the_webapp.mdwn @@ -0,0 +1,25 @@ +### Please describe the problem. +The watcher crashes. + +I only need to restart the thread in the pop-up to get everything to work again, but I'm reporting just in case that this issue has any other implications. + + +### What steps will reproduce the problem? +I open the webapp and in the minutes before it starts syncing (syncing is enabled) I disable it (clicking in the 'syncing enabled' text). + +This produces a crash every time. + + +### What version of git-annex are you using? On what operating system? +5.20140320 in Debian sid and testing + + +### Please provide any additional information below. +This is all I can see in the logs + +[[!format sh """ +Watcher crashed: PauseWatcher +[2014-03-26 08:54:57 CET] Watcher: warning Watcher crashed: PauseWatcher +"""]] + +[[!tag moreinfo]] diff --git a/doc/bugs/Crash_when_disabling_syncing_in_the_webapp/comment_1_e25dd80370820782f9c6a877101d8703._comment b/doc/bugs/Crash_when_disabling_syncing_in_the_webapp/comment_1_e25dd80370820782f9c6a877101d8703._comment new file mode 100644 index 0000000000..ce4af5caa5 --- /dev/null +++ b/doc/bugs/Crash_when_disabling_syncing_in_the_webapp/comment_1_e25dd80370820782f9c6a877101d8703._comment @@ -0,0 +1,10 @@ +[[!comment format=mdwn + username="http://joeyh.name/" + ip="209.250.56.41" + subject="comment 1" + date="2014-03-26T17:36:00Z" + content=""" +How did you install git-annex? Is this Debian Linux? + +I have not been able to reproduce a crash. It's indeed the case that a PauseWatcher exception is thrown, but the Watcher explicitly catches that exception. +"""]] diff --git a/doc/bugs/Crash_when_disabling_syncing_in_the_webapp/comment_2_4031c16362137747717e9595cb5c8a15._comment b/doc/bugs/Crash_when_disabling_syncing_in_the_webapp/comment_2_4031c16362137747717e9595cb5c8a15._comment new file mode 100644 index 0000000000..07efa451d7 --- /dev/null +++ b/doc/bugs/Crash_when_disabling_syncing_in_the_webapp/comment_2_4031c16362137747717e9595cb5c8a15._comment @@ -0,0 +1,10 @@ +[[!comment format=mdwn + username="https://www.google.com/accounts/o8/id?id=AItOawk9nck8WX8-ADF3Fdh5vFo4Qrw1I_bJcR8" + nickname="Jon Ander" + subject="comment 2" + date="2014-04-01T08:04:51Z" + content=""" +Yes, this is Debian Linux and I've been able to reproduce it in i386 and amd64. git-annex is installed from the Debian repositories. + +I'll try to continue testing the issue and will report back if I can find any useful info. +"""]] diff --git a/doc/bugs/Crash_when_disabling_syncing_in_the_webapp/comment_3_0667f39f60bdaba6670f5b8304a8a77c._comment b/doc/bugs/Crash_when_disabling_syncing_in_the_webapp/comment_3_0667f39f60bdaba6670f5b8304a8a77c._comment new file mode 100644 index 0000000000..60f82a5b30 --- /dev/null +++ b/doc/bugs/Crash_when_disabling_syncing_in_the_webapp/comment_3_0667f39f60bdaba6670f5b8304a8a77c._comment @@ -0,0 +1,12 @@ +[[!comment format=mdwn + username="http://joeyh.name/" + ip="209.250.56.244" + subject="comment 3" + date="2014-04-02T20:38:12Z" + content=""" +So we have the same version of git-annex from the same build, and only you see the problem. Hmm.. + +You mentioned that you see the problem if you disable syncing at a particular time. Does it only crash at that time, or at any time? + +If you create a brand new empty repository and run the webapp in it, can you reporoduce the problem there? Ie, \"mkdir test; cd test; git init; git annex init; git annex webapp\" +"""]] diff --git a/doc/bugs/Creating_an_encrypted_S3_does_not_check_for_presence_of_GPG.mdwn b/doc/bugs/Creating_an_encrypted_S3_does_not_check_for_presence_of_GPG.mdwn new file mode 100644 index 0000000000..611e86d613 --- /dev/null +++ b/doc/bugs/Creating_an_encrypted_S3_does_not_check_for_presence_of_GPG.mdwn @@ -0,0 +1,19 @@ +What steps will reproduce the problem? + +Don't have gpg installed/on your $PATH, and attempt to create an encrypted S3 remote via the web interface. + +What is the expected output? What do you see instead? + +Expected to be told to install GPG. Actual output was a Yesod error: + +Internal Server Error +user error (gpg ["--batch","--no-tty","--use-agent","--quiet","--trust-model","always","--gen-random","--armor","1","512"] exited 127) + +What version of git-annex are you using? On what operating system? + +3.20130107 on Fedora 17 (64-bit). + +Please provide any additional information below. + +[[!tag /design/assistant]] +[[!tag confirmed]] diff --git a/doc/bugs/DBG__58___running___96____47__Users__47__joey__47__homebrew__47__opt__47__gpg-agent__47__bin__47__gpg-agent__39___for_testing_failed.mdwn b/doc/bugs/DBG__58___running___96____47__Users__47__joey__47__homebrew__47__opt__47__gpg-agent__47__bin__47__gpg-agent__39___for_testing_failed.mdwn new file mode 100644 index 0000000000..3dc28d4e6e --- /dev/null +++ b/doc/bugs/DBG__58___running___96____47__Users__47__joey__47__homebrew__47__opt__47__gpg-agent__47__bin__47__gpg-agent__39___for_testing_failed.mdwn @@ -0,0 +1,31 @@ +### Please describe the problem. +Issue uploading to S3 remote (Dreamhost) + +### What steps will reproduce the problem? +git-annex copy massart/f16_Web1/screencaptures/IMG_5159.MOV --to=cloud +on my repo + +### What version of git-annex are you using? On what operating system? +6.20161031-g0a58e94 +OS-X 10.11.6 + +### Please provide any additional information below. +I am using a different WIFI I haven't used before. Maybe it is blocking something… + +[[!format sh """ +git-annex copy massart/f16_Web1/screencaptures/IMG_5159.MOV --to=cloud +copy massart/f16_Web1/screencaptures/IMG_5159.MOV (checking cloud...) (to cloud...) +17% 0.0 B/s 0sgpg: error running `/Users/joey/homebrew/opt/gpg-agent/bin/gpg-agent': probably not installed +gpg: DBG: running `/Users/joey/homebrew/opt/gpg-agent/bin/gpg-agent' for testing failed: Configuration error +gpg: can't connect to the agent: IPC connect call failed +gpg: problem with the agent: No agent running +35% 1021.8KB/s 30s + user error (gpg ["--quiet","--trust-model","always","--batch","--passphrase-fd","26","--symmetric","--force-mdc","--no-textmode"] exited 2) +failed +git-annex: copy: 1 failed +"""]] + +### Have you had any luck using git-annex before? (Sometimes we get tired of reading bug reports all day and a lil' positive end note does wonders) +Yes. + +[[done]] diff --git a/doc/bugs/DBG__58___running___96____47__Users__47__joey__47__homebrew__47__opt__47__gpg-agent__47__bin__47__gpg-agent__39___for_testing_failed/comment_1_39718e8a35e42421a8aaf3316ae1d76a._comment b/doc/bugs/DBG__58___running___96____47__Users__47__joey__47__homebrew__47__opt__47__gpg-agent__47__bin__47__gpg-agent__39___for_testing_failed/comment_1_39718e8a35e42421a8aaf3316ae1d76a._comment new file mode 100644 index 0000000000..02b62f46e8 --- /dev/null +++ b/doc/bugs/DBG__58___running___96____47__Users__47__joey__47__homebrew__47__opt__47__gpg-agent__47__bin__47__gpg-agent__39___for_testing_failed/comment_1_39718e8a35e42421a8aaf3316ae1d76a._comment @@ -0,0 +1,14 @@ +[[!comment format=mdwn + username="andrew" + avatar="http://cdn.libravatar.org/avatar/acc0ece1eedf07dd9631e7d7d343c435" + subject="RESOLVED" + date="2016-11-17T14:59:15Z" + content=""" +Ooops. I am on OS-X. I use brew for my gnupg installation. It appears I had removed gpg from the path when installing something. I just needed to run to fix: + + brew link gnupg2 + +Thanks, + +Andrew +"""]] diff --git a/doc/bugs/Data_loss_when_copying_files_with_running_assistant.mdwn b/doc/bugs/Data_loss_when_copying_files_with_running_assistant.mdwn new file mode 100644 index 0000000000..ba563953c3 --- /dev/null +++ b/doc/bugs/Data_loss_when_copying_files_with_running_assistant.mdwn @@ -0,0 +1,26 @@ +### Please describe the problem. + +I’ve just experienced something strange. When files are copied (instead of moved) into a repo with an assistant running, some of them get lost. + +### What steps will reproduce the problem? + +1. Start the assistant. +2. `cp -rv some-big-directory your-annex-repo/` +3. Wait for sync to finish. +4. Compare some-big-directory with your-annex-repo/some-big-directory. + +It might be of some importance that I’m syncing this repo to 5 different machines. + +### What version of git-annex are you using? On what operating system? + +git-annex version: 6.20170520 +local repository version: 6 +operating system: linux x86_64 + +### Please provide any additional information below. + +Hard to tell, sometimes it happens, sometimes not. + +### Have you had any luck using git-annex before? (Sometimes we get tired of reading bug reports all day and a lil' positive end note does wonders) + +Sure, very cool, but this data loss thing creeped me out! diff --git a/doc/bugs/Data_loss_when_copying_files_with_running_assistant/comment_1_51e7afb030dfb3052044b0fa95e08bed._comment b/doc/bugs/Data_loss_when_copying_files_with_running_assistant/comment_1_51e7afb030dfb3052044b0fa95e08bed._comment new file mode 100644 index 0000000000..f43dbe6f35 --- /dev/null +++ b/doc/bugs/Data_loss_when_copying_files_with_running_assistant/comment_1_51e7afb030dfb3052044b0fa95e08bed._comment @@ -0,0 +1,8 @@ +[[!comment format=mdwn + username="michalrus" + avatar="http://cdn.libravatar.org/avatar/83c0b6e7f9d20f09a892263c4903bbae" + subject="comment 1" + date="2017-08-20T15:47:03Z" + content=""" +These files can be seen in `git annex unused` on the machine that was to add them… +"""]] diff --git a/doc/bugs/Data_loss_when_copying_files_with_running_assistant/comment_2_b9cc9ae227a6dd883a2324b6d70b88ad._comment b/doc/bugs/Data_loss_when_copying_files_with_running_assistant/comment_2_b9cc9ae227a6dd883a2324b6d70b88ad._comment new file mode 100644 index 0000000000..dc3f82a0ca --- /dev/null +++ b/doc/bugs/Data_loss_when_copying_files_with_running_assistant/comment_2_b9cc9ae227a6dd883a2324b6d70b88ad._comment @@ -0,0 +1,111 @@ +[[!comment format=mdwn + username="michalrus" + avatar="http://cdn.libravatar.org/avatar/83c0b6e7f9d20f09a892263c4903bbae" + subject="comment 2" + date="2017-08-20T16:45:09Z" + content=""" +I’ve got some `git log -S'KEY'` of what happened. Names censored. So it seems like peers are deleting and re-adding this files, and at the end they end up deleted. :( The last commit is me manually re-adding them one last time. + +Please note, that these are completely new files, it’s not possible that merges are deleting them. + +``` +commit 38134b87fdd6957c578f2a9d06d6c50aefee361b +Author: Elzbieta +Date: Sun Aug 20 18:21:04 2017 +0200 + + Re-add unused files + + ...7e4a6e20cba47b04eefef5ad0a85e16b00e8f5bbc5bcfade24ad0b934738d790.docx | 1 + + 1 file changed, 1 insertion(+) + +commit 9d3be6aafe84fa7e114f5de12c0a15adc0b2704e +Author: Elzbieta +Date: Sun Aug 20 12:20:13 2017 +0200 + + git-annex in elzbietarus + + .../SA\305\201ATKA \305\232LEDZIOWA, SA\305\201ATKA Z SZYNKI.docx\" | 1 + + 1 file changed, 1 insertion(+) + +commit 92c1e62f8483c6b4fa263180a8a506bf5d0a46c7 +Author: Elzbieta +Date: Sun Aug 20 12:20:10 2017 +0200 + + git-annex in elzbietarus + + .../SA\305\201ATKA \305\232LEDZIOWA, SA\305\201ATKA Z SZYNKI.docx\" | 1 - + 1 file changed, 1 deletion(-) + +commit cd51d4dfed402a0929d357bb9557923618fdadff +Author: Robert +Date: Sun Aug 20 12:19:56 2017 +0200 + + git-annex in robertrus-qx511 + + .../SA\305\201ATKA \305\232LEDZIOWA, SA\305\201ATKA Z SZYNKI.docx\" | 1 + + 1 file changed, 1 insertion(+) + +commit 879256502d8047f0a3ed5487a57854076459f9a7 +Author: Robert +Date: Sun Aug 20 12:19:54 2017 +0200 + + git-annex in robertrus-qx511 + + .../SA\305\201ATKA \305\232LEDZIOWA, SA\305\201ATKA Z SZYNKI.docx\" | 1 - + 1 file changed, 1 deletion(-) + +commit 7196eb855f2ca3ca87959002c3e61c7e060d7ae6 +Author: Elzbieta +Date: Sun Aug 20 12:19:50 2017 +0200 + + git-annex in elzbietarus + + .../SA\305\201ATKA \305\232LEDZIOWA, SA\305\201ATKA Z SZYNKI.docx\" | 1 + + 1 file changed, 1 insertion(+) + +commit f566cb65b19aa6144e7587cdcb2a8b910584391c +Author: Elzbieta +Date: Sun Aug 20 12:19:42 2017 +0200 + + git-annex in elzbietarus + + .../SA\305\201ATKA \305\232LEDZIOWA, SA\305\201ATKA Z SZYNKI.docx\" | 1 - + 1 file changed, 1 deletion(-) + +commit de732718be3c25580463ddc5cd8abaf5eb6ca4be +Author: Michal Rus +Date: Sun Aug 20 12:19:28 2017 +0200 + + git-annex in michalrus + + .../SA\305\201ATKA \305\232LEDZIOWA, SA\305\201ATKA Z SZYNKI.docx\" | 1 - + 1 file changed, 1 deletion(-) + +commit b25b6274332bc5ac42dc0b6a0d69abe3bd32e0e0 +Author: Robert +Date: Sun Aug 20 12:19:24 2017 +0200 + + git-annex in robertrus-np300e5a + + .../SA\305\201ATKA \305\232LEDZIOWA, SA\305\201ATKA Z SZYNKI.docx\" | 1 + + 1 file changed, 1 insertion(+) + +commit f26399a8a62ae54eec45355f60228bc55527302d +Author: Robert +Date: Sun Aug 20 12:19:22 2017 +0200 + + git-annex in robertrus-np300e5a + + .../SA\305\201ATKA \305\232LEDZIOWA, SA\305\201ATKA Z SZYNKI.docx\" | 1 - + 1 file changed, 1 deletion(-) + +commit 103a62866df6fa8c7e18b727865c6c354fd6fc12 +Author: Elzbieta +Date: Sun Aug 20 12:16:29 2017 +0200 + + git-annex in elzbietarus + + .../SA\305\201ATKA \305\232LEDZIOWA, SA\305\201ATKA Z SZYNKI.docx\" | 1 + + 1 file changed, 1 insertion(+) +``` +"""]] diff --git a/doc/bugs/Data_loss_when_copying_files_with_running_assistant/comment_3_15f4fc02d574a2ed55e683dd99e9b72c._comment b/doc/bugs/Data_loss_when_copying_files_with_running_assistant/comment_3_15f4fc02d574a2ed55e683dd99e9b72c._comment new file mode 100644 index 0000000000..2acf16d380 --- /dev/null +++ b/doc/bugs/Data_loss_when_copying_files_with_running_assistant/comment_3_15f4fc02d574a2ed55e683dd99e9b72c._comment @@ -0,0 +1,8 @@ +[[!comment format=mdwn + username="michalrus" + avatar="http://cdn.libravatar.org/avatar/83c0b6e7f9d20f09a892263c4903bbae" + subject="comment 3" + date="2017-08-26T12:18:11Z" + content=""" +Related: [https://git-annex.branchable.com/bugs/Assistant_deleted_file_when_merging___40__another_report__41__/](https://git-annex.branchable.com/bugs/Assistant_deleted_file_when_merging___40__another_report__41__/) +"""]] diff --git a/doc/bugs/Data_loss_when_copying_files_with_running_assistant/comment_4_c16ecf8fabd0c63a7b59927fe0bb6e2c._comment b/doc/bugs/Data_loss_when_copying_files_with_running_assistant/comment_4_c16ecf8fabd0c63a7b59927fe0bb6e2c._comment new file mode 100644 index 0000000000..e55d4538e5 --- /dev/null +++ b/doc/bugs/Data_loss_when_copying_files_with_running_assistant/comment_4_c16ecf8fabd0c63a7b59927fe0bb6e2c._comment @@ -0,0 +1,94 @@ +[[!comment format=mdwn + username="michalrus" + avatar="http://cdn.libravatar.org/avatar/83c0b6e7f9d20f09a892263c4903bbae" + subject="comment 4" + date="2017-09-21T12:56:30Z" + content=""" +Look. What’s happening here? Reverse order. First, a file is saved and then assistant running on other machines makes a decision to remove it. Where does it come from? :o + +``` +commit cc61f6db3273c749ac2e4bdfb489457af919472c +Author: Elzbieta Rus +Date: Sat Sep 2 14:23:13 2017 +0200 + + git-annex in elzbietarus + + Dokumenty/Przepisy/SERNIK NA ZIMNO.docx | 1 - + 1 file changed, 1 deletion(-) + +commit 30965f29f756f02ad5d617d6fd07ec7013e01226 +Merge: d69d69901f ec07710223 +Author: Robert Rus +Date: Sat Sep 2 14:17:20 2017 +0200 + + Merge remote-tracking branch 'refs/remotes/origin/master' + +commit d69d69901fc2a4c958077cfbf8b27f595465627c +Author: Robert Rus +Date: Sat Sep 2 14:17:17 2017 +0200 + + git-annex in robertrus-asus-1225c + + Dokumenty/Przepisy/SERNIK NA ZIMNO.odt | 1 + + 1 file changed, 1 insertion(+) + +commit ec07710223770d1458de76004ec9baab5c20d508 +Merge: a60c530e9b 1cb58f2783 +Author: Robert Rus +Date: Sat Sep 2 14:17:17 2017 +0200 + + Merge remote-tracking branch 'refs/remotes/origin/master' + +commit 1e29dee284b5b9f4593cd7c0484c194ac2a9644c +Merge: 1a36f09308 1cb58f2783 +Author: Robert Rus +Date: Sat Sep 2 14:17:16 2017 +0200 + + Merge remote-tracking branch 'refs/remotes/origin/master' + +commit a60c530e9b167c17b625e66f42e7282a1e275703 +Author: Robert Rus +Date: Sat Sep 2 14:17:16 2017 +0200 + + git-annex in robertrus-acer + + Dokumenty/Przepisy/SERNIK NA ZIMNO.odt | 1 + + 1 file changed, 1 insertion(+) + +commit 1a36f09308c4134646b7074f2e827c909ee7033a +Author: Robert Rus +Date: Sat Sep 2 14:17:14 2017 +0200 + + git-annex in robertrus-asus-1225c + + Dokumenty/Przepisy/SERNIK NA ZIMNO.odt | 1 - + 1 file changed, 1 deletion(-) + +commit 1cb58f2783485e83d369ab9acb41f7e83cc3314c +Author: Mikolaj Rus +Date: Sat Sep 2 14:17:12 2017 +0200 + + git-annex in mikolajrus + + Dokumenty/Przepisy/SERNIK NA ZIMNO.odt | 1 + + 1 file changed, 1 insertion(+) + +commit 3542add42a050c5331cbfa67bffee38a05ce6bc6 +Author: Mikolaj Rus +Date: Sat Sep 2 14:17:11 2017 +0200 + + git-annex in mikolajrus + + Dokumenty/Przepisy/SERNIK NA ZIMNO.odt | 1 - + 1 file changed, 1 deletion(-) + +commit 956243cab7b865f9e633f51d4376154a3478210a +Author: Elzbieta Rus +Date: Sat Sep 2 14:17:00 2017 +0200 + + git-annex in elzbietarus + + Dokumenty/Przepisy/SERNIK NA ZIMNO.odt | 1 + + 1 file changed, 1 insertion(+) +``` +"""]] diff --git a/doc/bugs/Deasn__39__t_clean_up_ssh_keys_after_removing_remote_repo.mdwn b/doc/bugs/Deasn__39__t_clean_up_ssh_keys_after_removing_remote_repo.mdwn new file mode 100644 index 0000000000..0899ea4675 --- /dev/null +++ b/doc/bugs/Deasn__39__t_clean_up_ssh_keys_after_removing_remote_repo.mdwn @@ -0,0 +1,18 @@ +### Please describe the problem. +I created a remote repo on ssh server with the same git-annex version, no personal ssh keys for the repo (password authentication). But I put ~ in front of the repo name so it created in different place than I wanted. I deleted it from assistant and then deleted the remote repo dir. When I added it with the correct path again it always asks for server password (it has some old annex pub key in authorized_keys, but not the new one; it might have been prompted for the password also with the initial repo). During the whole thing it failed to connect a few times due to wrong password or me realizing too late that it asks for yet another one and it generated keys few times, but ultimately didn't put the last one in remote's authorized_keys. + +### What steps will reproduce the problem? +Create, delete and create again a new repo on remote ssh server with password auth. + +### What version of git-annex are you using? On what operating system? +git-annex 4.20130601 Gentoo amd64 + +### Please provide any additional information below. + +[[!format sh """ +# If you can, paste a complete transcript of the problem occurring here. +# If the problem is with the git-annex assistant, paste in .git/annex/daemon.log + + +# End of transcript or log. +"""]] diff --git a/doc/bugs/Deasn__39__t_clean_up_ssh_keys_after_removing_remote_repo/comment_1_88fbf70eae48484988dbb433a437c717._comment b/doc/bugs/Deasn__39__t_clean_up_ssh_keys_after_removing_remote_repo/comment_1_88fbf70eae48484988dbb433a437c717._comment new file mode 100644 index 0000000000..84686f1e45 --- /dev/null +++ b/doc/bugs/Deasn__39__t_clean_up_ssh_keys_after_removing_remote_repo/comment_1_88fbf70eae48484988dbb433a437c717._comment @@ -0,0 +1,14 @@ +[[!comment format=mdwn + username="http://joeyh.name/" + ip="4.152.108.193" + subject="comment 1" + date="2013-06-25T18:20:05Z" + content=""" +It's true that deleting a ssh repository in the webapp does not delete any ssh keys that the webapp configured to access that repository. I'm a bit reluctant to try to clean it up entirely automatically, because the ssh configuration might have been manually altered, or might be used by another local repository. Particularly, if another local repo is set up to use the same ssh remote, the webapp will reuse the ssh key. + +> Create, delete and create again a new repo on remote ssh server with password auth. + +I tried doing this, and did not encounter any problem. Since the new repo had a different path on the ssh server, I simply ended up with two ssh keys configured in `~/.ssh/config` for two different repositories. The presence or absence of the first key did not affect the second key. + +I think you need to go into more detail about exactly what you did, or show the files in .ssh that were set up, so I can see what the actual problem is. +"""]] diff --git a/doc/bugs/Debian_and_MacOSX_cannot_read_eachother__39__s_S3_credentials.mdwn b/doc/bugs/Debian_and_MacOSX_cannot_read_eachother__39__s_S3_credentials.mdwn new file mode 100644 index 0000000000..a351b0ae4e --- /dev/null +++ b/doc/bugs/Debian_and_MacOSX_cannot_read_eachother__39__s_S3_credentials.mdwn @@ -0,0 +1,63 @@ +### Please describe the problem. +I seem to be hitting a problem with the way git annex encrypts the credentials for S3 remotes. I am creating an S3 remote with shared encryption and embedded credentials. If I create the remote on MacOSX, enableremote fails on Debian. If I create the remote on Debian, enableremote fails on MacOSX with a similar error. I can enable MacOSX created remotes on other MacOSX systems and Debian created remotes on other Debian systems. +The enableremote creates a .git/annex/creds/uuid file, but the contents are garbled. + +Could this be an issue with different versions of GPG on the systems? + +### What steps will reproduce the problem? +- New git annex remote on Debian. +- Clone and git annex init on MacOSX +- Create S3 remote on MacOSX with encryption=shared embedcreds=yes +- Git annex sync +- On Debian enable remote + +Alternatively, create the S3 remote on Debian, sync and enable remote on MacOSX + +### What version of git-annex are you using? On what operating system? + MacOSX + gpg (GnuPG/MacGPG2) 2.0.30 + libgcrypt 1.6.6 + git-annex version: 6.20170101 + build flags: Assistant Webapp Pairing Testsuite S3(multipartupload)(storageclasses) WebDAV FsEvents ConcurrentOutput TorrentParser MagicMime Feeds Quvi + key/value backends: SHA256E SHA256 SHA512E SHA512 SHA224E SHA224 SHA384E SHA384 SHA3_256E SHA3_256 SHA3_512E SHA3_512 SHA3_224E SHA3_224 SHA3_384E SHA3_384 SKEIN256E SKEIN256 SKEIN512E SKEIN512 SHA1E SHA1 MD5E MD5 WORM URL + remote types: git gcrypt p2p S3 bup directory rsync web bittorrent webdav tahoe glacier ddar hook external + local repository version: 5 + supported repository versions: 3 5 6 + upgrade supported from repository versions: 0 1 2 3 4 5 + operating system: darwin x86_64 + + Debian 7 + gpg (GnuPG) 1.4.12 + git-annex version: 6.20170101+gitg93d69b1-1~ndall+1 + build flags: Assistant Webapp Pairing Testsuite S3(multipartupload)(storageclasses) WebDAV Inotify DBus DesktopNotify ConcurrentOutput TorrentParser MagicMime Feeds Quvi + key/value backends: SHA256E SHA256 SHA512E SHA512 SHA224E SHA224 SHA384E SHA384 SHA3_256E SHA3_256 SHA3_512E SHA3_512 SHA3_224E SHA3_224 SHA3_384E SHA3_384 SKEIN256E SKEIN256 SKEIN512E SKEIN512 SHA1E SHA1 MD5E MD5 WORM URL + remote types: git gcrypt p2p S3 bup directory rsync web bittorrent webdav tahoe glacier ddar hook external + local repository version: 5 + supported repository versions: 3 5 6 + upgrade supported from repository versions: 0 1 2 3 4 5 + operating system: linux x86_64 + +### Please provide any additional information below. + +[[!format sh """ +# Debian 7 + $ git annex enableremote S3 + enableremote S3 gpg: packet(2) with unknown version 13 + gpg error above was caused by an old git-annex bug in credentials storage. Working around it.. + git-annex: bad creds + CallStack (from HasCallStack): + error, called at ./Creds.hs:116:22 in main:Creds + + # MacOSX + git annex enableremote S34 + enableremote S34 gpg: [don't know]: invalid packet (ctb=6b) + gpg error above was caused by an old git-annex bug in credentials storage. Working around it.. + git-annex: bad creds + CallStack (from HasCallStack): + error, called at ./Creds.hs:116:22 in main:Creds + +"""]] + +### Have you had any luck using git-annex before? (Sometimes we get tired of reading bug reports all day and a lil' positive end note does wonders) +Yes! git annex has been enormously helpful. Thanks so much for this tool. + diff --git a/doc/bugs/Debian_and_MacOSX_cannot_read_eachother__39__s_S3_credentials/comment_1_e307e5d0ec02ed76dc7c3f0894770539._comment b/doc/bugs/Debian_and_MacOSX_cannot_read_eachother__39__s_S3_credentials/comment_1_e307e5d0ec02ed76dc7c3f0894770539._comment new file mode 100644 index 0000000000..840bfdaa2e --- /dev/null +++ b/doc/bugs/Debian_and_MacOSX_cannot_read_eachother__39__s_S3_credentials/comment_1_e307e5d0ec02ed76dc7c3f0894770539._comment @@ -0,0 +1,28 @@ +[[!comment format=mdwn + username="joey" + subject="""comment 1""" + date="2017-02-13T18:27:02Z" + content=""" +The .git/annex/creds/uuid files for S3 special remotes are supposed to +contain the S3 access credentials, the same values you set +`AWS_ACCESS_KEY_ID` and `AWS_SECRET_ACCESS_KEY` to. Since it's just some +high-entropy credentials, perhaps that's why you thought it looked garbled? + +I doubt that the error message about the cause of the gpg error is +right; that is a workaround for an old bug in git-annex 5.20140916. Unless +you used such an old version to set up the special remote, it's probably +not related to that problem at all. + +The actual problem seems to be: + + gpg: packet(2) with unknown version 13 + gpg: [don't know]: invalid packet (ctb=6b) + +This probably points to an incompatability between the gpg versions. +The easiest fix is probably to upgrade Debian to gpg 2. + +With embedcreds, the S3 creds are symmetrically encrypted/decrypted +using gpg --symmetric. This is a less common way to use gpg, +and it could be that there's an incompatability with how different +versions of gpg do it. +"""]] diff --git a/doc/bugs/Default_startup_command.mdwn b/doc/bugs/Default_startup_command.mdwn new file mode 100644 index 0000000000..2c16868c6b --- /dev/null +++ b/doc/bugs/Default_startup_command.mdwn @@ -0,0 +1,31 @@ +[[!meta title="Android Default startup command"]] + +### Please describe the problem. +On a CM11 4.4.4 install, the startup will fail with the default startup command "git annex webapp". +The error message shown is: +``` +Falling back to hardcoded app location; cannot find expected files in /data/app-lib +git annex webapp +``` + +However, using "git-annex webapp", git-annex will run fine. + +### What steps will reproduce the problem? + +Launch the beta or nightly build for 4.3 and 4.4, wait for error message + +### What version of git-annex are you using? On what operating system? + +Version "current" as of 2014-11-15 and also version "nightly build" as of 2014-11-15 on Android 4.4.4 CM11, device: Oppo Find 7a + +### Please provide any additional information below. + +[[!format sh """ +# If you can, paste a complete transcript of the problem occurring here. +# If the problem is with the git-annex assistant, paste in .git/annex/daemon.log + + +# End of transcript or log. +"""]] + +> Closing as this was a bug in the deprecated Android app. [[done]] --[[Joey]] diff --git a/doc/bugs/Default_startup_command/comment_1_2d63fb301586c9a97ceff1d1b9fafbee._comment b/doc/bugs/Default_startup_command/comment_1_2d63fb301586c9a97ceff1d1b9fafbee._comment new file mode 100644 index 0000000000..47487f01a2 --- /dev/null +++ b/doc/bugs/Default_startup_command/comment_1_2d63fb301586c9a97ceff1d1b9fafbee._comment @@ -0,0 +1,29 @@ +[[!comment format=mdwn + username="git-annex@53a027ebda399714df9272a135f22ac774f3c9f1" + nickname="git-annex" + subject="i can second this" + date="2015-07-26T10:29:32Z" + content=""" +I have the same behaviour on CM11 4.4.4 on a Moto G as well. +When i installed git annex i would be greeted by the same message, but then could start the webapp from the menu successfully. +i then proceeded to setup my repo, which synced flawlessly, if i had started the webapp as stated above. +however my files were never automatically synced when just starting the app (terminal window). + +Therefore i resorted to \"cd\" to my repo and \"git annex add *\" and \"git annex sync --content\" every time i wanted to sync. + +i tried changing the startup command to \"git annex assistant --autostart\", \"git annex assistant --autostart --foreground\", \"cd ../DCIM;git annex sync --content\", \"echo test\" but nothing seems to start (not quite sure how to check for that). + +for my shell i had the default/data/data/ga.androidterm/lib/lib.start.so and /data/data/ga.androidterm/runshell if that makes any difference? + +not sure if that's related +when trying to manually invoke \"git annex --assistant [--foreground]\" i get: + + git annex autostart in /sdcard/DCIM + usage: ionice [none|rt|be|idle] [prio] + failed + +and back to prompt. Not sure if this is CM(11) only? + +my git annex version is: 5.20150527-gfc92f13 for android 4.3 and 4.4. if you need more information i'll be happy to provide it, to get this out of the way :D + +"""]] diff --git a/doc/bugs/Default_startup_command/comment_2_296de7f38161ebc21da954d46c924c98._comment b/doc/bugs/Default_startup_command/comment_2_296de7f38161ebc21da954d46c924c98._comment new file mode 100644 index 0000000000..af5cfc44db --- /dev/null +++ b/doc/bugs/Default_startup_command/comment_2_296de7f38161ebc21da954d46c924c98._comment @@ -0,0 +1,8 @@ +[[!comment format=mdwn + username="git-annex@53a027ebda399714df9272a135f22ac774f3c9f1" + nickname="git-annex" + subject="small correction to the above" + date="2015-07-26T10:33:28Z" + content=""" +i can not start the webapp via the menu (firefox opens but cant connect to the webapp) but if i do git annex webapp manually in the terminal it works as expected +"""]] diff --git a/doc/bugs/Default_startup_command/comment_3_f388be6121f3449f28aa3d6ca53fa88d._comment b/doc/bugs/Default_startup_command/comment_3_f388be6121f3449f28aa3d6ca53fa88d._comment new file mode 100644 index 0000000000..eda3046276 --- /dev/null +++ b/doc/bugs/Default_startup_command/comment_3_f388be6121f3449f28aa3d6ca53fa88d._comment @@ -0,0 +1,14 @@ +[[!comment format=mdwn + username="joey" + subject="""comment 3""" + date="2018-05-08T18:54:32Z" + content=""" +I'm closing this bug report because the git-annex Android app that it +was reported on is now deprecated. Instead, we have +a way to run the regular git-annex build for linux on Android in termux: + + +There were a lot of problems with the way the git-annex Android app was +put together, and I hope this new approach avoids them. If you try it and +still have the bug you reported, please followup and I'll reopen it. +"""]] diff --git a/doc/bugs/Delete_data__47__update_location_log_when_a_special_remote_fails_to_fsck.mdwn b/doc/bugs/Delete_data__47__update_location_log_when_a_special_remote_fails_to_fsck.mdwn new file mode 100644 index 0000000000..a7c2c77775 --- /dev/null +++ b/doc/bugs/Delete_data__47__update_location_log_when_a_special_remote_fails_to_fsck.mdwn @@ -0,0 +1,72 @@ +### Please describe the problem. + +After all the work I did [here](https://git-annex.branchable.com/bugs/Hybrid_encryption_can__39__t_generate_the_right_key_after_moving_files/), I discvovered that I have a Git Annex-encrypted special remote that has some corrupted data on it. I can run `git annex fsck --from=remotename` and have Git Annex complain about all the files that have failed to fsck. But Git Annex is not updating the location log. It still thinks the data is in the special remote, after fscking it and not getting the right data out. + +This makes sense if the failure was from a download corrupted in transit, but I think the files are actually corrupted in the remote. How do I make `git annex fsck` update the location log to say that files aren't there when they fail to fsck? Alternately, how do I get a nice list of all the filenames that failed to fsck, so I can have a script drop them from the offending remote? + +### What steps will reproduce the problem? + +1. Make a Git Annex encrypted special remote. + +2. Put some data in it. + +3. Modify the encrypted data, to corrupt it. + +4. Use `git annex fsck` on the file you broke. + +5. See that `git annex whereis` still thinks the file is in the special remote. + +### What version of git-annex are you using? On what operating system? + +This is Git Annex 6.20170408+gitg804f06baa-1~ndall+1 on Ubuntu 14.04. + +### Please provide any additional information below. + +[[!format sh """ +# If you can, paste a complete transcript of the problem occurring here. +# If the problem is with the git-annex assistant, paste in .git/annex/daemon.log + + +$ git annex whereis info.txt +whereis info.txt (6 copies) + ... + ... + 21a0c4ba-7255-4a9e-9baa-c638f7df68e5 -- gdrive [amazon] + ... + ... + ... +ok +$ git annex fsck --from=amazon info.txt +... +gpg: decryption failed: bad key +2017/06/13 20:14:54 Local file system at /tmp/tmp.Cu2Dsk0jY3: Waiting for checks to finish +2017/06/13 20:14:54 Local file system at /tmp/tmp.Cu2Dsk0jY3: Waiting for transfers to finish +2017/06/13 20:14:54 Attempt 1/3 failed with 0 errors and: directory not found +2017/06/13 20:14:54 Local file system at /tmp/tmp.Cu2Dsk0jY3: Waiting for checks to finish +2017/06/13 20:14:54 Local file system at /tmp/tmp.Cu2Dsk0jY3: Waiting for transfers to finish +2017/06/13 20:14:54 Attempt 2/3 failed with 0 errors and: directory not found +2017/06/13 20:14:55 Local file system at /tmp/tmp.Cu2Dsk0jY3: Waiting for checks to finish +2017/06/13 20:14:55 Local file system at /tmp/tmp.Cu2Dsk0jY3: Waiting for transfers to finish + failed to download file from remote +2017/06/13 20:14:55 Attempt 3/3 failed with 0 errors and: directory not found +2017/06/13 20:14:55 Failed to copy: directory not found +git-annex: fsck: 1 failed +$ git annex whereis info.txt +whereis info.txt (6 copies) + ... + ... + 21a0c4ba-7255-4a9e-9baa-c638f7df68e5 -- gdrive [amazon] + ... + ... + ... +ok + + +# End of transcript or log. +"""]] + +### Have you had any luck using git-annex before? (Sometimes we get tired of reading bug reports all day and a lil' positive end note does wonders) + +Yeah, it works great! If not for it I would not have noticed this data corruption until it was too late. + + diff --git a/doc/bugs/Delete_data__47__update_location_log_when_a_special_remote_fails_to_fsck/comment_1_203ebe6fa1bb8d3c6e7c0b948fc7dd6b._comment b/doc/bugs/Delete_data__47__update_location_log_when_a_special_remote_fails_to_fsck/comment_1_203ebe6fa1bb8d3c6e7c0b948fc7dd6b._comment new file mode 100644 index 0000000000..60fd9b8df8 --- /dev/null +++ b/doc/bugs/Delete_data__47__update_location_log_when_a_special_remote_fails_to_fsck/comment_1_203ebe6fa1bb8d3c6e7c0b948fc7dd6b._comment @@ -0,0 +1,33 @@ +[[!comment format=mdwn + username="joey" + subject="""comment 1""" + date="2017-06-26T17:14:56Z" + content=""" +`fsck --from remote` is supposed to update the location log when it +determines that the remote does not contain the file. + +But in your case, the decryption failure appears to fsck as a transfer +failure, which as you note can be transient. So it doesn't update the +location log. + +It seems that what's needed is different errors to be returned when +download fails, vs when download succeeds but decryption/verification fails. +Then fsck could mark the file as not being present in the remote +in the latter case. + +Although, that would leave the presumably corrupted encrypted data in the +remote. (Unless fsck also tried to delete it.) + +Also, decryption can fail for other reasons, eg missing gpg keys, +and in such a case, it would be bad for fsck to decide that the remote +didn't contain any content! (And super bad for it to delete it from the +remote!!) + +So hmm, I'm not sure about that idea. + +Your idea of getting a list of files that fsck failed to download +is certianly useful. Perhaps a good way would be to make `git annex fsck +--from remote --json` work, then the json output could be parsed to get a list of +files, and you could use `git annex drop --from remote` to remove the bad +data. That was the easiest possible thing, so I've made that change. +"""]] diff --git a/doc/bugs/Direct_mode_repositories_still_use_symlinks_sometimes.mdwn b/doc/bugs/Direct_mode_repositories_still_use_symlinks_sometimes.mdwn new file mode 100644 index 0000000000..10b25d774b --- /dev/null +++ b/doc/bugs/Direct_mode_repositories_still_use_symlinks_sometimes.mdwn @@ -0,0 +1,36 @@ +### Please describe the problem. + +When a repository is set in direct mode it will still replace files with symlinks when it becomes aware of a change but still hasn't been able to sync the file contents. This can create repositories that are temporarily unusable with files replaced with broken symlinks. + +### What steps will reproduce the problem? + +1. Create two repositories with each other as remotes +2. Run the assistant on both +3. Create some file changes in one and watch the directory in another. +4. For a brief (or sometimes long) time the destination repository will have it's old version of the file replaced by a broken symlink + +This is particularly noticeable when using XMPP as it can often be the case that the two repositories can't connect to each other directly but can talk through XMPP. This breaks using git-annex in direct mode for things like having a synced config directory across machines. Something like having "~/.bashrc" linked into "~/annex-repository/bashrc", doesn't work as there will be times when a machine is broken because .bashrc is linked to a broken symlink while it fetches a new version. + +The desired behavior would be to have git-annex in direct mode only replace older versions of files with newer versions of files. + +### What version of git-annex are you using? On what operating system? + +[[!format sh """ +$ git annex version +git-annex version: 4.20130516.1 +build flags: Assistant Webapp Pairing Testsuite S3 WebDAV Inotify DBus XMPP +local repository version: 4 +default repository version: 3 +supported repository versions: 3 4 +upgrade supported from repository versions: 0 1 2 +$ lsb_release -a +No LSB modules are available. +Distributor ID: Ubuntu +Description: Ubuntu 12.04.2 LTS +Release: 12.04 +Codename: precise +"""]] + +[[!meta title="git-annex does not guarantee modified files are available when merging with a new version of a repository"]] + +[[!tag confirmed]] diff --git a/doc/bugs/Disconcerting_warning_from_git-annex.mdwn b/doc/bugs/Disconcerting_warning_from_git-annex.mdwn new file mode 100644 index 0000000000..ef662441c1 --- /dev/null +++ b/doc/bugs/Disconcerting_warning_from_git-annex.mdwn @@ -0,0 +1,8 @@ +I did a "git annex add" of a bunch of files on a storage server with low IOPS, and saw this: + + git-annex: /tank/Media/Pictures/.git/annex/tmp/430_32b_SHA256E-s4464838--c1785a76ee1451f602e93c99c147e214705004e541de8256d74a3be3717d15e5.jpg.log: openBinaryFile: resource busy (file is locked) +failed + +How is that even possible, when the server is doing nothing else? + +[[!tag moreinfo]] diff --git a/doc/bugs/Disconcerting_warning_from_git-annex/comment_1_58cebd377bfdf247b6c4fee27a3ba461._comment b/doc/bugs/Disconcerting_warning_from_git-annex/comment_1_58cebd377bfdf247b6c4fee27a3ba461._comment new file mode 100644 index 0000000000..3076b38de1 --- /dev/null +++ b/doc/bugs/Disconcerting_warning_from_git-annex/comment_1_58cebd377bfdf247b6c4fee27a3ba461._comment @@ -0,0 +1,8 @@ +[[!comment format=mdwn + username="http://joeyh.name/" + ip="4.152.108.211" + subject="comment 1" + date="2013-01-05T21:13:09Z" + content=""" +Is this a Solaris system? +"""]] diff --git a/doc/bugs/Disconcerting_warning_from_git-annex/comment_2_dc7407044d4c739d05248300c58d8ef2._comment b/doc/bugs/Disconcerting_warning_from_git-annex/comment_2_dc7407044d4c739d05248300c58d8ef2._comment new file mode 100644 index 0000000000..96d495e66c --- /dev/null +++ b/doc/bugs/Disconcerting_warning_from_git-annex/comment_2_dc7407044d4c739d05248300c58d8ef2._comment @@ -0,0 +1,8 @@ +[[!comment format=mdwn + username="https://me.yahoo.com/a/2grhJvAC049fJnvALDXek.6MRZMTlg--#eec89" + nickname="John" + subject="comment 2" + date="2013-01-20T03:38:43Z" + content=""" +No, it's CentOS 6.3 x86_64., +"""]] diff --git a/doc/bugs/Disconcerting_warning_from_git-annex/comment_3_13999207f4ddac2f9c345415f25f7ada._comment b/doc/bugs/Disconcerting_warning_from_git-annex/comment_3_13999207f4ddac2f9c345415f25f7ada._comment new file mode 100644 index 0000000000..142e6ecd11 --- /dev/null +++ b/doc/bugs/Disconcerting_warning_from_git-annex/comment_3_13999207f4ddac2f9c345415f25f7ada._comment @@ -0,0 +1,28 @@ +[[!comment format=mdwn + username="http://joeyh.name/" + ip="209.250.56.102" + subject="moreinfo" + date="2014-03-19T20:49:47Z" + content=""" +What I don't understand about this is, how does `open` fail due to a file being locked? This is Linux, it doesn't have mandatory locking that I know of, and git-annex certianly doesn't use such a thing. + +I really need a way to reproduce this and/or a strace. As it is, I've never seen this reported by anyone else and don't understand the failure mode at all. + +The relevant part of the code seems to be here: + +[[!format haskell \"\"\" +setJournalFile :: JournalLocked -> FilePath -> String -> Annex () +setJournalFile _jl file content = do + tmp <- fromRepo gitAnnexTmpMiscDir + createAnnexDirectory =<< fromRepo gitAnnexJournalDir + createAnnexDirectory tmp + -- journal file is written atomically + jfile <- fromRepo $ journalFile file + let tmpfile = tmp takeFileName jfile + liftIO $ do + writeBinaryFile tmpfile content + moveFile tmpfile jfile +\"\"\"]] + +While there is some ctnl locking going on, it locks a special sentinal file, not the file it's writing to. +"""]] diff --git a/doc/bugs/Doesn__39__t_build_with_hinotify__44__0.3.10___47___fsnotify__44__0.2.1.2.mdwn b/doc/bugs/Doesn__39__t_build_with_hinotify__44__0.3.10___47___fsnotify__44__0.2.1.2.mdwn new file mode 100644 index 0000000000..9c81ffe172 --- /dev/null +++ b/doc/bugs/Doesn__39__t_build_with_hinotify__44__0.3.10___47___fsnotify__44__0.2.1.2.mdwn @@ -0,0 +1,196 @@ +### Please describe the problem. +Build failure with hinotify,0.3.10 / fsnotify,0.2.1.2 + +### What version of git-annex are you using? On what operating system? +git-annex 6.20180427 on Arch x86_64. + +### Please provide any additional information below. + +[[!format sh """ +[124 of 594] Compiling Utility.DirWatcher.INotify ( Utility/DirWatcher/INotify.hs, dist/build/git-annex/git-annex-tmp/Utility/DirWatcher/INotify.dyn_o ) + +Utility/DirWatcher/INotify.hs:58:54: error: + • Couldn't match type ‘[Char]’ + with ‘Data.ByteString.Internal.ByteString’ + Expected type: System.Posix.ByteString.FilePath.RawFilePath + Actual type: FilePath + • In the third argument of ‘addWatch’, namely ‘dir’ + In the first argument of ‘void’, namely + ‘(addWatch i watchevents dir handler)’ + In the first argument of ‘catchIO’, namely + ‘void (addWatch i watchevents dir handler)’ + | +58 | void (addWatch i watchevents dir handler) + | ^^^ + +Utility/DirWatcher/INotify.hs:97:41: error: + • Couldn't match type ‘Data.ByteString.Internal.ByteString’ + with ‘[Char]’ + Expected type: FilePath + Actual type: System.Posix.ByteString.FilePath.RawFilePath + • In the first argument of ‘indir’, namely ‘f’ + In the second argument of ‘($)’, namely ‘indir f’ + In the expression: recurse $ indir f + | +97 | | isd = recurse $ indir f + | ^ + +Utility/DirWatcher/INotify.hs:99:41: error: + • Couldn't match type ‘Data.ByteString.Internal.ByteString’ + with ‘[Char]’ + Expected type: FilePath + Actual type: System.Posix.ByteString.FilePath.RawFilePath + • In the first argument of ‘getstatus’, namely ‘f’ + In a stmt of a 'do' block: ms <- getstatus f + In the expression: + do ms <- getstatus f + case ms of + Just s + | isSymbolicLink s + -> when (hashook addSymlinkHook) $ runhook addSymlinkHook f ms + | isRegularFile s -> when (hashook addHook) $ runhook addHook f ms + _ -> noop + | +99 | ms <- getstatus f + | ^ + +Utility/DirWatcher/INotify.hs:104:80: error: + • Couldn't match type ‘Data.ByteString.Internal.ByteString’ + with ‘[Char]’ + Expected type: FilePath + Actual type: System.Posix.ByteString.FilePath.RawFilePath + • In the second argument of ‘runhook’, namely ‘f’ + In the second argument of ‘($)’, namely + ‘runhook addSymlinkHook f ms’ + In the expression: + when (hashook addSymlinkHook) $ runhook addSymlinkHook f ms + | +104 | runhook addSymlinkHook f ms + | ^ + +Utility/DirWatcher/INotify.hs:107:73: error: + • Couldn't match type ‘Data.ByteString.Internal.ByteString’ + with ‘[Char]’ + Expected type: FilePath + Actual type: System.Posix.ByteString.FilePath.RawFilePath + • In the second argument of ‘runhook’, namely ‘f’ + In the second argument of ‘($)’, namely ‘runhook addHook f ms’ + In the expression: when (hashook addHook) $ runhook addHook f ms + | +107 | runhook addHook f ms + | ^ + +Utility/DirWatcher/INotify.hs:112:67: error: + • Couldn't match type ‘Data.ByteString.Internal.ByteString’ + with ‘[Char]’ + Expected type: FilePath + Actual type: System.Posix.ByteString.FilePath.RawFilePath + • In the third argument of ‘checkfiletype’, namely ‘f’ + In the expression: checkfiletype isRegularFile addHook f + In an equation for ‘go’: + go (Closed {isDirectory = False, maybeFilePath = Just f}) + = checkfiletype isRegularFile addHook f + | +112 | checkfiletype Files.isRegularFile addHook f + | ^ + +Utility/DirWatcher/INotify.hs:115:46: error: + • Couldn't match type ‘Data.ByteString.Internal.ByteString’ + with ‘[Char]’ + Expected type: FilePath + Actual type: System.Posix.ByteString.FilePath.RawFilePath + • In the first argument of ‘scan’, namely ‘f’ + In the expression: scan f + In an equation for ‘go’: go (MovedIn {filePath = f}) = scan f + | +115 | go (MovedIn { filePath = f }) = scan f + | ^ + +Utility/DirWatcher/INotify.hs:117:44: error: + • Couldn't match type ‘Data.ByteString.Internal.ByteString’ + with ‘[Char]’ + Expected type: FilePath + Actual type: System.Posix.ByteString.FilePath.RawFilePath + • In the second argument of ‘runhook’, namely ‘f’ + In the expression: runhook delDirHook f Nothing + In an equation for ‘go’: + go (MovedOut {isDirectory = isd, filePath = f}) + | isd = runhook delDirHook f Nothing + | otherwise = runhook delHook f Nothing + | +117 | | isd = runhook delDirHook f Nothing + | ^ + +Utility/DirWatcher/INotify.hs:118:47: error: + • Couldn't match type ‘Data.ByteString.Internal.ByteString’ + with ‘[Char]’ + Expected type: FilePath + Actual type: System.Posix.ByteString.FilePath.RawFilePath + • In the second argument of ‘runhook’, namely ‘f’ + In the expression: runhook delHook f Nothing + In an equation for ‘go’: + go (MovedOut {isDirectory = isd, filePath = f}) + | isd = runhook delDirHook f Nothing + | otherwise = runhook delHook f Nothing + | +118 | | otherwise = runhook delHook f Nothing + | ^ + +Utility/DirWatcher/INotify.hs:124:54: error: + • Couldn't match type ‘Data.ByteString.Internal.ByteString’ + with ‘[Char]’ + Expected type: FilePath + Actual type: System.Posix.ByteString.FilePath.RawFilePath + • In the second argument of ‘runhook’, namely ‘f’ + In the second argument of ‘($)’, namely + ‘runhook delDirHook f Nothing’ + In the expression: guarded $ runhook delDirHook f Nothing + | +124 | | isd = guarded $ runhook delDirHook f Nothing + | ^ + +Utility/DirWatcher/INotify.hs:125:57: error: + • Couldn't match type ‘Data.ByteString.Internal.ByteString’ + with ‘[Char]’ + Expected type: FilePath + Actual type: System.Posix.ByteString.FilePath.RawFilePath + • In the second argument of ‘runhook’, namely ‘f’ + In the second argument of ‘($)’, namely ‘runhook delHook f Nothing’ + In the expression: guarded $ runhook delHook f Nothing + | +125 | | otherwise = guarded $ runhook delHook f Nothing + | ^ + +Utility/DirWatcher/INotify.hs:127:58: error: + • Couldn't match type ‘Data.ByteString.Internal.ByteString’ + with ‘[Char]’ + Expected type: FilePath + Actual type: System.Posix.ByteString.FilePath.RawFilePath + • In the second argument of ‘filetype’, namely ‘f’ + In the first argument of ‘unlessM’, namely + ‘(filetype (const True) f)’ + In the expression: unlessM (filetype (const True) f) + | +127 | guarded = unlessM (filetype (const True) f) + | ^ + +Utility/DirWatcher/INotify.hs:130:50: error: + • Couldn't match type ‘Data.ByteString.Internal.ByteString’ + with ‘[Char]’ + Expected type: FilePath + Actual type: System.Posix.ByteString.FilePath.RawFilePath + • In the second argument of ‘runhook’, namely ‘f’ + In the expression: runhook modifyHook f Nothing + In an equation for ‘go’: + go (Modified {isDirectory = isd, maybeFilePath = Just f}) + | isd = noop + | otherwise = runhook modifyHook f Nothing + | +130 | | otherwise = runhook modifyHook f Nothing + | ^ + +"""]] + +### Have you had any luck using git-annex before? (Sometimes we get tired of reading bug reports all day and a lil' positive end note does wonders) + +Yes! diff --git a/doc/bugs/Empty_files_make_git_status_slow.mdwn b/doc/bugs/Empty_files_make_git_status_slow.mdwn new file mode 100644 index 0000000000..64e4d171ac --- /dev/null +++ b/doc/bugs/Empty_files_make_git_status_slow.mdwn @@ -0,0 +1,28 @@ +### Please describe the problem. + +[[!meta title="Empty files make git status slow in v6 repo"]] + +When empty files are committed to a repository, git status becomes slow because git annex smudge is run for every empty file under git annex even with clean HEAD. I know git annex is for large files (rather than infinity small ones), but I'm using it to manage a GRASS GIS database, which oddly uses empty files for some things:/ + +### What steps will reproduce the problem? +``` +# slow +touch emptyfile +git add emptyfile +git commit -a -m 'Added empty file.' +GIT_TRACE=1 git status + +# fast +cat 1 > emptyfile +git commit -a -m 'Added a 1 to emptyfile.' +GIT_TRACE=1 git status +``` + +### What version of git-annex are you using? On what operating system? +- git-annex version: 6.20170228-g7a32e08c4 +- operating system: linux x86_64 (SLE 12.2) +- local repository version: 6 + + +### Have you had any luck using git-annex before? (Sometimes we get tired of reading bug reports all day and a lil' positive end note does wonders) +Yes, git with git annex has revolutionised my scientific project file organisation and thats why I want to improve it. diff --git a/doc/bugs/Empty_files_make_git_status_slow/comment_1_67662686361d345b3b82d1c80adc2452._comment b/doc/bugs/Empty_files_make_git_status_slow/comment_1_67662686361d345b3b82d1c80adc2452._comment new file mode 100644 index 0000000000..8c16331523 --- /dev/null +++ b/doc/bugs/Empty_files_make_git_status_slow/comment_1_67662686361d345b3b82d1c80adc2452._comment @@ -0,0 +1,15 @@ +[[!comment format=mdwn + username="joey" + subject="""comment 1""" + date="2017-03-02T17:20:24Z" + content=""" +I don't know why git is repeatedly smudging the empty file. +(I was able to reproduce it.) + +git-annex does not treat an empty file any differently than any other file. +It seems that this is probably a bug in git. + +Note that v6 git repos are known to be innefficient due to git's smudge +interface not being very good. This is why they are not yet used by +default. +"""]] diff --git a/doc/bugs/Empty_files_make_git_status_slow/comment_2_39876be38934e9056e9aa2a9fcdeaedf._comment b/doc/bugs/Empty_files_make_git_status_slow/comment_2_39876be38934e9056e9aa2a9fcdeaedf._comment new file mode 100644 index 0000000000..777f26ac92 --- /dev/null +++ b/doc/bugs/Empty_files_make_git_status_slow/comment_2_39876be38934e9056e9aa2a9fcdeaedf._comment @@ -0,0 +1,8 @@ +[[!comment format=mdwn + username="Michel" + avatar="http://cdn.libravatar.org/avatar/cf9fdab9aa34af56f769bf584c8e7011" + subject="Thanks" + date="2017-03-09T07:34:25Z" + content=""" +Thanks for the looking into it. I guess it will have to be sorted out at the git end then or I will need to fill my empty files with a single character. I must say I love the v6 repo tho and hope it will become the default soon. +"""]] diff --git a/doc/bugs/Empty_files_make_git_status_slow/comment_3_17b400bc03eb176f63009b9e01a862f4._comment b/doc/bugs/Empty_files_make_git_status_slow/comment_3_17b400bc03eb176f63009b9e01a862f4._comment new file mode 100644 index 0000000000..892eb6b728 --- /dev/null +++ b/doc/bugs/Empty_files_make_git_status_slow/comment_3_17b400bc03eb176f63009b9e01a862f4._comment @@ -0,0 +1,13 @@ +[[!comment format=mdwn + username="Michel" + avatar="http://cdn.libravatar.org/avatar/cf9fdab9aa34af56f769bf584c8e7011" + subject="Solution" + date="2017-03-11T00:44:18Z" + content=""" +I found a relatively simple solution by setting the following in .gitattributes: + +``` +* annex.largefiles=(largerthan=0kb) +``` + +"""]] diff --git a/doc/bugs/Empty_files_make_git_status_slow/comment_4_d3a05649e8d7d5d62cc3e80ce0f45fe4._comment b/doc/bugs/Empty_files_make_git_status_slow/comment_4_d3a05649e8d7d5d62cc3e80ce0f45fe4._comment new file mode 100644 index 0000000000..9c43129ea6 --- /dev/null +++ b/doc/bugs/Empty_files_make_git_status_slow/comment_4_d3a05649e8d7d5d62cc3e80ce0f45fe4._comment @@ -0,0 +1,20 @@ +[[!comment format=mdwn + username="joey" + subject="""comment 4""" + date="2018-08-27T16:17:32Z" + content=""" +Verified to still be the case with current git-annex +and git 2.18.0. + +I suppose one workaround would be to default annex.largefiles to your +setting. + +But, the user might want to use `git annex export`, which currently exports +only annexed file, not in-git files. Or there might be a workflow with +git-annex metadata, which likewise can only be added to annexed files. Such +a special case by default may be more trouble than it's worth. + +I'm guessing that git uses 0 size in the index to indicate it doesn't know +the stat information for the file, or something like that. Perhaps git +could be changed to use 0 inode instead. +"""]] diff --git a/doc/bugs/Empty_folders_don__39__t_get_remove.mdwn b/doc/bugs/Empty_folders_don__39__t_get_remove.mdwn new file mode 100644 index 0000000000..8562f23393 --- /dev/null +++ b/doc/bugs/Empty_folders_don__39__t_get_remove.mdwn @@ -0,0 +1,4 @@ +### Please describe the problem. +When you rename, move or delete a folder (with files in it) the old folder doesn't get deleted in the other clients + +[[!tag moreinfo]] diff --git a/doc/bugs/Empty_folders_don__39__t_get_remove/comment_1_9f41638299c214b2ee13f23ab41349da._comment b/doc/bugs/Empty_folders_don__39__t_get_remove/comment_1_9f41638299c214b2ee13f23ab41349da._comment new file mode 100644 index 0000000000..e32059fb62 --- /dev/null +++ b/doc/bugs/Empty_folders_don__39__t_get_remove/comment_1_9f41638299c214b2ee13f23ab41349da._comment @@ -0,0 +1,10 @@ +[[!comment format=mdwn + username="http://joeyh.name/" + ip="209.250.56.87" + subject="comment 1" + date="2013-12-12T19:16:34Z" + content=""" +I can't reproduce this. Tested in direct mode (which I assume you're using, but you didn't say), using both command-line `git annex sync`, and using the assistant. In all cases, when the last file in a directory was removed, the directory was deleted. + +Please write back with a version number, and an example showing the problem happening. +"""]] diff --git a/doc/bugs/Empty_folders_don__39__t_get_remove/comment_2_35f59adc30fe743221f29bbfecff639b._comment b/doc/bugs/Empty_folders_don__39__t_get_remove/comment_2_35f59adc30fe743221f29bbfecff639b._comment new file mode 100644 index 0000000000..e471d33664 --- /dev/null +++ b/doc/bugs/Empty_folders_don__39__t_get_remove/comment_2_35f59adc30fe743221f29bbfecff639b._comment @@ -0,0 +1,15 @@ +[[!comment format=mdwn + username="jgoerzen" + subject="Experienced this" + date="2016-06-14T22:08:43Z" + content=""" +Hi, + +I experienced this just now. Have a git-annex assistant repo set up (so using direct mode). + +On one machine, I mv'd entire directories out of the git-annex repo. + +The other machine saw the files deleted, but the directories remained. + +On the machine on which I mv'd directories out, git-annex actually re-created some of them with symlinks to now-nonexistant hashes! I have no idea why it did that. +"""]] diff --git a/doc/bugs/Empty_folders_don__39__t_get_remove/comment_3_ccc703f8667805ce2886719c57e76ef8._comment b/doc/bugs/Empty_folders_don__39__t_get_remove/comment_3_ccc703f8667805ce2886719c57e76ef8._comment new file mode 100644 index 0000000000..93bb20f255 --- /dev/null +++ b/doc/bugs/Empty_folders_don__39__t_get_remove/comment_3_ccc703f8667805ce2886719c57e76ef8._comment @@ -0,0 +1,7 @@ +[[!comment format=mdwn + username="jgoerzen" + subject="Version" + date="2016-06-14T22:35:59Z" + content=""" +Forgot to add: 5.20151208 +"""]] diff --git a/doc/bugs/Enable__47__paus_syncing_to_remote_ssh_server_with_multiple_directories.mdwn b/doc/bugs/Enable__47__paus_syncing_to_remote_ssh_server_with_multiple_directories.mdwn new file mode 100644 index 0000000000..823466f65e --- /dev/null +++ b/doc/bugs/Enable__47__paus_syncing_to_remote_ssh_server_with_multiple_directories.mdwn @@ -0,0 +1,19 @@ +What steps will reproduce the problem? + +Add two remote repositories to same server using ssh. Each directory is added to repositories list and starts syncing. +After both repositories are scanned click on "syncing enabled" to disable syncing for one of the two remote repositories. + +What is the expected output? What do you see instead? + +Only remote repository which i clicked should stop syncing, but both remote repos start to stop syncing. +Its also working the other way around (enable -> pause -> enable -> ..) + +What version of git-annex are you using? On what operating system? + +Version: 4.20130314, Debian + +Please provide any additional information below. + +I am an "webinterface only" user. + +[[!tag /design/assistant moreinfo]] diff --git a/doc/bugs/Enable__47__paus_syncing_to_remote_ssh_server_with_multiple_directories/comment_1_e8affeca873c2ef73255f8f77e0ac16f._comment b/doc/bugs/Enable__47__paus_syncing_to_remote_ssh_server_with_multiple_directories/comment_1_e8affeca873c2ef73255f8f77e0ac16f._comment new file mode 100644 index 0000000000..d25a46a98a --- /dev/null +++ b/doc/bugs/Enable__47__paus_syncing_to_remote_ssh_server_with_multiple_directories/comment_1_e8affeca873c2ef73255f8f77e0ac16f._comment @@ -0,0 +1,10 @@ +[[!comment format=mdwn + username="http://joeyh.name/" + nickname="joey" + subject="comment 1" + date="2013-03-16T21:56:37Z" + content=""" +Hmm, I can't quite reproduce this, but I did just fix a bug that, among other manifestations, caused the display of whether syncing is enabled for a remote to be wrong sometimes. + +Can you send a copy of your repository's .git/config file to this bug report? (That's `annex/.git/config`) +"""]] diff --git a/doc/bugs/Encoding_error_in_webapp.mdwn b/doc/bugs/Encoding_error_in_webapp.mdwn new file mode 100644 index 0000000000..ee57e8b06c --- /dev/null +++ b/doc/bugs/Encoding_error_in_webapp.mdwn @@ -0,0 +1,24 @@ +### Please describe the problem. + +In the webapp, file names with UTF-8 characters are displayed correctly when they are in the queue but not when they are being transferred. + + +### What steps will reproduce the problem? + +- Create a file with accented characters (éèà...) in their path +- Run the webapp :) + + +### What version of git-annex are you using? On what operating system? + +git-annex version: 6.20170612-ge4100fd60e (on Arch Linux) + + +### Please provide any additional information below. + +![Screenshot](http://i.imgur.com/xHgdVKc.png) + + +### Have you had any luck using git-annex before? (Sometimes we get tired of reading bug reports all day and a lil' positive end note does wonders) + +Oh yes! I love git-annex :) I've written the hubiC special remote for git-annex, the zsh completion, contributed to the crowdfunding campaigns, and I'm a supporter on Patreon :) diff --git a/doc/bugs/Error_running_git-annex.app_on_OS_X_10.13.1___40__17B1003__41__.mdwn b/doc/bugs/Error_running_git-annex.app_on_OS_X_10.13.1___40__17B1003__41__.mdwn new file mode 100644 index 0000000000..7c48be0e39 --- /dev/null +++ b/doc/bugs/Error_running_git-annex.app_on_OS_X_10.13.1___40__17B1003__41__.mdwn @@ -0,0 +1,29 @@ +### Please describe the problem. +git-annex doesn't start. + +### What steps will reproduce the problem? +launch git-annex from terminal (if it is ran from GUI it doesn't open without notifying any error) + +### What version of git-annex are you using? On what operating system? +Version: 6.20171128-g58b04cd2e on OS X 10.13.1 (17B1003) + +### Please provide any additional information below. +I solved the issue by manually changing the file /Applications/git-annex.app/Contents/MacOS/bundle/B which is an alias pointing to the file /Applications/git-annex.app/Contents/MacOS/bundle/usr/lib/libz.1.dylib with a new alias having the same name 'B' which link to /usr/lib/libz.1.dylb. +I'm pretty sure this is not an optimal and general solution, but it is useful in order to prove what's the problem. Further, for it worked. + +[[!format sh """ +# If you can, paste a complete transcript of the problem occurring here. +# If the problem is with the git-annex assistant, paste in .git/annex/daemon.log +dyld: Symbol not found: _inflateValidate + Referenced from: /System/Library/Frameworks/ImageIO.framework/Versions/A/Resources/libPng.dylib + Expected in: /Applications/git-annex.app/Contents/MacOS/bundle/B + in /System/Library/Frameworks/ImageIO.framework/Versions/A/Resources/libPng.dylib +Abort trap: 6 + +# End of transcript or log. +"""]] + +### Have you had any luck using git-annex before? (Sometimes we get tired of reading bug reports all day and a lil' positive end note does wonders) + +> OSX autobuild is updated and no longer contains the copy of libz. +> [[done]] diff --git a/doc/bugs/Error_running_git-annex.app_on_OS_X_10.13.1___40__17B1003__41__/comment_1_0f013027500f03b52e021716e493fbff._comment b/doc/bugs/Error_running_git-annex.app_on_OS_X_10.13.1___40__17B1003__41__/comment_1_0f013027500f03b52e021716e493fbff._comment new file mode 100644 index 0000000000..c6784f79b4 --- /dev/null +++ b/doc/bugs/Error_running_git-annex.app_on_OS_X_10.13.1___40__17B1003__41__/comment_1_0f013027500f03b52e021716e493fbff._comment @@ -0,0 +1,12 @@ +[[!comment format=mdwn + username="rameshvenk" + avatar="http://cdn.libravatar.org/avatar/185911818a3eb3795072b1aa1256449c" + subject="dyld: Symbol not found: _inflateValidate" + date="2018-03-04T03:59:42Z" + content=""" +Can confirm that this fix did resolve the issue. My version of git annex is as follows + +git-annex version: 6.20180226-g15a11fedf + +running on 10.13.3 +"""]] diff --git a/doc/bugs/Error_running_git-annex.app_on_OS_X_10.13.1___40__17B1003__41__/comment_2_e15b265d67cad6fb914f626e646c2fe7._comment b/doc/bugs/Error_running_git-annex.app_on_OS_X_10.13.1___40__17B1003__41__/comment_2_e15b265d67cad6fb914f626e646c2fe7._comment new file mode 100644 index 0000000000..ebf5867e25 --- /dev/null +++ b/doc/bugs/Error_running_git-annex.app_on_OS_X_10.13.1___40__17B1003__41__/comment_2_e15b265d67cad6fb914f626e646c2fe7._comment @@ -0,0 +1,20 @@ +[[!comment format=mdwn + username="joey" + subject="""comment 2""" + date="2018-03-05T14:40:08Z" + content=""" +This is a bit puzzling, since git-annex does not link to libpng. + +I think what must be going on os that Cocoa, which git-annex does link to, +links to libpng. + +The dmg contains its own copy of zlib, not for git-annex, +which does not directly link to it, but for the embedded copy of curl. +But the linker is told to prefer the bundled libraries, so it tries to use +the bundled zlib with Cocoa. + +Upgrading the OSX autobuilder to 10.13 would be the cleanest fix, but +installing a newer zlib on there with homebrew should also work. +Hmm, but I tried homebrew and their zlib does not contain the +`_inflateValidate` symbol. +"""]] diff --git a/doc/bugs/Error_running_git-annex.app_on_OS_X_10.13.1___40__17B1003__41__/comment_3_1346c5aa68b06d512aced0ef72cc92fc._comment b/doc/bugs/Error_running_git-annex.app_on_OS_X_10.13.1___40__17B1003__41__/comment_3_1346c5aa68b06d512aced0ef72cc92fc._comment new file mode 100644 index 0000000000..cd256b8e4c --- /dev/null +++ b/doc/bugs/Error_running_git-annex.app_on_OS_X_10.13.1___40__17B1003__41__/comment_3_1346c5aa68b06d512aced0ef72cc92fc._comment @@ -0,0 +1,8 @@ +[[!comment format=mdwn + username="joey" + subject="""comment 3""" + date="2018-03-22T15:52:04Z" + content=""" +I've excluded libz.1.dylb from git-annex.apk, on the assumuption that OSX +always ships with libz, and so git-annex does not need to provide it. +"""]] diff --git a/doc/bugs/Errors_on_Android__58___referenceTable_GDEF_length__61__778__44___referenceTable_GSUB_length__61__6388__44___referenceTable_GPOS_length__61__76868.mdwn b/doc/bugs/Errors_on_Android__58___referenceTable_GDEF_length__61__778__44___referenceTable_GSUB_length__61__6388__44___referenceTable_GPOS_length__61__76868.mdwn new file mode 100644 index 0000000000..b7b3ec4900 --- /dev/null +++ b/doc/bugs/Errors_on_Android__58___referenceTable_GDEF_length__61__778__44___referenceTable_GSUB_length__61__6388__44___referenceTable_GPOS_length__61__76868.mdwn @@ -0,0 +1,49 @@ +### Please describe the problem. + +Opening the Android app, the following message is shown: + + referenceTable GDEF length=778 + referenceTable GSUB length=6388 + referenceTable GPOS length=76868 + + [Terminal session finished] + +I am not able to use git-annex. Trying to open a new window in git-annex, I get similar errors: + + [Terminal session finished] + referenceTable GDEF length=778 1 + referenceTable GSUB length=6388 1 + referenceTable GPOS length=76868 1 + referenceTable head length=54 1 + referenceTable GDEF length=670 + referenceTable GSUB length=7168 + referenceTable GPOS length=24560 + referenceTable head length=54 1 + git annex webapp + + +### What steps will reproduce the problem? + +Download the most recent Android 5.0 app (I tried both [[http://downloads.kitenet.net/git-annex/android/current/5.0/git-annex.apk]] and [[http://downloads.kitenet.net/git-annex/autobuild/android/5.0/git-annex.apk]]). +Install it and open it. + +### What version of git-annex are you using? On what operating system? + +OS: Android 5.1.1. +git-annex: I tried the currently available stable 5.0 package (6.20170101), as well as the nightly build one (it shows version 6.20170102). + +### Please provide any additional information below. + +[[!format sh """ +# If you can, paste a complete transcript of the problem occurring here. +# If the problem is with the git-annex assistant, paste in .git/annex/daemon.log + + +# End of transcript or log. +"""]] + +### Have you had any luck using git-annex before? (Sometimes we get tired of reading bug reports all day and a lil' positive end note does wonders) + +Yes, I used it successfully until recently. I remember that I installed a new version of git-annex recently. I used it successfully after the update. However, I am not sure whether I closed and reopened the git-annex app after the update. + +> Closing as this was a bug in the deprecated Android app. [[done]] --[[Joey]] diff --git a/doc/bugs/Errors_on_Android__58___referenceTable_GDEF_length__61__778__44___referenceTable_GSUB_length__61__6388__44___referenceTable_GPOS_length__61__76868/comment_1_0014a7f4cf8d3c39b96bfb93c906c93f._comment b/doc/bugs/Errors_on_Android__58___referenceTable_GDEF_length__61__778__44___referenceTable_GSUB_length__61__6388__44___referenceTable_GPOS_length__61__76868/comment_1_0014a7f4cf8d3c39b96bfb93c906c93f._comment new file mode 100644 index 0000000000..a5da7d13dd --- /dev/null +++ b/doc/bugs/Errors_on_Android__58___referenceTable_GDEF_length__61__778__44___referenceTable_GSUB_length__61__6388__44___referenceTable_GPOS_length__61__76868/comment_1_0014a7f4cf8d3c39b96bfb93c906c93f._comment @@ -0,0 +1,14 @@ +[[!comment format=mdwn + username="joey" + subject="""comment 1""" + date="2018-05-08T18:45:59Z" + content=""" +I'm closing this bug report because the git-annex Android app that it +was reported on is now deprecated. Instead, we have +a way to run the regular git-annex build for linux on Android in termux: + + +There were a lot of problems with the way the git-annex Android app was +put together, and I hope this new approach avoids them. If you try it and +still have the bug you reported, please followup and I'll reopen it. +"""]] diff --git a/doc/bugs/EvilLinker_needs_to_escape_spaces_in_user_name.mdwn b/doc/bugs/EvilLinker_needs_to_escape_spaces_in_user_name.mdwn new file mode 100644 index 0000000000..1abe3e6918 --- /dev/null +++ b/doc/bugs/EvilLinker_needs_to_escape_spaces_in_user_name.mdwn @@ -0,0 +1,18 @@ +### Please describe the problem. + +git-annex build on Windows does not complete when user name has a space in it + +### What steps will reproduce the problem? + +Build git-annex from scratch as described on the Windows page of the guide when running as user Joe Blogs + + +### What version of git-annex are you using? On what operating system? + +Windows 7 Prof, Cygwin install on 28/5/2014 + +### Please provide any additional information below. + +The problem arises in escapeDosPaths of EvilLinker: windows usernames can have spaces in them which also need escaping + +[[!tag confirmed]] diff --git a/doc/bugs/EvilLinker_needs_to_escape_spaces_in_user_name/comment_1_6b289221a65a750444b1b5850df3386c._comment b/doc/bugs/EvilLinker_needs_to_escape_spaces_in_user_name/comment_1_6b289221a65a750444b1b5850df3386c._comment new file mode 100644 index 0000000000..db335239b8 --- /dev/null +++ b/doc/bugs/EvilLinker_needs_to_escape_spaces_in_user_name/comment_1_6b289221a65a750444b1b5850df3386c._comment @@ -0,0 +1,10 @@ +[[!comment format=mdwn + username="http://joeyh.name/" + ip="209.250.56.176" + subject="comment 1" + date="2014-05-28T22:29:05Z" + content=""" +Unfortunately, this is fairly intractable. It would be more productive to fix the [ghc bug](https://ghc.haskell.org/trac/ghc/ticket/8596) that the EvilLinker works around. + +Recommend building in a different directory in the meantime. +"""]] diff --git a/doc/bugs/Excessively_long_temp_names_preventing___34__get__34___on_Windows.mdwn b/doc/bugs/Excessively_long_temp_names_preventing___34__get__34___on_Windows.mdwn new file mode 100644 index 0000000000..4c6c85e514 --- /dev/null +++ b/doc/bugs/Excessively_long_temp_names_preventing___34__get__34___on_Windows.mdwn @@ -0,0 +1,39 @@ +### Please describe the problem. + +Getting a file with a long name on Windows fails due to doubling the size of the path in the "misctmp" directory. + +### What steps will reproduce the problem? + +``` +get [Connectivity and Thought - The Influence of Semantic Network Structure in a Neurodynamical Model of Thinking] marupaka_creativity_NN12.pdf (from origin...) +SHA256-s588307--41bf10d5c3a924808c6219395758354d7b6c4ff0a75e0d1a90e798fac015536c + 588,307 100% 670.38kB/s 0:00:00 (xfr#1, to-chk=0/1) +(checksum...) +git-annex: MoveFileEx "..\\..\\.git\\annex\\tmp\\SHA256-s588307--41bf10d5c3a924808c6219395758354d7b6c4ff0a75e0d1a90e798fac015536c" "..\\..\\.git\\annex\\misctmp\\[Connectivity and Thought - The Influence of Semantic Network Structure in a Neurodynamical Model of Thinking] marupaka_creativ.0\\[Connectivity and Thought - The Influence of Semantic Network Structure in a Neurodynamical Model of Thinking] marupaka_creativ": does not exist (The system cannot find the path specified.) +failed +git-annex: get: 1 failed +``` + +Note that the second argument to MoveFileEx is longer than 256 characters, because the name of the file gets doubled for the temp directory. + + +### What version of git-annex are you using? On what operating system? + +``` +git-annex version: 6.20160427-gd0036b9 +build flags: Assistant Webapp Pairing Testsuite S3(multipartupload) WebDAV ConcurrentOutput TorrentParser Feeds Quvi +key/value backends: SHA256E SHA256 SHA512E SHA512 SHA224E SHA224 SHA384E SHA384 SHA3_256E SHA3_256 SHA3_512E SHA3_512 SHA3_224E SHA3_224 SHA3_384E SHA3_384 SKEIN256E SKEIN256 SKEIN512E SKEIN512 SHA1E SHA1 MD5E MD5 WORM URL +remote types: git gcrypt S3 bup directory rsync web bittorrent webdav tahoe glacier ddar hook external +local repository version: 5 +supported repository versions: 5 6 +upgrade supported from repository versions: 2 3 4 5 +``` + +OS: Windows 10 + + +### Have you had any luck using git-annex before? (Sometimes we get tired of reading bug reports all day and a lil' positive end note does wonders) + +Yes, managing 30000 files, on operating systems other than Windows though... + +> [[fixed|done]] --[[Joey]] diff --git a/doc/bugs/Excessively_long_temp_names_preventing___34__get__34___on_Windows/comment_1_fafaff54e55e0b82df52656d8afb1c3e._comment b/doc/bugs/Excessively_long_temp_names_preventing___34__get__34___on_Windows/comment_1_fafaff54e55e0b82df52656d8afb1c3e._comment new file mode 100644 index 0000000000..b672dea8a7 --- /dev/null +++ b/doc/bugs/Excessively_long_temp_names_preventing___34__get__34___on_Windows/comment_1_fafaff54e55e0b82df52656d8afb1c3e._comment @@ -0,0 +1,8 @@ +[[!comment format=mdwn + username="joey" + subject="""comment 1""" + date="2016-05-06T16:46:22Z" + content=""" +Thanks for reporting this. I can't fix all the too long path problems +on windows currently, but I can fix this particular one at least. +"""]] diff --git a/doc/bugs/Excessively_long_temp_names_preventing___34__get__34___on_Windows/comment_2_7d691fe40bb134d0f270ed3a9cfbf659._comment b/doc/bugs/Excessively_long_temp_names_preventing___34__get__34___on_Windows/comment_2_7d691fe40bb134d0f270ed3a9cfbf659._comment new file mode 100644 index 0000000000..7ba0734027 --- /dev/null +++ b/doc/bugs/Excessively_long_temp_names_preventing___34__get__34___on_Windows/comment_2_7d691fe40bb134d0f270ed3a9cfbf659._comment @@ -0,0 +1,9 @@ +[[!comment format=mdwn + username="kubaello@d561f15ff5c07a78b706b096375cd89d6d706066" + nickname="kubaello" + avatar="http://cdn.libravatar.org/avatar/df2d1d9871b92552b99c3bacf442d739" + subject="comment 2" + date="2017-03-16T01:22:44Z" + content=""" +I came across the same problem today and I think I've found a workaround: [long paths on windows 10 - workaround](https://git-annex.branchable.com/forum/long_paths_on_windows_10_-_workaround/). It turns out Windows 10 allows enabling support for long paths with a single registry modification - unfortunately it is a system level setting, so it can potentially break some other programs. +"""]] diff --git a/doc/bugs/Export_fails_for_files_directly_in_git_repo.mdwn b/doc/bugs/Export_fails_for_files_directly_in_git_repo.mdwn new file mode 100644 index 0000000000..3d9a402ae4 --- /dev/null +++ b/doc/bugs/Export_fails_for_files_directly_in_git_repo.mdwn @@ -0,0 +1,32 @@ +### Please describe the problem. +I'm playing around with the new export function and it's working really fine so far. But just now I ran into problems when `git add`ing a file, see below. + +### What steps will reproduce the problem? + +[[!format sh """ + +$ echo test > file.txt +$ git add file.txt +$ git annex sync +commit +[master a05faa1] git-annex in lykos@**:/tmp/testrepo2 + 1 file changed, 1 insertion(+) + create mode 100644 file.txt +ok +$ git annex export --tracking master --to exp +export exp file.txt +git-annex: Sorry, this file cannot be stored on an external special remote because its key's name contains a space. To avoid this problem, you can run: git-annex migrate --backend=SHA1 and pass it the name of the file +failed +(recording state in git...) +git-annex: export: 1 failed + +"""]] + + +### What version of git-annex are you using? On what operating system? +6.20171026-g43d011a52 on Arch Linux + +### Have you had any luck using git-annex before? (Sometimes we get tired of reading bug reports all day and a lil' positive end note does wonders) +Of course ;) All the time + +> [[fixed|done]] --[[Joey]] diff --git a/doc/bugs/Export_fails_for_files_directly_in_git_repo/comment_1_0077e5e36896116059070e26143f03d0._comment b/doc/bugs/Export_fails_for_files_directly_in_git_repo/comment_1_0077e5e36896116059070e26143f03d0._comment new file mode 100644 index 0000000000..7105d42aaa --- /dev/null +++ b/doc/bugs/Export_fails_for_files_directly_in_git_repo/comment_1_0077e5e36896116059070e26143f03d0._comment @@ -0,0 +1,13 @@ +[[!comment format=mdwn + username="joey" + subject="""comment 1""" + date="2017-10-30T16:11:42Z" + content=""" +The limitation on spaces in key names only impacts +exporting to external special remotes, so "exp" must be some kind +of external special remote. (I was not aware of any that +supported exports yet..) + +I was able to reproduce this using an external special remote, +but not otherwise. +"""]] diff --git a/doc/bugs/Export_fails_for_files_directly_in_git_repo/comment_2_d98990c87e18b0dd99d78e3d136c4094._comment b/doc/bugs/Export_fails_for_files_directly_in_git_repo/comment_2_d98990c87e18b0dd99d78e3d136c4094._comment new file mode 100644 index 0000000000..a6b45f1174 --- /dev/null +++ b/doc/bugs/Export_fails_for_files_directly_in_git_repo/comment_2_d98990c87e18b0dd99d78e3d136c4094._comment @@ -0,0 +1,10 @@ +[[!comment format=mdwn + username="Lykos153" + avatar="http://cdn.libravatar.org/avatar/085df7b04d3408ba23c19f9c49be9ea2" + subject="comment 2" + date="2017-10-30T18:18:56Z" + content=""" +Thanks for your reply. I'm currently working on a python version of [git-annex-remote-gdrive](https://github.com/Lykos153/git-annex-remote-gdrive/tree/v2) and I just completed support for export. That's how I came across this issue. + + +"""]] diff --git a/doc/bugs/Export_fails_for_files_directly_in_git_repo/comment_3_dce8cbd3a216da6b4d09c2445b04d060._comment b/doc/bugs/Export_fails_for_files_directly_in_git_repo/comment_3_dce8cbd3a216da6b4d09c2445b04d060._comment new file mode 100644 index 0000000000..d2fdb3fa93 --- /dev/null +++ b/doc/bugs/Export_fails_for_files_directly_in_git_repo/comment_3_dce8cbd3a216da6b4d09c2445b04d060._comment @@ -0,0 +1,8 @@ +[[!comment format=mdwn + username="joey" + subject="""comment 3""" + date="2017-10-30T18:23:04Z" + content=""" +The problem is fixed in git-annex's master now. Thanks for working on the +special remote and for the testing! +"""]] diff --git a/doc/bugs/Exporting_subdirs_fails.mdwn b/doc/bugs/Exporting_subdirs_fails.mdwn new file mode 100644 index 0000000000..9d21e0a942 --- /dev/null +++ b/doc/bugs/Exporting_subdirs_fails.mdwn @@ -0,0 +1,23 @@ +### Please describe the problem. +git annex won't export subdirectories claiming they don't exist. `git rev-parse` however, returns hashes for said directories which then are accepted by `git annex export`. + + +### What steps will reproduce the problem? + +[[!format sh """ +$ git annex export master:folder --to exp +fatal: Path 'folder:' does not exist in 'master' +git-annex: unknown tree +$ git rev-parse master:folder +943218c141ab2a315b533fa9f92fe610916b5d1b +$ git annex export 943218c141ab2a315b533fa9f92fe610916b5d1b --to exp +export exp testfile2 +ok +(recording state in git...) +"""]] + + +### What version of git-annex are you using? On what operating system? +git-annex version: 6.20171026-g43d011a52 on Arch Linux + +> [[fixed|done]] --[[Joey]] diff --git a/doc/bugs/Exporting_subdirs_fails/comment_1_2113ddd317cd5d11cf1d7a56b30bb452._comment b/doc/bugs/Exporting_subdirs_fails/comment_1_2113ddd317cd5d11cf1d7a56b30bb452._comment new file mode 100644 index 0000000000..c3bddf2ffa --- /dev/null +++ b/doc/bugs/Exporting_subdirs_fails/comment_1_2113ddd317cd5d11cf1d7a56b30bb452._comment @@ -0,0 +1,8 @@ +[[!comment format=mdwn + username="joey" + subject="""comment 1""" + date="2017-11-07T21:08:48Z" + content=""" +This got fixed in [[!commit 24883e01cd487feb1be95146811d65e4f240465d]], +the next release of git-annex will have the fix. +"""]] diff --git a/doc/bugs/Failing_to_execute_bash_remotes_windows.mdwn b/doc/bugs/Failing_to_execute_bash_remotes_windows.mdwn new file mode 100644 index 0000000000..15ecbf4884 --- /dev/null +++ b/doc/bugs/Failing_to_execute_bash_remotes_windows.mdwn @@ -0,0 +1,58 @@ +### Please describe the problem. + +Trying to execute an external remote shell script on windows fails. + + +### What steps will reproduce the problem? + + $git annex initremote test type=external externaltype=rclone encryption=none + initremote test git-annex: Cannot run git-annex-remote-rclone -- Make sure it's in your PATH and is executable. + + +### What version of git-annex are you using? On what operating system? + +windows, using git-bash + + git-annex version: 6.20170214-g2233a50 + build flags: Assistant Webapp Pairing Testsuite S3(multipartupload)(storageclasses) WebDAV ConcurrentOutput TorrentParser Feeds Quvi + key/value backends: SHA256E SHA256 SHA512E SHA512 SHA224E SHA224 SHA384E SHA384 SHA3_256E SHA3_256 SHA3_512E SHA3_512 SHA3_224E SHA3_224 SHA3_384E SHA3_384 SKEIN256E SKEIN256 SKEIN512E SKEIN512 SHA1E SHA1 MD5E MD5 WORM URL + remote types: git gcrypt p2p S3 bup directory rsync web bittorrent webdav tahoe glacier ddar hook external + local repository version: 5 + supported repository versions: 3 5 6 + upgrade supported from repository versions: 2 3 4 5 + operating system: mingw32 i386 + + + +### Please provide any additional information below. + +I've also tried fooling around with the proxy, which may help with pinpointing the issue? + +[[!format sh """ +$ git-annex-remote-rclone +VERSION 1 + +$ git annex proxy -- git-annex-remote-rclone +git-annex: git-annex-remote-rclone: createProcess: does not exist (No such file or directory) +failed +git-annex: proxy: 1 failed + +$ git annex proxy -- which git-annex-remote-rclone +/d/bin/git-annex-remote-rclone + +$ git annex proxy -- /d/bin/git-annex-remote-rclone +git-annex: D:/bin/git-annex-remote-rclone: createProcess: invalid argument (Exec format error) +failed +git-annex: proxy: 1 failed + +$ git annex proxy -- sh /d/bin/git-annex-remote-rclone +VERSION 1 + +# End of transcript or log. +"""]] + +### Have you had any luck using git-annex before? (Sometimes we get tired of reading bug reports all day and a lil' positive end note does wonders) + +Oh yeah! This software is awesome. After getting used to having "dummy" shortcuts to content I don't currently have, with the simple ability to get/drop that content, I can't believe I haven't seen this anywhere before. If there is anything more impressive than this software, it's the support it has had from Joey over all this time. I'd have pulled my hair out long ago. :P + +> [[fixed|done]] although untested --[[Joey]] diff --git a/doc/bugs/Failing_to_execute_bash_remotes_windows/comment_1_f3d4899a90757132c79659b2c85f3e48._comment b/doc/bugs/Failing_to_execute_bash_remotes_windows/comment_1_f3d4899a90757132c79659b2c85f3e48._comment new file mode 100644 index 0000000000..7b231b8aad --- /dev/null +++ b/doc/bugs/Failing_to_execute_bash_remotes_windows/comment_1_f3d4899a90757132c79659b2c85f3e48._comment @@ -0,0 +1,11 @@ +[[!comment format=mdwn + username="jason.dixon.email@aa0e536a2ec2877d6f666108dbbc6e39bbe87ac0" + nickname="jason.dixon.email" + avatar="http://cdn.libravatar.org/avatar/fbe9050fc83bbd536d307d87ea14d4bc" + subject="Perhaps relevant" + date="2017-03-08T17:31:30Z" + content=""" +Not sure if this is helpful or relevant, but it may be? + +https://github.com/jgm/pandoc/issues/3247#issuecomment-262036875 +"""]] diff --git a/doc/bugs/Failing_to_execute_bash_remotes_windows/comment_2_7b8d82a32d6b82f1d50a54455aa4f643._comment b/doc/bugs/Failing_to_execute_bash_remotes_windows/comment_2_7b8d82a32d6b82f1d50a54455aa4f643._comment new file mode 100644 index 0000000000..1ebf64afd1 --- /dev/null +++ b/doc/bugs/Failing_to_execute_bash_remotes_windows/comment_2_7b8d82a32d6b82f1d50a54455aa4f643._comment @@ -0,0 +1,16 @@ +[[!comment format=mdwn + username="joey" + subject="""comment 2""" + date="2017-03-08T19:20:21Z" + content=""" +git-annex on Windows already checks if external special remotes start +with #! and tries to handle the shebang, since Windows does not do that +natively. That was added in version 6.20160907. +([[todo/refactor_shebang_handling_code_for_wider_use]]) + +`git-annex proxy` does not do that, and I feel it's out of scope for it to +do so. So the proxy stuff you tried is a red herring. + +It might be useful to use --debug to see what command git-annex tries to +execute when running the external special remote program. +"""]] diff --git a/doc/bugs/Failing_to_execute_bash_remotes_windows/comment_3_691fe409f869c46e3716924e958f431a._comment b/doc/bugs/Failing_to_execute_bash_remotes_windows/comment_3_691fe409f869c46e3716924e958f431a._comment new file mode 100644 index 0000000000..0d27c86e12 --- /dev/null +++ b/doc/bugs/Failing_to_execute_bash_remotes_windows/comment_3_691fe409f869c46e3716924e958f431a._comment @@ -0,0 +1,14 @@ +[[!comment format=mdwn + username="joey" + subject="""comment 3""" + date="2017-03-08T19:36:57Z" + content=""" +Looking into it a bit more, the problem seems to be that findShellCommand +expects a path to a file to examine, but when it's used for an external +special remote, it's only given the name of the command. + +So, I fixed it by searching for the command in the PATH. + +I have still not tested if this works on Windows, but probably, I think. +As long as PATH is set on Windows at least. +"""]] diff --git a/doc/bugs/Failing_to_execute_bash_remotes_windows/comment_4_d05069125823e7602a3441306e12db48._comment b/doc/bugs/Failing_to_execute_bash_remotes_windows/comment_4_d05069125823e7602a3441306e12db48._comment new file mode 100644 index 0000000000..38b449fdee --- /dev/null +++ b/doc/bugs/Failing_to_execute_bash_remotes_windows/comment_4_d05069125823e7602a3441306e12db48._comment @@ -0,0 +1,9 @@ +[[!comment format=mdwn + username="jason.dixon.email@aa0e536a2ec2877d6f666108dbbc6e39bbe87ac0" + nickname="jason.dixon.email" + avatar="http://cdn.libravatar.org/avatar/fbe9050fc83bbd536d307d87ea14d4bc" + subject="re: comment 3" + date="2017-03-08T21:58:14Z" + content=""" +Awesome, thanks Joey. Yeah that sounds plausible. I'll check it out whenever the next version comes by. Thanks. Keep up the amazing work! :) +"""]] diff --git a/doc/bugs/Files_disappear_from_locally_paired_annexes_when_edited.mdwn b/doc/bugs/Files_disappear_from_locally_paired_annexes_when_edited.mdwn new file mode 100644 index 0000000000..16109fb237 --- /dev/null +++ b/doc/bugs/Files_disappear_from_locally_paired_annexes_when_edited.mdwn @@ -0,0 +1,36 @@ +**What steps will reproduce the problem?** + +Create two annexes from the command line on two separate machines: + + mkdir ~/Files.annex + cd ~/Files.annex + git init + git annex init + git annex untrust . + git annex direct + +Add remote to each one pointing to the other: + + git remote add [remote] [remote hostname]:Files.annex + +Start assistant on both repos: + + git annex assistant + +Fill one repository with a few text files, and wait for them to propagate. + +Edit one of the text files using vim, and save. + +**What is the expected output? What do you see instead?** + +Edited file should remain in the repo, but a significant portion of the time, the file disappeared from the repo in which it was edited (the file is present and properly synced on the other repo). + +**What version of git-annex are you using? On what operating system?** + +git-annex 4.20130323, Mac OS X on the repo where the file was edited, Arch Linux for the other paired repo. + +**Please provide any additional information below.** + +I have observed this problem setting up the repos through the webapp as well, so I don't think it is related to setting up the repos manually. I think the way vim is writing the files seems to be tickling a race condition (both command-line vim and MacVim produce the behavior). I started trying to work around it by switching to emacs to edit those files, and the files haven't disappeared from the edited repo (so far at least). + +[[!tag /design/assistant moreinfo]] diff --git a/doc/bugs/Files_disappear_from_locally_paired_annexes_when_edited/comment_1_bdc97db9dc9954331e4c400baf9e5541._comment b/doc/bugs/Files_disappear_from_locally_paired_annexes_when_edited/comment_1_bdc97db9dc9954331e4c400baf9e5541._comment new file mode 100644 index 0000000000..46b3332714 --- /dev/null +++ b/doc/bugs/Files_disappear_from_locally_paired_annexes_when_edited/comment_1_bdc97db9dc9954331e4c400baf9e5541._comment @@ -0,0 +1,10 @@ +[[!comment format=mdwn + username="http://joeyh.name/" + nickname="joey" + subject="comment 1" + date="2013-04-08T17:14:23Z" + content=""" +When you say that the file \"disappeared from the repo\", do you mean that the actual file is no longer present on disk, or that the file is present on disk, but is no longer staged in the git repository? + +Version 4.20130405 fixes a bug that can result in the latter behavior. +"""]] diff --git a/doc/bugs/Files_reappear_after_delete.mdwn b/doc/bugs/Files_reappear_after_delete.mdwn new file mode 100644 index 0000000000..f53963945b --- /dev/null +++ b/doc/bugs/Files_reappear_after_delete.mdwn @@ -0,0 +1,25 @@ +### Please describe the problem. +I normally use two clients with the assistant (lets call them A and B) and they are usually online at different times. +Sometimes, after deleting a file in A, I turn on the client in B and when rescanning, instead of deleting that same file from B, it adds it back to A. + +I can always delete the file again, but it's a bit annoying. Anyway, it's a lot better than data loss ;) + +Might be worthy to mention that the computer running client B is quite slow and has a slow HD. I have 4 git-annex repositories and when I turn on the computer the rescan can easily take up a couple of hours. + + +### What steps will reproduce the problem? +B -> is turned off + +A -> delete file + +B -> turn on and wait for rescan + +B -> readds the file instead of deleting it + + +### What version of git-annex are you using? On what operating system? +6.20160923 (Debian Sid) + +### Have you had any luck using git-annex before? (Sometimes we get tired of reading bug reports all day and a lil' positive end note does wonders) + +I've been using git-annex for years with 4 different repositories, 3 of them pretty big (> 50 GB) and I'm really happy! :) diff --git a/doc/bugs/Files_unaccessible_in___40__some__63____41___views_on_a_crippled_filesystem.mdwn b/doc/bugs/Files_unaccessible_in___40__some__63____41___views_on_a_crippled_filesystem.mdwn new file mode 100644 index 0000000000..ac047ab1f6 --- /dev/null +++ b/doc/bugs/Files_unaccessible_in___40__some__63____41___views_on_a_crippled_filesystem.mdwn @@ -0,0 +1,80 @@ +(note: this is a more clear version of a previously reported bug. sorry if this breaks bug-reporting etiquette, but the title was incredibly unclear) + +### Please describe the problem. +In a v6 repository on a FAT filesystem, the view `/=*` replaces locally available files with placeholders normally used for unavailable files. + +### What steps will reproduce the problem? +[[!format sh """ +[leo60228@digitaleo:~]$ fallocate -l $((1024*1024*1024*2)) demo.img + +[leo60228@digitaleo:~]$ mkfs.vfat demo.img +mkfs.fat 4.1 (2017-01-24) + +[leo60228@digitaleo:~]$ mkdir demo + +[leo60228@digitaleo:~]$ sudo mount -o loop,uid=${UID},gid=$(id -g $UID) demo.img demo + +[leo60228@digitaleo:~]$ cd demo + +[leo60228@digitaleo:~/demo]$ git init +Initialized empty Git repository in /home/leo60228/demo/.git/ + +[leo60228@digitaleo:~/demo]$ git commit --allow-empty -m 'init' +[master (root-commit) 8dc8e0a] init + +[leo60228@digitaleo:~/demo]$ git annex init +init + Detected a filesystem without fifo support. + + Disabling ssh connection caching. + + Detected a crippled filesystem. + + Enabling direct mode. +ok +(recording state in git...) + +[leo60228@digitaleo:~/demo]$ mkdir subdir + +[leo60228@digitaleo:~/demo]$ echo hi > subdir/file + +[leo60228@digitaleo:~/demo]$ git annex upgrade +upgrade (v5 to v6...) (scanning for unlocked files...) +ok +(recording state in git...) + +[leo60228@digitaleo:~/demo]$ git annex add subdir/ +add subdir/file ok +(recording state in git...) + +[leo60228@digitaleo:~/demo]$ git commit -m 'add file' +[adjusted/master(unlocked) 0e870b3] add file + 1 file changed, 1 insertion(+) + create mode 100644 subdir/file + +[leo60228@digitaleo:~/demo]$ ls subdir/ +file + +[leo60228@digitaleo:~/demo]$ cat subdir/file +hi + +[leo60228@digitaleo:~/demo]$ git-annex view /=* +view (searching...) +Switched to branch 'views/_=_' +ok + +[leo60228@digitaleo:~/demo]$ ls subdir/ +file_%subdir% + +[leo60228@digitaleo:~/demo]$ cat subdir/file_%subdir% +../.git/annex/objects/zQ/MQ/SHA256E-s3--98ea6e4f216f2fb4b69fff9b3a44842c38686ca685f3f55dc48c5d3fb1107be4/SHA256E-s3--98ea6e4f216f2fb4b69fff9b3a44842c38686ca685f3f55dc48c5d3fb1107be4 +"""]] + +### What version of git-annex are you using? On what operating system? +6.20180427 on NixOS. Installed via `nix-env -iA nixos.gitAndTools.git-annex`. + +### Please provide any additional information below. +The repository is in v6 mode, because I was having strange issues with normal git commands such as `git commit` in direct mode. I know I needed to proxy the command, but are *all* git commands supposed to be broken in direct mode? + +### Have you had any luck using git-annex before? (Sometimes we get tired of reading bug reports all day and a lil' positive end note does wonders) +Yep! I already use it to move files between my laptop's HDD and SSD, and to copy files between my many SD cards. I was trying this to see if I could not have to scroll as far on my 3D printer's menu. diff --git a/doc/bugs/GHC_8.4.3_build_failure_in_Types__47__DesktopNotify.hs.mdwn b/doc/bugs/GHC_8.4.3_build_failure_in_Types__47__DesktopNotify.hs.mdwn new file mode 100644 index 0000000000..61ac77f0ea --- /dev/null +++ b/doc/bugs/GHC_8.4.3_build_failure_in_Types__47__DesktopNotify.hs.mdwn @@ -0,0 +1,139 @@ +### Please describe the problem. +The build fails with GHC 8.4.3 + +### What steps will reproduce the problem? +Be sure to use esqueleto HEAD to avoid getting stuck on dependencies. + +### What version of git-annex are you using? On what operating system? +git-annex-6.20180529 on macOS + +### Please provide any additional information below. + +[[!format sh """ +==> cabal install --jobs=8 --max-backjumps=100000 --prefix=/usr/local/Cellar/git-annex/6.20180529 --flags=s3 webapp +clang: warning: -Wl,-headerpad_max_install_names: 'linker' input unused +clang: warning: argument unused during compilation: '-L/usr/local/opt/gettext/lib' +clang: warning: argument unused during compilation: '-L/usr/local/opt/libffi/lib' +clang: warning: argument unused during compilation: '-L/usr/local/opt/icu4c/lib' +clang: warning: argument unused during compilation: '-L/usr/local/opt/openssl/lib' +clang: warning: argument unused during compilation: '-L/usr/local/opt/readline/lib' +clang: warning: argument unused during compilation: '-L/usr/local/opt/sqlite/lib' +clang: warning: argument unused during compilation: '-L/usr/local/lib' +clang: warning: argument unused during compilation: '-L/System/Library/Frameworks/OpenGL.framework/Versions/Current/Libraries' +Resolving dependencies... +Notice: installing into a sandbox located at +/private/tmp/git-annex-20180529-26197-14ebtme/git-annex-6.20180529/.cabal-sandbox +Configuring git-annex-6.20180529... +Building git-annex-6.20180529... +Failed to install git-annex-6.20180529 +Build log ( /private/tmp/git-annex-20180529-26197-14ebtme/git-annex-6.20180529/.cabal-sandbox/logs/ghc-8.4.3/git-annex-6.20180529-9O50etRsVNH1q2HUlIF2Gu.log ): +cabal: Entering directory '.' +[ 1 of 34] Compiling Utility.Applicative ( Utility/Applicative.hs, dist/dist-sandbox-669b35af/setup/Utility/Applicative.o ) +[ 2 of 34] Compiling Utility.Data ( Utility/Data.hs, dist/dist-sandbox-669b35af/setup/Utility/Data.o ) +[ 3 of 34] Compiling Utility.Exception ( Utility/Exception.hs, dist/dist-sandbox-669b35af/setup/Utility/Exception.o ) +[ 4 of 34] Compiling Utility.Env.Basic ( Utility/Env/Basic.hs, dist/dist-sandbox-669b35af/setup/Utility/Env/Basic.o ) +[ 5 of 34] Compiling Build.Mans ( Build/Mans.hs, dist/dist-sandbox-669b35af/setup/Build/Mans.o ) +[ 6 of 34] Compiling Utility.FileSize ( Utility/FileSize.hs, dist/dist-sandbox-669b35af/setup/Utility/FileSize.o ) +[ 7 of 34] Compiling Utility.Misc ( Utility/Misc.hs, dist/dist-sandbox-669b35af/setup/Utility/Misc.o ) +[ 8 of 34] Compiling Utility.Monad ( Utility/Monad.hs, dist/dist-sandbox-669b35af/setup/Utility/Monad.o ) +[ 9 of 34] Compiling Utility.PartialPrelude ( Utility/PartialPrelude.hs, dist/dist-sandbox-669b35af/setup/Utility/PartialPrelude.o ) +[10 of 34] Compiling Utility.Process.Shim ( Utility/Process/Shim.hs, dist/dist-sandbox-669b35af/setup/Utility/Process/Shim.o ) +[11 of 34] Compiling Utility.Process ( Utility/Process.hs, dist/dist-sandbox-669b35af/setup/Utility/Process.o ) +[12 of 34] Compiling Utility.Network ( Utility/Network.hs, dist/dist-sandbox-669b35af/setup/Utility/Network.o ) +[13 of 34] Compiling Utility.Split ( Utility/Split.hs, dist/dist-sandbox-669b35af/setup/Utility/Split.o ) +[14 of 34] Compiling Utility.SafeCommand ( Utility/SafeCommand.hs, dist/dist-sandbox-669b35af/setup/Utility/SafeCommand.o ) +[15 of 34] Compiling Utility.ExternalSHA ( Utility/ExternalSHA.hs, dist/dist-sandbox-669b35af/setup/Utility/ExternalSHA.o ) +[16 of 34] Compiling Utility.FileSystemEncoding ( Utility/FileSystemEncoding.hs, dist/dist-sandbox-669b35af/setup/Utility/FileSystemEncoding.o ) +[17 of 34] Compiling Utility.SystemDirectory ( Utility/SystemDirectory.hs, dist/dist-sandbox-669b35af/setup/Utility/SystemDirectory.o ) +[18 of 34] Compiling Utility.Tmp ( Utility/Tmp.hs, dist/dist-sandbox-669b35af/setup/Utility/Tmp.o ) +[19 of 34] Compiling Utility.Directory ( Utility/Directory.hs, dist/dist-sandbox-669b35af/setup/Utility/Directory.o ) +[20 of 34] Compiling Build.Version ( Build/Version.hs, dist/dist-sandbox-669b35af/setup/Build/Version.o ) +[21 of 34] Compiling Utility.UserInfo ( Utility/UserInfo.hs, dist/dist-sandbox-669b35af/setup/Utility/UserInfo.o ) +[22 of 34] Compiling Utility.Path ( Utility/Path.hs, dist/dist-sandbox-669b35af/setup/Utility/Path.o ) +[23 of 34] Compiling Common ( Common.hs, dist/dist-sandbox-669b35af/setup/Common.o ) +[24 of 34] Compiling Utility.DottedVersion ( Utility/DottedVersion.hs, dist/dist-sandbox-669b35af/setup/Utility/DottedVersion.o ) +[25 of 34] Compiling Git.Version ( Git/Version.hs, dist/dist-sandbox-669b35af/setup/Git/Version.o ) +[26 of 34] Compiling Build.TestConfig ( Build/TestConfig.hs, dist/dist-sandbox-669b35af/setup/Build/TestConfig.o ) +[27 of 34] Compiling Build.Configure ( Build/Configure.hs, dist/dist-sandbox-669b35af/setup/Build/Configure.o ) +[28 of 34] Compiling Utility.OSX ( Utility/OSX.hs, dist/dist-sandbox-669b35af/setup/Utility/OSX.o ) +[29 of 34] Compiling Utility.FreeDesktop ( Utility/FreeDesktop.hs, dist/dist-sandbox-669b35af/setup/Utility/FreeDesktop.o ) +[30 of 34] Compiling Config.Files ( Config/Files.hs, dist/dist-sandbox-669b35af/setup/Config/Files.o ) +[31 of 34] Compiling Assistant.Install.Menu ( Assistant/Install/Menu.hs, dist/dist-sandbox-669b35af/setup/Assistant/Install/Menu.o ) +[32 of 34] Compiling Assistant.Install.AutoStart ( Assistant/Install/AutoStart.hs, dist/dist-sandbox-669b35af/setup/Assistant/Install/AutoStart.o ) +[33 of 34] Compiling Build.DesktopFile ( Build/DesktopFile.hs, dist/dist-sandbox-669b35af/setup/Build/DesktopFile.o ) +[34 of 34] Compiling Main ( dist/dist-sandbox-669b35af/setup/setup.hs, dist/dist-sandbox-669b35af/setup/Main.o ) +Linking ./dist/dist-sandbox-669b35af/setup/setup ... + checking version...fatal: Not a git repository (or any of the parent directories): .git + 6.20180529 + checking UPGRADE_LOCATION... not available + checking git... yes + checking git version... 2.10.1 (Apple Git-78) + checking cp -a... yes + checking cp -p... yes + checking cp --preserve=timestamps... no + checking cp --reflink=auto... no + checking xargs -0... yes + checking rsync... yes + checking curl... yes + checking bup... no + checking nice... yes + checking ionice... no + checking nocache... no + checking gpg... not available + checking lsof... lsof + checking git-remote-gcrypt... not available + checking ssh connection caching... yes + checking sha1... not available + checking sha256... not available + checking sha512... not available + checking sha224... not available + checking sha384... not available +Configuring git-annex-6.20180529... +clang: warning: -Wl,-headerpad_max_install_names: 'linker' input unused +clang: warning: argument unused during compilation: '-L/usr/local/opt/gettext/lib' +clang: warning: argument unused during compilation: '-L/usr/local/opt/libffi/lib' +clang: warning: argument unused during compilation: '-L/usr/local/opt/icu4c/lib' +clang: warning: argument unused during compilation: '-L/usr/local/opt/openssl/lib' +clang: warning: argument unused during compilation: '-L/usr/local/opt/readline/lib' +clang: warning: argument unused during compilation: '-L/usr/local/opt/sqlite/lib' +clang: warning: argument unused during compilation: '-L/usr/local/lib' +clang: warning: argument unused during compilation: '-L/System/Library/Frameworks/OpenGL.framework/Versions/Current/Libraries' +Preprocessing executable 'git-annex' for git-annex-6.20180529.. +Building executable 'git-annex' for git-annex-6.20180529.. +[ 1 of 593] Compiling Assistant.Types.BranchChange ( Assistant/Types/BranchChange.hs, dist/dist-sandbox-669b35af/build/git-annex/git-annex-tmp/Assistant/Types/BranchChange.o ) +[ 2 of 593] Compiling Assistant.Types.ThreadName ( Assistant/Types/ThreadName.hs, dist/dist-sandbox-669b35af/build/git-annex/git-annex-tmp/Assistant/Types/ThreadName.o ) +[ 3 of 593] Compiling Assistant.Types.TransferSlots ( Assistant/Types/TransferSlots.hs, dist/dist-sandbox-669b35af/build/git-annex/git-annex-tmp/Assistant/Types/TransferSlots.o ) +[ 4 of 593] Compiling BuildFlags ( BuildFlags.hs, dist/dist-sandbox-669b35af/build/git-annex/git-annex-tmp/BuildFlags.o ) +[ 5 of 593] Compiling BuildInfo ( BuildInfo.hs, dist/dist-sandbox-669b35af/build/git-annex/git-annex-tmp/BuildInfo.o ) +[ 6 of 593] Compiling Build.BundledPrograms ( Build/BundledPrograms.hs, dist/dist-sandbox-669b35af/build/git-annex/git-annex-tmp/Build/BundledPrograms.o ) +[ 7 of 593] Compiling Config.Cost ( Config/Cost.hs, dist/dist-sandbox-669b35af/build/git-annex/git-annex-tmp/Config/Cost.o ) +[ 8 of 593] Compiling Logs.Line ( Logs/Line.hs, dist/dist-sandbox-669b35af/build/git-annex/git-annex-tmp/Logs/Line.o ) +[ 9 of 593] Compiling Types.Availability ( Types/Availability.hs, dist/dist-sandbox-669b35af/build/git-annex/git-annex-tmp/Types/Availability.o ) +[ 10 of 593] Compiling Types.BranchState ( Types/BranchState.hs, dist/dist-sandbox-669b35af/build/git-annex/git-annex-tmp/Types/BranchState.o ) +[ 11 of 593] Compiling Types.Concurrency ( Types/Concurrency.hs, dist/dist-sandbox-669b35af/build/git-annex/git-annex-tmp/Types/Concurrency.o ) +[ 12 of 593] Compiling Types.Creds ( Types/Creds.hs, dist/dist-sandbox-669b35af/build/git-annex/git-annex-tmp/Types/Creds.o ) +[ 13 of 593] Compiling Assistant.Types.CredPairCache ( Assistant/Types/CredPairCache.hs, dist/dist-sandbox-669b35af/build/git-annex/git-annex-tmp/Assistant/Types/CredPairCache.o ) +[ 14 of 593] Compiling Types.DesktopNotify ( Types/DesktopNotify.hs, dist/dist-sandbox-669b35af/build/git-annex/git-annex-tmp/Types/DesktopNotify.o ) + +Types/DesktopNotify.hs:19:10: error: + • No instance for (Semigroup DesktopNotify) + arising from the superclasses of an instance declaration + • In the instance declaration for ‘Monoid DesktopNotify’ + | +19 | instance Monoid DesktopNotify where + | ^^^^^^^^^^^^^^^^^^^^ +cabal: Leaving directory '.' +cabal: Error: some packages failed to install: +git-annex-6.20180529-9O50etRsVNH1q2HUlIF2Gu failed during the building phase. +The exception was: +ExitFailure 1 +"""]] + +Full log https://gist.github.com/ilovezfs/e3af135ed0362e0253a786c71f2a914f + +### Have you had any luck using git-annex before? (Sometimes we get tired of reading bug reports all day and a lil' positive end note does wonders) + +Yes! Thanks :) + +> Consequence of the Semigroup Monoid change. Fixed. [[done]] --[[Joey]] + diff --git a/doc/bugs/GIT__95__ANNEX__95__SHELL__95__DIRECTORY_won__39__t_match.mdwn b/doc/bugs/GIT__95__ANNEX__95__SHELL__95__DIRECTORY_won__39__t_match.mdwn new file mode 100644 index 0000000000..63c8d16c3c --- /dev/null +++ b/doc/bugs/GIT__95__ANNEX__95__SHELL__95__DIRECTORY_won__39__t_match.mdwn @@ -0,0 +1,33 @@ +### Please describe the problem. + +Trying to replace my rrsync setup with a restricted git-annex-shell, I ran into the following problem (slightly obfuscated): + + ± git annex get . + get secret (from secrets...) + git-annex-shell: Only allowed to access ~/store not ~/store/secrets/6a8/d0c/GPGHMACSHA1--0000000000000000000000000000000000000000/GPGHMACSHA1--0000000000000000000000000000000000000000 + rsync: connection unexpectedly closed (0 bytes received so far) [Receiver] + rsync error: error in rsync protocol data stream (code 12) at io.c(235) [Receiver=3.1.2] + +It does not matter if GIT_ANNEX_SHELL_DIRECTORY is just "store", "store/secrets" or the full absolute path to any of the two directories, the output and the results are the same. + + +### What steps will reproduce the problem? + +Create an encrypted rsync remote in ~/store/secrets, add + + restrict,command="GIT_ANNEX_SHELL_DIRECTORY=store GIT_ANNEX_SHELL_LIMITED=true GIT_ANNEX_SHELL_READONLY=true git-annex-shell -c \"$SSH_ORIGINAL_COMMAND\"" ssh-rsa ... + +to ~/.ssh/authorized keys and try to retrieve files. + + +### What version of git-annex are you using? On what operating system? + +git-annex version: 6.20180227-g32d682dd8 (standalone version) on Debian stretch + + +### Have you had any luck using git-annex before? + +As always, I'm a fan :> + + +> [[fixed|done]] --[[Joey]] diff --git a/doc/bugs/GIT__95__ANNEX__95__SHELL__95__DIRECTORY_won__39__t_match/comment_1_9523af238c6cd2ab66f83474dd580427._comment b/doc/bugs/GIT__95__ANNEX__95__SHELL__95__DIRECTORY_won__39__t_match/comment_1_9523af238c6cd2ab66f83474dd580427._comment new file mode 100644 index 0000000000..9d567f3762 --- /dev/null +++ b/doc/bugs/GIT__95__ANNEX__95__SHELL__95__DIRECTORY_won__39__t_match/comment_1_9523af238c6cd2ab66f83474dd580427._comment @@ -0,0 +1,18 @@ +[[!comment format=mdwn + username="joey" + subject="""comment 1""" + date="2018-03-07T19:46:27Z" + content=""" +A rsync special remote needs ssh to run rsync --server, +you have it running git-annex-shell. + +That can't work. It is possible to lock down generic rsync over +ssh to only operate in one directory, but git-annex-shell +is not the tool to do it. + +If there's a bug here, it's whatever led you to expect this would work.. + +(But, if you make a gcrypt special remote, git-annex-shell can be used +to access it, and it'll be all encrypted and files stored with rsync, +and can be locked down.) +"""]] diff --git a/doc/bugs/GIT__95__ANNEX__95__SHELL__95__DIRECTORY_won__39__t_match/comment_2_888d17279dc33de32d7433f1f88e94e1._comment b/doc/bugs/GIT__95__ANNEX__95__SHELL__95__DIRECTORY_won__39__t_match/comment_2_888d17279dc33de32d7433f1f88e94e1._comment new file mode 100644 index 0000000000..ba7b1fdedd --- /dev/null +++ b/doc/bugs/GIT__95__ANNEX__95__SHELL__95__DIRECTORY_won__39__t_match/comment_2_888d17279dc33de32d7433f1f88e94e1._comment @@ -0,0 +1,13 @@ +[[!comment format=mdwn + username="https://tribut.de/" + nickname="Felix" + avatar="http://cdn.libravatar.org/avatar/d7508a030fd9902a76b02f7feff1327e80400a1317ee98e463c68ef4c9c40bb8" + subject="comment 2" + date="2018-03-07T20:02:07Z" + content=""" +OK, sorry for making this weird bug report then. I was getting errors like + + rrsync: SSH_ORIGINAL_COMMAND='git-annex-shell' is not rsync + +so I assumed that my rrsync setup was incorrect and maybe it had something to do with your [recent blog](https://git-annex.branchable.com/devblog/day_486__time_to_ditch_rsync/). Unfortunately I cannot reproduce it right now, so I guess I'll wait until it happens again. +"""]] diff --git a/doc/bugs/GIT__95__ANNEX__95__SHELL__95__DIRECTORY_won__39__t_match/comment_3_56eeb148889bc95a464f00208775aae1._comment b/doc/bugs/GIT__95__ANNEX__95__SHELL__95__DIRECTORY_won__39__t_match/comment_3_56eeb148889bc95a464f00208775aae1._comment new file mode 100644 index 0000000000..10ed73864e --- /dev/null +++ b/doc/bugs/GIT__95__ANNEX__95__SHELL__95__DIRECTORY_won__39__t_match/comment_3_56eeb148889bc95a464f00208775aae1._comment @@ -0,0 +1,34 @@ +[[!comment format=mdwn + username="https://tribut.de/" + nickname="Felix" + avatar="http://cdn.libravatar.org/avatar/d7508a030fd9902a76b02f7feff1327e80400a1317ee98e463c68ef4c9c40bb8" + subject="comment 3" + date="2018-03-07T20:15:00Z" + content=""" +Got it to happen at least (though I do not believe this is was caused me to investigate this)... + +This works: + + ± git annex sync --content + commit + On branch master + Your branch is up to date with 'origin/master'. + + nothing to commit, working tree clean + ok + pull origin + ok + +This gives the error: + + ± git annex sync -J4 --content + /usr/local/bin/rrsync: SSH_ORIGINAL_COMMAND='git-annex-shell inannex .' is not rsync + + Unable to run git-annex-shell on remote . + On branch master + Your branch is up to date with 'origin/master'. + + nothing to commit, working tree clean + commit ok + pull origin ok +"""]] diff --git a/doc/bugs/GIT__95__ANNEX__95__SHELL__95__DIRECTORY_won__39__t_match/comment_4_57d694df6c22c42cbdc98ffc5dbc0945._comment b/doc/bugs/GIT__95__ANNEX__95__SHELL__95__DIRECTORY_won__39__t_match/comment_4_57d694df6c22c42cbdc98ffc5dbc0945._comment new file mode 100644 index 0000000000..e344830b01 --- /dev/null +++ b/doc/bugs/GIT__95__ANNEX__95__SHELL__95__DIRECTORY_won__39__t_match/comment_4_57d694df6c22c42cbdc98ffc5dbc0945._comment @@ -0,0 +1,18 @@ +[[!comment format=mdwn + username="joey" + subject="""comment 4""" + date="2018-03-07T20:25:10Z" + content=""" +Ok, this involves the fairly recently added support for +avoding overlapping ssh password prompts when running multiple ssh +processes concurrently. + +In -J mode, git-annex warms up the ssh connection by running +"git-annex-shell inannex". Your rrsync config prevents that being run. + +This doesn't actually cause a problem, the connection is still warmed up. +If you needed a password to connect, it would have prompted for it before +the error message from rrsync. + +But it's ugly.. I've committed a fix that will avoid the ugly messages. +"""]] diff --git a/doc/bugs/GPG_passphrase_repeated_prompt.mdwn b/doc/bugs/GPG_passphrase_repeated_prompt.mdwn new file mode 100644 index 0000000000..56b2bba4ed --- /dev/null +++ b/doc/bugs/GPG_passphrase_repeated_prompt.mdwn @@ -0,0 +1,25 @@ +#### What steps will reproduce the problem? + +1. Create a new repository with a directory +2. Add files +3. Select "Store your data in the cloud" with the "Remote server" option +4. Enter host, user, directory +5. Select "Use an encrypted rsync repository on the server" (Will there be an option for unencrypted later?) +6. GPG Passphrase prompt comes up for every file + +#### What is the expected output? What do you see instead? + +I expect to enter a passphase once and then it will sync all files with the remote server. + +Instead, it begins syncing the files to the server but prompts for a GPG passphase for every single file. + +#### What version of git-annex are you using? On what operating system? + +3.20121017 precompiled binary on Arch Linux + +#### Please provide any additional information below. + +Not sure if I'm just missing a setting for GPG, but I would think I should only need to use the web app to configure the remote server. + +[[!tag /design/assistant]] +[[!tag confirmed]] diff --git a/doc/bugs/GPG_passphrase_repeated_prompt/comment_1_6ef1c9725befc84ad57bce196ef630ef._comment b/doc/bugs/GPG_passphrase_repeated_prompt/comment_1_6ef1c9725befc84ad57bce196ef630ef._comment new file mode 100644 index 0000000000..016ecb9945 --- /dev/null +++ b/doc/bugs/GPG_passphrase_repeated_prompt/comment_1_6ef1c9725befc84ad57bce196ef630ef._comment @@ -0,0 +1,16 @@ +[[!comment format=mdwn + username="http://joeyh.name/" + ip="4.152.108.194" + subject="you should install a gpg agent" + date="2012-10-31T17:39:53Z" + content=""" +A gpg agent will cache your passphrase. It is beyond the scope of the git-annex package to provide one (though it does bundle gpg), but it should be easy to install gpg-agent on your distribution. + +That's all that's needed for normal git-annex use, but the assistant does seem to have a larger problem in this area, since it can need to unlock a remote's key at any time to sync files from it. Since a separate +process is spawned for each transfer, this defeats git-annex's normal in-process caching of encryption keys of remotes. So I think it needs to unlock any encrypted special remotes at startup, or when first accessing them, and pass the cached keys to the transfer processes it spawns. This is now on my todo list. + +However, none of the special remotes set up by the assistant will use +password protected gpg keys, even when it's using encryption it's using a +non-password protected shared key. So only encrypted special remotes set up +at the command line cause this problem. +"""]] diff --git a/doc/bugs/GPG_subkeys.mdwn b/doc/bugs/GPG_subkeys.mdwn new file mode 100644 index 0000000000..917ec90029 --- /dev/null +++ b/doc/bugs/GPG_subkeys.mdwn @@ -0,0 +1,47 @@ +### Please describe the problem. +Despite the very fast, much appreciated, and excellent response to my initial question [here](http://git-annex.branchable.com/encryption/#comment-cb9c20121e570f80fcd799c1619ced69), I still can't get encryption working with subkeys. + +I have a few different machines, each with its own subkey and each configured with the master key stripped out. I'm hoping to be able to encrypt and decrypt from the same special remotes with each of them. + +The issue is that git-annex consistently tries to encrypt with the master key, which is not available, and so encryption and the `initremote` fail. + +I'm submitting a bug report but I should say that I'm not sure this is the intended (or documented) way to use GPG subkeys and it may not be a git-annex concern. The [Debian Wiki](https://wiki.debian.org/Subkeys) does say that it won't work but the [password store](https://lists.zx2c4.com/pipermail/password-store/2014-September/001131.html) project has implemented it. + +Side note: this is not working in gcrypt either. + +### What steps will reproduce the problem? + +I've tried the following in a few different combinations, with both short and long KEYIDs, and with or without the appended '!' that's supposed to tell gpg which key to use for encryption: +`git annex initremote cloud type=S3 encryption=hybrid keyid=EFEFEFE host=storage.googleapis.com port=80` + +In each case I get the same error: +`gpg: decryption failed: No secret key` +`git-annex: user error (/usr/local/bin/gpg ["--quiet","--trust-model","always","--decrypt"] exited 2)` +`failed` + +### What version of git-annex are you using? On what operating system? +`git-annex version: 6.20171109` +OS X 10.10.5 + +### Please provide any additional information below. + +### Have you had any luck using git-annex before? (Sometimes we get tired of reading bug reports all day and a lil' positive end note does wonders) +I love git annex. Thank you for your work on it. + +> Yeah, so I had that working before but then I tightened the +> code to determine what is a forced subkey and broke it. +> +> [Fixed now; use a "!" at the _end_ to force subkey use: + + joey@darkstar:~/tmp/a>git annex initremote testing type=directory directory=../d encryption=hybrid keyid='A4FEC0B5F031BA70!' + initremote testing (encryption setup) gpg: A4FEC0B5F031BA70!: skipped: Unusable public key + gpg: [stdin]: encryption failed: Unusable public key + +> It failed to use that subkey because it is a sign-only key, +> and git-annex needs to encrypt, but that certainly shows I got +> git-annex to use the subkey.. +> +> I made an encrypt-only subkey, and git-annex is able to use it, +> and it fully works as far as I can tell. +> +> [[done]] --[[Joey]] diff --git a/doc/bugs/GPG_subkeys/comment_1_9be3e177b0b02c3988d5af64dc6e67ed._comment b/doc/bugs/GPG_subkeys/comment_1_9be3e177b0b02c3988d5af64dc6e67ed._comment new file mode 100644 index 0000000000..230ed65506 --- /dev/null +++ b/doc/bugs/GPG_subkeys/comment_1_9be3e177b0b02c3988d5af64dc6e67ed._comment @@ -0,0 +1,11 @@ +[[!comment format=mdwn + username="Yurt" + avatar="http://cdn.libravatar.org/avatar/28d2177f7678c6cc0b677c34927eba06" + subject="comment 1" + date="2017-12-20T22:33:03Z" + content=""" +Hi, + +I've tested this and can confirm it's working for me in 6.20171214. I can encrypt with a subkey when the master key is not present. +Thanks so much for your work on this and git-annex in general (and mr and etckeeper and everything else). +"""]] diff --git a/doc/bugs/Git-annex_and_Microsoft_Office_files_on_OS_X.mdwn b/doc/bugs/Git-annex_and_Microsoft_Office_files_on_OS_X.mdwn new file mode 100644 index 0000000000..22a0bb65b7 --- /dev/null +++ b/doc/bugs/Git-annex_and_Microsoft_Office_files_on_OS_X.mdwn @@ -0,0 +1,28 @@ +### Please describe the problem. + +On OS X 10.11.6, if you save any MS office file (.pptx, .docx, .xlsx) in a git-annex directory it won't be synced by the assistant unless you restart the daemon. If you look in the logs it says "/Users/me/annex/test.pptx still has writers, not adding" This probably has something to do with temporary/lockfiles used by MS office apps (Office 2011 for Mac). Any ideas on a workaround, other than restarting the daemon constantly? + + +### What steps will reproduce the problem? + +1. Create any MS office file (.docx, .pptx, .xlsx). Save in a directory managed by git-annex assistant. Error message "still has writers, not adding" will show up in git-annex log. File will not be synced to any other computer running git-annex assistant until Office application quit and daemon restarted. + + +### What version of git-annex are you using? On what operating system? + +6.20161211-gf58b134, OS X 10.11.6 + + +### Please provide any additional information below. + +[[!format sh """ +# If you can, paste a complete transcript of the problem occurring here. +# If the problem is with the git-annex assistant, paste in .git/annex/daemon.log + + +# End of transcript or log. +"""]] + +### Have you had any luck using git-annex before? (Sometimes we get tired of reading bug reports all day and a lil' positive end note does wonders) + +Yes! It's wonderful otherwise! diff --git a/doc/bugs/Git-annex_and_Microsoft_Office_files_on_OS_X/comment_1_0cf11096ceeb6cf93db5609a42a70641._comment b/doc/bugs/Git-annex_and_Microsoft_Office_files_on_OS_X/comment_1_0cf11096ceeb6cf93db5609a42a70641._comment new file mode 100644 index 0000000000..ae27704b28 --- /dev/null +++ b/doc/bugs/Git-annex_and_Microsoft_Office_files_on_OS_X/comment_1_0cf11096ceeb6cf93db5609a42a70641._comment @@ -0,0 +1,41 @@ +[[!comment format=mdwn + username="joey" + subject="""comment 1""" + date="2017-06-09T17:52:00Z" + content=""" +This must have to do with the fsevents interface used on OSX. + +In Assistant.Threads.Committer.safeToAdd, when lsof detects +a file is still open for write by some process, it cancels +the add. This relies on events being received when files +get closed (closingTracked). + +In Utility.DirWatcher.FSEvents.watchDir, when an event has +eventFlagItemModified set, it treats that as a file add event. +The intent is to emulate inotify's handling of file add events when +files are closed. + +So, two theories: + +1. Perhaps eventFlagItemModified only gets set if the file + is actually modified. Ie, if MS office writes the file + and while it's being written another process opens it to read + it (perhaps to index the content), then if the other process + doesn't modify it, eventFlagItemModified is not set. + +2. Perhaps the way the assistant hard links/moves the file around + confuses the FSEvents handling. Perhaps there is an event with + eventFlagItemModified, but it's for the locked down file, or + something like that, so git-annex ignores it. + +In any case, I'm leaning toward thinking that closingTracked should +not be True for FSEvents. This bug report seems to show, conclusively, +that FSEvents does not have that property. If closingTracked was False, +as it is for KQueue, the assistant would postpone adding the file, +and keep retrying, around once per second, until it no longer had +any writers, and then add it. + +So, I've made that change. I suspect it fixes the bug, but it would +be pretty hard for me to test it. Could you please download tomorrow's +daily build of git-annex for OSX, and see if it fixes the problem? +"""]] diff --git a/doc/bugs/Git-annex_not_properly_detecting_symlink_capability_in_CryFS.mdwn b/doc/bugs/Git-annex_not_properly_detecting_symlink_capability_in_CryFS.mdwn new file mode 100644 index 0000000000..bb62802283 --- /dev/null +++ b/doc/bugs/Git-annex_not_properly_detecting_symlink_capability_in_CryFS.mdwn @@ -0,0 +1,63 @@ +### Please describe the problem. + +When creating a new git-annex repository (or upgrading, or doing fsck) inside a CryFS container (FUSE-based encrypted local filesystem, similiar to EncFS), git-annex goes into direct(or adjusted, in v6) mode despite CryFS working just fine with symlinks. + +Indeed, a created-outside-cryfs git-annex repository works as expected when copied inside + +### What steps will reproduce the problem? +1. Install cryfs (https://www.cryfs.org/, packaged for ubuntu,debian, or build from source) +2. Create a CryFS container: $ cryfs container container-data +3. Create a git annex repository inside the container: + + $ cd container + $ git init . + $ git annex init --version 6 + +#### Cloning from v6 repository, you get + +git annex init --version 6 +init + Detected a filesystem without fifo support. + + Disabling ssh connection caching. + + Detected a crippled filesystem. + + Disabling core.symlinks. +(merging origin/git-annex into git-annex...) +(recording state in git...) +(scanning for unlocked files...) + + Entering an adjusted branch where files are unlocked as this filesystem does not support locked files. + +Switched to branch 'adjusted/master(unlocked)' +ok +(recording state in git...) + +#### Cloning from v5 repository and upgrading, you get + +git annex upgrade +upgrade . + Detected a filesystem without fifo support. + + Disabling ssh connection caching. + + Detected a crippled filesystem. + + Disabling core.symlinks. +(merging origin/git-annex into git-annex...) +(recording state in git...) + + Enabling direct mode. +(v5 to v6...) (scanning for unlocked files...) +ok +(recording state in git...) + +### What version of git-annex are you using? On what operating system? +git-annex-6.20170519-1.fc26.x86_64 + +### Have you had any luck using git-annex before? (Sometimes we get tired of reading bug reports all day and a lil' positive end note does wonders) + +I love git-annex. + +> [[done]] --[[Joey]] diff --git a/doc/bugs/Git-annex_not_properly_detecting_symlink_capability_in_CryFS/comment_1_78f6f047578974df0dd73740ab05ee8c._comment b/doc/bugs/Git-annex_not_properly_detecting_symlink_capability_in_CryFS/comment_1_78f6f047578974df0dd73740ab05ee8c._comment new file mode 100644 index 0000000000..dae2a9d099 --- /dev/null +++ b/doc/bugs/Git-annex_not_properly_detecting_symlink_capability_in_CryFS/comment_1_78f6f047578974df0dd73740ab05ee8c._comment @@ -0,0 +1,17 @@ +[[!comment format=mdwn + username="joey" + subject="""comment 1""" + date="2017-08-28T17:06:35Z" + content=""" +git-annex has detected problems with the filesystem, since symlinks +work I think it detected that the filesystem allows writing to a +file even when the file's write bit is not set. + +In that situation, a git-annex repository using symlinks will apprear to +work, but if you have an annexed file "foo" in there, and run "echo hi > +foo", it will follow the symlink, and overwrite the annexed file, ignoring +the lack of write bit. This can result in data loss in an unexpected way, +which is why git-annex avoids using the symlinks in this situation. + +I've added an additional message to make more clear what is happening. +"""]] diff --git a/doc/bugs/Git_copy_fails_with_absolute_path_on_mavericks.mdwn b/doc/bugs/Git_copy_fails_with_absolute_path_on_mavericks.mdwn new file mode 100644 index 0000000000..5704bf1ccc --- /dev/null +++ b/doc/bugs/Git_copy_fails_with_absolute_path_on_mavericks.mdwn @@ -0,0 +1,12 @@ +### Please describe the problem. + +git-annex version: 6.20160126 + +Problem happens on Mavericks, not on Yosemite with same version. + +Can do a relative or absolute path on a git annex add. + +But on the git annex copy to an S3 remote, if I use an absolute path for the file descriptor (e.g. /Users/name/git_directory/test.txt) it will not report an error, but will also not print the "copied..." string. A subsequent call with a relative path (e.g. test.txt) will work, show the copied message and indeed show up on S3. + + +Good news: It's working great on Yosemite! diff --git a/doc/bugs/Git_copy_fails_with_absolute_path_on_mavericks/comment_1_2f0c74aba502533b85b922ea81ded70d._comment b/doc/bugs/Git_copy_fails_with_absolute_path_on_mavericks/comment_1_2f0c74aba502533b85b922ea81ded70d._comment new file mode 100644 index 0000000000..e538a75c62 --- /dev/null +++ b/doc/bugs/Git_copy_fails_with_absolute_path_on_mavericks/comment_1_2f0c74aba502533b85b922ea81ded70d._comment @@ -0,0 +1,23 @@ +[[!comment format=mdwn + username="joey" + subject="""comment 1""" + date="2016-03-14T17:34:21Z" + content=""" +Using absolute paths to files is an unusual thing to do with git in my +experience. But, git does seem to support it, at least some of the time. + +I think that git-annex's support for it comes down to what `git ls-files` +does. For example: + + joey@darkstar:~/tmp/me> git ls-files ~/tmp/me/foo + foo + +Since git-annex uses git ls-files, it sees a relative file and so +all or most all git-annex commands can work with the absolute file input. + +Does git ls-files convert the absolute path to a relative path +when you run it on the Mavericks system? + +What version of git is installed on the Mavericks system? Is +git-annex installed from the git-annex app, or from homebrew, or how? +"""]] diff --git a/doc/bugs/Handling_of_files_inside_and_outside_archive_directory_at_the_same_time.mdwn b/doc/bugs/Handling_of_files_inside_and_outside_archive_directory_at_the_same_time.mdwn new file mode 100644 index 0000000000..b0bf84a601 --- /dev/null +++ b/doc/bugs/Handling_of_files_inside_and_outside_archive_directory_at_the_same_time.mdwn @@ -0,0 +1,18 @@ +Hi, + +some time ago, I accidentally copied some files from the archive to the non-archive part of my (indirect, type 'client') repository (instead of moving), with the effect that assistant always kept downloading and afterwards immediately dropping the files. Now this was not really surprising once I found the duplicate folder, but maybe git annex could detect this case and refuse to run in circles or at least complain about it. + +Best +Karsten + +[[!tag /design/assistant]] + +> Update: Current status is this is fixed for direct mode. +> +> In indirect mode, the startup scan will still download and then drop +> content if a file outside and inside the archive directory has the +> same content. It doesn't loop like it did in direct mode, only +> happens once (or once per duplicate file, really). Is still potentially +> annoying and a bug. --[[Joey]] + +[[!tag confirmed]] diff --git a/doc/bugs/Handling_of_files_inside_and_outside_archive_directory_at_the_same_time/comment_1_e8bb3d6a2318402b985caed08282d473._comment b/doc/bugs/Handling_of_files_inside_and_outside_archive_directory_at_the_same_time/comment_1_e8bb3d6a2318402b985caed08282d473._comment new file mode 100644 index 0000000000..0606fdffbb --- /dev/null +++ b/doc/bugs/Handling_of_files_inside_and_outside_archive_directory_at_the_same_time/comment_1_e8bb3d6a2318402b985caed08282d473._comment @@ -0,0 +1,12 @@ +[[!comment format=mdwn + username="http://joeyh.name/" + nickname="joey" + subject="comment 1" + date="2013-04-11T16:32:43Z" + content=""" +This is a known problem. + +It seems possible to fix it for direct mode. After all, direct mode tracks all files associated with a key, so it could expose this to preferred content expressions, and the expression check if any of the associated files was in an archive directory. + +Unsure how to deal with it in indirect mode. Short of making indirect mode do all the same tracking direct mode does, or otherwise build a key to file lookup table. +"""]] diff --git a/doc/bugs/Handling_of_files_inside_and_outside_archive_directory_at_the_same_time/comment_2_ead9fa75a12ef36be9a92637b144e74f._comment b/doc/bugs/Handling_of_files_inside_and_outside_archive_directory_at_the_same_time/comment_2_ead9fa75a12ef36be9a92637b144e74f._comment new file mode 100644 index 0000000000..2be39d451a --- /dev/null +++ b/doc/bugs/Handling_of_files_inside_and_outside_archive_directory_at_the_same_time/comment_2_ead9fa75a12ef36be9a92637b144e74f._comment @@ -0,0 +1,14 @@ +[[!comment format=mdwn + username="http://joeyh.name/" + nickname="joey" + subject="comment 2" + date="2013-06-15T17:52:36Z" + content=""" +This turns out to be much worse in direct mode than in indirect mode. + +In indirect mode, it only does extra work during the full startup scan. Suppose there are 3 files with the same content, 1, archive/2, and 3. It will download 1, and then will drop archive/2, and then will download 3. This certainly is not ideal, especially when the file content is large. + +In indirect mode, it continally and repeatedly downloads the drops the files, as long as it's running. Which is beyond unacceptable. + +What seems to be going on is that when archive/2 gets dropped, it necessary needs to convert 1 and 3 to broken symlinks. But the watcher than sees those file changes, thinks these are new or renamed files that have appeared, and promptly re-downloads them. That, in turn triggers an update of archive/2, to convert it back from symlink to direct mode file, and that in turn is noticed by the watcher. Round and round we go! +"""]] diff --git a/doc/bugs/Hard_links_not_synced_in_direct_mode.mdwn b/doc/bugs/Hard_links_not_synced_in_direct_mode.mdwn new file mode 100644 index 0000000000..6ede618605 --- /dev/null +++ b/doc/bugs/Hard_links_not_synced_in_direct_mode.mdwn @@ -0,0 +1,128 @@ +[[!meta title="hard links not synced"]] + +### Please describe the problem. + +Direct mode repositories seem to initially ignore hard linked files and then when changes are done to them sync them as separate files. However, changes to one file are only propagated to that file and not to any of the others that are hardlinked to it. + +### What steps will reproduce the problem? + +Inside a direct mode repository linked to a ssh remote: + +[[!format sh """ +$ ls -l +total 0 +$ echo "something" > foo +$ ln foo bar +$ ls -l +total 8 +-rw-r--r-- 2 pedrocr pedrocr 10 May 29 12:08 bar +-rw-r--r-- 2 pedrocr pedrocr 10 May 29 12:08 foo +$ tail .git/annex/daemon.log + 6c0fbd7..0bb8ef9 git-annex -> synced/git-annex + 0bae1b4..bfedc45 master -> synced/master + +sent 77 bytes received 31 bytes 72.00 bytes/sec +total size is 10 speedup is 0.09 +[2013-05-29 12:08:03 WEST] Transferrer: Uploaded foo +Already up-to-date. +[2013-05-29 12:08:05 WEST] Pusher: Syncing with golias +To ssh://golias.git-annex/home/pedrocr/testsync + 0bb8ef9..2ce5013 git-annex -> synced/git-annex +$ git status +# On branch master +# Changes not staged for commit: +# (use "git add ..." to update what will be committed) +# (use "git checkout -- ..." to discard changes in working directory) +# +# typechange: foo +# +# Untracked files: +# (use "git add ..." to include in what will be committed) +# +# bar +no changes added to commit (use "git add" and/or "git commit -a") +"""]] + +On the remote repository: + +[[!format sh """ +$ ls -l +total 4 +-rw-r--r-- 1 pedrocr pedrocr 10 May 29 12:08 foo +"""]] + +If I now just touch the linked file on the repository: + +[[!format sh """ +$ touch bar +$ tail .git/annex/daemon.log + +(merging synced/git-annex into git-annex...) +(Recording state in git...) +add bar (checksum...) [2013-05-29 12:12:49 WEST] Committer: Committing changes to git +[2013-05-29 12:12:49 WEST] Pusher: Syncing with golias +Already up-to-date. +To ssh://golias.git-annex/home/pedrocr/testsync + 2ce5013..d36166b git-annex -> synced/git-annex + bfedc45..ee3a7a1 master -> synced/master +Already up-to-date. +"""]] + +On the remote repository: + +[[!format sh """ +$ ls -l +total 8 +-rw-r--r-- 1 pedrocr pedrocr 10 May 29 12:08 bar +-rw-r--r-- 1 pedrocr pedrocr 10 May 29 12:08 foo +"""]] + +Note that now bar has been synced as a new file and not a hardlink as it should be (the 1's after the permissions). + +The sync also isn't acting properly on the linked files. For example. + +First in the origin repository: + +[[!format sh """ +$ cat bar +something +$ cat foo +something +$ echo "someotherthing" > bar +$ cat bar +someotherthing +$ cat foo +someotherthing +"""]] + +The result in the destination: + +[[!format sh """ +$ cat bar +someotherthing +$ cat foo +something +"""]] + +So even if the intended behavior is for hardlinked files to be synced as two separate files the sync isn't correct because the two files changed in the origin and only one of them changed in the destination. This probably needs to be fixed with actual hard links for real filesystems and with some copying for crippled filesystems. + +### What version of git-annex are you using? On what operating system? + +[[!format sh """ +$ git annex version +git-annex version: 4.20130516.1 +build flags: Assistant Webapp Pairing Testsuite S3 WebDAV Inotify DBus XMPP +local repository version: 4 +default repository version: 3 +supported repository versions: 3 4 +upgrade supported from repository versions: 0 1 2 +$ lsb_release -a +No LSB modules are available. +Distributor ID: Ubuntu +Description: Ubuntu 12.04.2 LTS +Release: 12.04 +Codename: precise +"""]] + + +> [[!taglink confirmed]] (but may be out of scope for git-annex) --[[Joey]] diff --git a/doc/bugs/Hard_links_not_synced_in_direct_mode/comment_1_aaa781664ae0c62c4f6530cb075ed367._comment b/doc/bugs/Hard_links_not_synced_in_direct_mode/comment_1_aaa781664ae0c62c4f6530cb075ed367._comment new file mode 100644 index 0000000000..5d7422c08d --- /dev/null +++ b/doc/bugs/Hard_links_not_synced_in_direct_mode/comment_1_aaa781664ae0c62c4f6530cb075ed367._comment @@ -0,0 +1,17 @@ +[[!comment format=mdwn + username="https://www.google.com/accounts/o8/id?id=AItOawkx5V3MTbzCXS3J7Mn9FEq8M9bPPYMkAHY" + nickname="Pedro" + subject="comment 1" + date="2013-05-29T11:33:05Z" + content=""" +Implementing hard links will probably also require handling the edge case where the user has setup two repositories where one of them spans filesystems. So: + +- Repository A is all in a single filesystem +- Repository B has it's root in one filesystem and them /someotherdirectory/ is another filesystem. + +If the user in repository A does a \"ln /somefile /someotherdirectory/otherfile\" you'd need to treat this as in the crippled case as repository B can't do the hardlink spanning filesystems. + +Another edge case may also be that OSX supports hardlinked directories for use with their TimeMachine feature. The feature isn't exposed through the normal ln command but users may sometimes hack around that[1]. Also, any TimeMachine backups will naturally have hardlinked directories. + +[1] http://stackoverflow.com/questions/1432540/creating-directory-hard-links-in-macos-x +"""]] diff --git a/doc/bugs/Hard_links_not_synced_in_direct_mode/comment_2_213aa10909d1fd0f20ed078a7ed93e79._comment b/doc/bugs/Hard_links_not_synced_in_direct_mode/comment_2_213aa10909d1fd0f20ed078a7ed93e79._comment new file mode 100644 index 0000000000..39d601dba1 --- /dev/null +++ b/doc/bugs/Hard_links_not_synced_in_direct_mode/comment_2_213aa10909d1fd0f20ed078a7ed93e79._comment @@ -0,0 +1,8 @@ +[[!comment format=mdwn + username="http://joeyh.name/" + nickname="joey" + subject="comment 2" + date="2013-05-29T15:39:48Z" + content=""" +git does not support hard linked files, so neither can git-annex. +"""]] diff --git a/doc/bugs/Hard_links_not_synced_in_direct_mode/comment_3_e6b783d9aaae20c0d35e9888d878716a._comment b/doc/bugs/Hard_links_not_synced_in_direct_mode/comment_3_e6b783d9aaae20c0d35e9888d878716a._comment new file mode 100644 index 0000000000..179d80b217 --- /dev/null +++ b/doc/bugs/Hard_links_not_synced_in_direct_mode/comment_3_e6b783d9aaae20c0d35e9888d878716a._comment @@ -0,0 +1,10 @@ +[[!comment format=mdwn + username="http://joeyh.name/" + nickname="joey" + subject="comment 3" + date="2013-05-29T16:11:46Z" + content=""" +It would be possible to make the assistant use the inotify CREATE event (which it currently ignores) to add a file to the repository when a hard link is created. However, when a hard linked file is modified, inotify only sends an event for the file that was changed, not for other hard links to it. So, without keeping track of all hard links that exist on my own, there's no way for the assistant to automatically handle that case. And even if it tried to, hard links to files in the repository from outside the repository would still allow modifying them without the assistant being able to detect it. + +Since hard links cannot be propigated over git anyway, I don't want to get into this mess. It's best to wontfix this I think. +"""]] diff --git a/doc/bugs/Hard_links_not_synced_in_direct_mode/comment_4_b008ae7b1cf8685d92c9a87a7609de1e._comment b/doc/bugs/Hard_links_not_synced_in_direct_mode/comment_4_b008ae7b1cf8685d92c9a87a7609de1e._comment new file mode 100644 index 0000000000..8e8acd9cb7 --- /dev/null +++ b/doc/bugs/Hard_links_not_synced_in_direct_mode/comment_4_b008ae7b1cf8685d92c9a87a7609de1e._comment @@ -0,0 +1,18 @@ +[[!comment format=mdwn + username="https://www.google.com/accounts/o8/id?id=AItOawkx5V3MTbzCXS3J7Mn9FEq8M9bPPYMkAHY" + nickname="Pedro" + subject="comment 4" + date="2013-05-29T18:48:10Z" + content=""" +I agree with your assessment for the traditional git-annex, as it's an extension of git. For the assistant (in direct mode at least) it seems like broken behavior, there will be files that exist on one side but not the other and files that have the same content on one side but not the other. + +I just had a look and unfortunately there doesn't seem to be a general way to do inode to path lookup in UNIX, so an inode cache would really be needed. It would look something like: + +* On inotify CREATE detect that the file has other hardlinked instances, find out if they are in the repository by keeping a inode->path cache of all files +* When the file is a hardlink to another file already in the repository commit some kind of special file to git that states \"hardlink to /path/to/otherfile\" and add metadata to the original file saying what files are hardlinks to it +* When a remote gets a special file after a sync it will itself run a \"ln /path/to/otherfile /path/to/newfile\" or alteratively a copy when the two paths span filesystems or the filesystem is crippled +* When a crippled remote gets a write to a file with hardlinks, it will replicate those writes to the other files (this is where the metadata is needed) +* When a crippled remote does a write to a file with hardlinks that file becomes independent in the remotes with hardlinks since the content is now different between the two (or more) files. + +This does get pretty hairy with the corner cases. Right now the simple case of \"sync between two non-crippled filesystems\" shows pretty surprising results though so I'd argue something needs to change. I think that for direct mode to really be a \"transparent folder sync\" kind of solution this should be fixed. +"""]] diff --git a/doc/bugs/Hard_links_not_synced_in_direct_mode/comment_5_949c891209713a2c0a5e66af11ed4c79._comment b/doc/bugs/Hard_links_not_synced_in_direct_mode/comment_5_949c891209713a2c0a5e66af11ed4c79._comment new file mode 100644 index 0000000000..52f3e30f7d --- /dev/null +++ b/doc/bugs/Hard_links_not_synced_in_direct_mode/comment_5_949c891209713a2c0a5e66af11ed4c79._comment @@ -0,0 +1,14 @@ +[[!comment format=mdwn + username="http://joeyh.name/" + ip="4.152.108.193" + subject="comment 5" + date="2013-06-26T16:34:23Z" + content=""" +To fix [[Problems_with_syncing_gnucash]], I have made some changes to how hard links are dealt with. + +Assistant will now notice when a hard link is created, and add the same thing to git it would add for any other file. The hard link is not propigated to other repositories. + +Files remain hard linked locally. This means that a change to one will affect the contents of the other. The assistant, lacking a hard link cache, will not notice this, and so will commit the change to the file that was written to, but not commit its hard link. Running `git annex add` manually (or restarting the assistant) will make it finally notice the other file has changed. + +So, the assistant still does not keep hard links in sync on an ongoing basis. This bug is still unsolved. +"""]] diff --git a/doc/bugs/How_to_clone_git-annex__63__.mdwn b/doc/bugs/How_to_clone_git-annex__63__.mdwn new file mode 100644 index 0000000000..e71e6b18bb --- /dev/null +++ b/doc/bugs/How_to_clone_git-annex__63__.mdwn @@ -0,0 +1,59 @@ +### Please describe the problem. + +When I clone git-annex with: + +``` +git clone git://git-annex.branchable.com/ git-annex +``` + +I get errors about filenames being too long and the checkout fails after the clone: + + + $ git clone git://git-annex.branchable.com/ git-annex + Cloning into 'git-annex'... + remote: Counting objects: 183837, done. + remote: Compressing objects: 100% (51419/51419), done. + remote: Total 183837 (delta 131729), reused 183837 (delta 131729) + Receiving objects: 100% (183837/183837), 46.20 MiB | 4.90 MiB/s, done. + Resolving deltas: 100% (131729/131729), done. + error: unable to create file doc/bugs/Errors_on_Android__58___referenceTable_GDEF_length__61__778__44___referenceTable_GSUB_length__61__6388__44___referenceTable_GPOS_length__61__76868.mdwn: File name too long + error: unable to create file doc/bugs/On_Lubuntu_14.04_assistant_fails_to_create_new_setup_or_actually_work___40__fixed_by_regular_lxsession_package_update_from_2014-06-30__41__.mdwn: File name too long + error: unable to create file doc/bugs/__39__git_annex_get__39___fails_for_unlocked_files_with_special_characters___40__e.g._umlauts__41___when_using_precompiled_version_6.20160126-g2336107_.mdwn: File name too long + fatal: cannot create directory at 'doc/bugs/__39__git_annex_get__39___fails_for_unlocked_files_with_special_characters___40__e.g._umlauts__41___when_using_precompiled_version_6.20160126-g2336107_': File name too long + warning: Clone succeeded, but checkout failed. + You can inspect what was checked out with 'git status' + and retry the checkout with 'git checkout -f HEAD' + +I would like to contribute, but I was a bit surprised by this problem right when I wanted to poke around in the code. + +After the clone attempt running `git status` in the checkout directory produces a **huge** amount of output about deleted files and changes to the repo: + + $ git status + On branch master + Your branch is up-to-date with 'origin/master'. + Changes to be committed: + (use "git reset HEAD ..." to unstage) + + deleted: .ghci + deleted: .gitattributes + deleted: .gitignore + deleted: .mailmap + deleted: Annex.hs + deleted: Annex/Action.hs + deleted: Annex/AdjustedBranch.hs + ... + + +Note that I have snipped off over 10K lines of output from above! + + +### What steps will reproduce the problem? + +Just try to clone it. + + +### Have you had any luck using git-annex before? (Sometimes we get tired of reading bug reports all day and a lil' positive end note does wonders) + +git-annex looks like a really promising project for my personal backups solution involving usb disks, and some external services such as Backblaze B2. I wanted to look into improving Backblaze B2 support (large file support, etc). + +> [[done]]; ecryptfs is not supported --[[Joey]] diff --git a/doc/bugs/How_to_clone_git-annex__63__/comment_1_a55d9717cffca2817254026cc9407b20._comment b/doc/bugs/How_to_clone_git-annex__63__/comment_1_a55d9717cffca2817254026cc9407b20._comment new file mode 100644 index 0000000000..76da8fd96b --- /dev/null +++ b/doc/bugs/How_to_clone_git-annex__63__/comment_1_a55d9717cffca2817254026cc9407b20._comment @@ -0,0 +1,9 @@ +[[!comment format=mdwn + username="aditya.mmy@be7b2fbd14a6ce2b6b8588f6719672725a11ed35" + nickname="aditya.mmy" + avatar="http://cdn.libravatar.org/avatar/ec711faaa06edd971f00a6d107fd3206" + subject="Nevermind - I got it working" + date="2017-08-27T10:18:03Z" + content=""" +The problem was that I am using ecryptfs - I was not aware of any file-length limitations in it until now, but here they are - https://unix.stackexchange.com/questions/32795/what-is-the-maximum-allowed-filename-and-folder-size-with-ecryptfs +"""]] diff --git a/doc/bugs/How_to_clone_git-annex__63__/comment_2_9b235c0a8f0e299f84e97857cb09c4cb._comment b/doc/bugs/How_to_clone_git-annex__63__/comment_2_9b235c0a8f0e299f84e97857cb09c4cb._comment new file mode 100644 index 0000000000..ad7845d71c --- /dev/null +++ b/doc/bugs/How_to_clone_git-annex__63__/comment_2_9b235c0a8f0e299f84e97857cb09c4cb._comment @@ -0,0 +1,9 @@ +[[!comment format=mdwn + username="joey" + subject="""comment 2""" + date="2017-08-28T17:00:21Z" + content=""" +You didn't say what OS/filesystem you're using that has trouble with +filenames that are 144 characters long. Even Windows supports up to 255 +character long filenames IIRC. +"""]] diff --git a/doc/bugs/How_to_clone_git-annex__63__/comment_3_6ac90de5faa03726b760eda831fd5203._comment b/doc/bugs/How_to_clone_git-annex__63__/comment_3_6ac90de5faa03726b760eda831fd5203._comment new file mode 100644 index 0000000000..3ffcbd1022 --- /dev/null +++ b/doc/bugs/How_to_clone_git-annex__63__/comment_3_6ac90de5faa03726b760eda831fd5203._comment @@ -0,0 +1,8 @@ +[[!comment format=mdwn + username="joey" + subject="""comment 3""" + date="2017-08-28T17:01:21Z" + content=""" +Ah yes, ecryptfs.. It has some problems. I am not going to try to limit +filenames to support it. +"""]] diff --git a/doc/bugs/Hybrid_encryption_can__39__t_generate_the_right_key_after_moving_files.mdwn b/doc/bugs/Hybrid_encryption_can__39__t_generate_the_right_key_after_moving_files.mdwn new file mode 100644 index 0000000000..b11da5e9da --- /dev/null +++ b/doc/bugs/Hybrid_encryption_can__39__t_generate_the_right_key_after_moving_files.mdwn @@ -0,0 +1,115 @@ +### Please describe the problem. + +I had to migrate a special remote using hybrid encryption from an Amazon Cloud Drive-hosted rclone remote to a Google Drive-hosted rclone remote. Now I am testing it by trying to fsck a particular file. + +I've left all of the special remote settings on the git annex side the same, and just changed my `.rclone.conf` to have "amazondrive", the rclone remote name, actually point to the new Google Drive location. + +When I try to fsck my file, git annex generates an HMAC key, successfully downloads it from the remote, but then fails to decrypt it. After that, it looks like it retries: it generates a **different HMAC key**, which it can't find the file for. + +I've compared the files in Amazon and in Google Drive and they are the same. + +Using the [independent decryption script](https://git-annex.branchable.com/tips/Decrypting_files_in_special_remotes_without_git-annex/), modified to use the SHA512 MAC that my repo uses, and to not use every matching line from `remote.log`, I can generate the *second* HMAC key that git annex appears to fall back to, but not the first one. I think that this means I can't generate the decryption key either; I've had no luck manually decrypting the file with the characters after the first 256 of the decrypted shared key as the GPG passphrase. + +How is Git Annex generating that first HMAC, if not by the process used in the decryption script? Why is it able to get an HMAC that exists but not a valid decryption key? What is the thinking behind trying multiple HMAC keys for the same file? + +### What steps will reproduce the problem? + +Have an encrypted, hybrid Git Annex repo in an Amazon Drive rclone remote. +Copy all the files over to Google Drive, and make a Google Drive rclone remote in rclone.conf with the same name that the Amazon Drive one had. +Try to retrieve files from the remote with Git Annex. + +### What version of git-annex are you using? On what operating system? + +`6.20170525+gitge1cf095ae-1~ndall+1` on Ubuntu 16.04 + +### Please provide any additional information below. + +[[!format sh """ +# If you can, paste a complete transcript of the problem occurring here. +# If the problem is with the git-annex assistant, paste in .git/annex/daemon.log + +$ git annex fsck info.txt --from=amazon --debug +[2017-06-03 11:24:31.368563623] read: git ["--git-dir=.git","--work-tree=.","--literal-pathspecs","show-ref","git-annex"] +[2017-06-03 11:24:31.373358653] process done ExitSuccess +[2017-06-03 11:24:31.373520685] read: git ["--git-dir=.git","--work-tree=.","--literal-pathspecs","show-ref","--hash","refs/heads/git-annex"] +[2017-06-03 11:24:31.379319755] process done ExitSuccess +[2017-06-03 11:24:31.382954929] chat: git ["--git-dir=.git","--work-tree=.","--literal-pathspecs","cat-file","--batch"] +[2017-06-03 11:24:31.386196477] chat: git ["--git-dir=.git","--work-tree=.","--literal-pathspecs","cat-file","--batch-check=%(objectname) %(objecttype) %(objectsize)"] +[2017-06-03 11:24:31.392852081] read: git ["config","--null","--list"] +[2017-06-03 11:24:31.397686571] process done ExitSuccess +[2017-06-03 11:24:31.400187312] read: git ["--git-dir=.git","--work-tree=.","--literal-pathspecs","ls-files","--cached","-z","--","info.txt"] +[2017-06-03 11:24:31.424951599] chat: git ["--git-dir=.git","--work-tree=.","--literal-pathspecs","check-attr","-z","--stdin","annex.backend","annex.numcopies","annex.largefiles","--"] +[2017-06-03 11:24:31.425544622] read: git ["--version"] +[2017-06-03 11:24:31.428009953] process done ExitSuccess +fsck info.txt [2017-06-03 11:24:31.479360829] chat: gpg ["--batch","--no-tty","--use-agent","--quiet","--trust-model","always","--decrypt"] +[2017-06-03 11:24:31.503040759] process done ExitSuccess +[2017-06-03 11:24:31.506638987] chat: /home/anovak/bin/git-annex-remote-rclone [] +[2017-06-03 11:24:31.510094668] git-annex-remote-rclone[1] --> VERSION 1 +[2017-06-03 11:24:31.510281634] git-annex-remote-rclone[1] <-- PREPARE +[2017-06-03 11:24:31.511006098] git-annex-remote-rclone[1] --> GETCONFIG prefix +[2017-06-03 11:24:31.511148645] git-annex-remote-rclone[1] <-- VALUE data/annex-rclone +[2017-06-03 11:24:31.515487261] git-annex-remote-rclone[1] --> GETCONFIG target +[2017-06-03 11:24:31.515770189] git-annex-remote-rclone[1] <-- VALUE amazondrive +[2017-06-03 11:24:31.519503001] git-annex-remote-rclone[1] --> GETCONFIG rclone_layout +[2017-06-03 11:24:31.519666709] git-annex-remote-rclone[1] <-- VALUE lower +[2017-06-03 11:24:31.523726589] git-annex-remote-rclone[1] --> PREPARE-SUCCESS +[2017-06-03 11:24:31.523902589] git-annex-remote-rclone[1] <-- CHECKPRESENT GPGHMACSHA512--4354571263cd241ac8300e0bd8fc3643c184b922500c3d9318b7ce590684e1de820d41c0b245f1595e0bd38b2844a5151192981b4f9ab41bda135d5f7a184d16 +[2017-06-03 11:24:31.530946791] git-annex-remote-rclone[1] --> DIRHASH-LOWER GPGHMACSHA512--4354571263cd241ac8300e0bd8fc3643c184b922500c3d9318b7ce590684e1de820d41c0b245f1595e0bd38b2844a5151192981b4f9ab41bda135d5f7a184d16 +[2017-06-03 11:24:31.531236912] git-annex-remote-rclone[1] <-- VALUE b95/785/ +Total objects: 1 Total size: 119 Bytes (119 Bytes) 2017/06/03 11:24:42 Transferred: 0 Bytes (0 Bytes/s) Errors: 0 Checks: 0 Transferred: 0 Elapsed time: 10.4s +[2017-06-03 11:24:42.04576211] git-annex-remote-rclone[1] --> CHECKPRESENT-SUCCESS GPGHMACSHA512--4354571263cd241ac8300e0bd8fc3643c184b922500c3d9318b7ce590684e1de820d41c0b245f1595e0bd38b2844a5151192981b4f9ab41bda135d5f7a184d16 + +[2017-06-03 11:24:42.049286934] git-annex-remote-rclone[1] <-- TRANSFER RETRIEVE GPGHMACSHA512--4354571263cd241ac8300e0bd8fc3643c184b922500c3d9318b7ce590684e1de820d41c0b245f1595e0bd38b2844a5151192981b4f9ab41bda135d5f7a184d16 .git/annex/tmp/GPGHMACSHA512--4354571263cd241ac8300e0bd8fc3643c184b922500c3d9318b7ce590684e1de820d41c0b245f1595e0bd38b2844a5151192981b4f9ab41bda135d5f7a184d16 +[2017-06-03 11:24:42.050298028] git-annex-remote-rclone[1] --> DIRHASH-LOWER GPGHMACSHA512--4354571263cd241ac8300e0bd8fc3643c184b922500c3d9318b7ce590684e1de820d41c0b245f1595e0bd38b2844a5151192981b4f9ab41bda135d5f7a184d16 +[2017-06-03 11:24:42.05045573] git-annex-remote-rclone[1] <-- VALUE b95/785/ +2017/06/03 11:24:52 Local file system at /tmp/tmp.ITXaCoES4i: Waiting for checks to finish +2017/06/03 11:24:52 Local file system at /tmp/tmp.ITXaCoES4i: Waiting for transfers to finish +2017/06/03 11:24:54 +Transferred: 119 Bytes (9 Bytes/s) +Errors: 0 +Checks: 0 +Transferred: 1 +Elapsed time: 12s +[2017-06-03 11:24:54.083003717] git-annex-remote-rclone[1] --> TRANSFER-SUCCESS RETRIEVE GPGHMACSHA512--4354571263cd241ac8300e0bd8fc3643c184b922500c3d9318b7ce590684e1de820d41c0b245f1595e0bd38b2844a5151192981b4f9ab41bda135d5f7a184d16 +[2017-06-03 11:24:54.083554696] chat: gpg ["--batch","--no-tty","--use-agent","--quiet","--trust-model","always","--batch","--passphrase-fd","22","--decrypt"] +gpg: decryption failed: bad key +[2017-06-03 11:24:54.086768586] process done ExitFailure 2 +[2017-06-03 11:24:54.088931923] git-annex-remote-rclone[1] <-- TRANSFER RETRIEVE GPGHMACSHA512--7439fc103f30635cd914af0ce14370aef2f9a81f415ba7a652bc20454c90893bcd5701ebbed855f6a0285fce413e5c039c15a7049738cf519e2719c9d485ce28 .git/annex/tmp/GPGHMACSHA512--7439fc103f30635cd914af0ce14370aef2f9a81f415ba7a652bc20454c90893bcd5701ebbed855f6a0285fce413e5c039c15a7049738cf519e2719c9d485ce28 +[2017-06-03 11:24:54.089806756] git-annex-remote-rclone[1] --> DIRHASH-LOWER GPGHMACSHA512--7439fc103f30635cd914af0ce14370aef2f9a81f415ba7a652bc20454c90893bcd5701ebbed855f6a0285fce413e5c039c15a7049738cf519e2719c9d485ce28 +[2017-06-03 11:24:54.090050153] git-annex-remote-rclone[1] <-- VALUE b47/e54/ +2017/06/03 11:25:03 Local file system at /tmp/tmp.5lapDbIoZy: Waiting for checks to finish +2017/06/03 11:25:03 Local file system at /tmp/tmp.5lapDbIoZy: Waiting for transfers to finish +2017/06/03 11:25:03 Attempt 1/3 failed with 0 errors and: directory not found +2017/06/03 11:25:03 Local file system at /tmp/tmp.5lapDbIoZy: Waiting for checks to finish +2017/06/03 11:25:03 Local file system at /tmp/tmp.5lapDbIoZy: Waiting for transfers to finish +2017/06/03 11:25:03 Attempt 2/3 failed with 0 errors and: directory not found +2017/06/03 11:25:04 Local file system at /tmp/tmp.5lapDbIoZy: Waiting for checks to finish +2017/06/03 11:25:04 Local file system at /tmp/tmp.5lapDbIoZy: Waiting for transfers to finish +2017/06/03 11:25:04 Attempt 3/3 failed with 0 errors and: directory not found +2017/06/03 11:25:04 Failed to copy: directory not found +[2017-06-03 11:25:04.479940212] git-annex-remote-rclone[1] --> TRANSFER-FAILURE RETRIEVE GPGHMACSHA512--7439fc103f30635cd914af0ce14370aef2f9a81f415ba7a652bc20454c90893bcd5701ebbed855f6a0285fce413e5c039c15a7049738cf519e2719c9d485ce28 + + failed to download file from remote +failed +[2017-06-03 11:25:04.492702185] chat: git ["--git-dir=.git","--work-tree=.","--literal-pathspecs","hash-object","-w","--stdin-paths","--no-filters"] +[2017-06-03 11:25:04.494054942] feed: git ["--git-dir=.git","--work-tree=.","--literal-pathspecs","update-index","-z","--index-info"] +[2017-06-03 11:25:04.672344301] process done ExitSuccess +[2017-06-03 11:25:04.67255166] read: git ["--git-dir=.git","--work-tree=.","--literal-pathspecs","show-ref","--hash","refs/heads/git-annex"] +[2017-06-03 11:25:04.68070068] process done ExitSuccess +(recording state in git...) +[2017-06-03 11:25:04.680983817] read: git ["--git-dir=.git","--work-tree=.","--literal-pathspecs","write-tree"] +[2017-06-03 11:25:04.900456464] process done ExitSuccess +[2017-06-03 11:25:04.900643429] chat: git ["--git-dir=.git","--work-tree=.","--literal-pathspecs","commit-tree","a8fc26b91dab1ffac0499d3e21c0f691858d3a6a","--no-gpg-sign","-p","refs/heads/git-annex"] +[2017-06-03 11:25:04.965122443] process done ExitSuccess +[2017-06-03 11:25:04.965288386] call: git ["--git-dir=.git","--work-tree=.","--literal-pathspecs","update-ref","refs/heads/git-annex","c0b297184826849d8a24c899faa9f4f47c1f8e53"] +[2017-06-03 11:25:04.970235879] process done ExitSuccess +[2017-06-03 11:25:04.971493023] process done ExitSuccess +git-annex: fsck: 1 failed + + +# End of transcript or log. +"""]] + +### Have you had any luck using git-annex before? (Sometimes we get tired of reading bug reports all day and a lil' positive end note does wonders) + +Yeah, it works great most of the time, and it was working great on the amazon remote before I tried to pull these migration shenanigans. diff --git a/doc/bugs/Hybrid_encryption_can__39__t_generate_the_right_key_after_moving_files/comment_1_bbc1e7205d7701afd405c6e62a1c0aa3._comment b/doc/bugs/Hybrid_encryption_can__39__t_generate_the_right_key_after_moving_files/comment_1_bbc1e7205d7701afd405c6e62a1c0aa3._comment new file mode 100644 index 0000000000..7d03495191 --- /dev/null +++ b/doc/bugs/Hybrid_encryption_can__39__t_generate_the_right_key_after_moving_files/comment_1_bbc1e7205d7701afd405c6e62a1c0aa3._comment @@ -0,0 +1,13 @@ +[[!comment format=mdwn + username="yibe" + avatar="http://cdn.libravatar.org/avatar/649c9e417b6c509a17f7b48caf065c91" + subject="comment 1" + date="2017-06-04T20:07:31Z" + content=""" +Hello, + +I have no idea why git-annex fails to decrypt your file, but as for the two different HMAC keys, I guess you have chunking enabled on that remote (at least when you uploaded the file) and that first HMAC key is the right key for the 1st chunk of your file. That decryption script does not take chunks into account, so you were only able to generate the second HMAC key, which should be the right key if the file was uploaded without chunking enabled. I've just [posted][1] a modified version of the script that supports chunks. + +[1]: http://git-annex.branchable.com/tips/Decrypting_files_in_special_remotes_without_git-annex/#comment-ea2df7b4739f3d66c169bf297e339e9d + +"""]] diff --git a/doc/bugs/Hybrid_encryption_can__39__t_generate_the_right_key_after_moving_files/comment_2_52a1800537f1244ad6cf417c5f25ebe0._comment b/doc/bugs/Hybrid_encryption_can__39__t_generate_the_right_key_after_moving_files/comment_2_52a1800537f1244ad6cf417c5f25ebe0._comment new file mode 100644 index 0000000000..07fa485449 --- /dev/null +++ b/doc/bugs/Hybrid_encryption_can__39__t_generate_the_right_key_after_moving_files/comment_2_52a1800537f1244ad6cf417c5f25ebe0._comment @@ -0,0 +1,29 @@ +[[!comment format=mdwn + username="joey" + subject="""comment 2""" + date="2017-06-06T15:38:24Z" + content=""" +Your remote has chunking enabled, so git-annex first tries generating a +HMAC for a chunked key. When decrypting the content fails for some reason, +it falls back to trying the HMAC for an unchunked key. This is done because +chunking can be enabled after some content has been uploaded to a remote, +so it always tries the unchunked location just in case. + +It looks like gpg is successfully decrypting the hybrid encryption key +that's embedded in your git repository. That is the first, successful +call tp gpg in your log. + +The "bad key" error then comes when gpg is asked to use the hybrid +encryption key to decrypt the content. This seems to indicate it's not +using the same hybrid encryption key that was used to encrypt it. + +The fact that it was able to generate the right HMAC key to download the +content though, indicates that it did get the right hybrid encryption key +(since half of that key is used to generate the HMAC). + +So hmm, I don't understand what is going on. + +Are you able to retrieve the same file successfully when the rclone.conf +is configured to use the Amazon Cloud Drive? (Assuming that the content of +the file is still present over there.) +"""]] diff --git a/doc/bugs/Hybrid_encryption_can__39__t_generate_the_right_key_after_moving_files/comment_3_414bd487619e46b25e8d9a57d9aef1f6._comment b/doc/bugs/Hybrid_encryption_can__39__t_generate_the_right_key_after_moving_files/comment_3_414bd487619e46b25e8d9a57d9aef1f6._comment new file mode 100644 index 0000000000..bc136ec452 --- /dev/null +++ b/doc/bugs/Hybrid_encryption_can__39__t_generate_the_right_key_after_moving_files/comment_3_414bd487619e46b25e8d9a57d9aef1f6._comment @@ -0,0 +1,17 @@ +[[!comment format=mdwn + username="interfect@b151490178830f44348aa57b77ad58c7d18e8fe7" + nickname="interfect" + avatar="http://cdn.libravatar.org/avatar/3ba541a3204f4ea4b274049a152489c5" + subject="comment 3" + date="2017-06-09T01:40:00Z" + content=""" +Unfortunately even though the content is still over at Amazon, ACD can no longer be accessed through rclone, so I can't get Git Annex to go over there and download it. + +With the new chunk-supporting decryption script (which I further modified to actually use the timestamps on the log entries), I am able to generate the right key for my test file. I was also able to generate the key for and decrypt another test file, and then testing with git annex again shows that I can successfully fsck other file in the remote. + +I think what is happening is that my small file I was testing with somehow became corrupted or was modified while on Amazon's servers. I downloaded from Amazon before the transfer and from Google after and did a diff, and both files are identical, so I think I moved over corrupt data. + +It looks like git annex is just successfully doing its job and identifying some data corruption here. The bug can probably be closed. + + +"""]] diff --git a/doc/bugs/Hybrid_encryption_can__39__t_generate_the_right_key_after_moving_files/comment_4_81abd2627672911ba6367effbf5c487b._comment b/doc/bugs/Hybrid_encryption_can__39__t_generate_the_right_key_after_moving_files/comment_4_81abd2627672911ba6367effbf5c487b._comment new file mode 100644 index 0000000000..66c23ee4ff --- /dev/null +++ b/doc/bugs/Hybrid_encryption_can__39__t_generate_the_right_key_after_moving_files/comment_4_81abd2627672911ba6367effbf5c487b._comment @@ -0,0 +1,9 @@ +[[!comment format=mdwn + username="interfect@b151490178830f44348aa57b77ad58c7d18e8fe7" + nickname="interfect" + avatar="http://cdn.libravatar.org/avatar/3ba541a3204f4ea4b274049a152489c5" + subject="comment 4" + date="2017-06-09T01:48:15Z" + content=""" +I'm actually seeing a lot of files (mostly smaller ones, for some reason) failing to fsck because GPG decryption failed. I can't tell at the moment whether they got corrupted in the transfer, or corrupted in the initial upload somehow. I'm pretty sure the problem here isn't with git annex itself, or more people would have noticed, but I'm definitely going to be fscking my cloud remotes more frequently. +"""]] diff --git a/doc/bugs/Hybrid_encryption_can__39__t_generate_the_right_key_after_moving_files/comment_5_ce0342e38a87017ad58c9a79b17d759a._comment b/doc/bugs/Hybrid_encryption_can__39__t_generate_the_right_key_after_moving_files/comment_5_ce0342e38a87017ad58c9a79b17d759a._comment new file mode 100644 index 0000000000..d20a6e6a13 --- /dev/null +++ b/doc/bugs/Hybrid_encryption_can__39__t_generate_the_right_key_after_moving_files/comment_5_ce0342e38a87017ad58c9a79b17d759a._comment @@ -0,0 +1,26 @@ +[[!comment format=mdwn + username="joey" + subject="""comment 5""" + date="2017-06-09T17:04:09Z" + content=""" +"I think what is happening is that my small file I was testing with somehow +became corrupted or was modified while on Amazon's servers." + +That was also kind of my guess. It's hard to imagine how +the way that a file is downloaded from The Cloud changes +how git-annex decrypts it. As long as the content is the same, +the decrpytion step should behave identially no matter where +the file is downloaded from. + +But, multiple small files getting corrupted seems like it must +have a cause other than a bit flip. Perhaps something about how +they were transferred between the two clouds corrupted them.. + +I suppose there could also be a bug in git-annex or rclone that somehow +corrupts uploads of small files. Perhaps something to do with chunking.. +What does `git annex info theremote --fast` say about its configuration? + +What is the range of sizes of small files that you've found to be +corrupted? Is there a cut-off point after which all larger files are +not corrupted? Are any small files not corrupted? +"""]] diff --git a/doc/bugs/Importing__95__Commands__95__Destructive__95__by__95__default.mdwn b/doc/bugs/Importing__95__Commands__95__Destructive__95__by__95__default.mdwn new file mode 100644 index 0000000000..0fd8de1c53 --- /dev/null +++ b/doc/bugs/Importing__95__Commands__95__Destructive__95__by__95__default.mdwn @@ -0,0 +1,30 @@ +### Please describe the problem. +the import and reinject commands are needlessly destructive by default. +this caused me some grief as it repeatedly "destroyed" data on my cammeras sdcard +as i tried to import new data nad/or reinject missing data correctly +without having to scan 40gb of pictures for hases repeatedly. + +### software versions +[[!format sh """ +$ git-annex version +git-annex version: 6.20180626-gdf91a5cff +build flags: Assistant Webapp Pairing S3(multipartupload)(storageclasses) WebDAV Inotify DBus DesktopNotify ConcurrentOutput TorrentParser MagicMime Feeds Testsuite +dependency versions: aws-0.16 bloomfilter-2.0.1.0 cryptonite-0.23 DAV-1.3.1 feed-0.3.12.0 ghc-8.0.2 http-client-0.5.7.1 persistent-sqlite-2.6.3 torrent-10000.1.1 uuid-1.3.13 yesod-1.4.5 +key/value backends: SHA256E SHA256 SHA512E SHA512 SHA224E SHA224 SHA384E SHA384 SHA3_256E SHA3_256 SHA3_512E SHA3_512 SHA3_224E SHA3_224 SHA3_384E SHA3_384 SKEIN256E SKEIN256 SKEIN512E SKEIN512 BLAKE2B256E BLAKE2B256 BLAKE2B512E BLAKE2B512 BLAKE2B160E BLAKE2B160 BLAKE2B224E BLAKE2B224 BLAKE2B384E BLAKE2B384 BLAKE2S256E BLAKE2S256 BLAKE2S160E BLAKE2S160 BLAKE2S224E BLAKE2S224 BLAKE2SP256E BLAKE2SP256 BLAKE2SP224E BLAKE2SP224 SHA1E SHA1 MD5E MD5 WORM URL +remote types: git gcrypt p2p S3 bup directory rsync web bittorrent webdav adb tahoe glacier ddar hook external +operating system: linux x86_64 +supported repository versions: 3 5 6 +upgrade supported from repository versions: 0 1 2 3 4 5 +"""]]] + +### what i would like to see + +* better documentation for commands that sort out non-destructive importing + (@joeyh already pointed out `cp` followed by `git annex add` on irc, but that doesn't cover known files/keys) +* comprehensible options that disable deletion for `reinject`/`import`, i consider the semantics of "move" for commands with actionable names like "import" and "reinject" a bit deceptive even if they are documented that way + + +for me personally the ideal case would be a cp command for git annex that would copy in new data +in a cp style allowing to use options like `noclobber` and perhaps `reinject` + +it may be sensible to leave supporting this to the importtree feature diff --git a/doc/bugs/Importing__95__Commands__95__Destructive__95__by__95__default/comment_1_96fae5927db6badfe13466cd27cfc371._comment b/doc/bugs/Importing__95__Commands__95__Destructive__95__by__95__default/comment_1_96fae5927db6badfe13466cd27cfc371._comment new file mode 100644 index 0000000000..8eba862c35 --- /dev/null +++ b/doc/bugs/Importing__95__Commands__95__Destructive__95__by__95__default/comment_1_96fae5927db6badfe13466cd27cfc371._comment @@ -0,0 +1,14 @@ +[[!comment format=mdwn + username="CandyAngel" + avatar="http://cdn.libravatar.org/avatar/15c0aade8bec5bf004f939dd73cf9ed8" + subject="comment 1" + date="2018-07-05T13:53:46Z" + content=""" +This shouldn't be under bugs, it is the intended behaviour. + +Both the words `import` and `reinject` relate to the movement of something from one place to another (importing goods, injecting fluids). Descriptions for both these commands state that data will be *moved* into the annex in the first sentence. If you aren't reading docs before using these commands, better docs are hardly going to have helped.. + +What you are trying to do can be done with `import --skip-duplicates` then `cp` -> `reinject`, albeit a bit of a pain. Personally, I would use `calckey` to generate the hashes (and keep them in a file), add hashes for files not listed in that file, then use that file to find out what keys were unknown and `import --duplicate` just those files. + +..or [following a suggestion I made a while ago](http://git-annex.branchable.com/todo/Alternative_mode_control_for_import/) `git annex import --mode=Di,Nsi` would do what you want (injects duplicates without deleting, adds new without deleting) :P +"""]] diff --git a/doc/bugs/Importing__95__Commands__95__Destructive__95__by__95__default/comment_2_432795dbe2308013b76488f273ce31e9._comment b/doc/bugs/Importing__95__Commands__95__Destructive__95__by__95__default/comment_2_432795dbe2308013b76488f273ce31e9._comment new file mode 100644 index 0000000000..cebf3d3e1b --- /dev/null +++ b/doc/bugs/Importing__95__Commands__95__Destructive__95__by__95__default/comment_2_432795dbe2308013b76488f273ce31e9._comment @@ -0,0 +1,15 @@ +[[!comment format=mdwn + username="git-annex.branchable.com@07c0f8919010cc703ae7eea746d9b494c153291f" + nickname="git-annex.branchable.com" + avatar="http://cdn.libravatar.org/avatar/c5379a3fe2188b7571858c49f9db63c6" + subject="comment 2" + date="2018-07-05T15:47:14Z" + content=""" +thanks for the outline + +i believe by refining the arguments of `import` my use-case could be sorted out +im happy to move this to a todo + +basically i'm looking for `--no-clobber` to easily skip existing known files `--reinject-missing` and `--remove-source==never` or something like that +i'd prefer to decompose the features into flags over the condensed examples you suggested +"""]] diff --git a/doc/bugs/Importing__95__Commands__95__Destructive__95__by__95__default/comment_3_3aefeda27ae0d6a34d26fc4bce56c48b._comment b/doc/bugs/Importing__95__Commands__95__Destructive__95__by__95__default/comment_3_3aefeda27ae0d6a34d26fc4bce56c48b._comment new file mode 100644 index 0000000000..9b8fcb0844 --- /dev/null +++ b/doc/bugs/Importing__95__Commands__95__Destructive__95__by__95__default/comment_3_3aefeda27ae0d6a34d26fc4bce56c48b._comment @@ -0,0 +1,8 @@ +[[!comment format=mdwn + username="joey" + subject="""comment 3""" + date="2018-07-06T16:43:15Z" + content=""" +When you say it «"destroyed" data on my cammeras sdcard» +do you mean you actually lost data, and if so, how? +"""]] diff --git a/doc/bugs/Importing__95__Commands__95__Destructive__95__by__95__default/comment_4_bc6fa5cf986502ef36d949ff3c7daa2f._comment b/doc/bugs/Importing__95__Commands__95__Destructive__95__by__95__default/comment_4_bc6fa5cf986502ef36d949ff3c7daa2f._comment new file mode 100644 index 0000000000..ba29827acf --- /dev/null +++ b/doc/bugs/Importing__95__Commands__95__Destructive__95__by__95__default/comment_4_bc6fa5cf986502ef36d949ff3c7daa2f._comment @@ -0,0 +1,10 @@ +[[!comment format=mdwn + username="git-annex.branchable.com@07c0f8919010cc703ae7eea746d9b494c153291f" + nickname="git-annex.branchable.com" + avatar="http://cdn.libravatar.org/avatar/c5379a3fe2188b7571858c49f9db63c6" + subject="comment 4" + date="2018-07-06T20:13:32Z" + content=""" +the data was not lost as i had a copy, but i was quite surprised that the data was completely gone at first as i misunderstood the command +im under the impression it was pure luck on my side that this didn't end in me getting an actual data loss +"""]] diff --git a/doc/bugs/Importing__95__Commands__95__Destructive__95__by__95__default/comment_5_b9de159a02d0ec8e6466154302c39bf3._comment b/doc/bugs/Importing__95__Commands__95__Destructive__95__by__95__default/comment_5_b9de159a02d0ec8e6466154302c39bf3._comment new file mode 100644 index 0000000000..5ff8ee4d5b --- /dev/null +++ b/doc/bugs/Importing__95__Commands__95__Destructive__95__by__95__default/comment_5_b9de159a02d0ec8e6466154302c39bf3._comment @@ -0,0 +1,12 @@ +[[!comment format=mdwn + username="andrew" + avatar="http://cdn.libravatar.org/avatar/acc0ece1eedf07dd9631e7d7d343c435" + subject="comment 5" + date="2018-07-08T16:37:04Z" + content=""" +I was also confused the first time I read the documentation of `git-annex-import`. By default it *moves* files, IE copies files into your annex and then deletes them from the source. Since `git-annex` commands seems to be eager to preserve multiple copies of files I did find it surprising that with this command it does not. I suppose one motivation for deleting the files at the source is so `git-annex` or the user can keep track of which files have been imported, IE they are using the import command to put unorganized files under the control of an annex. + +I would have expected `git-annex import` to *copy* files by default, then have an option like `--move` to *move* files instead. I'm not sure this is a compelling enough reason to change an existing command though… + +Personally I always do `git-annex import --duplicate` +"""]] diff --git a/doc/bugs/Importing__95__Commands__95__Destructive__95__by__95__default/comment_6_61ca3d5bb350e52f31d8b75dd3ece1d9._comment b/doc/bugs/Importing__95__Commands__95__Destructive__95__by__95__default/comment_6_61ca3d5bb350e52f31d8b75dd3ece1d9._comment new file mode 100644 index 0000000000..3439a759fe --- /dev/null +++ b/doc/bugs/Importing__95__Commands__95__Destructive__95__by__95__default/comment_6_61ca3d5bb350e52f31d8b75dd3ece1d9._comment @@ -0,0 +1,11 @@ +[[!comment format=mdwn + username="CandyAngel" + avatar="http://cdn.libravatar.org/avatar/15c0aade8bec5bf004f939dd73cf9ed8" + subject="comment 6" + date="2018-07-08T17:58:33Z" + content=""" + By default it moves files, IE copies files into your annex and then deletes them from the source. + +Only if you move from one filesystem to another (because it has to). +If it is on the same filesystem, *move* doesn't do any copy/delete of the data, only the references to the file. +"""]] diff --git a/doc/bugs/Importing__95__Commands__95__Destructive__95__by__95__default/comment_7_c8a0a64aa4253ba2eea525ce9fc67d4f._comment b/doc/bugs/Importing__95__Commands__95__Destructive__95__by__95__default/comment_7_c8a0a64aa4253ba2eea525ce9fc67d4f._comment new file mode 100644 index 0000000000..ed4f904d0b --- /dev/null +++ b/doc/bugs/Importing__95__Commands__95__Destructive__95__by__95__default/comment_7_c8a0a64aa4253ba2eea525ce9fc67d4f._comment @@ -0,0 +1,25 @@ +[[!comment format=mdwn + username="joey" + subject="""comment 7""" + date="2018-07-11T19:41:15Z" + content=""" +Seems to me that the documentation of git-annex import, git-annex reinject, +and git-annex setkey is clear, that these commands all move content into the +annex. + +If you don't want it to move, you can always make a copy of the file first, +and run the command on the copy. But git-annex is about managing large +files, and making a second local copy of a large file would be surprising +behavior if the user didn't explicitly ask for it. + +When you set up a remote somewhere, and ask git-annex to ensure numcopies +is 2+, it is *then* eager to preserve multiple copies, because it's been +given a way to do so, and been told to. It's not eager by default though. + +Given that people get confused with the existing number of options to +git-annex import, I'm reluctant to add more to it, especially if the same +behavior as the suggested options can be obtained by running `cp` first. + +I still don't fully understand what your proposed options are supposed +to do.. +"""]] diff --git a/doc/bugs/Importing__95__Commands__95__Destructive__95__by__95__default/comment_8_f24b544018d84b0182621d07bb9c8c3d._comment b/doc/bugs/Importing__95__Commands__95__Destructive__95__by__95__default/comment_8_f24b544018d84b0182621d07bb9c8c3d._comment new file mode 100644 index 0000000000..f3e7a385fe --- /dev/null +++ b/doc/bugs/Importing__95__Commands__95__Destructive__95__by__95__default/comment_8_f24b544018d84b0182621d07bb9c8c3d._comment @@ -0,0 +1,10 @@ +[[!comment format=mdwn + username="andrew" + avatar="http://cdn.libravatar.org/avatar/acc0ece1eedf07dd9631e7d7d343c435" + subject="comment 8" + date="2018-07-13T15:04:13Z" + content=""" +That all makes sense Joey/CandyAngel. I do wonder how this command will change though if [import tree](https://git-annex.branchable.com/todo/import_tree/) is ever implemented. I presume the default behavior of `git annex import treeish` would preserve the source tree as if the option `--duplicate` had been given? Whereas `git annex import path` would continue to move files by default. But maybe this is another reason not to do `git annex import treeish`, maybe that command should be `git annex importtree treeish`? + +In terms of the anonymous poster's suggestions of changing the options to import, I would say all of your suggestions already exist in a usable form “skip existing known files” is already an option `--skip-duplicates`, “--remove-source==never” is already an option `--duplicate` and presumably “--reinject-missing” is similar to the existing option `--force`. +"""]] diff --git a/doc/bugs/Impossible_to_enable_an_existing_gcrypt_repo_in_the_webapp.mdwn b/doc/bugs/Impossible_to_enable_an_existing_gcrypt_repo_in_the_webapp.mdwn new file mode 100644 index 0000000000..d9bae50190 --- /dev/null +++ b/doc/bugs/Impossible_to_enable_an_existing_gcrypt_repo_in_the_webapp.mdwn @@ -0,0 +1,23 @@ +### Please describe the problem. +As described earlier in [[tips/fully_encrypted_git_repositories_with_gcrypt]] + +### What steps will reproduce the problem? +- A: use the webapp to create a new repository +- A: add a remote server to the repository using the 'gcrypt' method +- A: add a jabber account + +- B: use the webapp to create a new repository +- B: add the jabber account +- B: see the previously created 'cloud repository' with status 'not enabled' +- B: click enable, see that the stored credentials are correct, and press "verify this server" +- B: enter the ssh password twice +- B: get redirected to a blank screen (on the url /config/repository/enable/gcrypt/UUID "x"?auth=y) + +The assistent logfiles show nothing after the "Your public key has been saved in", the server shows that no public key for B was added to the account. + +This is with git-annex installed on the remote server; without it the process gets stuck after clicking "encrypt repository" in step 2, it will just indefinitely keep prompting for the SSH password. + +### What version of git-annex are you using? On what operating system? +Latest nightly build on ubuntu 13.10 + +[[!tag moreinfo]] diff --git a/doc/bugs/Impossible_to_enable_an_existing_gcrypt_repo_in_the_webapp/comment_1_17814787e333d15da3ab4e57c7d31d4b._comment b/doc/bugs/Impossible_to_enable_an_existing_gcrypt_repo_in_the_webapp/comment_1_17814787e333d15da3ab4e57c7d31d4b._comment new file mode 100644 index 0000000000..5bd45d6866 --- /dev/null +++ b/doc/bugs/Impossible_to_enable_an_existing_gcrypt_repo_in_the_webapp/comment_1_17814787e333d15da3ab4e57c7d31d4b._comment @@ -0,0 +1,12 @@ +[[!comment format=mdwn + username="http://joeyh.name/" + ip="209.250.56.246" + subject="comment 1" + date="2013-11-13T16:51:40Z" + content=""" +I've tried to reproduce this, and cannot; it enabled the repository without trouble. + +Also, I have never seen the webapp fail with a blank screen, so that's strange. + +I think you need to show your `~/annex/.git/annex/daemon.log` from B. +"""]] diff --git a/doc/bugs/Inconsistent_results_between_git-annex-fsck_and_git-annex-whereis.mdwn b/doc/bugs/Inconsistent_results_between_git-annex-fsck_and_git-annex-whereis.mdwn new file mode 100644 index 0000000000..c9037b575b --- /dev/null +++ b/doc/bugs/Inconsistent_results_between_git-annex-fsck_and_git-annex-whereis.mdwn @@ -0,0 +1,53 @@ +### Please describe the problem. + +I'm seeing some inconsistent results between runs of `git annex fsck` and `git annex whereis` that I'm not able to explain. When I run `git annex fsck`, it reports a few keys that only have 1 copy, and advises me to make more copies. If I run `git annex whereis --key `, git annex confirms that it only knows about 1 copy of this key. If I then use `git log --stat -S''` to find the actual file that it refers to, and run `git annex whereis `, git annex report 9 copies of this file. Checking on remotes shows that these files do exist on the remote, so why does `git annex fsck` and `git annex whereis` mis-report the number of copies when querying for the key - but not for the actual filename? Additionally, `git annex find --lackingcopies 1` doesn't return any results, but should if there are actually files with not enough copies? + + +### What steps will reproduce the problem? + + +### What version of git-annex are you using? On what operating system? + +5.20151208-1build1 on Ubuntu Xenial, one remote running 5.20141024~bpo70+1 on Debian Wheezy + +### Please provide any additional information below. + +[[!format sh """ +# If you can, paste a complete transcript of the problem occurring here. +# If the problem is with the git-annex assistant, paste in .git/annex/daemon.log + +[william@hactar ~/Pictures/Photo Library]$ git annex whereis SHA256E-s1071765--dbaa7f32ee44c28d6a1f0c8095e8dfd8b4ec433b144085d5097425303a510ea9 +git-annex: SHA256E-s1071765--dbaa7f32ee44c28d6a1f0c8095e8dfd8b4ec433b144085d5097425303a510ea9 not found +git-annex: whereis: 1 failed +[william@hactar ~/Pictures/Photo Library]$ git annex whereis --key SHA256E-s1071765--dbaa7f32ee44c28d6a1f0c8095e8dfd8b4ec433b144085d5097425303a510ea9 +whereis SHA256E-s1071765--dbaa7f32ee44c28d6a1f0c8095e8dfd8b4ec433b144085d5097425303a510ea9 (1 copy) + 7691934f-2542-4103-9122-2db4e6cfc887 -- hactar [here] +ok +[william@hactar ~/Pictures/Photo Library]$ git annex fsck --key SHA256E-s1071765--dbaa7f32ee44c28d6a1f0c8095e8dfd8b4ec433b144085d5097425303a510ea9 +fsck SHA256E-s1071765--dbaa7f32ee44c28d6a1f0c8095e8dfd8b4ec433b144085d5097425303a510ea9 + Only 1 of 3 trustworthy copies exist of SHA256E-s1071765--dbaa7f32ee44c28d6a1f0c8095e8dfd8b4ec433b144085d5097425303a510ea9 + Back it up with git-annex copy. +failed +(recording state in git...) +git-annex: fsck: 1 failed +[william@hactar ~/Pictures/Photo Library]$ git log --stat -S'SHA256E-s1071765--dbaa7f32ee44c28d6a1f0c8095e8dfd8b4ec433b144085d5097425303a510ea9' +[william@hactar ~/Pictures/Photo Library]$ git annex whereis 2009/05/05/P1040890.JPG +whereis 2009/05/05/P1040890.JPG (9 copies) + 0e825a69-1927-4f62-b731-6f3e98bba998 -- william@marvin:/media/backup/annex/photos [marvin] + 1b728ab5-1e32-45a6-bc11-2a4bfdc9d6ab -- backup1 + 5c0caa42-b489-467b-a612-9590fa9d5a94 -- backup2 + 7691934f-2542-4103-9122-2db4e6cfc887 -- hactar [here] + 894b2216-72e0-40e1-8765-1386e1e9e4b4 -- backup3 + 96f19fa8-d385-4e8b-b000-61ee15993a70 -- backup3 + a862b121-d794-4af4-bb56-21adfe8962f2 -- S3 + b083f8ae-42fb-41f0-a2a3-4e7c9f93aadb -- [guide] + bf021ce9-465b-4419-86e7-bddfd208fca4 -- git@newzaphod:~/repositories/annex/photos.git [zaphod] +ok + + +# End of transcript or log. +"""]] + +### Have you had any luck using git-annex before? (Sometimes we get tired of reading bug reports all day and a lil' positive end note does wonders) + +I trust Git Annex to keep hundreds of GB of data safe, and it has never failed me - despite my best efforts diff --git a/doc/bugs/Inconsistent_results_between_git-annex-fsck_and_git-annex-whereis/comment_1_bd56607f228f3480f1355e3bdb755410._comment b/doc/bugs/Inconsistent_results_between_git-annex-fsck_and_git-annex-whereis/comment_1_bd56607f228f3480f1355e3bdb755410._comment new file mode 100644 index 0000000000..d65f39fd0a --- /dev/null +++ b/doc/bugs/Inconsistent_results_between_git-annex-fsck_and_git-annex-whereis/comment_1_bd56607f228f3480f1355e3bdb755410._comment @@ -0,0 +1,12 @@ +[[!comment format=mdwn + username="joey" + subject="""comment 1""" + date="2016-12-13T16:42:08Z" + content=""" +The obvious reason for this would be if the file no longer points to that +same key. Perhaps the file got modified and the key is the old version of +the file. + +That would explain everything you showed, so currently I don't see any +bug.. +"""]] diff --git a/doc/bugs/Infinite_loop_when_synchronizing_between_many_machines.mdwn b/doc/bugs/Infinite_loop_when_synchronizing_between_many_machines.mdwn new file mode 100644 index 0000000000..5fc8da66b7 --- /dev/null +++ b/doc/bugs/Infinite_loop_when_synchronizing_between_many_machines.mdwn @@ -0,0 +1,27 @@ +### Please describe the problem. + +When an assistant adds a lot of new files, and there’s an assistant running for the repo on each of some 5 machines, infinite loop happens on those receiving the new files. + +### What steps will reproduce the problem? + +1. Have 5 machines with a repo checked out. +2. Run git-annex-assistant on each. +3. Copy a lot of files into the repo on one machine. +4. Look at the webapp UIs on other machines. +5. In the UI, the same files get enqueued and dequeued repeatedly and nothing else happens. + +### What version of git-annex are you using? On what operating system? + +git-annex version: 6.20170520 + +local repository version: 6 + +operating system: linux x86_64 + +### Please provide any additional information below. + +— + +### Have you had any luck using git-annex before? (Sometimes we get tired of reading bug reports all day and a lil' positive end note does wonders) + +Yes! But do you test it for similar races on more than two simultaneous machines? diff --git a/doc/bugs/Infinite_loop_when_synchronizing_between_many_machines/comment_1_6d8f42446749f3e697af20626f22cf8b._comment b/doc/bugs/Infinite_loop_when_synchronizing_between_many_machines/comment_1_6d8f42446749f3e697af20626f22cf8b._comment new file mode 100644 index 0000000000..037f632c80 --- /dev/null +++ b/doc/bugs/Infinite_loop_when_synchronizing_between_many_machines/comment_1_6d8f42446749f3e697af20626f22cf8b._comment @@ -0,0 +1,8 @@ +[[!comment format=mdwn + username="michalrus" + avatar="http://cdn.libravatar.org/avatar/83c0b6e7f9d20f09a892263c4903bbae" + subject="comment 1" + date="2017-08-20T15:27:31Z" + content=""" +Restarting the daemon helps, but I can’t reasonably expect my parents to do this regularly… :c +"""]] diff --git a/doc/bugs/Initial___34__Make_repository__34___fails_due_to___34__empty_ident_name___40__for___60__email-address__62____41___not_allowed__34__.mdwn b/doc/bugs/Initial___34__Make_repository__34___fails_due_to___34__empty_ident_name___40__for___60__email-address__62____41___not_allowed__34__.mdwn new file mode 100644 index 0000000000..b15cfad68e --- /dev/null +++ b/doc/bugs/Initial___34__Make_repository__34___fails_due_to___34__empty_ident_name___40__for___60__email-address__62____41___not_allowed__34__.mdwn @@ -0,0 +1,43 @@ +### Please describe the problem. + +During an initial run of "git annex webapp", it gets stuck at the "Make repository" stage. Entering a directory and pressing "Make repository" takes the user back to the same page repeatedly, preventing them from advancing. + +### What steps will reproduce the problem? + +Download https://downloads.kitenet.net/git-annex/linux/current/git-annex-standalone-i386.tar.gz from 2015-10-19 (sha1: 3b30de86d6790fdf80de90b2f54967de8345fe98 ) on an Ubuntu Trusty Tahr server, untar, run git-annex-webapp, attempt to make a Repository in ~/annex. "Make repository" fails to advance. + +### What version of git-annex are you using? On what operating system? + +git-annex-standalone-i386.tar.gz downloaded on 2015-10-19 (sha1: 3b30de86d6790fdf80de90b2f54967de8345fe98), on Ubuntu Trusty Tahr (14.04.3). + +### Please provide any additional information below. + +The following is printed to the console (no daemon.log is generated): + +[[!format sh """ +~/src/git-annex.linux$ ./git-annex-webapp +Launching web browser on file:///tmp/webapp8296.html + + +*** Please tell me who you are. + +Run + + git config --global user.email "you@example.com" + git config --global user.name "Your Name" + +to set your account's default identity. +Omit --global to set the identity only in this repository. + +fatal: empty ident name (for ) not allowed +(recording state in git...) + + +# End of transcript or log. +"""]] + +Manually running the "git config --global user.name" will allow the process to proceed. It will just be confusing to users who launched the webapp via a menu item. + +### Have you had any luck using git-annex before? (Sometimes we get tired of reading bug reports all day and a lil' positive end note does wonders) + +Works well using the debian testing package on a debian laptop! diff --git a/doc/bugs/Initial___34__Make_repository__34___fails_due_to___34__empty_ident_name___40__for___60__email-address__62____41___not_allowed__34__/comment_1_267b252368dca4803d706c70a3ef3244._comment b/doc/bugs/Initial___34__Make_repository__34___fails_due_to___34__empty_ident_name___40__for___60__email-address__62____41___not_allowed__34__/comment_1_267b252368dca4803d706c70a3ef3244._comment new file mode 100644 index 0000000000..376ea92efe --- /dev/null +++ b/doc/bugs/Initial___34__Make_repository__34___fails_due_to___34__empty_ident_name___40__for___60__email-address__62____41___not_allowed__34__/comment_1_267b252368dca4803d706c70a3ef3244._comment @@ -0,0 +1,29 @@ +[[!comment format=mdwn + username="joey" + subject="""comment 1""" + date="2015-11-10T19:45:36Z" + content=""" +I was able to completely reproduce this in every particular, except for one +important thing: + +The webapp *was* able to advance; the repository was created, and +when I looked inside its .git/config, I saw as expected: + + [user] + name = foo + email = bar + +So, as far as I can see, git-annex's existing code to detect when the +system is so badly configured that git commit doesn't work, and fix the git +configuration of the repository well enough for git commit to proceed, is +working. The git error just occurs when git-annex is doing that probing. + +And, in your transcript, it says "(recording state in git...)" so +it seems to have gotten past that git error just fine as well. + +If the webapp didn't advance beyond creating the repository, what error +message was displayed **in the webapp**? + +Currently, my guess is you saw scary stuff on the console and assumed it +meant it wouldn't work, but it did. +"""]] diff --git a/doc/bugs/Installing_git-annex_on_macOS_10.13_HighSierra_via_brew_install_git-annex_require_to_install_lib_magic_via_the_command_brew_install_lib_magic.mdwn b/doc/bugs/Installing_git-annex_on_macOS_10.13_HighSierra_via_brew_install_git-annex_require_to_install_lib_magic_via_the_command_brew_install_lib_magic.mdwn new file mode 100644 index 0000000000..310761e690 --- /dev/null +++ b/doc/bugs/Installing_git-annex_on_macOS_10.13_HighSierra_via_brew_install_git-annex_require_to_install_lib_magic_via_the_command_brew_install_lib_magic.mdwn @@ -0,0 +1,29 @@ +### Please describe the problem. + +Installing git annex on MacOS 10.13 HighSierra via brew install git-annex issues an error: +... +Trap 6 + +The fix is to install lib magic with this command: + +brew install libmagic + + +### What steps will reproduce the problem? + + +### What version of git-annex are you using? On what operating system? + + +### Please provide any additional information below. + +[[!format sh """ +# If you can, paste a complete transcript of the problem occurring here. +# If the problem is with the git-annex assistant, paste in .git/annex/daemon.log + + +# End of transcript or log. +"""]] + +### Have you had any luck using git-annex before? (Sometimes we get tired of reading bug reports all day and a lil' positive end note does wonders) + diff --git a/doc/bugs/Installing_git-annex_on_macOS_10.13_HighSierra_via_brew_install_git-annex_require_to_install_lib_magic_via_the_command_brew_install_lib_magic/comment_1_43614460d92cfac8a40ad2d6654926b9._comment b/doc/bugs/Installing_git-annex_on_macOS_10.13_HighSierra_via_brew_install_git-annex_require_to_install_lib_magic_via_the_command_brew_install_lib_magic/comment_1_43614460d92cfac8a40ad2d6654926b9._comment new file mode 100644 index 0000000000..5be4a0a662 --- /dev/null +++ b/doc/bugs/Installing_git-annex_on_macOS_10.13_HighSierra_via_brew_install_git-annex_require_to_install_lib_magic_via_the_command_brew_install_lib_magic/comment_1_43614460d92cfac8a40ad2d6654926b9._comment @@ -0,0 +1,9 @@ +[[!comment format=mdwn + username="joey" + subject="""comment 1""" + date="2018-03-16T15:55:48Z" + content=""" +I don't think I can make any changes to git-annex to fix this, it's an +issue with the homebrew packaging of it, which should pull in the libmagic +dependency. Can you file a bug on homebrew about this? +"""]] diff --git a/doc/bugs/Is_there_any_way_to_rate_limit_uploads_to_an_S3_backend__63__.mdwn b/doc/bugs/Is_there_any_way_to_rate_limit_uploads_to_an_S3_backend__63__.mdwn new file mode 100644 index 0000000000..0e7d61b2f8 --- /dev/null +++ b/doc/bugs/Is_there_any_way_to_rate_limit_uploads_to_an_S3_backend__63__.mdwn @@ -0,0 +1,20 @@ +What steps will reproduce the problem? + +Adding files to a local annex set up to sync to a remote S3 one + + +What is the expected output? What do you see instead? + +It syncs, but maxes out the uplink + + +What version of git-annex are you using? On what operating system? + +3.20121112 on Debian testing + + +Please provide any additional information below. + +The man page lists how to configure rate limiting for rsync, not sure how to do it for this + +[[!tag confirmed]] diff --git a/doc/bugs/Is_there_any_way_to_rate_limit_uploads_to_an_S3_backend__63__/comment_1_ef97e735ce308f7bcc03f5d9fda588bf._comment b/doc/bugs/Is_there_any_way_to_rate_limit_uploads_to_an_S3_backend__63__/comment_1_ef97e735ce308f7bcc03f5d9fda588bf._comment new file mode 100644 index 0000000000..67de2becf4 --- /dev/null +++ b/doc/bugs/Is_there_any_way_to_rate_limit_uploads_to_an_S3_backend__63__/comment_1_ef97e735ce308f7bcc03f5d9fda588bf._comment @@ -0,0 +1,10 @@ +[[!comment format=mdwn + username="https://www.google.com/accounts/o8/id?id=AItOawmBUR4O9mofxVbpb8JV9mEbVfIYv670uJo" + nickname="Justin" + subject="comment 1" + date="2012-11-13T15:10:45Z" + content=""" +There's always trickle.. + + trickle -u 50 git annex ... +"""]] diff --git a/doc/bugs/Is_there_any_way_to_rate_limit_uploads_to_an_S3_backend__63__/comment_2_539b89de8743e435386b86119d1e982f._comment b/doc/bugs/Is_there_any_way_to_rate_limit_uploads_to_an_S3_backend__63__/comment_2_539b89de8743e435386b86119d1e982f._comment new file mode 100644 index 0000000000..122bafbba5 --- /dev/null +++ b/doc/bugs/Is_there_any_way_to_rate_limit_uploads_to_an_S3_backend__63__/comment_2_539b89de8743e435386b86119d1e982f._comment @@ -0,0 +1,8 @@ +[[!comment format=mdwn + username="http://joeyh.name/" + ip="4.252.11.120" + subject="comment 2" + date="2012-11-13T17:09:31Z" + content=""" +It's not there yet. I don't think it will be hard to add, most of the same things need to be done to support upload progress bars with S3. +"""]] diff --git a/doc/bugs/Issue_fewer_S3_GET_requests.mdwn b/doc/bugs/Issue_fewer_S3_GET_requests.mdwn new file mode 100644 index 0000000000..8bbcfa1792 --- /dev/null +++ b/doc/bugs/Issue_fewer_S3_GET_requests.mdwn @@ -0,0 +1,9 @@ +It appears that git-annex issues one GET request to S3 / Google cloud for every file it tries to copy, if you don't pass --fast. (I could be wrong; I'm basing this on the fact that each "checking " takes about the same amount of time, and that it's slow enough to be hitting the network.) + +Amazon lets you GET 1000 objects in one GET request, and afaict a request that returns 1000 objects costs just as much as a request that returns 1 object. The cost of GET'ing every file in my annex is nontrivial -- Google charges 0.01 per 1000 GETs, and my repo has 130k objects, so that's $1.3, compared to a monthly cost for storage of under $10. This means that if I want to back up my files more than, say, once a week, I need to write a script that parses the JSON output of git annex whereis and uploads with --fast only the files that aren't present in the cloud. It also means that I have to trust the output of whereis. + +All those GETs also slow down the non-fast copy, and this also applies to other kinds of remotes. + +There are a number of ways one could implement this. One way would be to have a command that updates the whereis data from the remote and then to add a parameter (maybe you already have it) to copy that's like --fast but skips files that are already present (maybe this is what --fast already does, but I did a quick check and it doesn't seem to). Because of the way git annex names files, I think it would be hard to coalesce GETs during a copy command, but it could be done. + +Anyway, please don't consider this a high-priority request; I can get by as-is, and I <3 git annex. diff --git a/doc/bugs/Issue_fewer_S3_GET_requests/comment_1_59a6b87d01c003bf55cde4c882e1778c._comment b/doc/bugs/Issue_fewer_S3_GET_requests/comment_1_59a6b87d01c003bf55cde4c882e1778c._comment new file mode 100644 index 0000000000..b0c6b76132 --- /dev/null +++ b/doc/bugs/Issue_fewer_S3_GET_requests/comment_1_59a6b87d01c003bf55cde4c882e1778c._comment @@ -0,0 +1,18 @@ +[[!comment format=mdwn + username="http://joeyh.name/" + ip="209.250.56.96" + subject="comment 1" + date="2014-10-22T21:36:27Z" + content=""" +The man page documents this: + +> To avoid contacting the remote to check if it has every +> file when copying --to the repository, specify --fast + +As you've noted, this has to rely on the location tracking information being up-to-date, so if it's not it might miss copying a file to the remote that the remote doesn't currently have but used to. Otherwise, it's fine to use `copy --fast --to --remote` or `copy --not --in remote --to remote`, which is functionally identical. + +The check is not a GET request, it's a HEAD request, to check if the file +is present. Does S3 have a way to combine multiple HEAD requests in a +single http request? That seems unlikely. Maybe it is enough to reuse an +open http connection for multiple HEADs? Anything needing a single HEAD request would not fit well into git-annex, but ways to do more caching of open http connections are being considered. +"""]] diff --git a/doc/bugs/Issue_fewer_S3_GET_requests/comment_2_8fdeb3352ac00a76625ec9c007e5b76a._comment b/doc/bugs/Issue_fewer_S3_GET_requests/comment_2_8fdeb3352ac00a76625ec9c007e5b76a._comment new file mode 100644 index 0000000000..b3b2594450 --- /dev/null +++ b/doc/bugs/Issue_fewer_S3_GET_requests/comment_2_8fdeb3352ac00a76625ec9c007e5b76a._comment @@ -0,0 +1,14 @@ +[[!comment format=mdwn + username="https://www.google.com/accounts/o8/id?id=AItOawmUJBh1lYmvfCCiGr3yrdx-QhuLCSRnU5c" + nickname="Justin" + subject="comment 2" + date="2014-10-24T04:49:20Z" + content=""" +Oh jeez, I screwed that up wrt HEAD and GET. Sorry. The cost per HEAD on Google is 1/10 the price of GET, so we're talking $.13 to HEAD my 130k-file annex, which is totally reasonable. + +One can GET a bucket, which is what I was looking at. This returns up to 1000 elements of its contents (and there's a way to iterate over larger buckets). Of course this would only be useful if the majority of files in the bucket were of interest to git-annex, and it sounds like more trouble than it's worth at the prices I'm seeing. + +There might be a throughput improvement to be had by keeping the connection alive, although in my brief investigation, I think there may be a larger gain to be had by pipelining the various steps. Based on the fact that git-annex oomed when trying to upload a large file from my rpi, it seems like maybe the whole file is encrypted in memory before it's uploaded? And certainly the HEAD(s) appear not to be done in parallel with the upload. + +Sorry again for that HEAD/GET fail. +"""]] diff --git a/doc/bugs/Issue_fewer_S3_GET_requests/comment_3_6ccbb1cff7bc6b4640220d98f7ce21c3._comment b/doc/bugs/Issue_fewer_S3_GET_requests/comment_3_6ccbb1cff7bc6b4640220d98f7ce21c3._comment new file mode 100644 index 0000000000..353281db65 --- /dev/null +++ b/doc/bugs/Issue_fewer_S3_GET_requests/comment_3_6ccbb1cff7bc6b4640220d98f7ce21c3._comment @@ -0,0 +1,12 @@ +[[!comment format=mdwn + username="http://joeyh.name/" + ip="209.250.56.96" + subject="comment 3" + date="2014-10-24T16:02:23Z" + content=""" +The OOM is [[S3_memory_leaks]]; fixed in the s3-aws branch. + +Yeah, GET of a bucket is doable. Another problem with it though is, if the bucket has a lot of contents, such as many files, or large files split into many chunks, that all has to be buffered in memory or processed as a stream. It would make sense in operations where git-annex knows it wants to check every key in a bucket. `git annex unused --from $s3remote` is the case that springs to mind where it could be quite useful to do that. Integrating it with `get`, not so much. + +I'd be inclined to demote this to a wishlist todo item to try to use bucket GET for `unused`. And/or rethink whether it makes sense for `copy --to` to run in --fast mode by default. I've been back and forth on that question before, but just from a runtime perspective, not from a 13 cents perspective. ;) +"""]] diff --git a/doc/bugs/Issue_on_OSX_with_some_system_limits.mdwn b/doc/bugs/Issue_on_OSX_with_some_system_limits.mdwn new file mode 100644 index 0000000000..3942132208 --- /dev/null +++ b/doc/bugs/Issue_on_OSX_with_some_system_limits.mdwn @@ -0,0 +1,27 @@ +I was dumping ~gigs of files of approximately 3-6megs a pop (my music collection) so I could track the files that I want to listen to when I'm on the go. I had the git watch command running from the assistant branch. + +I was getting something along the lines of... + + /Users/jtang/annex/.git/annex/tmp/: openTempFile: resource exhausted (Too many open files) + +and + + git-annex: createPipe: resource exhausted (Too many open files) + +I also noticed that I somehow ended up with 256 ssh-agent's running on one of my machines, I'm not sure if the two issues are related or not, I had not noticed this type of behaviour up until recently. + +Also this was appearing in the logs + + x00:annex jtang$ tail -f .git/annex/daemon.log + (scanning...) Already up-to-date. + kqueue: Too many open files + +To be precise, I suspect that the kqueue limit is 256, I had 325 files in the 'queue', I ended up doing a _git annex add_ manually and all was fine. + +[[!meta title="kqueue system limits"]] + +> This affects BSD systems that use Kqueue. It no longer affects OSX, +> since we use FSEvents there instead. --[[Joey]] + +[[!tag /design/assistant]] +[[!tag confirmed]] diff --git a/doc/bugs/Issue_on_OSX_with_some_system_limits/comment_1_5fc1eedb5231edc37c87a2d9b91313b9._comment b/doc/bugs/Issue_on_OSX_with_some_system_limits/comment_1_5fc1eedb5231edc37c87a2d9b91313b9._comment new file mode 100644 index 0000000000..d30cddca50 --- /dev/null +++ b/doc/bugs/Issue_on_OSX_with_some_system_limits/comment_1_5fc1eedb5231edc37c87a2d9b91313b9._comment @@ -0,0 +1,12 @@ +[[!comment format=mdwn + username="http://joeyh.name/" + ip="4.153.2.25" + subject="comment 1" + date="2012-06-25T15:42:48Z" + content=""" +Yes, this is a known problem with kqueue, it has to keep every directory in the tree open. On [[design/assistant/inotify]] I have a note that it may need to fork off extra watcher processes to deal with this. Of course that adds significant complication. + +In the meantime, you may be able to increase your system's maximum allowed number of open files per process somehow. + +(I doubt that the ssh-agent is related; git-annex does not use ssh-agent directly anyway..) +"""]] diff --git a/doc/bugs/Issue_on_OSX_with_some_system_limits/comment_2_b14e697c211843163285aaa8de5bf4c6._comment b/doc/bugs/Issue_on_OSX_with_some_system_limits/comment_2_b14e697c211843163285aaa8de5bf4c6._comment new file mode 100644 index 0000000000..17dcf76343 --- /dev/null +++ b/doc/bugs/Issue_on_OSX_with_some_system_limits/comment_2_b14e697c211843163285aaa8de5bf4c6._comment @@ -0,0 +1,12 @@ +[[!comment format=mdwn + username="https://www.google.com/accounts/o8/id?id=AItOawkSq2FDpK2n66QRUxtqqdbyDuwgbQmUWus" + nickname="Jimmy" + subject="comment 2" + date="2012-06-29T12:02:48Z" + content=""" +Doing, + + sudo sysctl -w kern.maxfilesperproc=400000 + +Somewhat works for me, git-annex watch at least starts up and takes a while to scan the directory, but it's not ideal. Also, creating files seems to work okay, when I remove a file the changes don't seem to get pushed across my other repos, running a sync on the remote repo fixes things. +"""]] diff --git a/doc/bugs/Issue_on_OSX_with_some_system_limits/comment_3_18ddf8b5934dd6fb1676cd6adc7d103b._comment b/doc/bugs/Issue_on_OSX_with_some_system_limits/comment_3_18ddf8b5934dd6fb1676cd6adc7d103b._comment new file mode 100644 index 0000000000..eb886acf6b --- /dev/null +++ b/doc/bugs/Issue_on_OSX_with_some_system_limits/comment_3_18ddf8b5934dd6fb1676cd6adc7d103b._comment @@ -0,0 +1,19 @@ +[[!comment format=mdwn + username="http://joeyh.name/" + subject="comment 3" + date="2012-07-04T12:32:44Z" + content=""" +Jimmy, sounds like I could use something like this to get the current limit: + + sysctl kern.maxfilesperproc + +Probably prints \"sysctl kern.maxfilesperproc = 256\" or such.. can you verify? +Once I have the limit, I can make the kqueue code use subset of it, and print out a message when it needs to be increased, like the inotify code does. + +(Also, the kqueue code only opens directories, not files, so unless you have 400000 directories, that's +a little high.) + +--- + +On file removal not propigating, does this still happen? When you remove a file does a git commit automatically happen, or is that broken with kqueue? +"""]] diff --git a/doc/bugs/Issue_on_OSX_with_some_system_limits/comment_4_c25a8eb369e546f65e1a72d89f43066f._comment b/doc/bugs/Issue_on_OSX_with_some_system_limits/comment_4_c25a8eb369e546f65e1a72d89f43066f._comment new file mode 100644 index 0000000000..509752bf18 --- /dev/null +++ b/doc/bugs/Issue_on_OSX_with_some_system_limits/comment_4_c25a8eb369e546f65e1a72d89f43066f._comment @@ -0,0 +1,12 @@ +[[!comment format=mdwn + username="http://joeyh.name/" + subject="comment 4" + date="2012-07-08T18:13:58Z" + content=""" +On kFreeBSD, I get this: + + $ sysctl kern.maxfilesperproc + kern.maxfilesperproc: 11095 + +But ulimit still has 1024 limit, so you'd need to adjust both, as root. Messy.. +"""]] diff --git a/doc/bugs/Issue_on_OSX_with_some_system_limits/comment_5_6407a3e7aa0316cba2994bfef0e3c633._comment b/doc/bugs/Issue_on_OSX_with_some_system_limits/comment_5_6407a3e7aa0316cba2994bfef0e3c633._comment new file mode 100644 index 0000000000..30ea6b310e --- /dev/null +++ b/doc/bugs/Issue_on_OSX_with_some_system_limits/comment_5_6407a3e7aa0316cba2994bfef0e3c633._comment @@ -0,0 +1,37 @@ +[[!comment format=mdwn + username="https://www.google.com/accounts/o8/id?id=AItOawkSq2FDpK2n66QRUxtqqdbyDuwgbQmUWus" + nickname="Jimmy" + subject="comment 4" + date="2012-07-04T13:17:05Z" + content=""" +In relation to the system limits, + + laplace:~ jtang$ sysctl kern.maxfilesperproc + kern.maxfilesperproc: 10240 + +Also, the maxfiles for the whole system is + + laplace:~ jtang$ sysctl kern.maxfiles + kern.maxfiles: 12288 + +the above was the defaults as far as I recall. What you probably would be interested is the ulimits that the user see + + laplace:~ jtang$ ulimit -a + core file size (blocks, -c) 0 + data seg size (kbytes, -d) unlimited + file size (blocks, -f) unlimited + max locked memory (kbytes, -l) unlimited + max memory size (kbytes, -m) unlimited + open files (-n) 256 + pipe size (512 bytes, -p) 1 + stack size (kbytes, -s) 8192 + cpu time (seconds, -t) unlimited + max user processes (-u) 709 + virtual memory (kbytes, -v) unlimited + +I would imagine the limit that you are looking for is 256. Hope this helps. + +---- + +On the point about deletions not being propagated, it does do a commit. I suspect that the kqueue code is just not picking up the changes and pushing the changes out. The watch command on a single annex with no remotes functions as expected. +"""]] diff --git a/doc/bugs/Issue_on_OSX_with_some_system_limits/comment_6_f01887695e8b8386e125464c6d401565._comment b/doc/bugs/Issue_on_OSX_with_some_system_limits/comment_6_f01887695e8b8386e125464c6d401565._comment new file mode 100644 index 0000000000..cd5c73a7a5 --- /dev/null +++ b/doc/bugs/Issue_on_OSX_with_some_system_limits/comment_6_f01887695e8b8386e125464c6d401565._comment @@ -0,0 +1,8 @@ +[[!comment format=mdwn + username="https://www.google.com/accounts/o8/id?id=AItOawkSq2FDpK2n66QRUxtqqdbyDuwgbQmUWus" + nickname="Jimmy" + subject="comment 2" + date="2012-06-25T22:36:39Z" + content=""" +On the system limits side, I think if you want to make it more approachable by more users then adjusting system limits might scare users away. On the note of the ssh-agents spawning like no tomorrow on my machine, it turned out that i had a symlink from my .bashrc to .bash_profile, I guess I should not be too lazy and have two seperate files. +"""]] diff --git a/doc/bugs/Issue_on_OSX_with_some_system_limits/comment_7_c7776d5b2d073e0d2ae36515185c25aa._comment b/doc/bugs/Issue_on_OSX_with_some_system_limits/comment_7_c7776d5b2d073e0d2ae36515185c25aa._comment new file mode 100644 index 0000000000..b65dc4dfcc --- /dev/null +++ b/doc/bugs/Issue_on_OSX_with_some_system_limits/comment_7_c7776d5b2d073e0d2ae36515185c25aa._comment @@ -0,0 +1,17 @@ +[[!comment format=mdwn + username="http://edheil.wordpress.com/" + ip="173.162.44.162" + subject="Just encountered this today" + date="2012-11-29T19:21:50Z" + content=""" +& it's killing my assistant. + +Found a workaround here: https://github.com/mockko/livereload/issues/1 + +This fixes it for now: + +ulimit -n 4096 + +Just posting here in case anybody else runs into this issue and, like me, finds themselves here via search engine. + +"""]] diff --git a/doc/bugs/It_is_very_easy_to_turn_git-annex_into_a_zombie.mdwn b/doc/bugs/It_is_very_easy_to_turn_git-annex_into_a_zombie.mdwn new file mode 100644 index 0000000000..9148252091 --- /dev/null +++ b/doc/bugs/It_is_very_easy_to_turn_git-annex_into_a_zombie.mdwn @@ -0,0 +1,25 @@ +What steps will reproduce the problem? + +Run the git-annex assitant, and then "sudo kill" it. + +What is the expected output? What do you see instead? + +I expect it to die, instead I end up with: + + 14604 ?? S 0:00.64 ga assistant + 14623 ?? Z 0:00.00 (git) + 14624 ?? Z 0:00.00 (git) + 14936 ?? Z 0:00.00 (git-annex) + +The only way to clear these zombies is to reboot. Perhaps there is some resource not being correctly terminated under exceptional conditions? + +Note that on OpenIndiana the problem is even more severe: Aborting git-annex at the wrong time leaves behind both zombie processes and lock files which cause the machine to suddenly halt if I try to access them in any way (via mv, rsync, etc)! + +What version of git-annex are you using? On what operating system? + +4d1e0c9 on OS X 10.8.2. + +Please provide any additional information below. + +[[!meta title="strange OSX behavior when killed"]] +[[!tag /design/assistant/OSX moreinfo]] diff --git a/doc/bugs/It_is_very_easy_to_turn_git-annex_into_a_zombie/comment_1_d5fba6c061fb21795021ea83070dbfa2._comment b/doc/bugs/It_is_very_easy_to_turn_git-annex_into_a_zombie/comment_1_d5fba6c061fb21795021ea83070dbfa2._comment new file mode 100644 index 0000000000..8d7eea6c52 --- /dev/null +++ b/doc/bugs/It_is_very_easy_to_turn_git-annex_into_a_zombie/comment_1_d5fba6c061fb21795021ea83070dbfa2._comment @@ -0,0 +1,8 @@ +[[!comment format=mdwn + username="https://me.yahoo.com/a/2grhJvAC049fJnvALDXek.6MRZMTlg--#eec89" + nickname="John" + subject="Correction" + date="2012-10-20T05:34:47Z" + content=""" +If I \"sudo kill -9\" all the processes in the above group after deleting the annex directories, then they do go away without needing to reboot. +"""]] diff --git a/doc/bugs/It_is_very_easy_to_turn_git-annex_into_a_zombie/comment_2_12cba707239018989e8d5b6f456fa754._comment b/doc/bugs/It_is_very_easy_to_turn_git-annex_into_a_zombie/comment_2_12cba707239018989e8d5b6f456fa754._comment new file mode 100644 index 0000000000..2b552c6aa4 --- /dev/null +++ b/doc/bugs/It_is_very_easy_to_turn_git-annex_into_a_zombie/comment_2_12cba707239018989e8d5b6f456fa754._comment @@ -0,0 +1,10 @@ +[[!comment format=mdwn + username="http://joeyh.name/" + ip="4.154.0.23" + subject="comment 2" + date="2012-10-22T14:01:42Z" + content=""" +It's not at all clear to me from this what process you killed. + +(I can't comment on files that somehow crash a kernel on access.. That's not in any UNIX spec I'm aware of.) +"""]] diff --git a/doc/bugs/Linux_standalone__39__s_metadata_--batch_can__39__t_parse_UTF-8.mdwn b/doc/bugs/Linux_standalone__39__s_metadata_--batch_can__39__t_parse_UTF-8.mdwn new file mode 100644 index 0000000000..73c7ae8641 --- /dev/null +++ b/doc/bugs/Linux_standalone__39__s_metadata_--batch_can__39__t_parse_UTF-8.mdwn @@ -0,0 +1,88 @@ +### Please describe the problem. +I had also commented this on [[another bug|bugs/git-annex_fromkey_barfs_on_utf-8_input]], but the original issue there is fixed now. +I tested `fromkey`, `calckey --batch`, `lookupkey --batch` (in standalone) after your fix, they work nicely. + +However, `git-annex metadata --batch --json` using the [[linux standalone|install/Linux_standalone]] (autobuild) still fails when it encounters UTF-8 characters (e.g. ü, ç, ä). +Also, `git-annex metadata --json` gives `"file":"��.txt"` for `ü.txt`. + +This happens only in the standalone builds. + +### What steps will reproduce the problem? + +[[!format sh """ +$ .../git-annex.linux/runshell +$ touch u.txt ü.txt +$ git-annex add . + +$ git-annex metadata --batch --json +{"file":"ü.txt"} +git-annex: Batch input parse failure: Error in $: Failed reading: Cannot decode byte '\xb3': Data.Text.Internal.Encoding.decodeUtf8: Invalid UTF-8 stream + +$ git-annex metadata --batch --json +{"file":"u.txt","fields":{"ç":["b"]}} +git-annex: Batch input parse failure: Error in $: Failed reading: Cannot decode byte '\xb3': Data.Text.Internal.Encoding.decodeUtf8: Invalid UTF-8 stream + +$ git-annex metadata --batch --json +{"file":"u.txt","fields":{"b":["ä"]}} +git-annex: Batch input parse failure: Error in $: Failed reading: Cannot decode byte '\xb3': Data.Text.Internal.Encoding.decodeUtf8: Invalid UTF-8 stream + +$ git-annex metadata --json +{"command":"metadata","note":"","success":true,"key":"SHA256E-s0--e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855.txt","file":"u.txt","fields":{}} +{"command":"metadata","note":"","success":true,"key":"SHA256E-s0--e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855.txt","file":"��.txt","fields":{}} +# success, but the second line should have "file":"ü.txt" +"""]] + +It's the same even if I call `.../git-annex.linux/git-annex` directly (without `runshell`) + +### What version of git-annex are you using? On what operating system? +Using the Linux standalone: [git-annex-standalone-amd64.tar.gz](https://downloads.kitenet.net/git-annex/autobuild/amd64/git-annex-standalone-amd64.tar.gz) on Xubuntu 16.04 + +[[!format sh """ +$ git-annex version +git-annex version: 6.20161213-g55a34b493 +build flags: Assistant Webapp Pairing Testsuite S3(multipartupload)(storageclasses) WebDAV Inotify DBus DesktopNotify XMPP ConcurrentOutput TorrentParser MagicMime Feeds Quvi +key/value backends: SHA256E SHA256 SHA512E SHA512 SHA224E SHA224 SHA384E SHA384 SHA3_256E SHA3_256 SHA3_512E SHA3_512 SHA3_224E SHA3_224 SHA3_384E SHA3_384 SKEIN256E SKEIN256 SKEIN512E SKEIN512 SHA1E SHA1 MD5E MD5 WORM URL +remote types: git gcrypt p2p S3 bup directory rsync web bittorrent webdav tahoe glacier ddar hook external +local repository version: 5 +supported repository versions: 3 5 6 +upgrade supported from repository versions: 0 1 2 3 4 5 +operating system: linux x86_64 +"""]] + +### Please provide any additional information below. + +None of the characters I used have `\xb3` in it, but all the errors happen due to it: +[[!format sh """ +$ .../git-annex.linux/runshell +$ echo -n ü | xxd +00000000: c3bc .. +$ echo -n ç | xxd +00000000: c3a7 .. +$ echo -n ä | xxd +00000000: c3a4 .. +"""]] + +In `runshell`, `ls` can't show UTF-8, but `git-annex status` can: +[[!format sh """ +$ .../git-annex.linux/runshell +$ ls +u.txt ??.txt +$ git-annex status +A u.txt +A ü.txt +"""]] + +`man` complains about locale in `runshell` as well: +[[!format sh """ +$ .../git-annex.linux/runshell +$ man +man: can\'t set the locale; make sure $LC_* and $LANG are correct +What manual page do you want? +# I escaped that \', formatting was messy otherwise +$ set | grep LANG +GDM_LANG='en_GB' +LANG='en_GB.UTF-8' +LANGUAGE='en_GB:en' +$ set | grep LC +# nothing +"""]] diff --git a/doc/bugs/Linux_standalone__39__s_metadata_--batch_can__39__t_parse_UTF-8/comment_1_1765400777911cc61eb591b76c84ae89._comment b/doc/bugs/Linux_standalone__39__s_metadata_--batch_can__39__t_parse_UTF-8/comment_1_1765400777911cc61eb591b76c84ae89._comment new file mode 100644 index 0000000000..4a15b19871 --- /dev/null +++ b/doc/bugs/Linux_standalone__39__s_metadata_--batch_can__39__t_parse_UTF-8/comment_1_1765400777911cc61eb591b76c84ae89._comment @@ -0,0 +1,45 @@ +[[!comment format=mdwn + username="joey" + subject="""comment 1""" + date="2016-12-19T20:37:56Z" + content=""" +runshell was recently changed to bypass using the system locales, it +includes its own locale data and attempts to generate a locale definition +file for the locale. The code that did that was failing to notice that +en_GB.UTF-8 was a UTF-8 locale (en_GB.utf8 would work though), which +explains why the locale is not set inside runshell +(git-annex.linux/git-annex is a script that uses runshell). I've corrected +that problem, and verified it fixes the problem you reported. + +---- + +However.. The same thing happens when using LANG=C with git-annex +installed by any method and --json --batch. So the deeper problem is that +it's forcing the batch input to be decoded as utf8 via the current locale. +This happens in Command/MetaData.hs parseJSONInput which uses +`BU.fromString`. + +I tried swapping in `encodeBS` for `BU.fromString`. That prevented the +decoding error, but made git-annex complain that the file was not annexed, +due to a Mojibake problem: + +With `encodeBS`, the input `{"file":"ü.txt"}` is encoded as +`"{\"file\":\"\195\188.txt\"}"`. Aeson parses that input to this: + + JSONActionItem {itemCommand = Nothing, itemKey = Nothing, itemFile = Just "\252.txt", itemAdded = Nothing} + +Note that the first two bytes have been +parsed by Aeson as unicode (since JSON is unicode encoded), +yielding character 252 (ü). + +In a unicode locale, this works ok, because the encoding layer is able to +convert that unicode character back to two bytes 195 188 +and finds the file on disk. But in a non-unicode locale, it doesn't know +what to do with the unicode character, and in fact it gets discarded +and so it looks for a file named ".txt". + +So, to make --batch --json input work in non-unicode locales, it would +need, after parsing the json, to re-encode filenames (and perhaps other +data), from utf8 to the filesystem encoding. I have not yet worked out how +to do that. +"""]] diff --git a/doc/bugs/Local_network___40__ssh__41___fails_to_pair__47__sync.mdwn b/doc/bugs/Local_network___40__ssh__41___fails_to_pair__47__sync.mdwn new file mode 100644 index 0000000000..bff665a5e9 --- /dev/null +++ b/doc/bugs/Local_network___40__ssh__41___fails_to_pair__47__sync.mdwn @@ -0,0 +1,177 @@ +### Please describe the problem. +I am trying to set out two computers on the same network to synchronise. + +### What steps will reproduce the problem? +Install Git-Annex. Start the webapp. Try to connect. Enter a secret phrase on both. The Mythbuntu machine shows "Failed to sync with Inspiron 14z" (the laptop). The laptop shows "Pairing in progress" forever. + +The machines can normally connect together passwordlessly through ssh with public key encryption. + +### What version of git-annex are you using? On what operating system? +Ubuntu Raring Version: 3.20121112ubuntu2 from the repos on my laptop. Install version 4.20130627 from the PPA on Mythbuntu Precise (which I use as a home server). + +### Please provide any additional information below. +From the the laptop, where I started the pairing: +[[!format sh """ +# If you can, paste a complete transcript of the problem occurring here. +# If the problem is with the git-annex assistant, paste in .git/annex/daemon.log +$ git-annex webapp + +(process:3084): GLib-CRITICAL **: g_slice_set_config: assertion `sys_page_size == 0' failed + +** (firefox:3084): WARNING **: Failed to find domain member of JSON manifest +Running global cleanup code from study base classes. +(Recording state in git...) + +Launching web browser on file:///tmp/webapp3075.html +(scanning...) + dbus failed; falling back to mtab polling (ClientError {clientErrorMessage = "Call failed: The name org.gtk.Private.GduVolumeMonitor was not provided by any .service files", clientErrorFatal = False}) +(started...) Generating public/private rsa key pair. +Your identification has been saved in /tmp/git-annex-keygen3075.0/key. +Your public key has been saved in /tmp/git-annex-keygen3075.0/key.pub. +The key fingerprint is: +79:00:67:a4:f0:5f:62:26:78:ed:09:97:e4:c4:dd:56 aaron@Inspiron-14z +The key's randomart image is: ++--[ RSA 2048]----+ +| . .o*. . .E | +| + X... o | +| . * X .. | +| . O * | +| S . | +| . | +| | +| | +| | ++-----------------+ +Control socket connect(/home/shared/annex/.git/annex/ssh/mythbuntu@git-annex-mythbuntu-server.local-mythbuntu): Connection refused +Failed to connect to new control master +warning: no common commits + + Remote mythbuntuserver.local_annex does not have git-annex installed; setting remote.mythbuntuserver.local_annex.annex-ignore +Already up-to-date. +Counting objects: 13, done. +Delta compression using up to 4 threads. +Compressing objects: 100% (9/9), done. +Writing objects: 100% (11/11), 1.06 KiB, done. +Total 11 (delta 2), reused 0 (delta 0) +To ssh://mythbuntu@git-annex-mythbuntu-server.local-mythbuntu/~/annex/ + * [new branch] git-annex -> synced/git-annex + * [new branch] master -> synced/master +Already up-to-date! +Merge made by the 'recursive' strategy. +Already up-to-date. +Already up-to-date. +Counting objects: 2, done. +Delta compression using up to 4 threads. +Compressing objects: 100% (2/2), done. +Writing objects: 100% (2/2), 326 bytes, done. +Total 2 (delta 1), reused 0 (delta 0) +To ssh://mythbuntu@git-annex-mythbuntu-server.local-mythbuntu/~/annex/ + 82bf946..dfe0bd1 master -> synced/master +Already up-to-date. +SendMessage (77594660, 0x101f, (nil), (nil)) +SendMessage (0, 0x1203, (nil), 0x7fffc68a1850) +SendMessage (0, 0x1204, (nil), 0x7fffc68a1850) +SendMessage (0, 0x1203, 0x1, 0x7fffc68a1850) +SendMessage (0, 0x1204, 0x1, 0x7fffc68a1850) +SendMessage (0, 0x1203, 0x2, 0x7fffc68a1850) +SendMessage (0, 0x1204, 0x2, 0x7fffc68a1850) +SendMessage (0, 0x1203, 0x3, 0x7fffc68a1850) +SendMessage (0, 0x1204, 0x3, 0x7fffc68a1850) +SendMessage (0, 0x1203, 0x4, 0x7fffc68a1850) +SendMessage (0, 0x1204, 0x4, 0x7fffc68a1850) +SendMessage (77594660, 0x101f, (nil), (nil)) +SendMessage (0, 0x1203, (nil), 0x7fffc68a1810) +SendMessage (0, 0x1204, (nil), 0x7fffc68a1810) +SendMessage (0, 0x1203, 0x1, 0x7fffc68a1810) +SendMessage (0, 0x1204, 0x1, 0x7fffc68a1810) +SendMessage (0, 0x1203, 0x2, 0x7fffc68a1810) +SendMessage (0, 0x1204, 0x2, 0x7fffc68a1810) +SendMessage (0, 0x1203, 0x3, 0x7fffc68a1810) +SendMessage (0, 0x1204, 0x3, 0x7fffc68a1810) +SendMessage (0, 0x1203, 0x4, 0x7fffc68a1810) +SendMessage (0, 0x1204, 0x4, 0x7fffc68a1810) +SendMessage (77594660, 0x101f, (nil), (nil)) +SendMessage (0, 0x1203, (nil), 0x7fffc68a2920) +SendMessage (0, 0x1204, (nil), 0x7fffc68a2920) +SendMessage (0, 0x1203, 0x1, 0x7fffc68a2920) +SendMessage (0, 0x1204, 0x1, 0x7fffc68a2920) +SendMessage (0, 0x1203, 0x2, 0x7fffc68a2920) +SendMessage (0, 0x1204, 0x2, 0x7fffc68a2920) +SendMessage (0, 0x1203, 0x3, 0x7fffc68a2920) +SendMessage (0, 0x1204, 0x3, 0x7fffc68a2920) +SendMessage (0, 0x1203, 0x4, 0x7fffc68a2920) +SendMessage (0, 0x1204, 0x4, 0x7fffc68a2920) +Redirection loop trying to set HTTPS on: + http://www.aol.com/favicon.ico +(falling back to HTTP) +SendMessage (77594652, 0x444, 0x1, 0x3652e00) +SendMessage (77594652, 0x444, 0x1, 0x3647de0) +SendMessage (77594652, 0x444, 0x1, 0x3667a80) +SendMessage (77594652, 0x444, 0x1, 0x3667a80) + +# End of transcript or log. +# End of transcript or log. +"""]] + +On the Mythbuntu machine: +[[!format sh """ +$ git-annex webapp +Launching web browser on file:///tmp/webapp8399.html +(Recording state in git...) +"""]] + +Unless I'm going mad, there doesn't seem to be a daemon.log on my laptop. + +Daemon.log on the Mythbuntu machine: +[[!format sh """ +[2013-07-14 10:38:56 BST] main: starting assistant version 4.20130627 +(scanning...) [2013-07-14 10:38:56 BST] Watcher: Performing startup scan +(started...) [2013-07-14 10:38:57 BST] PairListener: aaron@Inspiron-14z:/home/shared/annex is sending a pair request. +Generating public/private rsa key pair. +Your identification has been saved in /tmp/git-annex-keygen.0/key. +Your public key has been saved in /tmp/git-annex-keygen.0/key.pub. +The key fingerprint is: +bb:8c:66:05:22:8e:fa:e1:10:33:6d:cb:d6:57:e2:47 mythbuntu@mythbuntu-server +The key's randomart image is: ++--[ RSA 2048]----+ +| | +| | +| | +| .. . . | +|+oo. ...E | +|.*.o . +.. | +|o = . o.o | +|.+ . .o+ . | +| .o o. o | ++-----------------+ +[2013-07-14 10:39:13 BST] main: Pairing with aaron@Inspiron-14z:/home/shared/annex in progress +ssh: connect to host Inspiron-14z.local port 22: Connection refused +ssh: connect to host Inspiron-14z.local port 22: Connection refused +fatal: The remote end hung up unexpectedly +[2013-07-14 10:39:17 BST] PairListener: Syncing with Inspiron14z.local__home_shared_annex +ssh: connect to host Inspiron-14z.local port 22: Connection refused +fatal: The remote end hung up unexpectedly +ssh: connect to host Inspiron-14z.local port 22: Connection refused +fatal: The remote end hung up unexpectedly +ssh: connect to host Inspiron-14z.local port 22: Connection refused +fatal: The remote end hung up unexpectedly +Already up-to-date. +Already up-to-date. +ssh: connect to host Inspiron-14z.local port 22: Connection refused +fatal: The remote end hung up unexpectedly +ssh: connect to host Inspiron-14z.local port 22: Connection refused +fatal: The remote end hung up unexpectedly +Updating 82bf946..dfe0bd1 +Fast-forward +[2013-07-14 10:39:56 BST] Pusher: Syncing with Inspiron14z.local__home_shared_annex +ssh: connect to host Inspiron-14z.local port 22: Connection refused +fatal: The remote end hung up unexpectedly +ssh: connect to host Inspiron-14z.local port 22: Connection refused +fatal: The remote end hung up unexpectedly +ssh: connect to host Inspiron-14z.local port 22: Connection refused +fatal: The remote end hung up unexpectedly +ssh: connect to host Inspiron-14z.local port 22: Connection refused +fatal: The remote end hung up unexpectedly +"""]] + +[[!taglink moreinfo]] diff --git a/doc/bugs/Local_network___40__ssh__41___fails_to_pair__47__sync/comment_1_bab9cd5bdcffec3c48b9e8657cd9bbf7._comment b/doc/bugs/Local_network___40__ssh__41___fails_to_pair__47__sync/comment_1_bab9cd5bdcffec3c48b9e8657cd9bbf7._comment new file mode 100644 index 0000000000..272cf26840 --- /dev/null +++ b/doc/bugs/Local_network___40__ssh__41___fails_to_pair__47__sync/comment_1_bab9cd5bdcffec3c48b9e8657cd9bbf7._comment @@ -0,0 +1,14 @@ +[[!comment format=mdwn + username="https://www.google.com/accounts/o8/id?id=AItOawnFjuvfPpi1kf6l54bxfFUm0Aw_Gf_IO0o" + nickname="Aaron" + subject="Some things working" + date="2013-07-14T10:06:13Z" + content=""" +Oh, that's odd... + +The Mythbuntu machine says it fails and the laptop screen still says pairing, but a new window on the laptop says that it synced with the Mythbuntu server. On the Mythbuntu server, it has a blank entry for the laptop, which if you click to edit says \"Internal Server Error\" \"Unknown UUID\". + +When I add things on the laptop, they immediately become shortcuts and an equivalent shortcut appears on the Mythbuntu machine, though updates to the content (in this case a text file) are not sychronised. + +Adding a file on the Mythbuntu machine does not turn into a shortcut and nothing appears on the laptop. +"""]] diff --git a/doc/bugs/Local_network___40__ssh__41___fails_to_pair__47__sync/comment_2_104898dce3c67c082a9f2b36e2f45ff8._comment b/doc/bugs/Local_network___40__ssh__41___fails_to_pair__47__sync/comment_2_104898dce3c67c082a9f2b36e2f45ff8._comment new file mode 100644 index 0000000000..09d7a6d59a --- /dev/null +++ b/doc/bugs/Local_network___40__ssh__41___fails_to_pair__47__sync/comment_2_104898dce3c67c082a9f2b36e2f45ff8._comment @@ -0,0 +1,10 @@ +[[!comment format=mdwn + username="http://joeyh.name/" + ip="4.152.246.110" + subject="comment 2" + date="2013-07-28T00:27:23Z" + content=""" +Remote mythbuntuserver.local_annex does not have git-annex installed; setting remote.mythbuntuserver.local_annex.annex-ignore + +So, how did you install git-annex on mythbuntuserver? What does the `.ssh/authorized_keys` on mythbuntuserver look like? (In particular the key that git-annex adds to it, which should be set to run git-annex-shell.) +"""]] diff --git a/doc/bugs/Location_change_of_remote_DNS_ignored.mdwn b/doc/bugs/Location_change_of_remote_DNS_ignored.mdwn new file mode 100644 index 0000000000..b35f660685 --- /dev/null +++ b/doc/bugs/Location_change_of_remote_DNS_ignored.mdwn @@ -0,0 +1,28 @@ +### Please describe the problem. +git-annex ignores changing of remote location in .git/config + +### What steps will reproduce the problem? +1. Change Hostname of previously working remote so that the existing remote will no longer work and produce "ssh: connect to host port 22: Network is unreachable" errors. +2. Stop all running git-annex processes. +3. Edit DNS name in corresponding remote .git/config. +4. Restart git-annex. +5. Turns out: git-annex still uses the old remote's DNS name. + +### What version of git-annex are you using? On what operating system? +newest one available. +5.20151116-gbe86081 + +### Please provide any additional information below. +See following entry in log, occurring plenty often: +[[!format sh """ +ssh: connect to host some.unreachable.dns.net port 22: Network is unreachable +rsync: connection unexpec + rsync failed -- run git annex again to resume file transfer +tedly closed (0 bytes received so far) [Receiver] +rsync error: unexplained error (code 255) at io.c(226) [Receiver=3.1.1] +"""]] + +### Have you had any luck using git-annex before? (Sometimes we get tired of reading bug reports all day and a lil' positive end note does wonders) +Trust me I love git-annex. Had dreams of something like git-annex for almost 10 years. I instantly got stuck on git-annex when I have read the first few sentences about it. Since then, things in my life have changed in a quite serious manner. + +THANKS for your help! diff --git a/doc/bugs/Location_change_of_remote_DNS_ignored/comment_1_449bfb3a594f99cf5e71a6e18d6d3dbc._comment b/doc/bugs/Location_change_of_remote_DNS_ignored/comment_1_449bfb3a594f99cf5e71a6e18d6d3dbc._comment new file mode 100644 index 0000000000..4f81c4348e --- /dev/null +++ b/doc/bugs/Location_change_of_remote_DNS_ignored/comment_1_449bfb3a594f99cf5e71a6e18d6d3dbc._comment @@ -0,0 +1,8 @@ +[[!comment format=mdwn + username="torpidus" + subject="comment 1" + date="2015-12-06T20:29:25Z" + content=""" +**self-edit**: +Changed location of .git/config once again as described but instead of re-starting assistant, I ran \"git-annex sync\" which ended with no errors. After that, assistant started correctly and also had the corrected remote location. From my point of view, this thread has been resolved - even though I wonder what changed the value back to the old one in the first place. +"""]] diff --git a/doc/bugs/Location_change_of_remote_DNS_ignored/comment_2_b097dec73d4965f62c61003ed27e4209._comment b/doc/bugs/Location_change_of_remote_DNS_ignored/comment_2_b097dec73d4965f62c61003ed27e4209._comment new file mode 100644 index 0000000000..d099ea8046 --- /dev/null +++ b/doc/bugs/Location_change_of_remote_DNS_ignored/comment_2_b097dec73d4965f62c61003ed27e4209._comment @@ -0,0 +1,9 @@ +[[!comment format=mdwn + username="joey" + subject="""comment 2""" + date="2015-12-11T15:24:44Z" + content=""" +It seems likely that you didn't stop the assistant. There's no +caching of urls to git remotes; git-annex just uses whatever's there in +.git/config. +"""]] diff --git a/doc/bugs/Low_disk_space_corrupts_state.mdwn b/doc/bugs/Low_disk_space_corrupts_state.mdwn new file mode 100644 index 0000000000..ecdfc48a2e --- /dev/null +++ b/doc/bugs/Low_disk_space_corrupts_state.mdwn @@ -0,0 +1,61 @@ +### Please describe the problem. + +When there are low disk space left, changes in the annex repo are only semi updated, leaving the dir and git annex state out of sync, leaving error messages with "invalid objects" and "fatal: git-write-tree: error building trees". + +I ignored these errors, and kept on trying to copy over all the files to a remote disk, since I wanted a backup, which resultet in symlinks pointing to files which aren't there. + +Maybe git-annex should stop if it sees that it's not enough disk space to perform a certain operation? E.g. cache space needed for syncing. + +### What steps will reproduce the problem? + +I'm not certain about what command that created the issue, but i ran various commands: + +- `git annex sync EXTERNALREPO` +- `git annex copy --to EXTERNALREPO` +- `git annex unlock Videos/` +- `git annex lock Videos/` + +My disk had 280MB left, and the repo were at a few GBs. Somewhere in between these commands, git-annex started producing a lot of error messages for various files. + +### What version of git-annex are you using? On what operating system? + +git-annex version: 5.20140717 (Fedora 23) + +### Please provide any additional information below. + +Unfortunately, I don't have the full transcript of my badly behaved commands. Instead I add some of the error messages: + +[[!format sh """ +# If you can, paste a complete transcript of the problem occurring here. +# If the problem is with the git-annex assistant, paste in .git/annex/daemon.log + + + $ git annex lock Videos/ + error: invalid object 100644 56715f46d9256bdcef0cd387364818e597dc9f41 for '003/a56/SHA256-s23430--26dc3b33a5101e4ead217241a371b17f15e7a2f37bbcaedd0d35a0a1aa4eb9b0.log.cnk' + fatal: git-write-tree: error building trees + git-annex: failed to read sha from git write-tree + + $ git annex sync + error: refs/remotes/remote_annex/synced/master does not point to a valid object! + error: refs/remotes/home/synced/master does not point to a valid object! + error: refs/remotes/work_annex/synced/master does not point to a valid object! + error: refs/remotes/home_annex/synced/master does not point to a valid object! + error: refs/remotes/home/synced/master does not point to a valid object! + error: refs/remotes/work_annex/synced/master does not point to a valid object! + error: Could not read bf3d6640fa32460032926ae6... + fatal: revision walk setup failed + error: Could not read 8cdec1808723971eaf30e32... + fatal: revision walk setup failed + (merging work/git-annex into git-annex...) + fatal: unable to read tree 21e0681190de239f41d... + (Recording state in git...) + error: invalid object 100644 56715f36d9256bdeff... for '003/a56/SHA256-s23430--2dc3b411e5ead...log.cnk' + fatal: git-write-tree: error building trees + git-annex: failed to read sha from git write-tree + +# End of transcript or log. +"""]] + +### Have you had any luck using git-annex before? (Sometimes we get tired of reading bug reports all day and a lil' positive end note does wonders) + +It worked as a charm until my disk got full! (Maybe it is better to split it up in various, smaller repos, and sync them individually?) diff --git a/doc/bugs/Low_disk_space_corrupts_state/comment_1_d8235498e2206c3b5284455e5c4182b9._comment b/doc/bugs/Low_disk_space_corrupts_state/comment_1_d8235498e2206c3b5284455e5c4182b9._comment new file mode 100644 index 0000000000..334a0fd94b --- /dev/null +++ b/doc/bugs/Low_disk_space_corrupts_state/comment_1_d8235498e2206c3b5284455e5c4182b9._comment @@ -0,0 +1,20 @@ +[[!comment format=mdwn + username="joakim.hovlandsvag@ad788ffa13d1ccbf03f2c485653900f8baa33950" + nickname="joakim.hovlandsvag" + subject="Repairing ignores disk space issues too" + date="2015-12-13T09:13:24Z" + content=""" +It seems that `git annex repair` also ignores such issues: + + $ git annex repair + Running git fsck ... + Initialized empty Git repository in /tmp/tmprepo.0/.git/ + Trying to recover missing objects from remote home. + Unpacking all pack files. + error: file write error (No space left on device) + fatal: unable to write sha1 file + Successfully recovered repository! + You should run \"git fsck\" to make sure, but it looks like everything was recovered ok. + ok + +"""]] diff --git a/doc/bugs/Low_disk_space_corrupts_state/comment_2_0e47c63a48e10ae760754df0ce165388._comment b/doc/bugs/Low_disk_space_corrupts_state/comment_2_0e47c63a48e10ae760754df0ce165388._comment new file mode 100644 index 0000000000..3ee40ef92e --- /dev/null +++ b/doc/bugs/Low_disk_space_corrupts_state/comment_2_0e47c63a48e10ae760754df0ce165388._comment @@ -0,0 +1,17 @@ +[[!comment format=mdwn + username="joey" + subject="""comment 2""" + date="2016-04-13T19:23:29Z" + content=""" +I suggest that you first free up some disk space, and only then try to run +`git annex repair`. + +I checked, and git-annex repair actually has a legitimate reason for +ignoring the error to unpack a pack file. The pack file could be corrupt. +It can't tell why git fails to unpack a pack file, so can't detect that +it's due to low disk space. + +I don't know if git-repair managed to get your repository back to a working +state (although with git history lost) or not. It seems like it thinks +it managed to repair the repo enough for it to work, is that not the case? +"""]] diff --git a/doc/bugs/MacOSX__58___archive_folders_not_working_as_expected.mdwn b/doc/bugs/MacOSX__58___archive_folders_not_working_as_expected.mdwn new file mode 100644 index 0000000000..4b45cafd2d --- /dev/null +++ b/doc/bugs/MacOSX__58___archive_folders_not_working_as_expected.mdwn @@ -0,0 +1,123 @@ +### Please describe the problem. + +I've added a backup USB Drive and a small archive USB drive and created a archive folder in the root of my ~/annex folder. + +If I move a file into the ~/annex folder and then move the same file into the ~/annex/archive folder the file gets uploaded to the small archive USB drive, but not dropped from my ~/annex/archive folder. + +[[!format sh """ +[2015-08-14 19:03:47 CEST] Committer: Adding git-annex..iving.ogv +ok +(recording state in git...) +(recording state in git...) +(recording state in git...) +add /Users/markus/annex/git-annex-assistant-archiving.ogv [2015-08-14 19:03:47 CEST] Committer: Committing changes to git +[2015-08-14 19:03:47 CEST] Pusher: Syncing with annexbackup, annexarchive + +git-annex-assistant-archiving.ogv +To /Volumes/annex-archive/annex-archive0 + 6ef9226..6c87a7d git-annex -> synced/git-annex + b8855cc..bc20ee2 annex/direct/master -> synced/master +To /Volumes/annex-backup/annex-backup + 6ef9226..6c87a7d git-annex -> synced/git-annex + b8855cc..bc20ee2 annex/direct/master -> synced/master + 15428954 100% 20.71MB/s 0:00:00 (xfer#1, to-check=0/1) + +sent 15430948 bytes received 42 bytes 10287326.67 bytes/sec +total size is 15428954 speedup is 1.00 +[2015-08-14 19:03:48 CEST] Transferrer: Uploaded git-annex..iving.ogv +[2015-08-14 19:03:49 CEST] Pusher: Syncing with annexbackup, annexarchive +To /Volumes/annex-backup/annex-backup + 6c87a7d..d24edd5 git-annex -> synced/git-annex +To /Volumes/annex-archive/annex-archive + 6c87a7d..d24edd5 git-annex -> synced/git-annex +[2015-08-14 19:04:23 CEST] Committer: Adding git-annex..iving.ogv +ok +(recording state in git...) +(recording state in git...) +(recording state in git...) +add /Users/markus/annex/archive/git-annex-assistant-archiving.ogv [2015-08-14 19:04:23 CEST] Committer: Committing changes to git +[2015-08-14 19:04:23 CEST] Pusher: Syncing with annexbackup, annexarchive + +git-annex-assistant-archiving.ogv +To /Volumes/annex-backup/annex-backup:00 + d24edd5..507be0c git-annex -> synced/git-annex + bc20ee2..8fea81a annex/direct/master -> synced/master +To /Volumes/annex-archive/annex-archive + d24edd5..507be0c git-annex -> synced/git-annex + bc20ee2..8fea81a annex/direct/master -> synced/master + 15428954 100% 30.72MB/s 0:00:00 (xfer#1, to-check=0/1) + +sent 15430948 bytes received 42 bytes 30861980.00 bytes/sec +total size is 15428954 speedup is 1.00 +[2015-08-14 19:04:23 CEST] Transferrer: Uploaded git-annex..iving.ogv +[2015-08-14 19:04:25 CEST] Pusher: Syncing with annexbackup, annexarchive +To /Volumes/annex-archive/annex-archive + 507be0c..ba58f77 git-annex -> synced/git-annex +To /Volumes/annex-backup/annex-backup + 507be0c..ba58f77 git-annex -> synced/git-annex +"""]] + +If I directly move a file e.g. from ~/Downloads to ~/annex/archive the file gets uploaded to the small archive USB drive and then dropped from my drive (as expected). + +[[!format sh """ +[2015-08-14 19:04:25 CEST] Pusher: Syncing with annexbackup, annexarchive +To /Volumes/annex-archive/annex-archive + 507be0c..ba58f77 git-annex -> synced/git-annex +To /Volumes/annex-backup/annex-backup + 507be0c..ba58f77 git-annex -> synced/git-annex +[2015-08-14 19:09:44 CEST] Committer: Adding Oracle_VM..x-extpack +ok +(recording state in git...) +(recording state in git...) +(recording state in git...) +add /Users/markus/annex/archive/Oracle_VM_VirtualBox_Extension_Pack-5.0.0-101573.vbox-extpack [2015-08-14 19:09:44 CEST] Committer: Committing changes to git +[2015-08-14 19:09:44 CEST] Pusher: Syncing with annexbackup, annexarchive + +Oracle_VM_VirtualBox_Extension_Pack-5.0.0-101573.vbox-extpack +To /Volumes/annex-backup/annex-backup:00 + ba58f77..73eed9e git-annex -> synced/git-annex + 8fea81a..68526f7 annex/direct/master -> synced/master +To /Volumes/annex-archive/annex-archive + ba58f77..73eed9e git-annex -> synced/git-annex + 8fea81a..68526f7 annex/direct/master -> synced/master + 18028050 100% 25.65MB/s 0:00:00 (xfer#1, to-check=0/1) + +sent 18030392 bytes received 42 bytes 12020289.33 bytes/sec +total size is 18028050 speedup is 1.00 +[2015-08-14 19:09:45 CEST] Transferrer: Uploaded Oracle_VM..x-extpack +ok +(recording state in git...) +(recording state in git...) +drop /Users/markus/annex/archive/Oracle_VM_VirtualBox_Extension_Pack-5.0.0-101573.vbox-extpack [2015-08-14 19:09:47 CEST] Pusher: Syncing with annexbackup, annexarchive +To /Volumes/annex-backup/annex-backup + 73eed9e..93dc4a3 git-annex -> synced/git-annex +To /Volumes/annex-archive/annex-archive + 73eed9e..93dc4a3 git-annex -> synced/git-annex +"""]] + +But if I move the file-link from ~/annex/archive to ~/annex the file stays a link and don't get transferred back to my drive. + +[[!format sh """ +[2015-08-14 19:12:39 CEST] Committer: Committing changes to git +[2015-08-14 19:12:39 CEST] Pusher: Syncing with annexbackup, annexarchive +To /Volumes/annex-backup/annex-backup + 68526f7..082e0d8 annex/direct/master -> synced/master +To /Volumes/annex-archive/annex-archive + 68526f7..082e0d8 annex/direct/master -> synced/master +"""]] + + +### What version of git-annex are you using? On what operating system? + +Im using MacOSX 10.11 Beta (15A243d) with the most resend MacOSX App. + +[[!format sh """ +$ /Applications/git-annex.app/Contents/MacOS/git-annex version +git-annex version: 5.20150812-ga6f7b36 +build flags: Assistant Webapp Webapp-secure Pairing Testsuite S3 WebDAV FsEvents XMPP DNS Feeds Quvi TDFA TorrentParser Database +key/value backends: SHA256E SHA256 SHA512E SHA512 SHA224E SHA224 SHA384E SHA384 SKEIN256E SKEIN256 SKEIN512E SKEIN512 SHA1E SHA1 MD5E MD5 WORM URL +remote types: git gcrypt S3 bup directory rsync web bittorrent webdav tahoe glacier ddar hook external +local repository version: 5 +supported repository version: 5 +upgrade supported from repository versions: 0 1 2 4 +"""]] diff --git a/doc/bugs/MacOSX__58___archive_folders_not_working_as_expected/comment_1_c8ce2a1978fc1bfa5f31cff9f48690f6._comment b/doc/bugs/MacOSX__58___archive_folders_not_working_as_expected/comment_1_c8ce2a1978fc1bfa5f31cff9f48690f6._comment new file mode 100644 index 0000000000..d1380f1c52 --- /dev/null +++ b/doc/bugs/MacOSX__58___archive_folders_not_working_as_expected/comment_1_c8ce2a1978fc1bfa5f31cff9f48690f6._comment @@ -0,0 +1,57 @@ +[[!comment format=mdwn + username="mhubig" + subject="comment 1" + date="2015-08-17T11:42:18Z" + content=""" +Here's some more infos from running git-annex with debug: + +Moving a 'symlinked' file from `~/annex/archive` to `~/annex`. I expected that the file get's automagically pulled from the archive drive and the symlink is replaced with the real file. +but what happens is this: + +[[!format sh \"\"\" +[2015-08-17 13:29:48 CEST] Watcher: file deleted /Users/markus/annex/archive/Crisi\'s Trail und Stubenfelsen Jam.gpx +[2015-08-17 13:29:48 CEST] Committer: committing 1 changes +[2015-08-17 13:29:48 CEST] Committer: Committing changes to git +(recording state in git...) +[2015-08-17 13:29:48 CEST] feed: git [\"--git-dir=.git\",\"--work-tree=.\",\"--literal-pathspecs\",\"-c\",\"core.bare=false\",\"update-index\",\"-z\",\"--index-info\"] +[2015-08-17 13:29:48 CEST] read: git [\"--git-dir=.git\",\"--work-tree=.\",\"--literal-pathspecs\",\"-c\",\"core.bare=false\",\"symbolic-ref\",\"-q\",\"HEAD\"] +[2015-08-17 13:29:48 CEST] read: git [\"--git-dir=.git\",\"--work-tree=.\",\"--literal-pathspecs\",\"-c\",\"core.bare=false\",\"show-ref\",\"--hash\",\"refs/heads/annex/direct/master\"] +[2015-08-17 13:29:48 CEST] read: git [\"--git-dir=.git\",\"--work-tree=.\",\"--literal-pathspecs\",\"-c\",\"core.bare=false\",\"write-tree\"] +[2015-08-17 13:29:48 CEST] read: git [\"--git-dir=.git\",\"--work-tree=.\",\"--literal-pathspecs\",\"-c\",\"core.bare=false\",\"rev-parse\",\"df537468c185961a9785d9288d00b0bc5f628a28:\"] +[2015-08-17 13:29:48 CEST] chat: git [\"--git-dir=.git\",\"--work-tree=.\",\"--literal-pathspecs\",\"-c\",\"core.bare=false\",\"commit-tree\",\"be3e4a4a4477d6a1aad9fcbd333b43d84295e9b6\",\"--no-gpg-sign\",\"-p\",\"df537468c185961a9785d9288d00b0bc5f628a28\"] +[2015-08-17 13:29:48 CEST] call: git [\"--git-dir=.git\",\"--work-tree=.\",\"--literal-pathspecs\",\"-c\",\"core.bare=false\",\"update-ref\",\"refs/heads/annex/direct/master\",\"3436556450500cf342154e9b03bfd1825402f5eb\"] +[2015-08-17 13:29:48 CEST] read: git [\"--git-dir=.git\",\"--work-tree=.\",\"--literal-pathspecs\",\"-c\",\"core.bare=false\",\"symbolic-ref\",\"-q\",\"HEAD\"] +[2015-08-17 13:29:48 CEST] read: git [\"--git-dir=.git\",\"--work-tree=.\",\"--literal-pathspecs\",\"-c\",\"core.bare=false\",\"show-ref\",\"refs/heads/annex/direct/master\"] +[2015-08-17 13:29:48 CEST] call: git [\"--git-dir=.git\",\"--work-tree=.\",\"--literal-pathspecs\",\"-c\",\"core.bare=false\",\"branch\",\"-f\",\"synced/master\"] +[2015-08-17 13:29:48 CEST] call: git [\"--git-dir=.git\",\"--work-tree=.\",\"--literal-pathspecs\",\"-c\",\"core.bare=false\",\"branch\",\"-f\",\"master\"] +[2015-08-17 13:29:48 CEST] Pusher: Syncing with annexbackup, annexarchive +[2015-08-17 13:29:48 CEST] read: git [\"--git-dir=.git\",\"--work-tree=.\",\"--literal-pathspecs\",\"-c\",\"core.bare=false\",\"symbolic-ref\",\"-q\",\"HEAD\"] +[2015-08-17 13:29:48 CEST] read: git [\"--git-dir=.git\",\"--work-tree=.\",\"--literal-pathspecs\",\"-c\",\"core.bare=false\",\"show-ref\",\"refs/heads/annex/direct/master\"] +[2015-08-17 13:29:48 CEST] Pusher: pushing to [Remote { name =\"annexbackup\" },Remote { name =\"annexarchive\" }] +[2015-08-17 13:29:48 CEST] call: git [\"--git-dir=.git\",\"--work-tree=.\",\"--literal-pathspecs\",\"-c\",\"core.bare=false\",\"push\",\"annexbackup\",\"+git-annex:synced/git-annex\",\"annex/direct/master:synced/master\"] +[2015-08-17 13:29:48 CEST] call: git [\"--git-dir=.git\",\"--work-tree=.\",\"--literal-pathspecs\",\"-c\",\"core.bare=false\",\"push\",\"annexarchive\",\"+git-annex:synced/git-annex\",\"annex/direct/master:synced/master\"] +To /Volumes/annex-archive/annex-archive + df53746..3436556 annex/direct/master -> synced/master +[2015-08-17 13:29:48 CEST] read: git [\"--git-dir=.git\",\"--work-tree=.\",\"--literal-pathspecs\",\"-c\",\"core.bare=false\",\"push\",\"annexarchive\",\"git-annex\",\"master\"] +To /Volumes/annex-backup/annex-backup + 464ebc5..2413398 git-annex -> synced/git-annex + df53746..3436556 annex/direct/master -> synced/master +[2015-08-17 13:29:48 CEST] read: git [\"--git-dir=.git\",\"--work-tree=.\",\"--literal-pathspecs\",\"-c\",\"core.bare=false\",\"push\",\"annexbackup\",\"git-annex\",\"master\"] +[2015-08-17 13:29:49 CEST] read: git [\"--git-dir=.git\",\"--work-tree=.\",\"--literal-pathspecs\",\"-c\",\"core.bare=false\",\"symbolic-ref\",\"-q\",\"HEAD\"] +[2015-08-17 13:29:49 CEST] read: git [\"--git-dir=.git\",\"--work-tree=.\",\"--literal-pathspecs\",\"-c\",\"core.bare=false\",\"show-ref\",\"refs/heads/annex/direct/master\"] +[2015-08-17 13:29:49 CEST] read: git [\"--git-dir=.git\",\"--work-tree=.\",\"--literal-pathspecs\",\"-c\",\"core.bare=false\",\"log\",\"refs/heads/annex/direct/master..refs/heads/synced/master\",\"-n1\",\"--pretty=%H\"] +[2015-08-17 13:29:49 CEST] read: git [\"--git-dir=.git\",\"--work-tree=.\",\"--literal-pathspecs\",\"-c\",\"core.bare=false\",\"symbolic-ref\",\"-q\",\"HEAD\"] +[2015-08-17 13:29:49 CEST] read: git [\"--git-dir=.git\",\"--work-tree=.\",\"--literal-pathspecs\",\"-c\",\"core.bare=false\",\"show-ref\",\"refs/heads/annex/direct/master\"] +[2015-08-17 13:29:49 CEST] read: git [\"--git-dir=.git\",\"--work-tree=.\",\"--literal-pathspecs\",\"-c\",\"core.bare=false\",\"log\",\"refs/heads/annex/direct/master..refs/remotes/annexarchive/synced/master\",\"-n1\",\"--pretty=%H\"] +[2015-08-17 13:29:49 CEST] read: git [\"--git-dir=.git\",\"--work-tree=.\",\"--literal-pathspecs\",\"-c\",\"core.bare=false\",\"show-ref\",\"git-annex\"] +[2015-08-17 13:29:49 CEST] read: git [\"--git-dir=.git\",\"--work-tree=.\",\"--literal-pathspecs\",\"-c\",\"core.bare=false\",\"show-ref\",\"--hash\",\"refs/heads/git-annex\"] +[2015-08-17 13:29:49 CEST] read: git [\"--git-dir=.git\",\"--work-tree=.\",\"--literal-pathspecs\",\"-c\",\"core.bare=false\",\"log\",\"refs/heads/git-annex..24133980364c5ae86e939c214deea1b6ab63135c\",\"-n1\",\"--pretty=%H\"] +[2015-08-17 13:29:49 CEST] read: git [\"--git-dir=.git\",\"--work-tree=.\",\"--literal-pathspecs\",\"-c\",\"core.bare=false\",\"log\",\"refs/heads/git-annex..d42c35efe8b1333930587a41aec8744aa0e90ae3\",\"-n1\",\"--pretty=%H\"] +[2015-08-17 13:29:49 CEST] read: git [\"--git-dir=.git\",\"--work-tree=.\",\"--literal-pathspecs\",\"-c\",\"core.bare=false\",\"log\",\"refs/heads/git-annex..796db681152a16121be271668d5f79c845e24b8c\",\"-n1\",\"--pretty=%H\"] +[2015-08-17 13:29:49 CEST] read: git [\"--git-dir=.git\",\"--work-tree=.\",\"--literal-pathspecs\",\"-c\",\"core.bare=false\",\"ls-tree\",\"--full-tree\",\"-z\",\"--\",\"refs/heads/git-annex\",\"uuid.log\",\"remote.log\",\"trust.log\",\"group.log\",\"numcopies.log\",\"schedule.log\",\"preferred-content.log\",\"required-content.log\",\"group-preferred-content.log\"] +[2015-08-17 13:29:49 CEST] read: git [\"--git-dir=.git\",\"--work-tree=.\",\"--literal-pathspecs\",\"-c\",\"core.bare=false\",\"symbolic-ref\",\"-q\",\"HEAD\"] +[2015-08-17 13:29:49 CEST] read: git [\"--git-dir=.git\",\"--work-tree=.\",\"--literal-pathspecs\",\"-c\",\"core.bare=false\",\"show-ref\",\"refs/heads/annex/direct/master\"] +[2015-08-17 13:29:49 CEST] read: git [\"--git-dir=.git\",\"--work-tree=.\",\"--literal-pathspecs\",\"-c\",\"core.bare=false\",\"log\",\"refs/heads/annex/direct/master..refs/remotes/annexbackup/synced/master\",\"-n1\",\"--pretty=%H\"] +\"\"\"]] + +"""]] diff --git a/doc/bugs/MacOSX__58___archive_folders_not_working_as_expected/comment_2_cf9a679c7e6ec5cae4109e748b20b8f9._comment b/doc/bugs/MacOSX__58___archive_folders_not_working_as_expected/comment_2_cf9a679c7e6ec5cae4109e748b20b8f9._comment new file mode 100644 index 0000000000..8398805bed --- /dev/null +++ b/doc/bugs/MacOSX__58___archive_folders_not_working_as_expected/comment_2_cf9a679c7e6ec5cae4109e748b20b8f9._comment @@ -0,0 +1,14 @@ +[[!comment format=mdwn + username="joey" + subject="""comment 2""" + date="2015-09-21T19:29:26Z" + content=""" +If this is OSX specific, which it seems to be (I've tested archive folders +working as expected plenty on Linux), it might have to do with the +assistant not seeing an event for the file being added when it's renamed +to/from the archive folder. In the --debug log, it looks like the file +got deleted but never added back. + +If this is the case, `git annex status` should show the file as not being +checked into git. +"""]] diff --git a/doc/bugs/MacOSX__58___archive_folders_not_working_as_expected/comment_3_4026607087a9da2759f841e59aadfcdc._comment b/doc/bugs/MacOSX__58___archive_folders_not_working_as_expected/comment_3_4026607087a9da2759f841e59aadfcdc._comment new file mode 100644 index 0000000000..ae32c97888 --- /dev/null +++ b/doc/bugs/MacOSX__58___archive_folders_not_working_as_expected/comment_3_4026607087a9da2759f841e59aadfcdc._comment @@ -0,0 +1,9 @@ +[[!comment format=mdwn + username="nanotech" + subject="comment 3" + date="2016-03-17T08:39:36Z" + content=""" +I think this is the same bug as [Moved files are not picked up by the assistant on OS X](https://git-annex.branchable.com/bugs/Moved_files_are_not_picked_up_by_the_assistant_on_OS_X/). To reproduce, run the first transcript there but drop the file before moving it. + +FSEvents fails to produce add events for dangling symlinks (), although it does produce removal events. Recompiling and using Kqueue with git-annex instead seems to detect dangling symlinks and add them correctly. +"""]] diff --git a/doc/bugs/MacOSX__58___archive_folders_not_working_as_expected/comment_4_45c1e00e64f916ef8a5f9200144861eb._comment b/doc/bugs/MacOSX__58___archive_folders_not_working_as_expected/comment_4_45c1e00e64f916ef8a5f9200144861eb._comment new file mode 100644 index 0000000000..4d33e50579 --- /dev/null +++ b/doc/bugs/MacOSX__58___archive_folders_not_working_as_expected/comment_4_45c1e00e64f916ef8a5f9200144861eb._comment @@ -0,0 +1,37 @@ +[[!comment format=mdwn + username="olaf" + avatar="http://cdn.libravatar.org/avatar/4ae498d3d6ee558d6b65caa658f72572" + subject="comment 4" + date="2017-06-22T04:50:13Z" + content=""" +I can confirm this is still a problem. + +## OS: +macOS Sierra, version 10.12.5 (16F73) + +## git-annex versions: + +### OSX DMG install +``` +Version: 6.20170611-gb493ac8 +Build flags: Assistant Webapp Pairing Testsuite S3(multipartupload)(storageclasses) WebDAV FsEvents ConcurrentOutput TorrentParser MagicMime Feeds Quvi Dependency versions: aws-0.16 bloomfilter-2.0.1.0 cryptonite-0.21 DAV-1.3.1 feed-0.3.12.0 ghc-8.0.2 http-client-0.5.6.1 persistent-sqlite-2.6.2 torrent-10000.1.1 uuid-1.3.13 yesod-1.4.5 +``` + +### Homebrew install +``` +git-annex version: 6.20170521 +build flags: Assistant Webapp Pairing Testsuite S3(multipartupload)(storageclasses) WebDAV FsEvents ConcurrentOutput TorrentParser MagicMime Feeds Quvi +dependency versions: aws-0.16 bloomfilter-2.0.1.0 cryptonite-0.23 DAV-1.3.1 feed-0.3.12.0 ghc-8.0.2 http-client-0.5.7.0 persistent-sqlite-2.6.2 torrent-10000.1.1 uuid-1.3.13 yesod-1.4.5 +key/value backends: SHA256E SHA256 SHA512E SHA512 SHA224E SHA224 SHA384E SHA384 SHA3_256E SHA3_256 SHA3_512E SHA3_512 SHA3_224E SHA3_224 SHA3_384E SHA3_384 SKEIN256E SKEIN256 SKEIN512E SKEIN512 SHA1E SHA1 MD5E MD5 WORM URL +remote types: git gcrypt p2p S3 bup directory rsync web bittorrent webdav tahoe glacier ddar hook external +local repository version: 5 +supported repository versions: 3 5 6 +upgrade supported from repository versions: 0 1 2 3 4 5 +operating system: darwin x86_64 +``` + + +## Question + +@nanotech - can you elaborate on your recompile with Kqueue comment above? +"""]] diff --git a/doc/bugs/MacOSX__58___archive_folders_not_working_as_expected/comment_5_9501e259e7d813545ff9cfdac22748f6._comment b/doc/bugs/MacOSX__58___archive_folders_not_working_as_expected/comment_5_9501e259e7d813545ff9cfdac22748f6._comment new file mode 100644 index 0000000000..b0c11e7625 --- /dev/null +++ b/doc/bugs/MacOSX__58___archive_folders_not_working_as_expected/comment_5_9501e259e7d813545ff9cfdac22748f6._comment @@ -0,0 +1,37 @@ +[[!comment format=mdwn + username="olaf" + avatar="http://cdn.libravatar.org/avatar/4ae498d3d6ee558d6b65caa658f72572" + subject="Still a problem" + date="2017-11-23T04:34:17Z" + content=""" +Not wanting to repeat myself, but this is still an issue. + +**ISSUE** Moving files to the `archive` folder on MacOS does not result in them being dropped. + +## To reproduce + +- Created two linked repos with the `webapp`, one is `client` the other is `full archive` type. +- Using MacOS' GUI drag some files into the `client` directory. + - All good. They get added and copied over to the `full archive` (looking at the `webapp` and `git annex status`) +- Drag one of the files into the `archive` folder the `webapp`/`assistant` helpfully created for me. + - File move is noticed and `git m[o]v[ed]` (looking at the `webapp` and `git annex status`) + - However, file is not dropped from `client` + + +## Version + + git-annex version: 6.20171109 + build flags: Assistant Webapp Pairing Testsuite S3(multipartupload)(storageclasses) WebDAV FsEvents TorrentParser MagicMime Feeds Quvi + dependency versions: aws-0.17.1 bloomfilter-2.0.1.0 cryptonite-0.24 DAV-1.3.1 feed-1.0.0.0 ghc-8.2.1 http-client-0.5.7.0 persistent-sqlite-2.6.3 torrent-10000.1.1 uuid-1.3.13 yesod-1.4.5 + key/value backends: SHA256E SHA256 SHA512E SHA512 SHA224E SHA224 SHA384E SHA384 SHA3_256E SHA3_256 SHA3_512E SHA3_512 SHA3_224E SHA3_224 SHA3_384E SHA3_384 SKEIN256E SKEIN256 SKEIN512E SKEIN512 SHA1E SHA1 MD5E MD5 WORM URL + remote types: git gcrypt p2p S3 bup directory rsync web bittorrent webdav tahoe glacier ddar hook external + local repository version: 6 + supported repository versions: 3 5 6 + upgrade supported from repository versions: 0 1 2 3 4 5 + operating system: darwin x86_64 + + +## I want to help! + +What can I do to help debug this Joey? +"""]] diff --git a/doc/bugs/Massive_git_add_produces_sqlite_crashes.mdwn b/doc/bugs/Massive_git_add_produces_sqlite_crashes.mdwn new file mode 100644 index 0000000000..3e3d749a19 --- /dev/null +++ b/doc/bugs/Massive_git_add_produces_sqlite_crashes.mdwn @@ -0,0 +1,62 @@ +### Please describe the problem. + +When adding plenty of files to my git annex repository, I encounter recurring sqlite errors. + +### What steps will reproduce the problem? +Create a git annex repo, add thousands of annexed binary files, and add thousands of small files tracked only with git. + + +### What version of git-annex are you using? On what operating system? + + >git annex version + git-annex version: 6.20171003 + build flags: Assistant Webapp Pairing Testsuite S3(multipartupload)(storageclasses) WebDAV Inotify DBus DesktopNotify ConcurrentOutput TorrentParser MagicMime Feeds Quvi + dependency versions: aws-0.13.0 bloomfilter-2.0.1.0 cryptonite-0.10 DAV-1.2 feed-0.3.10.4 ghc-7.10.3 http-client-0.4.26.2 persistent-sqlite-2.2 torrent-10000.0.0 uuid-1.3.11 yesod-1.4.2 + key/value backends: SHA256E SHA256 SHA512E SHA512 SHA224E SHA224 SHA384E SHA384 SHA3_256E SHA3_256 SHA3_512E SHA3_512 SHA3_224E SHA3_224 SHA3_384E SHA3_384 SKEIN256E SKEIN256 SKEIN512E SKEIN512 SHA1E SHA1 MD5E MD5 WORM URL + remote types: git gcrypt p2p S3 bup directory rsync web bittorrent webdav tahoe glacier ddar hook external + local repository version: 6 + supported repository versions: 3 5 6 + upgrade supported from repository versions: 0 1 2 3 4 5 + operating system: linux x86_64 + + >lsb_release -d + Description: Ubuntu 16.04.3 LTS + + >uname -r + 4.4.0-43-Microsoft + #(I am using Bash on Windows.) + +### Please provide any additional information below. + + + # If you can, paste a complete transcript of the problem occurring here. + >git ls-files --others | grep txt | wc -l + 1953 + >git add $(git ls-files --others | grep txt) + sqlite worker thread crashed: SQLite3 returned ErrorIO while attempting to perform prepare "SELECT null from content limit 1": disk I/O error + git-annex: sqlite query crashed + error: external filter git-annex smudge --clean %f failed 1 + error: external filter git-annex smudge --clean %f failed + # [...] plenty of errors follow + >git ls-files --others | grep txt | wc -l + 1953 + # End of transcript or log. + + +Triying to solve this problem, I found a part of answer in the form of a similar problem encountered here : + +Deleting git annex databases and running git annex fsck didnt do the trick: + + rm -rf .git/annex/keys/db .git/annex/keys/db-wal + git annex fsck --incremental -J4 + git add $(git ls-files --others | grep txt) + # Again, plenty of sqlite errors :() + +It seems like a big overhead to add files tracked only by git in git annex repo. I know there are hooks/filters that catch and recover annexed files after modification but is it possible to disable these git annex hooks/filters when adding files that shouldn't be annexed ? + +### Have you had any luck using git-annex before? (Sometimes we get tired of reading bug reports all day and a lil' positive end note does wonders) +Oh yeah, I am still discovering this powerfull git annex tool. + +In fact, collegues and I are forming a group during the process to exchange about different use cases, encountered problems and help each other. + +[[!meta title="sqlite crash on Windows running linux binary via WSL"]] diff --git a/doc/bugs/Massive_git_add_produces_sqlite_crashes/comment_1_6f4179fe22d85b4a51e4aad60c4ff00d._comment b/doc/bugs/Massive_git_add_produces_sqlite_crashes/comment_1_6f4179fe22d85b4a51e4aad60c4ff00d._comment new file mode 100644 index 0000000000..e22ea1d93e --- /dev/null +++ b/doc/bugs/Massive_git_add_produces_sqlite_crashes/comment_1_6f4179fe22d85b4a51e4aad60c4ff00d._comment @@ -0,0 +1,27 @@ +[[!comment format=mdwn + username="joey" + subject="""comment 1""" + date="2017-11-07T17:48:28Z" + content=""" +There have been several bug reports with that same error message +and different causes (permissions problems; filesystems on which sqlite +doesn't work etc). So all that I know from the error is something went +wrong with the sqlite database, which causes queries of it to fail. + +Using the linux version of git-annex on Winows via the (misleadingly named) +"bash for windows" is pretty unusual. I have never seen it work; Microsoft +were still adding missing linux system calls to their emulation layer the +last time I tried it. It's quite possible that there remains a bug in that +emulation layer, that prevents sqlite from working, or makes it unreliable, +etc. I cannot reproduce the problem running git-annex on Linux. + +Note that since it's a v6 repository, running `git add` actually adds the +files to git-annex. `git add` unfortunately runs `git-annex smudge` once +per file. As you note, this is slow; this is one of the several reasons +documented in [[todo/smudge]] why v6 mode is still considered experimental; +and it will need changes in git to improve the speed). + +It would be useful to know if it's failing on the first file, +or if several files get processed ok before it begins to fail. +You could set `GIT_TRACE=1` in the environment to find out. +"""]] diff --git a/doc/bugs/Massive_git_add_produces_sqlite_crashes/comment_2_74f3520d5eaecea35e52f3ffc2bbc559._comment b/doc/bugs/Massive_git_add_produces_sqlite_crashes/comment_2_74f3520d5eaecea35e52f3ffc2bbc559._comment new file mode 100644 index 0000000000..8b00153de4 --- /dev/null +++ b/doc/bugs/Massive_git_add_produces_sqlite_crashes/comment_2_74f3520d5eaecea35e52f3ffc2bbc559._comment @@ -0,0 +1,31 @@ +[[!comment format=mdwn + username="webanck" + avatar="http://cdn.libravatar.org/avatar/cd273f76ef8c4218510b4f50ef7e1f3d" + subject="reply to comment 1" + date="2017-11-07T19:16:08Z" + content=""" +Hello joey, and thanks for the quick answer, I am honored ! +It's refreshing to enter an active community :) + +About the usage of \"bash on Windows\", well, git-annex had too many problems, especially with the symlinks, until the recent \"Windows 10 Fall Creators Update\". +Now it goes pretty well. + +My main concern for the moment is the symlinks created in the \"bash\" can't be used by windows whereas symlinks created in windows can be used in \"bash\". +Thus working in an annexed directory with Windows programs needs to unlock files. One note though, I didn't try with hardlinks. + +About the v6. I just figured that some minutes ago and, following , created a .gitattributes file. + + *.txt annex.largefiles=nothing + +It works better now. + +About the smudge/sqlite errors. I encountered those only after a big number of adds. +I even managed to add all my files eventually with a little loop ignoring the errors (for 1000 files): + + for i in {1..10} + do + git add $(git ls-files --others | grep txt | head -100) 2> /dev/null + done + + +"""]] diff --git a/doc/bugs/Massive_git_add_produces_sqlite_crashes/comment_3_24eadf45e4d8690470ed34686569a323._comment b/doc/bugs/Massive_git_add_produces_sqlite_crashes/comment_3_24eadf45e4d8690470ed34686569a323._comment new file mode 100644 index 0000000000..ee761a7cef --- /dev/null +++ b/doc/bugs/Massive_git_add_produces_sqlite_crashes/comment_3_24eadf45e4d8690470ed34686569a323._comment @@ -0,0 +1,11 @@ +[[!comment format=mdwn + username="joey" + subject="""comment 3""" + date="2017-11-07T19:45:51Z" + content=""" +The sqlite failure does not seem to be due to the database getting too +large or anything like that, because your loop does eventually manage to +add all the files. + +Perhaps sqlite is failing intermittently for whatever reason.. +"""]] diff --git a/doc/bugs/Massive_git_add_produces_sqlite_crashes/comment_4_3f44cf6a9251f664cd0e00d168232696._comment b/doc/bugs/Massive_git_add_produces_sqlite_crashes/comment_4_3f44cf6a9251f664cd0e00d168232696._comment new file mode 100644 index 0000000000..7c5d61e968 --- /dev/null +++ b/doc/bugs/Massive_git_add_produces_sqlite_crashes/comment_4_3f44cf6a9251f664cd0e00d168232696._comment @@ -0,0 +1,12 @@ +[[!comment format=mdwn + username="webanck" + avatar="http://cdn.libravatar.org/avatar/cd273f76ef8c4218510b4f50ef7e1f3d" + subject="Similar issue again" + date="2018-08-13T14:18:17Z" + content=""" +Hello, it has been a while since I posted here about this issue with sqlite but it keeps following me! I randomly get errors while trying to lock files: +```sqlite worker thread crashed: SQLite3 returned ErrorIO while attempting to perform prepare \"SELECT null from content limit 1\": disk I/O error``` + +Should I worry about the state of my hard drive? And I don't know if it is intended, but when this happens, the process doesn't stop with a failure code, it just freezes. +I checked with top, and git-annex seems to continue doing stuff as it is still using a full core. +"""]] diff --git a/doc/bugs/Metadata_charset_not_uniform.mdwn b/doc/bugs/Metadata_charset_not_uniform.mdwn new file mode 100644 index 0000000000..80b643d5b3 --- /dev/null +++ b/doc/bugs/Metadata_charset_not_uniform.mdwn @@ -0,0 +1,66 @@ +### Please describe the problem. + +Metadata are not stored in a consistent format. It seems more like git-annex chooses the "smallest" charset able to hold the data, i.e. US-ASCII, unless there are latin1 characters, and only UTF-8 if there are UTF-8 characters that are not in latin1 + +### What steps will reproduce the problem? + + % git init + Initialized empty Git repository in /home/madduck/.tmp/cdt.GlIevu/.git/ + + % git annex init + init ok + (recording state in git...) + + % date > a + + % git annex add a + add a ok + (recording state in git...) + + % git annex metadata -s one=$(echo US-ASCII | iconv -tus-ascii) a + metadata a + lastchanged=2016-09-25@13-18-57 + one=US-ASCII + one-lastchanged=2016-09-25@13-18-57 + ok + (recording state in git...) + + % git annex metadata -s two=$(echo lätin1 | iconv -tlatin1) a + metadata a + lastchanged=2016-09-25@13-19-37 + one=US-ASCII + one-lastchanged=2016-09-25@13-18-57 + two=lätin1 + two-lastchanged=2016-09-25@13-19-37 + ok + (recording state in git...) + + % git annex metadata -s three=$(echo unicode… | iconv -tutf8) a + metadata a + lastchanged=2016-09-25@13-19-41 + one=US-ASCII + one-lastchanged=2016-09-25@13-18-57 + three=unicode… + three-lastchanged=2016-09-25@13-19-41 + two=lätin1 + two-lastchanged=2016-09-25@13-19-37 + ok + (recording state in git...) + + % git annex metadata -g three a | iconv -tutf8 + unicode… + + % git annex metadata -g two a | iconv -tutf8 + liconv: illegal input sequence at position 1 + + % git annex metadata -g one a | iconv -tutf8 + US-ASCII + + % git annex metadata -g two a | iconv -flatin1 -tutf8 + lätin1 + +### What version of git-annex are you using? On what operating system? + +6.20160808-1 + +[[!tag moreinfo]] diff --git a/doc/bugs/Metadata_charset_not_uniform/comment_1_bb6a2016801687ef38522611d7b6f2bc._comment b/doc/bugs/Metadata_charset_not_uniform/comment_1_bb6a2016801687ef38522611d7b6f2bc._comment new file mode 100644 index 0000000000..e0ac56c47d --- /dev/null +++ b/doc/bugs/Metadata_charset_not_uniform/comment_1_bb6a2016801687ef38522611d7b6f2bc._comment @@ -0,0 +1,12 @@ +[[!comment format=mdwn + username="joey" + subject="""comment 1""" + date="2016-09-26T20:50:26Z" + content=""" +The metadata you get out should always be encoded the same as the metadata +you put in. The encoding, or encodings used are up to you. + +Are you seeing metadata queries returning a different sequence of bytes +than the sequence of bytes that were originally stored? If not, I don't +think this is a bug. +"""]] diff --git a/doc/bugs/Metadata_views_in_v6_repo_upgraded_from_direct_mode_act_strangely.mdwn b/doc/bugs/Metadata_views_in_v6_repo_upgraded_from_direct_mode_act_strangely.mdwn new file mode 100644 index 0000000000..015f2c2ed6 --- /dev/null +++ b/doc/bugs/Metadata_views_in_v6_repo_upgraded_from_direct_mode_act_strangely.mdwn @@ -0,0 +1,79 @@ +### Please describe the problem. +In a v6 repository upgraded from direct mode on a FAT filesystem, the view `/=*` replaces locally available files with placeholders and embeds their directory in the name. For example, `x3gs/one.x3g` becomes `x3gs/one_%x3gs%.x3g`. + +### What steps will reproduce the problem? +Following http://git-annex.branchable.com/forum/How_to_hide_broken_symlinks/ on a v6 repository upgraded from direct mode. + +### What version of git-annex are you using? On what operating system? +6.20180427 on NixOS. Installed via `nix-env -iA nixos.gitAndTools.git-annex`. + +### Please provide any additional information below. + +[[!format sh """ +[leo60228@digitaleo:~]$ fallocate -l $((1024*1024*1024*2)) demo.img + +[leo60228@digitaleo:~]$ mkfs.vfat demo.img +mkfs.fat 4.1 (2017-01-24) + +[leo60228@digitaleo:~]$ mkdir demo + +[leo60228@digitaleo:~]$ sudo mount -o loop,uid=${UID},gid=$(id -g $UID) demo.img demo + +[leo60228@digitaleo:~]$ cd demo + +[leo60228@digitaleo:~/demo]$ git init +Initialized empty Git repository in /home/leo60228/demo/.git/ + +[leo60228@digitaleo:~/demo]$ git commit --allow-empty -m 'init' +[master (root-commit) 8dc8e0a] init + +[leo60228@digitaleo:~/demo]$ git annex init +init + Detected a filesystem without fifo support. + + Disabling ssh connection caching. + + Detected a crippled filesystem. + + Enabling direct mode. +ok +(recording state in git...) + +[leo60228@digitaleo:~/demo]$ mkdir subdir + +[leo60228@digitaleo:~/demo]$ echo hi > subdir/file + +[leo60228@digitaleo:~/demo]$ git annex upgrade +upgrade (v5 to v6...) (scanning for unlocked files...) +ok +(recording state in git...) + +[leo60228@digitaleo:~/demo]$ git annex add subdir/ +add subdir/file ok +(recording state in git...) + +[leo60228@digitaleo:~/demo]$ git commit -m 'add file' +[adjusted/master(unlocked) 0e870b3] add file + 1 file changed, 1 insertion(+) + create mode 100644 subdir/file + +[leo60228@digitaleo:~/demo]$ ls subdir/ +file + +[leo60228@digitaleo:~/demo]$ cat subdir/file +hi + +[leo60228@digitaleo:~/demo]$ git-annex view /=* +view (searching...) +Switched to branch 'views/_=_' +ok + +[leo60228@digitaleo:~/demo]$ ls subdir/ +file_%subdir% + +[leo60228@digitaleo:~/demo]$ cat subdir/file_%subdir% +../.git/annex/objects/zQ/MQ/SHA256E-s3--98ea6e4f216f2fb4b69fff9b3a44842c38686ca685f3f55dc48c5d3fb1107be4/SHA256E-s3--98ea6e4f216f2fb4b69fff9b3a44842c38686ca685f3f55dc48c5d3fb1107be4 +"""]] + +### Have you had any luck using git-annex before? (Sometimes we get tired of reading bug reports all day and a lil' positive end note does wonders) +Yep! I already use it to move files between my laptop's HDD and SSD, and to copy files between my many SD cards. I was trying this to see if I could not have to scroll as far on my 3D printer's menu. diff --git a/doc/bugs/Metadata_views_in_v6_repo_upgraded_from_direct_mode_act_strangely/comment_1_9b274536dd5b2f29207fa2bebf3cad28._comment b/doc/bugs/Metadata_views_in_v6_repo_upgraded_from_direct_mode_act_strangely/comment_1_9b274536dd5b2f29207fa2bebf3cad28._comment new file mode 100644 index 0000000000..648d7c84ff --- /dev/null +++ b/doc/bugs/Metadata_views_in_v6_repo_upgraded_from_direct_mode_act_strangely/comment_1_9b274536dd5b2f29207fa2bebf3cad28._comment @@ -0,0 +1,8 @@ +[[!comment format=mdwn + username="https://openid-provider.appspot.com/iakornfeld" + nickname="iakornfeld" + subject="Another weird bit" + date="2018-05-15T14:59:42Z" + content=""" +I forgot to put this in the example, but if the file has an extension the directory name is put *before* the extension. +"""]] diff --git a/doc/bugs/Metadata_views_in_v6_repo_upgraded_from_direct_mode_act_strangely/comment_2_b129b1642f5d21a5b933c8c51807b33c._comment b/doc/bugs/Metadata_views_in_v6_repo_upgraded_from_direct_mode_act_strangely/comment_2_b129b1642f5d21a5b933c8c51807b33c._comment new file mode 100644 index 0000000000..e66d61afc6 --- /dev/null +++ b/doc/bugs/Metadata_views_in_v6_repo_upgraded_from_direct_mode_act_strangely/comment_2_b129b1642f5d21a5b933c8c51807b33c._comment @@ -0,0 +1,22 @@ +[[!comment format=mdwn + username="joey" + subject="""comment 2""" + date="2018-05-15T15:37:57Z" + content=""" +This has nothing to do with direct mode or v6, or the filesystem. +It is behaving more or less as intended. + +Filenames in views are always mangled to make them guaranteed to be +unique, since two files with base name "foo" could be collected into the +same view directory. This is done by embedding the directory structure +inside the filename as you show. + +In the specific case of "/=*" that is not necessary since the view +is replicating the full directory structure. But most views do need +the mangling, and I feel it's better to always do the mangling than try to +work out when it's not needed. + +And "/=*" is a mostly not useful edge case in the actual useful functionality +of views. Note my comment in the http://git-annex.branchable.com/forum/How_to_hide_broken_symlinks/ +forum post; it's not really a good way to hide missing files. +"""]] diff --git a/doc/bugs/Metadata_views_in_v6_repo_upgraded_from_direct_mode_act_strangely/comment_3_e9c819759cc930088c82e6ed7a038a26._comment b/doc/bugs/Metadata_views_in_v6_repo_upgraded_from_direct_mode_act_strangely/comment_3_e9c819759cc930088c82e6ed7a038a26._comment new file mode 100644 index 0000000000..4a4440b755 --- /dev/null +++ b/doc/bugs/Metadata_views_in_v6_repo_upgraded_from_direct_mode_act_strangely/comment_3_e9c819759cc930088c82e6ed7a038a26._comment @@ -0,0 +1,8 @@ +[[!comment format=mdwn + username="https://openid-provider.appspot.com/iakornfeld" + nickname="iakornfeld" + subject="Looking at code" + date="2018-05-15T19:01:10Z" + content=""" +I was going to try to program this, and https://git.kitenet.net/index.cgi/git-annex.git/tree/Annex/AdjustedBranch.hs appears to have (some?) code already. If the problem has to do with it not being immutable, I was thinking of adding a `filter.*.drop` option. This would, for all filters that have this property matching the file, be called like a normal git filter when the file gets dropped. On the v6 repository filter, there would be a drop filter that's a no-op outside an adjusted branch and would remove it from the adjusted branch when on one. Other uses could include backing up dropped files to a slow, local drive (like a portable NAS with a low-RPM drive?) when they're dropped in the background, so that they wouldn't have to be redownloaded when your main server is unavailable/remote. +"""]] diff --git a/doc/bugs/Missing_automounts_block_every_command.mdwn b/doc/bugs/Missing_automounts_block_every_command.mdwn new file mode 100644 index 0000000000..53af6f15a4 --- /dev/null +++ b/doc/bugs/Missing_automounts_block_every_command.mdwn @@ -0,0 +1,47 @@ +### Please describe the problem. +When a remote is located on a device (network) that systemd is configured to automount but fails to do so, every git-annex command blocks/waits until the automount times out. + +Commands that have to access such a remote (e.g., `sync`, `move`) are are allowed to block, but commands that only operate on the local repository (e.g., `version`, `add`, `calckey`, `find`) or another one (`sync not-doesnotexist`, `move --to=not-doesnotexist`) should not. + +The Bash completion is also affected and blocks at every tab. + +Probably related: git-annex causes **not missing** idle hard drives (as remotes) to spin up for no reason – even for local commands and completions. + +### What steps will reproduce the problem? +Add a non-existing mount point to `/etc/fstab`: + + /dev/sdoesnotexist /mnt/doesnotexist ext4 defaults,noauto,x-systemd.automount,x-systemd.device-timeout=10 0 0 + +Add a remote pointing to a path on `/mnt/doesnotexist`: + + $ git remote add doesnotexist /mnt/doesnotexist/path/to/repository + +Use any git-annex command and wait for at least `x-systemd.device-timeout`: + + $ time git-annex version > /dev/null + real 0m10.433s + user 0m0.171s + sys 0m0.028s + +### What version of git-annex are you using? On what operating system? + + git-annex version: 6.20171214-g61b515d71d + build flags: Assistant Webapp Pairing Testsuite S3(multipartupload)(storageclasses) WebDAV Inotify DBus DesktopNotify ConcurrentOutput TorrentParser MagicMime Feeds + dependency versions: aws-0.18 bloomfilter-2.0.1.0 cryptonite-0.24 DAV-1.3.1 feed-1.0.0.0 ghc-8.2.2 http-client-0.5.7.1 persistent-sqlite-2.6.4 torrent-10000.1.1 uuid-1.3.13 yesod-1.4.5 + key/value backends: SHA256E SHA256 SHA512E SHA512 SHA224E SHA224 SHA384E SHA384 SHA3_256E SHA3_256 SHA3_512E SHA3_512 SHA3_224E SHA3_224 SHA3_384E SHA3_384 SKEIN256E SKEIN256 SKEIN512E SKEIN512 SHA1E SHA1 MD5E MD5 WORM URL + remote types: git gcrypt p2p S3 bup directory rsync web bittorrent webdav tahoe glacier ddar hook external + local repository version: 5 + supported repository versions: 3 5 6 + upgrade supported from repository versions: 0 1 2 3 4 5 + operating system: linux x86_64 + +### Please provide any additional information below. + +`strace` always includes a call to `stat("/mnt/doesnotexist/path/to/repository")`. + +### Have you had any luck using git-annex before? (Sometimes we get tired of reading bug reports all day and a lil' positive end note does wonders) + +I’m very happy with git-annex (thanks) and use it frequently enough to notice this behavior. + +> [[fixed|done]] via the `remote..annex-checkuuid` config setting +> that can disable this behavior. --[[Joey]] diff --git a/doc/bugs/Missing_automounts_block_every_command/comment_1_3e9ac639a2f15cc3b0d277b5fbf17db7._comment b/doc/bugs/Missing_automounts_block_every_command/comment_1_3e9ac639a2f15cc3b0d277b5fbf17db7._comment new file mode 100644 index 0000000000..620e061e82 --- /dev/null +++ b/doc/bugs/Missing_automounts_block_every_command/comment_1_3e9ac639a2f15cc3b0d277b5fbf17db7._comment @@ -0,0 +1,18 @@ +[[!comment format=mdwn + username="joey" + subject="""comment 1""" + date="2018-01-09T17:02:41Z" + content=""" +There are a couple of parts to this, so let's get this one out of the way +first: Tab completion etc should not be looking at remotes. + +It seems that even `git annex --help` does for some reason; so does +stuff like `git annex examinekey`. So it's happening in a core code-path. + +Ah, ok.. Git.Config.read uses Git.Construct.fromRemotes, +which uses Git.Construct.fromAbsPath, which stats +the remote directory to handle ".git" canonicalization. + +Fixed this part of it; now only when the remoteList is built does it +stat remotes. +"""]] diff --git a/doc/bugs/Missing_automounts_block_every_command/comment_2_94e118e60c74e6ac44aa6a396d41a939._comment b/doc/bugs/Missing_automounts_block_every_command/comment_2_94e118e60c74e6ac44aa6a396d41a939._comment new file mode 100644 index 0000000000..f60b4d7062 --- /dev/null +++ b/doc/bugs/Missing_automounts_block_every_command/comment_2_94e118e60c74e6ac44aa6a396d41a939._comment @@ -0,0 +1,39 @@ +[[!comment format=mdwn + username="joey" + subject="""comment 2""" + date="2018-01-09T19:56:42Z" + content=""" +With the above dealt with, the remaining problem is with commands +like `git annex whereis` or `git annex info`, which don't really +any on any remote, but still need to examine the remotes as part of +building the remoteList. + +git-annex supports remotes that point to a mount point that might have +different drives mounted at it at different times. So, it needs to +check the git config of the remote each time, to see what repository is +currently there. + +Even commands like "whereis" and "info" have output that depends on +what repository a remote is currently pointing to. In some cases, +"whereis" might not output anything that depends on a given remote, +so in theory it could avoid looking at the config of that remote. +And a command like "git annex copy --to origin" doesn't really +need to look at the configs of any other remotes. + +But to avoid unncessarily checking the git configs of remotes that a +command does not use would need each use of the current remoteList +to be replaced with something else that does the minimal needed work, +instead of building the whole remoteList. I think this would be quite +complicated. + +And, I don't know that it would address the bug report adequequately, even +if it were done. Running `git annex info` would +still block waiting for the automount; `git annex whereis` would +only *sometimes* block, depending on where content is. + +So instead of that approach, perhaps a config setting will do? +A per-remote config that tells git-annex that only one repository +should ever be mounted at its location. That would make git-annex +avoid checking the git config of that remote each time, except +when it's actually storing/dropping content on it. +"""]] diff --git a/doc/bugs/Missing_automounts_block_every_command/comment_3_c7d5015db5aea0dd4d2ed67803ee9c92._comment b/doc/bugs/Missing_automounts_block_every_command/comment_3_c7d5015db5aea0dd4d2ed67803ee9c92._comment new file mode 100644 index 0000000000..a2dae9620c --- /dev/null +++ b/doc/bugs/Missing_automounts_block_every_command/comment_3_c7d5015db5aea0dd4d2ed67803ee9c92._comment @@ -0,0 +1,25 @@ +[[!comment format=mdwn + username="joey" + subject="""comment 3""" + date="2018-01-09T20:29:40Z" + content=""" +There would still be some cases where a git-annex command blocks somewhat +unexpectedly on the automount. + +For one, `git annex drop` can need to +check if content is in a remote, and so would block, despite not acting +directly on that remote. + +And, `git annex get` of a file that's located in +such an locally automounted remote and a network remote will +default to trying the local remote first, and so would block. + +The cost of the automounted remote could be adjusted to make these commands +prefer some other remote, but then you've configured git-annex to not +use the automounted remote much, which is probably not what you really want +to do if it's a fast drive. + +Of course, there are also ways to automount removable drives when they get +plugged in, rather than using automounts that block on access, and so +neatly avoid all blocking problems. +"""]] diff --git a/doc/bugs/Missing_automounts_block_every_command/comment_4_c3cfb1c27806c45dfa6ebf275c175d3e._comment b/doc/bugs/Missing_automounts_block_every_command/comment_4_c3cfb1c27806c45dfa6ebf275c175d3e._comment new file mode 100644 index 0000000000..31f97beeba --- /dev/null +++ b/doc/bugs/Missing_automounts_block_every_command/comment_4_c3cfb1c27806c45dfa6ebf275c175d3e._comment @@ -0,0 +1,13 @@ +[[!comment format=mdwn + username="joey" + subject="""comment 4""" + date="2018-01-10T17:37:42Z" + content=""" +Added remote..annex-checkuuid config, which can be set to false +to disable the default checking of the uuid. It will still check before +making any modification of the remote repository. + +There may still be situations where using this kind of automount is suboptimal +with git-annex, as outlined in comment 3, but I think this is as far as it makes +sense to change git-annex to deal with them. +"""]] diff --git a/doc/bugs/Missing_backend_extension_in_files_with_long_extensions.mdwn b/doc/bugs/Missing_backend_extension_in_files_with_long_extensions.mdwn new file mode 100644 index 0000000000..4f937d14e7 --- /dev/null +++ b/doc/bugs/Missing_backend_extension_in_files_with_long_extensions.mdwn @@ -0,0 +1,42 @@ +### Please describe the problem. + +Adding a file with a long extension, like Apple Numbers file (for example budget.numbers) results in it being added to the backend without file extension at all. + +### What steps will reproduce the problem? +[[!format sh """ +$ touch budget.numbers +$ git-annex add budget.numbers +add budget.numbers ok +(recording state in git...) +$ git-annex lookupkey budget.numbers +SKEIN256E-s0--c8877087da56e072870daa843f176e9453115929094c3a40c463a196c29bf7ba +$ touch budget.num +$ git annex add budget.num +add budget.num ok +(recording state in git...) +$ git annex lookupkey budget.num +SKEIN256E-s0--c8877087da56e072870daa843f176e9453115929094c3a40c463a196c29bf7ba.num +"""]] +### What version of git-annex are you using? On what operating system? +[[!format sh """ +$ git-annex version +git-annex version: 6.20180807 +build flags: Assistant Webapp Pairing S3(multipartupload)(storageclasses) WebDAV FsEvents ConcurrentOutput TorrentParser MagicMime Feeds Testsuite +dependency versions: aws-0.19 bloomfilter-2.0.1.0 cryptonite-0.25 DAV-1.3.2 feed-1.0.0.0 ghc-8.2.2 http-client-0.5.13.1 persistent-sqlite-2.6.4 torrent-10000.1.1 uuid-1.3.13 yesod-1.4.5 +key/value backends: SHA256E SHA256 SHA512E SHA512 SHA224E SHA224 SHA384E SHA384 SHA3_256E SHA3_256 SHA3_512E SHA3_512 SHA3_224E SHA3_224 SHA3_384E SHA3_384 SKEIN256E SKEIN256 SKEIN512E SKEIN512 BLAKE2B256E BLAKE2B256 BLAKE2B512E BLAKE2B512 BLAKE2B160E BLAKE2B160 BLAKE2B224E BLAKE2B224 BLAKE2B384E BLAKE2B384 BLAKE2S256E BLAKE2S256 BLAKE2S160E BLAKE2S160 BLAKE2S224E BLAKE2S224 BLAKE2SP256E BLAKE2SP256 BLAKE2SP224E BLAKE2SP224 SHA1E SHA1 MD5E MD5 WORM URL +remote types: git gcrypt p2p S3 bup directory rsync web bittorrent webdav adb tahoe glacier ddar hook external +operating system: darwin x86_64 +supported repository versions: 3 5 6 +upgrade supported from repository versions: 0 1 2 3 4 5 +local repository version: 5 +"""]] + +macOS 10.13.6 + +### Please provide any additional information below. + +macOS isn't able to open a file without extension. In order to just view it, I have to `git annex unlock` it, so that there's a real file with an extension. Is there a way to tell macOS to use symlink extension instead? Or other workaround? + +### Have you had any luck using git-annex before? (Sometimes we get tired of reading bug reports all day and a lil' positive end note does wonders) + +Yup, I've been using it for 5 years. It's awesome! Thank you. diff --git a/doc/bugs/Missing_sanity_check_of_group_names._Special_group_names_should_not_be_allowed..mdwn b/doc/bugs/Missing_sanity_check_of_group_names._Special_group_names_should_not_be_allowed..mdwn new file mode 100644 index 0000000000..bf2972257b --- /dev/null +++ b/doc/bugs/Missing_sanity_check_of_group_names._Special_group_names_should_not_be_allowed..mdwn @@ -0,0 +1,37 @@ +### Please describe the problem. + +As documented in [[git-annex-matching-options]], `--copies` accepts `trustlevel` or `groupname` in the following format: + +* `--copies=trustlevel:number` +* `--copies=groupname:number` + +This is ambiguous in the unlikely case where a user might come up with the idea to create a group called `trusted`, `semitrusted` or `untrusted` (`remote..annex-trustlevel`). It should thereby not be allowed to use such special groups. + +### What steps will reproduce the problem? + + +[[!format sh """ +## In an annex repo: + +git annex group here trusted + +## What does this command do now? +## For reference, it interprets it as `--copies=trustlevel:number` +git annex find --copies=trusted:1 + +"""]] + +### What version of git-annex are you using? On what operating system? + + git-annex version: 6.20180509 + build flags: Assistant Webapp Pairing S3(multipartupload)(storageclasses) WebDAV Inotify DBus DesktopNotify ConcurrentOutput TorrentParser MagicMime Feeds Testsuite + dependency versions: aws-0.14.1 bloomfilter-2.0.1.0 cryptonite-0.20 DAV-1.3.1 feed-0.3.11.1 ghc-8.0.1 http-client-0.4.31.1 persistent-sqlite-2.6 torrent-10000.0.0 uuid-1.3.12 yesod-1.4.3 + key/value backends: SHA256E SHA256 SHA512E SHA512 SHA224E SHA224 SHA384E SHA384 SHA3_256E SHA3_256 SHA3_512E SHA3_512 SHA3_224E SHA3_224 SHA3_384E SHA3_384 SKEIN256E SKEIN256 SKEIN512E SKEIN512 SHA1E SHA1 MD5E MD5 WORM URL + remote types: git gcrypt p2p S3 bup directory rsync web bittorrent webdav adb tahoe glacier ddar hook external + operating system: linux x86_64 + +### Have you had any luck using git-annex before? (Sometimes we get tired of reading bug reports all day and a lil' positive end note does wonders) + +Yes! Thanks so much for your work! + +> Closing per comments. [[done]] --[[Joey]] diff --git a/doc/bugs/Missing_sanity_check_of_group_names._Special_group_names_should_not_be_allowed./comment_1_b4ba41b6e0e611d90ddda92bbe991266._comment b/doc/bugs/Missing_sanity_check_of_group_names._Special_group_names_should_not_be_allowed./comment_1_b4ba41b6e0e611d90ddda92bbe991266._comment new file mode 100644 index 0000000000..4bd9247927 --- /dev/null +++ b/doc/bugs/Missing_sanity_check_of_group_names._Special_group_names_should_not_be_allowed./comment_1_b4ba41b6e0e611d90ddda92bbe991266._comment @@ -0,0 +1,22 @@ +[[!comment format=mdwn + username="joey" + subject="""comment 1""" + date="2018-06-04T16:06:57Z" + content=""" +Thanks for pointing out this ambiguity, which I don't remember having +ever considered. + +The code for this does look for the name of a trust level first, +and only a group name if it's not a trust level. So, adding a group +with a colliding name will never change the behavior of such a command +or preferred content expression. + +The only problem then would be that you couldn't match +on a group by that name. But if you run into that problem, +you can simply rename your group. + +So I don't think that git-annex needs a sanity check, really. + +(I have added a comment in the code to make clear that the order of +parsing this does matter.) +"""]] diff --git a/doc/bugs/Missing_sanity_check_of_group_names._Special_group_names_should_not_be_allowed./comment_2_73b2d20a793a8a17cf0a3d593b3b4ed4._comment b/doc/bugs/Missing_sanity_check_of_group_names._Special_group_names_should_not_be_allowed./comment_2_73b2d20a793a8a17cf0a3d593b3b4ed4._comment new file mode 100644 index 0000000000..c9b6f38a71 --- /dev/null +++ b/doc/bugs/Missing_sanity_check_of_group_names._Special_group_names_should_not_be_allowed./comment_2_73b2d20a793a8a17cf0a3d593b3b4ed4._comment @@ -0,0 +1,8 @@ +[[!comment format=mdwn + username="ypid" + avatar="http://cdn.libravatar.org/avatar/4286841cddc77176758dde874afe9a3a" + subject="comment 2" + date="2018-06-04T19:42:30Z" + content=""" +Works for me, thanks. I also think this is not a real issue, I just noticed the edge case. +"""]] diff --git a/doc/bugs/More_build_oddities_under_OpenBSD.mdwn b/doc/bugs/More_build_oddities_under_OpenBSD.mdwn new file mode 100644 index 0000000000..f36604c575 --- /dev/null +++ b/doc/bugs/More_build_oddities_under_OpenBSD.mdwn @@ -0,0 +1,41 @@ +### Please describe the problem. +I have managed to get most things working under OpenBSD 5.4 now. + +One of the last hurdles is that if I enable XMPP the build fails on "Loading package gnuidn-0.2.1..." +See the error below. + +I suspect this is an error in git-annex because network-protocol-xmpp AND gnuidn compiles (and links) fine. + +I will gladly do anything I can to get this working, but I'm at a loss what to do right now. It's the last major piece of the puzzle before I get it properly functioning under OpenBSD. + +### What steps will reproduce the problem? +Building with XMPP support under OpenBSD 5.4 + +### What version of git-annex are you using? On what operating system? +5.20140129 under OpenBSD 5.4 + +### Please provide any additional information below. + +[[!format sh """ +# If you can, paste a complete transcript of the problem occurring here. +# If the problem is with the git-annex assistant, paste in .git/annex/daemon.log +Loading package gnuidn-0.2.1 ... + +GHCi runtime linker: fatal error: I found a duplicate definition for symbol + c_isascii +whilst processing object file + /usr/local/lib/libidn.a +This could be caused by: + * Loading two different object files which export the same symbol + * Specifying the same object file twice on the GHCi command line + * An incorrect `package.conf' entry, causing some object to be + loaded twice. +GHCi cannot safely continue in this situation. Exiting now. Sorry. + + +# End of transcript or log. +"""]] + +> Since this bug was filed, git-annex has dropped xmpp and the problimatic +> libraries. Since the bug comments say the build worked without xmpp, +> closing this bug [[done]] --[[Joey]] diff --git a/doc/bugs/More_build_oddities_under_OpenBSD/comment_10_09297f99f3c1c081738ca4ab32808fde._comment b/doc/bugs/More_build_oddities_under_OpenBSD/comment_10_09297f99f3c1c081738ca4ab32808fde._comment new file mode 100644 index 0000000000..e3871ea612 --- /dev/null +++ b/doc/bugs/More_build_oddities_under_OpenBSD/comment_10_09297f99f3c1c081738ca4ab32808fde._comment @@ -0,0 +1,8 @@ +[[!comment format=mdwn + username="http://joeyh.name/" + ip="209.250.56.163" + subject="comment 10" + date="2014-02-08T18:31:23Z" + content=""" +But you said that setSocketOption failed when you were using XMPP, not when starting the webapp, so I think it's more likely to be one of the setSocketOption calls in the network library, or possibly somewhere else. +"""]] diff --git a/doc/bugs/More_build_oddities_under_OpenBSD/comment_11_1407efc78b92a3c6156154f54e4a14e2._comment b/doc/bugs/More_build_oddities_under_OpenBSD/comment_11_1407efc78b92a3c6156154f54e4a14e2._comment new file mode 100644 index 0000000000..78c430533d --- /dev/null +++ b/doc/bugs/More_build_oddities_under_OpenBSD/comment_11_1407efc78b92a3c6156154f54e4a14e2._comment @@ -0,0 +1,97 @@ +[[!comment format=mdwn + username="https://www.google.com/accounts/o8/id?id=AItOawkzwmw_zyMpZC9_J7ey--woeYPoZkAOgGw" + nickname="dxtrish" + subject="comment 11" + date="2014-02-08T19:18:17Z" + content=""" +I honestly have no idea why that move works because + + % ls -lh /usr/lib|grep -E '(gsasl|xml2|gnutls|idn)' + +returns nothing. But couldn't those symbols already be in the other libraries considering, from what I've read at least, haskell stuff are statically compiled by default? + +Anyway, you are completely right that this happened when I try to use XMPP. The reason I was looking in the wrong place to begin with was because the webapp spit out the error messsage. I have redirected my attention to the network library and the xmpp library. + +But I might have found something interesting in the network library. Keep in mind that I just learned a little today, so do correct me if I'm wrong. + +Looking at http://hackage.haskell.org/package/network-2.2.1.8/docs/src/Network-Socket.html I found: + + setSocketOption :: Socket + -> SocketOption -- Option Name + -> Int -- Option Value + -> IO () + setSocketOption (MkSocket s _ _ _ _) so v = do + with (fromIntegral v) $ \ptr_v -> do + throwErrnoIfMinus1_ \"setSocketOption\" $ + c_setsockopt s (socketOptLevel so) (packSocketOption so) ptr_v + (fromIntegral (sizeOf v)) + return () + +Everything here looks good. So I decided to take a look at SocketOption, socketOptLevel and packSocketOption. + + data SocketOption + = DummySocketOption__ + | Debug {- SO_DEBUG -} + | ReuseAddr {- SO_REUSEADDR -} + | Type {- SO_TYPE -} + | SoError {- SO_ERROR -} + | DontRoute {- SO_DONTROUTE -} + | Broadcast {- SO_BROADCAST -} + | SendBuffer {- SO_SNDBUF -} + | RecvBuffer {- SO_RCVBUF -} + | KeepAlive {- SO_KEEPALIVE -} + | OOBInline {- SO_OOBINLINE -} + | TimeToLive {- IP_TTL -} + | MaxSegment {- TCP_MAXSEG -} + | NoDelay {- TCP_NODELAY -} + | Linger {- SO_LINGER -} + | RecvLowWater {- SO_RCVLOWAT -} + | SendLowWater {- SO_SNDLOWAT -} + | RecvTimeOut {- SO_RCVTIMEO -} + | SendTimeOut {- SO_SNDTIMEO -} + + socketOptLevel :: SocketOption -> CInt + socketOptLevel so = + case so of + TimeToLive -> 0 + MaxSegment -> 6 + NoDelay -> 6 + _ -> 1 + + packSocketOption :: SocketOption -> CInt + packSocketOption so = + case so of + Debug -> 1 + ReuseAddr -> 2 + Type -> 3 + SoError -> 4 + DontRoute -> 5 + Broadcast -> 6 + SendBuffer -> 7 + RecvBuffer -> 8 + KeepAlive -> 9 + OOBInline -> 10 + TimeToLive -> 2 + MaxSegment -> 2 + NoDelay -> 1 + Linger -> 13 + RecvLowWater -> 18 + SendLowWater -> 19 + RecvTimeOut -> 20 + SendTimeOut -> 21 + +Everything looks good so I thought long and hard about this. Then, by chance, I just looked at the man page for setsockopt() and it mentioned SOL_SOCKET and I was like \"Hmm...\" + + % grep -R SOL_SOCKET /usr/include + /usr/include/openssl/e_os.h:#define ioctlsocket(a,b,c) setsockopt((a),SOL_SOCKET,(b),(c),sizeof(*(c))) + /usr/include/sys/socket.h:#define SOL_SOCKET 0xffff /* options for socket level */ + /usr/include/sys/socket.h:/* Read using getsockopt() with SOL_SOCKET, SO_PEERCRED */ + +Wat? + + #define SOL_SOCKET 0xffff + +Going back to the Haskell code above I realized that SetSocketOption will NEVER feed 0xffff as level to setsockopt() because socketOptLevel returns 1 unless optname is TimeToLive, MaxSegment or NoDelay. + +Am I way off? +"""]] diff --git a/doc/bugs/More_build_oddities_under_OpenBSD/comment_12_fdec033e37652c51fbcd74438586d285._comment b/doc/bugs/More_build_oddities_under_OpenBSD/comment_12_fdec033e37652c51fbcd74438586d285._comment new file mode 100644 index 0000000000..77c8edc68a --- /dev/null +++ b/doc/bugs/More_build_oddities_under_OpenBSD/comment_12_fdec033e37652c51fbcd74438586d285._comment @@ -0,0 +1,12 @@ +[[!comment format=mdwn + username="http://joeyh.name/" + ip="209.250.56.163" + subject="comment 12" + date="2014-02-08T20:03:30Z" + content=""" +WRT haskell static linking, AFAIK that only affects haskell libraries. If they in turn link with C libs, the linking is still dynamic. At least this is the case on both Linux and the limited BSDs I've used it on. Have no OpenBSD experience. + +`SOL_SOCKET` is 1 on linux, so you may have the culprit. + +However.. That's a really old version of network! 2.2.1.8 is from 2010. In a more current 2.4.x version, packSocketOption uses the `SOL_SOCKET` value and not 1, so should work AFAICS. +"""]] diff --git a/doc/bugs/More_build_oddities_under_OpenBSD/comment_13_ed3716baf787ca17d227ce2e327a1959._comment b/doc/bugs/More_build_oddities_under_OpenBSD/comment_13_ed3716baf787ca17d227ce2e327a1959._comment new file mode 100644 index 0000000000..3981d32bfd --- /dev/null +++ b/doc/bugs/More_build_oddities_under_OpenBSD/comment_13_ed3716baf787ca17d227ce2e327a1959._comment @@ -0,0 +1,8 @@ +[[!comment format=mdwn + username="http://joeyh.name/" + ip="209.250.56.163" + subject="comment 13" + date="2014-02-08T20:12:12Z" + content=""" +Does your binary end up dynamically linked to libxml2 etc? If not, it certianly seems plausible that the haskell libs statically linked with the C libs, and then at binary link time it tried to redundantly link with the libs again and failed. If this is the case, it seems it would probably be a bug in ghc. You can use `cabal build --ghc-options=-v` to get a look at how the linker is run. +"""]] diff --git a/doc/bugs/More_build_oddities_under_OpenBSD/comment_14_cf5f92e5cdfc738e7f6178c1d7a73ceb._comment b/doc/bugs/More_build_oddities_under_OpenBSD/comment_14_cf5f92e5cdfc738e7f6178c1d7a73ceb._comment new file mode 100644 index 0000000000..bb4d095a37 --- /dev/null +++ b/doc/bugs/More_build_oddities_under_OpenBSD/comment_14_cf5f92e5cdfc738e7f6178c1d7a73ceb._comment @@ -0,0 +1,11 @@ +[[!comment format=mdwn + username="https://www.google.com/accounts/o8/id?id=AItOawkzwmw_zyMpZC9_J7ey--woeYPoZkAOgGw" + nickname="dxtrish" + subject="comment 14" + date="2014-02-08T20:29:46Z" + content=""" +Sigh.. Ofcourse it was an old version. It never occurred to me to check that. I was just Googling around and stumbled over that page. +Do you have any pointers on how I would debug this further? + +Regarding the linking; I haven't checked it any further actually. I'm planning on investigating that further once I can get this error sorted out. But my hypothesis is that it's all statically linked all the way through, like I said. That would also explain those error messages I got during linking before. +"""]] diff --git a/doc/bugs/More_build_oddities_under_OpenBSD/comment_15_ad4b7191c9b8f67def33b26a1d762a5d._comment b/doc/bugs/More_build_oddities_under_OpenBSD/comment_15_ad4b7191c9b8f67def33b26a1d762a5d._comment new file mode 100644 index 0000000000..4aeda69ebf --- /dev/null +++ b/doc/bugs/More_build_oddities_under_OpenBSD/comment_15_ad4b7191c9b8f67def33b26a1d762a5d._comment @@ -0,0 +1,26 @@ +[[!comment format=mdwn + username="http://joeyh.name/" + ip="209.250.56.163" + subject="comment 15" + date="2014-02-08T20:56:10Z" + content=""" +I think you need to find which calls to setSocketOption are failing and/or what value is causing the crash. It's a bit of a pain to get a backtrace on error (would need to build everything with profiling enabled and run git-annex with +RTS -xc options). Probably simplest to modify the setSocketOption code: + +[[!format patch \"\"\" +diff --git a/Network/Socket.hsc b/Network/Socket.hsc +index 2fe62ee..0c66432 100644 +--- a/Network/Socket.hsc ++++ b/Network/Socket.hsc +@@ -963,7 +963,7 @@ setSocketOption :: Socket + setSocketOption (MkSocket s _ _ _ _) so v = do + (level, opt) <- packSocketOption' \"setSocketOption\" so + with (fromIntegral v) $ \ptr_v -> do +- throwSocketErrorIfMinus1_ \"setSocketOption\" $ ++ throwSocketErrorIfMinus1_ (\"setSocketOption \" ++ show so ++ \" \" ++ show v) $ + c_setsockopt s level opt ptr_v + (fromIntegral (sizeOf (undefined :: CInt))) + return () +\"\"\"]] + +Which should make it print out a quite nice symbolic name of the option being used on failure, which will make it easy to find any call sites and more importantly, determine what's wrong with that option. +"""]] diff --git a/doc/bugs/More_build_oddities_under_OpenBSD/comment_16_2e765b5286d816bea00880a17a20cbfb._comment b/doc/bugs/More_build_oddities_under_OpenBSD/comment_16_2e765b5286d816bea00880a17a20cbfb._comment new file mode 100644 index 0000000000..def40653da --- /dev/null +++ b/doc/bugs/More_build_oddities_under_OpenBSD/comment_16_2e765b5286d816bea00880a17a20cbfb._comment @@ -0,0 +1,10 @@ +[[!comment format=mdwn + username="https://www.google.com/accounts/o8/id?id=AItOawkzwmw_zyMpZC9_J7ey--woeYPoZkAOgGw" + nickname="dxtrish" + subject="comment 16" + date="2014-02-08T21:08:38Z" + content=""" +Thanks for that code snippet! I'll try it out within the next few minutes. + +Also, how would I go about rebuilding everything with profiling support? I'm doing this testing in a virtual machine so I can always start over from scratch. +"""]] diff --git a/doc/bugs/More_build_oddities_under_OpenBSD/comment_17_ded9011dcdbe4de05189a0e8d040f045._comment b/doc/bugs/More_build_oddities_under_OpenBSD/comment_17_ded9011dcdbe4de05189a0e8d040f045._comment new file mode 100644 index 0000000000..f361268419 --- /dev/null +++ b/doc/bugs/More_build_oddities_under_OpenBSD/comment_17_ded9011dcdbe4de05189a0e8d040f045._comment @@ -0,0 +1,10 @@ +[[!comment format=mdwn + username="http://joeyh.name/" + ip="209.250.56.163" + subject="comment 17" + date="2014-02-08T21:23:02Z" + content=""" +Blow away ~/.ghc and ~/.cabal; cabal update; put `library-profiling: True` in ~/.cabal/config; and reinstall everything. + +Note that you may need to first build ghc's own bundled libraries with profiling support, if your ghc installation does not already include them. I don't know how to do that since on debian I can just `apt-get install ghc-prof`. If that's needed, ghc will tell you and refuse to build stuff with profiling. +"""]] diff --git a/doc/bugs/More_build_oddities_under_OpenBSD/comment_18_f7a85b46bf7afaaf431d6771219c66b0._comment b/doc/bugs/More_build_oddities_under_OpenBSD/comment_18_f7a85b46bf7afaaf431d6771219c66b0._comment new file mode 100644 index 0000000000..d8d73a0b77 --- /dev/null +++ b/doc/bugs/More_build_oddities_under_OpenBSD/comment_18_f7a85b46bf7afaaf431d6771219c66b0._comment @@ -0,0 +1,16 @@ +[[!comment format=mdwn + username="https://www.google.com/accounts/o8/id?id=AItOawkzwmw_zyMpZC9_J7ey--woeYPoZkAOgGw" + nickname="dxtrish" + subject="comment 18" + date="2014-02-08T23:25:29Z" + content=""" +I applied your patch and rebuilt. I even tried doing it from scratch just to get it right. + +I have three news: + +1) I got it completely working in a virtual machine + +2) On my actual (physical) openbsd machine it still isn't working, even though they're almost identical. The only difference is that the physical one also has IPv6 connectivity, while the virtual machine doesn't. Could that be it? I don't know yet, although I'm about to try it. + +3) Your patch did not help. It still didn't print out anything more useful, suggesting that it in fact isn't that function that is throwing the error. [This](http://i.imgur.com/tmBiseE.png) is what the actual error looks like and shows up after I enter the jabber account details. +"""]] diff --git a/doc/bugs/More_build_oddities_under_OpenBSD/comment_19_217be2000e423e844241d405ba9f64c8._comment b/doc/bugs/More_build_oddities_under_OpenBSD/comment_19_217be2000e423e844241d405ba9f64c8._comment new file mode 100644 index 0000000000..7ea52c150a --- /dev/null +++ b/doc/bugs/More_build_oddities_under_OpenBSD/comment_19_217be2000e423e844241d405ba9f64c8._comment @@ -0,0 +1,10 @@ +[[!comment format=mdwn + username="http://joeyh.name/" + ip="209.250.56.172" + subject="comment 19" + date="2014-02-09T02:13:20Z" + content=""" +I'll bet you didn't rebuild all the libraries that depend on network. (Which all that static linking makes necessary...) + +IPv6 was my first guess; see http://git-annex.branchable.com/bugs/More_build_oddities_under_OpenBSD/#comment-84ee81cd162d22283fcccc1a41c8f8b3 +"""]] diff --git a/doc/bugs/More_build_oddities_under_OpenBSD/comment_1_4ffea64907656ff2ec65ff4450aadda7._comment b/doc/bugs/More_build_oddities_under_OpenBSD/comment_1_4ffea64907656ff2ec65ff4450aadda7._comment new file mode 100644 index 0000000000..f2ac7af8f0 --- /dev/null +++ b/doc/bugs/More_build_oddities_under_OpenBSD/comment_1_4ffea64907656ff2ec65ff4450aadda7._comment @@ -0,0 +1,11 @@ +[[!comment format=mdwn + username="http://joeyh.name/" + ip="71.80.94.56" + subject="comment 1" + date="2014-02-07T18:49:38Z" + content=""" +This is definitely not an issue in git-annex's code. Two C libraries are exporting the same symbol (gnulib might be the other one, or it could be part of the OpenBSD libc as it's some deprecated POSIX symbol, I don't know) and this is simply not apparent until the linker tries to make a binary linking with both. + +I have dealt with a similar issue on Android by modifying C libraries to not export colliding symbols. See: + +"""]] diff --git a/doc/bugs/More_build_oddities_under_OpenBSD/comment_20_df72e5698ba2bf2eb4fa39c5b2c5be83._comment b/doc/bugs/More_build_oddities_under_OpenBSD/comment_20_df72e5698ba2bf2eb4fa39c5b2c5be83._comment new file mode 100644 index 0000000000..29e58e1564 --- /dev/null +++ b/doc/bugs/More_build_oddities_under_OpenBSD/comment_20_df72e5698ba2bf2eb4fa39c5b2c5be83._comment @@ -0,0 +1,10 @@ +[[!comment format=mdwn + username="http://joeyh.name/" + ip="209.250.56.172" + subject="comment 20" + date="2014-02-20T20:26:03Z" + content=""" +Any further luck on this? + +It would be nice if a page on openbsd could be added to the install page documenting what is needed to get it to build. +"""]] diff --git a/doc/bugs/More_build_oddities_under_OpenBSD/comment_21_8a7f5a87bbd4289362f8f4609c02322d._comment b/doc/bugs/More_build_oddities_under_OpenBSD/comment_21_8a7f5a87bbd4289362f8f4609c02322d._comment new file mode 100644 index 0000000000..5befc39dbe --- /dev/null +++ b/doc/bugs/More_build_oddities_under_OpenBSD/comment_21_8a7f5a87bbd4289362f8f4609c02322d._comment @@ -0,0 +1,8 @@ +[[!comment format=mdwn + username="joey" + subject="""comment 21""" + date="2015-02-06T20:34:17Z" + content=""" +C-Keen on irc: +for the record git-annex works fine on openbsd-current (5.7) without xmpp +"""]] diff --git a/doc/bugs/More_build_oddities_under_OpenBSD/comment_2_4fb96984757b3d37a1a5ebce664aa8fe._comment b/doc/bugs/More_build_oddities_under_OpenBSD/comment_2_4fb96984757b3d37a1a5ebce664aa8fe._comment new file mode 100644 index 0000000000..248f7fb335 --- /dev/null +++ b/doc/bugs/More_build_oddities_under_OpenBSD/comment_2_4fb96984757b3d37a1a5ebce664aa8fe._comment @@ -0,0 +1,8 @@ +[[!comment format=mdwn + username="https://www.google.com/accounts/o8/id?id=AItOawkzwmw_zyMpZC9_J7ey--woeYPoZkAOgGw" + nickname="dxtrish" + subject="comment 2" + date="2014-02-07T19:38:14Z" + content=""" +I do realize that it's not a fault in git-annex' code. I'm sorry if it was a stupid idea to post it here, but I was thinking if there exists some kind of workaround one could implement in the build system. I mean.. This isn't the first time someone compiles a program with libidn, gettext and/or gnutls (According to 'grep -R c_isascii /usr') +"""]] diff --git a/doc/bugs/More_build_oddities_under_OpenBSD/comment_3_c5fdf29499a02be83850d1238fc8ce23._comment b/doc/bugs/More_build_oddities_under_OpenBSD/comment_3_c5fdf29499a02be83850d1238fc8ce23._comment new file mode 100644 index 0000000000..ee7c537d91 --- /dev/null +++ b/doc/bugs/More_build_oddities_under_OpenBSD/comment_3_c5fdf29499a02be83850d1238fc8ce23._comment @@ -0,0 +1,8 @@ +[[!comment format=mdwn + username="https://www.google.com/accounts/o8/id?id=AItOawkzwmw_zyMpZC9_J7ey--woeYPoZkAOgGw" + nickname="dxtrish" + subject="comment 3" + date="2014-02-07T20:57:54Z" + content=""" +I have researched this a little more and I am not entirely convinced it is an actual conflict anywhere. I did, in fact, compile a patched version of libidn WITHOUT the c_isascii symbols and.. It suddenly started complaining about even MORE symbols (stringprep_utf8_to_unichar). +"""]] diff --git a/doc/bugs/More_build_oddities_under_OpenBSD/comment_4_d42106128c3dac2dd7761a82cc03912f._comment b/doc/bugs/More_build_oddities_under_OpenBSD/comment_4_d42106128c3dac2dd7761a82cc03912f._comment new file mode 100644 index 0000000000..9677848d3d --- /dev/null +++ b/doc/bugs/More_build_oddities_under_OpenBSD/comment_4_d42106128c3dac2dd7761a82cc03912f._comment @@ -0,0 +1,9 @@ +[[!comment format=mdwn + username="https://www.google.com/accounts/o8/id?id=AItOawkzwmw_zyMpZC9_J7ey--woeYPoZkAOgGw" + nickname="dxtrish" + subject="comment 4" + date="2014-02-07T22:12:43Z" + content=""" +Couldn't it be possible that something is passed twice on the command line, like that message says could be the reason? Because I have inspected my systems and the only thing I can find that even contains such a string (stringprep_utf8_to_unichar) is libidn so there shouldn't be any conflicts. + +"""]] diff --git a/doc/bugs/More_build_oddities_under_OpenBSD/comment_5_71166beb796f22dcee065a167cd5e0ed._comment b/doc/bugs/More_build_oddities_under_OpenBSD/comment_5_71166beb796f22dcee065a167cd5e0ed._comment new file mode 100644 index 0000000000..a2772452d2 --- /dev/null +++ b/doc/bugs/More_build_oddities_under_OpenBSD/comment_5_71166beb796f22dcee065a167cd5e0ed._comment @@ -0,0 +1,12 @@ +[[!comment format=mdwn + username="https://www.google.com/accounts/o8/id?id=AItOawkzwmw_zyMpZC9_J7ey--woeYPoZkAOgGw" + nickname="dxtrish" + subject="comment 5" + date="2014-02-08T14:46:10Z" + content=""" +Okay, so I did work around this with an ugly hack (I'm not even sure it will work properly) so I now have XMPP support, according to git-annex. + +But.. When I try to play around with XMPP I get: + + setSocketOption: invalid argument (Invalid argument) +"""]] diff --git a/doc/bugs/More_build_oddities_under_OpenBSD/comment_6_65913a2de8bbe981beaa66c58d2429b5._comment b/doc/bugs/More_build_oddities_under_OpenBSD/comment_6_65913a2de8bbe981beaa66c58d2429b5._comment new file mode 100644 index 0000000000..ff76fb3ded --- /dev/null +++ b/doc/bugs/More_build_oddities_under_OpenBSD/comment_6_65913a2de8bbe981beaa66c58d2429b5._comment @@ -0,0 +1,8 @@ +[[!comment format=mdwn + username="https://www.google.com/accounts/o8/id?id=AItOawkzwmw_zyMpZC9_J7ey--woeYPoZkAOgGw" + nickname="dxtrish" + subject="comment 6" + date="2014-02-08T17:23:54Z" + content=""" +Googling around I found [this](http://lpaste.net/77947) codesnippet that suggests setSocketOption is broken under OpenBSD +"""]] diff --git a/doc/bugs/More_build_oddities_under_OpenBSD/comment_7_8dd46cec230125d1410d8e6824aeddf2._comment b/doc/bugs/More_build_oddities_under_OpenBSD/comment_7_8dd46cec230125d1410d8e6824aeddf2._comment new file mode 100644 index 0000000000..72b427357a --- /dev/null +++ b/doc/bugs/More_build_oddities_under_OpenBSD/comment_7_8dd46cec230125d1410d8e6824aeddf2._comment @@ -0,0 +1,12 @@ +[[!comment format=mdwn + username="http://joeyh.name/" + ip="209.250.56.163" + subject="comment 7" + date="2014-02-08T17:42:52Z" + content=""" +What was the ugly hack that got it to link? + +I've seen setSocketOption fail on other OS's for various portability reasons. The haskell library that is responsible for this is , and you can find several setSocketOption calls in it. I've had good luck ifdefing those out when they don't work. + +Here's a patch where I disable the IPv6Only setting on Android (amoung other unrelated porting) +"""]] diff --git a/doc/bugs/More_build_oddities_under_OpenBSD/comment_8_275d3e62cb5667a2d6ddd90db7a40bff._comment b/doc/bugs/More_build_oddities_under_OpenBSD/comment_8_275d3e62cb5667a2d6ddd90db7a40bff._comment new file mode 100644 index 0000000000..f7f7e34291 --- /dev/null +++ b/doc/bugs/More_build_oddities_under_OpenBSD/comment_8_275d3e62cb5667a2d6ddd90db7a40bff._comment @@ -0,0 +1,18 @@ +[[!comment format=mdwn + username="https://www.google.com/accounts/o8/id?id=AItOawkzwmw_zyMpZC9_J7ey--woeYPoZkAOgGw" + nickname="dxtrish" + subject="comment 8" + date="2014-02-08T18:13:33Z" + content=""" +What I did was to temporarily move away (or rename) the offending libs. What I essentially did was this: + + cabal configure + cabal build + sudo mv /usr/local/lib/lib{xml2,gnutls,gsasl,idn}.a /tmp + cabal install + sudo mv /tmp/lib{xml2,gnutls,gsasl,idn}.a /usr/local/lib + +but I've also had to patch network-info. I've contacted the maintainer of that package but I haven't received anything. I'm considering creating an actual fork with my changes but that would almost seem kind of silly as I don't know *ANY* haskell.. But I am looking at the Haskell wiki as I'm typing this so I can see what I'm looking at :). + +Won't break anything by not setting SO_REUSEADDR? I suspect you're setting it for a reason? +"""]] diff --git a/doc/bugs/More_build_oddities_under_OpenBSD/comment_9_ec6a1eb6c7b264c23ec4bbd45465d7d8._comment b/doc/bugs/More_build_oddities_under_OpenBSD/comment_9_ec6a1eb6c7b264c23ec4bbd45465d7d8._comment new file mode 100644 index 0000000000..dee77a69f8 --- /dev/null +++ b/doc/bugs/More_build_oddities_under_OpenBSD/comment_9_ec6a1eb6c7b264c23ec4bbd45465d7d8._comment @@ -0,0 +1,12 @@ +[[!comment format=mdwn + username="http://joeyh.name/" + ip="209.250.56.163" + subject="comment 9" + date="2014-02-08T18:29:41Z" + content=""" +So you moved away libs in /usr/local to expose usable ones in /usr? + +I've had luck before sending the network-info mantainer pull requests: + +git-annex does set ReuseAddr in one place; in Utility/Webapp.hs. The only time that would have a benefit would be when using `git annex webapp --listen=address:port` and starting and restarting the webapp. Normally the webapp chooses a random free port so it shouldn't need that. +"""]] diff --git a/doc/bugs/Move_out_of_shared_repository_as___34__maintenance_user__34___gives_permission_denied_for_files_edited_by_others.mdwn b/doc/bugs/Move_out_of_shared_repository_as___34__maintenance_user__34___gives_permission_denied_for_files_edited_by_others.mdwn new file mode 100644 index 0000000000..f67d73ba43 --- /dev/null +++ b/doc/bugs/Move_out_of_shared_repository_as___34__maintenance_user__34___gives_permission_denied_for_files_edited_by_others.mdwn @@ -0,0 +1,251 @@ +### Please describe the problem. + +For shared repositories git annex refuses to move file out of repository (git annex move --to) if user triggering the move is not the owner of the file. + +### What steps will reproduce the problem? + + * Init two shared repositories and create a file + * Modify that file in repo1 as user u1 + * Move the file to repo2 as "repository maintenance" user u2 + - this gives permission denied error + +Alternatively -- mabye better understandable usecase(repository maintainer moves unused files to backup): + + * Init two shared repositories and create a file + * Modify that file in repo1 as user u1 + * Modify that file in repo1 a second time (regardless which user) -- this makes the version edited by u1 unused + * Display unused files as repository maintenance user u2 + - this gives permission denied error + * Move unused files to repo2 as "repository maintenance" user u2 + - this gives permission denied error + +### What version of git-annex are you using? On what operating system? + +[[!format sh """ +stefan@atom-linux:/tmp/git-annex-test/a$ git --version +git version 2.6.2 +stefan@atom-linux:/tmp/git-annex-test/a$ git annex version +git-annex version: 5.20150610+gitg608172f-1~ndall+1 +build flags: Assistant Webapp Webapp-secure Pairing Testsuite S3 WebDAV Inotify DBus DesktopNotify XMPP DNS Feeds Quvi TDFA +key/value backends: SHA256E SHA1E SHA512E SHA224E SHA384E SKEIN256E SKEIN512E MD5E SHA256 SHA1 SHA512 SHA224 SHA384 SKEIN256 SKEIN512 MD5 WORM URL +remote types: git gcrypt S3 bup directory rsync web bittorrent webdav tahoe glacier ddar hook external +local repository version: 5 +supported repository version: 5 +upgrade supported from repository versions: 0 1 2 4 +stefan@atom-linux:/tmp/git-annex-test/a$ uname -a +Linux atom-linux 3.13.0-66-generic #108-Ubuntu SMP Wed Oct 7 15:21:40 UTC 2015 i686 i686 i686 GNU/Linux +"""]] + +### Please provide any additional information below. + +[[!format sh """ + +# init two shared repositories and create a file "file.txt" + +stefan@atom-linux:/tmp/git-annex-test$ mkdir a +stefan@atom-linux:/tmp/git-annex-test$ mkdir b +stefan@atom-linux:/tmp/git-annex-test$ sudo chown :fileadmin-data * +stefan@atom-linux:/tmp/git-annex-test$ sudo chmod g+s * +stefan@atom-linux:/tmp/git-annex-test$ cd a +stefan@atom-linux:/tmp/git-annex-test/a$ git init --shared . +Initialized empty shared Git repository in /tmp/git-annex-test/a/.git/ +stefan@atom-linux:/tmp/git-annex-test/a$ git config core.sharedrepository group +stefan@atom-linux:/tmp/git-annex-test/a$ echo "test" >> file.txt +stefan@atom-linux:/tmp/git-annex-test/a$ ls -al +total 16 +drwxrwsr-x 3 stefan fileadmin-data 4096 Nov 1 16:41 . +drwxrwxr-x 4 stefan stefan 4096 Nov 1 16:39 .. +-rw-rw-r-- 1 stefan fileadmin-data 5 Nov 1 16:41 file.txt +drwxrwsr-x 7 stefan fileadmin-data 4096 Nov 1 16:40 .git +stefan@atom-linux:/tmp/git-annex-test/a$ git annex init +init ok +(recording state in git...) +stefan@atom-linux:/tmp/git-annex-test/a$ git annex add file.txt +add file.txt ok +(recording state in git...) +stefan@atom-linux:/tmp/git-annex-test/a$ git commit -m "new file" +[master (root-commit) 7aeaa5f] new file + 1 file changed, 1 insertion(+) + create mode 120000 file.txt +stefan@atom-linux:/tmp/git-annex-test/a$ cd .. +stefan@atom-linux:/tmp/git-annex-test$ cd b +stefan@atom-linux:/tmp/git-annex-test/b$ git clone -o a ssh://127.0.0.1/tmp/git-annex-test/a . +Cloning into '.'... +remote: Counting objects: 13, done. +remote: Compressing objects: 100% (9/9), done. +remote: Total 13 (delta 0), reused 0 (delta 0) +Receiving objects: 100% (13/13), done. +Checking connectivity... done. +stefan@atom-linux:/tmp/git-annex-test/b$ git config core.sharedrepository group +stefan@atom-linux:/tmp/git-annex-test/b$ git annex init "B" +init B (merging a/git-annex into git-annex...) +(recording state in git...) +ok +(recording state in git...) +stefan@atom-linux:/tmp/git-annex-test/b$ cd .. +stefan@atom-linux:/tmp/git-annex-test$ cd a +stefan@atom-linux:/tmp/git-annex-test/a$ git remote add b ssh://127.0.0.1/tmp/git-annex-test/b +stefan@atom-linux:/tmp/git-annex-test/a$ git annex sync +commit ok +pull b +remote: Counting objects: 6, done. +remote: Compressing objects: 100% (5/5), done. +remote: Total 6 (delta 0), reused 1 (delta 0) +Unpacking objects: 100% (6/6), done. +From ssh://127.0.0.1/tmp/git-annex-test/b + * [new branch] git-annex -> b/git-annex + * [new branch] master -> b/master +ok +(merging b/git-annex into git-annex...) +push b +Total 0 (delta 0), reused 0 (delta 0) +To ssh://127.0.0.1/tmp/git-annex-test/b + * [new branch] git-annex -> synced/git-annex + * [new branch] master -> synced/master +ok +stefan@atom-linux:/tmp/git-annex-test/a$ git annex move file.txt --to b +move file.txt (checking b...) (to b...) +SHA256E-s5--f2ca1bb6c7e907d06dafe4687e579fce76b37e4e93b7605022da52e6ccc26fd2.txt + 5 100% 0.00kB/s 0:00:00 (xfr#1, to-chk=0/1) +ok +(recording state in git...) + +## ok, everything fine so far - we have a file "file.txt" which was created in a and moved to b successfully. +## Now we go to a different user account and modify "file.txt" +## -------------------------------------- userswitch + +unison@atom-linux:/tmp/git-annex-test/a$ git annex get file.txt +get file.txt (from b...) +SHA256E-s5--f2ca1bb6c7e907d06dafe4687e579fce76b37e4e93b7605022da52e6ccc26fd2.txt + 5 100% 4.88kB/s 0:00:00 (xfr#1, to-chk=0/1) +ok +(recording state in git...) +unison@atom-linux:/tmp/git-annex-test/a$ git annex unlock file.txt +unlock file.txt (copying...) ok +unison@atom-linux:/tmp/git-annex-test/a$ ls -al +total 16 +drwxrwsr-x 3 stefan fileadmin-data 4096 Nov 1 16:48 . +drwxrwxr-x 4 stefan stefan 4096 Nov 1 16:39 .. +-rw-rw-r-- 1 unison fileadmin-data 5 Nov 1 16:47 file.txt +drwxrwsr-x 9 stefan fileadmin-data 4096 Nov 1 16:44 .git +unison@atom-linux:/tmp/git-annex-test/a$ echo "a change by unison user" >> file.txt +unison@atom-linux:/tmp/git-annex-test/a$ git annex add . +add file.txt ok +(recording state in git...) +unison@atom-linux:/tmp/git-annex-test/a$ git commit -m "unison change" +[master 37bd55d] unison change + 1 file changed, 1 insertion(+), 1 deletion(-) + +## now the original user / repository maintainer wants to move the file +## out to b which gives an permission denied error for setFileMode +## git annex does copy it to b, but not remove it from a +## -------------------------------------- userswitch + +stefan@atom-linux:/tmp/git-annex-test/a$ ls -al +total 16 +drwxrwsr-x 3 stefan fileadmin-data 4096 Nov 1 16:48 . +drwxrwxr-x 4 stefan stefan 4096 Nov 1 16:39 .. +lrwxrwxrwx 1 unison fileadmin-data 188 Nov 1 16:48 file.txt -> .git/annex/objects/8x/7w/SHA256E-s29--6aec68aad4745c6eb7babaa5f66fb727e896ec47414edfd4fec8136a1db8484e.tx +t/SHA256E-s29--6aec68aad4745c6eb7babaa5f66fb727e896ec47414edfd4fec8136a1db8484e.txt +drwxrwsr-x 9 stefan fileadmin-data 4096 Nov 1 16:48 .git +stefan@atom-linux:/tmp/git-annex-test/a$ git annex move file.txt --to b +move file.txt (checking b...) (to b...) +SHA256E-s29--6aec68aad4745c6eb7babaa5f66fb727e896ec47414edfd4fec8136a1db8484e.txt + 29 100% 0.00kB/s 0:00:00 (xfr#1, to-chk=0/1) +git-annex: failed to lock content: .git/annex/objects/8x/7w/SHA256E-s29--6aec68aad4745c6eb7babaa5f66fb727e896ec47414edfd4fec8136a1db8484e.txt/SHA256E-s29--6aec68aad4745c6 +eb7babaa5f66fb727e896ec47414edfd4fec8136a1db8484e.txt: setFileMode: permission denied (Operation not permitted) + +# which makes sense - the file is owned by other user: +stefan@atom-linux:/tmp/git-annex-test/a$ ls -al .git/annex/objects/8x/7w/SHA256E-s29--6aec68aad4745c6eb7babaa5f66fb727e896ec47414edfd4fec8136a1db8484e.txt/SHA256E-s29--6 +aec68aad4745c6eb7babaa5f66fb727e896ec47414edfd4fec8136a1db8484e.txt +-r--r--r-- 1 unison fileadmin-data 29 Nov 1 16:48 .git/annex/objects/8x/7w/SHA256E-s29--6aec68aad4745c6eb7babaa5f66fb727e896ec47414edfd4fec8136a1db8484e.txt/SHA256E-s29- +-6aec68aad4745c6eb7babaa5f66fb727e896ec47414edfd4fec8136a1db8484e.txt + +## the last editor of the file can move it, though: +## -------------------------------------- userswitch +unison@atom-linux:/tmp/git-annex-test/a$ git annex move file.txt --to b +move file.txt (checking b...) ok +(recording state in git...) + +## continued - for second usecase... +## edit file two more times as unison user: + +unison@atom-linux:/tmp/git-annex-test/a$ git annex unlock file.txt +unlock file.txt (copying...) ok +unison@atom-linux:/tmp/git-annex-test/a$ echo "another unison modification" >> file.txt +unison@atom-linux:/tmp/git-annex-test/a$ ls +file.txt +unison@atom-linux:/tmp/git-annex-test/a$ git annex add +add file.txt ok +(recording state in git...) +unison@atom-linux:/tmp/git-annex-test/a$ git commit -m "another unison change" +[master df85491] another unison change + 1 file changed, 1 insertion(+), 1 deletion(-) +unison@atom-linux:/tmp/git-annex-test/a$ git annex unlock file.txt +unlock file.txt (copying...) ok +(recording state in git...) +unison@atom-linux:/tmp/git-annex-test/a$ echo "changes once more" >> file.txt +unison@atom-linux:/tmp/git-annex-test/a$ git annex add . +add file.txt ok +(recording state in git...) +unison@atom-linux:/tmp/git-annex-test/a$ git commit -m "1 more" +[master 099a2c7] 1 more + 1 file changed, 1 insertion(+), 1 deletion(-) + +unison@atom-linux:/tmp/git-annex-test/a$ git annex unused +unused . (checking for unused data...) (checking b/master...) (checking master...) + Some annexed data is no longer used by any files: + NUMBER KEY + 1 SHA256E-s29--6aec68aad4745c6eb7babaa5f66fb727e896ec47414edfd4fec8136a1db8484e.txt + 2 SHA256E-s57--dc131ed3874c3b51a6801c43f5d5c0706a9aea58134459eeddf3436626adc89f.txt + (To see where data was previously used, try: git log --stat -S'KEY') + + To remove unwanted data: git-annex dropunused NUMBER + +ok + +## switch to repository maintenance user and list unused: +## -------------------------------------- userswitch + +stefan@atom-linux:/tmp/git-annex-test/a$ git annex unused --debug +unused . (checking for unused data...) [2015-11-01 17:35:09 CET] read: git ["--git-dir=.git","--work-tree=.","--literal-pathspecs","ls-files","--cached","--others","-z"," +--","."] +[2015-11-01 17:35:09 CET] read: git ["--git-dir=.git","--work-tree=.","--literal-pathspecs","symbolic-ref","-q","HEAD"] +[2015-11-01 17:35:09 CET] read: git ["--git-dir=.git","--work-tree=.","--literal-pathspecs","show-ref","--hash","refs/heads/master"] +[2015-11-01 17:35:09 CET] read: git ["--git-dir=.git","--work-tree=.","--literal-pathspecs","show-ref"] +(checking b/master...) [2015-11-01 17:35:09 CET] read: git ["--git-dir=.git","--work-tree=.","--literal-pathspecs","show-ref","--head"] +[2015-11-01 17:35:09 CET] read: git ["--git-dir=.git","--work-tree=.","--literal-pathspecs","show-ref","--head"] +[2015-11-01 17:35:09 CET] read: git ["--git-dir=.git","--work-tree=.","--literal-pathspecs","diff-index","-z","--raw","--no-renames","-l0","refs/remotes/b/master","--"] +[2015-11-01 17:35:09 CET] chat: git ["--git-dir=.git","--work-tree=.","--literal-pathspecs","cat-file","--batch"] +(checking master...) [2015-11-01 17:35:09 CET] read: git ["--git-dir=.git","--work-tree=.","--literal-pathspecs","show-ref","--head"] +[2015-11-01 17:35:09 CET] read: git ["--git-dir=.git","--work-tree=.","--literal-pathspecs","show-ref","--head"] +[2015-11-01 17:35:09 CET] read: git ["--git-dir=.git","--work-tree=.","--literal-pathspecs","diff-index","-z","--raw","--no-renames","-l0","refs/heads/master","--"] + + Some annexed data is no longer used by any files: + NUMBER KEY + 1 SHA256E-s29--6aec68aad4745c6eb7babaa5f66fb727e896ec47414edfd4fec8136a1db8484e.txt + 2 SHA256E-s57--dc131ed3874c3b51a6801c43f5d5c0706a9aea58134459eeddf3436626adc89f.txt + (To see where data was previously used, try: git log --stat -S'KEY') + + To remove unwanted data: git-annex dropunused NUMBER + + +git-annex: .git/annex/unused: openFile: permission denied (Permission denied) +failed +git-annex: unused: 1 failed + + +stefan@atom-linux:/tmp/git-annex-test/a$ git annex move --unused --to b +git-annex: .git/annex/unused: openFile: permission denied (Permission denied) +stefan@atom-linux:/tmp/git-annex-test/a$ git annex move --debug --unused --to b +[2015-11-01 17:38:48 CET] read: git ["--git-dir=.git","--work-tree=.","--literal-pathspecs","show-ref","git-annex"] +[2015-11-01 17:38:48 CET] read: git ["--git-dir=.git","--work-tree=.","--literal-pathspecs","show-ref","--hash","refs/heads/git-annex"] +[2015-11-01 17:38:48 CET] read: git ["--git-dir=.git","--work-tree=.","--literal-pathspecs","log","refs/heads/git-annex..4cdcc1b4550b2737e018d382022f3ce3e1dcb029","-n1"," +--pretty=%H"] +[2015-11-01 17:38:48 CET] chat: git ["--git-dir=.git","--work-tree=.","--literal-pathspecs","cat-file","--batch"] +git-annex: .git/annex/unused: openFile: permission denied (Permission denied) + +# End of transcript or log. +"""]] + +> [[fixed|done]] --[[Joey]] diff --git a/doc/bugs/Move_out_of_shared_repository_as___34__maintenance_user__34___gives_permission_denied_for_files_edited_by_others/comment_1_5c06ab75371237a263c836da45106707._comment b/doc/bugs/Move_out_of_shared_repository_as___34__maintenance_user__34___gives_permission_denied_for_files_edited_by_others/comment_1_5c06ab75371237a263c836da45106707._comment new file mode 100644 index 0000000000..90491e5629 --- /dev/null +++ b/doc/bugs/Move_out_of_shared_repository_as___34__maintenance_user__34___gives_permission_denied_for_files_edited_by_others/comment_1_5c06ab75371237a263c836da45106707._comment @@ -0,0 +1,28 @@ +[[!comment format=mdwn + username="joey" + subject="""comment 1""" + date="2015-11-18T19:35:52Z" + content=""" +More simply stated, user A adds a file, which sets its perms to 444, and +user B can't change those perms to lock the file for removal. + +In sharedRepository mode, the object directory's perms are already +weakened, to eg 775 rather than the default 555, for the same reason; +another user with shared access can't chmod the object directory to allow +writing to it. That just needs to be extended from object directory to +object file to fix this. + +But, that means that the object file will be mode 664, rather than +444, and so git-annex can't prevent accidental direct modifications of the +content of objects when in sharedRepository mode, like it normally does. + +Since that's a belt and suspenders protection, and since the object +directory permissions weakening already lost a similar protection against +accidental deletion of object files, shrug, I guess we'll do that. + +I do feel that sharedRepository mode rarely ever makes sense to use. It's +very fiddly to get the permissions set up right and keep them right, and +there are much better ways to share a centralized repo between users, eg +use gitolite or a dedicated account that's locked down to only let +git/git-annex commands be run. +"""]] diff --git a/doc/bugs/Moved_files_are_not_picked_up_by_the_assistant_on_OS_X.mdwn b/doc/bugs/Moved_files_are_not_picked_up_by_the_assistant_on_OS_X.mdwn new file mode 100644 index 0000000000..0a3815cb96 --- /dev/null +++ b/doc/bugs/Moved_files_are_not_picked_up_by_the_assistant_on_OS_X.mdwn @@ -0,0 +1,24 @@ +### Please describe the problem. + +On indirect repos on OS X, when a symlink is moved asisstant does not pick it up it. Even though assistant syncs after delete/move changes are not committed. git annex status returns ?? for the moved files. when I run git annex add on the files it says it added them but instead they still show up as ??. the only way to solve the problem is to manually restart the daemon which commits them. + +### What steps will reproduce the problem? + +Moving an annex file. + + +### What version of git-annex are you using? On what operating system? + +5.20140703 g3cfcd54 + +### Please provide any additional information below. + +[[!format sh """ +# If you can, paste a complete transcript of the problem occurring here. +# If the problem is with the git-annex assistant, paste in .git/annex/daemon.log + + +# End of transcript or log. +"""]] + +[[!tag moreinfo]] diff --git a/doc/bugs/Moved_files_are_not_picked_up_by_the_assistant_on_OS_X/comment_1_75c14b405929a8f771a7c261dcc4b7a2._comment b/doc/bugs/Moved_files_are_not_picked_up_by_the_assistant_on_OS_X/comment_1_75c14b405929a8f771a7c261dcc4b7a2._comment new file mode 100644 index 0000000000..5346cb9495 --- /dev/null +++ b/doc/bugs/Moved_files_are_not_picked_up_by_the_assistant_on_OS_X/comment_1_75c14b405929a8f771a7c261dcc4b7a2._comment @@ -0,0 +1,42 @@ +[[!comment format=mdwn + username="http://joeyh.name/" + ip="209.250.56.2" + subject="unable to reproduce, please provide transcript" + date="2014-07-11T19:36:37Z" + content=""" +I have tried to reproduce this problem on OSX, but it seems to work fine: + +
    +oberon:xx joeyh$ git annex assistant 
+oberon:xx joeyh$ date > foo
+oberon:xx joeyh$ git log --stat
+commit 93a208c6aa0080e70a636181606f53af5f2c4441
+Author: Joey Hess 
+Date:   Fri Jul 11 15:33:50 2014 -0400
+
+ foo | 1 +
+ 1 file changed, 1 insertion(+)
+oberon:xx joeyh$ mv foo bar
+oberon:xx joeyh$ git log --stat
+commit 495592b2977ee96029db99761d20396577ff3c51
+Author: Joey Hess 
+Date:   Fri Jul 11 15:33:54 2014 -0400
+
+ bar | 1 +
+ foo | 1 -
+ 2 files changed, 1 insertion(+), 1 deletion(-)
+
+commit 93a208c6aa0080e70a636181606f53af5f2c4441
+Author: Joey Hess 
+Date:   Fri Jul 11 15:33:50 2014 -0400
+
+ foo | 1 +
+ 1 file changed, 1 insertion(+)
+oberon:xx joeyh$ git annex status
+oberon:xx joeyh$ 
+
    + +> when I run git annex add on the files it says it added them but instead they still show up as ?? + +This must be the root of whatever the problem is, so please provide a transcript of you doing that. +"""]] diff --git a/doc/bugs/Moved_files_are_not_picked_up_by_the_assistant_on_OS_X/comment_2_f52483415c623ea0649c3805728ce761._comment b/doc/bugs/Moved_files_are_not_picked_up_by_the_assistant_on_OS_X/comment_2_f52483415c623ea0649c3805728ce761._comment new file mode 100644 index 0000000000..9b7f17789a --- /dev/null +++ b/doc/bugs/Moved_files_are_not_picked_up_by_the_assistant_on_OS_X/comment_2_f52483415c623ea0649c3805728ce761._comment @@ -0,0 +1,35 @@ +[[!comment format=mdwn + username="https://me.yahoo.com/a/FHnTlSBo1eCGJRwueeKeB6.RCaPbGMPr5jxx8A--#ce0d8" + nickname="Hamza" + subject="comment 2" + date="2014-07-12T10:00:59Z" + content=""" +[[!format sh \"\"\" +host annex/(master) $ ls -a Staging/Rise* +Staging/Rise and Fall of the Berlin Wall - History Channel.mkv +host annex/(master) $ mv Staging/Rise\ and\ Fall\ of\ the\ Berlin\ Wall\ -\ History\ Channel.mkv Documentary/ +host annex/(master) $ git status +On branch master +Untracked files: + (use \"git add ...\" to include in what will be committed) + + Documentary/Rise and Fall of the Berlin Wall - History Channel.mkv + +nothing added to commit but untracked files present (use \"git add\" to track) +host annex/(master) $ ga status +D Documentary/Rise and Fall of the Berlin Wall - History Channel.mkv +host annex/(master) $ git annex add . +add Documentary/Rise and Fall of the Berlin Wall - History Channel.mkv ok +(Recording state in git...) +host annex/(master) $ ga status +D Documentary/Rise and Fall of the Berlin Wall - History Channel.mkv +host annex/(master) $ git annex sync +commit ok +pull server +ok +host annex/(master) $ ga status +D Documentary/Rise and Fall of the Berlin Wall - History Channel.mkv +host annex/(master) $ +\"\"\"]] + +"""]] diff --git a/doc/bugs/Moved_files_are_not_picked_up_by_the_assistant_on_OS_X/comment_3_fd73fbeef61df106f084ac235fca904a._comment b/doc/bugs/Moved_files_are_not_picked_up_by_the_assistant_on_OS_X/comment_3_fd73fbeef61df106f084ac235fca904a._comment new file mode 100644 index 0000000000..a659ad2c14 --- /dev/null +++ b/doc/bugs/Moved_files_are_not_picked_up_by_the_assistant_on_OS_X/comment_3_fd73fbeef61df106f084ac235fca904a._comment @@ -0,0 +1,8 @@ +[[!comment format=mdwn + username="https://me.yahoo.com/a/FHnTlSBo1eCGJRwueeKeB6.RCaPbGMPr5jxx8A--#ce0d8" + nickname="Hamza" + subject="comment 3" + date="2014-07-12T10:03:07Z" + content=""" +BTW the mv operation is picked up by the assistant it start syncing immediately after mv but the symlink link is left dangling. I've also waited for the assistant to sync before readding the file. +"""]] diff --git a/doc/bugs/Moved_files_are_not_picked_up_by_the_assistant_on_OS_X/comment_4_c5e9843a956984efd22bad629930f6bd._comment b/doc/bugs/Moved_files_are_not_picked_up_by_the_assistant_on_OS_X/comment_4_c5e9843a956984efd22bad629930f6bd._comment new file mode 100644 index 0000000000..a437142af2 --- /dev/null +++ b/doc/bugs/Moved_files_are_not_picked_up_by_the_assistant_on_OS_X/comment_4_c5e9843a956984efd22bad629930f6bd._comment @@ -0,0 +1,8 @@ +[[!comment format=mdwn + username="http://joeyh.name/" + ip="209.250.56.2" + subject="comment 4" + date="2014-07-14T20:15:07Z" + content=""" +Well, I don't see the \"??\" that you were talking about before in your transcript. I do not understand why the file's status shows as \"D\" which means it's deleted. What does `git ls-files` say? Does this stange behavior persist if you stop the assistant from running and re-run the same git-annex commands? +"""]] diff --git a/doc/bugs/Nearline_bucket_stopped_working___40__can__39__t_even_HEAD_files__41__.mdwn b/doc/bugs/Nearline_bucket_stopped_working___40__can__39__t_even_HEAD_files__41__.mdwn new file mode 100644 index 0000000000..1cc8e096b1 --- /dev/null +++ b/doc/bugs/Nearline_bucket_stopped_working___40__can__39__t_even_HEAD_files__41__.mdwn @@ -0,0 +1,69 @@ +### Please describe the problem. + +I added a google-cloud nearline special remote. It worked great for about two weeks. But now I cannot do any operations on the bucket; they all fail with 400s. + +### What steps will reproduce the problem? + +(I've XXX'ed out things that look like they might contain secrets.) + + $ AWS_ACCESS_KEY_ID=GOOGXXX AWS_SECRET_ACCESS_KEY=XXX git annex copy --to goog-cloud-nearline2 photos/raw-photos/2007/Birds\ at\ GovCo/ --debug --json --verbose + [2016-11-02 10:09:43.617264] read: git ["--git-dir=.git","--work-tree=.","--literal-pathspecs","show-ref","git-annex"] + [2016-11-02 10:09:43.628311] process done ExitSuccess + [2016-11-02 10:09:43.62842] read: git ["--git-dir=.git","--work-tree=.","--literal-pathspecs","show-ref","--hash","refs/heads/git-annex"] + [2016-11-02 10:09:43.638154] process done ExitSuccess + [2016-11-02 10:09:43.639472] chat: git ["--git-dir=.git","--work-tree=.","--literal-pathspecs","cat-file","--batch"] + [2016-11-02 10:09:43.670284] read: git ["config","--null","--list"] + [2016-11-02 10:09:43.684014] read: git ["--git-dir=.git","--work-tree=.","--literal-pathspecs","ls-files","--cached","-z","--","photos/raw-photos/2007/Birds at GovCo/"] + copy photos/raw-photos/2007/Birds at GovCo/IMG_4186.dng (checking goog-cloud-nearline2...) [2016-11-02 10:09:43.845838] String to sign: "HEAD\n\n\nWed, 2 Nov 2016 17:09:43 GMT\n/jlebar-annex-nearline2/GPGHMACSHA1--07b92ba4ec937b88911ce8128974ba449382113d" + [2016-11-02 10:09:43.845906] Host: "jlebar-annex-nearline2.storage.googleapis.com" + [2016-11-02 10:09:43.845938] Path: "/GPGHMACSHA1--07b92ba4ec937b88911ce8128974ba449382113d" + [2016-11-02 10:09:43.845966] Query string: "" + [2016-11-02 10:09:43.954556] Response status: Status {statusCode = 400, statusMessage = "Bad Request"} + [2016-11-02 10:09:43.954689] Response header 'X-GUploader-UploadID': 'XXX' + [2016-11-02 10:09:43.954818] Response header 'Content-Type': 'application/xml; charset=UTF-8' + [2016-11-02 10:09:43.954893] Response header 'Content-Length': '178' + [2016-11-02 10:09:43.95498] Response header 'Date': 'Wed, 02 Nov 2016 17:09:41 GMT' + [2016-11-02 10:09:43.955017] Response header 'Expires': 'Wed, 02 Nov 2016 17:09:41 GMT' + [2016-11-02 10:09:43.955047] Response header 'Cache-Control': 'private, max-age=0' + [2016-11-02 10:09:43.955143] Response header 'Server': 'UploadServer' + [2016-11-02 10:09:43.95518] Response metadata: S3: request ID=, x-amz-id-2= + (StatusCodeException (Status {statusCode = 400, statusMessage = "Bad Request"}) [("X-GUploader-UploadID","XXX"),("Content-Type","application/xml; charset=UTF-8"),("Content-Length","178"),("Date","Wed, 02 Nov 2016 17:09:41 GMT"),("Expires","Wed, 02 Nov 2016 17:09:41 GMT"),("Cache-Control","private, max-age=0"),("Server","UploadServer")] (CJ {expose = []})) failed + git-annex: copy: 1 failed + CallStack (from HasCallStack): + error, called at ./CmdLine/Action.hs:41:28 in main:CmdLine.Action + +### What version of git-annex are you using? On what operating system? + + $ git annex version + git-annex version: 6.20160923 + build flags: Assistant Webapp Pairing Testsuite S3(multipartupload)(storageclasses) WebDAV FsEvents XMPP ConcurrentOutput TorrentParser MagicMime Feeds Quvi + key/value backends: SHA256E SHA256 SHA512E SHA512 SHA224E SHA224 SHA384E SHA384 SHA3_256E SHA3_256 SHA3_512E SHA3_512 SHA3_224E SHA3_224 SHA3_384E SHA3_384 SKEIN256E SKEIN256 SKEIN512E SKEIN512 SHA1E SHA1 MD5E MD5 WORM URL + remote types: git gcrypt S3 bup directory rsync web bittorrent webdav tahoe glacier ddar hook external + local repository version: 6 + supported repository versions: 5 6 + upgrade supported from repository versions: 0 1 2 3 4 5 + operating system: darwin x86_64 + +### Please provide any additional information below. + +As I read their docs, GCS should be sending back a more helpful error message +in the body of the response: +https://cloud.google.com/storage/docs/xml-api/reference-status. I tried to +figure out how to get git annex to print that out, but...did you know git annex +is a nontrivial Haskell program? :) + +If we could get git annex to print out the body of the response and there is in +fact an error message in there, I expect the reason for this error would be +obvious. Or if we could get git annex to do a "dry run" and print the HTTP +requeset it *would have* made, I could probably run it in curl and get the +response that way. + +(I actually considered setting up wireshark to capture the response, but I'd +have to set up an HTTPS MITM...) + +I cannot figure out what made this stop working all of a sudden. I don't think it was a git annex version update, and I checked remotes.log, and this remote's metadata hasn't changed. (The line for this remote has been updated a few times, but afaict only the timestamp changed.) + +### Have you had any luck using git-annex before? (Sometimes we get tired of reading bug reports all day and a lil' positive end note does wonders) + +It was working so great up until it wasn't -- I was very happy. Thank you +for all your work on this software; there is nothing like it. diff --git a/doc/bugs/Nearline_bucket_stopped_working___40__can__39__t_even_HEAD_files__41__/comment_1_2a1c20b432dd839e0153827e151fbbc0._comment b/doc/bugs/Nearline_bucket_stopped_working___40__can__39__t_even_HEAD_files__41__/comment_1_2a1c20b432dd839e0153827e151fbbc0._comment new file mode 100644 index 0000000000..ed426f3e30 --- /dev/null +++ b/doc/bugs/Nearline_bucket_stopped_working___40__can__39__t_even_HEAD_files__41__/comment_1_2a1c20b432dd839e0153827e151fbbc0._comment @@ -0,0 +1,17 @@ +[[!comment format=mdwn + username="joey" + subject="""comment 1""" + date="2016-11-07T17:34:32Z" + content=""" +If you pass the --debug option, the debug output includes a transcript of +the S3 http session. Edit: I see you did that, so you can in fact see +the query and response in the debug output. + +I can confirm there have been no recent changes to git-annex's S3 support. +Of course it's possible that if you upgraded git-annex the version of + used changed, but that seems +unlikely to cause such breakage. + +Seems likely that if it's stopped working, something changed on Google's +end.. +"""]] diff --git a/doc/bugs/Nearline_bucket_stopped_working___40__can__39__t_even_HEAD_files__41__/comment_2_c227071f23a96ed9928f128e7f77e503._comment b/doc/bugs/Nearline_bucket_stopped_working___40__can__39__t_even_HEAD_files__41__/comment_2_c227071f23a96ed9928f128e7f77e503._comment new file mode 100644 index 0000000000..820e6b0401 --- /dev/null +++ b/doc/bugs/Nearline_bucket_stopped_working___40__can__39__t_even_HEAD_files__41__/comment_2_c227071f23a96ed9928f128e7f77e503._comment @@ -0,0 +1,17 @@ +[[!comment format=mdwn + username="justin.lebar@7a36fcafc322d9a381e89f08ab6289033c6dde91" + nickname="justin.lebar" + avatar="http://cdn.libravatar.org/avatar/9fca4b61a1ab555f231851e7543f9a3e" + subject="comment 2" + date="2016-11-20T03:47:23Z" + content=""" +Thanks for your reply, Joey. Sorry for the delay getting back to this -- I didn't realize I hadn't enabled notifications on the thread. + +The GCS docs suggest that 400 errors should be accompanied by an explanation in the reply body. + +> Error responses usually include a JSON document in the response body, which contains information about the error. + +https://cloud.google.com/storage/docs/json_api/v1/status-codes + +Do you think we're not getting an http response body here, or that it's not being printed out? +"""]] diff --git a/doc/bugs/Nearline_bucket_stopped_working___40__can__39__t_even_HEAD_files__41__/comment_3_5ac676877feaa7cdb9e05d6b71b1a4c3._comment b/doc/bugs/Nearline_bucket_stopped_working___40__can__39__t_even_HEAD_files__41__/comment_3_5ac676877feaa7cdb9e05d6b71b1a4c3._comment new file mode 100644 index 0000000000..67b215bc72 --- /dev/null +++ b/doc/bugs/Nearline_bucket_stopped_working___40__can__39__t_even_HEAD_files__41__/comment_3_5ac676877feaa7cdb9e05d6b71b1a4c3._comment @@ -0,0 +1,11 @@ +[[!comment format=mdwn + username="justin.lebar@7a36fcafc322d9a381e89f08ab6289033c6dde91" + nickname="justin.lebar" + avatar="http://cdn.libravatar.org/avatar/9fca4b61a1ab555f231851e7543f9a3e" + subject="comment 3" + date="2016-12-04T04:30:38Z" + content=""" +For a while things were working, but now it's not working again, same problem as before. + +Do you think maybe it's a timestamp bug in the signature or something? That could explain this \"mysteriously works then stops working\" behavior. +"""]] diff --git a/doc/bugs/Nearline_bucket_stopped_working___40__can__39__t_even_HEAD_files__41__/comment_4_3853bebdfcb7dac647edefd6f55ba8d5._comment b/doc/bugs/Nearline_bucket_stopped_working___40__can__39__t_even_HEAD_files__41__/comment_4_3853bebdfcb7dac647edefd6f55ba8d5._comment new file mode 100644 index 0000000000..9b3bc55354 --- /dev/null +++ b/doc/bugs/Nearline_bucket_stopped_working___40__can__39__t_even_HEAD_files__41__/comment_4_3853bebdfcb7dac647edefd6f55ba8d5._comment @@ -0,0 +1,20 @@ +[[!comment format=mdwn + username="joey" + subject="""comment 4""" + date="2017-02-20T17:00:55Z" + content=""" +I'd expect that the respone body is indeed not shown by --debug. + +A timestamp bug seems as good a theory as anything. In either the S3 +library git-annex is using, or the S3 emulation Google Cloud Storage is +using. Or perhaps google has different versions of the servers that behave +differently. + +Over on [[tips/using_Google_Cloud_Storage]], @Dosenpfand +reports what looks like the same 400 when initializing a nearline remote. + +Since this is using a S3 interoperability layer in Google Cloud Storage, +I'd not be surprised if there were bugs or limitations in it. +Using the native GCS API seems like an increasingly good idea, +and implements that. +"""]] diff --git a/doc/bugs/Non-annexed_files_being_annexed_and_not_stored.mdwn b/doc/bugs/Non-annexed_files_being_annexed_and_not_stored.mdwn new file mode 100644 index 0000000000..e05a6181d7 --- /dev/null +++ b/doc/bugs/Non-annexed_files_being_annexed_and_not_stored.mdwn @@ -0,0 +1,18 @@ +### Please describe the problem. +Somehow in my repo, perhaps since I started using the webapp, some of the files which have been commited plain into git have become annexed and turned into broken symlinks. However, they are not marked as stored in any repositories, so it is difficult to recover them (they need to be checked out again from git history). + +### What steps will reproduce the problem? +I'm not sure how this happened. + +### What version of git-annex are you using? On what operating system? +I've been using builds of git annex between february and april of 2016, on linux. + +### Please provide any additional information below. + +It looks like the problem occurred on Sun Apr 10 00:49:27 2016 -0400. I'm mostly recording here that this problem happened, as a lot is on my backlog atm, but if somebody wanted to let me know what logs to investigate, I plan to still be using this repository into the future. + +It looks like the system which caused this problem had a looser largefiles setting than the system which added the files to git. It looks like it was using the 6.20160229-g37a89cc build. + +### Have you had any luck using git-annex before? (Sometimes we get tired of reading bug reports all day and a lil' positive end note does wonders) + +Git-annex rocks, I am a monthly flatr'r. diff --git a/doc/bugs/Non-annexed_files_being_annexed_and_not_stored/comment_1_ef6c2dee191e80a46e348c78ba054fc2._comment b/doc/bugs/Non-annexed_files_being_annexed_and_not_stored/comment_1_ef6c2dee191e80a46e348c78ba054fc2._comment new file mode 100644 index 0000000000..7db4ea2233 --- /dev/null +++ b/doc/bugs/Non-annexed_files_being_annexed_and_not_stored/comment_1_ef6c2dee191e80a46e348c78ba054fc2._comment @@ -0,0 +1,19 @@ +[[!comment format=mdwn + username="joey" + subject="""comment 1""" + date="2016-04-20T18:09:37Z" + content=""" +Seems that there are two potential bugs here; a file that was stored in git +somehow got stored in the annex instead, and something seems to have +happened to the content of the file in the annex. + +Try using `git annex log` to see where the content of the file was +stored in the past and when the content got marked as no longer being +present. + +A looser annex.largefiles setting could result in a modified file +that had been stored in git being stored in the annex instead. +Look at the symlink that was checked into git for the annexed file +and compare that with the sha256sum of the file that you rescued +from the git history. Are they the same or different? +"""]] diff --git a/doc/bugs/Non-annexed_files_being_annexed_and_not_stored/comment_2_6d73b11c55e7b2ed4003e1908558426d._comment b/doc/bugs/Non-annexed_files_being_annexed_and_not_stored/comment_2_6d73b11c55e7b2ed4003e1908558426d._comment new file mode 100644 index 0000000000..a71e3809c6 --- /dev/null +++ b/doc/bugs/Non-annexed_files_being_annexed_and_not_stored/comment_2_6d73b11c55e7b2ed4003e1908558426d._comment @@ -0,0 +1,10 @@ +[[!comment format=mdwn + username="xloem" + subject="comment 2" + date="2016-04-25T21:25:58Z" + content=""" +The checksum of a file is the same as the generated symlink name. +For a file which had the issue, `git annex log` appears empty. +I used `git checkout [revision] [filename]` to get the old version of the file in the newly versioned repo, not sure if that would confuse `git annex log`. + +"""]] diff --git a/doc/bugs/OSX_Assistant_will_not_automatically_drop.mdwn b/doc/bugs/OSX_Assistant_will_not_automatically_drop.mdwn new file mode 100644 index 0000000000..7e43f089fb --- /dev/null +++ b/doc/bugs/OSX_Assistant_will_not_automatically_drop.mdwn @@ -0,0 +1,172 @@ +### Please describe the problem. +I cannot reproduce the archive behaviour explained in the screencast: git annex assistant archiving. + +### What steps will reproduce the problem? +Install the OSX dmg. Open the webapp. Make a client repo `~/annex` on the laptop and a full archive repo `/Volumes/Seagate/annex` on an external hard drive. Disregard the errors in the webapp. Restart daemon. Finish configuration of the archive repo as full archive. +Put a test file in `~/annex`, watch it copy over in the webapp. Verify copies with `git-annex whereis`. +**Use the GUI to move the test file to `~/annex/archive`. The file does not turn into a symlink as expected.** +Run the command `git-annex drop --auto`. the archived file is now replaced by a symlink. `git annex whereis` verifies that the only copy is in the archive repo. +Now move the symlink back to `~/annex`. The symlink is not replaced by the file. +Restart the assistant using the webapp. The symlink is replaced by the file. +Repeat above steps with the same effect. + +### What version of git-annex are you using? On what operating system? +OSX 10.11.6 El Capitan +git-annex version: 6.20180409-gfb0780266 +build flags: Assistant Webapp Pairing S3(multipartupload)(storageclasses) WebDAV FsEvents ConcurrentOutput TorrentParser MagicMime Feeds Testsuite +dependency versions: aws-0.17.1 bloomfilter-2.0.1.0 cryptonite-0.23 DAV-1.3.1 feed-0.3.12.0 ghc-8.0.2 http-client-0.5.7.0 persistent-sqlite-2.6.2 torrent-10000.1.1 uuid-1.3.13 yesod-1.4.5 +key/value backends: SHA256E SHA256 SHA512E SHA512 SHA224E SHA224 SHA384E SHA384 SHA3_256E SHA3_256 SHA3_512E SHA3_512 SHA3_224E SHA3_224 SHA3_384E SHA3_384 SKEIN256E SKEIN256 SKEIN512E SKEIN512 BLAKE2B256E BLAKE2B256 BLAKE2B512E BLAKE2B512 BLAKE2B160E BLAKE2B160 BLAKE2B224E BLAKE2B224 BLAKE2B384E BLAKE2B384 BLAKE2S256E BLAKE2S256 BLAKE2S160E BLAKE2S160 BLAKE2S224E BLAKE2S224 BLAKE2SP256E BLAKE2SP256 BLAKE2SP224E BLAKE2SP224 SHA1E SHA1 MD5E MD5 WORM URL +remote types: git gcrypt p2p S3 bup directory rsync web bittorrent webdav adb tahoe glacier ddar hook external + +Version 6.20180426 also produces the same result, however the webapp does not launch a browser window. + + +### Please provide any additional information below. + +[[!format sh """ +# If you can, paste a complete transcript of the problem occurring here. +# If the problem is with the git-annex assistant, paste in .git/annex/daemon.log + +[2018-04-29 22:12:25.754703] main: starting assistant version 6.20180409-gfb0780266 +[2018-04-29 22:12:25.780218] Cronner: You should enable consistency checking to protect your data. +(scanning...) [2018-04-29 22:12:25.901493] Watcher: Performing startup scan +(started...) +gpg: assuming signed data in '/var/folders/vt/jdjpydy51cb4c9743nvk2s7m0000gp/T/git-annex.tmpvDEpq7/info' +gpg: Signature made Fri Apr 27 20:03:18 2018 EAT using DSA key ID 89C809CB +gpg: /var/folders/vt/jdjpydy51cb4c9743nvk2s7m0000gp/T/git-annex-gpg.tmpBe1DxC/trustdb.gpg: trustdb created +gpg: Good signature from "git-annex distribution signing key (for Joey Hess) " [unknown] +gpg: WARNING: This key is not certified with a trusted signature! +gpg: There is no indication that the signature belongs to the owner. +Primary key fingerprint: 4005 5C6A FD2D 526B 2961 E78F 5EE1 DBA7 89C8 09CB +[2018-04-29 22:12:28.884998] Upgrader: An upgrade of git-annex is available. (version 6.20180426) +(recording state in git...) +29/Apr/2018:22:13:13 +0300 [Error#yesod-core] there is no available git remote named "SeagateBlack" @(yesod-core-1.4.37-GCI7RasEcSMIU2vku0fHJ1:Yesod.Core.Class.Yesod ./Yesod/Core/Class/Yesod.hs:693:5) +fatal: remote SeagateBlack already exists. +29/Apr/2018:22:13:21 +0300 [Error#yesod-core] there is no available git remote named "SeagateBlack" @(yesod-core-1.4.37-GCI7RasEcSMIU2vku0fHJ1:Yesod.Core.Class.Yesod ./Yesod/Core/Class/Yesod.hs:693:5) +fatal: remote SeagateBlack already exists. +29/Apr/2018:22:13:24 +0300 [Error#yesod-core] there is no available git remote named "SeagateBlack" @(yesod-core-1.4.37-GCI7RasEcSMIU2vku0fHJ1:Yesod.Core.Class.Yesod ./Yesod/Core/Class/Yesod.hs:693:5) + +##restarted daemon via webapp## + +[2018-04-29 22:13:33.426914] main: starting assistant version 6.20180409-gfb0780266 +[2018-04-29 22:13:33.474857] Cronner: You should enable consistency checking to protect your data. +[2018-04-29 22:13:33.494878] TransferScanner: Syncing with SeagateBlack +(scanning...) [2018-04-29 22:13:33.646686] Watcher: Performing startup scan +(started...) +warning: no common commits +From /Volumes/SeagateBlack/annex + * [new branch] git-annex -> SeagateBlack/git-annex +(merging SeagateBlack/git-annex into git-annex...) +(recording state in git...) +To /Volumes/SeagateBlack/annex + * [new branch] git-annex -> synced/git-annex + * [new branch] annex/direct/master -> synced/master +gpg: assuming signed data in '/var/folders/vt/jdjpydy51cb4c9743nvk2s7m0000gp/T/git-annex.tmpIrPVZb/info' +gpg: Signature made Fri Apr 27 20:03:18 2018 EAT using DSA key ID 89C809CB +gpg: /var/folders/vt/jdjpydy51cb4c9743nvk2s7m0000gp/T/git-annex-gpg.tmptALBpe/trustdb.gpg: trustdb created +gpg: Good signature from "git-annex distribution signing key (for Joey Hess) " [unknown] +gpg: WARNING: This key is not certified with a trusted signature! +gpg: There is no indication that the signature belongs to the owner. +Primary key fingerprint: 4005 5C6A FD2D 526B 2961 E78F 5EE1 DBA7 89C8 09CB +[2018-04-29 22:13:38.218204] Upgrader: An upgrade of git-annex is available. (version 6.20180426) +[2018-04-29 22:14:33.641316] Pusher: Syncing with SeagateBlack +Everything up-to-date + +##restarted daemon via webapp## + +[2018-04-29 22:14:04.840648] main: starting assistant version 6.20180409-gfb0780266 +[2018-04-29 22:14:04.914622] Cronner: You should enable consistency checking to protect your data. +[2018-04-29 22:14:04.938372] TransferScanner: Syncing with SeagateBlack +(scanning...) [2018-04-29 22:14:05.129225] Watcher: Performing startup scan +(recording state in git...) +(started...) +To /Volumes/SeagateBlack/annex + 96eb102..56b72d6 git-annex -> synced/git-annex +gpg: assuming signed data in '/var/folders/vt/jdjpydy51cb4c9743nvk2s7m0000gp/T/git-annex.tmp6iw9TV/info' +gpg: Signature made Fri Apr 27 20:03:18 2018 EAT using DSA key ID 89C809CB +gpg: /var/folders/vt/jdjpydy51cb4c9743nvk2s7m0000gp/T/git-annex-gpg.tmpFugMM8/trustdb.gpg: trustdb created +gpg: Good signature from "git-annex distribution signing key (for Joey Hess) " [unknown] +gpg: WARNING: This key is not certified with a trusted signature! +gpg: There is no indication that the signature belongs to the owner. +Primary key fingerprint: 4005 5C6A FD2D 526B 2961 E78F 5EE1 DBA7 89C8 09CB +[2018-04-29 22:14:13.443848] Upgrader: An upgrade of git-annex is available. (version 6.20180426) +[2018-04-29 22:14:29.453797] Committer: Adding testing.txt +add /Users/jhnichol/Desktop/annex/testing.txt ok +[2018-04-29 22:14:29.458807] Committer: Committing changes to git +(recording state in git...) +[2018-04-29 22:14:29.508458] Pusher: Syncing with SeagateBlack +(recording state in git...) + +testing.txt + 13 100% 0.00kB/s 0:00:00 + 13 100% 0.00kB/s 0:00:0(checksum...) 0 (xfer#1, to-check=0/1) + +sent 105 bytes received 42 bytes 294.00 bytes/sec +total size is 13 speedup is 0.09 +[2018-04-29 22:14:29.908335] Transferrer: Uploaded testing.txt +To /Volumes/SeagateBlack/annex + 56b72d6..97076de git-annex -> synced/git-annex + f981f0e..903f744 annex/direct/master -> synced/master +[2018-04-29 22:14:31.993469] Pusher: Syncing with SeagateBlack +(recording state in git...) +To /Volumes/SeagateBlack/annex + 97076de..351ae72 git-annex -> synced/git-annex +[2018-04-29 22:14:37.442779] Committer: Adding testing.txt +add /Users/jhnichol/Desktop/annex/archive/testing.txt ok +[2018-04-29 22:14:37.451934] Committer: Committing changes to git +(recording state in git...) +[2018-04-29 22:14:37.506885] Pusher: Syncing with SeagateBlack +To /Volumes/SeagateBlack/annex + 903f744..0ace0a4 annex/direct/master -> synced/master +[2018-04-29 22:15:05.080814] Pusher: Syncing with SeagateBlack +To /Volumes/SeagateBlack/annex + 351ae72..acc6f66 git-annex -> synced/git-annex +[2018-04-29 22:15:07.643433] Committer: Committing changes to git +(recording state in git...) +[2018-04-29 22:15:07.716942] Pusher: Syncing with SeagateBlack +To /Volumes/SeagateBlack/annex + 0ace0a4..216adc1 annex/direct/master -> synced/master + +##restarted daemon via webapp## + +[2018-04-29 22:15:12.721042] main: starting assistant version 6.20180409-gfb0780266 +[2018-04-29 22:15:12.780184] Cronner: You should enable consistency checking to protect your data. +[2018-04-29 22:15:12.802662] TransferScanner: Syncing with SeagateBlack +(scanning...) [2018-04-29 22:15:12.995192] Watcher: Performing startup scan +Everything up-to-date +(started...) +[2018-04-29 22:15:13.807031] Committer: Committing changes to git +(recording state in git...) + +SHA256E-s13--091dfb50686a31a1137099832271eb582fbb745aeca5c0c8153ffc9acc7cd0fe.txt + 13 100% 0.00kB/s 0:00:00 + 13 100% 0.00kB/s 0:00:00 (xfer#1, to-check=0/1) + +sent 175 bytes received 42 bytes 434.00 bytes/sec +total size is 13 speedup is 0.06 +(checksum...) [2018-04-29 22:15:13.980745] Transferrer: Downloaded testing.txt +[2018-04-29 22:15:14.805402] Pusher: Syncing with SeagateBlack +(recording state in git...) +To /Volumes/SeagateBlack/annex + acc6f66..f62818b git-annex -> synced/git-annex + 216adc1..bebd3d1 annex/direct/master -> synced/master +[2018-04-29 22:15:15.384765] Committer: Adding testing.txt +add /Users/jhnichol/Desktop/annex/testing.txt ok +[2018-04-29 22:15:15.447495] Committer: Committing changes to git +(recording state in git...) +gpg: assuming signed data in '/var/folders/vt/jdjpydy51cb4c9743nvk2s7m0000gp/T/git-annex.tmpxY52C5/info' +gpg: Signature made Fri Apr 27 20:03:18 2018 EAT using DSA key ID 89C809CB +gpg: /var/folders/vt/jdjpydy51cb4c9743nvk2s7m0000gp/T/git-annex-gpg.tmpqPe64c/trustdb.gpg: trustdb created +gpg: Good signature from "git-annex distribution signing key (for Joey Hess) " [unknown] +gpg: WARNING: This key is not certified with a trusted signature! +gpg: There is no indication that the signature belongs to the owner. +Primary key fingerprint: 4005 5C6A FD2D 526B 2961 E78F 5EE1 DBA7 89C8 09CB +[2018-04-29 22:15:15.908187] Upgrader: An upgrade of git-annex is available. (version 6.20180426) +[2018-04-29 22:15:16.983499] Pusher: Syncing with SeagateBlack +Everything up-to-date + +# End of transcript or log. +"""]] + +### Have you had any luck using git-annex before? (Sometimes we get tired of reading bug reports all day and a lil' positive end note does wonders) +I have not used it before, but I'm determined to make it work! I'm nomadic and archive my pictures/videos/files and therefore this software must be perfect for me! diff --git a/doc/bugs/OSX_Assistant_will_not_automatically_drop/comment_1_9a3a594f983048a9035f385101b394cf._comment b/doc/bugs/OSX_Assistant_will_not_automatically_drop/comment_1_9a3a594f983048a9035f385101b394cf._comment new file mode 100644 index 0000000000..2f32bb5225 --- /dev/null +++ b/doc/bugs/OSX_Assistant_will_not_automatically_drop/comment_1_9a3a594f983048a9035f385101b394cf._comment @@ -0,0 +1,20 @@ +[[!comment format=mdwn + username="andrew" + avatar="http://cdn.libravatar.org/avatar/acc0ece1eedf07dd9631e7d7d343c435" + subject="comment 1" + date="2018-05-04T15:58:59Z" + content=""" +Hmmm. I am testing on macOS 10.12.6 with git-annex 6.20180409-gfb0780266 installed from dmg in /Applications + +The webapp launch behavior seems a bit odd. If I launch from the command-line `git-annex webapp` from my `/Users/Shared/a1` directory, then a webpage loads (for me in Chrome) and no assistant daemons are started, or at least I can't find any with `ps aux`. If I add files to `/Users/Shared/a1` the assistant sees them, but it doesn't transfer to my backup repo at `/Users/Shared/a3`. It seems to only see files from the repo I started git-annex webapp from. Is this the intended behavior Joey? + +I have three repos listed at `~/.config/git-annex/autostart`. If I run `git-annex assistant --autostart` from one of my repo directories, then an assistant daemon is started for each repo, I can verify that with `ps aux | grep \"git-annex assistant --startdelay\"` which lists one assistant process per repository (3 total). If I start the assistants this way, they seem to stick around. Then, I can follow the mentioned archiving video and successfully sync/archive as shown. + +I have these errors in `system.log`: `bumblebee git-annex[766]: no path for address 0x107d40000`, I don't have anything interesting in any of my daemon.log files. I can't find any other interesting system logs. + +—Andrew + + + + +"""]] diff --git a/doc/bugs/OSX_Assistant_will_not_automatically_drop/comment_2_7901e1211ab86429ef8bd95c72ed89f5._comment b/doc/bugs/OSX_Assistant_will_not_automatically_drop/comment_2_7901e1211ab86429ef8bd95c72ed89f5._comment new file mode 100644 index 0000000000..057e3c6b19 --- /dev/null +++ b/doc/bugs/OSX_Assistant_will_not_automatically_drop/comment_2_7901e1211ab86429ef8bd95c72ed89f5._comment @@ -0,0 +1,10 @@ +[[!comment format=mdwn + username="andrew" + avatar="http://cdn.libravatar.org/avatar/acc0ece1eedf07dd9631e7d7d343c435" + subject="thoughts?" + date="2018-05-20T17:41:28Z" + content=""" +Any thoughts Joey? I am happy to troubleshoot this further, but I am not really sure what to test. I do have git-annex building from source on a mac, but I don't have any experience with Haskell, so don't really know what I could be checking. + +Thanks. Andrew +"""]] diff --git a/doc/bugs/OSX_Assistant_will_not_automatically_drop/comment_3_1ba89b95a028ba4a84b22dfd983049fd._comment b/doc/bugs/OSX_Assistant_will_not_automatically_drop/comment_3_1ba89b95a028ba4a84b22dfd983049fd._comment new file mode 100644 index 0000000000..21b93d0419 --- /dev/null +++ b/doc/bugs/OSX_Assistant_will_not_automatically_drop/comment_3_1ba89b95a028ba4a84b22dfd983049fd._comment @@ -0,0 +1,16 @@ +[[!comment format=mdwn + username="jhnichol@cce81d2a480707652a3340ea2f24b3dc4b1f808c" + nickname="jhnichol" + avatar="http://cdn.libravatar.org/avatar/2d05fd7e681bf4838bba8bab538ac65d" + subject="comment 3" + date="2018-05-24T19:28:27Z" + content=""" +> Andrew: +> \" I have three repos listed at `~/.config/git-annex/autostart`. If I run `git-annex assistant --autostart` from one of my repo directories, then an assistant daemon is started for each repo, I can verify that with `ps aux | grep \"git-annex assistant --startdelay\"` which lists one assistant process per repository (3 total). If I start the assistants this way, they seem to stick around. Then, I can follow the mentioned archiving video and successfully sync/archive as shown. \" + +I have only one repo listed at `~/.config/git-annex/autostart` which is my laptop. The other repo was created with the webapp on a USB drive; it is bare and will not allow me to start an assistant on it. Again, I cannot successfully sync/archive as shown in the archiving video. + +I tried using `git-annex version: 6.20180509-gd801a3318` but no help. + +FYI, I also noticed that clicking the .app now successfully opens the webapp again. +"""]] diff --git a/doc/bugs/OSX_Assistant_will_not_automatically_drop/comment_4_704ef9c0ed78a00030f7d4d151ecbe5b._comment b/doc/bugs/OSX_Assistant_will_not_automatically_drop/comment_4_704ef9c0ed78a00030f7d4d151ecbe5b._comment new file mode 100644 index 0000000000..5e4ff10197 --- /dev/null +++ b/doc/bugs/OSX_Assistant_will_not_automatically_drop/comment_4_704ef9c0ed78a00030f7d4d151ecbe5b._comment @@ -0,0 +1,12 @@ +[[!comment format=mdwn + username="andrew" + avatar="http://cdn.libravatar.org/avatar/acc0ece1eedf07dd9631e7d7d343c435" + subject="comment 4" + date="2018-05-30T15:24:47Z" + content=""" +So [bare repositories](https://git-annex.branchable.com/bare_repositories/) should work fine with Assistant. + +You said `it is bare and will not allow me to start an assistant on it`. Do you get an error message from the webapp? + +What happens if you manually add the path to your bare USB drive repo to the `~/.config/git-annex/autostart` file, then run `git-annex assistant --autostart`? Then is assistant running on it? +"""]] diff --git a/doc/bugs/OSX_Assistant_will_not_automatically_drop/comment_5_47aa5de003b409d3d2a4c58a0f08a14a._comment b/doc/bugs/OSX_Assistant_will_not_automatically_drop/comment_5_47aa5de003b409d3d2a4c58a0f08a14a._comment new file mode 100644 index 0000000000..722d92dd39 --- /dev/null +++ b/doc/bugs/OSX_Assistant_will_not_automatically_drop/comment_5_47aa5de003b409d3d2a4c58a0f08a14a._comment @@ -0,0 +1,45 @@ +[[!comment format=mdwn + username="jhnichol@cce81d2a480707652a3340ea2f24b3dc4b1f808c" + nickname="jhnichol" + avatar="http://cdn.libravatar.org/avatar/2d05fd7e681bf4838bba8bab538ac65d" + subject="comment 5" + date="2018-05-30T17:36:30Z" + content=""" +I started over, making new repos with the webapp & version: 6.20180527-gc3064edac . +I still get an error in the webapp when adding the USB repo . +The USB repo `/Volumes/SeagateBlack/annex` was not in `~/.config/git-annex/autostart` , so I added it . +I enabled debugging in the webapp, shutdown the webapp and assistants, and used the terminal: + + JamesMac:annex jhnichol$ cd /Volumes/SeagateBlack/annex + JamesMac:annex jhnichol$ git-annex assistant --autostart + git-annex: You cannot run this command in a bare repository. + JamesMac:annex jhnichol$ cd /Users/jhnichol/Desktop/annex + JamesMac:annex jhnichol$ git-annex assistant --autostart + git-annex autostart in /Users/jhnichol/Desktop/annex + [2018-05-30 20:13:45.005716] call: /Applications/git-annex.app/Contents/MacOS/bundle/git-annex [\"assistant\",\"--stop\"] + [2018-05-30 20:13:45.064878] process done ExitSuccess + [2018-05-30 20:13:45.065006] call: /Applications/git-annex.app/Contents/MacOS/bundle/git-annex [\"assistant\",\"--startdelay=5s\"] + [2018-05-30 20:13:45.117653] logging to .git/annex/daemon.log + [2018-05-30 20:13:45.507796] read: git [\"--git-dir=.git\",\"--work-tree=.\",\"--literal-pathspecs\",\"-c\",\"core.bare=false\",\"show-ref\",\"git-annex\"] + [2018-05-30 20:13:45.520686] process done ExitSuccess + [2018-05-30 20:13:45.520806] read: git [\"--git-dir=.git\",\"--work-tree=.\",\"--literal-pathspecs\",\"-c\",\"core.bare=false\",\"show-ref\",\"--hash\",\"refs/heads/git-annex\"] + [2018-05-30 20:13:45.526297] process done ExitSuccess + [2018-05-30 20:13:45.526857] read: git [\"--git-dir=.git\",\"--work-tree=.\",\"--literal-pathspecs\",\"-c\",\"core.bare=false\",\"log\",\"refs/heads/git-annex..2f0d4e1b6fe058a5cc899012ef99870cd2877d2c\",\"--pretty=%H\",\"-n1\"] + [2018-05-30 20:13:45.537446] process done ExitSuccess + [2018-05-30 20:13:45.537572] read: git [\"--git-dir=.git\",\"--work-tree=.\",\"--literal-pathspecs\",\"-c\",\"core.bare=false\",\"log\",\"refs/heads/git-annex..31d29d92a8a48146cdf9dd4517f5a034daf4a467\",\"--pretty=%H\",\"-n1\"] + [2018-05-30 20:13:45.543984] process done ExitSuccess + [2018-05-30 20:13:45.544626] chat: git [\"--git-dir=.git\",\"--work-tree=.\",\"--literal-pathspecs\",\"-c\",\"core.bare=false\",\"cat-file\",\"--batch\"] + [2018-05-30 20:13:45.546186] chat: git [\"--git-dir=.git\",\"--work-tree=.\",\"--literal-pathspecs\",\"-c\",\"core.bare=false\",\"cat-file\",\"--batch-check=%(objectname) %(objecttype) %(objectsize)\"] + [2018-05-30 20:13:45.550793] read: git [\"config\",\"--null\",\"--list\"] + [2018-05-30 20:13:45.556288] process done ExitSuccess + [2018-05-30 20:13:45.557233] logging to .git/annex/daemon.log + [2018-05-30 20:13:45.565149] process done ExitSuccess + ok + git-annex autostart in /Volumes/SeagateBlack/annex + failed + +so you see I got the \"You cannot run this command in a bare repository\" error from the assistant. +Also, the assistant for the USB drive just fails with no error details. + +Thank you so much for your comments! I feel so close to having working software! +"""]] diff --git a/doc/bugs/OSX_Assistant_will_not_automatically_drop/comment_6_69781b2f00afe6150bc5997335b8af90._comment b/doc/bugs/OSX_Assistant_will_not_automatically_drop/comment_6_69781b2f00afe6150bc5997335b8af90._comment new file mode 100644 index 0000000000..88d00cd336 --- /dev/null +++ b/doc/bugs/OSX_Assistant_will_not_automatically_drop/comment_6_69781b2f00afe6150bc5997335b8af90._comment @@ -0,0 +1,17 @@ +[[!comment format=mdwn + username="andrew" + avatar="http://cdn.libravatar.org/avatar/acc0ece1eedf07dd9631e7d7d343c435" + subject="bare remotes, workflow" + date="2018-05-31T12:50:55Z" + content=""" +Aaah. + +So although git-annex does support storing files in bare repositories it treats them in much the same way as git in general. Namely, they are meant to be added as remotes from other full repositories. Assistant is responsible for monitoring a working directory for changes and automatically committing and syncing those changes. Bare repositories don't actually need assistant monitoring them because they don't have a working directory where users are adding and removing files from. + +In your case, you want assistant monitoring `~/Desktop/annex` which it now is. If you then add `/Volumes/SeagateBlack/annex` as a remote to `~/Desktop/annex` then assistant, by watching just `~/Desktop/annex` will sync with all remotes whenever you add remove files in `~/Desktop/annex`. I imagine you already have your USB drive correctly setup as a remote. Just run `git annex info` from your `~/Desktop/annex` directory to see what remotes it knows about. + +Once USB is a remote of `~/Desktop/annex`, then if you launch the webapp from your `~/Desktop/annex` directory,`git-annex webapp` you should actually see both `~/Desktop/annex` and `/Volumes/SeagateBlack/annex` listed. The webapp automatically lists all remotes your repo knows about. + +In terms of workflow, I am not actually sure that using a bare repo is what you want on your USB drive. I always setup normal full git repositories on USB drives (not bare repos). That way I can use Finder to actually browse out to the files on the USB drive, add files when I am on the go, copy files etc… If you use a bare repo, then the files won't actually be listed in a usable form. But, if you don't actually want to ever look at the files on your USB drive (from Finder) then using a bare remote on a USB drive has the advantage that you don't need assistant monitoring it, and is probably marginally faster in some extreme cases since there is no working tree to update. + +"""]] diff --git a/doc/bugs/OSX__58___Assistant_leaves_repo_in_inconsistent_state.mdwn b/doc/bugs/OSX__58___Assistant_leaves_repo_in_inconsistent_state.mdwn new file mode 100644 index 0000000000..c7231cd1b1 --- /dev/null +++ b/doc/bugs/OSX__58___Assistant_leaves_repo_in_inconsistent_state.mdwn @@ -0,0 +1,151 @@ +### Please describe the problem. +Shortly after starting the assistant the local client repository is in an inconsistent state. Just using the cli works fine without destroying the repository. + +### What steps will reproduce the problem? +- Working git annex repository + +- Start the Git-Annex Assistant +> git annex assistant + +- Wait a short time and then check the repository state: +> ➜ annex git:(master) ✗ gst +> On branch master +> error: Could not read ea63b58d1252e3432280cdce1282eb02035c926e +> error: Could not read ea63b58d1252e3432280cdce1282eb02035c926e +> fatal: Failed to traverse parents of commit 108fa97a989aeabff16738558e1aa636c836598b + +- refs/heads/synced/git-annex links to an unknown commit +> ➜ .git git:(master) cat refs/heads/synced/git-annex +> be8e6023fbb172b12cec8fdbaa8654194f71c4f5% + +> git show be8e6023fbb172b12cec8fdbaa8654194f71c4f5 +> fatal: unable to read source tree (d2959525362dcadd4dbec96a8f18d80cc689a851) + +### What version of git-annex are you using? On what operating system? +- git-annex version: 5.20150508-gf71c23f +- Mac OSX 10.10.2 + +### Please provide any additional information below. + +[[!format sh """ + +[2015-05-15 07:06:32 CEST] main: starting assistant version 5.20150508-gf71c23f +[2015-05-15 07:06:32 CEST] Cronner: Consistency check in progress +fsck 2015-05-01 07.56.02.jpg (checksum…) +ok +fsck 2015-05-01 09.55.58.jpg (checksum…) +ok +fsck 2015-05-14 12.38.24.jpg (checksum…) +ok +fsck 2015-05-14 12.39.30.jpg (checksum…) +ok +fsck 2015-05-14 12.40.49.jpg (checksum…) +ok +fsck 2015-05-14 12.40.53.jpg (checksum…) +ok +fsck 2015-05-14 12.43.42.jpg (checksum…) +ok +fsck 2015-05-14 12.44.16.jpg (checksum…) +ok +fsck 2015-05-14 12.45.11.jpg (checksum…) +ok +(recording state in git…) +[2015-05-15 07:06:39 CEST] TransferScanner: Syncing with nas +(scanning…) [2015-05-15 07:06:39 CEST] Watcher: Performing startup scan +gpg: Unterschrift vom Fr 8 Mai 22:10:28 2015 CEST mittels DSA-Schlüssel ID 89C809CB +gpg: Unterschrift kann nicht geprüft werden: Öffentlicher Schlüssel nicht gefunden +(started…) +[2015-05-15 07:06:41 CEST] Committer: Committing changes to git +(recording state in git…) +[2015-05-15 07:06:41 CEST] Pusher: Syncing with nas + + +0% 0.0 B/s 0s +2% 32.0KB/s 47s +... +90% 467.6KB/s 0s + +[2015-05-15 07:06:47 CEST] Transferrer: Uploaded 2015-05-1..38.24.jpg + + +0% 0.0 B/s 0s +1% 32.0KB/s 54s +... +91% 534.5KB/s 0s + +[2015-05-15 07:06:54 CEST] Transferrer: Uploaded 2015-05-1..45.11.jpg + + +0% 0.0 B/s 0s +2% 16.0KB/s 1m30s +… +90% 440.4KB/s 0s + +[2015-05-15 07:07:01 CEST] Transferrer: Uploaded 2015-05-1..44.16.jpg + + +0% 0.0 B/s 0s +1% 16.0KB/s 1m42s +... +91% 508.2KB/s 0s + +[2015-05-15 07:07:07 CEST] Transferrer: Uploaded 2015-05-1..43.42.jpg + + +0% 0.0 B/s 0s +1% 16.0KB/s 1m40s +... +91% 371.6KB/s 0s + +[2015-05-15 07:07:13 CEST] Transferrer: Uploaded 2015-05-1..40.53.jpg + + +0% 0.0 B/s 0s +1% 32.0KB/s 50s +... +91% 495.1KB/s 0s + +[2015-05-15 07:07:19 CEST] Transferrer: Uploaded 2015-05-1..40.49.jpg + + +0% 0.0 B/s 0s +1% 32.0KB/s 53s +... +92% 528.2KB/s 0s + +[2015-05-15 07:07:25 CEST] Transferrer: Uploaded 2015-05-1..39.30.jpg +ssh: connect to host 1.2.3.43 port 22: Operation timed out +fatal: Could not read from remote repository. + +Please make sure you have the correct access rights +and the repository exists. +fatal: ‚/Volumes/WD1500GB/annex‘ does not appear to be a git repository +fatal: Could not read from remote repository. + +Please make sure you have the correct access rights +and the repository exists. +ssh: connect to host 1.2.3.43 port 22: Operation timed out +s(recording state in git…) +fatal: ‚/Volumes/WD1500GB/annex‘ does not appear to be a git repository +fatal: Could not read from remote repository. + +Please make sure you have the correct access rights +and the repository exists. +ssh: connect to host 1.2.3.43 port 22: Operation timed out +fatal: Could not read from remote repository. + +Please make sure you have the correct access rights +and the repository exists. +[2015-05-15 07:08:16 CEST] Cronner: Attempting to repair aether [here] +Unpacking all pack files. +ssh: connect to host 1.2.3.43 port 22: Operation timed out +fatal: Could not read from remote repository. + +Please make sure you have the correct access rights +and the repository exists. +ssh: connect to host 1.2.3.36 port 22:sssssh: connect to host 1.2.3.43 port 22: Operation timed out +fatal: early EOF + + +# End of transcript or log. +"""]] diff --git a/doc/bugs/OSX__58___Assistant_leaves_repo_in_inconsistent_state/comment_1_0dfd9eedccb48f1f3d7939677dc96446._comment b/doc/bugs/OSX__58___Assistant_leaves_repo_in_inconsistent_state/comment_1_0dfd9eedccb48f1f3d7939677dc96446._comment new file mode 100644 index 0000000000..be48ca87fb --- /dev/null +++ b/doc/bugs/OSX__58___Assistant_leaves_repo_in_inconsistent_state/comment_1_0dfd9eedccb48f1f3d7939677dc96446._comment @@ -0,0 +1,9 @@ +[[!comment format=mdwn + username="Marco" + subject="git annex repair --force" + date="2015-05-15T08:01:25Z" + content=""" +This command removed all history. It looks like no file was ever added. + +Any idea how I should handle this client repository? I could clone again from a backup. Is it possible to keep the annex UUID for a new clone? +"""]] diff --git a/doc/bugs/OSX__58___Assistant_leaves_repo_in_inconsistent_state/comment_2_5c5e7356cae8ddf8d2c8964fb69000f5._comment b/doc/bugs/OSX__58___Assistant_leaves_repo_in_inconsistent_state/comment_2_5c5e7356cae8ddf8d2c8964fb69000f5._comment new file mode 100644 index 0000000000..5d39e8ac22 --- /dev/null +++ b/doc/bugs/OSX__58___Assistant_leaves_repo_in_inconsistent_state/comment_2_5c5e7356cae8ddf8d2c8964fb69000f5._comment @@ -0,0 +1,11 @@ +[[!comment format=mdwn + username="Marco" + subject="Disaster recovery" + date="2015-05-15T09:40:16Z" + content=""" +Ah, found it: + +https://git-annex.branchable.com/design/assistant/disaster_recovery/ + +> As long as the git repository has at least one remote, another method is to clone the remote, sync from all other remotes, move over .git/config and .git/annex/objects, and tar up the old broken git repo and git annex add it +"""]] diff --git a/doc/bugs/OSX__58___Assistant_leaves_repo_in_inconsistent_state/comment_3_1e0841d71c33fd3919310f1711b6e0b4._comment b/doc/bugs/OSX__58___Assistant_leaves_repo_in_inconsistent_state/comment_3_1e0841d71c33fd3919310f1711b6e0b4._comment new file mode 100644 index 0000000000..319b001f82 --- /dev/null +++ b/doc/bugs/OSX__58___Assistant_leaves_repo_in_inconsistent_state/comment_3_1e0841d71c33fd3919310f1711b6e0b4._comment @@ -0,0 +1,12 @@ +[[!comment format=mdwn + username="Marco" + subject="A possible solution" + date="2015-05-19T11:03:22Z" + content=""" +While searching for the problem I found this: + +http://feeding.cloud.geek.nz/posts/error-while-running-git-gc/ +> git reflog expire --all --stale-fix + +I can't verify this because I nuked the broken repository after recreating it. +"""]] diff --git a/doc/bugs/OSX__58___Assistant_leaves_repo_in_inconsistent_state/comment_4_eb5b0f8259d861000510fcfd58f66617._comment b/doc/bugs/OSX__58___Assistant_leaves_repo_in_inconsistent_state/comment_4_eb5b0f8259d861000510fcfd58f66617._comment new file mode 100644 index 0000000000..395c31b948 --- /dev/null +++ b/doc/bugs/OSX__58___Assistant_leaves_repo_in_inconsistent_state/comment_4_eb5b0f8259d861000510fcfd58f66617._comment @@ -0,0 +1,16 @@ +[[!comment format=mdwn + username="Marco" + subject="Once again" + date="2015-06-08T15:45:15Z" + content=""" +I was using the assistant on two machines in parallel and both git repositories got broken. Before that I was just using the command line tools and just one assistant. Today the history got inconsistent when firing up the second assistant. + +Repos in use: +- client 1 with active assistant +- client 2 with active assistant +- Synology NAS with git-annex installed (armel-based) + +I could recover everything pretty fast by cloning again from the nas device. Copying the .git/config and moving over the .git/annex directory. + +I will send you the log-files via mail. +"""]] diff --git a/doc/bugs/OSX__58___Assistant_leaves_repo_in_inconsistent_state/comment_5_4343ee35e4091fc50268f9fc611f5148._comment b/doc/bugs/OSX__58___Assistant_leaves_repo_in_inconsistent_state/comment_5_4343ee35e4091fc50268f9fc611f5148._comment new file mode 100644 index 0000000000..0ec1370819 --- /dev/null +++ b/doc/bugs/OSX__58___Assistant_leaves_repo_in_inconsistent_state/comment_5_4343ee35e4091fc50268f9fc611f5148._comment @@ -0,0 +1,10 @@ +[[!comment format=mdwn + username="Marco" + subject="How to debug?" + date="2015-07-25T18:10:08Z" + content=""" +I want to give the two assistants one more try with the newest version. + +But I want the assistant not to try a repair once a failure is detected. I want to check the state of the repository to get a clue what goes wrong. +As I understood the code there is no configuration to prevent the repair. Am I wrong? If not I could try to introduce one if it makes sense. +"""]] diff --git a/doc/bugs/OSX__58___addurl_--batch_--json_spits_out_shortened_output_string__dies_off_with_4.mdwn b/doc/bugs/OSX__58___addurl_--batch_--json_spits_out_shortened_output_string__dies_off_with_4.mdwn new file mode 100644 index 0000000000..5c72b00959 --- /dev/null +++ b/doc/bugs/OSX__58___addurl_--batch_--json_spits_out_shortened_output_string__dies_off_with_4.mdwn @@ -0,0 +1,41 @@ +### Please describe the problem. + +See transcript. Didn't bisect fully but 6.20160128-g599a3ba seems to work ok, 6.20160211-g479a094 already dies off + +[[!format sh """ + +datalads-imac:~ yoh$ git annex version +git-annex version: 6.20160221-g3d1895e +... +datalads-imac:~ yoh$ mkdir -p /tmp/123; cd /tmp/123; git init; git annex init; +Initialized empty Git repository in /private/tmp/123/.git/ +init ok +(recording state in git...) + +datalads-imac:123 yoh$ rm -f /tmp/pipe; mkfifo /tmp/pipe; cat /tmp/pipe | git -c receive.autogc=false annex addurl -c annex.largefiles=exclude=*.txt --with-files --json --batch +{"command":"addurl","file":"1.png","note":"downloading http://www.onerussian.com/tmp/banner.png ..."error: git-annex died of signal 4 + +# now with --debug +datalads-imac:123 yoh$ rm -f /tmp/pipe; mkfifo /tmp/pipe; cat /tmp/pipe | git -c receive.autogc=false annex addurl -c annex.largefiles=exclude=*.txt --with-files --json --batch --debug +[2016-02-23 08:32:49.160686] read: git ["--git-dir=.git","--work-tree=.","--literal-pathspecs","-c","annex.largefiles=exclude=*.txt","show-ref","git-annex"] +[2016-02-23 08:32:49.167372] process done ExitSuccess +[2016-02-23 08:32:49.167479] read: git ["--git-dir=.git","--work-tree=.","--literal-pathspecs","-c","annex.largefiles=exclude=*.txt","show-ref","--hash","refs/heads/git-annex"] +[2016-02-23 08:32:49.172795] process done ExitSuccess +[2016-02-23 08:32:49.173039] read: git ["--git-dir=.git","--work-tree=.","--literal-pathspecs","-c","annex.largefiles=exclude=*.txt","log","refs/heads/git-annex..b2892e67682c5240dadb5d439da810edbdb882df","-n1","--pretty=%H"] +[2016-02-23 08:32:49.178586] process done ExitSuccess +[2016-02-23 08:32:49.179085] chat: git ["--git-dir=.git","--work-tree=.","--literal-pathspecs","-c","annex.largefiles=exclude=*.txt","cat-file","--batch"] +[2016-02-23 08:32:49.183985] read: quvi ["--version"] +{"command":"addurl","file":"1.png","note":"downloading http://www.onerussian.com/tmp/banner.png ..."[2016-02-23 08:32:49.255949] call: curl ["-s","-f","-L","-C","-","-#","-o","/private/tmp/123/.git/annex/tmp/URL-s25319--http&c%%www.onerussian.com%tmp%banner.png","http://www.onerussian.com/tmp/banner.png","--user-agent","git-annex/6.20160221-g3d1895e"] +[2016-02-23 08:32:49.302577] process done ExitSuccess +[2016-02-23 08:32:49.303332] chat: git ["--git-dir=.git","--work-tree=.","--literal-pathspecs","-c","annex.largefiles=exclude=*.txt","check-attr","-z","--stdin","annex.backend","annex.numcopies","annex.largefiles","--"] +[2016-02-23 08:32:49.305653] read: git ["--version"] +[2016-02-23 08:32:49.310314] process done ExitSuccess +error: git-annex died of signal 4 + +"""]] + +[[!meta author=yoh]] + +> [[done]]; this was indeed due to libmagic, and I've fixed it using brew +> --build-bottle to make a portable one. Then had to disable the build +> flag. --[[Joey]] diff --git a/doc/bugs/OSX_assistant_fails_to_download_new_file_after_initial_pass.mdwn b/doc/bugs/OSX_assistant_fails_to_download_new_file_after_initial_pass.mdwn new file mode 100644 index 0000000000..bb76e96e52 --- /dev/null +++ b/doc/bugs/OSX_assistant_fails_to_download_new_file_after_initial_pass.mdwn @@ -0,0 +1,119 @@ +### Please describe the problem. + +git-annex assistant on Mac OSX 10.8.5 fails to download new repo files from SSH remote/rsync shared-encryption annex after initial startup. The assistant does sync with the repo and create symlinks for new files, and file contents can manually be retrieved using 'git annex get $file' once the file is in the repository. However, assistant makes no subsequent attempt to download the new files until assistant is restarted. Once restarted, assistant downloads all new files for which is previously only had symlinks. + +git-annex whereis $file does not indicate that the files represented as symlinks are located on the Mac OSX clients in question until after files are manually retrieved or assistant is retarted and files are automatically downloaded, replacing the symlinks. + +git-annex assistant on Mac OSX does not have problems uploading files to the remote as they are added. + +git-annex 5.20140517.4 on Ubuntu 12.04 behaves as expected and downloads new files (with content) properly as they are added to the repo. + +### What steps will reproduce the problem? + +(all using git-annex webapp) + +--create client repo on Mac OSX, direct mode (used two 10.8.5 machines) + +--create client repo on Ubuntu, direct mode (used several 12.04 VMs) + +--create repo (transfer mode) on VPS + +--create shared encryption repo (backup mode) on same VPS + +--add VPS transfer repo to each client (shared encryption repo appears and syncing is enabled for each client) + +--create file in Mac OSX annex - file gets distributed to VPS backup repo and each linux system. Symlink is created on 2nd Mac OS X machine. + +--create file in Ubuntu VM annex - file gets distributed to VPS backup repo and each linux system. Symlink is created on both Mac OSX machines. + + + +### What version of git-annex are you using? On what operating system? +git-annex version: 5.20140421 (installed using brew), Mac OSX 10.8.5 + +I tested 5.20140613 (from brew) also and it appears to also not download new files automatically. + + + +### Please provide any additional information below. +I've tested a few things I thought might be related. Since restarting assistant seems to temporarily "fix" the problem, I thought there may be some SSH client caching problems... as in the existing connection not being reused properly. However, I disabled annex.sshcaching as described elsewhere and that did not have any affect. + +I've enabled XMPP and it only introduced more confusion as the XMPP messaging was not reliably consistent. + +daemon.log for Mac OS systems does not show the "Pusher" process getting initiated once new files are added to the repo if the assistant is currently running, however it is logged immediately after assistant is started if there are new files in the repo. + +daemon.log for Ubuntu systems does contain the Pusher process event following the repo sync that takes place once the file has been added to the repo. + + + +[[!format sh """ +# If you can, paste a complete transcript of the problem occurring here. +# If the problem is with the git-annex assistant, paste in .git/annex/daemon.log + +MACOSX:user$git-annex assistant --autostart + +daemon.log: + +[2014-06-25 11:52:04 CDT] main: starting assistant version 5.20140421 +[2014-06-25 11:52:09 CDT] TransferScanner: Syncing with serverrepo + +Already up-to-date. +(scanning...) [2014-06-25 11:52:09 CDT] Watcher: Performing startup scan + +Already up-to-date. +(started...) +[2014-06-25 11:52:10 CDT] Committer: Committing changes to git +(Recording state in git...) +(gpg) +Already up-to-date. + +Already up-to-date. +[2014-06-25 11:52:11 CDT] Pusher: Syncing with serverrepo + +GPGHMACSHA1--3e18c6a4ee5aece84f4f00fc2e7fcb828d8fa7d8 +Everything up-to-date0.00kB/s 0:00:00 +Everything up-to-date4.56MB/s 0:00:00 + 8409197 100% 4.94MB/s 0:00:01 (xfer#1, to-check=0/1) + +sent 38 bytes received 8411383 bytes 2403263.14 bytes/sec +total size is 8409197 speedup is 1.00 +[2014-06-25 11:52:13 CDT] Transferrer: Downloaded 67-more-fromubuntu +[2014-06-25 11:52:14 CDT] Committer: Adding 67-more-fromubuntu +add /Users/user/computer_annex/67-more-fromubuntu ok +[2014-06-25 11:52:14 CDT] Committer: Committing changes to git +(Recording state in git...) +[2014-06-25 11:52:20 CDT] Pusher: Syncing with serverrepo +(Recording state in git...) +To ssh://user@git-annex-server.remotehost.net-user_.2Fhome.2Fuser.2Fserver_repo/home/user/server_repo/ + 7e375a8..1a64e4c git-annex -> synced/git-annex +[2014-06-25 11:52:36 CDT] RemoteControl: Syncing with serverrepo +From ssh://git-annex-server.remotehost.net-user_.2Fhome.2Fuser.2Fserver_repo/home/user/server_repo + 1a64e4c..dee9f84 synced/git-annex -> serverrepo/synced/git-annex + d424057..989003e synced/master -> serverrepo/synced/master +(merging serverrepo/synced/git-annex into git-annex...) + +<<>> + +Updating d424057..989003e +Fast-forward + 89-more-fromubuntu | 1 + + 1 file changed, 1 insertion(+) + create mode 120000 89-more-fromubuntu +[2014-06-25 11:58:42 CDT] RemoteControl: Syncing with serverrepo +From ssh://git-annex-server.remotehost.net-user_.2Fhome.2Fuser.2Fserver_repo/home/user/server_repo + dee9f84..b24c8bf synced/git-annex -> serverrepo/synced/git-annex +(merging serverrepo/synced/git-annex into git-annex...) +[2014-06-25 11:58:47 CDT] RemoteControl: Syncing with serverrepo +From ssh://git-annex-server.remotehost.net-user_.2Fhome.2Fuser.2Fserver_repo/home/user/server_repo + b24c8bf..6488b45 synced/git-annex -> serverrepo/synced/git-annex +(merging serverrepo/synced/git-annex into git-annex...) +[2014-06-25 11:58:56 CDT] RemoteControl: Syncing with serverrepo +From ssh://git-annex-server.remotehost.net-user_.2Fhome.2Fuser.2Fserver_repo/home/user/server_repo + 6488b45..c73c180 synced/git-annex -> serverrepo/synced/git-annex +(merging serverrepo/synced/git-annex into git-annex...) + + + + +# End of transcript or log. +"""]] diff --git a/doc/bugs/OSX_assistant_fails_to_download_new_file_after_initial_pass/comment_10_3feba4ba84efb77bd4f8f46b6b4600f1._comment b/doc/bugs/OSX_assistant_fails_to_download_new_file_after_initial_pass/comment_10_3feba4ba84efb77bd4f8f46b6b4600f1._comment new file mode 100644 index 0000000000..37aa47c37e --- /dev/null +++ b/doc/bugs/OSX_assistant_fails_to_download_new_file_after_initial_pass/comment_10_3feba4ba84efb77bd4f8f46b6b4600f1._comment @@ -0,0 +1,8 @@ +[[!comment format=mdwn + username="https://www.google.com/accounts/o8/id?id=AItOawncukUQl56TwiBJb7dIyAqP1YirNg_wjR4" + nickname="Matthew" + subject="Actions -> Sync Now in webapp does properly download new files" + date="2014-07-17T13:50:06Z" + content=""" +confirmed - your understanding is correct +"""]] diff --git a/doc/bugs/OSX_assistant_fails_to_download_new_file_after_initial_pass/comment_1_a33fcd088e419d8e6c459e42f21f8bbe._comment b/doc/bugs/OSX_assistant_fails_to_download_new_file_after_initial_pass/comment_1_a33fcd088e419d8e6c459e42f21f8bbe._comment new file mode 100644 index 0000000000..b78fd5aa07 --- /dev/null +++ b/doc/bugs/OSX_assistant_fails_to_download_new_file_after_initial_pass/comment_1_a33fcd088e419d8e6c459e42f21f8bbe._comment @@ -0,0 +1,10 @@ +[[!comment format=mdwn + username="https://www.google.com/accounts/o8/id?id=AItOawnu1NjsjNS3m0OxiYLKO-5SKNpSERxpi6k" + nickname="Matthew" + subject="comment 1" + date="2014-07-09T13:56:31Z" + content=""" +\"workaround\" is just a cron job that runs 'git annex sync --content' every N minutes... + +Is there any possible configuration error I've made that would cause the OSX clients to behave differently than the linux clients despite ostensibly being set up identically?? +"""]] diff --git a/doc/bugs/OSX_assistant_fails_to_download_new_file_after_initial_pass/comment_2_47196f7e781137751ebd1a1d7083838a._comment b/doc/bugs/OSX_assistant_fails_to_download_new_file_after_initial_pass/comment_2_47196f7e781137751ebd1a1d7083838a._comment new file mode 100644 index 0000000000..ee5ad138e5 --- /dev/null +++ b/doc/bugs/OSX_assistant_fails_to_download_new_file_after_initial_pass/comment_2_47196f7e781137751ebd1a1d7083838a._comment @@ -0,0 +1,18 @@ +[[!comment format=mdwn + username="http://joeyh.name/" + ip="209.250.56.2" + subject="comment 2" + date="2014-07-10T19:19:36Z" + content=""" +Could you please go into some more detail about these two things: + +> --create repo (transfer mode) on VPS +> +> --create shared encryption repo (backup mode) on same VPS + +It's not clear to me what kind of repositories these are. Are they bare git repositories? What version of git-annex, if any, is installed on that VPS? + +> git-annex whereis $file does not indicate that the files represented as symlinks are located on the Mac OSX clients in question until after files are manually retrieved + +Well, I'd not expect whereis to say that the file is in a repository until it's retrieved into that repository. However, does whereis, on the OSX clients, ever say that the file has reached one of the repos on the VPS? +"""]] diff --git a/doc/bugs/OSX_assistant_fails_to_download_new_file_after_initial_pass/comment_3_672d98ee06e051430f8e01faa93bb4cf._comment b/doc/bugs/OSX_assistant_fails_to_download_new_file_after_initial_pass/comment_3_672d98ee06e051430f8e01faa93bb4cf._comment new file mode 100644 index 0000000000..e26492dd7c --- /dev/null +++ b/doc/bugs/OSX_assistant_fails_to_download_new_file_after_initial_pass/comment_3_672d98ee06e051430f8e01faa93bb4cf._comment @@ -0,0 +1,28 @@ +[[!comment format=mdwn + username="https://www.google.com/accounts/o8/id?id=AItOawncukUQl56TwiBJb7dIyAqP1YirNg_wjR4" + nickname="Matthew" + subject="comment 3" + date="2014-07-11T17:06:43Z" + content=""" +I created an local repository on each of 2 OS X (10.8.5) systems running git-annex version 5.20140421, and each of 2 Ubuntu 12.04 systems running version 5.20140517.4 + +I then created a remote SSH repository by selecting \"Make an unencrypted git repository on the server\" - this repo is in \"transfer\" mode. + +On the 2nd OSX client, I added a remote SSH repository, providing the same host and path information and elected to \"combine\" my local repo with the remote repo so that files would stay in sync. + +I did the same on each of the linux clients. + +git-annex indicates that local repos on each machine are in direct mode and show all expected semi trusted repos: web, SSH remote, other clients that have combined with the SSH remote. + +git-annex indicates that the SSH remote repo is in indirect mode + +git indicates that all repos are in bare mode + +git-annex version 5.20140412ubuntu1 is installed on the VPS, and 'git-annex shell notify changes' processes are running. + +Agreed on the behavior of git-annex whereis… just pointing out that the location info is accurate. + +The 2nd, encrypted repo may have no bearing at all because even when I remove it, the client behavior on OSX doesn't change. + +Thanks for taking the time to consider this! +"""]] diff --git a/doc/bugs/OSX_assistant_fails_to_download_new_file_after_initial_pass/comment_4_06fb3031b838cd443326f4ecd689b600._comment b/doc/bugs/OSX_assistant_fails_to_download_new_file_after_initial_pass/comment_4_06fb3031b838cd443326f4ecd689b600._comment new file mode 100644 index 0000000000..0f77607a04 --- /dev/null +++ b/doc/bugs/OSX_assistant_fails_to_download_new_file_after_initial_pass/comment_4_06fb3031b838cd443326f4ecd689b600._comment @@ -0,0 +1,10 @@ +[[!comment format=mdwn + username="https://www.google.com/accounts/o8/id?id=AItOawncukUQl56TwiBJb7dIyAqP1YirNg_wjR4" + nickname="Matthew" + subject="comment 4" + date="2014-07-11T17:11:41Z" + content=""" +I should add that in the web app on OSX, the behavior is consistent; syncing the local repository on an OSX client create a symlink (if that hasn't already been done in the background). resyncing \"here\" (local client) doesn't download the file. syncing the remote SSH repo, however does download the file content (as does git-annex sync --content) + +thanks! +"""]] diff --git a/doc/bugs/OSX_assistant_fails_to_download_new_file_after_initial_pass/comment_5_42d447400c15acf6ca031d165b2c781c._comment b/doc/bugs/OSX_assistant_fails_to_download_new_file_after_initial_pass/comment_5_42d447400c15acf6ca031d165b2c781c._comment new file mode 100644 index 0000000000..04a5de75c8 --- /dev/null +++ b/doc/bugs/OSX_assistant_fails_to_download_new_file_after_initial_pass/comment_5_42d447400c15acf6ca031d165b2c781c._comment @@ -0,0 +1,8 @@ +[[!comment format=mdwn + username="https://www.google.com/accounts/o8/id?id=AItOawncukUQl56TwiBJb7dIyAqP1YirNg_wjR4" + nickname="Matthew" + subject="comment 5" + date="2014-07-11T18:10:06Z" + content=""" +Sorry to miss this previously, but, yes, whereis does indicate that it has made it to both of the remotes. +"""]] diff --git a/doc/bugs/OSX_assistant_fails_to_download_new_file_after_initial_pass/comment_6_edd7d5d5c761ff665840f0ef7bea50c9._comment b/doc/bugs/OSX_assistant_fails_to_download_new_file_after_initial_pass/comment_6_edd7d5d5c761ff665840f0ef7bea50c9._comment new file mode 100644 index 0000000000..9b28909867 --- /dev/null +++ b/doc/bugs/OSX_assistant_fails_to_download_new_file_after_initial_pass/comment_6_edd7d5d5c761ff665840f0ef7bea50c9._comment @@ -0,0 +1,10 @@ +[[!comment format=mdwn + username="http://joeyh.name/" + ip="209.250.56.2" + subject="comment 6" + date="2014-07-16T20:51:36Z" + content=""" +5.20140421 happens to be the very first version of git-annex to support notifychanges. I forget what bugs that first attempt may have had. Have you tried upgrading the macs to a newer version? + + +"""]] diff --git a/doc/bugs/OSX_assistant_fails_to_download_new_file_after_initial_pass/comment_7_18e5334ab89efcf89ba8847436d55065._comment b/doc/bugs/OSX_assistant_fails_to_download_new_file_after_initial_pass/comment_7_18e5334ab89efcf89ba8847436d55065._comment new file mode 100644 index 0000000000..197fca67af --- /dev/null +++ b/doc/bugs/OSX_assistant_fails_to_download_new_file_after_initial_pass/comment_7_18e5334ab89efcf89ba8847436d55065._comment @@ -0,0 +1,13 @@ +[[!comment format=mdwn + username="https://www.google.com/accounts/o8/id?id=AItOawncukUQl56TwiBJb7dIyAqP1YirNg_wjR4" + nickname="Matthew" + subject="comment 7" + date="2014-07-16T21:23:00Z" + content=""" +git-annex version: 5.20140613 also behaves the same way, however it generates a new log message during the sync attempt: + +Automatic merge went well; stopped before committing as requested + + + +"""]] diff --git a/doc/bugs/OSX_assistant_fails_to_download_new_file_after_initial_pass/comment_8_35b2bbdc24a7bd686527cd1839dee7d0._comment b/doc/bugs/OSX_assistant_fails_to_download_new_file_after_initial_pass/comment_8_35b2bbdc24a7bd686527cd1839dee7d0._comment new file mode 100644 index 0000000000..f551fe6275 --- /dev/null +++ b/doc/bugs/OSX_assistant_fails_to_download_new_file_after_initial_pass/comment_8_35b2bbdc24a7bd686527cd1839dee7d0._comment @@ -0,0 +1,10 @@ +[[!comment format=mdwn + username="http://joeyh.name/" + ip="209.250.56.2" + subject="comment 8" + date="2014-07-16T22:23:50Z" + content=""" +Hmm, it looks to me like the transfer scan may not be getting run after the git branches get pulled. I would expect to see \"starting scan of\" in the debug log when it does that scan. Do you see that in the log? + +AIUI, when you click on Actions -> Sync Now in the webapp for the repository that contains the content, it gets downloaded right? That does the same scan that should be automatically run after a branch is pulled. +"""]] diff --git a/doc/bugs/OSX_assistant_fails_to_download_new_file_after_initial_pass/comment_9_a771c6b453e6a4b3895dd69a53093440._comment b/doc/bugs/OSX_assistant_fails_to_download_new_file_after_initial_pass/comment_9_a771c6b453e6a4b3895dd69a53093440._comment new file mode 100644 index 0000000000..bf57b46be3 --- /dev/null +++ b/doc/bugs/OSX_assistant_fails_to_download_new_file_after_initial_pass/comment_9_a771c6b453e6a4b3895dd69a53093440._comment @@ -0,0 +1,246 @@ +[[!comment format=mdwn + username="https://www.google.com/accounts/o8/id?id=AItOawncukUQl56TwiBJb7dIyAqP1YirNg_wjR4" + nickname="Matthew" + subject="comment 9" + date="2014-07-17T01:58:29Z" + content=""" +Hi Joey, + +Thanks for taking the time to follow up on this - I sincerely appreciate it! + +Only immediately after I start the assistant does the \"starting scan of\" message get logged: + +[2014-07-16 20:38:07 CDT] TransferScanner: starting scan of [Remote { name =\"hostrepo\" }] + +and, at that point, new files are downloaded. + +If the assistant has been running for a while, I then create a new file in an annex on another machine, and the OS X assistant has synced and then created a symlink for new files, the following log is generated (NOTE the groovy scroodled first line if it's at all of interest...): + + +[2014-07-16 20:30:[312 0C1D4T-] 0c7a-ll1:6 g2i0t: 3[0\":--3g1i tC-DdTi]r =R/eUmsoetresC/omnhtirlotly:/ sShYiNnCiIeNrG_ asnsnhe:x///.mgaitte\"o,@\"g-i-tw-oarnkn-etxr-egeo=t/aU.sienrfsr/omnhtiiletrys/.snheitn-imeart_eaon_ngeoxt\"a,_\"r-ecp\"o,/\"~c/ogroet.ab_arreep=of/a +lse\",\"fetch\",\"hostrepo\"] + +[2014-07-16 20:30:31 CDT] RemoteControl: Syncing with hostrepo +From ssh://git-annex-host.vpshost.net-someuser_host_repo/~/host_repo + fdd1eff..e1057e0 synced/git-annex -> hostrepo/synced/git-annex + fa3cfb0..e4ba411 synced/master -> hostrepo/synced/master + +[2014-07-16 20:30:32 CDT] RemoteControl: DONESYNCING ssh://someuser@git-annex-host.vpshost.net-someuser_host_repo/~/host_repo/ 1 + +[2014-07-16 20:30:32 CDT] read: git [\"--git-dir=/Users/someuser/local_annex/.git\",\"--work-tree=/Users/someuser/local_annex\",\"-c\",\"core.bare=false\",\"show-ref\",\"git-annex\"] + +[2014-07-16 20:30:32 CDT] read: git [\"--git-dir=/Users/someuser/local_annex/.git\",\"--work-tree=/Users/someuser/local_annex\",\"-c\",\"core.bare=false\",\"show-ref\",\"--hash\",\"refs/heads/git-annex\"] + +[2014-07-16 20:30:32 CDT] read: git [\"--git-dir=/Users/someuser/local_annex/.git\",\"--work-tree=/Users/someuser/local_annex\",\"-c\",\"core.bare=false\",\"log\",\"refs/heads/git-annex..fdd1effb8d2b8073e94725874e16056d6557f199\",\"--oneline\",\"-n1\"] + +[2014-07-16 20:30:32 CDT] read: git [\"--git-dir=/Users/someuser/local_annex/.git\",\"--work-tree=/Users/someuser/local_annex\",\"-c\",\"core.bare=false\",\"log\",\"refs/heads/git-annex..bfef6b84186d59431377343ab968e255db8c76fa\",\"--oneline\",\"-n1\"] + +[2014-07-16 20:30:32 CDT] read: git [\"--git-dir=/Users/someuser/local_annex/.git\",\"--work-tree=/Users/someuser/local_annex\",\"-c\",\"core.bare=false\",\"log\",\"refs/heads/git-annex..e1057e037f03ba804f021c8c39073b2efd1fad08\",\"--oneline\",\"-n1\"] +(merging hostrepo/synced/git-annex into git-annex...) + +[2014-07-16 20:30:32 CDT] feed: git [\"--git-dir=/Users/someuser/local_annex/.git\",\"--work-tree=/Users/someuser/local_annex\",\"-c\",\"core.bare=false\",\"update-index\",\"-z\",\"--index-info\"] + +[2014-07-16 20:30:32 CDT] read: git [\"--git-dir=/Users/someuser/local_annex/.git\",\"--work-tree=/Users/someuser/local_annex\",\"-c\",\"core.bare=false\",\"diff-index\",\"--raw\",\"-z\",\"-r\",\"--no-renames\",\"-l0\",\"--cached\",\"e1057e037f03ba804f021c8c39073b2efd1fad08\"] + +[2014-07-16 20:30:32 CDT] read: git [\"--git-dir=/Users/someuser/local_annex/.git\",\"--work-tree=/Users/someuser/local_annex\",\"-c\",\"core.bare=false\",\"log\",\"e1057e037f03ba804f021c8c39073b2efd1fad08..refs/heads/git-annex\",\"--oneline\",\"-n1\"] + +[2014-07-16 20:30:32 CDT] call: git [\"--git-dir=/Users/someuser/local_annex/.git\",\"--work-tree=/Users/someuser/local_annex\",\"-c\",\"core.bare=false\",\"update-ref\",\"refs/heads/git-annex\",\"e1057e037f03ba804f021c8c39073b2efd1fad08\"] + +[2014-07-16 20:30:32 CDT] read: git [\"--git-dir=/Users/someuser/local_annex/.git\",\"--work-tree=/Users/someuser/local_annex\",\"-c\",\"core.bare=false\",\"ls-tree\",\"--full-tree\",\"-z\",\"--\",\"refs/heads/git-annex\",\"uuid.log\",\"remote.log\",\"trust.log\",\"group.log\",\"numcopies.log\",\"schedule.log\",\"preferred-content.log\",\"required-content.log\",\"group-preferred-content.log\"] + +[2014-07-16 20:30:32 CDT] read: git [\"--git-dir=/Users/someuser/local_annex/.git\",\"--work-tree=/Users/someuser/local_annex\",\"-c\",\"core.bare=false\",\"symbolic-ref\",\"HEAD\"] + +[2014-07-16 20:30:32 CDT] read: git [\"--git-dir=/Users/someuser/local_annex/.git\",\"--work-tree=/Users/someuser/local_annex\",\"-c\",\"core.bare=false\",\"show-ref\",\"refs/heads/annex/direct/master\"] + +[2014-07-16 20:30:32 CDT] Merger: merging refs/remotes/hostrepo/synced/master into refs/heads/annex/direct/master + +[2014-07-16 20:30:32 CDT] read: git [\"--git-dir=/Users/someuser/local_annex/.git\",\"--work-tree=/Users/someuser/local_annex\",\"-c\",\"core.bare=false\",\"show-ref\",\"--hash\",\"refs/heads/annex/direct/master\"] + +[2014-07-16 20:30:32 CDT] call: git [\"--git-dir=/Users/someuser/local_annex/.git\",\"--work-tree=/Users/someuser/local_annex/.git/annex/merge/\",\"-c\",\"core.bare=false\",\"merge\",\"--no-edit\",\"refs/remotes/hostrepo/synced/master\"] + +Updating fa3cfb0..e4ba411 +Fast-forward + from-ubuntu | 1 + + 1 file changed, 1 insertion(+) + create mode 120000 from-ubuntu + +[2014-07-16 20:30:32 CDT] read: git [\"--git-dir=/Users/someuser/local_annex/.git\",\"--work-tree=/Users/someuser/local_annex\",\"-c\",\"core.bare=false\",\"diff-tree\",\"-z\",\"--raw\",\"--no-renames\",\"-l0\",\"-r\",\"fa3cfb01cc4493ed8a7cf84dff38bbba1fc042f9\",\"HEAD\"] + +[2014-07-16 20:30:33 CDT] read: git [\"--git-dir=/Users/someuser/local_annex/.git\",\"--work-tree=/Users/someuser/local_annex\",\"-c\",\"core.bare=false\",\"show-ref\",\"git-annex\"] + +[2014-07-16 20:30:33 CDT] read: git [\"--git-dir=/Users/someuser/local_annex/.git\",\"--work-tree=/Users/someuser/local_annex\",\"-c\",\"core.bare=false\",\"show-ref\",\"--hash\",\"refs/heads/git-annex\"] + +[2014-07-16 20:30:33 CDT] read: git [\"--git-dir=/Users/someuser/local_annex/.git\",\"--work-tree=/Users/someuser/local_annex\",\"-c\",\"core.bare=false\",\"log\",\"refs/heads/git-annex..e1057e037f03ba804f021c8c39073b2efd1fad08\",\"--oneline\",\"-n1\"] + +[2014-07-16 20:30:33 CDT] read: git [\"--git-dir=/Users/someuser/local_annex/.git\",\"--work-tree=/Users/someuser/local_annex\",\"-c\",\"core.bare=false\",\"log\",\"refs/heads/git-annex..bfef6b84186d59431377343ab968e255db8c76fa\",\"--oneline\",\"-n1\"] + +[2014-07-16 20:30:33 CDT] feed: git [\"--git-dir=/Users/someuser/local_annex/.git\",\"--work-tree=/Users/someuser/local_annex\",\"-c\",\"core.bare=false\",\"update-index\",\"-z\",\"--index-info\"] + +[2014-07-16 20:30:33 CDT] read: git [\"--git-dir=/Users/someuser/local_annex/.git\",\"--work-tree=/Users/someuser/local_annex\",\"-c\",\"core.bare=false\",\"diff-index\",\"--raw\",\"-z\",\"-r\",\"--no-renames\",\"-l0\",\"--cached\",\"refs/heads/git-annex\"] + +[2014-07-16 20:30:45 CDT][ cal2l0:1 4g-i0t7 -[16\" -2-0g:i3t0-:di4r5= /CUDsTe]r sR/emmhoitletCyo/nsthrionli:e rS_YaNnCnIeNxG/ .sgsiht:\"/,/\"m-a-tweoor@kg-ittr-eaen=n/eUxs-egrost/am.hiinlftryo/nsthiienrise.rn_eatn-nmeaxt\"e,o\"_-gco\"t,a\"_croerpeo./b~a/rgeo=tfaa_lrseep\"o,/\" +fetch\",\"hostrepo\"] + +[2014-07-16 20:30:45 CDT] RemoteControl: Syncing with hostrepo +From ssh://git-annex-host.vpshost.net-someuser_host_repo/~/host_repo + e1057e0..d44a67a synced/git-annex -> hostrepo/synced/git-annex + +[2014-07-16 20:30:47 CDT] RemoteControl: DONESYNCING ssh://someuser@git-annex-host.vpshost.net-someuser_host_repo/~/host_repo/ 1 + +[2014-07-16 20:30:47 CDT] read: git [\"--git-dir=/Users/someuser/local_annex/.git\",\"--work-tree=/Users/someuser/local_annex\",\"-c\",\"core.bare=false\",\"show-ref\",\"git-annex\"] + +[2014-07-16 20:30:47 CDT] read: git [\"--git-dir=/Users/someuser/local_annex/.git\",\"--work-tree=/Users/someuser/local_annex\",\"-c\",\"core.bare=false\",\"show-ref\",\"--hash\",\"refs/heads/git-annex\"] + +[2014-07-16 20:30:47 CDT] read: git [\"--git-dir=/Users/someuser/local_annex/.git\",\"--work-tree=/Users/someuser/local_annex\",\"-c\",\"core.bare=false\",\"log\",\"refs/heads/git-annex..e1057e037f03ba804f021c8c39073b2efd1fad08\",\"--oneline\",\"-n1\"] + +[2014-07-16 20:30:47 CDT] read: git [\"--git-dir=/Users/someuser/local_annex/.git\",\"--work-tree=/Users/someuser/local_annex\",\"-c\",\"core.bare=false\",\"log\",\"refs/heads/git-annex..bfef6b84186d59431377343ab968e255db8c76fa\",\"--oneline\",\"-n1\"] + +[2014-07-16 20:30:47 CDT] read: git [\"--git-dir=/Users/someuser/local_annex/.git\",\"--work-tree=/Users/someuser/local_annex\",\"-c\",\"core.bare=false\",\"log\",\"refs/heads/git-annex..d44a67a5795c28e423a9ac018bead9dac793be5c\",\"--oneline\",\"-n1\"] +(merging hostrepo/synced/git-annex into git-annex...) + +[2014-07-16 20:30:47 CDT] feed: git [\"--git-dir=/Users/someuser/local_annex/.git\",\"--work-tree=/Users/someuser/local_annex\",\"-c\",\"core.bare=false\",\"update-index\",\"-z\",\"--index-info\"] + +[2014-07-16 20:30:47 CDT] read: git [\"--git-dir=/Users/someuser/local_annex/.git\",\"--work-tree=/Users/someuser/local_annex\",\"-c\",\"core.bare=false\",\"diff-index\",\"--raw\",\"-z\",\"-r\",\"--no-renames\",\"-l0\",\"--cached\",\"d44a67a5795c28e423a9ac018bead9dac793be5c\"] + +[2014-07-16 20:30:47 CDT] read: git [\"--git-dir=/Users/someuser/local_annex/.git\",\"--work-tree=/Users/someuser/local_annex\",\"-c\",\"core.bare=false\",\"log\",\"d44a67a5795c28e423a9ac018bead9dac793be5c..refs/heads/git-annex\",\"--oneline\",\"-n1\"] + +[2014-07-16 20:30:47 CDT] call: git [\"--git-dir=/Users/someuser/local_annex/.git\",\"--work-tree=/Users/someuser/local_annex\",\"-c\",\"core.bare=false\",\"update-ref\",\"refs/heads/git-annex\",\"d44a67a5795c28e423a9ac018bead9dac793be5c\"] + +[2014-07-16 20:30:48 CDT] read: git [\"--git-dir=/Users/someuser/local_annex/.git\",\"--work-tree=/Users/someuser/local_annex\",\"-c\",\"core.bare=false\",\"show-ref\",\"git-annex\"] + +[2014-07-16 20:30:48 CDT] read: git [\"--git-dir=/Users/someuser/local_annex/.git\",\"--work-tree=/Users/someuser/local_annex\",\"-c\",\"core.bare=false\",\"show-ref\",\"--hash\",\"refs/heads/git-annex\"] + +[2014-07-16 20:30:48 CDT] read: git [\"--git-dir=/Users/someuser/local_annex/.git\",\"--work-tree=/Users/someuser/local_annex\",\"-c\",\"core.bare=false\",\"log\",\"refs/heads/git-annex..d44a67a5795c28e423a9ac018bead9dac793be5c\",\"--oneline\",\"-n1\"] + +[2014-07-16 20:30:48 CDT] read: git [\"--git-dir=/Users/someuser/local_annex/.git\",\"--work-tree=/Users/someuser/local_annex\",\"-c\",\"core.bare=false\",\"log\",\"refs/heads/git-annex..bfef6b84186d59431377343ab968e255db8c76fa\",\"--oneline\",\"-n1\"] + +[2014-07-16 20:30:48 CDT] feed: git [\"--git-dir=/Users/someuser/local_annex/.git\",\"--work-tree=/Users/someuser/local_annex\",\"-c\",\"core.bare=false\",\"update-index\",\"-z\",\"--index-info\"] + +[2014-07-16 20:30:48 CDT] read: git [\"--git-dir=/Users/someuser/local_annex/.git\",\"--work-tree=/Users/someuser/local_annex\",\"-c\",\"core.bare=false\",\"diff-index\",\"--raw\",\"-z\",\"-r\",\"--no-renames\",\"-l0\",\"--cached\",\"refs/heads/git-annex\"] + +[2014-07-16 20:30:51 CD[T]2 0c1a4l-l:0 7g-it1 6[ \"2-0-:g3i0t:-d5i1r =C/DUTs]e rRse/mmohtielCtoyn/tsrhoiln:i eSrY_NaCnInNeGx /s.sghi:t/\"/,m\"a-t-ewoo@rgki-tt-raenen=e/xU-sgeortsa/.mihniflrtoyn/tsiheirnsi.enre_ta-nmnaetxe\"o,_\"g-oct\"a,_\"rceoproe/.~b/agroet=af_arlespeo\"/, +\"fetch\",\"hostrepo\"] + +[2014-07-16 20:30:51 CDT] RemoteControl: Syncing with hostrepo +From ssh://git-annex-host.vpshost.net-someuser_host_repo/~/host_repo + d44a67a..660d71a synced/git-annex -> hostrepo/synced/git-annex + +[2014-07-16 20:30:51 CDT] RemoteControl: DONESYNCING ssh://someuser@git-annex-host.vpshost.net-someuser_host_repo/~/host_repo/ 1 + +[2014-07-16 20:30:51 CDT] read: git [\"--git-dir=/Users/someuser/local_annex/.git\",\"--work-tree=/Users/someuser/local_annex\",\"-c\",\"core.bare=false\",\"show-ref\",\"git-annex\"] + +[2014-07-16 20:30:51 CDT] read: git [\"--git-dir=/Users/someuser/local_annex/.git\",\"--work-tree=/Users/someuser/local_annex\",\"-c\",\"core.bare=false\",\"show-ref\",\"--hash\",\"refs/heads/git-annex\"] + +[2014-07-16 20:30:51 CDT] read: git [\"--git-dir=/Users/someuser/local_annex/.git\",\"--work-tree=/Users/someuser/local_annex\",\"-c\",\"core.bare=false\",\"log\",\"refs/heads/git-annex..d44a67a5795c28e423a9ac018bead9dac793be5c\",\"--oneline\",\"-n1\"] + +[2014-07-16 20:30:51 CDT] read: git [\"--git-dir=/Users/someuser/local_annex/.git\",\"--work-tree=/Users/someuser/local_annex\",\"-c\",\"core.bare=false\",\"log\",\"refs/heads/git-annex..bfef6b84186d59431377343ab968e255db8c76fa\",\"--oneline\",\"-n1\"] + +[2014-07-16 20:30:52 CDT] read: git [\"--git-dir=/Users/someuser/local_annex/.git\",\"--work-tree=/Users/someuser/local_annex\",\"-c\",\"core.bare=false\",\"log\",\"refs/heads/git-annex..660d71ac55b47061c531e5c8fa2af7910f2e13aa\",\"--oneline\",\"-n1\"] +(merging hostrepo/synced/git-annex into git-annex...) + +[2014-07-16 20:30:52 CDT] feed: git [\"--git-dir=/Users/someuser/local_annex/.git\",\"--work-tree=/Users/someuser/local_annex\",\"-c\",\"core.bare=false\",\"update-index\",\"-z\",\"--index-info\"] + +[2014-07-16 20:30:52 CDT] read: git [\"--git-dir=/Users/someuser/local_annex/.git\",\"--work-tree=/Users/someuser/local_annex\",\"-c\",\"core.bare=false\",\"diff-index\",\"--raw\",\"-z\",\"-r\",\"--no-renames\",\"-l0\",\"--cached\",\"660d71ac55b47061c531e5c8fa2af7910f2e13aa\"] + +[2014-07-16 20:30:52 CDT] read: git [\"--git-dir=/Users/someuser/local_annex/.git\",\"--work-tree=/Users/someuser/local_annex\",\"-c\",\"core.bare=false\",\"log\",\"660d71ac55b47061c531e5c8fa2af7910f2e13aa..refs/heads/git-annex\",\"--oneline\",\"-n1\"] + +[2014-07-16 20:30:52 CDT] call: git [\"--git-dir=/Users/someuser/local_annex/.git\",\"--work-tree=/Users/someuser/local_annex\",\"-c\",\"core.bare=false\",\"update-ref\",\"refs/heads/git-annex\",\"660d71ac55b47061c531e5c8fa2af7910f2e13aa\"] + +[2014-07-16 20:30:52 CDT] read: git [\"--git-dir=/Users/someuser/local_annex/.git\",\"--work-tree=/Users/someuser/local_annex\",\"-c\",\"core.bare=false\",\"show-ref\",\"git-annex\"] + +[2014-07-16 20:30:52 CDT] read: git [\"--git-dir=/Users/someuser/local_annex/.git\",\"--work-tree=/Users/someuser/local_annex\",\"-c\",\"core.bare=false\",\"show-ref\",\"--hash\",\"refs/heads/git-annex\"] + +[2014-07-16 20:30:52 CDT] read: git [\"--git-dir=/Users/someuser/local_annex/.git\",\"--work-tree=/Users/someuser/local_annex\",\"-c\",\"core.bare=false\",\"log\",\"refs/heads/git-annex..660d71ac55b47061c531e5c8fa2af7910f2e13aa\",\"--oneline\",\"-n1\"] + +[2014-07-16 20:30:53 CDT] read: git [\"--git-dir=/Users/someuser/local_annex/.git\",\"--work-tree=/Users/someuser/local_annex\",\"-c\",\"core.bare=false\",\"log\",\"refs/heads/git-annex..bfef6b84186d59431377343ab968e255db8c76fa\",\"--oneline\",\"-n1\"] + +[2014-07-16 20:30:53 CDT] feed: git [\"--git-dir=/Users/someuser/local_annex/.git\",\"--work-tree=/Users/someuser/local_annex\",\"-c\",\"core.bare=false\",\"update-index\",\"-z\",\"--index-info\"] + +[2014-07-16 20:30:53 CDT] read: git [\"--git-dir=/Users/someuser/local_annex/.git\",\"--work-tree=/Users/someuser/local_annex\",\"-c\",\"core.bare=false\",\"diff-index\",\"--raw\",\"-z\",\"-r\",\"--no-renames\",\"-l0\",\"--cached\",\"refs/heads/git-annex\"] + +[2014-07-16 20[:32101:4-007-71 6 C2D0T:]3 1:ca0l7l :C DgTi]t Re[m\"ot-e-Cognitrtol-: dSiYNrCI=NG/ Usssh:e/r/msa/tmeoh@giitl-tayn/nesx-hgoitan.iinefrron_tiaerns.nneetx-/m.agtieto\"_,g\"o-t-a_wroepro/k~/-gottar_reepeo/= +/Users/someuser/local_ann[ex\"2,0\"1-4c-\",0\"7c-o1r6e .2b0a:r3e1=:fa0l7s eC\"D,T\"]f eRtecmho\"t,e\"Cgoonttarroelp:o \"] +Syncing with hostrepo +From ssh://git-annex-host.vpshost.net-someuser_host_repo/~/host_repo + 660d71a..273bc7f synced/git-annex -> hostrepo/synced/git-annex + +[2014-07-16 20:31:09 CDT] RemoteControl: DONESYNCING ssh://someuser@git-annex-host.vpshost.net-someuser_host_repo/~/host_repo/ 1 + +[2014-07-16 20:31:09 CDT] read: git [\"--git-dir=/Users/someuser/local_annex/.git\",\"--work-tree=/Users/someuser/local_annex\",\"-c\",\"core.bare=false\",\"show-ref\",\"git- +annex\"] + +[2014-07-16 20:31:09 CDT] read: git [\"--git-dir=/Users/someuser/local_annex/.git\",\"--work-tree=/Users/someuser/local_annex\",\"-c\",\"core.bare=false\",\"show-ref\",\"--hash\",\"refs/heads/git-annex\"] + +[2014-07-16 20:31:09 CDT] read: git [\"--git-dir=/Users/someuser/local_annex/.git\",\"--work-tree=/Users/someuser/local_annex\",\"-c\",\"core.bare=false\",\"log\",\"refs/heads/git-annex..660d71ac55b47061c531e5c8fa2af7910f2e13aa\",\"--oneline\",\"-n1\"] + +[2014-07-16 20:31:09 CDT] read: git [\"--git-dir=/Users/someuser/local_annex/.git\",\"--work-tree=/Users/someuser/local_annex\",\"-c\",\"core.bare=false\",\"log\",\"refs/heads/git-annex..bfef6b84186d59431377343ab968e255db8c76fa\",\"--oneline\",\"-n1\"] + +[2014-07-16 20:31:09 CDT] read: git [\"--git-dir=/Users/someuser/local_annex/.git\",\"--work-tree=/Users/someuser/local_annex\",\"-c\",\"core.bare=false\",\"log\",\"refs/heads/git-annex..273bc7f261a79539abcc237f1f0bbf5cd4f57add\",\"--oneline\",\"-n1\"] +(merging hostrepo/synced/git-annex into git-annex...) + +[2014-07-16 20:31:09 CDT] feed: git [\"--git-dir=/Users/someuser/local_annex/.git\",\"--work-tree=/Users/someuser/local_annex\",\"-c\",\"core.bare=false\",\"update-index\",\"-z\",\"--index-info\"] + +[2014-07-16 20:31:09 CDT] read: git [\"--git-dir=/Users/someuser/local_annex/.git\",\"--work-tree=/Users/someuser/local_annex\",\"-c\",\"core.bare=false\",\"diff-index\",\"--raw\",\"-z\",\"-r\",\"--no-renames\",\"-l0\",\"--cached\",\"273bc7f261a79539abcc237f1f0bbf5cd4f57add\"] + +[2014-07-16 20:31:09 CDT] read: git [\"--git-dir=/Users/someuser/local_annex/.git\",\"--work-tree=/Users/someuser/local_annex\",\"-c\",\"core.bare=false\",\"log\",\"273bc7f261a79539abcc237f1f0bbf5cd4f57add..refs/heads/git-annex\",\"--oneline\",\"-n1\"] + +[2014-07-16 20:31:09 CDT] call: git [\"--git-dir=/Users/someuser/local_annex/.git\",\"--work-tree=/Users/someuser/local_annex\",\"-c\",\"core.bare=false\",\"update-ref\",\"refs/heads/git-annex\",\"273bc7f261a79539abcc237f1f0bbf5cd4f57add\"] + +[2014-07-16 20:31:10 CDT] read: git [\"--git-dir=/Users/someuser/local_annex/.git\",\"--work-tree=/Users/someuser/local_annex\",\"-c\",\"core.bare=false\",\"show-ref\",\"git-annex\"] + +[2014-07-16 20:31:10 CDT] read: git [\"--git-dir=/Users/someuser/local_annex/.git\",\"--work-tree=/Users/someuser/local_annex\",\"-c\",\"core.bare=false\",\"show-ref\",\"--hash\",\"refs/heads/git-annex\"] + +[2014-07-16 20:31:10 CDT] read: git [\"--git-dir=/Users/someuser/local_annex/.git\",\"--work-tree=/Users/someuser/local_annex\",\"-c\",\"core.bare=false\",\"log\",\"refs/heads/git-annex..273bc7f261a79539abcc237f1f0bbf5cd4f57add\",\"--oneline\",\"-n1\"] + +[2014-07-16 20:31:10 CDT] read: git [\"--git-dir=/Users/someuser/local_annex/.git\",\"--work-tree=/Users/someuser/local_annex\",\"-c\",\"core.bare=false\",\"log\",\"refs/heads/git-annex..bfef6b84186d59431377343ab968e255db8c76fa\",\"--oneline\",\"-n1\"] + +[2014-07-16 20:31:10 CDT] feed: git [\"--git-dir=/Users/someuser/local_annex/.git\",\"--work-tree=/Users/someuser/local_annex\",\"-c\",\"core.bare=false\",\"update-index\",\"-z\",\"--index-info\"] + +[2014-07-16 20:31:10 CDT] read: git [\"--git-dir=/Users/someuser/local_annex/.git\",\"--work-tree=/Users/someuser/local_annex\",\"-c\",\"core.bare=false\",\"diff-index\",\"--raw\",\"-z\",\"-r\",\"--no-renames\",\"-l0\",\"--cached\",\"refs/heads/git-annex\"] + +[2014-07-16 20:31:11 CDT] Watcher: directory deleted /Users/someuser/local_annex/local_annex-test + +[2014-07-16 20:31:11 CDT] read: git [\"--git-dir=/Users/someuser/local_annex/.git\",\"--work-tree=/Users/someuser/local_annex\",\"-c\",\"core.bare=false\",\"ls-files\",\"--deleted\",\"-z\",\"--\",\"/Users/someuser/local_annex/local_annex-test\"] + +[2014-07-16 20:31[:1270 1C4D-T]0 7c-al1l6: 2g0i:t3 1[:\"-1-7g iCtD-Td]i rR=e/mUosteerCso/nmthriollt:y /SsYhNiCnIiNeGr _sasnhn:e/x//m.agtieto\"@,g\"i-t--waonrnke-xt-rgeoet=a/.Uisnefrrso/nmthiielrtsy./nsehti-nmiaetre_oa_ngnoetxa\"_r,e\"po-/c~\"/g,o\"tcao_rreep.ob/a +re=false\",\"fetch\",\"g[ot2a0r1e4p-o0\"7]- +16 20:31:17 CDT] RemoteControl: Syncing with hostrepo +From ssh://git-annex-host.vpshost.net-someuser_host_repo/~/host_repo + 273bc7f..ce6dd68 synced/git-annex -> hostrepo/synced/git-annex + +[2014-07-16 20:31:18 CDT] RemoteControl: DONESYNCING ssh://someuser@git-annex-host.vpshost.net-someuser_host_repo/~/host_repo/ 1 + +[2014-07-16 20:31:18 CDT] read: git [\"--git-dir=/Users/someuser/local_annex/.git\",\"--work-tree=/Users/someuser/local_annex\",\"-c\",\"core.bare=false\",\"show-ref\",\"git-annex\"] + +[2014-07-16 20:31:18 CDT] read: git [\"--git-dir=/Users/someuser/local_annex/.git\",\"--work-tree=/Users/someuser/local_annex\",\"-c\",\"core.bare=false\",\"show-ref\",\"--hash\",\"refs/heads/git-annex\"] + +[2014-07-16 20:31:18 CDT] read: git [\"--git-dir=/Users/someuser/local_annex/.git\",\"--work-tree=/Users/someuser/local_annex\",\"-c\",\"core.bare=false\",\"log\",\"refs/heads/git-annex..273bc7f261a79539abcc237f1f0bbf5cd4f57add\",\"--oneline\",\"-n1\"] + +[2014-07-16 20:31:18 CDT] read: git [\"--git-dir=/Users/someuser/local_annex/.git\",\"--work-tree=/Users/someuser/local_annex\",\"-c\",\"core.bare=false\",\"log\",\"refs/heads/git-annex..bfef6b84186d59431377343ab968e255db8c76fa\",\"--oneline\",\"-n1\"] + +[2014-07-16 20:31:18 CDT] read: git [\"--git-dir=/Users/someuser/local_annex/.git\",\"--work-tree=/Users/someuser/local_annex\",\"-c\",\"core.bare=false\",\"log\",\"refs/heads/git-annex..ce6dd68adec97bbf73f5da408c17511aca8840c5\",\"--oneline\",\"-n1\"] +(merging hostrepo/synced/git-annex into git-annex...) + +[2014-07-16 20:31:18 CDT] feed: git [\"--git-dir=/Users/someuser/local_annex/.git\",\"--work-tree=/Users/someuser/local_annex\",\"-c\",\"core.bare=false\",\"update-index\",\"-z\",\"--index-info\"] + +[2014-07-16 20:31:18 CDT] read: git [\"--git-dir=/Users/someuser/local_annex/.git\",\"--work-tree=/Users/someuser/local_annex\",\"-c\",\"core.bare=false\",\"diff-index\",\"--raw\",\"-z\",\"-r\",\"--no-renames\",\"-l0\",\"--cached\",\"ce6dd68adec97bbf73f5da408c17511aca8840c5\"] + +[2014-07-16 20:31:18 CDT] read: git [\"--git-dir=/Users/someuser/local_annex/.git\",\"--work-tree=/Users/someuser/local_annex\",\"-c\",\"core.bare=false\",\"log\",\"ce6dd68adec97bbf73f5da408c17511aca8840c5..refs/heads/git-annex\",\"--oneline\",\"-n1\"] + +[2014-07-16 20:31:18 CDT] call: git [\"--git-dir=/Users/someuser/local_annex/.git\",\"--work-tree=/Users/someuser/local_annex\",\"-c\",\"core.bare=false\",\"update-ref\",\"refs/heads/git-annex\",\"ce6dd68adec97bbf73f5da408c17511aca8840c5\"] + +[2014-07-16 20:31:19 CDT] read: git [\"--git-dir=/Users/someuser/local_annex/.git\",\"--work-tree=/Users/someuser/local_annex\",\"-c\",\"core.bare=false\",\"show-ref\",\"git-annex\"] + +[2014-07-16 20:31:19 CDT] read: git [\"--git-dir=/Users/someuser/local_annex/.git\",\"--work-tree=/Users/someuser/local_annex\",\"-c\",\"core.bare=false\",\"show-ref\",\"--hash\",\"refs/heads/git-annex\"] + +[2014-07-16 20:31:19 CDT] read: git [\"--git-dir=/Users/someuser/local_annex/.git\",\"--work-tree=/Users/someuser/local_annex\",\"-c\",\"core.bare=false\",\"log\",\"refs/heads/git-annex..ce6dd68adec97bbf73f5da408c17511aca8840c5\",\"--oneline\",\"-n1\"] + +[2014-07-16 20:31:19 CDT] read: git [\"--git-dir=/Users/someuser/local_annex/.git\",\"--work-tree=/Users/someuser/local_annex\",\"-c\",\"core.bare=false\",\"log\",\"refs/heads/git-annex..bfef6b84186d59431377343ab968e255db8c76fa\",\"--oneline\",\"-n1\"] + +[2014-07-16 20:31:19 CDT] feed: git [\"--git-dir=/Users/someuser/local_annex/.git\",\"--work-tree=/Users/someuser/local_annex\",\"-c\",\"core.bare=false\",\"update-index\",\"-z\",\"--index-info\"] + +[2014-07-16 20:31:19 CDT] read: git [\"--git-dir=/Users/someuser/local_annex/.git\",\"--work-tree=/Users/someuser/local_annex\",\"-c\",\"core.bare=false\",\"diff-index\",\"--raw\",\"-z\",\"-r\",\"--no-renames\",\"-l0\",\"--cached\",\"refs/heads/git-annex\"] + +[2014-07-16 20:31:32 CDT] read: git [\"--git-dir=/Users/someuser/local_annex/.git\",\"--work-tree=/Users/someuser/local_annex\",\"-c\",\"core.bare=false\",\"ls-tree\",\"--full-tree\",\"-z\",\"--\",\"refs/heads/git-annex\",\"uuid.log\",\"remote.log\",\"trust.log\",\"group.log\",\"numcopies.log\",\"schedule.log\",\"preferred-content.log\",\"required-content.log\",\"group-preferred-content.log\"] + +"""]] diff --git a/doc/bugs/OSX_case_insensitive_filesystem.mdwn b/doc/bugs/OSX_case_insensitive_filesystem.mdwn new file mode 100644 index 0000000000..cbdcea99ba --- /dev/null +++ b/doc/bugs/OSX_case_insensitive_filesystem.mdwn @@ -0,0 +1,56 @@ +### Please describe the problem. + +I copied one entire folder from my OSX local machine to a linux server. +That folder was version controlled by git with git annex big files. +On the linux machine the symlinks were broken. + +In particular, on my local machine, a file like: +``` +file.vcf.gz.tbi -> ../../.git/annex/objects/J4/Pg/SHA256E-s572463--85b357849ddad75fc1138b27d6af62cf410876e329ff035f21a631bd53146224.gz.tbi/SHA256E-s572463--85b357849ddad75fc1138b27d6af62cf410876e329ff035f21a631bd53146224.gz.tbi +``` + +but the file resides in: + +``` +../../.git/annex/objects/j4/Pg/SHA256E-s572463--85b357849ddad75fc1138b27d6af62cf410876e329ff035f21a631bd53146224.gz.tbi/SHA256E-s572463--85b357849ddad75fc1138b27d6af62cf410876e329ff035f21a631bd53146224.gz.tbi +``` + +Notice the difference between `J4` and `j4`. This is not a problem on my OSX but becomes one on a linux machine. + +### What steps will reproduce the problem? + +on local machine: +```rsync -aztv folder/ remote-machine:folder/``` + +on server: +```find . -type l -exec sh -c "file -b {} | grep -q ^broken" \; -print``` +listed every symlink as broken. + +### What version of git-annex are you using? On what operating system? + +git annex version 6.20160318 + +my special remote is rsync.net + +OSX 10.11.4 + +Not sure of which option were used when creating ther file system, but I suspect a HFS+ case insensitive. + +### Please provide any additional information below. + +I wanted to do this full copy because the transfer speed from the special remote was too slow. +Since then I got decent speeds, and if I do `git annex get .` on the server, all is fine. +So, my "problem" is solved but I'm still wondering why the discrepant folder capitalization. + +[[!format sh """ +# If you can, paste a complete transcript of the problem occurring here. +# If the problem is with the git-annex assistant, paste in .git/annex/daemon.log + + +# End of transcript or log. +"""]] + +### Have you had any luck using git-annex before? (Sometimes we get tired of reading bug reports all day and a lil' positive end note does wonders) + +Sure, it works marvels :-) Also what I was trying to do is perhaps not by the book... + diff --git a/doc/bugs/OSX_case_insensitive_filesystem/comment_1_2e81165ac03e1d0566c81016e7728ee6._comment b/doc/bugs/OSX_case_insensitive_filesystem/comment_1_2e81165ac03e1d0566c81016e7728ee6._comment new file mode 100644 index 0000000000..01f355ec1d --- /dev/null +++ b/doc/bugs/OSX_case_insensitive_filesystem/comment_1_2e81165ac03e1d0566c81016e7728ee6._comment @@ -0,0 +1,29 @@ +[[!comment format=mdwn + username="joey" + subject="""comment 1""" + date="2016-04-20T18:14:25Z" + content=""" +So the directory structure got lower cased when you copied it from OSX to +linux. OSX remembered a lower-case name for the J4 directory, for example, +and propigated that over to linux. + +Unfortunately, git-annex is stuck using mixed case hash directories for +backwards compatability reasons. Changing to all lower-case hash +directories would need every git-annex repo to be converted; would +invalidate all old tags and branches and history in the repos, etc. +This is discussed in [[design/new_repo_versions]]. + +It's actually possible to make brand-new git-annex repos use all lower case +hash directories today, by setting `git config annex.tune.objecthashlower true` +before you run `git annex init` for the first time. + +If you know you will need to move a repository between case-insensative and +case-sensative filesystems, you could use that configuration. But that +would be very forward looking, and instead users are just going to stumble +over the mixed case directories from time to time. + +What I'd recommend you do is, move the repository back to OSX, and then +make a clone of it on the linux system, and use `git annex move --all +--from origin` to move all the annexed file contents over from OSX to +linux. This method avoids the problem entirely. +"""]] diff --git a/doc/bugs/OSX_case_insensitive_filesystem/comment_2_4472d5db7a1cf7242077184912e32e17._comment b/doc/bugs/OSX_case_insensitive_filesystem/comment_2_4472d5db7a1cf7242077184912e32e17._comment new file mode 100644 index 0000000000..4f841513eb --- /dev/null +++ b/doc/bugs/OSX_case_insensitive_filesystem/comment_2_4472d5db7a1cf7242077184912e32e17._comment @@ -0,0 +1,10 @@ +[[!comment format=mdwn + username="http://id.clacke.se/" + nickname="clacke" + subject="Case-sensitive on OSX" + date="2016-04-22T07:53:56Z" + content=""" +> What I'd recommend you do is, move the repository back to OSX, and then make a clone of it on the linux system + +Alternatively, create a case-sensitive FS on the Mac (just in a .dmg) and do the clone and copy to there, before rsync'ing to the remote Linux. +"""]] diff --git a/doc/bugs/OSX_case_insensitive_filesystem/comment_3_4be758c2b70c0aadfe4fc9ecff8e4522._comment b/doc/bugs/OSX_case_insensitive_filesystem/comment_3_4be758c2b70c0aadfe4fc9ecff8e4522._comment new file mode 100644 index 0000000000..dc6b89f910 --- /dev/null +++ b/doc/bugs/OSX_case_insensitive_filesystem/comment_3_4be758c2b70c0aadfe4fc9ecff8e4522._comment @@ -0,0 +1,10 @@ +[[!comment format=mdwn + username="http://id.clacke.se/" + nickname="clacke" + subject="objecthashlower" + date="2016-04-22T07:55:40Z" + content=""" +> It's actually possible to make brand-new git-annex repos use all lower case hash directories today, by setting git config annex.tune.objecthashlower true before you run git annex init for the first time. + +Am I interpreting this correctly, that this sets some attribute in the git-annex branch, so that clones of this repo will use the same annex layout? +"""]] diff --git a/doc/bugs/OSX_case_insensitive_filesystem/comment_4_0b0c95a902718dd4e0976bf4e2cf265b._comment b/doc/bugs/OSX_case_insensitive_filesystem/comment_4_0b0c95a902718dd4e0976bf4e2cf265b._comment new file mode 100644 index 0000000000..29a53e04f3 --- /dev/null +++ b/doc/bugs/OSX_case_insensitive_filesystem/comment_4_0b0c95a902718dd4e0976bf4e2cf265b._comment @@ -0,0 +1,8 @@ +[[!comment format=mdwn + username="http://id.clacke.se/" + nickname="clacke" + subject="objecthashlower" + date="2016-04-22T07:58:33Z" + content=""" +... i.e., can I just `git config --global annex.tune.objecthashlower true` and it will Just Work, managing old repos correctly and using the feature for new repos? And remote machines will handle things correctly just as long as they run a modern enough `git-annex`? +"""]] diff --git a/doc/bugs/OSX_case_insensitive_filesystem/comment_5_2123944418123c3aaf187d34df5c11e0._comment b/doc/bugs/OSX_case_insensitive_filesystem/comment_5_2123944418123c3aaf187d34df5c11e0._comment new file mode 100644 index 0000000000..e8c9c3efe0 --- /dev/null +++ b/doc/bugs/OSX_case_insensitive_filesystem/comment_5_2123944418123c3aaf187d34df5c11e0._comment @@ -0,0 +1,11 @@ +[[!comment format=mdwn + username="joey" + subject="""comment 5""" + date="2016-04-22T18:41:22Z" + content=""" +You should not set annex.tune.* globally. Only set it when +initializing a new git-annex repo for the first time. Clones of the repo will +automatically inherit the tunings. + +See [[tuning]]. +"""]] diff --git a/doc/bugs/OSX_case_insensitive_filesystem/comment_6_f82c803d725829722842c6d54e388719._comment b/doc/bugs/OSX_case_insensitive_filesystem/comment_6_f82c803d725829722842c6d54e388719._comment new file mode 100644 index 0000000000..b65c0787b5 --- /dev/null +++ b/doc/bugs/OSX_case_insensitive_filesystem/comment_6_f82c803d725829722842c6d54e388719._comment @@ -0,0 +1,8 @@ +[[!comment format=mdwn + username="http://id.clacke.se/" + nickname="clacke" + subject="comment 6" + date="2016-04-22T19:01:45Z" + content=""" +But it only checks the parameter at `annex init` time? What will the bad consequences be if it is set at other times? +"""]] diff --git a/doc/bugs/OSX_case_insensitive_filesystem/comment_7_45e0178ce9598d086f34a89701536eb5._comment b/doc/bugs/OSX_case_insensitive_filesystem/comment_7_45e0178ce9598d086f34a89701536eb5._comment new file mode 100644 index 0000000000..e1a5fec74d --- /dev/null +++ b/doc/bugs/OSX_case_insensitive_filesystem/comment_7_45e0178ce9598d086f34a89701536eb5._comment @@ -0,0 +1,43 @@ +[[!comment format=mdwn + username="thowz" + subject="create symlinks inside .git/annex/objects" + date="2016-04-27T00:36:57Z" + content=""" +I also ran into this problem when copying a folder from Mac to Linux. Because the HFS+ filesystem is case-insensitive, directories that should be multiple directories are instead collapsed into one. For example, if there are four different files that are supposed to go into folders FG, fG, Fg, and fg, they will all end up in just one folder (whichever of the four was created first). But the symlinks will still point to different cases, so when everything gets moved to a case-sensitive filesystem, the symlinks break. + +I did a quick fix on the Linux side by making symlinks within .git/annex/objects to account for all the case variations (if folder FG exists and has more than one sub-folder, create symlinks fG --> FG, Fg --> FG, fg --> FG). + +To do this for the whole set of directories and sub-directories, I made a perl script: + + #!/usr/bin/perl + + $dir=$ARGV[0]; + $dir=substr($dir,2,2); + $link=$dir; + + if($dir =~ /[a-zA-Z]\d/){ + substr($link,0,1) ^= \" \"; + system(\"ln\", \"-s\", $dir, $link); + } + + if($dir =~ /\d[a-zA-Z]/){ + substr($link,1,1) ^= \" \"; + system(\"ln\", \"-s\", $dir, $link); + } + + if($dir =~ /[a-zA-Z]{2}/){ + substr($link,0,1) ^= \" \"; + system(\"ln\", \"-s\", $dir, $link); + substr($link,1,1) ^= \" \"; + system(\"ln\", \"-s\", $dir, $link); + substr($link,0,1) ^= \" \"; + system(\"ln\", \"-s\", $dir, $link); + } + + +And ran it this way (with script made executable and placed in $PATH): + + find .git/annex/objects -mindepth 1 -maxdepth 2 -links +3 -type d -execdir makelinks.pl {} \; + +It might be nice if git-annex-fsck could check whether a file was \"misfiled\" into the wrong directory due to case-insensitivity (before concluding that the file is missing). Although it also makes sense just to recommend cloning instead of copying when moving to a different filesystem (or remembering to set `annex.tune.objecthashlower true` when initiating a repo on a case-insensitive system). +"""]] diff --git a/doc/bugs/OSX_case_insensitive_filesystem/comment_8_3384bca070d84220efe166a398eed8a4._comment b/doc/bugs/OSX_case_insensitive_filesystem/comment_8_3384bca070d84220efe166a398eed8a4._comment new file mode 100644 index 0000000000..0dfa655077 --- /dev/null +++ b/doc/bugs/OSX_case_insensitive_filesystem/comment_8_3384bca070d84220efe166a398eed8a4._comment @@ -0,0 +1,17 @@ +[[!comment format=mdwn + username="thowz" + subject="reinject files" + date="2016-05-04T00:52:38Z" + content=""" +Here's a way to put files into their proper case-sensitive folders using `git-annex reinject --known`: + + # Enable write permissions on directories containing misfiled items + chmod u+w `find -xtype l -printf \"%l\n\" | sed -r \"s/.*(\.git\/annex\/objects\/)[^\/]*\/[^\/]*\/([^\/]*).*/\1*\/*\/\2/\" | sort | uniq` + + # Reinject those files into the annex + git-annex reinject --known `find -xtype l -printf \"%l\n\" | sed -r \"s/.*(\.git\/annex\/objects\/)[^\/]*\/[^\/]*\/([^\/]*.*)/\1*\/*\/\2/\" | sort | uniq` + + # Remove empty directories (rmdir will fail on the non-empty directories) + find .git/annex/objects -mindepth 3 -maxdepth 3 -type d -exec rmdir {} \; + find .git/annex/objects -mindepth 2 -maxdepth 2 -type d -exec rmdir {} \; +"""]] diff --git a/doc/bugs/Offline_editing_in_Android_removes_files_and_creates_links.mdwn b/doc/bugs/Offline_editing_in_Android_removes_files_and_creates_links.mdwn new file mode 100644 index 0000000000..9dd05d1d40 --- /dev/null +++ b/doc/bugs/Offline_editing_in_Android_removes_files_and_creates_links.mdwn @@ -0,0 +1,45 @@ +### Please describe the problem. +Two androids, A and B, and a computer, C. All running git annex assistant. +If you edit a file on A while A is offline, when A is back online and reconnects, that file will disappear from C and, in B, it will be substituted by a link +If you create a new file on A, that file will appear as a link in B and in C + + +### What steps will reproduce the problem? +- Have A, B, and C be connected sharing some repo. Create a file, file1. Let it propagate, so file1 is on A, B, C. + +- Take A offline (e.g., turn wifi off). + +- Edit file1 in A. + +- Create file2 in A. + +- (Let those files appear in the repo in A as in +[2015-02-27 20:55:04 CET] Committer: Adding file1 +add file1 ok +) + +- Turn the wifi of A on again. + +- Sync A from the webapp (clik on sync now) + +- In B (the other android) both file1 and file2 will contain just links, not the actual content of file1 and file2. + +- In the computer, file1 will have disappeared and file2 will be a link that points nowhere. (The link that file2 points too, as a string, are the contents of file2 in B). + + + +### What version of git-annex are you using? On what operating system? +Computer: 5.20141125 + +Androids: 5.20150226-g9c72d37 and 5.20150224-g9dca034 + + +### Please provide any additional information below. + +[[!format sh """ +# If you can, paste a complete transcript of the problem occurring here. +# If the problem is with the git-annex assistant, paste in .git/annex/daemon.log + + +# End of transcript or log. +"""]] diff --git a/doc/bugs/Offline_editing_in_Android_removes_files_and_creates_links/comment_1_a15eb9cd1a7bcdfa4b16459ef978a019._comment b/doc/bugs/Offline_editing_in_Android_removes_files_and_creates_links/comment_1_a15eb9cd1a7bcdfa4b16459ef978a019._comment new file mode 100644 index 0000000000..236f7f07d3 --- /dev/null +++ b/doc/bugs/Offline_editing_in_Android_removes_files_and_creates_links/comment_1_a15eb9cd1a7bcdfa4b16459ef978a019._comment @@ -0,0 +1,19 @@ +[[!comment format=mdwn + username="https://www.google.com/accounts/o8/id?id=AItOawnbPKLjBONawBd74MKJZo05juCqdsP1jAU" + nickname="Ramon" + subject="comment 1" + date="2015-02-27T22:06:51Z" + content=""" +I can confirm the above using, in the computer + +5.20150219-g52daae5 + +and 5.20150226-g9c72d37 + +In the computer, files that were edited in the Android while offline, might not always disappear, but might reappear as content after first being shown as links that point nowhere. Or may remain as links that point nowhere. + +In the Androids files that contain links (instead of original content) continue containint the links for over 1 hour. + +Binary (e.g., PDFs) and plain text files do not always behave the same. Plain text files are more easily affected (whereas PDFs are back to what they should be sooner). + +"""]] diff --git a/doc/bugs/Offline_editing_in_Android_removes_files_and_creates_links/comment_2_1c061621f0abe435182d763cd871f405._comment b/doc/bugs/Offline_editing_in_Android_removes_files_and_creates_links/comment_2_1c061621f0abe435182d763cd871f405._comment new file mode 100644 index 0000000000..2408ccf266 --- /dev/null +++ b/doc/bugs/Offline_editing_in_Android_removes_files_and_creates_links/comment_2_1c061621f0abe435182d763cd871f405._comment @@ -0,0 +1,15 @@ +[[!comment format=mdwn + username="joey" + subject="""comment 2""" + date="2015-07-07T19:26:27Z" + content=""" +These links are what is put in place if the content of a file has not yet +reached the local repository. On a more featureful system than Android, +it's a broken symlink, which is nicer than a link target. + +So, unless the content has reached the local repository, this is normal. + +There's a feature request somewhere, which I am having trouble finding, +that only files whose content is present be shown. I'm inclined to merge +this bug with that. +"""]] diff --git a/doc/bugs/On_Lubuntu_14.04_assistant_fails_to_create_new_setup_or_actually_work___40__fixed_by_regular_lxsession_package_update_from_2014-06-30__41__.mdwn b/doc/bugs/On_Lubuntu_14.04_assistant_fails_to_create_new_setup_or_actually_work___40__fixed_by_regular_lxsession_package_update_from_2014-06-30__41__.mdwn new file mode 100644 index 0000000000..d7788fa93f --- /dev/null +++ b/doc/bugs/On_Lubuntu_14.04_assistant_fails_to_create_new_setup_or_actually_work___40__fixed_by_regular_lxsession_package_update_from_2014-06-30__41__.mdwn @@ -0,0 +1,93 @@ +### Please describe the problem. + +Following steps from http://git-annex.branchable.com/assistant/quickstart/ does not get past the first screen. + +### What steps will reproduce the problem? + +* On a nearly fresh Lubuntu system, nearly fresh account, Lubuntu session, run `git-annex` from the start menu. +* Opens firefox, in `Create a git-annex repository` select my `~/Documents` folder, press `Make Repository`. + +* Expected: a new, different page opens. +* Observed: the same page opens, only difference the path displayed is `/home/mylogin/Documents`. Pressing `Make Repository` again shows the same page again. + +I couldn't find any logs at the time. Now I see that they are in .git/annex/daemon.log but content from that time is already gone. + +### What version of git-annex are you using? On what operating system? + +A fresh Ubuntu 14.04 (actually Lubuntu but this shouldn't change anything, right ?), create a user, login. + + + LC_ALL=C apt-cache policy git-annex + git-annex: + Installed: 5.20140412ubuntu1 + Candidate: 5.20140412ubuntu1 + Version table: + *** 5.20140412ubuntu1 0 + 500 http://fr.archive.ubuntu.com/ubuntu/ trusty/universe i386 Packages + 100 /var/lib/dpkg/status + + +### Please provide any additional information below. + +[[!format sh """ +# No command line in bug occurrence. This is the transcript of the workaround I found. +# Actually, the PC was rebooted then I went command line. +cd ~/Documents +git status +fatal: This operation must be run in a work tree +# oh I see it's a bare repo +git annex add . +add (my file names... one line for each) ok +(Recording state in git...) + +Now the webapp seems to run okay, I can see many pages, schedule checks, etc. + +# If the problem is with the git-annex assistant, paste in .git/annex/daemon.log +-> Ok but log shows only events after I manually dit git annex add. + +[2014-06-08 14:28:46 CEST] main: starting assistant version 5.20140412ubuntu1 +[2014-06-08 14:28:46 CEST] Cronner: You should enable consistency checking to protect your data. + + dbus failed; falling back to mtab polling (ClientError {clientErrorMessage = "runClient: unable to determine DBUS address", clientErrorFatal = True}) +(scanning...) [2014-06-08 14:28:46 CEST] Watcher: Performing startup scan +(started...) [2014-06-08 14:28:47 CEST] Committer: Adding (myfiles) + +add (my files...) ok +[2014-06-08 14:32:51 CEST] Committer: Committing changes to git +[2014-06-08 14:34:12 CEST] Committer: Committing changes to git +[2014-06-08 14:34:13 CEST] Committer: Adding (myfiles) +ok +(Recording state in git...) +(Recording state in git...) +(Recording state in git...) +(Recording state in git...) +(Recording state in git...) +add (myfile) ok +add (myfile) [2014-06-08 14:34:13 CEST] Committer: Committing changes to git +[2014-06-08 14:42:48 CEST] Cronner: Consistency check in progress +fsck (myfile) (checksum...) +ok + +# End of transcript or log. +"""]] + + + +# Question: + + cat /etc/xdg/autostart/git-annex.desktop + [Desktop Entry] + Type=Application + Version=1.0 + Name=Git Annex Assistant + Comment=Autostart + Terminal=false + Exec=/usr/bin/git-annex assistant --autostart + Categories= + +Should there be a git-annex process whenever I log in even if I don't launch the webapp ? +I can check that on next login. + +At the moment there is "git-annex webapp" and several child processes. + +[[!tag moreinfo]] diff --git a/doc/bugs/On_Lubuntu_14.04_assistant_fails_to_create_new_setup_or_actually_work___40__fixed_by_regular_lxsession_package_update_from_2014-06-30__41__/comment_1_7508685b6f676c72e316642b80e40ee8._comment b/doc/bugs/On_Lubuntu_14.04_assistant_fails_to_create_new_setup_or_actually_work___40__fixed_by_regular_lxsession_package_update_from_2014-06-30__41__/comment_1_7508685b6f676c72e316642b80e40ee8._comment new file mode 100644 index 0000000000..e92e1b1007 --- /dev/null +++ b/doc/bugs/On_Lubuntu_14.04_assistant_fails_to_create_new_setup_or_actually_work___40__fixed_by_regular_lxsession_package_update_from_2014-06-30__41__/comment_1_7508685b6f676c72e316642b80e40ee8._comment @@ -0,0 +1,10 @@ +[[!comment format=mdwn + username="http://joeyh.name/" + ip="108.236.230.124" + subject="comment 1" + date="2014-06-09T18:57:25Z" + content=""" +It sounds rather like the webapp is failing to create the repository. Although I would expect for it to show an error message in red beneath the repository path. OTOH, it also sounds like something (either the webapp or eg a manual git-annex run) set up a git-annex repository in ~/Documents. + +git-annex will be started automatically on login as long as `~/.config/git-annex/autostart` lists a repository to use. If that file exists and contains ~/Documents then we'll know that the webapp successfully set up the repository. +"""]] diff --git a/doc/bugs/On_Lubuntu_14.04_assistant_fails_to_create_new_setup_or_actually_work___40__fixed_by_regular_lxsession_package_update_from_2014-06-30__41__/comment_2_c4dd12ea578d1f07464e1b9d68ec96cf._comment b/doc/bugs/On_Lubuntu_14.04_assistant_fails_to_create_new_setup_or_actually_work___40__fixed_by_regular_lxsession_package_update_from_2014-06-30__41__/comment_2_c4dd12ea578d1f07464e1b9d68ec96cf._comment new file mode 100644 index 0000000000..a94d0c13a8 --- /dev/null +++ b/doc/bugs/On_Lubuntu_14.04_assistant_fails_to_create_new_setup_or_actually_work___40__fixed_by_regular_lxsession_package_update_from_2014-06-30__41__/comment_2_c4dd12ea578d1f07464e1b9d68ec96cf._comment @@ -0,0 +1,20 @@ +[[!comment format=mdwn + username="https://www.google.com/accounts/o8/id?id=AItOawm7eqCMh_B7mxE0tnchbr0JoYu11FUAFRY" + nickname="Stéphane" + subject="comment 2" + date="2014-06-09T20:34:55Z" + content=""" +> It sounds rather like the webapp is failing to create the repository. Although I would expect for it to show an error message in red beneath the repository path. + +No message in red at any time. + +> OTOH, it also sounds like something (either the webapp or eg a manual git-annex run) set up a git-annex repository in ~/Documents. + +There was none before the webapp. +After the webapp failure, there was one, but without any file. +Only after I manually did git annex add, it looked \"unlocked\". + +> git-annex will be started automatically on login as long as ~/.config/git-annex/autostart lists a repository to use. If that file exists and contains ~/Documents then we'll know that the webapp successfully set up the repository. + +Can't check now, the machine is off. Should check soon. Thanks. +"""]] diff --git a/doc/bugs/On_Lubuntu_14.04_assistant_fails_to_create_new_setup_or_actually_work___40__fixed_by_regular_lxsession_package_update_from_2014-06-30__41__/comment_3_8687c1d1c44d88a8ac13208273565d6c._comment b/doc/bugs/On_Lubuntu_14.04_assistant_fails_to_create_new_setup_or_actually_work___40__fixed_by_regular_lxsession_package_update_from_2014-06-30__41__/comment_3_8687c1d1c44d88a8ac13208273565d6c._comment new file mode 100644 index 0000000000..4e497e240a --- /dev/null +++ b/doc/bugs/On_Lubuntu_14.04_assistant_fails_to_create_new_setup_or_actually_work___40__fixed_by_regular_lxsession_package_update_from_2014-06-30__41__/comment_3_8687c1d1c44d88a8ac13208273565d6c._comment @@ -0,0 +1,37 @@ +[[!comment format=mdwn + username="https://www.google.com/accounts/o8/id?id=AItOawm7eqCMh_B7mxE0tnchbr0JoYu11FUAFRY" + nickname="Stéphane" + subject="No git-annex process." + date="2014-06-11T17:55:56Z" + content=""" +> It sounds rather like the webapp is failing to create the repository. Although I would expect for it to show an error message in red beneath the repository path. OTOH, it also sounds like something (either the webapp or eg a manual git-annex run) set up a git-annex repository in ~/Documents. + +As told before, there was definitely no git repository there before the webapp tried to create one. + +> git-annex will be started automatically on login as long as ~/.config/git-annex/autostart lists a repository to use. If that file exists and contains ~/Documents then we'll know that the webapp successfully set up the repository. + +There exists a ~/.config/git-annex/autostart containing one line, the correct /home/mylogin/Documents path to the folder. + +There is no process with git or annex in its name when user logs on his desktop session. + + + $ grep . $(dpkg -L git-annex | grep -i desktop) + /usr/share/applications/git-annex.desktop:[Desktop Entry] + /usr/share/applications/git-annex.desktop:Type=Application + /usr/share/applications/git-annex.desktop:Version=1.0 + /usr/share/applications/git-annex.desktop:Name=Git Annex + /usr/share/applications/git-annex.desktop:Comment=Track and sync the files in your Git Annex + /usr/share/applications/git-annex.desktop:Terminal=false + /usr/share/applications/git-annex.desktop:Exec=/usr/bin/git-annex webapp + /usr/share/applications/git-annex.desktop:Icon=git-annex + /usr/share/applications/git-annex.desktop:Categories=Network;FileTransfer; + /etc/xdg/autostart/git-annex.desktop:[Desktop Entry] + /etc/xdg/autostart/git-annex.desktop:Type=Application + /etc/xdg/autostart/git-annex.desktop:Version=1.0 + /etc/xdg/autostart/git-annex.desktop:Name=Git Annex Assistant + /etc/xdg/autostart/git-annex.desktop:Comment=Autostart + /etc/xdg/autostart/git-annex.desktop:Terminal=false + /etc/xdg/autostart/git-annex.desktop:Exec=/usr/bin/git-annex assistant --autostart + /etc/xdg/autostart/git-annex.desktop:Categories= + +"""]] diff --git a/doc/bugs/On_Lubuntu_14.04_assistant_fails_to_create_new_setup_or_actually_work___40__fixed_by_regular_lxsession_package_update_from_2014-06-30__41__/comment_4_62be3dd4092b15cdf85cf9a231b2863a._comment b/doc/bugs/On_Lubuntu_14.04_assistant_fails_to_create_new_setup_or_actually_work___40__fixed_by_regular_lxsession_package_update_from_2014-06-30__41__/comment_4_62be3dd4092b15cdf85cf9a231b2863a._comment new file mode 100644 index 0000000000..748b57df5c --- /dev/null +++ b/doc/bugs/On_Lubuntu_14.04_assistant_fails_to_create_new_setup_or_actually_work___40__fixed_by_regular_lxsession_package_update_from_2014-06-30__41__/comment_4_62be3dd4092b15cdf85cf9a231b2863a._comment @@ -0,0 +1,8 @@ +[[!comment format=mdwn + username="http://joeyh.name/" + ip="108.236.230.124" + subject="comment 4" + date="2014-06-11T20:09:14Z" + content=""" +What happens if you run `git-annex assistant --autostart` ? +"""]] diff --git a/doc/bugs/On_Lubuntu_14.04_assistant_fails_to_create_new_setup_or_actually_work___40__fixed_by_regular_lxsession_package_update_from_2014-06-30__41__/comment_5_cca4905426a3e01da6e12be855c7a418._comment b/doc/bugs/On_Lubuntu_14.04_assistant_fails_to_create_new_setup_or_actually_work___40__fixed_by_regular_lxsession_package_update_from_2014-06-30__41__/comment_5_cca4905426a3e01da6e12be855c7a418._comment new file mode 100644 index 0000000000..50b6e5dfa1 --- /dev/null +++ b/doc/bugs/On_Lubuntu_14.04_assistant_fails_to_create_new_setup_or_actually_work___40__fixed_by_regular_lxsession_package_update_from_2014-06-30__41__/comment_5_cca4905426a3e01da6e12be855c7a418._comment @@ -0,0 +1,27 @@ +[[!comment format=mdwn + username="https://www.google.com/accounts/o8/id?id=AItOawm7eqCMh_B7mxE0tnchbr0JoYu11FUAFRY" + nickname="Stéphane" + subject="Can't reproduce" + date="2014-06-30T20:06:30Z" + content=""" +I tried to reproduce the bug on another account on the same machine, same LXDE environment. +The bug did not occur again. + +> What happens if you run `git-annex assistant --autostart` ? + +To answer that, I went back to the first account. Just logging in. +The following git-related processes are automatically launched without running `git-annex assistant --autostart`. + + git-annex assistant --startdelay=5s + git --git-dir=/home/somelogin/Documents/.git --work-tree=/home/somelogin/Documents -c core.bare=false cat-file --batch + git --git-dir=/home/somelogin/Documents/.git --work-tree=/home/somelogin/Documents -c core.bare=false cat-file --batch + git --git-dir=/home/somelogin/Documents/.git --work-tree=/home/somelogin/Documents -c core.bare=false check-ignore -z --stdin --verbose --non-matching + git --git-dir=/home/somelogin/Documents/.git --work-tree=/home/somelogin/Documents -c core.bare=false check-attr -z --stdin annex.backend annex.numcopies -- + +So, somehow things went back in order by themselves ? + +There were quite a bunch of software upgrades from Ubuntu, including `lxsession`. Perhaps something was wrong in handling autostart desktop files ? + +Since things appear to work now, should we close the bug ? + +"""]] diff --git a/doc/bugs/On_Lubuntu_14.04_assistant_fails_to_create_new_setup_or_actually_work___40__fixed_by_regular_lxsession_package_update_from_2014-06-30__41__/comment_6_ae2ca07169321c4a51b7e8e581fda5e2._comment b/doc/bugs/On_Lubuntu_14.04_assistant_fails_to_create_new_setup_or_actually_work___40__fixed_by_regular_lxsession_package_update_from_2014-06-30__41__/comment_6_ae2ca07169321c4a51b7e8e581fda5e2._comment new file mode 100644 index 0000000000..6349de01e2 --- /dev/null +++ b/doc/bugs/On_Lubuntu_14.04_assistant_fails_to_create_new_setup_or_actually_work___40__fixed_by_regular_lxsession_package_update_from_2014-06-30__41__/comment_6_ae2ca07169321c4a51b7e8e581fda5e2._comment @@ -0,0 +1,72 @@ +[[!comment format=mdwn + username="https://www.google.com/accounts/o8/id?id=AItOawm7eqCMh_B7mxE0tnchbr0JoYu11FUAFRY" + nickname="Stéphane" + subject="Cconfirmed it was a LXDE issue, now fixed." + date="2014-07-03T09:58:24Z" + content=""" +Since this is a little strange, I just looked again at the machine. +Your software attracts archivist kind of git-annex user, so don't be surprise that I kept logs of the machine. + +Short answer: **confirmed it was a LXDE issue, now fixed**. + +## From what date has git-annex started to work ? + + git log | grep Date: + + (many other lines...) + Date: Mon Jun 30 21:43:49 2014 +0200 + Date: Mon Jun 30 21:42:07 2014 +0200 + Date: Mon Jun 30 21:38:22 2014 +0200 + Date: Sun Jun 8 14:34:13 2014 +0200 + Date: Sun Jun 8 14:28:48 2014 +0200 + Date: Sat Jun 7 16:21:06 2014 +0200 + +So I tried to enable it on June 7, reported bug on June 8. +The the machine was used on a regular basis but git-annex assistant was ineffective. +Then it started working as expected from June 30th. + +## Any package install near June 30th ? + +Updates installed regularly. + + cd /var/log ; zgrep -h \" installed.*\" dpkg.log* | grep '2014-0[67]-..' -o | sort | uniq -c + + 2 2014-06-02 + 3 2014-06-03 + 3 2014-06-04 + 3 2014-06-06 + 49 2014-06-07 + 3 2014-06-10 + 8 2014-06-11 + 45 2014-06-12 + 7 2014-06-13 + 2 2014-06-16 + 42 2014-06-21 + 20 2014-06-25 + 3 2014-06-26 + 23 2014-06-27 + 13 2014-06-28 + 6 2014-06-30 + 15 2014-07-02 + 3 2014-07-03 + +Which packages were installed on 2014-06-30 ? + + cd /var/log ; zgrep -h \" installed.*\" dpkg.log* | grep '2014-06-(29|30)' -E + + 2014-06-30 17:29:15 status installed lxsession-data:all 0.4.9.2+git20140410-0ubuntu1.1 + 2014-06-30 17:29:29 status installed desktop-file-utils:i386 0.22-1ubuntu1 + 2014-06-30 17:29:30 status installed mime-support:all 3.54ubuntu1 + 2014-06-30 17:29:30 status installed lxsession-default-apps:i386 0.4.9.2+git20140410-0ubuntu1.1 + 2014-06-30 17:29:44 status installed lxsession-logout:i386 0.4.9.2+git20140410-0ubuntu1.1 + 2014-06-30 17:29:58 status installed lxsession:i386 0.4.9.2+git20140410-0ubuntu1.1 + +And indeed [0.4.9.2+git20140410-0ubuntu1.1 : “lxsession” package : Ubuntu](https://launchpad.net/ubuntu/+source/lxsession/0.4.9.2+git20140410-0ubuntu1.1) + +So it was indeed a lxde bug. + +[Diff text](http://launchpadlibrarian.net/177201602/lxsession_0.4.9.2%2Bgit20140410-0ubuntu1_0.4.9.2%2Bgit20140410-0ubuntu1.1.diff.gz) + +I added a comment there (#69) which may serve as a hint to other people having not updated their system. [Bug #1308348 “network settings indicator missing from panel” : Bugs : “lxsession” package : Ubuntu](https://bugs.launchpad.net/ubuntu/+source/lxsession/+bug/1308348) + +"""]] diff --git a/doc/bugs/On_Lubuntu_14.04_assistant_fails_to_create_new_setup_or_actually_work___40__fixed_by_regular_lxsession_package_update_from_2014-06-30__41__/comment_7_ae5c434a9c94aa000b604095f52e3d3c._comment b/doc/bugs/On_Lubuntu_14.04_assistant_fails_to_create_new_setup_or_actually_work___40__fixed_by_regular_lxsession_package_update_from_2014-06-30__41__/comment_7_ae5c434a9c94aa000b604095f52e3d3c._comment new file mode 100644 index 0000000000..4ae5454078 --- /dev/null +++ b/doc/bugs/On_Lubuntu_14.04_assistant_fails_to_create_new_setup_or_actually_work___40__fixed_by_regular_lxsession_package_update_from_2014-06-30__41__/comment_7_ae5c434a9c94aa000b604095f52e3d3c._comment @@ -0,0 +1,14 @@ +[[!comment format=mdwn + username="http://joeyh.name/" + ip="209.250.56.55" + subject="comment 7" + date="2014-07-03T19:03:52Z" + content=""" +That lubuntu bug explains why git-annex would not autostart on login. + +However, that does not explain the original problem description, which was that the `Make Repository` button seemed to make a repository but not switch to the webapp running in that repository. In the case of making a new repository, the webapp is started in it by git-annex, not by the desktop autostart mechanism. + +Can you reproduce the original problem? + + +"""]] diff --git a/doc/bugs/On_Windows__44___can__39__t_use_a_USB_disk_annex_created_on_Linux.mdwn b/doc/bugs/On_Windows__44___can__39__t_use_a_USB_disk_annex_created_on_Linux.mdwn new file mode 100644 index 0000000000..b997a8c725 --- /dev/null +++ b/doc/bugs/On_Windows__44___can__39__t_use_a_USB_disk_annex_created_on_Linux.mdwn @@ -0,0 +1,18 @@ +### Please describe the problem. +On Windows, git-annex is unable to 'annex get' from a repository hosted on an external USB disk. + +### What steps will reproduce the problem? +1. Create an annex (A1) on a Linux system +2. Clone A1 to a new annex (A2) to an external USB drive. add A1 as a remote, "annex get ." +3. Mount the USB drive to a windows system. Clone A2 to a new annex (A3). +4. add A2 as a remote to A3 +5. "git annex get ." +6. annex get will fail with file not found errors. This appears to be due to case sensitivity on Windows. + + +### What version of git-annex are you using? On what operating system? +Windows 7: 4.20140627-g8a36ec5 (from the git-annex download page) + +Debian Linux: 3.20120629 (from the package manager) + +[[!tag moreinfo]] diff --git a/doc/bugs/On_Windows__44___can__39__t_use_a_USB_disk_annex_created_on_Linux/comment_1_f224f4155d857a59595658357f97dac1._comment b/doc/bugs/On_Windows__44___can__39__t_use_a_USB_disk_annex_created_on_Linux/comment_1_f224f4155d857a59595658357f97dac1._comment new file mode 100644 index 0000000000..403c9a1b91 --- /dev/null +++ b/doc/bugs/On_Windows__44___can__39__t_use_a_USB_disk_annex_created_on_Linux/comment_1_f224f4155d857a59595658357f97dac1._comment @@ -0,0 +1,12 @@ +[[!comment format=mdwn + username="http://joeyh.name/" + ip="4.153.254.222" + subject="comment 1" + date="2013-07-07T17:50:40Z" + content=""" +Is A2 a bare or a non-bare git repository? + +What filesystem is used on the USB drive? + +Why do you think it has to do with case sensativity? +"""]] diff --git a/doc/bugs/On_Windows__44___can__39__t_use_repository_that_has_a_unix-style_local_remote_configured.mdwn b/doc/bugs/On_Windows__44___can__39__t_use_repository_that_has_a_unix-style_local_remote_configured.mdwn new file mode 100644 index 0000000000..92de1dce3e --- /dev/null +++ b/doc/bugs/On_Windows__44___can__39__t_use_repository_that_has_a_unix-style_local_remote_configured.mdwn @@ -0,0 +1,22 @@ +### Please describe the problem. +On Windows, I try to "annex get" from a Linux based repository, but it fails with no useful error message. + +### What steps will reproduce the problem? +1. Create annex (A1) on a Linux system. +2. clone A1 to another annex (A2) on the same file tree (on a mounted USB drive, for example) +3. add A2 as a remote to A1. +4. clone A1 to an annex (A3) on a Windows 7 system. +5. attempt to "git annex get ." +6. annex reports that it can find no repositories, and asks to make one available. + + +### What version of git-annex are you using? On what operating system? +Windows 7: 4.20140627-g8a36ec5 (from the git-annex download page) +Debian Linux: 3.20120629 (from the package manager) + +### Please provide any additional information below. +I experienced this bug several times with a plain git-annex install, and the only workaround was removing the unix-style remotes from remote repos. + +After other hackery to get git-annex working on Windows, I can't currently reproduce the issue. + +[[!tag moreinfo]] diff --git a/doc/bugs/On_Windows__44___can__39__t_use_repository_that_has_a_unix-style_local_remote_configured/comment_1_95655915ff6ba9fb5d873358ff047496._comment b/doc/bugs/On_Windows__44___can__39__t_use_repository_that_has_a_unix-style_local_remote_configured/comment_1_95655915ff6ba9fb5d873358ff047496._comment new file mode 100644 index 0000000000..d727bcafcb --- /dev/null +++ b/doc/bugs/On_Windows__44___can__39__t_use_repository_that_has_a_unix-style_local_remote_configured/comment_1_95655915ff6ba9fb5d873358ff047496._comment @@ -0,0 +1,10 @@ +[[!comment format=mdwn + username="http://joeyh.name/" + ip="4.154.2.134" + subject="comment 1" + date="2013-09-13T19:14:55Z" + content=""" +I don't understand what is meant by a \"unix-style\" remote. + +Generally it helps to provide some commands I can run on Windows to replicate the problem. +"""]] diff --git a/doc/bugs/One_Client_Not_Syncing_Content.mdwn b/doc/bugs/One_Client_Not_Syncing_Content.mdwn new file mode 100644 index 0000000000..2535074bee --- /dev/null +++ b/doc/bugs/One_Client_Not_Syncing_Content.mdwn @@ -0,0 +1,54 @@ +### Please describe the problem. + +I am fairly new to git annex - so I may have a misconception about how it should work. + +I have two laptop clients A and B that each have a single ssh remote, server C, setup as a full backup. Both of the laptops were setup through the web and have the assistant running. + +The problem I am seeing is that laptop A does not seem to sync contents created on laptop B. If i do the following: + + laptopA# echo "foo" > bar + laptopB# cat bar + > foo + + laptopB# echo "baz" > bang + laptopA# cat bang + > No such file or directory + +Laptop A has the symlink for bang but no contents. + +if i run the following: + + laptopA# git annex find --want-get --not --in . +it returns 'bang' + +So it seems that laptopA wants the contents of the file created on laptopB, but doesn't sync it. If I manually run git annex sync --content everything is good. But when running the assistant should I not care about this? + + +### What steps will reproduce the problem? + +See above. + +### What version of git-annex are you using? On what operating system? + +laptopA: OSX 10.10 + + git-annex version: 5.20150205 + build flags: Assistant Webapp Webapp-secure Pairing Testsuite S3 WebDAV FsEvents XMPP DNS Feeds Quvi TDFA TorrentParser + key/value backends: SHA256E SHA1E SHA512E SHA224E SHA384E SKEIN256E SKEIN512E MD5E SHA256 SHA1 SHA512 SHA224 SHA384 SKEIN256 SKEIN512 MD5 WORM URL + remote types: git gcrypt S3 bup directory rsync web bittorrent webdav tahoe glacier ddar hook external + local repository version: 5 + supported repository version: 5 + upgrade supported from repository versions: 0 1 2 4 + +laptopB: Archlinux + + git-annex version: 5.20150219-g7751a05 + build flags: Assistant Webapp Webapp-secure Pairing Testsuite S3 Inotify DBus DesktopNotify XMPP DNS Feeds Quvi TDFA TorrentParser + key/value backends: SHA256E SHA1E SHA512E SHA224E SHA384E SKEIN256E SKEIN512E MD5E SHA256 SHA1 SHA512 SHA224 SHA384 SKEIN256 SKEIN512 MD5 WORM URL + remote types: git gcrypt S3 bup directory rsync web bittorrent glacier ddar hook external + local repository version: 5 + supported repository version: 5 + upgrade supported from repository versions: 0 1 2 4 + + +### Please provide any additional information below. diff --git a/doc/bugs/One_Client_Not_Syncing_Content/comment_1_b8c8a9d10a12b72ea828b6b35d87fa5e._comment b/doc/bugs/One_Client_Not_Syncing_Content/comment_1_b8c8a9d10a12b72ea828b6b35d87fa5e._comment new file mode 100644 index 0000000000..8f60325e3f --- /dev/null +++ b/doc/bugs/One_Client_Not_Syncing_Content/comment_1_b8c8a9d10a12b72ea828b6b35d87fa5e._comment @@ -0,0 +1,14 @@ +[[!comment format=mdwn + username="joey" + subject="""comment 1""" + date="2015-03-26T18:17:57Z" + content=""" +What version of git-annex is installed on laptopA? + +If it's too old, it won't be getting change notifications from the server +when laptopB pushes to it. + +Since laptopB immediately notices pushes from laptopA, I guess that +both laptopB and the server have new enough versions of git-annex +for that change noitification to work. +"""]] diff --git a/doc/bugs/One_Client_Not_Syncing_Content/comment_2_59921abb184bffb5ac1656cc50054e11._comment b/doc/bugs/One_Client_Not_Syncing_Content/comment_2_59921abb184bffb5ac1656cc50054e11._comment new file mode 100644 index 0000000000..fda5be00c1 --- /dev/null +++ b/doc/bugs/One_Client_Not_Syncing_Content/comment_2_59921abb184bffb5ac1656cc50054e11._comment @@ -0,0 +1,8 @@ +[[!comment format=mdwn + username="https://www.google.com/accounts/o8/id?id=AItOawm69GSNODRsVx40rPWDWyecnZn2r3bmOlY" + nickname="Jeffery" + subject="comment 2" + date="2015-03-28T03:02:24Z" + content=""" +laptop A has 5.20150205 while B has 5.20150219-g7751a05 ... they are fairly close. Could this cause a problem? +"""]] diff --git a/doc/bugs/Operating_system_should_be_contained_in___96__git_annex_version__96___output_regardless_if_in_annex_repo_or_not.mdwn b/doc/bugs/Operating_system_should_be_contained_in___96__git_annex_version__96___output_regardless_if_in_annex_repo_or_not.mdwn new file mode 100644 index 0000000000..b4d84b6e52 --- /dev/null +++ b/doc/bugs/Operating_system_should_be_contained_in___96__git_annex_version__96___output_regardless_if_in_annex_repo_or_not.mdwn @@ -0,0 +1,49 @@ +### Please describe the problem. + +When looking over the [[bug reports|bugs]] here some of them are missing the "Operating system" info in the `git annex version` output. The reason for this is that the `git annex version` output depends on whether it was run inside an annex repo or not. + +### What steps will reproduce the problem? + +Run `git annex version` outside of an annex repo: + + git-annex version: 6.20180509 + build flags: Assistant Webapp Pairing S3(multipartupload)(storageclasses) WebDAV Inotify DBus DesktopNotify ConcurrentOutput TorrentParser MagicMime Feeds Testsuite + dependency versions: aws-0.14.1 bloomfilter-2.0.1.0 cryptonite-0.20 DAV-1.3.1 feed-0.3.11.1 ghc-8.0.1 http-client-0.4.31.1 persistent-sqlite-2.6 torrent-10000.0.0 uuid-1.3.12 yesod-1.4.3 + key/value backends: SHA256E SHA256 SHA512E SHA512 SHA224E SHA224 SHA384E SHA384 SHA3_256E SHA3_256 SHA3_512E SHA3_512 SHA3_224E SHA3_224 SHA3_384E SHA3_384 SKEIN256E SKEIN256 SKEIN512E SKEIN512 SHA1E SHA1 MD5E MD5 WORM URL + remote types: git gcrypt p2p S3 bup directory rsync web bittorrent webdav adb tahoe glacier ddar hook external + + +Run `git annex version` inside an annex repo: + + git-annex version: 6.20180509 + build flags: Assistant Webapp Pairing S3(multipartupload)(storageclasses) WebDAV Inotify DBus DesktopNotify ConcurrentOutput TorrentParser MagicMime Feeds Testsuite + dependency versions: aws-0.14.1 bloomfilter-2.0.1.0 cryptonite-0.20 DAV-1.3.1 feed-0.3.11.1 ghc-8.0.1 http-client-0.4.31.1 persistent-sqlite-2.6 torrent-10000.0.0 uuid-1.3.12 yesod-1.4.3 + key/value backends: SHA256E SHA256 SHA512E SHA512 SHA224E SHA224 SHA384E SHA384 SHA3_256E SHA3_256 SHA3_512E SHA3_512 SHA3_224E SHA3_224 SHA3_384E SHA3_384 SKEIN256E SKEIN256 SKEIN512E SKEIN512 SHA1E SHA1 MD5E MD5 WORM URL + remote types: git gcrypt p2p S3 bup directory rsync web bittorrent webdav adb tahoe glacier ddar hook external + local repository version: 5 + supported repository versions: 3 5 6 + upgrade supported from repository versions: 0 1 2 3 4 5 + operating system: linux x86_64 + + +### What version of git-annex are you using? On what operating system? + + git-annex version: 6.20180509 + build flags: Assistant Webapp Pairing S3(multipartupload)(storageclasses) WebDAV Inotify DBus DesktopNotify ConcurrentOutput TorrentParser MagicMime Feeds Testsuite + dependency versions: aws-0.14.1 bloomfilter-2.0.1.0 cryptonite-0.20 DAV-1.3.1 feed-0.3.11.1 ghc-8.0.1 http-client-0.4.31.1 persistent-sqlite-2.6 torrent-10000.0.0 uuid-1.3.12 yesod-1.4.3 + key/value backends: SHA256E SHA256 SHA512E SHA512 SHA224E SHA224 SHA384E SHA384 SHA3_256E SHA3_256 SHA3_512E SHA3_512 SHA3_224E SHA3_224 SHA3_384E SHA3_384 SKEIN256E SKEIN256 SKEIN512E SKEIN512 SHA1E SHA1 MD5E MD5 WORM URL + remote types: git gcrypt p2p S3 bup directory rsync web bittorrent webdav adb tahoe glacier ddar hook external + supported repository versions: 3 5 6 + upgrade supported from repository versions: 0 1 2 3 4 5 + operating system: linux x86_64 + + +### Please provide any additional information below. + +It might make sense to also always output "supported repository versions" and "upgrade supported from repository versions" because they also do not depend on the current annex repo. + +### Have you had any luck using git-annex before? (Sometimes we get tired of reading bug reports all day and a lil' positive end note does wonders) + +I am a long time happy user of git-annex. Thank you for your never ending effort! + +> [[done]] --[[Joey]] diff --git a/doc/bugs/Operating_system_should_be_contained_in___96__git_annex_version__96___output_regardless_if_in_annex_repo_or_not/comment_1_17ab67ebfd21e0844973461bad1d40f9._comment b/doc/bugs/Operating_system_should_be_contained_in___96__git_annex_version__96___output_regardless_if_in_annex_repo_or_not/comment_1_17ab67ebfd21e0844973461bad1d40f9._comment new file mode 100644 index 0000000000..e618fbaf41 --- /dev/null +++ b/doc/bugs/Operating_system_should_be_contained_in___96__git_annex_version__96___output_regardless_if_in_annex_repo_or_not/comment_1_17ab67ebfd21e0844973461bad1d40f9._comment @@ -0,0 +1,7 @@ +[[!comment format=mdwn + username="joey" + subject="""comment 1""" + date="2018-06-04T16:16:39Z" + content=""" +Good catch. +"""]] diff --git a/doc/bugs/Out_of_memory_error_when_adding_large_files_to_v6_repository.mdwn b/doc/bugs/Out_of_memory_error_when_adding_large_files_to_v6_repository.mdwn new file mode 100644 index 0000000000..568b2bf5f4 --- /dev/null +++ b/doc/bugs/Out_of_memory_error_when_adding_large_files_to_v6_repository.mdwn @@ -0,0 +1,57 @@ +### Please describe the problem. + +When trying to add a large file (compared to the amount of RAM on the machine) to a v6 repository using `git add large_file.iso`, I get `fatal: Out of memory, realloc failed`. Adding the file with `git annex add large_file.iso` works fine. + +### What steps will reproduce the problem? + +On a machine with 4 GiB of RAM, I'm trying to add a 2.8 GiB file called `large_file.iso` to a freshly created v6 repository: + +``` +$ mkdir annex +$ cd annex +$ git init +$ git annex init --version=6 +$ cp /tmp/large_file.iso . +$ git add large_file.iso +fatal: Out of memory, realloc failed +``` + + +### What version of git-annex are you using? On what operating system? + +- git-annex version: 6.20170101 +- git version 2.11.0 +- Debian GNU/Linux 9.0 (stretch) + +I also tried git-annex 6.20170321, with no success. + +### Please provide any additional information below. + +``` +$ GIT_TRACE=1 git add large_file.iso +06:48:23.048295 git.c:371 trace: built-in: git 'add' 'large_file.iso' +fatal: Out of memory, realloc failed +06:48:23.048725 run-command.c:350 trace: run_command: 'git-annex smudge --clean '\''large_file.iso'\''' +``` + +So the problem seems to be with Git trying to allocate memory the size of the file, rather that git-annex itself: + +``` +$ ltrace git add large_file.iso 2>&1 | grep realloc | tail -n3 +realloc(0, 2902458369) = 0 +realloc(0, 2902458369) = 0 +fputc('\n', 0x7ff2c98ce520Out of memory, realloc failed +``` + +``` +$ git-annex smudge --clean large_file.iso < large_file.iso +/annex/objects/SHA256E-s2902458368--f8ef73594445624eabc1ea4a567ebedb8841fd248347c1aa01538a8de705c5c1.iso +``` + +I have no idea why it needs to do that, though. + +### Have you had any luck using git-annex before? (Sometimes we get tired of reading bug reports all day and a lil' positive end note does wonders) + +I've been using git-annex v5 repositories without any issues, and with smaller files, repository v6 works great! + +> dup; [[done]] --[[Joey]] diff --git a/doc/bugs/Out_of_memory_error_when_adding_large_files_to_v6_repository/comment_1_b9154c38406fca40c4c0edb716707c3a._comment b/doc/bugs/Out_of_memory_error_when_adding_large_files_to_v6_repository/comment_1_b9154c38406fca40c4c0edb716707c3a._comment new file mode 100644 index 0000000000..9ef59ed710 --- /dev/null +++ b/doc/bugs/Out_of_memory_error_when_adding_large_files_to_v6_repository/comment_1_b9154c38406fca40c4c0edb716707c3a._comment @@ -0,0 +1,19 @@ +[[!comment format=mdwn + username="joey" + subject="""comment 1""" + date="2017-06-09T17:34:40Z" + content=""" +Unfortunately, git add tries to load the whole file content +into memory (or at least allocates a buffer for it all), +even if it's only going to stream it through the clean filter +used in v6 mode, and even though the git-annex smudge +filter reads the file content from disk itself. + +I submitted a patch to git over a year ago, that IIRC fixes this kind of +problem, but it was not accepted. Getting an updated version of that patch +accepted into git is the main blocker for v6 repositories to not be +experimental. + +[[todo/smudge]] documents this problem. I'm going to close this bug +as it's a duplicate of stuff discussed there. +"""]] diff --git a/doc/bugs/Out_of_memory_error_when_adding_large_files_to_v6_repository/comment_2_97d5a2571679d5f6dc90694e5be89eaa._comment b/doc/bugs/Out_of_memory_error_when_adding_large_files_to_v6_repository/comment_2_97d5a2571679d5f6dc90694e5be89eaa._comment new file mode 100644 index 0000000000..8eec02fa43 --- /dev/null +++ b/doc/bugs/Out_of_memory_error_when_adding_large_files_to_v6_repository/comment_2_97d5a2571679d5f6dc90694e5be89eaa._comment @@ -0,0 +1,9 @@ +[[!comment format=mdwn + username="yves.noirjean@3f9b06d19a920fbf5c82340c362e5971b00d4af2" + nickname="yves.noirjean" + avatar="http://cdn.libravatar.org/avatar/2f1ad9d443c037337d91f29781560344" + subject="comment 2" + date="2018-06-22T07:53:26Z" + content=""" +I stumbled over the same problem. With \"git annex add\" I can add the file. But in unlocked mode, I get the \"out of memory\" error when trying to commit. Is this also a known problem? Is there a workaround? +"""]] diff --git a/doc/bugs/Packfile_does_not_match_digest__58___gcrypt_with_assistant.mdwn b/doc/bugs/Packfile_does_not_match_digest__58___gcrypt_with_assistant.mdwn new file mode 100644 index 0000000000..9a8eed58fa --- /dev/null +++ b/doc/bugs/Packfile_does_not_match_digest__58___gcrypt_with_assistant.mdwn @@ -0,0 +1,24 @@ +### Please describe the problem. +I tried to sync a folder between two laptops, using the webapp and GPG encryption. I started by seting up the repository on laptop1, and then, while all the files were uploading, I went over to laptop2 and set things up there as well. + +At first, everything looked fine, laptop1 was uploading and laptop2 was downloading. Then, laptop2 reported "Failed to sync with VPS, and the log file showed a gcrypt error: "Packfile long-hash does not match digest!" + +### What version of git-annex are you using? On what operating system? + + + +Laptop1 is running wheezy with git annex version 6.20160307+gitgb095561-1~ndall+1 from neurodebian. + +VPS is running wheezy + +Laptop2 is runing jessie with git annex version 6.20160307+gitgb095561-1~ndall+1 from neurodebian. + +### Please provide any additional information below. + +http://denisa.hobbs.cz/laptop1.daemon.log +http://denisa.hobbs.cz/laptop2.daemon.log + +### Have you had any luck using git-annex before? + +If I could get this syncing to work, then that would be great! I don't want to use unison, because that wouldn't be encrypted... So this would be wonderful. + diff --git a/doc/bugs/Packfile_does_not_match_digest__58___gcrypt_with_assistant/comment_10_9d4fdddd7ab05de9dfa4cc90f1051ef5._comment b/doc/bugs/Packfile_does_not_match_digest__58___gcrypt_with_assistant/comment_10_9d4fdddd7ab05de9dfa4cc90f1051ef5._comment new file mode 100644 index 0000000000..2bb9552ff2 --- /dev/null +++ b/doc/bugs/Packfile_does_not_match_digest__58___gcrypt_with_assistant/comment_10_9d4fdddd7ab05de9dfa4cc90f1051ef5._comment @@ -0,0 +1,17 @@ +[[!comment format=mdwn + username="joey" + subject="""comment 10""" + date="2016-09-21T21:36:18Z" +content=""" +@Don and @pot, did you actually see un-encrypted data end up in the gcrypt +repository? Or by "same issue" do you mean you saw the same error message, +perhaps due to a very different cause? + +@jgoerzen, when you say that the gcrypt reposotory had an "unencrypted +index", are you referring to a regular git index file, or to a pack's .idx +file? + +It would be very strange if a gcrypt repository had an index file. +I've never been able to use git-remote-gcrypt with a non-bare repository, +and bare repositories don't normally have index files. +"""]] diff --git a/doc/bugs/Packfile_does_not_match_digest__58___gcrypt_with_assistant/comment_11_bbd8b537e277d24df254ed058ad40e24._comment b/doc/bugs/Packfile_does_not_match_digest__58___gcrypt_with_assistant/comment_11_bbd8b537e277d24df254ed058ad40e24._comment new file mode 100644 index 0000000000..adff369df4 --- /dev/null +++ b/doc/bugs/Packfile_does_not_match_digest__58___gcrypt_with_assistant/comment_11_bbd8b537e277d24df254ed058ad40e24._comment @@ -0,0 +1,9 @@ +[[!comment format=mdwn + username="pot" + subject="inodes...." + date="2016-09-30T22:00:31Z" + content=""" +I think I may have tracked this down partly, I ran out of inodes on a removeable drive and started getting the same error. Will try and rectify underlying inode issue and then see. + +(secondary point, how on earth has my docs directory used up all 59 million inodes available to it when put into an annex???) +"""]] diff --git a/doc/bugs/Packfile_does_not_match_digest__58___gcrypt_with_assistant/comment_12_5090e41cf96dfe542d1d2326aebc556c._comment b/doc/bugs/Packfile_does_not_match_digest__58___gcrypt_with_assistant/comment_12_5090e41cf96dfe542d1d2326aebc556c._comment new file mode 100644 index 0000000000..af336a193b --- /dev/null +++ b/doc/bugs/Packfile_does_not_match_digest__58___gcrypt_with_assistant/comment_12_5090e41cf96dfe542d1d2326aebc556c._comment @@ -0,0 +1,20 @@ +[[!comment format=mdwn + username="http://schnouki.net/" + nickname="Schnouki" + avatar="http://cdn.libravatar.org/avatar/5f6406e9db28564121169f0051645b8c30a12a20ca7bc40287ac9bf2cd3ad283" + subject="comment 12" + date="2017-05-29T09:56:59Z" + content=""" +Not sure how I did it, but I have 2 repos that give me the same error. (Perhaps it happened when killing the assistant in the middle of a sync? No idea…) + +However I \"solved\" the issue for one repo: + +1. cloned the gcrypt remote to a temporary directory +2. removed all the (encrypted) files with `git rm`, committed that, and pushed it (the repo is hosted on Gitlab and I can't just remove the branch or do something like that) +3. removed the `gcrypt-id| from my `.git/config` +4. ran `git annex sync` again + +And voilà! It works :) + +I don't have any content in this remote, only metadata, so it's easy to do. But I still don't know what caused this. +"""]] diff --git a/doc/bugs/Packfile_does_not_match_digest__58___gcrypt_with_assistant/comment_13_2bfa76c4083ff9c1d2a4adfe38ced2f0._comment b/doc/bugs/Packfile_does_not_match_digest__58___gcrypt_with_assistant/comment_13_2bfa76c4083ff9c1d2a4adfe38ced2f0._comment new file mode 100644 index 0000000000..1a2bc7cd18 --- /dev/null +++ b/doc/bugs/Packfile_does_not_match_digest__58___gcrypt_with_assistant/comment_13_2bfa76c4083ff9c1d2a4adfe38ced2f0._comment @@ -0,0 +1,10 @@ +[[!comment format=mdwn + username="jgoerzen" + avatar="http://cdn.libravatar.org/avatar/090740822c9dcdb39ffe506b890981b4" + subject="A hint?" + date="2017-10-02T02:05:32Z" + content=""" +Hi folks, + +I no longer use git-annex so I can't comment directly on this. However, in some testing with git-remote-gcrypt, I came across an issue where changes seem to be lost. In short, every git push behaves as if --force were given. Details in [Debian but #877464](https://bugs.debian.org/cgi-bin/bugreport.cgi?bug=877464). +"""]] diff --git a/doc/bugs/Packfile_does_not_match_digest__58___gcrypt_with_assistant/comment_1_bf61defadd3be9bc867be961b3149be6._comment b/doc/bugs/Packfile_does_not_match_digest__58___gcrypt_with_assistant/comment_1_bf61defadd3be9bc867be961b3149be6._comment new file mode 100644 index 0000000000..cd3735334c --- /dev/null +++ b/doc/bugs/Packfile_does_not_match_digest__58___gcrypt_with_assistant/comment_1_bf61defadd3be9bc867be961b3149be6._comment @@ -0,0 +1,19 @@ +[[!comment format=mdwn + username="joey" + subject="""comment 1""" + date="2016-04-13T18:10:03Z" + content=""" +That error message is certainly not coming from git-annex. It sounds like a +git error message, but I don't find it in the current git source code. + +What it sounds like is some form of corruption of the git repository, +probably to a pack file. Since git-annex doesn't have anything to do with +writing such files, it's hard to see how this could be a bug in git-annex. + +This kind of corruption problem tends to happen when a disk loses data, +perhaps having to do with an unclean shutdown. Or perhaps it received +bad git repository data from the VPS. + +Suggest you run `git fsck` and if it reports problems, you may be ableo +to fix them by running `git annex repair`. +"""]] diff --git a/doc/bugs/Packfile_does_not_match_digest__58___gcrypt_with_assistant/comment_2_8a8ead857917e7b843f1bdfe07b5d52d._comment b/doc/bugs/Packfile_does_not_match_digest__58___gcrypt_with_assistant/comment_2_8a8ead857917e7b843f1bdfe07b5d52d._comment new file mode 100644 index 0000000000..77a1e4d997 --- /dev/null +++ b/doc/bugs/Packfile_does_not_match_digest__58___gcrypt_with_assistant/comment_2_8a8ead857917e7b843f1bdfe07b5d52d._comment @@ -0,0 +1,7 @@ +[[!comment format=mdwn + username="Don" + subject="comment 2" + date="2016-04-14T19:31:52Z" + content=""" +I've been watching this thread because I have the same issue. Joey, it seems like this error message is coming from the `get_verify_decrypt_pack()` function in git-remote-gcrypt. +"""]] diff --git a/doc/bugs/Packfile_does_not_match_digest__58___gcrypt_with_assistant/comment_3_2432df64f920f3962b6ba02adaa9b61d._comment b/doc/bugs/Packfile_does_not_match_digest__58___gcrypt_with_assistant/comment_3_2432df64f920f3962b6ba02adaa9b61d._comment new file mode 100644 index 0000000000..bfdddc9c7f --- /dev/null +++ b/doc/bugs/Packfile_does_not_match_digest__58___gcrypt_with_assistant/comment_3_2432df64f920f3962b6ba02adaa9b61d._comment @@ -0,0 +1,15 @@ +[[!comment format=mdwn + username="joey" + subject="""comment 3""" + date="2016-04-14T19:57:58Z" + content=""" +Ok, so definitely not a bug in git-annex then. + +`get_verify_decrypt_pack` downloads an encrypted pack file, and then +uses gpg to hash the pack file and compares this to the hash +encoded in the name of the pack file. + +So, this could happen if the pack files in the gcrypt remote have gotten +the wrong data into them. Or it could be a bug of some kind in +git-remote-gcrypt. +"""]] diff --git a/doc/bugs/Packfile_does_not_match_digest__58___gcrypt_with_assistant/comment_4_536fbc38a7f2e7ab42188b5a8700e2cf._comment b/doc/bugs/Packfile_does_not_match_digest__58___gcrypt_with_assistant/comment_4_536fbc38a7f2e7ab42188b5a8700e2cf._comment new file mode 100644 index 0000000000..524f507e62 --- /dev/null +++ b/doc/bugs/Packfile_does_not_match_digest__58___gcrypt_with_assistant/comment_4_536fbc38a7f2e7ab42188b5a8700e2cf._comment @@ -0,0 +1,7 @@ +[[!comment format=mdwn + username="jgoerzen" + subject="A caution on this" + date="2016-06-23T21:24:49Z" + content=""" +Although I believe Joey's analysis is correct, I want to interject a caution: while investigating this issue on my own system, I discovered that a gcrypt remote has an unencrypted pack file and unencrypted index, which somehow contained actual data of mine. I would consider it a git-annex bug that data (file contents) wound up in a pack file (this is a git-annex assistant repo) but a gcrypt bug that it made it there unencrypted. +"""]] diff --git a/doc/bugs/Packfile_does_not_match_digest__58___gcrypt_with_assistant/comment_5_f45d09939527be362c586ae1d470afe4._comment b/doc/bugs/Packfile_does_not_match_digest__58___gcrypt_with_assistant/comment_5_f45d09939527be362c586ae1d470afe4._comment new file mode 100644 index 0000000000..4fbbfe1cad --- /dev/null +++ b/doc/bugs/Packfile_does_not_match_digest__58___gcrypt_with_assistant/comment_5_f45d09939527be362c586ae1d470afe4._comment @@ -0,0 +1,7 @@ +[[!comment format=mdwn + username="pot" + subject="Same issue" + date="2016-07-25T18:23:13Z" + content=""" +Anyone got any ideas on how to fix this, or move forward without starting again and trashing the gcrypt repos? Bit of a serious bug this, if it can't be repaired. I was using git-annex for backups, but finding your backups are corrupt would be a bit of a nightmare! +"""]] diff --git a/doc/bugs/Packfile_does_not_match_digest__58___gcrypt_with_assistant/comment_6_88178fa312f351a7667b3b2c48335b02._comment b/doc/bugs/Packfile_does_not_match_digest__58___gcrypt_with_assistant/comment_6_88178fa312f351a7667b3b2c48335b02._comment new file mode 100644 index 0000000000..c31f2345c9 --- /dev/null +++ b/doc/bugs/Packfile_does_not_match_digest__58___gcrypt_with_assistant/comment_6_88178fa312f351a7667b3b2c48335b02._comment @@ -0,0 +1,7 @@ +[[!comment format=mdwn + username="jgoerzen" + subject="comment 6" + date="2016-07-25T19:09:09Z" + content=""" +This was concerning enough to me that I wound up switching to Syncthing for this particular use case. +"""]] diff --git a/doc/bugs/Packfile_does_not_match_digest__58___gcrypt_with_assistant/comment_7_cd6a850119dd1aa0d9bb34c6d8fe560c._comment b/doc/bugs/Packfile_does_not_match_digest__58___gcrypt_with_assistant/comment_7_cd6a850119dd1aa0d9bb34c6d8fe560c._comment new file mode 100644 index 0000000000..e7d8b41a33 --- /dev/null +++ b/doc/bugs/Packfile_does_not_match_digest__58___gcrypt_with_assistant/comment_7_cd6a850119dd1aa0d9bb34c6d8fe560c._comment @@ -0,0 +1,11 @@ +[[!comment format=mdwn + username="branchable@1f03006a73aeda463532c8138da071037870395b" + nickname="branchable" + subject="Transcrypt? " + date="2016-08-05T07:28:27Z" + content=""" +Any experience with using transcrypt instead of git crypt? + + +https://github.com/elasticdog/transcrypt +"""]] diff --git a/doc/bugs/Packfile_does_not_match_digest__58___gcrypt_with_assistant/comment_8_39139e25175ae265a4dc15f0b6b7b618._comment b/doc/bugs/Packfile_does_not_match_digest__58___gcrypt_with_assistant/comment_8_39139e25175ae265a4dc15f0b6b7b618._comment new file mode 100644 index 0000000000..7629d45df8 --- /dev/null +++ b/doc/bugs/Packfile_does_not_match_digest__58___gcrypt_with_assistant/comment_8_39139e25175ae265a4dc15f0b6b7b618._comment @@ -0,0 +1,9 @@ +[[!comment format=mdwn + username="pot" + subject="Hmmm, seems odd such a serious bug is allowed to linger without serious investigation" + date="2016-08-09T19:22:48Z" + content=""" +Should a developer not be all over this, would seem like a very major issue with the gcrypt backend (and annex's interaction with it) that this outcome could even be possible. You can't be using an encrypted store, which becomes irrepairably corrupted and with some of your data sat outside, unencrypted! + +Any thoughts from more knowledgeable developers on how to fix, investigate further, or do something useful on this front? +"""]] diff --git a/doc/bugs/Packfile_does_not_match_digest__58___gcrypt_with_assistant/comment_9_16c004fe7c77cbbba113787ce6410c9e._comment b/doc/bugs/Packfile_does_not_match_digest__58___gcrypt_with_assistant/comment_9_16c004fe7c77cbbba113787ce6410c9e._comment new file mode 100644 index 0000000000..6c76241229 --- /dev/null +++ b/doc/bugs/Packfile_does_not_match_digest__58___gcrypt_with_assistant/comment_9_16c004fe7c77cbbba113787ce6410c9e._comment @@ -0,0 +1,9 @@ +[[!comment format=mdwn + username="spwhitton" + subject="comment 9" + date="2016-08-09T20:17:47Z" + content=""" +I'm the git-remote-gcrypt maintainer, but as it stands it's not really possible to do anything about this bug without steps to reliably reproduce the corrupted/unencrypted remote repository. + +I have been using git-remote-gcrypt with git-annex and with plain git repos for more than two years and I have never encountered anything like this issue. So there's no starting point to work on a fix for the issue. +"""]] diff --git a/doc/bugs/Pairing_locally_shows__58_____34__bad_comment_in_ssh_public_key_ssh-rsa__34__.mdwn b/doc/bugs/Pairing_locally_shows__58_____34__bad_comment_in_ssh_public_key_ssh-rsa__34__.mdwn new file mode 100644 index 0000000000..6999d714c9 --- /dev/null +++ b/doc/bugs/Pairing_locally_shows__58_____34__bad_comment_in_ssh_public_key_ssh-rsa__34__.mdwn @@ -0,0 +1,23 @@ +Open two webapp sessions (in two different *screen* windows or whatever) + + name@name-Computermodel-1000:~/test/annex1$ git-annex webapp + name@name-Computermodel-1000:~/test/annex2$ git-annex webapp + +In annex1's web UI, choose *Configuration* -> *Manage repositories* -> *Local computer*. + +Fill in the secret phrase and press *Start pairing*. + +In annex2's web UI, a pairing request will show on the left notifications bar. Choose *Respond*, fill in the phrase field and press *Finish pairing*. + +I was expecting a pairing between the two running session. But I got this after pressing *Finish pairing* in annex2's web UI: + +**start of output** +# Internal Server Error + +bad comment in ssh public key ssh-rsa [very long GPG key jibber jabber] name@name-Computermodel-1000 + +**end of output** + +git-annex version: 3.20121017, Ubuntu 12.04 + +> [[Done]], allowed dash and underscore in there now. diff --git a/doc/bugs/Permission_problem_in_second_user_account_on_Android.mdwn b/doc/bugs/Permission_problem_in_second_user_account_on_Android.mdwn new file mode 100644 index 0000000000..57789e77d1 --- /dev/null +++ b/doc/bugs/Permission_problem_in_second_user_account_on_Android.mdwn @@ -0,0 +1,17 @@ +I get the following error message upon starting git-annex in a second user account on Android: + + Falling back to hardcoded app location: cannot find expected files in /data/app-lib + git annex webapp + lib/lib.runshell.so: line 133: git: Permission denied + + [Terminal session finished] + +The same version of git-annex works just fine for the primary user. +(The primary user has root access which unfortunately can't be enabled for other user accounts.) + +### What version of git-annex are you using? On what operating system? + + * git-annex: 5.20140710 + * OS: CyanogenMod 10.1.3-p3110 + +> Closing as this was a bug in the deprecated Android app. [[done]] --[[Joey]] diff --git a/doc/bugs/Permission_problem_in_second_user_account_on_Android/comment_1_4ac4f94354b6c5c4370f792689107ddc._comment b/doc/bugs/Permission_problem_in_second_user_account_on_Android/comment_1_4ac4f94354b6c5c4370f792689107ddc._comment new file mode 100644 index 0000000000..a4f56553c2 --- /dev/null +++ b/doc/bugs/Permission_problem_in_second_user_account_on_Android/comment_1_4ac4f94354b6c5c4370f792689107ddc._comment @@ -0,0 +1,8 @@ +[[!comment format=mdwn + username="http://joeyh.name/" + ip="209.250.56.7" + subject="comment 1" + date="2014-08-15T17:52:57Z" + content=""" +I'm afraid all I can do to help with this is to say that the git-annex Android app does not do anything I know of to prevent running the programs, such as git, that are included in it. If your user cannot access them, it must be your OS configuration preventing it. +"""]] diff --git a/doc/bugs/Permission_problem_in_second_user_account_on_Android/comment_2_ef775b5fceb4caf00227978318f73470._comment b/doc/bugs/Permission_problem_in_second_user_account_on_Android/comment_2_ef775b5fceb4caf00227978318f73470._comment new file mode 100644 index 0000000000..761bca635e --- /dev/null +++ b/doc/bugs/Permission_problem_in_second_user_account_on_Android/comment_2_ef775b5fceb4caf00227978318f73470._comment @@ -0,0 +1,8 @@ +[[!comment format=mdwn + username="http://inkwell.za.net/" + nickname="kwill" + subject="Confirmation" + date="2014-11-11T08:01:48Z" + content=""" +I can confirm this occurs for me on Android 4.4 with exactly the same symptoms: primary user account can install and run Git Annex (triggers repo setup, webapp opens), secondary user account can install, but gets this error when starting Git Annex (webapp does not open). Is there any additional info I can provide to resolve this? +"""]] diff --git a/doc/bugs/Permission_problem_in_second_user_account_on_Android/comment_3_2977076f0f4f26daa7d6a087d8af18b3._comment b/doc/bugs/Permission_problem_in_second_user_account_on_Android/comment_3_2977076f0f4f26daa7d6a087d8af18b3._comment new file mode 100644 index 0000000000..45700fda04 --- /dev/null +++ b/doc/bugs/Permission_problem_in_second_user_account_on_Android/comment_3_2977076f0f4f26daa7d6a087d8af18b3._comment @@ -0,0 +1,43 @@ +[[!comment format=mdwn + username="http://inkwell.za.net/" + nickname="kwill" + subject="Logfile with errors" + date="2015-01-06T11:43:12Z" + content=""" +Found the logfile for the attempted installation at `/storage/emulated/10/git-annex.home/git-annex-install.log` + +It looks like there's a problem quite early on. BTW this is on Android 4.4 running on a Nexus 7 (2012). Only modification is installation of F-Droid. Is it worth / possible to install the recent Android 5.0 build? + + Installation starting to /data/data/ga.androidterm + abcdef_SOME_LONG_HEX_STRING + mv: can't rename '/data/data/ga.androidterm/bin': Permission denied + installing busybox + ln: /data/data/ga.androidterm/bin/busybox: Permission denied + installing git-annex + ln: /data/data/ga.androidterm/bin/git-annex: Permission denied + + ... (more of the same) + + busybox: /data/data/ga.androidterm/bin/[: Permission denied + busybox: /data/data/ga.androidterm/bin/[[: Permission denied + busybox: /data/data/ga.androidterm/bin/ar: Permission denied + busybox: /data/data/ga.androidterm/bin/arp: Permission denied + + ... (more of the same) + + busybox: /data/data/ga.androidterm/bin/zcat: Permission denied + tar: can't remove old file ./links/git-shell: Permission denied + tar: write: Broken pipe + cat: can't open '/data/data/ga.androidterm/links/git': Permission denied + rm: can't stat '/data/data/ga.androidterm/links/git': Permission denied + cat: can't open '/data/data/ga.androidterm/links/git-shell': Permission denied + rm: can't stat '/data/data/ga.androidterm/links/git-shell': Permission denied + cat: can't open '/data/data/ga.androidterm/links/git-upload-pack': Permission denied + rm: can't stat '/data/data/ga.androidterm/links/git-upload-pack': Permission denied + lib/lib.runshell.so: line 133: can't create /data/data/ga.androidterm/runshell: Permission denied + lib/lib.runshell.so: line 133: can't create /data/data/ga.androidterm/runshell: Permission denied + chmod: runshell: Operation not permitted + lib/lib.runshell.so: line 133: can't create /data/data/ga.androidterm/bin/trustedkeys.gpg: Permission denied + lib/lib.runshell.so: line 133: can't create /data/data/ga.androidterm/installed-version: Permission denied + Installation complete +"""]] diff --git a/doc/bugs/Permission_problem_in_second_user_account_on_Android/comment_4_cccf1f3aca5edbd82661a9eb91c72de5._comment b/doc/bugs/Permission_problem_in_second_user_account_on_Android/comment_4_cccf1f3aca5edbd82661a9eb91c72de5._comment new file mode 100644 index 0000000000..c392095404 --- /dev/null +++ b/doc/bugs/Permission_problem_in_second_user_account_on_Android/comment_4_cccf1f3aca5edbd82661a9eb91c72de5._comment @@ -0,0 +1,15 @@ +[[!comment format=mdwn + username="joey" + subject="""comment 4""" + date="2018-05-08T18:56:00Z" + content=""" +I'm closing this bug report because the git-annex Android app that it +was reported on is now deprecated. Instead, we have +a way to run the regular git-annex build for linux on Android in termux: + + +There were a lot of problems with the way the git-annex Android app was +put together, and I hope this new approach avoids them. If you try it and +still have the bug you reported, please followup and I'll reopen it. + +"""]] diff --git a/doc/bugs/Podcast_filename_encoding_breaks_Android_client.mdwn b/doc/bugs/Podcast_filename_encoding_breaks_Android_client.mdwn new file mode 100644 index 0000000000..73a9351fe3 --- /dev/null +++ b/doc/bugs/Podcast_filename_encoding_breaks_Android_client.mdwn @@ -0,0 +1,34 @@ +### Please describe the problem. + +Imported podcast files with iso-8859-1 encoding and swedish +characters break repo on Android client with errors like. + +``error: unable to create file pod/Pennypodden___om_bostadspolitik/E54___�se_Richard__Gr�nby_.mp3 (Invalid argument)`` + +Android clients empty direct repo is then distributed to all clients. Whole +annex repo appears empty on all clients. Old revision has to be checked out to +get files back. As soon as Android client syncs again however, all repos are +emptied. + +### What steps will reproduce the problem? +Set up indirect repo on linux laptop and direct repo on Android phone. On the +laptop do: + +``git-annex importfeed http://www.pennygangen.se/podcast/podcast.xml`` + +Wait for changes to propagate if using the assistant or sync manually. + +### What version of git-annex are you using? On what operating system? + + - Debian sid and testing with 5.20141125 on various clients and + - Android 4.4 with 5.20150219-gd24cfd3 on the phone. + +### Please provide any additional information below. + +After failing to write the files with problematic filenames git annex appears +to belive it should commit the fact that the repo is empty and sync this +emptiness to everyone. + +``convmv -f iso-8859-1 -t utf8 $problemfile --notest`` on the laptop fixes the +filenames and the Android client stops wrecking everything. + diff --git a/doc/bugs/Podcast_filename_encoding_breaks_Android_client/comment_1_1bb8e3f3a3d190ed557302df8b8e0335._comment b/doc/bugs/Podcast_filename_encoding_breaks_Android_client/comment_1_1bb8e3f3a3d190ed557302df8b8e0335._comment new file mode 100644 index 0000000000..c2e7795423 --- /dev/null +++ b/doc/bugs/Podcast_filename_encoding_breaks_Android_client/comment_1_1bb8e3f3a3d190ed557302df8b8e0335._comment @@ -0,0 +1,16 @@ +[[!comment format=mdwn + username="joey" + subject="""comment 1""" + date="2016-04-04T18:59:05Z" + content=""" +The bug that caused merge to empty the repo was fixed in a newer +version of git-annex than the 5.20150219 that you're using +on your android device. So, you should upgrade. + +git-annex is supposed to be completely filename encoding agnostic. +Need to investigate what might be breaking that on Android. +Perhaps it's really git, and not git-annex, that's +crashing with the "unable to create file" error? + +Seems so; git's entry.c contains that message. +"""]] diff --git a/doc/bugs/Problem_setting_up_encrypted_repository_using_the_assistant_with___40__outside_of_git-annex__41___shared_pgp_key.mdwn b/doc/bugs/Problem_setting_up_encrypted_repository_using_the_assistant_with___40__outside_of_git-annex__41___shared_pgp_key.mdwn new file mode 100644 index 0000000000..1d4ea21053 --- /dev/null +++ b/doc/bugs/Problem_setting_up_encrypted_repository_using_the_assistant_with___40__outside_of_git-annex__41___shared_pgp_key.mdwn @@ -0,0 +1,387 @@ +### Please describe the problem. +Using the assistant on two computers to setup a shared encrypted repository (while sharing the same pgp key) on a third computer leads to files not propagating between one and two. + +The first and second computer does not get changes done on the other. If new files are added on the first computer it appears as if everything works (no error messages) but the files never reach the second computer (and vice versa). + + +### What steps will reproduce the problem? + +Three computers needed. + +* Computer A: Use the assistant to create a repository +* Computer A: Use the assitant to setup a remote repository on Computer C (Add another repository - Remote server - Encrypt with GnuPG key/Encript repository with a new encryption key - Save changes) + +[At this point files propagate from A to C] + +* Computer A: Export the private and public gpg keys to files +* Computer B: Import these private and public gpg files, fix trust to ultimate +* Computer B: Use the assistant to create a repository +* Computer B: Use the assitant to connect with the remote repository on Computer C (Add another repository - Remote server - Combine the repositories) + +[Files created on A before adding B now appear on B] + +[New files created on A do not appear on B, new files created on B do not appear on A. Files from A and B seem to propagate to C (the number of files/directories in the object sub directory on C goes up after adding files on A or B)] + + + +### What version of git-annex are you using? On what operating system? +Computer A: +[[!format sh """ +dirk@A:~$ lsb_release -a +No LSB modules are available. +Distributor ID: Ubuntu +Description: Ubuntu 14.04.1 LTS +Release: 14.04 +Codename: trusty +dirk@A:~$ git-annex version +git-annex version: 5.20140818-g10bf03a +build flags: Assistant Webapp Webapp-secure Pairing Testsuite S3 WebDAV Inotify DBus DesktopNotify XMPP DNS Feeds Quvi TDFA CryptoHash +key/value backends: SHA256E SHA1E SHA512E SHA224E SHA384E SKEIN256E SKEIN512E SHA256 SHA1 SHA512 SHA224 SHA384 SKEIN256 SKEIN512 WORM URL +remote types: git gcrypt S3 bup directory rsync web webdav tahoe glacier ddar hook external +local repository version: 5 +supported repository version: 5 +upgrade supported from repository versions: 0 1 2 4 +dirk@A:~$ + +dirk@A:~$ gpg --list-keys --list-options show-uid-validity +/home/dirk/.gnupg/pubring.gpg +----------------------------- +pub 4096R/0A7AA2A4 2014-08-23 +uid [ultimate] dirk's git-annex encryption key + +dirk@A:~$ gpg --list-secret-keys --list-options show-uid-validity +/home/dirk/.gnupg/secring.gpg +----------------------------- +sec 4096R/0A7AA2A4 2014-08-23 +uid dirk's git-annex encryption key + +dirk@A:~$ +"""]] + +Computer B: +[[!format sh """ +dirk@B:~$ lsb_release -a +No LSB modules are available. +Distributor ID: Ubuntu +Description: Ubuntu 14.04.1 LTS +Release: 14.04 +Codename: trusty +dirk@B:~$ git-annex version +git-annex version: 5.20140818-g10bf03a +build flags: Assistant Webapp Webapp-secure Pairing Testsuite S3 WebDAV Inotify DBus DesktopNotify XMPP DNS Feeds Quvi TDFA CryptoHash +key/value backends: SHA256E SHA1E SHA512E SHA224E SHA384E SKEIN256E SKEIN512E SHA256 SHA1 SHA512 SHA224 SHA384 SKEIN256 SKEIN512 WORM URL +remote types: git gcrypt S3 bup directory rsync web webdav tahoe glacier ddar hook external +dirk@B:~$ + +dirk@B:~$ gpg --list-keys --list-options show-uid-validity +/home/dirk/.gnupg/pubring.gpg +----------------------------- +pub 4096R/0A7AA2A4 2014-08-23 +uid [ultimate] dirk's git-annex encryption key + +dirk@B:~$ gpg --list-secret-keys --list-options show-uid-validity +/home/dirk/.gnupg/secring.gpg +----------------------------- +sec 4096R/0A7AA2A4 2014-08-23 +uid dirk's git-annex encryption key + +dirk@B:~$ +"""]] + +Computer C: +[[!format sh """ +dirk@C:~$ lsb_release -a +No LSB modules are available. +Distributor ID: Debian +Description: Debian GNU/Linux 7.6 (wheezy) +Release: 7.6 +Codename: wheezy +dirk@C:~$ git-annex version +git-annex version: 5.20140717~bpo70+1 +build flags: Assistant Webapp Pairing S3 Inotify DBus XMPP Feeds Quvi TDFA +key/value backends: SHA256E SHA1E SHA512E SHA224E SHA384E SHA256 SHA1 SHA512 SHA224 SHA384 WORM URL +remote types: git gcrypt S3 bup directory rsync web tahoe glacier ddar hook external +dirk@C:~$ +"""]] + +### Please provide any additional information below. + +.git/annex/daemon.log - Computer A +[[!format sh """ +# If you can, paste a complete transcript of the problem occurring here. +# If the problem is with the git-annex assistant, paste in .git/annex/daemon.log +[2014-08-23 15:15:01 CEST] main: starting assistant version 5.20140818-g10bf03a +[2014-08-23 15:15:01 CEST] Cronner: You should enable consistency checking to protect your data. +(scanning...) [2014-08-23 15:15:01 CEST] Watcher: Performing startup scan +(started...) +gpg: new configuration file `/home/dirk/.gnupg/gpg.conf' created +gpg: WARNING: options in `/home/dirk/.gnupg/gpg.conf' are not yet active during this run + +Not enough random bytes available. Please do some other work to give +the OS a chance to collect more entropy! (Need 235 more bytes) +....+++++ + +Not enough random bytes available. Please do some other work to give +the OS a chance to collect more entropy! (Need 196 more bytes) +.......+++++ +gpg: /home/dirk/.gnupg/trustdb.gpg: trustdb created +gpg: key 0A7AA2A4 marked as ultimately trusted +Generating public/private rsa key pair. +Your identification has been saved in /tmp/git-annex-keygen.0/key. +Your public key has been saved in /tmp/git-annex-keygen.0/key.pub. +The key fingerprint is: +7d:02:34:56:d4:86:b6:e5:82:b0:d9:4f:3b:51:b3:c7 dirk@A +The key's randomart image is: ++--[ RSA 2048]----+ +| +ooo | +| .o .o * | +| =.o * + | +| o oo= o E | +| Soo+.. | +| +o | +| . | +| | +| | ++-----------------+ +(encryption setup) (hybrid cipher with gpg key 7815EA570A7AA2A4) gcrypt: Development version -- Repository format MAY CHANGE +gpg: checking the trustdb +gpg: 3 marginal(s) needed, 1 complete(s) needed, PGP trust model +gpg: depth: 0 valid: 1 signed: 0 trust: 0-, 0q, 0n, 0m, 0f, 1u +gcrypt: Repository not found: ssh://dirk@git-annex-C-dirk_1022_annex/~/annex/ +gcrypt: Development version -- Repository format MAY CHANGE +gcrypt: Repository not found: ssh://dirk@git-annex-C-dirk_1022_annex/~/annex/ +gcrypt: Setting up new repository +gcrypt: Remote ID is :id:00RaA3cNQu+nZDMERYMM +gcrypt: Encrypting to: -r 7815EA570A7AA2A4 +gcrypt: Requesting manifest signature +To gcrypt::ssh://dirk@git-annex-C-dirk_1022_annex/~/annex/ + * [new branch] git-annex -> git-annex +ok +[2014-08-23 15:25:46 CEST] main: Syncing with C_annex +gcrypt: Development version -- Repository format MAY CHANGE +gcrypt: Decrypting manifest +gpg: Signature made Sat 23 Aug 2014 03:25:45 PM CEST using RSA key ID 0A7AA2A4 +gpg: Good signature from "dirk's git-annex encryption key" +(Recording state in git...) +gcrypt: Development version -- Repository format MAY CHANGE +gcrypt: Decrypting manifest +gpg: Signature made Sat 23 Aug 2014 03:25:45 PM CEST using RSA key ID 0A7AA2A4 +gpg: Good signature from "dirk's git-annex encryption key" +gcrypt: Encrypting to: -r 7815EA570A7AA2A4 +gcrypt: Requesting manifest signature +To gcrypt::ssh://dirk@git-annex-C-dirk_1022_annex/~/annex/ + * [new branch] git-annex -> synced/git-annex + * [new branch] annex/direct/master -> synced/master +[2014-08-23 15:26:46 CEST] Pusher: Syncing with C_annex +gcrypt: Development version -- Repository format MAY CHANGE +gcrypt: Decrypting manifest +gpg: Signature made Sat 23 Aug 2014 03:25:49 PM CEST using RSA key ID 0A7AA2A4 +gpg: Good signature from "dirk's git-annex encryption key" +Everything up-to-date +[2014-08-23 15:34:01 CEST] Committer: Adding hhhhn.txt +add hhhhn.txt ok +add hhhhn.txt ok +[2014-08-23 15:34:01 CEST] Committer: Committing changes to git +(Recording state in git...) +[2014-08-23 15:34:01 CEST] Pusher: Syncing with C_annex +(Recording state in git...) +gcrypt: Development version -- Repository format MAY CHANGE +(gpg) +GPGHMACSHA1--7a46226ea53e4043cb45e8df6a2382ac2696164e + + 74 100% 0.00kB/s 0:00:00 + 74 100% 0.00kB/s 0:00:00 (xfr#1, to-chk=0/1) +[2014-08-23 15:34:01 CEST] Transferrer: Uploaded hhhhn.txt +gcrypt: Decrypting manifest +gpg: Signature made Sat 23 Aug 2014 03:33:27 PM CEST using RSA key ID 0A7AA2A4 +gpg: Good signature from "dirk's git-annex encryption key" +gcrypt: WARNING: +gcrypt: WARNING: Remote ID has changed! +gcrypt: WARNING: from :id:00RaA3cNQu+nZDMERYMM +gcrypt: WARNING: to :id:h/BFJbR+mE8CEkASZ/tx +gcrypt: WARNING: +gcrypt: Encrypting to: -r 7815EA570A7AA2A4 +gcrypt: Requesting manifest signature +To gcrypt::ssh://dirk@git-annex-C-dirk_1022_annex/~/annex/ + 85b70d6..e1d6871 annex/direct/master -> synced/master + + 99dc810...a7a89ff git-annex -> synced/git-annex (forced update) +[2014-08-23 15:34:07 CEST] Pusher: Syncing with C_annex +(Recording state in git...) +gcrypt: Development version -- Repository format MAY CHANGE +gcrypt: Decrypting manifest +gpg: Signature made Sat 23 Aug 2014 03:34:04 PM CEST using RSA key ID 0A7AA2A4 +gpg: Good signature from "dirk's git-annex encryption key" +gcrypt: Encrypting to: -r 7815EA570A7AA2A4 +gcrypt: Requesting manifest signature +To gcrypt::ssh://dirk@git-annex-C-dirk_1022_annex/~/annex/ + a7a89ff..e68b5a9 git-annex -> synced/git-annex +[2014-08-23 15:48:30 CEST] main: warning git-annex has been shut down +# End of transcript or log. +"""]] + +.git/annex/daemon.log - Computer B +[[!format sh """ +# If you can, paste a complete transcript of the problem occurring here. +# If the problem is with the git-annex assistant, paste in .git/annex/daemon.log +[2014-08-23 15:30:11 CEST] main: starting assistant version 5.20140818-g10bf03a +[2014-08-23 15:30:11 CEST] Cronner: You should enable consistency checking to protect your data. + + dbus failed; falling back to mtab polling (ClientError {clientErrorMessage = "runClient: unable to determine DBUS address", clientErrorFatal = True}) + + No known network monitor available through dbus; falling back to polling +(scanning...) [2014-08-23 15:30:11 CEST] Watcher: Performing startup scan +(started...) +Generating public/private rsa key pair. +Your identification has been saved in /tmp/git-annex-keygen.0/key. +Your public key has been saved in /tmp/git-annex-keygen.0/key.pub. +The key fingerprint is: +b5:c3:6b:af:fc:fe:82:f2:a6:f3:42:e9:50:4b:63:9e dirk@A +The key's randomart image is: ++--[ RSA 2048]----+ +| | +| | +| . | +| =o . | +| =S=+ | +| . E o | +| + o. | +| =oo.. | +| .O=++o. | ++-----------------+ +gcrypt: Development version -- Repository format MAY CHANGE +gcrypt: Decrypting manifest +gpg: Signature made Sat 23 Aug 2014 03:25:49 PM CEST using RSA key ID 0A7AA2A4 +gpg: checking the trustdb +gpg: 3 marginal(s) needed, 1 complete(s) needed, PGP trust model +gpg: depth: 0 valid: 1 signed: 0 trust: 0-, 0q, 0n, 0m, 0f, 1u +gpg: Good signature from "dirk's git-annex encryption key" +gcrypt: Remote ID is :id:00RaA3cNQu+nZDMERYMM +gcrypt: Development version -- Repository format MAY CHANGE +gcrypt: Decrypting manifest +gpg: Signature made Sat 23 Aug 2014 03:25:49 PM CEST using RSA key ID 0A7AA2A4 +gpg: Good signature from "dirk's git-annex encryption key" +gcrypt: Remote ID is :id:00RaA3cNQu+nZDMERYMM +Receiving objects: 14% (1/7) +Receiving objects: 28% (2/7) +Receiving objects: 42% (3/7) +Receiving objects: 57% (4/7) +Receiving objects: 71% (5/7) +Receiving objects: 85% (6/7) +Receiving objects: 100% (7/7) +Receiving objects: 100% (7/7), done. +Receiving objects: 12% (1/8) +Receiving objects: 25% (2/8) +Receiving objects: 37% (3/8) +Receiving objects: 50% (4/8) +Receiving objects: 62% (5/8) +Receiving objects: 75% (6/8) +Receiving objects: 87% (7/8) +Receiving objects: 100% (8/8) +Receiving objects: 100% (8/8), done. +From gcrypt::ssh://dirk@git-annex-C-dirk_1022_annex/~/annex + * [new branch] git-annex -> tmpgcryptremote/git-annex + * [new branch] synced/git-annex -> tmpgcryptremote/synced/git-annex + * [new branch] synced/master -> tmpgcryptremote/synced/master + * [new branch] master -> tmpgcryptremote/master +(merging tmpgcryptremote/git-annex tmpgcryptremote/synced/git-annex into git-annex...) +(Recording state in git...) +(encryption update) (hybrid cipher with gpg key 7815EA570A7AA2A4) gcrypt: Development version -- Repository format MAY CHANGE +gcrypt: Decrypting manifest +gpg: Signature made Sat 23 Aug 2014 03:25:49 PM CEST using RSA key ID 0A7AA2A4 +gpg: Good signature from "dirk's git-annex encryption key" +gcrypt: Remote ID is :id:00RaA3cNQu+nZDMERYMM +From gcrypt::ssh://dirk@git-annex-C-dirk_1022_annex/~/annex + * [new branch] git-annex -> C_annex/git-annex + * [new branch] synced/git-annex -> C_annex/synced/git-annex + * [new branch] synced/master -> C_annex/synced/master + * [new branch] master -> C_annex/master +gcrypt: Development version -- Repository format MAY CHANGE +gcrypt: Decrypting manifest +gpg: Signature made Sat 23 Aug 2014 03:25:49 PM CEST using RSA key ID 0A7AA2A4 +gpg: Good signature from "dirk's git-annex encryption key" +gcrypt: Encrypting to: -r 7815EA570A7AA2A4 +gcrypt: Requesting manifest signature +remote: error: denying non-fast-forward refs/heads/master (you should pull first) +To ssh://dirk@git-annex-C-dirk_1022_annex/~/annex/ + ! [remote rejected] refs/gcrypt/gitception+ -> master (non-fast-forward) +error: failed to push some refs to 'ssh://dirk@git-annex-C-dirk_1022_annex/~/annex/' +error: failed to push some refs to 'gcrypt::ssh://dirk@git-annex-C-dirk_1022_annex/~/annex/' +ok +[2014-08-23 15:31:36 CEST] main: Syncing with C_annex + +Automatic merge went well; stopped before committing as requested +Already up-to-date! +gcrypt: Development version -- Repository format MAY CHANGE +[2014-08-23 15:31:37 CEST] Pusher: Syncing with C_annex +(Recording state in git...) +gcrypt: Development version -- Repository format MAY CHANGE +gcrypt: Decrypting manifest +gcrypt: Repository not found: ssh://dirk@git-annex-C-dirk_1022_annex/~/annex/ +gcrypt: Setting up new repository +gpg: Signature made Sat 23 Aug 2014 03:25:49 PM CEST using RSA key ID 0A7AA2A4 +gpg: Good signature from "dirk's git-annex encryption key" +fatal: ambiguous argument 'refs/heads/synced/master..refs/remotes/C_annex/synced/master': unknown revision or path not in the working tree. +Use '--' to separate paths from revisions, like this: +'git [...] -- [...]' +gcrypt: Development version -- Repository format MAY CHANGE +gcrypt: Remote ID is :id:h/BFJbR+mE8CEkASZ/tx +gcrypt: Encrypting to: -r 7815EA570A7AA2A4 +gcrypt: Requesting manifest signature +gcrypt: Decrypting manifest +gpg: Signature made Sat 23 Aug 2014 03:25:49 PM CEST using RSA key ID 0A7AA2A4 +gpg: Good signature from "dirk's git-annex encryption key" +gcrypt: Encrypting to: -r 7815EA570A7AA2A4 +gcrypt: Requesting manifest signature +To gcrypt::ssh://dirk@git-annex-C-dirk_1022_annex/~/annex/ + * [new branch] git-annex -> synced/git-annex + * [new branch] annex/direct/master -> synced/master +fatal: Not a valid object name refs/gcrypt/gitception+ +To gcrypt::ssh://dirk@git-annex-C-dirk_1022_annex/~/annex/ + 5d2eb63..e4763b8 git-annex -> synced/git-annex + da18915..3068bad annex/direct/master -> synced/master +[2014-08-23 15:32:37 CEST] Pusher: Syncing with C_annex +gcrypt: Development version -- Repository format MAY CHANGE +gcrypt: Decrypting manifest +gpg: Signature made Sat 23 Aug 2014 03:31:43 PM CEST using RSA key ID 0A7AA2A4 +gpg: Good signature from "dirk's git-annex encryption key" +gcrypt: WARNING: +gcrypt: WARNING: Remote ID has changed! +gcrypt: WARNING: from :id:00RaA3cNQu+nZDMERYMM +gcrypt: WARNING: to :id:h/BFJbR+mE8CEkASZ/tx +gcrypt: WARNING: +Everything up-to-date +[2014-08-23 15:33:17 CEST] Committer: Adding fmksmxxs.txt +add fmksmxxs.txt ok +add fmksmxxs.txt ok +[2014-08-23 15:33:18 CEST] Committer: Committing changes to git +(Recording state in git...) +[2014-08-23 15:33:18 CEST] Pusher: Syncing with C_annex +(Recording state in git...) +gcrypt: Development version -- Repository format MAY CHANGE +(gpg) gcrypt: Decrypting manifest +gpg: Signature made Sat 23 Aug 2014 03:31:43 PM CEST using RSA key ID 0A7AA2A4 +gpg: Good signature from "dirk's git-annex encryption key" +gcrypt: Encrypting to: -r 7815EA570A7AA2A4 +gcrypt: Requesting manifest signature + +GPGHMACSHA1--f605f108429ffba3058a2fcf0bc006a1fbe600be + + 70 100% 0.00kB/s 0:00:00 + 70 100% 0.00kB/s 0:00:00 (xfr#1, to-chk=0/1) +[2014-08-23 15:33:20 CEST] Transferrer: Uploaded fmksmxxs.txt +To gcrypt::ssh://dirk@git-annex-C-dirk_1022_annex/~/annex/ + e4763b8..85dbfc5 git-annex -> synced/git-annex + 3068bad..85b70d6 annex/direct/master -> synced/master +[2014-08-23 15:33:25 CEST] Pusher: Syncing with C_annex +(Recording state in git...) +gcrypt: Development version -- Repository format MAY CHANGE +gcrypt: Decrypting manifest +gpg: Signature made Sat 23 Aug 2014 03:33:22 PM CEST using RSA key ID 0A7AA2A4 +gpg: Good signature from "dirk's git-annex encryption key" +gcrypt: Encrypting to: -r 7815EA570A7AA2A4 +gcrypt: Requesting manifest signature +To gcrypt::ssh://dirk@git-annex-C-dirk_1022_annex/~/annex/ + 85dbfc5..99dc810 git-annex -> synced/git-annex +[2014-08-23 15:48:39 CEST] main: warning git-annex has been shut down +# End of transcript or log. +"""]] diff --git a/doc/bugs/Problem_setting_up_encrypted_repository_using_the_assistant_with___40__outside_of_git-annex__41___shared_pgp_key/comment_1_c8e7be58222afff2a4c1df60f657d2ed._comment b/doc/bugs/Problem_setting_up_encrypted_repository_using_the_assistant_with___40__outside_of_git-annex__41___shared_pgp_key/comment_1_c8e7be58222afff2a4c1df60f657d2ed._comment new file mode 100644 index 0000000000..5b26906daf --- /dev/null +++ b/doc/bugs/Problem_setting_up_encrypted_repository_using_the_assistant_with___40__outside_of_git-annex__41___shared_pgp_key/comment_1_c8e7be58222afff2a4c1df60f657d2ed._comment @@ -0,0 +1,35 @@ +[[!comment format=mdwn + username="https://www.google.com/accounts/o8/id?id=AItOawk7iPiqWr3BVPLWEDvJhSSvcOqheLEbLNo" + nickname="Dirk" + subject="comment 1" + date="2014-08-23T18:13:06Z" + content=""" +Restarting the two git-annex instances actually now leads to an error message on computer B. + +[[!format sh \"\"\" +[2014-08-23 20:02:00 CEST] main: starting assistant version 5.20140818-g10bf03a +[2014-08-23 20:02:00 CEST] Cronner: You should enable consistency checking to protect your data. + + dbus failed; falling back to mtab polling (ClientError {clientErrorMessage = \"runClient: unable to determine DBUS address\", clientErrorFatal = True}) +[2014-08-23 20:02:00 CEST] TransferScanner: Syncing with C_annex + + No known network monitor available through dbus; falling back to polling +(scanning...) [2014-08-23 20:02:00 CEST] Watcher: Performing startup scan +gcrypt: Development version -- Repository format MAY CHANGE +(started...) +gcrypt: Decrypting manifest +gpg: Signature made Sat 23 Aug 2014 03:34:08 PM CEST using RSA key ID 0A7AA2A4 +gpg: Good signature from \"dirk's git-annex encryption key\" +gcrypt: Packfile 59a8d97d3d252effb044625e020f9dc8621804649186a5c33c4e47f9e961cc1a does not match digest! +fatal: early EOF +gcrypt: Development version -- Repository format MAY CHANGE +gcrypt: Decrypting manifest +gpg: Signature made Sat 23 Aug 2014 03:34:08 PM CEST using RSA key ID 0A7AA2A4 +gpg: Good signature from \"dirk's git-annex encryption key\" +gcrypt: Encrypting to: -r 7815EA570A7AA2A4 +gcrypt: Requesting manifest signature +To gcrypt::ssh://dirk@git-annex-C-dirk_1022_annex/~/annex/ + e1d6871..85b70d6 annex/direct/master -> synced/master + + e68b5a9...99dc810 git-annex -> synced/git-annex (forced update) +\"\"\"]] +"""]] diff --git a/doc/bugs/Problem_with_windows_version__58___1.9.5.msysgit/comment_1_720dbf9d959a0050a159713cd1386913._comment b/doc/bugs/Problem_with_windows_version__58___1.9.5.msysgit/comment_1_720dbf9d959a0050a159713cd1386913._comment new file mode 100644 index 0000000000..da503f4131 --- /dev/null +++ b/doc/bugs/Problem_with_windows_version__58___1.9.5.msysgit/comment_1_720dbf9d959a0050a159713cd1386913._comment @@ -0,0 +1,11 @@ +[[!comment format=mdwn + username="joey" + subject="""comment 1""" + date="2015-02-10T16:08:21Z" + content=""" +I have tried a clean install of msysgit 1.9.5, and verified that it still +puts `cmd` in the PATH, as long as you follow the instructions and tell it +to. + +Closing this bug. +"""]] diff --git a/doc/bugs/R__47__O_permissions_are_stalking_me_somehow.mdwn b/doc/bugs/R__47__O_permissions_are_stalking_me_somehow.mdwn new file mode 100644 index 0000000000..df480901d0 --- /dev/null +++ b/doc/bugs/R__47__O_permissions_are_stalking_me_somehow.mdwn @@ -0,0 +1,55 @@ +### Please describe the problem. + +I have got data which I was trying to add to an existing annex. Unfortunately I have missed first that some files were not having write permissions so initial 'annex add' failed with similar to below messages, so I have pruned misctmp, adjusted permissions and have tried to add again -- but the same error keeps coming back + +[[!format sh """ +% chmod +w -R S1/DDA-Freeviewing/surf2anat + +% ls -l S1/DDA-Freeviewing/surf2anat/S1_run1_lh* +-rw-r----- 2 yoh retino 241147956 Feb 28 21:46 S1/DDA-Freeviewing/surf2anat/S1_run1_lh_process_surf.nii.gz + +% rm -rf .git/annex/misctmp/* +zsh: sure you want to delete all the files in /mnt/datasets/mikemp/data-annex/.git/annex/misctmp [yn]? y + +% git annex add S1/DDA-Freeviewing/surf2anat/S1_run1_lh_process_surf.nii.gz +add S1/DDA-Freeviewing/surf2anat/S1_run1_lh_process_surf.nii.gz +git-annex: .git/annex/misctmp/S1_run1_lh1804289383846930886: rename: permission denied (Permission denied) +failed +git-annex: add: 1 failed + +% ls -l S1/DDA-Freeviewing/surf2anat/S1_run1_lh* +-r--r----- 2 yoh retino 241147956 Feb 28 21:46 S1/DDA-Freeviewing/surf2anat/S1_run1_lh_process_surf.nii.gz + +% git annex version +git-annex version: 6.20160307+gitgb095561-1~ndall+1 +build flags: Assistant Webapp Pairing Testsuite S3(multipartupload)(storageclasses) WebDAV Inotify DBus DesktopNotify XMPP ConcurrentOutput TorrentParser MagicMime Feeds Quvi +key/value backends: SHA256E SHA256 SHA512E SHA512 SHA224E SHA224 SHA384E SHA384 SHA3_256E SHA3_256 SHA3_512E SHA3_512 SHA3_224E SHA3_224 SHA3_384E SHA3_384 SKEIN256E SKEIN256 SKEIN512E SKEIN512 SHA1E SHA1 MD5E MD5 WORM URL +remote types: git gcrypt S3 bup directory rsync web bittorrent webdav tahoe glacier ddar hook external +local repository version: 5 +supported repository versions: 5 6 +upgrade supported from repository versions: 0 1 2 4 5 + + +% rm -rf .git/annex/misctmp/* +zsh: sure you want to delete all the files in /mnt/datasets/mikemp/data-annex/.git/annex/misctmp [yn]? y + +% chmod +w -R S1/DDA-Freeviewing/surf2anat + +% git annex add --debug S1/DDA-Freeviewing/surf2anat/S1_run1_lh_process_surf.nii.gz +[2016-03-10 10:15:22.578786] read: git ["--git-dir=.git","--work-tree=.","--literal-pathspecs","ls-files","--others","--exclude-standard","-z","--","S1/DDA-Freeviewing/surf2anat/S1_run1_lh_process_surf.nii.gz"] +[2016-03-10 10:15:22.583854] chat: git ["--git-dir=.git","--work-tree=.","--literal-pathspecs","check-attr","-z","--stdin","annex.backend","annex.numcopies","annex.largefiles","--"] +[2016-03-10 10:15:22.584443] read: git ["--version"] +[2016-03-10 10:15:22.588703] process done ExitSuccess +add S1/DDA-Freeviewing/surf2anat/S1_run1_lh_process_surf.nii.gz [2016-03-10 10:15:23.186056] read: sha256sum [".git/annex/misctmp/S1_run1_lh1804289383846930886"] +[2016-03-10 10:15:24.611133] process done ExitSuccess + +git-annex: .git/annex/misctmp/S1_run1_lh1804289383846930886: rename: permission denied (Permission denied) +failed +[2016-03-10 10:15:24.648808] read: git ["--git-dir=.git","--work-tree=.","--literal-pathspecs","diff","--name-only","--diff-filter=T","-z","--","S1/DDA-Freeviewing/surf2anat/S1_run1_lh_process_surf.nii.gz"] +git-annex: add: 1 failed +% ls -l .git/annex/misctmp/S1_run1_lh1804289383846930886 +-r--r----- 2 yoh retino 241147956 Feb 28 21:46 .git/annex/misctmp/S1_run1_lh1804289383846930886 + +"""]] + +[[!meta author=yoh]] diff --git a/doc/bugs/R__47__O_permissions_are_stalking_me_somehow/comment_1_9e4ba1e03a89c219c5e62280757311ba._comment b/doc/bugs/R__47__O_permissions_are_stalking_me_somehow/comment_1_9e4ba1e03a89c219c5e62280757311ba._comment new file mode 100644 index 0000000000..6e77495297 --- /dev/null +++ b/doc/bugs/R__47__O_permissions_are_stalking_me_somehow/comment_1_9e4ba1e03a89c219c5e62280757311ba._comment @@ -0,0 +1,12 @@ +[[!comment format=mdwn + username="joey" + subject="""comment 1""" + date="2016-03-14T17:53:32Z" + content=""" +A file's permissions cannot prevent it being renamed; so I diagnose some +kind of directory permission problem. + +(Also AFAICS git-annex add deals with files whose permissions don't allow +being read by turning on the read bit. And it should never need the write +bit. I can `chmod 000 foo; git annex add foo` and it succeeds..) +"""]] diff --git a/doc/bugs/Race_condition_between_watch__47__assistant_and_addurl.mdwn b/doc/bugs/Race_condition_between_watch__47__assistant_and_addurl.mdwn new file mode 100644 index 0000000000..1fee439281 --- /dev/null +++ b/doc/bugs/Race_condition_between_watch__47__assistant_and_addurl.mdwn @@ -0,0 +1,195 @@ +Addurl can fail due to an apparent race condition when watch or assistant is running and the repository is in direct mode. The following stress test script encounters the bug consistently on my system. I am running git-annex 5.20140320 on on Ubuntu 13.10. + +[[!format sh """ +#!/bin/sh +set -eu + +cleanup() { + local dir + dir="$1"; shift + if [ -d "$dir" ]; then + ( + set -x + fuser -k -w "$dir/annex/.git/annex/daemon.log" || : + find "$dir" -type d -exec chmod 700 '{}' '+' + find "$dir" -type f -exec chmod 600 '{}' '+' + rm -fr "$dir" + ) + fi +} + +go() { + local dir + dir="$(mktemp -d "${TMP:-/tmp}/stress-annex.XXXXXXXXXX")" + trap "cleanup '$dir'" 0 1 2 13 15 + + ( + cd "$dir" + mkdir annex + cd annex + set -x + + git init + git annex init + git annex direct + git annex watch + + for n in $(seq 100); do + git annex addurl --file=foo http://heh.fi/robots.txt + git annex sync + rm -f foo + git annex sync + done + + git annex watch --stop + git annex uninit + ) + + cleanup "$dir" + trap - 0 1 2 13 14 +} + +go +"""]] + +Script output: + +[[!format sh """ +% ./stress-annex ++ git init +Initialized empty Git repository in /tmp/stress-annex.OKj6D8kVmV/annex/.git/ ++ git annex init +init ok +(Recording state in git...) ++ git annex direct +commit +On branch master + +Initial commit + +nothing to commit +ok +direct ok ++ git annex watch ++ seq 100 ++ git annex addurl --file=foo http://heh.fi/robots.txt +addurl foo (downloading http://heh.fi/robots.txt ...) +--2014-03-27 03:14:29-- http://heh.fi/robots.txt +Resolving heh.fi (heh.fi)... 83.145.237.222 +Connecting to heh.fi (heh.fi)|83.145.237.222|:80... connected. +HTTP request sent, awaiting response... 200 OK +Length: 0 [text/plain] +Saving to: ‘/tmp/stress-annex.OKj6D8kVmV/annex/.git/annex/tmp/URL--http&c%%heh.fi%robots.txt’ + + [ <=> ] 0 --.-K/s in 0s + +2014-03-27 03:14:29 (0.00 B/s) - ‘/tmp/stress-annex.OKj6D8kVmV/annex/.git/annex/tmp/URL--http&c%%heh.fi%robots.txt’ saved [0/0] + +(Recording state in git...) +ok +(Recording state in git...) ++ git annex sync +commit ok ++ rm -f foo ++ git annex sync +commit (Recording state in git...) +ok +(Recording state in git...) ++ git annex addurl --file=foo http://heh.fi/robots.txt +addurl foo (downloading http://heh.fi/robots.txt ...) +--2014-03-27 03:14:29-- http://heh.fi/robots.txt +Resolving heh.fi (heh.fi)... 83.145.237.222 +Connecting to heh.fi (heh.fi)|83.145.237.222|:80... connected. +HTTP request sent, awaiting response... 200 OK +Length: 0 [text/plain] +Saving to: ‘/tmp/stress-annex.OKj6D8kVmV/annex/.git/annex/tmp/URL--http&c%%heh.fi%robots.txt’ + + [ <=> ] 0 --.-K/s in 0s + +2014-03-27 03:14:29 (0.00 B/s) - ‘/tmp/stress-annex.OKj6D8kVmV/annex/.git/annex/tmp/URL--http&c%%heh.fi%robots.txt’ saved [0/0] + +(Recording state in git...) +ok +(Recording state in git...) ++ git annex sync +commit ok ++ rm -f foo ++ git annex sync +commit (Recording state in git...) +ok +(Recording state in git...) ++ git annex addurl --file=foo http://heh.fi/robots.txt +addurl foo (downloading http://heh.fi/robots.txt ...) +--2014-03-27 03:14:29-- http://heh.fi/robots.txt +Resolving heh.fi (heh.fi)... 83.145.237.222 +Connecting to heh.fi (heh.fi)|83.145.237.222|:80... connected. +HTTP request sent, awaiting response... 200 OK +Length: 0 [text/plain] +Saving to: ‘/tmp/stress-annex.OKj6D8kVmV/annex/.git/annex/tmp/URL--http&c%%heh.fi%robots.txt’ + + [ <=> ] 0 --.-K/s in 0s + +2014-03-27 03:14:29 (0.00 B/s) - ‘/tmp/stress-annex.OKj6D8kVmV/annex/.git/annex/tmp/URL--http&c%%heh.fi%robots.txt’ saved [0/0] + + +git-annex: /tmp/stress-annex.OKj6D8kVmV/annex/.git/annex/objects/pX/ZJ/SHA256E-s0--e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855/: openTempFile: permission denied (Permission denied) +failed +git-annex: addurl: 1 failed ++ fuser -k -w /tmp/stress-annex.OKj6D8kVmV/annex/.git/annex/daemon.log +/tmp/stress-annex.OKj6D8kVmV/annex/.git/annex/daemon.log: 30704 30709 30735 30738 30778 ++ find /tmp/stress-annex.OKj6D8kVmV -type d -exec chmod 700 {} + ++ find /tmp/stress-annex.OKj6D8kVmV -type f -exec chmod 600 {} + ++ rm -fr /tmp/stress-annex.OKj6D8kVmV +"""]] + +The script also seems to encounter another issue. The output when seq 100 is changed to seq 1 and addurl happens to succeed: + +[[!format sh """ ++ git init +Initialized empty Git repository in /tmp/stress-annex.QEs0pNyS9z/annex/.git/ ++ git annex init +init ok +(Recording state in git...) ++ git annex direct +commit +On branch master + +Initial commit + +nothing to commit +ok +direct ok ++ git annex watch ++ seq 1 ++ git annex addurl --file=foo http://heh.fi/robots.txt +addurl foo (downloading http://heh.fi/robots.txt ...) +--2014-03-27 03:17:20-- http://heh.fi/robots.txt +Resolving heh.fi (heh.fi)... 83.145.237.222 +Connecting to heh.fi (heh.fi)|83.145.237.222|:80... connected. +HTTP request sent, awaiting response... 200 OK +Length: 0 [text/plain] +Saving to: ‘/tmp/stress-annex.QEs0pNyS9z/annex/.git/annex/tmp/URL--http&c%%heh.fi%robots.txt’ + + [ <=> ] 0 --.-K/s in 0s + +2014-03-27 03:17:20 (0.00 B/s) - ‘/tmp/stress-annex.QEs0pNyS9z/annex/.git/annex/tmp/URL--http&c%%heh.fi%robots.txt’ saved [0/0] + +(Recording state in git...) +ok +(Recording state in git...) ++ git annex sync +commit ok ++ rm -f foo ++ git annex sync +commit (Recording state in git...) +ok +(Recording state in git...) ++ git annex watch --stop ++ git annex uninit +git-annex: /tmp/stress-annex.QEs0pNyS9z/annex/.git/annex/objects/pX/ZJ/SHA256E-s0--e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855/SHA256E-s0--e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855.map: removeLink: permission denied (Permission denied) ++ fuser -k -w /tmp/stress-annex.QEs0pNyS9z/annex/.git/annex/daemon.log ++ : ++ find /tmp/stress-annex.QEs0pNyS9z -type d -exec chmod 700 {} + ++ find /tmp/stress-annex.QEs0pNyS9z -type f -exec chmod 600 {} + ++ rm -fr /tmp/stress-annex.QEs0pNyS9z +"""]] diff --git a/doc/bugs/Race_condition_between_watch__47__assistant_and_addurl/comment_1_8f56b8661a600729d7a9d569e8a0ba70._comment b/doc/bugs/Race_condition_between_watch__47__assistant_and_addurl/comment_1_8f56b8661a600729d7a9d569e8a0ba70._comment new file mode 100644 index 0000000000..cf010cd253 --- /dev/null +++ b/doc/bugs/Race_condition_between_watch__47__assistant_and_addurl/comment_1_8f56b8661a600729d7a9d569e8a0ba70._comment @@ -0,0 +1,55 @@ +[[!comment format=mdwn + username="http://johan.kiviniemi.name/" + nickname="Johan" + subject="Another race condition" + date="2014-03-31T03:42:42Z" + content=""" +Here’s another race condition which seems related: + +[[!format sh \"\"\" +% git annex addurl 'quvi:http://youtu.be/-CbFj9K9AQg' +addurl David_Raymond_Christiansen___Dependently_Typed_Programming_in_Idris___A_Demo.webm +--2014-03-31 05:45:49-- http://r4---sn-oxc0a5-ixae.googlevideo.com/videoplayback? +Resolving r4---sn-oxc0a5-ixae.googlevideo.com (r4---sn-oxc0a5-ixae.googlevideo.com)... 83.145.196.143, 2001:1bc8:100:1b::f +Connecting to r4---sn-oxc0a5-ixae.googlevideo.com (r4---sn-oxc0a5-ixae.googlevideo.com)|83.145.196.143|:80... connected. +HTTP request sent, awaiting response... 302 Found +Location: http://r13---sn-5go7dn7s.googlevideo.com/videoplayback? [following] +--2014-03-31 05:45:49-- http://r13---sn-5go7dn7s.googlevideo.com/videoplayback? +Resolving r13---sn-5go7dn7s.googlevideo.com (r13---sn-5go7dn7s.googlevideo.com)... 173.194.48.18, 2a00:1450:400f::12 +Connecting to r13---sn-5go7dn7s.googlevideo.com (r13---sn-5go7dn7s.googlevideo.com)|173.194.48.18|:80... connected. +HTTP request sent, awaiting response... 200 OK +Length: 320557466 (306M) [video/webm] +Saving to: ‘/home/ion/nobackup/media/video/.git/annex/tmp/URL--quvi&chttp&c%%youtu.be%-CbFj9K9AQg’ + +100%[======================================>] 320,557,466 895KB/s in 6m 14s + +2014-03-31 05:52:04 (837 KB/s) - ‘/home/ion/nobackup/media/video/.git/annex/tmp/URL--quvi&chttp&c%%youtu.be%-CbFj9K9AQg’ saved [320557466/320557466] + +(Recording state in git...) +fatal: Unable to create '/home/ion/nobackup/media/video/.git/index.lock': File exists. + +If no other git process is currently running, this probably means a +git process crashed in this repository earlier. Make sure no other git +process is running and remove the file manually to continue. + +git-annex: user error (xargs [\"-0\",\"git\",\"--git-dir=/home/ion/nobackup/media/video/.git\",\"--work-tree=/home/ion/nobackup/media/video\",\"-c\",\"core.bare=false\",\"add\",\"--\"] exited 123) +failed +git-annex: addurl: 1 failed + +% ls -l /home/ion/nobackup/media/video/.git/index.lock +ls: cannot access /home/ion/nobackup/media/video/.git/index.lock: No such file or directory +\"\"\"]] + +The only lines in daemon.log from that time: + +[[!format sh \"\"\" +[2014-03-31 05:52:04 EEST] Committer: Committing changes to git +[2014-03-31 05:52:04 EEST] Pusher: Syncing with heh.fi +Already up-to-date. +To heh.fi:/storage/ion/media/video + 3df241b..095d6c3 git-annex -> synced/git-annex + 10b3166..98074c1 annex/direct/master -> synced/master +Already up-to-date. +\"\"\"]] + +"""]] diff --git a/doc/bugs/Race_condition_between_watch__47__assistant_and_addurl/comment_2_46dc67bdcd174cd50ccc421ec56735ad._comment b/doc/bugs/Race_condition_between_watch__47__assistant_and_addurl/comment_2_46dc67bdcd174cd50ccc421ec56735ad._comment new file mode 100644 index 0000000000..ec3c974d98 --- /dev/null +++ b/doc/bugs/Race_condition_between_watch__47__assistant_and_addurl/comment_2_46dc67bdcd174cd50ccc421ec56735ad._comment @@ -0,0 +1,14 @@ +[[!comment format=mdwn + username="http://joeyh.name/" + ip="209.250.56.244" + subject="comment 2" + date="2014-04-02T20:17:19Z" + content=""" +These races look beniegn, as far as I can see it doesn't cause any data to be lost, or indeed anything to happen that wouldn't have happened if addurl had been run without the assistant running. + +The first race probably has addurl and the assistant both trying to move the file object into the annex at the same time. One wins and moves it; the other loses and sulks. + +The second race has addurl trying to `git add` the file, while the assistant has already noticed the file appeared, `git add`ed it, and committed the add. + +The only way to really avoid these races would be to add a lot of lock checking. Or just make `git annex addurl` and presumably also `git annex add` and maybe several other commands refuse to run when the assistant is running. +"""]] diff --git a/doc/bugs/Recreating_remote_repository__39__s_annex.mdwn b/doc/bugs/Recreating_remote_repository__39__s_annex.mdwn new file mode 100644 index 0000000000..58003f6245 --- /dev/null +++ b/doc/bugs/Recreating_remote_repository__39__s_annex.mdwn @@ -0,0 +1,34 @@ +### Please describe the problem. +My remote repository's annex was lost. In the webapp, I can now click to recreate it, that fails half-way. + +### What steps will reproduce the problem? +Create a remote repo, delete the annex directory. + +### What version of git-annex are you using? On what operating system? +git-annex version: 5.20140517-g0aed6d9 +build flags: Assistant Webapp Webapp-secure Pairing Testsuite S3 WebDAV FsEvents XMPP DNS Feeds Quvi TDFA CryptoHash +key/value backends: SHA256E SHA1E SHA512E SHA224E SHA384E SKEIN256E SKEIN512E SHA256 SHA1 SHA512 SHA224 SHA384 SKEIN256 SKEIN512 WORM URL +remote types: git gcrypt S3 bup directory rsync web webdav tahoe glacier ddar hook external +local repository version: unknown +supported repository version: 5 +upgrade supported from repository versions: 0 1 2 4 + +### Please provide any additional information below. + +[[!format sh """ +gcrypt: Development version -- Repository format MAY CHANGE +gcrypt: Repository not found: ssh://lhunath@satura.lyndir.com/~/annex/ +gcrypt: Setting up new repository +gcrypt: Remote ID is :id:ROTs3SImZEdvdO2yCMku +gcrypt: Encrypting to: -R 0EF21226F43EA6BC +gcrypt: Requesting manifest signature +Warning: remote port forwarding failed for listen port 2222 +Warning: remote port forwarding failed for listen port 37218 +fatal: '~/annex/' does not appear to be a git repository +fatal: Could not read from remote repository. + +Please make sure you have the correct access rights +and the repository exists. +"""]] + +[[!tag moreinfo]] diff --git a/doc/bugs/Recreating_remote_repository__39__s_annex/comment_1_9fffb1329a4a06111cf2afd8552aea62._comment b/doc/bugs/Recreating_remote_repository__39__s_annex/comment_1_9fffb1329a4a06111cf2afd8552aea62._comment new file mode 100644 index 0000000000..94404fca37 --- /dev/null +++ b/doc/bugs/Recreating_remote_repository__39__s_annex/comment_1_9fffb1329a4a06111cf2afd8552aea62._comment @@ -0,0 +1,8 @@ +[[!comment format=mdwn + username="https://www.google.com/accounts/o8/id?id=AItOawmZilYULa6CDEGfuagoDlesyakBgnf-dF8" + nickname="Maarten" + subject="comment 1" + date="2014-05-21T22:21:37Z" + content=""" +I'd delete the repo and recreate it but I can't do that either - deleting works from webapp but when recreating it says there is already a repo by that name. enableing that repo leads me to this situation again. +"""]] diff --git a/doc/bugs/Recreating_remote_repository__39__s_annex/comment_2_b7f5634e136294ea1a3d4ce6de58fc5d._comment b/doc/bugs/Recreating_remote_repository__39__s_annex/comment_2_b7f5634e136294ea1a3d4ce6de58fc5d._comment new file mode 100644 index 0000000000..887fca94af --- /dev/null +++ b/doc/bugs/Recreating_remote_repository__39__s_annex/comment_2_b7f5634e136294ea1a3d4ce6de58fc5d._comment @@ -0,0 +1,12 @@ +[[!comment format=mdwn + username="http://joeyh.name/" + ip="24.159.78.125" + subject="comment 2" + date="2014-05-23T14:27:01Z" + content=""" +What are you clicking on in the webapp? + +What kind of repository did the remote have before it got deleted? Was it a repository encrypted with git-remote-gcrypt? + +Show the contents of ~/annex/.git/config on your local system. +"""]] diff --git a/doc/bugs/Remote_does_not_work_with_UTF-8_filenames.mdwn b/doc/bugs/Remote_does_not_work_with_UTF-8_filenames.mdwn new file mode 100644 index 0000000000..41c7a7d30f --- /dev/null +++ b/doc/bugs/Remote_does_not_work_with_UTF-8_filenames.mdwn @@ -0,0 +1,193 @@ +### Please describe the problem. + +I'm using git-annex-remote-googledrive to sync a large music library. Some files contain special characters like: +chillgressive/\[Astropilot\ Music\]/Unusual\ Cosmic\ Process\ -\ Spacetime/Unusual\ Cosmic\ Process\ -\ Spacetime\ -\ 03\ Souvenirs\ Éphémère.flac + +When running: +"git annex sync google --content -v -d" + + +### What steps will reproduce the problem? + +The problem seems to be, that git annex sends non utf-8 data to the remote script which crashes due to decoding errors. This seems to be git annex bug as the remote should not have to guess which encoding is used but should be able to use utf-8 always. + +environment: +LANG=en_US.UTF-8 +LANGUAGE=en_US.UTF-8 + + +### What version of git-annex are you using? On what operating system? + +nixos + +git-annex version: 6.20180807 +build flags: Assistant Pairing WebDAV Inotify DBus DesktopNotify ConcurrentOutput TorrentParser MagicMime Feeds Testsuite +dependency versions: bloomfilter-2.0.1.0 cryptonite-0.25 DAV-1.3.2 feed-1.0.0.0 ghc-8.4.3 http-client-0.5.13.1 persistent-sqlite-2.8.1.2 torrent-10000.1.1 uuid-1.3.13 +key/value backends: SHA256E SHA256 SHA512E SHA512 SHA224E SHA224 SHA384E SHA384 SHA3_256E SHA3_256 SHA3_512E SHA3_512 SHA3_224E SHA3_224 SHA3_384E SHA3_384 SKEIN256E SKEIN256 SKEIN512E SKEIN512 BLAKE2B256E BLAKE2B256 BLAKE2B512E BLAKE2B512 BLAKE2B160E BLAKE2B160 BLAKE2B224E BLAKE2B224 BLAKE2B384E BLAKE2B384 BLAKE2S256E BLAKE2S256 BLAKE2S160E BLAKE2S160 BLAKE2S224E BLAKE2S224 BLAKE2SP256E BLAKE2SP256 BLAKE2SP224E BLAKE2SP224 SHA1E SHA1 MD5E MD5 WORM URL +remote types: git gcrypt p2p bup directory rsync web bittorrent webdav adb tahoe glacier ddar hook external +operating system: linux x86_64 +supported repository versions: 3 5 6 +upgrade supported from repository versions: 0 1 2 3 4 5 +local repository version: 5 + + +### Please provide any additional information below. + +[[!format sh """ +# If you can, paste a complete transcript of the problem occurring here. +# If the problem is with the git-annex assistant, paste in .git/annex/daemon.log +git annex sync google --content -v -d 1.8m  Tue 18 Sep 2018 12:29:32 PM CEST +[2018-09-18 12:29:41.854290811] read: git ["--git-dir=.git","--work-tree=.","--literal-pathspecs","show-ref","git-annex"] +[2018-09-18 12:29:41.856338835] process done ExitSuccess +[2018-09-18 12:29:41.85643436] read: git ["--git-dir=.git","--work-tree=.","--literal-pathspecs","show-ref","--hash","refs/heads/git-annex"] +[2018-09-18 12:29:41.858259966] process done ExitSuccess +[2018-09-18 12:29:41.859926397] read: git ["--git-dir=.git","--work-tree=.","--literal-pathspecs","log","refs/heads/git-annex..97d762166d735b12c5393a21d5d2eeb27b33eb8d","--pretty=%H","-n1"] +[2018-09-18 12:29:41.861562154] process done ExitSuccess +[2018-09-18 12:29:41.864078093] chat: git ["--git-dir=.git","--work-tree=.","--literal-pathspecs","cat-file","--batch"] +[2018-09-18 12:29:41.864393271] chat: git ["--git-dir=.git","--work-tree=.","--literal-pathspecs","cat-file","--batch-check=%(objectname) %(objecttype) %(objectsize)"] +[2018-09-18 12:29:41.866339921] read: git ["config","--null","--list"] +[2018-09-18 12:29:41.867406295] process done ExitSuccess +[2018-09-18 12:29:41.867536122] read: git ["config","--null","--list"] +[2018-09-18 12:29:41.86768986] read: git ["config","--null","--list"] +[2018-09-18 12:29:41.86875252] chat: /home/poelzi/.nix-profile/bin/git-annex-remote-googledrive [] +[2018-09-18 12:29:42.10211189] git-annex-remote-googledrive[1] --> VERSION 1 +[2018-09-18 12:29:42.102224585] git-annex-remote-googledrive[1] <-- EXTENSIONS INFO +[2018-09-18 12:29:42.10234806] git-annex-remote-googledrive[1] --> EXTENSIONS +[2018-09-18 12:29:42.10240219] git-annex-remote-googledrive[1] <-- EXPORTSUPPORTED +[2018-09-18 12:29:42.102579476] git-annex-remote-googledrive[1] --> EXPORTSUPPORTED-SUCCESS +commit +[2018-09-18 12:29:42.106079317] call: git ["--git-dir=.git","--work-tree=.","--literal-pathspecs","commit","-a","-m","git-annex in poelzi@galaxy:/run/media/poelzi/backup-1/Music"] +On branch master +Your branch is ahead of 'origin/master' by 3 commits. + (use "git push" to publish your local commits) + +nothing to commit, working tree clean +[2018-09-18 12:29:42.325422783] process done ExitFailure 1 +ok +[2018-09-18 12:29:42.325517478] read: git ["--git-dir=.git","--work-tree=.","--literal-pathspecs","symbolic-ref","-q","HEAD"] +[2018-09-18 12:29:42.326719154] process done ExitSuccess +[2018-09-18 12:29:42.326780286] read: git ["--git-dir=.git","--work-tree=.","--literal-pathspecs","show-ref","refs/heads/master"] +[2018-09-18 12:29:42.328496909] process done ExitSuccess +[2018-09-18 12:29:42.328549028] call: git ["--git-dir=.git","--work-tree=.","--literal-pathspecs","show-ref","--verify","-q","refs/heads/synced/master"] +[2018-09-18 12:29:42.329769099] process done ExitSuccess +[2018-09-18 12:29:42.329841385] read: git ["--git-dir=.git","--work-tree=.","--literal-pathspecs","log","refs/heads/master..refs/heads/synced/master","--pretty=%H","-n1"] +[2018-09-18 12:29:42.33163463] process done ExitSuccess +[2018-09-18 12:29:42.331711233] read: git ["--git-dir=.git","--work-tree=.","--literal-pathspecs","show-ref","git-annex"] +[2018-09-18 12:29:42.333577663] process done ExitSuccess +[2018-09-18 12:29:42.333638691] read: git ["--git-dir=.git","--work-tree=.","--literal-pathspecs","show-ref","--hash","refs/heads/git-annex"] +[2018-09-18 12:29:42.335399941] process done ExitSuccess +[2018-09-18 12:29:42.337137236] read: git ["--git-dir=.git","--work-tree=.","--literal-pathspecs","log","refs/heads/git-annex..97d762166d735b12c5393a21d5d2eeb27b33eb8d","--pretty=%H","-n1"] +[2018-09-18 12:29:42.338812954] process done ExitSuccess +[2018-09-18 12:29:42.338954942] read: git ["--git-dir=.git","--work-tree=.","--literal-pathspecs","ls-files","--cached","-z","--"] +[2018-09-18 12:29:42.349381649] chat: git ["--git-dir=.git","--work-tree=.","--literal-pathspecs","check-attr","-z","--stdin","annex.backend","annex.numcopies","annex.largefiles","--"] +[2018-09-18 12:29:42.349788044] read: git ["--version"] +[2018-09-18 12:29:42.351035475] process done ExitSuccess +[2018-09-18 12:30:46.421010356] read: git ["--git-dir=.git","--work-tree=.","--literal-pathspecs","rev-parse","master:"] +[2018-09-18 12:30:46.423208056] process done ExitSuccess +[2018-09-18 12:30:46.426223794] read: git ["--git-dir=.git","--work-tree=.","--literal-pathspecs","show-ref","--hash","refs/heads/git-annex"] +[2018-09-18 12:30:46.428877128] process done ExitSuccess +[2018-09-18 12:30:46.428974324] read: git ["--git-dir=.git","--work-tree=.","--literal-pathspecs","ls-tree","--full-tree","-z","-r","-t","--","97d762166d735b12c5393a21d5d2eeb27b33eb8d"] +[2018-09-18 12:30:50.326843882] process done ExitSuccess +[2018-09-18 12:30:50.326943526] chat: git ["--git-dir=.git","--work-tree=.","--literal-pathspecs","mktree","--batch","-z"] +[2018-09-18 12:30:50.357438031] process done ExitSuccess +[2018-09-18 12:30:50.357526145] chat: git ["--git-dir=.git","--work-tree=.","--literal-pathspecs","commit-tree","ee08aadd5604249699ce157df47426c70f62312b","--no-gpg-sign","-p","97d762166d735b12c5393a21d5d2eeb27b33eb8d"] +[2018-09-18 12:30:50.359994305] process done ExitSuccess +[2018-09-18 12:30:50.360058172] chat: git ["--git-dir=.git","--work-tree=.","--literal-pathspecs","mktree","--batch","-z"] +[2018-09-18 12:30:50.383728957] process done ExitSuccess +[2018-09-18 12:30:50.383984605] chat: git ["--git-dir=.git","--work-tree=.","--literal-pathspecs","commit-tree","acf111be942ccf1cfda391d7ecffc4a9c6a27d6f","--no-gpg-sign","-p","46bf4e6b14c01c587d5eb1be2db75c66479b6e15"] +[2018-09-18 12:30:50.388013862] process done ExitSuccess +[2018-09-18 12:30:50.388101083] call: git ["--git-dir=.git","--work-tree=.","--literal-pathspecs","update-ref","refs/heads/git-annex","125e9dfe8129591025117cc9dc0c752265be7e70"] +[2018-09-18 12:30:50.391058496] process done ExitSuccess +[2018-09-18 12:30:50.391473142] read: git ["--git-dir=.git","--work-tree=.","--literal-pathspecs","diff-tree","-z","--raw","--no-renames","-l0","-r","9557efae9721f725d73f2b2776a2070aad7b0a39","9557efae9721f725d73f2b2776a2070aad7b0a39","--"] +[2018-09-18 12:30:50.402405642] process done ExitSuccess +[2018-09-18 12:30:50.402521039] read: git ["--git-dir=.git","--work-tree=.","--literal-pathspecs","diff-tree","-z","--raw","--no-renames","-l0","-r","33bf55cdf1f58109ee31097308693f833c4cca97","9557efae9721f725d73f2b2776a2070aad7b0a39","--"] +[2018-09-18 12:30:50.415691771] chat: git ["--git-dir=.git","--work-tree=.","--literal-pathspecs","cat-file","--batch"] +[2018-09-18 12:30:50.41669191] chat: git ["--git-dir=.git","--work-tree=.","--literal-pathspecs","cat-file","--batch-check=%(objectname) %(objecttype) %(objectsize)"] +[2018-09-18 12:30:50.4222699] process done ExitSuccess +rename google chillgressive/[Astropilot Music]/Unusual Cosmic Process - Spacetime/Unusual Cosmic Process - Spacetime - 03 Souvenirs �ph�m�re.flac -> .git-annex-tmp-content-SHA256E-s97469487--14f5f5a819093a154f8d660ae1d986332108eb556a9c27dfd36d8f8dd90d30ae.flac [2018-09-18 12:30:50.422844753] git-annex-remote-googledrive[1] <-- PREPARE +[2018-09-18 12:30:50.423065052] git-annex-remote-googledrive[1] --> DEBUG Running .git-annex-remote-googledrive-wrapped version 0.11.1 +[2018-09-18 12:30:50.423111151] Running .git-annex-remote-googledrive-wrapped version 0.11.1 +[2018-09-18 12:30:50.423145446] git-annex-remote-googledrive[1] --> DEBUG Using AnnexRemote version 1.2.0 +[2018-09-18 12:30:50.423239295] Using AnnexRemote version 1.2.0 +[2018-09-18 12:30:50.423278034] git-annex-remote-googledrive[1] --> GETCONFIG prefix +[2018-09-18 12:30:50.423319342] git-annex-remote-googledrive[1] <-- VALUE Music +[2018-09-18 12:30:50.423424438] git-annex-remote-googledrive[1] --> GETCONFIG root_id +[2018-09-18 12:30:50.423486989] git-annex-remote-googledrive[1] <-- VALUE 1vnnTRDDMJWqTjm5hc0BGzxhCW4S7-j7b +[2018-09-18 12:30:50.42358159] git-annex-remote-googledrive[1] --> GETCREDS credentials +[2018-09-18 12:30:50.423727488] git-annex-remote-googledrive[1] <-- CREDS {"access_token":"ya...8Q","client_id":"...","client_secret":"...","refresh_token":"..."} +[2018-09-18 12:30:51.007447341] git-annex-remote-googledrive[1] --> SETCREDS credentials {"access_token":...} +[2018-09-18 12:30:51.348855667] git-annex-remote-googledrive[1] --> PREPARE-SUCCESS +[2018-09-18 12:30:51.348957671] git-annex-remote-googledrive[1] <-- EXPORT chillgressive/[Astropilot Music]/Unusual Cosmic Process - Spacetime/Unusual Cosmic Process - Spacetime - 03 Souvenirs �ph�m�re.flac +[2018-09-18 12:30:51.34902213] git-annex-remote-googledrive[1] <-- RENAMEEXPORT SHA256E-s97469487--14f5f5a819093a154f8d660ae1d986332108eb556a9c27dfd36d8f8dd90d30ae.flac .git-annex-tmp-content-SHA256E-s97469487--14f5f5a819093a154f8d660ae1d986332108eb556a9c27dfd36d8f8dd90d30ae.flac +Traceback (most recent call last): + File "/nix/store/478kqwy2yfrx61swlsnhjajna7sl8lhh-git-annex-remote-googledrive-0.11.1/bin/.git-annex-remote-googledrive-wrapped", line 584, in + main() + File "/nix/store/478kqwy2yfrx61swlsnhjajna7sl8lhh-git-annex-remote-googledrive-0.11.1/bin/.git-annex-remote-googledrive-wrapped", line 580, in main + master.Listen() + File "/nix/store/5wncs360nzq0fgqmv30p8qn0cga8zi0d-python3.6-annexremote-1.2.0/lib/python3.6/site-packages/annexremote/annexremote.py", line 394, in Listen + for line in self.input: + File "/nix/store/hy65mn4wjswqih75gfr6g4q3xgqdm325-python3-3.6.6/lib/python3.6/codecs.py", line 321, in decode + (result, consumed) = self._buffer_decode(data, self.errors, final) +UnicodeDecodeError: 'utf-8' codec can't decode byte 0xc9 in position 125: invalid continuation byte +git-annex: external special remote protocol error, unexpectedly received "" (unable to parse command) + + rename failed; deleting instead +[2018-09-18 12:30:51.384823525] git-annex-remote-googledrive[1] <-- EXPORT chillgressive/[Astropilot Music]/Unusual Cosmic Process - Spacetime/Unusual Cosmic Process - Spacetime - 03 Souvenirs �ph�m�re.flac +git-annex: fd:16: hFlush: resource vanished (Broken pipe) +failed +rename google chillgressive/[Astropilot Music]/Unusual Cosmic Process - Elysium/Unusual Cosmic Process - Elysium - 04 Souvenirs �ph�m�re (Trance Version).flac -> .git-annex-tmp-content-SHA256E-s90158047--578d406e3db07835d3c4045096c7f8333c5adbbfe1003d37121d5b0656f45688.flac [2018-09-18 12:30:51.385017728] git-annex-remote-googledrive[1] <-- EXPORT chillgressive/[Astropilot Music]/Unusual Cosmic Process - Elysium/Unusual Cosmic Process - Elysium - 04 Souvenirs �ph�m�re (Trance Version).flac +git-annex: fd:16: hFlush: resource vanished (Broken pipe) + + rename failed; deleting instead +[2018-09-18 12:30:51.385128278] git-annex-remote-googledrive[1] <-- EXPORT chillgressive/[Astropilot Music]/Unusual Cosmic Process - Elysium/Unusual Cosmic Process - Elysium - 04 Souvenirs �ph�m�re (Trance Version).flac +git-annex: fd:16: hFlush: resource vanished (Broken pipe) +failed +rename google chillgressive/[Astropilot Music]/Dream Twice - Enta/Dream Twice - Enta - 01 Artem�sia.flac -> .git-annex-tmp-content-SHA256E-s41053932--7bb9b80766e1d445bb39442e39e3e727ba7329d00186cccb5b2b1e2b28b493b1.flac [2018-09-18 12:30:51.38521185] git-annex-remote-googledrive[1] <-- EXPORT chillgressive/[Astropilot Music]/Dream Twice - Enta/Dream Twice - Enta - 01 Artem�sia.flac +git-annex: fd:16: hFlush: resource vanished (Broken pipe) + + rename failed; deleting instead +[2018-09-18 12:30:51.385281886] git-annex-remote-googledrive[1] <-- EXPORT chillgressive/[Astropilot Music]/Dream Twice - Enta/Dream Twice - Enta - 01 Artem�sia.flac +git-annex: fd:16: hFlush: resource vanished (Broken pipe) +failed +rename google psychill/[Badger Records]/Badgers Records - VA - Inner Holographic Reality/Badgers Records - VA - Inner Holographic Reality - 01 Cord & Frankie L�t� - Lullaby.flac -> .git-annex-tmp-content-SHA256E-s43901186--9d51034d81247e3e0890d81c1e1c20c06149aac3bf4451d880523dcc89c3de82.flac [2018-09-18 12:30:51.385360183] git-annex-remote-googledrive[1] <-- EXPORT psychill/[Badger Records]/Badgers Records - VA - Inner Holographic Reality/Badgers Records - VA - Inner Holographic Reality - 01 Cord & Frankie L�t� - Lullaby.flac +git-annex: fd:16: hFlush: resource vanished (Broken pipe) + + rename failed; deleting instead +[2018-09-18 12:30:51.385429424] git-annex-remote-googledrive[1] <-- EXPORT psychill/[Badger Records]/Badgers Records - VA - Inner Holographic Reality/Badgers Records - VA - Inner Holographic Reality - 01 Cord & Frankie L�t� - Lullaby.flac +git-annex: fd:16: hFlush: resource vanished (Broken pipe) +failed +rename google chillgressive/[Astropilot Music]/Skytechnic - Astral Insıgnias/Skytechnic - Astral Ins?gnias - 08 Vac�o.flac -> .git-annex-tmp-content-SHA256E-s89899720--afbc07b561f058db6da5849f728d665d14cc6ea42e2768fecf6396d0009e2861.flac [2018-09-18 12:30:51.385513597] git-annex-remote-googledrive[1] <-- EXPORT chillgressive/[Astropilot Music]/Skytechnic - Astral Insıgnias/Skytechnic - Astral Ins?gnias - 08 Vac�o.flac +git-annex: fd:16: hFlush: resource vanished (Broken pipe) + + rename failed; deleting instead +[2018-09-18 12:30:51.385594622] git-annex-remote-googledrive[1] <-- EXPORT chillgressive/[Astropilot Music]/Skytechnic - Astral Insıgnias/Skytechnic - Astral Ins?gnias - 08 Vac�o.flac +git-annex: fd:16: hFlush: resource vanished (Broken pipe) +failed +rename google chillgressive/[Astropilot Music]/Unusual Cosmic Process - Elysium/Unusual Cosmic Process - Elysium - 03 Souvenirs �ph�m�re (Ambient Version).flac -> .git-annex-tmp-content-SHA256E-s81128227--c4222d79979c8562c44847869c20d172f51a8d4670192d860aa2dae1ba3d54c8.flac [2018-09-18 12:30:51.385768001] git-annex-remote-googledrive[1] <-- EXPORT chillgressive/[Astropilot Music]/Unusual Cosmic Process - Elysium/Unusual Cosmic Process - Elysium - 03 Souvenirs �ph�m�re (Ambient Version).flac +git-annex: fd:16: hFlush: resource vanished (Broken pipe) + + rename failed; deleting instead +[2018-09-18 12:30:51.385830411] git-annex-remote-googledrive[1] <-- EXPORT chillgressive/[Astropilot Music]/Unusual Cosmic Process - Elysium/Unusual Cosmic Process - Elysium - 03 Souvenirs �ph�m�re (Ambient Version).flac +git-annex: fd:16: hFlush: resource vanished (Broken pipe) +failed +[2018-09-18 12:30:51.38631869] read: git ["--git-dir=.git","--work-tree=.","--literal-pathspecs","ls-tree","--full-tree","-z","-r","--","9557efae9721f725d73f2b2776a2070aad7b0a39"] +export google .gitignore (not available) failed +export google bin/bandcamp-rip-album.variant-5bf5 (not available) failed +export google bin/flac2opus.variant-1201.py (not available) failed +export google chillgressive/[Astropilot Music]/Dream Twice - Enta/Dream Twice - Enta - 01 Artemísia.flac +[2018-09-18 12:30:59.661174242] git-annex-remote-googledrive[1] <-- EXPORT chillgressive/[Astropilot Music]/Dream Twice - Enta/Dream Twice - Enta - 01 Artemísia.flac +git-annex: fd:16: hFlush: resource vanished (Broken pipe) +failed +export google chillgressive/[Astropilot Music]/Dream Twice - Enta/Dream Twice - Enta - 02 Marwell.flac +[2018-09-18 12:30:59.663165427] git-annex-remote-googledrive[1] <-- EXPORT chillgressive/[Astropilot Music]/Dream Twice - Enta/Dream Twice - Enta - 02 Marwell.flac +git-annex: fd:16: hFlush: resource vanished (Broken pipe) +failed + + +# End of transcript or log. +"""]] + +### Have you had any luck using git-annex before? (Sometimes we get tired of reading bug reports all day and a lil' positive end note does wonders) + +I love git annex, using it very day :) + diff --git a/doc/bugs/Remote_repositories_have_to_be_setup_encrypted.mdwn b/doc/bugs/Remote_repositories_have_to_be_setup_encrypted.mdwn new file mode 100644 index 0000000000..bcac962ee6 --- /dev/null +++ b/doc/bugs/Remote_repositories_have_to_be_setup_encrypted.mdwn @@ -0,0 +1,28 @@ +What steps will reproduce the problem? + +Create a new remote repository in the webapp. Get to the final phase of the setup where it asks you if you want to encrypt it, yet no other option is given to continue. + +What is the expected output? What do you see instead? + +At least two options: + +1. Use an encrypted rsync repository on the server (the existing one) +2. Use an unencrypted rsync repository on the server + +What version of git-annex are you using? On what operating system? + + $ ./git-annex version + git-annex version: 3.20130102 + + $ uname -a + Linux wintermute 3.2.0-35-generic #55-Ubuntu SMP Wed Dec 5 17:45:18 UTC 2012 i686 i686 i386 GNU/Linux + + $ lsb_release -a + Distributor ID: Ubuntu + Description: Ubuntu 12.04.1 LTS + Release: 12.04 + Codename: precise + +[[!meta title="webapp does not allow disabling encryption on rsync special remotes"]] +[[!tag /design/assistant]] +[[!tag confirmed]] diff --git a/doc/bugs/Remote_repositories_have_to_be_setup_encrypted/comment_1_95f73315657bc35a8d3ff9b4ba207af0._comment b/doc/bugs/Remote_repositories_have_to_be_setup_encrypted/comment_1_95f73315657bc35a8d3ff9b4ba207af0._comment new file mode 100644 index 0000000000..f54b4626b2 --- /dev/null +++ b/doc/bugs/Remote_repositories_have_to_be_setup_encrypted/comment_1_95f73315657bc35a8d3ff9b4ba207af0._comment @@ -0,0 +1,8 @@ +[[!comment format=mdwn + username="http://joeyh.name/" + ip="4.152.108.211" + subject="comment 1" + date="2013-01-03T18:02:21Z" + content=""" +Specifically, this affects only rsync special remotes. All the other ones have a check box that allows enabling encryption or not. I didn't get around to adding that for rsync due to some complications in the code. Of course, you can use `git annex initremote` at the command line to set up non-encrypted rsync remotes. +"""]] diff --git a/doc/bugs/Remote_webapp_on_debian_stretch_not_responding.mdwn b/doc/bugs/Remote_webapp_on_debian_stretch_not_responding.mdwn new file mode 100644 index 0000000000..9109ae3464 --- /dev/null +++ b/doc/bugs/Remote_webapp_on_debian_stretch_not_responding.mdwn @@ -0,0 +1,16 @@ +In a fresh repository installation on debian stretch (arm) (git-annex version: 6.20170101) +the remote webapp will start and present the dashboard page, +however any request made by clicking any link does not receive a response. + +I have a workaround: to ssh into my server and access the webapp locally using lynx browser, and that does seem to work well. + +It presents me with a warning: + +warning + + WebApp crashed: xdg-open: createProcess: runInteractiveProcess: exec: + does not exist (No such file or directory) + +I'm not getting any errors, just the requests not returning responses. + +I'm pretty new to this software. diff --git a/doc/bugs/Remote_webapp_on_debian_stretch_not_responding/comment_1_e2a0d0f458d263eae15640f36d59f071._comment b/doc/bugs/Remote_webapp_on_debian_stretch_not_responding/comment_1_e2a0d0f458d263eae15640f36d59f071._comment new file mode 100644 index 0000000000..8ba6d9946b --- /dev/null +++ b/doc/bugs/Remote_webapp_on_debian_stretch_not_responding/comment_1_e2a0d0f458d263eae15640f36d59f071._comment @@ -0,0 +1,59 @@ +[[!comment format=mdwn + username="moaxey" + avatar="http://cdn.libravatar.org/avatar/06abae5b84ce9bbcd39c1cbf49817d29" + subject="I also found this log" + date="2018-04-17T22:52:15Z" + content=""" +``` + Here's everything that has been logged by the git-annex assistant, as + well as by programs it has run. +[2018-04-17 21:55:06.712567732] main: starting assistant version 6.20170101 +[2018-04-17 21:55:06.754681296] Cronner: You should enable consistency checking +to protect your data. + + No known network monitor available through dbus; falling back to polling + + No known volume monitor available through dbus; falling back to mtab polling +(scanning...) [2018-04-17 21:55:06.920804804] Watcher: Performing startup scan +(started...) +WebApp crashed: unable to bind to local socket +CallStack (from HasCallStack): + error, called at ./Utility/WebApp.hs:124:19 in main:Utility.WebApp +[2018-04-17 21:55:07.19622235] WebApp: warning WebApp crashed: unable to bind to + local socket +CallStack (from HasCallStack): + error, called at ./Utility/WebApp.hs:124:19 in main:Utility.WebApp +[2018-04-17 21:57:06.821805552] main: starting assistant version 6.20170101 +[2018-04-17 21:57:06.858432032] Cronner: You should enable consistency checking +to protect your data. + + No known network monitor available through dbus; falling back to polling +(scanning...) [2018-04-17 21:57:07.025115165] Watcher: Performing startup scan +(started...) + + No known volume monitor available through dbus; falling back to mtab polling +WebApp crashed: unable to bind to local socket +CallStack (from HasCallStack): + error, called at ./Utility/WebApp.hs:124:19 in main:Utility.WebApp +[2018-04-17 21:57:07.243030677] WebApp: warning WebApp crashed: unable to bind t +o local socket +CallStack (from HasCallStack): + error, called at ./Utility/WebApp.hs:124:19 in main:Utility.WebApp +[2018-04-17 21:59:07.137839178] main: starting assistant version 6.20170101 +[2018-04-17 21:59:07.177856054] Cronner: You should enable consistency checking +to protect your data. + + No known network monitor available through dbus; falling back to polling + + No known volume monitor available through dbus; falling back to mtab polling + +(scanning...) [2018-04-17 22:24:10.365052622] Watcher: Performing startup scan +(started...) +(merging synced/git-annex into git-annex...) +(recording state in git...) + + Performed startup scan + +``` + +"""]] diff --git a/doc/bugs/Renamed_special_remote_cannot_be_reactivated_by_the_webapp.mdwn b/doc/bugs/Renamed_special_remote_cannot_be_reactivated_by_the_webapp.mdwn new file mode 100644 index 0000000000..ccf6ef1caa --- /dev/null +++ b/doc/bugs/Renamed_special_remote_cannot_be_reactivated_by_the_webapp.mdwn @@ -0,0 +1,30 @@ +Setup: + +* fresh install of Debian Wheezy with git-annex 4.20130227 pulled in from unstable +* clone existing repository and activate assistant +* repository has encrypted rsync remote originally setup with the name `metaarray` +* this remote was renamed to `ma` a long time ago, using the webapp +* had to perform this rename on each client + +Steps: + +* attempt to reactivate special remote using webapp repositories page, on reinstalled machine + +Expected: + +* special remote starts working +* renaming special remotes ought to survive clones + +Actual: + +* firstly, special remote activation page has blank hostname box and the hostname of the machine is in the username box +* form gives error "cannot change encryption type of existing remote" + +Workaround: + +* execute `git annex initremote metaarray` +* rename `metaarray` to `ma` again using the webapp + +Perhaps the renaming of the remote not surviving clones is unavoidable, but the webapp should be able to cope with the situation. Thanks. + +[[!tag /design/assistant]] diff --git a/doc/bugs/Resource_exhausted.mdwn b/doc/bugs/Resource_exhausted.mdwn new file mode 100644 index 0000000000..3f125b08aa --- /dev/null +++ b/doc/bugs/Resource_exhausted.mdwn @@ -0,0 +1,45 @@ +What steps will reproduce the problem? +My annex dir has 23459 files and uses 749MB disk space. +Just create a repository put this dir inside, and git-annex will crash. + +What is the expected output? What do you see instead? +I expect git-annex handles large number of files, and does not watch every single file of it. + +What version of git-annex are you using? On what operating system? +I'm using git-annex linux build, version 2013.04.17. + +Please provide any additional information below. + + [2013-04-17 23:52:35 CEST] Transferrer: Downloaded pappas_hu..di_44.jpg + git-annex: runInteractiveProcess: pipe: Too many open files + Committer crashed: lsof: createProcess: resource exhausted (Too many open files) + [2013-04-17 23:53:52 CEST] Committer: warning Committer crashed: lsof: createProcess: resource exhausted (Too many open files) + git-annex: runInteractiveProcess: pipe: Too many open files + git: createProcess: resource exhausted (Too many open files) + DaemonStatus crashed: /home/user/Desktop/down/annex_test/.git/annex/daemon.status.tmp21215: openFile: resource exhausted (Too many open files) + [2013-04-17 23:57:24 CEST] DaemonStatus: warning DaemonStatus crashed: /home/user/Desktop/down/annex_test/.git/annex/daemon.status.tmp21215: openFile: resource exhausted (Too many open files) + git-annex: runInteractiveProcess: pipe: Too many open files + git: createProcess: resource exhausted (Too many open files) + git-annex: runInteractiveProcess: pipe: Too many open files + NetWatcherFallback crashed: git: createProcess: resource exhausted (Too many open files) + [2013-04-18 00:27:17 CEST] NetWatcherFallback: warning NetWatcherFallback crashed: git: createProcess: resource exhausted (Too many open files) + git-annex: runInteractiveProcess: pipe: Too many open files + git-annex: git: createProcess: resource exhausted (Too many open files) + git-annex: accept: resource exhausted (Too many open files) + +Instead of raising system's limit (which is a neverending story), can we make git-annex only watch a directory and not every file of it? + +Or could the user specify some directory which he knows it is rarely change, to not be watched only check it once a day? + +The best would be if git annex could automatically adapt itself. +Ie. it watches eg. 200 files, and if some of it does not change for three days, then it drops from the watching basket, and those who changed (noticed while sanity checked) it adds to the basket. + +I don't really want to raise the ulimit, because my ultimate goal is to have git-annex on multiple raspberry pi with external harddrive (one at my home, one at my mom's home, one at my friends home, etc, etc). And raspberry is fairly low on resource. + +I'm interested in your thoughts. + +Best, + Laszlo + +[[!tag /design/assistant]] +[[!meta title="assistant can try to add too many files at once in batch add mode"]] diff --git a/doc/bugs/Resource_exhausted/comment_10_bccf9528ffe963154c92ce49762e7ea6._comment b/doc/bugs/Resource_exhausted/comment_10_bccf9528ffe963154c92ce49762e7ea6._comment new file mode 100644 index 0000000000..da2a15ba2a --- /dev/null +++ b/doc/bugs/Resource_exhausted/comment_10_bccf9528ffe963154c92ce49762e7ea6._comment @@ -0,0 +1,10 @@ +[[!comment format=mdwn + username="https://www.google.com/accounts/o8/id?id=AItOawln3ckqKx0x_xDZMYwa9Q1bn4I06oWjkog" + nickname="Michael" + subject="comment 10" + date="2013-08-26T18:33:40Z" + content=""" +@Joey: it was a \"pretty large\" transfer, several hundred gigabytes in perhaps ~100000 files. The copying was going to a GPG-encrypted directory remote. +The error only happened once or twice so far. Point taken about find in /proc; I'll do that if it happens next time. + +"""]] diff --git a/doc/bugs/Resource_exhausted/comment_11_24ad3a76a25a787ac59e2c5270709e72._comment b/doc/bugs/Resource_exhausted/comment_11_24ad3a76a25a787ac59e2c5270709e72._comment new file mode 100644 index 0000000000..9cf8d04e01 --- /dev/null +++ b/doc/bugs/Resource_exhausted/comment_11_24ad3a76a25a787ac59e2c5270709e72._comment @@ -0,0 +1,12 @@ +[[!comment format=mdwn + username="http://joeyh.name/" + ip="209.250.56.87" + subject="comment 11" + date="2013-12-16T17:20:34Z" + content=""" +I now understand the problem described in comment 6, where once it started failing, it would leak one file descriptor per failure. + +I think that failure mode was fixed by accident in the changes in [[!commit 2fd63f3cfac705f0a18f4bcbe0489ce8ea1800d7]]. + +This doesn't explain what would open so many files to get it into that failure mode, however. +"""]] diff --git a/doc/bugs/Resource_exhausted/comment_12_1943ff77f3bccf885229ecc10c82399d._comment b/doc/bugs/Resource_exhausted/comment_12_1943ff77f3bccf885229ecc10c82399d._comment new file mode 100644 index 0000000000..a2b82023dc --- /dev/null +++ b/doc/bugs/Resource_exhausted/comment_12_1943ff77f3bccf885229ecc10c82399d._comment @@ -0,0 +1,12 @@ +[[!comment format=mdwn + username="https://www.google.com/accounts/o8/id?id=AItOawkipQLNyt8RHREHpg2k5wdYeRSCCvSNSBg" + nickname="Tim" + subject="Having "too many open files" message show up" + date="2014-01-13T14:57:11Z" + content=""" +This is happening to me now about twice per day. I've got a repo that is completely synced already and I'm working on a project in IntelliJ. When I check git-annex I often see this message when it is doing a startup scan or consistency check. + +The message can be seen here: http://imgur.com/Xb4LA73 + +Are there logs somewhere that I can gather and supply to you to help track this down? I'm using the 2014-01-07 release of git-annex for Mac OS. +"""]] diff --git a/doc/bugs/Resource_exhausted/comment_13_c3cac0717232e04e89df62efa1db0870._comment b/doc/bugs/Resource_exhausted/comment_13_c3cac0717232e04e89df62efa1db0870._comment new file mode 100644 index 0000000000..5e8ca65f13 --- /dev/null +++ b/doc/bugs/Resource_exhausted/comment_13_c3cac0717232e04e89df62efa1db0870._comment @@ -0,0 +1,8 @@ +[[!comment format=mdwn + username="https://www.google.com/accounts/o8/id?id=AItOawkipQLNyt8RHREHpg2k5wdYeRSCCvSNSBg" + nickname="Tim" + subject="Happened four times yesterday" + date="2014-01-14T14:33:02Z" + content=""" +Maybe you can e-mail me and we can discuss this? +"""]] diff --git a/doc/bugs/Resource_exhausted/comment_1_a5ef7a62d4ed9365f9448520bb17e3b5._comment b/doc/bugs/Resource_exhausted/comment_1_a5ef7a62d4ed9365f9448520bb17e3b5._comment new file mode 100644 index 0000000000..9bd4b0779f --- /dev/null +++ b/doc/bugs/Resource_exhausted/comment_1_a5ef7a62d4ed9365f9448520bb17e3b5._comment @@ -0,0 +1,9 @@ +[[!comment format=mdwn + username="https://www.google.com/accounts/o8/id?id=AItOawm5iosFbL2By7UFeViqkc6v-hoAtqILeDA" + nickname="Laszlo" + subject="comment 1" + date="2013-04-18T06:20:13Z" + content=""" +Once resource exhausted git-annex has git [defunct] process too. + +"""]] diff --git a/doc/bugs/Resource_exhausted/comment_2_cdba2015e603f3c21f3e1697dd6fafcd._comment b/doc/bugs/Resource_exhausted/comment_2_cdba2015e603f3c21f3e1697dd6fafcd._comment new file mode 100644 index 0000000000..fb0f4c16b1 --- /dev/null +++ b/doc/bugs/Resource_exhausted/comment_2_cdba2015e603f3c21f3e1697dd6fafcd._comment @@ -0,0 +1,18 @@ +[[!comment format=mdwn + username="http://joeyh.name/" + nickname="joey" + subject="comment 2" + date="2013-04-23T19:44:04Z" + content=""" +I have tried repeatedly to reproduce this problem, and I cannot. + +git-annex does *not* keep every file open. It tends to have less than 10 open file descriptors at any one time. + +I thought perhaps `lsof` opened every file, but it does not seem to, either. + +So far, I have no indication that the problem had to do with git-annex at all. If some other program on the system opened a great many files, it could cause this to happen to git-annex. + +You pasted a debug log that shows that the problem persisted for several minutes. So you should make it happen again, and in that time period, investigate what program has so many files open. You can do this with lsof, or, if lsof won't run, by looking in /proc/$pid/fd/ + +Or, of course, give me enough information to reproduce the problem. \"I have 23459 files\" isn't much help.. +"""]] diff --git a/doc/bugs/Resource_exhausted/comment_3_747d16d050fdcf69dd3d2bc5ca469a2e._comment b/doc/bugs/Resource_exhausted/comment_3_747d16d050fdcf69dd3d2bc5ca469a2e._comment new file mode 100644 index 0000000000..07badd83a7 --- /dev/null +++ b/doc/bugs/Resource_exhausted/comment_3_747d16d050fdcf69dd3d2bc5ca469a2e._comment @@ -0,0 +1,39 @@ +[[!comment format=mdwn + username="http://adamspiers.myopenid.com/" + nickname="Adam" + subject="I just reproduced this" + date="2013-07-26T23:37:26Z" + content=""" +On openSUSE 12.3 with this version (which I'm sure is horribly old): + + git-annex version: 4.20130314 + local repository version: 3 + default repository version: 3 + supported repository versions: 3 4 + upgrade supported from repository versions: 0 1 2 + build flags: Assistant Webapp Pairing Testsuite S3 WebDAV Inotify DBus XMPP DNS + +I ran 'git annex get' on a large repository, and got this: + + [snipped] + git-annex: /home/adam/music/.git/annex/transfer/failed/download/9c930baf-6b25-4294-b1f9-a90ba9663fb7/SHA1-s7227476--49d203960b062ec00c0d1d7042c4b6aa6720b976: openFile: resource exhausted (Too many open files) + failed + git-annex: runInteractiveProcess: pipe: Too many open files + + git-annex: git: runInteractiveProcess: resource exhausted (Too many open files) + failed + git-annex: get: 1207 failed + +Then I ran it again and saw that after every file retrieved, git-annex leaks another lockfile. lsof shows an ever increasing number of files like this: + + [snipped] + git-annex 32498 adam 86rR REG 253,3 0 787353 /home/adam/music/.git/annex/ssh/adam@atlantic.lock + git-annex 32498 adam 87rR REG 253,3 0 787353 /home/adam/music/.git/annex/ssh/adam@atlantic.lock + git-annex 32498 adam 88rR REG 253,3 0 787353 /home/adam/music/.git/annex/ssh/adam@atlantic.lock + git-annex 32498 adam 89rR REG 253,3 0 787353 /home/adam/music/.git/annex/ssh/adam@atlantic.lock + git-annex 32498 adam 90rR REG 253,3 0 787353 /home/adam/music/.git/annex/ssh/adam@atlantic.lock + git-annex 32498 adam 91rR REG 253,3 0 787353 /home/adam/music/.git/annex/ssh/adam@atlantic.lock + git-annex 32498 adam 92rR REG 253,3 0 787353 /home/adam/music/.git/annex/ssh/adam@atlantic.lock + git-annex 32498 adam 93r FIFO 0,8 0t0 28752703 pipe + git-annex 32498 adam 94r FIFO 0,8 0t0 28765910 pipe +"""]] diff --git a/doc/bugs/Resource_exhausted/comment_4_1e9b74e60da57c3d5f08c1eb3801c1d2._comment b/doc/bugs/Resource_exhausted/comment_4_1e9b74e60da57c3d5f08c1eb3801c1d2._comment new file mode 100644 index 0000000000..eee206168f --- /dev/null +++ b/doc/bugs/Resource_exhausted/comment_4_1e9b74e60da57c3d5f08c1eb3801c1d2._comment @@ -0,0 +1,10 @@ +[[!comment format=mdwn + username="http://joeyh.name/" + ip="4.152.246.110" + subject="comment 4" + date="2013-07-28T00:23:08Z" + content=""" +Hmm, Adam your version is older than the bug reporter's version. OTOH, while there were several FD leak fixes after your version, none of them were to Annex.LockPool, which is what's used for the ssh lock files. + +I can't reproduce it with `git annex get` and the current release.. can you? +"""]] diff --git a/doc/bugs/Resource_exhausted/comment_5_f55d933bce77fd2185ebd0cc46fe57ec._comment b/doc/bugs/Resource_exhausted/comment_5_f55d933bce77fd2185ebd0cc46fe57ec._comment new file mode 100644 index 0000000000..19bcdb3763 --- /dev/null +++ b/doc/bugs/Resource_exhausted/comment_5_f55d933bce77fd2185ebd0cc46fe57ec._comment @@ -0,0 +1,64 @@ +[[!comment format=mdwn + username="https://www.google.com/accounts/o8/id?id=AItOawln3ckqKx0x_xDZMYwa9Q1bn4I06oWjkog" + nickname="Michael" + subject="I just reproduced this with the latest version" + date="2013-08-11T03:17:11Z" + content=""" +At some point during a large copy, there's an ever increasing number of pipes in /proc/git-annex-pid/fd +As soon as it hits the limit (1023 in my case), copies start failing + + etc + r-x------ 1 michael michael 64 Aug 10 20:14 851 -> pipe:[2250609] + l-wx------ 1 michael michael 64 Aug 10 20:14 852 -> pipe:[2251549] + lr-x------ 1 michael michael 64 Aug 10 20:14 853 -> pipe:[2251550] + l-wx------ 1 michael michael 64 Aug 10 20:14 854 -> pipe:[2250612] + lr-x------ 1 michael michael 64 Aug 10 20:14 855 -> pipe:[2250613] + l-wx------ 1 michael michael 64 Aug 10 20:14 856 -> pipe:[2246639] + lr-x------ 1 michael michael 64 Aug 10 20:14 857 -> pipe:[2246640] + l-wx------ 1 michael michael 64 Aug 10 20:14 858 -> pipe:[2246642] + lr-x------ 1 michael michael 64 Aug 10 20:14 859 -> pipe:[2246643] + l-wx------ 1 michael michael 64 Aug 10 20:13 86 -> pipe:[2241378] + l-wx------ 1 michael michael 64 Aug 10 20:14 860 -> pipe:[2246645] + lr-x------ 1 michael michael 64 Aug 10 20:14 861 -> pipe:[2246646] + l-wx------ 1 michael michael 64 Aug 10 20:14 862 -> pipe:[2246648] + lr-x------ 1 michael michael 64 Aug 10 20:14 863 -> pipe:[2246649] + l-wx------ 1 michael michael 64 Aug 10 20:14 864 -> pipe:[2246653] + lr-x------ 1 michael michael 64 Aug 10 20:14 865 -> pipe:[2246654] + lr-x------ 1 michael michael 64 Aug 10 20:14 866 -> pipe:[2249407] + l-wx------ 1 michael michael 64 Aug 10 20:14 867 -> pipe:[2251789] + l-wx------ 1 michael michael 64 Aug 10 20:14 868 -> pipe:[2250627] + lr-x------ 1 michael michael 64 Aug 10 20:14 869 -> pipe:[2250628] + lr-x------ 1 michael michael 64 Aug 10 20:13 87 -> pipe:[2241379] + l-wx------ 1 michael michael 64 Aug 10 20:14 870 -> pipe:[2251778] + lr-x------ 1 michael michael 64 Aug 10 20:14 871 -> pipe:[2251779] + l-wx------ 1 michael michael 64 Aug 10 20:14 872 -> pipe:[2251781] + lr-x------ 1 michael michael 64 Aug 10 20:14 873 -> pipe:[2251782] + l-wx------ 1 michael michael 64 Aug 10 20:14 874 -> pipe:[2250635] + lr-x------ 1 michael michael 64 Aug 10 20:14 875 -> pipe:[2250636] + l-wx------ 1 michael michael 64 Aug 10 20:14 876 -> pipe:[2251575] + lr-x------ 1 michael michael 64 Aug 10 20:14 877 -> pipe:[2251576] + l-wx------ 1 michael michael 64 Aug 10 20:14 878 -> pipe:[2251785] + lr-x------ 1 michael michael 64 Aug 10 20:14 879 -> pipe:[2251786] + l-wx------ 1 michael michael 64 Aug 10 20:13 88 -> pipe:[2247853] + l-wx------ 1 michael michael 64 Aug 10 20:14 880 -> pipe:[2249430] + lr-x------ 1 michael michael 64 Aug 10 20:14 881 -> pipe:[2249431] + l-wx------ 1 michael michael 64 Aug 10 20:14 882 -> pipe:[2251581] + lr-x------ 1 michael michael 64 Aug 10 20:14 883 -> pipe:[2251582] + l-wx------ 1 michael michael 64 Aug 10 20:14 884 -> pipe:[2250653] + lr-x------ 1 michael michael 64 Aug 10 20:14 885 -> pipe:[2250654] + lr-x------ 1 michael michael 64 Aug 10 20:14 886 -> pipe:[2251790] + l-wx------ 1 michael michael 64 Aug 10 20:14 887 -> pipe:[2250670] + l-wx------ 1 michael michael 64 Aug 10 20:14 888 -> pipe:[2250663] + lr-x------ 1 michael michael 64 Aug 10 20:14 889 -> pipe:[2250664] + lr-x------ 1 michael michael 64 Aug 10 20:13 89 -> pipe:[2247854] + l-wx------ 1 michael michael 64 Aug 10 20:14 890 -> pipe:[2250668] + lr-x------ 1 michael michael 64 Aug 10 20:14 891 -> pipe:[2250669] + lr-x------ 1 michael michael 64 Aug 10 20:14 892 -> pipe:[2250671] + l-wx------ 1 michael michael 64 Aug 10 20:14 894 -> pipe:[2251601] + lr-x------ 1 michael michael 64 Aug 10 20:14 895 -> pipe:[2251602] + lr-x------ 1 michael michael 64 Aug 10 20:13 9 -> pipe:[2241268] + lr-x------ 1 michael michael 64 Aug 10 20:13 90 -> pipe:[2242273] + l-wx------ 1 michael michael 64 Aug 10 20:13 91 -> pipe:[2245776] + etc + +"""]] diff --git a/doc/bugs/Resource_exhausted/comment_6_26c98fca45b029a527f9684873db4be5._comment b/doc/bugs/Resource_exhausted/comment_6_26c98fca45b029a527f9684873db4be5._comment new file mode 100644 index 0000000000..b95b6cfa28 --- /dev/null +++ b/doc/bugs/Resource_exhausted/comment_6_26c98fca45b029a527f9684873db4be5._comment @@ -0,0 +1,18 @@ +[[!comment format=mdwn + username="https://www.google.com/accounts/o8/id?id=AItOawln3ckqKx0x_xDZMYwa9Q1bn4I06oWjkog" + nickname="Michael" + subject="comment 6" + date="2013-08-11T03:20:37Z" + content=""" +with every + + git annex + copy blah/blah/blah (to testremote...) + git-annex: runInteractiveProcess: pipe: Too many open files + failed + +The number of open fd's by git-annex increases by 1. + +4.20130802 built with cabal on Ubuntu 13.04 + +"""]] diff --git a/doc/bugs/Resource_exhausted/comment_7_8bab413b472f900e04977db2bc3951b6._comment b/doc/bugs/Resource_exhausted/comment_7_8bab413b472f900e04977db2bc3951b6._comment new file mode 100644 index 0000000000..1b050aa333 --- /dev/null +++ b/doc/bugs/Resource_exhausted/comment_7_8bab413b472f900e04977db2bc3951b6._comment @@ -0,0 +1,8 @@ +[[!comment format=mdwn + username="https://www.google.com/accounts/o8/id?id=AItOawln3ckqKx0x_xDZMYwa9Q1bn4I06oWjkog" + nickname="Michael" + subject="comment 7" + date="2013-08-11T03:21:45Z" + content=""" +...increases in the failing case, that is. This doesn't happen all the time. +"""]] diff --git a/doc/bugs/Resource_exhausted/comment_8_e9bec0b80179b1229b6af0979a21c727._comment b/doc/bugs/Resource_exhausted/comment_8_e9bec0b80179b1229b6af0979a21c727._comment new file mode 100644 index 0000000000..2135211cd6 --- /dev/null +++ b/doc/bugs/Resource_exhausted/comment_8_e9bec0b80179b1229b6af0979a21c727._comment @@ -0,0 +1,9 @@ +[[!comment format=mdwn + username="https://www.google.com/accounts/o8/id?id=AItOawln3ckqKx0x_xDZMYwa9Q1bn4I06oWjkog" + nickname="Michael" + subject="comment 8" + date="2013-08-11T20:44:19Z" + content=""" +It also looks like the location log has got corrupted (files are actually present, but not recorded in the location log) somewhere along the lines as I was trying to get/drop to figure out what's going on. +Explicitly dropping files then getting files fixes the location log issue. +"""]] diff --git a/doc/bugs/Resource_exhausted/comment_9_419e24e0b91f569294ece28c42daa246._comment b/doc/bugs/Resource_exhausted/comment_9_419e24e0b91f569294ece28c42daa246._comment new file mode 100644 index 0000000000..0e06f5af92 --- /dev/null +++ b/doc/bugs/Resource_exhausted/comment_9_419e24e0b91f569294ece28c42daa246._comment @@ -0,0 +1,15 @@ +[[!comment format=mdwn + username="http://joeyh.name/" + ip="4.154.0.63" + subject="comment 9" + date="2013-08-24T19:10:21Z" + content=""" +@Michael how large a copy are you doing? And what kind of remote are you copying the files to? +It would be helpful if you could be more specific about something I could do to reproduce the problem. Without a test case, I am unlikely to fix the bug. With a test case, I'd be surprised if it took long to fix it. + +If you have a process running that is experiencing the problem, you can also narrow it down a *lot* by looking at what these leaking pipe file descriptors are pipes to. For example, if you have: + +lr-x------ 1 michael michael 64 Aug 10 20:14 895 -> pipe:[2251602] + +You can run `find /proc/ -ls 2251602` and find the process at other end of the pipe, and look its pid up in ps to see what command it is. +"""]] diff --git a/doc/bugs/Resource_temporarily_unavailable_when_running_enableremote.mdwn b/doc/bugs/Resource_temporarily_unavailable_when_running_enableremote.mdwn new file mode 100644 index 0000000000..071782d518 --- /dev/null +++ b/doc/bugs/Resource_temporarily_unavailable_when_running_enableremote.mdwn @@ -0,0 +1,23 @@ +### Please describe the problem. + +enableremote gives this weird error when trying to change settings on the s3 remote: + +
    +$ git annex enableremote s3
+enableremote s3
+git-annex: .git/annex/creds/SOME-LONG-HASH-I-AM-NOT-SURE-SHOULD-BE-PUBLIC: openFile: resource exhausted (Resource temporarily unavailable)
+failed
+git-annex: enableremote: 1 failed
+
    + +### What steps will reproduce the problem? + +Unclear. I tried to update to the latest neurodebian packages to make use of the new [[tips/public_Amazon_S3_remote]] but it seems something is going wrong here. + +The git-annex repository is on a "persistent amazon volume" (EBS, presumably?) so there may be something fishy with the filesystem there... + +### What version of git-annex are you using? On what operating system? + +5.20150610+gitg608172f-1~ndall+1 on Ubuntu precise, in amazon EC2. --[[anarcat]] + +A workaround is to run this on a different machine. I suspect some amazon foul play at hand... so this could be invalid. :( diff --git a/doc/bugs/Resource_temporarily_unavailable_when_running_enableremote/comment_1_14a5135c2e822129972feb570e453bc8._comment b/doc/bugs/Resource_temporarily_unavailable_when_running_enableremote/comment_1_14a5135c2e822129972feb570e453bc8._comment new file mode 100644 index 0000000000..6fce74e31f --- /dev/null +++ b/doc/bugs/Resource_temporarily_unavailable_when_running_enableremote/comment_1_14a5135c2e822129972feb570e453bc8._comment @@ -0,0 +1,10 @@ +[[!comment format=mdwn + username="joey" + subject="""comment 1""" + date="2015-07-02T15:45:13Z" + content=""" +So it's writing the creds file that's failing. The only thing unusual +about this file write is that it's done with a umask of 0077. + +Seeing what strace has to say about it would probably be useful.. +"""]] diff --git a/doc/bugs/S3_remote___8212___un-embedding_creds__63__.mdwn b/doc/bugs/S3_remote___8212___un-embedding_creds__63__.mdwn new file mode 100644 index 0000000000..d26cf3f8c2 --- /dev/null +++ b/doc/bugs/S3_remote___8212___un-embedding_creds__63__.mdwn @@ -0,0 +1,34 @@ +### Please describe the problem. + +After once embedding credentials into a S3 remote, there’s no way to un-embed them. + +### What steps will reproduce the problem? + +1. Add a S3 remote with `embedcreds=yes`. +1. Try to set `embedcreds=no`. +1. `git annex sync` here, and in some other clone. +1. Check `git annex info my-s3-remote` for the other clone. Credentials are still embedded. + +### What version of git-annex are you using? On what operating system? + +6.20170101 on NixOS 16.09 (stable). + +### Please provide any additional information below. + +[[!format sh """ +% g annex info test2 | grep '^creds' +creds: embedded in git repository (gpg encrypted) + +% g annex enableremote test2 embedcreds=no +enableremote test2 (encryption update) (to gpg keys: 134164FA6FAFE075) ok +(recording state in git...) + +% g annex info test2 | grep '^creds' +creds: embedded in git repository (gpg encrypted) + +% +"""]] + +### Have you had any luck using git-annex before? (Sometimes we get tired of reading bug reports all day and a lil' positive end note does wonders) + +Yes, it’s awesome! Thank you! diff --git a/doc/bugs/S3_remote___8212___un-embedding_creds__63__/comment_1_336e1c2a5c2a367cba0ad74896b3895b._comment b/doc/bugs/S3_remote___8212___un-embedding_creds__63__/comment_1_336e1c2a5c2a367cba0ad74896b3895b._comment new file mode 100644 index 0000000000..79a44d7c52 --- /dev/null +++ b/doc/bugs/S3_remote___8212___un-embedding_creds__63__/comment_1_336e1c2a5c2a367cba0ad74896b3895b._comment @@ -0,0 +1,19 @@ +[[!comment format=mdwn + username="joey" + subject="""comment 1""" + date="2017-02-20T18:38:16Z" + content=""" +When you used embedcreds=yes, it *committed* the creds to the git-annex +branch of the git repository. For embedcreds=no to do anything useful, +it would need to remove that data from the git repository history. + +Removing data from a git repository tends to involve rewriting commits and +forced pushes to all remotes, it's not a simple process and not ameanable +to automation. It will be much easier, and more secure, to go into S3 +and generate new credentials, and revoke access to the old ones. + +What `git annex enableremote` with `embedcreds=no` does do is prevent +any new creds from being embedded into the repository. Otherwise, +`git annex enableremote` will update the embedded creds +with whatever new ones are set in the environment when you run it. +"""]] diff --git a/doc/bugs/S3_remote___8212___un-embedding_creds__63__/comment_2_282aaaed5e95e8062460f94ac3a68dea._comment b/doc/bugs/S3_remote___8212___un-embedding_creds__63__/comment_2_282aaaed5e95e8062460f94ac3a68dea._comment new file mode 100644 index 0000000000..9c2e141f37 --- /dev/null +++ b/doc/bugs/S3_remote___8212___un-embedding_creds__63__/comment_2_282aaaed5e95e8062460f94ac3a68dea._comment @@ -0,0 +1,14 @@ +[[!comment format=mdwn + username="michalrus" + avatar="http://cdn.libravatar.org/avatar/83c0b6e7f9d20f09a892263c4903bbae" + subject="comment 2" + date="2017-02-25T18:47:35Z" + content=""" +Sure, that’s what I did at the very beginning (revoking the creds on AWS). + +But still, when I clone this repo, `git annex info S3` in this fresh clone, says that credentials are embedded. This is a bit misleading. + +Maybe updating S3 configuration with `embedcreds=no` could change that behavior? + +It wouldn’t matter from the security standpoint (which is solved by changing permissions directly on AWS), but would simply be beneficial for usability. =) +"""]] diff --git a/doc/bugs/S3_remote___8212___un-embedding_creds__63__/comment_3_07fb8558238285883d11d8805884bdae._comment b/doc/bugs/S3_remote___8212___un-embedding_creds__63__/comment_3_07fb8558238285883d11d8805884bdae._comment new file mode 100644 index 0000000000..fb97783f72 --- /dev/null +++ b/doc/bugs/S3_remote___8212___un-embedding_creds__63__/comment_3_07fb8558238285883d11d8805884bdae._comment @@ -0,0 +1,9 @@ +[[!comment format=mdwn + username="joey" + subject="""comment 3""" + date="2017-03-06T17:34:29Z" + content=""" +If `git annex info` had said that the creds were not embedded after +you tried `embedcreds=no`, then wouldn't you have belived it, and so not +revoked the creds on AWS? +"""]] diff --git a/doc/bugs/SMB__58___git_annex_clone_works__44___get_fails_on_transfer_lock.mdwn b/doc/bugs/SMB__58___git_annex_clone_works__44___get_fails_on_transfer_lock.mdwn new file mode 100644 index 0000000000..ef08680d4e --- /dev/null +++ b/doc/bugs/SMB__58___git_annex_clone_works__44___get_fails_on_transfer_lock.mdwn @@ -0,0 +1,249 @@ +### Please describe the problem. + +I have a Synology DS216+ NAS, which can export shares via SMB2, AFP or NFS. Since I have a lot of "media" data in git-annex, I'd like to be able to put a useful copy of that data (ie, original file names) onto the NAS, and have git-annex track that content as another "local" copy of the content that is available (rather than having to track it by hand). + +Because the NFS shares have UID mapping issues (AFAICT NFS v4 id mapping is not supported without Kerberos authentication, due to changes since about Linux 3.4; the Synology is based on a Linux 3.10 kernel), I have been trying to use SMBv2 shares, with symlinks enabled ("Allow symbolic links within shared folders"). I'm aware that `git-annex` on a network file system is less supported, but this is still a use case that would be quite useful to manage larger pools of files. + +After updating to the latest release (to [fix git annex init timeouts](https://git-annex.branchable.com/forum/git_annex_init_timeout/)), I am able to complete the `git clone ...`, `git annex init`, and `git sync` steps. (The SMB mount supports symlinks, but not hard links; hence pidlocks being needed.) + +But `git annex get` or `git annex copy` to transfer any of the content files fails with: + + get README (transfer already in progress, or unable to take transfer lock) + Unable to access these remotes: ashram-data + +whether initiated from the SMB mounted `git-annex` or a `git-annex` with the data on a local file system. No content files get transferred. + +Of note, on this "SMB share with symlinks enabled" I can manually create symlinks, but `git annex` still detects symlinks as not possible and switches to direct mode. However it leaves the existing symlinks from `git clone ...` in place. It is unclear to me whether this is part of the cause of the "transfer lock" issue, or whether the transfer locks are, eg, not using pidlocks and thus failing because flock() does not work. + +### What steps will reproduce the problem? + +* Mount a remote file system via SMB v2, with symlinks enabled (eg mounting from a Samba share should be sufficient, as that's what is running on the Synology NAS) + +* `git clone` an existing `git-annex` onto a directory on that SMB mounted filesystem + +* `git remote ....` configure one or more remotes with the file data + +* `git annex init "SMB share"` + +* `git annex sync` + +* `git annex get ....` of an existing file + +or alternatively after the init/sync, go to a source `git-annex` and do: + +* `git remote add smb ....` + +* `git annex sync` + +* `git annex copy --to=smb ...` of an existing file with content in that `git-annex` + +### What version of git-annex are you using? On what operating system? + +`git-annex` 6.20170320 (downloaded 2017-04-23) on Mac OS X 10.11 (El Capitan): + + ewen@ashram:~$ uname -a + Darwin ashram 15.6.0 Darwin Kernel Version 15.6.0: Fri Feb 17 10:21:18 PST 2017; root:xnu-3248.60.11.4.1~1/RELEASE_X86_64 x86_64 + ewen@ashram:~$ git annex version + git-annex version: 6.20170320-g41c5d9d + build flags: Assistant Webapp Pairing Testsuite S3(multipartupload)(storageclasses) WebDAV FsEvents ConcurrentOutput TorrentParser MagicMime Feeds Quvi + key/value backends: SHA256E SHA256 SHA512E SHA512 SHA224E SHA224 SHA384E SHA384 SHA3_256E SHA3_256 SHA3_512E SHA3_512 SHA3_224E SHA3_224 SHA3_384E SHA3_384 SKEIN256E SKEIN256 SKEIN512E SKEIN512 SHA1E SHA1 MD5E MD5 WORM URL + remote types: git gcrypt p2p S3 bup directory rsync web bittorrent webdav tahoe glacier ddar hook external + ewen@ashram:~$ + +### Please provide any additional information below. + +[[!format sh """ +# Example session +ewen@ashram:/nas01/annex/purchased$ git clone /bkup/purchased/mediabytes +Cloning into 'mediabytes'... +done. +ewen@ashram:/nas01/annex/purchased$ cd mediabytes/ +ewen@ashram:/nas01/annex/purchased/mediabytes$ git remote add ashram-data /data/purchased/mediabytes +ewen@ashram:/nas01/annex/purchased/mediabytes$ git remote add ashram /bkup/purchased/mediabytes +ewen@ashram:/nas01/annex/purchased/mediabytes$ git remote remove origin +ewen@ashram:/nas01/annex/purchased/mediabytes$ git annex init 'nas01 file server' +init nas01 file server + Detected a filesystem without POSIX fcntl lock support. + + Enabling annex.pidlock. + + Detected a filesystem without fifo support. + + Disabling ssh connection caching. + + Detected a crippled filesystem. + + Disabling core.symlinks. + + Enabling direct mode. +ok +(recording state in git...) +ewen@ashram:/nas01/annex/purchased/mediabytes$ git annex sync +commit ok +pull ashram +From /bkup/purchased/mediabytes + * [new branch] git-annex -> ashram/git-annex + * [new branch] master -> ashram/master + * [new branch] synced/git-annex -> ashram/synced/git-annex + * [new branch] synced/master -> ashram/synced/master +ok +pull ashram-data +From /data/purchased/mediabytes + * [new branch] git-annex -> ashram-data/git-annex + * [new branch] master -> ashram-data/master + * [new branch] synced/git-annex -> ashram-data/synced/git-annex + * [new branch] synced/master -> ashram-data/synced/master +ok +(merging ashram-data/git-annex into git-annex...) +(recording state in git...) +push ashram +Counting objects: 8, done. +Delta compression using up to 8 threads. +Compressing objects: 100% (6/6), done. +Writing objects: 100% (8/8), 806 bytes | 0 bytes/s, done. +Total 8 (delta 2), reused 1 (delta 0) +To /bkup/purchased/mediabytes + 02b7699..0edf575 git-annex -> synced/git-annex +ok +push ashram-data +Counting objects: 8, done. +Delta compression using up to 8 threads. +Compressing objects: 100% (6/6), done. +Writing objects: 100% (8/8), 806 bytes | 0 bytes/s, done. +Total 8 (delta 2), reused 1 (delta 0) +To /data/purchased/mediabytes + 02b7699..0edf575 git-annex -> synced/git-annex +ok +ewen@ashram:/nas01/annex/purchased/mediabytes$ git annex sync +commit ok +pull ashram +ok +pull ashram-data +ok +ewen@ashram:/nas01/annex/purchased/mediabytes$ git annex get . +get README (transfer already in progress, or unable to take transfer lock) + Unable to access these remotes: ashram-data + + Try making some of these repositories available: + 00420eb2-1c9a-47e8-abfa-675b9ffb0782 -- naos_wd_1 backup drive + 0ac1326e-9e03-4c1e-bd51-6cd664988caa -- Naos (colo) fileserver + 0e5fd428-ba5b-4817-8489-9780bc62e655 -- naos_wd_2 backup drive + 1144edcc-b18f-4cb1-a828-6c272d7788d6 -- naos_wd_3 backup drive + 6142cfd4-83ba-410f-bf54-fd4b79e49fdb -- ashram external data drive [ashram-data] + 77aeca6a-a4f8-4734-befc-2a0b50687c52 -- tv home fileserver + cc1c45f3-7c96-4103-b9c0-cce0f5f2cc36 -- bethel_data_drive + eb0ca2b6-e8c6-43b0-a805-de17a4d7eeae -- naos_wd_4 backup drive +failed +get alex-lindsay-1.m4a (transfer already in progress, or unable to take transfer lock) + Unable to access these remotes: ashram-data + + Try making some of these repositories available: + 00420eb2-1c9a-47e8-abfa-675b9ffb0782 -- naos_wd_1 backup drive + 0ac1326e-9e03-4c1e-bd51-6cd664988caa -- Naos (colo) fileserver + 0e5fd428-ba5b-4817-8489-9780bc62e655 -- naos_wd_2 backup drive + 1144edcc-b18f-4cb1-a828-6c272d7788d6 -- naos_wd_3 backup drive + 6142cfd4-83ba-410f-bf54-fd4b79e49fdb -- ashram external data drive [ashram-data] + 77aeca6a-a4f8-4734-befc-2a0b50687c52 -- tv home fileserver + cc1c45f3-7c96-4103-b9c0-cce0f5f2cc36 -- bethel_data_drive + eb0ca2b6-e8c6-43b0-a805-de17a4d7eeae -- naos_wd_4 backup drive +failed +[...] +[...] +failed +git-annex: get: 59 failed +ewen@ashram:/nas01/annex/purchased/mediabytes$ +ewen@ashram:/nas01/annex/purchased/mediabytes$ ls -l | head +total 118 +lrwx------ 1 ewen staff 182 23 Apr 18:53 README -> .git/annex/objects/3M/8w/SHA256E-s273--1eb5c07e12d99c99af8c163b5e005162820518020cc9922152852bd1422858ae/SHA256E-s273--1eb5c07e12d99c99af8c163b5e005162820518020cc9922152852bd1422858ae +lrwx------ 1 ewen staff 200 23 Apr 18:53 alex-lindsay-1.m4a -> .git/annex/objects/9M/xJ/SHA256E-s26905448--c8ecda8ca67c3ff8ab1061b34a2974be41eb4b44421e3c39d1dda63f4de6b910.m4a/SHA256E-s26905448--c8ecda8ca67c3ff8ab1061b34a2974be41eb4b44421e3c39d1dda63f4de6b910.m4a +lrwx------ 1 ewen staff 194 23 Apr 18:53 alexlindsay.txt -> .git/annex/objects/P7/7g/SHA256E-s17467--2e7ae57eff6db5a832681443eb3787d1d7f9e57a52e4fedb2b693136a3df19f0.txt/SHA256E-s17467--2e7ae57eff6db5a832681443eb3787d1d7f9e57a52e4fedb2b693136a3df19f0.txt +lrwx------ 1 ewen staff 198 23 Apr 18:53 bourne-1.mp3 -> .git/annex/objects/98/vg/SHA256E-s7560841--a24ce7acabcb0dd46b9a9df6df8aff50145d8b1d5253308a51573a7742652943.mp3/SHA256E-s7560841--a24ce7acabcb0dd46b9a9df6df8aff50145d8b1d5253308a51573a7742652943.mp3 +lrwx------ 1 ewen staff 192 23 Apr 18:53 bourne.txt -> .git/annex/objects/QK/2p/SHA256E-s1031--d45a0f0aba935126bf29d6265e9c252faae8b735abdc20edbec69eb3cc9edcd7.txt/SHA256E-s1031--d45a0f0aba935126bf29d6265e9c252faae8b735abdc20edbec69eb3cc9edcd7.txt +lrwx------ 1 ewen staff 200 23 Apr 18:53 chasejarvis-1.m4a -> .git/annex/objects/M1/8z/SHA256E-s80616681--d1b63892a9d68c51ddef9eb5c0a56ced795a449afe13d4884663ca6ecc76cade.m4a/SHA256E-s80616681--d1b63892a9d68c51ddef9eb5c0a56ced795a449afe13d4884663ca6ecc76cade.m4a +lrwx------ 1 ewen staff 194 23 Apr 18:53 chasejarvis.txt -> .git/annex/objects/35/jv/SHA256E-s50443--687b0cf63cc8f5724e3a694e1de6221ca4d1dd3449a6e6ad9b6f34501bfa30ce.txt/SHA256E-s50443--687b0cf63cc8f5724e3a694e1de6221ca4d1dd3449a6e6ad9b6f34501bfa30ce.txt +lrwx------ 1 ewen staff 184 23 Apr 18:53 checksums -> .git/annex/objects/3Q/vv/SHA256E-s1672--31b3cb7ccbcafc96d5061775dc31aa974bb4ce2f16984b2ab0104eb66674d81b/SHA256E-s1672--31b3cb7ccbcafc96d5061775dc31aa974bb4ce2f16984b2ab0104eb66674d81b +lrwx------ 1 ewen staff 200 23 Apr 18:53 chrismarquardt-1.m4a -> .git/annex/objects/k4/f3/SHA256E-s61861112--c14bacb5adfe3eca237547760183a986499808084dd1925a0202598981e4406b.m4a/SHA256E-s61861112--c14bacb5adfe3eca237547760183a986499808084dd1925a0202598981e4406b.m4a +ewen@ashram:/nas01/annex/purchased/mediabytes$ +ewen@ashram:/nas01/annex/purchased/mediabytes$ git annex info +repository mode: direct +trusted repositories: 0 +semitrusted repositories: 13 + 00000000-0000-0000-0000-000000000001 -- web + 00000000-0000-0000-0000-000000000002 -- bittorrent + 00420eb2-1c9a-47e8-abfa-675b9ffb0782 -- naos_wd_1 backup drive + 0ac1326e-9e03-4c1e-bd51-6cd664988caa -- Naos (colo) fileserver + 0ba93724-85a8-4ac7-855b-e656a6abaf09 -- ashram (laptop) internal drive [ashram] + 0e5fd428-ba5b-4817-8489-9780bc62e655 -- naos_wd_2 backup drive + 1144edcc-b18f-4cb1-a828-6c272d7788d6 -- naos_wd_3 backup drive + 4e5bfc02-1adc-44d2-a60a-885d9120ed5d -- nas01 file server [here] + 6142cfd4-83ba-410f-bf54-fd4b79e49fdb -- ashram external data drive [ashram-data] + 77aeca6a-a4f8-4734-befc-2a0b50687c52 -- tv home fileserver + bde87f8e-4740-411f-b068-3dbbd9db03d4 -- nas01 file server + cc1c45f3-7c96-4103-b9c0-cce0f5f2cc36 -- bethel_data_drive + eb0ca2b6-e8c6-43b0-a805-de17a4d7eeae -- naos_wd_4 backup drive +untrusted repositories: 0 +transfers in progress: none +available local disk space: 3.63 terabytes (+1 megabyte reserved) +local annex keys: 0 +local annex size: 0 bytes +annexed files in working tree: 59 +size of annexed files in working tree: 1.67 gigabytes +bloom filter size: 32 mebibytes (0% full) +backend usage: + SHA256E: 59 +ewen@ashram:/nas01/annex/purchased/mediabytes$ +"""]] + +For reference Synology DS216+ `/etc/samba/smb.conf`: + + ewen@nas01:/volume1$ cat /etc/samba/smb.conf + [global] + printcap name=cups + winbind enum groups=yes + include=/var/tmp/nginx/smb.netbios.aliases.conf + min protocol=SMB2 + security=user + local master=no + realm=* + passdb backend=smbpasswd + printing=cups + max protocol=SMB3 + winbind enum users=yes + load printers=yes + workgroup=WORKGROUP + ewen@nas01:/volume1$ + +and the relevant share: + + ewen@nas01:/volume1$ cat /etc/samba/smb.share.conf + [annex] + recycle bin admin only=no + ftp disable modify=no + ftp disable download=no + write list=nobody,nobody + browseable=yes + mediaindex=no + hide unreadable=no + win share=yes + enable recycle bin=no + invalid users=nobody,nobody + read list=nobody,nobody + ftp disable list=no + edit synoacl=yes + valid users=nobody,nobody + writeable=yes + guest ok=yes + path=/volume1/annex + skip smb perm=yes + comment="Git Annexes" + [music] + [...] + ewen@nas01:/volume1$ + +### Have you had any luck using git-annex before? (Sometimes we get tired of reading bug reports all day and a lil' positive end note does wonders) + +I'm an extremely regular user of git-annex on OS X and Linux, for several years, using it as a podcatcher and to manage most of my "large file" media. It's one of those "couldn't live without" tools. Thanks for writing it. + +(Touched, to ask for comments to be emailed to me) + +> [[fixed|done]] --[[Joey]] diff --git a/doc/bugs/SMB__58___git_annex_clone_works__44___get_fails_on_transfer_lock/comment_1_d165e4895e7043192888751218261282._comment b/doc/bugs/SMB__58___git_annex_clone_works__44___get_fails_on_transfer_lock/comment_1_d165e4895e7043192888751218261282._comment new file mode 100644 index 0000000000..7b92837e35 --- /dev/null +++ b/doc/bugs/SMB__58___git_annex_clone_works__44___get_fails_on_transfer_lock/comment_1_d165e4895e7043192888751218261282._comment @@ -0,0 +1,16 @@ +[[!comment format=mdwn + username="ewen" + avatar="http://cdn.libravatar.org/avatar/605b2981cb52b4af268455dee7a4f64e" + subject="git annex standalone on Synology NAS" + date="2017-04-23T07:52:19Z" + content=""" +After posting this bug, I found a [git annex tip on running a standalone build on the Synology NAS](http://git-annex.branchable.com/tips/Synology_NAS_and_git_annex/), so that may provide another workaround for me. Particularly since the Synology DS216+ is actually a x86-64 based system: + + ewen@nas01:/volume1$ uname -a + Linux nas01 3.10.102 #15047 SMP Thu Feb 23 02:23:28 CST 2017 x86_64 GNU/Linux synology_braswell_216+II + ewen@nas01:/volume1$ + +but it is probably still worth figuring out the cause of the transfer lock issue on SMB for other SMB use cases. + +Ewen +"""]] diff --git a/doc/bugs/SMB__58___git_annex_clone_works__44___get_fails_on_transfer_lock/comment_2_8831965ea7abf5ac6fe860d93133fbb1._comment b/doc/bugs/SMB__58___git_annex_clone_works__44___get_fails_on_transfer_lock/comment_2_8831965ea7abf5ac6fe860d93133fbb1._comment new file mode 100644 index 0000000000..c422dc903e --- /dev/null +++ b/doc/bugs/SMB__58___git_annex_clone_works__44___get_fails_on_transfer_lock/comment_2_8831965ea7abf5ac6fe860d93133fbb1._comment @@ -0,0 +1,42 @@ +[[!comment format=mdwn + username="joey" + subject="""comment 2""" + date="2017-05-11T16:37:35Z" + content=""" +`git-annex init` is probably detecting a problem with the filesystem +allowing writes to files whose permissions don't allow them to be written +to. Since indirect mode relies on being able to prevent writing to files, +it enables direct mode. The disabling of symlinks is a side effect of that. +You can probably get away with using indirect mode, but there is the risk +of annexed files getting modified and so the old version of files being +lost. + +The pid lock is used for transfer locks when that's enabled. + +So, the pid lock creation is failing for some reason. The two likely causes +would be: + +* The pid locking code not working well on OSX. This would not be too + surprising. + + But, I gave pid locking on OSX a quick test, by making a repository + in my OSX home directory, `git config annex.pidlock true` and doing the + same in a clone. Transfers between the repositories worked ok so pid + locking seems to at least basically work on OSX. + +* Something to do with the SMB filesystem breaking the pid locking code. + Perhaps something to do with permissions since as noted `git annex init` + seems to have identified a problem with permissions handling. + +Could you possibly get a `dtruss` dump of git-annex when this error +happens? That would narrow down the problem a lot. + +I could try to set up SMB etc, but I don't have root access to an OSX +machine, so am not confident I could replicate the problem. + +---- + +Also, it might overall be better to run git-annex on the Synology NAS. +Then you don't have to mess with networked filesystems. People have done +this successfully before. +"""]] diff --git a/doc/bugs/SMB__58___git_annex_clone_works__44___get_fails_on_transfer_lock/comment_3_24c6874757783e3c3838db35174870d1._comment b/doc/bugs/SMB__58___git_annex_clone_works__44___get_fails_on_transfer_lock/comment_3_24c6874757783e3c3838db35174870d1._comment new file mode 100644 index 0000000000..c549c19723 --- /dev/null +++ b/doc/bugs/SMB__58___git_annex_clone_works__44___get_fails_on_transfer_lock/comment_3_24c6874757783e3c3838db35174870d1._comment @@ -0,0 +1,86 @@ +[[!comment format=mdwn + username="ewen" + avatar="http://cdn.libravatar.org/avatar/605b2981cb52b4af268455dee7a4f64e" + subject="dtruss OS X 10.11 git annex on SMB mount" + date="2017-05-14T05:04:29Z" + content=""" +A tar file with: + + ewen@ashram:~/Desktop$ tar -tzf dtruss-git-annex-osx-smb-2017-05-14.tar.gz + dtruss-git-annex-osx-smb-2017-05-14/ + dtruss-git-annex-osx-smb-2017-05-14/00-ga-test-smb-osx-symlinks-2017-05-14.script + dtruss-git-annex-osx-smb-2017-05-14/01-ga-test-smb-osx-symlinks-dtruss-git-annex-init-2017-05-14.txt + dtruss-git-annex-osx-smb-2017-05-14/02-ga-test-smb-osx-symlinks-dtruss-git-annex-sync-2017-05-14.txt + dtruss-git-annex-osx-smb-2017-05-14/03-ga-test-smb-osx-symlinks-dtruss-git-annex-get-all-2017-05-14.txt + dtruss-git-annex-osx-smb-2017-05-14/04-ga-test-smb-osx-symlinks-dtruss-git-annex-sync-other-way-2017-05-14.txt + dtruss-git-annex-osx-smb-2017-05-14/05-ga-test-smb-osx-symlinks-dtruss-git-annex-copy-to-smb-2017-05-14.txt + dtruss-git-annex-osx-smb-2017-05-14/dtruss-ga + ewen@ashram:~/Desktop$ + +is available at [dtruss-git-annex-osx-smb-2017-05-14.tar.gz](http://ewen.mcneill.gen.nz/temp/dtruss-git-annex-osx-smb-2017-05-14.tar.gz) (at least until I next tidy up the `/temp/` directory :-) ). + +The script file is the output of \"`script ....`\" showing all the commands (including typos :-) ) run, except the `dtruss` ones themselves. The various `0n-ga-tst-smb-osx-symlinks-dtruss...` files contain the output of \"`sudo dtruss -af PID`\" where the `PID` was printed out by the script `dtruss-ga` that is included in the archive. `dtruss-ga` was used to enable running git-annex as non-root (to ensure that the SMB access wasn't altered by running as a different user), since `dtruss` itself can only run as root. The script is just: + + ewen@ashram:~/Desktop$ cat dtruss-git-annex-osx-smb-2017-05-14/dtruss-ga + #! /bin/sh + # Allow dtruss of non-root git annex + # + # Based on hints at: + # http://sec.cs.ucl.ac.uk/users/smurdoch/blog/2015/08/fixing-slow-start-mc-on-osx.html + #--------------------------------------------------------------------------- + + echo \"Run: sudo dtruss -af -p $$\" + echo \"Then press enter\" + read + exec git annex \"$@\" + ewen@ashram:~/Desktop$ + +so basically \"print PID, wait for go ahead, run git annex command\". + +From a quick glance through, it looks like: + + 7003/0x16c3126: 121576 14 11 write(0x12, \"PidLock {lockingPid = 7003, lockingHost = \\"ashram\\"}\0\", 0x33) = 51 0 + 7003/0x16c3126: 121775 3350 191 close(0x12) = 0 0 + 7003/0x16c3126: 121792 10 6 stat64(\"/Volumes/music/ga-test/.git/annex/locktmp16807282475249\0\", 0x200004940, 0x33) = 0 0 + 7003/0x16c3126: 122041 2341 235 link(\"/Volumes/music/ga-test/.git/annex/locktmp16807282475249\0\", \"/Volumes/music/ga-test/.git/annex/locktmp16807282475249.lnk\0\") = -1 Err#45 + 7003/0x16c3126: 122115 885 49 unlink(\"/Volumes/music/ga-test/.git/annex/locktmp16807282475249.lnk\0\", 0x200004D80, 0x33) = -1 Err#2 + +(in `03-...`, the first `get`) probably isn't helping -- that's trying to do a *hardlink* on a SMB file system, and appearing to give up when it fails... (`/Volumes/music` is the SMB mount I was using for testing). + +Also since I didn't realise until after the `script` file/`tar` file were made, there *is* an `annex/pidlock` file: + + ewen@ashram:~$ ls -l /Volumes/music/ga-test/.git/annex/pidlock + -rwx------ 1 ewen staff 51 14 May 16:25 /Volumes/music/ga-test/.git/annex/pidlock + ewen@ashram:~$ cat /Volumes/music/ga-test/.git/annex/pidlock + PidLock {lockingPid = 7143, lockingHost = \"ashram\"}ewen@ashram:~$ + ewen@ashram:~$ + +created within the SMB `git-annex`, which looks vaguely sensible. And FTR, there is *not* one in the HFS (local OS X) `git-annex`: + + ewen@ashram:~$ ls -l /var/tmp/ga-test/.git/annex/pidlock + ls: /var/tmp/ga-test/.git/annex/pidlock: No such file or directory + ewen@ashram:~$ + +so it definitely seems to *try* to do pidlocking on the SMB mount on transfers, but it looks like maybe it's also trying to do something *not* supported. + +Looking through the OS X `open(2)` manpage, it appears that: + + O_EXCL error if O_CREAT and the file exists + [...] + O_EXLOCK atomically obtain an exclusive lock + +might be plausible non-hardlink locking primitives that'd work better than nothing. The manpage also says: + + When opening a file, a lock with flock(2) semantics can be obtained by + setting O_SHLOCK for a shared lock, or O_EXLOCK for an exclusive lock. + If creating a file with O_CREAT, the request for the lock will never fail + (provided that the underlying filesystem supports locking). + +so maybe that's something to work with? (SMB as a protocol supports All Kinds Of Locking (tm), and hopefully the OS X SMB file system driver would also translate O_EXLOCK, etc, through to those...) + +Hope that's of some help. + +Ewen + +PS: Yes, I do plan to try to install the stand alone x86-64 git-annex on my Synology when I get a spare few hours to sort out all the details. It seems like it would be the most robust solution for my specific problem. But generalised SMB support could also be useful, even if it was \"user needs to be careful about running commands in parallel as there are races\". +"""]] diff --git a/doc/bugs/SMB__58___git_annex_clone_works__44___get_fails_on_transfer_lock/comment_4_33087a19874447a1cb07aa794c0f030b._comment b/doc/bugs/SMB__58___git_annex_clone_works__44___get_fails_on_transfer_lock/comment_4_33087a19874447a1cb07aa794c0f030b._comment new file mode 100644 index 0000000000..041fe926b4 --- /dev/null +++ b/doc/bugs/SMB__58___git_annex_clone_works__44___get_fails_on_transfer_lock/comment_4_33087a19874447a1cb07aa794c0f030b._comment @@ -0,0 +1,10 @@ +[[!comment format=mdwn + username="ewen" + avatar="http://cdn.libravatar.org/avatar/605b2981cb52b4af268455dee7a4f64e" + subject="Standalone git-annex on Synology DS216+ NAS" + date="2017-05-28T01:24:43Z" + content=""" +For the benefit of future readers, I did manage to [get the stand alone `git-annex` running on my Synology DS216+](http://ewen.mcneill.gen.nz/blog/entry/2017-05-28-git-annex-on-synology-ds216+/). With a little bit of sorting out details, including adding some symlinks in `/usr/bin`, it runs like `git-annex` on any other Linux-based system. At this time that is definitely much easier than trying to use a SMB share if you are able to enable `ssh` access -- but note that `ssh` access to the Synology NAS requires an administrator account, so may not be available to everyone. (Plus of course this work around is useful mainly to relatively open Linux-based NAS solutions.) + +Ewen +"""]] diff --git a/doc/bugs/SMB__58___git_annex_clone_works__44___get_fails_on_transfer_lock/comment_5_8b5d03f64127aa6b4073ff10796b9771._comment b/doc/bugs/SMB__58___git_annex_clone_works__44___get_fails_on_transfer_lock/comment_5_8b5d03f64127aa6b4073ff10796b9771._comment new file mode 100644 index 0000000000..64b1feb8b2 --- /dev/null +++ b/doc/bugs/SMB__58___git_annex_clone_works__44___get_fails_on_transfer_lock/comment_5_8b5d03f64127aa6b4073ff10796b9771._comment @@ -0,0 +1,59 @@ +[[!comment format=mdwn + username="joey" + subject="""comment 5""" + date="2017-06-06T17:54:42Z" + content=""" +Here's the relevant parts of the dtruss: + + 7003/0x16c3126: 121483 3371 127 open("/Volumes/music/ga-test/.git/annex/locktmp16807282475249\0", 0x20A06, 0x180) = 18 0 + ... + 7003/0x16c3126: 121792 10 6 stat64("/Volumes/music/ga-test/.git/annex/locktmp16807282475249\0", 0x200004940, 0x33) = 0 0 + 7003/0x16c3126: 122041 2341 235 link("/Volumes/music/ga-test/.git/annex/locktmp16807282475249\0", "/Volumes/music/ga-test/.git/annex/locktmp16807282475249.lnk\0") = -1 Err#45 + 7003/0x16c3126: 122115 885 49 unlink("/Volumes/music/ga-test/.git/annex/locktmp16807282475249.lnk\0", 0x200004D80, 0x33) = -1 Err#2 + 7003/0x16c3126: 122126 5 1 sigreturn(0x7FFF56C9E390, 0x1E, 0x33) = 0 Err#-2 + 7003/0x16c3126: 122247 1050 90 open("/Volumes/music/ga-test/.git/annex/pidlock\0", 0xA01, 0x124) = 18 0 + 7003/0x16c3126: 122256 4 1 fcntl(0x12, 0x3, 0x0) = 1 0 + 7003/0x16c3126: 122260 5 2 fstat64(0x12, 0x20001A120, 0x0) = 0 0 + 7003/0x16c3126: 122273 4 1 ioctl(0x12, 0x4004667A, 0x7FFF56C9E3FC) = -1 Err#25 + 7003/0x16c3126: 122274 2 0 ioctl(0x12, 0x40487413, 0x7FFF56C9E400) = -1 Err#25 + 7003/0x16c3126: 122314 677 32 open("/Volumes/music/ga-test/.git/annex/locktmp16807282475249\0", 0x20004, 0x1B6) = 19 0 + 7003/0x16c3126: 122319 5 2 fstat64(0x13, 0x20001A310, 0x1B6) = 0 0 + 7003/0x16c3126: 122326 2 0 ioctl(0x13, 0x4004667A, 0x7FFF56C9E3FC) = -1 Err#25 + 7003/0x16c3126: 122327 2 0 ioctl(0x13, 0x40487413, 0x7FFF56C9E400) = -1 Err#25 + 7003/0x16c3126: 122381 1455 46 read(0x13, "PidLock {lockingPid = 7003, lockingHost = \"ashram\"}\0", 0x1FA0) = 51 0 + 7003/0x16c3126: 122392 5 2 read(0x13, "\0", 0x1FA0) = 0 0 + 7003/0x16c3126: 122427 1339 27 close(0x13) = 0 0 + 7003/0x16c3126: 122441 10 4 select(0x13, 0x7FFF56C9E340, 0x7FFF56C9E3C0, 0x0, 0x7FFF56C9E330) = 1 0 + 7003/0x16c3126: 122469 661 23 write(0x12, "PidLock {lockingPid = 7003, lockingHost = \"ashram\"}\0", 0x33) = 51 0 + 7003/0x16c3126: 122577 6028 99 close(0x12) = 0 0 + +Avove shows hard linking failed, and it fell back to using open to create +the pidlock file. That seemed to finish successfully. + + 7003/0x16c3126: 122583 4 0 sigreturn(0x7FFF56C9E3B0, 0x1E, 0x33) = 0 Err#-2 + +Unsure what this is? + + 7003/0x16c3126: 122697 2126 101 unlink("/Volumes/music/ga-test/.git/annex/locktmp16807282475249\0", 0x1E, 0x33) = 0 0 + +Here linkToLock must has succeeded, because it deletes the tmp file that was passed to that function. +So, it looks like tryLock succeeded. + + 7003/0x16c3126: 122746 1423 28 stat64(".git/annex/pidlock\0", 0x20001A510, 0x33) = 0 0 + 7003/0x16c3126: 122775 10 5 select(0x2, 0x7FFF56C9E340, 0x7FFF56C9E3C0, 0x0, 0x7FFF56C9E330) = 1 0 + 7003/0x16c3126: 122786 10 8 write(0x1, "(transfer already in progress, or unable to take transfer lock) \0", 0x40) = 64 0 + +I think this is a call to checkSaneLock, which calls getFileStatus, +and since the pid lock was created earlier, that succeeds. +runTransfer does call checkSaneLock, and if it fails +we get the "transfer already in progress, or unable to take transfer lock" +message. + +So, why did checkSaneLock fail? + +Well, the device and inode of the lock file +are compared with the device and inode of the temp file +that was passed to linkToLock. However, since the hard link didn't +happen, the device and inode will be different! +And that's the bug. +"""]] diff --git a/doc/bugs/SSL_repos_does_not_show_up_in_Assistant.mdwn b/doc/bugs/SSL_repos_does_not_show_up_in_Assistant.mdwn new file mode 100644 index 0000000000..6f21cb5342 --- /dev/null +++ b/doc/bugs/SSL_repos_does_not_show_up_in_Assistant.mdwn @@ -0,0 +1,27 @@ +### Please describe the problem. + +I have recently switched to self hosted git. Gitbucket over SSL. When I swithced origin from rsync ssh to ssl url assistant stopped showing origin in the repositories list it also does not sync to it when a file changes. (It does sync to it every 5 6 hours.). Running sync on the command line does sync to the repo. Authentication is done using netrc file. Replacing the ssl based url to ssh based url, repo shows up in the list and assistant syncs to it. + + +### What steps will reproduce the problem? + +Replacing the ssh url with ssl url. + + +### What version of git-annex are you using? On what operating system? + +5.20150823-geb17375 + +### Please provide any additional information below. + +[[!format sh """ +# If you can, paste a complete transcript of the problem occurring here. +# If the problem is with the git-annex assistant, paste in .git/annex/daemon.log + + +# End of transcript or log. +"""]] + +### Have you had any luck using git-annex before? (Sometimes we get tired of reading bug reports all day and a lil' positive end note does wonders) + + diff --git a/doc/bugs/SSL_repos_does_not_show_up_in_Assistant/comment_1_5a8e599d6e22e7c1960bd1715666657b._comment b/doc/bugs/SSL_repos_does_not_show_up_in_Assistant/comment_1_5a8e599d6e22e7c1960bd1715666657b._comment new file mode 100644 index 0000000000..d394ca039d --- /dev/null +++ b/doc/bugs/SSL_repos_does_not_show_up_in_Assistant/comment_1_5a8e599d6e22e7c1960bd1715666657b._comment @@ -0,0 +1,17 @@ +[[!comment format=mdwn + username="joey" + subject="""comment 1""" + date="2015-10-12T18:52:42Z" + content=""" +You seem to be talking about using https to access a git repository. + +git-annex is not generally able to use https as a transport -- ie, it +can maybe download files from a git repository over https, if +the repository is set up just right. But it can't upload file to that +repository, nor can it promptly notice when changes are synced to that +repository from elsewhere. + +So, there's not much that the assistant can do with such a repository. + +You'll be better off using ssh for your git-annex repositories. +"""]] diff --git a/doc/bugs/SSL_repos_does_not_show_up_in_Assistant/comment_2_2ad5098c5e2ac9b8651cb464adb974c1._comment b/doc/bugs/SSL_repos_does_not_show_up_in_Assistant/comment_2_2ad5098c5e2ac9b8651cb464adb974c1._comment new file mode 100644 index 0000000000..0759fb7771 --- /dev/null +++ b/doc/bugs/SSL_repos_does_not_show_up_in_Assistant/comment_2_2ad5098c5e2ac9b8651cb464adb974c1._comment @@ -0,0 +1,8 @@ +[[!comment format=mdwn + username="https://me.yahoo.com/a/FHnTlSBo1eCGJRwueeKeB6.RCaPbGMPr5jxx8A--#ce0d8" + nickname="Hamza" + subject="comment 2" + date="2015-10-13T16:57:24Z" + content=""" +You are right about https access, server does not support git-annex so it is set to meta data only. when using ssh changes are sync (git changes only) but with https changes are not synced (meta data). It is not even listed in repos. When using ssh it is listed in repos as meta data only. I do not store files in the repo just meta data. +"""]] diff --git a/doc/bugs/Selfsigned_certificates_with_jabber_fail_miserably.mdwn b/doc/bugs/Selfsigned_certificates_with_jabber_fail_miserably.mdwn new file mode 100644 index 0000000000..a6c258332a --- /dev/null +++ b/doc/bugs/Selfsigned_certificates_with_jabber_fail_miserably.mdwn @@ -0,0 +1,26 @@ +### Please describe the problem. +Entering a jabber address which's server got a selfsigned certificate, the process just fails, without asking for acceptance for that certificate. This is quite a showstopper. +(for example: jabber.ccc.de) + + +### What steps will reproduce the problem? +Try with an account from e.g. jabber.ccc.de + + +### What version of git-annex are you using? On what operating system? +Arch Linux, aur/git-annex-standalone 4.20130709-1 + +### Please provide any additional information below. +There is no logoutput to add... I'm sorry. + +[[!format sh """ +# If you can, paste a complete transcript of the problem occurring here. +# If the problem is with the git-annex assistant, paste in .git/annex/daemon.log + + +# End of transcript or log. +"""]] + +[[!meta title="XMPP does not work with jabber.ccc.de"]] + +[[!tag moreinfo]] diff --git a/doc/bugs/Selfsigned_certificates_with_jabber_fail_miserably/comment_1_13d27ba41d9ef78c8db534b6bc26314e._comment b/doc/bugs/Selfsigned_certificates_with_jabber_fail_miserably/comment_1_13d27ba41d9ef78c8db534b6bc26314e._comment new file mode 100644 index 0000000000..a591b51e1b --- /dev/null +++ b/doc/bugs/Selfsigned_certificates_with_jabber_fail_miserably/comment_1_13d27ba41d9ef78c8db534b6bc26314e._comment @@ -0,0 +1,12 @@ +[[!comment format=mdwn + username="http://joeyh.name/" + ip="4.154.4.90" + subject="comment 1" + date="2013-07-16T17:33:23Z" + content=""" +Hmm, actually the XMPP library it's using does not fail on an invalid or self-signed cert. This is something that SSL using libraries sadly often get wrong, and yeah, I see I contacted the library author a while ago, and a \"sessionIsSecure\" has been added to it, which can be used to check if the cert was valid. + +This leaves open the question of what's happening with you on the CCC server. Did you get an error message? + + +"""]] diff --git a/doc/bugs/Selfsigned_certificates_with_jabber_fail_miserably/comment_2_018eed99e71680be9e7c0844020419bb._comment b/doc/bugs/Selfsigned_certificates_with_jabber_fail_miserably/comment_2_018eed99e71680be9e7c0844020419bb._comment new file mode 100644 index 0000000000..9caee8a735 --- /dev/null +++ b/doc/bugs/Selfsigned_certificates_with_jabber_fail_miserably/comment_2_018eed99e71680be9e7c0844020419bb._comment @@ -0,0 +1,8 @@ +[[!comment format=mdwn + username="http://joeyh.name/" + ip="4.154.4.90" + subject="comment 2" + date="2013-07-16T17:42:42Z" + content=""" +Actually, sessionIsSecure only tells me if it's using SSL at all; still waiting on the XMPP library to add cert checking.. +"""]] diff --git a/doc/bugs/Selfsigned_certificates_with_jabber_fail_miserably/comment_3_1e7578dd1321f399b12197056495b0b6._comment b/doc/bugs/Selfsigned_certificates_with_jabber_fail_miserably/comment_3_1e7578dd1321f399b12197056495b0b6._comment new file mode 100644 index 0000000000..62d723c7b3 --- /dev/null +++ b/doc/bugs/Selfsigned_certificates_with_jabber_fail_miserably/comment_3_1e7578dd1321f399b12197056495b0b6._comment @@ -0,0 +1,8 @@ +[[!comment format=mdwn + username="https://launchpad.net/~psycojoker" + nickname="psycojoker" + subject="comment 3" + date="2013-11-25T09:54:33Z" + content=""" +Just to confirm that I'm also affected. Maybe you should add the possibility in the error message next to \"is the password correct?\". +"""]] diff --git a/doc/bugs/Selfsigned_certificates_with_jabber_fail_miserably/comment_4_a1948b7cd6ea990c8f1be5e483c835fa._comment b/doc/bugs/Selfsigned_certificates_with_jabber_fail_miserably/comment_4_a1948b7cd6ea990c8f1be5e483c835fa._comment new file mode 100644 index 0000000000..6eec4ae9a1 --- /dev/null +++ b/doc/bugs/Selfsigned_certificates_with_jabber_fail_miserably/comment_4_a1948b7cd6ea990c8f1be5e483c835fa._comment @@ -0,0 +1,14 @@ +[[!comment format=mdwn + username="http://xlogon.net/yminus" + nickname="http://xlogon.net/yminus" + subject="comment 4" + date="2014-05-08T21:12:49Z" + content=""" +When will this issue be fixed? Currently my Jabber account is on jabber.ccc.de. + +This is the error message: + + Unable to connect to the Jabber server. Maybe you entered the wrong password? (Error message: host jabberd.jabber.ccc.de.:5222 failed: AuthenticationFailure (Element {elementName = Name {nameLocalName = \"failure\", nameNamespace = Just \"urn:ietf:params:xml:ns:xmpp-sasl\", namePrefix = Nothing}, elementAttributes = [], elementNodes = [NodeElement (Element {elementName = Name {nameLocalName = \"bad-protocol\", nameNamespace = Just \"urn:ietf:params:xml:ns:xmpp-sasl\", namePrefix = Nothing}, elementAttributes = [], elementNodes = []})]}); host jabberd.jabber.ccc.de.:80 failed: AuthenticationFailure (Element {elementName = Name {nameLocalName = \"failure\", nameNamespace = Just \"urn:ietf:params:xml:ns:xmpp-sasl\", namePrefix = Nothing}, elementAttributes = [], elementNodes = [NodeElement (Element {elementName = Name {nameLocalName = \"bad-protocol\", nameNamespace = Just \"urn:ietf:params:xml:ns:xmpp-sasl\", namePrefix = Nothing}, elementAttributes = [], elementNodes = []})]})) + +Can you advise me another trustworthy non-profit and free (free as in free beer\" as well as \"free as in free speech\") jabber server? +"""]] diff --git a/doc/bugs/Selfsigned_certificates_with_jabber_fail_miserably/comment_5_293b333134c97dc666a825cc7a8b2b62._comment b/doc/bugs/Selfsigned_certificates_with_jabber_fail_miserably/comment_5_293b333134c97dc666a825cc7a8b2b62._comment new file mode 100644 index 0000000000..37c86cc80b --- /dev/null +++ b/doc/bugs/Selfsigned_certificates_with_jabber_fail_miserably/comment_5_293b333134c97dc666a825cc7a8b2b62._comment @@ -0,0 +1,12 @@ +[[!comment format=mdwn + username="http://joeyh.name/" + ip="209.250.56.36" + subject="comment 5" + date="2014-05-24T20:03:46Z" + content=""" +AFAIKS, this is not a failure from a self-signed certificate. git-annex contacted the jabber server, accepted its certificate, and tried to log in. The jabber server rejected the login. + +This kind of problem has in the past turned out to involve incompatabilities in SASL authentication using some newish authentication schemes. Often it's been a bug in the XMPP server that's tickled by the haksell TLS library. See for example + +So, what XMPP server is used by CCC? +"""]] diff --git a/doc/bugs/Set_some_reasonable_requirements_lower-bound_for_linux_standalone_builds__58___no_longer_usable_on_CentOS_6.5.mdwn b/doc/bugs/Set_some_reasonable_requirements_lower-bound_for_linux_standalone_builds__58___no_longer_usable_on_CentOS_6.5.mdwn new file mode 100644 index 0000000000..f2ff387026 --- /dev/null +++ b/doc/bugs/Set_some_reasonable_requirements_lower-bound_for_linux_standalone_builds__58___no_longer_usable_on_CentOS_6.5.mdwn @@ -0,0 +1,8 @@ +### Please describe the problem. + +standalone builds leap forward a bit too fast in terms of their demand on modern kernel. Up until a month ago, standalone build worked at least on CentOS 6.5 with 2.6.32-431.11.2.el6.x86_64 (although [already didn't on ancient but still in use RHEL5 with 2.6.18](https://github.com/datalad/datalad/issues/176#issuecomment-114612365)). Current build from 20150916 doesn't work on CentOS 6.5 any longer. No matter how much I like people to use more recent releases of their distributions, it is infeasible to demand people to do that. It would be great if standalone builds were carried out on some elder kernel/libc environment to make git-annex standalone builds usable there. + +> [[done]]; the ancient build provides this. It's now build with stack, so +> it gets most features enabled. (Except xmpp, notably.) This does mean +> it will only get library security fixes once git-annex's stack.yaml +> is updated to require a new version of the affected library. --[[Joey]] diff --git a/doc/bugs/Set_some_reasonable_requirements_lower-bound_for_linux_standalone_builds__58___no_longer_usable_on_CentOS_6.5/comment_1_625268287fe55e6f2254a6f31c637164._comment b/doc/bugs/Set_some_reasonable_requirements_lower-bound_for_linux_standalone_builds__58___no_longer_usable_on_CentOS_6.5/comment_1_625268287fe55e6f2254a6f31c637164._comment new file mode 100644 index 0000000000..b59365274a --- /dev/null +++ b/doc/bugs/Set_some_reasonable_requirements_lower-bound_for_linux_standalone_builds__58___no_longer_usable_on_CentOS_6.5/comment_1_625268287fe55e6f2254a6f31c637164._comment @@ -0,0 +1,31 @@ +[[!comment format=mdwn + username="joey" + subject="""comment 1""" + date="2015-09-23T16:00:38Z" + content=""" +This is entirely down to the libc used on the build system and included in +the tarball. + +Currently the builds use unstable or testing because this lets me get new +library-driven features like sha3 into the build (nearly) ASAP. + +It would be possible to switch to using Debian stable. But then I either +lose new library-driven features in the builds, or I have to start +installing haskell libraries from source on the build machines, which would +be a lot of additional work compared to letting Debian maintain hundreds of +haskell library packages for me. + +Then too, while I am pretty much committed to keeping git-annex building on +Debian stable (which is actually a lot of work, up to several thousand +lines of ifdefs, and prevents me from using newer ghc features which would +make my code happier), Debian stable does itself have a habit of updating to a +new libc release every couple of years. So no matter what, as long as libc +keeps requiring slightly less ancient kernels for whatever reasons it does, +everything isn't going to be supported forever. + +(OTOH, I'd expect that some enterprise level ancient distros might start to +include enterprise level ancient builds of git-annex themselves some time soon? +git-annex version 3, which is broadly compatible with current versions was +released a full 4 years ago already. Linux 2.6.32 is only, er, 3 years +older than that.) +"""]] diff --git a/doc/bugs/Set_some_reasonable_requirements_lower-bound_for_linux_standalone_builds__58___no_longer_usable_on_CentOS_6.5/comment_2_3ed7589687fe251066677ad5a01bbbb0._comment b/doc/bugs/Set_some_reasonable_requirements_lower-bound_for_linux_standalone_builds__58___no_longer_usable_on_CentOS_6.5/comment_2_3ed7589687fe251066677ad5a01bbbb0._comment new file mode 100644 index 0000000000..01f83a095b --- /dev/null +++ b/doc/bugs/Set_some_reasonable_requirements_lower-bound_for_linux_standalone_builds__58___no_longer_usable_on_CentOS_6.5/comment_2_3ed7589687fe251066677ad5a01bbbb0._comment @@ -0,0 +1,11 @@ +[[!comment format=mdwn + username="joey" + subject="""comment 2""" + date="2015-09-23T17:04:38Z" + content=""" +Actually, git-annex doesn't currently build on debian stable because it +needs a newer version of optparse-applicative. The new version builds just +fine on stable, so could easily be backported. I could probably get +git-annex building with the old version of optparse-applicative, with +sufficient ifdef pain, but a backport would be a nicer solution. +"""]] diff --git a/doc/bugs/Set_some_reasonable_requirements_lower-bound_for_linux_standalone_builds__58___no_longer_usable_on_CentOS_6.5/comment_2_557d428d2aa6bf8e5d435e7f5350f5c6._comment b/doc/bugs/Set_some_reasonable_requirements_lower-bound_for_linux_standalone_builds__58___no_longer_usable_on_CentOS_6.5/comment_2_557d428d2aa6bf8e5d435e7f5350f5c6._comment new file mode 100644 index 0000000000..179c70f58d --- /dev/null +++ b/doc/bugs/Set_some_reasonable_requirements_lower-bound_for_linux_standalone_builds__58___no_longer_usable_on_CentOS_6.5/comment_2_557d428d2aa6bf8e5d435e7f5350f5c6._comment @@ -0,0 +1,7 @@ +[[!comment format=mdwn + username="https://me.yahoo.com/a/EbvxpTI_xP9Aod7Mg4cwGhgjrCrdM5s-#7c0f4" + subject="comment 2" + date="2015-09-23T16:36:08Z" + content=""" +Shouldn't it be possible reasonably easily to create env which would just use stock haskell-platform? that one should provide all recent libraries you would need? and then just do that on e.g. debian stable +"""]] diff --git a/doc/bugs/Set_some_reasonable_requirements_lower-bound_for_linux_standalone_builds__58___no_longer_usable_on_CentOS_6.5/comment_4_641389c7c52fa4f624ae74b5d32a6b1c._comment b/doc/bugs/Set_some_reasonable_requirements_lower-bound_for_linux_standalone_builds__58___no_longer_usable_on_CentOS_6.5/comment_4_641389c7c52fa4f624ae74b5d32a6b1c._comment new file mode 100644 index 0000000000..4dfe0f4608 --- /dev/null +++ b/doc/bugs/Set_some_reasonable_requirements_lower-bound_for_linux_standalone_builds__58___no_longer_usable_on_CentOS_6.5/comment_4_641389c7c52fa4f624ae74b5d32a6b1c._comment @@ -0,0 +1,18 @@ +[[!comment format=mdwn + username="joey" + subject="""comment 4""" + date="2015-09-23T17:14:14Z" + content=""" +It might be possible to use stackage to get a build of all libraries from +the base haskell platform that are not too far out of date and don't break +too often. There is currently no way to upgrade such a build to get newer +version of libraries except for throwing the whole build tree away and +building all libraries again. So, it would be a lot of CPU on an ongoing basis, +or a source of unfixed security bugs. + +And, I think it would not be entirely non-fragile. It's not exactly unheard +of for a new version of a haskell library to accidentally break +compatibility with the old version of ghc which is generally unused in the +haskell community outside of debian stable. Or, to need a newer version of +some C library headers than is in stable, or ... +"""]] diff --git a/doc/bugs/Set_some_reasonable_requirements_lower-bound_for_linux_standalone_builds__58___no_longer_usable_on_CentOS_6.5/comment_5_8d5e7b7aa25a82e14611e0b852512adf._comment b/doc/bugs/Set_some_reasonable_requirements_lower-bound_for_linux_standalone_builds__58___no_longer_usable_on_CentOS_6.5/comment_5_8d5e7b7aa25a82e14611e0b852512adf._comment new file mode 100644 index 0000000000..d3662569cd --- /dev/null +++ b/doc/bugs/Set_some_reasonable_requirements_lower-bound_for_linux_standalone_builds__58___no_longer_usable_on_CentOS_6.5/comment_5_8d5e7b7aa25a82e14611e0b852512adf._comment @@ -0,0 +1,8 @@ +[[!comment format=mdwn + username="joey" + subject="""comment 5""" + date="2015-09-23T17:35:51Z" + content=""" +I've added an "ancient" autobuild, which is run on Debian stable plus +minimal backports (currently of optparse-applicative only). +"""]] diff --git a/doc/bugs/Set_some_reasonable_requirements_lower-bound_for_linux_standalone_builds__58___no_longer_usable_on_CentOS_6.5/comment_6_391d0ba1cbe59de8e0cca2c29085d9fd._comment b/doc/bugs/Set_some_reasonable_requirements_lower-bound_for_linux_standalone_builds__58___no_longer_usable_on_CentOS_6.5/comment_6_391d0ba1cbe59de8e0cca2c29085d9fd._comment new file mode 100644 index 0000000000..70f53ea04c --- /dev/null +++ b/doc/bugs/Set_some_reasonable_requirements_lower-bound_for_linux_standalone_builds__58___no_longer_usable_on_CentOS_6.5/comment_6_391d0ba1cbe59de8e0cca2c29085d9fd._comment @@ -0,0 +1,7 @@ +[[!comment format=mdwn + username="https://me.yahoo.com/a/EbvxpTI_xP9Aod7Mg4cwGhgjrCrdM5s-#7c0f4" + subject="comment 6" + date="2015-09-23T17:57:43Z" + content=""" +so far we are not bottlenecked in CPU power, so if needed could setup some CPU-intensive rebuilding of such environments (e.g. in a docker) so we could may be even have some \"collection\" of previous environments laying around +"""]] diff --git a/doc/bugs/Set_some_reasonable_requirements_lower-bound_for_linux_standalone_builds__58___no_longer_usable_on_CentOS_6.5/comment_7_a72bed153e91e32ebd8944e5ee17c186._comment b/doc/bugs/Set_some_reasonable_requirements_lower-bound_for_linux_standalone_builds__58___no_longer_usable_on_CentOS_6.5/comment_7_a72bed153e91e32ebd8944e5ee17c186._comment new file mode 100644 index 0000000000..86043c49f3 --- /dev/null +++ b/doc/bugs/Set_some_reasonable_requirements_lower-bound_for_linux_standalone_builds__58___no_longer_usable_on_CentOS_6.5/comment_7_a72bed153e91e32ebd8944e5ee17c186._comment @@ -0,0 +1,15 @@ +[[!comment format=mdwn + username="joey" + subject="""comment 7""" + date="2015-09-30T17:09:49Z" + content=""" +I got stack working to build git-annex over the weekend, and it's now the +recommended way to build from source, since it avoids library churn issues +and automates everything. Upgrades can also be made to happen by increasing +the lts version in the stack.yaml file. + +So using that and the haskell platform would be a way to get an even more +ancient distribution autobuilding. (But, I had to leave out xmpp and dbus +support to make the stack recipe work everywhere, so I don't want to make +the regular autobuilds built using stack.) +"""]] diff --git a/doc/bugs/Set_some_reasonable_requirements_lower-bound_for_linux_standalone_builds__58___no_longer_usable_on_CentOS_6.5/comment_8_be1d950835ee47774063e1335dd043e3._comment b/doc/bugs/Set_some_reasonable_requirements_lower-bound_for_linux_standalone_builds__58___no_longer_usable_on_CentOS_6.5/comment_8_be1d950835ee47774063e1335dd043e3._comment new file mode 100644 index 0000000000..218126f252 --- /dev/null +++ b/doc/bugs/Set_some_reasonable_requirements_lower-bound_for_linux_standalone_builds__58___no_longer_usable_on_CentOS_6.5/comment_8_be1d950835ee47774063e1335dd043e3._comment @@ -0,0 +1,7 @@ +[[!comment format=mdwn + username="https://me.yahoo.com/a/EbvxpTI_xP9Aod7Mg4cwGhgjrCrdM5s-#7c0f4" + subject="standalone builds" + date="2015-10-08T16:35:10Z" + content=""" +Thanks Joey. So the standalone builds you currently have will still be full featured builds, but where could we get those ones with limited functionality? may be they could be automated and made available alongside with corresponding description (for older systems, without assistant functionalities such as X and Y)? +"""]] diff --git a/doc/bugs/Set_some_reasonable_requirements_lower-bound_for_linux_standalone_builds__58___no_longer_usable_on_CentOS_6.5/comment_9_805c0e3d4f3d0f3d4c1d4e6e9a5cf032._comment b/doc/bugs/Set_some_reasonable_requirements_lower-bound_for_linux_standalone_builds__58___no_longer_usable_on_CentOS_6.5/comment_9_805c0e3d4f3d0f3d4c1d4e6e9a5cf032._comment new file mode 100644 index 0000000000..c1d9a876ec --- /dev/null +++ b/doc/bugs/Set_some_reasonable_requirements_lower-bound_for_linux_standalone_builds__58___no_longer_usable_on_CentOS_6.5/comment_9_805c0e3d4f3d0f3d4c1d4e6e9a5cf032._comment @@ -0,0 +1,7 @@ +[[!comment format=mdwn + username="https://me.yahoo.com/a/EbvxpTI_xP9Aod7Mg4cwGhgjrCrdM5s-#7c0f4" + subject="ah -- the 32bit build I guess" + date="2015-10-08T17:50:50Z" + content=""" +ah -- now I saw the https://downloads.kitenet.net/git-annex/linux/current/git-annex-standalone-i386-ancient.tar.gz which is I guess as ancient as it could get ;) unfortunately \"was built without its test suite; not testing\" so can't test but let's see how it performs ;) +"""]] diff --git a/doc/bugs/Should_try_again_when_network_fails___40__esp._DNS__41__.mdwn b/doc/bugs/Should_try_again_when_network_fails___40__esp._DNS__41__.mdwn new file mode 100644 index 0000000000..988f986e56 --- /dev/null +++ b/doc/bugs/Should_try_again_when_network_fails___40__esp._DNS__41__.mdwn @@ -0,0 +1,50 @@ +### Please describe the problem. + +If you have a flaky connection big uploads and downloads will fail. git-annex should try again few times. + +This is an example of failed download. + +[[!format sh """ +$ git annex get --not --in here +get File1.bin (from s3...) (gpg) +You need a passphrase to unlock the secret key for +user: "Gioele" +4096-bit RSA key, .... + +gpg: gpg-agent is not available in this session + + ErrorMisc ": hGetBuf: resource vanished (Connection reset by peer)" + + Unable to access these remotes: s3 + + Try making some of these repositories available: + 331fa184-799d-4511-1725-ef2a17ace8b4 -- s3 + c2a0cfa0-8871-9721-9b81-5649281fabdc -- other +failed +get File2.bin (from s3...) + + Unable to access these remotes: s3 + + Try making some of these repositories available: + 331fa184-799d-4511-1725-ef2a17ace8b4 -- s3 + c2a0cfa0-8871-9721-9b81-5649281fabdc -- other +failed +git-annex: get: 2 failed +"""]] + +This is especially annoying when the DNS is out of order for a few seconds every now and then. In such cases, git-annex will complain, skip very fast to the next file, and repeat this process until it runs out of files. In the end it will have uploaded or downloaded very few files. + +Please not that it may not possible to write a simple shell loop to try again as the are GPG passwords to be entered. + +Git-annex should try again to upload or download a file in case something goes wrong. + +### What version of git-annex are you using? On what operating system? + + git-annex version: 4.20130709.1 + build flags: Assistant Webapp Pairing Testsuite S3 WebDAV Inotify DBus XMPP + local repository version: unknown + default repository version: 3 + supported repository versions: 3 4 + upgrade supported from repository versions: 0 1 2 + +Ubuntu 12.04.2 LTS diff --git a/doc/bugs/Should_try_again_when_network_fails___40__esp._DNS__41__/comment_1_dd792bd98a48554a65150c06401ed3e5._comment b/doc/bugs/Should_try_again_when_network_fails___40__esp._DNS__41__/comment_1_dd792bd98a48554a65150c06401ed3e5._comment new file mode 100644 index 0000000000..620f5e82e4 --- /dev/null +++ b/doc/bugs/Should_try_again_when_network_fails___40__esp._DNS__41__/comment_1_dd792bd98a48554a65150c06401ed3e5._comment @@ -0,0 +1,12 @@ +[[!comment format=mdwn + username="http://joeyh.name/" + ip="4.154.4.90" + subject="comment 1" + date="2013-07-17T18:54:47Z" + content=""" +Trying again one time does not seem like it would help, given the example you show. Trying multiple times by default would, I think, be annoying in lots of use cases where one just wants to get whatever is available, and having it get stuck retrying to download a file from a remote that is offline would not be desired when it could move on and get another file from a remote that is online. + +I'm willing to consider some kind of option to control how much it retries on error. But I'm not 100% sold on it being better than a simple loop. At least in most cases, using a gpg agent and a loop would work. I suppose the case it would not work as well is if enough time has elapsed for the gpg agent to re-lock the key. + +One approach that might work well is to add a --retry-failures-at-end option. It turns out that all failed downloads are already logged (the assistant uses this to automatically retry them), and so it would be easy to add. And rather than retrying immediately after a failure, when transferring multiple files, this puts some space in between, in which the problem may correct itself. +"""]] diff --git a/doc/bugs/SmartGit__39__s_Cherry-Pick_function_throws_error_with_git-annex.mdwn b/doc/bugs/SmartGit__39__s_Cherry-Pick_function_throws_error_with_git-annex.mdwn new file mode 100644 index 0000000000..9bad58b804 --- /dev/null +++ b/doc/bugs/SmartGit__39__s_Cherry-Pick_function_throws_error_with_git-annex.mdwn @@ -0,0 +1,43 @@ +### Please describe the problem. + +In SmartGit, doing a cherry-pick invokes the following 2 commands. + +``git cherry-pick --no-commit `` +``git commit "--author=''" "--date=''" --file=/var/folders/bw/jh4pq43918gfd_r7qxzkkt_40000gq/T/smartgit3129597356340000806tmp/commit7268815197345344355.tmp`` + +The error is _git 'annex' is not a git command_. + + +### What steps will reproduce the problem? + +``git-annex init `` + +Do some commits in 2 branches. +Do a cherry-pick in SmartGit. + + +### What version of git-annex are you using? On what operating system? + +PLEASE provide instructions here on how to get the git-annex version. +(It is ``git-annex version``. And why is ``git-annex | less`` blanking out when I press any key?) + +git-annex: 5.20151019 + +OS: Mac OS El Capitan. + + +### Please provide any additional information below. + +[[!format sh """ +# If you can, paste a complete transcript of the problem occurring here. +# If the problem is with the git-annex assistant, paste in .git/annex/daemon.log + + +# End of transcript or log. +"""]] + +### Have you had any luck using git-annex before? (Sometimes we get tired of reading bug reports all day and a lil' positive end note does wonders) + +Yes. Lots. Am an expert at it now. + +This can either be a problem with SmartGit, or a problem with the hacks that git-annex puts into a standard git repo. diff --git a/doc/bugs/SmartGit__39__s_Cherry-Pick_function_throws_error_with_git-annex/comment_1_6fd91f615e8d5212674a9748d1b4135b._comment b/doc/bugs/SmartGit__39__s_Cherry-Pick_function_throws_error_with_git-annex/comment_1_6fd91f615e8d5212674a9748d1b4135b._comment new file mode 100644 index 0000000000..541eefc41d --- /dev/null +++ b/doc/bugs/SmartGit__39__s_Cherry-Pick_function_throws_error_with_git-annex/comment_1_6fd91f615e8d5212674a9748d1b4135b._comment @@ -0,0 +1,12 @@ +[[!comment format=mdwn + username="joey" + subject="""comment 1""" + date="2015-12-19T17:51:55Z" + content=""" +`git-annex init` sets up .git/hooks/pre-commit to run `git annex pre-commit`. +It looks like when git commit is run, the git-annex program is not in PATH. +This probably has something to do with the way you installed git-annex. + +(Why is git-annex | less blanking out? Because git-annex outputs usage +messages to stderr, and less pages stdout. Use git-annex 2>&1 |less) +"""]] diff --git a/doc/bugs/SmartGit__39__s_Cherry-Pick_function_throws_error_with_git-annex/comment_2_3dbaf39065c2f0941f8f7092c27fd4cb._comment b/doc/bugs/SmartGit__39__s_Cherry-Pick_function_throws_error_with_git-annex/comment_2_3dbaf39065c2f0941f8f7092c27fd4cb._comment new file mode 100644 index 0000000000..7f401acd6a --- /dev/null +++ b/doc/bugs/SmartGit__39__s_Cherry-Pick_function_throws_error_with_git-annex/comment_2_3dbaf39065c2f0941f8f7092c27fd4cb._comment @@ -0,0 +1,16 @@ +[[!comment format=mdwn + username="jhannwong@c9c7a67b5632a4bbc0c959cfeb3d340e02f28565" + nickname="jhannwong" + subject="A problem with SmartGit then?" + date="2015-12-21T03:21:58Z" + content=""" +> git-annex init sets up .git/hooks/pre-commit to run git annex pre-commit. It looks like when git commit is run, the git-annex program is not in PATH. This probably has something to do with the way you installed git-annex. + +``which git-annex`` shows ``/usr/local/bin/git-annex``. I think I installed via Homebrew. + +Seems to be a problem with SmartGit then. The pre-commit hooks work just fine in Terminal sessions. + +> (Why is git-annex | less blanking out? Because git-annex outputs usage messages to stderr, and less pages stdout. Use git-annex 2>&1 |less) + +Ah. Oops. +"""]] diff --git a/doc/bugs/SmartGit__39__s_Cherry-Pick_function_throws_error_with_git-annex/comment_3_996139be157e23e17ece9f131b17240a._comment b/doc/bugs/SmartGit__39__s_Cherry-Pick_function_throws_error_with_git-annex/comment_3_996139be157e23e17ece9f131b17240a._comment new file mode 100644 index 0000000000..3af1b89bcf --- /dev/null +++ b/doc/bugs/SmartGit__39__s_Cherry-Pick_function_throws_error_with_git-annex/comment_3_996139be157e23e17ece9f131b17240a._comment @@ -0,0 +1,12 @@ +[[!comment format=mdwn + username="joey" + subject="""comment 3""" + date="2016-01-15T20:04:19Z" + content=""" +If SmartGit is started from the OSX GUI, and +/usr/local/bin is added to your PATH in eg .bashrc, there's a good chance +that the PATH won't include it. + +If this were a Linux system, I'd suggest setting the PATH in `.xsessionrc`. +I don't know what the equivilant is for OSX. +"""]] diff --git a/doc/bugs/Strange_case_of_data_loss__44___possibly_linked_to_git-annex_with_encrypted_rsync_remote.mdwn b/doc/bugs/Strange_case_of_data_loss__44___possibly_linked_to_git-annex_with_encrypted_rsync_remote.mdwn new file mode 100644 index 0000000000..be0060c582 --- /dev/null +++ b/doc/bugs/Strange_case_of_data_loss__44___possibly_linked_to_git-annex_with_encrypted_rsync_remote.mdwn @@ -0,0 +1,55 @@ +This is not really a proper bug report, but I thought I should post this here +in case someone can find any sane, non-supernatural reason for a strange case +of data loss I have experienced with git-annex. + +Some time ago I cloned a bunch of git-annex repos from an external drive (let's +call it disk1) to a new computer (computer3). On one of my repos git-annex +marked a bunch of files corrupt and moved them to .git/annex/bad. Oops, I +thought, I must have a failing disk. Luckily I had offsite backups -- no less +than two other external hard disks (disk2-3), each having a full copy of the +repo in question. However, **both of these** had the same, corrupt files. The +files have the correct size, but are filled with zeroes. Other files in the +repo are fine, and so are other repos. + +I have been trying to wrap my head around this but I can't think of any reason +how this could occur. However the files have gotten corrupted in the first +place, the corruption should have been picked up when copying the content to +the external drives disk2 and disk3, right? I have to rule out NSA/MIB/aliens +from messing with me because these files are not that valuable or sensitive. + +The files in question were added to git-annex back in 2012, so the trail is +cold on this one. Naturally, I have no idea on how to reproduce this, nor can I +reliably say that git-annex is to blame. I can gather some hints though. The +files were all added on the same commit in 2012, but not all files from that +commit are corrupted. The corrupted files have consecutive file names. The +files were never modified since (except for the corruption), and the content +*may* have been copied via an encrypted rsync transfer repository. I have +always used git-annex on Arch Linux and in indirect more. The files used the +SHA-1 backend. + +All these files have a similar tracking log that looks something like this +(uuids replaced with symbolic names): + + 1356690700.542152s 1 computer1 <- first added + 1356691074.253815s 1 disk1 <- copied to disk1 + 1356719321.145126s 1 rsync <- copied to rsync repo + 1358070999.435676s 1 rsync <- copied to rsync repo (again?) + 1362166895.310332s 1 disk2 <- copied to disk2 + 1362906850.555869s 1 computer2 (dead) <- copied to another computer + 1364926664.362195s 0 computer1 <- dropped from computer1 as enough copies in disks + 1374412057.409496s 0 computer2 (dead) <- dropped from computer2, now dead + 1445691595.764108s 1 disk3 <- copied to disk3 + 1445770764.165792s 0 rsync <- dropped from rsync repo to save space + 1482077052.217353646s 0 disk1 <- first noticed as corrupted on disk1 + 1482741278.318274404s 0 disk3 <- WTF, also corrupted on disk3 + 1482926246.268440532s 0 disk2 <- double-WTF, also corrupted on disk2 + +The only thing that strikes odd to me is the double entry with the rsync +remote. The non-corrupted files from the same commit do not seem to have such a +double entry. + +So my main question is, has there ever been a bug in git-annex that could have +caused this behavior? Or is there any other realistic explanation for this? In +case this is an existing bug, is there any other evidence I can gather? +Needless to say, the lesson here is to run `git annex fsck` regularly even if +you have offsite backups... diff --git a/doc/bugs/Strange_case_of_data_loss__44___possibly_linked_to_git-annex_with_encrypted_rsync_remote/comment_1_4038261a4d49694374c37ff029d3540f._comment b/doc/bugs/Strange_case_of_data_loss__44___possibly_linked_to_git-annex_with_encrypted_rsync_remote/comment_1_4038261a4d49694374c37ff029d3540f._comment new file mode 100644 index 0000000000..7c448d6d7b --- /dev/null +++ b/doc/bugs/Strange_case_of_data_loss__44___possibly_linked_to_git-annex_with_encrypted_rsync_remote/comment_1_4038261a4d49694374c37ff029d3540f._comment @@ -0,0 +1,20 @@ +[[!comment format=mdwn + username="user4" + avatar="http://cdn.libravatar.org/avatar/cd0825fd95c541c0a48c7138b59c240a" + subject="remote.log" + date="2017-06-11T19:53:27Z" + content=""" +Since the most likely culprit (after aliens) is the encrypted rsync remote, here is the remote.log. I've replaced +sensitive information with variables in double curly braces + + {{uuid}} = cipher={{cipher}} cipherkeys={{keyid}} {{url}}= name={{name}} rsyncurl={{url}} type=rsync timestamp=1345706291.523459s + {{uuid}} = cipher={{cipher}} cipherkeys={{keyid}} {{url}}= name={{name}} rsyncurl={{url}} type=rsync timestamp=1482178826.391400069s + {{uuid}} = cipher={{cipher}} cipherkeys={{keyid}} {{url}}= name={{name}} rsyncurl={{url}} type=rsync timestamp=1482742412.058733547s + {{uuid}} = cipher={{cipher}} cipherkeys={{keyid}} {{url}}= name={{name}} rsyncurl={{url}} type=rsync timestamp=1482742686.603751605s + {{uuid}} cipher={{cipher}} cipherkeys={{keyid}} name={{name}} rsyncurl={{url}} type=rsync timestamp=1343199142.018003s + {{uuid}} cipher={{cipher}} cipherkeys={{keyid}} name={{name}} rsyncurl={{url}} type=rsync timestamp=1343199310.53916s + {{uuid}} cipher={{cipher}} cipherkeys={{keyid}} name={{name}} rsyncurl={{url}} type=rsync timestamp=1343486259.34504s + +Here {{uuid}} is the uuid of the encrypted rsync remote. It's odd that there are several lines for the same remote, and not all in the same format. +Could this be a PEBKAC involving a misconfigured rsync remote? +"""]] diff --git a/doc/bugs/Stress_test.mdwn b/doc/bugs/Stress_test.mdwn new file mode 100644 index 0000000000..3bc701bbda --- /dev/null +++ b/doc/bugs/Stress_test.mdwn @@ -0,0 +1,45 @@ +What steps will reproduce the problem? + +mkdir annex_stress; cd annex_stress, +then execute the following script: + + #! /bin/sh + + # creating a directory, in which we dump all the files. + mkdir probes; cd probes + + for i in `seq -w 1 25769`; do + mkdir probe$i + echo "This is an important file, which saved also in backup ('back') directory too.\n Content changes: $i" > probe$i/probe$i.txt + echo "This is just an identical content file. Saved in each subdir." > probe$i/defaults.txt + echo "This is a variable ($i) content file, which is not backed up in 'back' directory." > probe$i/probe-nb$i.txt + mkdir probe$i/back + cp probe$i/probe$i.txt probe$i/back/probe$i.txt + done + + +It creates about 25000 directory and 3 files in each, two of them are identical. + +What is the expected output? What do you see instead? + +I expect git annex could import the directory within 12 hours. +Yet, it just crashes the gui (starting webapp, uses the cpu 100% and it does not finish after 28hours.) + + +What version of git-annex are you using? On what operating system? + +version 2013.04.17 + +Please provide any additional information below. + +I do hope git-annex can be fixed to handle large number of files. +This stress test models well enough my own directory structure, +relatively high number of files relatively low disk space usage +(my own directory structure: 750MB, this test creates 605MB). + + +Best, + Laszlo + +[[!meta title="assistant Stress test"]] +[[!tag /design/assistant]] diff --git a/doc/bugs/Stress_test/comment_10_1694e990eab6592159309c231c6dcc16._comment b/doc/bugs/Stress_test/comment_10_1694e990eab6592159309c231c6dcc16._comment new file mode 100644 index 0000000000..ec50b58c5a --- /dev/null +++ b/doc/bugs/Stress_test/comment_10_1694e990eab6592159309c231c6dcc16._comment @@ -0,0 +1,12 @@ +[[!comment format=mdwn + username="http://joeyh.name/" + nickname="joey" + subject="comment 10" + date="2013-05-06T16:54:36Z" + content=""" +My estimate was indeed slightly optimistic. While I did not run the whole import, it did run slower for the later batches of files. As far as I can see, that slowdown is just because git gets slower as it has more files. So nothing I can do about it. git-annex is now scaling well itself, though. + +Re checksumming on startup: There was a bug that caused the assistant to re-checksum all direct mode files on startup. This bug was fixed in version 4.20130417. If you're using that version and still see it re-checksumming files, please file a new bug report about it, as this is not intended behavior. + +You seem to be saying that the assistant is failing to add some files, and then when stopped and restarted it finds and adds them. I don't quite know how that would happen. If you can provide a test case that I can use to reproduce that behavior, I will try to debug it. +"""]] diff --git a/doc/bugs/Stress_test/comment_11_ab4cb6eefd279e6c1f229e089f703581._comment b/doc/bugs/Stress_test/comment_11_ab4cb6eefd279e6c1f229e089f703581._comment new file mode 100644 index 0000000000..05c409a514 --- /dev/null +++ b/doc/bugs/Stress_test/comment_11_ab4cb6eefd279e6c1f229e089f703581._comment @@ -0,0 +1,25 @@ +[[!comment format=mdwn + username="https://www.google.com/accounts/o8/id?id=AItOawm5iosFbL2By7UFeViqkc6v-hoAtqILeDA" + nickname="Laszlo" + subject="comment 11" + date="2013-05-11T05:36:48Z" + content=""" +rechecksuming: it seem like it is indeed fixed in the newest (2013.05.01) version downloaded from here: +http://downloads.kitenet.net/git-annex/linux/ + +I tried to add the big stress test dir as a secondary repository into git-annex (along with my real data dir), but +seems like some library is not matching on my system, so some curl is complaining: + + curl: /lib/tls/i686/cmov/libc.so.6: version `GLIBC_2.12' not found (required by /home/user/Desktop/down/git-annex.linux//usr/lib/i386-linux-gnu/libldap_r-2.4.so.2) + +I'm on ubuntu 10.04. + +And the log file is starting to fill up, so maybe once a problem occur, it should only write into the log file once. + +I will redone this stress test next week, without combining with any repository. +Thank you very much for your response, I do appreciate you are bothering/dealing with my complains!:) + +Best, + Laszlo + +"""]] diff --git a/doc/bugs/Stress_test/comment_1_c4c764488ac082f5c48d3a6b4b5fba42._comment b/doc/bugs/Stress_test/comment_1_c4c764488ac082f5c48d3a6b4b5fba42._comment new file mode 100644 index 0000000000..42750808b7 --- /dev/null +++ b/doc/bugs/Stress_test/comment_1_c4c764488ac082f5c48d3a6b4b5fba42._comment @@ -0,0 +1,17 @@ +[[!comment format=mdwn + username="http://joeyh.name/" + nickname="joey" + subject="comment 1" + date="2013-04-23T20:00:31Z" + content=""" +Is this related or unrelated to the bug you filed at [[Resource_exhausted]]? + +I tried this test, and noticed that it was taking the assistant rather a long time to get to the 10 thousand file threshhold where it makes a batch commit. A small change to a better data structure for its queue reduced that time from probably 10 minutes to 2.5. + +I was unable to reproduce any problem with the webapp. Please provide lots of details to back up \"it just crashes the GUI\". + +The main problem with this directory tree is that it has more directories than inotify can watch, in the default configuration. +So after it adds the first 8192 directories, it begins failing to watch any more, and printing a message about you needing to increase the inotify limits for each additional directory. I don't think that 51 thousand directories is a particularly realistic amount for any real-world usage of git-annex. (It will also break file manager, dropbox, etc, which all use inotify in the same way.) + +The other main time sink is that git-annex needs to run `git hash-object` once per file to stage its symlink. That is a lot of processes to run, and perhaps it could be sped up by using `git fast-import`. +"""]] diff --git a/doc/bugs/Stress_test/comment_2_42125bba09a0ea9821cda7183e458100._comment b/doc/bugs/Stress_test/comment_2_42125bba09a0ea9821cda7183e458100._comment new file mode 100644 index 0000000000..26ffe2d62d --- /dev/null +++ b/doc/bugs/Stress_test/comment_2_42125bba09a0ea9821cda7183e458100._comment @@ -0,0 +1,47 @@ +[[!comment format=mdwn + username="https://www.google.com/accounts/o8/id?id=AItOawm5iosFbL2By7UFeViqkc6v-hoAtqILeDA" + nickname="Laszlo" + subject="comment 2" + date="2013-04-24T06:30:16Z" + content=""" +Hi, + +First of all thank you for your time looking into my bug. I try to research more from my side. + +The 'Resource exhausted' bugreport +(which lost its title, and could not click on it to add this testcase as a comment) +was tested on real data, my own working directory (a copy of it). +This bugreport is tested on the output of this small shell script. + +None of them succeeded to import, and I quickly assumed it is the exact same. + +So I will test again, raising the ulimit to 81920, and report. + + The main problem with this directory tree is that it has more directories than inotify can watch, in the default configuration + +I would be perfectly fine if I could configure git-annex to sync those directory only once a month or once a week +(ie. check for update once a week). So no need to watch it real time, those are my archived work files. + + I don't think that 51 thousand directories is a particularly realistic amount for any real-world usage of git-annex. + +Well, it is not 25000 dir in a single a folder, but rather something like this: + + work_done/2009/workname/back9/back8/back7/back6/back5 + +Where each 'backX' contains a whole backup the work until it. +So the directory structure is a bit more deep, and no 25000 subdirectory in a single dir. +But the overall numbers are right. + +If I could somehow mark this **work_done** dir to not sync real time (or work_done/2008,work_done/2009,work_done/2010,work_done/2011,work_done/2012 subdir in them), +then my whole issue would vanish. + +I only want to use git-annex to have a backup of this directory. +In case of laptop theft, or misfunction I could have a backup. +I dont need live sync anywhere, I have directories which I know I will not touch for months. + +Best, + Laszlo + + + +"""]] diff --git a/doc/bugs/Stress_test/comment_3_8240e61106b494d3600ad91f16eb5b1c._comment b/doc/bugs/Stress_test/comment_3_8240e61106b494d3600ad91f16eb5b1c._comment new file mode 100644 index 0000000000..8469f489fd --- /dev/null +++ b/doc/bugs/Stress_test/comment_3_8240e61106b494d3600ad91f16eb5b1c._comment @@ -0,0 +1,20 @@ +[[!comment format=mdwn + username="https://www.google.com/accounts/o8/id?id=AItOawm5iosFbL2By7UFeViqkc6v-hoAtqILeDA" + nickname="Laszlo" + subject="comment 3" + date="2013-04-24T09:10:20Z" + content=""" + (It will also break file manager, dropbox, etc, which all use inotify in the same way.) + +I beg to differ: with dropbox I handle my scrapbook(1) folder, +which means 130 thousand files for over 2 years now without problem between three computers. + + ~/Dropbox/scrapbook$ ls -R -1 |wc -l + 130263 + +Don't get me wrong. I'm not complaining, I only give you a completely unrelated usecase, +which requires also high number of files handling. And in that case the 81 thousand ulimit would not help either. + +(1): https://addons.mozilla.org/hu/firefox/addon/scrapbook/ + +"""]] diff --git a/doc/bugs/Stress_test/comment_4_c38d84e0dcc834931804c44bce7f7b7a._comment b/doc/bugs/Stress_test/comment_4_c38d84e0dcc834931804c44bce7f7b7a._comment new file mode 100644 index 0000000000..c7cdd4bfd9 --- /dev/null +++ b/doc/bugs/Stress_test/comment_4_c38d84e0dcc834931804c44bce7f7b7a._comment @@ -0,0 +1,11 @@ +[[!comment format=mdwn + username="http://joeyh.name/" + nickname="joey" + subject="comment 4" + date="2013-04-24T15:05:33Z" + content=""" +You're confusing number of files (inotify doesn't care) with number of directories (inotify does care). + +Dropbox is on record about being limited in the number of directories it can watch without adjusting the inotify limit. + +"""]] diff --git a/doc/bugs/Stress_test/comment_5_60ce20ee255451c4ea809ba475561adb._comment b/doc/bugs/Stress_test/comment_5_60ce20ee255451c4ea809ba475561adb._comment new file mode 100644 index 0000000000..820f04ba99 --- /dev/null +++ b/doc/bugs/Stress_test/comment_5_60ce20ee255451c4ea809ba475561adb._comment @@ -0,0 +1,15 @@ +[[!comment format=mdwn + username="http://joeyh.name/" + nickname="joey" + subject="comment 5" + date="2013-04-24T15:30:04Z" + content=""" +I found a bug in the webapp thanks to this stress test. When inotify goes over limit, it displays a message about how to fix it.. +But it displays that message over and over for each file. The result is a constantly updating very large web page. + +Unless you tell me differently, I'm going to assume that's what the GUI crash you referred to was, since it can make a web browser very slow. + +I've fixed this problem. Now when it goes over limit, the webapp will just display this: + +[[/assistant/inotify_max_limit_alert.png]] +"""]] diff --git a/doc/bugs/Stress_test/comment_6_1371562e201393986cd41597f6f288cb._comment b/doc/bugs/Stress_test/comment_6_1371562e201393986cd41597f6f288cb._comment new file mode 100644 index 0000000000..26f14ef092 --- /dev/null +++ b/doc/bugs/Stress_test/comment_6_1371562e201393986cd41597f6f288cb._comment @@ -0,0 +1,14 @@ +[[!comment format=mdwn + username="http://joeyh.name/" + nickname="joey" + subject="comment 6" + date="2013-04-24T17:26:39Z" + content=""" +I put in a further change to reduce the number of alerts shown in the webapp when bulk adding files. This probably quadrupled the speed or more, even when the webapp was not running, as updating an alert every time a file was added was a lot of unnecessary work. + +After these changes, it adds the first 10 thousand files in 35 minutes, on my five year old netbook. It should scale linear +(aside from git's own scalability issues with a lot of files, which I don't think are very bad under 1 million files), +so adding all 100 thousand files should take 6 hours or so. + +I'm interested to see what results you get, compared with before.. +"""]] diff --git a/doc/bugs/Stress_test/comment_7_a14be7699da224a8f6c9b34f1b911219._comment b/doc/bugs/Stress_test/comment_7_a14be7699da224a8f6c9b34f1b911219._comment new file mode 100644 index 0000000000..3d02a27184 --- /dev/null +++ b/doc/bugs/Stress_test/comment_7_a14be7699da224a8f6c9b34f1b911219._comment @@ -0,0 +1,8 @@ +[[!comment format=mdwn + username="http://joeyh.name/" + nickname="joey" + subject="comment 7" + date="2013-04-24T21:02:55Z" + content=""" +A few more changes got the rate down to 21 minutes per 10 thousand files. Estimate 3.5 hours for all. +"""]] diff --git a/doc/bugs/Stress_test/comment_8_a01995bdca7ade7dde9842b53fbc4e0c._comment b/doc/bugs/Stress_test/comment_8_a01995bdca7ade7dde9842b53fbc4e0c._comment new file mode 100644 index 0000000000..d12b7ebfea --- /dev/null +++ b/doc/bugs/Stress_test/comment_8_a01995bdca7ade7dde9842b53fbc4e0c._comment @@ -0,0 +1,57 @@ +[[!comment format=mdwn + username="https://www.google.com/accounts/o8/id?id=AItOawm5iosFbL2By7UFeViqkc6v-hoAtqILeDA" + nickname="Laszlo" + subject="Definite improvement" + date="2013-05-03T06:27:12Z" + content=""" +Hi, + +I have just tried it out again with the latest (20130501) version. + +It is really nice to see you have been working on it, and it have improved tremendously! +The logging issue solved, and logrotates even, and it finished importing without crashing! + +Remaining polishing things: + +a) +The import time is not as good (as you write), it slowes itself down. +It is true the first 10000 files import in about an hour, but it finishes with everything +in 9 hours 20 minutes. +(on a normal laptop, the last 5000 file portion took more then 2 hours) + +b) +Every startup means rechecksuming everything, so it means the second start took also around 8-12 hours. +(I don't know exactly because it finished somewhere during the night, but it was longer then 8 hours) +I don't think rechecksuming is necessary at all, if the filename, size and date have not modified, +then why rechecksuming (sha) it? + + +c) +It is leaking. +At the second startup, it reported it successfully added: + Added 2375 files 5 files probe25366.txt + +I have not touched the directory. ls confirms leaking: + + After first start (importing): + annex_many/.git$ ls -lR |wc -l + 770199 + + After second startup: + annex_many/.git$ ls -lR |wc -l + 788351 + +d) Without ulimit raise, it does not work at all. +I think it could be solved by not watching each and every directory all the time. +Every users will likely have a working directory and some which he don't intend to touch/modify at all. +Some usecases: photo archiving, video archiving, finished work archiving, etc + +All the above results with the stress test script. +I would love to have a confirmation by a thirdparty. + +Overall I'm impressed with the work you have done. + +Best, + Laszlo + +"""]] diff --git a/doc/bugs/Stress_test/comment_9_9f7efe81b7e40aaa04a865394c53e20f._comment b/doc/bugs/Stress_test/comment_9_9f7efe81b7e40aaa04a865394c53e20f._comment new file mode 100644 index 0000000000..a82b98611c --- /dev/null +++ b/doc/bugs/Stress_test/comment_9_9f7efe81b7e40aaa04a865394c53e20f._comment @@ -0,0 +1,52 @@ +[[!comment format=mdwn + username="https://www.google.com/accounts/o8/id?id=AItOawm5iosFbL2By7UFeViqkc6v-hoAtqILeDA" + nickname="Laszlo" + subject="Maybe it is not leaking after all" + date="2013-05-03T18:37:48Z" + content=""" +I have been working the whole day zipping up (tar.gz) all the unused directories. +Now my real data dir looks like this: + + ./annex_real/work_done$ du -hs . + 1,1G . + Has 9088 files and 1608 directories in total: + ./annex_real/work_done$ ls -R1l |grep \\-r |wc -l + 9088 + ./annex_real/work_done$ ls -R1l |grep ^d |wc -l + 1608 + +When I first started git annex, it added 5492 files, then next time it added the missing 3596 files. Then it stopped adding files. +From the gui everything looked fine even at the first start (performed startup scan), even in the log files (daemon.log.x) was nothing suspicious. + + ./annex_real/work_done$ for i in ../.git/annex/daemon.log.*; do echo $i; cat $i |grep files; done + ../.git/annex/daemon.log.1 + ../.git/annex/daemon.log.2 + ../.git/annex/daemon.log.3 + ../.git/annex/daemon.log.4 + [2013-05-03 20:03:34 CEST] Committer: Adding 3596 files + ../.git/annex/daemon.log.5 + [2013-05-03 19:15:22 CEST] Committer: Adding 5492 files + +As you can see, this case is not a stress test at all, +it is really the minimal test case, 1.1GB diskspace, 9088 files and a thousand dirs. +The real question is, why git-annex miss at the first startup 3492 files (ie. adding all the files). + +It would help tremendously, if it would display at startup how many files he found, +and when it adds, then how many left to be added. +Something like this: + + (scanning...) [2013-05-03 20:03:14 CEST] Watcher: Performing startup scan + (started...) + [2013-05-03 20:03:34 CEST] Committer: Found 9088 files + [2013-05-03 20:03:34 CEST] Committer: Adding 3596 files of 9088 remaining files (9088 in total) + .... + [2013-05-03 20:05:04 CEST] Committer: Adding 1492 files of 5492 remaining files (9088 in total) + .... + [2013-05-03 20:06:02 CEST] Committer: Adding 4000 files of 4000 remaining files (9088 in total) + +So it is definietly a bug, and I stuck how to debug it further. Everything looks just fine. + +Best, + Laszlo + +"""]] diff --git a/doc/bugs/Symlink_support_on_Windows_10_Creators_Update_with_Developer_Mode.mdwn b/doc/bugs/Symlink_support_on_Windows_10_Creators_Update_with_Developer_Mode.mdwn new file mode 100644 index 0000000000..b0ea928205 --- /dev/null +++ b/doc/bugs/Symlink_support_on_Windows_10_Creators_Update_with_Developer_Mode.mdwn @@ -0,0 +1,26 @@ +### Please describe the problem. +Since the Win10 Creators Update, if Developer Mode is enabled, symlinks can be created without elevated privileges. During cloning of a repository with symlinks, git detects this and creates symbolic links, whereas in the past it would simply create text-like files with a path. + +When symlinks are created, git-annex breaks in many wonderful ways. The symlinks are not detected as annexed files and `git annex info` shows that no files are in the annex. + +### What steps will reproduce the problem? +On a Windows 10 machine with Creators update (v1709) and Developer mode enabled, do a `git clone` of a git-annex repository. Then `git annex init` and `git annex info`. + +Additionally, if a `git annex sync` is performed, the symlinks are deleted. + +### What version of git-annex are you using? On what operating system? +v6.20180112 on Windows 10 with update 1709. + +### Please provide any additional information below. +Since the aforementioned Windows update (commonly referred to as Creators Update), the mklink command can be executed without requiring elevated privileges if Developer mode is enabled. +See the following two articles for some background: + +- [Windows 10 Creators Update: Symlinks without elevation + [ghacks.net]](https://www.ghacks.net/2016/12/04/windows-10-creators-update-symlinks-without-elevation/) +- [What Is “Developer Mode” in Windows 10? [howtogeek.com]](https://www.howtogeek.com/292914/what-is-developer-mode-in-windows-10/) + +It would seem that git detects this and does not disable symlinks as git-annex expects it to do on Windows. + +### Have you had any luck using git-annex before? (Sometimes we get tired of reading bug reports all day and a lil' positive end note does wonders) +Yes. I use it frequently on multiple platforms. + diff --git a/doc/bugs/Syncing_bare_repository_doesn__39__t_work_after_using_watch_on_Windows.mdwn b/doc/bugs/Syncing_bare_repository_doesn__39__t_work_after_using_watch_on_Windows.mdwn new file mode 100644 index 0000000000..34f310b43f --- /dev/null +++ b/doc/bugs/Syncing_bare_repository_doesn__39__t_work_after_using_watch_on_Windows.mdwn @@ -0,0 +1,93 @@ +### Please describe the problem. + +After initializing a (indirect) repository on my Linux PC and git cloning it on a FAT32 formatted USB drive (direct mode), git annex sync worked fine. +After using it on a Windows system (git annex watch), I took it back to the Linux machine. Now doing git annex sync on the USB drive repository lead to: + + (usb drive) $ git annex sync + commit ok + + There are no commits yet in the currently checked out branch, so cannot merge any remote changes into it. + failed + pull PC + .... + merge: refs/remotes/PC/annex/direct/master - not something we can merge + merge: refs/remotes/PC/synced/master - not something we can merge + failed + (merging origin/git-annex into git-annex...) + git-annex: sync: 2 failed + $ git fsck + Checking object directories: 100% (256/256), done. + $ git annex fsck + fsck blah.pdf (checksum...) ok + fsck foo.pdf (checksum...) ok + fsck bar.pdf (checksum...) ok + ... + (recording state in git...) + +Note that for some reason it mentions the ref refs/remotes/PC/annex/direct/master - which doesn't exist, because the Linux PC repo is in indirect mode. + +### What steps will reproduce the problem? + +1. git init / git annex init on a ext* drive under Linux +2. git sync it to a FAT32-formatted (USB) drive +3. git annex init on the USB drive +4. add each others as remote +5. git sync each other - works fine +6. Change into a Windows system +7. git annex watch inside the git annex repository under Windows +8. add some files, they are committed (says daemon.log) +9. Ctrl+C git annex watch, unmount drive... +10. Change back to a Linux system +11. git annex sync from the annex repository on USB + + +### What version of git-annex are you using? On what operating system? + +Gentoo: + + git-annex version: 6.20170611-gb493ac8d3 + build flags: Assistant Webapp Pairing Testsuite S3(multipartupload)(storageclasses) WebDAV Inotify DBus DesktopNotify ConcurrentOutput TorrentParser MagicMime Feeds Quvi + dependency versions: aws-0.14.1 bloomfilter-2.0.1.0 cryptonite-0.20 DAV-1.3.1 feed-0.3.11.1 ghc-8.0.1 http-client-0.4.31.1 persistent-sqlite-2.6 torrent-10000.0.0 uuid-1.3.12 yesod-1.4.3 + key/value backends: SHA256E SHA256 SHA512E SHA512 SHA224E SHA224 SHA384E SHA384 SHA3_256E SHA3_256 SHA3_512E SHA3_512 SHA3_224E SHA3_224 SHA3_384E SHA3_384 SKEIN256E SKEIN256 SKEIN512E SKEIN512 SHA1E SHA1 MD5E MD5 WORM URL + remote types: git gcrypt p2p S3 bup directory rsync web bittorrent webdav tahoe glacier ddar hook external + local repository version: 5 + supported repository versions: 3 5 6 + upgrade supported from repository versions: 0 1 2 3 4 5 + operating system: linux x86_64 + +Windows 10: + + git-annex version: 6.20170611-gb493ac8 + build flags: Assistant Webapp Pairing Testsuite S3(multipartupload)(storageclasses) WebDAV ConcurrentOutput TorrentParser Feeds Quvi + dependency versions: aws-0.14.0 bloomfilter-2.0.1.0 cryptonite-0.7 DAV-1.3.1 feed-0.3.11.1 ghc-7.10.2 http-client-0.4.31.1 persistent-sqlite-2.2 torrent-10000.0.0 uuid-1.3.12 yesod-1.4.3 + key/value backends: SHA256E SHA256 SHA512E SHA512 SHA224E SHA224 SHA384E SHA384 SHA3_256E SHA3_256 SHA3_512E SHA3_512 SHA3_224E SHA3_224 SHA3_384E SHA3_384 SKEIN256E SKEIN256 SKEIN512E SKEIN512 SHA1E SHA1 MD5E MD5 WORM URL + remote types: git gcrypt p2p S3 bup directory rsync web bittorrent webdav tahoe glacier ddar hook external + local repository version: 5 + supported repository versions: 3 5 6 + upgrade supported from repository versions: 2 3 4 5 + operating system: mingw32 i386 + +### Please provide any additional information below. + +[[!format sh """ +# daemon.log from git watch +[2017-08-16 13:16:01.6773524] main: starting watch version 6.20170611-gb493ac8 +(scanning...) [2017-08-16 13:16:02.0049693] Watcher: Performing startup scan +(started...) +[2017-08-16 13:16:03.0007698] Committer: Committing changes to git +(recording state in git...) +[2017-08-16 13:16:51.420494] Committer: Adding annex3.txt +add annex3.txt ok +[2017-08-16 13:16:51.771892] Committer: Committing changes to git +(recording state in git...) + + +# End of transcript or log. +"""]] + +### Have you had any luck using git-annex before? (Sometimes we get tired of reading bug reports all day and a lil' positive end note does wonders) + +Used it under only-linux before. I especially like the location tracking. + + +Also tried using special remotes (mega.nz) with encryption - although I was very confused at first that you need the private key for the hybrid approach. After some reading I think I need sharedpubkey? (I just wanna use it for myself, but don't wanna insert the smartcard where my private key is stored every time for sending the files to the remote). diff --git a/doc/bugs/Syncing_bare_repository_doesn__39__t_work_after_using_watch_on_Windows/comment_1_e197b8dc0eae0dd6dea4febdc2ecce53._comment b/doc/bugs/Syncing_bare_repository_doesn__39__t_work_after_using_watch_on_Windows/comment_1_e197b8dc0eae0dd6dea4febdc2ecce53._comment new file mode 100644 index 0000000000..d386792800 --- /dev/null +++ b/doc/bugs/Syncing_bare_repository_doesn__39__t_work_after_using_watch_on_Windows/comment_1_e197b8dc0eae0dd6dea4febdc2ecce53._comment @@ -0,0 +1,11 @@ +[[!comment format=mdwn + username="https://openid.stackexchange.com/user/8a69a637-97cb-41e6-8f45-00f08ba54d6e" + nickname="Andreas Duering" + avatar="http://cdn.libravatar.org/avatar/915cf485815ded2f7f4b68736994c185ecda8c31dd293acd16f5e21ca7db23e1" + subject="comment 1" + date="2017-08-16T17:10:58Z" + content=""" +Oh, and by the way: The bug does *not* occur if you don't use git annex watch, but git annex add the files manually under Windows. + +(In this case, I have a weird drive letter problem, probably (can't figure out how to make the link work) bugs/Windows_to_Linux_clone_-_Windows_drive_letters_cause_git_annex_get_to_fail) +"""]] diff --git a/doc/bugs/Syncing_of_file_contents_seems_to_be_broken_in_recent_versions_of_the_assistant.mdwn b/doc/bugs/Syncing_of_file_contents_seems_to_be_broken_in_recent_versions_of_the_assistant.mdwn new file mode 100644 index 0000000000..590c862057 --- /dev/null +++ b/doc/bugs/Syncing_of_file_contents_seems_to_be_broken_in_recent_versions_of_the_assistant.mdwn @@ -0,0 +1,134 @@ +### Please describe the problem. + +Since I last tested git-annex (version 4.20130516.1) my setup seems to no longer work in current versions (5.20131130) + +### What steps will reproduce the problem? + +[[!format sh """ +$ mkdir 1 2 +$ cd 1; git init; git-annex init; git annex direct; echo "file added to 1" > file_from_1; cd .. +Initialized empty Git repository in /home/pedrocr/testannexbug/1/.git/ +init ok +(Recording state in git...) +commit +# On branch master +# +# Initial commit +# +nothing to commit (create/copy files and use "git add" to track) +ok +direct ok +$ cd 2; git init; git-annex init; git annex direct; echo "file added to 2" > file_from_2; cd .. +Initialized empty Git repository in /home/pedrocr/testannexbug/2/.git/ +init ok +(Recording state in git...) +commit +# On branch master +# +# Initial commit +# +nothing to commit (create/copy files and use "git add" to track) +ok +direct ok +$ cd 1; git remote add 2 ssh://localhost/~pedrocr/Hacks/test-git-annex/2; git annex assistant; cd .. +$ cd 2; git remote add 1 ssh://localhost/~pedrocr/Hacks/test-git-annex/1; git annex assistant; cd .. +$ ls -la 1 +total 20 +drwxrwxr-x 3 pedrocr pedrocr 4096 Dec 15 11:33 . +drwxrwxr-x 4 pedrocr pedrocr 4096 Dec 15 11:33 .. +-rw-rw-r-- 1 pedrocr pedrocr 16 Dec 15 11:33 file_from_1 +-rw-r--r-- 1 pedrocr pedrocr 16 Dec 15 11:33 file_from_2 +drwxrwxr-x 9 pedrocr pedrocr 4096 Dec 15 11:33 .git +$ ls -la 2 +total 20 +drwxrwxr-x 3 pedrocr pedrocr 4096 Dec 15 11:33 . +drwxrwxr-x 4 pedrocr pedrocr 4096 Dec 15 11:33 .. +-rw-r--r-- 1 pedrocr pedrocr 16 Dec 15 11:33 file_from_1 +-rw-rw-r-- 1 pedrocr pedrocr 16 Dec 15 11:33 file_from_2 +drwxrwxr-x 9 pedrocr pedrocr 4096 Dec 15 11:33 .git +"""]] + +so far so good + +[[!format sh """ +$ date | tee 1/file_from_1 +Sun Dec 15 11:53:18 WET 2013 +$ date | tee 2/file_from_2 +Sun Dec 15 11:53:22 WET 2013 +$ cat 1/* +Sun Dec 15 11:53:18 WET 2013 +file added to 2 +$ cat 2/* +file added to 1 +Sun Dec 15 11:53:22 WET 2013 +$ +"""]] + +But then syncing doesn't work. There doesn't seem to be anything special in the logs + +[[!format sh """ +$ tail -n 20 1/.git/annex/daemon.log +(Recording state in git...) +add file_from_1 (checksum...) [2013-12-15 11:53:18 WET] Committer: Committing changes to git +[2013-12-15 11:53:18 WET] Pusher: Syncing with 2 + +file_from_1 + 29 100% 0.00kB/s 0:00:00 (xfer#1, to-check=0/1) + +sent 104 bytes received 31 bytes 270.00 bytes/sec +total size is 29 speedup is 0.21 +[2013-12-15 11:53:18 WET] Transferrer: Uploaded file_from_1 +Already up-to-date. +To ssh://localhost/~pedrocr/Hacks/test-git-annex/2 + f025062..431ac37 git-annex -> synced/git-annex + d393bb3..fef60ca annex/direct/master -> synced/master +Already up-to-date. +[2013-12-15 11:53:21 WET] Pusher: Syncing with 2 +To ssh://localhost/~pedrocr/Hacks/test-git-annex/2 + 431ac37..3cc7045 git-annex -> synced/git-annex + + +$ tail -n 20 2/.git/annex/daemon.log +(Recording state in git...) +add file_from_2 (checksum...) [2013-12-15 11:53:22 WET] Committer: Committing changes to git +[2013-12-15 11:53:22 WET] Pusher: Syncing with 1 + +file_from_2 + 29 100% 0.00kB/s 0:00:00 (xfer#1, to-check=0/1) +Already up-to-date. + +sent 104 bytes received 31 bytes 270.00 bytes/sec +total size is 29 speedup is 0.21 +[2013-12-15 11:53:22 WET] Transferrer: Uploaded file_from_2 +To ssh://localhost/~pedrocr/Hacks/test-git-annex/1 + ebbcd4c..dae7a77 git-annex -> synced/git-annex + ffd5a90..67ad7c1 annex/direct/master -> synced/master +Already up-to-date. +[2013-12-15 11:53:25 WET] Pusher: Syncing with 1 +To ssh://localhost/~pedrocr/Hacks/test-git-annex/1 + dae7a77..976c899 git-annex -> synced/git-annex +"""]] + + + +### What version of git-annex are you using? On what operating system? + +I'm running git-annex from the Ubuntu PPA on Ubuntu 12.04 LTS. + +[[!format sh """ +$ lsb_release -v +No LSB modules are available. +$ lsb_release -a +No LSB modules are available. +Distributor ID: Ubuntu +Description: Ubuntu 12.04.3 LTS +Release: 12.04 +Codename: precise +$ git annex version +git-annex version: 5.20131130 +build flags: Assistant Webapp Pairing S3 WebDAV Inotify DBus XMPP Feeds Quvi TDFA +key/value backends: SHA256E SHA1E SHA512E SHA224E SHA384E SHA256 SHA1 SHA512 SHA224 SHA384 WORM URL +remote types: git gcrypt S3 bup directory rsync web webdav glacier hook +"""]] + +> [[!taglink moreinfo]] needed. --[[Joey]] diff --git a/doc/bugs/Syncing_of_file_contents_seems_to_be_broken_in_recent_versions_of_the_assistant/comment_1_8100505a7ac74646e0767d03fe643a45._comment b/doc/bugs/Syncing_of_file_contents_seems_to_be_broken_in_recent_versions_of_the_assistant/comment_1_8100505a7ac74646e0767d03fe643a45._comment new file mode 100644 index 0000000000..337bd12b05 --- /dev/null +++ b/doc/bugs/Syncing_of_file_contents_seems_to_be_broken_in_recent_versions_of_the_assistant/comment_1_8100505a7ac74646e0767d03fe643a45._comment @@ -0,0 +1,8 @@ +[[!comment format=mdwn + username="http://joeyh.name/" + ip="209.250.56.87" + subject="comment 1" + date="2013-12-16T20:48:12Z" + content=""" +Sounds to me like it could be a messed up direct mode mapping file due to bugs in version 4.20130516. But I am not sure I follow your transcript; you don't seem to show how things looked when it went wrong. If it was the mapping, `git annex fsck` would correct the problem. +"""]] diff --git a/doc/bugs/Syncing_of_file_contents_seems_to_be_broken_in_recent_versions_of_the_assistant/comment_2_9833e8f77c6148db9572316066a67eee._comment b/doc/bugs/Syncing_of_file_contents_seems_to_be_broken_in_recent_versions_of_the_assistant/comment_2_9833e8f77c6148db9572316066a67eee._comment new file mode 100644 index 0000000000..db7c82fd36 --- /dev/null +++ b/doc/bugs/Syncing_of_file_contents_seems_to_be_broken_in_recent_versions_of_the_assistant/comment_2_9833e8f77c6148db9572316066a67eee._comment @@ -0,0 +1,8 @@ +[[!comment format=mdwn + username="https://www.google.com/accounts/o8/id?id=AItOawkx5V3MTbzCXS3J7Mn9FEq8M9bPPYMkAHY" + nickname="Pedro" + subject="comment 2" + date="2013-12-16T23:00:48Z" + content=""" +What my transcript shows is that after the initial sync 1 doesn't get the changes from 2 and vice versa. This test is from scratch so it can't be something related to the old version. I'll try the fsck on my other machines to see if it fixes the issues there. +"""]] diff --git a/doc/bugs/Syncing_of_file_contents_seems_to_be_broken_in_recent_versions_of_the_assistant/comment_3_1504f8767f1f4415222d8c315c734e81._comment b/doc/bugs/Syncing_of_file_contents_seems_to_be_broken_in_recent_versions_of_the_assistant/comment_3_1504f8767f1f4415222d8c315c734e81._comment new file mode 100644 index 0000000000..07d6eec3a6 --- /dev/null +++ b/doc/bugs/Syncing_of_file_contents_seems_to_be_broken_in_recent_versions_of_the_assistant/comment_3_1504f8767f1f4415222d8c315c734e81._comment @@ -0,0 +1,9 @@ +[[!comment format=mdwn + username="http://joeyh.name/" + ip="209.250.56.146" + subject="comment 3" + date="2014-03-07T17:21:02Z" + content=""" +Any news on this? Does it happen with the current version? + +"""]] diff --git a/doc/bugs/Synology_NAS__58___timer__95__create__58___Bad_address.mdwn b/doc/bugs/Synology_NAS__58___timer__95__create__58___Bad_address.mdwn new file mode 100644 index 0000000000..f638638e48 --- /dev/null +++ b/doc/bugs/Synology_NAS__58___timer__95__create__58___Bad_address.mdwn @@ -0,0 +1,58 @@ +### Please describe the problem. + +After updating my Synology DS216+ NAS from [6.2-23739-1](https://www.synology.com/en-uk/releaseNote/DS216+II#ver_23739-1) to [6.2-23739-2](https://www.synology.com/en-uk/releaseNote/DS216+II#ver_23739-2), running `git-annex` with any non-trivial command, locally on the NAS or remotely on the NAS results in: + + git-annex: timer_create: Bad address + +I'm guessing that this means that a library function the standalone binary needs is no longer present, but I'm unsure why. The `-1` to `-2` update seems to only have two security fixes ([SA 18-36](https://www.synology.com/en-uk/support/security/Synology_SA_18_36), and [SA 18-01](https://www.synology.com/en-uk/support/security/Synology_SA_18_01) -- the latter being a Spectre/Meltdown bug). + +Do you know if `timer_create` is a kernel system call? Or a libc (etc) library function? + +### What steps will reproduce the problem? + +Update Synology NAS to [6.2-23739-2](https://www.synology.com/en-uk/releaseNote/DS216+II#ver_23739-2), run `git annex sync`, `git annex version` or similar (remotely or locally). + +### What version of git-annex are you using? On what operating system? + +`x86-32`, stand alone build, for ancient-kernels, as that the 64-bit stand alone build [no longer seemed to work due to locale issues](https://ewen.mcneill.gen.nz/blog/entry/2017-05-28-git-annex-on-synology-ds216+/) (see note added at the end). + +I believe it is `6.20180626`, but `git annex version` currently also fails... + +### Please provide any additional information below. + + ewen@nas01:/volume1/music/podcasts$ hostname --fqdn + nas01 + ewen@nas01:/volume1/music/podcasts$ git annex sync + git-annex: timer_create: Bad address + ewen@nas01:/volume1/music/podcasts$ + ewen@nas01:/volume1/music/podcasts$ git annex version + git-annex: timer_create: Bad address + ewen@nas01:/volume1/music/podcasts$ + ewen@nas01:/volume1/music/podcasts$ uname -mr + 3.10.105 x86_64 + ewen@nas01:/volume1/music/podcasts$ + +There is a [6.2.1 firmware release](https://www.synology.com/en-uk/releaseNote/DS216+) for the Synology NAS released a couple of days ago, but it does not yet seem to be visible to my NAS. I can try that one if it'd help. + +I can also try switching back to one of the more modern `x86-64` / `x86-32` stand alone builds if that'd help. But then I'd need some assistance with working around the: + + sh: loadlocale.c:129: _nl_intern_locale_data: Assertion `cnt < (sizeof (_nl_value_type_LC_TIME) / sizeof (_nl_value_type_LC_TIME[0]))' failed. + error: git-annex died of signal 6 + +error I was seeing on those builds. (From other work elsewhere it feels a lot like the LC_TIME structure changed in size in some libc definition somewhere in the last 6-12 months; in another project we had a similar break on Ubuntu 18.04 issues which didn't affect Ubuntu 16.04 and earlier.) + +In case it helps, out of the box defaults give: + + ewen@nas01:/volume1/music/podcasts$ echo $LANG + en_US.utf8 + ewen@nas01:/volume1/music/podcasts$ ls /volume1/thirdparty/git-annex.linux/locales/en_US.utf8/ + LC_ADDRESS LC_IDENTIFICATION LC_MONETARY LC_PAPER + LC_COLLATE LC_MEASUREMENT LC_NAME LC_TELEPHONE + LC_CTYPE LC_MESSAGES LC_NUMERIC LC_TIME + ewen@nas01:/volume1/music/podcasts$ + +(although those files are currently generated with the now broken x86-32 legacy standalone build). + +### Have you had any luck using git-annex before? (Sometimes we get tired of reading bug reports all day and a lil' positive end note does wonders) + +Definitely. It was working until I updated the NAS firmware image this afternoon... for this repository holding years of podcasts and many others. diff --git a/doc/bugs/Synology_NAS__58___timer__95__create__58___Bad_address/comment_1_8ce2cbdf5b5c31fd42b0ed270c54a824._comment b/doc/bugs/Synology_NAS__58___timer__95__create__58___Bad_address/comment_1_8ce2cbdf5b5c31fd42b0ed270c54a824._comment new file mode 100644 index 0000000000..8970688f90 --- /dev/null +++ b/doc/bugs/Synology_NAS__58___timer__95__create__58___Bad_address/comment_1_8ce2cbdf5b5c31fd42b0ed270c54a824._comment @@ -0,0 +1,23 @@ +[[!comment format=mdwn + username="ewen" + avatar="http://cdn.libravatar.org/avatar/605b2981cb52b4af268455dee7a4f64e" + subject="timer_create" + date="2018-09-03T22:06:00Z" + content=""" +From some digging around it looks like [`timer_create`](http://man7.org/linux/man-pages/man2/timer_create.2.html) is a Linux system call. So presumably the issue with the Synology NAS is being caused by the kernel being updated. + +Of note, it appears that [Microsoft Windows System for Linux does not support `timer_create` either](https://github.com/Microsoft/WSL/issues/307). And that [system call is apparently required for Haskell binaries](https://github.com/Microsoft/WSL/issues/307#issuecomment-217051052)... Apparently (from this thread) other people have hit this issue trying to run `git-annex` too. + +There's a [suggestion of some work around flags](https://github.com/Microsoft/WSL/issues/307#issuecomment-239617836), but they do not seem to work with the `git-annex` stand alone binary as it is currently built: + + ewen@nas01:~$ git-annex +RTS -V0 -RTS help + git-annex: Most RTS options are disabled. Link with -rtsopts to enable them. + ewen@nas01:~$ export GHCRTS=-V0 + ewen@nas01:~$ git annex help + git-annex: Most RTS options are disabled. Link with -rtsopts to enable them. + ewen@nas01:~$ + +Possibly it'll be necessary to try to persuade Synology to re-enable the `timer_create` system call. But my guess is maybe they turned it off as a Spectre/Meltdown fix, so it may be non-trivial to persuade them to change it back. + +Ewen +"""]] diff --git a/doc/bugs/Synology_NAS__58___timer__95__create__58___Bad_address/comment_2_1316ea67dc7df8d3d6f681ac3caf6d3c._comment b/doc/bugs/Synology_NAS__58___timer__95__create__58___Bad_address/comment_2_1316ea67dc7df8d3d6f681ac3caf6d3c._comment new file mode 100644 index 0000000000..e849792a78 --- /dev/null +++ b/doc/bugs/Synology_NAS__58___timer__95__create__58___Bad_address/comment_2_1316ea67dc7df8d3d6f681ac3caf6d3c._comment @@ -0,0 +1,12 @@ +[[!comment format=mdwn + username="ewen" + avatar="http://cdn.libravatar.org/avatar/605b2981cb52b4af268455dee7a4f64e" + subject="Synology bug report" + date="2018-09-03T22:36:07Z" + content=""" +FWIW, I've created a Synology Account, to create a Synology Support Request, [#2082132](https://account.synology.com/en-global/support/2082132/detail) for the missing timer_create system call on DSM 6.2-23739-2. I'm not holding out a lot of hope given it's a third party application running on the NAS, and not even one in their app store, but maybe there's a chance it'll get the system call turned back on again. That Synology support request includes a link to this bug report. + +Ewen + +PS: I expect that bug report will only be visible if you're logged in with my Synology account. But at least someone else reporting it can reference the same issue to Synology Support. +"""]] diff --git a/doc/bugs/Synology_NAS__58___timer__95__create__58___Bad_address/comment_3_0e098ee460a0ad70dbe34d74c5e9bf3b._comment b/doc/bugs/Synology_NAS__58___timer__95__create__58___Bad_address/comment_3_0e098ee460a0ad70dbe34d74c5e9bf3b._comment new file mode 100644 index 0000000000..782e7272ae --- /dev/null +++ b/doc/bugs/Synology_NAS__58___timer__95__create__58___Bad_address/comment_3_0e098ee460a0ad70dbe34d74c5e9bf3b._comment @@ -0,0 +1,44 @@ +[[!comment format=mdwn + username="ewen" + avatar="http://cdn.libravatar.org/avatar/605b2981cb52b4af268455dee7a4f64e" + subject="x86-64 build working on Synology NAS (with work around)" + date="2018-09-08T02:03:21Z" + content=""" +The Synology reply was \"our developers suspected that this is related to some library linking problem in the kernel\", and they asked for remote access to my device -- which I wasn't very keen on so I did some experimentation of my own. + +With the latest standalone `x86-64` build, I do *not* get the `timer_create` error. So it does seem like there's a linking issue around accessing `timer_create` from the \"ancient\" builds on the current Synology DS216+ kernel/libc. + +However, the standalone `x86-64` builds *do* fail with `LC_TIME` issues by default: + + ewen@nas01:/volume1/thirdparty$ git-annex help + warning: /bin/sh: setlocale: LC_ALL: cannot change locale (en_US.utf8) + sh: loadlocale.c:129: _nl_intern_locale_data: Assertion `cnt < (sizeof (_nl_value_type_LC_TIME) / sizeof (_nl_value_type_LC_TIME[0]))' failed. + Aborted + ewen@nas01:/volume1/thirdparty$ + +(and have for a while; it was why I switched to trying the Ancient 32-bit build earlier this year). It appears at some point the `LC_TIME` structure changed in size, which means that mixing statically linked binaries with `locale` information that is older/newer causes problems. + +Fortunately that problem is now [common](https://unix.stackexchange.com/questions/444102/loadlocale-c-nl-intern-locale-data-assertion-error) [enough](https://github.com/hadolint/hadolint/issues/173) ([etc](https://github.com/ProteoWizard/pwiz/issues/29)), due to people encountering it with, eg Ubuntu 18.04 LTS, that it's widely known, and there's a [pretty easy work around](https://stackoverflow.com/questions/49301627/android-7-1-2-armv7): + + export LC_ALL=C + +(which overrides the Synology shell default of `LC_ALL=en_US.utf8`). Just setting `LC_TIME` does not seem to be sufficient if either `LC_ALL` or `LANG` are set to other values (with the Synology shell does by default). + +That `LC_ALL=C` override works for me, so I've tweaked my `git-annex-shell`, `git-annex-wrapper`, and `git-annex` shell scripts to do that before carrying on to call the rest of the `git-annex` tools, and everything seems to work again. Possibly the `standalone` builds should do that automatically? + + ewen@nas01:~$ set | egrep 'LANG|LC_' + LANG=en_US.utf8 + LC_ALL=en_US.utf8 + ewen@nas01:~$ LC_ALL=C git-annex version + git-annex version: 6.20180807-g48d11a5df + build flags: Assistant Webapp Pairing S3(multipartupload)(storageclasses) WebDAV Inotify DBus DesktopNotify ConcurrentOutput TorrentParser MagicMime Feeds Testsuite + dependency versions: aws-0.19 bloomfilter-2.0.1.0 cryptonite-0.25 DAV-1.3.2 feed-1.0.0.0 ghc-8.2.2 http-client-0.5.13 persistent-sqlite-2.8.1.2 torrent-10000.1.1 uuid-1.3.13 yesod-1.6.0 + key/value backends: SHA256E SHA256 SHA512E SHA512 SHA224E SHA224 SHA384E SHA384 SHA3_256E SHA3_256 SHA3_512E SHA3_512 SHA3_224E SHA3_224 SHA3_384E SHA3_384 SKEIN256E SKEIN256 SKEIN512E SKEIN512 BLAKE2B256E BLAKE2B256 BLAKE2B512E BLAKE2B512 BLAKE2B160E BLAKE2B160 BLAKE2B224E BLAKE2B224 BLAKE2B384E BLAKE2B384 BLAKE2S256E BLAKE2S256 BLAKE2S160E BLAKE2S160 BLAKE2S224E BLAKE2S224 BLAKE2SP256E BLAKE2SP256 BLAKE2SP224E BLAKE2SP224 SHA1E SHA1 MD5E MD5 WORM URL + remote types: git gcrypt p2p S3 bup directory rsync web bittorrent webdav adb tahoe glacier ddar hook external + operating system: linux x86_64 + supported repository versions: 3 5 6 + upgrade supported from repository versions: 0 1 2 3 4 5 + ewen@nas01:~$ + +Ewen +"""]] diff --git a/doc/bugs/Synology_NAS__58___timer__95__create__58___Bad_address/comment_4_64fd17f3537fec02f8356e535a1d0bb4._comment b/doc/bugs/Synology_NAS__58___timer__95__create__58___Bad_address/comment_4_64fd17f3537fec02f8356e535a1d0bb4._comment new file mode 100644 index 0000000000..6ce52c04b1 --- /dev/null +++ b/doc/bugs/Synology_NAS__58___timer__95__create__58___Bad_address/comment_4_64fd17f3537fec02f8356e535a1d0bb4._comment @@ -0,0 +1,21 @@ +[[!comment format=mdwn + username="joey" + subject="""comment 4""" + date="2018-09-11T17:20:18Z" + content=""" +IIRC the haskell runtime stopped using `timer_create` after ghc 7 or +something like that. These days it uses poll. + +That said, `timer_create` is part of POSIX and part of the linux API +and that is unlikely to change. There might be something in the libc/kernel +interface that has changed in newer kernels, if so it may be that the +i386ancient build is not going to work with new kernels.. But it's there to +support old kernels anyway. + +As for the setlocale failure, while LC_ALL=C avoids it, that's not +something I want to set by default, because people do use git-annex with +unicode filenames and that can impact that, as well as of course preventing +display of any translated messages by anything git-annex runs (though +git-annex itself is not translated). It would be good to get to the bottom +of the setlocale failure. +"""]] diff --git a/doc/bugs/Synology_NAS__58___timer__95__create__58___Bad_address/comment_5_dacfb426f7e67e984bb9281a40427e85._comment b/doc/bugs/Synology_NAS__58___timer__95__create__58___Bad_address/comment_5_dacfb426f7e67e984bb9281a40427e85._comment new file mode 100644 index 0000000000..3dbaa43558 --- /dev/null +++ b/doc/bugs/Synology_NAS__58___timer__95__create__58___Bad_address/comment_5_dacfb426f7e67e984bb9281a40427e85._comment @@ -0,0 +1,26 @@ +[[!comment format=mdwn + username="ewen" + avatar="http://cdn.libravatar.org/avatar/605b2981cb52b4af268455dee7a4f64e" + subject="timer_create" + date="2018-09-11T21:37:24Z" + content=""" +It does very much look like something has removed `timer_create` from the Synology NAS. Eg, if I statically compile the [example `timer_create` program in the man page](http://man7.org/linux/man-pages/man2/timer_create.2.html) on Debian Jessie/Stretch with: + + gcc -o test_timer_create.jessie -static test_timer_create.c -lrt -lpthread + +and try to run it on the Synology NAS now, I get effectively the same symptoms: + + ewen@nas01:~$ ./test_timer_create.jessie 1 100 + Establishing handler for signal 34 + Blocking signal 34 + timer_create: Bad address + ewen@nas01:~$ + +(Unfortunately I have no easy way to check if with a previous Synology NAS version.) + +The same manpage notes that it *can* be configured out of the kernel: \"Since Linux 4.10, support for POSIX timers is a configurable option that is enabled by default. Kernel support can be disabled via the CONFIG_POSIX_TIMERS option.\" for which [the patch](https://patchwork.kernel.org/patch/9422453/) notes that \"Some embedded systems have no use for them\". Obviously the Synology NAS is an embedded kernel situation. Given that the tripping point was the [release where Synology introduced their Meltdown/Spectre fixes](https://www.synology.com/en-uk/releaseNote/DS216+II#ver_23739-2), I imagine that they backported a *lot* of related fixes from later kernels, and it seems likely that they did deliberately turn off `CONFIG_POSIX_TIMERS`, even if their 1st level/2nd level helpdesk did not know about it, since timing RAM accesses is pretty much key to the common Meltdown/Spectre exploits, and thus various projects removed \"high resolution\" timers. (Unfortunately there's no `/proc/config.gz` or similar that I can see, and while I can find some [Synology open source bits](https://sourceforge.net/projects/dsgpl/files/) I got bored digging to try to find a kernel config file.) + +The fact that later Haskell switched to `poll` probably explains why the later build works. At least that seems to a viable path forward for using `git-annex` on my NAS for now. + +Ewen +"""]] diff --git a/doc/bugs/Synology_NAS__58___timer__95__create__58___Bad_address/comment_6_c85f2c69aa2cff05859e92afc95e9ccf._comment b/doc/bugs/Synology_NAS__58___timer__95__create__58___Bad_address/comment_6_c85f2c69aa2cff05859e92afc95e9ccf._comment new file mode 100644 index 0000000000..beb30c1193 --- /dev/null +++ b/doc/bugs/Synology_NAS__58___timer__95__create__58___Bad_address/comment_6_c85f2c69aa2cff05859e92afc95e9ccf._comment @@ -0,0 +1,45 @@ +[[!comment format=mdwn + username="ewen" + avatar="http://cdn.libravatar.org/avatar/605b2981cb52b4af268455dee7a4f64e" + subject="LC_TIME" + date="2018-09-11T22:23:49Z" + content=""" +From some hunting around I can find reports of that `loadlocale.c` issue with `LC_TIME` throughout 2018. It seems to be more commonly encountered now that things like Ubuntu 18.04 are released with the newer glibc. (There's also a `LC_COLLATE` change somewhere around [mid](https://www.reddit.com/r/archlinux/comments/3g9dzf/loadlocalec130_error_on_everything/)/[late](https://www.linuxquestions.org/questions/slackware-arm-108/login-error-after-upgrade-of-current-4175562326/) 2015, which seems to be the previous change in data format impacting people in this way.) + +The generally understood wisdom seems to be that it's caused by locale data compiled by older (glibc) tool versions than the ones being used to load it (eg, statically linked into the binaries). This is at least the second context in which I've come across it, in the last month, myself. Both being caused by statically linked binaries built on Linux versions older/newer than the one on which they were being run, approximately across the 2017/2018 calendar boundary. + +On the Synology NAS this seems to be the locale tools version: + + ewen@nas01:~$ locale --version + locale (crosstool-NG 1.20.0) 2.20-2014.11 + Copyright (C) 2014 Free Software Foundation, Inc. + This is free software; see the source for copying conditions. There is NO + warranty; not even for MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. + Written by Ulrich Drepper. + ewen@nas01:~$ + +which is a tiny bit newer than Debian Jessie: + + ewen@noc:~$ cat /etc/debian_version + 8.11 + ewen@noc:~$ locale --version + locale (Debian GLIBC 2.19-18+deb8u10) 2.19 + Copyright (C) 2014 Free Software Foundation, Inc. + This is free software; see the source for copying conditions. There is NO + warranty; not even for MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. + Written by Ulrich Drepper. + ewen@noc:~$ + +and somewhat older than Debian Stretch (2.24), Debian Unstable (2.27), or [Ubuntu 18.04](https://packages.ubuntu.com/bionic/locales) which is also 2.27. My guess from the timing is that something in the `lc_time` structure size changed between about glibc 2.24 and 2.27, maybe late 2017 or early 2018. + +Digging in the glibc git tree turns up [commit f301e5334065e93aace667fd4a87bce6fc1dbd13](https://sourceware.org/git/?p=glibc.git;a=commit;h=f301e5334065e93aace667fd4a87bce6fc1dbd13), from 2017-10-27 which foreshadows a change with \"Now when we are about to add alternative month +names to LC_TIME (BZ#10871) this will fail again.\". This appears to be [BugZilla #10871](https://sourceware.org/bugzilla/show_bug.cgi?id=10871) which had been open forever, but the patches seem to have been [finally pushed on 2018-01-22](https://sourceware.org/bugzilla/show_bug.cgi?id=10871#c137). Which is about the right timing for the problems seen. Building locales with a glibc after that and loading with an older libc (statically linked), or vice versa, is going to break. (I think this [might be the commit that breaks things](https://sourceware.org/git/?p=glibc.git;a=commit;h=95cb863a1ef7760a11272bbd7ba5fe62dc41be54), and if not, it's very nearby that commit in the history. Applied 2018-01-22.) + +I don't see a work around for statically linked binaries using system locale files, short of building versions with code from on either side of that flag date and suggesting people use the right version depending on their locale files... + +... the best kludge I've thought of so far that *might* work is to turn `LC_ALL` into `LC_...` settings for each individual one, except `LC_TIME=C`, in the hope that maybe that'll cause the (changed) time part not to load. But I haven't tested that myself. + +Ewen + +PS: One might hope that this change could have been done in a backwards compatible manner. But that does not appear to be the case here. glibc seem to have been perfectly fine with creating a flag day, presumably reasoning distros can always force the locales to be recompiled so they'll always stay in sync. +"""]] diff --git a/doc/bugs/Synology_NAS__58___timer__95__create__58___Bad_address/comment_7_e8c76f8b9645733a793676dc274744ef._comment b/doc/bugs/Synology_NAS__58___timer__95__create__58___Bad_address/comment_7_e8c76f8b9645733a793676dc274744ef._comment new file mode 100644 index 0000000000..445089f3bd --- /dev/null +++ b/doc/bugs/Synology_NAS__58___timer__95__create__58___Bad_address/comment_7_e8c76f8b9645733a793676dc274744ef._comment @@ -0,0 +1,14 @@ +[[!comment format=mdwn + username="ewen" + avatar="http://cdn.libravatar.org/avatar/605b2981cb52b4af268455dee7a4f64e" + subject="LC_TIME" + date="2018-09-11T23:52:23Z" + content=""" +In case it helps, I think this is the [change to the `lc_time` structure definition](https://sourceware.org/git/?p=glibc.git;a=blobdiff;f=locale/categories.def;h=3cbb4e673882eb4ff827fa2b6d9eec2faafceff4;hp=47947f76862ea77779b821a7ca294a3eff67136e;hb=95cb863a1ef7760a11272bbd7ba5fe62dc41be54;hpb=4612268a0ad8e3409d8ce2314dd2dd8ee0af5269), and this is an example of the [change needed to a program loading them](https://sourceware.org/git/?p=glibc.git;a=blobdiff;f=locale/programs/ld-time.c;h=41864482eccb14e31e0ff6cbe27c7008f687160d;hp=67d055a1c41d859b651e3af60f55b6a1499ed107;hb=95cb863a1ef7760a11272bbd7ba5fe62dc41be54;hpb=4612268a0ad8e3409d8ce2314dd2dd8ee0af5269). Looks like two new arrays (one `char *`, one wide char), each 12 entries long, and maybe a flag for whether the \"alt month\" (names) are present (`alt_mon_defined`), but I'm not clear whether or not that is in the loaded files. Both linked from the commit I mentioned in the previous comment. + +While the parser in that program seems to be able to cope with various formats to some extent, the [loadlocale.c code has an assert check on size](https://sourceware.org/git/?p=glibc.git;a=blob;f=locale/loadlocale.c;h=15f93bec80e1f11c114eaac1c85f77e6ed071712;hb=95cb863a1ef7760a11272bbd7ba5fe62dc41be54#l126) that makes it more brittle. (And at this point I think I've seen it fail both with \"newer locale files, older binary\" and \"older locale files, newer binary\", so I suspect both are at least \"untested\".) + +For the Synology NAS case building on Debian Stretch will *might* still work, and building on Debian Jessie looks like it should still work, but building on Debian Testing/Unstable presumably will not. Presumably this incompatibility issue will just get more common over 2018/2019 as more things are/aren't upgraded. Joy. + +Ewen +"""]] diff --git a/doc/bugs/Tests_fail_on_Windows_10.mdwn b/doc/bugs/Tests_fail_on_Windows_10.mdwn new file mode 100644 index 0000000000..4df7bcbed7 --- /dev/null +++ b/doc/bugs/Tests_fail_on_Windows_10.mdwn @@ -0,0 +1,2004 @@ +### Please describe the problem. + +Just tried to install git-annex on a friend's Windows machine to share a repo, but a lot of the tests failed. I have no idea how to debug things on Windows, maybe someone has an idea. AFAIK Windows has only crippled file systems, I tried with NTFS and ReFS. + +It says "Permission denied. File is being used by another process". Maybe some kind of indexing? git annex assistant was turned off and it was the only test running. + +### What steps will reproduce the problem? + +Install git, install git-annex, run "git annex test" + +### What version of git-annex are you using? On what operating system? + +Windows 10. git 32 bit + +git-annex version: 6.20170611-gb493ac8 +build flags: Assistant Webapp Pairing Testsuite S3(multipartupload)(storageclasses) WebDAV ConcurrentOutput TorrentParser Feeds Quvi +dependency versions: aws-0.14.0 bloomfilter-2.0.1.0 cryptonite-0.7 DAV-1.3.1 feed-0.3.11.1 ghc-7.10.2 http-client-0.4.31.1 persistent-sqlite-2.2 torrent-10000.0.0 uuid-1.3.12 yesod +-1.4.3 +key/value backends: SHA256E SHA256 SHA512E SHA512 SHA224E SHA224 SHA384E SHA384 SHA3_256E SHA3_256 SHA3_512E SHA3_512 SHA3_224E SHA3_224 SHA3_384E SHA3_384 SKEIN256E SKEIN256 SKEIN +512E SKEIN512 SHA1E SHA1 MD5E MD5 WORM URL +remote types: git gcrypt p2p S3 bup directory rsync web bittorrent webdav tahoe glacier ddar hook external + +### Please provide any additional information below. + +[[!format sh """ + +PS G:\test2> git annex test +Tests + QuickCheck + prop_isomorphic_deencode_git: OK (0.11s) + +++ OK, passed 1000 tests. + prop_isomorphic_deencode: OK (0.11s) + +++ OK, passed 1000 tests. + prop_isomorphic_fileKey: OK (0.04s) + +++ OK, passed 1000 tests. + prop_isomorphic_key_encode: OK (0.06s) + +++ OK, passed 1000 tests. + prop_isomorphic_key_decode: OK (0.04s) + +++ OK, passed 1000 tests. + prop_isomorphic_shellEscape: OK (0.05s) + +++ OK, passed 1000 tests. + prop_isomorphic_shellEscape_multiword: OK (1.61s) + +++ OK, passed 1000 tests. + prop_isomorphic_configEscape: OK (0.04s) + +++ OK, passed 1000 tests. + prop_parse_show_Config: OK (0.11s) + +++ OK, passed 1000 tests. + prop_upFrom_basics: OK (0.04s) + +++ OK, passed 1000 tests. + prop_relPathDirToFile_basics: OK (0.09s) + +++ OK, passed 1000 tests. + prop_relPathDirToFile_regressionTest: OK + +++ OK, passed 1000 tests. + prop_cost_sane: OK + +++ OK, passed 1000 tests. + prop_matcher_sane: OK + +++ OK, passed 1000 tests. + prop_HmacSha1WithCipher_sane: OK + +++ OK, passed 1000 tests. + prop_TimeStamp_sane: OK + +++ OK, passed 1000 tests. + prop_addMapLog_sane: OK + +++ OK, passed 1000 tests. + prop_verifiable_sane: OK (0.14s) + +++ OK, passed 1000 tests. + prop_segment_regressionTest: OK + +++ OK, passed 1000 tests. + prop_read_write_transferinfo: OK (0.05s) + +++ OK, passed 1000 tests. + prop_read_show_inodecache: OK (0.02s) + +++ OK, passed 1000 tests. + prop_parse_show_log: OK (2.05s) + +++ OK, passed 1000 tests. + prop_read_show_TrustLevel: OK + +++ OK, passed 1000 tests. + prop_parse_show_TrustLog: OK + +++ OK, passed 1000 tests. + prop_hashes_stable: OK + +++ OK, passed 1000 tests. + prop_mac_stable: OK + +++ OK, passed 1000 tests. + prop_schedule_roundtrips: OK (0.02s) + +++ OK, passed 1000 tests. + prop_past_sane: OK + +++ OK, passed 1000 tests. + prop_duration_roundtrips: OK + +++ OK, passed 1000 tests. + prop_metadata_sane: OK (6.90s) + +++ OK, passed 1000 tests. + prop_metadata_serialize: OK (6.37s) + +++ OK, passed 1000 tests. + prop_branchView_legal: OK (10.22s) + +++ OK, passed 1000 tests. + prop_viewPath_roundtrips: OK (0.11s) + +++ OK, passed 1000 tests. + prop_view_roundtrips: OK (1.28s) + +++ OK, passed 1000 tests. + prop_viewedFile_rountrips: OK (0.04s) + +++ OK, passed 1000 tests. + prop_b64_roundtrips: OK + +++ OK, passed 1000 tests. + prop_standardGroups_parse: OKInit Tests + init: + +++ OK, passed 1000 tests. + Unit Tests v6 unlocked + add dup: Detected a filesystem without fifo support. + Disabling ssh connection caching. + Detected a crippled filesystem. + Entering an adjusted branch where files are unlocked as this filesystem does not support locked files. +Switched to branch 'adjusted/master(unlocked)' +OK (1.32s) + add: OK (1.12s) + +All 2 tests passed (2.44s) + Detected a filesystem without fifo support. + Disabling ssh connection caching. + Detected a crippled filesystem. +OK (1.33s) + add extras: Detected a filesystem without fifo support. + Disabling ssh connection caching. + Detected a crippled filesystem. +OK (1.46s) + shared clone: Detected a filesystem without fifo support. + Disabling ssh connection caching. + Detected a crippled filesystem. +OK (0.94s) + log: Detected a filesystem without fifo support. + Disabling ssh connection caching. + Detected a crippled filesystem. ++ Sun, 30 Jul 2017 02:07:24 Mitteleurop├Áische Sommerzeit foo | ccfb8066-6f33-425e-9459-7fda7a8b9117 -- test repo [origi +n] +OK (0.94s) + import: Detected a filesystem without fifo support. + Disabling ssh connection caching. + Detected a crippled filesystem. +git-annex: MoveFileEx "C:\\Users\\\\AppData\\Local\\Temp\\importtest.0\\import1\\f" "import1\\f": permission denie +d (Der Prozess kann nicht auf die Datei zugreifen, da sie von einem anderen Prozess verwendet wird.) +FAIL (4.98s) + import failed + reinject: Detected a filesystem without fifo support. + Disabling ssh connection caching. + Detected a crippled filesystem. + Detected a filesystem without fifo support. + Disabling ssh connection caching. + Detected a crippled filesystem. +git-annex: MoveFileEx "tmpfile" ".git\\annex\\objects\\ecb\\0b5\\SHA1-s25--ee80d2cec57a3810db83b80e1b320df3a3721ffa\\SHA +1-s25--ee80d2cec57a3810db83b80e1b320df3a3721ffa": permission denied (Der Prozess kann nicht auf die Datei zugreifen, da +sie von einem anderen Prozess verwendet wird.) +FAIL (3.55s) + reinject failed + unannex (no copy): Detected a filesystem without fifo support. + Disabling ssh connection caching. + Detected a crippled filesystem. +OK (1.20s) + unannex (with copy): Detected a filesystem without fifo support. + Disabling ssh connection caching. + Detected a crippled filesystem. +OK (1.51s) + drop (no remote): Detected a filesystem without fifo support. + Disabling ssh connection caching. + Detected a crippled filesystem. +OK (1.81s) + drop (with remote): Detected a filesystem without fifo support. + Disabling ssh connection caching. + Detected a crippled filesystem. +numcopies 2 ok +(recording state in git...) +numcopies 1 ok +(recording state in git...) +OK (2.31s) + drop (untrusted remote): Detected a filesystem without fifo support. + Disabling ssh connection caching. + Detected a crippled filesystem. +OK (1.73s) + get: Detected a filesystem without fifo support. + Disabling ssh connection caching. + Detected a crippled filesystem. +OK (1.86s) + get (ssh remote): Detected a filesystem without fifo support. + Disabling ssh connection caching. + Detected a crippled filesystem. +Der Befehl "git-annex-shell" ist entweder falsch geschrieben oder +konnte nicht gefunden werden. +rsync: safe_read failed to read 4 bytes [Receiver]: Connection reset by peer (104) +rsync error: error in rsync protocol data stream (code 12) at io.c(276) [Receiver=3.1.1] +FAIL (1.21s) + get of file failed + move: Detected a filesystem without fifo support. + Disabling ssh connection caching. + Detected a crippled filesystem. +OK (3.05s) + move (ssh remote): Detected a filesystem without fifo support. + Disabling ssh connection caching. + Detected a crippled filesystem. +Der Befehl "git-annex-shell" ist entweder falsch geschrieben oder +konnte nicht gefunden werden. +rsync: safe_read failed to read 4 bytes [Receiver]: Connection reset by peer (104) +rsync error: error in rsync protocol data stream (code 12) at io.c(276) [Receiver=3.1.1] +FAIL (1.19s) + move --from of file failed + copy: Detected a filesystem without fifo support. + Disabling ssh connection caching. + Detected a crippled filesystem. +OK (2.56s) + lock: Detected a filesystem without fifo support. + Disabling ssh connection caching. + Detected a crippled filesystem. + Detected a filesystem without fifo support. + Disabling ssh connection caching. + Detected a crippled filesystem. +OK (3.44s) + lock (v6 --force): Detected a filesystem without fifo support. + Disabling ssh connection caching. + Detected a crippled filesystem. + Detected a filesystem without fifo support. + Disabling ssh connection caching. + Detected a crippled filesystem. +FAIL + Exception: removeDirectoryRecursive: does not exist (Das System kann die angegebene Datei nicht finden.) + edit (no pre-commit): Detected a filesystem without fifo support. + Disabling ssh connection caching. + Detected a crippled filesystem. + Detected a filesystem without fifo support. + Disabling ssh connection caching. + Detected a crippled filesystem. +OK (2.22s) + edit (pre-commit): Detected a filesystem without fifo support. + Disabling ssh connection caching. + Detected a crippled filesystem. + Detected a filesystem without fifo support. + Disabling ssh connection caching. + Detected a crippled filesystem. +OK (2.07s) + partial commit: Detected a filesystem without fifo support. + Disabling ssh connection caching. + Detected a crippled filesystem. + Detected a filesystem without fifo support. + Disabling ssh connection caching. + Detected a crippled filesystem. +OK (2.03s) + fix: Detected a filesystem without fifo support. + Disabling ssh connection caching. + Detected a crippled filesystem. + Detected a filesystem without fifo support. + Disabling ssh connection caching. + Detected a crippled filesystem. +OK (1.11s) + direct: Detected a filesystem without fifo support. + Disabling ssh connection caching. + Detected a crippled filesystem. + Detected a filesystem without fifo support. + Disabling ssh connection caching. + Detected a crippled filesystem. +OK (1.57s) + trust: Detected a filesystem without fifo support. + Disabling ssh connection caching. + Detected a crippled filesystem. +OK (2.59s) + fsck (basics): Detected a filesystem without fifo support. + Disabling ssh connection caching. + Detected a crippled filesystem. +numcopies 2 ok +(recording state in git...) + Only 1 of 2 trustworthy copies exist of foo + Back it up with git-annex copy. + Only 1 of 2 trustworthy copies exist of sha1foo + Back it up with git-annex copy. +numcopies 1 ok +(recording state in git...) +OK (3.09s) + fsck (bare): Detected a filesystem without fifo support. + Disabling ssh connection caching. + Detected a crippled filesystem. +OK (1.45s) + fsck (local untrusted): Detected a filesystem without fifo support. + Disabling ssh connection caching. + Detected a crippled filesystem. + Only these untrusted locations may have copies of foo + 3a088e2e-d913-45c7-a6f4-187c6c23d52d -- .t\tmprepo26 [here] + ccfb8066-6f33-425e-9459-7fda7a8b9117 -- test repo [origin] + Back it up to trusted locations with git-annex copy. + Only these untrusted locations may have copies of sha1foo + ccfb8066-6f33-425e-9459-7fda7a8b9117 -- test repo [origin] + Back it up to trusted locations with git-annex copy. +OK (2.05s) + fsck (remote untrusted): Detected a filesystem without fifo support. + Disabling ssh connection caching. + Detected a crippled filesystem. +numcopies 2 ok +(recording state in git...) + Only 1 of 2 trustworthy copies exist of foo + Back it up with git-annex copy. + The following untrusted locations may also have copies: + ccfb8066-6f33-425e-9459-7fda7a8b9117 -- test repo [origin] + Only 1 of 2 trustworthy copies exist of sha1foo + Back it up with git-annex copy. + The following untrusted locations may also have copies: + ccfb8066-6f33-425e-9459-7fda7a8b9117 -- test repo [origin] +OK (2.21s) + fsck --from remote: Detected a filesystem without fifo support. + Disabling ssh connection caching. + Detected a crippled filesystem. +OK (1.38s) + migrate: Detected a filesystem without fifo support. + Disabling ssh connection caching. + Detected a crippled filesystem. + Detected a filesystem without fifo support. + Disabling ssh connection caching. + Detected a crippled filesystem. + MoveFileEx ".git\\annex\\misctmp\\t.0\\t" ".git\\annex\\objects\\6cd\\e82\\SHA256E-s20--e394a389d787383843decc5d3d99b6 +d184ffa5fddeec23b911f9ee7fc8b9ea77\\SHA256E-s20--e394a389d787383843decc5d3d99b6d184ffa5fddeec23b911f9ee7fc8b9ea77": perm +ission denied (Zugriff verweigert) +FAIL (2.41s) + migrate annexedfile failed + migrate (via gitattributes): Detected a filesystem without fifo support. + Disabling ssh connection caching. + Detected a crippled filesystem. + Detected a filesystem without fifo support. + Disabling ssh connection caching. + Detected a crippled filesystem. + MoveFileEx ".git\\annex\\misctmp\\t.0\\t" ".git\\annex\\objects\\6cd\\e82\\SHA256E-s20--e394a389d787383843decc5d3d99b6 +d184ffa5fddeec23b911f9ee7fc8b9ea77\\SHA256E-s20--e394a389d787383843decc5d3d99b6d184ffa5fddeec23b911f9ee7fc8b9ea77": perm +ission denied (Zugriff verweigert) +FAIL (2.56s) + migrate annexedfile failed + unused: Detected a filesystem without fifo support. + Disabling ssh connection caching. + Detected a crippled filesystem. + Detected a filesystem without fifo support. + Disabling ssh connection caching. + Detected a crippled filesystem. + bloomfilter capacity too large to represent; falling back to sane value +Unlink of file 'foo' failed. Should I try again? (y/n) y +Unlink of file 'foo' failed. Should I try again? (y/n) y +Unlink of file 'foo' failed. Should I try again? (y/n) y +Unlink of file 'foo' failed. Should I try again? (y/n) y +Unlink of file 'foo' failed. Should I try again? (y/n) y +Unlink of file 'foo' failed. Should I try again? (y/n) y +Unlink of file 'foo' failed. Should I try again? (y/n) n +fatal: git rm: 'foo': Invalid argument +FAIL (129.88s) + git rm failed + describe: Detected a filesystem without fifo support. + Disabling ssh connection caching. + Detected a crippled filesystem. +OK (1.83s) + find: Detected a filesystem without fifo support. + Disabling ssh connection caching. + Detected a crippled filesystem. +foo +foo +sha1foo +sha1foo +dir/subfile +OK (2.89s) + merge: Detected a filesystem without fifo support. + Disabling ssh connection caching. + Detected a crippled filesystem. +OK (0.96s) + info: Detected a filesystem without fifo support. + Disabling ssh connection caching. + Detected a crippled filesystem. + bloomfilter capacity too large to represent; falling back to sane value + bloomfilter capacity too large to represent; falling back to sane value +{"transfers in progress":[],"local annex keys":0,"available local disk space":"840.12 gigabytes (+1 megabyte reserved)", +"annexed files in working tree":2,"file":null,"trusted repositories":[],"size of annexed files in working tree":"45 byte +s","local annex size":"0 bytes","command":"info","untrusted repositories":[],"semitrusted repositories":[{"description": +"web","here":false,"uuid":"00000000-0000-0000-0000-000000000001"},{"description":"bittorrent","here":false,"uuid":"00000 +000-0000-0000-0000-000000000002"},{"description":".t\\tmprepo35","here":true,"uuid":"1493e4c5-cbc7-4571-9c87-696e2a91861 +b"},{"description":"test repo [origin]","here":false,"uuid":"ccfb8066-6f33-425e-9459-7fda7a8b9117"}],"success":true,"blo +om filter size":"32 mebibytes (0% full)","backend usage":{"SHA1":1,"SHA256E":1},"repository mode":"indirect"} +OK (1.13s) + version: Detected a filesystem without fifo support. + Disabling ssh connection caching. + Detected a crippled filesystem. +git-annex version: 6.20170611-gb493ac8 +build flags: Assistant Webapp Pairing Testsuite S3(multipartupload)(storageclasses) WebDAV ConcurrentOutput TorrentParse +r Feeds Quvi +dependency versions: aws-0.14.0 bloomfilter-2.0.1.0 cryptonite-0.7 DAV-1.3.1 feed-0.3.11.1 ghc-7.10.2 http-client-0.4.31 +.1 persistent-sqlite-2.2 torrent-10000.0.0 uuid-1.3.12 yesod-1.4.3 +key/value backends: SHA256E SHA256 SHA512E SHA512 SHA224E SHA224 SHA384E SHA384 SHA3_256E SHA3_256 SHA3_512E SHA3_512 SH +A3_224E SHA3_224 SHA3_384E SHA3_384 SKEIN256E SKEIN256 SKEIN512E SKEIN512 SHA1E SHA1 MD5E MD5 WORM URL +remote types: git gcrypt p2p S3 bup directory rsync web bittorrent webdav tahoe glacier ddar hook external +local repository version: 6 +supported repository versions: 3 5 6 +upgrade supported from repository versions: 2 3 4 5 +operating system: mingw32 i386 +OK (0.97s) + sync: Detected a filesystem without fifo support. + Disabling ssh connection caching. + Detected a crippled filesystem. +On branch adjusted/master(unlocked) +Your branch is up-to-date with 'origin/adjusted/master(unlocked)'. +nothing to commit, working tree clean +Counting objects: 10, done. +Delta compression using up to 4 threads. +Compressing objects: 100% (9/9), done. +Writing objects: 100% (10/10), 1.11 KiB | 0 bytes/s, done. +Total 10 (delta 2), reused 0 (delta 0) +To G:/test2/.t\repo + * [new branch] git-annex -> synced/git-annex + * [new branch] master -> synced/master +foo +wanted . ok +(recording state in git...) +On branch adjusted/master(unlocked) +Your branch and 'origin/adjusted/master(unlocked)' have diverged, +and have 2 and 2 different commits each, respectively. + (use "git pull" to merge the remote branch into yours) +nothing to commit, working tree clean +Counting objects: 8, done. +Delta compression using up to 4 threads. +Compressing objects: 100% (7/7), done. +Writing objects: 100% (8/8), 764 bytes | 0 bytes/s, done. +Total 8 (delta 2), reused 0 (delta 0) +To G:/test2/.t\repo + d94bd7a..4751eeb git-annex -> synced/git-annex +foo +On branch adjusted/master(unlocked) +Your branch and 'origin/adjusted/master(unlocked)' have diverged, +and have 2 and 2 different commits each, respectively. + (use "git pull" to merge the remote branch into yours) +nothing to commit, working tree clean +Counting objects: 5, done. +Delta compression using up to 4 threads. +Compressing objects: 100% (4/4), done. +Writing objects: 100% (5/5), 522 bytes | 0 bytes/s, done. +Total 5 (delta 1), reused 0 (delta 0) +To G:/test2/.t\repo + 4751eeb..68a9352 git-annex -> synced/git-annex +OK (4.66s) + union merge regression: Detected a filesystem without fifo support. + Disabling ssh connection caching. + Detected a crippled filesystem. + Detected a filesystem without fifo support. + Disabling ssh connection caching. + Detected a crippled filesystem. + Detected a filesystem without fifo support. + Disabling ssh connection caching. + Detected a crippled filesystem. +On branch adjusted/master(unlocked) +nothing to commit, working tree clean +remote: Counting objects: 11, done. +remote: Compressing objects: 100% (9/9), done. +remote: Total 11 (delta 3), reused 0 (delta 0) +Unpacking objects: 100% (11/11), done. +From ../../.t\tmprepo39 + * [new branch] adjusted/master(unlocked) -> r2/adjusted/master(unlocked) + * [new branch] git-annex -> r2/git-annex + * [new branch] master -> r2/master +remote: Counting objects: 9, done. +remote: Compressing objects: 100% (8/8), done. +remote: Total 9 (delta 3), reused 0 (delta 0) +Unpacking objects: 100% (9/9), done. +From ../../.t\tmprepo38 + * [new branch] adjusted/master(unlocked) -> r1/adjusted/master(unlocked) + * [new branch] git-annex -> r1/git-annex + * [new branch] master -> r1/master +Counting objects: 30, done. +Delta compression using up to 4 threads. +Compressing objects: 100% (26/26), done. +Writing objects: 100% (30/30), 2.87 KiB | 0 bytes/s, done. +Total 30 (delta 12), reused 0 (delta 0) +To ../../.t\tmprepo39 + * [new branch] git-annex -> synced/git-annex + * [new branch] master -> synced/master +To ../../.t\tmprepo39 + ! [rejected] master -> master (non-fast-forward) +error: failed to push some refs to '../../.t\tmprepo39' +hint: Updates were rejected because a pushed branch tip is behind its remote +hint: counterpart. Check out this branch and integrate the remote changes +hint: (e.g. 'git pull ...') before pushing again. +hint: See the 'Note about fast-forwards' in 'git push --help' for details. +Counting objects: 30, done. +Delta compression using up to 4 threads. +Compressing objects: 100% (26/26), done. +Writing objects: 100% (30/30), 2.83 KiB | 0 bytes/s, done. +Total 30 (delta 13), reused 0 (delta 0) +To ../../.t\tmprepo38 + * [new branch] git-annex -> synced/git-annex + * [new branch] master -> synced/master +To ../../.t\tmprepo38 + ! [rejected] master -> master (non-fast-forward) +error: failed to push some refs to '../../.t\tmprepo38' +hint: Updates were rejected because a pushed branch tip is behind its remote +hint: counterpart. Check out this branch and integrate the remote changes +hint: (e.g. 'git pull ...') before pushing again. +hint: See the 'Note about fast-forwards' in 'git push --help' for details. +On branch adjusted/master(unlocked) +nothing to commit, working tree clean +Auto-merging sha1foo +CONFLICT (add/add): Merge conflict in sha1foo +Auto-merging foo +CONFLICT (add/add): Merge conflict in foo +Automatic merge failed; fix conflicts and then commit the result. +[detached HEAD cd6b062] git-annex automatic merge conflict fix +Updating 75480f7..672d8f6 +error: Your local changes to the following files would be overwritten by merge: + foo +Please commit your changes or stash them before you merge. +Aborting +remote: Counting objects: 4, done. +remote: Compressing objects: 100% (4/4), done. +remote: Total 4 (delta 0), reused 0 (delta 0) +Unpacking objects: 100% (4/4), done. +From ../../.t\tmprepo40 + * [new branch] adjusted/master(unlocked) -> r3/adjusted/master(unlocked) + * [new branch] git-annex -> r3/git-annex + * [new branch] master -> r3/master + * [new branch] synced/master -> r3/synced/master +From ../../.t\tmprepo38 + * [new branch] adjusted/master(unlocked) -> r1/adjusted/master(unlocked) + * [new branch] git-annex -> r1/git-annex + * [new branch] master -> r1/master + * [new branch] synced/git-annex -> r1/synced/git-annex + * [new branch] synced/master -> r1/synced/master +Merge made by the 'recursive' strategy. + bar.c | 1 - + foo | 1 - + sha1foo | 1 - + 3 files changed, 3 deletions(-) + delete mode 100644 bar.c + delete mode 120000 foo + delete mode 120000 sha1foo +Updating 75480f7..2773470 +error: Your local changes to the following files would be overwritten by merge: + foo +Please commit your changes or stash them before you merge. +Aborting +Counting objects: 9, done. +Delta compression using up to 4 threads. +Compressing objects: 100% (9/9), done. +Writing objects: 100% (9/9), 954 bytes | 0 bytes/s, done. +Total 9 (delta 5), reused 0 (delta 0) +To ../../.t\tmprepo40 + 9588f09..b8df995 master -> synced/master + * [new branch] git-annex -> synced/git-annex +Counting objects: 9, done. +Delta compression using up to 4 threads. +Compressing objects: 100% (9/9), done. +Writing objects: 100% (9/9), 954 bytes | 0 bytes/s, done. +Total 9 (delta 5), reused 0 (delta 0) +To ../../.t\tmprepo38 + 9588f09..b8df995 master -> synced/master +FAIL (8.41s) + sync failed in .t\tmprepo39 + adjusted branch merge regression: Detected a filesystem without fifo support. + Disabling ssh connection caching. + Detected a crippled filesystem. + Detected a filesystem without fifo support. + Disabling ssh connection caching. + Detected a crippled filesystem. +Already on 'adjusted/master(unlocked)' +M foo +[adjusted/master(unlocked) 22ff3ea] git-annex in .t\tmprepo41 + 2 files changed, 2 insertions(+), 1 deletion(-) + create mode 100644 conflictor +remote: Counting objects: 11, done. +remote: Compressing objects: 100% (10/10), done. +remote: Total 11 (delta 3), reused 0 (delta 0) +Unpacking objects: 100% (11/11), done. +From ../../.t\tmprepo42 + * [new branch] adjusted/master(unlocked) -> r2/adjusted/master(unlocked) + * [new branch] git-annex -> r2/git-annex + * [new branch] master -> r2/master +Counting objects: 18, done. +Delta compression using up to 4 threads. +Compressing objects: 100% (16/16), done. +Writing objects: 100% (18/18), 1.72 KiB | 0 bytes/s, done. +Total 18 (delta 7), reused 0 (delta 0) +To ../../.t\tmprepo42 + * [new branch] git-annex -> synced/git-annex + * [new branch] master -> synced/master +Already on 'adjusted/master(unlocked)' +D conflictor +M foo +[adjusted/master(unlocked) b0c16b3] git-annex in .t\tmprepo42 + 2 files changed, 2 insertions(+), 2 deletions(-) +remote: Counting objects: 5, done. +remote: Compressing objects: 100% (5/5), done. +remote: Total 5 (delta 0), reused 0 (delta 0) +Unpacking objects: 100% (5/5), done. +From ../../.t\tmprepo41 + * [new branch] adjusted/master(unlocked) -> r1/adjusted/master(unlocked) + * [new branch] git-annex -> r1/git-annex + * [new branch] master -> r1/master + * [new branch] synced/master -> r1/synced/master +Counting objects: 9, done. +Delta compression using up to 4 threads. +Compressing objects: 100% (8/8), done. +Writing objects: 100% (9/9), 806 bytes | 0 bytes/s, done. +Total 9 (delta 4), reused 0 (delta 0) +To ../../.t\tmprepo41 + 6f2f330..2f4c792 master -> synced/master + * [new branch] git-annex -> synced/git-annex +On branch adjusted/master(unlocked) +nothing to commit, working tree clean +Updating 6f2f330..2f4c792 +Fast-forward + conflictor | 2 +- + foo | 2 +- + 2 files changed, 2 insertions(+), 2 deletions(-) +Updating a6b8bec..222127f +Fast-forward + conflictor | 2 +- + foo | 2 +- + 2 files changed, 2 insertions(+), 2 deletions(-) +remote: Counting objects: 5, done. +remote: Compressing objects: 100% (5/5), done. +remote: Total 5 (delta 0), reused 0 (delta 0) +Unpacking objects: 100% (5/5), done. +From ../../.t\tmprepo42 + + 75480f7...62cd69f adjusted/master(unlocked) -> r2/adjusted/master(unlocked) (forced update) + 14acc87..16ac4ee git-annex -> r2/git-annex + 6f2f330..2f4c792 master -> r2/master + 6f2f330..2f4c792 synced/master -> r2/synced/master +On branch adjusted/master(unlocked) +nothing to commit, working tree clean +remote: Counting objects: 5, done. +remote: Compressing objects: 100% (5/5), done. +remote: Total 5 (delta 0), reused 0 (delta 0) +Unpacking objects: 100% (5/5), done. +From ../../.t\tmprepo41 + + a6b8bec...29bc72f adjusted/master(unlocked) -> r1/adjusted/master(unlocked) (forced update) +OK (7.19s) + adjusted branch subtree regression: Detected a filesystem without fifo support. + Disabling ssh connection caching. + Detected a crippled filesystem. +Already on 'adjusted/master(unlocked)' +M foo +[adjusted/master(unlocked) dd3d179] git-annex in .t\tmprepo43 + 2 files changed, 2 insertions(+), 1 deletion(-) + create mode 100644 a/b/c/d +[adjusted/master(unlocked) 543a653] git-annex in .t\tmprepo43 + 1 file changed, 1 insertion(+) + create mode 100644 a/b/x/y +Switched to branch 'master' +OK (2.58s) + conflict resolution: Detected a filesystem without fifo support. + Disabling ssh connection caching. + Detected a crippled filesystem. + Detected a filesystem without fifo support. + Disabling ssh connection caching. + Detected a crippled filesystem. +[adjusted/master(unlocked) 5cb38e6] git-annex in .t\tmprepo44 + 1 file changed, 1 insertion(+) + create mode 100644 conflictor +[adjusted/master(unlocked) dbe9298] git-annex in .t\tmprepo45 + 1 file changed, 1 insertion(+) + create mode 100644 conflictor +On branch adjusted/master(unlocked) +nothing to commit, working tree clean +remote: Counting objects: 23, done. +remote: Compressing objects: 100% (21/21), done. +remote: Total 23 (delta 5), reused 0 (delta 0) +Unpacking objects: 100% (23/23), done. +From ../../.t\tmprepo45 + * [new branch] adjusted/master(unlocked) -> r2/adjusted/master(unlocked) + * [new branch] git-annex -> r2/git-annex + * [new branch] master -> r2/master + * [new branch] synced/master -> r2/synced/master +Auto-merging sha1foo +CONFLICT (add/add): Merge conflict in sha1foo +Auto-merging foo +CONFLICT (add/add): Merge conflict in foo +Auto-merging conflictor +CONFLICT (add/add): Merge conflict in conflictor +Automatic merge failed; fix conflicts and then commit the result. +conflictor: needs merge +[detached HEAD 106cd9d] git-annex automatic merge conflict fix +Updating 085b7b5..5a26f64 +Fast-forward + conflictor.variant-75dc | 1 + + conflictor => conflictor.variant-a507 | 2 +- + foo | 2 +- + 3 files changed, 3 insertions(+), 2 deletions(-) + create mode 100644 conflictor.variant-75dc + rename conflictor => conflictor.variant-a507 (98%) +Counting objects: 23, done. +Delta compression using up to 4 threads. +Compressing objects: 100% (22/22), done. +Writing objects: 100% (23/23), 2.27 KiB | 0 bytes/s, done. +Total 23 (delta 10), reused 0 (delta 0) +To ../../.t\tmprepo45 + b9ebb44..106cd9d master -> synced/master + * [new branch] git-annex -> synced/git-annex +On branch adjusted/master(unlocked) +nothing to commit, working tree clean +Updating b9ebb44..106cd9d +Fast-forward + conflictor | 1 - + conflictor.variant-75dc | 1 + + conflictor.variant-a507 | 1 + + foo | 2 +- + sha1foo | 2 +- + 5 files changed, 4 insertions(+), 3 deletions(-) + delete mode 120000 conflictor + create mode 120000 conflictor.variant-75dc + create mode 120000 conflictor.variant-a507 +Updating 834afb8..a855bbb +Fast-forward + conflictor => conflictor.variant-75dc | 2 +- + conflictor.variant-a507 | 1 + + foo | 2 +- + 3 files changed, 3 insertions(+), 2 deletions(-) + rename conflictor => conflictor.variant-75dc (98%) + create mode 100644 conflictor.variant-a507 +From ../../.t\tmprepo44 + * [new branch] adjusted/master(unlocked) -> r1/adjusted/master(unlocked) + * [new branch] git-annex -> r1/git-annex + * [new branch] master -> r1/master + * [new branch] synced/master -> r1/synced/master +[adjusted/master(unlocked) a544d12] git-annex in .t\tmprepo44 + 1 file changed, 1 insertion(+), 1 deletion(-) +From ../../.t\tmprepo45 + + 834afb8...ba3658e adjusted/master(unlocked) -> r2/adjusted/master(unlocked) (forced update) +Counting objects: 1, done. +Writing objects: 100% (1/1), 191 bytes | 0 bytes/s, done. +Total 1 (delta 0), reused 0 (delta 0) +To ../../.t\tmprepo45 + 106cd9d..55808e6 master -> synced/master +conflictor.variant-a507 +conflictor.variant-75dc +conflictor.variant-a507 +conflictor.variant-75dc +OK (8.56s) + conflict resolution (adjusted branch): Detected a filesystem without fifo support. + Disabling ssh connection caching. + Detected a crippled filesystem. + Detected a filesystem without fifo support. + Disabling ssh connection caching. + Detected a crippled filesystem. +[adjusted/master(unlocked) c828dc1] git-annex in .t\tmprepo46 + 1 file changed, 1 insertion(+) + create mode 100644 conflictor +[adjusted/master(unlocked) f2a9fab] git-annex in .t\tmprepo47 + 1 file changed, 1 insertion(+) + create mode 100644 conflictor +Already on 'adjusted/master(unlocked)' +M conflictor +M foo +On branch adjusted/master(unlocked) +nothing to commit, working tree clean +remote: Counting objects: 23, done. +remote: Compressing objects: 100% (21/21), done. +remote: Total 23 (delta 5), reused 0 (delta 0) +Unpacking objects: 100% (23/23), done. +From ../../.t\tmprepo47 + * [new branch] adjusted/master(unlocked) -> r2/adjusted/master(unlocked) + * [new branch] git-annex -> r2/git-annex + * [new branch] master -> r2/master + * [new branch] synced/master -> r2/synced/master +Auto-merging sha1foo +CONFLICT (add/add): Merge conflict in sha1foo +Auto-merging foo +CONFLICT (add/add): Merge conflict in foo +Auto-merging conflictor +CONFLICT (add/add): Merge conflict in conflictor +Automatic merge failed; fix conflicts and then commit the result. +conflictor: needs merge +[detached HEAD 5c75014] git-annex automatic merge conflict fix +Updating 21164f0..40932a6 +Fast-forward + conflictor.variant-75dc | 1 + + conflictor => conflictor.variant-a507 | 2 +- + foo | 2 +- + 3 files changed, 3 insertions(+), 2 deletions(-) + create mode 100644 conflictor.variant-75dc + rename conflictor => conflictor.variant-a507 (98%) +Counting objects: 23, done. +Delta compression using up to 4 threads. +Compressing objects: 100% (22/22), done. +Writing objects: 100% (23/23), 2.31 KiB | 0 bytes/s, done. +Total 23 (delta 9), reused 0 (delta 0) +To ../../.t\tmprepo47 + f48a2ad..5c75014 master -> synced/master + * [new branch] git-annex -> synced/git-annex +[adjusted/master(unlocked) 5582940] git-annex in .t\tmprepo47 + 2 files changed, 2 insertions(+), 2 deletions(-) +Removing conflictor +Merge made by the 'recursive' strategy. + conflictor | 1 - + conflictor.variant-75dc | 1 + + conflictor.variant-a507 | 1 + + foo | 2 +- + sha1foo | 2 +- + 5 files changed, 4 insertions(+), 3 deletions(-) + delete mode 120000 conflictor + create mode 120000 conflictor.variant-75dc + create mode 120000 conflictor.variant-a507 +Updating 5582940..95b64c3 +Fast-forward + conflictor => conflictor.variant-75dc | 2 +- + conflictor.variant-a507 | 1 + + foo | 2 +- + 3 files changed, 3 insertions(+), 2 deletions(-) + rename conflictor => conflictor.variant-75dc (98%) + create mode 100644 conflictor.variant-a507 +remote: Counting objects: 6, done. +remote: Compressing objects: 100% (6/6), done. +remote: Total 6 (delta 0), reused 0 (delta 0) +Unpacking objects: 100% (6/6), done. +From ../../.t\tmprepo46 + * [new branch] adjusted/master(unlocked) -> r1/adjusted/master(unlocked) + * [new branch] git-annex -> r1/git-annex + * [new branch] master -> r1/master + * [new branch] synced/master -> r1/synced/master +Counting objects: 2, done. +Delta compression using up to 4 threads. +Compressing objects: 100% (2/2), done. +Writing objects: 100% (2/2), 407 bytes | 0 bytes/s, done. +Total 2 (delta 0), reused 0 (delta 0) +To ../../.t\tmprepo46 + 5c75014..0be7ee2 master -> synced/master + * [new branch] git-annex -> synced/git-annex +[adjusted/master(unlocked) e579d2b] git-annex in .t\tmprepo46 + 1 file changed, 1 insertion(+), 1 deletion(-) +Already up-to-date! +Merge made by the 'recursive' strategy. +Updating e579d2b..5dc5ec6 +Fast-forward + conflictor.variant-a507 | 2 +- + 1 file changed, 1 insertion(+), 1 deletion(-) +remote: Counting objects: 6, done. +remote: Compressing objects: 100% (6/6), done. +remote: Total 6 (delta 0), reused 0 (delta 0) +Unpacking objects: 100% (6/6), done. +From ../../.t\tmprepo47 + + 844c7ad...1d42500 adjusted/master(unlocked) -> r2/adjusted/master(unlocked) (forced update) + 5c75014..0be7ee2 master -> r2/master + 5c75014..0be7ee2 synced/master -> r2/synced/master +Counting objects: 2, done. +Delta compression using up to 4 threads. +Compressing objects: 100% (2/2), done. +Writing objects: 100% (2/2), 334 bytes | 0 bytes/s, done. +Total 2 (delta 1), reused 0 (delta 0) +To ../../.t\tmprepo47 + 0be7ee2..b481b8e master -> synced/master +conflictor.variant-a507 +conflictor.variant-75dc +conflictor.variant-a507 +conflictor.variant-75dc +OK (9.46s) + conflict resolution movein regression: Detected a filesystem without fifo support. + Disabling ssh connection caching. + Detected a crippled filesystem. + Detected a filesystem without fifo support. + Disabling ssh connection caching. + Detected a crippled filesystem. +[adjusted/master(unlocked) 98268ad] git-annex in .t\tmprepo48 + 2 files changed, 2 insertions(+), 2 deletions(-) +remote: Counting objects: 14, done. +remote: Compressing objects: 100% (11/11), done. +remote: Total 14 (delta 4), reused 0 (delta 0) +Unpacking objects: 100% (14/14), done. +From ../../.t\tmprepo49 + * [new branch] adjusted/master(unlocked) -> r2/adjusted/master(unlocked) + * [new branch] git-annex -> r2/git-annex + * [new branch] master -> r2/master +Counting objects: 33, done. +Delta compression using up to 4 threads. +Compressing objects: 100% (27/27), done. +Writing objects: 100% (33/33), 3.04 KiB | 0 bytes/s, done. +Total 33 (delta 9), reused 0 (delta 0) +To ../../.t\tmprepo49 + * [new branch] git-annex -> synced/git-annex + * [new branch] master -> synced/master +To ../../.t\tmprepo49 + ! [rejected] master -> master (non-fast-forward) +error: failed to push some refs to '../../.t\tmprepo49' +hint: Updates were rejected because a pushed branch tip is behind its remote +hint: counterpart. Check out this branch and integrate the remote changes +hint: (e.g. 'git pull ...') before pushing again. +hint: See the 'Note about fast-forwards' in 'git push --help' for details. +[adjusted/master(unlocked) 4839b05] git-annex in .t\tmprepo49 + 2 files changed, 2 insertions(+), 2 deletions(-) +Auto-merging sha1foo +CONFLICT (add/add): Merge conflict in sha1foo +Auto-merging foo +CONFLICT (add/add): Merge conflict in foo +Automatic merge failed; fix conflicts and then commit the result. +foo: needs merge +[detached HEAD 4d9c158] git-annex automatic merge conflict fix +Updating 4839b05..941b7f7 +Fast-forward + foo.variant-0b0e | 1 + + foo => foo.variant-bc25 | 2 +- + sha1foo | 2 +- + 3 files changed, 3 insertions(+), 2 deletions(-) + create mode 100644 foo.variant-0b0e + rename foo => foo.variant-bc25 (98%) +remote: Counting objects: 4, done. +remote: Compressing objects: 100% (4/4), done. +remote: Total 4 (delta 0), reused 0 (delta 0) +Unpacking objects: 100% (4/4), done. +From ../../.t\tmprepo48 + * [new branch] adjusted/master(unlocked) -> r1/adjusted/master(unlocked) + * [new branch] git-annex -> r1/git-annex + * [new branch] master -> r1/master + * [new branch] synced/master -> r1/synced/master +Counting objects: 10, done. +Delta compression using up to 4 threads. +Compressing objects: 100% (10/10), done. +Writing objects: 100% (10/10), 1.18 KiB | 0 bytes/s, done. +Total 10 (delta 2), reused 0 (delta 0) +To ../../.t\tmprepo48 + dcae980..4d9c158 master -> synced/master + * [new branch] git-annex -> synced/git-annex +On branch adjusted/master(unlocked) +nothing to commit, working tree clean +Updating dcae980..4d9c158 +Fast-forward + foo | 1 - + foo.variant-0b0e | 1 + + foo.variant-bc25 | 1 + + sha1foo | 2 +- + 4 files changed, 3 insertions(+), 2 deletions(-) + delete mode 120000 foo + create mode 120000 foo.variant-0b0e + create mode 120000 foo.variant-bc25 +Updating 93aeb67..c5dd322 +Fast-forward + foo => foo.variant-0b0e | 2 +- + foo.variant-bc25 | 1 + + sha1foo | 2 +- + 3 files changed, 3 insertions(+), 2 deletions(-) + rename foo => foo.variant-0b0e (98%) + create mode 100644 foo.variant-bc25 +remote: Counting objects: 5, done. +remote: Compressing objects: 100% (4/4), done. +remote: Total 5 (delta 1), reused 0 (delta 0) +Unpacking objects: 100% (5/5), done. +From ../../.t\tmprepo49 + + 75480f7...834d96c adjusted/master(unlocked) -> r2/adjusted/master(unlocked) (forced update) + 2411485..288a846 git-annex -> r2/git-annex + + 685e9b5...4d9c158 master -> r2/master (forced update) + dcae980..4d9c158 synced/master -> r2/synced/master +OK (8.20s) + conflict resolution (mixed directory and file): Detected a filesystem without fifo support. + Disabling ssh connection caching. + Detected a crippled filesystem. + Detected a filesystem without fifo support. + Disabling ssh connection caching. + Detected a crippled filesystem. +[adjusted/master(unlocked) 77f9c58] git-annex in .t\tmprepo50 + 1 file changed, 1 insertion(+) + create mode 100644 conflictor +[adjusted/master(unlocked) e1fa8b8] git-annex in .t\tmprepo51 + 1 file changed, 1 insertion(+) + create mode 100644 conflictor/subfile +On branch adjusted/master(unlocked) +nothing to commit, working tree clean +remote: Counting objects: 25, done. +remote: Compressing objects: 100% (21/21), done. +remote: Total 25 (delta 4), reused 0 (delta 0) +Unpacking objects: 100% (25/25), done. +From ../../.t\tmprepo51 + * [new branch] adjusted/master(unlocked) -> r2/adjusted/master(unlocked) + * [new branch] git-annex -> r2/git-annex + * [new branch] master -> r2/master + * [new branch] synced/master -> r2/synced/master +Auto-merging sha1foo +CONFLICT (add/add): Merge conflict in sha1foo +Auto-merging foo +CONFLICT (add/add): Merge conflict in foo +Adding conflictor/subfile +CONFLICT (file/directory): There is a directory with name conflictor in refs/remotes/r2/master. Adding conflictor as con +flictor~HEAD +Automatic merge failed; fix conflicts and then commit the result. +conflictor: needs merge +[detached HEAD 724aad5] git-annex automatic merge conflict fix +Updating 7ceb7ad..1c51b66 +Fast-forward + conflictor => conflictor.variant-cc12 | 2 +- + conflictor/subfile | 1 + + foo | 2 +- + 3 files changed, 3 insertions(+), 2 deletions(-) + rename conflictor => conflictor.variant-cc12 (98%) + create mode 100644 conflictor/subfile +Counting objects: 22, done. +Delta compression using up to 4 threads. +Compressing objects: 100% (21/21), done. +Writing objects: 100% (22/22), 2.15 KiB | 0 bytes/s, done. +Total 22 (delta 10), reused 0 (delta 0) +To ../../.t\tmprepo51 + f4b3387..724aad5 master -> synced/master + * [new branch] git-annex -> synced/git-annex +On branch adjusted/master(unlocked) +nothing to commit, working tree clean +Updating f4b3387..724aad5 +Fast-forward + conflictor.variant-cc12 | 1 + + foo | 2 +- + sha1foo | 2 +- + 3 files changed, 3 insertions(+), 2 deletions(-) + create mode 120000 conflictor.variant-cc12 +Updating 98b13e3..6ab96da +Fast-forward + conflictor.variant-cc12 | 1 + + conflictor/subfile | 2 +- + foo | 2 +- + 3 files changed, 3 insertions(+), 2 deletions(-) + create mode 100644 conflictor.variant-cc12 +From ../../.t\tmprepo50 + * [new branch] adjusted/master(unlocked) -> r1/adjusted/master(unlocked) + * [new branch] git-annex -> r1/git-annex + * [new branch] master -> r1/master + * [new branch] synced/master -> r1/synced/master +conflictor/subfile +conflictor.variant-cc12 +conflictor/subfile +conflictor.variant-cc12 + Detected a filesystem without fifo support. + Disabling ssh connection caching. + Detected a crippled filesystem. + Detected a filesystem without fifo support. + Disabling ssh connection caching. + Detected a crippled filesystem. +[adjusted/master(unlocked) de2bac8] git-annex in .t\tmprepo52 + 1 file changed, 1 insertion(+) + create mode 100644 conflictor +[adjusted/master(unlocked) eb3a2f9] git-annex in .t\tmprepo53 + 1 file changed, 1 insertion(+) + create mode 100644 conflictor/subfile +On branch adjusted/master(unlocked) +nothing to commit, working tree clean +remote: Counting objects: 23, done. +remote: Compressing objects: 100% (21/21), done. +remote: Total 23 (delta 3), reused 0 (delta 0) +Unpacking objects: 100% (23/23), done. +From ../../.t\tmprepo52 + * [new branch] adjusted/master(unlocked) -> r1/adjusted/master(unlocked) + * [new branch] git-annex -> r1/git-annex + * [new branch] master -> r1/master + * [new branch] synced/master -> r1/synced/master +Auto-merging sha1foo +CONFLICT (add/add): Merge conflict in sha1foo +Auto-merging foo +CONFLICT (add/add): Merge conflict in foo +Adding conflictor/subfile +CONFLICT (directory/file): There is a directory with name conflictor in HEAD. Adding conflictor as conflictor~refs_remot +es_r1_master +Automatic merge failed; fix conflicts and then commit the result. +conflictor: needs merge +[detached HEAD 80b8489] git-annex automatic merge conflict fix +Updating f4714af..8c0e1af +Fast-forward + conflictor.variant-cc12 | 1 + + conflictor/subfile | 2 +- + foo | 2 +- + 3 files changed, 3 insertions(+), 2 deletions(-) + create mode 100644 conflictor.variant-cc12 +Counting objects: 24, done. +Delta compression using up to 4 threads. +Compressing objects: 100% (22/22), done. +Writing objects: 100% (24/24), 2.34 KiB | 0 bytes/s, done. +Total 24 (delta 9), reused 0 (delta 0) +To ../../.t\tmprepo52 + c785d3b..80b8489 master -> synced/master + * [new branch] git-annex -> synced/git-annex +On branch adjusted/master(unlocked) +nothing to commit, working tree clean +Updating c785d3b..80b8489 +Fast-forward + conflictor | 1 - + conflictor.variant-cc12 | 1 + + conflictor/subfile | 1 + + foo | 2 +- + sha1foo | 2 +- + 5 files changed, 4 insertions(+), 3 deletions(-) + delete mode 120000 conflictor + create mode 120000 conflictor.variant-cc12 + create mode 120000 conflictor/subfile +Updating f9dc04b..83e640a +Fast-forward + conflictor => conflictor.variant-cc12 | 2 +- + conflictor/subfile | 1 + + foo | 2 +- + 3 files changed, 3 insertions(+), 2 deletions(-) + rename conflictor => conflictor.variant-cc12 (98%) + create mode 100644 conflictor/subfile +From ../../.t\tmprepo53 + * [new branch] adjusted/master(unlocked) -> r2/adjusted/master(unlocked) + * [new branch] git-annex -> r2/git-annex + * [new branch] master -> r2/master + * [new branch] synced/master -> r2/synced/master +conflictor/subfile +conflictor.variant-cc12 +conflictor/subfile +conflictor.variant-cc12 +OK (15.63s) + conflict resolution symlink bit: OK + conflict resolution (uncommitted local file): Detected a filesystem without fifo support. + Disabling ssh connection caching. + Detected a crippled filesystem. + Detected a filesystem without fifo support. + Disabling ssh connection caching. + Detected a crippled filesystem. +[adjusted/master(unlocked) 4cfaea2] git-annex in .t\tmprepo54 + 1 file changed, 1 insertion(+) + create mode 100644 conflictor +On branch adjusted/master(unlocked) +Untracked files: + conflictor + +nothing added to commit but untracked files present +remote: Counting objects: 23, done. +remote: Compressing objects: 100% (21/21), done. +remote: Total 23 (delta 5), reused 0 (delta 0) +Unpacking objects: 100% (23/23), done. +From ../../.t\tmprepo54 + * [new branch] adjusted/master(unlocked) -> r1/adjusted/master(unlocked) + * [new branch] git-annex -> r1/git-annex + * [new branch] master -> r1/master + * [new branch] synced/master -> r1/synced/master +Auto-merging sha1foo +CONFLICT (add/add): Merge conflict in sha1foo +Auto-merging foo +CONFLICT (add/add): Merge conflict in foo +Automatic merge failed; fix conflicts and then commit the result. +[detached HEAD 2a880d5] git-annex automatic merge conflict fix +Updating 75480f7..c84a79d +error: The following untracked working tree files would be overwritten by merge: + conflictor +Please move or remove them before you merge. +Aborting +Already up-to-date. +Updating 75480f7..4acfac6 +error: The following untracked working tree files would be overwritten by merge: + conflictor +Please move or remove them before you merge. +Aborting +Counting objects: 18, done. +Delta compression using up to 4 threads. +Compressing objects: 100% (18/18), done. +Writing objects: 100% (18/18), 1.77 KiB | 0 bytes/s, done. +Total 18 (delta 10), reused 0 (delta 0) +To ../../.t\tmprepo54 + bda2c4e..5fd3f84 master -> synced/master + * [new branch] git-annex -> synced/git-annex + Detected a filesystem without fifo support. + Disabling ssh connection caching. + Detected a crippled filesystem. + Detected a filesystem without fifo support. + Disabling ssh connection caching. + Detected a crippled filesystem. +[adjusted/master(unlocked) a7ba016] git-annex in .t\tmprepo56 + 1 file changed, 1 insertion(+) + create mode 100644 conflictor/file +On branch adjusted/master(unlocked) +Untracked files: + conflictor + +nothing added to commit but untracked files present +remote: Counting objects: 25, done. +remote: Compressing objects: 100% (21/21), done. +remote: Total 25 (delta 3), reused 0 (delta 0) +Unpacking objects: 100% (25/25), done. +From ../../.t\tmprepo56 + * [new branch] adjusted/master(unlocked) -> r1/adjusted/master(unlocked) + * [new branch] git-annex -> r1/git-annex + * [new branch] master -> r1/master + * [new branch] synced/master -> r1/synced/master +Auto-merging sha1foo +CONFLICT (add/add): Merge conflict in sha1foo +Auto-merging foo +CONFLICT (add/add): Merge conflict in foo +Automatic merge failed; fix conflicts and then commit the result. +[detached HEAD f13554c] git-annex automatic merge conflict fix +Updating 75480f7..01da6d6 +error: The following untracked working tree files would be overwritten by merge: + conflictor +Please move or remove them before you merge. +Aborting +Already up-to-date. +Updating 75480f7..2b83f70 +error: The following untracked working tree files would be overwritten by merge: + conflictor +Please move or remove them before you merge. +Aborting +Counting objects: 18, done. +Delta compression using up to 4 threads. +Compressing objects: 100% (18/18), done. +Writing objects: 100% (18/18), 1.76 KiB | 0 bytes/s, done. +Total 18 (delta 10), reused 0 (delta 0) +To ../../.t\tmprepo56 + 363622c..0103a78 master -> synced/master + * [new branch] git-annex -> synced/git-annex +OK (10.59s) + conflict resolution (removed file): Detected a filesystem without fifo support. + Disabling ssh connection caching. + Detected a crippled filesystem. + Detected a filesystem without fifo support. + Disabling ssh connection caching. + Detected a crippled filesystem. +[adjusted/master(unlocked) e7f1054] git-annex in .t\tmprepo58 + 1 file changed, 1 insertion(+) + create mode 100644 conflictor +On branch adjusted/master(unlocked) +nothing to commit, working tree clean +remote: Counting objects: 23, done. +remote: Compressing objects: 100% (21/21), done. +remote: Total 23 (delta 4), reused 0 (delta 0) +Unpacking objects: 100% (23/23), done. +From ../../.t\tmprepo58 + * [new branch] adjusted/master(unlocked) -> r1/adjusted/master(unlocked) + * [new branch] git-annex -> r1/git-annex + * [new branch] master -> r1/master + * [new branch] synced/master -> r1/synced/master +Auto-merging sha1foo +CONFLICT (add/add): Merge conflict in sha1foo +Auto-merging foo +CONFLICT (add/add): Merge conflict in foo +Automatic merge failed; fix conflicts and then commit the result. +[detached HEAD 55cc242] git-annex automatic merge conflict fix +Updating 75480f7..bfe5776 +Fast-forward + conflictor | 1 + + foo | 2 +- + 2 files changed, 2 insertions(+), 1 deletion(-) + create mode 100644 conflictor +Counting objects: 14, done. +Delta compression using up to 4 threads. +Compressing objects: 100% (14/14), done. +Writing objects: 100% (14/14), 1.42 KiB | 0 bytes/s, done. +Total 14 (delta 8), reused 0 (delta 0) +To ../../.t\tmprepo58 + 34511ad..55cc242 master -> synced/master + * [new branch] git-annex -> synced/git-annex +[adjusted/master(unlocked) 937a71e] git-annex in .t\tmprepo58 + 1 file changed, 1 deletion(-) + delete mode 100644 conflictor +Merge made by the 'recursive' strategy. + foo | 2 +- + sha1foo | 2 +- + 2 files changed, 2 insertions(+), 2 deletions(-) +Updating 937a71e..e992316 +Fast-forward + foo | 2 +- + 1 file changed, 1 insertion(+), 1 deletion(-) +remote: Counting objects: 10, done. +remote: Compressing objects: 100% (9/9), done. +remote: Total 10 (delta 2), reused 0 (delta 0) +Unpacking objects: 100% (10/10), done. +From ../../.t\tmprepo59 + * [new branch] adjusted/master(unlocked) -> r2/adjusted/master(unlocked) + * [new branch] git-annex -> r2/git-annex + * [new branch] master -> r2/master + * [new branch] synced/master -> r2/synced/master +Counting objects: 4, done. +Delta compression using up to 4 threads. +Compressing objects: 100% (4/4), done. +Writing objects: 100% (4/4), 482 bytes | 0 bytes/s, done. +Total 4 (delta 2), reused 0 (delta 0) +To ../../.t\tmprepo59 + 55cc242..f01e945 master -> synced/master + * [new branch] git-annex -> synced/git-annex +[adjusted/master(unlocked) d60d0d5] git-annex in .t\tmprepo59 + 1 file changed, 1 insertion(+), 1 deletion(-) +CONFLICT (modify/delete): conflictor deleted in refs/heads/synced/master and modified in HEAD. Version HEAD of conflicto +r left in tree. +Automatic merge failed; fix conflicts and then commit the result. +conflictor: needs merge +[detached HEAD af662a3] git-annex automatic merge conflict fix +Updating d60d0d5..2584833 +Fast-forward + conflictor => conflictor.variant-0cbf | 2 +- + 1 file changed, 1 insertion(+), 1 deletion(-) + rename conflictor => conflictor.variant-0cbf (98%) +remote: Counting objects: 4, done. +remote: Compressing objects: 100% (4/4), done. +remote: Total 4 (delta 0), reused 0 (delta 0) +Unpacking objects: 100% (4/4), done. +From ../../.t\tmprepo58 + + 8291a6a...c7986fa adjusted/master(unlocked) -> r1/adjusted/master(unlocked) (forced update) + b2dee3f..5c73f12 git-annex -> r1/git-annex + 55cc242..f01e945 master -> r1/master + 55cc242..f01e945 synced/master -> r1/synced/master +Counting objects: 5, done. +Delta compression using up to 4 threads. +Compressing objects: 100% (5/5), done. +Writing objects: 100% (5/5), 680 bytes | 0 bytes/s, done. +Total 5 (delta 1), reused 0 (delta 0) +To ../../.t\tmprepo58 + b2dee3f..5c73f12 git-annex -> synced/git-annex + f01e945..af662a3 master -> synced/master +On branch adjusted/master(unlocked) +nothing to commit, working tree clean +Updating f01e945..af662a3 +Fast-forward + conflictor.variant-0cbf | 1 + + 1 file changed, 1 insertion(+) + create mode 120000 conflictor.variant-0cbf +Updating c7986fa..2965015 +Fast-forward + conflictor.variant-0cbf | 1 + + 1 file changed, 1 insertion(+) + create mode 100644 conflictor.variant-0cbf +remote: Counting objects: 5, done. +remote: Compressing objects: 100% (4/4), done. +remote: Total 5 (delta 1), reused 0 (delta 0) +Unpacking objects: 100% (5/5), done. +From ../../.t\tmprepo59 + + e665450...49f18da adjusted/master(unlocked) -> r2/adjusted/master(unlocked) (forced update) + 5c73f12..317f667 git-annex -> r2/git-annex + f01e945..af662a3 master -> r2/master + f01e945..af662a3 synced/master -> r2/synced/master + Detected a filesystem without fifo support. + Disabling ssh connection caching. + Detected a crippled filesystem. + Detected a filesystem without fifo support. + Disabling ssh connection caching. + Detected a crippled filesystem. +[adjusted/master(unlocked) 4933941] git-annex in .t\tmprepo60 + 1 file changed, 1 insertion(+) + create mode 100644 conflictor +On branch adjusted/master(unlocked) +nothing to commit, working tree clean +remote: Counting objects: 23, done. +remote: Compressing objects: 100% (21/21), done. +remote: Total 23 (delta 3), reused 0 (delta 0) +Unpacking objects: 100% (23/23), done. +From ../../.t\tmprepo60 + * [new branch] adjusted/master(unlocked) -> r1/adjusted/master(unlocked) + * [new branch] git-annex -> r1/git-annex + * [new branch] master -> r1/master + * [new branch] synced/master -> r1/synced/master +Auto-merging sha1foo +CONFLICT (add/add): Merge conflict in sha1foo +Auto-merging foo +CONFLICT (add/add): Merge conflict in foo +Automatic merge failed; fix conflicts and then commit the result. +[detached HEAD a57143f] git-annex automatic merge conflict fix +Updating 75480f7..b8f5d68 +Fast-forward + conflictor | 1 + + foo | 2 +- + 2 files changed, 2 insertions(+), 1 deletion(-) + create mode 100644 conflictor +Counting objects: 14, done. +Delta compression using up to 4 threads. +Compressing objects: 100% (14/14), done. +Writing objects: 100% (14/14), 1.46 KiB | 0 bytes/s, done. +Total 14 (delta 7), reused 0 (delta 0) +To ../../.t\tmprepo60 + 41acfd6..a57143f master -> synced/master + * [new branch] git-annex -> synced/git-annex +[adjusted/master(unlocked) 5f6a63e] git-annex in .t\tmprepo61 + 1 file changed, 1 insertion(+), 1 deletion(-) +Counting objects: 8, done. +Delta compression using up to 4 threads. +Compressing objects: 100% (7/7), done. +Writing objects: 100% (8/8), 853 bytes | 0 bytes/s, done. +Total 8 (delta 2), reused 0 (delta 0) +To ../../.t\tmprepo60 + 261f2b0..0908b36 git-annex -> synced/git-annex + a57143f..828e91b master -> synced/master +[adjusted/master(unlocked) fdcc819] git-annex in .t\tmprepo60 + 1 file changed, 1 deletion(-) + delete mode 100644 conflictor +CONFLICT (modify/delete): conflictor deleted in HEAD and modified in refs/heads/synced/master. Version refs/heads/synced +/master of conflictor left in tree. +Automatic merge failed; fix conflicts and then commit the result. +conflictor: needs merge +[detached HEAD 4151827] git-annex automatic merge conflict fix +Updating fdcc819..975b799 +Fast-forward + conflictor.variant-0cbf | 1 + + foo | 2 +- + 2 files changed, 2 insertions(+), 1 deletion(-) + create mode 100644 conflictor.variant-0cbf +remote: Counting objects: 10, done. +remote: Compressing objects: 100% (9/9), done. +remote: Total 10 (delta 1), reused 0 (delta 0) +Unpacking objects: 100% (10/10), done. +From ../../.t\tmprepo61 + * [new branch] adjusted/master(unlocked) -> r2/adjusted/master(unlocked) + * [new branch] git-annex -> r2/git-annex + * [new branch] master -> r2/master + * [new branch] synced/master -> r2/synced/master +Counting objects: 5, done. +Delta compression using up to 4 threads. +Compressing objects: 100% (5/5), done. +Writing objects: 100% (5/5), 695 bytes | 0 bytes/s, done. +Total 5 (delta 1), reused 0 (delta 0) +To ../../.t\tmprepo61 + 828e91b..4151827 master -> synced/master + * [new branch] git-annex -> synced/git-annex +On branch adjusted/master(unlocked) +nothing to commit, working tree clean +Updating 828e91b..4151827 +Fast-forward + conflictor | 1 - + conflictor.variant-0cbf | 1 + + 2 files changed, 1 insertion(+), 1 deletion(-) + delete mode 120000 conflictor + create mode 120000 conflictor.variant-0cbf +Updating 8780774..c4c9055 +Fast-forward + conflictor => conflictor.variant-0cbf | 2 +- + 1 file changed, 1 insertion(+), 1 deletion(-) + rename conflictor => conflictor.variant-0cbf (98%) +From ../../.t\tmprepo60 + + 7ceafed...f885d4b adjusted/master(unlocked) -> r1/adjusted/master(unlocked) (forced update) + 0908b36..45e8d64 git-annex -> r1/git-annex + 828e91b..4151827 master -> r1/master + 828e91b..4151827 synced/master -> r1/synced/master +OK (18.00s) + conflict resolution (nonannexed file): Detected a filesystem without fifo support. + Disabling ssh connection caching. + Detected a crippled filesystem. + Detected a filesystem without fifo support. + Disabling ssh connection caching. + Detected a crippled filesystem. +[adjusted/master(unlocked) 4438cd9] git-annex in .t\tmprepo62 + 1 file changed, 1 insertion(+) + create mode 100644 conflictor +[adjusted/master(unlocked) 7d984b7] git-annex in .t\tmprepo63 + 1 file changed, 1 insertion(+) + create mode 100644 conflictor +On branch adjusted/master(unlocked) +nothing to commit, working tree clean +remote: Counting objects: 17, done. +remote: Compressing objects: 100% (15/15), done. +remote: Total 17 (delta 2), reused 0 (delta 0) +Unpacking objects: 100% (17/17), done. +From ../../.t\tmprepo63 + * [new branch] adjusted/master(unlocked) -> r2/adjusted/master(unlocked) + * [new branch] git-annex -> r2/git-annex + * [new branch] master -> r2/master + * [new branch] synced/master -> r2/synced/master +Auto-merging sha1foo +CONFLICT (add/add): Merge conflict in sha1foo +Auto-merging foo +CONFLICT (add/add): Merge conflict in foo +Auto-merging conflictor +CONFLICT (add/add): Merge conflict in conflictor +Automatic merge failed; fix conflicts and then commit the result. +conflictor: needs merge +[detached HEAD ddbc6a3] git-annex automatic merge conflict fix +Updating c42b305..90fd358 +Fast-forward + conflictor | 2 +- + conflictor.variant-cc12 | 1 + + foo | 2 +- + 3 files changed, 3 insertions(+), 2 deletions(-) + create mode 100644 conflictor.variant-cc12 +Counting objects: 22, done. +Delta compression using up to 4 threads. +Compressing objects: 100% (21/21), done. +Writing objects: 100% (22/22), 2.25 KiB | 0 bytes/s, done. +Total 22 (delta 9), reused 0 (delta 0) +To ../../.t\tmprepo63 + 6ae609a..ddbc6a3 master -> synced/master + * [new branch] git-annex -> synced/git-annex +On branch adjusted/master(unlocked) +nothing to commit, working tree clean +Updating 6ae609a..ddbc6a3 +Fast-forward + conflictor.variant-cc12 | 1 + + foo | 2 +- + sha1foo | 2 +- + 3 files changed, 3 insertions(+), 2 deletions(-) + create mode 120000 conflictor.variant-cc12 +Updating 8714c6e..d884f3f +Fast-forward + conflictor.variant-cc12 | 1 + + foo | 2 +- + 2 files changed, 2 insertions(+), 1 deletion(-) + create mode 100644 conflictor.variant-cc12 +From ../../.t\tmprepo62 + * [new branch] adjusted/master(unlocked) -> r1/adjusted/master(unlocked) + * [new branch] git-annex -> r1/git-annex + * [new branch] master -> r1/master + * [new branch] synced/master -> r1/synced/master + Detected a filesystem without fifo support. + Disabling ssh connection caching. + Detected a crippled filesystem. + Detected a filesystem without fifo support. + Disabling ssh connection caching. + Detected a crippled filesystem. +[adjusted/master(unlocked) 99fd1b3] git-annex in .t\tmprepo64 + 1 file changed, 1 insertion(+) + create mode 100644 conflictor +[adjusted/master(unlocked) 30452d4] git-annex in .t\tmprepo65 + 1 file changed, 1 insertion(+) + create mode 100644 conflictor +On branch adjusted/master(unlocked) +nothing to commit, working tree clean +remote: Counting objects: 23, done. +remote: Compressing objects: 100% (21/21), done. +remote: Total 23 (delta 4), reused 0 (delta 0) +Unpacking objects: 100% (23/23), done. +From ../../.t\tmprepo64 + * [new branch] adjusted/master(unlocked) -> r1/adjusted/master(unlocked) + * [new branch] git-annex -> r1/git-annex + * [new branch] master -> r1/master + * [new branch] synced/master -> r1/synced/master +Auto-merging sha1foo +CONFLICT (add/add): Merge conflict in sha1foo +Auto-merging foo +CONFLICT (add/add): Merge conflict in foo +Auto-merging conflictor +CONFLICT (add/add): Merge conflict in conflictor +Automatic merge failed; fix conflicts and then commit the result. +conflictor: needs merge +[detached HEAD 46ec327] git-annex automatic merge conflict fix +Updating 818dce1..9788dab +Fast-forward + conflictor.variant-cc12 | 1 + + foo | 2 +- + 2 files changed, 2 insertions(+), 1 deletion(-) + create mode 100644 conflictor.variant-cc12 +Counting objects: 18, done. +Delta compression using up to 4 threads. +Compressing objects: 100% (17/17), done. +Writing objects: 100% (18/18), 1.69 KiB | 0 bytes/s, done. +Total 18 (delta 10), reused 0 (delta 0) +To ../../.t\tmprepo64 + 6ae34c0..46ec327 master -> synced/master + * [new branch] git-annex -> synced/git-annex +On branch adjusted/master(unlocked) +nothing to commit, working tree clean +Updating 6ae34c0..46ec327 +Fast-forward + conflictor | 2 +- + conflictor.variant-cc12 | 1 + + foo | 2 +- + sha1foo | 2 +- + 4 files changed, 4 insertions(+), 3 deletions(-) + mode change 120000 => 100644 conflictor + create mode 120000 conflictor.variant-cc12 +Updating aa27001..eb304ba +Fast-forward + conflictor | 2 +- + conflictor.variant-cc12 | 1 + + foo | 2 +- + 3 files changed, 3 insertions(+), 2 deletions(-) + create mode 100644 conflictor.variant-cc12 +From ../../.t\tmprepo65 + * [new branch] adjusted/master(unlocked) -> r2/adjusted/master(unlocked) + * [new branch] git-annex -> r2/git-annex + * [new branch] master -> r2/master + * [new branch] synced/master -> r2/synced/master + Detected a filesystem without fifo support. + Disabling ssh connection caching. + Detected a crippled filesystem. + Detected a filesystem without fifo support. + Disabling ssh connection caching. + Detected a crippled filesystem. +[adjusted/master(unlocked) 34af785] git-annex in .t\tmprepo66 + 1 file changed, 1 insertion(+) + create mode 100644 conflictor +[adjusted/master(unlocked) d1ad368] git-annex in .t\tmprepo67 + 1 file changed, 1 insertion(+) + create mode 100644 conflictor +On branch adjusted/master(unlocked) +nothing to commit, working tree clean +remote: Counting objects: 17, done. +remote: Compressing objects: 100% (15/15), done. +remote: Total 17 (delta 1), reused 0 (delta 0) +Unpacking objects: 100% (17/17), done. +From ../../.t\tmprepo67 + * [new branch] adjusted/master(unlocked) -> r2/adjusted/master(unlocked) + * [new branch] git-annex -> r2/git-annex + * [new branch] master -> r2/master + * [new branch] synced/master -> r2/synced/master +Auto-merging sha1foo +CONFLICT (add/add): Merge conflict in sha1foo +Auto-merging foo +CONFLICT (add/add): Merge conflict in foo +Auto-merging conflictor +CONFLICT (add/add): Merge conflict in conflictor +Automatic merge failed; fix conflicts and then commit the result. +conflictor: needs merge +[detached HEAD b6643ac] git-annex automatic merge conflict fix +Updating 58a264d..7f1e818 +Fast-forward + conflictor | 2 +- + conflictor.variant-cc12 | 1 + + foo | 2 +- + 3 files changed, 3 insertions(+), 2 deletions(-) + create mode 100644 conflictor.variant-cc12 +Counting objects: 22, done. +Delta compression using up to 4 threads. +Compressing objects: 100% (21/21), done. +Writing objects: 100% (22/22), 2.30 KiB | 0 bytes/s, done. +Total 22 (delta 8), reused 0 (delta 0) +To ../../.t\tmprepo67 + ad93099..b6643ac master -> synced/master + * [new branch] git-annex -> synced/git-annex +On branch adjusted/master(unlocked) +nothing to commit, working tree clean +Updating ad93099..b6643ac +Fast-forward + conflictor.variant-cc12 | 1 + + foo | 2 +- + sha1foo | 2 +- + 3 files changed, 3 insertions(+), 2 deletions(-) + create mode 120000 conflictor.variant-cc12 +Updating f042552..0b591c1 +Fast-forward + conflictor.variant-cc12 | 1 + + foo | 2 +- + 2 files changed, 2 insertions(+), 1 deletion(-) + create mode 100644 conflictor.variant-cc12 +From ../../.t\tmprepo66 + * [new branch] adjusted/master(unlocked) -> r1/adjusted/master(unlocked) + * [new branch] git-annex -> r1/git-annex + * [new branch] master -> r1/master + * [new branch] synced/master -> r1/synced/master + Detected a filesystem without fifo support. + Disabling ssh connection caching. + Detected a crippled filesystem. + Detected a filesystem without fifo support. + Disabling ssh connection caching. + Detected a crippled filesystem. +[adjusted/master(unlocked) aabedb4] git-annex in .t\tmprepo68 + 1 file changed, 1 insertion(+) + create mode 100644 conflictor +[adjusted/master(unlocked) 0666b67] git-annex in .t\tmprepo69 + 1 file changed, 1 insertion(+) + create mode 100644 conflictor +On branch adjusted/master(unlocked) +nothing to commit, working tree clean +remote: Counting objects: 23, done. +remote: Compressing objects: 100% (21/21), done. +remote: Total 23 (delta 4), reused 0 (delta 0) +Unpacking objects: 100% (23/23), done. +From ../../.t\tmprepo68 + * [new branch] adjusted/master(unlocked) -> r1/adjusted/master(unlocked) + * [new branch] git-annex -> r1/git-annex + * [new branch] master -> r1/master + * [new branch] synced/master -> r1/synced/master +Auto-merging sha1foo +CONFLICT (add/add): Merge conflict in sha1foo +Auto-merging foo +CONFLICT (add/add): Merge conflict in foo +Auto-merging conflictor +CONFLICT (add/add): Merge conflict in conflictor +Automatic merge failed; fix conflicts and then commit the result. +conflictor: needs merge +[detached HEAD 15337ba] git-annex automatic merge conflict fix +Updating d8e1daa..2bdecc4 +Fast-forward + conflictor.variant-cc12 | 1 + + foo | 2 +- + 2 files changed, 2 insertions(+), 1 deletion(-) + create mode 100644 conflictor.variant-cc12 +Counting objects: 18, done. +Delta compression using up to 4 threads. +Compressing objects: 100% (17/17), done. +Writing objects: 100% (18/18), 1.69 KiB | 0 bytes/s, done. +Total 18 (delta 10), reused 0 (delta 0) +To ../../.t\tmprepo68 + 0322df0..15337ba master -> synced/master + * [new branch] git-annex -> synced/git-annex +On branch adjusted/master(unlocked) +nothing to commit, working tree clean +Updating 0322df0..15337ba +Fast-forward + conflictor | 2 +- + conflictor.variant-cc12 | 1 + + foo | 2 +- + sha1foo | 2 +- + 4 files changed, 4 insertions(+), 3 deletions(-) + mode change 120000 => 100644 conflictor + create mode 120000 conflictor.variant-cc12 +Updating f638189..62dac7e +Fast-forward + conflictor | 2 +- + conflictor.variant-cc12 | 1 + + foo | 2 +- + 3 files changed, 3 insertions(+), 2 deletions(-) + create mode 100644 conflictor.variant-cc12 +From ../../.t\tmprepo69 + * [new branch] adjusted/master(unlocked) -> r2/adjusted/master(unlocked) + * [new branch] git-annex -> r2/git-annex + * [new branch] master -> r2/master + * [new branch] synced/master -> r2/synced/master +OK (25.61s) + conflict resolution (nonannexed symlink): Detected a filesystem without fifo support. + Disabling ssh connection caching. + Detected a crippled filesystem. + Detected a filesystem without fifo support. + Disabling ssh connection caching. + Detected a crippled filesystem. + Detected a filesystem without fifo support. + Disabling ssh connection caching. + Detected a crippled filesystem. + Detected a filesystem without fifo support. + Disabling ssh connection caching. + Detected a crippled filesystem. + Detected a filesystem without fifo support. + Disabling ssh connection caching. + Detected a crippled filesystem. + Detected a filesystem without fifo support. + Disabling ssh connection caching. + Detected a crippled filesystem. + Detected a filesystem without fifo support. + Disabling ssh connection caching. + Detected a crippled filesystem. + Detected a filesystem without fifo support. + Disabling ssh connection caching. + Detected a crippled filesystem. +OK (7.43s) + conflict resolution (mixed locked and unlocked file): Detected a filesystem without fifo support. + Disabling ssh connection caching. + Detected a crippled filesystem. + Detected a filesystem without fifo support. + Disabling ssh connection caching. + Detected a crippled filesystem. +[adjusted/master(unlocked) 6c05187] git-annex in .t\tmprepo78 + 1 file changed, 1 insertion(+) + create mode 100644 conflictor +[adjusted/master(unlocked) 3968c28] git-annex in .t\tmprepo79 + 1 file changed, 1 insertion(+) + create mode 100644 conflictor +On branch adjusted/master(unlocked) +nothing to commit, working tree clean +remote: Counting objects: 23, done. +remote: Compressing objects: 100% (21/21), done. +remote: Total 23 (delta 4), reused 0 (delta 0) +Unpacking objects: 100% (23/23), done. +From ../../.t\tmprepo79 + * [new branch] adjusted/master(unlocked) -> r2/adjusted/master(unlocked) + * [new branch] git-annex -> r2/git-annex + * [new branch] master -> r2/master + * [new branch] synced/master -> r2/synced/master +Auto-merging sha1foo +CONFLICT (add/add): Merge conflict in sha1foo +Auto-merging foo +CONFLICT (add/add): Merge conflict in foo +Auto-merging conflictor +CONFLICT (add/add): Merge conflict in conflictor +Automatic merge failed; fix conflicts and then commit the result. +[detached HEAD 54748a5] git-annex automatic merge conflict fix +Updating 0cafe66..1f3fc72 +Fast-forward + conflictor | 2 +- + foo | 2 +- + 2 files changed, 2 insertions(+), 2 deletions(-) +Counting objects: 24, done. +Delta compression using up to 4 threads. +Compressing objects: 100% (22/22), done. +Writing objects: 100% (24/24), 2.07 KiB | 0 bytes/s, done. +Total 24 (delta 14), reused 0 (delta 0) +To ../../.t\tmprepo79 + 9223b4e..54748a5 master -> synced/master + * [new branch] git-annex -> synced/git-annex +On branch adjusted/master(unlocked) +nothing to commit, working tree clean +Updating 9223b4e..54748a5 +Fast-forward + conflictor | 2 +- + foo | 2 +- + sha1foo | 2 +- + 3 files changed, 3 insertions(+), 3 deletions(-) +Updating 592ec16..001733a +Fast-forward + conflictor | 2 +- + foo | 2 +- + 2 files changed, 2 insertions(+), 2 deletions(-) +From ../../.t\tmprepo78 + * [new branch] adjusted/master(unlocked) -> r1/adjusted/master(unlocked) + * [new branch] git-annex -> r1/git-annex + * [new branch] master -> r1/master + * [new branch] synced/master -> r1/synced/master +[adjusted/master(unlocked) 43e95b6] git-annex in .t\tmprepo78 + 1 file changed, 1 insertion(+), 1 deletion(-) +From ../../.t\tmprepo79 + + 592ec16...7fbe351 adjusted/master(unlocked) -> r2/adjusted/master(unlocked) (forced update) +Counting objects: 1, done. +Writing objects: 100% (1/1), 191 bytes | 0 bytes/s, done. +Total 1 (delta 0), reused 0 (delta 0) +To ../../.t\tmprepo79 + 54748a5..a15aa8c master -> synced/master +conflictor +conflictor +OK (7.72s) + map: Detected a filesystem without fifo support. + Disabling ssh connection caching. + Detected a crippled filesystem. +OK (1.25s) + uninit: Detected a filesystem without fifo support. + Disabling ssh connection caching. + Detected a crippled filesystem. +OK (1.63s) + uninit (in git-annex branch): Detected a filesystem without fifo support. + Disabling ssh connection caching. + Detected a crippled filesystem. + Detected a filesystem without fifo support. + Disabling ssh connection caching. + Detected a crippled filesystem. +Switched to branch 'git-annex' +OK (1.49s) + upgrade: Detected a filesystem without fifo support. + Disabling ssh connection caching. + Detected a crippled filesystem. +OK (0.99s) + whereis: Detected a filesystem without fifo support. + Disabling ssh connection caching. + Detected a crippled filesystem. +OK (2.34s) + hook remote: Detected a filesystem without fifo support. + Disabling ssh connection caching. + Detected a crippled filesystem. +OK (0.97s) + directory remote: Detected a filesystem without fifo support. + Disabling ssh connection caching. + Detected a crippled filesystem. +OK (2.65s) + rsync remote: Detected a filesystem without fifo support. + Disabling ssh connection caching. + Detected a crippled filesystem. +OK (3.92s) + bup remote: Detected a filesystem without fifo support. + Disabling ssh connection caching. + Detected a crippled filesystem. +OKgpg testing not implemented on Windows + (0.99s) + crypto: OK + preferred content: Detected a filesystem without fifo support. + Disabling ssh connection caching. + Detected a crippled filesystem. +wanted . ok +(recording state in git...) +group . ok +(recording state in git...) +wanted . ok +(recording state in git...) +group . ok +(recording state in git...) +wanted . ok +(recording state in git...) +OK (5.19s) + add subdirs: Detected a filesystem without fifo support. + Disabling ssh connection caching. + Detected a crippled filesystem. +OK (1.96s) + addurl: Detected a filesystem without fifo support. + Disabling ssh connection caching. + Detected a crippled filesystem. +curl: (37) Couldn't open file /test2/.t/tmprepo91/myurl +removeDirectoryRecursive: permission denied (Zugriff verweigert) +sleeping 10 seconds and will retry directory cleanup +FAIL + Exception: removeDirectoryRecursive: permission denied (Zugriff verweigert) + Unit Tests v5 direct + add dup: Init Tests + init: Detected a filesystem without fifo support. + Disabling ssh connection caching. + Detected a crippled filesystem. + Entering an adjusted branch where files are unlocked as this filesystem does not support locked files. + adjusted branch adjusted/master(unlocked) already exists. + Aborting because that branch may have changes that have not yet reached master + You can check out the adjusted branch manually to enter it, or delete the adjusted branch and re-run this command. + Failed to enter adjusted branch! +FAIL (0.59s) + git annex direct failed + add: FAIL (0.18s) + foo is not a (crippled) symlink + +2 out of 2 tests failed (0.76s) +FAIL + Exception: init tests failed! cannot continue + add extras: FAIL + Exception: init tests failed! cannot continue + shared clone: FAIL + Exception: init tests failed! cannot continue + log: FAIL + Exception: init tests failed! cannot continue + import: FAIL + Exception: init tests failed! cannot continue + reinject: FAIL + Exception: init tests failed! cannot continue + unannex (no copy): FAIL + Exception: init tests failed! cannot continue + unannex (with copy): FAIL + Exception: init tests failed! cannot continue + drop (no remote): FAIL + Exception: init tests failed! cannot continue + drop (with remote): FAIL + Exception: init tests failed! cannot continue + drop (untrusted remote): FAIL + Exception: init tests failed! cannot continue + get: FAIL + Exception: init tests failed! cannot continue + get (ssh remote): FAIL + Exception: init tests failed! cannot continue + move: FAIL + Exception: init tests failed! cannot continue + move (ssh remote): FAIL + Exception: init tests failed! cannot continue + copy: FAIL + Exception: init tests failed! cannot continue + lock: FAIL + Exception: init tests failed! cannot continue + lock (v6 --force): FAIL + Exception: init tests failed! cannot continue + edit (no pre-commit): FAIL + Exception: init tests failed! cannot continue + edit (pre-commit): FAIL + Exception: init tests failed! cannot continue + partial commit: FAIL + Exception: init tests failed! cannot continue + fix: FAIL + Exception: init tests failed! cannot continue + direct: FAIL + Exception: init tests failed! cannot continue + trust: FAIL + Exception: init tests failed! cannot continue + fsck (basics): FAIL + Exception: init tests failed! cannot continue + fsck (bare): FAIL + Exception: init tests failed! cannot continue + fsck (local untrusted): FAIL + Exception: init tests failed! cannot continue + fsck (remote untrusted): FAIL + Exception: init tests failed! cannot continue + fsck --from remote: FAIL + Exception: init tests failed! cannot continue + migrate: FAIL + Exception: init tests failed! cannot continue + migrate (via gitattributes): FAIL + Exception: init tests failed! cannot continue + unused: FAIL + Exception: init tests failed! cannot continue + describe: FAIL + Exception: init tests failed! cannot continue + find: FAIL + Exception: init tests failed! cannot continue + merge: FAIL + Exception: init tests failed! cannot continue + info: FAIL + Exception: init tests failed! cannot continue + version: FAIL + Exception: init tests failed! cannot continue + sync: FAIL + Exception: init tests failed! cannot continue + union merge regression: FAIL + Exception: init tests failed! cannot continue + adjusted branch merge regression: FAIL + Exception: init tests failed! cannot continue + adjusted branch subtree regression: FAIL + Exception: init tests failed! cannot continue + conflict resolution: FAIL + Exception: init tests failed! cannot continue + conflict resolution (adjusted branch): FAIL + Exception: init tests failed! cannot continue + conflict resolution movein regression: FAIL + Exception: init tests failed! cannot continue + conflict resolution (mixed directory and file): FAIL + Exception: init tests failed! cannot continue + conflict resolution symlink bit: FAIL + Exception: init tests failed! cannot continue + conflict resolution (uncommitted local file): FAIL + Exception: init tests failed! cannot continue + conflict resolution (removed file): FAIL + Exception: init tests failed! cannot continue + conflict resolution (nonannexed file): FAIL + Exception: init tests failed! cannot continue + conflict resolution (nonannexed symlink): FAIL + Exception: init tests failed! cannot continue + conflict resolution (mixed locked and unlocked file): FAIL + Exception: init tests failed! cannot continue + map: FAIL + Exception: init tests failed! cannot continue + uninit: FAIL + Exception: init tests failed! cannot continue + uninit (in git-annex branch): FAIL + Exception: init tests failed! cannot continue + upgrade: FAIL + Exception: init tests failed! cannot continue + whereis: FAIL + Exception: init tests failed! cannot continue + hook remote: FAIL + Exception: init tests failed! cannot continue + directory remote: FAIL + Exception: init tests failed! cannot continue + rsync remote: FAIL + Exception: init tests failed! cannot continue + bup remote: FAIL + Exception: init tests failed! cannot continue + crypto: FAIL + Exception: init tests failed! cannot continue + preferred content: FAIL + Exception: init tests failed! cannot continue + add subdirs: FAIL + Exception: init tests failed! cannot continue + addurl: FAIL + Exception: init tests failed! cannot continue + +74 out of 165 tests failed (406.30s) + (This could be due to a bug in git-annex, or an incompatibility + with utilities, such as git, installed on this system.) +PS G:\test2> + +# End of transcript or log. +"""]] + +### Have you had any luck using git-annex before? (Sometimes we get tired of reading bug reports all day and a lil' positive end note does wonders) + +Of course! On Linux, it runs perfectly. I manage all my data with git-annex. diff --git a/doc/bugs/Three_tests_fail_when_annex.backends_is_defined.mdwn b/doc/bugs/Three_tests_fail_when_annex.backends_is_defined.mdwn new file mode 100644 index 0000000000..11f4a9a3e1 --- /dev/null +++ b/doc/bugs/Three_tests_fail_when_annex.backends_is_defined.mdwn @@ -0,0 +1,111 @@ +### Please describe the problem. + +I noticed three tests failed when running "git annex test", and it seems +it's because I have `annex.backends` set to `SHA256` in `~/.gitconfig`. +When I disable that option, all tests succeed. + +### What steps will reproduce the problem? + +Add + + [annex] + backends = SHA256 + +to `~/.gitconfig`. + +### What version of git-annex are you using? On what operating system? + +Newest git-annex (6.20160211 amd64) from downloads.kitenet.net. + +- Debian GNU/Linux 8.3 (64 bit). Installed yesterday, so it's pretty pristine. +- git version 2.7.1.287.g4943984 (Newest version from 'master' in + git.git) + +The first version with this problem is 6.20160114. 5.20151218 works. + +### Please provide any additional information below. + +[[!format text """ +# If you can, paste a complete transcript of the problem occurring here. +# If the problem is with the git-annex assistant, paste in .git/annex/daemon.log + +$ git annex test +Tests + +[Removed 188 lines] + + migrate (via gitattributes): /etc/magic, 4: Warning: using regular magic file `/usr/share/misc/magic' +OK (3.11s) + unused: /etc/magic, 4: Warning: using regular magic file `/usr/share/misc/magic' +FAIL (1.84s) + unused keys differ after origin branches are gone + expected: [Key {keyName = "e394a389d787383843decc5d3d99b6d184ffa5fddeec23b911f9ee7fc8b9ea77", keyBackendName = "SHA256E", keySize = Just 20, keyMtime = Nothing, keyChunkSize = Nothing, keyChunkNum = Nothing}] + but got: [Key {keyName = "e394a389d787383843decc5d3d99b6d184ffa5fddeec23b911f9ee7fc8b9ea77", keyBackendName = "SHA256", keySize = Just 20, keyMtime = Nothing, keyChunkSize = Nothing, keyChunkNum = Nothing}] + describe: /etc/magic, 4: Warning: using regular magic file `/usr/share/misc/magic' +OK (0.84s) + +[Removed 1108 lines] + + migrate (via gitattributes): /etc/magic, 4: Warning: using regular magic file `/usr/share/misc/magic' +OK (3.34s) + unused: /etc/magic, 4: Warning: using regular magic file `/usr/share/misc/magic' +/etc/magic, 4: Warning: using regular magic file `/usr/share/misc/magic' +/etc/magic, 4: Warning: using regular magic file `/usr/share/misc/magic' +FAIL (1.82s) + unused keys differ after origin branches are gone + expected: [Key {keyName = "e394a389d787383843decc5d3d99b6d184ffa5fddeec23b911f9ee7fc8b9ea77", keyBackendName = "SHA256E", keySize = Just 20, keyMtime = Nothing, keyChunkSize = Nothing, keyChunkNum = Nothing}] + but got: [Key {keyName = "e394a389d787383843decc5d3d99b6d184ffa5fddeec23b911f9ee7fc8b9ea77", keyBackendName = "SHA256", keySize = Just 20, keyMtime = Nothing, keyChunkSize = Nothing, keyChunkNum = Nothing}] + describe: /etc/magic, 4: Warning: using regular magic file `/usr/share/misc/magic' +OK (0.78s) + +[Removed 1062 lines] + + migrate (via gitattributes): OK (2.64s) + unused: FAIL (1.53s) + unused keys differ after origin branches are gone + expected: [Key {keyName = "e394a389d787383843decc5d3d99b6d184ffa5fddeec23b911f9ee7fc8b9ea77", keyBackendName = "SHA256E", keySize = Just 20, keyMtime = Nothing, keyChunkSize = Nothing, keyChunkNum = Nothing}] + but got: [Key {keyName = "e394a389d787383843decc5d3d99b6d184ffa5fddeec23b911f9ee7fc8b9ea77", keyBackendName = "SHA256", keySize = Just 20, keyMtime = Nothing, keyChunkSize = Nothing, keyChunkNum = Nothing}] + describe: OK (0.62s) + +[Removed 1667 lines] + +3 out of 269 tests failed (640.58s) + (This could be due to a bug in git-annex, or an incompatibility + with utilities, such as git, installed on this system.) +$ + +# End of transcript or log. +"""]] + +### Have you had any luck using git-annex before? (Sometimes we get tired of reading bug reports all day and a lil' positive end note does wonders) + +Roses are red
    +Violets are blue
    +git-annex is awesome
    +and so are you + +`;-)` + +But bloody hell, it's hard to get this thing to build. As you've +mentioned earlier, building on Debian 7 isn't supported anymore, so I +installed Debian 8.3 yesterday for the sole purpose of compiling git-annex, +but still no luck: + + [32 of 32] Compiling Main ( dist/setup/setup.hs, dist/setup/Main.o ) + Linking ./dist/setup/setup ... + unrecognized option `--extra-prog-path=/home/sunny/.cabal/bin' + Makefile:19: recipe for target 'Build/SysConfig.hs' failed + make: *** [Build/SysConfig.hs] Error 1 + +I was hoping to get it to build so I could find the exact commit that +introduced this, and also provide some patches with some functionality +I've been thinking of. Is building on Debian 8.3 (jessie) supported? I +found a bug report from you on + that seems to +be related, but I don't know if that's the case. The newest version I'm +able to build is 5.20150219, somewhere after that various things start +to fail. + +> [[fixed|done]]; I found a way to isolate the test suite from global git +> configs, by setting `GIT_CONFIG_NOSYSTEM` and also setting +> `HOME` and `XDG_CONFIG_HOME` to an empty directory. --[[Joey]] diff --git a/doc/bugs/Three_tests_fail_when_annex.backends_is_defined/comment_1_153ba2b98b65fcc6156b1f2f967839b2._comment b/doc/bugs/Three_tests_fail_when_annex.backends_is_defined/comment_1_153ba2b98b65fcc6156b1f2f967839b2._comment new file mode 100644 index 0000000000..194439668d --- /dev/null +++ b/doc/bugs/Three_tests_fail_when_annex.backends_is_defined/comment_1_153ba2b98b65fcc6156b1f2f967839b2._comment @@ -0,0 +1,23 @@ +[[!comment format=mdwn + username="joey" + subject="""comment 1""" + date="2016-02-17T16:16:53Z" + content=""" +The solution to your build failure is to edit ~/.cabal/config and remove +the extra-prog-path line. (It's unfortunate that Debian shipped 8.3 with +that bug. I told them about it before the release, but they didn't get it +fixed in time.) + +There are probably several other config settings that also break +the test suite if set in a global gitconfig. For example the +annex.addunlocked setting I added yesterday will. + +I think the best approach is to isolate the the test suite from global git +config settings. Making the test suite set `GIT_CONFIG` to the config of +each test repo it runs git-annex in seems like it would +work; this prevents git from reading the user and global configs. + +The test suite could set all config values to defaults +in the test git repos. But this would need some restructuring of the code to +be able to not just query, but also set those values. +"""]] diff --git a/doc/bugs/Three_tests_fail_when_annex.backends_is_defined/comment_2_20bd7de1902f7e3e4afdb8a8ff0ca1b7._comment b/doc/bugs/Three_tests_fail_when_annex.backends_is_defined/comment_2_20bd7de1902f7e3e4afdb8a8ff0ca1b7._comment new file mode 100644 index 0000000000..917a88c1fc --- /dev/null +++ b/doc/bugs/Three_tests_fail_when_annex.backends_is_defined/comment_2_20bd7de1902f7e3e4afdb8a8ff0ca1b7._comment @@ -0,0 +1,9 @@ +[[!comment format=mdwn + username="joey" + subject="""comment 2""" + date="2016-02-17T19:19:18Z" + content=""" +Setting `GIT_CONFIG` won't work well, because sometimes git-annex in one +repo needs to look at the config of another repo, which would be prevented +by that setting. +"""]] diff --git a/doc/bugs/Too_difficult_if_not_impossible_to_explicitly_add__47__keep_file_under_git___40__not_annex__41___in_v6_without_employing_.gitattributes.mdwn b/doc/bugs/Too_difficult_if_not_impossible_to_explicitly_add__47__keep_file_under_git___40__not_annex__41___in_v6_without_employing_.gitattributes.mdwn new file mode 100644 index 0000000000..32760fa521 --- /dev/null +++ b/doc/bugs/Too_difficult_if_not_impossible_to_explicitly_add__47__keep_file_under_git___40__not_annex__41___in_v6_without_employing_.gitattributes.mdwn @@ -0,0 +1,110 @@ +### Please describe the problem. + +Even if -c annex.largefiles=nothing is used with git add, then git commit commits file into annex. + +This script demonstrates initial finding: http://www.onerussian.com/tmp/ga.sh + +Even if we use -c annex.largefiles=nothing with both add and commit, subsequent git status migrates that file into annex (which is even more weird) + +Adjusted script demonstrates it: http://www.onerussian.com/tmp/ga-2.sh + +### What steps will reproduce the problem? + +see above + +### What version of git-annex are you using? On what operating system? + +6.20170101+gitg93d69b1-1~ndall+1 + + +### Please provide any additional information below. + +For completeness here is the output of the 2nd script run: + +[[!format sh """ +$> /tmp/ga-2.sh +++ mktemp -d ++ d=/home/yoh/.tmp/tmp.WXTv2cULD0 ++ echo 'directory: /home/yoh/.tmp/tmp.WXTv2cULD0' +directory: /home/yoh/.tmp/tmp.WXTv2cULD0 ++ cd /home/yoh/.tmp/tmp.WXTv2cULD0 ++ git init +Initialized empty Git repository in /tmp/tmp.WXTv2cULD0/.git/ ++ git annex init --version=6 +init ok +(recording state in git...) ++ sed -i -e 's,pre-commit ,pre-commit --debug ,g' .git/hooks/pre-commit ++ echo 'I: creating a file' +I: creating a file ++ echo whatever ++ git -c annex.largefiles=nothing add file5 ++ git annex lookupkey file5 ++ echo 'not in annex as it should' +not in annex as it should ++ git annex find ++ ls -lR .git/annex/objects +ls: cannot access '.git/annex/objects': No such file or directory ++ : ++ git -c annex.largefiles=nothing commit -m sdf +[2017-01-27 09:43:27.569648245] read: git ["--git-dir=.git","--work-tree=.","--literal-pathspecs","diff","--cached","--name-only","-z","--diff-filter=ACMRT","--","."] +[2017-01-27 09:43:27.576498829] read: git ["--git-dir=.git","--work-tree=.","--literal-pathspecs","symbolic-ref","-q","HEAD"] +[2017-01-27 09:43:27.581039152] process done ExitSuccess +[2017-01-27 09:43:27.581134039] read: git ["--git-dir=.git","--work-tree=.","--literal-pathspecs","show-ref","refs/heads/master"] +[2017-01-27 09:43:27.585093046] process done ExitFailure 1 +[master (root-commit) 9ec3fe6] sdf + 1 file changed, 1 insertion(+) + create mode 100644 file5 ++ ls -lR .git/annex/objects +ls: cannot access '.git/annex/objects': No such file or directory ++ : ++ echo 'I: before git status' +I: before git status ++ ls -lR .git/annex/objects +ls: cannot access '.git/annex/objects': No such file or directory ++ : ++ git status +On branch master +Changes not staged for commit: + (use "git add ..." to update what will be committed) + (use "git checkout -- ..." to discard changes in working directory) + + modified: file5 + +no changes added to commit (use "git add" and/or "git commit -a") ++ echo 'I: after git status' +I: after git status ++ ls -lR .git/annex/objects +.git/annex/objects: +total 0 +drwx------ 3 yoh yoh 60 Jan 27 09:43 XF + +.git/annex/objects/XF: +total 0 +drwx------ 3 yoh yoh 60 Jan 27 09:43 pp + +.git/annex/objects/XF/pp: +total 0 +dr-x------ 2 yoh yoh 60 Jan 27 09:43 SHA256E-s9--cd293be6cea034bd45a0352775a219ef5dc7825ce55d1f7dae9762d80ce64411 + +.git/annex/objects/XF/pp/SHA256E-s9--cd293be6cea034bd45a0352775a219ef5dc7825ce55d1f7dae9762d80ce64411: +total 4 +-rw------- 1 yoh yoh 9 Jan 27 09:43 SHA256E-s9--cd293be6cea034bd45a0352775a219ef5dc7825ce55d1f7dae9762d80ce64411 ++ git diff +diff --git a/file5 b/file5 +index 982793c..8fdffc0 100644 +--- a/file5 ++++ b/file5 +@@ -1 +1 @@ +-whatever ++/annex/objects/SHA256E-s9--cd293be6cea034bd45a0352775a219ef5dc7825ce55d1f7dae9762d80ce64411 + +"""]] + +### Have you had any luck using git-annex before? (Sometimes we get tired of reading bug reports all day and a lil' positive end note does wonders) + +yeap + +[[!meta author=yoh]] + +> [[done]]; clean filter defaults to preserving git/annex state of file. +> --[[Joey]] diff --git a/doc/bugs/Too_difficult_if_not_impossible_to_explicitly_add__47__keep_file_under_git___40__not_annex__41___in_v6_without_employing_.gitattributes/comment_10_5eb0622b326a8094b06f5f0de627e288._comment b/doc/bugs/Too_difficult_if_not_impossible_to_explicitly_add__47__keep_file_under_git___40__not_annex__41___in_v6_without_employing_.gitattributes/comment_10_5eb0622b326a8094b06f5f0de627e288._comment new file mode 100644 index 0000000000..dcba5d1420 --- /dev/null +++ b/doc/bugs/Too_difficult_if_not_impossible_to_explicitly_add__47__keep_file_under_git___40__not_annex__41___in_v6_without_employing_.gitattributes/comment_10_5eb0622b326a8094b06f5f0de627e288._comment @@ -0,0 +1,20 @@ +[[!comment format=mdwn + username="joey" + subject="""comment 10""" + date="2018-08-09T19:52:50Z" + content=""" +Ben, I can reproduce that, but the file appearing modified in git status +is a known problem documented in [[todo/smudge]]. It's one +of the primary reasons that v6 mode remains experiemental. + +While `git commit -a` in that clone does cause the file to be converted +from git to annex, touching the file and committing has the same effect. If +you want to juggle annexed and non-annexed files in a v6 repository without +letting annex.largefiles tell git-annex what to do, you have to manually +tell it what to do every time the file is staged. When you `git commit -a`, +you stage the file and so you need to include `-c annex.largefiles=nothing` +to keep it from transitioning to the annex. + +It think it might make sense to get v6 working to the point that it's +non-experimental before worrying about such a marginal edge case as this. +"""]] diff --git a/doc/bugs/Too_difficult_if_not_impossible_to_explicitly_add__47__keep_file_under_git___40__not_annex__41___in_v6_without_employing_.gitattributes/comment_11_a3b5e2047185bbd3972ada07ddb79172._comment b/doc/bugs/Too_difficult_if_not_impossible_to_explicitly_add__47__keep_file_under_git___40__not_annex__41___in_v6_without_employing_.gitattributes/comment_11_a3b5e2047185bbd3972ada07ddb79172._comment new file mode 100644 index 0000000000..45ddfa1014 --- /dev/null +++ b/doc/bugs/Too_difficult_if_not_impossible_to_explicitly_add__47__keep_file_under_git___40__not_annex__41___in_v6_without_employing_.gitattributes/comment_11_a3b5e2047185bbd3972ada07ddb79172._comment @@ -0,0 +1,31 @@ +[[!comment format=mdwn + username="joey" + subject="""comment 11""" + date="2018-08-27T17:20:19Z" + content=""" +Actually the "some" in-git file showing as modified in status is not the +same as the git-annex get/drop showing modified in status problem. I've +fixed the latter and the former still happens. + +What's happening is, git runs the clean filter, I think because this +is a fresh clone and it's not cleaned it yet. That looks at +annex.largefiles (lack of) configuration and concludes this file belongs in +the annex, so it ingests it. Rest follows the same as what I described in +comment #2. + +So yeah, that's a real problem, you clone a repo and there are suddenly +changes that mess up the painstakingly set up in-annex/in-git division. +(Note that it does not need to involve an upgrade.) + +Only fix I can imagine is my old idea: + +> I suppose one way this could be improved is for git annex smudge --clean to +> check if a file was checked into git as a non-annexed file before, and then +> avoid cleaning it at all. But then if someone had a non-annexed file and it got +> big and they wanted to add it annexed, such a change would cause a problem.. + +I suppose we could get around that problem with a new git-annex command +that converts a non-annexed file to an annexed file. Without a command, +this would also work: mv the file to a temp name, followed by git-annex add, +and then git mv back. +"""]] diff --git a/doc/bugs/Too_difficult_if_not_impossible_to_explicitly_add__47__keep_file_under_git___40__not_annex__41___in_v6_without_employing_.gitattributes/comment_12_db64ae67b8aa974bd6d00aad005058ac._comment b/doc/bugs/Too_difficult_if_not_impossible_to_explicitly_add__47__keep_file_under_git___40__not_annex__41___in_v6_without_employing_.gitattributes/comment_12_db64ae67b8aa974bd6d00aad005058ac._comment new file mode 100644 index 0000000000..1b373f801e --- /dev/null +++ b/doc/bugs/Too_difficult_if_not_impossible_to_explicitly_add__47__keep_file_under_git___40__not_annex__41___in_v6_without_employing_.gitattributes/comment_12_db64ae67b8aa974bd6d00aad005058ac._comment @@ -0,0 +1,31 @@ +[[!comment format=mdwn + username="joey" + subject="""comment 12""" + date="2018-08-27T18:08:24Z" + content=""" +Hmm, I think if annex.largefiles is configured, it should be honored. So, +the ways to convert documented on + will work +with only a minor modification. + +If annex.largefiles is not configured, it can check if the file was annexed +or not before, and maintain the status quo. + +At least to start with I am only going to do it in the clean filter, so +`git annex add` behavior won't change, it default to annexing. It may make +sense to make `git annex add` consistent with `git add`, but that would +also need to affect v5 repositories, probably, and I don't want to entangle +a v6 bug fix with a v5 behavior change. Also, the risk of a +`git annex add` with the wrong annex.largefiles temp setting +is much smaller than the risk of forgetting to temp set annex.largefiles +when running `git commit -a`. Also, I seem to remember another todo item +discussing making this change to `git annex add` and would need to revisit +the thinking in that. + +Note that a .gitattributes file might only configure largefiles for eg, +"*.c" and not for other files, thus implicitly accepting the default +largefiles for the rest. Should then `Makefile` be treated as having +largefiles configured or not? I lean toward treating it as not configured, +because then when the user temporarily overrides largefiles to add `Makefile` +to git, a modification won't accidentially go to the annex. +"""]] diff --git a/doc/bugs/Too_difficult_if_not_impossible_to_explicitly_add__47__keep_file_under_git___40__not_annex__41___in_v6_without_employing_.gitattributes/comment_13_5f11262639d60ef63a44de09b191e0b6._comment b/doc/bugs/Too_difficult_if_not_impossible_to_explicitly_add__47__keep_file_under_git___40__not_annex__41___in_v6_without_employing_.gitattributes/comment_13_5f11262639d60ef63a44de09b191e0b6._comment new file mode 100644 index 0000000000..3b68a560cf --- /dev/null +++ b/doc/bugs/Too_difficult_if_not_impossible_to_explicitly_add__47__keep_file_under_git___40__not_annex__41___in_v6_without_employing_.gitattributes/comment_13_5f11262639d60ef63a44de09b191e0b6._comment @@ -0,0 +1,21 @@ +[[!comment format=mdwn + username="joey" + subject="""comment 13""" + date="2018-08-28T14:02:37Z" + content=""" +However, that leaves the case where .gitattributes configures +annex.largefiles, but that's been overridden for a file to add it to git, +and then the repo is cloned and initted with --version=6 (or upgraded). + +Turns out that calling git status before enabling the smudge filter +prevents git from getting confused about the file being modified in this +case. + +In the fresh clone, git has not populated the index with stat info +yet, and so it later runs the clean filter on the file, and that +respects the largefiles configuration, so the way the file is +stored in git is not taken into account. + +Worked around this by adding a `git status` call to the v6 +initialization/upgrade. +"""]] diff --git a/doc/bugs/Too_difficult_if_not_impossible_to_explicitly_add__47__keep_file_under_git___40__not_annex__41___in_v6_without_employing_.gitattributes/comment_1_045f90b5693de55958c6ad1b825cbc9e._comment b/doc/bugs/Too_difficult_if_not_impossible_to_explicitly_add__47__keep_file_under_git___40__not_annex__41___in_v6_without_employing_.gitattributes/comment_1_045f90b5693de55958c6ad1b825cbc9e._comment new file mode 100644 index 0000000000..b1f3d49b43 --- /dev/null +++ b/doc/bugs/Too_difficult_if_not_impossible_to_explicitly_add__47__keep_file_under_git___40__not_annex__41___in_v6_without_employing_.gitattributes/comment_1_045f90b5693de55958c6ad1b825cbc9e._comment @@ -0,0 +1,10 @@ +[[!comment format=mdwn + username="joey" + subject="""comment 1""" + date="2017-01-30T16:07:31Z" + content=""" +> Even if -c annex.largefiles=nothing is used with git add, then git commit commits file into annex. + +I don't reproduce that problem; as long as the file is staged with that -c, +git commit commits what's staged. +"""]] diff --git a/doc/bugs/Too_difficult_if_not_impossible_to_explicitly_add__47__keep_file_under_git___40__not_annex__41___in_v6_without_employing_.gitattributes/comment_2_ead7e348547da0598a82be8d064c0c08._comment b/doc/bugs/Too_difficult_if_not_impossible_to_explicitly_add__47__keep_file_under_git___40__not_annex__41___in_v6_without_employing_.gitattributes/comment_2_ead7e348547da0598a82be8d064c0c08._comment new file mode 100644 index 0000000000..f3f287c009 --- /dev/null +++ b/doc/bugs/Too_difficult_if_not_impossible_to_explicitly_add__47__keep_file_under_git___40__not_annex__41___in_v6_without_employing_.gitattributes/comment_2_ead7e348547da0598a82be8d064c0c08._comment @@ -0,0 +1,36 @@ +[[!comment format=mdwn + username="joey" + subject="""comment 2""" + date="2017-01-30T16:11:04Z" + content=""" +What's really going on with `git status`: + +* It runs `git annex smudge --clean` +* That ingests the file into the annex, but does not change what's staged at all. +* When you run `git diff`, it again cleans the file and displays + the difference between what is staged (the unannexed file) and the + cleaned version (the annexed file). +* Notice that if you run `git annex add file12 -c annex.largefiles=nothing` + after this, the file gets staged in non-annexed file, as you'd expect. + +So the only problem I see here is `git annex smudge --clean` +is ingesting a file into the annex when `git status` is not going to +update the working tree to use the pointer to that file. But the clean +hook interface doesn't provide it any way to know why git is asking for a +file to be cleaned, so it has to always do that. + +If this is a small file suitable for being checked into git, the overhead of +having a copy of it in the annex shouldn't matter much, and annex.thin +will even make that copy be a hard link. + +I suppose one way this could be improved is for `git annex smudge --clean` +to check if a file was checked into git as a non-annexed file before, +and then avoid cleaning it at all. But then if someone had a non-annexed +file and it got big and they wanted to add it annexed, such a change +would cause a problem.. + +Or git's smudge/clean interface could be improved so that the clean +filter can know why it's being called, and so avoid ingesting files +unnecessarily. IIRC my patch to improve the git interface did do that, but +unfortunately it's stalled getting accepted into git. +"""]] diff --git a/doc/bugs/Too_difficult_if_not_impossible_to_explicitly_add__47__keep_file_under_git___40__not_annex__41___in_v6_without_employing_.gitattributes/comment_3_ff0746f08600fb9729b8e1773317a52d._comment b/doc/bugs/Too_difficult_if_not_impossible_to_explicitly_add__47__keep_file_under_git___40__not_annex__41___in_v6_without_employing_.gitattributes/comment_3_ff0746f08600fb9729b8e1773317a52d._comment new file mode 100644 index 0000000000..3f823cab4f --- /dev/null +++ b/doc/bugs/Too_difficult_if_not_impossible_to_explicitly_add__47__keep_file_under_git___40__not_annex__41___in_v6_without_employing_.gitattributes/comment_3_ff0746f08600fb9729b8e1773317a52d._comment @@ -0,0 +1,9 @@ +[[!comment format=mdwn + username="yarikoptic" + avatar="http://cdn.libravatar.org/avatar/f11e9c84cb18d26a1748c33b48c924b4" + subject="comment 3" + date="2017-02-06T20:46:29Z" + content=""" +thought about it... +I guess it would then be a limitation, sorry -- a feature, of v6 mode indeed that all instructions on what files should go into annex vs git should always reside within .gitattributes, to be in effect. Although very powerful, might be limiting in case of a naive user. Any chance you would pick up on pushing your suggested changes forward (i.e. keep nagging like I do I guess) within git? ;) +"""]] diff --git a/doc/bugs/Too_difficult_if_not_impossible_to_explicitly_add__47__keep_file_under_git___40__not_annex__41___in_v6_without_employing_.gitattributes/comment_4_598622df322c56da71320d1b15e0348f._comment b/doc/bugs/Too_difficult_if_not_impossible_to_explicitly_add__47__keep_file_under_git___40__not_annex__41___in_v6_without_employing_.gitattributes/comment_4_598622df322c56da71320d1b15e0348f._comment new file mode 100644 index 0000000000..182d19e2e8 --- /dev/null +++ b/doc/bugs/Too_difficult_if_not_impossible_to_explicitly_add__47__keep_file_under_git___40__not_annex__41___in_v6_without_employing_.gitattributes/comment_4_598622df322c56da71320d1b15e0348f._comment @@ -0,0 +1,10 @@ +[[!comment format=mdwn + username="benjamin.poldrack@d09ccff6d42dd20277610b59867cf7462927b8e3" + nickname="benjamin.poldrack" + avatar="http://cdn.libravatar.org/avatar/5c1a901caa7c2cfeeb7e17e786c5230d" + subject="comment 4" + date="2017-02-09T09:05:34Z" + content=""" +I want to point out an issue, that seems to be unnoticed: +While we (datalad) might be able to deal with this effect regarding git status in terms of space, it has another consequence. As the scripts are showing it also leads to the file marked as modified, if it is rewritten (very same content) and therefore leaves the repository dirty. To me this a pretty weird effect and I don't understand yet what this is caused by. +"""]] diff --git a/doc/bugs/Too_difficult_if_not_impossible_to_explicitly_add__47__keep_file_under_git___40__not_annex__41___in_v6_without_employing_.gitattributes/comment_5_9707d9dda7ebb2c94c71f1ea2f99064d._comment b/doc/bugs/Too_difficult_if_not_impossible_to_explicitly_add__47__keep_file_under_git___40__not_annex__41___in_v6_without_employing_.gitattributes/comment_5_9707d9dda7ebb2c94c71f1ea2f99064d._comment new file mode 100644 index 0000000000..cf1822b592 --- /dev/null +++ b/doc/bugs/Too_difficult_if_not_impossible_to_explicitly_add__47__keep_file_under_git___40__not_annex__41___in_v6_without_employing_.gitattributes/comment_5_9707d9dda7ebb2c94c71f1ea2f99064d._comment @@ -0,0 +1,8 @@ +[[!comment format=mdwn + username="db48x" + avatar="http://cdn.libravatar.org/avatar/ad2688127feb555a92154b16d8eeb5d3" + subject="comment 5" + date="2017-02-14T21:10:13Z" + content=""" +I've got a repository that has a small file that gets updated regularly. I've given up trying to keep it unannexed and just started unlocking it every time I run the automated script that updates it. +"""]] diff --git a/doc/bugs/Too_difficult_if_not_impossible_to_explicitly_add__47__keep_file_under_git___40__not_annex__41___in_v6_without_employing_.gitattributes/comment_6_56c8ddb604b54b8a5cfd077225971582._comment b/doc/bugs/Too_difficult_if_not_impossible_to_explicitly_add__47__keep_file_under_git___40__not_annex__41___in_v6_without_employing_.gitattributes/comment_6_56c8ddb604b54b8a5cfd077225971582._comment new file mode 100644 index 0000000000..8999250aa1 --- /dev/null +++ b/doc/bugs/Too_difficult_if_not_impossible_to_explicitly_add__47__keep_file_under_git___40__not_annex__41___in_v6_without_employing_.gitattributes/comment_6_56c8ddb604b54b8a5cfd077225971582._comment @@ -0,0 +1,13 @@ +[[!comment format=mdwn + username="joey" + subject="""comment 6""" + date="2017-03-02T17:58:22Z" + content=""" +The several comments after my analysis above seem to still be under the +impression that it's somehow hard to keep files stored in git, not annexed +in v6 mode. But as my analysis shows, that is not what is happening at all +in the test case given. + +If you have a situation there that problem occurs, please share it. +Otherwise, I will close this bug report. +"""]] diff --git a/doc/bugs/Too_difficult_if_not_impossible_to_explicitly_add__47__keep_file_under_git___40__not_annex__41___in_v6_without_employing_.gitattributes/comment_7_6b2a763f3d07f0cdc035008638e08194._comment b/doc/bugs/Too_difficult_if_not_impossible_to_explicitly_add__47__keep_file_under_git___40__not_annex__41___in_v6_without_employing_.gitattributes/comment_7_6b2a763f3d07f0cdc035008638e08194._comment new file mode 100644 index 0000000000..7c7d1b5afe --- /dev/null +++ b/doc/bugs/Too_difficult_if_not_impossible_to_explicitly_add__47__keep_file_under_git___40__not_annex__41___in_v6_without_employing_.gitattributes/comment_7_6b2a763f3d07f0cdc035008638e08194._comment @@ -0,0 +1,14 @@ +[[!comment format=mdwn + username="benjamin.poldrack@d09ccff6d42dd20277610b59867cf7462927b8e3" + nickname="benjamin.poldrack" + avatar="http://cdn.libravatar.org/avatar/5c1a901caa7c2cfeeb7e17e786c5230d" + subject="comment 7" + date="2017-03-27T09:03:10Z" + content=""" +You are right, if we stage that file again with annex.largefiles option, it stays in git. +But still keeping a file in git is troublesome. For example: Take a repository with an annexed file and a file in git, clone it and the call git annex init --version=6 on the clone. +This will lead to a dirty repository, where git status as well as git annex status are stating, that the file in git has unstaged modifications. +I'm not sure, whether this is actually related but I guess it is. Causing fresh clones to be dirty is at least a strange consequence for having files directly in git. + +[[ben]] +"""]] diff --git a/doc/bugs/Too_difficult_if_not_impossible_to_explicitly_add__47__keep_file_under_git___40__not_annex__41___in_v6_without_employing_.gitattributes/comment_8_5aa63bf0d8e4145974702a86b36bd435._comment b/doc/bugs/Too_difficult_if_not_impossible_to_explicitly_add__47__keep_file_under_git___40__not_annex__41___in_v6_without_employing_.gitattributes/comment_8_5aa63bf0d8e4145974702a86b36bd435._comment new file mode 100644 index 0000000000..24067287d0 --- /dev/null +++ b/doc/bugs/Too_difficult_if_not_impossible_to_explicitly_add__47__keep_file_under_git___40__not_annex__41___in_v6_without_employing_.gitattributes/comment_8_5aa63bf0d8e4145974702a86b36bd435._comment @@ -0,0 +1,25 @@ +[[!comment format=mdwn + username="joey" + subject="""@ben: what?""" + date="2017-04-07T16:37:31Z" + content=""" +I don't know what you mean with `git annex init` in a v6 repo somehow +doing something to a non-annexed file. That would be extemely surprising +if it happened, and it does not happen when I try to follow your +instructions. + + joey@darkstar:~/tmp/v62>git annex init --version=6 + init (merging origin/git-annex into git-annex...) + (recording state in git...) + ok + (recording state in git...) + joey@darkstar:~/tmp/v62>ls + x y@ + joey@darkstar:~/tmp/v62>git annex get y + get y (from origin...) (checksum...) ok + (recording state in git...) + joey@darkstar:~/tmp/v62>git status + On branch master + Your branch is up-to-date with 'origin/master'. + nothing to commit, working tree clean +"""]] diff --git a/doc/bugs/Too_difficult_if_not_impossible_to_explicitly_add__47__keep_file_under_git___40__not_annex__41___in_v6_without_employing_.gitattributes/comment_9_5a22839b8dc11965a879dd2654bd5d60._comment b/doc/bugs/Too_difficult_if_not_impossible_to_explicitly_add__47__keep_file_under_git___40__not_annex__41___in_v6_without_employing_.gitattributes/comment_9_5a22839b8dc11965a879dd2654bd5d60._comment new file mode 100644 index 0000000000..c2a1a3dc1b --- /dev/null +++ b/doc/bugs/Too_difficult_if_not_impossible_to_explicitly_add__47__keep_file_under_git___40__not_annex__41___in_v6_without_employing_.gitattributes/comment_9_5a22839b8dc11965a879dd2654bd5d60._comment @@ -0,0 +1,76 @@ +[[!comment format=mdwn + username="benjamin.poldrack@d09ccff6d42dd20277610b59867cf7462927b8e3" + nickname="benjamin.poldrack" + avatar="http://cdn.libravatar.org/avatar/5c1a901caa7c2cfeeb7e17e786c5230d" + subject="@joey: Sorry ..." + date="2017-09-14T12:00:09Z" + content=""" +... I somehow managed to miss your response. Now, since a somewhat related topic is emerging again with datalad, I looked into this one again. +I reproduced, what I described before, but I noticed that it involves kind of an implicit upgrade from a V5 to V6 repository. + +First, let's have v5 repo with a file in git and a file in annex: + + + ben@tree /tmp % mkdir origin + ben@tree /tmp % cd origin + ben@tree /tmp/origin % git init + Initialized empty Git repository in /tmp/origin/.git/ + ben@tree /tmp/origin % git annex init + init ok + (recording state in git...) + ben@tree /tmp/origin % echo some > some + ben@tree /tmp/origin % git add some + ben@tree /tmp/origin % echo something different > annex + ben@tree /tmp/origin % git annex add annex + add annex ok + (recording state in git...) + ben@tree /tmp/origin % git commit -m \"initial\" + [master (root-commit) 8b96354] initial + 2 files changed, 2 insertions(+) + create mode 120000 annex + create mode 100644 some + ben@tree /tmp/origin % ll + total 376 + drwxr-xr-x 3 ben ben 4096 Sep 14 13:34 . + drwxrwxrwt 24 root root 364544 Sep 14 13:33 .. + lrwxrwxrwx 1 ben ben 180 Sep 14 13:34 annex -> .git/annex/objects/g7/4P/SHA256E-s20--b6105173f468fc7afa866aa469220cd56e5200db590be89922239a38631379c9/SHA256E-s20--b6105173f468fc7afa866aa469220cd56e5200db590be89922239a38631379c9 + drwxr-xr-x 9 ben ben 4096 Sep 14 13:34 .git + -rw-r--r-- 1 ben ben 5 Sep 14 13:34 some + ben@tree /tmp/origin % git ls-files + annex + some + ben@tree /tmp/origin % git annex find + annex + +Now, clone this repository: + + ben@tree /tmp/origin % cd .. + ben@tree /tmp % git clone origin cloned + Cloning into 'cloned'... + done. + ben@tree /tmp % cd cloned + +And annex-init as a v6 repository: + + ben@tree /tmp/cloned % git annex init --version=6 + init (merging origin/git-annex into git-annex...) + (recording state in git...) + (scanning for unlocked files...) + ok + (recording state in git...) + ben@tree /tmp/cloned % git status + On branch master + Your branch is up-to-date with 'origin/master'. + + Changes not staged for commit: + (use \"git add ...\" to update what will be committed) + (use \"git checkout -- ...\" to discard changes in working directory) + + modified: some + + no changes added to commit (use \"git add\" and/or \"git commit -a\") + + +This kind of \"implicit\" upgrade might not be a common use case, but the result seems to be a bit weird nonetheless. + +"""]] diff --git a/doc/bugs/TransferScanner_crashed__58___fd__58__59__58___hGetLine__58___end_of_file.mdwn b/doc/bugs/TransferScanner_crashed__58___fd__58__59__58___hGetLine__58___end_of_file.mdwn new file mode 100644 index 0000000000..40cd3ebb9c --- /dev/null +++ b/doc/bugs/TransferScanner_crashed__58___fd__58__59__58___hGetLine__58___end_of_file.mdwn @@ -0,0 +1,50 @@ +### Please describe the problem. +git-annex assistant reports a warning: +TransferScanner crashed: fd:59: hGetLine: end of file + +### What steps will reproduce the problem? +I don't know. Just run git-annex assistant and wait for this to happen. + +### What version of git-annex are you using? On what operating system? +Version: 5.20140709 +Build flags: Assistant Webapp Webapp-secure Pairing Testsuite S3 WebDAV Inotify DBus XMPP DNS Feeds Quvi TDFA CryptoHash + +Gentoo + +### Please provide any additional information below. + + +[[!format sh """ +# If you can, paste a complete transcript of the problem occurring here. +# If the problem is with the git-annex assistant, paste in .git/annex/daemon.log +error: object file /home/crabman/annex/.git/objects/97/3b57761ebe88b11e1d0d2c70c8d3b48530202f is empty +fatal: loose object 973b57761ebe88b11e1d0d2c70c8d3b48530202f (stored in /home/crabman/annex/.git/objects/97/3b57761ebe88b11e1d0d2c70c8d3b48530202f) is corrupt +error: object file /home/crabman/annex/.git/objects/97/3b57761ebe88b11e1d0d2c70c8d3b48530202f is empty +fatal: loose object 973b57761ebe88b11e1d0d2c70c8d3b48530202f (stored in /home/crabman/annex/.git/objects/97/3b57761ebe88b11e1d0d2c70c8d3b48530202f) is corrupt +error: object file /home/crabman/annex/.git/objects/97/3b57761ebe88b11e1d0d2c70c8d3b48530202f is empty +fatal: loose object 973b57761ebe88b11e1d0d2c70c8d3b48530202f (stored in /home/crabman/annex/.git/objects/97/3b57761ebe88b11e1d0d2c70c8d3b48530202f) is corrupt +error: object file /home/crabman/annex/.git/objects/97/3b57761ebe88b11e1d0d2c70c8d3b48530202f is empty +fatal: loose object 973b57761ebe88b11e1d0d2c70c8d3b48530202f (stored in /home/crabman/annex/.git/objects/97/3b57761ebe88b11e1d0d2c70c8d3b48530202f) is corrupt +fd:31: hGetLine: end of file +fd:32: hGetLine: end of file +fd:33: hGetLine: end of file +fd:34: hGetLine: end of file +fd:38: hGetLine: end of file +fd:41: hGetLine: end of file +fd:41: hGetLine: end of file +fd:41: hGetLine: end of file +fd:41: hGetLine: end of file +fd:41: hGetLine: end of file +fd:41: hGetLine: end of file +fd:43: hGetLine: end of file +fd:41: hGetLine: end of file +fd:41: hGetLine: end of file +fd:41: hGetLine: end of file +fd:41: hGetLine: end of file +fd:41: hGetLine: end of file +fd:41: hGetLine: end of file +fd:43: hGetLine: end of file + + +# End of transcript or log. +"""]] diff --git a/doc/bugs/TransferScanner_crashed__58___fd__58__59__58___hGetLine__58___end_of_file/comment_1_a1392b92efdff82783a4b0cc2c3c7f2f._comment b/doc/bugs/TransferScanner_crashed__58___fd__58__59__58___hGetLine__58___end_of_file/comment_1_a1392b92efdff82783a4b0cc2c3c7f2f._comment new file mode 100644 index 0000000000..101b234047 --- /dev/null +++ b/doc/bugs/TransferScanner_crashed__58___fd__58__59__58___hGetLine__58___end_of_file/comment_1_a1392b92efdff82783a4b0cc2c3c7f2f._comment @@ -0,0 +1,14 @@ +[[!comment format=mdwn + username="http://joeyh.name/" + ip="209.250.56.2" + subject="comment 1" + date="2014-07-14T18:47:21Z" + content=""" +You did not seem to paste the whole log, but it's clear that your git repository is corrupt. You might try running `git annex repair` inside it. + +This was probably caused by a disk corruption or if you're lucky just a hard shutdown that caused files to get truncated. + +The assistant tries to detect corrupt repositories on startup, but it's hard to detect when a repository has a few corrupt objects like this. This is why the webapp suggests you set up periodic repository consistency checks, which should eventually repair this kind of problem. + +Perhaps it would make sense for the webapp, when a thread crashes, to provide a button to run an consistency check immediately. +"""]] diff --git a/doc/bugs/TransferScanner_crashed__58___fd__58__59__58___hGetLine__58___end_of_file/comment_2_5e153b7c59c474988fe551a505e545bc._comment b/doc/bugs/TransferScanner_crashed__58___fd__58__59__58___hGetLine__58___end_of_file/comment_2_5e153b7c59c474988fe551a505e545bc._comment new file mode 100644 index 0000000000..c8d4a988af --- /dev/null +++ b/doc/bugs/TransferScanner_crashed__58___fd__58__59__58___hGetLine__58___end_of_file/comment_2_5e153b7c59c474988fe551a505e545bc._comment @@ -0,0 +1,8 @@ +[[!comment format=mdwn + username="jg123h12jh3y12g3y" + ip="86.62.100.131" + subject="comment 2" + date="2014-07-15T07:58:15Z" + content=""" +But I do have periodice consistency checks turned on. I think git-annex assistant should tell the user what the heck is going on and maybe try to repair it automatically. +"""]] diff --git a/doc/bugs/Transfers_continue_after_daemon_stopped.mdwn b/doc/bugs/Transfers_continue_after_daemon_stopped.mdwn new file mode 100644 index 0000000000..0d9b4f6488 --- /dev/null +++ b/doc/bugs/Transfers_continue_after_daemon_stopped.mdwn @@ -0,0 +1,5 @@ +After creating a new pairing, I stopped the daemon through the webapp while it was syncing. The webapp shut down and was no longer accessible, but git-annex continued running in the background, along with git-annex-shell, and they continued to run new transfers with new rsync processes. This continued until I killed them all. + +I expected that when I stopped the daemon in the webapp, all git-annex processes and all transfers would stop. + +Using the 20131101 tarball. diff --git a/doc/bugs/Transfers_continue_after_daemon_stopped/comment_1_39eb527d64367e6762281246f1d49b1f._comment b/doc/bugs/Transfers_continue_after_daemon_stopped/comment_1_39eb527d64367e6762281246f1d49b1f._comment new file mode 100644 index 0000000000..9d504f743c --- /dev/null +++ b/doc/bugs/Transfers_continue_after_daemon_stopped/comment_1_39eb527d64367e6762281246f1d49b1f._comment @@ -0,0 +1,12 @@ +[[!comment format=mdwn + username="http://joeyh.name/" + ip="209.250.56.47" + subject="comment 1" + date="2013-11-05T16:01:14Z" + content=""" +I've checked, and shutting down the daemon does cause it to stop any transfers it is running. + +However, this does not stop other transfers initiated by the other paired computer. + +I'm ambivilant about whether local pairing should only allow transfers when both daemons are running. +"""]] diff --git a/doc/bugs/Tries_to_upload_to_remote_although_remote_is_dead.mdwn b/doc/bugs/Tries_to_upload_to_remote_although_remote_is_dead.mdwn new file mode 100644 index 0000000000..99055e7c5d --- /dev/null +++ b/doc/bugs/Tries_to_upload_to_remote_although_remote_is_dead.mdwn @@ -0,0 +1,51 @@ +What steps will reproduce the problem? + +I added a (encrypted) ssh remote and everything worked fine. Now I marked the remote as dead, but git-annex still tries to upload to this remote. I recognize this because it asks for my ssh and gpg keys passwords. + +While transfering (or asking for the password), `git annex status` shows the following: +
    +supported backends: SHA256E SHA1E SHA512E SHA224E SHA384E SHA256 SHA1 SHA512 SHA224 SHA384 WORM URL
+supported remote types: git S3 bup directory rsync web hook
+trusted repositories: 0
+semitrusted repositories: 2
+	00000000-0000-0000-0000-000000000001 -- web
+ 	cd16b9c6-f464-11e1-9845-8749687232d2 -- here (Dell)
+untrusted repositories: 0
+dead repositories: 7
+	11379fa0-ecd6-49e2-9bec-24fc19cc7b9f -- vserver.dbruhn.de_annex
+ 	2195e036-d2ef-4357-8c89-a9aaec23ebdc -- vserver-plain
+ 	4d066ea1-fb9f-45fd-990a-5c5c836f530e -- inTmp
+ 	bb276045-6ba6-488f-88d0-39a3c5f5134d -- vserver-enc
+ 	c49f3372-3fcf-49fc-b626-73ba4454c172 -- annexBare (bareAnnex)
+ 	e52645b3-bfb6-457d-b281-967353919e29 -- AnnexUSBFAT
+ 	ea3d6acc-716c-48e8-9b6b-993b90dcc1db -- vserver2
+transfers in progress: 
+	uploading Schmidt/somefile.m4a
+
+
+ to vserver2
+available local disk space: 43 gigabytes (+1 megabyte reserved)
+temporary directory size: 389 megabytes (clean up with git-annex unused)
+local annex keys: 23
+local annex size: 396 megabytes
+known annex keys: 19
+known annex size: 396 megabytes
+bloom filter size: 16 mebibytes (0% full)
+backend usage: 
+	SHA256E: 42
+
    + +As you can see, the `vserver2` remote is marked as dead but git-annex still tries to upload. This problem keeps occuring even after restarts. + +What is the expected output? What do you see instead? + +If I do not get the `dead` status wrong, git-annex should not use these remotes. + + +What version of git-annex are you using? On what operating system? + +git-annex HEAD from yesterdays git. Ubuntu 12.10 + +Please provide any additional information below. + +[[!tag /design/assistant moreinfo]] diff --git a/doc/bugs/Tries_to_upload_to_remote_although_remote_is_dead/comment_1_108b3984891f82429430b503cddfb3c1._comment b/doc/bugs/Tries_to_upload_to_remote_although_remote_is_dead/comment_1_108b3984891f82429430b503cddfb3c1._comment new file mode 100644 index 0000000000..f8a85cb166 --- /dev/null +++ b/doc/bugs/Tries_to_upload_to_remote_although_remote_is_dead/comment_1_108b3984891f82429430b503cddfb3c1._comment @@ -0,0 +1,10 @@ +[[!comment format=mdwn + username="http://joeyh.name/" + ip="4.152.108.194" + subject="comment 1" + date="2012-11-04T20:01:35Z" + content=""" +Are you using the git-annex assistant? It's supposed to have code in it to prevent it from using dead remotes. Perhaps that is not working. + +When not using the git-annex assistant, nothing prevents git-annex from trying to continue to use dead remotes. I think this is ok; maybe the dead remote is not 100% dead yet; if it is, the remote can be removed from git's configuration to prevent it being used of course. +"""]] diff --git a/doc/bugs/Tries_to_upload_to_remote_although_remote_is_dead/comment_2_fa5b1bc26ed3e5bfe48441490c94fe3a._comment b/doc/bugs/Tries_to_upload_to_remote_although_remote_is_dead/comment_2_fa5b1bc26ed3e5bfe48441490c94fe3a._comment new file mode 100644 index 0000000000..fd0d9d2ab9 --- /dev/null +++ b/doc/bugs/Tries_to_upload_to_remote_although_remote_is_dead/comment_2_fa5b1bc26ed3e5bfe48441490c94fe3a._comment @@ -0,0 +1,8 @@ +[[!comment format=mdwn + username="http://www.dbruhn.de/" + nickname="Dominik" + subject="comment 2" + date="2012-11-08T14:13:00Z" + content=""" +Actually this happens when using the assistant. +"""]] diff --git a/doc/bugs/Tries_to_upload_to_remote_although_remote_is_dead/comment_3_0a785b5dfbf4eef30854d6bedb12b7d1._comment b/doc/bugs/Tries_to_upload_to_remote_although_remote_is_dead/comment_3_0a785b5dfbf4eef30854d6bedb12b7d1._comment new file mode 100644 index 0000000000..e839e7a156 --- /dev/null +++ b/doc/bugs/Tries_to_upload_to_remote_although_remote_is_dead/comment_3_0a785b5dfbf4eef30854d6bedb12b7d1._comment @@ -0,0 +1,10 @@ +[[!comment format=mdwn + username="http://joeyh.name/" + ip="4.252.11.120" + subject="comment 3" + date="2012-11-13T18:00:42Z" + content=""" +I've tried & failed to reproduce this using the assistant. When I mark a repo as dead, and go into Configuration -> Manage repositories, the repository is not shown in the list of repositories it'll sync to, and it certainly doesn't upload any files to it. + + +"""]] diff --git a/doc/bugs/Unable_to_take_transfer_lock.mdwn b/doc/bugs/Unable_to_take_transfer_lock.mdwn new file mode 100644 index 0000000000..e1d4f0a117 --- /dev/null +++ b/doc/bugs/Unable_to_take_transfer_lock.mdwn @@ -0,0 +1,46 @@ +### Please describe the problem. + +`git annex get --from=myremote' show "transfer already in progress, or unable to take transfer lock". + +**Likely Problem discovered!** I copied the whole remote repo (was residing on a Linux machine accessed via SMB) and pasted it into my Mac. The problem was resolved. It is likely something to do with the fact that Mac OS filesystem is **case-insensitive**. I discovered this issue when I tried to copy that remote repo with ``cp`` rather than ``tar``ing it first. + + +### What steps will reproduce the problem? + +Remote is on a Linux machine. +Do `git annex get' on a Mac OS. +Do the same on another Linux machine. + +The git annex client on the Linux machine works. +Mac OS doesn't. + +rsync version for all machines: 3.1.1 + +### What version of git-annex are you using? On what operating system? + +git-annex 5.20151019 + +Lubuntu 15.10. +Mac OS X 10.11.1 (El Capitan) + + +### Please provide any additional information below. + +[[!format sh """ +# If you can, paste a complete transcript of the problem occurring here. +# If the problem is with the git-annex assistant, paste in .git/annex/daemon.log + + +# End of transcript or log. +"""]] + +### Have you had any luck using git-annex before? (Sometimes we get tired of reading bug reports all day and a lil' positive end note does wonders) + +Yes. + +It's wonderful. This should be a problem with Mac OS. + +> [[done]]; this appears to be the same bug as +> [[parallel_get_can_fail_some_downloads_and_require_re-getting_]], +> which was fixed in 6.20170520. +> --[[Joey]] diff --git a/doc/bugs/Unable_to_take_transfer_lock/comment_1_4eff7f261659202084d32e068b93f77b._comment b/doc/bugs/Unable_to_take_transfer_lock/comment_1_4eff7f261659202084d32e068b93f77b._comment new file mode 100644 index 0000000000..3db482bc19 --- /dev/null +++ b/doc/bugs/Unable_to_take_transfer_lock/comment_1_4eff7f261659202084d32e068b93f77b._comment @@ -0,0 +1,20 @@ +[[!comment format=mdwn + username="joey" + subject="""comment 1""" + date="2015-11-30T19:46:05Z" + content=""" +The most likely reason for that error message would be if a transfer +of the same file was already in progress by another git-annex process. + +It might also be the case that if you're trying to access a git-annex +repository via SMB, the POSIX fcntl locking that git-annex needs to do +don't work. This is often the case with various network filesystems. +Solution is to make the url for the git remote be a ssh:// url instead. + +I don't know what OSX's case-insensative complication could have to do with +this, although it certianly seems to be a good way to get completely +unexpected behavior. + +Is there an actual bug here? What are the steps to reproduce +it? +"""]] diff --git a/doc/bugs/Unable_to_take_transfer_lock/comment_2_88a6132696bd299fc40359ff56657596._comment b/doc/bugs/Unable_to_take_transfer_lock/comment_2_88a6132696bd299fc40359ff56657596._comment new file mode 100644 index 0000000000..3a7426cf07 --- /dev/null +++ b/doc/bugs/Unable_to_take_transfer_lock/comment_2_88a6132696bd299fc40359ff56657596._comment @@ -0,0 +1,14 @@ +[[!comment format=mdwn + username="jhannwong@c9c7a67b5632a4bbc0c959cfeb3d340e02f28565" + nickname="jhannwong" + subject="Am pretty sure this is a filesystem issue" + date="2015-12-01T12:45:50Z" + content=""" +Same git-annex repo. One resides on Linux; the other on Mac OS. + +The one on Mac OS can be accessed (`git annex get') by git clones on Linux and on Mac OS. + +The one on Linux can only be accessed by git clones on Linux. + +I even tracked it down to folders having the same name --- 'Fs' vs 'FS'. +"""]] diff --git a/doc/bugs/Unable_to_take_transfer_lock/comment_3_8ba3d4eb6ff857d6a33e5a84808fe4fa._comment b/doc/bugs/Unable_to_take_transfer_lock/comment_3_8ba3d4eb6ff857d6a33e5a84808fe4fa._comment new file mode 100644 index 0000000000..39b3f7a9e8 --- /dev/null +++ b/doc/bugs/Unable_to_take_transfer_lock/comment_3_8ba3d4eb6ff857d6a33e5a84808fe4fa._comment @@ -0,0 +1,7 @@ +[[!comment format=mdwn + username="joey" + subject="""comment 3""" + date="2015-12-02T17:01:36Z" + content=""" +What specific path names in the git repository did you track it down to? +"""]] diff --git a/doc/bugs/Unable_to_take_transfer_lock/comment_4_7468862a6d4ec47320071aa52ae4072a._comment b/doc/bugs/Unable_to_take_transfer_lock/comment_4_7468862a6d4ec47320071aa52ae4072a._comment new file mode 100644 index 0000000000..a3ef3c0de0 --- /dev/null +++ b/doc/bugs/Unable_to_take_transfer_lock/comment_4_7468862a6d4ec47320071aa52ae4072a._comment @@ -0,0 +1,16 @@ +[[!comment format=mdwn + username="luca@b7e273474d087a5c7fc4428503f21d5624636d9f" + nickname="luca" + subject="comment 4" + date="2016-02-05T06:43:11Z" + content=""" +I get this error on OS X when setting the `-J` parameter to > 1, this works: + + git annex copy --to usb -J1 --auto + +This consistently throughs the error (but seems to copy everything): + + git annex copy --to usb -J2 --auto + +I'm running 6.20160126 on version 5 repositories. +"""]] diff --git a/doc/bugs/Unable_to_take_transfer_lock/comment_5_e33957c5a75b06d77b188a10b69a39fd._comment b/doc/bugs/Unable_to_take_transfer_lock/comment_5_e33957c5a75b06d77b188a10b69a39fd._comment new file mode 100644 index 0000000000..89ac13197d --- /dev/null +++ b/doc/bugs/Unable_to_take_transfer_lock/comment_5_e33957c5a75b06d77b188a10b69a39fd._comment @@ -0,0 +1,18 @@ +[[!comment format=mdwn + username="joey" + subject="""comment 5""" + date="2016-02-05T17:01:24Z" + content=""" +@luca you might get that with -J2 if you have two work tree files that +point to the same key. The first thread will start copying the first file, +and then the second thread tries to copy the second file but sees the key +is already being copied. + +It shouldn't happen otherwise; the way -J works is it splits files between +threads. So, no two threads will be working on the same file, except in the +situation described above. + +I doubt that whatever you're seeing is the same problem originally +described in this bug report. I'm still waiting for a followup with missing +information about the originally described bug. +"""]] diff --git a/doc/bugs/Unable_to_take_transfer_lock/comment_6_4778b7004983ccea7a6ad87cfd61a0f1._comment b/doc/bugs/Unable_to_take_transfer_lock/comment_6_4778b7004983ccea7a6ad87cfd61a0f1._comment new file mode 100644 index 0000000000..0318092613 --- /dev/null +++ b/doc/bugs/Unable_to_take_transfer_lock/comment_6_4778b7004983ccea7a6ad87cfd61a0f1._comment @@ -0,0 +1,9 @@ +[[!comment format=mdwn + username="aer1PheiZa" + avatar="http://cdn.libravatar.org/avatar/1fb7b6779a544f27abf147fc614d6633" + subject="6.20170101 - still unable. " + date="2017-08-07T11:58:05Z" + content=""" +In 'git annex get . -J1' anything more than one given to -J causes annex to \"unable to transfer\", and the files are not copied. The higher the number, the more likely it is not to transfer any of the involved files. At the end it has to be run with -J1, because even running it multiple times does not really help. +And yes, all the files are different, as they are photos, and there are no copies. For me it looks like a race. +"""]] diff --git a/doc/bugs/Unable_to_take_transfer_lock/comment_7_051d76ff1ac98fc1e1f70872390dadbf._comment b/doc/bugs/Unable_to_take_transfer_lock/comment_7_051d76ff1ac98fc1e1f70872390dadbf._comment new file mode 100644 index 0000000000..98018439d8 --- /dev/null +++ b/doc/bugs/Unable_to_take_transfer_lock/comment_7_051d76ff1ac98fc1e1f70872390dadbf._comment @@ -0,0 +1,8 @@ +[[!comment format=mdwn + username="joey" + subject="""comment 7""" + date="2017-10-02T16:33:34Z" + content=""" +aer1PheiZa, I think this was fixed in 6.20170520. If you (or anyone else +still watching this) sees it with a newer version than that, let me know. +"""]] diff --git a/doc/bugs/Unchanged_files_are_detected_as_modified_in_unlocked_branch.mdwn b/doc/bugs/Unchanged_files_are_detected_as_modified_in_unlocked_branch.mdwn new file mode 100644 index 0000000000..7580e0d213 --- /dev/null +++ b/doc/bugs/Unchanged_files_are_detected_as_modified_in_unlocked_branch.mdwn @@ -0,0 +1,25 @@ +### Please describe the problem. + +I have a git-annex repo with lots of images and keep a unlocked v6 branch around to access them. When new files are added (i.e. after running `git annex sync --content`), they are often detected as "modified": + + $ git annex status . + M image.jpg + +They are however identical to the checked-in files: + + $ cp image.jpg image-changed.jpg + $ git checkout -- image.jpg + $ cmp image.jpg image-changed.jpg && echo "No change" + No change + +This seems to happen to older files from time to time as well, but I cannot reproduce that. + +The only way to rectify this I can find is `git checkout` - but that means I have no way to know whether I am actually throwing away changes. It also has the unfortunate side effect of changing the mtime, leading to previews having to be regenerated. + +### What version of git-annex are you using? On what operating system? + +git-annex version: 6.20171026-gd451d333d (standalone binary) on Debian stretch. + +[[!meta title="v6 get/drop make git think files are changed"]] + +> [[fixed|done]] --[[Joey]] diff --git a/doc/bugs/Unchanged_files_are_detected_as_modified_in_unlocked_branch/comment_1_d5e90cf5edf189bb023deb46b5ffe96c._comment b/doc/bugs/Unchanged_files_are_detected_as_modified_in_unlocked_branch/comment_1_d5e90cf5edf189bb023deb46b5ffe96c._comment new file mode 100644 index 0000000000..d635819d7d --- /dev/null +++ b/doc/bugs/Unchanged_files_are_detected_as_modified_in_unlocked_branch/comment_1_d5e90cf5edf189bb023deb46b5ffe96c._comment @@ -0,0 +1,11 @@ +[[!comment format=mdwn + username="joey" + subject="""comment 1""" + date="2017-11-09T15:56:41Z" + content=""" +This is one of the primary known problems with v6 mode. On the page +[[todo/smudge]] that collects all such issues, it's the second item. +Basically, when git-annex gets/drops the content of the file, it +updates the work tree file, and so git thinks the file has changed. +Currently the best way to deal with it is to run `git add` on the file. +"""]] diff --git a/doc/bugs/UnicodeDecodeError_while_archiving_files_with_git-annex_to_glacier.mdwn b/doc/bugs/UnicodeDecodeError_while_archiving_files_with_git-annex_to_glacier.mdwn new file mode 100644 index 0000000000..b7e8754f76 --- /dev/null +++ b/doc/bugs/UnicodeDecodeError_while_archiving_files_with_git-annex_to_glacier.mdwn @@ -0,0 +1,43 @@ +I'll just started using git-annex and now I'm trying archiving to glacier. Unfortunately if I put a simple 2mb pdf with the name 'test.txt' into the archive folder, I get the following error: + +[[!format sh """ +Traceback (most recent call last): + File "/usr/local/bin/glacier", line 9, in + load_entry_point('glacier-cli==0.1.0', 'console_scripts', 'glacier-cli')() + File "build/bdist.macosx-10.11-x86_64/egg/glacier.py", line 732, in entry_point + File "build/bdist.macosx-10.11-x86_64/egg/glacier.py", line 718, in main + File "build/bdist.macosx-10.11-x86_64/egg/glacier.py", line 500, in archive_upload + File "/Users/markus/.virtualenvs/glacier-cli/lib/python2.7/site-packages/boto-2.38.0-py2.7.egg/boto/glacier/vault.py", line 178, in create_archive_from_file + writer.close() + File "/Users/markus/.virtualenvs/glacier-cli/lib/python2.7/site-packages/boto-2.38.0-py2.7.egg/boto/glacier/writer.py", line 228, in close + self.partitioner.flush() + File "/Users/markus/.virtualenvs/glacier-cli/lib/python2.7/site-packages/boto-2.38.0-py2.7.egg/boto/glacier/writer.py", line 79, in flush + self._send_part() + File "/Users/markus/.virtualenvs/glacier-cli/lib/python2.7/site-packages/boto-2.38.0-py2.7.egg/boto/glacier/writer.py", line 75, in _send_part + self.send_fn(part) + File "/Users/markus/.virtualenvs/glacier-cli/lib/python2.7/site-packages/boto-2.38.0-py2.7.egg/boto/glacier/writer.py", line 222, in _upload_part + self.uploader.upload_part(self.next_part_index, part_data) + File "/Users/markus/.virtualenvs/glacier-cli/lib/python2.7/site-packages/boto-2.38.0-py2.7.egg/boto/glacier/writer.py", line 129, in upload_part + content_range, part_data) + File "/Users/markus/.virtualenvs/glacier-cli/lib/python2.7/site-packages/boto-2.38.0-py2.7.egg/boto/glacier/layer1.py", line 1279, in upload_part + response_headers=response_headers) + File "/Users/markus/.virtualenvs/glacier-cli/lib/python2.7/site-packages/boto-2.38.0-py2.7.egg/boto/glacier/layer1.py", line 114, in make_request + data=data) + File "/Users/markus/.virtualenvs/glacier-cli/lib/python2.7/site-packages/boto-2.38.0-py2.7.egg/boto/connection.py", line 1071, in make_request + retry_handler=retry_handler) + File "/Users/markus/.virtualenvs/glacier-cli/lib/python2.7/site-packages/boto-2.38.0-py2.7.egg/boto/connection.py", line 943, in _mexe + request.body, request.headers) + File "/usr/local/Cellar/python/2.7.10_2/Frameworks/Python.framework/Versions/2.7/lib/python2.7/httplib.py", line 1053, in request + self._send_request(method, url, body, headers) + File "/usr/local/Cellar/python/2.7.10_2/Frameworks/Python.framework/Versions/2.7/lib/python2.7/httplib.py", line 1093, in _send_request + self.endheaders(body) + File "/usr/local/Cellar/python/2.7.10_2/Frameworks/Python.framework/Versions/2.7/lib/python2.7/httplib.py", line 1049, in endheaders + self._send_output(message_body) + File "/usr/local/Cellar/python/2.7.10_2/Frameworks/Python.framework/Versions/2.7/lib/python2.7/httplib.py", line 891, in _send_output + msg += message_body +UnicodeDecodeError: 'ascii' codec can't decode byte 0x8c in position 0: ordinal not in range(128) +"""]] + +I also filled a bugreport here: https://github.com/basak/glacier-cli/issues/61 + +I'm using git-annex from homebrew and the most recent version of glacier-cli. diff --git a/doc/bugs/UnicodeDecodeError_while_archiving_files_with_git-annex_to_glacier/comment_1_f68d5ad86d67aa73074389cf9a21469e._comment b/doc/bugs/UnicodeDecodeError_while_archiving_files_with_git-annex_to_glacier/comment_1_f68d5ad86d67aa73074389cf9a21469e._comment new file mode 100644 index 0000000000..3e5468e63a --- /dev/null +++ b/doc/bugs/UnicodeDecodeError_while_archiving_files_with_git-annex_to_glacier/comment_1_f68d5ad86d67aa73074389cf9a21469e._comment @@ -0,0 +1,22 @@ +[[!comment format=mdwn + username="joey" + subject="""comment 1""" + date="2015-08-14T18:07:10Z" + content=""" +Doesn't seem likely that this is a bug in git-annex. +git-annex is clearly operating on the file just fine until it passes +it off to glacier-cli. + +I suggest you pass --debug to git-annex, and see what glacier-cli command +it's running. That will probably be useful information for the glacier-cli +developers, to reproduce the problem without involving git-annex. + +One possibility that comes to mind, given + is that your git-annex +repository might be in a path with some characters that glacier-cli doesn't +like in its name. If git-annex passes the full path to the file to +glacier-cli, it might then run into this problem. I'm not sure ATM if +git-annex does pass the full path to the file; it seems like a relative +path would avoid such a problem. The --debug output should answer this +too.. +"""]] diff --git a/doc/bugs/UnicodeDecodeError_while_archiving_files_with_git-annex_to_glacier/comment_2_f0f1f18265ebb83caf4e6f1fb23ec876._comment b/doc/bugs/UnicodeDecodeError_while_archiving_files_with_git-annex_to_glacier/comment_2_f0f1f18265ebb83caf4e6f1fb23ec876._comment new file mode 100644 index 0000000000..c288b02783 --- /dev/null +++ b/doc/bugs/UnicodeDecodeError_while_archiving_files_with_git-annex_to_glacier/comment_2_f0f1f18265ebb83caf4e6f1fb23ec876._comment @@ -0,0 +1,7 @@ +[[!comment format=mdwn + username="mhubig" + subject="comment 2" + date="2015-08-17T11:44:40Z" + content=""" +Yes it's a glacier.cli related bug ... maybe git-annex should use boto3 for glacier support ... +"""]] diff --git a/doc/bugs/Upgrade_impossible_om_Mac_OSX.mdwn b/doc/bugs/Upgrade_impossible_om_Mac_OSX.mdwn new file mode 100644 index 0000000000..de0392eaa0 --- /dev/null +++ b/doc/bugs/Upgrade_impossible_om_Mac_OSX.mdwn @@ -0,0 +1,22 @@ +### Please describe the problem. +When starting git-annex Version: 5.20140420-ga25b8bb Build flags: Assistant Webapp Webapp-secure Pairing Testsuite S3 WebDAV FsEvents XMPP DNS Feeds Quvi TDFA CryptoHash on My MBP running OSX 10.9.2 I get a notification about: An upgrade of git-annex is available. (version 5.20140421) + +I then click UPGRADE and then get this message: Internal Server Error Cannot find old distribution bundle; not upgrading. + +### What steps will reproduce the problem? +See problem description above, not sure what else to say. + +### What version of git-annex are you using? On what operating system? +See problem description above, not sure what else to say. + +### Please provide any additional information below. + +[[!format sh """ +# If you can, paste a complete transcript of the problem occurring here. +# If the problem is with the git-annex assistant, paste in .git/annex/daemon.log + + +# End of transcript or log. +"""]] + +[[!tag moreinfo]] diff --git a/doc/bugs/Upgrade_impossible_om_Mac_OSX/comment_10_8c1487635f724d017ebe2f8b7bc10e8d._comment b/doc/bugs/Upgrade_impossible_om_Mac_OSX/comment_10_8c1487635f724d017ebe2f8b7bc10e8d._comment new file mode 100644 index 0000000000..13427f4663 --- /dev/null +++ b/doc/bugs/Upgrade_impossible_om_Mac_OSX/comment_10_8c1487635f724d017ebe2f8b7bc10e8d._comment @@ -0,0 +1,8 @@ +[[!comment format=mdwn + username="https://www.google.com/accounts/o8/id?id=AItOawkAUMhKOSkh9JaBA6xst3XxQIIsDEq5Zd4" + nickname="Ovidiu" + subject="OK" + date="2014-05-21T18:19:30Z" + content=""" +so I'll watch this behavior with future updates and reply if I have more info +"""]] diff --git a/doc/bugs/Upgrade_impossible_om_Mac_OSX/comment_1_84e52a4d1502fd622bdb25e04b459292._comment b/doc/bugs/Upgrade_impossible_om_Mac_OSX/comment_1_84e52a4d1502fd622bdb25e04b459292._comment new file mode 100644 index 0000000000..6ddb5ef30d --- /dev/null +++ b/doc/bugs/Upgrade_impossible_om_Mac_OSX/comment_1_84e52a4d1502fd622bdb25e04b459292._comment @@ -0,0 +1,13 @@ +[[!comment format=mdwn + username="https://www.google.com/accounts/o8/id?id=AItOawkAUMhKOSkh9JaBA6xst3XxQIIsDEq5Zd4" + nickname="Ovidiu" + subject="fixed" + date="2014-05-09T13:12:59Z" + content=""" +re-downloaded the old version, re-isntalled and now it actually starts downlaoding the new version => http://screencast.com/t/id2Ng57Z9 + +NOT sure though, why it now uploads the new version to all repositories? i.e. box.com? IS there a reason for this? + +Transfers +git-annex.dmg (for upgrade)→box.com 42% of 22.52 MiB +"""]] diff --git a/doc/bugs/Upgrade_impossible_om_Mac_OSX/comment_2_f2b109f6cf92f101d7f0afeb91605240._comment b/doc/bugs/Upgrade_impossible_om_Mac_OSX/comment_2_f2b109f6cf92f101d7f0afeb91605240._comment new file mode 100644 index 0000000000..9ed8dade9f --- /dev/null +++ b/doc/bugs/Upgrade_impossible_om_Mac_OSX/comment_2_f2b109f6cf92f101d7f0afeb91605240._comment @@ -0,0 +1,24 @@ +[[!comment format=mdwn + username="https://www.google.com/accounts/o8/id?id=AItOawkAUMhKOSkh9JaBA6xst3XxQIIsDEq5Zd4" + nickname="Ovidiu" + subject="spoke too soon" + date="2014-05-09T13:23:52Z" + content=""" +Stuck in a loop, now with every restart of git-annex it downloads the update and then uploads it to all my repositories again: +http://screencast.com/t/Ow7SlPVaS68 + +And then it says: + Finished upgrading git-annex to version 5.20140420-ga25b8bb +BUT that is the OLD version not the new one :-/ + +NEXT TRY: + +killed all processes of git and git-annex, restart. + +Was greeted by: + An upgrade of git-annex is available. +(version 5.20140421) + +Klicked the Upgrade button. +Nothing happens, except that the upgrade notice disappeared :-/ +"""]] diff --git a/doc/bugs/Upgrade_impossible_om_Mac_OSX/comment_3_df9b111aefea82b9d108495b79d7ffb4._comment b/doc/bugs/Upgrade_impossible_om_Mac_OSX/comment_3_df9b111aefea82b9d108495b79d7ffb4._comment new file mode 100644 index 0000000000..32b067a97c --- /dev/null +++ b/doc/bugs/Upgrade_impossible_om_Mac_OSX/comment_3_df9b111aefea82b9d108495b79d7ffb4._comment @@ -0,0 +1,24 @@ +[[!comment format=mdwn + username="https://www.google.com/accounts/o8/id?id=AItOawkAUMhKOSkh9JaBA6xst3XxQIIsDEq5Zd4" + nickname="Ovidiu" + subject="comment 3" + date="2014-05-09T13:28:38Z" + content=""" +here is what the log looks like: + +[2014-05-09 15:23:22 SAST] main: starting assistant version 5.20140420-ga25b8bb +(scanning...) [2014-05-09 15:23:23 SAST] Watcher: Performing startup scan +[2014-05-09 15:23:24 SAST] Upgrader: An upgrade of git-annex is available. (version 5.20140421) +(started...) [2014-05-09 15:25:10 SAST] main: warning git-annex has been shut down +[2014-05-09 15:25:31 SAST] main: starting assistant version 5.20140420-ga25b8bb +(scanning...) [2014-05-09 15:25:31 SAST] Watcher: Performing startup scan +[2014-05-09 15:25:32 SAST] Upgrader: An upgrade of git-annex is available. (version 5.20140421) +(started...) [2014-05-09 15:25:59 SAST] main: warning git-annex has been shut down + +[2014-05-09 15:27:14 SAST] main: starting assistant version 5.20140420-ga25b8bb +(scanning...) [2014-05-09 15:27:14 SAST] Watcher: Performing startup scan +[2014-05-09 15:27:15 SAST] Upgrader: An upgrade of git-annex is available. (version 5.20140421) +(started...) [2014-05-09 15:27:46 SAST] main: starting assistant version 5.20140420-ga25b8bb +(scanning...) [2014-05-09 15:27:46 SAST] Watcher: Performing startup scan +(started...) [2014-05-09 15:27:48 SAST] Upgrader: An upgrade of git-annex is available. (version 5.20140421) +"""]] diff --git a/doc/bugs/Upgrade_impossible_om_Mac_OSX/comment_4_e3b5bc815376383bbd1f312cc1c32a41._comment b/doc/bugs/Upgrade_impossible_om_Mac_OSX/comment_4_e3b5bc815376383bbd1f312cc1c32a41._comment new file mode 100644 index 0000000000..e55d370d99 --- /dev/null +++ b/doc/bugs/Upgrade_impossible_om_Mac_OSX/comment_4_e3b5bc815376383bbd1f312cc1c32a41._comment @@ -0,0 +1,12 @@ +[[!comment format=mdwn + username="http://joeyh.name/" + ip="108.236.230.124" + subject="comment 4" + date="2014-05-16T17:36:10Z" + content=""" +\"Cannot find old distribution bundle; not upgrading\" + +This message means that git-annex was unable to find where the old version was installed, which it needs to do in order to replace it with the new version. It looks for a `git-annex.app` directory, and could not find one. How did you install the old version, and to what location? + +I think it might try to upload the dmg file to a repository if that repository is set as a full backup and so wants all files even ones not in the work tree. +"""]] diff --git a/doc/bugs/Upgrade_impossible_om_Mac_OSX/comment_5_cd9b77e6670840a9902f5be80e1e658a._comment b/doc/bugs/Upgrade_impossible_om_Mac_OSX/comment_5_cd9b77e6670840a9902f5be80e1e658a._comment new file mode 100644 index 0000000000..8320e53a2b --- /dev/null +++ b/doc/bugs/Upgrade_impossible_om_Mac_OSX/comment_5_cd9b77e6670840a9902f5be80e1e658a._comment @@ -0,0 +1,87 @@ +[[!comment format=mdwn + username="https://www.google.com/accounts/o8/id?id=AItOawkAUMhKOSkh9JaBA6xst3XxQIIsDEq5Zd4" + nickname="Ovidiu" + subject="comment 5" + date="2014-05-16T17:45:48Z" + content=""" +I gave up on this. After doing this a few times, I stopped. Living in Africa, bandwidth is really expensive :-/ + +Initially I installed via the downloaded .dmg file. +After poking around a few times, I found a folder inside my Applications folder: + + ls -al /Applications/gi* + /Applications/git-annex.app: + total 0 + drwxr-xr-x@ 3 ovi admin 102 May 12 13:10 . + drwxrwxr-x+ 155 root admin 5270 May 15 04:19 .. + drwxr-xr-x 5 ovi admin 170 May 9 15:45 Contents + + /Applications/git-annex.app.5.20140421: + total 0 + drwxr-xr-x 2 ovi admin 68 May 11 11:26 . + drwxrwxr-x+ 155 root admin 5270 May 15 04:19 .. + +Every time I now open git-annex assistant I am greeted with: + + An upgrade of git-annex is available. + (version 5.20140421) + +I just did it again and it starts downloading again arrrghh! + + Transfers + git-annex.dmg (for upgrade)←web 12% of 22.52 MiB + +restarted the assistant and it now says: + + Finished upgrading git-annex to version 5.20140420-ga25b8bb + +Then I check and find this: + +ls -al /Applications/gi* +/Applications/git-annex.app: +total 0 +drwxr-xr-x@ 3 ovi admin 102 May 12 13:10 . +drwxrwxr-x+ 155 root admin 5270 May 15 04:19 .. +drwxr-xr-x 5 ovi admin 170 May 9 15:45 Contents + +/Applications/git-annex.app.5.20140421: +total 0 +drwxr-xr-x 2 ovi admin 68 May 11 11:26 . +drwxrwxr-x+ 155 root admin 5270 May 15 04:19 .. +shiny-2:~ ovi$ ls -al /Applications/gi* +/Applications/git-annex.app: +total 0 +drwxr-xr-x@ 3 ovi admin 102 May 12 13:10 . +drwxrwxr-x+ 155 root admin 5270 May 15 04:19 .. +drwxr-xr-x 5 ovi admin 170 May 9 15:45 Contents + +/Applications/git-annex.app.5.20140421: +total 0 +drwxr-xr-x 2 ovi admin 68 May 11 11:26 . +drwxrwxr-x+ 155 root admin 5270 May 15 04:19 .. + +LOL. restarting the assistant am greeted with click here to finish the upgrade. did so. + +now only 2 folders not 3 left: + +ls -al /Applications/gi* +/Applications/git-annex.app: +total 0 +drwxr-xr-x@ 3 ovi admin 102 May 12 13:10 . +drwxrwxr-x+ 155 root admin 5270 May 15 04:19 .. +drwxr-xr-x 5 ovi admin 170 May 9 15:45 Contents + +/Applications/git-annex.app.5.20140421: +total 0 +drwxr-xr-x 2 ovi admin 68 May 11 11:26 . +drwxrwxr-x+ 155 root admin 5270 May 15 04:19 .. + +Started the asistant again, am greeted with: + + An upgrade of git-annex is available. +(version 5.20140421) + +Thank you very much... not going to downlaod it again... + +COULD YOU PLEASE JUST SHARE THE DOWNLOAD LINK TO THE NEWEST ONE? +"""]] diff --git a/doc/bugs/Upgrade_impossible_om_Mac_OSX/comment_6_4584393333d377532d8607b42319abd9._comment b/doc/bugs/Upgrade_impossible_om_Mac_OSX/comment_6_4584393333d377532d8607b42319abd9._comment new file mode 100644 index 0000000000..023ccebfef --- /dev/null +++ b/doc/bugs/Upgrade_impossible_om_Mac_OSX/comment_6_4584393333d377532d8607b42319abd9._comment @@ -0,0 +1,8 @@ +[[!comment format=mdwn + username="https://www.google.com/accounts/o8/id?id=AItOawkAUMhKOSkh9JaBA6xst3XxQIIsDEq5Zd4" + nickname="Ovidiu" + subject="failed upgrade log file" + date="2014-05-16T17:55:34Z" + content=""" +Just in case it helps diagnose the problem: http://pastebin.com/PUpPRURb +"""]] diff --git a/doc/bugs/Upgrade_impossible_om_Mac_OSX/comment_7_d9d36ff62267dde98a27c5981951df7f._comment b/doc/bugs/Upgrade_impossible_om_Mac_OSX/comment_7_d9d36ff62267dde98a27c5981951df7f._comment new file mode 100644 index 0000000000..b38502bf40 --- /dev/null +++ b/doc/bugs/Upgrade_impossible_om_Mac_OSX/comment_7_d9d36ff62267dde98a27c5981951df7f._comment @@ -0,0 +1,10 @@ +[[!comment format=mdwn + username="http://joeyh.name/" + ip="108.236.230.124" + subject="comment 7" + date="2014-05-16T17:57:13Z" + content=""" +The repeated attempts to upgrade to 21 when it only downloads 20 are due to a different bug: [[bugs/git-annex_auto_upgrade_is_redundant]]. I've just put in a quick fix that should avoid that. + +However, for the purposes of this bug report, I'm interested in the original problem of \"Cannot find old distribution bundle; not upgrading\", and to fix that, I need to know how you originally installed git-annex. +"""]] diff --git a/doc/bugs/Upgrade_impossible_om_Mac_OSX/comment_8_5b26711c737eeb0b302060f098cb320b._comment b/doc/bugs/Upgrade_impossible_om_Mac_OSX/comment_8_5b26711c737eeb0b302060f098cb320b._comment new file mode 100644 index 0000000000..b2cab60970 --- /dev/null +++ b/doc/bugs/Upgrade_impossible_om_Mac_OSX/comment_8_5b26711c737eeb0b302060f098cb320b._comment @@ -0,0 +1,8 @@ +[[!comment format=mdwn + username="https://www.google.com/accounts/o8/id?id=AItOawkAUMhKOSkh9JaBA6xst3XxQIIsDEq5Zd4" + nickname="Ovidiu" + subject="comment 8" + date="2014-05-16T17:59:13Z" + content=""" +download 20.dmg, double clicked it, dragged it into my Applications folder. +"""]] diff --git a/doc/bugs/Upgrade_impossible_om_Mac_OSX/comment_9_1bf050056dcac7c1e0f497ee6e87a95a._comment b/doc/bugs/Upgrade_impossible_om_Mac_OSX/comment_9_1bf050056dcac7c1e0f497ee6e87a95a._comment new file mode 100644 index 0000000000..245bdc56a8 --- /dev/null +++ b/doc/bugs/Upgrade_impossible_om_Mac_OSX/comment_9_1bf050056dcac7c1e0f497ee6e87a95a._comment @@ -0,0 +1,10 @@ +[[!comment format=mdwn + username="http://joeyh.name/" + ip="209.250.56.3" + subject="comment 9" + date="2014-05-21T17:28:16Z" + content=""" +I have adjusted the \"Cannot find old distribution bundle; not upgrading\" message to say where it looked, and failed to find the git-annex.app directory. + +I am going to have to moreinfo this, since without knowing where git-annex was installed to, I can't say why it failed to find a git-annex.app directory. +"""]] diff --git a/doc/bugs/Upload_to_S3_fails_.mdwn b/doc/bugs/Upload_to_S3_fails_.mdwn new file mode 100644 index 0000000000..de74bfd7f8 --- /dev/null +++ b/doc/bugs/Upload_to_S3_fails_.mdwn @@ -0,0 +1,59 @@ +### Please describe the problem. + +Uploading a 21GB file to an S3 special remote fails. It will generally fail somewhere at about 3-15%. I am using the new chunking feature, with chunks set to 25MiB. + +### What steps will reproduce the problem? + + $ git annex copy my-big-file.tar.bz --to s3 + copy my-big-file.tar.bz (gpg) (checking s3...) (to s3...) + 13% 863.8KB/s 6h0m + ErrorClosed + failed + git-annex: copy: 1 failed + +### What version of git-annex are you using? On what operating system? + +Running on Arch Linux. + + git-annex version: 5.20140818-g10bf03a + build flags: Assistant Webapp Webapp-secure Pairing Testsuite S3 WebDAV Inotify DBus DesktopNotify XMPP DNS Feeds Quvi TDFA CryptoHash + key/value backends: SHA256E SHA1E SHA512E SHA224E SHA384E SKEIN256E SKEIN512E SHA256 SHA1 SHA512 SHA224 SHA384 SKEIN256 SKEIN512 WORM URL + remote types: git gcrypt S3 bup directory rsync web webdav tahoe glacier ddar hook external + local repository version: 5 + supported repository version: 5 + upgrade supported from repository versions: 0 1 2 4 + +### Please provide any additional information below. + +If I fire up the web app and open the log, the end looks like this: + + +[[!format sh """ +... + +3% 857.3KB/s 6h46m +3% 857.3KB/s 6h46m +3% 857.3KB/s 6h46m +3% 857.4KB/s 6h46m +3% 857.4KB/s 6h46m +3% 857.5KB/s 6h46m +3% 857.5KB/s 6h46m +3% 857.6KB/s 6h46m +3% 857.6KB/s 6h46m +3% 857.6KB/s 6h46m +3% 857.7KB/s 6h46m +3% 857.7KB/s 6h46m +3% 857.8KB/s 6h46m +3% 857.8KB/s 6h46m +3% 857.8KB/s 6h46m +3% 857.9KB/s 6h46m +3% 857.9KB/s 6h46m +3% 858.0KB/s 6h46m +3% 858.0KB/s 6h46m +3% 858.1KB/s 6h46m +3% 858.1KB/s 6h45m +3% 858.1KB/s 6h45mmux_client_request_session: read from master failed: Broken pipe + +"""]] + +[[!tag moreinfo]] diff --git a/doc/bugs/Upload_to_S3_fails_/comment_10_b7e912bac673bdffa5775b71d5d39937._comment b/doc/bugs/Upload_to_S3_fails_/comment_10_b7e912bac673bdffa5775b71d5d39937._comment new file mode 100644 index 0000000000..ecc34d4871 --- /dev/null +++ b/doc/bugs/Upload_to_S3_fails_/comment_10_b7e912bac673bdffa5775b71d5d39937._comment @@ -0,0 +1,8 @@ +[[!comment format=mdwn + username="https://www.google.com/accounts/o8/id?id=AItOawkvSZ1AFJdY_1FeutZr_KWeqtzjZta1PNE" + nickname="Thedward" + subject="comment 10" + date="2014-10-21T21:25:57Z" + content=""" +The only files that succeeded were small text files. The other files — 3-200MiB — all failed. +"""]] diff --git a/doc/bugs/Upload_to_S3_fails_/comment_11_65cb9b1dd69cd657f2cfde036eb60417._comment b/doc/bugs/Upload_to_S3_fails_/comment_11_65cb9b1dd69cd657f2cfde036eb60417._comment new file mode 100644 index 0000000000..ec59d6a207 --- /dev/null +++ b/doc/bugs/Upload_to_S3_fails_/comment_11_65cb9b1dd69cd657f2cfde036eb60417._comment @@ -0,0 +1,11 @@ +[[!comment format=mdwn + username="http://joeyh.name/" + ip="209.250.56.96" + subject="comment 11" + date="2014-10-23T21:05:15Z" + content=""" +When it resumes, it will start at 0% but jump forward to the resume point pretty quickly, after verifying which chunks have already been sent. +If any full chunk gets transferred, I'd expect it to resume. This may not be very obvious it's happening for smaller files. + +I have been running `git annex testremote` against S3 special remotes today, and have not managed to reproduce this problem (using either the old S3 or the new AWS libraries). It could be anything, including a problem with your network or the network between you and the S3 endpoint. Have you tried using a different S3 region? +"""]] diff --git a/doc/bugs/Upload_to_S3_fails_/comment_1_398c014921f9af957fb5e9a92ed0ef4d._comment b/doc/bugs/Upload_to_S3_fails_/comment_1_398c014921f9af957fb5e9a92ed0ef4d._comment new file mode 100644 index 0000000000..a18ca1d05a --- /dev/null +++ b/doc/bugs/Upload_to_S3_fails_/comment_1_398c014921f9af957fb5e9a92ed0ef4d._comment @@ -0,0 +1,10 @@ +[[!comment format=mdwn + username="http://joeyh.name/" + ip="108.236.230.124" + subject="comment 1" + date="2014-09-18T18:49:43Z" + content=""" +This is using the old hS3 library. So, each chunk is sent using a new http connection. It seems that the connection must be being closed by S3 part way through the upload of a chunk. + +It may be that the new aws library somehow avoids this problem. So, a git-annex built with the `s3-aws` branch merged in may help with this bug. OTOH, that new branch makes a single http connection be reused for all the chunks in a file, so it might also make things worse. +"""]] diff --git a/doc/bugs/Upload_to_S3_fails_/comment_2_f33ce058c9460cf7d151e739bff0440a._comment b/doc/bugs/Upload_to_S3_fails_/comment_2_f33ce058c9460cf7d151e739bff0440a._comment new file mode 100644 index 0000000000..dcf719bba7 --- /dev/null +++ b/doc/bugs/Upload_to_S3_fails_/comment_2_f33ce058c9460cf7d151e739bff0440a._comment @@ -0,0 +1,10 @@ +[[!comment format=mdwn + username="http://joeyh.name/" + ip="108.236.230.124" + subject="comment 2" + date="2014-09-18T18:52:17Z" + content=""" +If you're using the new chunking system, git-annex should support resuming the upload to S3. Next time you try to send the file, it should find the chunks that were successfully sent, and resume at the chunk where it failed. + +Supporting this even for encrypted uploads was a major benefit of the new chunking system, so I hope it works...? +"""]] diff --git a/doc/bugs/Upload_to_S3_fails_/comment_3_cd1e768fe1e67daf08b5afd460620922._comment b/doc/bugs/Upload_to_S3_fails_/comment_3_cd1e768fe1e67daf08b5afd460620922._comment new file mode 100644 index 0000000000..5efb786853 --- /dev/null +++ b/doc/bugs/Upload_to_S3_fails_/comment_3_cd1e768fe1e67daf08b5afd460620922._comment @@ -0,0 +1,8 @@ +[[!comment format=mdwn + username="annexuser" + ip="71.198.212.13" + subject="comment 3" + date="2014-09-19T04:43:42Z" + content=""" +Was the version I listed above not using the new chunking from the `s3-aws` branch? How do I determine if my version of git-annex was built with the new or old chunking? +"""]] diff --git a/doc/bugs/Upload_to_S3_fails_/comment_4_0cdd2e8d6e83c03de717ecd3253e753d._comment b/doc/bugs/Upload_to_S3_fails_/comment_4_0cdd2e8d6e83c03de717ecd3253e753d._comment new file mode 100644 index 0000000000..216aea5751 --- /dev/null +++ b/doc/bugs/Upload_to_S3_fails_/comment_4_0cdd2e8d6e83c03de717ecd3253e753d._comment @@ -0,0 +1,8 @@ +[[!comment format=mdwn + username="http://joeyh.name/" + ip="67.223.1.203" + subject="comment 4" + date="2014-09-19T18:33:17Z" + content=""" +Your version supports both new and old style chunking. Which is used depends on how the S3 remote was configured when it was set up. It can't really be changed w/o re-setting up the remote. You can check which is used by `git show git-annex:remote.log`, find the line for the UUID of the remote, and see if it has chunk= (new chunking) or chunksize= (old chunking). +"""]] diff --git a/doc/bugs/Upload_to_S3_fails_/comment_5_020c055f6c06860dda27c1debb123742._comment b/doc/bugs/Upload_to_S3_fails_/comment_5_020c055f6c06860dda27c1debb123742._comment new file mode 100644 index 0000000000..368d2853f7 --- /dev/null +++ b/doc/bugs/Upload_to_S3_fails_/comment_5_020c055f6c06860dda27c1debb123742._comment @@ -0,0 +1,43 @@ +[[!comment format=mdwn + username="https://www.google.com/accounts/o8/id?id=AItOawkvSZ1AFJdY_1FeutZr_KWeqtzjZta1PNE" + nickname="Thedward" + subject="comment 5" + date="2014-10-21T04:10:16Z" + content=""" + +I am experiencing similar behavior on Ubuntu Trusty (x86_64) using a prebuilt Linux release: + + Linux hostname 3.13.0-36-generic #63-Ubuntu SMP Wed Sep 3 21:30:07 UTC 2014 x86_64 x86_64 x86_64 GNU/Linux + + Distributor ID: Ubuntu + Description: Ubuntu 14.04.1 LTS + Release: 14.04 + Codename: trusty + + git-annex version: 5.20141016-g26b38fd + build flags: Assistant Webapp Webapp-secure Pairing Testsuite S3 WebDAV Inotify DBus DesktopNotify XMPP DNS Feeds Quvi TDFA CryptoHash + key/value backends: SHA256E SHA1E SHA512E SHA224E SHA384E SKEIN256E SKEIN512E SHA256 SHA1 SHA512 SHA224 SHA384 SKEIN256 SKEIN512 WORM URL + remote types: git gcrypt S3 bup directory rsync web webdav tahoe glacier ddar hook external + +Copying files to S3 consistently fails both from the command line and via the assistant: + + [2014-10-20 22:34:32 CDT] read: git [\"--git-dir=/home/user/git-annex/.git\",\"--work-tree=/home/user/git-annex\",\"-c\",\"core.bare=false\",\"show-ref\",\"git-annex\"] + [2014-10-20 22:34:32 CDT] read: git [\"--git-dir=/home/user/git-annex/.git\",\"--work-tree=/home/user/git-annex\",\"-c\",\"core.bare=false\",\"show-ref\",\"--hash\",\"refs/heads/git-annex\"] + [2014-10-20 22:34:32 CDT] read: git [\"--git-dir=/home/user/git-annex/.git\",\"--work-tree=/home/user/git-annex\",\"-c\",\"core.bare=false\",\"log\",\"refs/heads/git-annex..78e9b6b85f3b453d8ed4f66f63ff09e03ce13d06\",\"-n1\",\"--pretty=%H\"] + [2014-10-20 22:34:32 CDT] read: git [\"--git-dir=/home/user/git-annex/.git\",\"--work-tree=/home/user/git-annex\",\"-c\",\"core.bare=false\",\"log\",\"refs/heads/git-annex..658720ba59a2fefee89c908b972971ca901f84dc\",\"-n1\",\"--pretty=%H\"] + [2014-10-20 22:34:32 CDT] chat: git [\"--git-dir=/home/user/git-annex/.git\",\"--work-tree=/home/user/git-annex\",\"-c\",\"core.bare=false\",\"cat-file\",\"--batch\"] + [2014-10-20 22:34:32 CDT] read: git [\"--git-dir=/home/user/git-annex/.git\",\"--work-tree=/home/user/git-annex\",\"-c\",\"core.bare=false\",\"ls-files\",\"--cached\",\"-z\",\"--\",\"storage/data.bin\"] + [2014-10-20 22:34:32 CDT] chat: git [\"--git-dir=/home/user/git-annex/.git\",\"--work-tree=/home/user/git-annex\",\"-c\",\"core.bare=false\",\"cat-file\",\"--batch\"] + copy storage/data.bin (gpg) (checking S3git-annex...) (to S3git-annex...) + 0% 0.0 B/s 0s[2014-10-20 22:34:33 CDT] chat: gpg [\"--batch\",\"--no-tty\",\"--use-agent\",\"--quiet\",\"--trust-model\",\"always\",\"--batch\",\"--passphrase-fd\",\"14\",\"--symmetric\",\"--force-mdc\",\"--no-textmode\"] + 8% 512.0KB/s 21s[2014-10-20 22:34:35 CDT] chat: gpg [\"--batch\",\"--no-tty\",\"--use-agent\",\"--quiet\",\"--trust-model\",\"always\",\"--batch\",\"--passphrase-fd\",\"14\",\"--symmetric\",\"--force-mdc\",\"--no-textmode\"] + 8% 528.0KB/s 21s + ErrorClosed + failed + git-annex: copy: 1 failed + +Two files (out of several hundred) have succeeded. + +Any ideas? + +"""]] diff --git a/doc/bugs/Upload_to_S3_fails_/comment_6_8bc023fca8cedfc517856cdcd20b7f10._comment b/doc/bugs/Upload_to_S3_fails_/comment_6_8bc023fca8cedfc517856cdcd20b7f10._comment new file mode 100644 index 0000000000..ed21bb8fb6 --- /dev/null +++ b/doc/bugs/Upload_to_S3_fails_/comment_6_8bc023fca8cedfc517856cdcd20b7f10._comment @@ -0,0 +1,10 @@ +[[!comment format=mdwn + username="http://joeyh.name/" + ip="209.250.56.96" + subject="comment 6" + date="2014-10-21T16:31:02Z" + content=""" +I need to know if the S3 remote is configured to use the new style chunking feature, and what size chunks it is configured to use. I have already explained how to check that in this thread. + +I also need to know if retrying the upload after it fails lets it resume where it left off. +"""]] diff --git a/doc/bugs/Upload_to_S3_fails_/comment_7_32685258748a7cdd177e7af2105f128e._comment b/doc/bugs/Upload_to_S3_fails_/comment_7_32685258748a7cdd177e7af2105f128e._comment new file mode 100644 index 0000000000..4d66552245 --- /dev/null +++ b/doc/bugs/Upload_to_S3_fails_/comment_7_32685258748a7cdd177e7af2105f128e._comment @@ -0,0 +1,10 @@ +[[!comment format=mdwn + username="https://www.google.com/accounts/o8/id?id=AItOawkvSZ1AFJdY_1FeutZr_KWeqtzjZta1PNE" + nickname="Thedward" + subject="comment 7" + date="2014-10-21T17:35:16Z" + content=""" +It is running the new style chunking (chunk=1MiB). + +It does not appear to resume when it tries again. If I try copying a file to the remote from the command line, it always starts at 0% and dies at some point before 100% even if it has tried to copy that file before. +"""]] diff --git a/doc/bugs/Upload_to_S3_fails_/comment_8_841fd94d0f599c71a76fd22b07944366._comment b/doc/bugs/Upload_to_S3_fails_/comment_8_841fd94d0f599c71a76fd22b07944366._comment new file mode 100644 index 0000000000..61e201c9db --- /dev/null +++ b/doc/bugs/Upload_to_S3_fails_/comment_8_841fd94d0f599c71a76fd22b07944366._comment @@ -0,0 +1,8 @@ +[[!comment format=mdwn + username="https://www.google.com/accounts/o8/id?id=AItOawkvSZ1AFJdY_1FeutZr_KWeqtzjZta1PNE" + nickname="Thedward" + subject="comment 8" + date="2014-10-21T17:47:35Z" + content=""" +Additional info: I created both of the related git-annex repositories yesterday via the webapp. Then imported a number of files to each one. Then connected them via xmpp. Then created the S3 remote via the webapp so they could actually share files. I am using an IAM identity for S3 instead of my root access key; it has full S3 access (and data IS showing up in the bucket, so it's not a *simple* permissions problem). +"""]] diff --git a/doc/bugs/Upload_to_S3_fails_/comment_9_dd837a1cb2146224b9c000cbeea4f3b3._comment b/doc/bugs/Upload_to_S3_fails_/comment_9_dd837a1cb2146224b9c000cbeea4f3b3._comment new file mode 100644 index 0000000000..4cfda38c2f --- /dev/null +++ b/doc/bugs/Upload_to_S3_fails_/comment_9_dd837a1cb2146224b9c000cbeea4f3b3._comment @@ -0,0 +1,8 @@ +[[!comment format=mdwn + username="http://joeyh.name/" + ip="209.250.56.96" + subject="comment 9" + date="2014-10-21T20:22:52Z" + content=""" +How big is the file that it fails to copy? +"""]] diff --git a/doc/bugs/Upload_to_box_very_slow.mdwn b/doc/bugs/Upload_to_box_very_slow.mdwn new file mode 100644 index 0000000000..006fe1afa7 --- /dev/null +++ b/doc/bugs/Upload_to_box_very_slow.mdwn @@ -0,0 +1,122 @@ +### Please describe the problem. + +In recent version of git-annex (including the most recent standalone version), uploading to box is slow. Typically, it will take 25 seconds or so for git-annex to begin the upload, whereas with previous versions it happened almost instantly. + +I used git bisect and rebuilds via stack to locate the commit that introduced this problem. This was the result: + + Bisecting: 0 revisions left to test after this (roughly 0 steps) + [cf51f40f0eaed2c5968f35b94bebab4a1c819491] webdav: Changed path used on webdav server for temporary files. + +Please note: this is distinct from the bug I reported here (https://git-annex.branchable.com/bugs/git-annex_can_no_longer_copy_files_to_box/), since it also affects the standalone version. + +### What steps will reproduce the problem? + +Use a version of git annex built after the commit above from Sept. 15 (e.g., a recent standalone): + +Use a repo with a box remote set up with encryption and chunking: + + WEBDAV_USERNAME=[username] WEBDAV_PASSWORD=[passwd] git annex initremote box type=webdav url=https://dav.box.com/dav/mystuff/annex chunk=100mb keyid=[keyid] + + echo "Hello, world!" > test.txt + git add test.txt + git annex --verbose --debug copy -t box test.txt + +### What version of git-annex are you using? On what operating system? + +Tested with the nightly standalone build on arch linux: + + git-annex version: 6.20171006-gbeca3ce2b + build flags: Assistant Webapp Pairing Testsuite S3(multipartupload)(storageclasses) WebDAV Inotify DBus DesktopNotify ConcurrentOutput TorrentParser MagicMime Feeds Quvi + dependency versions: aws-0.14.1 bloomfilter-2.0.1.0 cryptonite-0.20 DAV-1.3.1 feed-0.3.11.1 ghc-8.0.1 http-client-0.4.31.1 persistent-sqlite-2.6 torrent-10000.0.0 uuid-1.3.12 yesod-1.4.3 + key/value backends: SHA256E SHA256 SHA512E SHA512 SHA224E SHA224 SHA384E SHA384 SHA3_256E SHA3_256 SHA3_512E SHA3_512 SHA3_224E SHA3_224 SHA3_384E SHA3_384 SKEIN256E SKEIN256 SKEIN512E SKEIN512 SHA1E SHA1 MD5E MD5 WORM URL + remote types: git gcrypt p2p S3 bup directory rsync web bittorrent webdav tahoe glacier ddar hook external + +### Please provide any additional information below. + +Here is the output: + + % ~/git-annex.linux/git-annex --verbose --debug copy -t box test.txt + [2017-10-07 10:13:25.283802979] read: git ["--git-dir=.git","--work-tree=.","--literal-pathspecs","show-ref","git-annex"] + [2017-10-07 10:13:25.294784901] process done ExitSuccess + [2017-10-07 10:13:25.295173712] read: git ["--git-dir=.git","--work-tree=.","--literal-pathspecs","show-ref","--hash","refs/heads/git-annex"] + [2017-10-07 10:13:25.305375915] process done ExitSuccess + [2017-10-07 10:13:25.306659057] chat: git ["--git-dir=.git","--work-tree=.","--literal-pathspecs","cat-file","--batch"] + [2017-10-07 10:13:25.307617362] chat: git ["--git-dir=.git","--work-tree=.","--literal-pathspecs","cat-file","--batch-check=%(objectname) %(objecttype) %(objectsize)"] + [2017-10-07 10:13:25.338084356] read: git ["--git-dir=.git","--work-tree=.","--literal-pathspecs","ls-files","--cached","-z","--","test.txt"] + copy test.txt [2017-10-07 10:13:25.360741244] chat: gpg ["--quiet","--trust-model","always","--decrypt"] + [2017-10-07 10:13:25.454260122] process done ExitSuccess + (checking box...) (to box...) + [2017-10-07 10:13:26.215000484] chat: gpg ["--quiet","--trust-model","always","--batch","--passphrase-fd","18","--symmetric","--force-mdc","--no-textmode"] + 100% 2 B/s 7s[2017-10-07 10:13:58.059203157] process done ExitSuccess + ok + +Note the 32 second delay from 10:13:26 to 10:13:58 here + + [2017-10-07 10:13:58.066819166] chat: git ["--git-dir=.git","--work-tree=.","--literal-pathspecs","hash-object","-w","--stdin-paths","--no-filters"] + [2017-10-07 10:13:58.067584497] feed: git ["--git-dir=.git","--work-tree=.","--literal-pathspecs","update-index","-z","--index-info"] + [2017-10-07 10:13:58.216568109] process done ExitSuccess + [2017-10-07 10:13:58.216839725] read: git ["--git-dir=.git","--work-tree=.","--literal-pathspecs","show-ref","--hash","refs/heads/git-annex"] + [2017-10-07 10:13:58.227738805] process done ExitSuccess + (recording state in git...) + [2017-10-07 10:13:58.227942884] read: git ["--git-dir=.git","--work-tree=.","--literal-pathspecs","write-tree"] + [2017-10-07 10:13:58.386915393] process done ExitSuccess + [2017-10-07 10:13:58.387325088] chat: git ["--git-dir=.git","--work-tree=.","--literal-pathspecs","commit-tree","bf90d33cdcf43119d2608e90fa6fdc23c497950d","--no-gpg-sign","-p","refs/heads/git-annex"] + [2017-10-07 10:13:58.400174021] process done ExitSuccess + [2017-10-07 10:13:58.400315661] call: git ["--git-dir=.git","--work-tree=.","--literal-pathspecs","update-ref","refs/heads/git-annex","b9583c0cf24933de01063281108f6d0c3a98053b"] + [2017-10-07 10:13:58.413220259] process done ExitSuccess + [2017-10-07 10:13:58.414476091] process done ExitSuccess + [2017-10-07 10:13:58.414890325] process done ExitSuccess + [2017-10-07 10:13:58.415545095] process done ExitSuccess + +Compare this with the results of git-annex built with stack that does not include the problematic commit: + + git-annex version: 6.20170915-g122396029 + build flags: Assistant Webapp Pairing Testsuite S3(multipartupload)(storageclasses) WebDAV Inotify ConcurrentOutput TorrentParser Feeds Quvi + dependency versions: aws-0.16 bloomfilter-2.0.1.0 cryptonite-0.21 DAV-1.3.1 feed-0.3.12.0 ghc-8.0.2 http-client-0.5.6.1 persistent-sqlite-2.6.2 torrent-10000.1.1 uuid-1.3.13 yesod-1.4.5 + key/value backends: SHA256E SHA256 SHA512E SHA512 SHA224E SHA224 SHA384E SHA384 SHA3_256E SHA3_256 SHA3_512E SHA3_512 SHA3_224E SHA3_224 SHA3_384E SHA3_384 SKEIN256E SKEIN256 SKEIN512E SKEIN512 SHA1E SHA1 MD5E MD5 WORM URL + remote types: git gcrypt p2p S3 bup directory rsync web bittorrent webdav tahoe glacier ddar hook external + local repository version: 5 + supported repository versions: 3 5 6 + upgrade supported from repository versions: 0 1 2 3 4 5 + operating system: linux x86_64 + +Here is the output: + + % git annex --verbose --debug copy -t box test.txt + [2017-10-07 10:51:28.863004037] read: git ["--git-dir=.git","--work-tree=.","--literal-pathspecs","show-ref","git-annex"] + [2017-10-07 10:51:28.866087687] process done ExitSuccess + [2017-10-07 10:51:28.866184907] read: git ["--git-dir=.git","--work-tree=.","--literal-pathspecs","show-ref","--hash","refs/heads/git-annex"] + [2017-10-07 10:51:28.868674623] process done ExitSuccess + [2017-10-07 10:51:28.869107711] read: git ["--git-dir=.git","--work-tree=.","--literal-pathspecs","log","refs/heads/git-annex..5cc79198b2eb9e281d64b9eed7b1b6a2e869fb20","--pretty=%H","-n1"] + [2017-10-07 10:51:28.87271685] process done ExitSuccess + [2017-10-07 10:51:28.873655522] chat: git ["--git-dir=.git","--work-tree=.","--literal-pathspecs","cat-file","--batch"] + [2017-10-07 10:51:28.874173468] chat: git ["--git-dir=.git","--work-tree=.","--literal-pathspecs","cat-file","--batch-check=%(objectname) %(objecttype) %(objectsize)"] + [2017-10-07 10:51:28.896038454] read: git ["--git-dir=.git","--work-tree=.","--literal-pathspecs","ls-files","--cached","-z","--","test.txt"] + copy test.txt [2017-10-07 10:51:28.909441282] chat: gpg ["--quiet","--trust-model","always","--decrypt"] + [2017-10-07 10:51:28.997627719] process done ExitSuccess + (checking box...) (to box...) + [2017-10-07 10:51:29.838627434] chat: gpg ["--quiet","--trust-model","always","--batch","--passphrase-fd","18","--symmetric","--force-mdc","--no-textmode"] + 100% 85 B/s 0s[2017-10-07 10:51:34.465610054] process done ExitSuccess + ok + +Notice the 5 second gap here: + + + [2017-10-07 10:51:34.469743494] chat: git ["--git-dir=.git","--work-tree=.","--literal-pathspecs","hash-object","-w","--stdin-paths","--no-filters"] + [2017-10-07 10:51:34.47009347] feed: git ["--git-dir=.git","--work-tree=.","--literal-pathspecs","update-index","-z","--index-info"] + [2017-10-07 10:51:34.588535947] process done ExitSuccess + [2017-10-07 10:51:34.588699027] read: git ["--git-dir=.git","--work-tree=.","--literal-pathspecs","show-ref","--hash","refs/heads/git-annex"] + [2017-10-07 10:51:34.591436472] process done ExitSuccess + (recording state in git...) + [2017-10-07 10:51:34.59158293] read: git ["--git-dir=.git","--work-tree=.","--literal-pathspecs","write-tree"] + [2017-10-07 10:51:34.767927859] process done ExitSuccess + [2017-10-07 10:51:34.768081231] chat: git ["--git-dir=.git","--work-tree=.","--literal-pathspecs","commit-tree","b7c89dee187ca46d21ad05937d746acb149c323e","--no-gpg-sign","-p","refs/heads/git-annex"] + [2017-10-07 10:51:34.799274341] process done ExitSuccess + [2017-10-07 10:51:34.799458862] call: git ["--git-dir=.git","--work-tree=.","--literal-pathspecs","update-ref","refs/heads/git-annex","676897d54eeecff41c28432e903671400b3c0b84"] + [2017-10-07 10:51:34.802489572] process done ExitSuccess + +### Have you had any luck using git-annex before? (Sometimes we get tired of reading bug reports all day and a lil' positive end note does wonders) + +Yes! Thank you! + +> [[fixed|done]] --[[Joey]] diff --git a/doc/bugs/Upload_to_box_very_slow/comment_1_602697f6ea6cfd07764b1e9ed040736c._comment b/doc/bugs/Upload_to_box_very_slow/comment_1_602697f6ea6cfd07764b1e9ed040736c._comment new file mode 100644 index 0000000000..a5e9b69c3d --- /dev/null +++ b/doc/bugs/Upload_to_box_very_slow/comment_1_602697f6ea6cfd07764b1e9ed040736c._comment @@ -0,0 +1,34 @@ +[[!comment format=mdwn + username="joey" + subject="""comment 1""" + date="2017-10-07T17:03:56Z" + content=""" +I suspect that the problem dealt with in +[[!commit 2ca1d3cc0134134314649c21822bf1352df52e93]] +is responsible for this behavior. If many redirects are +followed before git-annex gives up, it would explain why it's taking so long. + +In redirecting from a non-existant file to some index.php file that itself +doesn't exist, and so on forever, box.com is clearly a horrible, broken, +very bad webdav server. Given that it's so broken, it's not terribly +surprising that small changes might expose that or another brokenness in +different ways. Changing the location of a temp file from a tmp +subdirectory to the parent directory is not the kind of thing that causes +30 second delays with non-broken servers. + +I have not been able to reproduce the problem here though so that is just a +theory. + +I am not inclined to make random changes to try to work around random +breakage in box.com. That is a game you lose before you start playing. + +It may be that deleting line 132 from Remote/WebDAV.hs avoids +the problem. + + maybe noop (void . mkColRecursive) (locationParent tmp) + +Since the parent directory of the temp file is the top of the webdav +repository, and so already exists, that line is now unnecessary. +Since you can reproduce the problem and build from source, can you test +that change? +"""]] diff --git a/doc/bugs/Upload_to_box_very_slow/comment_2_380443c65e576959dddae499745bff09._comment b/doc/bugs/Upload_to_box_very_slow/comment_2_380443c65e576959dddae499745bff09._comment new file mode 100644 index 0000000000..6dbd775ba3 --- /dev/null +++ b/doc/bugs/Upload_to_box_very_slow/comment_2_380443c65e576959dddae499745bff09._comment @@ -0,0 +1,8 @@ +[[!comment format=mdwn + username="joey" + subject="""comment 2""" + date="2017-10-07T18:05:44Z" + content=""" +Also, I've added logging of all webdav operations with --debug, which +should help with determining what operation is being slow. +"""]] diff --git a/doc/bugs/Upload_to_box_very_slow/comment_3_08d3bee435eb0950b3f6ac3b458dda18._comment b/doc/bugs/Upload_to_box_very_slow/comment_3_08d3bee435eb0950b3f6ac3b458dda18._comment new file mode 100644 index 0000000000..f4f9fc6669 --- /dev/null +++ b/doc/bugs/Upload_to_box_very_slow/comment_3_08d3bee435eb0950b3f6ac3b458dda18._comment @@ -0,0 +1,46 @@ +[[!comment format=mdwn + username="madalu" + avatar="http://cdn.libravatar.org/avatar/c00d4aa29fd053f08a2ef35531592914" + subject="First test" + date="2017-10-09T14:22:22Z" + content=""" +I'm sorry to hear that Box has a buggy WebDav server. I would gladly shift away from Box except that I have unlimited storage through my university. So thanks for being willing to help debug this problem! + +Here is the first test *without* making the change above but with the new debug information. You can see the big gap after 9:04:54 and \"getProps .\". This test actually failed, so it could be that this bug is related to the earlier one I reported. I cannot reproduce this reliably with the latest build using stack. Sometimes git annex uploads the file after a long delay, sometimes it fails. + + % git annex --verbose --debug copy -t box test.txt + [2017-10-09 09:04:52.566913246] read: git [\"--git-dir=.git\",\"--work-tree=.\",\"--literal-pathspecs\",\"show-ref\",\"git-annex\"] + [2017-10-09 09:04:52.570241399] process done ExitSuccess + [2017-10-09 09:04:52.570335828] read: git [\"--git-dir=.git\",\"--work-tree=.\",\"--literal-pathspecs\",\"show-ref\",\"--hash\",\"refs/heads/git-annex\"] + [2017-10-09 09:04:52.573016175] process done ExitSuccess + [2017-10-09 09:04:52.57406831] chat: git [\"--git-dir=.git\",\"--work-tree=.\",\"--literal-pathspecs\",\"cat-file\",\"--batch\"] + [2017-10-09 09:04:52.57444107] chat: git [\"--git-dir=.git\",\"--work-tree=.\",\"--literal-pathspecs\",\"cat-file\",\"--batch-check=%(objectname) %(objecttype) %(objectsize)\"] + [2017-10-09 09:04:52.595364899] read: git [\"--git-dir=.git\",\"--work-tree=.\",\"--literal-pathspecs\",\"ls-files\",\"--cached\",\"-z\",\"--\",\"test.txt\"] + copy test.txt [2017-10-09 09:04:52.609509008] chat: gpg [\"--quiet\",\"--trust-model\",\"always\",\"--decrypt\"] + [2017-10-09 09:04:52.699246422] process done ExitSuccess + (checking box...) [2017-10-09 09:04:52.739720313] getProps 90a/d4d/GPGHMACSHA1--53636e5e7a50bc58eae478ddc260bb5abd899d03/GPGHMACSHA1--53636e5e7a50bc58eae478ddc260bb5abd899d03 + (to box...) + [2017-10-09 09:04:54.134452518] chat: gpg [\"--quiet\",\"--trust-model\",\"always\",\"--batch\",\"--passphrase-fd\",\"19\",\"--symmetric\",\"--force-mdc\",\"--no-textmode\"] + [2017-10-09 09:04:54.188323123] getProps . + [2017-10-09 09:05:24.214747951] mkCol . + [2017-10-09 09:05:24.900782145] process done ExitSuccess + + DAV failure: Status {statusCode = 405, statusMessage = \"Method Not Allowed\"} \"\n\n Sabre_DAV_Exception_MethodNotAllowed\n The resource you tried to create already exists\n\n\" HTTP request: \"MKCOL\" \"/dav/mystuff/annex/.\" + failed + [2017-10-09 09:05:24.903485338] process done ExitSuccess + [2017-10-09 09:05:24.904346521] process done ExitSuccess + git-annex: copy: 1 failed + +git-annex-version + + git-annex version: 6.20171007-g7613a5e81 + build flags: Assistant Webapp Pairing Testsuite S3(multipartupload)(storageclasses) WebDAV Inotify ConcurrentOutput TorrentParser Feeds Quvi + dependency versions: aws-0.16 bloomfilter-2.0.1.0 cryptonite-0.21 DAV-1.3.1 feed-0.3.12.0 ghc-8.0.2 http-client-0.5.6.1 persistent-sqlite-2.6.2 torrent-10000.1.1 uuid-1.3.13 yesod-1.4.5 + key/value backends: SHA256E SHA256 SHA512E SHA512 SHA224E SHA224 SHA384E SHA384 SHA3_256E SHA3_256 SHA3_512E SHA3_512 SHA3_224E SHA3_224 SHA3_384E SHA3_384 SKEIN256E SKEIN256 SKEIN512E SKEIN512 SHA1E SHA1 MD5E MD5 WORM URL + remote types: git gcrypt p2p S3 bup directory rsync web bittorrent webdav tahoe glacier ddar hook external + local repository version: 5 + supported repository versions: 3 5 6 + upgrade supported from repository versions: 0 1 2 3 4 5 + operating system: linux x86_64 + +"""]] diff --git a/doc/bugs/Upload_to_box_very_slow/comment_4_9c503745b9c450f422ce47d9fd43e9b8._comment b/doc/bugs/Upload_to_box_very_slow/comment_4_9c503745b9c450f422ce47d9fd43e9b8._comment new file mode 100644 index 0000000000..0b3f640c7a --- /dev/null +++ b/doc/bugs/Upload_to_box_very_slow/comment_4_9c503745b9c450f422ce47d9fd43e9b8._comment @@ -0,0 +1,51 @@ +[[!comment format=mdwn + username="madalu" + avatar="http://cdn.libravatar.org/avatar/c00d4aa29fd053f08a2ef35531592914" + subject="Second test (without line 134)" + date="2017-10-09T14:40:45Z" + content=""" +I removed line 134 and rebuilt using Stack: + + maybe noop (void . mkColRecursive) (locationParent tmp) + +The file uploaded right away: + + % git annex --verbose --debug copy -t box test.txt + [2017-10-09 09:36:32.711847016] read: git [\"--git-dir=.git\",\"--work-tree=.\",\"--literal-pathspecs\",\"show-ref\",\"git-annex\"] + [2017-10-09 09:36:32.809619031] process done ExitSuccess + [2017-10-09 09:36:32.809888889] read: git [\"--git-dir=.git\",\"--work-tree=.\",\"--literal-pathspecs\",\"show-ref\",\"--hash\",\"refs/heads/git-annex\"] + [2017-10-09 09:36:32.812814521] process done ExitSuccess + [2017-10-09 09:36:32.829791929] chat: git [\"--git-dir=.git\",\"--work-tree=.\",\"--literal-pathspecs\",\"cat-file\",\"--batch\"] + [2017-10-09 09:36:32.830463363] chat: git [\"--git-dir=.git\",\"--work-tree=.\",\"--literal-pathspecs\",\"cat-file\",\"--batch-check=%(objectname) %(objecttype) %(objectsize)\"] + [2017-10-09 09:36:32.862446411] read: git [\"--git-dir=.git\",\"--work-tree=.\",\"--literal-pathspecs\",\"ls-files\",\"--cached\",\"-z\",\"--\",\"test.txt\"] + copy test.txt [2017-10-09 09:36:32.904592591] chat: gpg [\"--quiet\",\"--trust-model\",\"always\",\"--decrypt\"] + [2017-10-09 09:36:33.015053401] process done ExitSuccess + (checking box...) [2017-10-09 09:36:33.127413417] getProps 90a/d4d/GPGHMACSHA1--53636e5e7a50bc58eae478ddc260bb5abd899d03/GPGHMACSHA1--53636e5e7a50bc58eae478ddc260bb5abd899d03 + (to box...) + [2017-10-09 09:36:35.646456289] chat: gpg [\"--quiet\",\"--trust-model\",\"always\",\"--batch\",\"--passphrase-fd\",\"19\",\"--symmetric\",\"--force-mdc\",\"--no-textmode\"] + [2017-10-09 09:36:35.700070757] putContent git-annex-webdav-tmp-GPGHMACSHA1--b7d5df88040626033a471159a0652c9e3777d624 + 100% 153 B/s 0s[2017-10-09 09:36:37.830031092] delContent 18f/335/GPGHMACSHA1--b7d5df88040626033a471159a0652c9e3777d624/GPGHMACSHA1--b7d5df88040626033a471159a0652c9e3777d624 + [2017-10-09 09:36:38.453637089] getProps 18f/335/GPGHMACSHA1--b7d5df88040626033a471159a0652c9e3777d624 + [2017-10-09 09:36:38.891805895] mkCol 18f/335/GPGHMACSHA1--b7d5df88040626033a471159a0652c9e3777d624 + [2017-10-09 09:36:39.47474337] moveContent git-annex-webdav-tmp-GPGHMACSHA1--b7d5df88040626033a471159a0652c9e3777d624 https://dav.box.com/dav/mystuff/annex/18f/335/GPGHMACSHA1--b7d5df88040626033a471159a0652c9e3777d624/GPGHMACSHA1--b7d5df88040626033a471159a0652c9e3777d624 + [2017-10-09 09:36:40.325543883] process done ExitSuccess + ok + [2017-10-09 09:36:40.343371234] chat: git [\"--git-dir=.git\",\"--work-tree=.\",\"--literal-pathspecs\",\"hash-object\",\"-w\",\"--stdin-paths\",\"--no-filters\"] + [2017-10-09 09:36:40.346018836] feed: git [\"--git-dir=.git\",\"--work-tree=.\",\"--literal-pathspecs\",\"update-index\",\"-z\",\"--index-info\"] + [2017-10-09 09:36:40.574634504] process done ExitSuccess + [2017-10-09 09:36:40.574833127] read: git [\"--git-dir=.git\",\"--work-tree=.\",\"--literal-pathspecs\",\"show-ref\",\"--hash\",\"refs/heads/git-annex\"] + [2017-10-09 09:36:40.579083747] process done ExitSuccess + (recording state in git...) + [2017-10-09 09:36:40.579508229] read: git [\"--git-dir=.git\",\"--work-tree=.\",\"--literal-pathspecs\",\"write-tree\"] + [2017-10-09 09:36:40.908596805] process done ExitSuccess + [2017-10-09 09:36:40.908746261] chat: git [\"--git-dir=.git\",\"--work-tree=.\",\"--literal-pathspecs\",\"commit-tree\",\"890c8c30542bfb566cb0f6c18948661f9dba3477\",\"--no-gpg-sign\",\"-p\",\"refs/heads/git-annex\"] + [2017-10-09 09:36:40.911636345] process done ExitSuccess + [2017-10-09 09:36:40.911746341] call: git [\"--git-dir=.git\",\"--work-tree=.\",\"--literal-pathspecs\",\"update-ref\",\"refs/heads/git-annex\",\"8f02270fa945f2381b63ed2f8280f8f966628a70\"] + [2017-10-09 09:36:40.916033767] process done ExitSuccess + [2017-10-09 09:36:40.917348489] process done ExitSuccess + [2017-10-09 09:36:40.917759841] process done ExitSuccess + [2017-10-09 09:36:40.918326516] process done ExitSuccess + + + +"""]] diff --git a/doc/bugs/VFAT_crazy_limit_on_max_filenames_in_directory.mdwn b/doc/bugs/VFAT_crazy_limit_on_max_filenames_in_directory.mdwn new file mode 100644 index 0000000000..8c526ea620 --- /dev/null +++ b/doc/bugs/VFAT_crazy_limit_on_max_filenames_in_directory.mdwn @@ -0,0 +1,2 @@ +VFAT limits have been hit when the .git/annex/journal/ +directory gets a lot of stuff in it. See diff --git a/doc/bugs/Watcher_crashed__58___.git__92__annex__92__objects__92__...sha256....pdf__92____58___openTempFile__58___does_not_exist.mdwn b/doc/bugs/Watcher_crashed__58___.git__92__annex__92__objects__92__...sha256....pdf__92____58___openTempFile__58___does_not_exist.mdwn new file mode 100644 index 0000000000..347f8b3d5e --- /dev/null +++ b/doc/bugs/Watcher_crashed__58___.git__92__annex__92__objects__92__...sha256....pdf__92____58___openTempFile__58___does_not_exist.mdwn @@ -0,0 +1,10 @@ +### Please describe the problem. + +On windows 7 under assistant/webapp, we have the following message displayed : + +warning +Watcher crashed: .git\annex\objects\56b\ff2\SHA256E-s298110--0ff71fda9236dbf84954969add770bfbf90c800916450823335a1bd6c6229a5e.pdf\: openTempFile: does not exist (No such file or directory) + +git annex fsck indeed complains that several such files are missing. + +Hope this helps. diff --git a/doc/bugs/Watcher_crashed__58___.git__92__annex__92__objects__92__...sha256....pdf__92____58___openTempFile__58___does_not_exist/comment_1_9892f6b80597f0b637aeb96d212164d9._comment b/doc/bugs/Watcher_crashed__58___.git__92__annex__92__objects__92__...sha256....pdf__92____58___openTempFile__58___does_not_exist/comment_1_9892f6b80597f0b637aeb96d212164d9._comment new file mode 100644 index 0000000000..a4715c7937 --- /dev/null +++ b/doc/bugs/Watcher_crashed__58___.git__92__annex__92__objects__92__...sha256....pdf__92____58___openTempFile__58___does_not_exist/comment_1_9892f6b80597f0b637aeb96d212164d9._comment @@ -0,0 +1,7 @@ +[[!comment format=mdwn + username="OlivierBerger" + subject="More details" + date="2015-11-09T14:52:24Z" + content=""" +AFAIU, this happened when we moved a lot of files at once, trying to dispatch files into subdirs, in direct mode. Hope this helps +"""]] diff --git a/doc/bugs/Watcher_crashed__58___.git__92__annex__92__objects__92__...sha256....pdf__92____58___openTempFile__58___does_not_exist/comment_2_687a9fd1c0b760dd2bef9647cd16175c._comment b/doc/bugs/Watcher_crashed__58___.git__92__annex__92__objects__92__...sha256....pdf__92____58___openTempFile__58___does_not_exist/comment_2_687a9fd1c0b760dd2bef9647cd16175c._comment new file mode 100644 index 0000000000..23ea64bd31 --- /dev/null +++ b/doc/bugs/Watcher_crashed__58___.git__92__annex__92__objects__92__...sha256....pdf__92____58___openTempFile__58___does_not_exist/comment_2_687a9fd1c0b760dd2bef9647cd16175c._comment @@ -0,0 +1,11 @@ +[[!comment format=mdwn + username="joey" + subject="""comment 2""" + date="2015-12-02T17:32:32Z" + content=""" +What version of git-annex? + +Is it possible that the .git/annex/objects directory, +or something related to it (maybe .git itself) is one of the +files you moved? +"""]] diff --git a/doc/bugs/Watcher_crashed__58___.git__92__annex__92__objects__92__...sha256....pdf__92____58___openTempFile__58___does_not_exist/comment_3_1ef917c10f944e6c874aa40affae7602._comment b/doc/bugs/Watcher_crashed__58___.git__92__annex__92__objects__92__...sha256....pdf__92____58___openTempFile__58___does_not_exist/comment_3_1ef917c10f944e6c874aa40affae7602._comment new file mode 100644 index 0000000000..9830d3dce7 --- /dev/null +++ b/doc/bugs/Watcher_crashed__58___.git__92__annex__92__objects__92__...sha256....pdf__92____58___openTempFile__58___does_not_exist/comment_3_1ef917c10f944e6c874aa40affae7602._comment @@ -0,0 +1,11 @@ +[[!comment format=mdwn + username="OlivierBerger" + subject="Difficult to tell" + date="2015-12-04T10:21:26Z" + content=""" +Sorry, Joey, but I don't have the details of the version which crahed on us, and we updated afterwards : no time to try and reproduce fully : really busy producing videos, and anxious to find a solution on backing up videos reliably (which by the way I solved like this : https://www-public.tem-tsp.eu/~berger_o/weblog/2015/11/26/handling-video-files-produced-for-a-mooc-on-windows-with-git-and-git-annex/). + +I don't see however how we could have touched the contents of the .git/ : we were only doing Windows explorer actions : dragging files here and there, renaming dirs, saving stuff in Camtasia, but all inside subdirs of the git clone. + +Sorry for the poor quality of the report, but we were in a real hurry and had to find solutions without distracting us from the main topic : video editing :-/ +"""]] diff --git a/doc/bugs/Watcher_crashed_in_Android_on___47__storage__47__sdcard1_-_bug__63__.mdwn b/doc/bugs/Watcher_crashed_in_Android_on___47__storage__47__sdcard1_-_bug__63__.mdwn new file mode 100644 index 0000000000..270a69ecb2 --- /dev/null +++ b/doc/bugs/Watcher_crashed_in_Android_on___47__storage__47__sdcard1_-_bug__63__.mdwn @@ -0,0 +1,45 @@ +### Please describe the problem. + +In webapp UI, added on first install, the location for repository: /storage/sdcard1 + +!warning + +Watcher crashed: addWatch: + +permission denied (Permission denied) + +[Restart Thread] + +:Performing startup scan + +In terminal Window 1: + +nex webapp < + +Detected a crippled filesystem. + +Enabling direct mode. + +Detected a filesystem without fifo support. + +Disabling ssh connection caching. + +### What steps will reproduce the problem? + +In webapp UI, added on first install, the location for repository: /storage/sdcard1 + +### What version of git-annex are you using? On what operating system? + +Android 4.1.1 Huawei Y300 Annex.apk v1.0.52 version 4.20130723 + +### Please provide any additional information below. + +[[!format sh """ +# If you can, paste a complete transcript of the problem occurring here. +# If the problem is with the git-annex assistant, paste in .git/annex/daemon.log + + +# End of transcript or log. +"""]] + +[[!tag moreinfo]] diff --git a/doc/bugs/Watcher_crashed_in_Android_on___47__storage__47__sdcard1_-_bug__63__/comment_1_71b052be40fbdaca09ca3ede8c59ac7a._comment b/doc/bugs/Watcher_crashed_in_Android_on___47__storage__47__sdcard1_-_bug__63__/comment_1_71b052be40fbdaca09ca3ede8c59ac7a._comment new file mode 100644 index 0000000000..0622a24391 --- /dev/null +++ b/doc/bugs/Watcher_crashed_in_Android_on___47__storage__47__sdcard1_-_bug__63__/comment_1_71b052be40fbdaca09ca3ede8c59ac7a._comment @@ -0,0 +1,8 @@ +[[!comment format=mdwn + username="http://joeyh.name/" + ip="4.154.0.21" + subject="comment 1" + date="2013-07-30T18:42:32Z" + content=""" +You should have a `/storage/sdcard1/.git/annex/daemon.log` file. Please paste it here. +"""]] diff --git a/doc/bugs/Watcher_crashed_in_Android_on___47__storage__47__sdcard1_-_bug__63__/comment_3_0f7cc02e0193c969c9b6ceb27e71af8a._comment b/doc/bugs/Watcher_crashed_in_Android_on___47__storage__47__sdcard1_-_bug__63__/comment_3_0f7cc02e0193c969c9b6ceb27e71af8a._comment new file mode 100644 index 0000000000..0ea47dc402 --- /dev/null +++ b/doc/bugs/Watcher_crashed_in_Android_on___47__storage__47__sdcard1_-_bug__63__/comment_3_0f7cc02e0193c969c9b6ceb27e71af8a._comment @@ -0,0 +1,20 @@ +[[!comment format=mdwn + username="https://www.google.com/accounts/o8/id?id=AItOawnaH44G3QbxBAYyDwy0PbvL0ls60XoaR3Y" + nickname="Nigel" + subject="/storage/sdcard1/.git/annex/daemon.log" + date="2013-07-30T23:11:29Z" + content=""" + [2013-07-30 23:17:43 BST] main: starting assistant version 4.20130723-ged05a63 + (scanning...) [2013-07-30 23:17:44 BST] Watcher: Performing startup scan + Watcher crashed: addWatch: permission denied (Permission denied) + [2013-07-30 23:17:45 BST] Watcher: warning Watcher crashed: addWatch: permission denied (Permission denied) + [2013-07-30 23:17:45 BST] Committer: Adding 229286 229381 + add LOST.DIR/229286 (checksum...) ok + add LOST.DIR/229381 (checksum...) [2013-07-30 23:17:47 BST] Committer: Committing changes to git + [2013-07-30 23:20:22 BST] Committer: Adding git-annex..stall.log + ok + (Recording state in git...) + (Recording state in git...) + add git-annex-install.log (checksum...) ok + add git-annex-install.log (checksum...) [2013-07-30 23:20:22 BST] Committer: Committing changes to git +"""]] diff --git a/doc/bugs/Watcher_crashed_in_Android_on___47__storage__47__sdcard1_-_bug__63__/comment_3_1a7542249b9c37507126e97441057c12._comment b/doc/bugs/Watcher_crashed_in_Android_on___47__storage__47__sdcard1_-_bug__63__/comment_3_1a7542249b9c37507126e97441057c12._comment new file mode 100644 index 0000000000..9bc0c90e64 --- /dev/null +++ b/doc/bugs/Watcher_crashed_in_Android_on___47__storage__47__sdcard1_-_bug__63__/comment_3_1a7542249b9c37507126e97441057c12._comment @@ -0,0 +1,12 @@ +[[!comment format=mdwn + username="http://joeyh.name/" + ip="209.250.56.246" + subject="comment 3" + date="2013-11-10T18:18:09Z" + content=""" +Seems to me this was probably fixed in [[!commit 6b37fcffd872b62fb78047a77e4cee5ab0bb57f1]]. + +However, I don't know why permission problems would happen on Android, which typically uses a filesystem that does not have restricted permissions for the sdcard. + +If you upgrade, you should be able to see if it fixed the crash, and it will also mention any file or directory that it's unable to read in the log. +"""]] diff --git a/doc/bugs/When_stopping___96__git_annex_get__96___files_left_broken.mdwn b/doc/bugs/When_stopping___96__git_annex_get__96___files_left_broken.mdwn new file mode 100644 index 0000000000..a8df723548 --- /dev/null +++ b/doc/bugs/When_stopping___96__git_annex_get__96___files_left_broken.mdwn @@ -0,0 +1,40 @@ +### Please describe the problem. + +``` +> git annex get Narnia/ +get Narnia/Course of a Generation/01 Sail Around the World.mp3 (from Seagate...) +SHA256E-s8395599--2fea961006a279f0765c45755b35a06f0a4fc6bfbab6118182ebc693d7b47a91.mp3 + 8,395,599 100% 29.65MB/s 0:00:00 (xfr#1, to-chk=0/1) +(checksum...) ^C⏎ +``` + +``` +> mpv ~/Music/sorted/Narnia/Course\ of\ a\ Generation/ +Playing: /home/philip/Music/sorted/Narnia/Course of a Generation/ +[file] This is a directory - adding to playlist. + +Playing: /home/philip/Music/sorted/Narnia/Course of a Generation/01 Sail Around the World.mp3 +Failed to recognize file format. + +Playing: /home/philip/Music/sorted/Narnia/Course of a Generation/02 When the Stars Are Falling.mp3 +``` + +``` +> git annex version +git-annex version: 6.20161012 +build flags: Assistant Webapp Pairing Testsuite S3(multipartupload)(storageclasses) WebDAV Inotify DBus DesktopNotify XMPP ConcurrentOutput TorrentParser MagicMime Feeds Quvi +key/value backends: SHA256E SHA256 SHA512E SHA512 SHA224E SHA224 SHA384E SHA384 SHA3_256E SHA3_256 SHA3_512E SHA3_512 SHA3_224E SHA3_224 SHA3_384E SHA3_384 SKEIN256E SKEIN256 SKEIN512E SKEIN512 SHA1E SHA1 MD5E MD5 WORM URL +remote types: git gcrypt S3 bup directory rsync web bittorrent webdav tahoe glacier ddar hook external +local repository version: 6 +supported repository versions: 3 5 6 +upgrade supported from repository versions: 0 1 2 3 4 5 +operating system: linux x86_64 +``` + +Any consecutive `git annex get` commands don’t notice that the file is not completely transferred and leave it in a broken state. +`git annex get --failed` does not correct the problem. + + +### Have you had any luck using git-annex before? (Sometimes we get tired of reading bug reports all day and a lil' positive end note does wonders) + +Yes, it (kind of) works for keeping my music library in sync. diff --git a/doc/bugs/When_stopping___96__git_annex_get__96___files_left_broken/comment_1_9392346203c561b88f30fa2ce7540b76._comment b/doc/bugs/When_stopping___96__git_annex_get__96___files_left_broken/comment_1_9392346203c561b88f30fa2ce7540b76._comment new file mode 100644 index 0000000000..4f90ddfa69 --- /dev/null +++ b/doc/bugs/When_stopping___96__git_annex_get__96___files_left_broken/comment_1_9392346203c561b88f30fa2ce7540b76._comment @@ -0,0 +1,22 @@ +[[!comment format=mdwn + username="joey" + subject="""comment 1""" + date="2016-11-16T18:36:34Z" + content=""" +Thing is, git-annex get does not update the file in place. Only once the +entire file is downloaded, and its content is verified correct is it moved +into a place where you can access it. + +So, it seems much more likely to me that the content of the file, as +originally added to git-annex, was bad, and the it had just finished +verifying the content and moving it into place when you interruped the +command. + +Please check with `git annex fsck` on the file and see if it determines +it has the content git-annex expects it to have. + +However, I notice you're using a v6 repository. Is the file an unlocked +file? It's possible that in that specific case there could be a bug. +I've interrupted `git annex get` on a nearly daily basis for years, but +v6 is still experimental and not as well tested. +"""]] diff --git a/doc/bugs/Windows__58___Doesn__39__t_want_to_handle_files_with_2_dots_in_filename.mdwn b/doc/bugs/Windows__58___Doesn__39__t_want_to_handle_files_with_2_dots_in_filename.mdwn new file mode 100644 index 0000000000..3a0da6b1dc --- /dev/null +++ b/doc/bugs/Windows__58___Doesn__39__t_want_to_handle_files_with_2_dots_in_filename.mdwn @@ -0,0 +1,29 @@ +### Please describe the problem. + +Git annex assistant on Windows 7 won't handle files with 2 dots in the filename + +### What steps will reproduce the problem? + +Create a file like "3.2 Restriction et projection.mp4" which is in a directory synced by Assistant. + +The file is unmanaged. + +Renaming it to "3-2 Restriction et projection.mp4", and there it goes : synced with the remote repo. + +### What version of git-annex are you using? On what operating system? + +Latext git-annex on Win 7 + +### Please provide any additional information below. + +[[!format sh """ +# If you can, paste a complete transcript of the problem occurring here. +# If the problem is with the git-annex assistant, paste in .git/annex/daemon.log + + +# End of transcript or log. +"""]] + +### Have you had any luck using git-annex before? (Sometimes we get tired of reading bug reports all day and a lil' positive end note does wonders) + + diff --git a/doc/bugs/Windows__58___Doesn__39__t_want_to_handle_files_with_2_dots_in_filename/comment_1_43f6de166d1811e9f9bd6e6d8b152e05._comment b/doc/bugs/Windows__58___Doesn__39__t_want_to_handle_files_with_2_dots_in_filename/comment_1_43f6de166d1811e9f9bd6e6d8b152e05._comment new file mode 100644 index 0000000000..1012cc729f --- /dev/null +++ b/doc/bugs/Windows__58___Doesn__39__t_want_to_handle_files_with_2_dots_in_filename/comment_1_43f6de166d1811e9f9bd6e6d8b152e05._comment @@ -0,0 +1,9 @@ +[[!comment format=mdwn + username="joey" + subject="""comment 1""" + date="2015-09-22T15:56:37Z" + content=""" +Unable to reproduce this on Windows XP. + +Is there anything in .git/annex/daemon.log that might help with debugging? +"""]] diff --git a/doc/bugs/Windows__58___git_annex_get_failed.mdwn b/doc/bugs/Windows__58___git_annex_get_failed.mdwn new file mode 100644 index 0000000000..6c7416653f --- /dev/null +++ b/doc/bugs/Windows__58___git_annex_get_failed.mdwn @@ -0,0 +1,90 @@ +### Please describe the problem. +I'm using git-annex v6 in windows and seems `git annex get` fails. +Deleting the pointer files and doing a hard reset seems to fix the problem. +Another bug report named "v6 repo can not restore files with executable permission" seems to point to the same problem but a little different. + +[[!retitle "git annex get fails sometimes in v6 repository on Windows"]] + +### What steps will reproduce the problem? +I have a repo in D:\annex with a file `test` +Now I create another repo: + + $ cd H:\annex + + $ git init + Initialized empty Git repository in H:/annex/.git/ + + $ git annex init "portable drive" + init portable drive + Detected a filesystem without fifo support. + + Disabling ssh connection caching. + + Detected a crippled filesystem. + + Enabling direct mode. + ok + (recording state in git...) + + $ git annex upgrade + upgrade . (v5 to v6...) + Upgraded repository out of direct mode. + + Changes have been staged for all annexed files in this repository; you should run `git commit` to commit these changes. + + Any other clones of this repository that use direct mode need to be upgraded now, too. + ok + + $ git remote add laptop D:/annex + + $ git annex sync + commit ok + pull laptop + warning: no common commits + remote: Counting objects: 21, done. + remote: Compressing objects: 100% (15/15), done. + remote: Total 21 (delta 3), reused 0 (delta 0) + Unpacking objects: 100% (21/21), done. + From D:/annex + * [new branch] git-annex -> laptop/git-annex + * [new branch] master -> laptop/master + * [new branch] synced/git-annex -> laptop/synced/git-annex + * [new branch] synced/master -> laptop/synced/master + + + Already up-to-date. + ok + (merging laptop/git-annex laptop/synced/git-annex into git-annex...) + (recording state in git...) + push laptop + Counting objects: 8, done. + Delta compression using up to 8 threads. + Compressing objects: 100% (6/6), done. + Writing objects: 100% (8/8), 928 bytes | 0 bytes/s, done. + Total 8 (delta 0), reused 0 (delta 0) + To D:/annex + c1aee82..980dc01 git-annex -> synced/git-annex + ok + + $ git annex get . + get test (from laptop...) + SHA256E-s14488367--4391729b982439764813156e1bfc12e9626ae89452ab812f5180c376fbd57fc0 + 14,488,367 100% 63.24MB/s 0:00:00 (xfr#1, to-chk=0/1) + (checksum...) + git-annex: DeleteFile ".\\test": permission denied (The process cannot access the file because it is being used by another process.) + failed + git-annex: get: 1 failed + +It seems to try to delete the pointer file, but finds the file in use. Maybe fsck is using it? + + $ cat test + /annex/objects/SHA256E-s14488367--4391729b982439764813156e1bfc12e9626ae89452ab812f5180c376fbd57fc0 + + $ git annex lock + lock test git-annex: content not present; cannot lock + +And `git annex unlock` will do nothing. + +If you can't reproduce the problem, I'll run the tests for you. +### What version of git-annex are you using? On what operating system? +Latest compile from source, Windows 8.1 diff --git a/doc/bugs/Windows__58___git_annex_get_failed/comment_2_0866038e64aa20c3b9ba3b3190074293._comment b/doc/bugs/Windows__58___git_annex_get_failed/comment_2_0866038e64aa20c3b9ba3b3190074293._comment new file mode 100644 index 0000000000..4e77a505c8 --- /dev/null +++ b/doc/bugs/Windows__58___git_annex_get_failed/comment_2_0866038e64aa20c3b9ba3b3190074293._comment @@ -0,0 +1,20 @@ +[[!comment format=mdwn + username="joey" + subject="""comment 2""" + date="2016-04-04T19:24:08Z" + content=""" +Note that v6 mode is still considered experimental. + +I know that I saw a problem like this, where git-annex was reading the +pointer file and didn't always get around to closing it before it tried to +replace it. I thought I fixed it back in February though -- but that fix +may have bit rotted; looking at the current code, I think it could leave +the file open until a GC pass, so if the GC doesn't always +run in time, that could explain the reported indeterminacy. + +Please paste the actual version of git-annex you're using, +"latest build from source" is a little bit vague. + +Since you are building from source, you might try pulling; +I've made a change that may fix it again. +"""]] diff --git a/doc/bugs/Windows__58___git_annex_get_failed/comment_3_22809831226478bac7f9361ad8d0c79d._comment b/doc/bugs/Windows__58___git_annex_get_failed/comment_3_22809831226478bac7f9361ad8d0c79d._comment new file mode 100644 index 0000000000..7ca9875f1a --- /dev/null +++ b/doc/bugs/Windows__58___git_annex_get_failed/comment_3_22809831226478bac7f9361ad8d0c79d._comment @@ -0,0 +1,10 @@ +[[!comment format=mdwn + username="ExGen" + subject="Found?" + date="2016-04-03T20:15:01Z" + content=""" +I think I found the problem:
    +After I edit the file and commit, if I **lock** the file and **commit again**, the problem won't happen.
    +I thought there shouldn't be a difference between committing a locked file and an unlocked file?

    +There is another problem too: When you unlock the file, git thinks you changed it. Maybe you could fix this too? +"""]] diff --git a/doc/bugs/Windows__58___git_annex_get_failed/comment_3_8a19ca100d44ab8131e40932f1bcdf29._comment b/doc/bugs/Windows__58___git_annex_get_failed/comment_3_8a19ca100d44ab8131e40932f1bcdf29._comment new file mode 100644 index 0000000000..3140fee364 --- /dev/null +++ b/doc/bugs/Windows__58___git_annex_get_failed/comment_3_8a19ca100d44ab8131e40932f1bcdf29._comment @@ -0,0 +1,10 @@ +[[!comment format=mdwn + username="ExGen" + subject="comment 3^^" + date="2016-04-08T21:03:15Z" + content=""" +It's 6.20160401-g083dd8f, Sorry for the delay. +And btw my v6 repo is not in adjusted mode. But I think adjusted mode has the same problem too. + + +"""]] diff --git a/doc/bugs/Windows__58___git_annex_get_failed/comment_4_b70fa3da7e776a6cd6d8467df30839c9._comment b/doc/bugs/Windows__58___git_annex_get_failed/comment_4_b70fa3da7e776a6cd6d8467df30839c9._comment new file mode 100644 index 0000000000..a6d7ebecdc --- /dev/null +++ b/doc/bugs/Windows__58___git_annex_get_failed/comment_4_b70fa3da7e776a6cd6d8467df30839c9._comment @@ -0,0 +1,35 @@ +[[!comment format=mdwn + username="ExGen" + subject="comment 4" + date="2016-04-08T22:00:33Z" + content=""" +I pulled and checked 6.20160408-gf970490.
    +There are several problems now:
    + +1. When I `sync` on a new repo it `get`s all the files
    +2. `drop` doesn't drop the file +3. `lock` doesn't work
    +4. I don't know if the problem with `get` really fixed because it isn't needed + +Btw, look at my comment on /todo/smudge (maybe put on a wrong page)
    +I think using hardlinks on windows can fix many problems here ( adjusted/direct mode which I hate :)) )
    +And it should need little work as tools already accept it on windows + +I'll put all the commands needed to reproduce the problem: + + cd D:/annex + git init + git annex init \"laptop\" + git annex upgrade + git add . + git commit -a -m \"First\" + git annex lock + git annex sync + + cd H:/annex + git init + git annex init \"usb\" + git annex upgrade + git remote add laptop D:/annex + git annex sync +"""]] diff --git a/doc/bugs/Windows___58___git-annex__58___failed_to_read_sha_from_git_write-tree.mdwn b/doc/bugs/Windows___58___git-annex__58___failed_to_read_sha_from_git_write-tree.mdwn new file mode 100644 index 0000000000..57f95d992b --- /dev/null +++ b/doc/bugs/Windows___58___git-annex__58___failed_to_read_sha_from_git_write-tree.mdwn @@ -0,0 +1,46 @@ +### Please describe the problem. + +On windows, git-annex webapp reports a crash. + +On the command-line, I get : + +$ git annex fsck +fatal: unable to read tree 6d692a016b7d772800ddc3993ffd68fc6aaa21f3 +error: invalid object 100644 39e3d739a15dc3ed11a0c324781bf3658ccb933a for '001/e8c/SHA256E-s38500--9017f32c23b4de6363d3e5105bc1c47dd3cd983e471e71e0a9d420daaf0e3e7b.html.log' +fatal: git-write-tree: error building trees +(merging origin/git-annex into git-annex...) +(recording state in git...) +git-annex: failed to read sha from git write-tree + +Then, I try : + +$ git annex repair +Running git fsck ... +Unpacking all pack files. + +git-annex: DeleteFile ".git\\objects\\pack\\pack-c53ea32e9514b6dda95af1d8d879d91d32240d76.idx": permission denied (Accès refusé.) +failed +git-annex: repair: 1 failed + + +### What steps will reproduce the problem? + +Dunno how that got messed up, and what to do now... + +### What version of git-annex are you using? On what operating system? + +latest nightly build (2015/10/19) on win 7 + +### Please provide any additional information below. + +[[!format sh """ +# If you can, paste a complete transcript of the problem occurring here. +# If the problem is with the git-annex assistant, paste in .git/annex/daemon.log + + +# End of transcript or log. +"""]] + +### Have you had any luck using git-annex before? (Sometimes we get tired of reading bug reports all day and a lil' positive end note does wonders) + + diff --git a/doc/bugs/Windows___58___git-annex__58___failed_to_read_sha_from_git_write-tree/comment_1_5829b551f5e1413ee236055eb3345760._comment b/doc/bugs/Windows___58___git-annex__58___failed_to_read_sha_from_git_write-tree/comment_1_5829b551f5e1413ee236055eb3345760._comment new file mode 100644 index 0000000000..3ed4e18868 --- /dev/null +++ b/doc/bugs/Windows___58___git-annex__58___failed_to_read_sha_from_git_write-tree/comment_1_5829b551f5e1413ee236055eb3345760._comment @@ -0,0 +1,15 @@ +[[!comment format=mdwn + username="joey" + subject="""comment 1""" + date="2015-10-26T17:34:09Z" + content=""" +Why can't you read files in your git repository? This must be due to some +permissions issue, etc. I know nothing about permissons in Windows, so +can't offer any useful advice, beyond the fact that, for git (or git-annex) +to work, you need to be able to read all the files in the git repository. + +You may also have some form of corruption of the files in the repository. +Perhaps your disk is going bad or something? + +I don't see any indications of a git-annex bug here. +"""]] diff --git a/doc/bugs/Windows___58___git-annex__58___failed_to_read_sha_from_git_write-tree/comment_2_535016130f3307c8104168e6bb8a583d._comment b/doc/bugs/Windows___58___git-annex__58___failed_to_read_sha_from_git_write-tree/comment_2_535016130f3307c8104168e6bb8a583d._comment new file mode 100644 index 0000000000..bda2a400dc --- /dev/null +++ b/doc/bugs/Windows___58___git-annex__58___failed_to_read_sha_from_git_write-tree/comment_2_535016130f3307c8104168e6bb8a583d._comment @@ -0,0 +1,7 @@ +[[!comment format=mdwn + username="aloukian" + subject="Your directory depth is too long" + date="2015-11-19T00:13:21Z" + content=""" +I've had a similar problem when the location of my annex was too deep. Windows only allows 256 char filenames. Git annex takes up a fair bit of that by storing files named according to their hash. I worked around the problem by creating my repo at the drive root and then creating a \"junction\" between the place where I wanted it to be and that location. It seems to work now. +"""]] diff --git a/doc/bugs/Windows___58___git-annex__58___failed_to_read_sha_from_git_write-tree/comment_3_79c66569c22a1280b807a3c21c2cdc4f._comment b/doc/bugs/Windows___58___git-annex__58___failed_to_read_sha_from_git_write-tree/comment_3_79c66569c22a1280b807a3c21c2cdc4f._comment new file mode 100644 index 0000000000..c95034fdfe --- /dev/null +++ b/doc/bugs/Windows___58___git-annex__58___failed_to_read_sha_from_git_write-tree/comment_3_79c66569c22a1280b807a3c21c2cdc4f._comment @@ -0,0 +1,20 @@ +[[!comment format=mdwn + username="joey" + subject="""comment 3""" + date="2015-12-02T17:22:01Z" + content=""" +Windows certianly does cause problems with its short path limits. The +SHA512 hash backend is known to not work at all on windows, for example. + +However, recent versions of git-annex use relative paths to files, so it +shouldn't much matter if they're run in a directory deep in a tree; it'll +refer to '.git/foo' rather than 'c:/verylong/path/to/.git/foo' + +Also, a long path is certianly not the cause of + + DeleteFile ".git\\objects\\pack\\pack-c53ea32e9514b6dda95af1d8d879d91d32240d76.idx": permission denied + +Since the path there is relative, is plenty short to not bother Windows's +tiny mind, and there's clearly some sort of permission problem preventing +this user from modifying this file. +"""]] diff --git a/doc/bugs/Windows___58___git-annex__58___failed_to_read_sha_from_git_write-tree/comment_4_2e5791aa8f568f86d0185a553227afa8._comment b/doc/bugs/Windows___58___git-annex__58___failed_to_read_sha_from_git_write-tree/comment_4_2e5791aa8f568f86d0185a553227afa8._comment new file mode 100644 index 0000000000..31a51db950 --- /dev/null +++ b/doc/bugs/Windows___58___git-annex__58___failed_to_read_sha_from_git_write-tree/comment_4_2e5791aa8f568f86d0185a553227afa8._comment @@ -0,0 +1,7 @@ +[[!comment format=mdwn + username="OlivierBerger" + subject="Again, difficult to tell" + date="2015-12-04T12:24:06Z" + content=""" +Well, I really don't understand either what may have caused the crashes... but the path issue might be a hint. I think our repo was cloned in a path something like : \"C:\Users\Majdi\git\git-annex.tem-tsp.eu\testscpo\" which already takes around 48 chars. +"""]] diff --git a/doc/bugs/Windows_build_test_failures.mdwn b/doc/bugs/Windows_build_test_failures.mdwn new file mode 100644 index 0000000000..c276c294e1 --- /dev/null +++ b/doc/bugs/Windows_build_test_failures.mdwn @@ -0,0 +1,1232 @@ +Given that others can build+run OK I wonder if this is a build environment setup problem rather than a code issue. +This might be two bugs, but I rather suspect the fix to one is the fix to the other. + +### Please describe the problem. +Some tests fail with: +ssh: Could not resolve hostname C: no address associated with name + +then if you try and do a copy to/from an ssh remote then it fails with simply 'copy: 1 failed' + +### What steps will reproduce the problem? + +For the tests, simply sh standalone/windows/build.sh + +For the other, I'm using a modified version of the other windows user's script, the set -x'd output is: + +
    ++ git-annex version
+git-annex version: 4.20130621-g36258de^M
+build flags: Pairing Testsuite S3 WebDAV DNS^M
++ ssh gitremote sh testrepo.sh
+git-annex version: 4.20130621-g36258de
+build flags: Assistant Webapp Pairing Testsuite S3 WebDAV Inotify DBus XMPP DNS
+Initialized empty Git repository in /media/backup/git/repo.git/
+init origin ok
+(Recording state in git...)
+commit
+ok
+git-annex: no branch is checked out
++ rm -rf repo
++ git init repo
+Initialized empty Git repository in c:/Users/Oliver/repo/.git/
++ cd repo
++ git-annex init
+init  ^M
+  Detected a crippled filesystem.^M
+^M
+  Enabling direct mode.^M
+^M
+  Detected a filesystem without fifo support.^M
+^M
+  Disabling ssh connection caching.^M
+ok^M
+(Recording state in git...)^M
++ git remote add origin ssh://git@remote/~/repo.git
++ echo hello
++ git-annex add .
+add foo.txt (checksum...) ok^M
+(Recording state in git...)^M
++ git commit -m .
+[master (root-commit) f34a076] .
+ 1 file changed, 1 insertion(+)
+ create mode 120000 foo.txt
++ git-annex sync
+commit  ^M
+ok^M
+pull origin warning: no common commits
+From ssh://remote/~/repo
+ * [new branch]      git-annex  -> origin/git-annex
+^M
+ok^M
+(merging origin/git-annex into git-annex...)^M
+(Recording state in git...)^M
+push origin To ssh://git@remote/~/repo.git
+ * [new branch]      git-annex -> synced/git-annex
+ * [new branch]      master -> synced/master
+^M
+ok^M
++ git-annex copy --to origin
+copy foo.txt (checking origin...) (to origin...) ^M
+failed^M
+git-annex.exe: copy: 1 failed
+
    + + + + + +### What version of git-annex are you using? On what operating system? +Windows (8 x64): +git-annex version: 4.20130621-g36258de +build flags: Pairing Testsuite S3 WebDAV DNS +Linux (debian wheezy i386): +git-annex version: 4.20130621-g36258de +build flags: Assistant Webapp Pairing Testsuite S3 WebDAV Inotify DBus XMPP DNS + +other versions +git-bash: git(1.8.0.msysgit.0),ssh(ssh 4.6p1, ssl 0.9.8e 23 feb 2007) +msys: git(none) ssh(ssh 5.4p1, ssl 1.0.0 29 Mar 2010) +cygwin: git(1.7.9) ssh(none) + +### Please provide any additional information below. + +[[!format sh """ +# If you can, paste a complete transcript of the problem occurring here. +# If the problem is with the git-annex assistant, paste in .git/annex/daemon.log + +build.log: ++ set -e ++ HP='/c/Program Files (x86)/Haskell Platform/2012.4.0.0' ++ FLAGS='-Webapp -Assistant -XMPP' ++ PATH='/c/Program Files (x86)/Haskell Platform/2012.4.0.0/bin:/c/Program Files (x86)/Haskell Platform/2012.4.0.0/lib/extralibs/bin:/c/Program Files (x86)/NSIS:/home/Oliver/bin:.:/usr/local/bin:/mingw/bin:/bin:/c/Program Files (x86)/Haskell/bin:/c/Program Files (x86)/Haskell Platform/2013.2.0.0/lib/extralibs/bin:/c/Program Files (x86)/Haskell Platform/2013.2.0.0/bin:/c/Program Files (x86)/AMD APP/bin/x86_64:/c/Program Files (x86)/AMD APP/bin/x86:/c/Windows/system32:/c/Windows:/c/Windows/System32/Wbem:/c/Windows/System32/WindowsPowerShell/v1.0/:/c/Program Files (x86)/ATI Technologies/ATI.ACE/Core-Static:/c/Program Files (x86)/Windows Kits/8.0/Windows Performance Toolkit/:/c/Program Files/Microsoft/Web Platform Installer/:/c/Program Files (x86)/Haskell Platform/2013.2.0.0/mingw/bin:/c/Users/Oliver/AppData/Roaming/cabal/bin' ++ rm -f git-annex-installer.exe ++ cabal update +Downloading the latest package list from hackage.haskell.org ++ rm -rf MissingH-1.2.0.0 ++ cabal unpack MissingH +Unpacking to MissingH-1.2.0.0\ ++ cd MissingH-1.2.0.0 ++ withcyg patch -p1 ++ PATH='/c/Program Files (x86)/Haskell Platform/2012.4.0.0/bin:/c/Program Files (x86)/Haskell Platform/2012.4.0.0/lib/extralibs/bin:/c/Program Files (x86)/NSIS:/home/Oliver/bin:.:/usr/local/bin:/mingw/bin:/bin:/c/Program Files (x86)/Haskell/bin:/c/Program Files (x86)/Haskell Platform/2013.2.0.0/lib/extralibs/bin:/c/Program Files (x86)/Haskell Platform/2013.2.0.0/bin:/c/Program Files (x86)/AMD APP/bin/x86_64:/c/Program Files (x86)/AMD APP/bin/x86:/c/Windows/system32:/c/Windows:/c/Windows/System32/Wbem:/c/Windows/System32/WindowsPowerShell/v1.0/:/c/Program Files (x86)/ATI Technologies/ATI.ACE/Core-Static:/c/Program Files (x86)/Windows Kits/8.0/Windows Performance Toolkit/:/c/Program Files/Microsoft/Web Platform Installer/:/c/Program Files (x86)/Haskell Platform/2013.2.0.0/mingw/bin:/c/Users/Oliver/AppData/Roaming/cabal/bin:/c/cygwin/bin' ++ patch -p1 +patching file src/System/IO/WindowsCompat.hs +Hunk #1 succeeded at 119 (offset -1 lines). +Hunk #2 succeeded at 132 (offset -1 lines). ++ cabal install +Resolving dependencies... +In order, the following would be installed: +MissingH-1.2.0.0 (reinstall) +cabal.exe: The following packages are likely to be broken by the reinstalls: +hS3-0.5.7 +Use --force-reinstalls if you want to install anyway. ++ true ++ cd .. ++ cabal install --only-dependencies '-f-Webapp -Assistant -XMPP' +Resolving dependencies... +All the requested packages are already installed: +Use --reinstall if you want to reinstall anyway. ++ '[' -e last-incremental-failed ']' ++ touch last-incremental-failed ++ withcyg cabal configure '-f-Webapp -Assistant -XMPP' ++ PATH='/c/Program Files (x86)/Haskell Platform/2012.4.0.0/bin:/c/Program Files (x86)/Haskell Platform/2012.4.0.0/lib/extralibs/bin:/c/Program Files (x86)/NSIS:/home/Oliver/bin:.:/usr/local/bin:/mingw/bin:/bin:/c/Program Files (x86)/Haskell/bin:/c/Program Files (x86)/Haskell Platform/2013.2.0.0/lib/extralibs/bin:/c/Program Files (x86)/Haskell Platform/2013.2.0.0/bin:/c/Program Files (x86)/AMD APP/bin/x86_64:/c/Program Files (x86)/AMD APP/bin/x86:/c/Windows/system32:/c/Windows:/c/Windows/System32/Wbem:/c/Windows/System32/WindowsPowerShell/v1.0/:/c/Program Files (x86)/ATI Technologies/ATI.ACE/Core-Static:/c/Program Files (x86)/Windows Kits/8.0/Windows Performance Toolkit/:/c/Program Files/Microsoft/Web Platform Installer/:/c/Program Files (x86)/Haskell Platform/2013.2.0.0/mingw/bin:/c/Users/Oliver/AppData/Roaming/cabal/bin:/c/cygwin/bin' ++ cabal configure '-f-Webapp -Assistant -XMPP' +Resolving dependencies... +[ 1 of 24] Compiling Utility.Applicative ( Utility\Applicative.hs, dist\setup\Utility\Applicative.o ) +[ 2 of 24] Compiling Utility.PartialPrelude ( Utility\PartialPrelude.hs, dist\setup\Utility\PartialPrelude.o ) +[ 3 of 24] Compiling Utility.FileSystemEncoding ( Utility\FileSystemEncoding.hs, dist\setup\Utility\FileSystemEncoding.o ) +[ 4 of 24] Compiling Utility.Exception ( Utility\Exception.hs, dist\setup\Utility\Exception.o ) +[ 5 of 24] Compiling Utility.Misc ( Utility\Misc.hs, dist\setup\Utility\Misc.o ) +[ 6 of 24] Compiling Utility.Process ( Utility\Process.hs, dist\setup\Utility\Process.o ) +[ 7 of 24] Compiling Utility.Env ( Utility\Env.hs, dist\setup\Utility\Env.o ) +[ 8 of 24] Compiling Utility.UserInfo ( Utility\UserInfo.hs, dist\setup\Utility\UserInfo.o ) +[ 9 of 24] Compiling Utility.OSX ( Utility\OSX.hs, dist\setup\Utility\OSX.o ) +[10 of 24] Compiling Utility.Tmp ( Utility\Tmp.hs, dist\setup\Utility\Tmp.o ) +[11 of 24] Compiling Utility.Monad ( Utility\Monad.hs, dist\setup\Utility\Monad.o ) +[12 of 24] Compiling Utility.Path ( Utility\Path.hs, dist\setup\Utility\Path.o ) +[13 of 24] Compiling Utility.FreeDesktop ( Utility\FreeDesktop.hs, dist\setup\Utility\FreeDesktop.o ) +[16 of 24] Compiling Utility.SafeCommand ( Utility\SafeCommand.hs, dist\setup\Utility\SafeCommand.o ) +[17 of 24] Compiling Utility.ExternalSHA ( Utility\ExternalSHA.hs, dist\setup\Utility\ExternalSHA.o ) +[18 of 24] Compiling Utility.Directory ( Utility\Directory.hs, dist\setup\Utility\Directory.o ) +Linking .\dist\setup\setup.exe ... + checking version... 4.20130621-g36258de + checking git... yes + checking git version... 1.7.9 + checking cp -a... yes + checking cp -p... yes + checking cp --reflink=auto... no + checking xargs -0... yes + checking rsync... yes + checking curl... no + checking wget... yes + checking bup... no + checking gpg... not available + checking lsof... not available + checking ssh connection caching... no + checking sha1... sha1sum + checking sha256... sha256sum + checking sha512... sha512sum + checking sha224... sha224sum + checking sha384... sha384sum +Configuring git-annex-4.20130601... ++ withcyg cabal build ++ PATH='/c/Program Files (x86)/Haskell Platform/2012.4.0.0/bin:/c/Program Files (x86)/Haskell Platform/2012.4.0.0/lib/extralibs/bin:/c/Program Files (x86)/NSIS:/home/Oliver/bin:.:/usr/local/bin:/mingw/bin:/bin:/c/Program Files (x86)/Haskell/bin:/c/Program Files (x86)/Haskell Platform/2013.2.0.0/lib/extralibs/bin:/c/Program Files (x86)/Haskell Platform/2013.2.0.0/bin:/c/Program Files (x86)/AMD APP/bin/x86_64:/c/Program Files (x86)/AMD APP/bin/x86:/c/Windows/system32:/c/Windows:/c/Windows/System32/Wbem:/c/Windows/System32/WindowsPowerShell/v1.0/:/c/Program Files (x86)/ATI Technologies/ATI.ACE/Core-Static:/c/Program Files (x86)/Windows Kits/8.0/Windows Performance Toolkit/:/c/Program Files/Microsoft/Web Platform Installer/:/c/Program Files (x86)/Haskell Platform/2013.2.0.0/mingw/bin:/c/Users/Oliver/AppData/Roaming/cabal/bin:/c/cygwin/bin' ++ cabal build +Building git-annex-4.20130601... +Preprocessing executable 'git-annex' for git-annex-4.20130601... +Touch.hsc:117:2: warning: #warning "utimensat and lutimes not available; building without symlink timestamp preservation support" +Touch.hsc: In function 'main': +Touch.hsc:117:2: warning: #warning "utimensat and lutimes not available; building without symlink timestamp preservation support" +Touch.hsc:117:2: warning: #warning "utimensat and lutimes not available; building without symlink timestamp preservation support" +In file included from Mounts.hsc:23:0: +Utility/libmounts.h:17:3: warning: #warning mounts listing code not available for this OS + +Utility\libdiskfree.c:36:3: + warning: #warning free space checking code not available for this OS + +In file included from Utility\libmounts.c:35:0: + +Utility\libmounts.h:17:3: + warning: #warning mounts listing code not available for this OS +[ 1 of 217] Compiling Utility.Dot ( Utility\Dot.hs, dist\build\git-annex\git-annex-tmp\Utility\Dot.o ) +[ 2 of 217] Compiling BuildFlags ( BuildFlags.hs, dist\build\git-annex\git-annex-tmp\BuildFlags.o ) +[ 3 of 217] Compiling Utility.Percentage ( Utility\Percentage.hs, dist\build\git-annex\git-annex-tmp\Utility\Percentage.o ) +[ 4 of 217] Compiling Utility.Base64 ( Utility\Base64.hs, dist\build\git-annex\git-annex-tmp\Utility\Base64.o ) +[ 5 of 217] Compiling Utility.JSONStream ( Utility\JSONStream.hs, dist\build\git-annex\git-annex-tmp\Utility\JSONStream.o ) +[ 6 of 217] Compiling Git.Types ( Git\Types.hs, dist\build\git-annex\git-annex-tmp\Git\Types.o ) +[ 7 of 217] Compiling Utility.DataUnits ( Utility\DataUnits.hs, dist\build\git-annex\git-annex-tmp\Utility\DataUnits.o ) +[ 8 of 217] Compiling Types.FileMatcher ( Types\FileMatcher.hs, dist\build\git-annex\git-annex-tmp\Types\FileMatcher.o ) +[ 9 of 217] Compiling Types.BranchState ( Types\BranchState.hs, dist\build\git-annex\git-annex-tmp\Types\BranchState.o ) +[ 10 of 217] Compiling Messages.JSON ( Messages\JSON.hs, dist\build\git-annex\git-annex-tmp\Messages\JSON.o ) +[ 11 of 217] Compiling Types.UUID ( Types\UUID.hs, dist\build\git-annex\git-annex-tmp\Types\UUID.o ) +[ 12 of 217] Compiling Types.Group ( Types\Group.hs, dist\build\git-annex\git-annex-tmp\Types\Group.o ) +[ 13 of 217] Compiling Utility.Shell ( Utility\Shell.hs, dist\build\git-annex\git-annex-tmp\Utility\Shell.o ) +[ 14 of 217] Compiling Utility.QuickCheck ( Utility\QuickCheck.hs, dist\build\git-annex\git-annex-tmp\Utility\QuickCheck.o ) +[ 15 of 217] Compiling Utility.PartialPrelude ( Utility\PartialPrelude.hs, dist\build\git-annex\git-annex-tmp\Utility\PartialPrelude.o ) +[ 16 of 217] Compiling Utility.HumanTime ( Utility\HumanTime.hs, dist\build\git-annex\git-annex-tmp\Utility\HumanTime.o ) +[ 17 of 217] Compiling Utility.FileSystemEncoding ( Utility\FileSystemEncoding.hs, dist\build\git-annex\git-annex-tmp\Utility\FileSystemEncoding.o ) +[ 18 of 217] Compiling Utility.Touch ( dist\build\git-annex\git-annex-tmp\Utility\Touch.hs, dist\build\git-annex\git-annex-tmp\Utility\Touch.o ) + +UtilityTouch.hsc:17:1: Warning: + The import of `Utility.FileSystemEncoding' is redundant + except perhaps to import instances from `Utility.FileSystemEncoding' + To import instances alone, use: import Utility.FileSystemEncoding() + +UtilityTouch.hsc:19:1: Warning: + The import of `Foreign' is redundant + except perhaps to import instances from `Foreign' + To import instances alone, use: import Foreign() + +UtilityTouch.hsc:21:1: Warning: + The import of `Control.Monad' is redundant + except perhaps to import instances from `Control.Monad' + To import instances alone, use: import Control.Monad() +[ 19 of 217] Compiling Utility.Applicative ( Utility\Applicative.hs, dist\build\git-annex\git-annex-tmp\Utility\Applicative.o ) +[ 20 of 217] Compiling Utility.Monad ( Utility\Monad.hs, dist\build\git-annex\git-annex-tmp\Utility\Monad.o ) +[ 21 of 217] Compiling Utility.Exception ( Utility\Exception.hs, dist\build\git-annex\git-annex-tmp\Utility\Exception.o ) +[ 22 of 217] Compiling Utility.Tmp ( Utility\Tmp.hs, dist\build\git-annex\git-annex-tmp\Utility\Tmp.o ) +[ 23 of 217] Compiling Utility.Env ( Utility\Env.hs, dist\build\git-annex\git-annex-tmp\Utility\Env.o ) +[ 24 of 217] Compiling Utility.UserInfo ( Utility\UserInfo.hs, dist\build\git-annex\git-annex-tmp\Utility\UserInfo.o ) +[ 25 of 217] Compiling Utility.Misc ( Utility\Misc.hs, dist\build\git-annex\git-annex-tmp\Utility\Misc.o ) + +Utility\Misc.hs:22:1: Warning: + The import of `Utility.Exception' is redundant + except perhaps to import instances from `Utility.Exception' + To import instances alone, use: import Utility.Exception() +[ 26 of 217] Compiling Utility.Process ( Utility\Process.hs, dist\build\git-annex\git-annex-tmp\Utility\Process.o ) + +Utility\Process.hs:44:1: Warning: + The import of `Data.Maybe' is redundant + except perhaps to import instances from `Data.Maybe' + To import instances alone, use: import Data.Maybe() +[ 27 of 217] Compiling Utility.Network ( Utility\Network.hs, dist\build\git-annex\git-annex-tmp\Utility\Network.o ) +[ 28 of 217] Compiling Utility.Verifiable ( Utility\Verifiable.hs, dist\build\git-annex\git-annex-tmp\Utility\Verifiable.o ) +[ 29 of 217] Compiling Utility.Format ( Utility\Format.hs, dist\build\git-annex\git-annex-tmp\Utility\Format.o ) +[ 30 of 217] Compiling Build.SysConfig ( Build\SysConfig.hs, dist\build\git-annex\git-annex-tmp\Build\SysConfig.o ) +[ 31 of 217] Compiling Utility.Path ( Utility\Path.hs, dist\build\git-annex\git-annex-tmp\Utility\Path.o ) +[ 32 of 217] Compiling Config.Cost ( Config\Cost.hs, dist\build\git-annex\git-annex-tmp\Config\Cost.o ) +[ 33 of 217] Compiling Types.Messages ( Types\Messages.hs, dist\build\git-annex\git-annex-tmp\Types\Messages.o ) +[ 34 of 217] Compiling Types.TrustLevel ( Types\TrustLevel.hs, dist\build\git-annex\git-annex-tmp\Types\TrustLevel.o ) +[ 35 of 217] Compiling Utility.SafeCommand ( Utility\SafeCommand.hs, dist\build\git-annex\git-annex-tmp\Utility\SafeCommand.o ) +[ 36 of 217] Compiling Utility.Directory ( Utility\Directory.hs, dist\build\git-annex\git-annex-tmp\Utility\Directory.o ) +[ 37 of 217] Compiling Utility.ExternalSHA ( Utility\ExternalSHA.hs, dist\build\git-annex\git-annex-tmp\Utility\ExternalSHA.o ) +[ 38 of 217] Compiling Common ( Common.hs, dist\build\git-annex\git-annex-tmp\Common.o ) +[ 39 of 217] Compiling Git.Filename ( Git\Filename.hs, dist\build\git-annex\git-annex-tmp\Git\Filename.o ) +[ 40 of 217] Compiling Logs.UUIDBased ( Logs\UUIDBased.hs, dist\build\git-annex\git-annex-tmp\Logs\UUIDBased.o ) +[ 41 of 217] Compiling Types.Key ( Types\Key.hs, dist\build\git-annex\git-annex-tmp\Types\Key.o ) +[ 42 of 217] Compiling Utility.FileMode ( Utility\FileMode.hs, dist\build\git-annex\git-annex-tmp\Utility\FileMode.o ) +[ 43 of 217] Compiling Git ( Git.hs, dist\build\git-annex\git-annex-tmp\Git.o ) + +Git.hs:41:1: Warning: + The import of `Utility.FileMode' is redundant + except perhaps to import instances from `Utility.FileMode' + To import instances alone, use: import Utility.FileMode() +[ 44 of 217] Compiling Utility.InodeCache ( Utility\InodeCache.hs, dist\build\git-annex\git-annex-tmp\Utility\InodeCache.o ) +[ 45 of 217] Compiling Types.KeySource ( Types\KeySource.hs, dist\build\git-annex\git-annex-tmp\Types\KeySource.o ) +[ 46 of 217] Compiling Types.Backend ( Types\Backend.hs, dist\build\git-annex\git-annex-tmp\Types\Backend.o ) +[ 47 of 217] Compiling Utility.Gpg ( Utility\Gpg.hs, dist\build\git-annex\git-annex-tmp\Utility\Gpg.o ) + +Utility\Gpg.hs:12:1: Warning: + The import of `System.Posix.Types' is redundant + except perhaps to import instances from `System.Posix.Types' + To import instances alone, use: import System.Posix.Types() + +Utility\Gpg.hs:14:1: Warning: + The import of `Control.Concurrent' is redundant + except perhaps to import instances from `Control.Concurrent' + To import instances alone, use: import Control.Concurrent() + +Utility\Gpg.hs:15:1: Warning: + The import of `Control.Exception' is redundant + except perhaps to import instances from `Control.Exception' + To import instances alone, use: import Control.Exception() + +Utility\Gpg.hs:16:1: Warning: + The import of `System.Path' is redundant + except perhaps to import instances from `System.Path' + To import instances alone, use: import System.Path() + +Utility\Gpg.hs:19:1: Warning: + The import of `Utility.Env' is redundant + except perhaps to import instances from `Utility.Env' + To import instances alone, use: import Utility.Env() +[ 48 of 217] Compiling Types.Crypto ( Types\Crypto.hs, dist\build\git-annex\git-annex-tmp\Types\Crypto.o ) +[ 49 of 217] Compiling Utility.Matcher ( Utility\Matcher.hs, dist\build\git-annex\git-annex-tmp\Utility\Matcher.o ) +[ 50 of 217] Compiling Utility.Metered ( Utility\Metered.hs, dist\build\git-annex\git-annex-tmp\Utility\Metered.o ) +[ 51 of 217] Compiling Git.FilePath ( Git\FilePath.hs, dist\build\git-annex\git-annex-tmp\Git\FilePath.o ) +[ 52 of 217] Compiling Git.Url ( Git\Url.hs, dist\build\git-annex\git-annex-tmp\Git\Url.o ) +[ 53 of 217] Compiling Git.Construct ( Git\Construct.hs, dist\build\git-annex\git-annex-tmp\Git\Construct.o ) +[ 54 of 217] Compiling Git.Config ( Git\Config.hs, dist\build\git-annex\git-annex-tmp\Git\Config.o ) +[ 55 of 217] Compiling Git.CurrentRepo ( Git\CurrentRepo.hs, dist\build\git-annex\git-annex-tmp\Git\CurrentRepo.o ) + +Git\CurrentRepo.hs:16:1: Warning: + The import of `Utility.Env' is redundant + except perhaps to import instances from `Utility.Env' + To import instances alone, use: import Utility.Env() + +Git\CurrentRepo.hs:43:17: Warning: Defined but not used: `s' +[ 56 of 217] Compiling Git.SharedRepository ( Git\SharedRepository.hs, dist\build\git-annex\git-annex-tmp\Git\SharedRepository.o ) +[ 57 of 217] Compiling Types.GitConfig ( Types\GitConfig.hs, dist\build\git-annex\git-annex-tmp\Types\GitConfig.o ) +[ 58 of 217] Compiling Types.Remote ( Types\Remote.hs, dist\build\git-annex\git-annex-tmp\Types\Remote.o ) +[ 59 of 217] Compiling Types.StandardGroups ( Types\StandardGroups.hs, dist\build\git-annex\git-annex-tmp\Types\StandardGroups.o ) +[ 60 of 217] Compiling Utility.Gpg.Types ( Utility\Gpg\Types.hs, dist\build\git-annex\git-annex-tmp\Utility\Gpg\Types.o ) +[ 61 of 217] Compiling Git.Sha ( Git\Sha.hs, dist\build\git-annex\git-annex-tmp\Git\Sha.o ) +[ 62 of 217] Compiling Utility.CoProcess ( Utility\CoProcess.hs, dist\build\git-annex\git-annex-tmp\Utility\CoProcess.o ) +[ 63 of 217] Compiling Git.Command ( Git\Command.hs, dist\build\git-annex\git-annex-tmp\Git\Command.o ) +[ 64 of 217] Compiling Git.LsFiles ( Git\LsFiles.hs, dist\build\git-annex\git-annex-tmp\Git\LsFiles.o ) +[ 65 of 217] Compiling Git.CatFile ( Git\CatFile.hs, dist\build\git-annex\git-annex-tmp\Git\CatFile.o ) +[ 66 of 217] Compiling Git.UpdateIndex ( Git\UpdateIndex.hs, dist\build\git-annex\git-annex-tmp\Git\UpdateIndex.o ) +[ 67 of 217] Compiling Git.Queue ( Git\Queue.hs, dist\build\git-annex\git-annex-tmp\Git\Queue.o ) +[ 68 of 217] Compiling Git.Version ( Git\Version.hs, dist\build\git-annex\git-annex-tmp\Git\Version.o ) +[ 69 of 217] Compiling Git.CheckAttr ( Git\CheckAttr.hs, dist\build\git-annex\git-annex-tmp\Git\CheckAttr.o ) +[ 70 of 217] Compiling Annex ( Annex.hs, dist\build\git-annex\git-annex-tmp\Annex.o ) +[ 71 of 217] Compiling Types.Option ( Types\Option.hs, dist\build\git-annex\git-annex-tmp\Types\Option.o ) +[ 72 of 217] Compiling Types ( Types.hs, dist\build\git-annex\git-annex-tmp\Types.o ) +[ 73 of 217] Compiling Locations ( Locations.hs, dist\build\git-annex\git-annex-tmp\Locations.o ) +[ 74 of 217] Compiling Messages ( Messages.hs, dist\build\git-annex\git-annex-tmp\Messages.o ) +[ 75 of 217] Compiling Common.Annex ( Common\Annex.hs, dist\build\git-annex\git-annex-tmp\Common\Annex.o ) +[ 76 of 217] Compiling Crypto ( Crypto.hs, dist\build\git-annex\git-annex-tmp\Crypto.o ) +[ 77 of 217] Compiling Annex.CatFile ( Annex\CatFile.hs, dist\build\git-annex\git-annex-tmp\Annex\CatFile.o ) +[ 78 of 217] Compiling Backend.SHA ( Backend\SHA.hs, dist\build\git-annex\git-annex-tmp\Backend\SHA.o ) +[ 79 of 217] Compiling Backend.WORM ( Backend\WORM.hs, dist\build\git-annex\git-annex-tmp\Backend\WORM.o ) +[ 80 of 217] Compiling Backend.URL ( Backend\URL.hs, dist\build\git-annex\git-annex-tmp\Backend\URL.o ) +[ 81 of 217] Compiling Annex.Exception ( Annex\Exception.hs, dist\build\git-annex\git-annex-tmp\Annex\Exception.o ) +[ 82 of 217] Compiling Remote.Helper.Special ( Remote\Helper\Special.hs, dist\build\git-annex\git-annex-tmp\Remote\Helper\Special.o ) +[ 83 of 217] Compiling Remote.Helper.Chunked ( Remote\Helper\Chunked.hs, dist\build\git-annex\git-annex-tmp\Remote\Helper\Chunked.o ) +[ 84 of 217] Compiling Annex.Environment ( Annex\Environment.hs, dist\build\git-annex\git-annex-tmp\Annex\Environment.o ) + +Annex\Environment.hs:13:1: Warning: + The import of `Utility.Env' is redundant + except perhaps to import instances from `Utility.Env' + To import instances alone, use: import Utility.Env() + +Annex\Environment.hs:14:1: Warning: + The import of `Utility.UserInfo' is redundant + except perhaps to import instances from `Utility.UserInfo' + To import instances alone, use: import Utility.UserInfo() +[ 85 of 217] Compiling Types.Command ( Types\Command.hs, dist\build\git-annex\git-annex-tmp\Types\Command.o ) +[ 86 of 217] Compiling Usage ( Usage.hs, dist\build\git-annex\git-annex-tmp\Usage.o ) +[ 87 of 217] Compiling Annex.Queue ( Annex\Queue.hs, dist\build\git-annex\git-annex-tmp\Annex\Queue.o ) +[ 88 of 217] Compiling Annex.BranchState ( Annex\BranchState.hs, dist\build\git-annex\git-annex-tmp\Annex\BranchState.o ) +[ 89 of 217] Compiling Remote.Helper.Encryptable ( Remote\Helper\Encryptable.hs, dist\build\git-annex\git-annex-tmp\Remote\Helper\Encryptable.o ) +[ 90 of 217] Compiling Fields ( Fields.hs, dist\build\git-annex\git-annex-tmp\Fields.o ) +[ 91 of 217] Compiling Annex.CheckAttr ( Annex\CheckAttr.hs, dist\build\git-annex\git-annex-tmp\Annex\CheckAttr.o ) +[ 92 of 217] Compiling Git.HashObject ( Git\HashObject.hs, dist\build\git-annex\git-annex-tmp\Git\HashObject.o ) +[ 93 of 217] Compiling Annex.Link ( Annex\Link.hs, dist\build\git-annex\git-annex-tmp\Annex\Link.o ) +[ 94 of 217] Compiling Utility.CopyFile ( Utility\CopyFile.hs, dist\build\git-annex\git-annex-tmp\Utility\CopyFile.o ) +[ 95 of 217] Compiling Git.Ref ( Git\Ref.hs, dist\build\git-annex\git-annex-tmp\Git\Ref.o ) +[ 96 of 217] Compiling Git.Branch ( Git\Branch.hs, dist\build\git-annex\git-annex-tmp\Git\Branch.o ) +[ 97 of 217] Compiling Git.UnionMerge ( Git\UnionMerge.hs, dist\build\git-annex\git-annex-tmp\Git\UnionMerge.o ) +[ 98 of 217] Compiling Git.Merge ( Git\Merge.hs, dist\build\git-annex\git-annex-tmp\Git\Merge.o ) +[ 99 of 217] Compiling Git.DiffTree ( Git\DiffTree.hs, dist\build\git-annex\git-annex-tmp\Git\DiffTree.o ) +[100 of 217] Compiling Utility.DiskFree ( Utility\DiskFree.hs, dist\build\git-annex\git-annex-tmp\Utility\DiskFree.o ) +[101 of 217] Compiling Utility.Url ( Utility\Url.hs, dist\build\git-annex\git-annex-tmp\Utility\Url.o ) +[102 of 217] Compiling Utility.Rsync ( Utility\Rsync.hs, dist\build\git-annex\git-annex-tmp\Utility\Rsync.o ) +[103 of 217] Compiling Utility.LogFile ( Utility\LogFile.hs, dist\build\git-annex\git-annex-tmp\Utility\LogFile.o ) + +Utility\LogFile.hs:67:1: Warning: + Top-level binding with no type signature: + redir :: forall t t1 t2. t -> t1 -> t2 +[104 of 217] Compiling Utility.Daemon ( Utility\Daemon.hs, dist\build\git-annex\git-annex-tmp\Utility\Daemon.o ) + +Utility\Daemon.hs:13:1: Warning: + The import of `Utility.LogFile' is redundant + except perhaps to import instances from `Utility.LogFile' + To import instances alone, use: import Utility.LogFile() + +Utility\Daemon.hs:19:1: Warning: + The import of `System.Posix.Types' is redundant + except perhaps to import instances from `System.Posix.Types' + To import instances alone, use: import System.Posix.Types() +[105 of 217] Compiling Git.AutoCorrect ( Git\AutoCorrect.hs, dist\build\git-annex\git-annex-tmp\Git\AutoCorrect.o ) +[106 of 217] Compiling Utility.ThreadScheduler ( Utility\ThreadScheduler.hs, dist\build\git-annex\git-annex-tmp\Utility\ThreadScheduler.o ) +[107 of 217] Compiling Git.LsTree ( Git\LsTree.hs, dist\build\git-annex\git-annex-tmp\Git\LsTree.o ) +[108 of 217] Compiling Config ( Config.hs, dist\build\git-annex\git-annex-tmp\Config.o ) +[109 of 217] Compiling Annex.UUID ( Annex\UUID.hs, dist\build\git-annex\git-annex-tmp\Annex\UUID.o ) +[110 of 217] Compiling Backend ( Backend.hs, dist\build\git-annex\git-annex-tmp\Backend.o ) +[111 of 217] Compiling Annex.Version ( Annex\Version.hs, dist\build\git-annex\git-annex-tmp\Annex\Version.o ) +[112 of 217] Compiling Annex.Perms ( Annex\Perms.hs, dist\build\git-annex\git-annex-tmp\Annex\Perms.o ) +[113 of 217] Compiling Logs.Transfer ( Logs\Transfer.hs, dist\build\git-annex\git-annex-tmp\Logs\Transfer.o ) + +Logs\Transfer.hs:126:20: Warning: Defined but not used: `mode' + +Logs\Transfer.hs:146:29: Warning: Defined but not used: `fd' +[114 of 217] Compiling Annex.ReplaceFile ( Annex\ReplaceFile.hs, dist\build\git-annex\git-annex-tmp\Annex\ReplaceFile.o ) +[115 of 217] Compiling Annex.Journal ( Annex\Journal.hs, dist\build\git-annex\git-annex-tmp\Annex\Journal.o ) + +Annex\Journal.hs:89:23: Warning: Defined but not used: `mode' +[116 of 217] Compiling Annex.Branch ( Annex\Branch.hs, dist\build\git-annex\git-annex-tmp\Annex\Branch.o ) +[117 of 217] Compiling Logs.Remote ( Logs\Remote.hs, dist\build\git-annex\git-annex-tmp\Logs\Remote.o ) +[118 of 217] Compiling Logs.Presence ( Logs\Presence.hs, dist\build\git-annex\git-annex-tmp\Logs\Presence.o ) +[119 of 217] Compiling Logs.UUID ( Logs\UUID.hs, dist\build\git-annex\git-annex-tmp\Logs\UUID.o ) +[120 of 217] Compiling Logs.Location ( Logs\Location.hs, dist\build\git-annex\git-annex-tmp\Logs\Location.o ) +[121 of 217] Compiling Annex.Content.Direct ( Annex\Content\Direct.hs, dist\build\git-annex\git-annex-tmp\Annex\Content\Direct.o ) +[122 of 217] Compiling Annex.Content ( Annex\Content.hs, dist\build\git-annex\git-annex-tmp\Annex\Content.o ) + +Annex\Content.hs:50:1: Warning: + The import of `Annex.Exception' is redundant + except perhaps to import instances from `Annex.Exception' + To import instances alone, use: import Annex.Exception() + +Annex\Content.hs:89:21: Warning: Defined but not used: `f' + +Annex\Content.hs:96:21: Warning: Defined but not used: `h' + +Annex\Content.hs:106:9: Warning: Defined but not used: `is_locked' + +Annex\Content.hs:113:13: Warning: Defined but not used: `key' +[123 of 217] Compiling Annex.Direct ( Annex\Direct.hs, dist\build\git-annex\git-annex-tmp\Annex\Direct.o ) +[124 of 217] Compiling Init ( Init.hs, dist\build\git-annex\git-annex-tmp\Init.o ) + +Init.hs:29:1: Warning: + The import of `Utility.UserInfo' is redundant + except perhaps to import instances from `Utility.UserInfo' + To import instances alone, use: import Utility.UserInfo() + +Init.hs:31:1: Warning: + The import of `Utility.FileMode' is redundant + except perhaps to import instances from `Utility.FileMode' + To import instances alone, use: import Utility.FileMode() +[125 of 217] Compiling Logs.Web ( Logs\Web.hs, dist\build\git-annex\git-annex-tmp\Logs\Web.o ) +[126 of 217] Compiling Logs.Group ( Logs\Group.hs, dist\build\git-annex\git-annex-tmp\Logs\Group.o ) +[127 of 217] Compiling Upgrade.V2 ( Upgrade\V2.hs, dist\build\git-annex\git-annex-tmp\Upgrade\V2.o ) +[128 of 217] Compiling Upgrade ( Upgrade.hs, dist\build\git-annex\git-annex-tmp\Upgrade.o ) +[129 of 217] Compiling Creds ( Creds.hs, dist\build\git-annex\git-annex-tmp\Creds.o ) + +Creds.hs:18:1: Warning: + The import of `Utility.Env' is redundant + except perhaps to import instances from `Utility.Env' + To import instances alone, use: import Utility.Env() +[130 of 217] Compiling Remote.Helper.AWS ( Remote\Helper\AWS.hs, dist\build\git-annex\git-annex-tmp\Remote\Helper\AWS.o ) +[131 of 217] Compiling Annex.LockPool ( Annex\LockPool.hs, dist\build\git-annex\git-annex-tmp\Annex\LockPool.o ) + +Annex\LockPool.hs:17:1: Warning: + The import of `Annex.Perms' is redundant + except perhaps to import instances from `Annex.Perms' + To import instances alone, use: import Annex.Perms() + +Annex\LockPool.hs:39:12: Warning: Defined but not used: `fd' +[132 of 217] Compiling Remote.Helper.Hooks ( Remote\Helper\Hooks.hs, dist\build\git-annex\git-annex-tmp\Remote\Helper\Hooks.o ) + +Remote\Helper\Hooks.hs:18:1: Warning: + The import of `Annex.Perms' is redundant + except perhaps to import instances from `Annex.Perms' + To import instances alone, use: import Annex.Perms() + +Remote\Helper\Hooks.hs:74:17: Warning: Defined but not used: `lck' +[133 of 217] Compiling Remote.S3 ( Remote\S3.hs, dist\build\git-annex\git-annex-tmp\Remote\S3.o ) +[134 of 217] Compiling Remote.Directory ( Remote\Directory.hs, dist\build\git-annex\git-annex-tmp\Remote\Directory.o ) +[135 of 217] Compiling Remote.Web ( Remote\Web.hs, dist\build\git-annex\git-annex-tmp\Remote\Web.o ) +[136 of 217] Compiling Remote.WebDAV ( Remote\WebDAV.hs, dist\build\git-annex\git-annex-tmp\Remote\WebDAV.o ) +[137 of 217] Compiling Remote.Glacier ( Remote\Glacier.hs, dist\build\git-annex\git-annex-tmp\Remote\Glacier.o ) +[138 of 217] Compiling Remote.Hook ( Remote\Hook.hs, dist\build\git-annex\git-annex-tmp\Remote\Hook.o ) +[139 of 217] Compiling Annex.Ssh ( Annex\Ssh.hs, dist\build\git-annex\git-annex-tmp\Annex\Ssh.o ) + +Annex\Ssh.hs:21:1: Warning: + The import of `Annex.Perms' is redundant + except perhaps to import instances from `Annex.Perms' + To import instances alone, use: import Annex.Perms() +[140 of 217] Compiling Remote.Rsync ( Remote\Rsync.hs, dist\build\git-annex\git-annex-tmp\Remote\Rsync.o ) +[141 of 217] Compiling Remote.Helper.Ssh ( Remote\Helper\Ssh.hs, dist\build\git-annex\git-annex-tmp\Remote\Helper\Ssh.o ) +[142 of 217] Compiling Remote.Git ( Remote\Git.hs, dist\build\git-annex\git-annex-tmp\Remote\Git.o ) + +Remote\Git.hs:20:1: Warning: + The import of `Utility.CopyFile' is redundant + except perhaps to import instances from `Utility.CopyFile' + To import instances alone, use: import Utility.CopyFile() + +Remote\Git.hs:360:21: Warning: Defined but not used: `r' + +Remote\Git.hs:360:23: Warning: Defined but not used: `key' + +Remote\Git.hs:360:27: Warning: Defined but not used: `file' +[143 of 217] Compiling Remote.Bup ( Remote\Bup.hs, dist\build\git-annex\git-annex-tmp\Remote\Bup.o ) +[144 of 217] Compiling Remote.List ( Remote\List.hs, dist\build\git-annex\git-annex-tmp\Remote\List.o ) +[145 of 217] Compiling Logs.Trust ( Logs\Trust.hs, dist\build\git-annex\git-annex-tmp\Logs\Trust.o ) +[146 of 217] Compiling Remote ( Remote.hs, dist\build\git-annex\git-annex-tmp\Remote.o ) +[147 of 217] Compiling Limit ( Limit.hs, dist\build\git-annex\git-annex-tmp\Limit.o ) +[148 of 217] Compiling Option ( Option.hs, dist\build\git-annex\git-annex-tmp\Option.o ) +[149 of 217] Compiling Annex.FileMatcher ( Annex\FileMatcher.hs, dist\build\git-annex\git-annex-tmp\Annex\FileMatcher.o ) +[150 of 217] Compiling Logs.PreferredContent ( Logs\PreferredContent.hs, dist\build\git-annex\git-annex-tmp\Logs\PreferredContent.o ) +[151 of 217] Compiling Annex.Wanted ( Annex\Wanted.hs, dist\build\git-annex\git-annex-tmp\Annex\Wanted.o ) +[152 of 217] Compiling Seek ( Seek.hs, dist\build\git-annex\git-annex-tmp\Seek.o ) +[153 of 217] Compiling Checks ( Checks.hs, dist\build\git-annex\git-annex-tmp\Checks.o ) +[154 of 217] Compiling Command ( Command.hs, dist\build\git-annex\git-annex-tmp\Command.o ) +[155 of 217] Compiling Logs.Unused ( Logs\Unused.hs, dist\build\git-annex\git-annex-tmp\Logs\Unused.o ) +[156 of 217] Compiling CmdLine ( CmdLine.hs, dist\build\git-annex\git-annex-tmp\CmdLine.o ) +[157 of 217] Compiling Command.ConfigList ( Command\ConfigList.hs, dist\build\git-annex\git-annex-tmp\Command\ConfigList.o ) +[158 of 217] Compiling Command.InAnnex ( Command\InAnnex.hs, dist\build\git-annex\git-annex-tmp\Command\InAnnex.o ) +[159 of 217] Compiling Command.DropKey ( Command\DropKey.hs, dist\build\git-annex\git-annex-tmp\Command\DropKey.o ) +[160 of 217] Compiling Command.SendKey ( Command\SendKey.hs, dist\build\git-annex\git-annex-tmp\Command\SendKey.o ) +[161 of 217] Compiling Command.RecvKey ( Command\RecvKey.hs, dist\build\git-annex\git-annex-tmp\Command\RecvKey.o ) +[162 of 217] Compiling Command.TransferInfo ( Command\TransferInfo.hs, dist\build\git-annex\git-annex-tmp\Command\TransferInfo.o ) +[163 of 217] Compiling Command.Commit ( Command\Commit.hs, dist\build\git-annex\git-annex-tmp\Command\Commit.o ) +[164 of 217] Compiling GitAnnex.Options ( GitAnnex\Options.hs, dist\build\git-annex\git-annex-tmp\GitAnnex\Options.o ) +[165 of 217] Compiling Command.Unannex ( Command\Unannex.hs, dist\build\git-annex\git-annex-tmp\Command\Unannex.o ) +[166 of 217] Compiling Command.FromKey ( Command\FromKey.hs, dist\build\git-annex\git-annex-tmp\Command\FromKey.o ) +[167 of 217] Compiling Command.Fix ( Command\Fix.hs, dist\build\git-annex\git-annex-tmp\Command\Fix.o ) +[168 of 217] Compiling Command.Init ( Command\Init.hs, dist\build\git-annex\git-annex-tmp\Command\Init.o ) +[169 of 217] Compiling Command.Describe ( Command\Describe.hs, dist\build\git-annex\git-annex-tmp\Command\Describe.o ) +[170 of 217] Compiling Command.InitRemote ( Command\InitRemote.hs, dist\build\git-annex\git-annex-tmp\Command\InitRemote.o ) +[171 of 217] Compiling Command.EnableRemote ( Command\EnableRemote.hs, dist\build\git-annex\git-annex-tmp\Command\EnableRemote.o ) +[172 of 217] Compiling Command.Unused ( Command\Unused.hs, dist\build\git-annex\git-annex-tmp\Command\Unused.o ) +[173 of 217] Compiling Command.Unlock ( Command\Unlock.hs, dist\build\git-annex\git-annex-tmp\Command\Unlock.o ) +[174 of 217] Compiling Command.Lock ( Command\Lock.hs, dist\build\git-annex\git-annex-tmp\Command\Lock.o ) +[175 of 217] Compiling Command.Find ( Command\Find.hs, dist\build\git-annex\git-annex-tmp\Command\Find.o ) +[176 of 217] Compiling Command.Whereis ( Command\Whereis.hs, dist\build\git-annex\git-annex-tmp\Command\Whereis.o ) +[177 of 217] Compiling Command.Log ( Command\Log.hs, dist\build\git-annex\git-annex-tmp\Command\Log.o ) +[178 of 217] Compiling Command.Merge ( Command\Merge.hs, dist\build\git-annex\git-annex-tmp\Command\Merge.o ) +[179 of 217] Compiling Command.Uninit ( Command\Uninit.hs, dist\build\git-annex\git-annex-tmp\Command\Uninit.o ) +[180 of 217] Compiling Command.Trust ( Command\Trust.hs, dist\build\git-annex\git-annex-tmp\Command\Trust.o ) +[181 of 217] Compiling Command.Untrust ( Command\Untrust.hs, dist\build\git-annex\git-annex-tmp\Command\Untrust.o ) +[182 of 217] Compiling Command.Semitrust ( Command\Semitrust.hs, dist\build\git-annex\git-annex-tmp\Command\Semitrust.o ) +[183 of 217] Compiling Command.Dead ( Command\Dead.hs, dist\build\git-annex\git-annex-tmp\Command\Dead.o ) +[184 of 217] Compiling Command.Group ( Command\Group.hs, dist\build\git-annex\git-annex-tmp\Command\Group.o ) +[185 of 217] Compiling Command.Content ( Command\Content.hs, dist\build\git-annex\git-annex-tmp\Command\Content.o ) +[186 of 217] Compiling Command.Ungroup ( Command\Ungroup.hs, dist\build\git-annex\git-annex-tmp\Command\Ungroup.o ) +[187 of 217] Compiling Command.Vicfg ( Command\Vicfg.hs, dist\build\git-annex\git-annex-tmp\Command\Vicfg.o ) +[188 of 217] Compiling Command.RmUrl ( Command\RmUrl.hs, dist\build\git-annex\git-annex-tmp\Command\RmUrl.o ) +[189 of 217] Compiling Command.Map ( Command\Map.hs, dist\build\git-annex\git-annex-tmp\Command\Map.o ) +[190 of 217] Compiling Command.Upgrade ( Command\Upgrade.hs, dist\build\git-annex\git-annex-tmp\Command\Upgrade.o ) +[191 of 217] Compiling Command.Version ( Command\Version.hs, dist\build\git-annex\git-annex-tmp\Command\Version.o ) +[192 of 217] Compiling Command.Test ( Command\Test.hs, dist\build\git-annex\git-annex-tmp\Command\Test.o ) +[193 of 217] Compiling Command.Add ( Command\Add.hs, dist\build\git-annex\git-annex-tmp\Command\Add.o ) +[194 of 217] Compiling Command.ReKey ( Command\ReKey.hs, dist\build\git-annex\git-annex-tmp\Command\ReKey.o ) +[195 of 217] Compiling Command.AddUnused ( Command\AddUnused.hs, dist\build\git-annex\git-annex-tmp\Command\AddUnused.o ) +[196 of 217] Compiling Command.PreCommit ( Command\PreCommit.hs, dist\build\git-annex\git-annex-tmp\Command\PreCommit.o ) +[197 of 217] Compiling Command.Import ( Command\Import.hs, dist\build\git-annex\git-annex-tmp\Command\Import.o ) +[198 of 217] Compiling Command.Drop ( Command\Drop.hs, dist\build\git-annex\git-annex-tmp\Command\Drop.o ) +[199 of 217] Compiling Command.Move ( Command\Move.hs, dist\build\git-annex\git-annex-tmp\Command\Move.o ) +[200 of 217] Compiling Command.Copy ( Command\Copy.hs, dist\build\git-annex\git-annex-tmp\Command\Copy.o ) +[201 of 217] Compiling Command.Get ( Command\Get.hs, dist\build\git-annex\git-annex-tmp\Command\Get.o ) +[202 of 217] Compiling Command.TransferKey ( Command\TransferKey.hs, dist\build\git-annex\git-annex-tmp\Command\TransferKey.o ) +[203 of 217] Compiling Command.DropUnused ( Command\DropUnused.hs, dist\build\git-annex\git-annex-tmp\Command\DropUnused.o ) +[204 of 217] Compiling Command.Fsck ( Command\Fsck.hs, dist\build\git-annex\git-annex-tmp\Command\Fsck.o ) +[205 of 217] Compiling Command.Reinject ( Command\Reinject.hs, dist\build\git-annex\git-annex-tmp\Command\Reinject.o ) +[206 of 217] Compiling Command.Migrate ( Command\Migrate.hs, dist\build\git-annex\git-annex-tmp\Command\Migrate.o ) +[207 of 217] Compiling Command.Status ( Command\Status.hs, dist\build\git-annex\git-annex-tmp\Command\Status.o ) +[208 of 217] Compiling Command.Sync ( Command\Sync.hs, dist\build\git-annex\git-annex-tmp\Command\Sync.o ) +[209 of 217] Compiling Command.Help ( Command\Help.hs, dist\build\git-annex\git-annex-tmp\Command\Help.o ) +[210 of 217] Compiling Command.AddUrl ( Command\AddUrl.hs, dist\build\git-annex\git-annex-tmp\Command\AddUrl.o ) +[211 of 217] Compiling Command.Direct ( Command\Direct.hs, dist\build\git-annex\git-annex-tmp\Command\Direct.o ) +[212 of 217] Compiling Command.Indirect ( Command\Indirect.hs, dist\build\git-annex\git-annex-tmp\Command\Indirect.o ) +[213 of 217] Compiling Command.FuzzTest ( Command\FuzzTest.hs, dist\build\git-annex\git-annex-tmp\Command\FuzzTest.o ) +[214 of 217] Compiling Test ( Test.hs, dist\build\git-annex\git-annex-tmp\Test.o ) +[215 of 217] Compiling GitAnnexShell ( GitAnnexShell.hs, dist\build\git-annex\git-annex-tmp\GitAnnexShell.o ) +[216 of 217] Compiling GitAnnex ( GitAnnex.hs, dist\build\git-annex\git-annex-tmp\GitAnnex.o ) +[217 of 217] Compiling Main ( git-annex.hs, dist\build\git-annex\git-annex-tmp\Main.o ) +Linking dist\build\git-annex\git-annex.exe ... ++ cabal install nsis +Resolving dependencies... +All the requested packages are already installed: +nsis-0.2.2 +Use --reinstall if you want to reinstall anyway. ++ ghc --make Build/NullSoftInstaller.hs +[15 of 18] Compiling Build.SysConfig ( Build\SysConfig.hs, Build\SysConfig.o ) +Linking Build\NullSoftInstaller.exe ... ++ withcyg Build/NullSoftInstaller.exe ++ PATH='/c/Program Files (x86)/Haskell Platform/2012.4.0.0/bin:/c/Program Files (x86)/Haskell Platform/2012.4.0.0/lib/extralibs/bin:/c/Program Files (x86)/NSIS:/home/Oliver/bin:.:/usr/local/bin:/mingw/bin:/bin:/c/Program Files (x86)/Haskell/bin:/c/Program Files (x86)/Haskell Platform/2013.2.0.0/lib/extralibs/bin:/c/Program Files (x86)/Haskell Platform/2013.2.0.0/bin:/c/Program Files (x86)/AMD APP/bin/x86_64:/c/Program Files (x86)/AMD APP/bin/x86:/c/Windows/system32:/c/Windows:/c/Windows/System32/Wbem:/c/Windows/System32/WindowsPowerShell/v1.0/:/c/Program Files (x86)/ATI Technologies/ATI.ACE/Core-Static:/c/Program Files (x86)/Windows Kits/8.0/Windows Performance Toolkit/:/c/Program Files/Microsoft/Web Platform Installer/:/c/Program Files (x86)/Haskell Platform/2013.2.0.0/mingw/bin:/c/Users/Oliver/AppData/Roaming/cabal/bin:/c/cygwin/bin' ++ Build/NullSoftInstaller.exe +MakeNSIS v2.46 - Copyright 1995-2009 Contributors +See the file COPYING for license details. +Credits can be found in the Users Manual. + +Processing config: +Processing plugin dlls: "c:\Program Files (x86)\NSIS\Plugins\*.dll" + - AdvSplash::show + - Banner::destroy + - Banner::getWindow + - Banner::show + - BgImage::AddImage + - BgImage::AddText + - BgImage::Clear + - BgImage::Destroy + - BgImage::Redraw + - BgImage::SetBg + - BgImage::SetReturn + - BgImage::Sound + - Dialer::AttemptConnect + - Dialer::AutodialHangup + - Dialer::AutodialOnline + - Dialer::AutodialUnattended + - Dialer::GetConnectedState + - InstallOptions::dialog + - InstallOptions::initDialog + - InstallOptions::show + - LangDLL::LangDialog + - Math::Script + - NSISdl::download + - NSISdl::download_quiet + - Splash::show + - StartMenu::Init + - StartMenu::Select + - StartMenu::Show + - System::Alloc + - System::Call + - System::Copy + - System::Free + - System::Get + - System::Int64Op + - System::Store + - TypeLib::GetLibVersion + - TypeLib::Register + - TypeLib::UnRegister + - UserInfo::GetAccountType + - UserInfo::GetName + - UserInfo::GetOriginalAccountType + - VPatch::GetFileCRC32 + - VPatch::GetFileMD5 + - VPatch::vpatchfile + - nsDialogs::Create + - nsDialogs::CreateControl + - nsDialogs::CreateItem + - nsDialogs::CreateTimer + - nsDialogs::GetUserData + - nsDialogs::KillTimer + - nsDialogs::OnBack + - nsDialogs::OnChange + - nsDialogs::OnClick + - nsDialogs::OnNotify + - nsDialogs::SelectFileDialog + - nsDialogs::SelectFolderDialog + - nsDialogs::SetRTL + - nsDialogs::SetUserData + - nsDialogs::Show + - nsExec::Exec + - nsExec::ExecToLog + - nsExec::ExecToStack + +!define: "MUI_INSERT_NSISCONF"="" + +Changing directory to: "C:\MinGW\msys\1.0\home\Oliver\src\git-annex" + +Processing script file: "git-annex.nsi" +!include: "c:\Program Files (x86)\NSIS\Include\MUI2.nsh" +!include: "c:\Program Files (x86)\NSIS\Contrib\Modern UI 2\MUI2.nsh" +NSIS Modern User Interface version 2.0 - Copyright 2002-2009 Joost Verburg (c:\Program Files (x86)\NSIS\Contrib\Modern UI 2\MUI2.nsh:8) +!define: "MUI_INCLUDED"="" +!define: "MUI_SYSVERSION"="2.0" +!define: "MUI_VERBOSE"="3" +!include: closed: "c:\Program Files (x86)\NSIS\Contrib\Modern UI 2\MUI2.nsh" +!include: closed: "c:\Program Files (x86)\NSIS\Include\MUI2.nsh" +Name: "git-annex" +OutFile: "git-annex-installer.exe" +InstallDir: "$PROGRAMFILES\Git\cmd" +!insertmacro: MUI_PAGE_DIRECTORY +!insertmacro: end of MUI_PAGE_DIRECTORY +!insertmacro: MUI_PAGE_LICENSE +!insertmacro: end of MUI_PAGEDECLARATION_LICENSE +!insertmacro: end of MUI_PAGE_LICENSE +!insertmacro: MUI_PAGE_INSTFILES +!insertmacro: end of MUI_PAGE_INSTFILES +!insertmacro: MUI_LANGUAGE +!insertmacro: end of MUI_LANGUAGE +Section: "main" ->(_sec10) +SetOutPath: "$INSTDIR" +File: "git-annex.exe" [compress] 7664822/31704766 bytes +File: "git-annex-licenses.txt" [compress] 60509/237415 bytes +File: "cp.exe" [compress] 62591/116736 bytes +File: "xargs.exe" [compress] 17002/33280 bytes +File: "rsync.exe" [compress] 188346/359424 bytes +File: "ssh.exe" [compress] 157190/320000 bytes +File: "wget.exe" [compress] 155666/349184 bytes +File: "sha1sum.exe" [compress] 20337/39424 bytes +File: "sha256sum.exe" [compress] 18279/37390 bytes +File: "sha512sum.exe" [compress] 30393/92174 bytes +File: "sha224sum.exe" [compress] 18279/37390 bytes +File: "sha384sum.exe" [compress] 30392/92174 bytes +File: "cygwin1.dll" [compress] 1036075/2874639 bytes +File: "cygasn1-8.dll" [compress] 154881/459293 bytes +File: "cygattr-1.dll" [compress] 5460/13838 bytes +File: "cygheimbase-1.dll" [compress] 4441/10781 bytes +File: "cygroken-18.dll" [compress] 26103/52253 bytes +File: "cygcom_err-2.dll" [compress] 3984/9757 bytes +File: "cygheimntlm-0.dll" [compress] 9018/20509 bytes +File: "cygsqlite3-0.dll" [compress] 334366/601629 bytes +File: "cygcrypt-0.dll" [compress] 3352/7182 bytes +File: "cyghx509-5.dll" [compress] 95839/216093 bytes +File: "cygssp-0.dll" [compress] 3377/8206 bytes +File: "cygcrypto-1.0.0.dll" [compress] 652766/1553920 bytes +File: "cygiconv-2.dll" [compress] 738899/1008654 bytes +File: "cyggcc_s-1.dll" [compress] 40241/80910 bytes +File: "cygintl-8.dll" [compress] 17737/35342 bytes +File: "cygwind-0.dll" [compress] 73172/160797 bytes +File: "cyggssapi-3.dll" [compress] 81662/183837 bytes +File: "cygkrb5-26.dll" [compress] 170204/381469 bytes +File: "cygz.dll" [compress] 42634/74269 bytes +WriteUninstaller: "git-annex-uninstall.exe" +SectionEnd +Section: "Uninstall" ->(_sec11) +Delete: /REBOOTOK "$INSTDIR\git-annex.exe" +Delete: /REBOOTOK "$INSTDIR\git-annex-licenses.txt" +Delete: /REBOOTOK "$INSTDIR\git-annex-uninstall.exe" +Delete: /REBOOTOK "$INSTDIR\cp.exe" +Delete: /REBOOTOK "$INSTDIR\xargs.exe" +Delete: /REBOOTOK "$INSTDIR\rsync.exe" +Delete: /REBOOTOK "$INSTDIR\ssh.exe" +Delete: /REBOOTOK "$INSTDIR\wget.exe" +Delete: /REBOOTOK "$INSTDIR\sha1sum.exe" +Delete: /REBOOTOK "$INSTDIR\sha256sum.exe" +Delete: /REBOOTOK "$INSTDIR\sha512sum.exe" +Delete: /REBOOTOK "$INSTDIR\sha224sum.exe" +Delete: /REBOOTOK "$INSTDIR\sha384sum.exe" +Delete: /REBOOTOK "$INSTDIR\cygwin1.dll" +Delete: /REBOOTOK "$INSTDIR\cygasn1-8.dll" +Delete: /REBOOTOK "$INSTDIR\cygattr-1.dll" +Delete: /REBOOTOK "$INSTDIR\cygheimbase-1.dll" +Delete: /REBOOTOK "$INSTDIR\cygroken-18.dll" +Delete: /REBOOTOK "$INSTDIR\cygcom_err-2.dll" +Delete: /REBOOTOK "$INSTDIR\cygheimntlm-0.dll" +Delete: /REBOOTOK "$INSTDIR\cygsqlite3-0.dll" +Delete: /REBOOTOK "$INSTDIR\cygcrypt-0.dll" +Delete: /REBOOTOK "$INSTDIR\cyghx509-5.dll" +Delete: /REBOOTOK "$INSTDIR\cygssp-0.dll" +Delete: /REBOOTOK "$INSTDIR\cygcrypto-1.0.0.dll" +Delete: /REBOOTOK "$INSTDIR\cygiconv-2.dll" +Delete: /REBOOTOK "$INSTDIR\cyggcc_s-1.dll" +Delete: /REBOOTOK "$INSTDIR\cygintl-8.dll" +Delete: /REBOOTOK "$INSTDIR\cygwind-0.dll" +Delete: /REBOOTOK "$INSTDIR\cyggssapi-3.dll" +Delete: /REBOOTOK "$INSTDIR\cygkrb5-26.dll" +Delete: /REBOOTOK "$INSTDIR\cygz.dll" +SectionEnd +Function: ".onInit" +IfFileExists: "$PROGRAMFILES\Git\cmd" ? _lbl3 : 0 +MessageBox: 48: "You need git installed to use git-annex. Looking at $PROGRAMFILES\Git\cmd , it seems to not be installed, or may be installed in another location. You can install git from http://git-scm.com/" (on IDOK goto 0) +FunctionEnd + +Processed 1 file, writing output: +Processing pages... Done! +Removing unused resources... Done! +Generating language tables... Done! +Generating uninstaller... Done! + +Output: "C:\MinGW\msys\1.0\home\Oliver\src\git-annex\git-annex-installer.exe" +Install: 4 pages (256 bytes), 1 section (1048 bytes), 98 instructions (2744 bytes), 133 strings (239643 bytes), 1 language table (278 bytes). +Uninstall: 2 pages (128 bytes), +1 section (1048 bytes), 33 instructions (924 bytes), 72 strings (1154 bytes), 1 language table (194 bytes). + +Using zlib compression. + +EXE header size: 48640 / 35840 bytes +Install code: 63270 / 244345 bytes +Install data: 11918141 / 41172867 bytes +Uninstall code+data: 10537 / 14897 bytes +CRC (0x090A4FD3): 4 / 4 bytes + +Total size: 12040592 / 41467953 bytes (29.0%) ++ rm -f last-incremental-failed ++ rm -rf .t ++ withcyg dist/build/git-annex/git-annex.exe test ++ PATH='/c/Program Files (x86)/Haskell Platform/2012.4.0.0/bin:/c/Program Files (x86)/Haskell Platform/2012.4.0.0/lib/extralibs/bin:/c/Program Files (x86)/NSIS:/home/Oliver/bin:.:/usr/local/bin:/mingw/bin:/bin:/c/Program Files (x86)/Haskell/bin:/c/Program Files (x86)/Haskell Platform/2013.2.0.0/lib/extralibs/bin:/c/Program Files (x86)/Haskell Platform/2013.2.0.0/bin:/c/Program Files (x86)/AMD APP/bin/x86_64:/c/Program Files (x86)/AMD APP/bin/x86:/c/Windows/system32:/c/Windows:/c/Windows/System32/Wbem:/c/Windows/System32/WindowsPowerShell/v1.0/:/c/Program Files (x86)/ATI Technologies/ATI.ACE/Core-Static:/c/Program Files (x86)/Windows Kits/8.0/Windows Performance Toolkit/:/c/Program Files/Microsoft/Web Platform Installer/:/c/Program Files (x86)/Haskell Platform/2013.2.0.0/mingw/bin:/c/Users/Oliver/AppData/Roaming/cabal/bin:/c/cygwin/bin' ++ dist/build/git-annex/git-annex.exe test +(0 tests) (1 test) (2 tests) (3 tests) (4 tests) (5 tests) (6 tests) (7 tests) (8 tests) (9 tests) (10 tests) (11 tests) (12 tests) (13 tests) (14 tests) (15 tests) (16 tests) (17 tests) (18 tests) (19 tests) (20 tests) (21 tests) (22 tests) (23 tests) (24 tests) (25 tests) (26 tests) (27 tests) (28 tests) (29 tests) (30 tests) (31 tests) (32 tests) (33 tests) (34 tests) (35 tests) (36 tests) (37 tests) (38 tests) (39 tests) (40 tests) (41 tests) (42 tests) (43 tests) (44 tests) (45 tests) (46 tests) (47 tests) (48 tests) (49 tests) (50 tests) (51 tests) (52 tests) (53 tests) (54 tests) (55 tests) (56 tests) (57 tests) (58 tests) (59 tests) (60 tests) (61 tests) (62 tests) (63 tests) (64 tests) (65 tests) (66 tests) (67 tests) (68 tests) (69 tests) (70 tests) (71 tests) (72 tests) (73 tests) (74 tests) (75 tests) (76 tests) (77 tests) (78 tests) (79 tests) (80 tests) (81 tests) (82 tests) (83 tests) (84 tests) (85 tests) (86 tests) (87 tests) (88 tests) (89 tests) (90 tests) (91 tests) (92 tests) (93 tests) (94 tests) (95 tests) (96 tests) (97 tests) (98 tests) (99 tests) ---------------------------------------------------------------------- +First, some automated quick checks of properties ... +---------------------------------------------------------------------- +prop_idempotent_deencode_git ++++ OK, passed 100 tests. +(0 tests) (1 test) (2 tests) (3 tests) (4 tests) (5 tests) (6 tests) (7 tests) (8 tests) (9 tests) (10 tests) (11 tests) (12 tests) (13 tests) (14 tests) (15 tests) (16 tests) (17 tests) (18 tests) (19 tests) (20 tests) (21 tests) (22 tests) (23 tests) (24 tests) (25 tests) (26 tests) (27 tests) (28 tests) (29 tests) (30 tests) (31 tests) (32 tests) (33 tests) (34 tests) (35 tests) (36 tests) (37 tests) (38 tests) (39 tests) (40 tests) (41 tests) (42 tests) (43 tests) (44 tests) (45 tests) (46 tests) (47 tests) (48 tests) (49 tests) (50 tests) (51 tests) (52 tests) (53 tests) (54 tests) (55 tests) (56 tests) (57 tests) (58 tests) (59 tests) (60 tests) (61 tests) (62 tests) (63 tests) (64 tests) (65 tests) (66 tests) (67 tests) (68 tests) (69 tests) (70 tests) (71 tests) (72 tests) (73 tests) (74 tests) (75 tests) (76 tests) (77 tests) (78 tests) (79 tests) (80 tests) (81 tests) (82 tests) (83 tests) (84 tests) (85 tests) (86 tests) (87 tests) (88 tests) (89 tests) (90 tests) (91 tests) (92 tests) (93 tests) (94 tests) (95 tests) (96 tests) (97 tests) (98 tests) (99 tests) prop_idempotent_deencode ++++ OK, passed 100 tests. +(0 tests) (1 test) (2 tests) (3 tests) (4 tests) (5 tests) (6 tests) (7 tests) (8 tests) (9 tests) (10 tests) (11 tests) (12 tests) (13 tests) (14 tests) (15 tests) (16 tests) (17 tests) (18 tests) (19 tests) (20 tests) (21 tests) (22 tests) (23 tests) (24 tests) (25 tests) (26 tests) (27 tests) (28 tests) (29 tests) (30 tests) (31 tests) (32 tests) (33 tests) (34 tests) (35 tests) (36 tests) (37 tests) (38 tests) (39 tests) (40 tests) (41 tests) (42 tests) (43 tests) (44 tests) (45 tests) (46 tests) (47 tests) (48 tests) (49 tests) (50 tests) (51 tests) (52 tests) (53 tests) (54 tests) (55 tests) (56 tests) (57 tests) (58 tests) (59 tests) (60 tests) (61 tests) (62 tests) (63 tests) (64 tests) (65 tests) (66 tests) (67 tests) (68 tests) (69 tests) (70 tests) (71 tests) (72 tests) (73 tests) (74 tests) (75 tests) (76 tests) (77 tests) (78 tests) (79 tests) (80 tests) (81 tests) (82 tests) (83 tests) (84 tests) (85 tests) (86 tests) (87 tests) (88 tests) (89 tests) (90 tests) (91 tests) (92 tests) (93 tests) (94 tests) (95 tests) (96 tests) (97 tests) (98 tests) (99 tests) prop_idempotent_fileKey ++++ OK, passed 100 tests. +(0 tests) (1 test) (2 tests) (3 tests) (4 tests) (5 tests) (6 tests) (7 tests) (8 tests) (9 tests) (10 tests) (11 tests) (12 tests) (13 tests) (14 tests) (15 tests) (16 tests) (17 tests) (18 tests) (19 tests) (20 tests) (21 tests) (22 tests) (23 tests) (24 tests) (25 tests) (26 tests) (27 tests) (28 tests) (29 tests) (30 tests) (31 tests) (32 tests) (33 tests) (34 tests) (35 tests) (36 tests) (37 tests) (38 tests) (39 tests) (40 tests) (41 tests) (42 tests) (43 tests) (44 tests) (45 tests) (46 tests) (47 tests) (48 tests) (49 tests) (50 tests) (51 tests) (52 tests) (53 tests) (54 tests) (55 tests) (56 tests) (57 tests) (58 tests) (59 tests) (60 tests) (61 tests) (62 tests) (63 tests) (64 tests) (65 tests) (66 tests) (67 tests) (68 tests) (69 tests) (70 tests) (71 tests) (72 tests) (73 tests) (74 tests) (75 tests) (76 tests) (77 tests) (78 tests) (79 tests) (80 tests) (81 tests) (82 tests) (83 tests) (84 tests) (85 tests) (86 tests) (87 tests) (88 tests) (89 tests) (90 tests) (91 tests) (92 tests) (93 tests) (94 tests) (95 tests) (96 tests) (97 tests) (98 tests) (99 tests) prop_idempotent_key_encode ++++ OK, passed 100 tests. +(0 tests) (1 test) (2 tests) (3 tests) (4 tests) (5 tests) (6 tests) (7 tests) (8 tests) (9 tests) (10 tests) (11 tests) (12 tests) (13 tests) (14 tests) (15 tests) (16 tests) (17 tests) (18 tests) (19 tests) (20 tests) (21 tests) (22 tests) (23 tests) (24 tests) (25 tests) (26 tests) (27 tests) (28 tests) (29 tests) (30 tests) (31 tests) (32 tests) (33 tests) (34 tests) (35 tests) (36 tests) (37 tests) (38 tests) (39 tests) (40 tests) (41 tests) (42 tests) (43 tests) (44 tests) (45 tests) (46 tests) (47 tests) (48 tests) (49 tests) (50 tests) (51 tests) (52 tests) (53 tests) (54 tests) (55 tests) (56 tests) (57 tests) (58 tests) (59 tests) (60 tests) (61 tests) (62 tests) (63 tests) (64 tests) (65 tests) (66 tests) (67 tests) (68 tests) (69 tests) (70 tests) (71 tests) (72 tests) (73 tests) (74 tests) (75 tests) (76 tests) (77 tests) (78 tests) (79 tests) (80 tests) (81 tests) (82 tests) (83 tests) (84 tests) (85 tests) (86 tests) (87 tests) (88 tests) (89 tests) (90 tests) (91 tests) (92 tests) (93 tests) (94 tests) (95 tests) (96 tests) (97 tests) (98 tests) (99 tests) prop_idempotent_shellEscape ++++ OK, passed 100 tests. +(0 tests) (1 test) (2 tests) (3 tests) (4 tests) (5 tests) (6 tests) (7 tests) (8 tests) (9 tests) (10 tests) (11 tests) (12 tests) (13 tests) (14 tests) (15 tests) (16 tests) (17 tests) (18 tests) (19 tests) (20 tests) (21 tests) (22 tests) (23 tests) (24 tests) (25 tests) (26 tests) (27 tests) (28 tests) (29 tests) (30 tests) (31 tests) (32 tests) (33 tests) (34 tests) (35 tests) (36 tests) (37 tests) (38 tests) (39 tests) (40 tests) (41 tests) (42 tests) (43 tests) (44 tests) (45 tests) (46 tests) (47 tests) (48 tests) (49 tests) (50 tests) (51 tests) (52 tests) (53 tests) (54 tests) (55 tests) (56 tests) (57 tests) (58 tests) (59 tests) (60 tests) (61 tests) (62 tests) (63 tests) (64 tests) (65 tests) (66 tests) (67 tests) (68 tests) (69 tests) (70 tests) (71 tests) (72 tests) (73 tests) (74 tests) (75 tests) (76 tests) (77 tests) (78 tests) (79 tests) (80 tests) (81 tests) (82 tests) (83 tests) (84 tests) (85 tests) (86 tests) (87 tests) (88 tests) (89 tests) (90 tests) (91 tests) (92 tests) (93 tests) (94 tests) (95 tests) (96 tests) (97 tests) (98 tests) (99 tests) prop_idempotent_shellEscape_multiword ++++ OK, passed 100 tests. +(0 tests) (1 test) (2 tests) (3 tests) (4 tests) (5 tests) (6 tests) (7 tests) (8 tests) (9 tests) (10 tests) (11 tests) (12 tests) (13 tests) (14 tests) (15 tests) (16 tests) (17 tests) (18 tests) (19 tests) (20 tests) (21 tests) (22 tests) (23 tests) (24 tests) (25 tests) (26 tests) (27 tests) (28 tests) (29 tests) (30 tests) (31 tests) (32 tests) (33 tests) (34 tests) (35 tests) (36 tests) (37 tests) (38 tests) (39 tests) (40 tests) (41 tests) (42 tests) (43 tests) (44 tests) (45 tests) (46 tests) (47 tests) (48 tests) (49 tests) (50 tests) (51 tests) (52 tests) (53 tests) (54 tests) (55 tests) (56 tests) (57 tests) (58 tests) (59 tests) (60 tests) (61 tests) (62 tests) (63 tests) (64 tests) (65 tests) (66 tests) (67 tests) (68 tests) (69 tests) (70 tests) (71 tests) (72 tests) (73 tests) (74 tests) (75 tests) (76 tests) (77 tests) (78 tests) (79 tests) (80 tests) (81 tests) (82 tests) (83 tests) (84 tests) (85 tests) (86 tests) (87 tests) (88 tests) (89 tests) (90 tests) (91 tests) (92 tests) (93 tests) (94 tests) (95 tests) (96 tests) (97 tests) (98 tests) (99 tests) prop_idempotent_configEscape ++++ OK, passed 100 tests. +(0 tests) (1 test) (2 tests) (3 tests) (4 tests) (5 tests) (6 tests) (7 tests) (8 tests) (9 tests) (10 tests) (11 tests) (12 tests) (13 tests) (14 tests) (15 tests) (16 tests) (17 tests) (18 tests) (19 tests) (20 tests) (21 tests) (22 tests) (23 tests) (24 tests) (25 tests) (26 tests) (27 tests) (28 tests) (29 tests) (30 tests) (31 tests) (32 tests) (33 tests) (34 tests) (35 tests) (36 tests) (37 tests) (38 tests) (39 tests) (40 tests) (41 tests) (42 tests) (43 tests) (44 tests) (45 tests) (46 tests) (47 tests) (48 tests) (49 tests) (50 tests) (51 tests) (52 tests) (53 tests) (54 tests) (55 tests) (56 tests) (57 tests) (58 tests) (59 tests) (60 tests) (61 tests) (62 tests) (63 tests) (64 tests) (65 tests) (66 tests) (67 tests) (68 tests) (69 tests) (70 tests) (71 tests) (72 tests) (73 tests) (74 tests) (75 tests) (76 tests) (77 tests) (78 tests) (79 tests) (80 tests) (81 tests) (82 tests) (83 tests) (84 tests) (85 tests) (86 tests) (87 tests) (88 tests) (89 tests) (90 tests) (91 tests) (92 tests) (93 tests) (94 tests) (95 tests) (96 tests) (97 tests) (98 tests) (99 tests) prop_parse_show_Config ++++ OK, passed 100 tests. +(0 tests) (1 test) (2 tests) (3 tests) (4 tests) (5 tests) (6 tests) (7 tests) (8 tests) (9 tests) (10 tests) (11 tests) (12 tests) (13 tests) (14 tests) (15 tests) (16 tests) (17 tests) (18 tests) (19 tests) (20 tests) (21 tests) (22 tests) (23 tests) (24 tests) (25 tests) (26 tests) (27 tests) (28 tests) (29 tests) (30 tests) (31 tests) (32 tests) (33 tests) (34 tests) (35 tests) (36 tests) (37 tests) (38 tests) (39 tests) (40 tests) (41 tests) (42 tests) (43 tests) (44 tests) (45 tests) (46 tests) (47 tests) (48 tests) (49 tests) (50 tests) (51 tests) (52 tests) (53 tests) (54 tests) (55 tests) (56 tests) (57 tests) (58 tests) (59 tests) (60 tests) (61 tests) (62 tests) (63 tests) (64 tests) (65 tests) (66 tests) (67 tests) (68 tests) (69 tests) (70 tests) (71 tests) (72 tests) (73 tests) (74 tests) (75 tests) (76 tests) (77 tests) (78 tests) (79 tests) (80 tests) (81 tests) (82 tests) (83 tests) (84 tests) (85 tests) (86 tests) (87 tests) (88 tests) (89 tests) (90 tests) (91 tests) (92 tests) (93 tests) (94 tests) (95 tests) (96 tests) (97 tests) (98 tests) (99 tests) prop_parentDir_basics ++++ OK, passed 100 tests. +(0 tests) (1 test) (2 tests) (3 tests) (4 tests) (5 tests) (6 tests) (7 tests) (8 tests) (9 tests) (10 tests) (11 tests) (12 tests) (13 tests) (14 tests) (15 tests) (16 tests) (17 tests) (18 tests) (19 tests) (20 tests) (21 tests) (22 tests) (23 tests) (24 tests) (25 tests) (26 tests) (27 tests) (28 tests) (29 tests) (30 tests) (31 tests) (32 tests) (33 tests) (34 tests) (35 tests) (36 tests) (37 tests) (38 tests) (39 tests) (40 tests) (41 tests) (42 tests) (43 tests) (44 tests) (45 tests) (46 tests) (47 tests) (48 tests) (49 tests) (50 tests) (51 tests) (52 tests) (53 tests) (54 tests) (55 tests) (56 tests) (57 tests) (58 tests) (59 tests) (60 tests) (61 tests) (62 tests) (63 tests) (64 tests) (65 tests) (66 tests) (67 tests) (68 tests) (69 tests) (70 tests) (71 tests) (72 tests) (73 tests) (74 tests) (75 tests) (76 tests) (77 tests) (78 tests) (79 tests) (80 tests) (81 tests) (82 tests) (83 tests) (84 tests) (85 tests) (86 tests) (87 tests) (88 tests) (89 tests) (90 tests) (91 tests) (92 tests) (93 tests) (94 tests) (95 tests) (96 tests) (97 tests) (98 tests) (99 tests) prop_relPathDirToFile_basics ++++ OK, passed 100 tests. +(0 tests) prop_relPathDirToFile_regressionTest ++++ OK, passed 1 tests. +(0 tests) prop_cost_sane ++++ OK, passed 1 tests. +(0 tests) prop_matcher_sane ++++ OK, passed 1 tests. +(0 tests) prop_HmacSha1WithCipher_sane ++++ OK, passed 1 tests. +(0 tests) prop_TimeStamp_sane ++++ OK, passed 1 tests. +(0 tests) prop_addLog_sane ++++ OK, passed 1 tests. +(0 tests) (1 test) (2 tests) (3 tests) (4 tests) (5 tests) (6 tests) (7 tests) (8 tests) (9 tests) (10 tests) (11 tests) (12 tests) (13 tests) (14 tests) (15 tests) (16 tests) (17 tests) (18 tests) (19 tests) (20 tests) (21 tests) (22 tests) (23 tests) (24 tests) (25 tests) (26 tests) (27 tests) (28 tests) (29 tests) (30 tests) (31 tests) (32 tests) (33 tests) (34 tests) (35 tests) (36 tests) (37 tests) (38 tests) (39 tests) (40 tests) (41 tests) (42 tests) (43 tests) (44 tests) (45 tests) (46 tests) (47 tests) (48 tests) (49 tests) (50 tests) (51 tests) (52 tests) (53 tests) (54 tests) (55 tests) (56 tests) (57 tests) (58 tests) (59 tests) (60 tests) (61 tests) (62 tests) (63 tests) (64 tests) (65 tests) (66 tests) (67 tests) (68 tests) (69 tests) (70 tests) (71 tests) (72 tests) (73 tests) (74 tests) (75 tests) (76 tests) (77 tests) (78 tests) (79 tests) (80 tests) (81 tests) (82 tests) (83 tests) (84 tests) (85 tests) (86 tests) (87 tests) (88 tests) (89 tests) (90 tests) (91 tests) (92 tests) (93 tests) (94 tests) (95 tests) (96 tests) (97 tests) (98 tests) (99 tests) prop_verifiable_sane ++++ OK, passed 100 tests. +(0 tests) prop_segment_regressionTest ++++ OK, passed 1 tests. +(0 tests) (1 test) (2 tests) (3 tests) (4 tests) (5 tests) (6 tests) (7 tests) (8 tests) (9 tests) (10 tests) (11 tests) (12 tests) (13 tests) (14 tests) (15 tests) (16 tests) (17 tests) (18 tests) (19 tests) (20 tests) (21 tests) (22 tests) (23 tests) (24 tests) (25 tests) (26 tests) (27 tests) (28 tests) (29 tests) (30 tests) (31 tests) (32 tests) (33 tests) (34 tests) (35 tests) (36 tests) (37 tests) (38 tests) (39 tests) (40 tests) (41 tests) (42 tests) (43 tests) (44 tests) (45 tests) (46 tests) (47 tests) (48 tests) (49 tests) (50 tests) (51 tests) (52 tests) (53 tests) (54 tests) (55 tests) (56 tests) (57 tests) (58 tests) (59 tests) (60 tests) (61 tests) (62 tests) (63 tests) (64 tests) (65 tests) (66 tests) (67 tests) (68 tests) (69 tests) (70 tests) (71 tests) (72 tests) (73 tests) (74 tests) (75 tests) (76 tests) (77 tests) (78 tests) (79 tests) (80 tests) (81 tests) (82 tests) (83 tests) (84 tests) (85 tests) (86 tests) (87 tests) (88 tests) (89 tests) (90 tests) (91 tests) (92 tests) (93 tests) (94 tests) (95 tests) (96 tests) (97 tests) (98 tests) (99 tests) prop_read_write_transferinfo ++++ OK, passed 100 tests. +(0 tests) (1 test) (2 tests) (3 tests) (4 tests) (5 tests) (6 tests) (7 tests) (8 tests) (9 tests) (10 tests) (11 tests) (12 tests) (13 tests) (14 tests) (15 tests) (16 tests) (17 tests) (18 tests) (19 tests) (20 tests) (21 tests) (22 tests) (23 tests) (24 tests) (25 tests) (26 tests) (27 tests) (28 tests) (29 tests) (30 tests) (31 tests) (32 tests) (33 tests) (34 tests) (35 tests) (36 tests) (37 tests) (38 tests) (39 tests) (40 tests) (41 tests) (42 tests) (43 tests) (44 tests) (45 tests) (46 tests) (47 tests) (48 tests) (49 tests) (50 tests) (51 tests) (52 tests) (53 tests) (54 tests) (55 tests) (56 tests) (57 tests) (58 tests) (59 tests) (60 tests) (61 tests) (62 tests) (63 tests) (64 tests) (65 tests) (66 tests) (67 tests) (68 tests) (69 tests) (70 tests) (71 tests) (72 tests) (73 tests) (74 tests) (75 tests) (76 tests) (77 tests) (78 tests) (79 tests) (80 tests) (81 tests) (82 tests) (83 tests) (84 tests) (85 tests) (86 tests) (87 tests) (88 tests) (89 tests) (90 tests) (91 tests) (92 tests) (93 tests) (94 tests) (95 tests) (96 tests) (97 tests) (98 tests) (99 tests) prop_read_show_inodecache ++++ OK, passed 100 tests. +(0 tests) (1 test) (2 tests) (3 tests) (4 tests) (5 tests) (6 tests) (7 tests) (8 tests) (9 tests) (10 tests) (11 tests) (12 tests) (13 tests) (14 tests) (15 tests) (16 tests) (17 tests) (18 tests) (19 tests) (20 tests) (21 tests) (22 tests) (23 tests) (24 tests) (25 tests) (26 tests) (27 tests) (28 tests) (29 tests) (30 tests) (31 tests) (32 tests) (33 tests) (34 tests) (35 tests) (36 tests) (37 tests) (38 tests) (39 tests) (40 tests) (41 tests) (42 tests) (43 tests) (44 tests) (45 tests) (46 tests) (47 tests) (48 tests) (49 tests) (50 tests) (51 tests) (52 tests) (53 tests) (54 tests) (55 tests) (56 tests) (57 tests) (58 tests) (59 tests) (60 tests) (61 tests) (62 tests) (63 tests) (64 tests) (65 tests) (66 tests) (67 tests) (68 tests) (69 tests) (70 tests) (71 tests) (72 tests) (73 tests) (74 tests) (75 tests) (76 tests) (77 tests) (78 tests) (79 tests) (80 tests) (81 tests) (82 tests) (83 tests) (84 tests) (85 tests) (86 tests) (87 tests) (88 tests) (89 tests) (90 tests) (91 tests) (92 tests) (93 tests) (94 tests) (95 tests) (96 tests) (97 tests) (98 tests) (99 tests) prop_parse_show_log ++++ OK, passed 100 tests. +(0 tests) prop_read_show_TrustLevel ++++ OK, passed 1 tests. +(0 tests) prop_parse_show_TrustLog ++++ OK, passed 1 tests. + +Cases: 1 Tried: 0 Errors: 0 Failures: 0init test repo + Detected a crippled filesystem. + + Disabling core.symlinks. + + Enabling direct mode. + + Detected a filesystem without fifo support. + + Disabling ssh connection caching. +ok +(Recording state in git...) + + +Cases: 1 Tried: 1 Errors: 0 Failures: 0 + +Cases: 3 Tried: 0 Errors: 0 Failures: 0add foo (checksum...) ok +(Recording state in git...) +add sha1foo (checksum...) ok +(Recording state in git...) +add apple ok +(Recording state in git...) + +Cases: 3 Tried: 1 Errors: 0 Failures: 0ssh: Could not resolve hostname C: no address associated with name +fatal: The remote end hung up unexpectedly + + +### Failure in: git-annex add:1 +git clone failed + +Cases: 3 Tried: 2 Errors: 0 Failures: 1ssh: Could not resolve hostname C: no address associated with name +fatal: The remote end hung up unexpectedly + + +### Failure in: git-annex add:2 +git clone failed +Cases: 3 Tried: 3 Errors: 0 Failures: 2 + +Cases: 1 Tried: 0 Errors: 0 Failures: 0ssh: Could not resolve hostname C: no address associated with name +fatal: The remote end hung up unexpectedly + + +### Failure in: git-annex reinject/fromkey +git clone failed +Cases: 1 Tried: 1 Errors: 0 Failures: 1 + +Cases: 2 Tried: 0 Errors: 0 Failures: 0ssh: Could not resolve hostname C: no address associated with name +fatal: The remote end hung up unexpectedly + + +### Failure in: git-annex unannex:0:no content +git clone failed + +Cases: 2 Tried: 1 Errors: 0 Failures: 1ssh: Could not resolve hostname C: no address associated with name +fatal: The remote end hung up unexpectedly + + +### Failure in: git-annex unannex:1:with content +git clone failed +Cases: 2 Tried: 2 Errors: 0 Failures: 2 + +Cases: 3 Tried: 0 Errors: 0 Failures: 0ssh: Could not resolve hostname C: no address associated with name +fatal: The remote end hung up unexpectedly + + +### Failure in: git-annex drop:0:no remotes +git clone failed + +Cases: 3 Tried: 1 Errors: 0 Failures: 1ssh: Could not resolve hostname C: no address associated with name +fatal: The remote end hung up unexpectedly + + +### Failure in: git-annex drop:1:with remote +git clone failed + +Cases: 3 Tried: 2 Errors: 0 Failures: 2ssh: Could not resolve hostname C: no address associated with name +fatal: The remote end hung up unexpectedly + + +### Failure in: git-annex drop:2:untrusted remote +git clone failed +Cases: 3 Tried: 3 Errors: 0 Failures: 3 + +Cases: 1 Tried: 0 Errors: 0 Failures: 0ssh: Could not resolve hostname C: no address associated with name +fatal: The remote end hung up unexpectedly + + +### Failure in: git-annex get +git clone failed +Cases: 1 Tried: 1 Errors: 0 Failures: 1 + +Cases: 1 Tried: 0 Errors: 0 Failures: 0ssh: Could not resolve hostname C: no address associated with name +fatal: The remote end hung up unexpectedly + + +### Failure in: git-annex move +git clone failed +Cases: 1 Tried: 1 Errors: 0 Failures: 1 + +Cases: 1 Tried: 0 Errors: 0 Failures: 0ssh: Could not resolve hostname C: no address associated with name +fatal: The remote end hung up unexpectedly + + +### Failure in: git-annex copy +git clone failed +Cases: 1 Tried: 1 Errors: 0 Failures: 1 + +Cases: 1 Tried: 0 Errors: 0 Failures: 0ssh: Could not resolve hostname C: no address associated with name +fatal: The remote end hung up unexpectedly + + +### Failure in: git-annex unlock/lock +git clone failed +Cases: 1 Tried: 1 Errors: 0 Failures: 1 + +Cases: 2 Tried: 0 Errors: 0 Failures: 0ssh: Could not resolve hostname C: no address associated with name +fatal: The remote end hung up unexpectedly + + +### Failure in: git-annex edit/commit:0 +git clone failed + +Cases: 2 Tried: 1 Errors: 0 Failures: 1ssh: Could not resolve hostname C: no address associated with name +fatal: The remote end hung up unexpectedly + + +### Failure in: git-annex edit/commit:1 +git clone failed +Cases: 2 Tried: 2 Errors: 0 Failures: 2 + +Cases: 1 Tried: 0 Errors: 0 Failures: 0ssh: Could not resolve hostname C: no address associated with name +fatal: The remote end hung up unexpectedly + + +### Failure in: git-annex fix +git clone failed +Cases: 1 Tried: 1 Errors: 0 Failures: 1 + +Cases: 1 Tried: 0 Errors: 0 Failures: 0ssh: Could not resolve hostname C: no address associated with name +fatal: The remote end hung up unexpectedly + + +### Failure in: git-annex trust/untrust/semitrust/dead +git clone failed +Cases: 1 Tried: 1 Errors: 0 Failures: 1 + +Cases: 4 Tried: 0 Errors: 0 Failures: 0ssh: Could not resolve hostname C: no address associated with name +fatal: The remote end hung up unexpectedly + + +### Failure in: git-annex fsck:0 +git clone failed + +Cases: 4 Tried: 1 Errors: 0 Failures: 1ssh: Could not resolve hostname C: no address associated with name +fatal: The remote end hung up unexpectedly + + +### Failure in: git-annex fsck:1 +git clone failed + +Cases: 4 Tried: 2 Errors: 0 Failures: 2ssh: Could not resolve hostname C: no address associated with name +fatal: The remote end hung up unexpectedly + + +### Failure in: git-annex fsck:2 +git clone failed + +Cases: 4 Tried: 3 Errors: 0 Failures: 3ssh: Could not resolve hostname C: no address associated with name +fatal: The remote end hung up unexpectedly + + +### Failure in: git-annex fsck:3 +git clone failed +Cases: 4 Tried: 4 Errors: 0 Failures: 4 + +Cases: 2 Tried: 0 Errors: 0 Failures: 0ssh: Could not resolve hostname C: no address associated with name +fatal: The remote end hung up unexpectedly + + +### Failure in: git-annex migrate:0 +git clone failed + +Cases: 2 Tried: 1 Errors: 0 Failures: 1ssh: Could not resolve hostname C: no address associated with name +fatal: The remote end hung up unexpectedly + + +### Failure in: git-annex migrate:1 +git clone failed +Cases: 2 Tried: 2 Errors: 0 Failures: 2 + +Cases: 1 Tried: 0 Errors: 0 Failures: 0ssh: Could not resolve hostname C: no address associated with name +fatal: The remote end hung up unexpectedly + + +### Failure in: git-annex unused/dropunused +git clone failed +Cases: 1 Tried: 1 Errors: 0 Failures: 1 + +Cases: 1 Tried: 0 Errors: 0 Failures: 0ssh: Could not resolve hostname C: no address associated with name +fatal: The remote end hung up unexpectedly + + +### Failure in: git-annex describe +git clone failed +Cases: 1 Tried: 1 Errors: 0 Failures: 1 + +Cases: 1 Tried: 0 Errors: 0 Failures: 0ssh: Could not resolve hostname C: no address associated with name +fatal: The remote end hung up unexpectedly + + +### Failure in: git-annex find +git clone failed +Cases: 1 Tried: 1 Errors: 0 Failures: 1 + +Cases: 1 Tried: 0 Errors: 0 Failures: 0ssh: Could not resolve hostname C: no address associated with name +fatal: The remote end hung up unexpectedly + + +### Failure in: git-annex merge +git clone failed +Cases: 1 Tried: 1 Errors: 0 Failures: 1 + +Cases: 1 Tried: 0 Errors: 0 Failures: 0ssh: Could not resolve hostname C: no address associated with name +fatal: The remote end hung up unexpectedly + + +### Failure in: git-annex status +git clone failed +Cases: 1 Tried: 1 Errors: 0 Failures: 1 + +Cases: 1 Tried: 0 Errors: 0 Failures: 0ssh: Could not resolve hostname C: no address associated with name +fatal: The remote end hung up unexpectedly + + +### Failure in: git-annex version +git clone failed +Cases: 1 Tried: 1 Errors: 0 Failures: 1 + +Cases: 1 Tried: 0 Errors: 0 Failures: 0ssh: Could not resolve hostname C: no address associated with name +fatal: The remote end hung up unexpectedly + + +### Failure in: git-annex sync +git clone failed +Cases: 1 Tried: 1 Errors: 0 Failures: 1 + +Cases: 1 Tried: 0 Errors: 0 Failures: 0ssh: Could not resolve hostname C: no address associated with name +fatal: The remote end hung up unexpectedly + + +### Failure in: union merge regression +git clone failed +Cases: 1 Tried: 1 Errors: 0 Failures: 1 + +Cases: 1 Tried: 0 Errors: 0 Failures: 0ssh: Could not resolve hostname C: no address associated with name +fatal: The remote end hung up unexpectedly + + +### Failure in: automatic conflict resolution +git clone failed +Cases: 1 Tried: 1 Errors: 0 Failures: 1 + +Cases: 1 Tried: 0 Errors: 0 Failures: 0ssh: Could not resolve hostname C: no address associated with name +fatal: The remote end hung up unexpectedly + + +### Failure in: git-annex map +git clone failed +Cases: 1 Tried: 1 Errors: 0 Failures: 1 + +Cases: 1 Tried: 0 Errors: 0 Failures: 0ssh: Could not resolve hostname C: no address associated with name +fatal: The remote end hung up unexpectedly + + +### Failure in: git-annex uninit +git clone failed +Cases: 1 Tried: 1 Errors: 0 Failures: 1 + +Cases: 1 Tried: 0 Errors: 0 Failures: 0ssh: Could not resolve hostname C: no address associated with name +fatal: The remote end hung up unexpectedly + + +### Failure in: git-annex upgrade +git clone failed +Cases: 1 Tried: 1 Errors: 0 Failures: 1 + +Cases: 1 Tried: 0 Errors: 0 Failures: 0ssh: Could not resolve hostname C: no address associated with name +fatal: The remote end hung up unexpectedly + + +### Failure in: git-annex whereis +git clone failed +Cases: 1 Tried: 1 Errors: 0 Failures: 1 + +Cases: 1 Tried: 0 Errors: 0 Failures: 0ssh: Could not resolve hostname C: no address associated with name +fatal: The remote end hung up unexpectedly + + +### Failure in: git-annex hook remote +git clone failed +Cases: 1 Tried: 1 Errors: 0 Failures: 1 + +Cases: 1 Tried: 0 Errors: 0 Failures: 0ssh: Could not resolve hostname C: no address associated with name +fatal: The remote end hung up unexpectedly + + +### Failure in: git-annex directory remote +git clone failed +Cases: 1 Tried: 1 Errors: 0 Failures: 1 + +Cases: 1 Tried: 0 Errors: 0 Failures: 0ssh: Could not resolve hostname C: no address associated with name +fatal: The remote end hung up unexpectedly + + +### Failure in: git-annex rsync remote +git clone failed +Cases: 1 Tried: 1 Errors: 0 Failures: 1 + +Cases: 1 Tried: 0 Errors: 0 Failures: 0ssh: Could not resolve hostname C: no address associated with name +fatal: The remote end hung up unexpectedly + + +### Failure in: git-annex bup remote +git clone failed +Cases: 1 Tried: 1 Errors: 0 Failures: 1 + +Cases: 1 Tried: 0 Errors: 0 Failures: 0ssh: Could not resolve hostname C: no address associated with name +fatal: The remote end hung up unexpectedly + + +### Failure in: git-annex crypto +git clone failed +Cases: 1 Tried: 1 Errors: 0 Failures: 1 + +Cases: 1 Tried: 0 Errors: 0 Failures: 0ssh: Could not resolve hostname C: no address associated with name +fatal: The remote end hung up unexpectedly + + +### Failure in: git-annex preferred-content +git clone failed +Cases: 1 Tried: 1 Errors: 0 Failures: 1 +---------------------------------------------------------------------- +Now, some broader checks ... + (Do not be alarmed by odd output here; it's normal. + wait for the last line to see how it went.) +---------------------------------------------------------------------- +init +---------------------------------------------------------------------- +add +---------------------------------------------------------------------- +reinject +---------------------------------------------------------------------- +unannex +---------------------------------------------------------------------- +drop +---------------------------------------------------------------------- +get +---------------------------------------------------------------------- +move +---------------------------------------------------------------------- +copy +---------------------------------------------------------------------- +lock +---------------------------------------------------------------------- +edit +---------------------------------------------------------------------- +fix +---------------------------------------------------------------------- +trust +---------------------------------------------------------------------- +fsck +---------------------------------------------------------------------- +migrate +---------------------------------------------------------------------- + unused +---------------------------------------------------------------------- +describe +---------------------------------------------------------------------- +find +---------------------------------------------------------------------- +merge +---------------------------------------------------------------------- +status +---------------------------------------------------------------------- +version +---------------------------------------------------------------------- +sync +---------------------------------------------------------------------- +union merge regression +---------------------------------------------------------------------- +conflict resolution +---------------------------------------------------------------------- +map +---------------------------------------------------------------------- +uninit +---------------------------------------------------------------------- +upgrade +---------------------------------------------------------------------- +whereis +---------------------------------------------------------------------- +hook remote +---------------------------------------------------------------------- +directory remote +---------------------------------------------------------------------- +rsync remote +---------------------------------------------------------------------- +bup remote +---------------------------------------------------------------------- +crypto +---------------------------------------------------------------------- +preferred content +---------------------------------------------------------------------- +Some tests failed! + (This could be due to a bug in git-annex, or an incompatibility + with utilities, such as git, installed on this system.) + + +# End of transcript or log. +"""]] + +[[!tag moreinfo]] diff --git a/doc/bugs/Windows_build_test_failures/comment_1_ea7523fdbafdc8be2971df52d9038826._comment b/doc/bugs/Windows_build_test_failures/comment_1_ea7523fdbafdc8be2971df52d9038826._comment new file mode 100644 index 0000000000..dc09fb89b0 --- /dev/null +++ b/doc/bugs/Windows_build_test_failures/comment_1_ea7523fdbafdc8be2971df52d9038826._comment @@ -0,0 +1,10 @@ +[[!comment format=mdwn + username="http://joeyh.name/" + ip="4.154.2.134" + subject="comment 1" + date="2013-09-13T19:18:26Z" + content=""" +The Windows port has improved quite a lot since this bug was filed. Can you still reproduce it? + +If you install the pre-built git-annex for Windows, you can run `git annex test` to run the test suite. Does that work? It works for some others on Windows. +"""]] diff --git a/doc/bugs/Windows_installer_includes_curl_and_wget__44___but_not_required_DLLs.mdwn b/doc/bugs/Windows_installer_includes_curl_and_wget__44___but_not_required_DLLs.mdwn new file mode 100644 index 0000000000..a21813500e --- /dev/null +++ b/doc/bugs/Windows_installer_includes_curl_and_wget__44___but_not_required_DLLs.mdwn @@ -0,0 +1,17 @@ +### Please describe the problem. +On a default, clean install on Windows, annex get over HTTP fails, as neither wget nor curl have all necessary DLLs. + +### What steps will reproduce the problem? +1. Install git from git-scm.org, and git-annex from git-annex.branchable.com. +2. Don't install, or remove if installed, Cygwin. +3. clone an annex over HTTP +4. attempt to 'annex get' a file +5. get dialog box, explaining missing DLLs. + +### What version of git-annex are you using? On what operating system? +Windows 7. + +### Please provide any additional information below. +Installing Cygwin and adding to path seems to fix issue. + +[[!tag moreinfo]] diff --git a/doc/bugs/Windows_installer_includes_curl_and_wget__44___but_not_required_DLLs/comment_1_a7bf0f027f2209e5632e292afd7214d0._comment b/doc/bugs/Windows_installer_includes_curl_and_wget__44___but_not_required_DLLs/comment_1_a7bf0f027f2209e5632e292afd7214d0._comment new file mode 100644 index 0000000000..05639ef3a9 --- /dev/null +++ b/doc/bugs/Windows_installer_includes_curl_and_wget__44___but_not_required_DLLs/comment_1_a7bf0f027f2209e5632e292afd7214d0._comment @@ -0,0 +1,10 @@ +[[!comment format=mdwn + username="http://joeyh.name/" + ip="4.153.254.222" + subject="comment 1" + date="2013-07-07T17:47:54Z" + content=""" +The Windows installer includes numerous DLLs from Cygwin. I have tested it in a non-cygwin environment and it seems to work for me, and for other users. At least generally.. perhaps it *is* missing something used by wget or curl? + +Can you please provide the names of the DLLs that seem to be missing? +"""]] diff --git a/doc/bugs/Windows_to_Linux_clone_-_Windows_drive_letters_cause_git_annex_get_to_fail.mdwn b/doc/bugs/Windows_to_Linux_clone_-_Windows_drive_letters_cause_git_annex_get_to_fail.mdwn new file mode 100644 index 0000000000..6e5fd8433d --- /dev/null +++ b/doc/bugs/Windows_to_Linux_clone_-_Windows_drive_letters_cause_git_annex_get_to_fail.mdwn @@ -0,0 +1,134 @@ +[[!tag moreinfo]] + +### Please describe the problem. +git annex get on a clone of a repository created on Windows fails. + +### What steps will reproduce the problem? +On Windows: +1. Create an annex using git annex init + +2. git add files + +3. git annex sync (because I forgot about commit) + +On Linux: + +1. Mount the Windows annex via SMB (mine was read only, but probably won't matter) + +2. git clone the folder (works fine) + +3. git annex get (fails, due to Windows drive letter) + +### What version of git-annex are you using? On what operating system? + +*Windows:* + +git-annex version: 4.20130802-g0a52f02 + +build flags: Pairing Testsuite S3 WebDAV DNS + +*Linux: (Debian wheezy on ARMel)* + +git-annex version: 3.20120629 + +local repository version: 3 + +default repository version: 3 + +supported repository versions: 3 + +upgrade supported from repository versions: 0 1 2 + + +### Please provide any additional information below. + +[[!format sh """ +# If you can, paste a complete transcript of the problem occurring here. +# If the problem is with the git-annex assistant, paste in .git/annex/daemon.log + +#output of git annex get on the Linux side. + +get Purchased/Caravan Palace/Panic/01 Queens.mp3 (not available) + Try making some of these repositories available: + d6c78593-79d2-455a-8202-7be2133e2d48 -- ICARUS:E:\music +failed +get Purchased/Caravan Palace/Panic/02 Maniac.mp3 (not available) + Try making some of these repositories available: + d6c78593-79d2-455a-8202-7be2133e2d48 -- ICARUS:E:\music +failed +get Purchased/Caravan Palace/Panic/03 The Dirty Side of the Street.mp3 (not available) + Try making some of these repositories available: + d6c78593-79d2-455a-8202-7be2133e2d48 -- ICARUS:E:\music +failed +get Purchased/Caravan Palace/Panic/04 12 juin 3049.mp3 (not available) + Try making some of these repositories available: + d6c78593-79d2-455a-8202-7be2133e2d48 -- ICARUS:E:\music +failed +get Purchased/Caravan Palace/Panic/05 Rock It for Me.mp3 (not available) + Try making some of these repositories available: + d6c78593-79d2-455a-8202-7be2133e2d48 -- ICARUS:E:\music +failed +get Purchased/Caravan Palace/Panic/06 Clash.mp3 (not available) + Try making some of these repositories available: + d6c78593-79d2-455a-8202-7be2133e2d48 -- ICARUS:E:\music +failed +get Purchased/Caravan Palace/Panic/07 Newbop.mp3 (not available) + Try making some of these repositories available: + d6c78593-79d2-455a-8202-7be2133e2d48 -- ICARUS:E:\music +failed +get Purchased/Caravan Palace/Panic/08 Glory of Nelly.mp3 (not available) + Try making some of these repositories available: + d6c78593-79d2-455a-8202-7be2133e2d48 -- ICARUS:E:\music +failed +get Purchased/Caravan Palace/Panic/09 Dramophone.mp3 (not available) + Try making some of these repositories available: + d6c78593-79d2-455a-8202-7be2133e2d48 -- ICARUS:E:\music +failed +get Purchased/Caravan Palace/Panic/10 Cotton Heads.mp3 (not available) + Try making some of these repositories available: + d6c78593-79d2-455a-8202-7be2133e2d48 -- ICARUS:E:\music +failed +get Purchased/Caravan Palace/Panic/11 Panic.mp3 (not available) + Try making some of these repositories available: + d6c78593-79d2-455a-8202-7be2133e2d48 -- ICARUS:E:\music +failed +get Purchased/Caravan Palace/Panic/12 Pirates.mp3 (not available) + Try making some of these repositories available: + d6c78593-79d2-455a-8202-7be2133e2d48 -- ICARUS:E:\music +failed +get Purchased/Caravan Palace/Panic/13 Beatophone.mp3 (not available) + Try making some of these repositories available: + d6c78593-79d2-455a-8202-7be2133e2d48 -- ICARUS:E:\music +failed +get Purchased/Caravan Palace/Panic/14 Sydney.mp3 (not available) + Try making some of these repositories available: + d6c78593-79d2-455a-8202-7be2133e2d48 -- ICARUS:E:\music +failed +get Purchased/Caravan Palace/Panic/Thumbs.db (not available) + Try making some of these repositories available: + d6c78593-79d2-455a-8202-7be2133e2d48 -- ICARUS:E:\music + + +And the (broken) symlinks: + +lrwxrwxrwx 1 voltagex voltagex 198 Aug 11 23:05 01 Queens.mp3 -> .git/annex/objects/5P/vq/SHA256E-s9886015--dfb8f452d47f997a9141de3f152de375fad1d5ccb4a20d5c022064a630eaba88.mp3/SHA256E-s9886015--dfb8f452d47f997a9141de3f152de375fad1d5ccb4a20d5c022064a630eaba88.mp3 +lrwxrwxrwx 1 voltagex voltagex 200 Aug 11 23:05 02 Maniac.mp3 -> .git/annex/objects/J0/WV/SHA256E-s10116938--7ce35f4a54c4f74bd2e3813445cb3a07fae76b2854df06ab00876cd0067f34a5.mp3/SHA256E-s10116938--7ce35f4a54c4f74bd2e3813445cb3a07fae76b2854df06ab00876cd0067f34a5.mp3 +lrwxrwxrwx 1 voltagex voltagex 198 Aug 11 23:05 03 The Dirty Side of the Street.mp3 -> .git/annex/objects/xz/4F/SHA256E-s8812949--9e15761ae8c6231d11d1f5a67de8a89142e2e4017b43b8ffac1088fd71842ca7.mp3/SHA256E-s8812949--9e15761ae8c6231d11d1f5a67de8a89142e2e4017b43b8ffac1088fd71842ca7.mp3 +lrwxrwxrwx 1 voltagex voltagex 198 Aug 11 23:05 04 12 juin 3049.mp3 -> .git/annex/objects/Kp/k6/SHA256E-s8236133--3a6465bea3272ae7a22b77b67e96a91ae389390cc95642fb1456ce80e3313033.mp3/SHA256E-s8236133--3a6465bea3272ae7a22b77b67e96a91ae389390cc95642fb1456ce80e3313033.mp3 +lrwxrwxrwx 1 voltagex voltagex 198 Aug 11 23:05 05 Rock It for Me.mp3 -> .git/annex/objects/Wk/Qv/SHA256E-s7701150--8197e499408e64601157df95927ac6b09bd37834a41eceecc5d12667bd7c66a4.mp3/SHA256E-s7701150--8197e499408e64601157df95927ac6b09bd37834a41eceecc5d12667bd7c66a4.mp3 +lrwxrwxrwx 1 voltagex voltagex 200 Aug 11 23:05 06 Clash.mp3 -> .git/annex/objects/ZQ/P4/SHA256E-s10201572--c2a1a8892fcaaab1ed59ca7f5ab9d45f0c3bb3c6d6450b95228039b9e8d7a0b4.mp3/SHA256E-s10201572--c2a1a8892fcaaab1ed59ca7f5ab9d45f0c3bb3c6d6450b95228039b9e8d7a0b4.mp3 +lrwxrwxrwx 1 voltagex voltagex 198 Aug 11 23:05 07 Newbop.mp3 -> .git/annex/objects/Fk/wV/SHA256E-s6850587--b5ba0347f6a09a7ff8bec2dd1fbe8076fdff645d4bc1909ddeadfc035bf19fda.mp3/SHA256E-s6850587--b5ba0347f6a09a7ff8bec2dd1fbe8076fdff645d4bc1909ddeadfc035bf19fda.mp3 +lrwxrwxrwx 1 voltagex voltagex 198 Aug 11 23:05 08 Glory of Nelly.mp3 -> .git/annex/objects/Pm/K3/SHA256E-s9002048--18149581cbfc3b4e3e72b784777fe34c53f1580e78e97f66058be0b1eb40e809.mp3/SHA256E-s9002048--18149581cbfc3b4e3e72b784777fe34c53f1580e78e97f66058be0b1eb40e809.mp3 +lrwxrwxrwx 1 voltagex voltagex 198 Aug 11 23:05 09 Dramophone.mp3 -> .git/annex/objects/k3/Jf/SHA256E-s8199558--925117d9fc47a65e8e5324f3d0638a3c24bf51fd6c0b5d8ac2f63951c893cc48.mp3/SHA256E-s8199558--925117d9fc47a65e8e5324f3d0638a3c24bf51fd6c0b5d8ac2f63951c893cc48.mp3 +lrwxrwxrwx 1 voltagex voltagex 198 Aug 11 23:05 10 Cotton Heads.mp3 -> .git/annex/objects/9f/32/SHA256E-s8801425--c6926238c0b1a3bbea1a5d17841ceac591e53e223e4c4c45a2077cabffc85d81.mp3/SHA256E-s8801425--c6926238c0b1a3bbea1a5d17841ceac591e53e223e4c4c45a2077cabffc85d81.mp3 +lrwxrwxrwx 1 voltagex voltagex 198 Aug 11 23:05 11 Panic.mp3 -> .git/annex/objects/XF/WF/SHA256E-s9833770--5e837c7fa3ec096f7c0507efbcf398067029749f8a0fc77a2badf864b9ffbb7c.mp3/SHA256E-s9833770--5e837c7fa3ec096f7c0507efbcf398067029749f8a0fc77a2badf864b9ffbb7c.mp3 +lrwxrwxrwx 1 voltagex voltagex 198 Aug 11 23:05 12 Pirates.mp3 -> .git/annex/objects/7Z/jz/SHA256E-s8017742--5fd49b2d89577266d2fb740e7e7def9338475af90c2ca99f9d6d513465b2bfac.mp3/SHA256E-s8017742--5fd49b2d89577266d2fb740e7e7def9338475af90c2ca99f9d6d513465b2bfac.mp3 +lrwxrwxrwx 1 voltagex voltagex 198 Aug 11 23:05 13 Beatophone.mp3 -> .git/annex/objects/81/xv/SHA256E-s9339544--a1eb8404ecf0503b9f635378fec4e2c95ec08bb1428c2cd4c0cedf492811577d.mp3/SHA256E-s9339544--a1eb8404ecf0503b9f635378fec4e2c95ec08bb1428c2cd4c0cedf492811577d.mp3 +lrwxrwxrwx 1 voltagex voltagex 198 Aug 11 23:05 14 Sydney.mp3 -> .git/annex/objects/3K/99/SHA256E-s8459731--4ff44b25c912e914c79124ff9074c576c0024152442fc96c9bad65f5a50a40d9.mp3/SHA256E-s8459731--4ff44b25c912e914c79124ff9074c576c0024152442fc96c9bad65f5a50a40d9.mp3 +lrwxrwxrwx 1 voltagex voltagex 192 Aug 11 23:05 Thumbs.db -> .git/annex/objects/28/Zv/SHA256E-s85504--3604669cd3a55234516191eb4f19434829c1634d6dd69a9981185f095d2bbaba.db/SHA256E-s85504--3604669cd3a55234516191eb4f19434829c1634d6dd69a9981185f095d2bbaba.db + +# End of transcript or log. +"""]] + +> Closing since bug submitter did not follow up with requested information. +> [[done]] --[[Joey]] diff --git a/doc/bugs/Windows_to_Linux_clone_-_Windows_drive_letters_cause_git_annex_get_to_fail/comment_1_c87bae87b7902db60a3fef41e1fca85d._comment b/doc/bugs/Windows_to_Linux_clone_-_Windows_drive_letters_cause_git_annex_get_to_fail/comment_1_c87bae87b7902db60a3fef41e1fca85d._comment new file mode 100644 index 0000000000..0577531b7e --- /dev/null +++ b/doc/bugs/Windows_to_Linux_clone_-_Windows_drive_letters_cause_git_annex_get_to_fail/comment_1_c87bae87b7902db60a3fef41e1fca85d._comment @@ -0,0 +1,9 @@ +[[!comment format=mdwn + username="https://www.google.com/accounts/o8/id?id=AItOawlzWwnBfgJrkhPQakBo6DbPXutJIVDHkj0" + nickname="Adam" + subject="comment 1" + date="2013-08-11T14:15:40Z" + content=""" +Confirming the same errors happen with git-annex version: 4.20130802 from sid. + +"""]] diff --git a/doc/bugs/Windows_to_Linux_clone_-_Windows_drive_letters_cause_git_annex_get_to_fail/comment_2_9e3c1f1ba05d8996b5a95829ce32c07e._comment b/doc/bugs/Windows_to_Linux_clone_-_Windows_drive_letters_cause_git_annex_get_to_fail/comment_2_9e3c1f1ba05d8996b5a95829ce32c07e._comment new file mode 100644 index 0000000000..d98e7976e3 --- /dev/null +++ b/doc/bugs/Windows_to_Linux_clone_-_Windows_drive_letters_cause_git_annex_get_to_fail/comment_2_9e3c1f1ba05d8996b5a95829ce32c07e._comment @@ -0,0 +1,10 @@ +[[!comment format=mdwn + username="http://joeyh.name/" + ip="4.154.0.63" + subject="bad bug report title 101" + date="2013-08-24T19:17:29Z" + content=""" +I don't understand why you think the problem has something to do with Windows drive letters. There are no Windows drive letters in the symlinks you show. The only place I see any Windows drive letter is in the descripton of the remote that `git annex get` displays when it fails to get the file. That description is purely informative, it's not a path that git-annex is trying to use. + +I'd suggest that you run `git annex get --debug` to see if it is doing anything obviously wrong. The mostly likely culprit is your SMB setup, which I am not going to be able to replicate. +"""]] diff --git a/doc/bugs/Windows_to_Linux_clone_-_Windows_drive_letters_cause_git_annex_get_to_fail/comment_3_3a0787912f4a3a8797b7786f5ce38590._comment b/doc/bugs/Windows_to_Linux_clone_-_Windows_drive_letters_cause_git_annex_get_to_fail/comment_3_3a0787912f4a3a8797b7786f5ce38590._comment new file mode 100644 index 0000000000..16f240cf29 --- /dev/null +++ b/doc/bugs/Windows_to_Linux_clone_-_Windows_drive_letters_cause_git_annex_get_to_fail/comment_3_3a0787912f4a3a8797b7786f5ce38590._comment @@ -0,0 +1,8 @@ +[[!comment format=mdwn + username="https://www.google.com/accounts/o8/id?id=AItOawlzWwnBfgJrkhPQakBo6DbPXutJIVDHkj0" + nickname="Adam" + subject="comment 3" + date="2013-08-26T06:56:33Z" + content=""" +You're correct. I can see in .git/config that the remote references z:\ which of course will break on the Linux side. Maybe this is a case of the error messages not quite telling me the right thing? +"""]] diff --git a/doc/bugs/Windows_to_Linux_clone_-_Windows_drive_letters_cause_git_annex_get_to_fail/comment_4_c4249f32d65594d79ea01145b93ec948._comment b/doc/bugs/Windows_to_Linux_clone_-_Windows_drive_letters_cause_git_annex_get_to_fail/comment_4_c4249f32d65594d79ea01145b93ec948._comment new file mode 100644 index 0000000000..31bc7c1a16 --- /dev/null +++ b/doc/bugs/Windows_to_Linux_clone_-_Windows_drive_letters_cause_git_annex_get_to_fail/comment_4_c4249f32d65594d79ea01145b93ec948._comment @@ -0,0 +1,9 @@ +[[!comment format=mdwn + username="http://joeyh.name/" + ip="4.154.2.134" + subject="comment 4" + date="2013-09-13T19:29:25Z" + content=""" +I'd suggest that you run git annex get --debug to see if it is doing anything obviously wrong. The mostly likely culprit is your SMB setup, which I am not going to be able to replicate. + +"""]] diff --git a/doc/bugs/Windows_to_Linux_clone_-_Windows_drive_letters_cause_git_annex_get_to_fail/comment_5_9308d5ef037afdaf48cc3378cfa10afb._comment b/doc/bugs/Windows_to_Linux_clone_-_Windows_drive_letters_cause_git_annex_get_to_fail/comment_5_9308d5ef037afdaf48cc3378cfa10afb._comment new file mode 100644 index 0000000000..2964382d8e --- /dev/null +++ b/doc/bugs/Windows_to_Linux_clone_-_Windows_drive_letters_cause_git_annex_get_to_fail/comment_5_9308d5ef037afdaf48cc3378cfa10afb._comment @@ -0,0 +1,79 @@ +[[!comment format=mdwn + username="https://openid.stackexchange.com/user/8a69a637-97cb-41e6-8f45-00f08ba54d6e" + nickname="Andreas Duering" + avatar="http://cdn.libravatar.org/avatar/915cf485815ded2f7f4b68736994c185ecda8c31dd293acd16f5e21ca7db23e1" + subject="comment 5" + date="2017-08-16T17:16:13Z" + content=""" +I think I have a similar problem - especially with the drive letters. I don't need any SMB shares, though. + +### What steps will reproduce the problem? + +1. git init / git annex init on a ext* drive under Linux +2. git sync it to a FAT32-formatted (USB) drive +3. git annex init on the USB drive +4. add each others as remote +5. git sync each other - works fine +6. Change into a Windows system +7. git annex add some files +8. Change back to a Linux system +9. git annex sync from the annex repository on USB +10. git annex sync from the linux-side + +now, if I do ls -l: + + $ ls -l + lrwxrwxrwx 1 me me 208 Aug 16 18:54 version2.win.txt -> G:/annex2/.git/annex/objects/j1/0J/SHA256E-s846--e46ff540d59e80b419798a53d6d97313a5e04b94f9708168a3d371be1ccd635c.win.txt/SHA256E-s846--e46ff540d59e80b419798a53d6d97313a5e04b94f9708168a3d371be1ccd635c.win.txt + +note the g:\, which was the Windows drive letter. + +git annex get worked fine for me: + + $ git annex get version2.win.txt + get version2.win.txt (from usb...) + version2.win.txt + 846 100% 0.00kB/s 0:00:00 (xfr#1, to-chk=0/1) + (checksum...) ok + (recording state in git...) + +However, the link is still broken (same ls -l output) + + $ stat version2.win.txt + File: 'version2.win.txt' -> 'G:/annex2/.git/annex/objects/j1/0J/SHA256E-s846--e46ff540d59e80b419798a53d6d97313a5e04b94f9708168a3d371be1ccd635c.win.txt/SHA256E-s846--e46ff540d59e80b419798a53d6d97313a5e04b94f9708168a3d371be1ccd635c.win.txt' + Size: 208 Blocks: 8 IO Block: 4096 symbolic link + Device: 81bh/2075d Inode: 1200146 Links: 1 + Access: (0777/lrwxrwxrwx) Uid: ( 1000/ me) Gid: ( 1000/ me) + Access: 2017-08-16 18:54:11.479490622 +0200 + Modify: 2017-08-16 18:54:11.479490622 +0200 + Change: 2017-08-16 18:54:11.479490622 +0200 + Birth: - + + $ cat version2.win.txt + cat: version2.win.txt: No such file or directory + +### What version of git-annex are you using? On what operating system? + +Gentoo: + + git-annex version: 6.20170611-gb493ac8d3 + build flags: Assistant Webapp Pairing Testsuite S3(multipartupload)(storageclasses) WebDAV Inotify DBus DesktopNotify ConcurrentOutput TorrentParser MagicMime Feeds Quvi + dependency versions: aws-0.14.1 bloomfilter-2.0.1.0 cryptonite-0.20 DAV-1.3.1 feed-0.3.11.1 ghc-8.0.1 http-client-0.4.31.1 persistent-sqlite-2.6 torrent-10000.0.0 uuid-1.3.12 yesod-1.4.3 + key/value backends: SHA256E SHA256 SHA512E SHA512 SHA224E SHA224 SHA384E SHA384 SHA3_256E SHA3_256 SHA3_512E SHA3_512 SHA3_224E SHA3_224 SHA3_384E SHA3_384 SKEIN256E SKEIN256 SKEIN512E SKEIN512 SHA1E SHA1 MD5E MD5 WORM URL + remote types: git gcrypt p2p S3 bup directory rsync web bittorrent webdav tahoe glacier ddar hook external + local repository version: 5 + supported repository versions: 3 5 6 + upgrade supported from repository versions: 0 1 2 3 4 5 + operating system: linux x86_64 + +Windows 10: + + git-annex version: 6.20170611-gb493ac8 + build flags: Assistant Webapp Pairing Testsuite S3(multipartupload)(storageclasses) WebDAV ConcurrentOutput TorrentParser Feeds Quvi + dependency versions: aws-0.14.0 bloomfilter-2.0.1.0 cryptonite-0.7 DAV-1.3.1 feed-0.3.11.1 ghc-7.10.2 http-client-0.4.31.1 persistent-sqlite-2.2 torrent-10000.0.0 uuid-1.3.12 yesod-1.4.3 + key/value backends: SHA256E SHA256 SHA512E SHA512 SHA224E SHA224 SHA384E SHA384 SHA3_256E SHA3_256 SHA3_512E SHA3_512 SHA3_224E SHA3_224 SHA3_384E SHA3_384 SKEIN256E SKEIN256 SKEIN512E SKEIN512 SHA1E SHA1 MD5E MD5 WORM URL + remote types: git gcrypt p2p S3 bup directory rsync web bittorrent webdav tahoe glacier ddar hook external + local repository version: 5 + supported repository versions: 3 5 6 + upgrade supported from repository versions: 2 3 4 5 + operating system: mingw32 i386 +"""]] diff --git a/doc/bugs/Windows_to_Linux_clone_-_Windows_drive_letters_cause_git_annex_get_to_fail/comment_6_616aebe7392c9f0e41fb33ed6a490d2a._comment b/doc/bugs/Windows_to_Linux_clone_-_Windows_drive_letters_cause_git_annex_get_to_fail/comment_6_616aebe7392c9f0e41fb33ed6a490d2a._comment new file mode 100644 index 0000000000..b751921812 --- /dev/null +++ b/doc/bugs/Windows_to_Linux_clone_-_Windows_drive_letters_cause_git_annex_get_to_fail/comment_6_616aebe7392c9f0e41fb33ed6a490d2a._comment @@ -0,0 +1,9 @@ +[[!comment format=mdwn + username="https://openid.stackexchange.com/user/8a69a637-97cb-41e6-8f45-00f08ba54d6e" + nickname="Andreas Duering" + avatar="http://cdn.libravatar.org/avatar/915cf485815ded2f7f4b68736994c185ecda8c31dd293acd16f5e21ca7db23e1" + subject="comment 6" + date="2017-08-16T18:18:27Z" + content=""" +Adding to the previous comment - a git annex fsck fixes the link +"""]] diff --git a/doc/bugs/Windows_to_Linux_clone_-_Windows_drive_letters_cause_git_annex_get_to_fail/comment_7_6cdc019321e75ee269a4884f303d966d._comment b/doc/bugs/Windows_to_Linux_clone_-_Windows_drive_letters_cause_git_annex_get_to_fail/comment_7_6cdc019321e75ee269a4884f303d966d._comment new file mode 100644 index 0000000000..ec701c2f1b --- /dev/null +++ b/doc/bugs/Windows_to_Linux_clone_-_Windows_drive_letters_cause_git_annex_get_to_fail/comment_7_6cdc019321e75ee269a4884f303d966d._comment @@ -0,0 +1,15 @@ +[[!comment format=mdwn + username="joey" + subject="""comment 7""" + date="2017-10-30T17:07:18Z" + content=""" +@Andreas Duering, your problem was indeed a bug in git-annex; +I think it got introduced somewhere around April 2017, and I just fixed it +last week. + +However, it does not seem related to the original bug report from 2013, which, +despite the title, did not seem to have anything to do with drive letters. + +Since this bug report has languished since 2013 without the submitter +providing any information, I suppose I'll close this bug report. +"""]] diff --git a/doc/bugs/Wrong_backend_extension_in_files_with_multiple_dots.mdwn b/doc/bugs/Wrong_backend_extension_in_files_with_multiple_dots.mdwn new file mode 100644 index 0000000000..b92b32ab42 --- /dev/null +++ b/doc/bugs/Wrong_backend_extension_in_files_with_multiple_dots.mdwn @@ -0,0 +1,39 @@ +### Please describe the problem. +When `git-annex add`ing files with multiple dots in them, the `SHA256E`, `MD5E` (and presumably other `*E`) backends take the extension from the second-to-last dot, instead of the last dot. +This annoyed me because I have some photographs with names like `YYYY-mm-dd HH.MM.SS.jpg`. However, it might be intentional considering a `file.tar.gz` would have `tar.gz`. + +### What steps will reproduce the problem? +[[!format sh """ +$ touch a a.b a.b.c a.b.c.d +$ git-annex add . +add a ok +add a.b ok +add a.b.c ok +add a.b.c.d ok +$ git-annex lookupkey * +SHA256E-s0--e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 +SHA256E-s0--e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855.b +SHA256E-s0--e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855.b.c +SHA256E-s0--e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855.c.d +"""]] + +### What version of git-annex are you using? On what operating system? +[[!format sh """ +$ git-annex version +git-annex version: 6.20170101+gitg93d69b1-1~ndall+1 +build flags: Assistant Webapp Pairing Testsuite S3(multipartupload)(storageclasses) WebDAV Inotify DBus DesktopNotify ConcurrentOutput TorrentParser MagicMime Feeds Quvi +key/value backends: SHA256E SHA256 SHA512E SHA512 SHA224E SHA224 SHA384E SHA384 SHA3_256E SHA3_256 SHA3_512E SHA3_512 SHA3_224E SHA3_224 SHA3_384E SHA3_384 SKEIN256E SKEIN256 SKEIN512E SKEIN512 SHA1E SHA1 MD5E MD5 WORM URL +remote types: git gcrypt p2p S3 bup directory rsync web bittorrent webdav tahoe glacier ddar hook external +local repository version: 5 +supported repository versions: 3 5 6 +upgrade supported from repository versions: 0 1 2 3 4 5 +operating system: linux x86_64 +"""]] + +NeuroDebian's `git-annex-standalone` package on Xubuntu 16.04. (Also with a Debian sid chroot with their own `git-annex 6.20161210-1`) + +### Have you had any luck using git-annex before? (Sometimes we get tired of reading bug reports all day and a lil' positive end note does wonders) + +I'm trying to analyze and organize my huge `Photos` folder into a neat git-annex repository (with yet another [project of mine](https://www.github.com/alpernebbi/albumin)). It's a huge mess. + +Keep up the great work! Also thanks for fixing [my UTF-8 problem](https://git-annex.branchable.com/bugs/Linux_standalone__39__s_metadata_--batch_can__39__t_parse_UTF-8/) as well. diff --git a/doc/bugs/Wrong_backend_extension_in_files_with_multiple_dots/comment_1_e7b8aca1e65affc993572003b0f1b532._comment b/doc/bugs/Wrong_backend_extension_in_files_with_multiple_dots/comment_1_e7b8aca1e65affc993572003b0f1b532._comment new file mode 100644 index 0000000000..a997f73ba4 --- /dev/null +++ b/doc/bugs/Wrong_backend_extension_in_files_with_multiple_dots/comment_1_e7b8aca1e65affc993572003b0f1b532._comment @@ -0,0 +1,27 @@ +[[!comment format=mdwn + username="joey" + subject="""comment 1""" + date="2017-02-20T15:53:29Z" + content=""" +It is indeed to support things like `.tar.gz`, `.pdf.gz` etc. + +There's really no way to distinguish between what's desired to be part of +the extension and not, other than heuristics. The heuristics for this are +fairly narrow. Other than not allowing purely numeric parts in extenssions, +or only taking the last extension, I don't see any change that could help +with this case. + +Bear in mind that the only reason the extension is included at all is +because some strange programs, especially on OSX, follow symlinks and look +at the symlink of the link destination to try to guess what kind of file it +is. There's no good reason for a program to do that, and if you're not +using programs with that problem, you can just use the SHA256 backend and +leave off the extensions. + +Bearing in mind that the extension is part of a big long ugly key name that +you don't generally need to do anything with, including an extra short +possibly not-really-extension part doesn't seem like a usability problem. +The only real problem would be if you had two files with the same content, +but different "extensions", since this would prevent them being +deduplicated to the same key. +"""]] diff --git a/doc/bugs/__34__Adding_4923_files__34___is_really_slow.mdwn b/doc/bugs/__34__Adding_4923_files__34___is_really_slow.mdwn new file mode 100644 index 0000000000..6a0b5ced9f --- /dev/null +++ b/doc/bugs/__34__Adding_4923_files__34___is_really_slow.mdwn @@ -0,0 +1,103 @@ +Wow, what a great archiving system. Thank you for all your work on git annex! + +### Please describe the problem. + +I was using 'git annex assistant' on a brand-new annex that I created today. I had previously added about 20GB of data and a couple thousand files, mostly MP4 videos and MP3 music. + +I then used regular 'mv' to add a folder containing about 20GB of music. This went well for a while—git annex assistant added two groups of files, containing roughly 700 and 1000 MP3s each. But the third group contained 4,923 files, and it's taking a really long time to import. + +CPU usage is pretty consistently near 100%. According to the log, the files are being processed slowly. + +### What steps will reproduce the problem? + +I don't want to try to reproduce this problem until the MP3s finish being imported. I can try again later with thousands of digital photos if that would help. + +### What version of git-annex are you using? On what operating system? + +Version: 4.20130709.1 +Build flags: Assistant Webapp Pairing Testsuite S3 WebDAV Inotify DBus XMPP + +Ubuntu 12.04.2 LTS + +### Please provide any additional information below. + +Here's the 'top' output and a snippet of the log. Let me know if you need anything else. + +[[!format sh """ +# CPU usage + + PID USER PR NI VIRT RES SHR S %CPU %MEM TIME+ COMMAND + 584 ...me... 20 0 776m 147m 16m S 100 0.9 181:16.87 git-annex + + +# If you can, paste a complete transcript of the problem occurring here. +# If the problem is with the git-annex assistant, paste in .git/annex/daemon.log + +[201ok +add music/Pop/Various/Like, Omigod! The 80s Pop Culture Box (totally)/._3-06 Words.mp3 3-0(checksum...) 7-22 14:52:14 EDT] TransferScanner: queued Upload UUID "8dbe75a4-b065-46fd-99f7-22599b2eaf06" music/Rock/Aerosmith/A Little South of Sanity/._2-08 Walk This Way.mp3 Nothing : expensive scan found missing object +[2013-07-22 14:52:34 EDT] Transferrer: Transferring: Upload UUID "8dbe75a4-b065-46fd-99f7-22599b2eaf06" music/Rock/Aerosmith/A Little South of Sanity/._2-08 Walk This Way.mp3 Nothing +[2013-07-22 14:52:34 EDT] chat: git ["--git-dir=/mnt/storage/private/annex/.git","--work-tree=/mnt/storage/private/annex","hash-object","-t","blob","-w","--stdin","--no-filters"] +[ok +add music/Pop/Various/Like, Omigod! The 80s Pop Culture Box (totally)/._4-05 Our House.mp3 2013-0(checksum...) 7-22 14:52:34 EDT] TransferWatcher: transfer starting: Upload UUID "8dbe75a4-b065-46fd-99f7-22599b2eaf06" music/Rock/Aerosmith/A Little South of Sanity/._2-08 Walk This Way.mp3 Nothing +[2013-07-22 14:52:54 EDT] TransferWatcher: transfer finishing: Transfer {transferDirection = Upload, transferUUID = UUID "8dbe75a4-b065-46fd-99f7-22599b2eaf06", transferKey = Key {keyName = "a8ddf79be61cf4a5ab3c7c8e95d8c259ceb102410dff50eb1260e7d818f8c5a8.mp3", keyBackendName = "SHA256E", keySize = Just 70, keyMtime = Nothing}} +[2013-07-22 14:52:54 EDT] chat: git ["--git-dir=/mnt/storage/private/annex/.git","--work-tree=/mnt/storage/private/annex","hash-object","-t","blob","-w","--stdin","--no-filters"] +ok +add music/Pop/Various/Like, Omigod! The 80s Pop Culture Box (totally)/7-15 Never Gonna Give You Up.mp3 (checksum...) [2013-07-22 14:52:54 EDT] read: sha256sum ["/mnt/storage/private/annex/.git/annex/tmp/7-15 Never Gonna Give You Up584.mp3"] +[2013-07-22 14:52:55 EDT] chat: git ["--git-dir=/mnt/storage/private/annex/.git","--work-tree=/mnt/storage/private/annex","hash-object","-t","blob","-w","--stdin","--no-filters"] +[ok +add music/Pop/Various/Like, Omigod! The 80s Pop Culture Box (totally)/3-09 Down Under.mp3 2013-07-(checksum...) 22 14:52:55 EDT] TransferScanner: queued Upload UUID "8dbe75a4-b065-46fd-99f7-22599b2eaf06" music/Rock/Aerosmith/A Little South of Sanity/._2-09 Dude (Looks Like A Lady).mp3 Nothing : expensive scan found missing object +[2013-07-22 14:52:55 EDT] read: sha256sum ["/mnt/storage/private/annex/.git/annex/tmp/3-09 Down Under584.mp3"] +[2013-07-22 14:52:55 EDT] Transferrer: Transferring: Upload UUID "8dbe75a4-b065-46fd-99f7-22599b2eaf06" music/Rock/Aerosmith/A Little South of Sanity/._2-09 Dude (Looks Like A Lady).mp3 Nothing +[2013-07-22 14:52:55 EDT] chat: git ["--git-dir=/mnt/storage/private/annex/.git","--work-tree=/mnt/storage/private/annex","hash-object","-t","blob","-w","--stdin","--no-filters"] +[ok +add music/Pop/Various/Like, Omigod! The 80s Pop Culture Box (totally)/7-19 Right Here Waiting.mp3 2013(checksum...) -07-22 14:52:55 EDT] TransferWatcher: transfer starting: Upload UUID "8dbe75a4-b065-46fd-99f7-22599b2eaf06" music/Rock/Aerosmith/A Little South of Sanity/._2-09 Dude (Looks Like A Lady).mp3 Nothing +[2013-07-22 14:52:55 EDT] read: sha256sum ["/mnt/storage/private/annex/.git/annex/tmp/7-19 Right Here Waiting584.mp3"] +[2013-07-22 14:52:55 EDT] TransferWatcher: transfer finishing: Transfer {transferDirection = Upload, transferUUID = UUID "8dbe75a4-b065-46fd-99f7-22599b2eaf06", transferKey = Key {keyName = "a8ddf79be61cf4a5ab3c7c8e95d8c259ceb102410dff50eb1260e7d818f8c5a8.mp3", keyBackendName = "SHA256E", keySize = Just 70, keyMtime = Nothing}} +[2013-07-22 14:52:55 EDT] chat: git ["--git-dir=/mnt/storage/private/annex/.git","--work-tree=/mnt/storage/private/annex","hash-object","-t","blob","-w","--stdin","--no-filters"] +ok +add music/Pop/Various/Like, Omigod! The 80s Pop Culture Box (totally)/._1-04 Another One Bites The Dust.mp3 (checksum...) [2013-07-22 14:53:15 EDT] chat: git ["--git-dir=/mnt/storage/private/annex/.git","--work-tree=/mnt/storage/private/annex","hash-object","-t","blob","-w","--stdin","--no-filters"] +[2013ok +add music/Pop/Various/Like, Omigod! The 80s Pop Culture Box (totally)/2-08 Hold On Loosely.mp3 -07(checksum...) -22 14:53:15 EDT] TransferScanner: queued Upload UUID "8dbe75a4-b065-46fd-99f7-22599b2eaf06" music/Rock/Aerosmith/A Little South of Sanity/._2-10 What It Takes.mp3 Nothing : expensive scan found missing object +[2013-07-22 14:53:15 EDT] read: sha256sum ["/mnt/storage/private/annex/.git/annex/tmp/2-08 Hold On Loosely584.mp3"] +[2013-07-22 14:53:15 EDT] Transferrer: Transferring: Upload UUID "8dbe75a4-b065-46fd-99f7-22599b2eaf06" music/Rock/Aerosmith/A Little South of Sanity/._2-10 What It Takes.mp3 Nothing +[2013-07-22 14:53:15 EDT] chat: git ["--git-dir=/mnt/storage/private/annex/.git","--work-tree=/mnt/storage/private/annex","hash-object","-t","blob","-w","--stdin","--no-filters"] +[ok +add music/Pop/Various/Like, Omigod! The 80s Pop Culture Box (totally)/._6-11 Kyrie.mp3 201(checksum...) 3-07-22 14:53:15 EDT] TransferWatcher: transfer starting: Upload UUID "8dbe75a4-b065-46fd-99f7-22599b2eaf06" music/Rock/Aerosmith/A Little South of Sanity/._2-10 What It Takes.mp3 Nothing +[2013-07-22 14:53:36 EDT] TransferWatcher: transfer finishing: Transfer {transferDirection = Upload, transferUUID = UUID "8dbe75a4-b065-46fd-99f7-22599b2eaf06", transferKey = Key {keyName = "a8ddf79be61cf4a5ab3c7c8e95d8c259ceb102410dff50eb1260e7d818f8c5a8.mp3", keyBackendName = "SHA256E", keySize = Just 70, keyMtime = Nothing}} +[2013-07-22 14:53:36 EDT] chat: git ["--git-dir=/mnt/storage/private/annex/.git","--work-tree=/mnt/storage/private/annex","hash-object","-t","blob","-w","--stdin","--no-filters"] +ok +add music/Pop/Various/Like, Omigod! The 80s Pop Culture Box (totally)/._5-03 I'm So Excited.mp3 (checksum...) [2013-07-22 14:53:56 EDT] chat: git ["--git-dir=/mnt/storage/private/annex/.git","--work-tree=/mnt/storage/private/annex","hash-object","-t","blob","-w","--stdin","--no-filters"] +[2013ok +add music/Pop/Various/Like, Omigod! The 80s Pop Culture Box (totally)/7-20 Roam.mp3 -07-(checksum...) 22 14:53:56 EDT] TransferScanner: queued Upload UUID "8dbe75a4-b065-46fd-99f7-22599b2eaf06" music/Rock/Aerosmith/A Little South of Sanity/._2-11 Sweet Emotion.mp3 Nothing : expensive scan found missing object +[2013-07-22 14:53:56 EDT] read: sha256sum ["/mnt/storage/private/annex/.git/annex/tmp/7-20 Roam584.mp3"] +[2013-07-22 14:53:57 EDT] Transferrer: Transferring: Upload UUID "8dbe75a4-b065-46fd-99f7-22599b2eaf06" music/Rock/Aerosmith/A Little South of Sanity/._2-11 Sweet Emotion.mp3 Nothing +[2013-07-22 14:53:57 EDT] chat: git ["--git-dir=/mnt/storage/private/annex/.git","--work-tree=/mnt/storage/private/annex","hash-object","-t","blob","-w","--stdin","--no-filters"] +[2ok +add music/Pop/Various/Like, Omigod! The 80s Pop Culture Box (totally)/._3-20 Goodbye To You.mp3 013-07(checksum...) -22 14:53:57 EDT] TransferWatcher: transfer finishing: Transfer {transferDirection = Upload, transferUUID = UUID "8dbe75a4-b065-46fd-99f7-22599b2eaf06", transferKey = Key {keyName = "a8ddf79be61cf4a5ab3c7c8e95d8c259ceb102410dff50eb1260e7d818f8c5a8.mp3", keyBackendName = "SHA256E", keySize = Just 70, keyMtime = Nothing}} +[2013-07-22 14:54:17 EDT] chat: git ["--git-dir=/mnt/storage/private/annex/.git","--work-tree=/mnt/storage/private/annex","hash-object","-t","blob","-w","--stdin","--no-filters"] +ok +add music/Pop/Various/Like, Omigod! The 80s Pop Culture Box (totally)/._7-18 Don't Worry Be Happy.mp3 (checksum...) [2013-07-22 14:54:37 EDT] chat: git ["--git-dir=/mnt/storage/private/annex/.git","--work-tree=/mnt/storage/private/annex","hash-object","-t","blob","-w","--stdin","--no-filters"] +[ok +add music/Pop/Various/Like, Omigod! The 80s Pop Culture Box (totally)/._3-04 Rock This Town.mp3 2013-(checksum...) 07-22 14:54:37 EDT] TransferScanner: queued Upload UUID "8dbe75a4-b065-46fd-99f7-22599b2eaf06" music/Rock/Aerosmith/A Little South of Sanity/1-01 Eat The Rich.mp3 Nothing : expensive scan found missing object +[2013-07-22 14:54:57 EDT] Transferrer: Transferring: Upload UUID "8dbe75a4-b065-46fd-99f7-22599b2eaf06" music/Rock/Aerosmith/A Little South of Sanity/1-01 Eat The Rich.mp3 Nothing +[2013-07-22 14:54:57 EDT] chat: git ["--git-dir=/mnt/storage/private/annex/.git","--work-tree=/mnt/storage/private/annex","hash-object","-t","blob","-w","--stdin","--no-filters"] +ok +add music/Pop/Various/Like, Omigod! The 80s Pop Culture Box (totally)/._7-13 Since You've Been Gone.mp3 [2013-(checksum...) 07-22 14:54:57 EDT] TransferWatcher: transfer finishing: Transfer {transferDirection = Upload, transferUUID = UUID "8dbe75a4-b065-46fd-99f7-22599b2eaf06", transferKey = Key {keyName = "2cb6e7b6ee77f9f98e01e942185265dfe18868503e93d78201485672e6939ab7.mp3", keyBackendName = "SHA256E", keySize = Just 6354105, keyMtime = Nothing}} +[2013-07-22 14:55:18 EDT] chat: git ["--git-dir=/mnt/storage/private/annex/.git","--work-tree=/mnt/storage/private/annex","hash-object","-t","blob","-w","--stdin","--no-filters"] +ok +add music/Pop/Various/Like, Omigod! The 80s Pop Culture Box (totally)/._2-09 Believe It Or Not (Theme From _Greatest American Hero_).mp3 (checksum...) [2013-07-22 14:55:38 EDT] chat: git ["--git-dir=/mnt/storage/private/annex/.git","--work-tree=/mnt/storage/private/annex","hash-object","-t","blob","-w","--stdin","--no-filters"] +[ok +add music/Pop/Various/Like, Omigod! The 80s Pop Culture Box (totally)/7-14 Only In My Dreams.mp3 2013(checksum...) -07-22 14:55:38 EDT] TransferScanner: queued Upload UUID "8dbe75a4-b065-46fd-99f7-22599b2eaf06" music/Rock/Aerosmith/A Little South of Sanity/1-02 Love In An Elevator.mp3 Nothing : expensive scan found missing object +[2013-07-22 14:55:38 EDT] read: sha256sum ["/mnt/storage/private/annex/.git/annex/tmp/7-14 Only In My Dreams584.mp3"] +[2013-07-22 14:55:38 EDT] Transferrer: Transferring: Upload UUID "8dbe75a4-b065-46fd-99f7-22599b2eaf06" music/Rock/Aerosmith/A Little South of Sanity/1-02 Love In An Elevator.mp3 Nothing +[2013-07-22 14:55:38 EDT] chat: git ["--git-dir=/mnt/storage/private/annex/.git","--work-tree=/mnt/storage/private/annex","hash-object","-t","blob","-w","--stdin","--no-filters"] +[2013-ok +add music/Pop/Various/Like, Omigod! The 80s Pop Culture Box (totally)/._4-08 Talking In Your Sleep.mp3 07-2(checksum...) 2 14:55:38 EDT] TransferWatcher: transfer finishing: Transfer {transferDirection = Upload, transferUUID = UUID "8dbe75a4-b065-46fd-99f7-22599b2eaf06", transferKey = Key {keyName = "7cd4b9aefb99f044c5f3b24e9890f45673a63ada3b71e7399e17eb1d710ea0f6.mp3", keyBackendName = "SHA256E", keySize = Just 7181660, keyMtime = Nothing}} + +# End of transcript or log. +"""]] + +[[!meta title="direct mode mappings scale badly with thousands of identical files"]] + +[[!tag confirmed]] +[[!meta tag=deprecateddirectmode]] diff --git a/doc/bugs/__34__Adding_4923_files__34___is_really_slow/comment_2_5f3b9f00bc31ce71d695c008971ed7fd._comment b/doc/bugs/__34__Adding_4923_files__34___is_really_slow/comment_2_5f3b9f00bc31ce71d695c008971ed7fd._comment new file mode 100644 index 0000000000..9030d73f7b --- /dev/null +++ b/doc/bugs/__34__Adding_4923_files__34___is_really_slow/comment_2_5f3b9f00bc31ce71d695c008971ed7fd._comment @@ -0,0 +1,16 @@ +[[!comment format=mdwn + username="http://emk.myopenid.com/" + ip="24.2.150.84" + subject="Interesting speedup" + date="2013-07-22T19:45:58Z" + content=""" +I noticed that there were a lot of junk files with names of the form \"._*\" in the \"music\" directory I had added. These are created by older MacOS X systems storing Mac-related files on Unix file systems. (I think they're supposed to contain the old resource fork or something weird like that.) + +So while the endless import was running, I decided to live dangerous and delete the offending files: + + find music/ -name ._\* -print0 | xargs -0 rm + +A few seconds later, the 4,923 file import finished. + +If I discover anything else interesting, I'll mention it. Once again, many thanks! +"""]] diff --git a/doc/bugs/__34__Adding_4923_files__34___is_really_slow/comment_2_708b02dd06a1eed6b5ded9eb7aa9e7a8._comment b/doc/bugs/__34__Adding_4923_files__34___is_really_slow/comment_2_708b02dd06a1eed6b5ded9eb7aa9e7a8._comment new file mode 100644 index 0000000000..8ad833ca9a --- /dev/null +++ b/doc/bugs/__34__Adding_4923_files__34___is_really_slow/comment_2_708b02dd06a1eed6b5ded9eb7aa9e7a8._comment @@ -0,0 +1,16 @@ +[[!comment format=mdwn + username="http://joeyh.name/" + ip="4.154.1.10" + subject="comment 2" + date="2013-07-22T20:15:03Z" + content=""" +I doubt that removing the junk files did anything, unless perhaps it deleted all the files that it had not yet gotten around to adding, and so short-circuited the slow part of the process. + +Adding a lot of files in a repository in direct mode can be slowed down some because it has to run `git hash-object` once per file to stage the symlink. It may be possible to speed this up by changing the code to write the symlink to disk in a temporary location, which would then allow it to use the single `git hash-object --batch` it keeps running, and just tell it to hash that file. + +It seems likely to me though that the sha256sum it has to run on every file before adding it is responsible for more of the slowdown. After all, that has to read every file from disk (here apparently from `/mnt` which, if it's a USB device etc could be pretty slow.) You can benchmark this at home: First look at the debug log to find the start and finish times of it adding all those files. Then clear all disk caches. Then run sha256sum on every file and time that. Compare & post here. ;) + +Adding to the overhead, the assistant is uploading every file it adds to the remote with uuid 8dbe75a4-b065-46fd-99f7-22599b2eaf06. Which means yet another read of the file, and more work depending on what kind of remote that is and how expensive it is to transfer to it. + +So, in summary, hashing, recording in git, and backing up a lot of files is really slow. It would be good to work out which parts are the slow parts, so we can think about whether that speed is acceptable for that part. +"""]] diff --git a/doc/bugs/__34__Adding_4923_files__34___is_really_slow/comment_3_6a735b7875d2a0c92df6786dd649985d._comment b/doc/bugs/__34__Adding_4923_files__34___is_really_slow/comment_3_6a735b7875d2a0c92df6786dd649985d._comment new file mode 100644 index 0000000000..f95c368d63 --- /dev/null +++ b/doc/bugs/__34__Adding_4923_files__34___is_really_slow/comment_3_6a735b7875d2a0c92df6786dd649985d._comment @@ -0,0 +1,28 @@ +[[!comment format=mdwn + username="http://emk.myopenid.com/" + ip="24.2.150.84" + subject="That particular "Adding 4923 files" was unusually slow" + date="2013-07-23T11:46:03Z" + content=""" +That particular add took most of an afternoon, and didn't complete until I deleted all the junk files. + +I later did \"adding ~10,000 files\" and \"adding ~15,000 files\", all of which ran in an hour or so at most, even though I had added a second remote. So this isn't solely a disk I/O bottleneck. + +One peculiar thing about the garbage files: There were 1,626 of them, and they were all tiny: + +``` +00000000 00 05 16 07 00 02 00 00 00 00 00 00 00 00 00 00 |................| +00000010 00 00 00 00 00 00 00 00 00 01 00 00 00 09 00 00 |................| +00000020 00 26 00 00 00 20 4d 50 47 33 68 6f 6f 6b 00 00 |.&... MPG3hook..| +00000030 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 |................| +* +00000046 +``` + +1,581 of these files had the SHA1 hash code 5460c190ef495baf43e2cd001687be272cd6a9d2, and all but one of the rest had the hash code d4b1d36c67149c981ea4b3d8392050188673817c. So if there's anything weird about this repository, it was thousands of tiny identical files in the same 'git annex assistant' adding pass. Once I scrubbed those files on later imports, things were considerably faster even when the file count increased dramatically. + +You can go ahead and close this bug report if that seems reasonable—I just thought the weird behavior was worth reporting, in case somebody else runs it into again. + +And thank you for an awesome program! + +"""]] diff --git a/doc/bugs/__34__Adding_4923_files__34___is_really_slow/comment_4_7e768908ba6983ea13af27635c4a947f._comment b/doc/bugs/__34__Adding_4923_files__34___is_really_slow/comment_4_7e768908ba6983ea13af27635c4a947f._comment new file mode 100644 index 0000000000..0971f1c342 --- /dev/null +++ b/doc/bugs/__34__Adding_4923_files__34___is_really_slow/comment_4_7e768908ba6983ea13af27635c4a947f._comment @@ -0,0 +1,12 @@ +[[!comment format=mdwn + username="http://joeyh.name/" + ip="4.152.246.110" + subject="comment 4" + date="2013-07-26T22:28:37Z" + content=""" +Did some playing around, and I am seeing a progressive slowdown when adding lots of identical files in direct mode. +This does not happen when the files have different content, and thinking about it, it's pretty clear what's going on: + +When there are a ton of files with the same content, the map file that has to list all the files using this content +get larger and larger. Each file added has to read and re-write the map file, which is obviously not going to scale well to thousands of duplicate files. +"""]] diff --git a/doc/bugs/__34__byte-progress__34___could_jump_down_upon_initiating_re-download_--_report_actual_one_first__63__.mdwn b/doc/bugs/__34__byte-progress__34___could_jump_down_upon_initiating_re-download_--_report_actual_one_first__63__.mdwn new file mode 100644 index 0000000000..6b0cb62a85 --- /dev/null +++ b/doc/bugs/__34__byte-progress__34___could_jump_down_upon_initiating_re-download_--_report_actual_one_first__63__.mdwn @@ -0,0 +1,28 @@ +### Please describe the problem. + +someone kinda could consider it a 'feature' but it complicates use of the output since then progressbar should jump down and some progressbar libraries +do not "support" that + +### What steps will reproduce the problem? + +initiate download, interrupt it, try to redownload it... I guess in some cases redownload doesn't start at the point where it was previously interrupted but somewhat before, or restarts altogether. But annex first reports in --json-progress the size of previously downloaded portion and then goes down. see below + +so, my life would be easier, if annex did not report "unconfirmed initial progress" at all I guess + +### What version of git-annex are you using? On what operating system? + +6.20170307+gitg24ade8a25-1~ndall+1 + +### Please provide any additional information below. + +[[!format sh """ +$> ls -l .git/annex/tmp; k=$(/bin/ls .git/annex/tmp | head -n 1); [ -z "$k" ] || git annex get --json --json-progress --key $k +total 3796 +-rw------- 1 yoh yoh 3887104 Mar 24 17:06 MD5E-s4108657--e055fc250b37b313c0904f3687bbed1c +{"byte-progress":3887104,"action":{"command":"get","note":"from origin...","key":"MD5E-s4108657--e055fc250b37b313c0904f3687bbed1c","file":null},"total-size":4108657,"percent-progress":"94.61%"} +{"byte-progress":2068480,"action":{"command":"get","note":"from origin...","key":"MD5E-s4108657--e055fc250b37b313c0904f3687bbed1c","file":null},"total-size":4108657,"percent-progress":"50.34%"} +{"command":"get","note":"checksum...","success":true,"key":"MD5E-s4108657--e055fc250b37b313c0904f3687bbed1c","file":null} + +"""]] + + diff --git a/doc/bugs/__34__byte-progress__34___could_jump_down_upon_initiating_re-download_--_report_actual_one_first__63__/comment_1_ee95564fafba601246df3de57500eb1c._comment b/doc/bugs/__34__byte-progress__34___could_jump_down_upon_initiating_re-download_--_report_actual_one_first__63__/comment_1_ee95564fafba601246df3de57500eb1c._comment new file mode 100644 index 0000000000..7f87e22b41 --- /dev/null +++ b/doc/bugs/__34__byte-progress__34___could_jump_down_upon_initiating_re-download_--_report_actual_one_first__63__/comment_1_ee95564fafba601246df3de57500eb1c._comment @@ -0,0 +1,29 @@ +[[!comment format=mdwn + username="joey" + subject="""comment 1""" + date="2017-05-25T17:56:48Z" + content=""" +That looks like a git remote accessed perhaps by rsync, or perhaps locally? + +I'd be surprised if a rsync transfer did this, because AFAIK all progress +updates come from rsync's own progress display, and that does not jump +backward. + +Local file copies (when not using rsync), and some other types of remotes, +poll the size of the temp file to determine how much data has been +received, and so if the transfer doesn't resume, they will do this. **I've +made it avoid reporting the file size until the file size has changed once, +which avoids the problem in this case.** + +Another way it could happen is when a transfer fails partway and git-annex +immediately retries and the retry fails to resume. In +this case, the progress would go to some percent for the first transfer, +and then could reset to a lower percent for the retry, and that +reflects what's really happening. Eg, 50% of it transferred and now +we've unfortunately started over at 0%. + +I could make the reported progress always be monotonically increasing, but +then in that retry cases it would just seem to stall, perhaps for a long +period of time. Not sure that's better than a progress display that while +annoying, reflects what's really going on. +"""]] diff --git a/doc/bugs/__34__commitBuffer__58___invalid_argument___40__invalid_character__41____34___during___34__git_annex_sync__34__.mdwn b/doc/bugs/__34__commitBuffer__58___invalid_argument___40__invalid_character__41____34___during___34__git_annex_sync__34__.mdwn new file mode 100644 index 0000000000..d4e3ad4716 --- /dev/null +++ b/doc/bugs/__34__commitBuffer__58___invalid_argument___40__invalid_character__41____34___during___34__git_annex_sync__34__.mdwn @@ -0,0 +1,52 @@ +### Please describe the problem. + +In my unlocked adjusted branch, I get a lot of errors during "git annex sync". It appears to work fine otherwise (the files actually get synced). Below is what I see on the terminal. The repository is otherwise clean (no local or remote changes). +This has started to happen around a month ago, though I cannot pinpoint the exact version. This is in the same repo you used to debug the disappearing files in direct mode recently (thanks a lot btw!). + +### What version of git-annex are you using? On what operating system? + +[[!format sh """ +$ git annex version +git-annex version: 6.20161110-gd48f4ca +build flags: Assistant Webapp Pairing Testsuite S3(multipartupload)(storageclasses) WebDAV Inotify DBus DesktopNotify XMPP ConcurrentOutput TorrentParser MagicMime Feeds Quvi +key/value backends: SHA256E SHA256 SHA512E SHA512 SHA224E SHA224 SHA384E SHA384 SHA3_256E SHA3_256 SHA3_512E SHA3_512 SHA3_224E SHA3_224 SHA3_384E SHA3_384 SKEIN256E SKEIN256 SKEIN512E SKEIN512 SHA1E SHA1 MD5E MD5 WORM URL +remote types: git gcrypt S3 bup directory rsync web bittorrent webdav tahoe glacier ddar hook external +$ lsb_release -a +No LSB modules are available. +Distributor ID: Debian +Description: Debian GNU/Linux 8.6 (jessie) +Release: 8.6 +Codename: jessie +"""]] + +### Please provide any additional information below. + +[[!format sh """ +$ git annex sync --content +commit +On branch adjusted/master(unlocked) +nothing to commit, working tree clean +ok +pull origin +remote: Counting objects: 113, done. +remote: Compressing objects: 100% (113/113), done. +remote: Total 113 (delta 112), reused 0 (delta 0) +Receiving objects: 100% (113/113), 7.16 KiB | 0 bytes/s, done. +Resolving deltas: 100% (112/112), completed with 112 local objects. +From /srv/annex/bilder + 97a4806..78cb4ef git-annex -> origin/git-annex +ok +(merging origin/git-annex into git-annex...) + +git-annex: fd:25: commitBuffer: invalid argument (invalid character) +failed + +git-annex: fd:25: commitBuffer: invalid argument (invalid character) +failed + +[...] + +git-annex: fd:25: commitBuffer: invalid argument (invalid character) +failed +git-annex: sync: 2653 failed +"""]] diff --git a/doc/bugs/__34__error__58___invalid_object__34____44___after_add__59___cannot_commit.mdwn b/doc/bugs/__34__error__58___invalid_object__34____44___after_add__59___cannot_commit.mdwn new file mode 100644 index 0000000000..f6265f05a6 --- /dev/null +++ b/doc/bugs/__34__error__58___invalid_object__34____44___after_add__59___cannot_commit.mdwn @@ -0,0 +1,24 @@ +### Please describe the problem. +After having added new content (SHA1E backend), when trying to commit, git commit fails with the following error: + +[[!format sh """ +(Recording state in git...) +error: invalid object 100644 5d471129a031f0f493de3736eaea6f2f4056aeee for '000/091/WORM-s1493-m1321288671--scrapbook%data%20111114173520%horiz-menu-tab-r_001.png.log' +fatal: git-write-tree: error building trees +git-annex: failed to read sha from git write-tree +"""]] + +The commit subsequently fails and the index is left as is. When I did git-annex add, I got the same error, but the additions seem to have been staged, at least. + +What’s curious about this is that I migrated all keys to SHA1E earlier and dropped all WORM keys. git annex info also says that all my keys are SHA1E. + +Can this be related to your changes to the WORM backend? I upgraded to git-annex 5.20140818 today. Rolling back to 5.20140716 didn’t allow me to commit, either, though. + +Any way I could resolve this? I don’t want to git reset for now, since this will leave the added objects in the annex store. + +### What version of git-annex are you using? On what operating system? +git-annex 5.20140818 + +Linux 3.16.1 + +[[!tag moreinfo]] diff --git a/doc/bugs/__34__error__58___invalid_object__34____44___after_add__59___cannot_commit/comment_10_8abfe7417167df54b686960319465a65._comment b/doc/bugs/__34__error__58___invalid_object__34____44___after_add__59___cannot_commit/comment_10_8abfe7417167df54b686960319465a65._comment new file mode 100644 index 0000000000..a8d501682f --- /dev/null +++ b/doc/bugs/__34__error__58___invalid_object__34____44___after_add__59___cannot_commit/comment_10_8abfe7417167df54b686960319465a65._comment @@ -0,0 +1,8 @@ +[[!comment format=mdwn + username="CandyAngel" + ip="81.111.193.130" + subject="comment 10" + date="2014-09-08T08:08:50Z" + content=""" +Removing .git/annex/index is safe, it is a step in getting git-annex to [forget a commit entirely](http://git-annex.branchable.com/forum/How_to_get_git-annex_to_forget_a_commit__63__). +"""]] diff --git a/doc/bugs/__34__error__58___invalid_object__34____44___after_add__59___cannot_commit/comment_11_7776659e257a97c9a3855c8ad008207a._comment b/doc/bugs/__34__error__58___invalid_object__34____44___after_add__59___cannot_commit/comment_11_7776659e257a97c9a3855c8ad008207a._comment new file mode 100644 index 0000000000..4632a5df3b --- /dev/null +++ b/doc/bugs/__34__error__58___invalid_object__34____44___after_add__59___cannot_commit/comment_11_7776659e257a97c9a3855c8ad008207a._comment @@ -0,0 +1,10 @@ +[[!comment format=mdwn + username="http://joeyh.name/" + ip="209.250.56.54" + subject="comment 11" + date="2014-10-02T16:09:37Z" + content=""" +It seems to me that the problem must be with .git/annex/index. + +I would be interested in looking at this git repository, if there's a way to get a copy (no .git/annex/objects needed). +"""]] diff --git a/doc/bugs/__34__error__58___invalid_object__34____44___after_add__59___cannot_commit/comment_12_e8bbfb4eb26dae31f90a5fa53b4dc948._comment b/doc/bugs/__34__error__58___invalid_object__34____44___after_add__59___cannot_commit/comment_12_e8bbfb4eb26dae31f90a5fa53b4dc948._comment new file mode 100644 index 0000000000..dd6b5c1960 --- /dev/null +++ b/doc/bugs/__34__error__58___invalid_object__34____44___after_add__59___cannot_commit/comment_12_e8bbfb4eb26dae31f90a5fa53b4dc948._comment @@ -0,0 +1,9 @@ +[[!comment format=mdwn + username="joey" + subject="""ping?""" + date="2015-02-10T17:37:06Z" + content=""" +Do you still have this repository? + +Marking this bug moreinfo. +"""]] diff --git a/doc/bugs/__34__error__58___invalid_object__34____44___after_add__59___cannot_commit/comment_13_aee32c53d2198ba1e82acd0c617d517c._comment b/doc/bugs/__34__error__58___invalid_object__34____44___after_add__59___cannot_commit/comment_13_aee32c53d2198ba1e82acd0c617d517c._comment new file mode 100644 index 0000000000..d0ca7f5542 --- /dev/null +++ b/doc/bugs/__34__error__58___invalid_object__34____44___after_add__59___cannot_commit/comment_13_aee32c53d2198ba1e82acd0c617d517c._comment @@ -0,0 +1,10 @@ +[[!comment format=mdwn + username="zenture@51f656b89d89727211c54a15824fd8393ba11a79" + nickname="zenture" + subject="same problem" + date="2015-05-29T08:30:42Z" + content=""" +Hello, I've had the same problem, removing the index fixed the issue. + +I have a backup of it, it's 88MB, how shall I share it with you, Joey? +"""]] diff --git a/doc/bugs/__34__error__58___invalid_object__34____44___after_add__59___cannot_commit/comment_14_f41116524fb7b423ff03c9440b5f278b._comment b/doc/bugs/__34__error__58___invalid_object__34____44___after_add__59___cannot_commit/comment_14_f41116524fb7b423ff03c9440b5f278b._comment new file mode 100644 index 0000000000..fb65220efe --- /dev/null +++ b/doc/bugs/__34__error__58___invalid_object__34____44___after_add__59___cannot_commit/comment_14_f41116524fb7b423ff03c9440b5f278b._comment @@ -0,0 +1,8 @@ +[[!comment format=mdwn + username="konubinix" + avatar="http://cdn.libravatar.org/avatar/72f2376231d98f52c59abd26745174fc" + subject="Same here" + date="2017-01-26T11:19:01Z" + content=""" +Instead of index, I removed index.lck, it looks like it resolved the problem also. +"""]] diff --git a/doc/bugs/__34__error__58___invalid_object__34____44___after_add__59___cannot_commit/comment_15_0f61544accba5002edef2bcd4131e69f._comment b/doc/bugs/__34__error__58___invalid_object__34____44___after_add__59___cannot_commit/comment_15_0f61544accba5002edef2bcd4131e69f._comment new file mode 100644 index 0000000000..3ce7b7bb41 --- /dev/null +++ b/doc/bugs/__34__error__58___invalid_object__34____44___after_add__59___cannot_commit/comment_15_0f61544accba5002edef2bcd4131e69f._comment @@ -0,0 +1,8 @@ +[[!comment format=mdwn + username="konubinix" + avatar="http://cdn.libravatar.org/avatar/72f2376231d98f52c59abd26745174fc" + subject="Same here" + date="2017-01-26T11:19:24Z" + content=""" +Instead of index, I removed index.lck, it looks like it resolved the problem also. +"""]] diff --git a/doc/bugs/__34__error__58___invalid_object__34____44___after_add__59___cannot_commit/comment_16_9dc2f74dc69ccf867ac4e68cc502fda4._comment b/doc/bugs/__34__error__58___invalid_object__34____44___after_add__59___cannot_commit/comment_16_9dc2f74dc69ccf867ac4e68cc502fda4._comment new file mode 100644 index 0000000000..598b75b12a --- /dev/null +++ b/doc/bugs/__34__error__58___invalid_object__34____44___after_add__59___cannot_commit/comment_16_9dc2f74dc69ccf867ac4e68cc502fda4._comment @@ -0,0 +1,8 @@ +[[!comment format=mdwn + username="konubinix" + avatar="http://cdn.libravatar.org/avatar/72f2376231d98f52c59abd26745174fc" + subject="Same here" + date="2017-01-26T11:19:50Z" + content=""" +Instead of index, I removed index.lck, it looks like it resolved the problem also. +"""]] diff --git a/doc/bugs/__34__error__58___invalid_object__34____44___after_add__59___cannot_commit/comment_17_0525b887cd75c183a34de0f486ec022c._comment b/doc/bugs/__34__error__58___invalid_object__34____44___after_add__59___cannot_commit/comment_17_0525b887cd75c183a34de0f486ec022c._comment new file mode 100644 index 0000000000..7c2cf5e36a --- /dev/null +++ b/doc/bugs/__34__error__58___invalid_object__34____44___after_add__59___cannot_commit/comment_17_0525b887cd75c183a34de0f486ec022c._comment @@ -0,0 +1,8 @@ +[[!comment format=mdwn + username="konubinix" + avatar="http://cdn.libravatar.org/avatar/72f2376231d98f52c59abd26745174fc" + subject="Same here" + date="2017-01-26T11:20:12Z" + content=""" +Instead of index, I removed index.lck, it looks like it resolved the problem also. +"""]] diff --git a/doc/bugs/__34__error__58___invalid_object__34____44___after_add__59___cannot_commit/comment_18_fd55979b686b5f83ba2b95300abd1852._comment b/doc/bugs/__34__error__58___invalid_object__34____44___after_add__59___cannot_commit/comment_18_fd55979b686b5f83ba2b95300abd1852._comment new file mode 100644 index 0000000000..c7b2756a08 --- /dev/null +++ b/doc/bugs/__34__error__58___invalid_object__34____44___after_add__59___cannot_commit/comment_18_fd55979b686b5f83ba2b95300abd1852._comment @@ -0,0 +1,8 @@ +[[!comment format=mdwn + username="konubinix" + avatar="http://cdn.libravatar.org/avatar/72f2376231d98f52c59abd26745174fc" + subject="Same here" + date="2017-01-26T11:20:32Z" + content=""" +Instead of index, I removed index.lck, it looks like it resolved the problem also. +"""]] diff --git a/doc/bugs/__34__error__58___invalid_object__34____44___after_add__59___cannot_commit/comment_1_2a64a2da445a64149da7335f35142a08._comment b/doc/bugs/__34__error__58___invalid_object__34____44___after_add__59___cannot_commit/comment_1_2a64a2da445a64149da7335f35142a08._comment new file mode 100644 index 0000000000..af980d1bce --- /dev/null +++ b/doc/bugs/__34__error__58___invalid_object__34____44___after_add__59___cannot_commit/comment_1_2a64a2da445a64149da7335f35142a08._comment @@ -0,0 +1,35 @@ +[[!comment format=mdwn + username="zardoz" + ip="78.48.163.229" + subject="comment 1" + date="2014-08-22T09:27:34Z" + content=""" +git fsck only shows a few dangling blobs from a branch I did earlier and left behind, but otherwise reports no errors. + +git annex fsck --fast ultimately fails with the original error message at some point: + +[[!format sh \"\"\" +# If you can, paste a complete transcript of the problem occurring here. +# If the problem is with the git-annex assistant, paste in .git/annex/daemon.log + +# nx fsck --fast|egrep -v 'ok$' +[2014-08-22 11:14:43 CEST] read: git [\"--git-dir=/home/seb/Webmirror/.git\",\"--work-tree=/home/seb/Webmirror\",\"ls-files\",\"--cached\",\"-z\",\"--\"] +[2014-08-22 11:14:43 CEST] chat: git [\"--git-dir=/home/seb/Webmirror/.git\",\"--work-tree=/home/seb/Webmirror\",\"check-attr\",\"-z\",\"--stdin\",\"annex.backend\",\"annex.numcopies\",\"--\"] +[2014-08-22 11:14:43 CEST] read: git [\"--git-dir=/home/seb/Webmirror/.git\",\"--work-tree=/home/seb/Webmirror\",\"show-ref\",\"git-annex\"] +[2014-08-22 11:14:43 CEST] read: git [\"--git-dir=/home/seb/Webmirror/.git\",\"--work-tree=/home/seb/Webmirror\",\"show-ref\",\"--hash\",\"refs/heads/git-annex\"] +[2014-08-22 11:14:43 CEST] read: git [\"--git-dir=/home/seb/Webmirror/.git\",\"--work-tree=/home/seb/Webmirror\",\"log\",\"refs/heads/git-annex..dda9b068ac5c075e79ab63a531770ad772ae8491\",\"-n1\",\"--pretty=%H\"] +[2014-08-22 11:14:43 CEST] chat: git [\"--git-dir=/home/seb/Webmirror/.git\",\"--work-tree=/home/seb/Webmirror\",\"cat-file\",\"--batch\"] +[2014-08-22 11:25:24 CEST] chat: git [\"--git-dir=/home/seb/Webmirror/.git\",\"--work-tree=/home/seb/Webmirror\",\"hash-object\",\"-w\",\"--stdin-paths\",\"--no-filters\"] +[2014-08-22 11:25:24 CEST] feed: git [\"--git-dir=/home/seb/Webmirror/.git\",\"--work-tree=/home/seb/Webmirror\",\"update-index\",\"-z\",\"--index-info\"] +[2014-08-22 11:25:24 CEST] read: git [\"--git-dir=/home/seb/Webmirror/.git\",\"--work-tree=/home/seb/Webmirror\",\"show-ref\",\"--hash\",\"refs/heads/git-annex\"] +[2014-08-22 11:25:24 CEST] read: git [\"--git-dir=/home/seb/Webmirror/.git\",\"--work-tree=/home/seb/Webmirror\",\"write-tree\"] +error: invalid object 100644 5d471129a031f0f493de3736eaea6f2f4056aeee for '000/091/WORM-s1493-m1321288671--scrapbook%data%20111114173520%horiz-menu-tab-r_001.png.log' +fatal: git-write-tree: error building trees +git-annex: failed to read sha from git write-tree +(Recording state in git...) + +# End of transcript or log. +\"\"\"]] + + +"""]] diff --git a/doc/bugs/__34__error__58___invalid_object__34____44___after_add__59___cannot_commit/comment_2_decb1689b8cc2541077e2d0ae273b5e7._comment b/doc/bugs/__34__error__58___invalid_object__34____44___after_add__59___cannot_commit/comment_2_decb1689b8cc2541077e2d0ae273b5e7._comment new file mode 100644 index 0000000000..b4feb654f0 --- /dev/null +++ b/doc/bugs/__34__error__58___invalid_object__34____44___after_add__59___cannot_commit/comment_2_decb1689b8cc2541077e2d0ae273b5e7._comment @@ -0,0 +1,30 @@ +[[!comment format=mdwn + username="zardoz" + ip="78.48.163.229" + subject="comment 2" + date="2014-08-22T09:38:03Z" + content=""" +git commit with git-annex debug output enabled: + + +[[!format sh \"\"\" +# If you can, paste a complete transcript of the problem occurring here. +# If the problem is with the git-annex assistant, paste in .git/annex/daemon.log + +[2014-08-22 11:36:46 CEST] read: git [\"--git-dir=/home/seb/Webmirror/.git\",\"--work-tree=/home/seb/Webmirror\",\"diff\",\"--cached\",\"--name-only\",\"-z\",\"--diff-filter=ACMRT\",\"--\",\".\"] +[2014-08-22 11:36:46 CEST] read: git [\"--git-dir=/home/seb/Webmirror/.git\",\"--work-tree=/home/seb/Webmirror\",\"diff\",\"--name-only\",\"--diff-filter=T\",\"-z\",\"--cached\",\"--\",\".\"] +[2014-08-22 11:36:46 CEST] read: git [\"--git-dir=/home/seb/Webmirror/.git\",\"--work-tree=/home/seb/Webmirror\",\"symbolic-ref\",\"HEAD\"] +[2014-08-22 11:36:46 CEST] read: git [\"--git-dir=/home/seb/Webmirror/.git\",\"--work-tree=/home/seb/Webmirror\",\"show-ref\",\"refs/heads/master\"] +[2014-08-22 11:36:46 CEST] chat: git [\"--git-dir=/home/seb/Webmirror/.git\",\"--work-tree=/home/seb/Webmirror\",\"hash-object\",\"-w\",\"--stdin-paths\",\"--no-filters\"] +[2014-08-22 11:36:46 CEST] feed: git [\"--git-dir=/home/seb/Webmirror/.git\",\"--work-tree=/home/seb/Webmirror\",\"update-index\",\"-z\",\"--index-info\"] +[2014-08-22 11:36:46 CEST] read: git [\"--git-dir=/home/seb/Webmirror/.git\",\"--work-tree=/home/seb/Webmirror\",\"show-ref\",\"--hash\",\"refs/heads/git-annex\"] +(Recording state in git...) +[2014-08-22 11:36:46 CEST] read: git [\"--git-dir=/home/seb/Webmirror/.git\",\"--work-tree=/home/seb/Webmirror\",\"write-tree\"] +error: invalid object 100644 5d471129a031f0f493de3736eaea6f2f4056aeee for '000/091/WORM-s1493-m1321288671--scrapbook%data%20111114173520%horiz-menu-tab-r_001.png.log' +fatal: git-write-tree: error building trees +git-annex: failed to read sha from git write-tree + +# End of transcript or log. +\"\"\"]] + +"""]] diff --git a/doc/bugs/__34__error__58___invalid_object__34____44___after_add__59___cannot_commit/comment_3_1f6443e495cc16a13e2e4175e73dc8f1._comment b/doc/bugs/__34__error__58___invalid_object__34____44___after_add__59___cannot_commit/comment_3_1f6443e495cc16a13e2e4175e73dc8f1._comment new file mode 100644 index 0000000000..78e7f4138a --- /dev/null +++ b/doc/bugs/__34__error__58___invalid_object__34____44___after_add__59___cannot_commit/comment_3_1f6443e495cc16a13e2e4175e73dc8f1._comment @@ -0,0 +1,27 @@ +[[!comment format=mdwn + username="zardoz" + ip="78.48.163.229" + subject="comment 3" + date="2014-08-22T09:58:05Z" + content=""" +Doing a git annex fsck on a new clone of the repository succeded; the problem must somehow with the .git/annex/index then, I presume? + +I did a git reset to restore to the sane state state before adding, but the problem is that I cannot unannex the files I added. :( + +[[!format sh \"\"\" +nx unannex scrapbook/data/20140822101558/1.jpg +[2014-08-22 11:56:16 CEST] read: git [\"--git-dir=/home/seb/Webmirror/.git\",\"--work-tree=/home/seb/Webmirror\",\"show-ref\",\"--head\"] +[2014-08-22 11:56:16 CEST] read: git [\"--git-dir=/home/seb/Webmirror/.git\",\"--work-tree=/home/seb/Webmirror\",\"diff-index\",\"-z\",\"--raw\",\"--no-renames\",\"-l0\",\"--cached\",\"HEAD\"] +[2014-08-22 11:56:16 CEST] read: git [\"--git-dir=/home/seb/Webmirror/.git\",\"--work-tree=/home/seb/Webmirror\",\"ls-files\",\"--cached\",\"-z\",\"--\",\"scrapbook/data/20140822101558/1.jpg\"] +[2014-08-22 11:56:16 CEST] call: git [\"--git-dir=/home/seb/Webmirror/.git\",\"--work-tree=/home/seb/Webmirror\",\"commit\",\"-q\",\"--allow-empty\",\"--no-verify\",\"-m\",\"content removed from git annex\"] +[2014-08-22 11:56:16 CEST] chat: git [\"--git-dir=/home/seb/Webmirror/.git\",\"--work-tree=/home/seb/Webmirror\",\"hash-object\",\"-w\",\"--stdin-paths\",\"--no-filters\"] +[2014-08-22 11:56:16 CEST] feed: git [\"--git-dir=/home/seb/Webmirror/.git\",\"--work-tree=/home/seb/Webmirror\",\"update-index\",\"-z\",\"--index-info\"] +[2014-08-22 11:56:16 CEST] read: git [\"--git-dir=/home/seb/Webmirror/.git\",\"--work-tree=/home/seb/Webmirror\",\"show-ref\",\"--hash\",\"refs/heads/git-annex\"] +(Recording state in git...) +[2014-08-22 11:56:16 CEST] read: git [\"--git-dir=/home/seb/Webmirror/.git\",\"--work-tree=/home/seb/Webmirror\",\"write-tree\"] +error: invalid object 100644 5d471129a031f0f493de3736eaea6f2f4056aeee for '000/091/WORM-s1493-m1321288671--scrapbook%data%20111114173520%horiz-menu-tab-r_001.png.log' +fatal: git-write-tree: error building trees +git-annex: failed to read sha from git write-tree +\"\"\"]] + +"""]] diff --git a/doc/bugs/__34__error__58___invalid_object__34____44___after_add__59___cannot_commit/comment_4_522020e71393434834def6c80b82e39e._comment b/doc/bugs/__34__error__58___invalid_object__34____44___after_add__59___cannot_commit/comment_4_522020e71393434834def6c80b82e39e._comment new file mode 100644 index 0000000000..bfab0d1be6 --- /dev/null +++ b/doc/bugs/__34__error__58___invalid_object__34____44___after_add__59___cannot_commit/comment_4_522020e71393434834def6c80b82e39e._comment @@ -0,0 +1,16 @@ +[[!comment format=mdwn + username="zardoz" + ip="78.48.163.229" + subject="comment 4" + date="2014-08-22T10:15:51Z" + content=""" +The file referred to in the error message seems to be in good shape: + +[[!format sh \"\"\" +git --no-pager show git-annex:000/091/WORM-s1493-m1321288671--scrapbook%data%20111114173520%horiz-menu-tab-r_001.png.log +1408605730.57892s 0 b25f42de-f4be-4d31-84d1-ab0b71dfec01 +1408562938.526946s 0 e148ea91-0eb6-4f47-86e9-db2136a15279 +\"\"\"]] + +Strangely, the SHA1 of the blob is different from the one reported in the write-tree error. +"""]] diff --git a/doc/bugs/__34__error__58___invalid_object__34____44___after_add__59___cannot_commit/comment_5_cc4cba022869b32d298cdafed9545a34._comment b/doc/bugs/__34__error__58___invalid_object__34____44___after_add__59___cannot_commit/comment_5_cc4cba022869b32d298cdafed9545a34._comment new file mode 100644 index 0000000000..0be1eb958b --- /dev/null +++ b/doc/bugs/__34__error__58___invalid_object__34____44___after_add__59___cannot_commit/comment_5_cc4cba022869b32d298cdafed9545a34._comment @@ -0,0 +1,8 @@ +[[!comment format=mdwn + username="zardoz" + ip="78.48.163.229" + subject="comment 5" + date="2014-08-22T13:07:34Z" + content=""" +I remembered I keep an hourly snapshot regimen and was able to get back the repository from before doing the «add» this morning. Both git fsck and git annex fsck return no errors, and yet, whenever anything is done to the git-annex branch (I tried add and forget), I get the above error. +"""]] diff --git a/doc/bugs/__34__error__58___invalid_object__34____44___after_add__59___cannot_commit/comment_6_e71b251db2ff1f52a40fec40303cdefc._comment b/doc/bugs/__34__error__58___invalid_object__34____44___after_add__59___cannot_commit/comment_6_e71b251db2ff1f52a40fec40303cdefc._comment new file mode 100644 index 0000000000..3fcbf200ba --- /dev/null +++ b/doc/bugs/__34__error__58___invalid_object__34____44___after_add__59___cannot_commit/comment_6_e71b251db2ff1f52a40fec40303cdefc._comment @@ -0,0 +1,15 @@ +[[!comment format=mdwn + username="zardoz" + ip="78.48.163.229" + subject="comment 6" + date="2014-08-22T13:15:06Z" + content=""" +I tried git annex repair on the repo (before doing any adds). It reports no fsck errors, but the repair then dies from a stack overflow. + +[[!format sh \"\"\" +Running git fsck ... +No problems found. +Stack space overflow: current size 8388608 bytes. +Use `+RTS -Ksize -RTS' to increase it. +\"\"\"]] +"""]] diff --git a/doc/bugs/__34__error__58___invalid_object__34____44___after_add__59___cannot_commit/comment_7_0b6413f9ca403be3d83bb3306d1e7f8f._comment b/doc/bugs/__34__error__58___invalid_object__34____44___after_add__59___cannot_commit/comment_7_0b6413f9ca403be3d83bb3306d1e7f8f._comment new file mode 100644 index 0000000000..eb2dea9563 --- /dev/null +++ b/doc/bugs/__34__error__58___invalid_object__34____44___after_add__59___cannot_commit/comment_7_0b6413f9ca403be3d83bb3306d1e7f8f._comment @@ -0,0 +1,8 @@ +[[!comment format=mdwn + username="zardoz" + ip="78.48.163.229" + subject="comment 7" + date="2014-08-22T14:00:42Z" + content=""" +I experimented on my snapshot a bit and found out something odd: When I reset the git-annex branch from dda9b06 to git-annex~1 (4246f73) my local file additions succeed, even though git-annex will fast-forward the branch to dda9b06 again before adding (when merging from origin/git-annex). dda9b06 is a large commit in which I dropped many unused WORM keys from another remote. +"""]] diff --git a/doc/bugs/__34__error__58___invalid_object__34____44___after_add__59___cannot_commit/comment_8_f951981f0bf8cbaecfc46e7b9c903d70._comment b/doc/bugs/__34__error__58___invalid_object__34____44___after_add__59___cannot_commit/comment_8_f951981f0bf8cbaecfc46e7b9c903d70._comment new file mode 100644 index 0000000000..ec876d2ea4 --- /dev/null +++ b/doc/bugs/__34__error__58___invalid_object__34____44___after_add__59___cannot_commit/comment_8_f951981f0bf8cbaecfc46e7b9c903d70._comment @@ -0,0 +1,8 @@ +[[!comment format=mdwn + username="zardoz" + ip="78.48.163.229" + subject="comment 8" + date="2014-08-22T18:57:37Z" + content=""" +I just checked my other large git annex repo and noticed that here too I could no longer add files to the annex. The same observations as above apply. Here too on the tip of the git-anenx branch I had one huge commit in which I dropped the last of the unused WORM keys from another remote. Resetting the git-annex branch to git-annex~1 allowed me to make additions again, even though the reset tip was subsequently merged in again from the remote tracking branch. +"""]] diff --git a/doc/bugs/__34__error__58___invalid_object__34____44___after_add__59___cannot_commit/comment_9_7c98c8b291fdf28a09ccb6c7e4001d5a._comment b/doc/bugs/__34__error__58___invalid_object__34____44___after_add__59___cannot_commit/comment_9_7c98c8b291fdf28a09ccb6c7e4001d5a._comment new file mode 100644 index 0000000000..1a49e449ff --- /dev/null +++ b/doc/bugs/__34__error__58___invalid_object__34____44___after_add__59___cannot_commit/comment_9_7c98c8b291fdf28a09ccb6c7e4001d5a._comment @@ -0,0 +1,22 @@ +[[!comment format=mdwn + username="zardoz" + ip="78.48.163.229" + subject="comment 9" + date="2014-09-07T14:04:51Z" + content=""" +Any ideas? I noticed one alternative way (cf. the reset workaround +above) to make «git annex add» work again is by deleting +.git/annex/index*. Is this safe? + +In both repos, I had not even staged annex additions before the index +was corrupted; the corruption must somehow have been left-over from +earlier actions, altough all previous additions succeeded at the time, +before both repositories mysteriously stopped working (in the context +of backend-migration). + +I still have the original snapshots around if you’d like to debug +this. As noted, «git fsck» succeeds, and all the block-level checksums +check out, so the problem can’t be on the block device or file-system +level. + +"""]] diff --git a/doc/bugs/__34__git-annex__58___direct__58___1_failed__34___on_Windows.mdwn b/doc/bugs/__34__git-annex__58___direct__58___1_failed__34___on_Windows.mdwn new file mode 100644 index 0000000000..d127934fae --- /dev/null +++ b/doc/bugs/__34__git-annex__58___direct__58___1_failed__34___on_Windows.mdwn @@ -0,0 +1,109 @@ +### Please describe the problem. + +When attempting to clone and use a git repository in a subdirectory several levels deep on Windows, I observe symptoms very similar to those described at http://git-annex.branchable.com/direct_mode/#comment-8feee726df4e287dd3751bc77fd1441f. By contrast, when I attempt the same operation in a subdirectory higher up, the operation is successful. Logs of both sessions are given below. + +My suspicion is that this has to do with exceeding the maximum path length limitation (MAX_PATH) of 260 characters on Windows, as described here: http://msdn.microsoft.com/en-us/library/aa365247.aspx. + + +### What steps will reproduce the problem? + +See above. + + +### What version of git-annex are you using? On what operating system? + +>git annex version +git-annex version: 5.20140517-gee56d21 +build flags: Assistant Webapp Webapp-secure Pairing Testsuite S3 WebDAV DNS Feeds Quvi TDFA CryptoHash +key/value backends: SHA256E SHA1E SHA512E SHA224E SHA384E SKEIN256E SKEIN512E SHA256 SHA1 SHA512 SHA224 SHA384 SKEIN256 SKEIN512 WORM URL +remote types: git gcrypt S3 bup directory rsync web webdav tahoe glacier ddar hook external +local repository version: 5 +supported repository version: 5 +upgrade supported from repository versions: 2 3 4 + +>git version +git version 1.9.0.msysgit.0 + +Operating system: Windows 7 Professional (64-bit), Service Pack 1 + + +### Please provide any additional information below. + + +[[!format sh """ +# If you can, paste a complete transcript of the problem occurring here. +# If the problem is with the git-annex assistant, paste in .git/annex/daemon.log +C:\Users\areeves\Documents\Work\MyDirectoryHere\git>git clone ssh://areeves@myserver:/home/work/git/sbv +Cloning into 'sbv'... +remote: Counting objects: 65, done. +remote: Compressing objects: 100% (57/57), done. +remote: Total 65 (delta 26), reused 0 (delta 0) +Receiving objects: 100% (65/65), 9.25 KiB | 0 bytes/s, done. +Resolving deltas: 100% (26/26), done. +Checking connectivity... done. + + +C:\Users\areeves\Documents\Work\MyDirectoryHere\git>cd sbv +C:\Users\areeves\Documents\Work\MyDirectoryHere\git\sbv>git annex get + + Detected a filesystem without fifo support. + + Disabling ssh connection caching. + + Detected a crippled filesystem. + + Enabling direct mode. +git-annex: C:\Users\areeves\Documents\Work\MyDirectoryHere\git\sbv\.git\annex\objects\3de\5f4\SHA256-s765223180--c9e2eebd915b4ade9429b00a7a893df928389b3fb4ab759ea9f00b0e05e18de6\: openTempFile: does not exist (No such file or directory) + + +C:\Users\areeves\Documents\Work\MyDirectoryHere\git\sbv>git annex direct +commit +On branch master +Your branch is up-to-date with 'origin/master'. + +nothing to commit, working directory clean +ok + +git-annex: C:\Users\areeves\Documents\Work\MyDirectoryHere\git\sbv\.git\annex\objects\3de\5f4\SHA256-s765223180--c9e2eebd915b4ade9429b00a7a893df928389b3fb4ab759ea9f00b0e05e18de6\: openTempFile: does not exist (No such file or directory) +failed +git-annex: direct: 1 failed + + +C:\Users\areeves\Documents\Work\MyDirectoryHere\git\sbv>cd c:\temp +c:\temp>git clone ssh://areeves@myserver:/home/work/git/sbv +Cloning into 'sbv'... +remote: Counting objects: 65, done. +remote: Compressing objects: 100% (57/57), done. +remote: Total 65 (delta 26), reused 0 (delta 0) +Receiving objects: 100% (65/65), 9.25 KiB | 0 bytes/s, done. +Resolving deltas: 100% (26/26), done. +Checking connectivity... done. + +c:\temp>cd sbv +c:\temp\sbv>git annex direct + + Detected a filesystem without fifo support. + + Disabling ssh connection caching. + + Detected a crippled filesystem. + + Enabling direct mode. +(Recording state in git...) + + +c:\temp\sbv>git annex get +get BigBinaryFile_Data_Package_2012-03-31.tar.bz2.gpg (merging origin/git-annex into git-annex...) +(Recording state in git...) +sent 30 bytes received 765316741 bytes 11011752.10 bytes/sec +total size is 765223180 speedup is 1.00 +ok +(Recording state in git...) + + +c:\temp\sbv> + +# End of transcript or log. +"""]] + +[[!meta title="window's tiny mind is confused by some long paths used by git-annex"]] diff --git a/doc/bugs/__34__git-annex__58___direct__58___1_failed__34___on_Windows/comment_10_5638e0090598425d67c073b2f04e56d5._comment b/doc/bugs/__34__git-annex__58___direct__58___1_failed__34___on_Windows/comment_10_5638e0090598425d67c073b2f04e56d5._comment new file mode 100644 index 0000000000..f9b689974e --- /dev/null +++ b/doc/bugs/__34__git-annex__58___direct__58___1_failed__34___on_Windows/comment_10_5638e0090598425d67c073b2f04e56d5._comment @@ -0,0 +1,14 @@ +[[!comment format=mdwn + username="davclark" + avatar="http://cdn.libravatar.org/avatar/375c43ce3d79c0955a645e522cdab7af" + subject="Still getting this error" + date="2018-02-02T16:45:50Z" + content=""" +I enabled long filename support (running Windows 10 1709 build 16299.192), did a reboot and I'm still getting this error: + + git-annex: ..\..\.git\annex\objects\ada\ebe\SHA256E-s418503869--4ea1d3209d3199fed9b0e75e97cf299f59f37de2f204da2c3192ce04f69677ae.csv\SHA256E-s418503869--4ea1d3209d3199fed9b0e75e97cf299f59f37de2f204da2c3192ce04f69677ae.csv.cache: openFile: does not exist (No such file or directory) + failed + git-annex: add: 1 failed + +That filename is only 216 characters too... is there any way to diagnose *why* the file creation failed? +"""]] diff --git a/doc/bugs/__34__git-annex__58___direct__58___1_failed__34___on_Windows/comment_1_ce2355485f2610b6a7a79914dcd365be._comment b/doc/bugs/__34__git-annex__58___direct__58___1_failed__34___on_Windows/comment_1_ce2355485f2610b6a7a79914dcd365be._comment new file mode 100644 index 0000000000..127d0603f9 --- /dev/null +++ b/doc/bugs/__34__git-annex__58___direct__58___1_failed__34___on_Windows/comment_1_ce2355485f2610b6a7a79914dcd365be._comment @@ -0,0 +1,17 @@ +[[!comment format=mdwn + username="edward" + subject="another example" + date="2014-12-06T21:01:03Z" + content=""" +I'm having the same problem: + +> git-annex: c:\Users\TV\annex\.git\annex\objects\566\a33\URL--quvi&chttps&c%%www.youtube.com%watch,63v,61XS-kKX9wQk0,38index,615,38list,61PLQ-uHSnFig5NCQkhJfkn8ogXFwzrP4SIf\: openTempFile: does not exist (No such file or directory) +> failed +> git-annex: init: 1 failed + +In my case the filename is slightly shorter, 154 characters, for Aaron the offending filename was 162 characters. + +I think the full filename that git annex is trying to write is 270 characters: + +> c:\Users\TV\annex\.git\annex\objects\566\a33\URL--quvi&chttps&c%%www.youtube.com%watch,63v,61XS-kKX9wQk0,38index,615,38list,61PLQ-uHSnFig5NCQkhJfkn8ogXFwzrP4SIf/URL--quvi&chttps&c%%www.youtube.com%watch,63v,61XS-kKX9wQk0,38index,615,38list,61PLQ-uHSnFig5NCQkhJfkn8ogXFwzrP4SIf +"""]] diff --git a/doc/bugs/__34__git-annex__58___direct__58___1_failed__34___on_Windows/comment_2_dfc398002e2ffbe0b63ce422a1e16d67._comment b/doc/bugs/__34__git-annex__58___direct__58___1_failed__34___on_Windows/comment_2_dfc398002e2ffbe0b63ce422a1e16d67._comment new file mode 100644 index 0000000000..558a1ce23a --- /dev/null +++ b/doc/bugs/__34__git-annex__58___direct__58___1_failed__34___on_Windows/comment_2_dfc398002e2ffbe0b63ce422a1e16d67._comment @@ -0,0 +1,25 @@ +[[!comment format=mdwn + username="http://joeyh.name/" + subject="comment 2" + date="2015-01-06T19:18:26Z" + content=""" +On Linux and OSX, there is a maximum filename size, typically 255 bytes. git-annex always ensures that keys it generates are a maximum of 255 bytes long, no matter the platform. But, in dir/subdir/file, each of the 3 segments of the path is allowed to be that long. The limit on the total path size on Linux is a more reasonable 4096 bytes; OSX has only 1024 bytes. + +I don't know what to do about Windows having such an absurdly small `MAX_PATH` compared to more modern systems. + +The length of just a SHA512 checksum is 128 bytes; that means SHA512 backend cannot be used on windows, at all, since the paths git-annex generates will be at least twice that long, and will easily overflow `PATH_MAX`. I've confirmed this; just adding a file with --backend=SHA512 fails with a \"No such file or directory\" error when it tries to use the path. + +A SHA256 is a more manageable 64 bytes long. So a typical path to such an object will end with eg \".git\annex\objects\566\a33\SHA256E--d728a4c4727febe1c28509482ae1b7b2215798218e544eed7cb7b4dc988f838b\SHA256E--d728a4c4727febe1c28509482ae1b7b2215798218e544eed7cb7b4dc988f838b\" -- 174 bytes long (or a bit longer when there are also extension and size in the key) and leaving only 86 bytes or so for `c:\path\to\repo`. + +Perhaps git-annex should reduce its maximum key size from 255 to 64 bytes, the same as SHA256. Then url keys would work on Windows, except for in deep paths, where git-annex cannot work at all. This would be an easy change. + +git-annex could also avoid using absolute paths, which it currently uses extensively for simplicity (and possiibly robustness against renames of repositories and changes of working directory?), and use relative paths instead. This would probably solve the two examples given in the bug report, and it would make git-annex work better when in a deep path in Windows. It would not make SHA512 work though; with keys that long, the relative path is still too long. (And, it's still possible to get a relative path that has so many '../../' and subdirectories etc that it overflows `PATH_MAX`. It would probably take a really crazy repository directory structure though.) + +The MSDN article has one very interesting bit: + +> The Windows API has many functions that also have Unicode versions to permit an extended-length path for a maximum total path length of 32,767 characters. This type of path is composed of components separated by backslashes, each up to the value returned in the lpMaximumComponentLength parameter of the GetVolumeInformation function (this value is commonly 255 characters). To specify an extended-length path, use the \"\\?\\" prefix. For example, \"\\?\D:\very long path\". + +(It seems that, when using that prefix, `/` is not converted to `\` .. I think git-annex is quite good about getting the slashes the right way round these days.) + +So it might be possible for git-annex to use that prefix and avoid this issue entirely. Haskell's FilePath library does understand that prefix (treats it as part of the drive). Since git-annex always uses the path to the top of the Repo when constructing the problematic FilePaths, I might be able to just change the Repo constructor to add that prefix, and everything follow from that. I tried doing that, unfortunately this makes *git* fail, with \"fatal: relative path syntax cannot be used outside working tree\" when operating on such a repo. Cause git doesn't understand that prefix. +"""]] diff --git a/doc/bugs/__34__git-annex__58___direct__58___1_failed__34___on_Windows/comment_3_1d23e9760782a8d6d2ea2dd5a4c6253a._comment b/doc/bugs/__34__git-annex__58___direct__58___1_failed__34___on_Windows/comment_3_1d23e9760782a8d6d2ea2dd5a4c6253a._comment new file mode 100644 index 0000000000..153c48db23 --- /dev/null +++ b/doc/bugs/__34__git-annex__58___direct__58___1_failed__34___on_Windows/comment_3_1d23e9760782a8d6d2ea2dd5a4c6253a._comment @@ -0,0 +1,9 @@ +[[!comment format=mdwn + username="http://joeyh.name/" + subject="comment 3" + date="2015-01-06T20:51:20Z" + content=""" +I've started a `relativepaths` branch that uses all relative paths to the git repo. After working on it for several hours, there are still 16 test suite failures (update: 10) (update: 1). The potential for uncaught breakage is much higher than I am happy with. (Amoung other problems, git-annex does call setCurrentDirectory in several places, and this utterly breaks the relative paths). + +Using that branch on windows, I am still unable to add files with --backend=SHA512; even relative paths don't make it short enough for such keys. +"""]] diff --git a/doc/bugs/__34__git-annex__58___direct__58___1_failed__34___on_Windows/comment_4_108f3e4449fc9591bcdeb490b486357f._comment b/doc/bugs/__34__git-annex__58___direct__58___1_failed__34___on_Windows/comment_4_108f3e4449fc9591bcdeb490b486357f._comment new file mode 100644 index 0000000000..3aa56f0048 --- /dev/null +++ b/doc/bugs/__34__git-annex__58___direct__58___1_failed__34___on_Windows/comment_4_108f3e4449fc9591bcdeb490b486357f._comment @@ -0,0 +1,9 @@ +[[!comment format=mdwn + username="http://joeyh.name/" + subject="comment 4" + date="2015-01-06T21:59:15Z" + content=""" +Even with relative paths, Edward's example would use a path of 253 characters, and so a slightly longer url would still break it, even with relative paths. + +So, I think reducing url key length needs to be done anyway, and I've done that. Which hardly closes this bug. +"""]] diff --git a/doc/bugs/__34__git-annex__58___direct__58___1_failed__34___on_Windows/comment_5_a1c8ac1d7884d676f05db588b2894603._comment b/doc/bugs/__34__git-annex__58___direct__58___1_failed__34___on_Windows/comment_5_a1c8ac1d7884d676f05db588b2894603._comment new file mode 100644 index 0000000000..54c29af3b5 --- /dev/null +++ b/doc/bugs/__34__git-annex__58___direct__58___1_failed__34___on_Windows/comment_5_a1c8ac1d7884d676f05db588b2894603._comment @@ -0,0 +1,11 @@ +[[!comment format=mdwn + username="http://joeyh.name/" + subject="comment 5" + date="2015-01-07T02:35:46Z" + content=""" +I've beat on the relativepaths branch some more and am probably as confident about it as I'm going to get. Will have to merge it and see what else it breaks. + +Also, I've documented that SHA512 and other large hashes are not recommended if one wants to interop with Windows. + +None of which completely fixes this bug, but short of teaching git about the magic filename prefix to make windows not be so broken, I don't see anything more I can do. +"""]] diff --git a/doc/bugs/__34__git-annex__58___direct__58___1_failed__34___on_Windows/comment_6_739b4fd2156de7570ec71417f41eb188._comment b/doc/bugs/__34__git-annex__58___direct__58___1_failed__34___on_Windows/comment_6_739b4fd2156de7570ec71417f41eb188._comment new file mode 100644 index 0000000000..b20fe2ab98 --- /dev/null +++ b/doc/bugs/__34__git-annex__58___direct__58___1_failed__34___on_Windows/comment_6_739b4fd2156de7570ec71417f41eb188._comment @@ -0,0 +1,23 @@ +[[!comment format=sh + username="https://www.google.com/accounts/o8/id?id=AItOawkDqgw0JLrxLH3GIpg36Mp79F_1pxZxWxU" + nickname="Benjamin" + subject="comment 6" + date="2015-02-04T14:56:36Z" + content=""" +Hi, + +I experienced the same problem. In my case the path to the local clone is \"C:\Users\user\Documents\dev\testplay\studyforrest\". Trying to get a file within that repo (\"c:\Users\user\Documents\dev\testplay\studyforrest\stimulus\task001\annotations>git annex get german_audio_description.csv\") results in \"couldn't find path\". +More precise git annex says: \"git-annex: MoveFileEx \"..\\..\\..\\.git\\annex\\tmp\\SHA256E-s49358--49140697bfd54e0d384b30efb7256c246b99f8c2cd63a48d54078e7d03e26286.csv\" \"..\\..\\..\\.git\\annex\\objects\\885\\a97\\SHA256E-s49358-- +49140697bfd54e0d384b30efb7256c246b99f8c2cd63a48d54078e7d03e26286.csv\\SHA256E-s49358--49140697bfd54e0d384b30efb7256c246b99f8c2cd63a48d54078e7d03e26286.csv\": does not exist (Das System kann den angegeb +enen Pfad nicht finden.) +failed +git-annex: get: 1 failed\" + + +Cloning to c:\studyforrest works. + +Now, I wonder why, since none of the mentioned paths exceeds the limit of 260. But as git annex mentioned it seems to use that relative path \"..\\..\\..\\.git\\annex\ [...]\". May be it internally composes it to something like +\"c:\Users\user\Documents\dev\testplay\studyforrest\stimulus\task001\annotations\..\\..\\..\\.git\\annex\ [...]\" where the part \"stimulus\task001\annotations\..\..\..\\" is actually not needed to adress the desired file. So, if that is the case may be you could give some more room by eliminating this? + + +"""]] diff --git a/doc/bugs/__34__git-annex__58___direct__58___1_failed__34___on_Windows/comment_7_c561d2eb75a2579db620bf7877c98502._comment b/doc/bugs/__34__git-annex__58___direct__58___1_failed__34___on_Windows/comment_7_c561d2eb75a2579db620bf7877c98502._comment new file mode 100644 index 0000000000..6243b9129b --- /dev/null +++ b/doc/bugs/__34__git-annex__58___direct__58___1_failed__34___on_Windows/comment_7_c561d2eb75a2579db620bf7877c98502._comment @@ -0,0 +1,8 @@ +[[!comment format=mdwn + username="https://www.google.com/accounts/o8/id?id=AItOawnx8kHW66N3BqmkVpgtXDlYMvr8TJ5VvfY" + nickname="Yaroslav" + subject="comment 7" + date="2015-02-04T15:53:48Z" + content=""" +It seems that is where where having no read-only key/ directory would also be of help to shorten filename paths. (as discussed in https://github.com/datalad/datalad/issues/32) +"""]] diff --git a/doc/bugs/__34__git-annex__58___direct__58___1_failed__34___on_Windows/comment_8_169423dd4f3292e503b285f088f6701f._comment b/doc/bugs/__34__git-annex__58___direct__58___1_failed__34___on_Windows/comment_8_169423dd4f3292e503b285f088f6701f._comment new file mode 100644 index 0000000000..c88e3b1b5f --- /dev/null +++ b/doc/bugs/__34__git-annex__58___direct__58___1_failed__34___on_Windows/comment_8_169423dd4f3292e503b285f088f6701f._comment @@ -0,0 +1,8 @@ +[[!comment format=mdwn + username="https://www.google.com/accounts/o8/id?id=AItOawkDqgw0JLrxLH3GIpg36Mp79F_1pxZxWxU" + nickname="Benjamin" + subject="comment 8" + date="2015-02-05T13:05:31Z" + content=""" +Despite all things, that may or may not be possible to do to shorten the path lengths used by git-annex, it would be very helpful to properly detect that this problem is occuring and give a reasonable error message. Since windows' \"couldn't find path\"-message doesn't tell you what's going on. +"""]] diff --git a/doc/bugs/__34__git-annex__58___direct__58___1_failed__34___on_Windows/comment_9_c85eff61e8f48f4d58e0e9a11e72e20d._comment b/doc/bugs/__34__git-annex__58___direct__58___1_failed__34___on_Windows/comment_9_c85eff61e8f48f4d58e0e9a11e72e20d._comment new file mode 100644 index 0000000000..38657121c4 --- /dev/null +++ b/doc/bugs/__34__git-annex__58___direct__58___1_failed__34___on_Windows/comment_9_c85eff61e8f48f4d58e0e9a11e72e20d._comment @@ -0,0 +1,14 @@ +[[!comment format=mdwn + username="joey" + subject="""comment 9""" + date="2017-06-06T19:52:31Z" + content=""" +Workaround: Enable long paths in the windows registry. See + + +It would be good to make git-annex enable that automatically, perhaps by +using the manifest file that is described on that page. I don't know +how to make windows use such a manifest file. It seems to have to be +embedded into the exe file. GHC has a open ticket to get it to do that: + +"""]] diff --git a/doc/bugs/__34__invalid_object__34___errors_cropping_up.mdwn b/doc/bugs/__34__invalid_object__34___errors_cropping_up.mdwn new file mode 100644 index 0000000000..5a06bf0920 --- /dev/null +++ b/doc/bugs/__34__invalid_object__34___errors_cropping_up.mdwn @@ -0,0 +1,91 @@ +### Please describe the problem. +In my repo network, multiple repos are cropping up with an "invalid object" error. The object in question changes and appears to be from git-annex's hidden branch. + +### What steps will reproduce the problem? +This is happening after I add a mess of new files and sync, within a couple times of doing this, after a day or so. + +### What version of git-annex are you using? On what operating system? +6.20160518-g766728c on linux + +### Please provide any additional information below. + +[[!format sh """ +# Example error, file in question will change + (recording state in git...) + error: invalid object 100644 d547f60ac6c53f8bf38999d93ce954e3dfca1656 for + '051/0b4/SHA3_512-s12447441634--b84ff2ead694b9d355d8deb4eae620b5979f0127f127718d53d02cd32f1020fba45f1e63723484462beb10110328dab29d875ea1788f949ce541640603fa73c0.log' + fatal: git-write-tree: error building trees + git-annex: failed to read sha from git write-tree +# End of transcript or log. + +# log of attempting to determine the issue yesterday +2016-05-22 + +Issue history: +Original annex started being unable to finish commit. +Second annex, I cloned and copied objects folder over. Soon, same issue. +Third annex, cloned from elsewhere, use git annex get to get objects from +broken repo. Same issue developed. +Fourth annex, cloned from elsewhere, was able to commit successfully +prior to adding objects from broken repo. +Some object in the broken repo seems to be causing this issue somehow. + +Plan to resolve: +- copy all objects over again. verify that issue immediately develops. + + I expect the issue will trigger when I add a set of objects with litelog. + So I could narrow it either by narrowing the objects I add with litelog, + or by narrowing the object I copy from the broken repo. + + I'll narrow the issue between the two, first. + +annex: used to be annex.2, busted +annex.3: third repository, busted, holds objects +annex.4: + 1. cloned from delta + 2. added logs, no issue + 3. 10:24:24 EDT rsync --delete-during'd objects in from annex.3 + 4. 10:25:10 EDT added small test file, sync'd with delta fine + 5. 2016-05-22 10:27:38 + I ran fetchall but ran by root by mistake. + files were added with wrong permissions + fixed permissions. only two files were cpied, voicerecorder and muse. + sync works fine.' + 6. ran fetchall (as root) again, got 4 sensorium files + 7. add and sync seems to work fine. + So now this repository is functioning with additional objects copied into + its annex folder. It must have been something else which caused the issue + on the other repo I initialized this way. + 8. I foolishly tried to git annex add the alphabetically first object from + the annex/objects folder to get it annex's logs. It failed when trying to + _delete_ it to replace it with a link to itself, luckily it was not writable. + + I'll try using this repo - 2016-05-22 10:36:24 EDT + + I'll add the remotes one at a time and sync with them, adding litelog after + each one. + + 9. 2016-05-23 00:08 EDT + Issue reoccurred uploading data from phone. Was a larger block of data. + First fetched + then added + then copied to delta + issue showed at end of copy to delta + + perhaps the issue resides on delta ??? but we synced to delta fine before + the issue must happen over time while I am away from the computer + or respond to a large amount of logs being uploaded + or it could have happened as a result of my work on the nested repo + 10. 2016-05-23 00:10:15 EDT + $ git annex copy --to=gitlab + 06:55 EDT + $ git annex sync gitlab + both commands ran without issue + delta also fsck'd fine + 11. 2016-05-23 07:01 EDT + local repo fsck'd fine +"""]] + +### Have you had any luck using git-annex before? (Sometimes we get tired of reading bug reports all day and a lil' positive end note does wonders) +Very stable. Still excited. + diff --git a/doc/bugs/__34__invalid_object__34___errors_cropping_up/comment_1_d146a64ef8d76c2a7e45bf85d8943456._comment b/doc/bugs/__34__invalid_object__34___errors_cropping_up/comment_1_d146a64ef8d76c2a7e45bf85d8943456._comment new file mode 100644 index 0000000000..6c7b8e2689 --- /dev/null +++ b/doc/bugs/__34__invalid_object__34___errors_cropping_up/comment_1_d146a64ef8d76c2a7e45bf85d8943456._comment @@ -0,0 +1,17 @@ +[[!comment format=mdwn + username="joey" + subject="""comment 1""" + date="2016-05-23T16:52:23Z" + content=""" +Try running `git fsck` and see if it complains about a corrupt object file. +You may then need to delete the object file and fix up the repository to +not refer to it. `git annex repair` can probably handle that. + +> Some object in the broken repo seems to be causing this issue somehow. + +Maybe, but you also say that the object it complains about has varied. +Perhaps you have multiple corrupt objects in the git repo.. + +You mention "litelog". What is that and how is it relevant? Is Android +involved somehow? +"""]] diff --git a/doc/bugs/__34__invalid_object__34___errors_cropping_up/comment_2_6585b15aa7ae63175482d08b2b5b79fc._comment b/doc/bugs/__34__invalid_object__34___errors_cropping_up/comment_2_6585b15aa7ae63175482d08b2b5b79fc._comment new file mode 100644 index 0000000000..72417ff8b8 --- /dev/null +++ b/doc/bugs/__34__invalid_object__34___errors_cropping_up/comment_2_6585b15aa7ae63175482d08b2b5b79fc._comment @@ -0,0 +1,11 @@ +[[!comment format=mdwn + username="xloem" + subject="comment 2" + date="2016-05-29T02:02:38Z" + content=""" +This has occurred for me yet again, after starting a new remote. + +Strangely 'git fsck' succeeds, showing only dangling objects, but 'git annex sync' fails on commit with this error. + +litelog is a set of service scripts I'm making which automatically record and log from devices when they are connected. Voice recorder and sensor logs are copied off of android phones from a handful of supported apps. +"""]] diff --git a/doc/bugs/__34__invalid_object__34___errors_cropping_up/comment_3_973d90a6fde1c6f26fcf594610693fad._comment b/doc/bugs/__34__invalid_object__34___errors_cropping_up/comment_3_973d90a6fde1c6f26fcf594610693fad._comment new file mode 100644 index 0000000000..92ae9f2d55 --- /dev/null +++ b/doc/bugs/__34__invalid_object__34___errors_cropping_up/comment_3_973d90a6fde1c6f26fcf594610693fad._comment @@ -0,0 +1,17 @@ +[[!comment format=mdwn + username="joey" + subject="""comment 3""" + date="2016-05-31T15:59:49Z" + content=""" +It may be that .git/annex/index is referring to an object that is somehow +not present in the git object store. In this case, deleting the index file +may recover. + +I can't see any way this can normally happen (short of the object file +getting deleted somehow); git should always write +objects to the object store before their sha1s are available to be put in +an index file. But, a couple of people have reported something like this +before. + +Any clues to reproducing it would be useful. +"""]] diff --git a/doc/bugs/__34__metadata_only__34___git-remote-gcrypt_syncing_files_anyway.mdwn b/doc/bugs/__34__metadata_only__34___git-remote-gcrypt_syncing_files_anyway.mdwn new file mode 100644 index 0000000000..a318692b4e --- /dev/null +++ b/doc/bugs/__34__metadata_only__34___git-remote-gcrypt_syncing_files_anyway.mdwn @@ -0,0 +1,69 @@ +### Please describe the problem. +The git-annex assistant queues files for transfer to gcrypt git (not SSH) remotes, which shouldn't happen. `git-annex sync gcrypt` doesn't. +I think the problem might be that the assistant triggers a content sync, despite showing "metadata only", and that sync --content circumvents gcrypt. + +I have a gcrypt remote set up with github, but I can reprodue this issue using a directory as the destination as well. + +When I set up the remote and sync it, it works as it should: + + ~/annex (git)-[annex/direct/master] % git remote add gcrypt gcrypt::/tmp/test + ~/annex (git)-[annex/direct/master] % git-annex sync gcrypt + commit (recording state in git...) + ok + pull gcrypt + gcrypt: Development version -- Repository format MAY CHANGE + gcrypt: Repository not found: /tmp/test + ok + push gcrypt + gcrypt: Development version -- Repository format MAY CHANGE + gcrypt: Repository not found: /tmp/test + gcrypt: Setting up new repository + gcrypt: Remote ID is :id:thetwentycharacterid + Counting objects: 469459, done. + Compressing objects: 100% (122643/122643), done. + Total 469459 (delta 342415), reused 468156 (delta 341616) + gcrypt: Encrypting to: -r fngerprintremoved + gcrypt: Requesting manifest signature + To gcrypt::/tmp/test + * [new branch] git-annex -> synced/git-annex + * [new branch] annex/direct/master -> synced/master + ok + +But when I launch the assistant/webapp, it starts queuing and syncing file contents, even though the remote is listed as "metadata only". + +After letting the assistant run for a minute: + + ~ % ls /tmp/test/annex/objects/ + 000 043 091 0ed 130 18e 1d8 21c 26c 2b2 2f4 334 371 3ad 3e3 439 471 4b4 525 565 5c3 61b 691 724 788 ... + +*And the files aren't encrypted!* + + find /tmp/test/annex/objects/ -type f -exec file {} \; | head -n10 + /tmp/test/annex/objects/247/100/SHA256E-s5310--06c62006efde5abd7d03dbb15e3725982c80c0eaffde90e3b566fab26d810d6d.opf/SHA256E-s5310--06c62006efde5abd7d03dbb15e3725982c80c0eaffde90e3b566fab26d810d6d.opf: XML 1.0 document text + /tmp/test/annex/objects/5f8/851/SHA256E-s36705--a3b71efc462876709de6c95a7b21218fe437fe45ed39cf5da2709be546c360bc.jpg/SHA256E-s36705--a3b71efc462876709de6c95a7b21218fe437fe45ed39cf5da2709be546c360bc.jpg: JPEG image data, JFIF standard 1.01, aspect ratio, density 1x1, segment length 16, baseline, precision 8, 310x500, frames 3 + /tmp/test/annex/objects/f09/b22/SHA256E-s358928--7e7680e24baf28a82b94cf9931d73a8353ae5c009d81cb4d8c6e313bdc0b22e0/SHA256E-s358928--7e7680e24baf28a82b94cf9931d73a8353ae5c009d81cb4d8c6e313bdc0b22e0: EPUB document + /tmp/test/annex/objects/512/38c/SHA256E-s4246--f7dda72a07aa9f11e329e180c873ba1edd45ac80aa270e379ac52e09302ccfa0.opf/SHA256E-s4246--f7dda72a07aa9f11e329e180c873ba1edd45ac80aa270e379ac52e09302ccfa0.opf: XML 1.0 document text + /tmp/test/annex/objects/079/b2d/SHA256E-s496807--d5ba2b9a564a199ea33da246d70f43c8f531c53745130c4ae82ac8b5b5180684.epub/SHA256E-s496807--d5ba2b9a564a199ea33da246d70f43c8f531c53745130c4ae82ac8b5b5180684.epub: EPUB document + /tmp/test/annex/objects/6e3/e05/SHA256E-s5145--07a4770b8d1c10e46834b895484c20fca7f7e0850a51f8eb1f7c91f175ab2f8a.opf/SHA256E-s5145--07a4770b8d1c10e46834b895484c20fca7f7e0850a51f8eb1f7c91f175ab2f8a.opf: XML 1.0 document text + /tmp/test/annex/objects/b65/708/SHA256E-s271061--dc839391472cf5f08edc2807f7dd016a1ce4f5e3113a964882585fd6dcc51ce2.epub/SHA256E-s271061--dc839391472cf5f08edc2807f7dd016a1ce4f5e3113a964882585fd6dcc51ce2.epub: EPUB document + /tmp/test/annex/objects/9a4/9d3/SHA256E-s38917--5a1a770c758f2f9b254ad7f2f6b6e33b87f14486322d04b5d25b09569772b9e1.jpg/SHA256E-s38917--5a1a770c758f2f9b254ad7f2f6b6e33b87f14486322d04b5d25b09569772b9e1.jpg: JPEG image data, JFIF standard 1.01, aspect ratio, density 1x1, segment length 16, baseline, precision 8, 305x500, frames 3 + /tmp/test/annex/objects/4ec/b23/SHA256E-s307073--cbdef5840ad0addde500d2bfbb5e916c71dea6394f647f6663771abdb05a1776/SHA256E-s307073--cbdef5840ad0addde500d2bfbb5e916c71dea6394f647f6663771abdb05a1776: EPUB document + /tmp/test/annex/objects/074/106/SHA256E-s307165--9a379564df17b9e1a0b7a2221e81180db447a05a9f1123b52fe2a618462a922e.epub/SHA256E-s307165--9a379564df17b9e1a0b7a2221e81180db447a05a9f1123b52fe2a618462a922e.epub: EPUB document + +The above files are part of my Calibre library. + +For my github repo, the assistant still shows files queuing but they don't actually show up in the repo. I think it's trying to sync using SSH but failing. + +### What steps will reproduce the problem? +1. Create a git-remote-gcrypt remote on a git server or in a local directory +2. Open the git-annex webapp, ensuring that syncing is enabled + +### What version of git-annex are you using? On what operating system? + + ~ % git-annex version + git-annex version: 5.20150519-g87f28bb + build flags: Assistant Webapp Webapp-secure Pairing Testsuite S3 Inotify DBus DesktopNotify DNS Feeds Quvi TDFA TorrentParser + key/value backends: SHA256E SHA1E SHA512E SHA224E SHA384E SKEIN256E SKEIN512E MD5E SHA256 SHA1 SHA512 SHA224 SHA384 SKEIN256 SKEIN512 MD5 WORM URL + remote types: git gcrypt S3 bup directory rsync web bittorrent glacier ddar hook external + +I'm running Arch Linux diff --git a/doc/bugs/__34__metadata_only__34___git-remote-gcrypt_syncing_files_anyway/comment_1_30a0a715fb9d8738e5ec53296375c1b3._comment b/doc/bugs/__34__metadata_only__34___git-remote-gcrypt_syncing_files_anyway/comment_1_30a0a715fb9d8738e5ec53296375c1b3._comment new file mode 100644 index 0000000000..515079a79c --- /dev/null +++ b/doc/bugs/__34__metadata_only__34___git-remote-gcrypt_syncing_files_anyway/comment_1_30a0a715fb9d8738e5ec53296375c1b3._comment @@ -0,0 +1,8 @@ +[[!comment format=mdwn + username="chris@f4ea67aa5ae4709d79959f782fcebb5edae9a79b" + nickname="chris" + subject="similar problem" + date="2016-06-02T16:28:18Z" + content=""" +For me it's a version 6 repository, but I also get lots of queued transfers to my \"metadata only\" git-remote-gcrypt repo when I launch the webapp. Luckily they don't actually seem to turn up amongst the remote repo's objects (unless I'm missing something), but it causes the assistant to be busy being useless for a while. +"""]] diff --git a/doc/bugs/__34__metadata_only__34___git-remote-gcrypt_syncing_files_anyway/comment_2_b91f9337b7ed536539bb3236b7552d82._comment b/doc/bugs/__34__metadata_only__34___git-remote-gcrypt_syncing_files_anyway/comment_2_b91f9337b7ed536539bb3236b7552d82._comment new file mode 100644 index 0000000000..db88d6b5d4 --- /dev/null +++ b/doc/bugs/__34__metadata_only__34___git-remote-gcrypt_syncing_files_anyway/comment_2_b91f9337b7ed536539bb3236b7552d82._comment @@ -0,0 +1,8 @@ +[[!comment format=mdwn + username="chris@f4ea67aa5ae4709d79959f782fcebb5edae9a79b" + nickname="chris" + subject="comment 2" + date="2016-06-07T13:40:24Z" + content=""" +The way I got round this was to give the metadata-only repository an annex-uuid of 00000000-0000-0000-0000-000000000000, and to run \"git annex wanted remotename nothing\". Now everything works as I want, whether with the assistant or when I run git annex sync --content [--all] manually. +"""]] diff --git a/doc/bugs/__34__metadata_only__34___git-remote-gcrypt_syncing_files_anyway/comment_3_79268506a1653220ddfbb45f9c61d8a7._comment b/doc/bugs/__34__metadata_only__34___git-remote-gcrypt_syncing_files_anyway/comment_3_79268506a1653220ddfbb45f9c61d8a7._comment new file mode 100644 index 0000000000..e24cb210b9 --- /dev/null +++ b/doc/bugs/__34__metadata_only__34___git-remote-gcrypt_syncing_files_anyway/comment_3_79268506a1653220ddfbb45f9c61d8a7._comment @@ -0,0 +1,19 @@ +[[!comment format=mdwn + username="joey" + subject="""comment 3""" + date="2016-06-09T20:27:07Z" + content=""" +This was fixed in [[!commit fbf5045d4f17accde9e20fa528e52cb1dce61c47]] +for `git annex sync --content` + +I don't remember the immediate cause of it being in a code that that the +webapp would call, but I did add a belt-and-suspenders fix at a lower level +which I'd hope would prevent the webapp from uploading anything in any +case. + +Sounds like the webapp tries to queue transfers to a NoUUID remote, and +then presumably gives up before the object gets uploaded. + +Looking at the code, calcSyncRemotes does not filter out NoUUID remotes +when populating syncDataRemotes. So, I've fixed that too now. +"""]] diff --git a/doc/bugs/__34__sha256sum_failed__34___for_some_files_with_newest_Android_client.mdwn b/doc/bugs/__34__sha256sum_failed__34___for_some_files_with_newest_Android_client.mdwn new file mode 100644 index 0000000000..87e91d9a3a --- /dev/null +++ b/doc/bugs/__34__sha256sum_failed__34___for_some_files_with_newest_Android_client.mdwn @@ -0,0 +1,57 @@ +### Please describe the problem. +git annex fsck and git annex get are showing (false?) sha256sum failed messages on Android with the 2016-07-19 android 5.0 build. I haven't seen this before but have been using git-annex on Android, with the same repository for a year, so I'd guess regression. + +I ran fsck on the files on another machine (Watt: Debian testing, 6.20160511-1) and no errors occur. Note that isn't the machine that the Android tablet transfers from (Einstein: Debian stable + backports, 5.20151208-1~bpo8+1). The Android shell didn't have a sha256sum command, but it had an md5 command—and I tried one of the files (Management Report.pdf) and the md5 matches on both Watt and the Android tablet. + +I also ran git annex fsck on Einstein (the entire repository, since it's a bare repo) and the only problem it found was a single (different) key with an insufficient number of trusted copies. + +So I think the file is actually fine. + +### What steps will reproduce the problem? +Unsure. Dropping and re-getting those files didn't help; it showed a sha256 error with get too. + +The only difference I can think of between the files that fail and work is the size: + +[[!format text """ +u0_a180@manta:/sdcard/Westerley-Board/Board Packets & Reports/2016-08-25 (Regular) $ ls -l +-rw-rw---- root sdcard_r 116225 2016-08-24 05:40 Agenda 16-08-25.pdf +-rw-rw---- root sdcard_r 10128521 2016-08-24 06:02 Management Report scan 2 (annotated; Anthony).pdf +-rw-rw---- root sdcard_r 10128521 2016-08-24 06:02 Management Report scan 2.pdf +-rw-rw---- root sdcard_r 53313352 2016-08-24 06:02 Management Report.pdf +-rw-rw---- root sdcard_r 27154 2016-08-24 06:02 WHOA Chart of Accounts Spreadsheet.xlsx +"""]] + +### What version of git-annex are you using? On what operating system? + + +### Please provide any additional information below. + +[[!format text """ +u0_a180@manta:/sdcard/Westerley-Board/Board Packets & Reports/2016-08-25 (Regular) $ git annex fsck * +WARNING: linker: git-annex has text relocations. This is wasting memory and prevents security hardening. Please fix. +fsck Agenda 16-08-25.pdf (checksum...) ok +fsck Management Report scan 2 (annotated; Anthony).pdf (checksum...) +sha256sum failed +ok +fsck Management Report scan 2.pdf (checksum...) +sha256sum failed +ok +fsck Management Report.pdf (checksum...) +sha256sum failed +ok +fsck WHOA Chart of Accounts Spreadsheet.xlsx (checksum...) ok +(recording state in git...) +__bionic_open_tzdata_path: ANDROID_ROOT not set! +__bionic_open_tzdata_path: ANDROID_ROOT not set! +__bionic_open_tzdata_path: ANDROID_ROOT not set! +__bionic_open_tzdata_path: ANDROID_ROOT not set! +__bionic_open_tzdata_path: ANDROID_ROOT not set! +__bionic_open_tzdata_path: ANDROID_ROOT not set! +u0_a180@manta:/sdcard/Westerley-Board/Board Packets & Reports/2016-08-25 (Regular) $ +"""]] + +### Have you had any luck using git-annex before? (Sometimes we get tired of reading bug reports all day and a lil' positive end note does wonders) + +Yes, I've been using git annex for I think a year and a half now, on several repositories. It works pretty well. I have a total of around 315GB and 23K annexed keys across them (counting each annex only once, even though they're cloned on a bunch of machines). + +> Closing as this was a bug in the deprecated Android app. [[done]] --[[Joey]] diff --git a/doc/bugs/__34__sha256sum_failed__34___for_some_files_with_newest_Android_client/comment_1_f5643bafe724c98a6dc810adb2ea931a._comment b/doc/bugs/__34__sha256sum_failed__34___for_some_files_with_newest_Android_client/comment_1_f5643bafe724c98a6dc810adb2ea931a._comment new file mode 100644 index 0000000000..7fe44973f5 --- /dev/null +++ b/doc/bugs/__34__sha256sum_failed__34___for_some_files_with_newest_Android_client/comment_1_f5643bafe724c98a6dc810adb2ea931a._comment @@ -0,0 +1,24 @@ +[[!comment format=mdwn + username="joey" + subject="""comment 1""" + date="2016-09-05T17:19:43Z" + content=""" +That warning message indicates that the `sha256sum` command is exiting nonzero. +git-annex handles that failure by using its internal SHA, which apparently +succeeds, because the `fsck` output ends with "ok". Indeed, I don't see any +indication that this is causing any problems, other than a warning +message. + +The size variation is due to git-annex only using `sha256sum` for +larger files, where it can be faster than the internal SHA. + +Android is supposed to have `sha256sum` and `sha1sum` available +(but not some of the other sizes). They are included in the git-annex +bundle, in eg /data/data/ga.androidterm/bin/ along with lots of other +busybox utilities. + +So, the problem seems to be that either those commands are not in your +android device somehow, or indeed a reversion in the git-annex Android +build has lost them, or perhaps they're included but are always failing to +work. +"""]] diff --git a/doc/bugs/__34__sha256sum_failed__34___for_some_files_with_newest_Android_client/comment_2_9a8c76e6043d30e77c7aa2513d5af45a._comment b/doc/bugs/__34__sha256sum_failed__34___for_some_files_with_newest_Android_client/comment_2_9a8c76e6043d30e77c7aa2513d5af45a._comment new file mode 100644 index 0000000000..4ea42c1509 --- /dev/null +++ b/doc/bugs/__34__sha256sum_failed__34___for_some_files_with_newest_Android_client/comment_2_9a8c76e6043d30e77c7aa2513d5af45a._comment @@ -0,0 +1,10 @@ +[[!comment format=mdwn + username="https://openid.stackexchange.com/user/3ee5cf54-f022-4a71-8666-3c2b5ee231dd" + nickname="Anthony DeRobertis" + subject="comment 2" + date="2016-09-09T06:51:45Z" + content=""" +`sha256sum` isn't available on the tablet—at least not in the git-annex shell's PATH. + +I tried checking if it's present in the apk, and I don't see it in there—but I'm also not sure exactly where in the jar file I should find it. Unfortunately, I couldn't find an older Android git-annex build to check—seems the download site only keeps the most recent. +"""]] diff --git a/doc/bugs/__34__sha256sum_failed__34___for_some_files_with_newest_Android_client/comment_3_82369bb5a705dbf24b6511b864c98a73._comment b/doc/bugs/__34__sha256sum_failed__34___for_some_files_with_newest_Android_client/comment_3_82369bb5a705dbf24b6511b864c98a73._comment new file mode 100644 index 0000000000..b078e5aa29 --- /dev/null +++ b/doc/bugs/__34__sha256sum_failed__34___for_some_files_with_newest_Android_client/comment_3_82369bb5a705dbf24b6511b864c98a73._comment @@ -0,0 +1,14 @@ +[[!comment format=mdwn + username="joey" + subject="""comment 3""" + date="2018-05-08T18:47:38Z" + content=""" +I'm closing this bug report because the git-annex Android app that it +was reported on is now deprecated. Instead, we have +a way to run the regular git-annex build for linux on Android in termux: + + +There were a lot of problems with the way the git-annex Android app was +put together, and I hope this new approach avoids them. If you try it and +still have the bug you reported, please followup and I'll reopen it. +"""]] diff --git a/doc/bugs/__39__add__39___results_in_max_cpu__44___long_run_and_huge_repo.mdwn b/doc/bugs/__39__add__39___results_in_max_cpu__44___long_run_and_huge_repo.mdwn new file mode 100644 index 0000000000..cb3c7ef813 --- /dev/null +++ b/doc/bugs/__39__add__39___results_in_max_cpu__44___long_run_and_huge_repo.mdwn @@ -0,0 +1,40 @@ +### Please describe the problem. +massive repo, max cpu using + + git annex add . + +had to interrupt the job as it was processing 1 small file per 5 seconds after about 3h run. + +I am running it on the root of a large (currently 1TB) exFAT-based drive used for archiving + +The repo grew to 28G. + +Is this a regular issue with exFAT? I've done quite a bit of searching. I'll do more. + +### What steps will reproduce the problem? +- install on El Capitan (latest) via homebrew +- create 1TB exFAT file store +- follow walk through to setup annex locally and on external +- add + +### What version of git-annex are you using? On what operating system? +git-annex version: 6.20160126 +build flags: Assistant Webapp Pairing Testsuite S3(multipartupload)(storageclasses) WebDAV FsEvents XMPP ConcurrentOutput TorrentParser Feeds Quvi +key/value backends: SHA256E SHA256 SHA512E SHA512 SHA224E SHA224 SHA384E SHA384 SHA3_256E SHA3_256 SHA3_512E SHA3_512 SHA3_224E SHA3_224 SHA3_384E SHA3_384 SKEIN256E SKEIN256 SKEIN512E SKEIN512 SHA1E SHA1 MD5E MD5 WORM URL +remote types: git gcrypt S3 bup directory rsync web bittorrent webdav tahoe glacier ddar hook external + +El Capitan 10.11.3 + + +### Please provide any additional information below. + +[[!format sh """ +# If you can, paste a complete transcript of the problem occurring here. +# If the problem is with the git-annex assistant, paste in .git/annex/daemon.log + + +# End of transcript or log. +"""]] + +### Have you had any luck using git-annex before? (Sometimes we get tired of reading bug reports all day and a lil' positive end note does wonders) +I'd love to say I have. You'll hear my shout of joy when I do. diff --git a/doc/bugs/__39__add__39___results_in_max_cpu__44___long_run_and_huge_repo/comment_1_3233c29405da296360d57af7d5eb418d._comment b/doc/bugs/__39__add__39___results_in_max_cpu__44___long_run_and_huge_repo/comment_1_3233c29405da296360d57af7d5eb418d._comment new file mode 100644 index 0000000000..4bd283aa22 --- /dev/null +++ b/doc/bugs/__39__add__39___results_in_max_cpu__44___long_run_and_huge_repo/comment_1_3233c29405da296360d57af7d5eb418d._comment @@ -0,0 +1,46 @@ +[[!comment format=mdwn + username="joey" + subject="""comment 1""" + date="2016-03-14T17:59:08Z" + content=""" +If I've done the math right, 5 files per second over 3 hours is only 2000 files. +The size of the files does matter, since git-annex has to read them all. +You said the repo grew to 28 gb; does that mean you added 2000 files +totalling 28 gb in size? + +I can add 2000 tiny files (5 bytes each) in 2 seconds on a SSD on Linux. + +By using a FAT filesystem, you've forced git-annex to use direct mode. +Direct mode can be a little slower, but not a great deal. Adding 2000 files +to a direct mode repo takes around 11 seconds here. (I did a little +optimisation and sped that up to 7 seconds.) + +Doing the same benchmark on a removable USB stick with a FAT filesystem +was still not slow; 7 seconds again. + +But then I had linux mount that FAT filesystem sync (so, it flushes each +file write to disk, not buffering them), and I start getting closer to your +slow speed; benchmark took 53 minutes. + +So, I think the slow speed you're seeing is quite likely due to a +combination of, in order from most to least important: + +1. Synchronous writes to your disk drive. Fixable in linux by eg, running + "mount -o remount,async /path/to/repo" and there's probably something + similar for OSX. +2. External drive being slow to access. (And if a spinning disk, slow to + seek.) +3. git-annex using direct mode on FAT + +Also there is a fair amount of faff that git-annex does when adding a file +around calling rename, stat, mkdir, etc multiple times. It may be possible +to optimize some of that to get at some speedup on synchronous disks. +But, I'd not expect more than a few percentage points speedup from such +optimisation. + +One other possiblity is you could be hitting an edge case where direct mode's +performace is bad. One known such edge case is if you have a lot of files +that all have the same content. For example, I made 2000 files that were +all empty; adding them to a direct mode repository gets slower and slower +to the point it's spending 10 or more seconds per file. +"""]] diff --git a/doc/bugs/__39__annex_add__39___locks_unmodified_file_in_V5_but_not_V6.mdwn b/doc/bugs/__39__annex_add__39___locks_unmodified_file_in_V5_but_not_V6.mdwn new file mode 100644 index 0000000000..22f9270a57 --- /dev/null +++ b/doc/bugs/__39__annex_add__39___locks_unmodified_file_in_V5_but_not_V6.mdwn @@ -0,0 +1,42 @@ +### Please describe the problem. + +In a V6 repo, if I unlock a file, modify it, and `git annex add` it, +it ends up in a locked state: + + echo foo >foo && git annex add foo && git commit -mfoo + git annex unlock foo && echo more >>foo && git annex add foo + git cat-file -p :foo + # .git/annex/objects/60/QW/SHA256E-s9--323409d9a706bc08d0b2c7f71309e21a757367c81cffb405a88e61749d79952d/SHA256E-s9--323409d9a706bc08d0b2c7f71309e21a757367c81cffb405a88e61749d79952d + +However, if I do the same, minus the modification, the file stays +unlocked: + + git reset --hard + git annex unlock foo && git annex add foo + git cat-file -p :foo + # /annex/objects/SHA256E-s4--b5bb9d8014a0f9b1d61e21e796d78dccdf1352f23cd32812f4850b878ae4944c + +I'd expect the second action to end up with a locked file because (1) +it does with a modified file and (2) it does in V5. + +From a user's perspective, I suppose this is a minor inconsistency, +and it's easy enough to call `git annex lock` here instead `git annex +add`. But in the case of DataLad, it makes the handling of our +internal `git annex add` calls trickier because we assume the V5 +"annex add unconditionally locks" behavior and we'd have to add +special handling for the V6 behavior. + +### What version of git-annex are you using? On what operating system? + + git-annex version: 6.20180808-gdad627fa9 + build flags: Assistant Webapp Pairing S3(multipartupload)(storageclasses) WebDAV Inotify ConcurrentOutput TorrentParser Feeds Testsuite + dependency versions: aws-0.17.1 bloomfilter-2.0.1.0 cryptonite-0.23 DAV-1.3.1 feed-0.3.12.0 ghc-8.0.2 http-client-0.5.7.0 persistent-sqlite-2.6.2 torrent-10000.1.1 uuid-1.3.13 yesod-1.4.5 + key/value backends: SHA256E SHA256 SHA512E SHA512 SHA224E SHA224 SHA384E SHA384 SHA3_256E SHA3_256 SHA3_512E SHA3_512 SHA3_224E SHA3_224 SHA3_384E SHA3_384 SKEIN256E SKEIN256 SKEIN512E SKEIN512 BLAKE2B256E BLAKE2B256 BLAKE2B512E BLAKE2B512 BLAKE2B160E BLAKE2B160 BLAKE2B224E BLAKE2B224 BLAKE2B384E BLAKE2B384 BLAKE2S256E BLAKE2S256 BLAKE2S160E BLAKE2S160 BLAKE2S224E BLAKE2S224 BLAKE2SP256E BLAKE2SP256 BLAKE2SP224E BLAKE2SP224 SHA1E SHA1 MD5E MD5 WORM URL + remote types: git gcrypt p2p S3 bup directory rsync web bittorrent webdav adb tahoe glacier ddar hook external + operating system: linux x86_64 + supported repository versions: 3 5 6 + upgrade supported from repository versions: 0 1 2 3 4 5 + +on Debian Stretch, built from source with `stack build` + +> [[fixed|done]] --[[Joey]] diff --git a/doc/bugs/__39__annex_add__39___locks_unmodified_file_in_V5_but_not_V6/comment_1_a54ff1147b025b58e65ce0be6e795029._comment b/doc/bugs/__39__annex_add__39___locks_unmodified_file_in_V5_but_not_V6/comment_1_a54ff1147b025b58e65ce0be6e795029._comment new file mode 100644 index 0000000000..d702e6226b --- /dev/null +++ b/doc/bugs/__39__annex_add__39___locks_unmodified_file_in_V5_but_not_V6/comment_1_a54ff1147b025b58e65ce0be6e795029._comment @@ -0,0 +1,23 @@ +[[!comment format=mdwn + username="joey" + subject="""comment 1""" + date="2018-09-12T16:29:55Z" + content=""" +In fact, `git annex add` does not process the v6 unlocked file at all +since it only looks for unstaged changes to files and the unlocked file's +type change has been staged already. + +In v5 mode there is a separate pass to add unlocked files, which is +necessary since they have to be converted back to locked files before they +can be committed. + +It would need a separate pass in v6 too, since the main pass looks only +at unstaged modifications and git can't be queried for staged modifications +at the same time as unstaged. + +Hmm, this would though mean that `git annex add` would now be changing +what's staged. It has never done that before; it's only staged new changes. +Not convinced by that argument, but something to keep in mind. + +I'm feeling this is ok to change, and the patch is not difficult. +"""]] diff --git a/doc/bugs/__39__web__39___remote_does_not_work_on_Android.mdwn b/doc/bugs/__39__web__39___remote_does_not_work_on_Android.mdwn new file mode 100644 index 0000000000..c88b63341e --- /dev/null +++ b/doc/bugs/__39__web__39___remote_does_not_work_on_Android.mdwn @@ -0,0 +1,22 @@ +### Please describe the problem. +On Android, any attempt by git-annex to use the 'web' special remote won't work, instead wget will complain about the user-agent parameter. This is very annoying when I have web URLs registered for numerous files. + +### What steps will reproduce the problem? +Try to grab a file via git-annex that has an attached web URL on Android. + +### What version of git-annex are you using? On what operating system? +Latest version of git-annex app on an Android 5 tablet. + +### Please provide any additional information below. + +[[!format sh """ +# If you can, paste a complete transcript of the problem occurring here. +# If the problem is with the git-annex assistant, paste in .git/annex/daemon.log + + +# End of transcript or log. +"""]] + +### Have you had any luck using git-annex before? (Sometimes we get tired of reading bug reports all day and a lil' positive end note does wonders) + +> Closing as this was a bug in the deprecated Android app. [[done]] --[[Joey]] diff --git a/doc/bugs/__39__web__39___remote_does_not_work_on_Android/comment_1_11d3af70288da83aee72a48cb46d22e9._comment b/doc/bugs/__39__web__39___remote_does_not_work_on_Android/comment_1_11d3af70288da83aee72a48cb46d22e9._comment new file mode 100644 index 0000000000..d5b18226ab --- /dev/null +++ b/doc/bugs/__39__web__39___remote_does_not_work_on_Android/comment_1_11d3af70288da83aee72a48cb46d22e9._comment @@ -0,0 +1,14 @@ +[[!comment format=mdwn + username="joey" + subject="""comment 1""" + date="2018-05-08T18:45:14Z" + content=""" +I'm closing this bug report because the git-annex Android app that it +was reported on is now deprecated. Instead, we have +a way to run the regular git-annex build for linux on Android in termux: + + +There were a lot of problems with the way the git-annex Android app was +put together, and I hope this new approach avoids them. If you try it and +still have the bug you reported, please followup and I'll reopen it. +"""]] diff --git a/doc/bugs/__91__Android__93_____34__Make_Camera_Repository__34___button_goes_to_blank_page.mdwn b/doc/bugs/__91__Android__93_____34__Make_Camera_Repository__34___button_goes_to_blank_page.mdwn new file mode 100644 index 0000000000..2261624387 --- /dev/null +++ b/doc/bugs/__91__Android__93_____34__Make_Camera_Repository__34___button_goes_to_blank_page.mdwn @@ -0,0 +1,18 @@ +### Please describe the problem. + +When first installing the Android app, it opens the browser to the "Welcome to git-annex!" page. There is a big button to "Make Camera Repository". I click it, and am brought to a completely blank page. If I re-open the web app, it does appear to have created the DCIM repository (it shows in the repository list), but should probably show something (a confirmation, at least?) on that page. + +### What steps will reproduce the problem? + +1. Install the android app +2. Run it +3. Click the "Make Camera Repository" button + +### What version of git-annex are you using? On what operating system? + +Current (2014-11-14) Android 4.3/4.4 build on a non-rooted Moto X. I don't see a way to find version info about the Android app itself, but the terminal output for "version" says it's running git-annex version 5.20141104-gcaafd06 + +### Please provide any additional information below. + + +> Closing as this was a bug in the deprecated Android app. [[done]] --[[Joey]] diff --git a/doc/bugs/__91__Android__93_____34__Make_Camera_Repository__34___button_goes_to_blank_page/comment_1_0543bcdd46565275790069c572331dab._comment b/doc/bugs/__91__Android__93_____34__Make_Camera_Repository__34___button_goes_to_blank_page/comment_1_0543bcdd46565275790069c572331dab._comment new file mode 100644 index 0000000000..8a7a553700 --- /dev/null +++ b/doc/bugs/__91__Android__93_____34__Make_Camera_Repository__34___button_goes_to_blank_page/comment_1_0543bcdd46565275790069c572331dab._comment @@ -0,0 +1,15 @@ +[[!comment format=mdwn + username="joey" + subject="""comment 1""" + date="2018-05-08T18:54:47Z" + content=""" +I'm closing this bug report because the git-annex Android app that it +was reported on is now deprecated. Instead, we have +a way to run the regular git-annex build for linux on Android in termux: + + +There were a lot of problems with the way the git-annex Android app was +put together, and I hope this new approach avoids them. If you try it and +still have the bug you reported, please followup and I'll reopen it. + +"""]] diff --git a/doc/bugs/__91__PATCH__93___Build_Build__47__InstallDesktopFile_at___34__make_all__34___time.mdwn b/doc/bugs/__91__PATCH__93___Build_Build__47__InstallDesktopFile_at___34__make_all__34___time.mdwn new file mode 100644 index 0000000000..ef4e63ffae --- /dev/null +++ b/doc/bugs/__91__PATCH__93___Build_Build__47__InstallDesktopFile_at___34__make_all__34___time.mdwn @@ -0,0 +1,29 @@ + commit 69138285fd4671855184a2de68e1b99aa0a4f3a8 + Author: Eric Siegerman + Date: Tue Oct 31 02:17:27 2017 -0400 + + Build Build/InstallDesktopFile at "make all" time + + If you run stack as root (e.g. for "make install"), any files it + creates under ./ will, of course, be owned by root. That's a + problem for subsequent runs as non-root. + + Reduce the likelihood of that happening by building + Build/InstallDesktopFile during "make all", so that it needn't be + built by "make install". + + diff --git a/Makefile b/Makefile + index aceb65cae..6ac241f67 100644 + --- a/Makefile + +++ b/Makefile + @@ -1,4 +1,4 @@ + -all=git-annex git-annex-shell mans docs + +all=git-annex git-annex-shell mans docs Build/InstallDesktopFile + + # set to "./Setup" if you lack a cabal program. Or can be set to "stack" + BUILDER?=cabal + +> Applied [[done]]. Note that I had to do a considerable amount of editing to +> get that in to a form that `git am` would accept. In the future, +> providing a patch in a form that `git am` can use would be better. +> --[[Joey]] diff --git a/doc/bugs/__91__PATCH__93___Cosmetic__58___clarify_a_warning_message.mdwn b/doc/bugs/__91__PATCH__93___Cosmetic__58___clarify_a_warning_message.mdwn new file mode 100644 index 0000000000..f62e75d9ee --- /dev/null +++ b/doc/bugs/__91__PATCH__93___Cosmetic__58___clarify_a_warning_message.mdwn @@ -0,0 +1,24 @@ + commit 3ee8dc86cd831e975c80844924ef062b79e763b6 + Author: Eric Siegerman + Date: Tue Oct 31 21:12:38 2017 -0400 + + Make a Makefile warning ... more obviously only a warning + + diff --git a/Makefile b/Makefile + index aceb65cae..0381e7383 100644 + --- a/Makefile + +++ b/Makefile + @@ -34,7 +34,10 @@ git-annex: tmp/configure-stamp + # Work around https://github.com/haskell/cabal/issues/3524 + # when not linked dynamically to haskell libs + @if ! ldd git-annex | grep -q libHS; then \ + - chrpath -d git-annex || echo "** unable to chrpath git-annex; it will be a little bit slower than necessary"; \ + + chrpath -d git-annex || { \ + + echo "** warning: unable to chrpath git-annex; it will run OK..."; \ + + echo "** ... but maybe a little bit slower than necessary"; \ + + } \ + fi + + git-annex-shell: git-annex + +> Added "warning:" [[done]] --[[Joey]] diff --git a/doc/bugs/__91__PATCH__93___Cosmetic__58___clarify_a_warning_message/comment_1_2ec5e2dfe502b3357f7cb224bb28219a._comment b/doc/bugs/__91__PATCH__93___Cosmetic__58___clarify_a_warning_message/comment_1_2ec5e2dfe502b3357f7cb224bb28219a._comment new file mode 100644 index 0000000000..fba3a63f75 --- /dev/null +++ b/doc/bugs/__91__PATCH__93___Cosmetic__58___clarify_a_warning_message/comment_1_2ec5e2dfe502b3357f7cb224bb28219a._comment @@ -0,0 +1,8 @@ +[[!comment format=mdwn + username="joey" + subject="""comment 1""" + date="2017-11-07T20:39:22Z" + content=""" +Perhaps I'm blind as the one who wrote it, but I don't see anything +unclear about the message. +"""]] diff --git a/doc/bugs/__91__PATCH__93___Cosmetic__58___clarify_a_warning_message/comment_2_16c12a55f7d4b862e021e9bc1f8c9788._comment b/doc/bugs/__91__PATCH__93___Cosmetic__58___clarify_a_warning_message/comment_2_16c12a55f7d4b862e021e9bc1f8c9788._comment new file mode 100644 index 0000000000..f100492b37 --- /dev/null +++ b/doc/bugs/__91__PATCH__93___Cosmetic__58___clarify_a_warning_message/comment_2_16c12a55f7d4b862e021e9bc1f8c9788._comment @@ -0,0 +1,10 @@ +[[!comment format=mdwn + username="erics" + avatar="http://cdn.libravatar.org/avatar/6e5f74c742128e5d98fd672ed6ea4865" + subject="comment 2" + date="2017-12-08T21:11:40Z" + content=""" +It wasn't clear to me that this was just a warning, i.e. that functionality wasn't compromised. The part after the semicolon kind of implies that, but it seems clearer to say so explicitly. + +On second thought, though, the rephrasing probably isn't necessary. Just adding the \"Warning:\" prefix would suffice... +"""]] diff --git a/doc/bugs/__91__PATCH__93___Cosmetic__58___only_print_ikiwiki_command_if_it__39__s_run.mdwn b/doc/bugs/__91__PATCH__93___Cosmetic__58___only_print_ikiwiki_command_if_it__39__s_run.mdwn new file mode 100644 index 0000000000..f2cea95f50 --- /dev/null +++ b/doc/bugs/__91__PATCH__93___Cosmetic__58___only_print_ikiwiki_command_if_it__39__s_run.mdwn @@ -0,0 +1,44 @@ +No intended functional change; only what *make* prints should be different. + + commit bb43afb0d70311dc9fd7633133c3c4fec32511e6 + Author: Eric Siegerman + Date: Tue Oct 31 02:33:13 2017 -0400 + + Refactor "make docs" to eliminate confusing output + + The the many lines of arguments to the ikiwiki command would + always be printed -- even if ikiwiki was unavailable. Now + you'll only see them if they're accomplishing something. + + diff --git a/Makefile b/Makefile + index aceb65cae..121b19a99 100644 + --- a/Makefile + +++ b/Makefile + @@ -88,16 +88,21 @@ tags: + # If ikiwiki is available, build static html docs suitable for being + # shipped in the software package. + ifeq ($(shell which ikiwiki),) + -IKIWIKI=echo "** ikiwiki not found, skipping building docs" >&2; true + +BUILD_DOCS=_skip_building_docs + else + -IKIWIKI=ikiwiki + +BUILD_DOCS = _build_docs + endif + + mans: Build/MakeMans + ./Build/MakeMans + + -docs: mans + - LC_ALL=C TZ=UTC $(IKIWIKI) doc html -v --wikiname git-annex \ + +docs: mans $(BUILD_DOCS) + + + +_skip_building_docs: + + @echo "** ikiwiki not found, skipping building docs" >&2 + + + +_build_docs: + + LC_ALL=C TZ=UTC ikiwiki doc html -v --wikiname git-annex \ + --plugin=goodstuff \ + --no-usedirs --disable-plugin=openid --plugin=sidebar \ + --plugin theme --set theme=actiontabs --set deterministic=1 \ + +> [[done]] --[[Joey]] diff --git a/doc/bugs/__91__PATCH__93___Cosmetic__58___only_print_ikiwiki_command_if_it__39__s_run/comment_1_b8304ff302805f7bec977002d733436a._comment b/doc/bugs/__91__PATCH__93___Cosmetic__58___only_print_ikiwiki_command_if_it__39__s_run/comment_1_b8304ff302805f7bec977002d733436a._comment new file mode 100644 index 0000000000..31a6700866 --- /dev/null +++ b/doc/bugs/__91__PATCH__93___Cosmetic__58___only_print_ikiwiki_command_if_it__39__s_run/comment_1_b8304ff302805f7bec977002d733436a._comment @@ -0,0 +1,9 @@ +[[!comment format=mdwn + username="joey" + subject="""comment 1""" + date="2017-11-07T20:31:00Z" + content=""" +I don't like this amount of complication for a build system cosmetic +improvement. Instead, I have added a "@" to the objectionably long +command line in the Makefile. +"""]] diff --git a/doc/bugs/__91__meta__93___ikiwiki_is___42__slow__42__.mdwn b/doc/bugs/__91__meta__93___ikiwiki_is___42__slow__42__.mdwn new file mode 100644 index 0000000000..8f10b83bca --- /dev/null +++ b/doc/bugs/__91__meta__93___ikiwiki_is___42__slow__42__.mdwn @@ -0,0 +1,3 @@ +### Please describe the problem. + +Adding a comment or a bug takes easily 15 seconds after pressing “Submit” or “Save”… diff --git a/doc/bugs/__91__meta__93___ikiwiki_is___42__slow__42__/comment_1_3afce2aadeb2221c66f397e631d0e4a6._comment b/doc/bugs/__91__meta__93___ikiwiki_is___42__slow__42__/comment_1_3afce2aadeb2221c66f397e631d0e4a6._comment new file mode 100644 index 0000000000..6ec99e8635 --- /dev/null +++ b/doc/bugs/__91__meta__93___ikiwiki_is___42__slow__42__/comment_1_3afce2aadeb2221c66f397e631d0e4a6._comment @@ -0,0 +1,8 @@ +[[!comment format=mdwn + username="michalrus" + avatar="http://cdn.libravatar.org/avatar/83c0b6e7f9d20f09a892263c4903bbae" + subject="comment 1" + date="2017-08-26T12:20:58Z" + content=""" +^ This one took 21 s. +"""]] diff --git a/doc/bugs/__91__meta__93___ikiwiki_is___42__slow__42__/comment_2_220b3ed61dfcce4d6d33fa8516cd1ec1._comment b/doc/bugs/__91__meta__93___ikiwiki_is___42__slow__42__/comment_2_220b3ed61dfcce4d6d33fa8516cd1ec1._comment new file mode 100644 index 0000000000..48aab919cb --- /dev/null +++ b/doc/bugs/__91__meta__93___ikiwiki_is___42__slow__42__/comment_2_220b3ed61dfcce4d6d33fa8516cd1ec1._comment @@ -0,0 +1,20 @@ +[[!comment format=mdwn + username="joey" + subject="""comment 2""" + date="2017-09-29T16:27:40Z" + content=""" +ikiwiki gets slow the more pages it has to deal with. With only a few +thousand pages, it's pretty fast. Including comments, there are currently +nearly 10k pages here. (The server might also have been overloaded +for some reason when this bug was filed. There were some stability problems +in august related to apache bugs.) + +In the past I've deleted old closed bugs to speed it back up. +There around 3k pages in bugs and todo including comments. + +Deleted bugs closed before this year and removed almost 1k pages. + +Also, needing to update the rss and atom feeds when adding a bug +meant writing over 10 mb to disk every time. Limited the size of those +feeds, which should help a lot. +"""]] diff --git a/doc/bugs/__91__meta__93___ikiwiki_is___42__slow__42__/comment_3_8fb126d6583014d3e16137e9c18057e6._comment b/doc/bugs/__91__meta__93___ikiwiki_is___42__slow__42__/comment_3_8fb126d6583014d3e16137e9c18057e6._comment new file mode 100644 index 0000000000..7fb7dd096f --- /dev/null +++ b/doc/bugs/__91__meta__93___ikiwiki_is___42__slow__42__/comment_3_8fb126d6583014d3e16137e9c18057e6._comment @@ -0,0 +1,8 @@ +[[!comment format=mdwn + username="michalrus" + avatar="http://cdn.libravatar.org/avatar/83c0b6e7f9d20f09a892263c4903bbae" + subject="comment 3" + date="2017-09-29T20:38:31Z" + content=""" +Thank you! =) +"""]] diff --git a/doc/bugs/__91__meta__93___ikiwiki_is___42__slow__42__/comment_4_72204d0647ec808145b6838d5e621107._comment b/doc/bugs/__91__meta__93___ikiwiki_is___42__slow__42__/comment_4_72204d0647ec808145b6838d5e621107._comment new file mode 100644 index 0000000000..493d3e63ad --- /dev/null +++ b/doc/bugs/__91__meta__93___ikiwiki_is___42__slow__42__/comment_4_72204d0647ec808145b6838d5e621107._comment @@ -0,0 +1,8 @@ +[[!comment format=mdwn + username="michalrus" + avatar="http://cdn.libravatar.org/avatar/83c0b6e7f9d20f09a892263c4903bbae" + subject="comment 4" + date="2017-09-29T20:38:59Z" + content=""" +^ This comment took only 10 seconds, so there’s improvement. (ﾉ^_^)ﾉ +"""]] diff --git a/doc/bugs/_git-annex__58___bad_url_ssh__58____47____47__git__64__gitlab.com__58____126____47__gitlabname__47__reponame.git.mdwn b/doc/bugs/_git-annex__58___bad_url_ssh__58____47____47__git__64__gitlab.com__58____126____47__gitlabname__47__reponame.git.mdwn new file mode 100644 index 0000000000..74e9235dd9 --- /dev/null +++ b/doc/bugs/_git-annex__58___bad_url_ssh__58____47____47__git__64__gitlab.com__58____126____47__gitlabname__47__reponame.git.mdwn @@ -0,0 +1,33 @@ +### Please describe the problem. +An encrypted remote is added to a working git annex repository (mind ":~/" in the remote add command). However, after that any git annex command fails. + +### What steps will reproduce the problem? + > git remote add encrypted gcrypt::ssh://git@gitlab.com:~/gitlabname/reponame.git + > git push encrypted master + gcrypt: Repository not found: ssh://git@gitlab.com:~/gitlabname/reponame.git + gcrypt: Setting up new repository + gcrypt: Remote ID is :id:abcdefghijklmnopqrst + Counting objects: 53, done. + Compressing objects: 100% (52/52), done. + Total 53 (delta 12), reused 0 (delta 0) + gcrypt: Encrypting to: --throw-keyids --default-recipient-self + gcrypt: Requesting manifest signature + ... + To gcrypt::ssh://git@gitlab.com:~/gitlabname/reponame.git + * [new branch] master -> master + > + > git annex sync + git-annex: bad url ssh://git@gitlab.com:~/gitlabname/reponame.git + +### What version of git-annex are you using? On what operating system? +5.20150419-g900e1b6 on Mac OS X + +### Please provide any additional information below. + +[[!format sh """ +# If you can, paste a complete transcript of the problem occurring here. +# If the problem is with the git-annex assistant, paste in .git/annex/daemon.log +- + +# End of transcript or log. +"""]] diff --git a/doc/bugs/_git-annex__58___bad_url_ssh__58____47____47__git__64__gitlab.com__58____126____47__gitlabname__47__reponame.git/comment_1_60db0d6bfc71a62b3c1021527a8d2d60._comment b/doc/bugs/_git-annex__58___bad_url_ssh__58____47____47__git__64__gitlab.com__58____126____47__gitlabname__47__reponame.git/comment_1_60db0d6bfc71a62b3c1021527a8d2d60._comment new file mode 100644 index 0000000000..4b7b0e90a7 --- /dev/null +++ b/doc/bugs/_git-annex__58___bad_url_ssh__58____47____47__git__64__gitlab.com__58____126____47__gitlabname__47__reponame.git/comment_1_60db0d6bfc71a62b3c1021527a8d2d60._comment @@ -0,0 +1,23 @@ +[[!comment format=mdwn + username="joey" + subject="""comment 1""" + date="2015-04-29T15:16:31Z" + content=""" +According to the git-fetch man page, the syntax to use +for this kind of url is: + + ssh://[user@]host.xz[:port]/~[user]/path/to/repo.git/ + +Your url is missing the leading slash before the `~`, and has +a : with no port specified. + + ssh://git@gitlab.com:~/gitlabname/reponame.git + +It is in fact, not a legal url. + +Now, git might accept it despite not documenting it as an accepted form, +but why wander into undefined territory when there are legal ways to write +this url that work fine? + +Does GitLab promote using these malformed urls? +"""]] diff --git a/doc/bugs/_git-annex__58___bad_url_ssh__58____47____47__git__64__gitlab.com__58____126____47__gitlabname__47__reponame.git/comment_2_49682a91147990a578929678880bd226._comment b/doc/bugs/_git-annex__58___bad_url_ssh__58____47____47__git__64__gitlab.com__58____126____47__gitlabname__47__reponame.git/comment_2_49682a91147990a578929678880bd226._comment new file mode 100644 index 0000000000..e937086445 --- /dev/null +++ b/doc/bugs/_git-annex__58___bad_url_ssh__58____47____47__git__64__gitlab.com__58____126____47__gitlabname__47__reponame.git/comment_2_49682a91147990a578929678880bd226._comment @@ -0,0 +1,26 @@ +[[!comment format=mdwn + username="http://rfhbuk.pip.verisignlabs.com.pip.verisignlabs.com/" + nickname="rfhb" + subject="comment 2" + date="2015-04-29T15:45:34Z" + content=""" +Thank you. I was wondering about the URL as flagged above. When I specify the URL as I did, the git commands work but when I change it to ssh://git@gitlab.com:/~/gitlabname/reponame.git, git command fails with + + > git push encrypted master + gcrypt: Development version -- Repository format MAY CHANGE + gcrypt: Repository not found: ssh://git@gitlab.com:/~/gitlabname/reponame.git + gcrypt: Setting up new repository + gcrypt: Remote ID is :id:xxxxxxxxxxxxxxxxx + Counting objects: 3, done. + Total 3 (delta 0), reused 0 (delta 0) + gcrypt: Encrypting to: --throw-keyids --default-recipient-self + gcrypt: Requesting manifest signature + .... passphase entered ... + GitLab: No such project + fatal: Could not read from remote repository. + Please make sure you have the correct access rights + and the repository exists. + error: failed to push some refs to 'gcrypt::ssh://git@gitlab.com:/~/gitlabname/reponame.git' + +So perhaps I need to investigate how to get gcrypt to work with remote git(lab) repositories. Thanks. +"""]] diff --git a/doc/bugs/_git-annex__58___bad_url_ssh__58____47____47__git__64__gitlab.com__58____126____47__gitlabname__47__reponame.git/comment_3_bf0fa06596338444a3772d1865b17e26._comment b/doc/bugs/_git-annex__58___bad_url_ssh__58____47____47__git__64__gitlab.com__58____126____47__gitlabname__47__reponame.git/comment_3_bf0fa06596338444a3772d1865b17e26._comment new file mode 100644 index 0000000000..0d7fb9c2da --- /dev/null +++ b/doc/bugs/_git-annex__58___bad_url_ssh__58____47____47__git__64__gitlab.com__58____126____47__gitlabname__47__reponame.git/comment_3_bf0fa06596338444a3772d1865b17e26._comment @@ -0,0 +1,7 @@ +[[!comment format=mdwn + username="joey" + subject="""comment 3""" + date="2015-04-29T17:09:07Z" + content=""" +This works: ssh://git@gitlab.com/username/reponame.git +"""]] diff --git a/doc/bugs/_impossible_to_switch_repositories_on_android__in_webapp.mdwn b/doc/bugs/_impossible_to_switch_repositories_on_android__in_webapp.mdwn new file mode 100644 index 0000000000..21bcb6f6c6 --- /dev/null +++ b/doc/bugs/_impossible_to_switch_repositories_on_android__in_webapp.mdwn @@ -0,0 +1,23 @@ +### Please describe the problem. +I didn't spot this bugs page before so here is my report I have commented on android page + +In addition to two existing repositories (1 local /sdcard/annex, which is also avail at/storage/sdcard0/annex + 1 remote) I have added one more local (and said to keep it in sync with original local). But it didn't work -- it "Synced with onerussian.com_annex but not with Annex" and claimed that the /external/extSdCard/Annex doesn't exist, although it is there (and with .git generated etc). When I restarted the deamon I got into a "new" Repository: /storage/extSdCard/Annex which also listed the 1st local but with "Failed to sync with localhost" message -- no remote one listed. Whenever I try to "Switch repository" to /sdcard/annex (the original local) -- it starts loading a new page but gets stuck right there. The only way to revive webui is to go back to Dashboard. Log there says (retyping from the screen so typos might be there): + +error: cannot run git-receive-pack '/storage/sdcard0/annex': No such file or directory fatal: unable to fork +### What steps will reproduce the problem? + + +### What version of git-annex are you using? On what operating system? +android + +### Please provide any additional information below. + +[[!format sh """ +# If you can, paste a complete transcript of the problem occurring here. +# If the problem is with the git-annex assistant, paste in .git/annex/daemon.log + + +# End of transcript or log. +"""]] + +> Closing as this was a bug in the deprecated Android app. [[done]] --[[Joey]] diff --git a/doc/bugs/_impossible_to_switch_repositories_on_android__in_webapp/comment_1_d488d71a72eb54d7711d2a867db6172f._comment b/doc/bugs/_impossible_to_switch_repositories_on_android__in_webapp/comment_1_d488d71a72eb54d7711d2a867db6172f._comment new file mode 100644 index 0000000000..aedeaabf3e --- /dev/null +++ b/doc/bugs/_impossible_to_switch_repositories_on_android__in_webapp/comment_1_d488d71a72eb54d7711d2a867db6172f._comment @@ -0,0 +1,8 @@ +[[!comment format=mdwn + username="https://www.google.com/accounts/o8/id?id=AItOawnRai_qFYPVvEgC6i1nlM1bh-C__jbhqS0" + nickname="Matthew" + subject="comment 1" + date="2013-07-28T11:21:42Z" + content=""" +I can confirm it's virtually impossible to change repository on Android (Stock Galaxy Nexus GSM). you click on the files link (which is hard to see on phone as it is in the notification blocks) and then click click \"Switch repository\", on the next page, select a different one. I think it does actually change, occasionally, you have to kill and restart the daemon at least and I think you also have to restart the phone. +"""]] diff --git a/doc/bugs/_impossible_to_switch_repositories_on_android__in_webapp/comment_2_85b31db6d0fb2d20018db3d8c8258bf4._comment b/doc/bugs/_impossible_to_switch_repositories_on_android__in_webapp/comment_2_85b31db6d0fb2d20018db3d8c8258bf4._comment new file mode 100644 index 0000000000..0ee4e81669 --- /dev/null +++ b/doc/bugs/_impossible_to_switch_repositories_on_android__in_webapp/comment_2_85b31db6d0fb2d20018db3d8c8258bf4._comment @@ -0,0 +1,8 @@ +[[!comment format=mdwn + username="https://www.google.com/accounts/o8/id?id=AItOawnRai_qFYPVvEgC6i1nlM1bh-C__jbhqS0" + nickname="Matthew" + subject="comment 2" + date="2013-07-28T11:46:30Z" + content=""" +For notification +"""]] diff --git a/doc/bugs/_impossible_to_switch_repositories_on_android__in_webapp/comment_3_9ffafbeb572e110b3e072029d1ce177c._comment b/doc/bugs/_impossible_to_switch_repositories_on_android__in_webapp/comment_3_9ffafbeb572e110b3e072029d1ce177c._comment new file mode 100644 index 0000000000..b181a4fa13 --- /dev/null +++ b/doc/bugs/_impossible_to_switch_repositories_on_android__in_webapp/comment_3_9ffafbeb572e110b3e072029d1ce177c._comment @@ -0,0 +1,8 @@ +[[!comment format=mdwn + username="yasin.zaehringer" + ip="90.218.200.128" + subject="comment 3" + date="2014-04-02T11:43:52Z" + content=""" +The bug still exists. It is not possible to change the repository in the WebApp. +"""]] diff --git a/doc/bugs/_impossible_to_switch_repositories_on_android__in_webapp/comment_4_69c0068218586ee22d3bd29dd30d0ae0._comment b/doc/bugs/_impossible_to_switch_repositories_on_android__in_webapp/comment_4_69c0068218586ee22d3bd29dd30d0ae0._comment new file mode 100644 index 0000000000..4374c0793e --- /dev/null +++ b/doc/bugs/_impossible_to_switch_repositories_on_android__in_webapp/comment_4_69c0068218586ee22d3bd29dd30d0ae0._comment @@ -0,0 +1,8 @@ +[[!comment format=mdwn + username="https://www.google.com/accounts/o8/id?id=AItOawnbPKLjBONawBd74MKJZo05juCqdsP1jAU" + nickname="Ramon" + subject="bug still there" + date="2015-02-22T14:58:41Z" + content=""" +As of today (with v 5.20150219-gd24cfd3, on two different Android tablets, one with Android 4.1.1 and the other 4.4.1) it is still impossible to switch repos in Android webapp. So configuration, etc, of different repos becomes somewhat of a pain. +"""]] diff --git a/doc/bugs/_impossible_to_switch_repositories_on_android__in_webapp/comment_5_4522150470f42aee27233bfd35664b9e._comment b/doc/bugs/_impossible_to_switch_repositories_on_android__in_webapp/comment_5_4522150470f42aee27233bfd35664b9e._comment new file mode 100644 index 0000000000..e74e137548 --- /dev/null +++ b/doc/bugs/_impossible_to_switch_repositories_on_android__in_webapp/comment_5_4522150470f42aee27233bfd35664b9e._comment @@ -0,0 +1,21 @@ +[[!comment format=mdwn + username="ano.nymous@12ebd53e5933cd1730c84027a7cb905e7c3fdd9c" + nickname="ano.nymous" + avatar="http://cdn.libravatar.org/avatar/f75c9e80591de02a07a1156d45cacdda" + subject="Workaround for switching repos from WebApp" + date="2017-06-07T19:18:51Z" + content=""" +With version 6.20170104 the following workaround seems to work: + + - Start the webapp + + - Click on the menu and select \"Switch Repository\" (as ususal) + + - Click on the link of the repo you wish to switch to (as usual); at this point, the WebApp freezes and the repo is not changed. + + - Keep cool :-) Select the Git Annex terminal session from the slide-down menu. + + - In the Git Annex terminal create a new window by clicking the (+) icon. This will open a new WebApp Dashboard with the newly selected repository. + + - Enjoy :-) +"""]] diff --git a/doc/bugs/_impossible_to_switch_repositories_on_android__in_webapp/comment_6_ee75ed3c65e0c3a0aaf22ea2ad7e9832._comment b/doc/bugs/_impossible_to_switch_repositories_on_android__in_webapp/comment_6_ee75ed3c65e0c3a0aaf22ea2ad7e9832._comment new file mode 100644 index 0000000000..1ada17de47 --- /dev/null +++ b/doc/bugs/_impossible_to_switch_repositories_on_android__in_webapp/comment_6_ee75ed3c65e0c3a0aaf22ea2ad7e9832._comment @@ -0,0 +1,9 @@ +[[!comment format=mdwn + username="Crystalvonwerder@a141a1e27afcd463daccce74ba4df918a01dfd9e" + nickname="Crystalvonwerder" + avatar="http://cdn.libravatar.org/avatar/508da5fc6a8669b0c7dc674259f78075" + subject="Android" + date="2017-07-18T14:43:43Z" + content=""" +Not working +"""]] diff --git a/doc/bugs/_impossible_to_switch_repositories_on_android__in_webapp/comment_7_828dd78475a11c38b58162485fc1a243._comment b/doc/bugs/_impossible_to_switch_repositories_on_android__in_webapp/comment_7_828dd78475a11c38b58162485fc1a243._comment new file mode 100644 index 0000000000..8aee671c72 --- /dev/null +++ b/doc/bugs/_impossible_to_switch_repositories_on_android__in_webapp/comment_7_828dd78475a11c38b58162485fc1a243._comment @@ -0,0 +1,14 @@ +[[!comment format=mdwn + username="joey" + subject="""comment 7""" + date="2018-05-08T18:44:11Z" + content=""" +I'm closing this bug report because the git-annex Android app that it +was reported on is now deprecated. Instead, we have +a way to run the regular git-annex build for linux on Android in termux: + + +There were a lot of problems with the way the git-annex Android app was +put together, and I hope this new approach avoids them. If you try it and +still have the bug you reported, please followup and I'll reopen it. +"""]] diff --git a/doc/bugs/acl_not_honoured_in_rsync_remote.mdwn b/doc/bugs/acl_not_honoured_in_rsync_remote.mdwn new file mode 100644 index 0000000000..edc4e67d7d --- /dev/null +++ b/doc/bugs/acl_not_honoured_in_rsync_remote.mdwn @@ -0,0 +1,59 @@ +in a setup where an rsync(+gnupg) remote is shared among different users of the same git-annex repository (ie, the people copying to there use different accounts on the rsync server), acls are not honored under some circumstances. + +the error message reads as follows: + + copy …filename… (to prometheus...) Reading passphrase from file descriptor 11 + + sending incremental file list + rsync: recv_generator: mkdir "/home/shared/photos/encrypted_storage/9a6/0ff" failed: Permission denied (13) + *** Skipping any contents from this failed directory *** + 9a6/0ff/ + rsync error: some files/attrs were not transferred (see previous errors) (code 23) at main.c(1070) [sender=3.0.8] + + sent 185 bytes received 18 bytes 135.33 bytes/sec + total size is 2119419 speedup is 10440.49 + + This could have failed because --fast is enabled. + failed + +the acl used in my particular case is: + + # file: . + # owner: chrysn + # group: chrysn + user::rwx + group::rwx + group:family:rwx + mask::rwx + other::r-x + default:user::rwx + default:group::rwx + default:group:family:rwx + default:mask::rwx + default:other::r-x + +sub-directories are observed to have diverging permissions, though: + + # file: 794 + # owner: chrysn + # group: chrysn + user::rwx + group::rwx #effective:r-x + group:family:rwx #effective:r-x + mask::r-x + other::r-x + default:user::rwx + default:group::rwx + default:group:family:rwx + default:mask::rwx + default:other::r-x + +something seems to apply the umask (default 022) and revoke group write access from the files, overruling the acl. this is not what a umask is normally used for, and smells of [coreutils slavishly observing posix specs that don't consider all features](http://savannah.gnu.org/bugs/?19546) -- the observed effect is exactly what's described there. + +the git annex version used is 3.20121017 as in debian, the receiving site uses rsync 3.0.7; the affected directories come from a time when these very versions are known to have been used. + +this is probably not a bug of git-annex alone, but affects its operation and might be solvable by invoking rsync differently. + +(this is kind of a follow-up on [[forum/__34__permission_denied__34___in_fsck_on_shared_repo]]) + +[[!tag forwarded]] diff --git a/doc/bugs/acl_not_honoured_in_rsync_remote/comment_1_aa6fe1d7b029eae7ee71c97e0f0937a6._comment b/doc/bugs/acl_not_honoured_in_rsync_remote/comment_1_aa6fe1d7b029eae7ee71c97e0f0937a6._comment new file mode 100644 index 0000000000..c24b021572 --- /dev/null +++ b/doc/bugs/acl_not_honoured_in_rsync_remote/comment_1_aa6fe1d7b029eae7ee71c97e0f0937a6._comment @@ -0,0 +1,8 @@ +[[!comment format=mdwn + username="http://joeyh.name/" + ip="4.152.108.27" + subject="comment 1" + date="2012-11-06T20:49:47Z" + content=""" +I don't know much about ACLs, but you might try setting `remote.prometheus.annex-rsync-options` with rsync options such as --acls or --no-acls +"""]] diff --git a/doc/bugs/acl_not_honoured_in_rsync_remote/comment_2_ffb9424e966ee10a4fe2d446b3042cb2._comment b/doc/bugs/acl_not_honoured_in_rsync_remote/comment_2_ffb9424e966ee10a4fe2d446b3042cb2._comment new file mode 100644 index 0000000000..96dbc6a2e2 --- /dev/null +++ b/doc/bugs/acl_not_honoured_in_rsync_remote/comment_2_ffb9424e966ee10a4fe2d446b3042cb2._comment @@ -0,0 +1,10 @@ +[[!comment format=mdwn + username="http://christian.amsuess.com/chrysn" + nickname="chrysn" + subject="further tracking" + date="2012-11-11T01:15:41Z" + content=""" +the behavior is rooted in rsync, which behaves similar to `mkdir -p`. it can probably be worked around by configuring the rsync option `--chmod=775`, but that would add executability to files. + +i've opened a bug in rsync's bug tracker at , let's see what the developer says. +"""]] diff --git a/doc/bugs/acl_not_honoured_in_rsync_remote/comment_3_f93177593a2d90627672647fd5f065c9._comment b/doc/bugs/acl_not_honoured_in_rsync_remote/comment_3_f93177593a2d90627672647fd5f065c9._comment new file mode 100644 index 0000000000..6b20bbc3a7 --- /dev/null +++ b/doc/bugs/acl_not_honoured_in_rsync_remote/comment_3_f93177593a2d90627672647fd5f065c9._comment @@ -0,0 +1,7 @@ +[[!comment format=mdwn + username="eigengrau" + subject="comment 3" + date="2016-01-03T07:52:36Z" + content=""" +I recently experimented with shared repos a bit and came across this rsync issue. I’ve found that instead of setting `rsync-options` to `--chmod=755`, you can also set it to `--chmod=ugo=rX`. This will cause the executable bit to be set only when the source file is already executable. +"""]] diff --git a/doc/bugs/add_-J_fails_with_not_found.mdwn b/doc/bugs/add_-J_fails_with_not_found.mdwn new file mode 100644 index 0000000000..05e42c48fc --- /dev/null +++ b/doc/bugs/add_-J_fails_with_not_found.mdwn @@ -0,0 +1,40 @@ +here, caught one for you for add (git annex version is tiny bit dated: 6.20170815+gitg22da64d0f-1~ndall+1 ) + +(Pdb) print e.msg +Failed to run ['git', '-c', 'receive.autogc=0', '-c', 'gc.auto=0', 'annex', 'add', '--json', '-J6', '082-sn/000001.dcm', '078-sn/000001.dcm', '080-sn/000002.dcm', '076-sn/000002.dcm', '080-sn/000004.dcm', '080-sn/000001.dcm', '079-sn/000003.dcm', '082-sn/000003.dcm', '073-sn/000002.dcm', '079-sn/000002.dcm', '079-sn/000001.dcm', '077-sn/000002.dcm', '074-sn/000001.dcm', '080-sn/000003.dcm', '077-sn/000001.dcm', '076-sn/000001.dcm', '081-sn/000002.dcm', '078-sn/000002.dcm', '081-sn/000001.dcm', '081-sn/000003.dcm', '073-sn/000001.dcm', '075-sn/000001.dcm', '079-sn/000004.dcm', '082-sn/000002.dcm', '075-sn/000002.dcm', '074-sn/000002.dcm'] under '/mnt/DICOM/test2/inbox/2016/12/12/unknown'. Exit code=1. out={"command":"add","success":true,"key":"MD5E-s193740--3da2e91e0888c05e01daf8ef9ae79570.dcm","file":"076-sn/000002.dcm"} +{"command":"add","success":true,"key":"MD5E-s205064--851ce819ea44cabd66923d902e55cd2c.dcm","file":"082-sn/000001.dcm"} +{"command":"add","success":true,"key":"MD5E-s226874--8acaba7ff0f57a3e69a5c1afb8bc0ba3.dcm","file":"080-sn/000001.dcm"} +{"command":"add","success":true,"key":"MD5E-s193746--faf49ea7b403c8f6191fa5b7521ebbd5.dcm","file":"078-sn/000001.dcm"} +{"command":"add","success":true,"key":"MD5E-s205066--c5af824ecd895a342cba7ffefe019fde.dcm","file":"082-sn/000003.dcm"} +{"command":"add","success":true,"key":"MD5E-s226872--abdaf5312c4c11da68cc9aaead6bc93a.dcm","file":"080-sn/000004.dcm"} +{"command":"add","success":true,"key":"MD5E-s226768--352d7bf9be4b45316cb75c2fc5bfbdd2.dcm","file":"079-sn/000002.dcm"} +{"command":"add","success":true,"key":"MD5E-s226872--755e5e693b07fd7cab07fa55e0f17fd2.dcm","file":"080-sn/000002.dcm"} +{"command":"add","success":true,"key":"MD5E-s193636--47816f992ddf6167549b0d20bf430036.dcm","file":"073-sn/000002.dcm"} +{"command":"add","success":true,"key":"MD5E-s193750--465de1ebab376633842ac9e36f6fcc35.dcm","file":"074-sn/000001.dcm"} +{"command":"add","success":true,"key":"MD5E-s226766--04006df494da6adbe7b3eea90e4dba1b.dcm","file":"079-sn/000003.dcm"} +{"command":"add","success":true,"key":"MD5E-s226768--3624982179994bbaa781ee8c72c5d6b9.dcm","file":"079-sn/000001.dcm"} +{"command":"add","success":true,"key":"MD5E-s193636--98a6efe8bea17485297e5d600c4c01e6.dcm","file":"077-sn/000002.dcm"} +{"command":"add","success":true,"key":"MD5E-s193642--15770ed37bedb6bd90a7a4b39761a1fa.dcm","file":"077-sn/000001.dcm"} +{"command":"add","success":true,"key":"MD5E-s226870--621a7d3a2be10b99d45a39ef2526ab1c.dcm","file":"080-sn/000003.dcm"} +{"command":"add","success":true,"key":"MD5E-s204960--e28aac5eb5e52fe47c2671e8b87edc8c.dcm","file":"081-sn/000002.dcm"} +{"command":"add","success":true,"key":"MD5E-s193750--25cadf2f0fc7a4ca324687a742a2d55e.dcm","file":"076-sn/000001.dcm"} +{"command":"add","success":true,"key":"MD5E-s193740--0424fce22306f7590f6d94561eb74e93.dcm","file":"078-sn/000002.dcm"} +{"command":"add","success":true,"key":"MD5E-s204956--039309e288379d98480390043ea84ecb.dcm","file":"081-sn/000001.dcm"} +{"command":"add","success":true,"key":"MD5E-s204960--7b32e2620301909a74adea3d29c00ed5.dcm","file":"081-sn/000003.dcm"} +{"command":"add","success":true,"key":"MD5E-s193646--ff7337b5330f852063652f1e75099b27.dcm","file":"073-sn/000001.dcm"} +{"command":"add","success":true,"key":"MD5E-s226768--40821b65d32878f5859ed1b19c5633d0.dcm","file":"079-sn/000004.dcm"} +{"command":"add","success":true,"key":"MD5E-s193646--02704fbd9d37349c4ee96c1509f1a20d.dcm","file":"075-sn/000001.dcm"} +{"command":"add","success":true,"key":"MD5E-s193636--7c727d28c56da2476b744e8aca421b5c.dcm","file":"075-sn/000002.dcm"} +{"command":"add","success":true,"key":"MD5E-s193740--c0b3731d6a46df9000df32eeb78ae894.dcm","file":"074-sn/000002.dcm"} +{"command":"add","success":true,"key":"MD5E-s205066--11b6f6e4cb2f5879a530f83d288bb7fa.dcm","file":"082-sn/000002.dcm"} + err=git-annex: 075-sn/000001.dcm not found +git-annex: add: 1 failed + +(Pdb) +[1]+ Stopped ../test/addall.sh +(dev) [yoh@rolando test2]$ ls -l '/mnt/DICOM/test2/inbox/2016/12/12/unknown/075-sn/000001.dcm' +lrwxrwxrwx 1 yoh users 129 Dec 12 2016 /mnt/DICOM/test2/inbox/2016/12/12/unknown/075-sn/000001.dcm -> ../.git/annex/objects/ZQ/3G/MD5E-s193646--02704fbd9d37349c4ee96c1509f1a20d.dcm/MD5E-s193646--02704fbd9d37349c4ee96c1509f1a20d.dcm + +so it does report success in json, but complains in stderr that file is not found... I guess some race condition between workers so it manages to catch the moment when file is moved into a key or smth like that? + +> [[done]] --[[Joey]] diff --git a/doc/bugs/add_-J_fails_with_not_found/comment_1_daf1f53ac77f664ca0ce572c1bdc45ee._comment b/doc/bugs/add_-J_fails_with_not_found/comment_1_daf1f53ac77f664ca0ce572c1bdc45ee._comment new file mode 100644 index 0000000000..078d14b6a8 --- /dev/null +++ b/doc/bugs/add_-J_fails_with_not_found/comment_1_daf1f53ac77f664ca0ce572c1bdc45ee._comment @@ -0,0 +1,31 @@ +[[!comment format=mdwn + username="joey" + subject="""comment 1""" + date="2017-10-16T16:08:18Z" + content=""" +The " not found" error message comes from +CmdLine.Seek.checkFileOrDirectoryExists, which is intended to catch +git-annex being run with a parameter that does not exist on disk and let +the user know about their mistake. + +Seems like that that's being called from withFilesOldUnlocked, +or withFilesMaybeModified. Both of which Command.Add +calls after withFilesNotInGit. + +With -J, I suppose there could be worker threads still running +to ingest withFilesNotInGit when it moves on to +withFilesOldUnlocked. + +There is a window during file ingestion where the file has been +removed from the working tree and the annex symlink has not been +created yet. Probably that is triggering checkFileOrDirectoryExists. +Although I'd expect that window to be small, so it's somewhat surprising +that yoh could reproduce this problem multiple times. + +The problem could be fixed in several different ways. Could wait for +worker threads to finish before moving on to the next `with*` seek. +Could make checkFileOrDirectoryExists only be run once, rather than 3 +times in `git annex add` (which is surely unncessary work..). Or could +try to eliminate the window where the file is not present in the working +tree. It may be worth doing several of those. +"""]] diff --git a/doc/bugs/add_-J_fails_with_not_found/comment_2_441ded7e239f26ab3a27a2c43a6d0856._comment b/doc/bugs/add_-J_fails_with_not_found/comment_2_441ded7e239f26ab3a27a2c43a6d0856._comment new file mode 100644 index 0000000000..5a1f106182 --- /dev/null +++ b/doc/bugs/add_-J_fails_with_not_found/comment_2_441ded7e239f26ab3a27a2c43a6d0856._comment @@ -0,0 +1,44 @@ +[[!comment format=mdwn + username="joey" + subject="""comment 2""" + date="2017-10-16T16:47:57Z" + content=""" +Here's a way to reproduce it reliably: + + joey@darkstar:~/tmp/tt>for x in $(seq 1 25); do echo $x > $x.f;done + joey@darkstar:~/tmp/tt>git annex add --json -J6 *.f + {"command":"add","success":true,"key":"SHA256E-s2--4355a46b19d348dc2f57c046f8ef63d4538ebb936000f3c9ee954a27460dd865.f","file":"1.f"} + {"command":"add","success":true,"key":"SHA256E-s3--917df3320d778ddbaa5c5c7742bc4046bf803c36ed2b050f30844ed206783469.f","file":"10.f"} + {"command":"add","success":true,"key":"SHA256E-s3--a1fb50e6c86fae1679ef3351296fd6713411a08cf8dd1790a4fd05fae8688164.f","file":"12.f"} + {"command":"add","success":true,"key":"SHA256E-s3--1a252402972f6057fa53cc172b52b9ffca698e18311facd0f3b06ecaaef79e17.f","file":"13.f"} + {"command":"add","success":true,"key":"SHA256E-s3--9a92adbc0cee38ef658c71ce1b1bf8c65668f166bfb213644c895ccb1ad07a25.f","file":"14.f"} + {"command":"add","success":true,"key":"SHA256E-s3--25d4f2a86deb5e2574bb3210b67bb24fcc4afb19f93a7b65a057daa874a9d18e.f","file":"11.f"} + {"command":"add","success":true,"key":"SHA256E-s3--238903180cc104ec2c5d8b3f20c5bc61b389ec0a967df8cc208cdc7cd454174f.f","file":"15.f"} + {"command":"add","success":true,"key":"SHA256E-s3--e6c21e8d260fe71882debdb339d2402a2ca7648529bc2303f48649bce0380017.f","file":"16.f"} + {"command":"add","success":true,"key":"SHA256E-s3--54183f4323f377b737433a1e98229ead0fdc686f93bab057ecb612daa94002b5.f","file":"17.f"} + {"command":"add","success":true,"key":"SHA256E-s3--7ee29791fc17e986b97128845622b077fb45e349fdb80523fac9dba879b4ad60.f","file":"18.f"} + {"command":"add","success":true,"key":"SHA256E-s3--a9742eb8ee320e006666aef25ae9aeed948247f3125c9cafa7cf97b7e7467dd5.f","file":"19.f"} + {"command":"add","success":true,"key":"SHA256E-s2--53c234e5e8472b6ac51c1ae1cab3fe06fad053beb8ebfd8977b010655bfdd3c3.f","file":"2.f"} + {"command":"add","success":true,"key":"SHA256E-s3--5378796307535df3ec8d8b15a2e2dc5641419c3d3060cfe32238c0fa973f7aa3.f","file":"20.f"} + {"command":"add","success":true,"key":"SHA256E-s3--6e2ae11dad0616f66bbb2b6e6556f580bb987fd911d7132aa6bee2bfc7cc7b52.f","file":"21.f"} + {"command":"add","success":true,"key":"SHA256E-s3--f14b4987904bcb5814e4459a057ed4d20f58a633152288a761214dcd28780b56.f","file":"22.f"} + {"command":"add","success":true,"key":"SHA256E-s3--076320a2a08267b4c026d06573bba408ea68841e73cdc20e62cce59de165ece3.f","file":"23.f"} + {"command":"add","success":true,"key":"SHA256E-s3--68ca3fba3b7e864770cb61aeb306d4bd4354b68ab4dd38450860c5d823e42a53.f","file":"24.f"} + {"command":"add","success":true,"key":"SHA256E-s3--64aeb9975f234becd55bb4635e6e2f2da7a6b7bf0a896f0c07763bdfbfb31420.f","file":"25.f"} + {"command":"add","success":true,"key":"SHA256E-s2--06e9d52c1720fca412803e3b07c4b228ff113e303f4c7ab94665319d832bbfb7.f","file":"6.f"} + {"command":"add","success":true,"key":"SHA256E-s2--aa67a169b0bba217aa0aa88a65346920c84c42447c36ba5f7ea65f422c1fe5d8.f","file":"8.f"} + {"command":"add","success":true,"key":"SHA256E-s2--f0b5c2c2211c8d67ed15e75e656c7862d086e9245420892a7de62cd9ec582a06.f","file":"5.f"} + git-annex: 9.f not found + {"command":"add","success":true,"key":"SHA256E-s2--10159baf262b43a92d95db59dae1f72c645127301661e0a3ce4e38b295a97c58.f","file":"7.f"} + {"command":"add","success":true,"key":"SHA256E-s2--7de1555df0c2700329e815b93b32c571c3ea54dc967b89e81ab73b9972b72d1d.f","file":"4.f"} + {"command":"add","success":true,"key":"SHA256E-s2--1121cfccd5913f0a63fec40a6ffd44ea64f9dc135c66634ba001d10bcf4302a2.f","file":"3.f"} + {"command":"add","success":true,"key":"SHA256E-s2--2e6d31a5983a91251bfae5aefa1c0a19d8ba3cf601d0e8a706b4cfa9661a6b8a.f","file":"9.f"} + git-annex: add: 1 failed + +Which file it fails on varies, but it fails on one of the files +every time I've tried this. + +And, there's always a "success" line printed for the same file after +it failed on it. That confirms my hypothesis that the worker thread is +still running at the same time checkFileOrDirectoryExists gets run. +"""]] diff --git a/doc/bugs/add_-J_fails_with_not_found/comment_3_9749c2722b5520e13448128b68fcfaed._comment b/doc/bugs/add_-J_fails_with_not_found/comment_3_9749c2722b5520e13448128b68fcfaed._comment new file mode 100644 index 0000000000..b2ecb4c812 --- /dev/null +++ b/doc/bugs/add_-J_fails_with_not_found/comment_3_9749c2722b5520e13448128b68fcfaed._comment @@ -0,0 +1,20 @@ +[[!comment format=mdwn + username="joey" + subject="""comment 2""" + date="2017-10-16T16:25:39Z" + content=""" +It would be good to avoid the window where the file is not present in the +working tree during ingestion, because interupting `git annex add` during +that window causes the file to go missing, with no record of it yet in git. + +I tried making the window longer by adding a 10 second sleep, and indeed +interrupting `git annex add` during the window is *bad*. + +Looking at the code, `makeLink` uses `replaceFile` to atomicallty +replace the file with the symlink. But `ingestAdd` for some reason +calls `nukeFile` before `makeLink`. I could not find any good reason +for it to do that. So, I've removed the `nukeFile`, closing the window. + +That change also fixed the "file not found" error. But I'm not sure +if it's entirely dealt with the problems raised by this bug report.. +"""]] diff --git a/doc/bugs/add_-J_fails_with_not_found/comment_4_983aa78a20672e4d2b1b74a922eeba0a._comment b/doc/bugs/add_-J_fails_with_not_found/comment_4_983aa78a20672e4d2b1b74a922eeba0a._comment new file mode 100644 index 0000000000..f997b7193a --- /dev/null +++ b/doc/bugs/add_-J_fails_with_not_found/comment_4_983aa78a20672e4d2b1b74a922eeba0a._comment @@ -0,0 +1,22 @@ +[[!comment format=mdwn + username="joey" + subject="""comment 4""" + date="2017-10-16T16:58:46Z" + content=""" +I was worried there could be further races in the seeking +done by withFilesOldUnlocked and withFilesMaybeModified if those +run while files are still being ingested by actions run earlier +in the `git annex add`. Seems this is not a problem though -- + +withFilesOldUnlocked looks for typeChanged files, but the files +that were just/are currently being added were not in git before, +so are not typeChanged. + +withFilesMaybeModified looks for modified files, and again these +files were/are just being added for the first time, so it won't stumble +over them. + +So, I don't think a synchronization point is needed. In fact, +all three seeks could actually be run more concurrently than they are not +without stepping on one-another's toes. +"""]] diff --git a/doc/bugs/add_-J_fails_with_not_found/comment_5_c18b38456d58900f5710a311eced9f34._comment b/doc/bugs/add_-J_fails_with_not_found/comment_5_c18b38456d58900f5710a311eced9f34._comment new file mode 100644 index 0000000000..352cc7a26f --- /dev/null +++ b/doc/bugs/add_-J_fails_with_not_found/comment_5_c18b38456d58900f5710a311eced9f34._comment @@ -0,0 +1,13 @@ +[[!comment format=mdwn + username="joey" + subject="""comment 5""" + date="2017-10-16T17:06:43Z" + content=""" +That leaves only the innefficiency of checkFileOrDirectoryExists being +run three times per parameter passed to `git annex add`. + +There are some other commands that also run checkFileOrDirectoryExists +multiple times. `git annex lock` being one. + +So, I factored that out into a separate pass, that's only done once. +"""]] diff --git a/doc/bugs/add_fails_with_v6_repo_when_four_levels_deep.mdwn b/doc/bugs/add_fails_with_v6_repo_when_four_levels_deep.mdwn new file mode 100644 index 0000000000..34798dc2d1 --- /dev/null +++ b/doc/bugs/add_fails_with_v6_repo_when_four_levels_deep.mdwn @@ -0,0 +1,138 @@ +### Please describe the problem. +I've noticed that adding a modified file to a version 6 repo will throw a number of errors if 'git annex add' is executed four or more directories deep from the git directory. +Though it throws a number of errors, it seems the file is still added normally. +I have also experienced a problem whereby adding a file will replace it with its key (as if I had run 'git annex drop'). +I'm not sure if that problem is related. + +### What steps will reproduce the problem? +In order to cause the exception, a file already in the repo has to be unlocked, edited, and re-added; this re-adding must take place from four levels below the git directory. +Here is a minimal working example: + + git init + git annex init --version=6 + mkdir -p 1/2/3/4 + touch 1/2/3/4/foo + git annex add 1/2/3/4/foo + git annex sync + git annex unlock 1/2/3/4/foo + echo "bar" >> 1/2/3/4/foo + cd 1/2/3/4 + git annex add foo + +Specifically, trying to run 'git annex add foo' will result in the following errors being thrown: + + fatal: '../1/2/3/4/foo' is outside repository + fatal: '../1/2/3/4/foo' is outside repository + fatal: '../1/2/3/4/foo' is outside repository + fatal: '../1/2/3/4/foo' is outside repository + fatal: '../1/2/3/4/foo' is outside repository + fatal: '../1/2/3/4/foo' is outside repository + fatal: '../1/2/3/4/foo' is outside repository + fatal: '../1/2/3/4/foo' is outside repository + fatal: '../1/2/3/4/foo' is outside repository + fatal: '../1/2/3/4/foo' is outside repository + fatal: '../1/2/3/4/foo' is outside repository + git-annex: git check-attr EOF: user error + git-annex: smudge: 1 failed + error: external filter git-annex smudge --clean %f failed 1 + error: external filter git-annex smudge --clean %f failed + add foo ok + (recording state in git...) + + +### What version of git-annex are you using? On what operating system? +I'm currently running version 6.20160720-g9f0428e on Arch Linux (x86), though I've experienced this problem since at least February. + +### Please provide any additional information below. + +[[!format sh """ +# If you can, paste a complete transcript of the problem occurring here. +# If the problem is with the git-annex assistant, paste in .git/annex/daemon.log +$ git init; git annex init --version=6 +Initialized empty Git repository in /mnt/.git/ +init ok +(recording state in git...) +$ mkdir -p 1/2/3/4 +$ touch 1/2/3/4/foo +$ git annex add 1/2/3/4/foo +add 1/2/3/4/foo ok +(recording state in git...) +$ git annex sync +commit +[master (root-commit) 25e1676] git-annex in +:/mntt + 1 file changed, 1 insertion(+) + create mode 120000 1/2/3/4/foo +ok +$ git annex unlock 1/2/3/4/foo +unlock 1/2/3/4/foo ok +(recording state in git...) +$ echo "bar" >> 1/2/3/4/foo +$ cd 1/2/3/4 +$ git annex add foo --debug +[2016-08-23 14:47:21.764271] read: git +["--git-dir=../../../../.git","--work-tree=../../../..","--literal-pathspecs","ls-files","--others","--exclude-standard","-z","--","foo"] +[2016-08-23 14:47:21.766481] read: git +["--git-dir=../../../../.git","--work-tree=../../../..","--literal-pathspecs","ls-files","--modified","-z","--","foo"] +fatal: '../1/2/3/4/foo' is outside repository +fatal: '../1/2/3/4/foo' is outside repository +fatal: '../1/2/3/4/foo' is outside repository +fatal: '../1/2/3/4/foo' is outside repository +fatal: '../1/2/3/4/foo' is outside repository +fatal: '../1/2/3/4/foo' is outside repository +fatal: '../1/2/3/4/foo' is outside repository +fatal: '../1/2/3/4/foo' is outside repository +fatal: '../1/2/3/4/foo' is outside repository +fatal: '../1/2/3/4/foo' is outside repository +fatal: '../1/2/3/4/foo' is outside repository +git-annex: git check-attr EOF: user error +git-annex: smudge: 1 failed +error: external filter git-annex smudge --clean %f failed 1 +error: external filter git-annex smudge --clean %f failed +[2016-08-23 14:47:21.806128] chat: git +["--git-dir=../../../../.git","--work-tree=../../../..","--literal-pathspecs","check-attr","-z","--stdin","annex.backend","annex.numcopies","annex.largefiles","--"] +[2016-08-23 14:47:21.806624] read: git ["--version"] +[2016-08-23 14:47:21.809352] process done ExitSuccess +[2016-08-23 14:47:21.813764] chat: git +["--git-dir=../../../../.git","--work-tree=../../../..","--literal-pathspecs","cat-file","--batch"] +add foo [2016-08-23 14:47:21.818268] read: git +["--git-dir=../../../../.git","--work-tree=../../../..","--literal-pathspecs","symbolic-ref","-q","HEAD"] +[2016-08-23 14:47:21.82027] process done ExitSuccess +[2016-08-23 14:47:21.822862] read: git +["--git-dir=../../../../.git","--work-tree=../../../..","--literal-pathspecs","show-ref","git-annex"] +[2016-08-23 14:47:21.825102] process done ExitSuccess +[2016-08-23 14:47:21.825233] read: git +["--git-dir=../../../../.git","--work-tree=../../../..","--literal-pathspecs","show-ref","--hash","refs/heads/git-annex"] +[2016-08-23 14:47:21.82715] process done ExitSuccess +[2016-08-23 14:47:21.828549] chat: git +["--git-dir=../../../../.git","--work-tree=../../../..","--literal-pathspecs","cat-file","--batch"] +ok +(recording state in git...) +[2016-08-23 14:47:21.832848] feed: xargs +["-0","git","--git-dir=../../../../.git","--work-tree=../../../..","--literal-pathspecs","add","--"] +[2016-08-23 14:47:21.836822] process done ExitSuccess +[2016-08-23 14:47:21.837518] chat: git +["--git-dir=../../../.git","--work-tree=../../../..","--literal-pathspecs","hash-object","-w","--stdin-paths","--no-filters"] +[2016-08-23 14:47:21.838061] feed: git +["--git-dir=../../../.git","--work-tree=../../../..","--literal-pathspecs","update-index","-z","--index-info"] +[2016-08-23 14:47:21.843259] process done ExitSuccess +[2016-08-23 14:47:21.843444] read: git +["--git-dir=../../../.git","--work-tree=../../../..","--literal-pathspecs","show-ref","--hash","refs/heads/git-annex"] +[2016-08-23 14:47:21.847287] process done ExitSuccess +[2016-08-23 14:47:21.847602] read: git +["--git-dir=../../../.git","--work-tree=../../../..","--literal-pathspecs","write-tree"] +[2016-08-23 14:47:21.85106] process done ExitSuccess +[2016-08-23 14:47:21.851221] chat: git +["--git-dir=../../../.git","--work-tree=../../../..","--literal-pathspecs","commit-tree","b4a158d15da89e28ef5c2f1782c5b1e3c6f1176c","--no-gpg-sign","-p","refs/heads/git-annex"] +[2016-08-23 14:47:21.85892] process done ExitSuccess +[2016-08-23 14:47:21.85907] call: git +["--git-dir=../../../.git","--work-tree=../../../..","--literal-pathspecs","update-ref","refs/heads/git-annex","68381bac2ba0f559d37214c30b5e41a404b9c98b"] +[2016-08-23 14:47:21.861978] process done ExitSuccess +# End of transcript or log. +"""]] + +### Have you had any luck using git-annex before? (Sometimes we get tired of reading bug reports all day and a lil' positive end note does wonders) +I've been happily using git-annex on my laptop and server for almost a year; it's saved me a great deal of time and effort. +Thanks for all your work! + +> [[fixed|done]] finally... --[[Joey]] diff --git a/doc/bugs/add_fails_with_v6_repo_when_four_levels_deep/comment_10_43aa4935ee42abc90546d166042d379b._comment b/doc/bugs/add_fails_with_v6_repo_when_four_levels_deep/comment_10_43aa4935ee42abc90546d166042d379b._comment new file mode 100644 index 0000000000..3bf8896294 --- /dev/null +++ b/doc/bugs/add_fails_with_v6_repo_when_four_levels_deep/comment_10_43aa4935ee42abc90546d166042d379b._comment @@ -0,0 +1,16 @@ +[[!comment format=mdwn + username="t.z.mates" + avatar="http://cdn.libravatar.org/avatar/90f15fad216078fd08d62cc676487925" + subject="Error messages changed" + date="2017-06-15T22:59:42Z" + content=""" +Just a quick follow-up; it seems that one of the recent updates (I believe the version published of 11-June-2017) modified the error messages shown. Running the same code above, I no longer receive a string of `fatal: '../1/2/3/4/foo' is outside repository` followed by `git-annex: git check-attr EOF: user error`. Instead, it shows `git-annex: 1/2/3/4/foo: getFileStatus: does not exist (No such file or directory)`, followed by the rest of the output I originally posted. That is, the output is now: + + git-annex: 1/2/3/4/foo: getFileStatus: does not exist (No such file or directory) + git-annex: smudge: 1 failed + error: external filter 'git-annex smudge --clean %f' failed 1 + error: external filter 'git-annex smudge --clean %f' failed + add foo ok + +I don't know what this represents, but I find it interesting that something was changed recently. +"""]] diff --git a/doc/bugs/add_fails_with_v6_repo_when_four_levels_deep/comment_11_499cdb675327aaed59877226f77b7077._comment b/doc/bugs/add_fails_with_v6_repo_when_four_levels_deep/comment_11_499cdb675327aaed59877226f77b7077._comment new file mode 100644 index 0000000000..a92d8821d6 --- /dev/null +++ b/doc/bugs/add_fails_with_v6_repo_when_four_levels_deep/comment_11_499cdb675327aaed59877226f77b7077._comment @@ -0,0 +1,8 @@ +[[!comment format=mdwn + username="t.z.mates" + avatar="http://cdn.libravatar.org/avatar/90f15fad216078fd08d62cc676487925" + subject="comment 11" + date="2017-06-15T23:07:48Z" + content=""" +More specifically, I noticed it first on version 6.20170519-gc6079c3ce8 (it's possible it was one or two commits before that, but I pull updates daily, so the change occured at most a day before). +"""]] diff --git a/doc/bugs/add_fails_with_v6_repo_when_four_levels_deep/comment_12_d54f9cd6470cbaf6f066513e6fc66f69._comment b/doc/bugs/add_fails_with_v6_repo_when_four_levels_deep/comment_12_d54f9cd6470cbaf6f066513e6fc66f69._comment new file mode 100644 index 0000000000..924e0a4c0b --- /dev/null +++ b/doc/bugs/add_fails_with_v6_repo_when_four_levels_deep/comment_12_d54f9cd6470cbaf6f066513e6fc66f69._comment @@ -0,0 +1,80 @@ +[[!comment format=mdwn + username="joey" + subject="""comment 12""" + date="2018-07-17T15:52:47Z" + content=""" +Reproduced this today. ext4, git 2.18.0, git-annex 6.20180627-gbd6799ebe. +In an adjusted unlocked v6 repository. + +Running git annex add in 1/2/3/4/5/6/ failed, I then cd'd down to the top +of the repository, and it worked there. + +The bug seems to be intermittent; I was then not able to reproduce it for +a while, and now can again. + + joey@darkstar:~/tmp/t2/1/2/3/4/5/6#master(unlocked)>git annex add meow --debug + [2018-07-17 12:15:35.536935635] read: git ["--git-dir=../../../../../../.git","--work-tree=../../../../../..","--literal-pathspecs","ls-files","--others","--exclude-standard","-z","--","meow"] + [2018-07-17 12:15:35.542108221] read: git ["--git-dir=../../../../../../.git","--work-tree=../../../../../..","--literal-pathspecs","ls-files","--modified","-z","--","meow"] + git-annex: 1/2/3/4/5/6/meow: getFileStatus: does not exist (No such file or directory) + git-annex: smudge: 1 failed + error: external filter 'git-annex smudge --clean %f' failed 1 + error: external filter 'git-annex smudge --clean %f' failed + +This is git ls-files running the smudge filter which then fails: + + joey@darkstar:~/tmp/t2/1/2/3/4/5/6#master(unlocked)>GIT_TRACE=1 git + --git-dir=../../../../../../.git --work-tree=../../../../../.. + --literal-pathspecs ls-files --modified + + 12:19:49.691639 git.c:415 trace: built-in: git ls-files --modified + 12:19:49.692166 run-command.c:637 trace: run_command: 'git-annex smudge --clean '\''1/2/3/4/5/6/meow'\''' + 12:19:49.700739 git.c:415 trace: built-in: git config --null --list + 12:19:49.704813 git.c:415 trace: built-in: git check-attr -z --stdin annex.backend annex.numcopies annex.largefiles -- + 12:19:49.705020 git.c:415 trace: built-in: git version + 12:19:49.707887 git.c:415 trace: built-in: git cat-file --batch + 12:19:49.707880 git.c:415 trace: built-in: git cat-file '--batch-check=%(objectname) %(objecttype) %(objectsize)' + git-annex: 1/2/3/4/5/6/meow: getFileStatus: does not exist (No such file or directory) + git-annex: smudge: 1 failed + +Note that the smudge filter is being passed the path from the top of the +repo to the file, despite being in the same directory as the file. +Same thing happens during a successful add though, so why is it +failing to process the filename here? Maybe git is normally running +the smudge filter with cwd in the top of the repo, but when this occurs +it's elsewhere? No, I checked and the smudge filter is always being run +at the top of the repository. + +Huh! It's somehow caused by the way that the path to the work-tree is +specified. + + joey@darkstar:~/tmp/t2/1/2/3/4/5/6#master(unlocked)>GIT_TRACE=1 git --work-tree=/home/joey/tmp/t2 --literal-pathspecs ls-files --modified12:32:34.639710 git.c:415 trace: built-in: git ls-files --modified + 12:32:34.640571 run-command.c:637 trace: run_command: 'git-annex smudge --clean '\''1/2/3/4/5/6/meow'\''' + # succeeds no error + +That's two different ways to specifiy the same path, one succeeds and one +fails. + +Hmm.. when git runs the smudge filter, pwd is the top of the repository +and `GIT_WORK_TREE=../../../../../..` +So the work tree in this case points outside the repository, +indeed it points to `/`. I guess this is why the smudge filter is +failing, since it's not finding the file in that wrong worktree. + +This is absolutely a bug in git, I posted to the git mailing list +in message-id 20180717165834.GA5615@kitenet.net about it. + +But why does this only happen sometimes? Well, running `git add` +in a subdirectory does not pass `GIT_WORK_TREE`, because none +was specified. So, the problem does not occur then. +When git-annex runs a git command such as ls-files --modified and passes +a relative work tree, the problem also sometimes doesn't occur. +Sometimes git doesn't need to run the smudge filter +(touching the file makes it need to). +I think it may have to do with the number of subdirectories somehow as +well. + +Since git 1.7.6, `GIT_PREFIX` is set when it runs these filters, +and it contains the path to the subdirectory of the working tree that +it was originally run in. So, git-annex can work around the +problem using that. Done. +"""]] diff --git a/doc/bugs/add_fails_with_v6_repo_when_four_levels_deep/comment_13_61f13e76890340bd5ae97dab6d6c0141._comment b/doc/bugs/add_fails_with_v6_repo_when_four_levels_deep/comment_13_61f13e76890340bd5ae97dab6d6c0141._comment new file mode 100644 index 0000000000..ada3fd8e18 --- /dev/null +++ b/doc/bugs/add_fails_with_v6_repo_when_four_levels_deep/comment_13_61f13e76890340bd5ae97dab6d6c0141._comment @@ -0,0 +1,12 @@ +[[!comment format=mdwn + username="t.z.mates" + avatar="http://cdn.libravatar.org/avatar/90f15fad216078fd08d62cc676487925" + subject="Thanks!" + date="2018-07-19T19:44:25Z" + content=""" +Wow, I'm really appreciate the in-depth analysis. Unfortunately, I'm not familiar enough with the inner workings of git/git-annex to dig that deep, so I'm happy you were able to (intermittently) reproduce it. In particular, it seems you've done some great work with the smudge filters, but that's all way above my head still. + +That's good to know that it was actually a bug in git itself. Thanks for submitting the bug report; hopefully that takes care of the problem. + +Using `GIT_PREFIX` is a clever work-around. I appreciate all your help with this! +"""]] diff --git a/doc/bugs/add_fails_with_v6_repo_when_four_levels_deep/comment_1_e308245bf81a536db6f9a2b743d912bf._comment b/doc/bugs/add_fails_with_v6_repo_when_four_levels_deep/comment_1_e308245bf81a536db6f9a2b743d912bf._comment new file mode 100644 index 0000000000..a5d988faeb --- /dev/null +++ b/doc/bugs/add_fails_with_v6_repo_when_four_levels_deep/comment_1_e308245bf81a536db6f9a2b743d912bf._comment @@ -0,0 +1,10 @@ +[[!comment format=mdwn + username="joey" + subject="""comment 1""" + date="2016-11-16T18:49:09Z" + content=""" +I'm not able to reproduce the problem with your test case and git-annex +version 6.20161012. + +Can you still reproduce it after upgrading? +"""]] diff --git a/doc/bugs/add_fails_with_v6_repo_when_four_levels_deep/comment_2_b3998823aca4266089dcbcf325d8f8c1._comment b/doc/bugs/add_fails_with_v6_repo_when_four_levels_deep/comment_2_b3998823aca4266089dcbcf325d8f8c1._comment new file mode 100644 index 0000000000..1046fb0665 --- /dev/null +++ b/doc/bugs/add_fails_with_v6_repo_when_four_levels_deep/comment_2_b3998823aca4266089dcbcf325d8f8c1._comment @@ -0,0 +1,12 @@ +[[!comment format=mdwn + username="t.z.mates" + avatar="http://cdn.libravatar.org/avatar/90f15fad216078fd08d62cc676487925" + subject="comment 2" + date="2016-11-19T04:42:25Z" + content=""" +Thanks for looking into it; I just checked again, and even on the newest version (6.20161118 binary), I'm still experiencing the behavior. However, I checked on an older OpenSuse box I have, and there it works (6.20161031 from OpenSuse repo). + +Since my two machines experiencing the problem are both running arch, it seems it's somehow related to that distro. I've checked both installing via the binary (from kitenet) and from the arch community repo, but both produce the same behavior. Further, the OpenSuse install has the same build flags as the binaries, so that doesn't seem to be it. Are there any other diagnostics I can run? + +This particular problem isn't very troublesome (it doesn't seem to have any material impact aside from error messages); however, I also occasionally experience a more serious bug. Namely, when certain (seemingly random) files are added to the repo locked, their content disappears and the symlink is broken (this is the other problem I alluded to in the description). I suspect that problem is related to this one though, since it also only affects my arch machines. I haven't yet submitted a report for that bug yet, though, since I can't reliably replicate it. +"""]] diff --git a/doc/bugs/add_fails_with_v6_repo_when_four_levels_deep/comment_3_d74835534f52c7f123b14e5d74194733._comment b/doc/bugs/add_fails_with_v6_repo_when_four_levels_deep/comment_3_d74835534f52c7f123b14e5d74194733._comment new file mode 100644 index 0000000000..918bb4bac7 --- /dev/null +++ b/doc/bugs/add_fails_with_v6_repo_when_four_levels_deep/comment_3_d74835534f52c7f123b14e5d74194733._comment @@ -0,0 +1,7 @@ +[[!comment format=mdwn + username="joey" + subject="""comment 3""" + date="2016-12-13T16:54:11Z" + content=""" +Perhaps it's caused by a particular/old version of git? +"""]] diff --git a/doc/bugs/add_fails_with_v6_repo_when_four_levels_deep/comment_4_f9d6dffb2617715c58216f54016de3a4._comment b/doc/bugs/add_fails_with_v6_repo_when_four_levels_deep/comment_4_f9d6dffb2617715c58216f54016de3a4._comment new file mode 100644 index 0000000000..af0b2030b9 --- /dev/null +++ b/doc/bugs/add_fails_with_v6_repo_when_four_levels_deep/comment_4_f9d6dffb2617715c58216f54016de3a4._comment @@ -0,0 +1,13 @@ +[[!comment format=mdwn + username="t.z.mates" + avatar="http://cdn.libravatar.org/avatar/90f15fad216078fd08d62cc676487925" + subject="comment 4" + date="2016-12-20T23:08:44Z" + content=""" +Hmm, I don't think an old version of git is the cause. I'm currently running the most recent build of git (2.11.0), but have used a number of versions over the past year. + +I'm not sure if this is relevant, but this other bug reports similar behavior: [sync --content, fatal is outside repository errors](https://git-annex.branchable.com/forum/sync_--content__44___fatal_is_outside_repository_errors/). Specifically, it notes that there is an odd use of relative paths: +> The relative path ../Users is curious + +My error also appends an extra period. In particular, the path should be \"./1/2/3/4/foo\" but prints \"../1/2/3/4/foo\". +"""]] diff --git a/doc/bugs/add_fails_with_v6_repo_when_four_levels_deep/comment_5_9b9a369fd07cf966bdb9a44699c0d8a4._comment b/doc/bugs/add_fails_with_v6_repo_when_four_levels_deep/comment_5_9b9a369fd07cf966bdb9a44699c0d8a4._comment new file mode 100644 index 0000000000..340d3e351c --- /dev/null +++ b/doc/bugs/add_fails_with_v6_repo_when_four_levels_deep/comment_5_9b9a369fd07cf966bdb9a44699c0d8a4._comment @@ -0,0 +1,25 @@ +[[!comment format=mdwn + username="t.z.mates" + avatar="http://cdn.libravatar.org/avatar/90f15fad216078fd08d62cc676487925" + subject="comment 5" + date="2017-01-08T06:43:48Z" + content=""" +So, I've done a bit more investigating, and it seems the specific command that causes the error is + + git --git-dir=../../../../.git --work-tree=../../../.. --literal-pathspecs ls-files --modified -- foo +In particular, if I execute the code: + + git init + git annex init --version=6 + mkdir -p 1/2/3/4 + touch 1/2/3/4/foo + git annex add 1/2/3/4/foo + git annex sync + git annex unlock 1/2/3/4/foo + echo \"bar\" >> 1/2/3/4/foo + cd 1/2/3/4 + git --git-dir=../../../../.git --work-tree=../../../.. --literal-pathspecs ls-files --modified -- foo +I get the above mentioned error. However, if I run the exact same code without any of the \"git annex\" commands (i.e. only initializing a standard git repository), the ls-files commands returns without error. + +I'm not sure what to make of this though; it doesn't seem to be any sort of corrupted log or bad config options (I ran the same commands on a different, working computer, copied the .git directory to the non-working computer, and still couldn't run the ls-files command). I'm rather at a loss of what to check next. +"""]] diff --git a/doc/bugs/add_fails_with_v6_repo_when_four_levels_deep/comment_6_ce19be99100dfadc9e69729a7af98dff._comment b/doc/bugs/add_fails_with_v6_repo_when_four_levels_deep/comment_6_ce19be99100dfadc9e69729a7af98dff._comment new file mode 100644 index 0000000000..bba777c388 --- /dev/null +++ b/doc/bugs/add_fails_with_v6_repo_when_four_levels_deep/comment_6_ce19be99100dfadc9e69729a7af98dff._comment @@ -0,0 +1,19 @@ +[[!comment format=mdwn + username="joey" + subject="""comment 6""" + date="2017-02-20T16:27:39Z" + content=""" +I also can't reproduce any problem running the script from comment #5 +here. + +(The reason that `git ls-files --modified` is +throwing the error is because that ends up running git-annex +when in a v6 repo. + +Seems like it must come down to the version of git, or some +other part of the environment, since it is only happening on one computer +and not others. + +Are there any symlinks in the path to the repository where this happens? +Does this happen if you run the script in `/tmp/newrepo`? +"""]] diff --git a/doc/bugs/add_fails_with_v6_repo_when_four_levels_deep/comment_7_fc053d15c8a634fc7be744ee51470fc6._comment b/doc/bugs/add_fails_with_v6_repo_when_four_levels_deep/comment_7_fc053d15c8a634fc7be744ee51470fc6._comment new file mode 100644 index 0000000000..e2d331ef5a --- /dev/null +++ b/doc/bugs/add_fails_with_v6_repo_when_four_levels_deep/comment_7_fc053d15c8a634fc7be744ee51470fc6._comment @@ -0,0 +1,10 @@ +[[!comment format=mdwn + username="t.z.mates" + avatar="http://cdn.libravatar.org/avatar/90f15fad216078fd08d62cc676487925" + subject="comment 7" + date="2017-04-02T04:17:55Z" + content=""" +Thanks for the suggestions! Just to clarrify, I've tried this on three of my machines: machines 1 and 2 both run arch linux and both experience the same error (even in `/tmp/newrepo`); machine 3 runs opensuse but does not experience the error. + +However, all three machines are using BTRFS for the underlying filesystem. I next tried running the script (on machine 1) in a partition that has Copy-on-Write turned off, and it completed successfully. So it seems that CoW has something to do with it (though the script runs fine even with CoW enabled on machine 3). So it seems we got a bit closer, but I'm still not sure what to make of it. +"""]] diff --git a/doc/bugs/add_fails_with_v6_repo_when_four_levels_deep/comment_8_a6c330f5ad2f64d86d8a24c23c49115a._comment b/doc/bugs/add_fails_with_v6_repo_when_four_levels_deep/comment_8_a6c330f5ad2f64d86d8a24c23c49115a._comment new file mode 100644 index 0000000000..5ea68b6850 --- /dev/null +++ b/doc/bugs/add_fails_with_v6_repo_when_four_levels_deep/comment_8_a6c330f5ad2f64d86d8a24c23c49115a._comment @@ -0,0 +1,21 @@ +[[!comment format=mdwn + username="joey" + subject="""comment 8""" + date="2017-04-05T15:26:59Z" + content=""" +git-annex does use CoW in a few situations; it does so by running `cp +--reflink` and letting it use CoW features when available. However, +I don't see where `git annex add` would use that, and of course I don't see +why that feature would break for you even if it did run cp that way. + +git also uses CoW, at least it uses `mmap` with `MAP_PRIVATE`. I'm not +clear if/how that involves the filesystem layer. + +This remains puzzling, but knowing it's limited to btrfs on Arch Linux +with CoW is certianly a good start. + +It seems like a bug in btrfs would not be out of the question. +Trying some different kernel versions might be useful. + +It would perhaps be useful to get `strace -ff` logs. +"""]] diff --git a/doc/bugs/add_fails_with_v6_repo_when_four_levels_deep/comment_9_9b91f93a66d3e38edc4490be20445a6d._comment b/doc/bugs/add_fails_with_v6_repo_when_four_levels_deep/comment_9_9b91f93a66d3e38edc4490be20445a6d._comment new file mode 100644 index 0000000000..586dd6b3a7 --- /dev/null +++ b/doc/bugs/add_fails_with_v6_repo_when_four_levels_deep/comment_9_9b91f93a66d3e38edc4490be20445a6d._comment @@ -0,0 +1,14 @@ +[[!comment format=mdwn + username="t.z.mates" + avatar="http://cdn.libravatar.org/avatar/90f15fad216078fd08d62cc676487925" + subject="comment 9" + date="2017-05-13T20:59:34Z" + content=""" +Thanks for the response! I've [linked](https://github.com/tzmates/GitAnnexStrace) to a repo where I posted the strace results (only for the offending line, otherwise there were a lot of files) as well as the code that generated it. I'm not very familiar with strace results, so let me know if I should supply anything else. + +It certainly could be a BTRFS bug, though it doesn't seem to be tied to a specific kernel version; I've been continually updating kernels for the last 2 years now. + +If it's any help, I've never had this sort of problem using `cp --reflink` day-to-day. I have no experience with `mmap` though, so that's a distinct possibility. + +Thanks for all your input on this so far! +"""]] diff --git a/doc/bugs/added_branches_makes___39__git_annex_unused__39___slow.mdwn b/doc/bugs/added_branches_makes___39__git_annex_unused__39___slow.mdwn new file mode 100644 index 0000000000..95751527b3 --- /dev/null +++ b/doc/bugs/added_branches_makes___39__git_annex_unused__39___slow.mdwn @@ -0,0 +1,87 @@ +Creating additional branches in history seems to slow down the 'git annex unused' command quadratically, even if the location of the branches should be irrelevant as far as unused data goes. + +This was tested on: + + $ git annex version + git-annex version: 3.20130216 + local repository version: 3 + default repository version: 3 + supported repository versions: 3 + upgrade supported from repository versions: 0 1 2 + +What steps will reproduce the problem? + + $ mkdir a + $ cd a + $ git init + $ git annex init + $ i=0 ; while test $i -lt 1000; do dd if=/dev/urandom of=$i.img bs=1M count=1; i=$(($i+1)); done + $ git annex add . + $ git commit -m"foo" + $ git rm 1* + $ git commit -m"bar" + $ git log --oneline --decorate + ffcca3a (HEAD, master) bar + 3e7793d foo + $ time -p git annex unused + unused . (checking for unused data...) (checking master...) + (...) + real 0.76 + user 0.40 + sys 0.06 + git commit --allow-empty -m"baz" + $ git log --oneline --decorate + 4390c32 (HEAD, master) baz + ffcca3a bar + 3e7793d foo + $ time -p git annex unused + unused . (checking for unused data...) (checking master...) + (...) + real 0.75 + user 0.38 + sys 0.07 + $ git branch boo HEAD^ + $ time -p git annex unused + unused . (checking for unused data...) (checking boo...) (checking master...) + (...) + real 1.29 + user 0.62 + sys 0.08 + arand@mas:~/tmp/more/a(master)$ git branch beeboo HEAD^ + 4390c32 (HEAD, master) baz + ffcca3a (boo, beeboo) bar + 3e7793d foo + arand@mas:~/tmp/more/a(master)$ time -p git annex unused + unused . (checking for unused data...) (checking beeboo...) (checking master...) + (...) + real 2.50 + user 1.12 + sys 0.14 + $ git branch -d boo beeboo + $ git log --oneline --decorate + 4390c32 (HEAD, master) baz + ffcca3a bar + 3e7793d foo + $ time -p git annex unused + unused . (checking for unused data...) (checking master...) + (...) + real 0.77 + user 0.42 + sys 0.04 + +What is the expected output? What do you see instead? + +I would expect the time to be the same in all the above cases. + +What version of git-annex are you using? On what operating system? + + $ git annex version + git-annex version: 3.20130216 + +On current Debian sid/experimental + +> [[Done]], thanks to guilhem. We ended up using a different algorythm +> which is faster yet, basically it now does a diff-index between the +> index and each branch for its second stage bloom filter. +> Speedup is 30x with 0 (or 1?) branch, and then massive for each +> additional branch. --[[Joey]] diff --git a/doc/bugs/added_branches_makes___39__git_annex_unused__39___slow/comment_1_d350c39c67031c500e3224e92c0029ea._comment b/doc/bugs/added_branches_makes___39__git_annex_unused__39___slow/comment_1_d350c39c67031c500e3224e92c0029ea._comment new file mode 100644 index 0000000000..f8b7e789b5 --- /dev/null +++ b/doc/bugs/added_branches_makes___39__git_annex_unused__39___slow/comment_1_d350c39c67031c500e3224e92c0029ea._comment @@ -0,0 +1,19 @@ +[[!comment format=mdwn + username="http://joeyh.name/" + ip="4.152.108.210" + subject="comment 1" + date="2013-02-26T20:29:05Z" + content=""" +`git annex unused` finds content that is not used by the working tree or by *any* branch is unused. For the working tree, it can look at the symlinks on disk, which is the fastest option. + +For branches, it has to first use `git ls-tree` to find the files on the branch, and then use `git cat-file` to look up the key used by each file. It does this as fast as it can (eg, it runs a single `git cat-file --batch`, rather than one process per file). Still, this is pulling potentially a lot of data out of git, and it gets pretty slow. + +I've spent a lot of time optimising this as much as is possible with these constraints. One nice one is that, if it finds no unused keys after checking the working tree, it can stop, rather than checking any branches. Your example avoids this optimisation. + +Another optimisation is to only check each git ref once, even if multiple branches refer to it. You can see this optmisation firing in your transcript, when it only shows it's checking one branch of the two identical branches you've made. + +Indeed, if you go on and add 100 identical branches, you'll find it runs in just about the same time it ran with 2 branches. (There's a little overhead in getting the list of branches and throwing out the duplicates, but that's all.) + +What then explains your numbers? Well, I have no idea. I cannot replicate them; I tend to see about the same amount of time taken with two duplicate branches as with one branch. I suspect you just didn't get statistically valid results, which playing around with `time` at the command line often doesn't, +due to caching, other active processes, etc. +"""]] diff --git a/doc/bugs/added_branches_makes___39__git_annex_unused__39___slow/comment_2_b2d2b1caa51ffec3d87c36b373cb8d4a._comment b/doc/bugs/added_branches_makes___39__git_annex_unused__39___slow/comment_2_b2d2b1caa51ffec3d87c36b373cb8d4a._comment new file mode 100644 index 0000000000..0954e89880 --- /dev/null +++ b/doc/bugs/added_branches_makes___39__git_annex_unused__39___slow/comment_2_b2d2b1caa51ffec3d87c36b373cb8d4a._comment @@ -0,0 +1,20 @@ +[[!comment format=mdwn + username="https://launchpad.net/~arand" + nickname="arand" + subject="comment 2" + date="2013-02-27T15:53:37Z" + content=""" +Hmm, indeed, after further testing it seems like the increased time due to the duplicate branch seems to have been a random quirk, bleh :( + +But shouldn't it theoretically be possible to optimize out much of the overhead of multiple very-similar (though not identical) branches though? + +I've experimented around with ls-tree and cat-file in a bash script[1], and in this primitive implementation the running time seems to be considerably lower (~0.3s vs ~4s), with much less overhead for extra very-similar branches (~0.7s vs ~37s) + +Am I missing some key element that's the reason for the time taken by git annex unused? + + +[1] primitive annex unused script: [https://gitorious.org/arand-scripts/arand-scripts/blobs/master/annex-funused](https://gitorious.org/arand-scripts/arand-scripts/blobs/master/annex-funused) + +timing script: [https://gitorious.org/arand-scripts/arand-scripts/blobs/master/annex-testunused](https://gitorious.org/arand-scripts/arand-scripts/blobs/master/annex-testunused) + +"""]] diff --git a/doc/bugs/added_branches_makes___39__git_annex_unused__39___slow/comment_3_12b20cbbc2b4cd1ab8af7e3eec9589b4._comment b/doc/bugs/added_branches_makes___39__git_annex_unused__39___slow/comment_3_12b20cbbc2b4cd1ab8af7e3eec9589b4._comment new file mode 100644 index 0000000000..9c1da8eeaa --- /dev/null +++ b/doc/bugs/added_branches_makes___39__git_annex_unused__39___slow/comment_3_12b20cbbc2b4cd1ab8af7e3eec9589b4._comment @@ -0,0 +1,30 @@ +[[!comment format=mdwn + username="arand" + ip="130.243.226.21" + subject="comment 3" + date="2013-08-10T17:00:21Z" + content=""" +So, if I've understood it correctly (please correct me if that's not the case :) ) + +Currently git-annex unused goes through this process + +* Look through all files in the index and find those which are git-annex keys (git ls-tree + git cat-file) +* Look through all files the current ref and find those which are git-annex keys (git ls-tree + git cat-file) +* For each ref in the repo + - Look through all files and find those which are git-annex keys (git ls-tree + git cat-file) +* Then at the end + - Compare this list of keys with what is stored in .git/annex/objects + - Print out any objects which does not match a key. + +If that's the case, it means if that if you have multiple refs, even is they only differ by single empty commits, git-annex will end up doing a cat-file for the same file multiple times (one per ref), which is expensive. + +Would it be possible to change the algorithm for git-annex unused into instead something like: + +* For the index, HEAD, and all refs + - Create a list all files and remove those which are duplicates based on their sha1 hash (git ls-tree | uniq) +* Then Look through this reduced list to find those which are git-annex keys (git cat-file) +* Then check as before + +Unless this bypasses some safety or case I've overlooked, I think it should be possible to speed up git-annex unused quite a bit. + +"""]] diff --git a/doc/bugs/added_branches_makes___39__git_annex_unused__39___slow/comment_4_a50b43c15d2650df90f0fa1ced47f532._comment b/doc/bugs/added_branches_makes___39__git_annex_unused__39___slow/comment_4_a50b43c15d2650df90f0fa1ced47f532._comment new file mode 100644 index 0000000000..3de85407e3 --- /dev/null +++ b/doc/bugs/added_branches_makes___39__git_annex_unused__39___slow/comment_4_a50b43c15d2650df90f0fa1ced47f532._comment @@ -0,0 +1,10 @@ +[[!comment format=mdwn + username="http://joeyh.name/" + ip="2001:4978:f:21a::2" + subject="comment 4" + date="2013-08-11T12:48:47Z" + content=""" +I think that could work. It would probably tend to use more memory than the current method, but only a small constant multiplier more. And unused is already the one command that necessarily needs to hold information about the whole repository in memory. + +Note that git cat-file is only needed when dealing with branches other than the current working tree. In that special case, it can, and AFAIK does have the optimisation of looking directly at the symlink target instead. Your method may turn out to be both slower and use more memory in that case. It may make sense to special case handling of that case and keep the current code paths there (most of the necessary code for it is used by other stuff anyway). +"""]] diff --git a/doc/bugs/added_branches_makes___39__git_annex_unused__39___slow/comment_5_7328bc51bd001f2b732a92a2ae175839._comment b/doc/bugs/added_branches_makes___39__git_annex_unused__39___slow/comment_5_7328bc51bd001f2b732a92a2ae175839._comment new file mode 100644 index 0000000000..21890a4e1e --- /dev/null +++ b/doc/bugs/added_branches_makes___39__git_annex_unused__39___slow/comment_5_7328bc51bd001f2b732a92a2ae175839._comment @@ -0,0 +1,114 @@ +[[!comment format=mdwn + username="arand" + ip="130.243.226.21" + subject="comment 5" + date="2013-08-11T20:43:22Z" + content=""" +I've compared my bash/coreutils implementation mentioned above [annex-funused](https://gitorious.org/arand-scripts/arand-scripts/blobs/918cc79b99e22cbdca01ea4de10e2ca64abfc27a/annex-funused) with `git annex unused` in various situations, and from what I've seen `annex-funused` is pretty much always faster. + +In the case of no unused files they seem to be about the same. + +In all other cases there is a very considerable difference, for example, in my current main annex I get: + + $ time git annex unused + unused . (checking for unused data...) (checking master...) (checking synced/master...) (checking barracuda160G/master...) ok + + real 5m13.830s + user 2m0.444s + sys 0m28.344s + + +whereas + + $ time annex-funused + == WARNING == + This program should NOT be trusted to reliably find unused files in the + git annex. + + + real 0m1.569s + user 0m2.024s + sys 0m0.184s + +I tried to check memory usage via `/usr/bin/time -v` as well, and that showed (re-running in the same annex as above) + +annex-funused + + Maximum resident set size (kbytes): 13560 + +git annex unused + + Maximum resident set size (kbytes): 29120 + + +I've also written a comparison script [annex-testunused](https://gitorious.org/arand-scripts/arand-scripts/blobs/918cc79b99e22cbdca01ea4de10e2ca64abfc27a/annex-testunused) (needs annex-funused in $PATH) which creates an annex with a bunch of unused files and compares the running time for both versions: + +
    +$ annex-testunused
+Initialized empty Git repository in /tmp/tmp.fmsAvsPTcd/.git/
+init  ok
+(Recording state in git...)
+###
+* b2840d7 (HEAD, master) delete ~1100 files
+* c4a1e3a add 3000 files
+* bc19777 (git-annex) update
+* b3e6539 update
+* bec2c8f branch created
+annex unused
+real 0m4.154s
+real 0m2.029s
+real 0m2.044s
+annex funused
+real 0m0.923s
+real 0m0.933s
+real 0m0.905s
+Initialized empty Git repository in /tmp/tmp.7qFoCRWzB3/.git/
+init  ok
+(Recording state in git...)
+###
+* a5ff392 (HEAD, master) empty
+* cca4810 (1) delete ~1100 files
+* 587c406 add 3000 files
+* de0afeb (git-annex) update
+* 37b7881 update
+* 1735062 branch created
+annex unused
+real 0m3.499s
+real 0m3.443s
+real 0m3.435s
+annex funused
+real 0m0.956s
+real 0m0.956s
+real 0m0.874s
+Initialized empty Git repository in /tmp/tmp.L5fjdAgnFv/.git/
+init  ok
+(Recording state in git...)
+###
+* 94463a0 (HEAD, master) empty
+* e115619 (10) empty
+* 20686d4 (9) empty
+* 2e01a3f (8) empty
+* 043289d (7) empty
+* 6a52966 (6) empty
+* 0dc866d (5) empty
+* 35db331 (4) empty
+* 48504bc (3) empty
+* e25cac7 (2) empty
+* 655d026 (1) delete ~1100 files
+* 91a07d1 add 3000 files
+* 3c9ac62 (git-annex) update
+* c5736e0 update
+* 862d5b8 branch created
+annex unused
+real 0m16.242s
+real 0m16.277s
+real 0m16.246s
+annex funused
+real 0m0.960s
+real 0m0.960s
+real 0m0.927s
+
    + +So, unless I've missed something fundamental (I keep thinking I might have...), it seems to be very consistently faster, and scale ok where `git annex unused` scales rather poorly. + +"""]] diff --git a/doc/bugs/added_branches_makes___39__git_annex_unused__39___slow/comment_6_880ef2ee797221332dbb629b2d55522f._comment b/doc/bugs/added_branches_makes___39__git_annex_unused__39___slow/comment_6_880ef2ee797221332dbb629b2d55522f._comment new file mode 100644 index 0000000000..a4fea313f9 --- /dev/null +++ b/doc/bugs/added_branches_makes___39__git_annex_unused__39___slow/comment_6_880ef2ee797221332dbb629b2d55522f._comment @@ -0,0 +1,10 @@ +[[!comment format=mdwn + username="http://joeyh.name/" + ip="2001:4978:f:21a::2" + subject="comment 6" + date="2013-08-12T04:14:18Z" + content=""" +The memory usage is probably lower because `sort` and `comm` and bash's `<(command)` all have particularly well tuned memory usage with 37 years of history behind them. Particularly GNU `sort` will transparently use a temp file rather than storing too much data in memory, and does rather sophisticated stuff to make that work efficiently. It's rather harder to get that kind of behavior when not using the unix tools and instead using stock programming language primatives like sort() and hashes. + +I still suspect that `git cat-file` is slower than a direct readlink(2) of the symlink, when that can be done. +"""]] diff --git a/doc/bugs/added_branches_makes___39__git_annex_unused__39___slow/comment_7_826fd82cdf9b1c79c9b555ca26c2c176._comment b/doc/bugs/added_branches_makes___39__git_annex_unused__39___slow/comment_7_826fd82cdf9b1c79c9b555ca26c2c176._comment new file mode 100644 index 0000000000..c2dbd18109 --- /dev/null +++ b/doc/bugs/added_branches_makes___39__git_annex_unused__39___slow/comment_7_826fd82cdf9b1c79c9b555ca26c2c176._comment @@ -0,0 +1,8 @@ +[[!comment format=mdwn + username="arand" + ip="130.243.226.21" + subject="comment 7" + date="2013-08-12T18:07:44Z" + content=""" +Hmm, probably, do you think this way of getting everything and then doing the filtering could work reasonably in git-annex? Assuming \"git cat-file --batch\" is the main bottleneck, reducing the amount of runs for it would still likely be an improvement? +"""]] diff --git a/doc/bugs/adding_a_remote_server_fails.mdwn b/doc/bugs/adding_a_remote_server_fails.mdwn new file mode 100644 index 0000000000..06e6af4dae --- /dev/null +++ b/doc/bugs/adding_a_remote_server_fails.mdwn @@ -0,0 +1,27 @@ +### Please describe the problem. +I tried to add a remote server to my annex repo using the webapp. + +### What steps will reproduce the problem? +Configuration -> Repositories -> Remote server -> Fill in stuff, use public key auth -> click "check this server" -> enjoy + +### What version of git-annex are you using? On what operating system? +OpenSUSE Tumbleweed, git-annex 5.20140709-gc75193e + +### Please provide any additional information below. +Error message: `user error (gpg ["--batch","--no-tty","--use-agent","--quiet","--trust-model","always","--with-colons","--list-secret-keys","--fixed-list-mode"] exited 2)` + +[[!format sh """ +# If you can, paste a complete transcript of the problem occurring here. +# If the problem is with the git-annex assistant, paste in .git/annex/daemon.log +[2014-07-11 12:03:52 CEST] main: starting assistant version 5.20140709-gc75193e +[2014-07-11 12:03:53 CEST] UpgradeWatcher: Finished upgrading git-annex to version 5.20140709-gc75193e +(scanning...) [2014-07-11 12:03:53 CEST] Watcher: Performing startup scan +(started...) gpg: /home/zilti/.gnupg/gpg.conf:200: argument not expected +gpg: /home/zilti/.gnupg/gpg.conf:201: invalid option +gpg: /home/zilti/.gnupg/gpg.conf:200: argument not expected +gpg: /home/zilti/.gnupg/gpg.conf:201: invalid option + +# End of transcript or log. +"""]] + +[[!tag moreinfo]] diff --git a/doc/bugs/adding_a_remote_server_fails/comment_1_cf7ea9171f002c5793a882b42d33a77d._comment b/doc/bugs/adding_a_remote_server_fails/comment_1_cf7ea9171f002c5793a882b42d33a77d._comment new file mode 100644 index 0000000000..1dc6d69548 --- /dev/null +++ b/doc/bugs/adding_a_remote_server_fails/comment_1_cf7ea9171f002c5793a882b42d33a77d._comment @@ -0,0 +1,10 @@ +[[!comment format=mdwn + username="http://joeyh.name/" + ip="209.250.56.2" + subject="obvious questions" + date="2014-07-11T18:12:44Z" + content=""" +From what source did you install git-annex? + +What version of gpg is installed on your system? What is the contents of line 200 and 201 of/home/zilti/.gnupg/gpg.conf? +"""]] diff --git a/doc/bugs/adding_remote_server_using_ssh_on_a_4.1_device.mdwn b/doc/bugs/adding_remote_server_using_ssh_on_a_4.1_device.mdwn new file mode 100644 index 0000000000..c2e0b5588c --- /dev/null +++ b/doc/bugs/adding_remote_server_using_ssh_on_a_4.1_device.mdwn @@ -0,0 +1,38 @@ +[[!meta title="adding remote server using ssh on an Android 4.1 device"]] + +### Please describe the problem. + +Unable to add remote server using ssh on a 4.1 device. + +The error message on the android is: Failed to ssh to the server. Transcript: Could not create directory '(null)/.ssh'. + +The message from sshd on the server is: Feb 20 11:32:37 thrain sshd[1662]: Did not receive identification string from 10.1.0.16 + +(thrain is the sshd server, 10.1.0.16 is the android) + +### What steps will reproduce the problem? + +On the android, go into the get-annex webpage, select add remote repository, +add the particulars + +hit check this server. + +### What version of git-annex are you using? On what operating system? + +The android version of git-annex is 5.20150219-gd24cgd3 +The version of address is 4.1.1 + +The sshd server is debian wheezy + + +### Please provide any additional information below. + +[[!format sh """ +# If you can, paste a complete transcript of the problem occurring here. +# If the problem is with the git-annex assistant, paste in .git/annex/daemon.log + + +# End of transcript or log. +"""]] + +> Closing as this was a bug in the deprecated Android app. [[done]] --[[Joey]] diff --git a/doc/bugs/adding_remote_server_using_ssh_on_a_4.1_device/comment_1_ab8a227df71c70c05074b50dcb798acd._comment b/doc/bugs/adding_remote_server_using_ssh_on_a_4.1_device/comment_1_ab8a227df71c70c05074b50dcb798acd._comment new file mode 100644 index 0000000000..9edb912099 --- /dev/null +++ b/doc/bugs/adding_remote_server_using_ssh_on_a_4.1_device/comment_1_ab8a227df71c70c05074b50dcb798acd._comment @@ -0,0 +1,15 @@ +[[!comment format=mdwn + username="https://www.google.com/accounts/o8/id?id=AItOawnbPKLjBONawBd74MKJZo05juCqdsP1jAU" + nickname="Ramon" + subject="comment 1" + date="2015-02-27T01:43:42Z" + content=""" +I had the same problem, also with Android 4.1.1 (an Asus TF201). This is what I did: + +- Download version for Android 4.3 and 4.4 (autobuilds) and install that one over the previous. +- Add the remote, which worked just fine. +- Reinstall the version for Android 4.1 (autobuilds) + +And things seem to be working (things are syncing to the Android) + +"""]] diff --git a/doc/bugs/adding_remote_server_using_ssh_on_a_4.1_device/comment_2_f88081ca768d0108936f1da1e509ff32._comment b/doc/bugs/adding_remote_server_using_ssh_on_a_4.1_device/comment_2_f88081ca768d0108936f1da1e509ff32._comment new file mode 100644 index 0000000000..a9ad7a4918 --- /dev/null +++ b/doc/bugs/adding_remote_server_using_ssh_on_a_4.1_device/comment_2_f88081ca768d0108936f1da1e509ff32._comment @@ -0,0 +1,14 @@ +[[!comment format=mdwn + username="joey" + subject="""comment 2""" + date="2018-05-08T18:53:46Z" + content=""" +I'm closing this bug report because the git-annex Android app that it +was reported on is now deprecated. Instead, we have +a way to run the regular git-annex build for linux on Android in termux: + + +There were a lot of problems with the way the git-annex Android app was +put together, and I hope this new approach avoids them. If you try it and +still have the bug you reported, please followup and I'll reopen it. +"""]] diff --git a/doc/bugs/adds_file_destined_for_annex_into_git_in___39__addurl__39__.mdwn b/doc/bugs/adds_file_destined_for_annex_into_git_in___39__addurl__39__.mdwn new file mode 100644 index 0000000000..a5cdfd6d6d --- /dev/null +++ b/doc/bugs/adds_file_destined_for_annex_into_git_in___39__addurl__39__.mdwn @@ -0,0 +1,41 @@ +### Please describe the problem. + +When addurl'ing a big file with .gitattributes configured to add only some files directly into git (and 'git annex add' operating correctly), addurl adds large files straight into git. + +### What version of git-annex are you using? On what operating system? + +git-annex version: 6.20161018+gitgf3c366a-1~ndall+1 + + +### Please provide any additional information below. + +[[!format sh """ +$> cat .gitattributes +* annex.backend=MD5E +* annex.largefiles=(largerthan=100kb) +*.json annex.largefiles=nothing +*.txt annex.largefiles=nothing +*.tsv annex.largefiles=nothing +*.nii.gz annex.largefiles=(largerthan=0kb) +*.tgz annex.largefiles=(largerthan=0kb) +*.tar.gz annex.largefiles=(largerthan=0kb) +*.gz annex.largefiles=(largerthan=0kb) + +$> git annex addurl http://fcp-indi.s3.amazonaws.com/data/Projects/HBNSSI/RawDataTars/sub-0031121_baseline.tar.gz\?versionId\=7FvexHgyazWF.dUo238FA7XRiK0FWQDw. +addurl fcp_indi.s3.amazonaws.com_data_Projects_HBNSSI_RawDataTars_sub_0031121_baseline.tar.gz_versionId_7FvexHgyazWF.dUo238FA7XRiK0FWQDw. (downloading http://fcp-indi.s3.amazonaws.com/data/Projects/HBNSSI/RawDataTars/sub-0031121_baseline.tar.gz?versionId=7FvexHgyazWF.dUo238FA7XRiK0FWQDw. ...) +/mnt/btrfs/datasets/datalad/crawl-misc/indi/ 100%[==============================================================================================>] 195.44M 21.2MB/s in 12s +(non-large file; adding content to git repository) ok +(recording state in git...) +cached/staged changes: + \u2026r.gz_versionId_7FvexHgyazWF.dUo238FA7XRiK0FWQDw. | Bin 0 -> 204937338 bytes + +$> ls -l fcp_indi.s3.amazonaws.com_data_Projects_HBNSSI_RawDataTars_sub_0031121_baseline.tar.gz_versionId_7FvexHgyazWF.dUo238FA7XRiK0FWQDw. +-rw------- 1 yoh datalad 204937338 Oct 25 17:30 fcp_indi.s3.amazonaws.com_data_Projects_HBNSSI_RawDataTars_sub_0031121_baseline.tar.gz_versionId_7FvexHgyazWF.dUo238FA7XRiK0FWQDw. +cached/staged changes: + \u2026r.gz_versionId_7FvexHgyazWF.dUo238FA7XRiK0FWQDw. | Bin 0 -> 204937338 bytes + +"""]] + +[[!meta author=yoh]] + +> [[fixed|done]] --[[Joey]] diff --git a/doc/bugs/addurl_--batch__--with-files_doesn__39__t_add_file_into_git_until_pipe_is_closed.mdwn b/doc/bugs/addurl_--batch__--with-files_doesn__39__t_add_file_into_git_until_pipe_is_closed.mdwn new file mode 100644 index 0000000000..59dda810f0 --- /dev/null +++ b/doc/bugs/addurl_--batch__--with-files_doesn__39__t_add_file_into_git_until_pipe_is_closed.mdwn @@ -0,0 +1,98 @@ +### Please describe the problem. + +only happens if file doesn't exist yet and gets downloaded (interestingly, as I reported in another report, if file exists it gets re-downloaded but this bug doesn't repeat) + +[[!format sh """ + +$> chmod a+w -R /tmp/123; rm -rf /tmp/123; mkdir /tmp/123; cd /tmp/123; git init; git annex init; +Initialized empty Git repository in /tmp/123/.git/ +init ok +(recording state in git...) + +$> mkfifo /tmp/pipe + +$> cat /tmp/pipe | git annex --debug addurl --batch --with-files +^Z +[2] + 1084 suspended cat /tmp/pipe | + 1085 suspended git annex --debug addurl --batch --with-files + +$> bg +[2] - 1084 continued cat /tmp/pipe | + 1085 continued git annex --debug addurl --batch --with-files + +$> exec 3>/tmp/pipe + +$> echo "http://www.onerussian.com/tmp/banner.png banner.png" >&3 +[2016-01-11 22:25:34.423037] read: git ["--git-dir=.git","--work-tree=.","--literal-pathspecs","show-ref","git-annex"] +[2016-01-11 22:25:34.427649] process done ExitSuccess +[2016-01-11 22:25:34.428067] read: git ["--git-dir=.git","--work-tree=.","--literal-pathspecs","show-ref","--hash","refs/heads/git-annex"] +[2016-01-11 22:25:34.432199] process done ExitSuccess + +$> [2016-01-11 22:25:34.432834] read: git ["--git-dir=.git","--work-tree=.","--literal-pathspecs","log","refs/heads/git-annex..87efc349dceb06df635c8485ef536ae11f48736c","-n1","--pretty=%H"] +[2016-01-11 22:25:34.438533] process done ExitSuccess +[2016-01-11 22:25:34.439666] chat: git ["--git-dir=.git","--work-tree=.","--literal-pathspecs","cat-file","--batch"] +[2016-01-11 22:25:34.444875] read: quvi ["--version"] +[2016-01-11 22:25:34.452158] process done ExitSuccess +[2016-01-11 22:25:34.452563] call: quvi ["--verbosity","mute","--support","http://www.onerussian.com/tmp/banner.png"] +[2016-01-11 22:25:34.468071] process done ExitFailure 65 +addurl banner.png (downloading http://www.onerussian.com/tmp/banner.png ...) +[2016-01-11 22:25:34.500939] call: wget ["-q","--show-progress","--clobber","-c","-O","/tmp/123/.git/annex/tmp/URL-s25319--http&c%%www.onerussian.com%tmp%banner.png","http://www.onerussian.com/tmp/banner.png","--user-agent","git-annex/6.20160104+gitg0cf96be-1~ndall+1"] +/tmp/123/.git/annex/tmp/URL-s25319--http&c%%www.on 100%[==============================================================================================================>] 24.73K --.-KB/s in 0s +[2016-01-11 22:25:34.518335] process done ExitSuccess +[2016-01-11 22:25:34.519195] chat: git ["--git-dir=.git","--work-tree=.","--literal-pathspecs","check-attr","-z","--stdin","annex.backend","annex.numcopies","--"] +[2016-01-11 22:25:34.5241] read: git ["--version"] +[2016-01-11 22:25:34.527001] process done ExitSuccess +ok + +$> git status # note that file is untracked, although it is a symlink into .git/annex/objects +On branch master + +Initial commit + +Untracked files: + (use "git add ..." to include in what will be committed) + + banner.png + +nothing added to commit but untracked files present (use "git add" to track) + +$> # closing the pipe now +$> exec 3>&- +(recording state in git...) +[2016-01-11 22:26:02.947133] feed: xargs ["-0","git","--git-dir=.git","--work-tree=.","--literal-pathspecs","add","--"] +$> [2016-01-11 22:26:02.95701] process done ExitSuccess +[2016-01-11 22:26:02.961588] chat: git ["--git-dir=.git","--work-tree=.","--literal-pathspecs","hash-object","-w","--stdin-paths","--no-filters"] +[2016-01-11 22:26:02.964507] feed: git ["--git-dir=.git","--work-tree=.","--literal-pathspecs","update-index","-z","--index-info"] +[2016-01-11 22:26:02.971853] process done ExitSuccess +[2016-01-11 22:26:02.972524] process done ExitSuccess +[2016-01-11 22:26:02.973043] read: git ["--git-dir=.git","--work-tree=.","--literal-pathspecs","show-ref","--hash","refs/heads/git-annex"] +[2016-01-11 22:26:02.975871] process done ExitSuccess +[2016-01-11 22:26:02.977613] read: git ["--git-dir=.git","--work-tree=.","--literal-pathspecs","write-tree"] +[2016-01-11 22:26:02.982308] process done ExitSuccess +[2016-01-11 22:26:02.982744] chat: git ["--git-dir=.git","--work-tree=.","--literal-pathspecs","commit-tree","0b4d294f555bc86c17de89bca35193433408bbef","--no-gpg-sign","-p","refs/heads/git-annex"] +[2016-01-11 22:26:02.990485] process done ExitSuccess +[2016-01-11 22:26:02.990941] call: git ["--git-dir=.git","--work-tree=.","--literal-pathspecs","update-ref","refs/heads/git-annex","1359dc4d9296bc4f5941e7299376319899ea222c"] +[2016-01-11 22:26:02.995434] process done ExitSuccess + +[2] - 1084 done cat /tmp/pipe | + 1085 done git annex --debug addurl --batch --with-files + +$> git status +On branch master + +Initial commit + +Changes to be committed: + (use "git rm --cached ..." to unstage) + + new file: banner.png + +$> git annex version +git-annex version: 6.20160104+gitg0cf96be-1~ndall+1 +... + +"""]] + +[[!meta author=yoh]] + +> closing as not a bug [[done]] --[[Joey]] diff --git a/doc/bugs/addurl_--batch___40__--json_or_not__41___doesn__39__t_report_failure_correctly_if_non-annexed_file_exists.mdwn b/doc/bugs/addurl_--batch___40__--json_or_not__41___doesn__39__t_report_failure_correctly_if_non-annexed_file_exists.mdwn new file mode 100644 index 0000000000..c135f7f324 --- /dev/null +++ b/doc/bugs/addurl_--batch___40__--json_or_not__41___doesn__39__t_report_failure_correctly_if_non-annexed_file_exists.mdwn @@ -0,0 +1,21 @@ +### Please describe the problem. + +Complained elsewhere (http://git-annex.branchable.com/bugs/inconsistent_output_upon_addurl_--batch_complicates_if_not_forbids_reliable_parsing_of_output/) haven't mentioned it was marked fixed, so reiterating as independent bugreport + +With today's version +[[!format sh """ + +$> echo "http://www.onerussian.com/tmp/banner.png 123" | git annex addurl --batch --with-files +addurl 123 git-annex: 123 already exists and is not annexed; not overwriting + +$> echo "http://www.onerussian.com/tmp/banner.png 123" | git annex addurl --batch --with-files 2>/dev/null +addurl 123 % + +$> echo "http://www.onerussian.com/tmp/banner.png 123" | git annex addurl --batch --with-files --json 2>/dev/null +{"command":"addurl","file":"123"% + +"""]] + +[[!meta author=yoh]] + +> [[fixed|done]] --[[Joey]] diff --git a/doc/bugs/addurl_--batch_decides_to_talk_to_ssh_remotes_for_some_reason.mdwn b/doc/bugs/addurl_--batch_decides_to_talk_to_ssh_remotes_for_some_reason.mdwn new file mode 100644 index 0000000000..cd2e2b5fbf --- /dev/null +++ b/doc/bugs/addurl_--batch_decides_to_talk_to_ssh_remotes_for_some_reason.mdwn @@ -0,0 +1,21 @@ +### What version of git-annex are you using? On what operating system? + +6.20160425+gitgffe2ea2-1~ndall+1 +(reconfirmed with 6.20160523+gitg33c00ab-1~ndall+1) + +### Please provide any additional information below. +Here is a debug output from datalad which shows the interaction + +[[!format sh """ +2016-05-23 16:41:59,955 [DEBUG ] Initiating a new process for BatchedAnnex(annex_cmd='addurl', annex_options=<<['-c', 'annex.largefil...>>, git_options=[], output_proc=, path=<<'/mnt/btrfs/datasets/d...>>) (annexrepo.py:1177) +2016-05-23 16:41:59,956 [Level 5] Command: ['git', '-c', 'receive.autogc=0', '-c', 'gc.auto=0', 'annex', 'addurl', '-c', 'annex.largefiles=exclude=Makefile and exclude=LICENSE* and exclude=ISSUES* and exclude=CHANGES* and exclude=README* and exclude=*.[mc] and exclude=dataset*.json and (exclude=*.txt or include=*/*.txt) and (exclude=*.json or include=*/*.json) and (exclude=*.tsv or include=*/*.tsv)', '--with-files', '--json', '--batch'] (annexrepo.py:1180) +2016-05-23 16:41:59,967 [Level 5] Sending u'http://openfmri.s3.amazonaws.com/tarballs/ds052_R2.0.0_metadata_derivatives.tgz?versionId=nrIMjS3lH6TMoSkA.27U.Md_k2BSva3i ds052_R2.0.0_metadata_derivatives.tgz\n' to batched annex BatchedAnnex(annex_cmd='addurl', annex_options=<<['-c', 'annex.largefil...>>, git_options=[], output_proc=, path=<<'/mnt/btrfs/datasets/d...>>) (annexrepo.py:1234) +2016-05-23 16:41:59,968 [Level 5] Done sending. (annexrepo.py:1242) +yoh@datasets.datalad.org's password: +"""]] + +That repository indeed has a remote (ssh) setup pointing to datasets.datalad.org (which carries no load for annex, besides git-annex repository, ATM), but that remote should not be consulted IMHO for addurl operation (not to mention in the --batch mode shouldn't request any user interaction) + +[[!meta author=yoh]] + +> [[done]] --[[Joey]] diff --git a/doc/bugs/addurl_--file__causes_file_redownload_even_if_it_already_present.mdwn b/doc/bugs/addurl_--file__causes_file_redownload_even_if_it_already_present.mdwn new file mode 100644 index 0000000000..c3645b6fa6 --- /dev/null +++ b/doc/bugs/addurl_--file__causes_file_redownload_even_if_it_already_present.mdwn @@ -0,0 +1,32 @@ +### Please describe the problem. + +IMHO annex shouldn't redownload the file (not yet under annex/git control) entirely if pointed by --file=FILE FILE exists. + +[[!format sh """ + +$> chmod a+w -R /tmp/123; rm -rf /tmp/123; mkdir /tmp/123; cd /tmp/123; git init; git annex init; +Initialized empty Git repository in /tmp/123/.git/ +init ok + +$> wget -q http://www.onerussian.com/tmp/banner.png + +$> ls -l +total 28 +-rw------- 1 yoh yoh 25319 Sep 17 13:49 banner.png + +$> git annex addurl --file=banner.png http://www.onerussian.com/tmp/banner.png +addurl banner.png (downloading http://www.onerussian.com/tmp/banner.png ...) +/tmp/123/.git/annex/tmp/URL-s25319--http&c%%w 100%[=================================================================================================>] 24.73K --.-KB/s in 0.003s +ok + +$> git annex version +git-annex version: 6.20160104+gitg0cf96be-1~ndall+1 + +"""] + +[[!meta author=yoh]] + +> I don't think that the re-download is the bug. The actual problem +> is that the file is present and not an annexed file, so git-annex addurl +> should avoid overwriting it, whatever its content. [[fixed|done]] +> --[[Joey]] diff --git a/doc/bugs/addurl_results_in_different_file_to_wget.mdwn b/doc/bugs/addurl_results_in_different_file_to_wget.mdwn new file mode 100644 index 0000000000..10dec894ea --- /dev/null +++ b/doc/bugs/addurl_results_in_different_file_to_wget.mdwn @@ -0,0 +1,50 @@ +### Please describe the problem. +addurl results in a different file than wget. Looks like git-annex is either decompressing it or triggering the server to serve it decompressed. This only happens with this one file in my usage, the other 32 .tar.gz that are added at the same time md5sum correctly. + +### What steps will reproduce the problem? + $ git annex addurl http://www.greenwoodsoftware.com/less/less-530.tar.gz + $ wget http://www.greenwoodsoftware.com/less/less-530.tar.gz + $ md5sum * + +### What version of git-annex are you using? On what operating system? +* git-annex version: 6.20180509-g0632c49c22 +* Arch Linux + +### Please provide any additional information below. + +[[!format sh """ +$ git annex init +init ok +(recording state in git...) +$ git annex addurl http://www.greenwoodsoftware.com/less/less-530.tar.gz +addurl http://www.greenwoodsoftware.com/less/less-530.tar.gz + +(to www.greenwoodsoftware.com_less_less_530.tar.gz) ok +(recording state in git...) +$ wget http://www.greenwoodsoftware.com/less/less-530.tar.gz +--2018-05-16 22:10:19-- http://www.greenwoodsoftware.com/less/less-530.tar.gz +Resolving www.greenwoodsoftware.com (www.greenwoodsoftware.com)... 104.200.21.227 +Connecting to www.greenwoodsoftware.com (www.greenwoodsoftware.com)|104.200.21.227|:80... connected. +HTTP request sent, awaiting response... 200 OK +Length: 339723 (332K) [application/x-gzip] +Saving to: ‘less-530.tar.gz’ + +less-530.tar.gz 100%[==============================================>] 331.76K 573KB/s in 0.6s + +2018-05-16 22:10:20 (573 KB/s) - ‘less-530.tar.gz’ saved [339723/339723] + +$ md5sum * +6a39bccf420c946b0fd7ffc64961315b less-530.tar.gz +3d8def818aa59f10218e2549dc16f6b1 www.greenwoodsoftware.com_less_less_530.tar.gz +$ file -L * +less-530.tar.gz: gzip compressed data, last modified: Tue Dec 5 22:56:50 2017, from Unix +www.greenwoodsoftware.com_less_less_530.tar.gz: POSIX tar archive (GNU) +$ du -L * +332 less-530.tar.gz +1272 www.greenwoodsoftware.com_less_less_530.tar.gz +"""]] + +### Have you had any luck using git-annex before? (Sometimes we get tired of reading bug reports all day and a lil' positive end note does wonders) +Of course! Still love git-annex to bits <3 + +> [[fixed|done]] --[[Joey]] diff --git a/doc/bugs/addurl_results_in_different_file_to_wget/comment_1_43449ff547a70f5fbfc6fa1d1d4bd464._comment b/doc/bugs/addurl_results_in_different_file_to_wget/comment_1_43449ff547a70f5fbfc6fa1d1d4bd464._comment new file mode 100644 index 0000000000..99c037c8e6 --- /dev/null +++ b/doc/bugs/addurl_results_in_different_file_to_wget/comment_1_43449ff547a70f5fbfc6fa1d1d4bd464._comment @@ -0,0 +1,84 @@ +[[!comment format=mdwn + username="CandyAngel" + avatar="http://cdn.libravatar.org/avatar/15c0aade8bec5bf004f939dd73cf9ed8" + subject="comment 1" + date="2018-05-17T19:14:25Z" + content=""" +I thought the working ones might be because they are being redirected to https, but I found another one that works correctly that isn't behind https. + +# Broken (http://www.greenwoodsoftware.com/less/less-530.tar.gz) +## git-annex + + GET /less/less-530.tar.gz HTTP/1.1 + Host: www.greenwoodsoftware.com + Accept-Encoding: gzip + + HTTP/1.1 200 OK + Date: Thu, 17 May 2018 18:43:43 GMT + Server: Apache/2 + Last-Modified: Tue, 05 Dec 2017 22:57:13 GMT + ETag: \"52f0b-55f9fc1798dca\" + Accept-Ranges: bytes + Content-Length: 339723 + Vary: User-Agent + Content-Type: application/x-gzip + Content-Encoding: gzip + +## wget + + GET /less/less-530.tar.gz HTTP/1.1 + User-Agent: Wget/1.19.5 (linux-gnu) + Accept: */* + Accept-Encoding: identity + Host: www.greenwoodsoftware.com + Connection: Keep-Alive + + HTTP/1.1 200 OK + Date: Thu, 17 May 2018 18:43:53 GMT + Server: Apache/2 + Upgrade: h2,h2c + Connection: Upgrade, Keep-Alive + Last-Modified: Tue, 05 Dec 2017 22:57:13 GMT + ETag: \"52f0b-55f9fc1798dca\" + Accept-Ranges: bytes + Content-Length: 339723 + Vary: User-Agent + Keep-Alive: timeout=2, max=100 + Content-Type: application/x-gzip + Content-Encoding: x-gzip + +# Working (http://www.trout.me.uk/perl/Authen-Htpasswd-0.171.tar.gz) +## git-annex + GET /perl/Authen-Htpasswd-0.171.tar.gz HTTP/1.1 + Host: www.trout.me.uk + Accept-Encoding: gzip + + HTTP/1.1 200 OK + Date: Thu, 17 May 2018 19:09:55 GMT + Server: Apache/2.2.16 (Debian) + Last-Modified: Tue, 09 Aug 2011 12:09:31 GMT + ETag: \"10269-2096-4aa116fa394c0\" + Accept-Ranges: bytes + Content-Length: 8342 + Content-Type: application/x-gzip + +## wget + + GET /perl/Authen-Htpasswd-0.171.tar.gz HTTP/1.1 + User-Agent: Wget/1.19.5 (linux-gnu) + Accept: */* + Accept-Encoding: identity + Host: www.trout.me.uk + Connection: Keep-Alive + + HTTP/1.1 200 OK + Date: Thu, 17 May 2018 19:09:58 GMT + Server: Apache/2.2.16 (Debian) + Last-Modified: Tue, 09 Aug 2011 12:09:31 GMT + ETag: \"10269-2096-4aa116fa394c0\" + Accept-Ranges: bytes + Content-Length: 8342 + Keep-Alive: timeout=15, max=100 + Connection: Keep-Alive + Content-Type: application/x-gzip +"""]] diff --git a/doc/bugs/addurl_results_in_different_file_to_wget/comment_2_846c236798af74eaa071f59512b0ce6c._comment b/doc/bugs/addurl_results_in_different_file_to_wget/comment_2_846c236798af74eaa071f59512b0ce6c._comment new file mode 100644 index 0000000000..41222b90d6 --- /dev/null +++ b/doc/bugs/addurl_results_in_different_file_to_wget/comment_2_846c236798af74eaa071f59512b0ce6c._comment @@ -0,0 +1,14 @@ +[[!comment format=mdwn + username="CandyAngel" + avatar="http://cdn.libravatar.org/avatar/15c0aade8bec5bf004f939dd73cf9ed8" + subject="comment 2" + date="2018-05-17T20:21:47Z" + content=""" +Forcing the use of curl by setting + + git config annex.web-options \"-4\" + +results in the correct download of the file with addurl. Looks like a bug or misbehaviour in http-conduit! + +(Sorry for the noise..) +"""]] diff --git a/doc/bugs/addurl_results_in_different_file_to_wget/comment_3_c8fd9ca4e8b22f6ba04a3530efc3da62._comment b/doc/bugs/addurl_results_in_different_file_to_wget/comment_3_c8fd9ca4e8b22f6ba04a3530efc3da62._comment new file mode 100644 index 0000000000..ee90e27883 --- /dev/null +++ b/doc/bugs/addurl_results_in_different_file_to_wget/comment_3_c8fd9ca4e8b22f6ba04a3530efc3da62._comment @@ -0,0 +1,21 @@ +[[!comment format=mdwn + username="joey" + subject="""comment 3""" + date="2018-05-21T18:30:53Z" + content=""" +haskell http-client has a strange default handling of compressed files. +It seems to want to decompress them unless the content-type is +"application/x-tar". It defaults to accept-encoding of gzip. + +That seems to be targeting being used to implement a web browser or +something, although I don't entirely understand how that behavior would +make sense for a web browser either; I'd expect it to only decompress +content that was transparently compressed in transit, but not other +content. Firefox does not decompress that tarball when downloading it; nor +does it display a foo.html.gz as a web page; it downloads it as-is. + +Very strange default for a general purpose http library; IMHO it's a bug. + +`Accept-Encoding: identity` and no transparent decompression seems to be +the way to go here, just like wget. +"""]] diff --git a/doc/bugs/addurl_unittest_failing_under_windows.mdwn b/doc/bugs/addurl_unittest_failing_under_windows.mdwn new file mode 100644 index 0000000000..8458ed83be --- /dev/null +++ b/doc/bugs/addurl_unittest_failing_under_windows.mdwn @@ -0,0 +1,54 @@ +### Please describe the problem. +git annex works perfectly fine under windows now; awesome work! One of the tests in the suite still fails (which might affect some features, that I don't use (yet?)). + +### What steps will reproduce the problem? +[[!format sh """ +$ git annex test -p addurl +"""]] + +### What version of git-annex are you using? On what operating system? + +[[!format sh """ +$ git --version +git version 2.6.4.windows.1 + +$ git annex version +git-annex version: 5.20151218-g5008846 +build flags: Assistant Webapp Webapp-secure Pairing Testsuite S3(multipartupload) WebDAV ConcurrentOutput DNS Feeds Quvi TDFA TorrentParser Database +key/value backends: SHA256E SHA256 SHA512E SHA512 SHA224E SHA224 SHA384E SHA384 SHA3_256E SHA3_256 SHA3_512E SHA3_512 SHA3_224E SHA3_224 SHA3_384E SHA3_384 SKEIN256E SKEIN256 SKEIN512E SKEIN512 SHA1E SHA1 MD5E MD5 WORM URL +remote types: git gcrypt S3 bup directory rsync web bittorrent webdav tahoe glacier ddar hook external +"""]] + +Windows 10 (OS Build 10586.0) + +### Please provide any additional information below. + +[[!format sh """ +$ git annex test -p addurl +Tests + Unit Tests +Init Tests + addurl: init: Detected a filesystem without fifo support. + Disabling ssh connection caching. + Detected a crippled filesystem. + Enabling direct mode. +OK (2.74s) + add: fatal: This operation must be run in a work tree +OK (6.00s) + +All 2 tests passed (8.79s) + Detected a filesystem without fifo support. + Disabling ssh connection caching. + Detected a crippled filesystem. + Enabling direct mode. +removeDirectoryRecursive: permission denied (The process cannot access the file because it is being used by another process.) +sleeping 10 seconds and will retry directory cleanup +FAIL (7.16s) + addurl failed on file:///Users/born/.t/tmprepo0/myurl + +1 out of 1 tests failed (28.13s) + (This could be due to a bug in git-annex, or an incompatibility + with utilities, such as git, installed on this system.) +"""]] + + diff --git a/doc/bugs/addurl_unittest_failing_under_windows/comment_1_8f723e986d80754a8b68308ab8b15c20._comment b/doc/bugs/addurl_unittest_failing_under_windows/comment_1_8f723e986d80754a8b68308ab8b15c20._comment new file mode 100644 index 0000000000..de5b45a1d6 --- /dev/null +++ b/doc/bugs/addurl_unittest_failing_under_windows/comment_1_8f723e986d80754a8b68308ab8b15c20._comment @@ -0,0 +1,10 @@ +[[!comment format=mdwn + username="joey" + subject="""comment 1""" + date="2016-01-15T19:17:54Z" + content=""" +That test needs curl to be available. git-annex relies on Git for Windows +containing curl. + +Is the curl command available in the shell where you ran git-annex test? +"""]] diff --git a/doc/bugs/addurl_unittest_failing_under_windows/comment_2_e0bbb72af2852b2210bb1ce520e88f1d._comment b/doc/bugs/addurl_unittest_failing_under_windows/comment_2_e0bbb72af2852b2210bb1ce520e88f1d._comment new file mode 100644 index 0000000000..8f1c3a21a7 --- /dev/null +++ b/doc/bugs/addurl_unittest_failing_under_windows/comment_2_e0bbb72af2852b2210bb1ce520e88f1d._comment @@ -0,0 +1,10 @@ +[[!comment format=mdwn + username="yarikoptic" + avatar="http://cdn.libravatar.org/avatar/f11e9c84cb18d26a1748c33b48c924b4" + subject="comment 2" + date="2018-09-24T19:33:41Z" + content=""" +Just ran into the same situation! Happens in both v6 and direct modes. I have `curl` but not wget available in that shell (Anaconda's shell). +Here are examples: +[v6](http://www.onerussian.com/tmp/gkrellShoot_09-24-18_152906.png), [direct](http://www.onerussian.com/tmp/gkrellShoot_09-24-18_153055.png) +"""]] diff --git a/doc/bugs/addurl_youtube-dl_behavior_change.mdwn b/doc/bugs/addurl_youtube-dl_behavior_change.mdwn new file mode 100644 index 0000000000..17455358b4 --- /dev/null +++ b/doc/bugs/addurl_youtube-dl_behavior_change.mdwn @@ -0,0 +1,19 @@ +I have often used a terminal window and prefixed my input with "git-annex +addurl" then drag links to the window for pasting. Often, I have to press the +up-arrow and run the command again. The addurl behavior with urls that are +already locally present, quvi responds "ok." However, when repeating a command +using yt-dl, when the url is already local, yt-dl refuses to overwrite, yet +returns "failed." I didn't know if you were aware of this. This isn't a +show-stopper, but just something I noticed. I generally do "addurl" manually. + +> I tried this, and it's not youtube-dl failing; but it re-downloads +> the whole content of the already present file, and then git-annex fails: +> +> whatever.mp4 already exists; not overwriting +> +> So I think it needs to ask youtube-dl for the filename first, and check +> if the local file already exists and already has the url, to get back to +> the old behavior. +> -- [[Joey]] + +[[fixed|done]] --[[Joey]] diff --git a/doc/bugs/after_git_annex_add_and_commit__44___git_annex_fsck_fails__58___no_known_copies.mdwn b/doc/bugs/after_git_annex_add_and_commit__44___git_annex_fsck_fails__58___no_known_copies.mdwn new file mode 100644 index 0000000000..cb0ed5edf6 --- /dev/null +++ b/doc/bugs/after_git_annex_add_and_commit__44___git_annex_fsck_fails__58___no_known_copies.mdwn @@ -0,0 +1,16 @@ +I added new content to my git annex repo on my laptop, in the usual way: copying the new dir into the repo, then cd into the new directory, 'git annex add .' and then 'git commit -m '. Everything went well, with no errors, which means that the files were correctly moved to .git/annex/objects/ and symbolic links were created in the new dir. But now, if I do 'git annex fsck .', I get an error for each file: +[[!format sh """ +fsck (fixing location log) (checksum...) + + ** No known copies exist of +failed +"""]] +Unfortunately, I am not able to reproduce the problem on a toy-example repository. This means that my git annex repo may be broken. How do I fix it? +I use git v1.9.1 on Ubuntu 14.04 LTS, and git-annex version: 5.20150406-gb2814bc + +OK, in all honesty, I did a 'git annex sync' between the 'add' and the 'commit' above. But I synced with a clone of the repository that I keep on an external drive, which is less updated than my laptop. It is less updated because I always add content on my laptop and then sync/get from the external disk. So the sync did no harm, apparently. + +> Since this seems to be only a problem with messaging about accidentally +> marked dead repositories, I've made fsck mention when a file is only +> located in a dead repo, and I've made info tell when it's run in a +> supposedly-dead repo. [[done]] --[[Joey]] diff --git a/doc/bugs/after_git_annex_add_and_commit__44___git_annex_fsck_fails__58___no_known_copies/comment_1_b23c42004a3486d2409c1f96afa819aa._comment b/doc/bugs/after_git_annex_add_and_commit__44___git_annex_fsck_fails__58___no_known_copies/comment_1_b23c42004a3486d2409c1f96afa819aa._comment new file mode 100644 index 0000000000..c059040e19 --- /dev/null +++ b/doc/bugs/after_git_annex_add_and_commit__44___git_annex_fsck_fails__58___no_known_copies/comment_1_b23c42004a3486d2409c1f96afa819aa._comment @@ -0,0 +1,18 @@ +[[!comment format=mdwn + username="emanuele.olivetti@47d88ed185b03191e25329caa6fabc2efb3118b2" + nickname="emanuele.olivetti" + subject="comment 1" + date="2016-02-13T22:27:56Z" + content=""" +Additional information: I have 62 files in the added directory. Git-annex info return this, with a suspicious \"numcopies -2: 62\": + + > git-annex info . + directory: . + local annex keys: 62 + local annex size: 377.19 megabytes + annexed files in working tree: 62 + size of annexed files in working tree: 377.19 megabytes + numcopies stats: + numcopies -2: 62 + +"""]] diff --git a/doc/bugs/after_git_annex_add_and_commit__44___git_annex_fsck_fails__58___no_known_copies/comment_2_e379c25d18ee1ae4ea7e0a5a33e1c75e._comment b/doc/bugs/after_git_annex_add_and_commit__44___git_annex_fsck_fails__58___no_known_copies/comment_2_e379c25d18ee1ae4ea7e0a5a33e1c75e._comment new file mode 100644 index 0000000000..c9e046aa62 --- /dev/null +++ b/doc/bugs/after_git_annex_add_and_commit__44___git_annex_fsck_fails__58___no_known_copies/comment_2_e379c25d18ee1ae4ea7e0a5a33e1c75e._comment @@ -0,0 +1,11 @@ +[[!comment format=mdwn + username="joey" + subject="""comment 2""" + date="2016-02-14T19:13:05Z" + content=""" +Is the content of the files locally present? Either fsck incorrectly thinks +it's not present despite it being present, or fsck correctly noticed that +it somehow went missing.. + +Is this repository using direct mode? +"""]] diff --git a/doc/bugs/after_git_annex_add_and_commit__44___git_annex_fsck_fails__58___no_known_copies/comment_3_063e27ec1f2dd23fbf914a08213316df._comment b/doc/bugs/after_git_annex_add_and_commit__44___git_annex_fsck_fails__58___no_known_copies/comment_3_063e27ec1f2dd23fbf914a08213316df._comment new file mode 100644 index 0000000000..6146fad5c3 --- /dev/null +++ b/doc/bugs/after_git_annex_add_and_commit__44___git_annex_fsck_fails__58___no_known_copies/comment_3_063e27ec1f2dd23fbf914a08213316df._comment @@ -0,0 +1,12 @@ +[[!comment format=mdwn + username="emanuele.olivetti@47d88ed185b03191e25329caa6fabc2efb3118b2" + nickname="emanuele.olivetti" + subject="comment 3" + date="2016-02-15T09:47:24Z" + content=""" +Yes, the content is locally present. In the sense that each symlink in the current directory points to an actual file in .git/annex/objects/ . That is why, in my opinion, fsck incorrectly thinks the content is not present, despite it being present. + +Initially I thought that a possible explanation was that 'git commit' went wrong, so that I would have to do it again. I tried to git commit again, but git says there is nothing to commit. + +My repository is in indirect mode. All files in the repository, including the troubling ones are in .git/annex/objects. +"""]] diff --git a/doc/bugs/after_git_annex_add_and_commit__44___git_annex_fsck_fails__58___no_known_copies/comment_4_8688477bb694dbc9e6c7768f5f375f3f._comment b/doc/bugs/after_git_annex_add_and_commit__44___git_annex_fsck_fails__58___no_known_copies/comment_4_8688477bb694dbc9e6c7768f5f375f3f._comment new file mode 100644 index 0000000000..1fa409f9c3 --- /dev/null +++ b/doc/bugs/after_git_annex_add_and_commit__44___git_annex_fsck_fails__58___no_known_copies/comment_4_8688477bb694dbc9e6c7768f5f375f3f._comment @@ -0,0 +1,28 @@ +[[!comment format=mdwn + username="emanuele.olivetti@47d88ed185b03191e25329caa6fabc2efb3118b2" + nickname="emanuele.olivetti" + subject="repository marked as dead (!)" + date="2016-02-18T14:33:56Z" + content=""" +Update: I've just tried to add a 'test' directory to the repository with only a 'foo' file inside, then git commit. Surprisingly, if I do 'git annex fsck .' of it, the repository is marked as **dead** and no known copies of foo are available: + + (master)> mkdir test ; cd test + test (master) > cat > foo + bar + test (master)> git annex add . + add foo ok + (recording state in git...) + test (master)> git commit -m \"added fake content to test git annex repo\" + [master b9f0a8f] added fake content to test git annex repo + 1 file changed, 1 insertion(+) + create mode 120000 events/2015/test/foo + test (master)> git annex fsck . + Warning: Fscking a repository that is currently marked as dead. + fsck foo (checksum...) + ** No known copies exist of foo + failed + (recording state in git...) + git-annex: fsck: 1 failed + +At this point, I suspect that the repository on my laptop has some serious problem. Shall I move this discussion to the forum? It may be a git-annex bug but it can just be that the repository is damaged for other reasons. Moreover, what is the best course of action, at this point? +"""]] diff --git a/doc/bugs/after_git_annex_add_and_commit__44___git_annex_fsck_fails__58___no_known_copies/comment_5_a256583bd9b3815a23cd1ca40d6c19ca._comment b/doc/bugs/after_git_annex_add_and_commit__44___git_annex_fsck_fails__58___no_known_copies/comment_5_a256583bd9b3815a23cd1ca40d6c19ca._comment new file mode 100644 index 0000000000..785d6c7b77 --- /dev/null +++ b/doc/bugs/after_git_annex_add_and_commit__44___git_annex_fsck_fails__58___no_known_copies/comment_5_a256583bd9b3815a23cd1ca40d6c19ca._comment @@ -0,0 +1,10 @@ +[[!comment format=mdwn + username="emanuele.olivetti@47d88ed185b03191e25329caa6fabc2efb3118b2" + nickname="emanuele.olivetti" + subject="'git-annex semitrust' seems to solve the issue" + date="2016-02-18T21:51:00Z" + content=""" +Given that the repository is marked as dead, I tried to bring it back with 'git-annex semitrust laptop' - 'laptop' being the name of the git annex repository on my laptop. Apparently, this action solved the problem. Fsck now works as expected, and I could sync and get the new files on the clone on the external hard disk as usual. + +Now, if this the correct solution to my problem, I am wondering how it is possible that the repository was marked as dead. I carefully checked the history of previous command, but no sign of 'git annex dead' o similar. Are there scenarios where a repository is automatically marked as dead? +"""]] diff --git a/doc/bugs/after_git_annex_add_and_commit__44___git_annex_fsck_fails__58___no_known_copies/comment_6_0999f9ec9c3d6f51889141344d4cfcb6._comment b/doc/bugs/after_git_annex_add_and_commit__44___git_annex_fsck_fails__58___no_known_copies/comment_6_0999f9ec9c3d6f51889141344d4cfcb6._comment new file mode 100644 index 0000000000..a2e678c1b8 --- /dev/null +++ b/doc/bugs/after_git_annex_add_and_commit__44___git_annex_fsck_fails__58___no_known_copies/comment_6_0999f9ec9c3d6f51889141344d4cfcb6._comment @@ -0,0 +1,16 @@ +[[!comment format=mdwn + username="joey" + subject="""comment 6""" + date="2016-02-19T18:52:25Z" + content=""" +Ok, the repository being dead certianly explains why fsck found a problem. + +It may be that fsck should have a better message in this case. Like +"only dead repository had copies of the file". + +Repositories are only ever marked dead on user request. There are a few +ways besides `git annex dead` that you could have done it. Perhaps trying +to delete the repo in the webapp, or changing it in `git annex vicfg`. +You can check out the git-annex branch and examine the history of the +trust.log file to see when it was changed to dead. +"""]] diff --git a/doc/bugs/android_4.3_install_failed_.mdwn b/doc/bugs/android_4.3_install_failed_.mdwn new file mode 100644 index 0000000000..c073e2d2f2 --- /dev/null +++ b/doc/bugs/android_4.3_install_failed_.mdwn @@ -0,0 +1,22 @@ +### Please describe the problem. +Impossible installation on Android 4.3 + + +### What version of git-annex are you using? On what operating system? +The lastest version of git-annex, and Android 4.3, **without sdcard** (Wiko Wax) + +### Please provide any additional information below. + +The message given by git-annex: + + + Falling back to hardcoded app location; cannot find expected files in /data/app-lib + mkdir: can't create directory '/sdcard/git-annex.home': Permission denied + mkdir of /sdcard/git-annex.home failed ! + lib/lib.runshell.so: line 133: can't create /sdcard/git-annex.home/git-annex-install.log: Permission denied + Installation failed ! Please report a but and attach /sdcard/git-annex.home/git-annex-install.log + +[[!meta title="android 4.3 install failed on android device without sdcard"]] +[[!tag moreinfo]] + +> Closing as this was a bug in the deprecated Android app. [[done]] --[[Joey]] diff --git a/doc/bugs/android_4.3_install_failed_/comment_1_82447f1e24d7e8df8048464d1b7df117._comment b/doc/bugs/android_4.3_install_failed_/comment_1_82447f1e24d7e8df8048464d1b7df117._comment new file mode 100644 index 0000000000..682780cb2d --- /dev/null +++ b/doc/bugs/android_4.3_install_failed_/comment_1_82447f1e24d7e8df8048464d1b7df117._comment @@ -0,0 +1,8 @@ +[[!comment format=mdwn + username="http://joeyh.name/" + ip="24.159.78.125" + subject="comment 1" + date="2014-06-02T16:28:42Z" + content=""" +git-annex needs a directory to use to store its repositories. If your android device does not have a /sdcard directory (which does not have to be a literal sd card, but just someplace you can write to), does it have any equivilant directory? +"""]] diff --git a/doc/bugs/android_4.3_install_failed_/comment_2_67ace7c454c7e962ca69e42178142e80._comment b/doc/bugs/android_4.3_install_failed_/comment_2_67ace7c454c7e962ca69e42178142e80._comment new file mode 100644 index 0000000000..09374e8ee6 --- /dev/null +++ b/doc/bugs/android_4.3_install_failed_/comment_2_67ace7c454c7e962ca69e42178142e80._comment @@ -0,0 +1,8 @@ +[[!comment format=mdwn + username="https://www.google.com/accounts/o8/id?id=AItOawkPSASwemBzccJmjkotESlbUSs5GPFiPCs" + nickname="Lin" + subject="Ok, but ?" + date="2014-06-02T18:22:24Z" + content=""" +You mean that I have to create the missed files ? +"""]] diff --git a/doc/bugs/android_4.3_install_failed_/comment_3_051e39129a38e439f24703385f503cf4._comment b/doc/bugs/android_4.3_install_failed_/comment_3_051e39129a38e439f24703385f503cf4._comment new file mode 100644 index 0000000000..3f6e86ce4e --- /dev/null +++ b/doc/bugs/android_4.3_install_failed_/comment_3_051e39129a38e439f24703385f503cf4._comment @@ -0,0 +1,8 @@ +[[!comment format=mdwn + username="http://joeyh.name/" + ip="108.236.230.124" + subject="comment 3" + date="2014-06-04T18:08:39Z" + content=""" +I asked if your Android device has an eqivilant directory to /sdcard that you can write to. If so, I can fix git-annex to use that directory. But you need to tell me what it is first! +"""]] diff --git a/doc/bugs/android_4.3_install_failed_/comment_4_5083a8a3fa21c00f70b24c29ed8b6652._comment b/doc/bugs/android_4.3_install_failed_/comment_4_5083a8a3fa21c00f70b24c29ed8b6652._comment new file mode 100644 index 0000000000..ef6d01709c --- /dev/null +++ b/doc/bugs/android_4.3_install_failed_/comment_4_5083a8a3fa21c00f70b24c29ed8b6652._comment @@ -0,0 +1,15 @@ +[[!comment format=mdwn + username="joey" + subject="""comment 4""" + date="2018-05-08T18:56:42Z" + content=""" +I'm closing this bug report because the git-annex Android app that it +was reported on is now deprecated. Instead, we have +a way to run the regular git-annex build for linux on Android in termux: + + +There were a lot of problems with the way the git-annex Android app was +put together, and I hope this new approach avoids them. If you try it and +still have the bug you reported, please followup and I'll reopen it. + +"""]] diff --git a/doc/bugs/android__58___cannot_link_executable.mdwn b/doc/bugs/android__58___cannot_link_executable.mdwn new file mode 100644 index 0000000000..8c8f60cac4 --- /dev/null +++ b/doc/bugs/android__58___cannot_link_executable.mdwn @@ -0,0 +1,26 @@ +### Please describe the problem. / What steps will reproduce the problem? +1. Download http://downloads.kitenet.net/git-annex/android/current/5.0/git-annex.apk +2. Start the App +3. Error Message + * CANNOT LINK EXECUTABLE "git-annex": /data/app/ga.androidterm-1/lib/arm/lib.git-annex.so has text relocations + * error: git-annex died of signal 6 + +### What version of git-annex are you using? On what operating system? +* Android 7.0 +* Nexus 5X +* no root +* git-annex 5.0 + +### Please provide any additional information below. + +[[!format sh """ +# If you can, paste a complete transcript of the problem occurring here. +# If the problem is with the git-annex assistant, paste in .git/annex/daemon.log + + +# End of transcript or log. +"""]] + +### Have you had any luck using git-annex before? (Sometimes we get tired of reading bug reports all day and a lil' positive end note does wonders) + +> Closing as this was a bug in the deprecated Android app. [[done]] --[[Joey]] diff --git a/doc/bugs/android__58___cannot_link_executable/comment_1_17a502aba0bd260801c9ca01664abd1e._comment b/doc/bugs/android__58___cannot_link_executable/comment_1_17a502aba0bd260801c9ca01664abd1e._comment new file mode 100644 index 0000000000..ef682de31a --- /dev/null +++ b/doc/bugs/android__58___cannot_link_executable/comment_1_17a502aba0bd260801c9ca01664abd1e._comment @@ -0,0 +1,9 @@ +[[!comment format=mdwn + username="joey" + subject="""comment 1""" + date="2016-09-05T16:31:56Z" + content=""" +People have reported similar problems for years, and always on Nexus +devices. So it seems that something done to Android on Nexus is the root of +the problem. +"""]] diff --git a/doc/bugs/android__58___cannot_link_executable/comment_2_1057c0477050e52e463c36e03fcab09d._comment b/doc/bugs/android__58___cannot_link_executable/comment_2_1057c0477050e52e463c36e03fcab09d._comment new file mode 100644 index 0000000000..a8469cfe0f --- /dev/null +++ b/doc/bugs/android__58___cannot_link_executable/comment_2_1057c0477050e52e463c36e03fcab09d._comment @@ -0,0 +1,9 @@ +[[!comment format=mdwn + username="moc514@eb7af2cd9147722b29f32b6606feb2b8563dfac8" + nickname="moc514" + avatar="http://cdn.libravatar.org/avatar/c8c98fc66ef014e61c163375ca9e7422" + subject="Nexus 6p" + date="2016-12-16T02:08:21Z" + content=""" +I also have the same issue with the Nexus 6p with 7.1.1 +"""]] diff --git a/doc/bugs/android__58___cannot_link_executable/comment_3_0039743728e02ca27f5346c74e8bad50._comment b/doc/bugs/android__58___cannot_link_executable/comment_3_0039743728e02ca27f5346c74e8bad50._comment new file mode 100644 index 0000000000..0bb2b832f1 --- /dev/null +++ b/doc/bugs/android__58___cannot_link_executable/comment_3_0039743728e02ca27f5346c74e8bad50._comment @@ -0,0 +1,13 @@ +[[!comment format=mdwn + username="https://itorres.net/" + nickname="Ignacio Torres " + avatar="http://cdn.libravatar.org/avatar/525cc09b4eb4dc770fe20209f4a240268cad5b027135556bd27acbca9d0d3bc8" + subject="Issue appeared on Android 6.0 (SDK 23)" + date="2016-12-31T14:33:25Z" + content=""" +> On previous versions of Android, if your app requested the system to load a shared library with text relocations, the system displayed a warning but still allowed the library to be loaded. Beginning in this release, the system rejects this library if your app's target SDK version is 23 or higher. + +Source: https://developer.android.com/about/versions/marshmallow/android-6.0-changes.html#behavior-runtime + +I experience the same issue on a CM14 (Android 7) Moto X +"""]] diff --git a/doc/bugs/android__58___cannot_link_executable/comment_4_1e86ba33f6b709bf8bc72b70adbc73dd._comment b/doc/bugs/android__58___cannot_link_executable/comment_4_1e86ba33f6b709bf8bc72b70adbc73dd._comment new file mode 100644 index 0000000000..ea86f4d99d --- /dev/null +++ b/doc/bugs/android__58___cannot_link_executable/comment_4_1e86ba33f6b709bf8bc72b70adbc73dd._comment @@ -0,0 +1,10 @@ +[[!comment format=mdwn + username="https://christian.amsuess.com/chrysn" + avatar="http://christian.amsuess.com/avatar/c6c0d57d63ac88f3541522c4b21198c3c7169a665a2f2d733b4f78670322ffdc" + subject="Issue also affects Samsung devices, git unaffected" + date="2017-09-11T18:07:51Z" + content=""" +I'm experiencing this on a Samsung SM-T813 (arm64) with Android 7.0. + +Running `git` commands or busybox commands in the shipped shell works, this seems to affect the git-annex binary only. +"""]] diff --git a/doc/bugs/android__58___cannot_link_executable/comment_5_6c94c327f6dc1c039f9bc5cfaea455ee._comment b/doc/bugs/android__58___cannot_link_executable/comment_5_6c94c327f6dc1c039f9bc5cfaea455ee._comment new file mode 100644 index 0000000000..f39e4edb38 --- /dev/null +++ b/doc/bugs/android__58___cannot_link_executable/comment_5_6c94c327f6dc1c039f9bc5cfaea455ee._comment @@ -0,0 +1,19 @@ +[[!comment format=mdwn + username="https://christian.amsuess.com/chrysn" + avatar="http://christian.amsuess.com/avatar/c6c0d57d63ac88f3541522c4b21198c3c7169a665a2f2d733b4f78670322ffdc" + subject="Issue also affects LineageOS" + date="2017-12-17T12:04:52Z" + content=""" +I've installed LineageOS (Android 7.1.2 Nightly) on aforementioned SM-T813. + +The issue affects that setup as well, both in the \"current\" and the \"autobuild\" version for Android 5.0. + +Full message: + + Falling back to hardcoded app location; cannot find expected files in /data/app/ga.androidterm-1/lib + git annex webapp + gts210vewifi:/sdcard/git-annex.home $ git annex webapp + CANNOT LINK EXECUTABLE \"git-annex\": /data/app/ga.androidterm-1/lib/arm/lib.git-annex.so: has text relocations + error: git-annex died of signal 6 + 134|gts210vewifi:/sdcard/git-annex.home $ +"""]] diff --git a/doc/bugs/android__58___cannot_link_executable/comment_6_059393465b26596c0af8cfaaef46c11d._comment b/doc/bugs/android__58___cannot_link_executable/comment_6_059393465b26596c0af8cfaaef46c11d._comment new file mode 100644 index 0000000000..8563f0d709 --- /dev/null +++ b/doc/bugs/android__58___cannot_link_executable/comment_6_059393465b26596c0af8cfaaef46c11d._comment @@ -0,0 +1,14 @@ +[[!comment format=mdwn + username="joey" + subject="""comment 6""" + date="2018-04-16T20:36:19Z" + content=""" +Apparently this might have to do with ghc passing -no-pie to GCC, +and they added a -fPIE flag to request a PIE. + + +Since the git-annex for android autobuilder uses a hacked up ghc +cross compiler for android, which is quite out of date, that would need to +be updated, as well as probably refeshing all the cabal library patches and +hacks for android. +"""]] diff --git a/doc/bugs/android__58___cannot_link_executable/comment_7_1d37b251f1f9bf31647eb13322b92cce._comment b/doc/bugs/android__58___cannot_link_executable/comment_7_1d37b251f1f9bf31647eb13322b92cce._comment new file mode 100644 index 0000000000..51d6a1f127 --- /dev/null +++ b/doc/bugs/android__58___cannot_link_executable/comment_7_1d37b251f1f9bf31647eb13322b92cce._comment @@ -0,0 +1,15 @@ +[[!comment format=mdwn + username="joey" + subject="""comment 7""" + date="2018-04-16T23:49:20Z" + content=""" +Even with gcc and ld wrapped to pass `-pie` and strip `-no-pie`, +ghc is not producing PIE binaries. Verified with +"readelf -s git-annex | grep main", and the address for +main is a full address not an offset. + +The same android gcc toolchain does produce +PIE executables by default when building C code. + +Unclear why.. +"""]] diff --git a/doc/bugs/android__58___cannot_link_executable/comment_8_ce4c16b0cc064fd6f8314167255c470a._comment b/doc/bugs/android__58___cannot_link_executable/comment_8_ce4c16b0cc064fd6f8314167255c470a._comment new file mode 100644 index 0000000000..3c34530372 --- /dev/null +++ b/doc/bugs/android__58___cannot_link_executable/comment_8_ce4c16b0cc064fd6f8314167255c470a._comment @@ -0,0 +1,15 @@ +[[!comment format=mdwn + username="joey" + subject="""comment 8""" + date="2018-04-25T17:49:30Z" + content=""" +We may be moving away from the git-annex Android app, and to running +git-annex in Termux on Android. + +I think that will avoid this particular problem, since it uses the linux +build of git-annex which bundles the linux linker, so the behavior of the +android linker is no longer an issue. + +If the people who have been bitten by this bug want to give it a try, +see +"""]] diff --git a/doc/bugs/android__58___cannot_link_executable/comment_9_fe04f4cff3d3cb3471edebb25ee37a4d._comment b/doc/bugs/android__58___cannot_link_executable/comment_9_fe04f4cff3d3cb3471edebb25ee37a4d._comment new file mode 100644 index 0000000000..fa38741377 --- /dev/null +++ b/doc/bugs/android__58___cannot_link_executable/comment_9_fe04f4cff3d3cb3471edebb25ee37a4d._comment @@ -0,0 +1,14 @@ +[[!comment format=mdwn + username="joey" + subject="""comment 9""" + date="2018-05-08T18:46:53Z" + content=""" +I'm closing this bug report because the git-annex Android app that it +was reported on is now deprecated. Instead, we have +a way to run the regular git-annex build for linux on Android in termux: + + +There were a lot of problems with the way the git-annex Android app was +put together, and I hope this new approach avoids them. If you try it and +still have the bug you reported, please followup and I'll reopen it. +"""]] diff --git a/doc/bugs/android_ed25519_algorithm.mdwn b/doc/bugs/android_ed25519_algorithm.mdwn new file mode 100644 index 0000000000..392e1d780a --- /dev/null +++ b/doc/bugs/android_ed25519_algorithm.mdwn @@ -0,0 +1,14 @@ +### Please describe the problem. +Openssh was not compiled to support ed25519 algorithm + +### What steps will reproduce the problem? +only enable ed25519 on server and try to connect via ssh. +fails with "no hostkey alg" + +### What version of git-annex are you using? On what operating system? +5.20150219-gd24cfd3, Android 5.0.1 + +regards, +David + +> Closing as this was a bug in the deprecated Android app. [[done]] --[[Joey]] diff --git a/doc/bugs/android_ed25519_algorithm/comment_1_d60c9a7c3a3f12c14a30a8790be6c65b._comment b/doc/bugs/android_ed25519_algorithm/comment_1_d60c9a7c3a3f12c14a30a8790be6c65b._comment new file mode 100644 index 0000000000..a431dfc3a4 --- /dev/null +++ b/doc/bugs/android_ed25519_algorithm/comment_1_d60c9a7c3a3f12c14a30a8790be6c65b._comment @@ -0,0 +1,14 @@ +[[!comment format=mdwn + username="joey" + subject="""comment 1""" + date="2018-05-08T18:53:06Z" + content=""" +I'm closing this bug report because the git-annex Android app that it +was reported on is now deprecated. Instead, we have +a way to run the regular git-annex build for linux on Android in termux: + + +There were a lot of problems with the way the git-annex Android app was +put together, and I hope this new approach avoids them. If you try it and +still have the bug you reported, please followup and I'll reopen it. +"""]] diff --git a/doc/bugs/android_ed25519_algorithm/comment_2_7669b8e5bbbd2c474f0516e7da407f09._comment b/doc/bugs/android_ed25519_algorithm/comment_2_7669b8e5bbbd2c474f0516e7da407f09._comment new file mode 100644 index 0000000000..3e4e9604a8 --- /dev/null +++ b/doc/bugs/android_ed25519_algorithm/comment_2_7669b8e5bbbd2c474f0516e7da407f09._comment @@ -0,0 +1,11 @@ +[[!comment format=mdwn + username="gdkags@28c74e22834edcf63c178e595b8649e5af5151d4" + nickname="gdkags" + avatar="http://cdn.libravatar.org/avatar/d2cf888e824158a3b2d0efe35e163cf2" + subject="fine with me!" + date="2018-05-10T13:02:23Z" + content=""" +Thank you for the feedback! Have fun with the spring-cleaning :D +regards, +David +"""]] diff --git a/doc/bugs/annex-checkuuid_renders_remotes_inaccessible.mdwn b/doc/bugs/annex-checkuuid_renders_remotes_inaccessible.mdwn new file mode 100644 index 0000000000..812d8e00fc --- /dev/null +++ b/doc/bugs/annex-checkuuid_renders_remotes_inaccessible.mdwn @@ -0,0 +1,42 @@ +### Please describe the problem. + +Setting `remote..annex-checkuuid` to `false` renders remote `` inaccessible for git-annex, i.e., when using commands such as `get`. + +### What steps will reproduce the problem? + + $ cat git-annex-checkuuid.sh + pushd $(mktemp -d) + + git init origin + pushd origin + git annex init origin + touch blob + git annex add blob + git annex sync + + popd + git clone origin clone + pushd clone + git annex init clone + git config remote.origin.annex-checkuuid false + git annex get blob + + $ bash git-annex-checkuuid.sh + … + get blob + Unable to access these remotes: origin + + Try making some of these repositories available: + d27a9c2d-af76-4b01-8335-bc2d72a1d28a -- [origin] + failed + git-annex: get: 1 failed + +### What version of git-annex are you using? On what operating system? + + git-annex version: 6.20180529-g33834140e + build flags: Assistant Webapp Pairing S3(multipartupload)(storageclasses) WebDAV Inotify DBus DesktopNotify ConcurrentOutput TorrentParser MagicMime Feeds Testsuite + dependency versions: aws-0.20 bloomfilter-2.0.1.0 cryptonite-0.25 DAV-1.3.2 feed-1.0.0.0 ghc-8.4.2 http-client-0.5.12.1 persistent-sqlite-2.8.1.2 torrent-10000.1.1 uuid-1.3.13 yesod-1.6.0 + key/value backends: SHA256E SHA256 SHA512E SHA512 SHA224E SHA224 SHA384E SHA384 SHA3_256E SHA3_256 SHA3_512E SHA3_512 SHA3_224E SHA3_224 SHA3_384E SHA3_384 SKEIN256E SKEIN256 SKEIN512E SKEIN512 BLAKE2B256E BLAKE2B256 BLAKE2B512E BLAKE2B512 BLAKE2B160E BLAKE2B160 BLAKE2B224E BLAKE2B224 BLAKE2B384E BLAKE2B384 BLAKE2S256E BLAKE2S256 BLAKE2S160E BLAKE2S160 BLAKE2S224E BLAKE2S224 BLAKE2SP256E BLAKE2SP256 BLAKE2SP224E BLAKE2SP224 SHA1E SHA1 MD5E MD5 WORM URL + remote types: git gcrypt p2p S3 bup directory rsync web bittorrent webdav adb tahoe glacier ddar hook external + +> [[fixed|done]] --[[Joey]] diff --git a/doc/bugs/annex-checkuuid_renders_remotes_inaccessible/comment_1_451498fab01ae745eb33d42da8682ad3._comment b/doc/bugs/annex-checkuuid_renders_remotes_inaccessible/comment_1_451498fab01ae745eb33d42da8682ad3._comment new file mode 100644 index 0000000000..b88e5f7d87 --- /dev/null +++ b/doc/bugs/annex-checkuuid_renders_remotes_inaccessible/comment_1_451498fab01ae745eb33d42da8682ad3._comment @@ -0,0 +1,28 @@ +[[!comment format=mdwn + username="joey" + subject="""comment 1""" + date="2018-06-04T16:44:39Z" + content=""" +I think checkuuid=false has always had this problem; it's never been +actually usable. + +Problem is that checkuuid=false prevents reading the git config, so +the Repo is LocalUnknown, and such repositories are assumed to not be +currently available to use. + +To get from LocalUnknown to Local, it would have to read the repo's +config to determine if the repo is bare or not. But reading the repo's +config every time is exactly what checkuuid=false is intended to prevent. + +Only path I can see is to make the DeferredUUIDCheck that's done +with checkuuid=false return the Local Repo. That could be done in +Remote.Git. But, other modules use Remote.repo to look at the Repo too, +including Command.Sync, and Assistant.TransferSlots. Remote.repo can't +be updated after it's constructed without some mess of updating the remote +list, which does not seem like a good idea. Seems that Remote.repo would +need to be converted to an IO action. + +(So would Remote.gitconfig, which incidentally doesn't contain the config +of the remote when checkuuid=false, which could also be considered a bug +with checkuuid=false.) +"""]] diff --git a/doc/bugs/annex-checkuuid_renders_remotes_inaccessible/comment_2_298b1a8418380d932464962ca00fb2f3._comment b/doc/bugs/annex-checkuuid_renders_remotes_inaccessible/comment_2_298b1a8418380d932464962ca00fb2f3._comment new file mode 100644 index 0000000000..625ed48de2 --- /dev/null +++ b/doc/bugs/annex-checkuuid_renders_remotes_inaccessible/comment_2_298b1a8418380d932464962ca00fb2f3._comment @@ -0,0 +1,12 @@ +[[!comment format=mdwn + username="joey" + subject="""comment 2""" + date="2018-06-04T17:40:23Z" + content=""" +I've changed Remote.repo to Remote.getRepo so IO can be done there. + +The remaining problem is the remoteGitConfig part +of RemoteGitConfig. That is dependent on the git config of the remote being +read when a Remote is constructed, and when checkuuid=false, that won't be +done. Only a few things use it, but my first try at removing it failed. +"""]] diff --git a/doc/bugs/annex-checkuuid_renders_remotes_inaccessible/comment_3_4e68ba8e7bc1e3d935093bf45c5afb10._comment b/doc/bugs/annex-checkuuid_renders_remotes_inaccessible/comment_3_4e68ba8e7bc1e3d935093bf45c5afb10._comment new file mode 100644 index 0000000000..74868844e6 --- /dev/null +++ b/doc/bugs/annex-checkuuid_renders_remotes_inaccessible/comment_3_4e68ba8e7bc1e3d935093bf45c5afb10._comment @@ -0,0 +1,12 @@ +[[!comment format=mdwn + username="joey" + subject="""comment 3""" + date="2018-06-04T20:43:13Z" + content=""" +Since I have been working on this for 4 hours now and want to +end with something usable today, I went ahead with the fix. + +Opened a todo to keep track of the remaining problems with the remote's git +config not being read: +[[todo/need_to_remove_remoteGitConfig_for_checkuuid_support]] +"""]] diff --git a/doc/bugs/annex-lookupkey_fails_to_deal_with_absolute_paths.mdwn b/doc/bugs/annex-lookupkey_fails_to_deal_with_absolute_paths.mdwn new file mode 100644 index 0000000000..f08d822b61 --- /dev/null +++ b/doc/bugs/annex-lookupkey_fails_to_deal_with_absolute_paths.mdwn @@ -0,0 +1,56 @@ +### Please describe the problem. +Apparently, git-annex-lookupkey can't handle absolute paths to files to look for. + +### What steps will reproduce the problem? + + /tmp % mkdir some + /tmp % cd some + /tmp/some % git init + Initialized empty Git repository in /tmp/some/.git/ + /tmp/some % git annex init + init ok + (recording state in git...) + /tmp/some % echo some > some + /tmp/some % git annex add some + add some ok + (recording state in git...) + /tmp/some % git annex lookupkey some + SHA256E-s5--2922bee6973370915cc63ab5ab8b7a57e1cab909477d7a030b2e4661e7aa2202 + /tmp/some % echo $? + 0 + /tmp/some % git annex lookupkey /tmp/some/some + /tmp/some % echo $? + 1 + + + +### What version of git-annex are you using? On what operating system? + % git annex version + git-annex version: 6.20171018+gitgbb20b1ed3-1~ndall+1 + build flags: Assistant Webapp Pairing Testsuite S3(multipartupload)(storageclasses) WebDAV Inotify DBus DesktopNotify ConcurrentOutput TorrentParser MagicMime Feeds Quvi + dependency versions: aws-0.14.1 bloomfilter-2.0.1.0 cryptonite-0.20 DAV-1.3.1 feed-0.3.11.1 ghc-8.0.1 http-client-0.4.31.1 persistent-sqlite-2.6 torrent-10000.0.0 uuid-1.3.12 yesod-1.4.3 + key/value backends: SHA256E SHA256 SHA512E SHA512 SHA224E SHA224 SHA384E SHA384 SHA3_256E SHA3_256 SHA3_512E SHA3_512 SHA3_224E SHA3_224 SHA3_384E SHA3_384 SKEIN256E SKEIN256 SKEIN512E SKEIN512 SHA1E SHA1 MD5E MD5 WORM URL + remote types: git gcrypt p2p S3 bup directory rsync web bittorrent webdav tahoe glacier ddar hook external + local repository version: 5 + supported repository versions: 3 5 6 + upgrade supported from repository versions: 0 1 2 3 4 5 + operating system: linux x86_64 + + +### Please provide any additional information below. + +[[!format sh """ +# If you can, paste a complete transcript of the problem occurring here. +# If the problem is with the git-annex assistant, paste in .git/annex/daemon.log + + +# End of transcript or log. +"""]] + +### Have you had any luck using git-annex before? (Sometimes we get tired of reading bug reports all day and a lil' positive end note does wonders) + +I do! I wouldn't even have my job, if it wasn't for git-annex. ;-) + +> Which warms the cockles of my heart, Ben! :) +> +> [[fixed|done]] --[[Joey]] diff --git a/doc/bugs/annex-lookupkey_fails_to_deal_with_absolute_paths/comment_1_b9aa0c69b8841c8fe885a5e2e9ec9a06._comment b/doc/bugs/annex-lookupkey_fails_to_deal_with_absolute_paths/comment_1_b9aa0c69b8841c8fe885a5e2e9ec9a06._comment new file mode 100644 index 0000000000..72ff677fc4 --- /dev/null +++ b/doc/bugs/annex-lookupkey_fails_to_deal_with_absolute_paths/comment_1_b9aa0c69b8841c8fe885a5e2e9ec9a06._comment @@ -0,0 +1,25 @@ +[[!comment format=mdwn + username="joey" + subject="""comment 1""" + date="2017-12-08T18:58:18Z" + content=""" +This is due to lookupkey not passing the filename through git ls-files like +most (all?) other commands do. + +Using git ls-files would lead to other behavior changes though. It recurses into +directories. I don't think it makes sense for lookupkey to recurse into +directories, because its output format does not include the filename, so +listing a bunch of keys for annexed files is not clear. git annex find +can already recurse and can use a format with the key and the filename +that's suited for directory recursion. git annex lookupkey, as plumbing, +is supposed to be simpler than that. + +I suppose lookupkey could normalize absolute file paths, checking if they +point into the git repository. I don't think that git-annex contains +such normalization code, and it might be a lot more complicated than it at +first seems -- git has a lot of wrinkles with submodules, symlinks, etc etc. +git does not seem to have a suitable command to do it. + +So seems the best way is to use git ls-files and detect when it's recursing, +and exit nonzero then. +"""]] diff --git a/doc/bugs/annex.autocommit_seems_ignored_for_new_files.mdwn b/doc/bugs/annex.autocommit_seems_ignored_for_new_files.mdwn new file mode 100644 index 0000000000..a06d3aa5a6 --- /dev/null +++ b/doc/bugs/annex.autocommit_seems_ignored_for_new_files.mdwn @@ -0,0 +1,39 @@ +### Please describe the problem. + +When `annex.autocommit` is set to `false`, the assistant continues to commit to the repository, at least in indirect mode (direct mode not tested). I'd like it to not even `git annex add`, but it should at least not `git commit`. + +### What steps will reproduce the problem? + +I started the assistant with `git annex assistant`. Then I also opened the webapp with `git annex website`. + +Next, I ran `git config --bool --local --replace-all annex.autocommit false`. Supposedly that'll stop the assistant from committing things. + +Then I did a `echo "hello world" > test`. + +The webapp showed the file syncing, and `ls -l test` confirmed it'd been added the annex. `git status` confirmed it'd been committed as well. + +I did a `git rm test`, and then a `git status`... and it turns out the annex committed that as well. + +NOTE: My repository is in indirect mode. + +I tried `--global` instead of `--local` for the config, and that didn't make a difference. + + +### What version of git-annex are you using? On what operating system? + +5.20141125 on Debian testing, from the package. (And, thank you Joey, for all your work on Debian. You'll be missed.) + +### Please provide any additional information below. + +> The assistant does not notice changes to the +> git configuration that are made while it's already running. +> +> I have verified that setting `git config annex.autocommit true` +> and then starting the assistant behaves as documented. +> +> So will asking the webapp to restart. +> +> Retitling bug report appropriately. --[[Joey]] + +[[!meta title="git-annex assistant could watch .git/config and react to config changes on the fly"]] +[[!tag confirmed]] diff --git a/doc/bugs/annex.genmetadata_should_default_to_true.mdwn b/doc/bugs/annex.genmetadata_should_default_to_true.mdwn new file mode 100644 index 0000000000..afc004ed0a --- /dev/null +++ b/doc/bugs/annex.genmetadata_should_default_to_true.mdwn @@ -0,0 +1,11 @@ +### Please describe the problem. +annex.genmetadata is off by default, which will potentially lose you a lot of timestamp metadata if you don't pay attention. It should be on by default. + +### What steps will reproduce the problem? +Add a file to a fresh annex, observe it has no metadata. + +### What version of git-annex are you using? On what operating system? +6.20180227 on Debian + +### Have you had any luck using git-annex before? (Sometimes we get tired of reading bug reports all day and a lil' positive end note does wonders) +Yes I love it! (Except for its spotty timestamp support) diff --git a/doc/bugs/annex.genmetadata_should_default_to_true/comment_1_215ca5e67a1bc04bcb8bea6062aebf3f._comment b/doc/bugs/annex.genmetadata_should_default_to_true/comment_1_215ca5e67a1bc04bcb8bea6062aebf3f._comment new file mode 100644 index 0000000000..48822f22a8 --- /dev/null +++ b/doc/bugs/annex.genmetadata_should_default_to_true/comment_1_215ca5e67a1bc04bcb8bea6062aebf3f._comment @@ -0,0 +1,23 @@ +[[!comment format=mdwn + username="joey" + subject="""comment 1""" + date="2018-04-04T17:50:02Z" + content=""" +I think there are two potential problems with this change: + +1. Bloat. If someone is not going to use that metadata, they may not want + the overhead of attaching it to every file. + +2. It's entirely possible that a user has chosen to use year/month/day + metadata fields for their own purposes. Perhaps they are using it to + keep track of the original publication date of files, and so would not + want "wrong" values to be automatically added. + +And that's essentially the reason why all parts of git-annex that deal +with specific metadata fields are optional, so there's no default +hard-coded semantics to the metadata fields. + +(With the exception of git-annex importfeed, which always +stores the itemid from the rss feed in the metadata, but only because +it's needed to detect buggy feeds that change their item urls.) +"""]] diff --git a/doc/bugs/annex.genmetadata_should_default_to_true/comment_2_0966f8bf944ded0a5199285ed3c12930._comment b/doc/bugs/annex.genmetadata_should_default_to_true/comment_2_0966f8bf944ded0a5199285ed3c12930._comment new file mode 100644 index 0000000000..cea41ad1db --- /dev/null +++ b/doc/bugs/annex.genmetadata_should_default_to_true/comment_2_0966f8bf944ded0a5199285ed3c12930._comment @@ -0,0 +1,10 @@ +[[!comment format=mdwn + username="vrs+annex@ea5fa24dbb279be61a8e50adb638bf8366300717" + nickname="vrs+annex" + avatar="http://cdn.libravatar.org/avatar/74219abcec6eece8e2c9d4351c2c912c" + subject="comment 2" + date="2018-04-05T01:32:32Z" + content=""" +1. Not losing data (or alternatively having a great big warning in the manual, or requiring it as a configuration step) should be the default in software that manages files, especially software that advertises a backup usecase on its front page. I do think the current implementation could be improved, which is what I opened for. If that's not enough, a simple binary format should do the trick at about the same overhead as regular filesystems while staying future-proof. +2. Implementing timestamps with a field name that doesn't clash with existing fields should avoid this issue. +"""]] diff --git a/doc/bugs/annex.genmetadata_should_default_to_true/comment_3_7322a9f7d8d01e8daf67bfb08ce7042c._comment b/doc/bugs/annex.genmetadata_should_default_to_true/comment_3_7322a9f7d8d01e8daf67bfb08ce7042c._comment new file mode 100644 index 0000000000..8d0efd3ecb --- /dev/null +++ b/doc/bugs/annex.genmetadata_should_default_to_true/comment_3_7322a9f7d8d01e8daf67bfb08ce7042c._comment @@ -0,0 +1,8 @@ +[[!comment format=mdwn + username="CandyAngel" + avatar="http://cdn.libravatar.org/avatar/15c0aade8bec5bf004f939dd73cf9ed8" + subject="comment 3" + date="2018-04-05T15:45:52Z" + content=""" +As far as I am aware, git-annex doesn't store *any* information about any files (including their content!) without fairly explicit instruction to do so.. this would be exception to that general behaviour. +"""]] diff --git a/doc/bugs/annex_add_ignores_.-prefixed_directories.mdwn b/doc/bugs/annex_add_ignores_.-prefixed_directories.mdwn new file mode 100644 index 0000000000..5db40f4dd6 --- /dev/null +++ b/doc/bugs/annex_add_ignores_.-prefixed_directories.mdwn @@ -0,0 +1,78 @@ +### Please describe the problem. + +annex add seems to ignore content under directories having . prefix. + +We thought to unify (across direct/indirect/v6) adding files to annex repository by using 'git annex add' with corresponding setting for largefiles for any addition, but it seems to ignore content under .-prefixed directories, unlike git + +### What version of git-annex are you using? On what operating system? + +6.20161122+gitg9f179ae-1~ndall+1 + +### Please provide any additional information below. + +[[!format sh """ +hopa:/tmp/datalad_temp_test_annex_add_no_dotfilesqMXck8 +$> git status +On branch master + +Initial commit + +nothing to commit (create/copy files and use "git add" to track) + +$> mkdir .dir dir; echo 123 > .dir/123; echo 124 > dir/124 + +$> git status +On branch master + +Initial commit + +Untracked files: + (use "git add ..." to include in what will be committed) + + .dir/ + dir/ + +nothing added to commit but untracked files present (use "git add" to track) + +$> git annex add -c 'annex.largefiles=nothing' . +add dir/124 (non-large file; adding content to git repository) ok +(recording state in git...) + +$> git status +On branch master + +Initial commit + +Changes to be committed: + (use "git rm --cached ..." to unstage) + + new file: dir/124 + +Untracked files: + (use "git add ..." to include in what will be committed) + + .dir/ + + +# and with regular git +$> git -c 'annex.largefiles=nothing' add . + +$> git status +On branch master + +Initial commit + +Changes to be committed: + (use "git rm --cached ..." to unstage) + + new file: .dir/123 + new file: dir/124 + + +"""]] + +Ref: https://github.com/datalad/datalad/issues/1027 + +[[!meta author=yoh]] + +[[done]]; oh -- it is RTFM: --include-dotfiles --[[yoh]] diff --git a/doc/bugs/annex_copy_might_not_report_percent-progress_when_it_has_actual_key_file.mdwn b/doc/bugs/annex_copy_might_not_report_percent-progress_when_it_has_actual_key_file.mdwn new file mode 100644 index 0000000000..b4c8177098 --- /dev/null +++ b/doc/bugs/annex_copy_might_not_report_percent-progress_when_it_has_actual_key_file.mdwn @@ -0,0 +1,24 @@ +### Please describe the problem. + +git-annex does not report percent-progress in the json-progress output if size is not listed in the key. But the file is available so git annex could easily report the %s. We will need to workaround in datalad atm where we assumed that percents are always reported + +[[!format sh """ +$> git annex copy --to=localhost --json --json-progress Why_is_git_annex_awesome__This_is_why_.webm +{"byte-progress":32768,"action":{"command":"copy","note":"to localhost...","key":"URL--quvi:https://www.youtube.com/watch,63v,614qCZFW_uGU0","file":"Why_is_git_annex_awesome__This_is_why_.webm"}} +{"command":"copy","note":"to localhost...","success":true,"key":"URL--quvi:https://www.youtube.com/watch,63v,614qCZFW_uGU0","file":"Why_is_git_annex_awesome__This_is_why_.webm"} + +$> du -scmL Why_is_git_annex_awesome__This_is_why_.webm +6 Why_is_git_annex_awesome__This_is_why_.webm +6 total + +$> git annex version +git-annex version: 6.20171018+gitgbb20b1ed3-1~ndall+1 +... + +$> ls -l Why_is_git_annex_awesome__This_is_why_.webm +lrwxrwxrwx 1 yoh yoh 150 Nov 3 09:02 Why_is_git_annex_awesome__This_is_why_.webm -> ../../.git/annex/objects/8f/XP/URL--quvi&chttps&c%%www.youtube.com%watch,63v,614qCZFW_uGU0/URL--quvi&chttps&c%%www.youtube.com%watch,63v,614qCZFW_uGU0 + +"""]] + +> [[done]], but see my caveat about needing to handle lack of progress +> output anyway. --[[Joey]] diff --git a/doc/bugs/annex_copy_might_not_report_percent-progress_when_it_has_actual_key_file/comment_1_8e8dd79385b523502b39247135385bc4._comment b/doc/bugs/annex_copy_might_not_report_percent-progress_when_it_has_actual_key_file/comment_1_8e8dd79385b523502b39247135385bc4._comment new file mode 100644 index 0000000000..631155c20a --- /dev/null +++ b/doc/bugs/annex_copy_might_not_report_percent-progress_when_it_has_actual_key_file/comment_1_8e8dd79385b523502b39247135385bc4._comment @@ -0,0 +1,13 @@ +[[!comment format=mdwn + username="joey" + subject="""comment 1""" + date="2017-11-14T19:19:30Z" + content=""" +It indeed would be possible for `copy --to` to check the actual file +size when the key does not have a known size, and use that for progress. +I don't know how hard it would be. + +Note that, even if that were done, there's no guarantee that a given remote +will update progress information, and if it doesn't, --json-progress +won't result in any. So your code certianly needs to handle that case. +"""]] diff --git a/doc/bugs/annex_copy_might_not_report_percent-progress_when_it_has_actual_key_file/comment_2_85c2ca68a4611e684c147fc005470fab._comment b/doc/bugs/annex_copy_might_not_report_percent-progress_when_it_has_actual_key_file/comment_2_85c2ca68a4611e684c147fc005470fab._comment new file mode 100644 index 0000000000..29c1248453 --- /dev/null +++ b/doc/bugs/annex_copy_might_not_report_percent-progress_when_it_has_actual_key_file/comment_2_85c2ca68a4611e684c147fc005470fab._comment @@ -0,0 +1,20 @@ +[[!comment format=mdwn + username="joey" + subject="""comment 2""" + date="2017-11-14T19:29:21Z" + content=""" +Messages.Progress.metered is what looks at the keySize, and when it's +not known, displays no meter. So it would need an additional Maybe FilePath +that's the file being uploaded, to look at when the keySize is not known. + +That does not seem too hard a change to make; I'm not convinced the extra +complexity is worth it, since this would only add progress for uploads, +and not for downloads. + +There is something to be said for consistency; +and if some transfers of a key have progress and others do not, +it seems the user might get confused, while if nothing does, the user +can conclude that git-annex is not able to provide progress for a key +that does not contain a size, and if they don't like that, avoid +things that generate such keys. +"""]] diff --git a/doc/bugs/annex_copy_might_not_report_percent-progress_when_it_has_actual_key_file/comment_3_5866f9e2f21151f36d4fcf1b7d0ea83e._comment b/doc/bugs/annex_copy_might_not_report_percent-progress_when_it_has_actual_key_file/comment_3_5866f9e2f21151f36d4fcf1b7d0ea83e._comment new file mode 100644 index 0000000000..1034a63580 --- /dev/null +++ b/doc/bugs/annex_copy_might_not_report_percent-progress_when_it_has_actual_key_file/comment_3_5866f9e2f21151f36d4fcf1b7d0ea83e._comment @@ -0,0 +1,9 @@ +[[!comment format=mdwn + username="joey" + subject="""comment 3""" + date="2017-11-14T20:17:10Z" + content=""" +I suppose that, since some remotes don't have progress display implemented, +in paricular some external special remotes, there's no point in worrying +about interface consistency. So, may as well display progress when we can. +"""]] diff --git a/doc/bugs/annex_copy_might_not_report_percent-progress_when_it_has_actual_key_file/comment_4_bc55ede32beee2a74b943132e6377005._comment b/doc/bugs/annex_copy_might_not_report_percent-progress_when_it_has_actual_key_file/comment_4_bc55ede32beee2a74b943132e6377005._comment new file mode 100644 index 0000000000..0f36e43963 --- /dev/null +++ b/doc/bugs/annex_copy_might_not_report_percent-progress_when_it_has_actual_key_file/comment_4_bc55ede32beee2a74b943132e6377005._comment @@ -0,0 +1,29 @@ +[[!comment format=mdwn + username="yarikoptic" + avatar="http://cdn.libravatar.org/avatar/f11e9c84cb18d26a1748c33b48c924b4" + subject="thanks, what about 'find' with the same issue?" + date="2017-11-16T19:11:51Z" + content=""" +Shoudn't the same apply to \"find\" output? + +[[!format sh \"\"\" +$> git annex find --not --in localhost --json Brainhack2012__Share_your_tools__But_fear_the_wombat_.mp4 +{\"bytesize\":\"unknown\",\"mtime\":\"unknown\",\"keyname\":\"quvi:https://www.youtube.com/watch,63v,618t6znEOEDVo\",\"backend\":\"URL\",\"key\":\"URL--quvi:https://www.youtube.com/watch,63v,618t6znEOEDVo\",\"humansize\":\"unknown\",\"hashdirmixed\":\"kq/PM/\",\"file\":\"Brainhack2012__Share_your_tools__But_fear_the_wombat_.mp4\",\"hashdirlower\":\"8ba/bf9/\"} + +$> sudo dpkg -i /tmp/git-annex-standalone_6.20171114+gitg5e6c3ba30-1\~ndall+1_amd64.deb +[sudo] password for yoh: +(Reading database ... 706557 files and directories currently installed.) +Preparing to unpack .../git-annex-standalone_6.20171114+gitg5e6c3ba30-1~ndall+1_amd64.deb ... +Unpacking git-annex-standalone (6.20171114+gitg5e6c3ba30-1~ndall+1) over (6.20171018+gitgbb20b1ed3-1~ndall+1) ... +Setting up git-annex-standalone (6.20171114+gitg5e6c3ba30-1~ndall+1) ... +... + +$> git annex find --not --in localhost --json Brainhack2012__Share_your_tools__But_fear_the_wombat_.mp4 +{\"bytesize\":\"unknown\",\"mtime\":\"unknown\",\"keyname\":\"quvi:https://www.youtube.com/watch,63v,618t6znEOEDVo\",\"backend\":\"URL\",\"key\":\"URL--quvi:https://www.youtube.com/watch,63v,618t6znEOEDVo\",\"humansize\":\"unknown\",\"hashdirmixed\":\"kq/PM/\",\"file\":\"Brainhack2012__Share_your_tools__But_fear_the_wombat_.mp4\",\"hashdirlower\":\"8ba/bf9/\"} + +$> ls -lL Brainhack2012__Share_your_tools__But_fear_the_wombat_.mp4 +-r-------- 1 yoh yoh 64354577 Mar 22 2014 Brainhack2012__Share_your_tools__But_fear_the_wombat_.mp4 + + +\"\"\"]] +"""]] diff --git a/doc/bugs/annex_copy_might_not_report_percent-progress_when_it_has_actual_key_file/comment_5_8c33fc01dd1de2abcd3782ff89b437de._comment b/doc/bugs/annex_copy_might_not_report_percent-progress_when_it_has_actual_key_file/comment_5_8c33fc01dd1de2abcd3782ff89b437de._comment new file mode 100644 index 0000000000..5645cb6755 --- /dev/null +++ b/doc/bugs/annex_copy_might_not_report_percent-progress_when_it_has_actual_key_file/comment_5_8c33fc01dd1de2abcd3782ff89b437de._comment @@ -0,0 +1,17 @@ +[[!comment format=mdwn + username="joey" + subject="""comment 5""" + date="2017-11-20T18:18:07Z" + content=""" +I don't understand what you mean with the find command. Are you talking +about the "unknown" values in the json? + +Oh, I suppose you mean particularly that the bytesize is unknown. + +Well, this would make `find` output differ depending on whether the key is +present or not, in cases where it would otherwise be the same. And it would +change machine-consumable output in a way that for all I know would break +stuff. + +So, changing that seems like a bad idea. +"""]] diff --git a/doc/bugs/annex_copy_might_not_report_percent-progress_when_it_has_actual_key_file/comment_6_4d5764187b03962cf7910ef805c57f06._comment b/doc/bugs/annex_copy_might_not_report_percent-progress_when_it_has_actual_key_file/comment_6_4d5764187b03962cf7910ef805c57f06._comment new file mode 100644 index 0000000000..10f8e3444b --- /dev/null +++ b/doc/bugs/annex_copy_might_not_report_percent-progress_when_it_has_actual_key_file/comment_6_4d5764187b03962cf7910ef805c57f06._comment @@ -0,0 +1,13 @@ +[[!comment format=mdwn + username="joey" + subject="""comment 6""" + date="2017-12-05T16:26:09Z" + content=""" +I'd be more comfortable with adding a new "disksize" value +with the current space taken up by the key. + +However, adding that to find --json would mean that find would have to stat +the file to get its size, which would slow it down. Find often does already +stat the file via its inAnnex check, but eg `git annex find --in remote` does +not, and slowing that down further seems inadvisable. +"""]] diff --git a/doc/bugs/annex_doesn__39__t_fixup_symlinks_when___34__git_commit_path__95__to__95__repo__34___is_used.mdwn b/doc/bugs/annex_doesn__39__t_fixup_symlinks_when___34__git_commit_path__95__to__95__repo__34___is_used.mdwn new file mode 100644 index 0000000000..b72788f8ff --- /dev/null +++ b/doc/bugs/annex_doesn__39__t_fixup_symlinks_when___34__git_commit_path__95__to__95__repo__34___is_used.mdwn @@ -0,0 +1,79 @@ +### Please describe the problem. + +whenever we 'git mv' some files from one dir to another, "git commit path_to_that_repo" seems to cause git annex to not fix up the symlinks + +### What steps will reproduce the problem? + +see below complete examples + +### What version of git-annex are you using? On what operating system? + +6.20170307+gitg24ade8a25-1~ndall+1 + +### Please provide any additional information below. + +correct operation: + +[[!format sh """ + +hopa:/tmp/testl +$> builtin cd /tmp/; chmod +w -R /tmp/testl; rm -rf /tmp/testl; mkdir /tmp/testl && cd /tmp/testl && git init && git annex init && mkdir d && echo 123 > d/123 && git annex add d/123 && git commit -m added && git mv d/123 123 && git annex add . && echo -e '#!/bin/sh\ngit annex pre-commit --debug . >/tmp/precommit.log 2>&1\n' >| .git/hooks/pre-commit && git commit -m msg && ls -l 123 && git status && cat /tmp/precommit.log +Initialized empty Git repository in /tmp/testl/.git/ +init ok +(recording state in git...) +add d/123 ok +(recording state in git...) +[master (root-commit) a04d6e9] added + 1 file changed, 1 insertion(+) + create mode 120000 d/123 +[master 1be2c88] msg + 2 files changed, 1 insertion(+), 1 deletion(-) + create mode 120000 123 + delete mode 120000 d/123 +lrwxrwxrwx 1 yoh yoh 178 Mar 22 14:52 123 -> .git/annex/objects/G6/qW/SHA256E-s4--181210f8f9c779c26da1d9b2075bde0127302ee0e3fca38c9a83f5b1dd8e5d3b/SHA256E-s4--181210f8f9c779c26da1d9b2075bde0127302ee0e3fca38c9a83f5b1dd8e5d3b +On branch master +nothing to commit, working tree clean +[2017-03-22 14:52:45.459507064] read: git ["--git-dir=.git","--work-tree=.","--literal-pathspecs","diff","--cached","--name-only","-z","--diff-filter=ACMRT","--","."] +fix 123 ok +[2017-03-22 14:52:45.46373033] read: git ["--git-dir=.git","--work-tree=.","--literal-pathspecs","diff","--name-only","--diff-filter=T","-z","--cached","--","."] +[2017-03-22 14:52:45.46776695] read: git ["--git-dir=.git","--work-tree=.","--literal-pathspecs","symbolic-ref","-q","HEAD"] +[2017-03-22 14:52:45.471690183] process done ExitSuccess +[2017-03-22 14:52:45.471781853] read: git ["--git-dir=.git","--work-tree=.","--literal-pathspecs","show-ref","refs/heads/master"] +[2017-03-22 14:52:45.475456411] process done ExitSuccess +(recording state in git...) +[2017-03-22 14:52:45.475535994] feed: xargs ["-0","git","--git-dir=.git","--work-tree=.","--literal-pathspecs","add","--force","--"] +[2017-03-22 14:52:45.482800008] process done ExitSuccess +"""]] + +and this time commit with the path: (git commit -m msg /tmp/testl) + +[[!format sh """ +$> builtin cd /tmp/; chmod +w -R /tmp/testl; rm -rf /tmp/testl; mkdir /tmp/testl && cd /tmp/testl && git init && git annex init && mkdir d && echo 123 > d/123 && git annex add d/123 && git commit -m added && git mv d/123 123 && git annex add . && echo -e '#!/bin/sh\ngit annex pre-commit --debug . >/tmp/precommit.log 2>&1\n' >| .git/hooks/pre-commit && git commit -m msg /tmp/testl && ls -l 123 && git status && cat /tmp/precommit.log +Initialized empty Git repository in /tmp/testl/.git/ +init ok +(recording state in git...) +add d/123 ok +(recording state in git...) +[master (root-commit) 1438090] added + 1 file changed, 1 insertion(+) + create mode 120000 d/123 +[master 8912aa3] msg + 1 file changed, 0 insertions(+), 0 deletions(-) + rename d/123 => 123 (100%) +lrwxrwxrwx 1 yoh yoh 181 Mar 22 14:52 123 -> ../.git/annex/objects/G6/qW/SHA256E-s4--181210f8f9c779c26da1d9b2075bde0127302ee0e3fca38c9a83f5b1dd8e5d3b/SHA256E-s4--181210f8f9c779c26da1d9b2075bde0127302ee0e3fca38c9a83f5b1dd8e5d3b +On branch master +nothing to commit, working tree clean +[2017-03-22 14:52:55.857221368] read: git ["--git-dir=.git","--work-tree=.","--literal-pathspecs","diff","--name-only","--diff-filter=T","-z","--cached","--","."] +[2017-03-22 14:52:55.861708907] process done ExitSuccess +[2017-03-22 14:52:55.861809182] read: git ["--git-dir=.git","--work-tree=.","--literal-pathspecs","symbolic-ref","-q","HEAD"] +[2017-03-22 14:52:55.865393029] process done ExitSuccess +[2017-03-22 14:52:55.865471585] read: git ["--git-dir=.git","--work-tree=.","--literal-pathspecs","show-ref","refs/heads/master"] +[2017-03-22 14:52:55.869145816] process done ExitSuccess + + +"""]] + + +note that there is no "(recording state in git...) ..." portion in the output! + +[[!meta author=yoh]] diff --git a/doc/bugs/annex_doesn__39__t_fixup_symlinks_when___34__git_commit_path__95__to__95__repo__34___is_used/comment_1_b90f22ce0ab931658856e949ac227985._comment b/doc/bugs/annex_doesn__39__t_fixup_symlinks_when___34__git_commit_path__95__to__95__repo__34___is_used/comment_1_b90f22ce0ab931658856e949ac227985._comment new file mode 100644 index 0000000000..e5a1ac2784 --- /dev/null +++ b/doc/bugs/annex_doesn__39__t_fixup_symlinks_when___34__git_commit_path__95__to__95__repo__34___is_used/comment_1_b90f22ce0ab931658856e949ac227985._comment @@ -0,0 +1,23 @@ +[[!comment format=mdwn + username="joey" + subject="""comment 1""" + date="2017-04-07T20:16:45Z" + content=""" +There's nothing special about using the absolute path; "git commit ." +or "git commit thefile" will behave the same. + +This is the git false index problem. Since git commit in these situations +runs the pre-commit hook with a false index file, changes made to that +index file won't be visible after the commit. + +So, if `git annex pre-commit` fixes symlinks in this situation, +the right thing will be committed, but then the old index will have the old +symlinks staged, which will result in `git status` after the commit showing +modification to the files you just staged and committed! + +Short of having a post-commit hook come along and fix up the index to match +what was committed, I don't see anything git-annex can do better. Well, it +could prevent such commits even being made, I suppose, or warn. + +It's a pity git uses this false index file in this situation. +"""]] diff --git a/doc/bugs/annex_drop_fails_to_determine_availability_on_a_http_url_redirecting_to_ftp.mdwn b/doc/bugs/annex_drop_fails_to_determine_availability_on_a_http_url_redirecting_to_ftp.mdwn new file mode 100644 index 0000000000..9ecab381f1 --- /dev/null +++ b/doc/bugs/annex_drop_fails_to_determine_availability_on_a_http_url_redirecting_to_ftp.mdwn @@ -0,0 +1,131 @@ +### Please describe the problem. + +Need to annex some files from a http link which forwards to ftp. addurl works out fine (besides not carrying about redirected filename), but drop fails to verify presence of the file + + +### What version of git-annex are you using? On what operating system? + +6.20160706+gitgc4229be-1~ndall+1 + +### Please provide any additional information below. + +[[!format sh """ +smaug:/tmp +$> mkdir /tmp/123; cd /tmp/123; git init; git annex init; +Initialized empty Git repository in /tmp/123/.git/ +init ok +(recording state in git...) +1 10080.....................................:Tue 12 Jul 2016 02:18:46 PM EDT:. +(git)smaug:/tmp/123[master] +$> git annex addurl --debug --pathdepth=-1 http://www.nitrc.org/frs/downloadlink.php/1637 +[2016-07-12 14:18:54.8195] read: git ["--git-dir=.git","--work-tree=.","--literal-pathspecs","show-ref","git-annex"] +[2016-07-12 14:18:54.83021] process done ExitSuccess +[2016-07-12 14:18:54.830357] read: git ["--git-dir=.git","--work-tree=.","--literal-pathspecs","show-ref","--hash","refs/heads/git-annex"] +[2016-07-12 14:18:54.841375] process done ExitSuccess +[2016-07-12 14:18:54.841694] read: git ["--git-dir=.git","--work-tree=.","--literal-pathspecs","log","refs/heads/git-annex..787586641a9027e772b91989eb649cb9303cd975","--pretty=%H","-n1"] +[2016-07-12 14:18:54.851263] process done ExitSuccess +[2016-07-12 14:18:54.852043] chat: git ["--git-dir=.git","--work-tree=.","--literal-pathspecs","cat-file","--batch"] +[2016-07-12 14:18:54.862122] read: quvi ["--version"] +[2016-07-12 14:18:54.869939] process done ExitSuccess +[2016-07-12 14:18:54.870062] call: quvi ["--verbosity","mute","--support","http://www.nitrc.org/frs/downloadlink.php/1637"] +[2016-07-12 14:18:54.891339] process done ExitFailure 65 +addurl 1637 (downloading http://www.nitrc.org/frs/downloadlink.php/1637 ...) +[2016-07-12 14:18:55.830611] call: wget ["-q","--show-progress","--clobber","-c","-O","/tmp/123/.git/annex/tmp/URL--http&c%%www.nitrc.org%frs%downloadlink.php%1637","http://www.nitrc.org/frs/downloadlink.php/1637","--user-agent","git-annex/6.20160706+gitgc4229be-1~ndall+1"] +1637 100%[===========================================================================================>] 268.32M 12.9MB/s in 27s +[2016-07-12 14:19:31.32811] process done ExitSuccess +[2016-07-12 14:19:31.328909] chat: git ["--git-dir=.git","--work-tree=.","--literal-pathspecs","check-attr","-z","--stdin","annex.backend","annex.numcopies","annex.largefiles","--"] +[2016-07-12 14:19:31.329444] read: git ["--version"] +[2016-07-12 14:19:31.338195] process done ExitSuccess +[2016-07-12 14:19:31.338947] read: sha256sum [".git/annex/tmp/URL--http&c%%www.nitrc.org%frs%downloadlink.php%1637"] +[2016-07-12 14:19:32.860187] process done ExitSuccess +ok +(recording state in git...) +[2016-07-12 14:19:32.872749] feed: xargs ["-0","git","--git-dir=.git","--work-tree=.","--literal-pathspecs","add","--"] +[2016-07-12 14:19:32.890096] process done ExitSuccess +[2016-07-12 14:19:32.890573] chat: git ["--git-dir=.git","--work-tree=.","--literal-pathspecs","hash-object","-w","--stdin-paths","--no-filters"] +[2016-07-12 14:19:32.891376] feed: git ["--git-dir=.git","--work-tree=.","--literal-pathspecs","update-index","-z","--index-info"] +[2016-07-12 14:19:32.901206] process done ExitSuccess +[2016-07-12 14:19:32.901288] read: git ["--git-dir=.git","--work-tree=.","--literal-pathspecs","show-ref","--hash","refs/heads/git-annex"] +[2016-07-12 14:19:32.909396] process done ExitSuccess +[2016-07-12 14:19:32.909691] read: git ["--git-dir=.git","--work-tree=.","--literal-pathspecs","write-tree"] +[2016-07-12 14:19:32.91827] process done ExitSuccess +[2016-07-12 14:19:32.918324] chat: git ["--git-dir=.git","--work-tree=.","--literal-pathspecs","commit-tree","6c1ce1871151c12f6930733e7693eac2fd862677","--no-gpg-sign","-p","refs/heads/git-annex"] +[2016-07-12 14:19:32.927217] process done ExitSuccess +[2016-07-12 14:19:32.927281] call: git ["--git-dir=.git","--work-tree=.","--literal-pathspecs","update-ref","refs/heads/git-annex","2453a1c23977e4f17ac5fd9668cdc7bdb93e34e8"] +[2016-07-12 14:19:32.936074] process done ExitSuccess +cached/staged changes: + 1637 | 1 + +1 10081.....................................:Tue 12 Jul 2016 02:19:32 PM EDT:. +(git)smaug:/tmp/123[master] +$> git commit -m 'added' +[master (root-commit) b212851] added + 1 file changed, 1 insertion(+) + create mode 120000 1637 +1 10082.....................................:Tue 12 Jul 2016 02:19:38 PM EDT:. +(git)smaug:/tmp/123[master]git +$> git annex drop 1637 +drop 1637 (checking http://www.nitrc.org/frs/downloadlink.php/1637...) (unsafe) + Could only verify the existence of 0 out of 1 necessary copies + + Rather than dropping this file, try using: git annex move + + (Use --force to override this check, or adjust numcopies.) +failed +git-annex: drop: 1 failed +1 10083 ->1.....................................:Tue 12 Jul 2016 02:19:48 PM EDT:. +(git)smaug:/tmp/123[master]git +$> wget -S http://www.nitrc.org/frs/downloadlink.php/1637 +--2016-07-12 14:19:54-- http://www.nitrc.org/frs/downloadlink.php/1637 +Resolving www.nitrc.org (www.nitrc.org)... 132.239.16.23 +Connecting to www.nitrc.org (www.nitrc.org)|132.239.16.23|:80... connected. +HTTP request sent, awaiting response... + HTTP/1.1 302 Found + Date: Tue, 12 Jul 2016 18:19:54 GMT + Server: Apache/2.2.15 (CentOS) + X-Powered-By: PHP/5.3.3 + Set-Cookie: PHPSESSID=vhcpo1fmi205cfv0h4jgbnn9a0; path=/ + Expires: Thu, 19 Nov 1981 08:52:00 GMT + Cache-Control: no-store, no-cache, must-revalidate, post-check=0, pre-check=0 + Pragma: no-cache + Location: ftp://www.nitrc.org/fcon_1000/htdocs/Ontario.tar + Content-Length: 0 + Keep-Alive: timeout=2, max=100 + Connection: Keep-Alive + Content-Type: text/html; charset=UTF-8 +Location: ftp://www.nitrc.org/fcon_1000/htdocs/Ontario.tar [following] +--2016-07-12 14:19:55-- ftp://www.nitrc.org/fcon_1000/htdocs/Ontario.tar + => ‘Ontario.tar’ +Connecting to www.nitrc.org (www.nitrc.org)|132.239.16.23|:21... connected. +Logging in as anonymous ... +220 (vsFTPd 2.2.2) +--> USER anonymous + +331 Please specify the password. +--> PASS Turtle Power! + +230 Login successful. +--> SYST + +215 UNIX Type: L8 +--> PWD + +257 "/" +--> TYPE I + +200 Switching to Binary mode. +--> CWD /fcon_1000/htdocs + +250 Directory successfully changed. +--> SIZE Ontario.tar + +213 281354240 +--> PASV + +227 Entering Passive Mode (132,239,16,23,106,251). +--> RETR Ontario.tar + +"""]] + +[[!meta author=yoh]] + +> [[fixed|done]] --[[Joey]] diff --git a/doc/bugs/annex_drop_is_not___34__in_effect__34___for_load_which_was___34__addurl_--batch__34__ed_but_not_yet_committed.mdwn b/doc/bugs/annex_drop_is_not___34__in_effect__34___for_load_which_was___34__addurl_--batch__34__ed_but_not_yet_committed.mdwn new file mode 100644 index 0000000000..2ce7c18996 --- /dev/null +++ b/doc/bugs/annex_drop_is_not___34__in_effect__34___for_load_which_was___34__addurl_--batch__34__ed_but_not_yet_committed.mdwn @@ -0,0 +1,66 @@ +### Please describe the problem. + +Usecase (you might suggest a better way) is to 'annex' the content of files (via addurl) without actually committing symlinks to git (git-annex branch obviously should progress forward). So the sequence is akin 'annex addurl', 'annex drop' both ran with annex.alwayscommit=false . The issue is that it seems that if addurl is --batched, then symlink is not 'git add'ed until batched process finishes, and thus subsequent to 'addurl' 'drop' does nothing, leaving key behind not dropped. + +Related recent TODO wishlist is [[http://git-annex.branchable.com/todo/drop_--batch/]] + +### What steps will reproduce the problem? + +FWIW below is a somewhat noisy output from datalad's unittest (which addurl into the archive's content, so custom special remote is involved). At the end of the run content is not dropped, symlink is not dangling. + +### What version of git-annex are you using? On what operating system? + +originally was a version from March but tried also with fresh 6.20160425+gitgffe2ea2-1~ndall+1 + +### Please provide any additional information below. + +[[!format sh """ + +2016-04-26 14:13:05,545 [DEBUG ] Initiating a new process for BatchedAnnex(annex_cmd='addurl', annex_options=['--with-files', '--json'], git_options=[], output_proc=, path=<<'/home/yoh/.tmp/datala...>>) (annexrepo.py:1145) +2016-04-26 14:13:05,545 [Level 5] Command: ['git', '-c', 'receive.autogc=0', '-c', 'gc.auto=0', 'annex', 'addurl', '--with-files', '--json', '--batch'] (annexrepo.py:1148) +2016-04-26 14:13:05,548 [Level 5] Sending 'dl+archive:SHA256E-s10240--0d52ce580b5ea9324dfc2ac7c714130f02499439e6b990282adb6580a97fe1d0.tar/1/file.txt#size=4 .dataladMCsP4y/file.txt\n' to batched annex BatchedAnnex(annex_cmd='addurl', annex_options=['--with-files', '--json'], git_options=[], output_proc=, path=<<'/home/yoh/.tmp/datala...>>) (annexrepo.py:1202) +2016-04-26 14:13:05,548 [Level 5] Done sending. (annexrepo.py:1210) +2016-04-26 14:13:05,869 [DEBUG] Importing the rest of datalad.__init__ (__init__.py:15) +2016-04-26 14:13:05,870 [DEBUG] Reading files: ['/etc/datalad/datalad.cfg', '/etc/xdg/datalad/config', '/home/yoh/.config/datalad/datalad.cfg', '.datalad/config'] (configparserinc.py:144) +2016-04-26 14:13:05,959 [DEBUG] UI set to ConsoleLog(out=) (__init__.py:48) +2016-04-26 14:13:05,960 [DEBUG] UI set to UnderAnnexUI(out=) (__init__.py:48) +2016-04-26 14:13:05,962 [DEBUG] Not initiating existing cache for the archives under /tmp/datalad_temp_tree_setupmRy8cc/.git/datalad/tmp/archives (archives.py:262) +2016-04-26 14:13:05,962 [Level 4] Sending 'VERSION 1' (base.py:312) +2016-04-26 14:13:05,962 [Level 4] Received ['PREPARE'] (base.py:312) +2016-04-26 14:13:05,962 [Level 4] Sending 'PREPARE-SUCCESS' (base.py:312) +2016-04-26 14:13:05,962 [Level 4] Received ['CLAIMURL', 'dl+archive:SHA256E-s10240--0d52ce580b5ea9324dfc2ac7c714130f02499439e6b990282adb6580a97fe1d0.tar/1/file.txt#size=4'] (base.py:312) +2016-04-26 14:13:05,963 [DEBUG] Claiming url 'dl+archive:SHA256E-s10240--0d52ce580b5ea9324dfc2ac7c714130f02499439e6b990282adb6580a97fe1d0.tar/1/file.txt#size=4' (base.py:316) +2016-04-26 14:13:05,963 [Level 4] Sending "DEBUG Claiming url 'dl+archive:SHA256E-s10240--0d52ce580b5ea9324dfc2ac7c714130f02499439e6b990282adb6580a97fe1d0.tar/1/file.txt#size=4'" (base.py:312) +2016-04-26 14:13:05,963 [Level 4] Sending 'CLAIMURL-SUCCESS' (base.py:312) +2016-04-26 14:13:05,963 [Level 4] Received ['CHECKURL', 'dl+archive:SHA256E-s10240--0d52ce580b5ea9324dfc2ac7c714130f02499439e6b990282adb6580a97fe1d0.tar/1/file.txt#size=4'] (base.py:312) +2016-04-26 14:13:05,964 [DEBUG] Current directory: /tmp/datalad_temp_tree_setupmRy8cc, url: dl+archive:SHA256E-s10240--0d52ce580b5ea9324dfc2ac7c714130f02499439e6b990282adb6580a97fe1d0.tar/1/file.txt#size=4 (archives.py:163) +2016-04-26 14:13:05,964 [DEBUG] Initiating a new process for BatchedAnnex(annex_cmd='contentlocation', annex_options=[], git_options=[], output_proc=, path=<<'/tmp/datalad_temp_tre...>>) (annexrepo.py:1145) +2016-04-26 14:13:05,964 [Level 5] Command: ['git', '-c', 'receive.autogc=0', '-c', 'gc.auto=0', 'annex', 'contentlocation', '--batch'] (annexrepo.py:1148) +2016-04-26 14:13:05,967 [Level 5] Sending 'SHA256E-s10240--0d52ce580b5ea9324dfc2ac7c714130f02499439e6b990282adb6580a97fe1d0.tar\n' to batched annex BatchedAnnex(annex_cmd='contentlocation', annex_options=[], git_options=[], output_proc=, path=<<'/tmp/datalad_temp_tre...>>) (annexrepo.py:1202) +2016-04-26 14:13:05,967 [Level 5] Done sending. (annexrepo.py:1210) +2016-04-26 14:13:06,007 [Level 5] Received output: '.git/annex/objects/7p/mj/SHA256E-s10240--0d52ce580b5ea9324dfc2ac7c714130f02499439e6b990282adb6580a97fe1d0.tar/SHA256E-s10240--0d52ce580b5ea9324dfc2ac7c714130f02499439e6b990282adb6580a97fe1d0.tar' (annexrepo.py:1220) +2016-04-26 14:13:06,008 [Level 4] Sending 'CHECKURL-CONTENTS 4' (base.py:312) +2016-04-26 14:13:06,009 [Level 4] Received ['TRANSFER', 'RETRIEVE', 'URL-s4--dl,43archive:SHA256E-s10240--0d-d5401228f11172416c78bab2f3f31f61', '.git/annex/tmp/URL-s4--dl,43archive&cSHA256E-s10240--0d-d5401228f11172416c78bab2f3f31f61'] (base.py:312) +2016-04-26 14:13:06,009 [INFO] RETRIEVE key URL-s4--dl,43archive:SHA256E-s10240--0d-d5401228f11172416c78bab2f3f31f61 into/from .git/annex/tmp/URL-s4--dl,43archive&cSHA256E-s10240--0d-d5401228f11172416c78bab2f3f31f61 (base.py:429) +2016-04-26 14:13:06,009 [Level 4] Sending 'GETURLS URL-s4--dl,43archive:SHA256E-s10240--0d-d5401228f11172416c78bab2f3f31f61 dl+archive:' (base.py:312) +2016-04-26 14:13:06,011 [Level 4] Received ['VALUE', 'dl+archive:SHA256E-s10240--0d52ce580b5ea9324dfc2ac7c714130f02499439e6b990282adb6580a97fe1d0.tar/1/file.txt#size=4'] (base.py:312) +2016-04-26 14:13:06,011 [Level 4] Received ['VALUE'] (base.py:312) +2016-04-26 14:13:06,011 [Level 4] Received URLS: ['dl+archive:SHA256E-s10240--0d52ce580b5ea9324dfc2ac7c714130f02499439e6b990282adb6580a97fe1d0.tar/1/file.txt#size=4'] (base.py:312) +2016-04-26 14:13:06,011 [DEBUG] Getting file 1/file.txt from /tmp/datalad_temp_tree_setupmRy8cc/.git/annex/objects/7p/mj/SHA256E-s10240--0d52ce580b5ea9324dfc2ac7c714130f02499439e6b990282adb6580a97fe1d0.tar/SHA256E-s10240--0d52ce580b5ea9324dfc2ac7c714130f02499439e6b990282adb6580a97fe1d0.tar while PWD=/tmp/datalad_temp_tree_setupmRy8cc (archives.py:299) +2016-04-26 14:13:06,011 [DEBUG] Requested file 1/file.txt from archive /tmp/datalad_temp_tree_setupmRy8cc/.git/annex/objects/7p/mj/SHA256E-s10240--0d52ce580b5ea9324dfc2ac7c714130f02499439e6b990282adb6580a97fe1d0.tar/SHA256E-s10240--0d52ce580b5ea9324dfc2ac7c714130f02499439e6b990282adb6580a97fe1d0.tar (archives.py:454) +2016-04-26 14:13:06,012 [Level 2] Verifying that /tmp/datalad_temp_tree_setupmRy8cc/.git/datalad/tmp/archives/60e44d2ccc/1/file.txt exists (archives.py:462) +2016-04-26 14:13:06,012 [DEBUG] Hardlinking /tmp/datalad_temp_tree_setupmRy8cc/.git/datalad/tmp/archives/60e44d2ccc/1/file.txt under .git/annex/tmp/URL-s4--dl,43archive&cSHA256E-s10240--0d-d5401228f11172416c78bab2f3f31f61 (cmd.py:372) +2016-04-26 14:13:06,012 [Level 2] Hardlinking finished (cmd.py:382) +2016-04-26 14:13:06,012 [Level 4] Sending 'TRANSFER-SUCCESS RETRIEVE URL-s4--dl,43archive:SHA256E-s10240--0d-d5401228f11172416c78bab2f3f31f61' (base.py:312) +2016-04-26 14:13:06,022 [Level 5] Received output: {u'note': u'from datalad-archives', u'success': True, u'command': u'addurl', u'key': u'SHA256E-s4--0cf67fc72b3c86c7a454f6d86b43ed245a8e491d0e5288d4da8c7ff43a7bcdb0.txt', u'file': u'.dataladMCsP4y/file.txt'} (annexrepo.py:1220) +2016-04-26 14:13:06,023 [DEBUG ] Running: ['git', '-c', 'receive.autogc=0', '-c', 'gc.auto=0', '-c', 'annex.alwayscommit=false', 'annex', 'drop', '--debug', '.dataladMCsP4y/file.txt'] (cmd.py:351) +2016-04-26 14:13:06,075 [ERROR ] stderr| [2016-04-26 14:13:06.069266] read: git ["--git-dir=.git","--work-tree=.","--literal-pathspecs","ls-files","--cached","-z","--",".dataladMCsP4y/file.txt"] (cmd.py:351) +2016-04-26 14:13:06,075 [Level 8] Finished running ['git', '-c', 'receive.autogc=0', '-c', 'gc.auto=0', '-c', 'annex.alwayscommit=false', 'annex', 'drop', '--debug', '.dataladMCsP4y/file.txt'] with status 0 (cmd.py:351) +t sh + +"""]] + + +[[!meta author=yoh]] + +> [[done]] per comments --[[Joey]] diff --git a/doc/bugs/annex_get_-J_16_via_ssh_stalls_.mdwn b/doc/bugs/annex_get_-J_16_via_ssh_stalls_.mdwn new file mode 100644 index 0000000000..0edd892980 --- /dev/null +++ b/doc/bugs/annex_get_-J_16_via_ssh_stalls_.mdwn @@ -0,0 +1,96 @@ + +[[!format sh """ +(git)smaug:/mnt/btrfs/datasets/datalad/crawl/dbic/QA[master]git +$> git annex get -J 16 --debug --json-progress . +[2018-09-25 09:40:26.76476951] read: git ["--git-dir=.git","--work-tree=.","--literal-pathspecs","ls-files","--cached","-z","--","."] +[2018-09-25 09:40:26.769184442] read: git ["--git-dir=.git","--work-tree=.","--literal-pathspecs","show-ref","git-annex"] +[2018-09-25 09:40:26.771844846] process done ExitSuccess +[2018-09-25 09:40:26.771993108] read: git ["--git-dir=.git","--work-tree=.","--literal-pathspecs","show-ref","--hash","refs/heads/git-annex"] +[2018-09-25 09:40:26.775598151] process done ExitSuccess +[2018-09-25 09:40:26.776318887] read: git ["--git-dir=.git","--work-tree=.","--literal-pathspecs","log","refs/heads/git-annex..db88e4b0bf3d633885a4e202e0c113942856c47f","--pretty=%H","-n1"] +[2018-09-25 09:40:26.780382394] process done ExitSuccess +[2018-09-25 09:40:26.780553211] read: git ["--git-dir=.git","--work-tree=.","--literal-pathspecs","log","refs/heads/git-annex..4f0e8cd9a43b003228c3898133b1e200f9bdfc22","--pretty=%H","-n1"] +[2018-09-25 09:40:26.785109549] process done ExitSuccess +[2018-09-25 09:40:26.785992031] chat: git ["--git-dir=.git","--work-tree=.","--literal-pathspecs","cat-file","--batch"] +[2018-09-25 09:40:26.787017719] chat: git ["--git-dir=.git","--work-tree=.","--literal-pathspecs","cat-file","--batch-check=%(objectname) %(objecttype) %(objectsize)"] +[2018-09-25 09:40:26.838015294] chat: git ["--git-dir=.git","--work-tree=.","--literal-pathspecs","cat-file","--batch"] +[2018-09-25 09:40:26.838108997] chat: git ["--git-dir=.git","--work-tree=.","--literal-pathspecs","cat-file","--batch"] +[2018-09-25 09:40:26.838146759] chat: git ["--git-dir=.git","--work-tree=.","--literal-pathspecs","cat-file","--batch"] +[2018-09-25 09:40:26.838125814] chat: git ["--git-dir=.git","--work-tree=.","--literal-pathspecs","cat-file","--batch"] +[2018-09-25 09:40:26.840522185] chat: git ["--git-dir=.git","--work-tree=.","--literal-pathspecs","cat-file","--batch"] +[2018-09-25 09:40:26.840544359] chat: git ["--git-dir=.git","--work-tree=.","--literal-pathspecs","cat-file","--batch"] +[2018-09-25 09:40:26.840612031] chat: git ["--git-dir=.git","--work-tree=.","--literal-pathspecs","cat-file","--batch-check=%(objectname) %(objecttype) %(objectsize)"] +[2018-09-25 09:40:26.843542095] chat: git ["--git-dir=.git","--work-tree=.","--literal-pathspecs","cat-file","--batch-check=%(objectname) %(objecttype) %(objectsize)"] +[2018-09-25 09:40:26.843885927] chat: git ["--git-dir=.git","--work-tree=.","--literal-pathspecs","cat-file","--batch-check=%(objectname) %(objecttype) %(objectsize)"] +[2018-09-25 09:40:26.844793008] chat: git ["--git-dir=.git","--work-tree=.","--literal-pathspecs","cat-file","--batch-check=%(objectname) %(objecttype) %(objectsize)"] +[2018-09-25 09:40:26.84691911] chat: git ["--git-dir=.git","--work-tree=.","--literal-pathspecs","cat-file","--batch-check=%(objectname) %(objecttype) %(objectsize)"] +[2018-09-25 09:40:26.848525571] chat: git ["--git-dir=.git","--work-tree=.","--literal-pathspecs","cat-file","--batch-check=%(objectname) %(objecttype) %(objectsize)"] +[2018-09-25 09:40:26.876936065] read: ssh ["-O","stop","-S","bids@rolando.cns","-o","ControlMaster=auto","-o","ControlPersist=yes","localhost"] +[2018-09-25 09:40:26.89165743] process done ExitSuccess +{"command":"get","wanted":[{"here":false,"uuid":"6384a551-a41d-4290-b186-9258befede97","description":"bids@rolando:/inbox/BIDS/dbic/QA [origin]"}],"note":"transfer already in progress, or unable to take transfer lock\nUnable to access these remotes: origin\nTry making some of these repositories available:\n\t6384a551-a41d-4290-b186-9258befede97 -- bids@rolando:/inbox/BIDS/dbic/QA [origin]\n","skipped":[],"success":false,"key":"SHA256E-s1633--1bc7a100b82cd31d084aae5b25b8d90af128324d10d92cfc9e2196d28829d375.tsv","file":".heudiconv/qa/ses-20171106/info/dicominfo_ses-20171106.tsv"} +[2018-09-25 09:40:26.89294943] read: ssh ["-o","BatchMode=true","-S",".git/annex/ssh/bids@rolando.cns","-o","ControlMaster=auto","-o","ControlPersist=yes","-n","-T","bids@rolando.cns","true"] +[2018-09-25 09:40:26.892976736] read: ssh ["-o","BatchMode=true","-S",".git/annex/ssh/bids@rolando.cns","-o","ControlMaster=auto","-o","ControlPersist=yes","-n","-T","bids@rolando.cns","true"] +[2018-09-25 09:40:26.892951965] read: ssh ["-o","BatchMode=true","-S",".git/annex/ssh/bids@rolando.cns","-o","ControlMaster=auto","-o","ControlPersist=yes","-n","-T","bids@rolando.cns","true"] +[2018-09-25 09:40:26.901388917] read: ssh ["-o","BatchMode=true","-S",".git/annex/ssh/bids@rolando.cns","-o","ControlMaster=auto","-o","ControlPersist=yes","-n","-T","bids@rolando.cns","true"] +[2018-09-25 09:40:26.901683012] read: ssh ["-o","BatchMode=true","-S",".git/annex/ssh/bids@rolando.cns","-o","ControlMaster=auto","-o","ControlPersist=yes","-n","-T","bids@rolando.cns","true"] +[2018-09-25 09:40:28.479898885] process done ExitSuccess +[2018-09-25 09:40:28.480093679] chat: ssh ["bids@rolando.cns","-S",".git/annex/ssh/bids@rolando.cns","-o","ControlMaster=auto","-o","ControlPersist=yes","-T","git-annex-shell 'p2pstdio' '/inbox/BIDS/dbic/QA' '--debug' 'ba8f2cea-f229-422c-82be-6580e5e07ed5' --uuid 6384a551-a41d-4290-b186-9258befede97"] +[2018-09-25 09:40:28.505377708] process done ExitSuccess +[2018-09-25 09:40:28.505676376] chat: ssh ["bids@rolando.cns","-S",".git/annex/ssh/bids@rolando.cns","-o","ControlMaster=auto","-o","ControlPersist=yes","-T","git-annex-shell 'p2pstdio' '/inbox/BIDS/dbic/QA' '--debug' 'ba8f2cea-f229-422c-82be-6580e5e07ed5' --uuid 6384a551-a41d-4290-b186-9258befede97"] +[2018-09-25 09:40:28.508740142] process done ExitSuccess +[2018-09-25 09:40:28.508761163] process done ExitSuccess +[2018-09-25 09:40:28.508782029] process done ExitSuccess +[2018-09-25 09:40:28.50891246] chat: ssh ["bids@rolando.cns","-S",".git/annex/ssh/bids@rolando.cns","-o","ControlMaster=auto","-o","ControlPersist=yes","-T","git-annex-shell 'p2pstdio' '/inbox/BIDS/dbic/QA' '--debug' 'ba8f2cea-f229-422c-82be-6580e5e07ed5' --uuid 6384a551-a41d-4290-b186-9258befede97"] +[2018-09-25 09:40:28.508944895] chat: ssh ["bids@rolando.cns","-S",".git/annex/ssh/bids@rolando.cns","-o","ControlMaster=auto","-o","ControlPersist=yes","-T","git-annex-shell 'p2pstdio' '/inbox/BIDS/dbic/QA' '--debug' 'ba8f2cea-f229-422c-82be-6580e5e07ed5' --uuid 6384a551-a41d-4290-b186-9258befede97"] +[2018-09-25 09:40:28.509074882] chat: ssh ["bids@rolando.cns","-S",".git/annex/ssh/bids@rolando.cns","-o","ControlMaster=auto","-o","ControlPersist=yes","-T","git-annex-shell 'p2pstdio' '/inbox/BIDS/dbic/QA' '--debug' 'ba8f2cea-f229-422c-82be-6580e5e07ed5' --uuid 6384a551-a41d-4290-b186-9258befede97"] +[2018-09-25 09:40:29.936552742] P2P < AUTH-SUCCESS 6384a551-a41d-4290-b186-9258befede97 +[2018-09-25 09:40:29.93677054] P2P > VERSION 1 +[2018-09-25 09:40:29.937971781] P2P < VERSION 1 +[2018-09-25 09:40:29.938155534] P2P > GET 0 .heudiconv/qa/ses-20171127/info/filegroup_ses-20171127.json SHA256E-s429026--12edfd6e2c58f05803cb5bff0ff3665f46229b5593354324fbecdd5b131506fc.json +[2018-09-25 09:40:29.940554579] P2P < GET 0 .heudiconv/qa/ses-20171127/info/filegroup_ses-20171127.json SHA256E-s429026--12edfd6e2c58f05803cb5bff0ff3665f46229b5593354324fbecdd5b131506fc.json[2018-09-25 09:40:29.967840409] P2P < AUTH-SUCCESS 6384a551-a41d-4290-b186-9258befede97 +[2018-09-25 09:40:29.967989255] P2P > VERSION 1 +[2018-09-25 09:40:29.969234907] P2P < VERSION 1 +[2018-09-25 09:40:29.96933411] P2P > GET 0 .heudiconv/qa/ses-20171204/info/dicominfo_ses-20171204.tsv SHA256E-s1633--7aee5bc34c95def9972b1f99f1a9ad976a3acb9da9f18e8bfa8888f579c111f6.tsv +[2018-09-25 09:40:29.971575677] P2P < GET 0 .heudiconv/qa/ses-20171204/info/dicominfo_ses-20171204.tsv SHA256E-s1633--7aee5bc34c95def9972b1f99f1a9ad976a3acb9da9f18e8bfa8888f579c111f6.tsv[2018-09-25 09:40:29.972619373] P2P > DATA 429026[2018-09-25 09:40:29.971577514] P2P < DATA 429026 +[2018-09-25 09:40:29.981195211] P2P < AUTH-SUCCESS 6384a551-a41d-4290-b186-9258befede97 +[2018-09-25 09:40:29.981346807] P2P > VERSION 1 +[2018-09-25 09:40:29.982599704] P2P < VERSION 1 +[2018-09-25 09:40:29.982851645] P2P > GET 0 .heudiconv/qa/ses-20171113/info/filegroup_ses-20171113.json SHA256E-s429026--2f0daf77e8ff5d902c00f88343b3cad51a2c7abc1e4702d450e5aea8dafc4ca6.json +[2018-09-25 09:40:29.98372615] P2P < AUTH-SUCCESS 6384a551-a41d-4290-b186-9258befede97 +[2018-09-25 09:40:29.983855252] P2P > VERSION 1 +[2018-09-25 09:40:29.985098551] P2P < GET 0 .heudiconv/qa/ses-20171113/info/filegroup_ses-20171113.json SHA256E-s429026--2f0daf77e8ff5d902c00f88343b3cad51a2c7abc1e4702d450e5aea8dafc4ca6.json[2018-09-25 09:40:29.985099291] P2P < VERSION 1 +[2018-09-25 09:40:29.985268868] P2P > GET 0 .heudiconv/qa/ses-20171113/info/dicominfo_ses-20171113.tsv SHA256E-s1633--76397f694088587dfbe2cbea5d35341c7588dbcb262a0ff91f94a07892c83409.tsv +[2018-09-25 09:40:29.986356073] P2P < AUTH-SUCCESS 6384a551-a41d-4290-b186-9258befede97 +[2018-09-25 09:40:29.9876026] P2P < GET 0 .heudiconv/qa/ses-20171113/info/dicominfo_ses-20171113.tsv SHA256E-s1633--76397f694088587dfbe2cbea5d35341c7588dbcb262a0ff91f94a07892c83409.tsv[2018-09-25 09:40:29.986449339] P2P > VERSION 1 +[2018-09-25 09:40:29.987893014] P2P < VERSION 1 +[2018-09-25 09:40:29.988057111] P2P > GET 0 .heudiconv/qa/ses-20171106/info/filegroup_ses-20171106.json SHA256E-s429026--cd00d181cf477405db3b5243fbd5d7969efffccf16b49599a1cd1fde32879a66.json +[2018-09-25 09:40:29.990480076] P2P < GET 0 .heudiconv/qa/ses-20171106/info/filegroup_ses-20171106.json SHA256E-s429026--cd00d181cf477405db3b5243fbd5d7969efffccf16b49599a1cd1fde32879a66.json{"byte-progress":32752,"action":{"command":"get","note":"from origin...","key":"SHA256E-s429026--12edfd6e2c58f05803cb5bff0ff3665f46229b5593354324fbecdd5b131506fc.json","file":".heudiconv/qa/ses-20171127/info/filegroup_ses-20171127.json"},"total-size":429026,"percent-progress":"7.63%"} +[2018-09-25 09:40:30.109171443] P2P > VALID{"byte-progress":429026,"action":{"command":"get","note":"from origin...","key":"SHA256E-s429026--12edfd6e2c58f05803cb5bff0ff3665f46229b5593354324fbecdd5b131506fc.json","file":".heudiconv/qa/ses-20171127/info/filegroup_ses-20171127.json"},"total-size":429026,"percent-progress":"100%"} +[2018-09-25 09:40:30.109206914] P2P < VALID +[2018-09-25 09:40:30.109391344] P2P > SUCCESS +[2018-09-25 09:40:30.111586675] P2P < SUCCESS{"command":"get","note":"from origin...\nchecksum...","success":true,"key":"SHA256E-s429026--12edfd6e2c58f05803cb5bff0ff3665f46229b5593354324fbecdd5b131506fc.json","file":".heudiconv/qa/ses-20171127/info/filegroup_ses-20171127.json"} + +"""]] + +on the remote I see 5 sleeping annex-shells: +[[!format sh """ +[bids@rolando ~] > ps auxw | grep git-anne[x] | nl + 1 bids 17254 0.0 0.0 1074095936 12788 ? Ssl 09:40 0:00 /afs/dbic.dartmouth.edu/usr/pkg/git-annex/git-annex_v2/exe/git-annex-shell --library-path /afs/dbic.dartmouth.edu/usr/pkg/git-annex/git-annex_v2//usr/lib/x86_64-linux-gnu/gconv:/afs/dbic.dartmouth.edu/usr/pkg/git-annex/git-annex_v2//usr/lib/x86_64-linux-gnu/audit:/afs/dbic.dartmouth.edu/usr/pkg/git-annex/git-annex_v2//etc/ld.so.conf.d:/afs/dbic.dartmouth.edu/usr/pkg/git-annex/git-annex_v2//lib64:/afs/dbic.dartmouth.edu/usr/pkg/git-annex/git-annex_v2//lib/x86_64-linux-gnu:/afs/dbic.dartmouth.edu/usr/pkg/git-annex/git-annex_v2//usr/lib/x86_64-linux-gnu: /afs/dbic.dartmouth.edu/usr/pkg/git-annex/git-annex_v2/shimmed/git-annex-shell/git-annex-shell p2pstdio /inbox/BIDS/dbic/QA --debug ba8f2cea-f229-422c-82be-6580e5e07ed5 --uuid 6384a551-a41d-4290-b186-9258befede97 + 2 bids 17259 0.0 0.0 1074095936 12004 ? Ssl 09:40 0:00 /afs/dbic.dartmouth.edu/usr/pkg/git-annex/git-annex_v2/exe/git-annex-shell --library-path /afs/dbic.dartmouth.edu/usr/pkg/git-annex/git-annex_v2//usr/lib/x86_64-linux-gnu/gconv:/afs/dbic.dartmouth.edu/usr/pkg/git-annex/git-annex_v2//usr/lib/x86_64-linux-gnu/audit:/afs/dbic.dartmouth.edu/usr/pkg/git-annex/git-annex_v2//etc/ld.so.conf.d:/afs/dbic.dartmouth.edu/usr/pkg/git-annex/git-annex_v2//lib64:/afs/dbic.dartmouth.edu/usr/pkg/git-annex/git-annex_v2//lib/x86_64-linux-gnu:/afs/dbic.dartmouth.edu/usr/pkg/git-annex/git-annex_v2//usr/lib/x86_64-linux-gnu: /afs/dbic.dartmouth.edu/usr/pkg/git-annex/git-annex_v2/shimmed/git-annex-shell/git-annex-shell p2pstdio /inbox/BIDS/dbic/QA --debug ba8f2cea-f229-422c-82be-6580e5e07ed5 --uuid 6384a551-a41d-4290-b186-9258befede97 + 3 bids 17260 0.0 0.0 1074095936 12012 ? Ssl 09:40 0:00 /afs/dbic.dartmouth.edu/usr/pkg/git-annex/git-annex_v2/exe/git-annex-shell --library-path /afs/dbic.dartmouth.edu/usr/pkg/git-annex/git-annex_v2//usr/lib/x86_64-linux-gnu/gconv:/afs/dbic.dartmouth.edu/usr/pkg/git-annex/git-annex_v2//usr/lib/x86_64-linux-gnu/audit:/afs/dbic.dartmouth.edu/usr/pkg/git-annex/git-annex_v2//etc/ld.so.conf.d:/afs/dbic.dartmouth.edu/usr/pkg/git-annex/git-annex_v2//lib64:/afs/dbic.dartmouth.edu/usr/pkg/git-annex/git-annex_v2//lib/x86_64-linux-gnu:/afs/dbic.dartmouth.edu/usr/pkg/git-annex/git-annex_v2//usr/lib/x86_64-linux-gnu: /afs/dbic.dartmouth.edu/usr/pkg/git-annex/git-annex_v2/shimmed/git-annex-shell/git-annex-shell p2pstdio /inbox/BIDS/dbic/QA --debug ba8f2cea-f229-422c-82be-6580e5e07ed5 --uuid 6384a551-a41d-4290-b186-9258befede97 + 4 bids 17261 0.0 0.0 1074095936 12012 ? Ssl 09:40 0:00 /afs/dbic.dartmouth.edu/usr/pkg/git-annex/git-annex_v2/exe/git-annex-shell --library-path /afs/dbic.dartmouth.edu/usr/pkg/git-annex/git-annex_v2//usr/lib/x86_64-linux-gnu/gconv:/afs/dbic.dartmouth.edu/usr/pkg/git-annex/git-annex_v2//usr/lib/x86_64-linux-gnu/audit:/afs/dbic.dartmouth.edu/usr/pkg/git-annex/git-annex_v2//etc/ld.so.conf.d:/afs/dbic.dartmouth.edu/usr/pkg/git-annex/git-annex_v2//lib64:/afs/dbic.dartmouth.edu/usr/pkg/git-annex/git-annex_v2//lib/x86_64-linux-gnu:/afs/dbic.dartmouth.edu/usr/pkg/git-annex/git-annex_v2//usr/lib/x86_64-linux-gnu: /afs/dbic.dartmouth.edu/usr/pkg/git-annex/git-annex_v2/shimmed/git-annex-shell/git-annex-shell p2pstdio /inbox/BIDS/dbic/QA --debug ba8f2cea-f229-422c-82be-6580e5e07ed5 --uuid 6384a551-a41d-4290-b186-9258befede97 + 5 bids 17262 0.0 0.0 1074095936 12008 ? Ssl 09:40 0:00 /afs/dbic.dartmouth.edu/usr/pkg/git-annex/git-annex_v2/exe/git-annex-shell --library-path /afs/dbic.dartmouth.edu/usr/pkg/git-annex/git-annex_v2//usr/lib/x86_64-linux-gnu/gconv:/afs/dbic.dartmouth.edu/usr/pkg/git-annex/git-annex_v2//usr/lib/x86_64-linux-gnu/audit:/afs/dbic.dartmouth.edu/usr/pkg/git-annex/git-annex_v2//etc/ld.so.conf.d:/afs/dbic.dartmouth.edu/usr/pkg/git-annex/git-annex_v2//lib64:/afs/dbic.dartmouth.edu/usr/pkg/git-annex/git-annex_v2//lib/x86_64-linux-gnu:/afs/dbic.dartmouth.edu/usr/pkg/git-annex/git-annex_v2//usr/lib/x86_64-linux-gnu: /afs/dbic.dartmouth.edu/usr/pkg/git-annex/git-annex_v2/shimmed/git-annex-shell/git-annex-shell p2pstdio /inbox/BIDS/dbic/QA --debug ba8f2cea-f229-422c-82be-6580e5e07ed5 --uuid 6384a551-a41d-4290-b186-9258befede97 +"""]] + +local version is 6.20180913+git33-g2cd5a723f-1~ndall+1 and remote 6.20180808-ga1327779a (it is like Sep 13, current standalone build). +There was/is no other transfer in progress when calling it, but some previously failed runs. That file gets just fine later on if run get on it alone +[[!format sh """ +$> git annex get .heudiconv/qa/ses-20171113/info/filegroup_ses-20171113.json +get .heudiconv/qa/ses-20171113/info/filegroup_ses-20171113.json (from origin...) +(checksum...) ok +(recording state in git...) +"""]] + +so to me smells like some race condition due to high -J value. + +[[!meta author=yoh]] diff --git a/doc/bugs/annex_get_-J_16_via_ssh_stalls_/comment_1_4916bb94a700e72d42302d2938693daa._comment b/doc/bugs/annex_get_-J_16_via_ssh_stalls_/comment_1_4916bb94a700e72d42302d2938693daa._comment new file mode 100644 index 0000000000..4d20d7a00c --- /dev/null +++ b/doc/bugs/annex_get_-J_16_via_ssh_stalls_/comment_1_4916bb94a700e72d42302d2938693daa._comment @@ -0,0 +1,8 @@ +[[!comment format=mdwn + username="yarikoptic" + avatar="http://cdn.libravatar.org/avatar/f11e9c84cb18d26a1748c33b48c924b4" + subject="same as another one?" + date="2018-09-25T15:35:15Z" + content=""" +might relate to my other recent report [about needing to retry while transfering over ssh/rsync](http://git-annex.branchable.com/bugs/some_transfers_for_get_-J4_fail_over_ssh__47__rsync_and_require_retry/) +"""]] diff --git a/doc/bugs/annex_get_-J_16_via_ssh_stalls_/comment_2_8d391bacf291492846d9253057653082._comment b/doc/bugs/annex_get_-J_16_via_ssh_stalls_/comment_2_8d391bacf291492846d9253057653082._comment new file mode 100644 index 0000000000..1116dca82a --- /dev/null +++ b/doc/bugs/annex_get_-J_16_via_ssh_stalls_/comment_2_8d391bacf291492846d9253057653082._comment @@ -0,0 +1,12 @@ +[[!comment format=mdwn + username="joey" + subject="""comment 2""" + date="2018-09-25T17:01:00Z" + content=""" +All this debug output means nothing to me, what operation is it stalling +on? + + {"command":"get","wanted":[{"here":false,"uuid":"6384a551-a41d-4290-b186-9258befede97","description":"bids@rolando:/inbox/BIDS/dbic/QA [origin]"}],"note":"transfer already in progress, or unable to take transfer lock\nUnable to access these remotes: origin\nTry making some of these repositories available:\n\t6384a551-a41d-4290-b186-9258befede97 -- bids@rolando:/inbox/BIDS/dbic/QA [origin]\n","skipped":[],"success":false,"key":"SHA256E-s1633--1bc7a100b82cd31d084aae5b25b8d90af128324d10d92cfc9e2196d28829d375.tsv","file":".heudiconv/qa/ses-20171106/info/dicominfo_ses-20171106.tsv"} + +Are two files in the set of files you operated on both pointing at this key? +"""]] diff --git a/doc/bugs/annex_get_-J_16_via_ssh_stalls_/comment_3_6f228fa6770aad308d0f99647ac21347._comment b/doc/bugs/annex_get_-J_16_via_ssh_stalls_/comment_3_6f228fa6770aad308d0f99647ac21347._comment new file mode 100644 index 0000000000..92daaeba35 --- /dev/null +++ b/doc/bugs/annex_get_-J_16_via_ssh_stalls_/comment_3_6f228fa6770aad308d0f99647ac21347._comment @@ -0,0 +1,26 @@ +[[!comment format=mdwn + username="yarikoptic" + avatar="http://cdn.libravatar.org/avatar/f11e9c84cb18d26a1748c33b48c924b4" + subject="comment 3" + date="2018-09-25T22:28:49Z" + content=""" +> All this debug output means nothing to me, what operation is it stalling on? + +Do you mean that command I have pasted in the beginning `git annex get -J 16 --debug --json-progress .`? if something else, I cannot add beyond that command and `--debug` output. + +> Are two files in the set of files you operated on both pointing at this key? + +Ha, good question... seems to be not?: + +[[!format sh \"\"\" +$> git clone http://datasets.datalad.org/dbic/QA/.git +Cloning into 'QA'... + +$> cd QA + +$> find -lname */SHA256E-s1633--1bc7a100b82cd31d084aae5b25b8d90af128324d10d92cfc9e2196d28829d375.tsv +./.heudiconv/qa/ses-20171106/info/dicominfo_ses-20171106.tsv + +\"\"\"]] +the only thing is that I have interrupted some previous stalled `get` invocations so may be it is somehow related? +"""]] diff --git a/doc/bugs/annex_get_-J_16_via_ssh_stalls_/comment_4_3eaebe28f2925cac21a49f30aed3fa3c._comment b/doc/bugs/annex_get_-J_16_via_ssh_stalls_/comment_4_3eaebe28f2925cac21a49f30aed3fa3c._comment new file mode 100644 index 0000000000..328c047313 --- /dev/null +++ b/doc/bugs/annex_get_-J_16_via_ssh_stalls_/comment_4_3eaebe28f2925cac21a49f30aed3fa3c._comment @@ -0,0 +1,239 @@ +[[!comment format=mdwn + username="yarikoptic" + avatar="http://cdn.libravatar.org/avatar/f11e9c84cb18d26a1748c33b48c924b4" + subject="reproducible, no locking message, many "duplicate" files" + date="2018-09-26T00:06:29Z" + content=""" +tried to replicate from other hosts, but without success, so might be something specific about that one (original repo is on NFS (mount options rw,relatime,vers=3,rsize=131072,wsize=131072,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,mountaddr=10.232.11.5,mountvers=3,mountport=1009,mountproto=udp,local_lock=none,addr=10.232.11.5), CentOS Linux release 7.4.1708 (Core), git annex 6.20180808-ga1327779a, OpenSSH_7.4p1, OpenSSL 1.0.2k-fips). This time it is also without any message about being unable to lock: + +[[!format sh \"\"\" +$> git clone bids@rolando.cns:/inbox/BIDS/dbic/QA +Cloning into 'QA'... + + + Dartmouth College, Department of Psychological and Brain Sciences + Authorized access only + + + + +remote: Counting objects: 26086, done. +remote: Compressing objects: 100% (9257/9257), done. +remote: Total 26086 (delta 11366), reused 25687 (delta 10968) +Receiving objects: 100% (26086/26086), 30.63 MiB | 53.17 MiB/s, done. +Resolving deltas: 100% (11366/11366), done. +1 13074.....................................:Tue 25 Sep 2018 07:39:11 PM EDT:. +smaug:/tmp +$> cd QA +CHANGES dataset_description.json sub-emmet/ task-rest_acq-3mm_bold.json task-rest_acq-p2Xs4X35mm_bold.json +README participants.tsv sub-qa/ task-rest_acq-noXgrappa_bold.json task-rest_acq-p2_bold.json +README.md sourcedata/ sub-sid000143/ task-rest_acq-p22_bold.json task-rest_acq-p2noprescannormalize_bold.json +code/ sub-amit/ task-rest_acq-24mm64sl1000tr32te600dyn_bold.json task-rest_acq-p2XB0shimXadvanced_bold.json +1 13075.....................................:Tue 25 Sep 2018 07:39:12 PM EDT:. +(git)smaug:/tmp/QA[master] +$> git annex get -J16 --debug --json --json-progress . +[2018-09-25 19:39:24.076576505] read: git [\"--git-dir=.git\",\"--work-tree=.\",\"--literal-pathspecs\",\"show-ref\",\"git-annex\"] +[2018-09-25 19:39:24.08073842] process done ExitSuccess +[2018-09-25 19:39:24.080921239] read: git [\"--git-dir=.git\",\"--work-tree=.\",\"--literal-pathspecs\",\"show-ref\",\"--hash\",\"refs/heads/git-annex\"] +[2018-09-25 19:39:24.083832166] process done ExitFailure 1 +[2018-09-25 19:39:24.084026056] call: git [\"--git-dir=.git\",\"--work-tree=.\",\"--literal-pathspecs\",\"show-ref\",\"--verify\",\"-q\",\"origin/git-annex\"] +[2018-09-25 19:39:24.086534115] process done ExitFailure 1 +[2018-09-25 19:39:24.087406723] read: git [\"--git-dir=.git\",\"--work-tree=.\",\"--literal-pathspecs\",\"write-tree\"] +[2018-09-25 19:39:24.090577834] process done ExitSuccess +[2018-09-25 19:39:24.090805078] chat: git [\"--git-dir=.git\",\"--work-tree=.\",\"--literal-pathspecs\",\"commit-tree\",\"4b825dc642cb6eb9a060e54bf8d69288fbee4904\",\"--no-gpg-sign\"] +[2018-09-25 19:39:24.094384723] process done ExitSuccess +[2018-09-25 19:39:24.094571341] call: git [\"--git-dir=.git\",\"--work-tree=.\",\"--literal-pathspecs\",\"update-ref\",\"refs/heads/git-annex\",\"0fd8155db835617a672760453bf6fd0786d16ea7\"] +[2018-09-25 19:39:24.099179252] process done ExitSuccess +[2018-09-25 19:39:24.163625058] call: git [\"--git-dir=.git\",\"--work-tree=.\",\"--literal-pathspecs\",\"config\",\"annex.uuid\",\"e57c285c-4145-4c57-9e9c-73c29a6dc115\"] +[2018-09-25 19:39:24.167183166] process done ExitSuccess +[2018-09-25 19:39:24.167313312] read: git [\"config\",\"--null\",\"--list\"] +[2018-09-25 19:39:24.171253975] process done ExitSuccess +[2018-09-25 19:39:24.182032449] read: git [\"--git-dir=.git\",\"--work-tree=.\",\"--literal-pathspecs\",\"show-ref\",\"git-annex\"] +[2018-09-25 19:39:24.183696686] process done ExitSuccess +[2018-09-25 19:39:24.183777495] read: git [\"--git-dir=.git\",\"--work-tree=.\",\"--literal-pathspecs\",\"show-ref\",\"--hash\",\"refs/heads/git-annex\"] +[2018-09-25 19:39:24.185400318] process done ExitSuccess +[2018-09-25 19:39:24.185531597] read: git [\"--git-dir=.git\",\"--work-tree=.\",\"--literal-pathspecs\",\"log\",\"refs/heads/git-annex..0fd8155db835617a672760453bf6fd0786d16ea7\",\"--pretty=%H\",\"-n1\"] +[2018-09-25 19:39:24.18734038] process done ExitSuccess +[2018-09-25 19:39:24.187415347] read: git [\"--git-dir=.git\",\"--work-tree=.\",\"--literal-pathspecs\",\"log\",\"refs/heads/git-annex..70f0788641a4b3b77353b14ee6db0986a1c1e682\",\"--pretty=%H\",\"-n1\"] +[2018-09-25 19:39:24.191055467] process done ExitSuccess +[2018-09-25 19:39:24.191255165] chat: git [\"--git-dir=.git\",\"--work-tree=.\",\"--literal-pathspecs\",\"cat-file\",\"--batch\"] +[2018-09-25 19:39:24.191633582] chat: git [\"--git-dir=.git\",\"--work-tree=.\",\"--literal-pathspecs\",\"cat-file\",\"--batch-check=%(objectname) %(objecttype) %(objectsize)\"] +[2018-09-25 19:39:24.193406759] chat: git [\"--git-dir=.git\",\"--work-tree=.\",\"--literal-pathspecs\",\"hash-object\",\"-w\",\"--stdin-paths\",\"--no-filters\"] +[2018-09-25 19:39:24.193797713] feed: git [\"--git-dir=.git\",\"--work-tree=.\",\"--literal-pathspecs\",\"update-index\",\"-z\",\"--index-info\"] +[2018-09-25 19:39:24.19424715] read: git [\"--git-dir=.git\",\"--work-tree=.\",\"--literal-pathspecs\",\"diff-index\",\"--raw\",\"-z\",\"-r\",\"--no-renames\",\"-l0\",\"--cached\",\"70f0788641a4b3b77353b14ee6db0986a1c1e682\",\"--\"] +[2018-09-25 19:39:24.402505793] process done ExitSuccess +[2018-09-25 19:39:24.411127976] process done ExitSuccess +[2018-09-25 19:39:24.411819307] read: git [\"--git-dir=.git\",\"--work-tree=.\",\"--literal-pathspecs\",\"log\",\"70f0788641a4b3b77353b14ee6db0986a1c1e682..refs/heads/git-annex\",\"--pretty=%H\",\"-n1\"] +[2018-09-25 19:39:24.414936679] process done ExitSuccess +[2018-09-25 19:39:24.415127364] read: git [\"--git-dir=.git\",\"--work-tree=.\",\"--literal-pathspecs\",\"write-tree\"] +[2018-09-25 19:39:24.426606063] process done ExitSuccess +[2018-09-25 19:39:24.426728558] chat: git [\"--git-dir=.git\",\"--work-tree=.\",\"--literal-pathspecs\",\"commit-tree\",\"961159c1de057d7120f7cea5c0f151954ab400c1\",\"--no-gpg-sign\",\"-p\",\"refs/heads/git-annex\",\"-p\",\"70f0788641a4b3b77353b14ee6db0986a1c1e682\"] +[2018-09-25 19:39:24.42982717] process done ExitSuccess +[2018-09-25 19:39:24.429968237] call: git [\"--git-dir=.git\",\"--work-tree=.\",\"--literal-pathspecs\",\"update-ref\",\"refs/heads/git-annex\",\"691a2081d0839b84e286f50330ffbb0fa4dfc22f\"] +[2018-09-25 19:39:24.433255645] process done ExitSuccess +[2018-09-25 19:39:24.434830328] call: git [\"--git-dir=.git\",\"--work-tree=.\",\"--literal-pathspecs\",\"config\",\"annex.version\",\"5\"] +[2018-09-25 19:39:24.436846163] process done ExitSuccess +[2018-09-25 19:39:24.436926679] read: git [\"config\",\"--null\",\"--list\"] +[2018-09-25 19:39:24.449483362] process done ExitSuccess +[2018-09-25 19:39:24.449648554] read: git [\"--git-dir=.git\",\"--work-tree=.\",\"--literal-pathspecs\",\"symbolic-ref\",\"-q\",\"HEAD\"] +[2018-09-25 19:39:24.452355348] process done ExitSuccess +[2018-09-25 19:39:24.452485849] read: git [\"--git-dir=.git\",\"--work-tree=.\",\"--literal-pathspecs\",\"show-ref\",\"refs/heads/master\"] +[2018-09-25 19:39:24.455464092] process done ExitSuccess +[2018-09-25 19:39:24.455563403] read: git [\"--git-dir=.git\",\"--work-tree=.\",\"--literal-pathspecs\",\"symbolic-ref\",\"-q\",\"HEAD\"] +[2018-09-25 19:39:24.458025223] process done ExitSuccess +[2018-09-25 19:39:24.458139592] read: git [\"--git-dir=.git\",\"--work-tree=.\",\"--literal-pathspecs\",\"show-ref\",\"--hash\",\"refs/heads/master\"] +[2018-09-25 19:39:24.461541709] process done ExitSuccess +[2018-09-25 19:39:24.461682534] call: git [\"--git-dir=.git\",\"--work-tree=.\",\"--literal-pathspecs\",\"checkout\",\"-q\",\"-B\",\"master\"] +[2018-09-25 19:39:24.523918634] process done ExitSuccess +[2018-09-25 19:39:24.525217891] read: uname [\"-n\"] +[2018-09-25 19:39:24.527130923] process done ExitSuccess +[2018-09-25 19:39:24.528961091] read: git [\"--git-dir=.git\",\"--work-tree=.\",\"--literal-pathspecs\",\"ls-files\",\"--cached\",\"-z\",\"--\",\".\"] +[2018-09-25 19:39:24.53401854] read: ssh [\"-o\",\"BatchMode=true\",\"-S\",\".git/annex/ssh/bids@rolando.cns\",\"-o\",\"ControlMaster=auto\",\"-o\",\"ControlPersist=yes\",\"-n\",\"-T\",\"bids@rolando.cns\",\"true\"] +[2018-09-25 19:39:26.071148456] process done ExitSuccess +[2018-09-25 19:39:26.071456884] read: ssh [\"bids@rolando.cns\",\"-S\",\".git/annex/ssh/bids@rolando.cns\",\"-o\",\"ControlMaster=auto\",\"-o\",\"ControlPersist=yes\",\"-n\",\"-T\",\"git-annex-shell 'configlist' '/inbox/BIDS/dbic/QA' '--debug'\"] +[2018-09-25 19:39:27.533523078] process done ExitSuccess +[2018-09-25 19:39:27.533732629] call: git [\"--git-dir=.git\",\"--work-tree=.\",\"--literal-pathspecs\",\"config\",\"remote.origin.annex-uuid\",\"6384a551-a41d-4290-b186-9258befede97\"] +[2018-09-25 19:39:27.541031187] process done ExitSuccess +[2018-09-25 19:39:27.541185336] read: git [\"config\",\"--null\",\"--list\"] +[2018-09-25 19:39:27.546614539] process done ExitSuccess +[2018-09-25 19:39:27.548934829] chat: git [\"--git-dir=.git\",\"--work-tree=.\",\"--literal-pathspecs\",\"cat-file\",\"--batch\"] +[2018-09-25 19:39:27.54893425] chat: git [\"--git-dir=.git\",\"--work-tree=.\",\"--literal-pathspecs\",\"cat-file\",\"--batch\"] +[2018-09-25 19:39:27.5490026] chat: git [\"--git-dir=.git\",\"--work-tree=.\",\"--literal-pathspecs\",\"cat-file\",\"--batch\"] +[2018-09-25 19:39:27.54908919] chat: git [\"--git-dir=.git\",\"--work-tree=.\",\"--literal-pathspecs\",\"cat-file\",\"--batch\"] +[2018-09-25 19:39:27.549123562] chat: git [\"--git-dir=.git\",\"--work-tree=.\",\"--literal-pathspecs\",\"cat-file\",\"--batch\"] +[2018-09-25 19:39:27.549156199] chat: git [\"--git-dir=.git\",\"--work-tree=.\",\"--literal-pathspecs\",\"cat-file\",\"--batch\"] +[2018-09-25 19:39:27.549209727] chat: git [\"--git-dir=.git\",\"--work-tree=.\",\"--literal-pathspecs\",\"cat-file\",\"--batch\"] +[2018-09-25 19:39:27.549329392] chat: git [\"--git-dir=.git\",\"--work-tree=.\",\"--literal-pathspecs\",\"cat-file\",\"--batch\"] +[2018-09-25 19:39:27.549398039] chat: git [\"--git-dir=.git\",\"--work-tree=.\",\"--literal-pathspecs\",\"cat-file\",\"--batch\"] +[2018-09-25 19:39:27.549438971] chat: git [\"--git-dir=.git\",\"--work-tree=.\",\"--literal-pathspecs\",\"cat-file\",\"--batch\"] +[2018-09-25 19:39:27.549469661] chat: git [\"--git-dir=.git\",\"--work-tree=.\",\"--literal-pathspecs\",\"cat-file\",\"--batch\"] +[2018-09-25 19:39:27.549491832] chat: git [\"--git-dir=.git\",\"--work-tree=.\",\"--literal-pathspecs\",\"cat-file\",\"--batch\"] +[2018-09-25 19:39:27.549500665] chat: git [\"--git-dir=.git\",\"--work-tree=.\",\"--literal-pathspecs\",\"cat-file\",\"--batch-check=%(objectname) %(objecttype) %(objectsize)\"] +[2018-09-25 19:39:27.549677068] chat: git [\"--git-dir=.git\",\"--work-tree=.\",\"--literal-pathspecs\",\"cat-file\",\"--batch-check=%(objectname) %(objecttype) %(objectsize)\"] +[2018-09-25 19:39:27.549933199] chat: git [\"--git-dir=.git\",\"--work-tree=.\",\"--literal-pathspecs\",\"cat-file\",\"--batch-check=%(objectname) %(objecttype) %(objectsize)\"] +[2018-09-25 19:39:27.550251508] chat: git [\"--git-dir=.git\",\"--work-tree=.\",\"--literal-pathspecs\",\"cat-file\",\"--batch-check=%(objectname) %(objecttype) %(objectsize)\"] +[2018-09-25 19:39:27.550728146] chat: git [\"--git-dir=.git\",\"--work-tree=.\",\"--literal-pathspecs\",\"cat-file\",\"--batch-check=%(objectname) %(objecttype) %(objectsize)\"] +[2018-09-25 19:39:27.552286946] chat: git [\"--git-dir=.git\",\"--work-tree=.\",\"--literal-pathspecs\",\"cat-file\",\"--batch-check=%(objectname) %(objecttype) %(objectsize)\"] +[2018-09-25 19:39:27.552666906] chat: git [\"--git-dir=.git\",\"--work-tree=.\",\"--literal-pathspecs\",\"cat-file\",\"--batch-check=%(objectname) %(objecttype) %(objectsize)\"] +[2018-09-25 19:39:27.552969316] chat: git [\"--git-dir=.git\",\"--work-tree=.\",\"--literal-pathspecs\",\"cat-file\",\"--batch-check=%(objectname) %(objecttype) %(objectsize)\"] +[2018-09-25 19:39:27.553792735] chat: git [\"--git-dir=.git\",\"--work-tree=.\",\"--literal-pathspecs\",\"cat-file\",\"--batch-check=%(objectname) %(objecttype) %(objectsize)\"] +[2018-09-25 19:39:27.55697274] chat: git [\"--git-dir=.git\",\"--work-tree=.\",\"--literal-pathspecs\",\"cat-file\",\"--batch-check=%(objectname) %(objecttype) %(objectsize)\"] +[2018-09-25 19:39:27.557179172] chat: git [\"--git-dir=.git\",\"--work-tree=.\",\"--literal-pathspecs\",\"cat-file\",\"--batch-check=%(objectname) %(objecttype) %(objectsize)\"] +[2018-09-25 19:39:27.557586095] chat: git [\"--git-dir=.git\",\"--work-tree=.\",\"--literal-pathspecs\",\"cat-file\",\"--batch-check=%(objectname) %(objecttype) %(objectsize)\"] +[2018-09-25 19:39:27.559528852] chat: ssh [\"bids@rolando.cns\",\"-S\",\".git/annex/ssh/bids@rolando.cns\",\"-o\",\"ControlMaster=auto\",\"-o\",\"ControlPersist=yes\",\"-T\",\"git-annex-shell 'p2pstdio' '/inbox/BIDS/dbic/QA' '--debug' 'e57c285c-4145-4c57-9e9c-73c29a6dc115' --uuid 6384a551-a41d-4290-b186-9258befede97\"] +[2018-09-25 19:39:27.560166953] chat: ssh [\"bids@rolando.cns\",\"-S\",\".git/annex/ssh/bids@rolando.cns\",\"-o\",\"ControlMaster=auto\",\"-o\",\"ControlPersist=yes\",\"-T\",\"git-annex-shell 'p2pstdio' '/inbox/BIDS/dbic/QA' '--debug' 'e57c285c-4145-4c57-9e9c-73c29a6dc115' --uuid 6384a551-a41d-4290-b186-9258befede97\"] +[2018-09-25 19:39:27.560470671] chat: ssh [\"bids@rolando.cns\",\"-S\",\".git/annex/ssh/bids@rolando.cns\",\"-o\",\"ControlMaster=auto\",\"-o\",\"ControlPersist=yes\",\"-T\",\"git-annex-shell 'p2pstdio' '/inbox/BIDS/dbic/QA' '--debug' 'e57c285c-4145-4c57-9e9c-73c29a6dc115' --uuid 6384a551-a41d-4290-b186-9258befede97\"] +[2018-09-25 19:39:27.609120855] chat: ssh [\"bids@rolando.cns\",\"-S\",\".git/annex/ssh/bids@rolando.cns\",\"-o\",\"ControlMaster=auto\",\"-o\",\"ControlPersist=yes\",\"-T\",\"git-annex-shell 'p2pstdio' '/inbox/BIDS/dbic/QA' '--debug' 'e57c285c-4145-4c57-9e9c-73c29a6dc115' --uuid 6384a551-a41d-4290-b186-9258befede97\"] +[2018-09-25 19:39:27.628746035] chat: ssh [\"bids@rolando.cns\",\"-S\",\".git/annex/ssh/bids@rolando.cns\",\"-o\",\"ControlMaster=auto\",\"-o\",\"ControlPersist=yes\",\"-T\",\"git-annex-shell 'p2pstdio' '/inbox/BIDS/dbic/QA' '--debug' 'e57c285c-4145-4c57-9e9c-73c29a6dc115' --uuid 6384a551-a41d-4290-b186-9258befede97\"] +[2018-09-25 19:39:27.629098606] chat: ssh [\"bids@rolando.cns\",\"-S\",\".git/annex/ssh/bids@rolando.cns\",\"-o\",\"ControlMaster=auto\",\"-o\",\"ControlPersist=yes\",\"-T\",\"git-annex-shell 'p2pstdio' '/inbox/BIDS/dbic/QA' '--debug' 'e57c285c-4145-4c57-9e9c-73c29a6dc115' --uuid 6384a551-a41d-4290-b186-9258befede97\"] +[2018-09-25 19:39:27.642438219] chat: ssh [\"bids@rolando.cns\",\"-S\",\".git/annex/ssh/bids@rolando.cns\",\"-o\",\"ControlMaster=auto\",\"-o\",\"ControlPersist=yes\",\"-T\",\"git-annex-shell 'p2pstdio' '/inbox/BIDS/dbic/QA' '--debug' 'e57c285c-4145-4c57-9e9c-73c29a6dc115' --uuid 6384a551-a41d-4290-b186-9258befede97\"] +[2018-09-25 19:39:27.642444075] chat: ssh [\"bids@rolando.cns\",\"-S\",\".git/annex/ssh/bids@rolando.cns\",\"-o\",\"ControlMaster=auto\",\"-o\",\"ControlPersist=yes\",\"-T\",\"git-annex-shell 'p2pstdio' '/inbox/BIDS/dbic/QA' '--debug' 'e57c285c-4145-4c57-9e9c-73c29a6dc115' --uuid 6384a551-a41d-4290-b186-9258befede97\"] +[2018-09-25 19:39:27.642633512] chat: ssh [\"bids@rolando.cns\",\"-S\",\".git/annex/ssh/bids@rolando.cns\",\"-o\",\"ControlMaster=auto\",\"-o\",\"ControlPersist=yes\",\"-T\",\"git-annex-shell 'p2pstdio' '/inbox/BIDS/dbic/QA' '--debug' 'e57c285c-4145-4c57-9e9c-73c29a6dc115' --uuid 6384a551-a41d-4290-b186-9258befede97\"] +[2018-09-25 19:39:27.642685766] chat: ssh [\"bids@rolando.cns\",\"-S\",\".git/annex/ssh/bids@rolando.cns\",\"-o\",\"ControlMaster=auto\",\"-o\",\"ControlPersist=yes\",\"-T\",\"git-annex-shell 'p2pstdio' '/inbox/BIDS/dbic/QA' '--debug' 'e57c285c-4145-4c57-9e9c-73c29a6dc115' --uuid 6384a551-a41d-4290-b186-9258befede97\"] +[2018-09-25 19:39:27.64756999] chat: ssh [\"bids@rolando.cns\",\"-S\",\".git/annex/ssh/bids@rolando.cns\",\"-o\",\"ControlMaster=auto\",\"-o\",\"ControlPersist=yes\",\"-T\",\"git-annex-shell 'p2pstdio' '/inbox/BIDS/dbic/QA' '--debug' 'e57c285c-4145-4c57-9e9c-73c29a6dc115' --uuid 6384a551-a41d-4290-b186-9258befede97\"] +[2018-09-25 19:39:27.64783317] chat: ssh [\"bids@rolando.cns\",\"-S\",\".git/annex/ssh/bids@rolando.cns\",\"-o\",\"ControlMaster=auto\",\"-o\",\"ControlPersist=yes\",\"-T\",\"git-annex-shell 'p2pstdio' '/inbox/BIDS/dbic/QA' '--debug' 'e57c285c-4145-4c57-9e9c-73c29a6dc115' --uuid 6384a551-a41d-4290-b186-9258befede97\"] +[2018-09-25 19:39:29.140327887] P2P < AUTH-SUCCESS 6384a551-a41d-4290-b186-9258befede97 +[2018-09-25 19:39:29.140500055] P2P > VERSION 1 +[2018-09-25 19:39:29.141624424] P2P < VERSION 1 +[2018-09-25 19:39:29.141769124] P2P > GET 0 .heudiconv/amit/ses-20180508/info/dicominfo_ses-20180508.tsv SHA256E-s2072--84342b6857db244be0bfa97d4289ee6206ba8e34bb11dc23262f609b6478e783.tsv +[2018-09-25 19:39:29.1419719] P2P < AUTH-SUCCESS 6384a551-a41d-4290-b186-9258befede97 +[2018-09-25 19:39:29.142163513] P2P > VERSION 1 +[2018-09-25 19:39:29.143097033] P2P < GET 0 .heudiconv/amit/ses-20180508/info/dicominfo_ses-20180508.tsv SHA256E-s2072--84342b6857db244be0bfa97d4289ee6206ba8e34bb11dc23262f609b6478e783.tsv[2018-09-25 19:39:29.143589471] P2P < VERSION 1 +[2018-09-25 19:39:29.14370964] P2P > GET 0 .heudiconv/amit/ses-20180508/info/amit_ses-20180508.edit.txt SHA256E-s551--bc05c5596a6cf03ad7d302aab294dac7b45ecfcb96f7469085520660bf1c3e29.edit.txt +[2018-09-25 19:39:29.145075908] P2P < GET 0 .heudiconv/amit/ses-20180508/info/amit_ses-20180508.edit.txt SHA256E-s551--bc05c5596a6cf03ad7d302aab294dac7b45ecfcb96f7469085520660bf1c3e29.edit.txt[2018-09-25 19:39:29.172581646] P2P > DATA 2072[2018-09-25 19:39:29.172391553] P2P < DATA 2072 +[2018-09-25 19:39:29.17275537] P2P < AUTH-SUCCESS 6384a551-a41d-4290-b186-9258befede97 +[2018-09-25 19:39:29.172869445] P2P > VERSION 1 +[2018-09-25 19:39:29.1739641] P2P < VERSION 1 +[2018-09-25 19:39:29.174135119] P2P > GET 0 .heudiconv/emmet/ses-20180508/info/dicominfo_ses-20180508.tsv SHA256E-s2086--b3e49ddbe4c33b7872c031722b771e66ecfc67468c9d13e9abbaa5fdde2a2851.tsv +[2018-09-25 19:39:29.17581668] P2P > VALID[2018-09-25 19:39:29.175562895] P2P < GET 0 .heudiconv/emmet/ses-20180508/info/dicominfo_ses-20180508.tsv SHA256E-s2086--b3e49ddbe4c33b7872c031722b771e66ecfc67468c9d13e9abbaa5fdde2a2851.tsv{\"byte-progress\":2072,\"action\":{\"command\":\"get\",\"note\":\"from origin...\",\"key\":\"SHA256E-s2072--84342b6857db244be0bfa97d4289ee6206ba8e34bb11dc23262f609b6478e783.tsv\",\"file\":\".heudiconv/amit/ses-20180508/info/dicominfo_ses-20180508.tsv\"},\"total-size\":2072,\"percent-progress\":\"100%\"} +[2018-09-25 19:39:29.179265924] P2P < AUTH-SUCCESS 6384a551-a41d-4290-b186-9258befede97 +[2018-09-25 19:39:29.179393396] P2P > VERSION 1 +[2018-09-25 19:39:29.180519354] P2P < VERSION 1 +[2018-09-25 19:39:29.180693554] P2P > GET 0 .heudiconv/emmet/ses-20180521/info/emmet_ses-20180521.auto.txt SHA256E-s2658--42b972cbc99464913e8819e4944071a949914ec2cc7e786cb48fc26b41f665f1.auto.txt +[2018-09-25 19:39:29.181047474] P2P < AUTH-SUCCESS 6384a551-a41d-4290-b186-9258befede97 +[2018-09-25 19:39:29.18117293] P2P > VERSION 1 +[2018-09-25 19:39:29.182208685] P2P < VERSION 1 +[2018-09-25 19:39:29.18233901] P2P > GET 0 .heudiconv/amit/ses-20180508/info/amit_ses-20180508.auto.txt SHA256E-s551--bc05c5596a6cf03ad7d302aab294dac7b45ecfcb96f7469085520660bf1c3e29.auto.txt +[2018-09-25 19:39:29.183648585] P2P < GET 0 .heudiconv/amit/ses-20180508/info/amit_ses-20180508.auto.txt SHA256E-s551--bc05c5596a6cf03ad7d302aab294dac7b45ecfcb96f7469085520660bf1c3e29.auto.txt[2018-09-25 19:39:29.18996796] P2P < AUTH-SUCCESS 6384a551-a41d-4290-b186-9258befede97 +[2018-09-25 19:39:29.190121603] P2P > VERSION 1 +[2018-09-25 19:39:29.19043361] P2P < AUTH-SUCCESS 6384a551-a41d-4290-b186-9258befede97 +[2018-09-25 19:39:29.190512631] P2P > VERSION 1 +[2018-09-25 19:39:29.191211455] P2P < VERSION 1 +[2018-09-25 19:39:29.191380774] P2P > GET 0 .heudiconv/emmet/ses-20180508/info/filegroup_ses-20180508.json SHA256E-s222552--1d8be851f4aa959df6cf8b39313f9f8245d4f5c7897f3028eaf671c3140e07db.json +[2018-09-25 19:39:29.191418345] P2P < VERSION 1 +[2018-09-25 19:39:29.191614568] P2P > GET 0 .heudiconv/emmet/ses-20180521/info/emmet_ses-20180521.edit.txt SHA256E-s2658--42b972cbc99464913e8819e4944071a949914ec2cc7e786cb48fc26b41f665f1.edit.txt +[2018-09-25 19:39:29.192744192] P2P < GET 0 .heudiconv/emmet/ses-20180508/info/filegroup_ses-20180508.json SHA256E-s222552--1d8be851f4aa959df6cf8b39313f9f8245d4f5c7897f3028eaf671c3140e07db.json[2018-09-25 19:39:29.194287137] P2P < AUTH-SUCCESS 6384a551-a41d-4290-b186-9258befede97 +[2018-09-25 19:39:29.194417333] P2P > VERSION 1 +[2018-09-25 19:39:29.195415113] P2P < VERSION 1 +[2018-09-25 19:39:29.1956202] P2P < AUTH-SUCCESS 6384a551-a41d-4290-b186-9258befede97 +[2018-09-25 19:39:29.195628464] P2P > GET 0 .heudiconv/emmet/ses-20180508/info/reproin.py SHA256E-s39141--59c2abfbe8184c851079d1cc0d8b87ddb38239a20d08002610ed81866aff7680.py +[2018-09-25 19:39:29.195755609] P2P > VERSION 1 +[2018-09-25 19:39:29.196329099] P2P < AUTH-SUCCESS 6384a551-a41d-4290-b186-9258befede97 +[2018-09-25 19:39:29.196428881] P2P > VERSION 1 +[2018-09-25 19:39:29.197074793] P2P < GET 0 .heudiconv/emmet/ses-20180508/info/reproin.py SHA256E-s39141--59c2abfbe8184c851079d1cc0d8b87ddb38239a20d08002610ed81866aff7680.py[2018-09-25 19:39:29.196872107] P2P < VERSION 1 +[2018-09-25 19:39:29.197063591] P2P > GET 0 .heudiconv/emmet/ses-20180521/info/filegroup_ses-20180521.json SHA256E-s505696--ebc027b49ed92d8020c9f566239782519c9eb2b44945476fd534d3290feab14f.json +[2018-09-25 19:39:29.197414808] P2P < VERSION 1 +[2018-09-25 19:39:29.19757817] P2P > GET 0 .heudiconv/amit/ses-20180508/info/filegroup_ses-20180508.json SHA256E-s219789--d98a66036a9e3eb96308430cb3b4ac4e741a3cf13373e396ba353dbc46ddb3ec.json +[2018-09-25 19:39:29.19848677] P2P < GET 0 .heudiconv/emmet/ses-20180521/info/filegroup_ses-20180521.json SHA256E-s505696--ebc027b49ed92d8020c9f566239782519c9eb2b44945476fd534d3290feab14f.json[2018-09-25 19:39:29.198940575] P2P < GET 0 .heudiconv/amit/ses-20180508/info/filegroup_ses-20180508.json SHA256E-s219789--d98a66036a9e3eb96308430cb3b4ac4e741a3cf13373e396ba353dbc46ddb3ec.json[2018-09-25 19:39:29.285402553] P2P < AUTH-SUCCESS 6384a551-a41d-4290-b186-9258befede97 +[2018-09-25 19:39:29.285533748] P2P > VERSION 1 +[2018-09-25 19:39:29.286629667] P2P < VERSION 1 +[2018-09-25 19:39:29.286785793] P2P > GET 0 .heudiconv/emmet/ses-20180531/info/dicominfo_ses-20180531.tsv SHA256E-s2569--52f497d72d0ccd90f9405ec9ba26ff01d37257e618d9d836ebc56d60aa90413f.tsv +[2018-09-25 19:39:29.288268229] P2P < GET 0 .heudiconv/emmet/ses-20180531/info/dicominfo_ses-20180531.tsv SHA256E-s2569--52f497d72d0ccd90f9405ec9ba26ff01d37257e618d9d836ebc56d60aa90413f.tsv[2018-09-25 19:39:29.179261499] P2P > AUTH-SUCCESS 6384a551-a41d-4290-b186-9258befede97[2018-09-25 19:39:29.190593096] P2P > AUTH-SUCCESS 6384a551-a41d-4290-b186-9258befede97[2018-09-25 19:39:29.180612292] P2P < VERSION 1[2018-09-25 19:39:29.191640719] P2P < VERSION 1[2018-09-25 19:39:29.180837225] P2P > VERSION 1[2018-09-25 19:39:29.191763909] P2P > VERSION 1[2018-09-25 19:39:29.182014203] P2P < GET 0 .heudiconv/emmet/ses-20180521/info/emmet_ses-20180521.auto.txt SHA256E-s2658--42b972cbc99464913e8819e4944071a949914ec2cc7e786cb48fc26b41f665f1.auto.txt[2018-09-25 19:39:29.192947297] P2P < GET 0 .heudiconv/emmet/ses-20180521/info/emmet_ses-20180521.edit.txt SHA256E-s2658--42b972cbc99464913e8819e4944071a949914ec2cc7e786cb48fc26b41f665f1.edit.txt[2018-09-25 19:39:29.319628794] P2P < VALID +[2018-09-25 19:39:29.31975345] P2P > SUCCESS +[2018-09-25 19:39:29.321050683] P2P < SUCCESS{\"command\":\"get\",\"note\":\"from origin...\nchecksum...\",\"success\":true,\"key\":\"SHA256E-s2072--84342b6857db244be0bfa97d4289ee6206ba8e34bb11dc23262f609b6478e783.tsv\",\"file\":\".heudiconv/amit/ses-20180508/info/dicominfo_ses-20180508.tsv\"} +[2018-09-25 19:39:29.326554415] P2P > GET 0 .heudiconv/emmet/ses-20180531/info/emmet_ses-20180531.auto.txt SHA256E-s1317--7eca0e85c908fbf855e085d4514d025ab401c3673fe63f4af04e24ac5cf049fa.auto.txt + + +\"\"\"]] +and nothing more... + +indeed though there is a good number of files pointing to the same annex keys + +[[!format sh \"\"\" +(git)smaug:/tmp/QA[master]git +$> find -type l | while read f; do readlink $f | xargs basename || echo ERROR; done | sort | uniq -c | sort -n | nl | tail -n 20 + 1168 1 SHA256E-s9234655--441a4f30bdda1378ab9a55c682e075af8f6660ddf3c20b6bdd0ade5e788183c0.nii.gz + 1169 1 SHA256E-s9236756--b761702178efdb66a7165ac21433ea65cee844231d85ce2b3699dbf6f9f10439.nii.gz + 1170 1 SHA256E-s9251135--903bfe4fa818f8773a3617090c6545d9bbc45f7287cbb4727fa784365bb17ccc.nii.gz + 1171 1 SHA256E-s9266850--97c97de7fcb552893dfd48e78d98d8300bfa26a51d56ef112675b91df55f1aca.nii.gz + 1172 1 SHA256E-s93--ddf04c5c23b143330b4e4ea83dfbbdf07b48acb64f3cfd309147a5cc2c6487f8 + 1173 1 SHA256E-s9678246--6048a63eac04b294cd4ef06c217da1ec19b9733ece68991da0ab4200363dad1c.tgz + 1174 2 SHA256E-s11626616--d9b4f8bc420f09c2b43a0967f12421295fb500c0311b40d1cd2de0f7881d2f48.nii.gz + 1175 2 SHA256E-s43413657--dc9a866b5100af7129acac14d3fb31e3404767349dda2d7cbde42bcb2cd5fa45.nii.gz + 1176 2 SHA256E-s551--bc05c5596a6cf03ad7d302aab294dac7b45ecfcb96f7469085520660bf1c3e29.auto.txt + 1177 2 SHA256E-s551--bc05c5596a6cf03ad7d302aab294dac7b45ecfcb96f7469085520660bf1c3e29.edit.txt + 1178 4 SHA256E-s39141--59c2abfbe8184c851079d1cc0d8b87ddb38239a20d08002610ed81866aff7680.py + 1179 7 SHA256E-s736--bb53d9a3cf00df009e6a53d1960ec76796e46dc32dd92e5349a48c8c00d50df9.auto.txt + 1180 7 SHA256E-s736--bb53d9a3cf00df009e6a53d1960ec76796e46dc32dd92e5349a48c8c00d50df9.edit.txt + 1181 9 SHA256E-s685--fc402d82b86dcfb0b1736d176948e7b9d368af063f92fcdb730f068b5e30e04f.auto.txt + 1182 9 SHA256E-s685--fc402d82b86dcfb0b1736d176948e7b9d368af063f92fcdb730f068b5e30e04f.edit.txt + 1183 31 SHA256E-s598--492217a4a3191ce3d7848e60138c31315152fdf85d902fdbc98b6f642cd4b859.auto.txt + 1184 31 SHA256E-s598--492217a4a3191ce3d7848e60138c31315152fdf85d902fdbc98b6f642cd4b859.edit.txt + 1185 34 SHA256E-s857--15709030cff9f43cd0b8f7e91a184dfde283215a017704da01380202d2f55425.auto.txt + 1186 34 SHA256E-s857--15709030cff9f43cd0b8f7e91a184dfde283215a017704da01380202d2f55425.edit.txt + 1187 92 SHA256E-s39894--637a4b00403a5c88835c6d1ffb3f611d68b6377d95c5f7751e33539b598b18a3.py +1 13093.....................................:Tue 25 Sep 2018 08:04:56 PM EDT:. +(git)smaug:/tmp/QA[master]git +$> git annex info +repository mode: indirect +trusted repositories: 0 +semitrusted repositories: 6 + 00000000-0000-0000-0000-000000000001 -- web + 00000000-0000-0000-0000-000000000002 -- bittorrent + 6384a551-a41d-4290-b186-9258befede97 -- bids@rolando:/inbox/BIDS/dbic/QA [origin] + 7d9ed214-3e5f-4cc8-ac88-f397145b2d4c -- yoh@falkor:/srv/datasets.datalad.org/www/dbic/QA + ba8f2cea-f229-422c-82be-6580e5e07ed5 -- yoh@smaug:/mnt/datasets/datalad/crawl/dbic/QA + e57c285c-4145-4c57-9e9c-73c29a6dc115 -- yoh@smaug:/tmp/QA [here] +untrusted repositories: 0 +transfers in progress: none +available local disk space: 3.35 gigabytes (+1 megabyte reserved) +local annex keys: 4 +local annex size: 246.75 kilobytes +annexed files in working tree: 1439 +size of annexed files in working tree: 39.52 gigabytes +bloom filter size: 32 mebibytes (0% full) +backend usage: + SHA256E: 1439 +\"\"\"]] +so only 1187 unique keys for 1439 files. +"""]] diff --git a/doc/bugs/annex_get_-J_16_via_ssh_stalls_/comment_5_c31884e8d52324c0e04cdbbee42d4d35._comment b/doc/bugs/annex_get_-J_16_via_ssh_stalls_/comment_5_c31884e8d52324c0e04cdbbee42d4d35._comment new file mode 100644 index 0000000000..379e51b555 --- /dev/null +++ b/doc/bugs/annex_get_-J_16_via_ssh_stalls_/comment_5_c31884e8d52324c0e04cdbbee42d4d35._comment @@ -0,0 +1,43 @@ +[[!comment format=mdwn + username="yarikoptic" + avatar="http://cdn.libravatar.org/avatar/f11e9c84cb18d26a1748c33b48c924b4" + subject="comment 5" + date="2018-09-26T13:47:35Z" + content=""" +FWIW, some anecdotal evidences/observations: + +After upgrade to 6.20180913+git149-g23bd27773-1~ndall+1 (resolving that `get` issue over ssh with elderly annexes) I thought that I started to see that `get` does not stall in a fresh git clone of the repository +while `get` was still stalling in the git clone where I have tried to get (and stalled) with 6.20180913+git52-gdb1644808-1~ndall+1. Removing files under `.git/annex/tmp` seemed to make `get` on the next run behave better... But then I still started to observe stalling from time to time, although it felt that it started to happen **much** more rarely. + +Despite -J flag, and starting a large number of `git-annex-shell 'p2pstdio'`, transmission seems to happen one file at a time, nothing is really parallelized. I see a single file being downloaded reported by `datalad` (which processes --json --json-progress output) or just observing output of `git annex get -J16 .` (or even just -J2) having only a single % line running at the bottom. + +On one run I got a new beasty error reported: `hGetChar: illegal operation`: + +[[!format sh \"\"\" +$> git annex get -J16 . +get sourcedata/sub-qa/ses-20170807/func/sub-qa_ses-20170807_task-rest_acq-p2Xs4X35mm_bold.dicom.tgz (from origin...) (checksum...) ok +get sourcedata/sub-qa/ses-20170828/func/sub-qa_ses-20170828_task-rest_acq-p2Xs4X35mm_bold.dicom.tgz (from origin...) (checksum...) ok +get sourcedata/sub-qa/ses-20170905/func/sub-qa_ses-20170905_task-rest_acq-p2Xs4X35mm_bold.dicom.tgz (from origin...) + Lost connection (fd:167: hGetChar: illegal operation (handle is closed)) + + Unable to access these remotes: origin + + Try making some of these repositories available: + 6384a551-a41d-4290-b186-9258befede97 -- bids@rolando:/inbox/BIDS/dbic/QA [origin] + 7d9ed214-3e5f-4cc8-ac88-f397145b2d4c -- yoh@falkor:/srv/datasets.datalad.org/www/dbic/QA + ba8f2cea-f229-422c-82be-6580e5e07ed5 -- yoh@smaug:/mnt/datasets/datalad/crawl/dbic/QA +failed +get sourcedata/sub-qa/ses-20170905/dwi/sub-qa_ses-20170905_acq-DTIX30Xp2_dwi.dicom.tgz (from origin...) (checksum...) ok +get sourcedata/sub-qa/ses-20170911/dwi/sub-qa_ses-20170911_acq-DTIX30Xp2Xs4_dwi.dicom.tgz (from origin...) (checksum...) ok +.... +42% 34.86 MiB 52 MiB/s 0s +\"\"\"]] +may be it is somehow related. + +Raising -J to some obscene value (32) causes a collection of the `Unable to access these remotes: origin`, some include that `Lost connection (fd:323: hGetChar: illegal operation (handle is closed))`, some not. Someone could argue that it would be expected... + +Overall: +- I think that stalling situation did improve, although not entirely resolved. +- Not sure if annex is actually doing any parallel transfer over ssh + +"""]] diff --git a/doc/bugs/annex_get_-J_16_via_ssh_stalls_/comment_6_0707380cc043d9bbb84f0232eb1c1322._comment b/doc/bugs/annex_get_-J_16_via_ssh_stalls_/comment_6_0707380cc043d9bbb84f0232eb1c1322._comment new file mode 100644 index 0000000000..32945294fe --- /dev/null +++ b/doc/bugs/annex_get_-J_16_via_ssh_stalls_/comment_6_0707380cc043d9bbb84f0232eb1c1322._comment @@ -0,0 +1,105 @@ +[[!comment format=mdwn + username="yarikoptic" + avatar="http://cdn.libravatar.org/avatar/f11e9c84cb18d26a1748c33b48c924b4" + subject=""minimal" reproducible case" + date="2018-09-26T16:03:13Z" + content=""" +Finally I've boiled down to a small reproducible case for the stall with the bleeding edge 6.20180913+git149-g23bd27773-1~ndall+1 build: + +[[!format sh \"\"\" +(git)smaug:/mnt/btrfs/scrap/tmp/QA[master]git +$> git annex drop --fast sub-qa/ses-20161205/dwi/sub-qa_ses-20161205_acq-DTIX30Xp2Xs4_dwi.nii.gz sub-qa/ses-20161205/dwi/sub-qa_ses-20161205_acq-DTIX30Xp2_dwi.nii.gz sub-qa/ses-20161205/func/sub-qa_ses-20161205_task-rest_acq-p2Xs4X35mm_bold.nii.gz +drop sub-qa/ses-20161205/dwi/sub-qa_ses-20161205_acq-DTIX30Xp2Xs4_dwi.nii.gz (locking origin...) ok +drop sub-qa/ses-20161205/dwi/sub-qa_ses-20161205_acq-DTIX30Xp2_dwi.nii.gz (locking origin...) ok +drop sub-qa/ses-20161205/func/sub-qa_ses-20161205_task-rest_acq-p2Xs4X35mm_bold.nii.gz (locking origin...) ok +(recording state in git...) + +$> git annex get --debug -c remote.origin.annex-ssh-options='-o ControlMaster=auto -S /home/yoh/.cache/datalad/sockets/478606ac' --json --json-progress -J2 -- sub-qa/ses-20161205/dwi/sub-qa_ses-20161205_acq-DTIX30Xp2Xs4_dwi.nii.gz sub-qa/ses-20161205/dwi/sub-qa_ses-20161205_acq-DTIX30Xp2_dwi.nii.gz sub-qa/ses-20161205/func/sub-qa_ses-20161205_task-rest_acq-p2Xs4X35mm_bold.nii.gz +{\"byte-progress\":32752,\"action\":{\"command\":\"get\",\"note\":\"from origin...\",\"key\":\"SHA256E-s24797353--d1bb6541067f4ba3cdb6a698e2e6e7dbd483d8179007ebd2f2774d11972daf04.nii.gz\",\"file\":\"sub-qa/ses-20161205/dwi/sub-qa_ses-20161205_acq-DTIX30Xp2Xs4_dwi.nii.gz\"},\"total-size\":24797353,\"percent-progress\":\"0.13%\"} +{\"byte-progress\":4159504,\"action\":{\"command\":\"get\",\"note\":\"from origin...\",\"key\":\"SHA256E-s24797353--d1bb6541067f4ba3cdb6a698e2e6e7dbd483d8179007ebd2f2774d11972daf04.nii.gz\",\"file\":\"sub-qa/ses-20161205/dwi/sub-qa_ses-20161205_acq-DTIX30Xp2Xs4_dwi.nii.gz\"},\"total-size\":24797353,\"percent-progress\":\"16.77%\"} +{\"byte-progress\":8253504,\"action\":{\"command\":\"get\",\"note\":\"from origin...\",\"key\":\"SHA256E-s24797353--d1bb6541067f4ba3cdb6a698e2e6e7dbd483d8179007ebd2f2774d11972daf04.nii.gz\",\"file\":\"sub-qa/ses-20161205/dwi/sub-qa_ses-20161205_acq-DTIX30Xp2Xs4_dwi.nii.gz\"},\"total-size\":24797353,\"percent-progress\":\"33.28%\"} +{\"byte-progress\":12347504,\"action\":{\"command\":\"get\",\"note\":\"from origin...\",\"key\":\"SHA256E-s24797353--d1bb6541067f4ba3cdb6a698e2e6e7dbd483d8179007ebd2f2774d11972daf04.nii.gz\",\"file\":\"sub-qa/ses-20161205/dwi/sub-qa_ses-20161205_acq-DTIX30Xp2Xs4_dwi.nii.gz\"},\"total-size\":24797353,\"percent-progress\":\"49.79%\"} +{\"byte-progress\":16539760,\"action\":{\"command\":\"get\",\"note\":\"from origin...\",\"key\":\"SHA256E-s24797353--d1bb6541067f4ba3cdb6a698e2e6e7dbd483d8179007ebd2f2774d11972daf04.nii.gz\",\"file\":\"sub-qa/ses-20161205/dwi/sub-qa_ses-20161205_acq-DTIX30Xp2Xs4_dwi.nii.gz\"},\"total-size\":24797353,\"percent-progress\":\"66.7%\"} +{\"byte-progress\":20699264,\"action\":{\"command\":\"get\",\"note\":\"from origin...\",\"key\":\"SHA256E-s24797353--d1bb6541067f4ba3cdb6a698e2e6e7dbd483d8179007ebd2f2774d11972daf04.nii.gz\",\"file\":\"sub-qa/ses-20161205/dwi/sub-qa_ses-20161205_acq-DTIX30Xp2Xs4_dwi.nii.gz\"},\"total-size\":24797353,\"percent-progress\":\"83.47%\"} +{\"byte-progress\":24797353,\"action\":{\"command\":\"get\",\"note\":\"from origin...\",\"key\":\"SHA256E-s24797353--d1bb6541067f4ba3cdb6a698e2e6e7dbd483d8179007ebd2f2774d11972daf04.nii.gz\",\"file\":\"sub-qa/ses-20161205/dwi/sub-qa_ses-20161205_acq-DTIX30Xp2Xs4_dwi.nii.gz\"},\"total-size\":24797353,\"percent-progress\":\"100%\"} +{\"command\":\"get\",\"note\":\"from origin...\nchecksum...\",\"success\":true,\"key\":\"SHA256E-s24797353--d1bb6541067f4ba3cdb6a698e2e6e7dbd483d8179007ebd2f2774d11972daf04.nii.gz\",\"file\":\"sub-qa/ses-20161205/dwi/sub-qa_ses-20161205_acq-DTIX30Xp2Xs4_dwi.nii.gz\"} +{\"byte-progress\":32752,\"action\":{\"command\":\"get\",\"note\":\"from origin...\",\"key\":\"SHA256E-s33352666--36efd3512b1dd33192ec0d4e1a615c2a6eb36277fbe817827eb0be0a2c6934ea.nii.gz\",\"file\":\"sub-qa/ses-20161205/func/sub-qa_ses-20161205_task-rest_acq-p2Xs4X35mm_bold.nii.gz\"},\"total-size\":33352666,\"percent-progress\":\"0.1%\"} +{\"byte-progress\":4683536,\"action\":{\"command\":\"get\",\"note\":\"from origin...\",\"key\":\"SHA256E-s33352666--36efd3512b1dd33192ec0d4e1a615c2a6eb36277fbe817827eb0be0a2c6934ea.nii.gz\",\"file\":\"sub-qa/ses-20161205/func/sub-qa_ses-20161205_task-rest_acq-p2Xs4X35mm_bold.nii.gz\"},\"total-size\":33352666,\"percent-progress\":\"14.04%\"} +{\"byte-progress\":9694592,\"action\":{\"command\":\"get\",\"note\":\"from origin...\",\"key\":\"SHA256E-s33352666--36efd3512b1dd33192ec0d4e1a615c2a6eb36277fbe817827eb0be0a2c6934ea.nii.gz\",\"file\":\"sub-qa/ses-20161205/func/sub-qa_ses-20161205_task-rest_acq-p2Xs4X35mm_bold.nii.gz\"},\"total-size\":33352666,\"percent-progress\":\"29.07%\"} +{\"byte-progress\":11790720,\"action\":{\"command\":\"get\",\"note\":\"from origin...\",\"key\":\"SHA256E-s33352666--36efd3512b1dd33192ec0d4e1a615c2a6eb36277fbe817827eb0be0a2c6934ea.nii.gz\",\"file\":\"sub-qa/ses-20161205/func/sub-qa_ses-20161205_task-rest_acq-p2Xs4X35mm_bold.nii.gz\"},\"total-size\":33352666,\"percent-progress\":\"35.35%\"} +{\"byte-progress\":18373872,\"action\":{\"command\":\"get\",\"note\":\"from origin...\",\"key\":\"SHA256E-s33352666--36efd3512b1dd33192ec0d4e1a615c2a6eb36277fbe817827eb0be0a2c6934ea.nii.gz\",\"file\":\"sub-qa/ses-20161205/func/sub-qa_ses-20161205_task-rest_acq-p2Xs4X35mm_bold.nii.gz\"},\"total-size\":33352666,\"percent-progress\":\"55.09%\"} +{\"byte-progress\":23417680,\"action\":{\"command\":\"get\",\"note\":\"from origin...\",\"key\":\"SHA256E-s33352666--36efd3512b1dd33192ec0d4e1a615c2a6eb36277fbe817827eb0be0a2c6934ea.nii.gz\",\"file\":\"sub-qa/ses-20161205/func/sub-qa_ses-20161205_task-rest_acq-p2Xs4X35mm_bold.nii.gz\"},\"total-size\":33352666,\"percent-progress\":\"70.21%\"} +{\"byte-progress\":28461488,\"action\":{\"command\":\"get\",\"note\":\"from origin...\",\"key\":\"SHA256E-s33352666--36efd3512b1dd33192ec0d4e1a615c2a6eb36277fbe817827eb0be0a2c6934ea.nii.gz\",\"file\":\"sub-qa/ses-20161205/func/sub-qa_ses-20161205_task-rest_acq-p2Xs4X35mm_bold.nii.gz\"},\"total-size\":33352666,\"percent-progress\":\"85.33%\"} +{\"byte-progress\":33352666,\"action\":{\"command\":\"get\",\"note\":\"from origin...\",\"key\":\"SHA256E-s33352666--36efd3512b1dd33192ec0d4e1a615c2a6eb36277fbe817827eb0be0a2c6934ea.nii.gz\",\"file\":\"sub-qa/ses-20161205/func/sub-qa_ses-20161205_task-rest_acq-p2Xs4X35mm_bold.nii.gz\"},\"total-size\":33352666,\"percent-progress\":\"100%\"} +{\"command\":\"get\",\"note\":\"from origin...\nchecksum...\",\"success\":true,\"key\":\"SHA256E-s33352666--36efd3512b1dd33192ec0d4e1a615c2a6eb36277fbe817827eb0be0a2c6934ea.nii.gz\",\"file\":\"sub-qa/ses-20161205/func/sub-qa_ses-20161205_task-rest_acq-p2Xs4X35mm_bold.nii.gz\"} + + +\"\"\"]] + +So it gets stuck after having 2 out of 3 files transmitted. With -J8 (in above it is -J2) it stalls after the first one. Similarly stalls (there is more --debug output) if I do not reuse the ssh socket: +[[!format sh \"\"\" +$> git annex get --debug --json --json-progress -J2 -- sub-qa/ses-20161205/dwi/sub-qa_ses-20161205_acq-DTIX30Xp2Xs4_dwi.nii.gz sub-qa/ses-20161205/dwi/sub-qa_ses-20161205_acq-DTIX30Xp2_dwi.nii.gz sub-qa/ses-20161205/func/sub-qa_ses-20161205_task-rest_acq-p2Xs4X35mm_bold.nii.gz +[2018-09-26 11:59:52.123811672] read: git [\"--git-dir=.git\",\"--work-tree=.\",\"--literal-pathspecs\",\"ls-files\",\"--cached\",\"-z\",\"--\",\"sub-qa/ses-20161205/dwi/sub-qa_ses-20161205_acq-DTIX30Xp2Xs4_dwi.nii.gz\",\"sub-qa/ses-20161205/dwi/sub-qa_ses-20161205_acq-DTIX30Xp2_dwi.nii.gz\",\"sub-qa/ses-20161205/func/sub-qa_ses-20161205_task-rest_acq-p2Xs4X35mm_bold.nii.gz\"] +[2018-09-26 11:59:52.127664351] read: git [\"--git-dir=.git\",\"--work-tree=.\",\"--literal-pathspecs\",\"show-ref\",\"git-annex\"] +[2018-09-26 11:59:52.132089355] process done ExitSuccess +[2018-09-26 11:59:52.132238783] read: git [\"--git-dir=.git\",\"--work-tree=.\",\"--literal-pathspecs\",\"show-ref\",\"--hash\",\"refs/heads/git-annex\"] +[2018-09-26 11:59:52.135559006] process done ExitSuccess +[2018-09-26 11:59:52.135835934] read: git [\"--git-dir=.git\",\"--work-tree=.\",\"--literal-pathspecs\",\"log\",\"refs/heads/git-annex..eb4d1df5604157f25b3f2f272413e4a3fb59dc2d\",\"--pretty=%H\",\"-n1\"] +[2018-09-26 11:59:52.139675103] process done ExitSuccess +[2018-09-26 11:59:52.140319876] chat: git [\"--git-dir=.git\",\"--work-tree=.\",\"--literal-pathspecs\",\"cat-file\",\"--batch\"] +[2018-09-26 11:59:52.140817819] chat: git [\"--git-dir=.git\",\"--work-tree=.\",\"--literal-pathspecs\",\"cat-file\",\"--batch-check=%(objectname) %(objecttype) %(objectsize)\"] +[2018-09-26 11:59:52.146834989] chat: git [\"--git-dir=.git\",\"--work-tree=.\",\"--literal-pathspecs\",\"cat-file\",\"--batch\"] +[2018-09-26 11:59:52.146890494] chat: git [\"--git-dir=.git\",\"--work-tree=.\",\"--literal-pathspecs\",\"cat-file\",\"--batch\"] +[2018-09-26 11:59:52.147529819] chat: git [\"--git-dir=.git\",\"--work-tree=.\",\"--literal-pathspecs\",\"cat-file\",\"--batch-check=%(objectname) %(objecttype) %(objectsize)\"] +[2018-09-26 11:59:52.147813727] chat: git [\"--git-dir=.git\",\"--work-tree=.\",\"--literal-pathspecs\",\"cat-file\",\"--batch-check=%(objectname) %(objecttype) %(objectsize)\"] +[2018-09-26 11:59:52.154567936] read: ssh [\"-o\",\"BatchMode=true\",\"-S\",\".git/annex/ssh/bids@rolando.cns\",\"-o\",\"ControlMaster=auto\",\"-o\",\"ControlPersist=yes\",\"-n\",\"-T\",\"bids@rolando.cns\",\"true\"] +[2018-09-26 11:59:52.155552005] read: ssh [\"-o\",\"BatchMode=true\",\"-S\",\".git/annex/ssh/bids@rolando.cns\",\"-o\",\"ControlMaster=auto\",\"-o\",\"ControlPersist=yes\",\"-n\",\"-T\",\"bids@rolando.cns\",\"true\"] +[2018-09-26 11:59:53.680319247] process done ExitSuccess +[2018-09-26 11:59:53.680436536] process done ExitSuccess +[2018-09-26 11:59:53.680671127] chat: ssh [\"bids@rolando.cns\",\"-S\",\".git/annex/ssh/bids@rolando.cns\",\"-o\",\"ControlMaster=auto\",\"-o\",\"ControlPersist=yes\",\"-T\",\"git-annex-shell 'p2pstdio' '/inbox/BIDS/dbic/QA' '--debug' 'c807100d-3e57-4198-9a25-a9d434f493a9' --uuid 6384a551-a41d-4290-b186-9258befede97\"] +[2018-09-26 11:59:53.680730098] chat: ssh [\"bids@rolando.cns\",\"-S\",\".git/annex/ssh/bids@rolando.cns\",\"-o\",\"ControlMaster=auto\",\"-o\",\"ControlPersist=yes\",\"-T\",\"git-annex-shell 'p2pstdio' '/inbox/BIDS/dbic/QA' '--debug' 'c807100d-3e57-4198-9a25-a9d434f493a9' --uuid 6384a551-a41d-4290-b186-9258befede97\"] +[2018-09-26 11:59:55.15893547] P2P < AUTH-SUCCESS 6384a551-a41d-4290-b186-9258befede97 +[2018-09-26 11:59:55.15895813] P2P < AUTH-SUCCESS 6384a551-a41d-4290-b186-9258befede97 +[2018-09-26 11:59:55.159205411] P2P > VERSION 1 +[2018-09-26 11:59:55.159313796] P2P > VERSION 1 +[2018-09-26 11:59:55.160624999] P2P < VERSION 1 +[2018-09-26 11:59:55.160742428] P2P < VERSION 1 +[2018-09-26 11:59:55.162001851] P2P > VERSION 1[2018-09-26 11:59:55.160845243] P2P > GET 0 sub-qa/ses-20161205/dwi/sub-qa_ses-20161205_acq-DTIX30Xp2_dwi.nii.gz SHA256E-s26287753--02448aa9a07ef300dcd7b445d178c558bef8048dced539e0cf97a51250c937c8.nii.gz +[2018-09-26 11:59:55.160943268] P2P > GET 0 sub-qa/ses-20161205/dwi/sub-qa_ses-20161205_acq-DTIX30Xp2Xs4_dwi.nii.gz SHA256E-s24797353--d1bb6541067f4ba3cdb6a698e2e6e7dbd483d8179007ebd2f2774d11972daf04.nii.gz +[2018-09-26 11:59:55.163615313] P2P < GET 0 sub-qa/ses-20161205/dwi/sub-qa_ses-20161205_acq-DTIX30Xp2_dwi.nii.gz SHA256E-s26287753--02448aa9a07ef300dcd7b445d178c558bef8048dced539e0cf97a51250c937c8.nii.gz[2018-09-26 11:59:55.163667046] P2P < GET 0 sub-qa/ses-20161205/dwi/sub-qa_ses-20161205_acq-DTIX30Xp2Xs4_dwi.nii.gz SHA256E-s24797353--d1bb6541067f4ba3cdb6a698e2e6e7dbd483d8179007ebd2f2774d11972daf04.nii.gz[2018-09-26 11:59:55.196799694] P2P > DATA 24797353[2018-09-26 11:59:55.19553292] P2P < DATA 24797353 +{\"byte-progress\":32752,\"action\":{\"command\":\"get\",\"note\":\"from origin...\",\"key\":\"SHA256E-s24797353--d1bb6541067f4ba3cdb6a698e2e6e7dbd483d8179007ebd2f2774d11972daf04.nii.gz\",\"file\":\"sub-qa/ses-20161205/dwi/sub-qa_ses-20161205_acq-DTIX30Xp2Xs4_dwi.nii.gz\"},\"total-size\":24797353,\"percent-progress\":\"0.13%\"} +{\"byte-progress\":3930240,\"action\":{\"command\":\"get\",\"note\":\"from origin...\",\"key\":\"SHA256E-s24797353--d1bb6541067f4ba3cdb6a698e2e6e7dbd483d8179007ebd2f2774d11972daf04.nii.gz\",\"file\":\"sub-qa/ses-20161205/dwi/sub-qa_ses-20161205_acq-DTIX30Xp2Xs4_dwi.nii.gz\"},\"total-size\":24797353,\"percent-progress\":\"15.85%\"} +{\"byte-progress\":8351760,\"action\":{\"command\":\"get\",\"note\":\"from origin...\",\"key\":\"SHA256E-s24797353--d1bb6541067f4ba3cdb6a698e2e6e7dbd483d8179007ebd2f2774d11972daf04.nii.gz\",\"file\":\"sub-qa/ses-20161205/dwi/sub-qa_ses-20161205_acq-DTIX30Xp2Xs4_dwi.nii.gz\"},\"total-size\":24797353,\"percent-progress\":\"33.68%\"} +{\"byte-progress\":12675024,\"action\":{\"command\":\"get\",\"note\":\"from origin...\",\"key\":\"SHA256E-s24797353--d1bb6541067f4ba3cdb6a698e2e6e7dbd483d8179007ebd2f2774d11972daf04.nii.gz\",\"file\":\"sub-qa/ses-20161205/dwi/sub-qa_ses-20161205_acq-DTIX30Xp2Xs4_dwi.nii.gz\"},\"total-size\":24797353,\"percent-progress\":\"51.11%\"} +{\"byte-progress\":17293056,\"action\":{\"command\":\"get\",\"note\":\"from origin...\",\"key\":\"SHA256E-s24797353--d1bb6541067f4ba3cdb6a698e2e6e7dbd483d8179007ebd2f2774d11972daf04.nii.gz\",\"file\":\"sub-qa/ses-20161205/dwi/sub-qa_ses-20161205_acq-DTIX30Xp2Xs4_dwi.nii.gz\"},\"total-size\":24797353,\"percent-progress\":\"69.74%\"} +{\"byte-progress\":22009344,\"action\":{\"command\":\"get\",\"note\":\"from origin...\",\"key\":\"SHA256E-s24797353--d1bb6541067f4ba3cdb6a698e2e6e7dbd483d8179007ebd2f2774d11972daf04.nii.gz\",\"file\":\"sub-qa/ses-20161205/dwi/sub-qa_ses-20161205_acq-DTIX30Xp2Xs4_dwi.nii.gz\"},\"total-size\":24797353,\"percent-progress\":\"88.76%\"} +[2018-09-26 11:59:55.759916936] P2P > VALID{\"byte-progress\":24797353,\"action\":{\"command\":\"get\",\"note\":\"from origin...\",\"key\":\"SHA256E-s24797353--d1bb6541067f4ba3cdb6a698e2e6e7dbd483d8179007ebd2f2774d11972daf04.nii.gz\",\"file\":\"sub-qa/ses-20161205/dwi/sub-qa_ses-20161205_acq-DTIX30Xp2Xs4_dwi.nii.gz\"},\"total-size\":24797353,\"percent-progress\":\"100%\"} +[2018-09-26 11:59:55.759776216] P2P < VALID +[2018-09-26 11:59:55.759965583] P2P > SUCCESS +[2018-09-26 11:59:55.762270429] P2P < SUCCESS{\"command\":\"get\",\"note\":\"from origin...\nchecksum...\",\"success\":true,\"key\":\"SHA256E-s24797353--d1bb6541067f4ba3cdb6a698e2e6e7dbd483d8179007ebd2f2774d11972daf04.nii.gz\",\"file\":\"sub-qa/ses-20161205/dwi/sub-qa_ses-20161205_acq-DTIX30Xp2Xs4_dwi.nii.gz\"} +[2018-09-26 11:59:55.902055144] P2P > GET 0 sub-qa/ses-20161205/func/sub-qa_ses-20161205_task-rest_acq-p2Xs4X35mm_bold.nii.gz SHA256E-s33352666--36efd3512b1dd33192ec0d4e1a615c2a6eb36277fbe817827eb0be0a2c6934ea.nii.gz +[2018-09-26 11:59:55.904705166] P2P < GET 0 sub-qa/ses-20161205/func/sub-qa_ses-20161205_task-rest_acq-p2Xs4X35mm_bold.nii.gz SHA256E-s33352666--36efd3512b1dd33192ec0d4e1a615c2a6eb36277fbe817827eb0be0a2c6934ea.nii.gz[2018-09-26 11:59:55.93085693] P2P > DATA 33352666[2018-09-26 11:59:55.929729548] P2P < DATA 33352666 +{\"byte-progress\":32752,\"action\":{\"command\":\"get\",\"note\":\"from origin...\",\"key\":\"SHA256E-s33352666--36efd3512b1dd33192ec0d4e1a615c2a6eb36277fbe817827eb0be0a2c6934ea.nii.gz\",\"file\":\"sub-qa/ses-20161205/func/sub-qa_ses-20161205_task-rest_acq-p2Xs4X35mm_bold.nii.gz\"},\"total-size\":33352666,\"percent-progress\":\"0.1%\"} +{\"byte-progress\":5207568,\"action\":{\"command\":\"get\",\"note\":\"from origin...\",\"key\":\"SHA256E-s33352666--36efd3512b1dd33192ec0d4e1a615c2a6eb36277fbe817827eb0be0a2c6934ea.nii.gz\",\"file\":\"sub-qa/ses-20161205/func/sub-qa_ses-20161205_task-rest_acq-p2Xs4X35mm_bold.nii.gz\"},\"total-size\":33352666,\"percent-progress\":\"15.61%\"} +{\"byte-progress\":10382384,\"action\":{\"command\":\"get\",\"note\":\"from origin...\",\"key\":\"SHA256E-s33352666--36efd3512b1dd33192ec0d4e1a615c2a6eb36277fbe817827eb0be0a2c6934ea.nii.gz\",\"file\":\"sub-qa/ses-20161205/func/sub-qa_ses-20161205_task-rest_acq-p2Xs4X35mm_bold.nii.gz\"},\"total-size\":33352666,\"percent-progress\":\"31.13%\"} +{\"byte-progress\":15720960,\"action\":{\"command\":\"get\",\"note\":\"from origin...\",\"key\":\"SHA256E-s33352666--36efd3512b1dd33192ec0d4e1a615c2a6eb36277fbe817827eb0be0a2c6934ea.nii.gz\",\"file\":\"sub-qa/ses-20161205/func/sub-qa_ses-20161205_task-rest_acq-p2Xs4X35mm_bold.nii.gz\"},\"total-size\":33352666,\"percent-progress\":\"47.14%\"} +{\"byte-progress\":21157792,\"action\":{\"command\":\"get\",\"note\":\"from origin...\",\"key\":\"SHA256E-s33352666--36efd3512b1dd33192ec0d4e1a615c2a6eb36277fbe817827eb0be0a2c6934ea.nii.gz\",\"file\":\"sub-qa/ses-20161205/func/sub-qa_ses-20161205_task-rest_acq-p2Xs4X35mm_bold.nii.gz\"},\"total-size\":33352666,\"percent-progress\":\"63.44%\"} +{\"byte-progress\":26463616,\"action\":{\"command\":\"get\",\"note\":\"from origin...\",\"key\":\"SHA256E-s33352666--36efd3512b1dd33192ec0d4e1a615c2a6eb36277fbe817827eb0be0a2c6934ea.nii.gz\",\"file\":\"sub-qa/ses-20161205/func/sub-qa_ses-20161205_task-rest_acq-p2Xs4X35mm_bold.nii.gz\"},\"total-size\":33352666,\"percent-progress\":\"79.34%\"} +{\"byte-progress\":27511680,\"action\":{\"command\":\"get\",\"note\":\"from origin...\",\"key\":\"SHA256E-s33352666--36efd3512b1dd33192ec0d4e1a615c2a6eb36277fbe817827eb0be0a2c6934ea.nii.gz\",\"file\":\"sub-qa/ses-20161205/func/sub-qa_ses-20161205_task-rest_acq-p2Xs4X35mm_bold.nii.gz\"},\"total-size\":33352666,\"percent-progress\":\"82.49%\"} +[2018-09-26 11:59:56.729414189] P2P > VALID{\"byte-progress\":33352666,\"action\":{\"command\":\"get\",\"note\":\"from origin...\",\"key\":\"SHA256E-s33352666--36efd3512b1dd33192ec0d4e1a615c2a6eb36277fbe817827eb0be0a2c6934ea.nii.gz\",\"file\":\"sub-qa/ses-20161205/func/sub-qa_ses-20161205_task-rest_acq-p2Xs4X35mm_bold.nii.gz\"},\"total-size\":33352666,\"percent-progress\":\"100%\"} +[2018-09-26 11:59:56.729435631] P2P < VALID +[2018-09-26 11:59:56.729716652] P2P > SUCCESS +[2018-09-26 11:59:56.732037464] P2P < SUCCESS{\"command\":\"get\",\"note\":\"from origin...\nchecksum...\",\"success\":true,\"key\":\"SHA256E-s33352666--36efd3512b1dd33192ec0d4e1a615c2a6eb36277fbe817827eb0be0a2c6934ea.nii.gz\",\"file\":\"sub-qa/ses-20161205/func/sub-qa_ses-20161205_task-rest_acq-p2Xs4X35mm_bold.nii.gz\"} +\"\"\"]] + +all of those 3 files point to different keys +[[!format sh \"\"\" +$> ls -l sub-qa/ses-20161205/dwi/sub-qa_ses-20161205_acq-DTIX30Xp2Xs4_dwi.nii.gz sub-qa/ses-20161205/dwi/sub-qa_ses-20161205_acq-DTIX30Xp2_dwi.nii.gz sub-qa/ses-20161205/func/sub-qa_ses-20161205_task-rest_acq-p2Xs4X35mm_bold.nii.gz +lrwxrwxrwx 1 yoh yoh 215 Sep 26 09:25 sub-qa/ses-20161205/dwi/sub-qa_ses-20161205_acq-DTIX30Xp2Xs4_dwi.nii.gz -> ../../../.git/annex/objects/8p/qK/SHA256E-s24797353--d1bb6541067f4ba3cdb6a698e2e6e7dbd483d8179007ebd2f2774d11972daf04.nii.gz/SHA256E-s24797353--d1bb6541067f4ba3cdb6a698e2e6e7dbd483d8179007ebd2f2774d11972daf04.nii.gz +lrwxrwxrwx 1 yoh yoh 215 Sep 26 09:25 sub-qa/ses-20161205/dwi/sub-qa_ses-20161205_acq-DTIX30Xp2_dwi.nii.gz -> ../../../.git/annex/objects/gq/WX/SHA256E-s26287753--02448aa9a07ef300dcd7b445d178c558bef8048dced539e0cf97a51250c937c8.nii.gz/SHA256E-s26287753--02448aa9a07ef300dcd7b445d178c558bef8048dced539e0cf97a51250c937c8.nii.gz +lrwxrwxrwx 1 yoh yoh 215 Sep 26 09:25 sub-qa/ses-20161205/func/sub-qa_ses-20161205_task-rest_acq-p2Xs4X35mm_bold.nii.gz -> ../../../.git/annex/objects/8K/85/SHA256E-s33352666--36efd3512b1dd33192ec0d4e1a615c2a6eb36277fbe817827eb0be0a2c6934ea.nii.gz/SHA256E-s33352666--36efd3512b1dd33192ec0d4e1a615c2a6eb36277fbe817827eb0be0a2c6934ea.nii.gz +\"\"\"]] + +`.git/tmp/annex` is clean (I thought it could have mattered, seems not) +What other information could I provide? +"""]] diff --git a/doc/bugs/annex_ignores_pushurl_and_uses_only_url_upon___34__copy_--to__34__.mdwn b/doc/bugs/annex_ignores_pushurl_and_uses_only_url_upon___34__copy_--to__34__.mdwn new file mode 100644 index 0000000000..c7cd3addda --- /dev/null +++ b/doc/bugs/annex_ignores_pushurl_and_uses_only_url_upon___34__copy_--to__34__.mdwn @@ -0,0 +1,84 @@ +### Please describe the problem. + +Cannot copy content to the remote whenever its url is http and pushurl is ssh + + +### What version of git-annex are you using? On what operating system? + +6.20170101+gitg93d69b1-1~ndall+1 + +### Please provide any additional information below. + +Having a remote which has Read-only http url but ssh pushurl: + +[[!format sh """ + + +hopa:/tmp/tmp.uCP6SjbQYt/orig +$> cat .git/config +[core] + repositoryformatversion = 0 + filemode = true + bare = false + logallrefupdates = true +[annex] + uuid = 1c81ccbf-e814-44cb-b2a4-3aef93d159e4 + version = 5 + backends = MD5E +[remote "target1"] + url = http://localhost:8082/demoannex/.git + fetch = +refs/heads/*:refs/remotes/target1/* + pushurl = localhost:/home/yoh/.tmp/tmp.uCP6SjbQYt/public_html/demoannex + annex-ignore = false + annex-bare = false + annex-uuid = d3720112-3b83-450d-a30e-cda370bf5a13 + push = master + push = git-annex + + +"""]] + +copy seems to try to push via http url, not succeeding. Interestingly, without --fast option, it doesn't even give a meaningful error message -- just says that it fails to find the file at the remote end (here are both calls outputs): + +[[!format sh """ + +hopa:/tmp/tmp.uCP6SjbQYt/orig +$> git annex copy --to target1 --debug --json --fast +[2017-02-03 09:58:09.433515455] read: git ["--git-dir=.git","--work-tree=.","--literal-pathspecs","show-ref","git-annex"] +[2017-02-03 09:58:09.43918477] process done ExitSuccess +[2017-02-03 09:58:09.43929589] read: git ["--git-dir=.git","--work-tree=.","--literal-pathspecs","show-ref","--hash","refs/heads/git-annex"] +[2017-02-03 09:58:09.443217439] process done ExitSuccess +[2017-02-03 09:58:09.444030163] chat: git ["--git-dir=.git","--work-tree=.","--literal-pathspecs","cat-file","--batch"] +[2017-02-03 09:58:09.444468521] chat: git ["--git-dir=.git","--work-tree=.","--literal-pathspecs","cat-file","--batch-check=%(objectname) %(objecttype) %(objectsize)"] +[2017-02-03 09:58:09.449625506] read: git ["--git-dir=.git","--work-tree=.","--literal-pathspecs","ls-files","--cached","-z","--"] + copying to non-ssh repo not supported + copying to non-ssh repo not supported + This could have failed because --fast is enabled. +{"command":"copy","note":"to target1...","success":false,"key":"MD5E-s0--d41d8cd98f00b204e9800998ecf8427e","file":"probe"} +git-annex: copy: 1 failed + + +$> git annex copy --to target1 --debug --json +[2017-02-03 09:57:51.713147198] read: git ["--git-dir=.git","--work-tree=.","--literal-pathspecs","show-ref","git-annex"] +[2017-02-03 09:57:51.717451798] process done ExitSuccess +[2017-02-03 09:57:51.717690883] read: git ["--git-dir=.git","--work-tree=.","--literal-pathspecs","show-ref","--hash","refs/heads/git-annex"] +[2017-02-03 09:57:51.721647463] process done ExitSuccess +[2017-02-03 09:57:51.72243661] chat: git ["--git-dir=.git","--work-tree=.","--literal-pathspecs","cat-file","--batch"] +[2017-02-03 09:57:51.722822231] chat: git ["--git-dir=.git","--work-tree=.","--literal-pathspecs","cat-file","--batch-check=%(objectname) %(objecttype) %(objectsize)"] +[2017-02-03 09:57:51.726838506] read: git ["--git-dir=.git","--work-tree=.","--literal-pathspecs","ls-files","--cached","-z","--"] +{"command":"copy","note":"not found","success":false,"key":"MD5E-s0--d41d8cd98f00b204e9800998ecf8427e","file":"probe"} +git-annex: copy: 1 failed + + +"""]] + +So far we found the way to rectify it by providing value of pushurl as a remote..url value for that git annex call. But it would be more "kosher" if git annex itself respected pushurl in such cases. + +### Have you had any luck using git-annex before? (Sometimes we get tired of reading bug reports all day and a lil' positive end note does wonders) + +lots + +[[!meta author="yoh"]] + +> [[done]] because there is already an annexUrl config to meet this need. +> --[[Joey]] diff --git a/doc/bugs/annex_ignores_pushurl_and_uses_only_url_upon___34__copy_--to__34__/comment_1_92ea74beb2f8beb449769404a7f57a77._comment b/doc/bugs/annex_ignores_pushurl_and_uses_only_url_upon___34__copy_--to__34__/comment_1_92ea74beb2f8beb449769404a7f57a77._comment new file mode 100644 index 0000000000..55fe57947c --- /dev/null +++ b/doc/bugs/annex_ignores_pushurl_and_uses_only_url_upon___34__copy_--to__34__/comment_1_92ea74beb2f8beb449769404a7f57a77._comment @@ -0,0 +1,22 @@ +[[!comment format=mdwn + username="joey" + subject="""comment 1""" + date="2017-02-07T19:13:28Z" + content=""" +Thing is, pushUrl is documented in git-push as "The is used for +pushes only.". And while git-annex is sending data to the remote, +that's not really the same as a push. + +Following that logic, if git has a pullUrl, then git-annex should +use that for `git annex get`, since that's similar to a pull. +But then there are other operations git-annex does to remotes +that are similar to neither a push or a pull. And if git doesn't have a +pullUrl, then `git annex get` probably shouldn't use pushUrl, because +perhaps git will later get a pullUrl. + +That's why annexUrl was added in 2011. Which is just used for all annex +operations. + +So, I suggest you just set annexUrl at the same time you set pushUrl, +and be happy. ;) +"""]] diff --git a/doc/bugs/annex_metadata___40__not_--batch__39__ed__41___is_not_aware_of_files_added_via_addurls_--batch.mdwn b/doc/bugs/annex_metadata___40__not_--batch__39__ed__41___is_not_aware_of_files_added_via_addurls_--batch.mdwn new file mode 100644 index 0000000000..b4567e84a6 --- /dev/null +++ b/doc/bugs/annex_metadata___40__not_--batch__39__ed__41___is_not_aware_of_files_added_via_addurls_--batch.mdwn @@ -0,0 +1,35 @@ +### Please describe the problem. + +From our inspection, `annex metadata` call relies on `ls-files` to determine if the file is known to annex. But if there is an `addurl --batch` running which is adding those files, `git` is not immediately aware of them, thus `git annex metadata` returns "peacefully" without actually performing any operation on metadata. + +### What steps will reproduce the problem? + +this is an extract from what we are doing in datalad, so might be insufficient (let us know if so) + +1. run `git annex addurl --batch` and add a file +2. run `git annex metadata` to assign some metadata to the just addurl'ed file + + +### What version of git-annex are you using? On what operating system? + + +6.20180316+gitg308f3ecf6-1~ndall+1 + +### Please provide any additional information below. + +[[!format sh """ +$> git -c receive.autogc=0 -c gc.auto=0 annex --debug metadata --json -s verbal_iq+=116 -s performance_iq+=89 -s age+=16.77 -s handedness+=ambidextrous -s site_id+=pitt -s session_count+=1 -s dsm_iv_tr+=autism -s sex+=Male -s project+=abide_initiative -s full_iq+=103 -s MRI+=yes -s diagnosis+=autism -s participant_id+=50002 -s session_id+=1 -s species+=homo-sapiens -- sub-50002/ses-1/T1_rep-0.mgz [2018-03-27 11:25:15.953624157] read: git ["--git-dir=.git","--work-tree=.","--literal-pathspecs","ls-files","--cached","-z","--","sub-50002/ses-1/T1_rep-0.mgz"] +[2018-03-27 11:25:15.957761277] process done ExitSuccess + +# no error exit code +$> px | grep addurl.*batch +yoh 26767 0.0 0.0 17892 2184 pts/16 T 11:05 0:00 | \_ /usr/lib/git-annex.linux/exe/git --library-path /usr/lib/git-annex.linux//usr/lib/x86_64-linux-gnu/gconv:/usr/lib/git-annex.linux//usr/lib/x86_64-linux-gnu/audit:/usr/lib/git-annex.linux//etc/ld.so.conf.d:/usr/lib/git-annex.linux//lib64:/usr/lib/git-annex.linux//usr/lib/x86_64-linux-gnu:/usr/lib/git-annex.linux//lib/x86_64-linux-gnu: /usr/lib/git-annex.linux/shimmed/git/git -c receive.autogc=0 -c gc.auto=0 annex addurl --with-files --json --batch +yoh 26790 0.0 0.3 1074180296 63604 pts/16 Tl 11:05 0:00 | \_ /usr/lib/git-annex.linux/exe/git-annex --library-path /usr/lib/git-annex.linux//usr/lib/x86_64-linux-gnu/gconv:/usr/lib/git-annex.linux//usr/lib/x86_64-linux-gnu/audit:/usr/lib/git-annex.linux//etc/ld.so.conf.d:/usr/lib/git-annex.linux//lib64:/usr/lib/git-annex.linux//usr/lib/x86_64-linux-gnu:/usr/lib/git-annex.linux//lib/x86_64-linux-gnu: /usr/lib/git-annex.linux/shimmed/git-annex/git-annex addurl --with-files --json --batch + +"""]] + + +P.S. It might be a related observation that git-annex metadata does exit with non-0 exit code whenever it is ran on a non-existing file, but it exits with 0 exit code (but without performing any action) when ran on the existing but not known to git file. I wondered if there could be if not a change in behavior but a flag to make `annex metadata PATHs` exit with non-0 code if it didn't handle some path(s) from the provided. Then we could use it within our "set metadata" to guarantee that we do not omit any file for which we thought we would get metadata operation performed. + +[[!meta author=yoh]] + diff --git a/doc/bugs/annex_metadata___40__not_--batch__39__ed__41___is_not_aware_of_files_added_via_addurls_--batch/comment_1_4169fceac6ef95400ddc79652558dc13._comment b/doc/bugs/annex_metadata___40__not_--batch__39__ed__41___is_not_aware_of_files_added_via_addurls_--batch/comment_1_4169fceac6ef95400ddc79652558dc13._comment new file mode 100644 index 0000000000..f02fd8669e --- /dev/null +++ b/doc/bugs/annex_metadata___40__not_--batch__39__ed__41___is_not_aware_of_files_added_via_addurls_--batch/comment_1_4169fceac6ef95400ddc79652558dc13._comment @@ -0,0 +1,20 @@ +[[!comment format=mdwn + username="joey" + subject="""comment 1""" + date="2018-04-04T16:01:21Z" + content=""" +I guess you expected annex.queuesize to be set to 1. +However, that would mean every single file that addurl adds +needs the whole git index file to be rewritten so other commands can +immediately know about it. Which could be very slow, which is why that is +not the default. + +What you can do is use `git annex addurl --batch --json` and observe the +key that it reports it's added. Then pass that key into `git annex metadata --batch --json` +to add metadata to the key, which will work before the file ever gets +added to the git index, and much more efficiently than relying on the +index. + +(Pretty sure this has came up before and that I suggested the same thing +then.) +"""]] diff --git a/doc/bugs/annex_remotedaemon_100__37___cpu_hungry.mdwn b/doc/bugs/annex_remotedaemon_100__37___cpu_hungry.mdwn new file mode 100644 index 0000000000..6b20db9fda --- /dev/null +++ b/doc/bugs/annex_remotedaemon_100__37___cpu_hungry.mdwn @@ -0,0 +1,81 @@ +### Please describe the problem. + +Rebooted my laptop recently (after dunno how long of uptime) and annex assistant was shut off in previous uptime, but this time decided to leave it running. spotted that laptop is hot today to see that git-annex is busy. And I am not sure what it is really doing: top says + + 3087 yoh 30 10 520700 21084 14748 S 100.0 0.1 80:50.60 /usr/bin/git-annex remotedaemon + + +[[!format sh """ + +$> tail -20 /proc/3087/cwd/.git/annex/daemon.log + +Please make sure you have the correct access rights +and the repository exists. +ssh: connect to host vagus.cns.dartmouth.edu port 22: Connection timed out +fatal: Could not read from remote repository. + +Please make sure you have the correct access rights +and the repository exists. +ssh: connect to host vagus.cns.dartmouth.edu port 22: Connection timed out +fatal: Could not read from remote repository. + +Please make sure you have the correct access rights +and the repository exists. +[2015-03-28 20:27:53 EDT] main: Syncing with debandy_Vault +hostname: Name or service not known +hostname: Name or service not known +To ssh://yoh@git-annex-andy-yoh_.2Fmedia.2FVault.2Fannex/media/Vault/annex/ + 8c181af..a0b0bd6 git-annex -> synced/git-annex + 8352761..537d4a6 master -> synced/master + +"""]] + +in webapp I saw it struggling to connect to vagus which is offline, I turned it off, then reanabled syncing to above yoh@git-annex-andy-yoh_.2Fmedia.2FVault.2Fannex . I don't know if annex was busier or not before, but it is 100% busy now, but nothing seems to be done -- no traffic, no changing fd's for that annex process + +strace shows busy reading from fd 24: + +[[!format sh """ +[pid 3110] read(24, "", 8096) = 0 +[pid 3110] read(24, "", 8096) = 0 +[pid 3110] read(24, "", 8096) = 0 +[pid 3110] read(24, "", 8096) = 0 +"""]] + + + $> ls -l /proc/3087/fd/24 + lr-x------ 1 yoh yoh 64 Mar 28 20:31 /proc/3087/fd/24 -> pipe:[794930] + + +so what could it be doing? + +[[!format sh """ + 2807 yoh 20 0 693M 110M 36948 S 1.4 0.7 11:15.32 ├─ /usr/bin/git-annex assistant --startdelay=5s + 3359 yoh 39 19 20996 3412 3096 S 0.0 0.0 0:00.01 │ ├─ git --git-dir=.git --work-tree=. check-attr -z --stdin annex.backend annex.numcopies -- + 3140 yoh 20 0 21128 3504 3188 S 0.0 0.0 0:00.01 │ ├─ git --git-dir=.git --work-tree=. check-ignore -z --stdin --verbose --non-matching + 3117 yoh 20 0 693M 110M 36948 S 0.0 0.7 0:00.00 │ ├─ /usr/bin/git-annex assistant --startdelay=5s + 3108 yoh 20 0 693M 110M 36948 S 0.0 0.7 0:00.15 │ ├─ /usr/bin/git-annex assistant --startdelay=5s + 3087 yoh 30 10 508M 21084 14748 S 97.9 0.1 1h29:16 │ ├─ /usr/bin/git-annex remotedaemon +24361 yoh 30 10 508M 21084 14748 S 0.0 0.1 0:00.00 │ │ ├─ /usr/bin/git-annex remotedaemon +19799 yoh 30 10 0 0 0 Z 0.0 0.0 0:00.01 │ │ ├─ ssh +19440 yoh 30 10 508M 21084 14748 S 0.0 0.1 0:00.00 │ │ ├─ /usr/bin/git-annex remotedaemon + 3222 yoh 30 10 27368 5796 3456 S 0.0 0.0 0:00.00 │ │ ├─ git --git-dir=.git --work-tree=. cat-file --batch + 3110 yoh 30 10 508M 21084 14748 R 97.5 0.1 1h29:10 │ │ ├─ /usr/bin/git-annex remotedaemon + 3090 yoh 30 10 508M 21084 14748 S 0.0 0.1 0:00.98 │ │ ├─ /usr/bin/git-annex remotedaemon + 3089 yoh 30 10 508M 21084 14748 S 0.0 0.1 0:00.03 │ │ └─ /usr/bin/git-annex remotedaemon +"""]] + +how could I figure out what is doing??? meanwhile I have just sent STOP signal to 3087 + +### What version of git-annex are you using? On what operating system? + +5.20150327+git27-g6af24b6-1 + +### Please provide any additional information below. + +[[!format sh """ +# If you can, paste a complete transcript of the problem occurring here. +# If the problem is with the git-annex assistant, paste in .git/annex/daemon.log + + +# End of transcript or log. +"""]] diff --git a/doc/bugs/annex_sync___40__in_direct_mode_only__41___does_not_actually_update_local_branch__47__tree.mdwn b/doc/bugs/annex_sync___40__in_direct_mode_only__41___does_not_actually_update_local_branch__47__tree.mdwn new file mode 100644 index 0000000000..b768f030ac --- /dev/null +++ b/doc/bugs/annex_sync___40__in_direct_mode_only__41___does_not_actually_update_local_branch__47__tree.mdwn @@ -0,0 +1,116 @@ +### Please describe the problem. + +ref: [datalad#1713](https://github.com/datalad/datalad/issues/1713#issuecomment-368091458) so it seems to work when origin remote on http, but fails (as demoed below) for local paths (and ssh) + +### What steps will reproduce the problem? + +It works correctly in indirect mode (commands below are the same besides switching to direct mode in the clone, sorry for "sudo rm" but that was the fast non-datalad method to remove them ;) ) + +[[!format sh """ +*yoh@hopa:/tmp/test-clone> cd /tmp/; sudo rm -r /tmp/test-{clone,origin}; mkdir /tmp/test-origin; cd /tmp/test-origin; git init; git annex init; echo "1" > 1; git add 1; git commit -m 'added 1'; git clone /tmp/test-origin /tmp/test-clone; echo 2 > 2; git add 2; git commit -m 'added 2'; cd /tmp/test-clone; git annex sync --no-content --no-push --pull origin; git show +Initialized empty Git repository in /tmp/test-origin/.git/ +init ok +(recording state in git...) +[master (root-commit) 2fc9be3] added 1 + 1 file changed, 1 insertion(+) + create mode 100644 1 +Cloning into '/tmp/test-clone'... +done. +[master 1caaa64] added 2 + 1 file changed, 1 insertion(+) + create mode 100644 2 +(merging origin/git-annex into git-annex...) +commit (recording state in git...) + +On branch master +Your branch is up-to-date with 'origin/master'. +nothing to commit, working tree clean +ok +pull origin +remote: Counting objects: 3, done. +remote: Compressing objects: 100% (2/2), done. +remote: Total 3 (delta 0), reused 0 (delta 0) +Unpacking objects: 100% (3/3), done. +From /tmp/test-origin + 2fc9be3..1caaa64 master -> origin/master + +Updating 2fc9be3..1caaa64 +Fast-forward + 2 | 1 + + 1 file changed, 1 insertion(+) + create mode 100644 2 +ok +commit 1caaa64da380d82641356be46a5f6bc4b012a54d (HEAD -> master, origin/master, origin/HEAD, synced/master) +Author: Yaroslav Halchenko +Date: Fri Feb 23 14:16:43 2018 -0500 + + added 2 + +diff --git a/2 b/2 +new file mode 100644 +index 0000000..0cfbf08 +--- /dev/null ++++ b/2 +@@ -0,0 +1 @@ ++2 + +"""]] + +but in direct you will see that you are still on 1: + +[[!format sh """ +*yoh@hopa:/tmp/test-clone> cd /tmp/; sudo rm -r /tmp/test-{clone,origin}; mkdir /tmp/test-origin; cd /tmp/test-origin; git init; git annex init; echo "1" > 1; git add 1; git commit -m 'added 1'; git clone /tmp/test-origin /tmp/test-clone; echo 2 > 2; git add 2; git commit -m 'added 2'; cd /tmp/test-clone; git annex direct; git annex sync --no-content --no-push --pull origin; git show +Initialized empty Git repository in /tmp/test-origin/.git/ +init ok +(recording state in git...) +[master (root-commit) 06b75e1] added 1 + 1 file changed, 1 insertion(+) + create mode 100644 1 +Cloning into '/tmp/test-clone'... +done. +[master e147893] added 2 + 1 file changed, 1 insertion(+) + create mode 100644 2 +(merging origin/git-annex into git-annex...) +(recording state in git...) +commit +(recording state in git...) +On branch master +Your branch is up-to-date with 'origin/master'. +nothing to commit, working tree clean +ok +direct ok +commit (recording state in git...) +ok +pull origin +remote: Counting objects: 3, done. +remote: Compressing objects: 100% (2/2), done. +remote: Total 3 (delta 0), reused 0 (delta 0) +Unpacking objects: 100% (3/3), done. +From /tmp/test-origin + 06b75e1..e147893 master -> origin/master +ok +commit 06b75e18594f89d5af0bc2b083a90b02dd5e3f21 (HEAD -> annex/direct/master, synced/master, master) +Author: Yaroslav Halchenko +Date: Fri Feb 23 14:18:27 2018 -0500 + + added 1 + +diff --git a/1 b/1 +new file mode 100644 +index 0000000..d00491f +--- /dev/null ++++ b/1 +@@ -0,0 +1 @@ ++1 + +"""]] + + +### What version of git-annex are you using? On what operating system? + +6.20180206+gitg638032f3a-1~ndall+1 + +[[!meta author=yoh]] + +> [[fixed|done]] --[[Joey]] diff --git a/doc/bugs/annex_sync___40__in_direct_mode_only__41___does_not_actually_update_local_branch__47__tree/comment_1_6fb5a4fec435eccc0528744c55b7468c._comment b/doc/bugs/annex_sync___40__in_direct_mode_only__41___does_not_actually_update_local_branch__47__tree/comment_1_6fb5a4fec435eccc0528744c55b7468c._comment new file mode 100644 index 0000000000..04b6d62027 --- /dev/null +++ b/doc/bugs/annex_sync___40__in_direct_mode_only__41___does_not_actually_update_local_branch__47__tree/comment_1_6fb5a4fec435eccc0528744c55b7468c._comment @@ -0,0 +1,15 @@ +[[!comment format=mdwn + username="joey" + subject="""comment 1""" + date="2018-02-26T17:38:05Z" + content=""" +Seems that it's only merging from remotes/origin/synced/master, not +remotes/origin/master. When `git annex sync` is used in the first +repository, that works, but not when `git commit` is used. + +And it's due to being in direct mode because the annex/direct/master +branch is checked out, so it tries to merge from +remotes/origin/direct/master, which doesn't exist. + +This does not seem to be a reversion, but a long-ago oversight. +"""]] diff --git a/doc/bugs/annex_tries_to_start_old_binaray.mdwn b/doc/bugs/annex_tries_to_start_old_binaray.mdwn new file mode 100644 index 0000000000..76f9350015 --- /dev/null +++ b/doc/bugs/annex_tries_to_start_old_binaray.mdwn @@ -0,0 +1,45 @@ +### Please describe the problem. +See the logs. git-annex-shell tries to use not existing runshell + + +### What steps will reproduce the problem? +I am on Debian testing and have, some month ago, tried the tarball distribution. +I have returned to deb packages later and deleted the tarball installation. +Seems that there some traces left. + +I have tried to find the runshell configuration, but failed to do so. + +I have destroyed the repo completely, has not helped. + +### What version of git-annex are you using? On what operating system? +ii git-annex 5.20140831 amd64 + +### Please provide any additional information below. + +[[!format sh """ +# If you can, paste a complete transcript of the problem occurring here. +# If the problem is with the git-annex assistant, paste in .git/annex/daemon.log + +[2014-09-07 17:15:04 CEST] main: starting assistant version 5.20140831 +[2014-09-07 17:15:04 CEST] Cronner: Consistency check in progress +/home//.ssh/git-annex-shell: 4: exec: /home//git-annex.linux.5.20131213/runshell: not found +/home//.ssh/git-annex-shell: 4: exec: /home//git-annex.linux.5.20131213/runshell: not found +fatal: Could not read from remote repository. + +Please make sure you have the correct access rights +and the repository exists. +/home//.ssh/git-annex-shell: 4: exec: /home//git-annex.linux.5.20131213/runshell: not found +/home//.ssh/git-annex-shell: 4: exec: /home//git-annex.linux.5.20131213/runshell: not found +fatal: Could not read from remote repository. + +Please make sure you have the correct access rights +and the repository exists. +/home//.ssh/git-annex-shell: 4: exec: /home//git-annex.linux.5.20131213/runshell: not found +(scanning...) [2014-09-07 17:16:47 CEST] Watcher: Performing startup scan +/home//.ssh/git-annex-shell: 4: exec: /home//git-annex.linux.5.20131213/runshell: not found +/home//.ssh/git-annex-shell: 4: exec: /home//git-annex.linux.5.2013121/ +/home//.ssh/git-annex-shell: 4: exec: /home//git-annex.linux.5.20131213/runshell: not found +/home//.ssh/git-annex-shell: 4: exec: /home//git-annex.linux.5.20131213/runshell: not found + +# End of transcript or log. +"""]] diff --git a/doc/bugs/annex_tries_to_start_old_binaray/comment_1_209c36b737a48a6136eeb85d1f3d01b3._comment b/doc/bugs/annex_tries_to_start_old_binaray/comment_1_209c36b737a48a6136eeb85d1f3d01b3._comment new file mode 100644 index 0000000000..753b9a9980 --- /dev/null +++ b/doc/bugs/annex_tries_to_start_old_binaray/comment_1_209c36b737a48a6136eeb85d1f3d01b3._comment @@ -0,0 +1,10 @@ +[[!comment format=mdwn + username="http://joeyh.name/" + ip="209.250.56.132" + subject="comment 1" + date="2014-09-11T17:55:28Z" + content=""" +~/.ssh/git-annex-shell is a wrapper script that gets installed when you use the standalone build. You can delete it and your problem will be fixed. + +It would probably be good if the standalone build came with an uninstallation script. +"""]] diff --git a/doc/bugs/annex_unannex__47__uninit_should_handle_copies.mdwn b/doc/bugs/annex_unannex__47__uninit_should_handle_copies.mdwn new file mode 100644 index 0000000000..e830f11564 --- /dev/null +++ b/doc/bugs/annex_unannex__47__uninit_should_handle_copies.mdwn @@ -0,0 +1,20 @@ +Just starting using v3, even more awesome, thanks! + +With git-annex, I take the habit to do copies of files without restriction, as they end up into (cheap) symlink copies. +However, if 2 copies are unannexed, only one is restored, the other becomes a broken symlink, so I kind of loose some information +(my use case: I have a repo on which I recently started using annex, but most of the files, which i would want to be annexed, are only in git, +so my plan is to unninit this repo, delete the .git dir, and then annex everything, as I don't mind the history). + +Rafaël + +> The only way for git-annex to support this in its current state would be +> for the unannex command to copy the file content from the annex, rather +> than moving it out. Then multiple links to the same content could be +> unannexed. +> +> But, this would be slower, and would depend on a later `unused` and +> `dropunused` to actually remove the content. While doable, by use case +> for unannex is more to quickly undo a mistaken add, and it's unlikely there +> are multiple symlinks to the same content in this situation. --[[Joey]] + +[[!tag done]] diff --git a/doc/bugs/annex_unannex__47__uninit_should_handle_copies/comment_1_c896ff6589f62178b60e606771e4f2bf._comment b/doc/bugs/annex_unannex__47__uninit_should_handle_copies/comment_1_c896ff6589f62178b60e606771e4f2bf._comment new file mode 100644 index 0000000000..839992477a --- /dev/null +++ b/doc/bugs/annex_unannex__47__uninit_should_handle_copies/comment_1_c896ff6589f62178b60e606771e4f2bf._comment @@ -0,0 +1,10 @@ +[[!comment format=mdwn + username="https://www.google.com/accounts/o8/id?id=AItOawnpdM9F8VbtQ_H5PaPMpGSxPe_d5L1eJ6w" + nickname="Rafaël" + subject="comment 1" + date="2011-07-04T16:57:25Z" + content=""" +You convince me for unannex, but isn't the goal of uninit to revert all annex operations? In the current state, a clean revert is not possible (because of the broken symlinks after uninit). Instead of copying, using hard links is out of question? + +For my needs, is the command \"git annex unlock .\" (from the root of the repo) a correct workaround? +"""]] diff --git a/doc/bugs/annex_unannex__47__uninit_should_handle_copies/comment_2_9249609f83f8e9c7521cd2f007c1a39e._comment b/doc/bugs/annex_unannex__47__uninit_should_handle_copies/comment_2_9249609f83f8e9c7521cd2f007c1a39e._comment new file mode 100644 index 0000000000..21c0c449b0 --- /dev/null +++ b/doc/bugs/annex_unannex__47__uninit_should_handle_copies/comment_2_9249609f83f8e9c7521cd2f007c1a39e._comment @@ -0,0 +1,8 @@ +[[!comment format=mdwn + username="https://www.google.com/accounts/o8/id?id=AItOawmJfIszzreLNvCqzqzvTayA9_9L6gb9RtY" + nickname="Joey" + subject="comment 2" + date="2011-07-04T20:25:38Z" + content=""" +Indeed, uninit needed to be improved. I've done so. Also, unannex --fast can be used to make hard links to content left in the annex. +"""]] diff --git a/doc/bugs/annex_view_barfs__fatal__58___Unable_to_add___40__null__41___to_database.mdwn b/doc/bugs/annex_view_barfs__fatal__58___Unable_to_add___40__null__41___to_database.mdwn new file mode 100644 index 0000000000..2d9b7f39bd --- /dev/null +++ b/doc/bugs/annex_view_barfs__fatal__58___Unable_to_add___40__null__41___to_database.mdwn @@ -0,0 +1,35 @@ +### Please describe the problem. + + +[[!format sh """ +(git-annex)hopa:/tmp/1038_hyperface[master]git-annex +$> git annex view 'needed-by=*' +view (searching...) fatal: Unable to add (null) to database +fatal: Unable to add (null) to database +fatal: Unable to add (null) to database +fatal: Unable to add (null) to database +fatal: Unable to add (null) to database +fatal: Unable to add (null) to database +fatal: Unable to add (null) to database +fatal: Unable to add (null) to database +fatal: Unable to add (null) to database +fatal: Unable to add (null) to database +fatal: Unable to add (null) to database + +git-annex: fd:14: hGetLine: end of file +failed +git-annex: user error (git ["--git-dir=.git","--work-tree=.","--literal-pathspecs","hash-object","-w","--stdin-paths","--no-filters"] exited 128) + + +$> git annex version +git-annex version: 6.20180416+gitg86b18966f-1~ndall+1 + +"""]] + +copy of the repository will be provided via email + +[[!meta author=yoh]] + + + +> [[fixed|done]] --[[Joey]] diff --git a/doc/bugs/annex_view_barfs__fatal__58___Unable_to_add___40__null__41___to_database/comment_1_9c91eef41c51737f85d4df3c737fc6c7._comment b/doc/bugs/annex_view_barfs__fatal__58___Unable_to_add___40__null__41___to_database/comment_1_9c91eef41c51737f85d4df3c737fc6c7._comment new file mode 100644 index 0000000000..6436841e82 --- /dev/null +++ b/doc/bugs/annex_view_barfs__fatal__58___Unable_to_add___40__null__41___to_database/comment_1_9c91eef41c51737f85d4df3c737fc6c7._comment @@ -0,0 +1,14 @@ +[[!comment format=mdwn + username="joey" + subject="""comment 1""" + date="2018-05-08T17:56:54Z" + content=""" +I have not received the database by email yet. + +Note that this is a sqlite error message. + +I wonder if this is a symptom of the same bug as + +since it popped up at the same time, and that also involved a NULL +dereference. +"""]] diff --git a/doc/bugs/annex_view_barfs__fatal__58___Unable_to_add___40__null__41___to_database/comment_2_f6759af83ddd62293593a276e089188d._comment b/doc/bugs/annex_view_barfs__fatal__58___Unable_to_add___40__null__41___to_database/comment_2_f6759af83ddd62293593a276e089188d._comment new file mode 100644 index 0000000000..98ac9d8901 --- /dev/null +++ b/doc/bugs/annex_view_barfs__fatal__58___Unable_to_add___40__null__41___to_database/comment_2_f6759af83ddd62293593a276e089188d._comment @@ -0,0 +1,8 @@ +[[!comment format=mdwn + username="yarikoptic" + avatar="http://cdn.libravatar.org/avatar/f11e9c84cb18d26a1748c33b48c924b4" + subject="details" + date="2018-05-09T01:12:30Z" + content=""" +not sure what happened with the email (was sent on 3rd of may to id@joeyh.name, I have resent it again, both times encrypted). Also now uploaded to http://www.onerussian.com/tmp/1038_hyperface.tgz.gpg encrypted with your key. Let me know when you get it - I will remove it +"""]] diff --git a/doc/bugs/annex_view_barfs__fatal__58___Unable_to_add___40__null__41___to_database/comment_3_798629780fa04dc1a8bf1c21ca813d40._comment b/doc/bugs/annex_view_barfs__fatal__58___Unable_to_add___40__null__41___to_database/comment_3_798629780fa04dc1a8bf1c21ca813d40._comment new file mode 100644 index 0000000000..cab739d835 --- /dev/null +++ b/doc/bugs/annex_view_barfs__fatal__58___Unable_to_add___40__null__41___to_database/comment_3_798629780fa04dc1a8bf1c21ca813d40._comment @@ -0,0 +1,11 @@ +[[!comment format=mdwn + username="joey" + subject="""comment 3""" + date="2018-05-14T17:14:17Z" + content=""" +I have the file. I've reproduced the crash using that repo and the most +recent git-annex. + +I don't think sqlite is involved after all; seems that git hash-object +is what's crashing on a null pointer. +"""]] diff --git a/doc/bugs/annex_view_barfs__fatal__58___Unable_to_add___40__null__41___to_database/comment_4_b7c2abf63c9f350402b3176e028bda03._comment b/doc/bugs/annex_view_barfs__fatal__58___Unable_to_add___40__null__41___to_database/comment_4_b7c2abf63c9f350402b3176e028bda03._comment new file mode 100644 index 0000000000..c991bd3b6a --- /dev/null +++ b/doc/bugs/annex_view_barfs__fatal__58___Unable_to_add___40__null__41___to_database/comment_4_b7c2abf63c9f350402b3176e028bda03._comment @@ -0,0 +1,21 @@ +[[!comment format=mdwn + username="joey" + subject="""comment 4""" + date="2018-05-14T17:30:25Z" + content=""" +It's caused by git submodules with names starting with dots. Not at all +related to the recent null pointer dereference bug in git-annex. + +The .heudiconv/ directory is a submodule and git ls-files lists it; the view +generation code assumes it's a dotfile and asks git to hash it. + + joey@darkstar:~/tmp/1038_hyperface.x>git hash-object -w --stdin-paths --no-filters + .heudiconv + fatal: Unable to add (null) to database + +Views only include dotfiles to make .gitattributes etc work, and they don't +include submodules normally, but they do include dot-dirs too; indeed I see +this repository uses that with a .datalad/ that gets included in the view. +Perhaps .heudiconv/ is a similar dot-dir. Seems that such submodule +dot-dirs ought to be included in the view. +"""]] diff --git a/doc/bugs/apparent_regression_in_git_annex_p2p_--pair_usage_of_magic_wormhole_invocation.mdwn b/doc/bugs/apparent_regression_in_git_annex_p2p_--pair_usage_of_magic_wormhole_invocation.mdwn new file mode 100644 index 0000000000..7f5b2c9315 --- /dev/null +++ b/doc/bugs/apparent_regression_in_git_annex_p2p_--pair_usage_of_magic_wormhole_invocation.mdwn @@ -0,0 +1,74 @@ +### Please describe the problem. + +git annex p2p --pair does not successfully complete pairing + +### What steps will reproduce the problem? + +- run `git annex p2p --pair` +- optionally run `wormhole receive` on other host, and input pairing code + +### What version of git-annex are you using? On what operating system? + +6.20170101.1 on Debian 9/stretch + +magic wormhole version 0.9.1 + +### Please provide any additional information below. + +[[!format sh """ +git annex init +sudo git annex enable-tor $( id -u ) +git annex p2p --gen-addresses +git annex p2p --debug --pair +"""]] + +[[!format txt """ +p2p pair peer1 (using Magic Wormhole) +[2018-07-04 19:45:28.712958619] chat: wormhole ["receive","--accept-file","--output-file","/tmp/pairedEj5Y/recv"] +[2018-07-04 19:45:28.715141955] chat: wormhole ["send","/tmp/pairedEj5Y/send"] +Sending 105 Bytes file named 'send' +On the other computer, please run: wormhole receive +Wormhole code is: 3-quantity-fracture +"""]] + +Running the "wormhole receive" command gives a local file: +[[!format txt """ +user@chat:/tmp/bar$ wormhole receive +Enter receive wormhole code: 3-quantity-fracture +Receiving file (105 Bytes) into: send +ok? (y/n): y +Receiving (->tcp:10.137.6.42:36089).. +100%|| 105/105 [00:00<00:00, 519B/s] +Received file written to send +user@chat:/tmp/bar$ cat send +XXX +tor-annex::XXX.onion:18055 +"""]] + +and the `p2p --pair` command persists, but does not appear to be accepting input at any stage. + +[[!format txt """ +Sending (<-10.137.6.42:48778).. +100%|| 105/105 [00:00<00:00, 198KB/s] +File sent.. waiting for confirmation +Confirmation received. Transfer complete. +"""]] + +at this point `p2p --pair` seems to hang, but will exit on sigint w/ following error, and there's no need to press enter despite what's printed, not sure if that's annex or wormhole's output): + +[[!format txt """ +^C +Command interrupted: please press Return to quit +"""]] + +I do not see the expected message `Enter receive wormhole code:` message, which I do see when manually re-invoking `wormhole receive --accept-file --output-file /tmp/pairedEj5Y/recv` (in accordance with what `Wormhole.receiveFile` does), so I suspect some version incompatibility relating either to line buffering or pty mode assumed by magic wormhole itself, causing the `p2p --pair` command to wait indefinitely even though the `wormhole receive` process is not able to receive input. + +### Have you had any luck using git-annex before? (Sometimes we get tired of reading bug reports all day and a lil' positive end note does wonders) + +Yes, but not `p2p --pair`. + +`p2p --link` works, including syncing, no problems with the remote daemon itself. + +... and I wouldn't call it luck, i both trust and rely on git annex =) + +> [[fixed|done]] diff --git a/doc/bugs/apparent_regression_in_git_annex_p2p_--pair_usage_of_magic_wormhole_invocation/comment_1_d403b087693d766b8f902c27964268e2._comment b/doc/bugs/apparent_regression_in_git_annex_p2p_--pair_usage_of_magic_wormhole_invocation/comment_1_d403b087693d766b8f902c27964268e2._comment new file mode 100644 index 0000000000..e4efcb5b53 --- /dev/null +++ b/doc/bugs/apparent_regression_in_git_annex_p2p_--pair_usage_of_magic_wormhole_invocation/comment_1_d403b087693d766b8f902c27964268e2._comment @@ -0,0 +1,22 @@ +[[!comment format=mdwn + username="joey" + subject="""comment 1""" + date="2018-07-05T15:28:15Z" + content=""" +Thanks much for reporting this bug. It turns out that shortly after I +first implemented this, magic-wormhole switched to outputting that text on +stderr, which defeated git-annex's attempt to extract the code phrase. + +[[!commit 3dd7f450c1968f8bc2b9cc73c701b1f12f7e326a]] fixes the bug. + +Really unfortunate this feature has been broken since almost the beginning. +Especially because pairing with a friend will fail if they have an older +version of git-annex. Added a note about this to the documentation. + +Clearly some better testing of this is needed. Testing network stuff like +this is outside the bounds of git-annex's usual test suite, and perhaps it +needs an additional test suite that is allowed to make network connections. +(testremote is allowed to, but needs the remote to be manually set up, +so this is also outside its scope). Opened [[todo/network_test_suite]] +about this. +"""]] diff --git a/doc/bugs/aria2c_display_broken_in_git-annex.mdwn b/doc/bugs/aria2c_display_broken_in_git-annex.mdwn new file mode 100644 index 0000000000..c08fdeba0b --- /dev/null +++ b/doc/bugs/aria2c_display_broken_in_git-annex.mdwn @@ -0,0 +1,147 @@ +[[!meta title="aria2c output very verbose (non-console mode)"]] +[[!tag confirmed]] + +### Please describe the problem. + +Instead of displaying what are probably carriage returns, git-annex strips those out and outputs a *lot* of lines when downloading stuff through aria2c. On small downloads, it's not a big deal, but on large downloads it can flood the screen pretty badly. + +### What steps will reproduce the problem? + +Just download a torrent with git-annex. + +### What version of git-annex are you using? On what operating system? + +`5.20150205-gbf9058a` on current debian jessie. + +### Please provide any additional information below. + +[[!format sh """ +# If you can, paste a complete transcript of the problem occurring here. +# If the problem is with the git-annex assistant, paste in .git/annex/daemon.log +anarcat@marcos:iso(master +)$ /srv/downloads.kitenet.net/git-annex/linux/current/git-annex.linux/git-annex addurl http://images.kali.org/kali-linux-1.1.0-amd64.torrent +(downloading torrent file...) +--2015-02-09 22:12:51-- http://images.kali.org/kali-linux-1.1.0-amd64.torrent +Résolution de images.kali.org (images.kali.org)… 50.7.37.130 +Connexion à images.kali.org (images.kali.org)|50.7.37.130|:80… connecté. +requête HTTP transmise, en attente de la réponse… 200 OK +Taille : 233152 (228K) [application/octet-stream] +Sauvegarde en : « ../.git/annex/misctmp/torrent32659 » + +100%[=====================================================================================================================================================================================================>] 233 152 610KB/s ds 0,4s + +2015-02-09 22:12:52 (610 KB/s) — « ../.git/annex/misctmp/torrent32659 » sauvegardé [233152/233152] + +addurl images.kali.org_kali_linux_1.1.0_amd64.torrent/kali_linux_1.1.0_amd64.iso (from bittorrent) + + +02/09 22:12:52 [NOTICE] IPv4 DHT: listening on UDP port 6940 + +02/09 22:12:52 [ERROR] Erreur d'intégrité détectée. fichier=../.git/annex/misctmp/URL--http&c%%images.kali.org%kali-linux-1.1.0-amd64.torrent/kali-linux-1.1.0-amd64 + +02/09 22:12:52 [NOTICE] IPv4 BitTorrent: listening on TCP port 6923 + +02/09 22:12:52 [NOTICE] IPv6 BitTorrent: listening on TCP port 6923 +[#066a28 0B/2.8GiB(0%) CN:0 SD:0 DL:0B] +[#066a28 0B/2.8GiB(0%) CN:44 SD:0 DL:0B] +[#066a28 0B/2.8GiB(0%) CN:44 SD:0 DL:0B] +[#066a28 0B/2.8GiB(0%) CN:44 SD:1 DL:0B] +[#066a28 0B/2.8GiB(0%) CN:44 SD:2 DL:0B] +[#066a28 16KiB/2.8GiB(0%) CN:44 SD:2 DL:11KiB ETA:72h44m54s] +[#066a28 32KiB/2.8GiB(0%) CN:44 SD:4 DL:11KiB ETA:72h55m47s] +[#066a28 48KiB/2.8GiB(0%) CN:44 SD:5 DL:10KiB ETA:79h7m17s] +[#066a28 64KiB/2.8GiB(0%) CN:49 SD:6 DL:10KiB ETA:77h27m] +[#066a28 96KiB/2.8GiB(0%) CN:47 SD:6 DL:13KiB ETA:60h20m41s] +[#066a28 160KiB/2.8GiB(0%) CN:45 SD:6 DL:20KiB ETA:41h20m43s] +[#066a28 304KiB/2.8GiB(0%) CN:45 SD:6 DL:33KiB ETA:24h28m29s] +[#066a28 384KiB/2.8GiB(0%) CN:45 SD:6 DL:38KiB ETA:21h34m21s] +[#066a28 416KiB/2.8GiB(0%) CN:44 SD:6 DL:37KiB ETA:21h54m17s] +[#066a28 512KiB/2.8GiB(0%) CN:44 SD:6 DL:42KiB ETA:19h26m19s] +[#066a28 544KiB/2.8GiB(0%) CN:44 SD:6 DL:41KiB ETA:19h48m32s] +[#066a28 624KiB/2.8GiB(0%) CN:44 SD:6 DL:44KiB ETA:18h38m8s] +[#066a28 720KiB/2.8GiB(0%) CN:44 SD:6 DL:52KiB ETA:15h42m33s] +[#066a28 736KiB/2.8GiB(0%) CN:44 SD:6 DL:50KiB ETA:16h30m42s] +[#066a28 768KiB/2.8GiB(0%) CN:44 SD:7 DL:52KiB ETA:15h42m44s] +[#066a28 896KiB/2.8GiB(0%) CN:44 SD:7 DL:57KiB ETA:14h20m4s] +[#066a28 1.0MiB/2.8GiB(0%) CN:44 SD:7 DL:69KiB ETA:11h58m54s] +[#066a28 1.1MiB/2.8GiB(0%) CN:44 SD:7 DL:79KiB ETA:10h21m22s] +[#066a28 1.2MiB/2.8GiB(0%) CN:44 SD:7 DL:87KiB ETA:9h24m15s] +[#066a28 1.4MiB/2.8GiB(0%) CN:44 SD:6 DL:90KiB ETA:9h8m26s] +[#066a28 1.4MiB/2.8GiB(0%) CN:44 SD:6 DL:88KiB ETA:9h23m40s] +[#066a28 1.5MiB/2.8GiB(0%) CN:44 SD:6 DL:85KiB ETA:9h41m21s] +[#066a28 1.6MiB/2.8GiB(0%) CN:44 SD:6 DL:89KiB ETA:9h12m17s] +[#066a28 1.6MiB/2.8GiB(0%) CN:44 SD:6 DL:86KiB ETA:9h32m9s] +[#066a28 1.7MiB/2.8GiB(0%) CN:44 SD:6 DL:85KiB UL:451KiB(288KiB) ETA:9h37m58s] +[#066a28 1.8MiB/2.8GiB(0%) CN:44 SD:6 DL:89KiB UL:168KiB(288KiB) ETA:9h16m53s] +[#066a28 1.8MiB/2.8GiB(0%) CN:44 SD:6 DL:84KiB UL:105KiB(288KiB) ETA:9h47m6s] +[#066a28 1.8MiB/2.8GiB(0%) CN:44 SD:6 DL:89KiB UL:76KiB(288KiB) ETA:9h16m44s] +[#066a28 1.9MiB/2.8GiB(0%) CN:44 SD:7 DL:85KiB UL:59KiB(288KiB) ETA:9h36m53s] +[#066a28 1.9MiB/2.8GiB(0%) CN:44 SD:7 DL:79KiB UL:49KiB(288KiB) ETA:10h24m56s] +[#066a28 2.1MiB/2.8GiB(0%) CN:44 SD:7 DL:80KiB UL:42KiB(288KiB) ETA:10h15m19s] +[#066a28 2.2MiB/2.8GiB(0%) CN:44 SD:7 DL:75KiB UL:68KiB(544KiB) ETA:10h56m] +[#066a28 2.3MiB/2.8GiB(0%) CN:44 SD:7 DL:75KiB UL:60KiB(544KiB) ETA:10h54m12s] +[#066a28 2.4MiB/2.8GiB(0%) CN:44 SD:7 DL:70KiB UL:54KiB(544KiB) ETA:11h39m51s] +[#066a28 2.4MiB/2.8GiB(0%) CN:44 SD:7 DL:71KiB UL:49KiB(544KiB) ETA:11h29m45s] +[#066a28 2.6MiB/2.8GiB(0%) CN:44 SD:7 DL:74KiB UL:44KiB(544KiB) ETA:11h1m53s] +[#066a28 2.6MiB/2.8GiB(0%) CN:44 SD:7 DL:73KiB UL:41KiB(544KiB) ETA:11h14m27s] +[#066a28 2.7MiB/2.8GiB(0%) CN:44 SD:7 DL:76KiB UL:38KiB(544KiB) ETA:10h52m4s] +[#066a28 2.7MiB/2.8GiB(0%) CN:44 SD:7 DL:73KiB UL:19KiB(544KiB) ETA:11h18m54s] +[#066a28 2.8MiB/2.8GiB(0%) CN:44 SD:7 DL:78KiB UL:27KiB(544KiB) ETA:10h31m16s] +[#066a28 2.9MiB/2.8GiB(0%) CN:44 SD:7 DL:74KiB UL:25KiB(544KiB) ETA:11h7m40s] +[#066a28 2.9MiB/2.8GiB(0%) CN:44 SD:7 DL:72KiB UL:22KiB(544KiB) ETA:11h22m19s] +[#066a28 3.0MiB/2.8GiB(0%) CN:44 SD:7 DL:77KiB UL:41KiB(800KiB) ETA:10h42m53s] +[#066a28 3.0MiB/2.8GiB(0%) CN:44 SD:7 DL:79KiB UL:38KiB(800KiB) ETA:10h24m56s] +[#066a28 3.2MiB/2.8GiB(0%) CN:44 SD:7 DL:82KiB UL:35KiB(800KiB) ETA:10h4m14s] +[#066a28 3.2MiB/2.8GiB(0%) CN:44 SD:7 DL:76KiB UL:72KiB(800KiB) ETA:10h44m14s] +[#066a28 3.2MiB/2.8GiB(0%) CN:44 SD:7 DL:69KiB UL:113KiB(1.0MiB) ETA:11h48m20s] +[#066a28 3.3MiB/2.8GiB(0%) CN:44 SD:8 DL:69KiB UL:92KiB(1.0MiB) ETA:11h54m37s] +[#066a28 3.4MiB/2.8GiB(0%) CN:44 SD:8 DL:73KiB UL:78KiB(1.0MiB) ETA:11h17m38s] +[#066a28 3.6MiB/2.8GiB(0%) CN:44 SD:8 DL:78KiB UL:67KiB(1.0MiB) ETA:10h27m51s] +[#066a28 3.7MiB/2.8GiB(0%) CN:44 SD:9 DL:76KiB UL:59KiB(1.0MiB) ETA:10h46m10s] +[#066a28 3.7MiB/2.8GiB(0%) CN:44 SD:9 DL:72KiB UL:53KiB(1.0MiB) ETA:11h21m24s] +[#066a28 3.8MiB/2.8GiB(0%) CN:44 SD:10 DL:75KiB UL:72KiB(1.2MiB) ETA:10h55m4s] +[#066a28 3.9MiB/2.8GiB(0%) CN:44 SD:10 DL:74KiB UL:66KiB(1.2MiB) ETA:11h9m6s] +[#066a28 4.0MiB/2.8GiB(0%) CN:44 SD:10 DL:80KiB UL:62KiB(1.3MiB) ETA:10h12m8s] +[#066a28 4.1MiB/2.8GiB(0%) CN:44 SD:10 DL:84KiB UL:58KiB(1.3MiB) ETA:9h45m9s] +[#066a28 4.2MiB/2.8GiB(0%) CN:44 SD:9 DL:88KiB UL:54KiB(1.3MiB) ETA:9h17m20s] +[#066a28 4.2MiB/2.8GiB(0%) CN:44 SD:9 DL:83KiB UL:46KiB(1.3MiB) ETA:9h53m47s] +[#066a28 4.3MiB/2.8GiB(0%) CN:44 SD:8 DL:75KiB UL:42KiB(1.3MiB) ETA:10h55m25s] +[#066a28 4.4MiB/2.8GiB(0%) CN:44 SD:8 DL:82KiB UL:39KiB(1.3MiB) ETA:10h3m52s] +[#066a28 4.4MiB/2.8GiB(0%) CN:44 SD:7 DL:80KiB UL:36KiB(1.3MiB) ETA:10h18m43s] +[#066a28 4.4MiB/2.8GiB(0%) CN:44 SD:7 DL:76KiB UL:53KiB(1.5MiB) ETA:10h49m36s] +[#066a28 4.4MiB/2.8GiB(0%) CN:44 SD:7 DL:66KiB UL:48KiB(1.5MiB) ETA:12h22m22s] +[#066a28 4.5MiB/2.8GiB(0%) CN:44 SD:7 DL:57KiB UL:43KiB(1.5MiB) ETA:14h27m44s] +[#066a28 4.5MiB/2.8GiB(0%) CN:44 SD:7 DL:55KiB UL:39KiB(1.5MiB) ETA:14h52m9s] +[#066a28 4.5MiB/2.8GiB(0%) CN:44 SD:8 DL:52KiB UL:37KiB(1.5MiB) ETA:15h48m10s] +[#066a28 4.6MiB/2.8GiB(0%) CN:44 SD:8 DL:50KiB UL:21KiB(1.5MiB) ETA:16h13m54s] +[#066a28 4.6MiB/2.8GiB(0%) CN:44 SD:9 DL:42KiB UL:30KiB(1.5MiB) ETA:19h24m47s] +[#066a28 4.6MiB/2.8GiB(0%) CN:44 SD:9 DL:41KiB UL:53KiB(1.8MiB) ETA:19h44m57s] +[#066a28 4.7MiB/2.8GiB(0%) CN:44 SD:9 DL:32KiB UL:48KiB(1.8MiB) ETA:25h20m56s] +[#066a28 4.7MiB/2.8GiB(0%) CN:44 SD:10 DL:38KiB UL:44KiB(1.8MiB) ETA:21h24m33s] +[#066a28 4.8MiB/2.8GiB(0%) CN:44 SD:10 DL:34KiB UL:40KiB(1.8MiB) ETA:23h45m37s] +[#066a28 4.8MiB/2.8GiB(0%) CN:48 SD:10 DL:31KiB UL:56KiB(2.0MiB) ETA:26h11m56s] +[#066a28 4.8MiB/2.8GiB(0%) CN:46 SD:10 DL:30KiB UL:52KiB(2.0MiB) ETA:27h15m40s] +[#066a28 4.9MiB/2.8GiB(0%) CN:44 SD:10 DL:32KiB UL:75KiB(2.0MiB) ETA:25h13m45s] +[#066a28 4.9MiB/2.8GiB(0%) CN:44 SD:10 DL:36KiB UL:66KiB(2.0MiB) ETA:22h32m9s] +[#066a28 4.9MiB/2.8GiB(0%) CN:44 SD:10 DL:35KiB UL:61KiB(2.0MiB) ETA:23h9m10s] +[#066a28 5.0MiB/2.8GiB(0%) CN:44 SD:9 DL:36KiB UL:80KiB(2.3MiB) ETA:22h43m56s] +[#066a28 5.0MiB/2.8GiB(0%) CN:44 SD:9 DL:35KiB UL:69KiB(2.3MiB) ETA:23h6m29s] +[#066a28 5.1MiB/2.8GiB(0%) CN:44 SD:10 DL:37KiB UL:62KiB(2.3MiB) ETA:21h58m37s] +[#066a28 5.1MiB/2.8GiB(0%) CN:44 SD:10 DL:38KiB UL:58KiB(2.3MiB) ETA:21h19m1s] +[#066a28 5.1MiB/2.8GiB(0%) CN:49 SD:12 DL:35KiB UL:55KiB(2.3MiB) ETA:23h3s] +[#066a28 5.2MiB/2.8GiB(0%) CN:48 SD:11 DL:38KiB UL:48KiB(2.3MiB) ETA:21h41m12s] +[#066a28 5.2MiB/2.8GiB(0%) CN:45 SD:11 DL:32KiB UL:64KiB(2.6MiB) ETA:25h8m51s] +[#066a28 5.2MiB/2.8GiB(0%) CN:44 SD:12 DL:32KiB UL:59KiB(2.6MiB) ETA:25h17m48s] +^C +anarcat@marcos:iso(master +%)$ aria2c kali-linux-1.1.0-amd64.torrent + +02/09 22:17:16 [NOTICE] IPv4 DHT: listening on UDP port 6963 + +02/09 22:17:16 [NOTICE] IPv4 BitTorrent: listening on TCP port 6950 + +02/09 22:17:16 [NOTICE] IPv6 BitTorrent: listening on TCP port 6950 +[#51d0f6 5.2MiB/2.8GiB(0%) CN:44 SD:12 DL:241KiB ETA:3h25m7s] + +# End of transcript or log. +"""]] + +Could be caused by [[bittorrent_special_url_double-encoding]]? --[[anarcat]] diff --git a/doc/bugs/aria2c_display_broken_in_git-annex/comment_1_770e1d657bba2a3f5782eac2e8bdca57._comment b/doc/bugs/aria2c_display_broken_in_git-annex/comment_1_770e1d657bba2a3f5782eac2e8bdca57._comment new file mode 100644 index 0000000000..2fe9bf475b --- /dev/null +++ b/doc/bugs/aria2c_display_broken_in_git-annex/comment_1_770e1d657bba2a3f5782eac2e8bdca57._comment @@ -0,0 +1,14 @@ +[[!comment format=mdwn + username="joey" + subject="""comment 1""" + date="2015-02-10T16:11:09Z" + content=""" +This is because aria2c detects it's not outputting to a terminal, so it +stops using \r for progress displays. You can get the same effect by piping +aria2c to less. There does not seem to be any switch to force console-style +output. + +The only solution would be the complexity of making git-annex use a pty +internally, or just not showing aria2c progress output (or not intercepting +it for metering). +"""]] diff --git a/doc/bugs/aria2c_display_broken_in_git-annex/comment_2_2f26037c5a67f6f8928204ab0b7b7965._comment b/doc/bugs/aria2c_display_broken_in_git-annex/comment_2_2f26037c5a67f6f8928204ab0b7b7965._comment new file mode 100644 index 0000000000..4f12f3681f --- /dev/null +++ b/doc/bugs/aria2c_display_broken_in_git-annex/comment_2_2f26037c5a67f6f8928204ab0b7b7965._comment @@ -0,0 +1,7 @@ +[[!comment format=mdwn + username="anarcat" + subject="comment 2" + date="2015-02-10T17:22:08Z" + content=""" +maybe adding `\r` chars ourselves would fix it? +"""]] diff --git a/doc/bugs/assistant_bails_when_adding_encrypted_usbdrive_repo_on_mac.mdwn b/doc/bugs/assistant_bails_when_adding_encrypted_usbdrive_repo_on_mac.mdwn new file mode 100644 index 0000000000..23babd7975 --- /dev/null +++ b/doc/bugs/assistant_bails_when_adding_encrypted_usbdrive_repo_on_mac.mdwn @@ -0,0 +1,53 @@ +### Please describe the problem. +Tried adding a removable drive repository through git-annex assistant on Mac, asked to encrypt it, got Internal server error with explanation: unable to determine gcrypt-id of remote + +### What steps will reproduce the problem? +current (today's) version of git-annex build for Mountain Lion installed +app launched +default annex initialized, a couple file added +a removable 2GB USB drive (ExFAT) mounted in OS +click Add another repository +choose removable drive +choose to encrypt +a window explaining the need to wait for entropy shows +then we get the Internal server error: unable to determine gcrypt-id of remote + +### What version of git-annex are you using? On what operating system? +git-annex version 4.20130922-g7dc188a +Mac OS X 10.8.5 + +### Please provide any additional information below. + +looking at transcript below, it appears the root cause is the lack of initial git/gcrypt configuration. perhaps this needs to be caught/addressed somehow + +[[!format sh """ +# If you can, paste a complete transcript of the problem occurring here. +# If the problem is with the git-annex assistant, paste in .git/annex/daemon.log + +gpg: can't open `/usr/local/share/gnupg/options.skel': No such file or directory +.......+++++ +.+++++ +gpg: key 71BFBC31 marked as ultimately trusted +ok +(Recording state in git...) +(Recording state in git...) +(encryption setup) (hybrid cipher with gpg key C4B2EA8D71BFBC31) gcrypt: Development version -- Repository format MAY CHANGE +gpg: checking the trustdb +gpg: 3 marginal(s) needed, 1 complete(s) needed, PGP trust model +gpg: depth: 0 valid: 1 signed: 0 trust: 0-, 0q, 0n, 0m, 0f, 1u +gcrypt: WARNING: Skipping missing key C4B2EA8D71BFBC31 +gcrypt: You have not configured any keys you can encrypt to for this repository +gcrypt: Use :: +gcrypt: git config gcrypt.participants YOURKEYID +gcrypt: Development version -- Repository format MAY CHANGE +gcrypt: WARNING: Skipping missing key C4B2EA8D71BFBC31 +gcrypt: You have not configured any keys you can encrypt to for this repository +gcrypt: Use :: +gcrypt: git config gcrypt.participants YOURKEYID +03/Oct/2013:00:05:24 +0400 [Error#yesod-core] unable to determine gcrypt-id of remote @(yesod-core-1.1.8.3:Yesod.Internal.Core ./Yesod/Internal/Core.hs:550:5) + + +# End of transcript or log. +"""]] + +[[!tag moreinfo]] diff --git a/doc/bugs/assistant_bails_when_adding_encrypted_usbdrive_repo_on_mac/comment_1_4ea192e57f86a33087997746722e6acf._comment b/doc/bugs/assistant_bails_when_adding_encrypted_usbdrive_repo_on_mac/comment_1_4ea192e57f86a33087997746722e6acf._comment new file mode 100644 index 0000000000..2d0802cae7 --- /dev/null +++ b/doc/bugs/assistant_bails_when_adding_encrypted_usbdrive_repo_on_mac/comment_1_4ea192e57f86a33087997746722e6acf._comment @@ -0,0 +1,10 @@ +[[!comment format=mdwn + username="http://joeyh.name/" + ip="108.236.230.124" + subject="comment 1" + date="2013-10-02T20:37:03Z" + content=""" +Can you take a look at ~/annex/.git/config and if there is an entry for this git remote, post it? (Or post the whole thing if unsure..) + +Also, can you get gpg to list your secret keys, and see if key 71BFBC31 is in the list? You can do so by running `gpg --list-secret-keys` ; if you don't have gpg in your path at the terminal, you can use the version from the git-annex DMG by first running `/Volumes/git-annex/git-annex.app/Contents/MacOS/runshell` +"""]] diff --git a/doc/bugs/assistant_bails_when_adding_encrypted_usbdrive_repo_on_mac/comment_2_622ad5b34780fc8468c5c515ad9f27fa._comment b/doc/bugs/assistant_bails_when_adding_encrypted_usbdrive_repo_on_mac/comment_2_622ad5b34780fc8468c5c515ad9f27fa._comment new file mode 100644 index 0000000000..2ba7a59845 --- /dev/null +++ b/doc/bugs/assistant_bails_when_adding_encrypted_usbdrive_repo_on_mac/comment_2_622ad5b34780fc8468c5c515ad9f27fa._comment @@ -0,0 +1,8 @@ +[[!comment format=mdwn + username="https://www.google.com/accounts/o8/id?id=AItOawnyMzZZLS1xGW1raqc_9Md6Ksdkvx5rUJU" + nickname="Michael" + subject="comment 2" + date="2013-10-04T10:59:30Z" + content=""" +Sorry Joey, I was tinkering around trying to get the assistant off the ground and have destroyed the evidence of the bug in the process. I will try to recreate it and post the details. (most of my problems are/were related to bundled git/gpg/other software, I posted a separate comment in OSX Install page) +"""]] diff --git a/doc/bugs/assistant_crashes_in_TransferScanner.mdwn b/doc/bugs/assistant_crashes_in_TransferScanner.mdwn new file mode 100644 index 0000000000..53147bcef9 --- /dev/null +++ b/doc/bugs/assistant_crashes_in_TransferScanner.mdwn @@ -0,0 +1,34 @@ +After updating git annex from version 6.20160613 to 6.20161011 on my Archlinux machine, the assistant crashes while scanning the files for transfer. + +I'm using a v6 repository. My repositories can be synced perfectly on the command line, only the assistant keeps crashing. + +Here is the daemon.log: + +[[!format sh """ +[2016-10-18 11:49:59.482873] main: starting assistant version 6.20161011-ge2dcbe6 +[2016-10-18 11:50:05.181469] TransferScanner: Syncing with annexsyncbackup, origin + + No known network monitor available through dbus; falling back to polling + +From /media/backup/annex-backup/annex + 318e381..9cc5e23 git-annex -> annexsyncbackup/git-annex +(merging annexsyncbackup/git-annex into git-annex...) +From ssh://FOO/annex-sync + 318e381..81ac41f git-annex -> origin/git-annex +(merging origin/git-annex into git-annex...) +(recording state in git...) +To /media/backup/annex-backup/annex + 318e381..a4931de git-annex -> synced/git-annex +To ssh://FOO/annex-sync.git + 318e381..a4931de git-annex -> synced/git-annex +(started...) +[2016-10-18 11:50:12.364643] Committer: Committing changes to git +(recording state in git...) +[2016-10-18 11:50:12.464943] Pusher: Syncing with annexsyncbackup, origin +Everything up-to-date +Everything up-to-date +TransferScanner crashed: fd:47: commitBuffer: invalid argument (invalid character) +[2016-10-18 11:50:15.016862] TransferScanner: warning TransferScanner crashed: fd:47: commitBuffer: invalid argument (invalid character) +"""]] + +[[!meta title="v6 repository problems with filename encoding when not in unicode locale"]] diff --git a/doc/bugs/assistant_crashes_in_TransferScanner/comment_10_420d16e3d5343cf5dcc9dfcbb34689f4._comment b/doc/bugs/assistant_crashes_in_TransferScanner/comment_10_420d16e3d5343cf5dcc9dfcbb34689f4._comment new file mode 100644 index 0000000000..03ae5c254b --- /dev/null +++ b/doc/bugs/assistant_crashes_in_TransferScanner/comment_10_420d16e3d5343cf5dcc9dfcbb34689f4._comment @@ -0,0 +1,13 @@ +[[!comment format=mdwn + username="joey" + subject="""comment 10""" + date="2016-10-31T20:45:05Z" + content=""" +Was able to reproduce this pattern of output w/o filenames +by throwing an IOError from Command.Sync.syncFile, +in code that is outside of callCommandAction or includeCommandAction block. + +Unfortunately that does not narrow it down +enough to know there the problem lies. +I need a way to reproduce the bug to get any further. +"""]] diff --git a/doc/bugs/assistant_crashes_in_TransferScanner/comment_11_6a5fe92c433ca7b6e66bbb86a16aa3b1._comment b/doc/bugs/assistant_crashes_in_TransferScanner/comment_11_6a5fe92c433ca7b6e66bbb86a16aa3b1._comment new file mode 100644 index 0000000000..f7f4f84f1c --- /dev/null +++ b/doc/bugs/assistant_crashes_in_TransferScanner/comment_11_6a5fe92c433ca7b6e66bbb86a16aa3b1._comment @@ -0,0 +1,11 @@ +[[!comment format=mdwn + username="johannes@12f1850a57e13cc234b5ebf88a5d3ac68915a6c2" + nickname="johannes" + avatar="http://cdn.libravatar.org/avatar/7acaf4a71b0b93cc419195f58f4cd54c" + subject="comment 11" + date="2016-10-31T20:59:30Z" + content=""" +Here is the complete log (which, imho, does not give any hint): + +(elided, I appreciate the info, but not needed. --[[Joey]]) +"""]] diff --git a/doc/bugs/assistant_crashes_in_TransferScanner/comment_12_7c7267ab724b0da325d4cf15e33e6192._comment b/doc/bugs/assistant_crashes_in_TransferScanner/comment_12_7c7267ab724b0da325d4cf15e33e6192._comment new file mode 100644 index 0000000000..ff2cb5c1d2 --- /dev/null +++ b/doc/bugs/assistant_crashes_in_TransferScanner/comment_12_7c7267ab724b0da325d4cf15e33e6192._comment @@ -0,0 +1,12 @@ +[[!comment format=mdwn + username="joey" + subject="""comment 12""" + date="2016-10-31T21:16:53Z" + content=""" +Aha! It's something to do with unlocked files in v6 mode. + +Reproduced by unlocking a file, and LANG=C git annex sync --content + +With 1 unlocked file, I see the message twice, which makes sense +given --content's 2 passes. +"""]] diff --git a/doc/bugs/assistant_crashes_in_TransferScanner/comment_13_2ff6938256176b668b7d611a16b7a73e._comment b/doc/bugs/assistant_crashes_in_TransferScanner/comment_13_2ff6938256176b668b7d611a16b7a73e._comment new file mode 100644 index 0000000000..3413c1dd1f --- /dev/null +++ b/doc/bugs/assistant_crashes_in_TransferScanner/comment_13_2ff6938256176b668b7d611a16b7a73e._comment @@ -0,0 +1,8 @@ +[[!comment format=mdwn + username="joey" + subject="""comment 13""" + date="2016-10-31T21:26:52Z" + content=""" +It's getFileNumCopies crashing, where the unlocked worktree +filename contains some non-ascii character. +"""]] diff --git a/doc/bugs/assistant_crashes_in_TransferScanner/comment_14_16d967c0ba173d122f645af2a6a487e6._comment b/doc/bugs/assistant_crashes_in_TransferScanner/comment_14_16d967c0ba173d122f645af2a6a487e6._comment new file mode 100644 index 0000000000..52b226f7f8 --- /dev/null +++ b/doc/bugs/assistant_crashes_in_TransferScanner/comment_14_16d967c0ba173d122f645af2a6a487e6._comment @@ -0,0 +1,17 @@ +[[!comment format=mdwn + username="joey" + subject="""comment 14""" + date="2016-10-31T21:45:37Z" + content=""" +Tracked it back to Git.CheckAttr.checkAttr crashing when it sends the +filename "foö" (note the non-ascii) to the handle. +Which is odd as that handle is in filesystem encoding mode. + +But, I then unlocked an additional file in my repo and it stopped +crashing, and I have yet to reproduce it again. Urgh. + +Also, found an unrelated filename encoding crash: + + LANG=C git annex smudge --clean xx.oök < xx.oök + git-annex: recoverEncode: invalid argument (invalid character) +"""]] diff --git a/doc/bugs/assistant_crashes_in_TransferScanner/comment_15_d940f66df8aa5850ed9425666ec08185._comment b/doc/bugs/assistant_crashes_in_TransferScanner/comment_15_d940f66df8aa5850ed9425666ec08185._comment new file mode 100644 index 0000000000..bd5f270974 --- /dev/null +++ b/doc/bugs/assistant_crashes_in_TransferScanner/comment_15_d940f66df8aa5850ed9425666ec08185._comment @@ -0,0 +1,49 @@ +[[!comment format=mdwn + username="joey" + subject="""comment 15""" + date="2016-11-01T18:04:33Z" + content=""" +Having difficulty reproducing this again. + +However, I did find one way that the git check-attr handle could not be in +raw mode. If for some reason the process was started and crashed, it would +be restarted and not be in raw mode. +Fixed in [[!commit e23028d19b493c7abae0dffb96da44591aacca7b]]. + +---- + +As to the `git annex smudge --clean` encoding problem, here's a full test +case: + + joey@darkstar:/tmp>git init repo3 + Initialized empty Git repository in /tmp/repo3/.git/ + joey@darkstar:/tmp>cd repo3 + joey@darkstar:/tmp/repo3>LANG=en_US.utf8 + joey@darkstar:/tmp/repo3>git annex init --version=6 + init ok + (recording state in git...) + joey@darkstar:/tmp/repo3>touch xx.oöo + joey@darkstar:/tmp/repo3>git annex add xx.oöo + add xx.oöo ok + (recording state in git...) + joey@darkstar:/tmp/repo3>LANG=C + joey@darkstar:/tmp/repo3>git annex unlock xx.oöo + unlock xx.oöo ok + (recording state in git...) + joey@darkstar:/tmp/repo3>git annex smudge --clean xx.oöo < xx.oöo + git-annex: recoverEncode: invalid argument (invalid character) + git-annex: smudge: 1 failed + +Only happens when the filename extension contains unicode, which then becomes part of the key. + +If the unlock is done in a unicode locale, the later smudge succeeds even +when done in LANG=C. + +Dumping the sqlite database, in the problem case, I see mokibake: + + INSERT INTO "associated" VALUES(1,'SHA256E-s0--e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855.o��o','"xx.o\56515\56502o"'); + +Compare with the working case: + + INSERT INTO "associated" VALUES(11,'SHA256E-s30--58ed27fa666794aa15b72609483e9b488b984b568d668959136302faa8bb2fde.oöo','good.oöo'); +"""]] diff --git a/doc/bugs/assistant_crashes_in_TransferScanner/comment_16_0d32f7938c071553c3d3cec2b85ccb4f._comment b/doc/bugs/assistant_crashes_in_TransferScanner/comment_16_0d32f7938c071553c3d3cec2b85ccb4f._comment new file mode 100644 index 0000000000..574e2891ee --- /dev/null +++ b/doc/bugs/assistant_crashes_in_TransferScanner/comment_16_0d32f7938c071553c3d3cec2b85ccb4f._comment @@ -0,0 +1,56 @@ +[[!comment format=mdwn + username="johannes@12f1850a57e13cc234b5ebf88a5d3ac68915a6c2" + nickname="johannes" + avatar="http://cdn.libravatar.org/avatar/7acaf4a71b0b93cc419195f58f4cd54c" + subject="comment 16" + date="2016-11-04T10:27:26Z" + content=""" +FYI, with 20161101 I get the following errors: + + fatal: '../home/johannes/annex-sync/Documents/personal/Kontoauszüge/Konto_XXXXXXXXXX-Auszug_2016_016.PDF' is outside repository + fatal: '../home/johannes/annex-sync/Documents/personal/Kontoauszüge/Konto_XXXXXXXXXX-Auszug_2016_016.PDF' is outside repository + fatal: '../home/johannes/annex-sync/Documents/personal/Kontoauszüge/Konto_XXXXXXXXXX-Auszug_2016_016.PDF' is outside repository + fatal: '../home/johannes/annex-sync/Documents/personal/Kontoauszüge/Konto_XXXXXXXXXX-Auszug_2016_016.PDF' is outside repository + fatal: '../home/johannes/annex-sync/Documents/personal/Kontoauszüge/Konto_XXXXXXXXXX-Auszug_2016_016.PDF' is outside repository + fatal: '../home/johannes/annex-sync/Documents/personal/Kontoauszüge/Konto_XXXXXXXXXX-Auszug_2016_016.PDF' is outside repository + fatal: '../home/johannes/annex-sync/Documents/personal/Kontoauszüge/Konto_XXXXXXXXXX-Auszug_2016_016.PDF' is outside repository + fatal: '../home/johannes/annex-sync/Documents/personal/Kontoauszüge/Konto_XXXXXXXXXX-Auszug_2016_016.PDF' is outside repository + fatal: '../home/johannes/annex-sync/Documents/personal/Kontoauszüge/Konto_XXXXXXXXXX-Auszug_2016_016.PDF' is outside repository + fatal: '../home/johannes/annex-sync/Documents/personal/Kontoauszüge/Konto_XXXXXXXXXX-Auszug_2016_016.PDF' is outside repository + fatal: '../home/johannes/annex-sync/Documents/personal/Kontoauszüge/Konto_XXXXXXXXXX-Auszug_2016_016.PDF' is outside repository + + + git-annex: git check-attr EOF: user error + failed + + git-annex: fd:30: hFlush: resource vanished (Broken pipe) + failed + + git-annex: fd:30: hFlush: resource vanished (Broken pipe) + failed + + [...] + + git-annex: fd:30: commitBuffer: resource vanished (Broken pipe) + failed + + git-annex: fd:30: commitBuffer: resource vanished (Broken pipe) + failed + + git-annex: fd:30: hFlush: resource vanished (Broken pipe) + failed + + git-annex: fd:30: commitBuffer: resource vanished (Broken pipe) + failed + + git-annex: fd:30: commitBuffer: resource vanished (Broken pipe) + failed + + [...] + + git-annex: sync: 836 failed + +The `[...]` indicates that the last error appears a lot of times. + +The first (repeated) error might be unrelated though. Note that this is a file I recently added and locked. +"""]] diff --git a/doc/bugs/assistant_crashes_in_TransferScanner/comment_17_fc141f396d2b9a27f6668d1b10836ea5._comment b/doc/bugs/assistant_crashes_in_TransferScanner/comment_17_fc141f396d2b9a27f6668d1b10836ea5._comment new file mode 100644 index 0000000000..7978244f9a --- /dev/null +++ b/doc/bugs/assistant_crashes_in_TransferScanner/comment_17_fc141f396d2b9a27f6668d1b10836ea5._comment @@ -0,0 +1,13 @@ +[[!comment format=mdwn + username="johannes@12f1850a57e13cc234b5ebf88a5d3ac68915a6c2" + nickname="johannes" + avatar="http://cdn.libravatar.org/avatar/7acaf4a71b0b93cc419195f58f4cd54c" + subject="Solution" + date="2017-01-06T12:19:46Z" + content=""" +After the update to 20161231, I tried again. Unfortunately, I am still getting the same error. + +However, I noticed that removing the file for which I get the \"is outside repository\" error actually fixes the problem. Once I dropped the file from the annex as well, I was able to re-add the file without triggering this error again. + +Please consider this issues as fixed. +"""]] diff --git a/doc/bugs/assistant_crashes_in_TransferScanner/comment_1_e626d6fc90a4af8906973121149eef7d._comment b/doc/bugs/assistant_crashes_in_TransferScanner/comment_1_e626d6fc90a4af8906973121149eef7d._comment new file mode 100644 index 0000000000..514c85a233 --- /dev/null +++ b/doc/bugs/assistant_crashes_in_TransferScanner/comment_1_e626d6fc90a4af8906973121149eef7d._comment @@ -0,0 +1,13 @@ +[[!comment format=mdwn + username="johannes@12f1850a57e13cc234b5ebf88a5d3ac68915a6c2" + nickname="johannes" + avatar="http://cdn.libravatar.org/avatar/7acaf4a71b0b93cc419195f58f4cd54c" + subject="comment 1" + date="2016-10-19T10:28:13Z" + content=""" +I just noticed that I get the same error when executing `git annex sync --content`. Nevertheless, the following commands are running flawlessly: + +* `git annex sync` +* `git annex get --auto` +* `git annex copy -t origin --auto`. +"""]] diff --git a/doc/bugs/assistant_crashes_in_TransferScanner/comment_2_0beeaa83cbcea07b522c7b274e328a25._comment b/doc/bugs/assistant_crashes_in_TransferScanner/comment_2_0beeaa83cbcea07b522c7b274e328a25._comment new file mode 100644 index 0000000000..c885252ede --- /dev/null +++ b/doc/bugs/assistant_crashes_in_TransferScanner/comment_2_0beeaa83cbcea07b522c7b274e328a25._comment @@ -0,0 +1,14 @@ +[[!comment format=mdwn + username="joey" + subject="""comment 2""" + date="2016-10-26T18:21:13Z" + content=""" +This kind of thing tends to be due to a problem with locales, or a +filename in the repository that can't be represented under the current +locale. + +Just so happens that the version you upgraded to changed how the standalone +tarball for linux handles locales. Did you install using that tarball? + +What does `locale` say? +"""]] diff --git a/doc/bugs/assistant_crashes_in_TransferScanner/comment_3_a56c8acec75c5862d3a407ffe9a385b3._comment b/doc/bugs/assistant_crashes_in_TransferScanner/comment_3_a56c8acec75c5862d3a407ffe9a385b3._comment new file mode 100644 index 0000000000..5507091ed4 --- /dev/null +++ b/doc/bugs/assistant_crashes_in_TransferScanner/comment_3_a56c8acec75c5862d3a407ffe9a385b3._comment @@ -0,0 +1,26 @@ +[[!comment format=mdwn + username="johannes@12f1850a57e13cc234b5ebf88a5d3ac68915a6c2" + nickname="johannes" + avatar="http://cdn.libravatar.org/avatar/7acaf4a71b0b93cc419195f58f4cd54c" + subject="comment 3" + date="2016-10-28T10:17:19Z" + content=""" +Yes, I installed the standalone tarball. + +My locale is set to en_US.utf8, here is what `locale` tells me: + + LANG=en_US.utf8 + LC_CTYPE=\"en_US.utf8\" + LC_NUMERIC=\"en_US.utf8\" + LC_TIME=\"en_US.utf8\" + LC_COLLATE=\"en_US.utf8\" + LC_MONETARY=\"en_US.utf8\" + LC_MESSAGES=\"en_US.utf8\" + LC_PAPER=\"en_US.utf8\" + LC_NAME=\"en_US.utf8\" + LC_ADDRESS=\"en_US.utf8\" + LC_TELEPHONE=\"en_US.utf8\" + LC_MEASUREMENT=\"en_US.utf8\" + LC_IDENTIFICATION=\"en_US.utf8\" + LC_ALL= +"""]] diff --git a/doc/bugs/assistant_crashes_in_TransferScanner/comment_4_cdc659b30d1cbcfdc9ce0d311ebc52c5._comment b/doc/bugs/assistant_crashes_in_TransferScanner/comment_4_cdc659b30d1cbcfdc9ce0d311ebc52c5._comment new file mode 100644 index 0000000000..fd1be667bb --- /dev/null +++ b/doc/bugs/assistant_crashes_in_TransferScanner/comment_4_cdc659b30d1cbcfdc9ce0d311ebc52c5._comment @@ -0,0 +1,19 @@ +[[!comment format=mdwn + username="joey" + subject="""comment 4""" + date="2016-10-31T18:16:51Z" + content=""" +What is the output when using `git annex sync --content` ? +I know, same basic error as the assistant, but it probably displays +some filenames which will hint at the particular filename it's crashing on. + +Investigating, it looks like there may be a recently +introduced bug in the standalone tarball where it contains +`git-annex.linux/i18n/i18n`. If you move the contents of +`git-annex.linux/i18n/i18n` to `git-annex.linux/i18n`, you might find that +it causes the crash to go away. I'm committing a fix for that problem. + +(I'd still like information about the filename that causes the crash +though, since that same crash could happen if the locale was misconfigured +or if you were using a non-utf8 locale.) +"""]] diff --git a/doc/bugs/assistant_crashes_in_TransferScanner/comment_5_6565e422393e9c1f40d075af7058020e._comment b/doc/bugs/assistant_crashes_in_TransferScanner/comment_5_6565e422393e9c1f40d075af7058020e._comment new file mode 100644 index 0000000000..eb2e53fa5f --- /dev/null +++ b/doc/bugs/assistant_crashes_in_TransferScanner/comment_5_6565e422393e9c1f40d075af7058020e._comment @@ -0,0 +1,45 @@ +[[!comment format=mdwn + username="johannes@12f1850a57e13cc234b5ebf88a5d3ac68915a6c2" + nickname="johannes" + avatar="http://cdn.libravatar.org/avatar/7acaf4a71b0b93cc419195f58f4cd54c" + subject="comment 5" + date="2016-10-31T18:51:17Z" + content=""" +The output of `git annex sync --content` looks like this: + + commit + [master 4b31758] git-annex in new latitude + 1 file changed, 2 insertions(+), 1 deletion(-) + ok + pull annexsyncbackup + ok + pull origin + Enter passphrase for key '/home/johanness/.ssh/id_rsa': + ok + + git-annex: fd:30: commitBuffer: invalid argument (invalid character) + failed + + git-annex: fd:30: commitBuffer: invalid argument (invalid character) + failed + + git-annex: fd:30: commitBuffer: invalid argument (invalid character) + failed + + git-annex: fd:30: commitBuffer: invalid argument (invalid character) + failed + + git-annex: fd:30: commitBuffer: invalid argument (invalid character) + failed + +These lines are repeated a bunch of times. In between git annex successfully uploads and drops a couple of files to/from the remotes. After that it successfully records the state in git and pushes to the remotes finishing with: + + git-annex: sync: 645 failed + +I moved the i18n folder but am still getting the same errors. + +`ls -l /opt/git-annex.linux/i18n`: + + charmaps locales SUPPORTED + +"""]] diff --git a/doc/bugs/assistant_crashes_in_TransferScanner/comment_6_86389af01aaeabe8eb6c2af26449ec50._comment b/doc/bugs/assistant_crashes_in_TransferScanner/comment_6_86389af01aaeabe8eb6c2af26449ec50._comment new file mode 100644 index 0000000000..dec8161f3d --- /dev/null +++ b/doc/bugs/assistant_crashes_in_TransferScanner/comment_6_86389af01aaeabe8eb6c2af26449ec50._comment @@ -0,0 +1,13 @@ +[[!comment format=mdwn + username="johannes@12f1850a57e13cc234b5ebf88a5d3ac68915a6c2" + nickname="johannes" + avatar="http://cdn.libravatar.org/avatar/7acaf4a71b0b93cc419195f58f4cd54c" + subject="SOLVED" + date="2016-10-31T19:08:16Z" + content=""" +I fixed it. It came to my mind that I'm using a rather antiquated PKGBUILD on my ArchLinux system to package the current git-annex tarball. + +(Fortunately, git-annex moved to the community repository a few months ago. Unfortunately, it's not maintained so that I was still using and updating the old PKGBUILD from the Archlinux User Repository (AUR).) + +Turned out this PKGBUILD removed some executables from the tarball. When I unpack the unmodified tarball, git annex works perfectly. +"""]] diff --git a/doc/bugs/assistant_crashes_in_TransferScanner/comment_7_d39a75177cc8fda77f3b4bcda6129a12._comment b/doc/bugs/assistant_crashes_in_TransferScanner/comment_7_d39a75177cc8fda77f3b4bcda6129a12._comment new file mode 100644 index 0000000000..262451a471 --- /dev/null +++ b/doc/bugs/assistant_crashes_in_TransferScanner/comment_7_d39a75177cc8fda77f3b4bcda6129a12._comment @@ -0,0 +1,19 @@ +[[!comment format=mdwn + username="joey" + subject="""comment 7""" + date="2016-10-31T19:26:38Z" + content=""" +Ok so you're using that monstrosity where Arch repackages the git-annex +standalone tarball into an Arch package? I never liked that because it's +prone to this kind of breakage. AIUI, `pacman -S git-annex` will install +a proper from-source build of git-annex and is the right thing to use. + +Anyway, I'm still wanting to get to the bottom of the problem +since as I noted the same problem could occur if using a non-UTF8 locale. + +Did you leave out the filenames from the `git annex sync` output, +or is it actually not printing any filenames there? It's very hard for me +to understand output where some lines have been left out. It's fine +if you replace all ASCII characters in your filenames with `X` in the paste +you share. +"""]] diff --git a/doc/bugs/assistant_crashes_in_TransferScanner/comment_8_099d19ffa8f769b69807fd2f39f77ed3._comment b/doc/bugs/assistant_crashes_in_TransferScanner/comment_8_099d19ffa8f769b69807fd2f39f77ed3._comment new file mode 100644 index 0000000000..56d8166627 --- /dev/null +++ b/doc/bugs/assistant_crashes_in_TransferScanner/comment_8_099d19ffa8f769b69807fd2f39f77ed3._comment @@ -0,0 +1,16 @@ +[[!comment format=mdwn + username="johannes@12f1850a57e13cc234b5ebf88a5d3ac68915a6c2" + nickname="johannes" + avatar="http://cdn.libravatar.org/avatar/7acaf4a71b0b93cc419195f58f4cd54c" + subject="comment 8" + date="2016-10-31T19:50:23Z" + content=""" +It's not printing any filenames. I modified my old PKGBUILD to create a package that only wraps the /opt/git-annex.linux directory. It still left me with the same error. I first suspected the Archlinux packaging of modifying the binaries in some awkward manner. But guess what, it's caused by the restrictive file/directory permissions on /opt/git-annex.linux. + +So here is how I can reproduce the error: + +- Step 1: Extract the git-annex standalone tarball to /opt/. +- Step 2: Change the owner and group of /opt/git-annex.linux to `root` (`chown -R root:root /opt/git-annex.linux`). + +So, what happens here? On the first execution of git-annex, it creates a `locales` directory in /opt/git-annex.linux. If the permissions are too restrictive, it silently fails to do that. I get the above error messages if the locales directory is not initialised. +"""]] diff --git a/doc/bugs/assistant_crashes_in_TransferScanner/comment_9_941f1a2911f25e74b0f610b5dbc9641c._comment b/doc/bugs/assistant_crashes_in_TransferScanner/comment_9_941f1a2911f25e74b0f610b5dbc9641c._comment new file mode 100644 index 0000000000..3570410e20 --- /dev/null +++ b/doc/bugs/assistant_crashes_in_TransferScanner/comment_9_941f1a2911f25e74b0f610b5dbc9641c._comment @@ -0,0 +1,14 @@ +[[!comment format=mdwn + username="joey" + subject="""comment 9""" + date="2016-10-31T20:17:59Z" + content=""" +You say it prints no filenames, but earlier you said: "In between git annex +successfully uploads and drops a couple of files to/from the remotes." +... Which sounds like it must be printing some filenames. + +Even when I make the standalone tarball owned by root, and move all +system-wide locale files out of the way, I still can't reproduce this. +So I think it must have something to do with the filenames in your git +repository, or something else particular to your git repository. +"""]] diff --git a/doc/bugs/assistant_does_not_allow_adding_an_existing_repo.mdwn b/doc/bugs/assistant_does_not_allow_adding_an_existing_repo.mdwn new file mode 100644 index 0000000000..8d2c3c9964 --- /dev/null +++ b/doc/bugs/assistant_does_not_allow_adding_an_existing_repo.mdwn @@ -0,0 +1,10 @@ +The assistant does not allow adding an existing repo: I tried to add a "remote server", to enter the credits, and then I add to choose between "git-annex" or "rsync". Choosing "git-annex" tries to create a new repo and then fails. The repo on the server is a bare one, accessed via gitolite. Setting the remote manually ("git remote add origin git@example.org:my-annex.git" works (I can pull, push, ...)) + +Mac OS 10.7 version 2013-09-10 + + +[[!meta title="assistant does not interoperate with gitolite when adding a repository"]] + +[2013-09-13 17:00:55 CEST] chat: ssh ["-p","22","git@example.org","sh -c 'mkdir -p '\"'\"'my-annex.git'\"'\"'&&cd '\"'\"'my-annex.git'\"'\"'&&if [ ! -d .git ]; then git init --bare --shared; fi&&git annex init'"] + +[[!tag confirmed]] diff --git a/doc/bugs/assistant_does_not_allow_adding_an_existing_repo/comment_1_87e84d56d56abefe8cac8a52b76c9003._comment b/doc/bugs/assistant_does_not_allow_adding_an_existing_repo/comment_1_87e84d56d56abefe8cac8a52b76c9003._comment new file mode 100644 index 0000000000..b260b8df51 --- /dev/null +++ b/doc/bugs/assistant_does_not_allow_adding_an_existing_repo/comment_1_87e84d56d56abefe8cac8a52b76c9003._comment @@ -0,0 +1,8 @@ +[[!comment format=mdwn + username="http://joeyh.name/" + ip="4.154.2.134" + subject="comment 1" + date="2013-09-13T18:19:03Z" + content=""" +I suppose the assistant could try to add a git remote and pull from it, and only if this pull fails go try to run shell commands to create it. This might work with gitolite, gitosis, github, etc. Although only partially for ones that don't support running git-annex-shell.. +"""]] diff --git a/doc/bugs/assistant_does_not_allow_adding_an_existing_repo/comment_2_55fcf44eefe6abac3f61a14e9ce9206c._comment b/doc/bugs/assistant_does_not_allow_adding_an_existing_repo/comment_2_55fcf44eefe6abac3f61a14e9ce9206c._comment new file mode 100644 index 0000000000..8ebdb4ff98 --- /dev/null +++ b/doc/bugs/assistant_does_not_allow_adding_an_existing_repo/comment_2_55fcf44eefe6abac3f61a14e9ce9206c._comment @@ -0,0 +1,34 @@ +[[!comment format=mdwn + username="effigies" + subject="Strategy for getting up and running" + date="2015-03-09T17:32:19Z" + content=""" +The following is the set of steps I use when setting assistant up on a new repository: + + git clone user@host:repo.git + pushd repo + git annex init + touch EMPTY + git annex add EMPTY + git commit -m 'Initial commit' + git push --all + git annex copy --to origin + git annex direct + git annex sync + git annex untrust . + popd + +Entering the folder path now lets the assistant take over. + +For an existing repository: + + git clone user@host:repo.git + pushd repo + git annex get . + git annex direct + git annex sync + git annex untrust . + popd + +Not sure if this would be helpful for working out what the assistant behavior should be, but maybe it'll help others get to the point where the assistant works with a gitolite managed repo. +"""]] diff --git a/doc/bugs/assistant_does_not_always_use_repo_cost_info_when_queueing_downloads.mdwn b/doc/bugs/assistant_does_not_always_use_repo_cost_info_when_queueing_downloads.mdwn new file mode 100644 index 0000000000..d7002a2f7a --- /dev/null +++ b/doc/bugs/assistant_does_not_always_use_repo_cost_info_when_queueing_downloads.mdwn @@ -0,0 +1,18 @@ +git-annex has nice repository cost tracking, but the assistant doesn't +always use this when queuing downloads. + +For example, if a network comes up, it'll queue a lot of downloads from a +remote, perhaps expensive server. If a removable drive is then attached, +which has a lower cost, it'll queue downloads from there, but +only once it works through all the ones it's already queued from the +network. + +There is a workaround: Since the webapp allows disabling syncing to a +remote, the user can go in and disable syncing to the network remote +temporarily. + +[[!tag /design/assistant]] + +--[[Joey]] + +[[!tag confirmed]] diff --git a/doc/bugs/assistant_doesn__39__t_sync_empty_directories.mdwn b/doc/bugs/assistant_doesn__39__t_sync_empty_directories.mdwn new file mode 100644 index 0000000000..53ee5a1050 --- /dev/null +++ b/doc/bugs/assistant_doesn__39__t_sync_empty_directories.mdwn @@ -0,0 +1,32 @@ +### Please describe the problem. + +The assistant seems to only sync files and not directories, so empty directories will not be synced. + +### What steps will reproduce the problem? + +- Create an empty directory on HostA +- See that nothing gets synced (daemon.log is silent) and the directory doesn't exist on the other HostB +- Create an empty file inside the empty directory on HostA +- See that now HostB has the empty file inside the directory + +I'm sure this comes directly from the git behavior but is at least surprising for a direct mode repository using the assistant. + +### What version of git-annex are you using? On what operating system? + +[[!format sh """ +$ git annex version +git-annex version: 4.20130516.1 +build flags: Assistant Webapp Pairing Testsuite S3 WebDAV Inotify DBus XMPP +local repository version: 4 +default repository version: 3 +supported repository versions: 3 4 +upgrade supported from repository versions: 0 1 2 +$ lsb_release -a +No LSB modules are available. +Distributor ID: Ubuntu +Description: Ubuntu 12.04.2 LTS +Release: 12.04 +Codename: precise +"""]] + +> [[!taglink confirmed]] (but may be out of scope) --[[Joey]] diff --git a/doc/bugs/assistant_doesn__39__t_sync_empty_directories/comment_1_78a3bde607f43c0f518bd2d3d7196022._comment b/doc/bugs/assistant_doesn__39__t_sync_empty_directories/comment_1_78a3bde607f43c0f518bd2d3d7196022._comment new file mode 100644 index 0000000000..b31955ed38 --- /dev/null +++ b/doc/bugs/assistant_doesn__39__t_sync_empty_directories/comment_1_78a3bde607f43c0f518bd2d3d7196022._comment @@ -0,0 +1,8 @@ +[[!comment format=mdwn + username="http://joeyh.name/" + nickname="joey" + subject="comment 1" + date="2013-05-31T21:14:11Z" + content=""" +Git does not have a concept of an empty directory. The typical workaround is to put a .gitignore file in the directory. +"""]] diff --git a/doc/bugs/assistant_doesn__39__t_sync_empty_directories/comment_2_83777384b72732b1d0a19b32686d3d1f._comment b/doc/bugs/assistant_doesn__39__t_sync_empty_directories/comment_2_83777384b72732b1d0a19b32686d3d1f._comment new file mode 100644 index 0000000000..a62cba1e37 --- /dev/null +++ b/doc/bugs/assistant_doesn__39__t_sync_empty_directories/comment_2_83777384b72732b1d0a19b32686d3d1f._comment @@ -0,0 +1,8 @@ +[[!comment format=mdwn + username="https://www.google.com/accounts/o8/id?id=AItOawkx5V3MTbzCXS3J7Mn9FEq8M9bPPYMkAHY" + nickname="Pedro" + subject="comment 2" + date="2013-05-31T21:48:24Z" + content=""" +Adding .gitignore files makes sense for a git standpoint but whatever workaround is needed to make the assistant works properly should be done by the assistant itself no? Maybe having direct mode commit a shaddow \".this_is_a_magic_empty_file\" in empty directories and then not actually writing that file into direct mode repositories on the other side. +"""]] diff --git a/doc/bugs/assistant_doesn__39__t_sync_empty_directories/comment_3_f9b2a700c060707fae1bcb2ec0e4e4dc._comment b/doc/bugs/assistant_doesn__39__t_sync_empty_directories/comment_3_f9b2a700c060707fae1bcb2ec0e4e4dc._comment new file mode 100644 index 0000000000..757a1c7276 --- /dev/null +++ b/doc/bugs/assistant_doesn__39__t_sync_empty_directories/comment_3_f9b2a700c060707fae1bcb2ec0e4e4dc._comment @@ -0,0 +1,9 @@ +[[!comment format=mdwn + username="https://www.google.com/accounts/o8/id?id=AItOawmOsy6nbvPyXLd--qqjPMLnVIzxgZwtKlQ" + nickname="Nicolas" + subject="comment 3" + date="2013-09-11T23:19:26Z" + content=""" +This behaviour is indeed very confusing when one attempts to use git-annex with the assistant only, without thinking about the underlying machinery. +Having the assistant automatically put a .gitignore file in empty directories is a possible solution, but maybe printing a warning in the log when the watcher sees an empty directory could be an unobtrusive and helpful solution? (At least it would have helped me) +"""]] diff --git a/doc/bugs/assistant_doesn__39__t_sync_empty_directories/comment_4_014d213a959dd7993bdd247722a8817e._comment b/doc/bugs/assistant_doesn__39__t_sync_empty_directories/comment_4_014d213a959dd7993bdd247722a8817e._comment new file mode 100644 index 0000000000..f6be08ae4b --- /dev/null +++ b/doc/bugs/assistant_doesn__39__t_sync_empty_directories/comment_4_014d213a959dd7993bdd247722a8817e._comment @@ -0,0 +1,8 @@ +[[!comment format=mdwn + username="https://www.google.com/accounts/o8/id?id=AItOawmOsy6nbvPyXLd--qqjPMLnVIzxgZwtKlQ" + nickname="Nicolas" + subject="comment 4" + date="2013-09-11T23:24:58Z" + content=""" +Another problem with the current behaviour is that when deleting a directory and its contents on a repository, the contents are deleted on the other repositories but empty directories are left behind... +"""]] diff --git a/doc/bugs/assistant_doesn__39__t_sync_empty_directories/comment_5_440f349781d7d9ca2d1ed81386f7dd26._comment b/doc/bugs/assistant_doesn__39__t_sync_empty_directories/comment_5_440f349781d7d9ca2d1ed81386f7dd26._comment new file mode 100644 index 0000000000..0d5f0aa295 --- /dev/null +++ b/doc/bugs/assistant_doesn__39__t_sync_empty_directories/comment_5_440f349781d7d9ca2d1ed81386f7dd26._comment @@ -0,0 +1,8 @@ +[[!comment format=mdwn + username="https://www.google.com/accounts/o8/id?id=AItOawmNu4V5fvpLlBhaCUfXXOB0MI5NXwh8SkU" + nickname="Adam" + subject="comment 5" + date="2013-11-01T22:12:04Z" + content=""" +I agree that this is very confusing. Also having to manually delete empty, renamed directories is...less than ideal. :) +"""]] diff --git a/doc/bugs/assistant_doesn__39__t_sync_file_permissions.mdwn b/doc/bugs/assistant_doesn__39__t_sync_file_permissions.mdwn new file mode 100644 index 0000000000..48a99d7c24 --- /dev/null +++ b/doc/bugs/assistant_doesn__39__t_sync_file_permissions.mdwn @@ -0,0 +1,47 @@ +### Please describe the problem. + +The assistant does not sync file permissions so if you set something as executable it won't be set the same on the other repositories. + +### What steps will reproduce the problem? + +On the first host: + +[[!format sh """ +$ echo -e '#!/bin/sh\necho "Hello World!"' > testscript +$ chmod ugo+x testscript +$ ./testscript +Hello World! +$ ls -l +total 4 +-rwxr-xr-x 1 pedrocr pedrocr 30 May 31 20:00 testscript +"""]] + +And on the second host: + +[[!format sh """ +$ ls -l +total 4 +-rw-r--r-- 1 pedrocr pedrocr 30 May 31 20:00 testscript +$ ./testscript +bash: ./testscript: Permission denied +"""]] + +### What version of git-annex are you using? On what operating system? + +[[!format sh """ +$ git annex version +git-annex version: 4.20130516.1 +build flags: Assistant Webapp Pairing Testsuite S3 WebDAV Inotify DBus XMPP +local repository version: 4 +default repository version: 3 +supported repository versions: 3 4 +upgrade supported from repository versions: 0 1 2 +$ lsb_release -a +No LSB modules are available. +Distributor ID: Ubuntu +Description: Ubuntu 12.04.2 LTS +Release: 12.04 +Codename: precise +"""]] + +> [[!taglink confirmed]] (but may be out of scope) --[[Joey]] diff --git a/doc/bugs/assistant_doesn__39__t_sync_file_permissions/comment_1_fc8d3ea209a2ab39c1aeff52452d4c58._comment b/doc/bugs/assistant_doesn__39__t_sync_file_permissions/comment_1_fc8d3ea209a2ab39c1aeff52452d4c58._comment new file mode 100644 index 0000000000..9e1b801a4e --- /dev/null +++ b/doc/bugs/assistant_doesn__39__t_sync_file_permissions/comment_1_fc8d3ea209a2ab39c1aeff52452d4c58._comment @@ -0,0 +1,10 @@ +[[!comment format=mdwn + username="http://joeyh.name/" + nickname="joey" + subject="comment 1" + date="2013-05-31T21:17:17Z" + content=""" +More generally, symbolic links cannot have permissions of their own, and git in fact indicates a file is a symlink by setting the tree object's mode to a specific magic number. So there is no git metadata that git-annex can use to track whether a file is supposed to be executable. + +Therefore, this will only get fixed when there's some kind of full-fledged metadata storage for git. +"""]] diff --git a/doc/bugs/assistant_doesn__39__t_sync_file_permissions/comment_2_1a364c422e0dd7418f74e1cc3d543a3c._comment b/doc/bugs/assistant_doesn__39__t_sync_file_permissions/comment_2_1a364c422e0dd7418f74e1cc3d543a3c._comment new file mode 100644 index 0000000000..8585e309c1 --- /dev/null +++ b/doc/bugs/assistant_doesn__39__t_sync_file_permissions/comment_2_1a364c422e0dd7418f74e1cc3d543a3c._comment @@ -0,0 +1,8 @@ +[[!comment format=mdwn + username="https://www.google.com/accounts/o8/id?id=AItOawkx5V3MTbzCXS3J7Mn9FEq8M9bPPYMkAHY" + nickname="Pedro" + subject="comment 2" + date="2013-05-31T21:45:39Z" + content=""" +Note that the file I created wasn't a symlink. I assume you mention symlinks because git-annex itself checks in symlinks to git instead of actual files? +"""]] diff --git a/doc/bugs/assistant_doesn__39__t_sync_file_permissions/comment_3_4d5ae51b4c7e6177d934d7c9f21b912c._comment b/doc/bugs/assistant_doesn__39__t_sync_file_permissions/comment_3_4d5ae51b4c7e6177d934d7c9f21b912c._comment new file mode 100644 index 0000000000..6bf1894145 --- /dev/null +++ b/doc/bugs/assistant_doesn__39__t_sync_file_permissions/comment_3_4d5ae51b4c7e6177d934d7c9f21b912c._comment @@ -0,0 +1,8 @@ +[[!comment format=mdwn + username="https://www.google.com/accounts/o8/id?id=AItOawmNu4V5fvpLlBhaCUfXXOB0MI5NXwh8SkU" + nickname="Adam" + subject="comment 3" + date="2013-11-02T23:49:49Z" + content=""" +Dropbox handles permissions, e.g. if I \"chmod -x\" a file on one system, it does the same to my other systems. It would be a bit of a step backward if git-annex can't do this. :/ Just my two cents. +"""]] diff --git a/doc/bugs/assistant_doesn__39__t_sync_file_permissions/comment_4_ec7b491ad3c9a1f253d886682bff8b82._comment b/doc/bugs/assistant_doesn__39__t_sync_file_permissions/comment_4_ec7b491ad3c9a1f253d886682bff8b82._comment new file mode 100644 index 0000000000..4b287c9df6 --- /dev/null +++ b/doc/bugs/assistant_doesn__39__t_sync_file_permissions/comment_4_ec7b491ad3c9a1f253d886682bff8b82._comment @@ -0,0 +1,11 @@ +[[!comment format=mdwn + username="https://www.google.com/accounts/o8/id?id=AItOawmkBwMWvNKZZCge_YqobCSILPMeK6xbFw8" + nickname="TobiasTheViking" + subject="comment 4" + date="2014-01-12T19:16:07Z" + content=""" +Yeah, now i just hit this. kinda annoying. + +Couldn't this at least be solved for direct mode repositories? + +"""]] diff --git a/doc/bugs/assistant_doesn__39__t_sync_file_permissions/comment_5_e04ddcd7a3d260c1dac54e41911bb4bc._comment b/doc/bugs/assistant_doesn__39__t_sync_file_permissions/comment_5_e04ddcd7a3d260c1dac54e41911bb4bc._comment new file mode 100644 index 0000000000..73f3d00d86 --- /dev/null +++ b/doc/bugs/assistant_doesn__39__t_sync_file_permissions/comment_5_e04ddcd7a3d260c1dac54e41911bb4bc._comment @@ -0,0 +1,10 @@ +[[!comment format=mdwn + username="http://cstork.org/" + nickname="Chris Stork" + subject="comment 5" + date="2014-01-14T08:33:21Z" + content=""" +@Pedro: What Joey is refering to is that git-annex uses sym-links behind the scenes to store even such a small file. (BTW, that can be changed with the large-file config option, IIRC.) + +@Joey: Even a \"full-fledged metadata storage for git\" wouldn't solve this problem since from git's POV it's still just a sym-link without permissions. +"""]] diff --git a/doc/bugs/assistant_doesn__39__t_sync_file_permissions/comment_6_13db3474113b157b7431eb1c835e5814._comment b/doc/bugs/assistant_doesn__39__t_sync_file_permissions/comment_6_13db3474113b157b7431eb1c835e5814._comment new file mode 100644 index 0000000000..6f7283be26 --- /dev/null +++ b/doc/bugs/assistant_doesn__39__t_sync_file_permissions/comment_6_13db3474113b157b7431eb1c835e5814._comment @@ -0,0 +1,14 @@ +[[!comment format=mdwn + username="Xyem" + ip="178.79.137.64" + subject="comment 6" + date="2014-06-24T10:05:26Z" + content=""" +Now that git-annex has a proper metadata system now, can this be handled? + +Perhaps doing this: + +git annex metadata -s __executable 001 + +could cause git-annex to set the executable bit (on get, fsck etc.) and that be set automatically on git annex add if the file has executable bit set (numbers are a bitmask for which execute bit to set..?). +"""]] diff --git a/doc/bugs/assistant_doesn__39__t_sync_file_permissions/comment_7_3cc6eeb8eae14ac3727b1e420f96ee7d._comment b/doc/bugs/assistant_doesn__39__t_sync_file_permissions/comment_7_3cc6eeb8eae14ac3727b1e420f96ee7d._comment new file mode 100644 index 0000000000..8b62a7d93c --- /dev/null +++ b/doc/bugs/assistant_doesn__39__t_sync_file_permissions/comment_7_3cc6eeb8eae14ac3727b1e420f96ee7d._comment @@ -0,0 +1,10 @@ +[[!comment format=mdwn + username="yarikoptic" + avatar="http://cdn.libravatar.org/avatar/f11e9c84cb18d26a1748c33b48c924b4" + subject="no generic solution is possible in indirect mode BUT still would be nice to have a 99% solution" + date="2018-08-03T21:51:22Z" + content=""" +just ran into this as well, so was looking around. +I am afraid that in indirect mode no \"proper\" solution is possible since for the same content (git-annex key) there could originally be multiple files with different permissions -- e.g. one executable and one not. +**BUT** IMHO even though no proper solution possible, if would indeed be very useful to have it resolved to work for 99% of cases, where such collisions aren't likely and a \"union\" of executable bit across present files in the repo could be used (so if one is executable, all others with the same content are as well). Since git annex by default inherits/propagates metadata changes across \"editions\" of the files it would already be handy even if e.g. executable shell scripts gets modified which is kinda a neat side effect ;-) +"""]] diff --git a/doc/bugs/assistant_expensive_scan_unnecessarily_queues_files.mdwn b/doc/bugs/assistant_expensive_scan_unnecessarily_queues_files.mdwn new file mode 100644 index 0000000000..0196b7d7d4 --- /dev/null +++ b/doc/bugs/assistant_expensive_scan_unnecessarily_queues_files.mdwn @@ -0,0 +1,65 @@ +### Please describe the problem. +Summary: The assistant pulls files from remote repos although they are stored on trusted repos (remote machine may be up or down). + +After starting the assistant via "git annex webapp --listen #ip# --verbose --debug" it starts to check for pending tasks and starts to queue lots of files (could be all of them). I checked the daemon.log and found the following line: + +[[!format sh """ +[2014-06-03 21:39:04 CEST] TransferScanner: queued Download UUID "541d2f88-16c3-11e2-aa7e-8f1a2c8e14c5" apps/Apache_OpenOffice_4.0.0_MacOS_x86_install_en-US.dmg Nothing : expensive scan found missing object +"""]] + +The file is available in a remote repo: + +[[!format sh """ +$ git annex whereis apps/Apache_OpenOffice_4.0.0_MacOS_x86_install_en-US.dmg +whereis apps/Apache_OpenOffice_4.0.0_MacOS_x86_install_en-US.dmg (1 copy) + 541d2f88-16c3-11e2-aa7e-8f1a2c8e14c5 -- dump +ok +"""]] + +The remote "dump" is trusted and is configured as "backup" repo. The repo running the assistant is configured as "manual". The configuration settings for wanted, required and scheduled remained untouched. + +Num copies is unset and defaults to 1: + +[[!format sh """ +$ git annex numcopies +global numcopies is not set +(default is 1) +"""]] + +When calling "git annex find --want-get" no files are listed as in my case, all are stored in the "dump" repo which is trusted. + +[[!format sh """ +$ git annex find --want-get +$ +"""]] + +### What steps will reproduce the problem? +Start the assistant and wait for it to finish startup checking. The assistant adds lots (all?) of the files due to “expensive scan found missing object”. + +### What version of git-annex are you using? On what operating system? +Snapshot build from the annex branchable webpage running on an OSX 10.9.3 machine: +[[!format sh """ +$ git annex version +git-annex version: 5.20140411-g2503f43 +build flags: Assistant Webapp Webapp-secure Pairing Testsuite S3 WebDAV FsEvents XMPP DNS Feeds Quvi TDFA CryptoHash +key/value backends: SHA256E SHA1E SHA512E SHA224E SHA384E SKEIN256E SKEIN512E SHA256 SHA1 SHA512 SHA224 SHA384 SKEIN256 SKEIN512 WORM URL +remote types: git gcrypt S3 bup directory rsync web webdav tahoe glacier hook external +local repository version: 5 +supported repository version: 5 +upgrade supported from repository versions: 0 1 2 4 +"""]] + +Note: Updated to 5.20140529-g68a56a6 but same behavior is observed. + +### Please provide any additional information below. + +The backup node was down while the assistant starts up. However also files from another trusted node that is online 24/7 is pulled. Repo was upgraded from version 4 to 5. Is there a way to further check why a file was added e.g. to dump all variables of the evaluated term (approxlackingcopies, copies, etc)? + +[[!format sh """ +[2014-06-03 21:39:04 CEST] TransferScanner: starting scan of [Remote { name ="origin" },Remote { name ="dump" }] +[2014-06-03 21:39:04 CEST] read: git ["--git-dir=/Volumes/DATA/annex/images/.git","--work-tree=/Volumes/DATA/annex/images","ls-files","--cached","-z","--"] +[2014-06-03 21:39:04 CEST] chat: git ["--git-dir=/Volumes/DATA/annex/images/.git","--work-tree=/Volumes/DATA/annex/images","check-attr","-z","--stdin","annex.backend","annex.numcopies","--"] +[2014-06-03 21:39:04 CEST] TransferScanner: queued Download UUID "541d2f88-16c3-11e2-aa7e-8f1a2c8e14c5" apps/Apache_OpenOffice_4.0.0_MacOS_x86_install_en-US.dmg Nothing : expensive scan found missing object +[2014-06-03 21:39:04 CEST] Transferrer: Transferring: Download UUID "541d2f88-16c3-11e2-aa7e-8f1a2c8e14c5" apps/Apache_OpenOffice_4.0.0_MacOS_x86_install_en-US.dmg Nothing +[2014-06-03 21:39:04 CEST] TransferScanner: queued Download UUID "541d2f88-16c3-11e2-aa7e-8f1a2c8e14c5" apps/Apache_OpenOffice_4.0.1_MacOS_x86_install_en-US.dmg Nothing : expensive scan found missing object +"""]] diff --git a/doc/bugs/assistant_expensive_scan_unnecessarily_queues_files/comment_1_bc874e077009b81ab36a6d43d793030a._comment b/doc/bugs/assistant_expensive_scan_unnecessarily_queues_files/comment_1_bc874e077009b81ab36a6d43d793030a._comment new file mode 100644 index 0000000000..91c8616c26 --- /dev/null +++ b/doc/bugs/assistant_expensive_scan_unnecessarily_queues_files/comment_1_bc874e077009b81ab36a6d43d793030a._comment @@ -0,0 +1,10 @@ +[[!comment format=mdwn + username="http://joeyh.name/" + ip="108.236.230.124" + subject="comment 1" + date="2014-06-12T20:07:30Z" + content=""" +This seems to me to be likely to be some kind of configuration problem. In particular, what is the \"wanted\" setting for the repo running the assistant? If it's not \"standard\" then the fact that it's in the \"manual\" group won't affect the assistant's behavior. + +If that's not it, it would probably help to paste the entire vicfg display, so I can replicate the config (or share the repo). +"""]] diff --git a/doc/bugs/assistant_expensive_scan_unnecessarily_queues_files/comment_2_a7e9d8ec500399dd6794e168f85e6a5c._comment b/doc/bugs/assistant_expensive_scan_unnecessarily_queues_files/comment_2_a7e9d8ec500399dd6794e168f85e6a5c._comment new file mode 100644 index 0000000000..589086f6e1 --- /dev/null +++ b/doc/bugs/assistant_expensive_scan_unnecessarily_queues_files/comment_2_a7e9d8ec500399dd6794e168f85e6a5c._comment @@ -0,0 +1,40 @@ +[[!comment format=mdwn + username="https://www.google.com/accounts/o8/id?id=AItOawmXSkgjC_ypUQafVwvHTLsStrkiXH8CfHU" + nickname="Matthias" + subject="comment 2" + date="2014-06-13T18:30:25Z" + content=""" +Thanks for your fast reply, Joey. I appended the shortened config below. Wanted and groupwanted were untouched and therefore still commented out. I removed the hash from wanted to explicitly set it standard. Now the assistant behaves as expected. However I’m wondering: + +* Shouldn’t ‘standard’ be the default rule to apply for wanted/groupwanted in case both are not explicitly set in the config? I assumed this to be the case. +* Shouldn’t “find --want-get” list the identical set of files that is also requested by the assistant? Especially this one confused me. + +[[!format sh \"\"\" +# Repository trust configuration +# (for macbook) +#trust 3938865a-ce40-4166-a918-2012078846c7 = semitrusted + +# Repository groups +# (for macbook) +group 3938865a-ce40-4166-a918-2012078846c7 = manual + +# Repository preferred contents +# (for macbook) +#wanted 3938865a-ce40-4166-a918-2012078846c7 = standard + +# Group preferred contents +# (Used by repositories with \"groupwanted\" in their preferred contents) +(config defaults - all commented out) + +# Standard preferred contents +(config defaults - all commented out) + +# Repository required contents +# (for macbook) +#required 3938865a-ce40-4166-a918-2012078846c7 = + +# Scheduled activities +# (for macbook) +#schedule 3938865a-ce40-4166-a918-2012078846c7 = +\"\"\"]] +"""]] diff --git a/doc/bugs/assistant_expensive_scan_unnecessarily_queues_files/comment_3_a0824e00f04f95c39823f29d6c76e7fe._comment b/doc/bugs/assistant_expensive_scan_unnecessarily_queues_files/comment_3_a0824e00f04f95c39823f29d6c76e7fe._comment new file mode 100644 index 0000000000..58f2069c18 --- /dev/null +++ b/doc/bugs/assistant_expensive_scan_unnecessarily_queues_files/comment_3_a0824e00f04f95c39823f29d6c76e7fe._comment @@ -0,0 +1,10 @@ +[[!comment format=mdwn + username="http://joeyh.name/" + ip="108.236.230.124" + subject="comment 3" + date="2014-06-13T18:35:53Z" + content=""" +I need to look into why --want-get didn't behave as expected. + +It would probably make sense to make standard be the default. There could be upgrade concerns though. +"""]] diff --git a/doc/bugs/assistant_ignores_foreground_switch.mdwn b/doc/bugs/assistant_ignores_foreground_switch.mdwn new file mode 100644 index 0000000000..b4e50ddba2 --- /dev/null +++ b/doc/bugs/assistant_ignores_foreground_switch.mdwn @@ -0,0 +1,35 @@ +### Please describe the problem. + +git annex assistant ignores the --foreground switch: + +``` +florian@marduk ~ % /usr/bin/git-annex assistant --foreground --autostart +git-annex autostart in /home/florian/.synced_configuration +ok +florian@marduk ~ % +``` + +The assistant ist started, but forks to background. This also breaks the systemd unit from https://git-annex.branchable.com/tips/Systemd_unit/ (add Type=forking to the service works around that) + +It seems to be connected to the autostart option, without --autostart it does not go background. + +### What steps will reproduce the problem? + + +### What version of git-annex are you using? On what operating system? + +6.20170101-15 at Arch. + +### Please provide any additional information below. + +[[!format sh """ +# If you can, paste a complete transcript of the problem occurring here. +# If the problem is with the git-annex assistant, paste in .git/annex/daemon.log + + +# End of transcript or log. +"""]] + +### Have you had any luck using git-annex before? (Sometimes we get tired of reading bug reports all day and a lil' positive end note does wonders) + +> Made it support this combination of switches. [[done]] --[[Joey]] diff --git a/doc/bugs/assistant_ignores_foreground_switch/comment_1_d9c812d6ea52e9fd45a7aed94921dc8a._comment b/doc/bugs/assistant_ignores_foreground_switch/comment_1_d9c812d6ea52e9fd45a7aed94921dc8a._comment new file mode 100644 index 0000000000..ab7dbb2e42 --- /dev/null +++ b/doc/bugs/assistant_ignores_foreground_switch/comment_1_d9c812d6ea52e9fd45a7aed94921dc8a._comment @@ -0,0 +1,10 @@ +[[!comment format=mdwn + username="joey" + subject="""comment 1""" + date="2017-02-07T17:10:05Z" + content=""" +--autostart can start multiple daemons, one per registered repository. + +So, --autostart --foreground would need to start the daemons and child +processes, and then wait on them all. +"""]] diff --git a/doc/bugs/assistant_locked_my_files.mdwn b/doc/bugs/assistant_locked_my_files.mdwn new file mode 100644 index 0000000000..07bafd6cc2 --- /dev/null +++ b/doc/bugs/assistant_locked_my_files.mdwn @@ -0,0 +1,30 @@ +### Please describe the problem. + +When the assistant is running, using "git annex unlock" on the commandline may lead to unexpected results. + +### What steps will reproduce the problem? + +Make sure the assitant is running on a repository. + +`git annex unlock somefiles`, try to edit them, wait a while, try to edit. + +If you're not lucky, the assistant will notice the unlocked files as new and will add them back, locking them in the process, and you won't be able to save them. + +### What version of git-annex are you using? On what operating system? + +debian wheezy. + +[[!format sh """ +git-annex version: 5.20131109-gf2cb5b9 +build flags: Assistant Webapp Pairing Testsuite S3 WebDAV Inotify DBus XMPP DNS Feeds Quvi TDFA CryptoHash +key/value backends: SHA256E SHA1E SHA512E SHA224E SHA384E SKEIN256E SKEIN512E SHA256 SHA1 SHA512 SHA224 SHA384 SKEIN256 SKEIN512 WORM URL +remote types: git gcrypt S3 bup directory rsync web webdav glacier hook +local repository version: 3 +default repository version: 3 +supported repository versions: 3 5 +upgrade supported from repository versions: 0 1 2 4 +"""]] + +### Please provide any additional information below. + +[[!tag confirmed]] diff --git a/doc/bugs/assistant_locked_my_files/comment_1_74ee7ea12df7bfbb44c9d0485f787b73._comment b/doc/bugs/assistant_locked_my_files/comment_1_74ee7ea12df7bfbb44c9d0485f787b73._comment new file mode 100644 index 0000000000..fdf921c471 --- /dev/null +++ b/doc/bugs/assistant_locked_my_files/comment_1_74ee7ea12df7bfbb44c9d0485f787b73._comment @@ -0,0 +1,8 @@ +[[!comment format=mdwn + username="http://joeyh.name/" + ip="209.250.56.64" + subject="comment 1" + date="2013-12-05T20:46:30Z" + content=""" +Right -- you can't use the assistant at the same time as git-annex commands that get/drop/unlock files, unless you set the assistant to run in manual mode (which prevents it undoing a get/drop) and/or disable syncing in the assistant entirely (which prevents it adding back unlocked files). +"""]] diff --git a/doc/bugs/assistant_memory_leak.mdwn b/doc/bugs/assistant_memory_leak.mdwn new file mode 100644 index 0000000000..ca23cf8ffb --- /dev/null +++ b/doc/bugs/assistant_memory_leak.mdwn @@ -0,0 +1,22 @@ +### Please describe the problem. + +The assistant is using gruesome amounts of resident memory: + +
    +USER       PID %CPU %MEM    VSZ   RSS TTY      STAT START   TIME COMMAND
+www-data 23898 29.0 90.4 7842888 5536740 ?     Sl   Jun17 2810:08 /usr/lib/git-annex.linux/exe/git-annex --library-path /usr/lib/git-annex.linu
+
    + +I had to stop the assistant because it ended up using all memory. + +### What steps will reproduce the problem? + +Unclear. The assistant has been running for a while and there's a big tansfer (~800GB) of files in progress. + +### What version of git-annex are you using? On what operating system? + +5.20150610+gitg608172f-1~ndall+1 on Debian 7 Wheezy. + +### Please provide any additional information below. + +daemon.log: http://paste2.org/YJVGvpy5 --[[anarcat]] diff --git a/doc/bugs/assistant_should_set_up_dedicated_ssh_keys_for_remotes_using_the_standalone_tarball.mdwn b/doc/bugs/assistant_should_set_up_dedicated_ssh_keys_for_remotes_using_the_standalone_tarball.mdwn new file mode 100644 index 0000000000..7e605ba7cc --- /dev/null +++ b/doc/bugs/assistant_should_set_up_dedicated_ssh_keys_for_remotes_using_the_standalone_tarball.mdwn @@ -0,0 +1,21 @@ +git remotes that have git-annex and git "installed" using the linux +standalone tarball (or the OSX dmg) may not have git-receive-pack in PATH. + +To make `git annex sync` and `git push` etc work on such a remote, +it needs to have a dedicated ssh key that runs git-annex-shell and uses it +to run the command. + +The assistant sets that up when it sets up a dedicated ssh key. However, if +the server can be sshed into without a password using an existing ssh key, +the assistant re-uses that key. + +The assistant should either always set up a dedicated ssh key, even when +not otherwise needed, or it should probe to see if git-receive-pack and +git-send-pack can be run as part of its probing, and set up a dedicated +ssh key when they cannot be. --[[Joey]] + +> After recent sshpassword changes, the webapp sets up a dedicated ssh key +> by default. If the user chooses to use an existing ssh key, it will be +> used. So this is less likely to be a problem. --[[Joey]] + +[[!tag confirmed]] diff --git a/doc/bugs/assistant_sometimes_does_not_merge_changes_from_remote.mdwn b/doc/bugs/assistant_sometimes_does_not_merge_changes_from_remote.mdwn new file mode 100644 index 0000000000..1abc7ffcb2 --- /dev/null +++ b/doc/bugs/assistant_sometimes_does_not_merge_changes_from_remote.mdwn @@ -0,0 +1,12 @@ +This bug seems to be somewhat reproducible: + +1. Set up ~/annex using webapp, keep empty. +2. Add another local repo, eg /tmp/annex, and combine the 2 repositories. + +Now files written to ~/annex should quickly appear in /tmp/annex, and +vice-versa. However, this seems not to happen at first. There's a pause +for some reason until changes start propigating. Apparently a push from +the repo that got the file to the other repo doesn't happen. +After this pause, syncing of changes starts happening quickly, as expected. + +--[[Joey]] diff --git a/doc/bugs/autoenable__61__true_seems_to_not_work_any_longer.mdwn b/doc/bugs/autoenable__61__true_seems_to_not_work_any_longer.mdwn new file mode 100644 index 0000000000..86ecda2ed2 --- /dev/null +++ b/doc/bugs/autoenable__61__true_seems_to_not_work_any_longer.mdwn @@ -0,0 +1,79 @@ +### Please describe the problem. + +autoenable for special remotes seems has stopped working (didn't check with prev versions though, so it might be just something changed in how repository is setup or smth else) + +### What version of git-annex are you using? On what operating system? + +6.20160902+gitgbc49d8a-1~ndall+1 + +### Please provide any additional information below. + +as you can see below autoenable=true is set for that remote, and it enables manually just fine + +[[!format sh """ +(venv-tests) % rm -rf fbirn_phaseIII; git clone http://datasets.datalad.org/nidm/fbirn_phaseIII/.git +Cloning into 'fbirn_phaseIII'... +Checking connectivity... done. + +(venv-tests) % cd fbirn_phaseIII + +(venv-tests) % git annex info +(merging origin/git-annex into git-annex...) +(recording state in git...) +repository mode: indirect +trusted repositories: 0 +semitrusted repositories: 6 + 00000000-0000-0000-0000-000000000001 -- web + 00000000-0000-0000-0000-000000000002 -- bittorrent + 225f46f1-c353-48ce-89da-ccc94dc59d01 -- yoh@falkor:/srv/datasets.datalad.org/www/nidm/fbirn_phaseIII [origin] + 72ce8ab3-19bd-4cef-95b0-5b150c53edc1 -- datalad-archives + d3ceb488-0266-4464-985d-4d4a265e4144 -- yoh@smaug:/mnt/datasets/datalad/crawl/nidm/fbirn_phaseIII + f779a37c-96a5-43b5-822b-0010651dc7b1 -- yoh@hopa:/tmp/autoenable/fbirn_phaseIII [here] +untrusted repositories: 0 +transfers in progress: none +available local disk space: 1.2 gigabytes (+1 megabyte reserved) +local annex keys: 0 +local annex size: 0 bytes +annexed files in working tree: 7521 +size of annexed files in working tree: 2.76 gigabytes +bloom filter size: 32 mebibytes (0% full) +backend usage: + MD5E: 7521 + +(venv-tests) % echo 'git-annex:remote.log' | git cat-file --batch +951989a46d53a17d9a2621f6af82def73c2dc96e blob 328 +72ce8ab3-19bd-4cef-95b0-5b150c53edc1 autoenable=true encryption=none externaltype=datalad-archives name=datalad-archives type=external timestamp=1473266618.950662s +72ce8ab3-19bd-4cef-95b0-5b150c53edc1 autoenable=true encryption=none externaltype=datalad-archives name=datalad-archives type=external timestamp=1473444735.988475s + +(venv-tests) % git annex enableremote datalad-archives +enableremote datalad-archives ok +(recording state in git...) +(venv-tests) % git annex info +repository mode: indirect +trusted repositories: 0 +semitrusted repositories: 6 + 00000000-0000-0000-0000-000000000001 -- web + 00000000-0000-0000-0000-000000000002 -- bittorrent + 225f46f1-c353-48ce-89da-ccc94dc59d01 -- yoh@falkor:/srv/datasets.datalad.org/www/nidm/fbirn_phaseIII [origin] + 72ce8ab3-19bd-4cef-95b0-5b150c53edc1 -- [datalad-archives] + d3ceb488-0266-4464-985d-4d4a265e4144 -- yoh@smaug:/mnt/datasets/datalad/crawl/nidm/fbirn_phaseIII + f779a37c-96a5-43b5-822b-0010651dc7b1 -- yoh@hopa:/tmp/autoenable/fbirn_phaseIII [here] +untrusted repositories: 0 +transfers in progress: none +available local disk space: 1.2 gigabytes (+1 megabyte reserved) +local annex keys: 0 +local annex size: 0 bytes +annexed files in working tree: 7521 +size of annexed files in working tree: 2.76 gigabytes +bloom filter size: 32 mebibytes (0% full) +backend usage: + MD5E: 7521 + +"""]] + +I am a little confused though since we do test for this scenario in datalad and test still passes, i.e. remote gets enabled... + + +[[!meta author=yoh]] + +[[!meta title="autoenable not done for implicit init"]] diff --git a/doc/bugs/autoenable__61__true_seems_to_not_work_any_longer/comment_1_457027471d09aa4ae8718c1508cfae1d._comment b/doc/bugs/autoenable__61__true_seems_to_not_work_any_longer/comment_1_457027471d09aa4ae8718c1508cfae1d._comment new file mode 100644 index 0000000000..0c8585d602 --- /dev/null +++ b/doc/bugs/autoenable__61__true_seems_to_not_work_any_longer/comment_1_457027471d09aa4ae8718c1508cfae1d._comment @@ -0,0 +1,27 @@ +[[!comment format=mdwn + username="joey" + subject="""comment 1""" + date="2016-09-21T17:37:39Z" + content=""" +`git annex init` does handle autoenable. When you bypass explicit init, +it does not do autoenabling. + +This is not a change AFAICS. The changelog entry for autoenable +says that it's done by `git annex init`. Presumably your test suite +does run `git annex init`. + +My original notes on why not to have automaitic init handle autoenable were: + +> There was also the question of what to do when git-annex auto-inits +> in a clone of a repository. It wouldn't do for a command like +> `git annex find`'s output to include any messages that might be shown +> while auto-enabling special remotes as a result of an auto-init. +> Since I can't guarantee enabling special remotes will be quiet, I've not +> tried to auto-enable special remotes in this case. +> +> I think I'd have to +> exec a git-annex init process with stdout sent to stderr to implement +> this in a safe way, and due to calls to ensureInitialized in Remote.Git, +> which can auto-init a local remote, that gets particularly tricky. Best, +> I feel, to wait and see if anyone needs that. +"""]] diff --git a/doc/bugs/automatically_adding_metadata_fails_on_files_with_UTF-8_umlauts.mdwn b/doc/bugs/automatically_adding_metadata_fails_on_files_with_UTF-8_umlauts.mdwn new file mode 100644 index 0000000000..ba55b41d17 --- /dev/null +++ b/doc/bugs/automatically_adding_metadata_fails_on_files_with_UTF-8_umlauts.mdwn @@ -0,0 +1,70 @@ +### Please describe the problem. +The recommended hook script does silently ignore files with non-ascii filenames. + +### What steps will reproduce the problem? +see below. + +### What version of git-annex are you using? On what operating system? +git-annex version: 6.20180112 +on NixOS i686 + +### Please provide any additional information below. +[[!format txt """ +[woffs@lapdoepp:/tmp] $ git init test +Leeres Git-Repository in /tmp/test/.git/ initialisiert + +[woffs@lapdoepp:/tmp] $ cd test +[woffs@lapdoepp:/tmp/test] (master #) $ git annex init +init ok +(recording state in git...) +[woffs@lapdoepp:/tmp/test] (master #) $ git config metadata.exiftool "CreateDate Model ImageSize FocusRange GPS FileType: NominalBitrate Title Artist Album Date: TrackNumber Year"l "CreateDate Model ImageSize FocusRange GPS FileType: NominalBitrate Title Artist Album Date: TrackNum + [woffs@lapdoepp:/tmp/test] (master #) $ git config annex.genmetadata true + [woffs@lapdoepp:/tmp/test] (master #) $ wget -O ./.git/hooks/pre-commit-annex http://git-annex.branchable.com/tips/automatically_adding_metadata/pre-commit-annex +--2018-02-26 10:28:42-- http://git-annex.branchable.com/tips/automatically_adding_metadata/pre-commit-annex +Auflösen des Hostnamens git-annex.branchable.com (git-annex.branchable.com)… 66.228.46.55, 2600:3c03::f03c:91ff:fedf:c0e5 +Verbindungsaufbau zu git-annex.branchable.com (git-annex.branchable.com)|66.228.46.55|:80 … verbunden. +HTTP-Anforderung gesendet, auf Antwort wird gewartet … 200 OK +Länge: 3035 (3,0K) +Wird in »./.git/hooks/pre-commit-annex« gespeichert. + +./.git/hooks/pre-commit-annex 100%[=====================================================================================>] 2,96K --.-KB/s in 0s + +2018-02-26 10:28:42 (75,8 MB/s) - »./.git/hooks/pre-commit-annex« gespeichert [3035/3035] + + [woffs@lapdoepp:/tmp/test] (master #) $ chmod +x ./.git/hooks/pre-commit-annex + wget http://git-a[woffs@lapdoepp:/tmp/test] (master #) $ wget http://git-annex.branchable.com/logo_small.png +--2018-02-26 10:28:52-- http://git-annex.branchable.com/logo_small.png +Auflösen des Hostnamens git-annex.branchable.com (git-annex.branchable.com)… 66.228.46.55, 2600:3c03::f03c:91ff:fedf:c0e5 +Verbindungsaufbau zu git-annex.branchable.com (git-annex.branchable.com)|66.228.46.55|:80 … verbunden. +HTTP-Anforderung gesendet, auf Antwort wird gewartet … 200 OK +Länge: 4749 (4,6K) [image/png] +Wird in »logo_small.png« gespeichert. + +logo_small.png 100%[=====================================================================================>] 4,64K --.-KB/s in 0s + +2018-02-26 10:28:52 (108 MB/s) - »logo_small.png« gespeichert [4749/4749] + + [woffs@lapdoepp:/tmp/test] (master #%) $ cp logo_small.png lögö_smäll.png +[woffs@lapdoepp:/tmp/test] (master #%) $ git annex add +add logo_small.png ok +add lögö_smäll.png ok +(recording state in git...) +[woffs@lapdoepp:/tmp/test] (master +) $ git annex sync +commit +adding metadata for logo_small.png +(recording state in git...) +[master (Basis-Commit) e3f8647] git-annex in woffs@lapdoepp:/tmp/test + 2 files changed, 2 insertions(+) + create mode 120000 logo_small.png + create mode 120000 "l\303\266g\303\266_sm\303\244ll.png" +ok + +[woffs@lapdoepp:/tmp/test] (master) $ +"""]] + +### Have you had any luck using git-annex before? (Sometimes we get tired of reading bug reports all day and a lil' positive end note does wonders) + +git annex is great. :) + +> [[fixed|done]]; caused by git diff-index defaulting to not emitting such +> characters, got it to by using -z. --[[Joey]] diff --git a/doc/bugs/autostart__47__git-annex.desktop_gets_resurrected.mdwn b/doc/bugs/autostart__47__git-annex.desktop_gets_resurrected.mdwn new file mode 100644 index 0000000000..6d0c2a87d9 --- /dev/null +++ b/doc/bugs/autostart__47__git-annex.desktop_gets_resurrected.mdwn @@ -0,0 +1,9 @@ +### Please describe the problem. + +Due to the very unstable kernel on my laptop atm I need to reboot quite frequently... and this issue with annex assistant immediately starting synchronizing (although I did disable all synchronizations in webapp, so it would be another issue I guess), I wanted just to disable autostart of git-annex altogether. I have removed $HOME/.config/autostart/git-annex.desktop but to my surprise upon gnome crashing/relogin I had the same ssh key passphrase dialog and that file reappeared. + +### What version of git-annex are you using? On what operating system? + +5.20151116+gitg5416a1a-1~ndall+1 + +[[!meta author=yoh]] diff --git a/doc/bugs/autostart__47__git-annex.desktop_gets_resurrected/comment_1_020c0c7a51638cf28efc518da4ccdb9d._comment b/doc/bugs/autostart__47__git-annex.desktop_gets_resurrected/comment_1_020c0c7a51638cf28efc518da4ccdb9d._comment new file mode 100644 index 0000000000..69de952217 --- /dev/null +++ b/doc/bugs/autostart__47__git-annex.desktop_gets_resurrected/comment_1_020c0c7a51638cf28efc518da4ccdb9d._comment @@ -0,0 +1,13 @@ +[[!comment format=mdwn + username="joey" + subject="""comment 1""" + date="2015-11-30T19:51:45Z" + content=""" +Well, if your system is crashing, the deletion of the file might not be +reflected on the disk or something. + +It is true that, when git-annex is installed via the standalone tarball, it +will try to install the autostart file whenever the assistant is started. +But something has to cause the assistant to start, and if you deleted the +autostart file, it wasn't that .. so what was it? +"""]] diff --git a/doc/bugs/autostart__47__git-annex.desktop_gets_resurrected/comment_2_0378843b9c7281d587304ac9c42d46f5._comment b/doc/bugs/autostart__47__git-annex.desktop_gets_resurrected/comment_2_0378843b9c7281d587304ac9c42d46f5._comment new file mode 100644 index 0000000000..01e468f3f6 --- /dev/null +++ b/doc/bugs/autostart__47__git-annex.desktop_gets_resurrected/comment_2_0378843b9c7281d587304ac9c42d46f5._comment @@ -0,0 +1,8 @@ +[[!comment format=mdwn + username="timothyhobbs@8b50ff958c937fa4b6de1f9009f464b9ddfc2991" + nickname="timothyhobbs" + subject="comment 2" + date="2016-04-04T06:07:20Z" + content=""" +It would be better to prompt the user \"Would you like to automatically start git annex when you log in? Yes No [ ] Do not display again.\" That way user's won't be surprised that there is disk churn when they startup their computer... +"""]] diff --git a/doc/bugs/autostart__47__git-annex.desktop_gets_resurrected/comment_3_935f2fe1f7ff1a46b1038cd4458317a4._comment b/doc/bugs/autostart__47__git-annex.desktop_gets_resurrected/comment_3_935f2fe1f7ff1a46b1038cd4458317a4._comment new file mode 100644 index 0000000000..a0a94fceb0 --- /dev/null +++ b/doc/bugs/autostart__47__git-annex.desktop_gets_resurrected/comment_3_935f2fe1f7ff1a46b1038cd4458317a4._comment @@ -0,0 +1,7 @@ +[[!comment format=mdwn + username="https://me.yahoo.com/a/EbvxpTI_xP9Aod7Mg4cwGhgjrCrdM5s-#7c0f4" + subject="comment 3" + date="2016-05-01T21:47:17Z" + content=""" +FWIW, I think for me it was the /etc/xdg/autostart/git-annex.desktop which kept it coming from dead (and laptop behaves a bit better lately so I don't reboot as often ;) ) +"""]] diff --git a/doc/bugs/awkward_error_from_annex_whenever_operating_on_a_clone_with_submodules.mdwn b/doc/bugs/awkward_error_from_annex_whenever_operating_on_a_clone_with_submodules.mdwn new file mode 100644 index 0000000000..720b0c0698 --- /dev/null +++ b/doc/bugs/awkward_error_from_annex_whenever_operating_on_a_clone_with_submodules.mdwn @@ -0,0 +1,60 @@ +### Please describe the problem. + +git annex init fails on a clone of a sub sub module from a git repo + +### What steps will reproduce the problem? + +This is the archive with sample repo which is used in example below +[http://www.onerussian.com/tmp/repl_submodule.tgz] + + +### What version of git-annex are you using? On what operating system? + +6.20160307+gitgb095561-1~ndall+1 + +[[!format sh """ +> tar -xzf repl_submodule.tgz + +> ls -ld datalad_temp_testrepo_Lb8lMD +drwx------ 5 yoh yoh 180 Mar 29 09:06 datalad_temp_testrepo_Lb8lMD/ + +> git clone datalad_temp_testrepo_Lb8lMD/sub1/sub1 +Cloning into 'sub1'... +done. + +> cd sub1 +INFO.txt sub1/ sub2/ test-annex.dat@ test.dat + +> git annex init +init fatal: Could not switch to '/sub1/': No such file or directory +ok +(recording state in git...) + +> cat .git/config +[core] + repositoryformatversion = 0 + filemode = true + bare = false + logallrefupdates = true +[remote "origin"] + url = /tmp/datalad_temp_testrepo_Lb8lMD/sub1/sub1 + fetch = +refs/heads/*:refs/remotes/origin/* +[branch "master"] + remote = origin + merge = refs/heads/master +[annex] + uuid = 7293cd84-98c5-4f8c-b1ff-1c4fadd90ba3 + version = 5 + +> git annex --debug init 2>&1 | grep -3 "Could not" +[2016-03-29 09:22:21.786669] read: git ["config","--null","--list"] +[2016-03-29 09:22:21.792047] process done ExitSuccess +[2016-03-29 09:22:21.792158] read: git ["--git-dir=../datalad_temp_testrepo_Lb8lMD/sub1/sub1/.git","--work-tree=/sub1/sub1","--literal-pathspecs","show-ref","git-annex"] +fatal: Could not switch to '/sub1/': No such file or directory +[2016-03-29 09:22:21.797597] process done ExitFailure 128 +ok +[2016-03-29 09:22:21.800514] chat: git ["--git-dir=.git","--work-tree=.","--literal-pathspecs","hash-object","-w","--stdin-paths","--no-filters"] + +"""]] + +[[!meta author=yoh]] diff --git a/doc/bugs/awkward_error_from_annex_whenever_operating_on_a_clone_with_submodules/comment_1_86b409dc4b7824f601deeed0a14cc8e4._comment b/doc/bugs/awkward_error_from_annex_whenever_operating_on_a_clone_with_submodules/comment_1_86b409dc4b7824f601deeed0a14cc8e4._comment new file mode 100644 index 0000000000..fb81b5b1e6 --- /dev/null +++ b/doc/bugs/awkward_error_from_annex_whenever_operating_on_a_clone_with_submodules/comment_1_86b409dc4b7824f601deeed0a14cc8e4._comment @@ -0,0 +1,11 @@ +[[!comment format=mdwn + username="joey" + subject="""comment 1""" + date="2016-04-13T18:31:00Z" + content=""" +I'm pretty sure this is a duplicate of +[[git-annex_confuses_Git_with_nested_submodules]], +which was filed by Michael Hanke. + +Error message is not identical but very close. +"""]] diff --git a/doc/bugs/aws_0.16_breaking_changes.mdwn b/doc/bugs/aws_0.16_breaking_changes.mdwn new file mode 100644 index 0000000000..e03180c347 --- /dev/null +++ b/doc/bugs/aws_0.16_breaking_changes.mdwn @@ -0,0 +1,44 @@ +### Please describe the problem. +aws-0.16 and 0.15.1 were just released. 0.16 has breaking changes. + +### What steps will reproduce the problem? +Try to build git-annex with the S3 flag with aws-0.16. + +Note that git-annex builds successfully with 0.15.1, and that 0.15 has its own problems since it constrains xml-conduit to < 1.4, which fails unless blaze-markup is also contrained to < 0.8.0.0. This is resolved by using 0.15.1 since it lifted the contraint on xml-conduit to < 1.5. + +### What version of git-annex are you using? On what operating system? +6.20170101 on macOS + +### Please provide any additional information below. +The build failure is +[[!format sh """ +[323 of 546] Compiling Remote.Glacier ( Remote/Glacier.hs, dist/dist-sandbox-8fbcd4b9/build/git-annex/git-annex-tmp/Remote/Glacier.o ) +[324 of 546] Compiling Remote.Helper.Http ( Remote/Helper/Http.hs, dist/dist-sandbox-8fbcd4b9/build/git-annex/git-annex-tmp/Remote/Helper/Http.o ) +[325 of 546] Compiling Remote.S3 ( Remote/S3.hs, dist/dist-sandbox-8fbcd4b9/build/git-annex/git-annex-tmp/Remote/S3.o ) + +Remote/S3.hs:224:49: error: + • The constructor ‘S3.UploadPartResponse’ should have 1 argument, but has been given 2 + • In the pattern: S3.UploadPartResponse _ etag + In a stmt of a 'do' block: + S3.UploadPartResponse _ etag <- sendS3Handle h req + In the expression: + do { let sz = ...; + let p' = offsetMeterUpdate p (toBytesProcessed pos); + let numchunks + = ceiling + (fromIntegral sz / fromIntegral defaultChunkSize :: Double); + let popper = handlePopper numchunks defaultChunkSize p' fh; + .... } +cabal: Leaving directory '.' +cabal: Error: some packages failed to install: +git-annex-6.20170101 failed during the building phase. The exception was: +ExitFailure 1 +/usr/local/Homebrew/Library/Homebrew/debrew.rb:11:in `raise' +BuildError: Failed executing: cabal install --jobs=8 --max-backjumps=100000 --prefix=/usr/local/Cellar/git-annex/6.20170101_1 --flags=s3\ webapp +"""]] + +### Have you had any luck using git-annex before? (Sometimes we get tired of reading bug reports all day and a lil' positive end note does wonders) + +Yes! + +> [[fixed|done]] thanks --[[Joey]] diff --git a/doc/bugs/aws_0.16_breaking_changes/comment_1_32b0d5d388f2925335a4dd83bae228c7._comment b/doc/bugs/aws_0.16_breaking_changes/comment_1_32b0d5d388f2925335a4dd83bae228c7._comment new file mode 100644 index 0000000000..b4afc0a48c --- /dev/null +++ b/doc/bugs/aws_0.16_breaking_changes/comment_1_32b0d5d388f2925335a4dd83bae228c7._comment @@ -0,0 +1,16 @@ +[[!comment format=mdwn + username="aristidb@4551cc1215222a56e271a796b92908680f3941b5" + nickname="aristidb" + avatar="http://cdn.libravatar.org/avatar/64e6cb35f87f56a7f13bac236afff510" + subject="comment 1" + date="2017-02-05T12:29:27Z" + content=""" +As explained in my duplicate report on : + +I suggest switching to {} pattern matching, like this: + +[[!format haskell \"\"\" +S3.UploadPartResponse { S3.uprEtag = etag } <- sendS3Handle h req +\"\"\"]] + +"""]] diff --git a/doc/bugs/aws_0.16_breaking_changes/comment_2_067794ef74712b4d6071df1ef5f3b314._comment b/doc/bugs/aws_0.16_breaking_changes/comment_2_067794ef74712b4d6071df1ef5f3b314._comment new file mode 100644 index 0000000000..986d3002f4 --- /dev/null +++ b/doc/bugs/aws_0.16_breaking_changes/comment_2_067794ef74712b4d6071df1ef5f3b314._comment @@ -0,0 +1,13 @@ +[[!comment format=mdwn + username="aristidb@4551cc1215222a56e271a796b92908680f3941b5" + nickname="aristidb" + avatar="http://cdn.libravatar.org/avatar/64e6cb35f87f56a7f13bac236afff510" + subject="comment 2" + date="2017-02-05T12:49:22Z" + content=""" +I made a typo in my previous response: It should be ETag, not Etag. Corrected: + +[[!haskell \"\"\" +S3.UploadPartResponse { S3.uprETag = etag } <- sendS3Handle h req +\"\"\"] +"""]] diff --git a/doc/bugs/bare_remote_is_not_automatically_synced_to_local_repo.mdwn b/doc/bugs/bare_remote_is_not_automatically_synced_to_local_repo.mdwn new file mode 100644 index 0000000000..459b08bcb5 --- /dev/null +++ b/doc/bugs/bare_remote_is_not_automatically_synced_to_local_repo.mdwn @@ -0,0 +1,15 @@ +### Please describe the problem. + +I have a git annex repository at `~/doc` which has a remote stored on a second drive at `/media/backup/doc.git`, which is a bare git annex repo. The webapp is run from `~/doc`. Changes are made to `/media/backup/doc.git` by syncing from other computers, but then those changes to the remote are not automatically propagated to `~/doc` by the webapp. + +The use case that lead to this situation is syncing several computers to a large hard drive on a server, utilizing a bare repo as a full backup, and then wishing to have a client repo in the home folder on the server. So other computers sync their repos to `/media/backup/doc.git` and then any changes should be propagated to `~/doc` on the server. + +Changes made to `~/doc` *are* synced to the backup repository on the backup hard drive, and those changes are almost immediately synced to the other machines by the webapp running on those respective machines. Those computers are configured to connect to `/media/backup/doc.git` on the server using SSH only (and git-annex-shell on the server), and do not use XMPP. + +### What version of git-annex are you using? On what operating system? + +The amd64 pre-built tarball, git-annex version: 6.20160613-g35dbe35, running on Ubuntu 16.04. + +### Have you had any luck using git-annex before? (Sometimes we get tired of reading bug reports all day and a lil' positive end note does wonders) + +Migrated all of my syncing to git annex about 10 months ago and haven't had any significant trouble :) diff --git a/doc/bugs/bare_remote_is_not_automatically_synced_to_local_repo/comment_1_66ffaa679ba0fe86bba114fd9cdc5a53._comment b/doc/bugs/bare_remote_is_not_automatically_synced_to_local_repo/comment_1_66ffaa679ba0fe86bba114fd9cdc5a53._comment new file mode 100644 index 0000000000..698338b1b2 --- /dev/null +++ b/doc/bugs/bare_remote_is_not_automatically_synced_to_local_repo/comment_1_66ffaa679ba0fe86bba114fd9cdc5a53._comment @@ -0,0 +1,9 @@ +[[!comment format=mdwn + username="Farhan" + subject="More Info" + date="2016-06-23T12:15:28Z" + content=""" +Can you please provide the following info? For curiosity's sake. + +When you are viewing the repos in the web app, on the right hand side for each repo is a drop down menu labelled \"actions\". Please click there and go to the option labelled \"Edit\". Click this option, and on the resulting page there should be a field labelled \"Repository group\". Can you tell us the \"Repository group\" for the repos you mentioned? What is the \"Repository group\" for the backup on the secondary drive, the repo in your home folder and the remote clients? My hunch is that these group types are not set up to reflect the configuration you want. +"""]] diff --git a/doc/bugs/bare_remote_is_not_automatically_synced_to_local_repo/comment_2_a7617cc63584dafb16bdc9373fb1e304._comment b/doc/bugs/bare_remote_is_not_automatically_synced_to_local_repo/comment_2_a7617cc63584dafb16bdc9373fb1e304._comment new file mode 100644 index 0000000000..01b0e4cd0a --- /dev/null +++ b/doc/bugs/bare_remote_is_not_automatically_synced_to_local_repo/comment_2_a7617cc63584dafb16bdc9373fb1e304._comment @@ -0,0 +1,7 @@ +[[!comment format=mdwn + username="Don" + subject="comment 2" + date="2016-06-24T05:40:23Z" + content=""" +The bare repo on the external drive is configured as `full backup` and the repo at `~/doc` is `client`. +"""]] diff --git a/doc/bugs/bare_remote_is_not_automatically_synced_to_local_repo/comment_3_ecb57e72c0bf583d1c3461313a83c4f2._comment b/doc/bugs/bare_remote_is_not_automatically_synced_to_local_repo/comment_3_ecb57e72c0bf583d1c3461313a83c4f2._comment new file mode 100644 index 0000000000..15662ca519 --- /dev/null +++ b/doc/bugs/bare_remote_is_not_automatically_synced_to_local_repo/comment_3_ecb57e72c0bf583d1c3461313a83c4f2._comment @@ -0,0 +1,13 @@ +[[!comment format=mdwn + username="Farhan" + subject="Thanks" + date="2016-06-24T06:35:10Z" + content=""" +And the other repositories (which are syncing correctly), they are also marked as client? Clients should sync with each other whenever they can connect to each other, but perhaps your client repositories cannot connect to each other? + +[Here is a comment from Joey](https://git-annex.branchable.com/forum/Two_computer_setup__58_____34__transfer__34___or___34__full_backup__34___repository_groups__63__/) about a similar setup. He says you might need to set up an XMPP account for the various clients to message each other when they have new changes? But that was a long time ago. And anyways it seems like the webapp would be doing this automatically (yet in your post you say the computers do not use XMPP?). + +I think everyone would benefit from more info still. For example, what does the webapp look like when you are syncing this repo? Are all the clients and the backup on the same page? Is each client only combined to the backup itself and no other repos? What kind of remote repos are the other computers? + +Hope you don't mind the questions, I'm also curious about this issue myself. Thanks for bringing it up. +"""]] diff --git a/doc/bugs/bare_remote_is_not_automatically_synced_to_local_repo/comment_4_bf0e16f9181438c2050521c5ff5bb7d0._comment b/doc/bugs/bare_remote_is_not_automatically_synced_to_local_repo/comment_4_bf0e16f9181438c2050521c5ff5bb7d0._comment new file mode 100644 index 0000000000..e34ea04670 --- /dev/null +++ b/doc/bugs/bare_remote_is_not_automatically_synced_to_local_repo/comment_4_bf0e16f9181438c2050521c5ff5bb7d0._comment @@ -0,0 +1,14 @@ +[[!comment format=mdwn + username="Don" + subject="comment 4" + date="2016-06-24T07:16:20Z" + content=""" +Good questions. The setup is essentially a star topology where the central node is the full-backup, bare git repository on the external drive on the server. It's not encrypted, if that matters. Since it's a bare repo, nothing actually changes in that node except when changes are pushed from elsewhere. + +Every other node is configured as a client and is connected to that repo on the server via SSH. When a change is made to one of the client repos, syncing is happening automatically and nearly instantaneously--not just to the server repo, but the changes are very quickly propagated to the other SSH-connected clients. (I think this is supposed to happen for SSH repos now without using XMPP. In the release notes for version 5.20140421: \"This release begins to deprecate XMPP support. In particular, if you use the assistant with a ssh remote that has this version of git-annex installed, you don't need XMPP any longer to get immediate syncing of changes.\") + +But then there's this one other node in the star topology---the client repo (non-bare) that is actually on the same system as the bare, backup repo. When changes are made to the client, they are synced immediately to the backup repo, and those changes are quickly synced to the other client repos on other computers connected by SSH. The problem is changes in the other direction; when a client repo on a remote system is changed, the updates are synced to the backup repo on the server, but no farther. The webapp running on the server never (or only very slowly) syncs those changes to that client repo that is on the same system. The webapp on the server is running with that local client repo as the \"main\" repo (I mean, the upper right hand corner says \"Repository: ~/doc\", which is the client repo). All of the status messages are green and say that the repos are synced. You can, of course, force a sync from there, and and then the changes are noticed and propagated to the client. + +I did some testing and it seems like repositories with non-bare remotes on the same filesystem are synced immediately using inotify or something. Maybe this just doesn't happen for bare repos? That's the variable that seems to make syncing not happen automatically, in my limited testing. I can test more to isolate exactly when it happens, but I was hoping Joey would know off the top of his head if this should be working or not. + +"""]] diff --git a/doc/bugs/bare_remote_is_not_automatically_synced_to_local_repo/comment_5_aeea3ff59c8493c87c313fbaf7614f20._comment b/doc/bugs/bare_remote_is_not_automatically_synced_to_local_repo/comment_5_aeea3ff59c8493c87c313fbaf7614f20._comment new file mode 100644 index 0000000000..92241bf51a --- /dev/null +++ b/doc/bugs/bare_remote_is_not_automatically_synced_to_local_repo/comment_5_aeea3ff59c8493c87c313fbaf7614f20._comment @@ -0,0 +1,36 @@ +[[!comment format=mdwn + username="joey" + subject="""comment 5""" + date="2017-06-07T16:40:18Z" + content=""" +This has nothing to do with repository groups; it's the git branch +not getting synced by the assistant when master is updated in the +bare repository. + +This is easy enough to repoduce. + +The assistant relies on `git annex remotedaemon` to notice changes in +remotes, and the remotedaemon only supports remotes accessed via ssh, not +on a local drive. + +The assistant has some special case handling for repositories on +removable drives, which lets those be plugged in and syncs with them. +Removable drives are also why the remotedaemon does not watch repositories +on a local drive. If the drive is removable, the remotedaemon would +keep it always busy, by having directories open for inotify, +and so prevent a clean removal of the drive. + +So, I don't think anything can be done about this, at least in the default +configuration. One workaround it using a ssh url for the remote will +make the remotedaemon watch it for changes. + +I could add a non-default configuration setting that +makes a remote on a local drive be watched for changes. + +But, it's not clear to me why you'd want to have the changes be pushed +in to a repository on a removable drive, rather than pushing/syncing +with the repository where the assistant is running. The assistant would +then sync changes it received on to the removable drive. That seems like a +better setup, will still work when the drive is removed for a while, +and is already supported. +"""]] diff --git a/doc/bugs/bash__58___git-annex-shell__58___command_not_found.mdwn b/doc/bugs/bash__58___git-annex-shell__58___command_not_found.mdwn new file mode 100644 index 0000000000..cc784bd2d3 --- /dev/null +++ b/doc/bugs/bash__58___git-annex-shell__58___command_not_found.mdwn @@ -0,0 +1,60 @@ +### Please describe the problem. + +I run a script (via cron) to sync my various git-annex repositories. Recently, I noticed the following error message (which I'd not seen before): + + bash: git-annex-shell: command not found + +I can't say for certain when it first appeared, but I first noticed it after recently upgrading to git-annex 6.20170519. + +### What steps will reproduce the problem? + + git annex sync -J5 + +(or using any other 'jobs' option via -Jx or --jobs=x, including setting 'jobs' to '1') + +### What version of git-annex are you using? On what operating system? + +6.20170519 on MacOS Sierra (installed via homebrew) + +### Please provide any additional information below. + +git annex sync (without a 'jobs' option) does not produce the error message. + +In the sample transcripts below, RAID10 is a local drive. unraid, manuel & drobo are all accessed via ssh. + +[[!format sh """ +# If you can, paste a complete transcript of the problem occurring here. +# If the problem is with the git-annex assistant, paste in .git/annex/daemon.log + +[tom:~/annex/dl] git annex sync -J1 +On branch master +nothing to commit, working tree clean +commit ok +bash: git-annex-shell: command not found +pull RAID10 ok +pull unraid ok +pull manuel ok +pull drobo ok + +[tom:~/annex/dl] git annex sync +commit +On branch master +nothing to commit, working tree clean +ok +pull RAID10 +ok +pull unraid +ok +pull manuel +ok +pull drobo +ok + +# End of transcript or log. +"""]] + +### Have you had any luck using git-annex before? (Sometimes we get tired of reading bug reports all day and a lil' positive end note does wonders) + +I've used git-annex for well over a year, syncing multiple repositories, without a problem. I don't know how I ever got along without it! :-) + +> [[fixed|done]] --[[Joey]] diff --git a/doc/bugs/bash__58___git-annex-shell__58___command_not_found/comment_1_461a8d85738d63f9d276bb33d5c24255._comment b/doc/bugs/bash__58___git-annex-shell__58___command_not_found/comment_1_461a8d85738d63f9d276bb33d5c24255._comment new file mode 100644 index 0000000000..fda3bdf3c3 --- /dev/null +++ b/doc/bugs/bash__58___git-annex-shell__58___command_not_found/comment_1_461a8d85738d63f9d276bb33d5c24255._comment @@ -0,0 +1,27 @@ +[[!comment format=mdwn + username="joey" + subject="""comment 1""" + date="2017-06-06T16:21:53Z" + content=""" +One of the remotes of your repository is on a host +where git-annex-shell is either not installed, or perhaps +it is installed but `ssh thehost git-annex-shell` does not +work -- eg due to the problem decribed in +[[tips/get_git-annex-shell_into_PATH]]. + +There was indeed a change in 6.20170519 that explains why this +only happens in -J mode. [[!commit 6992fe133b37ec6d64498f3dd2c69613c4c37469]] +made it run git-annex-shell at startup in that mode. + +Hmm, I suppose one of your remotes could intentionally not have +git-annex-shell on it, and yet you'd still want `git annex sync` +to work to it, and so this message about git-annex-shell +being displayed is not ideal. + +So, I've made it skip trying to run git-annex-shell in this case unless +the remote has an annex-uuid set. If the remote has never had +git-annex-shell installed, it can't have an annex-uuid set. + +And, when it does fail to run git-annex-shell, I've made it say which +remote it was unable to run it on. +"""]] diff --git a/doc/bugs/broken_repo_when_inodes_exhausted.mdwn b/doc/bugs/broken_repo_when_inodes_exhausted.mdwn new file mode 100644 index 0000000000..3c03478811 --- /dev/null +++ b/doc/bugs/broken_repo_when_inodes_exhausted.mdwn @@ -0,0 +1,62 @@ +### Please describe the problem. +I found one of my repos damaged after letting the webapp sync with it for a while. The filesystem inodes were exhausted and all the refs pointed to objects that do not exist. + +### What steps will reproduce the problem? +I expect this issue to be reproduced by syncing to a filesystem without enough free inodes to hold the incoming files. + +### What version of git-annex are you using? On what operating system? +This repo is using git-annex 6.20160229-g37a89cc + +### Please provide any additional information below. + +[[!format sh """ +# If you can, paste a complete transcript of the problem occurring here. +# If the problem is with the git-annex assistant, paste in .git/annex/daemon.log + +$ git status +fatal: bad object HEAD +$ git pull +error: refs/heads/master does not point to a valid object! +error: refs/heads/synced/git-annex does not point to a valid object! +error: refs/heads/synced/master does not point to a valid object! +error: refs/heads/master does not point to a valid object! +error: refs/heads/synced/git-annex does not point to a valid object! +error: refs/heads/synced/master does not point to a valid object! +error: refs/heads/master does not point to a valid object! +error: refs/heads/synced/git-annex does not point to a valid object! +error: refs/heads/synced/master does not point to a valid object! +error: refs/heads/master does not point to a valid object! +error: refs/heads/synced/git-annex does not point to a valid object! +error: refs/heads/synced/master does not point to a valid object! +remote: Counting objects: 3, done. +remote: Compressing objects: 100% (3/3), done. +remote: Total 3 (delta 0), reused 0 (delta 0) +Unpacking objects: 100% (3/3), done. +error: refs/heads/master does not point to a valid object! +error: refs/heads/synced/git-annex does not point to a valid object! +error: refs/heads/synced/master does not point to a valid object! +fatal: bad object HEAD +error: /home/olpc/annex-ext4 did not send all necessary objects + +Auto packing the repository in background for optimum performance. +See "git help gc" for manual housekeeping. +error: The last gc run reported the following. Please correct the root cause +and remove .git/gc.log. +Automatic cleanup will not be performed until the file is removed. + +error: refs/heads/master does not point to a valid object! +error: refs/heads/synced/git-annex does not point to a valid object! +error: refs/heads/synced/master does not point to a valid object! +error: refs/remotes/beta/master does not point to a valid object! +error: refs/remotes/beta/synced/master does not point to a valid object! +error: refs/remotes/halloween/master does not point to a valid object! +error: refs/remotes/halloween/synced/master does not point to a valid object! +fatal: bad object HEAD +error: failed to run repack + +# End of transcript or log. +"""]] + +### Have you had any luck using git-annex before? (Sometimes we get tired of reading bug reports all day and a lil' positive end note does wonders) + +I'm running into a few issues but I intend to work through them. diff --git a/doc/bugs/broken_repo_when_inodes_exhausted/comment_1_8ccfac4cea4c54029b96b436ac2114ed._comment b/doc/bugs/broken_repo_when_inodes_exhausted/comment_1_8ccfac4cea4c54029b96b436ac2114ed._comment new file mode 100644 index 0000000000..c71e59fcd5 --- /dev/null +++ b/doc/bugs/broken_repo_when_inodes_exhausted/comment_1_8ccfac4cea4c54029b96b436ac2114ed._comment @@ -0,0 +1,12 @@ +[[!comment format=mdwn + username="joey" + subject="""comment 1""" + date="2016-04-27T16:55:36Z" + content=""" +Pretty much anything is going to break when you run out of inodes; +all the errors shown here are due to the git repository getting +inconsistent because of it. I don't see any part that's git-annex specific. + +Is `git annex repair` able to repair it at all? I'd expect it would be able +to, as long as `git fsck` reports that there are problems. +"""]] diff --git a/doc/bugs/broken_repo_when_inodes_exhausted/comment_2_4c6ced0d3163735c7f4bd5a4a7053159._comment b/doc/bugs/broken_repo_when_inodes_exhausted/comment_2_4c6ced0d3163735c7f4bd5a4a7053159._comment new file mode 100644 index 0000000000..270968bbcc --- /dev/null +++ b/doc/bugs/broken_repo_when_inodes_exhausted/comment_2_4c6ced0d3163735c7f4bd5a4a7053159._comment @@ -0,0 +1,9 @@ +[[!comment format=mdwn + username="xloem" + subject="comment 2" + date="2016-05-18T09:43:13Z" + content=""" +Thanks. It turns out I'd deleted that repo in order to free the inodes up. Next time I will see if I can repair. + +But it would be nice if git-annex tracked inodes the way it tracks free space, so that it could refrain from causing the inode exhaustion. This also would have alerted me to how few inodes I had remaining. +"""]] diff --git a/doc/bugs/can__39__t_build_without_assistant.mdwn b/doc/bugs/can__39__t_build_without_assistant.mdwn new file mode 100644 index 0000000000..22171a2275 --- /dev/null +++ b/doc/bugs/can__39__t_build_without_assistant.mdwn @@ -0,0 +1,32 @@ +### Please describe the problem. +1. Building minimal variant of git-annex failed. +1. Seems, that instructions about [minimal building from source](https://git-annex.branchable.com/install/fromsource/) are out-of-date, at least because options 'xmpp' and 'dns' don't exist in git-annex.cabal. +1. Also seems, that file Utility/SRV.hs unused, may be remove it? + +In git-annex.cabal 'Assistant' flag doesn't influence on list of modules, only on compile flags and dependencies. 'Assistant/\*' modules includes into base variant of building (and some 'Assistant/\*' modules turn on with WebApp flag). + +### What steps will reproduce the problem? +Try to build as in [archlinux package](https://git.archlinux.org/svntogit/community.git/tree/repos/community-x86_64/PKGBUILD?h=packages/git-annex), but turn off assistant: + +`runhaskell Setup configure -O --prefix=/usr --enable-executable-dynamic --disable-library-vanilla --docdir="/usr/share/doc/git-annex" -fconcurrentoutput -ftorrentparser -f-androidsplice -f-android -fproduction -fpairing -fwebapp -f-assistant -fwebdav -fs3 -f-benchmark -fdbus -fmagicmime` + +`runhaskell Setup.hs build` + +After that I got error: + +`Utility/Mounts.hs:13:1: error:` + +` Could not find module ‘System.MountPoints’` + +haskell-mountpoints package is instaled. +If turn on assistant flag, building is ok. + +### Please provide any additional information below. +I don't know haskell, but try to make optional building of Assistant, see [patch](https://github.com/fftmp/git-annex-min-pkgbuild/blob/master/0002-optional-build-assistant-and-other-small-cabal-refin.patch). Seems, that it works for minimal building, but may be incomplete. +### What version of git-annex are you using? On what operating system? +6.20180807, Archlinux + + +### Have you had any luck using git-annex before? (Sometimes we get tired of reading bug reports all day and a lil' positive end note does wonders) +Yes, use it for several years and have approx 100G repo with 10k+ files. Works fine. + diff --git a/doc/bugs/can__39__t_build_without_assistant/comment_1_41b96b4e7f01b79a2e942b8098faa534._comment b/doc/bugs/can__39__t_build_without_assistant/comment_1_41b96b4e7f01b79a2e942b8098faa534._comment new file mode 100644 index 0000000000..e21ae48a4e --- /dev/null +++ b/doc/bugs/can__39__t_build_without_assistant/comment_1_41b96b4e7f01b79a2e942b8098faa534._comment @@ -0,0 +1,20 @@ +[[!comment format=mdwn + username="joey" + subject="""comment 1""" + date="2018-09-05T15:23:49Z" + content=""" +Thanks for the patch, but while it is moving in a good direction, +it gets some things wrong too. Problems I noticed: + +* It removes the "&& ! os(solaris) && ! os(gnu)" from the assistant flag + test, which are important to support building git-annex on hurd and + solaris. +* Utility.Android is used inside *linux* builds to detect android at + runtime. So moving it under the flag(Android) test is wrong and will + break all linux builds AFAICS. +* Build.DesktopFile is not linked into the assistant, like Build.Configure + it's only listed to get the source file into the cabal tarball. Best to + keep it with the other such files. + +I've applied the patch but reverted the above changes. +"""]] diff --git a/doc/bugs/can__39__t_connect_jabber_with_custom_google_apps_domain.mdwn b/doc/bugs/can__39__t_connect_jabber_with_custom_google_apps_domain.mdwn new file mode 100644 index 0000000000..b63f56ce83 --- /dev/null +++ b/doc/bugs/can__39__t_connect_jabber_with_custom_google_apps_domain.mdwn @@ -0,0 +1,21 @@ +### Please describe the problem. +In "Configuring jabber account" custom domains through google apps don't connect properly. + +### What steps will reproduce the problem? +Try to use a google account that uses a non-gmail domain, e.g. user@domain.com + +### What version of git-annex are you using? On what operating system? +Newest, on Mac OS 10.9 + +### Please provide any additional information below. +The issue is because git-annex is trying to connect to @domain.com as the jabber server, but the server should be talk.google.com:5223. + +[[!format sh """ +# If you can, paste a complete transcript of the problem occurring here. +# If the problem is with the git-annex assistant, paste in .git/annex/daemon.log + + +# End of transcript or log. +"""]] + +[[!tag moreinfo]] diff --git a/doc/bugs/can__39__t_connect_jabber_with_custom_google_apps_domain/comment_1_6537e928a0d6d5c41b55370f112f4afb._comment b/doc/bugs/can__39__t_connect_jabber_with_custom_google_apps_domain/comment_1_6537e928a0d6d5c41b55370f112f4afb._comment new file mode 100644 index 0000000000..26a63f87f0 --- /dev/null +++ b/doc/bugs/can__39__t_connect_jabber_with_custom_google_apps_domain/comment_1_6537e928a0d6d5c41b55370f112f4afb._comment @@ -0,0 +1,10 @@ +[[!comment format=mdwn + username="http://joeyh.name/" + ip="108.236.230.124" + subject="comment 1" + date="2014-07-02T17:28:01Z" + content=""" +This seems like the same problem as in [[bugs/googlemail]]. + +git-annex uses SRV records to find the jabber server for a domain. What is google doing that makes this not work? Do you have a sample domain? +"""]] diff --git a/doc/bugs/can__39__t_connect_jabber_with_custom_google_apps_domain/comment_2_b639ad750a4635d95f6ad16a1aa39a3e._comment b/doc/bugs/can__39__t_connect_jabber_with_custom_google_apps_domain/comment_2_b639ad750a4635d95f6ad16a1aa39a3e._comment new file mode 100644 index 0000000000..a705f68551 --- /dev/null +++ b/doc/bugs/can__39__t_connect_jabber_with_custom_google_apps_domain/comment_2_b639ad750a4635d95f6ad16a1aa39a3e._comment @@ -0,0 +1,8 @@ +[[!comment format=mdwn + username="https://www.google.com/accounts/o8/id?id=AItOawlU0H3uyacCnqWxjSI_chHBlHu8TDIkTt0" + nickname="Matt" + subject="cannot connect via google apps domain" + date="2014-08-14T15:55:07Z" + content=""" +Having the same issue with our domain: zebradog.com SRV records are correctly specified (as defined here: https://support.google.com/a/answer/34143?hl=en) +"""]] diff --git a/doc/bugs/can__39__t_get.mdwn b/doc/bugs/can__39__t_get.mdwn new file mode 100644 index 0000000000..d5a29a2aec --- /dev/null +++ b/doc/bugs/can__39__t_get.mdwn @@ -0,0 +1,81 @@ +### Please describe the problem. + + +### What steps will reproduce the problem? + +[[!format sh """ +$> git annex get 2read/ISNN2010__Tang.pdf +git-annex: Cannot mix --all or --unused with file names. +"""]] + +### What version of git-annex are you using? On what operating system? + + +[[!format sh """ +$> apt-cache policy git-annex +git-annex: + Installed: 5.20140116 + Candidate: 5.20140116 + Version table: + *** 5.20140116 0 + 600 http://debian.lcs.mit.edu/debian/ sid/main amd64 Packages + 100 /var/lib/dpkg/status +"""]] + +### Please provide any additional information below. + +[[!format sh """ +# If you can, paste a complete transcript of the problem occurring here. +# If the problem is with the git-annex assistant, paste in .git/annex/daemon.log + +$> git annex get 2read/ISNN2010__Tang.pdf +git-annex: Cannot mix --all or --unused with file names. + +but seems to start fetching some load if I do not specify any path and just run 'git annex get'. + +There seems to be some screw up: + +I have plenty of objects under .git/annex/objects/ (seems largely from +the directory above), nothing is now reported by unused (with obscure +msg): + +$> du -scmL * 2>/dev/null | tail -1 +1 total + +$> du -scm .git/annex/objects +334 .git/annex/objects +334 total + +$> git annex dropunused all +git-annex: Map.findMin: empty map has no minimal element + +Here is some portion of the history which lead to such a state (there +was git annex unused somewhere before) + +25954 git annex move --unused --to onerussian.com_annex +25955 git annex dropunused +25956 git annex dropunused all +25962 git annex unused +25963 git log --stat -SSHA256E-s5639442--67691e57cb4d6c51afe838590ad265ba4bea9c291cf52d58ed24f05b70bf33bf.mp3 +25965 git log --stat -SSHA256E-s143042--b4012bf03ed0a387a9e714390efa75f1dd769162cca4c9b77e516732342be3f9.html +25968 git annex move --unused --to onerussian.com_annex +25969 git annex dropunused all +25976 git annex unused +25978 git br +25980 git log --stat -Ss741707--7c215090893f1f0c994e2a9ad3088016676464bbad26768841dd08c07295a2fe.pdf.map +25981 git annex unused +25982 git annex fsck +25983 git annex unused +25984 git annex dropkey +25985 git log --stat -SSHA256E-s14534131--20de680eedb3e1fb687c9b00c154d978333b61f4ea122c632bdb5bcdbb1553ff.pdf +25986 git show de3ccae8304efbae4a7a8add49de638f64b821fc +25991 git annex fsck + +# End of transcript or log. +"""]] + +[[!tag moreinfo]] + +> Tagged moreinfo since I have a workable theory about how this happened, +> which would make it user configuration error and not a bug, but +> that has not been confirmed. --[[Joey]] diff --git a/doc/bugs/can__39__t_get/comment_1_ef32287828481c161bd913c9db9052a5._comment b/doc/bugs/can__39__t_get/comment_1_ef32287828481c161bd913c9db9052a5._comment new file mode 100644 index 0000000000..6530ae3c80 --- /dev/null +++ b/doc/bugs/can__39__t_get/comment_1_ef32287828481c161bd913c9db9052a5._comment @@ -0,0 +1,27 @@ +[[!comment format=mdwn + username="https://www.google.com/accounts/o8/id?id=AItOawnx8kHW66N3BqmkVpgtXDlYMvr8TJ5VvfY" + nickname="Yaroslav" + subject="git annex fix starts fixing but then spits bulk of errors" + date="2014-01-18T05:42:15Z" + content=""" +probably related: + +``` +fix books/Мои первые книжки/PDF/Благинина Е.А. - Лодочки (Мои первые книжки) - 1962.pdf ok +fix books/Мои первые книжки/PDF/Благинина Е.А. - Не мешайте мне трудиться (Мои первые книжки) - 1975.pdf fatal: This operation must be run in a work tree +ok +(Recording state in git...) + +git-annex: user error (xargs [\"-0\",\"git\",\"--git-dir=/home/yoh/annex/.git\",\"add\",\"--force\",\"--\"] exited 123) +fatal: This operation must be run in a work tree +failed +(Recording state in git...) + +git-annex: user error (xargs [\"-0\",\"git\",\"--git-dir=/home/yoh/annex/.git\",\"add\",\"--force\",\"--\"] exited 123) +fatal: This operation must be run in a work tree +failed +(Recording state in git...) + +.... +``` +"""]] diff --git a/doc/bugs/can__39__t_get/comment_2_31fe400f4bac516a5c1101612cb06a54._comment b/doc/bugs/can__39__t_get/comment_2_31fe400f4bac516a5c1101612cb06a54._comment new file mode 100644 index 0000000000..456bd0342d --- /dev/null +++ b/doc/bugs/can__39__t_get/comment_2_31fe400f4bac516a5c1101612cb06a54._comment @@ -0,0 +1,32 @@ +[[!comment format=mdwn + username="https://www.google.com/accounts/o8/id?id=AItOawnx8kHW66N3BqmkVpgtXDlYMvr8TJ5VvfY" + nickname="Yaroslav" + subject="repair seems to be also confused" + date="2014-01-18T05:47:02Z" + content=""" +[[[ +$> git annex repair +Running git fsck ... +No problems found. +fatal: '/home/yoh/annex/.git' is outside repository +Had to delete the .git/annex/index file as it was corrupt. +No data was lost. +ok + +$> ls +2enjoy/ 2read/ 2watch/ books/ hardware/ videos/ +2listen/ 2review/ abooks/ docs/ pics/ + +$> git annex repair +Running git fsck ... +No problems found. +fatal: '/home/yoh/annex/.git' is outside repository +Had to delete the .git/annex/index file as it was corrupt. +No data was lost. +ok + +$> git annex get 2read/ISNN2010__Tang.pdf +git-annex: Cannot mix --all or --unused with file names. + +]]] +"""]] diff --git a/doc/bugs/can__39__t_get/comment_3_87d123c04815d38abb92f967829c3a23._comment b/doc/bugs/can__39__t_get/comment_3_87d123c04815d38abb92f967829c3a23._comment new file mode 100644 index 0000000000..b7a7f3bc46 --- /dev/null +++ b/doc/bugs/can__39__t_get/comment_3_87d123c04815d38abb92f967829c3a23._comment @@ -0,0 +1,16 @@ +[[!comment format=mdwn + username="https://www.google.com/accounts/o8/id?id=AItOawnx8kHW66N3BqmkVpgtXDlYMvr8TJ5VvfY" + nickname="Yaroslav" + subject="could it be part/reason of the problem" + date="2014-01-18T06:05:50Z" + content=""" +not sure how that happened... definitely not me consciously! ;-) some commands are complaining that \"You cannot run this command in a bare repository\" which I thought is BS since it is not BARE! but then looked into .git/config and it does have core.bare = True ... yikes!.. + +This repository is also under assistant \"control\". + +changing to bare=False seems to start 'get'ing things, git annex repair doesn't produce obscure errors. + +git annex fix though now doesn't report any problems -- only 'ok', but none of those files mentioned 'ok' has a working symlink,,, but I guess that is a fluke after many upgrades -- just dropping everything locally and getting needed context after purging .git/annex/objects . + +So I guess issue is resolved by discovering that repository was set to 'bare' mode somehow although it was not and seemed like working but not quite +"""]] diff --git a/doc/bugs/can__39__t_get/comment_4_b99cff87dbe38f08f888200dfe7e2436._comment b/doc/bugs/can__39__t_get/comment_4_b99cff87dbe38f08f888200dfe7e2436._comment new file mode 100644 index 0000000000..0d0422fd35 --- /dev/null +++ b/doc/bugs/can__39__t_get/comment_4_b99cff87dbe38f08f888200dfe7e2436._comment @@ -0,0 +1,14 @@ +[[!comment format=mdwn + username="http://joeyh.name/" + ip="209.250.56.43" + subject="comment 4" + date="2014-01-18T15:42:59Z" + content=""" +git-annex sets core.bare=true for direct mode, but it also then sets annex.direct=true and so does not treat it as a bare mode repository. If you had eg, manually tried to change annex.direct to false, and left it in bare mode, that would explain everything. + +> git annex fix though now doesn't report any problems -- only 'ok', but none of those files mentioned 'ok' has a working symlink + +That is completely normal behavior; git annex fix does not care if the content is locally present or not; it just checks that the symlinks would point to it if it were present. + +(Fixed the partial function in dropunused.) +"""]] diff --git a/doc/bugs/can__39__t_launch_assistant_on_latest_OSX_build.mdwn b/doc/bugs/can__39__t_launch_assistant_on_latest_OSX_build.mdwn new file mode 100644 index 0000000000..0f11ddbb12 --- /dev/null +++ b/doc/bugs/can__39__t_launch_assistant_on_latest_OSX_build.mdwn @@ -0,0 +1,59 @@ +### Please describe the problem. +I can't launch the Assistant on the latest OSX build. Also mentioned by a user at [install/OSX](http://git-annex.branchable.com/install/OSX/). + +I can launch the assistant fine with `6.20180409-gfb0780266` + +### What steps will reproduce the problem? +Download git-annex.dmg (6.20180430-g393fc79d5) and copy to /Applications folder + +Add `/Applications/git-annex.app/Contents/MacOS` to bash path + +Run `git-annex-webapp` from command-line: + + andrew@bumblebee ~$ git-annex-webapp + andrew@bumblebee ~$ uname: illegal option -- o + usage: uname [-amnprsv] + git-annex: user error (uname ["-o"] exited 1) + WebApp crashed: user error (uname ["-o"] exited 1) + [2018-05-04 09:32:07.67968] WebApp: warning WebApp crashed: user error (uname ["-o"] exited 1) + +### What version of git-annex are you using? On what operating system? +macOS Sierra 10.12.6 + +git-annex version: 6.20180430-g393fc79d5 +build flags: Assistant Webapp Pairing S3(multipartupload)(storageclasses) WebDAV FsEvents ConcurrentOutput TorrentParser MagicMime Feeds Testsuite +dependency versions: aws-0.17.1 bloomfilter-2.0.1.0 cryptonite-0.23 DAV-1.3.1 feed-0.3.12.0 ghc-8.0.2 http-client-0.5.7.0 persistent-sqlite-2.6.2 torrent-10000.1.1 uuid-1.3.13 yesod-1.4.5 +key/value backends: SHA256E SHA256 SHA512E SHA512 SHA224E SHA224 SHA384E SHA384 SHA3_256E SHA3_256 SHA3_512E SHA3_512 SHA3_224E SHA3_224 SHA3_384E SHA3_384 SKEIN256E SKEIN256 SKEIN512E SKEIN512 BLAKE2B256E BLAKE2B256 BLAKE2B512E BLAKE2B512 BLAKE2B160E BLAKE2B160 BLAKE2B224E BLAKE2B224 BLAKE2B384E BLAKE2B384 BLAKE2S256E BLAKE2S256 BLAKE2S160E BLAKE2S160 BLAKE2S224E BLAKE2S224 BLAKE2SP256E BLAKE2SP256 BLAKE2SP224E BLAKE2SP224 SHA1E SHA1 MD5E MD5 WORM URL +remote types: git gcrypt p2p S3 bup directory rsync web bittorrent webdav adb tahoe glacier ddar hook external + +### Please provide any additional information below. + +[[!format sh """ +# If you can, paste a complete transcript of the problem occurring here. +# If the problem is with the git-annex assistant, paste in .git/annex/daemon.log + +andrew@bumblebee ~$ which git-annex-webapp +/Applications/git-annex.app/Contents/MacOS/git-annex-webapp +andrew@bumblebee ~$ git-annex version +git-annex version: 6.20180430-g393fc79d5 +build flags: Assistant Webapp Pairing S3(multipartupload)(storageclasses) WebDAV FsEvents ConcurrentOutput TorrentParser MagicMime Feeds Testsuite +dependency versions: aws-0.17.1 bloomfilter-2.0.1.0 cryptonite-0.23 DAV-1.3.1 feed-0.3.12.0 ghc-8.0.2 http-client-0.5.7.0 persistent-sqlite-2.6.2 torrent-10000.1.1 uuid-1.3.13 yesod-1.4.5 +key/value backends: SHA256E SHA256 SHA512E SHA512 SHA224E SHA224 SHA384E SHA384 SHA3_256E SHA3_256 SHA3_512E SHA3_512 SHA3_224E SHA3_224 SHA3_384E SHA3_384 SKEIN256E SKEIN256 SKEIN512E SKEIN512 BLAKE2B256E BLAKE2B256 BLAKE2B512E BLAKE2B512 BLAKE2B160E BLAKE2B160 BLAKE2B224E BLAKE2B224 BLAKE2B384E BLAKE2B384 BLAKE2S256E BLAKE2S256 BLAKE2S160E BLAKE2S160 BLAKE2S224E BLAKE2S224 BLAKE2SP256E BLAKE2SP256 BLAKE2SP224E BLAKE2SP224 SHA1E SHA1 MD5E MD5 WORM URL +remote types: git gcrypt p2p S3 bup directory rsync web bittorrent webdav adb tahoe glacier ddar hook external +andrew@bumblebee ~$ git-annex-webapp +andrew@bumblebee ~$ uname: illegal option -- o +usage: uname [-amnprsv] +git-annex: user error (uname ["-o"] exited 1) +WebApp crashed: user error (uname ["-o"] exited 1) +[2018-05-04 09:32:07.67968] WebApp: warning WebApp crashed: user error (uname ["-o"] exited 1) + +# End of transcript or log. +"""]] + +### Have you had any luck using git-annex before? (Sometimes we get tired of reading bug reports all day and a lil' positive end note does wonders) + +Yup! git-annex is great! Thanks for all your hard work on this project Joey!! + +—Andrew + +> [[fixed|done]] --[[Joey]] diff --git a/doc/bugs/cannot___34__install__34___standalone_git_annex_within_afs_mount.mdwn b/doc/bugs/cannot___34__install__34___standalone_git_annex_within_afs_mount.mdwn new file mode 100644 index 0000000000..32643a612f --- /dev/null +++ b/doc/bugs/cannot___34__install__34___standalone_git_annex_within_afs_mount.mdwn @@ -0,0 +1,24 @@ +### Please describe the problem. + +Cannot even extract the standalone linux tarball for deployment on our processing server: + +[[!format sh """ +[mvdoc@rolando bin] > tar -xzf git-annex-standalone-amd64.tar.gz +tar: git-annex.linux/shimmed/git-fsck/git-fsck: Cannot hard link to `git-annex.linux/shimmed/git-help/git-help': Invalid cross-device link +tar: git-annex.linux/shimmed/git-cat-file/git-cat-file: Cannot hard link to `git-annex.linux/shimmed/git-help/git-help': Invalid cross-device link +tar: git-annex.linux/shimmed/git-init/git-init: Cannot hard link to `git-annex.linux/shimmed/git-help/git-help': Invalid cross-device link +... +tar: Exiting with failure status due to previous errors + +[mvdoc@rolando bin] > ls -l git-annex.linux/shimmed/git-pack-redundant/git-pack-redundant git-annex.linux/shimmed/git-help/git-help +ls: cannot access git-annex.linux/shimmed/git-pack-redundant/git-pack-redundant: No such file or directory +-rwxr-xr-x 1 mvdoc users 2023056 Oct 29 22:44 git-annex.linux/shimmed/git-help/git-help +[mvdoc@rolando bin] > + + +# End of transcript or log. +"""]] + +[[!meta author=yoh]] + +> [[done]] --[[Joey]] diff --git a/doc/bugs/cannot___40__or_how__63____41___to_pass_socket_path_with_a_space_in_its_path_via_annex-ssh-options.mdwn b/doc/bugs/cannot___40__or_how__63____41___to_pass_socket_path_with_a_space_in_its_path_via_annex-ssh-options.mdwn new file mode 100644 index 0000000000..cc0eed824b --- /dev/null +++ b/doc/bugs/cannot___40__or_how__63____41___to_pass_socket_path_with_a_space_in_its_path_via_annex-ssh-options.mdwn @@ -0,0 +1,31 @@ +### Please describe the problem. + +git annex specifies its own socket path via -S. To overload with our socket path (to be reused by annex and other ssh invocations) we need to provide the path via annex-ssh-options option. And we cannot pass it as an overload -o ControlPath since -S specification provided by annex "overrides" it. +The difficulty comes in possibly not quite common but still possible case when path to the socket contains a space. I have tried all kinds of possible specifications but failed to find one which works... + +Could you give me a hint!? ;) + +### What version of git-annex are you using? On what operating system? + +6.20170307+gitg24ade8a25-1~ndall+1 + +### Please provide any additional information below. + +[[!format sh """ +# so -- ssh works +$> ssh -O check -oControlMaster=auto -S"/tmp/te st/socket" 'yohtest@smaug' +Master running (pid=20336) +# but can't get annex to use it: + +$> git annex init -c 'remote.origin.annex-ssh-options=-oControlMaster=auto -S"/tmp/te st/socket"' +init ssh: Could not resolve hostname st/socket": Name or service not known +yohtest@smaug's password: + +$> git annex init -c 'remote.origin.annex-ssh-options=-oControlMaster=auto "-S\"/tmp/te st/socket\""' +init ssh: Could not resolve hostname "-s\\"/tmp/te: Name or service not known +yohtest@smaug's password: + +# etc +"""]] + +[[!meta author=yoh]] diff --git a/doc/bugs/cannot___40__or_how__63____41___to_pass_socket_path_with_a_space_in_its_path_via_annex-ssh-options/comment_1_d773dee03276e9b0e0b75d0709b76278._comment b/doc/bugs/cannot___40__or_how__63____41___to_pass_socket_path_with_a_space_in_its_path_via_annex-ssh-options/comment_1_d773dee03276e9b0e0b75d0709b76278._comment new file mode 100644 index 0000000000..002f9d1151 --- /dev/null +++ b/doc/bugs/cannot___40__or_how__63____41___to_pass_socket_path_with_a_space_in_its_path_via_annex-ssh-options/comment_1_d773dee03276e9b0e0b75d0709b76278._comment @@ -0,0 +1,18 @@ +[[!comment format=mdwn + username="joey" + subject="""comment 1""" + date="2017-03-21T17:38:57Z" + content=""" +You can't accomplish this with `remote..annex-ssh-options`, +since it is not exposed to the shell, and the parser just breaks it up into +words. + +A smarter parser would be needed. Or you could configure it in +~/.ssh/config, or perhaps make a ssh config file elsewhere and use +annex-ssh-options to pass -F to ssh to make it use this other config file. + +Now that git-annex supports `GIT_SSH_COMMAND`, which is exposed to the +shell, you should be able to accomplish it that way. I don't know if that +would work in your use case, since the environment variable affects all ssh +remotes, not just one. +"""]] diff --git a/doc/bugs/cannot_add_a_files_with_an_accent_in_it.mdwn b/doc/bugs/cannot_add_a_files_with_an_accent_in_it.mdwn new file mode 100644 index 0000000000..f85453a88a --- /dev/null +++ b/doc/bugs/cannot_add_a_files_with_an_accent_in_it.mdwn @@ -0,0 +1,43 @@ +### Please describe the problem. + +I cannot add a file that has an accent in its name. If I change the file name to not have accents, everything works. + +### What steps will reproduce the problem? + +- create a file with an accent in its name +- "git annex status" shows the file is not checked in +- "git annex add" the file +- nothing happens, and "git annex status" still shows the file as not checked in + +### What version of git-annex are you using? On what operating system? + +% brew info git-annex +git-annex: stable 5.20151218 (bottled), HEAD + +OS X 10.11.2 + +### Please provide any additional information below. + +[[!format sh """ +# If you can, paste a complete transcript of the problem occurring here. +# If the problem is with the git-annex assistant, paste in .git/annex/daemon.log +[schmitta@charm-ecran:~/Documents/annex on master] +% git annex status +[schmitta@charm-ecran:~/Documents/annex on master] +% cat > é +foo +[schmitta@charm-ecran:~/Documents/annex on master] +% git annex status ✭ +? é +[schmitta@charm-ecran:~/Documents/annex on master] +% git annex add é ✭ +[schmitta@charm-ecran:~/Documents/annex on master] +% git annex status ✭ +? é + +# End of transcript or log. +"""]] + +### Have you had any luck using git-annex before? (Sometimes we get tired of reading bug reports all day and a lil' positive end note does wonders) + +Yes, git annex is great and I use it to store videos I want to watch later but that are too big to all be stored on my laptop. diff --git a/doc/bugs/cannot_add_a_files_with_an_accent_in_it/comment_1_d465d7f88c8f11b4b636fba56711d745._comment b/doc/bugs/cannot_add_a_files_with_an_accent_in_it/comment_1_d465d7f88c8f11b4b636fba56711d745._comment new file mode 100644 index 0000000000..0d47449a0d --- /dev/null +++ b/doc/bugs/cannot_add_a_files_with_an_accent_in_it/comment_1_d465d7f88c8f11b4b636fba56711d745._comment @@ -0,0 +1,23 @@ +[[!comment format=mdwn + username="joey" + subject="""comment 1""" + date="2016-01-11T16:46:42Z" + content=""" +I can't even type an e with an accent in it into the bash prompt on OSX. +However, I copied over such a file, and had no problem adding it: + + bash-3.2$ ls + e?? + bash-3.2$ git annex add . + add ë ok + bash-3.2$ git annex status + A ë + +I also tried tab-completing a file named "testë" and that also works: + + oberon:gd joeyh$ git annex add teste\314\210 + add testë ok + (recording state in git...) + +So, I can't reproduce your problem. +"""]] diff --git a/doc/bugs/cannot_add_a_files_with_an_accent_in_it/comment_2_b6886a5c38725d93630562a3172d1a5c._comment b/doc/bugs/cannot_add_a_files_with_an_accent_in_it/comment_2_b6886a5c38725d93630562a3172d1a5c._comment new file mode 100644 index 0000000000..7dc9497bc2 --- /dev/null +++ b/doc/bugs/cannot_add_a_files_with_an_accent_in_it/comment_2_b6886a5c38725d93630562a3172d1a5c._comment @@ -0,0 +1,20 @@ +[[!comment format=mdwn + username="Alan" + subject="Some additional information" + date="2016-01-12T17:48:58Z" + content=""" +Here is what `locale` returns here + + LANG= + LC_COLLATE=\"en_US.UTF-8\" + LC_CTYPE=\"en_US.UTF-8\" + LC_MESSAGES=\"en_US.UTF-8\" + LC_MONETARY=\"en_US.UTF-8\" + LC_NUMERIC=\"en_US.UTF-8\" + LC_TIME=\"en_US.UTF-8\" + LC_ALL=\"en_US.UTF-8\" + +I think it may matter as I see that your accents are not displayed correctly when you \"tab\". + +To type a é on OS X with qwerty, you can do `option-e` then e. +"""]] diff --git a/doc/bugs/cannot_add_a_files_with_an_accent_in_it/comment_3_7c34ada3c7b413f2b622149b60757f6e._comment b/doc/bugs/cannot_add_a_files_with_an_accent_in_it/comment_3_7c34ada3c7b413f2b622149b60757f6e._comment new file mode 100644 index 0000000000..a30e2259e0 --- /dev/null +++ b/doc/bugs/cannot_add_a_files_with_an_accent_in_it/comment_3_7c34ada3c7b413f2b622149b60757f6e._comment @@ -0,0 +1,37 @@ +[[!comment format=mdwn + username="Alan" + avatar="http://cdn.libravatar.org/avatar/9cbc26346f1c693d7df198e662a5fdae" + subject="Issue still present" + date="2017-03-10T07:32:16Z" + content=""" +I'm still having this issue, now with a folder. I added several files: + + % git annex add Théâtre\ Augustin\ 2016/* + add Théâtre Augustin 2016/Juin 2016 - Les Ptits du Madrigal - Bizarre, Bizarre.webm ok + add Théâtre Augustin 2016/Juin 2016 - Les Ptits du Madrigal - Conte A Rebours.webm ok + add Théâtre Augustin 2016/Juin 2016 - Les Ptits du Madrigal - Coup De Theatre.webm ok + add Théâtre Augustin 2016/Juin 2016 - Les Ptits du Madrigal - Inspecteur Toutou.webm ok + add Théâtre Augustin 2016/Juin 2016 - Les Ptits du Madrigal - La Princesse Qui Disait Toujours Non.webm ok + add Théâtre Augustin 2016/Juin 2016 - Les Ptits du Madrigal - Le Magicien Qui Aimait Les Bonbons.webm ok + (recording state in git...) + +and they show as both added and untracked + + % git annex status + A Théâtre Augustin 2016/Juin 2016 - Les Ptits du Madrigal - Bizarre, Bizarre.webm + A Théâtre Augustin 2016/Juin 2016 - Les Ptits du Madrigal - Conte A Rebours.webm + A Théâtre Augustin 2016/Juin 2016 - Les Ptits du Madrigal - Coup De Theatre.webm + A Théâtre Augustin 2016/Juin 2016 - Les Ptits du Madrigal - Inspecteur Toutou.webm + A Théâtre Augustin 2016/Juin 2016 - Les Ptits du Madrigal - La Princesse Qui Disait Toujours Non.webm + A Théâtre Augustin 2016/Juin 2016 - Les Ptits du Madrigal - Le Magicien Qui Aimait Les Bonbons.webm + ? Théâtre Augustin 2016/Juin 2016 - Les Ptits du Madrigal - Bizarre, Bizarre.webm + ? Théâtre Augustin 2016/Juin 2016 - Les Ptits du Madrigal - Conte A Rebours.webm + ? Théâtre Augustin 2016/Juin 2016 - Les Ptits du Madrigal - Coup De Theatre.webm + ? Théâtre Augustin 2016/Juin 2016 - Les Ptits du Madrigal - Inspecteur Toutou.webm + ? Théâtre Augustin 2016/Juin 2016 - Les Ptits du Madrigal - La Princesse Qui Disait Toujours Non.webm + ? Théâtre Augustin 2016/Juin 2016 - Les Ptits du Madrigal - Le Magicien Qui Aimait Les Bonbons.webm + +It seems to be related to this: https://stackoverflow.com/questions/11968183/how-to-handle-accented-characters-in-file-names-in-git-on-mac-os-x-converted-to + +I “solved“ the issue by doing a git add of the repository, but I'm wondering what options I need to use with git to avoid such issues in the future. +"""]] diff --git a/doc/bugs/cannot_add_local_readonly_repo_through_the_webapp.mdwn b/doc/bugs/cannot_add_local_readonly_repo_through_the_webapp.mdwn new file mode 100644 index 0000000000..9b1f726cf6 --- /dev/null +++ b/doc/bugs/cannot_add_local_readonly_repo_through_the_webapp.mdwn @@ -0,0 +1,98 @@ +### Please describe the problem. + +A readonly repository that I can add fine on the commandline (and sync content from) cannot be added through the webapp. + +### What steps will reproduce the problem? + +Say I have a readonly (owned by root) repository in `~/test/a` and I create a `~/test/b` (owned by my user). In the webapp, when to add `/home/anarcat/test/a` as a "local repository" (`Add another local repository`) to the `~/test/b` repo, it fails when i enter that path, with "Cannot write a repository there." I obviously can't sync content from there then. + +This works on the commandline, although with warnings. + +### What version of git-annex are you using? On what operating system? + +Version: 5.20140927 +Build flags: Assistant Webapp Webapp-secure Pairing Testsuite S3 WebDAV Inotify DBus DesktopNotify XMPP DNS Feeds Quvi TDFA CryptoHash + +Debian Jessie. + +### Please provide any additional information below. + +Here's the transcript of the commandline equivalent: + +~~~ +anarcat@marcos:test$ git init a +Dépôt Git vide initialisé dans /home/anarcat/test/a/.git/ +anarcat@marcos:test$ git init b +Dépôt Git vide initialisé dans /home/anarcat/test/b/.git/ +anarcat@marcos:test$ cd a +anarcat@marcos:a$ git annex init +init ok +(Recording state in git...) +anarcat@marcos:a$ echo hellow world > README +anarcat@marcos:a$ git annex add README +add README ok +(Recording state in git...) +anarcat@marcos:a$ git commit -m"test repo a" +[master (commit racine) 3ece2a1] test repo a + 1 file changed, 1 insertion(+) + create mode 120000 README +anarcat@marcos:a$ cd ../ ^C +anarcat@marcos:a$ sudo chown -R root . +[sudo] password for anarcat: +Sorry, try again. +[sudo] password for anarcat: +anarcat@marcos:a$ cd ../b +anarcat@marcos:b$ git annex init +init ok +(Recording state in git...) +anarcat@marcos:b$ git remote add a ../a +anarcat@marcos:b$ git annex sync a +commit ok +pull a +warning: no common commits +remote: Décompte des objets: 13, fait. +remote: Compression des objets: 100% (9/9), fait. +remote: Total 13 (delta 1), reused 0 (delta 0) +Dépaquetage des objets: 100% (13/13), fait. +Depuis ../a + * [nouvelle branche] git-annex -> a/git-annex + * [nouvelle branche] master -> a/master + + +merge: refs/remotes/a/synced/master - not something we can merge +failed +(merging a/git-annex into git-annex...) +(Recording state in git...) +push a +Décompte des objets: 8, fait. +Delta compression using up to 2 threads. +Compression des objets: 100% (6/6), fait. +Écriture des objets: 100% (8/8), 819 bytes | 0 bytes/s, fait. +Total 8 (delta 1), reused 0 (delta 0) +remote: error: insufficient permission for adding an object to repository database objects +remote: fatal: failed to write object +error: unpack failed: unpack-objects abnormal exit +To ../a + ! [remote rejected] git-annex -> synced/git-annex (unpacker error) + ! [remote rejected] master -> synced/master (unpacker error) +error: impossible de pousser des références vers '../a' + + Pushing to a failed. + + (non-fast-forward problems can be solved by setting receive.denyNonFastforwards to false in the remote's git config) +failed +git-annex: sync: 2 failed +anarcat@marcos:b$ ls +README +anarcat@marcos:b$ git annex copy --from a +copy README (from a...) ok +(Recording state in git...) +anarcat@marcos:b$ ls -al +total 16K +drwxr-xr-x 3 anarcat anarcat 4096 oct. 20 15:36 . +drwxr-xr-x 4 anarcat anarcat 4096 oct. 20 15:35 .. +drwxr-xr-x 9 anarcat anarcat 4096 oct. 20 15:36 .git +lrwxrwxrwx 1 anarcat anarcat 180 oct. 20 15:36 README -> .git/annex/objects/wz/Zq/SHA256E-s13--8c083c6897455257dfbace7a9012d92ca8ebfb6e6ebe8acddc6dfa8fc81226ed/SHA256E-s13--8c083c6897455257dfbace7a9012d92ca8ebfb6e6ebe8acddc6dfa8fc81226ed +~~~ + +This is part of the [[todo/read-only_removable_drives/]] series. --[[anarcat]] diff --git a/doc/bugs/cannot_commit___34__annex_add__34__ed_modified_file_which_switched_its_largefile_status_to_be_committed_to_git_now.mdwn b/doc/bugs/cannot_commit___34__annex_add__34__ed_modified_file_which_switched_its_largefile_status_to_be_committed_to_git_now.mdwn new file mode 100644 index 0000000000..9fdfa20124 --- /dev/null +++ b/doc/bugs/cannot_commit___34__annex_add__34__ed_modified_file_which_switched_its_largefile_status_to_be_committed_to_git_now.mdwn @@ -0,0 +1,70 @@ +### Please describe the problem. + +Leads to a failure of 'git commit' upon attempt to commit a file which went from "largefile" to small, according to .gitattributes settings, if we `git annex add file` before committing the change. + +### What version of git-annex are you using? On what operating system? + +6.20180720+gitg03978571f-1~ndall+1 + +### Please provide any additional information below. + +Here is a full script to reproduce it: +[[!format sh """ +#!/bin/bash + +set -ex + +builtin cd /tmp; + +if [ -e /tmp/repo ]; then + chmod -R +w /tmp/repo; + rm -rf /tmp/repo; +fi +mkdir /tmp/repo; +cd /tmp/repo; +git init; +git annex init; +echo '* annex.largefiles=(largerthan=5b)' >.gitattributes; +git add .gitattributes; +git commit -m 'added .gitattri'; +echo 123456 > file; +git annex add file; +git commit -m add1; +ls -l; + +git annex unlock file; +echo 123 >| file +git annex add file + +# this would work but commit to git-annex, not git despite .gitattributes settings +# git commit -m edit -a +# This one would fail to commit at all, complaining about "partial commit" +git commit -m edit file +ls -l file; + +git status + + +"""]] + +which leads to +[[!format sh """ +... ++ git annex add file +add file (non-large file; adding content to git repository) ok +add file (non-large file; adding content to git repository) ok +(recording state in git...) ++ git commit -m edit file +git-annex: Cannot make a partial commit with unlocked annexed files. You should `git annex add` the files you want to commit, and then run git commit. +"""]] + +additional observations: + +- works fine if remains large file (e.g. we just append to it) +- does not fail if we do `git commit -a` not `git commit file`, but it commits it to annex not to git, despite previous `git annex add` message rightfully says that "non-large file; adding content to git repository" + +Expected behavior: +- have consistent behavior between `commit -a` and `commit file` +- commit without a failure, committing to git (since .gitattributes instructs so and even `git annex add` reports that) + +[[!meta author=yoh]] diff --git a/doc/bugs/cannot_commit___34__annex_add__34__ed_modified_file_which_switched_its_largefile_status_to_be_committed_to_git_now/comment_1_ad19d064dd9c175debe647a6928e4f35._comment b/doc/bugs/cannot_commit___34__annex_add__34__ed_modified_file_which_switched_its_largefile_status_to_be_committed_to_git_now/comment_1_ad19d064dd9c175debe647a6928e4f35._comment new file mode 100644 index 0000000000..d6167d7893 --- /dev/null +++ b/doc/bugs/cannot_commit___34__annex_add__34__ed_modified_file_which_switched_its_largefile_status_to_be_committed_to_git_now/comment_1_ad19d064dd9c175debe647a6928e4f35._comment @@ -0,0 +1,16 @@ +[[!comment format=mdwn + username="joey" + subject="""comment 1""" + date="2018-08-09T15:57:57Z" + content=""" +I take this as not being a bug about the partial commit +blocking (as explained in +[[!commit adc5ca70a8095a389273e7c286cb32de6873a5a3]]), which is working +around a git behavior and so can't be fixed other than by going to v6. + +Instead, I think this is a bug about git annex add of an unlocked file +not converting it to a in-git file when annex.largefiles says it ought +to. If it did that it would not run into the partial commit blocking +at all. And, the observersion about git commit -a committing to the annex +not to git points at the same problem. +"""]] diff --git a/doc/bugs/cannot_commit___34__annex_add__34__ed_modified_file_which_switched_its_largefile_status_to_be_committed_to_git_now/comment_2_19de1d89dd0a266ce43e0f25cd2b74cd._comment b/doc/bugs/cannot_commit___34__annex_add__34__ed_modified_file_which_switched_its_largefile_status_to_be_committed_to_git_now/comment_2_19de1d89dd0a266ce43e0f25cd2b74cd._comment new file mode 100644 index 0000000000..50c512e369 --- /dev/null +++ b/doc/bugs/cannot_commit___34__annex_add__34__ed_modified_file_which_switched_its_largefile_status_to_be_committed_to_git_now/comment_2_19de1d89dd0a266ce43e0f25cd2b74cd._comment @@ -0,0 +1,12 @@ +[[!comment format=mdwn + username="joey" + subject="""comment 2""" + date="2018-08-09T16:05:27Z" + content=""" +The double output from `git-annex add file` is also some kind of minor bug. + +The double output seems to have the same root cause too: The file is left +typechanged form by the first pass of the add, and so the second pass +sees it again. When annex.largefiles lets the file be annexed, the doubled +output does not occur. +"""]] diff --git a/doc/bugs/cannot_commit___34__annex_add__34__ed_modified_file_which_switched_its_largefile_status_to_be_committed_to_git_now/comment_3_92b71940fe9d00bf58ee08b327f4a991._comment b/doc/bugs/cannot_commit___34__annex_add__34__ed_modified_file_which_switched_its_largefile_status_to_be_committed_to_git_now/comment_3_92b71940fe9d00bf58ee08b327f4a991._comment new file mode 100644 index 0000000000..25f6c61a23 --- /dev/null +++ b/doc/bugs/cannot_commit___34__annex_add__34__ed_modified_file_which_switched_its_largefile_status_to_be_committed_to_git_now/comment_3_92b71940fe9d00bf58ee08b327f4a991._comment @@ -0,0 +1,55 @@ +[[!comment format=mdwn + username="joey" + subject="""comment 3""" + date="2018-08-09T16:17:33Z" + content=""" +So the root problem is that when we have a typechanged file and want to +convert that to be not typechanged, we have to git commit it. +As long as the previous commit is a symlink and the file in the index is +not, the file will be typechanged by definition. + +When git-annex add runs `git add file`, it's doing the only thing it can +do, but it leaves the file typechanged, and so git-annex later has no way +to tell that this file is not supposed to be treated as an unlocked file. +I don't think we want `git annex add` to commit the file. That would be +very surprising behavior! + +What git-annex could do is have the pre-commit hook notice that the file +doesn't match annex.largefiles and not re-annex it, allowing the typechange to +get committed to git. Then the user would only need to unlock the file, +modify it to make it non-large, and commit it to get it checked into git. + +In a way, this is *too* easy, because if the user sees that working, they may +expect to be able to turn a small file back into an annexed file by +making the content large and running git commit on it w/o git-annex add. +Which would be bad because that would commit a large file to git. +I suppose the pre-commit could handle that too, but imagine that replacing +eg a `configure` script that's expected to be shipped in the git repository +with an annex symlink, which would be surprising. + +So it may be better to keep the conversion from annexed to in-git file +and back explicit. This could be done by `git annex add` detecting +this situation and erroring out with a message that suggests running +`git commit -n` if the user wants to change the annexed file to a in-git +file. That bypasses the pre-commit hook, so the typechange gets committed +to git as they desire. + +Which is better, the implicit conversion of the explicit? I am not +sure, but lean toward the explicit since it doesn't have this potential +to confuse users. Also, the implicit conversion would only work when +annex.largefiles is being used, but the explicit conversion can be done +irregardless. + +The explicit paths would be: + + # annex to git # git to annex + git annex unlock file + smallen file largen file + git add file git annex add file + git commit -n git commit + +Seems worth documenting somewhere. Or making a command that handles these +conversions, but the largen and smallen steps being manual, and the +possibility to combine multiple of these into a single commit argues +against a conversion command. +"""]] diff --git a/doc/bugs/cannot_commit___34__annex_add__34__ed_modified_file_which_switched_its_largefile_status_to_be_committed_to_git_now/comment_4_010b0b10ce6f9cd2b9a3f9847ff2a91a._comment b/doc/bugs/cannot_commit___34__annex_add__34__ed_modified_file_which_switched_its_largefile_status_to_be_committed_to_git_now/comment_4_010b0b10ce6f9cd2b9a3f9847ff2a91a._comment new file mode 100644 index 0000000000..e1ec8152d4 --- /dev/null +++ b/doc/bugs/cannot_commit___34__annex_add__34__ed_modified_file_which_switched_its_largefile_status_to_be_committed_to_git_now/comment_4_010b0b10ce6f9cd2b9a3f9847ff2a91a._comment @@ -0,0 +1,10 @@ +[[!comment format=mdwn + username="yarikoptic" + avatar="http://cdn.libravatar.org/avatar/f11e9c84cb18d26a1748c33b48c924b4" + subject="may be one more gitattribute to instruct on either conversion is desired for the file?" + date="2018-08-09T18:59:56Z" + content=""" +I will place implementation and possible tech difficulties aside for now. +I am afraid that here and [there](http://git-annex.branchable.com/bugs/Too_difficult_if_not_impossible_to_explicitly_add__47__keep_file_under_git___40__not_annex__41___in_v6_without_employing_.gitattributes/) we (well, me?) indeed wanted to see two conflicting behaviors somehow happen. On one hand (in [there](http://git-annex.branchable.com/bugs/Too_difficult_if_not_impossible_to_explicitly_add__47__keep_file_under_git___40__not_annex__41___in_v6_without_employing_.gitattributes/)) we would like to keep the file initially committed to git under git, regardless what .gitattributes instructs. +On the other, here I expected file to automagically jump between git and annex depending on `.gitattributes`. So, rather than explicit \"to git\" or \"to annex\" you outlined, to me the question sounds more like \"retain the same storage (git or annex) as before\" or \"possibly perform conversion according to .gitattributes\". And I see usecases where for some files (directories, e.g. `.datalad/metadata`) we would like to see one strategy (auto-conversion) and for the others (default?) the other (maintain git/annex). Given that in v6 there would only be `git add` (so no explicit `git` vs `git annex` add), and that `-n` for `git add` is a flag I was not even aware about, may be it is better to think about being able to explicitly set some additional gitattribute to allow (or disallow?) the conversion for given files, and then have consistent user-level `git annex add` (and in v6 `git add`) which would perform necessary actions across provided files according to `largefiles` and that additional attribute value to decide on the destiny of the file? +"""]] diff --git a/doc/bugs/cannot_commit___34__annex_add__34__ed_modified_file_which_switched_its_largefile_status_to_be_committed_to_git_now/comment_4_8ab0f0852a1dc6c51a502f07faba61eb._comment b/doc/bugs/cannot_commit___34__annex_add__34__ed_modified_file_which_switched_its_largefile_status_to_be_committed_to_git_now/comment_4_8ab0f0852a1dc6c51a502f07faba61eb._comment new file mode 100644 index 0000000000..7a3436b636 --- /dev/null +++ b/doc/bugs/cannot_commit___34__annex_add__34__ed_modified_file_which_switched_its_largefile_status_to_be_committed_to_git_now/comment_4_8ab0f0852a1dc6c51a502f07faba61eb._comment @@ -0,0 +1,8 @@ +[[!comment format=mdwn + username="joey" + subject="""comment 4""" + date="2018-08-09T19:04:28Z" + content=""" +Documented both conversions in + +"""]] diff --git a/doc/bugs/cannot_commit___34__annex_add__34__ed_modified_file_which_switched_its_largefile_status_to_be_committed_to_git_now/comment_6_7081b1386ca8807dff79e8613088b619._comment b/doc/bugs/cannot_commit___34__annex_add__34__ed_modified_file_which_switched_its_largefile_status_to_be_committed_to_git_now/comment_6_7081b1386ca8807dff79e8613088b619._comment new file mode 100644 index 0000000000..36ae0a0a5a --- /dev/null +++ b/doc/bugs/cannot_commit___34__annex_add__34__ed_modified_file_which_switched_its_largefile_status_to_be_committed_to_git_now/comment_6_7081b1386ca8807dff79e8613088b619._comment @@ -0,0 +1,17 @@ +[[!comment format=mdwn + username="joey" + subject="""comment 6""" + date="2018-08-09T19:16:10Z" + content=""" +In v6, `git add` rather than `git annex add` will also add it to the annex, +given annex.largefiles setting. Of course, `git annex add` can also be used +in v6 mode too. And `-n` is a flag to `git commit`, not `git add`. + +Let's please not entangle this bug with that other bug. Unless your goal is +that I merge them. Bear in mind that I consider the other bug a snake pit, +and probably should have closed it as utterly useless some time ago.a + +(Maybe that was uncharitable, but the other bug seems pretty well blocked +on a complete reimplementation of v6 mode leading to a v6 mode that is not +experimental, and entangling this bug into that does not seem wise.) +"""]] diff --git a/doc/bugs/cannot_commit___34__annex_add__34__ed_modified_file_which_switched_its_largefile_status_to_be_committed_to_git_now/comment_7_d987bcd1f589cd9f19a4df92461f9e1a._comment b/doc/bugs/cannot_commit___34__annex_add__34__ed_modified_file_which_switched_its_largefile_status_to_be_committed_to_git_now/comment_7_d987bcd1f589cd9f19a4df92461f9e1a._comment new file mode 100644 index 0000000000..23a34f0e4f --- /dev/null +++ b/doc/bugs/cannot_commit___34__annex_add__34__ed_modified_file_which_switched_its_largefile_status_to_be_committed_to_git_now/comment_7_d987bcd1f589cd9f19a4df92461f9e1a._comment @@ -0,0 +1,15 @@ +[[!comment format=mdwn + username="yarikoptic" + avatar="http://cdn.libravatar.org/avatar/f11e9c84cb18d26a1748c33b48c924b4" + subject="my 1c" + date="2018-08-10T19:17:16Z" + content=""" +> Let's please not entangle this bug with that other bug. + +Sure! I just (probably erroneously) felt that they stem from the same point of absent clear \"semantic\" on either conversion should happen or not. I am yet to fully digest what you are suggesting, and either and how we should address for this at datalad level, but meanwhile FWIW: + +- adding `-n` to the `commit` (and not to `add`) is as uncommon to me in my daily use of git/git-annex, and I hope that I would never have to use it while performing regular \"annex unlock file(s); annex add file(s); commit file(s)\" sequence in order to maintain a file(s) under annex. + +- either a file `smallen` according to git-annex/largefiles setting is unknown to the user (or some higher level tool using git-annex as datalad) without explicitly checking (not even sure yet how) or doing `git annex add`-ing it/them and seeing either it would now be added to git whenever it was added to annex before. So hopefully we do not need to do that either. + +"""]] diff --git a/doc/bugs/cannot_commit___34__annex_add__34__ed_modified_file_which_switched_its_largefile_status_to_be_committed_to_git_now/comment_8_a9100e161d004729db9e3157fcb78343._comment b/doc/bugs/cannot_commit___34__annex_add__34__ed_modified_file_which_switched_its_largefile_status_to_be_committed_to_git_now/comment_8_a9100e161d004729db9e3157fcb78343._comment new file mode 100644 index 0000000000..47d80128d1 --- /dev/null +++ b/doc/bugs/cannot_commit___34__annex_add__34__ed_modified_file_which_switched_its_largefile_status_to_be_committed_to_git_now/comment_8_a9100e161d004729db9e3157fcb78343._comment @@ -0,0 +1,18 @@ +[[!comment format=mdwn + username="joey" + subject="""comment 8""" + date="2018-08-29T16:14:09Z" + content=""" +That other bug is closed now, and +let to [[!commit 10138056dc240ad4265a4c473a32d4c715bbb629]]. + +So in v6, when annex.largefiles is not configured for a file, +git add of a modification to the file will store it however the file was +stored before. But when annex.largefiles is configured, it overrides that +default. + +That didn't change the behavior of git-annex add, but we could consider +doing so. Although I don't know if it's technically possible to support +that behavior in v5 mode, due to the difficulties discussed in comment #3 +above. +"""]] diff --git a/doc/bugs/cannot_remove___96__.t__96___directory.mdwn b/doc/bugs/cannot_remove___96__.t__96___directory.mdwn new file mode 100644 index 0000000000..cd7c15daff --- /dev/null +++ b/doc/bugs/cannot_remove___96__.t__96___directory.mdwn @@ -0,0 +1,37 @@ +### Please describe the problem. + +I am running `git-annex test`, it stops on an NFS mount with exhausted resources (too many open FDs). + +Then I try to remove the `.t` directory with `rm -rf .t` and get many errors like: + +``` +rm: cannot remove `.t/tmprepo61/.git/annex/objects/96/qw/SHA256E-s10--bcc2eba25d48be76c9d968f2d818196b0e10e13476dcff75371fc0ca4910450c/SHA256E-s10--bcc2eba25d48be76c9d968f2d818196b0e10e13476dcff75371fc0ca4910450c': Permission denied +``` + +The permissions on those files look normal (they are mine). + +### What steps will reproduce the problem? + +see above + +### What version of git-annex are you using? On what operating system? + +git HEAD (self-built). On Linux. + +### Please provide any additional information below. + +[[!format sh """ +# If you can, paste a complete transcript of the problem occurring here. +# If the problem is with the git-annex assistant, paste in .git/annex/daemon.log + + +# End of transcript or log. +"""]] + +### Have you had any luck using git-annex before? (Sometimes we get tired of reading bug reports all day and a lil' positive end note does wonders) + +On /tmp the self-test works. Maybe it is related to NFS? + +> Turns out that git-annex was keeping files in .t open in some +> circumstances after deleting them. I have fixed some of this. Possibly +> not all. --[[Joey]] diff --git a/doc/bugs/cannot_remove___96__.t__96___directory/comment_1_98e0522782acbe51fcc274a974e3e346._comment b/doc/bugs/cannot_remove___96__.t__96___directory/comment_1_98e0522782acbe51fcc274a974e3e346._comment new file mode 100644 index 0000000000..6ef5bd4713 --- /dev/null +++ b/doc/bugs/cannot_remove___96__.t__96___directory/comment_1_98e0522782acbe51fcc274a974e3e346._comment @@ -0,0 +1,12 @@ +[[!comment format=mdwn + username="joey" + subject="""comment 1""" + date="2016-01-29T17:20:38Z" + content=""" +`chmod -R u+w` will let you delete the directory. + +`git-annex test` normally takes care of removing the directory for you. +Only because it crashed did it not in this case. It would be helpful if you +could show the actual result of running it, including the real error +message. Perhaps there's something that could be improved. +"""]] diff --git a/doc/bugs/cause__58___fake_annex_repo___124___result__58___inflated_numcopies___124___workaround__58___initialize_properly.mdwn b/doc/bugs/cause__58___fake_annex_repo___124___result__58___inflated_numcopies___124___workaround__58___initialize_properly.mdwn new file mode 100644 index 0000000000..fc098bb84b --- /dev/null +++ b/doc/bugs/cause__58___fake_annex_repo___124___result__58___inflated_numcopies___124___workaround__58___initialize_properly.mdwn @@ -0,0 +1,84 @@ +
    +
+***** summary
+
+if I run sync, and there is an annex-less git remote in the network
+the annex-less git repo will gain a v5 annex branch
+it knows it can't store binary files
+but all the properly initialized annex repos in the network don't know that
+
+when I run "sync --content",
+the initialized annex repos think that the uninitialized repo contains copies.
+
+I suspect this results in inaccurate "copies" count
+I know it results in an inaccurate "list files' location" graph
+and also inaccurate "whereis" readout
+
+***** reproduction
+
+repo "Alpha" is a git-annex repo with file "music"
+repo "Zeta" is a git repo cloned from Alpha
+
+@Alpha:
+git annex sync --content
+git annex list
+[shows music present on Zeta]
+git annex whereis
+[shows music present on Zeta]
+
+@Zeta
+git annex list
+[shows nothing]
+git annex whereis
+[shows music present on Zeta]
+
+***** proving that the problem is "sync --content"
+
+Hypothesis:
+
+maybe the issue is that I cloned the repo
+rather than creating it normally,
+thus leaving it in a half-annexed state?
+
+Experiment:
+
+tested by creating an independent git repo
+and then adding it as a remote to Alpha.
+
+then ran sync --content
+
+Result:
+
+no different than before.
+
+Conclusion:
+
+The problem lies with 
+git-annex sync --content
+
+***** ramifications
+
+It would appear this error can cause data loss
+due to a false numcopies count.
+
+Yet GitHub is supposed to work.
+So this error should've already been noticed.
+Contradiction detected.
+
+Positivity: I am planning on becoming a git-annex evangelist as part of a larger project.
+Emacs offers ergonomic control via magit and a dired plugin.
+
+***** environment
+
+git-annex version: 6.20170301-ga9e1e17d40
+build flags: Assistant Webapp Pairing Testsuite S3(multipartupload)(storageclasses) WebDAV Inotify DBus DesktopNotify ConcurrentOutput TorrentParser MagicMime Feeds Quvi
+key/value backends: SHA256E SHA256 SHA512E SHA512 SHA224E SHA224 SHA384E SHA384 SHA3_256E SHA3_256 SHA3_512E SHA3_512 SHA3_224E SHA3_224 SHA3_384E SHA3_384 SKEIN256E SKEIN256 SKEIN512E SKEIN512 SHA1E SHA1 MD5E MD5 WORM URL
+remote types: git gcrypt p2p S3 bup directory rsync web bittorrent webdav tahoe glacier ddar hook external
+local repository version: unknown
+supported repository versions: 3 5 6
+upgrade supported from repository versions: 0 1 2 3 4 5
+operating system: linux x86_64
+
+
    + +> [[done]] appears to be a confused user, not a bug. --[[Joey]] diff --git a/doc/bugs/cause__58___fake_annex_repo___124___result__58___inflated_numcopies___124___workaround__58___initialize_properly/comment_1_6e85b0122fca46b9232f8fd34e8c9f6d._comment b/doc/bugs/cause__58___fake_annex_repo___124___result__58___inflated_numcopies___124___workaround__58___initialize_properly/comment_1_6e85b0122fca46b9232f8fd34e8c9f6d._comment new file mode 100644 index 0000000000..16ab9423f0 --- /dev/null +++ b/doc/bugs/cause__58___fake_annex_repo___124___result__58___inflated_numcopies___124___workaround__58___initialize_properly/comment_1_6e85b0122fca46b9232f8fd34e8c9f6d._comment @@ -0,0 +1,16 @@ +[[!comment format=mdwn + username="Cyberthal" + avatar="http://cdn.libravatar.org/avatar/1c619d65ee07d2343295c8f70f23c9df" + subject="knowing about annex-ignore and git-annex shell helps" + date="2017-03-26T10:06:35Z" + content=""" +When I wrote the above, +I was unaware of the annex-ignore flag +which can avert the above issue. + +This flag should've been automatically set. + +Also, I learned about the need for git-annex shell +which must be available at the default path +of remote servers. +"""]] diff --git a/doc/bugs/cause__58___fake_annex_repo___124___result__58___inflated_numcopies___124___workaround__58___initialize_properly/comment_2_13989894486a9ff9f243c6b2071d788f._comment b/doc/bugs/cause__58___fake_annex_repo___124___result__58___inflated_numcopies___124___workaround__58___initialize_properly/comment_2_13989894486a9ff9f243c6b2071d788f._comment new file mode 100644 index 0000000000..5ffd00d71f --- /dev/null +++ b/doc/bugs/cause__58___fake_annex_repo___124___result__58___inflated_numcopies___124___workaround__58___initialize_properly/comment_2_13989894486a9ff9f243c6b2071d788f._comment @@ -0,0 +1,25 @@ +[[!comment format=mdwn + username="joey" + subject="""comment 2""" + date="2017-04-07T16:42:11Z" + content=""" +This bug report is, to the best of my knowledge, totally wrong. + +Yes, git-annex sync pushes the git-annex branch to all git remotes. +This is intentional, and not a problem. That is just a git branch. +It's helpful to push it to eg, github, if you want to be able to pull it +from there into another clone. Pushing the git-annex branch to github does +*not* make git-annex think that github is holding the contents of annexed +files. + +In your "reproduction" section, I think you forgot to run `git annex sync` +in Zeta, so its working tree had not been updated with the files synced to +it from Alpha, which is why `git annex list` didn't show anything. Or +somethig like that. You did not provide quite enough information to guess +what you were doing. + +I'm going to close this. If you think it is still a problem, please provide +either a detailed transcript demontrating the problem or enough information +to reporoduce the problem, starting with an empty directory and +constructing the necessary git repositories to demonstrate the problem. +"""]] diff --git a/doc/bugs/cause__58___fake_annex_repo___124___result__58___inflated_numcopies___124___workaround__58___initialize_properly/comment_3_79ddc1a3c554efb375b9575687e1ee04._comment b/doc/bugs/cause__58___fake_annex_repo___124___result__58___inflated_numcopies___124___workaround__58___initialize_properly/comment_3_79ddc1a3c554efb375b9575687e1ee04._comment new file mode 100644 index 0000000000..b44311d105 --- /dev/null +++ b/doc/bugs/cause__58___fake_annex_repo___124___result__58___inflated_numcopies___124___workaround__58___initialize_properly/comment_3_79ddc1a3c554efb375b9575687e1ee04._comment @@ -0,0 +1,8 @@ +[[!comment format=mdwn + username="Cyberthal" + avatar="http://cdn.libravatar.org/avatar/1c619d65ee07d2343295c8f70f23c9df" + subject="PEBKAC" + date="2017-05-01T22:19:05Z" + content=""" +I was unable to duplicate the problem. Your explanation sounds likely. I shall strive to do better! +"""]] diff --git a/doc/bugs/checkpresentkey_batch_stops_at_97_or_98_keys.mdwn b/doc/bugs/checkpresentkey_batch_stops_at_97_or_98_keys.mdwn new file mode 100644 index 0000000000..ff761904db --- /dev/null +++ b/doc/bugs/checkpresentkey_batch_stops_at_97_or_98_keys.mdwn @@ -0,0 +1,63 @@ +### Please describe the problem. + +**git annex checkpresentkey --batch $remote** doesn't check all the keys it is provided with. + +Depending how this is run, given 8000 keys, it may return info only the first handful or ~97. + +### What steps will reproduce the problem? + +The directory contains 8000 symlinks whose filename is the same as their key. The content is not in 'spearmint' (or any other configured remotes, but is present 'here'). + + $ find . -type l -printf "%f\n" | git annex checkpresentkey --batch | wc -l + 8000 + $ find . -type l -printf "%f\n" | git annex checkpresentkey --batch spearmint | wc -l + 97 + $ find . -type l -printf "%f\n" | git annex checkpresentkey --batch spearmint | wc -l + 97 + +Without a remote, all get checked.. + + $ git annex find --format '${key}\n' . | git annex checkpresentkey --batch | wc -l + 8000 + $ git annex find --format '${key}\n' . | git annex checkpresentkey --batch | sort | uniq -c + 8000 0 + +Specifying a remote only checks a small handful of keys (count changes each time).. + + $ git annex find --format '${key}\n' . | git annex checkpresentkey --batch spearmint | wc -l + 6 + $ git annex find --format '${key}\n' . | git annex checkpresentkey --batch spearmint | wc -l + 14 + $ git annex find --format '${key}\n' . | git annex checkpresentkey --batch spearmint | wc -l + 7 + $ git annex find --format '${key}\n' . | git annex checkpresentkey --batch spearmint | wc -l + 8 + +Putting the keys into a file seems to make this more consistent (more in line with *find -type f*) + + $ git annex find --format '${key}\n' . > /tmp/keys.txt + $ cat /tmp/keys.txt | git annex checkpresentkey --batch spearmint | wc -l + 96 + $ cat /tmp/keys.txt | git annex checkpresentkey --batch spearmint | wc -l + 96 + +Shuffling the key order doesn't matter much either.. + + $ shuf /tmp/keys.txt | git annex checkpresentkey --batch spearmint | wc -l + 97 + $ shuf /tmp/keys.txt | git annex checkpresentkey --batch spearmint | wc -l + 96 + $ shuf /tmp/keys.txt | git annex checkpresentkey --batch spearmint | wc -l + 97 + + +### What version of git-annex are you using? On what operating system? +git-annex version: 6.20161231-g8740cd971 + +Arch Linux (installed from 'community') + +### Have you had any luck using git-annex before? (Sometimes we get tired of reading bug reports all day and a lil' positive end note does wonders) + +I only find (what I think are) bugs because I use it and I use it because I like it. I like it because it works (except for when I find actual bugs :]). + +> [[fixed|done]] --[[Joey]] diff --git a/doc/bugs/checkpresentkey_batch_stops_at_97_or_98_keys/comment_1_f0d17735d01a04c3c2adeb5ab4c2c0ce._comment b/doc/bugs/checkpresentkey_batch_stops_at_97_or_98_keys/comment_1_f0d17735d01a04c3c2adeb5ab4c2c0ce._comment new file mode 100644 index 0000000000..16d278f22a --- /dev/null +++ b/doc/bugs/checkpresentkey_batch_stops_at_97_or_98_keys/comment_1_f0d17735d01a04c3c2adeb5ab4c2c0ce._comment @@ -0,0 +1,26 @@ +[[!comment format=mdwn + username="joey" + subject="""comment 1""" + date="2017-02-15T18:04:29Z" + content=""" +I am able to reproduce this, and it only happens when the remote being +checked is a ssh remote, not a local directory. + +So, presumably something in the verification that the remote has the +content is sometimes consuming the rest of stdin. + +The different numbers processed each time are probably due to buffering. If +the command feeding the list of keys takes a while to print them all, and +parts of its output are being thrown away, then that would explain the +different numbers processed. + +Using ssh -n to run git-annex-shell checkpresentkey avoids the problem. + +This could also impact git-annex being used in some script, when the script +is intended to consume stdin, but git-annex runs ssh, which consumes it +instead. Other commands like `git annex drop` could be affected +too in such situations. + +I've put in a comprehensive fix to all of git-annex's calls to ssh +that don't provide some other stdin. +"""]] diff --git a/doc/bugs/clash_of_-j__in_copy_for_--json_--json-progress.mdwn b/doc/bugs/clash_of_-j__in_copy_for_--json_--json-progress.mdwn new file mode 100644 index 0000000000..43988d94dd --- /dev/null +++ b/doc/bugs/clash_of_-j__in_copy_for_--json_--json-progress.mdwn @@ -0,0 +1,18 @@ +### What version of git-annex are you using? On what operating system? + +6.20170101+gitg93d69b1-1~ndall+1 + +### Please provide any additional information below. + +[[!format sh """ +$> git annex copy --help +... + -j,--json enable JSON output + -j,--json-progress include progress in JSON output + +"""]] + +[[!meta author=yoh]] + +> Removed the short -j for --json-progress, I don't think that +> needs a short option. [[done]] --[[Joey]] diff --git a/doc/bugs/commitBuffer__58___invalid_argument___40__invalid_character__41__2.mdwn b/doc/bugs/commitBuffer__58___invalid_argument___40__invalid_character__41__2.mdwn new file mode 100644 index 0000000000..1be88f4a14 --- /dev/null +++ b/doc/bugs/commitBuffer__58___invalid_argument___40__invalid_character__41__2.mdwn @@ -0,0 +1,24 @@ +### Please describe the problem. + +git-annex fails if there are any "invalid characters" in the filename. That should have been fixed already? In one of my repos I cannot commit anymore: + +$ git commit -m u +git-annex: .git/annex/misctmp/jlog6281: commitBuffer: invalid argument (invalid character) + +This logfile is empty by the way. The obvious fix would be to move out of germany and to not listen to german podcasts, but there has to be another solution :). + +Because I did not have any particular problems with that in the past, this bug must have been introduced in one of the latest releases I guess. + +### What steps will reproduce the problem? + +Try + +git-annex importfeed --fast http://ulm.ccc.de/dev/radio/podcast-ogg.xml + +It will fail at entry "DR14: Verschwörungstheorien" + +### What version of git-annex are you using? On what operating system? + +5.20150327-g19a1a35 (standalone build) on Fedora 21 + +[[!tag moreinfo]] diff --git a/doc/bugs/commitBuffer__58___invalid_argument___40__invalid_character__41__2/comment_1_b56c847c5eda432a4330b4d853a25519._comment b/doc/bugs/commitBuffer__58___invalid_argument___40__invalid_character__41__2/comment_1_b56c847c5eda432a4330b4d853a25519._comment new file mode 100644 index 0000000000..43e6a390b2 --- /dev/null +++ b/doc/bugs/commitBuffer__58___invalid_argument___40__invalid_character__41__2/comment_1_b56c847c5eda432a4330b4d853a25519._comment @@ -0,0 +1,27 @@ +[[!comment format=mdwn + username="joey" + subject="""comment 1""" + date="2015-04-09T17:49:57Z" + content=""" +I cannot reproduce this, I get: + + addurl _dev_radio/DR14__Verschwörungstheorien.ogg ok + +Does the _dev_radio/DR14__Verschwörungstheorien.ogg file get created? +If so, how does it look? + +The jlog tells me it's trying to commit the git-annex branch journal. +Does .git/annex/journal/ contain any files? Any files containing German +characters? + +Do you have any git config settings for git-annex beyone the typical +annex.uuid? + +I noticed one place in the journal commit code where it does seem to +neglect to use filesystem encoding when dealing with writing filenames to +the jlog tmpfile. Which could lead to this crash theoretically. I've fixed +that, but since I couldn't reproduce the problem, I don't know if this will +fix your problem. Nor do I understand how annex journal log files could +have these characters in their names. You can try today's upcoming release +of git-annex to test the fix though. +"""]] diff --git a/doc/bugs/committing_files_into_git_doesn__39__t_work_with_explicitly_given_paths_in_V6.mdwn b/doc/bugs/committing_files_into_git_doesn__39__t_work_with_explicitly_given_paths_in_V6.mdwn new file mode 100644 index 0000000000..35f198d67b --- /dev/null +++ b/doc/bugs/committing_files_into_git_doesn__39__t_work_with_explicitly_given_paths_in_V6.mdwn @@ -0,0 +1,50 @@ +### Please describe the problem. +If there are staged changes in two files (one in git, one in annex) and I want to commit them by explicitly giving git-commit both paths (in order to exclude other changes that are possibly staged), the not annexed file stays staged, while committing everything staged by not giving any path to git-commit works just fine. + +### What steps will reproduce the problem? +[[!format sh """ +mkdir testv6 +cd testv6 +git init +git annex init --version=6 +echo some > file_in_git +git -c annex.largefiles=nothing annex add file_in_git +echo more > file_in_annex +git annex add file_in_annex +git commit -m "files added" file_in_git file_in_annex +git status +"""]] + + +### What version of git-annex are you using? On what operating system? +[[!format sh """ +% git annex version +git-annex version: 6.20170307+gitg24ade8a25-1~ndall+1 +build flags: Assistant Webapp Pairing Testsuite S3(multipartupload)(storageclasses) WebDAV Inotify DBus DesktopNotify ConcurrentOutput TorrentParser MagicMime Feeds Quvi +key/value backends: SHA256E SHA256 SHA512E SHA512 SHA224E SHA224 SHA384E SHA384 SHA3_256E SHA3_256 SHA3_512E SHA3_512 SHA3_224E SHA3_224 SHA3_384E SHA3_384 SKEIN256E SKEIN256 SKEIN512E SKEIN512 SHA1E SHA1 MD5E MD5 WORM URL +remote types: git gcrypt p2p S3 bup directory rsync web bittorrent webdav tahoe glacier ddar hook external +local repository version: 6 +supported repository versions: 3 5 6 +upgrade supported from repository versions: 0 1 2 3 4 5 +operating system: linux x86_64 + +"""]] + +### Please provide any additional information below. + +[[!format sh """ +# If you can, paste a complete transcript of the problem occurring here. +# If the problem is with the git-annex assistant, paste in .git/annex/daemon.log + + +# End of transcript or log. +"""]] + +### Have you had any luck using git-annex before? (Sometimes we get tired of reading bug reports all day and a lil' positive end note does wonders) + + + +[[ben]] + +> [[done]]; clean filter defaults to preserving git/annex state of file. +> --[[Joey]] diff --git a/doc/bugs/committing_files_into_git_doesn__39__t_work_with_explicitly_given_paths_in_V6/comment_1_1922f38b3620e94e90b16e3c14f59add._comment b/doc/bugs/committing_files_into_git_doesn__39__t_work_with_explicitly_given_paths_in_V6/comment_1_1922f38b3620e94e90b16e3c14f59add._comment new file mode 100644 index 0000000000..5faf6b71ce --- /dev/null +++ b/doc/bugs/committing_files_into_git_doesn__39__t_work_with_explicitly_given_paths_in_V6/comment_1_1922f38b3620e94e90b16e3c14f59add._comment @@ -0,0 +1,18 @@ +[[!comment format=mdwn + username="joey" + subject="""comment 1""" + date="2017-04-07T16:24:59Z" + content=""" +The root problem is, you have set annex.largefiles=nothing temporarily when +adding the file. But, when git commit re-smudges the file, that is not set, +so git-annex generates a v6 pointer, which is what gets committed. + +I don't think you will have these problems if use use .gitattributes to +configure annex.largefiles. + +(There is something going on that I don't quite understand with why +git status then thinks the file has changed. git diff --cached shows a diff +between the pointer that got committed and the actual file contents; +I would have expected that git would run the smudge/clean filter then and +not show that diff.) +"""]] diff --git a/doc/bugs/concurrent_git-annex_processes_can_lead_to_locking_issues.mdwn b/doc/bugs/concurrent_git-annex_processes_can_lead_to_locking_issues.mdwn new file mode 100644 index 0000000000..2485e7b19e --- /dev/null +++ b/doc/bugs/concurrent_git-annex_processes_can_lead_to_locking_issues.mdwn @@ -0,0 +1,53 @@ +When two git-annex processes are running and both modifying the git-annex +branch, it's possible one will fail due to git's locking. When this +happens, git-annex has already recorded its state in the journal (so no +data is lost), but git-annex does crash, which can be surprising. + +I feel that, in general, multiple git-annex processes should be able to run +concurrently. A big lock around all commands, or even all +repository-modifying commands is a bad idea. Also, it's probably best to +only worry about locking conflicts editing the git-annex branch. While `git +annex add` and a few other commands make changes to the main git repo, +and can have similar locking issues, so can any git commands that stage +changes (I think.. check). + +Probably should KISS. Just add a lock file that is taken before changes to +the git-annex branch, and if it's locked, wait. Changes to the git-annex +branch tend to happen quickly (unless it's committing an enormous set of +changes, and even that is relatively fast), so waiting seems ok. --[[Joey]] + +---- + +Commit 7981eb4cb512fbe3c49a3dd165c31be14ae4bc49 is more pessimistic, +describes some other potential issues. + +* The journal needs to be emptied (done) and kept locked (not done) during + a merge, since a merge operates at a level below the journal, and any + changes that are journaled during a merge can overwrite changes merged + in from another branch. + +* Two git-annex processes can be doing conflicting things and inconsistent + information be written to the journal. + + - One example would be concurrent get and drop of the same key. + But could this really race? If the key was already present, the get + would do nothing, so record no changes. If the key was not yet present, + the drop would do nothing, and record no changes. + + - Instead, consider two copys of a key to different locations. If the + slower copy starts first and ends last, it could cache the location + info, add the new location, and lose the other location it was copied to. + Tested it and the location is not cached during the whole copy (logChange + reads the current log immediatly before writing), so this + race's window is very small -- but does exist. + +---- + +## Updated plan + +Make Branch.change transactional, so it takes a lock, reads a file, +applies a function to it, and writes the changed file. + +Make Branch.update hold the same lock. + +> [[Done]]. diff --git a/doc/bugs/configuration_options_for_system_ssh_and_bundled_ssh_are_incompatible.mdwn b/doc/bugs/configuration_options_for_system_ssh_and_bundled_ssh_are_incompatible.mdwn new file mode 100644 index 0000000000..a61caca667 --- /dev/null +++ b/doc/bugs/configuration_options_for_system_ssh_and_bundled_ssh_are_incompatible.mdwn @@ -0,0 +1,54 @@ +### Please describe the problem. + +It appears that the bundled ssh is built without support for some configuration options supported by my system ssh. In particular it doesn't support GSSAPIKexAlgorithms. In principle it's fine to compile this ssh without support for GSS, since nobody uses it, but in practice it means that I can't even turn it off in my system config without breaking the bundled ssh. + +(I suppose the problem could also be that the bundled ssh tries to use the system ssh configuration instead of a bundled configuration.) + +### What version of git-annex are you using? On what operating system? + +[[!format sh """ +git-annex version: 6.20161231-gc8eeb17da +build flags: Assistant Webapp Pairing Testsuite S3(multipartupload)(storageclasses) WebDAV Inotify DBus DesktopNotify ConcurrentOutput TorrentParser MagicMime Feeds Quvi +key/value backends: SHA256E SHA256 SHA512E SHA512 SHA224E SHA224 SHA384E SHA384 SHA3_256E SHA3_256 SHA3_512E SHA3_512 SHA3_224E SHA3_224 SHA3_384E SHA3_384 SKEIN256E SKEIN256 SKEIN512E SKEIN512 SHA1E SHA1 MD5E MD5 WORM URL +remote types: git gcrypt p2p S3 bup directory rsync web bittorrent webdav tahoe glacier ddar hook external +local repository version: 6 +supported repository versions: 3 5 6 +upgrade supported from repository versions: 0 1 2 3 4 5 +operating system: linux x86_64 +"""]] + +### Please provide any additional information below. + +See also [[todo/git-annex_ignores_GIT__95__SSH]] + +[[!format sh """ + db48x  ~  video  git-annex sync +commit +On branch master +Your branch is ahead of 'anglachel/master' by 20 commits. + (use "git push" to publish your local commits) +nothing to commit, working tree clean +ok +pull anglachel +/etc/crypto-policies/back-ends/openssh.config: line 3: Bad configuration option: gssapikexalgorithms +/etc/crypto-policies/back-ends/openssh.config: terminating, 1 bad configuration options +fatal: Could not read from remote repository. +Please make sure you have the correct access rights +and the repository exists. +failed +push anglachel +/etc/crypto-policies/back-ends/openssh.config: line 3: Bad configuration option: gssapikexalgorithms +/etc/crypto-policies/back-ends/openssh.config: terminating, 1 bad configuration options +fatal: Could not read from remote repository. +Please make sure you have the correct access rights +and the repository exists. + Pushing to anglachel failed. + (non-fast-forward problems can be solved by setting receive.denyNonFastforwards to false in the remote's git config) +failed +git-annex: sync: 2 failed +"""]] + +### Have you had any luck using git-annex before? (Sometimes we get tired of reading bug reports all day and a lil' positive end note does wonders) + + +> [[done]]; moved ssh to extra directory. --[[Joey]] diff --git a/doc/bugs/configuration_options_for_system_ssh_and_bundled_ssh_are_incompatible/comment_1_abdd5622dc1733a8b59dc11885083799._comment b/doc/bugs/configuration_options_for_system_ssh_and_bundled_ssh_are_incompatible/comment_1_abdd5622dc1733a8b59dc11885083799._comment new file mode 100644 index 0000000000..2024b296f3 --- /dev/null +++ b/doc/bugs/configuration_options_for_system_ssh_and_bundled_ssh_are_incompatible/comment_1_abdd5622dc1733a8b59dc11885083799._comment @@ -0,0 +1,20 @@ +[[!comment format=mdwn + username="joey" + subject="""comment 1""" + date="2017-03-02T21:30:55Z" + content=""" +One way to deal with this would be to put the bundled ssh in the +git-annex.linux/extra/ directory. Programs in that directory are added to the +end of the path, so that system versions will be preferred when available. + +That was earlier done for gpg and didn't cause any problems. + +There is the potential for some sort of versioning problem. Currently the +only thing git-annex probes about the bundled ssh is if it supports +connection caching. That's quite an old feature by now, so a system ssh not +supporting it would either be super out of date and insecure openssh, or +perhaps some other ssh implementation. + +Seems worth moving the bundled ssh to the extra directory, and see what +breaks.. +"""]] diff --git a/doc/bugs/confirmed.mdwn b/doc/bugs/confirmed.mdwn new file mode 100644 index 0000000000..1bb3c864fc --- /dev/null +++ b/doc/bugs/confirmed.mdwn @@ -0,0 +1,5 @@ +This tag is for bugs that have been confirmed to be real bugs, and so are likely +to be the next bugs fixed. + +If your bug report is not tagged as confirmed, you probably need to provide more +information so that the bug can be reproduced. diff --git a/doc/bugs/copy_does_not_reflect_some_failed_copies_in_--json_output.mdwn b/doc/bugs/copy_does_not_reflect_some_failed_copies_in_--json_output.mdwn new file mode 100644 index 0000000000..1d216cc05f --- /dev/null +++ b/doc/bugs/copy_does_not_reflect_some_failed_copies_in_--json_output.mdwn @@ -0,0 +1,23 @@ +### Please describe the problem. + +see below. expected: entries on failed copies with "success"=false, and not just error messages + + +[[!format sh """ + +$> git annex copy --to=neurosmaug svgtune_0.2.0.orig.tar.gz nonexisting +git-annex: nonexisting not found +copy svgtune_0.2.0.orig.tar.gz (checking neurosmaug...) ok +git-annex: copy: 1 failed + +$> git annex copy --to=neurosmaug svgtune_0.2.0.orig.tar.gz nonexisting --json +git-annex: nonexisting not found +{"command":"copy","note":"checking neurosmaug...","success":true,"key":"SHA256E-s5121--6d8f7d10206a120a42bec2cd29bc2365d09889fdf070ac8c67d1cff0b1539f63.tar.gz","file":"svgtune_0.2.0.orig.tar.gz"} +git-annex: copy: 1 failed + +$> git annex version +git-annex version: 6.20170220+gitg75a15e1ad-1~ndall+1 + +"""]] + +[[!meta author=yoh]] diff --git a/doc/bugs/copy_does_not_reflect_some_failed_copies_in_--json_output/comment_1_4ed0e29f0b599135b1b1c9197d58873b._comment b/doc/bugs/copy_does_not_reflect_some_failed_copies_in_--json_output/comment_1_4ed0e29f0b599135b1b1c9197d58873b._comment new file mode 100644 index 0000000000..d41ce9ebd4 --- /dev/null +++ b/doc/bugs/copy_does_not_reflect_some_failed_copies_in_--json_output/comment_1_4ed0e29f0b599135b1b1c9197d58873b._comment @@ -0,0 +1,15 @@ +[[!comment format=mdwn + username="joey" + subject="""comment 1""" + date="2017-03-02T17:08:21Z" + content=""" +I get what you're saying, but there's a layering problem that prevents +the JSON display of a higher layer being used by a lower layer that is +finding files to operate on. + +Trying to copy a file that does not exist is a fundamentally +different class of error that an error that occurs when copying a file +that does exist. A memory read error resulting in a SIGBUS is a different +class of error, as is a git failure due to a corrupted repository. +git-annex fails in four entirely different ways in these four cases. +"""]] diff --git a/doc/bugs/corrupt_backend_upon_sync__63__.mdwn b/doc/bugs/corrupt_backend_upon_sync__63__.mdwn new file mode 100644 index 0000000000..e99dfcc88c --- /dev/null +++ b/doc/bugs/corrupt_backend_upon_sync__63__.mdwn @@ -0,0 +1,78 @@ +### Please describe the problem. + +When syncing with a remote, I get some files with "unknown backend". + +Original: + + $ ls -lh Pictures/2014/06/21/2014-06-21\ 13.52.34.png + lrwxrwxrwx 1 jean jean 214 Jun 21 2014 Pictures/2014/06/21/2014-06-21 13.52.34.png -> ../../../../.git/annex/objects/mx/Ff/SHA256E-s247069--d425c1049778880eb9b9fcab74a82ec86dd22882db9c026f8fc0e9cb6270d022.34.png/SHA256E-s247069--d425c1049778880eb9b9fcab74a82ec86dd22882db9c026f8fc0e9cb6270d022.34.png + +This picture and symlink is fine. + +Remote (broken symlink): + + $ ls -lh Pictures/2014/06/21/2014-06-21\ 13.52.34.png + lrwxrwxrwx 1 jean jean 214 Apr 6 12:50 Pictures/2014/06/21/2014-06-21 13.52.34.png -> ./../../.git/annex/objects/mx/Ff/SHA256E-s247069--d425c134ac2f9b0eb9b9fcab74a82ec86dd22882db9c026f8fc0e9cb6270d022.34.pn0.png/SHE-s247069--d425c1049778880eb9b9fcab74a82ec86dd22882db9c09efe84ca9cb6270d022.34.png + +In this case the paths differ: there is an extra `.pn0` component, and the final `SHA256E` has changed to `SHE`. + +### What steps will reproduce the problem? + +I don't know yet. + +### What version of git-annex are you using? On what operating system? + + $ git annex version + git-annex version: 5.20140412ubuntu1 + build flags: Assistant Webapp Webapp-secure Pairing Testsuite S3 WebDAV Inotify DBus DesktopNotify XMPP DNS Feeds Quvi TDFA CryptoHash + key/value backends: SHA256E SHA1E SHA512E SHA224E SHA384E SKEIN256E SKEIN512E SHA256 SHA1 SHA512 SHA224 SHA384 SKEIN256 SKEIN512 WORM URL + remote types: git gcrypt S3 bup directory rsync web webdav tahoe glacier hook external + local repository version: 5 + supported repository version: 5 + upgrade supported from repository versions: 0 1 2 4 + +Ubuntu 14.04.1 LTS + +### Please provide any additional information below. + +[[!format sh """ +# If you can, paste a complete transcript of the problem occurring here. +# If the problem is with the git-annex assistant, paste in .git/annex/daemon.log + +$ git annex info +repository mode: indirect +trusted repositories: 0 +semitrusted repositories: 6 + 00000000-0000-0000-0000-000000000001 -- web + ... +untrusted repositories: 0 +transfers in progress: none +available local disk space: 168.35 gigabytes (+1 megabyte reserved) +local annex keys: 16503 +local annex size: 39.15 gigabytes +annexed files in working tree: + skipping Pictures/2014/06/21/2014-06-21 13.52.34.png (unknown backend SHE) + + skipping Pictures/2014/06/21/2014-06-21 13.52.45.png (unknown backend SHE) + + skipping Pictures/2014/06/21/2014-06-21 13.58.03.png (unknown backend SHA256EE) +104552 +size of annexed files in working tree: 124.64 gigabytes +bloom filter size: 16 mebibytes (3.3% full) +backend usage: + SHA256E: 121054 + SHA256: 1 + +$ git annex get --from=laptop + + skipping Pictures/2014/06/21/2014-06-21 13.52.34.png (unknown backend SHE) + + skipping Pictures/2014/06/21/2014-06-21 13.52.45.png (unknown backend SHE) + + skipping Pictures/2014/06/21/2014-06-21 13.58.03.png (unknown backend SHA256EE) + + +# End of transcript or log. +"""]] + +[[!tag moreinfo]] diff --git a/doc/bugs/corrupt_backend_upon_sync__63__/comment_1_9f248d82f93040b739b56515d18458c7._comment b/doc/bugs/corrupt_backend_upon_sync__63__/comment_1_9f248d82f93040b739b56515d18458c7._comment new file mode 100644 index 0000000000..ddec07c1eb --- /dev/null +++ b/doc/bugs/corrupt_backend_upon_sync__63__/comment_1_9f248d82f93040b739b56515d18458c7._comment @@ -0,0 +1,24 @@ +[[!comment format=mdwn + username="joey" + subject="""comment 1""" + date="2015-04-09T17:39:28Z" + content=""" +The symink that you're showing is a file checked into git. + +So, you should be able to run `git log 'Pictures/2014/06/21/2014-06-21 13.52.34.png'` +on the remote and find a commit that somehow changed the symlink to the +broken one that the remote has. + +The only other possibilities are + +* Somehow the data in git in the remote got corrupted, and git didn't + notice. Seems very unlikely. +* Somehow git decided to munge up the symlink when checking it out on the + remote. While there are some git features like smudge filters that could + perhaps be configured to do that, I don't see how git could do it on its + own. + +I've never seen git do anything like this. +You're going to have to investigate this on your own and/or provide enough +information to reproduce the problem. +"""]] diff --git a/doc/bugs/could_standalone_copy_of_git_ship__47__deploy_hooks_samples_as_well_as_stock_git__63__.mdwn b/doc/bugs/could_standalone_copy_of_git_ship__47__deploy_hooks_samples_as_well_as_stock_git__63__.mdwn new file mode 100644 index 0000000000..9f81f8ab54 --- /dev/null +++ b/doc/bugs/could_standalone_copy_of_git_ship__47__deploy_hooks_samples_as_well_as_stock_git__63__.mdwn @@ -0,0 +1,61 @@ +### Please describe the problem. + +ATM, if bundled git is used instead of system-wide available (e.g. standalone git-annex's PATH is preceding system's /usr/bin), then user running 'git init' uses bundled version of git. In comparison to "full regular" installation of git, bundled version doesn't carry hooks, so any "git init"ed with it repository would lack those samples in .git/hooks + +### What steps will reproduce the problem? + +just use bundled git and run "git init" in some new dir + +### What version of git-annex are you using? On what operating system? + +6.20170209+gitg16be7b5cc-1~ndall+1 + + +### Please provide any additional information below. + +actually there is few more differences (/tmp/123 generated with bundled version) -- description, branches: + +[[!format sh """ +(git)hopa:/tmp/123[master] +$> find .git +.git +.git/objects +.git/objects/info +.git/objects/pack +.git/config +.git/HEAD +.git/refs +.git/refs/tags +.git/refs/heads + +(git)hopa:/tmp/123_system[master] +$> find .git +.git +.git/objects +.git/objects/info +.git/objects/pack +.git/config +.git/HEAD +.git/refs +.git/refs/tags +.git/refs/heads +.git/info +.git/info/exclude +.git/hooks +.git/hooks/prepare-commit-msg.sample +.git/hooks/pre-applypatch.sample +.git/hooks/pre-rebase.sample +.git/hooks/pre-receive.sample +.git/hooks/pre-commit.sample +.git/hooks/pre-push.sample +.git/hooks/commit-msg.sample +.git/hooks/post-update.sample +.git/hooks/applypatch-msg.sample +.git/hooks/update.sample +.git/description +.git/branches +"""]] + +[[!meta author=yoh]] + +> [[done]] --[[Joey]] diff --git a/doc/bugs/could_webdav_be_more_resilient_to_timeouts__63__.mdwn b/doc/bugs/could_webdav_be_more_resilient_to_timeouts__63__.mdwn new file mode 100644 index 0000000000..083f1351e6 --- /dev/null +++ b/doc/bugs/could_webdav_be_more_resilient_to_timeouts__63__.mdwn @@ -0,0 +1,44 @@ +### Please describe the problem. + +Upload to box.com (corporate account so filesizes could be large, chunking to 50MB anyways) WebDAV times out quite frequently... I wonder if annex could try more times etc? + + +### What version of git-annex are you using? On what operating system? + +6.20170525+gitge1cf095ae-1~ndall+1 + +### Please provide any additional information below. + +[[!format sh """ +hopa:/tmp/testbox +$> git annex copy -J4 --to=box-dartm3.com --json --json-progress video.mp4 +... +{"byte-progress":13166304,"action":{"command":"copy","note":"to box-dartm3.com...","key":"MD5E-s1073741824--e06da14afc6face003121641e60593bb.mp4","file":"video.mp4"},"total-size":1073741824,"percent-progress":"1.23%"} +{"command":"copy","note":"checking box-dartm3.com...","success":false,"key":"MD5E-s1073741824--e06da14afc6face003121641e60593bb.mp4","file":"video.mp4"} + DAV failure: 408 "Request Timeout" + CallStack (from HasCallStack): + error, called at ./Remote/WebDAV.hs:324:47 in main:Remote.WebDAV +git-annex: copy: 1 failed + +$> git annex copy -J4 --to=box-dartm3.com --json --json-progress video.mp4 +{"byte-progress":0,"action":{"command":"copy","note":"to box-dartm3.com...","key":"MD5E-s1073741824--e06da14afc6face003121641e60593bb.mp4","file":"video.mp4"},"total-size":1073741824,"percent-progress":"0%"} +{"byte-progress":32752,"action":{"command":"copy","note":"to box-dartm3.com...","key":"MD5E-s1073741824--e06da14afc6face003121641e60593bb.mp4","file":"video.mp4"},"total-size":1073741824,"percent-progress":"0%"} + +... +{"byte-progress":225153536,"action":{"command":"copy","note":"to box-dartm3.com...","key":"MD5E-s1073741824--e06da14afc6face003121641e60593bb.mp4","file":"video.mp4"},"total-size":1073741824,"percent-progress":"20.97%"} +{"command":"copy","note":"checking box-dartm3.com...","success":false,"key":"MD5E-s1073741824--e06da14afc6face003121641e60593bb.mp4","file":"video.mp4"} + DAV failure: 408 "Request Timeout" + CallStack (from HasCallStack): + error, called at ./Remote/WebDAV.hs:324:47 in main:Remote.WebDAV +git-annex: copy: 1 failed + +$> git annex copy -J4 --to=box-dartm3.com --json --json-progress video.mp4 +{"byte-progress":200000000,"action":{"command":"copy","note":"to box-dartm3.com...","key":"MD5E-s1073741824--e06da14afc6face003121641e60593bb.mp4","file":"video.mp4"},"total-size":1073741824,"percent-progress":"18.63%"} +... + +"""]] + +apparently it is actually timing out on checking (I guess after chunk completion?), not even when copying... could there be multiple attempts and some grace time period for webdav server possibly to "finish receiving" a particular file? + + +[[!meta author="yoh"]] diff --git a/doc/bugs/could_webdav_be_more_resilient_to_timeouts__63__/comment_1_bb4130ae787e19757b849a6301129c34._comment b/doc/bugs/could_webdav_be_more_resilient_to_timeouts__63__/comment_1_bb4130ae787e19757b849a6301129c34._comment new file mode 100644 index 0000000000..8fd509a675 --- /dev/null +++ b/doc/bugs/could_webdav_be_more_resilient_to_timeouts__63__/comment_1_bb4130ae787e19757b849a6301129c34._comment @@ -0,0 +1,29 @@ +[[!comment format=mdwn + username="joey" + subject="""comment 1""" + date="2017-08-15T17:02:20Z" + content=""" +This last came up in 2014, at the time I disabled the http response timeout +entirely with setResponseTimeout Nothing. + +Looking at Network.Protocol.HTTP.DAV.setResponseTimeout: + + setResponseTimeout :: MonadIO m => Maybe Int -> DAVT m () + #if MIN_VERSION_http_client(0,5,0) + setResponseTimeout rt = baseRequest %= \x -> x { responseTimeout = maybe responseTimeoutDefault responseTimeoutMicro rt } + #else + setResponseTimeout rt = baseRequest %= \x -> x { responseTimeout = rt } + #endif + +Looks like with recent http-client versions in the case of Nothing, rather +than disabling the timeout, it now defaults to responseTimeoutDefault. +Which is 30 seconds. Since I'm passing it Nothing to try to disable the +timeout, that's kind of a problem! + +I feel this is a bug in the DAV library. I could try to work around it +with `Just maxBound`, that's many years worth of microseconds on 64 bit, but +on 32 bit, it's somewhere under 10 minutes, which is really not good +enough when the goal is to disable timeouts at this level entirely. + +Bug filed on DAV: +"""]] diff --git a/doc/bugs/creating_a_plain_directory_where_a_mountpoint_should_have_been.mdwn b/doc/bugs/creating_a_plain_directory_where_a_mountpoint_should_have_been.mdwn new file mode 100644 index 0000000000..05f3f5b7d1 --- /dev/null +++ b/doc/bugs/creating_a_plain_directory_where_a_mountpoint_should_have_been.mdwn @@ -0,0 +1,29 @@ +### Please describe the problem. + +This is a one-off thing, not a reproducible bug. It was just so weird that I wanted to make a note of it in case somebody else had the same problem and it was reproducible. + +I have a remote on the a USB drive which I mount once in a while on /Volumes/TOSHIBAEXT which has a remote on /Volumes/TOSHIBAEXT/annex. It's a remote which I created by hand using "git clone" a long time ago, not something the assistant set up for me. + +Anyway, today the assistant kept reporting that it could not sync to it. I didn't know why, because I checked in my Finder sidebar and it showed a mounted drive called TOSHIBAEXT. It's only when I checked with the command line that I noticed something was up! There was now an ordinary directory at /Volumes/TOSHIBAEXT/annex. This had apparently been created at some point when the drive was unmounted, so when it re-mounted, because there was a directory in the way, it re-mounted at "/Volumes/TOSHIBAEXT 1". But it was displayed in the finder sidebar as just "TOSHIBAEXT" anyway because the finder hides this workaround from the user for whatever reason. + +Presumably git-annex assistant at some point, for some reason, created an ordinary directory where it expected to find the annex. + +I wonder if this may have to do with the fact that this is a non-bare, created-by-hand repo, from before I was using the assistant, not a normal bare remote that the assistant would have created for me if I'd been using the webapp. + +### What steps will reproduce the problem? + +I've only seen it once, and report the bug not as an outstanding issue but only as a heads-up that this has ever happened. + +### What version of git-annex are you using? On what operating system? + +git-annex version: 4.20130501-gb61740e + +OS X lion (10.7) + +### Please provide any additional information below. + +I wasn't logging when this happened. + +Again, just a heads-up; I'll keep my eye open for this happening again and post more info if it does. + +[[!tag confirmed]] diff --git a/doc/bugs/creating_a_plain_directory_where_a_mountpoint_should_have_been/comment_1_41cfd5e48426a6ef52bef70a06a6f46a._comment b/doc/bugs/creating_a_plain_directory_where_a_mountpoint_should_have_been/comment_1_41cfd5e48426a6ef52bef70a06a6f46a._comment new file mode 100644 index 0000000000..a4d0a5895e --- /dev/null +++ b/doc/bugs/creating_a_plain_directory_where_a_mountpoint_should_have_been/comment_1_41cfd5e48426a6ef52bef70a06a6f46a._comment @@ -0,0 +1,11 @@ +[[!comment format=mdwn + username="http://edheil.wordpress.com/" + ip="99.54.57.201" + subject="comment 1" + date="2013-05-05T20:55:19Z" + content=""" +Huh. This happened a second time today. I'll try and catch it in the act, and make a proper bug report, when I get a chance. + +Still not sure this is a bug rather than just a peculiarity of how I have things set up. + +"""]] diff --git a/doc/bugs/creating_a_plain_directory_where_a_mountpoint_should_have_been/comment_2_bd584ccbe128427fca99e61d66d301c9._comment b/doc/bugs/creating_a_plain_directory_where_a_mountpoint_should_have_been/comment_2_bd584ccbe128427fca99e61d66d301c9._comment new file mode 100644 index 0000000000..1d9e09ef95 --- /dev/null +++ b/doc/bugs/creating_a_plain_directory_where_a_mountpoint_should_have_been/comment_2_bd584ccbe128427fca99e61d66d301c9._comment @@ -0,0 +1,18 @@ +[[!comment format=mdwn + username="http://joeyh.name/" + nickname="joey" + subject="comment 2" + date="2013-05-06T14:50:01Z" + content=""" +Hmm.. One way I can think that this could happen is if git-annex was running, and doing something in the repository on the drive that involved making a directory, and the drive was removed. There are several places in git-annex where it has code, like this: + +
    +createAnnexDirectory
+-- some action here that expects to have the git-annex directory
+
    + +Is the repo you have on the drive a direct mode repo by any chance? This is the only obvious way I can see that would cause it to create just the top level directory of the repository, and not a deeper directory tree like `.git/annex/tmp/` + +The assistant also has a MountWatcher that detects when repositories that were not accessible get mounted, but it does not do anything to stop the repository being used when its drive gets unmounted. Even if it did, it couldn't go anything about code that is already running using the previously mounted repository. However, as things stand, it would probably also be possible for the drive to be removed, and some time to pass before an action was run that tried to do something to the no longer present repository. +So making the mountwatcher disable repositories when drives are unmounted would at least make this window narrower. +"""]] diff --git a/doc/bugs/creating_a_plain_directory_where_a_mountpoint_should_have_been/comment_3_5bb0347215b321444643646f25a35759._comment b/doc/bugs/creating_a_plain_directory_where_a_mountpoint_should_have_been/comment_3_5bb0347215b321444643646f25a35759._comment new file mode 100644 index 0000000000..d163925427 --- /dev/null +++ b/doc/bugs/creating_a_plain_directory_where_a_mountpoint_should_have_been/comment_3_5bb0347215b321444643646f25a35759._comment @@ -0,0 +1,10 @@ +[[!comment format=mdwn + username="http://edheil.wordpress.com/" + ip="173.162.44.162" + subject="comment 3" + date="2013-05-06T16:52:07Z" + content=""" +It's possible, even likely, that there were indeed more directories and I didn't notice because they were hidden and I just looked at them with a vanilla \"ls\" command with no flags like \"-a\". + +So it's probably exactly what you're thinking --t here was something going on, a transfer queued or something, that I didn't realize was happening and I removed it and it went and created the directory as if it were there. +"""]] diff --git a/doc/bugs/creating_a_plain_directory_where_a_mountpoint_should_have_been/comment_4_73848a9c783ecf3d9fccdd41b20fbe36._comment b/doc/bugs/creating_a_plain_directory_where_a_mountpoint_should_have_been/comment_4_73848a9c783ecf3d9fccdd41b20fbe36._comment new file mode 100644 index 0000000000..7b588ecc73 --- /dev/null +++ b/doc/bugs/creating_a_plain_directory_where_a_mountpoint_should_have_been/comment_4_73848a9c783ecf3d9fccdd41b20fbe36._comment @@ -0,0 +1,56 @@ +[[!comment format=mdwn + username="https://www.google.com/accounts/o8/id?id=AItOawkfHTPsiAcHEEN7Xl7WxiZmYq-vX7azxFY" + nickname="Vincent" + subject="comment 4" + date="2013-07-24T14:19:01Z" + content=""" +I saw this too, today. The repo in my case was created via the webapp, using the 'usb key' option. +I was messing about with deleting repos and had turned off synchronisation for the repo on the key as I didn't have it inserted. + +I only just stumbled on this bug report after removing the repository from the key entirely, so it's difficult to define the reproduction steps. + +* create a 'usb key' repo via webapp, transfer type +* let it sync +* disable sync +* change a file in another repo, so there is something to sync +* unmount key, unplug key +* turn on sync again - should see 'failed ot sync with ...' in webapp dashboard +* reinsert key +* let it sync +* unplug key without properly unmounting +* change a file in another repo, so there is something to sync + +I just did this. The key was mounting at /Volumes/VERBATIM4. Just after unplugging without unmounting, that directory was gone. +When I made the change that could be synced, I got a /Volumes/VERBATIM4 directory. The directory tree structure is similar to but different +from the structure on the usb key, see below. + +Now when I try to plug it in again os/x will spot the potential conflict and mount the key at /Volumes/VERBATIM4\ 1. + +If I instead rm -rf /Volumes/VERBATIM4 and then replug the usb key, everything syncs as expected. + +Version: 4.20130723-ge023649 (os/x 10.8) + +Build flags: Assistant Webapp Pairing Testsuite S3 WebDAV FsEvents XMPP DNS + +Tree structure of the phantom directory: + + /Volumes/VERBATIM4/annex/ + └── annex + ├── journal.lck + ├── objects + │   └── df3 + │   └── 043 + │   └── SHA256E-s17363--4c5ef74b4fcb9ecd962c6ecac694f87277e580836335e57924654762668a5448 + │   └── SHA256E-s17363--4c5ef74b4fcb9ecd962c6ecac694f87277e580836335e57924654762668a5448 + ├── tmp + └── transfer + ├── download + │   └── adf30a67-777a-45a6-a658-e2266770f01b + └── failed + └── download + └── adf30a67-777a-45a6-a658-e2266770f01b + └── SHA256E-s17363--4c5ef74b4fcb9ecd962c6ecac694f87277e580836335e57924654762668a5448 + + 12 directories, 3 files + +"""]] diff --git a/doc/bugs/creating_a_plain_directory_where_a_mountpoint_should_have_been/comment_5_64bf56f2b0ff206c3caf5cadebfd0cda._comment b/doc/bugs/creating_a_plain_directory_where_a_mountpoint_should_have_been/comment_5_64bf56f2b0ff206c3caf5cadebfd0cda._comment new file mode 100644 index 0000000000..49ab80ed4b --- /dev/null +++ b/doc/bugs/creating_a_plain_directory_where_a_mountpoint_should_have_been/comment_5_64bf56f2b0ff206c3caf5cadebfd0cda._comment @@ -0,0 +1,20 @@ +[[!comment format=mdwn + username="https://www.google.com/accounts/o8/id?id=AItOawmTNrhkVQ26GBLaLD5-zNuEiR8syTj4mI8" + nickname="Juan" + subject="Same happening here" + date="2014-07-09T14:45:32Z" + content=""" +Any hint about how to stop this from happening again? This time I think I was careful and tried to stop git-annex, however I got ghosts. +Whats the correct way to unplug the drive? + +My scenario is the following, I have two repos on HDD synced with a USB drive. I want to sync them to the USB drive from time to time when I plug it. I want a safe way to unplug drive without getting these ghost directories. + +This time, I entered both directories in the HDD and made a git-annex assistant --stop, but after doing a ps aux | grep git-annex I still see many processes running so I'm not sure if I truly stopped it. +The webapp gives me the impression of working on a per-repo basis, so I miss having some sort of general feedback about git-annex. Something like: \"at the moment you are syncing repos A and B\" or \"idle\". +So I never have a clue about whats going on. +I'm trying to set up this since long time, and I'm a very capable unix guy, and I still have the feeling that I can't trust git-annex. +I really like the tool but I'm not sure how to use it properly. +Thanks in advance. + + +"""]] diff --git a/doc/bugs/creating_a_remote_server_repository.mdwn b/doc/bugs/creating_a_remote_server_repository.mdwn new file mode 100644 index 0000000000..07367d7731 --- /dev/null +++ b/doc/bugs/creating_a_remote_server_repository.mdwn @@ -0,0 +1,24 @@ +What steps will reproduce the problem? + +I was trying to add a remote server repository. Unfortunately, this didn't work. Enter Host name, user name, directory, port +(left most of them at their default), but there is no way, to specify a password.) Clicking check this server failed: + + Failed to ssh to the server. Transcript: Permission denied, please try again. Permission denied, please try again. Permission denied (publickey,password). + +(Problem was, I could never enter a password. Interestingly, on the konsole, I get a prompt for a password, but I can't enter anything there). + + +What is the expected output? What do you see instead? + +Successfully create a connection and use the remote server. + + +What version of git-annex are you using? On what operating system? +Version: 3.20130124 + + + +Please provide any additional information below. + +[[!tag /design/assistant done]] +[[!meta title="ssh password prompting issue with assistant"]] diff --git a/doc/bugs/creating_a_remote_server_repository/comment_1_de1a370347428245bcfca60eaca96779._comment b/doc/bugs/creating_a_remote_server_repository/comment_1_de1a370347428245bcfca60eaca96779._comment new file mode 100644 index 0000000000..aad008fd01 --- /dev/null +++ b/doc/bugs/creating_a_remote_server_repository/comment_1_de1a370347428245bcfca60eaca96779._comment @@ -0,0 +1,10 @@ +[[!comment format=mdwn + username="http://joeyh.name/" + ip="4.154.3.125" + subject="comment 1" + date="2013-02-05T19:22:04Z" + content=""" +You're not really intended to start the git-annex assistant from a console. If you do, ssh will go ahead and prompt for passwords using that controlling console. + +When the assistant is not started from a console, ssh should use the `ssh-askpass` to prompt for the password. Assuming your system has that installed. +"""]] diff --git a/doc/bugs/creating_a_remote_server_repository/comment_2_482ac9b0f881099910f9bd9f7cda184d._comment b/doc/bugs/creating_a_remote_server_repository/comment_2_482ac9b0f881099910f9bd9f7cda184d._comment new file mode 100644 index 0000000000..2370f7d478 --- /dev/null +++ b/doc/bugs/creating_a_remote_server_repository/comment_2_482ac9b0f881099910f9bd9f7cda184d._comment @@ -0,0 +1,8 @@ +[[!comment format=mdwn + username="http://edward.myopenid.com/" + nickname="edward" + subject="Same problem on Android 4.3" + date="2013-12-01T08:47:06Z" + content=""" +I have the same error on Android. Is git-annex on Android meant to come with ssh-askpass? +"""]] diff --git a/doc/bugs/creating_a_remote_server_repository/comment_3_b7c7f684d0eef14fcc00cb2ac0566703._comment b/doc/bugs/creating_a_remote_server_repository/comment_3_b7c7f684d0eef14fcc00cb2ac0566703._comment new file mode 100644 index 0000000000..ca56e7e3f5 --- /dev/null +++ b/doc/bugs/creating_a_remote_server_repository/comment_3_b7c7f684d0eef14fcc00cb2ac0566703._comment @@ -0,0 +1,10 @@ +[[!comment format=mdwn + username="http://joeyh.name/" + ip="209.250.56.64" + subject="comment 3" + date="2013-12-01T20:10:02Z" + content=""" +The bug your are commenting on is over a year old, and is closed, and does not involve Android. If you're having trouble with android, file a new bug report. + +On android, git-annex runs from a terminal app. The ssh password prompting is currently done inside that app. +"""]] diff --git a/doc/bugs/crippled_filesystem_direct_mode_sync_loop.mdwn b/doc/bugs/crippled_filesystem_direct_mode_sync_loop.mdwn new file mode 100644 index 0000000000..d7db94c8cb --- /dev/null +++ b/doc/bugs/crippled_filesystem_direct_mode_sync_loop.mdwn @@ -0,0 +1,74 @@ +### Please describe the problem. +Two direct mode repos both on crippled filesystem (NTFS), although no file is modified, each "git annex sync" command will generate a new commit on master branch, which makes "git log" grow fast. + +### What steps will reproduce the problem? +Run the script below on NTFS filesystem + +### What version of git-annex are you using? On what operating system? +I tried multiple combination: +git version from 2.7.4 to 2.11.0; +git-annex version from 5.20150812 to 6.20161211; +OS includes ubuntu xenial&yakkety and Windows 10; +As long as the script is run on NTFS filesystem it reproudces the problem. +However, on non crippled file system the script works without problem. + +### Please provide any additional information below. + +[[!format sh """ +# If you can, paste a complete transcript of the problem occurring here. +# If the problem is with the git-annex assistant, paste in .git/annex/daemon.log +mkdir a +cd a +git init +git annex init first +git annex direct +echo foo > 1.txt +git annex add . +git annex sync +cd .. +git clone a b +cd b +git annex init second +git annex direct +git annex sync +cd ../a +git remote add second ../b +git annex sync +echo bar > 2.txt +git annex add 2.txt +git annex sync +cd ../b +git annex sync +cd ../a +git annex sync +cd ../b +git annex sync +cd ../a +git annex sync +cd ../b +git annex sync +cd ../a +git annex sync +cd ../b +git annex sync +cd ../a +git annex sync +cd ../b +git annex sync +cd ../a +git annex sync +cd ../b +git annex sync +cd ../a +git annex sync +cd ../b +git annex sync +git log | grep refs/heads/synced/master | wc + + +# End of transcript or log. +"""]] + +### Have you had any luck using git-annex before? (Sometimes we get tired of reading bug reports all day and a lil' positive end note does wonders) + +I'm new to git-annex and immediately astonished by its unique strength. I fit the Archivist use case, and this could be the solution I wanted for so long. I'm planning to deploy it on 2 Windows boxes and several USB disks, all of them on NTFS. I learnt the idea that v6 repo is not yet good for Win/NTFS (double disk space), so I guess direct mode is the way to go? I have already got some test repos running and practicing, indeed this sync loop problem is the only remaining case I'm not confident with. Is it a bug or some safety measure feature? Am I good to go? Thanks and oh, Merry X'mas! diff --git a/doc/bugs/crippled_filesystem_direct_mode_sync_loop/comment_1_54fbe98e49d949cb6bed6122fcaec048._comment b/doc/bugs/crippled_filesystem_direct_mode_sync_loop/comment_1_54fbe98e49d949cb6bed6122fcaec048._comment new file mode 100644 index 0000000000..baa1bb91ed --- /dev/null +++ b/doc/bugs/crippled_filesystem_direct_mode_sync_loop/comment_1_54fbe98e49d949cb6bed6122fcaec048._comment @@ -0,0 +1,8 @@ +[[!comment format=mdwn + username="Asureus" + avatar="http://cdn.libravatar.org/avatar/f544d481d1e5adcd3b68c27d18680535" + subject="one more thing" + date="2016-12-25T14:46:46Z" + content=""" +Forgot to mention it, \"git annex test\" never passed on my Windows setup, always got 10-20 failed tests. Is that OK? +"""]] diff --git a/doc/bugs/crypto-api_is_a_global_dependency_because_of_Utility.AuthToken.mdwn b/doc/bugs/crypto-api_is_a_global_dependency_because_of_Utility.AuthToken.mdwn new file mode 100644 index 0000000000..fcde456fd7 --- /dev/null +++ b/doc/bugs/crypto-api_is_a_global_dependency_because_of_Utility.AuthToken.mdwn @@ -0,0 +1,25 @@ +### Please describe the problem. + +Builds without Webapp and TestSuite fail at `Utility.AuthToken`. + + +### What steps will reproduce the problem? + +Build git-annex with Webapp and TestSuite disabled. + + +### What version of git-annex are you using? On what operating system? + +Version 6.20170101 on arch linux. + + +### Please provide any additional information below. + +We need to make `crypto-api` into a global dependency. Here is a [patch](https://gist.github.com/aroig/100c2977b6df8a2109823b715647d5fb). + + +### Have you had any luck using git-annex before? (Sometimes we get tired of reading bug reports all day and a lil' positive end note does wonders) + +Oh yes! I use it everyday to sync collections of binary files across computers and VM's! + +> [[fixed|done]], thanks! --[[Joey]] diff --git a/doc/bugs/dashed_ssh_hostname_security_hole.mdwn b/doc/bugs/dashed_ssh_hostname_security_hole.mdwn new file mode 100644 index 0000000000..056d049e66 --- /dev/null +++ b/doc/bugs/dashed_ssh_hostname_security_hole.mdwn @@ -0,0 +1,27 @@ +git-annex was vulnerable to the same class of security hole as +git's CVE-2017-1000117. In several cases, git-annex parses a repository +url, and uses it to generate a ssh command, with the hostname to ssh to +coming from the url. If the hostname it parses is something like +"-oProxyCommand=evil", this could result in arbitrary local code execution +via ssh. + +I have not bothered to try to exploit the problem, and some details of URL +parsing may prevent the exploit working in some cases. + +Exploiting this would involve the attacker tricking the victim into adding +a remote something like "ssh://-oProxyCommand=evil/blah". + +One possible avenue for an attacker that avoids exposing the URL to the +user is to use initremote with a ssh remote, so embedding the URL in the +git-annex branch. Then the victim would enable it with enableremote. + +This was fixed in version 6.20170818. Now there's a SshHost type that +is not allowed to start with a dash, and every invocation of ssh is +in a function that takes a SshHost. + +CVE-2017-12976 has been assigned for this issue. + +[[done]] + +--[[Joey]] + diff --git a/doc/bugs/data_loss_on_Windows__58___git_annex_sync_--no-content_drops_last_copy_unexpectedly.mdwn b/doc/bugs/data_loss_on_Windows__58___git_annex_sync_--no-content_drops_last_copy_unexpectedly.mdwn new file mode 100644 index 0000000000..0914278b8a --- /dev/null +++ b/doc/bugs/data_loss_on_Windows__58___git_annex_sync_--no-content_drops_last_copy_unexpectedly.mdwn @@ -0,0 +1,181 @@ +### Please describe the problem. +A seemingly harmless script causes data loss by dropping last copy of file content. + +In my test this script only drops file content on Windows. On Linux it's working good, even on a crippled filesystem. + +### What steps will reproduce the problem? +run the following script test.sh: +[[!format sh """ +mkdir a +cd a +git init +git annex init first +mkdir folder +echo foo > folder/1.txt +git annex add . +git annex sync +cd .. +git clone a b +cd b +git annex init second +git annex sync +cd ../a +git remote add second ../b +git annex sync +git annex move --to second +git annex sync +mv folder folder1 +git annex add +git annex sync +cd ../b +git annex sync +"""]] + +### What version of git-annex are you using? On what operating system? +git-annex version: 6.20161231-gc8eeb17 + +Windows 10.0.14393 and also Windows 8 + +### Please provide any additional information below. + +[[!format sh """ +# a complete transcript of the problem occurring. +$ ./test.sh +Initialized empty Git repository in A:/a/.git/ +init first + Detected a filesystem without fifo support. + + Disabling ssh connection caching. + + Detected a crippled filesystem. + + Enabling direct mode. +ok +(recording state in git...) +add folder/1.txt ok +(recording state in git...) +commit ok +Cloning into 'b'... +done. +init second + Detected a filesystem without fifo support. + + Disabling ssh connection caching. + + Detected a crippled filesystem. +(merging origin/git-annex into git-annex...) +(recording state in git...) + + Enabling direct mode. +ok +(recording state in git...) +commit ok +pull origin +ok +push origin +Counting objects: 6, done. +Delta compression using up to 8 threads. +Compressing objects: 100% (5/5), done. +Writing objects: 100% (6/6), 664 bytes | 0 bytes/s, done. +Total 6 (delta 0), reused 0 (delta 0) +To A:/a + * [new branch] git-annex -> synced/git-annex +ok +commit ok +pull second +From ../b + * [new branch] annex/direct/master -> second/annex/direct/master + * [new branch] git-annex -> second/git-annex + * [new branch] master -> second/master + * [new branch] synced/master -> second/synced/master +ok +move folder/1.txt (to second...) +1.txt + 4 100% 0.00kB/s 0:00:00 (xfr#1, to-chk=0/1) +(checksum...) ok +(recording state in git...) +commit ok +pull second +remote: Counting objects: 5, done. +remote: Compressing objects: 100% (4/4), done. +remote: Total 5 (delta 1), reused 0 (delta 0) +Unpacking objects: 100% (5/5), done. +From ../b + fd774cb..1aba4de git-annex -> second/git-annex +ok +(merging second/git-annex into git-annex...) +(recording state in git...) +push second +Counting objects: 10, done. +Delta compression using up to 8 threads. +Compressing objects: 100% (8/8), done. +Writing objects: 100% (10/10), 827 bytes | 0 bytes/s, done. +Total 10 (delta 3), reused 0 (delta 0) +To ../b + * [new branch] git-annex -> synced/git-annex +ok +add folder1/1.txt ok +(recording state in git...) +commit (recording state in git...) +ok +pull second +ok +push second +Counting objects: 7, done. +Delta compression using up to 8 threads. +Compressing objects: 100% (5/5), done. +Writing objects: 100% (7/7), 687 bytes | 0 bytes/s, done. +Total 7 (delta 0), reused 0 (delta 0) +To ../b + 7ba3e8a..ee8025b git-annex -> synced/git-annex + 0758cf9..6e91185 annex/direct/master -> synced/master +ok +commit ok +merge synced/master +Updating 0758cf9..6e91185 +Fast-forward + {folder => folder1}/1.txt | 0 + 1 file changed, 0 insertions(+), 0 deletions(-) + rename {folder => folder1}/1.txt (100%) +error: duplicate parent 6e91185c7c64569b275a09be1a104a1d8955e1fb ignored +ok +pull origin +From A:/a + 0758cf9..6e91185 annex/direct/master -> origin/annex/direct/master + fd774cb..ee8025b git-annex -> origin/git-annex + 0758cf9..6e91185 master -> origin/master + 0758cf9..6e91185 synced/master -> origin/synced/master +ok +push origin +Counting objects: 1, done. +Writing objects: 100% (1/1), 185 bytes | 0 bytes/s, done. +Total 1 (delta 0), reused 0 (delta 0) +To A:/a + fd774cb..ee8025b git-annex -> synced/git-annex + 6e91185..a886805 annex/direct/master -> synced/master +ok + +$ # script done here +$ cd b +$ git annex whereis +whereis folder1/1.txt (1 copy) + 2a9ef292-1729-4533-ac50-f68d2d0badb6 -- second [here] +ok + +$ cat folder1/1.txt +../.git/annex/objects/W5/55/SHA256E-s4--b5bb9d8014a0f9b1d61e21e796d78dccdf1352f23cd32812f4850b878ae4944c.txt/SHA256E-s4--b5bb9d8014a0f9b1d61e21e796d78dccdf1352f23cd32812f4850b878ae4944c.txt + +$ git annex get +get folder1/1.txt (not available) + No other repository is known to contain the file. +failed +git-annex: get: 1 failed + + +# End of transcript or log. +"""]] + + +### Conclusion + +I'm so attracted to git-annex's idea, but so sad it's still not robust enough to use on Windows platform - v5 direct mode repo is as far as I can get, yet it still throws away my data like this... diff --git a/doc/bugs/default_cabal_install_on_OSX_lacks_S3.mdwn b/doc/bugs/default_cabal_install_on_OSX_lacks_S3.mdwn new file mode 100644 index 0000000000..58ae5d9700 --- /dev/null +++ b/doc/bugs/default_cabal_install_on_OSX_lacks_S3.mdwn @@ -0,0 +1,15 @@ +A default cabal install on OS X in a sandbox of git-annex 6.20160511 will result in no S3 support, as reported to Homebrew in the following two issues: +https://github.com/Homebrew/homebrew-core/issues/1268 +https://github.com/Homebrew/legacy-homebrew/issues/47737 + +The underlying cause is that aws-0.13.0 lacks commit https://github.com/aristidb/aws/commit/402bfe5aa9ef4bec84186880faafcbfdae1ad91d, which allows data-default 0.6. + +I attempted to mitigate the issue using --flags="s3", but that does not seem to help (nor does it force the build to fail): still no s3 support. I'd expect that either to constrain data-default to 0.5.3 and produce a build with s3 support, or fail the build, but for some reason it doesn't. Is this not working because we're in a sandbox or some other reason? + +Currently, I'm planning to just patch aws with https://github.com/aristidb/aws/commit/402bfe5aa9ef4bec84186880faafcbfdae1ad91d rather than resorting to a fixed configuration (e.g., lts-5.5 or whatever), as you can see here: +https://github.com/Homebrew/homebrew-core/pull/1307 + +It would be great if git-annex could work around the issue itself, though. + +Meanwhile, I have also pinged aws to request a 0.13.1 release, which would solve the problem "the right way": +https://github.com/aristidb/aws/issues/202 diff --git a/doc/bugs/direct_command_leaves_repository_inconsistent_if_interrupted.mdwn b/doc/bugs/direct_command_leaves_repository_inconsistent_if_interrupted.mdwn new file mode 100644 index 0000000000..8a88811730 --- /dev/null +++ b/doc/bugs/direct_command_leaves_repository_inconsistent_if_interrupted.mdwn @@ -0,0 +1,47 @@ +### Please describe the problem. + +When `git annex direct` is interrupted (either through a power outage or deliberate `control-c`) it may leave the repository in an inconsistent state. + +A typical situation is `git-annex` believing that the repo is in `indirect` mode while the files are not symlinks anymore. + +I believe I have described this problem here before, but the bug report was deleted as part of the may 29th purge (222f78e9eadd3d2cc40ec94ab22241823a7d50d9, [[bugs/git_annex_indirect_can_fail_catastrophically]]). + +### What steps will reproduce the problem? + +`git annex direct` on a large repository, `control-c` before it finishes. + +Observe how a lot of files are now considered to be in the famous [[typechange status|forum/git-status_typechange_in_direct_mode/]] in git. + +### What version of git-annex are you using? On what operating system? + +5.20140717 on Debian Jessie, ext4 filesystem. + +### Please provide any additional information below. + +I wish i could resume the `git annex direct` command, but this will do a `git commit -a` and therefore commit all those files to git directly. It still seems to me that `git annex` should never run `git commit -a` for exactly that kind of situations. + +I think that's it for now. -- [[anarcat]] + +Update: i was able to get rid of the `typechange` situation by running `git annex lock` on the repository, but then all files are found to be missing by `git annex fsck`: + +[[!format txt """ +fsck films/God Hates Cartoons/VIDEO_TS/VTS_15_0.BUP (fixing location log) + ** Based on the location log, films/God Hates Cartoons/VIDEO_TS/VTS_15_0.BUP + ** was expected to be present, but its content is missing. + + Only 1 of 2 trustworthy copies exist of films/God Hates Cartoons/VIDEO_TS/VTS_15_0.BUP + Back it up with git-annex copy. +"""]] + +Oddly enough, the repo still uses hundreds of gigs, because all the files ended up in `.git/annex/misctmp`. Not sure I remember what happened there. + +Similar issues and discussions: + +* [[bugs/direct_mode_merge_interrupt/]] +* [[forum/Cleaning_up_after_aborted_sync_in_direct_mode/]] +* [[bugs/failure_to_return_to_indirect_mode_on_usb/]] +* [[forum/git-status_typechange_in_direct_mode/]] + +[[!meta title="git annex lock --force deletes only copy of content after interrupted switch to direct mode"] + +[[!meta tag=deprecateddirectmode]] diff --git a/doc/bugs/direct_command_leaves_repository_inconsistent_if_interrupted/comment_1_770e6ed8556fa6b259f517d5c398271f._comment b/doc/bugs/direct_command_leaves_repository_inconsistent_if_interrupted/comment_1_770e6ed8556fa6b259f517d5c398271f._comment new file mode 100644 index 0000000000..4d7246eda7 --- /dev/null +++ b/doc/bugs/direct_command_leaves_repository_inconsistent_if_interrupted/comment_1_770e6ed8556fa6b259f517d5c398271f._comment @@ -0,0 +1,16 @@ +[[!comment format=mdwn + username="http://joeyh.name/" + ip="209.250.56.7" + subject="comment 1" + date="2014-08-12T17:26:46Z" + content=""" +The way it's supposed to work is that `git annex direct` does not set the repository into direct mode until it's entirely done moving files around. So, if it is interrupted at any point, you are left with an indirect mode repository, with some unlocked files. Which can be put back to indirect by `git annex add`, or the conversion restarted with `git annex direct`. + +That seems to work in my tests; I can interrupt `git annex direct` and resume with `git annex direct` with a good result; `git annex add` reverts back to indirect mode. Even `git commit -a` reverts back to indirect mode, thanks to the pre-commit hook. I have tested that all these recovery methods work as intended. + +That leaves `git annex lock --force` (it has to be forced) after an interrupted switch to direct mode. I have reproduced that in this situation, that will delete your file's contents (I cannot reproduce them ending up in misctmp, but [[!commit d8be828734704c78f91029263b59eed75174e665]] may have had something to do with that). In a sense, `git annex lock --force` is doing what you told it to -- git-annex lock throws unlocked file contents away, under the assumption that they might contain modified changes. Since normally, `git annex unlock` makes a copy of the file, there is not normally data loss, unless the unlocked files got modified. But that's why it requires the --force; it can result in data loss. + +I a having a hard time thinking of a modification to `git annex lock` that would make sense. The best I can come up with is, if the file's content is not present in the annex, it could switch to what `git annex add` does, and re-add the file content to the annex if it's unchanged. While, I guess, throwing away the content if it is changed. That seems a bit complicated. + +(BTW, if you do still have the files in misctmp, you can `git annex import` their content back into the repository.) +"""]] diff --git a/doc/bugs/direct_command_leaves_repository_inconsistent_if_interrupted/comment_2_2f3fb399f976d96aa66310f11365207c._comment b/doc/bugs/direct_command_leaves_repository_inconsistent_if_interrupted/comment_2_2f3fb399f976d96aa66310f11365207c._comment new file mode 100644 index 0000000000..b55e3c2a39 --- /dev/null +++ b/doc/bugs/direct_command_leaves_repository_inconsistent_if_interrupted/comment_2_2f3fb399f976d96aa66310f11365207c._comment @@ -0,0 +1,10 @@ +[[!comment format=mdwn + username="http://joeyh.name/" + ip="209.250.56.7" + subject="comment 2" + date="2014-08-15T17:39:14Z" + content=""" +I still cannot see a way that more than one file's content could end up in misctemp, since `git annex direct` moves just one file there at a time, so max of one should be there if interrupted. However, there was really no reason to be moving files through misctemp at all, so `git annex direct` now moves them into place completely atomically. + +Bug report retitled appropriatly for the `git annex lock --force` suprise. +"""]] diff --git a/doc/bugs/direct_command_leaves_repository_inconsistent_if_interrupted/comment_3_2acff7b667e8618251075031cbef6b9a._comment b/doc/bugs/direct_command_leaves_repository_inconsistent_if_interrupted/comment_3_2acff7b667e8618251075031cbef6b9a._comment new file mode 100644 index 0000000000..5de6726aa4 --- /dev/null +++ b/doc/bugs/direct_command_leaves_repository_inconsistent_if_interrupted/comment_3_2acff7b667e8618251075031cbef6b9a._comment @@ -0,0 +1,8 @@ +[[!comment format=mdwn + username="http://joeyh.name/" + ip="131.252.200.111" + subject="comment 3" + date="2014-08-31T22:29:44Z" + content=""" +Occurs to me that your repo may not have a pre-commit hook; if not then `git commit -a` would not behave as I described.. +"""]] diff --git a/doc/bugs/direct_command_leaves_repository_inconsistent_if_interrupted/comment_4_c3a4a1ce24fcbe1087041850f490a58a._comment b/doc/bugs/direct_command_leaves_repository_inconsistent_if_interrupted/comment_4_c3a4a1ce24fcbe1087041850f490a58a._comment new file mode 100644 index 0000000000..82b77f78bf --- /dev/null +++ b/doc/bugs/direct_command_leaves_repository_inconsistent_if_interrupted/comment_4_c3a4a1ce24fcbe1087041850f490a58a._comment @@ -0,0 +1,8 @@ +[[!comment format=mdwn + username="https://andrew.aylett.co.uk/" + nickname="andrew" + subject="comment 4" + date="2014-09-07T18:41:28Z" + content=""" +I, too, have seen this issue -- took me a while to recover from it. I do (now, at least) have a pre-commit hook that calls git annex pre-commit; I didn't set that up myself. +"""]] diff --git a/doc/bugs/direct_mode_fails__44___left_in_an_inconsistent_state.mdwn b/doc/bugs/direct_mode_fails__44___left_in_an_inconsistent_state.mdwn new file mode 100644 index 0000000000..42746b9665 --- /dev/null +++ b/doc/bugs/direct_mode_fails__44___left_in_an_inconsistent_state.mdwn @@ -0,0 +1,61 @@ +### Please describe the problem. + +Running `git annex direct` in a repository may results with the following error message: + + git-annex: /home/mildred/Music/.git/annex/objects/2K/49/SHA256-s0--e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855/SHA256-s0--e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855.map898.tmp: rename: permission denied (Permission denied) +failed + git-annex: direct: 1 failed + + +The major problem is that git-annex doesn't roll back the changes it did for the files it could successfully put in direct mode. Running git status show many files with typechange. The solution was to run `git add` on those files (although the hashing backend changed, so a commit must be created) + +### What steps will reproduce the problem? + +Don't know yet why the rename failed, but the direct mode should be rolled back if there is a problem. Restarting `git-annex direct` didn't result in an error. + +### What version of git-annex are you using? On what operating system? + +git-annex 5.20140405-g8729abc +arch-linux Linux moiraine 3.15.3-1-ARCH #1 SMP PREEMPT Tue Jul 1 07:32:45 CEST 2014 x86_64 GNU/Linux + +### Please provide any additional information below. + +[[!format sh """ + +$ git annex direct +commit +(Recording state in git...) +On branch master +nothing to commit, working directory clean +ok +direct .gitrefs/heads/annex/direct/master ok +direct .gitrefs/heads/git-annex ok +direct .gitrefs/heads/master ok +direct .gitrefs/heads/synced/master ok +direct .gitrefs/remotes/ashley/git-annex ok +direct .gitrefs/remotes/ashley/master ok +direct .gitrefs/remotes/ashley/synced/git-annex ok +direct .gitrefs/remotes/ashley/synced/master ok +direct .gitrefs/remotes/kylae/git-annex ok +direct .gitrefs/remotes/kylae/master ok +direct .gitrefs/remotes/kylae/synced/git-annex ok +direct ... ok +direct ... ok +direct ... ok + /home/mildred/Music/.git/annex/objects/2K/49/SHA256-s0--e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855/SHA256-s0--e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855.map897.tmp: rename: permission denied (Permission denied) + + leaving this file as-is; correct this problem and run git annex fsck on it +direct ... ok +direct ... ok +direct ... ok +direct ... ok +direct ... ok + +git-annex: /home/mildred/Music/.git/annex/objects/2K/49/SHA256-s0--e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855/SHA256-s0--e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855.map898.tmp: rename: permission denied (Permission denied) +failed +git-annex: direct: 1 failed + +"""]] + +[[!tag moreinfo]] +[[!meta tag=deprecateddirectmode]] diff --git a/doc/bugs/direct_mode_fails__44___left_in_an_inconsistent_state/comment_1_be1302a006a66e501fe543f3af191fea._comment b/doc/bugs/direct_mode_fails__44___left_in_an_inconsistent_state/comment_1_be1302a006a66e501fe543f3af191fea._comment new file mode 100644 index 0000000000..c0455b9924 --- /dev/null +++ b/doc/bugs/direct_mode_fails__44___left_in_an_inconsistent_state/comment_1_be1302a006a66e501fe543f3af191fea._comment @@ -0,0 +1,22 @@ +[[!comment format=mdwn + username="http://joeyh.name/" + ip="209.250.56.2" + subject="comment 1" + date="2014-07-10T18:11:22Z" + content=""" +The most likely problem would be if your repository contained annexed objects owned by different user than the one running `git annex direct`. + +However, I cannot reproduce this problem: + +
    +direct foo 
+  /home/joey/tmp/r/.git/annex/objects/pV/7j/SHA256E-s30--2754b7f82f6994005b97256273756f14d4abc17165c8819c06c07340d03351fa: setFileMode: permission denied (Operation not permitted)
+
+  leaving this file as-is; correct this problem and run git annex fsck on it
+direct  ok
+
    + +Since version 4.20130921, any exception when moving a file to direct mode should be caught like that. + +I will need more information to reproduce your bug. Or are you sure you wrote down the right version of git-annex? +"""]] diff --git a/doc/bugs/direct_mode_merge_interrupt.mdwn b/doc/bugs/direct_mode_merge_interrupt.mdwn new file mode 100644 index 0000000000..f6bb795b6d --- /dev/null +++ b/doc/bugs/direct_mode_merge_interrupt.mdwn @@ -0,0 +1,56 @@ +Seems to me there is a bug in how merges are done in direct mode. This is +done in two steps: + +1. Merge the remote branch into the local branch, with work tree directed + to a temp dir. +2. Use the temp dir and the newly merged branch to update the work tree. + +If this is interrupted between 1 and 2, by eg the user ctrl-Cing or power +being lost, the result is a repository that thinks the current branch has +been merged, but does not have an updated work tree. The next sync in that +repository will see the files as deleted (or as being an old version), and +commit the current work tree state to the branch. + +Result is files appear to be lost, although `git revert` in an indirect +mode repo can get them back. + +To fix this, direct mode merge would need to avoid updating the current +branch when merging the remote branch into it (how?). It should first +update the whole work tree, and only after it's updated should it update +the index and the current branch to reflect the merge. + +This way, if the merge is interrupted, the work tree may have uncommitted +changed -- but it's fine if they get accidentally committed, since when +the merge is re-done, those changes will by the same ones made by the +merge. (I assume this is how `git merge` normally works.) --[[Joey]] + +> Implemented that. And then realized that even updating the index +> as part of a merge results in the work tree being out of sync with the +> index. Which will cause the next sync to again delete any files that +> are in the index but not the work tree. Urgh. +> +> Seems that a direct mode +> merge also needs to use a different index file to stage its changes? +> (Ugh) +> > done --[[Joey]] + +> > > I had to revert the fix on FAT/Windows due to +> > > a git bug: +> > > Once that bug's fixed, I can revisit this. --[[Joey]] + +[[!meta title="direct mode merge interrupt (fixed for all except FAT, Windows)"]] + +## other options + +> Or could perhaps use `git-merge-tree` +> and avoid staging the merge in the index until the work-tree is updated. +> +> Alternatively, could use another strategy.. Add a lock file which is held while +> the merge is in progress and contains the pre-merge sha. +> If the lock file is present but not held, state is inconsistent. +> `git-annex sync` and the SanityChecker should +> then run mergeDirectCleanup to recover, before any commits can be made +> from the inconsistent state. This approach seems to get complicated +> quickly.. --[[Joey]] + +[[!meta tag=deprecateddirectmode]] diff --git a/doc/bugs/direct_mode_should_refuse_to_merge_with_illegal_filenames.mdwn b/doc/bugs/direct_mode_should_refuse_to_merge_with_illegal_filenames.mdwn new file mode 100644 index 0000000000..f90710277b --- /dev/null +++ b/doc/bugs/direct_mode_should_refuse_to_merge_with_illegal_filenames.mdwn @@ -0,0 +1,38 @@ +Some filesystems have stupid rules about characters not allowed in filenames. For example, FAT doesn't allow '?' '*' ':' etc. + +The direct mode merge code lets `git merge` update a temp directory with the new files from the merge, before doing its work tree update and committing. This can fail: + +
    +error: unable to create file non-rus/Dance/Dream_Dance_Vol15/CD1/09-??.mp3 (Invalid argument)
+
    + +This leaves the work tree without the file, and the index knows about the file. Result is that the next time a commit is done, this file appears to have been deleted, and that is committed and propigates out. Which can be surprising. + +---- + +It would probably be better if, when the working tree cannot be updated, it left the repository in some state that would not make the next commit remove anything. + +Ie, direct mode should replicate this behavior: + +
    +root@darkstar:/home/joey/mnt>git init
+root@darkstar:/home/joey/mnt>git merge FETCH_HEAD 
+error: unable to create file foo? (Invalid argument)
+fatal: read-tree failed
+root@darkstar:/home/joey/mnt>git status
+On branch master
+
+Initial commit
+
+nothing to commit (create/copy files and use "git add" to track)
+
    + +Problem is, the call to `git merge` can also fail due to a conflict. In that case, git-annex wants to continue with automatic conflict resolution. +So, how to detect when `git merge` has skipped creating illegal filenames? + +---- + +Alternatively, git-annex could learn/probe the full set of characters not allowed in filenames, and examine merges before performing them, and refuse to do anything if the merge added an illegal filename.a + +[[!tag confirmed]] +[[!meta tag=deprecateddirectmode]] diff --git a/doc/bugs/does_not_build_on_OpenBSD_5.9.mdwn b/doc/bugs/does_not_build_on_OpenBSD_5.9.mdwn new file mode 100644 index 0000000000..3185c9fc88 --- /dev/null +++ b/doc/bugs/does_not_build_on_OpenBSD_5.9.mdwn @@ -0,0 +1,216 @@ +### Please describe the problem. + +git-annex will not build on OpenBSD 5.9-stable for amd64 + +### What steps will reproduce the problem? + +$ stack install git-annex + +### What version of git-annex are you using? On what operating system? + +git-annex-6.20160511, the version from stack resolver lts-6.7 + +### Please provide any additional information below. + +[[!format sh """ +$ stack install git-annex +Run from outside a project, using implicit global project config +git-annex-6.20160511: configure +git-annex-6.20160511: build + +-- While building package git-annex-6.20160511 using: + /tmp/stack29275/git-annex-6.20160511/.stack-work/dist/x86_64-openbsd/Cabal-1.22.5.0/setup/setup --builddir=.stack-work/dist/x86_64-openbsd/Cabal-1.22.5.0 build --ghc-options " -ddump-hi -ddump-to-file" + Process exited with code: ExitFailure 1 + Logs have been written to: /home/scott/.stack/global-project/.stack-work/logs/git-annex-6.20160511.log + + [ 1 of 32] Compiling Utility.FileSize ( Utility/FileSize.hs, /tmp/stack29275/git-annex-6.20160511/.stack-work/dist/x86_64-openbsd/Cabal-1.22.5.0/setup/Utility/FileSize.o ) + [ 2 of 32] Compiling Utility.Process.Shim ( Utility/Process/Shim.hs, /tmp/stack29275/git-annex-6.20160511/.stack-work/dist/x86_64-openbsd/Cabal-1.22.5.0/setup/Utility/Process/Shim.o ) + [ 3 of 32] Compiling Utility.Applicative ( Utility/Applicative.hs, /tmp/stack29275/git-annex-6.20160511/.stack-work/dist/x86_64-openbsd/Cabal-1.22.5.0/setup/Utility/Applicative.o ) + [ 4 of 32] Compiling Utility.PosixFiles ( Utility/PosixFiles.hs, /tmp/stack29275/git-annex-6.20160511/.stack-work/dist/x86_64-openbsd/Cabal-1.22.5.0/setup/Utility/PosixFiles.o ) + [ 5 of 32] Compiling Utility.Env ( Utility/Env.hs, /tmp/stack29275/git-annex-6.20160511/.stack-work/dist/x86_64-openbsd/Cabal-1.22.5.0/setup/Utility/Env.o ) + [ 6 of 32] Compiling Utility.UserInfo ( Utility/UserInfo.hs, /tmp/stack29275/git-annex-6.20160511/.stack-work/dist/x86_64-openbsd/Cabal-1.22.5.0/setup/Utility/UserInfo.o ) + [ 7 of 32] Compiling Utility.OSX ( Utility/OSX.hs, /tmp/stack29275/git-annex-6.20160511/.stack-work/dist/x86_64-openbsd/Cabal-1.22.5.0/setup/Utility/OSX.o ) + [ 8 of 32] Compiling Utility.PartialPrelude ( Utility/PartialPrelude.hs, /tmp/stack29275/git-annex-6.20160511/.stack-work/dist/x86_64-openbsd/Cabal-1.22.5.0/setup/Utility/PartialPrelude.o ) + [ 9 of 32] Compiling Utility.Data ( Utility/Data.hs, /tmp/stack29275/git-annex-6.20160511/.stack-work/dist/x86_64-openbsd/Cabal-1.22.5.0/setup/Utility/Data.o ) + [10 of 32] Compiling Utility.Exception ( Utility/Exception.hs, /tmp/stack29275/git-annex-6.20160511/.stack-work/dist/x86_64-openbsd/Cabal-1.22.5.0/setup/Utility/Exception.o ) + [11 of 32] Compiling Utility.FileSystemEncoding ( Utility/FileSystemEncoding.hs, /tmp/stack29275/git-annex-6.20160511/.stack-work/dist/x86_64-openbsd/Cabal-1.22.5.0/setup/Utility/FileSystemEncoding.o ) + [12 of 32] Compiling Utility.Tmp ( Utility/Tmp.hs, /tmp/stack29275/git-annex-6.20160511/.stack-work/dist/x86_64-openbsd/Cabal-1.22.5.0/setup/Utility/Tmp.o ) + [13 of 32] Compiling Utility.Monad ( Utility/Monad.hs, /tmp/stack29275/git-annex-6.20160511/.stack-work/dist/x86_64-openbsd/Cabal-1.22.5.0/setup/Utility/Monad.o ) + [14 of 32] Compiling Utility.Misc ( Utility/Misc.hs, /tmp/stack29275/git-annex-6.20160511/.stack-work/dist/x86_64-openbsd/Cabal-1.22.5.0/setup/Utility/Misc.o ) + [15 of 32] Compiling Utility.Process ( Utility/Process.hs, /tmp/stack29275/git-annex-6.20160511/.stack-work/dist/x86_64-openbsd/Cabal-1.22.5.0/setup/Utility/Process.o ) + [16 of 32] Compiling Utility.SafeCommand ( Utility/SafeCommand.hs, /tmp/stack29275/git-annex-6.20160511/.stack-work/dist/x86_64-openbsd/Cabal-1.22.5.0/setup/Utility/SafeCommand.o ) + [17 of 32] Compiling Utility.Directory ( Utility/Directory.hs, /tmp/stack29275/git-annex-6.20160511/.stack-work/dist/x86_64-openbsd/Cabal-1.22.5.0/setup/Utility/Directory.o ) + [18 of 32] Compiling Build.Version ( Build/Version.hs, /tmp/stack29275/git-annex-6.20160511/.stack-work/dist/x86_64-openbsd/Cabal-1.22.5.0/setup/Build/Version.o ) + [19 of 32] Compiling Utility.Network ( Utility/Network.hs, /tmp/stack29275/git-annex-6.20160511/.stack-work/dist/x86_64-openbsd/Cabal-1.22.5.0/setup/Utility/Network.o ) + [20 of 32] Compiling Utility.ExternalSHA ( Utility/ExternalSHA.hs, /tmp/stack29275/git-annex-6.20160511/.stack-work/dist/x86_64-openbsd/Cabal-1.22.5.0/setup/Utility/ExternalSHA.o ) + [21 of 32] Compiling Utility.Path ( Utility/Path.hs, /tmp/stack29275/git-annex-6.20160511/.stack-work/dist/x86_64-openbsd/Cabal-1.22.5.0/setup/Utility/Path.o ) + [22 of 32] Compiling Build.TestConfig ( Build/TestConfig.hs, /tmp/stack29275/git-annex-6.20160511/.stack-work/dist/x86_64-openbsd/Cabal-1.22.5.0/setup/Build/TestConfig.o ) + [23 of 32] Compiling Common ( Common.hs, /tmp/stack29275/git-annex-6.20160511/.stack-work/dist/x86_64-openbsd/Cabal-1.22.5.0/setup/Common.o ) + [24 of 32] Compiling Utility.DottedVersion ( Utility/DottedVersion.hs, /tmp/stack29275/git-annex-6.20160511/.stack-work/dist/x86_64-openbsd/Cabal-1.22.5.0/setup/Utility/DottedVersion.o ) + [25 of 32] Compiling Git.Version ( Git/Version.hs, /tmp/stack29275/git-annex-6.20160511/.stack-work/dist/x86_64-openbsd/Cabal-1.22.5.0/setup/Git/Version.o ) + [26 of 32] Compiling Utility.FreeDesktop ( Utility/FreeDesktop.hs, /tmp/stack29275/git-annex-6.20160511/.stack-work/dist/x86_64-openbsd/Cabal-1.22.5.0/setup/Utility/FreeDesktop.o ) + [27 of 32] Compiling Config.Files ( Config/Files.hs, /tmp/stack29275/git-annex-6.20160511/.stack-work/dist/x86_64-openbsd/Cabal-1.22.5.0/setup/Config/Files.o ) + [28 of 32] Compiling Assistant.Install.AutoStart ( Assistant/Install/AutoStart.hs, /tmp/stack29275/git-annex-6.20160511/.stack-work/dist/x86_64-openbsd/Cabal-1.22.5.0/setup/Assistant/Install/AutoStart.o ) + [29 of 32] Compiling Assistant.Install.Menu ( Assistant/Install/Menu.hs, /tmp/stack29275/git-annex-6.20160511/.stack-work/dist/x86_64-openbsd/Cabal-1.22.5.0/setup/Assistant/Install/Menu.o ) + [30 of 32] Compiling Build.Configure ( Build/Configure.hs, /tmp/stack29275/git-annex-6.20160511/.stack-work/dist/x86_64-openbsd/Cabal-1.22.5.0/setup/Build/Configure.o ) + [31 of 32] Compiling Build.DesktopFile ( Build/DesktopFile.hs, /tmp/stack29275/git-annex-6.20160511/.stack-work/dist/x86_64-openbsd/Cabal-1.22.5.0/setup/Build/DesktopFile.o ) + [32 of 32] Compiling Main ( /tmp/stack29275/git-annex-6.20160511/Setup.hs, /tmp/stack29275/git-annex-6.20160511/.stack-work/dist/x86_64-openbsd/Cabal-1.22.5.0/setup/Main.o ) + Linking /tmp/stack29275/git-annex-6.20160511/.stack-work/dist/x86_64-openbsd/Cabal-1.22.5.0/setup/setup ... + /usr/local/lib/ghc/rts/libHSrts.a(RtsFlags.o): In function `copyArg': + + rts/RtsFlags.c:1685:0: + warning: warning: strcpy() is almost always misused, please use strlcpy() + /usr/local/lib/ghc/base_HQfYBxpPvuw8OunzQu6JGM/libHSbase-4.8.2.0-HQfYBxpPvuw8OunzQu6JGM.a(IO__1.o): In function `ghczuwrapperZC0ZCbaseZCSystemziIOZCrand': + (.text+0x1): warning: warning: rand() may return deterministic values, is that what you want? + /usr/local/lib/ghc/rts/libHSrts.a(RtsUtils.o): In function `showStgWord64': + + rts/RtsUtils.c:204:0: + warning: warning: sprintf() is often misused, please use snprintf() + checking version...fatal: Not a git repository (or any parent up to mount point /tmp) + Stopping at filesystem boundary (GIT_DISCOVERY_ACROSS_FILESYSTEM not set). + 6.20160511 + checking UPGRADE_LOCATION... not available + checking git... yes + checking git version... 2.7.0 + checking cp -a... no + checking cp -p... yes + checking cp --preserve=timestamps... no + checking cp --reflink=auto... no + checking xargs -0... yes + checking rsync... yes + checking curl... yes + checking wget... yes + checking wget supports -q --show-progress... yes + checking bup... no + checking nice... yes + checking ionice... no + checking nocache... no + checking gpg... not available + checking lsof... not available + checking git-remote-gcrypt... not available + checking ssh connection caching... yes + checking sha1... not available + checking sha256... not available + checking sha512... not available + checking sha224... not available + checking sha384... not available + Configuring git-annex-6.20160511... + Linking /tmp/stack29275/git-annex-6.20160511/.stack-work/dist/x86_64-openbsd/Cabal-1.22.5.0/setup/setup ... + /usr/local/lib/ghc/rts/libHSrts.a(RtsFlags.o): In function `copyArg': + + rts/RtsFlags.c:1685:0: + warning: warning: strcpy() is almost always misused, please use strlcpy() + /usr/local/lib/ghc/base_HQfYBxpPvuw8OunzQu6JGM/libHSbase-4.8.2.0-HQfYBxpPvuw8OunzQu6JGM.a(IO__1.o): In function `ghczuwrapperZC0ZCbaseZCSystemziIOZCrand': + (.text+0x1): warning: warning: rand() may return deterministic values, is that what you want? + /usr/local/lib/ghc/rts/libHSrts.a(RtsUtils.o): In function `showStgWord64': + + rts/RtsUtils.c:204:0: + warning: warning: sprintf() is often misused, please use snprintf() + Building git-annex-6.20160511... + Preprocessing executable 'git-annex' for git-annex-6.20160511... + + /tmp/stack29275/git-annex-6.20160511/Assistant/Threads/MountWatcher.hs:35:0: + warning: #warning Building without dbus support; will use mtab polling + [ 1 of 538] Compiling Utility.Dot ( Utility/Dot.hs, .stack-work/dist/x86_64-openbsd/Cabal-1.22.5.0/build/git-annex/git-annex-tmp/Utility/Dot.o ) + [ 2 of 538] Compiling Utility.Mounts ( Utility/Mounts.hs, .stack-work/dist/x86_64-openbsd/Cabal-1.22.5.0/build/git-annex/git-annex-tmp/Utility/Mounts.o ) + [ 3 of 538] Compiling Utility.SRV ( Utility/SRV.hs, .stack-work/dist/x86_64-openbsd/Cabal-1.22.5.0/build/git-annex/git-annex-tmp/Utility/SRV.o ) + [ 4 of 538] Compiling BuildFlags ( BuildFlags.hs, .stack-work/dist/x86_64-openbsd/Cabal-1.22.5.0/build/git-annex/git-annex-tmp/BuildFlags.o ) + [ 5 of 538] Compiling Utility.Yesod ( Utility/Yesod.hs, .stack-work/dist/x86_64-openbsd/Cabal-1.22.5.0/build/git-annex/git-annex-tmp/Utility/Yesod.o ) + [ 6 of 538] Compiling Utility.Touch ( Utility/Touch.hs, .stack-work/dist/x86_64-openbsd/Cabal-1.22.5.0/build/git-annex/git-annex-tmp/Utility/Touch.o ) + [ 7 of 538] Compiling Assistant.Types.BranchChange ( Assistant/Types/BranchChange.hs, .stack-work/dist/x86_64-openbsd/Cabal-1.22.5.0/build/git-annex/git-annex-tmp/Assistant/Types/BranchChange.o ) + [ 8 of 538] Compiling Assistant.Types.TransferSlots ( Assistant/Types/TransferSlots.hs, .stack-work/dist/x86_64-openbsd/Cabal-1.22.5.0/build/git-annex/git-annex-tmp/Assistant/Types/TransferSlots.o ) + [ 9 of 538] Compiling Assistant.Types.ThreadName ( Assistant/Types/ThreadName.hs, .stack-work/dist/x86_64-openbsd/Cabal-1.22.5.0/build/git-annex/git-annex-tmp/Assistant/Types/ThreadName.o ) + [ 10 of 538] Compiling Utility.Tense ( Utility/Tense.hs, .stack-work/dist/x86_64-openbsd/Cabal-1.22.5.0/build/git-annex/git-annex-tmp/Utility/Tense.o ) + [ 11 of 538] Compiling Assistant.Types.Alert ( Assistant/Types/Alert.hs, .stack-work/dist/x86_64-openbsd/Cabal-1.22.5.0/build/git-annex/git-annex-tmp/Assistant/Types/Alert.o ) + [ 12 of 538] Compiling Utility.OptParse ( Utility/OptParse.hs, .stack-work/dist/x86_64-openbsd/Cabal-1.22.5.0/build/git-annex/git-annex-tmp/Utility/OptParse.o ) + [ 13 of 538] Compiling Utility.PID ( Utility/PID.hs, .stack-work/dist/x86_64-openbsd/Cabal-1.22.5.0/build/git-annex/git-annex-tmp/Utility/PID.o ) + [ 14 of 538] Compiling Utility.Shell ( Utility/Shell.hs, .stack-work/dist/x86_64-openbsd/Cabal-1.22.5.0/build/git-annex/git-annex-tmp/Utility/Shell.o ) + [ 15 of 538] Compiling Logs.TimeStamp ( Logs/TimeStamp.hs, .stack-work/dist/x86_64-openbsd/Cabal-1.22.5.0/build/git-annex/git-annex-tmp/Logs/TimeStamp.o ) + [ 16 of 538] Compiling Utility.JSONStream ( Utility/JSONStream.hs, .stack-work/dist/x86_64-openbsd/Cabal-1.22.5.0/build/git-annex/git-annex-tmp/Utility/JSONStream.o ) + [ 17 of 538] Compiling Utility.HumanNumber ( Utility/HumanNumber.hs, .stack-work/dist/x86_64-openbsd/Cabal-1.22.5.0/build/git-annex/git-annex-tmp/Utility/HumanNumber.o ) + [ 18 of 538] Compiling Utility.Percentage ( Utility/Percentage.hs, .stack-work/dist/x86_64-openbsd/Cabal-1.22.5.0/build/git-annex/git-annex-tmp/Utility/Percentage.o ) + [ 19 of 538] Compiling Utility.Glob ( Utility/Glob.hs, .stack-work/dist/x86_64-openbsd/Cabal-1.22.5.0/build/git-annex/git-annex-tmp/Utility/Glob.o ) + [ 20 of 538] Compiling Utility.DataUnits ( Utility/DataUnits.hs, .stack-work/dist/x86_64-openbsd/Cabal-1.22.5.0/build/git-annex/git-annex-tmp/Utility/DataUnits.o ) + [ 21 of 538] Compiling Types.Creds ( Types/Creds.hs, .stack-work/dist/x86_64-openbsd/Cabal-1.22.5.0/build/git-annex/git-annex-tmp/Types/Creds.o ) + [ 22 of 538] Compiling Assistant.Types.CredPairCache ( Assistant/Types/CredPairCache.hs, .stack-work/dist/x86_64-openbsd/Cabal-1.22.5.0/build/git-annex/git-annex-tmp/Assistant/Types/CredPairCache.o ) + [ 23 of 538] Compiling Types.Availability ( Types/Availability.hs, .stack-work/dist/x86_64-openbsd/Cabal-1.22.5.0/build/git-annex/git-annex-tmp/Types/Availability.o ) + [ 24 of 538] Compiling Utility.Bloom ( Utility/Bloom.hs, .stack-work/dist/x86_64-openbsd/Cabal-1.22.5.0/build/git-annex/git-annex-tmp/Utility/Bloom.o ) + [ 25 of 538] Compiling Utility.LockFile.LockStatus ( Utility/LockFile/LockStatus.hs, .stack-work/dist/x86_64-openbsd/Cabal-1.22.5.0/build/git-annex/git-annex-tmp/Utility/LockFile/LockStatus.o ) + [ 26 of 538] Compiling Utility.QuickCheck ( Utility/QuickCheck.hs, .stack-work/dist/x86_64-openbsd/Cabal-1.22.5.0/build/git-annex/git-annex-tmp/Utility/QuickCheck.o ) + + /tmp/stack29275/git-annex-6.20160511/Utility/QuickCheck.hs:19:1: Warning: + The qualified import of `Data.Map' is redundant + except perhaps to import instances from `Data.Map' + To import instances alone, use: import Data.Map() + + /tmp/stack29275/git-annex-6.20160511/Utility/QuickCheck.hs:20:1: Warning: + The qualified import of `Data.Set' is redundant + except perhaps to import instances from `Data.Set' + To import instances alone, use: import Data.Set() + [ 27 of 538] Compiling Types.DesktopNotify ( Types/DesktopNotify.hs, .stack-work/dist/x86_64-openbsd/Cabal-1.22.5.0/build/git-annex/git-annex-tmp/Types/DesktopNotify.o ) + [ 28 of 538] Compiling Types.UUID ( Types/UUID.hs, .stack-work/dist/x86_64-openbsd/Cabal-1.22.5.0/build/git-annex/git-annex-tmp/Types/UUID.o ) + [ 29 of 538] Compiling Types.Group ( Types/Group.hs, .stack-work/dist/x86_64-openbsd/Cabal-1.22.5.0/build/git-annex/git-annex-tmp/Types/Group.o ) + [ 30 of 538] Compiling Types.BranchState ( Types/BranchState.hs, .stack-work/dist/x86_64-openbsd/Cabal-1.22.5.0/build/git-annex/git-annex-tmp/Types/BranchState.o ) + [ 31 of 538] Compiling Utility.Process.Shim ( Utility/Process/Shim.hs, .stack-work/dist/x86_64-openbsd/Cabal-1.22.5.0/build/git-annex/git-annex-tmp/Utility/Process/Shim.o ) + [ 32 of 538] Compiling Utility.FileSize ( Utility/FileSize.hs, .stack-work/dist/x86_64-openbsd/Cabal-1.22.5.0/build/git-annex/git-annex-tmp/Utility/FileSize.o ) + [ 33 of 538] Compiling Utility.PosixFiles ( Utility/PosixFiles.hs, .stack-work/dist/x86_64-openbsd/Cabal-1.22.5.0/build/git-annex/git-annex-tmp/Utility/PosixFiles.o ) + [ 34 of 538] Compiling Utility.Applicative ( Utility/Applicative.hs, .stack-work/dist/x86_64-openbsd/Cabal-1.22.5.0/build/git-annex/git-annex-tmp/Utility/Applicative.o ) + [ 35 of 538] Compiling Utility.PartialPrelude ( Utility/PartialPrelude.hs, .stack-work/dist/x86_64-openbsd/Cabal-1.22.5.0/build/git-annex/git-annex-tmp/Utility/PartialPrelude.o ) + [ 36 of 538] Compiling Utility.ThreadScheduler ( Utility/ThreadScheduler.hs, .stack-work/dist/x86_64-openbsd/Cabal-1.22.5.0/build/git-annex/git-annex-tmp/Utility/ThreadScheduler.o ) + [ 37 of 538] Compiling Utility.HumanTime ( Utility/HumanTime.hs, .stack-work/dist/x86_64-openbsd/Cabal-1.22.5.0/build/git-annex/git-annex-tmp/Utility/HumanTime.o ) + [ 38 of 538] Compiling Utility.Hash ( Utility/Hash.hs, .stack-work/dist/x86_64-openbsd/Cabal-1.22.5.0/build/git-annex/git-annex-tmp/Utility/Hash.o ) + [ 39 of 538] Compiling Utility.Env ( Utility/Env.hs, .stack-work/dist/x86_64-openbsd/Cabal-1.22.5.0/build/git-annex/git-annex-tmp/Utility/Env.o ) + [ 40 of 538] Compiling Utility.UserInfo ( Utility/UserInfo.hs, .stack-work/dist/x86_64-openbsd/Cabal-1.22.5.0/build/git-annex/git-annex-tmp/Utility/UserInfo.o ) + [ 41 of 538] Compiling Utility.Verifiable ( Utility/Verifiable.hs, .stack-work/dist/x86_64-openbsd/Cabal-1.22.5.0/build/git-annex/git-annex-tmp/Utility/Verifiable.o ) + [ 42 of 538] Compiling Utility.Format ( Utility/Format.hs, .stack-work/dist/x86_64-openbsd/Cabal-1.22.5.0/build/git-annex/git-annex-tmp/Utility/Format.o ) + [ 43 of 538] Compiling Build.SysConfig ( Build/SysConfig.hs, .stack-work/dist/x86_64-openbsd/Cabal-1.22.5.0/build/git-annex/git-annex-tmp/Build/SysConfig.o ) + [ 44 of 538] Compiling Config.Cost ( Config/Cost.hs, .stack-work/dist/x86_64-openbsd/Cabal-1.22.5.0/build/git-annex/git-annex-tmp/Config/Cost.o ) + [ 45 of 538] Compiling Types.Messages ( Types/Messages.hs, .stack-work/dist/x86_64-openbsd/Cabal-1.22.5.0/build/git-annex/git-annex-tmp/Types/Messages.o ) + [ 46 of 538] Compiling Types.TrustLevel ( Types/TrustLevel.hs, .stack-work/dist/x86_64-openbsd/Cabal-1.22.5.0/build/git-annex/git-annex-tmp/Types/TrustLevel.o ) + [ 47 of 538] Compiling Types.Test ( Types/Test.hs, .stack-work/dist/x86_64-openbsd/Cabal-1.22.5.0/build/git-annex/git-annex-tmp/Types/Test.o ) + [ 48 of 538] Compiling Utility.Data ( Utility/Data.hs, .stack-work/dist/x86_64-openbsd/Cabal-1.22.5.0/build/git-annex/git-annex-tmp/Utility/Data.o ) + [ 49 of 538] Compiling Utility.Exception ( Utility/Exception.hs, .stack-work/dist/x86_64-openbsd/Cabal-1.22.5.0/build/git-annex/git-annex-tmp/Utility/Exception.o ) + [ 50 of 538] Compiling Utility.FileMode ( Utility/FileMode.hs, .stack-work/dist/x86_64-openbsd/Cabal-1.22.5.0/build/git-annex/git-annex-tmp/Utility/FileMode.o ) + [ 51 of 538] Compiling Git.FileMode ( Git/FileMode.hs, .stack-work/dist/x86_64-openbsd/Cabal-1.22.5.0/build/git-annex/git-annex-tmp/Git/FileMode.o ) + [ 52 of 538] Compiling Utility.FileSystemEncoding ( Utility/FileSystemEncoding.hs, .stack-work/dist/x86_64-openbsd/Cabal-1.22.5.0/build/git-annex/git-annex-tmp/Utility/FileSystemEncoding.o ) + [ 53 of 538] Compiling Utility.Base64 ( Utility/Base64.hs, .stack-work/dist/x86_64-openbsd/Cabal-1.22.5.0/build/git-annex/git-annex-tmp/Utility/Base64.o ) + [ 54 of 538] Compiling Utility.Tmp ( Utility/Tmp.hs, .stack-work/dist/x86_64-openbsd/Cabal-1.22.5.0/build/git-annex/git-annex-tmp/Utility/Tmp.o ) + [ 55 of 538] Compiling Database.Handle ( Database/Handle.hs, .stack-work/dist/x86_64-openbsd/Cabal-1.22.5.0/build/git-annex/git-annex-tmp/Database/Handle.o ) + [ 56 of 538] Compiling Utility.LockFile.Posix ( Utility/LockFile/Posix.hs, .stack-work/dist/x86_64-openbsd/Cabal-1.22.5.0/build/git-annex/git-annex-tmp/Utility/LockFile/Posix.o ) + [ 57 of 538] Compiling Utility.DiskFree ( Utility/DiskFree.hs, .stack-work/dist/x86_64-openbsd/Cabal-1.22.5.0/build/git-annex/git-annex-tmp/Utility/DiskFree.o ) + [ 58 of 538] Compiling Utility.Monad ( Utility/Monad.hs, .stack-work/dist/x86_64-openbsd/Cabal-1.22.5.0/build/git-annex/git-annex-tmp/Utility/Monad.o ) + [ 59 of 538] Compiling Utility.Misc ( Utility/Misc.hs, .stack-work/dist/x86_64-openbsd/Cabal-1.22.5.0/build/git-annex/git-annex-tmp/Utility/Misc.o ) + [ 60 of 538] Compiling Utility.Process ( Utility/Process.hs, .stack-work/dist/x86_64-openbsd/Cabal-1.22.5.0/build/git-annex/git-annex-tmp/Utility/Process.o ) + [ 61 of 538] Compiling Utility.SafeCommand ( Utility/SafeCommand.hs, .stack-work/dist/x86_64-openbsd/Cabal-1.22.5.0/build/git-annex/git-annex-tmp/Utility/SafeCommand.o ) + [ 62 of 538] Compiling Git.Types ( Git/Types.hs, .stack-work/dist/x86_64-openbsd/Cabal-1.22.5.0/build/git-annex/git-annex-tmp/Git/Types.o ) + [ 63 of 538] Compiling Utility.Network ( Utility/Network.hs, .stack-work/dist/x86_64-openbsd/Cabal-1.22.5.0/build/git-annex/git-annex-tmp/Utility/Network.o ) + [ 64 of 538] Compiling Utility.Scheduled ( Utility/Scheduled.hs, .stack-work/dist/x86_64-openbsd/Cabal-1.22.5.0/build/git-annex/git-annex-tmp/Utility/Scheduled.o ) + [ 65 of 538] Compiling Utility.Scheduled.QuickCheck ( Utility/Scheduled/QuickCheck.hs, .stack-work/dist/x86_64-openbsd/Cabal-1.22.5.0/build/git-annex/git-annex-tmp/Utility/Scheduled/QuickCheck.o ) + [ 66 of 538] Compiling Utility.ExternalSHA ( Utility/ExternalSHA.hs, .stack-work/dist/x86_64-openbsd/Cabal-1.22.5.0/build/git-annex/git-annex-tmp/Utility/ExternalSHA.o ) + [ 67 of 538] Compiling Utility.Directory ( Utility/Directory.hs, .stack-work/dist/x86_64-openbsd/Cabal-1.22.5.0/build/git-annex/git-annex-tmp/Utility/Directory.o ) + [ 68 of 538] Compiling Utility.Path ( Utility/Path.hs, .stack-work/dist/x86_64-openbsd/Cabal-1.22.5.0/build/git-annex/git-annex-tmp/Utility/Path.o ) + [ 69 of 538] Compiling Common ( Common.hs, .stack-work/dist/x86_64-openbsd/Cabal-1.22.5.0/build/git-annex/git-annex-tmp/Common.o ) + [ 70 of 538] Compiling Git ( Git.hs, .stack-work/dist/x86_64-openbsd/Cabal-1.22.5.0/build/git-annex/git-annex-tmp/Git.o ) + [ 71 of 538] Compiling Git.Filename ( Git/Filename.hs, .stack-work/dist/x86_64-openbsd/Cabal-1.22.5.0/build/git-annex/git-annex-tmp/Git/Filename.o ) + [ 72 of 538] Compiling Git.FilePath ( Git/FilePath.hs, .stack-work/dist/x86_64-openbsd/Cabal-1.22.5.0/build/git-annex/git-annex-tmp/Git/FilePath.o ) + [ 73 of 538] Compiling Git.DiffTreeItem ( Git/DiffTreeItem.hs, .stack-work/dist/x86_64-openbsd/Cabal-1.22.5.0/build/git-annex/git-annex-tmp/Git/DiffTreeItem.o ) + [ 74 of 538] Compiling Logs.MapLog ( Logs/MapLog.hs, .stack-work/dist/x86_64-openbsd/Cabal-1.22.5.0/build/git-annex/git-annex-tmp/Logs/MapLog.o ) + [ 75 of 538] Compiling Types.MetaData ( Types/MetaData.hs, .stack-work/dist/x86_64-openbsd/Cabal-1.22.5.0/build/git-annex/git-annex-tmp/Types/MetaData.o ) + [ 76 of 538] Compiling Annex.MetaData.StandardFields ( Annex/MetaData/StandardFields.hs, .stack-work/dist/x86_64-openbsd/Cabal-1.22.5.0/build/git-annex/git-annex-tmp/Annex/MetaData/StandardFields.o ) + [ 77 of 538] Compiling Types.Key ( Types/Key.hs, .stack-work/dist/x86_64-openbsd/Cabal-1.22.5.0/build/git-annex/git-annex-tmp/Types/Key.o ) + [ 78 of 538] Compiling Messages.JSON ( Messages/JSON.hs, .stack-work/dist/x86_64-openbsd/Cabal-1.22.5.0/build/git-annex/git-annex-tmp/Messages/JSON.o ) + [ 79 of 538] Compiling Utility.InodeCache ( Utility/InodeCache.hs, .stack-work/dist/x86_64-openbsd/Cabal-1.22.5.0/build/git-annex/git-annex-tmp/Utility/InodeCache.o ) + [ 80 of 538] Compiling Types.KeySource ( Types/KeySource.hs, .stack-work/dist/x86_64-openbsd/Cabal-1.22.5.0/build/git-annex/git-annex-tmp/Types/KeySource.o ) + [ 81 of 538] Compiling Types.Backend ( Types/Backend.hs, .stack-work/dist/x86_64-openbsd/Cabal-1.22.5.0/build/git-annex/git-annex-tmp/Types/Backend.o ) + [ 82 of 538] Compiling Database.Types ( Database/Types.hs, .stack-work/dist/x86_64-openbsd/Cabal-1.22.5.0/build/git-annex/git-annex-tmp/Database/Types.o ) + ghc: /usr/local/lib/libidn.a: unknown symbol `libintl_bindtextdomain' + ghc: unable to load package `gnuidn-0.2.2' +"""]] + +> So git-annex dropped xmpp and the libidn.a +> library will no longer be pulled in. +> +> Also, you were right about `stack install git-annex` +> being problimatic. The build instuctions have since been changed to +> run stack inside the git-annex source tree. +> Since that worked for you, closing this bug. [[done]] --[[Joey]] diff --git a/doc/bugs/does_not_build_on_OpenBSD_5.9/comment_1_e69ed5e0c9e474158da6764d27f67b67._comment b/doc/bugs/does_not_build_on_OpenBSD_5.9/comment_1_e69ed5e0c9e474158da6764d27f67b67._comment new file mode 100644 index 0000000000..5cac7c971a --- /dev/null +++ b/doc/bugs/does_not_build_on_OpenBSD_5.9/comment_1_e69ed5e0c9e474158da6764d27f67b67._comment @@ -0,0 +1,16 @@ +[[!comment format=mdwn + username="joey" + subject="""comment 1""" + date="2016-07-11T16:14:37Z" + content=""" +The build error suggests a problem with either the libidn.a library in +openbsd, or the haskell bindings to it in gnuidn. + +A similar problem is discussed in [[More_build_oddities_under_OpenBSD]]. + +Disabling XMPP support in git-annex makes it not need either library, +so should avoid the problem. Eg: `stack build --flag git-annex:-XMPP` + +(Although AFAICS, XMPP support is disabled by default in stack.yaml anyway, +so I am not sure why your stack build included it.) +"""]] diff --git a/doc/bugs/does_not_build_on_OpenBSD_5.9/comment_2_c6521306cbee2e398998cd2eec4e47ca._comment b/doc/bugs/does_not_build_on_OpenBSD_5.9/comment_2_c6521306cbee2e398998cd2eec4e47ca._comment new file mode 100644 index 0000000000..2466bd932d --- /dev/null +++ b/doc/bugs/does_not_build_on_OpenBSD_5.9/comment_2_c6521306cbee2e398998cd2eec4e47ca._comment @@ -0,0 +1,17 @@ +[[!comment format=mdwn + username="quisquous" + subject="comment 2" + date="2016-07-11T17:59:56Z" + content=""" +Perhaps running ````stack install git-annex```` (using the implicit global project) ignores the flags in stack.yaml, see [#1313](https://github.com/commercialhaskell/stack/issues/1313). Following your suggestion, I tried cloning the repo and building using `stack build` both with and without the XMPP flag. Without the flag works fine, it seems the default ````false```` setting form xmpp flag in stack.yaml is honored when building *this* way. + +In any case, this worked: + +```` +$ git clone git://git-annex.branchable.com/ git-annex +$ cd git-annex +$ git checkout 6.20160511 +$ stack build +$ stack install +```` +"""]] diff --git a/doc/bugs/does_not_build_on_OpenBSD_5.9/comment_3_8a563d9ccba5e912713812c77d34ea1f._comment b/doc/bugs/does_not_build_on_OpenBSD_5.9/comment_3_8a563d9ccba5e912713812c77d34ea1f._comment new file mode 100644 index 0000000000..87b1c4c127 --- /dev/null +++ b/doc/bugs/does_not_build_on_OpenBSD_5.9/comment_3_8a563d9ccba5e912713812c77d34ea1f._comment @@ -0,0 +1,15 @@ +[[!comment format=mdwn + username="joey" + subject="""comment 3""" + date="2016-07-12T18:31:17Z" + content=""" +Not sure if that stack bug you linked to is quite a match for what it's +doing, so I opened a new one: + + +I don't currently see anything I could do in git-annex to avoid this +problem. + +What version of stack are you using? It could be that a newer version of +stack doesn't have the problem. +"""]] diff --git a/doc/bugs/does_not_build_on_OpenBSD_5.9/comment_4_557f5a38ae78b21b18a63d3ba16bc099._comment b/doc/bugs/does_not_build_on_OpenBSD_5.9/comment_4_557f5a38ae78b21b18a63d3ba16bc099._comment new file mode 100644 index 0000000000..7ea3e42947 --- /dev/null +++ b/doc/bugs/does_not_build_on_OpenBSD_5.9/comment_4_557f5a38ae78b21b18a63d3ba16bc099._comment @@ -0,0 +1,10 @@ +[[!comment format=mdwn + username="quisquous" + subject="comment 4" + date="2016-07-12T18:59:55Z" + content=""" +```` +~ ❯❯❯ stack --version +Version 1.1.2 x86_64 hpack-0.14.0 +```` +"""]] diff --git a/doc/bugs/does_not_complain__47__fail_if_by_mistake_option_value_passed_within_arg_for__-c.mdwn b/doc/bugs/does_not_complain__47__fail_if_by_mistake_option_value_passed_within_arg_for__-c.mdwn new file mode 100644 index 0000000000..dbaceb9eaf --- /dev/null +++ b/doc/bugs/does_not_complain__47__fail_if_by_mistake_option_value_passed_within_arg_for__-c.mdwn @@ -0,0 +1,26 @@ +### Please describe the problem. + +spotted that in my script I have "incorrectly" passed both -c and its value within the same single argument ('-c annex....'). But surprisingly annex neither treated it correctly, nor complained that something was incorrectly passed in. Shouldn't it fail if option is not '-c' but '-c smthelse' or correctly treat the value in such as case? + +### What version of git-annex are you using? On what operating system? + +5.20151104+gitge9cdce6-1~ndall+1 + +### Please provide any additional information below. + +[[!format sh """ +$> ls -ld svgtune-0.2.0/README.rst +-rw------- 1 yoh yoh 9 Nov 12 12:03 svgtune-0.2.0/README.rst + +$> git-annex add --debug '-c annex.largefiles=exclude=*.rst' svgtune-0.2.0/README.rst +add svgtune-0.2.0/README.rst ok +(recording state in git...) + +$> ls -ld svgtune-0.2.0/README.rst +lrwxrwxrwx 1 yoh yoh 189 Nov 12 12:03 svgtune-0.2.0/README.rst -> ../.git/annex/objects/32/vx/SHA256E-s9--246c960678ee9a80aa6d2eff2f1df1debd590cb73aa5fed6cb4b13b8018599f5.rst/SHA256E-s9--246c960678ee9a80aa6d2eff2f1df1debd590cb73aa5fed6cb4b13b8018599f5.rst +"""]] + +[[!meta author=yoh]] + +> I guess it could be considered [[done]] --[[yoh]] + diff --git a/doc/bugs/does_not_handle_youtube_playlists.mdwn b/doc/bugs/does_not_handle_youtube_playlists.mdwn new file mode 100644 index 0000000000..213fa96328 --- /dev/null +++ b/doc/bugs/does_not_handle_youtube_playlists.mdwn @@ -0,0 +1,43 @@ +### Please describe the problem. + +Not really related to datalad -- thought to addurl youtube playlist + +youtube-dl seems might be capable of doing it + +### Please provide any additional information below. + +[[!format sh """ + +$> git annex addurl --debug 'https://www.youtube.com/playlist?list=PLBHioGD0U1Cjd-meZbEcz-9ZxK-mb50tZ' [2018-03-28 13:09:06.337738339] read: git ["--git-dir=.git","--work-tree=.","--literal-pathspecs","show-ref","git-annex"] +[2018-03-28 13:09:06.34708705] process done ExitSuccess +[2018-03-28 13:09:06.347202003] read: git ["--git-dir=.git","--work-tree=.","--literal-pathspecs","show-ref","--hash","refs/heads/git-annex"] +[2018-03-28 13:09:06.35303272] process done ExitSuccess +[2018-03-28 13:09:06.353205536] read: git ["--git-dir=.git","--work-tree=.","--literal-pathspecs","log","refs/heads/git-annex..734b368eab4f39d8494671657977952b02a35d9a","--pretty=%H","-n1"] +[2018-03-28 13:09:06.359101486] process done ExitSuccess +[2018-03-28 13:09:06.360005167] chat: git ["--git-dir=.git","--work-tree=.","--literal-pathspecs","cat-file","--batch"] +[2018-03-28 13:09:06.361006846] chat: git ["--git-dir=.git","--work-tree=.","--literal-pathspecs","cat-file","--batch-check=%(objectname) %(objecttype) %(objectsize)"] +addurl https://www.youtube.com/playlist?list=PLBHioGD0U1Cjd-meZbEcz-9ZxK-mb50tZ +[2018-03-28 13:09:06.409163359] call: wget ["-nv","--show-progress","--clobber","-c","-O","/tmp/testyt/.git/annex/tmp/URL--https&c%%www.youtube.com%playlis-5bc73cdf8dc8bd73b13addc290b160e6","https://www.youtube.com/playlist?list=PLBHioGD0U1Cjd-meZbEcz-9ZxK-mb50tZ","--user-agent","git-annex/6.20180316+gitg308f3ecf6-1~ndall+1"] +/tmp/testyt/.git/ann 100%[===================>] 211.54K --.-KB/s in 0.04s +2018-03-28 13:09:06 URL:https://www.youtube.com/playlist?list=PLBHioGD0U1Cjd-meZbEcz-9ZxK-mb50tZ [216616/216616] -> "/tmp/testyt/.git/annex/tmp/URL--https&c%%www.youtube.com%playlis-5bc73cdf8dc8bd73b13addc290b160e6" [1] +[2018-03-28 13:09:06.844196402] process done ExitSuccess +[2018-03-28 13:09:06.845246148] read: youtube-dl ["https://www.youtube.com/playlist?list=PLBHioGD0U1Cjd-meZbEcz-9ZxK-mb50tZ","--get-filename","--no-warnings"] +[2018-03-28 13:09:40.643440496] process done ExitSuccess +[2018-03-28 13:09:40.644420015] call: youtube-dl ["https://www.youtube.com/playlist?list=PLBHioGD0U1Cjd-meZbEcz-9ZxK-mb50tZ","--no-playlist","--playlist-items","0","--max-filesize","9046842816"] +[youtube:playlist] PLBHioGD0U1Cjd-meZbEcz-9ZxK-mb50tZ: Downloading webpage +[download] Downloading playlist: Canonical Computation in Brains and Machines +[youtube:playlist] playlist Canonical Computation in Brains and Machines: Downloading 0 videos +[download] Finished downloading playlist: Canonical Computation in Brains and Machines +[2018-03-28 13:09:41.925771815] process done ExitSuccess + + youtube-dl did not put any media in its work directory, perhaps it's been configured to store files somewhere else? +failed +[2018-03-28 13:09:41.926669401] process done ExitSuccess +[2018-03-28 13:09:41.926951446] process done ExitSuccess +git-annex: addurl: 1 failed +git annex addurl --debug 6.49s user 0.32s system 18% cpu 35.834 total + +"""]] + + +> importfeed supports youtube playlists, once you look up their rss feed. [[done]] --[[Joey]] diff --git a/doc/bugs/done.mdwn b/doc/bugs/done.mdwn new file mode 100644 index 0000000000..62b5fc9e93 --- /dev/null +++ b/doc/bugs/done.mdwn @@ -0,0 +1,4 @@ +recently fixed [[bugs]] + +[[!inline pages="./* and link(./done) and !*/Discussion" sort=mtime show=0 +archive=yes feedlimit=10]] diff --git a/doc/bugs/drop_blows_on_lustre__58___SQLite3_returned_ErrorIO.mdwn b/doc/bugs/drop_blows_on_lustre__58___SQLite3_returned_ErrorIO.mdwn new file mode 100644 index 0000000000..c90a9f21a7 --- /dev/null +++ b/doc/bugs/drop_blows_on_lustre__58___SQLite3_returned_ErrorIO.mdwn @@ -0,0 +1,51 @@ +[[!format sh """ +$> git clone http://data.pymvpa.org/datasets/tutorial_data/.git tutorial_data-2 +Cloning into 'tutorial_data-2'... +Checking connectivity... done. +cd +#35 !280 [0].....................................:Thu Jul 14 10:16:49:. +anthill:/home/ironfs/scratch/yarik +$> cd tutorial_data-2 +total 36 + 4 freesurfer/ 4 hyperalignment_tutorial_data.hdf5.gz@ 20 suma_surfaces/ + 4 haxby2001/ 4 results/ +(git)/home/ironfs/scratch/yarik/tutorial_data-2:[master] +#36 !281 [0].....................................:Thu Jul 14 10:16:50:. +anthill:/home/ironfs/scratch/yarik/tutorial_data-2 +$> git annex get hyperalignment_tutorial_data.hdf5.gz + + Detected a filesystem without POSIX fcntl lock support. + + Enabling annex.pidlock. +(merging origin/git-annex into git-annex...) +(recording state in git...) +/home/ironfs/scratch 100%[===================>] 15.04M 68.9MB/s in 0.2s +(checksum...) ok +(recording state in git...) +(git)/home/ironfs/scratch/yarik/tutorial_data-2:[master] +#37 !282 [0].....................................:Thu Jul 14 10:16:54:. +anthill:/home/ironfs/scratch/yarik/tutorial_data-2 +$> git annex drop hyperalignment_tutorial_data.hdf5.gz +drop hyperalignment_tutorial_data.hdf5.gz (checking origin...) git-annex: SQLite3 returned ErrorIO while attempting to perform prepare "PRAGMA journal_mode=WAL;": disk I/O error +(git)/home/ironfs/scratch/yarik/tutorial_data-2:[master] + +$> git annex fsck +fsck freesurfer/anat_nii.nii sqlite worker thread crashed: SQLite3 returned ErrorIO while attempting to perform prepare "SELECT null from content limit 1": disk I/O error +git-annex: sqlite query crashed + +$> git annex version +git-annex version: 6.20160613-g35dbe35 +build flags: Assistant Webapp Pairing Testsuite S3(multipartupload)(storageclasses) WebDAV Inotify DBus DesktopNotify XMPP ConcurrentOutput TorrentParser MagicMime Feeds Quvi +key/value backends: SHA256E SHA256 SHA512E SHA512 SHA224E SHA224 SHA384E SHA384 SHA3_256E SHA3_256 SHA3_512E SHA3_512 SHA3_224E SHA3_224 SHA3_384E SHA3_384 SKEIN256E SKEIN256 SKEIN512E SKEIN512 SHA1E SHA1 MD5E MD5 WORM URL +remote types: git gcrypt S3 bup directory rsync web bittorrent webdav tahoe glacier ddar hook external +local repository version: 5 +supported repository versions: 5 6 +upgrade supported from repository versions: 0 1 2 3 4 5 +operating system: linux x86_64 +(git)/home/ironfs/scratch/yarik/tutorial_data:[master] + +"""]] + + + +[[!meta author=yoh]] diff --git a/doc/bugs/drop_blows_on_lustre__58___SQLite3_returned_ErrorIO/comment_1_49e8920ad09ae71874686f50f566b77c._comment b/doc/bugs/drop_blows_on_lustre__58___SQLite3_returned_ErrorIO/comment_1_49e8920ad09ae71874686f50f566b77c._comment new file mode 100644 index 0000000000..a9734a4f5d --- /dev/null +++ b/doc/bugs/drop_blows_on_lustre__58___SQLite3_returned_ErrorIO/comment_1_49e8920ad09ae71874686f50f566b77c._comment @@ -0,0 +1,32 @@ +[[!comment format=mdwn + username="joey" + subject="""comment 1""" + date="2016-07-19T15:38:21Z" + content=""" +The v6 changes added a sqlite database. Some code will try to query or +write that database even in v5 mode, although it's meant to give up if the +database is not available. + +So, the easy fix, at least for the drop problem, is to avoid using the keys +database at all in v5 mode. I've made this change and it will probably fix +the case you reported. + +But, that won't help with v6 repos which need to use that sqlite database. +And, incremental fsck uses its own sqlite database too. And, +[[design/caching_database]] plans are to use sqlite databases more broadly +in the future. + +I'm sure that it's not a good idea for git-annex to catch "disk IO error" +exceptions from the database layer. So, it seems that most any other fix than +avoiding using the database would need to be made in sqlite or in lustre, +which it seems don't get along. At a guess, sqlite is trying to use some +POSIX filesystem functionality, likely related to locking, that lustre does +not support. + +Hmm, what could be done to hack in support for lustre is to +move the sqlite databases to a different filesystem. But, accessing the +same repo from different hosts which have different sqlite databases would +lead to inconsistent and buggy behavior. And repo setup would need to +decide where to put the sqlite databases and manually configure that +location. So this would be very much a caveat empror configuration. +"""]] diff --git a/doc/bugs/drop_from_special_hybrid_encryption_remote_fails.mdwn b/doc/bugs/drop_from_special_hybrid_encryption_remote_fails.mdwn new file mode 100644 index 0000000000..13e06bd287 --- /dev/null +++ b/doc/bugs/drop_from_special_hybrid_encryption_remote_fails.mdwn @@ -0,0 +1,57 @@ +### Please describe the problem. + +I am trying to drop a file from a special, hybrid encryption remote. It says "failed", but I cannot figure out why, the commands in --debug all seem to exit with ExitSuccess. + +### What steps will reproduce the problem? + +Not sure how to reproduce from scratch, see log below for what happens when I do +git annex drop file -f specialremote ... + +### What version of git-annex are you using? On what operating system? + +[[!format sh """ +$ git annex version +git-annex version: 5.20150916+gitg79661ef-1~ndall+1 +build flags: Assistant Webapp Webapp-secure Pairing Testsuite S3 WebDAV Inotify DBus DesktopNotify XMPP DNS Feeds Quvi TDFA Database +key/value backends: SHA256E SHA256 SHA512E SHA512 SHA224E SHA224 SHA384E SHA384 SHA3_256E SHA3_256 SHA3_512E SHA3_512 SHA3_224E SHA3_224 SHA3_384E SHA3_384 SKEIN256E SKEIN256 SKEIN512E SKEIN512 SHA1E SHA1 MD5E MD5 WORM URL +remote types: git gcrypt S3 bup directory rsync web bittorrent webdav tahoe glacier ddar hook external +local repository version: 5 +supported repository version: 5 +upgrade supported from repository versions: 0 1 2 4 +"""]] + +### Please provide any additional information below. + +[[!format sh """ +[2015-12-27 00:15:58.093036] read: git ["--git-dir=../../.git","--work-tree=../..","--literal-pathspecs","ls-files","--cached","-z","--","20150816_cccamp/DSC04537.ARW"] +[2015-12-27 00:15:58.147092] read: git ["--git-dir=../../.git","--work-tree=../..","--literal-pathspecs","show-ref","git-annex"] +[2015-12-27 00:15:58.150596] process done ExitSuccess +[2015-12-27 00:15:58.150901] read: git ["--git-dir=../../.git","--work-tree=../..","--literal-pathspecs","show-ref","--hash","refs/heads/git-annex"] +[2015-12-27 00:15:58.155607] process done ExitSuccess +[2015-12-27 00:15:58.157843] read: git ["--git-dir=../../.git","--work-tree=../..","--literal-pathspecs","log","refs/heads/git-annex..85eb4dbbebcc100c2414f5a71f4fb9a22f0239fb","-n1","--pretty=%H"] +[2015-12-27 00:15:58.162227] process done ExitSuccess +[2015-12-27 00:15:58.162985] read: git ["--git-dir=../../.git","--work-tree=../..","--literal-pathspecs","log","refs/heads/git-annex..c0747d4c69cf80aa0290f1385454eb76a4c21032","-n1","--pretty=%H"] +[2015-12-27 00:15:58.168515] process done ExitSuccess +[2015-12-27 00:15:58.168888] read: git ["--git-dir=../../.git","--work-tree=../..","--literal-pathspecs","log","refs/heads/git-annex..0357da603dc61b28814125650def17fb0e3f82ca","-n1","--pretty=%H"] +[2015-12-27 00:15:58.174589] process done ExitSuccess +[2015-12-27 00:15:58.1749] read: git ["--git-dir=../../.git","--work-tree=../..","--literal-pathspecs","log","refs/heads/git-annex..4673c36870f0a536b4bedc3f5826f2d6b39c9d6a","-n1","--pretty=%H"] +[2015-12-27 00:15:58.214221] process done ExitSuccess +[2015-12-27 00:15:58.215418] read: git ["--git-dir=../../.git","--work-tree=../..","--literal-pathspecs","log","refs/heads/git-annex..75de80c0acf77e04e5b2b4c6f7947aec75603989","-n1","--pretty=%H"] +[2015-12-27 00:15:58.243546] process done ExitSuccess +[2015-12-27 00:15:58.244412] chat: git ["--git-dir=../../.git","--work-tree=../..","--literal-pathspecs","cat-file","--batch"] +[2015-12-27 00:15:58.250349] read: git ["config","--null","--list"] +[2015-12-27 00:15:58.250859] read: git ["config","--null","--list"] +[2015-12-27 00:15:58.253927] process done ExitSuccess +[2015-12-27 00:15:58.254538] chat: git ["--git-dir=../../.git","--work-tree=../..","--literal-pathspecs","check-attr","-z","--stdin","annex.backend","annex.numcopies","--"] +[2015-12-27 00:15:58.255324] read: git ["--version"] +[2015-12-27 00:15:58.257803] process done ExitSuccess +drop stratohidrive 20150816_cccamp/DSC04537.ARW [2015-12-27 00:15:58.328068] chat: gpg ["--batch","--no-tty","--use-agent","--quiet","--trust-model","always","--decrypt"] +[2015-12-27 00:15:58.354229] process done ExitSuccess +failed +git-annex: drop: 1 failed +# End of transcript or log. +"""]] + +### Have you had any luck using git-annex before? (Sometimes we get tired of reading bug reports all day and a lil' positive end note does wonders) + +As mentioned before, I am very, very happy with git-annex :-) Discovery of 2015 for me. diff --git a/doc/bugs/drop_from_special_hybrid_encryption_remote_fails/comment_1_09e26fdacfb5c159997671680dc95ab7._comment b/doc/bugs/drop_from_special_hybrid_encryption_remote_fails/comment_1_09e26fdacfb5c159997671680dc95ab7._comment new file mode 100644 index 0000000000..96a96bd4fe --- /dev/null +++ b/doc/bugs/drop_from_special_hybrid_encryption_remote_fails/comment_1_09e26fdacfb5c159997671680dc95ab7._comment @@ -0,0 +1,13 @@ +[[!comment format=mdwn + username="joey" + subject="""comment 1""" + date="2016-01-20T19:14:23Z" + content=""" +What kind of special remote is this? In fact, can you just paste the output +of `git annex info thenameoftheremote` + +There are plenty of reasons that an operation like a drop can fail, +most of which are not bugs, but some kind of configuration problem, +an inaccessible remote, etc. Most or all of them should come with a +useful error message though.. +"""]] diff --git a/doc/bugs/dropkey_--batch_--json_--force_is_always_succesfull.mdwn b/doc/bugs/dropkey_--batch_--json_--force_is_always_succesfull.mdwn new file mode 100644 index 0000000000..8dcf3b6fc9 --- /dev/null +++ b/doc/bugs/dropkey_--batch_--json_--force_is_always_succesfull.mdwn @@ -0,0 +1,35 @@ +### Please describe the problem. + +just want to check if that is "by design"... --force was demanded seem operation is dangerous, but I expected that it would still report success false if key was not actually dropped, at least if a key is a completely wrong key + +so is that "by design" and I shouldn't care to check success field...? + +### What version of git-annex are you using? On what operating system? +6.20160425+gitgffe2ea2-1~ndall+1 + +### Please provide any additional information below. + +[[!format sh """ + +$> echo MD5E-s11--74d80f7d99b835e5189948c8d4297efd | git annex dropkey --batch --json --force +{"command":"dropkey","key":"MD5E-s11--74d80f7d99b835e5189948c8d4297efd","success":true} + +$> ls -l +total 0 +0 lrwxrwxrwx 1 yoh yoh 110 Apr 29 09:21 124 -> .git/annex/objects/MV/Jw/MD5E-s11--74d80f7d99b835e5189948c8d4297efd/MD5E-s11--74d80f7d99b835e5189948c8d4297efd + +$> echo MD5E-s11--74d80f7d99b835e5189948c8d4297efd | git annex dropkey --batch --json --force +{"command":"dropkey","key":"MD5E-s11--74d80f7d99b835e5189948c8d4297efd","success":true} + +$> echo MD5E-s11--74d80f7dsd99b835e5189948c8d4297efd | git annex dropkey --batch --json --force +{"command":"dropkey","key":"MD5E-s11--74d80f7dsd99b835e5189948c8d4297efd","success":true} + +$> echo MD5E-s11--74d80f7dsd99b835e5189948c8d4297efdsdfsdf | git annex dropkey --batch --json --force +{"command":"dropkey","key":"MD5E-s11--74d80f7dsd99b835e5189948c8d4297efdsdfsdf","success":true} + + +"""]] + +[[!meta author=yoh]] + +> [[notabug|done]] --[[Joey]] diff --git a/doc/bugs/duplicate_progress_reports_in_parallel___39__get__39__.mdwn b/doc/bugs/duplicate_progress_reports_in_parallel___39__get__39__.mdwn new file mode 100644 index 0000000000..21e1d44464 --- /dev/null +++ b/doc/bugs/duplicate_progress_reports_in_parallel___39__get__39__.mdwn @@ -0,0 +1,24 @@ +[[!format sh """ +$> git annex version +git-annex version: 6.20160213+gitg9597a21-1~ndall+1 +... +$> git annex get -J 5 . +get docs/freesurfer.groupanalysis.ppt (from origin...) (checksum...) ok +get docs/freesurfer.future_directions.2007.ppt (from origin...) (checksum...) ok +get docs/freesurfer.groupanalysis.short.ppt (from origin...) (checksum...) ok +get distribution/trctrain/trctraindata.tar.gz (from origin...) +47% 80.2MB/s 3s +47% 80.2MB/s 3s +get docs/freesurfer.inferring_architectonics.ppt (from origin...) +33% 8.1MB/s 4s +33% 8.1MB/s 4s +get docs/freesurfer.intro.2011.ppt (from origin...) +64% 7.3MB/s 1s +64% 7.3MB/s 1s +get docs/freesurfer.intro.mmclass.ppt (from origin...) +# End of transcript or log. +"""]] + +[[!meta author=yoh]] + +> [[fixed|done]] --[[Joey]] diff --git a/doc/bugs/easy_feature_request__58___enable_lan_sync_on_android.mdwn b/doc/bugs/easy_feature_request__58___enable_lan_sync_on_android.mdwn new file mode 100644 index 0000000000..54676b5710 --- /dev/null +++ b/doc/bugs/easy_feature_request__58___enable_lan_sync_on_android.mdwn @@ -0,0 +1,3 @@ +It would be nice to allow LAN sync on the android client. This would allow for easily syncing files when in an out-of-service area, or rapidly syncing to a device with a better network connection. + +> Closing as this was a bug in the deprecated Android app. [[done]] --[[Joey]] diff --git a/doc/bugs/easy_feature_request__58___enable_lan_sync_on_android/comment_1_de6b49a56df51ef872a3f80a3f9f7340._comment b/doc/bugs/easy_feature_request__58___enable_lan_sync_on_android/comment_1_de6b49a56df51ef872a3f80a3f9f7340._comment new file mode 100644 index 0000000000..14a634a011 --- /dev/null +++ b/doc/bugs/easy_feature_request__58___enable_lan_sync_on_android/comment_1_de6b49a56df51ef872a3f80a3f9f7340._comment @@ -0,0 +1,13 @@ +[[!comment format=mdwn + username="joey" + subject="""comment 1""" + date="2016-04-13T18:17:47Z" + content=""" +It would be "easy" except that + +a) Getting a ssh *server* running on Android is not exactly easy. + +b) Local pairing needs multicast to be supported, and IIRC this needs porting + to work with + Android's Bionic libc. +"""]] diff --git a/doc/bugs/easy_feature_request__58___enable_lan_sync_on_android/comment_2_471d2dfee80033072cf8732bb532eca5._comment b/doc/bugs/easy_feature_request__58___enable_lan_sync_on_android/comment_2_471d2dfee80033072cf8732bb532eca5._comment new file mode 100644 index 0000000000..94aa4a8787 --- /dev/null +++ b/doc/bugs/easy_feature_request__58___enable_lan_sync_on_android/comment_2_471d2dfee80033072cf8732bb532eca5._comment @@ -0,0 +1,13 @@ +[[!comment format=mdwn + username="Farhan" + subject="Interested" + date="2016-06-24T03:22:38Z" + content=""" +I would like to help (though I have not enough time to fully make the feature for sure). + +With regards to: + +1) It seems that there is a library which implements an SSH server on Android. I see it [at this repo](https://github.com/barryk/android_external_dropbear) as well as at [this repo](https://github.com/berserker/android_dropbear), both of which are owned by the makers of popular sshd apps on the Google Play Store + +2) Do you mean that that library needs to be ported to Android/Java? If so, I could take a crack at it! +"""]] diff --git a/doc/bugs/easy_feature_request__58___enable_lan_sync_on_android/comment_3_58e569cb01b6e9ce5bd6428ee106cdad._comment b/doc/bugs/easy_feature_request__58___enable_lan_sync_on_android/comment_3_58e569cb01b6e9ce5bd6428ee106cdad._comment new file mode 100644 index 0000000000..28297b2381 --- /dev/null +++ b/doc/bugs/easy_feature_request__58___enable_lan_sync_on_android/comment_3_58e569cb01b6e9ce5bd6428ee106cdad._comment @@ -0,0 +1,14 @@ +[[!comment format=mdwn + username="joey" + subject="""comment 3""" + date="2018-05-08T18:48:11Z" + content=""" +I'm closing this bug report because the git-annex Android app that it +was reported on is now deprecated. Instead, we have +a way to run the regular git-annex build for linux on Android in termux: + + +There were a lot of problems with the way the git-annex Android app was +put together, and I hope this new approach avoids them. If you try it and +still have the bug you reported, please followup and I'll reopen it. +"""]] diff --git a/doc/bugs/encfs_support_--_shouldn__39__t_it_be_treated_as_crippled_already__63__.mdwn b/doc/bugs/encfs_support_--_shouldn__39__t_it_be_treated_as_crippled_already__63__.mdwn new file mode 100644 index 0000000000..a4b4ffa461 --- /dev/null +++ b/doc/bugs/encfs_support_--_shouldn__39__t_it_be_treated_as_crippled_already__63__.mdwn @@ -0,0 +1,92 @@ +### Please describe the problem. + +In the past archived issue ([[https://webcache.googleusercontent.com/search?q=cache:t2N25kWAWEsJ:https://git-annex.branchable.com/bugs/encfs_accused_of_being_crippled/+&cd=1&hl=en&ct=clnk&gl=us]]) you have left a comment + + So, it seems worthwhile to break out lack of hard links support from the other limitations currently lumped into "cripped file system". I've done so. + +did it mean that encfs should be supported without assuming it crippled any longer? + +I have just tried with annex 6.20160425+gitgffe2ea2-1~ndall+1 and it still considers it crippled (FWIW and for the record encfs is 1.8.1-3+b2) + + +[[!format sh """ +$> git annex init --debug +init [2016-04-27 11:55:41.20934] read: git ["--git-dir=.git","--work-tree=.","--literal-pathspecs","show-ref","--hash","refs/heads/git-annex"] +[2016-04-27 11:55:41.214794] process done ExitFailure 1 +[2016-04-27 11:55:41.21487] call: git ["--git-dir=.git","--work-tree=.","--literal-pathspecs","show-ref","--verify","-q","origin/git-annex"] +[2016-04-27 11:55:41.218775] process done ExitFailure 1 +[2016-04-27 11:55:41.219362] read: git ["--git-dir=.git","--work-tree=.","--literal-pathspecs","write-tree"] +[2016-04-27 11:55:41.225289] process done ExitSuccess +[2016-04-27 11:55:41.225379] chat: git ["--git-dir=.git","--work-tree=.","--literal-pathspecs","commit-tree","4b825dc642cb6eb9a060e54bf8d69288fbee4904","--no-gpg-sign"] +[2016-04-27 11:55:41.232881] process done ExitSuccess +[2016-04-27 11:55:41.233028] call: git ["--git-dir=.git","--work-tree=.","--literal-pathspecs","update-ref","refs/heads/git-annex","243e083b39dbf3ae84014dfd9be82d93ab57feb1"] +[2016-04-27 11:55:41.240395] process done ExitSuccess +[2016-04-27 11:55:41.252322] call: git ["--git-dir=.git","--work-tree=.","--literal-pathspecs","config","annex.uuid","75f66b80-d877-4f33-8b32-53dff9ac7da8"] +[2016-04-27 11:55:41.25768] process done ExitSuccess +[2016-04-27 11:55:41.257822] read: git ["config","--null","--list"] +[2016-04-27 11:55:41.263729] process done ExitSuccess + + Detected a crippled filesystem. +[2016-04-27 11:55:41.283605] call: git ["--git-dir=.git","--work-tree=.","--literal-pathspecs","config","annex.crippledfilesystem","true"] +[2016-04-27 11:55:41.288741] process done ExitSuccess +[2016-04-27 11:55:41.289113] read: git ["config","--null","--list"] +[2016-04-27 11:55:41.294897] process done ExitSuccess + + Disabling core.symlinks. +[2016-04-27 11:55:41.295002] call: git ["--git-dir=.git","--work-tree=.","--literal-pathspecs","config","core.symlinks","false"] +[2016-04-27 11:55:41.29995] process done ExitSuccess +[2016-04-27 11:55:41.300036] read: git ["config","--null","--list"] +[2016-04-27 11:55:41.305809] process done ExitSuccess +[2016-04-27 11:55:41.30591] read: git ["--git-dir=.git","--work-tree=.","--literal-pathspecs","show-ref","git-annex"] +[2016-04-27 11:55:41.310985] process done ExitSuccess +[2016-04-27 11:55:41.311082] read: git ["--git-dir=.git","--work-tree=.","--literal-pathspecs","show-ref","--hash","refs/heads/git-annex"] +[2016-04-27 11:55:41.316091] process done ExitSuccess +[2016-04-27 11:55:41.316284] read: git ["--git-dir=.git","--work-tree=.","--literal-pathspecs","log","refs/heads/git-annex..243e083b39dbf3ae84014dfd9be82d93ab57feb1","--pretty=%H","-n1"] +[2016-04-27 11:55:41.322238] process done ExitSuccess +[2016-04-27 11:55:41.323989] chat: git ["--git-dir=.git","--work-tree=.","--literal-pathspecs","cat-file","--batch"] +[2016-04-27 11:55:41.330709] call: git ["--git-dir=.git","--work-tree=.","--literal-pathspecs","config","annex.version","5"] +[2016-04-27 11:55:41.335578] process done ExitSuccess +[2016-04-27 11:55:41.335656] read: git ["config","--null","--list"] +[2016-04-27 11:55:41.340418] process done ExitSuccess + + Enabling direct mode. +[2016-04-27 11:55:41.340519] read: git ["--git-dir=.git","--work-tree=.","--literal-pathspecs","ls-files","--cached","-z","--","."] +[2016-04-27 11:55:41.34474] process done ExitSuccess +[2016-04-27 11:55:41.344828] read: git ["--git-dir=.git","--work-tree=.","--literal-pathspecs","symbolic-ref","-q","HEAD"] +[2016-04-27 11:55:41.349993] process done ExitSuccess +[2016-04-27 11:55:41.352705] read: git ["--git-dir=.git","--work-tree=.","--literal-pathspecs","show-ref","--hash","refs/heads/master"] +[2016-04-27 11:55:41.357245] process done ExitFailure 1 +[2016-04-27 11:55:41.357309] call: git ["--git-dir=.git","--work-tree=.","--literal-pathspecs","checkout","-q","-B","annex/direct/master"] +[2016-04-27 11:55:41.362052] process done ExitSuccess +[2016-04-27 11:55:41.362117] call: git ["--git-dir=.git","--work-tree=.","--literal-pathspecs","config","core.bare","true"] +[2016-04-27 11:55:41.368411] process done ExitSuccess +[2016-04-27 11:55:41.368536] read: git ["config","--null","--list"] +[2016-04-27 11:55:41.373468] process done ExitSuccess +[2016-04-27 11:55:41.373556] call: git ["--git-dir=.git","--work-tree=.","--literal-pathspecs","config","annex.direct","true"] +[2016-04-27 11:55:41.378578] process done ExitSuccess +[2016-04-27 11:55:41.379147] read: git ["config","--null","--list"] +[2016-04-27 11:55:41.38818] process done ExitSuccess +[2016-04-27 11:55:41.388271] read: git ["--git-dir=.git","--work-tree=.","--literal-pathspecs","symbolic-ref","-q","HEAD"] +[2016-04-27 11:55:41.39251] process done ExitSuccess +[2016-04-27 11:55:41.392591] read: git ["--git-dir=.git","--work-tree=.","--literal-pathspecs","show-ref","refs/heads/annex/direct/master"] +[2016-04-27 11:55:41.397179] process done ExitFailure 1 +[2016-04-27 11:55:41.416782] read: uname ["-n"] +[2016-04-27 11:55:41.419832] process done ExitSuccess +ok +[2016-04-27 11:55:41.434629] chat: git ["--git-dir=.git","--work-tree=.","--literal-pathspecs","hash-object","-w","--stdin-paths","--no-filters"] +[2016-04-27 11:55:41.435418] feed: git ["--git-dir=.git","--work-tree=.","--literal-pathspecs","update-index","-z","--index-info"] +[2016-04-27 11:55:41.443936] process done ExitSuccess +[2016-04-27 11:55:41.444038] read: git ["--git-dir=.git","--work-tree=.","--literal-pathspecs","show-ref","--hash","refs/heads/git-annex"] +[2016-04-27 11:55:41.449543] process done ExitSuccess +(recording state in git...) +[2016-04-27 11:55:41.450041] read: git ["--git-dir=.git","--work-tree=.","--literal-pathspecs","write-tree"] +[2016-04-27 11:55:41.460223] process done ExitSuccess +[2016-04-27 11:55:41.46035] chat: git ["--git-dir=.git","--work-tree=.","--literal-pathspecs","commit-tree","a8504dac0ddbd154fff36b704b6d49c6eb573d6c","--no-gpg-sign","-p","refs/heads/git-annex"] +[2016-04-27 11:55:41.467958] process done ExitSuccess +[2016-04-27 11:55:41.468059] call: git ["--git-dir=.git","--work-tree=.","--literal-pathspecs","update-ref","refs/heads/git-annex","c163e24c747d665d51bfa3f13026193956ef1313"] +[2016-04-27 11:55:41.475227] process done ExitSuccess + +"""]] + +[[!meta author=yoh]] + diff --git a/doc/bugs/encfs_support_--_shouldn__39__t_it_be_treated_as_crippled_already__63__/comment_1_a3111068628efcfe8c5dbbe653805481._comment b/doc/bugs/encfs_support_--_shouldn__39__t_it_be_treated_as_crippled_already__63__/comment_1_a3111068628efcfe8c5dbbe653805481._comment new file mode 100644 index 0000000000..98455e1aed --- /dev/null +++ b/doc/bugs/encfs_support_--_shouldn__39__t_it_be_treated_as_crippled_already__63__/comment_1_a3111068628efcfe8c5dbbe653805481._comment @@ -0,0 +1,26 @@ +[[!comment format=mdwn + username="joey" + subject="""comment 1""" + date="2016-04-27T16:59:51Z" + content=""" +git-annex does not these days treat lack of of hard link support as a crippled +filesystem. It just falls back to copying files where it would make hard +links. + +encfs is coming up crippled because it ignores lack of write bits on files; +writing to a mode 444 file on an encfs filesystem by the owner of the file +is allowed. + +That breaks an important safeguard that git-annex relies on; +for example this would work in a non-direct mode repository on an encfs +filesystem, even though file permissions don't allow writing to annexed +file contents: + + joey@darkstar:/tmp/encfs/d> echo corrupt > bar + +So, you're really better off using direct mode on encfs. + +encfs has tons of other problems that make it not work well with +git-annex, and generally insecure. I heartily recommend you reconsider +using it. +"""]] diff --git a/doc/bugs/encryption__61__none_doesn__39__t_work_with_enableremote.mdwn b/doc/bugs/encryption__61__none_doesn__39__t_work_with_enableremote.mdwn new file mode 100644 index 0000000000..991d054933 --- /dev/null +++ b/doc/bugs/encryption__61__none_doesn__39__t_work_with_enableremote.mdwn @@ -0,0 +1,44 @@ +### Please describe the problem. +When cloning a remote client repo via ssh and then enableremote the encryptionsetting seem not be used + + +### What steps will reproduce the problem? +- create a repository (as client) on computer WO +- create a special remote via rsync+ssh on computer BA with encryption=none from WO +--> syncing works +- git clone via ssh from WO on computer XY, group is manual +- git-annex get on XY with source WO workes (OK) +- enableremote BA on computer XY +- shutdown computer WO +- try to get some file on computer XY. The download is first tried from WO, then from BA --> correct + + +- download from BA fails (ERROR) +--> Reason seems to be that encryption=none is not honored: +rsync: change_dir "XXXXXXXX/GPGHMACSHA1--398057f8bd37edf898aeae4557c6277f1162382b" failed: No such file or directory (2) + +I additionally could not find out where to manually change the encryption settings after enableremote + + + +### What version of git-annex are you using? On what operating system? +git-annex version: 5.20140412ubuntu1 +build flags: Assistant Webapp Webapp-secure Pairing Testsuite S3 WebDAV Inotify DBus DesktopNotify XMPP DNS Feeds Quvi TDFA CryptoHash +key/value backends: SHA256E SHA1E SHA512E SHA224E SHA384E SKEIN256E SKEIN512E SHA256 SHA1 SHA512 SHA224 SHA384 SKEIN256 SKEIN512 WORM URL +remote types: git gcrypt S3 bup directory rsync web webdav tahoe glacier hook external +local repository version: 5 +supported repository version: 5 +upgrade supported from repository versions: 0 1 2 4 + + +### Please provide any additional information below. + +[[!format sh """ +# If you can, paste a complete transcript of the problem occurring here. +# If the problem is with the git-annex assistant, paste in .git/annex/daemon.log + + +# End of transcript or log. +"""]] + +[[!tag moreinfo]] diff --git a/doc/bugs/encryption__61__none_doesn__39__t_work_with_enableremote/comment_1_ce62150a3bb8c6be05b9f2ddbe64ae8b._comment b/doc/bugs/encryption__61__none_doesn__39__t_work_with_enableremote/comment_1_ce62150a3bb8c6be05b9f2ddbe64ae8b._comment new file mode 100644 index 0000000000..410849ec5a --- /dev/null +++ b/doc/bugs/encryption__61__none_doesn__39__t_work_with_enableremote/comment_1_ce62150a3bb8c6be05b9f2ddbe64ae8b._comment @@ -0,0 +1,39 @@ +[[!comment format=mdwn + username="joey" + subject="""comment 1""" + date="2015-02-25T18:10:38Z" + content=""" +I cannot reproduce this, and you did not provide any kind of transcript, +which makes it hard to guess where you went wrong. + +Here is a transcript of it working. + +
    +joey@darkstar:~/tmp/1>mkdir /tmp/demo
+joey@darkstar:~/tmp/1>git annex initremote demo type=rsync encryption=none rsyncurl=localhost:/tmp/demo
+initremote demo ok
+(recording state in git...)
+joey@darkstar:~/tmp/1>git annex copy --to demo
+copy foo (checking demo...) (to demo...) 
+sending incremental file list
+f28/
+f28/734/
+f28/734/SHA256E-s30--20c71ffd77c7f25e52416eb6afd36bf8df9410a9942d8a42e7caf74f6cfa5fb8/
+f28/734/SHA256E-s30--20c71ffd77c7f25e52416eb6afd36bf8df9410a9942d8a42e7caf74f6cfa5fb8/SHA256E-s30--20c71ffd77c7f25e52416eb6afd36bf8df9410a9942d8a42e7caf74f6cfa5fb8
+             30 100%    0.00kB/s    0:00:00 (xfr#1, to-chk=0/5)
+ok
+(recording state in git...)
+joey@darkstar:~/tmp/1>cd ..
+joey@darkstar:~/tmp>cd 2
+joey@darkstar:~/tmp/2>git annex sync
+[...]
+joey@darkstar:~/tmp/2>git annex enableremote demo
+enableremote demo ok
+(recording state in git...)
+joey@darkstar:~/tmp/2>git annex get --from demo
+get foo (from demo...) 
+SHA256E-s30--20c71ffd77c7f25e52416eb6afd36bf8df9410a9942d8a42e7caf74f6cfa5fb8
+             30 100%   29.30kB/s    0:00:00 (xfr#1, to-chk=0/1)
+ok
+(recording state in git...)
+"""]]
diff --git a/doc/bugs/error_on_only_repository_copy_deletion.mdwn b/doc/bugs/error_on_only_repository_copy_deletion.mdwn
new file mode 100644
index 0000000000..7c7be888ed
--- /dev/null
+++ b/doc/bugs/error_on_only_repository_copy_deletion.mdwn
@@ -0,0 +1,16 @@
+**What steps will reproduce the problem?**
    
+Delete the last repository in the current repository (where it says: warning type in "yes please do as i say").
+Then try running git annex webapp
+

    
+**What is the expected output? What do you see instead?**
    
+Webapp starts. git-annex: Not in a git repository.
+

    
+**What version of git-annex are you using? On what operating system?**
    
+4.20130417. Debian wheezy/testing
+

    
+**Please provide any additional information below.**
    
+See also
+[[bugs/git-annex:_Not_in_a_git_repository._/]]
+where the same error message occurs
+
+[[!tag /design/assistant moreinfo]]
diff --git a/doc/bugs/error_on_only_repository_copy_deletion/comment_1_af394ac0956ab33a77256bcb02ef2a0f._comment b/doc/bugs/error_on_only_repository_copy_deletion/comment_1_af394ac0956ab33a77256bcb02ef2a0f._comment
new file mode 100644
index 0000000000..a60eee1379
--- /dev/null
+++ b/doc/bugs/error_on_only_repository_copy_deletion/comment_1_af394ac0956ab33a77256bcb02ef2a0f._comment
@@ -0,0 +1,14 @@
+[[!comment format=mdwn
+ username="http://joeyh.name/"
+ nickname="joey"
+ subject="comment 1"
+ date="2013-04-23T16:39:08Z"
+ content="""
+Do you still have a `~/annex` or `~/Desktop/annex`? What's in there?
+
+Also, please show me the content of your `~/.config/git-annex/autostart` file.
+
+The only bug I have been able to track down is that it fails to remove the repository from the autostart file, and so if the repository directory still exists, this will happen. I've fixed it to properly remove the repository from the file, but hesitate to close this bug as I don't understand how the repository directory could still exist after it's supposed to be deleted. 
+
+(When I try, it does get deleted, and starting the webapp again allows starting over with a new repository.)
+"""]]
diff --git a/doc/bugs/external_remote_hGetContents_handle_error.mdwn b/doc/bugs/external_remote_hGetContents_handle_error.mdwn
new file mode 100644
index 0000000000..a3414780fc
--- /dev/null
+++ b/doc/bugs/external_remote_hGetContents_handle_error.mdwn
@@ -0,0 +1,44 @@
+### Please describe the problem.
+
+A follow on from https://git-annex.branchable.com/bugs/Failing_to_execute_bash_remotes_windows/
+
+After noticing your note about it still being possible to run external scripts locally (don't know why I didn't try this!), I tried it. I guess this is related to the reading of the shebang?
+This may be fixed already, so I'm sorry if I'm rehashing things!
+
+
+### What steps will reproduce the problem?
+
+[[!format sh """
+$ git init
+$ git annex init
+$ cp `which git-annex-remote-rclone` .
+$ git annex initremote test type=external externaltype=rclone encryption=none
+
+initremote test
+git-annex: git-annex-remote-rclone: hGetContents: illegal operation (delayed read on closed handle)
+failed
+git-annex: initremote: 1 failed
+"""]]
+
+
+
+### What version of git-annex are you using? On what operating system?
+
+Same as before, windows, git-bash
+
+    git-annex version: 6.20170214-g2233a50
+    build flags: Assistant Webapp Pairing Testsuite S3(multipartupload)(storageclasses) WebDAV ConcurrentOutput TorrentParser Feeds Quvi
+    key/value backends: SHA256E SHA256 SHA512E SHA512 SHA224E SHA224 SHA384E SHA384 SHA3_256E SHA3_256 SHA3_512E SHA3_512 SHA3_224E SHA3_224 SHA3_384E SHA3_384 SKEIN256E SKEIN256  SKEIN512E SKEIN512 SHA1E SHA1 MD5E MD5 WORM URL
+    remote types: git gcrypt p2p S3 bup directory rsync web bittorrent webdav tahoe glacier ddar hook external
+    local repository version: 5
+    supported repository versions: 3 5 6
+    upgrade supported from repository versions: 2 3 4 5
+    operating system: mingw32 i386
+
+
+
+### Have you had any luck using git-annex before? (Sometimes we get tired of reading bug reports all day and a lil' positive end note does wonders)
+
+Thanks for the quick response earlier. I hope this is helpful information. Keep up the great work! :)
+
+> Reproduced this bug, and I've committed a fix. [[done]] --[[Joey]]
diff --git a/doc/bugs/external_special_remote_protocol_broken_by_key_with_spaces.mdwn b/doc/bugs/external_special_remote_protocol_broken_by_key_with_spaces.mdwn
new file mode 100644
index 0000000000..4180678c95
--- /dev/null
+++ b/doc/bugs/external_special_remote_protocol_broken_by_key_with_spaces.mdwn
@@ -0,0 +1,16 @@
+It's possible for a key to contain whitespace in its name. This breaks the
+external special remote protocol, which uses whitespace to separate the key
+parameter from subsequent parameters.
+
+I think that this only causes problems for WORM keys. --[[Joey]]
+
+> Ok, went with my last approach, rather than complicating all special
+> remotes due to this problem, we'll deprecate WORM keys with spaces in
+> their name, and provide a migratipon path.
+> 
+> The error message looks like this:
+
+> > Sorry, this file cannot be stored on an external special remote because its key's name contains a space. To avoid this problem, you can run: git-annex migrate --backend=WORM
+
+> This is fixed as well as is feasible, so while that kind of sucks,
+> calling it [[done]]. --[[Joey]] 
diff --git a/doc/bugs/external_special_remote_protocol_broken_by_key_with_spaces/comment_1_c9095cf16c9b044a4eae6b9b445b2512._comment b/doc/bugs/external_special_remote_protocol_broken_by_key_with_spaces/comment_1_c9095cf16c9b044a4eae6b9b445b2512._comment
new file mode 100644
index 0000000000..611508b6bd
--- /dev/null
+++ b/doc/bugs/external_special_remote_protocol_broken_by_key_with_spaces/comment_1_c9095cf16c9b044a4eae6b9b445b2512._comment
@@ -0,0 +1,60 @@
+[[!comment format=mdwn
+ username="joey"
+ subject="""comment 1"""
+ date="2017-08-15T18:45:21Z"
+ content="""
+It would be kind of nice to not have to worry about keys containing spaces.
+(Only space is allowed currently, not other whitespace.) But that ship has
+long sailed and it would be very ugly to force changing repositories using
+WORM keys with spaces to WORM keys without spaces. About as ugly as
+entirely dropping WORM, which might not be a bad long-term goal.
+
+Here's a few approaches for fixing this in the external special remote protocol:
+
+1. Mangle keys containing spaces as they are sent via the protocol
+   and de-mangle as they're received.
+   Eg, convert ' ' to 's' and convert 's' to 'ss' etc.  
+   (Perhaps pick a better letter than 's' -- '%' and ',' and '&' are
+   already used for other escaping.)  
+2. Make "VERSION 2" be the same as the current protocol, except Key
+   (and other parameters) are quoted in some way.
+3. Make "VERSION 2" be the same as the current protocol, except using NUL
+   rather than space to separate parameters. Perhaps also to separate
+   protocol commands from their parameters, but then "VERSION 2" would
+   be an exception to the rule.
+
+Of course the advantage of #1 is that it doesn't need any modifications to all
+the external special remotes. Such modifications would take quite a while,
+and probably wouldn't even start until a git-annex supporting "VERSION 2"
+was widely available, ie years even to start.
+
+Using NUL is conceptually the simplest implementation, but it may cause
+issues for implementing external special remotes in some languages. Shell
+scripts are probably not good at NULs, for example.
+
+One problem with approach #1 is, what if a key containing a space in the
+protocol happened to already work with the protocol parser in an external
+special remote? This seems at least possible. And so content would already
+be stored on the external special remote under the real key, not the
+mangled key. When git-annex starts mangling the key, retreiving the content
+from the special remote would then fail.
+
+For that to happen, the external special remote would need a parser that
+when it sees "TRANSFER STORE Key File", parses it with File as the last
+word, so supporting multi-word Key. The documentation does currently say
+that "The filename will not contain any whitespace." Actually, I tested it,
+and with a WORM key with spaces, that documentation is generally wrong,
+since the File is based on the Key, it also contains whitespace, most of
+the time. One exception is when using direct mode, if the work tree file
+has been renamed to not contain whitespace.
+
+So, approach #1 seems to call for auditing of external special remotes
+and/or a warning to their implementors about that issue. This would also
+be an opportunity to correct the documentation about spaces in the
+filename, and make sure that they all support that.
+
+Another problem with key mangling is if an external special remote takes
+the mangled key, and passes it to some other git-annex command, like `git annex
+setpresentkey` or something. I seem to remember datalad doing this kind of
+thing.
+"""]]
diff --git a/doc/bugs/external_special_remote_protocol_broken_by_key_with_spaces/comment_2_ed1299a6e12acf728bfa91e60a3706e9._comment b/doc/bugs/external_special_remote_protocol_broken_by_key_with_spaces/comment_2_ed1299a6e12acf728bfa91e60a3706e9._comment
new file mode 100644
index 0000000000..9ef839f1df
--- /dev/null
+++ b/doc/bugs/external_special_remote_protocol_broken_by_key_with_spaces/comment_2_ed1299a6e12acf728bfa91e60a3706e9._comment
@@ -0,0 +1,13 @@
+[[!comment format=mdwn
+ username="joey"
+ subject="""comment 2"""
+ date="2017-08-15T19:40:44Z"
+ content="""
+git-annex could set something in the environment to let the external
+special remote know that it supports version 2. Or, git-annex could
+reply to "VERSION 2" with a new request to indicate it would
+like to use NUL or whatever. That would not break older clients;
+they'd reply with "UNSUPPORTED-REQUEST". 
+
+Of these, the environment variable seems cleaner.
+"""]]
diff --git a/doc/bugs/external_special_remote_protocol_broken_by_key_with_spaces/comment_3_ec5eddd2f345bb8c22168ad34334565e._comment b/doc/bugs/external_special_remote_protocol_broken_by_key_with_spaces/comment_3_ec5eddd2f345bb8c22168ad34334565e._comment
new file mode 100644
index 0000000000..86f38917f0
--- /dev/null
+++ b/doc/bugs/external_special_remote_protocol_broken_by_key_with_spaces/comment_3_ec5eddd2f345bb8c22168ad34334565e._comment
@@ -0,0 +1,17 @@
+[[!comment format=mdwn
+ username="joey"
+ subject="""simpler approach"""
+ date="2017-08-17T18:35:33Z"
+ content="""
+Notice that nooone has ever complained about encountering this problem.
+Not many users are affected. So, a change that prevents the problem
+occuring going forward, without fixing old repositories, is not going to
+bother many users.
+
+* Make `preSanitizeKeyName` escape out spaces, so new WORM keys
+  won't have spaces.
+* Let WORM keys with spaces be able to be migrated to not have spaces,
+  using `git annex migrate`
+* Make the Remotes.External refuse to have anything to do with keys
+  with spaces, and suggest that the user migrate them.
+"""]]
diff --git a/doc/bugs/fails_to_verify_presence_via_http_while_wget_fetches_it_just_fine.mdwn b/doc/bugs/fails_to_verify_presence_via_http_while_wget_fetches_it_just_fine.mdwn
new file mode 100644
index 0000000000..2dc1e641bb
--- /dev/null
+++ b/doc/bugs/fails_to_verify_presence_via_http_while_wget_fetches_it_just_fine.mdwn
@@ -0,0 +1,51 @@
+### Please describe the problem.
+
+Fails to addurl --fast (there is additional problem there anyways since size is not provided by the server -- generated on the fly), but more important that refuses to drop even when addurl/wget fetch it just fine (although different one each time ATM -- probably time stamp within zip etc)
+
+### What steps will reproduce the problem?
+
+just try that url in the pasted output below
+
+### What version of git-annex are you using? On what operating system?
+6.20170525+gitge1cf095ae-1~ndall+1
+
+### Please provide any additional information below.
+
+[[!format sh """
+
+$> git annex whereis mriqcresults.zip 
+whereis mriqcresults.zip (2 copies)  
+  	00000000-0000-0000-0000-000000000001 -- web
+   	8da1dc60-6c81-4bdc-91f6-90f15368797c -- yoh@hopa:/tmp/test [here]
+
+  web: https://openneuro.org/crn/jobs/594d42a090c6d1000180660b/results/fileName?ticket=ticket
+ok
+2 31497.....................................:Wed 26 Jul 2017 03:08:00 PM EDT:.
+hopa:/tmp/test
+$> git annex drop --debug mriqcresults.zip
+[2017-07-26 15:08:11.696997009] read: git ["--git-dir=.git","--work-tree=.","--literal-pathspecs","ls-files","--cached","-z","--","mriqcresults.zip"]
+[2017-07-26 15:08:11.700695374] chat: git ["--git-dir=.git","--work-tree=.","--literal-pathspecs","check-attr","-z","--stdin","annex.backend","annex.numcopies","annex.largefiles","--"]
+[2017-07-26 15:08:11.700910913] read: git ["--version"]
+[2017-07-26 15:08:11.705537628] process done ExitSuccess
+[2017-07-26 15:08:11.705930374] read: git ["--git-dir=.git","--work-tree=.","--literal-pathspecs","show-ref","git-annex"]
+[2017-07-26 15:08:11.709905527] process done ExitSuccess
+[2017-07-26 15:08:11.709986403] read: git ["--git-dir=.git","--work-tree=.","--literal-pathspecs","show-ref","--hash","refs/heads/git-annex"]
+[2017-07-26 15:08:11.713535966] process done ExitSuccess
+[2017-07-26 15:08:11.714440039] chat: git ["--git-dir=.git","--work-tree=.","--literal-pathspecs","cat-file","--batch"]
+[2017-07-26 15:08:11.714790464] chat: git ["--git-dir=.git","--work-tree=.","--literal-pathspecs","cat-file","--batch-check=%(objectname) %(objecttype) %(objectsize)"]
+drop mriqcresults.zip (checking https://openneuro.org/crn/jobs/594d42a090c6d1000180660b/results/fileName?ticket=ticket...) (unsafe) 
+  Could only verify the existence of 0 out of 1 necessary copies
+
+  Rather than dropping this file, try using: git annex move
+
+  (Use --force to override this check, or adjust numcopies.)
+failed
+git-annex: drop: 1 failed
+
+
+"""]]
+
+[[!meta author=yoh]]
+
+
+> [[done]] --[[Joey]]
diff --git a/doc/bugs/fails_to_verify_presence_via_http_while_wget_fetches_it_just_fine/comment_1_fa6649208f1882a6bb412ba40cf57fec._comment b/doc/bugs/fails_to_verify_presence_via_http_while_wget_fetches_it_just_fine/comment_1_fa6649208f1882a6bb412ba40cf57fec._comment
new file mode 100644
index 0000000000..ed23e8902b
--- /dev/null
+++ b/doc/bugs/fails_to_verify_presence_via_http_while_wget_fetches_it_just_fine/comment_1_fa6649208f1882a6bb412ba40cf57fec._comment
@@ -0,0 +1,25 @@
+[[!comment format=mdwn
+ username="joey"
+ subject="""comment 1"""
+ date="2017-08-15T17:28:20Z"
+ content="""
+The normal reason for this to happen is if the size of the file
+on the website has changed. git-annex checks the reported size and if it
+differs from the versioned file, it knows that the website no longer
+contains the same file.
+
+In this case, it seems to be a cgi program generating a zip file, and the
+program actually generated two different zip files when I hit it twice with
+wget. (So if git-annex actually did drop the only copy of the version you
+downloaded, you'd not be able to download it again. Not that git-annex can know
+that; this kind of thing is why trusting the web is not a good idea..) They did
+have the same size, but it looks like the web server is not sending a size
+header anyway.
+
+The actual problem is the web server takes a long time to answer a HEAD request
+for this URL. It takes 35 seconds before curl is able to HEAD it. I suspect
+it's generating the 300 mb zip file before it gets around to finishing
+the HEAD request. Not the greatest server behavior, all around.
+
+That breaks http-client due to its default 30 second timeout. So, will remove that timeout then.
+"""]]
diff --git a/doc/bugs/false_positives_from_fsck_in_bare_repo.mdwn b/doc/bugs/false_positives_from_fsck_in_bare_repo.mdwn
new file mode 100644
index 0000000000..215db11780
--- /dev/null
+++ b/doc/bugs/false_positives_from_fsck_in_bare_repo.mdwn
@@ -0,0 +1,47 @@
+### Please describe the problem.
+
+git annex fsck complains about no known copies of files which seem to be there
+
+### What steps will reproduce the problem?
+
+run git annex fsck in a bare repo? At least I tried 3, two from one set of mirrors, and one from another
+
+### What version of git-annex are you using? On what operating system?
+
+╰─% apt-cache policy git-annex
+git-annex:
+  Installed: 5.20141125
+  Candidate: 5.20141125
+  Version table:
+ *** 5.20141125 0
+        900 http://http.debian.net/debian/ jessie/main amd64 Packages
+
+
+### Please provide any additional information below.
+
+[[!format sh """
+# If you can, paste a complete transcript of the problem occurring here.
+# If the problem is with the git-annex assistant, paste in .git/annex/daemon.log
+$ cd /media/usbdata3/data/audio.git/
+$ tail /tmp/usbdata3.audio.log
+  ** No known copies exist of SHA256E-s22382--ceaa24fd4ef186b90146cbfc48a2da261e85d63288520aa3efd80705f1976117.jpg
+  ** No known copies exist of SHA256E-s6532--4cf24963db72d1c06b03310a83922875bfff4d82b7285096a7bf76ecdba39552.jpg
+  ** No known copies exist of SHA256E-s9539--eb1db62f33125aee2093ecf530f94497f8c890b3ffddb5214cbd2ad99ce5a4c4.jpg
+  ** No known copies exist of SHA256E-s13710--5d69a49a290012f480e6848f88122e90f803177c610524234e16136d95bc7715.jpg
+  ** No known copies exist of SHA256E-s3515--3a225ae6b26571a27d7e70f49948e2be4d78f1fb29e64308880e208e20bc2868.jpg
+  ** No known copies exist of SHA256E-s8359--9291d7b0bf3a862901607ec7c56d5e1ee6a0f3889e775e4db23999b38322e83f.jpg
+  ** No known copies exist of SHA256E-s3157--4c43cf5618f939a09dada21b45b3860bcb4c6968fa166daf3215f879a25e504d.gif
+  ** No known copies exist of SHA256E-s2676--13b4b08525ce90b44ea9eb703630646f2d8f41d9a07b65d130aaf653a072d408.gif
+  ** No known copies exist of SHA256E-s5863--f0eb8c34ea1aa834280c78f1ec28d50aef81379ba62fb0bbf084664c7a7e2746.jpg
+git-annex: fsck: 10484 failed
+$ find . -name SHA256E-s5863--f0eb8c34ea1aa834280c7\*
+./annex/objects/5b3/5c3/SHA256E-s5863--f0eb8c34ea1aa834280c78f1ec28d50aef81379ba62fb0bbf084664c7a7e2746.jpg
+./annex/objects/5b3/5c3/SHA256E-s5863--f0eb8c34ea1aa834280c78f1ec28d50aef81379ba62fb0bbf084664c7a7e2746.jpg/SHA256E-s5863--f0eb8c34ea1aa834280c78f1ec28d50aef81379ba62fb0bbf084664c7a7e2746.jpg
+$ sha256sum ./annex/objects/5b3/5c3/SHA256E-s5863--f0eb8c34ea1aa834280c78f1ec28d50aef81379ba62fb0bbf084664c7a7e2746.jpg/SHA256E-s5863--f0eb8c34ea1aa834280c78f1ec28d50aef81379ba62fb0bbf084664c7a7e2746.jpg
+f0eb8c34ea1aa834280c78f1ec28d50aef81379ba62fb0bbf084664c7a7e2746  ./annex/objects/5b3/5c3/SHA256E-s5863--f0eb8c34ea1aa834280c78f1ec28d50aef81379ba62fb0bbf084664c7a7e2746.jpg/SHA256E-s5863--f0eb8c34ea1aa834280c78f1ec28d50aef81379ba62fb0bbf084664c7a7e2746.jpg
+$ 
+
+# End of transcript or log.
+"""]]
+
+[[!tag moreinfo]]
diff --git a/doc/bugs/false_positives_from_fsck_in_bare_repo/comment_1_e13d7e142e4f3c33afb1e445a1a4c093._comment b/doc/bugs/false_positives_from_fsck_in_bare_repo/comment_1_e13d7e142e4f3c33afb1e445a1a4c093._comment
new file mode 100644
index 0000000000..e369bb3d47
--- /dev/null
+++ b/doc/bugs/false_positives_from_fsck_in_bare_repo/comment_1_e13d7e142e4f3c33afb1e445a1a4c093._comment
@@ -0,0 +1,29 @@
+[[!comment format=mdwn
+ username="joey"
+ subject="""comment 1"""
+ date="2015-02-25T18:22:00Z"
+ content="""
+I tried to reproduce this by eg, making a bare repo that contained
+the only known copy of a file. No problem.
+
+
    +joey@darkstar:~/tmp/2.git>git annex fsck
+(merging synced/git-annex into git-annex...)
+fsck
+SHA256E-s30--20c71ffd77c7f25e52416eb6afd36bf8df9410a9942d8a42e7caf74f6cfa5fb8 (checksum...)
+ok
+
    
+
+I then purposefully told git-annex the file was not located in the bare
+repo, even though it was there, fscked again, and it had no trouble
+fixing the location log to reflect reality:
+
+
    +joey@darkstar:~/tmp/2.git>git annex fsck
+fsck SHA256E-s30--20c71ffd77c7f25e52416eb6afd36bf8df9410a9942d8a42e7caf74f6cfa5fb8 (fixing location log) (checksum...)
+ok
+(recording state in git...)
+
    
+
+You will need to investigate further why git-annex is not seeing the file.
+"""]]
diff --git a/doc/bugs/fat_support.mdwn b/doc/bugs/fat_support.mdwn
new file mode 100644
index 0000000000..70ee3b369c
--- /dev/null
+++ b/doc/bugs/fat_support.mdwn
@@ -0,0 +1,13 @@
+Klaus pointed out that there are two problems that keep
+git-annex from being used on USB keys, that would typically
+be VFAT formatted:
+
+- Use of symlinks, which VFAT does not support. Very hard to fix.
+  Instead, just use [[/bare_repositories]] on the key,
+  they're supported now.
+- Use of ":" in filenames of object files, also not supported.
+  Could easily be fixed by reorganizing the object directory.
+
+[[Done]]; in annex.version 2 repos, colons are entirely avoided in
+filenames. So a bare git clone can be put on VFAT, and git-annex
+used to move stuff --to and --from it, for sneakernet.
diff --git a/doc/bugs/fat_support/comment_1_04bcc4795d431e8cb32293aab29bbfe2._comment b/doc/bugs/fat_support/comment_1_04bcc4795d431e8cb32293aab29bbfe2._comment
new file mode 100644
index 0000000000..510e449842
--- /dev/null
+++ b/doc/bugs/fat_support/comment_1_04bcc4795d431e8cb32293aab29bbfe2._comment
@@ -0,0 +1,12 @@
+[[!comment format=mdwn
+ username="fmarier"
+ ip="121.73.248.43"
+ subject="Exporting to a FAT filesystem?"
+ date="2011-04-04T07:40:41Z"
+ content="""
+I'm using git-annex to keep my music in sync between all of my different machines. What I'd love to be able to do is to also keep it in sync with my iRiver player. Unfortunately, the firmware, Rockbox, doesn't support ext3, so I'm stuck with a FAT filesystem.
+
+I can see how the design of git-annex makes it rather difficult to get rid of the symlinks, so how about taking a different approach: something like a \"git annex export DEST\" which would take a destination (not a git remote) and rsync the content over to there as regular files.
+
+Maybe \"git annex sync DEST\" or \"git annex rsync DEST\" would be better names if we want to convey the idea that the destination will be made to look like the source repo, including performing the necessary deletions.
+"""]]
diff --git a/doc/bugs/fat_support/comment_2_bb4a97ebadb5c53809fc78431eabd7c8._comment b/doc/bugs/fat_support/comment_2_bb4a97ebadb5c53809fc78431eabd7c8._comment
new file mode 100644
index 0000000000..7618c9a7b6
--- /dev/null
+++ b/doc/bugs/fat_support/comment_2_bb4a97ebadb5c53809fc78431eabd7c8._comment
@@ -0,0 +1,14 @@
+[[!comment format=mdwn
+ username="http://joey.kitenet.net/"
+ nickname="joey"
+ subject="comment 2"
+ date="2011-04-04T18:20:45Z"
+ content="""
+Hey @fmarier. Well, this bug report is closed because you can already get rid of the symlinks. Just put a bare git repo on your fat filesystem, and use git-annex copy --to/--from there.
+
+Now, that puts all the files that are on the device in .git/annex/objects/xx/yy/blah.mp3 -- how well rockbox would support that I don't know. And if it tries to modify or delete those files, git annex also can't help you manage those changes.
+
+Another recent option is the [[special_remotes/directory]] special remote type, which again uses \"xx/yy/blah.mp3\" and can't track changes made to the files. This could perhaps be extended in the direction you suggest, although trying to fit this into the special remote infrastructure might not be a good fit really.
+
+The most likely way this has to get dealt with is really by using [[todo/smudge]] filters, which would eliminate the symlinks and allow copying a non-bare git repo onto vfat. 
+"""]]
diff --git a/doc/bugs/fat_support/comment_3_df3b943bc1081a8f3f7434ae0c8e061e._comment b/doc/bugs/fat_support/comment_3_df3b943bc1081a8f3f7434ae0c8e061e._comment
new file mode 100644
index 0000000000..f3db75c2f6
--- /dev/null
+++ b/doc/bugs/fat_support/comment_3_df3b943bc1081a8f3f7434ae0c8e061e._comment
@@ -0,0 +1,11 @@
+[[!comment format=mdwn
+ username="fmarier"
+ subject="comment 3"
+ date="2011-04-05T10:00:21Z"
+ content="""
+Thanks for the reply @joey.
+
+While it would certainly be possible for a bare repo to exist on my iRiver, the problem is that the music player uses the filesystem to organize files into directories like \"Artist/Album/Track.ogg\". So replacing that with \"..../xx/yy/Track.ogg\" would make it fairly difficult to browse my music collection and select the album/track I want to listen to :)
+
+So unless I have the files physically organized like the symlinks, then it's probably not going to work very for that particular workflow. Smudge filters are interesting though. In the meantime, I'll look into rsyncing from another box which has the right filesystem layout onto my iRiver directly.
+"""]]
diff --git a/doc/bugs/fat_support/comment_4_90a8a15bedd94480945a374f9d706b86._comment b/doc/bugs/fat_support/comment_4_90a8a15bedd94480945a374f9d706b86._comment
new file mode 100644
index 0000000000..722cbdd9e7
--- /dev/null
+++ b/doc/bugs/fat_support/comment_4_90a8a15bedd94480945a374f9d706b86._comment
@@ -0,0 +1,10 @@
+[[!comment format=mdwn
+ username="http://ethan.betacantrips.com/"
+ nickname="ethan.glasser.camp"
+ subject="no symlinks"
+ date="2011-06-08T20:59:38Z"
+ content="""
+If you try to clone a git repo that has a symlink over to a VFAT filesystem, you get (in its place) a regular file that contains the name of the symlink target.  So why can't git-annex use that?  I could still do git annex get on this file, git annex would still \"know\" that it's a symlink, and could replace it with a copy of the real file (instead of putting it in .git/annex).
+
+I know if it were that simple, someone would have done it already, so what am I missing?  I guess trying to get the file FROM the repository would fail because it wouldn't find the file in .git/annex?  Couldn't you store a reverse mapping?  You wouldn't be able to move the file around, but you already lose that once you give up symlinks.  It would also be a little harder to tell which symlinks were \"dangling\"; I don't see an easy way to get around that.  It would still be better than a bare repo..
+"""]]
diff --git a/doc/bugs/fat_support/comment_5_64bbf89de0836673224b83fdefa0407b._comment b/doc/bugs/fat_support/comment_5_64bbf89de0836673224b83fdefa0407b._comment
new file mode 100644
index 0000000000..1063b0f910
--- /dev/null
+++ b/doc/bugs/fat_support/comment_5_64bbf89de0836673224b83fdefa0407b._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="http://joey.kitenet.net/"
+ nickname="joey"
+ subject="comment 5"
+ date="2011-06-10T16:41:43Z"
+ content="""
+@ethan the reason that wouldn't work is because git would then see a file that was checked in and had its one line symlinkish content replaced with a huge binary blob. And git commit would try to commit that etc. The potential for foot-shooting is too high.
+"""]]
diff --git a/doc/bugs/fat_support/comment_6_a3b6000330c9c376611c228d746a1d55._comment b/doc/bugs/fat_support/comment_6_a3b6000330c9c376611c228d746a1d55._comment
new file mode 100644
index 0000000000..c7defd13ef
--- /dev/null
+++ b/doc/bugs/fat_support/comment_6_a3b6000330c9c376611c228d746a1d55._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="https://www.google.com/accounts/o8/id?id=AItOawkZRoTRyW3tox-FD2DQWxskgI6_tkEtHL4"
+ nickname="Ben"
+ subject="comment 6"
+ date="2012-07-23T16:11:52Z"
+ content="""
+The above would work fine for me but the files in my annex (e.g. .git/annex/objects/xx/yy/blah.ogg) don't have extensions like that, so my media player doesn't recognize them as media files. How do I get the files under \"objects\" to keep the extensions of the original files like in Joey's example?
+"""]]
diff --git a/doc/bugs/fat_support/comment_7_a0ac7f2c44efc8116940c7b94b35e9d0._comment b/doc/bugs/fat_support/comment_7_a0ac7f2c44efc8116940c7b94b35e9d0._comment
new file mode 100644
index 0000000000..11668615e6
--- /dev/null
+++ b/doc/bugs/fat_support/comment_7_a0ac7f2c44efc8116940c7b94b35e9d0._comment
@@ -0,0 +1,7 @@
+[[!comment format=mdwn
+ username="http://joeyh.name/"
+ subject="comment 7"
+ date="2012-07-24T14:51:50Z"
+ content="""
+You can get the extensions by migrating to the SHA1E (or SHA256E) backend. 
+"""]]
diff --git a/doc/bugs/fat_support/comment_8_acc947643a635eb10a1bff92083a3506._comment b/doc/bugs/fat_support/comment_8_acc947643a635eb10a1bff92083a3506._comment
new file mode 100644
index 0000000000..558e0ca105
--- /dev/null
+++ b/doc/bugs/fat_support/comment_8_acc947643a635eb10a1bff92083a3506._comment
@@ -0,0 +1,10 @@
+[[!comment format=mdwn
+ username="https://www.google.com/accounts/o8/id?id=AItOawmcYryijvlF8bJvM_eZNSrUPEkMlxMDGTQ"
+ nickname="Thiago"
+ subject="POSIX layer on top of VFAT using FUSE"
+ date="2012-11-24T00:21:23Z"
+ content="""
+I just found out about this project and didn't try it, but it looks like it would allow using git-annex on an usb stick with a normal repository:
+
+
+"""]]
diff --git a/doc/bugs/fatal__58___Cannot_handle_files_this_big.mdwn b/doc/bugs/fatal__58___Cannot_handle_files_this_big.mdwn
new file mode 100644
index 0000000000..7272bfc294
--- /dev/null
+++ b/doc/bugs/fatal__58___Cannot_handle_files_this_big.mdwn
@@ -0,0 +1,96 @@
+### Please describe the problem.
+
+Syncing a 20GB video file causes this error. I have no problems with 8GB files. 
+
+### What steps will reproduce the problem?
+
+See additional info
+
+### What version of git-annex are you using? On what operating system?
+
+git-annex version: 5.20140920-gb0c4300
+build flags: Assistant Webapp Webapp-secure Pairing Testsuite S3 WebDAV DNS Feeds Quvi TDFA CryptoHash
+key/value backends: SHA256E SHA1E SHA512E SHA224E SHA384E SKEIN256E SKEIN512E SHA256 SHA1 SHA512 SHA224 SHA384 SKEIN256 SKEIN512 WORM URL
+remote types: git gcrypt S3 bup directory rsync web webdav tahoe glacier ddar hook external
+local repository version: 5
+supported repository version: 5
+upgrade supported from repository versions: 2 3 4
+
+git version 1.9.4.msysgit.2
+
+Windows 7 64bit 
+
+### Please provide any additional information below.
+
+[[!format sh """
+
+Z:\>git clone L:\repositories\bigFilesTest.git-annex
+Cloning into 'bigFilesTest.git-annex'...
+done.
+
+Z:\>cd bigFilesTest.git-annex
+
+Z:\bigFilesTest.git-annex>git annex init "cloned"
+init cloned
+  Detected a filesystem without fifo support.
+
+  Disabling ssh connection caching.
+
+  Detected a crippled filesystem.
+
+  Enabling direct mode.
+ok
+(Recording state in git...)
+
+Z:\bigFilesTest.git-annex>git annex add test20GBVideo.mkv
+add test20GBVideo.mkv ok
+(Recording state in git...)
+
+Z:\bigFilesTest.git-annex>git annex sync --debug
+[2014-10-18 15:39:02 Mitteleuropäische Sommerzeit] read: git ["--git-dir=Z:\\bigFilesTest.git-annex\\.git","--work-tree=Z:\\bigFilesTest.git-annex","-c","core.bare=false","show-ref","git-annex"]
+[2014-10-18 15:39:02 Mitteleuropäische Sommerzeit] read: git ["--git-dir=Z:\\bigFilesTest.git-annex\\.git","--work-tree=Z:\\bigFilesTest.git-annex","-c","core.bare=false","show-ref","--hash","refs/heads/git-annex"]
+[2014-10-18 15:39:02 Mitteleuropäische Sommerzeit] read: git ["--git-dir=Z:\\bigFilesTest.git-annex\\.git","--work-tree=Z:\\bigFilesTest.git-annex","-c","core.bare=false","log","refs/heads/git-annex..54de336a3423f7f8f72f897effd29f952534c24e","-n1","--pretty=%H"]
+[2014-10-18 15:39:02 Mitteleuropäische Sommerzeit] read: git ["--git-dir=Z:\\bigFilesTest.git-annex\\.git","--work-tree=Z:\\bigFilesTest.git-annex","-c","core.bare=false","log","refs/heads/git-annex..53cfcf38b40247b3992b6007336b2c915a945ad4","-n1","--pretty=%H"]
+[2014-10-18 15:39:02 Mitteleuropäische Sommerzeit] chat: git ["--git-dir=Z:\\bigFilesTest.git-annex\\.git","--work-tree=Z:\\bigFilesTest.git-annex","-c","core.bare=false","cat-file","--batch"]
+[2014-10-18 15:39:02 Mitteleuropäische Sommerzeit] read: git ["config","--null","--list"]
+commit  [2014-10-18 15:39:02 Mitteleuropäische Sommerzeit] read: git ["--git-dir=Z:\\bigFilesTest.git-annex\\.git","--work-tree=Z:\\bigFilesTest.git-annex","-c","core.bare=false","ls-files","--stage","-z","--others","--exclude-standard","--","Z:\\bigFilesTest.git-annex"]
+[2014-10-18 15:39:02 Mitteleuropäische Sommerzeit] chat: git ["--git-dir=Z:\\bigFilesTest.git-annex\\.git","--work-tree=Z:\\bigFilesTest.git-annex","-c","core.bare=false","cat-file","--batch"]
+(Recording state in git...)
+[2014-10-18 15:39:03 Mitteleuropäische Sommerzeit] call: git ["--git-dir=Z:\\bigFilesTest.git-annex\\.git","--work-tree=Z:\\bigFilesTest.git-annex","-c","core.bare=false","add","-f","test20GBVideo.mkv"]
+fatal: Cannot handle files this big
+[2014-10-18 15:39:03 Mitteleuropäische Sommerzeit] read: git ["--git-dir=Z:\\bigFilesTest.git-annex\\.git","--work-tree=Z:\\bigFilesTest.git-annex","-c","core.bare=false","show-ref","--head"]
+[2014-10-18 15:39:03 Mitteleuropäische Sommerzeit] read: git ["--git-dir=Z:\\bigFilesTest.git-annex\\.git","--work-tree=Z:\\bigFilesTest.git-annex","-c","core.bare=false","diff-index","-z","--raw","--no-renames","-l0","--cached","HEAD"]
+[2014-10-18 15:39:03 Mitteleuropäische Sommerzeit] read: git ["--git-dir=Z:\\bigFilesTest.git-annex\\.git","--work-tree=Z:\\bigFilesTest.git-annex","-c","core.bare=false","symbolic-ref","HEAD"]
+[2014-10-18 15:39:03 Mitteleuropäische Sommerzeit] read: git ["--git-dir=Z:\\bigFilesTest.git-annex\\.git","--work-tree=Z:\\bigFilesTest.git-annex","-c","core.bare=false","show-ref","--hash","refs/heads/annex/direct/master"]
+[2014-10-18 15:39:03 Mitteleuropäische Sommerzeit] read: git ["--git-dir=Z:\\bigFilesTest.git-annex\\.git","--work-tree=Z:\\bigFilesTest.git-annex","-c","core.bare=false","write-tree"]
+[2014-10-18 15:39:03 Mitteleuropäische Sommerzeit] read: git ["--git-dir=Z:\\bigFilesTest.git-annex\\.git","--work-tree=Z:\\bigFilesTest.git-annex","-c","core.bare=false","rev-parse","b12e8477242df97be13c1395db143f860ce8e895:"]
+ok
+[2014-10-18 15:39:03 Mitteleuropäische Sommerzeit] read: git ["--git-dir=Z:\\bigFilesTest.git-annex\\.git","--work-tree=Z:\\bigFilesTest.git-annex","-c","core.bare=false","symbolic-ref","HEAD"]
+[2014-10-18 15:39:03 Mitteleuropäische Sommerzeit] read: git ["--git-dir=Z:\\bigFilesTest.git-annex\\.git","--work-tree=Z:\\bigFilesTest.git-annex","-c","core.bare=false","show-ref","refs/heads/annex/direct/master"]
+[2014-10-18 15:39:03 Mitteleuropäische Sommerzeit] call: git ["--git-dir=Z:\\bigFilesTest.git-annex\\.git","--work-tree=Z:\\bigFilesTest.git-annex","-c","core.bare=false","show-ref","--verify","-q","refs/heads/synced/master"]
+[2014-10-18 15:39:03 Mitteleuropäische Sommerzeit] read: git ["--git-dir=Z:\\bigFilesTest.git-annex\\.git","--work-tree=Z:\\bigFilesTest.git-annex","-c","core.bare=false","log","refs/heads/annex/direct/master..refs/heads/synced/master","-n1","--pretty=%H"]
+pull origin
+[2014-10-18 15:39:03 Mitteleuropäische Sommerzeit] call: git ["--git-dir=Z:\\bigFilesTest.git-annex\\.git","--work-tree=Z:\\bigFilesTest.git-annex","-c","core.bare=false","fetch","origin"]
+[2014-10-18 15:39:03 Mitteleuropäische Sommerzeit] call: git ["--git-dir=Z:\\bigFilesTest.git-annex\\.git","--work-tree=Z:\\bigFilesTest.git-annex","-c","core.bare=false","show-ref","--verify","-q","refs/remotes/origin/annex/direct/master"]
+[2014-10-18 15:39:03 Mitteleuropäische Sommerzeit] call: git ["--git-dir=Z:\\bigFilesTest.git-annex\\.git","--work-tree=Z:\\bigFilesTest.git-annex","-c","core.bare=false","show-ref","--verify","-q","refs/remotes/origin/synced/master"]
+[2014-10-18 15:39:03 Mitteleuropäische Sommerzeit] read: git ["--git-dir=Z:\\bigFilesTest.git-annex\\.git","--work-tree=Z:\\bigFilesTest.git-annex","-c","core.bare=false","log","refs/heads/synced/master..refs/remotes/origin/synced/master","-n1","--pretty=%H"]
+ok
+[2014-10-18 15:39:03 Mitteleuropäische Sommerzeit] read: git ["--git-dir=Z:\\bigFilesTest.git-annex\\.git","--work-tree=Z:\\bigFilesTest.git-annex","-c","core.bare=false","show-ref","git-annex"]
+[2014-10-18 15:39:03 Mitteleuropäische Sommerzeit] read: git ["--git-dir=Z:\\bigFilesTest.git-annex\\.git","--work-tree=Z:\\bigFilesTest.git-annex","-c","core.bare=false","show-ref","--hash","refs/heads/git-annex"]
+[2014-10-18 15:39:03 Mitteleuropäische Sommerzeit] read: git ["--git-dir=Z:\\bigFilesTest.git-annex\\.git","--work-tree=Z:\\bigFilesTest.git-annex","-c","core.bare=false","log","refs/heads/git-annex..54de336a3423f7f8f72f897effd29f952534c24e","-n1","--pretty=%H"]
+[2014-10-18 15:39:03 Mitteleuropäische Sommerzeit] read: git ["--git-dir=Z:\\bigFilesTest.git-annex\\.git","--work-tree=Z:\\bigFilesTest.git-annex","-c","core.bare=false","log","refs/heads/git-annex..53cfcf38b40247b3992b6007336b2c915a945ad4","-n1","--pretty=%H"]
+[2014-10-18 15:39:03 Mitteleuropäische Sommerzeit] call: git ["--git-dir=Z:\\bigFilesTest.git-annex\\.git","--work-tree=Z:\\bigFilesTest.git-annex","-c","core.bare=false","branch","-f","synced/master"]
+[2014-10-18 15:39:03 Mitteleuropäische Sommerzeit] call: git ["--git-dir=Z:\\bigFilesTest.git-annex\\.git","--work-tree=Z:\\bigFilesTest.git-annex","-c","core.bare=false","branch","-f","master"]
+[2014-10-18 15:39:03 Mitteleuropäische Sommerzeit] call: git ["--git-dir=Z:\\bigFilesTest.git-annex\\.git","--work-tree=Z:\\bigFilesTest.git-annex","-c","core.bare=false","show-ref","--verify","-q","refs/remotes/origin/synced/master"]
+[2014-10-18 15:39:03 Mitteleuropäische Sommerzeit] read: git ["--git-dir=Z:\\bigFilesTest.git-annex\\.git","--work-tree=Z:\\bigFilesTest.git-annex","-c","core.bare=false","log","refs/remotes/origin/synced/master..refs/heads/synced/master","-n1","--pretty=%H"]
+[2014-10-18 15:39:03 Mitteleuropäische Sommerzeit] call: git ["--git-dir=Z:\\bigFilesTest.git-annex\\.git","--work-tree=Z:\\bigFilesTest.git-annex","-c","core.bare=false","show-ref","--verify","-q","refs/remotes/origin/git-annex"]
+[2014-10-18 15:39:03 Mitteleuropäische Sommerzeit] read: git ["--git-dir=Z:\\bigFilesTest.git-annex\\.git","--work-tree=Z:\\bigFilesTest.git-annex","-c","core.bare=false","log","refs/remotes/origin/git-annex..git-annex","-n1","--pretty=%H"]
+push origin
+[2014-10-18 15:39:03 Mitteleuropäische Sommerzeit] call: git ["--git-dir=Z:\\bigFilesTest.git-annex\\.git","--work-tree=Z:\\bigFilesTest.git-annex","-c","core.bare=false","push","origin","+git-annex:synced/git-annex","annex/direct/master:synced/master"]
+Everything up-to-date
+[2014-10-18 15:39:03 Mitteleuropäische Sommerzeit] read: git ["--git-dir=Z:\\bigFilesTest.git-annex\\.git","--work-tree=Z:\\bigFilesTest.git-annex","-c","core.bare=false","push","origin","master"]
+ok
+
+
+
+"""]]
diff --git a/doc/bugs/fatal__58___Cannot_handle_files_this_big/comment_1_e4f03a86a7adc6c5421d1e70c37659e0._comment b/doc/bugs/fatal__58___Cannot_handle_files_this_big/comment_1_e4f03a86a7adc6c5421d1e70c37659e0._comment
new file mode 100644
index 0000000000..b580cdbe68
--- /dev/null
+++ b/doc/bugs/fatal__58___Cannot_handle_files_this_big/comment_1_e4f03a86a7adc6c5421d1e70c37659e0._comment
@@ -0,0 +1,17 @@
+[[!comment format=mdwn
+ username="joey"
+ subject="""comment 1"""
+ date="2015-02-10T17:29:44Z"
+ content="""
+I recently fixed a bug that would
+likewise cause git-annex to fail on windows with a large file, although I
+didn't see it failing with this message.
+
+There's a good chance that git-annex version 5.20150205 might fix this
+problem. 
+
+I was able to successfully use git-annex with a 6 gb file, after the fixes.
+
+Please check, and followup. I'm closing this bug provisionally in the
+meantime.
+"""]]
diff --git a/doc/bugs/fatal__58___Cannot_handle_files_this_big/comment_2_546782c644230741470f9a9de23bd019._comment b/doc/bugs/fatal__58___Cannot_handle_files_this_big/comment_2_546782c644230741470f9a9de23bd019._comment
new file mode 100644
index 0000000000..0ea9dc4d1d
--- /dev/null
+++ b/doc/bugs/fatal__58___Cannot_handle_files_this_big/comment_2_546782c644230741470f9a9de23bd019._comment
@@ -0,0 +1,24 @@
+[[!comment format=mdwn
+ username="bvaa"
+ subject="similar problem"
+ date="2016-03-01T08:12:27Z"
+ content="""
+I have a similar problem on Windows 7 64bit trying to add files that are around 5GB in size. I tried repository version 5 and 6 with same results.
+
+```
+$ git annex add bigfile
+add bigfile ok
+(recording state in git...)
+
+$ git annex status
+fatal: Cannot handle files this big
+```
+git-annex version: 6.20160229-g37a89cc
+build flags: Assistant Webapp Pairing Testsuite S3(multipartupload) WebDAV ConcurrentOutput TorrentParser Feeds Quvi
+key/value backends: SHA256E SHA256 SHA512E SHA512 SHA224E SHA224 SHA384E SHA384 SHA3_256E SHA3_256 SHA3_512E SHA3_512 SHA3_224E SHA3_224 SHA3_384E SHA3_384 SKEIN256E SKEIN256 SKEIN512E SKEIN512 SHA1E SHA1 MD5E MD5 WORM URL
+remote types: git gcrypt S3 bup directory rsync web bittorrent webdav tahoe glacier ddar hook external
+local repository version: 5
+supported repository versions: 5 6
+upgrade supported from repository versions: 2 3 4 5
+
+"""]]
diff --git a/doc/bugs/fatal__58___Cannot_handle_files_this_big/comment_3_151e7cf96c7d168e1397d111aa47f279._comment b/doc/bugs/fatal__58___Cannot_handle_files_this_big/comment_3_151e7cf96c7d168e1397d111aa47f279._comment
new file mode 100644
index 0000000000..e6ad551e42
--- /dev/null
+++ b/doc/bugs/fatal__58___Cannot_handle_files_this_big/comment_3_151e7cf96c7d168e1397d111aa47f279._comment
@@ -0,0 +1,20 @@
+[[!comment format=mdwn
+ username="joey"
+ subject="""comment 3"""
+ date="2016-03-01T14:41:45Z"
+ content="""
+git (not git-annex) will throw this error if a file size is greater than
+`size_t`.
+
+This bug report seemed to originally concern git add being run on such a
+file, but I can't see how git-annex would do that, it doesn't add large
+files to git.
+
+I think that in the case of git-annex status, when it runs git status, that
+looks at work tree files, and so falls over if they're large, even if
+what's checked into git is a nice small git-annex symlink. This would also
+probably affect other places where git looks at worktree files, perhaps git
+diff (in v6 repo mode).
+
+Reopening bug report.
+"""]]
diff --git a/doc/bugs/fatal__58___git-write-tree__58___error_building_trees.mdwn b/doc/bugs/fatal__58___git-write-tree__58___error_building_trees.mdwn
new file mode 100644
index 0000000000..c3d205faf4
--- /dev/null
+++ b/doc/bugs/fatal__58___git-write-tree__58___error_building_trees.mdwn
@@ -0,0 +1,110 @@
+### Please describe the problem.
+Not able to successfully git-annex sync with a remote due to a git fatal. Caused by masters diverging?
+
+### What steps will reproduce the problem?
+git-annex sync, or, letting the assistant try.
+
+### What version of git-annex are you using? On what operating system?
+git-annex version: 5.20131221+b1 on my laptop
+git-annex version: 5.20131224-g6ca5271 on the remote server
+
+### Please provide any additional information below.
+
+Output of a manual git-annex sync in the directory:
+
+[[!format sh """
+greg@x200s:~/Documents$ git-annex sync
+commit  (Recording state in git...)
+Copyright Office/Orphan Works/ARROW/170409_ARROW_Leaflet.pdf: unmerged (783afced6bc43138373fda43edfda0c33be36525)
+Copyright Office/Orphan Works/ARROW/ARROWproject_results1.pdf: unmerged (b536e5f3d93e7905e05510f26db1f743e9eae16e)
+Copyright Office/Orphan Works/ARROW/ARROWproject_results1.ppt: unmerged (5543049b8940cc5702d37aff18b03c67d9c8374d)
+Copyright Office/Orphan Works/ARROW/ARROWstandardPresent2010.pdf: unmerged (54d751bc98cb5da29d3d568856b74675e842072e)
+Copyright Office/Orphan Works/ARROW/ARROWstandardPresent2010.ppt: unmerged (efe0e94b51eccb9a6a0c352f4a210bd5a6105050)
+Copyright Office/Orphan Works/ARROW/ARROWtrifoldMAR2011.pdf: unmerged (b52ff16178e29261fe00a518c23610a3b0826482)
+Copyright Office/Orphan Works/Documentation/20110531/Documentation.doc.odt: unmerged (1348d5f42f7e34706407f7936f4fb0438e4b8ffa)
+Copyright Office/Orphan Works/Documentation/AAPpublishers.pdf: unmerged (3f448a03d31a38adb095e3031e4ee13771d22d70)
+Copyright Office/Orphan Works/Documentation/Documentation.doc: unmerged (265fdff7787f560e3ba20789a12e15ffb165ec7f)
+Copyright Office/Orphan Works/Documentation/Documentation.pdf: unmerged (7a9ff92663ed42b42b9baaefaf4721499d18d82d)
+...
+fatal: git-write-tree: error building trees
+git-annex: failed to read sha from git write-tree
+"""]]
+
+See also:
+
+1. the [partial daemon log](http://paste.debian.net/73176/) from the assistant running in that directory on the laptop and 
+2. the output of [git fsck](http://paste.debian.net/73175/) on the remote.
+
+git-annex repair on the laptop and the server:
+[[!format sh """
+greg@x200s:~/Documents$ git-annex repair
+Running git fsck ...
+No problems found.
+ok
+"""]]
+
+
+### How I ended up fixing it:
+[[!format sh """
+greg@x200s:~/Documents$ killall git-annex
+greg@x200s:~/Documents$ git-annex indirect
+blah...............
+indirect  ok
+ok
+greg@x200s:~/Documents$ git status
+On branch master
+Your branch and 'rose/master' have diverged,
+and have 294 and 1 different commit each, respectively.
+  (use "git pull" to merge the remote branch into yours)
+
+Untracked files:
+  (use "git add ..." to include in what will be committed)
+
+	.gitrefs/
+
+nothing added to commit but untracked files present (use "git add" to track)
+greg@x200s:~/Documents$ git pull
+Merge made by the 'recursive' strategy.
+ Copyright Office/Orphan Works/staging/reporting/process_report.txt.2 | 1 +
+ Copyright Office/Orphan Works/staging/reporting/with-title.xls       | 1 +
+ Copyright Office/Orphan Works/staging/with-title.xls                 | 1 +
+ Copyright Office/Orphan Works/worker_emails.txt                      | 1 +
+ git.fsck.log                                                         | 1 +
+ 5 files changed, 5 insertions(+)
+ create mode 120000 Copyright Office/Orphan Works/staging/reporting/process_report.txt.2
+ create mode 120000 Copyright Office/Orphan Works/staging/reporting/with-title.xls
+ create mode 120000 Copyright Office/Orphan Works/staging/with-title.xls
+ create mode 120000 Copyright Office/Orphan Works/worker_emails.txt
+ create mode 120000 git.fsck.log
+greg@x200s:~/Documents$ git-annex sync
+commit  ok
+pull rose 
+
+Already up-to-date.
+ok
+push rose 
+Counting objects: 1658, done.
+Delta compression using up to 2 threads.
+Compressing objects: 100% (904/904), done.
+Writing objects: 100% (1604/1604), 138.97 KiB | 0 bytes/s, done.
+Total 1604 (delta 892), reused 1298 (delta 688)
+To greg@rose.makesad.us:/home/greg/Documents/
+   f1d206e..e836b9b  master -> synced/master
+ok
+greg@x200s:~/Documents$
+"""]]
+
+I restarted the assistant and the daemon.log looks good.
+
+After sync'ing on the server, it appears that this has been the case for quite some time (based off of what symlinks were created).
+
+Lastly: Joey, this is probably what caused that weird behavior in the webapp where it showed the bad transfer each day after the fsck at noon. I never diagnosed that more but I bet I won't see it tomorrow.
+
+[[!tag moreinfo]]
+
+> I think this was a corrupt .git/annex/index. The "..." in the above
+> transcript is where the actual useful part of the error message would be,
+> so pity it was trimmed in this bug report. Based on @yarikoptic's
+> occurance of the problem below, deleting .git/annex/index will recover.
+> And I would expect `git annex repair` would also recover in this
+> situation. [[done]] --[[Joey]]
diff --git a/doc/bugs/fatal__58___git-write-tree__58___error_building_trees/comment_2_4412fe19280551109bed2fde7a921079._comment b/doc/bugs/fatal__58___git-write-tree__58___error_building_trees/comment_2_4412fe19280551109bed2fde7a921079._comment
new file mode 100644
index 0000000000..ba0aec5c83
--- /dev/null
+++ b/doc/bugs/fatal__58___git-write-tree__58___error_building_trees/comment_2_4412fe19280551109bed2fde7a921079._comment
@@ -0,0 +1,47 @@
+[[!comment format=mdwn
+ username="yarikoptic"
+ avatar="http://cdn.libravatar.org/avatar/f11e9c84cb18d26a1748c33b48c924b4"
+ subject="old issue bites back"
+ date="2018-04-16T05:53:00Z"
+ content="""
+Sorry to bring the old issue up but we just ran into the same situation, and I wonder if any idea on how to mediate -- unlike other cases where it happens due to corrupt index, here I guess it goes about \"git-annex\" branch (or its some kind of local index cache?)
+
+[[!format sh \"\"\"
+nastase@head1:~/attention/raw_bids/stimuli$ git add README.md
+nastase@head1:~/attention/raw_bids/stimuli$ git commit -m 'README for stimuli' README.md 
+(recording state in git...)
+error: invalid object 100644 fd601b9139518bd6737ad20e55198dd1854fa0f9 for '001/92f/MD5E-s1413--241f0946a6aecfe12a7c031cb2339f9d.pklz.log'
+fatal: git-write-tree: error building trees
+git-annex: failed to read sha from git write-tree
+CallStack (from HasCallStack):
+  error, called at ./Git/Sha.hs:18:15 in main:Git.Sha
+
+nastase@head1:~/attention/raw_bids/stimuli$ git annex fsck
+curl: (22) The requested URL returned error: 404 Not Found
+
+  Remote openfmri not usable by git-annex; setting annex-ignore
+fsck bird_eating_1.mp4 (checksum...) ok
+...
+fsck ungulate_swimming_2.mp4 (checksum...) ok
+(recording state in git...)
+error: invalid object 100644 fd601b9139518bd6737ad20e55198dd1854fa0f9 for '001/92f/MD5E-s1413--241f0946a6aecfe12a7c031cb2339f9d.pklz.log'
+fatal: git-write-tree: error building trees
+git-annex: failed to read sha from git write-tree
+CallStack (from HasCallStack):
+  error, called at ./Git/Sha.hs:18:15 in main:Git.Sha
+
+nastase@head1:~/attention/raw_bids/stimuli$ git --version
+git version 2.13.0.rc1.294.g07d810a77f
+
+nastase@head1:~/attention/raw_bids/stimuli$ git annex version
+git-annex version: 6.20180329+gitga5fe62bed-1~ndall+1
+
+\"\"\"]]
+
+NB The issue came earlier and I have just upgraded git from elderly 2.7.0 and git-annex-standalone from a few months back.
+
+Any advice would be welcome
+
+[[!meta author=yoh]]
+
+"""]]
diff --git a/doc/bugs/fatal__58___git-write-tree__58___error_building_trees/comment_2_77295c0b749e984a6fb200d3b73b5765._comment b/doc/bugs/fatal__58___git-write-tree__58___error_building_trees/comment_2_77295c0b749e984a6fb200d3b73b5765._comment
new file mode 100644
index 0000000000..dabc1d4f35
--- /dev/null
+++ b/doc/bugs/fatal__58___git-write-tree__58___error_building_trees/comment_2_77295c0b749e984a6fb200d3b73b5765._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="http://joeyh.name/"
+ ip="209.250.56.35"
+ subject="comment 2"
+ date="2014-01-06T16:14:29Z"
+ content="""
+As far as I can tell from git's code, this involves a \"cache-tree\" data structure stored in git's index file. git-annex has nothing to do with writing index files, beyond running git plumbing. So I don't see how this could be anything other than a corrupt index file (of a sort that git fsck doesn't detect), or a git bug. The best thing to do would be to send an example of a repository having this problem to the git developers for analysis.
+"""]]
diff --git a/doc/bugs/fatal__58___git-write-tree__58___error_building_trees/comment_3_6bd0caa81cb87d30355cc8ed9fc6d958._comment b/doc/bugs/fatal__58___git-write-tree__58___error_building_trees/comment_3_6bd0caa81cb87d30355cc8ed9fc6d958._comment
new file mode 100644
index 0000000000..389b9d79fc
--- /dev/null
+++ b/doc/bugs/fatal__58___git-write-tree__58___error_building_trees/comment_3_6bd0caa81cb87d30355cc8ed9fc6d958._comment
@@ -0,0 +1,20 @@
+[[!comment format=mdwn
+ username="joey"
+ subject="""comment 3"""
+ date="2018-04-16T20:24:33Z"
+ content="""
+@yarikoptic, yes, the filename there is a git-annex branch filename.
+So, .git/annex/index is where the problem lies in your case.
+
+(Probably in the original bug reporter's case too; they trimmed out the
+important part of the error message it seems. git fsck won't detect a
+problem with .git/annex/index.)
+
+The best thing to do in this case is to delete .git/annex/index,
+since git-annex can always recover from that by creating a new one that
+does not point to git objects that have mysteriously vanished from your
+disk.
+
+Since information that was being written to the git-annex branch has
+apparently been lost, you'll also want to `git annex fsck --fast`
+"""]]
diff --git a/doc/bugs/fatal__58___git-write-tree__58___error_building_trees/comment_4_f3af2244be7bf7461882b0e9e527922a._comment b/doc/bugs/fatal__58___git-write-tree__58___error_building_trees/comment_4_f3af2244be7bf7461882b0e9e527922a._comment
new file mode 100644
index 0000000000..f14e42d1b4
--- /dev/null
+++ b/doc/bugs/fatal__58___git-write-tree__58___error_building_trees/comment_4_f3af2244be7bf7461882b0e9e527922a._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="yarikoptic"
+ avatar="http://cdn.libravatar.org/avatar/f11e9c84cb18d26a1748c33b48c924b4"
+ subject="comment 4"
+ date="2018-04-17T03:52:55Z"
+ content="""
+thank you Joey.  It seems to be healed now (after ` rm ../.git/annex/index*`, since there also was .lck file, just killed them together, annex fsck didn't raise any alarm)
+"""]]
diff --git a/doc/bugs/fatal__58___unable_to_normalize_object_directory.mdwn b/doc/bugs/fatal__58___unable_to_normalize_object_directory.mdwn
new file mode 100644
index 0000000000..29e694f2c0
--- /dev/null
+++ b/doc/bugs/fatal__58___unable_to_normalize_object_directory.mdwn
@@ -0,0 +1,82 @@
+### Please describe the problem.
+
+I'm getting `fatal: unable to normalize object directory` while issuing `git annex copy` from an annex in my laptop to a vanilla remote annex located in a USB disk.
+As far as I can tell there are no hardware problems with the disk or the laptop.
+
+```
+$ git annex copy --auto --to usbdrive .
+copy vlc-record-2016-07-17-15h16m55s-hr-fernsehen-Die fantastische Reise der Vögel (3_6).ts (to usbdrive...) 
+SHA256E-s1256869488--28edfd2c4eb08e6782c806a85db7bc10595c16df6f64d3d46ebc2802b3ba5038.ts
+  1,256,869,488 100%   52.96MB/s    0:00:22 (xfr#1, to-chk=0/1)
+(checksum...) fatal: unable to normalize object directory: /home/jr/Videos/ts/../../../../../run/media/jr/TOSHIBA/jr/Videos.git/objects
+fatal: unable to normalize object directory: /home/jr/Videos/ts/../../../../../run/media/jr/TOSHIBA/jr/Videos.git/objects
+fatal: unable to normalize object directory: /home/jr/Videos/ts/../../../../../run/media/jr/TOSHIBA/jr/Videos.git/objects
+fatal: unable to normalize object directory: /home/jr/Videos/ts/../../../../../run/media/jr/TOSHIBA/jr/Videos.git/objects
+fatal: unable to normalize object directory: /home/jr/Videos/ts/../../../../../run/media/jr/TOSHIBA/jr/Videos.git/objects
+fatal: unable to normalize object directory: /home/jr/Videos/ts/../../../../../run/media/jr/TOSHIBA/jr/Videos.git/objects
+fatal: unable to normalize object directory: /home/jr/Videos/ts/../../../../../run/media/jr/TOSHIBA/jr/Videos.git/objects
+fatal: unable to normalize object directory: /home/jr/Videos/ts/../../../../../run/media/jr/TOSHIBA/jr/Videos.git/objects
+fatal: unable to normalize object directory: /home/jr/Videos/ts/../../../../../run/media/jr/TOSHIBA/jr/Videos.git/objects
+fatal: unable to normalize object directory: /home/jr/Videos/ts/../../../../../run/media/jr/TOSHIBA/jr/Videos.git/objects
+fatal: unable to normalize object directory: /home/jr/Videos/ts/../../../../../run/media/jr/TOSHIBA/jr/Videos.git/objects
+
+  fd:24: hGetLine: end of file
+
+SHA256E-s1256869488--28edfd2c4eb08e6782c806a85db7bc10595c16df6f64d3d46ebc2802b3ba5038.ts
+  1,256,869,488 100%   56.08MB/s    0:00:21 (xfr#1, to-chk=0/1)
+(checksum...) 
+  fd:23: hFlush: resource vanished (Broken pipe)
+
+  fd:23: hClose: resource vanished (Broken pipe)
+ok
+(recording state in git...)
+```
+
+But note that the command did not abort. After copying, `fsck` reports no error:
+
+```
+$ git annex fsck --from=usbdrive 'vlc-record-2016-07-17-15h16m55s-hr-fernsehen-Die fantastische Reise der Vögel (3_6).ts'
+fsck vlc-record-2016-07-17-15h16m55s-hr-fernsehen-Die fantastische Reise der Vögel (3_6).ts (checksum...) ok
+(recording state in git...)
+```
+
+I get the same errors with `drop`, which fails: 
+
+```
+$ git annex drop --from usbdrive 'vlc-record-2016-07-17-15h16m55s-hr-fernsehen-Die fantastische Reise der Vögel (3_6).ts'
+drop usbdrive vlc-record-2016-07-17-15h16m55s-hr-fernsehen-Die fantastische Reise der Vögel (3_6).ts fatal: unable to normalize object directory: /home/jr/Videos/ts/../../../../../run/media/jr/TOSHIBA/jr/Videos.git/objects
+fatal: unable to normalize object directory: /home/jr/Videos/ts/../../../../../run/media/jr/TOSHIBA/jr/Videos.git/objects
+fatal: unable to normalize object directory: /home/jr/Videos/ts/../../../../../run/media/jr/TOSHIBA/jr/Videos.git/objects
+fatal: unable to normalize object directory: /home/jr/Videos/ts/../../../../../run/media/jr/TOSHIBA/jr/Videos.git/objects
+fatal: unable to normalize object directory: /home/jr/Videos/ts/../../../../../run/media/jr/TOSHIBA/jr/Videos.git/objects
+fatal: unable to normalize object directory: /home/jr/Videos/ts/../../../../../run/media/jr/TOSHIBA/jr/Videos.git/objects
+fatal: unable to normalize object directory: /home/jr/Videos/ts/../../../../../run/media/jr/TOSHIBA/jr/Videos.git/objects
+fatal: unable to normalize object directory: /home/jr/Videos/ts/../../../../../run/media/jr/TOSHIBA/jr/Videos.git/objects
+fatal: unable to normalize object directory: /home/jr/Videos/ts/../../../../../run/media/jr/TOSHIBA/jr/Videos.git/objects
+fatal: unable to normalize object directory: /home/jr/Videos/ts/../../../../../run/media/jr/TOSHIBA/jr/Videos.git/objects
+fatal: unable to normalize object directory: /home/jr/Videos/ts/../../../../../run/media/jr/TOSHIBA/jr/Videos.git/objects
+
+git-annex: fd:27: hGetLine: end of file
+failed
+git-annex: drop: 1 failed
+```
+
+But the same `drop` works if executed again:
+
+```
+$ git annex drop --from usbdrive 'vlc-record-2016-07-17-15h16m55s-hr-fernsehen-Die fantastische Reise der Vögel (3_6).ts'
+drop usbdrive vlc-record-2016-07-17-15h16m55s-hr-fernsehen-Die fantastische Reise der Vögel (3_6).ts ok
+(recording state in git...)
+```
+
+### What version of git-annex are you using? On what operating system?
+
+git-annex version: 6.20161231-gc8eeb17da
+
+Linux XXX 4.8.13-1-ARCH #1 SMP PREEMPT Fri Dec 9 07:24:34 CET 2016 x86_64 GNU/Linux
+
+### Have you had any luck using git-annex before? (Sometimes we get tired of reading bug reports all day and a lil' positive end note does wonders)
+
+Yes! I use git-annex without a problem since years. This is the first time I have really no idea what's going on.
+
+> Seems to be fixed in git version 2.19.0 [[done]] --[[Joey]]
diff --git a/doc/bugs/fatal__58___unable_to_normalize_object_directory/comment_1_a2b90b2111bee705afa399fdb8fad23d._comment b/doc/bugs/fatal__58___unable_to_normalize_object_directory/comment_1_a2b90b2111bee705afa399fdb8fad23d._comment
new file mode 100644
index 0000000000..ebebdb3a9b
--- /dev/null
+++ b/doc/bugs/fatal__58___unable_to_normalize_object_directory/comment_1_a2b90b2111bee705afa399fdb8fad23d._comment
@@ -0,0 +1,10 @@
+[[!comment format=mdwn
+ username="jesrui@51c25da8d6f34e6df8e3e7ed0277335ed7ddf6a6"
+ nickname="jesrui"
+ avatar="http://cdn.libravatar.org/avatar/079b86efcbbb1c0c54be3759a0c2b223"
+ subject="comment 1"
+ date="2017-01-21T15:08:20Z"
+ content="""
+I found the problem: I was cd'ing into the local repository via a symbolic link, instead of using the \"canonical\" path.
+You can mark the bug as solved or delete it. Sorry for the noise.
+"""]]
diff --git a/doc/bugs/fatal__58___unable_to_normalize_object_directory/comment_2_bd39c124ebb1f505fc6a7fb6815ff49b._comment b/doc/bugs/fatal__58___unable_to_normalize_object_directory/comment_2_bd39c124ebb1f505fc6a7fb6815ff49b._comment
new file mode 100644
index 0000000000..1df44e8eb2
--- /dev/null
+++ b/doc/bugs/fatal__58___unable_to_normalize_object_directory/comment_2_bd39c124ebb1f505fc6a7fb6815ff49b._comment
@@ -0,0 +1,15 @@
+[[!comment format=mdwn
+ username="joey"
+ subject="""comment 2"""
+ date="2017-06-06T19:38:00Z"
+ content="""
+"fatal: unable to normalize object directory" is an error message that
+git emits, not git-annex.
+
+I wonder if git-annex's use of --work-tree and --git-dir with
+relative paths confuses git in some way?
+
+It would be good to explain how to reproduce this problem.
+I tried a few things involving symlinks in the path I `cd`ed to, 
+but was not able to cause the problem.
+"""]]
diff --git a/doc/bugs/fatal__58___unable_to_normalize_object_directory/comment_3_c49df6a33bbf8c3238f5eec1ef3e8152._comment b/doc/bugs/fatal__58___unable_to_normalize_object_directory/comment_3_c49df6a33bbf8c3238f5eec1ef3e8152._comment
new file mode 100644
index 0000000000..3910a24eda
--- /dev/null
+++ b/doc/bugs/fatal__58___unable_to_normalize_object_directory/comment_3_c49df6a33bbf8c3238f5eec1ef3e8152._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="rfourquet"
+ avatar="http://cdn.libravatar.org/avatar/2c78d7b5b3c6a417e5d666666ec40d51"
+ subject="comment 3"
+ date="2017-11-05T12:04:50Z"
+ content="""
+I ran into the same problem: assuming I have a repo at \"/mnt/hd/repo\", then `$ cd /; sudo ln -s /mnt/hd/ .; cd /hd/repo; git annex copy --to=someremote ./some-file` exhibits the problem. Thanks!
+"""]]
diff --git a/doc/bugs/fatal__58___unable_to_normalize_object_directory/comment_4_c4cf3df34625b17ed44077bec53b0ac4._comment b/doc/bugs/fatal__58___unable_to_normalize_object_directory/comment_4_c4cf3df34625b17ed44077bec53b0ac4._comment
new file mode 100644
index 0000000000..85e51a9633
--- /dev/null
+++ b/doc/bugs/fatal__58___unable_to_normalize_object_directory/comment_4_c4cf3df34625b17ed44077bec53b0ac4._comment
@@ -0,0 +1,10 @@
+[[!comment format=mdwn
+ username="joey"
+ subject="""comment 4"""
+ date="2017-11-07T18:19:39Z"
+ content="""
+Thanks for the reproducion recipe, @rfourquet. That worked for me.
+
+The bug submitter closed this bug, but I'm reopening it since this seems a
+legitimate bug report and we know how to reproduce it too.
+"""]]
diff --git a/doc/bugs/fatal__58___unable_to_normalize_object_directory/comment_5_cb5332119c54764b4ef1e000b3767b0e._comment b/doc/bugs/fatal__58___unable_to_normalize_object_directory/comment_5_cb5332119c54764b4ef1e000b3767b0e._comment
new file mode 100644
index 0000000000..5cc9dc6787
--- /dev/null
+++ b/doc/bugs/fatal__58___unable_to_normalize_object_directory/comment_5_cb5332119c54764b4ef1e000b3767b0e._comment
@@ -0,0 +1,36 @@
+[[!comment format=mdwn
+ username="joey"
+ subject="""comment 5"""
+ date="2017-11-07T18:21:20Z"
+ content="""
+With somerepo in ~/tmp/repo, and pwd in /hd/repo, which is a symlink from /tmp/hd/repo,
+this happens:
+
+	chat: git ["--git-dir=../../../home/joey/tmp/repo/.git","--work-tree=../../../home/joey/tmp/repo","--literal-pathspecs","--literal-pathspecs","cat-file","--batch"]
+	fatal: unable to normalize object directory: /hd/repo/../../../home/joey/tmp/repo/.git/objects
+
+Here it's making git operate on the remote git repo. The relative paths it uses
+to it seem legitimate, in that "ls ../.." shows the content of "/tmp" and
+"ls ../../.." shows the root directory.
+
+This is a very confusing situation, and different ways of getting the current
+directory give different results in this situation. In particular `$PWD`
+contains "/hd/repo" while getcwd(3) will return "/tmp/hd/repo". Paths
+need to be relative to "/tmp/hd/repo" as that's the *actual* cwd.
+
+It seems that git is looking at `$PWD` to get "/hd/repo" and normalizing
+the relative path via that. So there's no possible relative path
+that git will accept in this situation. This kind of seems like buggy
+behavior on git's part. I've posted about it to the git mailing list.
+
+
+git-annex could avoid the problem by not using relative paths of course,
+but there are reasons including shorter path length (generally) 
+for its use of relative paths. 
+
+git-annex could unset PWD before running git, which forces git to fall
+back to using getcwd(3). I've verified that does avoid the problem.
+Going to see if this just gets fixed in git before I put in such ugly
+and kind of expensive workarounds. (Expensive because, to unset PWD
+from the environment, git-annex would have to copy it.)
+"""]]
diff --git a/doc/bugs/fatal__58___unable_to_normalize_object_directory/comment_6_19c9dbdbcc2875cd6429f0ba06336fd2._comment b/doc/bugs/fatal__58___unable_to_normalize_object_directory/comment_6_19c9dbdbcc2875cd6429f0ba06336fd2._comment
new file mode 100644
index 0000000000..11d32e5f67
--- /dev/null
+++ b/doc/bugs/fatal__58___unable_to_normalize_object_directory/comment_6_19c9dbdbcc2875cd6429f0ba06336fd2._comment
@@ -0,0 +1,11 @@
+[[!comment format=mdwn
+ username="joey"
+ subject="""comment 6"""
+ date="2017-11-13T17:09:31Z"
+ content="""
+The git bug is fixed in git's `pu` branch, or as Peff says,
+at least "papered over", and I've tested the fix with git-annex and
+it seems to avoid the problem. So as long as
+5eb0e541132208a9027cc090943276fa52f29c71 gets into a git release,
+this bug can be closed without any changes to git-annex.
+"""]]
diff --git a/doc/bugs/feed_dependency.mdwn b/doc/bugs/feed_dependency.mdwn
new file mode 100644
index 0000000000..789a94f54e
--- /dev/null
+++ b/doc/bugs/feed_dependency.mdwn
@@ -0,0 +1,31 @@
+It looks like the latest version of `git-annex` currently published on
+Hackage has a dependency on `feed` package without an upper bound
+specified.
+
+FYI, as per https://github.com/bergmark/feed/pull/20, a new version of
+`feed` will be released soon, incorporating non-backwards compatible
+changes to `feed` interface (`String` was replaced with `Text`
+throughout and `xml-types` types were adopted). Changes are largely
+mechanical but you'll probably want to ensure that future version of
+`git-annex` can build with newer `feed`.
+
+To prevent breakage for users, you may want to release a new version
+of `git-annex` with an upper bound specified now, then push a version
+compatible with `feed-1.0` after that comes out later.
+
+Use the tip of the PR to test your package against:
+https://github.com/dzhus/feed/commit/259bab0 . With Stack, you just
+need to put the following in the `packages` section of your
+`stack.yaml`:
+
+	    - location:
+	        git: git@github.com:dzhus/feed
+	        commit: 259bab0dd16853656ce1d2a005c4009d4747edc1
+	      extra-dep: true
+
+Please report any feedback / issues to the aforementioned PR thread
+or here if that works for you.
+
+-- Dmitry Dzhus
+
+> [[fixed|done]] --[[Joey]]
diff --git a/doc/bugs/file_extension_not_used_for_URIs_from_special_remotes.mdwn b/doc/bugs/file_extension_not_used_for_URIs_from_special_remotes.mdwn
new file mode 100644
index 0000000000..78e80fe1ae
--- /dev/null
+++ b/doc/bugs/file_extension_not_used_for_URIs_from_special_remotes.mdwn
@@ -0,0 +1,14 @@
+### Please describe the problem.
+If a special remote has URIs that do not end in a file extension, then symlinks to annexed files fetched from that remote will have null extensions, even if
+CHECKURL returned a filename with an extension.
+git annex --verbose --debug addurl dx://file-BXF0vYQ0QyBF509G9J12g927
+creates the symlink
+ contaminants.fasta -> ../.git/annex/objects/ZF/Gj/SHA256E-s18932--c1ed24754e8b7736e275359a34b312d2d6ce2efc1d236061b04d27a9e8147c1a/SHA256E-s18932--c1ed24754e8b7736e275359a34b312d2d6ce2efc1d236061b04d27a9e8147c1a
+
+which doesn't have an extension even though 
+
+[2018-09-18 23:49:54.875801] git-annex-remote-dnanexus[1] --> CHECKURL-CONTENTS 18932 contaminants.fasta
+addurl dx://file-BXF0vYQ0QyBF509G9J12g927 (from dnanexus) (to contaminants.fasta) [2018-09-18 23:49:54.876016] read: git ["--version"]
+[2018-09-18 23:49:54.876825] process done ExitSuccess
+
+and the SHA256E backend is used.
diff --git a/doc/bugs/file_extensions_of___62__4_chars_ignored_by___42__E_backends.mdwn b/doc/bugs/file_extensions_of___62__4_chars_ignored_by___42__E_backends.mdwn
new file mode 100644
index 0000000000..5109204eab
--- /dev/null
+++ b/doc/bugs/file_extensions_of___62__4_chars_ignored_by___42__E_backends.mdwn
@@ -0,0 +1,38 @@
+### Please describe the problem.
+It seems that SHA256E, MD5E etc backends ignore file extensions longer than 4 characters, considering files with such extensions to have an empty extension.
+But it's not uncommon to have longer extensions; e.g. .fasta and .fasta.gz files are common in bioinformatics.
+Is it possible to remove this 4-character limit, or make it configurable?
+
+### What steps will reproduce the problem?
+(master_env_py27_v28) [12:37 PM /data/ilya-work/sw]$ cp c10.yyy c11.yyyy
+(master_env_py27_v28) [12:37 PM /data/ilya-work/sw]$ git annex calckey c11.yyyy
+SHA256E-s18841--9fd9a2607e019b7726c722d9d6f6171e6578f255bc60a0b79c525f8a3ffa05de.yyyy
+(master_env_py27_v28) [12:37 PM /data/ilya-work/sw]$ cp c10.yyy c12.yyyyy
+(master_env_py27_v28) [12:37 PM /data/ilya-work/sw]$ git annex calckey c12.yyyyy
+SHA256E-s18841--9fd9a2607e019b7726c722d9d6f6171e6578f255bc60a0b79c525f8a3ffa05de
+
+(master_env_py27_v28) [12:43 PM /data/ilya-work/sw]$ git annex calckey c10.yyyy.gz
+SHA256E-s2168--21bb6c514473754cc49a455f45bc84961fe4fceb2cb0527ba2a1cfabdce6bf80.yyyy.gz
+(master_env_py27_v28) [12:43 PM /data/ilya-work/sw]$ mv c10.yyyy.gz c10.yyyyy.gz
+(master_env_py27_v28) [12:43 PM /data/ilya-work/sw]$ git annex calckey c10.yyyyy.gz
+SHA256E-s2168--21bb6c514473754cc49a455f45bc84961fe4fceb2cb0527ba2a1cfabdce6bf80.gz
+
+### What version of git-annex are you using? On what operating system?
+
+git-annex version: 6.20180807-gebc1bb5
+build flags: Assistant Webapp Pairing S3(multipartupload)(storageclasses) WebDAV Inotify ConcurrentOutput TorrentParser MagicMime Feed\
+s Testsuite
+dependency versions: aws-0.17.1 bloomfilter-2.0.1.0 cryptonite-0.23 DAV-1.3.1 feed-0.3.12.0 ghc-8.0.2 http-client-0.5.7.0 persistent-s\
+qlite-2.6.2 torrent-10000.1.1 uuid-1.3.13 yesod-1.4.5
+key/value backends: SHA256E SHA256 SHA512E SHA512 SHA224E SHA224 SHA384E SHA384 SHA3_256E SHA3_256 SHA3_512E SHA3_512 SHA3_224E SHA3_2\
+24 SHA3_384E SHA3_384 SKEIN256E SKEIN256 SKEIN512E SKEIN512 BLAKE2B256E BLAKE2B256 BLAKE2B512E BLAKE2B512 BLAKE2B160E BLAKE2B160 BLAKE\
+2B224E BLAKE2B224 BLAKE2B384E BLAKE2B384 BLAKE2S256E BLAKE2S256 BLAKE2S160E BLAKE2S160 BLAKE2S224E BLAKE2S224 BLAKE2SP256E BLAKE2SP256\
+ BLAKE2SP224E BLAKE2SP224 SHA1E SHA1 MD5E MD5 WORM URL
+remote types: git gcrypt p2p S3 bup directory rsync web bittorrent webdav adb tahoe glacier ddar hook external
+operating system: linux x86_64
+supported repository versions: 3 5 6
+upgrade supported from repository versions: 0 1 2 3 4 5
+local repository version: 5
+
+
+
diff --git a/doc/bugs/file_modification_time_should_be_stored_in_exactly_one_metadata_field.mdwn b/doc/bugs/file_modification_time_should_be_stored_in_exactly_one_metadata_field.mdwn
new file mode 100644
index 0000000000..ae2ddf907f
--- /dev/null
+++ b/doc/bugs/file_modification_time_should_be_stored_in_exactly_one_metadata_field.mdwn
@@ -0,0 +1,10 @@
+### Please describe the problem.
+File modification metadata is stored separately as year, month and day according to the annex.genmetadata docs. This makes reading and writing it unnecessarily complicated. Also, every field has its own modification timestamp and two out of the three become meaningless once you change one. (Does the modification time field even need its own modification time field?)
+
+It would be easier to work with if it was represented as a UTC unix timestamp or ISO 8601 string or something similar, ideally something date(1) and/or touch(1) (GNU and BSD) will understand natively. Also, you could fit more precision in there for little cost.
+
+### What steps will reproduce the problem?
+`git annex metadata `
+
+### What version of git-annex are you using? On what operating system?
+git annex 6.20180227, various Linuxes.
diff --git a/doc/bugs/file_modification_time_should_be_stored_in_exactly_one_metadata_field/comment_1_7fef3484f3019539c0a305d5f4a801ce._comment b/doc/bugs/file_modification_time_should_be_stored_in_exactly_one_metadata_field/comment_1_7fef3484f3019539c0a305d5f4a801ce._comment
new file mode 100644
index 0000000000..6d49f7d77a
--- /dev/null
+++ b/doc/bugs/file_modification_time_should_be_stored_in_exactly_one_metadata_field/comment_1_7fef3484f3019539c0a305d5f4a801ce._comment
@@ -0,0 +1,15 @@
+[[!comment format=mdwn
+ username="lykos@d125a37d89b1cfac20829f12911656c40cb70018"
+ nickname="lykos"
+ avatar="http://cdn.libravatar.org/avatar/085df7b04d3408ba23c19f9c49be9ea2"
+ subject="comment 1"
+ date="2018-03-23T19:01:02Z"
+ content="""
+The year, month and day fields are used when creating views that sort files by modification date, eg. `git annex view year=* month=* day=*`. This cannot be done using timestamps. I agree that an additional timestamp field is a good idea, so I modified the pre-commit-annex hook by adding
+
+    field=\"mtime\"
+    value=$(stat -Lc %Y \"$f\")
+    addmeta \"$f\" \"$field\" \"$value\"
+
+at the end of the process function.
+"""]]
diff --git a/doc/bugs/file_modification_time_should_be_stored_in_exactly_one_metadata_field/comment_2_32285971869ec56460b3f0eee4edfa23._comment b/doc/bugs/file_modification_time_should_be_stored_in_exactly_one_metadata_field/comment_2_32285971869ec56460b3f0eee4edfa23._comment
new file mode 100644
index 0000000000..68c177c83b
--- /dev/null
+++ b/doc/bugs/file_modification_time_should_be_stored_in_exactly_one_metadata_field/comment_2_32285971869ec56460b3f0eee4edfa23._comment
@@ -0,0 +1,30 @@
+[[!comment format=mdwn
+ username="joey"
+ subject="""comment 2"""
+ date="2018-04-04T16:40:19Z"
+ content="""
+One use case for the separate fields is `git-annex view`,
+where they allow creating view branches with yyyy/mm/ subdirectories
+or artist/yyyy/ or whatever kind of thing the user wants.
+
+Part of the original metadata design, which never got implemented,
+was to allow pure functions on metadata, that added to the set of available
+metadata. So a unix timestamp could be what's stored in metadata, and then
+the individual fields (and more fields that are currently omitted to avoid
+bloat) derived from it.
+
+I'd be happy to see that part of the metadata design actually happen.
+There are some potential problems with it that need to be thought about,
+such as:
+
+* What should happen if the user runs `git annex metadata` to change
+  or delete such a derived metadata field?
+* If a unix timestamp is stored, it's harder for the user to make
+  some simple edit, such as adjusting the year of a file to the year they
+  want. Should a change to a derived year field propigate back to the unix
+  timestamp? (Would need reversible functions.)
+
+(BTW, the timestamps on metadata fields come for "free" in that the
+underlying metadata storage log necessarily contains timestamps of when
+each part of the metadata was changed.)
+"""]]
diff --git a/doc/bugs/file_modification_time_should_be_stored_in_exactly_one_metadata_field/comment_3_7bf423111829ec6b358c6925d012a3e2._comment b/doc/bugs/file_modification_time_should_be_stored_in_exactly_one_metadata_field/comment_3_7bf423111829ec6b358c6925d012a3e2._comment
new file mode 100644
index 0000000000..8c5973f255
--- /dev/null
+++ b/doc/bugs/file_modification_time_should_be_stored_in_exactly_one_metadata_field/comment_3_7bf423111829ec6b358c6925d012a3e2._comment
@@ -0,0 +1,11 @@
+[[!comment format=mdwn
+ username="vrs+annex@ea5fa24dbb279be61a8e50adb638bf8366300717"
+ nickname="vrs+annex"
+ avatar="http://cdn.libravatar.org/avatar/74219abcec6eece8e2c9d4351c2c912c"
+ subject="comment 3"
+ date="2018-04-05T02:11:41Z"
+ content="""
+If pure functions were part of the design anyway, why not avoid adding derived fields and introduce functions like `year()` etc instead? The semantics would be obvious (you don't assign to a function call) and existing workflows involving the year field would keep working.
+
+As for editing the timestamp in general, I'll post an answer to  later that should address this.
+"""]]
diff --git a/doc/bugs/filemanager_integration_broken.mdwn b/doc/bugs/filemanager_integration_broken.mdwn
new file mode 100644
index 0000000000..aef3896fd7
--- /dev/null
+++ b/doc/bugs/filemanager_integration_broken.mdwn
@@ -0,0 +1,44 @@
+### Please describe the problem.
+
+I tried now for hours to understand why the filemanager inegration is just not working.
+Debug output in Assistent/Install.hs is basically none existent so I don't even know if anything is run at all.
+There is no dedicated process to get the .desktop files etc created or an example that you could use if the process fails.
+I'm out of ideas now.
+
+I tried running with GIT_ANNEX_PACKAGE_INSTALL=1 and without, tried to ensure directories exist. 
+
+### What steps will reproduce the problem?
+
+git annex assistant
+
+
+
+### What version of git-annex are you using? On what operating system?
+
+nixos
+
+git-annex version: 6.20171003
+build flags: Assistant Webapp Pairing S3(multipartupload)(storageclasses) WebDAV Inotify DBus DesktopNotify ConcurrentOutput TorrentParser MagicMime Feeds Quvi
+dependency versions: aws-0.16 bloomfilter-2.0.1.0 cryptonite-0.23 DAV-1.3.1 feed-0.3.12.0 ghc-8.0.2 http-client-0.5.7.0 persistent-sqlite-2.6.2 torrent-10000.1.1 uuid-1.3.13 yesod-1.4.5
+key/value backends: SHA256E SHA256 SHA512E SHA512 SHA224E SHA224 SHA384E SHA384 SHA3_256E SHA3_256 SHA3_512E SHA3_512 SHA3_224E SHA3_224 SHA3_384E SHA3_384 SKEIN256E SKEIN256 SKEIN512E SKEIN512 SHA1E SHA1 MD5E MD5 WORM URL
+remote types: git gcrypt p2p S3 bup directory rsync web bittorrent webdav tahoe glacier ddar hook external
+local repository version: 5
+supported repository versions: 3 5 6
+upgrade supported from repository versions: 0 1 2 3 4 5
+operating system: linux x86_64
+
+
+### Please provide any additional information below.
+
+[[!format sh """
+# If you can, paste a complete transcript of the problem occurring here.
+# If the problem is with the git-annex assistant, paste in .git/annex/daemon.log
+
+
+# End of transcript or log.
+"""]]
+
+### Have you had any luck using git-annex before? (Sometimes we get tired of reading bug reports all day and a lil' positive end note does wonders)
+
+yes
+
diff --git a/doc/bugs/filemanager_integration_broken/comment_1_aa5afe9af21e534a6c0683064296d9e6._comment b/doc/bugs/filemanager_integration_broken/comment_1_aa5afe9af21e534a6c0683064296d9e6._comment
new file mode 100644
index 0000000000..3a4184791a
--- /dev/null
+++ b/doc/bugs/filemanager_integration_broken/comment_1_aa5afe9af21e534a6c0683064296d9e6._comment
@@ -0,0 +1,30 @@
+[[!comment format=mdwn
+ username="joey"
+ subject="""comment 1"""
+ date="2018-04-04T16:26:07Z"
+ content="""
+Did you install git-annex from the nix package?
+
+The desktop file is supposed to be shipped with the OS's package of
+git-annex, in /etc so it's available for file managers to use as soon as
+it's installed. `make install` arranges for that to happen. 
+
+If nix is building
+git-annex in some other way, they may be leaving out such ancillary files.
+(And from what I know of nix, there would be some complication providing 
+files in /etc.)
+
+The other way the desktop file is installed was added for
+the git-annex standalone bundle, and installs it into the home directory
+of the user who starts the git-annex webapp. You've correctly
+identified the (undocumented) `GIT_ANNEX_PACKAGE_INSTALL=1` that informs
+git-annex that it's running in such a situation. But there's also a
+`GIT_ANNEX_APP_BASE`
+that it expects to point to where the standalone bundle is located
+(in order to use an icon file from there in the desktop file),
+and it doesn't install the desktop file when that's not set.
+
+I think that the nix package maintainer should be looped into this,
+so we can come up with some solution that works out of the box with those
+packages.
+"""]]
diff --git a/doc/bugs/find_with_batch_does_not_apply_matching_options.mdwn b/doc/bugs/find_with_batch_does_not_apply_matching_options.mdwn
new file mode 100644
index 0000000000..813ae9caf1
--- /dev/null
+++ b/doc/bugs/find_with_batch_does_not_apply_matching_options.mdwn
@@ -0,0 +1,13 @@
+### Please describe the problem.
+Using `git annex find --batch` with matching options seems to not apply them.
+
+### What steps will reproduce the problem?
+
+    find -type l | git annex find --batch --copies 2
+    ... list of files that include files with only 1 copy ...
+
+### What version of git-annex are you using? On what operating system?
+I'd rather not say ~~because it is ancient~~. Joey says this is reproducible on recent git-annex versions though.
+
+> Not only find, but a bunch of commands supporting --batch had this
+> oversight. Fixed them all. [[done]] --[[Joey]]
diff --git a/doc/bugs/fingertree___62____61___0.1.2_causes_build_to_fail_on_reducers.mdwn b/doc/bugs/fingertree___62____61___0.1.2_causes_build_to_fail_on_reducers.mdwn
new file mode 100644
index 0000000000..a3169be3b1
--- /dev/null
+++ b/doc/bugs/fingertree___62____61___0.1.2_causes_build_to_fail_on_reducers.mdwn
@@ -0,0 +1,57 @@
+### Please describe the problem.
+git-annex's dependencies fail to install unless fingertree is constrained to < 0.1.2.0
+
+Note this has already been fixed upstream in reducers https://github.com/ekmett/reducers/commit/f18a111d66c343bd472f914baaa948191f8ecf55
+
+However, there hasn't been a new release yet of reducers so it still affects git-annex currently.
+
+### What steps will reproduce the problem?
+cabal install --jobs=8 --max-backjumps=100000 --only-dependencies --flags=s3\ webapp
+
+
+### What version of git-annex are you using? On what operating system?
+6.20171003 on macOS
+
+### Please provide any additional information below.
+
+The error is
+[[!format sh """
+Failed to install reducers-3.12.1
+Build log ( /private/tmp/git-annex-20171003-61223-1v5jyd4/git-annex-6.20171003/.cabal-sandbox/logs/ghc-8.2.1/reducers-3.12.1-5Of5cjMdWsgHjbctJQaaa4.log ):
+cabal: Entering directory '/tmp/cabal-tmp-62861/reducers-3.12.1'
+Configuring reducers-3.12.1...
+clang: warning: -Wl,-headerpad_max_install_names: 'linker' input unused
+clang: warning: argument unused during compilation: '-L/usr/local/opt/gettext/lib'
+clang: warning: argument unused during compilation: '-L/usr/local/opt/libffi/lib'
+clang: warning: argument unused during compilation: '-L/usr/local/opt/icu4c/lib'
+clang: warning: argument unused during compilation: '-L/usr/local/lib'
+clang: warning: argument unused during compilation: '-L/System/Library/Frameworks/OpenGL.framework/Versions/Current/Libraries'
+Preprocessing library for reducers-3.12.1..
+Building library for reducers-3.12.1..
+[ 1 of 14] Compiling Data.Semigroup.Instances ( src/Data/Semigroup/Instances.hs, dist/dist-sandbox-c97b5ef4/build/Data/Semigroup/Instances.o )
+
+src/Data/Semigroup/Instances.hs:11:10: error:
+    Duplicate instance declarations:
+      instance Measured v a => Semigroup (FingerTree v a)
+        -- Defined at src/Data/Semigroup/Instances.hs:11:10
+      instance [safe] Measured v a => Semigroup (FingerTree v a)
+        -- Defined in ‘Data.FingerTree’
+   |
+11 | instance Measured v a => Semigroup (FingerTree v a) where
+   |          ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+cabal: Leaving directory '/tmp/cabal-tmp-62861/reducers-3.12.1'
+"""]]
+
+Full log is here: https://gist.github.com/ilovezfs/544d546785addbe2c5ef41656fa7eda0
+
+### Have you had any luck using git-annex before? (Sometimes we get tired of reading bug reports all day and a lil' positive end note does wonders)
+
+Yes :)
+
+> I see this was fixed in the meantime in fingertree-0.1.2.
+> 
+> Also, I'd prefer not to complicate git-annex's dependencies with
+> versioning for indirect dependencies like this one, unless there's
+> some long drawn-out breakage to contend with.
+> 
+> [[done]] --[[Joey]]
diff --git a/doc/bugs/fingertree___62____61___0.1.2_causes_build_to_fail_on_reducers/comment_1_cbb3c9f16cf834466cbc31fd129b3559._comment b/doc/bugs/fingertree___62____61___0.1.2_causes_build_to_fail_on_reducers/comment_1_cbb3c9f16cf834466cbc31fd129b3559._comment
new file mode 100644
index 0000000000..31e7547849
--- /dev/null
+++ b/doc/bugs/fingertree___62____61___0.1.2_causes_build_to_fail_on_reducers/comment_1_cbb3c9f16cf834466cbc31fd129b3559._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="ilovezfs"
+ avatar="http://cdn.libravatar.org/avatar/f2b3954cf2ed0a551de9a49d3b6a64d0"
+ subject="comment 1"
+ date="2017-10-19T05:50:11Z"
+ content="""
+reducers 3.12.2 was released with the fix included.
+"""]]
diff --git a/doc/bugs/fix_git-annex_paths___47___objects___40__repository_not_available__41__.mdwn b/doc/bugs/fix_git-annex_paths___47___objects___40__repository_not_available__41__.mdwn
new file mode 100644
index 0000000000..3bd64aeede
--- /dev/null
+++ b/doc/bugs/fix_git-annex_paths___47___objects___40__repository_not_available__41__.mdwn
@@ -0,0 +1,87 @@
+### Please describe the problem.
+I cloned my git-annex repository to a bare repository on my server (and deleted the original to reinstall the OS). When I try to clone back to a new machine
+
+    $ git-annex get .
+    get programs/2017-06-drafts/About.txt (not available) 
+      Try making some of these repositories available:
+      	8ddb8c4d-06ac-4c93-bf28-15639e0ea600 -- MacBook
+    failed
+
+and so for many files I committed. The files are actually on the server as I can see from the size of the repo and I remember them being copied there.
+
+When I strace that command, I see that it stats a missing file
+
+    stat(".git/annex/objects/7v/5x/SHA256E-s81068--de1d8de99645d74ba1ea186b6cabd1fc116cb6c1823130756f33ff81807815ed.pdf/SHA256E-s81068--de1d8de99645d74ba1ea186b6cabd1fc116cb6c1823130756f33ff81807815ed.pdf", 0x200015bf0) = -1 ENOENT (No such file or directory)
+
+`.git/annex/objects` is absent in the cloned repository on the server (I test it there - on my machine it doesn't work either).
+
+However I can find that the SHA file really exists 
+
+    $ locate SHA256E-s81068--de1d8de99645d74ba1ea186b6cabd1fc116cb6c1823130756f33ff81807815ed.pdf/SHA256E-s81068--de1d8de99645d74ba1ea186b6cabd1fc116cb6c1823130756f33ff81807815ed.pdf
+    /home/yaroslav/work.git/d0d/994/SHA256E-s81068--de1d8de99645d74ba1ea186b6cabd1fc116cb6c1823130756f33ff81807815ed.pdf/SHA256E-s81068--de1d8de99645d74ba1ea186b6cabd1fc116cb6c1823130756f33ff81807815ed.pdf
+
+and that contains the necessary data.
+
+So I wrote this bash script to recover some files from the corrupt repo (put there your parameters like directory names etc)
+
+    fix_obj.sh
+[[!format  bash """
+NEWDIR=new
+while IFS=  read -r -d $'\0'; 
+do
+        trueloc=`sed 's/\/annex\/objects\///g' <"$REPLY"`  
+        # if you had swap files from editors, they may appear here. Fix them in advance.
+        # echo "$REPLY"':'$trueloc
+        truefil=`locate $trueloc/$trueloc | grep work.git-copy`
+        mkdir -p $NEWDIR/`dirname $REPLY`
+        cp -p $truefil $NEWDIR/$REPLY
+done < <(find $1/* -type f -print0)
+"""]]
+
+The cycle on all files is not a simple "for dir in \`find $1\`" because of possible newlines and spaces in directories. For my directories that worked, but I'm still not sure about possible bugs in that (actually it complains several times, but seems to work). The script doesn't work in sh, but can be launched via e.g.
+
+    . fix_obj.sh programs
+
+where 'programs' is a subdirectory (without a backslash) in your git repo that you want to recover.
+
+I don't know how this situation occured and how to fix that in other way. I tried `git gc`, `git-annex fsck`, `git-annex repair`, of course cloned (git and git-annex) and other things, but that didn't help. 
+
+I've read [disaster recovery](https://git-annex.branchable.com/design/assistant/disaster_recovery/)
+
+> # git repository repair
+> There are several ways git repositories can get damanged.
+> The most common is empty files in .git/annex/objects and commits that refer to those objects. When the objects have not yet been pushed anywhere. I've several times recovered from this manually by removing the bad files and resetting to before the commits that referred to them. Then re-staging any divergence in the working tree. This could perhaps be automated.
+> ...
+> This is useful outside git-annex as well, so make it a git-recover-repository command.
+
+I think that would be nice to provide some means to recover from these situations. At least those who face the same problem as me can use the script above.
+
+### What steps will reproduce the problem?
+
+1. Create git-annex on your local machine.
+2. Clone that to a remote server. Unfortunately I don't remember the exact commands - I think that was done with the `rsync` special backend.
+3. \* Delete all the data except on the server (better not do that).
+4. Try to clone that from the server to a new machine.
+
+### What version of git-annex are you using? On what operating system?
+
+On the original machine I used `version 6` with `thin`. git-annex was downloaded from this site as a binary maybe several months ago. The OS was `Fedora Core 24`, the FS was `ext4`.
+
+On the server where I cloned the repo I noticed that the git-annex version was `5`. On the server the git-annex version is `6.20171003-g14ffdd779`. The OS is `CentOS Linux 7 (Core)` (a virtual private server).
+
+### Please provide any additional information below.
+
+I managed to recover a unique part of my data, however I don't know how the repo could be recovered (which would be best). I will combine the data again from available pieces. 
+
+I'd also like to add that I'm not a git expert, I use quite few commands from that and now I may know it not better than git-annex. Only recently I realized that `special remotes` don't have a copy of the git repository (I found this only [here](http://git-annex.branchable.com/special_remotes/rsync/#comment-525d3951ab1f09fdf471f450a798b50e) and still can't see that on [special remotes](http://git-annex.branchable.com/special_remotes/) page). That would be great if we could understand not only basic things about e.g. special remotes, but also the underlying facts about git-annex, to better understand possible problems. It's not intuitive that some repositories are cloned via `git clone`, and some via `git-annex initremote` etc, or that could better pronounced if that is the only difference, otherwise things mess up: I still don't quite understand what would be the difference between `git-annex get .` and `git-annex sync --content` (because the latter showed me that my repo above was synced - even though it really missed the files I needed). 
+
+git-annex version 6 seems very promising, but I have a feeling that the documentation for the project should be a bit rewritten/restructured, because when I read comments from several years ago, I can't judge whether that is still appropriate or not. Sorry if I should had submitted that in another bug on the documentation. 
+
+And maybe one should rename my question, because I can't locate precisely the problem and mostly just used the keywords for a better search.
+
+### Have you had any luck using git-annex before? (Sometimes we get tired of reading bug reports all day and a lil' positive end note does wonders)
+
+Yes, I have a working git-annex repository on a server (with `gcrypt`) and on two laptops of mine. That works fine, though [not blazingly fast](http://git-annex.branchable.com/bugs/v6__58___git_status__47__add_after_unlock_is_linear_to_the_file_size/#comment-5b8baf74551f8cc192d5d87b8d6aefa3). 
+> and a lil' positive end note
+
+I think there are no other variants for me. A year ago I tried some 'out-of-the box' popular alternatives, but the only use from that was to better learn SELinux and process isolation. Now I think the best is the \*NIX style where you control everything and you can learn. I've spent many days learning git-annex and hope that will work (though the last link above). 
diff --git a/doc/bugs/fix_git-annex_paths___47___objects___40__repository_not_available__41__/comment_1_70e050ef94ab57e74b45b18f3f9ca1f3._comment b/doc/bugs/fix_git-annex_paths___47___objects___40__repository_not_available__41__/comment_1_70e050ef94ab57e74b45b18f3f9ca1f3._comment
new file mode 100644
index 0000000000..bae013a871
--- /dev/null
+++ b/doc/bugs/fix_git-annex_paths___47___objects___40__repository_not_available__41__/comment_1_70e050ef94ab57e74b45b18f3f9ca1f3._comment
@@ -0,0 +1,49 @@
+[[!comment format=mdwn
+ username="joey"
+ subject="""comment 1"""
+ date="2017-10-25T19:13:16Z"
+ content="""
+The strance output you pasted shows git-annex checking the *local*
+repository to see if it has a copy of the file. As far as I can see, that
+has nothing to do with any failure to get the file from a remote
+repository. 
+
+> Clone that to a remote server. Unfortunately I don't remember the exact commands - I think that was done with the rsync special backend.
+
+Are you talking about a git-annex rsync special remote,
+or a git repository accessed via a rsync:// url, or what?
+
+> /home/yaroslav/work.git/d0d/994/SHA256E-s81068--de1d8de99645d74ba1ea186b6cabd1fc116cb6c1823130756f33ff81807815ed.pdf/SHA256E-s81068--de1d8de99645d74ba1ea186b6cabd1fc116cb6c1823130756f33ff81807815ed.pdf
+
+What this path tells me is that, despite this being a bare git repository,
+the content of objects is not in work.git/annex/objects/ where it belongs,
+but at the top of the repository.
+
+Hypothesis: You created this bare git repository. But then rather than
+simply having git-annex send contents of files to it in the sane and
+regular way, you them proceeded to set up some kind of special remote,
+in the same directory. It could have been a rsync special remote, or
+a directory special remote. You copied the files to that special remote.
+
+Then, you deleted the original git repository, which was the only one that
+knew about the special remote, before syncing it to anywhere.
+
+That fits all the evidence. I don't think that's a mistake many people
+are likely to make.
+
+It should be possible to recover from this situation by:
+
+	cd /home/yaroslav/work.git/
+	mkdir -p annex/objects
+	mv d0d annex/objects
+	# also move any other 3 letter directories that have annex
+	# objects in them
+	git annex fsck
+
+Then after `git annex sync` in your clone, it should know where the content
+is and be able to get it from the bare repo.
+
+Assuming that works, I'm having a hard time treating this as any kind
+of bug in git-annex. You put the gun in a vise, bent its barrel
+to a 180 degree angle, and... ;)
+"""]]
diff --git a/doc/bugs/fix_git-annex_paths___47___objects___40__repository_not_available__41__/comment_2_35df3798ddb3424fb26e11df53420d2d._comment b/doc/bugs/fix_git-annex_paths___47___objects___40__repository_not_available__41__/comment_2_35df3798ddb3424fb26e11df53420d2d._comment
new file mode 100644
index 0000000000..547fe70095
--- /dev/null
+++ b/doc/bugs/fix_git-annex_paths___47___objects___40__repository_not_available__41__/comment_2_35df3798ddb3424fb26e11df53420d2d._comment
@@ -0,0 +1,13 @@
+[[!comment format=mdwn
+ username="ynikitenko"
+ avatar="http://cdn.libravatar.org/avatar/168d629704097ddc596f75ca32a687a3"
+ subject="thanks"
+ date="2017-10-29T16:28:33Z"
+ content="""
+Dear Joey, many thanks for your useful reply. Now I understand that better. I tried your fix and it worked - no there are no complaints and the files are being synced.
+
+However, in my /annex/objects there is 1.9Gb, while in the cloned repo there are only 20 Mb of data synced. Do you have any ideas of what could be done? Should I ask at the forum? Many thanks anyway. 
+
+Now I think this is not a bug of git-annex, but I still think that the documentation for that could be improved. Should I delete this report, move it to the forum or anything?
+
+"""]]
diff --git a/doc/bugs/fix_git-annex_paths___47___objects___40__repository_not_available__41__/comment_3_584fce9a8a501e380426c123505bfdf9._comment b/doc/bugs/fix_git-annex_paths___47___objects___40__repository_not_available__41__/comment_3_584fce9a8a501e380426c123505bfdf9._comment
new file mode 100644
index 0000000000..e7a1704bf3
--- /dev/null
+++ b/doc/bugs/fix_git-annex_paths___47___objects___40__repository_not_available__41__/comment_3_584fce9a8a501e380426c123505bfdf9._comment
@@ -0,0 +1,10 @@
+[[!comment format=mdwn
+ username="joey"
+ subject="""comment 3"""
+ date="2017-10-30T15:48:34Z"
+ content="""
+You could try running `git annex unused` in the repository of unusual size
+and see if it perhaps finds something to remove.
+
+Always happy to improve documentation if you have ideas.
+"""]]
diff --git a/doc/bugs/forwarded.mdwn b/doc/bugs/forwarded.mdwn
new file mode 100644
index 0000000000..1a544a4f7d
--- /dev/null
+++ b/doc/bugs/forwarded.mdwn
@@ -0,0 +1,2 @@
+This tag is for bugs that have been forwarded from git-annex to some other
+software, such as a library it uses.
diff --git a/doc/bugs/fsck_does_not_detect__47__fix_some_key_directories_correctly.mdwn b/doc/bugs/fsck_does_not_detect__47__fix_some_key_directories_correctly.mdwn
new file mode 100644
index 0000000000..8d6e304e4c
--- /dev/null
+++ b/doc/bugs/fsck_does_not_detect__47__fix_some_key_directories_correctly.mdwn
@@ -0,0 +1,43 @@
+### Please describe the problem.
+
+ATM I am chasing a problem that somehow one key "mutated" although I do not remember doing anything malicious, file seems to be also not  writable (itself).
+So I decided to fsck, and only spotted a problem when some warnings started to appear that I am not the owner of the (key)file.  So I looked inside and found
+that all key dirs are writeable BUT annex complains only about the ones where it can't change permissions since they don't belong to me
+
+
+### What version of git-annex are you using? On what operating system?
+
+6.20170815+gitg22da64d0f-1~ndall+1
+
+### Please provide any additional information below.
+
+[[!format sh """
+$> for f in sub00{1,3}/anatomy/highres001.nii.gz; do ls -ld $(realpath $f | xargs dirname ); git annex fsck $f; done 
+drwxrws--- 2 yoh famface 3 May 12 15:15 /data/famface/openfmri/data/.git/annex/objects/08/V3/SHA256E-s6498717--b850ac82ec9db2d399962609e9381d9c2bdf1f426012500b7005b173ea4d9102.nii.gz/
+fsck sub001/anatomy/highres001.nii.gz (checksum...) ok
+(recording state in git...)
+drwxrwsr-x 2 contematto famface 3 Jul 13  2015 /data/famface/openfmri/data/.git/annex/objects/x2/xX/SHA256E-s6592524--dccba651dc4cd104826a05a2efb6a257b39ca2d8b44d215027250221729f9434.nii.gz/
+fsck sub003/anatomy/highres001.nii.gz 
+  ** Unable to set correct write mode for .git/annex/objects/x2/xX/SHA256E-s6592524--dccba651dc4cd104826a05a2efb6a257b39ca2d8b44d215027250221729f9434.nii.gz/SHA256E-s6592524--dccba651dc4cd104826a05a2efb6a257b39ca2d8b44d215027250221729f9434.nii.gz ; perhaps you don't own that file
+(checksum...) ok
+(recording state in git...)
+
+# thought to see may be annex would complain then!?
+$> chmod a+rwx /data/famface/openfmri/data/.git/annex/objects/08/V3/SHA256E-s6498717--b850ac82ec9db2d399962609e9381d9c2bdf1f426012500b7005b173ea4d9102.nii.gz/
+
+$> for f in sub00{1,3}/anatomy/highres001.nii.gz; do ls -ld $(realpath $f | xargs dirname ); git annex fsck $f; done                                          
+drwxrwsrwx 2 yoh famface 3 May 12 15:15 /data/famface/openfmri/data/.git/annex/objects/08/V3/SHA256E-s6498717--b850ac82ec9db2d399962609e9381d9c2bdf1f426012500b7005b173ea4d9102.nii.gz/
+fsck sub001/anatomy/highres001.nii.gz (checksum...) ok
+(recording state in git...)
+drwxrwsr-x 2 contematto famface 3 Jul 13  2015 /data/famface/openfmri/data/.git/annex/objects/x2/xX/SHA256E-s6592524--dccba651dc4cd104826a05a2efb6a257b39ca2d8b44d215027250221729f9434.nii.gz/
+fsck sub003/anatomy/highres001.nii.gz 
+  ** Unable to set correct write mode for .git/annex/objects/x2/xX/SHA256E-s6592524--dccba651dc4cd104826a05a2efb6a257b39ca2d8b44d215027250221729f9434.nii.gz/SHA256E-s6592524--dccba651dc4cd104826a05a2efb6a257b39ca2d8b44d215027250221729f9434.nii.gz ; perhaps you don't own that file
+(checksum...) ok
+(recording state in git...)
+
+"""]]
+
+~~btw -- the same wrong permissions on the upper hash directories, and they do not get fixed/complained about at all (that is ok)~~
+
+
+[[!meta author=yoh]]
diff --git a/doc/bugs/fsck_does_not_detect__47__fix_some_key_directories_correctly/comment_1_1d627cb46d9ecdf0b526a9c8e9764011._comment b/doc/bugs/fsck_does_not_detect__47__fix_some_key_directories_correctly/comment_1_1d627cb46d9ecdf0b526a9c8e9764011._comment
new file mode 100644
index 0000000000..92fbb3ba59
--- /dev/null
+++ b/doc/bugs/fsck_does_not_detect__47__fix_some_key_directories_correctly/comment_1_1d627cb46d9ecdf0b526a9c8e9764011._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="yarikoptic"
+ avatar="http://cdn.libravatar.org/avatar/f11e9c84cb18d26a1748c33b48c924b4"
+ subject="comment 1"
+ date="2017-09-13T14:32:57Z"
+ content="""
+also note, that even though I am not the owner, I should have sufficient privileges to modify (member of the group(s))!
+"""]]
diff --git a/doc/bugs/fsck_does_not_detect__47__fix_some_key_directories_correctly/comment_2_eb7c6445b1ca53d5552506d8ae93b5d4._comment b/doc/bugs/fsck_does_not_detect__47__fix_some_key_directories_correctly/comment_2_eb7c6445b1ca53d5552506d8ae93b5d4._comment
new file mode 100644
index 0000000000..5d708f871d
--- /dev/null
+++ b/doc/bugs/fsck_does_not_detect__47__fix_some_key_directories_correctly/comment_2_eb7c6445b1ca53d5552506d8ae93b5d4._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="joey"
+ subject="""comment 2"""
+ date="2017-09-13T16:12:22Z"
+ content="""
+I think you must have core.sharedRepository set to group or all or
+something like that, otherwise fsck never complains about modes.
+"""]]
diff --git a/doc/bugs/fsck_does_not_detect__47__fix_some_key_directories_correctly/comment_3_c7d43b06f88d2000fcf574ebab971ae1._comment b/doc/bugs/fsck_does_not_detect__47__fix_some_key_directories_correctly/comment_3_c7d43b06f88d2000fcf574ebab971ae1._comment
new file mode 100644
index 0000000000..162052573b
--- /dev/null
+++ b/doc/bugs/fsck_does_not_detect__47__fix_some_key_directories_correctly/comment_3_c7d43b06f88d2000fcf574ebab971ae1._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="yarikoptic"
+ avatar="http://cdn.libravatar.org/avatar/f11e9c84cb18d26a1748c33b48c924b4"
+ subject="comment 3"
+ date="2017-09-13T17:58:54Z"
+ content="""
+yeap -- sharedrepository=1 I have.  should be legit right?
+"""]]
diff --git a/doc/bugs/fsck_does_not_detect__47__fix_some_key_directories_correctly/comment_4_9d7c83cfb6154bdfc51caae00da4c4b7._comment b/doc/bugs/fsck_does_not_detect__47__fix_some_key_directories_correctly/comment_4_9d7c83cfb6154bdfc51caae00da4c4b7._comment
new file mode 100644
index 0000000000..af7840112e
--- /dev/null
+++ b/doc/bugs/fsck_does_not_detect__47__fix_some_key_directories_correctly/comment_4_9d7c83cfb6154bdfc51caae00da4c4b7._comment
@@ -0,0 +1,38 @@
+[[!comment format=mdwn
+ username="joey"
+ subject="""comment 4"""
+ date="2017-10-02T20:07:23Z"
+ content="""
+In general, it's out of scope for fsck to make file permission sane,
+because "sane" has a fairly broad set of definitions when it comes
+to file permissions!
+
+See [[!commit bd516af734bf5e1f7a3d43c7e4dd0f6fb9fd5919]] for the
+backstory about why fsck wants to fix this one particular permission.
+In short, old versions of git-annex didn't set the write bit of content files
+in a shared repo, which prevented git-annex from locking the content
+files, which prevents dropping them or locking them to prevent removal
+while dropping them from another repo. So fsck is fixing up from that
+situation.
+
+With core.sharedrepository=1, isContentWritePermOk 
+wants owner and group write bits to be set on the content file.
+
+I can reproduce what seems to be the same problem as follows:
+
+	drwxrwxr-x 2 root netdev 4096 Oct  2 16:20 .git/annex/objects/2K/7F/SHA256E-s30--0d18b66aa72aeecad450963ec8f8951f4f5d27fc965d56bdfd3ff75e6d42ea3e/
+	-rw-r--r-- 1 root netdev 30 Oct  2 16:20 .git/annex/objects/2K/7F/SHA256E-s30--0d18b66aa72aeecad450963ec8f8951f4f5d27fc965d56bdfd3ff75e6d42ea3e/SHA256E-s30--0d18b66aa72aeecad450963ec8f8951f4f5d27fc965d56bdfd3ff75e6d42ea3e
+
+When I fsck as user joey, who is in group netdev, it complains it can't
+fix the permissions.
+
+While joey has write access to the directory containing
+the content file, this does not allow changing the permissions of the file.
+
+The directory perms do allow deleting the file and replacing it with a copy
+that has the permissions I want. But, that is an expensive operation, needing
+to copy a perhaps enormous file. So I don't think it's a reasonable thing for
+fsck to do.
+
+So, it seems to me that fsck complaining is ok.
+"""]]
diff --git a/doc/bugs/fsck_does_not_detect__47__fix_some_key_directories_correctly/comment_5_09bb754f4e7e19cd52d68f91368f57fc._comment b/doc/bugs/fsck_does_not_detect__47__fix_some_key_directories_correctly/comment_5_09bb754f4e7e19cd52d68f91368f57fc._comment
new file mode 100644
index 0000000000..8356dc2745
--- /dev/null
+++ b/doc/bugs/fsck_does_not_detect__47__fix_some_key_directories_correctly/comment_5_09bb754f4e7e19cd52d68f91368f57fc._comment
@@ -0,0 +1,12 @@
+[[!comment format=mdwn
+ username="yarikoptic"
+ avatar="http://cdn.libravatar.org/avatar/f11e9c84cb18d26a1748c33b48c924b4"
+ subject="comment 5"
+ date="2017-12-05T18:26:43Z"
+ content="""
+FWIW regarding
+
+> But, that is an expensive operation, needing to copy a perhaps enormous file. So I don't think it's a reasonable thing for fsck to do.
+
+I prefer machines to do the job instead of me, even though they need to sweat may be more than I for some of those ;)  not sure if needs to be an option or default behavior. But altogether, I am trying to get away from shared mode (ref: http://git-annex.branchable.com/bugs/--shared_setting_of_git_causes_annex__39__ed_files_to_be_writeable__33__/) with write permissions set on fiels since it is simply too dangerous if that repository is the single source of correct data/files.
+"""]]
diff --git a/doc/bugs/g-a_move_has_force_option_described_twice.mdwn b/doc/bugs/g-a_move_has_force_option_described_twice.mdwn
new file mode 100644
index 0000000000..93e7a2896d
--- /dev/null
+++ b/doc/bugs/g-a_move_has_force_option_described_twice.mdwn
@@ -0,0 +1,20 @@
+The **force** option in [[git-annex-move]] is described twice.
+
+* `--force`
+
+  Override numcopies and required content checking, and always remove
+  files from the source repository once the destination repository has a
+  copy.
+
+  Note that, even without this option, you can move the content of a file
+  from one repository to another when numcopies is not satisfied, as long
+  as the move does not result in there being fewer copies.
+
+* `--force`
+
+  When moving content from a remote, ignore location tracking information
+  and always check if the remote has content. Can be useful if the location
+  tracking information is out of date.
+
+> Removed the latter, and as far as I could gather on irc, the desired
+> behavior was the former. So, [[done]] --[[Joey]]
diff --git a/doc/bugs/g-a_move_has_force_option_described_twice/comment_1_1ff90a7b1df3ebad2e1f17726d7d4fd7._comment b/doc/bugs/g-a_move_has_force_option_described_twice/comment_1_1ff90a7b1df3ebad2e1f17726d7d4fd7._comment
new file mode 100644
index 0000000000..140c1ff52a
--- /dev/null
+++ b/doc/bugs/g-a_move_has_force_option_described_twice/comment_1_1ff90a7b1df3ebad2e1f17726d7d4fd7._comment
@@ -0,0 +1,23 @@
+[[!comment format=mdwn
+ username="joey"
+ subject="""comment 1"""
+ date="2018-05-21T16:42:51Z"
+ content="""
+Worse, it's forcing two unrelated behaviors and the user may only
+want one of them.
+
+The numcopies check override --force was only recently
+added.
+
+The location tracking ignore --force is older, but is unlike
+other uses of --force in git-annex. It's so off the radar that it was an
+undocumented option from 2013 until I noticed the documentation was missing
+in 2017! Also, it's not clear if/when anyone would use that. Seems like
+running `git annex sync` first would be more efficient and have the same
+result.
+
+So, I'm inclined to keep the numcopies check --force, which matches other
+uses of --force in git-annex (eg drop --force), and remove the other
+--force behavior, or make it be a differently named option. --slow
+would be a good name, by contrast with move --fast.
+"""]]
diff --git a/doc/bugs/g-a_move_has_force_option_described_twice/comment_2_71eed2cf7fd21634a2b60ef5cd9ccb93._comment b/doc/bugs/g-a_move_has_force_option_described_twice/comment_2_71eed2cf7fd21634a2b60ef5cd9ccb93._comment
new file mode 100644
index 0000000000..daa49e540c
--- /dev/null
+++ b/doc/bugs/g-a_move_has_force_option_described_twice/comment_2_71eed2cf7fd21634a2b60ef5cd9ccb93._comment
@@ -0,0 +1,32 @@
+[[!comment format=mdwn
+ username="joey"
+ subject="""comment 2"""
+ date="2018-05-21T17:04:56Z"
+ content="""
+Actually the --force documentation re location tracking is not a good
+description of the behavior. It only affects --from, and with --force
+it still contacts the remote to check if it has the content. Only
+if that check fails to return a result (ie, the remote can't be contacted),
+does it assume that the remote has the content, rather than the default
+behavior of falling back to looking at the location tracking information.
+
+It's hard to see how that could be useful at all; if it fails in
+communication with the remote, presumably the content transfer will later
+fail as well.
+
+The only actual behavior change would be when the remote cannot be
+contacted, and the location tracking information says it does not contain a
+file. Then `move --force --from remote` will fail, because it tries
+to perform the move and can't contact the remote, while `move --from
+remote` will succeed, because it assumes the location tracking is right.
+
+That does not seem a useful distinction. And that was the behavior all the
+way back to the first commit of this "feature" in 2013.
+
+So, removing that.
+
+----
+
+There may be room for a new option that actually does whatever you were
+hoping --force did. So let's talk about that..
+"""]]
diff --git a/doc/bugs/gcrypt__58___..but_repository_ID_is_set._Aborting..mdwn b/doc/bugs/gcrypt__58___..but_repository_ID_is_set._Aborting..mdwn
new file mode 100644
index 0000000000..eb1c1d6465
--- /dev/null
+++ b/doc/bugs/gcrypt__58___..but_repository_ID_is_set._Aborting..mdwn
@@ -0,0 +1,45 @@
+### Please describe the problem.
+After month of flawless operations, a gcrypted special (rsync) remote has suddenly stopped working. Below is what since happens on every git annex sync.
+
+    ...
+    push rsync.net 
+    gcrypt: Repository not found: ssh://usw-a123.rsync.net/data1/home/XXXX/annex/files
+    gcrypt: ..but repository ID is set. Aborting.
+
+At the file system level, the remote appears to be intact.
+
+    foo@bar:~$ ssh usw-a123.rsync.net ls -lah annex/files/
+    total 175
+    drwxr-xr-x    8 XXXX  XXXX    11B Sep  8 17:23 .
+    drwxr-xr-x    3 XXXX  XXXX     3B Mar 25 11:42 ..
+    -rw-r--r--    1 XXXX  XXXX    23B Mar 22 18:25 HEAD
+    drwxr-xr-x    3 XXXX  XXXX     3B Mar 22 18:26 annex
+    drwxr-xr-x    2 XXXX  XXXX     2B Mar 22 18:25 branches
+    -rw-r--r--    1 XXXX  XXXX   143B Mar 30 10:12 config
+    -rw-r--r--    1 XXXX  XXXX    73B Mar 22 18:25 description
+    drwxr-xr-x    2 XXXX  XXXX    12B Mar 22 18:25 hooks
+    drwxr-xr-x    2 XXXX  XXXX     3B Mar 22 18:25 info
+    drwxr-xr-x  255 XXXX  XXXX   255B Sep  8 04:56 objects
+    drwxr-xr-x    4 XXXX  XXXX     4B Mar 22 18:25 refs
+
+
+### What steps will reproduce the problem?
+Unknown
+
+### What version of git-annex are you using? On what operating system?
+* git-annex-standalone  6.20180807+git230-gaa291acfe-1~ndall+1 amd64  (Neurodebian)
+* Ubuntu 18.04
+
+### Please provide any additional information below.
+
+[[!format sh """
+# If you can, paste a complete transcript of the problem occurring here.
+# If the problem is with the git-annex assistant, paste in .git/annex/daemon.log
+
+
+# End of transcript or log.
+"""]]
+
+### Have you had any luck using git-annex before? (Sometimes we get tired of reading bug reports all day and a lil' positive end note does wonders)
+
+I am a vocal advocate of git-annex. :-)
diff --git a/doc/bugs/gcrypt__58___..but_repository_ID_is_set._Aborting./comment_1_ec28718db57922b7d0c01f3ef672e6d4._comment b/doc/bugs/gcrypt__58___..but_repository_ID_is_set._Aborting./comment_1_ec28718db57922b7d0c01f3ef672e6d4._comment
new file mode 100644
index 0000000000..de25721ddc
--- /dev/null
+++ b/doc/bugs/gcrypt__58___..but_repository_ID_is_set._Aborting./comment_1_ec28718db57922b7d0c01f3ef672e6d4._comment
@@ -0,0 +1,9 @@
+[[!comment format=mdwn
+ username="git-annex.branchable.com@79d6855760f61f7fbe0a401b45d8c791ef49b500"
+ nickname="git-annex.branchable.com"
+ avatar="http://cdn.libravatar.org/avatar/4bf61f9feda20e8b4fc09d52ee48af39"
+ subject="comment 1"
+ date="2018-09-14T16:42:59Z"
+ content="""
+It turns out that rsync.net silently changed the (absolute) path to my user's home directory and the path in .git/config was thus incorrect.
+"""]]
diff --git a/doc/bugs/gcrypt__58___..but_repository_ID_is_set._Aborting./comment_2_e4d97ac757e7da09c9c23f74177991ae._comment b/doc/bugs/gcrypt__58___..but_repository_ID_is_set._Aborting./comment_2_e4d97ac757e7da09c9c23f74177991ae._comment
new file mode 100644
index 0000000000..c103ce3842
--- /dev/null
+++ b/doc/bugs/gcrypt__58___..but_repository_ID_is_set._Aborting./comment_2_e4d97ac757e7da09c9c23f74177991ae._comment
@@ -0,0 +1,10 @@
+[[!comment format=mdwn
+ username="joey"
+ subject="""comment 2"""
+ date="2018-09-14T16:52:56Z"
+ content="""
+This is gcrypt displaying an error message, not git-annex.
+
+Do you have any indication it's somehow a git-annex bug? The obvious guess
+would be that it's a gcrypt bug.
+"""]]
diff --git a/doc/bugs/gcrypt__58___WARNING__58___Remote_ID_has_changed__33__.mdwn b/doc/bugs/gcrypt__58___WARNING__58___Remote_ID_has_changed__33__.mdwn
new file mode 100644
index 0000000000..556a468263
--- /dev/null
+++ b/doc/bugs/gcrypt__58___WARNING__58___Remote_ID_has_changed__33__.mdwn
@@ -0,0 +1,45 @@
+### Please describe the problem.
+
+For some reason, after a while I can't sync to any remotes (removeable drives, servers, etc.)
+
+
+### What steps will reproduce the problem?
+
+git annex sync, git annex assistant doing its autosync, etc.
+
+
+### What version of git-annex are you using? On what operating system?
+
+Arch Linux, git annex 6.20160613-g1e4e6f4
+
+
+### Please provide any additional information below.
+
+[[!format sh """
+# If you can, paste a complete transcript of the problem occurring here.
+
+Typing git annex synx in a terminal gives me:
+
+commit  ok
+pull disk0_annexes_docs 
+gcrypt: Decrypting manifest
+gpg: Signature made Mon 25 Jul 2016 00:13:00 BST using RSA key ID XXXXXXXXXXXXXXX
+gpg: Good signature from "XXX " [ultimate]
+gcrypt: WARNING:
+gcrypt: WARNING: Remote ID has changed!
+gcrypt: WARNING: from :id:lczpGq00kb1sF+cxgCNE
+gcrypt: WARNING: to   :id:2br4BOnJZkrGUsXTlQR8
+
+
+# If the problem is with the git-annex assistant, paste in .git/annex/daemon.log
+
+Not specific to assistant.
+
+
+# End of transcript or log.
+"""]]
+
+
+### Have you had any luck using git-annex before? (Sometimes we get tired of reading bug reports all day and a lil' positive end note does wonders)
+
+Yes, this worked for ages, a year or more, it's awesome! Haven't plugged the removeable drives in for a while, now when I do this happens...
diff --git a/doc/bugs/gcrypt_remote__58___every_sync_uploads_huge_manifest.mdwn b/doc/bugs/gcrypt_remote__58___every_sync_uploads_huge_manifest.mdwn
new file mode 100644
index 0000000000..4f2abacc0b
--- /dev/null
+++ b/doc/bugs/gcrypt_remote__58___every_sync_uploads_huge_manifest.mdwn
@@ -0,0 +1,30 @@
+### Please describe the problem.
+
+every sync (without --content) to a gcrypt remote uploads huge amount of data (>100MB) when doing
+
+    gcrypt: Requesting manifest signature
+
+It seems to upload a special git object every time, even if this object is apparently unchanged. An unencrypted, regular git remote is much faster and does not transfer such amounts of data.
+
+I wonder if this can be changed, because it renders that gcrypt remote almost unusable via ADSL upstream. In my case the sync duration was 36 Minutes, uploading ~250MB.
+
+### What steps will reproduce the problem?
+
+Have a (bare) gcrypt remote and a rather big (mine has 77668 keys, annexing 769GB of files) git-annex repository. Sync with the gcrypt remote. When pushing, the message "gcrypt: Requesting manifest signature" appears, and a very large amount of data is transferred to the remote, while the process chain
+
+    git-remote-gcrypt mygcrypt ssh://mygcrypt/home/my/annex
+     git push --quiet -f ssh://mygcrypt/home/my/annex refs/gcrypt/gitception+:refs/heads/master
+      ssh mygcrypt git-receive-pack '/home/my/annex'
+       git pack-objects --all-progress-implied --revs --stdout --thin --delta-base-offset -q
+
+is running. The upload of the actual changeset starts after this, the processes look the same, transferring again a more or less big amount of data (depending on the changeset size, I guess).
+
+### What version of git-annex are you using? On what operating system?
+
+6.20170101-1 on Debian Stretch (9.0)
+
+### Have you had any luck using git-annex before? (Sometimes we get tired of reading bug reports all day and a lil' positive end note does wonders)
+
+git-annex is great and revolutionized my file organization and backup structure (if they were even existing before)
+
+[[!meta tite="gcrypt special remotes should support rsync:// and perhaps also sftp://"]]
diff --git a/doc/bugs/gcrypt_remote__58___every_sync_uploads_huge_manifest/comment_1_45982ced836d2e0f41a5ddd7edd59936._comment b/doc/bugs/gcrypt_remote__58___every_sync_uploads_huge_manifest/comment_1_45982ced836d2e0f41a5ddd7edd59936._comment
new file mode 100644
index 0000000000..065ca658bc
--- /dev/null
+++ b/doc/bugs/gcrypt_remote__58___every_sync_uploads_huge_manifest/comment_1_45982ced836d2e0f41a5ddd7edd59936._comment
@@ -0,0 +1,13 @@
+[[!comment format=mdwn
+ username="joey"
+ subject="""comment 1"""
+ date="2017-04-05T15:44:47Z"
+ content="""
+I don't think this is a bug in git-annex. That is the behavior of
+`git-remote-gcrypt`. Could you please file a bug on that program instead?
+
+(I've also wondered about this; it could be that it's re-uploading the
+manifest every time for good security reasons, or it could be that a better
+file structure would allow more incremental uploads. It certianly
+seems like it could avoid re-uploading when nothing has changed!)
+"""]]
diff --git a/doc/bugs/gcrypt_remote__58___every_sync_uploads_huge_manifest/comment_2_02ae97849e2d9fc6d3d996500f264455._comment b/doc/bugs/gcrypt_remote__58___every_sync_uploads_huge_manifest/comment_2_02ae97849e2d9fc6d3d996500f264455._comment
new file mode 100644
index 0000000000..d9174ceb14
--- /dev/null
+++ b/doc/bugs/gcrypt_remote__58___every_sync_uploads_huge_manifest/comment_2_02ae97849e2d9fc6d3d996500f264455._comment
@@ -0,0 +1,17 @@
+[[!comment format=mdwn
+ username="git-annex@6f13b739194f758abc0b86556b7ce966c1bf3c00"
+ nickname="git-annex"
+ avatar="http://cdn.libravatar.org/avatar/198790d74209efe4896fd4cfc37ec2a6"
+ subject="git-remote-gcrypt recommends rsync:// or sftp:// transports"
+ date="2017-04-05T16:16:25Z"
+ content="""
+spwhitton says on :
+
+> \"Using an arbitrary  requires uploading the entire repository history with
+> each push. If your repository history is large or you are pushing over a slow link, consider using
+> either the rsync:// or sftp:// transports, which perform incremental pushes\"
+
+So it's a known performance. Would be great if rsync:// could be used when combining git-annex with
+git-remote-gcrypt?
+
+"""]]
diff --git a/doc/bugs/gcrypt_remote__58___every_sync_uploads_huge_manifest/comment_3_83fd8643b988fdf689ef40b819b48299._comment b/doc/bugs/gcrypt_remote__58___every_sync_uploads_huge_manifest/comment_3_83fd8643b988fdf689ef40b819b48299._comment
new file mode 100644
index 0000000000..2f46e29626
--- /dev/null
+++ b/doc/bugs/gcrypt_remote__58___every_sync_uploads_huge_manifest/comment_3_83fd8643b988fdf689ef40b819b48299._comment
@@ -0,0 +1,20 @@
+[[!comment format=mdwn
+ username="joey"
+ subject="""comment 3"""
+ date="2017-04-05T18:35:30Z"
+ content="""
+It should be possible for git-annex to support rsync:// for gcrypt special
+remotes; that would just need to reuse the rsync special remote for the
+git-annex objects. Retitling this bug report appropriately.
+
+In the meantime, it should work to set up a gcrypt git remote (not a
+git-annex special remote) using rsync:// or sftp:// and then use a
+git-annex rsync special remote on the same server to store the annex
+objects.
+
+But that doesn't help with large pushes to gcrypt remotes
+when git hosting providers are being used, which is a main 
+use case for using gcrypt (though generally not the gcrypt special remote).
+The lack of incrementals there seems like something worth finding a way to
+fix in gcrypt.
+"""]]
diff --git a/doc/bugs/gcrypt_remote__58___every_sync_uploads_huge_manifest/comment_4_0f82673281494b1cb084dce702525a01._comment b/doc/bugs/gcrypt_remote__58___every_sync_uploads_huge_manifest/comment_4_0f82673281494b1cb084dce702525a01._comment
new file mode 100644
index 0000000000..0245a0b166
--- /dev/null
+++ b/doc/bugs/gcrypt_remote__58___every_sync_uploads_huge_manifest/comment_4_0f82673281494b1cb084dce702525a01._comment
@@ -0,0 +1,16 @@
+[[!comment format=mdwn
+ username="spwhitton"
+ avatar="http://cdn.libravatar.org/avatar/9c3f08f80e67733fd506c353239569eb"
+ subject="comment 4"
+ date="2017-04-05T22:28:10Z"
+ content="""
+git-remote-gcrypt maintainer here.
+
+As Joey recommends for the meantime, I am successfully using an `rsync://` gcrypt remote plus a separate encrypted git-annex rsync remote to work around these performance issues.
+
+git-remote-gcrypt relies on rsync to implement the incremental upload, so the README is wrong to suggest that using an `sftp://` remote would work around this issue -- git-remote-gcrypt invokes curl for the sftp transaction, which as far as I know does nothing incremental (that's presumably why rsync exists).  I've just updated the README.
+
+If we wanted the gitception gcrypt remote to be incremental, we would need to implement rsync-like incremental uploads on top of the structure of git commits, such that we could push a git commit that represents the changes to the gcrypt packfiles and manifest since the previous commit.  But I don't think this is possible for binary files -- I don't think git can represent the deltas efficiently.
+
+I've come to think that git-remote-gcrypt's gitception mode is not actually very useful, simply due to the design of git.  But perhaps there is an alternative way to represent the manifest and packfiles that would be compatible with incremental git pushes.
+"""]]
diff --git a/doc/bugs/gcrypt_remote__58___every_sync_uploads_huge_manifest/comment_5_39d905d4577c9b2987bf5e6cdbace7f2._comment b/doc/bugs/gcrypt_remote__58___every_sync_uploads_huge_manifest/comment_5_39d905d4577c9b2987bf5e6cdbace7f2._comment
new file mode 100644
index 0000000000..72de2a5189
--- /dev/null
+++ b/doc/bugs/gcrypt_remote__58___every_sync_uploads_huge_manifest/comment_5_39d905d4577c9b2987bf5e6cdbace7f2._comment
@@ -0,0 +1,10 @@
+[[!comment format=mdwn
+ username="woffs"
+ avatar="http://cdn.libravatar.org/avatar/198790d74209efe4896fd4cfc37ec2a6"
+ subject="comment 5"
+ date="2017-04-06T07:29:14Z"
+ content="""
+I remember having tried this (gcrypt rsync:// git remote + rsync encrypted git-annex specialremote) but I was disappointed because git-annex-sync does not pull/push the git remote and I had to do this separately every time.
+
+Anyway, thank you for caring. :-)
+"""]]
diff --git a/doc/bugs/gcrypt_remote__58___every_sync_uploads_huge_manifest/comment_6_a7c02a4dfa74de8ad05bfaaee0b335b8._comment b/doc/bugs/gcrypt_remote__58___every_sync_uploads_huge_manifest/comment_6_a7c02a4dfa74de8ad05bfaaee0b335b8._comment
new file mode 100644
index 0000000000..a92c9ac9ae
--- /dev/null
+++ b/doc/bugs/gcrypt_remote__58___every_sync_uploads_huge_manifest/comment_6_a7c02a4dfa74de8ad05bfaaee0b335b8._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="woffs"
+ avatar="http://cdn.libravatar.org/avatar/198790d74209efe4896fd4cfc37ec2a6"
+ subject="comment 6"
+ date="2017-04-06T08:28:06Z"
+ content="""
+I was wrong. git-annex-sync DOES pull/push to the gcrypt rsync remote. So seems fine. Thank you. :)
+"""]]
diff --git a/doc/bugs/gcrypt_remote__58___every_sync_uploads_huge_manifest/comment_7_2ad1079ba07013fbb8adabc673042a8f._comment b/doc/bugs/gcrypt_remote__58___every_sync_uploads_huge_manifest/comment_7_2ad1079ba07013fbb8adabc673042a8f._comment
new file mode 100644
index 0000000000..b024787665
--- /dev/null
+++ b/doc/bugs/gcrypt_remote__58___every_sync_uploads_huge_manifest/comment_7_2ad1079ba07013fbb8adabc673042a8f._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="andrew"
+ avatar="http://cdn.libravatar.org/avatar/acc0ece1eedf07dd9631e7d7d343c435"
+ subject="gcrypt, git-annex and rsync requires absolute path"
+ date="2017-12-05T13:49:42Z"
+ content="""
+I struggled to get git-annex sync working for some time with gcrypt and the rsync transport protocol. It turns out I was using a relative path to my repo instead of an absolute path. In my .git/config with a url like this: `gcrypt::rsync://username@servername:relative-path-to-git-repo` where relative path is relative to my user's home directory, git push would work fine, but git-annex sync would complain it couldn't find the repository. Changing my url to an absolute path like `gcrypt::rsync://username@servername:absolute-path-to-git-repo` now allows both git-annex sync and git push to work. Andrew.
+"""]]
diff --git a/doc/bugs/gcrypt_repository_not_found.mdwn b/doc/bugs/gcrypt_repository_not_found.mdwn
new file mode 100644
index 0000000000..f54c050a67
--- /dev/null
+++ b/doc/bugs/gcrypt_repository_not_found.mdwn
@@ -0,0 +1,79 @@
+### Please describe the problem.
+
+I seem to be incapable of creating an encrypted git-annex repository with the instructions provided at [[tips/fully_encrypted_git_repositories_with_gcrypt/]]. The setup step fails with *gcrypt: Repository not found: (path-to-repo)*.
+
+### What steps will reproduce the problem?
+
+I *think* I naively followed the tips page, but I might be wrong. Here's the minimal reproducer I could find, but originally the problem occured with an SSH remote which I thought was the culprit.
+
+    git init a
+    git init --bare b
+    cd a
+    git annex init
+    echo > foo
+    git annex add foo
+    git commit -m'test repo' 
+    git annex initremote encrypted type=gcrypt gitrepo=~/tmp/b keyid=8DC901CE64146C048AD50FBB792152527B75921E
+
+
+### What version of git-annex are you using? On what operating system?
+
+This is Debian stretch with backports, so git-annex is actually from backports (6.20180509-1~bpo9+1):
+
+    git-annex version: 6.20180509
+    build flags: Assistant Webapp Pairing S3(multipartupload)(storageclasses) WebDAV Inotify DBus DesktopNotify ConcurrentOutput TorrentParser MagicMime Feeds Testsuite
+    dependency versions: aws-0.14.1 bloomfilter-2.0.1.0 cryptonite-0.20 DAV-1.3.1 feed-0.3.11.1 ghc-8.0.1 http-client-0.4.31.1 persistent-sqlite-2.6 torrent-10000.0.0 uuid-1.3.12 yesod-1.4.3
+    key/value backends: SHA256E SHA256 SHA512E SHA512 SHA224E SHA224 SHA384E SHA384 SHA3_256E SHA3_256 SHA3_512E SHA3_512 SHA3_224E SHA3_224 SHA3_384E SHA3_384 SKEIN256E SKEIN256 SKEIN512E SKEIN512 SHA1E SHA1 MD5E MD5 WORM URL
+    remote types: git gcrypt p2p S3 bup directory rsync web bittorrent webdav adb tahoe glacier ddar hook external
+    local repository version: unknown
+    supported repository versions: 3 5 6
+    upgrade supported from repository versions: 0 1 2 3 4 5
+    operating system: linux x86_64
+
+git-remote-gcrypt is the vanilla version from stretch (1.0.1-1) but the bug can be reproduced with the 1.1 version from sid.
+
+### Please provide any additional information below.
+
+[[!format sh """
+
+[18]anarcat@curie:tmp$ git init a
+Dépôt Git vide initialisé dans /home/anarcat/dist/tmp/a/.git/
+[19]anarcat@curie:tmp$ git init --bare b
+Dépôt Git vide initialisé dans /home/anarcat/dist/tmp/b/
+[20]anarcat@curie:tmp$ cd a
+/home/anarcat/dist/tmp/a
+[21]anarcat@curie:a$ git annex init
+init  ok
+(recording state in git...)
+[22]anarcat@curie:a$ echo > foo
+[23]anarcat@curie:a$ git annex add foo
+add foo ok
+(recording state in git...)
+[24]anarcat@curie:a$ git commit -m'test repo' 
+[master (commit racine) f759ebe] test repo
+ 1 file changed, 1 insertion(+)
+ create mode 120000 foo
+[25]anarcat@curie:a$ git annex initremote encrypted type=gcrypt gitrepo=~/tmp/b keyid=8DC901CE64146C048AD50FBB792152527B75921E
+initremote encrypted (encryption setup) (to gpg keys: 792152527B75921E) gcrypt: Repository not found: /home/anarcat/tmp/b
+gcrypt: Repository not found: /home/anarcat/tmp/b
+fatal: helper gcrypt does not support --signed=if-asked
+git-annex: unable to determine gcrypt-id of remote
+"""]]
+
+Note that this failure leaves the repository with an half-configured remote. Trying to rerun the setup gives this error:
+
+    git-annex: There is already a remote named "encrypted"
+
+Removing the remote seems to be sufficient to restore a working state and try again:
+
+    git remote rm encrypted
+
+I also note that the `encryption step` part takes a looong time when we try to reproduce the issue multiple time... I will report this in a separate issue.
+
+### Have you had any luck using git-annex before? (Sometimes we get tired of reading bug reports all day and a lil' positive end note does wonders)
+
+Of course! As you very likely know, I use git-annex daily and I'm a really happy user. :) This is, however, the first time I give gcrypt a shot.
+
+Thanks for your hard work! --[[anarcat]]
+
+Update: turns out this is a bug in git-remote-gcrypt and this probably doesn't need to be tracked in git-annex. the workaround is to comment out the `push.sign=ifAsked` entry in the git config, or to make git-remote-gcrypt ignore unknown options. so [[done]].  --[[anarcat]]
diff --git a/doc/bugs/gcrypt_repository_not_found/comment_1_40b31c79eedb59e66ef34e27ed7d137d._comment b/doc/bugs/gcrypt_repository_not_found/comment_1_40b31c79eedb59e66ef34e27ed7d137d._comment
new file mode 100644
index 0000000000..fa0a135f81
--- /dev/null
+++ b/doc/bugs/gcrypt_repository_not_found/comment_1_40b31c79eedb59e66ef34e27ed7d137d._comment
@@ -0,0 +1,12 @@
+[[!comment format=mdwn
+ username="spwhitton"
+ avatar="http://cdn.libravatar.org/avatar/9c3f08f80e67733fd506c353239569eb"
+ subject="analysis"
+ date="2018-05-17T19:49:04Z"
+ content="""
+\"Repository not found\" is expected when you have not yet pushed with gcrypt.
+
+I think the error is rather that git-annex is passing `--signed=if-asked` to git-remote-gcrypt and it does not like that, dies and never sets up the repository.
+
+I am not sure whether git-annex should stop passing that option, or git-remote-gcrypt should accept the option and do nothing with it.
+"""]]
diff --git a/doc/bugs/gcrypt_repository_not_found/comment_2_bc67619191380b6cb2555998b2da9e88._comment b/doc/bugs/gcrypt_repository_not_found/comment_2_bc67619191380b6cb2555998b2da9e88._comment
new file mode 100644
index 0000000000..f9b2ebd0f8
--- /dev/null
+++ b/doc/bugs/gcrypt_repository_not_found/comment_2_bc67619191380b6cb2555998b2da9e88._comment
@@ -0,0 +1,21 @@
+[[!comment format=mdwn
+ username="anarcat"
+ avatar="http://cdn.libravatar.org/avatar/4ad594c1e13211c1ad9edb81ce5110b7"
+ subject=".gitconfig was the cause, git-remote-gcrypt probably at fault"
+ date="2018-05-17T20:42:37Z"
+ content="""
+I understand, thanks for the analysis!
+
+As it turns out, the problem was that I had this block in my `~/.gitconfig`:
+
+    [push]
+        gpgSign = if-asked
+
+This means that git-annex is not the component that was passing the
+argument. That's why I couldn't find it anywhere in the source. git
+itself was passing this along.
+
+I would then make the argument that git-remote-gcrypt is the one that
+should be more tolerant to those arguments. Should I send a pull
+request for this?
+"""]]
diff --git a/doc/bugs/gcrypt_repository_not_found/comment_3_6278a373a323b621edfe1995a8685e8c._comment b/doc/bugs/gcrypt_repository_not_found/comment_3_6278a373a323b621edfe1995a8685e8c._comment
new file mode 100644
index 0000000000..eb94ebfac5
--- /dev/null
+++ b/doc/bugs/gcrypt_repository_not_found/comment_3_6278a373a323b621edfe1995a8685e8c._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="spwhitton"
+ avatar="http://cdn.libravatar.org/avatar/9c3f08f80e67733fd506c353239569eb"
+ subject="patches welcome"
+ date="2018-05-17T21:31:00Z"
+ content="""
+Patches would be very welcome!
+"""]]
diff --git a/doc/bugs/gcrypt_repository_not_found/comment_4_166597e046b4bc1d3770c1e8a4191832._comment b/doc/bugs/gcrypt_repository_not_found/comment_4_166597e046b4bc1d3770c1e8a4191832._comment
new file mode 100644
index 0000000000..05467d60c4
--- /dev/null
+++ b/doc/bugs/gcrypt_repository_not_found/comment_4_166597e046b4bc1d3770c1e8a4191832._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="anarcat"
+ avatar="http://cdn.libravatar.org/avatar/4ad594c1e13211c1ad9edb81ce5110b7"
+ subject="comment 4"
+ date="2018-05-22T20:54:13Z"
+ content="""
+sent a crude patch by email.
+"""]]
diff --git a/doc/bugs/gcrypt_special_remote_fails_on_bare_remote_repos.mdwn b/doc/bugs/gcrypt_special_remote_fails_on_bare_remote_repos.mdwn
new file mode 100644
index 0000000000..dcea28f651
--- /dev/null
+++ b/doc/bugs/gcrypt_special_remote_fails_on_bare_remote_repos.mdwn
@@ -0,0 +1,39 @@
+Creating a gcrypt remote on a remote bare repository fails: it tries to modify the config file in the wrong path:
+
+(problem is in last line)
+
+```
+$ git annex initremote qech type=gcrypt encryption=hybrid gitrepo=~/astro.git keyid=0BA74AC5B9494715
+initremote qech (encryption setup) (hybrid cipher with gpg key 0BA74AC5B9494715) gcrypt: Development version -- Repository format MAY CHANGE
+gcrypt: Decrypting manifest
+gpg: Signature made Son 13 Sep 2015 19:37:16 CEST using RSA key ID B9494715
+gpg: Good signature from "Arian Sanusi "
+gpg:                 aka "Arian Sanusi "
+gpg:                 aka "Arian Sanusi "
+gpg:                 aka "Arian Sanusi "
+gpg:                 aka "Arian Sanusi "
+gpg:                 aka "Arian Sanusi "
+gcrypt: Remote ID is :id:1+DP8KgrbsvpSycQ5QKO
+Von gcrypt::/home/arian-debian-ssd/astro
+ * [neuer Branch]    git-annex  -> qech/git-annex
+gcrypt: Development version -- Repository format MAY CHANGE
+gcrypt: Decrypting manifest
+gpg: Signature made Son 13 Sep 2015 19:37:16 CEST using RSA key ID B9494715
+gpg: Good signature from "Arian Sanusi "
+gpg:                 aka "Arian Sanusi "
+gpg:                 aka "Arian Sanusi "
+gpg:                 aka "Arian Sanusi "
+gpg:                 aka "Arian Sanusi "
+gpg:                 aka "Arian Sanusi "
+Everything up-to-date
+error: could not lock config file /home/arian-debian-ssd/astro.git/.git/config: Datei oder Verzeichnis nicht gefunden
+git-annex: git [Param "config",Param "core.gcrypt-id",Param ":id:1+DP8KgrbsvpSycQ5QKO"] failed
+```
+
+debian stretch *pre gcc5* on amd64 here:
+
+```
+i  git                 1:2.5.1-1    amd64
+ii  git-annex           5.20150812-2 amd64
+ii  git-remote-gcrypt   0.20130908-7 all
+```
diff --git a/doc/bugs/gcrypt_special_remote_fails_on_bare_remote_repos/comment_1_7e690402c8dd663d60f7fc659c323ed8._comment b/doc/bugs/gcrypt_special_remote_fails_on_bare_remote_repos/comment_1_7e690402c8dd663d60f7fc659c323ed8._comment
new file mode 100644
index 0000000000..e208b9bd11
--- /dev/null
+++ b/doc/bugs/gcrypt_special_remote_fails_on_bare_remote_repos/comment_1_7e690402c8dd663d60f7fc659c323ed8._comment
@@ -0,0 +1,26 @@
+[[!comment format=mdwn
+ username="joey"
+ subject="""comment 1"""
+ date="2015-09-18T16:56:42Z"
+ content="""
+So, the missing bit of data needed to reproduce this is that
+~/astro.git didn't exist at all before this initremote call.
+
+If you `git init --bare ~/astro.git` first, it works fine.
+
+The documentation says, of gitrepo:
+
+	This repository should be either empty, or an existing gcrypt repositry.
+
+So, I don't know if this is really a bug, you're not doing what the
+documentation says to do.
+
+I suppose git-annex initremote could create a bare repository when none exists
+in this case, but it would then have to parse the gitrepo parameter, normalize
+it, use the approptiate protocol (file, ssh, rsync) to check if it exists or
+create it, etc, instead of simply passing it to gcrypt.
+
+(What is actually going on here is, gcrypt supports remotes that are not git repositories at all.
+When the repo doesn't exist, it assumes such a remote is wanted and makes one.
+But, git-annex does not support working with that kind of gcrypt remote.)
+"""]]
diff --git a/doc/bugs/get_--json_fails_whenever_plain_get_works___40__with_https_urls__41__.mdwn b/doc/bugs/get_--json_fails_whenever_plain_get_works___40__with_https_urls__41__.mdwn
new file mode 100644
index 0000000000..6839e77a85
--- /dev/null
+++ b/doc/bugs/get_--json_fails_whenever_plain_get_works___40__with_https_urls__41__.mdwn
@@ -0,0 +1,54 @@
+### Please describe the problem.
+
+subject
+
+### What steps will reproduce the problem?
+
+repo from http://datasets.datalad.org/workshops/nih-2017/ds000114/derivatives/freesurfer/
+
+### What version of git-annex are you using? On what operating system?
+
+6.20170525+gitge1cf095ae-1~ndall+1
+
+
+### Please provide any additional information below.
+
+[[!format sh """
+[student3@helix freesurfer]$ 'git' '-c' 'receive.autogc=0' '-c' 'gc.auto=0' 'annex' 'get' '--debug' '--json' '--json-progress' 'sub-01/label/lh.aparc.a2009s.annot'
+[2017-07-30 09:33:42.479678681] read: git ["--git-dir=.git","--work-tree=.","--literal-pathspecs","ls-files","--cached","-z","--","sub-01/label/lh.aparc.a2009s.annot"]
+[2017-07-30 09:33:42.501443872] read: git ["--git-dir=.git","--work-tree=.","--literal-pathspecs","show-ref","git-annex"]
+[2017-07-30 09:33:42.523673251] process done ExitSuccess
+[2017-07-30 09:33:42.523765181] read: git ["--git-dir=.git","--work-tree=.","--literal-pathspecs","show-ref","--hash","refs/heads/git-annex"]
+[2017-07-30 09:33:42.544559473] process done ExitSuccess
+[2017-07-30 09:33:42.553475534] chat: git ["--git-dir=.git","--work-tree=.","--literal-pathspecs","cat-file","--batch"]
+[2017-07-30 09:33:42.554641292] chat: git ["--git-dir=.git","--work-tree=.","--literal-pathspecs","cat-file","--batch-check=%(objectname) %(objecttype) %(objectsize)"]
+[2017-07-30 09:33:42.591306199] call: curl ["-sS","-f","-L","-C","-","-#","-o","/scratch/nih-2017/ds000114/derivatives/freesurfer/.git/annex/tmp/MD5E-s831572--7437ad482af573329af54386a15007fe","https://dl.dropbox.com/s/85yms1fl56pq7io/lh.aparc.a2009s.annot?dl=0","--user-agent","git-annex/6.20170525+gitge1cf095ae-1~ndall+1"]
+curl: (77) error setting certificate verify locations:
+  CAfile: /etc/ssl/certs/ca-certificates.crt
+  CApath: /etc/ssl/certs
+[2017-07-30 09:33:42.693467186] process done ExitFailure 77
+{"command":"get","wanted":[{"here":false,"uuid":"00000000-0000-0000-0000-000000000001","description":"web"},{"here":false,"uuid":"5e47b3f3-f09c-4969-8885-920a49ff8a45","description":"yoh@smaug:/mnt/btrfs/datasets/datalad/crawl/workshops/nih-workshop-2017/ds000114/derivatives/freesurfer"}],"note":"Try making some of these repositories available:\n\t00000000-0000-0000-0000-000000000001 -- web\n \t5e47b3f3-f09c-4969-8885-920a49ff8a45 -- yoh@smaug:/mnt/btrfs/datasets/datalad/crawl/workshops/nih-workshop-2017/ds000114/derivatives/freesurfer\n","skipped":[],"success":false,"key":"MD5E-s831572--7437ad482af573329af54386a15007fe","file":"sub-01/label/lh.aparc.a2009s.annot"}
+git-annex: get: 1 failed
+[student3@helix freesurfer]$ 'git' '-c' 'receive.autogc=0' '-c' 'gc.auto=0' 'annex' 'get' '--json' '--json-progress' 'sub-01/label/lh.aparc.a2009s.annot'
+curl: (77) error setting certificate verify locations:
+  CAfile: /etc/ssl/certs/ca-certificates.crt
+  CApath: /etc/ssl/certs
+{"command":"get","wanted":[{"here":false,"uuid":"00000000-0000-0000-0000-000000000001","description":"web"},{"here":false,"uuid":"5e47b3f3-f09c-4969-8885-920a49ff8a45","description":"yoh@smaug:/mnt/btrfs/datasets/datalad/crawl/workshops/nih-workshop-2017/ds000114/derivatives/freesurfer"}],"note":"Try making some of these repositories available:\n\t00000000-0000-0000-0000-000000000001 -- web\n \t5e47b3f3-f09c-4969-8885-920a49ff8a45 -- yoh@smaug:/mnt/btrfs/datasets/datalad/crawl/workshops/nih-workshop-2017/ds000114/derivatives/freesurfer\n","skipped":[],"success":false,"key":"MD5E-s831572--7437ad482af573329af54386a15007fe","file":"sub-01/label/lh.aparc.a2009s.annot"}
+git-annex: get: 1 failed
+                        
+while works fine without:
+                 
+[student3@helix freesurfer]$ 'git' '-c' 'receive.autogc=0' '-c' 'gc.auto=0' 'annex' 'get'  'sub-01/label/lh.aparc.a2009s.annot'
+get sub-01/label/lh.aparc.a2009s.annot (from web...)
+/scratch/nih-2017/ds000114/ 100%[===========================================>] 812.08K  --.-KB/s    in 0.08s
+2017-07-30 09:34:02 URL:https://dl.dropboxusercontent.com/s/85yms1fl56pq7io/lh.aparc.a2009s.annot?dl=0 [831572/831572] -> "/scratch/nih-2017/ds000114/derivatives/freesurfer/.git/annex/tmp/MD5E-s831572--7437ad482af573329af54386a15007fe" [1]  
+(checksum...) 
+(recording state in git...)
+
+"""]]
+
+[[!meta author=yoh]]
+
+> git-annex no longer uses either curl or wget by default, and always uses
+> curl when configured to do so, so this kind of surprising behavior will
+> no longer occur [[done]] --[[Joey]]
diff --git a/doc/bugs/get_--json_fails_whenever_plain_get_works___40__with_https_urls__41__/comment_1_a9aee679a5069dc8ee8960e975bf020b._comment b/doc/bugs/get_--json_fails_whenever_plain_get_works___40__with_https_urls__41__/comment_1_a9aee679a5069dc8ee8960e975bf020b._comment
new file mode 100644
index 0000000000..22f1ad8c17
--- /dev/null
+++ b/doc/bugs/get_--json_fails_whenever_plain_get_works___40__with_https_urls__41__/comment_1_a9aee679a5069dc8ee8960e975bf020b._comment
@@ -0,0 +1,27 @@
+[[!comment format=mdwn
+ username="joey"
+ subject="""comment 1"""
+ date="2017-07-31T18:46:00Z"
+ content="""
+Using --json will have the same effect as --quiet in that commands have
+to be made to not output to stdout in that mode. In the case of
+downloading an url, git-annex usually uses wget, however when curl is
+available in path, it uses it in quiet mode, because curl displays error
+messages to stdout, whereas wget never displays error messages at all
+when run in quiet mode.
+
+So, by using --json, you're changing the command that git-annex uses to
+download files from wget to curl.
+
+But, the git-annex standalone build bundles both wget and curl, so
+I'd expect that any certificates problem would affect both equally.
+But I suppose wget may be better able to deal with the certs on this
+system than curl, for whatever reason.
+
+Bundling certs with git-annex seems wrong on multiple counts; it's
+moving a large security boundary inside git-annex; it's lots of work;
+there may be local policies on allowed certs that this would disallow.
+
+Suggestion: Remove curl from the standalone build you're using, so
+it will use the system-wide curl.
+"""]]
diff --git a/doc/bugs/get_--json_fails_whenever_plain_get_works___40__with_https_urls__41__/comment_2_f1e97aa8b9927c01e3365b940061d3c2._comment b/doc/bugs/get_--json_fails_whenever_plain_get_works___40__with_https_urls__41__/comment_2_f1e97aa8b9927c01e3365b940061d3c2._comment
new file mode 100644
index 0000000000..e27a5fa6fc
--- /dev/null
+++ b/doc/bugs/get_--json_fails_whenever_plain_get_works___40__with_https_urls__41__/comment_2_f1e97aa8b9927c01e3365b940061d3c2._comment
@@ -0,0 +1,10 @@
+[[!comment format=mdwn
+ username="yarikoptic"
+ avatar="http://cdn.libravatar.org/avatar/f11e9c84cb18d26a1748c33b48c924b4"
+ subject="comment 2"
+ date="2017-07-31T19:05:04Z"
+ content="""
+There is no guarantee that system would have curl installed,  e.g. I believe neither of debian docker images would have curl.  For neurodebian package I sure thing could just depend on curl.  But I am afraid those who might be using standalone git-annex in minimalistic environments would get surprised
+
+FWIW -- removing curl from the bundled git-annex indeed helped
+"""]]
diff --git a/doc/bugs/get_-JX__58____transfer_already_in_progress_..._for_some_files.mdwn b/doc/bugs/get_-JX__58____transfer_already_in_progress_..._for_some_files.mdwn
new file mode 100644
index 0000000000..4af968cb1b
--- /dev/null
+++ b/doc/bugs/get_-JX__58____transfer_already_in_progress_..._for_some_files.mdwn
@@ -0,0 +1,32 @@
+### Please describe the problem.
+
+Unclear reason why some files/keys fail to get in parallel.
+
+### What steps will reproduce the problem?
+
+In my case it is getting files from the datalad-archives special remote which fetches the tarball key, extracts, and copies for annex.
+I've implemented locking in datalad-archives on getting the tarball key and extracting the archive, so now we can run `get -JX` and it generally works. But when there is lots of files in the tarball, for some of them (seems to be up to the X in -JX) transfer fails.
+
+git annex simply reports
+[[!format sh """
+$> grep already git-annex-getJ5-5.log                               
+{"command":"get","wanted":[{"here":false,"uuid":"79080a38-0e94-4a0a-bd89-9022eada547b","description":"yoh@smaug:/mnt/btrfs/datasets/datalad/crawl/crcns/aa-1"},{"here":false,"uuid":"895b9a07-6613-4c8a-95ae-280d8119475c","description":"[datalad-archives]"}],"note":"transfer already in progress, or unable to take transfer lock\nUnable to access these remotes: datalad-archives\nTry making some of these repositories available:\n\t79080a38-0e94-4a0a-bd89-9022eada547b -- yoh@smaug:/mnt/btrfs/datasets/datalad/crawl/crcns/aa-1\n \t895b9a07-6613-4c8a-95ae-280d8119475c -- [datalad-archives]\n","skipped":[],"success":false,"key":"MD5E-s1001--2cd1bc42ddd745e7d5c00edb07d8c9d4","file":"MLd_cells/gg0304_6_A/conspecific/spike20"}
+.... 4 more
+
+$> find -lname */MD5E-s1001--2cd1bc42ddd745e7d5c00edb07d8c9d4 
+./MLd_cells/gg0304_6_A/conspecific/spike20
+"""]]
+
+so it is not that the key is used for multiple files (I remember we had that before, so checked for that first).
+According to the logs ([git-annex](http://www.onerussian.com/tmp/git-annex-getJ5-5.log) and [datalad](http://www.onerussian.com/tmp/datalad-getJ5-5.log)) that key is not even passed from annex to our special remote so somehow it freaks out and skips it.
+
+If needed, I could probably provide you a singularity image with the environment with datalad pre-installed with that branch so you could troubleshoot.
+
+
+### What version of git-annex are you using? On what operating system?
+
+Tried with bleeding edge 6.20180308+gitg3962ca71b-1~ndall+1 although originally detected with 6.20180220+gitg811d0d313-1~ndall+1
+
+[[!meta author=yoh]]
+
+> [[fixed|done]] --[[Joey]]
diff --git a/doc/bugs/get_-JX__58____transfer_already_in_progress_..._for_some_files/comment_1_57844bc289adacc6a53162cd224e6df3._comment b/doc/bugs/get_-JX__58____transfer_already_in_progress_..._for_some_files/comment_1_57844bc289adacc6a53162cd224e6df3._comment
new file mode 100644
index 0000000000..502fa12d3e
--- /dev/null
+++ b/doc/bugs/get_-JX__58____transfer_already_in_progress_..._for_some_files/comment_1_57844bc289adacc6a53162cd224e6df3._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="yarikoptic"
+ avatar="http://cdn.libravatar.org/avatar/f11e9c84cb18d26a1748c33b48c924b4"
+ subject="comment 1"
+ date="2018-03-09T18:55:51Z"
+ content="""
+May be would be of help: here datalad-archives spits out TRANSFER-SUCCESSes really fast since it takes it little time to just copy a little file... so it feels to me that issue is in some race condition within git-annex.  If I add 100ms delay before returning TRANSFER-SUCCESS from the special remote - number of failures goes down to 0-2 instead of 5ish. With 200ms delay seems to get no failures whatsoever.
+"""]]
diff --git a/doc/bugs/get_-JX__58____transfer_already_in_progress_..._for_some_files/comment_2_ea5d3efc56b70de03a5b0b5e908bf11f._comment b/doc/bugs/get_-JX__58____transfer_already_in_progress_..._for_some_files/comment_2_ea5d3efc56b70de03a5b0b5e908bf11f._comment
new file mode 100644
index 0000000000..b2348ff01a
--- /dev/null
+++ b/doc/bugs/get_-JX__58____transfer_already_in_progress_..._for_some_files/comment_2_ea5d3efc56b70de03a5b0b5e908bf11f._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="yarikoptic"
+ avatar="http://cdn.libravatar.org/avatar/f11e9c84cb18d26a1748c33b48c924b4"
+ subject="comment 2"
+ date="2018-03-09T19:00:27Z"
+ content="""
+FWIW: I was wrong with the statement that # of failures goes upto the X in -JX -- just got 13 (when no delay) with -J5.
+"""]]
diff --git a/doc/bugs/get_-JX__58____transfer_already_in_progress_..._for_some_files/comment_3_ed5fa01a2263ebb4a24bfe0e46ee820f._comment b/doc/bugs/get_-JX__58____transfer_already_in_progress_..._for_some_files/comment_3_ed5fa01a2263ebb4a24bfe0e46ee820f._comment
new file mode 100644
index 0000000000..579960fda9
--- /dev/null
+++ b/doc/bugs/get_-JX__58____transfer_already_in_progress_..._for_some_files/comment_3_ed5fa01a2263ebb4a24bfe0e46ee820f._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="joey"
+ subject="""comment 3"""
+ date="2018-03-14T19:43:38Z"
+ content="""
+I was able to reproduce this with two normal git-annex repos
+and `git annex get -J10`. 6.20180227.
+"""]]
diff --git a/doc/bugs/get_-JX__58____transfer_already_in_progress_..._for_some_files/comment_4_5964e40963325975e5de37506a2852ca._comment b/doc/bugs/get_-JX__58____transfer_already_in_progress_..._for_some_files/comment_4_5964e40963325975e5de37506a2852ca._comment
new file mode 100644
index 0000000000..e3c8243e43
--- /dev/null
+++ b/doc/bugs/get_-JX__58____transfer_already_in_progress_..._for_some_files/comment_4_5964e40963325975e5de37506a2852ca._comment
@@ -0,0 +1,11 @@
+[[!comment format=mdwn
+ username="joey"
+ subject="""comment 4"""
+ date="2018-03-14T20:17:31Z"
+ content="""
+tryLockExclusive is returning Nothing when this happens.
+
+Which seems very similar to the problem fixed in
+[[parallel_get_can_fail_some_downloads_and_require_re-getting_]].
+Which really seemed to be fixed back then..
+"""]]
diff --git a/doc/bugs/get_-JX__58____transfer_already_in_progress_..._for_some_files/comment_5_64705f2eddbb6fcb55edc797bbd3b22c._comment b/doc/bugs/get_-JX__58____transfer_already_in_progress_..._for_some_files/comment_5_64705f2eddbb6fcb55edc797bbd3b22c._comment
new file mode 100644
index 0000000000..4fb6884467
--- /dev/null
+++ b/doc/bugs/get_-JX__58____transfer_already_in_progress_..._for_some_files/comment_5_64705f2eddbb6fcb55edc797bbd3b22c._comment
@@ -0,0 +1,36 @@
+[[!comment format=mdwn
+ username="joey"
+ subject="""comment 5"""
+ date="2018-03-14T20:38:48Z"
+ content="""
+This is not the same as the previous bug; the STM code seems ok.
+It is finding an open exclusive lock handle in the STM lock pool
+for the transfer lock file, and so tryLockExclusive fails.
+
+Instrumented tryLockExclusive, and it's somehow being called twice or more
+on the same transfer lock file despite there being no duplicate keys.
+
+Aha.. sizeOfDownloadsInProgress calls getTransfers,
+which locks the transfer info files in order to check
+which transfers are running. So one of the other worker threads
+is calling that at just the wrong time, and so contending with
+the thread that is starting up the transfer.
+
+Bug was introduced by [[!commit 3cd47f997873ff9d50b35c0f4440763364766d93]].
+
+It's interesting that the transfer info file is being created before
+the transfer lock file. If the lock file were always created first,
+then getTransfers would not see the transfer before its lock file is
+locked, and this bug would be avoided. On the other hand,
+this exact ordering of file creation is why
+3cd47f997873ff9d50b35c0f4440763364766d93 is necessary.
+
+There are a couple of ways the files could be created in the wrong
+order. 3cd47f997873ff9d50b35c0f4440763364766d93 commit message describes
+one, which does not apply to the test case for this bug.
+
+Hmm, mkProgressUpdater was very recently changed to write the transfer
+info file, in 24df95f0f6ab474119aff3bbd942251373754ab2, and that comes
+before the transfer lock is created. That is probably the recent change
+that exposed this mess.
+"""]]
diff --git a/doc/bugs/get_-JX__58____transfer_already_in_progress_..._for_some_files/comment_6_e51841ae57dd9d2f48cd21fbdb90ac29._comment b/doc/bugs/get_-JX__58____transfer_already_in_progress_..._for_some_files/comment_6_e51841ae57dd9d2f48cd21fbdb90ac29._comment
new file mode 100644
index 0000000000..c4ab254675
--- /dev/null
+++ b/doc/bugs/get_-JX__58____transfer_already_in_progress_..._for_some_files/comment_6_e51841ae57dd9d2f48cd21fbdb90ac29._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="joey"
+ subject="""comment 6"""
+ date="2018-03-14T22:54:24Z"
+ content="""
+Ok, fixed the reversion in a fairly decent way,
+verified with 1000 files and -J10.
+"""]]
diff --git a/doc/bugs/get_-JX__58____transfer_already_in_progress_..._for_some_files/comment_7_2f7f87db7ca4d379d439ab88bdeee5b5._comment b/doc/bugs/get_-JX__58____transfer_already_in_progress_..._for_some_files/comment_7_2f7f87db7ca4d379d439ab88bdeee5b5._comment
new file mode 100644
index 0000000000..1fbd5ef75d
--- /dev/null
+++ b/doc/bugs/get_-JX__58____transfer_already_in_progress_..._for_some_files/comment_7_2f7f87db7ca4d379d439ab88bdeee5b5._comment
@@ -0,0 +1,9 @@
+[[!comment format=mdwn
+ username="yarikoptic"
+ avatar="http://cdn.libravatar.org/avatar/f11e9c84cb18d26a1748c33b48c924b4"
+ subject="comment 7"
+ date="2018-03-14T23:09:20Z"
+ content="""
+cool, thanks, will test it out
+would this one somehow reveal itself also in non-J mode? (just doublechecking)
+"""]]
diff --git a/doc/bugs/get_-JX__58____transfer_already_in_progress_..._for_some_files/comment_8_36bcdc23439c4f19c5aaedff85213db0._comment b/doc/bugs/get_-JX__58____transfer_already_in_progress_..._for_some_files/comment_8_36bcdc23439c4f19c5aaedff85213db0._comment
new file mode 100644
index 0000000000..4972081c38
--- /dev/null
+++ b/doc/bugs/get_-JX__58____transfer_already_in_progress_..._for_some_files/comment_8_36bcdc23439c4f19c5aaedff85213db0._comment
@@ -0,0 +1,10 @@
+[[!comment format=mdwn
+ username="joey"
+ subject="""comment 8"""
+ date="2018-03-14T23:14:46Z"
+ content="""
+This same bug can happen without -J if two git-annex processes are running
+at the same time and both downloading (different) files.
+
+So, the way datalad special remote runs git-annex, for example...
+"""]]
diff --git a/doc/bugs/get_-JX__58____transfer_already_in_progress_..._for_some_files/comment_9_7ce83c16509bd42b2460ce59470f70c0._comment b/doc/bugs/get_-JX__58____transfer_already_in_progress_..._for_some_files/comment_9_7ce83c16509bd42b2460ce59470f70c0._comment
new file mode 100644
index 0000000000..a92e576f36
--- /dev/null
+++ b/doc/bugs/get_-JX__58____transfer_already_in_progress_..._for_some_files/comment_9_7ce83c16509bd42b2460ce59470f70c0._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="yarikoptic"
+ avatar="http://cdn.libravatar.org/avatar/f11e9c84cb18d26a1748c33b48c924b4"
+ subject="comment 9"
+ date="2018-03-15T12:47:01Z"
+ content="""
+FWIW, confirming that the fix seems to work nicely for us (with a bit of git-annex on sid, although typically do on stable).
+"""]]
diff --git a/doc/bugs/get_-J___34__fails__34___to_get_files_with_the_same_key.mdwn b/doc/bugs/get_-J___34__fails__34___to_get_files_with_the_same_key.mdwn
new file mode 100644
index 0000000000..7008eb62d3
--- /dev/null
+++ b/doc/bugs/get_-J___34__fails__34___to_get_files_with_the_same_key.mdwn
@@ -0,0 +1,52 @@
+### What steps will reproduce the problem?
+
+ask annex get in parallel files which point to the same key
+
+### What version of git-annex are you using? On what operating system?
+
+6.20170815+gitg22da64d0f-1~ndall+1
+
+### Please provide any additional information below.
+
+[[!format sh """
+# works in serial mode
+
+$> git annex get rh.white{,_avg}
+get rh.white (from web...) 
+/mnt/btrfs/scrap/tmp/ds0001 100%[===========================================>] 360.31K  --.-KB/s    in 0.1s    
+2017-08-30 10:08:02 URL:https://dl.dropboxusercontent.com/s/0lww4tomnwfanwd/rh.white_avg?dl=0 [368962/368962] -> "/mnt/btrfs/scrap/tmp/ds000114/derivatives/freesurfer/.git/annex/tmp/MD5E-s368962--99a4db61cedffee686aef99b2d197794" [1]
+(checksum...) ok
+(recording state in git...)
+(dev)2 10016.....................................:Wed 30 Aug 2017 10:08:02 AM EDT:.
+(git)smaug:…/btrfs/scrap/tmp/ds000114/derivatives/freesurfer[master]fsaverage5/surf
+$> git annex drop --fast rh.white{,_avg}         
+drop rh.white (checking https://dl.dropbox.com/s/0lww4tomnwfanwd/rh.white_avg?dl=0...) ok
+(recording state in git...)
+
+# "fails" in parallel
+$> git annex get -J2 rh.white{,_avg}   
+get rh.white get rh.white_avg (transfer already in progress, or unable to take transfer lock) 
+  Unable to access these remotes: web
+(from web...) 
+
+  Try making some of these repositories available:
+        00000000-0000-0000-0000-000000000001 -- web
+        5e47b3f3-f09c-4969-8885-920a49ff8a45 -- yoh@smaug:/mnt/btrfs/datasets/datalad/crawl/workshops/nih-workshop-2017/ds000114/derivatives/freesurfer
+failed
+/mnt/btrfs/scrap/tmp/ds0001 100%[===========================================>] 360.31K  1.63MB/s    in 0.2s    
+2017-08-30 10:08:21 URL:https://dl.dropboxusercontent.com/s/0lww4tomnwfanwd/rh.white_avg?dl=0 [368962/368962] -> "/mnt/btrfs/scrap/tmp/ds000114/derivatives/freesurfer/.git/annex/tmp/MD5E-s368962--99a4db61cedffee686aef99b2d197794" [1]
+(checksum...) ok
+(recording state in git...)
+git-annex: get: 1 failed
+(dev)2 10018 ->1.....................................:Wed 30 Aug 2017 10:08:21 AM EDT:.
+
+"""]]
+
+so at the end we get a run of git-annex which exits with error 1... and in json mode also the error(s) reported etc.
+I wondered if annex should first analyze passed paths to get actual keys to be fetched?
+
+[[!meta author=yoh]]
+
+> [[fixed|done]]; also fixed for several other commands, but the final 
+> fix needed each command that could have the problem to be modified, so 
+> there could possibly be some I missed.. --[[Joey]]
diff --git a/doc/bugs/get_-J___34__fails__34___to_get_files_with_the_same_key/comment_1_5b3879d1bb992ddf2aafa3fcb41b968f._comment b/doc/bugs/get_-J___34__fails__34___to_get_files_with_the_same_key/comment_1_5b3879d1bb992ddf2aafa3fcb41b968f._comment
new file mode 100644
index 0000000000..11786bb777
--- /dev/null
+++ b/doc/bugs/get_-J___34__fails__34___to_get_files_with_the_same_key/comment_1_5b3879d1bb992ddf2aafa3fcb41b968f._comment
@@ -0,0 +1,19 @@
+[[!comment format=mdwn
+ username="joey"
+ subject="""comment 1"""
+ date="2017-09-29T16:47:18Z"
+ content="""
+The only way I can see to improve this would be to keep track of which
+keys already have a thread working on them, and avoid a second thread
+working on the same key.
+
+I've started this in the avoid-dup-threads branch.
+
+Getting key information to commandAction would be quite the plumbing job;
+there are something like 50 call sites.
+
+More difficult, the key is not known yet when commandAction is called in a
+lot of cases, and looking up the key redundantly will slow down all
+git-annex scanning. Seems that nontrivial changes would be needed to every
+command.
+"""]]
diff --git a/doc/bugs/get_-J___34__fails__34___to_get_files_with_the_same_key/comment_2_af7078b291d99569070f6a76579787aa._comment b/doc/bugs/get_-J___34__fails__34___to_get_files_with_the_same_key/comment_2_af7078b291d99569070f6a76579787aa._comment
new file mode 100644
index 0000000000..3412f1cb17
--- /dev/null
+++ b/doc/bugs/get_-J___34__fails__34___to_get_files_with_the_same_key/comment_2_af7078b291d99569070f6a76579787aa._comment
@@ -0,0 +1,10 @@
+[[!comment format=mdwn
+ username="yarikoptic"
+ avatar="http://cdn.libravatar.org/avatar/f11e9c84cb18d26a1748c33b48c924b4"
+ subject="also the same for add"
+ date="2017-10-05T17:00:51Z"
+ content="""
+Hi Joey
+
+I did remember that we had something similar but forgot that it was for \"get\" -- now I was getting similar problem for \"add --jobs\".  Will this fix also work for it as well?
+"""]]
diff --git a/doc/bugs/get_-J___34__fails__34___to_get_files_with_the_same_key/comment_3_86e1be495262a7f57428e92eb27a559f._comment b/doc/bugs/get_-J___34__fails__34___to_get_files_with_the_same_key/comment_3_86e1be495262a7f57428e92eb27a559f._comment
new file mode 100644
index 0000000000..066f8db4e0
--- /dev/null
+++ b/doc/bugs/get_-J___34__fails__34___to_get_files_with_the_same_key/comment_3_86e1be495262a7f57428e92eb27a559f._comment
@@ -0,0 +1,47 @@
+[[!comment format=mdwn
+ username="yarikoptic"
+ avatar="http://cdn.libravatar.org/avatar/f11e9c84cb18d26a1748c33b48c924b4"
+ subject="comment 3"
+ date="2017-10-05T20:36:50Z"
+ content="""
+here, caught one for you for add (git annex version is tiny bit dated: 6.20170815+gitg22da64d0f-1~ndall+1 )
+[[!format sh \"\"\"
+(Pdb) print e.msg
+Failed to run ['git', '-c', 'receive.autogc=0', '-c', 'gc.auto=0', 'annex', 'add', '--json', '-J6', '082-sn/000001.dcm', '078-sn/000001.dcm', '080-sn/000002.dcm', '076-sn/000002.dcm', '080-sn/000004.dcm', '080-sn/000001.dcm', '079-sn/000003.dcm', '082-sn/000003.dcm', '073-sn/000002.dcm', '079-sn/000002.dcm', '079-sn/000001.dcm', '077-sn/000002.dcm', '074-sn/000001.dcm', '080-sn/000003.dcm', '077-sn/000001.dcm', '076-sn/000001.dcm', '081-sn/000002.dcm', '078-sn/000002.dcm', '081-sn/000001.dcm', '081-sn/000003.dcm', '073-sn/000001.dcm', '075-sn/000001.dcm', '079-sn/000004.dcm', '082-sn/000002.dcm', '075-sn/000002.dcm', '074-sn/000002.dcm'] under '/mnt/DICOM/test2/inbox/2016/12/12/unknown'. Exit code=1. out={\"command\":\"add\",\"success\":true,\"key\":\"MD5E-s193740--3da2e91e0888c05e01daf8ef9ae79570.dcm\",\"file\":\"076-sn/000002.dcm\"}
+{\"command\":\"add\",\"success\":true,\"key\":\"MD5E-s205064--851ce819ea44cabd66923d902e55cd2c.dcm\",\"file\":\"082-sn/000001.dcm\"}
+{\"command\":\"add\",\"success\":true,\"key\":\"MD5E-s226874--8acaba7ff0f57a3e69a5c1afb8bc0ba3.dcm\",\"file\":\"080-sn/000001.dcm\"}
+{\"command\":\"add\",\"success\":true,\"key\":\"MD5E-s193746--faf49ea7b403c8f6191fa5b7521ebbd5.dcm\",\"file\":\"078-sn/000001.dcm\"}
+{\"command\":\"add\",\"success\":true,\"key\":\"MD5E-s205066--c5af824ecd895a342cba7ffefe019fde.dcm\",\"file\":\"082-sn/000003.dcm\"}
+{\"command\":\"add\",\"success\":true,\"key\":\"MD5E-s226872--abdaf5312c4c11da68cc9aaead6bc93a.dcm\",\"file\":\"080-sn/000004.dcm\"}
+{\"command\":\"add\",\"success\":true,\"key\":\"MD5E-s226768--352d7bf9be4b45316cb75c2fc5bfbdd2.dcm\",\"file\":\"079-sn/000002.dcm\"}
+{\"command\":\"add\",\"success\":true,\"key\":\"MD5E-s226872--755e5e693b07fd7cab07fa55e0f17fd2.dcm\",\"file\":\"080-sn/000002.dcm\"}
+{\"command\":\"add\",\"success\":true,\"key\":\"MD5E-s193636--47816f992ddf6167549b0d20bf430036.dcm\",\"file\":\"073-sn/000002.dcm\"}
+{\"command\":\"add\",\"success\":true,\"key\":\"MD5E-s193750--465de1ebab376633842ac9e36f6fcc35.dcm\",\"file\":\"074-sn/000001.dcm\"}
+{\"command\":\"add\",\"success\":true,\"key\":\"MD5E-s226766--04006df494da6adbe7b3eea90e4dba1b.dcm\",\"file\":\"079-sn/000003.dcm\"}
+{\"command\":\"add\",\"success\":true,\"key\":\"MD5E-s226768--3624982179994bbaa781ee8c72c5d6b9.dcm\",\"file\":\"079-sn/000001.dcm\"}
+{\"command\":\"add\",\"success\":true,\"key\":\"MD5E-s193636--98a6efe8bea17485297e5d600c4c01e6.dcm\",\"file\":\"077-sn/000002.dcm\"}
+{\"command\":\"add\",\"success\":true,\"key\":\"MD5E-s193642--15770ed37bedb6bd90a7a4b39761a1fa.dcm\",\"file\":\"077-sn/000001.dcm\"}
+{\"command\":\"add\",\"success\":true,\"key\":\"MD5E-s226870--621a7d3a2be10b99d45a39ef2526ab1c.dcm\",\"file\":\"080-sn/000003.dcm\"}
+{\"command\":\"add\",\"success\":true,\"key\":\"MD5E-s204960--e28aac5eb5e52fe47c2671e8b87edc8c.dcm\",\"file\":\"081-sn/000002.dcm\"}
+{\"command\":\"add\",\"success\":true,\"key\":\"MD5E-s193750--25cadf2f0fc7a4ca324687a742a2d55e.dcm\",\"file\":\"076-sn/000001.dcm\"}
+{\"command\":\"add\",\"success\":true,\"key\":\"MD5E-s193740--0424fce22306f7590f6d94561eb74e93.dcm\",\"file\":\"078-sn/000002.dcm\"}
+{\"command\":\"add\",\"success\":true,\"key\":\"MD5E-s204956--039309e288379d98480390043ea84ecb.dcm\",\"file\":\"081-sn/000001.dcm\"}
+{\"command\":\"add\",\"success\":true,\"key\":\"MD5E-s204960--7b32e2620301909a74adea3d29c00ed5.dcm\",\"file\":\"081-sn/000003.dcm\"}
+{\"command\":\"add\",\"success\":true,\"key\":\"MD5E-s193646--ff7337b5330f852063652f1e75099b27.dcm\",\"file\":\"073-sn/000001.dcm\"}
+{\"command\":\"add\",\"success\":true,\"key\":\"MD5E-s226768--40821b65d32878f5859ed1b19c5633d0.dcm\",\"file\":\"079-sn/000004.dcm\"}
+{\"command\":\"add\",\"success\":true,\"key\":\"MD5E-s193646--02704fbd9d37349c4ee96c1509f1a20d.dcm\",\"file\":\"075-sn/000001.dcm\"}
+{\"command\":\"add\",\"success\":true,\"key\":\"MD5E-s193636--7c727d28c56da2476b744e8aca421b5c.dcm\",\"file\":\"075-sn/000002.dcm\"}
+{\"command\":\"add\",\"success\":true,\"key\":\"MD5E-s193740--c0b3731d6a46df9000df32eeb78ae894.dcm\",\"file\":\"074-sn/000002.dcm\"}
+{\"command\":\"add\",\"success\":true,\"key\":\"MD5E-s205066--11b6f6e4cb2f5879a530f83d288bb7fa.dcm\",\"file\":\"082-sn/000002.dcm\"}
+ err=git-annex: 075-sn/000001.dcm not found
+git-annex: add: 1 failed
+
+(Pdb)
+[1]+  Stopped                 ../test/addall.sh
+(dev) [yoh@rolando test2]$ ls -l '/mnt/DICOM/test2/inbox/2016/12/12/unknown/075-sn/000001.dcm'
+lrwxrwxrwx 1 yoh users 129 Dec 12  2016 /mnt/DICOM/test2/inbox/2016/12/12/unknown/075-sn/000001.dcm -> ../.git/annex/objects/ZQ/3G/MD5E-s193646--02704fbd9d37349c4ee96c1509f1a20d.dcm/MD5E-s193646--02704fbd9d37349c4ee96c1509f1a20d.dcm
+
+\"\"\"]]
+
+so it does report success in json, but complains in stderr that file is not found... I guess some race condition between workers so it manages to catch the moment when file is moved into a key or smth like that?
+"""]]
diff --git a/doc/bugs/get_-J___34__fails__34___to_get_files_with_the_same_key/comment_4_3428942e9b52091b2297cd9823f17219._comment b/doc/bugs/get_-J___34__fails__34___to_get_files_with_the_same_key/comment_4_3428942e9b52091b2297cd9823f17219._comment
new file mode 100644
index 0000000000..e023db5e1c
--- /dev/null
+++ b/doc/bugs/get_-J___34__fails__34___to_get_files_with_the_same_key/comment_4_3428942e9b52091b2297cd9823f17219._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="yarikoptic"
+ avatar="http://cdn.libravatar.org/avatar/f11e9c84cb18d26a1748c33b48c924b4"
+ subject="comment 4"
+ date="2017-10-06T00:38:25Z"
+ content="""
+verified that it is still the case with annex 6.20171001+gitg542d0649f-1~ndall+1
+"""]]
diff --git a/doc/bugs/get_-J___34__fails__34___to_get_files_with_the_same_key/comment_5_525ac8695b6253fc435112f336238c41._comment b/doc/bugs/get_-J___34__fails__34___to_get_files_with_the_same_key/comment_5_525ac8695b6253fc435112f336238c41._comment
new file mode 100644
index 0000000000..ace650c101
--- /dev/null
+++ b/doc/bugs/get_-J___34__fails__34___to_get_files_with_the_same_key/comment_5_525ac8695b6253fc435112f336238c41._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="joey"
+ subject="""comment 5"""
+ date="2017-10-16T16:06:50Z"
+ content="""
+The add problem is clearly an entirely unrelated problem. Opened this
+bug report for it: 
+"""]]
diff --git a/doc/bugs/get_-J___34__fails__34___to_get_files_with_the_same_key/comment_6_5f2ed95bcad2e3d1c4260d1fb0440052._comment b/doc/bugs/get_-J___34__fails__34___to_get_files_with_the_same_key/comment_6_5f2ed95bcad2e3d1c4260d1fb0440052._comment
new file mode 100644
index 0000000000..32f2d4decc
--- /dev/null
+++ b/doc/bugs/get_-J___34__fails__34___to_get_files_with_the_same_key/comment_6_5f2ed95bcad2e3d1c4260d1fb0440052._comment
@@ -0,0 +1,19 @@
+[[!comment format=mdwn
+ username="joey"
+ subject="""comment 6"""
+ date="2017-10-17T17:21:27Z"
+ content="""
+Another way to approach the problem would be, when the transfer of
+the same key is already in progress by another thread of the same process, 
+wait for that thread to complete before running the requested transfer
+action.
+
+The assistant has a TransferMap of all transfers the process is running.
+That would need to be moved from the DaemonStatus to Annex state.
+
+To wait on the thread that's doing the transfer, would need to store
+a MVar or something in the TransferInfo; the ThreadId can't be waited on
+by itself.
+
+This seems much less intrusive, and just as fast as my initial approach.
+"""]]
diff --git a/doc/bugs/get_-J_cannot_be_used_with_password-based_authentication.mdwn b/doc/bugs/get_-J_cannot_be_used_with_password-based_authentication.mdwn
new file mode 100644
index 0000000000..852a6045c4
--- /dev/null
+++ b/doc/bugs/get_-J_cannot_be_used_with_password-based_authentication.mdwn
@@ -0,0 +1,39 @@
+### Please describe the problem.
+
+can't fetch in parallel from a host over ssh if authentication is password-based
+
+### What steps will reproduce the problem?
+
+try to  get -J4  from a host which has ssh authentication password-only (no key)
+
+
+### What version of git-annex are you using? On what operating system?
+
+6.20170101+gitg93d69b1-1~ndall+1
+with newer version (6.20170220+gitg75a15e1ad-1~ndall+1) looks slightly different but to the same "effect"
+
+### Please provide any additional information below.
+
+[[!format sh """
+$> git annex get -J4
+get sourcedata/sub-sid000004/ses-siemens0/anat/sub-sid000004_ses-siemens0_acq-MPRAGE_run-01_T1w.dicom.tgz get sourcedata/sub-sid000004/ses-siemens0/fmap/sub-sid000004_ses-siemens0_acq-3mm_run-01_phasediff.dicom.tgz get sourcedata/sub-sid000005/ses-siemens1/func/sub-sid000005_ses-siemens1_task-life_acq-2mm692_run-04_bold.dicom.tgz get sourcedata/sub-sid000005/ses-siemens1/func/sub-sid000005_ses-siemens1_task-life_acq-2mm748_run-03_bold.dicom.tgz (transfer already in progress, or unable to take transfer lock) 
+  Unable to access these remotes: origin
+(from origin...) (from origin...) 
+  Try making some of these repositories available:
+
+
+  	2e44be07-8f1a-4c11-a7cb-464802b87b26 -- mvdoc@smaug:/mnt/btrfs/dbic/inbox/dbic-ds-3mm/dbic/pulse_sequences
+   	b2ff2964-c31b-4784-b094-2bebb336da91 -- mvdoc@smaug:/mnt/btrfs/dbic/inbox/dbic-ds/dbic/pulse_sequences
+   	d486ea11-98dc-42d3-9640-e5713acfb675 -- yoh@rolando:/inbox/BIDS/dbic/1000-dbic-dataset [origin]
+failed
+get sourcedata/sub-sid000005/ses-siemens1/func/sub-sid000005_ses-siemens1_task-life_acq-2mm754_run-05_bold.dicom.tgz (from origin...) 
+(from origin...) 
+...
+yohtest@rolando.cns's password: yohtest@rolando.cns's password: yohtest@rolando.cns's password: yohtest@rolando.cns's password
+
+"""]]
+
+I have entered password just once -- didn't try to enter it multiple times into the void ;)  but I guess it would be neat if annex could handle this situation gracefully (e.g. initiate central ssh controller first before spawning parallel getters) and demand password once
+
+
+[[!meta author=yoh]]
diff --git a/doc/bugs/get_-J_cannot_be_used_with_password-based_authentication/comment_1_236d2b897a550e7db4b266814d4e778d._comment b/doc/bugs/get_-J_cannot_be_used_with_password-based_authentication/comment_1_236d2b897a550e7db4b266814d4e778d._comment
new file mode 100644
index 0000000000..40e7ee2ca0
--- /dev/null
+++ b/doc/bugs/get_-J_cannot_be_used_with_password-based_authentication/comment_1_236d2b897a550e7db4b266814d4e778d._comment
@@ -0,0 +1,25 @@
+[[!comment format=mdwn
+ username="joey"
+ subject="""comment 1"""
+ date="2017-04-07T19:58:41Z"
+ content="""
+Well let's see.. To fix this would need some way for ssh to outsource its
+password prompting to another program, which could then serialize
+concurrent password requests, and perhaps reuse the same password when
+reconnecting to the same host.
+
+Sounds an aweful lot like ssh-agent, doesn't it?
+
+Now, it does happen to be the case that without -J, the password is only
+prompted for once to download multiple files from the same host. That works
+because of ssh connection caching. But in the -J case, the
+connection caching does not help, because multiple sshed are started before
+there's a connection to reuse, so each tries to make a new connection and
+prompts.
+
+Even if connection caching worked with -J, the general problem would remain
+when it did concurrent downloads from different hosts.
+
+So I tend to feel that this is just not fixable; if the user wants to use
+-J, they ought to use ssh-agent so it doesn't prompt for passwords.
+"""]]
diff --git a/doc/bugs/get_-J_cannot_be_used_with_password-based_authentication/comment_2_c877de08f959dee4ace34e66f42c8615._comment b/doc/bugs/get_-J_cannot_be_used_with_password-based_authentication/comment_2_c877de08f959dee4ace34e66f42c8615._comment
new file mode 100644
index 0000000000..c60ba6d4ad
--- /dev/null
+++ b/doc/bugs/get_-J_cannot_be_used_with_password-based_authentication/comment_2_c877de08f959dee4ace34e66f42c8615._comment
@@ -0,0 +1,21 @@
+[[!comment format=mdwn
+ username="yarikoptic"
+ avatar="http://cdn.libravatar.org/avatar/f11e9c84cb18d26a1748c33b48c924b4"
+ subject="may be?"
+ date="2017-04-07T21:01:56Z"
+ content="""
+well, it kinda depends at either at which level parallelization is happening or how parallel jobs handling is done, or may be ...
+
+level of parallelization:
+I guess ATM annex just parallelizes at the level of \"get --key KEY\" jobs.
+But if central process decided to try to \"get --from=remote --key KEY\" -- call which it submits to parallel work pull -- then it could first check if remote is an ssh remote and connection caching is established, and if not -- establish it and then submit this and/or any subsequent get call.
+This would though over-complicate the design I guess considerably, so probably shouldn't be approached.
+
+jobs handling:
+if parallel jobs could 'yield' back to the original process (e.g. if there was some protocoled exchange between them and master process... somewhat similar to git annex special remotes in a way) demanding some action (e.g. - authenticate me to the host) and then proceed back with its dues, could work out I guess.
+But I guess that is also not current implementation
+
+
+may be...:
+since I guess (didn't check) GIT_SSH_COMMAND is used (or not yet but could be?) for ssh transfers, such activity as establishing shared ssh connection could be deferred to it (with some proper locking/waiting for parallel invocations)... or am I wrong?
+"""]]
diff --git a/doc/bugs/get_-J_cannot_be_used_with_password-based_authentication/comment_3_daac1d424bc9e5b56772fa49707bc5a5._comment b/doc/bugs/get_-J_cannot_be_used_with_password-based_authentication/comment_3_daac1d424bc9e5b56772fa49707bc5a5._comment
new file mode 100644
index 0000000000..26f1aedc62
--- /dev/null
+++ b/doc/bugs/get_-J_cannot_be_used_with_password-based_authentication/comment_3_daac1d424bc9e5b56772fa49707bc5a5._comment
@@ -0,0 +1,7 @@
+[[!comment format=mdwn
+ username="joey"
+ subject="""comment 3"""
+ date="2017-04-07T21:06:51Z"
+ content="""
+How do you check if ssh has established a cached ssh connection?
+"""]]
diff --git a/doc/bugs/get_-J_cannot_be_used_with_password-based_authentication/comment_4_05bf20db275b911e2d89311182f289f6._comment b/doc/bugs/get_-J_cannot_be_used_with_password-based_authentication/comment_4_05bf20db275b911e2d89311182f289f6._comment
new file mode 100644
index 0000000000..27d39a7d88
--- /dev/null
+++ b/doc/bugs/get_-J_cannot_be_used_with_password-based_authentication/comment_4_05bf20db275b911e2d89311182f289f6._comment
@@ -0,0 +1,7 @@
+[[!comment format=mdwn
+ username="joey"
+ subject="""comment 4"""
+ date="2017-04-07T21:07:44Z"
+ content="""
+`GIT_SSH_COMMAND` is used for *every* call to ssh in git-annex.
+"""]]
diff --git a/doc/bugs/get_-J_cannot_be_used_with_password-based_authentication/comment_5_b094509fe0194313666b5b1db0a68156._comment b/doc/bugs/get_-J_cannot_be_used_with_password-based_authentication/comment_5_b094509fe0194313666b5b1db0a68156._comment
new file mode 100644
index 0000000000..4d904ca499
--- /dev/null
+++ b/doc/bugs/get_-J_cannot_be_used_with_password-based_authentication/comment_5_b094509fe0194313666b5b1db0a68156._comment
@@ -0,0 +1,14 @@
+[[!comment format=mdwn
+ username="yarikoptic"
+ avatar="http://cdn.libravatar.org/avatar/f11e9c84cb18d26a1748c33b48c924b4"
+ subject="comment 5"
+ date="2017-04-08T03:16:48Z"
+ content="""
+> How do you check if ssh has established a cached ssh connection?
+
+ssh -O check  -- somewhat of an additional overhead, but possible
+
+> GIT_SSH_COMMAND is used for every call to ssh in git-annex.
+
+so then theoretically we could implement \"may be ...\" strategy on our end in our sshrun.
+"""]]
diff --git a/doc/bugs/get_-J_cannot_be_used_with_password-based_authentication/comment_6_f5fe8d4cecfceec5cb4a03dd054d2e0a._comment b/doc/bugs/get_-J_cannot_be_used_with_password-based_authentication/comment_6_f5fe8d4cecfceec5cb4a03dd054d2e0a._comment
new file mode 100644
index 0000000000..2daef87279
--- /dev/null
+++ b/doc/bugs/get_-J_cannot_be_used_with_password-based_authentication/comment_6_f5fe8d4cecfceec5cb4a03dd054d2e0a._comment
@@ -0,0 +1,42 @@
+[[!comment format=mdwn
+ username="joey"
+ subject="""comment 6"""
+ date="2017-05-11T18:09:04Z"
+ content="""
+All cases could be dealt with by having a single process-level prompt lock
+(not a lock file, but an MVar), that's taken when doing something
+that might prompt for input.
+
+Then `Annex.Ssh.prepSocket` could block to take the prompt lock, and once
+it has the prompt lock, start the ssh connection multiplexer and wait for
+the the ssh connection to be established.
+
+Thus, even if `git annex get -J` is connecting to multiple hosts that each
+need passwords, password prompting would be serialized.
+
+All message output could also be blocked while the prompt lock is held,
+and then concurrent output would not scramble with the ssh password prompt.
+
+`ssh -S path -O check` does indeed exit nonzero when ssh has not yet
+connected and is at a password prompt. Also, I noticed that the socket file
+is only created after the password prompt, so a less expensive check
+(though perhaps not as accurate) is to see if the socket file exists.
+(But, it seems we don't need to check, see below.)
+
+The real problem is starting the ssh connection multiplexer without
+blocking for eg a whole rsync transfer to run. There's
+not a `-O` command that starts the multiplexer. The only way to do it seems
+to be something like `ssh -S path -o ControlMaster=auto -o 
+ControlPersist=yes host true`. So, run a no-op command on the remote host just
+to get the connection up. Then prepSocket will know the cached connection
+is up, and can drop the prompt lock and return.
+
+It would only need to do this when concurrency is enabled, so
+non-concurrent uses the current, faster path.
+
+prepSocket takes a shared 
+file level lock of the socket's lock file, which is used to tell when
+another git-annex process is using the connection multiplexer.
+So, an optimisation would be for prepSocket to check if it's already
+taken that shared lock, and then it does not need to start the multiplexer.
+"""]]
diff --git a/doc/bugs/get_-J_cannot_be_used_with_password-based_authentication/comment_7_0118a107147f6b94a7da907e599e58db._comment b/doc/bugs/get_-J_cannot_be_used_with_password-based_authentication/comment_7_0118a107147f6b94a7da907e599e58db._comment
new file mode 100644
index 0000000000..24f34bcc01
--- /dev/null
+++ b/doc/bugs/get_-J_cannot_be_used_with_password-based_authentication/comment_7_0118a107147f6b94a7da907e599e58db._comment
@@ -0,0 +1,15 @@
+[[!comment format=mdwn
+ username="joey"
+ subject="""comment 7"""
+ date="2017-05-11T19:23:02Z"
+ content="""
+What about when `GIT_SSH` is used? `prepSocket` is not used then,
+and git-annex can only use the `GIT_SSH` interface to ssh to the host.
+So, the approach above won't work.
+
+git-annex could then try to use `GIT_SSH` to ssh to the host and run eg `true`,
+in hopes that `GIT_SSH` is enabling ssh connection caching and that will
+get the ssh connection set up. If `GIT_SSH` is not enabling connection
+caching, that might add an additional password prompt, and not avoid
+other password prompts from overlapping.
+"""]]
diff --git a/doc/bugs/get_-J_cannot_be_used_with_password-based_authentication/comment_8_5da63cf5fa93120c85b98077fba51488._comment b/doc/bugs/get_-J_cannot_be_used_with_password-based_authentication/comment_8_5da63cf5fa93120c85b98077fba51488._comment
new file mode 100644
index 0000000000..8860bc536e
--- /dev/null
+++ b/doc/bugs/get_-J_cannot_be_used_with_password-based_authentication/comment_8_5da63cf5fa93120c85b98077fba51488._comment
@@ -0,0 +1,11 @@
+[[!comment format=mdwn
+ username="joey"
+ subject="""status"""
+ date="2017-05-11T21:57:24Z"
+ content="""
+Current status: It's implemented, but not for `GIT_SSH` yet.
+
+The display is a bit ugly, because the ssh password prompt line
+confuses the concurrent-output region manager. Opened
+[[minor_display_glitch_with_ssh_password_prompting_and_-J]] bug for that.
+"""]]
diff --git a/doc/bugs/get_-J_from_ssh_remote_tries_to_lock_in_home_directory__63__.mdwn b/doc/bugs/get_-J_from_ssh_remote_tries_to_lock_in_home_directory__63__.mdwn
new file mode 100644
index 0000000000..77b2b1ccf3
--- /dev/null
+++ b/doc/bugs/get_-J_from_ssh_remote_tries_to_lock_in_home_directory__63__.mdwn
@@ -0,0 +1,49 @@
+### Please describe the problem.
+
+had `datalad get -J4` lockup ... interrupted it, ran git annex directly to observe some errors reported and what looks like to be an attempt to determine if the home directory on remote end is a git repo... see below for details of such a run.  
+
+may be unrelated but also experiencing some lock ups [while "interacting" with this remote  from datalad](https://github.com/datalad/datalad/issues/2128)
+
+### What version of git-annex are you using? On what operating system?
+6.20180206+gitg638032f3a-1~ndall+1 on the local
+6.20180115-g56b56033a on remote
+
+### Please provide any additional information below.
+
+[[!format sh """
+$> 'git' '-c' 'receive.autogc=0' '-c' 'gc.auto=0' 'annex' 'get' '--debug' '-c' 'remote.rolando.annex-ssh-options=-o ControlMaster=auto -S /home/yoh/.cache/datalad/sockets/810d3ac4' '--json' '--json-progress' -J4  --from rolando --key SHA256E-s4295588--8db39d197775c7372bb1afff197ea724f158f3217ad064eef7ff8427a1502f15.tgz    
+fatal: Not a git repository: '../../../home/bids/.git'
+git-annex-shell: Not a git-annex or gcrypt repository.
+  Unable to run git-annex-shell on remote .
+{"byte-progress":4295588,"action":{"command":"get","note":"from rolando...","key":"SHA256E-s4295588--8db39d197775c7372bb1afff197ea724f158f3217ad064eef7ff8427a1502f15.tgz","file":null},"total-size":4295588,"percent-progress":"100%"}
+{"byte-progress":4295588,"action":{"command":"get","note":"from rolando...","key":"SHA256E-s4295588--8db39d197775c7372bb1afff197ea724f158f3217ad064eef7ff8427a1502f15.tgz","file":null},"total-size":4295588,"percent-progress":"100%"}
+{"command":"get","note":"checksum...","success":true,"key":"SHA256E-s4295588--8db39d197775c7372bb1afff197ea724f1  Unable to run git-annex-shell on remote .
+
+$> git annex drop 'sourcedata/sub-qa/ses-20180102'
+drop sourcedata/sub-qa/ses-20180102/anat/sub-qa_ses-20180102_scout.dicom.tgz (locking rolando...) ok
+(recording state in git...)
+
+$> 'git' '-c' 'receive.autogc=0' '-c' 'gc.auto=0' 'annex' 'get' '--debug' '-c' 'remote.rolando.annex-ssh-options=-o ControlMaster=auto -S /home/yoh/.cache/datalad/sockets/810d3ac4' '--json' '--json-progress' --from rolando --key SHA256E-s4295588--8db39d197775c7372bb1afff197ea724f158f3217ad064eef7ff8427a1502f15.tgz     
+{"byte-progress":4295588,"action":{"command":"get","note":"from rolando...","key":"SHA256E-s4295588--8db39d197775c7372bb1afff197ea724f158f3217ad064eef7ff8427a1502f15.tgz","file":null},"total-size":4295588,"percent-progress":"100%"}
+{"byte-progress":4295588,"action":{"command":"get","note":"from rolando...","key":"SHA256E-s4295588--8db39d197775c7372bb1afff197ea724f158f3217ad064eef7ff8427a1502f15.tgz","file":null},"total-size":4295588,"percent-progress":"100%"}
+{"command":"get","note":"checksum...","success":true,"key":"SHA256E-s4295588--8db39d197775c7372bb1afff197ea724f158f3217ad064eef7ff8427a1502f15.tgz","file":null}
+
+$> cat .git/config
+ ...
+[remote "rolando"]
+	url = bids@rolando.cns.dartmouth.edu:/inbox/BIDS/dbic/QA
+	fetch = +refs/heads/*:refs/remotes/rolando/*
+	annex-uuid = 6384a551-a41d-4290-b186-9258befede97
+	annex-ignore = false
+
+"""]]
+
+To get to that host I have `ProxyCommand ssh -q -A smaug.dartmouth.edu 'nc -w1 %h %p'` in my ~/.ssh/config for it (if relevant)
+
+### Have you had any luck using git-annex before? (Sometimes we get tired of reading bug reports all day and a lil' positive end note does wonders)
+
+Most of the days of the week. Friday is a tricky one
+
+[[!meta author=yoh]]
+
+
diff --git a/doc/bugs/get_-J_from_ssh_remote_tries_to_lock_in_home_directory__63__/comment_1_362a858ce78fa0b8d7949fb39fb247d9._comment b/doc/bugs/get_-J_from_ssh_remote_tries_to_lock_in_home_directory__63__/comment_1_362a858ce78fa0b8d7949fb39fb247d9._comment
new file mode 100644
index 0000000000..27f8f3fef5
--- /dev/null
+++ b/doc/bugs/get_-J_from_ssh_remote_tries_to_lock_in_home_directory__63__/comment_1_362a858ce78fa0b8d7949fb39fb247d9._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="yarikoptic"
+ avatar="http://cdn.libravatar.org/avatar/f11e9c84cb18d26a1748c33b48c924b4"
+ subject="comment 1"
+ date="2018-02-10T02:48:14Z"
+ content="""
+BTW remote location is an NFS mount ... git annex didn't show any issue working with it on that host any time recently
+"""]]
diff --git a/doc/bugs/get_from_the_web_remote_fails___40__redirect__63____41___-_lack_of_further_debug_info.mdwn b/doc/bugs/get_from_the_web_remote_fails___40__redirect__63____41___-_lack_of_further_debug_info.mdwn
new file mode 100644
index 0000000000..8c1fea6fd6
--- /dev/null
+++ b/doc/bugs/get_from_the_web_remote_fails___40__redirect__63____41___-_lack_of_further_debug_info.mdwn
@@ -0,0 +1,93 @@
+### Please describe the problem.
+
+Originally reported on [https://github.com/datalad/datalad/issues/2472](datalad issue 2472)
+
+First I present full initial dump, then minimal set of commands and a few observations (e.g. seems to be related to running it within that docker environment, but cannot figure out why)
+
+[[!format sh """
+$> docker run -it --rm miykael/datalad_test
+neuro@d0008523faf7:~$ cd ds000114/derivatives/fmriprep/
+neuro@d0008523faf7:~/ds000114/derivatives/fmriprep$ git annex get --debug sub-01.html
+[2018-05-06 15:30:05.79497316] read: git ["--git-dir=.git","--work-tree=.","--literal-pathspecs","ls-files","--cached","-z","--","sub-01.html"]
+get sub-01.html [2018-05-06 15:30:05.801191505] read: git ["--git-dir=.git","--work-tree=.","--literal-pathspecs","show-ref","git-annex"]
+[2018-05-06 15:30:05.828819907] process done ExitSuccess
+[2018-05-06 15:30:05.82896273] read: git ["--git-dir=.git","--work-tree=.","--literal-pathspecs","show-ref","--hash","refs/heads/git-annex"]
+[2018-05-06 15:30:05.832716117] process done ExitSuccess
+[2018-05-06 15:30:05.833138385] read: git ["--git-dir=.git","--work-tree=.","--literal-pathspecs","log","refs/heads/git-annex..7d2ed09660cc91efbeb52b5792fb372622463813","--pretty=%H","-n1"]
+[2018-05-06 15:30:05.837375012] process done ExitSuccess
+[2018-05-06 15:30:05.837896891] chat: git ["--git-dir=.git","--work-tree=.","--literal-pathspecs","cat-file","--batch"]
+[2018-05-06 15:30:05.838257537] chat: git ["--git-dir=.git","--work-tree=.","--literal-pathspecs","cat-file","--batch-check=%(objectname) %(objecttype) %(objectsize)"]
+(from web...) 
+
+(from web...) 
+
+
+  Unable to access these remotes: web
+
+  Try making some of these repositories available:
+  	00000000-0000-0000-0000-000000000001 -- web
+   	b8b7bab3-c978-4f50-8ed4-e77900198eb9 -- yoh@smaug:/mnt/btrfs/datasets/datalad/crawl/workshops/nih-workshop-2017/ds000114/derivatives/fmriprep
+failed
+[2018-05-06 15:30:05.877509068] process done ExitSuccess
+[2018-05-06 15:30:05.877986803] process done ExitSuccess
+git-annex: get: 1 failed
+
+neuro@d0008523faf7:~/ds000114/derivatives/fmriprep$ git annex whereis sub-01.html
+whereis sub-01.html (2 copies) 
+  	00000000-0000-0000-0000-000000000001 -- web
+   	b8b7bab3-c978-4f50-8ed4-e77900198eb9 -- yoh@smaug:/mnt/btrfs/datasets/datalad/crawl/workshops/nih-workshop-2017/ds000114/derivatives/fmriprep
+
+  web: https://dl.dropbox.com/s/f03sxu7omhsibyc/sub-01.html?dl=0
+ok
+
+neuro@d0008523faf7:~/ds000114/derivatives/fmriprep$ curl https://dl.dropbox.com/s/f03sxu7omhsibyc/sub-01.html?dl=0
+
+  Found
+  
+    
    Found
    
+    
    The resource was found at https://dl.dropboxusercontent.com/s/f03sxu7omhsibyc/sub-01.html?dl=0;
+you should be redirected automatically.
+
+
    
+    
    
+    
    WSGI Server
    
+  
+
+
+neuro@d0008523faf7:~/ds000114/derivatives/fmriprep$ git annex version
+git-annex version: 6.20180416+gitg86b18966f-1~ndall+1
+build flags: Assistant Webapp Pairing S3(multipartupload)(storageclasses) WebDAV Inotify DBus DesktopNotify ConcurrentOutput TorrentParser MagicMime Feeds Testsuite
+dependency versions: aws-0.14.1 bloomfilter-2.0.1.0 cryptonite-0.20 DAV-1.3.1 feed-0.3.11.1 ghc-8.0.1 http-client-0.4.31.1 persistent-sqlite-2.6 torrent-10000.0.0 uuid-1.3.12 yesod-1.4.3
+key/value backends: SHA256E SHA256 SHA512E SHA512 SHA224E SHA224 SHA384E SHA384 SHA3_256E SHA3_256 SHA3_512E SHA3_512 SHA3_224E SHA3_224 SHA3_384E SHA3_384 SKEIN256E SKEIN256 SKEIN512E SKEIN512 SHA1E SHA1 MD5E MD5 WORM URL
+remote types: git gcrypt p2p S3 bup directory rsync web bittorrent webdav adb tahoe glacier ddar hook external
+local repository version: 5
+supported repository versions: 3 5 6
+upgrade supported from repository versions: 0 1 2 3 4 5
+operating system: linux x86_64
+
+"""]]
+
+Minimal set of commands to replicate
+
+[[!format sh """
+git clone http://datasets.datalad.org/workshops/nih-2017/ds000114/derivatives/fmriprep/.git
+cd fmriprep
+git annex get --debug --from=web sub-01.html
+"""]]
+
+- I think the issue is somehow specific to running it within that docker -- if those commands ran on laptop, it works just fine!
+- Issue persists with the `6.20180501-g0d2527e67` manually installed in the container
+- Since then I also provided that file from the webserver, so if you do `git fetch`, it would start fetching that file from the web server (thus obfuscating the issue), so you would need ' --from=web ' to force an attempt to download from the web
+- if before, with invoking wget/curl commands it was possible to see what is going on, ATM `--debug` of git-annex does not give any information about what is attempted to be fetched from web remote and what problem it is experiencing.  I think it would be very valuable to get more troubleshooting information output in `--debug`
+
+
+[[!meta author=yoh]]
+
+> This does not involve redirects. It's hitting the url a
+> couple of times on failure, thus the multiple "from web".
+> That was due to a bug in the forward retry code; `Just 0 > Nothing`.
+> Fixed.
+> 
+> I added a display of the error message from the web server.
+> 
+> [[fixed|done]] --[[Joey]]
diff --git a/doc/bugs/get_over_ssh_fails_with___fd__58__19__58___hClose__58___resource_vanished.mdwn b/doc/bugs/get_over_ssh_fails_with___fd__58__19__58___hClose__58___resource_vanished.mdwn
new file mode 100644
index 0000000000..570b12b970
--- /dev/null
+++ b/doc/bugs/get_over_ssh_fails_with___fd__58__19__58___hClose__58___resource_vanished.mdwn
@@ -0,0 +1,68 @@
+### Please describe the problem.
+
+I think I never saw that before.  The only thing which differs now I think is the version of git-annex on this box:
+
+[[!format sh """
+$> git annex get --from origin --debug 'sub-qa/ses-20180312/func/sub-qa_ses-20180312_task-rest_acq-p2Xs4X35mm_bold.nii.gz'                              
+[2018-09-21 23:26:38.299280879] read: git ["--git-dir=.git","--work-tree=.","--literal-pathspecs","show-ref","git-annex"]
+[2018-09-21 23:26:38.303220623] process done ExitSuccess
+[2018-09-21 23:26:38.311081622] read: git ["--git-dir=.git","--work-tree=.","--literal-pathspecs","show-ref","--hash","refs/heads/git-annex"]
+[2018-09-21 23:26:38.314302424] process done ExitSuccess
+[2018-09-21 23:26:38.314584504] read: git ["--git-dir=.git","--work-tree=.","--literal-pathspecs","log","refs/heads/git-annex..4f0e8cd9a43b003228c3898133b1e200f9bdfc22","--pretty=%H","-n1"]
+[2018-09-21 23:26:38.321260451] process done ExitSuccess
+[2018-09-21 23:26:38.321567411] chat: git ["--git-dir=.git","--work-tree=.","--literal-pathspecs","cat-file","--batch"]
+[2018-09-21 23:26:38.322235784] chat: git ["--git-dir=.git","--work-tree=.","--literal-pathspecs","cat-file","--batch-check=%(objectname) %(objecttype) %(objectsize)"]
+[2018-09-21 23:26:38.327995657] read: git ["--git-dir=.git","--work-tree=.","--literal-pathspecs","ls-files","--cached","-z","--","sub-qa/ses-20180312/func/sub-qa_ses-20180312_task-rest_acq-p2Xs4X35mm_bold.nii.gz"]
+get sub-qa/ses-20180312/func/sub-qa_ses-20180312_task-rest_acq-p2Xs4X35mm_bold.nii.gz (from origin...) 
+[2018-09-21 23:26:38.336441538] chat: ssh ["bids@rolando.cns","-S",".git/annex/ssh/bids@rolando.cns","-o","ControlMaster=auto","-o","ControlPersist=yes","-T","git-annex-shell 'p2pstdio' '/inbox/BIDS/dbic/QA' '--debug' 'ba8f2cea-f229-422c-82be-6580e5e07ed5' --uuid 6384a551-a41d-4290-b186-9258befede97"]
+[2018-09-21 23:26:39.94999901] P2P > ERROR auth failed
+
+  fd:19: hClose: resource vanished (Broken pipe)
+failed
+[2018-09-21 23:26:39.951838683] read: ssh ["-O","stop","-S","bids@rolando.cns","-o","ControlMaster=auto","-o","ControlPersist=yes","localhost"]
+[2018-09-21 23:26:39.960970417] process done ExitSuccess
+[2018-09-21 23:26:39.961786471] process done ExitSuccess
+[2018-09-21 23:26:39.962311637] process done ExitSuccess
+
+$> git annex version
+git-annex version: 6.20180913+git33-g2cd5a723f-1~ndall+1
+build flags: Assistant Webapp Pairing S3(multipartupload)(storageclasses) WebDAV Inotify DBus DesktopNotify ConcurrentOutput TorrentParser MagicMime Feeds Testsuite
+dependency versions: aws-0.19 bloomfilter-2.0.1.0 cryptonite-0.25 DAV-1.3.2 feed-1.0.0.0 ghc-8.2.2 http-client-0.5.12 persistent-sqlite-2.8.1.2 torrent-10000.1.1 uuid-1.3.13 yesod-1.6.0
+key/value backends: SHA256E SHA256 SHA512E SHA512 SHA224E SHA224 SHA384E SHA384 SHA3_256E SHA3_256 SHA3_512E SHA3_512 SHA3_224E SHA3_224 SHA3_384E SHA3_384 SKEIN256E SKEIN256 SKEIN512E SKEIN512 BLAKE2B256E BLAKE2B256 BLAKE2B512E BLAKE2B512 BLAKE2B160E BLAKE2B160 BLAKE2B224E BLAKE2B224 BLAKE2B384E BLAKE2B384 BLAKE2S256E BLAKE2S256 BLAKE2S160E BLAKE2S160 BLAKE2S224E BLAKE2S224 BLAKE2SP256E BLAKE2SP256 BLAKE2SP224E BLAKE2SP224 SHA1E SHA1 MD5E MD5 WORM URL
+remote types: git gcrypt p2p S3 bup directory rsync web bittorrent webdav adb tahoe glacier ddar hook external
+operating system: linux x86_64
+supported repository versions: 3 5 6
+upgrade supported from repository versions: 0 1 2 3 4 5
+local repository version: 5
+
+$> ssh -S .git/annex/ssh/bids@rolando.cns bids@rolando.cns ls /inbox/BIDS/dbic/QA
+                                                                                 
+
+    Dartmouth College, Department of Psychological and Brain Sciences
+                      Authorized access only
+
+
+
+
+CHANGES
+code
+...
+
+$> ssh -S .git/annex/ssh/bids@rolando.cns bids@rolando.cns git annex version     
+
+
+    Dartmouth College, Department of Psychological and Brain Sciences
+                      Authorized access only
+
+
+git-annex version: 6.20180115-g56b56033a
+
+"""]]
+
+so ssh connection seems to be ok
+
+what could it be?
+
+[[!meta author=yoh]]
+
+> [[fixed|done]] --[[Joey]]
diff --git a/doc/bugs/get_over_ssh_fails_with___fd__58__19__58___hClose__58___resource_vanished/comment_10_dd7883772bb37fa85aa1332633184071._comment b/doc/bugs/get_over_ssh_fails_with___fd__58__19__58___hClose__58___resource_vanished/comment_10_dd7883772bb37fa85aa1332633184071._comment
new file mode 100644
index 0000000000..b4f18dade4
--- /dev/null
+++ b/doc/bugs/get_over_ssh_fails_with___fd__58__19__58___hClose__58___resource_vanished/comment_10_dd7883772bb37fa85aa1332633184071._comment
@@ -0,0 +1,18 @@
+[[!comment format=mdwn
+ username="joey"
+ subject="""comment 1"""
+ date="2018-09-25T17:18:14Z"
+ content="""
+Ok, reproduced by downgrading git-annex, so it is a reversion in the
+fallback from p2pstdio.
+
+It is indeed caused by [[!commit b18fb1e343e9654207fbebacf686659c75d0fb4c]],
+because that adds a new ProtocolEOF value, and postAuth has a fallthrough
+case that tries to then send a ERROR message back over the connection, but
+by that time the pipe has closed.
+
+So that was an easy fix. I do still wonder how a system that displays
+/etc/issue or whatever to noninteractive logins could have ever worked with
+rsync or git-annex. But if that system doesn't work after this fix, it's an
+unrelated problem.
+"""]]
diff --git a/doc/bugs/get_over_ssh_fails_with___fd__58__19__58___hClose__58___resource_vanished/comment_11_375e6d23c43a2cc0ef8bb81b06c3bcc9._comment b/doc/bugs/get_over_ssh_fails_with___fd__58__19__58___hClose__58___resource_vanished/comment_11_375e6d23c43a2cc0ef8bb81b06c3bcc9._comment
new file mode 100644
index 0000000000..7887e73ef2
--- /dev/null
+++ b/doc/bugs/get_over_ssh_fails_with___fd__58__19__58___hClose__58___resource_vanished/comment_11_375e6d23c43a2cc0ef8bb81b06c3bcc9._comment
@@ -0,0 +1,47 @@
+[[!comment format=mdwn
+ username="yarikoptic"
+ avatar="http://cdn.libravatar.org/avatar/f11e9c84cb18d26a1748c33b48c924b4"
+ subject="comment 11"
+ date="2018-09-26T00:10:40Z"
+ content="""
+FWIW confirming that the issue is resolved for my attempts while interacting with that very old git-annex on a server side
+
+[[!format sh \"\"\"
+(git-annex)hopa:/tmp/repo[master]
+$> git annex get 1                              
+get 1 (from origin...) 
+ 
+  fd:19: hClose: resource vanished (Broken pipe)
+
+  Unable to access these remotes: origin
+
+  Try making some of these repositories available:
+  	f6bedfe8-0e58-4182-bca3-764c5c3ed17b -- yoh@washoe:/tmp/repo [origin]
+failed
+git-annex: get: 1 failed
+1 5943 ->1.....................................:Tue 25 Sep 2018 08:07:34 PM EDT:.
+(git-annex)hopa:/tmp/repo[master]
+$> sudo dpkg -i ../git-annex-standalone_6.20180913+git149-g23bd27773-1\~ndall+1_amd64.deb 
+[sudo] password for yoh: 
+(Reading database ... 825207 files and directories currently installed.)
+Preparing to unpack .../git-annex-standalone_6.20180913+git149-g23bd27773-1~ndall+1_amd64.deb ...
+Unpacking git-annex-standalone (6.20180913+git149-g23bd27773-1~ndall+1) over (6.20180913+git52-gdb1644808-1~ndall+1) ...
+Setting up git-annex-standalone (6.20180913+git149-g23bd27773-1~ndall+1) ...
+Processing triggers for gnome-menus (3.13.3-11) ...
+Processing triggers for desktop-file-utils (0.23-2) ...
+Processing triggers for mime-support (3.60) ...
+Processing triggers for doc-base (0.10.8) ...
+Processing 1 changed doc-base file...
+Registering documents with scrollkeeper...
+Processing triggers for hicolor-icon-theme (0.17-2) ...
+Processing triggers for man-db (2.8.2-1) ...
+sudo dpkg -i   4.98s user 1.95s system 65% cpu 10.632 total
+1 5944.....................................:Tue 25 Sep 2018 08:07:52 PM EDT:.
+(git-annex)hopa:/tmp/repo[master]
+$> git annex get 1                                                                       
+get 1 (from origin...) 
+(checksum...) ok                  
+(recording state in git...)
+
+\"\"\"]]
+"""]]
diff --git a/doc/bugs/get_over_ssh_fails_with___fd__58__19__58___hClose__58___resource_vanished/comment_1_032dfcb0dc7e5c54a0cbc6634ad77268._comment b/doc/bugs/get_over_ssh_fails_with___fd__58__19__58___hClose__58___resource_vanished/comment_1_032dfcb0dc7e5c54a0cbc6634ad77268._comment
new file mode 100644
index 0000000000..6bf1259cb0
--- /dev/null
+++ b/doc/bugs/get_over_ssh_fails_with___fd__58__19__58___hClose__58___resource_vanished/comment_1_032dfcb0dc7e5c54a0cbc6634ad77268._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="bremner"
+ avatar="http://cdn.libravatar.org/avatar/22c2d800db6a7699139df604a67cb221"
+ subject="Similar failure with one host running 6.20180913-1"
+ date="2018-09-22T18:38:23Z"
+ content="""
+I have two up to date debian testing machines on my desk, same kernel, both running 6.20180913-1 from Debian. One of them sees these \"auth failed\" but not the other.  The server they are trying to fetch from is running 6.20170101-1+deb9u2. The one that is failing  has a restrictive firewall on it; could that be the problem? In principle it is supposed to allow all outbound traffic. Certainly ssh to the server works.
+"""]]
diff --git a/doc/bugs/get_over_ssh_fails_with___fd__58__19__58___hClose__58___resource_vanished/comment_2_1c079d165831b790676dc6bec0549773._comment b/doc/bugs/get_over_ssh_fails_with___fd__58__19__58___hClose__58___resource_vanished/comment_2_1c079d165831b790676dc6bec0549773._comment
new file mode 100644
index 0000000000..8eb24f15bd
--- /dev/null
+++ b/doc/bugs/get_over_ssh_fails_with___fd__58__19__58___hClose__58___resource_vanished/comment_2_1c079d165831b790676dc6bec0549773._comment
@@ -0,0 +1,21 @@
+[[!comment format=mdwn
+ username="bremner"
+ avatar="http://cdn.libravatar.org/avatar/22c2d800db6a7699139df604a67cb221"
+ subject="Dropped the firewall, no change. Maybe it's IPv6?"
+ date="2018-09-22T18:56:12Z"
+ content="""
+
    +convex# iptables -L      
+Chain INPUT (policy ACCEPT)
+target     prot opt source               destination         
+
+Chain FORWARD (policy DROP)
+target     prot opt source               destination         
+
+Chain OUTPUT (policy ACCEPT)
+target     prot opt source               destination         
+convex# 
+
    
+
+The other difference is that the non-working machine is using IPv6 and IPv4, while the working machine is just IPv4
+"""]]
diff --git a/doc/bugs/get_over_ssh_fails_with___fd__58__19__58___hClose__58___resource_vanished/comment_3_d83330ad40f4f2c2c55f29acf6680620._comment b/doc/bugs/get_over_ssh_fails_with___fd__58__19__58___hClose__58___resource_vanished/comment_3_d83330ad40f4f2c2c55f29acf6680620._comment
new file mode 100644
index 0000000000..7526fc8e6e
--- /dev/null
+++ b/doc/bugs/get_over_ssh_fails_with___fd__58__19__58___hClose__58___resource_vanished/comment_3_d83330ad40f4f2c2c55f29acf6680620._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="gueux"
+ avatar="http://cdn.libravatar.org/avatar/47e44a21505727b2d6bb5d88f0468f34"
+ subject="comment 3"
+ date="2018-09-24T09:03:13Z"
+ content="""
+I'm seeing this bug, too. It happens between my laptop, which is also running 6.20180913-1, and two servers which both run 6.20170101-1+deb9u2. One of the servers is remote, while the other one is on my local network.
+"""]]
diff --git a/doc/bugs/get_over_ssh_fails_with___fd__58__19__58___hClose__58___resource_vanished/comment_4_129a1946d7c76b89ae2f45b8a804959f._comment b/doc/bugs/get_over_ssh_fails_with___fd__58__19__58___hClose__58___resource_vanished/comment_4_129a1946d7c76b89ae2f45b8a804959f._comment
new file mode 100644
index 0000000000..158800bcee
--- /dev/null
+++ b/doc/bugs/get_over_ssh_fails_with___fd__58__19__58___hClose__58___resource_vanished/comment_4_129a1946d7c76b89ae2f45b8a804959f._comment
@@ -0,0 +1,10 @@
+[[!comment format=mdwn
+ username="yarikoptic"
+ avatar="http://cdn.libravatar.org/avatar/f11e9c84cb18d26a1748c33b48c924b4"
+ subject="downgrading helped! "
+ date="2018-09-24T17:56:42Z"
+ content="""
+Downgrading (testing on local laptop) local git-annex all the way down (didn't try intermediates yet) to 6.20170101-1+deb9u2  helped - I can fetch data from that server.
+
+My wild guess is that it all relates to the fact that remote git-annex-shell, being outdated, has no clue about `p2pstdio` command, and local new git-annex doesn't fall back anyhow to use some other way to transfer.
+"""]]
diff --git a/doc/bugs/get_over_ssh_fails_with___fd__58__19__58___hClose__58___resource_vanished/comment_4_1713d56b81db533db4532f2a79250c87._comment b/doc/bugs/get_over_ssh_fails_with___fd__58__19__58___hClose__58___resource_vanished/comment_4_1713d56b81db533db4532f2a79250c87._comment
new file mode 100644
index 0000000000..35eadd4a57
--- /dev/null
+++ b/doc/bugs/get_over_ssh_fails_with___fd__58__19__58___hClose__58___resource_vanished/comment_4_1713d56b81db533db4532f2a79250c87._comment
@@ -0,0 +1,21 @@
+[[!comment format=mdwn
+ username="joey"
+ subject="""comment 4"""
+ date="2018-09-24T18:37:08Z"
+ content="""
+The only new change in 6.20180913 that might somehow affect this is 
+[[!commit b18fb1e343e9654207fbebacf686659c75d0fb4c]]. But I don't see how
+it could really.
+
+Since ssh seems to be displaying a motd when running a noninteractive
+command, I think that git-annex must be seeing that motd where it
+expects to see the P2P protocol.
+
+Although it may be that ssh -T avoids the motd display. (But unfortunately
+your transcript using -T trims out exactly that part!) I didn't think ssh
+normally displays motd when not running a login shell. Perhaps it's
+something to do with your shell and not ssh.
+
+git-annex will fall back to not using p2pstdio, but only if the command
+exits nonzero without ssh sending anything to stdout.
+"""]]
diff --git a/doc/bugs/get_over_ssh_fails_with___fd__58__19__58___hClose__58___resource_vanished/comment_5_a14f999b63905c1f29999bd68ded42f2._comment b/doc/bugs/get_over_ssh_fails_with___fd__58__19__58___hClose__58___resource_vanished/comment_5_a14f999b63905c1f29999bd68ded42f2._comment
new file mode 100644
index 0000000000..239e4e32fd
--- /dev/null
+++ b/doc/bugs/get_over_ssh_fails_with___fd__58__19__58___hClose__58___resource_vanished/comment_5_a14f999b63905c1f29999bd68ded42f2._comment
@@ -0,0 +1,16 @@
+[[!comment format=mdwn
+ username="yarikoptic"
+ avatar="http://cdn.libravatar.org/avatar/f11e9c84cb18d26a1748c33b48c924b4"
+ subject="comment 5"
+ date="2018-09-24T18:05:15Z"
+ content="""
+FWIW, here is what I see if I just try to invoke directly
+[[!format sh \"\"\"
+$> ssh bids@rolando.cns -T git-annex-shell 'p2pstdio' '/inbox/BIDS/dbic/QA' '--debug' 'ba8f2cea-f229-422c-82be-6580e5e07ed5' --uuid 6384a551-a41d-4290-b186-9258befede97 
+...
+fatal: Run with no arguments or with -c cmd
+git-annex-shell: git-shell failed
+1 5358 ->1
+\"\"\"]]
+so - exit code is `1` (happen annex decides on exit code about availability of p2pstdio)
+"""]]
diff --git a/doc/bugs/get_over_ssh_fails_with___fd__58__19__58___hClose__58___resource_vanished/comment_7_31ecf1c31511e6454a274e3c9a9bef75._comment b/doc/bugs/get_over_ssh_fails_with___fd__58__19__58___hClose__58___resource_vanished/comment_7_31ecf1c31511e6454a274e3c9a9bef75._comment
new file mode 100644
index 0000000000..e7aa3c6b99
--- /dev/null
+++ b/doc/bugs/get_over_ssh_fails_with___fd__58__19__58___hClose__58___resource_vanished/comment_7_31ecf1c31511e6454a274e3c9a9bef75._comment
@@ -0,0 +1,26 @@
+[[!comment format=mdwn
+ username="yarikoptic"
+ avatar="http://cdn.libravatar.org/avatar/f11e9c84cb18d26a1748c33b48c924b4"
+ subject="comment 7"
+ date="2018-09-25T16:15:54Z"
+ content="""
+indeed I do have /etc/issue (not motd) displayed even when with -T:
+
+[[!format sh \"\"\"
+
+        $> ssh bids@rolando.cns -T git-annex-shell 'p2pstdio' '/inbox/BIDS/dbic/QA' '--debug' 'ba8f2cea-f229-422c-82be-6580e5e07ed5' --uuid 6384a551-a41d-4290-b186-9258befede97
+
+
+                Dartmouth College, Department of Psychological and Brain Sciences
+                                                  Authorized access only
+
+
+
+
+        fatal: Run with no arguments or with -c cmd
+        git-annex-shell: git-shell failed
+        1 5383 ->1
+
+\"\"\"]]
+
+"""]]
diff --git a/doc/bugs/get_over_ssh_fails_with___fd__58__19__58___hClose__58___resource_vanished/comment_8_3aecc3467427b4ebecbd53012e111004._comment b/doc/bugs/get_over_ssh_fails_with___fd__58__19__58___hClose__58___resource_vanished/comment_8_3aecc3467427b4ebecbd53012e111004._comment
new file mode 100644
index 0000000000..8df2246133
--- /dev/null
+++ b/doc/bugs/get_over_ssh_fails_with___fd__58__19__58___hClose__58___resource_vanished/comment_8_3aecc3467427b4ebecbd53012e111004._comment
@@ -0,0 +1,42 @@
+[[!comment format=mdwn
+ username="yarikoptic"
+ avatar="http://cdn.libravatar.org/avatar/f11e9c84cb18d26a1748c33b48c924b4"
+ subject="comment 8"
+ date="2018-09-25T16:18:58Z"
+ content="""
+replicated on another host with a VERY outdated annex (5.20141125), but where there is no issue or motd:
+[[!format sh \"\"\"
+$> git annex get --debug 1
+[2018-09-25 12:17:02.454369526] read: git [\"--git-dir=.git\",\"--work-tree=.\",\"--literal-pathspecs\",\"ls-files\",\"--cached\",\"-z\",\"--\",\"1\"]
+get 1 [2018-09-25 12:17:02.463349835] read: git [\"--git-dir=.git\",\"--work-tree=.\",\"--literal-pathspecs\",\"show-ref\",\"git-annex\"]
+[2018-09-25 12:17:02.467436881] process done ExitSuccess
+[2018-09-25 12:17:02.467538938] read: git [\"--git-dir=.git\",\"--work-tree=.\",\"--literal-pathspecs\",\"show-ref\",\"--hash\",\"refs/heads/git-annex\"]
+[2018-09-25 12:17:02.471371867] process done ExitSuccess
+[2018-09-25 12:17:02.471566695] read: git [\"--git-dir=.git\",\"--work-tree=.\",\"--literal-pathspecs\",\"log\",\"refs/heads/git-annex..70d4b5032ee0c3d4de499d650c71bdb0d55d2dee\",\"--pretty=%H\",\"-n1\"]
+[2018-09-25 12:17:02.476070612] process done ExitSuccess
+[2018-09-25 12:17:02.476399866] chat: git [\"--git-dir=.git\",\"--work-tree=.\",\"--literal-pathspecs\",\"cat-file\",\"--batch\"]
+[2018-09-25 12:17:02.477000311] chat: git [\"--git-dir=.git\",\"--work-tree=.\",\"--literal-pathspecs\",\"cat-file\",\"--batch-check=%(objectname) %(objecttype) %(objectsize)\"]
+(from origin...) 
+[2018-09-25 12:17:02.484004025] chat: ssh [\"washoe\",\"-S\",\".git/annex/ssh/washoe\",\"-o\",\"ControlMaster=auto\",\"-o\",\"ControlPersist=yes\",\"-T\",\"git-annex-shell 'p2pstdio' '/tmp/repo' '--debug' '84085bc4-d338-41eb-be4d-9b0d96536860' --uuid f6bedfe8-0e58-4182-bca3-764c5c3ed17b\"]
+[2018-09-25 12:17:02.695407743] P2P > ERROR auth failed
+
+  fd:19: hClose: resource vanished (Broken pipe)
+
+  Unable to access these remotes: origin
+
+  Try making some of these repositories available:
+  	f6bedfe8-0e58-4182-bca3-764c5c3ed17b -- yoh@washoe:/tmp/repo [origin]
+failed
+[2018-09-25 12:17:02.699113109] read: ssh [\"-O\",\"stop\",\"-S\",\"washoe\",\"-o\",\"ControlMaster=auto\",\"-o\",\"ControlPersist=yes\",\"localhost\"]
+[2018-09-25 12:17:02.709050558] process done ExitSuccess
+[2018-09-25 12:17:02.709515542] process done ExitSuccess
+[2018-09-25 12:17:02.710090983] process done ExitSuccess
+git-annex: get: 1 failed
+
+$> ssh washoe -T git-annex-shell 'p2pstdio' '/inbox/BIDS/dbic/QA' '--debug' 'ba8f2cea-f229-422c-82be-6580e5e07ed5' --uuid 6384a551-a41d-4290-b186-9258befede97       
+fatal: Run with no arguments or with -c cmd
+git-annex-shell: git-shell failed
+
+\"\"\"]]
+so I think it is incorrectly handles this failure and doesn't fall back to ssh/rsync.  nothing to do with motd/issue
+"""]]
diff --git a/doc/bugs/get_over_ssh_fails_with___fd__58__19__58___hClose__58___resource_vanished/comment_9_dfb1b7ee4e674cf18d6f41e4f4b6c879._comment b/doc/bugs/get_over_ssh_fails_with___fd__58__19__58___hClose__58___resource_vanished/comment_9_dfb1b7ee4e674cf18d6f41e4f4b6c879._comment
new file mode 100644
index 0000000000..024d71f740
--- /dev/null
+++ b/doc/bugs/get_over_ssh_fails_with___fd__58__19__58___hClose__58___resource_vanished/comment_9_dfb1b7ee4e674cf18d6f41e4f4b6c879._comment
@@ -0,0 +1,21 @@
+[[!comment format=mdwn
+ username="joey"
+ subject="""comment 9"""
+ date="2018-09-25T16:37:53Z"
+ content="""
+I tried configuring sshd with PrintMotd yes, that actually resulted in
+double motd on interactive login, but still none on noninteractive login,
+and doesn't cause this problem.
+
+I tried making ~/.bashrc echo some stuff. That also didn't reproduce the
+problem, though debug does have this:
+
+	[2018-09-25 12:40:18.092800592] P2P > ERROR auth failed
+
+It does not give up, but goes on to use rsync. Although rsync unsurprisingly 
+also fails "protocol version mismatch -- is your shell clean?".
+
+Still, this seems to indicate that the motd display shown in your transcripts is
+at the root of the problem, although I've not fully replicated the circumstances
+and will need more information to get further on this.
+"""]]
diff --git a/doc/bugs/git-annex-export_treeish_subdir_path_does_not_exist__91____91__done__93____93__.mdwn b/doc/bugs/git-annex-export_treeish_subdir_path_does_not_exist__91____91__done__93____93__.mdwn
new file mode 100644
index 0000000000..83a418e59e
--- /dev/null
+++ b/doc/bugs/git-annex-export_treeish_subdir_path_does_not_exist__91____91__done__93____93__.mdwn
@@ -0,0 +1,53 @@
+### Please describe the problem.
+When exporting a treeish with a subdir to a dir special remote, it fails with the error: 
+[[!format sh """
+git annex export master:subdir --to exportdir
+
+fatal: Path 'subdir:' does not exist (neither on disk nor in the index).
+git-annex: unknown tree
+"""]]
+
+I tried other treeish variations such as :./subdir, master:subdir but it didn't work either.
+What is very strange to me is the ':' appended at the end of the path in the error.
+Any idea what is going on here ? 
+
+
+### What steps will reproduce the problem?
+[[!format sh """
+mkdir annex
+mv annex
+git init annex
+git annex init --version=6
+touch a.txt
+mkdir subdir
+touch subdir/b.txt
+git commit -m "initial commit"
+mkdir ../exportdir
+git annex initremote exportdir type=directory directory=../exportdir/ encryption=none exporttree=yes
+git annex export master:subdir --to exportdir
+
+fatal: Path 'subdir:' does not exist (neither on disk nor in the index).
+git-annex: unknown tree
+"""]]
+
+
+### What version of git-annex are you using? On what operating system?
+[[!format sh """
+git annex version
+
+git-annex version: 6.20171003
+build flags: Assistant Webapp Pairing Testsuite S3(multipartupload)(storageclasses) WebDAV Inotify DBus DesktopNotify ConcurrentOutput TorrentParser MagicMime Feeds Quvi
+dependency versions: aws-0.13.0 bloomfilter-2.0.1.0 cryptonite-0.10 DAV-1.2 feed-0.3.10.4 ghc-7.10.3 http-client-0.4.26.2 persistent-sqlite-2.2 torrent-10000.0.0 uuid-1.3.11 yesod-1.4.2
+key/value backends: SHA256E SHA256 SHA512E SHA512 SHA224E SHA224 SHA384E SHA384 SHA3_256E SHA3_256 SHA3_512E SHA3_512 SHA3_224E SHA3_224 SHA3_384E SHA3_384 SKEIN256E SKEIN256 SKEIN512E SKEIN512 SHA1E SHA1 MD5E MD5 WORM URL
+remote types: git gcrypt p2p S3 bup directory rsync web bittorrent webdav tahoe glacier ddar hook external
+local repository version: 6
+supported repository versions: 3 5 6
+upgrade supported from repository versions: 0 1 2 3 4 5
+operating system: linux x86_64
+"""]]
+
+
+### Have you had any luck using git-annex before? (Sometimes we get tired of reading bug reports all day and a lil' positive end note does wonders)
+Yep, it has been a long time since I used it but I am back to see what I can do to manage my files properly :)
+
+> [[done]] --[[Joey]]
diff --git a/doc/bugs/git-annex-export_treeish_subdir_path_does_not_exist__91____91__done__93____93__/comment_1_ef090d1e29d1724aa1751b6cf874c537._comment b/doc/bugs/git-annex-export_treeish_subdir_path_does_not_exist__91____91__done__93____93__/comment_1_ef090d1e29d1724aa1751b6cf874c537._comment
new file mode 100644
index 0000000000..42a4155453
--- /dev/null
+++ b/doc/bugs/git-annex-export_treeish_subdir_path_does_not_exist__91____91__done__93____93__/comment_1_ef090d1e29d1724aa1751b6cf874c537._comment
@@ -0,0 +1,11 @@
+[[!comment format=mdwn
+ username="webanck"
+ avatar="http://cdn.libravatar.org/avatar/cd273f76ef8c4218510b4f50ef7e1f3d"
+ subject="comment 1"
+ date="2018-07-31T12:41:44Z"
+ content="""
+Well, I can answer myself now.
+It wasn't working because my version of git was too old.
+I updated to version 2.18.0 and it works very well now !
+It might be a good idea to find and specify the minimal git version needed to make this work next in the man page of git annex export for instance ;)
+"""]]
diff --git a/doc/bugs/git-annex-export_treeish_subdir_path_does_not_exist__91____91__done__93____93__/comment_2_d3808b85328fbbc6f4aef5086e225187._comment b/doc/bugs/git-annex-export_treeish_subdir_path_does_not_exist__91____91__done__93____93__/comment_2_d3808b85328fbbc6f4aef5086e225187._comment
new file mode 100644
index 0000000000..68aea9ad74
--- /dev/null
+++ b/doc/bugs/git-annex-export_treeish_subdir_path_does_not_exist__91____91__done__93____93__/comment_2_d3808b85328fbbc6f4aef5086e225187._comment
@@ -0,0 +1,10 @@
+[[!comment format=mdwn
+ username="joey"
+ subject="""comment 2"""
+ date="2018-08-03T17:31:39Z"
+ content="""
+It wasn't upgrading git that fixed it. The bug you reported was present in
+git-annex versions before 6.20171109. You had an older version listed
+in the bug report, so I assume you also upgraded git-annex to the fixed
+version.
+"""]]
diff --git a/doc/bugs/git-annex-shell/comment_1_3d2c3827de34509c0a5595eda07dd18f._comment b/doc/bugs/git-annex-shell/comment_1_3d2c3827de34509c0a5595eda07dd18f._comment
new file mode 100644
index 0000000000..6123471eb6
--- /dev/null
+++ b/doc/bugs/git-annex-shell/comment_1_3d2c3827de34509c0a5595eda07dd18f._comment
@@ -0,0 +1,13 @@
+[[!comment format=mdwn
+ username="joey"
+ subject="""comment 1"""
+ date="2015-04-09T17:48:20Z"
+ content="""
+GIT_SSH is set to the full path of the binary, unless
+~/.config/git-annex/program overrides it.
+
+Finding the full path to the binary is not a trivial or error-free
+operation.
+
+Could you please follow up to this bug or close it?
+"""]]
diff --git a/doc/bugs/git-annex-shell_-c_git-annex-shell_doesn__39__t_work__44___while_git-annex_expects_it_to.mdwn b/doc/bugs/git-annex-shell_-c_git-annex-shell_doesn__39__t_work__44___while_git-annex_expects_it_to.mdwn
new file mode 100644
index 0000000000..6bd4b4b1ed
--- /dev/null
+++ b/doc/bugs/git-annex-shell_-c_git-annex-shell_doesn__39__t_work__44___while_git-annex_expects_it_to.mdwn
@@ -0,0 +1,26 @@
+### Please describe the problem.
+git-annex-shell -c git-annex-shell errors, and git-annex interprets this as meaning that git-annex-shell is normal git-shell.
+
+### What steps will reproduce the problem?
+Running git-annex-shell -c git-annex-shell.
+
+### What version of git-annex are you using? On what operating system?
+NixOS, 6.20180529.
+
+### Please provide any additional information below.
+
+[[!format sh """
+# If you can, paste a complete transcript of the problem occurring here.
+# If the problem is with the git-annex assistant, paste in .git/annex/daemon.log
+
+[leo60228@nixos:~/test]$ git-annex-shell -c git-annex-shell
+fatal: unrecognized command 'git-annex-shell'
+git-annex-shell: git-shell failed
+
+# End of transcript or log.
+"""]]
+
+### Have you had any luck using git-annex before? (Sometimes we get tired of reading bug reports all day and a lil' positive end note does wonders)
+Yes.
+
+[[moreinfo]]
diff --git a/doc/bugs/git-annex-shell_-c_git-annex-shell_doesn__39__t_work__44___while_git-annex_expects_it_to/comment_1_b0a4f459d40bca7a0b24d1bec1e0f4a8._comment b/doc/bugs/git-annex-shell_-c_git-annex-shell_doesn__39__t_work__44___while_git-annex_expects_it_to/comment_1_b0a4f459d40bca7a0b24d1bec1e0f4a8._comment
new file mode 100644
index 0000000000..de19505f5e
--- /dev/null
+++ b/doc/bugs/git-annex-shell_-c_git-annex-shell_doesn__39__t_work__44___while_git-annex_expects_it_to/comment_1_b0a4f459d40bca7a0b24d1bec1e0f4a8._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="joey"
+ subject="""comment 1"""
+ date="2018-08-03T18:18:34Z"
+ content="""
+Why are you trying to do this? Is there a normal git-annex use case that
+involves that happening?
+"""]]
diff --git a/doc/bugs/git-annex-shell_-c_git-annex-shell_doesn__39__t_work__44___while_git-annex_expects_it_to/comment_3_335fea47cef83b6dfad7ce8c53193f82._comment b/doc/bugs/git-annex-shell_-c_git-annex-shell_doesn__39__t_work__44___while_git-annex_expects_it_to/comment_3_335fea47cef83b6dfad7ce8c53193f82._comment
new file mode 100644
index 0000000000..7140359949
--- /dev/null
+++ b/doc/bugs/git-annex-shell_-c_git-annex-shell_doesn__39__t_work__44___while_git-annex_expects_it_to/comment_3_335fea47cef83b6dfad7ce8c53193f82._comment
@@ -0,0 +1,12 @@
+[[!comment format=mdwn
+ username="joey"
+ subject="""comment 3"""
+ date="2018-08-08T15:04:09Z"
+ content="""
+In what configuration?
+
+This clearly does not affect the general case, or everyone who is happily
+using git-annex would have reported the bug.
+
+Please provide enough information to reproduce your problem.
+"""]]
diff --git a/doc/bugs/git-annex-shell_-c_git-annex-shell_doesn__39__t_work__44___while_git-annex_expects_it_to/comment_3_abaeb3d5c0322deeec790d80b2948549._comment b/doc/bugs/git-annex-shell_-c_git-annex-shell_doesn__39__t_work__44___while_git-annex_expects_it_to/comment_3_abaeb3d5c0322deeec790d80b2948549._comment
new file mode 100644
index 0000000000..a02a758689
--- /dev/null
+++ b/doc/bugs/git-annex-shell_-c_git-annex-shell_doesn__39__t_work__44___while_git-annex_expects_it_to/comment_3_abaeb3d5c0322deeec790d80b2948549._comment
@@ -0,0 +1,9 @@
+[[!comment format=mdwn
+ username="https://openid-provider.appspot.com/iakornfeld"
+ nickname="iakornfeld"
+ avatar="http://cdn.libravatar.org/avatar/c0369f5727cad81d1ecf6c2e657b42a1b756284aad0229351f9027a2cfcb2037"
+ subject="Yes"
+ date="2018-08-07T14:07:56Z"
+ content="""
+git-annex does this when checking remotes for git-annex support.
+"""]]
diff --git a/doc/bugs/git-annex-shell_configlist_doesn__39__t_work_over_SSH.mdwn b/doc/bugs/git-annex-shell_configlist_doesn__39__t_work_over_SSH.mdwn
new file mode 100644
index 0000000000..cae97ba438
--- /dev/null
+++ b/doc/bugs/git-annex-shell_configlist_doesn__39__t_work_over_SSH.mdwn
@@ -0,0 +1,34 @@
+### Please describe the problem.
+git-annex-shell -c configlist /home/annex/test2.git, when ran via sudo (login shell or otherwise), works fine. When executed equivalently over SSH, this error is given:
+[[!format sh """
+git-annex-shell: failed to read git config of git repository in unknown on /home/annex/test2.git; perhaps this repository is not set up correctly or has moved
+"""]]
+
+(note: I am using a workaround necessary due to another bug with git-annex-shell, I will also post that bug)
+
+### What steps will reproduce the problem?
+See the transcript.
+
+### What version of git-annex are you using? On what operating system?
+6.20180529 on NixOS.
+
+### Please provide any additional information below.
+
+[[!format sh """
+# If you can, paste a complete transcript of the problem occurring here.
+# If the problem is with the git-annex assistant, paste in .git/annex/daemon.log
+
+[leo60228@nixos:~/test]$ ssh annex@localhost 'git-annex-shell -c configlist /home/annex/test2.git'
+git-annex-shell: failed to read git config of git repository in unknown on /home/annex/test2.git; perhaps this repository is not set up correctly or has moved
+
+[leo60228@nixos:~/test]$ sudo -i -u annex git-annex-shell -c configlist /home/annex/test2.git
+annex.uuid=425c1e27-97df-4c14-8bf3-d7473e78dbfd
+core.gcrypt-id=
+
+# End of transcript or log.
+"""]]
+
+### Have you had any luck using git-annex before? (Sometimes we get tired of reading bug reports all day and a lil' positive end note does wonders)
+Yes.
+
+[[done]]
diff --git a/doc/bugs/git-annex-shell_configlist_doesn__39__t_work_over_SSH/comment_1_5d80f357a7f6513ccd949e58da572637._comment b/doc/bugs/git-annex-shell_configlist_doesn__39__t_work_over_SSH/comment_1_5d80f357a7f6513ccd949e58da572637._comment
new file mode 100644
index 0000000000..f674929b8c
--- /dev/null
+++ b/doc/bugs/git-annex-shell_configlist_doesn__39__t_work_over_SSH/comment_1_5d80f357a7f6513ccd949e58da572637._comment
@@ -0,0 +1,9 @@
+[[!comment format=mdwn
+ username="https://openid-provider.appspot.com/iakornfeld"
+ nickname="iakornfeld"
+ avatar="http://cdn.libravatar.org/avatar/c0369f5727cad81d1ecf6c2e657b42a1b756284aad0229351f9027a2cfcb2037"
+ subject="Please close"
+ date="2018-07-22T19:07:16Z"
+ content="""
+This was my fault.
+"""]]
diff --git a/doc/bugs/git-annex-shell_configlist_doesn__39__t_work_over_SSH/comment_2_6d1fa3ebc5bf20ebc3422a660f2c9632._comment b/doc/bugs/git-annex-shell_configlist_doesn__39__t_work_over_SSH/comment_2_6d1fa3ebc5bf20ebc3422a660f2c9632._comment
new file mode 100644
index 0000000000..e4cfa8e130
--- /dev/null
+++ b/doc/bugs/git-annex-shell_configlist_doesn__39__t_work_over_SSH/comment_2_6d1fa3ebc5bf20ebc3422a660f2c9632._comment
@@ -0,0 +1,10 @@
+[[!comment format=mdwn
+ username="andrew"
+ avatar="http://cdn.libravatar.org/avatar/acc0ece1eedf07dd9631e7d7d343c435"
+ subject="comment 2"
+ date="2018-07-23T22:37:39Z"
+ content="""
+OK. You can add [[done]] to the end of the original bug description to close it for future reference. I'll do that now.
+
+Andrew
+"""]]
diff --git a/doc/bugs/git-annex-sync_sometimes_fails_in_submodule_in_V6_adjusted_branch.mdwn b/doc/bugs/git-annex-sync_sometimes_fails_in_submodule_in_V6_adjusted_branch.mdwn
new file mode 100644
index 0000000000..8a188bb3a9
--- /dev/null
+++ b/doc/bugs/git-annex-sync_sometimes_fails_in_submodule_in_V6_adjusted_branch.mdwn
@@ -0,0 +1,51 @@
+### Please describe the problem.
+I have a V6 annex in adjusted branch with a submodule, which also is a V6 annex in adjusted branch.
+There staged changes in the superproject, while the submodule is clean. Calling git-annex-sync in the superproject now leads to:
+
+[[!format sh """
+% git annex sync
+commit  (recording state in git...)
+
+[adjusted/master(unlocked) 3c8d350] git-annex in me@mycomputer:/tmp/datalad_temp_test_AnnexRepo_statusGmqM5E
+ 4 files changed, 4 insertions(+), 2 deletions(-)
+ create mode 100644 fifth
+ create mode 100644 sub/third
+ok
+fatal: entry 'submod' object type (blob) doesn't match mode type (commit)
+
+git-annex: user error (git ["--git-dir=.git","--work-tree=.","--literal-pathspecs","mktree","--batch","-z"] exited 128)
+failed
+git-annex: sync: 1 failed
+"""]]
+
+### What steps will reproduce the problem?
+
+Sadly I wasn't able to reproduce it outside the datalad test it is happening in.
+I will provide the steps to achieve that situation, whenever I figured them out.
+But may be just the error message is sufficient to address the problem ...
+
+### What version of git-annex are you using? On what operating system?
+[[!format sh """
+% git annex version
+git-annex version: 6.20170307+gitg24ade8a25-1~ndall+1
+build flags: Assistant Webapp Pairing Testsuite S3(multipartupload)(storageclasses) WebDAV Inotify DBus DesktopNotify ConcurrentOutput TorrentParser MagicMime Feeds Quvi
+key/value backends: SHA256E SHA256 SHA512E SHA512 SHA224E SHA224 SHA384E SHA384 SHA3_256E SHA3_256 SHA3_512E SHA3_512 SHA3_224E SHA3_224 SHA3_384E SHA3_384 SKEIN256E SKEIN256 SKEIN512E SKEIN512 SHA1E SHA1 MD5E MD5 WORM URL
+remote types: git gcrypt p2p S3 bup directory rsync web bittorrent webdav tahoe glacier ddar hook external
+local repository version: 6
+supported repository versions: 3 5 6
+upgrade supported from repository 
+"""]]
+### Please provide any additional information below.
+
+[[!format sh """
+# If you can, paste a complete transcript of the problem occurring here.
+# If the problem is with the git-annex assistant, paste in .git/annex/daemon.log
+
+
+# End of transcript or log.
+"""]]
+
+### Have you had any luck using git-annex before? (Sometimes we get tired of reading bug reports all day and a lil' positive end note does wonders)
+
+
+[[ben]]
diff --git a/doc/bugs/git-annex_6.20161031_fails_with_filenames_containing_newlines.mdwn b/doc/bugs/git-annex_6.20161031_fails_with_filenames_containing_newlines.mdwn
new file mode 100644
index 0000000000..03ae68fa69
--- /dev/null
+++ b/doc/bugs/git-annex_6.20161031_fails_with_filenames_containing_newlines.mdwn
@@ -0,0 +1,46 @@
+### Please describe the problem.
+
+I tried to add a bunch of files and one of the files contained a newline in its name.
+When doing 'git annex add .' it failed with an error that contained only the part of the filename before the newline.
+
+### What steps will reproduce the problem?
+
+Create a file with a newline in its name:
+
+    ~/tmp $ touch 'file with newline'$'\n''line 2'
+    ~/tmp $ mkdir annex
+    ~/tmp $ cd annex/
+    ~/tmp/annex $ git init
+    ~/tmp/annex $ git annex init
+    init  ok
+    (recording state in git...)
+    ~/tmp/annex $ cp ../file* .
+    ~/tmp/annex $ git annex add .
+    add file with newline
+    line 2 git-annex: unknown response from git cat-file ("HEAD:./file with newline missing",Ref "HEAD:./file with newline\nline 2")
+    ~/tmp/annex $
+
+### What version of git-annex are you using? On what operating system?
+
+6.20161031 on Gentoo Linux
+
+
+### Please provide any additional information below.
+
+[[!format sh """
+# If you can, paste a complete transcript of the problem occurring here.
+
+please see above
+
+# If the problem is with the git-annex assistant, paste in .git/annex/daemon.log
+
+
+# End of transcript or log.
+"""]]
+
+### Have you had any luck using git-annex before? (Sometimes we get tired of reading bug reports all day and a lil' positive end note does wonders)
+
+I hope to have very soon ;-)
+
+> This is a duplicate of [[this bug report|git_annex_import_fails_on_filenames_with_newlines_in_them]].
+> Closing as dup. [[done]] --[[Joey]]
diff --git a/doc/bugs/git-annex__58_____60__socket__58___16__62____58___hPutBuf__58___resource_vanished___40__Broken_pipe__41__.mdwn b/doc/bugs/git-annex__58_____60__socket__58___16__62____58___hPutBuf__58___resource_vanished___40__Broken_pipe__41__.mdwn
new file mode 100644
index 0000000000..6374844a1d
--- /dev/null
+++ b/doc/bugs/git-annex__58_____60__socket__58___16__62____58___hPutBuf__58___resource_vanished___40__Broken_pipe__41__.mdwn
@@ -0,0 +1,18 @@
+### Please describe the problem.
+git-annex will randomly crash.
+
+### What steps will reproduce the problem?
+Unknown. Keeping git-annex running for an extended period, failing to sync properly on XMPP(not sure if that is relevant, but given this haven't been found before it might be)
+
+### What version of git-annex are you using? On what operating system?
+git-annex version: 4.20130521-g20710d4 (And multiple prior versions)
+
+### Please provide any additional information below.
+
+.git/annex/daemon.log upload: http://paste.ubuntu.com/5694813/
+
+I could find no debug.log?
+
+> [[!taglink moreinfo]] until it's reproduced with a current version.. --[[Joey]] 
+
+>> Note that this seems like a similar untrappable crash as [[git_annex_daemon_crashes_when_authenticating_with_jabber.de]]. --[[Joey]] 
diff --git a/doc/bugs/git-annex__58_____60__socket__58___16__62____58___hPutBuf__58___resource_vanished___40__Broken_pipe__41__/comment_1_e962317a939bf76097ae1a3b53b146e6._comment b/doc/bugs/git-annex__58_____60__socket__58___16__62____58___hPutBuf__58___resource_vanished___40__Broken_pipe__41__/comment_1_e962317a939bf76097ae1a3b53b146e6._comment
new file mode 100644
index 0000000000..3300f17b96
--- /dev/null
+++ b/doc/bugs/git-annex__58_____60__socket__58___16__62____58___hPutBuf__58___resource_vanished___40__Broken_pipe__41__/comment_1_e962317a939bf76097ae1a3b53b146e6._comment
@@ -0,0 +1,14 @@
+[[!comment format=mdwn
+ username="http://joeyh.name/"
+ nickname="joey"
+ subject="comment 1"
+ date="2013-05-24T15:50:45Z"
+ content="""
+I have seen this once on a similar system (family computer; XMPP being used). Unfortunatly it could be coming from anywhere -- and it's not at all clear how a crash in one thread could take it all down, since there are global top-level per-thread exception handlers that should run and log which thread crashed -- and normally seem to do this quite well.
+
+I may need to make a management process that ensures the assistant stays alive.
+
+I have also seen this happen when a computer is shutting down. But presumably in that case it's not really a bug.
+
+One thing you might try is see what is using socket 16 when it's running, assuming the socket will be the same. (Also, if you've had repeated crashes, it would be good to know if it's 16 each time..). You could do this by looking at `/proc/$pid/fd/16` Also, check the old logs, `.git/annex/daemon.log.*`
+"""]]
diff --git a/doc/bugs/git-annex__58_____60__socket__58___16__62____58___hPutBuf__58___resource_vanished___40__Broken_pipe__41__/comment_2_b32472b4c9b61e7a33dca802ecafb05b._comment b/doc/bugs/git-annex__58_____60__socket__58___16__62____58___hPutBuf__58___resource_vanished___40__Broken_pipe__41__/comment_2_b32472b4c9b61e7a33dca802ecafb05b._comment
new file mode 100644
index 0000000000..b575ce977a
--- /dev/null
+++ b/doc/bugs/git-annex__58_____60__socket__58___16__62____58___hPutBuf__58___resource_vanished___40__Broken_pipe__41__/comment_2_b32472b4c9b61e7a33dca802ecafb05b._comment
@@ -0,0 +1,11 @@
+[[!comment format=mdwn
+ username="https://www.google.com/accounts/o8/id?id=AItOawmLB39PC89rfGaA8SwrsnB6tbumezj-aC0"
+ nickname="Tobias"
+ subject="It did it again."
+ date="2013-05-25T09:31:23Z"
+ content="""
+And this time it was socket 27.
+
+Sadly happened during the night and I didn't monitor the socket when it happened.
+
+"""]]
diff --git a/doc/bugs/git-annex__58_____60__socket__58___16__62____58___hPutBuf__58___resource_vanished___40__Broken_pipe__41__/comment_3_fcfea3216831df9afbd855fbd842c27e._comment b/doc/bugs/git-annex__58_____60__socket__58___16__62____58___hPutBuf__58___resource_vanished___40__Broken_pipe__41__/comment_3_fcfea3216831df9afbd855fbd842c27e._comment
new file mode 100644
index 0000000000..bccfae0d94
--- /dev/null
+++ b/doc/bugs/git-annex__58_____60__socket__58___16__62____58___hPutBuf__58___resource_vanished___40__Broken_pipe__41__/comment_3_fcfea3216831df9afbd855fbd842c27e._comment
@@ -0,0 +1,20 @@
+[[!comment format=mdwn
+ username="http://joeyh.name/"
+ nickname="joey"
+ subject="comment 3"
+ date="2013-05-27T05:03:05Z"
+ content="""
+I got a similar crash:
+
+
    +fd:19: hGetLine: end of file
+fd:18: hFlush: resource vanished (Broken pipe)
+fd:18: hFlush: resource vanished (Broken pipe)
+
    
+
+I was able to determine that fd 18/19 is reliably used for one of the git cat-file processes on this system. It's quite likely that fd 16/17 would be similar. fd 27 less likely. (But could easily be some other less long running git command.)
+
+This would be consistent with git cat-file crashing as it's trying to write to it and read from it. 
+
+This took down the transferscanner thread, but the assistant continued running. 
+"""]]
diff --git a/doc/bugs/git-annex__58_____60__socket__58___16__62____58___hPutBuf__58___resource_vanished___40__Broken_pipe__41__/comment_4_30d0b40efa59eeecb8a4be6d1baa1520._comment b/doc/bugs/git-annex__58_____60__socket__58___16__62____58___hPutBuf__58___resource_vanished___40__Broken_pipe__41__/comment_4_30d0b40efa59eeecb8a4be6d1baa1520._comment
new file mode 100644
index 0000000000..aa3a2a3075
--- /dev/null
+++ b/doc/bugs/git-annex__58_____60__socket__58___16__62____58___hPutBuf__58___resource_vanished___40__Broken_pipe__41__/comment_4_30d0b40efa59eeecb8a4be6d1baa1520._comment
@@ -0,0 +1,10 @@
+[[!comment format=mdwn
+ username="http://joeyh.name/"
+ nickname="joey"
+ subject="comment 4"
+ date="2013-05-27T15:31:53Z"
+ content="""
+I tried, as an experiment, killing on of the `git cat-file` child processes of the assistant. As hypothesized, that led to the same thing I saw logged before.
+
+So, why might git commands be dying, and which commands? It would be pretty easy for git-annex to detect git cat-file dying, and restart it. Other commands would be more difficult. Still this might be a git bug which would best be fixed there. It would be good to get a core dump from git.
+"""]]
diff --git a/doc/bugs/git-annex__58_____60__socket__58___16__62____58___hPutBuf__58___resource_vanished___40__Broken_pipe__41__/comment_5_4af107f3184bc2abd2c9693167018628._comment b/doc/bugs/git-annex__58_____60__socket__58___16__62____58___hPutBuf__58___resource_vanished___40__Broken_pipe__41__/comment_5_4af107f3184bc2abd2c9693167018628._comment
new file mode 100644
index 0000000000..2288b1d846
--- /dev/null
+++ b/doc/bugs/git-annex__58_____60__socket__58___16__62____58___hPutBuf__58___resource_vanished___40__Broken_pipe__41__/comment_5_4af107f3184bc2abd2c9693167018628._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="http://joeyh.name/"
+ nickname="joey"
+ subject="comment 5"
+ date="2013-05-31T18:17:59Z"
+ content="""
+It will now detect if any of the long-running git processes crash, and automatically restart them. I don't know if this addresses the crash that was originally reported.
+"""]]
diff --git a/doc/bugs/git-annex__58_____60__socket__58___16__62____58___hPutBuf__58___resource_vanished___40__Broken_pipe__41__/comment_6_f96027f1e3c405809fae42ce8533c6d1._comment b/doc/bugs/git-annex__58_____60__socket__58___16__62____58___hPutBuf__58___resource_vanished___40__Broken_pipe__41__/comment_6_f96027f1e3c405809fae42ce8533c6d1._comment
new file mode 100644
index 0000000000..447e9d80eb
--- /dev/null
+++ b/doc/bugs/git-annex__58_____60__socket__58___16__62____58___hPutBuf__58___resource_vanished___40__Broken_pipe__41__/comment_6_f96027f1e3c405809fae42ce8533c6d1._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="http://joeyh.name/"
+ nickname="joey"
+ subject="comment 6"
+ date="2013-06-14T18:15:27Z"
+ content="""
+Tobias, have you seen this at all since the last release.
+"""]]
diff --git a/doc/bugs/git-annex__58_____60__socket__58___16__62____58___hPutBuf__58___resource_vanished___40__Broken_pipe__41__/comment_7_b6fe89deb468a7e4f63f7faab147e3fb._comment b/doc/bugs/git-annex__58_____60__socket__58___16__62____58___hPutBuf__58___resource_vanished___40__Broken_pipe__41__/comment_7_b6fe89deb468a7e4f63f7faab147e3fb._comment
new file mode 100644
index 0000000000..5d567ae8ee
--- /dev/null
+++ b/doc/bugs/git-annex__58_____60__socket__58___16__62____58___hPutBuf__58___resource_vanished___40__Broken_pipe__41__/comment_7_b6fe89deb468a7e4f63f7faab147e3fb._comment
@@ -0,0 +1,12 @@
+[[!comment format=mdwn
+ username="https://www.google.com/accounts/o8/id?id=AItOawmkBwMWvNKZZCge_YqobCSILPMeK6xbFw8"
+ nickname="develop"
+ subject="comment 7"
+ date="2013-06-14T21:26:25Z"
+ content="""
+I'm sure i can still provoke it on some of my machines. But this weekend is completely blocked for me.
+
+I'll update everything to latest and give you an updated log. But will probably not happen till the middle of next week.
+
+
+"""]]
diff --git a/doc/bugs/git-annex__58_____60__socket__58___16__62____58___hPutBuf__58___resource_vanished___40__Broken_pipe__41__/comment_8_ebec5d9266604f03959dc16d933ff4a4._comment b/doc/bugs/git-annex__58_____60__socket__58___16__62____58___hPutBuf__58___resource_vanished___40__Broken_pipe__41__/comment_8_ebec5d9266604f03959dc16d933ff4a4._comment
new file mode 100644
index 0000000000..b44164a896
--- /dev/null
+++ b/doc/bugs/git-annex__58_____60__socket__58___16__62____58___hPutBuf__58___resource_vanished___40__Broken_pipe__41__/comment_8_ebec5d9266604f03959dc16d933ff4a4._comment
@@ -0,0 +1,13 @@
+[[!comment format=mdwn
+ username="https://www.google.com/accounts/o8/id?id=AItOawmkBwMWvNKZZCge_YqobCSILPMeK6xbFw8"
+ nickname="develop"
+ subject="comment 8"
+ date="2013-06-29T09:34:02Z"
+ content="""
+I haven't been able to replicate this on two of my computers with the latest git-annex(as of this message).
+
+It seemed to happen more on my Mac OS X Lion though. And nightlies haven't been build for some time.
+
+So i'm waiting for a updated package for OS X. If you would rather clean up feel free to close this, I can just open it again if i hit it again.
+
+"""]]
diff --git a/doc/bugs/git-annex__58___content_is_locked__while_trying_to_move_under_NFS_and_pidlock.mdwn b/doc/bugs/git-annex__58___content_is_locked__while_trying_to_move_under_NFS_and_pidlock.mdwn
new file mode 100644
index 0000000000..d3d75e8c8d
--- /dev/null
+++ b/doc/bugs/git-annex__58___content_is_locked__while_trying_to_move_under_NFS_and_pidlock.mdwn
@@ -0,0 +1,31 @@
+### Please describe the problem.
+
+Ideally annex should detect all "paranormal" cases such as running on NFS mounted partition, but according to [https://git-annex.branchable.com/bugs/huge_multiple_copies_of___39__.nfs__42____39___and___39__.panfs__42____39___being_created/](https://git-annex.branchable.com/bugs/huge_multiple_copies_of___39__.nfs__42____39___and___39__.panfs__42____39___being_created/).  Happily ignorant we were running annex (5.20151116-g76139a9) on NFS mounted partition until we filled up 2TB of allocated to us space with .nfs* files.  Well -- apparently according to above we should have tried pidlock... trying now but doesn't work :-/
+
+[[!format sh """
+*$> git clone smaug:/tmp/123 123-clone && cd 123-clone && git config annex.pidlock true && echo 124 > 124.dat && git annex add 124.dat && git commit -m 'added 124' && git annex move --to=origin 124.dat
+Initialized empty Git repository in /home/yhalchen/123-clone/.git/
+remote: Counting objects: 22, done.
+remote: Compressing objects: 100% (16/16), done.
+remote: Total 22 (delta 3), reused 0 (delta 0)
+Receiving objects: 100% (22/22), done.
+Resolving deltas: 100% (3/3), done.
+total 1
+1 123.dat@  1 README.txt
+(merging origin/git-annex into git-annex...)
+(recording state in git...)
+add 124.dat ok
+(recording state in git...)
+[master 0f1092a] added 124
+ 1 files changed, 1 insertions(+), 0 deletions(-)
+ create mode 120000 124.dat
+move 124.dat (checking origin...) git-annex: content is locked
+
+$> echo $?
+1
+
+"""]]
+
+BTW running move in our old now somewhat screwed up annex, results in a differently expressed error: [http://www.onerussian.com/tmp/2016-02-29.png](http://www.onerussian.com/tmp/2016-02-29.png)
+
+[[!meta author=yoh]]
diff --git a/doc/bugs/git-annex__58___content_is_locked__while_trying_to_move_under_NFS_and_pidlock/comment_10_d44de6a250694b25ce9c3169d62db8d1._comment b/doc/bugs/git-annex__58___content_is_locked__while_trying_to_move_under_NFS_and_pidlock/comment_10_d44de6a250694b25ce9c3169d62db8d1._comment
new file mode 100644
index 0000000000..f2ba4335c7
--- /dev/null
+++ b/doc/bugs/git-annex__58___content_is_locked__while_trying_to_move_under_NFS_and_pidlock/comment_10_d44de6a250694b25ce9c3169d62db8d1._comment
@@ -0,0 +1,20 @@
+[[!comment format=mdwn
+ username="joey"
+ subject="""comment 10"""
+ date="2016-03-01T20:52:38Z"
+ content="""
+	2456732 openat(AT_FDCWD, ".git/annex/ssh/", O_RDONLY|O_NONBLOCK|O_DIRECTORY|O_CLOEXEC) = -1 ENOENT (No such file or directory)
+	2456732 mkdir(".git/annex/ssh", 0777)   = 0
+	2456732 open(".git/annex/ssh/smaug.lock", O_RDONLY|O_CREAT, 0666) = 11
+	2456732 fcntl(11, F_GETFD)              = 0
+	2456732 fcntl(11, F_SETFD, FD_CLOEXEC)  = 0
+	2456732 close(11)                       = 0
+
+Backs up what I thought git-annex should be doing; it's not fcntl locking that file.
+
+Ah, I'll bet it's not git-annex at all this time.
+It runs ssh with -S .git/annex/ssh/smaug, and ssh probably
+does its own locking around setting up that control socket.
+
+If so, disabling annex.sshcaching will avoid the problem.
+"""]]
diff --git a/doc/bugs/git-annex__58___content_is_locked__while_trying_to_move_under_NFS_and_pidlock/comment_11_56ae0f15bbdea2331df3b261b74d0b0b._comment b/doc/bugs/git-annex__58___content_is_locked__while_trying_to_move_under_NFS_and_pidlock/comment_11_56ae0f15bbdea2331df3b261b74d0b0b._comment
new file mode 100644
index 0000000000..2ee2e6aa86
--- /dev/null
+++ b/doc/bugs/git-annex__58___content_is_locked__while_trying_to_move_under_NFS_and_pidlock/comment_11_56ae0f15bbdea2331df3b261b74d0b0b._comment
@@ -0,0 +1,7 @@
+[[!comment format=mdwn
+ username="https://me.yahoo.com/a/EbvxpTI_xP9Aod7Mg4cwGhgjrCrdM5s-#7c0f4"
+ subject="comment 11"
+ date="2016-03-01T22:40:44Z"
+ content="""
+would then may be annex not to use sshcaching if operating under pidlock, unless some nfs specific flag is used to tease it apart
+"""]]
diff --git a/doc/bugs/git-annex__58___content_is_locked__while_trying_to_move_under_NFS_and_pidlock/comment_12_d499a2f44e8ee2f39ee959f4d6373570._comment b/doc/bugs/git-annex__58___content_is_locked__while_trying_to_move_under_NFS_and_pidlock/comment_12_d499a2f44e8ee2f39ee959f4d6373570._comment
new file mode 100644
index 0000000000..5ab7c9e118
--- /dev/null
+++ b/doc/bugs/git-annex__58___content_is_locked__while_trying_to_move_under_NFS_and_pidlock/comment_12_d499a2f44e8ee2f39ee959f4d6373570._comment
@@ -0,0 +1,13 @@
+[[!comment format=mdwn
+ username="https://me.yahoo.com/a/EbvxpTI_xP9Aod7Mg4cwGhgjrCrdM5s-#7c0f4"
+ subject="comment 12"
+ date="2016-03-10T04:36:12Z"
+ content="""
+so -- shouldn't annex at least upon init sense if repo is under nfs?  if to be done platform independent way then it could do smth like
+
+[[!format sh \"\"\"
+python -c 'from glob import glob; import os; f=open(\".tmp-test\", \"w\"); os.unlink(\".tmp-test\"); assert(not glob(\".nfs*\"))'
+\"\"\"]]
+
+somewhere under .git/annex/tmp .. so if .nfs* file gets generated -- under nfs.  Seems to work for me in limited set of tests -- assertion fails all the time under NFS ;)
+"""]]
diff --git a/doc/bugs/git-annex__58___content_is_locked__while_trying_to_move_under_NFS_and_pidlock/comment_13_05687dca2462eb79872879bd7695d665._comment b/doc/bugs/git-annex__58___content_is_locked__while_trying_to_move_under_NFS_and_pidlock/comment_13_05687dca2462eb79872879bd7695d665._comment
new file mode 100644
index 0000000000..64df7bc578
--- /dev/null
+++ b/doc/bugs/git-annex__58___content_is_locked__while_trying_to_move_under_NFS_and_pidlock/comment_13_05687dca2462eb79872879bd7695d665._comment
@@ -0,0 +1,51 @@
+[[!comment format=mdwn
+ username="joey"
+ subject="""comment 13"""
+ date="2016-10-26T16:34:38Z"
+ content="""
+Looking at the strace finally...
+
+Here's git-annex creating the dummy posix lock file, soon after
+the ssh socket is removed. (I think that's ssh removing the socket before,
+but the jumble of strace output is hard to follow here.)
+
+	2456732 unlink(".git/annex/ssh/smaug" 
+	...
+	2456732 open(".git/annex/ssh/.nfs00000000099d85d000000002.lock", O_RDONLY|O_CREAT, 0666 
+
+And later ssh is told to stop using a related socket:
+
+	2456815 execve("/mnt/btrfs/scrap/datalad/test_nfs/git-annex.linux/bin/ssh", ["ssh", "-O", "stop", "-S", ".nfs00000000099d85d000000002", "-o", "ControlMaster=auto", "-o", "ControlPersist=yes", "localhost"], [/* 98 vars */] 
+
+Which it seems does not exist by then:
+
+	2456815 connect(3, {sa_family=AF_LOCAL, sun_path=".nfs00000000099d85d000000002"}, 31) = -1 ENOENT (No such file or directory)
+	2456732 unlink(".git/annex/ssh/.nfs00000000099d85d000000002") = -1 ENOENT (No such file or directory)
+
+Seems like this would have to be `sshCleanup` running, and `enumSocketFiles`
+seeing ".nfs00000000099d85d000000002" existing at that point due
+probably to the ssh/smaug socket being in the process of being
+removed. 
+
+So, it assumes it's a socket file and tries to clean it up, creating
+the dummy posix lock file. And posix lock files don't get deleted
+(it's impossible to do so in a race-free way), so this results in more
+and more `.nfs*.lock` files.
+
+Need to find a way to prevent `enumSocketFiles` from listing these
+files. Seems that ".nfs00000000099d85d000000002" is probably the deleted
+form of the "smaug" socket, so it really is a socket file, so checking
+if a file is a socket won't help. Of course it could
+ explicitly filter out ".nfs" prefixed files. That could be a case of
+kicking the can down the road, since NFS could always choose to use
+another name for these files.
+
+Hmm.. Before a ssh socket file gets created, git-annex always locks the
+associated lock file. And as noted, the posix lock file is never
+removed. (On Windows the lock file is also created and never deleted.) 
+So, if $file does not have an associated $file.lock 
+then $file must not be a ssh socket, and `enumSocketFiles` can skip it.
+
+I've added that check. Unable to test it because the NFS mount has
+been lost in the intervening time. @yoh can you test this fixed it?
+"""]]
diff --git a/doc/bugs/git-annex__58___content_is_locked__while_trying_to_move_under_NFS_and_pidlock/comment_14_cb1df663b03f3ce9cb27e311e5a70198._comment b/doc/bugs/git-annex__58___content_is_locked__while_trying_to_move_under_NFS_and_pidlock/comment_14_cb1df663b03f3ce9cb27e311e5a70198._comment
new file mode 100644
index 0000000000..42dad95c4b
--- /dev/null
+++ b/doc/bugs/git-annex__58___content_is_locked__while_trying_to_move_under_NFS_and_pidlock/comment_14_cb1df663b03f3ce9cb27e311e5a70198._comment
@@ -0,0 +1,15 @@
+[[!comment format=mdwn
+ username="joey"
+ subject="""comment 14"""
+ date="2016-10-26T17:19:39Z"
+ content="""
+Re detecting NFS, I'm not sure that ".nfs" files are always used for
+deleted files by all NFS configurations. And, there are probably NFS
+configurations that do properly support Posix locks, and others that
+don't. So connecting a check for ".nfs" files with turning on
+annex.pidlock seems problimatic.
+
+If we had a good way to detect systems that don't support Posix locks,
+annex.pidlock could be auto-enabled. But for some reason embedding a large
+portion of a Posix test suite into git-annex does not fill me with joy.
+"""]]
diff --git a/doc/bugs/git-annex__58___content_is_locked__while_trying_to_move_under_NFS_and_pidlock/comment_1_a98a54c04fa4e81f35fe958e746d61cb._comment b/doc/bugs/git-annex__58___content_is_locked__while_trying_to_move_under_NFS_and_pidlock/comment_1_a98a54c04fa4e81f35fe958e746d61cb._comment
new file mode 100644
index 0000000000..21304e2e8b
--- /dev/null
+++ b/doc/bugs/git-annex__58___content_is_locked__while_trying_to_move_under_NFS_and_pidlock/comment_1_a98a54c04fa4e81f35fe958e746d61cb._comment
@@ -0,0 +1,7 @@
+[[!comment format=mdwn
+ username="joey"
+ subject="""comment 1"""
+ date="2016-03-01T14:36:25Z"
+ content="""
+FYI, I think you could remove the .nfs files to free up space.
+"""]]
diff --git a/doc/bugs/git-annex__58___content_is_locked__while_trying_to_move_under_NFS_and_pidlock/comment_2_18169e7bbd2caba5ee4bb0286961ac95._comment b/doc/bugs/git-annex__58___content_is_locked__while_trying_to_move_under_NFS_and_pidlock/comment_2_18169e7bbd2caba5ee4bb0286961ac95._comment
new file mode 100644
index 0000000000..42b3265bd8
--- /dev/null
+++ b/doc/bugs/git-annex__58___content_is_locked__while_trying_to_move_under_NFS_and_pidlock/comment_2_18169e7bbd2caba5ee4bb0286961ac95._comment
@@ -0,0 +1,10 @@
+[[!comment format=mdwn
+ username="joey"
+ subject="""comment 2"""
+ date="2016-03-01T15:35:48Z"
+ content="""
+Oddly, I cannot reproduce this, although I can reproduce the behavior in
+a
+
+(smaug:/tmp/123 has permissions that do not let me access it.)
+"""]]
diff --git a/doc/bugs/git-annex__58___content_is_locked__while_trying_to_move_under_NFS_and_pidlock/comment_3_e3b623ff6714a9fe5fa0d332c72fe32f._comment b/doc/bugs/git-annex__58___content_is_locked__while_trying_to_move_under_NFS_and_pidlock/comment_3_e3b623ff6714a9fe5fa0d332c72fe32f._comment
new file mode 100644
index 0000000000..51f71c7ac8
--- /dev/null
+++ b/doc/bugs/git-annex__58___content_is_locked__while_trying_to_move_under_NFS_and_pidlock/comment_3_e3b623ff6714a9fe5fa0d332c72fe32f._comment
@@ -0,0 +1,9 @@
+[[!comment format=mdwn
+ username="joey"
+ subject="""comment 3"""
+ date="2016-03-01T16:52:16Z"
+ content="""
+I've fixed the STM transaction bug. Need either more info to reproduce this
+bug, or you could test and see if it still occurs when git-annex is
+upgraded to ad888a6b760e8f9d31f8d99c51912bcdaa7fb0c1
+"""]]
diff --git a/doc/bugs/git-annex__58___content_is_locked__while_trying_to_move_under_NFS_and_pidlock/comment_4_58eebd8cfd664b32ef6fd0ddc34c5e86._comment b/doc/bugs/git-annex__58___content_is_locked__while_trying_to_move_under_NFS_and_pidlock/comment_4_58eebd8cfd664b32ef6fd0ddc34c5e86._comment
new file mode 100644
index 0000000000..7a562f7efc
--- /dev/null
+++ b/doc/bugs/git-annex__58___content_is_locked__while_trying_to_move_under_NFS_and_pidlock/comment_4_58eebd8cfd664b32ef6fd0ddc34c5e86._comment
@@ -0,0 +1,9 @@
+[[!comment format=mdwn
+ username="https://me.yahoo.com/a/EbvxpTI_xP9Aod7Mg4cwGhgjrCrdM5s-#7c0f4"
+ subject="more info"
+ date="2016-03-01T17:31:29Z"
+ content="""
+If we could remove those .nfs* files, it would indeed be not that bad  but we can't
+
+smaug:/tmp/123 -- sorry about permissions but it is a regular annex nothing special, so the bug should show itself with other repos as well I think.  I gave you access to it now and also there is /tmp/123.tar.gz archive of it just in case.
+"""]]
diff --git a/doc/bugs/git-annex__58___content_is_locked__while_trying_to_move_under_NFS_and_pidlock/comment_5_e5e24428ac02b78d38cd4f197ae3807b._comment b/doc/bugs/git-annex__58___content_is_locked__while_trying_to_move_under_NFS_and_pidlock/comment_5_e5e24428ac02b78d38cd4f197ae3807b._comment
new file mode 100644
index 0000000000..75a45bdad1
--- /dev/null
+++ b/doc/bugs/git-annex__58___content_is_locked__while_trying_to_move_under_NFS_and_pidlock/comment_5_e5e24428ac02b78d38cd4f197ae3807b._comment
@@ -0,0 +1,31 @@
+[[!comment format=mdwn
+ username="https://me.yahoo.com/a/EbvxpTI_xP9Aod7Mg4cwGhgjrCrdM5s-#7c0f4"
+ subject="recent snapshot seems has fixed it"
+ date="2016-03-01T18:52:27Z"
+ content="""
+[[!format sh \"\"\"
+$> git clone smaug:/tmp/123 123-clone && cd 123-clone && git config annex.pidlock true && echo 124 > 124.dat && git annex add 124.dat && git commit -m 'added 124' && git annex move --to=origin 124.dat
+Cloning into '123-clone'...
+remote: Counting objects: 22, done.
+remote: Compressing objects: 100% (16/16), done.
+remote: Total 22 (delta 3), reused 0 (delta 0)
+Receiving objects: 100% (22/22), done.
+Resolving deltas: 100% (3/3), done.
+Checking connectivity... done.
+total 1
+1 123.dat@  1 README.txt
+(merging origin/git-annex into git-annex...)
+(recording state in git...)
+add 124.dat ok
+(recording state in git...)
+[master 6eca577] added 124
+ 1 file changed, 1 insertion(+)
+ create mode 120000 124.dat
+move 124.dat (checking origin...) ok
+(recording state in git...)
+
+$> git annex version
+git-annex version: 6.20160301+gitg647fffd-1~ndall+1
+
+\"\"\"]]
+"""]]
diff --git a/doc/bugs/git-annex__58___content_is_locked__while_trying_to_move_under_NFS_and_pidlock/comment_6_01dc7a1ff67783ce672d72cefe7b4bb5._comment b/doc/bugs/git-annex__58___content_is_locked__while_trying_to_move_under_NFS_and_pidlock/comment_6_01dc7a1ff67783ce672d72cefe7b4bb5._comment
new file mode 100644
index 0000000000..20eca2f62f
--- /dev/null
+++ b/doc/bugs/git-annex__58___content_is_locked__while_trying_to_move_under_NFS_and_pidlock/comment_6_01dc7a1ff67783ce672d72cefe7b4bb5._comment
@@ -0,0 +1,7 @@
+[[!comment format=mdwn
+ username="https://me.yahoo.com/a/EbvxpTI_xP9Aod7Mg4cwGhgjrCrdM5s-#7c0f4"
+ subject="comment 6"
+ date="2016-03-01T18:54:15Z"
+ content="""
+but then I found   ./.git/annex/ssh/.nfs000000000000f41600003608.lock left behind (removable, luckily to me ;) )
+"""]]
diff --git a/doc/bugs/git-annex__58___content_is_locked__while_trying_to_move_under_NFS_and_pidlock/comment_7_458518805b8d6613930b38b9ccc3c1bc._comment b/doc/bugs/git-annex__58___content_is_locked__while_trying_to_move_under_NFS_and_pidlock/comment_7_458518805b8d6613930b38b9ccc3c1bc._comment
new file mode 100644
index 0000000000..756b642500
--- /dev/null
+++ b/doc/bugs/git-annex__58___content_is_locked__while_trying_to_move_under_NFS_and_pidlock/comment_7_458518805b8d6613930b38b9ccc3c1bc._comment
@@ -0,0 +1,7 @@
+[[!comment format=mdwn
+ username="https://me.yahoo.com/a/EbvxpTI_xP9Aod7Mg4cwGhgjrCrdM5s-#7c0f4"
+ subject="comment 7"
+ date="2016-03-01T18:58:20Z"
+ content="""
+and those are breeding with next subsequent --move
+"""]]
diff --git a/doc/bugs/git-annex__58___content_is_locked__while_trying_to_move_under_NFS_and_pidlock/comment_8_853bc273b19bd6d84ca8f5da6c3dfb56._comment b/doc/bugs/git-annex__58___content_is_locked__while_trying_to_move_under_NFS_and_pidlock/comment_8_853bc273b19bd6d84ca8f5da6c3dfb56._comment
new file mode 100644
index 0000000000..d24d78d513
--- /dev/null
+++ b/doc/bugs/git-annex__58___content_is_locked__while_trying_to_move_under_NFS_and_pidlock/comment_8_853bc273b19bd6d84ca8f5da6c3dfb56._comment
@@ -0,0 +1,20 @@
+[[!comment format=mdwn
+ username="joey"
+ subject="""comment 8"""
+ date="2016-03-01T20:17:35Z"
+ content="""
+That ssh lock file is created by this code:
+
+	-- The posix lock file is created even when using pid locks, in order to
+	-- avoid complicating any code that might expect to be able to see that
+	-- lock file. But, it's not locked.
+	dummyPosixLock :: Maybe FileMode -> LockFile -> IO ()
+	dummyPosixLock m f = closeFd =<< openLockFile ReadLock m f
+
+But, that does not ever actually take a lock on the file, so
+NFS should not make its .nfs thing in this case. Unless NFS does it when a
+FD is simply opened with close-on-exec set. 
+
+Can you get a strace of the creation of files under .git/annex/ssh/
+that result in these .nfs things?
+"""]]
diff --git a/doc/bugs/git-annex__58___content_is_locked__while_trying_to_move_under_NFS_and_pidlock/comment_9_86656a409ab25c7fa24de8ac3e68b254._comment b/doc/bugs/git-annex__58___content_is_locked__while_trying_to_move_under_NFS_and_pidlock/comment_9_86656a409ab25c7fa24de8ac3e68b254._comment
new file mode 100644
index 0000000000..b862174b15
--- /dev/null
+++ b/doc/bugs/git-annex__58___content_is_locked__while_trying_to_move_under_NFS_and_pidlock/comment_9_86656a409ab25c7fa24de8ac3e68b254._comment
@@ -0,0 +1,7 @@
+[[!comment format=mdwn
+ username="https://me.yahoo.com/a/EbvxpTI_xP9Aod7Mg4cwGhgjrCrdM5s-#7c0f4"
+ subject="comment 9"
+ date="2016-03-01T20:43:17Z"
+ content="""
+ok -- see on smaug /mnt/nfs/scrap/datalad/test_nfs/123-clone-move.strace  .  Now you can experiment there as well -- the entire /mnt/btrfs/scrap is mounted also via nfs (under /mnt/nfs/scrap)
+"""]]
diff --git a/doc/bugs/git-annex__58___fd__58__14__58___hGetLine__58___end_of_file.mdwn b/doc/bugs/git-annex__58___fd__58__14__58___hGetLine__58___end_of_file.mdwn
new file mode 100644
index 0000000000..0d773b52cf
--- /dev/null
+++ b/doc/bugs/git-annex__58___fd__58__14__58___hGetLine__58___end_of_file.mdwn
@@ -0,0 +1,51 @@
+[[!tag moreinfo]]
+
+### Please describe the problem.
+
+git-annex webapp won't run 
+
+### What steps will reproduce the problem?
+
+[[!format sh """
+arthur@machine:~/annex$ git-annex --debug webapp
+[2013-07-29 15:02:15 CEST] read: git ["--git-dir=/home/arthur/annex/.git","--work-tree=/home/arthur/annex","show-ref","git-annex"]
+[2013-07-29 15:02:15 CEST] read: git ["--git-dir=/home/arthur/annex/.git","--work-tree=/home/arthur/annex","show-ref","--hash","refs/heads/git-annex"]
+[2013-07-29 15:02:15 CEST] read: git ["--git-dir=/home/arthur/annex/.git","--work-tree=/home/arthur/annex","log","refs/heads/git-annex..a2b8f10ef258dff1a91e0354b2e2a58241631c9a","--oneline","-n1"]
+error: object file /home/arthur/annex/.git/objects/a2/b8f10ef258dff1a91e0354b2e2a58241631c9a is empty
+fatal: loose object a2b8f10ef258dff1a91e0354b2e2a58241631c9a (stored in /home/arthur/annex/.git/objects/a2/b8f10ef258dff1a91e0354b2e2a58241631c9a) is corrupt
+[2013-07-29 15:02:15 CEST] read: git ["--git-dir=/home/arthur/annex/.git","--work-tree=/home/arthur/annex","log","refs/heads/git-annex..8d4b8e04ccf0092d625f680b42e73d7bf15c6517","--oneline","-n1"]
+error: object file /home/arthur/annex/.git/objects/a2/b8f10ef258dff1a91e0354b2e2a58241631c9a is empty
+fatal: loose object a2b8f10ef258dff1a91e0354b2e2a58241631c9a (stored in /home/arthur/annex/.git/objects/a2/b8f10ef258dff1a91e0354b2e2a58241631c9a) is corrupt
+[2013-07-29 15:02:15 CEST] read: git ["--git-dir=/home/arthur/annex/.git","--work-tree=/home/arthur/annex","log","refs/heads/git-annex..6b2665208c11c9ecf969294bf45baac31894d8a7","--oneline","-n1"]
+error: object file /home/arthur/annex/.git/objects/a2/b8f10ef258dff1a91e0354b2e2a58241631c9a is empty
+fatal: loose object a2b8f10ef258dff1a91e0354b2e2a58241631c9a (stored in /home/arthur/annex/.git/objects/a2/b8f10ef258dff1a91e0354b2e2a58241631c9a) is corrupt
+[2013-07-29 15:02:15 CEST] read: git ["--git-dir=/home/arthur/annex/.git","--work-tree=/home/arthur/annex","log","refs/heads/git-annex..9d8429668f2148ea43760fb430e5950fbf42751e","--oneline","-n1"]
+error: object file /home/arthur/annex/.git/objects/a2/b8f10ef258dff1a91e0354b2e2a58241631c9a is empty
+fatal: loose object a2b8f10ef258dff1a91e0354b2e2a58241631c9a (stored in /home/arthur/annex/.git/objects/a2/b8f10ef258dff1a91e0354b2e2a58241631c9a) is corrupt
+[2013-07-29 15:02:15 CEST] chat: git ["--git-dir=/home/arthur/annex/.git","--work-tree=/home/arthur/annex","cat-file","--batch"]
+
+git-annex: fd:14: hGetLine: end of file
+failed
+[2013-07-29 15:02:15 CEST] read: ssh ["-O","stop","-S","/home/arthur/annex/.git/annex/ssh/arthur@git-annex-hostname-arthur_annex","-o","ControlMaster=auto","-o","ControlPersist=yes","arthur@git-annex-hostname-arthur_annex"]
+git-annex: webapp: 1 failed
+"""]]
+
+
+### What version of git-annex are you using? On what operating system?
+
+
+$ git-annex version
+git-annex version: 4.20130516.1
+build flags: Assistant Webapp Pairing Testsuite S3 WebDAV Inotify DBus XMPP
+local repository version: 3
+default repository version: 3
+supported repository versions: 3 4
+upgrade supported from repository versions: 0 1 2
+
+$ git --version
+git version 1.7.9.5
+
+
+### Please provide any additional information below.
+
+
diff --git a/doc/bugs/git-annex__58___fd__58__14__58___hGetLine__58___end_of_file/comment_1_36756f5d9d591cc52113c5cc0c1eae91._comment b/doc/bugs/git-annex__58___fd__58__14__58___hGetLine__58___end_of_file/comment_1_36756f5d9d591cc52113c5cc0c1eae91._comment
new file mode 100644
index 0000000000..b7fc68ffd5
--- /dev/null
+++ b/doc/bugs/git-annex__58___fd__58__14__58___hGetLine__58___end_of_file/comment_1_36756f5d9d591cc52113c5cc0c1eae91._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="http://joeyh.name/"
+ ip="4.154.0.21"
+ subject="comment 1"
+ date="2013-07-30T19:00:43Z"
+ content="""
+You have a somewhat out of date version there. Can you please try again with a newer version? There have been several fixes to bugs that could present like this.
+"""]]
diff --git a/doc/bugs/git-annex__58___unable_to_decommit_memory__58___Invalid_argument.mdwn b/doc/bugs/git-annex__58___unable_to_decommit_memory__58___Invalid_argument.mdwn
new file mode 100644
index 0000000000..baf7d4355e
--- /dev/null
+++ b/doc/bugs/git-annex__58___unable_to_decommit_memory__58___Invalid_argument.mdwn
@@ -0,0 +1,30 @@
+### Please describe the problem.
+
+Received the following error upon syncing a large repo.
+
+    git-annex: unable to decommit memory: Invalid argument
+
+### What steps will reproduce the problem?
+
+Not sure. I tried creating a small test repo, adding files, and syncing, but it did not produce the error.
+
+### What version of git-annex are you using? On what operating system?
+
+    git-annex version: 6.20161211+gitgc3ab3c6-1~ndall+1
+
+... via the neurodebian repo. On Xubuntu 14.04 64x
+
+    uname -r: 3.13.0-106-generic
+
+### Please provide any additional information below.
+
+My files appear to be okay, and otherwise the annex seems to be functioning normally.
+
+Nonetheless I'm hesitant to use it while this error occurs. Is safe to keep using my annex in spite of the error?
+
+### Have you had any luck using git-annex before? (Sometimes we get tired of reading bug reports all day and a lil' positive end note does wonders)
+
+I've been using git-annex for at least a year now. It's one of the greatest tools I've ever found, and is indispensable for some of my work. I can't recall any other problems I've had that weren't user-caused. So thank you!
+
+> According to yoh, this is fixed in the neurodebian build now. [[done]]
+> --[[Joey]]
diff --git a/doc/bugs/git-annex__58___unable_to_decommit_memory__58___Invalid_argument/comment_1_9b81586657226bf010145d1cd661885a._comment b/doc/bugs/git-annex__58___unable_to_decommit_memory__58___Invalid_argument/comment_1_9b81586657226bf010145d1cd661885a._comment
new file mode 100644
index 0000000000..c00a7465f2
--- /dev/null
+++ b/doc/bugs/git-annex__58___unable_to_decommit_memory__58___Invalid_argument/comment_1_9b81586657226bf010145d1cd661885a._comment
@@ -0,0 +1,11 @@
+[[!comment format=mdwn
+ username="joey"
+ subject="""comment 1"""
+ date="2016-12-30T22:26:10Z"
+ content="""
+This is a known bug, that is supposed to be fixed in 6.20161210.
+
+However, the fix involved upgrading the ghc compiler. I think that probably
+this has not been done in the neurodebian build yet. I will mention it to
+yoh again.
+"""]]
diff --git a/doc/bugs/git-annex__58___unable_to_decommit_memory__58___Invalid_argument/comment_2_792e40149bb2ae55206ebe9cea151831._comment b/doc/bugs/git-annex__58___unable_to_decommit_memory__58___Invalid_argument/comment_2_792e40149bb2ae55206ebe9cea151831._comment
new file mode 100644
index 0000000000..67d5734986
--- /dev/null
+++ b/doc/bugs/git-annex__58___unable_to_decommit_memory__58___Invalid_argument/comment_2_792e40149bb2ae55206ebe9cea151831._comment
@@ -0,0 +1,18 @@
+[[!comment format=mdwn
+ username="ghen1"
+ avatar="http://cdn.libravatar.org/avatar/efd0e92b6198291138f0cd7aedbf86b6"
+ subject="To Reproduce"
+ date="2016-12-31T14:19:07Z"
+ content="""
+I've successfully reproduced the error.
+
+Steps:
+
+1. Create 5k small files using a script. (I used one given here:  )
+2. Init git and git-annex
+3. Add files to git-annex. **The error occurs here** after/while (recording state in git...)
+
+If no error occurs, try increasing the number of files to 10k or more.
+
+I ran both *status* and *fsck* in both git and git-annex, with no errors, so AFAIK it hasn't corrupted anything.
+"""]]
diff --git a/doc/bugs/git-annex__58___unable_to_decommit_memory__58___Invalid_argument/comment_3_d5aead37bf76aaa194b3aea9aab0dd19._comment b/doc/bugs/git-annex__58___unable_to_decommit_memory__58___Invalid_argument/comment_3_d5aead37bf76aaa194b3aea9aab0dd19._comment
new file mode 100644
index 0000000000..28e3296324
--- /dev/null
+++ b/doc/bugs/git-annex__58___unable_to_decommit_memory__58___Invalid_argument/comment_3_d5aead37bf76aaa194b3aea9aab0dd19._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="https://me.yahoo.com/a/EbvxpTI_xP9Aod7Mg4cwGhgjrCrdM5s-#7c0f4"
+ avatar="http://cdn.libravatar.org/avatar/8122123b4c2bc77187e32d7e025f7d445d7a08de1ba532237876a31159ac01da"
+ subject="fresh git annex standalone  was uploaded to NeuroDebian"
+ date="2017-01-03T16:03:52Z"
+ content="""
+just uploaded 6.20170101+gitg93d69b1-1~ndall+1 which was built using ghc 8.0.1-17 on stretch.  Enjoy
+"""]]
diff --git a/doc/bugs/git-annex_adds_unicode_characters_at_end_of_checksum.mdwn b/doc/bugs/git-annex_adds_unicode_characters_at_end_of_checksum.mdwn
new file mode 100644
index 0000000000..ec8d57b652
--- /dev/null
+++ b/doc/bugs/git-annex_adds_unicode_characters_at_end_of_checksum.mdwn
@@ -0,0 +1,52 @@
+### Please describe the problem.
+Files with special unicode characters(in this case japanese) for some reason have the characters added to the file key.
+
+This is an issue because it causes errors when using glacier-cli when uploading copies to Glacier vault.
+
+[[!meta title="kanji in key extension cause glacier-cli upload error"]]
+
+### What steps will reproduce the problem?
+Here's how it looks for me:
+
+[[!format sh """
+% ls -l 12.\ Change\ The\ World\ \(feat.\ 웅산\).mp3
+lrwxrwxrwx 1 sochan sochan 221 Mar  3 13:37 12. Change The World (feat. 웅산).mp3 -> ../../../.git/annex/objects/6G/8K/SHA256E-s7479642--957208748ae03fe4fc8d7877b2c9d82b7f31be0726e4a3dec9063b84cc64cf09.웅산.mp3/SHA256E-s7479642--957208748ae03fe4fc8d7877b2c9d82b7f31be0726e4a3dec9063b84cc64cf09.웅산.mp3
+ % g an info 12.\ Change\ The\ World\ \(feat.\ 웅산\).mp3   
+file: 12. Change The World (feat. 웅산).mp3
+size: 7.48 megabytes
+key: SHA256E-s7479642--957208748ae03fe4fc8d7877b2c9d82b7f31be0726e4a3dec9063b84cc64cf09.웅산.mp3
+present: true
+ % g an calckey 12.\ Change\ The\ World\ \(feat.\ 웅산\).mp3
+SHA256E-s7479642--957208748ae03fe4fc8d7877b2c9d82b7f31be0726e4a3dec9063b84cc64cf09.웅산.mp3
+ % sha256sum 12.\ Change\ The\ World\ \(feat.\ 웅산\).mp3
+957208748ae03fe4fc8d7877b2c9d82b7f31be0726e4a3dec9063b84cc64cf09  12. Change The World (feat. 웅산).mp3
+"""]]
+
+### What version of git-annex are you using? On what operating system?
+
+[[!format sh """
+ % git-annex version
+git-annex version: 6.20170101.1
+build flags: Assistant Webapp Pairing Testsuite S3(multipartupload)(storageclasses) WebDAV Inotify DBus DesktopNotify ConcurrentOutput TorrentParser MagicMime Feeds Quvi
+key/value backends: SHA256E SHA256 SHA512E SHA512 SHA224E SHA224 SHA384E SHA384 SHA3_256E SHA3_256 SHA3_512E SHA3_512 SHA3_224E SHA3_224 SHA3_384E SHA3_384 SKEIN256E SKEIN256 SKEIN512E SKEIN512 SHA1E SHA1 MD5E MD5 WORM URL
+remote types: git gcrypt p2p S3 bup directory rsync web bittorrent webdav tahoe glacier ddar hook external
+local repository version: 5
+supported repository versions: 3 5 6
+upgrade supported from repository versions: 0 1 2 3 4 5
+operating system: linux x86_64
+% lsb_release -a
+No LSB modules are available.
+Distributor ID:	Debian
+Description:	Debian GNU/Linux 9.3 (stretch)
+Release:	9.3
+Codename:	stretch
+"""]]
+
+### Please provide any additional information below.
+Here's a sample file:
+https://www.dropbox.com/s/kghlz41ooaqfr0h/12.%20Change%20The%20World%20%28feat.%20%EC%9B%85%EC%82%B0%29.mp3?dl=0
+
+### Have you had any luck using git-annex before?
+Dude, I love git-annex. I use it to have multiple copies of my HUGE music collection. I love that I can just have partial copies of my whole collection on my laptop/phone/remote servers/usb drive backup and have that all documented properly. This is some amazing software my man!
+
+> [[done]]; see comments --[[Joey]]
diff --git a/doc/bugs/git-annex_adds_unicode_characters_at_end_of_checksum/comment_1_360b2731bde42fc3fcb621b9fa467153._comment b/doc/bugs/git-annex_adds_unicode_characters_at_end_of_checksum/comment_1_360b2731bde42fc3fcb621b9fa467153._comment
new file mode 100644
index 0000000000..8a0b6120d9
--- /dev/null
+++ b/doc/bugs/git-annex_adds_unicode_characters_at_end_of_checksum/comment_1_360b2731bde42fc3fcb621b9fa467153._comment
@@ -0,0 +1,24 @@
+[[!comment format=mdwn
+ username="i@f4fc1d4ed8c7cc91fc284462cb631c270a5195e9"
+ nickname="i"
+ avatar="http://cdn.libravatar.org/avatar/661785c9bf4c87cc795f130b47a1c4ae"
+ subject="checked 6.20180112 and it's the same"
+ date="2018-03-03T16:21:06Z"
+ content="""
+I installed git-annex from Debian buster(6.20180112) and checked, I get the same thing from calckey:
+
+[[!format sh \"\"\"
+ % g an version                                             
+git-annex version: 6.20180112
+build flags: Assistant Webapp Pairing S3(multipartupload)(storageclasses) WebDAV Inotify DBus DesktopNotify ConcurrentOutput TorrentParser MagicMime Feeds Testsuite
+dependency versions: aws-0.16 bloomfilter-2.0.1.0 cryptonite-0.23 DAV-1.3.1 feed-0.3.12.0 ghc-8.0.2 http-client-0.5.7.1 persistent-sqlite-2.6.3 torrent-10000.1.1 uuid-1.3.13 yesod-1.4.5
+key/value backends: SHA256E SHA256 SHA512E SHA512 SHA224E SHA224 SHA384E SHA384 SHA3_256E SHA3_256 SHA3_512E SHA3_512 SHA3_224E SHA3_224 SHA3_384E SHA3_384 SKEIN256E SKEIN256 SKEIN512E SKEIN512 SHA1E SHA1 MD5E MD5 WORM URL
+remote types: git gcrypt p2p S3 bup directory rsync web bittorrent webdav tahoe glacier ddar hook external
+local repository version: 5
+supported repository versions: 3 5 6
+upgrade supported from repository versions: 0 1 2 3 4 5
+operating system: linux x86_64
+ % g an calckey 12.\ Change\ The\ World\ \(feat.\ 웅산\).mp3
+SHA256E-s7479642--957208748ae03fe4fc8d7877b2c9d82b7f31be0726e4a3dec9063b84cc64cf09.웅산.mp3
+\"\"\"]]
+"""]]
diff --git a/doc/bugs/git-annex_adds_unicode_characters_at_end_of_checksum/comment_2_2c0729d9beb09c0392be6d63b491d9ca._comment b/doc/bugs/git-annex_adds_unicode_characters_at_end_of_checksum/comment_2_2c0729d9beb09c0392be6d63b491d9ca._comment
new file mode 100644
index 0000000000..7d404f0574
--- /dev/null
+++ b/doc/bugs/git-annex_adds_unicode_characters_at_end_of_checksum/comment_2_2c0729d9beb09c0392be6d63b491d9ca._comment
@@ -0,0 +1,22 @@
+[[!comment format=mdwn
+ username="i@f4fc1d4ed8c7cc91fc284462cb631c270a5195e9"
+ nickname="i"
+ avatar="http://cdn.libravatar.org/avatar/661785c9bf4c87cc795f130b47a1c4ae"
+ subject="I've also upgraded the repo to version 6"
+ date="2018-03-03T16:23:37Z"
+ content="""
+[[!format sh \"\"\"
+ % g an upgrade
+upgrade (v5 to v6...) (scanning for unlocked files...)
+ok
+ % g an info 12.\ Change\ The\ World\ \(feat.\ 웅산\).mp3   
+file: 12. Change The World (feat. 웅산).mp3
+size: 7.48 megabytes
+key: SHA256E-s7479642--957208748ae03fe4fc8d7877b2c9d82b7f31be0726e4a3dec9063b84cc64cf09.웅산.mp3
+present: true
+ % g an calckey 12.\ Change\ The\ World\ \(feat.\ 웅산\).mp3
+SHA256E-s7479642--957208748ae03fe4fc8d7877b2c9d82b7f31be0726e4a3dec9063b84cc64cf09.웅산.mp3
+\"\"\"]]
+
+Same thing with upgraded repo version.
+"""]]
diff --git a/doc/bugs/git-annex_adds_unicode_characters_at_end_of_checksum/comment_3_4e95c616da82eea1437744aaf02f39dd._comment b/doc/bugs/git-annex_adds_unicode_characters_at_end_of_checksum/comment_3_4e95c616da82eea1437744aaf02f39dd._comment
new file mode 100644
index 0000000000..70e14e80f3
--- /dev/null
+++ b/doc/bugs/git-annex_adds_unicode_characters_at_end_of_checksum/comment_3_4e95c616da82eea1437744aaf02f39dd._comment
@@ -0,0 +1,30 @@
+[[!comment format=mdwn
+ username="i@f4fc1d4ed8c7cc91fc284462cb631c270a5195e9"
+ nickname="i"
+ avatar="http://cdn.libravatar.org/avatar/661785c9bf4c87cc795f130b47a1c4ae"
+ subject="this has something to do with the dot and space"
+ date="2018-03-03T16:43:27Z"
+ content="""
+I did some tests and apparently this happens only if the characters are proceeded by a dot and/or space:
+
+[[!format sh \"\"\"
+ sochan@melchior: 2011 - Kaleidoscope% g an calckey Change.\ 웅산\).mp3            
+SHA256E-s7479642--957208748ae03fe4fc8d7877b2c9d82b7f31be0726e4a3dec9063b84cc64cf09.웅산.mp3
+ sochan@melchior: 2011 - Kaleidoscope% g an calckey Change.\ 웅산.mp3              
+SHA256E-s7479642--957208748ae03fe4fc8d7877b2c9d82b7f31be0726e4a3dec9063b84cc64cf09.웅산.mp3
+ sochan@melchior: 2011 - Kaleidoscope% g an calckey Change.웅산.mp3         
+SHA256E-s7479642--957208748ae03fe4fc8d7877b2c9d82b7f31be0726e4a3dec9063b84cc64cf09.웅산.mp3
+ sochan@melchior: 2011 - Kaleidoscope% g an calckey test_.웅산.mp3 
+SHA256E-s7479642--957208748ae03fe4fc8d7877b2c9d82b7f31be0726e4a3dec9063b84cc64cf09.웅산.mp3
+ sochan@melchior: 2011 - Kaleidoscope% g an calckey test_웅산.mp3 
+SHA256E-s7479642--957208748ae03fe4fc8d7877b2c9d82b7f31be0726e4a3dec9063b84cc64cf09.mp3
+ sochan@melchior: 2011 - Kaleidoscope% g an calckey test_.\ 웅산.mp3 
+SHA256E-s7479642--957208748ae03fe4fc8d7877b2c9d82b7f31be0726e4a3dec9063b84cc64cf09.웅산.mp3
+ sochan@melchior: 2011 - Kaleidoscope% g an calckey test_.\ \ 웅산.mp3         
+SHA256E-s7479642--957208748ae03fe4fc8d7877b2c9d82b7f31be0726e4a3dec9063b84cc64cf09.웅산.mp3
+ sochan@melchior: 2011 - Kaleidoscope% g an calckey test_.\ \ \ 웅산.mp3            
+SHA256E-s7479642--957208748ae03fe4fc8d7877b2c9d82b7f31be0726e4a3dec9063b84cc64cf09.mp3
+\"\"\"]]
+
+This leads me to thinking that this MIGHT have something to do with code that deals with file extensions.
+"""]]
diff --git a/doc/bugs/git-annex_adds_unicode_characters_at_end_of_checksum/comment_5_7f5a6ba6ed7b6f720874f8ded6edaa3c._comment b/doc/bugs/git-annex_adds_unicode_characters_at_end_of_checksum/comment_5_7f5a6ba6ed7b6f720874f8ded6edaa3c._comment
new file mode 100644
index 0000000000..1d8e1cabeb
--- /dev/null
+++ b/doc/bugs/git-annex_adds_unicode_characters_at_end_of_checksum/comment_5_7f5a6ba6ed7b6f720874f8ded6edaa3c._comment
@@ -0,0 +1,28 @@
+[[!comment format=mdwn
+ username="joey"
+ subject="""comment 5"""
+ date="2018-03-05T14:47:20Z"
+ content="""
+The easy workaround to bugs like this migrate the file to the
+SHA256 backend rather than SHA256E.
+
+It may be obvious to us that a file ending in "(feat. xy).mp3"
+has an extension of ".mp3" and not of ". xy).mp3", but this is not very
+obvious to git-annex, which would like to treat a file ending in ".tar.gz"
+as having that compound extension.
+
+The only rule I can think of that would help git-annex understand this is
+to not allow punctuation (other than "." in file extensions). Which it
+actually already filters out of extensions, which is why the extension it
+comes up with is ".xy.mp3". But it could notice the space and closing paren
+in the filename and assume those are not part of an extension. It might
+bite some file with an extension like .foo_", I can't recall seeing many
+such extensions. Ok, made this change.
+
+It remains a bug in the glacier special remote if unicode characters
+prevent uploading to it. We can't limit file
+extensions to ascii, it's perfectly reasonable to use your native language
+characters in a file extension. Leaving bug open since my change does
+nothing about whatever upload bug glacier-cli has. Is the python program
+failing?
+"""]]
diff --git a/doc/bugs/git-annex_adds_unicode_characters_at_end_of_checksum/comment_5_a160bb3ce6b8b6e1fe03d649f710806e._comment b/doc/bugs/git-annex_adds_unicode_characters_at_end_of_checksum/comment_5_a160bb3ce6b8b6e1fe03d649f710806e._comment
new file mode 100644
index 0000000000..f3e504313c
--- /dev/null
+++ b/doc/bugs/git-annex_adds_unicode_characters_at_end_of_checksum/comment_5_a160bb3ce6b8b6e1fe03d649f710806e._comment
@@ -0,0 +1,25 @@
+[[!comment format=mdwn
+ username="i@f4fc1d4ed8c7cc91fc284462cb631c270a5195e9"
+ nickname="i"
+ avatar="http://cdn.libravatar.org/avatar/661785c9bf4c87cc795f130b47a1c4ae"
+ subject="fixed prev commend"
+ date="2018-03-03T16:55:16Z"
+ content="""
+Oh man, yes, yes it does have something to do with extensions, it does not matter what it is, it gets added at the end if it is prefixed by a dot, even if there is something like a closing bracket in there:
+
+[[!format sh \"\"\"
+ % g an calckey test.wtf.mp3 
+SHA256E-s7479642--957208748ae03fe4fc8d7877b2c9d82b7f31be0726e4a3dec9063b84cc64cf09.wtf.mp3
+ % g an calckey test.wtf\).mp3
+SHA256E-s7479642--957208748ae03fe4fc8d7877b2c9d82b7f31be0726e4a3dec9063b84cc64cf09.wtf.mp3
+\"\"\"]]
+
+So it looks like I can avoid this issue by simply adding a space within the last segment after the dot, for example:
+
+[[!format sh \"\"\"
+ % g an calckey 12.\ Change\ The\ World\ \(feat.\ 웅산\).mp3 
+SHA256E-s7479642--957208748ae03fe4fc8d7877b2c9d82b7f31be0726e4a3dec9063b84cc64cf09.웅산.mp3
+ % g an calckey 12.\ Change\ The\ World\ \(feat.\ 웅산\ \).mp3 
+SHA256E-s7479642--957208748ae03fe4fc8d7877b2c9d82b7f31be0726e4a3dec9063b84cc64cf09.mp3
+\"\"\"]]
+"""]]
diff --git a/doc/bugs/git-annex_adds_unicode_characters_at_end_of_checksum/comment_6_c39d87c005f82da6424064f61b8ccd58._comment b/doc/bugs/git-annex_adds_unicode_characters_at_end_of_checksum/comment_6_c39d87c005f82da6424064f61b8ccd58._comment
new file mode 100644
index 0000000000..a6b40418ff
--- /dev/null
+++ b/doc/bugs/git-annex_adds_unicode_characters_at_end_of_checksum/comment_6_c39d87c005f82da6424064f61b8ccd58._comment
@@ -0,0 +1,33 @@
+[[!comment format=mdwn
+ username="i@f4fc1d4ed8c7cc91fc284462cb631c270a5195e9"
+ nickname="i"
+ avatar="http://cdn.libravatar.org/avatar/661785c9bf4c87cc795f130b47a1c4ae"
+ subject="thanks for the quick response!"
+ date="2018-03-05T17:19:19Z"
+ content="""
+That was really quick, thanks. I had no idea that the E in SHA256E stands for adding extensions. That makes the fix much easier, and much neater than adding spaces to filenames. Although it does make me wonder, what's the advantage of having the file extension in the key?
+
+And yes, the python glacier-cli is failing with:
+
+[[!format sh \"\"\"
+% git annex copy 12.\ Change\ The\ World\ \(feat.\ 웅산\).mp3 --to glacier
+copy 12. Change The World (feat. 웅산).mp3 (checking glacier...) Traceback (most recent call last):
+  File \"/usr/local/bin/glacier\", line 737, in 
+    main()
+  File \"/usr/local/bin/glacier\", line 733, in main
+    App().main()
+  File \"/usr/local/bin/glacier\", line 719, in main
+    self.args.func()
+  File \"/usr/local/bin/glacier\", line 600, in archive_checkpresent
+    self.args.vault, self.args.name)
+  File \"/usr/local/bin/glacier\", line 161, in get_archive_last_seen
+    result = self._get_archive_query_by_ref(vault, ref).one()
+  File \"/usr/local/bin/glacier\", line 136, in _get_archive_query_by_ref
+    if ref.startswith('id:'):
+UnicodeDecodeError: 'ascii' codec can't decode byte 0xec in position 83: ordinal not in range(128)
+(user error (glacier [\"--region=eu-west-1\",\"archive\",\"checkpresent\",\"music\",\"--quiet\",\"SHA256E-s7479642--957208748ae03fe4fc8d7877b2c9d82b7f31be0726e4a3dec9063b84cc64cf09.\50885\49328.mp3\"] exited 1)) failed
+git-annex: copy: 1 failed
+\"\"\"]]
+
+I'll open an issue in their GitHub in a moment since now I can give them some more context. Cheers!
+"""]]
diff --git a/doc/bugs/git-annex_adds_unicode_characters_at_end_of_checksum/comment_7_9503988854499ceb75c39ec248f7b238._comment b/doc/bugs/git-annex_adds_unicode_characters_at_end_of_checksum/comment_7_9503988854499ceb75c39ec248f7b238._comment
new file mode 100644
index 0000000000..73aac58abf
--- /dev/null
+++ b/doc/bugs/git-annex_adds_unicode_characters_at_end_of_checksum/comment_7_9503988854499ceb75c39ec248f7b238._comment
@@ -0,0 +1,10 @@
+[[!comment format=mdwn
+ username="i@f4fc1d4ed8c7cc91fc284462cb631c270a5195e9"
+ nickname="i"
+ avatar="http://cdn.libravatar.org/avatar/661785c9bf4c87cc795f130b47a1c4ae"
+ subject="opened github issue with glacier-cli"
+ date="2018-03-05T17:28:53Z"
+ content="""
+Here's the issue:
+https://github.com/basak/glacier-cli/issues/72
+"""]]
diff --git a/doc/bugs/git-annex_adds_unicode_characters_at_end_of_checksum/comment_8_8dd44353d46d774ad376b24ecca3e56e._comment b/doc/bugs/git-annex_adds_unicode_characters_at_end_of_checksum/comment_8_8dd44353d46d774ad376b24ecca3e56e._comment
new file mode 100644
index 0000000000..31f2089212
--- /dev/null
+++ b/doc/bugs/git-annex_adds_unicode_characters_at_end_of_checksum/comment_8_8dd44353d46d774ad376b24ecca3e56e._comment
@@ -0,0 +1,18 @@
+[[!comment format=mdwn
+ username="joey"
+ subject="""comment 8"""
+ date="2018-03-06T17:40:50Z"
+ content="""
+There are a few programs that look at filename extensions and when asked to
+open a git-annex symlink, resolve the symlink and look at the extension of
+the file it *points* to. Mostly such programs are on OSX I think,
+although there may be one or two on Linux. The extension is in the keys by
+default because of that insanity.
+
+It might be that you could mess with your locale settings and get python to
+manage to decode the unicode character.
+
+Since it looks like everything that it makes sense to do in git-annex has
+been done, and you opened the glacier-cli bug, I suppose I'll close this
+one.
+"""]]
diff --git a/doc/bugs/git-annex_branch_not_being_pushed_after_git-annex_forget.mdwn b/doc/bugs/git-annex_branch_not_being_pushed_after_git-annex_forget.mdwn
new file mode 100644
index 0000000000..688b1ef7bf
--- /dev/null
+++ b/doc/bugs/git-annex_branch_not_being_pushed_after_git-annex_forget.mdwn
@@ -0,0 +1,30 @@
+### Please describe the problem.
+I have a repository that is being synced with a normal Bitbucket git repository (with `annex-ignore = true`). It's been created with the webapp in direct mode and I'm using the assistant.
+
+Everything is working fine, but after running `git annex forget` the git-annex branch in Bitbucket stopped being pushed. git-annex is still working correctly, but I have a lot of useless commits in the history.
+
+So currently I have this branches:
+
+    # git branch -a
+    * annex/direct/master
+      git-annex
+      master
+      synced/master
+      remotes/bitbucket/git-annex
+      remotes/bitbucket/master
+      remotes/bitbucket/synced/git-annex
+      remotes/bitbucket/synced/master
+
+And `remotes/bitbucket/git-annex` isn't being updated with the content of `git-annex`
+
+### What version of git-annex are you using? On what operating system?
+
+5.20150812 and 5.20150731 in Debian Sid and Testing
+
+### Please provide any additional information below.
+
+As I've said, git-annex is still working normally, so the solution might be to simply remove that branch.
+
+### Have you had any luck using git-annex before? (Sometimes we get tired of reading bug reports all day and a lil' positive end note does wonders)
+
+I use `git-annex` daily with 3 repositories with a total of ~112GB, and I'm really happy with it. Thank you very much for the effort.
diff --git a/doc/bugs/git-annex_branch_not_being_pushed_after_git-annex_forget/comment_1_babea9a16f25b471026732ea63dd78a8._comment b/doc/bugs/git-annex_branch_not_being_pushed_after_git-annex_forget/comment_1_babea9a16f25b471026732ea63dd78a8._comment
new file mode 100644
index 0000000000..437613ee75
--- /dev/null
+++ b/doc/bugs/git-annex_branch_not_being_pushed_after_git-annex_forget/comment_1_babea9a16f25b471026732ea63dd78a8._comment
@@ -0,0 +1,14 @@
+[[!comment format=mdwn
+ username="joey"
+ subject="""comment 1"""
+ date="2015-09-15T15:56:46Z"
+ content="""
+"You may need to force git to push the branch to any git repositories not
+running git-annex." -- git-annex-forget man page
+
+I guess bitbucket doesn't accept pushes that replace an existing branch
+with one with a disconnected history. git push --force should sort it out.
+
+I'm not clear how this is a bug. Is there some change needed in the 
+documentation or perhaps in git-annex sync?
+"""]]
diff --git a/doc/bugs/git-annex_branch_not_being_pushed_after_git-annex_forget/comment_2_dd24434631ae3898e8f5c1f878d1965f._comment b/doc/bugs/git-annex_branch_not_being_pushed_after_git-annex_forget/comment_2_dd24434631ae3898e8f5c1f878d1965f._comment
new file mode 100644
index 0000000000..264a1daf32
--- /dev/null
+++ b/doc/bugs/git-annex_branch_not_being_pushed_after_git-annex_forget/comment_2_dd24434631ae3898e8f5c1f878d1965f._comment
@@ -0,0 +1,9 @@
+[[!comment format=mdwn
+ username="Jonan"
+ subject="comment 2"
+ date="2015-09-15T16:13:20Z"
+ content="""
+Ok, sorry, must have miss that info.
+
+Couldn't git-annex take care of the `push -f`?
+"""]]
diff --git a/doc/bugs/git-annex_branch_not_being_pushed_after_git-annex_forget/comment_3_70f2b9e61e90528fdbef4d8a6e8dd972._comment b/doc/bugs/git-annex_branch_not_being_pushed_after_git-annex_forget/comment_3_70f2b9e61e90528fdbef4d8a6e8dd972._comment
new file mode 100644
index 0000000000..be0e490943
--- /dev/null
+++ b/doc/bugs/git-annex_branch_not_being_pushed_after_git-annex_forget/comment_3_70f2b9e61e90528fdbef4d8a6e8dd972._comment
@@ -0,0 +1,13 @@
+[[!comment format=mdwn
+ username="joey"
+ subject="""comment 3"""
+ date="2015-09-15T17:34:15Z"
+ content="""
+What git-annex command? 
+
+`git-annex sync` should already deal with this, AFAICS. It forces the push
+to the remote's synced/git-annex branch.
+
+Some remote's may have receive.denyNonFastForwards set, which will prevent
+even a forced push. git-annex sync prints out some hints if this happens.
+"""]]
diff --git a/doc/bugs/git-annex_branch_not_being_pushed_after_git-annex_forget/comment_4_11e9ceb40aa5492a1a3e9ebfbd246832._comment b/doc/bugs/git-annex_branch_not_being_pushed_after_git-annex_forget/comment_4_11e9ceb40aa5492a1a3e9ebfbd246832._comment
new file mode 100644
index 0000000000..3233e932f2
--- /dev/null
+++ b/doc/bugs/git-annex_branch_not_being_pushed_after_git-annex_forget/comment_4_11e9ceb40aa5492a1a3e9ebfbd246832._comment
@@ -0,0 +1,7 @@
+[[!comment format=mdwn
+ username="Jonan"
+ subject="comment 4"
+ date="2015-09-16T11:15:11Z"
+ content="""
+Yes, the `synced/git-annex` branch is being pushed correctly, it's the `git-annex` branch the one giving trouble.
+"""]]
diff --git a/doc/bugs/git-annex_branch_not_being_pushed_after_git-annex_forget/comment_5_20ee285d62c35e18b78b121d8e4de81b._comment b/doc/bugs/git-annex_branch_not_being_pushed_after_git-annex_forget/comment_5_20ee285d62c35e18b78b121d8e4de81b._comment
new file mode 100644
index 0000000000..0285ebbed6
--- /dev/null
+++ b/doc/bugs/git-annex_branch_not_being_pushed_after_git-annex_forget/comment_5_20ee285d62c35e18b78b121d8e4de81b._comment
@@ -0,0 +1,35 @@
+[[!comment format=mdwn
+ username="joey"
+ subject="""comment 5"""
+ date="2015-09-18T16:27:37Z"
+ content="""
+For all git-annex knows, the remote repository may be actively making
+changes to its own git-annex branch. So, forcing pushes of it could lose
+data, and are not a good idea, and so not done, whereas synced/git-annex
+is force pushed.
+
+If git-annex is indeed being run in the remote repository, it will eventually
+try to merge the newly pushed synced/git-annex with its local git-annex
+branch, notice the forget operation has started, and automatically handle
+updating its git-annex branch appropriatly to be a child of the new
+synced/git-annex branch. At that point, pushes of the git-annex branch will
+work again.
+
+Since your remote is on a hosting service that doesn't use git-annex, that
+doesn't happen, and you'll need to force push to get that branch to update.
+(Although it's somewhat optional to do so; other clients using that remote
+with git-annex sync will pull down the synced/git-annex branch and keep
+working fine even if you never force push the git-annex branch.)
+
+Clearly all this is pretty complicated, but I don't see a good way to improve
+the behavior, unless there's some way to detect that the remote repository
+is not running git-annex, and so assume it's safe to auto-force-push that
+branch. I suppose it could check to see if the remote repository has an
+annex.uuid set, and if not, assume it's not using git-annex. But, if this
+assumption turns out to be wrong (due to a race, or a problem communicating
+the annex.uuid, etc), data loss could result.
+
+Maybe a better fix would be to improve the note about this on git-annex
+forget's man page, and perhaps also make git-annex sync print some helpful
+message when it detects this situation.
+"""]]
diff --git a/doc/bugs/git-annex_branch_shows_commit_with_looong_commitlog.mdwn b/doc/bugs/git-annex_branch_shows_commit_with_looong_commitlog.mdwn
new file mode 100644
index 0000000000..718170a989
--- /dev/null
+++ b/doc/bugs/git-annex_branch_shows_commit_with_looong_commitlog.mdwn
@@ -0,0 +1,74 @@
+### Please describe the problem.
+
+I have found a really weird commit in my git-annex branch:
+
+    * a59dd1c update (il y a 8 heures) 
+    * 57f887a update (recovery from race) (recovery from race) (recovery from race) [...]
+
+it repeats that for a looong time. about 12 000 times, to be more precise:
+
+[[!format sh """
+anarcat@marcos:video$ git show 57f887a | wc
+      5   12686   88850
+"""]]
+
+### What steps will reproduce the problem?
+
+Now i have absolutely no idea how I managed that. I got through some pretty dark moments last night trying various levels of git-annex voodoo (including a duplicate repo which was rsync'd to a backup drive so the unique identifier applied to two distinct paths), so I have no idea exactly what happened here.
+
+### What version of git-annex are you using? On what operating system?
+
+debian jessie amd64 5.20140412
+
+### Please provide any additional information below.
+
+[[!format sh """
+anarcat@marcos:video$ git show 57f887a | tail -c 100
+very from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race)
+anarcat@marcos:video$ git show 57f887a | head -c 512
+commit 57f887a9d766829d00832ad1ee23b2785212d055
+Author: Antoine Beaupré 
+Date:   Sat Apr 19 01:48:18 2014 -0400
+
+    update (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery
+"""]]
+
+that's 80KB for only one commit here - maybe that should be cleaned up? --[[anarcat]]
+
+Ah! more information: it seems that 01:48 was the moment i shutdown the assistant in yet another panic... 
+
+[[!format sh """
+anarcat@marcos:video$ ls -al .git/annex/daemon.log*
+-rw-r--r-- 1 anarcat anarcat  17075 avril 19 09:28 .git/annex/daemon.log
+-rw-r--r-- 1 anarcat anarcat 128367 avril 19 01:48 .git/annex/daemon.log.1
+"""]]
+
+an extract from that second logfile:
+
+[[!format sh """
+19/Apr/2014:01:31:38 -0400 [Error#yesod-core] unknown response from git cat-file ("9a73bf01-ed01-450d-a0ab-f20fff47ed32 encryption=none name=stephc rsyncurl=192.168.0.104:video/ type=rsync timestamp=1397865844.925354s","refs/heads/git-annex:remote.log") @(yesod-core-1.2.3:Yesod.Core.Class.Yesod ./Yesod/Core/Class/Yesod.hs:471:5)
+19/Apr/2014:01:31:50 -0400 [Error#yesod-core] unknown response from git cat-file ("fe428a7a-25a2-4c2e-b01f-315c490cbe45 encryption=none name=myrsync rsyncurl=/home/anarcat/video/ type=rsync timestamp=1397868063.038898s","refs/heads/git-annex:remote.log") @(yesod-core-1.2.3:Yesod.Core.Class.Yesod ./Yesod/Core/Class/Yesod.hs:471:5)
+19/Apr/2014:01:31:57 -0400 [Error#yesod-core] unknown response from git cat-file ("","refs/heads/git-annex:remote.log") @(yesod-core-1.2.3:Yesod.Core.Class.Yesod ./Yesod/Core/Class/Yesod.hs:471:5)
+[2014-04-19 01:32:03 EDT] TransferScanner: Syncing with test, mnt
+Depuis /mnt/video
+ * [nouvelle branche] synced/git-annex -> test/synced/git-annex
+ * [nouvelle branche] synced/master -> test/synced/master
+fatal: 'mnt' does not appear to be a git repository
+fatal: Could not read from remote repository.
+
+Please make sure you have the correct access rights
+and the repository exists.
+Already up-to-date.
+[2014-04-19 01:32:21 EDT] main: warning git-annex has been shut down
+
+(Recording state in git...)
+(Recording state in git...)
+(Recording state in git...)
+(Recording state in git...)
+"""]]
+
+the last line repeats about 4000 times.
+
+i would love to paste the daemon.log.1 file, but it seems like it containts encryption credentials... which i have no idea how to get rid of or change.
+
+[[!tag moreinfo]]
diff --git a/doc/bugs/git-annex_branch_shows_commit_with_looong_commitlog/comment_1_b83888a98075125dd043f323c99da03b._comment b/doc/bugs/git-annex_branch_shows_commit_with_looong_commitlog/comment_1_b83888a98075125dd043f323c99da03b._comment
new file mode 100644
index 0000000000..75bfd73685
--- /dev/null
+++ b/doc/bugs/git-annex_branch_shows_commit_with_looong_commitlog/comment_1_b83888a98075125dd043f323c99da03b._comment
@@ -0,0 +1,25 @@
+[[!comment format=mdwn
+ username="http://joeyh.name/"
+ ip="209.250.56.191"
+ subject="comment 1"
+ date="2014-04-20T17:11:02Z"
+ content="""
+Is there one commit with this long message, or 12 thousand commits each adding another (recovery from race) to the pyramid?
+
+
    + - Also safely handles a race that can occur if a change is being pushed
+ - into the branch at the same time. When the race happens, the commit will
+ - be made on top of the newly pushed change, but without the index file
+ - being updated to include it. The result is that the newly pushed
+ - change is reverted. This race is detected and another commit made
+ - to fix it.
+
    
+
+If there is only one message, then it must have tried 12k times to commit to the git-annex branch and each time something else pushed or commited to the git-annex branch and overwrote its commit. This seems statistically unlikely. (Also there's locking to prevent multiple local git-annex processes from committing at the same time.)
+
+There have been a few other unexplained reports of this race detection code repeatedly triggering.
+
+> \"shutdown the assistant in yet another panic\"
+
+This implies some hasty, perhaps unusual shutdown method, and some unusual situation. I think you could tell me more about what was going on.
+"""]]
diff --git a/doc/bugs/git-annex_branch_shows_commit_with_looong_commitlog/comment_2_4a7d824b6e75693cf47f6efbf2c99e2e._comment b/doc/bugs/git-annex_branch_shows_commit_with_looong_commitlog/comment_2_4a7d824b6e75693cf47f6efbf2c99e2e._comment
new file mode 100644
index 0000000000..0eeabf070d
--- /dev/null
+++ b/doc/bugs/git-annex_branch_shows_commit_with_looong_commitlog/comment_2_4a7d824b6e75693cf47f6efbf2c99e2e._comment
@@ -0,0 +1,10 @@
+[[!comment format=mdwn
+ username="https://id.koumbit.net/anarcat"
+ ip="72.0.72.144"
+ subject="comment 2"
+ date="2014-04-20T23:58:00Z"
+ content="""
+this was a single commit.
+
+i am not sure i can extract much more information from my memory: details are hazy as i was working late on a problem that night... sorry!
+"""]]
diff --git a/doc/bugs/git-annex_branch_shows_commit_with_looong_commitlog/comment_3_cbbeaa691d102bd7d29f5e9bad9d6f53._comment b/doc/bugs/git-annex_branch_shows_commit_with_looong_commitlog/comment_3_cbbeaa691d102bd7d29f5e9bad9d6f53._comment
new file mode 100644
index 0000000000..8651405091
--- /dev/null
+++ b/doc/bugs/git-annex_branch_shows_commit_with_looong_commitlog/comment_3_cbbeaa691d102bd7d29f5e9bad9d6f53._comment
@@ -0,0 +1,32 @@
+[[!comment format=mdwn
+ username="Francois"
+ ip="2001:788:5:1:29c2:de49:9811:51c8"
+ subject="comment 3"
+ date="2014-08-15T20:45:38Z"
+ content="""
+I've been experiencing the exact same problem and searching for **recovery from race** lead me to this bug report. Thanks for reporting it!
+
+For a few months, a repo storing ~19'000 files (mostly immutable pictures) started to launch memory hungry \"git log\" processes. For example:
+
+    4797 francois  20   0 8118296 7.719g   2032 D  22.3 50.2   0:11.61 git                                                                                                   
+
+    4797 pts/1    D+     0:12 git --git-dir=~/Pictures/.git --work-tree=~/Pictures -c core.bare=false log refs/heads/git-annex..52e44b967ad5d316d832562be02c5555c1f6d2a4 --oneline -n1
+
+Thanks to the hints found in this report, I was able to find many huge commit messages such as this one:
+
+    $ git show 6357b208 
+    commit 6357b2081e7c85dfe1ccc10824b75f3e212e6386
+    Author: Francois Deppierraz 
+    Date:   Sat Jun 14 10:38:46 2014 +0200
+
+        update (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) [...]
+
+    $ git show 6357b208 | wc 
+      5  444026 3108236
+
+There were probably many new files added on Jun 14th and looking for a way to increase to sync speed, especially to a S3-like remote, I found the solution on this wiki for [multiple concurrent transfers](https://git-annex.branchable.com/forum/Feature_request:_Multiple_concurrent_transfers/).
+
+This looks like a likely culprit for generating race conditions. What do you think?
+
+    git-annex version: 5.20140412ubuntu1
+"""]]
diff --git a/doc/bugs/git-annex_branch_shows_commit_with_looong_commitlog/comment_4_094191b806ac76b2aef325733fe37136._comment b/doc/bugs/git-annex_branch_shows_commit_with_looong_commitlog/comment_4_094191b806ac76b2aef325733fe37136._comment
new file mode 100644
index 0000000000..e80d66b5a3
--- /dev/null
+++ b/doc/bugs/git-annex_branch_shows_commit_with_looong_commitlog/comment_4_094191b806ac76b2aef325733fe37136._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="Francois"
+ ip="2001:788:5:1:29c2:de49:9811:51c8"
+ subject="comment 4"
+ date="2014-08-15T20:51:12Z"
+ content="""
+Bonus question: is there any way to bring this repo back into a working state (for instance with git-filter-branch?) ?
+"""]]
diff --git a/doc/bugs/git-annex_branch_shows_commit_with_looong_commitlog/comment_5_c97926b15ba320f57a6441f9844cb139._comment b/doc/bugs/git-annex_branch_shows_commit_with_looong_commitlog/comment_5_c97926b15ba320f57a6441f9844cb139._comment
new file mode 100644
index 0000000000..62903f1f16
--- /dev/null
+++ b/doc/bugs/git-annex_branch_shows_commit_with_looong_commitlog/comment_5_c97926b15ba320f57a6441f9844cb139._comment
@@ -0,0 +1,7 @@
+[[!comment format=mdwn
+ username="joey"
+ subject="""comment 5"""
+ date="2015-02-09T21:10:58Z"
+ content="""
+I guess the thing to do in this case is to run `git annex forget`
+"""]]
diff --git a/doc/bugs/git-annex_branch_shows_commit_with_looong_commitlog/comment_6_3b70a60ef1c47871a3933176eac38174._comment b/doc/bugs/git-annex_branch_shows_commit_with_looong_commitlog/comment_6_3b70a60ef1c47871a3933176eac38174._comment
new file mode 100644
index 0000000000..25e6059f10
--- /dev/null
+++ b/doc/bugs/git-annex_branch_shows_commit_with_looong_commitlog/comment_6_3b70a60ef1c47871a3933176eac38174._comment
@@ -0,0 +1,40 @@
+[[!comment format=mdwn
+ username="joey"
+ subject="""comment 6"""
+ date="2015-02-09T21:59:31Z"
+ content="""
+[[forum/repair_stuck_on_ls-tree_command]] is another case of that, and I got ahold of
+that repository for analysis.
+
+In that case, there was indeed an inverse pyramid effect where each commit
+added one more " (recovery from race)" to its parent commit.
+
+The code can clearly loop 
+if it keeps detecting a race and somehow fails to recover from it. Leading
+to a whole stack of commits with progressively longer messages.
+I don't see any way to get just one commit with a long message, which
+comment #1 seems to say happened.
+
+Apparently loops for a while and then succeeds in recovering from
+the race, since it then stops looping.
+
+I have added additional debug info to the commit message, in hopes of detecting
+what might be going wrong that causes it to loop.
+
+Seems to me there are two possibilities.
+
+One is that something else is continually changing the git-annex
+branch in a way that keeps triggering the race. If so, it might make
+sense for git-annex to do a brief random sleep (a few hundredths of a
+second) before recovering, to let whatever it is quiet down. I've done so.
+
+The other is some kind of bug where it detects a race when none
+occurred. Perhaps it's misparsing the commit or git cat-file is failing
+to output it, and so it's not finding the expected parent refs, for example.
+But in that case, why would it detect a race for many commits
+in a row, and then eventually not detect a race anymore?
+
+Also, I've made these messages no longer stack up even if it does go into a
+loop, which will at least help with the object size bloat, though not with the
+number of commits bloat.
+"""]]
diff --git a/doc/bugs/git-annex_branch_shows_commit_with_looong_commitlog/comment_7_fc56ec87725c7427794b70ee2ba2822a._comment b/doc/bugs/git-annex_branch_shows_commit_with_looong_commitlog/comment_7_fc56ec87725c7427794b70ee2ba2822a._comment
new file mode 100644
index 0000000000..cd7510140a
--- /dev/null
+++ b/doc/bugs/git-annex_branch_shows_commit_with_looong_commitlog/comment_7_fc56ec87725c7427794b70ee2ba2822a._comment
@@ -0,0 +1,12 @@
+[[!comment format=mdwn
+ username="joey"
+ subject="""comment 7"""
+ date="2015-02-09T22:44:50Z"
+ content="""
+The only other thing I can think to do about this is to make git-annex block
+direct pushes to the git-annex branch, by setting up a pre-receive hook.
+
+Then, there should be no way for the race to occur
+(except for a locking failure, or a pull into the branch?),
+and git-annex wouldn't need to check for it
+"""]]
diff --git a/doc/bugs/git-annex_can__39__t_compile_on_FreeBSD.mdwn b/doc/bugs/git-annex_can__39__t_compile_on_FreeBSD.mdwn
new file mode 100644
index 0000000000..ffcf21cd1b
--- /dev/null
+++ b/doc/bugs/git-annex_can__39__t_compile_on_FreeBSD.mdwn
@@ -0,0 +1,33 @@
+### Please describe the problem.
+
+git-annex can't compile on FreeBSD; specifically, the build fails while satisfying dependencies.
+
+### What steps will reproduce the problem?
+
+1. git clone git://git-annex.branchable.com/ git-annex
+2. cd git-annex
+3. cabal install -j -f-assistant -webapp -webdav -pairing -xmpp -dns -dbus -magicmime --only-dependencies
+
+### What version of git-annex are you using? On what operating system?
+
+git-annex HEAD.
+
+FreeBSD 11.1-RELEASE r321309 GENERIC amd64
+
+### Please provide any additional information below.
+
+The full log is available at [https://gitlab.com/snippets/1743708](https://gitlab.com/snippets/1743708).  Summary below:
+
+    cabal: Error: some packages failed to install:
+    esqueleto-2.5.3-J2ccnERt7unG9UdXfc7jAa depends on esqueleto-2.5.3 which failed to install.
+    persistent-2.7.0-IWtmEvQAI3yHscMZvQrE6P failed during the building phase. The exception was: ExitFailure 1
+    persistent-sqlite-2.6.4-3aF88LYjPwqbsHGVQ1VUp depends on
+    persistent-sqlite-2.6.4 which failed to install.
+    persistent-template-2.5.4-2tn9hCQqx2e2mAPIKgHBFO depends on
+    persistent-template-2.5.4 which failed to install.
+
+### Have you had any luck using git-annex before? (Sometimes we get tired of reading bug reports all day and a lil' positive end note does wonders)
+
+No, I'm afraid.  But it looks really good!  I'm trying to use it to add a bunch of high-res images to my Jeykll website, all managed through Git, with the images stored in S3.
+
+> [[done]] --[[Joey]]
diff --git a/doc/bugs/git-annex_can__39__t_compile_on_FreeBSD/comment_1_a959c15b7144acc5d68bb3891107a480._comment b/doc/bugs/git-annex_can__39__t_compile_on_FreeBSD/comment_1_a959c15b7144acc5d68bb3891107a480._comment
new file mode 100644
index 0000000000..456fd77967
--- /dev/null
+++ b/doc/bugs/git-annex_can__39__t_compile_on_FreeBSD/comment_1_a959c15b7144acc5d68bb3891107a480._comment
@@ -0,0 +1,17 @@
+[[!comment format=mdwn
+ username="joey"
+ subject="""comment 1"""
+ date="2018-08-13T16:35:43Z"
+ content="""
+This is caused by this bug in esquelito: 
+https://github.com/bitemyapp/esqueleto/issues/80
+
+The best way to avoid this kind of transient breakage in the haskell
+dependencies of git-annex is to build it using stack, instead of cabal.
+stack pins packages to a consistent working set.
+
+I don't really see this as something that warrants a change to git-annex.
+Using bleeding edge versions of all build dependencies will break, that's
+why the build docs recommend not using cabal if you don't want to be involved
+in fixing that kind  of breakage.
+"""]]
diff --git a/doc/bugs/git-annex_can__39__t_compile_on_FreeBSD/comment_2_e93c0bf6b2134814b490c3bc1362425a._comment b/doc/bugs/git-annex_can__39__t_compile_on_FreeBSD/comment_2_e93c0bf6b2134814b490c3bc1362425a._comment
new file mode 100644
index 0000000000..ad12693a8a
--- /dev/null
+++ b/doc/bugs/git-annex_can__39__t_compile_on_FreeBSD/comment_2_e93c0bf6b2134814b490c3bc1362425a._comment
@@ -0,0 +1,10 @@
+[[!comment format=mdwn
+ username="duncan_bayne"
+ avatar="http://cdn.libravatar.org/avatar/2fc23e2009234ad965f9d5d796400417"
+ subject="RTFM"
+ date="2018-08-16T04:25:26Z"
+ content="""
+Ah, right, I might have missed the line in the docs that say \"and it's not uncommon for it to be broken from time to time.\" :-/
+
+Reinstalling with stack, thanks for the tip :)
+"""]]
diff --git a/doc/bugs/git-annex_can__39__t_compile_on_FreeBSD_using_stack.mdwn b/doc/bugs/git-annex_can__39__t_compile_on_FreeBSD_using_stack.mdwn
new file mode 100644
index 0000000000..e5ed9c881f
--- /dev/null
+++ b/doc/bugs/git-annex_can__39__t_compile_on_FreeBSD_using_stack.mdwn
@@ -0,0 +1,51 @@
+### Please describe the problem.
+
+git-annex can't compile on FreeBSD; specifically, the build fails while compiling Utility.DirWatcher.Kqueue.
+
+### What steps will reproduce the problem?
+
+1. git clone git://git-annex.branchable.com/ git-annex
+2. cd git-annex
+3. stack setup
+4. stack install
+
+### What version of git-annex are you using? On what operating system?
+
+git-annex HEAD.
+
+FreeBSD 11.1-RELEASE r321309 GENERIC amd64
+
+### Please provide any additional information below.
+
+Compilation failure is as follows:
+
+    [126 of 586] Compiling Utility.DirWatcher.Kqueue ( Utility/DirWatcher/Kqueue.hs, .stack-work/dist/x86_64-freebsd/Cabal-1.24.2.0/build/git-annex/git-annex-tmp/Utility/Di
+    rWatcher/Kqueue.o )
+
+    /usr/home/duncan/code/git-annex/Utility/DirWatcher/Kqueue.hs:112:49: error:
+        Variable not in scope:
+          openFd :: FilePath -> t0 -> Maybe a0 -> t1 -> IO t
+
+    /usr/home/duncan/code/git-annex/Utility/DirWatcher/Kqueue.hs:112:60: error:
+        Data constructor not in scope: ReadOnly
+
+    /usr/home/duncan/code/git-annex/Utility/DirWatcher/Kqueue.hs:112:77: error:
+        Variable not in scope: defaultFileFlags
+
+    /usr/home/duncan/code/git-annex/Utility/DirWatcher/Kqueue.hs:132:15: error:
+        Variable not in scope: closeFd :: Fd -> IO b0
+
+    /usr/home/duncan/code/git-annex/Utility/DirWatcher/Kqueue.hs:170:14: error:
+        Variable not in scope: closeFd :: Fd -> IO ()
+    Completed 210 action(s).
+
+    --  While building custom Setup.hs for package git-annex-6.20180807 using:
+          /usr/home/duncan/code/git-annex/.stack-work/dist/x86_64-freebsd/Cabal-1.24.2.0/setup/setup --builddir=.stack-work/dist/x86_64-freebsd/Cabal-1.24.2.0 build exe:git
+    -annex --ghc-options " -ddump-hi -ddump-to-file"
+        Process exited with code: ExitFailure 1
+
+### Have you had any luck using git-annex before? (Sometimes we get tired of reading bug reports all day and a lil' positive end note does wonders)
+
+Not yet.  But I'm used to issues compiling new tools on FreeBSD, so I'm in no way disheartened by this :)
+
+> [[done]] --[[Joey]]
diff --git a/doc/bugs/git-annex_can__39__t_compile_on_FreeBSD_using_stack/comment_1_ca05260b9dd2feb9530dab17250c0b8d._comment b/doc/bugs/git-annex_can__39__t_compile_on_FreeBSD_using_stack/comment_1_ca05260b9dd2feb9530dab17250c0b8d._comment
new file mode 100644
index 0000000000..4184ac6efc
--- /dev/null
+++ b/doc/bugs/git-annex_can__39__t_compile_on_FreeBSD_using_stack/comment_1_ca05260b9dd2feb9530dab17250c0b8d._comment
@@ -0,0 +1,11 @@
+[[!comment format=mdwn
+ username="joey"
+ subject="""comment 1"""
+ date="2018-08-29T16:07:48Z"
+ content="""
+I've fixed those problems.
+
+It looks like noone has compiled on FreeBSD in a long time. Maybe since I
+first developed that module in 2013. So there may be other problems. Please
+open bugs if so.
+"""]]
diff --git a/doc/bugs/git-annex_can__39__t_compile_on_FreeBSD_using_stack/comment_2_7a5ff239751a2dc0547ad19b5bfa761f._comment b/doc/bugs/git-annex_can__39__t_compile_on_FreeBSD_using_stack/comment_2_7a5ff239751a2dc0547ad19b5bfa761f._comment
new file mode 100644
index 0000000000..abd4e6f772
--- /dev/null
+++ b/doc/bugs/git-annex_can__39__t_compile_on_FreeBSD_using_stack/comment_2_7a5ff239751a2dc0547ad19b5bfa761f._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="duncan_bayne"
+ avatar="http://cdn.libravatar.org/avatar/2fc23e2009234ad965f9d5d796400417"
+ subject="Thanks :)"
+ date="2018-09-03T00:39:08Z"
+ content="""
+Thanks :)  Will open further bugs if necessary.
+"""]]
diff --git a/doc/bugs/git-annex_can__39__t_compile_on_FreeBSD_using_stack/comment_3_3b38a1dc9b5d3b74f937206a90967c57._comment b/doc/bugs/git-annex_can__39__t_compile_on_FreeBSD_using_stack/comment_3_3b38a1dc9b5d3b74f937206a90967c57._comment
new file mode 100644
index 0000000000..95317f9086
--- /dev/null
+++ b/doc/bugs/git-annex_can__39__t_compile_on_FreeBSD_using_stack/comment_3_3b38a1dc9b5d3b74f937206a90967c57._comment
@@ -0,0 +1,13 @@
+[[!comment format=mdwn
+ username="duncan_bayne"
+ avatar="http://cdn.libravatar.org/avatar/2fc23e2009234ad965f9d5d796400417"
+ subject="Bug raised"
+ date="2018-09-16T22:15:23Z"
+ content="""
+Hi Joey,
+
+I've raised a bug ( https://git-annex.branchable.com/bugs/Can__39__t_compile_on_FreeBSD__58____Issues_with_System.Posix.Files/ ) as requested.  I also happen to have a machine sitting around at home that I can configure for SSH access, if you or anyone would like a current (FreeBSD 11.1) dev environment to use.  Let me know.
+
+Yours,
+Duncan
+"""]]
diff --git a/doc/bugs/git-annex_can__39__t_find_gpg_if_it__39__s_named_gpg2.mdwn b/doc/bugs/git-annex_can__39__t_find_gpg_if_it__39__s_named_gpg2.mdwn
new file mode 100644
index 0000000000..9fd7fdc7c4
--- /dev/null
+++ b/doc/bugs/git-annex_can__39__t_find_gpg_if_it__39__s_named_gpg2.mdwn
@@ -0,0 +1,21 @@
+### Please describe the problem.
+
+git-annex requires a `gpg -> gpg2` alias, which is dangerous for other software to misuse.
+
+### What steps will reproduce the problem?
+
+Run any gpg-requiring operation on a machine that has only gpg2 installed and no gpg alias.
+
+    gpg: createProcess: runInteractiveProcess: exec: does not exist (No such file or directory)
+
+### What version of git-annex are you using? On what operating system?
+
+git-annex version: 5.20150710
+
+OS X, gpg2 installed with brew
+
+### Have you had any luck using git-annex before?
+
+git-annex took some time to get in the mentality and configure, but now it's a beautiful perfectly oiled file management system. Thanks!
+
+>  git.program support now implemented, [[done]] --[[Joey]]
diff --git a/doc/bugs/git-annex_can__39__t_find_gpg_if_it__39__s_named_gpg2/comment_1_b17661b0dbec3a72b2fd9608f0ba6823._comment b/doc/bugs/git-annex_can__39__t_find_gpg_if_it__39__s_named_gpg2/comment_1_b17661b0dbec3a72b2fd9608f0ba6823._comment
new file mode 100644
index 0000000000..b48cfbca3f
--- /dev/null
+++ b/doc/bugs/git-annex_can__39__t_find_gpg_if_it__39__s_named_gpg2/comment_1_b17661b0dbec3a72b2fd9608f0ba6823._comment
@@ -0,0 +1,23 @@
+[[!comment format=mdwn
+ username="joey"
+ subject="""comment 1"""
+ date="2015-09-09T21:12:01Z"
+ content="""
+git-annex should work ok with gpg version 2; there was one minor
+incompatibility vs gpg version 1, but it was ironed out in 2013.
+
+If you build it from source, and have only gpg2 in PATH, and not gpg, it
+will build a git-annex that runs gpg2.
+
+You're using OSX.. the git-annex.app for OSX bundles its own gpg command,
+and git-annex will use that one. I guess the brew build is built to use
+gpg, and not gpg2. Would it then make sense for the brew package of
+git-annex to depend on the package that contains gpg?
+
+I don't really think it makes sense for git-annex to probe
+around at runtime to find which of gpg and gpg2 is in PATH and pick which
+one to use.
+
+I suppose I could make git-annex look at git config gpg.program and use
+that program when it's set. This would mirror the behavior of git.
+"""]]
diff --git a/doc/bugs/git-annex_can_no_longer_copy_files_to_box.mdwn b/doc/bugs/git-annex_can_no_longer_copy_files_to_box.mdwn
new file mode 100644
index 0000000000..940ae3ad16
--- /dev/null
+++ b/doc/bugs/git-annex_can_no_longer_copy_files_to_box.mdwn
@@ -0,0 +1,47 @@
+### Please describe the problem.
+
+With the upgrade to git-annex 6.20170925 I can no longer copy files to box via webdav. I notice that the changelog suggests that there were many changes to the webdav backend, including a new path for temporary files and url-escaping of file names. I have been using webdav + Box successfully for almost two years. My box/webdav backed was set up with chunking and encryption:
+
+     WEBDAV_USERNAME=[username] WEBDAV_PASSWORD=[passwd] git annex initremote box type=webdav url=https://dav.box.com/dav/mystuff/annex chunk=100mb keyid=[keyid]
+
+Now when I try to add and copy a file to webdav I get the following error:
+
+     (checking box...)                         
+     DAV failure: Status {statusCode = 405, statusMessage = "Method Not Allowed"} "\n\n  Sabre_DAV_Exception_MethodNotAllowed\n  The resource you tried to create 
+     already exists\n\n"
+     CallStack (from HasCallStack):     
+     error, called at ./Remote/WebDAV.hs:381:78 in main:Remote.WebDAV failed            
+     (recording state in git...)      
+
+### What steps will reproduce the problem?
+
+Add a file to an annex repo with a webdav backend set up per the settings above:
+    
+     git add test.pdf
+
+Attempt to copy that file to the webdav backend:
+
+     git-annex copy -t box test.pdf
+
+### What version of git-annex are you using? On what operating system?
+
+OS = arch linux (Linux archbook 4.12.13-1-ARCH #1 SMP PREEMPT Fri Sep 15 06:36:43 UTC 2017 x86_64 GNU/Linux)
+
+git annex version =
+
+     git-annex version: 6.20170925-g76c9b580b
+     build flags: Assistant Webapp Pairing Testsuite S3(multipartupload)(storageclasses) WebDAV Inotify DBus DesktopNotify ConcurrentOutput TorrentParser MagicMime Feeds Quvi
+     dependency versions: aws-0.16 bloomfilter-2.0.1.0 cryptonite-0.24 DAV-1.3.1 feed-1.0.0.0 ghc-8.2.1 http-client-0.5.7.0 persistent-sqlite-2.6.2 torrent-10000.1.1 uuid-1.3.13 yesod-1.4.5
+     key/value backends: SHA256E SHA256 SHA512E SHA512 SHA224E SHA224 SHA384E SHA384 SHA3_256E SHA3_256 SHA3_512E SHA3_512 SHA3_224E SHA3_224 SHA3_384E SHA3_384 SKEIN256E SKEIN256 SKEIN512E SKEIN512 SHA1E SHA1 MD5E MD5 WORM URL
+     remote types: git gcrypt p2p S3 bup directory rsync web bittorrent webdav tahoe glacier ddar hook external
+     local repository version: unknown
+     supported repository versions: 3 5 6
+     upgrade supported from repository versions: 0 1 2 3 4 5
+     operating system: linux x86_64
+
+### Have you had any luck using git-annex before? (Sometimes we get tired of reading bug reports all day and a lil' positive end note does wonders)
+
+Yes! A huge word of thanks for this amazing software! Since I work on multiple linux boxes, git annex keeps track of all my big files and enables me to move them back and forth with minimal fuss.
+
+> [[done]]; seems this is fixed. --[[Joey]]
diff --git a/doc/bugs/git-annex_can_no_longer_copy_files_to_box/comment_10_dfe3426a2cb3622d405002e7275dbcd6._comment b/doc/bugs/git-annex_can_no_longer_copy_files_to_box/comment_10_dfe3426a2cb3622d405002e7275dbcd6._comment
new file mode 100644
index 0000000000..111bbf11c2
--- /dev/null
+++ b/doc/bugs/git-annex_can_no_longer_copy_files_to_box/comment_10_dfe3426a2cb3622d405002e7275dbcd6._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="madalu"
+ avatar="http://cdn.libravatar.org/avatar/c00d4aa29fd053f08a2ef35531592914"
+ subject="comment 10"
+ date="2017-10-09T15:46:09Z"
+ content="""
+I should add that the automated standalone build downloaded from the git-annex website reliably uploads to box, though slowly (per the other bug report). 
+"""]]
diff --git a/doc/bugs/git-annex_can_no_longer_copy_files_to_box/comment_11_c5b5e449eddd6fa7111a31041699b657._comment b/doc/bugs/git-annex_can_no_longer_copy_files_to_box/comment_11_c5b5e449eddd6fa7111a31041699b657._comment
new file mode 100644
index 0000000000..45d81bdae7
--- /dev/null
+++ b/doc/bugs/git-annex_can_no_longer_copy_files_to_box/comment_11_c5b5e449eddd6fa7111a31041699b657._comment
@@ -0,0 +1,16 @@
+[[!comment format=mdwn
+ username="joey"
+ subject="""comment 11"""
+ date="2017-10-11T15:10:51Z"
+ content="""
+Looks related to the other bug report, it's failing creating the
+directory for the temp file at the top of the repository, which
+already exists.
+
+So, the fix I just committed to fix the other bug report may have
+fixed this too.. Can you test?
+
+(It's also possible that the nonsensicalness of "MKCOL ."
+means it's not widely supported, or may even be out of the webdav spec
+entirely. The fix for the other bug report avoids that behavior.)
+"""]]
diff --git a/doc/bugs/git-annex_can_no_longer_copy_files_to_box/comment_12_9f50a33f47e73283c93e37fe85dd788e._comment b/doc/bugs/git-annex_can_no_longer_copy_files_to_box/comment_12_9f50a33f47e73283c93e37fe85dd788e._comment
new file mode 100644
index 0000000000..987bcc3587
--- /dev/null
+++ b/doc/bugs/git-annex_can_no_longer_copy_files_to_box/comment_12_9f50a33f47e73283c93e37fe85dd788e._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="madalu"
+ avatar="http://cdn.libravatar.org/avatar/c00d4aa29fd053f08a2ef35531592914"
+ subject="comment 12"
+ date="2017-10-12T00:58:04Z"
+ content="""
+I rebuilt using stack and everything works as expected now. Thanks!
+"""]]
diff --git a/doc/bugs/git-annex_can_no_longer_copy_files_to_box/comment_1_8d075cae09f3f4c4ef3f30b68099b39c._comment b/doc/bugs/git-annex_can_no_longer_copy_files_to_box/comment_1_8d075cae09f3f4c4ef3f30b68099b39c._comment
new file mode 100644
index 0000000000..ef61529f79
--- /dev/null
+++ b/doc/bugs/git-annex_can_no_longer_copy_files_to_box/comment_1_8d075cae09f3f4c4ef3f30b68099b39c._comment
@@ -0,0 +1,15 @@
+[[!comment format=mdwn
+ username="joey"
+ subject="""comment 1"""
+ date="2017-09-28T15:39:45Z"
+ content="""
+I'm not reproducing the problem here.
+
+The only change recent that affected storing files in any
+way was the temporary location change. But I don't see how that
+would lead to such a failure. 
+
+I've improved the webdav error message to include information about the
+request that failed. That should help with debugging the problem. An
+updated autobuild will be available within an hour or two.
+"""]]
diff --git a/doc/bugs/git-annex_can_no_longer_copy_files_to_box/comment_2_4471a1be63212346253729238a7da470._comment b/doc/bugs/git-annex_can_no_longer_copy_files_to_box/comment_2_4471a1be63212346253729238a7da470._comment
new file mode 100644
index 0000000000..e903f51ad1
--- /dev/null
+++ b/doc/bugs/git-annex_can_no_longer_copy_files_to_box/comment_2_4471a1be63212346253729238a7da470._comment
@@ -0,0 +1,10 @@
+[[!comment format=mdwn
+ username="madalu"
+ avatar="http://cdn.libravatar.org/avatar/c00d4aa29fd053f08a2ef35531592914"
+ subject="comment 2"
+ date="2017-09-29T13:00:40Z"
+ content="""
+Thank you for testing. The standalone works fine. This must be a problem with the way archlinux builds git-annex and/or its dependencies (e.g., haskell-dav). 
+
+Sorry for the false alarm. I will be sure to test with the standalone before reporting bugs next time.
+"""]]
diff --git a/doc/bugs/git-annex_can_no_longer_copy_files_to_box/comment_2_dc4345abb59fbfaf4982a88753286fe2._comment b/doc/bugs/git-annex_can_no_longer_copy_files_to_box/comment_2_dc4345abb59fbfaf4982a88753286fe2._comment
new file mode 100644
index 0000000000..e254ce7fcc
--- /dev/null
+++ b/doc/bugs/git-annex_can_no_longer_copy_files_to_box/comment_2_dc4345abb59fbfaf4982a88753286fe2._comment
@@ -0,0 +1,15 @@
+[[!comment format=mdwn
+ username="joey"
+ subject="""comment 2"""
+ date="2017-09-29T16:17:15Z"
+ content="""
+Hmm, the arch version info you posted originally shows the same version
+of the DAV library as I have here, so I had assumed it was not a
+version problem. May be something else in the dependency tree below
+DAV though.
+
+Are you sure it's not an intermittent problem; does it consistently fail
+with the arch build and work with the standalone build?
+
+Also, which arch build of git-annex are you using?
+"""]]
diff --git a/doc/bugs/git-annex_can_no_longer_copy_files_to_box/comment_4_1eec270febdd1a9fa28ac437c089c5e6._comment b/doc/bugs/git-annex_can_no_longer_copy_files_to_box/comment_4_1eec270febdd1a9fa28ac437c089c5e6._comment
new file mode 100644
index 0000000000..045234442a
--- /dev/null
+++ b/doc/bugs/git-annex_can_no_longer_copy_files_to_box/comment_4_1eec270febdd1a9fa28ac437c089c5e6._comment
@@ -0,0 +1,12 @@
+[[!comment format=mdwn
+ username="madalu"
+ avatar="http://cdn.libravatar.org/avatar/c00d4aa29fd053f08a2ef35531592914"
+ subject="comment 4"
+ date="2017-09-29T17:22:43Z"
+ content="""
+Yes, copying to box consistently fails with the arch linux build (6.20170925-g76c9b580b) and consistently succeeds with the standalone build (6.20170929-gffc127582).
+
+I am using the git-annex build in the official arch repositories (community). I've submitted an arch linux bug report here:
+
+https://bugs.archlinux.org/task/55808?project=5&cat%5B0%5D=33&string=git-annex
+"""]]
diff --git a/doc/bugs/git-annex_can_no_longer_copy_files_to_box/comment_5_c758613d1d407e8ff1565822a9f36951._comment b/doc/bugs/git-annex_can_no_longer_copy_files_to_box/comment_5_c758613d1d407e8ff1565822a9f36951._comment
new file mode 100644
index 0000000000..e8b556edff
--- /dev/null
+++ b/doc/bugs/git-annex_can_no_longer_copy_files_to_box/comment_5_c758613d1d407e8ff1565822a9f36951._comment
@@ -0,0 +1,10 @@
+[[!comment format=mdwn
+ username="joey"
+ subject="""comment 5"""
+ date="2017-09-29T19:41:08Z"
+ content="""
+https://git.archlinux.org/svntogit/community.git/commit/trunk?h=packages/haskell-dav&id=3053316bdef797ade4abd9e94c76258468deb9bc
+
+Could be the new version of aeson that broke it. The autobuilder is using
+aeson-1.1.2.0.
+"""]]
diff --git a/doc/bugs/git-annex_can_no_longer_copy_files_to_box/comment_6_11a5b8e90d7c5f3ec4bfe854b4778fea._comment b/doc/bugs/git-annex_can_no_longer_copy_files_to_box/comment_6_11a5b8e90d7c5f3ec4bfe854b4778fea._comment
new file mode 100644
index 0000000000..476350de57
--- /dev/null
+++ b/doc/bugs/git-annex_can_no_longer_copy_files_to_box/comment_6_11a5b8e90d7c5f3ec4bfe854b4778fea._comment
@@ -0,0 +1,47 @@
+[[!comment format=mdwn
+ username="madalu"
+ avatar="http://cdn.libravatar.org/avatar/c00d4aa29fd053f08a2ef35531592914"
+ subject="comment 6"
+ date="2017-10-06T23:45:13Z"
+ content="""
+I had a chance to do some more testing. I build the latest git-annex using stack (after uninstalling the arch-linux git annex and all its dependencies). I was able to reproduce the bug with the build.
+
+git-annex version:
+
+     git-annex version: 6.20171006-g676000770
+     build flags: Assistant Webapp Pairing Testsuite S3(multipartupload)(storageclasses) WebDAV Inotify ConcurrentOutput TorrentParser Feeds Quvi
+     dependency versions: aws-0.16 bloomfilter-2.0.1.0 cryptonite-0.21 DAV-1.3.1 feed-0.3.12.0 ghc-8.0.2 http-client-0.5.6.1 persistent-sqlite-2.6.2 torrent-10000.1.1 uuid-1.3.13 yesod-1.4.5
+     key/value backends: SHA256E SHA256 SHA512E SHA512 SHA224E SHA224 SHA384E SHA384 SHA3_256E SHA3_256 SHA3_512E SHA3_512 SHA3_224E SHA3_224 SHA3_384E SHA3_384 SKEIN256E SKEIN256 SKEIN512E SKEIN512 SHA1E SHA1 MD5E MD5 WORM URL
+     remote types: git gcrypt p2p S3 bup directory rsync web bittorrent webdav tahoe glacier ddar hook external
+
+Steps to reproduce (assumes the webdav repo initiation from the original bug report):
+
+Create a file test.txt that contains \"Hello, world!\".
+
+     git add test.txt
+     git annex --verbose --debug copy -t box test.txt
+
+This results in:
+
+     [2017-10-06 18:32:00.036289708] read: git [\"--git-dir=.git\",\"--work-tree=.\",\"--literal-pathspecs\",\"show-ref\",\"git-annex\"]
+     [2017-10-06 18:32:00.039256791] process done ExitSuccess
+     [2017-10-06 18:32:00.039344582] read: git [\"--git-dir=.git\",\"--work-tree=.\",\"--literal-pathspecs\",\"show-ref\",\"--hash\",\"refs/heads/git-annex\"]
+     [2017-10-06 18:32:00.041786875] process done ExitSuccess
+     [2017-10-06 18:32:00.042888138] chat: git [\"--git-dir=.git\",\"--work-tree=.\",\"--literal-pathspecs\",\"cat-file\",\"--batch\"]
+     [2017-10-06 18:32:00.043236927] chat: git [\"--git-dir=.git\",\"--work-tree=.\",\"--literal-pathspecs\",\"cat-file\",\"--batch-check=%(objectname) %(objecttype) %(objectsize)\"]
+     [2017-10-06 18:32:00.064572923] read: git [\"--git-dir=.git\",\"--work-tree=.\",\"--literal-pathspecs\",\"ls-files\",\"--cached\",\"-z\",\"--\",\"test.txt\"]
+     copy test.txt [2017-10-06 18:32:00.078377684] chat: gpg [\"--quiet\",\"--trust-model\",\"always\",\"--decrypt\"]
+     [2017-10-06 18:32:00.16669294] process done ExitSuccess
+     (checking box...) (to box...)
+     [2017-10-06 18:32:01.195900381] chat: gpg  [\"--quiet\",\"--trust-model\",\"always\",\"--batch\",\"--passphrase-fd\",\"19\",\"--symmetric\",\"--force-mdc\",\"--no-textmode\"]
+     [2017-10-06 18:32:32.159837783] process done ExitSuccess
+
+       DAV failure: Status {statusCode = 405, statusMessage = \"Method Not Allowed\"} \"\n\n  Sabre_DAV_Exception_MethodNotAllowed\n  The resource you tried to create already exists\n\n\" HTTP request: \"MKCOL\" \"/dav/mystuff/annex/.\"
+     failed
+     [2017-10-06 18:32:32.163143388] process done ExitSuccess
+     [2017-10-06 18:32:32.163958789] process done ExitSuccess
+     git-annex: copy: 1 failed
+
+
+
+"""]]
diff --git a/doc/bugs/git-annex_can_no_longer_copy_files_to_box/comment_7_654cda20c0775e16c14ae8b1134aa042._comment b/doc/bugs/git-annex_can_no_longer_copy_files_to_box/comment_7_654cda20c0775e16c14ae8b1134aa042._comment
new file mode 100644
index 0000000000..eeac2f5495
--- /dev/null
+++ b/doc/bugs/git-annex_can_no_longer_copy_files_to_box/comment_7_654cda20c0775e16c14ae8b1134aa042._comment
@@ -0,0 +1,20 @@
+[[!comment format=mdwn
+ username="joey"
+ subject="""comment 7"""
+ date="2017-10-07T18:08:49Z"
+ content="""
+It's interesting you reproduced it when building with stack. I'm a bit
+confused because in your other bug report, you seemed to have git-annex
+built with stack working without this bug?
+
+In any case, IIRC stack will use haskell libraries installed system-wide in
+some cases, so it may be picking up whatever the problimatic library is
+from Arch Linux.
+
+If you can reproduce it with stack on a system that does not have a
+system-wide ghc installed, I'd think I should also be able to build with
+stack and reproduce it..
+
+Also, I've just made --debug log all webdav operations, which should help
+track down what operation is failing..
+"""]]
diff --git a/doc/bugs/git-annex_can_no_longer_copy_files_to_box/comment_8_81ef2faa317c1f5d4d6d051baa48ceff._comment b/doc/bugs/git-annex_can_no_longer_copy_files_to_box/comment_8_81ef2faa317c1f5d4d6d051baa48ceff._comment
new file mode 100644
index 0000000000..3d0d3b6577
--- /dev/null
+++ b/doc/bugs/git-annex_can_no_longer_copy_files_to_box/comment_8_81ef2faa317c1f5d4d6d051baa48ceff._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="madalu"
+ avatar="http://cdn.libravatar.org/avatar/c00d4aa29fd053f08a2ef35531592914"
+ subject="comment 8"
+ date="2017-10-09T14:52:54Z"
+ content="""
+Yes, I am confused as well. Sometimes the stack build works, sometimes it doesn't. So I can't reliably reproduce the bug. You can see an example of a failure (with debug information) on the other bug report. I'll see if I can get stack without a system wide install and rebuild.
+"""]]
diff --git a/doc/bugs/git-annex_can_no_longer_copy_files_to_box/comment_9_999697b83fa2451b1cf28110b27f9a1a._comment b/doc/bugs/git-annex_can_no_longer_copy_files_to_box/comment_9_999697b83fa2451b1cf28110b27f9a1a._comment
new file mode 100644
index 0000000000..4cdf3b1b84
--- /dev/null
+++ b/doc/bugs/git-annex_can_no_longer_copy_files_to_box/comment_9_999697b83fa2451b1cf28110b27f9a1a._comment
@@ -0,0 +1,48 @@
+[[!comment format=mdwn
+ username="madalu"
+ avatar="http://cdn.libravatar.org/avatar/c00d4aa29fd053f08a2ef35531592914"
+ subject="Reproduction of bug with "pure" stack build"
+ date="2017-10-09T15:43:16Z"
+ content="""
+I am able to reproduce this bug with an entirely local stack build (i.e., no global haskell installs) on arch linux. To make sure I began with a clean build, I removed local directories (rm -r ~/.ghc, rm -r ~/.stack, rm -r ~/.cabal) and then uninstalled all arch linux haskell packages (including ghc and ghc-libs). I downloaded the standalone stack and then ran stack clean, stack setup, stack install in a cloned git-annex repo.
+
+git-annex-version
+                             
+     git-annex version: 6.20171009-g92577980f                      
+     build flags: Assistant Webapp Pairing Testsuite S3(multipartupload)(storageclasses) WebDAV Inotify ConcurrentOutput TorrentParser Feeds Quvi
+     dependency versions: aws-0.16 bloomfilter-2.0.1.0 cryptonite-0.21 DAV-1.3.1 feed-0.3.12.0 ghc-8.0.2 http-client-0.5.6.1 persistent-sqlite-2.6.2 torrent-10000.1.1 uuid-1.3.13 yesod-1.4.5
+     key/value backends: SHA256E SHA256 SHA512E SHA512 SHA224E SHA224 SHA384E SHA384 SHA3_256E SHA3_256 SHA3_512E SHA3_512 SHA3_224E SHA3_224 SHA3_384E SHA3_384 SKEIN256E SKEIN256 SKEIN512E SKEIN512 SHA1E SHA1 MD5E MD5 WORM URL
+     remote types: git gcrypt p2p S3 bup directory rsync web bittorrent webdav tahoe glacier ddar hook external
+     local repository version: 5
+     supported repository versions: 3 5 6
+     upgrade supported from repository versions: 0 1 2 3 4 5       
+     operating system: linux x86_64 
+
+Here is the debug information:
+
+git annex --verbose --debug copy -t box test.txt
+
+     [2017-10-09 10:37:02.174210625] read: git [\"--git-dir=.git\",\"--work-tree=.\",\"--literal-pathspecs\",\"show-ref\",\"git-annex\"]
+     [2017-10-09 10:37:02.1773328] process done ExitSuccess
+     [2017-10-09 10:37:02.177437562] read: git [\"--git-dir=.git\",\"--work-tree=.\",\"--literal-pathspecs\",\"show-ref\",\"--hash\",\"refs/heads/git-annex\"]
+     [2017-10-09 10:37:02.179955262] process done ExitSuccess
+     [2017-10-09 10:37:02.18033415] read: git [\"--git-dir=.git\",\"--work-tree=.\",\"--literal-pathspecs\",\"log\",\"refs/heads/git-annex..4c46223e7ccc1c3569fc43e3dbfe1c89fdf9a628\",\"--pretty=%H\",\"-n1\"]
+     [2017-10-09 10:37:02.18512832] process done ExitSuccess
+     [2017-10-09 10:37:02.185946927] chat: git [\"--git-dir=.git\",\"--work-tree=.\",\"--literal-pathspecs\",\"cat-file\",\"--batch\"]
+     [2017-10-09 10:37:02.186427574] chat: git [\"--git-dir=.git\",\"--work-tree=.\",\"--literal-pathspecs\",\"cat-file\",\"--batch-check=%(objectname) %(objecttype) %(objectsize)\"]
+     [2017-10-09 10:37:02.209479037] read: git [\"--git-dir=.git\",\"--work-tree=.\",\"--literal-pathspecs\",\"ls-files\",\"--cached\",\"-z\",\"--\",\"test.txt\"] copy test.txt 
+     [2017-10-09 10:37:02.222741576] chat: gpg [\"--quiet\",\"--trust-model\",\"always\",\"--decrypt\"]
+     [2017-10-09 10:37:02.31191458] process done ExitSuccess
+     (checking box...) [2017-10-09 10:37:02.355350066] getProps 90a/d4d/GPGHMACSHA1--53636e5e7a50bc58eae478ddc260bb5abd899d03/GPGHMACSHA1--53636e5e7a50bc58eae478ddc260bb5abd899d03
+     (to box...)
+     [2017-10-09 10:37:03.30749556] chat: gpg [\"--quiet\",\"--trust-model\",\"always\",\"--batch\",\"--passphrase-fd\",\"19\",\"--symmetric\",\"--force-mdc\",\"--no-textmode\"]
+     [2017-10-09 10:37:03.361698513] getProps .
+     [2017-10-09 10:37:33.376085151] mkCol .
+     [2017-10-09 10:37:34.066994173] process done ExitSuccess
+
+     DAV failure: Status {statusCode = 405, statusMessage = \"Method Not Allowed\"} \"\n\n  Sabre_DAV_Exception_MethodNotAllowed\n  The resource you tried to create already exists\n\n\" HTTP request: \"MKCOL\" \"/dav/mystuff/annex/.\" failed                                                        
+     [2017-10-09 10:37:34.069044148] process done ExitSuccess      
+     [2017-10-09 10:37:34.069370655] process done ExitSuccess      
+     git-annex: copy: 1 failed    
+
+"""]]
diff --git a/doc/bugs/git-annex_cannot_connect_to_freenet_cloud___40__webdav__41__.mdwn b/doc/bugs/git-annex_cannot_connect_to_freenet_cloud___40__webdav__41__.mdwn
new file mode 100644
index 0000000000..bc7c3e4292
--- /dev/null
+++ b/doc/bugs/git-annex_cannot_connect_to_freenet_cloud___40__webdav__41__.mdwn
@@ -0,0 +1,36 @@
+### Please describe the problem.
+
+git-annex's WebDAV support does not like (aka it does not work) the WebDAV server of the freenet cloud.
+    
+### What steps will reproduce the problem?
+
+My first attempt was:
+
+    WEBDAV_USERNAME='XXX' WEBDAV_PASSWORD='XXX' git annex initremote webdav type=webdav url='https://webmail.freenet.de/webdav' encryption=none
+    initremote webdav (testing WebDAV server...) 
+    git-annex: WebDAV test failed: StatusCodeException (Status {statusCode = 401, statusMessage = "Unauthorized"}) [("Date","Fri, 04 Dec 2015 12:20:57 GMT"),("Server","Apache/2.2.16 (Debian)"),("WWW-Authenticate","Basic realm=\"MD-Cloud\""),
+    ("Vary","Accept-Encoding"),("Content-Encoding","gzip"),("Content-Length","20"),("Connection","close"),("Content-Type","text/html; charset=iso-8859-15"),("X-Response-Body-Start",""),("X-Request-URL","MKCOL https://webmail.freenet.de:443/webdav/tmp")]
+    (CJ {expose = []}): user error failed
+    git-annex: initremote: 1 failed
+
+Ok this fails (what is the error?). However, it does create a folder "tmp" in the "cloud". A second attempt yields another error:
+
+    WEBDAV_USERNAME='XXX' WEBDAV_PASSWORD='XXX' git annex initremote webdav type=webdav url='https://webmail.freenet.de/webdav' encryption=none
+    initremote webdav (testing WebDAV server...) 
+    git-annex: WebDAV test failed: StatusCodeException (Status {statusCode = 501, statusMessage = "Not Implemented"}) [("Date","Fri, 04 Dec 2015 12:21:22 GMT"),("Server","Apache/2.2.16 (Debian)"),("Content-Length","349"),("Connection","close"),
+    ("Content-Type","application/xml; charset=utf-8"),("X-Response-Body-Start","\n\n  Sabre\\DAV\\Exception\\NotImplemented\n  
+    This server is not compatible with OS/X finder. Consider using a different WebDAV client or webserver.\n  1.8.6\n\n"),("X-Request-URL","PUT https://webmail.freenet.de:443/webdav
+    /tmp/git-annex-test")] (CJ {expose = []}): user error failed
+    git-annex: initremote: 1 failed
+
+which is I guess the same. The WebDAV server does support writing and locking files. I tried writing using davfs2 and locking-unlocking using cadaver. I guess in the end that it's the server's fault, but it would be great to know what exactly fails at this point :).
+
+### What version of git-annex are you using? On what operating system?
+
+git-annex version: 5.20151116-gbe86081
+
+Gentoo Linux
+
+### Have you had any luck using git-annex before? (Sometimes we get tired of reading bug reports all day and a lil' positive end note does wonders)
+
+First time trying WebDAV support.
diff --git a/doc/bugs/git-annex_cannot_connect_to_freenet_cloud___40__webdav__41__/comment_1_db7b584c1e932d24e0dcf464b023a659._comment b/doc/bugs/git-annex_cannot_connect_to_freenet_cloud___40__webdav__41__/comment_1_db7b584c1e932d24e0dcf464b023a659._comment
new file mode 100644
index 0000000000..a4028e4823
--- /dev/null
+++ b/doc/bugs/git-annex_cannot_connect_to_freenet_cloud___40__webdav__41__/comment_1_db7b584c1e932d24e0dcf464b023a659._comment
@@ -0,0 +1,41 @@
+[[!comment format=mdwn
+ username="sts"
+ subject="comment 1"
+ date="2015-12-07T08:12:26Z"
+ content="""
+ok, I could find the source of the problem: they use sabredav as WebDAV server and sabredav does not support chunked transfers:
+
+        // Intercepting the Finder problem
+        if (($expected = $request->getHeader('X-Expected-Entity-Length')) && $expected > 0) {
+
+            /*
+            Many webservers will not cooperate well with Finder PUT requests,
+            because it uses 'Chunked' transfer encoding for the request body.
+
+            The symptom of this problem is that Finder sends files to the
+            server, but they arrive as 0-length files in PHP.
+
+            If we don't do anything, the user might think they are uploading
+            files successfully, but they end up empty on the server. Instead,
+            we throw back an error if we detect this.
+
+            The reason Finder uses Chunked, is because it thinks the files
+            might change as it's being uploaded, and therefore the
+            Content-Length can vary.
+
+            Instead it sends the X-Expected-Entity-Length header with the size
+            of the file at the very start of the request. If this header is set,
+            but we don't get a request body we will fail the request to
+            protect the end-user.
+            */
+
+            // Only reading first byte
+            $firstByte = fread($body, 1);
+            if (strlen($firstByte) !== 1) {
+                throw new Exception\Forbidden('This server is not compatible with OS/X finder. Consider using a different WebDAV client or webserver.');
+            }
+          
+            ...
+
+Although, I did not told git-annex to chunk the transfer :-/, because I did not append a 'chunk' parameter. Any ideas how to fix that?
+"""]]
diff --git a/doc/bugs/git-annex_cannot_connect_to_freenet_cloud___40__webdav__41__/comment_2_72e9ecc6acdc555e36f5588120d070fa._comment b/doc/bugs/git-annex_cannot_connect_to_freenet_cloud___40__webdav__41__/comment_2_72e9ecc6acdc555e36f5588120d070fa._comment
new file mode 100644
index 0000000000..8df1c736dd
--- /dev/null
+++ b/doc/bugs/git-annex_cannot_connect_to_freenet_cloud___40__webdav__41__/comment_2_72e9ecc6acdc555e36f5588120d070fa._comment
@@ -0,0 +1,39 @@
+[[!comment format=mdwn
+ username="joey"
+ subject="""comment 2"""
+ date="2015-12-11T15:08:30Z"
+ content="""
+The first failure is git-annex sending MKCOL (make directory basically).
+The server fails with "Unauthorized". You say it also made the directory.
+That's got to be a bug in the server, no? It can't sanely have an
+authorization problem and also go on and do the unathorized action.
+(Sounds rather like a security hole..)
+
+As to the PUT failure, the chunked transfer encoding mentioned in that
+comment is a regular part of the HTTP protocol (this is not connected
+to git-annex's own chunking).
+
+
+Looks like this PHP webdav server might be delegating the actual HTTP
+to whatever web server it's running on somehow. Since chunked transfer
+encoding might not be supported by some web server, they are left trying to
+detect that. I don't know if their check for that is accurate.
+
+As to the implementation in git-annex,
+Network.Http.Client.RequestBodyStreamChunked is documented to be the only
+thing that causes a chunked request body to be sent, and git-annex is using
+RequestBodyLBS instead. Unless the documentation is wrong (and I also 
+looked at the http-client source code and the documentation seems accurate),
+I am doubtful that the chunked transfer encoding is actually being used by
+git-annex. If eg a protocol dump shows that it is in fact using chunked
+transfer encoding (ie, contains "Transfer-Encoding: chunked"),
+that would be grounds to file a bug on the http-client library.
+
+Aah, but.. git-annex is sending an empty file. And the webdav server's
+check consists of reading 1 byte. 
+Of course there's not a byte to read if an empty file is being sent!
+So that code you showed is certianly buggy.
+
+I've changed git-annex to send a non-empty file when testing the webdav
+server to work around this.
+"""]]
diff --git a/doc/bugs/git-annex_cannot_connect_to_freenet_cloud___40__webdav__41__/comment_3_b2ef283a3933e22542e05ce9a1acef7d._comment b/doc/bugs/git-annex_cannot_connect_to_freenet_cloud___40__webdav__41__/comment_3_b2ef283a3933e22542e05ce9a1acef7d._comment
new file mode 100644
index 0000000000..3403a59fc8
--- /dev/null
+++ b/doc/bugs/git-annex_cannot_connect_to_freenet_cloud___40__webdav__41__/comment_3_b2ef283a3933e22542e05ce9a1acef7d._comment
@@ -0,0 +1,38 @@
+[[!comment format=mdwn
+ username="sts"
+ subject="comment 3"
+ date="2015-12-20T19:49:26Z"
+ content="""
+hey joey!
+
+thanks, it (kinda) worked :). At least the initialization works now pretty fine. But uploading files does not work as expected. It does upload  some data, but does not move the data to the corresponding folder (from 'tmp' to $hash):
+
+    git annex copy --debug --to webdav 1354580391258.jpg
+    [2015-12-20 20:31:36.985579] read: git [\"--git-dir=.git\",\"--work-tree=.\",\"--literal-pathspecs\",\"show-ref\",\"git-annex\"]
+    [2015-12-20 20:31:36.990098] process done ExitSuccess
+    [2015-12-20 20:31:36.990584] read: git [\"--git-dir=.git\",\"--work-tree=.\",\"--literal-pathspecs\",\"show-ref\",\"--hash\",\"refs/heads/git-annex\"]
+    [2015-12-20 20:31:36.993635] process done ExitSuccess
+    [2015-12-20 20:31:36.994311] read: git [\"--git-dir=.git\",\"--work-tree=.\",\"--literal-pathspecs\",\"log\",\"refs/heads/git-annex..2ff0c487bc1c92c11a45131b55f6fb2ba034071d\",\"-n1\",\"--pretty=%H\"]
+    [2015-12-20 20:31:36.998675] process done ExitSuccess
+    [2015-12-20 20:31:36.999747] chat: git [\"--git-dir=.git\",\"--work-tree=.\",\"--literal-pathspecs\",\"cat-file\",\"--batch\"]
+    [2015-12-20 20:31:37.003483] read: git [\"--git-dir=.git\",\"--work-tree=.\",\"--literal-pathspecs\",\"ls-files\",\"--cached\",\"-z\",\"--\",\"1354580391258.jpg\"]
+    copy 1354580391258.jpg (checking webdav...) (to webdav...) 
+    100%        564.6KB/s 0s
+    DAV failure: 409 \"Conflict\"
+    failed
+    git-annex: copy: 1 failed
+
+In the 'tmp'-folder on the server I can find the file ('SHA256E-s1156230--...'), but this file is not 100% identical to the one I uploaded. Well, it seems that the first byte is missing ;) at least if I compare the files via vimdiff:
+
+    0000000: d8ff e000 104a 4649 4600 0101 0100 4800  .....JFIF.....H.               |  0000000: ffd8 ffe0 0010 4a46 4946 0001 0101 0048  ......JFIF.....H               
+    0000010: 4800 00ff db00 4300 0101 0101 0101 0101  H.....C.........               |  0000010: 0048 0000 ffdb 0043 0001 0101 0101 0101  .H.....C........               
+    0000020: 0101 0101 0102 0203 0202 0202 0204 0303  ................               |  0000020: 0101 0101 0101 0202 0302 0202 0202 0403  ................               
+    0000030: 0203 0504 0505 0504 0404 0506 0706 0505  ................               |  0000030: 0302 0305 0405 0505 0404 0405 0607 0605  ................               
+    0000040: 0706 0404 0609 0607 0808 0808 0805 0609  ................               |  0000040: 0507 0604 0406 0906 0708 0808 0808 0506  ................               
+    0000050: 0a09 080a 0708 0808 ffdb 0043 0101 0101  ...........C....               |  0000050: 090a 0908 0a07 0808 08ff db00 4301 0101  ............C...               
+    0000060: 0202 0204 0202 0408 0504 0508 0808 0808  ................               |  0000060: 0102 0202 0402 0204 0805 0405 0808 0808  ................
+
+As you can see, on the left side (aka 'the uploaded file') the first two hexadecimal values are missing in comparison to the right side (the source), so the first byte is missing.
+
+I guess in the end it is still the same problem, right :)?
+"""]]
diff --git a/doc/bugs/git-annex_cannot_connect_to_freenet_cloud___40__webdav__41__/comment_4_7a139eb2c11ab2e32d22ca0f8bb648a5._comment b/doc/bugs/git-annex_cannot_connect_to_freenet_cloud___40__webdav__41__/comment_4_7a139eb2c11ab2e32d22ca0f8bb648a5._comment
new file mode 100644
index 0000000000..d12f6cbc52
--- /dev/null
+++ b/doc/bugs/git-annex_cannot_connect_to_freenet_cloud___40__webdav__41__/comment_4_7a139eb2c11ab2e32d22ca0f8bb648a5._comment
@@ -0,0 +1,17 @@
+[[!comment format=mdwn
+ username="joey"
+ subject="""comment 4"""
+ date="2016-01-20T19:22:36Z"
+ content="""
+As to where the first byte went, it seems rather suggestive that the part
+of the server's code you pasted earlier reads out `$firstByte`. Perhaps it
+forgot to preserve that first byte in the file.
+
+Which is horrible, but then there's the "Conflict" error. All I can make of
+this is that the server is probably rejecting the MOVE request for some reason.
+That might be reasonable if the destination file already existed, 
+but I don't think it does in this case, so who knows.
+
+Only thing I'm sure of is, I never want to entrust this particular webdav
+server with any of my data!
+"""]]
diff --git a/doc/bugs/git-annex_confuses_Git_with_nested_submodules.mdwn b/doc/bugs/git-annex_confuses_Git_with_nested_submodules.mdwn
new file mode 100644
index 0000000000..7bc6d74439
--- /dev/null
+++ b/doc/bugs/git-annex_confuses_Git_with_nested_submodules.mdwn
@@ -0,0 +1,38 @@
+### Please describe the problem.
+The way git-annex deals with submodules (replacing the .git file in the submodule, with a link to the corresponding gitdir of the submodule) seems to confuse Git when creating another submodule in an annex-init'ed submodule.
+
+### What steps will reproduce the problem?
+    % mkdir some ; cd some; git init
+    Initialized empty Git repository in /tmp/some/.git/
+    % git submodule add /src/somegitrepo sub_lvl1
+    Cloning into 'sub_lvl1'...
+    done.
+    % cd sub_lvl1
+    % git annex init
+    init  (merging origin/git-annex into git-annex...)
+    (recording state in git...)
+    ok
+    (recording state in git...)
+    % git submodule add /src/somegitrepo sub_lvl2
+    Cloning into 'sub_lvl2'...
+    done.
+    fatal: Could not chdir to '../../../sub_lvl2': No such file or directory
+    Unable to checkout submodule 'sub_lvl2'
+
+### What version of git-annex are you using? On what operating system?
+    % apt-cache policy git-annex-standalone
+    git-annex-standalone:
+      Installed: 6.20160213+gitg9597a21-1~ndall+1
+
+Debian stretch, git-annex from NeuroDebian.
+
+### Have you had any luck using git-annex before? (Sometimes we get tired of reading bug reports all day and a lil' positive end note does wonders)
+
+Yes, lots! Using it for some of its original use cases for more than five years now -- I was actually surprised to learn, just now, that the oldest commit in my music repository is exactly 5 years and 6 days old. Thanks for longevity and reliability!
+
+More recently I aim exploring the use of git annex for managing datasets and their dependencies, i.e. going from raw to some processed state over multiple levels, where each level is a useful starting point for some analysis, and each previous level is a dependency (input) to the next. With just one level above "raw" this has massively improved collaboration workflows in student/teacher settings for me. Deeper nesting levels would allow for even more interesting applications, but see above ;-) I think Git seems needlessly confused, but I don't fully grasp what is happening yet. I'd appreciate any insight you may have. Although it is Git that shows the undesired behavior, it seems it is git-annex that ultimately confused it. Hence I came here first.
+
+BTW: What a nice idea to ask for something like this in a bug report.
+
+
+[[!meta author=mih]]
diff --git a/doc/bugs/git-annex_confuses_Git_with_nested_submodules/comment_1_fb01d4b5af500affc08a5c3b3b1849dd._comment b/doc/bugs/git-annex_confuses_Git_with_nested_submodules/comment_1_fb01d4b5af500affc08a5c3b3b1849dd._comment
new file mode 100644
index 0000000000..41068b5a9b
--- /dev/null
+++ b/doc/bugs/git-annex_confuses_Git_with_nested_submodules/comment_1_fb01d4b5af500affc08a5c3b3b1849dd._comment
@@ -0,0 +1,16 @@
+[[!comment format=mdwn
+ username="joey"
+ subject="""comment 1"""
+ date="2016-03-01T20:25:13Z"
+ content="""
+Reproduced this.
+
+This really does feel like a git bug. git is supposed to treat "gitlink"
+files and .git symlinks the same. While modern versions of git set up
+gitlink files for submodules, older versions of git used .git symlinks, and
+git should still support that.
+
+Looks like the problem can be worked around, by setting
+`GIT_DIR`. In your example, `GIT_DIR=../.git/modules/sub_lvl1/ git
+submodule add /src/somegitrepo sub_lvl2`
+"""]]
diff --git a/doc/bugs/git-annex_confuses_Git_with_nested_submodules/comment_2_094baf6c3738691879fd907dd1729c56._comment b/doc/bugs/git-annex_confuses_Git_with_nested_submodules/comment_2_094baf6c3738691879fd907dd1729c56._comment
new file mode 100644
index 0000000000..50d2f709a7
--- /dev/null
+++ b/doc/bugs/git-annex_confuses_Git_with_nested_submodules/comment_2_094baf6c3738691879fd907dd1729c56._comment
@@ -0,0 +1,17 @@
+[[!comment format=mdwn
+ username="joey"
+ subject="""comment 2"""
+ date="2016-03-01T20:36:43Z"
+ content="""
+Here's a more minimal test case, not involving git-annex at all:
+
+	git init gitdir
+	mkdir worktree
+	cd worktree
+	ln -s ../gitdir/.git .git
+	git submodule add /any/git/repo sub
+
+	fatal: Could not chdir to '../../../sub': No such file or directory
+
+I have forwarded that test case to the git ML.
+"""]]
diff --git a/doc/bugs/git-annex_confuses_Git_with_nested_submodules/comment_3_e1bc8eb7f6ce0d6f2d2f2b0ea6f20862._comment b/doc/bugs/git-annex_confuses_Git_with_nested_submodules/comment_3_e1bc8eb7f6ce0d6f2d2f2b0ea6f20862._comment
new file mode 100644
index 0000000000..49e883c7ca
--- /dev/null
+++ b/doc/bugs/git-annex_confuses_Git_with_nested_submodules/comment_3_e1bc8eb7f6ce0d6f2d2f2b0ea6f20862._comment
@@ -0,0 +1,29 @@
+[[!comment format=mdwn
+ username="joey"
+ subject="""comment 3"""
+ date="2016-03-02T16:48:24Z"
+ content="""
+[git bug report](http://news.gmane.org/find-root.php?message_id=20160301204218.GA4083%40kitenet.net)
+
+So far, the git devs admit this is a problem, but don't seem too keen on fixing
+it, even though it breaks backwards compatability with repositories git
+submodule add created (circa 2012).
+
+It might be that git-annex init could work around git's bugginess by,
+instead of making submodule/.git a symlink to ../.git/modules/dir, making
+submodule/.git be the git directory, and converting ../.git/modules/dir
+to a symlink. In very limited testing, that setup seems to work. 
+
+I don't know if all the submodule stuff would work, perhaps it would break moving
+submodules etc. And, since git likes to chdir around (not the best idea),
+if it expected to be able to chdir from .git/modules to dir and chdir .. to
+get back, changing that to a symlink would defeat it.
+
+BTW, I found another way, unrelated to git-annex or symlinks at all,
+that git submodule add's broken path handling makes it fall over with
+nested submodules.
+.
+
+(It's almost like myrepos was a better idea than this submodule stuff, or
+something...)
+""]]
diff --git a/doc/bugs/git-annex_confuses_Git_with_nested_submodules/comment_4_4bcd571dcd6c1e709e83e519135519b3._comment b/doc/bugs/git-annex_confuses_Git_with_nested_submodules/comment_4_4bcd571dcd6c1e709e83e519135519b3._comment
new file mode 100644
index 0000000000..d663f33700
--- /dev/null
+++ b/doc/bugs/git-annex_confuses_Git_with_nested_submodules/comment_4_4bcd571dcd6c1e709e83e519135519b3._comment
@@ -0,0 +1,9 @@
+[[!comment format=mdwn
+ username="mih"
+ subject="Thanks"
+ date="2016-03-02T19:30:49Z"
+ content="""
+Thanks for investigating this further.
+
+One aspect that may make switching the location of the .git directory into the worktree of the submodule less desirable is this: With the actual .git in ../.git/modules/... one can easily rm -rf the submodule, deinit it, and re-init/update from the (still present) ../.git/modules/... at a later point in time. Especially, when a submodule is a more complicated beast (e.g. with multiple configured remotes) the required steps to regenerate the same setup get more complex.
+"""]]
diff --git a/doc/bugs/git-annex_confuses_Git_with_nested_submodules/comment_5_c35478d4fbe6dde21d22369494414fe8._comment b/doc/bugs/git-annex_confuses_Git_with_nested_submodules/comment_5_c35478d4fbe6dde21d22369494414fe8._comment
new file mode 100644
index 0000000000..47cf7624dc
--- /dev/null
+++ b/doc/bugs/git-annex_confuses_Git_with_nested_submodules/comment_5_c35478d4fbe6dde21d22369494414fe8._comment
@@ -0,0 +1,7 @@
+[[!comment format=mdwn
+ username="https://me.yahoo.com/a/EbvxpTI_xP9Aod7Mg4cwGhgjrCrdM5s-#7c0f4"
+ subject="comment 5"
+ date="2016-04-19T14:47:58Z"
+ content="""
+Hi Joey,  since git people agreed that it was a regression, should we expect this issue to be fixed on their side?  Could you please follow up since discussion on the mailing list you have referenced has ended without closure
+"""]]
diff --git a/doc/bugs/git-annex_confuses_Git_with_nested_submodules/comment_6_11528378e8c509940190cf84db7ed3d6._comment b/doc/bugs/git-annex_confuses_Git_with_nested_submodules/comment_6_11528378e8c509940190cf84db7ed3d6._comment
new file mode 100644
index 0000000000..a6362816a7
--- /dev/null
+++ b/doc/bugs/git-annex_confuses_Git_with_nested_submodules/comment_6_11528378e8c509940190cf84db7ed3d6._comment
@@ -0,0 +1,14 @@
+[[!comment format=mdwn
+ username="joey"
+ subject="""comment 6"""
+ date="2016-05-16T20:14:57Z"
+ content="""
+I hope this will be fixed on the git side, but I don't know when it will
+be. Probably comes down to someone developing a patch.
+
+Occurs to me that git-annex could use adjusted branches to deal with this.
+When initializing in a submodule, it could enter an adjusted branch with
+`git annex adjust --fix`. This way, symlinks would be re-written to point
+to the submodule's git repository. The .git dotfile would not need to be
+converted to a symlink at all.
+"""]]
diff --git a/doc/bugs/git-annex_confuses_Git_with_nested_submodules/comment_7_a257e5ccf264b662ce9becd5f76d2f53._comment b/doc/bugs/git-annex_confuses_Git_with_nested_submodules/comment_7_a257e5ccf264b662ce9becd5f76d2f53._comment
new file mode 100644
index 0000000000..c253716ead
--- /dev/null
+++ b/doc/bugs/git-annex_confuses_Git_with_nested_submodules/comment_7_a257e5ccf264b662ce9becd5f76d2f53._comment
@@ -0,0 +1,50 @@
+[[!comment format=mdwn
+ username="yarikoptic"
+ avatar="http://cdn.libravatar.org/avatar/f11e9c84cb18d26a1748c33b48c924b4"
+ subject="comment 7"
+ date="2018-03-23T15:58:15Z"
+ content="""
+I think it is fixed now, either by git or git-annex . Here is my full protocol to reproduce (needed to adjust paths etc):
+[[!format sh \"\"\"
+$> mkdir some ; cd some; git init
+Initialized empty Git repository in /tmp/some/.git/
+1 9949.....................................:Fri 23 Mar 2018 11:55:02 EDT:.
+(git)hopa:/tmp/some[master]
+$> git submodule add /src/somegitrepo sub_lvl1
+
+fatal: repository '/src/somegitrepo' does not exist
+fatal: clone of '/src/somegitrepo' into submodule path '/tmp/some/sub_lvl1' failed
+1 9950 ->128.....................................:Fri 23 Mar 2018 11:55:13 EDT:.
+(git)hopa:/tmp/some[master]
+$> git clone http://datasets.datalad.org/.git /tmp/somegitrepo      
+Cloning into '/tmp/somegitrepo'...
+1 9951.....................................:Fri 23 Mar 2018 11:55:57 EDT:.
+(git)hopa:/tmp/some[master]git
+$> git submodule add /tmp/src/somegitrepo sub_lvl1                
+fatal: repository '/tmp/src/somegitrepo' does not exist
+fatal: clone of '/tmp/src/somegitrepo' into submodule path '/tmp/some/sub_lvl1' failed
+1 9952 ->128.....................................:Fri 23 Mar 2018 11:56:12 EDT:.
+(git)hopa:/tmp/some[master]git
+$> git submodule add /tmp/somegitrepo sub_lvl1    
+Cloning into '/tmp/some/sub_lvl1'...
+done.
+cached/staged changes:                                                                                          
+ .gitmodules | 3 +++
+ sub_lvl1    | 1 +
+1 9953.....................................:Fri 23 Mar 2018 11:56:18 EDT:.
+(git)hopa:/tmp/some[master]git
+$> cd sub_lvl1 
+abide/   adhd200/  crcns/            dbic/   dicoms/  indi/    labs/        nidm/      workshops/
+abide2/  corr/     datapackage.json  devel/  hbnssi/  kaggle/  neurovault/  openfmri/
+1 9954.....................................:Fri 23 Mar 2018 11:56:24 EDT:.
+(git)hopa:/tmp/some/sub_lvl1[master]git
+$> git annex init
+init  ok
+(recording state in git...)
+1 9955.....................................:Fri 23 Mar 2018 11:56:27 EDT:.
+(git)hopa:/tmp/some/sub_lvl1[master]git
+$> git submodule add /tmp/somegitrepo sub_lvl2
+Cloning into '/tmp/some/sub_lvl1/sub_lvl2'...
+done.
+\"\"\"]]
+"""]]
diff --git a/doc/bugs/git-annex_died_of_signal_11_when_syncing_content.mdwn b/doc/bugs/git-annex_died_of_signal_11_when_syncing_content.mdwn
new file mode 100644
index 0000000000..d0781d2816
--- /dev/null
+++ b/doc/bugs/git-annex_died_of_signal_11_when_syncing_content.mdwn
@@ -0,0 +1,30 @@
+### Please describe the problem.
+
+git-annex dies of signal 11 when syncing content with Box.
+
+I noticed that the assistant stopped syncing after a few files, so I tried to sync manually through the terminal and saw the error.
+
+### What steps will reproduce the problem?
+
+    $ git annex sync box.com --content
+    commit  ok
+    copy myfile1 (checking box.com...) (to box.com...) 
+    ok                      
+    copy myfile2 (checking box.com...) (to box.com...) 
+    ok                      
+    copy myfile3 (checking box.com...) (to box.com...) 
+    ok                      
+    copy myfile4 (checking box.com...) (to box.com...) 
+    ok                      
+    copy myfile5 (checking box.com...) error: git-annex died of signal 11
+
+The number of files that are copied correctly is quite random, some times it syncs 200 files and other times only 2 or 3.
+
+
+### What version of git-annex are you using? On what operating system?
+
+5.20141125 on Debian Sid (everything install from official packages)
+
+### Please provide any additional information below.
+
+I don't know how to get more details of the problem, I haven't manage to get a more precise error activating logs or verbose mode, if there's anything I can do to debug the problem I'll be more than willing.
diff --git a/doc/bugs/git-annex_died_of_signal_11_when_syncing_content/comment_1_e982f985e5ca57454725e9e9a482e30c._comment b/doc/bugs/git-annex_died_of_signal_11_when_syncing_content/comment_1_e982f985e5ca57454725e9e9a482e30c._comment
new file mode 100644
index 0000000000..f81f1d7328
--- /dev/null
+++ b/doc/bugs/git-annex_died_of_signal_11_when_syncing_content/comment_1_e982f985e5ca57454725e9e9a482e30c._comment
@@ -0,0 +1,12 @@
+[[!comment format=mdwn
+ username="joey"
+ subject="""comment 1"""
+ date="2015-02-04T18:38:09Z"
+ content="""
+Well, a signal 11 can be caused by several different things, including
+some messed up pointer stuff in a C library, or just bad memory on your
+computer. I think a memory test would be a good first step. 
+
+If the memory checks clean, running git-annex in gdb and getting a stack
+trace after a crash may help.
+"""]]
diff --git a/doc/bugs/git-annex_does_not_build_with_aws_0.16.mdwn b/doc/bugs/git-annex_does_not_build_with_aws_0.16.mdwn
new file mode 100644
index 0000000000..9af3726009
--- /dev/null
+++ b/doc/bugs/git-annex_does_not_build_with_aws_0.16.mdwn
@@ -0,0 +1,51 @@
+### Please describe the problem.
+
+aws 0.16 has a breaking change, causing git-annex' usage of S3.UploadPartResponse to fail.
+
+[[!format haskell """
+[325 of 546] Compiling Remote.S3        ( Remote/S3.hs, dist/dist-sandbox-8fbcd4b9/build/git-annex/git-annex-tmp/Remote/S3.o )
+
+Remote/S3.hs:224:49: error:
+    • The constructor ‘S3.UploadPartResponse’ should have 1 argument, but has been given 2
+    • In the pattern: S3.UploadPartResponse _ etag
+      In a stmt of a 'do' block:
+        S3.UploadPartResponse _ etag <- sendS3Handle h req
+      In the expression:
+        do { let sz = ...;
+             let p' = offsetMeterUpdate p (toBytesProcessed pos);
+             let numchunks
+                   = ceiling
+                       (fromIntegral sz / fromIntegral defaultChunkSize :: Double);
+             let popper = handlePopper numchunks defaultChunkSize p' fh;
+             .... }
+cabal: Leaving directory '.'
+cabal: Error: some packages failed to install:
+"""]]
+
+I suggest switching to {} pattern matching, like this:
+
+[[!format haskell """
+S3.UploadPartResponse { S3.uprEtag = etag } <- sendS3Handle h req
+"""]]
+
+### What steps will reproduce the problem?
+
+Try to build git-annex with aws 0.16.
+
+### What version of git-annex are you using? On what operating system?
+
+Not sure, I got this build failure report from ilovezfs who I think uses Max OS X, and git-annex master.
+
+### Please provide any additional information below.
+
+[[!format sh """
+# If you can, paste a complete transcript of the problem occurring here.
+# If the problem is with the git-annex assistant, paste in .git/annex/daemon.log
+
+
+# End of transcript or log.
+"""]]
+
+### Have you had any luck using git-annex before? (Sometimes we get tired of reading bug reports all day and a lil' positive end note does wonders)
+
+> [[fixed|done]], thanks --[[Joey]]
diff --git a/doc/bugs/git-annex_does_not_install_on_windows_without_admin_rights.mdwn b/doc/bugs/git-annex_does_not_install_on_windows_without_admin_rights.mdwn
new file mode 100644
index 0000000000..58b11deb32
--- /dev/null
+++ b/doc/bugs/git-annex_does_not_install_on_windows_without_admin_rights.mdwn
@@ -0,0 +1,24 @@
+### Please describe the problem.
+
+Installing on Windows requires installing git followed by git-annex.
+Installing the former works without admin rights, but the latter cannot be
+installed afterwards.
+
+### What steps will reproduce the problem?
+
+1. Create a Windows account without admin rights
+2. Install git
+3. Install git-annex
+
+### What version of git-annex are you using? On what operating system?
+
+Latest release on MS Windows.
+
+### Please provide any additional information below.
+
+
+Installing git creates read-only directories that cannot be used by the
+git-annex install afterwards. Without admin rights, the read-only flag of
+the git dir cannot be altered.
+
+[[!tag confirmed]]
diff --git a/doc/bugs/git-annex_does_not_install_on_windows_without_admin_rights/comment_1_2533800ab5a95c5d71c3b47a630e751a._comment b/doc/bugs/git-annex_does_not_install_on_windows_without_admin_rights/comment_1_2533800ab5a95c5d71c3b47a630e751a._comment
new file mode 100644
index 0000000000..32fc0311e7
--- /dev/null
+++ b/doc/bugs/git-annex_does_not_install_on_windows_without_admin_rights/comment_1_2533800ab5a95c5d71c3b47a630e751a._comment
@@ -0,0 +1,10 @@
+[[!comment format=mdwn
+ username="http://joeyh.name/"
+ ip="209.250.56.64"
+ subject="comment 1"
+ date="2013-11-26T16:19:41Z"
+ content="""
+Thanks, this had been reported in the forum before, but they did not point at the directory permissions as the problem.
+
+It would be possible to modify the installer to install git-annex somewhere else and add it to the PATH, but it seems this is a utter nightmare on windows and I'd have to pull in enormous NSIS scripts from their wiki, of unknown provenance. Which is why I am piggybacking on the git installation's PATH.
+"""]]
diff --git a/doc/bugs/git-annex_does_not_install_on_windows_without_admin_rights/comment_2_5b71785acf16a8d9ea457726599daef3._comment b/doc/bugs/git-annex_does_not_install_on_windows_without_admin_rights/comment_2_5b71785acf16a8d9ea457726599daef3._comment
new file mode 100644
index 0000000000..4bd5c2c0ed
--- /dev/null
+++ b/doc/bugs/git-annex_does_not_install_on_windows_without_admin_rights/comment_2_5b71785acf16a8d9ea457726599daef3._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="https://me.yahoo.com/a/bBy7WkgQicYHIiiyj.Vm0TcMbxi2quzbPFef#6f9f7"
+ nickname="Frederik Vanrenterghem"
+ subject="comment 2"
+ date="2013-11-26T21:57:28Z"
+ content="""
+Just wondering how git succeeds in installing as non-admin user. The folder it goes in is also read-only.
+"""]]
diff --git a/doc/bugs/git-annex_does_not_install_on_windows_without_admin_rights/comment_3_63e4ff79bc683a70dd9a85c66e26e56a._comment b/doc/bugs/git-annex_does_not_install_on_windows_without_admin_rights/comment_3_63e4ff79bc683a70dd9a85c66e26e56a._comment
new file mode 100644
index 0000000000..ac5e77d08d
--- /dev/null
+++ b/doc/bugs/git-annex_does_not_install_on_windows_without_admin_rights/comment_3_63e4ff79bc683a70dd9a85c66e26e56a._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="http://joeyh.name/"
+ ip="209.250.56.87"
+ subject="comment 3"
+ date="2013-12-12T19:33:50Z"
+ content="""
+I've made the installer now request admin rights if run by a non-admin. Hopefully good enough. However, the NSIS docs say that only works on Vista or higher. The old XP I'm using (that will be completely EOLed soon IIRC) still has the problem.
+"""]]
diff --git a/doc/bugs/git-annex_doesn__39__t_find_adb_device.mdwn b/doc/bugs/git-annex_doesn__39__t_find_adb_device.mdwn
new file mode 100644
index 0000000000..5a37559cc9
--- /dev/null
+++ b/doc/bugs/git-annex_doesn__39__t_find_adb_device.mdwn
@@ -0,0 +1,23 @@
+### Please describe the problem.
+git-annex claims that adb does not list any devices. `adb devices` however shows the device.
+
+### What steps will reproduce the problem?
+    $ git annex initremote phone type=adb androiddirectory=/storage/external/annex encryption=none exporttree=yes        
+    initremote handy git-annex: adb does not list any connected android devices. Plug in an Android device, or configure adb, and try again..
+    $ adb devices                                          
+    List of devices attached
+    6c12eba7d440	device
+
+
+### What version of git-annex are you using? On what operating system?
+    git-annex version: 6.20180529-g33834140e
+    build flags: Assistant Webapp Pairing S3(multipartupload)(storageclasses) WebDAV Inotify DBus DesktopNotify ConcurrentOutput TorrentParser MagicMime Feeds Testsuite
+    dependency versions: aws-0.20 bloomfilter-2.0.1.0 cryptonite-0.25 DAV-1.3.2 feed-1.0.0.0 ghc-8.4.3 http-client-0.5.12.1 persistent-sqlite-2.8.1.2 torrent-10000.1.1 uuid-1.3.13 yesod-1.6.0
+    key/value backends: SHA256E SHA256 SHA512E SHA512 SHA224E SHA224 SHA384E SHA384 SHA3_256E SHA3_256 SHA3_512E SHA3_512 SHA3_224E SHA3_224 SHA3_384E SHA3_384 SKEIN256E SKEIN256 SKEIN512E SKEIN512 BLAKE2B256E BLAKE2B256 BLAKE2B512E BLAKE2B512 BLAKE2B160E           BLAKE2B160 BLAKE2B224E BLAKE2B224 BLAKE2B384E BLAKE2B384 BLAKE2S256E BLAKE2S256 BLAKE2S160E BLAKE2S160 BLAKE2S224E BLAKE2S224 BLAKE2SP256E BLAKE2SP256 BLAKE2SP224E BLAKE2SP224 SHA1E SHA1 MD5E MD5 WORM URL
+    remote types: git gcrypt p2p S3 bup directory rsync web bittorrent webdav adb tahoe glacier ddar hook external
+    local repository version: 5
+    supported repository versions: 3 5 6
+    upgrade supported from repository versions: 0 1 2 3 4 5
+    operating system: linux x86_64
+
+> [[fixed|done]] --[[Joey]] 
diff --git a/doc/bugs/git-annex_doesn__39__t_find_adb_device/comment_1_56070e29ea7f19d24d4ff0f56adf08a0._comment b/doc/bugs/git-annex_doesn__39__t_find_adb_device/comment_1_56070e29ea7f19d24d4ff0f56adf08a0._comment
new file mode 100644
index 0000000000..f030295f5a
--- /dev/null
+++ b/doc/bugs/git-annex_doesn__39__t_find_adb_device/comment_1_56070e29ea7f19d24d4ff0f56adf08a0._comment
@@ -0,0 +1,10 @@
+[[!comment format=mdwn
+ username="joey"
+ subject="""comment 1"""
+ date="2018-06-12T17:39:51Z"
+ content="""
+It's expecting a serial number 16 digits long, 
+and your adb is using a 12 digit one.  I suppose this means that the length
+can vary. Hopefully because of a difference in the android device and not
+the version of adb. I'll change it to accept shorter serials.
+"""]]
diff --git a/doc/bugs/git-annex_doesn__39__t_find_adb_device/comment_2_7e59d3e5a6273e737626cf73013c334d._comment b/doc/bugs/git-annex_doesn__39__t_find_adb_device/comment_2_7e59d3e5a6273e737626cf73013c334d._comment
new file mode 100644
index 0000000000..843a74ceb5
--- /dev/null
+++ b/doc/bugs/git-annex_doesn__39__t_find_adb_device/comment_2_7e59d3e5a6273e737626cf73013c334d._comment
@@ -0,0 +1,9 @@
+[[!comment format=mdwn
+ username="lykos@d125a37d89b1cfac20829f12911656c40cb70018"
+ nickname="lykos"
+ avatar="http://cdn.libravatar.org/avatar/085df7b04d3408ba23c19f9c49be9ea2"
+ subject="comment 2"
+ date="2018-06-13T10:46:25Z"
+ content="""
+Thanks! I think it's independent from adb, as it's the same serial number shown by dmesg.
+"""]]
diff --git a/doc/bugs/git-annex_drop_fails_to_access_file__58____47____47____47___target_URL_on_Windows.mdwn b/doc/bugs/git-annex_drop_fails_to_access_file__58____47____47____47___target_URL_on_Windows.mdwn
new file mode 100644
index 0000000000..ada6621f9b
--- /dev/null
+++ b/doc/bugs/git-annex_drop_fails_to_access_file__58____47____47____47___target_URL_on_Windows.mdwn
@@ -0,0 +1,29 @@
+### Please describe the problem.
+
+If I addurl file pointing to file:///C:/... it seems to work just fine, but then refuses to drop it stating that can't verify its presence.
+
+Please see two screenshots (sorry for not cut/paste here since it was already was painful -- debugging under Virtualbox in remote vnc through debconf internet which for some reason drops for me quite often):
+
+http://www.onerussian.com/tmp/gkrellShoot_08-19-15_220150.png
+http://www.onerussian.com/tmp/gkrellShoot_08-19-15_184052.png
+
+as screenshot shows, apparently wget is clueless about file:// targets on windows (while curl does fine) -- may be related ;-)
+
+### What steps will reproduce the problem?
+
+addurl some file:///C:/ under windows pointing to existing file, and then try to drop that annexed file
+
+### What version of git-annex are you using? On what operating system?
+
+windows
+20150805 or so
+
+### Please provide any additional information below.
+
+[[!format sh """
+# If you can, paste a complete transcript of the problem occurring here.
+# If the problem is with the git-annex assistant, paste in .git/annex/daemon.log
+
+
+# End of transcript or log.
+"""]]
diff --git a/doc/bugs/git-annex_drop_fails_to_access_file__58____47____47____47___target_URL_on_Windows/comment_1_f0d30a953f072f8d9a929a4a6ba69914._comment b/doc/bugs/git-annex_drop_fails_to_access_file__58____47____47____47___target_URL_on_Windows/comment_1_f0d30a953f072f8d9a929a4a6ba69914._comment
new file mode 100644
index 0000000000..c7de2eb6db
--- /dev/null
+++ b/doc/bugs/git-annex_drop_fails_to_access_file__58____47____47____47___target_URL_on_Windows/comment_1_f0d30a953f072f8d9a929a4a6ba69914._comment
@@ -0,0 +1,48 @@
+[[!comment format=mdwn
+ username="https://me.yahoo.com/a/EbvxpTI_xP9Aod7Mg4cwGhgjrCrdM5s-#7c0f4"
+ subject="seems just ignore errors while adding urls to "unsupported" urls"
+ date="2015-08-19T21:59:09Z"
+ content="""
+actually situation is somewhat similar on linux as well in a sense that annex manages to addurl a file using e.g. file:///./data   url (not sure even if legit) without puking but contains wrong content (empty):
+
+[[!format  sh \"\"\"
+
+% mkdir XXX
+% cd XXX
+% git init; git annex init
+Initialized empty Git repository in /tmp/XXX/.git/
+init  ok
+(recording state in git...)
+% echo 123 > data
+% git annex addurl --file=annexed file:///./data
+addurl annexed (downloading file:///./data ...) 
+
+ok
+(recording state in git...)
+% cat annexed 
+% git annex drop annexed
+drop annexed (checking file:///./data...) (unsafe) 
+  Could only verify the existence of 0 out of 1 necessary copies
+
+  Rather than dropping this file, try using: git annex move
+
+  (Use --force to override this check, or adjust numcopies.)
+failed
+git-annex: drop: 1 failed
+% git annex addurl --file=annexedfull file://$PWD/data
+addurl annexedfull (downloading file:///tmp/XXX/data ...) 
+######################################################################## 100.0%
+ok
+(recording state in git...)
+% cat annexedfull
+123
+% git annex drop annexedfull                          
+drop annexedfull (checking file:///tmp/XXX/data...) ok
+(recording state in git...)
+% annex version
+zsh: command not found: annex
+% git annex version
+git-annex version: 5.20150812-2
+
+\"\"\"]]
+"""]]
diff --git a/doc/bugs/git-annex_drop_fails_to_access_file__58____47____47____47___target_URL_on_Windows/comment_2_504ea07f798838710cdbf6133135c815._comment b/doc/bugs/git-annex_drop_fails_to_access_file__58____47____47____47___target_URL_on_Windows/comment_2_504ea07f798838710cdbf6133135c815._comment
new file mode 100644
index 0000000000..5097e2e16a
--- /dev/null
+++ b/doc/bugs/git-annex_drop_fails_to_access_file__58____47____47____47___target_URL_on_Windows/comment_2_504ea07f798838710cdbf6133135c815._comment
@@ -0,0 +1,39 @@
+[[!comment format=mdwn
+ username="joey"
+ subject="""comment 2"""
+ date="2015-09-09T16:36:13Z"
+ content="""
+I can't reproduce that behavior on linux. 
+
+	joey@darkstar:~/tmp/xx>git annex addurl  --file=annexed file:///./data
+	addurl annexed (downloading file:///./data ...) 
+	curl: (37) Couldn't open file /data
+
+Here, curl seems to be doing the right thing; the url is not relative; it's 
+for `/./data`, which doesn't exist.
+
+Relative `file:` urls shouldn't be valid at all, I think?
+
+---
+
+For checking if a file: url exits, git-annex parses the url and stats
+the file itself. The first screenshot
+shows this check for file: url existance failing on Windows for 
+the url `file:///C:/tmp/test/test.dat`
+
+I guess this might come down to problems with parsing file: urls on
+Windows; seems especially complicated by drive letters. git-annex and curl
+seem to parse this url in different ways.
+
+Checking how that url parses, the uriScheme is "file:" and the uriPath is
+"/C:/tmp/test/test.dat". So, it seems clear why it fails to stat that file.
+
+Is there actually a valid way to produce a file: url that refers to a drive
+letter? curl seems to think so, since it found the file when `git annex
+addurl` ran it. I don't know if the above parse is valid, but it's not
+git-annex's code doing the parse, but the URI parsing library.
+
+(Possibly related bug report: 
+
+)
+"""]]
diff --git a/doc/bugs/git-annex_fsck_in_quiet_mode_don__39__t_report_the_files_it_failed_to_download.mdwn b/doc/bugs/git-annex_fsck_in_quiet_mode_don__39__t_report_the_files_it_failed_to_download.mdwn
new file mode 100644
index 0000000000..2496715957
--- /dev/null
+++ b/doc/bugs/git-annex_fsck_in_quiet_mode_don__39__t_report_the_files_it_failed_to_download.mdwn
@@ -0,0 +1,15 @@
+### Please describe the problem.
+
+When running 'git annex fsck --from $i --quiet' when $i is a special remote, git annex will report failure to download as 'failed to download file from remote' without telling me which file it failed to download.
+
+### What steps will reproduce the problem?
+
+Just running 'git annex fsck --from remote --quiet' on some special remote with connexion problem
+
+### What version of git-annex are you using? On what operating system?
+
+6.20170101-1+b1 from Debian stretch
+
+### Have you had any luck using git-annex before? (Sometimes we get tired of reading bug reports all day and a lil' positive end note does wonders)
+
+git annex is very efficient to sync my usb key, laptops and servers together.
diff --git a/doc/bugs/git-annex_fsck_in_quiet_mode_don__39__t_report_the_files_it_failed_to_download/comment_1_074e49431bd18c43164872de3733b0e8._comment b/doc/bugs/git-annex_fsck_in_quiet_mode_don__39__t_report_the_files_it_failed_to_download/comment_1_074e49431bd18c43164872de3733b0e8._comment
new file mode 100644
index 0000000000..7465aae6f4
--- /dev/null
+++ b/doc/bugs/git-annex_fsck_in_quiet_mode_don__39__t_report_the_files_it_failed_to_download/comment_1_074e49431bd18c43164872de3733b0e8._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="Rémi"
+ avatar="http://cdn.libravatar.org/avatar/0f25950c04685d6c7ccb22215268d6a7"
+ subject="comment 1"
+ date="2017-08-21T14:17:14Z"
+ content="""
+Running 'git annex fsck --more --from remote' allowed me to try again and find which file had the problem. 
+"""]]
diff --git a/doc/bugs/git-annex_get_with_umlaut_characters_in_filenames.mdwn b/doc/bugs/git-annex_get_with_umlaut_characters_in_filenames.mdwn
new file mode 100644
index 0000000000..c05130a694
--- /dev/null
+++ b/doc/bugs/git-annex_get_with_umlaut_characters_in_filenames.mdwn
@@ -0,0 +1,59 @@
+### Please describe the problem.
+
+Files with umlauts were not copied from local system, all other files were copied successfully.
+
+### What steps will reproduce the problem?
+
+Trying to sync content from a repository on the same machine.
+
+### What version of git-annex are you using? On what operating system?
+
+git-annex version: 5.20150727 / Darwin tba.lan 14.4.0 Darwin Kernel Version 14.4.0: Thu May 28 11:35:04 PDT 2015; root:xnu-2782.30.5~1/RELEASE_X86_64 x86_64
+
+### Please provide any additional information below.
+
+[[!format sh """
+$ git annex get .
+get Die Sterne/Flucht in die Flucht (Bonus Track Version)/03 Ihr wollt mich töten.m4a
+  Unable to access these remotes: tba
+
+  Try making some of these repositories available:
+  	2cabf5e0-00ae-4cc6-b9b7-5d303a7f3f06 -- Music [tba]
+   	8e315ed0-f318-45f7-98ca-1a791f9c92df -- jan@hostname:/srv/annex-Music
+failed
+get Die Sterne/Flucht in die Flucht (Bonus Track Version)/03 Ihr wollt mich töten.m4a
+  Unable to access these remotes: tba
+
+  Try making some of these repositories available:
+  	2cabf5e0-00ae-4cc6-b9b7-5d303a7f3f06 -- Music [tba]
+   	8e315ed0-f318-45f7-98ca-1a791f9c92df -- jan@hostname:/srv/annex-Music
+failed
+get Die Sterne/Flucht in die Flucht (Bonus Track Version)/10 Der Bär.m4a
+  Unable to access these remotes: tba
+
+  Try making some of these repositories available:
+  	2cabf5e0-00ae-4cc6-b9b7-5d303a7f3f06 -- Music [tba]
+   	8e315ed0-f318-45f7-98ca-1a791f9c92df -- jan@hostname:/srv/annex-Music
+failed
+get Die Sterne/Flucht in die Flucht (Bonus Track Version)/10 Der Bär.m4a
+  Unable to access these remotes: tba
+
+  Try making some of these repositories available:
+  	2cabf5e0-00ae-4cc6-b9b7-5d303a7f3f06 -- Music [tba]
+   	8e315ed0-f318-45f7-98ca-1a791f9c92df -- jan@hostname:/srv/annex-Music
+failed
+"""]]
+
+the *tba* repository is accessible since all the other files were synced correctly.
+
+Git status reports untracked files which look like they were renamed.
+
+[[!format sh """
+$ git status
+Untracked files:
+  (use "git add ..." to include in what will be committed)
+
+	"Ant\303\263nio Varia\303\247\303\265es/"
+	"B Fachada/B Fachada/01 sozinho no r\303\263que.mp3"
+	"B Fachada/B Fachada/03 D\303\241 mais m\303\272sica \303\240 b\303\263fia.mp3"
+"""]]
diff --git a/doc/bugs/git-annex_get_with_umlaut_characters_in_filenames/comment_1_0724d70364f7067d864a80a39e2e609a._comment b/doc/bugs/git-annex_get_with_umlaut_characters_in_filenames/comment_1_0724d70364f7067d864a80a39e2e609a._comment
new file mode 100644
index 0000000000..b5e8f21f52
--- /dev/null
+++ b/doc/bugs/git-annex_get_with_umlaut_characters_in_filenames/comment_1_0724d70364f7067d864a80a39e2e609a._comment
@@ -0,0 +1,17 @@
+[[!comment format=mdwn
+ username="joey"
+ subject="""comment 1"""
+ date="2015-08-13T21:29:41Z"
+ content="""
+`git annex get` only acts on files that are present in the working tree of
+the git repo, and which `git ls-files` lists. So, if git is somehow
+confused about these filenames, which is appears to be, it makes sense that
+git-annex wouldn't be operating on them.
+
+I don't know why git would have such a problem with files with umlaut in
+their names. AFAIK, git is quite agnostic about filename encodings.
+However, your git status seems to be the smoking gun; the files are either
+not known to git at all, or it thinks they were renamed. Does the git
+log --stat show the files being deleted. Have you tried (re)adding the
+files to git?
+"""]]
diff --git a/doc/bugs/git-annex_in_nixpkgs_fails_with_git-2.13.0.mdwn b/doc/bugs/git-annex_in_nixpkgs_fails_with_git-2.13.0.mdwn
new file mode 100644
index 0000000000..53e12b5fce
--- /dev/null
+++ b/doc/bugs/git-annex_in_nixpkgs_fails_with_git-2.13.0.mdwn
@@ -0,0 +1,359 @@
+### Please describe the problem.
+
+Tests fail on Nix when git is upgraded to 2.13.
+
+
+### What steps will reproduce the problem?
+
+On NixOS or a system with Nix installed, check out nixpkgs ce8662e693b5756e8457d94bb1765d853310afdb, try to build git-annex (`nix-build -I . -A pkgs.gitAndTools.gitAnnex`).
+
+
+### What version of git-annex are you using? On what operating system?
+
+git-annex 6.20170321 on Nix ce8662e6.
+
+
+### Please provide any additional information below.
+
+I ran a git bisect that concluded:
+
+[[!format sh """
+8 out of 285 tests failed (281.73s)
+  (This could be due to a bug in git-annex, or an incompatibility
+   with utilities, such as git, installed on this system.)
+builder for ‘/nix/store/908y9923fnjmi87apji6q14smgc2rf3d-git-annex-6.20170321.drv’ failed with exit code 1
+error: build of ‘/nix/store/908y9923fnjmi87apji6q14smgc2rf3d-git-annex-6.20170321.drv’ failed
+ce8662e693b5756e8457d94bb1765d853310afdb is the first bad commit
+commit ce8662e693b5756e8457d94bb1765d853310afdb
+Author: Tim Steinbach 
+Date:   Tue May 9 21:57:24 2017 -0400
+
+    git: 2.12.2 -> 2.13.0
+
+:040000 040000 4155c091e7156a4e15cf073640e0be3c76b1b7f3 48ca917e5578b405cf8517e1f113433b2d076e15 M      pkgs
+bisect run success
+"""]]
+
+
+[[!format sh """
+    adjusted branch merge regression:                     Switched to branch 'adjusted/master(unlocked)'
+[adjusted/master(unlocked) ca9719e] git-annex in .t/tmprepo39
+ 1 file changed, 1 insertion(+)
+ create mode 100644 conflictor
+From ../../.t/tmprepo40
+ * [new branch]      git-annex  -> r2/git-annex
+ * [new branch]      master     -> r2/master
+To ../../.t/tmprepo40
+ * [new branch]      git-annex -> synced/git-annex
+ * [new branch]      master -> synced/master
+Switched to branch 'adjusted/master(unlocked)'
+[adjusted/master(unlocked) b29e8de] git-annex in .t/tmprepo40
+ 1 file changed, 1 insertion(+)
+ create mode 100644 conflictor
+merge: refs/heads/synced/master - not something we can merge
+From ../../.t/tmprepo39
+ * [new branch]      adjusted/master(unlocked) -> r1/adjusted/master(unlocked)
+ * [new branch]      git-annex                 -> r1/git-annex
+ * [new branch]      master                    -> r1/master
+ * [new branch]      synced/master             -> r1/synced/master
+merge: refs/remotes/r1/master - not something we can merge
+merge: refs/remotes/r1/synced/master - not something we can merge
+To ../../.t/tmprepo39
+ * [new branch]      git-annex -> synced/git-annex
+ ! [rejected]        master -> synced/master (non-fast-forward)
+error: failed to push some refs to '../../.t/tmprepo39'
+hint: Updates were rejected because a pushed branch tip is behind its remote
+hint: counterpart. Check out this branch and integrate the remote changes
+hint: (e.g. 'git pull ...') before pushing again.
+hint: See the 'Note about fast-forwards' in 'git push --help' for details.
+To ../../.t/tmprepo39
+ ! [rejected]        master -> master (non-fast-forward)
+error: failed to push some refs to '../../.t/tmprepo39'
+hint: Updates were rejected because a pushed branch tip is behind its remote
+hint: counterpart. Check out this branch and integrate the remote changes
+hint: (e.g. 'git pull ...') before pushing again.
+hint: See the 'Note about fast-forwards' in 'git push --help' for details.
+  Pushing to r1 failed.
+FAIL (2.32s)
+"""]]
+
+
+[[!format sh """
+    conflict resolution (adjusted branch):                [master b0dd758] git-annex in .t/tmprepo44
+ 1 file changed, 1 insertion(+)
+ create mode 100644 conflictor
+[master 8295bae] git-annex in .t/tmprepo45
+ 1 file changed, 1 insertion(+)
+ create mode 100644 conflictor
+Switched to branch 'adjusted/master(unlocked)'
+On branch master
+nothing to commit, working tree clean
+From ../../.t/tmprepo45
+ * [new branch]      adjusted/master(unlocked) -> r2/adjusted/master(unlocked)
+ * [new branch]      git-annex                 -> r2/git-annex
+ * [new branch]      master                    -> r2/master
+ * [new branch]      synced/master             -> r2/synced/master
+Auto-merging conflictor
+CONFLICT (add/add): Merge conflict in conflictor
+Automatic merge failed; fix conflicts and then commit the result.
+conflictor: needs merge
+[master 14ffa24] git-annex automatic merge conflict fix
+Already up-to-date.
+To ../../.t/tmprepo45
+   8295bae..14ffa24  master -> synced/master
+ * [new branch]      git-annex -> synced/git-annex
+On branch adjusted/master(unlocked)
+nothing to commit, working tree clean
+merge: refs/heads/synced/master - not something we can merge
+From ../../.t/tmprepo44
+ * [new branch]      git-annex     -> r1/git-annex
+ * [new branch]      master        -> r1/master
+ * [new branch]      synced/master -> r1/synced/master
+merge: refs/remotes/r1/master - not something we can merge
+merge: refs/remotes/r1/synced/master - not something we can merge
+FAIL (2.34s)
+"""]]
+
+[[!format sh """
+    adjusted branch merge regression:                     Switched to branch 'adjusted/master(unlocked)'
+[adjusted/master(unlocked) 7ccad36] git-annex in .t/tmprepo39
+ 1 file changed, 1 insertion(+)
+ create mode 100644 conflictor
+From ../../.t/tmprepo40
+ * [new branch]      git-annex  -> r2/git-annex
+ * [new branch]      master     -> r2/master
+To ../../.t/tmprepo40
+ * [new branch]      git-annex -> synced/git-annex
+ * [new branch]      master -> synced/master
+Switched to branch 'adjusted/master(unlocked)'
+[adjusted/master(unlocked) cfb04fd] git-annex in .t/tmprepo40
+ 1 file changed, 1 insertion(+)
+ create mode 100644 conflictor
+merge: refs/heads/synced/master - not something we can merge
+From ../../.t/tmprepo39
+ * [new branch]      adjusted/master(unlocked) -> r1/adjusted/master(unlocked)
+ * [new branch]      git-annex                 -> r1/git-annex
+ * [new branch]      master                    -> r1/master
+ * [new branch]      synced/master             -> r1/synced/master
+merge: refs/remotes/r1/master - not something we can merge
+merge: refs/remotes/r1/synced/master - not something we can merge
+To ../../.t/tmprepo39
+ * [new branch]      git-annex -> synced/git-annex
+ ! [rejected]        master -> synced/master (non-fast-forward)
+error: failed to push some refs to '../../.t/tmprepo39'
+hint: Updates were rejected because a pushed branch tip is behind its remote
+hint: counterpart. Check out this branch and integrate the remote changes
+hint: (e.g. 'git pull ...') before pushing again.
+hint: See the 'Note about fast-forwards' in 'git push --help' for details.
+To ../../.t/tmprepo39
+ ! [rejected]        master -> master (non-fast-forward)
+error: failed to push some refs to '../../.t/tmprepo39'
+hint: Updates were rejected because a pushed branch tip is behind its remote
+hint: counterpart. Check out this branch and integrate the remote changes
+hint: (e.g. 'git pull ...') before pushing again.
+hint: See the 'Note about fast-forwards' in 'git push --help' for details.
+  Pushing to r1 failed.
+FAIL (1.79s)
+"""]]
+
+[[!format sh """
+    conflict resolution (adjusted branch):                [master ea63ee5] git-annex in .t/tmprepo44
+ 1 file changed, 1 insertion(+)
+ create mode 120000 conflictor
+[master 07d129c] git-annex in .t/tmprepo45
+ 1 file changed, 1 insertion(+)
+ create mode 120000 conflictor
+Switched to branch 'adjusted/master(unlocked)'
+On branch master
+nothing to commit, working tree clean
+From ../../.t/tmprepo45
+ * [new branch]      adjusted/master(unlocked) -> r2/adjusted/master(unlocked)
+ * [new branch]      git-annex                 -> r2/git-annex
+ * [new branch]      master                    -> r2/master
+ * [new branch]      synced/master             -> r2/synced/master
+Auto-merging conflictor
+CONFLICT (add/add): Merge conflict in conflictor
+Automatic merge failed; fix conflicts and then commit the result.
+conflictor: needs merge
+[master 5d5a674] git-annex automatic merge conflict fix
+Already up-to-date.
+To ../../.t/tmprepo45
+   07d129c..5d5a674  master -> synced/master
+ * [new branch]      git-annex -> synced/git-annex
+On branch adjusted/master(unlocked)
+nothing to commit, working tree clean
+merge: refs/heads/synced/master - not something we can merge
+From ../../.t/tmprepo44
+ * [new branch]      git-annex     -> r1/git-annex
+ * [new branch]      master        -> r1/master
+ * [new branch]      synced/master -> r1/synced/master
+merge: refs/remotes/r1/master - not something we can merge
+merge: refs/remotes/r1/synced/master - not something we can merge
+FAIL (1.35s)
+"""]]
+
+[[!format sh """
+    adjusted branch merge regression:                     Switched to branch 'adjusted/master(unlocked)'
+[adjusted/master(unlocked) 6601699] git-annex in .t/tmprepo39
+ 1 file changed, 1 insertion(+)
+ create mode 100644 conflictor
+From ../../.t/tmprepo40
+ * [new branch]      git-annex  -> r2/git-annex
+ * [new branch]      master     -> r2/master
+To ../../.t/tmprepo40
+ * [new branch]      git-annex -> synced/git-annex
+ * [new branch]      master -> synced/master
+Switched to branch 'adjusted/master(unlocked)'
+[adjusted/master(unlocked) b37ecc1] git-annex in .t/tmprepo40
+ 1 file changed, 1 insertion(+)
+ create mode 100644 conflictor
+merge: refs/heads/synced/master - not something we can merge
+From ../../.t/tmprepo39
+ * [new branch]      adjusted/master(unlocked) -> r1/adjusted/master(unlocked)
+ * [new branch]      git-annex                 -> r1/git-annex
+ * [new branch]      master                    -> r1/master
+ * [new branch]      synced/master             -> r1/synced/master
+merge: refs/remotes/r1/master - not something we can merge
+merge: refs/remotes/r1/synced/master - not something we can merge
+To ../../.t/tmprepo39
+ * [new branch]      git-annex -> synced/git-annex
+ ! [rejected]        master -> synced/master (non-fast-forward)
+error: failed to push some refs to '../../.t/tmprepo39'
+hint: Updates were rejected because a pushed branch tip is behind its remote
+hint: counterpart. Check out this branch and integrate the remote changes
+hint: (e.g. 'git pull ...') before pushing again.
+hint: See the 'Note about fast-forwards' in 'git push --help' for details.
+To ../../.t/tmprepo39
+ ! [rejected]        master -> master (non-fast-forward)
+error: failed to push some refs to '../../.t/tmprepo39'
+hint: Updates were rejected because a pushed branch tip is behind its remote
+hint: counterpart. Check out this branch and integrate the remote changes
+hint: (e.g. 'git pull ...') before pushing again.
+hint: See the 'Note about fast-forwards' in 'git push --help' for details.
+  Pushing to r1 failed.
+FAIL (1.54s)
+"""]]
+
+[[!format sh """
+    conflict resolution (adjusted branch):                [master dcd2368] git-annex in .t/tmprepo44
+ 1 file changed, 1 insertion(+)
+ create mode 120000 conflictor
+[master c933362] git-annex in .t/tmprepo45
+ 1 file changed, 1 insertion(+)
+ create mode 120000 conflictor
+Switched to branch 'adjusted/master(unlocked)'
+On branch master
+nothing to commit, working tree clean
+From ../../.t/tmprepo45
+ * [new branch]      adjusted/master(unlocked) -> r2/adjusted/master(unlocked)
+ * [new branch]      git-annex                 -> r2/git-annex
+ * [new branch]      master                    -> r2/master
+ * [new branch]      synced/master             -> r2/synced/master
+Auto-merging conflictor
+CONFLICT (add/add): Merge conflict in conflictor
+Automatic merge failed; fix conflicts and then commit the result.
+conflictor: needs merge
+[master 9f19aa7] git-annex automatic merge conflict fix
+Already up-to-date.
+To ../../.t/tmprepo45
+   c933362..9f19aa7  master -> synced/master
+ * [new branch]      git-annex -> synced/git-annex
+On branch adjusted/master(unlocked)
+nothing to commit, working tree clean
+merge: refs/heads/synced/master - not something we can merge
+From ../../.t/tmprepo44
+ * [new branch]      git-annex     -> r1/git-annex
+ * [new branch]      master        -> r1/master
+ * [new branch]      synced/master -> r1/synced/master
+merge: refs/remotes/r1/master - not something we can merge
+merge: refs/remotes/r1/synced/master - not something we can merge
+FAIL (1.44s)
+"""]]
+
+[[!format sh """
+    adjusted branch merge regression:                     On branch master
+nothing to commit, working tree clean
+On branch master
+nothing to commit, working tree clean
+Already on 'adjusted/master(unlocked)'
+[adjusted/master(unlocked) 70b8469] git-annex in .t/tmprepo39
+ 1 file changed, 1 insertion(+)
+ create mode 100644 conflictor
+From ../../.t/tmprepo40
+ * [new branch]      annex/direct/master -> r2/annex/direct/master
+ * [new branch]      git-annex           -> r2/git-annex
+ * [new branch]      master              -> r2/master
+To ../../.t/tmprepo40
+ * [new branch]      git-annex -> synced/git-annex
+ * [new branch]      master -> synced/master
+Already on 'adjusted/master(unlocked)'
+[adjusted/master(unlocked) 353957d] git-annex in .t/tmprepo40
+ 1 file changed, 1 insertion(+)
+ create mode 100644 conflictor
+merge: refs/heads/synced/master - not something we can merge
+From ../../.t/tmprepo39
+ * [new branch]      adjusted/master(unlocked) -> r1/adjusted/master(unlocked)
+ * [new branch]      git-annex                 -> r1/git-annex
+ * [new branch]      master                    -> r1/master
+ * [new branch]      synced/master             -> r1/synced/master
+merge: refs/remotes/r1/master - not something we can merge
+merge: refs/remotes/r1/synced/master - not something we can merge
+To ../../.t/tmprepo39
+ * [new branch]      git-annex -> synced/git-annex
+ ! [rejected]        master -> synced/master (non-fast-forward)
+error: failed to push some refs to '../../.t/tmprepo39'
+hint: Updates were rejected because a pushed branch tip is behind its remote
+hint: counterpart. Check out this branch and integrate the remote changes
+hint: (e.g. 'git pull ...') before pushing again.
+hint: See the 'Note about fast-forwards' in 'git push --help' for details.
+To ../../.t/tmprepo39
+ ! [rejected]        master -> master (non-fast-forward)
+error: failed to push some refs to '../../.t/tmprepo39'
+hint: Updates were rejected because a pushed branch tip is behind its remote
+hint: counterpart. Check out this branch and integrate the remote changes
+hint: (e.g. 'git pull ...') before pushing again.
+hint: See the 'Note about fast-forwards' in 'git push --help' for details.
+  Pushing to r1 failed.
+FAIL (1.87s)
+"""]]
+
+[[!format sh """
+    conflict resolution (adjusted branch):                On branch master
+nothing to commit, working tree clean
+On branch master
+nothing to commit, working tree clean
+Already on 'adjusted/master(unlocked)'
+From ../../.t/tmprepo45
+ * [new branch]      adjusted/master(unlocked) -> r2/adjusted/master(unlocked)
+ * [new branch]      git-annex                 -> r2/git-annex
+ * [new branch]      master                    -> r2/master
+ * [new branch]      synced/master             -> r2/synced/master
+Auto-merging conflictor
+CONFLICT (add/add): Merge conflict in conflictor
+Automatic merge failed; fix conflicts and then commit the result.
+conflictor: needs merge
+To ../../.t/tmprepo45
+   372f97c..cebbf06  annex/direct/master -> synced/master
+ * [new branch]      git-annex -> synced/git-annex
+On branch adjusted/master(unlocked)
+nothing to commit, working tree clean
+merge: refs/heads/synced/master - not something we can merge
+From ../../.t/tmprepo44
+ * [new branch]      annex/direct/master -> r1/annex/direct/master
+ * [new branch]      git-annex           -> r1/git-annex
+ * [new branch]      master              -> r1/master
+ * [new branch]      synced/master       -> r1/synced/master
+merge: refs/remotes/r1/master - not something we can merge
+merge: refs/remotes/r1/synced/master - not something we can merge
+FAIL (1.50s)
+"""]]
+
+[[!format sh """
+
+"""]]
+
+### Have you had any luck using git-annex before? (Sometimes we get tired of reading bug reports all day and a lil' positive end note does wonders)
+
+git-annex is an essential building block in my digital life style! It keeps backups of all my precious family photos. I'm a big git-annex shill when I get the chance, especially to nix, guix and decentralized hacker types.
+
+> Workaround is in [[!commit 9bcaef1ec496b4ffd3033ae5080949bd8cc3edd5]]. [[done]] --[[Joey]]
diff --git a/doc/bugs/git-annex_in_nixpkgs_fails_with_git-2.13.0/comment_1_364f69b47889486ab006811406324c44._comment b/doc/bugs/git-annex_in_nixpkgs_fails_with_git-2.13.0/comment_1_364f69b47889486ab006811406324c44._comment
new file mode 100644
index 0000000000..9e7d1928a2
--- /dev/null
+++ b/doc/bugs/git-annex_in_nixpkgs_fails_with_git-2.13.0/comment_1_364f69b47889486ab006811406324c44._comment
@@ -0,0 +1,38 @@
+[[!comment format=mdwn
+ username="joey"
+ subject="""comment 1"""
+ date="2017-05-16T15:42:53Z"
+ content="""
+Not nix-specific, I can reproduce with self-built git.
+
+Bisecting git itself (rather than nix):
+
+	f57f37e2e1bf11ab4cdfd221ad47e961ba9353a0 is the first bad commit
+	commit f57f37e2e1bf11ab4cdfd221ad47e961ba9353a0
+	Author: Nguyễn Thái Ngọc Duy 
+	Date:   Sun Mar 26 09:42:24 2017 +0700
+	
+	    files-backend: remove the use of git_path()
+
+That commit seems has to do with paths used for refs, and the test suite is
+failing due to something involving pushing/merging the
+synced/master branch. So, it's looking like a git bug.
+
+In particular, this just seems outright wrong:
+
+	From ../../.t/tmprepo44
+	 * [new branch]      git-annex     -> r1/git-annex
+	 * [new branch]      master        -> r1/master
+	 * [new branch]      synced/master -> r1/synced/master
+	merge: refs/remotes/r1/master - not something we can merge
+	merge: refs/remotes/r1/synced/master - not something we can merge
+
+So, it's just pulled synced/master to r1/synced/master, but then it claims
+refs/remotes/r1/synced/master is not mergable.
+
+This only affects adjusted branches because git has a reversion in the
+`GIT_COMMON_DIR` that they use.
+
+Posted on the git ML about this reversion, message-id
+<20170516171028.5eagqr2sw5a2i77d@kitenet.net>
+"""]]
diff --git a/doc/bugs/git-annex_merge_stalls.mdwn b/doc/bugs/git-annex_merge_stalls.mdwn
new file mode 100644
index 0000000000..80330a822a
--- /dev/null
+++ b/doc/bugs/git-annex_merge_stalls.mdwn
@@ -0,0 +1,18 @@
+### Please describe the problem.
+
+Running git-annex merge shows the output "git-annex merge ", followed by a blinking cursor. The command never seems to end.
+
+### What steps will reproduce the problem?
+
+
+### What version of git-annex are you using? On what operating system?
+
+4.20130827-gd04d9bb on MacOS X Mountain Lion
+
+### Please provide any additional information below.
+
+dtruss output at https://www.dropbox.com/s/4b3yqn7ajfz5el2/annex-merge.log
+
+[[!meta title="no indication when git-annex is stuck waiting for a lock"]]
+
+[[!tag confirmed]]
diff --git a/doc/bugs/git-annex_merge_stalls/comment_1_31578a754945bdcb902c62ff58704bcb._comment b/doc/bugs/git-annex_merge_stalls/comment_1_31578a754945bdcb902c62ff58704bcb._comment
new file mode 100644
index 0000000000..123212d5d4
--- /dev/null
+++ b/doc/bugs/git-annex_merge_stalls/comment_1_31578a754945bdcb902c62ff58704bcb._comment
@@ -0,0 +1,17 @@
+[[!comment format=mdwn
+ username="http://joeyh.name/"
+ ip="4.153.255.110"
+ subject="comment 1"
+ date="2013-09-09T15:45:17Z"
+ content="""
+The relevant part of the log is:
+
+
    +65332/0x53591a:  open(\"/Users/gsolsberry/annex/.git/annex/journal.lck\0\", 0x601, 0x1B6)          = 8 0
+65332/0x53591a:  fcntl(0x8, 0x9, 0x107009D60)            = -1 Err#-1
+
    
+
+waitToSetLock thinks fcntl is failing to lock the file due to something else having it locked, and retries, leading to the hang.
+
+I'm told on irc that this was installed using the prebuilt image, and that a previous version of it didn't have the problem.
+"""]]
diff --git a/doc/bugs/git-annex_merge_stalls/comment_2_f3b6bf180466b5931bfd20b2f0229422._comment b/doc/bugs/git-annex_merge_stalls/comment_2_f3b6bf180466b5931bfd20b2f0229422._comment
new file mode 100644
index 0000000000..2fe9b8ac04
--- /dev/null
+++ b/doc/bugs/git-annex_merge_stalls/comment_2_f3b6bf180466b5931bfd20b2f0229422._comment
@@ -0,0 +1,10 @@
+[[!comment format=mdwn
+ username="https://www.google.com/accounts/o8/id?id=AItOawl8B4Ima-VCCJ4y32Gvfii8EmvTyN9tFGM"
+ nickname="Glendon"
+ subject="comment 2"
+ date="2013-09-09T15:54:59Z"
+ content="""
+With latest 20130909 version, the following log from git-annex merge:
+
+https://www.dropbox.com/s/3hklzfsflpxuk5s/annex-merge.log.1
+"""]]
diff --git a/doc/bugs/git-annex_merge_stalls/comment_3_ced9b0d724fb55c756106b64c3721003._comment b/doc/bugs/git-annex_merge_stalls/comment_3_ced9b0d724fb55c756106b64c3721003._comment
new file mode 100644
index 0000000000..a0f452b5dc
--- /dev/null
+++ b/doc/bugs/git-annex_merge_stalls/comment_3_ced9b0d724fb55c756106b64c3721003._comment
@@ -0,0 +1,18 @@
+[[!comment format=mdwn
+ username="http://joeyh.name/"
+ ip="4.153.255.110"
+ subject="comment 3"
+ date="2013-09-09T19:35:25Z"
+ content="""
+This problem cleared up after a reboot (and a crash, apparently).
+
+
    +joeyh so, my thought is that perhaps you had a git-annex process before that was holding the lock.
+joeyh for example, if you ran it and ctrl-z'd at just the right time, it could be suspended and holding the lock
+joeyh (or the kernel coud have gotten confused, which given you also had a crash, who knows..)
+dp sounds logical
+joeyh forcing locks is always a problematic thing
+joeyh but git-annex could certainly notice if it seems to be stalled and print some useful messages
+joeyh and maybe have a way to run with locks forced
+
    
+"""]]
diff --git a/doc/bugs/git-annex_requires_an_SSH_remote_to_have_an_absolute_path.mdwn b/doc/bugs/git-annex_requires_an_SSH_remote_to_have_an_absolute_path.mdwn
new file mode 100644
index 0000000000..23720ebd84
--- /dev/null
+++ b/doc/bugs/git-annex_requires_an_SSH_remote_to_have_an_absolute_path.mdwn
@@ -0,0 +1,37 @@
+### Please describe the problem.
+git-annex prefixes all paths used with SSH remotes with a /, which breaks the default git behavior of using the home directory by default.
+
+### What steps will reproduce the problem?
+Start an SSH server. Use it as a remote with a URL like annex@localhost:test.git or annex@localhost:~/test.git.
+
+### What version of git-annex are you using? On what operating system?
+NixOS
+
+[[!format sh """
+[leo60228@nixos:~/test]$ git annex version
+git-annex version: 6.20180529
+build flags: Assistant Webapp Pairing S3(multipartupload)(storageclasses) WebDAV Inotify DBus DesktopNotify ConcurrentOutput TorrentParser MagicMime Feeds Testsuite
+dependency versions: aws-0.18 bloomfilter-2.0.1.0 cryptonite-0.25 DAV-1.3.2 feed-1.0.0.0 ghc-8.2.2 http-client-0.5.12.1 persistent-sqlite-2.6.4 torrent-10000.1.1 uuid-1.3.13 yesod-1.4.5
+key/value backends: SHA256E SHA256 SHA512E SHA512 SHA224E SHA224 SHA384E SHA384 SHA3_256E SHA3_256 SHA3_512E SHA3_512 SHA3_224E SHA3_224 SHA3_384E SHA3_384 SKEIN256E SKEIN256 SKEIN512E SKEIN512 BLAKE2B256E BLAKE2B256 BLAKE2B512E BLAKE2B512 BLAKE2B160E BLAKE2B160 BLAKE2B224E BLAKE2B224 BLAKE2B384E BLAKE2B384 BLAKE2S256E BLAKE2S256 BLAKE2S160E BLAKE2S160 BLAKE2S224E BLAKE2S224 BLAKE2SP256E BLAKE2SP256 BLAKE2SP224E BLAKE2SP224 SHA1E SHA1 MD5E MD5 WORM URL
+remote types: git gcrypt p2p S3 bup directory rsync web bittorrent webdav adb tahoe glacier ddar hook external
+local repository version: 5
+supported repository versions: 3 5 6
+upgrade supported from repository versions: 0 1 2 3 4 5
+operating system: linux x86_64
+"""]]
+
+### Please provide any additional information below.
+
+[[!format sh """
+# If you can, paste a complete transcript of the problem occurring here.
+# If the problem is with the git-annex assistant, paste in .git/annex/daemon.log
+
+# Don't know how to make a reduced test-case for an issue with a specific protocol.
+
+# End of transcript or log.
+"""]]
+
+### Have you had any luck using git-annex before? (Sometimes we get tired of reading bug reports all day and a lil' positive end note does wonders)
+Yes, I use it on my laptop without issues. I'm trying to set up a server on my desktop.
+
+> [[done]], apparently the reporter was mistaken --[[Joey]] 
diff --git a/doc/bugs/git-annex_requires_an_SSH_remote_to_have_an_absolute_path/comment_1_f328d7248e7eac4d69bc543feb90da7c._comment b/doc/bugs/git-annex_requires_an_SSH_remote_to_have_an_absolute_path/comment_1_f328d7248e7eac4d69bc543feb90da7c._comment
new file mode 100644
index 0000000000..73bd7de882
--- /dev/null
+++ b/doc/bugs/git-annex_requires_an_SSH_remote_to_have_an_absolute_path/comment_1_f328d7248e7eac4d69bc543feb90da7c._comment
@@ -0,0 +1,13 @@
+[[!comment format=mdwn
+ username="joey"
+ subject="""comment 1"""
+ date="2018-08-03T18:19:38Z"
+ content="""
+I use home-relative paths in ssh remotes with git-annex all the time.
+It works fine.
+
+Seems to me that this, as well as your other bug report about using
+[git-annex-shell in a very strange way](http://git-annex.branchable.com/bugs/git-annex-shell_-c_git-annex-shell_doesn__39__t_work__44___while_git-annex_expects_it_to/), suggest that you are doing something
+strange that you need to go into detail about in order for this to be a
+useful bug report.
+"""]]
diff --git a/doc/bugs/git-annex_requires_an_SSH_remote_to_have_an_absolute_path/comment_2_406afc475368b5680be6c99514fbda74._comment b/doc/bugs/git-annex_requires_an_SSH_remote_to_have_an_absolute_path/comment_2_406afc475368b5680be6c99514fbda74._comment
new file mode 100644
index 0000000000..341f72912f
--- /dev/null
+++ b/doc/bugs/git-annex_requires_an_SSH_remote_to_have_an_absolute_path/comment_2_406afc475368b5680be6c99514fbda74._comment
@@ -0,0 +1,9 @@
+[[!comment format=mdwn
+ username="https://openid-provider.appspot.com/iakornfeld"
+ nickname="iakornfeld"
+ avatar="http://cdn.libravatar.org/avatar/c0369f5727cad81d1ecf6c2e657b42a1b756284aad0229351f9027a2cfcb2037"
+ subject="Sorry"
+ date="2018-08-07T14:12:20Z"
+ content="""
+This was me misreading the debug output.
+"""]]
diff --git a/doc/bugs/git-annex_stack.yml_misses_extra-deps..mdwn b/doc/bugs/git-annex_stack.yml_misses_extra-deps..mdwn
new file mode 100644
index 0000000000..56674eeeab
--- /dev/null
+++ b/doc/bugs/git-annex_stack.yml_misses_extra-deps..mdwn
@@ -0,0 +1,92 @@
+### Please describe the problem.
+
+extra-deps in stack.yaml seems to miss a few packages? I am pretty new to stack, so maybe
+I am misinterpreting, but following stacks suggestion below and adding 
+
+- bloomfilter-2.0.1.0
+- network-multicast-0.2.0
+- torrent-10000.0.1
+
+to my stack.yaml lets me build the latest master branch on jessie.
+
+### What steps will reproduce the problem?
+
+Checking out the latest commit (661dae3649f10a07ba6091a9a376ae1049fed716) and running "stack setup && stack install"
+should reproduce it? See transcript below.
+
+### What version of git-annex are you using? On what operating system?
+
+6.20170101-android on Debian Jessie. Stack Version 1.3.2
+
+
+### Please provide any additional information below.
+
+[[!format sh """
+
+$ git clone git://git-annex.branchable.com/ git-annex                                                                                                                                
+Cloning into 'git-annex'...
+remote: Counting objects: 178001, done.
+remote: Compressing objects: 100% (49612/49612), done.
+remote: Total 178001 (delta 127721), reused 178001 (delta 127721)
+Receiving objects: 100% (178001/178001), 43.14 MiB | 1.47 MiB/s, done.
+Resolving deltas: 100% (127721/127721), done.
+Checking connectivity... done.
+$ cd git-annex
+$ stack setup
+Downloaded lts-7.18 build plan.    
+Fetching package index ...remote: Counting objects: 1887, done.
+remote: Compressing objects: 100% (164/164), done.
+remote: Total 1887 (delta 676), reused 641 (delta 641), pack-reused 1079
+Receiving objects: 100% (1887/1887), 1.59 MiB | 904.00 KiB/s, done.
+Resolving deltas: 100% (757/757), completed with 311 local objects.
+From https://github.com/commercialhaskell/all-cabal-hashes
+   3ae7c75..ca0bc8e  hackage    -> origin/hackage
+ - [tag update]      current-hackage -> current-hackage
+Fetched package index.    
+Populated index cache.    
+stack will use a sandboxed GHC it installed
+For more information on paths, see 'stack path' and 'stack exec env'
+To use this GHC and packages outside of a project, consider using:
+stack ghc, stack ghci, stack runghc, or stack exec
+$ stack install
+Cabal file warning in /home/phaer/projects/external/git-annex/git-annex.cabal: Ignoring unknown section type: custom-setup
+
+Error: While constructing the build plan, the following exceptions were encountered:
+
+In the dependencies for git-annex-6.20170101(-android
+-androidsplice
++assistant
++concurrentoutput
++cryptonite
+-dbus
+-magicmime
++network-uri
++pairing
++production
++s3
++testsuite
++torrentparser
++webapp
++webdav):
+    bloomfilter must match -any, but the stack configuration has no specified version (latest applicable is 2.0.1.0)
+    network-multicast must match -any, but the stack configuration has no specified version (latest applicable is 0.2.0)
+    torrent must match (>=10000.0.0), but the stack configuration has no specified version
+            (latest applicable is 10000.0.1)
+
+Recommended action: try adding the following to your extra-deps in /home/phaer/projects/external/git-annex/stack.yaml:
+- bloomfilter-2.0.1.0
+- network-multicast-0.2.0
+- torrent-10000.0.1
+
+You may also want to try the 'stack solver' command
+Plan construction failed.
+
+"""]]
+
+### Have you had any luck using git-annex before? (Sometimes we get tired of reading bug reports all day and a lil' positive end note does wonders)
+
+Yes, I am using it since a few years to manage my videos, photos and music. Thanks so much for this outstanding piece of software!
+
+> Seems that, since I have those packages globally installed, stack solver 
+> did not notice that they needed to be listed there. Hrm. Ok well, added
+> them by hand. [[done]] --[[Joey]] 
diff --git a/doc/bugs/git-annex_sucking_up_all_available_RAM_after_startup.mdwn b/doc/bugs/git-annex_sucking_up_all_available_RAM_after_startup.mdwn
new file mode 100644
index 0000000000..69383666c2
--- /dev/null
+++ b/doc/bugs/git-annex_sucking_up_all_available_RAM_after_startup.mdwn
@@ -0,0 +1,49 @@
+Hi.
+
+trying to manage my collection of digital music files using git-annex. The collection (113 gigs of flac files ripped from my CDs) should be stored on my three different machines and updated on all of them, if I add or change a file on only one of the machines.
+
+### Please describe the problem.
+
+Added a new external USB disk for sneaker transfer via web app, yesterday.
+
+Now for no apparent reason, after startup/login, git-annex would start and quickly suck up all available RAM. This is on a fairly well equipped machine (16G physical RAM, i5-2400), yet "top" tells me that there is one git process that sucks up more than 20G and climbing. It looked like this:
+
+    git --git-dir=/home/user/Sync/Audio/.git --work-tree=/home/user/Sync/Audio -c core.bare=false log refs/heads git-annex..13d365f16ffdb5a393f66362b840d3f21bb4c59c --oneline -n1
+
+The computer then slows down, grinds to halt, becomes unresponsive and it's difficult to even login on the console.
+
+Then, the OOM killer kicks in and kicks the git process, but git-annex quickly starts another which does the same.
+
+### What steps will reproduce the problem?
+
+I don't know what caused it. The symptoms remained after a reboot, "git annex watch --stop" didn't help either, since I'm a dumb web app user, I'm not sure if that's the right command to use anyway.
+
+For now, I have removed git-annex from the system.
+
+### What version of git-annex are you using? On what operating system?
+
+Last installed version: git-annex 5.20140127.1 on Ubuntu 13.10, amd64.
+
+### Please provide any additional information below.
+
+I'm fairly unsure where to look for the cause and what logs to provide you with to help fix this. Just guessing that it could be a symptom, but the daemon.log is full of entries like this:
+
+[[!format sh """
+# If you can, paste a complete transcript of the problem occurring here.
+# If the problem is with the git-annex assistant, paste in .git/annex/daemon.log
+
+("race detected",ca2cbdb84bcbd4aab895284b16fc72f693fbba90,[4a2e7c1d7d286a4da9e816b20368ce2f9b4177c4],"committing",(ca2cbdb84bcbd4aab895284b16fc72f693fbba90,[ca2cbdb84bcbd4aab895284b16fc72f693fbba90]))
+(Recording state in git...)
+("race detected",28c835634e65ced0e532c1a0e4f34dd0344193bc,[19597be0f49fb859fafa51e006459d5a95e3d005],"committing",(28c835634e65ced0e532c1a0e4f34dd0344193bc,[28c835634e65ced0e532c1a0e4f34dd0344193bc]))
+(Recording state in git...)
+("race detected",1f2b06c7001be38bd9595eb2205c91454597edaa,[398660279436246a698d6bd55eb06998999ed64f],"committing",(1f2b06c7001be38bd9595eb2205c91454597edaa,[1f2b06c7001be38bd9595eb2205c91454597edaa]))
+(Recording state in git...)
+("race detected",4c1510c3db41ff400526d5753c03bddc48f5c37e,[1989177cf24ec9151058ed99f05117e48c239001],"committing",(4c1510c3db41ff400526d5753c03bddc48f5c37e,[4c1510c3db41ff400526d5753c03bddc48f5c37e]))
+(Recording state in git...)
+("race detected",b82f41fcbf24c43fe9f1f9d6fb54ba5ef9ff8de0,[799e4434447b18be63bd097120e1fbf56eac48ce],"committing",(b82f41fcbf24c43fe9f1f9d6fb54ba5ef9ff8de0,[b82f41fcbf24c43fe9f1f9d6fb54ba5ef9ff8de0]))
+(Recording state in git...)
+
+# End of transcript or log.
+"""]]
+
+[[!tag moreinfo]]
diff --git a/doc/bugs/git-annex_sucking_up_all_available_RAM_after_startup/comment_1_b550f292359b44977481bf69abad4012._comment b/doc/bugs/git-annex_sucking_up_all_available_RAM_after_startup/comment_1_b550f292359b44977481bf69abad4012._comment
new file mode 100644
index 0000000000..662141d64a
--- /dev/null
+++ b/doc/bugs/git-annex_sucking_up_all_available_RAM_after_startup/comment_1_b550f292359b44977481bf69abad4012._comment
@@ -0,0 +1,14 @@
+[[!comment format=mdwn
+ username="https://www.google.com/accounts/o8/id?id=AItOawkVnM7Ol2kr_jfLg6h_oXpmNdxWHIiL9mk"
+ nickname="Hanno"
+ subject="comment 1"
+ date="2014-01-30T11:18:15Z"
+ content="""
+In case it helps:
+
+    $ grep \"race detected\" daemon.log.5 | wc -l
+    234986
+
+
+
+"""]]
diff --git a/doc/bugs/git-annex_sucking_up_all_available_RAM_after_startup/comment_2_76e6c1d4db27bcc1767ba34e13e8211c._comment b/doc/bugs/git-annex_sucking_up_all_available_RAM_after_startup/comment_2_76e6c1d4db27bcc1767ba34e13e8211c._comment
new file mode 100644
index 0000000000..dacaac9421
--- /dev/null
+++ b/doc/bugs/git-annex_sucking_up_all_available_RAM_after_startup/comment_2_76e6c1d4db27bcc1767ba34e13e8211c._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="http://joeyh.name/"
+ ip="209.250.56.199"
+ subject="comment 2"
+ date="2014-01-30T17:39:47Z"
+ content="""
+How many files are in the repository?
+"""]]
diff --git a/doc/bugs/git-annex_sucking_up_all_available_RAM_after_startup/comment_3_14007c8e927b75c5706e80cc4242fae4._comment b/doc/bugs/git-annex_sucking_up_all_available_RAM_after_startup/comment_3_14007c8e927b75c5706e80cc4242fae4._comment
new file mode 100644
index 0000000000..90101155d5
--- /dev/null
+++ b/doc/bugs/git-annex_sucking_up_all_available_RAM_after_startup/comment_3_14007c8e927b75c5706e80cc4242fae4._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="Hanno"
+ ip="85.183.3.94"
+ subject="comment 3"
+ date="2014-01-31T10:39:47Z"
+ content="""
+The folder contains a bit more than 15000 files (counted with find -type f)
+"""]]
diff --git a/doc/bugs/git-annex_sucking_up_all_available_RAM_after_startup/comment_4_f3266b74517b421e5310e67818fe3969._comment b/doc/bugs/git-annex_sucking_up_all_available_RAM_after_startup/comment_4_f3266b74517b421e5310e67818fe3969._comment
new file mode 100644
index 0000000000..faaa7ed09a
--- /dev/null
+++ b/doc/bugs/git-annex_sucking_up_all_available_RAM_after_startup/comment_4_f3266b74517b421e5310e67818fe3969._comment
@@ -0,0 +1,12 @@
+[[!comment format=mdwn
+ username="https://www.google.com/accounts/o8/id?id=AItOawkVnM7Ol2kr_jfLg6h_oXpmNdxWHIiL9mk"
+ nickname="Hanno"
+ subject="comment 4"
+ date="2014-01-31T23:17:49Z"
+ content="""
+I have no idea if it is related to this problem, but allow me to add that the \"Consistency check\" (that the web app specifically recommended me to activate) makes my computers unreasonably slow immediately after each bootup. This is both at the fairly fast computer at the office and the ok-ish old laptop at home.
+
+Also, the check will kick in and suck up the systems responsiveness right after connecting the external USB sneaker disk, making any regular file transfer to the disk extremely slow (more than once I was stung because I just wanted to quickly move a bigger file to the disk and walk home at the end of an office day but git-annex would go into its check after mounting).
+
+Thanks!
+"""]]
diff --git a/doc/bugs/git-annex_sucking_up_all_available_RAM_after_startup/comment_5_1a07f15eb0353768c1e67a1e47e2e494._comment b/doc/bugs/git-annex_sucking_up_all_available_RAM_after_startup/comment_5_1a07f15eb0353768c1e67a1e47e2e494._comment
new file mode 100644
index 0000000000..d94eda8168
--- /dev/null
+++ b/doc/bugs/git-annex_sucking_up_all_available_RAM_after_startup/comment_5_1a07f15eb0353768c1e67a1e47e2e494._comment
@@ -0,0 +1,9 @@
+[[!comment format=mdwn
+ username="Hanno"
+ ip="85.183.3.94"
+ subject="comment 5"
+ date="2014-02-27T16:10:54Z"
+ content="""
+Is there anything I can do to help making progress on this issue?
+Right now I'm afraid to reinstall git-annex on this machine.
+"""]]
diff --git a/doc/bugs/git-annex_sucking_up_all_available_RAM_after_startup/comment_6_7d0d49fd165af5e30606982e05335d34._comment b/doc/bugs/git-annex_sucking_up_all_available_RAM_after_startup/comment_6_7d0d49fd165af5e30606982e05335d34._comment
new file mode 100644
index 0000000000..bd6fcad6ab
--- /dev/null
+++ b/doc/bugs/git-annex_sucking_up_all_available_RAM_after_startup/comment_6_7d0d49fd165af5e30606982e05335d34._comment
@@ -0,0 +1,10 @@
+[[!comment format=mdwn
+ username="http://joeyh.name/"
+ ip="108.236.230.124"
+ subject="comment 6"
+ date="2014-03-10T19:24:45Z"
+ content="""
+I suppose it would be useful to look at the output of `git log git-annex..13d365f16ffdb5a393f66362b840d3f21bb4c59c --oneline -n1`
+
+Either than command is outputting a huge amount of stuff, or it's actually causing git (not git-annex) to use a lot of memory. Not sure which it is from the description.
+"""]]
diff --git a/doc/bugs/git-annex_sucking_up_all_available_RAM_after_startup/comment_7_2b0ba2a15af04731a966a029be9b81aa._comment b/doc/bugs/git-annex_sucking_up_all_available_RAM_after_startup/comment_7_2b0ba2a15af04731a966a029be9b81aa._comment
new file mode 100644
index 0000000000..39e0e44e9e
--- /dev/null
+++ b/doc/bugs/git-annex_sucking_up_all_available_RAM_after_startup/comment_7_2b0ba2a15af04731a966a029be9b81aa._comment
@@ -0,0 +1,11 @@
+[[!comment format=mdwn
+ username="https://launchpad.net/~stephane-gourichon-lpad"
+ nickname="stephane-gourichon-lpad"
+ avatar="http://cdn.libravatar.org/avatar/02d4a0af59175f9123720b4481d55a769ba954e20f6dd9b2792217d9fa0c6089"
+ subject="comment 7"
+ date="2017-07-20T06:16:46Z"
+ content="""
+Hello.
+
+Any progress on this? (With regular git annex I now have a big repository with some corruption and `git annex repair` causes oom-kill.)
+"""]]
diff --git a/doc/bugs/git-annex_wants_to_repair_because_of_duplicateEntries_in_git_fsck.mdwn b/doc/bugs/git-annex_wants_to_repair_because_of_duplicateEntries_in_git_fsck.mdwn
new file mode 100644
index 0000000000..a9ed43785d
--- /dev/null
+++ b/doc/bugs/git-annex_wants_to_repair_because_of_duplicateEntries_in_git_fsck.mdwn
@@ -0,0 +1,61 @@
+### Please describe the problem.
+One of my annexes has duplicate entries in one of the older trees. This causes git-annex repair to attempt to repair it.
+
+### What version of git-annex are you using? On what operating system?
+6.20161027-g27d824e (standalone x86_64 tarball) on ArchLinux latest as of 29th of Oct 2016 x86_64.
+
+### Please provide any additional information below.
+
+[[!format sh """
+[0 zerodogg@firefly annexed]$ git annex version
+git-annex version: 6.20161027-g27d824e
+build flags: Assistant Webapp Pairing Testsuite S3(multipartupload)(storageclasses) WebDAV Inotify DBus DesktopNotify XMPP ConcurrentOutput TorrentParser MagicMime Feeds Quvi
+key/value backends: SHA256E SHA256 SHA512E SHA512 SHA224E SHA224 SHA384E SHA384 SHA3_256E SHA3_256 SHA3_512E SHA3_512 SHA3_224E SHA3_224 SHA3_384E SHA3_384 SKEIN256E SKEIN256 SKEIN512E SKEIN512 SHA1E SHA1 MD5E MD5 WORM URL
+remote types: git gcrypt S3 bup directory rsync web bittorrent webdav tahoe glacier ddar hook external
+local repository version: 6
+supported repository versions: 3 5 6
+upgrade supported from repository versions: 0 1 2 3 4 5
+operating system: linux x86_64
+[0 zerodogg@firefly annexed]$ git annex info                             
+repository mode: indirect
+trusted repositories: 0
+semitrusted repositories: 23
+# …snip
+untrusted repositories: 0
+transfers in progress: none
+available local disk space: 47.89 gigabytes (+500 megabytes reserved)
+local annex keys: 2300
+local annex size: 1.72 gigabytes
+annexed files in working tree: 2492
+size of annexed files in working tree: 1.89 gigabytes
+bloom filter size: 32 mebibytes (0.5% full)
+backend usage:
+        SHA256E: 2492
+[0 zerodogg@firefly annexed]$ git fsck
+Checking object directories: 100% (256/256), done.
+error in tree 3818a8ea201197ed355aa87133a718afca7c4a49: duplicateEntries: contains duplicate file entries
+error in tree 255f43cff847013439117a3fc4cfa517a90f38cb: duplicateEntries: contains duplicate file entries
+error in tree 404b4124800ece4772296d1f7559d982d0bc4cbd: duplicateEntries: contains duplicate file entries
+Checking objects: 100% (334784/334784), done.
+Checking connectivity: 334784, done.
+[4 zerodogg@firefly annexed]$ git annex repair -d
+Running git fsck ...
+[2016-10-29 09:55:24.652234] read: git ["--version"]
+[2016-10-29 09:55:24.668729] process done ExitSuccess
+[2016-10-29 09:55:24.668879] read: git ["--git-dir=.git","--work-tree=.","--literal-pathspecs","fsck","--no-dangling","--no-reflogs"]
+[2016-10-29 09:55:34.306776] process done ExitFailure 4
+Unpacking all pack files.
+[2016-10-29 09:55:34.307446] call: mv ["-f",".git/objects/pack/pack-18d920ca3181f59af68dc6b0061ca2430884a9bb.pack","/tmp/packsFokxCF/pack-18d920ca3181f59af68dc6b0061ca243
+0884a9bb.pack1804289383846930886.tmp"]
+[2016-10-29 09:55:34.334238] process done ExitSuccess
+[2016-10-29 09:55:34.336992] feed: git ["--git-dir=.git","--work-tree=.","--literal-pathspecs","unpack-objects","-r"]
+Unpacking objects: 100% (334784/334784), done.
+[2016-10-29 09:56:39.505888] process done ExitSuccess
+[2016-10-29 09:56:39.560473] read: git ["--git-dir=.git","--work-tree=.","--literal-pathspecs","show","10c1cc2127a7be7cf3ce4a0d69ed53e4c375f888"]
+[2016-10-29 09:56:39.607717] process done ExitSuccess
+# ^C
+# End of transcript or log.
+"""]]
+
+### Have you had any luck using git-annex before? (Sometimes we get tired of reading bug reports all day and a lil' positive end note does wonders)
+I use it for everything, around 3TB of data at the moment. It is awesome.
diff --git a/doc/bugs/git-annex_wants_to_repair_because_of_duplicateEntries_in_git_fsck/comment_1_4cd4f4b5c34b3e1bf2989d0cc412ca4f._comment b/doc/bugs/git-annex_wants_to_repair_because_of_duplicateEntries_in_git_fsck/comment_1_4cd4f4b5c34b3e1bf2989d0cc412ca4f._comment
new file mode 100644
index 0000000000..ddda442e0f
--- /dev/null
+++ b/doc/bugs/git-annex_wants_to_repair_because_of_duplicateEntries_in_git_fsck/comment_1_4cd4f4b5c34b3e1bf2989d0cc412ca4f._comment
@@ -0,0 +1,34 @@
+[[!comment format=mdwn
+ username="joey"
+ subject="""comment 1"""
+ date="2016-10-31T17:07:43Z"
+ content="""
+This was supposed to be dealt with in version 6.20161027. But, I see now
+that the parser I thought was triggering on those missing object lines,
+was not really (`extractSha` does not match in this case because of the
+colon after the sha). 
+
+Instead, the problem seems to be that `git fsck` is exiting nonzero. So it
+assumes that fsck is failing without printing out any shas, which is a
+condition that calls for repairs.
+
+About all I can think to do is, if fsck outputs "duplicateEntries" and
+no other lines at all, and exits nonzero, treat this as a success.
+This risks ignoring other reasons fsck might exit nonzero, but hopefully
+it would output something else in such a case. I've implemented this.
+
+---
+
+I am interested in getting at the root cause of the problem of
+duplicate directory entries. It seems pretty likely to result from using
+adjusted branches.
+
+It would be useful to get more information about the trees that fsck
+is warning about; are they part of existing or past adjusted branches
+or not? Are they merge commits?
+
+(Previously: [[forum/how_to_disaster_recovery]])
+
+Leaving this bug open since we really need to get at the root cause of
+the problem.
+"""]]
diff --git a/doc/bugs/git-annex_wants_to_repair_because_of_duplicateEntries_in_git_fsck/comment_2_7c65a600fdcb43b9a3aabf7182b67b2f._comment b/doc/bugs/git-annex_wants_to_repair_because_of_duplicateEntries_in_git_fsck/comment_2_7c65a600fdcb43b9a3aabf7182b67b2f._comment
new file mode 100644
index 0000000000..c37ebb0394
--- /dev/null
+++ b/doc/bugs/git-annex_wants_to_repair_because_of_duplicateEntries_in_git_fsck/comment_2_7c65a600fdcb43b9a3aabf7182b67b2f._comment
@@ -0,0 +1,45 @@
+[[!comment format=mdwn
+ username="EskildHustvedt"
+ avatar="http://cdn.libravatar.org/avatar/0be1310904ded29624b9edb4824d451b"
+ subject="comment 2"
+ date="2016-11-04T07:34:30Z"
+ content="""
+Ok, first off, the latest release no longer wants to release, so at least the symptom is fixed.
+
+Here's a quick look at the problem:
+
+[[!format sh \"\"\"
+[0 zerodogg@firefly annexed]$ git fsck          
+Checking object directories: 100% (256/256), done.
+error in tree 3818a8ea201197ed355aa87133a718afca7c4a49: duplicateEntries: contains duplicate file entries
+error in tree 255f43cff847013439117a3fc4cfa517a90f38cb: duplicateEntries: contains duplicate file entries
+error in tree 404b4124800ece4772296d1f7559d982d0bc4cbd: duplicateEntries: contains duplicate file entries
+Checking objects: 100% (347890/347890), done.
+Checking connectivity: 347890, done.
+[4 zerodogg@firefly annexed]$ GIT_PAGER=cat git show 3818a8ea201197ed355aa87133a718afca7c4a49
+tree 3818a8ea201197ed355aa87133a718afca7c4a49
+
+Diverse/
+Dokument/
+Dokument/
+NBGI/
+Politikk/
+Skule/
+btfixie.ods
+endless space/
+monolith productions/
+my games/
+[4 zerodogg@firefly annexed]$ GIT_PAGER=cat git ls-tree 404b4124800ece4772296d1f7559d982d0bc4cbd 
+040000 tree 2efffc4a0eb7df3fde68e7a222926ff4b91ede4a    Diverse
+040000 tree cd854aba649cee8855ec72579f4c98100471f7cf    Dokument
+040000 tree 1ae499997746c8e976cf02edc2276532978dafc5    Dokument
+040000 tree ec8256eb106836a356aa7818f5b579f78393820e    NBGI
+040000 tree 384904d7414811bd41ee36b4d2ae2555192bd49f    Politikk
+040000 tree 0ac0cf99f02852d45518ffcbd42a0edc9e07dfd4    Skule
+040000 tree a20b8cc805d1612b60d0dfab69a49788e0467ca6    endless space
+040000 tree 9463eeb6cfcaa87491bf7aec03d10cf5edd1ac7d    monolith productions
+040000 tree 0ce07221be630cbbf038d8a5f8534cb599d0daf4    my games
+\"\"\"]]
+
+From here I'm not sure how to figure out where to go. I do have a copy of the original working copy where the problem first appeared (it has now propagated to all copies), if you want a copy to dig around in. I'll also be happy to provide more information if you specify what you need.
+"""]]
diff --git a/doc/bugs/git-annex_won__39__t_execute_on_WD_My_Cloud_NAS.mdwn b/doc/bugs/git-annex_won__39__t_execute_on_WD_My_Cloud_NAS.mdwn
new file mode 100644
index 0000000000..abda583809
--- /dev/null
+++ b/doc/bugs/git-annex_won__39__t_execute_on_WD_My_Cloud_NAS.mdwn
@@ -0,0 +1,58 @@
+### Please describe the problem.
+
+I'm trying to get git-annex running on a WD MyCloud EX2 NAS.
+
+There are a number of issues, I imagine to overcome, a few of which I've already managed to overcome.
+
+
+### What steps will reproduce the problem?
+
+I've managed to get it to work as special remote via rsync. But, it's nowhere near ideal. I was hoping to get working properly by installing git-annex locally on the machine.
+
+FYI, git 2.6.1 is already installed on the NAS.
+
+I can ssh into the NAS no problem. Only user sshd is allowed to ssh, but that's ok. Furthermore, the which command is missing, which prevented webapp lunched on my Mac from creating an rsync remote. Overcame that hurdle by "faking" it:
+[[!format sh """
+    > echo \#\!\/bin\/sh > /usr/bin/which
+    > echo command \-v \$\@ >> /usr/bin/which
+    > chmod +x /usr/bin/which
+"""]]
+
+I've downloaded the latest arm build. After untaring and running runshell successfully, I can't get through the next step, which is running git annex. It results in the following error:
+[[!format sh """
+    > git annex
+    /mnt/HD/HD_a2/git-annex/shimmed/git-annex/git-annex: error while loading shared libraries: /mnt/HD/HD_a2/git-annex/shimmed/git-annex/git-annex: ELF load command alignment not page-aligned
+    fatal: 'annex' appears to be a git command, but we were not able to execute it. Maybe git-annex is broken?
+"""]]
+
+same goes for
+[[!format sh """
+    > runshell git-annex-shell -c git annex:
+    /mnt/HD/HD_a2/Nas_Prog/git-annex/shimmed/git-annex-shell/git-annex-shell: error while loading shared libraries: /mnt/HD/HD_a2/Nas_Prog/git-annex/shimmed/git-annex-shell/git-annex-shell: ELF load command alignment not page-aligned
+"""]]
+
+and pretty much any of the git-annex binaries.
+
+Compiling from source is not an option.
+Any ideas? Thanks.
+
+### What version of git-annex are you using? On what operating system?
+
+I'm trying the latest version of git-annex-standalone-armel.tar.gz downloaded 2015-10-10.
+
+uname -a:  Linux WDMyCloudEX2 3.2.40 #4 Fri Jul 31 16:04:18 CST 2015 armv7l GNU/Linux
+
+
+### Please provide any additional information below.
+
+[[!format sh """
+# If you can, paste a complete transcript of the problem occurring here.
+# If the problem is with the git-annex assistant, paste in .git/annex/daemon.log
+
+
+# End of transcript or log.
+"""]]
+
+### Have you had any luck using git-annex before? (Sometimes we get tired of reading bug reports all day and a lil' positive end note does wonders)
+
+Sure, runs on my Mac laptop
diff --git a/doc/bugs/git-annex_won__39__t_execute_on_WD_My_Cloud_NAS/comment_1_b87b50ca08bf90ecb7d741a6baa1dc4d._comment b/doc/bugs/git-annex_won__39__t_execute_on_WD_My_Cloud_NAS/comment_1_b87b50ca08bf90ecb7d741a6baa1dc4d._comment
new file mode 100644
index 0000000000..00f2c08060
--- /dev/null
+++ b/doc/bugs/git-annex_won__39__t_execute_on_WD_My_Cloud_NAS/comment_1_b87b50ca08bf90ecb7d741a6baa1dc4d._comment
@@ -0,0 +1,17 @@
+[[!comment format=mdwn
+ username="joey"
+ subject="""comment 1"""
+ date="2015-10-15T17:04:07Z"
+ content="""
+I wonder if this ELF issue extends to other binaries included in the
+git-annex bundle, or does it only affect the git-annex (and
+git-annex-shell) binary?
+
+It looks like git is working when used inside runshell, and that is using a
+git binary from the git-annex bundle.
+
+If so, it could have something to do with how the haskell binary is being
+linked. I found this thread about fixing this problem in the gold linker:
+. IIRC
+the arm build is currently using the bfd linker, not gold.
+"""]]
diff --git a/doc/bugs/git-annex_won__39__t_execute_on_WD_My_Cloud_NAS/comment_2_6c95d0fd70fbf41eea4e6575486af3f5._comment b/doc/bugs/git-annex_won__39__t_execute_on_WD_My_Cloud_NAS/comment_2_6c95d0fd70fbf41eea4e6575486af3f5._comment
new file mode 100644
index 0000000000..a161f6da71
--- /dev/null
+++ b/doc/bugs/git-annex_won__39__t_execute_on_WD_My_Cloud_NAS/comment_2_6c95d0fd70fbf41eea4e6575486af3f5._comment
@@ -0,0 +1,48 @@
+[[!comment format=mdwn
+ username="PaulK"
+ subject="comment 2"
+ date="2015-10-16T01:32:37Z"
+ content="""
+Yup. It seems to affect mainly git-annex binaries...
+
+[[!format  sh \"\"\"
+
+# ./git-annex
+/mnt/HD/git-annex/shimmed/git-annex/git-annex: error while loading shared libraries: /mnt/HD/git-annex/shimmed/git-annex/git-annex: ELF load command alignment not page-aligned
+
+# ./git-annex-shell 
+/mnt/HD/git-annex/shimmed/git-annex-shell/git-annex-shell: error while loading shared libraries: /mnt/HD/git-annex/shimmed/git-annex-shell/git-annex-shell: ELF load command alignment not page-aligned
+
+# ./git-annex-webapp 
+/mnt/HD/git-annex/shimmed/git-annex/git-annex: error while loading shared libraries: /mnt/HD/git-annex/shimmed/git-annex/git-annex: ELF load command alignment not page-aligned
+
+# ./git-shell
+fatal: Interactive git shell is not enabled.
+hint: ~/git-shell-commands should exist and have read and execute access.
+
+# ./git-upload-pack
+usage: git upload-pack [--strict] [--timeout=] 
+
+# ./shimmed/wget/wget
+./shimmed/wget/wget: error while loading shared libraries: libpcre.so.3: cannot open shared object file: No such file or directory
+
+# ./runshell which wget
+/mnt/HD/git-annex/bin/wget
+
+# ./runshell wget
+wget: missing URL
+Usage: wget [OPTION]... [URL]...
+
+Try `wget --help' for more options.
+
+# ./runshell which git 
+/mnt/HD/git-annex/bin/git
+
+# ./runshell git annex
+/mnt/HD/git-annex/shimmed/git-annex/git-annex: error while loading shared libraries: /mnt/HD/git-annex/shimmed/git-annex/git-annex: ELF load command alignment not page-aligned
+fatal: 'annex' appears to be a git command, but we were not
+able to execute it. Maybe git-annex is broken?
+
+\"\"\"]]
+
+"""]]
diff --git a/doc/bugs/git-annex_won__39__t_execute_on_WD_My_Cloud_NAS/comment_3_6dd54179aad379bb22638db518bd377f._comment b/doc/bugs/git-annex_won__39__t_execute_on_WD_My_Cloud_NAS/comment_3_6dd54179aad379bb22638db518bd377f._comment
new file mode 100644
index 0000000000..73eea64896
--- /dev/null
+++ b/doc/bugs/git-annex_won__39__t_execute_on_WD_My_Cloud_NAS/comment_3_6dd54179aad379bb22638db518bd377f._comment
@@ -0,0 +1,7 @@
+[[!comment format=mdwn
+ username="sts"
+ subject="comment 3"
+ date="2016-06-23T12:07:10Z"
+ content="""
+Is there a way to link the objects with the gold linker? I am running into the same issue. Currently I am doing a workaround with mounting the drive via NFS, but that has quite poor performance (all git-related things).
+"""]]
diff --git a/doc/bugs/git-annex_won__39__t_execute_on_WD_My_Cloud_NAS/comment_4_614da8cad46df5c71539cc5a8ed11831._comment b/doc/bugs/git-annex_won__39__t_execute_on_WD_My_Cloud_NAS/comment_4_614da8cad46df5c71539cc5a8ed11831._comment
new file mode 100644
index 0000000000..6481abb733
--- /dev/null
+++ b/doc/bugs/git-annex_won__39__t_execute_on_WD_My_Cloud_NAS/comment_4_614da8cad46df5c71539cc5a8ed11831._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="PaulK"
+ subject="comment 4"
+ date="2016-09-19T04:39:07Z"
+ content="""
+So, does anyone out there have access to this gold linker, and could they please try to link it for the armv7l architecture?
+Thanks.
+"""]]
diff --git a/doc/bugs/git-annex_won__39__t_execute_on_WD_My_Cloud_NAS/comment_5_5fa9d10ebd639eb1a836866b99e90aab._comment b/doc/bugs/git-annex_won__39__t_execute_on_WD_My_Cloud_NAS/comment_5_5fa9d10ebd639eb1a836866b99e90aab._comment
new file mode 100644
index 0000000000..a4784ced58
--- /dev/null
+++ b/doc/bugs/git-annex_won__39__t_execute_on_WD_My_Cloud_NAS/comment_5_5fa9d10ebd639eb1a836866b99e90aab._comment
@@ -0,0 +1,10 @@
+[[!comment format=mdwn
+ username="joey"
+ subject="""comment 5"""
+ date="2016-09-21T16:44:08Z"
+ content="""
+I've updated the [arm daily build](https://downloads.kitenet.net/git-annex/autobuild/armel/git-annex-standalone-armel.tar.gz) to use ld.gold temporarily. 
+
+Please give it a try and report back if it works and if so I'll
+make the change permanant.
+"""]]
diff --git a/doc/bugs/git-annex_won__39__t_execute_on_WD_My_Cloud_NAS/comment_6_8e05f8769870b3f1aec77fca06f37284._comment b/doc/bugs/git-annex_won__39__t_execute_on_WD_My_Cloud_NAS/comment_6_8e05f8769870b3f1aec77fca06f37284._comment
new file mode 100644
index 0000000000..d56e90c83a
--- /dev/null
+++ b/doc/bugs/git-annex_won__39__t_execute_on_WD_My_Cloud_NAS/comment_6_8e05f8769870b3f1aec77fca06f37284._comment
@@ -0,0 +1,7 @@
+[[!comment format=mdwn
+ username="PaulK"
+ subject="comment 6"
+ date="2016-09-24T04:16:51Z"
+ content="""
+Nope, no difference. Same \"ELF load command alignment not page-aligned\" errors as before.
+"""]]
diff --git a/doc/bugs/git-annex_won__39__t_execute_on_WD_My_Cloud_NAS/comment_7_8da1f5fc6656e92dc2e3dabd9808f2d1._comment b/doc/bugs/git-annex_won__39__t_execute_on_WD_My_Cloud_NAS/comment_7_8da1f5fc6656e92dc2e3dabd9808f2d1._comment
new file mode 100644
index 0000000000..05fe1010a8
--- /dev/null
+++ b/doc/bugs/git-annex_won__39__t_execute_on_WD_My_Cloud_NAS/comment_7_8da1f5fc6656e92dc2e3dabd9808f2d1._comment
@@ -0,0 +1,13 @@
+[[!comment format=mdwn
+ username="PaulK"
+ subject="comment 7"
+ date="2016-09-25T03:08:40Z"
+ content="""
+As an aside, just tried syncthing's arm verion, and I get a \"runtime: kernel page size (32768) is larger than runtime page size (4096)\" error. While their arm64 version also won't run.  
+
+Perhaps it's a similar issue with git-annex? Seems the page-size comes up in a number of contexts of people trying to get software running on the WD NAS.
+
+Is there any way to adjust  this page-size in the linker?
+
+Thanks
+"""]]
diff --git a/doc/bugs/git-annex_won__39__t_execute_on_WD_My_Cloud_NAS/comment_8_48026cf7c187e97d53d15d35ed2c3670._comment b/doc/bugs/git-annex_won__39__t_execute_on_WD_My_Cloud_NAS/comment_8_48026cf7c187e97d53d15d35ed2c3670._comment
new file mode 100644
index 0000000000..493031115a
--- /dev/null
+++ b/doc/bugs/git-annex_won__39__t_execute_on_WD_My_Cloud_NAS/comment_8_48026cf7c187e97d53d15d35ed2c3670._comment
@@ -0,0 +1,14 @@
+[[!comment format=mdwn
+ username="joey"
+ subject="""comment 8"""
+ date="2016-11-16T21:48:49Z"
+ content="""
+The arm daily build now uses a 32kb page size. So try
+
+
+That has been verified to fix the problem on a Drobo 5N.
+
+This may still not be enough for some of the affected NAS devices, which
+use a 64kb page size. Unfortunately, gold fails to link with a 64kb page
+size: 
+"""]]
diff --git a/doc/bugs/git_annex___36__command_--help_not_recoginized.mdwn b/doc/bugs/git_annex___36__command_--help_not_recoginized.mdwn
new file mode 100644
index 0000000000..a48221edf5
--- /dev/null
+++ b/doc/bugs/git_annex___36__command_--help_not_recoginized.mdwn
@@ -0,0 +1,53 @@
+### Please describe the problem.
+
+"git annex help" says to run "git-annex command --help", but doing so reports unrecognized option.
+
+
+### What steps will reproduce the problem?
+
+15:25:14 -0500 adam@tooz:/tmp$ git annex help
+The most frequently used git-annex commands are:
+        init    initialize git-annex
+        add     add files to annex
+        drop    indicate content of files not currently wanted
+        get     make content of annexed files available
+        move    move content of files to/from another repository
+        copy    copy content of files to/from another repository
+        sync    synchronize local repository with remotes
+        whereis lists repositories that have file content
+        fsck    check for problems
+
+Run 'git-annex' for a complete command list.
+Run 'git-annex command --help' for help on a specific command.
+Run `git annex help options' for a list of common options.
+
+15:26:27 -0500 adam@tooz:/tmp$ git annex add --help
+git-annex: unrecognized option `--help'
+
+Usage: git-annex add [PATH ...] [option ...]
+    --include-dotfiles  don't skip dotfiles
+
+To see additional options common to all commands, run: git annex help options
+
+### What version of git-annex are you using? On what operating system?
+
+git-annex version: 5.20141125
+build flags: Assistant Webapp Webapp-secure Pairing Testsuite S3 WebDAV Inotify DBus DesktopNotify XMPP DNS Feeds Quvi TDFA CryptoHash
+key/value backends: SHA256E SHA1E SHA512E SHA224E SHA384E SKEIN256E SKEIN512E SHA256 SHA1 SHA512 SHA224 SHA384 SKEIN256 SKEIN512 WORM URL
+remote types: git gcrypt S3 bup directory rsync web webdav tahoe glacier ddar hook external
+
+
+### Please provide any additional information below.
+
+[[!format sh """
+# If you can, paste a complete transcript of the problem occurring here.
+# If the problem is with the git-annex assistant, paste in .git/annex/daemon.log
+
+
+# End of transcript or log.
+"""]]
+
+### Have you had any luck using git-annex before? (Sometimes we get tired of reading bug reports all day and a lil' positive end note does wonders)
+
+
+> [[done]]; ancient version; fixed long ago --[[Joey]]
diff --git a/doc/bugs/git_annex___36__command_--help_not_recoginized/comment_1_63aa3512fd49c0c3b232eb85ed0cf4f3._comment b/doc/bugs/git_annex___36__command_--help_not_recoginized/comment_1_63aa3512fd49c0c3b232eb85ed0cf4f3._comment
new file mode 100644
index 0000000000..7a39b1fe4a
--- /dev/null
+++ b/doc/bugs/git_annex___36__command_--help_not_recoginized/comment_1_63aa3512fd49c0c3b232eb85ed0cf4f3._comment
@@ -0,0 +1,10 @@
+[[!comment format=mdwn
+ username="joey"
+ subject="""comment 1"""
+ date="2017-08-28T17:28:10Z"
+ content="""
+You are using a version of git-annex from 2014! You really ought to
+upgrade. The current version has rather a lot of changes, including,
+it happens, a fix for this small bug. Lots of other much larger bugs,
+including at least one security hole, have also been fixed since 2014..
+"""]]
diff --git a/doc/bugs/git_annex_add_out_of_memory__63___mmap_failed__58___No_such_file_or_directory.mdwn b/doc/bugs/git_annex_add_out_of_memory__63___mmap_failed__58___No_such_file_or_directory.mdwn
new file mode 100644
index 0000000000..13ac5a98f6
--- /dev/null
+++ b/doc/bugs/git_annex_add_out_of_memory__63___mmap_failed__58___No_such_file_or_directory.mdwn
@@ -0,0 +1,43 @@
+### Please describe the problem.
+
+Adding my 43GB Music Studio folder with 85K files causes out of memory or file not found errors. I subsequently tried on a very small folder but it still fails. --debug gives nothing useful that I can see.
+
+
+### What steps will reproduce the problem?
+
+cd Studio
+git init
+git annex init
+git annex add
+
+
+### What version of git-annex are you using? On what operating system?
+git-annex version: 5.20140613-g5587055
+build flags: Assistant Webapp Webapp-secure Pairing Testsuite S3 WebDAV DNS Feeds Quvi TDFA CryptoHash
+key/value backends: SHA256E SHA1E SHA512E SHA224E SHA384E SKEIN256E SKEIN512E SHA256 SHA1 SHA512 SHA224 SHA384 SKEIN256 SKEIN512 WORM URL
+remote types: git gcrypt S3 bup directory rsync web webdav tahoe glacier ddar hook external
+local repository version: 5
+supported repository version: 5
+upgrade supported from repository versions: 2 3 4
+
+Running on Windows 8.1 x64, Git Bash, Direct Mode, WORM backend
+
+### Please provide any additional information below.
+
+[[!format sh """
+# If you can, paste a complete transcript of the problem occurring here.
+# If the problem is with the git-annex assistant, paste in .git/annex/daemon.log
+
+fatal: Out of memory? mmap failed: No such file or directory
+git-annex: add: 8562 failed
+
+Phil@DESKTOP /c/Studio (annex/direct/master)
+$ git annex add audacity
+fatal: Out of memory? mmap failed: No such file or directory
+(Recording state in git...)
+
+
+# End of transcript or log.
+"""]]
+
+[[!meta title="msysgit failure on Windows: mmap failed: No such file or directory "]]
diff --git a/doc/bugs/git_annex_add_out_of_memory__63___mmap_failed__58___No_such_file_or_directory/comment_1_984f75d8078f2809486f38ecb3b16be3._comment b/doc/bugs/git_annex_add_out_of_memory__63___mmap_failed__58___No_such_file_or_directory/comment_1_984f75d8078f2809486f38ecb3b16be3._comment
new file mode 100644
index 0000000000..3662fe04fc
--- /dev/null
+++ b/doc/bugs/git_annex_add_out_of_memory__63___mmap_failed__58___No_such_file_or_directory/comment_1_984f75d8078f2809486f38ecb3b16be3._comment
@@ -0,0 +1,10 @@
+[[!comment format=mdwn
+ username="https://www.google.com/accounts/o8/id?id=AItOawnOSgFb3l7nL3Fs7Y9gPGJJjFiV7aJ1tek"
+ nickname="Phil"
+ subject="comment 1"
+ date="2014-07-04T16:25:53Z"
+ content="""
+Looks like it might be related to msysGit:
+
+[bug on msysgit](http://osdir.com/ml/msysgit/2009-12/msg00007.html)
+"""]]
diff --git a/doc/bugs/git_annex_add_out_of_memory__63___mmap_failed__58___No_such_file_or_directory/comment_2_c0f07e2d4bb142389629050479dd1465._comment b/doc/bugs/git_annex_add_out_of_memory__63___mmap_failed__58___No_such_file_or_directory/comment_2_c0f07e2d4bb142389629050479dd1465._comment
new file mode 100644
index 0000000000..0601b0a7de
--- /dev/null
+++ b/doc/bugs/git_annex_add_out_of_memory__63___mmap_failed__58___No_such_file_or_directory/comment_2_c0f07e2d4bb142389629050479dd1465._comment
@@ -0,0 +1,10 @@
+[[!comment format=mdwn
+ username="http://joeyh.name/"
+ ip="209.250.56.55"
+ subject="comment 2"
+ date="2014-07-04T17:58:10Z"
+ content="""
+If you run git-annex with the --debug flag, you can see the git command that is presumably what is failing. You could then probably run the same git command manually and see if it fails that way too.
+
+Does your git repository have a large pack file (in `.git/objects/pack`) as mentioned in the msysgit bug report?
+"""]]
diff --git a/doc/bugs/git_annex_add_out_of_memory__63___mmap_failed__58___No_such_file_or_directory/comment_3_7a58a884aaacedca9697b17cd5248214._comment b/doc/bugs/git_annex_add_out_of_memory__63___mmap_failed__58___No_such_file_or_directory/comment_3_7a58a884aaacedca9697b17cd5248214._comment
new file mode 100644
index 0000000000..c6aa8360a5
--- /dev/null
+++ b/doc/bugs/git_annex_add_out_of_memory__63___mmap_failed__58___No_such_file_or_directory/comment_3_7a58a884aaacedca9697b17cd5248214._comment
@@ -0,0 +1,33 @@
+[[!comment format=mdwn
+ username="https://www.google.com/accounts/o8/id?id=AItOawnOSgFb3l7nL3Fs7Y9gPGJJjFiV7aJ1tek"
+ nickname="Phil"
+ subject="comment 3"
+ date="2014-07-04T22:28:27Z"
+ content="""
+I thought it might be related to using WORM but I've just hit it again on the default backend.
+
+--debug just seems to fail on the folder I tried to add:
+
+[[!format sh \"\"\"
+
+git annex --debug add samples
+
+[2014-07-04 23:24:42 GMT Summer Time] read: git [\"--git-dir=c:\\Studio\\.git\",\"--work-tree=c:\\Studio\",\"-c\",\"core.bare=false\",\"ls-files\",\"--others\",\"--exclude-standard\",\"-z\",\"--\",\"samples\"]
+[2014-07-04 23:24:42 GMT Summer Time] chat: git [\"--git-dir=c:\\Studio\\.git\",\"--work-tree=c:\\Studio\",\"-c\",\"core.bare=false\",\"cat-file\",\"--batch\"]
+add samples/cr_info.sh [2014-07-04 23:24:42 GMT Summer Time] chat: git [\"--git-dir=c:\\Studio\\.git\",\"--work-tree=c:\\Studio\",\"-c\",\"core.bare=false\",\"check-attr\",\"-z\",\"--stdin\",\"annex.backend\",\"annex.numcopies\",\"--\"]
+[2014-07-04 23:24:42 GMT Summer Time] chat: git [\"--git-dir=c:\\Studio\\.git\",\"--work-tree=c:\\Studio\",\"-c\",\"core.bare=false\",\"hash-object\",\"-t\",\"blob\",\"-w\",\"--stdin\",\"--no-filters\"]
+[2014-07-04 23:24:42 GMT Summer Time] chat: git [\"--git-dir=c:\\Studio\\.git\",\"--work-tree=c:\\Studio\",\"-c\",\"core.bare=false\",\"cat-file\",\"--batch\"]
+ok
+[2014-07-04 23:24:42 GMT Summer Time] read: git [\"--git-dir=c:\\Studio\\.git\",\"--work-tree=c:\\Studio\",\"-c\",\"core.bare=false\",\"ls-files\",\"--modified\",\"-z\",\"--\",\"samples\"]
+fatal: Out of memory? mmap failed: No such file or directory
+(Recording state in git...)
+[2014-07-04 23:25:39 GMT Summer Time] feed: git [\"--git-dir=c:\\Studio\\.git\",\"--work-tree=c:\\Studio\",\"-c\",\"core.bare=false\",\"update-index\",\"-z\",\"--index-info\"]
+[2014-07-04 23:25:39 GMT Summer Time] chat: git [\"--git-dir=c:\\Studio\\.git\",\"--work-tree=c:\\Studio\",\"-c\",\"core.bare=false\",\"hash-object\",\"-w\",\"--stdin-paths\",\"--no-filters\"]
+[2014-07-04 23:25:39 GMT Summer Time] feed: git [\"--git-dir=c:\\Studio\\.git\",\"--work-tree=c:\\Studio\",\"-c\",\"core.bare=false\",\"update-index\",\"-z\",\"--index-info\"]
+[2014-07-04 23:25:39 GMT Summer Time] read: git [\"--git-dir=c:\\Studio\\.git\",\"--work-tree=c:\\Studio\",\"-c\",\"core.bare=false\",\"show-ref\",\"--hash\",\"refs/heads/git-annex\"]
+[2014-07-04 23:25:39 GMT Summer Time] read: git [\"--git-dir=c:\\Studio\\.git\",\"--work-tree=c:\\Studio\",\"-c\",\"core.bare=false\",\"write-tree\"]
+[2014-07-04 23:25:39 GMT Summer Time] chat: git [\"--git-dir=c:\\Studio\\.git\",\"--work-tree=c:\\Studio\",\"-c\",\"core.bare=false\",\"commit-tree\",\"447be0c8a000f98e3c580c6dede7f0ba2208963d\",\"-p\",\"refs/heads/git-annex\"]
+[2014-07-04 23:25:39 GMT Summer Time] call: git [\"--git-dir=c:\\Studio\\.git\",\"--work-tree=c:\\Studio\",\"-c\",\"core.bare=false\",\"update-ref\",\"refs/heads/git-annex\",\"8551a4e2f5c4f42823f7da3169d1f630622359ef\"]
+
+\"\"\"]]
+"""]]
diff --git a/doc/bugs/git_annex_add_out_of_memory__63___mmap_failed__58___No_such_file_or_directory/comment_4_aa4f1806207138115d2a95935bb0546b._comment b/doc/bugs/git_annex_add_out_of_memory__63___mmap_failed__58___No_such_file_or_directory/comment_4_aa4f1806207138115d2a95935bb0546b._comment
new file mode 100644
index 0000000000..8e98152420
--- /dev/null
+++ b/doc/bugs/git_annex_add_out_of_memory__63___mmap_failed__58___No_such_file_or_directory/comment_4_aa4f1806207138115d2a95935bb0546b._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="https://www.google.com/accounts/o8/id?id=AItOawnOSgFb3l7nL3Fs7Y9gPGJJjFiV7aJ1tek"
+ nickname="Phil"
+ subject="comment 4"
+ date="2014-07-04T22:30:45Z"
+ content="""
+There is no pack file, .git/objects/pack is empty. This is the first commit. I've queued up a load of files to commit, could this be it?
+"""]]
diff --git a/doc/bugs/git_annex_add_out_of_memory__63___mmap_failed__58___No_such_file_or_directory/comment_5_fa95f93416e3d6e66af557df6562f1e5._comment b/doc/bugs/git_annex_add_out_of_memory__63___mmap_failed__58___No_such_file_or_directory/comment_5_fa95f93416e3d6e66af557df6562f1e5._comment
new file mode 100644
index 0000000000..749ee44ad9
--- /dev/null
+++ b/doc/bugs/git_annex_add_out_of_memory__63___mmap_failed__58___No_such_file_or_directory/comment_5_fa95f93416e3d6e66af557df6562f1e5._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="http://joeyh.name/"
+ ip="209.250.56.2"
+ subject="comment 5"
+ date="2014-07-10T19:59:15Z"
+ content="""
+For some reason, this bug is also being discussed in the forum. This makes it hard for me to keep up with what's going on, and generally is not a good idea. Currently, the forum currently seems to have better information. 
+"""]]
diff --git a/doc/bugs/git_annex_adjust_--unlock_seems_to_cause_migration_of_a_file_to_another_backend.mdwn b/doc/bugs/git_annex_adjust_--unlock_seems_to_cause_migration_of_a_file_to_another_backend.mdwn
new file mode 100644
index 0000000000..8cb93952a5
--- /dev/null
+++ b/doc/bugs/git_annex_adjust_--unlock_seems_to_cause_migration_of_a_file_to_another_backend.mdwn
@@ -0,0 +1,60 @@
+### Please describe the problem.
+
+Interestingly, on my first attempt I had two files which migrated from MD5 to MD5E backend, but may be I have done some steps differently which provoked also  also utils/test/test_data.tar.gz to get to the same destiny (actually later replicated on a fresh clone).
+I am also a bit confused why git diff reports change in the symlink if file is already a real file (I guess all the smudging magic)
+
+### Please provide any additional information below.
+
+[[!format sh """
+# If you can, paste a complete transcript of the problem occurring here.
+# If the problem is with the git-annex assistant, paste in .git/annex/daemon.log
+(git)smaug:~exppsy/freesurfer-upstream2[master]git
+$> git status
+On branch master
+Your branch is up-to-date with 'origin/master'.
+nothing to commit, working directory clean
+
+$> git annex adjust --unlock
+adjust
+Checking out files: 100% (468/468), done.
+Switched to branch 'adjusted/master(unlocked)'
+ok
+git annex adjust --unlock  23.70s user 35.25s system 32% cpu 2:59.41 total
+changes on filesystem:
+ vtkutils/vtkKWRGBATransferFunctionEditorTester-scalars.mgh | 2 +-
+
+$> git status
+On branch adjusted/master(unlocked)
+Changes not staged for commit:
+  (use "git add ..." to update what will be committed)
+  (use "git checkout -- ..." to discard changes in working directory)
+
+        modified:   vtkutils/vtkKWRGBATransferFunctionEditorTester-scalars.mgh
+
+no changes added to commit (use "git add" and/or "git commit -a")
+changes on filesystem:
+ vtkutils/vtkKWRGBATransferFunctionEditorTester-scalars.mgh | 2 +-
+
+$> git diff
+diff --git a/vtkutils/vtkKWRGBATransferFunctionEditorTester-scalars.mgh b/vtkutils/vtkKWRGBATransferFunctionEditorTester-scalars.mgh
+index 9ea90d3..7bc0177 100644
+--- a/vtkutils/vtkKWRGBATransferFunctionEditorTester-scalars.mgh
++++ b/vtkutils/vtkKWRGBATransferFunctionEditorTester-scalars.mgh
+@@ -1 +1 @@
+-/annex/objects/SHA256-s655672--85f9b50e8fb8a72a8783152c7ad098c0600222256a7244ccd595cbe67b9ea949
++/annex/objects/SHA256E-s655672--85f9b50e8fb8a72a8783152c7ad098c0600222256a7244ccd595cbe67b9ea949.mgh
+changes on filesystem:
+ vtkutils/vtkKWRGBATransferFunctionEditorTester-scalars.mgh | 2 +-
+
+$> ls -ld vtkutils/vtkKWRGBATransferFunctionEditorTester-scalars.mgh
+-rw------- 1 yoh yoh 655672 Jun  1 23:28 vtkutils/vtkKWRGBATransferFunctionEditorTester-scalars.mgh
+
+$> git annex version
+git-annex version: 6.20160523+gitg33c00ab-1~ndall+1
+
+"""]]
+
+
+[[!meta author=yoh]]
+
+> [[fixed|done]] --[[Joey]]
diff --git a/doc/bugs/git_annex_assistant_exits_with_failure.mdwn b/doc/bugs/git_annex_assistant_exits_with_failure.mdwn
new file mode 100644
index 0000000000..934149eaa0
--- /dev/null
+++ b/doc/bugs/git_annex_assistant_exits_with_failure.mdwn
@@ -0,0 +1,161 @@
+### Please describe the problem
+Git annex assistant on windows exits with a failure upon large sync. It has done so about 6 times while messing with it. (I've been seeing if it is a good solution for my photos or not before taking the leap.) I am syncing between windows 10 and debian servers using the backup group and assistant/webapp. Debian is quite out of date, so I first used stable, then upped it to jessie-backports. Jessie backports is still, however, major version 5, so I upped it to neuro debian's build to see if it needed to match version numbers. I haven't quite matched the date though, so don't know if that could be an issue. I have tried the webapp as well, and it goes down during this issue. I have also had failures on the linux side. The second to last run git annex was using 85% of my system memory and the load was 2.5. I had to kill git annex assistant and restart. The Windows git annex assistant just became unresponsive that time and I had to reboot the system to get it to sense any files again. These could be separate issues, but it involved the same data set.
+
+### What steps will reproduce the problem?
+Setup git annex with assistant on windows 10 as a client and debian 8 latest neurodebian build running assistant. Copy in a lightroom catalog with previews, smart previews, and photos. Mine amounted to about 39 GB, 22k files, and 17k folders
+### What version of git-annex are you using? On what operating system?
+client, windows 10 running git bash window: 
+
+git-annex version: 6.20161231-gc8eeb17
+build flags: Assistant Webapp Pairing Testsuite S3(multipartupload)(storageclasses) WebDAV ConcurrentOutput TorrentParser Feeds Quvi
+key/value backends: SHA256E SHA256 SHA512E SHA512 SHA224E SHA224 SHA384E SHA384 SHA3_256E SHA3_256 SHA3_512E SHA3_512 SHA3_224E SHA3_224 SHA3_384E SHA3_384 SKEIN256E SKEIN256 SKEIN512E SKEIN512 SHA1E SHA1 MD5E MD5 WORM URL
+remote types: git gcrypt p2p S3 bup directory rsync web bittorrent webdav tahoe glacier ddar hook external
+local repository version: 5
+supported repository versions: 3 5 6
+upgrade supported from repository versions: 2 3 4 5
+operating system: mingw32 i386
+
+server Debian 8:
+
+git-annex version: 6.20170209+gitg16be7b5cc-1~ndall+1
+build flags: Assistant Webapp Pairing Testsuite S3(multipartupload)(storageclasses) WebDAV Inotify DBus DesktopNotify ConcurrentOutput TorrentParser MagicMime Feeds Quvi
+key/value backends: SHA256E SHA256 SHA512E SHA512 SHA224E SHA224 SHA384E SHA384 SHA3_256E SHA3_256 SHA3_512E SHA3_512 SHA3_224E SHA3_224 SHA3_384E SHA3_384 SKEIN256E SKEIN256 SKEIN512E SKEIN512 SHA1E SHA1 MD5E MD5 WORM URL
+remote types: git gcrypt p2p S3 bup directory rsync web bittorrent webdav tahoe glacier ddar hook external
+local repository version: 5
+supported repository versions: 3 5 6
+upgrade supported from repository versions: 0 1 2 3 4 5
+operating system: linux x86_64
+
+### Please provide any additional information below.
+This is happens for both my large and smaller lightroom catalogs. I'm wondering if it is a peculiarity with how some of the files are formatted or the fact that it is a huge GUID for a file name. I can fairly consistently reproduce this issue on my setup, and I'm a software engineer (just don't know haskell yet.) So, I'm willing to do further debugging steps to help out.
+
+[[!format sh """
+# If you can, paste a complete transcript of the problem occurring here.
+$ git annex assistant --debug
+[2017-02-12 15:10:11.3151425] logging to .git\annex\daemon.log
+[2017-02-12 15:10:11.3151425] chat: C:\Program Files (x86)\Git\usr\bin\git-annex.exe ["assistant","--debug"]
+[2017-02-12 15:25:00.1479375] process done ExitFailure (-1073741819)
+
+# If the problem is with the git-annex assistant, paste in .git/annex/daemon.log
+I assume you don't want the entire log in here right? It is 40k lines. I searched for ExitFailure and found only 3 instances, but none of them were at the end of the log.
+
+[2017-02-12 15:10:29.0507794] call: git ["--git-dir=.git","--work-tree=.","--literal-pathspecs","-c","core.bare=false","show-ref","--verify","-q","refs/remotes/origin/annex/direct/master"]
+[2017-02-12 15:10:29.1132817] process done ExitFailure 1
+
+[2017-02-12 15:10:30.1769265] read: git ["--git-dir=.git","--work-tree=.","--literal-pathspecs","-c","core.bare=false","push","origin","git-annex","master"]
+[2017-02-12 15:10:30.5990043] process done ExitFailure 1
+
+Last error:
+[2017-02-12 15:20:07.8259699] Watcher: add .\LR Main\LR Main Previews.lrdata\6\6D0A\6D0A45BF-E2F7-4094-A3C5-C0659456A383-05daea287739b47c27cea4102e72ecd9.lrprev
+[2017-02-12 15:20:07.8259699] process done ExitFailure 1
+
+
+Here are the last 100 or so lines, let me know if you want more:
+add .\LR Main\LR Main Previews.lrdata\5\5E0E\5E0EACCF-16F7-4D95-A01F-BA17EA4BFA48-05daea287739b47c27cea4102e72ecd9.lrprev ok
+add .\LR Main\LR Main Previews.lrdata\5\5E05\5E05C2FD-107F-4CC8-8B22-375F0B09F43F-a1140a4f04c71f00fb85fd8ac71d471c.lrfprev ok
+add .\LR Main\LR Main Previews.lrdata\5\5E05\5E05C2FD-107F-4CC8-8B22-375F0B09F43F-269826a76ee94c785b873f891dc7aa04.lrfprev ok
+add .\LR Main\LR Main Previews.lrdata\5\5E05\5E05C2FD-107F-4CC8-8B22-375F0B09F43F-05daea287739b47c27cea4102e72ecd9.lrprev [2017-02-12 15:24:05.2466023] read: sha256sum [".\\LR Main\\LR Main Previews.lrdata\\5\\5E05\\5E05C2FD-107F-4CC8-8B22-375F0B09F43F-05daea287739b47c27cea4102e72ecd9.lrprev"]
+[2017-02-12 15:24:05.293479] process done ExitSuccess
+ok
+add .\LR Main\LR Main Previews.lrdata\5\5DF5\5DF55627-D6A1-4E4D-96F3-00BFFBE07DE3-1c47c3bee58f7f7ae5b0b166edb1df35.lrprev [2017-02-12 15:24:05.3091052] read: sha256sum [".\\LR Main\\LR Main Previews.lrdata\\5\\5DF5\\5DF55627-D6A1-4E4D-96F3-00BFFBE07DE3-1c47c3bee58f7f7ae5b0b166edb1df35.lrprev"]
+[2017-02-12 15:24:05.355981] process done ExitSuccess
+ok
+add .\LR Main\LR Main Previews.lrdata\5\5DB9\5DB91865-DCDD-4987-8FBE-126861B802FC-05daea287739b47c27cea4102e72ecd9.lrprev ok
+add .\LR Main\LR Main Previews.lrdata\5\5DB3\5DB317AF-CE90-4B37-A65A-E7394E0D8364-d28c9daea957f6c969fa48154e27ccfa.lrfprev ok
+add .\LR Main\LR Main Previews.lrdata\5\5DB3\5DB317AF-CE90-4B37-A65A-E7394E0D8364-0dcfb6f1963162fa2616cf6950b64f23.lrfprev ok
+add .\LR Main\LR Main Previews.lrdata\5\5DB3\5DB317AF-CE90-4B37-A65A-E7394E0D8364-05daea287739b47c27cea4102e72ecd9.lrprev ok
+add .\LR Main\LR Main Previews.lrdata\5\5DAF\5DAFD546-32A8-4087-9DA9-E0C13C2EDE0E-077b070982fd42045ed5e932196fb988.lrprev ok
+add .\LR Main\LR Main Previews.lrdata\5\5D9E\5D9E0F0C-082F-4A2B-AF9C-3A3288B3A2AE-91a29824ad0a349e888a7db35e28325f.lrfprev ok
+add .\LR Main\LR Main Previews.lrdata\5\5D9E\5D9E0F0C-082F-4A2B-AF9C-3A3288B3A2AE-05daea287739b47c27cea4102e72ecd9.lrprev [2017-02-12 15:24:05.5123925] read: sha256sum [".\\LR Main\\LR Main Previews.lrdata\\5\\5D9E\\5D9E0F0C-082F-4A2B-AF9C-3A3288B3A2AE-05daea287739b47c27cea4102e72ecd9.lrprev"]
+[2017-02-12 15:24:05.559269] process done ExitSuccess
+ok
+add .\LR Main\LR Main Previews.lrdata\5\5D71\5D71BB4D-8322-47C0-B779-3542311E59DE-077b070982fd42045ed5e932196fb988.lrprev [2017-02-12 15:24:05.5748955] read: sha256sum [".\\LR Main\\LR Main Previews.lrdata\\5\\5D71\\5D71BB4D-8322-47C0-B779-3542311E59DE-077b070982fd42045ed5e932196fb988.lrprev"]
+[2017-02-12 15:24:05.6217725] process done ExitSuccess
+ok
+add .\LR Main\LR Main Previews.lrdata\5\5D6C\5D6C374E-FA1E-4526-A215-63B315BA82B2-9453b0b0faf115dd4cbf4a4ca3fd5ee7.lrprev ok
+add .\LR Main\LR Main Previews.lrdata\5\5D69\5D69C130-CDB9-4B67-8061-C0C0EF345D26-077b070982fd42045ed5e932196fb988.lrprev [2017-02-12 15:24:05.6842751] read: sha256sum [".\\LR Main\\LR Main Previews.lrdata\\5\\5D69\\5D69C130-CDB9-4B67-8061-C0C0EF345D26-077b070982fd42045ed5e932196fb988.lrprev"]
+[2017-02-12 15:24:05.7155263] process done ExitSuccess
+ok
+add .\LR Main\LR Main Previews.lrdata\5\5D47\5D479A01-2B69-4B85-AF2C-2CA2BF6BA66D-dfea411588b2a11b7e314d4691ee3d2f.lrprev ok
+add .\LR Main\LR Main Previews.lrdata\5\5D43\5D43DB0F-30D9-4071-B96C-25D2D82F6E40-05daea287739b47c27cea4102e72ecd9.lrprev ok
+add .\LR Main\LR Main Previews.lrdata\5\5D42\5D42D098-917E-46F6-AE24-2FB6242BE60D-05daea287739b47c27cea4102e72ecd9.lrprev [2017-02-12 15:24:05.7936545] read: sha256sum [".\\LR Main\\LR Main Previews.lrdata\\5\\5D42\\5D42D098-917E-46F6-AE24-2FB6242BE60D-05daea287739b47c27cea4102e72ecd9.lrprev"]
+[2017-02-12 15:24:05.8092806] process done ExitSuccess
+ok
+add .\LR Main\LR Main Previews.lrdata\5\5D3D\5D3DF631-9D24-446F-84F9-C858F398B23B-05daea287739b47c27cea4102e72ecd9.lrprev ok
+add .\LR Main\LR Main Previews.lrdata\5\5D36\5D3642E1-BBA6-4C0B-89B6-9451B375D47A-05daea287739b47c27cea4102e72ecd9.lrprev ok
+add .\LR Main\LR Main Previews.lrdata\5\5D29\5D29AB0B-E5A7-4930-BA0E-592675856EE8-b422a73fe8f4257d07ddd8c5d62692c9.lrfprev ok
+add .\LR Main\LR Main Previews.lrdata\5\5D29\5D29AB0B-E5A7-4930-BA0E-592675856EE8-05daea287739b47c27cea4102e72ecd9.lrprev [2017-02-12 15:24:05.9066036] read: sha256sum [".\\LR Main\\LR Main Previews.lrdata\\5\\5D29\\5D29AB0B-E5A7-4930-BA0E-592675856EE8-05daea287739b47c27cea4102e72ecd9.lrprev"]
+[2017-02-12 15:24:05.9509479] process done ExitSuccess
+ok
+add .\LR Main\LR Main Previews.lrdata\5\5D21\5D21CA81-F662-4566-8E69-B7587441A60B-c55693145cbd7408f494d96dc6feac77.lrfprev ok
+add .\LR Main\LR Main Previews.lrdata\5\5D21\5D21CA81-F662-4566-8E69-B7587441A60B-05daea287739b47c27cea4102e72ecd9.lrprev [2017-02-12 15:24:05.9978244] read: sha256sum [".\\LR Main\\LR Main Previews.lrdata\\5\\5D21\\5D21CA81-F662-4566-8E69-B7587441A60B-05daea287739b47c27cea4102e72ecd9.lrprev"]
+[2017-02-12 15:24:06.0134498] process done ExitSuccess
+ok
+add .\LR Main\LR Main Previews.lrdata\5\5D21\5D2174E4-5440-46C9-B852-0164DFB9912A-05daea287739b47c27cea4102e72ecd9.lrprev ok
+add .\LR Main\LR Main Previews.lrdata\5\5CF0\5CF09660-99B4-409F-8CF7-ADE34A6785B3-05daea287739b47c27cea4102e72ecd9.lrprev [2017-02-12 15:24:06.0603269] read: sha256sum [".\\LR Main\\LR Main Previews.lrdata\\5\\5CF0\\5CF09660-99B4-409F-8CF7-ADE34A6785B3-05daea287739b47c27cea4102e72ecd9.lrprev"]
+[2017-02-12 15:24:06.1072033] process done ExitSuccess
+ok
+add .\LR Main\LR Main Previews.lrdata\5\5CDA\5CDA3518-0AC4-4B77-9485-AC40246094C9-05daea287739b47c27cea4102e72ecd9.lrprev [2017-02-12 15:24:06.1228291] read: sha256sum [".\\LR Main\\LR Main Previews.lrdata\\5\\5CDA\\5CDA3518-0AC4-4B77-9485-AC40246094C9-05daea287739b47c27cea4102e72ecd9.lrprev"]
+[2017-02-12 15:24:06.1384542] process done ExitSuccess
+ok
+add .\LR Main\LR Main Previews.lrdata\5\5CCE\5CCE8E71-8668-45D7-9053-DBC59DBECEDA-05daea287739b47c27cea4102e72ecd9.lrprev [2017-02-12 15:24:06.1697055] read: sha256sum [".\\LR Main\\LR Main Previews.lrdata\\5\\5CCE\\5CCE8E71-8668-45D7-9053-DBC59DBECEDA-05daea287739b47c27cea4102e72ecd9.lrprev"]
+[2017-02-12 15:24:06.2165819] process done ExitSuccess
+ok
+add .\LR Main\LR Main Previews.lrdata\5\5CB2\5CB2A82E-E4E4-4999-B976-2650F63003D4-05daea287739b47c27cea4102e72ecd9.lrprev [2017-02-12 15:24:06.2322077] read: sha256sum [".\\LR Main\\LR Main Previews.lrdata\\5\\5CB2\\5CB2A82E-E4E4-4999-B976-2650F63003D4-05daea287739b47c27cea4102e72ecd9.lrprev"]
+[2017-02-12 15:24:06.2790841] process done ExitSuccess
+ok
+add .\LR Main\LR Main Previews.lrdata\5\5C8B\5C8BBE9D-4E0C-429E-BB54-0B39AA043ABA-05daea287739b47c27cea4102e72ecd9.lrprev ok
+add .\LR Main\LR Main Previews.lrdata\5\5C55\5C553361-7EF4-4B77-B568-FEAD5381CFCC-05daea287739b47c27cea4102e72ecd9.lrprev [2017-02-12 15:24:06.3103357] read: sha256sum [".\\LR Main\\LR Main Previews.lrdata\\5\\5C55\\5C553361-7EF4-4B77-B568-FEAD5381CFCC-05daea287739b47c27cea4102e72ecd9.lrprev"]
+[2017-02-12 15:24:06.3415866] process done ExitSuccess
+ok
+add .\LR Main\LR Main Previews.lrdata\5\5C2E\5C2EAB4B-3986-4038-855D-F9DB3A5EE390-05daea287739b47c27cea4102e72ecd9.lrprev ok
+add .\LR Main\LR Main Previews.lrdata\5\5C2D\5C2DE3F6-D523-4729-81E2-34DA2D66843D-05daea287739b47c27cea4102e72ecd9.lrprev [2017-02-12 15:24:06.3728379] read: sha256sum [".\\LR Main\\LR Main Previews.lrdata\\5\\5C2D\\5C2DE3F6-D523-4729-81E2-34DA2D66843D-05daea287739b47c27cea4102e72ecd9.lrprev"]
+[2017-02-12 15:24:06.4036538] process done ExitSuccess
+ok
+add .\LR Main\LR Main Previews.lrdata\5\5C21\5C21C472-7230-464F-ADFA-8F93D30280F1-05daea287739b47c27cea4102e72ecd9.lrprev ok
+add .\LR Main\LR Main Previews.lrdata\5\5C1C\5C1C2A2C-BFA9-42EA-A647-6F5138F5CEB4-077b070982fd42045ed5e932196fb988.lrprev ok
+[2017-02-12 15:24:06.449992] Committer: committing 5214 changes
+[2017-02-12 15:24:06.449992] Committer: Committing changes to git
+(recording state in git...)
+[2017-02-12 15:24:06.449992] feed: git ["--git-dir=.git","--work-tree=.","--literal-pathspecs","-c","core.bare=false","update-index","-z","--index-info"]
+[2017-02-12 15:24:06.6687496] process done ExitSuccess
+[2017-02-12 15:24:06.6687496] read: git ["--git-dir=.git","--work-tree=.","--literal-pathspecs","-c","core.bare=false","symbolic-ref","-q","HEAD"]
+[2017-02-12 15:24:06.6843757] process done ExitSuccess
+[2017-02-12 15:24:06.6843757] read: git ["--git-dir=.git","--work-tree=.","--literal-pathspecs","-c","core.bare=false","show-ref","--hash","refs/heads/annex/direct/master"]
+[2017-02-12 15:24:06.7156263] process done ExitSuccess
+[2017-02-12 15:24:06.7156263] read: git ["--git-dir=.git","--work-tree=.","--literal-pathspecs","-c","core.bare=false","write-tree"]
+[2017-02-12 15:24:18.823257] process done ExitSuccess
+[2017-02-12 15:24:18.823257] read: git ["--git-dir=.git","--work-tree=.","--literal-pathspecs","-c","core.bare=false","rev-parse","e27617a27919a1942924c791c7963898b558f9c2:"]
+[2017-02-12 15:24:18.8388825] process done ExitSuccess
+[2017-02-12 15:24:18.8388825] chat: git ["--git-dir=.git","--work-tree=.","--literal-pathspecs","-c","core.bare=false","commit-tree","c3f9659d704711122cfd83d3f4742635116ca96d","-p","e27617a27919a1942924c791c7963898b558f9c2"]
+[2017-02-12 15:24:19.0888926] process done ExitSuccess
+[2017-02-12 15:24:19.0888926] call: git ["--git-dir=.git","--work-tree=.","--literal-pathspecs","-c","core.bare=false","update-ref","refs/heads/annex/direct/master","8af641f066e30426a591bcf51a4244f127847431"]
+[2017-02-12 15:24:19.338833] process done ExitSuccess
+[2017-02-12 15:24:19.338833] read: git ["--git-dir=.git","--work-tree=.","--literal-pathspecs","-c","core.bare=false","symbolic-ref","-q","HEAD"]
+[2017-02-12 15:24:19.3700836] process done ExitSuccess
+[2017-02-12 15:24:19.3700836] read: git ["--git-dir=.git","--work-tree=.","--literal-pathspecs","-c","core.bare=false","show-ref","refs/heads/annex/direct/master"]
+[2017-02-12 15:24:19.3857092] process done ExitSuccess
+[2017-02-12 15:24:19.3857092] call: git ["--git-dir=.git","--work-tree=.","--literal-pathspecs","-c","core.bare=false","branch","-f","synced/master","refs/heads/annex/direct/master"]
+[2017-02-12 15:24:19.6400219] process done ExitSuccess
+[2017-02-12 15:24:19.6400219] call: git ["--git-dir=.git","--work-tree=.","--literal-pathspecs","-c","core.bare=false","branch","-f","master","refs/heads/annex/direct/master"]
+[2017-02-12 15:24:19.9025509] process done ExitSuccess
+[2017-02-12 15:24:19.9025509] Pusher: Syncing with origin 
+[2017-02-12 15:24:19.9025509] feed: git ["--git-dir=.git","--work-tree=.","--literal-pathspecs","-c","core.bare=false","update-index","-z","--index-info"]
+[2017-02-12 15:24:34.1218776] process done ExitSuccess
+[2017-02-12 15:24:34.1218776] read: git ["--git-dir=.git","--work-tree=.","--literal-pathspecs","-c","core.bare=false","show-ref","--hash","refs/heads/git-annex"]
+[2017-02-12 15:24:34.1531288] process done ExitSuccess
+(recording state in git...)
+[2017-02-12 15:24:34.1531288] read: git ["--git-dir=.git","--work-tree=.","--literal-pathspecs","-c","core.bare=false","write-tree"]
+[2017-02-12 15:24:57.3948994] process done ExitSuccess
+[2017-02-12 15:24:57.3948994] chat: git ["--git-dir=.git","--work-tree=.","--literal-pathspecs","-c","core.bare=false","commit-tree","df397eef838882ccb0cba9d6ceca056aad905520","-p","refs/heads/git-annex"]
+[2017-02-12 15:24:57.6449095] process done ExitSuccess
+[2017-02-12 15:24:57.6449095] call: git ["--git-dir=.git","--work-tree=.","--literal-pathspecs","-c","core.bare=false","update-ref","refs/heads/git-annex","14b3e08b23f938d901447b1b7fcbf020a8bc517c"]
+[2017-02-12 15:24:57.8960758] process done ExitSuccess
+[2017-02-12 15:25:00.0854341] read: git ["--git-dir=.git","--work-tree=.","--literal-pathspecs","-c","core.bare=false","symbolic-ref","-q","HEAD"]
+[2017-02-12 15:25:00.1010592] process done ExitSuccess
+# End of transcript or log.
+"""]]
+
+### Have you had any luck using git-annex before? (Sometimes we get tired of reading bug reports all day and a lil' positive end note does wonders)
+I have had luck in manual mode using only linux. I only shied away from it, because I am chained to Windows in order to use some tools, e.g. lightroom, and it wasn't so nice there. Seeing support grow in Windows and the automatic features are amazing. It looks exactly like what I want, knowing where any of my files are at any given time and be able to setup more complicated transfer schemes to fit my needs exactly. Not to mention the ease of backing up over ssh to any of my linux boxes or offsite, etc. Oh and being able to checksum all of my photos and know they are all present and intact and being able to remove duplicate unorganized backups all over the place would be so great. I also need to work on a small subset on my macbook, so great solution for that as well.
+
diff --git a/doc/bugs/git_annex_assistant_exits_with_failure/comment_1_247d4da208cd700851d48f13534d5f7c._comment b/doc/bugs/git_annex_assistant_exits_with_failure/comment_1_247d4da208cd700851d48f13534d5f7c._comment
new file mode 100644
index 0000000000..f791798864
--- /dev/null
+++ b/doc/bugs/git_annex_assistant_exits_with_failure/comment_1_247d4da208cd700851d48f13534d5f7c._comment
@@ -0,0 +1,13 @@
+[[!comment format=mdwn
+ username="joey"
+ subject="""comment 1"""
+ date="2017-02-13T20:08:32Z"
+ content="""
+"ExitFailure (-1073741819)" looks kind of like an overflow of the exit
+status code, which is very strange.
+
+Is there an error message output anywhere when this happens?
+
+It looks like you might have a ton of files in this repository, and that
+might have something to do with the memory use and load.
+"""]]
diff --git a/doc/bugs/git_annex_assistant_exits_with_failure/comment_2_c6c04006b1d9aa6be92f5a970ba60618._comment b/doc/bugs/git_annex_assistant_exits_with_failure/comment_2_c6c04006b1d9aa6be92f5a970ba60618._comment
new file mode 100644
index 0000000000..987c928383
--- /dev/null
+++ b/doc/bugs/git_annex_assistant_exits_with_failure/comment_2_c6c04006b1d9aa6be92f5a970ba60618._comment
@@ -0,0 +1,15 @@
+[[!comment format=mdwn
+ username="lasitus"
+ avatar="http://cdn.libravatar.org/avatar/dfe778f28027aeb75876172022aa5de3"
+ subject="comment 2"
+ date="2017-02-14T18:46:22Z"
+ content="""
+\"\"ExitFailure (-1073741819)\" looks kind of like an overflow of the exit status code, which is very strange.\"
+It isn't the only number I have received. I believe it was 5 on one run and 3 on another, but that is just from memory. I don't think the error was ever printed to the daemon.log. I can try and run it again and get more definite results.
+
+\"Is there an error message output anywhere when this happens?\"
+I gave you the end of daemon.log and everything that was printed after the crash. Any other logs you would like?
+
+\"It looks like you might have a ton of files in this repository, and that might have something to do with the memory use and load.\"
+After restarting and getting it going again, it was happily moving along with not much memory usage and 0.5 load. It seemed like both the client and the server were hung up on something that time. I had left it to chew on the files for quite some time. (That time the client didn't crash, but it stopped sensing any changes. Restarts of the assistant also didn't fix the issue, only a reboot on the Windows side.) I suppose I could have found whatever was running instead in task manager, but I'm saying a ctrl-c and apparent restart didn't help.
+"""]]
diff --git a/doc/bugs/git_annex_assistant_exits_with_failure/comment_3_eabad530a55adfa4be3eae0f743d3309._comment b/doc/bugs/git_annex_assistant_exits_with_failure/comment_3_eabad530a55adfa4be3eae0f743d3309._comment
new file mode 100644
index 0000000000..5cf9b66004
--- /dev/null
+++ b/doc/bugs/git_annex_assistant_exits_with_failure/comment_3_eabad530a55adfa4be3eae0f743d3309._comment
@@ -0,0 +1,14 @@
+[[!comment format=mdwn
+ username="joey"
+ subject="""comment 3"""
+ date="2017-02-14T18:50:02Z"
+ content="""
+With a weird crash like this, I was thinking there might be an error message
+output to the console. I guess you showed the full console output though.
+
+Was the 85% of system memory happening on Windows or on Linux?
+
+I have no clues how to debug this. If there's useful information anywhere,
+it's probably in some windows crash log or system log thing that I know
+nothing about.
+"""]]
diff --git a/doc/bugs/git_annex_assistant_exits_with_failure/comment_4_204764bb40509d8650386227ea166a6e._comment b/doc/bugs/git_annex_assistant_exits_with_failure/comment_4_204764bb40509d8650386227ea166a6e._comment
new file mode 100644
index 0000000000..6aae58109a
--- /dev/null
+++ b/doc/bugs/git_annex_assistant_exits_with_failure/comment_4_204764bb40509d8650386227ea166a6e._comment
@@ -0,0 +1,12 @@
+[[!comment format=mdwn
+ username="lasitus"
+ avatar="http://cdn.libravatar.org/avatar/dfe778f28027aeb75876172022aa5de3"
+ subject="comment 4"
+ date="2017-02-14T19:08:38Z"
+ content="""
+The high memory use and load were on the Linux side.
+
+Again these were two different behaviors on separate runs. Most runs, the linux side was fine and the windows side crashed. This one I can reproduce easily by just copying my data into an empty repo. I'll try it a few times later tonight and see if it always happens on a particular file.
+
+One time, the windows side stopped sensing files and the Linux side had heavy cpu usage and high memory usage) Sadly, I don't recall the exact time as the hang up happened over night, so hard to find in the log file. So, I guess I will focus on the first one.
+"""]]
diff --git a/doc/bugs/git_annex_assistant_exits_with_failure/comment_5_0453ae9123b359757e63c6eb5f12811b._comment b/doc/bugs/git_annex_assistant_exits_with_failure/comment_5_0453ae9123b359757e63c6eb5f12811b._comment
new file mode 100644
index 0000000000..fe27fba426
--- /dev/null
+++ b/doc/bugs/git_annex_assistant_exits_with_failure/comment_5_0453ae9123b359757e63c6eb5f12811b._comment
@@ -0,0 +1,40 @@
+[[!comment format=mdwn
+ username="lasitus"
+ avatar="http://cdn.libravatar.org/avatar/dfe778f28027aeb75876172022aa5de3"
+ subject="comment 5"
+ date="2017-02-15T02:17:25Z"
+ content="""
+Got a decent result this last run:
+
+Console output
+
+```
+[2017-02-14 20:40:25.3493472] logging to .git\annex\daemon.log
+[2017-02-14 20:40:25.3493472] chat: C:\Program Files (x86)\Git\usr\bin\git-annex.exe [\"assistant\",\"--debug\"]
+[2017-02-14 20:47:21.2931135] process done ExitFailure 1
+```
+
+daemon.log ending
+
+```
+[2017-02-14 20:47:17.4915947] process done ExitSuccess
+[2017-02-14 20:47:19.7279831] read: git [\"--git-dir=.git\",\"--work-tree=.\",\"--literal-pathspecs\",\"-c\",\"core.bare=false\",\"symbolic-ref\",\"-q\",\"HEAD\"]
+[2017-02-14 20:47:19.7436092] process done ExitSuccess
+[2017-02-14 20:47:19.7436092] read: git [\"--git-dir=.git\",\"--work-tree=.\",\"--literal-pathspecs\",\"-c\",\"core.bare=false\",\"show-ref\",\"refs/heads/annex/direct/master\"]
+[2017-02-14 20:47:19.7748596] process done ExitSuccess
+[2017-02-14 20:47:19.7748596] Pusher: pushing to [Remote { name =\"origin\" }]
+[2017-02-14 20:47:19.8217363] Committer: queued Upload UUID \"b77aaec0-e75b-4eec-a08c-b09cbe2947d5\" LR Main\LR Main Smart Previews.lrdata\1\117C\117C0B4F-1D99-4DF6-9E7B-7C0B655876D9.dng Nothing : new file created
+[2017-02-14 20:47:19.8529873] call: git [\"--git-dir=.git\",\"--work-tree=.\",\"--literal-pathspecs\",\"-c\",\"core.bare=false\",\"push\",\"origin\",\"+git-annex:synced/git-annex\",\"annex/direct/master:synced/master\"]
+[2017-02-14 20:47:19.9154902] Transferrer: Transferring: Upload UUID \"b77aaec0-e75b-4eec-a08c-b09cbe2947d5\" LR Main\LR Main Smart Previews.lrdata\1\117C\117C0B4F-1D99-4DF6-9E7B-7C0B655876D9.dng Nothing
+[2017-02-14 20:47:19.9154902] Watcher: add LR Main\LR Main Smart Previews.lrdata\1\162B\162B90D0-CC7C-4BD9-AC2B-BA2D4C72B555.dng
+
+117C0B4F-1D99-4DF6-9E7B-7C0B655876D9.dng
+      1,792,000 100%  152.52MB/s    0:00:00 (xfr#1, to-chk=0/1)
+git-annex.exe: out of memory
+(recording state in git...)
+To storage.lan:/srv/annex/pictures
+   0623558..669c99c  git-annex -> synced/git-annex
+   6309934..b8e043d  annex/direct/master -> synced/master
+
+```
+"""]]
diff --git a/doc/bugs/git_annex_assistant_exits_with_failure/comment_6_5b0310038a8b45d2083f39ea75c1a979._comment b/doc/bugs/git_annex_assistant_exits_with_failure/comment_6_5b0310038a8b45d2083f39ea75c1a979._comment
new file mode 100644
index 0000000000..36a16339de
--- /dev/null
+++ b/doc/bugs/git_annex_assistant_exits_with_failure/comment_6_5b0310038a8b45d2083f39ea75c1a979._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="lasitus"
+ avatar="http://cdn.libravatar.org/avatar/dfe778f28027aeb75876172022aa5de3"
+ subject="comment 6"
+ date="2017-02-15T16:01:38Z"
+ content="""
+Note: The last run was the Windows side
+"""]]
diff --git a/doc/bugs/git_annex_assistant_exits_with_failure/comment_7_63b3ec113d52501237abbe6ee1ef5fa5._comment b/doc/bugs/git_annex_assistant_exits_with_failure/comment_7_63b3ec113d52501237abbe6ee1ef5fa5._comment
new file mode 100644
index 0000000000..9206ba3139
--- /dev/null
+++ b/doc/bugs/git_annex_assistant_exits_with_failure/comment_7_63b3ec113d52501237abbe6ee1ef5fa5._comment
@@ -0,0 +1,45 @@
+[[!comment format=mdwn
+ username="lasitus"
+ avatar="http://cdn.libravatar.org/avatar/dfe778f28027aeb75876172022aa5de3"
+ subject="comment 7"
+ date="2017-02-17T03:23:46Z"
+ content="""
+Ok, I have a script that generates the error. This generates a repository and 30 GB of random binary files with many folders 2 layers deep. Just put in an empty folder and run with python. No remotes are necessary. This was run in Windows 10 in a git bash window.
+
+```
+#!/usr/bin/env python
+
+import logging
+import os
+import shutil
+import subprocess
+import uuid
+
+logging.basicConfig(level=logging.DEBUG)
+
+repositoryPath = os.path.abspath(\"./bigRepoTest\")	
+os.makedirs(repositoryPath)
+
+subprocess.call(\"git init\", cwd=repositoryPath)
+subprocess.call(\"git annex init pc\", cwd=repositoryPath)
+
+def makeRandomDirectories(level1FolderCount, level2FolderCount, fileCount):
+	for directoryIndex in range(0, level1FolderCount):
+		logging.info(\"Adding top level folder \" + str(directoryIndex + 1) + \" of \" + str(level1FolderCount))
+		newDirectory = os.path.join(repositoryPath, str(uuid.uuid1()))
+		os.makedirs(newDirectory)
+		for directoryIndex in range(0, level2FolderCount):
+			newNestedDirectory = os.path.join(newDirectory, str(uuid.uuid1()))
+			os.makedirs(newNestedDirectory)
+			for fileIndex in range(0, fileCount):
+				newFile = os.path.join(newNestedDirectory, str(uuid.uuid1()) + \".bin\")
+				with open(newFile, 'wb') as fileOut:
+					fileOut.write(os.urandom(500000))
+
+makeRandomDirectories(32, 1000, 1)
+with open(os.path.join(repositoryPath, \"assistant.log\"), 'w') as output:
+	subprocess.Popen([\"git\", \"annex\", \"assistant\", \"--debug\"], cwd=repositoryPath, stdout=output, stderr=output)
+	makeRandomDirectories(32, 1000, 1)
+	subprocess.call(\"tail -f daemon.log\", cwd=os.path.join(repositoryPath, \".git\", \"annex\"))
+```
+"""]]
diff --git a/doc/bugs/git_annex_assistant_failing_with_ARM_NAS.mdwn b/doc/bugs/git_annex_assistant_failing_with_ARM_NAS.mdwn
new file mode 100644
index 0000000000..83b963343b
--- /dev/null
+++ b/doc/bugs/git_annex_assistant_failing_with_ARM_NAS.mdwn
@@ -0,0 +1,29 @@
+I'm trying to use a Synology NAS (ARM architecture, DiskStation 214+) as a remote repository for my laptop, but I'm failing to get a convenient configuration to work.
+
+I already set-up git-annex on the NAS following the explanations found [here](http://git-annex.branchable.com/tips/Synology_NAS_and_git_annex/). I installed version **5.20150714-g8695533**.
+
+On my laptop I have the version provided with Ubuntu 14.04: **5.20140412ubuntu1**.
+
+If calling git annex from my laptop's command line and doing everything manually (git remote add, copy file to the dir, git annex add, git commit, git push, git annex copy), then it works properly.
+
+But when trying with the assistant I get this error:
+
+```
+fatal: unrecognized command 'sh -c 'mkdir -p '"'"'annex'"'"'&&cd '"'"'annex'"'"'&&if [ ! -d .git ]; then git init --bare --shared && git config receive.denyNonFastforwards false; fi&&git annex init''
+git-annex-shell: git-shell failed
+```
+
+This is the content of daemon.log:
+[[!format sh """
+[2015-08-04 00:51:41 CEST] main: starting assistant version 5.20140412ubuntu1
+[2015-08-04 00:51:41 CEST] Cronner: You should enable consistency checking to protect your data. 
+(Recording state in git...)
+(scanning...) [2015-08-04 00:51:41 CEST] Watcher: Performing startup scan
+(started...) [2015-08-04 00:52:41 CEST] Cronner: Consistency check in progress
+[2015-08-04 00:59:12 CEST] read: ssh-keygen ["-F","git-annex-trusted"]
+[2015-08-04 00:59:12 CEST] read: ssh ["-oNumberOfPasswordPrompts=0","-oStrictHostKeyChecking=no","-n","-p","22","git-annex@git-annex-trusted","sh -c 'echo git-annex-probe loggedin;if which git-annex-shell; then echo git-annex-probe git-annex-shell; fi;if which git; then echo git-annex-probe git; fi;if which rsync; then echo git-annex-probe rsync; fi;if which ~/.ssh/git-annex-shell; then echo git-annex-probe ~/.ssh/git-annex-shell; fi;cd '\"'\"'annex'\"'\"' && git config --list'"]
+[2015-08-04 00:59:13 CEST] read: gpg ["--batch","--no-tty","--use-agent","--quiet","--trust-model","always","--with-colons","--list-secret-keys","--fixed-list-mode"]
+[2015-08-04 00:59:15 CEST] read: ssh ["-p","22","git-annex@git-annex-trusted","sh -c 'mkdir -p '\"'\"'annex'\"'\"'&&cd '\"'\"'annex'\"'\"'&&if [ ! -d .git ]; then git init --bare --shared && git config receive.denyNonFastforwards false; fi&&git annex init'"]
+"""]]
+
+Is there any problem with the version provided by Ubuntu that is producing this strange behavior?
diff --git a/doc/bugs/git_annex_assistant_failing_with_ARM_NAS/comment_1_00ca17dca9da3ddb81514d319cae0ef4._comment b/doc/bugs/git_annex_assistant_failing_with_ARM_NAS/comment_1_00ca17dca9da3ddb81514d319cae0ef4._comment
new file mode 100644
index 0000000000..0999585abe
--- /dev/null
+++ b/doc/bugs/git_annex_assistant_failing_with_ARM_NAS/comment_1_00ca17dca9da3ddb81514d319cae0ef4._comment
@@ -0,0 +1,14 @@
+[[!comment format=mdwn
+ username="joey"
+ subject="""comment 1"""
+ date="2015-08-04T20:09:36Z"
+ content="""
+The assistant's automation for setting up a remote ssh server is what looks
+to be failing. It seems that this NAS has a problem with ssh being used to
+run a sh -c command. Probably the NAS's shell is not POSIX compliant
+in some way.
+
+The best thing to do in this situation is to set up the git remote
+on the NAS manually. Once it's set up, the assistant should be able to use
+it.
+"""]]
diff --git a/doc/bugs/git_annex_assistant_leaks_a_lot_of_memory.mdwn b/doc/bugs/git_annex_assistant_leaks_a_lot_of_memory.mdwn
new file mode 100644
index 0000000000..3f701b788b
--- /dev/null
+++ b/doc/bugs/git_annex_assistant_leaks_a_lot_of_memory.mdwn
@@ -0,0 +1,9 @@
+Hi,
+
+I have a git-annex repository on a server managed by git annex assistant in indirect mode and as a backup group serving my org-mode files (organizer module for emacs) which are plain text files no more than a 100kb.
+
+I also have 2 clients in direct mode to sync the files across my computers.
+
+I save my files fairly often so the assistant might be a bit overwhelmed but I think it can managed that. At least it used to.
+
+Since maybe a couple months, the assistant running on the server has been leaking nearly 1Gb each day and I suspect it rises whenever I push files to it.
diff --git a/doc/bugs/git_annex_assistant_leaks_a_lot_of_memory/comment_1_a4805854c448e4bc70a47f5555e0f73c._comment b/doc/bugs/git_annex_assistant_leaks_a_lot_of_memory/comment_1_a4805854c448e4bc70a47f5555e0f73c._comment
new file mode 100644
index 0000000000..198731df9b
--- /dev/null
+++ b/doc/bugs/git_annex_assistant_leaks_a_lot_of_memory/comment_1_a4805854c448e4bc70a47f5555e0f73c._comment
@@ -0,0 +1,21 @@
+[[!comment format=mdwn
+ username="joey"
+ subject="""comment 1"""
+ date="2014-11-06T19:53:02Z"
+ content="""
+Does the assistant's memory use go back to being small when you stop 
+and restart the daemon, or does it immediately go back up?
+
+Are there a lot of files in `.git/annex/transfer`? One possibility
+is it might be getting flooded with failed/pending file transfers.
+
+If you feel up to building git-annex from source, you could
+enable the EKG remote monitoring interface. This would let you see
+graphs of the memory use, and probably get a good idea of what's
+using memory. Or, you could do a full memory profiling build of
+git-annex, which will let us nail down exactly what's using the memory.
+
+See [[this_page|ekg]] for instructions for building git-annex for EKG
+and memory profiling. If it's too hard for you to do that, I can 
+probably provide you with a prebuilt version with memory profiling.
+"""]]
diff --git a/doc/bugs/git_annex_assistant_leaks_a_lot_of_memory/comment_2_e9dca04e0456ca7356996365a4b87d59._comment b/doc/bugs/git_annex_assistant_leaks_a_lot_of_memory/comment_2_e9dca04e0456ca7356996365a4b87d59._comment
new file mode 100644
index 0000000000..d7d4dec394
--- /dev/null
+++ b/doc/bugs/git_annex_assistant_leaks_a_lot_of_memory/comment_2_e9dca04e0456ca7356996365a4b87d59._comment
@@ -0,0 +1,14 @@
+[[!comment format=mdwn
+ username="https://renaud.casenave.fr/"
+ subject="comment 2"
+ date="2014-11-08T07:18:48Z"
+ content="""
+Hi,
+
+Yes, the assistant memory goes back to normal when I do `git annex assistant --stop && git annex assistant`. But when I look at htop the next day, it has already taken 1Gb.
+
+There are quite a lot of files in `.git/annex/transfer`… wc reports more than 200 entries, mainly in the upload folder, although the repository only manages 10 or so files.
+
+I tried building git-annex using cabal but `cabal configure -fEKG` exits with the code 9 and even without `-fEKG` cabal refuses to install some of the dependencies. I'll try again some other time…
+
+"""]]
diff --git a/doc/bugs/git_annex_assistant_leaks_a_lot_of_memory/comment_3_45ae862d4f940341d48d179583e65d66._comment b/doc/bugs/git_annex_assistant_leaks_a_lot_of_memory/comment_3_45ae862d4f940341d48d179583e65d66._comment
new file mode 100644
index 0000000000..294ca5f982
--- /dev/null
+++ b/doc/bugs/git_annex_assistant_leaks_a_lot_of_memory/comment_3_45ae862d4f940341d48d179583e65d66._comment
@@ -0,0 +1,38 @@
+[[!comment format=mdwn
+ username="https://renaud.casenave.fr/"
+ subject="comment 3"
+ date="2014-11-15T02:07:38Z"
+ content="""
+Well it's been a week and still no luck compiling git-annex using cabal :(
+
+I'm blocked here (at pretty much the start):
+
+    % cabal install --only-dependencies
+    Resolving dependencies...
+    cabal: Could not resolve dependencies:
+    trying: git-annex-5.20141125 (user goal)
+    trying: git-annex-5.20141125:+tahoe
+    trying: git-annex-5.20141125:+dns
+    trying: aeson-0.6.0.2/installed-5b6... (dependency of
+    git-annex-5.20141125:+tahoe)
+    trying: vector-0.9.1/installed-5fe... (dependency of
+    aeson-0.6.0.2/installed-5b6...)
+    trying: dns-1.4.4 (dependency of git-annex-5.20141125:+dns)
+    next goal: network (dependency of dns-1.4.4)
+    rejecting: network-2.3.0.13/installed-0e3... (conflict:
+    git-annex-5.20141125:network-uri => network(>=2.6))
+    trying: network-2.6.0.2
+    next goal: conduit-extra (dependency of dns-1.4.4)
+    rejecting: conduit-extra-1.1.4.2, 1.1.4.1, 1.1.4, 1.1.3.4, 1.1.3.3, 1.1.3.2,
+    1.1.3.1, 1.1.3, 1.1.2, 1.1.1, 1.1.0.4 (conflict: vector =>
+    primitive==0.4.1/installed-505..., conduit-extra => primitive>=0.5)
+    rejecting: conduit-extra-1.1.0.3, 1.1.0.2, 1.1.0.1 (conflict: vector =>
+    primitive==0.4.1/installed-505..., conduit-extra => primitive>=0.5 && <0.6)
+    rejecting: conduit-extra-1.1.0 (conflict: network==2.6.0.2, conduit-extra =>
+    network>=2.3 && <2.5)
+    rejecting: conduit-extra-1.0.0.1, 1.0.0, 0.1.7, 0.1.6, 0.1.5, 0.1.4, 0.1.3,
+    0.1.2, 0.1.1, 0.1.0 (conflict: dns => conduit-extra>=1.1)
+    zsh: exit 1     cabal install --only-dependencies
+
+Should I just wait for the maintainers to fix the dependencies? Or is there another way?
+"""]]
diff --git a/doc/bugs/git_annex_assistant_leaks_a_lot_of_memory/comment_4_d597e15e9693ce6e8c1ca80cb225b044._comment b/doc/bugs/git_annex_assistant_leaks_a_lot_of_memory/comment_4_d597e15e9693ce6e8c1ca80cb225b044._comment
new file mode 100644
index 0000000000..ccd35305bc
--- /dev/null
+++ b/doc/bugs/git_annex_assistant_leaks_a_lot_of_memory/comment_4_d597e15e9693ce6e8c1ca80cb225b044._comment
@@ -0,0 +1,21 @@
+[[!comment format=mdwn
+ username="https://www.google.com/accounts/o8/id?id=AItOawntodrSgODU27WUCyN2PV7TC14YMkyaoxQ"
+ nickname="Dennis"
+ subject="comment 4"
+ date="2014-12-11T10:58:21Z"
+ content="""
+I suffer from the same issue.
+
+In ~/.profile I put the line `git-annex assistant --quiet --autostart --startdelay=5s > /dev/null &`.
+
+Htop shows:
+
+    26291 xxx      20   0 6584M 3123M  9864 S  0.0 81.6 29h29:17 git-annex assistant --startdelay=5s
+    26292 xxx      20   0 6584M 3123M  9864 S  0.0 81.6 12h38:32 git-annex assistant --startdelay=5s
+    26293 xxx      20   0 6584M 3123M  9864 S  0.0 81.6 21h45:49 git-annex assistant --startdelay=5s
+    26321 xxx      20   0 6584M 3123M  9864 D  0.0 81.6 18:34.12 git-annex assistant --startdelay=5s
+    21019 xxx      20   0 6584M 3123M  9864 S  0.0 81.6  0:00.08 git-annex assistant --startdelay=5s
+    21908 xxx      20   0 6584M 3123M  9864 S  0.0 81.6  0:00.00 git-annex assistant --startdelay=5s
+    26290 xxx      20   0 6584M 3123M  9864 S  0.0 81.6 64h33:50 git-annex assistant --startdelay=5s
+
+"""]]
diff --git a/doc/bugs/git_annex_cannot_get_my_files_after_clone.mdwn b/doc/bugs/git_annex_cannot_get_my_files_after_clone.mdwn
new file mode 100644
index 0000000000..764e7b1df3
--- /dev/null
+++ b/doc/bugs/git_annex_cannot_get_my_files_after_clone.mdwn
@@ -0,0 +1,36 @@
+### Please describe the problem.
+
+git annex cannot get me my file contents from my remotes
+
+### What steps will reproduce the problem?
+
+```
+git clone myserver:/path/to/repo
+cd repo
+git annex get
+```
+
+### What version of git-annex are you using? On what operating system?
+Using git-annex-5.20150327 on gentoo linux
+
+### Please provide any additional information below.
+
+The error messages I get egenrally look like:
+
+```
+get 2att/photos-glasgow/male/patrik.jpg (from clusterhost...) 
+git-annex: ../chymera/data/.git/annex/transfer/upload/b8415264-4b9a-40ca-b450-7e57507cdc06/lck.SHA256E-s814245--9dc6f1287ba683cae030e04ba7f94a73e566ce392c2d032f171094ddc342fa60.jpg: openFd: does not exist (No such file or directory)
+git-annex-shell: sendkey: 1 failed
+protocol version mismatch -- is your shell clean?
+(see the rsync man page for an explanation)
+rsync error: protocol incompatibility (code 2) at compat.c(176) [Receiver=3.1.1]
+
+  rsync failed -- run git annex again to resume file transfer
+
+  Unable to access these remotes: clusterhost
+
+  Try making some of these repositories available:
+  	809074b6-e079-4ea1-b2f8-2d7840deda7d -- zenbookhost
+   	a1ed6786-8b93-4a14-b00d-877b741e34da -- [clusterhost]
+failed
+```
diff --git a/doc/bugs/git_annex_cannot_get_my_files_after_clone/comment_1_52c91f8d2e8086b26a078a02d036c197._comment b/doc/bugs/git_annex_cannot_get_my_files_after_clone/comment_1_52c91f8d2e8086b26a078a02d036c197._comment
new file mode 100644
index 0000000000..2a375c56c1
--- /dev/null
+++ b/doc/bugs/git_annex_cannot_get_my_files_after_clone/comment_1_52c91f8d2e8086b26a078a02d036c197._comment
@@ -0,0 +1,28 @@
+[[!comment format=mdwn
+ username="joey"
+ subject="""comment 1"""
+ date="2015-08-19T16:12:12Z"
+ content="""
+From the rsync man page that the error directs you to:
+
+> rsync occasionally produces error messages that may seem a little cryp‐
+> tic. The one that seems to cause the most confusion is  "protocol  ver‐
+> sion mismatch -- is your shell clean?".
+
+> This  message is usually caused by your startup scripts or remote shell
+> facility producing unwanted garbage on the stream that rsync  is  using
+> for  its  transport.  The  way  to diagnose this problem is to run your
+> remote shell like this:
+
+>        ssh remotehost /bin/true > out.dat
+
+
+> then look at out.dat. If everything is working correctly  then  out.dat
+> should  be  a zero length file. If you are getting the above error from
+> rsync then you will probably find that out.dat contains  some  text  or
+> data.  Look  at  the contents and try to work out what is producing it.
+> The most common cause is incorrectly configured shell  startup  scripts
+> (such  as  .cshrc  or  .profile)  that  contain  output  statements for
+> non-interactive logins.
+
+"""]]
diff --git a/doc/bugs/git_annex_cannot_get_my_files_after_clone/comment_2_545c9daafc63c5cdb80763b929c8d622._comment b/doc/bugs/git_annex_cannot_get_my_files_after_clone/comment_2_545c9daafc63c5cdb80763b929c8d622._comment
new file mode 100644
index 0000000000..efbb2327e7
--- /dev/null
+++ b/doc/bugs/git_annex_cannot_get_my_files_after_clone/comment_2_545c9daafc63c5cdb80763b929c8d622._comment
@@ -0,0 +1,32 @@
+[[!comment format=mdwn
+ username="https://me.yahoo.com/a/yx5Y6EI1t.759Jsu63ZWqYclCmpOmxxd.ramtw--#7114a"
+ nickname="Ioanas"
+ subject="comment 2"
+ date="2015-08-20T09:37:21Z"
+ content="""
+Doesn't seem to be the case. I get nothing in my out.dat file, but still the same rsync complaint from git annex. 
+
+```
+guest@labhost ~ $ ssh clusterhost /bin/true > out.dat
+guest@labhost ~ $ cat out.dat
+guest@labhost ~ $ ls -l out.dat 
+-rw-r--r-- 1 guest guest 0 20. Aug 11:31 out.dat
+guest@labhost ~ $ cd /mnt/data/data/                
+guest@labhost /mnt/data/data $ git annex get gt.ep/pcr/
+get gt.ep/pcr/pcr_0000.csv (from clusterhost...) 
+protocol version mismatch -- is your shell clean?
+(see the rsync man page for an explanation)
+rsync error: protocol incompatibility (code 2) at compat.c(176) [Receiver=3.1.1]
+git-annex: ../chymera/data/.git/annex/transfer/upload/b8415264-4b9a-40ca-b450-7e57507cdc06/lck.SHA256E-s960--4689d3b876dfeabb3d9578204c8df7c4af0fd6c0f09691cffb3dfd86130e6a27.csv: openFd: does not exist (No such file or directory)
+git-annex-shell: sendkey: 1 failed
+
+  rsync failed -- run git annex again to resume file transfer
+
+  Unable to access these remotes: clusterhost
+
+  Try making some of these repositories available:
+  	809074b6-e079-4ea1-b2f8-2d7840deda7d -- zenbookhost
+   	a1ed6786-8b93-4a14-b00d-877b741e34da -- [clusterhost]
+failed
+```
+"""]]
diff --git a/doc/bugs/git_annex_cannot_get_my_files_after_clone/comment_3_ef9b32d9ba1b80c313db48be36cc90d1._comment b/doc/bugs/git_annex_cannot_get_my_files_after_clone/comment_3_ef9b32d9ba1b80c313db48be36cc90d1._comment
new file mode 100644
index 0000000000..4dc1985bff
--- /dev/null
+++ b/doc/bugs/git_annex_cannot_get_my_files_after_clone/comment_3_ef9b32d9ba1b80c313db48be36cc90d1._comment
@@ -0,0 +1,26 @@
+[[!comment format=mdwn
+ username="joey"
+ subject="""comment 3"""
+ date="2015-08-20T14:58:32Z"
+ content="""
+The error message about the transfer lock file is probably relevant, and
+seems to be coming from git-annex-shell on the clusterhost server.
+
+Since this code has recently been changed and partly rewritten,
+you ought to try upgrading git-annex on clusterhost to a more recent
+version.
+
+If that doesn't help, check if the specified lock file exists,
+and if its parent directory exists. It's possible that the directory it's
+trying to put the lock file in doesn't exist and this is causing the
+problem. If so, manually creating the directory would solve it.
+
+The other possibility seems to be that it's trying to open a lock file for
+read that doesn't exist. But I don't see how that can happen, at least
+not with the current code which catches any such exception.
+
+Stracing git-annex-shell would help narrow this down. 
+The git-annex-shell command that's failing is something like this:
+
+git-annex-shell sendkey ../chymera/data SHA256E-s814245--9dc6f1287ba683cae030e04ba7f94a73e566ce392c2d032f171094ddc342fa60.jpg
+"""]]
diff --git a/doc/bugs/git_annex_cannot_get_my_files_after_clone/comment_4_d1d13d1f3366026957d572b930cafa4b._comment b/doc/bugs/git_annex_cannot_get_my_files_after_clone/comment_4_d1d13d1f3366026957d572b930cafa4b._comment
new file mode 100644
index 0000000000..29853c313b
--- /dev/null
+++ b/doc/bugs/git_annex_cannot_get_my_files_after_clone/comment_4_d1d13d1f3366026957d572b930cafa4b._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="anntzer.lee@b6e099b6ef8ba8b53005ca7f0e473e825001129b"
+ nickname="anntzer.lee"
+ subject="comment 4"
+ date="2016-01-08T07:35:21Z"
+ content="""
+I got the same error (with the same no-output from `ssh remote /bin/true`) with a fresh install of git annex, *when I did not have write rights to the remote's .git folder*.  I temporarily solved the issue by giving myself write rights to that folder, but can this be avoided?
+"""]]
diff --git a/doc/bugs/git_annex_cannot_get_my_files_after_clone/comment_5_f8b58f9c5acb509871cc4a64b66de6f3._comment b/doc/bugs/git_annex_cannot_get_my_files_after_clone/comment_5_f8b58f9c5acb509871cc4a64b66de6f3._comment
new file mode 100644
index 0000000000..e23ad93b7c
--- /dev/null
+++ b/doc/bugs/git_annex_cannot_get_my_files_after_clone/comment_5_f8b58f9c5acb509871cc4a64b66de6f3._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="joey"
+ subject="""comment 5"""
+ date="2016-01-11T16:36:01Z"
+ content="""
+git-annex needs write access to the repository it's running in. This is a
+fairly pervasive assumption and it would not be at all easy to fix.
+"""]]
diff --git a/doc/bugs/git_annex_daemon_crashes_when_authenticating_with_jabber.de.mdwn b/doc/bugs/git_annex_daemon_crashes_when_authenticating_with_jabber.de.mdwn
new file mode 100644
index 0000000000..cc38813557
--- /dev/null
+++ b/doc/bugs/git_annex_daemon_crashes_when_authenticating_with_jabber.de.mdwn
@@ -0,0 +1,26 @@
+### Please describe the problem.
+In the webapp, I enter my credentials for my jabber.de account.  When I hit "use this account" the daemon and webapp crash.
+
+### What steps will reproduce the problem?
+Presumably, authenticate with a jabber.de account.
+
+### What version of git-annex are you using? On what operating system?
+git-annex version: 5.20140517-g0aed6d9
+build flags: Assistant Webapp Webapp-secure Pairing Testsuite S3 WebDAV FsEvents XMPP DNS Feeds Quvi TDFA CryptoHash
+key/value backends: SHA256E SHA1E SHA512E SHA224E SHA384E SKEIN256E SKEIN512E SHA256 SHA1 SHA512 SHA224 SHA384 SKEIN256 SKEIN512 WORM URL
+remote types: git gcrypt S3 bup directory rsync web webdav tahoe glacier ddar hook external
+local repository version: 5
+supported repository version: 5
+upgrade supported from repository versions: 0 1 2 4
+
+### Please provide any additional information below.
+
+[[!format sh """
+[2014-05-25 18:11:02 EDT] 127.0.0.1 POST /config/xmpp Mozilla/5.0 (Macintosh; Intel Mac OS X 10_9_3) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/35.0.1916.114 Safari/537.36
+git-annex: : hGetBuf: resource vanished (Connection reset by peer)
+git-annex: interrupted
+"""]]
+
+[[!tag confirmed forwarded]]
+
+[[!meta title="OSX xmpp crash with jabber.de"]]
diff --git a/doc/bugs/git_annex_daemon_crashes_when_authenticating_with_jabber.de/comment_1_5ba4f22bda2f5438fb08753cf149b649._comment b/doc/bugs/git_annex_daemon_crashes_when_authenticating_with_jabber.de/comment_1_5ba4f22bda2f5438fb08753cf149b649._comment
new file mode 100644
index 0000000000..003b60b9fb
--- /dev/null
+++ b/doc/bugs/git_annex_daemon_crashes_when_authenticating_with_jabber.de/comment_1_5ba4f22bda2f5438fb08753cf149b649._comment
@@ -0,0 +1,12 @@
+[[!comment format=mdwn
+ username="http://joeyh.name/"
+ ip="209.250.56.176"
+ subject="comment 1"
+ date="2014-05-27T18:45:35Z"
+ content="""
+I was able to use that server from git-annex on Linux successfully.
+
+I also tried on OSX, which appears to be the OS you're using. The assistant didn't crash. I may not have exactly replicated what you did though.
+
+Can you enable debug logging in Configuration -> Preferences and try to do this again, and send any parts of the log that mention XMPP?
+"""]]
diff --git a/doc/bugs/git_annex_daemon_crashes_when_authenticating_with_jabber.de/comment_2_b096cfdf26bfedfff16d882d7b2e060d._comment b/doc/bugs/git_annex_daemon_crashes_when_authenticating_with_jabber.de/comment_2_b096cfdf26bfedfff16d882d7b2e060d._comment
new file mode 100644
index 0000000000..87e6b746eb
--- /dev/null
+++ b/doc/bugs/git_annex_daemon_crashes_when_authenticating_with_jabber.de/comment_2_b096cfdf26bfedfff16d882d7b2e060d._comment
@@ -0,0 +1,10 @@
+[[!comment format=mdwn
+ username="https://www.google.com/accounts/o8/id?id=AItOawmZilYULa6CDEGfuagoDlesyakBgnf-dF8"
+ nickname="Maarten"
+ subject="comment 2"
+ date="2014-05-27T18:59:08Z"
+ content="""
+This was debug logging enabled, it appeared to be the only relevant parts in the log but I can try again and post more verbosely.
+
+If nothing more comes up, what else could I do to get you diagnostic data?
+"""]]
diff --git a/doc/bugs/git_annex_daemon_crashes_when_authenticating_with_jabber.de/comment_3_210be77aabb0ef5b85865cd08c51861e._comment b/doc/bugs/git_annex_daemon_crashes_when_authenticating_with_jabber.de/comment_3_210be77aabb0ef5b85865cd08c51861e._comment
new file mode 100644
index 0000000000..c2db30db0d
--- /dev/null
+++ b/doc/bugs/git_annex_daemon_crashes_when_authenticating_with_jabber.de/comment_3_210be77aabb0ef5b85865cd08c51861e._comment
@@ -0,0 +1,12 @@
+[[!comment format=mdwn
+ username="http://joeyh.name/"
+ ip="209.250.56.176"
+ subject="comment 3"
+ date="2014-05-27T19:03:20Z"
+ content="""
+Hmm, so it was.. There was so little logged that I missed it.
+
+Can you reproduce the crash?
+
+I think that OSX has some ways to get a dump of information about a crash, but I don't know details.
+"""]]
diff --git a/doc/bugs/git_annex_daemon_crashes_when_authenticating_with_jabber.de/comment_4_bb974d882fbb31aa54e6de38410a2318._comment b/doc/bugs/git_annex_daemon_crashes_when_authenticating_with_jabber.de/comment_4_bb974d882fbb31aa54e6de38410a2318._comment
new file mode 100644
index 0000000000..059eec0500
--- /dev/null
+++ b/doc/bugs/git_annex_daemon_crashes_when_authenticating_with_jabber.de/comment_4_bb974d882fbb31aa54e6de38410a2318._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="https://www.google.com/accounts/o8/id?id=AItOawmZilYULa6CDEGfuagoDlesyakBgnf-dF8"
+ nickname="Maarten"
+ subject="comment 4"
+ date="2014-05-27T19:07:45Z"
+ content="""
+I might attach wireshark but I don't know if it'll help much seeing as jabber.de as most XMPP services just went SSL only.
+"""]]
diff --git a/doc/bugs/git_annex_daemon_crashes_when_authenticating_with_jabber.de/comment_5_be655cbea27b6c87d298a75e1697a87d._comment b/doc/bugs/git_annex_daemon_crashes_when_authenticating_with_jabber.de/comment_5_be655cbea27b6c87d298a75e1697a87d._comment
new file mode 100644
index 0000000000..f6aa9d45b2
--- /dev/null
+++ b/doc/bugs/git_annex_daemon_crashes_when_authenticating_with_jabber.de/comment_5_be655cbea27b6c87d298a75e1697a87d._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="http://joeyh.name/"
+ ip="209.250.56.176"
+ subject="comment 5"
+ date="2014-05-30T19:09:31Z"
+ content="""
+If you can reproduce the crash, I can provide some debug program that will narrow down where the bug is..
+"""]]
diff --git a/doc/bugs/git_annex_daemon_crashes_when_authenticating_with_jabber.de/comment_6_72d1dac6627bbe22d5bd140215d82c47._comment b/doc/bugs/git_annex_daemon_crashes_when_authenticating_with_jabber.de/comment_6_72d1dac6627bbe22d5bd140215d82c47._comment
new file mode 100644
index 0000000000..909fdd3ae8
--- /dev/null
+++ b/doc/bugs/git_annex_daemon_crashes_when_authenticating_with_jabber.de/comment_6_72d1dac6627bbe22d5bd140215d82c47._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="https://www.google.com/accounts/o8/id?id=AItOawmZilYULa6CDEGfuagoDlesyakBgnf-dF8"
+ nickname="Maarten"
+ subject="comment 6"
+ date="2014-05-30T19:37:03Z"
+ content="""
+I can definitely reproduce it on my end
+"""]]
diff --git a/doc/bugs/git_annex_daemon_crashes_when_authenticating_with_jabber.de/comment_7_8a8dfb15684525e156d6334e9f67a55c._comment b/doc/bugs/git_annex_daemon_crashes_when_authenticating_with_jabber.de/comment_7_8a8dfb15684525e156d6334e9f67a55c._comment
new file mode 100644
index 0000000000..8e862d468f
--- /dev/null
+++ b/doc/bugs/git_annex_daemon_crashes_when_authenticating_with_jabber.de/comment_7_8a8dfb15684525e156d6334e9f67a55c._comment
@@ -0,0 +1,66 @@
+[[!comment format=mdwn
+ username="http://joeyh.name/"
+ ip="209.250.56.176"
+ subject="comment 7"
+ date="2014-05-30T20:47:02Z"
+ content="""
+I wrote a test program to try to connect to this server, using a test account, and dump some events. This works on linux. Then I tried it on mac, and I think I have reproduced the same failure!
+
+
    +oberon:~ joeyh$ ./xmpp
+xmpp: : hGetBuf: resource vanished (Connection reset by peer)
+
    
+
+Note that this exception is unable to be caught, it seems. Which would explain why the whole git-annex assistant crashes.
+
+Also, the same program, when modified to use some other xmpp server, does not crash.
+
+At this point, I think I need to forward this bug to network-protocol-xmpp author John Millikin. Which I've now done.
+
+My advice for the bug submitter: git-annex is going to be deprecating XMPP in the not too distant future anyway. If you have your own server, a much nicer way to use git-annex is to install it on the server and use ssh remotes. Recent versions do not need xmpp to sync between clients in such a configuration.
+
+Test program:
+
+[[!format haskell \"\"\"
+{-# LANGUAGE OverloadedStrings #-}
+
+import Network.Protocol.XMPP
+import Data.Maybe
+import Data.Either
+import Network
+import Control.Monad
+import Control.Monad.IO.Class
+import Control.Exception
+
+main = do
+        catch test
+                (\e -> do
+                        let err = show (e :: IOException)
+                        print (\"caught: \", err)
+                        return (Right ())
+                )
+        print \"clean exit\"
+
+test :: IO (Either Error ())
+test = runClient server (getjid jid) username password $ do
+        liftIO $ print \"connected to server\"
+        jid <- bindJID (getjid jid)
+        liftIO $ print (\"bound to jid\", jid)
+        forever $ do
+                s <- getStanza
+                liftIO $ print (\"got\", s)
+  where
+        getjid t = fromMaybe (error \"jid parse error\") (parseJID t)
+        server = Server
+                (getjid serverjid)
+                servername
+                (PortNumber port)
+
+        servername = \"jabber.de\"
+        serverjid = \"jabber.de\"
+        port = 5222
+        jid = \"haskellxmpptest@jabber.de\"
+        username = \"haskellxmpptest\"
+        password = \"stupidpassword\"
+\"\"\"]]
+"""]]
diff --git a/doc/bugs/git_annex_daemon_crashes_when_authenticating_with_jabber.de/comment_8_f1a6e413756066659020e20147373a11._comment b/doc/bugs/git_annex_daemon_crashes_when_authenticating_with_jabber.de/comment_8_f1a6e413756066659020e20147373a11._comment
new file mode 100644
index 0000000000..c4a8749543
--- /dev/null
+++ b/doc/bugs/git_annex_daemon_crashes_when_authenticating_with_jabber.de/comment_8_f1a6e413756066659020e20147373a11._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="http://joeyh.name/"
+ ip="209.250.56.2"
+ subject="comment 8"
+ date="2014-07-11T19:57:35Z"
+ content="""
+Unfortunately, this seems to be somewhat intermittent, so while I had a test case that worked once, John never saw it fail and has not been able to debug it. The test case has successfully connected every time I've tried it recently too. Perhaps jabber.de has changed whatever was causing the problem..
+"""]]
diff --git a/doc/bugs/git_annex_enableremote_gcrypt_failure_leaves_a_remote.mdwn b/doc/bugs/git_annex_enableremote_gcrypt_failure_leaves_a_remote.mdwn
new file mode 100644
index 0000000000..021d782deb
--- /dev/null
+++ b/doc/bugs/git_annex_enableremote_gcrypt_failure_leaves_a_remote.mdwn
@@ -0,0 +1,31 @@
+### Please describe the problem.
+
+Running `git-annex enableremote remote` without a suitable key available leaves a partially unconfigured git remote.
+
+Subsequently making the key available and running `git-annex enableremote remote` a second time fails as the remote now partially exists.
+
+Removing the remote with `git remote remove remote` and then re-running `git-annex enable remote` seems to do the trick.
+
+However, I notice that even after syncing my special remotes are missing some details in the git config file - namely the `annex-gcrypt` and `annex-uuid` (there may have been more).  I fixed by adding the details from a working repo.
+
+Finally, when I initially created the gcrypt repo (by hand) I specified the remote as a local directory on the machine in question i.e. `gcrypt::/matt/mnt/isilon/rss/zzalsmf3/git-annex/mus` I then decided to make it available and changed it (via `git remote rename` to `gcrypt::ssh://rss/home/matt/mnt/isilon/rss/zzalsmf3/git-annex/mus`.  Unfortunately on other machines when I `git enableremote` it recovers the original remote which obviously doesn't work - how can I update this setting?
+
+Phew!
+
+
+
+### What steps will reproduce the problem?
+
+
+### What version of git-annex are you using? On what operating system?
+
+
+### Please provide any additional information below.
+
+[[!format sh """
+# If you can, paste a complete transcript of the problem occurring here.
+# If the problem is with the git-annex assistant, paste in .git/annex/daemon.log
+
+
+# End of transcript or log.
+"""]]
diff --git a/doc/bugs/git_annex_enableremote_ignoring_encryption_changes.mdwn b/doc/bugs/git_annex_enableremote_ignoring_encryption_changes.mdwn
new file mode 100644
index 0000000000..92a454c187
--- /dev/null
+++ b/doc/bugs/git_annex_enableremote_ignoring_encryption_changes.mdwn
@@ -0,0 +1,120 @@
+### Please describe the problem.
+
+Git-Annex enableremote ignores encryption changes to a hybrid-cypher gcrypt remote. So it is not possible to add or remove keys.
+
+
+### What steps will reproduce the problem?
+
+Having a bare repo prepared do:
+
+    git annex initremote $remotename type=gcrypt encryption=hybrid gitrepo=ssh://giessen@magierenklave.de/annex/test keyid=$myfirstkey
+        initremote $remotename (encryption setup) (to gpg keys: $myfirstkey) $user@$hosts's password:
+    (...)
+    git annex sync $remotename
+        (...)
+    git annex info $remotename
+        (...)
+        uuid: $remoteuuid
+        (...)
+        encryption: hybrid (to gpg keys: $myfirstkey)
+        (...)
+    git config --get remote.$remotename.gcrypt-participants
+        $myfirstkey
+
+So far, so good. Now we try adding another key
+
+    git annex enableremote $remotename keyid+=$mysecondkey
+        enableremote $remotename ok
+
+The command returns instantly, no interaction with the remote.
+
+    git annex sync $remotename
+        (...)
+    git annex info $remotename
+        (...)
+        uuid: $remoteuuid
+        (...)
+        encryption: hybrid (to gpg keys: $myfirstkey)
+        (...)
+    git config --get remote.$remotename.gcrypt-participants
+        $myfirstkey
+
+Obviously our change had no effect. This can be verified by the repository being inaccessible to the second key.
+
+A very hacky workaround seems to be:
+
+    git remote remove server.test
+(we need this or git will complain about the remote already existing and annex will fail)
+    git annex enableremote $remoteuuid keyid+=$mysecondkey
+        enableremote $remoteuuid (encryption update) (to gpg keys: $myfirstkey $mysecondkey) $user@$hosts's password:
+        (...)
+    git annex info $remotename
+        (...)
+        encryption: hybrid (to gpg keys: $myfirstkey $mysecondkey)
+        (...)
+    git config --get remote.$remotename.gcrypt-participants
+        $myfirstkey $mysecondkey
+
+...which yields the expected results but is somewhat counterintuitive and probably a bad abuse of enableremote. 
+
+### What version of git-annex are you using? On what operating system?
+
+git annex version:
+
+    git-annex version: 6.20170302-gb35a50cca
+    build flags: Assistant Webapp Pairing Testsuite S3(multipartupload)(storageclasses) WebDAV Inotify DBus DesktopNotify ConcurrentOutput TorrentParser MagicMime Feeds Quvi
+    key/value backends: SHA256E SHA256 SHA512E SHA512 SHA224E SHA224 SHA384E SHA384 SHA3_256E SHA3_256 SHA3_512E SHA3_512 SHA3_224E SHA3_224 SHA3_384E SHA3_384 SKEIN256E SKEIN256 SKEIN512E SKEIN512 SHA1E SHA1 MD5E MD5 WORM URL
+    remote types: git gcrypt p2p S3 bup directory rsync web bittorrent webdav tahoe glacier ddar hook external
+    local repository version: 6
+    supported repository versions: 3 5 6
+    upgrade supported from repository versions: 0 1 2 3 4 5
+    operating system: linux x86_64
+
+uname -a
+
+    Linux $host.$domain 4.9.3-1-default #1 SMP PREEMPT Thu Jan 12 11:32:53 UTC 2017 (2c7dfab) x86_64 x86_64 x86_64 GNU/Linux
+
+### Please provide any additional information below.
+
+gpg is running with -vvv so please don't mind the extra-verbose gpg-output.
+
+[[!format sh """
+# If you can, paste a complete transcript of the problem occurring here.
+# If the problem is with the git-annex assistant, paste in .git/annex/daemon.log
+:~/test> git annex enableremote test keyid+=$mysecondkey
+enableremote test ok
+:~/test> git remote remove test
+:~/test> git annex enableremote 1291d1b4-dac1-5c44-b09e-e04c03b755d6 keyid+=$mysecondkey
+enableremote 1291d1b4-dac1-5c44-b09e-e04c03b755d6 (encryption update) (to gpg keys: $myfirstkey $mysecondkey) user@server's password: 
+gcrypt: Decrypting manifest
+gpg: Signatur vom Mi 22 Mär 2017 14:50:39 CET
+gpg:                mittels RSA-Schlüssel 0144XXXXXXXXXXXXXXXXXXXXXXX
+gpg: Korrekte Signatur von "XXXXXX" [ultimativ]
+gcrypt: Remote ID is :id:Zfhze/U8YGNRIWmfNUV5
+Von gcrypt::ssh://user@server/annex/test
+ * [neuer Branch]    synced/git-annex -> test/synced/git-annex
+ * [neuer Branch]    synced/master    -> test/synced/master
+ * [neuer Branch]    git-annex        -> test/git-annex
+ * [neuer Branch]    master           -> test/master
+user@server's password: 
+gcrypt: Decrypting manifest
+gpg: Signatur vom Mi 22 Mär 2017 14:50:39 CET
+gpg:                mittels RSA-Schlüssel 0144XXXXXXXXXXXXXXXXXXXXXX
+gpg: Korrekte Signatur von "XXXXXX" [ultimativ]
+Everything up-to-date
+user@server's password: 
+ok
+ok
+(recording state in git...)
+# End of transcript or log.
+"""]]
+
+### Have you had any luck using git-annex before? (Sometimes we get tired of reading bug reports all day and a lil' positive end note does wonders)
+
+Yes, definitively. I enjoy using annex to backup and manage my data. I would love to see it used more often and am currently trying to get my boss to introduce it for storage of the digital backups of our documents. While preparing an encrypted test-setup I stumbled upon this. :)
+
+Thanks
+
+Jörn
+
+> [[fixed|done]] --[[Joey]]
diff --git a/doc/bugs/git_annex_enableremote_ignoring_encryption_changes/comment_1_e3924c02186d962e4dc1e1af8e968891._comment b/doc/bugs/git_annex_enableremote_ignoring_encryption_changes/comment_1_e3924c02186d962e4dc1e1af8e968891._comment
new file mode 100644
index 0000000000..9d0193d9b4
--- /dev/null
+++ b/doc/bugs/git_annex_enableremote_ignoring_encryption_changes/comment_1_e3924c02186d962e4dc1e1af8e968891._comment
@@ -0,0 +1,13 @@
+[[!comment format=mdwn
+ username="joey"
+ subject="""comment 1"""
+ date="2017-04-07T16:54:19Z"
+ content="""
+This does not happen when I do the same thing with a directory special
+remote.
+
+I do reproduce it with gcrypt. It must be missing a call to the code that
+handles keyid+=.
+
+Thanks for an excellent bug report!
+"""]]
diff --git a/doc/bugs/git_annex_enableremote_ignoring_encryption_changes/comment_2_5b36cd7f8ec4dd4f8787b60959512157._comment b/doc/bugs/git_annex_enableremote_ignoring_encryption_changes/comment_2_5b36cd7f8ec4dd4f8787b60959512157._comment
new file mode 100644
index 0000000000..22a6cdf601
--- /dev/null
+++ b/doc/bugs/git_annex_enableremote_ignoring_encryption_changes/comment_2_5b36cd7f8ec4dd4f8787b60959512157._comment
@@ -0,0 +1,26 @@
+[[!comment format=mdwn
+ username="joey"
+ subject="""comment 2"""
+ date="2017-04-07T16:58:42Z"
+ content="""
+Huh, so it seems that for gcrypt remotes, enableremote just doesn't
+call their setup function at all!
+
+Ah, it's because it sees the remote has an url, so it is not treated
+as a special remote, but as a regular git remote, and so the
+special remote encryption changes are ignored. (Since 6.20160527)
+
+So, enableremote needs to fail when it thinks it's enabling a regular git
+remote and has been passed some parameters which cannot apply to such a
+remote. Done.
+
+And, enableremote needs fixed to treat existing gcrypt remotes as special
+remotes. Done.
+
+Also, gcrypt special remotes didn't actually support being re-enabled
+either. I made that work. When an encryption key is added, that
+automatically makes it change the gcrypt-participants, too.
+
+I suppose enableremote could even be made to do the `GCRYPT_FULL_REPACK`
+and forced push, but that seems like too much for it to do!
+"""]]
diff --git a/doc/bugs/git_annex_forget_--drop-dead_can_lose_group_information.mdwn b/doc/bugs/git_annex_forget_--drop-dead_can_lose_group_information.mdwn
new file mode 100644
index 0000000000..5c76e9494f
--- /dev/null
+++ b/doc/bugs/git_annex_forget_--drop-dead_can_lose_group_information.mdwn
@@ -0,0 +1,156 @@
+### Please describe the problem.
+
+If multiple remotes edit group information and one of them does `git annex forget --force --drop-dead` some of those edits can be lost on sync.
+
+### What steps will reproduce the problem?
+
+Make a temporary directory and `cd` into it. Then run this script:
+
+[[!format sh """
+#!/bin/bash
+
+git annex version
+
+git init a
+cd a
+git annex init
+touch test
+git annex add test
+git annex sync
+cd ..
+git clone a b
+cd b
+git annex sync
+
+cd ../a
+git annex group here ga
+git annex sync
+cd ../b
+git annex group here gb
+git annex forget --force --drop-dead
+git annex sync
+
+cd ../a
+git annex sync
+cd ../b
+git annex sync
+
+cd ../a
+echo "A IS IN GROUP:"
+git annex group .
+cd ../b
+echo "B IS IN GROUP:"
+git annex group .
+
+"""]]
+
+
+### What version of git-annex are you using? On what operating system?
+
+6.20170101-1+deb9u2 on Debian Stretch but I also tested this occurs in version 6.20180719
+
+### Please provide any additional information below.
+
+Here's the output of the above script. The interesting part is the last two lines which show that remote 'b' is not in any group despite being added to group 'gb'.
+
+[[!format sh """
+git-annex version: 6.20170101.1
+build flags: Assistant Webapp Pairing Testsuite S3(multipartupload)(storageclasses) WebDAV Inotify DBus DesktopNotify ConcurrentOutput TorrentParser MagicMime Feeds Quvi
+key/value backends: SHA256E SHA256 SHA512E SHA512 SHA224E SHA224 SHA384E SHA384 SHA3_256E SHA3_256 SHA3_512E SHA3_512 SHA3_224E SHA3_224 SHA3_384E SHA3_384 SKEIN256E SKEIN256 SKEIN512E SKEIN512 SHA1E SHA1 MD5E MD5 WORM URL
+remote types: git gcrypt p2p S3 bup directory rsync web bittorrent webdav tahoe glacier ddar hook external
+Initialized empty Git repository in /home/matthew/test-git-annex/a/.git/
+init  ok
+(recording state in git...)
+add test ok
+(recording state in git...)
+commit  
+[master (root-commit) ffcf48d] git-annex in matthew@thorium:~/test-git-annex/a
+ 1 file changed, 1 insertion(+)
+ create mode 120000 test
+ok
+Cloning into 'b'...
+done.
+(merging origin/git-annex into git-annex...)
+commit  (recording state in git...)
+
+On branch master
+Your branch is up-to-date with 'origin/master'.
+nothing to commit, working tree clean
+ok
+pull origin 
+ok
+push origin 
+Counting objects: 3, done.
+Delta compression using up to 4 threads.
+Compressing objects: 100% (3/3), done.
+Writing objects: 100% (3/3), 409 bytes | 0 bytes/s, done.
+Total 3 (delta 0), reused 0 (delta 0)
+To /home/matthew/test-git-annex/a
+ * [new branch]      git-annex -> synced/git-annex
+ok
+group here ok
+(recording state in git...)
+commit  
+On branch master
+nothing to commit, working tree clean
+ok
+group here ok
+(recording state in git...)
+forget git-annex (recording state in git...)
+ok
+(recording state in git...)
+commit  
+On branch master
+Your branch is up-to-date with 'origin/master'.
+nothing to commit, working tree clean
+ok
+pull origin 
+remote: Counting objects: 3, done.
+remote: Compressing objects: 100% (3/3), done.
+remote: Total 3 (delta 0), reused 0 (delta 0)
+Unpacking objects: 100% (3/3), done.
+From /home/matthew/test-git-annex/a
+   e9a879d..000eb8e  git-annex  -> origin/git-annex
+ok
+(merging origin/git-annex into git-annex...)
+(recording state in git...)
+(recording state in git...)
+push origin 
+Counting objects: 11, done.
+Delta compression using up to 4 threads.
+Compressing objects: 100% (10/10), done.
+Writing objects: 100% (11/11), 1.03 KiB | 0 bytes/s, done.
+Total 11 (delta 3), reused 0 (delta 0)
+To /home/matthew/test-git-annex/a
+ + e9a879d...00986fa git-annex -> synced/git-annex (forced update)
+ok
+(merging synced/git-annex into git-annex...)
+(recording state in git...)
+commit  
+On branch master
+nothing to commit, working tree clean
+ok
+commit  
+On branch master
+Your branch is up-to-date with 'origin/master'.
+nothing to commit, working tree clean
+ok
+pull origin 
+remote: Counting objects: 3, done.
+remote: Compressing objects: 100% (3/3), done.
+remote: Total 3 (delta 0), reused 0 (delta 0)
+Unpacking objects: 100% (3/3), done.
+From /home/matthew/test-git-annex/a
+ + 000eb8e...965b6af git-annex  -> origin/git-annex  (forced update)
+ok
+(merging origin/git-annex into git-annex...)
+A IS IN GROUP:
+ga
+B IS IN GROUP:
+"""]]
+
+### Have you had any luck using git-annex before? (Sometimes we get tired of reading bug reports all day and a lil' positive end note does wonders)
+
+Yes, git-annex is slowly replacing all of my other sync and backup systems I've cobbled together over the years.
+
+> [[fixed|done]] --[[Joey]]
diff --git a/doc/bugs/git_annex_forget_--drop-dead_can_lose_group_information/comment_1_f4b78aeaf9163b374254c08057222661._comment b/doc/bugs/git_annex_forget_--drop-dead_can_lose_group_information/comment_1_f4b78aeaf9163b374254c08057222661._comment
new file mode 100644
index 0000000000..75bd9bd1ac
--- /dev/null
+++ b/doc/bugs/git_annex_forget_--drop-dead_can_lose_group_information/comment_1_f4b78aeaf9163b374254c08057222661._comment
@@ -0,0 +1,25 @@
+[[!comment format=mdwn
+ username="joey"
+ subject="""comment 1"""
+ date="2018-08-06T19:49:06Z"
+ content="""
+Great bug report!
+
+There is a general data loss problem in this scenario, it's not specific to
+group changes at all. Changes that were only present in a sibling git-annex
+branch are not being preserved when the repository updates its git-annex
+branch index file for a transition.
+
+The index file lacking those changes then gets committed with the sibling
+branches as parent(s). So the changes are effectively reverted.
+
+The root cause is that the handleTransitions uses getRaw
+to get the contents of files. That uses git cat-file git-annex:$file, which
+gets the version last committed to the git-annex branch,
+not the version from the git-annex branch index file. And handleTransitions is
+run after all sibling branches have been union merged in the index file
+but not committed yet. 
+
+The fix is to instead use git cat-file :$file, so it will get the version
+from the index. 
+"""]]
diff --git a/doc/bugs/git_annex_forget_--drop-dead_can_lose_group_information/comment_2_1468e26d2da85bcaf50715fbe3c38490._comment b/doc/bugs/git_annex_forget_--drop-dead_can_lose_group_information/comment_2_1468e26d2da85bcaf50715fbe3c38490._comment
new file mode 100644
index 0000000000..8fe3936bc1
--- /dev/null
+++ b/doc/bugs/git_annex_forget_--drop-dead_can_lose_group_information/comment_2_1468e26d2da85bcaf50715fbe3c38490._comment
@@ -0,0 +1,19 @@
+[[!comment format=mdwn
+ username="joey"
+ subject="""comment 2"""
+ date="2018-08-06T21:26:36Z"
+ content="""
+If you lost things from the git-annex branch due this bug, 
+you can find the commit that contained them by `git log git-annex`,
+and look for the commit before the "continuing transition" commit.
+
+It's then possible to get those changes applied back to the git-annex
+brannch; there should be no permanent data loss due to this bug.
+
+Eg, here the commit that contained the lost group change was
+261d1be6a2093f1e4059ed3030016c365f29413f. To get that back into the
+git-annex branch, I ran:
+
+	git update-ref git-annex 261d1be6a2093f1e4059ed3030016c365f29413f
+	git annex merge
+"""]]
diff --git a/doc/bugs/git_annex_fsck_--from_web_removes_all_urls_of_a_file.mdwn b/doc/bugs/git_annex_fsck_--from_web_removes_all_urls_of_a_file.mdwn
new file mode 100644
index 0000000000..0e6f5d256a
--- /dev/null
+++ b/doc/bugs/git_annex_fsck_--from_web_removes_all_urls_of_a_file.mdwn
@@ -0,0 +1,63 @@
+### Please describe the problem.
+If `git annex fsck --from web` encounters a URL that is no longer available,
+it will remove all URLs from the appropriate file's location log.
+Surprisingly, adding one of the URLs the file was associated to before brings back all of them.
+
+### What steps will reproduce the problem?
+
+    $ git annex whereis file.txt
+    whereis file.txt (3 copies) 
+  	    ...
+
+      The following untrusted locations may also have copies:
+  	    00000000-0000-0000-0000-000000000001 -- web
+
+      web: http://example.org/dead/file.txt
+      web: http://example.org/alive/file.txt
+    ok
+    $ git annex fsck --from web
+    fsck file.txt (checking http://example.org/dead/file.txt...) (fixing location log)
+      ** Based on the location log, file.txt
+      ** was expected to be present, but its content is missing.
+    failed
+    (recording state in git...)
+    git-annex: fsck: 1 failed
+    $ git annex whereis file.txt
+    whereis file.txt (3 copies) 
+  	    ...
+    ok
+    $ git annex addurl http://example.org/alive/file.txt
+    addurl file.txt ok
+    (recording state in git...)
+    $ git annex whereis file.txt
+    whereis file.txt (3 copies) 
+  	    ...
+
+      The following untrusted locations may also have copies:
+  	    00000000-0000-0000-0000-000000000001 -- web
+
+      web: http://example.org/dead/file.txt
+      web: http://example.org/alive/file.txt
+    ok
+    
+
+The only fix that seems to fix this is using `rmurl` on the dead one.
+
+### What version of git-annex are you using? On what operating system?
+    $ git annex version
+    git-annex version: 6.20171026
+    build flags: Assistant Webapp Pairing S3(multipartupload)(storageclasses) WebDAV Inotify DBus DesktopNotify ConcurrentOutput TorrentParser MagicMime Feeds Quvi
+    dependency versions: aws-0.16 bloomfilter-2.0.1.0 cryptonite-0.23 DAV-1.3.1 feed-0.3.12.0 ghc-8.0.2 http-client-0.5.7.0 persistent-sqlite-2.6.3 torrent-10000.1.1 uuid-1.3.13 yesod-1.4.5
+    key/value backends: SHA256E SHA256 SHA512E SHA512 SHA224E SHA224 SHA384E SHA384 SHA3_256E SHA3_256 SHA3_512E SHA3_512 SHA3_224E SHA3_224 SHA3_384E SHA3_384 SKEIN256E SKEIN256 SKEIN512E SKEIN512 SHA1E SHA1 MD5E MD5 WORM URL
+    remote types: git gcrypt p2p S3 bup directory rsync web bittorrent webdav tahoe glacier ddar hook external
+    local repository version: 5
+    supported repository versions: 3 5 6
+    upgrade supported from repository versions: 0 1 2 3 4 5
+    operating system: linux x86_64
+
+I'm running NixOS btw.
+
+### Have you had any luck using git-annex before? (Sometimes we get tired of reading bug reports all day and a lil' positive end note does wonders)
+I'm lovin' it!
+
+> [[fixed|done]] --[[Joey]] 
diff --git a/doc/bugs/git_annex_fsck_--from_web_removes_all_urls_of_a_file/comment_1_e1503374a87da976c831cd7fa0749e58._comment b/doc/bugs/git_annex_fsck_--from_web_removes_all_urls_of_a_file/comment_1_e1503374a87da976c831cd7fa0749e58._comment
new file mode 100644
index 0000000000..6c6957f60c
--- /dev/null
+++ b/doc/bugs/git_annex_fsck_--from_web_removes_all_urls_of_a_file/comment_1_e1503374a87da976c831cd7fa0749e58._comment
@@ -0,0 +1,10 @@
+[[!comment format=mdwn
+ username="joey"
+ subject="""comment 1"""
+ date="2017-11-07T19:58:14Z"
+ content="""
+I think that the real bug here is that, as long as one url still has the
+content, it's still present in the web, so should not be marked as not
+present. The rest of the operations in the web special remote work that
+way, trying urls until one succeeds, and so should `checkKey`.
+"""]]
diff --git a/doc/bugs/git_annex_fsck_--from_web_removes_all_urls_of_a_file/comment_2_d941d47c79ae9b72a936703aaba50ddd._comment b/doc/bugs/git_annex_fsck_--from_web_removes_all_urls_of_a_file/comment_2_d941d47c79ae9b72a936703aaba50ddd._comment
new file mode 100644
index 0000000000..ae6ac88e4c
--- /dev/null
+++ b/doc/bugs/git_annex_fsck_--from_web_removes_all_urls_of_a_file/comment_2_d941d47c79ae9b72a936703aaba50ddd._comment
@@ -0,0 +1,10 @@
+[[!comment format=mdwn
+ username="robert.schuetz@7942237bf71a2ae4f5d3cb047d167612b8c9e311"
+ nickname="robert.schuetz"
+ avatar="http://cdn.libravatar.org/avatar/89879460a9e84b9c736d982d9489d3d9"
+ subject="comment 2"
+ date="2017-11-07T20:22:04Z"
+ content="""
+If you want it that way, that's fine with me.
+But I think it would make more sense to let `fsck` call `rmurl` on all URLs that don't hold the content anymore.
+"""]]
diff --git a/doc/bugs/git_annex_fsck_--from_web_removes_all_urls_of_a_file/comment_3_e0a2822d931a3129dde3e280a15cf500._comment b/doc/bugs/git_annex_fsck_--from_web_removes_all_urls_of_a_file/comment_3_e0a2822d931a3129dde3e280a15cf500._comment
new file mode 100644
index 0000000000..f124fe7f52
--- /dev/null
+++ b/doc/bugs/git_annex_fsck_--from_web_removes_all_urls_of_a_file/comment_3_e0a2822d931a3129dde3e280a15cf500._comment
@@ -0,0 +1,10 @@
+[[!comment format=mdwn
+ username="joey"
+ subject="""comment 3"""
+ date="2017-11-07T21:02:59Z"
+ content="""
+There may be one url that works in one situation, and another that always
+works. Eg, a url on the work LAN, and a public, slower url. In such a
+situation, the user would not want fsck to remove an url that is
+temporarily not available.
+"""]]
diff --git a/doc/bugs/git_annex_help_options_not_piped_through_PAGER.mdwn b/doc/bugs/git_annex_help_options_not_piped_through_PAGER.mdwn
new file mode 100644
index 0000000000..3125014711
--- /dev/null
+++ b/doc/bugs/git_annex_help_options_not_piped_through_PAGER.mdwn
@@ -0,0 +1,35 @@
+### Please describe the problem.
+
+Long help output doesn't honor PAGER
+
+### What steps will reproduce the problem?
+
+* Open a small terminal
+* Run "git annex help options"
+* Note how it displays the full text.
+* Run "git help add"
+* Docs are viewed via the PAGER
+
+
+### What version of git-annex are you using? On what operating system?
+
+git-annex version: 5.20141125
+build flags: Assistant Webapp Webapp-secure Pairing Testsuite S3 WebDAV Inotify DBus DesktopNotify XMPP DNS Feeds Quvi TDFA CryptoHash
+key/value backends: SHA256E SHA1E SHA512E SHA224E SHA384E SKEIN256E SKEIN512E SHA256 SHA1 SHA512 SHA224 SHA384 SKEIN256 SKEIN512 WORM URL
+remote types: git gcrypt S3 bup directory rsync web webdav tahoe glacier ddar hook external
+
+
+### Please provide any additional information below.
+
+[[!format sh """
+# If you can, paste a complete transcript of the problem occurring here.
+# If the problem is with the git-annex assistant, paste in .git/annex/daemon.log
+
+
+# End of transcript or log.
+"""]]
+
+### Have you had any luck using git-annex before? (Sometimes we get tired of reading bug reports all day and a lil' positive end note does wonders)
+
+
+> [[done]] per my comment --[[Joey]]
diff --git a/doc/bugs/git_annex_help_options_not_piped_through_PAGER/comment_1_c581374b7581708da315e53338ef40c7._comment b/doc/bugs/git_annex_help_options_not_piped_through_PAGER/comment_1_c581374b7581708da315e53338ef40c7._comment
new file mode 100644
index 0000000000..c5eb45ffcf
--- /dev/null
+++ b/doc/bugs/git_annex_help_options_not_piped_through_PAGER/comment_1_c581374b7581708da315e53338ef40c7._comment
@@ -0,0 +1,15 @@
+[[!comment format=mdwn
+ username="joey"
+ subject="""comment 1"""
+ date="2017-08-28T17:31:03Z"
+ content="""
+"git annex help options" is not a thing in the current version of
+git-annex. Please upgrade to a current version before filing bug reports.
+
+git-annex outputs to stderr when it's run with invalid
+parameters. So `git annex` will output to stderr (`2>&1 | less` to page
+it). `git annex add --help` outputs to stdout since the user requested
+usage help and didn't do anything wrong. `git annex help add` displays the
+man page, which uses a pager. That is all consistent with git's
+behavior, and is fairly typical for unix programs.
+"""]]
diff --git a/doc/bugs/git_annex_import_fails_on_filenames_with_newlines_in_them.mdwn b/doc/bugs/git_annex_import_fails_on_filenames_with_newlines_in_them.mdwn
new file mode 100644
index 0000000000..eb2f6fa9cd
--- /dev/null
+++ b/doc/bugs/git_annex_import_fails_on_filenames_with_newlines_in_them.mdwn
@@ -0,0 +1,50 @@
+### Please describe the problem.
+I am importing a lot of old documents into my annex.  Some of these old files apparently have newlines in their filename.  A run of `git annex import` aborts when it encounters such a file; the file is moved to the annex, but it is left unstaged.
+
+### What steps will reproduce the problem?
+[[!format sh """
+bram@durian% mkdir annex
+bram@durian% cd annex
+bram@durian% git init
+Initialized empty Git repository in /home/bram/tmp/t/annex/.git/
+bram@durian% git annex init
+init  ok
+(Recording state in git...)
+bram@durian% echo foo > ../$'foo\nbar'
+bram@durian% ls -lb ..
+total 8
+drwxr-xr-x 3 bram bram 4096 Jul 26 18:20 annex/
+-rw-r--r-- 1 bram bram    4 Jul 26 18:20 foo\nbar
+bram@durian% git annex import ../foo$'\n'bar
+import foo
+bar git-annex: unknown response from git cat-file ("HEAD:./foo missing","HEAD:./foo\nbar")
+bram@durian% ls -lb
+total 4
+-r--r--r-- 2 bram bram 4 Jul 26 18:20 foo\nbar
+bram@durian% git status
+On branch master
+
+Initial commit
+
+Untracked files:
+  (use "git add ..." to include in what will be committed)
+
+	"foo\nbar"
+
+nothing added to commit but untracked files present (use "git add" to track)
+bram@durian% cat $'foo\nbar'
+foo
+"""]]
+
+
+### What version of git-annex are you using? On what operating system?
+    Debian unstable
+    git-annex version: 5.20140717
+    git version 2.0.1
+    Linux durian 3.14-1-amd64 #1 SMP Debian 3.14.9-1 (2014-06-30) x86_64 GNU/Linux
+
+[[!tag confirmed git-bug]]
+[[!title git limitations prevent using git-annex on filenames containing newlines]]
+
+> [[fixed|done]] with a workaround that is less efficient at handling such
+> filenames, but does work at least as far as I've tested it. --[[Joey]]
diff --git a/doc/bugs/git_annex_import_fails_on_filenames_with_newlines_in_them/comment_1_249b198e1141e05fe39f49bd7ad8870e._comment b/doc/bugs/git_annex_import_fails_on_filenames_with_newlines_in_them/comment_1_249b198e1141e05fe39f49bd7ad8870e._comment
new file mode 100644
index 0000000000..c4d78fa03f
--- /dev/null
+++ b/doc/bugs/git_annex_import_fails_on_filenames_with_newlines_in_them/comment_1_249b198e1141e05fe39f49bd7ad8870e._comment
@@ -0,0 +1,16 @@
+[[!comment format=mdwn
+ username="http://joeyh.name/"
+ ip="209.250.56.7"
+ subject="comment 1"
+ date="2014-08-12T18:24:57Z"
+ content="""
+Congrats on being the guy with newlines in his filenames.. someone had to do it!
+
+Similarly, `git annex add` on the file will fail with the same error and leave it where it is and not added.
+
+The problem here is that while git-annex is careful to use git commands with -z, so it gets \"foo\nbar\" with a literal newline from git ls-files, `git cat-file --batch` speaks a line-based protocol. And, it parses filenames like `git ref-parse` does -- and AFAICS, that does not provide a way to input something like \"foo\\nbar\" with an escaped newline. Normally this doesn't matter, since the whole line of input is taken to be a filename, so there's no need to escape anything, but of course it fails with newlines.
+
+IMHO, the solution to this is to make `git cat-file --batch` have a -z option that enables NUL-delimited input (and probably output). If you want to see this happen, take it to the git developers..
+
+(Should git annex import put files back if it fails to add them rather than leaving them sitting in the work tree?)
+"""]]
diff --git a/doc/bugs/git_annex_import_fails_on_filenames_with_newlines_in_them/comment_2_7320c7806da93d0862f8f768092ef073._comment b/doc/bugs/git_annex_import_fails_on_filenames_with_newlines_in_them/comment_2_7320c7806da93d0862f8f768092ef073._comment
new file mode 100644
index 0000000000..e1e6b9283b
--- /dev/null
+++ b/doc/bugs/git_annex_import_fails_on_filenames_with_newlines_in_them/comment_2_7320c7806da93d0862f8f768092ef073._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="https://www.google.com/accounts/o8/id?id=AItOawm_YXzEdPHzbSGVwtmTR7g1BqDtTnIBB5s"
+ nickname="Matthias"
+ subject="git and newlines"
+ date="2014-11-25T11:56:18Z"
+ content="""
+git does not support file names with newlines in them, and probably never will. Linus decided that quite early.
+"""]]
diff --git a/doc/bugs/git_annex_import_fails_on_filenames_with_newlines_in_them/comment_3_032d4a64e2f010fcf9359e4f77b6d784._comment b/doc/bugs/git_annex_import_fails_on_filenames_with_newlines_in_them/comment_3_032d4a64e2f010fcf9359e4f77b6d784._comment
new file mode 100644
index 0000000000..184a77af24
--- /dev/null
+++ b/doc/bugs/git_annex_import_fails_on_filenames_with_newlines_in_them/comment_3_032d4a64e2f010fcf9359e4f77b6d784._comment
@@ -0,0 +1,12 @@
+[[!comment format=mdwn
+ username="joey"
+ subject="""comment 3"""
+ date="2014-12-01T22:33:23Z"
+ content="""
+Mattias seems to be wrong. It may be that some of the shell script parts of
+git don't support it, or that parts of git has bugs. But as far as git's
+data representation is concerned, \n is a character like any other, and 
+eg `git add` and `git commit` deal with it just fine.
+
+(\0 is a different story..)
+"""]]
diff --git a/doc/bugs/git_annex_import_fails_on_filenames_with_newlines_in_them/comment_4_3f5e17f5d73892784640e468578819c4._comment b/doc/bugs/git_annex_import_fails_on_filenames_with_newlines_in_them/comment_4_3f5e17f5d73892784640e468578819c4._comment
new file mode 100644
index 0000000000..af72fcb88a
--- /dev/null
+++ b/doc/bugs/git_annex_import_fails_on_filenames_with_newlines_in_them/comment_4_3f5e17f5d73892784640e468578819c4._comment
@@ -0,0 +1,15 @@
+[[!comment format=mdwn
+ username="graboluk@f6de53961ab0f884e203f602f65eb5cdc0fb7513"
+ nickname="graboluk"
+ subject="very annoying for new users"
+ date="2015-09-25T17:06:45Z"
+ content="""
+Hi, I'd just spent better part of a day trying to figure out why git-annex keeps crashing. I've set up everything as in the instructional video for git-annex assistant, and proceeded to add my document directory with 3000 files (latex projects, etc). git-latex started to complain about unrecognized response from git. I tried many times again, to make long story short, turns out one of my files had newline in its name (I don't even know how it happened)  
+
+This was extremely annoying because the message which the assistant was giving me was always about some other file, the name of the offending file did not appear. So I've spent a day trying to figure out a minimal offending example (there were more obvious potential culprits - like the fact that I have many read-only dirs and so on...)
+
+Also, it's not a great first impression considering I'm trying to migrate from dropbox, which hasn't had any problems.
+
+Anyway, it's a great project. Is there a way to donate? (if it replaces dropbox for me for a month of my usual workflow I'd be very happy to donate!)
+
+"""]]
diff --git a/doc/bugs/git_annex_import_fails_on_filenames_with_newlines_in_them/comment_5_b495622a0e115484c5b6f18f21b8bf4f._comment b/doc/bugs/git_annex_import_fails_on_filenames_with_newlines_in_them/comment_5_b495622a0e115484c5b6f18f21b8bf4f._comment
new file mode 100644
index 0000000000..709a1a7fce
--- /dev/null
+++ b/doc/bugs/git_annex_import_fails_on_filenames_with_newlines_in_them/comment_5_b495622a0e115484c5b6f18f21b8bf4f._comment
@@ -0,0 +1,11 @@
+[[!comment format=mdwn
+ username="joey"
+ subject="""comment 5"""
+ date="2016-09-05T17:12:11Z"
+ content="""
+A couple others have reported dups of this bug.
+
+While this needs git changes to fix, it would be possible for git-annex to
+eg, warn and skip whenever a filename contains a newline. Probably better
+than crashing...
+"""]]
diff --git a/doc/bugs/git_annex_import_is_dangerous_if_you_have_unused_objects.mdwn b/doc/bugs/git_annex_import_is_dangerous_if_you_have_unused_objects.mdwn
new file mode 100644
index 0000000000..4c0cf29231
--- /dev/null
+++ b/doc/bugs/git_annex_import_is_dangerous_if_you_have_unused_objects.mdwn
@@ -0,0 +1,42 @@
+### Please describe the problem.
+`git annex import` is dangerous when you have unused git objects in your git store. You have the potential to lose your filenames and only be left with git objects containing the data.
+
+
+### What steps will reproduce the problem?
+```
+echo "foo" > /tmp/foo
+echo "bar" > /tmp/bar
+echo "baz" > /tmp/baz
+cd ~/annex
+cp /tmp/{foo,bar,baz} .
+git annex add ./{foo,bar,baz}
+# I decide I want to abort this particular commit, so I reset
+git reset --hard
+# At this point, git reset removed the symlinks from our index, but the objects containing the file content still exist in the git store.
+
+# A few days later I decide to import another backup of my data from this location: /tmp/myotherbackup/files/{foo,bar,baz}
+
+# This command removes foo, bar and baz from the source directory, but does not add symlinks for them. They were considered duplicates because we had git objects associated with their content (even though they are unused git objects).
+git annex import --deduplicate
+
+# This command considers foo, bar and baz duplicates and removes them from the source directory. We are left with unused objects in the git store, but no idea what the filenames were.
+git annex import --clean-duplicates
+```
+
+### What version of git-annex are you using? On what operating system?
+$ ga version
+git-annex version: 5.20140717
+build flags: Assistant Inotify DBus TDFA
+key/value backends: SHA256E SHA1E SHA512E SHA224E SHA384E SHA256 SHA1 SHA512 SHA224 SHA384 WORM URL
+remote types: git gcrypt bup directory rsync web glacier ddar hook external
+local repository version: 5
+supported repository version: 5
+upgrade supported from repository versions: 0 1 2 4
+
+OS: Fedora 25
+
+### Please provide any additional information below.
+
+
+### Have you had any luck using git-annex before? (Sometimes we get tired of reading bug reports all day and a lil' positive end note does wonders)
+I love git-annex! I used it to manage my Pictures.
diff --git a/doc/bugs/git_annex_import_is_dangerous_if_you_have_unused_objects/comment_1_9f9d5c18c27dfd33a5bf2c593ee65b58._comment b/doc/bugs/git_annex_import_is_dangerous_if_you_have_unused_objects/comment_1_9f9d5c18c27dfd33a5bf2c593ee65b58._comment
new file mode 100644
index 0000000000..aac409e983
--- /dev/null
+++ b/doc/bugs/git_annex_import_is_dangerous_if_you_have_unused_objects/comment_1_9f9d5c18c27dfd33a5bf2c593ee65b58._comment
@@ -0,0 +1,31 @@
+[[!comment format=mdwn
+ username="joey"
+ subject="""comment 1"""
+ date="2017-03-02T21:20:02Z"
+ content="""
+A simpler way to get to the same end result, without using git-annex
+import:
+
+	mv ~/foo .
+	git annex add foo
+	git reset --hard
+
+Now "foo" is only present in the git-annex object store, and we don't
+know what its filename(s) were. 
+
+A way to get to the same end result, without using git-annex:
+
+	mv ~/foo .
+	git add foo
+	git reset --hard
+
+Now "foo" is only present in the git object store, and we don't
+know what its filename(s) were.
+
+So, it's not `git-annex import`, or `git-annex add`, or `git add`
+that is dangerous here. It's `git reset --hard`.
+
+git will happily lose lots of data until you commit it. Once it's
+committed, it's safe. That's the rule of thumb. Nothing much that git-annex
+can do about that.
+"""]]
diff --git a/doc/bugs/git_annex_import_is_dangerous_if_you_have_unused_objects/comment_2_f64891ff6fe0cd0be6a356ab8fe42817._comment b/doc/bugs/git_annex_import_is_dangerous_if_you_have_unused_objects/comment_2_f64891ff6fe0cd0be6a356ab8fe42817._comment
new file mode 100644
index 0000000000..0b459529bc
--- /dev/null
+++ b/doc/bugs/git_annex_import_is_dangerous_if_you_have_unused_objects/comment_2_f64891ff6fe0cd0be6a356ab8fe42817._comment
@@ -0,0 +1,11 @@
+[[!comment format=mdwn
+ username="joey"
+ subject="""comment 2"""
+ date="2017-03-02T21:26:57Z"
+ content="""
+One way to fix this would be to make `git reset --hard` first make a commit
+of the current state of the index, and store that commit in the reflog. 
+
+Of course, that's quite similar to `git stash` and probably most of us have
+just gotten in the habit of running `git stash` instead.
+"""]]
diff --git a/doc/bugs/git_annex_import_is_dangerous_if_you_have_unused_objects/comment_3_d980c1e2d788324049475e468a0d6fab._comment b/doc/bugs/git_annex_import_is_dangerous_if_you_have_unused_objects/comment_3_d980c1e2d788324049475e468a0d6fab._comment
new file mode 100644
index 0000000000..e86ff13de2
--- /dev/null
+++ b/doc/bugs/git_annex_import_is_dangerous_if_you_have_unused_objects/comment_3_d980c1e2d788324049475e468a0d6fab._comment
@@ -0,0 +1,23 @@
+[[!comment format=mdwn
+ username="mbroadhead"
+ avatar="http://cdn.libravatar.org/avatar/f3d801c0c943caab1152c4ebe8c99d51"
+ subject="comment 3"
+ date="2017-03-02T22:48:08Z"
+ content="""
+Thanks for the insight.
+
+The `git stash` solution works assuming you are either:
+
+ a. Going to keep it in your stash forever
+ b. You are going to commit your stash eventually
+
+I think there are situations where I want to completely abort a commit and not have to worry about it biting me down the road.
+
+IMO from a end user perspective I think the best solution would be to have data only count as duplicate if it has a reachable file in your annex for options `--deduplicate`, `--clean-duplicates` and `--skip-duplicates` of `git annex import`.
+
+What would be the downside to this?
+
+Worst case scenario this re-wires up some symlinks to once dangling git objects. They still aren't duplicates, there will only be one symlink per formerly dangling git object. This seems better than data loss.
+
+Thoughts?
+"""]]
diff --git a/doc/bugs/git_annex_import_is_dangerous_if_you_have_unused_objects/comment_4_09b860a14bb8bd49f2067e8ddb0aacfc._comment b/doc/bugs/git_annex_import_is_dangerous_if_you_have_unused_objects/comment_4_09b860a14bb8bd49f2067e8ddb0aacfc._comment
new file mode 100644
index 0000000000..877c0ff182
--- /dev/null
+++ b/doc/bugs/git_annex_import_is_dangerous_if_you_have_unused_objects/comment_4_09b860a14bb8bd49f2067e8ddb0aacfc._comment
@@ -0,0 +1,39 @@
+[[!comment format=mdwn
+ username="mbroadhead"
+ avatar="http://cdn.libravatar.org/avatar/f3d801c0c943caab1152c4ebe8c99d51"
+ subject="comment 4"
+ date="2017-03-03T16:59:52Z"
+ content="""
+> I think there are situations where I want to completely abort a commit and
+> not have to worry about it biting me down the road.
+
+If you don't want to have to worry about a `git reset --hard` biting you down
+the road the way that it works now, just make sure you clean up after yourself.
+Example:
+
+```
+cd ~/annex
+cp /tmp/foo .
+git annex add foo
+
+# Oh, I decided I don't want \"foo\" in my annex right now, so I do a reset.
+# This will leave the data associated with \"foo\" in a git object in the git
+# store.  When running 'git annex import' sometime in the future, this will
+# make any files that contain the same data as \"foo\" to be considered
+# duplicate.  This will cause \"foo\" to be considered a duplicate by git annex
+# import, which we don't want in this scenario.
+git reset --hard
+
+# In order to avoid \"foo\" being a duplicate, find the dangling git object:
+git annex unused
+
+# And drop it:
+git annex dropunused N
+
+# Now \"foo\" won't be marked as a duplicate if you run any of the following
+# commands:
+git annex import /tmp/backup/foo --skip-duplicates
+git annex import /tmp/backup/foo --clean-duplicates
+git annex import /tmp/backup/foo --deduplicate
+```
+"""]]
diff --git a/doc/bugs/git_annex_import_is_dangerous_if_you_have_unused_objects/comment_5_1e737b740bc7d95f3329e3481d55fd35._comment b/doc/bugs/git_annex_import_is_dangerous_if_you_have_unused_objects/comment_5_1e737b740bc7d95f3329e3481d55fd35._comment
new file mode 100644
index 0000000000..89b088f145
--- /dev/null
+++ b/doc/bugs/git_annex_import_is_dangerous_if_you_have_unused_objects/comment_5_1e737b740bc7d95f3329e3481d55fd35._comment
@@ -0,0 +1,20 @@
+[[!comment format=mdwn
+ username="joey"
+ subject="""comment 5"""
+ date="2017-03-06T16:25:53Z"
+ content="""
+The difficulty with checking if the content to be imported is referred to
+somewhere in the working tree is that there's no inexpensive way to
+determine that. It would have to run `git log -n1 -S$KEY` for each file.
+That can take quite a long time in repositories with a lot of history.
+I clocked it at 12 seconds per file on an SSD; will be quite a
+lot slower on a disc.
+
+I suppose that check could be added with a --fast to skip the check.
+
+PS, mbroadhead's is a good approach. Note though that the dropunused content
+will be considered a duplicate by import since git-annex
+version 6.20170214. Still, --deduplicate and --clean-duplicates won't
+delete the files from the import location in this case, since there
+are no copies of the content in the annex.
+"""]]
diff --git a/doc/bugs/git_annex_info_fails_on_NFS__58___waitToSetLock__58___resource_exhausted___40__No_locks_available__41__.mdwn b/doc/bugs/git_annex_info_fails_on_NFS__58___waitToSetLock__58___resource_exhausted___40__No_locks_available__41__.mdwn
new file mode 100644
index 0000000000..5bc480bb24
--- /dev/null
+++ b/doc/bugs/git_annex_info_fails_on_NFS__58___waitToSetLock__58___resource_exhausted___40__No_locks_available__41__.mdwn
@@ -0,0 +1,46 @@
+### Please describe the problem.
+
+git annex info  crashes in some directories of the NFS mounted partition but not in the others
+
+[[!format sh """
+# If you can, paste a complete transcript of the problem occurring here.
+# If the problem is with the git-annex assistant, paste in .git/annex/daemon.log
+
+yoh@rolando:/inbox/BIDS/Meng/Meng/1005-faceperception$ git annex info
+
+git-annex: waitToSetLock: resource exhausted (No locks available)
+failed
+git-annex: info: 1 failed
+yoh@rolando:/inbox/BIDS/Meng/Meng/1005-faceperception$ cd -
+/inbox/BIDS/Gobbini/Matteo/1002-faceangles
+
+yoh@rolando:/inbox/BIDS/Gobbini/Matteo/1002-faceangles$ git annex info
+repository mode: indirect
+trusted repositories: 0
+semitrusted repositories: 5
+	00000000-0000-0000-0000-000000000001 -- web
+ 	00000000-0000-0000-0000-000000000002 -- bittorrent
+ 	8ffcc6d4-1288-45cb-8e5b-0fa13a22c8c5 -- mvdoc@falkor:/srv/dbic.datalad.org/www/Gobbini/Matteo/1002-faceangles
+ 	b7872847-80ac-47d6-a712-fa695b24e85e -- yoh@rolando:/inbox/BIDS/Gobbini/Matteo/1002-faceangles [here]
+ 	db79e407-569d-43b4-8c39-c50f0eeb1fdd -- mvdoc@smaug:/mnt/btrfs/dbic/inbox/face-angles/Gobbini/Matteo
+untrusted repositories: 0
+transfers in progress: none
+available local disk space: 30.19 terabytes (+1 megabyte reserved)
+local annex keys: 40
+local annex size: 5.21 gigabytes
+annexed files in working tree: 40
+size of annexed files in working tree: 5.21 gigabytes
+bloom filter size: 32 mebibytes (0% full)
+backend usage: 
+	SHA256E: 40
+
+yoh@rolando:/inbox/BIDS/Gobbini/Matteo/1002-faceangles$ git annex version
+git-annex version: 6.20170101+gitg93d69b1-1~ndall+1
+
+"""]]
+
+
+so this one was a run from within singularity neurodebian environment on a centos box. but the same behaviour is if ran using 6.20161231-gc8eeb17da  standalone build ran under CentOS natively.
+
+[[!meta author=yoh]]
+
diff --git a/doc/bugs/git_annex_info_fails_on_NFS__58___waitToSetLock__58___resource_exhausted___40__No_locks_available__41__/comment_1_6c595bec40f6bf470e9170c8ae4bb573._comment b/doc/bugs/git_annex_info_fails_on_NFS__58___waitToSetLock__58___resource_exhausted___40__No_locks_available__41__/comment_1_6c595bec40f6bf470e9170c8ae4bb573._comment
new file mode 100644
index 0000000000..3835086719
--- /dev/null
+++ b/doc/bugs/git_annex_info_fails_on_NFS__58___waitToSetLock__58___resource_exhausted___40__No_locks_available__41__/comment_1_6c595bec40f6bf470e9170c8ae4bb573._comment
@@ -0,0 +1,14 @@
+[[!comment format=mdwn
+ username="joey"
+ subject="""comment 1"""
+ date="2017-02-07T17:48:05Z"
+ content="""
+These seem to be two separate git repositories and it's only crashing in
+one of them. 
+
+Perhaps annex.pidlock is set in the repository where it works, and not in
+the other repository?
+
+git-annex init probes to see if it can set a lock, and if not, enables
+annex.pidlock. Since 5.20151116.
+"""]]
diff --git a/doc/bugs/git_annex_info_fails_on_NFS__58___waitToSetLock__58___resource_exhausted___40__No_locks_available__41__/comment_2_4c2e143b8b3283734963e0a6e68755ac._comment b/doc/bugs/git_annex_info_fails_on_NFS__58___waitToSetLock__58___resource_exhausted___40__No_locks_available__41__/comment_2_4c2e143b8b3283734963e0a6e68755ac._comment
new file mode 100644
index 0000000000..9fd9af6500
--- /dev/null
+++ b/doc/bugs/git_annex_info_fails_on_NFS__58___waitToSetLock__58___resource_exhausted___40__No_locks_available__41__/comment_2_4c2e143b8b3283734963e0a6e68755ac._comment
@@ -0,0 +1,10 @@
+[[!comment format=mdwn
+ username="yarikoptic"
+ avatar="http://cdn.libravatar.org/avatar/f11e9c84cb18d26a1748c33b48c924b4"
+ subject="comment 2"
+ date="2017-02-10T01:38:33Z"
+ content="""
+indeed that was the case.... not clear though how that one ended up without pidlock not being set since all creation was uniform as far as I remember
+
+I wonder if annex could potentially detect for such a case and reissue check/assignment of the pidlock?
+"""]]
diff --git a/doc/bugs/git_annex_info_fails_on_NFS__58___waitToSetLock__58___resource_exhausted___40__No_locks_available__41__/comment_3_dac1401bd3661397a9388a5b6ce673a5._comment b/doc/bugs/git_annex_info_fails_on_NFS__58___waitToSetLock__58___resource_exhausted___40__No_locks_available__41__/comment_3_dac1401bd3661397a9388a5b6ce673a5._comment
new file mode 100644
index 0000000000..95f02857c6
--- /dev/null
+++ b/doc/bugs/git_annex_info_fails_on_NFS__58___waitToSetLock__58___resource_exhausted___40__No_locks_available__41__/comment_3_dac1401bd3661397a9388a5b6ce673a5._comment
@@ -0,0 +1,14 @@
+[[!comment format=mdwn
+ username="joey"
+ subject="""comment 3"""
+ date="2017-02-10T17:56:32Z"
+ content="""
+Seems unlikely that the init-time check would not have worked in this
+repository. Probably the repository was either initialized by an old
+version of git-annex, or perhaps init was run on the NFS server,
+or perhaps the repository was initted elsewhere and moved into place.
+
+I don't want to complicate locking with fallbacks involving changing
+configuration. ENOLCK can be raised for other reasons, so it's not
+necessarily an indication of a NFS system anyway.
+"""]]
diff --git a/doc/bugs/git_annex_should_check_that_stdout_isatty_when_reporting_progress.mdwn b/doc/bugs/git_annex_should_check_that_stdout_isatty_when_reporting_progress.mdwn
new file mode 100644
index 0000000000..87ffd321b3
--- /dev/null
+++ b/doc/bugs/git_annex_should_check_that_stdout_isatty_when_reporting_progress.mdwn
@@ -0,0 +1,33 @@
+### Please describe the problem.
+
+git annex reports progress the same when when stdout is a terminal or not.
+This is fine when stdout is actually a terminal, but produce lots of annoying characters when stdout is just a regular file.
+This is problematic mostly when running git-annex with cron.
+
+### What steps will reproduce the problem?
+
+    git annex copy --to myrepo > log
+
+then look at the content of `log`.
+
+### What version of git-annex are you using? On what operating system?
+
+git annex 5.20151019
+
+Mac OS X 10.10.5
+
+Ubuntu 14.04
+
+### Please provide any additional information below.
+
+[[!format sh """
+# If you can, paste a complete transcript of the problem occurring here.
+# If the problem is with the git-annex assistant, paste in .git/annex/daemon.log
+
+
+# End of transcript or log.
+"""]]
+
+### Have you had any luck using git-annex before? (Sometimes we get tired of reading bug reports all day and a lil' positive end note does wonders)
+
+
diff --git a/doc/bugs/git_annex_should_check_that_stdout_isatty_when_reporting_progress/comment_1_6c7f745ee77e9b7502cb13d3babf211b._comment b/doc/bugs/git_annex_should_check_that_stdout_isatty_when_reporting_progress/comment_1_6c7f745ee77e9b7502cb13d3babf211b._comment
new file mode 100644
index 0000000000..94307fec01
--- /dev/null
+++ b/doc/bugs/git_annex_should_check_that_stdout_isatty_when_reporting_progress/comment_1_6c7f745ee77e9b7502cb13d3babf211b._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="joey"
+ subject="""comment 1"""
+ date="2015-11-30T19:01:16Z"
+ content="""
+If you have a recent version of git-annex that's built with the
+ConcurrentOutput flag, pass -J1 and it will do what you want. 
+"""]]
diff --git a/doc/bugs/git_annex_status_fails_with_submodule_in_direct_mode.mdwn b/doc/bugs/git_annex_status_fails_with_submodule_in_direct_mode.mdwn
new file mode 100644
index 0000000000..eafd22e0b9
--- /dev/null
+++ b/doc/bugs/git_annex_status_fails_with_submodule_in_direct_mode.mdwn
@@ -0,0 +1,57 @@
+### Please describe the problem.
+If a submodule is in direct mode, `git annex status` fails.
+Apparently the recursive inspection of submodules is done by an internal call to `git status`, which then misses a working tree in the submodule.
+### What steps will reproduce the problem?
+[[!format sh """
+#!/bin/bash
+
+set -ex
+d=$(mktemp -d)
+echo "directory: $d"
+cd $d
+git init
+git annex init
+echo whatever > file
+git annex add file
+git commit -m "file added"
+mkdir sub
+cd sub
+git init
+git annex init
+echo something > subfile
+git annex add subfile
+git commit -m "subfile added."
+cd ..
+git submodule add ./sub sub
+git commit -m "submodule added"
+cd sub
+git annex direct
+cd ..
+git annex status
+"""]]
+### What version of git-annex are you using? On what operating system?
+[[!format sh """
+% git annex version
+git-annex version: 6.20170209+gitg16be7b5cc-1~ndall+1
+build flags: Assistant Webapp Pairing Testsuite S3(multipartupload)(storageclasses) WebDAV Inotify DBus DesktopNotify ConcurrentOutput TorrentParser MagicMime Feeds Quvi
+key/value backends: SHA256E SHA256 SHA512E SHA512 SHA224E SHA224 SHA384E SHA384 SHA3_256E SHA3_256 SHA3_512E SHA3_512 SHA3_224E SHA3_224 SHA3_384E SHA3_384 SKEIN256E SKEIN256 SKEIN512E SKEIN512 SHA1E SHA1 MD5E MD5 WORM URL
+remote types: git gcrypt p2p S3 bup directory rsync web bittorrent webdav tahoe glacier ddar hook external
+local repository version: unknown
+supported repository versions: 3 5 6
+upgrade supported from repository versions: 0 1 2 3 4 5
+operating system: linux x86_64
+"""]]
+
+### Please provide any additional information below.
+
+[[!format sh """
+# If you can, paste a complete transcript of the problem occurring here.
+# If the problem is with the git-annex assistant, paste in .git/annex/daemon.log
+
+
+# End of transcript or log.
+"""]]
+
+### Have you had any luck using git-annex before? (Sometimes we get tired of reading bug reports all day and a lil' positive end note does wonders)
+
+Yes! I think it's a great piece of software!
diff --git a/doc/bugs/git_annex_status_fails_with_submodule_in_direct_mode/comment_1_0258db936120767e8e41cd926cb5bfd5._comment b/doc/bugs/git_annex_status_fails_with_submodule_in_direct_mode/comment_1_0258db936120767e8e41cd926cb5bfd5._comment
new file mode 100644
index 0000000000..63cceedd9c
--- /dev/null
+++ b/doc/bugs/git_annex_status_fails_with_submodule_in_direct_mode/comment_1_0258db936120767e8e41cd926cb5bfd5._comment
@@ -0,0 +1,19 @@
+[[!comment format=mdwn
+ username="joey"
+ subject="""comment 1"""
+ date="2017-02-20T18:30:54Z"
+ content="""
+Error is:
+
+	fatal: This operation must be run in a work tree
+	fatal: 'git status --porcelain' failed in submodule sub
+
+`git status` fails the same way, and it seems that the only way
+to make it work would be to pass --ignore-submodules to it.
+
+But I suppose then, it would need to replicate git status's submodule
+traversal.
+
+I'm not too keen on adding complicated stuff involving submodules to direct
+mode. My goal with direct mode is to eliminate the need for it.
+"""]]
diff --git a/doc/bugs/git_annex_status_fails_with_submodule_in_direct_mode/comment_2_0d16964586347e6447d9193508f0d7f8._comment b/doc/bugs/git_annex_status_fails_with_submodule_in_direct_mode/comment_2_0d16964586347e6447d9193508f0d7f8._comment
new file mode 100644
index 0000000000..a3e7198a4e
--- /dev/null
+++ b/doc/bugs/git_annex_status_fails_with_submodule_in_direct_mode/comment_2_0d16964586347e6447d9193508f0d7f8._comment
@@ -0,0 +1,13 @@
+[[!comment format=mdwn
+ username="benjamin.poldrack@d09ccff6d42dd20277610b59867cf7462927b8e3"
+ nickname="benjamin.poldrack"
+ avatar="http://cdn.libravatar.org/avatar/5c1a901caa7c2cfeeb7e17e786c5230d"
+ subject="comment 2"
+ date="2017-02-20T19:06:58Z"
+ content="""
+Yes, that's what I assumed. Therefore this bug was linked in the datalad issue regarding --ignore-submodules.
+
+May be we are the only ones, who are actually effected, since we heavily use submodules. If it's too much of an effort, I will find a way around it, I guess.
+Either way: Thank you for having a look at it.
+[[ben]]
+"""]]
diff --git a/doc/bugs/git_annex_status_fails_with_submodule_in_direct_mode/comment_3_8cce6890bf13af8bfd1ed19ccd904162._comment b/doc/bugs/git_annex_status_fails_with_submodule_in_direct_mode/comment_3_8cce6890bf13af8bfd1ed19ccd904162._comment
new file mode 100644
index 0000000000..e15c59a51f
--- /dev/null
+++ b/doc/bugs/git_annex_status_fails_with_submodule_in_direct_mode/comment_3_8cce6890bf13af8bfd1ed19ccd904162._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="joey"
+ subject="""comment 3"""
+ date="2017-02-20T20:13:06Z"
+ content="""
+Would it suffice for datalad to have git-annex status --ignore-submodules
+pass the option on to git status?
+"""]]
diff --git a/doc/bugs/git_annex_status_fails_with_submodule_in_direct_mode/comment_4_9041ccdacce602c6c09c3fc09e95ce8d._comment b/doc/bugs/git_annex_status_fails_with_submodule_in_direct_mode/comment_4_9041ccdacce602c6c09c3fc09e95ce8d._comment
new file mode 100644
index 0000000000..c5bd51a1ea
--- /dev/null
+++ b/doc/bugs/git_annex_status_fails_with_submodule_in_direct_mode/comment_4_9041ccdacce602c6c09c3fc09e95ce8d._comment
@@ -0,0 +1,11 @@
+[[!comment format=mdwn
+ username="benjamin.poldrack@d09ccff6d42dd20277610b59867cf7462927b8e3"
+ nickname="benjamin.poldrack"
+ avatar="http://cdn.libravatar.org/avatar/5c1a901caa7c2cfeeb7e17e786c5230d"
+ subject="comment 4"
+ date="2017-02-21T06:20:34Z"
+ content="""
+Yes, it saves half the work and eases to deal with this one.
+
+[[ben]]
+"""]]
diff --git a/doc/bugs/git_annex_status_fails_with_submodule_in_direct_mode/comment_5_e7c09850009e961354e26f169ccdc6e3._comment b/doc/bugs/git_annex_status_fails_with_submodule_in_direct_mode/comment_5_e7c09850009e961354e26f169ccdc6e3._comment
new file mode 100644
index 0000000000..a2af5ca080
--- /dev/null
+++ b/doc/bugs/git_annex_status_fails_with_submodule_in_direct_mode/comment_5_e7c09850009e961354e26f169ccdc6e3._comment
@@ -0,0 +1,12 @@
+[[!comment format=mdwn
+ username="benjamin.poldrack@d09ccff6d42dd20277610b59867cf7462927b8e3"
+ nickname="benjamin.poldrack"
+ avatar="http://cdn.libravatar.org/avatar/5c1a901caa7c2cfeeb7e17e786c5230d"
+ subject="comment 5"
+ date="2017-02-22T16:48:04Z"
+ content="""
+Just noticed another thing: If that failure happens, git-annex exits zero. Therefore it's hard to detect for us. We have just an empty json as return value as if the repo was clean.
+It would be nice to have either a non-zero exit or something within the json to indicate that the command failed.
+
+[[ben]]
+"""]]
diff --git a/doc/bugs/git_annex_status_fails_with_submodule_in_direct_mode/comment_6_be8774764b7892dcbf8bfa8f1fc00ab8._comment b/doc/bugs/git_annex_status_fails_with_submodule_in_direct_mode/comment_6_be8774764b7892dcbf8bfa8f1fc00ab8._comment
new file mode 100644
index 0000000000..8d86421bcf
--- /dev/null
+++ b/doc/bugs/git_annex_status_fails_with_submodule_in_direct_mode/comment_6_be8774764b7892dcbf8bfa8f1fc00ab8._comment
@@ -0,0 +1,13 @@
+[[!comment format=mdwn
+ username="joey"
+ subject="""comment 6"""
+ date="2017-03-02T18:05:39Z"
+ content="""
+`git annex status --ignore-submodules=when` has been supported for a week
+or so.
+
+I've fixed the nonzero exit status propigation.
+
+Leaving bug open for the general problem but don't anticipate working on
+it.
+"""]]
diff --git a/doc/bugs/git_annex_status_fails_with_submodule_in_direct_mode/comment_7_4e4d2ae552e2394197c2688b0dd04f48._comment b/doc/bugs/git_annex_status_fails_with_submodule_in_direct_mode/comment_7_4e4d2ae552e2394197c2688b0dd04f48._comment
new file mode 100644
index 0000000000..85573f3a26
--- /dev/null
+++ b/doc/bugs/git_annex_status_fails_with_submodule_in_direct_mode/comment_7_4e4d2ae552e2394197c2688b0dd04f48._comment
@@ -0,0 +1,11 @@
+[[!comment format=mdwn
+ username="benjamin.poldrack@d09ccff6d42dd20277610b59867cf7462927b8e3"
+ nickname="benjamin.poldrack"
+ avatar="http://cdn.libravatar.org/avatar/5c1a901caa7c2cfeeb7e17e786c5230d"
+ subject="comment 7"
+ date="2017-03-06T09:03:56Z"
+ content="""
+Thank you!
+
+[[ben]]
+"""]]
diff --git a/doc/bugs/git_annex_still_deleting_content_when_merging.mdwn b/doc/bugs/git_annex_still_deleting_content_when_merging.mdwn
new file mode 100644
index 0000000000..872b699c0d
--- /dev/null
+++ b/doc/bugs/git_annex_still_deleting_content_when_merging.mdwn
@@ -0,0 +1,97 @@
+**Premise:**
+
+I've had this problem before 5.20140709-gc75193e and when the [ugly bug](http://git-annex.branchable.com/devblog/day_193-194__ugly_bug/) was discovered and fixed, I updated and thought it was gone.
+Yet, I'm still experiencing a content removal bug, so it might be something different altogether.
+I should also add that I don't know much about git log history and git-annex inner workings. I might need some help with finding the necessary information for debugging.
+
+**Bug description:**
+
+Two local computers, running Ubuntu-14.04 and Fedora-20 respectively, are paired through the assistant and functional with git annex 5.20140709. In direct mode. Most of the time, everything seems to work fine. Sometimes, *Computer A* will modify one file in its repository while *B* is offline. When *B* comes back online both repositories will lose their copy of the modified file (I'm not sure whether this also happens when they're both online when the change happens; it doesn't look like it anyway). The rest of the data in the repository is intact. This does not happen every time and I haven't been able to reproduce it consistently. Since 5.20140709 came out, it happened 2 or 3 times.
+
+Looking at 'git log --stat' in *B*:
+
+    commit d9eb9e94a39
+    Author: COMPUTER B
+    Date:   Thu Jul 17 18:17:16 2014 +0200
+      shared.skg | 1 -
+      1 file changed, 1 deletion(-)
+
+    commit 6fa27f08492
+    Author: COMPUTER A
+    Date:   Thu Jul 17 18:17:14 2014 +0200
+      shared.skg | 1 +
+      1 file changed, 1 insertion(+)
+
+    commit d25cc793739
+    Author: COMPUTER A
+    Date:   Thu Jul 17 18:17:12 2014 +0200
+      shared.skg | 1 -
+      1 file changed, 1 deletion(-)
+
+A change is apparently deleting and rewriting the whole file (maybe that's normal because it's a binary (encrypted) file?). The third most recent deletion is probably the one that shouldn't happen.
+
+Reverting "d9eb9e94a39" will only recover a placeholder, not really the original file. I can get the content back by going through *git annex unused* with *git log -S'key'*, then going to indirect mode, and then using *addunused*. This can only be done in the repository that changed the file in the first place (repo A), as the other repository only has the older version of the file in its unused list. This probably means that the change did not propagate before the file was deleted.
+
+**daemon.log transcript**
+
+[[!format sh """
+
+[2014-07-17 18:13:17 CEST] main: starting assistant version 5.20140709-gc75193e
+[2014-07-17 18:13:17 CEST] Cronner: You should enable consistency checking to protect your data. 
+[2014-07-17 18:13:55 CEST] TransferScanner: Syncing with asus.local_sincro 
+(scanning...) [2014-07-17 18:13:55 CEST] Watcher: Performing startup scan
+p11-kit: couldn't load module: /usr/lib/x86_64-linux-gnu/pkcs11/p11-kit-trust.so: /usr/lib/x86_64-linux-gnu/pkcs11/p11-kit-trust.so: cannot open shared object file: No such file or directory
+p11-kit: couldn't load module: /usr/lib/x86_64-linux-gnu/pkcs11/gnome-keyring-pkcs11.so: /usr/lib/x86_64-linux-gnu/pkcs11/gnome-keyring-pkcs11.so: cannot open shared object file: No such file or directory
+(started...) p11-kit: couldn't load module: /usr/lib/x86_64-linux-gnu/pkcs11/p11-kit-trust.so: /usr/lib/x86_64-linux-gnu/pkcs11/p11-kit-trust.so: cannot open shared object file: No such file or directory
+p11-kit: couldn't load module: /usr/lib/x86_64-linux-gnu/pkcs11/gnome-keyring-pkcs11.so: /usr/lib/x86_64-linux-gnu/pkcs11/gnome-keyring-pkcs11.so: cannot open shared object file: No such file or directory
+gpg: Signature made gio 17 lug 2014 18:02:09 CEST using DSA key ID 89C809CB
+gpg: /tmp/git-annex-gpg.tmp.0/trustdb.gpg: trustdb created
+gpg: Good signature from "git-annex distribution signing key (for Joey Hess) "
+gpg: WARNING: This key is not certified with a trusted signature!
+gpg:          There is no indication that the signature belongs to the owner.
+Primary key fingerprint: 4005 5C6A FD2D 526B 2961  E78F 5EE1 DBA7 89C8 09CB
+[2014-07-17 18:13:58 CEST] Upgrader: An upgrade of git-annex is available.  (version 5.20140716)
+Everything up-to-date
+git-annex-shell: key is already present in annex
+rsync: connection unexpectedly closed (0 bytes received so far) [sender]
+rsync error: error in rsync protocol data stream (code 12) at io.c(226) [sender=3.1.0]
+
+########## SYNCING STARTS HERE ###########
+
+[2014-07-17 18:17:12 CEST] RemoteControl: Syncing with asus.local_sincro 
+From ssh://git-annex-asus.local-USER-A_22_sincro/~/sincro
+   51815ea..d25cc79  annex/direct/master -> asus.local_sincro/annex/direct/master
+   0391880..49efa5f  git-annex  -> asus.local_sincro/git-annex
+   51815ea..d25cc79  master     -> asus.local_sincro/master
+   51815ea..d25cc79  synced/master -> asus.local_sincro/synced/master
+[2014-07-17 18:17:15 CEST] RemoteControl: Syncing with asus.local_sincro 
+Automatic merge went well; stopped before committing as requested
+Removing shared.skg
+From ssh://git-annex-asus.local-USER-A_22_sincro/~/sincro
+   d25cc79..6fa27f0  annex/direct/master -> asus.local_sincro/annex/direct/master
+   d25cc79..6fa27f0  master     -> asus.local_sincro/master
+   d25cc79..6fa27f0  synced/master -> asus.local_sincro/synced/master
+[2014-07-17 18:17:16 CEST] Committer: Committing changes to git
+[2014-07-17 18:17:16 CEST] Pusher: Syncing with asus.local_sincro 
+To ssh://USER-A@git-annex-asus.local-USER-A_22_sincro/~/sincro/
+   0391880..8d1c927  git-annex -> synced/git-annex
+   6fa27f0..d9eb9e9  annex/direct/master -> synced/master
+[2014-07-17 18:17:17 CEST] RemoteControl: Syncing with asus.local_sincro 
+From ssh://git-annex-asus.local-USER-A_22_sincro/~/sincro
+   49efa5f..e328661  git-annex  -> asus.local_sincro/git-annex
+[2014-07-17 18:17:20 CEST] RemoteControl: Syncing with asus.local_sincro 
+From ssh://git-annex-asus.local-USER-A_22_sincro/~/sincro
+   6fa27f0..d9eb9e9  annex/direct/master -> asus.local_sincro/annex/direct/master
+   e328661..32e44ce  git-annex  -> asus.local_sincro/git-annex
+
+# End of transcript or log.
+"""]]
+
+> [[Deleted_files_during_merge]] seems like much the same problem.
+> In both that bug report and this one, the file that gets deleted on B was
+> earlier modifed on A. There is a small difference, in that in this bug
+> report, A made 2 commits, first deleting the file and then adding back a
+> new version. In the other bug report, A made just 1 commit, changing the
+> file. --[[Joey]]
+
+[[!meta title="Assistant deleted file when merging"]]
diff --git a/doc/bugs/git_annex_still_deleting_content_when_merging/comment_1_cb51e25c5e1656dcbb73b3ff680341f0._comment b/doc/bugs/git_annex_still_deleting_content_when_merging/comment_1_cb51e25c5e1656dcbb73b3ff680341f0._comment
new file mode 100644
index 0000000000..76db9bcc55
--- /dev/null
+++ b/doc/bugs/git_annex_still_deleting_content_when_merging/comment_1_cb51e25c5e1656dcbb73b3ff680341f0._comment
@@ -0,0 +1,14 @@
+[[!comment format=mdwn
+ username="http://joeyh.name/"
+ ip="108.236.230.124"
+ subject="comment 1"
+ date="2014-07-21T17:37:22Z"
+ content="""
+Well, this is definitely a different bug or issue than the \"ugly bug\". In particular, it only affects a single file. Also, based on the commits, this may not be occuring in a merge commit at all (although I am not sure; I am not familiar with the log syntax shown).
+
+AFAICS, the problem occurs on machine B. Which machine is the transcript from?
+
+The \"Removing shared.skg\" is a good clue. This seems to be printed by `git`, in merge-recursive.c's `process_entry`. What is puzzling to me is that it's printed after the \"Automatic merge went well; stopped before committing as requested\", which  AFAICS is printed out by git last thing. Is this \"Removing\" message then printed out by another git command?
+
+Enabling debug logging would probably help a lot, to narrow that down the next time this occurs.
+"""]]
diff --git a/doc/bugs/git_annex_still_deleting_content_when_merging/comment_2_bbbcce7fc5f34d733126c42be8ec0a1d._comment b/doc/bugs/git_annex_still_deleting_content_when_merging/comment_2_bbbcce7fc5f34d733126c42be8ec0a1d._comment
new file mode 100644
index 0000000000..b8748318ef
--- /dev/null
+++ b/doc/bugs/git_annex_still_deleting_content_when_merging/comment_2_bbbcce7fc5f34d733126c42be8ec0a1d._comment
@@ -0,0 +1,27 @@
+[[!comment format=mdwn
+ username="https://www.google.com/accounts/o8/id?id=AItOawmMLeU-zCzx2mc5pL2XT8a1UNkQwHAHjg8"
+ nickname="daniele"
+ subject="comment 2"
+ date="2014-07-22T01:47:29Z"
+ content="""
+> I am not familiar with the log syntax shown
+
+You mean the \"git log --stat\" part? Which git command would yield the most helpful syntax in this case?
+
+> AFAICS, the problem occurs on machine B. Which machine is the transcript from?
+
+Sorry I forgot to mention it: yes, it's from machine B.
+
+> Is this \"Removing\" message then printed out by another git command?
+
+Sorry I have no clue here. I didn't issue any git command from the terminal (nor did the user on computer A) if that was part of the question. It was all done in automatic.
+
+> Enabling debug logging would probably help a lot, to narrow that down the next time this occurs.
+
+Will do. I'll set 'annex.debug' to true in .git/config. Sadly, computer A is (a laptop) on vacation at the moment (well outside the local network), so I'll have to wait a couple of weeks to get back to debugging this. I'll have the logs with debug enabled when it happens again.
+
+Thanks again for your support and for developing git-annex.
+
+
+
+"""]]
diff --git a/doc/bugs/git_annex_still_deleting_content_when_merging/comment_3_f31a73e0e2c43f6a7f158455eadaa56c._comment b/doc/bugs/git_annex_still_deleting_content_when_merging/comment_3_f31a73e0e2c43f6a7f158455eadaa56c._comment
new file mode 100644
index 0000000000..04621df63b
--- /dev/null
+++ b/doc/bugs/git_annex_still_deleting_content_when_merging/comment_3_f31a73e0e2c43f6a7f158455eadaa56c._comment
@@ -0,0 +1,27 @@
+[[!comment format=mdwn
+ username="https://www.google.com/accounts/o8/id?id=AItOawmMLeU-zCzx2mc5pL2XT8a1UNkQwHAHjg8"
+ nickname="daniele"
+ subject="comment 3"
+ date="2014-07-22T02:10:14Z"
+ content="""
+I was unsure as to what git-log command would best describe the problematic commits but in the meantime I did a:
+    
+     git log --graph --decorate=full --full-diff
+
+These are the only three commits of that afternoon (the surrounding history is from completely different hours and very likely unrelated, so it wasn't posted)
+
+    * commit d9eb9e94a3973598a847a5bdab1b65e459c1588a
+    | Author: COMPUTER B
+    | Date:   Thu Jul 17 18:17:16 2014 +0200
+    |  
+    * commit 6fa27f0849227c490ac4d4d62ca86e4befe5121e
+    | Author: COMPUTER A
+    | Date:   Thu Jul 17 18:17:14 2014 +0200
+    |  
+    * commit d25cc793739573057e475c92c8d37ce4ecc7bc9b
+    | Author: COMPUTER A
+    | Date:   Thu Jul 17 18:17:12 2014 +0200
+
+
+It's a straight line (fast forward?), I don't see any merging either. Is this normal? Shouldn't a change in Repo A bring a merge in Repo B (where everything stayed the same) when things are synchronized? I don't fully understand how annex syncs happen so don't mind this question if it's all normal.
+"""]]
diff --git a/doc/bugs/git_annex_sync_in_direct_mode_does_not_honor_skip-worktree.mdwn b/doc/bugs/git_annex_sync_in_direct_mode_does_not_honor_skip-worktree.mdwn
new file mode 100644
index 0000000000..7ff82b69d3
--- /dev/null
+++ b/doc/bugs/git_annex_sync_in_direct_mode_does_not_honor_skip-worktree.mdwn
@@ -0,0 +1,38 @@
+### Please describe the problem.
+
+In a direct mode repo (crippled/uncrippled filesystem does not matter), when a symlink is marked using `git update-index --skip-worktree ` and removed, git annex sync still `git rm`s the symlink. This does not happen in indirect mode (git annex sync leaves the symlink in git intact).
+
+### What steps will reproduce the problem?
+
+[[!format  sh """
+mkdir test-repo; cd test-repo
+git init
+git annex init
+echo file1 >file1
+git annex add
+git commit -m"update"
+cd ..
+git clone test-repo test-repo2; cd test-repo2
+git annex init
+git annex direct
+git update-index --skip-worktree file1
+rm file1
+git annex sync
+"""]]
+
+Output of `git annex sync` indicates file has been removed from git. Repeating these steps without the `git annex direct` above to set the second repo to direct mode will succeed in retaining the symlink in git.
+
+### What version of git-annex are you using? On what operating system?
+
+4.20130521 using git-annex-standalone AUR build (uses Linux executable tarball) on Arch Linux
+
+### Please provide any additional information below.
+
+I'd like to use the skip-worktree scheme in order to be able to rm the symlink files (from the filesystem, not git), specifically for my Android devices. Syncing my music annex creates .mp3 symlinks that aren't actually MP3s, which gives the stock apps some fits. This would only be for clearing out symlinks; I fully understand that trying to doing this for downloaded content in a direct repo would be a Class A no-no. :-)
+
+I did a little digging in the code, and it looks like the source of this is the stageDirect step done specifically by `git annex sync` in direct repos (which makes sense, since indirect repos work). It does `git ls-files --others --exclude-standard --stage`. This list includes files marked skip-worktree, which means skip-worktree files would be treated like normal, and deleted because it's no longer there. There is an additional `-t` argument that could be added to ls-files that would provide the tag field to indicate if a file was marked skip-worktree, and they could be filtered out of processing.
+
+I wonder if this would have side effects, or if there are other places in the code where skip-worktree files would need to be handled, though. I'm particularly motivated to solve this, so let me know if it doesn't look like it would get looked at right away, and I'll have an excuse to get a Haskell dev environment setup again and shake the rust off.
+
+[[!tag confirmed]]
+[[!meta tag=deprecateddirectmode]]
diff --git a/doc/bugs/git_annex_sync_in_direct_mode_does_not_honor_skip-worktree/comment_1_69baeb997086c885f34fd1dc385cf5d6._comment b/doc/bugs/git_annex_sync_in_direct_mode_does_not_honor_skip-worktree/comment_1_69baeb997086c885f34fd1dc385cf5d6._comment
new file mode 100644
index 0000000000..e7baee584a
--- /dev/null
+++ b/doc/bugs/git_annex_sync_in_direct_mode_does_not_honor_skip-worktree/comment_1_69baeb997086c885f34fd1dc385cf5d6._comment
@@ -0,0 +1,12 @@
+[[!comment format=mdwn
+ username="http://joeyh.name/"
+ nickname="joey"
+ subject="comment 1"
+ date="2013-05-29T16:45:03Z"
+ content="""
+I'm a little worried about -t being \"semi-deprecated\". I don't know if it would be possible to use either of the other commands the man page suggests to get the info needed by `stageDirect`. (Particularly the Sha of their staged contents.)
+
+All of git-annex's access to the repository tree goes via Git.LsFiles and AFAICS, all the other git commands used there do respect skip-worktree.
+
+I encourage you to work on this, since you're motivated.
+"""]]
diff --git a/doc/bugs/git_annex_sync_in_direct_mode_does_not_honor_skip-worktree/comment_2_fb8c0bebb9aaa75ee7eaf6999b1db49e._comment b/doc/bugs/git_annex_sync_in_direct_mode_does_not_honor_skip-worktree/comment_2_fb8c0bebb9aaa75ee7eaf6999b1db49e._comment
new file mode 100644
index 0000000000..a252c5ea99
--- /dev/null
+++ b/doc/bugs/git_annex_sync_in_direct_mode_does_not_honor_skip-worktree/comment_2_fb8c0bebb9aaa75ee7eaf6999b1db49e._comment
@@ -0,0 +1,10 @@
+[[!comment format=mdwn
+ username="EvanDeaubl"
+ ip="12.130.123.174"
+ subject="comment 2"
+ date="2013-05-31T13:38:43Z"
+ content="""
+I share the concerns about the semi-deprecated status, although it's probably safer than they make it sound in the man page. The git test suite for sparse checkouts uses the same command to find files with skip-worktree set, and there are other tools that use it as well (magit being one example). The git maintainers couldn't remove it, or even fully deprecate it for eventual removal. It's been semi-deprecated for almost 3 years.
+
+I have a patch that fixes `git annex sync` and `git annex get` (it retrieved files marked skip-worktree) and it passes `git annex test`, but I'm going to more rigorously test it out on my particular use case before calling it good. When it is ready, I didn't see instructions on how you would like patches submitted anywhere in the wiki. How would you like to receive it?
+"""]]
diff --git a/doc/bugs/git_annex_sync_in_direct_mode_does_not_honor_skip-worktree/comment_3_6bfd4e9a7853af93e72b717249de9439._comment b/doc/bugs/git_annex_sync_in_direct_mode_does_not_honor_skip-worktree/comment_3_6bfd4e9a7853af93e72b717249de9439._comment
new file mode 100644
index 0000000000..b92ec0f131
--- /dev/null
+++ b/doc/bugs/git_annex_sync_in_direct_mode_does_not_honor_skip-worktree/comment_3_6bfd4e9a7853af93e72b717249de9439._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="http://joeyh.name/"
+ nickname="joey"
+ subject="comment 3"
+ date="2013-05-31T18:22:12Z"
+ content="""
+I accept patches by email, or pull requests.
+"""]]
diff --git a/doc/bugs/git_annex_sync_in_direct_mode_does_not_honor_skip-worktree/comment_4_a7eab4171af7e46bcc637aacf630e9db._comment b/doc/bugs/git_annex_sync_in_direct_mode_does_not_honor_skip-worktree/comment_4_a7eab4171af7e46bcc637aacf630e9db._comment
new file mode 100644
index 0000000000..c593d41e15
--- /dev/null
+++ b/doc/bugs/git_annex_sync_in_direct_mode_does_not_honor_skip-worktree/comment_4_a7eab4171af7e46bcc637aacf630e9db._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="http://joeyh.name/"
+ ip="209.250.56.102"
+ subject="ping?"
+ date="2014-03-19T20:54:57Z"
+ content="""
+You had a patch, but never shared it. I'm curious to see it..
+"""]]
diff --git a/doc/bugs/git_annex_sync_in_direct_mode_does_not_honor_skip-worktree/comment_5_cb98789c50c58f01055183dbaf7b4eba._comment b/doc/bugs/git_annex_sync_in_direct_mode_does_not_honor_skip-worktree/comment_5_cb98789c50c58f01055183dbaf7b4eba._comment
new file mode 100644
index 0000000000..d40d2d7891
--- /dev/null
+++ b/doc/bugs/git_annex_sync_in_direct_mode_does_not_honor_skip-worktree/comment_5_cb98789c50c58f01055183dbaf7b4eba._comment
@@ -0,0 +1,11 @@
+[[!comment format=mdwn
+ username="EvanDeaubl"
+ ip="24.251.129.149"
+ subject="comment 5"
+ date="2014-04-09T03:28:24Z"
+ content="""
+I'm afraid I abandoned this patch. It worked, but was still fidgety for being able to ignore parts of the tree. I found another way to do what I wanted by loading an indirect repo into /data and taking advantage of a surprise side effect in how the /sdcard filesystem translated the symlinks from the ext4 filesystem.
+
+I can probably scare it up from my archives, but it hasn't been kept up to date. The good news is (as I recall) the patch was pretty small.
+
+"""]]
diff --git a/doc/bugs/git_annex_test_fails.mdwn b/doc/bugs/git_annex_test_fails.mdwn
new file mode 100644
index 0000000000..39668b60f6
--- /dev/null
+++ b/doc/bugs/git_annex_test_fails.mdwn
@@ -0,0 +1,32 @@
+### Please describe the problem.
+git annex test fails outside a git repository.
+
+[[!format sh """
+$ git annex test
+git-annex: Not in a git repository.
+"""]]
+
+and then some tests fail once you work around that.
+[[!format sh """
+Exception: getCurrentDirectory:getWorkingDirectory: resource exhausted (Too many open files)
+"""]]
+
+Exception: getCurrentDirectory:getWorkingDirectory: resource exhausted
+
+### What steps will reproduce the problem?
+Run `git annex test`.
+
+### What version of git-annex are you using? On what operating system?
+HEAD at 425a3a1 built with GHC 8.2.1.
+
+### Please provide any additional information below.
+
+Full log is here: https://gist.github.com/ilovezfs/1ed886b43d534b239be25f4aa8b7394e
+
+### Have you had any luck using git-annex before? (Sometimes we get tired of reading bug reports all day and a lil' positive end note does wonders)
+
+Yes!
+
+[[!meta title="OSX git-annex test fails: Too many open files"]]
+
+> [[fixed|done]] --[[Joey]]
diff --git a/doc/bugs/git_annex_test_fails/comment_1_5e107e34b28a48c16d1240ea02d09439._comment b/doc/bugs/git_annex_test_fails/comment_1_5e107e34b28a48c16d1240ea02d09439._comment
new file mode 100644
index 0000000000..c36a360411
--- /dev/null
+++ b/doc/bugs/git_annex_test_fails/comment_1_5e107e34b28a48c16d1240ea02d09439._comment
@@ -0,0 +1,20 @@
+[[!comment format=mdwn
+ username="joey"
+ subject="""comment 1"""
+ date="2017-09-29T18:26:39Z"
+ content="""
+[Two separate problems reported in one bug report always makes extra work and
+risks one of the probles being forgotten about.
+Separate bug reports for separate problems, please.]
+
+I can reproduce test not working outside of a git repository.
+That is a reversion from 6.20170818. Bisected to commit
+db2a06b66f0aaf5a8e8822a0c01aa614a8e7a5a9. Fixed.
+
+The too many open files was also mentioned by another OSX user (also in a
+bug report about something else; that bug was closed...). I have not quite
+reproduced it, but running git-annex test on OSX I did see it was using 600
+or more open files and had quite a few git processes stacked up. There may
+be a small leak there, that's more likely to trip over a smaller limit on
+OSX.
+"""]]
diff --git a/doc/bugs/git_annex_test_fails/comment_2_057ba348486dfb45eafeaea4260e8642._comment b/doc/bugs/git_annex_test_fails/comment_2_057ba348486dfb45eafeaea4260e8642._comment
new file mode 100644
index 0000000000..4e5b0688fa
--- /dev/null
+++ b/doc/bugs/git_annex_test_fails/comment_2_057ba348486dfb45eafeaea4260e8642._comment
@@ -0,0 +1,51 @@
+[[!comment format=mdwn
+ username="joey"
+ subject="""comment 2"""
+ date="2017-09-29T19:13:19Z"
+ content="""
+On linux I'm seeing only around 90 open files max (mostly pipes to git), 
+and maybe 30 git processes max. It does not seem to be leaking a
+git process per test directory or anything like that on linux.
+
+On OSX, more accurate look at the open files (75868 is the pid of the
+git-annex test process)
+
+	while sleep 1; do lsof -p 75868|wc -l; done
+	      24
+	      32
+	      36
+	      46
+	      47
+	      49
+	      41
+	      32
+	      24
+	      30
+	      27
+	...
+
+Never went above 80 open files. Similarly:
+
+	while sleep 1; do ps | grep git |wc -l; done
+	      20
+	      17
+	      18
+	      17
+	      15
+	      22
+	       9
+	      14
+	      11
+	      19
+	       9
+	      11
+	      17
+
+	...
+
+Never went over 30 processes.
+
+It would be very helpful to pause git-annex test when it starts failing
+and take a look at lsof and ps to see what open files and child processes
+it has.
+"""]]
diff --git a/doc/bugs/git_annex_test_fails/comment_3_8a60a3c79e6f0f3d4a29c886e63bffba._comment b/doc/bugs/git_annex_test_fails/comment_3_8a60a3c79e6f0f3d4a29c886e63bffba._comment
new file mode 100644
index 0000000000..7b3763139a
--- /dev/null
+++ b/doc/bugs/git_annex_test_fails/comment_3_8a60a3c79e6f0f3d4a29c886e63bffba._comment
@@ -0,0 +1,11 @@
+[[!comment format=mdwn
+ username="joey"
+ subject="""comment 3"""
+ date="2017-09-29T19:35:28Z"
+ content="""
+Above was with daily OSX build, which is currently built with ghc 8.0.2.
+My linux test was also with ghc 8.0.2.
+
+It may be that ghc 8.2.1 has different cleanup behavior which
+causes more open files. The other reporter was also using ghc 8.2.1.
+"""]]
diff --git a/doc/bugs/git_annex_test_fails/comment_4_a4bbb2efcf75167ac938b00128b340fd._comment b/doc/bugs/git_annex_test_fails/comment_4_a4bbb2efcf75167ac938b00128b340fd._comment
new file mode 100644
index 0000000000..7c132a808d
--- /dev/null
+++ b/doc/bugs/git_annex_test_fails/comment_4_a4bbb2efcf75167ac938b00128b340fd._comment
@@ -0,0 +1,22 @@
+[[!comment format=mdwn
+ username="joey"
+ subject="""comment 4"""
+ date="2017-09-30T01:24:35Z"
+ content="""
+Built with ghc 8.2.1 on linux, I am seeing significantly more sub-processes
+and open files. Up to 170 sub-processes and 350 open files.
+
+Most of the open files are pipes.
+Most of the sub-processes are git cat-file --batch, and these are
+definitely lingering from test repos it's no longer using.
+
+Kind of looks like the annex state created by Test.annexeval
+is not getting garbage collected, and so the handles it has open are
+not getting closed. Hmm, I don't normally rely on GC to close handles,
+and am a bit surprised that ever worked, since the handles are pipes to
+processes that have no reason to exit unless the handles are closed.
+
+Made it explicitly stop the co-processes, and that seems to have fixed it.
+Indeed, the number of git subprocesses dropped to mostly below 10 since
+they get cleaned up without waiting for GC now.
+"""]]
diff --git a/doc/bugs/git_annex_test_fails_when_run_through_powershell.mdwn b/doc/bugs/git_annex_test_fails_when_run_through_powershell.mdwn
new file mode 100644
index 0000000000..fd9e2af990
--- /dev/null
+++ b/doc/bugs/git_annex_test_fails_when_run_through_powershell.mdwn
@@ -0,0 +1,30 @@
+### Please describe the problem.
+
+When running "git annex test" in a Powershell window, two tests fail (listed in additional details). I am assuming it is because it is using powershell internal commands instead of msys utilities, at least in the case of the "mv" failure. Specifying "mv.exe" instead of just "mv" on Windows should work.
+
+### What steps will reproduce the problem?
+
+Run "git annex test" from the powershell prompt. Two tests fail. All tests pass when run through cmd.
+
+### What version of git-annex are you using? On what operating system?
+
+Windows 7 Enterprise SP1
+Powershell version 4.0
+git version 1.9.5.msysgit.1
+git annex I *think* is 5.20150617, but I don't know for sure.
+
+### Please provide any additional information below.
+
+[[!format sh """
+# If you can, paste a complete transcript of the problem occurring here.
+# If the problem is with the git-annex assistant, paste in .git/annex/daemon.log
+
+addurl failed on file:///ws/wcs-project-new/.t/tmprepo5/myurl
+
+"mv": cannot move `C:\\Users\\duffrw\\LOCALS~1\\Temp\\importtest.0\\import1\\f' to `import1\\f12956.tmp'
+git-annex: MoveFileEx "C:\\Users\\duffrw\\LOCALS~1\\Temp\\importtest.0\\import1\\f" "import1\\f": unsupported operation (The system cannot move the file to a different disk drive.)
+
+# End of transcript or log.
+"""]]
+
+[[!tag moreinfo]]
diff --git a/doc/bugs/git_annex_test_fails_when_run_through_powershell/comment_1_014466c5da1922d34a498648baeeba26._comment b/doc/bugs/git_annex_test_fails_when_run_through_powershell/comment_1_014466c5da1922d34a498648baeeba26._comment
new file mode 100644
index 0000000000..f4d67c7551
--- /dev/null
+++ b/doc/bugs/git_annex_test_fails_when_run_through_powershell/comment_1_014466c5da1922d34a498648baeeba26._comment
@@ -0,0 +1,9 @@
+[[!comment format=mdwn
+ username="joey"
+ subject="""comment 1"""
+ date="2015-07-07T17:23:51Z"
+ content="""
+The "cannot move" message is not an error message from an external "mv"
+command. This error comes from Win32.moveFileEx, which fails on cross-device
+renames.
+"""]]
diff --git a/doc/bugs/git_annex_test_fails_when_run_through_powershell/comment_2_241ab999007eecaa2cd5a854e722e315._comment b/doc/bugs/git_annex_test_fails_when_run_through_powershell/comment_2_241ab999007eecaa2cd5a854e722e315._comment
new file mode 100644
index 0000000000..157fed799e
--- /dev/null
+++ b/doc/bugs/git_annex_test_fails_when_run_through_powershell/comment_2_241ab999007eecaa2cd5a854e722e315._comment
@@ -0,0 +1,16 @@
+[[!comment format=mdwn
+ username="joey"
+ subject="""comment 2"""
+ date="2015-07-07T18:15:21Z"
+ content="""
+The import problem can be reproduced by running git annex import on a different
+drive in Windows. No powershell needed; it does fall back to the external
+mv command, but that fails for some reason. Was unable to determine why
+(running the same mv at the command line works ok). Fixed for Windows by
+avoiding using the external mv command there.
+
+Still need more details about the addurl failure. AFAICS, it does not
+involve "mv" at all. Probably it involves
+the generated file:// url not working for some reason. 
+What drive and path did you run the test suite in?
+"""]]
diff --git a/doc/bugs/git_annex_testremote_fails_on_OSX_because_native___39__mktemp_-d__39___doesn__39__t_work.mdwn b/doc/bugs/git_annex_testremote_fails_on_OSX_because_native___39__mktemp_-d__39___doesn__39__t_work.mdwn
new file mode 100644
index 0000000000..5a5bf61c9e
--- /dev/null
+++ b/doc/bugs/git_annex_testremote_fails_on_OSX_because_native___39__mktemp_-d__39___doesn__39__t_work.mdwn
@@ -0,0 +1,89 @@
+### Please describe the problem.
+
+Created a special remote using one drive.
+ran git annex testremote 
+got errors, apparently starting with:
+
+    retrieveKeyFile:                 usage: mktemp [-d] [-q] [-t prefix] [-u] template ...
+    mktemp [-d] [-q] [-u] -t prefix 
+    FAIL (0.01s
+
+### What steps will reproduce the problem?
+
+As above.
+
+### What version of git-annex are you using? On what operating system?
+OSX 10.10.5 https://downloads.kitenet.net/git-annex/OSX/current/10.10_Yosemite/git-annex.dmg 
+
+### Please provide any additional information below.
+
+Verified that using gnu coreutils mktemp fixes the problem.
+Suggestion: Bundle that with the distribution.
+Along those lines, consider also including rclone and the script git-annex-remote-rclone,
+
+[[!format sh """
+# If you can, paste a complete transcript of the problem occurring here.
+# If the problem is with the git-annex assistant, paste in .git/annex/daemon.log
+testremote alanr-onedrive (generating test keys...) Remote Tests
+  unavailable remote
+    removeKey:                         Cannot run git-annex-remote-!dne! -- Make sure it's in your PATH and is executable.
+OK
+    storeKey:                          Cannot run git-annex-remote-!dne! -- Make sure it's in your PATH and is executable.
+OK (0.43s)
+    checkPresent:                    OK
+    retrieveKeyFile:                 OK
+    retrieveKeyFileCheap:            OK
+  key size Just 1048576; NoChunks; encryption none
+    removeKey when not present:      2017/03/09 21:48:42 Waiting for deletions to finish 2017/03/09 21:48:42 directory not found
+OK (3.80s)
+    present False:                   2017/03/09 21:48:44 directory not found
+OK (1.89s)
+    storeKey:                        2017/03/09 21:48:46 One drive root 'git-annex-test/70c/e24': Waiting for checks to finish
+2017/03/09 21:48:46 One drive root 'git-annex-test/70c/e24': Waiting for transfers to finish
+2017/03/09 21:48:49 
+Transferred:     1 MBytes (206.541 kBytes/s)
+Errors:                 0
+Checks:                 0
+Transferred:            1
+Elapsed time:        4.9s
+OK (4.97s)
+    present True:                    Total objects: 1 Total size: 1 MBytes (1048576 Bytes)
+OK (2.40s)
+    storeKey when already present:   2017/03/09 21:48:53 One drive root 'git-annex-test/70c/e24': Waiting for checks to finish
+2017/03/09 21:48:53 One drive root 'git-annex-test/70c/e24': Waiting for transfers to finish
+2017/03/09 21:48:53 
+Transferred:      0 Bytes (0 Bytes/s)
+Errors:                 0
+Checks:                 1
+Transferred:            0
+Elapsed time:        1.1s
+OK (1.12s)
+    present True:                    Total objects: 1 Total size: 1 MBytes (1048576 Bytes)
+OK (2.06s)
+    retrieveKeyFile:                 mktemp -d
+usage: mktemp [-d] [-q] [-t prefix] [-u] template ...
+       mktemp [-d] [-q] [-u] -t prefix 
+FAIL (0.01s)
+      failed
+    fsck downloaded object:          OK
+    retrieveKeyFile resume from 33%: FAIL
+      Exception: .git/annex/objects/Kv/9j/SHA256E-s1048576--22c73497372ed998c1e84743a67601e2c9522dc9ce78f180f90a3896313c0f61.this-is-a-test-key/SHA256E-s1048576--22c73497372ed998c1e84743a67601e2c9522dc9ce78f180f90a3896313c0f61.this-is-a-test-key: openBinaryFile: does not exist (No such file or directory)
+    fsck downloaded object:          OK
+    retrieveKeyFile resume from 0:   FAIL
+      Exception: failed to lock content: .git/annex/objects/Kv/9j/SHA256E-s1048576--22c73497372ed998c1e84743a67601e2c9522dc9ce78f180f90a3896313c0f61.this-is-a-test-key/SHA256E-s1048576--22c73497372ed998c1e84743a67601e2c9522dc9ce78f180f90a3896313c0f61.this-is-a-test-key: getFileStatus: does not exist (No such file or directory)
+    fsck downloaded object:          OK
+    retrieveKeyFile resume from end: cp: .git/annex/objects/Kv/9j/SHA256E-s1048576--22c73497372ed998c1e84743a67601e2c9522dc9ce78f180f90a3896313c0f61.this-is-a-test-key/SHA256E-s1048576--22c73497372ed998c1e84743a67601e2c9522dc9ce78f180f90a3896313c0f61.this-is-a-test-key: No such file or directory
+FAIL
+      Exception: failed to lock content: .git/annex/objects/Kv/9j/SHA256E-s1048576--22c73497372ed998c1e84743a67601e2c9522dc9ce78f180f90a3896313c0f61.this-is-a-test-key/SHA256E-s1048576--22c73497372ed998c1e84743a67601e2c9522dc9ce78f180f90a3896313c0f61.this-is-a-test-key: getFileStatus: does not exist (No such file or directory)
+    fsck downloaded object:          OK
+    removeKey when present:          OK (2.78s)
+    present False:                   ^C
+
+# End of transcript or log.
+"""]]
+
+### Have you had any luck using git-annex before? (Sometimes we get tired of reading bug reports all day and a lil' positive end note does wonders)
+
+Just starting but have high hopes :-)
+
+> [[done]] per my comment --[[Joey]]
diff --git a/doc/bugs/git_annex_testremote_fails_on_OSX_because_native___39__mktemp_-d__39___doesn__39__t_work/comment_1_66f4f2e9e674b02b2a7dee77690d26fd._comment b/doc/bugs/git_annex_testremote_fails_on_OSX_because_native___39__mktemp_-d__39___doesn__39__t_work/comment_1_66f4f2e9e674b02b2a7dee77690d26fd._comment
new file mode 100644
index 0000000000..67c339b408
--- /dev/null
+++ b/doc/bugs/git_annex_testremote_fails_on_OSX_because_native___39__mktemp_-d__39___doesn__39__t_work/comment_1_66f4f2e9e674b02b2a7dee77690d26fd._comment
@@ -0,0 +1,17 @@
+[[!comment format=mdwn
+ username="joey"
+ subject="""comment 1"""
+ date="2017-03-13T20:29:21Z"
+ content="""
+git-annex does not contain any calls to the "mktemp" command.
+
+So, it must be your external special remote that uses that command.
+It's up to the authors of external special remotes to make them run
+portably on whatever OS's they want to support. Probably if you mention
+this to the author of whatever external special remote you're using (you
+neglected to say what one it is in your bug report!), they will easily be
+able to fix it.
+
+git-annex is not going to get in the business of bundling a bunch of stuff
+that it does not depend on, sorry.
+"""]]
diff --git a/doc/bugs/git_annex_upgrade_loses_track_of_files_with___34____38____34___character___40__and_probably_others__41__.mdwn b/doc/bugs/git_annex_upgrade_loses_track_of_files_with___34____38____34___character___40__and_probably_others__41__.mdwn
new file mode 100644
index 0000000000..9daf8a0cb4
--- /dev/null
+++ b/doc/bugs/git_annex_upgrade_loses_track_of_files_with___34____38____34___character___40__and_probably_others__41__.mdwn
@@ -0,0 +1,33 @@
+"git annex upgrade" has lost track of some of my files.  Most of them have "&" characters.  The others contain "%" characters (I haven't tried the testcase below with "%" however).
+
+Testcase:
+
+    # (With git annex v2)
+    mkdir ~/testannex1
+    cd ~/testannex1
+    git init
+    git annex init "testannex1"
+    touch '02 - Afternoons & Coffeespoons.mp3'
+    touch 'no ampersand.mp3'
+    git annex add '02 - Afternoons & Coffeespoons.mp3'
+    git annex add 'no ampersand.mp3'
+    git commit -m added
+    git annex whereis '02 - Afternoons & Coffeespoons.mp3'
+    git annex whereis 'no ampersand.mp3'
+    # (Upgrade git-annex binary to v3 and then...)
+    git annex upgrade
+    git annex whereis '02 - Afternoons & Coffeespoons.mp3'
+    git annex whereis 'no ampersand.mp3'
+
+This produces:
+
+    12:38:40 ~/testannex1 (master)$ git annex whereis '02 - Afternoons & Coffeespoons.mp3'
+    whereis 02 - Afternoons & Coffeespoons.mp3 (0 copies) 
+    failed
+    git-annex: 1 failed
+    12:38:40 ~/testannex1 (master)$ git annex whereis 'no ampersand.mp3'
+    whereis no ampersand.mp3 (1 copy) 
+      	a7b680fc-a8d0-11e0-b0fe-4f94e86d1fb7  -- testannex1 <-- here
+    ok
+
+[[!tag done]]
diff --git a/doc/bugs/git_annex_upgrade_loses_track_of_files_with___34____38____34___character___40__and_probably_others__41__/comment_1_861506e40e0d04d2be98bbfe9188be89._comment b/doc/bugs/git_annex_upgrade_loses_track_of_files_with___34____38____34___character___40__and_probably_others__41__/comment_1_861506e40e0d04d2be98bbfe9188be89._comment
new file mode 100644
index 0000000000..194b36ac10
--- /dev/null
+++ b/doc/bugs/git_annex_upgrade_loses_track_of_files_with___34____38____34___character___40__and_probably_others__41__/comment_1_861506e40e0d04d2be98bbfe9188be89._comment
@@ -0,0 +1,12 @@
+[[!comment format=mdwn
+ username="http://joey.kitenet.net/"
+ nickname="joey"
+ subject="comment 1"
+ date="2011-07-07T21:04:23Z"
+ content="""
+What an evil little bug. In retrospect, this probably bit my own test upgrades, but I ran `git annex fsck` everywhere and so avoided the location log breakage.
+
+I've fixed the bug, which also involved files with other punctuation in their names [&:%] when using the WORM backend. 
+
+The only way I have to recover repos that have already been upgraded is to run `git annex fsck --fast` in each clone of such a repo, which will let it rebuild the location log information. I think that is the best way to recover; ie I can't think of a way to recover that doesn't need to do everything fsck does anyway.
+"""]]
diff --git a/doc/bugs/git_annex_wont_work_with_git-dir_and_work-tree.mdwn b/doc/bugs/git_annex_wont_work_with_git-dir_and_work-tree.mdwn
new file mode 100644
index 0000000000..0ac3eacfaf
--- /dev/null
+++ b/doc/bugs/git_annex_wont_work_with_git-dir_and_work-tree.mdwn
@@ -0,0 +1,29 @@
+### Please describe the problem.
+If using git annex with external .git dir webapp report a tread crashed and ever trying to restart it crash again and webapp is not able to upload/download/sync (also the download of git-annex updates is affected)
+
+### What steps will reproduce the problem?
+
+
+### What version of git-annex are you using? On what operating system?
+
+git-annex version: 5.20150113-gcf247cf
+build flags: Assistant Webapp Webapp-secure Pairing Testsuite S3 WebDAV Inotify DBus DesktopNotify XMPP DNS Feeds Quvi TDFA CryptoHash
+key/value backends: SHA256E SHA1E SHA512E SHA224E SHA384E SKEIN256E SKEIN512E SHA256 SHA1 SHA512 SHA224 SHA384 SKEIN256 SKEIN512 WORM URL
+remote types: git gcrypt S3 bup directory rsync web bittorrent webdav tahoe glacier ddar hook external
+
+Precompiled tarball from official git-annex download site
+
+Gentoo linux
+
+### Please provide any additional information below.
+
+[[!format sh """
+
+Launching web browser on file://../.git_annex_repos/pippo.git/annex/webapp.html      <-- note wrong path! should be /home/gioacchino/.git_annex_repos/pippo.git/annex/webapp.html
+
+RemoteControl crashed: user error (nice ["ionice","-c3","/home/gioacchino/opt/git-annex.linux/git-annex","remotedaemon"] exited 1)
+
+WebApp crashed: : hPutStr: illegal operation (handle is closed
+"""]]
+
+[[!meta title="assistant does not support nonstandard --git-dir"]]
diff --git a/doc/bugs/git_annex_wont_work_with_git-dir_and_work-tree/comment_1_cc3ea43301cd17b55794f9b9dd1e91cf._comment b/doc/bugs/git_annex_wont_work_with_git-dir_and_work-tree/comment_1_cc3ea43301cd17b55794f9b9dd1e91cf._comment
new file mode 100644
index 0000000000..2e76dda05b
--- /dev/null
+++ b/doc/bugs/git_annex_wont_work_with_git-dir_and_work-tree/comment_1_cc3ea43301cd17b55794f9b9dd1e91cf._comment
@@ -0,0 +1,27 @@
+[[!comment format=mdwn
+ username="joey"
+ subject="""comment 1"""
+ date="2015-02-09T17:54:00Z"
+ content="""
+I've confirmed this behavior. Normally git-annex propigates --git-dir
+to git commands it runs, but this is not done in a few cases in the
+assistant, including when it starts `git-annex remotedaemon` but also
+`git-annex fsck`, `git annex unused`, and `git annex transferkeys`.
+
+(Just search for readProgramFile; Remote.Git.fsckOnRemote is the only
+place outside the assistant that uses it, and it does take care to
+propigate --git-dir and --work-tree.)
+
+Also, restarting the assistant, either in the web ui or automatically due to
+an upgrade also doesn't propigate --git-dir.
+
+And, the assistant's autostart file doesn't support --git-dir.
+
+All of which could be fixed, but I have to wonder, why? What's the use case
+for a separate --git-dir with the assistant? I guess not vcsh because the
+assistant would watch for all changes to the home directory, and add them,
+and that's not how vcsh is used.
+
+I'm tempted to just make the assistant refuse to start when given a
+nonstandard --git-dir..
+"""]]
diff --git a/doc/bugs/git_annex_wont_work_with_git-dir_and_work-tree/comment_2_fb75f5941d355947b3f06165825dd012._comment b/doc/bugs/git_annex_wont_work_with_git-dir_and_work-tree/comment_2_fb75f5941d355947b3f06165825dd012._comment
new file mode 100644
index 0000000000..b716500997
--- /dev/null
+++ b/doc/bugs/git_annex_wont_work_with_git-dir_and_work-tree/comment_2_fb75f5941d355947b3f06165825dd012._comment
@@ -0,0 +1,16 @@
+[[!comment format=mdwn
+ username="https://www.google.com/accounts/o8/id?id=AItOawlvmWVPaSpWJO5_rimCL0lEoJRoKOIjhHE"
+ nickname="Gioacchino"
+ subject="Why shoul git-annex support --git-dir"
+ date="2015-03-01T10:10:31Z"
+ content="""
+Hi!
+
+There are multiple use case as I explained on IRC and it is why I and probably more users need it, there are some git-annex repository that i share also with other programs like RetroShare and apache USERDIR (like http://efesto.eigenlab.org/~gioacchino/) i don't like .git directory to be shared by these programs as the file are publics but not the metadata stored on the .git ( http://efesto.eigenlab.org/~gioacchino/.git/config ) i know i can do that with other tricks like htaccess but it is application specific hack while the real solution is to keep the .git in a private place
+
+Another use case is when you are grepping or doing scripty stuff on your dir having the .git in another place allow you to do stuff without concern about what there is inside .git without using options like --exclude and similar
+
+The list of use cases can grow as they are all the one caused git to support --git-dir so for the sake of consistence and for your donors please propagate --git-dir on all git-annex components ;)
+
+P.S. I am not native english speaker so sorry if something sounds bad
+"""]]
diff --git a/doc/bugs/git_annex_wont_work_with_git-dir_and_work-tree/comment_3_8c7b21c8e500de719fc2909d0c9027e0._comment b/doc/bugs/git_annex_wont_work_with_git-dir_and_work-tree/comment_3_8c7b21c8e500de719fc2909d0c9027e0._comment
new file mode 100644
index 0000000000..dc9d1b6b01
--- /dev/null
+++ b/doc/bugs/git_annex_wont_work_with_git-dir_and_work-tree/comment_3_8c7b21c8e500de719fc2909d0c9027e0._comment
@@ -0,0 +1,9 @@
+[[!comment format=mdwn
+ username="joey"
+ subject="""comment 3"""
+ date="2015-07-06T19:47:47Z"
+ content="""
+@Gioacchino, git-annex already fully supports --git-dir. It's only the
+assistant that does not. If you're doing "scripty stuff", you are, 
+I'd guess, not using the assistant...
+"""]]
diff --git a/doc/bugs/git_annex_wont_work_with_git-dir_and_work-tree/comment_4_423126c9884267b3466470d4ec7301ce._comment b/doc/bugs/git_annex_wont_work_with_git-dir_and_work-tree/comment_4_423126c9884267b3466470d4ec7301ce._comment
new file mode 100644
index 0000000000..b0f44c6c5d
--- /dev/null
+++ b/doc/bugs/git_annex_wont_work_with_git-dir_and_work-tree/comment_4_423126c9884267b3466470d4ec7301ce._comment
@@ -0,0 +1,10 @@
+[[!comment format=mdwn
+ username="gmazzurco89@89ed4f74a057a9c71b2a00000fdb201a88f81bcd"
+ nickname="gmazzurco89"
+ subject="comment 4"
+ date="2015-11-05T07:28:02Z"
+ content="""
+The \"scripty stuff\" may be a simple find or grep on the annexed dir, that doesn't actually involve git-annex, but those simple and usual commands are fooled by the content of .git, because of that it would be nice to have git-annex-{assistant, webapp, ...} work in this setup, it is not only a matter of scripts, other applications may have unexpected behavior in presence of .git -_-
+
+To make the usage of other apps more complicated doesn't seems an objective of git-annex-webapp, am I wrong?
+"""]]
diff --git a/doc/bugs/git_fsck_duplicateEntries_errors_when_using_adjusted_branch.mdwn b/doc/bugs/git_fsck_duplicateEntries_errors_when_using_adjusted_branch.mdwn
new file mode 100644
index 0000000000..079b915a8a
--- /dev/null
+++ b/doc/bugs/git_fsck_duplicateEntries_errors_when_using_adjusted_branch.mdwn
@@ -0,0 +1,69 @@
+### Please describe the problem.
+
+`git fsck` gives many duplicateEntries errors.  One of my repository's two clones is using an adjusted branch on MacOS.
+
+It's hard to say exactly what created this problem, because you don't see the error immediately after it happens.  But I believe all I did was add some files to the repo with the adjusted branch, run `git annex add` (I think), and git annex sync.  I believe this is all I did because when I have a remote that is apparently running `git fsck` and is rejecting pushes, and I think I'd pushed to this remote before I did the above.
+
+    $ git fsck
+    Checking object directories: 100% (256/256), done.
+    error in tree 85dfa00afd3a57a2981dfe9c0bf489288a3822ae: duplicateEntries: contains duplicate file entries
+    error in tree b30abe06ad503de3515269b657330ae6f4bef059: duplicateEntries: contains duplicate file entries
+    error in tree 9a3143961333528bfe0a8997413f3952a9892b90: duplicateEntries: contains duplicate file entries
+    error in tree c544533d9f4d802574ca3024249ae8171ebc7906: duplicateEntries: contains duplicate file entries
+    error in tree a906491c49fd46cc48a8a8ad2e5c059b9909418d: duplicateEntries: contains duplicate file entries
+    error in tree 45e22b842e07260b464306fd54daaddb00fe8689: duplicateEntries: contains duplicate file entries
+    error in tree 14a7cd8bc15738a61800485f63664d598fa5e100: duplicateEntries: contains duplicate file entries
+    [...]
+
+What's particularly interesting is that the duplicate objects appear to be the same blobs.
+
+    $ git ls-tree 7171d50e236e541b03faacb3b7113858a3640c4c
+    120000 blob 52f9ac67c8e73aa22057ecb5a195ebc51f0df5ea    General Clarification.docx
+    120000 blob 52f9ac67c8e73aa22057ecb5a195ebc51f0df5ea    General Clarification.docx
+    120000 blob 19b187309ddedd120791d3b9e0779f8ec7a21c9e    Mockumentary_Classic Story Synopses.doc
+    120000 blob 19b187309ddedd120791d3b9e0779f8ec7a21c9e    Mockumentary_Classic Story Synopses.doc
+    120000 blob 9c3d92e749e34dd1ec43a2c0b36425ae1ff7b01f    Worksheet 2018.pdf
+    120000 blob 9c3d92e749e34dd1ec43a2c0b36425ae1ff7b01f    Worksheet 2018.pdf
+    120000 blob 8445929b506f916f885018bb5a4a170f3901b7fb    Worksheet 2018.xls
+    120000 blob 8445929b506f916f885018bb5a4a170f3901b7fb    Worksheet 2018.xls
+
+We can check systematically:
+
+    $ git fsck  |& grep duplicateEntries | cut -f 4 -d ' ' | xargs git ls-tree | uniq -u
+    (No output)
+
+I see two previous bugs about this bug and an open question for more information.
+
+ * 
+ * 
+
+### What steps will reproduce the problem?
+
+Don't really know, sadly.
+
+### What version of git-annex are you using? On what operating system?
+
+Both machines show the following:
+
+    $ git --version
+    git version 2.18.0
+
+    $ git annex version
+    git-annex version: 6.20180626
+    build flags: Assistant Webapp Pairing S3(multipartupload)(storageclasses) WebDAV FsEvents ConcurrentOutput TorrentParser MagicMime Feeds Testsuite
+    dependency versions: aws-0.19 bloomfilter-2.0.1.0 cryptonite-0.25 DAV-1.3.2 feed-1.0.0.0 ghc-8.2.2 http-client-0.5.13 persistent-sqlite-2.6.4 torrent-10000.1.1 uuid-1.3.13 yesod-1.4.5
+    key/value backends: SHA256E SHA256 SHA512E SHA512 SHA224E SHA224 SHA384E SHA384 SHA3_256E SHA3_256 SHA3_512E SHA3_512 SHA3_224E SHA3_224 SHA3_384E SHA3_384 SKEIN256E SKEIN256 SKEIN512E SKEIN512 BLAKE2B256E BLAKE2B256 BLAKE2B512E BLAKE2B512 BLAKE2B160E BLAKE2B160 BLAKE2B224E BLAKE2B224 BLAKE2B384E BLAKE2B384 BLAKE2S256E BLAKE2S256 BLAKE2S160E BLAKE2S160 BLAKE2S224E BLAKE2S224 BLAKE2SP256E BLAKE2SP256 BLAKE2SP224E BLAKE2SP224 SHA1E SHA1 MD5E MD5 WORM URL
+    remote types: git gcrypt p2p S3 bup directory rsync web bittorrent webdav adb tahoe glacier ddar hook external
+    operating system: darwin x86_64
+    supported repository versions: 3 5 6
+    upgrade supported from repository versions: 0 1 2 3 4 5
+
+However I just recently ran brew update on the two machines.  I don't know what versions of git and git-annex they were running before.
+
+### Please provide any additional information below.
+
+I'm happy to help debug this if I can.
+
+### Have you had any luck using git-annex before? (Sometimes we get tired of reading bug reports all day and a lil' positive end note does wonders)
+
+I <3 git-annex.
diff --git a/doc/bugs/git_fsck_duplicateEntries_errors_when_using_adjusted_branch/comment_1_1a6674245aed0c325361043d1100daec._comment b/doc/bugs/git_fsck_duplicateEntries_errors_when_using_adjusted_branch/comment_1_1a6674245aed0c325361043d1100daec._comment
new file mode 100644
index 0000000000..6d1685d5f4
--- /dev/null
+++ b/doc/bugs/git_fsck_duplicateEntries_errors_when_using_adjusted_branch/comment_1_1a6674245aed0c325361043d1100daec._comment
@@ -0,0 +1,17 @@
+[[!comment format=mdwn
+ username="justin.lebar@7a36fcafc322d9a381e89f08ab6289033c6dde91"
+ nickname="justin.lebar"
+ avatar="http://cdn.libravatar.org/avatar/9fca4b61a1ab555f231851e7543f9a3e"
+ subject="comment 1"
+ date="2018-07-08T19:53:08Z"
+ content="""
+OK, that systematic check for duplicates is not right.  A better check is, how many blobs are not duplicated, and how many are duplicated, in the `git ls-tree` output?
+
+    # Not duplicated blobs
+    $ git fsck  |& grep duplicateEntries | cut -f 4 -d ' ' | sed -e 's/://' | xargs -n1 git ls-tree | grep -v ' tree ' | uniq -u | wc -l
+    324
+
+    # Duplicated blobs
+    $ git fsck  |& grep duplicateEntries | cut -f 4 -d ' ' | sed -e 's/://' | xargs -n1 git ls-tree | grep -v ' tree ' | uniq -u | wc -l
+    416
+"""]]
diff --git a/doc/bugs/git_fsck_duplicateEntries_errors_when_using_adjusted_branch/comment_2_ccadd646ec0fe7abec461c110bfb7ae7._comment b/doc/bugs/git_fsck_duplicateEntries_errors_when_using_adjusted_branch/comment_2_ccadd646ec0fe7abec461c110bfb7ae7._comment
new file mode 100644
index 0000000000..f18935274d
--- /dev/null
+++ b/doc/bugs/git_fsck_duplicateEntries_errors_when_using_adjusted_branch/comment_2_ccadd646ec0fe7abec461c110bfb7ae7._comment
@@ -0,0 +1,20 @@
+[[!comment format=mdwn
+ username="joey"
+ subject="""comment 2"""
+ date="2018-07-16T15:27:13Z"
+ content="""
+Something more is involved than git annex add and sync on an adjusted
+branch, because when I try that the problem does not appear.
+(Also tried on OSX in case it was somehow OSX specific.)
+
+I see that the mode of the files in the problem tree you showed
+is 120000, which tells me that tree is one that was committed to the master
+branch, not the adjusted unlocked branch. That suggests that the problem is
+in reverseAdjustedTree. But I don't think I will be able to find it by
+staring at the code; I need a way to reproduce the problem.
+
+What *might* help is if you can show the full directory tree in the
+repository (with names mangled for privacy as needed). It may be that it's
+somehow caused by adding a file at a particular location in the tree,
+since a lot of the complication in the code is around handling such things.
+"""]]
diff --git a/doc/bugs/git_fsck_duplicateEntries_errors_when_using_adjusted_branch/comment_3_2a6ef51c96a914be5d248b526ad37394._comment b/doc/bugs/git_fsck_duplicateEntries_errors_when_using_adjusted_branch/comment_3_2a6ef51c96a914be5d248b526ad37394._comment
new file mode 100644
index 0000000000..d2edf68b24
--- /dev/null
+++ b/doc/bugs/git_fsck_duplicateEntries_errors_when_using_adjusted_branch/comment_3_2a6ef51c96a914be5d248b526ad37394._comment
@@ -0,0 +1,37 @@
+[[!comment format=mdwn
+ username="jlebar"
+ avatar="http://cdn.libravatar.org/avatar/9fca4b61a1ab555f231851e7543f9a3e"
+ subject="comment 3"
+ date="2018-07-17T05:12:03Z"
+ content="""
+Thanks a lot, Joey.
+
+I wrote a script that replaces each directory or filename with a salted hash:
+
+	#!/usr/bin/env python
+
+	import sys
+	import hashlib
+
+	def hash(s):
+	  m = hashlib.sha256()
+	  m.update(s)
+	  m.update('')
+	  return m.hexdigest()
+
+	for line in sys.stdin:
+	  print '/'.join(hash(p) for p in line.split('/'))
+
+Then I ran
+
+	$ git ls-files | python hash_paths.py | bzip2 > repo_paths.bz2  # attached
+
+To make something you can correlate with the git fsck errors, I ran
+
+	$ git fsck  |& grep duplicateEntries | cut -f 4 -d ' ' | sed -e 's/://' | xargs -n1 git ls-tree | grep -v ' tree '  > blobs
+	$ paste <(cut -f 1 blobs) <(cut -f 2 blobs | python hash_paths.py) | bzip2 > fsck_errors.bz2  # attached
+
+So the second column in fsck_errors is the salted+hashed filename like in the comment above.  You should be able to correlate the \"filenames\" in fsck_errors with the paths in repo_paths.
+
+I'll email you the relevant files.
+"""]]
diff --git a/doc/bugs/git_fsck_duplicateEntries_errors_when_using_adjusted_branch/comment_4_f9588887f75c0d17b87f2718b0584a94._comment b/doc/bugs/git_fsck_duplicateEntries_errors_when_using_adjusted_branch/comment_4_f9588887f75c0d17b87f2718b0584a94._comment
new file mode 100644
index 0000000000..675ac64b84
--- /dev/null
+++ b/doc/bugs/git_fsck_duplicateEntries_errors_when_using_adjusted_branch/comment_4_f9588887f75c0d17b87f2718b0584a94._comment
@@ -0,0 +1,37 @@
+[[!comment format=mdwn
+ username="joey"
+ subject="""comment 4"""
+ date="2018-07-17T14:59:26Z"
+ content="""
+Looking at those files, all the duplicated files
+are located at least 2 subdirectories deep, and most more like 5-7
+deep. But, almost all of the repo is inside such
+subdirectories, so that is not conclusive.
+
+Aha! I managed to reproduce it:
+
+	joey@darkstar:~/tmp/t#master(unlocked)>mkdir ook
+	joey@darkstar:~/tmp/t#master(unlocked)>cd ook
+	joey@darkstar:~/tmp/t/ook#master(unlocked)>mkdir boop
+	joey@darkstar:~/tmp/t/ook#master(unlocked)>cd boop
+	joey@darkstar:~/tmp/t/ook/boop#master(unlocked)>mkdir beep
+	joey@darkstar:~/tmp/t/ook/boop#master(unlocked)>cd beep
+	joey@darkstar:~/tmp/t/ook/boop/beep#master(unlocked)>mkdir yeep
+	joey@darkstar:~/tmp/t/ook/boop/beep#master(unlocked)>cd yeep
+	joey@darkstar:~/tmp/t/ook/boop/beep/yeep#master(unlocked)>date > X
+	joey@darkstar:~/tmp/t/ook/boop/beep/yeep#master(unlocked)>git annex add
+	add X ok
+	(recording state in git...)
+	joey@darkstar:~/tmp/t/ook/boop/beep/yeep#master(unlocked)>git annex sync
+	commit
+	[adjusted/master(unlocked) fe11872] git-annex in joey@darkstar:~/tmp/t
+	 1 file changed, 1 insertion(+)
+	 create mode 100644 ook/boop/beep/yeep/X
+	ok
+	joey@darkstar:~/tmp/t/ook/boop/beep/yeep#master(unlocked)>git fsck
+	error in tree a5fcd5b3aa5189ed8916f025cf035fce74098a1a: duplicateEntries: contains duplicate file entries
+	Checking object directories: 100% (256/256), done.
+
+Ok, it seems it's simply caused by deep paths in the tree and nothing
+else. Will debug from here.
+"""]]
diff --git a/doc/bugs/git_fsck_duplicateEntries_errors_when_using_adjusted_branch/comment_5_a9d6cf6cd38bfa6f391beaf6831b0ef4._comment b/doc/bugs/git_fsck_duplicateEntries_errors_when_using_adjusted_branch/comment_5_a9d6cf6cd38bfa6f391beaf6831b0ef4._comment
new file mode 100644
index 0000000000..92c96f6dfd
--- /dev/null
+++ b/doc/bugs/git_fsck_duplicateEntries_errors_when_using_adjusted_branch/comment_5_a9d6cf6cd38bfa6f391beaf6831b0ef4._comment
@@ -0,0 +1,45 @@
+[[!comment format=mdwn
+ username="joey"
+ subject="""comment 5"""
+ date="2018-07-17T15:12:12Z"
+ content="""
+I wrote a reproduction script, which worked several times, and then stopped
+working. Now I can't reproduce it using that script. Some kind of race condition?
+Only happens before coffee o-clock? I don't know. :-/
+
+The script:
+
+	#!/bin/sh
+	sudo rm -rf /tmp/repo
+	git init /tmp/repo
+	cd /tmp/repo
+	git annex init
+	date > foo
+	git annex add foo
+	git annex sync
+	git annex upgrade
+	git annex adjust --unlock
+	mkdir -p ook/boop/beep/yeep
+	date > ook/boop/beep/yeep/x
+	git annex add
+	git annex sync
+	git fsck
+
+I also noticed that, once a tree gets duplicate entries in it, they are
+carried forward into the new trees when other commits are made to that
+directory. This may explain why fsck is finding so many trees to complain
+about in your repsitory.
+
+The commit made to the adjusted branch does not have a duplicate in the tree.
+The reverse adjusted commit made to master does. So it must involve
+adjustTree somehow.
+
+I don't see anything likely to cause a race condition in adjustTree.
+However, I do think that v6 mode has many bugs some of which may be
+race conditions, and perhaps the root cause is not adjustTree.
+
+This seems likely related in some way to
+[[bugs/add_fails_with_v6_repo_when_four_levels_deep]],
+which I incidentially reproduced for the first time (and then fixed) while
+trying again to reproduce this bug.
+"""]]
diff --git a/doc/bugs/git_proxy_uses_ls-files_without_--exclude-standard.mdwn b/doc/bugs/git_proxy_uses_ls-files_without_--exclude-standard.mdwn
new file mode 100644
index 0000000000..dc0e140526
--- /dev/null
+++ b/doc/bugs/git_proxy_uses_ls-files_without_--exclude-standard.mdwn
@@ -0,0 +1,38 @@
+### Please describe the problem.
+
+I'm starting to slowly migrate my personal data collection (530GB 3.7M files) under git-annex.  I'm going piece by piece, and not yet giving up my other synchronization methods (yet); thus I need to stay in direct mode.  
+
+I initially found that `git annex` commands [were quite slow](https://github.com/datalad/datalad/issues/17), but I was able to address that by adding all my "not yet included" directories to `.gitignore` at the root of the working tree.  Unfortunately, `git annex proxy` remains super slow, because I notice that it does not include `--exclude-standard` in its calls to ls-files, and thus does not respect `.gitignore`.  Here's an example from the `--debug` log:
+
+```
+ read: git ["--git-dir=../../../../../.git","--work-tree=../../../../..","--literal-pathspecs","-c","core.bare=false","ls-files","--others","-z","--","../../../../.."]
+```
+
+As a result, I was very shocked to learn that (25 min later), `git annex proxy` was still setting up, and had duplicated 140GB of untracked files!
+
+My end goal is actually just to add files directly to the git repo, bypassing the annex, in spite of being in direct mode.  (I can do this with the largefiles attribute, but I'd like to be able to control it directly irrespective of size.)
+
+### What steps will reproduce the problem?
+
+```
+git annex proxy --debug -- git commit myfile -m foo
+```
+
+### What version of git-annex are you using? On what operating system?
+
+Version 6.20170520 on Mac OS 10.13.4.
+
+### Please provide any additional information below.
+
+[[!format sh """
+# If you can, paste a complete transcript of the problem occurring here.
+# If the problem is with the git-annex assistant, paste in .git/annex/daemon.log
+
+
+# End of transcript or log.
+"""]]
+
+### Have you had any luck using git-annex before? (Sometimes we get tired of reading bug reports all day and a lil' positive end note does wonders)
+
+Today is my first day trying it out!  It's fabulous so far, but I'm at the beginning of the learning curve.
+
diff --git a/doc/bugs/git_proxy_uses_ls-files_without_--exclude-standard/comment_1_81383cf73102346fa8d537e1dde30353._comment b/doc/bugs/git_proxy_uses_ls-files_without_--exclude-standard/comment_1_81383cf73102346fa8d537e1dde30353._comment
new file mode 100644
index 0000000000..d0bb11b071
--- /dev/null
+++ b/doc/bugs/git_proxy_uses_ls-files_without_--exclude-standard/comment_1_81383cf73102346fa8d537e1dde30353._comment
@@ -0,0 +1,41 @@
+[[!comment format=mdwn
+ username="joey"
+ subject="""comment 1"""
+ date="2018-05-21T15:46:22Z"
+ content="""
+It wouldn't have actually copied 140 gb of files, unless you're using git-annex
+on a filesystem that does not support hard links. If it used hard links,
+it would not waste much space while running.
+
+There may be edge cases where, if git-annex proxy did not copy/hard link
+ignored files from the work tree to its temporary directory, the proxied
+git command would not behave the same as an unproxed git command.
+
+Let's see, such edge cases would have to involve a gitignored file that
+is still somehow affected by the proxied git command.
+
+The obvious case is, you have `.*` gitignored, and you run `git annex proxy
+-- git add .foo --force` to add the ignored file. If git-annex didn't copy
+`.foo`, that would fail, albeit in a fairly obvious way.
+
+Another problem case: You have `.*` gitignored, and you have a local
+file `.foo` which is not checked in. You run 
+`git annex proxy -- git merge branch`, and the branch happens to add
+`.foo` with different contents. The merge would normally fail, because
+there are conflicting changes in the working tree. If proxy were
+changed, the proxied merge would succeed. The local changes in this
+case get lost. I've verified that this change causes data loss in this
+situation.
+
+So, the current behavior is the safe and right behavior; git-annex should
+not lose data by default to optimise for an unusual edge case.
+
+It could be an option, but it would have to be flagged as causing data
+loss in some situations involving local modifications to gitignored files,
+and causing proxied git behavior to differ from non-proxied git behavior
+in other situations. I don't know if the potential benefit is worth the
+foot-gun potential.
+
+The code change is very simple if you want to play with it. In
+Command/Proxy.hs find the Git.LsFiles line and change "True" to "False".
+"""]]
diff --git a/doc/bugs/git_rename_detection_on_file_move.mdwn b/doc/bugs/git_rename_detection_on_file_move.mdwn
new file mode 100644
index 0000000000..306641cc2c
--- /dev/null
+++ b/doc/bugs/git_rename_detection_on_file_move.mdwn
@@ -0,0 +1,14 @@
+It's unfortunate that git-annex sorta defeats git's rename detection.
+
+When an annexed file is moved to a different directory (specifically, a
+directory that is shallower or deeper than the old directory),
+the symlink often has to change. And so git log cannot --follow back
+through the rename history, since all it has to go on is that symlink,
+which it effectively sees as a one line file containing the symlink target.
+
+One way to fix this might be to do the `git annex fix` *after* the rename
+is committed. This would mean that a commit would result in new staged
+changes for another commit, which is perhaps startling behavior.
+
+The other way to fix it is to stop using symlinks, see [[todo/smudge]].
+[[!tag confirmed]]
diff --git a/doc/bugs/git_rename_detection_on_file_move/comment_10_5ec2f965c80cc5dd31ee3c4edb695664._comment b/doc/bugs/git_rename_detection_on_file_move/comment_10_5ec2f965c80cc5dd31ee3c4edb695664._comment
new file mode 100644
index 0000000000..6ea2677289
--- /dev/null
+++ b/doc/bugs/git_rename_detection_on_file_move/comment_10_5ec2f965c80cc5dd31ee3c4edb695664._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="https://www.google.com/accounts/o8/id?id=AItOawnpdM9F8VbtQ_H5PaPMpGSxPe_d5L1eJ6w"
+ nickname="Rafael"
+ subject="comment 10"
+ date="2012-05-15T07:36:25Z"
+ content="""
+Won't git itself be fixed on this issue? It was on my plans to look into that, however I don't know how difficult it will be.
+"""]]
diff --git a/doc/bugs/git_rename_detection_on_file_move/comment_1_0531dcfa833b0321a7009526efe3df33._comment b/doc/bugs/git_rename_detection_on_file_move/comment_1_0531dcfa833b0321a7009526efe3df33._comment
new file mode 100644
index 0000000000..8fec6bad72
--- /dev/null
+++ b/doc/bugs/git_rename_detection_on_file_move/comment_1_0531dcfa833b0321a7009526efe3df33._comment
@@ -0,0 +1,26 @@
+[[!comment format=mdwn
+ username="http://christian.amsuess.com/chrysn"
+ nickname="chrysn"
+ subject="use mini-branches"
+ date="2011-03-09T23:47:48Z"
+ content="""
+if you go for the two-commits version, small intermediate branches (or git-commit-tree) could be used to create a tree like this:
+
+
+    *   commit 106eef2
+    |\  Merge: 436e46f 9395665
+    | | 
+    | |     the main commit
+    | |   
+    | * commit 9395665
+    |/  
+    |       intermediate move
+    |  
+    * commit 436e46f
+    | 
+    |     ...
+
+while the first commit (436e46f) has a \"`/subdir/foo → ../.git-annex/where_foo_is`\", the intermediate (9395665) has \"`/subdir/deeper/foo → ../.git-annex/where_foo_is`\", and the inal commit (106eef2) has \"`/subdir/deeper/foo → ../../.git-annex/where_foo_is`\".
+
+`--follow` uses the intermediate commit to find the history, but the intermediate commit would neither show up in `git log --first-parent` nor affect `git diff HEAD^..` & co. (there could still be confusion over `git show`, though).
+"""]]
diff --git a/doc/bugs/git_rename_detection_on_file_move/comment_2_7101d07400ad5935f880dc00d89bf90e._comment b/doc/bugs/git_rename_detection_on_file_move/comment_2_7101d07400ad5935f880dc00d89bf90e._comment
new file mode 100644
index 0000000000..7d50c58d1b
--- /dev/null
+++ b/doc/bugs/git_rename_detection_on_file_move/comment_2_7101d07400ad5935f880dc00d89bf90e._comment
@@ -0,0 +1,27 @@
+[[!comment format=mdwn
+ username="praet"
+ ip="81.240.159.215"
+ subject="Use variable symlinks, relative to the repo's root ?"
+ date="2011-03-10T16:50:28Z"
+ content="""
+It all boils down to the fact that the path to a relative symlink's target is determined relative to the symlink itself.
+
+Now, if we define the symlink's target relative to the git repo's root (eg. using the $GIT_DIR environment variable, which can be a relative or absolute path itself), this unfortunately results in an absolute symlink, which would -for obvious reasons- only be usable locally:
+
+    user@host:~$ mkdir -p tmp/{.git/annex,somefolder}
+    user@host:~$ export GIT_DIR=~/tmp
+    user@host:~$ touch $GIT_DIR/.git/annex/realfile
+    user@host:~$ ln -s $GIT_DIR/.git/annex/realfile $GIT_DIR/somefolder/file
+    user@host:~$ ls -al $GIT_DIR/somefolder/
+    total 12
+    drwxr-x--- 2 user group 4096 2011-03-10 16:54 .
+    drwxr-x--- 4 user group 4096 2011-03-10 16:53 ..
+    lrwxrwxrwx 1 user group   33 2011-03-10 16:54 file -> /home/user/tmp/.git/annex/realfile
+    user@host:~$
+
+So, what we need is the ability to record the actual variable name (instead of it's value) in our symlinks.
+
+It *is* possible, using [variable/variant symlinks](http://en.wikipedia.org/wiki/Symbolic_link#Variable_symbolic_links), yet I'm unsure as to whether or not this is available on Linux systems, and even if it is, it would introduce compatibility issues in multi-OS environments.
+
+Thoughts on this?
+"""]]
diff --git a/doc/bugs/git_rename_detection_on_file_move/comment_3_57010bcaca42089b451ad8659a1e018e._comment b/doc/bugs/git_rename_detection_on_file_move/comment_3_57010bcaca42089b451ad8659a1e018e._comment
new file mode 100644
index 0000000000..534723254a
--- /dev/null
+++ b/doc/bugs/git_rename_detection_on_file_move/comment_3_57010bcaca42089b451ad8659a1e018e._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="http://joey.kitenet.net/"
+ nickname="joey"
+ subject="comment 3"
+ date="2011-03-16T03:03:19Z"
+ content="""
+Interesting, I had not heard of variable symlinks before. AFAIK linux does not have them.
+"""]]
diff --git a/doc/bugs/git_rename_detection_on_file_move/comment_4_79d96599f757757f34d7b784e6c0e81c._comment b/doc/bugs/git_rename_detection_on_file_move/comment_4_79d96599f757757f34d7b784e6c0e81c._comment
new file mode 100644
index 0000000000..c265b58995
--- /dev/null
+++ b/doc/bugs/git_rename_detection_on_file_move/comment_4_79d96599f757757f34d7b784e6c0e81c._comment
@@ -0,0 +1,34 @@
+[[!comment format=mdwn
+ username="praet"
+ ip="81.240.27.89"
+ subject="Brainfart"
+ date="2011-03-20T20:11:27Z"
+ content="""
+Haven't given these any serious thought (which will become apparent in a moment) but hoping they will give birth to some less retarded ideas:
+
+---
+
+### Bait'n'switch
+
+- pre-commit: Replace all staged symlinks (when pointing to annexed files) with plaintext files containing the key of their respective annexed content, re-stage, and add their paths (relative to repo root) to .gitignore.
+- post-commit: Replace the plaintext files with (git annex fix'ed) symlinks.
+
+In doing so, the blobs to be committed can remain unaltered, irrespective of their related files' depth in the directory hierarchy.
+
+To prevent git from reporting ALL annexed files as unstaged changes after running post-commit hook, their paths would need to be added to .gitignore.
+
+This wouldn't cause any issues when adding files, very little when modifying files (would need some alterations to \"git annex unlock\"), BUT would make git totally oblivious to removals...
+
+---
+
+### Manifest-based (re)population
+- Keep a manifest of all annexed files (key + relative path)
+- DON'T track the symlinks (.gitignore)
+- Populate/update the directory structure using a post-commit hook.
+
+... thus circumventing the issue entirely, yet diffstats (et al.) would be rather uninformative.
+
+---
+
+***Wide open to suggestions, criticism, mocking laughter and finger-pointing :)***
+"""]]
diff --git a/doc/bugs/git_rename_detection_on_file_move/comment_5_d61f5693d947b9736b29fca1dbc7ad76._comment b/doc/bugs/git_rename_detection_on_file_move/comment_5_d61f5693d947b9736b29fca1dbc7ad76._comment
new file mode 100644
index 0000000000..93db97e704
--- /dev/null
+++ b/doc/bugs/git_rename_detection_on_file_move/comment_5_d61f5693d947b9736b29fca1dbc7ad76._comment
@@ -0,0 +1,12 @@
+[[!comment format=mdwn
+ username="praet"
+ ip="81.242.56.203"
+ subject="comment 5"
+ date="2011-03-21T19:58:34Z"
+ content="""
+In the meantime, would it be acceptable to split the pre-commit hook
+into two discrete parts?
+
+This would allow to (if preferred) defer \"git annex fix\" until
+post-commit while still keeping the safety net for unlocked files.
+"""]]
diff --git a/doc/bugs/git_rename_detection_on_file_move/comment_6_f63de6fe2f7189c8c2908cc41c4bc963._comment b/doc/bugs/git_rename_detection_on_file_move/comment_6_f63de6fe2f7189c8c2908cc41c4bc963._comment
new file mode 100644
index 0000000000..7398ac5614
--- /dev/null
+++ b/doc/bugs/git_rename_detection_on_file_move/comment_6_f63de6fe2f7189c8c2908cc41c4bc963._comment
@@ -0,0 +1,19 @@
+[[!comment format=mdwn
+ username="http://adamspiers.myopenid.com/"
+ nickname="Adam"
+ subject="extra level of indirection"
+ date="2011-12-19T12:45:18Z"
+ content="""
+Surely this could be handled with an extra layer of indirection?
+
+git-annex would ensure that every directory containing annexed data contains a new symlink `.git-annex` which points to `$git_root/.git/annex`.  Then every symlink to an annexed object uses a relative symlink via this: `.git_annex/objects/xx/yy/ZZZZZZZZZZ`.  Even though this symlink is relative, moving it to a different directory would not break anything: if the move destination directory already contained other annexed data, it would also already contain `.git-annex` so git-annex wouldn't need to do anything.  And if it didn't, git-annex would simply create a new `.git-annex` symlink there.
+
+These `.git-annex` symlinks could either be added to `.gitignore`, or manually/automatically checked in to the current branch - I'm not sure which would be best.  There's also the option of using multiple levels of indirection:
+
+    foo/bar/baz/.git-annex -> ../.git-annex
+    foo/bar/.git-annex -> ../.git-annex
+    foo/.git-annex -> ../.git-annex
+    .git-annex -> .git/annex
+
+I'm not sure whether this would bring any advantages.  It might bring a performance hit due to the kernel having to traverse more symlinks, but without benchmarking it's difficult to say how much.  I'd expect it only to be an issue with a large number of deep directory trees.
+"""]]
diff --git a/doc/bugs/git_rename_detection_on_file_move/comment_7_7f20d0b2f6ed1c34021a135438037306._comment b/doc/bugs/git_rename_detection_on_file_move/comment_7_7f20d0b2f6ed1c34021a135438037306._comment
new file mode 100644
index 0000000000..0a045feb63
--- /dev/null
+++ b/doc/bugs/git_rename_detection_on_file_move/comment_7_7f20d0b2f6ed1c34021a135438037306._comment
@@ -0,0 +1,10 @@
+[[!comment format=mdwn
+ username="http://joey.kitenet.net/"
+ nickname="joey"
+ subject="comment 7"
+ date="2011-12-19T18:22:25Z"
+ content="""
+That seems an excellent idea, also eliminating the need for git annex fix after moving. 
+
+However, I think CVS and svn have taught us the pain associated with a version control system putting something in every subdirectory. Would this pain be worth avoiding the minor pain of needing git annex fix and sometimes being unable to follow renames?
+"""]]
diff --git a/doc/bugs/git_rename_detection_on_file_move/comment_8_6a00500b24ba53248c78e1ffc8d1a591._comment b/doc/bugs/git_rename_detection_on_file_move/comment_8_6a00500b24ba53248c78e1ffc8d1a591._comment
new file mode 100644
index 0000000000..d53022302d
--- /dev/null
+++ b/doc/bugs/git_rename_detection_on_file_move/comment_8_6a00500b24ba53248c78e1ffc8d1a591._comment
@@ -0,0 +1,21 @@
+[[!comment format=mdwn
+ username="http://adamspiers.myopenid.com/"
+ nickname="Adam"
+ subject="comment 8"
+ date="2011-12-20T12:00:11Z"
+ content="""
+Personally I'd rather have working rename detection but I agree it's not 100% ideal to be littering multiple directories like this, so perhaps you could make it optional, e.g. based on a git config setting?
+
+Here are a few more considerations, some in defence of the approach, some against it:
+
+* `.git-annex` is hidden; `CVS/` is not.
+* Unlike `CVS/` and `.svn/`, it's only a symlink, not a directory containing other files.
+* It doesn't contain any data specific to that directory and could easily be regenerated if deleted accidentally or otherwise.
+* If a whole directory containing `.git-annex` was moved within the repository:
+    * git-annex would need to fix up these symlinks if and only if it's moved to a different depth within the tree.
+    * However, if the multi-level indirection approach is used, `.git-annex` in any subdirectory is *always* a symlink to `../.git-annex` so instead you would need to check that all of the new ancestors contain this symlink too, and optionally remove any no longer needed symlinks.
+    * In either case, git-annex already goes to the trouble of fixing symlinks, and if anything, I *think* this approach would reduce the number of symlinks which need checking (right?)
+* find `$git_root/foo -follow`, `diff -r` etc. would traverse into `$git_root/.git/annex`
+
+This last point is the only downside to this approach I can think of which gives me any noticeable cause for concern.  However, people are already use to working around this from CVS and svn days, e.g. `diff -r -x .svn` so I don't think it's anywhere near bad enough to rule it out.
+"""]]
diff --git a/doc/bugs/git_rename_detection_on_file_move/comment_9_75e0973f6d573df615e01005ebcea87d._comment b/doc/bugs/git_rename_detection_on_file_move/comment_9_75e0973f6d573df615e01005ebcea87d._comment
new file mode 100644
index 0000000000..919455bdcc
--- /dev/null
+++ b/doc/bugs/git_rename_detection_on_file_move/comment_9_75e0973f6d573df615e01005ebcea87d._comment
@@ -0,0 +1,9 @@
+[[!comment format=mdwn
+ username="http://joey.kitenet.net/"
+ nickname="joey"
+ subject="comment 9"
+ date="2011-12-20T14:56:12Z"
+ content="""
+Git can follow the rename fine if the file is committed before `git annex fix` (you can git commit -n to see this), so 
+making git-annex pre-commit generate a fixup commit before the staged commit would be one way. Or the other two ways I originally mentioned when writing down this minor issue. I like all those approaches better than .git-annex clutter.
+"""]]
diff --git a/doc/bugs/glacier.mdwn b/doc/bugs/glacier.mdwn
new file mode 100644
index 0000000000..cca9d3923a
--- /dev/null
+++ b/doc/bugs/glacier.mdwn
@@ -0,0 +1,103 @@
+[[!meta title="glacier-cli fails with backtrace"]]
+
+### Please describe the problem.
+
+Amazon Glacier remote doesn't work as expected. It seems like glacier-cli no longer works with git-annex.
+
+### What steps will reproduce the problem?
+
+1. use git-annex from archlinux repo
+2. download and install this with pip: https://github.com/basak/glacier-cli
+3. follow these instructions https://git-annex.branchable.com/tips/using_Amazon_Glacier/
+4. the problems occurs when trying to move or copy content to glacier
+
+### What version of git-annex are you using? On what operating system?
+
+version: 6.20160418-geff8673
+on: Linux  4.10.13-1-ARCH #1 SMP PREEMPT Thu Apr 27 12:15:09 CEST 2017 x86_64 GNU/Linux
+
+
+### Please provide any additional information below.
+
+[[!format sh """
+# If you can, paste a complete transcript of the problem occurring here.
+# If the problem is with the git-annex assistant, paste in .git/annex/daemon.log
+
+# with python 3.6
+% git annex copy file.mp3 --to glacier
+copy file.mp3 (checking glacier...) (to glacier...) 
+98%           0.0 B/s 0sTraceback (most recent call last):
+  File "/home/me/.local/bin/glacier", line 736, in 
+    main()
+  File "/home/me/.local/bin/glacier", line 732, in main
+    App().main()
+  File "/home/me/.local/bin/glacier", line 718, in main
+    self.args.func()
+  File "/home/me/.local/bin/glacier", line 500, in archive_upload
+    file_obj=self.args.file, description=name)
+  File "/usr/lib/python3.6/site-packages/boto/glacier/vault.py", line 178, in create_archive_from_file
+    writer.close()
+  File "/usr/lib/python3.6/site-packages/boto/glacier/writer.py", line 231, in close
+    self.partitioner.flush()
+  File "/usr/lib/python3.6/site-packages/boto/glacier/writer.py", line 79, in flush
+    self._send_part()
+  File "/usr/lib/python3.6/site-packages/boto/glacier/writer.py", line 64, in _send_part
+    data = b''.join(self._buffer)
+TypeError: sequence item 0: expected a bytes-like object, str found
+failed                  
+git-annex: copy: 1 failed
+
+# --------------------------
+# with python2.7
+% git annex copy file.m3 --to glacier
+copy file.mp3 (checking glacier...) (to glacier...) 
+98%           0.0 B/s 0sTraceback (most recent call last):
+  File "/home/me/.local/bin/glacier", line 736, in 
+    main()
+  File "/home/me/.local/bin/glacier", line 732, in main
+    App().main()
+  File "/home/me/.local/bin/glacier", line 718, in main
+    self.args.func()
+  File "/home/me/.local/bin/glacier", line 500, in archive_upload
+    file_obj=self.args.file, description=name)
+  File "/usr/lib/python2.7/site-packages/boto/glacier/vault.py", line 178, in create_archive_from_file
+    writer.close()
+  File "/usr/lib/python2.7/site-packages/boto/glacier/writer.py", line 228, in close
+    self.partitioner.flush()
+  File "/usr/lib/python2.7/site-packages/boto/glacier/writer.py", line 79, in flush
+    self._send_part()
+  File "/usr/lib/python2.7/site-packages/boto/glacier/writer.py", line 75, in _send_part
+    self.send_fn(part)
+  File "/usr/lib/python2.7/site-packages/boto/glacier/writer.py", line 222, in _upload_part
+    self.uploader.upload_part(self.next_part_index, part_data)
+  File "/usr/lib/python2.7/site-packages/boto/glacier/writer.py", line 129, in upload_part
+    content_range, part_data)
+  File "/usr/lib/python2.7/site-packages/boto/glacier/layer1.py", line 1279, in upload_part
+    response_headers=response_headers)
+  File "/usr/lib/python2.7/site-packages/boto/glacier/layer1.py", line 114, in make_request
+    data=data)
+  File "/usr/lib/python2.7/site-packages/boto/connection.py", line 1071, in make_request
+    retry_handler=retry_handler)
+  File "/usr/lib/python2.7/site-packages/boto/connection.py", line 943, in _mexe
+    request.body, request.headers)
+  File "/usr/lib/python2.7/httplib.py", line 1042, in request
+    self._send_request(method, url, body, headers)
+  File "/usr/lib/python2.7/httplib.py", line 1082, in _send_request
+    self.endheaders(body)
+  File "/usr/lib/python2.7/httplib.py", line 1038, in endheaders
+    self._send_output(message_body)
+  File "/usr/lib/python2.7/httplib.py", line 880, in _send_output
+    msg += message_body
+UnicodeDecodeError: 'ascii' codec can't decode byte 0x8c in position 0: ordinal not in range(128)
+failed                  
+git-annex: copy: 1 failed
+
+# End of transcript or log.
+"""]]
+
+### Have you had any luck using git-annex before? (Sometimes we get tired of reading bug reports all day and a lil' positive end note does wonders)
+Yes it's a great tool :). Thanks!
+AWS Glacier integration would be a perfect addition and I could employ git-annex at my company.
+
+> Closing as it's not a bug in git-annex, but glacier-cli. [[done]]
+> --[[Joey]]
diff --git a/doc/bugs/glacier/comment_1_257c401ee74fd48f0b2930f18b82c8c9._comment b/doc/bugs/glacier/comment_1_257c401ee74fd48f0b2930f18b82c8c9._comment
new file mode 100644
index 0000000000..ad6c3e9f67
--- /dev/null
+++ b/doc/bugs/glacier/comment_1_257c401ee74fd48f0b2930f18b82c8c9._comment
@@ -0,0 +1,14 @@
+[[!comment format=mdwn
+ username="joey"
+ subject="""comment 1"""
+ date="2017-05-16T18:19:13Z"
+ content="""
+That looks rather more like a bug in glacier-cli than a bug in git-annex to
+me.
+
+"expected a bytes-like object, str found" has a ton of google hits, 
+and looks to be breakage related to python 3's changes to string handling.
+
+Please file a bug on glacier-cli, and consider making it use python 2 until
+it can be fixed. I'll bet python 2 will avoid this problem.
+"""]]
diff --git a/doc/bugs/glacier/comment_2_5475986eed257c535c21833c23b7661c._comment b/doc/bugs/glacier/comment_2_5475986eed257c535c21833c23b7661c._comment
new file mode 100644
index 0000000000..a6223338b0
--- /dev/null
+++ b/doc/bugs/glacier/comment_2_5475986eed257c535c21833c23b7661c._comment
@@ -0,0 +1,11 @@
+[[!comment format=mdwn
+ username="namsgorf@6b5ce57fbe9dc2a2c65d6817151f107dc22f438c"
+ nickname="namsgorf"
+ avatar="http://cdn.libravatar.org/avatar/b85cacece5b347853dc35d97371b9e0c"
+ subject="comment 2"
+ date="2017-05-21T13:40:07Z"
+ content="""
+IIRC, at the time I wrote glacier-cli, boto (the Python AWS library it uses) had no Python 3 support, so I targeted Python 2 only. I see that boto supports Python 3 now. Patches to glacier-cli for Python 3 support welcome.
+
+Your Python 2 error looks like https://bugs.python.org/issue11898 to me. I know that others have hit the same error in the past. It's a problem between boto and httplib and Python, not a bug in glacier-cli. I don't think it affects Debian or Ubuntu, so you may need to take it up with Arch developers (perhaps Debian or Ubuntu carries a patch?)
+"""]]
diff --git a/doc/bugs/glacier_from_multiple_repos.mdwn b/doc/bugs/glacier_from_multiple_repos.mdwn
new file mode 100644
index 0000000000..3b6bb57f72
--- /dev/null
+++ b/doc/bugs/glacier_from_multiple_repos.mdwn
@@ -0,0 +1,16 @@
+glacier-cli currently relies on a local cache of
+inventory information, and so other git-annexes using the same glacier
+repository are not able to access stuff in it, unless and until
+`glacier vault sync` is run.
+
+An example of this causing trouble is with the assistant. When a file is
+moved into archive/, the assistant that sends it to glacier is able to
+trust that it's in glacier and remove the local copy. But other assistants
+that also have a copy cannot trust that, and so don't remove their copies.
+
+I've discussed with glacier-cli's author making git-annex store enough info
+in its branch to be able to bootstrap glacier-cli to know about a file.
+This seems doable and he had a design; waiting on movement 
+on the glacier-cli side.
+
+[[!tag confirmed]]
diff --git a/doc/bugs/googlemail.mdwn b/doc/bugs/googlemail.mdwn
new file mode 100644
index 0000000000..2fdf6c2da1
--- /dev/null
+++ b/doc/bugs/googlemail.mdwn
@@ -0,0 +1,19 @@
+### Please describe the problem.
+Git-Annex crashes when configuring jabber account with foo.bar@googlemail.com
+
+### What steps will reproduce the problem?
+Configure the Jabber Account with foo.bar@googlemail.com instead of foo.bar@gmail.com. The domain googlemail was used for a long time in germany because of a license issue. 
+
+### What version of git-annex are you using? On what operating system?
+Mac OS X - 10.8.3 Mountain Lion
+
+Version: 4.20130709-g18e5f43 
+
+Build flags: Assistant Webapp Pairing Testsuite S3 WebDAV FsEvents XMPP DNS
+
+
+[[!meta title="xmpp fails to work with googlemail domain"]]
+[[!tag design/assistant]]
+
+> git-annex no longer contains XMPP support, so this bug is "fixed".
+> [[done]] --[[Joey]]
diff --git a/doc/bugs/googlemail/comment_1_5614fa85029f9f97be03cb74899a7099._comment b/doc/bugs/googlemail/comment_1_5614fa85029f9f97be03cb74899a7099._comment
new file mode 100644
index 0000000000..eb3c388118
--- /dev/null
+++ b/doc/bugs/googlemail/comment_1_5614fa85029f9f97be03cb74899a7099._comment
@@ -0,0 +1,19 @@
+[[!comment format=mdwn
+ username="https://www.google.com/accounts/o8/id?id=AItOawm8BAEUyzYhORZmMuocRTk4M-3IumDm5VU"
+ nickname="luciusf0"
+ subject="Bug still valid"
+ date="2014-07-31T08:35:29Z"
+ content="""
+The bug is still valid. A lot of german users had to use the @googlemail.com extension as google couldn't get the gmail domain in Germany. 
+So it might be bothering not just a few people, but a whole country! Now, if that doesn't count ...
+
+	Mac OSX 10.9.4
+	Version: 5.20140717-g5a7d4ff 
+	Build flags: Assistant Webapp Webapp-secure Pairing Testsuite S3 WebDAV FsEvents XMPP DNS Feeds Quvi TDFA CryptoHash
+
+This is the message I get
+
+	Unable to connect to the Jabber server. Maybe you entered the wrong password? (Error message: host xmpp.l.google.com.:5222 failed: AuthenticationFailure (Element {elementName = Name {nameLocalName = \"failure\", nameNamespace = Just \"urn:ietf:params:xml:ns:xmpp-sasl\", namePrefix = Nothing}, elementAttributes = [], elementNodes = [NodeElement (Element {elementName = Name {nameLocalName = \"not-authorized\", nameNamespace = Just \"urn:ietf:params:xml:ns:xmpp-sasl\", namePrefix = Nothing}, elementAttributes = [], elementNodes = []})]}); host alt2.xmpp.l.google.com.:5222 failed: AuthenticationFailure (Element {elementName = Name {nameLocalName = \"failure\", nameNamespace = Just \"urn:ietf:params:xml:ns:xmpp-sasl\", namePrefix = Nothing}, elementAttributes = [], elementNodes = [NodeElement (Element {elementName = Name {nameLocalName = \"not-authorized\", nameNamespace = Just \"urn:ietf:params:xml:ns:xmpp-sasl\", namePrefix = Nothing}, elementAttributes = [], elementNodes = []})]}); host alt1.xmpp.l.google.com.:5222 failed: AuthenticationFailure (Element {elementName = Name {nameLocalName = \"failure\", nameNamespace = Just \"urn:ietf:params:xml:ns:xmpp-sasl\", namePrefix = Nothing}, elementAttributes = [], elementNodes = [NodeElement (Element {elementName = Name {nameLocalName = \"not-authorized\", nameNamespace = Just \"urn:ietf:params:xml:ns:xmpp-sasl\", namePrefix = Nothing}, elementAttributes = [], elementNodes = []})]}); host alt4.xmpp.l.google.com.:5222 failed: AuthenticationFailure (Element {elementName = Name {nameLocalName = \"failure\", nameNamespace = Just \"urn:ietf:params:xml:ns:xmpp-sasl\", namePrefix = Nothing}, elementAttributes = [], elementNodes = [NodeElement (Element {elementName = Name {nameLocalName = \"not-authorized\", nameNamespace = Just \"urn:ietf:params:xml:ns:xmpp-sasl\", namePrefix = Nothing}, elementAttributes = [], elementNodes = []})]}); host alt3.xmpp.l.google.com.:5222 failed: AuthenticationFailure (Element {elementName = Name {nameLocalName = \"failure\", nameNamespace = Just \"urn:ietf:params:xml:ns:xmpp-sasl\", namePrefix = Nothing}, elementAttributes = [], elementNodes = [NodeElement (Element {elementName = Name {nameLocalName = \"not-authorized\", nameNamespace = Just \"urn:ietf:params:xml:ns:xmpp-sasl\", namePrefix = Nothing}, elementAttributes = [], elementNodes = []})]}))
+	
+
+"""]]
diff --git a/doc/bugs/googlemail/comment_2_bdb2b08346673f850709041d2f41be5c._comment b/doc/bugs/googlemail/comment_2_bdb2b08346673f850709041d2f41be5c._comment
new file mode 100644
index 0000000000..1303a2077b
--- /dev/null
+++ b/doc/bugs/googlemail/comment_2_bdb2b08346673f850709041d2f41be5c._comment
@@ -0,0 +1,12 @@
+[[!comment format=mdwn
+ username="http://joeyh.name/"
+ ip="209.250.56.7"
+ subject="comment 2"
+ date="2014-08-12T17:39:09Z"
+ content="""
+AFAICS, xmpp.l.google.com is the correct XMPP server; it's what the SRV record for googlemail.com says to use.
+
+Since it fails with an authentication error, I wonder if google's XMPP is rejecting a user@googlemail.com jid and expects the domain to be @gmail.com or something else. Would that be allowed by the XMPP spec? I don't know.
+
+
+"""]]
diff --git a/doc/bugs/gpg-agent.mdwn b/doc/bugs/gpg-agent.mdwn
new file mode 100644
index 0000000000..df5106a8c6
--- /dev/null
+++ b/doc/bugs/gpg-agent.mdwn
@@ -0,0 +1,5049 @@
+### Please describe the problem.
+I'm running git-annex on OSX 10.9.3. The problem is that during sync with an git-annex remote the system gets flooded with gpg-agent processes which are never stopped, eventually running out of user processes.
+
+[[!tag moreinfo]]
+
+### What steps will reproduce the problem?
+Any synchronization of a lot of files with a git-annex remote.
+
+### What version of git-annex are you using? On what operating system?
+5.20140517-g0aed6d9.
+
+The problem did not appear on any older version.
+
+### Please provide any additional information below.
+
+
+[[!format sh """
+# If you can, paste a complete transcript of the problem occurring here.
+# If the problem is with the git-annex assistant, paste in .git/annex/daemon.log
+[2014-05-25 18:15:53 CEST] main: starting assistant version 5.20140517-g0aed6d9
+[2014-05-25 18:16:02 CEST] main: Syncing with diskstation 
+Everything up-to-date
+[2014-05-25 18:16:16 CEST] UpgradeWatcher: Finished upgrading git-annex to version 5.20140517-g0aed6d9 
+[2014-05-25 18:16:16 CEST] TransferScanner: Syncing with diskstation 
+Already up-to-date.
+Already up-to-date.
+Everything up-to-date
+[2014-05-25 18:16:36 CEST] main: Syncing with diskstation 
+remote: merge git-annex (merging synced/git-annex into git-annex...)        
+remote: ok        
+To joern@annex.diskstation.local:/volume1/homes/joern/Annex.movies
+   95ae98a..16f51d5  git-annex -> synced/git-annex
+
+WORM-s196517551-m1400608062--An Introduction to d3.js - From Scattered to Scatterplot%01_introduction.mp4
+       32768   0%    0.00kB/s    0:00:00
+     3670016   1%    3.40MB/s    0:00:55
+     6750208   3%    3.17MB/s    0:00:58
+     9830400   5%    3.09MB/s    0:00:59
+    12976128   6%    3.03MB/s    0:00:59
+    15990784   8%    2.90MB/s    0:01:00
+    18644992   9%    2.79MB/s    0:01:02
+    21528576  10%    2.75MB/s    0:01:02
+    24543232  12%    2.73MB/s    0:01:01
+    27557888  14%    2.72MB/s    0:01:00
+    30834688  15%    2.85MB/s    0:00:56
+    33980416  17%    2.89MB/s    0:00:55
+    36995072  18%    2.88MB/s    0:00:54
+    40108032  20%    2.91MB/s    0:00:52
+    42991616  21%    2.83MB/s    0:00:52
+    45875200  23%    2.79MB/s    0:00:52
+    48857088  24%    2.78MB/s    0:00:51
+    51740672  26%    2.72MB/s    0:00:51
+    54755328  27%    2.75MB/s    0:00:50
+    57638912  29%    2.74MB/s    0:00:49
+    59998208  30%    2.60MB/s    0:00:51
+    62947328  32%    2.61MB/s    0:00:49
+    65961984  33%    2.60MB/s    0:00:49
+    68845568  35%    2.60MB/s    0:00:48
+    71729152  36%    2.73MB/s    0:00:44
+    74743808  38%    2.74MB/s    0:00:43
+    77627392  39%    2.72MB/s    0:00:42
+    80642048  41%    2.76MB/s    0:00:41
+    83656704  42%    2.78MB/s    0:00:39
+    86540288  44%    2.76MB/s    0:00:38
+    89292800  45%    2.72MB/s    0:00:38
+    91389952  46%    2.51MB/s    0:00:40
+    94404608  48%    2.50MB/s    0:00:39
+    97288192  49%    2.49MB/s    0:00:38
+   100040704  50%    2.49MB/s    0:00:37
+   102924288  52%    2.66MB/s    0:00:34
+   105807872  53%    2.65MB/s    0:00:33
+   108691456  55%    2.67MB/s    0:00:32
+   111443968  56%    2.65MB/s    0:00:31
+   114196480  58%    2.63MB/s    0:00:30
+   117342208  59%    2.66MB/s    0:00:29
+   120225792  61%    2.66MB/s    0:00:27
+   123109376  62%    2.70MB/s    0:00:26
+   125992960  64%    2.73MB/s    0:00:25
+   128745472  65%    2.67MB/s    0:00:24
+   131760128  67%    2.70MB/s    0:00:23
+   134643712  68%    2.70MB/s    0:00:22
+   137396224  69%    2.65MB/s    0:00:21
+   140410880  71%    2.72MB/s    0:00:20
+   143294464  72%    2.70MB/s    0:00:19
+   145915904  74%    2.60MB/s    0:00:19
+   148275200  75%    2.50MB/s    0:00:18
+   150896640  76%    2.41MB/s    0:00:18
+   153780224  78%    2.41MB/s    0:00:17
+   157057024  79%    2.60MB/s    0:00:14
+   160202752  81%    2.79MB/s    0:00:12
+   163217408  83%    2.87MB/s    0:00:11
+   166100992  84%    2.86MB/s    0:00:10
+   168984576  85%    2.75MB/s    0:00:09
+   171737088  87%    2.68MB/s    0:00:09
+   174817280  88%    2.68MB/s    0:00:07
+   177831936  90%    2.71MB/s    0:00:06
+   180846592  92%    2.75MB/s    0:00:05
+   183730176  93%    2.77MB/s    0:00:04
+   186875904  95%    2.80MB/s    0:00:03
+   190021632  96%    2.84MB/s    0:00:02
+   192970752  98%    2.85MB/s    0:00:01
+   195854336  99%    2.87MB/s    0:00:00
+   196517551 100%    2.72MB/s    0:01:08 (xfer#1, to-check=0/1)
+
+sent 196541725 bytes  received 42 bytes  2787826.48 bytes/sec
+total size is 196517551  speedup is 1.00
+[2014-05-25 18:17:58 CEST] Transferrer: Uploaded 01_introduction.mp4
+[2014-05-25 18:17:58 CEST] Pusher: Syncing with diskstation 
+
+WORM-s57838296-m1400607924--An Introduction to d3.js - From Scattered to Scatterplot%02_setting-up.mp4
+       32768   0%    0.00kB/s    0:00:00
+     4030464   6%    3.75MB/s    0:00:14
+     6782976  11%    3.15MB/s    0:00:15
+     9535488  16%    2.96MB/s    0:00:15
+    12288000  21%    2.88MB/s    0:00:15
+    15040512  26%    2.58MB/s    0:00:16
+    17793024  30%    2.60MB/s    0:00:15
+    20545536  35%    2.61MB/s    0:00:13
+    23560192  40%    2.66MB/s    0:00:12
+    26705920  46%    2.76MB/s    0:00:11
+    29720576  51%    2.80MB/s    0:00:09
+    32604160  56%    2.81MB/s    0:00:08
+remote: merge git-annex (merging synced/git-annex into git-annex...)        
+remote: (Recording state in git...)        
+remote: ok        
+To joern@annex.diskstation.local:/volume1/homes/joern/Annex.movies
+   16f51d5..b142966  git-annex -> synced/git-annex
+    35356672  61%    2.76MB/s    0:00:07
+    38371328  66%    2.73MB/s    0:00:06
+[2014-05-25 18:18:14 CEST] RemoteControl: Syncing with diskstation 
+    41648128  72%    2.79MB/s    0:00:05
+    44761088  77%    2.86MB/s    0:00:04
+    47841280  82%    2.92MB/s    0:00:03
+    50855936  87%    2.92MB/s    0:00:02
+    53870592  93%    2.86MB/s    0:00:01
+    56885248  98%    2.81MB/s    0:00:00
+    57838296 100%    2.80MB/s    0:00:19 (xfer#1, to-check=0/1)
+From annex.diskstation.local:/volume1/homes/joern/Annex.movies
+   95ae98a..5809ceb  git-annex  -> diskstation/git-annex
+error: Ref refs/remotes/diskstation/synced/git-annex is at b1429660ab35f95d5bc54e4d8b4f39a19fe66998 but expected 16f51d51d23bf026556435a5cea88488ae0ab108
+ ! 16f51d5..b142966  synced/git-annex -> diskstation/synced/git-annex  (unable to update local ref)
+
+sent 57845539 bytes  received 42 bytes  2690492.14 bytes/sec
+total size is 57838296  speedup is 1.00
+[2014-05-25 18:18:22 CEST] Transferrer: Uploaded 02_setting-up.mp4
+[2014-05-25 18:18:23 CEST] Pusher: Syncing with diskstation 
+
+WORM-s149069061-m1401033118--An Introduction to d3.js - From Scattered to Scatterplot%03_selecting-and-creating-elements.mp4
+       32768   0%    0.00kB/s    0:00:00
+     4063232   2%    3.60MB/s    0:00:39
+remote: merge git-annex      6815744   4%    3.12MB/s    0:00:44
+     9568256   6%    2.96MB/s    0:00:46
+(merging synced/git-annex into git-annex...)        
+remote: (Recording state in git...)        
+remote: ok        
+[2014-05-25 18:18:28 CEST] RemoteControl: Syncing with diskstation 
+To joern@annex.diskstation.local:/volume1/homes/joern/Annex.movies
+   b142966..7713a32  git-annex -> synced/git-annex
+    12582912   8%    2.93MB/s    0:00:45
+    15466496  10%    2.70MB/s    0:00:48
+From annex.diskstation.local:/volume1/homes/joern/Annex.movies
+   5809ceb..19d05e5  git-annex  -> diskstation/git-annex
+error: Ref refs/remotes/diskstation/synced/git-annex is at 7713a32f85cf1d993948f15859f0d035d63aa6e0 but expected b1429660ab35f95d5bc54e4d8b4f39a19fe66998
+ ! b142966..7713a32  synced/git-annex -> diskstation/synced/git-annex  (unable to update local ref)
+    18448384  12%    2.75MB/s    0:00:46
+    21594112  14%    2.83MB/s    0:00:44
+    24477696  16%    2.80MB/s    0:00:43
+    27492352  18%    2.84MB/s    0:00:41
+    30375936  20%    2.80MB/s    0:00:41
+    33587200  22%    2.83MB/s    0:00:39
+    36601856  24%    2.85MB/s    0:00:38
+    39616512  26%    2.82MB/s    0:00:37
+    42041344  28%    2.73MB/s    0:00:38
+    45056000  30%    2.67MB/s    0:00:37
+    48037888  32%    2.66MB/s    0:00:37
+    50855936  34%    2.64MB/s    0:00:36
+    52756480  35%    2.52MB/s    0:00:37
+    54853632  36%    2.26MB/s    0:00:40
+    56950784  38%    2.02MB/s    0:00:44
+    58916864  39%    1.81MB/s    0:00:48
+    61669376  41%    2.00MB/s    0:00:42
+    64552960  43%    2.23MB/s    0:00:36
+    67567616  45%    2.48MB/s    0:00:32
+    70713344  47%    2.77MB/s    0:00:27
+    73859072  49%    2.84MB/s    0:00:25
+    76873728  51%    2.85MB/s    0:00:24
+    79888384  53%    2.85MB/s    0:00:23
+    82903040  55%    2.81MB/s    0:00:22
+    85786624  57%    2.77MB/s    0:00:22
+    88571904  59%    2.73MB/s    0:00:21
+    91422720  61%    2.70MB/s    0:00:20
+    94502912  63%    2.73MB/s    0:00:19
+    97288192  65%    2.68MB/s    0:00:18
+   100335616  67%    2.72MB/s    0:00:17
+   103350272  69%    2.77MB/s    0:00:16
+   106364928  71%    2.73MB/s    0:00:15
+   109379584  73%    2.82MB/s    0:00:13
+   112394240  75%    2.79MB/s    0:00:12
+   115376128  77%    2.76MB/s    0:00:11
+   118390784  79%    2.76MB/s    0:00:10
+   121536512  81%    2.80MB/s    0:00:09
+   124682240  83%    2.85MB/s    0:00:08
+   127827968  85%    2.90MB/s    0:00:07
+   130842624  87%    2.91MB/s    0:00:06
+   133988352  89%    2.91MB/s    0:00:05
+   137003008  91%    2.88MB/s    0:00:04
+   140017664  93%    2.86MB/s    0:00:03
+   142901248  95%    2.85MB/s    0:00:02
+   145588224  97%    2.71MB/s    0:00:01
+   148537344  99%    2.71MB/s    0:00:00
+   149069061 100%    2.71MB/s    0:00:52 (xfer#1, to-check=0/1)
+
+sent 149087462 bytes  received 42 bytes  2786682.32 bytes/sec
+total size is 149069061  speedup is 1.00
+[2014-05-25 18:19:18 CEST] Transferrer: Uploaded 03_select..ments.mp4
+[2014-05-25 18:19:18 CEST] Pusher: Syncing with diskstation 
+
+WORM-s36798210-m1400608746--An Introduction to d3.js - From Scattered to Scatterplot%04_exercise-1.mp4
+       32768   0%    0.00kB/s    0:00:00
+     4227072  11%    3.89MB/s    0:00:08
+     7143424  19%    3.32MB/s    0:00:08
+    10158080  27%    3.15MB/s    0:00:08
+remote: merge git-annex     13172736  35%    3.07MB/s    0:00:07
+[2014-05-25 18:19:24 CEST] RemoteControl: Syncing with diskstation 
+(merging synced/git-annex into git-annex...)        
+remote: (Recording state in git...)        
+remote: ok        
+To joern@annex.diskstation.local:/volume1/homes/joern/Annex.movies
+   7713a32..c892127  git-annex -> synced/git-annex
+    16187392  43%    2.81MB/s    0:00:07
+    19136512  52%    2.83MB/s    0:00:06
+    22282240  60%    2.88MB/s    0:00:04
+From annex.diskstation.local:/volume1/homes/joern/Annex.movies
+   19d05e5..928f25d  git-annex  -> diskstation/git-annex
+error: Ref refs/remotes/diskstation/synced/git-annex is at c8921270e51c1eb7f5f88a288adbd778711d0ff0 but expected 7713a32f85cf1d993948f15859f0d035d63aa6e0
+ ! 7713a32..c892127  synced/git-annex -> diskstation/synced/git-annex  (unable to update local ref)
+    25296896  68%    2.88MB/s    0:00:03
+    28442624  77%    2.88MB/s    0:00:02
+    31457280  85%    2.88MB/s    0:00:01
+    34603008  94%    2.84MB/s    0:00:00
+    36798210 100%    2.94MB/s    0:00:11 (xfer#1, to-check=0/1)
+
+sent 36802881 bytes  received 42 bytes  2538132.62 bytes/sec
+total size is 36798210  speedup is 1.00
+[2014-05-25 18:19:32 CEST] Transferrer: Uploaded 04_exercise-1.mp4
+[2014-05-25 18:19:32 CEST] Pusher: Syncing with diskstation 
+
+WORM-s272759172-m1400608447--An Introduction to d3.js - From Scattered to Scatterplot%05_svg.mp4
+       32768   0%    0.00kB/s    0:00:00
+     4227072   1%    3.97MB/s    0:01:06
+     7241728   2%    3.39MB/s    0:01:16
+    10256384   3%    3.17MB/s    0:01:20
+remote: merge git-annex     13402112   4%    3.11MB/s    0:01:21
+    16416768   6%    2.83MB/s    0:01:28
+[2014-05-25 18:19:38 CEST] RemoteControl: Syncing with diskstation 
+(merging synced/git-annex into git-annex...)        
+remote: (Recording state in git...)        
+remote: ok        
+To joern@annex.diskstation.local:/volume1/homes/joern/Annex.movies
+   c892127..ff739c0  git-annex -> synced/git-annex
+    19431424   7%    2.83MB/s    0:01:27
+    22446080   8%    2.85MB/s    0:01:25
+From annex.diskstation.local:/volume1/homes/joern/Annex.movies
+   928f25d..383d246  git-annex  -> diskstation/git-annex
+error: Ref refs/remotes/diskstation/synced/git-annex is at ff739c08bd8881b7d1c3fd6c305b0d80b3213b0d but expected c8921270e51c1eb7f5f88a288adbd778711d0ff0
+ ! c892127..ff739c0  synced/git-annex -> diskstation/synced/git-annex  (unable to update local ref)
+    25591808   9%    2.83MB/s    0:01:25
+    28606464  10%    2.81MB/s    0:01:24
+    31621120  11%    2.82MB/s    0:01:23
+    34766848  12%    2.83MB/s    0:01:22
+    37781504  13%    2.83MB/s    0:01:21
+    40927232  15%    2.87MB/s    0:01:18
+    44072960  16%    2.89MB/s    0:01:17
+    47087616  17%    2.89MB/s    0:01:16
+    50397184  18%    2.95MB/s    0:01:13
+    53411840  19%    2.93MB/s    0:01:13
+    56295424  20%    2.89MB/s    0:01:13
+    59146240  21%    2.84MB/s    0:01:13
+    62160896  22%    2.78MB/s    0:01:13
+    65306624  23%    2.82MB/s    0:01:11
+    68321280  25%    2.84MB/s    0:01:10
+    71335936  26%    2.86MB/s    0:01:08
+    74350592  27%    2.85MB/s    0:01:07
+    77365248  28%    2.80MB/s    0:01:08
+    80379904  29%    2.80MB/s    0:01:07
+    83394560  30%    2.82MB/s    0:01:05
+    86540288  31%    2.84MB/s    0:01:04
+    89554944  32%    2.85MB/s    0:01:02
+    92569600  33%    2.86MB/s    0:01:01
+    95453184  34%    2.81MB/s    0:01:01
+    98336768  36%    2.76MB/s    0:01:01
+   101220352  37%    2.74MB/s    0:01:01
+   104103936  38%    2.70MB/s    0:01:00
+   107118592  39%    2.75MB/s    0:00:58
+   110002176  40%    2.74MB/s    0:00:58
+   113016832  41%    2.75MB/s    0:00:56
+   115113984  42%    2.57MB/s    0:00:59
+   117342208  43%    2.37MB/s    0:01:04
+   120356864  44%    2.40MB/s    0:01:02
+   123371520  45%    2.41MB/s    0:01:00
+   126386176  46%    2.61MB/s    0:00:54
+   129335296  47%    2.77MB/s    0:00:50
+   132481024  48%    2.80MB/s    0:00:48
+   135626752  49%    2.83MB/s    0:00:47
+   138772480  50%    2.86MB/s    0:00:45
+   141656064  51%    2.88MB/s    0:00:44
+   144572416  53%    2.83MB/s    0:00:44
+   147456000  54%    2.78MB/s    0:00:43
+   150732800  55%    2.84MB/s    0:00:41
+   153845760  56%    2.89MB/s    0:00:40
+   156860416  57%    2.88MB/s    0:00:39
+   159875072  58%    2.89MB/s    0:00:38
+   162889728  59%    2.83MB/s    0:00:37
+   165904384  60%    2.79MB/s    0:00:37
+   168919040  61%    2.80MB/s    0:00:36
+   172064768  63%    2.82MB/s    0:00:34
+   175079424  64%    2.79MB/s    0:00:34
+   178094080  65%    2.79MB/s    0:00:33
+   180977664  66%    2.77MB/s    0:00:32
+   183861248  67%    2.74MB/s    0:00:31
+   186875904  68%    2.76MB/s    0:00:30
+   190152704  69%    2.81MB/s    0:00:28
+   193298432  70%    2.86MB/s    0:00:27
+   196313088  71%    2.87MB/s    0:00:26
+   199458816  73%    2.92MB/s    0:00:24
+   202473472  74%    2.86MB/s    0:00:23
+   205455360  75%    2.82MB/s    0:00:23
+   208207872  76%    2.76MB/s    0:00:22
+   211091456  77%    2.69MB/s    0:00:22
+   214237184  78%    2.74MB/s    0:00:20
+   217251840  79%    2.77MB/s    0:00:19
+   220266496  80%    2.81MB/s    0:00:18
+   223019008  81%    2.77MB/s    0:00:17
+   225247232  82%    2.54MB/s    0:00:18
+   228261888  83%    2.54MB/s    0:00:17
+   231145472  84%    2.52MB/s    0:00:16
+   234160128  85%    2.58MB/s    0:00:14
+   237043712  86%    2.74MB/s    0:00:12
+   240091136  88%    2.76MB/s    0:00:11
+   243073024  89%    2.78MB/s    0:00:10
+   246087680  90%    2.79MB/s    0:00:09
+   248971264  91%    2.81MB/s    0:00:08
+   251723776  92%    2.71MB/s    0:00:07
+   254738432  93%    2.73MB/s    0:00:06
+   257884160  94%    2.75MB/s    0:00:05
+   261029888  95%    2.80MB/s    0:00:04
+   263913472  96%    2.85MB/s    0:00:03
+   266928128  97%    2.87MB/s    0:00:01
+   269942784  98%    2.83MB/s    0:00:00
+   272759172 100%    2.80MB/s    0:01:32 (xfer#1, to-check=0/1)
+
+sent 272792641 bytes  received 42 bytes  2886695.06 bytes/sec
+total size is 272759172  speedup is 1.00
+[2014-05-25 18:21:07 CEST] Transferrer: Uploaded 05_svg.mp4
+[2014-05-25 18:21:07 CEST] Pusher: Syncing with diskstation 
+
+WORM-s29712591-m1400608733--An Introduction to d3.js - From Scattered to Scatterplot%06_exercise-2.mp4
+       32768   0%    0.00kB/s    0:00:00
+     4259840  14%    4.00MB/s    0:00:06
+     7274496  24%    3.44MB/s    0:00:06
+    10289152  34%    3.24MB/s    0:00:05
+remote: merge git-annex     13434880  45%    3.17MB/s    0:00:05
+    16678912  56%    2.92MB/s    0:00:04
+[2014-05-25 18:21:13 CEST] RemoteControl: Syncing with diskstation 
+(merging synced/git-annex into git-annex...)        
+remote: (Recording state in git...)        
+remote: ok        
+To joern@annex.diskstation.local:/volume1/homes/joern/Annex.movies
+   ff739c0..07153a2  git-annex -> synced/git-annex
+    19693568  66%    2.89MB/s    0:00:03
+    22740992  76%    2.86MB/s    0:00:02
+From annex.diskstation.local:/volume1/homes/joern/Annex.movies
+   383d246..1749643  git-annex  -> diskstation/git-annex
+error: Ref refs/remotes/diskstation/synced/git-annex is at 07153a2d49264b9792593fc016c00499ce87f70b but expected ff739c08bd8881b7d1c3fd6c305b0d80b3213b0d
+ ! ff739c0..07153a2  synced/git-annex -> diskstation/synced/git-annex  (unable to update local ref)
+    25624576  86%    2.81MB/s    0:00:01
+    28639232  96%    2.75MB/s    0:00:00
+    29712591 100%    2.94MB/s    0:00:09 (xfer#1, to-check=0/1)
+
+sent 29716398 bytes  received 42 bytes  2584038.26 bytes/sec
+total size is 29712591  speedup is 1.00
+[2014-05-25 18:21:18 CEST] Transferrer: Uploaded 06_exercise-2.mp4
+[2014-05-25 18:21:18 CEST] Pusher: Syncing with diskstation 
+
+WORM-s276749404-m1400608181--An Introduction to d3.js - From Scattered to Scatterplot%07_binding-data-and-creating-elements-from-data.mp4
+       32768   0%    0.00kB/s    0:00:00
+     4030464   1%    3.60MB/s    0:01:13
+     7307264   2%    3.29MB/s    0:01:19
+remote: merge git-annex     10190848   3%    3.11MB/s    0:01:23
+    13336576   4%    3.06MB/s    0:01:24
+[2014-05-25 18:21:24 CEST] RemoteControl: Syncing with diskstation 
+(merging synced/git-annex into git-annex...)        
+remote: (Recording state in git...)        
+remote: ok        
+To joern@annex.diskstation.local:/volume1/homes/joern/Annex.movies
+   07153a2..0a42aa9  git-annex -> synced/git-annex
+    16482304   5%    2.89MB/s    0:01:27
+    19496960   7%    2.85MB/s    0:01:28
+    22642688   8%    2.89MB/s    0:01:25
+From annex.diskstation.local:/volume1/homes/joern/Annex.movies
+   1749643..e6798ff  git-annex  -> diskstation/git-annex
+error: Ref refs/remotes/diskstation/synced/git-annex is at 0a42aa9412e0a0f03e8edaa7a37eb0d6273bc4d9 but expected 07153a2d49264b9792593fc016c00499ce87f70b
+ ! 07153a2..0a42aa9  synced/git-annex -> diskstation/synced/git-annex  (unable to update local ref)
+    25788416   9%    2.88MB/s    0:01:25
+    28803072  10%    2.84MB/s    0:01:25
+    31817728  11%    2.84MB/s    0:01:24
+    34832384  12%    2.83MB/s    0:01:23
+    37978112  13%    2.84MB/s    0:01:22
+    40828928  14%    2.82MB/s    0:01:21
+    43974656  15%    2.83MB/s    0:01:20
+    46956544  16%    2.84MB/s    0:01:18
+    50233344  18%    2.88MB/s    0:01:16
+    53248000  19%    2.91MB/s    0:01:15
+    56262656  20%    2.88MB/s    0:01:14
+    59146240  21%    2.84MB/s    0:01:14
+    62291968  22%    2.81MB/s    0:01:14
+    65699840  23%    2.89MB/s    0:01:11
+    68812800  24%    2.91MB/s    0:01:09
+    71892992  25%    2.97MB/s    0:01:07
+    75038720  27%    2.95MB/s    0:01:06
+    78020608  28%    2.87MB/s    0:01:07
+    81166336  29%    2.88MB/s    0:01:06
+    84049920  30%    2.83MB/s    0:01:06
+    86769664  31%    2.76MB/s    0:01:07
+    89653248  32%    2.73MB/s    0:01:06
+    92536832  33%    2.66MB/s    0:01:07
+    95420416  34%    2.66MB/s    0:01:06
+    98304000  35%    2.69MB/s    0:01:04
+   101187584  36%    2.69MB/s    0:01:03
+   104202240  37%    2.75MB/s    0:01:01
+   107216896  38%    2.76MB/s    0:00:59
+   110100480  39%    2.77MB/s    0:00:58
+   113016832  40%    2.78MB/s    0:00:57
+   115867648  41%    2.74MB/s    0:00:57
+   118620160  42%    2.71MB/s    0:00:57
+   121503744  43%    2.69MB/s    0:00:56
+   124452864  44%    2.67MB/s    0:00:55
+   127467520  46%    2.70MB/s    0:00:53
+   130220032  47%    2.69MB/s    0:00:53
+   132972544  48%    2.68MB/s    0:00:52
+   135987200  49%    2.70MB/s    0:00:50
+   138870784  50%    2.68MB/s    0:00:50
+   141754368  51%    2.72MB/s    0:00:48
+   144474112  52%    2.71MB/s    0:00:47
+   147357696  53%    2.68MB/s    0:00:47
+   150372352  54%    2.70MB/s    0:00:45
+   153387008  55%    2.71MB/s    0:00:44
+   156401664  56%    2.76MB/s    0:00:42
+   159416320  57%    2.79MB/s    0:00:41
+   162299904  58%    2.78MB/s    0:00:40
+   165445632  59%    2.80MB/s    0:00:38
+   168460288  60%    2.82MB/s    0:00:37
+   171343872  61%    2.79MB/s    0:00:36
+   174227456  62%    2.78MB/s    0:00:36
+   177373184  64%    2.78MB/s    0:00:34
+   180387840  65%    2.79MB/s    0:00:33
+   183533568  66%    2.86MB/s    0:00:31
+   186548224  67%    2.87MB/s    0:00:30
+   189562880  68%    2.81MB/s    0:00:30
+   192446464  69%    2.75MB/s    0:00:29
+   195461120  70%    2.71MB/s    0:00:29
+   198475776  71%    2.73MB/s    0:00:28
+   201359360  72%    2.74MB/s    0:00:26
+   204308480  73%    2.78MB/s    0:00:25
+   207060992  74%    2.72MB/s    0:00:24
+   210206720  75%    2.77MB/s    0:00:23
+   212828160  76%    2.64MB/s    0:00:23
+   215318528  77%    2.54MB/s    0:00:23
+   218333184  78%    2.58MB/s    0:00:22
+   221347840  79%    2.53MB/s    0:00:21
+   224362496  81%    2.67MB/s    0:00:19
+   227377152  82%    2.77MB/s    0:00:17
+   230129664  83%    2.74MB/s    0:00:16
+   233078784  84%    2.74MB/s    0:00:15
+   236093440  85%    2.74MB/s    0:00:14
+   239108096  86%    2.76MB/s    0:00:13
+   242122752  87%    2.83MB/s    0:00:11
+   245006336  88%    2.81MB/s    0:00:11
+   247824384  89%    2.77MB/s    0:00:10
+   250707968  90%    2.70MB/s    0:00:09
+   253460480  91%    2.63MB/s    0:00:08
+   256606208  92%    2.69MB/s    0:00:07
+   259620864  93%    2.73MB/s    0:00:06
+   262504448  94%    2.75MB/s    0:00:05
+   264994816  95%    2.63MB/s    0:00:04
+   266960896  96%    2.37MB/s    0:00:04
+   268795904  97%    2.11MB/s    0:00:03
+   271155200  97%    1.99MB/s    0:00:02
+   274038784  99%    2.14MB/s    0:00:01
+   276749404 100%    2.75MB/s    0:01:36 (xfer#1, to-check=0/1)
+
+sent 276783402 bytes  received 42 bytes  2838804.55 bytes/sec
+total size is 276749404  speedup is 1.00
+[2014-05-25 18:22:56 CEST] Transferrer: Uploaded 07_bindin..-data.mp4
+[2014-05-25 18:22:56 CEST] Pusher: Syncing with diskstation 
+
+WORM-s58893321-m1400608603--An Introduction to d3.js - From Scattered to Scatterplot%08_exercise-3.mp4
+       32768   0%    0.00kB/s    0:00:00
+     4227072   7%    3.98MB/s    0:00:13
+     7241728  12%    3.38MB/s    0:00:14
+    10125312  17%    3.14MB/s    0:00:15
+    13139968  22%    3.05MB/s    0:00:14
+    16023552  27%    2.74MB/s    0:00:15
+remote: merge git-annex     18907136  32%    2.72MB/s    0:00:14
+[2014-05-25 18:23:04 CEST] RemoteControl: Syncing with diskstation 
+(merging synced/git-annex into git-annex...)        
+remote: (Recording state in git...)        
+remote: ok        
+To joern@annex.diskstation.local:/volume1/homes/joern/Annex.movies
+   0a42aa9..961714e  git-annex -> synced/git-annex
+    21790720  37%    2.72MB/s    0:00:13
+    24870912  42%    2.72MB/s    0:00:12
+    27754496  47%    2.72MB/s    0:00:11
+    30638080  52%    2.70MB/s    0:00:10
+From annex.diskstation.local:/volume1/homes/joern/Annex.movies
+   e6798ff..c074ab0  git-annex  -> diskstation/git-annex
+error: Ref refs/remotes/diskstation/synced/git-annex is at 961714e5af3653435346ed0f7ae7e19efb899c6f but expected 0a42aa9412e0a0f03e8edaa7a37eb0d6273bc4d9
+ ! 0a42aa9..961714e  synced/git-annex -> diskstation/synced/git-annex  (unable to update local ref)
+    33980416  57%    2.81MB/s    0:00:08
+    36864000  62%    2.78MB/s    0:00:07
+    39616512  67%    2.76MB/s    0:00:06
+    42369024  71%    2.75MB/s    0:00:05
+    45252608  76%    2.64MB/s    0:00:05
+    48136192  81%    2.64MB/s    0:00:03
+    51150848  86%    2.67MB/s    0:00:02
+    53903360  91%    2.65MB/s    0:00:01
+    56655872  96%    2.63MB/s    0:00:00
+    58893321 100%    2.76MB/s    0:00:20 (xfer#1, to-check=0/1)
+
+sent 58900692 bytes  received 42 bytes  2617810.40 bytes/sec
+total size is 58893321  speedup is 1.00
+[2014-05-25 18:23:18 CEST] Transferrer: Uploaded 08_exercise-3.mp4
+[2014-05-25 18:23:18 CEST] Pusher: Syncing with diskstation 
+
+WORM-s134947390-m1400608565--An Introduction to d3.js - From Scattered to Scatterplot%09_transitions.mp4
+       32768   0%    0.00kB/s    0:00:00
+     4161536   3%    3.82MB/s    0:00:33
+     7045120   5%    3.24MB/s    0:00:38
+remote: merge git-annex      9928704   7%    3.04MB/s    0:00:40
+    13205504   9%    3.05MB/s    0:00:38
+(merging synced/git-annex into git-annex...)        
+remote: (Recording state in git...)        
+remote: ok        
+To joern@annex.diskstation.local:/volume1/homes/joern/Annex.movies
+   961714e..dbdf944  git-annex -> synced/git-annex
+[2014-05-25 18:23:24 CEST] RemoteControl: Syncing with diskstation 
+    16220160  12%    2.79MB/s    0:00:41
+    19103744  14%    2.80MB/s    0:00:40
+    21856256  16%    2.79MB/s    0:00:39
+    24576000  18%    2.66MB/s    0:00:40
+From annex.diskstation.local:/volume1/homes/joern/Annex.movies
+   c074ab0..1e5f9f1  git-annex  -> diskstation/git-annex
+    27459584  20%    2.64MB/s    0:00:39
+    30474240  22%    2.68MB/s    0:00:38
+    33357824  24%    2.70MB/s    0:00:36
+    36372480  26%    2.76MB/s    0:00:34
+    39256064  29%    2.76MB/s    0:00:33
+    42139648  31%    2.73MB/s    0:00:33
+    45023232  33%    2.73MB/s    0:00:32
+    47775744  35%    2.66MB/s    0:00:31
+    50659328  37%    2.66MB/s    0:00:30
+    53542912  39%    2.65MB/s    0:00:29
+    56557568  41%    2.66MB/s    0:00:28
+    59310080  43%    2.67MB/s    0:00:27
+    62029824  45%    2.64MB/s    0:00:26
+    64782336  48%    2.62MB/s    0:00:26
+    67665920  50%    2.61MB/s    0:00:25
+    70680576  52%    2.64MB/s    0:00:23
+    73695232  54%    2.70MB/s    0:00:22
+    76709888  56%    2.77MB/s    0:00:20
+    79593472  58%    2.76MB/s    0:00:19
+    82608128  61%    2.79MB/s    0:00:18
+    85622784  63%    2.81MB/s    0:00:17
+    88506368  65%    2.76MB/s    0:00:16
+    91619328  67%    2.81MB/s    0:00:15
+    94502912  70%    2.78MB/s    0:00:14
+    97255424  72%    2.72MB/s    0:00:13
+   100270080  74%    2.76MB/s    0:00:12
+   102760448  76%    2.61MB/s    0:00:12
+   105349120  78%    2.54MB/s    0:00:11
+   108232704  80%    2.57MB/s    0:00:10
+   111247360  82%    2.56MB/s    0:00:09
+   114262016  84%    2.68MB/s    0:00:07
+   117145600  86%    2.76MB/s    0:00:06
+   120029184  88%    2.75MB/s    0:00:05
+   123043840  91%    2.76MB/s    0:00:04
+   125927424  93%    2.74MB/s    0:00:03
+   128942080  95%    2.76MB/s    0:00:02
+   131694592  97%    2.74MB/s    0:00:01
+   134447104  99%    2.64MB/s    0:00:00
+   134947390 100%    2.72MB/s    0:00:47 (xfer#1, to-check=0/1)
+
+sent 134964047 bytes  received 42 bytes  2726547.25 bytes/sec
+total size is 134947390  speedup is 1.00
+[2014-05-25 18:24:08 CEST] Transferrer: Uploaded 09_transitions.mp4
+[2014-05-25 18:24:08 CEST] Pusher: Syncing with diskstation 
+
+WORM-s106802484-m1399168180--An Introduction to d3.js - From Scattered to Scatterplot%10_scales-2.mp4
+       32768   0%    0.00kB/s    0:00:00
+     4128768   3%    3.75MB/s    0:00:26
+     7143424   6%    3.27MB/s    0:00:29
+    10027008   9%    3.10MB/s    0:00:30
+remote: merge git-annex     13008896  12%    3.01MB/s    0:00:30
+(merging synced/git-annex into git-annex...)        
+remote: (Recording state in git...)        
+remote: ok        
+To joern@annex.diskstation.local:/volume1/homes/joern/Annex.movies
+[2014-05-25 18:24:13 CEST] RemoteControl: Syncing with diskstation 
+   dbdf944..cef70b3  git-annex -> synced/git-annex
+    16154624  15%    2.81MB/s    0:00:31
+    19169280  17%    2.81MB/s    0:00:30
+    22183936  20%    2.83MB/s    0:00:29
+From annex.diskstation.local:/volume1/homes/joern/Annex.movies
+   1e5f9f1..af18e69  git-annex  -> diskstation/git-annex
+error: Ref refs/remotes/diskstation/synced/git-annex is at cef70b30ddce59c2b22b370b25420da2fa7e2b49 but expected dbdf94438b7e48971a594c7647721f4821c8cff2
+ ! dbdf944..cef70b3  synced/git-annex -> diskstation/synced/git-annex  (unable to update local ref)
+    24936448  23%    2.78MB/s    0:00:28
+    27688960  25%    2.70MB/s    0:00:28
+    30834688  28%    2.73MB/s    0:00:27
+    32899072  30%    2.50MB/s    0:00:28
+    35749888  33%    2.53MB/s    0:00:27
+    38764544  36%    2.56MB/s    0:00:25
+    41779200  39%    2.53MB/s    0:00:25
+    44072960  41%    2.60MB/s    0:00:23
+    47415296  44%    2.71MB/s    0:00:21
+    50036736  46%    2.65MB/s    0:00:20
+    52789248  49%    2.59MB/s    0:00:20
+    55672832  52%    2.72MB/s    0:00:18
+    58687488  54%    2.64MB/s    0:00:17
+    61571072  57%    2.69MB/s    0:00:16
+    64585728  60%    2.75MB/s    0:00:14
+    67207168  62%    2.65MB/s    0:00:14
+    69304320  64%    2.44MB/s    0:00:14
+    72318976  67%    2.46MB/s    0:00:13
+    75071488  70%    2.42MB/s    0:00:12
+    78020608  73%    2.53MB/s    0:00:11
+    80904192  75%    2.71MB/s    0:00:09
+    83787776  78%    2.68MB/s    0:00:08
+    86540288  81%    2.66MB/s    0:00:07
+    89423872  83%    2.64MB/s    0:00:06
+    92372992  86%    2.67MB/s    0:00:05
+    95256576  89%    2.68MB/s    0:00:04
+    98140160  91%    2.72MB/s    0:00:03
+   101154816  94%    2.74MB/s    0:00:02
+   104169472  97%    2.73MB/s    0:00:00
+   106802484 100%    2.70MB/s    0:00:37 (xfer#1, to-check=0/1)
+
+sent 106815702 bytes  received 42 bytes  2704196.05 bytes/sec
+total size is 106802484  speedup is 1.00
+[2014-05-25 18:24:47 CEST] Transferrer: Uploaded 10_scales-2.mp4
+[2014-05-25 18:24:47 CEST] Pusher: Syncing with diskstation 
+
+WORM-s106802484-m1399168180--An Introduction to d3.js - From Scattered to Scatterplot%10_scales.mp4
+       32768   0%    0.00kB/s    0:00:00
+     3964928   3%    3.74MB/s    0:00:26
+     6848512   6%    3.19MB/s    0:00:30
+     9994240   9%    3.08MB/s    0:00:30
+remote: merge git-annex     12877824  12%    2.97MB/s    0:00:30
+    15892480  14%    2.74MB/s    0:00:32
+(merging synced/git-annex into git-annex...)        
+remote: (Recording state in git...)        
+remote: ok        
+To joern@annex.diskstation.local:/volume1/homes/joern/Annex.movies
+[2014-05-25 18:24:53 CEST] RemoteControl: Syncing with diskstation 
+   cef70b3..bc92e4a  git-annex -> synced/git-annex
+    18644992  17%    2.71MB/s    0:00:31
+    20873216  19%    2.51MB/s    0:00:33
+    23429120  21%    2.39MB/s    0:00:34
+From annex.diskstation.local:/volume1/homes/joern/Annex.movies
+   af18e69..7b0c500  git-annex  -> diskstation/git-annex
+error: Ref refs/remotes/diskstation/synced/git-annex is at bc92e4a867338760dec749af77a9262591b9c43a but expected cef70b30ddce59c2b22b370b25420da2fa7e2b49
+ ! cef70b3..bc92e4a  synced/git-annex -> diskstation/synced/git-annex  (unable to update local ref)
+    27099136  25%    2.52MB/s    0:00:30
+    29982720  28%    2.56MB/s    0:00:29
+    32866304  30%    2.72MB/s    0:00:26
+    35880960  33%    2.88MB/s    0:00:24
+    38633472  36%    2.69MB/s    0:00:24
+    41517056  38%    2.68MB/s    0:00:23
+    44662784  41%    2.74MB/s    0:00:22
+    47546368  44%    2.72MB/s    0:00:21
+    50298880  47%    2.72MB/s    0:00:20
+    53182464  49%    2.72MB/s    0:00:19
+    56066048  52%    2.66MB/s    0:00:18
+    58949632  55%    2.67MB/s    0:00:17
+    61308928  57%    2.59MB/s    0:00:17
+    63799296  59%    2.51MB/s    0:00:16
+    66781184  62%    2.54MB/s    0:00:15
+    69697536  65%    2.53MB/s    0:00:14
+    72712192  68%    2.67MB/s    0:00:12
+    75595776  70%    2.77MB/s    0:00:10
+    78479360  73%    2.74MB/s    0:00:10
+    81428480  76%    2.75MB/s    0:00:09
+    84312064  78%    2.73MB/s    0:00:08
+    87195648  81%    2.73MB/s    0:00:07
+    90210304  84%    2.77MB/s    0:00:05
+    93192192  87%    2.76MB/s    0:00:04
+    96075776  89%    2.74MB/s    0:00:03
+    98959360  92%    2.71MB/s    0:00:02
+   101974016  95%    2.68MB/s    0:00:01
+   104857600  98%    2.68MB/s    0:00:00
+   106802484 100%    2.71MB/s    0:00:37 (xfer#1, to-check=0/1)
+
+sent 106815700 bytes  received 42 bytes  2704196.00 bytes/sec
+total size is 106802484  speedup is 1.00
+[2014-05-25 18:25:26 CEST] Transferrer: Uploaded 10_scales.mp4
+[2014-05-25 18:25:26 CEST] Pusher: Syncing with diskstation 
+
+WORM-s132805496-m1399168071--An Introduction to d3.js - From Scattered to Scatterplot%11_axes.mp4
+       32768   0%    0.00kB/s    0:00:00
+     4096000   3%    3.82MB/s    0:00:32
+     7110656   5%    3.32MB/s    0:00:37
+     9863168   7%    3.07MB/s    0:00:39
+remote: merge git-annex     12877824   9%    3.01MB/s    0:00:38
+    15826944  11%    2.76MB/s    0:00:41
+(merging synced/git-annex into git-annex...)        
+remote: (Recording state in git...)        
+remote: ok        
+[2014-05-25 18:25:33 CEST] RemoteControl: Syncing with diskstation 
+To joern@annex.diskstation.local:/volume1/homes/joern/Annex.movies
+   bc92e4a..cf6d3d1  git-annex -> synced/git-annex
+    18710528  14%    2.74MB/s    0:00:40
+    21594112  16%    2.78MB/s    0:00:39
+    24477696  18%    2.75MB/s    0:00:38
+From annex.diskstation.local:/volume1/homes/joern/Annex.movies
+   7b0c500..8393191  git-annex  -> diskstation/git-annex
+error: Ref refs/remotes/diskstation/synced/git-annex is at cf6d3d1e7ec71ad4d9c4a0e5256f361c58ca5e75 but expected bc92e4a867338760dec749af77a9262591b9c43a
+ ! bc92e4a..cf6d3d1  synced/git-annex -> diskstation/synced/git-annex  (unable to update local ref)
+    27230208  20%    2.70MB/s    0:00:38
+    29949952  22%    2.63MB/s    0:00:38
+    32964608  24%    2.64MB/s    0:00:36
+    35848192  26%    2.64MB/s    0:00:35
+    38862848  29%    2.69MB/s    0:00:34
+    41877504  31%    2.77MB/s    0:00:32
+    44761088  33%    2.73MB/s    0:00:31
+    47513600  35%    2.71MB/s    0:00:30
+    50135040  37%    2.61MB/s    0:00:30
+    53280768  40%    2.66MB/s    0:00:29
+    56164352  42%    2.67MB/s    0:00:28
+    59179008  44%    2.71MB/s    0:00:26
+    62062592  46%    2.79MB/s    0:00:24
+    64946176  48%    2.73MB/s    0:00:24
+    68091904  51%    2.82MB/s    0:00:22
+    71106560  53%    2.84MB/s    0:00:21
+    74055680  55%    2.83MB/s    0:00:20
+    77070336  58%    2.85MB/s    0:00:19
+    79953920  60%    2.77MB/s    0:00:18
+    82837504  62%    2.74MB/s    0:00:17
+    85590016  64%    2.72MB/s    0:00:16
+    88342528  66%    2.63MB/s    0:00:16
+    91095040  68%    2.60MB/s    0:00:15
+    94044160  70%    2.62MB/s    0:00:14
+    97058816  73%    2.66MB/s    0:00:13
+    99549184  74%    2.60MB/s    0:00:12
+   101515264  76%    2.41MB/s    0:00:12
+   103481344  77%    2.18MB/s    0:00:13
+   105578496  79%    1.93MB/s    0:00:13
+   107806720  81%    1.89MB/s    0:00:12
+   110821376  83%    2.12MB/s    0:00:10
+   113704960  85%    2.34MB/s    0:00:07
+   116719616  87%    2.60MB/s    0:00:06
+   119603200  90%    2.77MB/s    0:00:04
+   122486784  92%    2.73MB/s    0:00:03
+   125435904  94%    2.74MB/s    0:00:02
+   128450560  96%    2.74MB/s    0:00:01
+   131268608  98%    2.70MB/s    0:00:00
+   132805496 100%    2.67MB/s    0:00:47 (xfer#1, to-check=0/1)
+
+sent 132821882 bytes  received 42 bytes  2683271.19 bytes/sec
+total size is 132805496  speedup is 1.00
+[2014-05-25 18:26:16 CEST] Transferrer: Uploaded 11_axes.mp4
+[2014-05-25 18:26:16 CEST] Pusher: Syncing with diskstation 
+
+WORM-s120549770-m1399667258--An Introduction to d3.js - From Scattered to Scatterplot%12_advanced-techniques.mp4
+       32768   0%    0.00kB/s    0:00:00
+     4096000   3%    3.77MB/s    0:00:30
+     7110656   5%    3.31MB/s    0:00:33
+    10125312   8%    3.15MB/s    0:00:34
+remote: merge git-annex     13008896  10%    3.04MB/s    0:00:34
+[2014-05-25 18:26:22 CEST] RemoteControl: Syncing with diskstation 
+(merging synced/git-annex into git-annex...)        
+remote: (Recording state in git...)        
+remote: ok        
+To joern@annex.diskstation.local:/volume1/homes/joern/Annex.movies
+   cf6d3d1..fef357a  git-annex -> synced/git-annex
+    15892480  13%    2.78MB/s    0:00:36
+    18776064  15%    2.75MB/s    0:00:36
+    21659648  17%    2.72MB/s    0:00:35
+    24608768  20%    2.73MB/s    0:00:34
+From annex.diskstation.local:/volume1/homes/joern/Annex.movies
+   8393191..9ef8d81  git-annex  -> diskstation/git-annex
+error: Ref refs/remotes/diskstation/synced/git-annex is at fef357aaed77f1b94344c0aeea71b30a50de247d but expected cf6d3d1e7ec71ad4d9c4a0e5256f361c58ca5e75
+ ! cf6d3d1..fef357a  synced/git-annex -> diskstation/synced/git-annex  (unable to update local ref)
+    27623424  22%    2.75MB/s    0:00:33
+    30507008  25%    2.74MB/s    0:00:32
+    33390592  27%    2.72MB/s    0:00:31
+    36405248  30%    2.74MB/s    0:00:30
+    39419904  32%    2.73MB/s    0:00:29
+    42434560  35%    2.76MB/s    0:00:27
+    45580288  37%    2.83MB/s    0:00:25
+    48594944  40%    2.83MB/s    0:00:24
+    51576832  42%    2.83MB/s    0:00:23
+    54231040  44%    2.76MB/s    0:00:23
+    57245696  47%    2.74MB/s    0:00:22
+    60260352  49%    2.73MB/s    0:00:21
+    63275008  52%    2.73MB/s    0:00:20
+    66289664  54%    2.80MB/s    0:00:18
+    69304320  57%    2.80MB/s    0:00:17
+    72187904  59%    2.77MB/s    0:00:17
+    75071488  62%    2.74MB/s    0:00:16
+    78086144  64%    2.73MB/s    0:00:15
+    80969728  67%    2.70MB/s    0:00:14
+    84115456  69%    2.75MB/s    0:00:12
+    87130112  72%    2.79MB/s    0:00:11
+    90013696  74%    2.77MB/s    0:00:10
+    92766208  76%    2.73MB/s    0:00:09
+    95780864  79%    2.70MB/s    0:00:08
+    98664448  81%    2.68MB/s    0:00:07
+   101679104  84%    2.71MB/s    0:00:06
+   104693760  86%    2.78MB/s    0:00:05
+   107511808  89%    2.74MB/s    0:00:04
+   110526464  91%    2.74MB/s    0:00:03
+   113508352  94%    2.75MB/s    0:00:02
+   116654080  96%    2.77MB/s    0:00:01
+   119668736  99%    2.82MB/s    0:00:00
+   120549770 100%    2.79MB/s    0:00:41 (xfer#1, to-check=0/1)
+
+sent 120564675 bytes  received 42 bytes  2771602.69 bytes/sec
+total size is 120549770  speedup is 1.00
+[2014-05-25 18:26:59 CEST] Transferrer: Uploaded 12_advanc..iques.mp4
+[2014-05-25 18:27:00 CEST] Pusher: Syncing with diskstation 
+
+WORM-s17591948-m1399167639--An Introduction to d3.js - From Scattered to Scatterplot%13_conclusion.mp4
+       32768   0%    0.00kB/s    0:00:00
+     3997696  22%    3.68MB/s    0:00:03
+     6881280  39%    3.16MB/s    0:00:03
+     9764864  55%    2.99MB/s    0:00:02
+remote: merge git-annex     12910592  73%    2.99MB/s    0:00:01
+    15794176  89%    2.74MB/s    0:00:00
+(merging synced/git-annex into git-annex...)        
+remote: (Recording state in git...)        
+remote: ok        
+[2014-05-25 18:27:07 CEST] RemoteControl: Syncing with diskstation 
+To joern@annex.diskstation.local:/volume1/homes/joern/Annex.movies
+    17591948 100%    2.92MB/s    0:00:05 (xfer#1, to-check=0/1)
+   fef357a..28d7559  git-annex -> synced/git-annex
+
+sent 17594275 bytes  received 42 bytes  2345908.93 bytes/sec
+total size is 17591948  speedup is 1.00
+[2014-05-25 18:27:08 CEST] Transferrer: Uploaded 13_conclusion.mp4
+
+WORM-s1250093-m1389398522--C,43,43 Memory Management%01_introduction.mp4
+       32768   2%    0.00kB/s    0:00:00
+     1250093 100%   25.24MB/s    0:00:00 (xfer#1, to-check=0/1)
+From annex.diskstation.local:/volume1/homes/joern/Annex.movies
+   9ef8d81..740fecd  git-annex  -> diskstation/git-annex
+error: Ref refs/remotes/diskstation/synced/git-annex is at 28d7559836e91e16f98a532c30e5a4cbaef12250 but expected fef357aaed77f1b94344c0aeea71b30a50de247d
+ ! fef357a..28d7559  synced/git-annex -> diskstation/synced/git-annex  (unable to update local ref)
+
+sent 1250398 bytes  received 42 bytes  833626.67 bytes/sec
+total size is 1250093  speedup is 1.00
+[2014-05-25 18:27:10 CEST] Transferrer: Uploaded 01_introduction.mp4
+[2014-05-25 18:27:11 CEST] Pusher: Syncing with diskstation 
+
+WORM-s4072461-m1389398387--C,43,43 Memory Management%02_leaks-and-overruns.mp4
+       32768   0%    0.00kB/s    0:00:00
+     3801088  93%    3.52MB/s    0:00:00
+     4072461 100%    3.35MB/s    0:00:01 (xfer#1, to-check=0/1)
+remote: merge git-annex 
+sent 4073116 bytes  received 42 bytes  1629263.20 bytes/sec
+total size is 4072461  speedup is 1.00
+[2014-05-25 18:27:17 CEST] Transferrer: Uploaded 02_leaks-..rruns.mp4
+(merging synced/git-annex into git-annex...)        
+remote: (Recording state in git...)        
+remote: ok        
+To joern@annex.diskstation.local:/volume1/homes/joern/Annex.movies
+
+WORM-s10885066-m1389398446--C,43,43 Memory Management%03_stl-vector-methods-range-checking-and-sentinels.mp4
+       32768   0%    0.00kB/s    0:00:00
+[2014-05-25 18:27:18 CEST] RemoteControl: Syncing with diskstation 
+   28d7559..d7a7f79  git-annex -> synced/git-annex
+     3964928  36%    3.71MB/s    0:00:01
+     6979584  64%    3.28MB/s    0:00:01
+     9797632  90%    3.08MB/s    0:00:00
+    10885066 100%    3.07MB/s    0:00:03 (xfer#1, to-check=0/1)
+From annex.diskstation.local:/volume1/homes/joern/Annex.movies
+   740fecd..8700b31  git-annex  -> diskstation/git-annex
+error: Ref refs/remotes/diskstation/synced/git-annex is at d7a7f7988fd67d9cd827311157f7633da55d2e85 but expected 28d7559836e91e16f98a532c30e5a4cbaef12250
+ ! 28d7559..d7a7f79  synced/git-annex -> diskstation/synced/git-annex  (unable to update local ref)
+
+sent 10886583 bytes  received 42 bytes  1979386.36 bytes/sec
+total size is 10885066  speedup is 1.00
+[2014-05-25 18:27:22 CEST] Transferrer: Uploaded 03_stl-ve..inels.mp4
+
+WORM-s6830209-m1389398548--C,43,43 Memory Management%04_memory-leaks-part-1.mp4
+       32768   0%    0.00kB/s    0:00:00
+[2014-05-25 18:27:23 CEST] Pusher: Syncing with diskstation 
+     3964928  58%    3.72MB/s    0:00:00
+     6830209 100%    3.23MB/s    0:00:00
+     6830209 100%    3.23MB/s    0:00:02 (xfer#1, to-check=0/1)
+
+sent 6831201 bytes  received 42 bytes  1951783.71 bytes/sec
+total size is 6830209  speedup is 1.00
+[2014-05-25 18:27:25 CEST] Transferrer: Uploaded 04_memory..art-1.mp4
+
+WORM-s4500501-m1389398494--C,43,43 Memory Management%05_memory-leaks-part-2.mp4
+       32768   0%    0.00kB/s    0:00:00
+remote: merge git-annex      3833856  85%    3.57MB/s    0:00:00
+     4500501 100%    3.39MB/s    0:00:01 (xfer#1, to-check=0/1)
+(merging synced/git-annex into git-annex...)        
+remote: (Recording state in git...)        
+remote: ok        
+To joern@annex.diskstation.local:/volume1/homes/joern/Annex.movies
+   d7a7f79..963ef9d  git-annex -> synced/git-annex
+[2014-05-25 18:27:28 CEST] RemoteControl: Syncing with diskstation 
+
+sent 4501209 bytes  received 42 bytes  1286071.71 bytes/sec
+total size is 4500501  speedup is 1.00
+[2014-05-25 18:27:28 CEST] Transferrer: Uploaded 05_memory..art-2.mp4
+
+WORM-s4479532-m1389398463--C,43,43 Memory Management%06_pointers-pitfalls-and-best-practices.mp4
+       32768   0%    0.00kB/s    0:00:00
+     4292608  95%    4.06MB/s    0:00:00
+From annex.diskstation.local:/volume1/homes/joern/Annex.movies
+   8700b31..8a3838b  git-annex  -> diskstation/git-annex
+     4479532 100%    4.05MB/s    0:00:01 (xfer#1, to-check=0/1)
+
+sent 4480253 bytes  received 42 bytes  1280084.29 bytes/sec
+total size is 4479532  speedup is 1.00
+[2014-05-25 18:27:31 CEST] Transferrer: Uploaded 06_pointe..tices.mp4
+
+WORM-s9775707-m1389398514--C,43,43 Memory Management%07_initialize-everything-trust-but-verify.mp4
+       32768   0%    0.00kB/s    0:00:00
+[2014-05-25 18:27:32 CEST] Pusher: Syncing with diskstation 
+     4096000  41%    3.77MB/s    0:00:01
+     7241728  74%    3.37MB/s    0:00:00
+     9775707 100%    3.22MB/s    0:00:02 (xfer#1, to-check=0/1)
+
+sent 9777078 bytes  received 42 bytes  2793462.86 bytes/sec
+total size is 9775707  speedup is 1.00
+[2014-05-25 18:27:34 CEST] Transferrer: Uploaded 07_initia..erify.mp4
+remote: merge git-annex 
+WORM-s15498925-m1389398484--C,43,43 Memory Management%08_reading-and-writing-shared-assets.mp4
+       32768   0%    0.00kB/s    0:00:00
+     3964928  25%    3.69MB/s    0:00:03
+[2014-05-25 18:27:36 CEST] RemoteControl: Syncing with diskstation 
+(merging synced/git-annex into git-annex...)        
+remote: (Recording state in git...)        
+remote: ok        
+To joern@annex.diskstation.local:/volume1/homes/joern/Annex.movies
+   963ef9d..d7cb480  git-annex -> synced/git-annex
+     6848512  44%    3.21MB/s    0:00:02
+     9863168  63%    3.07MB/s    0:00:01
+    13008896  83%    3.05MB/s    0:00:00
+From annex.diskstation.local:/volume1/homes/joern/Annex.movies
+   8a3838b..9dc2149  git-annex  -> diskstation/git-annex
+error: Ref refs/remotes/diskstation/synced/git-annex is at d7cb4801b1d5172fa94c5ef5b952650404806c5e but expected 963ef9d37e0ca48e88afccc16e3d53a7dbe94a15
+ ! 963ef9d..d7cb480  synced/git-annex -> diskstation/synced/git-annex  (unable to update local ref)
+    15498925 100%    3.04MB/s    0:00:04 (xfer#1, to-check=0/1)
+
+sent 15500988 bytes  received 42 bytes  2818369.09 bytes/sec
+total size is 15498925  speedup is 1.00
+[2014-05-25 18:27:41 CEST] Transferrer: Uploaded 08_readin..ssets.mp4
+[2014-05-25 18:27:41 CEST] Pusher: Syncing with diskstation 
+
+WORM-s7479394-m1389398424--C,43,43 Memory Management%09_undefined-behaviour-and-other-bad-practices.mp4
+       32768   0%    0.00kB/s    0:00:00
+     4030464  53%    3.77MB/s    0:00:00
+remote: merge git-annex      7045120  94%    3.28MB/s    0:00:00
+     7479394 100%    3.24MB/s    0:00:02 (xfer#1, to-check=0/1)
+
+sent 7480490 bytes  received 42 bytes  2137294.86 bytes/sec
+total size is 7479394  speedup is 1.00
+[2014-05-25 18:27:47 CEST] RemoteControl: Syncing with diskstation 
+[2014-05-25 18:27:47 CEST] Transferrer: Uploaded 09_undefi..tices.mp4
+
+WORM-s5620037-m1389398373--C,43,43 Memory Management%10_the-heap-and-stl-wrappers.mp4
+       32768   0%    0.00kB/s    0:00:00
+(merging synced/git-annex into git-annex...)        
+remote: (Recording state in git...)        
+remote: ok        
+remote: (Recording state in git...)        
+To joern@annex.diskstation.local:/volume1/homes/joern/Annex.movies
+   d7cb480..cebe41e  git-annex -> synced/git-annex
+     4390912  78%    4.02MB/s    0:00:00
+From annex.diskstation.local:/volume1/homes/joern/Annex.movies
+   9dc2149..2c100e7  git-annex  -> diskstation/git-annex
+error: Ref refs/remotes/diskstation/synced/git-annex is at cebe41ecefec0ba03838b77300b8213abff6e46f but expected d7cb4801b1d5172fa94c5ef5b952650404806c5e
+ ! d7cb480..cebe41e  synced/git-annex -> diskstation/synced/git-annex  (unable to update local ref)
+[2014-05-25 18:27:48 CEST] RemoteControl: Syncing with diskstation 
+     5620037 100%    3.71MB/s    0:00:01 (xfer#1, to-check=0/1)
+
+sent 5620887 bytes  received 42 bytes  2248371.60 bytes/sec
+total size is 5620037  speedup is 1.00
+[2014-05-25 18:27:50 CEST] Transferrer: Uploaded 10_the-he..ppers.mp4
+From annex.diskstation.local:/volume1/homes/joern/Annex.movies
+   2c100e7..7d585c6  git-annex  -> diskstation/git-annex
+
+WORM-s1014080-m1389398563--C,43,43 Memory Management%11_about-the-author.mp4
+       32768   3%    0.00kB/s    0:00:00
+     1014080 100%   20.80MB/s    0:00:00 (xfer#1, to-check=0/1)
+
+sent 1014357 bytes  received 42 bytes  676266.00 bytes/sec
+total size is 1014080  speedup is 1.00
+[2014-05-25 18:27:51 CEST] Transferrer: Uploaded 11_about-..uthor.mp4
+[2014-05-25 18:27:52 CEST] Pusher: Syncing with diskstation 
+remote: merge git-annex (merging synced/git-annex into git-annex...)        
+remote: (Recording state in git...)        
+remote: ok        
+To joern@annex.diskstation.local:/volume1/homes/joern/Annex.movies
+[2014-05-25 18:27:56 CEST] RemoteControl: Syncing with diskstation 
+   cebe41e..1f16e00  git-annex -> synced/git-annex
+From annex.diskstation.local:/volume1/homes/joern/Annex.movies
+   7d585c6..25d816f  git-annex  -> diskstation/git-annex
+error: Ref refs/remotes/diskstation/synced/git-annex is at 1f16e00081100c31ba110cc6655ba98ce9c3ccf9 but expected cebe41ecefec0ba03838b77300b8213abff6e46f
+ ! cebe41e..1f16e00  synced/git-annex -> diskstation/synced/git-annex  (unable to update local ref)
+[2014-05-25 19:02:45 CEST] main: Syncing with diskstation 
+remote: merge git-annex ok        
+To joern@annex.diskstation.local:/volume1/homes/joern/Annex.movies
+   1f16e00..25d816f  git-annex -> synced/git-annex
+[2014-05-25 19:16:16 CEST] NetWatcherFallback: Syncing with diskstation 
+Everything up-to-date
+[2014-05-25 19:44:23 CEST] main: Syncing with diskstation 
+remote: merge git-annex (merging synced/git-annex into git-annex...)        
+remote: ok        
+To joern@annex.diskstation.local:/volume1/homes/joern/Annex.movies
+   25d816f..af358a2  git-annex -> synced/git-annex
+
+WORM-s311796310-m1396397568--Lean UX Workshop%01_introduction.mp4
+       32768   0%    0.00kB/s    0:00:00
+     3440640   1%    3.23MB/s    0:01:33
+     6062080   1%    2.75MB/s    0:01:48
+    10518528   3%    3.15MB/s    0:01:33
+    13926400   4%    3.15MB/s    0:01:32
+    16023552   5%    2.73MB/s    0:01:45
+    20611072   6%    3.22MB/s    0:01:28
+    22315008   7%    2.58MB/s    0:01:49
+    26509312   8%    2.76MB/s    0:01:40
+    29655040   9%    3.10MB/s    0:01:28
+    33325056  10%    2.61MB/s    0:01:44
+    36470784  11%    2.96MB/s    0:01:30
+    40402944  12%    2.88MB/s    0:01:32
+    44072960  14%    2.96MB/s    0:01:28
+    47218688  15%    3.17MB/s    0:01:21
+    49840128  15%    3.04MB/s    0:01:24
+    54296576  17%    3.12MB/s    0:01:20
+    58228736  18%    3.20MB/s    0:01:17
+    61636608  19%    3.23MB/s    0:01:15
+    64782336  20%    3.40MB/s    0:01:11
+    66879488  21%    2.92MB/s    0:01:21
+    71335936  22%    3.07MB/s    0:01:16
+    75005952  24%    3.15MB/s    0:01:13
+    78151680  25%    3.14MB/s    0:01:12
+    80510976  25%    3.01MB/s    0:01:15
+    83918848  26%    2.76MB/s    0:01:20
+    86540288  27%    2.48MB/s    0:01:28
+    91291648  29%    2.83MB/s    0:01:15
+    92831744  29%    2.75MB/s    0:01:17
+    95977472  30%    2.66MB/s    0:01:19
+   100433920  32%    3.08MB/s    0:01:06
+   103841792  33%    2.77MB/s    0:01:13
+   107446272  34%    3.30MB/s    0:01:00
+   109608960  35%    3.12MB/s    0:01:03
+   113803264  36%    3.09MB/s    0:01:02
+   115900416  37%    2.68MB/s    0:01:11
+   119308288  38%    2.62MB/s    0:01:11
+   123240448  39%    2.97MB/s    0:01:01
+   126648320  40%    2.61MB/s    0:01:09
+   130580480  41%    3.09MB/s    0:00:57
+   134643712  43%    3.26MB/s    0:00:52
+   135561216  43%    2.65MB/s    0:01:04
+   140017664  44%    3.11MB/s    0:00:53
+   142639104  45%    2.82MB/s    0:00:58
+   146866176  47%    2.87MB/s    0:00:56
+   150241280  48%    3.46MB/s    0:00:45
+   152338432  48%    2.80MB/s    0:00:55
+   157220864  50%    3.31MB/s    0:00:45
+   160464896  51%    2.88MB/s    0:00:51
+   165085184  52%    3.14MB/s    0:00:45
+   167542784  53%    3.35MB/s    0:00:42
+   171638784  55%    3.18MB/s    0:00:43
+   172785664  55%    2.78MB/s    0:00:48
+   176979968  56%    2.67MB/s    0:00:49
+   179601408  57%    2.48MB/s    0:00:52
+   184254464  59%    2.60MB/s    0:00:47
+   187432960  60%    3.16MB/s    0:00:38
+   190742528  61%    2.96MB/s    0:00:39
+   194412544  62%    3.48MB/s    0:00:32
+   197525504  63%    3.12MB/s    0:00:35
+   201064448  64%    3.19MB/s    0:00:33
+   203980800  65%    3.01MB/s    0:00:35
+   208699392  66%    3.25MB/s    0:00:31
+   212369408  68%    3.36MB/s    0:00:28
+   215515136  69%    3.27MB/s    0:00:28
+   217350144  69%    3.04MB/s    0:00:30
+   221806592  71%    2.99MB/s    0:00:29
+   225378304  72%    2.94MB/s    0:00:28
+   228622336  73%    2.98MB/s    0:00:27
+   231243776  74%    3.13MB/s    0:00:25
+   233603072  74%    2.59MB/s    0:00:29
+   237273088  76%    2.35MB/s    0:00:30
+   239894528  76%    2.22MB/s    0:00:31
+   242515968  77%    2.07MB/s    0:00:32
+   244350976  78%    1.97MB/s    0:00:33
+   246448128  79%    1.87MB/s    0:00:34
+   250118144  80%    1.73MB/s    0:00:34
+   253263872  81%    2.00MB/s    0:00:28
+   254312448  81%    1.80MB/s    0:00:31
+   255885312  82%    1.58MB/s    0:00:34
+   257720320  82%    1.33MB/s    0:00:39
+   259817472  83%    1.01MB/s    0:00:50
+   261914624  84%    1.06MB/s    0:00:45
+   264011776  84%    1.08MB/s    0:00:43
+   265584640  85%    1.16MB/s    0:00:39
+   267419648  85%    1.21MB/s    0:00:35
+   272662528  87%    2.00MB/s    0:00:19
+   276856832  88%    2.77MB/s    0:00:12
+   278953984  89%    2.81MB/s    0:00:11
+   283410432  90%    3.51MB/s    0:00:07
+   288358400  92%    3.47MB/s    0:00:06
+   292323328  93%    3.32MB/s    0:00:05
+   296714240  95%    3.93MB/s    0:00:03
+   300908544  96%    3.95MB/s    0:00:02
+   303333376  97%    3.30MB/s    0:00:02
+   307789824  98%    3.53MB/s    0:00:01
+   310673408  99%    2.89MB/s    0:00:00
+   311796310 100%    2.73MB/s    0:01:48 (xfer#1, to-check=0/1)
+
+sent 311834516 bytes  received 42 bytes  2822032.20 bytes/sec
+total size is 311796310  speedup is 1.00
+[2014-05-25 19:46:46 CEST] Transferrer: Uploaded 01_introduction.mp4
+[2014-05-25 19:46:46 CEST] Pusher: Syncing with diskstation 
+
+WORM-s197776451-m1396396404--Lean UX Workshop%02_nordstrom-case-study.mp4
+       32768   0%    0.00kB/s    0:00:00
+     5177344   2%    4.78MB/s    0:00:39
+     8683520   4%    3.74MB/s    0:00:49
+    13139968   6%    3.33MB/s    0:00:54
+remote: merge git-annex     18382848   9%    3.63MB/s    0:00:48
+    22315008  11%    3.16MB/s    0:00:54
+(merging synced/git-annex into git-annex...)        
+remote: (Recording state in git...)        
+remote: ok        
+To joern@annex.diskstation.local:/volume1/homes/joern/Annex.movies
+   af358a2..3b93fc5  git-annex -> synced/git-annex
+[2014-05-25 19:46:54 CEST] RemoteControl: Syncing with diskstation 
+    26902528  13%    3.47MB/s    0:00:48
+    29917184  15%    3.51MB/s    0:00:46
+    31653888  16%    2.82MB/s    0:00:57
+    33849344  17%    2.64MB/s    0:01:00
+    36995072  18%    2.30MB/s    0:01:08
+    41189376  20%    2.62MB/s    0:00:58
+    44859392  22%    2.97MB/s    0:00:50
+From annex.diskstation.local:/volume1/homes/joern/Annex.movies
+   25d816f..b7c06ab  git-annex  -> diskstation/git-annex
+    50102272  25%    3.67MB/s    0:00:39
+    52199424  26%    3.41MB/s    0:00:41
+    56655872  28%    3.47MB/s    0:00:39
+    60719104  30%    3.64MB/s    0:00:36
+    64520192  32%    3.31MB/s    0:00:39
+    68714496  34%    3.85MB/s    0:00:32
+    72876032  36%    3.79MB/s    0:00:32
+    76054528  38%    3.60MB/s    0:00:32
+    80216064  40%    3.70MB/s    0:00:31
+    83656704  42%    3.25MB/s    0:00:34
+    88899584  44%    3.48MB/s    0:00:30
+    92995584  47%    3.69MB/s    0:00:27
+    96763904  48%    3.62MB/s    0:00:27
+   100925440  51%    4.08MB/s    0:00:23
+   104628224  52%    3.73MB/s    0:00:24
+   109084672  55%    3.80MB/s    0:00:22
+   111443968  56%    3.40MB/s    0:00:24
+   116752384  59%    3.66MB/s    0:00:21
+   119046144  60%    3.33MB/s    0:00:23
+   124026880  62%    3.44MB/s    0:00:20
+   128483328  64%    3.98MB/s    0:00:16
+   132710400  67%    3.73MB/s    0:00:17
+   136871936  69%    4.17MB/s    0:00:14
+   141131776  71%    4.00MB/s    0:00:13
+   145522688  73%    4.01MB/s    0:00:12
+   149716992  75%    4.02MB/s    0:00:11
+   154173440  77%    4.08MB/s    0:00:10
+   158433280  80%    4.11MB/s    0:00:09
+   162758656  82%    4.08MB/s    0:00:08
+   166952960  84%    4.08MB/s    0:00:07
+   170459136  86%    3.85MB/s    0:00:06
+   174817280  88%    3.87MB/s    0:00:05
+   179011584  90%    3.85MB/s    0:00:04
+   183238656  92%    3.86MB/s    0:00:03
+   187367424  94%    4.01MB/s    0:00:02
+   191561728  96%    3.96MB/s    0:00:01
+   195887104  99%    3.98MB/s    0:00:00
+   197776451 100%    3.62MB/s    0:00:52 (xfer#1, to-check=0/1)
+
+sent 197800745 bytes  received 42 bytes  3697210.97 bytes/sec
+total size is 197776451  speedup is 1.00
+[2014-05-25 19:47:40 CEST] Transferrer: Uploaded 02_nordst..study.mp4
+[2014-05-25 19:47:40 CEST] Pusher: Syncing with diskstation 
+
+WORM-s401429383-m1396397422--Lean UX Workshop%03_assumptions-and-hypotheses.mp4
+       32768   0%    0.00kB/s    0:00:00
+     5472256   1%    5.18MB/s    0:01:14
+remote: merge git-annex      9895936   2%    4.61MB/s    0:01:22
+    14188544   3%    4.41MB/s    0:01:25
+[2014-05-25 19:47:45 CEST] RemoteControl: Syncing with diskstation 
+(merging synced/git-annex into git-annex...)        
+remote: (Recording state in git...)        
+remote: ok        
+To joern@annex.diskstation.local:/volume1/homes/joern/Annex.movies
+   3b93fc5..02a8a6a  git-annex -> synced/git-annex
+    18808832   4%    4.39MB/s    0:01:25
+    22315008   5%    3.92MB/s    0:01:34
+    26935296   6%    3.98MB/s    0:01:31
+From annex.diskstation.local:/volume1/homes/joern/Annex.movies
+   b7c06ab..c1859f2  git-annex  -> diskstation/git-annex
+error: Ref refs/remotes/diskstation/synced/git-annex is at 02a8a6a6016fd1f36acc39b27cd9e2f1b24fd394 but expected 3b93fc5ace21818579a161200a925cf8e2120c4d
+ ! 3b93fc5..02a8a6a  synced/git-annex -> diskstation/synced/git-annex  (unable to update local ref)
+    31752192   7%    4.10MB/s    0:01:28
+    36110336   8%    4.04MB/s    0:01:28
+    39616512   9%    4.02MB/s    0:01:27
+    44826624  11%    4.13MB/s    0:01:24
+    47742976  11%    3.71MB/s    0:01:33
+    52854784  13%    3.89MB/s    0:01:27
+    57180160  14%    3.87MB/s    0:01:26
+    62423040  15%    3.92MB/s    0:01:24
+    66617344  16%    4.20MB/s    0:01:17
+    71008256  17%    4.04MB/s    0:01:19
+    75333632  18%    4.30MB/s    0:01:14
+    79659008  19%    4.08MB/s    0:01:17
+    83918848  20%    4.07MB/s    0:01:16
+    88375296  22%    4.07MB/s    0:01:15
+    92700672  23%    4.04MB/s    0:01:14
+    97058816  24%    4.05MB/s    0:01:13
+   101253120  25%    4.06MB/s    0:01:12
+   104759296  26%    3.84MB/s    0:01:15
+   109084672  27%    3.84MB/s    0:01:14
+   112820224  28%    3.70MB/s    0:01:16
+   117473280  29%    3.79MB/s    0:01:13
+   120094720  29%    3.53MB/s    0:01:17
+   125435904  31%    3.79MB/s    0:01:11
+   128483328  32%    3.46MB/s    0:01:17
+   133988352  33%    3.67MB/s    0:01:11
+   137134080  34%    3.68MB/s    0:01:10
+   142639104  35%    3.70MB/s    0:01:08
+   145522688  36%    3.72MB/s    0:01:07
+   150962176  37%    3.69MB/s    0:01:06
+   154435584  38%    3.72MB/s    0:01:04
+   159875072  39%    3.73MB/s    0:01:03
+   163610624  40%    3.76MB/s    0:01:01
+   168755200  42%    3.72MB/s    0:01:01
+   172523520  42%    3.67MB/s    0:01:00
+   177995776  44%    3.66MB/s    0:00:59
+   181960704  45%    3.69MB/s    0:00:58
+   187400192  46%    3.74MB/s    0:00:55
+   191397888  47%    3.79MB/s    0:00:54
+   196739072  49%    3.77MB/s    0:00:53
+   200310784  49%    3.72MB/s    0:00:52
+   205553664  51%    3.69MB/s    0:00:51
+   209780736  52%    4.06MB/s    0:00:46
+   213778432  53%    3.76MB/s    0:00:48
+   217088000  54%    3.72MB/s    0:00:48
+   222560256  55%    3.76MB/s    0:00:46
+   225214464  56%    3.09MB/s    0:00:55
+   227573760  56%    2.76MB/s    0:01:01
+   232816640  57%    3.35MB/s    0:00:49
+   236486656  58%    2.94MB/s    0:00:54
+   241336320  60%    3.77MB/s    0:00:41
+   243826688  60%    3.50MB/s    0:00:44
+   249069568  62%    3.47MB/s    0:00:42
+   253001728  63%    3.18MB/s    0:00:45
+   258146304  64%    3.23MB/s    0:00:43
+   262438912  65%    3.84MB/s    0:00:35
+   263487488  65%    3.00MB/s    0:00:44
+   267943936  66%    3.41MB/s    0:00:38
+   271876096  67%    2.87MB/s    0:00:44
+   277118976  69%    3.07MB/s    0:00:39
+   281542656  70%    3.77MB/s    0:00:31
+   285507584  71%    3.74MB/s    0:00:30
+   290226176  72%    4.25MB/s    0:00:25
+   293371904  73%    3.74MB/s    0:00:28
+   298614784  74%    3.64MB/s    0:00:27
+   304119808  75%    4.00MB/s    0:00:23
+   307527680  76%    3.39MB/s    0:00:27
+   312508416  77%    3.67MB/s    0:00:23
+   317882368  79%    3.97MB/s    0:00:20
+   320372736  79%    3.32MB/s    0:00:23
+   325550080  81%    4.07MB/s    0:00:18
+   329842688  82%    4.06MB/s    0:00:17
+   333217792  83%    3.54MB/s    0:00:18
+   338460672  84%    4.23MB/s    0:00:14
+   342949888  85%    4.06MB/s    0:00:14
+   346062848  86%    3.50MB/s    0:00:15
+   350519296  87%    3.72MB/s    0:00:13
+   355926016  88%    3.75MB/s    0:00:11
+   360185856  89%    3.70MB/s    0:00:10
+   364380160  90%    4.24MB/s    0:00:08
+   368607232  91%    4.26MB/s    0:00:07
+   372015104  92%    3.72MB/s    0:00:07
+   377520128  94%    3.97MB/s    0:00:05
+   381976576  95%    4.03MB/s    0:00:04
+   386236416  96%    4.03MB/s    0:00:03
+   389578752  97%    4.10MB/s    0:00:02
+   394330112  98%    3.97MB/s    0:00:01
+   398589952  99%    3.94MB/s    0:00:00
+   401429383 100%    3.77MB/s    0:01:41 (xfer#1, to-check=0/1)
+
+sent 401478543 bytes  received 42 bytes  3879020.14 bytes/sec
+total size is 401429383  speedup is 1.00
+[2014-05-25 19:49:23 CEST] Transferrer: Uploaded 03_assump..heses.mp4
+[2014-05-25 19:49:23 CEST] Pusher: Syncing with diskstation 
+
+WORM-s341574274-m1396396677--Lean UX Workshop%04_personas.mp4
+       32768   0%    0.00kB/s    0:00:00
+     5013504   1%    3.92MB/s    0:01:23
+    10452992   3%    4.49MB/s    0:01:12
+remote: merge git-annex     14778368   4%    4.38MB/s    0:01:12
+    18382848   5%    4.15MB/s    0:01:16
+[2014-05-25 19:49:29 CEST] RemoteControl: Syncing with diskstation 
+(merging synced/git-annex into git-annex...)        
+remote: (Recording state in git...)        
+remote: ok        
+To joern@annex.diskstation.local:/volume1/homes/joern/Annex.movies
+   02a8a6a..399d085  git-annex -> synced/git-annex
+    22577152   6%    4.16MB/s    0:01:14
+    26509312   7%    3.60MB/s    0:01:25
+    31293440   9%    3.70MB/s    0:01:21
+From annex.diskstation.local:/volume1/homes/joern/Annex.movies
+   c1859f2..5a46cc4  git-annex  -> diskstation/git-annex
+error: Ref refs/remotes/diskstation/synced/git-annex is at 399d0856bfb9fef74a18eb9d1d57c0151267323d but expected 02a8a6a6016fd1f36acc39b27cd9e2f1b24fd394
+ ! 02a8a6a..399d085  synced/git-annex -> diskstation/synced/git-annex  (unable to update local ref)
+    35684352  10%    3.88MB/s    0:01:17
+    40108032  11%    3.92MB/s    0:01:15
+    43352064  12%    3.95MB/s    0:01:13
+    46956544  13%    3.67MB/s    0:01:18
+    51675136  15%    3.55MB/s    0:01:19
+    57180160  16%    3.80MB/s    0:01:13
+    61472768  17%    4.03MB/s    0:01:07
+    65044480  19%    4.00MB/s    0:01:07
+    70057984  20%    4.29MB/s    0:01:01
+    74383360  21%    4.03MB/s    0:01:04
+    78675968  23%    4.03MB/s    0:01:03
+    82739200  24%    4.18MB/s    0:01:00
+    86999040  25%    4.01MB/s    0:01:02
+    90734592  26%    3.76MB/s    0:01:05
+    95977472  28%    3.96MB/s    0:01:00
+   100433920  29%    4.04MB/s    0:00:58
+   104792064  30%    4.04MB/s    0:00:57
+   108986368  31%    4.24MB/s    0:00:53
+   111443968  32%    3.60MB/s    0:01:02
+   116686848  34%    3.74MB/s    0:00:58
+   120487936  35%    3.62MB/s    0:00:59
+   123764736  36%    3.25MB/s    0:01:05
+   128974848  37%    3.88MB/s    0:00:53
+   132415488  38%    3.52MB/s    0:00:58
+   136347648  39%    3.31MB/s    0:01:00
+   141721600  41%    3.92MB/s    0:00:49
+   144998400  42%    3.48MB/s    0:00:55
+   147619840  43%    3.27MB/s    0:00:57
+   152862720  44%    3.81MB/s    0:00:48
+   156008448  45%    3.29MB/s    0:00:55
+   160464896  46%    3.25MB/s    0:00:54
+   165904384  48%    3.88MB/s    0:00:44
+   169246720  49%    3.48MB/s    0:00:48
+   172523520  50%    3.36MB/s    0:00:49
+   178028544  52%    3.92MB/s    0:00:40
+   181600256  53%    3.51MB/s    0:00:44
+   184582144  54%    3.28MB/s    0:00:46
+   188514304  55%    3.60MB/s    0:00:41
+   192970752  56%    3.36MB/s    0:00:43
+   196902912  57%    3.19MB/s    0:00:44
+   202113024  59%    3.81MB/s    0:00:35
+   205684736  60%    3.72MB/s    0:00:35
+   208175104  60%    3.27MB/s    0:00:39
+   213581824  62%    3.87MB/s    0:00:32
+   216825856  63%    3.41MB/s    0:00:35
+   219709440  64%    3.22MB/s    0:00:36
+   225083392  65%    3.91MB/s    0:00:29
+   228622336  66%    3.48MB/s    0:00:31
+   232292352  68%    3.31MB/s    0:00:32
+   237666304  69%    3.89MB/s    0:00:26
+   241139712  70%    3.48MB/s    0:00:28
+   245137408  71%    3.29MB/s    0:00:28
+   250380288  73%    3.49MB/s    0:00:25
+   255361024  74%    3.42MB/s    0:00:24
+   258899968  75%    3.43MB/s    0:00:23
+   262176768  76%    3.38MB/s    0:00:22
+   267452416  78%    3.79MB/s    0:00:19
+   270434304  79%    3.36MB/s    0:00:20
+   274235392  80%    3.24MB/s    0:00:20
+   279478272  81%    3.89MB/s    0:00:15
+   282886144  82%    3.45MB/s    0:00:16
+   286818304  83%    3.38MB/s    0:00:15
+   292061184  85%    3.50MB/s    0:00:13
+   297304064  87%    3.50MB/s    0:00:12
+   300711936  88%    3.49MB/s    0:00:11
+   304152576  89%    3.66MB/s    0:00:09
+   308543488  90%    3.86MB/s    0:00:08
+   312770560  91%    3.60MB/s    0:00:07
+   316178432  92%    3.63MB/s    0:00:06
+   319586304  93%    3.44MB/s    0:00:06
+   325025792  95%    3.66MB/s    0:00:04
+   328466432  96%    3.50MB/s    0:00:03
+   331120640  96%    3.24MB/s    0:00:03
+   336592896  98%    3.87MB/s    0:00:01
+   340000768  99%    3.42MB/s    0:00:00
+   341574274 100%    3.66MB/s    0:01:29 (xfer#1, to-check=0/1)
+
+sent 341616112 bytes  received 42 bytes  3733509.88 bytes/sec
+total size is 341574274  speedup is 1.00
+[2014-05-25 19:50:54 CEST] Transferrer: Uploaded 04_personas.mp4
+[2014-05-25 19:50:54 CEST] Pusher: Syncing with diskstation 
+
+WORM-s112588009-m1396396463--Lean UX Workshop%05_outcomes-and-features.mp4
+       32768   0%    0.00kB/s    0:00:00
+     5537792   4%    5.11MB/s    0:00:20
+     8683520   7%    3.73MB/s    0:00:27
+    14057472  12%    4.16MB/s    0:00:23
+remote: merge git-annex     17334272  15%    3.90MB/s    0:00:23
+(merging synced/git-annex into git-annex...)        
+remote: (Recording state in git...)        
+remote: ok        
+[2014-05-25 19:51:00 CEST] RemoteControl: Syncing with diskstation 
+To joern@annex.diskstation.local:/volume1/homes/joern/Annex.movies
+    21266432  18%    3.26MB/s    0:00:27
+   399d085..8395b06  git-annex -> synced/git-annex
+    26509312  23%    3.85MB/s    0:00:21
+    29655040  26%    3.34MB/s    0:00:24
+From annex.diskstation.local:/volume1/homes/joern/Annex.movies
+   5a46cc4..c29a46c  git-annex  -> diskstation/git-annex
+error: Ref refs/remotes/diskstation/synced/git-annex is at 8395b065d53211a1d1da518dafd09a81ba42f11c but expected 399d0856bfb9fef74a18eb9d1d57c0151267323d
+ ! 399d085..8395b06  synced/git-annex -> diskstation/synced/git-annex  (unable to update local ref)
+    33062912  29%    3.16MB/s    0:00:24
+    38502400  34%    3.76MB/s    0:00:19
+    41910272  37%    3.36MB/s    0:00:20
+    45121536  40%    3.21MB/s    0:00:20
+    50561024  44%    3.88MB/s    0:00:15
+    53772288  47%    3.36MB/s    0:00:17
+    57442304  51%    3.22MB/s    0:00:16
+    62881792  55%    3.88MB/s    0:00:12
+    66093056  58%    3.39MB/s    0:00:13
+    70451200  62%    3.69MB/s    0:00:11
+    72908800  64%    3.15MB/s    0:00:12
+    77627392  68%    2.87MB/s    0:00:11
+    83132416  73%    3.33MB/s    0:00:08
+    84541440  75%    2.75MB/s    0:00:09
+    89423872  79%    3.71MB/s    0:00:06
+    90570752  80%    3.08MB/s    0:00:06
+    95911936  85%    3.02MB/s    0:00:05
+    97550336  86%    3.06MB/s    0:00:04
+   102531072  91%    2.60MB/s    0:00:03
+   107872256  95%    3.43MB/s    0:00:01
+   109871104  97%    2.78MB/s    0:00:00
+   112588009 100%    3.35MB/s    0:00:31 (xfer#1, to-check=0/1)
+
+sent 112601904 bytes  received 42 bytes  3361252.12 bytes/sec
+total size is 112588009  speedup is 1.00
+[2014-05-25 19:51:27 CEST] Transferrer: Uploaded 05_outcom..tures.mp4
+[2014-05-25 19:51:27 CEST] Pusher: Syncing with diskstation 
+
+WORM-s78565449-m1396396837--Lean UX Workshop%06_writing-hypotheses.mp4
+       32768   0%    0.00kB/s    0:00:00
+     5275648   6%    4.94MB/s    0:00:14
+     6848512   8%    3.18MB/s    0:00:22
+remote: merge git-annex     11796480  15%    3.64MB/s    0:00:17
+    13402112  17%    3.12MB/s    0:00:20
+[2014-05-25 19:51:32 CEST] RemoteControl: Syncing with diskstation 
+(merging synced/git-annex into git-annex...)        
+remote: (Recording state in git...)        
+remote: ok        
+To joern@annex.diskstation.local:/volume1/homes/joern/Annex.movies
+   8395b06..bbec44b  git-annex -> synced/git-annex
+    17858560  22%    2.94MB/s    0:00:20
+    19693568  25%    3.01MB/s    0:00:19
+    24412160  31%    2.96MB/s    0:00:17
+From annex.diskstation.local:/volume1/homes/joern/Annex.movies
+   c29a46c..d52fe33  git-annex  -> diskstation/git-annex
+error: Ref refs/remotes/diskstation/synced/git-annex is at bbec44b1f18107dc3aaab411ddf2ada1aa273741 but expected 8395b065d53211a1d1da518dafd09a81ba42f11c
+ ! 8395b06..bbec44b  synced/git-annex -> diskstation/synced/git-annex  (unable to update local ref)
+    27033600  34%    3.00MB/s    0:00:16
+    31490048  40%    2.69MB/s    0:00:17
+    36470784  46%    2.96MB/s    0:00:13
+    41910272  53%    3.09MB/s    0:00:11
+    43286528  55%    2.97MB/s    0:00:11
+    48529408  61%    3.00MB/s    0:00:09
+    53510144  68%    3.06MB/s    0:00:07
+    58753024  74%    2.78MB/s    0:00:06
+    64159744  81%    3.51MB/s    0:00:04
+    67141632  85%    3.23MB/s    0:00:03
+    72548352  92%    3.61MB/s    0:00:01
+    76840960  97%    3.79MB/s    0:00:00
+    78565449 100%    3.17MB/s    0:00:23 (xfer#1, to-check=0/1)
+
+sent 78575188 bytes  received 42 bytes  3081381.57 bytes/sec
+total size is 78565449  speedup is 1.00
+[2014-05-25 19:51:52 CEST] Transferrer: Uploaded 06_writin..heses.mp4
+[2014-05-25 19:51:52 CEST] Pusher: Syncing with diskstation 
+
+WORM-s237023281-m1396397192--Lean UX Workshop%07_design-studio.mp4
+       32768   0%    0.00kB/s    0:00:00
+     3964928   1%    3.16MB/s    0:01:12
+remote: merge git-annex      9207808   3%    3.35MB/s    0:01:06
+(merging synced/git-annex into git-annex...)        
+remote: (Recording state in git...)        
+remote: ok        
+To joern@annex.diskstation.local:/volume1/homes/joern/Annex.movies
+[2014-05-25 19:51:57 CEST] RemoteControl: Syncing with diskstation 
+    14712832   6%    3.19MB/s    0:01:08
+   bbec44b..423bfb2  git-annex -> synced/git-annex
+    19169280   8%    2.99MB/s    0:01:11
+    23887872  10%    3.20MB/s    0:01:05
+From annex.diskstation.local:/volume1/homes/joern/Annex.movies
+   d52fe33..b408181  git-annex  -> diskstation/git-annex
+error: Ref refs/remotes/diskstation/synced/git-annex is at 423bfb2586ebc18e8149998862ee1d0060d962ec but expected bbec44b1f18107dc3aaab411ddf2ada1aa273741
+ ! bbec44b..423bfb2  synced/git-annex -> diskstation/synced/git-annex  (unable to update local ref)
+    26771456  11%    2.98MB/s    0:01:08
+    30965760  13%    2.91MB/s    0:01:09
+    35946496  15%    3.04MB/s    0:01:04
+    40927232  17%    2.76MB/s    0:01:09
+    45187072  19%    3.01MB/s    0:01:02
+    48005120  20%    3.04MB/s    0:01:00
+    50364416  21%    2.73MB/s    0:01:06
+    55607296  23%    2.77MB/s    0:01:03
+    60948480  25%    3.01MB/s    0:00:57
+    62423040  26%    2.75MB/s    0:01:01
+    66355200  27%    2.96MB/s    0:00:56
+    71335936  30%    2.89MB/s    0:00:55
+    76742656  32%    2.90MB/s    0:00:53
+    79200256  33%    3.06MB/s    0:00:50
+    82870272  34%    3.28MB/s    0:00:45
+    86802432  36%    3.60MB/s    0:00:40
+    91521024  38%    3.44MB/s    0:00:41
+    95191040  40%    3.74MB/s    0:00:37
+    97026048  40%    3.23MB/s    0:00:42
+   100171776  42%    2.87MB/s    0:00:46
+   104628224  44%    2.80MB/s    0:00:46
+   109903872  46%    3.14MB/s    0:00:39
+   113541120  47%    3.66MB/s    0:00:32
+   116686848  49%    3.89MB/s    0:00:30
+   121667584  51%    3.68MB/s    0:00:30
+   126386176  53%    3.51MB/s    0:00:30
+   131530752  55%    3.83MB/s    0:00:26
+   135823360  57%    4.07MB/s    0:00:24
+   140214272  59%    4.31MB/s    0:00:21
+   144474112  60%    4.26MB/s    0:00:21
+   148144128  62%    3.82MB/s    0:00:22
+   153124864  64%    3.97MB/s    0:00:20
+   154959872  65%    3.40MB/s    0:00:23
+   160071680  67%    3.59MB/s    0:00:20
+   161513472  68%    3.14MB/s    0:00:23
+   166494208  70%    3.15MB/s    0:00:21
+   169639936  71%    3.11MB/s    0:00:21
+   175046656  73%    3.17MB/s    0:00:19
+   178028544  75%    3.30MB/s    0:00:17
+   183271424  77%    3.33MB/s    0:00:15
+   186941440  78%    3.34MB/s    0:00:14
+   191922176  80%    3.25MB/s    0:00:13
+   195330048  82%    3.22MB/s    0:00:12
+   200835072  84%    3.27MB/s    0:00:10
+   202211328  85%    3.22MB/s    0:00:10
+   207585280  87%    3.31MB/s    0:00:08
+   210534400  88%    3.58MB/s    0:00:07
+   215154688  90%    3.40MB/s    0:00:06
+   215777280  91%    3.21MB/s    0:00:06
+   221020160  93%    3.03MB/s    0:00:05
+   225738752  95%    2.96MB/s    0:00:03
+   230227968  97%    2.94MB/s    0:00:02
+   235175936  99%    3.25MB/s    0:00:00
+   237023281 100%    3.26MB/s    0:01:09 (xfer#1, to-check=0/1)
+
+sent 237052360 bytes  received 42 bytes  3315418.21 bytes/sec
+total size is 237023281  speedup is 1.00
+[2014-05-25 19:53:03 CEST] Transferrer: Uploaded 07_design..tudio.mp4
+[2014-05-25 19:53:04 CEST] Pusher: Syncing with diskstation 
+
+WORM-s113325561-m1396396724--Lean UX Workshop%08_experiments-and-mvp-s.mp4
+       32768   0%    0.00kB/s    0:00:00
+     4227072   3%    3.88MB/s    0:00:27
+remote: merge git-annex      7897088   6%    3.69MB/s    0:00:27
+[2014-05-25 19:53:08 CEST] RemoteControl: Syncing with diskstation 
+(merging synced/git-annex into git-annex...)        
+remote: (Recording state in git...)        
+remote: ok        
+To joern@annex.diskstation.local:/volume1/homes/joern/Annex.movies
+   423bfb2..e9861a8  git-annex -> synced/git-annex
+    13139968  11%    4.11MB/s    0:00:23
+    16023552  14%    3.39MB/s    0:00:28
+    21266432  18%    3.62MB/s    0:00:24
+From annex.diskstation.local:/volume1/homes/joern/Annex.movies
+   b408181..6b97695  git-annex  -> diskstation/git-annex
+error: Ref refs/remotes/diskstation/synced/git-annex is at e9861a8445e7859f23e90dbc1e6113af547c0515 but expected 423bfb2586ebc18e8149998862ee1d0060d962ec
+ ! 423bfb2..e9861a8  synced/git-annex -> diskstation/synced/git-annex  (unable to update local ref)
+    23101440  20%    3.22MB/s    0:00:27
+    28082176  24%    3.16MB/s    0:00:26
+    31490048  27%    3.25MB/s    0:00:24
+    35946496  31%    3.09MB/s    0:00:24
+    41189376  36%    3.80MB/s    0:00:18
+    42500096  37%    3.03MB/s    0:00:22
+    47972352  42%    3.88MB/s    0:00:16
+    51675136  45%    3.70MB/s    0:00:16
+    55345152  48%    3.13MB/s    0:00:18
+    60948480  53%    4.11MB/s    0:00:12
+    63209472  55%    3.10MB/s    0:00:15
+    67698688  59%    3.28MB/s    0:00:13
+    72646656  64%    3.26MB/s    0:00:12
+    78315520  69%    3.26MB/s    0:00:10
+    81297408  71%    3.36MB/s    0:00:09
+    86900736  76%    3.55MB/s    0:00:07
+    90472448  79%    3.68MB/s    0:00:06
+    94928896  83%    3.23MB/s    0:00:05
+   100433920  88%    3.87MB/s    0:00:03
+   104628224  92%    3.40MB/s    0:00:02
+   110067712  97%    3.85MB/s    0:00:00
+   113325561 100%    3.52MB/s    0:00:30 (xfer#1, to-check=0/1)
+
+sent 113339548 bytes  received 42 bytes  3487372.00 bytes/sec
+total size is 113325561  speedup is 1.00
+[2014-05-25 19:53:36 CEST] Transferrer: Uploaded 08_experi..mvp-s.mp4
+[2014-05-25 19:53:37 CEST] Pusher: Syncing with diskstation 
+
+WORM-s171315323-m1396397078--Lean UX Workshop%09_research.mp4
+       32768   0%    0.00kB/s    0:00:00
+remote: merge git-annex      3964928   2%    3.72MB/s    0:00:43
+     9469952   5%    4.48MB/s    0:00:35
+(merging synced/git-annex into git-annex...)        
+remote: (Recording state in git...)        
+remote: ok        
+To joern@annex.diskstation.local:/volume1/homes/joern/Annex.movies
+   e9861a8..0967b9e  git-annex -> synced/git-annex
+[2014-05-25 19:53:41 CEST] RemoteControl: Syncing with diskstation 
+    12353536   7%    3.86MB/s    0:00:40
+    16252928   9%    3.83MB/s    0:00:39
+    19693568  11%    3.50MB/s    0:00:42
+From annex.diskstation.local:/volume1/homes/joern/Annex.movies
+   6b97695..b05c1b5  git-annex  -> diskstation/git-annex
+    25198592  14%    3.50MB/s    0:00:40
+    29392896  17%    3.78MB/s    0:00:36
+    31227904  18%    3.31MB/s    0:00:41
+    35979264  21%    3.82MB/s    0:00:34
+    38043648  22%    2.87MB/s    0:00:45
+    43548672  25%    3.18MB/s    0:00:39
+    47218688  27%    3.16MB/s    0:00:38
+    52723712  30%    3.26MB/s    0:00:35
+    55607296  32%    3.33MB/s    0:00:33
+    59801600  34%    3.09MB/s    0:00:35
+    65208320  38%    3.87MB/s    0:00:26
+    66879488  39%    3.10MB/s    0:00:32
+    72122368  42%    3.87MB/s    0:00:25
+    75530240  44%    3.32MB/s    0:00:28
+    80510976  46%    3.20MB/s    0:00:27
+    84180992  49%    3.32MB/s    0:00:25
+    89686016  52%    3.39MB/s    0:00:23
+    93880320  54%    3.59MB/s    0:00:21
+    99483648  58%    3.74MB/s    0:00:18
+   103317504  60%    3.83MB/s    0:00:17
+   108822528  63%    3.85MB/s    0:00:15
+   112754688  65%    3.82MB/s    0:00:14
+   118292480  69%    3.81MB/s    0:00:13
+   121405440  70%    3.93MB/s    0:00:12
+   126124032  73%    3.76MB/s    0:00:11
+   128745472  75%    3.68MB/s    0:00:11
+   133988352  78%    3.59MB/s    0:00:10
+   138182656  80%    3.53MB/s    0:00:09
+   143687680  83%    3.66MB/s    0:00:07
+   146833408  85%    3.70MB/s    0:00:06
+   152338432  88%    3.77MB/s    0:00:04
+   155484160  90%    3.70MB/s    0:00:04
+   160989184  93%    3.65MB/s    0:00:02
+   164397056  95%    3.71MB/s    0:00:01
+   169639936  99%    3.67MB/s    0:00:00
+   171315323 100%    3.60MB/s    0:00:45 (xfer#1, to-check=0/1)
+
+sent 171336377 bytes  received 42 bytes  3607082.51 bytes/sec
+total size is 171315323  speedup is 1.00
+[2014-05-25 19:54:24 CEST] Transferrer: Uploaded 09_research.mp4
+[2014-05-25 19:54:24 CEST] Pusher: Syncing with diskstation 
+
+WORM-s254857031-m1396396965--Lean UX Workshop%10_combining-lean-and-agile.mp4
+       32768   0%    0.00kB/s    0:00:00
+     5275648   2%    4.91MB/s    0:00:49
+    10027008   3%    4.72MB/s    0:00:50
+remote: merge git-annex     13139968   5%    3.93MB/s    0:01:00
+    18710528   7%    4.26MB/s    0:00:54
+[2014-05-25 19:54:30 CEST] RemoteControl: Syncing with diskstation 
+(merging synced/git-annex into git-annex...)        
+remote: (Recording state in git...)        
+remote: ok        
+To joern@annex.diskstation.local:/volume1/homes/joern/Annex.movies
+    22052864   8%    3.51MB/s    0:01:04
+   0967b9e..8e9de0b  git-annex -> synced/git-annex
+    27033600  10%    3.53MB/s    0:01:02
+    30441472  11%    3.72MB/s    0:00:58
+From annex.diskstation.local:/volume1/homes/joern/Annex.movies
+   b05c1b5..9285684  git-annex  -> diskstation/git-annex
+error: Ref refs/remotes/diskstation/synced/git-annex is at 8e9de0bb0562a8c9e6a72dcff4ad92f93403ad43 but expected 0967b9e587fc2d85662e7b0c1d047ea77bd1e540
+ ! 0967b9e..8e9de0b  synced/git-annex -> diskstation/synced/git-annex  (unable to update local ref)
+    35815424  14%    3.66MB/s    0:00:58
+    39092224  15%    3.99MB/s    0:00:52
+    43286528  16%    3.83MB/s    0:00:54
+    45645824  17%    3.39MB/s    0:01:00
+    50888704  19%    3.34MB/s    0:00:59
+    54034432  21%    3.04MB/s    0:01:04
+    57966592  22%    3.00MB/s    0:01:04
+    63373312  24%    3.79MB/s    0:00:49
+    64520192  25%    2.95MB/s    0:01:02
+    67403776  26%    3.06MB/s    0:00:59
+    72908800  28%    3.40MB/s    0:00:52
+    76840960  30%    3.06MB/s    0:00:56
+    79986688  31%    3.51MB/s    0:00:48
+    85196800  33%    4.18MB/s    0:00:39
+    87326720  34%    3.20MB/s    0:00:51
+    92831744  36%    3.55MB/s    0:00:44
+    95977472  37%    3.15MB/s    0:00:49
+   100696064  39%    2.97MB/s    0:00:50
+   105676800  41%    3.27MB/s    0:00:44
+   110395392  43%    3.13MB/s    0:00:45
+   115376128  45%    3.84MB/s    0:00:35
+   116686848  45%    3.27MB/s    0:00:41
+   121667584  47%    3.72MB/s    0:00:34
+   125861888  49%    3.56MB/s    0:00:35
+   129007616  50%    2.92MB/s    0:00:42
+   134250496  52%    3.74MB/s    0:00:31
+   137396224  53%    3.01MB/s    0:00:38
+   142835712  56%    3.30MB/s    0:00:33
+   146571264  57%    3.21MB/s    0:00:32
+   150765568  59%    3.02MB/s    0:00:33
+   156270592  61%    3.80MB/s    0:00:25
+   158072832  62%    3.08MB/s    0:00:30
+   163282944  64%    3.87MB/s    0:00:23
+   165445632  64%    2.81MB/s    0:00:31
+   169279488  66%    2.52MB/s    0:00:33
+   173572096  68%    2.96MB/s    0:00:26
+   177242112  69%    2.28MB/s    0:00:33
+   179601408  70%    2.36MB/s    0:00:31
+   183959552  72%    2.44MB/s    0:00:28
+   187203584  73%    2.27MB/s    0:00:29
+   191135744  74%    2.68MB/s    0:00:23
+   196640768  77%    3.84MB/s    0:00:14
+   197427200  77%    2.79MB/s    0:00:20
+   200310784  78%    2.76MB/s    0:00:19
+   205815808  80%    3.13MB/s    0:00:15
+   207650816  81%    2.33MB/s    0:00:19
+   213123072  83%    3.64MB/s    0:00:11
+   217579520  85%    3.99MB/s    0:00:09
+   219185152  86%    3.08MB/s    0:00:11
+   222068736  87%    3.17MB/s    0:00:10
+   227508224  89%    3.15MB/s    0:00:08
+   230457344  90%    2.83MB/s    0:00:08
+   234389504  91%    3.27MB/s    0:00:06
+   238583808  93%    3.81MB/s    0:00:04
+   243204096  95%    3.63MB/s    0:00:03
+   247496704  97%    3.94MB/s    0:00:01
+   249593856  97%    3.61MB/s    0:00:01
+   254312448  99%    3.71MB/s    0:00:00
+   254857031 100%    3.29MB/s    0:01:13 (xfer#1, to-check=0/1)
+
+sent 254888297 bytes  received 42 bytes  3376004.49 bytes/sec
+total size is 254857031  speedup is 1.00
+[2014-05-25 19:55:40 CEST] Transferrer: Uploaded 10_combin..agile.mp4
+[2014-05-25 19:55:40 CEST] Pusher: Syncing with diskstation 
+
+WORM-s67182998-m1396396768--Lean UX Workshop%11_tools-and-techniques.mp4
+       32768   0%    0.00kB/s    0:00:00
+     2392064   3%    2.20MB/s    0:00:28
+remote: merge git-annex      7897088  11%    3.69MB/s    0:00:15
+    12353536  18%    3.75MB/s    0:00:14
+(merging synced/git-annex into git-annex...)        
+remote: (Recording state in git...)        
+remote: ok        
+To joern@annex.diskstation.local:/volume1/homes/joern/Annex.movies
+   8e9de0b..10f88fb  git-annex -> synced/git-annex
+[2014-05-25 19:55:45 CEST] RemoteControl: Syncing with diskstation 
+    13664256  20%    3.12MB/s    0:00:16
+    18644992  27%    3.71MB/s    0:00:12
+    20217856  30%    2.79MB/s    0:00:16
+    24510464  36%    2.82MB/s    0:00:14
+From annex.diskstation.local:/volume1/homes/joern/Annex.movies
+   9285684..b2bad3b  git-annex  -> diskstation/git-annex
+    27557888  41%    3.04MB/s    0:00:12
+    32931840  49%    3.15MB/s    0:00:10
+    34373632  51%    3.09MB/s    0:00:10
+    38830080  57%    3.12MB/s    0:00:08
+    44072960  65%    3.80MB/s    0:00:05
+    45907968  68%    2.96MB/s    0:00:07
+    50626560  75%    3.81MB/s    0:00:04
+    53248000  79%    3.15MB/s    0:00:04
+    57442304  85%    2.95MB/s    0:00:03
+    62717952  93%    3.72MB/s    0:00:01
+    64520192  96%    3.07MB/s    0:00:00
+    67182998 100%    3.28MB/s    0:00:19 (xfer#1, to-check=0/1)
+
+sent 67191351 bytes  received 42 bytes  3125181.07 bytes/sec
+total size is 67182998  speedup is 1.00
+[2014-05-25 19:56:01 CEST] Transferrer: Uploaded 11_tools-..iques.mp4
+[2014-05-25 19:56:01 CEST] Pusher: Syncing with diskstation 
+remote: merge git-annex (merging synced/git-annex into git-annex...)        
+remote: (Recording state in git...)        
+remote: ok        
+To joern@annex.diskstation.local:/volume1/homes/joern/Annex.movies
+   10f88fb..aa5471b  git-annex -> synced/git-annex
+[2014-05-25 19:56:06 CEST] RemoteControl: Syncing with diskstation 
+From annex.diskstation.local:/volume1/homes/joern/Annex.movies
+   b2bad3b..34ef046  git-annex  -> diskstation/git-annex
+[2014-05-25 20:16:23 CEST] NetWatcherFallback: Syncing with diskstation 
+remote: merge git-annex ok        
+To joern@annex.diskstation.local:/volume1/homes/joern/Annex.movies
+   aa5471b..34ef046  git-annex -> synced/git-annex
+[2014-05-25 21:01:01 CEST] main: Syncing with diskstation 
+remote: merge git-annex (merging synced/git-annex into git-annex...)        
+remote: ok        
+To joern@annex.diskstation.local:/volume1/homes/joern/Annex.movies
+   34ef046..2f32e55  git-annex -> synced/git-annex
+
+WORM-s14123840-m1398381779--Learning Corel Painter X3%01_a-quick-tour-of-painter.mp4
+       32768   0%    0.00kB/s    0:00:00
+     5799936  41%    5.46MB/s    0:00:01
+     9994240  70%    4.70MB/s    0:00:00
+    14123840 100%    4.71MB/s    0:00:02 (xfer#1, to-check=0/1)
+
+sent 14125729 bytes  received 42 bytes  4035934.57 bytes/sec
+total size is 14123840  speedup is 1.00
+[2014-05-25 21:01:34 CEST] Transferrer: Uploaded 01_a-quic..inter.mp4
+[2014-05-25 21:01:34 CEST] Pusher: Syncing with diskstation 
+
+WORM-s23356431-m1398381923--Learning Corel Painter X3%02_brushes-and-custom-palettes.mp4
+       32768   0%    0.00kB/s    0:00:00
+     5275648  22%    4.47MB/s    0:00:03
+    10715136  45%    4.81MB/s    0:00:02
+    15237120  65%    4.65MB/s    0:00:01
+    19955712  85%    4.60MB/s    0:00:00
+remote: merge git-annex     23356431 100%    4.51MB/s    0:00:04 (xfer#1, to-check=0/1)
+[2014-05-25 21:01:40 CEST] RemoteControl: Syncing with diskstation 
+
+sent 23359448 bytes  received 42 bytes  4247180.00 bytes/sec
+total size is 23356431  speedup is 1.00
+[2014-05-25 21:01:40 CEST] Transferrer: Uploaded 02_brushe..ettes.mp4
+
+WORM-s16835823-m1398381229--Learning Corel Painter X3%03_managing-layers.mp4
+       32768   0%    0.00kB/s    0:00:00
+(merging synced/git-annex into git-annex...)        
+remote: (Recording state in git...)        
+remote: ok        
+remote: (Recording state in git...)        
+To joern@annex.diskstation.local:/volume1/homes/joern/Annex.movies
+   2f32e55..f2e1569  git-annex -> synced/git-annex
+     6094848  36%    5.76MB/s    0:00:01
+    11010048  65%    5.22MB/s    0:00:01
+    15761408  93%    4.99MB/s    0:00:00
+    16835823 100%    4.96MB/s    0:00:03 (xfer#1, to-check=0/1)
+
+sent 16838032 bytes  received 42 bytes  3741794.22 bytes/sec
+total size is 16835823  speedup is 1.00
+[2014-05-25 21:01:45 CEST] Transferrer: Uploaded 03_managi..ayers.mp4
+
+WORM-s26947174-m1398380993--Learning Corel Painter X3%04_project-red-chairs.mp4
+       32768   0%    0.00kB/s    0:00:00
+[2014-05-25 21:01:46 CEST] Pusher: Syncing with diskstation 
+     5799936  21%    5.47MB/s    0:00:03
+     8421376  31%    3.79MB/s    0:00:04
+From annex.diskstation.local:/volume1/homes/joern/Annex.movies
+   34ef046..28ecf68  git-annex  -> diskstation/git-annex
+error: Ref refs/remotes/diskstation/synced/git-annex is at f2e15690121bf48eba8288426bf72c0661ea444a but expected 2f32e55b5e8bd953707452ada9ad57fbeb20b7b1
+ ! 2f32e55..f2e1569  synced/git-annex -> diskstation/synced/git-annex  (unable to update local ref)
+[2014-05-25 21:01:48 CEST] RemoteControl: Syncing with diskstation 
+    13139968  48%    3.94MB/s    0:00:03
+    17956864  66%    4.10MB/s    0:00:02
+    22577152  83%    3.83MB/s    0:00:01
+From annex.diskstation.local:/volume1/homes/joern/Annex.movies
+   28ecf68..644a4ea  git-annex  -> diskstation/git-annex
+    25460736  94%    3.94MB/s    0:00:00
+    26947174 100%    3.97MB/s    0:00:06 (xfer#1, to-check=0/1)
+
+sent 26950622 bytes  received 42 bytes  3593421.87 bytes/sec
+total size is 26947174  speedup is 1.00
+remote: merge git-annex [2014-05-25 21:01:52 CEST] Transferrer: Uploaded 04_projec..hairs.mp4
+
+WORM-s14550387-m1398382074--Learning Corel Painter X3%05_the-toolbox-part-1.mp4
+       32768   0%    0.00kB/s    0:00:00
+[2014-05-25 21:01:53 CEST] RemoteControl: Syncing with diskstation 
+(merging synced/git-annex into git-annex...)        
+remote: (Recording state in git...)        
+remote: ok        
+To joern@annex.diskstation.local:/volume1/homes/joern/Annex.movies
+   f2e1569..7bf9116  git-annex -> synced/git-annex
+     5668864  38%    5.09MB/s    0:00:01
+    10289152  70%    4.74MB/s    0:00:00
+    14550387 100%    4.56MB/s    0:00:03 (xfer#1, to-check=0/1)
+
+sent 14552323 bytes  received 42 bytes  3233858.89 bytes/sec
+total size is 14550387  speedup is 1.00
+[2014-05-25 21:01:57 CEST] Transferrer: Uploaded 05_the-to..art-1.mp4
+From annex.diskstation.local:/volume1/homes/joern/Annex.movies
+   644a4ea..f3817af  git-annex  -> diskstation/git-annex
+error: Ref refs/remotes/diskstation/synced/git-annex is at 7bf9116bbeb90b398ac29d7b38e78dbe493f081f but expected f2e15690121bf48eba8288426bf72c0661ea444a
+ ! f2e1569..7bf9116  synced/git-annex -> diskstation/synced/git-annex  (unable to update local ref)
+
+WORM-s5537051-m1398381582--Learning Corel Painter X3%06_the-toolbox-part-2.mp4
+       32768   0%    0.00kB/s    0:00:00
+     4227072  76%    4.00MB/s    0:00:00
+[2014-05-25 21:01:58 CEST] Pusher: Syncing with diskstation 
+     5537051 100%    4.47MB/s    0:00:01 (xfer#1, to-check=0/1)
+
+sent 5537882 bytes  received 42 bytes  3691949.33 bytes/sec
+total size is 5537051  speedup is 1.00
+[2014-05-25 21:02:00 CEST] Transferrer: Uploaded 06_the-to..art-2.mp4
+
+WORM-s16270822-m1398381489--Learning Corel Painter X3%07_color-panels.mp4
+       32768   0%    0.00kB/s    0:00:00
+     4882432  30%    4.62MB/s    0:00:02
+remote: merge git-annex      7634944  46%    3.29MB/s    0:00:02
+[2014-05-25 21:02:03 CEST] RemoteControl: Syncing with diskstation 
+(merging synced/git-annex into git-annex...)        
+remote: (Recording state in git...)        
+remote: ok        
+To joern@annex.diskstation.local:/volume1/homes/joern/Annex.movies
+   7bf9116..a826827  git-annex -> synced/git-annex
+     9994240  61%    2.94MB/s    0:00:02
+    13369344  82%    3.01MB/s    0:00:00
+    15761408  96%    2.44MB/s    0:00:00
+    16270822 100%    2.91MB/s    0:00:05 (xfer#1, to-check=0/1)
+
+sent 16272960 bytes  received 42 bytes  2503538.77 bytes/sec
+total size is 16270822  speedup is 1.00
+[2014-05-25 21:02:06 CEST] Transferrer: Uploaded 07_color-panels.mp4
+From annex.diskstation.local:/volume1/homes/joern/Annex.movies
+   f3817af..bff569c  git-annex  -> diskstation/git-annex
+error: Ref refs/remotes/diskstation/synced/git-annex is at a8268279606cf4fdee4e0fabeef0c9075d840f71 but expected 7bf9116bbeb90b398ac29d7b38e78dbe493f081f
+ ! 7bf9116..a826827  synced/git-annex -> diskstation/synced/git-annex  (unable to update local ref)
+
+WORM-s21555439-m1398381255--Learning Corel Painter X3%08_paper-and-media-libraries.mp4
+       32768   0%    0.00kB/s    0:00:00
+     5111808  23%    4.81MB/s    0:00:03
+[2014-05-25 21:02:08 CEST] Pusher: Syncing with diskstation 
+     9175040  42%    4.29MB/s    0:00:02
+    13369344  62%    4.18MB/s    0:00:01
+    17563648  81%    4.10MB/s    0:00:00
+    21555439 100%    4.05MB/s    0:00:05 (xfer#1, to-check=0/1)
+
+sent 21558234 bytes  received 42 bytes  3316657.85 bytes/sec
+total size is 21555439  speedup is 1.00
+[2014-05-25 21:02:12 CEST] Transferrer: Uploaded 08_paper-..aries.mp4
+remote: merge git-annex 
+WORM-s24881920-m1398382026--Learning Corel Painter X3%09_menu-commands-part-1.mp4
+       32768   0%    0.00kB/s    0:00:00
+     5210112  20%    4.80MB/s    0:00:03
+(merging synced/git-annex into git-annex...)        
+remote: (Recording state in git...)        
+remote: ok        
+[2014-05-25 21:02:14 CEST] RemoteControl: Syncing with diskstation 
+To joern@annex.diskstation.local:/volume1/homes/joern/Annex.movies
+   a826827..e65041f  git-annex -> synced/git-annex
+     9142272  36%    4.28MB/s    0:00:03
+    13041664  52%    4.08MB/s    0:00:02
+    17235968  69%    4.03MB/s    0:00:01
+From annex.diskstation.local:/volume1/homes/joern/Annex.movies
+   bff569c..b552f10  git-annex  -> diskstation/git-annex
+error: Ref refs/remotes/diskstation/synced/git-annex is at e65041fc45b14af68426581456deb8d3e9efa1d9 but expected a8268279606cf4fdee4e0fabeef0c9075d840f71
+ ! a826827..e65041f  synced/git-annex -> diskstation/synced/git-annex  (unable to update local ref)
+    21397504  85%    3.81MB/s    0:00:00
+    24881920 100%    4.05MB/s    0:00:05 (xfer#1, to-check=0/1)
+[2014-05-25 21:02:19 CEST] Pusher: Syncing with diskstation 
+
+sent 24885118 bytes  received 42 bytes  3318021.33 bytes/sec
+total size is 24881920  speedup is 1.00
+[2014-05-25 21:02:20 CEST] Transferrer: Uploaded 09_menu-c..art-1.mp4
+
+WORM-s29078392-m1398382059--Learning Corel Painter X3%10_menu-commands-part-2.mp4
+       32768   0%    0.00kB/s    0:00:00
+remote: merge git-annex      5210112  17%    4.81MB/s    0:00:04
+     9437184  32%    4.41MB/s    0:00:04
+[2014-05-25 21:02:24 CEST] RemoteControl: Syncing with diskstation 
+(merging synced/git-annex into git-annex...)        
+remote: (Recording state in git...)        
+remote: ok        
+To joern@annex.diskstation.local:/volume1/homes/joern/Annex.movies
+   e65041f..530d23c  git-annex -> synced/git-annex
+    13762560  47%    4.30MB/s    0:00:03
+    17924096  61%    4.20MB/s    0:00:02
+    22970368  78%    4.16MB/s    0:00:01
+From annex.diskstation.local:/volume1/homes/joern/Annex.movies
+   b552f10..4dc60d5  git-annex  -> diskstation/git-annex
+error: Ref refs/remotes/diskstation/synced/git-annex is at 530d23cf75a7f936041cdb408fd4f3516fe1514b but expected e65041fc45b14af68426581456deb8d3e9efa1d9
+ ! e65041f..530d23c  synced/git-annex -> diskstation/synced/git-annex  (unable to update local ref)
+    27688960  95%    4.26MB/s    0:00:00
+    29078392 100%    4.33MB/s    0:00:06 (xfer#1, to-check=0/1)
+
+sent 29082102 bytes  received 42 bytes  3421428.71 bytes/sec
+total size is 29078392  speedup is 1.00
+[2014-05-25 21:02:29 CEST] Transferrer: Uploaded 10_menu-c..art-2.mp4
+
+WORM-s18704564-m1398381759--Learning Corel Painter X3%11_cloner-brushes.mp4
+       32768   0%    0.00kB/s    0:00:00
+[2014-05-25 21:02:29 CEST] Pusher: Syncing with diskstation 
+     5996544  32%    5.68MB/s    0:00:02
+    10321920  55%    4.86MB/s    0:00:01
+    14581760  77%    4.55MB/s    0:00:00
+    18704564 100%    4.39MB/s    0:00:00
+    18704564 100%    4.39MB/s    0:00:04 (xfer#1, to-check=0/1)
+
+sent 18707000 bytes  received 42 bytes  4157120.44 bytes/sec
+total size is 18704564  speedup is 1.00
+[2014-05-25 21:02:33 CEST] Transferrer: Uploaded 11_cloner..ushes.mp4
+
+WORM-s16483665-m1398381099--Learning Corel Painter X3%12_project-clone-painting-a-pear.mp4
+       32768   0%    0.00kB/s    0:00:00
+remote: merge git-annex      3964928  24%    3.69MB/s    0:00:03
+(merging synced/git-annex into git-annex...)        
+remote: (Recording state in git...)        
+remote: ok        
+[2014-05-25 21:02:36 CEST] RemoteControl: Syncing with diskstation 
+To joern@annex.diskstation.local:/volume1/homes/joern/Annex.movies
+   530d23c..821b643  git-annex -> synced/git-annex
+     8683520  52%    3.90MB/s    0:00:01
+    12877824  78%    3.90MB/s    0:00:00
+    16483665 100%    4.01MB/s    0:00:03 (xfer#1, to-check=0/1)
+
+sent 16485848 bytes  received 42 bytes  2997434.55 bytes/sec
+total size is 16483665  speedup is 1.00
+[2014-05-25 21:02:38 CEST] Transferrer: Uploaded 12_projec..-pear.mp4
+
+WORM-s13729745-m1398381011--Learning Corel Painter X3%13_dab-stroke-and-method.mp4
+       32768   0%    0.00kB/s    0:00:00
+From annex.diskstation.local:/volume1/homes/joern/Annex.movies
+   4dc60d5..b35b714  git-annex  -> diskstation/git-annex
+error: Ref refs/remotes/diskstation/synced/git-annex is at 821b643cf14025648cceac17c2de27e1a17840e3 but expected 530d23cf75a7f936041cdb408fd4f3516fe1514b
+ ! 530d23c..821b643  synced/git-annex -> diskstation/synced/git-annex  (unable to update local ref)
+     5013504  36%    4.35MB/s    0:00:01
+[2014-05-25 21:02:41 CEST] Pusher: Syncing with diskstation 
+     9469952  68%    4.23MB/s    0:00:00
+    13402112  97%    4.06MB/s    0:00:00
+    13729745 100%    3.98MB/s    0:00:03 (xfer#1, to-check=0/1)
+
+sent 13731580 bytes  received 42 bytes  3923320.57 bytes/sec
+total size is 13729745  speedup is 1.00
+[2014-05-25 21:02:43 CEST] Transferrer: Uploaded 13_dab-st..ethod.mp4
+
+WORM-s15796088-m1398381069--Learning Corel Painter X3%14_advanced-controls.mp4
+       32768   0%    0.00kB/s    0:00:00
+     5013504  31%    4.63MB/s    0:00:02
+     9207808  58%    4.25MB/s    0:00:01
+    13664256  86%    4.21MB/s    0:00:00
+    15796088 100%    4.21MB/s    0:00:03 (xfer#1, to-check=0/1)
+
+sent 15798175 bytes  received 42 bytes  3510714.89 bytes/sec
+total size is 15796088  speedup is 1.00
+[2014-05-25 21:02:47 CEST] Transferrer: Uploaded 14_advanc..trols.mp4
+
+WORM-s14529732-m1398382119--Learning Corel Painter X3%15_custom-variants.mp4
+       32768   0%    0.00kB/s    0:00:00
+     5996544  41%    5.49MB/s    0:00:01
+remote: merge git-annex     10551296  72%    4.90MB/s    0:00:00
+    14529732 100%    4.82MB/s    0:00:02 (xfer#1, to-check=0/1)
+
+sent 14531661 bytes  received 42 bytes  2235646.62 bytes/sec
+total size is 14529732  speedup is 1.00
+[2014-05-25 21:02:54 CEST] RemoteControl: Syncing with diskstation 
+[2014-05-25 21:02:54 CEST] Transferrer: Uploaded 15_custom..iants.mp4
+(merging synced/git-annex into git-annex...)        
+remote: (Recording state in git...)        
+remote: (Recording state in git...)        
+remote: ok        
+To joern@annex.diskstation.local:/volume1/homes/joern/Annex.movies
+   821b643..d1a88a1  git-annex -> synced/git-annex
+
+WORM-s20256713-m1398381124--Learning Corel Painter X3%16_abstract-painting.mp4
+       32768   0%    0.00kB/s    0:00:00
+     5242880  25%    4.82MB/s    0:00:03
+     9437184  46%    4.38MB/s    0:00:02
+    13729792  67%    4.28MB/s    0:00:01
+From annex.diskstation.local:/volume1/homes/joern/Annex.movies
+   b35b714..5d9e777  git-annex  -> diskstation/git-annex
+error: Ref refs/remotes/diskstation/synced/git-annex is at d1a88a1d1fb37d139753d3b96cbd570810a61a51 but expected 821b643cf14025648cceac17c2de27e1a17840e3
+ ! 821b643..d1a88a1  synced/git-annex -> diskstation/synced/git-annex  (unable to update local ref)
+    17924096  88%    4.19MB/s    0:00:00
+    20256713 100%    4.15MB/s    0:00:04 (xfer#1, to-check=0/1)
+
+sent 20259344 bytes  received 42 bytes  3116828.62 bytes/sec
+total size is 20256713  speedup is 1.00
+[2014-05-25 21:03:00 CEST] Transferrer: Uploaded 16_abstra..nting.mp4
+
+WORM-s15821561-m1398381954--Learning Corel Painter X3%17_tonal-controls.mp4
+       32768   0%    0.00kB/s    0:00:00
+[2014-05-25 21:03:01 CEST] Pusher: Syncing with diskstation 
+     3964928  25%    3.50MB/s    0:00:03
+     6848512  43%    3.13MB/s    0:00:02
+    11829248  74%    3.62MB/s    0:00:01
+    15821561 100%    3.70MB/s    0:00:04 (xfer#1, to-check=0/1)
+
+sent 15823645 bytes  received 42 bytes  2877034.00 bytes/sec
+total size is 15821561  speedup is 1.00
+[2014-05-25 21:03:05 CEST] Transferrer: Uploaded 17_tonal-..trols.mp4
+remote: merge git-annex 
+WORM-s27686314-m1398381815--Learning Corel Painter X3%18_surface-controls.mp4
+       32768   0%    0.00kB/s    0:00:00
+     5210112  18%    4.91MB/s    0:00:04
+[2014-05-25 21:03:08 CEST] RemoteControl: Syncing with diskstation 
+     9142272  33%    4.29MB/s    0:00:04
+(merging synced/git-annex into git-annex...)        
+remote: (Recording state in git...)        
+remote: ok        
+To joern@annex.diskstation.local:/volume1/homes/joern/Annex.movies
+   d1a88a1..2f6a023  git-annex -> synced/git-annex
+    12943360  46%    4.06MB/s    0:00:03
+    17104896  61%    4.01MB/s    0:00:02
+From annex.diskstation.local:/volume1/homes/joern/Annex.movies
+   5d9e777..5732a07  git-annex  -> diskstation/git-annex
+error: Ref refs/remotes/diskstation/synced/git-annex is at 2f6a0230e37a27a287290d40a66a61284abd958e but expected d1a88a1d1fb37d139753d3b96cbd570810a61a51
+ ! d1a88a1..2f6a023  synced/git-annex -> diskstation/synced/git-annex  (unable to update local ref)
+    21266432  76%    3.76MB/s    0:00:01
+    25591808  92%    3.85MB/s    0:00:00
+[2014-05-25 21:06:06 CEST] Pusher: Syncing with diskstation 
+    26640384  96%   73.39kB/s    0:00:14
+    27686314 100%  145.74kB/s    0:03:05 (xfer#1, to-check=0/1)
+
+sent 27689848 bytes  received 42 bytes  148471.26 bytes/sec
+total size is 27686314  speedup is 1.00
+[2014-05-25 21:06:11 CEST] Transferrer: Uploaded 18_surfac..trols.mp4
+remote: merge git-annex 
+WORM-s20041059-m1398382199--Learning Corel Painter X3%19_esoterica.mp4
+       32768   0%    0.00kB/s    0:00:00
+[2014-05-25 21:06:13 CEST] RemoteControl: Syncing with diskstation 
+(merging synced/git-annex into git-annex...)        
+remote: (Recording state in git...)        
+remote: ok        
+To joern@annex.diskstation.local:/volume1/homes/joern/Annex.movies
+     5013504  25%    4.74MB/s    0:00:03
+   2f6a023..3de6568  git-annex -> synced/git-annex
+     8421376  42%    3.99MB/s    0:00:02
+    12681216  63%    4.01MB/s    0:00:01
+From annex.diskstation.local:/volume1/homes/joern/Annex.movies
+   5732a07..22c09fa  git-annex  -> diskstation/git-annex
+error: Ref refs/remotes/diskstation/synced/git-annex is at 3de6568fc3634c50a42c935f315c763106a22e3c but expected 2f6a0230e37a27a287290d40a66a61284abd958e
+ ! 2f6a023..3de6568  synced/git-annex -> diskstation/synced/git-annex  (unable to update local ref)
+    17596416  87%    4.18MB/s    0:00:00
+    20041059 100%    4.24MB/s    0:00:04 (xfer#1, to-check=0/1)
+[2014-05-25 21:06:17 CEST] Pusher: Syncing with diskstation 
+
+sent 20043654 bytes  received 42 bytes  3083645.54 bytes/sec
+total size is 20041059  speedup is 1.00
+[2014-05-25 21:06:18 CEST] Transferrer: Uploaded 19_esoterica.mp4
+
+WORM-s20049901-m1398382237--Learning Corel Painter X3%20_esoterica-mosaic.mp4
+       32768   0%    0.00kB/s    0:00:00
+     4227072  21%    3.93MB/s    0:00:03
+remote: merge git-annex      7372800  36%    3.43MB/s    0:00:03
+(merging synced/git-annex into git-annex...)        
+remote: (Recording state in git...)        
+remote: ok        
+[2014-05-25 21:06:22 CEST] RemoteControl: Syncing with diskstation 
+To joern@annex.diskstation.local:/volume1/homes/joern/Annex.movies
+   3de6568..91bf58d  git-annex -> synced/git-annex
+    11042816  55%    3.45MB/s    0:00:02
+    14712832  73%    3.45MB/s    0:00:01
+    17858560  89%    3.20MB/s    0:00:00
+    20049901 100%    3.44MB/s    0:00:05 (xfer#1, to-check=0/1)
+From annex.diskstation.local:/volume1/homes/joern/Annex.movies
+   22c09fa..f2cac44  git-annex  -> diskstation/git-annex
+error: Ref refs/remotes/diskstation/synced/git-annex is at 91bf58d7240d46f8eee363bd3b56dde6deefc012 but expected 3de6568fc3634c50a42c935f315c763106a22e3c
+ ! 3de6568..91bf58d  synced/git-annex -> diskstation/synced/git-annex  (unable to update local ref)
+
+sent 20052503 bytes  received 42 bytes  2673672.67 bytes/sec
+total size is 20049901  speedup is 1.00
+[2014-05-25 21:06:25 CEST] Transferrer: Uploaded 20_esoter..osaic.mp4
+
+WORM-s18181651-m1398381889--Learning Corel Painter X3%21_selection-tools-and-commands.mp4
+       32768   0%    0.00kB/s    0:00:00
+[2014-05-25 21:06:26 CEST] Pusher: Syncing with diskstation 
+     4751360  26%    4.38MB/s    0:00:02
+     8421376  46%    3.89MB/s    0:00:02
+    12091392  66%    3.71MB/s    0:00:01
+    15761408  86%    3.64MB/s    0:00:00
+    18181651 100%    3.65MB/s    0:00:04 (xfer#1, to-check=0/1)
+remote: merge git-annex 
+sent 18184037 bytes  received 42 bytes  2797550.62 bytes/sec
+total size is 18181651  speedup is 1.00
+[2014-05-25 21:06:31 CEST] Transferrer: Uploaded 21_select..mands.mp4
+
+WORM-s16431574-m1398381365--Learning Corel Painter X3%22_text-properties.mp4
+       32768   0%    0.00kB/s    0:00:00
+     5013504  30%    4.71MB/s    0:00:02
+[2014-05-25 21:06:32 CEST] RemoteControl: Syncing with diskstation 
+(merging synced/git-annex into git-annex...)        
+remote: (Recording state in git...)        
+remote: ok        
+To joern@annex.diskstation.local:/volume1/homes/joern/Annex.movies
+   91bf58d..9341c6c  git-annex -> synced/git-annex
+     8355840  50%    3.82MB/s    0:00:02
+    11763712  71%    3.60MB/s    0:00:01
+    15466496  94%    3.57MB/s    0:00:00
+From annex.diskstation.local:/volume1/homes/joern/Annex.movies
+   f2cac44..70eb4e9  git-annex  -> diskstation/git-annex
+error: Ref refs/remotes/diskstation/synced/git-annex is at 9341c6cf9e5fe11181099779b7248a9debecaa26 but expected 91bf58d7240d46f8eee363bd3b56dde6deefc012
+ ! 91bf58d..9341c6c  synced/git-annex -> diskstation/synced/git-annex  (unable to update local ref)
+    16431574 100%    3.62MB/s    0:00:04 (xfer#1, to-check=0/1)
+
+sent 16433735 bytes  received 42 bytes  2987959.45 bytes/sec
+total size is 16431574  speedup is 1.00
+[2014-05-25 21:06:36 CEST] Transferrer: Uploaded 22_text-p..rties.mp4
+
+WORM-s17364419-m1398381850--Learning Corel Painter X3%23_text-effects.mp4
+       32768   0%    0.00kB/s    0:00:00
+[2014-05-25 21:06:37 CEST] Pusher: Syncing with diskstation 
+     5046272  29%    4.76MB/s    0:00:02
+     8421376  48%    3.97MB/s    0:00:02
+    12255232  70%    3.87MB/s    0:00:01
+    16023552  92%    3.79MB/s    0:00:00
+remote: merge git-annex     17364419 100%    3.71MB/s    0:00:04 (xfer#1, to-check=0/1)
+
+sent 17366689 bytes  received 42 bytes  3157587.45 bytes/sec
+total size is 17364419  speedup is 1.00
+[2014-05-25 21:06:42 CEST] RemoteControl: Syncing with diskstation 
+[2014-05-25 21:06:42 CEST] Transferrer: Uploaded 23_text-effects.mp4
+
+WORM-s19708183-m1398382290--Learning Corel Painter X3%24_postcard-graphic-part-1.mp4
+       32768   0%    0.00kB/s    0:00:00
+(merging synced/git-annex into git-annex...)        
+remote: (Recording state in git...)        
+remote: ok        
+remote: (Recording state in git...)        
+To joern@annex.diskstation.local:/volume1/homes/joern/Annex.movies
+   9341c6c..08cb8d8  git-annex -> synced/git-annex
+     4489216  22%    3.60MB/s    0:00:04
+From annex.diskstation.local:/volume1/homes/joern/Annex.movies
+   70eb4e9..dce2d61  git-annex  -> diskstation/git-annex
+error: Ref refs/remotes/diskstation/synced/git-annex is at 08cb8d8ba4b56c67984027263d5213133dcdaad2 but expected 9341c6cf9e5fe11181099779b7248a9debecaa26
+ ! 9341c6c..08cb8d8  synced/git-annex -> diskstation/synced/git-annex  (unable to update local ref)
+[2014-05-25 21:06:44 CEST] RemoteControl: Syncing with diskstation 
+     9469952  48%    4.08MB/s    0:00:02
+From annex.diskstation.local:/volume1/homes/joern/Annex.movies
+   dce2d61..ac9201d  git-annex  -> diskstation/git-annex
+    13664256  69%    3.97MB/s    0:00:01
+    17301504  87%    3.86MB/s    0:00:00
+[2014-05-25 21:06:47 CEST] Pusher: Syncing with diskstation 
+    19708183 100%    3.96MB/s    0:00:04 (xfer#1, to-check=0/1)
+
+sent 19710752 bytes  received 42 bytes  3032429.85 bytes/sec
+total size is 19708183  speedup is 1.00
+[2014-05-25 21:06:48 CEST] Transferrer: Uploaded 24_postca..art-1.mp4
+
+WORM-s21565207-m1398381554--Learning Corel Painter X3%25_postcard-graphic-part-2.mp4
+       32768   0%    0.00kB/s    0:00:00
+     5439488  25%    5.07MB/s    0:00:03
+remote: merge git-annex      9306112  43%    4.32MB/s    0:00:02
+[2014-05-25 21:06:51 CEST] RemoteControl: Syncing with diskstation 
+(merging synced/git-annex into git-annex...)        
+remote: (Recording state in git...)        
+remote: ok        
+To joern@annex.diskstation.local:/volume1/homes/joern/Annex.movies
+   08cb8d8..1212159  git-annex -> synced/git-annex
+    13107200  60%    4.05MB/s    0:00:02
+    16449536  76%    3.84MB/s    0:00:01
+From annex.diskstation.local:/volume1/homes/joern/Annex.movies
+   ac9201d..a885109  git-annex  -> diskstation/git-annex
+error: Ref refs/remotes/diskstation/synced/git-annex is at 12121595714360b6216de28d0beef79305ec276d but expected 08cb8d8ba4b56c67984027263d5213133dcdaad2
+ ! 08cb8d8..1212159  synced/git-annex -> diskstation/synced/git-annex  (unable to update local ref)
+    20250624  93%    3.46MB/s    0:00:00
+    21565207 100%    3.75MB/s    0:00:05 (xfer#1, to-check=0/1)
+
+sent 21568004 bytes  received 42 bytes  2875739.47 bytes/sec
+total size is 21565207  speedup is 1.00
+[2014-05-25 21:06:55 CEST] Transferrer: Uploaded 25_postca..art-2.mp4
+
+WORM-s20239101-m1398380964--Learning Corel Painter X3%26_dynamic-layers.mp4
+       32768   0%    0.00kB/s    0:00:00
+[2014-05-25 21:06:56 CEST] Pusher: Syncing with diskstation 
+     5275648  26%    4.78MB/s    0:00:03
+     9404416  46%    4.35MB/s    0:00:02
+    12943360  63%    4.03MB/s    0:00:01
+    16384000  80%    3.85MB/s    0:00:00
+remote: merge git-annex     20152320  99%    3.52MB/s    0:00:00
+    20239101 100%    3.77MB/s    0:00:05 (xfer#1, to-check=0/1)
+[2014-05-25 21:07:01 CEST] RemoteControl: Syncing with diskstation 
+
+sent 20241725 bytes  received 42 bytes  3114118.00 bytes/sec
+total size is 20239101  speedup is 1.00
+[2014-05-25 21:07:02 CEST] Transferrer: Uploaded 26_dynami..ayers.mp4
+(merging synced/git-annex into git-annex...)        
+remote: (Recording state in git...)        
+remote: ok        
+remote: (Recording state in git...)        
+To joern@annex.diskstation.local:/volume1/homes/joern/Annex.movies
+   1212159..4c20688  git-annex -> synced/git-annex
+
+WORM-s17613999-m1398382264--Learning Corel Painter X3%27_auto-painting.mp4
+       32768   0%    0.00kB/s    0:00:00
+     5275648  29%    4.78MB/s    0:00:02
+From annex.diskstation.local:/volume1/homes/joern/Annex.movies
+   a885109..ac9fbda  git-annex  -> diskstation/git-annex
+error: Ref refs/remotes/diskstation/synced/git-annex is at 4c206880fc1f99dd380b0fb28ebb780b72171633 but expected 12121595714360b6216de28d0beef79305ec276d
+ ! 1212159..4c20688  synced/git-annex -> diskstation/synced/git-annex  (unable to update local ref)
+     9175040  52%    4.26MB/s    0:00:01
+    12845056  72%    4.00MB/s    0:00:01
+    16384000  93%    3.82MB/s    0:00:00
+[2014-05-25 21:07:07 CEST] Pusher: Syncing with diskstation 
+    17613999 100%    3.79MB/s    0:00:04 (xfer#1, to-check=0/1)
+
+sent 17616302 bytes  received 42 bytes  3202971.64 bytes/sec
+total size is 17613999  speedup is 1.00
+[2014-05-25 21:07:08 CEST] Transferrer: Uploaded 27_auto-p..nting.mp4
+
+WORM-s15562581-m1398381274--Learning Corel Painter X3%28_smart-stroke-cloning.mp4
+       32768   0%    0.00kB/s    0:00:00
+remote: merge git-annex      5275648  33%    4.77MB/s    0:00:02
+     9469952  60%    4.34MB/s    0:00:01
+[2014-05-25 21:07:11 CEST] RemoteControl: Syncing with diskstation 
+(merging synced/git-annex into git-annex...)        
+remote: (Recording state in git...)        
+remote: ok        
+To joern@annex.diskstation.local:/volume1/homes/joern/Annex.movies
+   4c20688..f7be7ec  git-annex -> synced/git-annex
+    13139968  84%    4.06MB/s    0:00:00
+    15562581 100%    3.89MB/s    0:00:03 (xfer#1, to-check=0/1)
+
+sent 15564639 bytes  received 42 bytes  3458818.00 bytes/sec
+total size is 15562581  speedup is 1.00
+[2014-05-25 21:07:13 CEST] Transferrer: Uploaded 28_smart-..oning.mp4
+From annex.diskstation.local:/volume1/homes/joern/Annex.movies
+   ac9fbda..4f23f91  git-annex  -> diskstation/git-annex
+error: Ref refs/remotes/diskstation/synced/git-annex is at f7be7ececec0dd7a424dfc71571fc2a6cfc63bea but expected 4c206880fc1f99dd380b0fb28ebb780b72171633
+ ! 4c20688..f7be7ec  synced/git-annex -> diskstation/synced/git-annex  (unable to update local ref)
+
+WORM-s28205928-m1398382150--Learning Corel Painter X3%29_mirror-and-kaleidoscope.mp4
+       32768   0%    0.00kB/s    0:00:00
+     5537792  19%    4.94MB/s    0:00:04
+[2014-05-25 21:07:15 CEST] Pusher: Syncing with diskstation 
+     9994240  35%    4.54MB/s    0:00:03
+    14155776  50%    4.34MB/s    0:00:03
+    17956864  63%    4.16MB/s    0:00:02
+    21725184  77%    3.79MB/s    0:00:01
+remote: merge git-annex     25198592  89%    3.58MB/s    0:00:00
+    28205928 100%    3.85MB/s    0:00:06 (xfer#1, to-check=0/1)
+(merging synced/git-annex into git-annex...)        
+remote: (Recording state in git...)        
+remote: ok        
+To joern@annex.diskstation.local:/volume1/homes/joern/Annex.movies
+[2014-05-25 21:07:21 CEST] RemoteControl: Syncing with diskstation 
+
+sent 28209533 bytes  received 42 bytes  3761276.67 bytes/sec
+total size is 28205928  speedup is 1.00
+   f7be7ec..5c452a1  git-annex -> synced/git-annex
+[2014-05-25 21:07:21 CEST] Transferrer: Uploaded 29_mirror..scope.mp4
+
+WORM-s21976288-m1398382005--Learning Corel Painter X3%30_introduction-to-mixed-media.mp4
+       32768   0%    0.00kB/s    0:00:00
+     5406720  24%    5.07MB/s    0:00:03
+From annex.diskstation.local:/volume1/homes/joern/Annex.movies
+   4f23f91..1ca4624  git-annex  -> diskstation/git-annex
+error: Ref refs/remotes/diskstation/synced/git-annex is at 5c452a15506c41b30a4ff7099c02f5dae74db8db but expected f7be7ececec0dd7a424dfc71571fc2a6cfc63bea
+ ! f7be7ec..5c452a1  synced/git-annex -> diskstation/synced/git-annex  (unable to update local ref)
+[2014-05-25 21:22:27 CEST] Pusher: Syncing with diskstation 
+remote: merge git-annex (merging synced/git-annex into git-annex...)        
+remote: (Recording state in git...)        
+remote: ok        
+To joern@annex.diskstation.local:/volume1/homes/joern/Annex.movies
+   5c452a1..de02d45  git-annex -> synced/git-annex
+     8486912  38%    9.03kB/s    0:24:53
+[2014-05-25 21:22:35 CEST] RemoteControl: Syncing with diskstation 
+    12353536  56%   13.15kB/s    0:12:11
+    15761408  71%   16.76kB/s    0:06:10
+    18644992  84%   14.11kB/s    0:03:56
+From annex.diskstation.local:/volume1/homes/joern/Annex.movies
+   1ca4624..0fddca6  git-annex  -> diskstation/git-annex
+    21921792  99%    3.02MB/s    0:00:00
+    21976288 100%   23.34kB/s    0:15:18 (xfer#1, to-check=0/1)
+
+sent 21979137 bytes  received 42 bytes  23903.40 bytes/sec
+total size is 21976288  speedup is 1.00
+[2014-05-25 21:22:40 CEST] Transferrer: Uploaded 30_introd..media.mp4
+[2014-05-25 21:22:40 CEST] Pusher: Syncing with diskstation 
+
+WORM-s20759393-m1398382174--Learning Corel Painter X3%31_portrait-of-stephen.mp4
+       32768   0%    0.00kB/s    0:00:00
+     4194304  20%    3.79MB/s    0:00:04
+     6815744  32%    3.15MB/s    0:00:04
+remote: merge git-annex      9306112  44%    2.89MB/s    0:00:03
+    12320768  59%    2.87MB/s    0:00:02
+[2014-05-25 21:22:45 CEST] RemoteControl: Syncing with diskstation 
+/Applications/git-annex.app/Contents/MacOS/git-annex: fork: Resource temporarily unavailable
+fatal: Could not read from remote repository.
+
+Please make sure you have the correct access rights
+and the repository exists.
+(merging synced/git-annex into git-annex...)        
+remote: (Recording state in git...)        
+remote: ok        
+To joern@annex.diskstation.local:/volume1/homes/joern/Annex.movies
+   de02d45..251064b  git-annex -> synced/git-annex
+    15073280  72%    2.55MB/s    0:00:02
+    17563648  84%    2.52MB/s    0:00:01
+    20316160  97%    2.56MB/s    0:00:00
+    20759393 100%    2.71MB/s    0:00:07 (xfer#1, to-check=0/1)
+
+sent 20762086 bytes  received 42 bytes  2185487.16 bytes/sec
+total size is 20759393  speedup is 1.00
+[2014-05-25 21:22:49 CEST] Transferrer: Uploaded 31_portra..ephen.mp4
+
+WORM-s25441460-m1398381650--Learning Corel Painter X3%32_cel-animation.mp4
+       32768   0%    0.00kB/s    0:00:00
+[2014-05-25 21:22:50 CEST] Pusher: Syncing with diskstation 
+Pusher crashed: git: createProcess: resource exhausted (Resource temporarily unavailable)
+[2014-05-25 21:22:50 CEST] Pusher: warning Pusher crashed: git: createProcess: resource exhausted (Resource temporarily unavailable)
+     4096000  16%    3.83MB/s    0:00:05
+     6979584  27%    3.27MB/s    0:00:05
+     9469952  37%    2.97MB/s    0:00:05
+    12091392  47%    2.78MB/s    0:00:04
+    13664256  53%    2.19MB/s    0:00:05
+    16023552  62%    2.04MB/s    0:00:04
+    18644992  73%    2.07MB/s    0:00:03
+    21266432  83%    2.10MB/s    0:00:01
+    23625728  92%    2.25MB/s    0:00:00
+    25441460 100%    2.36MB/s    0:00:10 (xfer#1, to-check=0/1)
+
+sent 25444719 bytes  received 42 bytes  2212587.91 bytes/sec
+total size is 25441460  speedup is 1.00
+[2014-05-25 21:23:00 CEST] Transferrer: Uploaded 32_cel-an..ation.mp4
+
+WORM-s23123916-m1398381154--Learning Corel Painter X3%33_rotoscope-techniques.mp4
+       32768   0%    0.00kB/s    0:00:00
+     2129920   9%    1.98MB/s    0:00:10
+     5275648  22%    2.45MB/s    0:00:07
+     8945664  38%    2.77MB/s    0:00:04
+    12353536  53%    2.89MB/s    0:00:03
+    15269888  66%    3.06MB/s    0:00:02
+    18022400  77%    2.99MB/s    0:00:01
+    20742144  89%    2.78MB/s    0:00:00
+    23123916 100%    2.74MB/s    0:00:08 (xfer#1, to-check=0/1)
+
+sent 23126898 bytes  received 42 bytes  2434414.74 bytes/sec
+total size is 23123916  speedup is 1.00
+[2014-05-25 21:23:10 CEST] Transferrer: Uploaded 33_rotosc..iques.mp4
+
+WORM-s16168686-m1398381345--Learning Corel Painter X3%34_portrait-of-hines-part-1.mp4
+       32768   0%    0.00kB/s    0:00:00
+     4194304  25%    3.85MB/s    0:00:03
+     7340032  45%    3.40MB/s    0:00:02
+    10256384  63%    3.19MB/s    0:00:01
+    13500416  83%    3.16MB/s    0:00:00
+    16168686 100%    3.09MB/s    0:00:04 (xfer#1, to-check=0/1)
+
+sent 16170824 bytes  received 42 bytes  2940157.45 bytes/sec
+total size is 16168686  speedup is 1.00
+[2014-05-25 21:23:16 CEST] Transferrer: Uploaded 34_portra..art-1.mp4
+
+WORM-s16185635-m1398381044--Learning Corel Painter X3%35_portrait-of-hines-part-2.mp4
+       32768   0%    0.00kB/s    0:00:00
+     4390912  27%    4.09MB/s    0:00:02
+     7372800  45%    3.37MB/s    0:00:02
+    10518528  64%    3.23MB/s    0:00:01
+    13139968  81%    3.02MB/s    0:00:00
+    16185635 100%    3.13MB/s    0:00:04 (xfer#1, to-check=0/1)
+
+sent 16187773 bytes  received 42 bytes  2490433.08 bytes/sec
+total size is 16185635  speedup is 1.00
+[2014-05-25 21:23:22 CEST] Transferrer: Uploaded 35_portra..art-2.mp4
+
+WORM-s30681625-m1398381209--Learning Corel Painter X3%36_portrait-of-hines-part-3.mp4
+       32768   0%    0.00kB/s    0:00:00
+     4489216  14%    4.25MB/s    0:00:06
+     7798784  25%    3.65MB/s    0:00:06
+    11206656  36%    3.49MB/s    0:00:05
+    14483456  47%    3.40MB/s    0:00:04
+    17858560  58%    3.14MB/s    0:00:03
+    21299200  69%    3.17MB/s    0:00:02
+    24576000  80%    3.12MB/s    0:00:01
+    27951104  91%    3.15MB/s    0:00:00
+    30681625 100%    3.26MB/s    0:00:08 (xfer#1, to-check=0/1)
+
+sent 30685535 bytes  received 42 bytes  3230060.74 bytes/sec
+total size is 30681625  speedup is 1.00
+[2014-05-25 21:23:32 CEST] Transferrer: Uploaded 36_portra..art-3.mp4
+
+WORM-s22528765-m1398382099--Learning Corel Painter X3%37_digital-watercolor.mp4
+       32768   0%    0.00kB/s    0:00:00
+     4390912  19%    4.08MB/s    0:00:04
+     7536640  33%    3.55MB/s    0:00:04
+    10813440  47%    3.39MB/s    0:00:03
+    14090240  62%    3.30MB/s    0:00:02
+    17334272  76%    3.02MB/s    0:00:01
+    21495808  95%    3.25MB/s    0:00:00
+    22528765 100%    3.39MB/s    0:00:06 (xfer#1, to-check=0/1)
+
+sent 22531673 bytes  received 42 bytes  3004228.67 bytes/sec
+total size is 22528765  speedup is 1.00
+[2014-05-25 21:23:39 CEST] Transferrer: Uploaded 37_digita..color.mp4
+
+WORM-s26374030-m1398381726--Learning Corel Painter X3%38_real-watercolor-part-1.mp4
+       32768   0%    0.00kB/s    0:00:00
+     5210112  19%    4.87MB/s    0:00:04
+     8945664  33%    4.18MB/s    0:00:04
+    13074432  49%    4.08MB/s    0:00:03
+    16547840  62%    3.89MB/s    0:00:02
+    20676608  78%    3.65MB/s    0:00:01
+    24412160  92%    3.63MB/s    0:00:00
+    26374030 100%    3.82MB/s    0:00:06 (xfer#1, to-check=0/1)
+
+sent 26377410 bytes  received 42 bytes  3103229.65 bytes/sec
+total size is 26374030  speedup is 1.00
+[2014-05-25 21:23:47 CEST] Transferrer: Uploaded 38_real-w..art-1.mp4
+
+WORM-s19723496-m1398381429--Learning Corel Painter X3%39_real-watercolor-part-2.mp4
+       32768   0%    0.00kB/s    0:00:00
+     4751360  24%    4.50MB/s    0:00:03
+     8683520  44%    4.11MB/s    0:00:02
+    12517376  63%    3.96MB/s    0:00:01
+    16384000  83%    3.88MB/s    0:00:00
+    19723496 100%    3.86MB/s    0:00:04 (xfer#1, to-check=0/1)
+
+sent 19726064 bytes  received 42 bytes  3586564.73 bytes/sec
+total size is 19723496  speedup is 1.00
+[2014-05-25 21:23:53 CEST] Transferrer: Uploaded 39_real-w..art-2.mp4
+
+WORM-s19493451-m1398381312--Learning Corel Painter X3%40_caricature-part-1.mp4
+       32768   0%    0.00kB/s    0:00:00
+     4980736  25%    4.62MB/s    0:00:03
+     8683520  44%    4.04MB/s    0:00:02
+    12615680  64%    3.91MB/s    0:00:01
+    16547840  84%    3.84MB/s    0:00:00
+    19493451 100%    3.84MB/s    0:00:04 (xfer#1, to-check=0/1)
+
+sent 19495986 bytes  received 42 bytes  2999388.92 bytes/sec
+total size is 19493451  speedup is 1.00
+[2014-05-25 21:23:59 CEST] Transferrer: Uploaded 40_carica..art-1.mp4
+rsync: fork: Resource temporarily unavailable (35)
+rsync error: error in IPC code (code 14) at /SourceCache/rsync/rsync-42/rsync/pipe.c(65) [sender=2.6.9]
+rsync: fork: Resource temporarily unavailable (35)
+rsync error: error in IPC code (code 14) at /SourceCache/rsync/rsync-42/rsync/pipe.c(65) [sender=2.6.9]
+[2014-05-25 21:36:13 CEST] NetWatcherFallback: Syncing with diskstation 
+From annex.diskstation.local:/volume1/homes/joern/Annex.movies
+   0fddca6..08995a9  git-annex  -> diskstation/git-annex
+remote: merge git-annex (merging synced/git-annex into git-annex...)        
+remote: (Recording state in git...)        
+remote: ok        
+To joern@annex.diskstation.local:/volume1/homes/joern/Annex.movies
+[2014-05-25 21:36:38 CEST] RemoteControl: Syncing with diskstation 
+   251064b..9043046  git-annex -> synced/git-annex
+From annex.diskstation.local:/volume1/homes/joern/Annex.movies
+   08995a9..d2359ee  git-annex  -> diskstation/git-annex
+error: Ref refs/remotes/diskstation/synced/git-annex is at 90430461e599b7aad5bc8f4891f1e0eac38766b0 but expected 251064bd591813a5e813bd9cefafb8b4e6788f77
+ ! 251064b..9043046  synced/git-annex -> diskstation/synced/git-annex  (unable to update local ref)
+
+
+  rsync failed -- run git annex again to resume file transfer
+
+
+  rsync failed -- run git annex again to resume file transfer
+
+
+  rsync failed -- run git annex again to resume file transfer
+
+WORM-s14012455-m1398381979--Learning Corel Painter X3%41_caricature-part-2.mp4
+       32768   0%    0.00kB/s    0:00:00
+     4227072  30%    3.81MB/s    0:00:02
+     8159232  58%    3.69MB/s    0:00:01
+     9994240  71%    3.04MB/s    0:00:01
+    11829248  84%    2.67MB/s    0:00:00
+    13664256  97%    2.15MB/s    0:00:00
+    14012455 100%    2.50MB/s    0:00:05 (xfer#1, to-check=0/1)
+
+sent 14014322 bytes  received 42 bytes  1868581.87 bytes/sec
+total size is 14012455  speedup is 1.00
+[2014-05-25 21:37:01 CEST] Transferrer: Uploaded 41_carica..art-2.mp4
+
+WORM-s2813795-m1398381163--Learning Corel Painter X3%42_about-the-author.mp4
+       32768   1%    0.00kB/s    0:00:00
+     2813795 100%    6.18MB/s    0:00:00 (xfer#1, to-check=0/1)
+
+sent 2814292 bytes  received 42 bytes  1125733.60 bytes/sec
+total size is 2813795  speedup is 1.00
+[2014-05-25 21:37:03 CEST] Transferrer: Uploaded 42_about-..uthor.mp4
+
+WORM-s15151010-m1398381679--Learning Corel Painter X3%43_painter-resources-and-the-painter-community.mp4
+       32768   0%    0.00kB/s    0:00:00
+     5111808  33%    4.69MB/s    0:00:02
+     9109504  60%    4.20MB/s    0:00:01
+    12779520  84%    3.97MB/s    0:00:00
+    15151010 100%    3.95MB/s    0:00:03 (xfer#1, to-check=0/1)
+
+sent 15153043 bytes  received 42 bytes  3367352.22 bytes/sec
+total size is 15151010  speedup is 1.00
+[2014-05-25 21:37:07 CEST] Transferrer: Uploaded 43_painte..unity.mp4
+Watcher crashed: ResumeWatcher
+[2014-05-25 21:39:08 CEST] Watcher: warning Watcher crashed: ResumeWatcher
+
+
+(Recording state in git...)
+(merging diskstation/git-annex into git-annex...)
+(Recording state in git...)
+(merging diskstation/git-annex into git-annex...)
+(Recording state in git...)
+(merging diskstation/git-annex into git-annex...)
+(Recording state in git...)
+(merging diskstation/git-annex into git-annex...)
+(Recording state in git...)
+(merging diskstation/git-annex into git-annex...)
+(Recording state in git...)
+(merging diskstation/git-annex into git-annex...)
+(Recording state in git...)
+(merging diskstation/git-annex into git-annex...)
+(Recording state in git...)
+(merging diskstation/git-annex into git-annex...)
+(Recording state in git...)
+(merging diskstation/git-annex into git-annex...)
+(Recording state in git...)
+(merging diskstation/git-annex into git-annex...)
+(Recording state in git...)
+(merging diskstation/git-annex into git-annex...)
+(Recording state in git...)
+(merging diskstation/git-annex into git-annex...)
+(Recording state in git...)
+(merging diskstation/git-annex into git-annex...)
+(Recording state in git...)
+(merging diskstation/git-annex into git-annex...)
+(Recording state in git...)
+(merging diskstation/git-annex into git-annex...)
+(Recording state in git...)
+(merging diskstation/git-annex into git-annex...)
+(Recording state in git...)
+(merging diskstation/git-annex into git-annex...)
+(Recording state in git...)
+(merging diskstation/git-annex into git-annex...)
+(Recording state in git...)
+(merging diskstation/git-annex into git-annex...)
+(Recording state in git...)
+(merging diskstation/git-annex into git-annex...)
+(Recording state in git...)
+(merging diskstation/git-annex into git-annex...)
+(Recording state in git...)
+(merging diskstation/git-annex into git-annex...)
+(Recording state in git...)
+(merging diskstation/git-annex into git-annex...)
+(Recording state in git...)
+(merging diskstation/git-annex into git-annex...)
+(Recording state in git...)
+(merging diskstation/git-annex into git-annex...)
+(Recording state in git...)
+(merging diskstation/git-annex into git-annex...)
+(Recording state in git...)
+(merging diskstation/git-annex into git-annex...)
+(Recording state in git...)
+(merging diskstation/git-annex into git-annex...)
+(Recording state in git...)
+(merging diskstation/git-annex into git-annex...)
+(Recording state in git...)
+(merging diskstation/git-annex into git-annex...)
+(Recording state in git...)
+(Recording state in git...)
+(merging diskstation/git-annex into git-annex...)
+(Recording state in git...)
+(merging diskstation/git-annex into git-annex...)
+(Recording state in git...)
+(merging diskstation/git-annex into git-annex...)
+(Recording state in git...)
+(merging diskstation/git-annex into git-annex...)
+(Recording state in git...)
+(merging diskstation/git-annex into git-annex...)
+(Recording state in git...)
+(merging diskstation/git-annex into git-annex...)
+(Recording state in git...)
+(Recording state in git...)
+(merging diskstation/git-annex into git-annex...)
+(Recording state in git...)
+(merging diskstation/git-annex into git-annex...)
+(Recording state in git...)
+(Recording state in git...)
+(merging diskstation/git-annex into git-annex...)
+(Recording state in git...)
+(merging diskstation/git-annex into git-annex...)
+(Recording state in git...)
+(merging diskstation/git-annex into git-annex...)
+(Recording state in git...)
+(merging diskstation/git-annex into git-annex...)
+(Recording state in git...)
+(merging diskstation/git-annex into git-annex...)
+(Recording state in git...)
+(merging diskstation/git-annex into git-annex...)
+(Recording state in git...)
+(merging diskstation/git-annex into git-annex...)
+(Recording state in git...)
+(Recording state in git...)
+(merging diskstation/git-annex into git-annex...)
+(Recording state in git...)
+(merging diskstation/git-annex into git-annex...)
+(Recording state in git...)
+(merging diskstation/git-annex into git-annex...)
+(Recording state in git...)
+(merging diskstation/git-annex into git-annex...)
+(Recording state in git...)
+(merging diskstation/git-annex into git-annex...)
+(Recording state in git...)
+(merging diskstation/git-annex into git-annex...)
+(scanning...) [2014-05-25 21:39:16 CEST] Watcher: Performing startup scan
+[2014-05-25 21:39:20 CEST] Pusher: Syncing with diskstation 
+remote: merge git-annex (merging synced/git-annex into git-annex...)        
+remote: (Recording state in git...)        
+remote: ok        
+To joern@annex.diskstation.local:/volume1/homes/joern/Annex.movies
+   9043046..57f7b8b  git-annex -> synced/git-annex
+[2014-05-25 21:39:28 CEST] RemoteControl: Syncing with diskstation 
+From annex.diskstation.local:/volume1/homes/joern/Annex.movies
+   d2359ee..e459e2f  git-annex  -> diskstation/git-annex
+error: Ref refs/remotes/diskstation/synced/git-annex is at 57f7b8bb8d2dd5418eda74deef4a0338b00390d9 but expected 90430461e599b7aad5bc8f4891f1e0eac38766b0
+ ! 9043046..57f7b8b  synced/git-annex -> diskstation/synced/git-annex  (unable to update local ref)
+(Recording state in git...)
+(merging diskstation/git-annex into git-annex...)
+(started...) [2014-05-25 21:39:46 CEST] Committer: Committing changes to git
+[2014-05-25 21:39:46 CEST] Pusher: Syncing with diskstation 
+Already up-to-date.
+remote: merge git-annex ok        
+remote: merge synced/master Updating 53c007f..6f2044d        
+remote: Fast-forward        
+remote:  .../01_introduction.mp4                                                  | 1 +        
+remote:  .../02_setting-up.mp4                                                    | 1 +        
+remote:  .../03_selecting-and-creating-elements.mp4                               | 1 +        
+remote:  .../04_exercise-1.mp4                                                    | 1 +        
+remote:  An Introduction to d3.js - From Scattered to Scatterplot/05_svg.mp4      | 1 +        
+remote:  .../06_exercise-2.mp4                                                    | 1 +        
+remote:  .../07_binding-data-and-creating-elements-from-data.mp4                  | 1 +        
+remote:  .../08_exercise-3.mp4                                                    | 1 +        
+remote:  .../09_transitions.mp4                                                   | 1 +        
+remote:  An Introduction to d3.js - From Scattered to Scatterplot/10_scales-2.mp4 | 1 +        
+remote:  An Introduction to d3.js - From Scattered to Scatterplot/10_scales.mp4   | 1 +        
+remote:  An Introduction to d3.js - From Scattered to Scatterplot/11_axes.mp4     | 1 +        
+remote:  .../12_advanced-techniques.mp4                                           | 1 +        
+remote:  .../13_conclusion.mp4                                                    | 1 +        
+remote:  C++ Memory Management/01_introduction.mp4                                | 1 +        
+remote:  C++ Memory Management/02_leaks-and-overruns.mp4                          | 1 +        
+remote:  .../03_stl-vector-methods-range-checking-and-sentinels.mp4               | 1 +        
+remote:  C++ Memory Management/04_memory-leaks-part-1.mp4                         | 1 +        
+remote:  C++ Memory Management/05_memory-leaks-part-2.mp4                         | 1 +        
+remote:  C++ Memory Management/06_pointers-pitfalls-and-best-practices.mp4        | 1 +        
+remote:  C++ Memory Management/07_initialize-everything-trust-but-verify.mp4      | 1 +        
+remote:  C++ Memory Management/08_reading-and-writing-shared-assets.mp4           | 1 +        
+remote:  C++ Memory Management/09_undefined-behaviour-and-other-bad-practices.mp4 | 1 +        
+remote:  C++ Memory Management/10_the-heap-and-stl-wrappers.mp4                   | 1 +        
+remote:  C++ Memory Management/11_about-the-author.mp4                            | 1 +        
+remote:  Lean UX Workshop/01_introduction.mp4                                     | 1 +        
+remote:  Lean UX Workshop/02_nordstrom-case-study.mp4                             | 1 +        
+remote:  Lean UX Workshop/03_assumptions-and-hypotheses.mp4                       | 1 +        
+remote:  Lean UX Workshop/04_personas.mp4                                         | 1 +        
+remote:  Lean UX Workshop/05_outcomes-and-features.mp4                            | 1 +        
+remote:  Lean UX Workshop/06_writing-hypotheses.mp4                               | 1 +        
+remote:  Lean UX Workshop/07_design-studio.mp4                                    | 1 +        
+remote:  Lean UX Workshop/08_experiments-and-mvp-s.mp4                            | 1 +        
+remote:  Lean UX Workshop/09_research.mp4                                         | 1 +        
+remote:  Lean UX Workshop/10_combining-lean-and-agile.mp4                         | 1 +        
+remote:  Lean UX Workshop/11_tools-and-techniques.mp4                             | 1 +        
+remote:  Learning Corel Painter X3/01_a-quick-tour-of-painter.mp4                 | 1 +        
+remote:  Learning Corel Painter X3/02_brushes-and-custom-palettes.mp4             | 1 +        
+remote:  Learning Corel Painter X3/03_managing-layers.mp4                         | 1 +        
+remote:  Learning Corel Painter X3/04_project-red-chairs.mp4                      | 1 +        
+remote:  Learning Corel Painter X3/05_the-toolbox-part-1.mp4                      | 1 +        
+remote:  Learning Corel Painter X3/06_the-toolbox-part-2.mp4                      | 1 +        
+remote:  Learning Corel Painter X3/07_color-panels.mp4                            | 1 +        
+remote:  Learning Corel Painter X3/08_paper-and-media-libraries.mp4               | 1 +        
+remote:  Learning Corel Painter X3/09_menu-commands-part-1.mp4                    | 1 +        
+remote:  Learning Corel Painter X3/10_menu-commands-part-2.mp4                    | 1 +        
+remote:  Learning Corel Painter X3/11_cloner-brushes.mp4                          | 1 +        
+remote:  Learning Corel Painter X3/12_project-clone-painting-a-pear.mp4           | 1 +        
+remote:  Learning Corel Painter X3/13_dab-stroke-and-method.mp4                   | 1 +        
+remote:  Learning Corel Painter X3/14_advanced-controls.mp4                       | 1 +        
+remote:  Learning Corel Painter X3/15_custom-variants.mp4                         | 1 +        
+remote:  Learning Corel Painter X3/16_abstract-painting.mp4                       | 1 +        
+remote:  Learning Corel Painter X3/17_tonal-controls.mp4                          | 1 +        
+remote:  Learning Corel Painter X3/18_surface-controls.mp4                        | 1 +        
+remote:  Learning Corel Painter X3/19_esoterica.mp4                               | 1 +        
+remote:  Learning Corel Painter X3/20_esoterica-mosaic.mp4                        | 1 +        
+remote:  Learning Corel Painter X3/21_selection-tools-and-commands.mp4            | 1 +        
+remote:  Learning Corel Painter X3/22_text-properties.mp4                         | 1 +        
+remote:  Learning Corel Painter X3/23_text-effects.mp4                            | 1 +        
+remote:  Learning Corel Painter X3/24_postcard-graphic-part-1.mp4                 | 1 +        
+remote:  Learning Corel Painter X3/25_postcard-graphic-part-2.mp4                 | 1 +        
+remote:  Learning Corel Painter X3/26_dynamic-layers.mp4                          | 1 +        
+remote:  Learning Corel Painter X3/27_auto-painting.mp4                           | 1 +        
+remote:  Learning Corel Painter X3/28_smart-stroke-cloning.mp4                    | 1 +        
+remote:  Learning Corel Painter X3/29_mirror-and-kaleidoscope.mp4                 | 1 +        
+remote:  Learning Corel Painter X3/30_introduction-to-mixed-media.mp4             | 1 +        
+remote:  Learning Corel Painter X3/31_portrait-of-stephen.mp4                     | 1 +        
+remote:  Learning Corel Painter X3/32_cel-animation.mp4                           | 1 +        
+remote:  Learning Corel Painter X3/33_rotoscope-techniques.mp4                    | 1 +        
+remote:  Learning Corel Painter X3/34_portrait-of-hines-part-1.mp4                | 1 +        
+remote:  Learning Corel Painter X3/35_portrait-of-hines-part-2.mp4                | 1 +        
+remote:  Learning Corel Painter X3/36_portrait-of-hines-part-3.mp4                | 1 +        
+remote:  Learning Corel Painter X3/37_digital-watercolor.mp4                      | 1 +        
+remote:  Learning Corel Painter X3/38_real-watercolor-part-1.mp4                  | 1 +        
+remote:  Learning Corel Painter X3/39_real-watercolor-part-2.mp4                  | 1 +        
+remote:  Learning Corel Painter X3/40_caricature-part-1.mp4                       | 1 +        
+remote:  Learning Corel Painter X3/41_caricature-part-2.mp4                       | 1 +        
+remote:  Learning Corel Painter X3/42_about-the-author.mp4                        | 1 +        
+remote:  .../43_painter-resources-and-the-painter-community.mp4                   | 1 +        
+remote:  79 files changed, 79 insertions(+)        
+remote:  create mode 120000 An Introduction to d3.js - From Scattered to Scatterplot/01_introduction.mp4        
+remote:  create mode 120000 An Introduction to d3.js - From Scattered to Scatterplot/02_setting-up.mp4        
+remote:  create mode 120000 An Introduction to d3.js - From Scattered to Scatterplot/03_selecting-and-creating-elements.mp4        
+remote:  create mode 120000 An Introduction to d3.js - From Scattered to Scatterplot/04_exercise-1.mp4        
+remote:  create mode 120000 An Introduction to d3.js - From Scattered to Scatterplot/05_svg.mp4        
+remote:  create mode 120000 An Introduction to d3.js - From Scattered to Scatterplot/06_exercise-2.mp4        
+remote:  create mode 120000 An Introduction to d3.js - From Scattered to Scatterplot/07_binding-data-and-creating-elements-from-data.mp4        
+remote:  create mode 120000 An Introduction to d3.js - From Scattered to Scatterplot/08_exercise-3.mp4        
+remote:  create mode 120000 An Introduction to d3.js - From Scattered to Scatterplot/09_transitions.mp4        
+remote:  create mode 120000 An Introduction to d3.js - From Scattered to Scatterplot/10_scales-2.mp4        
+remote:  create mode 120000 An Introduction to d3.js - From Scattered to Scatterplot/10_scales.mp4        
+remote:  create mode 120000 An Introduction to d3.js - From Scattered to Scatterplot/11_axes.mp4        
+remote:  create mode 120000 An Introduction to d3.js - From Scattered to Scatterplot/12_advanced-techniques.mp4        
+remote:  create mode 120000 An Introduction to d3.js - From Scattered to Scatterplot/13_conclusion.mp4        
+remote:  create mode 120000 C++ Memory Management/01_introduction.mp4        
+remote:  create mode 120000 C++ Memory Management/02_leaks-and-overruns.mp4        
+remote:  create mode 120000 C++ Memory Management/03_stl-vector-methods-range-checking-and-sentinels.mp4        
+remote:  create mode 120000 C++ Memory Management/04_memory-leaks-part-1.mp4        
+remote:  create mode 120000 C++ Memory Management/05_memory-leaks-part-2.mp4        
+remote:  create mode 120000 C++ Memory Management/06_pointers-pitfalls-and-best-practices.mp4        
+remote:  create mode 120000 C++ Memory Management/07_initialize-everything-trust-but-verify.mp4        
+remote:  create mode 120000 C++ Memory Management/08_reading-and-writing-shared-assets.mp4        
+remote:  create mode 120000 C++ Memory Management/09_undefined-behaviour-and-other-bad-practices.mp4        
+remote:  create mode 120000 C++ Memory Management/10_the-heap-and-stl-wrappers.mp4        
+remote:  create mode 120000 C++ Memory Management/11_about-the-author.mp4        
+remote:  create mode 120000 Lean UX Workshop/01_introduction.mp4        
+remote:  create mode 120000 Lean UX Workshop/02_nordstrom-case-study.mp4        
+remote:  create mode 120000 Lean UX Workshop/03_assumptions-and-hypotheses.mp4        
+remote:  create mode 120000 Lean UX Workshop/04_personas.mp4        
+remote:  create mode 120000 Lean UX Workshop/05_outcomes-and-features.mp4        
+remote:  create mode 120000 Lean UX Workshop/06_writing-hypotheses.mp4        
+remote:  create mode 120000 Lean UX Workshop/07_design-studio.mp4        
+remote:  create mode 120000 Lean UX Workshop/08_experiments-and-mvp-s.mp4        
+remote:  create mode 120000 Lean UX Workshop/09_research.mp4        
+remote:  create mode 120000 Lean UX Workshop/10_combining-lean-and-agile.mp4        
+remote:  create mode 120000 Lean UX Workshop/11_tools-and-techniques.mp4        
+remote:  create mode 120000 Learning Corel Painter X3/01_a-quick-tour-of-painter.mp4        
+remote:  create mode 120000 Learning Corel Painter X3/02_brushes-and-custom-palettes.mp4        
+remote:  create mode 120000 Learning Corel Painter X3/03_managing-layers.mp4        
+remote:  create mode 120000 Learning Corel Painter X3/04_project-red-chairs.mp4        
+remote:  create mode 120000 Learning Corel Painter X3/05_the-toolbox-part-1.mp4        
+remote:  create mode 120000 Learning Corel Painter X3/06_the-toolbox-part-2.mp4        
+remote:  create mode 120000 Learning Corel Painter X3/07_color-panels.mp4        
+remote:  create mode 120000 Learning Corel Painter X3/08_paper-and-media-libraries.mp4        
+remote:  create mode 120000 Learning Corel Painter X3/09_menu-commands-part-1.mp4        
+remote:  create mode 120000 Learning Corel Painter X3/10_menu-commands-part-2.mp4        
+remote:  create mode 120000 Learning Corel Painter X3/11_cloner-brushes.mp4        
+remote:  create mode 120000 Learning Corel Painter X3/12_project-clone-painting-a-pear.mp4        
+remote:  create mode 120000 Learning Corel Painter X3/13_dab-stroke-and-method.mp4        
+remote:  create mode 120000 Learning Corel Painter X3/14_advanced-controls.mp4        
+remote:  create mode 120000 Learning Corel Painter X3/15_custom-variants.mp4        
+remote:  create mode 120000 Learning Corel Painter X3/16_abstract-painting.mp4        
+remote:  create mode 120000 Learning Corel Painter X3/17_tonal-controls.mp4        
+remote:  create mode 120000 Learning Corel Painter X3/18_surface-controls.mp4        
+remote:  create mode 120000 Learning Corel Painter X3/19_esoterica.mp4        
+remote:  create mode 120000 Learning Corel Painter X3/20_esoterica-mosaic.mp4        
+remote:  create mode 120000 Learning Corel Painter X3/21_selection-tools-and-commands.mp4        
+remote:  create mode 120000 Learning Corel Painter X3/22_text-properties.mp4        
+remote:  create mode 120000 Learning Corel Painter X3/23_text-effects.mp4        
+remote:  create mode 120000 Learning Corel Painter X3/24_postcard-graphic-part-1.mp4        
+remote:  create mode 120000 Learning Corel Painter X3/25_postcard-graphic-part-2.mp4        
+remote:  create mode 120000 Learning Corel Painter X3/26_dynamic-layers.mp4        
+remote:  create mode 120000 Learning Corel Painter X3/27_auto-painting.mp4        
+remote:  create mode 120000 Learning Corel Painter X3/28_smart-stroke-cloning.mp4        
+remote:  create mode 120000 Learning Corel Painter X3/29_mirror-and-kaleidoscope.mp4        
+remote:  create mode 120000 Learning Corel Painter X3/30_introduction-to-mixed-media.mp4        
+remote:  create mode 120000 Learning Corel Painter X3/31_portrait-of-stephen.mp4        
+remote:  create mode 120000 Learning Corel Painter X3/32_cel-animation.mp4        
+remote:  create mode 120000 Learning Corel Painter X3/33_rotoscope-techniques.mp4        
+remote:  create mode 120000 Learning Corel Painter X3/34_portrait-of-hines-part-1.mp4        
+remote:  create mode 120000 Learning Corel Painter X3/35_portrait-of-hines-part-2.mp4        
+remote:  create mode 120000 Learning Corel Painter X3/36_portrait-of-hines-part-3.mp4        
+remote:  create mode 120000 Learning Corel Painter X3/37_digital-watercolor.mp4        
+remote:  create mode 120000 Learning Corel Painter X3/38_real-watercolor-part-1.mp4        
+remote:  create mode 120000 Learning Corel Painter X3/39_real-watercolor-part-2.mp4        
+remote:  create mode 120000 Learning Corel Painter X3/40_caricature-part-1.mp4        
+remote:  create mode 120000 Learning Corel Painter X3/41_caricature-part-2.mp4        
+remote:  create mode 120000 Learning Corel Painter X3/42_about-the-author.mp4        
+remote:  create mode 120000 Learning Corel Painter X3/43_painter-resources-and-the-painter-community.mp4        
+remote: 
+remote: ok        
+To joern@annex.diskstation.local:/volume1/homes/joern/Annex.movies
+   57f7b8b..e459e2f  git-annex -> synced/git-annex
+   53c007f..6f2044d  master -> synced/master
+Already up-to-date.
+[2014-05-25 21:41:22 CEST] Committer: Adding 01_introduction.mp4 02_settin..plate.mp4 03_settin..lbars.mp4 04_how-to..files.mp4 05_line-tool.mp4 06_rectan..-tool.mp4 07_circle-tool.mp4 08_polygon-tool.mp4 09_arc-tool.mp4 and 34 other files
+
+(Recording state in git...)
+
+
+add Learning SketchUp 2013/01_introduction.mp4 ok
+add Learning SketchUp 2013/02_setting-up-the-template.mp4 ok
+add Learning SketchUp 2013/03_setting-up-the-toolbars.mp4 ok
+add Learning SketchUp 2013/04_how-to-access-your-working-files.mp4 ok
+add Learning SketchUp 2013/05_line-tool.mp4 ok
+add Learning SketchUp 2013/06_rectangle-tool.mp4 ok
+add Learning SketchUp 2013/07_circle-tool.mp4 ok
+add Learning SketchUp 2013/08_polygon-tool.mp4 ok
+add Learning SketchUp 2013/09_arc-tool.mp4 ok
+add Learning SketchUp 2013/100_adding-lighting.mp4 ok
+add Learning SketchUp 2013/101_add-reflections-to-existing-textures.mp4 ok
+add Learning SketchUp 2013/102_model-presets-create-a-preview-rendering.mp4 ok
+add Learning SketchUp 2013/103_rendering-a-high-quality-image.mp4 ok
+add Learning SketchUp 2013/104_post-processing-in-photoshop.mp4 ok
+add Learning SketchUp 2013/105_what-is-layout.mp4 ok
+add Learning SketchUp 2013/106_adjusting-the-model-settings.mp4 ok
+add Learning SketchUp 2013/108_adding-additional-pages-and-content.mp4 ok
+add Learning SketchUp 2013/109_updating-the-model-references.mp4 ok
+add Learning SketchUp 2013/10_freehand-tool.mp4 ok
+add Learning SketchUp 2013/111_about-dan-brown.mp4 ok
+add Learning SketchUp 2013/11_navigating-in-3d.mp4 ok
+add Learning SketchUp 2013/12_push-pull-and-move-tool.mp4 ok
+add Learning SketchUp 2013/13_using-follow-me-complex-roof-pitches.mp4 ok
+add Learning SketchUp 2013/14_creating-a-mansard-roof.mp4 ok
+add Learning SketchUp 2013/15_creating-a-gambrel-roof.mp4 ok
+add Learning SketchUp 2013/16_measurements.mp4 ok
+add Learning SketchUp 2013/18_extruding-your-walls-push-pull-and-offset.mp4 ok
+add Learning SketchUp 2013/19_creating-doors-and-openings.mp4 ok
+add Learning SketchUp 2013/20_clean-up-using-the-eraser-to-remove-unwanted-edges.mp4 ok
+add Learning SketchUp 2013/21_why-groups.mp4 ok
+add Learning SketchUp 2013/22_creating-the-second-floor.mp4 ok
+add Learning SketchUp 2013/23_creating-additional-openings.mp4 ok
+add Learning SketchUp 2013/24_hide-model-and-other-useful-shortcuts.mp4 ok
+add Learning SketchUp 2013/25_using-the-outliner.mp4 ok
+add Learning SketchUp 2013/26_using-layers-to-control-visibility.mp4 ok
+add Learning SketchUp 2013/27_viewing-layer-states-of-the-model.mp4 ok
+add Learning SketchUp 2013/28_field-of-view.mp4 ok
+add Learning SketchUp 2013/29_dividing-faces-to-create-your-rooms.mp4 ok
+add Learning SketchUp 2013/30_area-tool-for-calculations.mp4 ok
+add Learning SketchUp 2013/31_using-paint-bucket-for-floor-texture-2.mp4 ok
+add Learning SketchUp 2013/31_using-paint-bucket-for-floor-texture.mp4 ok
+add Learning SketchUp 2013/32_importing-and-applying-a-texture.mp4 ok
+add Learning SketchUp 2013/33_positioning-an-imported-texture.mp4 [2014-05-25 21:41:29 CEST] Committer: Adding 34_editin..tchup.mp4 35_editin..oshop.mp4 36_hide-e..-tool.mp4 37_creati..iling.mp4 38_assign..roups.mp4 39_adding..ter-7.mp4 40_creati..lding.mp4 41_copy-a..place.mp4 42_why-we..nents.mp4 and 57 other files
+ok
+add Learning SketchUp 2013/34_editing-texture-colour-in-sketchup.mp4 ok
+add Learning SketchUp 2013/35_editing-a-texture-in-photoshop.mp4 ok
+add Learning SketchUp 2013/36_hide-edges-and-lines-with-the-eraser-tool.mp4 ok
+add Learning SketchUp 2013/37_creating-the-ceiling.mp4 ok
+add Learning SketchUp 2013/38_assigning-layers-within-groups.mp4 ok
+add Learning SketchUp 2013/39_adding-detail-chapter-7.mp4 ok
+add Learning SketchUp 2013/40_creating-crown-moulding.mp4 ok
+add Learning SketchUp 2013/41_copy-and-paste-in-place.mp4 ok
+add Learning SketchUp 2013/42_why-we-use-groups-and-components.mp4 ok
+add Learning SketchUp 2013/43_creating-the-staircase.mp4 ok
+add Learning SketchUp 2013/44_adding-riser-detail.mp4 ok
+add Learning SketchUp 2013/45_making-a-unique-bottom-step.mp4 ok
+add Learning SketchUp 2013/46_adding-spindles-with-move-and-copy.mp4 ok
+add Learning SketchUp 2013/47_adding-railing.mp4 ok
+add Learning SketchUp 2013/48_review-of-components-and-nested-groups.mp4 ok
+add Learning SketchUp 2013/49_creating-doors.mp4 ok
+add Learning SketchUp 2013/50_creating-the-front-window-frame.mp4 ok
+add Learning SketchUp 2013/51_double-hung-windows.mp4 ok
+add Learning SketchUp 2013/52_installing-bit-tools-plugins.mp4 ok
+add Learning SketchUp 2013/53_creating-walls-and-door-openings.mp4 ok
+add Learning SketchUp 2013/54_using-the-window-feature.mp4 ok
+add Learning SketchUp 2013/55_starting-at-the-base.mp4 ok
+add Learning SketchUp 2013/56_building-the-mantle.mp4 ok
+add Learning SketchUp 2013/57_mantle-columns.mp4 ok
+add Learning SketchUp 2013/58_mantle-support.mp4 ok
+add Learning SketchUp 2013/59_adding-textures.mp4 ok
+add Learning SketchUp 2013/60_base-cabinets.mp4 ok
+add Learning SketchUp 2013/61_wall-cabinets.mp4 ok
+add Learning SketchUp 2013/62_cabinet-doors.mp4 ok
+add Learning SketchUp 2013/63_creating-the-countertop.mp4 ok
+add Learning SketchUp 2013/64_adding-materials.mp4 ok
+add Learning SketchUp 2013/65_adding-appliances.mp4 ok
+add Learning SketchUp 2013/66_adding-the-fridge-and-sink.mp4 ok
+add Learning SketchUp 2013/67_using-the-3d-warehouse.mp4 ok
+add Learning SketchUp 2013/68_using-form-fonts.mp4 ok
+add Learning SketchUp 2013/69_using-the-podium-browser.mp4 ok
+add Learning SketchUp 2013/70_creating-a-library.mp4 ok
+add Learning SketchUp 2013/71_setting-up-to-match-a-new-photo.mp4 ok
+add Learning SketchUp 2013/72_creating-the-massing-model.mp4 ok
+add Learning SketchUp 2013/73_applying-textures.mp4 ok
+add Learning SketchUp 2013/74_adding-detail-chapter-13.mp4 ok
+add Learning SketchUp 2013/75_saving-and-placing-into-the-model.mp4 ok
+add Learning SketchUp 2013/76_creating-the-exterior.mp4 ok
+add Learning SketchUp 2013/77_setting-the-building-in-place-geolocation.mp4 ok
+add Learning SketchUp 2013/78_reset-axis-to-align-building.mp4 ok
+add Learning SketchUp 2013/79_creating-the-front-facade.mp4 ok
+add Learning SketchUp 2013/80_adding-context.mp4 ok
+add Learning SketchUp 2013/81_get-photo-google-street-view.mp4 ok
+add Learning SketchUp 2013/82_using-3d-text.mp4 ok
+add Learning SketchUp 2013/83_add-solar-panels-with-skelion-plugin.mp4 ok
+add Learning SketchUp 2013/84_creating-scenes-and-updating.mp4 ok
+add Learning SketchUp 2013/85_styles.mp4 ok
+add Learning SketchUp 2013/86_scenes-for-animation.mp4 ok
+add Learning SketchUp 2013/87_section-cut-for-interior-elevations.mp4 ok
+add Learning SketchUp 2013/88_creating-a-group.mp4 ok
+add Learning SketchUp 2013/89_creating-the-animation.mp4 ok
+add Learning SketchUp 2013/90_jpeg-and-png-files.mp4 ok
+add Learning SketchUp 2013/91_dwg-elevation-and-plan-pro-feature.mp4 ok
+add Learning SketchUp 2013/92_kmz-for-google-earth.mp4 ok
+add Learning SketchUp 2013/93_collada-files.mp4 ok
+add Learning SketchUp 2013/94_animation-options.mp4 ok
+add Learning SketchUp 2013/95_add-final-touches.mp4 ok
+add Learning SketchUp 2013/96_export-display-options.mp4 ok
+add Learning SketchUp 2013/97_stacking-images-in-photoshop.mp4 ok
+add Learning SketchUp 2013/98_multiply-and-opacity.mp4 ok
+add Learning SketchUp 2013/99_saving-the-image.mp4 [2014-05-25 21:41:31 CEST] Committer: Committing changes to git
+[2014-05-25 21:41:32 CEST] Pusher: Syncing with diskstation 
+
+WORM-s8756441-m1398385688--Learning SketchUp 2013%34_editing-texture-colour-in-sketchup.mp4
+       32768   0%    0.00kB/s    0:00:00
+     2654208  30%    1.97MB/s    0:00:03
+     5734400  65%    2.32MB/s    0:00:01
+Already up-to-date.
+     8756441 100%    2.66MB/s    0:00:03 (xfer#1, to-check=0/1)
+
+sent 8757681 bytes  received 42 bytes  1592313.27 bytes/sec
+total size is 8756441  speedup is 1.00
+[2014-05-25 21:41:37 CEST] Transferrer: Uploaded 34_editin..tchup.mp4
+
+WORM-s16632826-m1398386674--Learning SketchUp 2013%33_positioning-an-imported-texture.mp4
+       32768   0%    0.00kB/s    0:00:00
+     4423680  26%    4.12MB/s    0:00:02
+     8486912  51%    3.96MB/s    0:00:02
+    12517376  75%    3.90MB/s    0:00:01
+remote: merge git-annex     16580608  99%    3.87MB/s    0:00:00
+    16632826 100%    3.89MB/s    0:00:04 (xfer#1, to-check=0/1)
+[2014-05-25 21:41:42 CEST] RemoteControl: Syncing with diskstation 
+
+sent 16635024 bytes  received 42 bytes  3024557.45 bytes/sec
+total size is 16632826  speedup is 1.00
+[2014-05-25 21:41:43 CEST] Transferrer: Uploaded 33_positi..xture.mp4
+(merging synced/git-annex into git-annex...)        
+remote: (Recording state in git...)        
+remote: ok        
+remote: merge synced/master 
+WORM-s13603741-m1398386183--Learning SketchUp 2013%32_importing-and-applying-a-texture.mp4
+       32768   0%    0.00kB/s    0:00:00
+     3244032  23%    3.06MB/s    0:00:03
+From annex.diskstation.local:/volume1/homes/joern/Annex.movies
+   e459e2f..297826c  git-annex  -> diskstation/git-annex
+   e459e2f..26c6449  synced/git-annex -> diskstation/synced/git-annex
+   6f2044d..7500a88  synced/master -> diskstation/synced/master
+Already up-to-date.
+Updating 6f2044d..7500a88        
+remote: Fast-forward        
+     6520832  47%    3.08MB/s    0:00:02
+remote:  Learning SketchUp 2013/01_introduction.mp4                               | 1 +        
+remote:  Learning SketchUp 2013/02_setting-up-the-template.mp4                    | 1 +        
+remote:  Learning SketchUp 2013/03_setting-up-the-toolbars.mp4                    | 1 +        
+remote:  Learning SketchUp 2013/04_how-to-access-your-working-files.mp4           | 1 +        
+remote:  Learning SketchUp 2013/05_line-tool.mp4                                  | 1 +        
+remote:  Learning SketchUp 2013/06_rectangle-tool.mp4                             | 1 +        
+remote:  Learning SketchUp 2013/07_circle-tool.mp4                                | 1 +        
+remote:  Learning SketchUp 2013/08_polygon-tool.mp4                               | 1 +        
+remote:  Learning SketchUp 2013/09_arc-tool.mp4                                   | 1 +        
+remote:  Learning SketchUp 2013/100_adding-lighting.mp4                           | 1 +        
+remote:  Learning SketchUp 2013/101_add-reflections-to-existing-textures.mp4      | 1 +        
+remote:  Learning SketchUp 2013/102_model-presets-create-a-preview-rendering.mp4  | 1 +        
+remote:  Learning SketchUp 2013/103_rendering-a-high-quality-image.mp4            | 1 +        
+remote:  Learning SketchUp 2013/104_post-processing-in-photoshop.mp4              | 1 +        
+remote:  Learning SketchUp 2013/105_what-is-layout.mp4                            | 1 +        
+remote:  Learning SketchUp 2013/106_adjusting-the-model-settings.mp4              | 1 +        
+remote:  Learning SketchUp 2013/108_adding-additional-pages-and-content.mp4       | 1 +        
+remote:  Learning SketchUp 2013/109_updating-the-model-references.mp4             | 1 +        
+remote:  Learning SketchUp 2013/10_freehand-tool.mp4                              | 1 +        
+remote:  Learning SketchUp 2013/111_about-dan-brown.mp4                           | 1 +        
+remote:  Learning SketchUp 2013/11_navigating-in-3d.mp4                           | 1 +        
+remote:  Learning SketchUp 2013/12_push-pull-and-move-tool.mp4                    | 1 +        
+remote:  Learning SketchUp 2013/13_using-follow-me-complex-roof-pitches.mp4       | 1 +        
+remote:  Learning SketchUp 2013/14_creating-a-mansard-roof.mp4                    | 1 +        
+remote:  Learning SketchUp 2013/15_creating-a-gambrel-roof.mp4                    | 1 +        
+remote:  Learning SketchUp 2013/16_measurements.mp4                               | 1 +        
+remote:  Learning SketchUp 2013/18_extruding-your-walls-push-pull-and-offset.mp4  | 1 +        
+remote:  Learning SketchUp 2013/19_creating-doors-and-openings.mp4                | 1 +        
+remote:  .../20_clean-up-using-the-eraser-to-remove-unwanted-edges.mp4            | 1 +        
+remote:  Learning SketchUp 2013/21_why-groups.mp4                                 | 1 +        
+remote:  Learning SketchUp 2013/22_creating-the-second-floor.mp4                  | 1 +        
+remote:  Learning SketchUp 2013/23_creating-additional-openings.mp4               | 1 +        
+remote:  Learning SketchUp 2013/24_hide-model-and-other-useful-shortcuts.mp4      | 1 +        
+remote:  Learning SketchUp 2013/25_using-the-outliner.mp4                         | 1 +        
+remote:  Learning SketchUp 2013/26_using-layers-to-control-visibility.mp4         | 1 +        
+remote:  Learning SketchUp 2013/27_viewing-layer-states-of-the-model.mp4          | 1 +        
+remote:  Learning SketchUp 2013/28_field-of-view.mp4                              | 1 +        
+remote:  Learning SketchUp 2013/29_dividing-faces-to-create-your-rooms.mp4        | 1 +        
+remote:  Learning SketchUp 2013/30_area-tool-for-calculations.mp4                 | 1 +        
+remote:  Learning SketchUp 2013/31_using-paint-bucket-for-floor-texture-2.mp4     | 1 +        
+remote:  Learning SketchUp 2013/31_using-paint-bucket-for-floor-texture.mp4       | 1 +        
+remote:  Learning SketchUp 2013/32_importing-and-applying-a-texture.mp4           | 1 +        
+remote:  Learning SketchUp 2013/33_positioning-an-imported-texture.mp4            | 1 +        
+remote:  Learning SketchUp 2013/34_editing-texture-colour-in-sketchup.mp4         | 1 +        
+remote:  Learning SketchUp 2013/35_editing-a-texture-in-photoshop.mp4             | 1 +        
+remote:  Learning SketchUp 2013/36_hide-edges-and-lines-with-the-eraser-tool.mp4  | 1 +        
+remote:  Learning SketchUp 2013/37_creating-the-ceiling.mp4                       | 1 +        
+remote:  Learning SketchUp 2013/38_assigning-layers-within-groups.mp4             | 1 +        
+remote:  Learning SketchUp 2013/39_adding-detail-chapter-7.mp4                    | 1 +        
+remote:  Learning SketchUp 2013/40_creating-crown-moulding.mp4                    | 1 +        
+remote:  Learning SketchUp 2013/41_copy-and-paste-in-place.mp4                    | 1 +        
+remote:  Learning SketchUp 2013/42_why-we-use-groups-and-components.mp4           | 1 +        
+remote:  Learning SketchUp 2013/43_creating-the-staircase.mp4                     | 1 +        
+remote:  Learning SketchUp 2013/44_adding-riser-detail.mp4                        | 1 +        
+remote:  Learning SketchUp 2013/45_making-a-unique-bottom-step.mp4                | 1 +        
+remote:  Learning SketchUp 2013/46_adding-spindles-with-move-and-copy.mp4         | 1 +        
+remote:  Learning SketchUp 2013/47_adding-railing.mp4                             | 1 +        
+remote:  Learning SketchUp 2013/48_review-of-components-and-nested-groups.mp4     | 1 +        
+remote:  Learning SketchUp 2013/49_creating-doors.mp4                             | 1 +        
+remote:  Learning SketchUp 2013/50_creating-the-front-window-frame.mp4            | 1 +        
+remote:  Learning SketchUp 2013/51_double-hung-windows.mp4                        | 1 +        
+remote:  Learning SketchUp 2013/52_installing-bit-tools-plugins.mp4               | 1 +        
+remote:  Learning SketchUp 2013/53_creating-walls-and-door-openings.mp4           | 1 +        
+remote:  Learning SketchUp 2013/54_using-the-window-feature.mp4                   | 1 +        
+remote:  Learning SketchUp 2013/55_starting-at-the-base.mp4                       | 1 +        
+remote:  Learning SketchUp 2013/56_building-the-mantle.mp4                        | 1 +        
+remote:  Learning SketchUp 2013/57_mantle-columns.mp4                             | 1 +        
+remote:  Learning SketchUp 2013/58_mantle-support.mp4                             | 1 +        
+remote:  Learning SketchUp 2013/59_adding-textures.mp4                            | 1 +        
+remote:  Learning SketchUp 2013/60_base-cabinets.mp4                              | 1 +        
+remote:  Learning SketchUp 2013/61_wall-cabinets.mp4                              | 1 +        
+remote:  Learning SketchUp 2013/62_cabinet-doors.mp4                              | 1 +        
+remote:  Learning SketchUp 2013/63_creating-the-countertop.mp4                    | 1 +        
+remote:  Learning SketchUp 2013/64_adding-materials.mp4                           | 1 +        
+remote:  Learning SketchUp 2013/65_adding-appliances.mp4                          | 1 +        
+remote:  Learning SketchUp 2013/66_adding-the-fridge-and-sink.mp4                 | 1 +        
+remote:  Learning SketchUp 2013/67_using-the-3d-warehouse.mp4                     | 1 +        
+remote:  Learning SketchUp 2013/68_using-form-fonts.mp4                           | 1 +        
+remote:  Learning SketchUp 2013/69_using-the-podium-browser.mp4                   | 1 +        
+remote:  Learning SketchUp 2013/70_creating-a-library.mp4                         | 1 +        
+remote:  Learning SketchUp 2013/71_setting-up-to-match-a-new-photo.mp4            | 1 +        
+remote:  Learning SketchUp 2013/72_creating-the-massing-model.mp4                 | 1 +        
+remote:  Learning SketchUp 2013/73_applying-textures.mp4                          | 1 +        
+remote:  Learning SketchUp 2013/74_adding-detail-chapter-13.mp4                   | 1 +        
+remote:  Learning SketchUp 2013/75_saving-and-placing-into-the-model.mp4          | 1 +        
+remote:  Learning SketchUp 2013/76_creating-the-exterior.mp4                      | 1 +        
+remote:  Learning SketchUp 2013/77_setting-the-building-in-place-geolocation.mp4  | 1 +        
+remote:  Learning SketchUp 2013/78_reset-axis-to-align-building.mp4               | 1 +        
+remote:  Learning SketchUp 2013/79_creating-the-front-facade.mp4                  | 1 +        
+remote:  Learning SketchUp 2013/80_adding-context.mp4                             | 1 +        
+remote:  Learning SketchUp 2013/81_get-photo-google-street-view.mp4               | 1 +        
+remote:  Learning SketchUp 2013/82_using-3d-text.mp4                              | 1 +        
+remote:  Learning SketchUp 2013/83_add-solar-panels-with-skelion-plugin.mp4       | 1 +        
+remote:  Learning SketchUp 2013/84_creating-scenes-and-updating.mp4               | 1 +        
+remote:  Learning SketchUp 2013/85_styles.mp4                                     | 1 +        
+remote:  Learning SketchUp 2013/86_scenes-for-animation.mp4                       | 1 +        
+remote:  Learning SketchUp 2013/87_section-cut-for-interior-elevations.mp4        | 1 +        
+remote:  Learning SketchUp 2013/88_creating-a-group.mp4                           | 1 +        
+remote:  Learning SketchUp 2013/89_creating-the-animation.mp4                     | 1 +        
+remote:  Learning SketchUp 2013/90_jpeg-and-png-files.mp4                         | 1 +        
+remote:  Learning SketchUp 2013/91_dwg-elevation-and-plan-pro-feature.mp4         | 1 +        
+remote:  Learning SketchUp 2013/92_kmz-for-google-earth.mp4                       | 1 +        
+remote:  Learning SketchUp 2013/93_collada-files.mp4                              | 1 +        
+remote:  Learning SketchUp 2013/94_animation-options.mp4                          | 1 +        
+remote:  Learning SketchUp 2013/95_add-final-touches.mp4                          | 1 +        
+remote:  Learning SketchUp 2013/96_export-display-options.mp4                     | 1 +        
+remote:  Learning SketchUp 2013/97_stacking-images-in-photoshop.mp4               | 1 +        
+remote:  Learning SketchUp 2013/98_multiply-and-opacity.mp4                       | 1 +        
+remote:  Learning SketchUp 2013/99_saving-the-image.mp4                           | 1 +        
+remote:  109 files changed, 109 insertions(+)        
+remote:  create mode 120000 Learning SketchUp 2013/01_introduction.mp4        
+remote:  create mode 120000 Learning SketchUp 2013/02_setting-up-the-template.mp4        
+remote:  create mode 120000 Learning SketchUp 2013/03_setting-up-the-toolbars.mp4        
+remote:  create mode 120000 Learning SketchUp 2013/04_how-to-access-your-working-files.mp4        
+remote:  create mode 120000 Learning SketchUp 2013/05_line-tool.mp4        
+remote:  create mode 120000 Learning SketchUp 2013/06_rectangle-tool.mp4        
+remote:  create mode 120000 Learning SketchUp 2013/07_circle-tool.mp4        
+remote:  create mode 120000 Learning SketchUp 2013/08_polygon-tool.mp4        
+remote:  create mode 120000 Learning SketchUp 2013/09_arc-tool.mp4        
+remote:  create mode 120000 Learning SketchUp 2013/100_adding-lighting.mp4        
+remote:  create mode 120000 Learning SketchUp 2013/101_add-reflections-to-existing-textures.mp4        
+remote:  create mode 120000 Learning SketchUp 2013/102_model-presets-create-a-preview-rendering.mp4        
+remote:  create mode 120000 Learning SketchUp 2013/103_rendering-a-high-quality-image.mp4        
+remote:  create mode 120000 Learning SketchUp 2013/104_post-processing-in-photoshop.mp4        
+remote:  create mode 120000 Learning SketchUp 2013/105_what-is-layout.mp4        
+remote:  create mode 120000 Learning SketchUp 2013/106_adjusting-the-model-settings.mp4        
+remote:  create mode 120000 Learning SketchUp 2013/108_adding-additional-pages-and-content.mp4        
+remote:  create mode 120000 Learning SketchUp 2013/109_updating-the-model-references.mp4        
+remote:  create mode 120000 Learning SketchUp 2013/10_freehand-tool.mp4        
+remote:  create mode 120000 Learning SketchUp 2013/111_about-dan-brown.mp4        
+remote:  create mode 120000 Learning SketchUp 2013/11_navigating-in-3d.mp4        
+remote:  create mode 120000 Learning SketchUp 2013/12_push-pull-and-move-tool.mp4        
+remote:  create mode 120000 Learning SketchUp 2013/13_using-follow-me-complex-roof-pitches.mp4        
+remote:  create mode 120000 Learning SketchUp 2013/14_creating-a-mansard-roof.mp4        
+remote:  create mode 120000 Learning SketchUp 2013/15_creating-a-gambrel-roof.mp4        
+remote:  create mode 120000 Learning SketchUp 2013/16_measurements.mp4        
+remote:  create mode 120000 Learning SketchUp 2013/18_extruding-your-walls-push-pull-and-offset.mp4        
+remote:  create mode 120000 Learning SketchUp 2013/19_creating-doors-and-openings.mp4        
+remote:  create mode 120000 Learning SketchUp 2013/20_clean-up-using-the-eraser-to-remove-unwanted-edges.mp4        
+remote:  create mode 120000 Learning SketchUp 2013/21_why-groups.mp4        
+remote:  create mode 120000 Learning SketchUp 2013/22_creating-the-second-floor.mp4        
+remote:  create mode 120000 Learning SketchUp 2013/23_creating-additional-openings.mp4        
+remote:  create mode 120000 Learning SketchUp 2013/24_hide-model-and-other-useful-shortcuts.mp4        
+remote:  create mode 120000 Learning SketchUp 2013/25_using-the-outliner.mp4        
+remote:  create mode 120000 Learning SketchUp 2013/26_using-layers-to-control-visibility.mp4        
+remote:  create mode 120000 Learning SketchUp 2013/27_viewing-layer-states-of-the-model.mp4        
+remote:  create mode 120000 Learning SketchUp 2013/28_field-of-view.mp4        
+remote:  create mode 120000 Learning SketchUp 2013/29_dividing-faces-to-create-your-rooms.mp4        
+remote:  create mode 120000 Learning SketchUp 2013/30_area-tool-for-calculations.mp4        
+remote:  create mode 120000 Learning SketchUp 2013/31_using-paint-bucket-for-floor-texture-2.mp4        
+remote:  create mode 120000 Learning SketchUp 2013/31_using-paint-bucket-for-floor-texture.mp4        
+remote:  create mode 120000 Learning SketchUp 2013/32_importing-and-applying-a-texture.mp4        
+remote:  create mode 120000 Learning SketchUp 2013/33_positioning-an-imported-texture.mp4        
+remote:  create mode 120000 Learning SketchUp 2013/34_editing-texture-colour-in-sketchup.mp4        
+remote:  create mode 120000 Learning SketchUp 2013/35_editing-a-texture-in-photoshop.mp4        
+remote:  create mode 120000 Learning SketchUp 2013/36_hide-edges-and-lines-with-the-eraser-tool.mp4        
+remote:  create mode 120000 Learning SketchUp 2013/37_creating-the-ceiling.mp4        
+remote:  create mode 120000 Learning SketchUp 2013/38_assigning-layers-within-groups.mp4        
+remote:  create mode 120000 Learning SketchUp 2013/39_adding-detail-chapter-7.mp4        
+remote:  create mode 120000 Learning SketchUp 2013/40_creating-crown-moulding.mp4        
+remote:  create mode 120000 Learning SketchUp 2013/41_copy-and-paste-in-place.mp4        
+remote:  create mode 120000 Learning SketchUp 2013/42_why-we-use-groups-and-components.mp4        
+remote:  create mode 120000 Learning SketchUp 2013/43_creating-the-staircase.mp4        
+remote:  create mode 120000 Learning SketchUp 2013/44_adding-riser-detail.mp4        
+remote:  create mode 120000 Learning SketchUp 2013/45_making-a-unique-bottom-step.mp4        
+remote:  create mode 120000 Learning SketchUp 2013/46_adding-spindles-with-move-and-copy.mp4        
+remote:  create mode 120000 Learning SketchUp 2013/47_adding-railing.mp4        
+remote:  create mode 120000 Learning SketchUp 2013/48_review-of-components-and-nested-groups.mp4        
+remote:  create mode 120000 Learning SketchUp 2013/49_creating-doors.mp4        
+remote:  create mode 120000 Learning SketchUp 2013/50_creating-the-front-window-frame.mp4        
+remote:  create mode 120000 Learning SketchUp 2013/51_double-hung-windows.mp4        
+remote:  create mode 120000 Learning SketchUp 2013/52_installing-bit-tools-plugins.mp4        
+remote:  create mode 120000 Learning SketchUp 2013/53_creating-walls-and-door-openings.mp4        
+remote:  create mode 120000 Learning SketchUp 2013/54_using-the-window-feature.mp4        
+remote:  create mode 120000 Learning SketchUp 2013/55_starting-at-the-base.mp4        
+remote:  create mode 120000 Learning SketchUp 2013/56_building-the-mantle.mp4        
+remote:  create mode 120000 Learning SketchUp 2013/57_mantle-columns.mp4        
+remote:  create mode 120000 Learning SketchUp 2013/58_mantle-support.mp4        
+remote:  create mode 120000 Learning SketchUp 2013/59_adding-textures.mp4        
+remote:  create mode 120000 Learning SketchUp 2013/60_base-cabinets.mp4        
+remote:  create mode 120000 Learning SketchUp 2013/61_wall-cabinets.mp4        
+remote:  create mode 120000 Learning SketchUp 2013/62_cabinet-doors.mp4        
+remote:  create mode 120000 Learning SketchUp 2013/63_creating-the-countertop.mp4        
+remote:  create mode 120000 Learning SketchUp 2013/64_adding-materials.mp4        
+remote:  create mode 120000 Learning SketchUp 2013/65_adding-appliances.mp4        
+remote:  create mode 120000 Learning SketchUp 2013/66_adding-the-fridge-and-sink.mp4        
+remote:  create mode 120000 Learning SketchUp 2013/67_using-the-3d-warehouse.mp4        
+remote:  create mode 120000 Learning SketchUp 2013/68_using-form-fonts.mp4        
+remote:  create mode 120000 Learning SketchUp 2013/69_using-the-podium-browser.mp4        
+remote:  create mode 120000 Learning SketchUp 2013/70_creating-a-library.mp4        
+remote:  create mode 120000 Learning SketchUp 2013/71_setting-up-to-match-a-new-photo.mp4        
+remote:  create mode 120000 Learning SketchUp 2013/72_creating-the-massing-model.mp4        
+remote:  create mode 120000 Learning SketchUp 2013/73_applying-textures.mp4        
+remote:  create mode 120000 Learning SketchUp 2013/74_adding-detail-chapter-13.mp4        
+remote:  create mode 120000 Learning SketchUp 2013/75_saving-and-placing-into-the-model.mp4        
+remote:  create mode 120000 Learning SketchUp 2013/76_creating-the-exterior.mp4        
+remote:  create mode 120000 Learning SketchUp 2013/77_setting-the-building-in-place-geolocation.mp4        
+remote:  create mode 120000 Learning SketchUp 2013/78_reset-axis-to-align-building.mp4        
+remote:  create mode 120000 Learning SketchUp 2013/79_creating-the-front-facade.mp4        
+remote:  create mode 120000 Learning SketchUp 2013/80_adding-context.mp4        
+remote:  create mode 120000 Learning SketchUp 2013/81_get-photo-google-street-view.mp4        
+remote:  create mode 120000 Learning SketchUp 2013/82_using-3d-text.mp4        
+remote:  create mode 120000 Learning SketchUp 2013/83_add-solar-panels-with-skelion-plugin.mp4        
+remote:  create mode 120000 Learning SketchUp 2013/84_creating-scenes-and-updating.mp4        
+remote:  create mode 120000 Learning SketchUp 2013/85_styles.mp4        
+remote:  create mode 120000 Learning SketchUp 2013/86_scenes-for-animation.mp4        
+remote:  create mode 120000 Learning SketchUp 2013/87_section-cut-for-interior-elevations.mp4        
+remote:  create mode 120000 Learning SketchUp 2013/88_creating-a-group.mp4        
+remote:  create mode 120000 Learning SketchUp 2013/89_creating-the-animation.mp4        
+remote:  create mode 120000 Learning SketchUp 2013/90_jpeg-and-png-files.mp4        
+remote:  create mode 120000 Learning SketchUp 2013/91_dwg-elevation-and-plan-pro-feature.mp4        
+remote:  create mode 120000 Learning SketchUp 2013/92_kmz-for-google-earth.mp4        
+remote:  create mode 120000 Learning SketchUp 2013/93_collada-files.mp4        
+remote:  create mode 120000 Learning SketchUp 2013/94_animation-options.mp4        
+remote:  create mode 120000 Learning SketchUp 2013/95_add-final-touches.mp4        
+remote:  create mode 120000 Learning SketchUp 2013/96_export-display-options.mp4        
+remote:  create mode 120000 Learning SketchUp 2013/97_stacking-images-in-photoshop.mp4        
+remote:  create mode 120000 Learning SketchUp 2013/98_multiply-and-opacity.mp4        
+remote:  create mode 120000 Learning SketchUp 2013/99_saving-the-image.mp4        
+     8224768  60%    2.54MB/s    0:00:02
+remote: 
+remote: ok        
+remote: (Recording state in git...)        
+[2014-05-25 21:41:47 CEST] RemoteControl: Syncing with diskstation 
+To joern@annex.diskstation.local:/volume1/homes/joern/Annex.movies
+   e459e2f..26c6449  git-annex -> synced/git-annex
+   6f2044d..7500a88  master -> synced/master
+    10911744  80%    2.52MB/s    0:00:01
+    13603741 100%    2.69MB/s    0:00:04 (xfer#1, to-check=0/1)
+
+sent 13605572 bytes  received 42 bytes  2473748.00 bytes/sec
+total size is 13603741  speedup is 1.00
+[2014-05-25 21:41:49 CEST] Transferrer: Uploaded 32_import..xture.mp4
+
+WORM-s13731971-m1398386799--Learning SketchUp 2013%31_using-paint-bucket-for-floor-texture.mp4
+       32768   0%    0.00kB/s    0:00:00
+     2752512  20%    2.48MB/s    0:00:04
+From annex.diskstation.local:/volume1/homes/joern/Annex.movies
+   297826c..ee2d7a6  git-annex  -> diskstation/git-annex
+error: Ref refs/remotes/diskstation/master is at 7500a886b58db7f0a3d2d279a946215525109cd2 but expected 6f2044d781d283993104edb9f8b96041801c1e76
+ ! 6f2044d..7500a88  master     -> diskstation/master  (unable to update local ref)
+[2014-05-25 21:41:51 CEST] Pusher: Syncing with diskstation 
+     5373952  39%    2.45MB/s    0:00:03
+     9404416  68%    2.88MB/s    0:00:01
+    12681216  92%    2.63MB/s    0:00:00
+    13731971 100%    2.84MB/s    0:00:04 (xfer#1, to-check=0/1)
+
+sent 13733822 bytes  received 42 bytes  2497066.18 bytes/sec
+total size is 13731971  speedup is 1.00
+[2014-05-25 21:41:54 CEST] Transferrer: Uploaded 31_using-..xture.mp4
+
+WORM-s13731971-m1398386799--Learning SketchUp 2013%31_using-paint-bucket-for-floor-texture-2.mp4
+       32768   0%    0.00kB/s    0:00:00
+remote: merge git-annex      3211264  23%    2.96MB/s    0:00:03
+     7634944  55%    3.55MB/s    0:00:01
+[2014-05-25 21:41:57 CEST] RemoteControl: Syncing with diskstation 
+(merging synced/git-annex into git-annex...)        
+remote: (Recording state in git...)        
+remote: ok        
+To joern@annex.diskstation.local:/volume1/homes/joern/Annex.movies
+   26c6449..9d66740  git-annex -> synced/git-annex
+    11370496  82%    3.54MB/s    0:00:00
+    13731971 100%    3.62MB/s    0:00:03 (xfer#1, to-check=0/1)
+
+sent 13733824 bytes  received 42 bytes  2497066.55 bytes/sec
+total size is 13731971  speedup is 1.00
+[2014-05-25 21:41:59 CEST] Transferrer: Uploaded 31_using-..ure-2.mp4
+From annex.diskstation.local:/volume1/homes/joern/Annex.movies
+   ee2d7a6..be030de  git-annex  -> diskstation/git-annex
+error: Ref refs/remotes/diskstation/synced/git-annex is at 9d667409bdd47098b595460d26a4a47a7477b602 but expected 26c64492572c692eb6c5e3af0922849d42142cc5
+ ! 26c6449..9d66740  synced/git-annex -> diskstation/synced/git-annex  (unable to update local ref)
+
+WORM-s6958518-m1398385748--Learning SketchUp 2013%30_area-tool-for-calculations.mp4
+       32768   0%    0.00kB/s    0:00:00
+     4096000  58%    3.77MB/s    0:00:00
+[2014-05-25 21:42:01 CEST] Pusher: Syncing with diskstation 
+     5505024  79%    2.56MB/s    0:00:00
+     6958518 100%    2.63MB/s    0:00:02 (xfer#1, to-check=0/1)
+
+sent 6959530 bytes  received 42 bytes  1988449.14 bytes/sec
+total size is 6958518  speedup is 1.00
+[2014-05-25 21:42:02 CEST] Transferrer: Uploaded 30_area-t..tions.mp4
+
+WORM-s17894336-m1398385618--Learning SketchUp 2013%29_dividing-faces-to-create-your-rooms.mp4
+       32768   0%    0.00kB/s    0:00:00
+remote: merge git-annex      5210112  29%    4.90MB/s    0:00:02
+     8617984  48%    3.99MB/s    0:00:02
+    10715136  59%    3.34MB/s    0:00:02
+[2014-05-25 21:42:07 CEST] RemoteControl: Syncing with diskstation 
+(merging synced/git-annex into git-annex...)        
+remote: (Recording state in git...)        
+remote: ok        
+To joern@annex.diskstation.local:/volume1/homes/joern/Annex.movies
+   9d66740..2f65ea4  git-annex -> synced/git-annex
+    13336576  74%    3.11MB/s    0:00:01
+    16842752  94%    2.72MB/s    0:00:00
+    17894336 100%    3.16MB/s    0:00:05 (xfer#1, to-check=0/1)
+
+sent 17896694 bytes  received 42 bytes  2753344.00 bytes/sec
+total size is 17894336  speedup is 1.00
+[2014-05-25 21:42:09 CEST] Transferrer: Uploaded 29_dividi..rooms.mp4
+From annex.diskstation.local:/volume1/homes/joern/Annex.movies
+   be030de..afc435c  git-annex  -> diskstation/git-annex
+error: Ref refs/remotes/diskstation/synced/git-annex is at 2f65ea464ab39db40b09478a6857d41450b394d3 but expected 9d667409bdd47098b595460d26a4a47a7477b602
+ ! 9d66740..2f65ea4  synced/git-annex -> diskstation/synced/git-annex  (unable to update local ref)
+[2014-05-25 21:42:10 CEST] Committer: Adding 107_addin..sions.mp4 110_prese..aving.mp4
+ok
+(Recording state in git...)
+(Recording state in git...)
+
+(merging diskstation/git-annex into git-annex...)
+(Recording state in git...)
+
+(merging diskstation/git-annex into git-annex...)
+(Recording state in git...)
+(merging diskstation/git-annex into git-annex...)
+(Recording state in git...)
+(merging diskstation/git-annex into git-annex...)
+(Recording state in git...)
+add Learning SketchUp 2013/107_adding-annotations-and-dimensions.mp4 ok
+add Learning SketchUp 2013/110_presenting-and-saving.mp4 [2014-05-25 21:42:10 CEST] Committer: Committing changes to git
+
+WORM-s16183756-m1398386984--Learning SketchUp 2013%28_field-of-view.mp4
+       32768   0%    0.00kB/s    0:00:00
+[2014-05-25 21:42:11 CEST] Pusher: Syncing with diskstation 
+     5177344  31%    4.80MB/s    0:00:02
+Already up-to-date.
+     9306112  57%    4.38MB/s    0:00:01
+    13434880  83%    4.21MB/s    0:00:00
+remote: merge git-annex     16183756 100%    3.91MB/s    0:00:03 (xfer#1, to-check=0/1)
+[2014-05-25 21:42:16 CEST] RemoteControl: Syncing with diskstation 
+
+sent 16185880 bytes  received 42 bytes  2942894.91 bytes/sec
+total size is 16183756  speedup is 1.00
+[2014-05-25 21:42:16 CEST] Transferrer: Uploaded 28_field-..-view.m(merging synced/git-annex into git-annex...)        
+premote: (Recording state in git...)        
+4remote: ok        
+remote: merge synced/master 
+
+WORM-s8653832-m1398386968--Learning SketchUp 2013%110_presenting-and-saving.mp4
+       32768   0%    0.00kB/s    0:00:00
+From annex.diskstation.local:/volume1/homes/joern/Annex.movies
+   afc435c..2feeb97  git-annex  -> diskstation/git-annex
+   2f65ea4..de80d6d  synced/git-annex -> diskstation/synced/git-annex
+   7500a88..af4c173  synced/master -> diskstation/synced/master
+     4390912  50%    4.13MB/s    0:00:01
+Already up-to-date.
+     7045120  81%    3.30MB/s    0:00:00
+Updating 7500a88..af4c173        
+remote: Fast-forward        
+remote:  Learning SketchUp 2013/107_adding-annotations-and-dimensions.mp4 | 1 +        
+remote:  Learning SketchUp 2013/110_presenting-and-saving.mp4             | 1 +        
+remote:  2 files changed, 2 insertions(+)        
+remote:  create mode 120000 Learning SketchUp 2013/107_adding-annotations-and-dimensions.mp4        
+remote:  create mode 120000 Learning SketchUp 2013/110_presenting-and-saving.mp4        
+     8653832 100%    3.26MB/s    0:00:02 (xfer#1, to-check=0/1)
+remote: 
+remote: ok        
+remote: (Recording state in git...)        
+To joern@annex.diskstation.local:/volume1/homes/joern/Annex.movies
+   2f65ea4..de80d6d  git-annex -> synced/git-annex
+   7500a88..af4c173  master -> synced/master
+
+sent 8655048 bytes  received 42 bytes  1923353.33 bytes/sec
+total size is 8653832  speedup is 1.00
+[2014-05-25 21:42:20 CEST] Transferrer: Uploaded 110_prese..aving.mp4
+[2014-05-25 21:42:20 CEST] RemoteControl: Syncing with diskstation 
+
+WORM-s12740321-m1398386533--Learning SketchUp 2013%107_adding-annotations-and-dimensions.mp4
+       32768   0%    0.00kB/s    0:00:00
+From annex.diskstation.local:/volume1/homes/joern/Annex.movies
+   2feeb97..c393949  git-annex  -> diskstation/git-annex
+   7500a88..af4c173  master     -> diskstation/master
+     4685824  36%    4.30MB/s    0:00:01
+     7667712  60%    3.56MB/s    0:00:01
+    10518528  82%    3.25MB/s    0:00:00
+[2014-05-25 21:42:24 CEST] Pusher: Syncing with diskstation 
+    12740321 100%    3.20MB/s    0:00:03 (xfer#1, to-check=0/1)
+
+sent 12742046 bytes  received 42 bytes  2316743.27 bytes/sec
+total size is 12740321  speedup is 1.00
+[2014-05-25 21:42:25 CEST] Transferrer: Uploaded 107_addin..sions.mp4
+
+WORM-s21683651-m1398384839--Learning SketchUp 2013%27_viewing-layer-states-of-the-model.mp4
+       32768   0%    0.00kB/s    0:00:00
+remote: merge git-annex      4358144  20%    4.11MB/s    0:00:04
+     7438336  34%    3.47MB/s    0:00:04
+[2014-05-25 21:42:29 CEST] RemoteControl: Syncing with diskstation 
+(merging synced/git-annex into git-annex...)        
+remote: (Recording state in git...)        
+remote: ok        
+To joern@annex.diskstation.local:/volume1/homes/joern/Annex.movies
+   de80d6d..da93668  git-annex -> synced/git-annex
+     9273344  42%    2.81MB/s    0:00:04
+    12189696  56%    2.78MB/s    0:00:03
+    15532032  71%    2.54MB/s    0:00:02
+From annex.diskstation.local:/volume1/homes/joern/Annex.movies
+   c393949..9f19ee8  git-annex  -> diskstation/git-annex
+error: Ref refs/remotes/diskstation/synced/git-annex is at da936685ca807cb619e2d6073c857cdcdc345dde but expected de80d6dd537ca7e4bfade9d40eade18afc58878b
+ ! de80d6d..da93668  synced/git-annex -> diskstation/synced/git-annex  (unable to update local ref)
+    19300352  89%    2.70MB/s    0:00:00
+    21683651 100%    2.95MB/s    0:00:06 (xfer#1, to-check=0/1)
+[2014-05-25 21:42:33 CEST] Pusher: Syncing with diskstation 
+
+sent 21686467 bytes  received 42 bytes  2551354.00 bytes/sec
+total size is 21683651  speedup is 1.00
+[2014-05-25 21:42:33 CEST] Transferrer: Uploaded 27_viewin..model.mp4
+
+WORM-s16669986-m1398385184--Learning SketchUp 2013%26_using-layers-to-control-visibility.mp4
+       32768   0%    0.00kB/s    0:00:00
+remote: merge git-annex      4259840  25%    3.95MB/s    0:00:03
+     7274496  43%    3.37MB/s    0:00:02
+[2014-05-25 21:42:37 CEST] RemoteControl: Syncing with diskstation 
+(merging synced/git-annex into git-annex...)        
+remote: (Recording state in git...)        
+remote: ok        
+To joern@annex.diskstation.local:/volume1/homes/joern/Annex.movies
+   da93668..fc553e7  git-annex -> synced/git-annex
+    10387456  62%    3.21MB/s    0:00:01
+    13500416  80%    3.14MB/s    0:00:00
+    16613376  99%    2.87MB/s    0:00:00
+    16669986 100%    3.09MB/s    0:00:05 (xfer#1, to-check=0/1)
+From annex.diskstation.local:/volume1/homes/joern/Annex.movies
+   9f19ee8..a8b9203  git-annex  -> diskstation/git-annex
+error: Ref refs/remotes/diskstation/synced/git-annex is at fc553e72a5ea5a077a254df3f557b3d6241d9342 but expected da936685ca807cb619e2d6073c857cdcdc345dde
+ ! da93668..fc553e7  synced/git-annex -> diskstation/synced/git-annex  (unable to update local ref)
+
+sent 16672191 bytes  received 42 bytes  2564958.92 bytes/sec
+total size is 16669986  speedup is 1.00
+[2014-05-25 21:42:40 CEST] Transferrer: Uploaded 26_using-..ility.mp4
+
+WORM-s8161367-m1398386819--Learning SketchUp 2013%25_using-the-outliner.mp4
+       32768   0%    0.00kB/s    0:00:00
+     4620288  56%    4.32MB/s    0:00:00
+[2014-05-25 21:42:41 CEST] Pusher: Syncing with diskstation 
+     8161367 100%    3.89MB/s    0:00:01 (xfer#1, to-check=0/1)
+
+sent 8162519 bytes  received 42 bytes  3265024.40 bytes/sec
+total size is 8161367  speedup is 1.00
+[2014-05-25 21:42:42 CEST] Transferrer: Uploaded 25_using-..liner.mp4
+
+WORM-s26081165-m1398387035--Learning SketchUp 2013%24_hide-model-and-other-useful-shortcuts.mp4
+       32768   0%    0.00kB/s    0:00:00
+     4620288  17%    4.27MB/s    0:00:04
+remote: merge git-annex      6193152  23%    2.82MB/s    0:00:06
+     9568256  36%    2.93MB/s    0:00:05
+    13631488  52%    3.16MB/s    0:00:03
+[2014-05-25 21:42:47 CEST] RemoteControl: Syncing with diskstation 
+(merging synced/git-annex into git-annex...)        
+remote: (Recording state in git...)        
+remote: ok        
+To joern@annex.diskstation.local:/volume1/homes/joern/Annex.movies
+   fc553e7..7eccd72  git-annex -> synced/git-annex
+    17203200  65%    2.92MB/s    0:00:02
+    21561344  82%    3.62MB/s    0:00:01
+    25362432  97%    3.72MB/s    0:00:00
+    26081165 100%    3.37MB/s    0:00:07 (xfer#1, to-check=0/1)
+From annex.diskstation.local:/volume1/homes/joern/Annex.movies
+   a8b9203..829ce2e  git-annex  -> diskstation/git-annex
+error: Ref refs/remotes/diskstation/synced/git-annex is at 7eccd720d952d1ec3a1ee00cc524732ec970285f but expected fc553e72a5ea5a077a254df3f557b3d6241d9342
+ ! fc553e7..7eccd72  synced/git-annex -> diskstation/synced/git-annex  (unable to update local ref)
+
+sent 26084521 bytes  received 42 bytes  2745743.47 bytes/sec
+total size is 26081165  speedup is 1.00
+[2014-05-25 21:42:51 CEST] Transferrer: Uploaded 24_hide-m..tcuts.mp4
+
+WORM-s24710536-m1398386140--Learning SketchUp 2013%23_creating-additional-openings.mp4
+       32768   0%    0.00kB/s    0:00:00
+[2014-05-25 21:42:52 CEST] Pusher: Syncing with diskstation 
+     5111808  20%    4.80MB/s    0:00:03
+     9142272  36%    4.25MB/s    0:00:03
+    12648448  51%    3.92MB/s    0:00:03
+    16023552  64%    3.71MB/s    0:00:02
+remote: merge git-annex     19365888  78%    3.27MB/s    0:00:01
+    22478848  90%    3.05MB/s    0:00:00
+[2014-05-25 21:42:58 CEST] RemoteControl: Syncing with diskstation 
+(merging synced/git-annex into git-annex...)        
+remote: (Recording state in git...)        
+remote: ok        
+To joern@annex.diskstation.local:/volume1/homes/joern/Annex.movies
+   7eccd72..5dd7ceb  git-annex -> synced/git-annex
+    24710536 100%    3.34MB/s    0:00:07 (xfer#1, to-check=0/1)
+
+sent 24713719 bytes  received 42 bytes  2907501.29 bytes/sec
+total size is 24710536  speedup is 1.00
+[2014-05-25 21:42:59 CEST] Transferrer: Uploaded 23_creati..nings.mp4
+
+WORM-s27007278-m1398386399--Learning SketchUp 2013%22_creating-the-second-floor.mp4
+       32768   0%    0.00kB/s    0:00:00
+From annex.diskstation.local:/volume1/homes/joern/Annex.movies
+   829ce2e..432e319  git-annex  -> diskstation/git-annex
+error: Ref refs/remotes/diskstation/synced/git-annex is at 5dd7ceb17a57b1f11edb692c3c20eb05f7c89aa7 but expected 7eccd720d952d1ec3a1ee00cc524732ec970285f
+ ! 7eccd72..5dd7ceb  synced/git-annex -> diskstation/synced/git-annex  (unable to update local ref)
+     4194304  15%    3.94MB/s    0:00:05
+     6946816  25%    3.24MB/s    0:00:06
+[2014-05-25 21:43:02 CEST] Pusher: Syncing with diskstation 
+     9830400  36%    3.06MB/s    0:00:05
+    12582912  46%    2.96MB/s    0:00:04
+remote: merge git-annex     15761408  58%    2.73MB/s    0:00:04
+    18481152  68%    2.71MB/s    0:00:03
+[2014-05-25 21:43:06 CEST] RemoteControl: Syncing with diskstation 
+(merging synced/git-annex into git-annex...)        
+remote: (Recording state in git...)        
+remote: ok        
+To joern@annex.diskstation.local:/volume1/homes/joern/Annex.movies
+   5dd7ceb..114c7a8  git-annex -> synced/git-annex
+    21889024  81%    2.82MB/s    0:00:01
+    24805376  91%    2.86MB/s    0:00:00
+    27007278 100%    2.89MB/s    0:00:08 (xfer#1, to-check=0/1)
+From annex.diskstation.local:/volume1/homes/joern/Annex.movies
+   432e319..ee3c95a  git-annex  -> diskstation/git-annex
+error: Ref refs/remotes/diskstation/synced/git-annex is at 114c7a8e0d0873ea44ebfd8a5989b7e4f6fe300d but expected 5dd7ceb17a57b1f11edb692c3c20eb05f7c89aa7
+ ! 5dd7ceb..114c7a8  synced/git-annex -> diskstation/synced/git-annex  (unable to update local ref)
+
+sent 27010738 bytes  received 42 bytes  2572455.24 bytes/sec
+total size is 27007278  speedup is 1.00
+[2014-05-25 21:43:09 CEST] Transferrer: Uploaded 22_creati..floor.mp4
+
+WORM-s11952523-m1398387004--Learning SketchUp 2013%21_why-groups.mp4
+       32768   0%    0.00kB/s    0:00:00
+     3538944  29%    3.13MB/s    0:00:02
+[2014-05-25 21:43:11 CEST] Pusher: Syncing with diskstation 
+     4849664  40%    2.11MB/s    0:00:03
+     8257536  69%    2.44MB/s    0:00:01
+    11272192  94%    2.54MB/s    0:00:00
+remote: merge git-annex     11952523 100%    2.56MB/s    0:00:04 (xfer#1, to-check=0/1)
+[2014-05-25 21:43:15 CEST] RemoteControl: Syncing with diskstation 
+
+sent 11954128 bytes  received 42 bytes  2173485.45 bytes/sec
+total size is 11952523  speedup is 1.00
+[2014-05-25 21:43:15 CEST] Transferrer: Uploaded 21_why-groups.mp4
+
+WORM-s14698958-m1401045757--Learning SketchUp 2013%20_clean-up-using-the-eraser-to-remove-unwanted-edges.mp4
+       32768   0%    0.00kB/s    0:00:00
+(merging synced/git-annex into git-annex...)        
+remote: (Recording state in git...)        
+remote: ok        
+remote: (Recording state in git...)        
+To joern@annex.diskstation.local:/volume1/homes/joern/Annex.movies
+   114c7a8..70d2843  git-annex -> synced/git-annex
+From annex.diskstation.local:/volume1/homes/joern/Annex.movies
+   ee3c95a..292456c  git-annex  -> diskstation/git-annex
+error: Ref refs/remotes/diskstation/synced/git-annex is at 70d2843ca3fd83291be9e3a83c8a2b1d393344fc but expected 114c7a8e0d0873ea44ebfd8a5989b7e4f6fe300d
+ ! 114c7a8..70d2843  synced/git-annex -> diskstation/synced/git-annex  (unable to update local ref)
+[2014-05-25 21:43:17 CEST] RemoteControl: Syncing with diskstation 
+     4489216  30%    4.09MB/s    0:00:02
+     7634944  51%    3.53MB/s    0:00:01
+    10715136  72%    3.32MB/s    0:00:01
+From annex.diskstation.local:/volume1/homes/joern/Annex.movies
+   292456c..489a02f  git-annex  -> diskstation/git-annex
+    14254080  96%    3.32MB/s    0:00:00
+    14698958 100%    3.27MB/s    0:00:04 (xfer#1, to-check=0/1)
+
+sent 14700939 bytes  received 42 bytes  2261689.38 bytes/sec
+total size is 14698958  speedup is 1.00
+[2014-05-25 21:43:21 CEST] Transferrer: Uploaded 20_clean-..edges.mp4
+[2014-05-25 21:43:21 CEST] Pusher: Syncing with diskstation 
+
+WORM-s53078315-m1401045818--Learning SketchUp 2013%19_creating-doors-and-openings.mp4
+       32768   0%    0.00kB/s    0:00:00
+     5046272   9%    4.66MB/s    0:00:10
+     9109504  17%    4.21MB/s    0:00:10
+remote: merge git-annex     12779520  24%    3.96MB/s    0:00:09
+    16187392  30%    3.76MB/s    0:00:09
+    20676608  38%    3.66MB/s    0:00:08
+[2014-05-25 21:43:27 CEST] RemoteControl: Syncing with diskstation 
+(merging synced/git-annex into git-annex...)        
+remote: (Recording state in git...)        
+remote: ok        
+To joern@annex.diskstation.local:/volume1/homes/joern/Annex.movies
+   70d2843..0d8c767  git-annex -> synced/git-annex
+    24739840  46%    3.67MB/s    0:00:07
+    28573696  53%    3.71MB/s    0:00:06
+    32243712  60%    3.76MB/s    0:00:05
+From annex.diskstation.local:/volume1/homes/joern/Annex.movies
+   489a02f..7a7f5ca  git-annex  -> diskstation/git-annex
+error: Ref refs/remotes/diskstation/synced/git-annex is at 0d8c76720fa6b51cf2cf5edf847e732dd550ab40 but expected 70d2843ca3fd83291be9e3a83c8a2b1d393344fc
+ ! 70d2843..0d8c767  synced/git-annex -> diskstation/synced/git-annex  (unable to update local ref)
+    36765696  69%    3.74MB/s    0:00:04
+    40697856  76%    3.72MB/s    0:00:03
+    44564480  83%    3.73MB/s    0:00:02
+    47316992  89%    3.53MB/s    0:00:01
+    50331648  94%    3.18MB/s    0:00:00
+    53078315 100%    3.62MB/s    0:00:13 (xfer#1, to-check=0/1)
+
+sent 53084957 bytes  received 42 bytes  3424838.65 bytes/sec
+total size is 53078315  speedup is 1.00
+[2014-05-25 21:43:36 CEST] Transferrer: Uploaded 19_creati..nings.mp4
+[2014-05-25 21:43:36 CEST] Pusher: Syncing with diskstation 
+
+WORM-s30352251-m1398385646--Learning SketchUp 2013%18_extruding-your-walls-push-pull-and-offset.mp4
+       32768   0%    0.00kB/s    0:00:00
+     4784128  15%    4.50MB/s    0:00:05
+remote: merge git-annex      8159232  26%    3.81MB/s    0:00:05
+    10256384  33%    3.17MB/s    0:00:06
+[2014-05-25 21:43:40 CEST] RemoteControl: Syncing with diskstation 
+(merging synced/git-annex into git-annex...)        
+remote: (Recording state in git...)        
+remote: ok        
+To joern@annex.diskstation.local:/volume1/homes/joern/Annex.movies
+   0d8c767..6f8ae3d  git-annex -> synced/git-annex
+    13271040  43%    3.06MB/s    0:00:05
+    16318464  53%    2.65MB/s    0:00:05
+From annex.diskstation.local:/volume1/homes/joern/Annex.movies
+   7a7f5ca..bef94a6  git-annex  -> diskstation/git-annex
+error: Ref refs/remotes/diskstation/synced/git-annex is at 6f8ae3d49d1006e2d5cf957d7539c0818db4df12 but expected 0d8c76720fa6b51cf2cf5edf847e732dd550ab40
+ ! 0d8c767..6f8ae3d  synced/git-annex -> diskstation/synced/git-annex  (unable to update local ref)
+    19070976  62%    2.51MB/s    0:00:04
+    21954560  72%    2.71MB/s    0:00:03
+    24969216  82%    2.72MB/s    0:00:01
+    27852800  91%    2.69MB/s    0:00:00
+    30352251 100%    2.87MB/s    0:00:10 (xfer#1, to-check=0/1)
+
+sent 30356135 bytes  received 42 bytes  2639667.57 bytes/sec
+total size is 30352251  speedup is 1.00
+[2014-05-25 21:43:48 CEST] Transferrer: Uploaded 18_extrud..ffset.mp4
+[2014-05-25 21:43:48 CEST] Pusher: Syncing with diskstation 
+
+WORM-s14102612-m1398386513--Learning SketchUp 2013%16_measurements.mp4
+       32768   0%    0.00kB/s    0:00:00
+     4554752  32%    4.23MB/s    0:00:02
+     7634944  54%    3.59MB/s    0:00:01
+remote: merge git-annex     10584064  75%    3.29MB/s    0:00:01
+[2014-05-25 21:43:52 CEST] RemoteControl: Syncing with diskstation 
+(merging synced/git-annex into git-annex...)        
+remote: (Recording state in git...)        
+remote: ok        
+To joern@annex.diskstation.local:/volume1/homes/joern/Annex.movies
+   6f8ae3d..32eee1a  git-annex -> synced/git-annex
+    13434880  95%    3.13MB/s    0:00:00
+    14102612 100%    3.14MB/s    0:00:04 (xfer#1, to-check=0/1)
+
+sent 14104483 bytes  received 42 bytes  2564459.09 bytes/sec
+total size is 14102612  speedup is 1.00
+[2014-05-25 21:43:53 CEST] Transferrer: Uploaded 16_measurements.mp4
+
+WORM-s16792198-m1398384994--Learning SketchUp 2013%15_creating-a-gambrel-roof.mp4
+       32768   0%    0.00kB/s    0:00:00
+From annex.diskstation.local:/volume1/homes/joern/Annex.movies
+   bef94a6..10025f1  git-annex  -> diskstation/git-annex
+error: Ref refs/remotes/diskstation/synced/git-annex is at 32eee1aff2a03a33e7cc200fefcef41ce775767e but expected 6f8ae3d49d1006e2d5cf957d7539c0818db4df12
+ ! 6f8ae3d..32eee1a  synced/git-annex -> diskstation/synced/git-annex  (unable to update local ref)
+     4128768  24%    3.88MB/s    0:00:03
+     7012352  41%    3.26MB/s    0:00:02
+[2014-05-25 21:43:56 CEST] Pusher: Syncing with diskstation 
+     9895936  58%    3.07MB/s    0:00:02
+    12648448  75%    2.95MB/s    0:00:01
+remote: merge git-annex     14745600  87%    2.48MB/s    0:00:00
+    16416768  97%    2.20MB/s    0:00:00
+    16792198 100%    2.58MB/s    0:00:06 (xfer#1, to-check=0/1)
+[2014-05-25 21:44:01 CEST] RemoteControl: Syncing with diskstation 
+
+sent 16794408 bytes  received 42 bytes  1975817.65 bytes/sec
+total size is 16792198  speedup is 1.00
+[2014-05-25 21:44:01 CEST] Transferrer: Uploaded 15_creati..-roof.mp4
+
+WORM-s33139501-m1398386446--Learning SketchUp 2013%14_creating-a-mansard-roof.mp4
+       32768   0%    0.00kB/s    0:00:00
+(merging synced/git-annex into git-annex...)        
+remote: (Recording state in git...)        
+remote: ok        
+remote: (Recording state in git...)        
+To joern@annex.diskstation.local:/volume1/homes/joern/Annex.movies
+   32eee1a..cc461b8  git-annex -> synced/git-annex
+     4685824  14%    4.39MB/s    0:00:06
+From annex.diskstation.local:/volume1/homes/joern/Annex.movies
+   10025f1..f39db09  git-annex  -> diskstation/git-annex
+error: Ref refs/remotes/diskstation/synced/git-annex is at cc461b8c48eb38d8f0b020a9c22440dada4973f0 but expected 32eee1aff2a03a33e7cc200fefcef41ce775767e
+ ! 32eee1a..cc461b8  synced/git-annex -> diskstation/synced/git-annex  (unable to update local ref)
+[2014-05-25 21:44:03 CEST] RemoteControl: Syncing with diskstation 
+     8159232  24%    3.84MB/s    0:00:06
+    11730944  35%    3.67MB/s    0:00:05
+From annex.diskstation.local:/volume1/homes/joern/Annex.movies
+   f39db09..d356ca4  git-annex  -> diskstation/git-annex
+    16351232  49%    3.85MB/s    0:00:04
+[2014-05-25 21:44:07 CEST] Pusher: Syncing with diskstation 
+    20119552  60%    3.63MB/s    0:00:03
+    23265280  70%    3.52MB/s    0:00:02
+    25952256  78%    3.31MB/s    0:00:02
+remote: merge git-annex     28803072  86%    2.89MB/s    0:00:01
+(merging synced/git-annex into git-annex...)        
+remote: ok        
+To joern@annex.diskstation.local:/volume1/homes/joern/Annex.movies
+   cc461b8..f90e094  git-annex -> synced/git-annex
+    31653888  95%    2.67MB/s    0:00:00
+    33139501 100%    3.28MB/s    0:00:09 (xfer#1, to-check=0/1)
+
+sent 33143707 bytes  received 42 bytes  2882065.13 bytes/sec
+total size is 33139501  speedup is 1.00
+[2014-05-25 21:44:12 CEST] Transferrer: Uploaded 14_creati..-roof.mp4
+
+WORM-s32543230-m1398386371--Learning SketchUp 2013%13_using-follow-me-complex-roof-pitches.mp4
+       32768   0%    0.00kB/s    0:00:00
+     4718592  14%    4.45MB/s    0:00:06
+[2014-05-25 21:44:14 CEST] Pusher: Syncing with diskstation 
+     7340032  22%    3.45MB/s    0:00:07
+    10125312  31%    3.18MB/s    0:00:06
+    12877824  39%    3.03MB/s    0:00:06
+remote: merge git-annex     15892480  48%    2.64MB/s    0:00:06
+    18579456  57%    2.64MB/s    0:00:05
+    21561344  66%    2.67MB/s    0:00:04
+[2014-05-25 21:44:20 CEST] RemoteControl: Syncing with diskstation 
+(merging synced/git-annex into git-annex...)        
+remote: (Recording state in git...)        
+remote: ok        
+To joern@annex.diskstation.local:/volume1/homes/joern/Annex.movies
+   f90e094..795020d  git-annex -> synced/git-annex
+    24412160  75%    2.70MB/s    0:00:02
+    27164672  83%    2.61MB/s    0:00:02
+From annex.diskstation.local:/volume1/homes/joern/Annex.movies
+   d356ca4..7d763cd  git-annex  -> diskstation/git-annex
+error: Ref refs/remotes/diskstation/synced/git-annex is at 795020d6aa856441f602de8d1f2f7efb98c00f20 but expected f90e0945bc55e8e50115bccec9672d1df7c23e69
+ ! f90e094..795020d  synced/git-annex -> diskstation/synced/git-annex  (unable to update local ref)
+    28737536  88%    2.28MB/s    0:00:01
+    30703616  94%    2.05MB/s    0:00:00
+    32543230 100%    2.58MB/s    0:00:12 (xfer#1, to-check=0/1)
+
+sent 32547377 bytes  received 42 bytes  2410919.93 bytes/sec
+total size is 32543230  speedup is 1.00
+[2014-05-25 21:44:25 CEST] Transferrer: Uploaded 13_using-..tches.mp4
+[2014-05-25 21:44:25 CEST] Pusher: Syncing with diskstation 
+
+WORM-s22461160-m1401045786--Learning SketchUp 2013%12_push-pull-and-move-tool.mp4
+       32768   0%    0.00kB/s    0:00:00
+     4161536  18%    3.80MB/s    0:00:04
+remote: merge git-annex      6782976  30%    3.13MB/s    0:00:04
+     9666560  43%    2.95MB/s    0:00:04
+[2014-05-25 21:44:29 CEST] RemoteControl: Syncing with diskstation 
+(merging synced/git-annex into git-annex...)        
+remote: (Recording state in git...)        
+remote: ok        
+To joern@annex.diskstation.local:/volume1/homes/joern/Annex.movies
+   795020d..496b7bd  git-annex -> synced/git-annex
+    12681216  56%    2.91MB/s    0:00:03
+    15859712  70%    2.70MB/s    0:00:02
+From annex.diskstation.local:/volume1/homes/joern/Annex.movies
+   7d763cd..2bf57cb  git-annex  -> diskstation/git-annex
+error: Ref refs/remotes/diskstation/synced/git-annex is at 496b7bd40f4c979efdd6196eccadeeb370bb4ec3 but expected 795020d6aa856441f602de8d1f2f7efb98c00f20
+ ! 795020d..496b7bd  synced/git-annex -> diskstation/synced/git-annex  (unable to update local ref)
+    18579456  82%    2.72MB/s    0:00:01
+    20414464  90%    2.50MB/s    0:00:00
+    22461160 100%    2.69MB/s    0:00:07 (xfer#1, to-check=0/1)
+
+sent 22464062 bytes  received 42 bytes  2364642.53 bytes/sec
+total size is 22461160  speedup is 1.00
+[2014-05-25 21:44:34 CEST] Transferrer: Uploaded 12_push-p..-tool.mp4
+[2014-05-25 21:44:34 CEST] Pusher: Syncing with diskstation 
+
+WORM-s15320679-m1398386653--Learning SketchUp 2013%11_navigating-in-3d.mp4
+       32768   0%    0.00kB/s    0:00:00
+     4587520  29%    4.18MB/s    0:00:02
+remote: merge git-annex      7471104  48%    3.45MB/s    0:00:02
+    10223616  66%    3.16MB/s    0:00:01
+[2014-05-25 21:44:38 CEST] RemoteControl: Syncing with diskstation 
+(merging synced/git-annex into git-annex...)        
+remote: (Recording state in git...)        
+remote: ok        
+To joern@annex.diskstation.local:/volume1/homes/joern/Annex.movies
+   496b7bd..eefc387  git-annex -> synced/git-annex
+    12976128  84%    3.00MB/s    0:00:00
+    15320679 100%    2.92MB/s    0:00:04 (xfer#1, to-check=0/1)
+From annex.diskstation.local:/volume1/homes/joern/Annex.movies
+   2bf57cb..310d31e  git-annex  -> diskstation/git-annex
+error: Ref refs/remotes/diskstation/synced/git-annex is at eefc387772b3e6ba46f51fbecd86cdabcd27db96 but expected 496b7bd40f4c979efdd6196eccadeeb370bb4ec3
+ ! 496b7bd..eefc387  synced/git-annex -> diskstation/synced/git-annex  (unable to update local ref)
+
+sent 15322702 bytes  received 42 bytes  2357345.23 bytes/sec
+total size is 15320679  speedup is 1.00
+[2014-05-25 21:44:40 CEST] Transferrer: Uploaded 11_naviga..in-3d.mp4
+
+WORM-s6065317-m1398386928--Learning SketchUp 2013%111_about-dan-brown.mp4
+       32768   0%    0.00kB/s    0:00:00
+     4292608  70%    3.98MB/s    0:00:00
+[2014-05-25 21:44:42 CEST] Pusher: Syncing with diskstation 
+     6065317 100%    3.02MB/s    0:00:01 (xfer#1, to-check=0/1)
+
+sent 6066211 bytes  received 42 bytes  1733215.14 bytes/sec
+total size is 6065317  speedup is 1.00
+[2014-05-25 21:44:43 CEST] Transferrer: Uploaded 111_about..brown.mp4
+
+WORM-s2764724-m1398386941--Learning SketchUp 2013%10_freehand-tool.mp4
+       32768   1%    0.00kB/s    0:00:00
+     2764724 100%    5.18MB/s    0:00:00 (xfer#1, to-check=0/1)
+
+sent 2765211 bytes  received 42 bytes  1843502.00 bytes/sec
+total size is 2764724  speedup is 1.00
+[2014-05-25 21:44:45 CEST] Transferrer: Uploaded 10_freeha..-tool.mp4
+remote: merge git-annex 
+WORM-s6834151-m1398385049--Learning SketchUp 2013%109_updating-the-model-references.mp4
+       32768   0%    0.00kB/s    0:00:00
+     4227072  61%    3.81MB/s    0:00:00
+     6834151 100%    3.46MB/s    0:00:01 (xfer#1, to-check=0/1)
+[2014-05-25 21:44:47 CEST] RemoteControl: Syncing with diskstation 
+(merging synced/git-annex into git-annex...)        
+remote: (Recording state in git...)        
+remote: ok        
+To joern@annex.diskstation.local:/volume1/homes/joern/Annex.movies
+   eefc387..dc4c11c  git-annex -> synced/git-annex
+
+sent 6835151 bytes  received 42 bytes  1952912.29 bytes/sec
+total size is 6834151  speedup is 1.00
+[2014-05-25 21:44:48 CEST] Transferrer: Uploaded 109_updat..ences.mp4
+
+WORM-s6878166-m1398386493--Learning SketchUp 2013%108_adding-additional-pages-and-content.mp4
+       32768   0%    0.00kB/s    0:00:00
+     4259840  61%    3.87MB/s    0:00:00
+     6878166 100%    3.80MB/s    0:00:01 (xfer#1, to-check=0/1)
+From annex.diskstation.local:/volume1/homes/joern/Annex.movies
+   310d31e..e2f5d0d  git-annex  -> diskstation/git-annex
+error: Ref refs/remotes/diskstation/synced/git-annex is at dc4c11c618002775bb0e8dcf4b132113c40875ed but expected eefc387772b3e6ba46f51fbecd86cdabcd27db96
+ ! eefc387..dc4c11c  synced/git-annex -> diskstation/synced/git-annex  (unable to update local ref)
+
+sent 6879176 bytes  received 42 bytes  1965490.86 bytes/sec
+total size is 6878166  speedup is 1.00
+[2014-05-25 21:44:51 CEST] Transferrer: Uploaded 108_addin..ntent.mp4
+
+WORM-s8253246-m1398386838--Learning SketchUp 2013%106_adjusting-the-model-settings.mp4
+       32768   0%    0.00kB/s    0:00:00
+[2014-05-25 21:44:52 CEST] Pusher: Syncing with diskstation 
+     2588672  31%    2.32MB/s    0:00:02
+     4554752  55%    2.10MB/s    0:00:01
+     8093696  98%    2.51MB/s    0:00:00
+     8253246 100%    2.52MB/s    0:00:03 (xfer#1, to-check=0/1)
+
+sent 8254417 bytes  received 42 bytes  1834324.22 bytes/sec
+total size is 8253246  speedup is 1.00
+[2014-05-25 21:44:55 CEST] Transferrer: Uploaded 106_adjus..tings.mp4
+
+WORM-s5364896-m1398386068--Learning SketchUp 2013%105_what-is-layout.mp4
+       32768   0%    0.00kB/s    0:00:00
+remote: merge git-annex      4816896  89%    4.56MB/s    0:00:00
+     5364896 100%    4.39MB/s    0:00:01 (xfer#1, to-check=0/1)
+
+sent 5365701 bytes  received 42 bytes  2146297.20 bytes/sec
+total size is 5364896  speedup is 1.00
+[2014-05-25 21:44:58 CEST] RemoteControl: Syncing with diskstation 
+[2014-05-25 21:44:58 CEST] Transferrer: Uploaded 105_what-..ayout.mp4
+(merging synced/git-annex into git-annex...)        
+remote: (Recording state in git...)        
+remote: ok        
+remote: (Recording state in git...)        
+To joern@annex.diskstation.local:/volume1/homes/joern/Annex.movies
+   dc4c11c..a19dfa9  git-annex -> synced/git-annex
+
+WORM-s7390741-m1398386693--Learning SketchUp 2013%104_post-processing-in-photoshop.mp4
+       32768   0%    0.00kB/s    0:00:00
+     4227072  57%    3.91MB/s    0:00:00
+From annex.diskstation.local:/volume1/homes/joern/Annex.movies
+   e2f5d0d..73400d3  git-annex  -> diskstation/git-annex
+error: Ref refs/remotes/diskstation/synced/git-annex is at a19dfa9bd0b4aee28bf67773a9f578a9bb94f2e9 but expected dc4c11c618002775bb0e8dcf4b132113c40875ed
+ ! dc4c11c..a19dfa9  synced/git-annex -> diskstation/synced/git-annex  (unable to update local ref)
+     7241728  97%    3.34MB/s    0:00:00
+     7390741 100%    3.35MB/s    0:00:02 (xfer#1, to-check=0/1)
+
+sent 7391808 bytes  received 42 bytes  1642633.33 bytes/sec
+total size is 7390741  speedup is 1.00
+[2014-05-25 21:45:02 CEST] Transferrer: Uploaded 104_post-..oshop.mp4
+
+WORM-s12544086-m1398385948--Learning SketchUp 2013%103_rendering-a-high-quality-image.mp4
+       32768   0%    0.00kB/s    0:00:00
+[2014-05-25 21:45:03 CEST] Pusher: Syncing with diskstation 
+     4292608  34%    3.96MB/s    0:00:02
+     7307264  58%    3.39MB/s    0:00:01
+     9535488  76%    2.97MB/s    0:00:00
+remote: merge git-annex     12419072  99%    2.91MB/s    0:00:00
+    12544086 100%    2.90MB/s    0:00:04 (xfer#1, to-check=0/1)
+
+sent 12545784 bytes  received 42 bytes  2281059.27 bytes/sec
+total size is 12544086  speedup is 1.00
+[2014-05-25 21:45:07 CEST] RemoteControl: Syncing with diskstation 
+[2014-05-25 21:45:07 CEST] Transferrer: Uploaded 103_rende..image.mp4
+(merging synced/git-annex into git-annex...)        
+remote: (Recording state in git...)        
+remote: ok        
+remote: (Recording state in git...)        
+To joern@annex.diskstation.local:/volume1/homes/joern/Annex.movies
+   a19dfa9..96ceb2c  git-annex -> synced/git-annex
+
+WORM-s6567293-m1398384854--Learning SketchUp 2013%102_model-presets-create-a-preview-rendering.mp4
+       32768   0%    0.00kB/s    0:00:00
+     4128768  62%    3.83MB/s    0:00:00
+     6567293 100%    3.26MB/s    0:00:01 (xfer#1, to-check=0/1)
+From annex.diskstation.local:/volume1/homes/joern/Annex.movies
+   73400d3..3e5d1b8  git-annex  -> diskstation/git-annex
+error: Ref refs/remotes/diskstation/synced/git-annex is at 96ceb2c0e06ff0f1372950ab38c2aa2ebb0f00d7 but expected a19dfa9bd0b4aee28bf67773a9f578a9bb94f2e9
+ ! a19dfa9..96ceb2c  synced/git-annex -> diskstation/synced/git-annex  (unable to update local ref)
+
+sent 6568272 bytes  received 42 bytes  1876661.14 bytes/sec
+total size is 6567293  speedup is 1.00
+[2014-05-25 21:45:11 CEST] Transferrer: Uploaded 102_model..ering.mp4
+
+WORM-s8030582-m1398385367--Learning SketchUp 2013%101_add-reflections-to-existing-textures.mp4
+       32768   0%    0.00kB/s    0:00:00
+     4423680  55%    4.18MB/s    0:00:00
+[2014-05-25 21:45:12 CEST] Pusher: Syncing with diskstation 
+     7569408  94%    3.54MB/s    0:00:00
+     8030582 100%    3.53MB/s    0:00:02 (xfer#1, to-check=0/1)
+
+sent 8031737 bytes  received 42 bytes  2294794.00 bytes/sec
+total size is 8030582  speedup is 1.00
+[2014-05-25 21:45:14 CEST] Transferrer: Uploaded 101_add-r..tures.mp4
+
+WORM-s13509940-m1398384818--Learning SketchUp 2013%100_adding-lighting.mp4
+       32768   0%    0.00kB/s    0:00:00
+     4587520  33%    4.25MB/s    0:00:02
+remote: merge git-annex      7831552  57%    3.62MB/s    0:00:01
+    10420224  77%    3.24MB/s    0:00:00
+[2014-05-25 21:45:18 CEST] RemoteControl: Syncing with diskstation 
+(merging synced/git-annex into git-annex...)        
+remote: (Recording state in git...)        
+remote: ok        
+To joern@annex.diskstation.local:/volume1/homes/joern/Annex.movies
+   96ceb2c..a5f9c8b  git-annex -> synced/git-annex
+    13509940 100%    3.28MB/s    0:00:03 (xfer#1, to-check=0/1)
+
+sent 13511743 bytes  received 42 bytes  2456688.18 bytes/sec
+total size is 13509940  speedup is 1.00
+[2014-05-25 21:45:19 CEST] Transferrer: Uploaded 100_addin..hting.mp4
+From annex.diskstation.local:/volume1/homes/joern/Annex.movies
+   3e5d1b8..6ea05e0  git-annex  -> diskstation/git-annex
+error: Ref refs/remotes/diskstation/synced/git-annex is at a5f9c8b9874d4d3fe571687365c391050325bc87 but expected 96ceb2c0e06ff0f1372950ab38c2aa2ebb0f00d7
+ ! 96ceb2c..a5f9c8b  synced/git-annex -> diskstation/synced/git-annex  (unable to update local ref)
+
+WORM-s6047748-m1398386548--Learning SketchUp 2013%09_arc-tool.mp4
+       32768   0%    0.00kB/s    0:00:00
+     4751360  78%    4.47MB/s    0:00:00
+     6047748 100%    4.50MB/s    0:00:01 (xfer#1, to-check=0/1)
+
+sent 6048630 bytes  received 42 bytes  2419468.80 bytes/sec
+total size is 6047748  speedup is 1.00
+[2014-05-25 21:45:22 CEST] Pusher: Syncing with diskstation 
+[2014-05-25 21:45:22 CEST] Transferrer: Uploaded 09_arc-tool.mp4
+
+WORM-s7257160-m1398385203--Learning SketchUp 2013%08_polygon-tool.mp4
+       32768   0%    0.00kB/s    0:00:00
+     5079040  69%    4.65MB/s    0:00:00
+remote: merge git-annex      7257160 100%    4.35MB/s    0:00:01 (xfer#1, to-check=0/1)
+[2014-05-25 21:45:25 CEST] RemoteControl: Syncing with diskstation 
+
+sent 7258194 bytes  received 42 bytes  2073781.71 bytes/sec
+total size is 7257160  speedup is 1.00
+[2014-05-25 21:45:25 CEST] Transferrer: Uploaded 08_polygon-tool.mp4
+
+WORM-s6789048-m1398386463--Learning SketchUp 2013%07_circle-tool.mp4
+       32768   0%    0.00kB/s    0:00:00
+(merging synced/git-annex into git-annex...)        
+remote: (Recording state in git...)        
+remote: ok        
+remote: (Recording state in git...)        
+To joern@annex.diskstation.local:/volume1/homes/joern/Annex.movies
+   a5f9c8b..05787b8  git-annex -> synced/git-annex
+     2654208  39%    2.46MB/s    0:00:01
+From annex.diskstation.local:/volume1/homes/joern/Annex.movies
+   6ea05e0..b78228d  git-annex  -> diskstation/git-annex
+error: Ref refs/remotes/diskstation/synced/git-annex is at 05787b8b7c6dd1aad45fdb0a019a2ed5e56b26bb but expected a5f9c8b9874d4d3fe571687365c391050325bc87
+ ! a5f9c8b..05787b8  synced/git-annex -> diskstation/synced/git-annex  (unable to update local ref)
+[2014-05-25 21:45:28 CEST] RemoteControl: Syncing with diskstation 
+     5013504  73%    2.35MB/s    0:00:00
+     6789048 100%    2.62MB/s    0:00:02 (xfer#1, to-check=0/1)
+
+sent 6790025 bytes  received 42 bytes  1940019.14 bytes/sec
+total size is 6789048  speedup is 1.00
+[2014-05-25 21:45:29 CEST] Transferrer: Uploaded 07_circle-tool.mp4
+From annex.diskstation.local:/volume1/homes/joern/Annex.movies
+   b78228d..f1fd60c  git-annex  -> diskstation/git-annex
+
+WORM-s3766669-m1398384958--Learning SketchUp 2013%06_rectangle-tool.mp4
+       32768   0%    0.00kB/s    0:00:00
+[2014-05-25 21:45:31 CEST] Pusher: Syncing with diskstation 
+     3702784  98%    2.95MB/s    0:00:00
+     3766669 100%    3.00MB/s    0:00:01 (xfer#1, to-check=0/1)
+
+sent 3767277 bytes  received 42 bytes  1506927.60 bytes/sec
+total size is 3766669  speedup is 1.00
+[2014-05-25 21:45:31 CEST] Transferrer: Uploaded 06_rectan..-tool.mp4
+
+WORM-s5571735-m1398386867--Learning SketchUp 2013%05_line-tool.mp4
+       32768   0%    0.00kB/s    0:00:00
+     5144576  92%    4.78MB/s    0:00:00
+     5571735 100%    4.70MB/s    0:00:01 (xfer#1, to-check=0/1)
+remote: merge git-annex 
+sent 5572562 bytes  received 42 bytes  2229041.60 bytes/sec
+total size is 5571735  speedup is 1.00
+[2014-05-25 21:45:33 CEST] Transferrer: Uploaded 05_line-tool.mp4
+(merging synced/git-annex into git-annex...)        
+remote: (Recording state in git...)        
+remote: ok        
+[2014-05-25 21:45:35 CEST] RemoteConTo joern@annex.diskstation.local:/volume1/homes/joern/Annex.movies
+trol: Syncing with diskstati   05787b8..2a723a6  ogit-annex -> synced/git-annexn
+ 
+
+WORM-s13796050-m1398386633--Learning SketchUp 2013%04_how-to-access-your-working-files.mp4
+       32768   0%    0.00kB/s    0:00:00
+     4915200  35%    4.65MB/s    0:00:01
+     8159232  59%    3.77MB/s    0:00:01
+From annex.diskstation.local:/volume1/homes/joern/Annex.movies
+   f1fd60c..bdd7ed6  git-annex  -> diskstation/git-annex
+    11304960  81%    3.45MB/s    0:00:00
+    13796050 100%    3.57MB/s    0:00:03 (xfer#1, to-check=0/1)
+[2014-05-25 21:45:38 CEST] Pusher: Syncing with diskstation 
+
+sent 13797905 bytes  received 42 bytes  2508717.64 bytes/sec
+total size is 13796050  speedup is 1.00
+[2014-05-25 21:45:39 CEST] Transferrer: Uploaded 04_how-to..files.mp4
+
+WORM-s5852944-m1398386582--Learning SketchUp 2013%03_setting-up-the-toolbars.mp4
+       32768   0%    0.00kB/s    0:00:00
+     4947968  84%    4.65MB/s    0:00:00
+     5852944 100%    4.42MB/s    0:00:01 (xfer#1, to-check=0/1)
+remote: merge git-annex [2014-05-25 21:45:42 CEST] RemoteControl: Syncing with diskstation 
+
+sent 5853817 bytes  received 42 bytes  2341543.60 bytes/sec
+total size is 5852944  speedup is 1.00
+[2014-05-25 21:45:42 CEST] Transferrer: Uploaded 03_settin..lbars.mp4
+(merging synced/git-annex into git-annex...)        
+remote: (Recording state in git...)        
+remote: ok        
+remote: (Recording state in git...)        
+To joern@annex.diskstation.local:/volume1/homes/joern/Annex.movies
+   2a723a6..dc76c23  git-annex -> synced/git-annex
+
+WORM-s3829366-m1398386198--Learning SketchUp 2013%02_setting-up-the-template.mp4
+       32768   0%    0.00kB/s    0:00:00
+     3829366 100%    4.68MB/s    0:00:00 (xfer#1, to-check=0/1)
+From annex.diskstation.local:/volume1/homes/joern/Annex.movies
+   bdd7ed6..84f8e25  git-annex  -> diskstation/git-annex
+error: Ref refs/remotes/diskstation/synced/git-annex is at dc76c234409bd5cfac50be70dd21ef8be88aed9b but expected 2a723a6697455228d8b2e95a7015aafb4be4f003
+ ! 2a723a6..dc76c23  synced/git-annex -> diskstation/synced/git-annex  (unable to update local ref)
+
+sent 3829991 bytes  received 42 bytes  1532013.20 bytes/sec
+total size is 3829366  speedup is 1.00
+[2014-05-25 21:45:45 CEST] Transferrer: Uploaded 02_settin..plate.mp4
+
+WORM-s6528377-m1398386753--Learning SketchUp 2013%01_introduction.mp4
+       32768   0%    0.00kB/s    0:00:00
+     4718592  72%    4.42MB/s    0:00:00
+[2014-05-25 21:45:47 CEST] Pusher: Syncing with diskstation 
+     6528377 100%    3.21MB/s    0:00:01 (xfer#1, to-check=0/1)
+
+sent 6529323 bytes  received 42 bytes  2611746.00 bytes/sec
+total size is 6528377  speedup is 1.00
+[2014-05-25 21:45:48 CEST] Transferrer: Uploaded 01_introduction.mp4
+
+WORM-s5900041-m1398386278--Learning SketchUp 2013%99_saving-the-image.mp4
+       32768   0%    0.00kB/s    0:00:00
+     5079040  86%    4.72MB/s    0:00:00
+     5900041 100%    4.60MB/s    0:00:01 (xfer#1, to-check=0/1)
+remote: merge git-annex 
+sent 5900915 bytes  received 42 bytes  2360382.80 bytes/sec
+total size is 5900041  speedup is 1.00
+[2014-05-25 21:45:50 CEST] Transferrer: Uploaded 99_saving..image.mp4
+
+WORM-s7221607-m1398385119--Learning SketchUp 2013%98_multiply-and-opacity.mp4
+       32768   0%    0.00kB/s    0:00:00
+[2014-05-25 21:45:52 CEST] RemoteControl: Syncing with diskstation 
+(merging synced/git-annex into git-annex...)        
+remote: (Recording state in git...)        
+remote: ok        
+To joern@annex.diskstation.local:/volume1/homes/joern/Annex.movies
+   dc76c23..60c1e82  git-annex -> synced/git-annex
+     4456448  61%    4.07MB/s    0:00:00
+     7221607 100%    4.25MB/s    0:00:01 (xfer#1, to-check=0/1)
+
+sent 7222645 bytes  received 42 bytes  2889074.80 bytes/sec
+total size is 7221607  speedup is 1.00
+[2014-05-25 21:45:54 CEST] Transferrer: Uploaded 98_multip..acity.mp4
+From annex.diskstation.local:/volume1/homes/joern/Annex.movies
+   84f8e25..3261510  git-annex  -> diskstation/git-annex
+error: Ref refs/remotes/diskstation/synced/git-annex is at 60c1e827e829b642d4618889b425c0bfc259b2c9 but expected dc76c234409bd5cfac50be70dd21ef8be88aed9b
+ ! dc76c23..60c1e82  synced/git-annex -> diskstation/synced/git-annex  (unable to update local ref)
+
+WORM-s6262129-m1398386258--Learning SketchUp 2013%97_stacking-images-in-photoshop.mp4
+       32768   0%    0.00kB/s    0:00:00
+     4947968  79%    4.69MB/s    0:00:00
+     6262129 100%    4.40MB/s    0:00:01 (xfer#1, to-check=0/1)
+[2014-05-25 21:45:56 CEST] Pusher: Syncing with diskstation 
+
+sent 6263059 bytes  received 42 bytes  2505240.40 bytes/sec
+total size is 6262129  speedup is 1.00
+[2014-05-25 21:45:56 CEST] Transferrer: Uploaded 97_stacki..oshop.mp4
+
+WORM-s9298049-m1398385793--Learning SketchUp 2013%96_export-display-options.mp4
+       32768   0%    0.00kB/s    0:00:00
+     5177344  55%    4.84MB/s    0:00:00
+remote: merge git-annex      8519680  91%    4.01MB/s    0:00:00
+     9298049 100%    3.97MB/s    0:00:02 (xfer#1, to-check=0/1)
+[2014-05-25 21:46:00 CEST] RemoteControl: Syncing with diskstation 
+
+sent 9299341 bytes  received 42 bytes  2656966.57 bytes/sec
+total size is 9298049  speedup is 1.00
+[2014-05-25 21:46:00 CEST] Transferrer: Uploaded 96_export..tions.mp4
+
+WORM-s7895136-m1398385533--Learning SketchUp 2013%95_add-final-touches.mp4
+       32768   0%    0.00kB/s    0:00:00
+(merging synced/git-annex into git-annex...)        
+remote: (Recording state in git...)        
+remote: ok        
+remote: (Recording state in git...)        
+To joern@annex.diskstation.local:/volume1/homes/joern/Annex.movies
+   60c1e82..f08968c  git-annex -> synced/git-annex
+     4325376  54%    4.06MB/s    0:00:00
+From annex.diskstation.local:/volume1/homes/joern/Annex.movies
+   3261510..c9b4dc3  git-annex  -> diskstation/git-annex
+error: Ref refs/remotes/diskstation/synced/git-annex is at f08968c15cbe86c49c1fb71f22f578ad7cecd680 but expected 60c1e827e829b642d4618889b425c0bfc259b2c9
+ ! 60c1e82..f08968c  synced/git-annex -> diskstation/synced/git-annex  (unable to update local ref)
+[2014-05-25 21:46:02 CEST] RemoteControl: Syncing with diskstation 
+     7895136 100%    3.99MB/s    0:00:01 (xfer#1, to-check=0/1)
+
+sent 7896251 bytes  received 42 bytes  2256083.71 bytes/sec
+total size is 7895136  speedup is 1.00
+[2014-05-25 21:46:03 CEST] Transferrer: Uploaded 95_add-fi..uches.mp4
+From annex.diskstation.local:/volume1/homes/joern/Annex.movies
+   c9b4dc3..78519cd  git-annex  -> diskstation/git-annex
+
+WORM-s13569035-m1398385318--Learning SketchUp 2013%94_animation-options.mp4
+       32768   0%    0.00kB/s    0:00:00
+     5373952  39%    5.00MB/s    0:00:01
+[2014-05-25 21:46:05 CEST] Pusher: Syncing with diskstation 
+     9437184  69%    4.45MB/s    0:00:00
+    13565952  99%    4.26MB/s    0:00:00
+    13569035 100%    4.26MB/s    0:00:03 (xfer#1, to-check=0/1)
+
+sent 13570847 bytes  received 42 bytes  3015753.11 bytes/sec
+total size is 13569035  speedup is 1.00
+[2014-05-25 21:46:07 CEST] Transferrer: Uploaded 94_animat..tions.mp4
+
+WORM-s8278464-m1398386213--Learning SketchUp 2013%93_collada-files.mp4
+       32768   0%    0.00kB/s    0:00:00
+remote: merge git-annex      5341184  64%    4.98MB/s    0:00:00
+     8278464 100%    4.37MB/s    0:00:01 (xfer#1, to-check=0/1)
+[2014-05-25 21:46:10 CEST] RemoteControl: Syncing with diskstation 
+
+sent 8279623 bytes  received 42 bytes  2365618.57 bytes/sec
+total size is 8278464  speedup is 1.00
+[2014-05-25 21:46:10 CEST] Transferrer: Uploaded 93_collad..files.mp4
+(merging synced/git-annex into git-annex...)        
+remote: (Recording state in git...)        
+remote: ok        
+remote: (Recording state in git...)        
+To joern@annex.diskstation.local:/volume1/homes/joern/Annex.movies
+   f08968c..f99a466  git-annex -> synced/git-annex
+
+WORM-s8934010-m1398386414--Learning SketchUp 2013%92_kmz-for-google-earth.mp4
+       32768   0%    0.00kB/s    0:00:00
+     4784128  53%    4.43MB/s    0:00:00
+From annex.diskstation.local:/volume1/homes/joern/Annex.movies
+   78519cd..e30d06f  git-annex  -> diskstation/git-annex
+error: Ref refs/remotes/diskstation/synced/git-annex is at f99a466d7568addbbc99cb058f66a2c03c571fba but expected f08968c15cbe86c49c1fb71f22f578ad7cecd680
+ ! f08968c..f99a466  synced/git-annex -> diskstation/synced/git-annex  (unable to update local ref)
+     8934010 100%    4.24MB/s    0:00:02 (xfer#1, to-check=0/1)
+
+sent 8935256 bytes  received 42 bytes  2552942.29 bytes/sec
+total size is 8934010  speedup is 1.00
+[2014-05-25 21:46:13 CEST] Transferrer: Uploaded 92_kmz-fo..earth.mp4
+
+WORM-s9716042-m1398384915--Learning SketchUp 2013%91_dwg-elevation-and-plan-pro-feature.mp4
+       32768   0%    0.00kB/s    0:00:00
+     5341184  54%    5.00MB/s    0:00:00
+[2014-05-25 21:46:15 CEST] Pusher: Syncing with diskstation 
+     9371648  96%    4.39MB/s    0:00:00
+     9716042 100%    4.44MB/s    0:00:02 (xfer#1, to-check=0/1)
+
+sent 9717398 bytes  received 42 bytes  2776411.43 bytes/sec
+total size is 9716042  speedup is 1.00
+[2014-05-25 21:46:16 CEST] Transferrer: Uploaded 91_dwg-el..ature.mp4
+
+WORM-s14343030-m1398386888--Learning SketchUp 2013%90_jpeg-and-png-files.mp4
+       32768   0%    0.00kB/s    0:00:00
+remote: merge git-annex      5079040  35%    4.81MB/s    0:00:01
+     9535488  66%    4.50MB/s    0:00:01
+[2014-05-25 21:46:19 CEST] RemoteControl: Syncing with diskstation 
+(merging synced/git-annex into git-annex...)        
+remote: (Recording state in git...)        
+remote: ok        
+To joern@annex.diskstation.local:/volume1/homes/joern/Annex.movies
+   f99a466..af84b87  git-annex -> synced/git-annex
+    13565952  94%    4.25MB/s    0:00:00
+    14343030 100%    3.99MB/s    0:00:03 (xfer#1, to-check=0/1)
+
+sent 14344935 bytes  received 42 bytes  2608177.64 bytes/sec
+total size is 14343030  speedup is 1.00
+[2014-05-25 21:46:21 CEST] Transferrer: Uploaded 90_jpeg-a..files.mp4
+From annex.diskstation.local:/volume1/homes/joern/Annex.movies
+   e30d06f..c6bcde6  git-annex  -> diskstation/git-annex
+error: Ref refs/remotes/diskstation/synced/git-annex is at af84b876d0902aa7da89ca414becf5b9d98d8a19 but expected f99a466d7568addbbc99cb058f66a2c03c571fba
+ ! f99a466..af84b87  synced/git-annex -> diskstation/synced/git-annex  (unable to update local ref)
+
+WORM-s10692771-m1398385558--Learning SketchUp 2013%89_creating-the-animation.mp4
+       32768   0%    0.00kB/s    0:00:00
+     5341184  49%    4.94MB/s    0:00:01
+[2014-05-25 21:46:23 CEST] Pusher: Syncing with diskstation 
+     9535488  89%    4.43MB/s    0:00:00
+    10692771 100%    4.41MB/s    0:00:02 (xfer#1, to-check=0/1)
+
+sent 10694236 bytes  received 42 bytes  3055508.00 bytes/sec
+total size is 10692771  speedup is 1.00
+[2014-05-25 21:46:24 CEST] Transferrer: Uploaded 89_creati..ation.mp4
+
+WORM-s14320839-m1398385514--Learning SketchUp 2013%88_creating-a-group.mp4
+       32768   0%    0.00kB/s    0:00:00
+     5308416  37%    5.02MB/s    0:00:01
+remote: merge git-annex      9469952  66%    4.47MB/s    0:00:01
+    13631488  95%    4.28MB/s    0:00:00
+    14320839 100%    4.31MB/s    0:00:03 (xfer#1, to-check=0/1)
+
+sent 14322742 bytes  received 42 bytes  3182840.89 bytes/sec
+total size is 14320839  speedup is 1.00
+[2014-05-25 21:46:28 CEST] RemoteControl: Syncing with diskstation 
+[2014-05-25 21:46:28 CEST] Transferrer: Uploaded 88_creati..group.mp4
+(merging synced/git-annex into git-annex...)        
+remote: (Recording state in git...)        
+remote: ok        
+remote: (Recording state in git...)        
+To joern@annex.diskstation.local:/volume1/homes/joern/Annex.movies
+   af84b87..7f1137a  git-annex -> synced/git-annex
+
+WORM-s20730987-m1398385414--Learning SketchUp 2013%87_section-cut-for-interior-elevations.mp4
+       32768   0%    0.00kB/s    0:00:00
+     5341184  25%    4.91MB/s    0:00:03
+From annex.diskstation.local:/volume1/homes/joern/Annex.movies
+   c6bcde6..a8341a2  git-annex  -> diskstation/git-annex
+error: Ref refs/remotes/diskstation/synced/git-annex is at 7f1137a28d8b17c5cdc10e856578da6337d1143c but expected af84b876d0902aa7da89ca414becf5b9d98d8a19
+ ! af84b87..7f1137a  synced/git-annex -> diskstation/synced/git-annex  (unable to update local ref)
+     9371648  45%    4.38MB/s    0:00:02
+    13860864  66%    4.33MB/s    0:00:01
+[2014-05-25 21:46:33 CEST] Pusher: Syncing with diskstation 
+    18055168  87%    4.23MB/s    0:00:00
+    20730987 100%    4.19MB/s    0:00:04 (xfer#1, to-check=0/1)
+
+sent 20733689 bytes  received 42 bytes  3189804.77 bytes/sec
+total size is 20730987  speedup is 1.00
+[2014-05-25 21:46:35 CEST] Transferrer: Uploaded 87_sectio..tions.mp4
+
+WORM-s21661288-m1398385144--Learning SketchUp 2013%86_scenes-for-animation.mp4
+       32768   0%    0.00kB/s    0:00:00
+remote: merge git-annex      5177344  23%    4.89MB/s    0:00:03
+     9404416  43%    4.46MB/s    0:00:02
+[2014-05-25 21:46:38 CEST] RemoteControl: Syncing with diskstation 
+    13631488  62%    4.28MB/s    0:00:01
+(merging synced/git-annex into git-annex...)        
+remote: (Recording state in git...)        
+remote: ok        
+To joern@annex.diskstation.local:/volume1/homes/joern/Annex.movies
+   7f1137a..9338de4  git-annex -> synced/git-annex
+    17170432  79%    4.04MB/s    0:00:01
+    20905984  96%    3.69MB/s    0:00:00
+    21661288 100%    3.90MB/s    0:00:05 (xfer#1, to-check=0/1)
+From annex.diskstation.local:/volume1/homes/joern/Annex.movies
+   a8341a2..72c4e65  git-annex  -> diskstation/git-annex
+error: Ref refs/remotes/diskstation/synced/git-annex is at 9338de4e5a82415622d40ef588287666f8757fc0 but expected 7f1137a28d8b17c5cdc10e856578da6337d1143c
+ ! 7f1137a..9338de4  synced/git-annex -> diskstation/synced/git-annex  (unable to update local ref)
+
+sent 21664091 bytes  received 42 bytes  3332943.54 bytes/sec
+total size is 21661288  speedup is 1.00
+[2014-05-25 21:46:41 CEST] Transferrer: Uploaded 86_scenes..ation.mp4
+
+WORM-s23994784-m1398385930--Learning SketchUp 2013%85_styles.mp4
+       32768   0%    0.00kB/s    0:00:00
+     5177344  21%    4.83MB/s    0:00:03
+[2014-05-25 21:46:42 CEST] Pusher: Syncing with diskstation 
+     9371648  39%    4.39MB/s    0:00:03
+    13565952  56%    4.26MB/s    0:00:02
+    17661952  73%    4.17MB/s    0:00:01
+remote: merge git-annex     21856256  91%    3.94MB/s    0:00:00
+    23994784 100%    4.08MB/s    0:00:05 (xfer#1, to-check=0/1)
+
+sent 23997857 bytes  received 42 bytes  3691984.46 bytes/sec
+total size is 23994784  speedup is 1.00
+[2014-05-25 21:46:48 CEST] RemoteControl: Syncing with diskstation 
+[2014-05-25 21:46:48 CEST] Transferrer: Uploaded 85_styles.mp4
+(merging synced/git-annex into git-annex...)        
+remote: (Recording state in git...)        
+remote: ok        
+remote: (Recording state in git...)        
+To joern@annex.diskstation.local:/volume1/homes/joern/Annex.movies
+   9338de4..47c3025  git-annex -> synced/git-annex
+
+WORM-s9421737-m1398386334--Learning SketchUp 2013%84_creating-scenes-and-updating.mp4
+       32768   0%    0.00kB/s    0:00:00
+     4128768  43%    3.80MB/s    0:00:01
+From annex.diskstation.local:/volume1/homes/joern/Annex.movies
+   72c4e65..3862dfd  git-annex  -> diskstation/git-annex
+error: Ref refs/remotes/diskstation/synced/git-annex is at 47c3025b26597ab96c6553b17d40df2473761367 but expected 9338de4e5a82415622d40ef588287666f8757fc0
+ ! 9338de4..47c3025  synced/git-annex -> diskstation/synced/git-annex  (unable to update local ref)
+     7045120  74%    3.29MB/s    0:00:00
+     9421737 100%    3.42MB/s    0:00:02 (xfer#1, to-check=0/1)
+
+sent 9423051 bytes  received 42 bytes  2094020.67 bytes/sec
+total size is 9421737  speedup is 1.00
+[2014-05-25 21:46:52 CEST] Transferrer: Uploaded 84_creati..ating.mp4
+
+WORM-s15890796-m1398387058--Learning SketchUp 2013%83_add-solar-panels-with-skelion-plugin.mp4
+       32768   0%    0.00kB/s    0:00:00
+     3932160  24%    3.64MB/s    0:00:03
+[2014-05-25 21:46:54 CEST] Pusher: Syncing with diskstation 
+     8093696  50%    3.80MB/s    0:00:02
+    11304960  71%    3.54MB/s    0:00:01
+    14516224  91%    3.40MB/s    0:00:00
+    15890796 100%    3.45MB/s    0:00:04 (xfer#1, to-check=0/1)
+
+sent 15892907 bytes  received 42 bytes  2889627.09 bytes/sec
+total size is 15890796  speedup is 1.00
+[2014-05-25 21:46:58 CEST] Transferrer: Uploaded 83_add-so..lugin.mp4
+remote: merge git-annex 
+WORM-s9278449-m1398386852--Learning SketchUp 2013%82_using-3d-text.mp4
+       32768   0%    0.00kB/s    0:00:00
+     5439488  58%    5.07MB/s    0:00:00
+     9278449 100%    4.51MB/s    0:00:01 (xfer#1, to-check=0/1)
+[2014-05-25 21:47:00 CEST] RemoteControl: Syncing with diskstation 
+
+sent 9279732 bytes  received 42 bytes  3711909.60 bytes/sec
+total size is 9278449  speedup is 1.00
+[2014-05-25 21:47:00 CEST] Transferrer: Uploaded 82_using-..-text.mp4
+
+WORM-s11983231-m1398385259--Learning SketchUp 2013%81_get-photo-google-street-view.mp4
+       32768   0%    0.00kB/s    0:00:00
+(merging synced/git-annex into git-annex...)        
+remote: (Recording state in git...)        
+remote: ok        
+remote: (Recording state in git...)        
+To joern@annex.diskstation.local:/volume1/homes/joern/Annex.movies
+   47c3025..a7b0dc9  git-annex -> synced/git-annex
+     5079040  42%    4.63MB/s    0:00:01
+From annex.diskstation.local:/volume1/homes/joern/Annex.movies
+   3862dfd..5f94bb2  git-annex  -> diskstation/git-annex
+error: Ref refs/remotes/diskstation/synced/git-annex is at a7b0dc9c64973369eb06a57d14ed6e64ec20fa07 but expected 47c3025b26597ab96c6553b17d40df2473761367
+ ! 47c3025..a7b0dc9  synced/git-annex -> diskstation/synced/git-annex  (unable to update local ref)
+[2014-05-25 21:47:03 CEST] RemoteControl: Syncing with diskstation 
+     9404416  78%    4.32MB/s    0:00:00
+    11983231 100%    4.36MB/s    0:00:02 (xfer#1, to-check=0/1)
+
+sent 11984858 bytes  received 42 bytes  2663311.11 bytes/sec
+total size is 11983231  speedup is 1.00
+[2014-05-25 21:47:04 CEST] Transferrer: Uploaded 81_get-ph..-view.mp4
+From annex.diskstation.local:/volume1/homes/joern/Annex.movies
+   5f94bb2..521740c  git-annex  -> diskstation/git-annex
+
+WORM-s10154793-m1398385298--Learning SketchUp 2013%80_adding-context.mp4
+       32768   0%    0.00kB/s    0:00:00
+     5242880  51%    4.94MB/s    0:00:00
+[2014-05-25 21:47:06 CEST] Pusher: Syncing with diskstation 
+     9437184  92%    4.40MB/s    0:00:00
+    10154793 100%    4.41MB/s    0:00:02 (xfer#1, to-check=0/1)
+
+sent 10156182 bytes  received 42 bytes  2901778.29 bytes/sec
+total size is 10154793  speedup is 1.00
+[2014-05-25 21:47:07 CEST] Transferrer: Uploaded 80_adding..ntext.mp4
+
+WORM-s15455988-m1398386244--Learning SketchUp 2013%79_creating-the-front-facade.mp4
+       32768   0%    0.00kB/s    0:00:00
+     5308416  34%    4.93MB/s    0:00:02
+remote: merge git-annex      9306112  60%    4.38MB/s    0:00:01
+    12124160  78%    3.73MB/s    0:00:00
+[2014-05-25 21:47:12 CEST] RemoteControl: Syncing with diskstation 
+(merging synced/git-annex into git-annex...)        
+remote: (Recording state in git...)        
+remote: ok        
+To joern@annex.diskstation.local:/volume1/homes/joern/Annex.movies
+   a7b0dc9..7a6f85f  git-annex -> synced/git-annex
+    13434880  86%    3.11MB/s    0:00:00
+    15455988 100%    3.14MB/s    0:00:04 (xfer#1, to-check=0/1)
+
+sent 15458036 bytes  received 42 bytes  2810559.64 bytes/sec
+total size is 15455988  speedup is 1.00
+[2014-05-25 21:47:13 CEST] Transferrer: Uploaded 79_creati..acade.mp4
+
+WORM-s7511926-m1398385903--Learning SketchUp 2013%78_reset-axis-to-align-building.mp4
+       32768   0%    0.00kB/s    0:00:00
+From annex.diskstation.local:/volume1/homes/joern/Annex.movies
+   521740c..57437fb  git-annex  -> diskstation/git-annex
+error: Ref refs/remotes/diskstation/synced/git-annex is at 7a6f85f7864920fbcac361f47558fc352f1d2d16 but expected a7b0dc9c64973369eb06a57d14ed6e64ec20fa07
+ ! a7b0dc9..7a6f85f  synced/git-annex -> diskstation/synced/git-annex  (unable to update local ref)
+     4554752  60%    4.27MB/s    0:00:00
+     7511926 100%    4.13MB/s    0:00:01 (xfer#1, to-check=0/1)
+[2014-05-25 21:47:16 CEST] Pusher: Syncing with diskstation 
+
+sent 7513008 bytes  received 42 bytes  2146585.71 bytes/sec
+total size is 7511926  speedup is 1.00
+[2014-05-25 21:47:16 CEST] Transferrer: Uploaded 78_reset-..lding.mp4
+
+WORM-s9197360-m1398386099--Learning SketchUp 2013%77_setting-the-building-in-place-geolocation.mp4
+       32768   0%    0.00kB/s    0:00:00
+     4521984  49%    4.04MB/s    0:00:01
+remote: merge git-annex      7962624  86%    3.66MB/s    0:00:00
+     9197360 100%    3.73MB/s    0:00:02 (xfer#1, to-check=0/1)
+[2014-05-25 21:47:21 CEST] RemoteControl: Syncing with diskstation 
+
+sent 9198659 bytes  received 42 bytes  2628200.29 bytes/sec
+total size is 9197360  speedup is 1.00
+[2014-05-25 21:47:21 CEST] Transferrer: Uploaded 77_settin..ation.mp4
+(merging synced/git-annex into git-annex...)        
+remote: (Recording state in git...)        
+remote: ok        
+remote: (Recording state in git...)        
+To joern@annex.diskstation.local:/volume1/homes/joern/Annex.movies
+   7a6f85f..7457693  git-annex -> synced/git-annex
+
+WORM-s11971891-m1398387083--Learning SketchUp 2013%76_creating-the-exterior.mp4
+       32768   0%    0.00kB/s    0:00:00
+     4784128  39%    4.52MB/s    0:00:01
+     8355840  69%    3.95MB/s    0:00:00
+From annex.diskstation.local:/volume1/homes/joern/Annex.movies
+   57437fb..394b254  git-annex  -> diskstation/git-annex
+error: Ref refs/remotes/diskstation/synced/git-annex is at 7457693b085f06be2a38cfaafb7c76b34eb62960 but expected 7a6f85f7864920fbcac361f47558fc352f1d2d16
+ ! 7a6f85f..7457693  synced/git-annex -> diskstation/synced/git-annex  (unable to update local ref)
+    11971891 100%    4.15MB/s    0:00:02 (xfer#1, to-check=0/1)
+
+sent 11973511 bytes  received 42 bytes  3421015.14 bytes/sec
+total size is 11971891  speedup is 1.00
+[2014-05-25 21:47:25 CEST] Transferrer: Uploaded 76_creati..erior.mp4
+[2014-05-25 21:47:26 CEST] Pusher: Syncing with diskstation 
+
+WORM-s6407146-m1398386568--Learning SketchUp 2013%75_saving-and-placing-into-the-model.mp4
+       32768   0%    0.00kB/s    0:00:00
+     4390912  68%    4.14MB/s    0:00:00
+     6324224  98%    2.99MB/s    0:00:00
+     6407146 100%    3.02MB/s    0:00:02 (xfer#1, to-check=0/1)
+
+sent 6408097 bytes  received 42 bytes  1830896.86 bytes/sec
+total size is 6407146  speedup is 1.00
+[2014-05-25 21:47:28 CEST] Transferrer: Uploaded 75_saving..model.mp4
+
+WORM-s8962345-m1398386088--Learning SketchUp 2013%74_adding-detail-chapter-13.mp4
+       32768   0%    0.00kB/s    0:00:00
+remote: merge git-annex      5144576  57%    4.78MB/s    0:00:00
+     8962345 100%    4.67MB/s    0:00:01 (xfer#1, to-check=0/1)
+(merging synced/git-annex into git-annex...)        
+remote: (Recording state in git...)        
+remote: ok        
+To joern@annex.diskstation.local:/volume1/homes/joern/Annex.movies
+
+sent 8963599 bytes  received 42 bytes  2561040.29 bytes/sec
+total size is 8962345  speedup is 1.00
+   7457693..ea4b518  git-annex -> synced/git-annex
+[2014-05-25 21:47:31 CEST] RemoteControl: Syncing with diskstation 
+[2014-05-25 21:47:31 CEST] Transferrer: Uploaded 74_adding..er-13.mp4
+
+WORM-s7160845-m1398385888--Learning SketchUp 2013%73_applying-textures.mp4
+       32768   0%    0.00kB/s    0:00:00
+     4816896  67%    4.45MB/s    0:00:00
+From annex.diskstation.local:/volume1/homes/joern/Annex.movies
+   394b254..c52b780  git-annex  -> diskstation/git-annex
+     7160845 100%    4.44MB/s    0:00:01 (xfer#1, to-check=0/1)
+
+sent 7161872 bytes  received 42 bytes  2046261.14 bytes/sec
+total size is 7160845  speedup is 1.00
+[2014-05-25 21:47:34 CEST] Transferrer: Uploaded 73_applyi..tures.mp4
+
+WORM-s11661338-m1398385993--Learning SketchUp 2013%72_creating-the-massing-model.mp4
+       32768   0%    0.00kB/s    0:00:00
+[2014-05-25 21:47:35 CEST] Pusher: Syncing with diskstation 
+     5341184  45%    4.97MB/s    0:00:01
+     8028160  68%    3.71MB/s    0:00:00
+     9601024  82%    2.97MB/s    0:00:00
+    11661338 100%    3.00MB/s    0:00:03 (xfer#1, to-check=0/1)
+
+sent 11662923 bytes  received 42 bytes  2120539.09 bytes/sec
+total size is 11661338  speedup is 1.00
+[2014-05-25 21:47:39 CEST] Transferrer: Uploaded 72_creati..model.mp4
+
+WORM-s18351413-m1398385439--Learning SketchUp 2013%71_setting-up-to-match-a-new-photo.mp4
+       32768   0%    0.00kB/s    0:00:00
+remote: merge git-annex      4816896  26%    4.56MB/s    0:00:02
+     8945664  48%    4.23MB/s    0:00:02
+[2014-05-25 21:47:42 CEST] RemoteControl: Syncing with diskstation 
+(merging synced/git-annex into git-annex...)        
+remote: (Recording state in git...)        
+remote: ok        
+To joern@annex.diskstation.local:/volume1/homes/joern/Annex.movies
+   ea4b518..d1f5994  git-annex -> synced/git-annex
+    13467648  73%    4.25MB/s    0:00:01
+    17104896  93%    4.04MB/s    0:00:00
+    18351413 100%    4.03MB/s    0:00:04 (xfer#1, to-check=0/1)
+
+sent 18353823 bytes  received 42 bytes  3337066.36 bytes/sec
+total size is 18351413  speedup is 1.00
+[2014-05-25 21:47:44 CEST] Transferrer: Uploaded 71_settin..photo.mp4
+From annex.diskstation.local:/volume1/homes/joern/Annex.movies
+   c52b780..c25cca9  git-annex  -> diskstation/git-annex
+error: Ref refs/remotes/diskstation/synced/git-annex is at d1f5994466b35d4c32ad5a9bbb9dd071e461e3e2 but expected ea4b518d3f3234433b2819aa9965b5613366f1bb
+ ! ea4b518..d1f5994  synced/git-annex -> diskstation/synced/git-annex  (unable to update local ref)
+
+WORM-s13344990-m1398385353--Learning SketchUp 2013%70_creating-a-library.mp4
+       32768   0%    0.00kB/s    0:00:00
+     3506176  26%    3.17MB/s    0:00:03
+[2014-05-25 21:47:47 CEST] Pusher: Syncing with diskstation 
+     6914048  51%    3.20MB/s    0:00:01
+    11337728  84%    3.52MB/s    0:00:00
+    13344990 100%    3.56MB/s    0:00:03 (xfer#1, to-check=0/1)
+
+sent 13346775 bytes  received 42 bytes  2426694.00 bytes/sec
+total size is 13344990  speedup is 1.00
+[2014-05-25 21:47:49 CEST] Transferrer: Uploaded 70_creati..brary.mp4
+
+WORM-s9842222-m1398386008--Learning SketchUp 2013%69_using-the-podium-browser.mp4
+       32768   0%    0.00kB/s    0:00:00
+remote: merge git-annex      5275648  53%    4.88MB/s    0:00:00
+     8323072  84%    3.89MB/s    0:00:00
+     9842222 100%    3.92MB/s    0:00:02 (xfer#1, to-check=0/1)
+[2014-05-25 21:47:52 CEST] RemoteControl: Syncing with diskstation 
+(merging synced/git-annex into git-annex...)        
+remote: (Recording state in git...)        
+remote: ok        
+To joern@annex.diskstation.local:/volume1/homes/joern/Annex.movies
+   d1f5994..33198fa  git-annex -> synced/git-annex
+
+sent 9843584 bytes  received 42 bytes  2812464.57 bytes/sec
+total size is 9842222  speedup is 1.00
+[2014-05-25 21:47:53 CEST] Transferrer: Uploaded 69_using-..owser.mp4
+
+WORM-s11662587-m1398385968--Learning SketchUp 2013%68_using-form-fonts.mp4
+       32768   0%    0.00kB/s    0:00:00
+     4653056  39%    4.33MB/s    0:00:01
+From annex.diskstation.local:/volume1/homes/joern/Annex.movies
+   c25cca9..2496e60  git-annex  -> diskstation/git-annex
+error: Ref refs/remotes/diskstation/synced/git-annex is at 33198fa7a8195245180bb53efbdec32359506368 but expected d1f5994466b35d4c32ad5a9bbb9dd071e461e3e2
+ ! d1f5994..33198fa  synced/git-annex -> diskstation/synced/git-annex  (unable to update local ref)
+     9437184  80%    4.41MB/s    0:00:00
+    11662587 100%    4.32MB/s    0:00:02 (xfer#1, to-check=0/1)
+[2014-05-25 21:47:57 CEST] Pusher: Syncing with diskstation 
+
+sent 11664162 bytes  received 42 bytes  3332629.71 bytes/sec
+total size is 11662587  speedup is 1.00
+[2014-05-25 21:47:57 CEST] Transferrer: Uploaded 68_using-..fonts.mp4
+
+WORM-s8289672-m1398385702--Learning SketchUp 2013%67_using-the-3d-warehouse.mp4
+       32768   0%    0.00kB/s    0:00:00
+     5439488  65%    4.99MB/s    0:00:00
+remote: merge git-annex      8289672 100%    4.48MB/s    0:00:01 (xfer#1, to-check=0/1)
+[2014-05-25 21:48:00 CEST] RemoteControl: Syncing with diskstation 
+
+sent 8290840 bytes  received 42 bytes  3316352.80 bytes/sec
+total size is 8289672  speedup is 1.00
+[2014-05-25 21:48:00 CEST] Transferrer: Uploaded 67_using-..house.mp4
+
+WORM-s12841748-m1398385284--Learning SketchUp 2013%66_adding-the-fridge-and-sink.mp4
+       32768   0%    0.00kB/s    0:00:00
+(merging synced/git-annex into git-annex...)        
+remote: (Recording state in git...)        
+remote: ok        
+remote: (Recording state in git...)        
+To joern@annex.diskstation.local:/volume1/homes/joern/Annex.movies
+   33198fa..f4258a2  git-annex -> synced/git-annex
+     3899392  30%    3.62MB/s    0:00:02
+From annex.diskstation.local:/volume1/homes/joern/Annex.movies
+   2496e60..8f91b25  git-annex  -> diskstation/git-annex
+error: Ref refs/remotes/diskstation/synced/git-annex is at f4258a2cd12ee4f23f3a9c9b9c635b9d59075f21 but expected 33198fa7a8195245180bb53efbdec32359506368
+ ! 33198fa..f4258a2  synced/git-annex -> diskstation/synced/git-annex  (unable to update local ref)
+[2014-05-25 21:48:03 CEST] RemoteControl: Syncing with diskstation 
+     7897088  61%    3.69MB/s    0:00:01
+    11698176  91%    3.66MB/s    0:00:00
+    12841748 100%    3.72MB/s    0:00:03 (xfer#1, to-check=0/1)
+
+sent 12843477 bytes  received 42 bytes  2335185.27 bytes/sec
+total size is 12841748  speedup is 1.00
+[2014-05-25 21:48:05 CEST] Transferrer: Uploaded 66_adding..-sink.mp4
+From annex.diskstation.local:/volume1/homes/joern/Annex.movies
+   8f91b25..c4dc443  git-annex  -> diskstation/git-annex
+
+WORM-s15640646-m1398385334--Learning SketchUp 2013%65_adding-appliances.mp4
+       32768   0%    0.00kB/s    0:00:00
+     5505024  35%    5.08MB/s    0:00:01
+[2014-05-25 21:48:07 CEST] Pusher: Syncing with diskstation 
+     9601024  61%    4.50MB/s    0:00:01
+    13860864  88%    4.32MB/s    0:00:00
+    15640646 100%    4.28MB/s    0:00:03 (xfer#1, to-check=0/1)
+
+sent 15642710 bytes  received 42 bytes  3476167.11 bytes/sec
+total size is 15640646  speedup is 1.00
+[2014-05-25 21:48:09 CEST] Transferrer: Uploaded 65_adding..ances.mp4
+
+WORM-s14906076-m1398385849--Learning SketchUp 2013%64_adding-materials.mp4
+       32768   0%    0.00kB/s    0:00:00
+remote: merge git-annex      4620288  30%    4.38MB/s    0:00:02
+[2014-05-25 21:48:12 CEST] RemoteControl: Syncing with diskstation 
+(merging synced/git-annex into git-annex...)        
+remote: (Recording state in git...)        
+remote: ok        
+To joern@annex.diskstation.local:/volume1/homes/joern/Annex.movies
+   f4258a2..89ce825  git-annex -> synced/git-annex
+     7700480  51%    3.61MB/s    0:00:01
+    10649600  71%    3.30MB/s    0:00:01
+    13467648  90%    3.13MB/s    0:00:00
+From annex.diskstation.local:/volume1/homes/joern/Annex.movies
+   c4dc443..6fdf232  git-annex  -> diskstation/git-annex
+error: Ref refs/remotes/diskstation/synced/git-annex is at 89ce825f4a57b9bc5490b2c0f7dc77e9b3cf0016 but expected f4258a2cd12ee4f23f3a9c9b9c635b9d59075f21
+ ! f4258a2..89ce825  synced/git-annex -> diskstation/synced/git-annex  (unable to update local ref)
+    14906076 100%    3.07MB/s    0:00:04 (xfer#1, to-check=0/1)
+
+sent 14908047 bytes  received 42 bytes  2293552.15 bytes/sec
+total size is 14906076  speedup is 1.00
+[2014-05-25 21:48:15 CEST] Transferrer: Uploaded 64_adding..rials.mp4
+
+WORM-s9700735-m1398385733--Learning SketchUp 2013%63_creating-the-countertop.mp4
+       32768   0%    0.00kB/s    0:00:00
+     4554752  46%    4.18MB/s    0:00:01
+[2014-05-25 21:48:17 CEST] Pusher: Syncing with diskstation 
+     7569408  78%    3.53MB/s    0:00:00
+     9700735 100%    3.40MB/s    0:00:02 (xfer#1, to-check=0/1)
+
+sent 9702080 bytes  received 42 bytes  2156027.11 bytes/sec
+total size is 9700735  speedup is 1.00
+[2014-05-25 21:48:19 CEST] Transferrer: Uploaded 63_creati..ertop.mp4
+
+WORM-s15089043-m1398386778--Learning SketchUp 2013%62_cabinet-doors.mp4
+       32768   0%    0.00kB/s    0:00:00
+remote: merge git-annex      4456448  29%    4.05MB/s    0:00:02
+     7340032  48%    3.36MB/s    0:00:02
+(merging synced/git-annex into git-annex...)        
+remote: (Recording state in git...)        
+remote: ok        
+To joern@annex.diskstation.local:/volume1/homes/joern/Annex.movies
+   89ce825..be1967a  git-annex -> synced/git-annex
+[2014-05-25 21:48:22 CEST] RemoteControl: Syncing with diskstation 
+    10223616  67%    3.12MB/s    0:00:01
+    13369344  88%    3.08MB/s    0:00:00
+    15089043 100%    3.02MB/s    0:00:04 (xfer#1, to-check=0/1)
+
+sent 15091035 bytes  received 42 bytes  2321704.15 bytes/sec
+total size is 15089043  speedup is 1.00
+[2014-05-25 21:48:25 CEST] Transferrer: Uploaded 62_cabine..doors.mp4
+From annex.diskstation.local:/volume1/homes/joern/Annex.movies
+   6fdf232..5d94435  git-annex  -> diskstation/git-annex
+error: Ref refs/remotes/diskstation/synced/git-annex is at be1967a15a951b38c76e4e8f674812a01df4c8c8 but expected 89ce825f4a57b9bc5490b2c0f7dc77e9b3cf0016
+ ! 89ce825..be1967a  synced/git-annex -> diskstation/synced/git-annex  (unable to update local ref)
+
+WORM-s10872872-m1398384973--Learning SketchUp 2013%61_wall-cabinets.mp4
+       32768   0%    0.00kB/s    0:00:00
+     4194304  38%    3.90MB/s    0:00:01
+[2014-05-25 21:48:27 CEST] Pusher: Syncing with diskstation 
+     7208960  66%    3.34MB/s    0:00:01
+    10190848  93%    3.16MB/s    0:00:00
+    10872872 100%    3.14MB/s    0:00:03 (xfer#1, to-check=0/1)
+
+sent 10874348 bytes  received 42 bytes  2416531.11 bytes/sec
+total size is 10872872  speedup is 1.00
+remote: merge git-annex [2014-05-25 21:48:29 CEST] Transferrer: Uploaded 61_wall-c..inets.mp4
+
+WORM-s14169006-m1398385488--Learning SketchUp 2013%60_base-cabinets.mp4
+       32768   0%    0.00kB/s    0:00:00
+     4292608  30%    3.96MB/s    0:00:02
+[2014-05-25 21:48:31 CEST] RemoteControl: Syncing with diskstation 
+(merging synced/git-annex into git-annex...)        
+remote: (Recording state in git...)        
+remote: ok        
+To joern@annex.diskstation.local:/volume1/homes/joern/Annex.movies
+   be1967a..0c0e83a  git-annex -> synced/git-annex
+     7143424  50%    3.29MB/s    0:00:02
+    10289152  72%    3.16MB/s    0:00:01
+    13139968  92%    3.03MB/s    0:00:00
+From annex.diskstation.local:/volume1/homes/joern/Annex.movies
+   5d94435..c7aef43  git-annex  -> diskstation/git-annex
+error: Ref refs/remotes/diskstation/synced/git-annex is at 0c0e83a4b729ed1ef86e4854405ec752226bbd09 but expected be1967a15a951b38c76e4e8f674812a01df4c8c8
+ ! be1967a..0c0e83a  synced/git-annex -> diskstation/synced/git-annex  (unable to update local ref)
+    14169006 100%    3.08MB/s    0:00:04 (xfer#1, to-check=0/1)
+
+sent 14170886 bytes  received 42 bytes  2180142.77 bytes/sec
+total size is 14169006  speedup is 1.00
+[2014-05-25 21:48:35 CEST] Transferrer: Uploaded 60_base-c..inets.mp4
+[2014-05-25 21:48:35 CEST] Pusher: Syncing with diskstation 
+
+WORM-s13699702-m1398385658--Learning SketchUp 2013%59_adding-textures.mp4
+       32768   0%    0.00kB/s    0:00:00
+     4063232  29%    3.80MB/s    0:00:02
+remote: merge git-annex      7503872  54%    3.54MB/s    0:00:01
+    11599872  84%    3.65MB/s    0:00:00
+    13699702 100%    3.75MB/s    0:00:03 (xfer#1, to-check=0/1)
+[2014-05-25 21:48:41 CEST] RemoteControl: Syncing with diskstation 
+(merging synced/git-annex into git-annex...)        
+remote: (Recording state in git...)        
+remote: ok        
+To joern@annex.diskstation.local:/volume1/homes/joern/Annex.movies
+   0c0e83a..cd5b9c0  git-annex -> synced/git-annex
+
+sent 13701528 bytes  received 42 bytes  3044793.33 bytes/sec
+total size is 13699702  speedup is 1.00
+[2014-05-25 21:48:42 CEST] Transferrer: Uploaded 59_adding..tures.mp4
+
+WORM-s10444949-m1398386483--Learning SketchUp 2013%58_mantle-support.mp4
+       32768   0%    0.00kB/s    0:00:00
+     5177344  49%    4.89MB/s    0:00:01
+From annex.diskstation.local:/volume1/homes/joern/Annex.movies
+   c7aef43..e499593  git-annex  -> diskstation/git-annex
+error: Ref refs/remotes/diskstation/synced/git-annex is at cd5b9c01988d7c7b10730356ed55cc8b1422ce52 but expected 0c0e83a4b729ed1ef86e4854405ec752226bbd09
+ ! 0c0e83a..cd5b9c0  synced/git-annex -> diskstation/synced/git-annex  (unable to update local ref)
+     9175040  87%    4.34MB/s    0:00:00
+    10444949 100%    3.92MB/s    0:00:02 (xfer#1, to-check=0/1)
+
+sent 10446374 bytes  received 42 bytes  2984690.29 bytes/sec
+total size is 10444949  speedup is 1.00
+[2014-05-25 21:48:46 CEST] Transferrer: Uploaded 58_mantle..pport.mp4
+[2014-05-25 21:48:46 CEST] Pusher: Syncing with diskstation 
+
+WORM-s24021797-m1398385079--Learning SketchUp 2013%57_mantle-columns.mp4
+       32768   0%    0.00kB/s    0:00:00
+     5079040  21%    4.81MB/s    0:00:03
+     9011200  37%    4.25MB/s    0:00:03
+    13041664  54%    4.09MB/s    0:00:02
+remote: merge git-annex     17006592  70%    3.97MB/s    0:00:01
+    20643840  85%    3.61MB/s    0:00:00
+[2014-05-25 21:48:52 CEST] RemoteControl: Syncing with diskstation 
+(merging synced/git-annex into git-annex...)        
+remote: (Recording state in git...)        
+remote: ok        
+To joern@annex.diskstation.local:/volume1/homes/joern/Annex.movies
+   cd5b9c0..7f4755f  git-annex -> synced/git-annex
+    23887872  99%    3.44MB/s    0:00:00
+    24021797 100%    3.69MB/s    0:00:06 (xfer#1, to-check=0/1)
+
+sent 24024882 bytes  received 42 bytes  3203323.20 bytes/sec
+total size is 24021797  speedup is 1.00
+[2014-05-25 21:48:53 CEST] Transferrer: Uploaded 57_mantle..lumns.mp4
+
+WORM-s20301926-m1398385822--Learning SketchUp 2013%56_building-the-mantle.mp4
+       32768   0%    0.00kB/s    0:00:00
+From annex.diskstation.local:/volume1/homes/joern/Annex.movies
+   e499593..a7e1df5  git-annex  -> diskstation/git-annex
+error: Ref refs/remotes/diskstation/synced/git-annex is at 7f4755fe1b2fbe86253a5854d83948a479df1302 but expected cd5b9c01988d7c7b10730356ed55cc8b1422ce52
+ ! cd5b9c0..7f4755f  synced/git-annex -> diskstation/synced/git-annex  (unable to update local ref)
+     4358144  21%    4.07MB/s    0:00:03
+[2014-05-25 21:48:55 CEST] Pusher: Syncing with diskstation 
+     7241728  35%    3.34MB/s    0:00:03
+    10125312  49%    3.14MB/s    0:00:03
+    13271040  65%    3.10MB/s    0:00:02
+remote: merge git-annex     16154624  79%    2.73MB/s    0:00:01
+    18972672  93%    2.74MB/s    0:00:00
+[2014-05-25 21:49:00 CEST] RemoteControl: Syncing with diskstation 
+(merging synced/git-annex into git-annex...)        
+remote: (Recording state in git...)        
+remote: ok        
+To joern@annex.diskstation.local:/volume1/homes/joern/Annex.movies
+   7f4755f..367f02c  git-annex -> synced/git-annex
+    20301926 100%    2.92MB/s    0:00:06 (xfer#1, to-check=0/1)
+
+sent 20304560 bytes  received 42 bytes  2388776.71 bytes/sec
+total size is 20301926  speedup is 1.00
+[2014-05-25 21:49:01 CEST] Transferrer: Uploaded 56_buildi..antle.mp4
+
+WORM-s15278630-m1398386298--Learning SketchUp 2013%55_starting-at-the-base.mp4
+       32768   0%    0.00kB/s    0:00:00
+From annex.diskstation.local:/volume1/homes/joern/Annex.movies
+   a7e1df5..200c73f  git-annex  -> diskstation/git-annex
+error: Ref refs/remotes/diskstation/synced/git-annex is at 367f02ceec6d27fdb719156715a8a24a9f0cc149 but expected 7f4755fe1b2fbe86253a5854d83948a479df1302
+ ! 7f4755f..367f02c  synced/git-annex -> diskstation/synced/git-annex  (unable to update local ref)
+     4653056  30%    4.41MB/s    0:00:02
+     7733248  50%    3.61MB/s    0:00:02
+[2014-05-25 21:49:04 CEST] Pusher: Syncing with diskstation 
+    10616832  69%    3.32MB/s    0:00:01
+    13500416  88%    3.14MB/s    0:00:00
+    15278630 100%    3.14MB/s    0:00:04 (xfer#1, to-check=0/1)
+remote: merge git-annex 
+sent 15280653 bytes  received 42 bytes  2350876.15 bytes/sec
+total size is 15278630  speedup is 1.00
+[2014-05-25 21:49:07 CEST] Transferrer: Uploaded 55_starti..-base.mp4
+(merging synced/git-annex into git-annex...)        
+remote: (Recording state in git...)        
+remote: ok        
+To joern@annex.diskstation.local:/volume1/homes/joern/Annex.movies
+   367f02c..e9cb764  git-annex -> synced/git-annex
+
+WORM-s17900868-m1398386914--Learning SketchUp 2013%54_using-the-window-feature.mp4
+       32768   0%    0.00kB/s    0:00:00
+[2014-05-25 21:49:08 CEST] RemoteControl: Syncing with diskstation 
+     4718592  26%    4.44MB/s    0:00:02
+     8290304  46%    3.92MB/s    0:00:02
+    12189696  68%    3.85MB/s    0:00:01
+From annex.diskstation.local:/volume1/homes/joern/Annex.movies
+   200c73f..e2eb7b5  git-annex  -> diskstation/git-annex
+    16482304  92%    3.88MB/s    0:00:00
+    17900868 100%    3.91MB/s    0:00:04 (xfer#1, to-check=0/1)
+
+sent 17903215 bytes  received 42 bytes  2754347.23 bytes/sec
+total size is 17900868  speedup is 1.00
+[2014-05-25 21:49:13 CEST] Pusher: Syncing with diskstation 
+[2014-05-25 21:49:13 CEST] Transferrer: Uploaded 54_using-..ature.mp4
+
+WORM-s13516185-m1398386734--Learning SketchUp 2013%53_creating-walls-and-door-openings.mp4
+       32768   0%    0.00kB/s    0:00:00
+     5242880  38%    4.94MB/s    0:00:01
+     9273344  68%    4.36MB/s    0:00:00
+remote: merge git-annex     13369344  98%    4.17MB/s    0:00:00
+    13516185 100%    4.17MB/s    0:00:03 (xfer#1, to-check=0/1)
+[2014-05-25 21:49:18 CEST] RemoteControl: Syncing with diskstation 
+
+sent 13518004 bytes  received 42 bytes  3004010.22 bytes/sec
+total size is 13516185  speedup is 1.00
+[2014-05-25 21:49:18 CEST] Transferrer: Uploaded 53_creati..nings.mp4
+(merging synced/git-annex into git-annex...)        
+remote: (Recording state in git...)        
+remote: ok        
+remote: (Recording state in git...)        
+To joern@annex.diskstation.local:/volume1/homes/joern/Annex.movies
+
+WORM-s18590095-m1398385163--Learning SketchUp 2013%52_installing-bit-tools-plugins.mp4
+   e9cb764..c7f1783  git-annex -> synced/git-annex
+       32768   0%    0.00kB/s    0:00:00
+From annex.diskstation.local:/volume1/homes/joern/Annex.movies
+   e2eb7b5..ff506d2  git-annex  -> diskstation/git-annex
+error: Ref refs/remotes/diskstation/synced/git-annex is at c7f178361d32682b60332b5c508a3ce0aec3925c but expected e9cb7644b684d418952b4132c725211eb045b260
+ ! e9cb764..c7f1783  synced/git-annex -> diskstation/synced/git-annex  (unable to update local ref)
+[2014-05-25 21:49:19 CEST] RemoteControl: Syncing with diskstation 
+     4259840  22%    3.88MB/s    0:00:03
+     6684672  35%    3.09MB/s    0:00:03
+From annex.diskstation.local:/volume1/homes/joern/Annex.movies
+   ff506d2..ff2bd4c  git-annex  -> diskstation/git-annex
+    10813440  58%    3.36MB/s    0:00:02
+[2014-05-25 21:49:23 CEST] Pusher: Syncing with diskstation 
+    14876672  80%    3.46MB/s    0:00:01
+    18590095 100%    3.53MB/s    0:00:05 (xfer#1, to-check=0/1)
+
+sent 18592530 bytes  received 42 bytes  2860395.69 bytes/sec
+total size is 18590095  speedup is 1.00
+[2014-05-25 21:49:24 CEST] Transferrer: Uploaded 52_instal..ugins.mp4
+
+WORM-s28284719-m1398385600--Learning SketchUp 2013%51_double-hung-windows.mp4
+       32768   0%    0.00kB/s    0:00:00
+     5046272  17%    4.72MB/s    0:00:04
+remote: merge git-annex      9076736  32%    4.27MB/s    0:00:04
+    13303808  47%    4.16MB/s    0:00:03
+[2014-05-25 21:49:29 CEST] RemoteControl: Syncing with diskstation 
+(merging synced/git-annex into git-annex...)        
+remote: (Recording state in git...)        
+remote: ok        
+To joern@annex.diskstation.local:/volume1/homes/joern/Annex.movies
+   c7f1783..a39803a  git-annex -> synced/git-annex
+    16711680  59%    3.89MB/s    0:00:02
+    18841600  66%    3.14MB/s    0:00:02
+    21331968  75%    2.78MB/s    0:00:02
+From annex.diskstation.local:/volume1/homes/joern/Annex.movies
+   ff2bd4c..338767b  git-annex  -> diskstation/git-annex
+error: Ref refs/remotes/diskstation/synced/git-annex is at a39803aef5ce739ae6a6472a7de156efac31bebc but expected c7f178361d32682b60332b5c508a3ce0aec3925c
+ ! c7f1783..a39803a  synced/git-annex -> diskstation/synced/git-annex  (unable to update local ref)
+    26050560  92%    2.89MB/s    0:00:00
+    28284719 100%    3.45MB/s    0:00:07 (xfer#1, to-check=0/1)
+
+sent 28288329 bytes  received 42 bytes  2977723.26 bytes/sec
+total size is 28284719  speedup is 1.00
+[2014-05-25 21:49:34 CEST] Transferrer: Uploaded 51_double..ndows.mp4
+
+WORM-s16987083-m1398385579--Learning SketchUp 2013%50_creating-the-front-window-frame.mp4
+       32768   0%    0.00kB/s    0:00:00
+[2014-05-25 21:49:35 CEST] Pusher: Syncing with diskstation 
+     5144576  30%    4.85MB/s    0:00:02
+     9371648  55%    4.39MB/s    0:00:01
+    11468800  67%    3.58MB/s    0:00:01
+    15859712  93%    3.72MB/s    0:00:00
+    16987083 100%    3.75MB/s    0:00:04 (xfer#1, to-check=0/1)
+
+sent 16989325 bytes  received 42 bytes  3088975.82 bytes/sec
+total size is 16987083  speedup is 1.00
+[2014-05-25 21:49:39 CEST] Transferrer: Uploaded 50_creati..frame.mp4
+remote: merge git-annex 
+WORM-s19055363-m1398385037--Learning SketchUp 2013%49_creating-doors.mp4
+       32768   0%    0.00kB/s    0:00:00
+     5013504  26%    4.68MB/s    0:00:02
+[2014-05-25 21:49:41 CEST] RemoteControl: Syncing with diskstation 
+(merging synced/git-annex into git-annex...)        
+remote: (Recording state in git...)        
+remote: ok        
+To joern@annex.diskstation.local:/volume1/homes/joern/Annex.movies
+   a39803a..8462dc1  git-annex -> synced/git-annex
+     9011200  47%    4.21MB/s    0:00:02
+    13041664  68%    4.05MB/s    0:00:01
+    16351232  85%    3.83MB/s    0:00:00
+From annex.diskstation.local:/volume1/homes/joern/Annex.movies
+   338767b..47c33b6  git-annex  -> diskstation/git-annex
+error: Ref refs/remotes/diskstation/synced/git-annex is at 8462dc1c099c64ff94d5d35cb2f2efaff89b028e but expected a39803aef5ce739ae6a6472a7de156efac31bebc
+ ! a39803a..8462dc1  synced/git-annex -> diskstation/synced/git-annex  (unable to update local ref)
+    19055363 100%    3.97MB/s    0:00:04 (xfer#1, to-check=0/1)
+
+sent 19057840 bytes  received 42 bytes  2931981.85 bytes/sec
+total size is 19055363  speedup is 1.00
+[2014-05-25 21:49:46 CEST] Pusher: Syncing with diskstation 
+[2014-05-25 21:49:46 CEST] Transferrer: Uploaded 49_creati..doors.mp4
+
+WORM-s11184398-m1398385463--Learning SketchUp 2013%48_review-of-components-and-nested-groups.mp4
+       32768   0%    0.00kB/s    0:00:00
+     3375104  30%    2.72MB/s    0:00:02
+     5603328  50%    2.38MB/s    0:00:02
+remote: merge git-annex      9371648  83%    2.75MB/s    0:00:00
+    11184398 100%    2.89MB/s    0:00:03 (xfer#1, to-check=0/1)
+
+sent 11185939 bytes  received 42 bytes  2033814.73 bytes/sec
+total size is 11184398  speedup is 1.00
+[2014-05-25 21:49:52 CEST] RemoteControl: Syncing with diskstation 
+[2014-05-25 21:49:52 CEST] Transferrer: Uploaded 48_review..roups.mp4
+(merging synced/git-annex into git-annex...)        
+remote: (Recording state in git...)        
+remote: ok        
+remote: (Recording state in git...)        
+To joern@annex.diskstation.local:/volume1/homes/joern/Annex.movies
+   8462dc1..877c07a  git-annex -> synced/git-annex
+
+WORM-s19612237-m1398385719--Learning SketchUp 2013%47_adding-railing.mp4
+       32768   0%    0.00kB/s    0:00:00
+     3997696  20%    3.75MB/s    0:00:04
+From annex.diskstation.local:/volume1/homes/joern/Annex.movies
+   47c33b6..bb182f5  git-annex  -> diskstation/git-annex
+error: Ref refs/remotes/diskstation/synced/git-annex is at 877c07ad78a25ddf859ac96049ec938fa9b0b5f3 but expected 8462dc1c099c64ff94d5d35cb2f2efaff89b028e
+ ! 8462dc1..877c07a  synced/git-annex -> diskstation/synced/git-annex  (unable to update local ref)
+     8257536  42%    3.88MB/s    0:00:02
+    12845056  65%    4.02MB/s    0:00:01
+    17039360  86%    4.00MB/s    0:00:00
+[2014-05-25 21:49:57 CEST] Pusher: Syncing with diskstation 
+    19612237 100%    3.99MB/s    0:00:04 (xfer#1, to-check=0/1)
+
+sent 19614782 bytes  received 42 bytes  3017665.23 bytes/sec
+total size is 19612237  speedup is 1.00
+[2014-05-25 21:49:58 CEST] Transferrer: Uploaded 47_adding..iling.mp4
+
+WORM-s23427266-m1398386041--Learning SketchUp 2013%46_adding-spindles-with-move-and-copy.mp4
+       32768   0%    0.00kB/s    0:00:00
+     4128768  17%    3.80MB/s    0:00:04
+remote: merge git-annex      7176192  30%    3.33MB/s    0:00:04
+    11304960  48%    3.53MB/s    0:00:03
+[2014-05-25 21:50:02 CEST] RemoteControl: Syncing with diskstation 
+(merging synced/git-annex into git-annex...)        
+remote: (Recording state in git...)        
+remote: ok        
+To joern@annex.diskstation.local:/volume1/homes/joern/Annex.movies
+   877c07a..873eda1  git-annex -> synced/git-annex
+    15138816  64%    3.53MB/s    0:00:02
+    18644992  79%    3.42MB/s    0:00:01
+    21135360  90%    3.29MB/s    0:00:00
+From annex.diskstation.local:/volume1/homes/joern/Annex.movies
+   bb182f5..470a1af  git-annex  -> diskstation/git-annex
+error: Ref refs/remotes/diskstation/synced/git-annex is at 873eda1fd0b52f509bbd03a02ec5a3d6d8085488 but expected 877c07ad78a25ddf859ac96049ec938fa9b0b5f3
+ ! 877c07a..873eda1  synced/git-annex -> diskstation/synced/git-annex  (unable to update local ref)
+    23427266 100%    3.17MB/s    0:00:07 (xfer#1, to-check=0/1)
+
+sent 23430295 bytes  received 42 bytes  2756510.24 bytes/sec
+total size is 23427266  speedup is 1.00
+[2014-05-25 21:50:06 CEST] Transferrer: Uploaded 46_adding..-copy.mp4
+
+WORM-s28584390-m1398385775--Learning SketchUp 2013%45_making-a-unique-bottom-step.mp4
+       32768   0%    0.00kB/s    0:00:00
+[2014-05-25 21:50:07 CEST] Pusher: Syncing with diskstation 
+     5472256  19%    4.96MB/s    0:00:04
+     9895936  34%    4.53MB/s    0:00:04
+    14221312  49%    4.40MB/s    0:00:03
+remote: merge git-annex     18055168  63%    4.21MB/s    0:00:02
+    22609920  79%    4.05MB/s    0:00:01
+[2014-05-25 21:50:12 CEST] RemoteControl: Syncing with diskstation 
+(merging synced/git-annex into git-annex...)        
+remote: (Recording state in git...)        
+remote: ok        
+To joern@annex.diskstation.local:/volume1/homes/joern/Annex.movies
+   873eda1..274fd99  git-annex -> synced/git-annex
+    26279936  91%    3.88MB/s    0:00:00
+    28584390 100%    4.06MB/s    0:00:06 (xfer#1, to-check=0/1)
+
+sent 28588044 bytes  received 42 bytes  3363304.24 bytes/sec
+total size is 28584390  speedup is 1.00
+[2014-05-25 21:50:14 CEST] Transferrer: Uploaded 45_making..-step.mp4
+From annex.diskstation.local:/volume1/homes/joern/Annex.movies
+   470a1af..989e8b5  git-annex  -> diskstation/git-annex
+error: Ref refs/remotes/diskstation/synced/git-annex is at 274fd99803c01b7911e45dcb050480eb4bfe2a82 but expected 873eda1fd0b52f509bbd03a02ec5a3d6d8085488
+ ! 873eda1..274fd99  synced/git-annex -> diskstation/synced/git-annex  (unable to update local ref)
+
+WORM-s24162552-m1398385100--Learning SketchUp 2013%44_adding-riser-detail.mp4
+       32768   0%    0.00kB/s    0:00:00
+     5341184  22%    5.06MB/s    0:00:03
+[2014-05-25 21:50:17 CEST] Pusher: Syncing with diskstation 
+     9240576  38%    4.38MB/s    0:00:03
+    13959168  57%    4.38MB/s    0:00:02
+    18153472  75%    4.28MB/s    0:00:01
+remote: merge git-annex     22347776  92%    4.00MB/s    0:00:00
+    24162552 100%    4.15MB/s    0:00:05 (xfer#1, to-check=0/1)
+[2014-05-25 21:50:22 CEST] RemoteControl: Syncing with diskstation 
+
+sent 24165658 bytes  received 42 bytes  3222093.33 bytes/sec
+total size is 24162552  speedup is 1.00
+[2014-05-25 21:50:22 CEST] Transferrer: Uploaded 44_adding..etail.mp4
+(merging synced/git-annex into git-annex...)        
+remote: (Recording state in git...)        
+remote: ok        
+remote: (Recording state in git...)        
+To joern@annex.diskstation.local:/volume1/homes/joern/Annex.movies
+   274fd99..baa9cda  git-annex -> synced/git-annex
+
+WORM-s14814183-m1398385868--Learning SketchUp 2013%43_creating-the-staircase.mp4
+       32768   0%    0.00kB/s    0:00:00
+     4947968  33%    4.69MB/s    0:00:02
+From annex.diskstation.local:/volume1/homes/joern/Annex.movies
+   989e8b5..c15470f  git-annex  -> diskstation/git-annex
+error: Ref refs/remotes/diskstation/synced/git-annex is at baa9cdae739f6bc1f928d9abe82c0f83b6645093 but expected 274fd99803c01b7911e45dcb050480eb4bfe2a82
+ ! 274fd99..baa9cda  synced/git-annex -> diskstation/synced/git-annex  (unable to update local ref)
+     9011200  60%    4.25MB/s    0:00:01
+    13729792  92%    4.31MB/s    0:00:00
+    14814183 100%    4.30MB/s    0:00:03 (xfer#1, to-check=0/1)
+
+sent 14816152 bytes  received 42 bytes  2693853.45 bytes/sec
+total size is 14814183  speedup is 1.00
+[2014-05-25 21:50:27 CEST] Transferrer: Uploaded 43_creati..rcase.mp4
+
+WORM-s14506973-m1398386318--Learning SketchUp 2013%42_why-we-use-groups-and-components.mp4
+       32768   0%    0.00kB/s    0:00:00
+[2014-05-25 21:50:27 CEST] Pusher: Syncing with diskstation 
+     5341184  36%    5.04MB/s    0:00:01
+     9633792  66%    4.55MB/s    0:00:01
+    13926400  95%    4.37MB/s    0:00:00
+    14506973 100%    4.38MB/s    0:00:03 (xfer#1, to-check=0/1)
+
+sent 14508912 bytes  received 42 bytes  3224212.00 bytes/sec
+total size is 14506973  speedup is 1.00
+[2014-05-25 21:50:31 CEST] Transferrer: Uploaded 42_why-we..nents.mp4
+
+WORM-s11494983-m1398385388--Learning SketchUp 2013%41_copy-and-paste-in-place.mp4
+       32768   0%    0.00kB/s    0:00:00
+remote: merge git-annex      5210112  45%    4.82MB/s    0:00:01
+     9404416  81%    4.37MB/s    0:00:00
+    11494983 100%    4.21MB/s    0:00:02 (xfer#1, to-check=0/1)
+
+sent 11496545 bytes  received 42 bytes  3284739.14 bytes/sec
+total size is 11494983  speedup is 1.00
+[2014-05-25 21:50:34 CEST] RemoteControl: Syncing with diskstation 
+[2014-05-25 21:50:34 CEST] Transferrer: Uploaded 41_copy-a..place.mp4
+
+
+# End of transcript or log.
+"""]]
diff --git a/doc/bugs/gpg-agent/comment_1_86860841aaa38541968693ec02f6a506._comment b/doc/bugs/gpg-agent/comment_1_86860841aaa38541968693ec02f6a506._comment
new file mode 100644
index 0000000000..382b3a8b77
--- /dev/null
+++ b/doc/bugs/gpg-agent/comment_1_86860841aaa38541968693ec02f6a506._comment
@@ -0,0 +1,10 @@
+[[!comment format=mdwn
+ username="http://joeyh.name/"
+ ip="209.250.56.176"
+ subject="comment 1"
+ date="2014-05-30T19:15:33Z"
+ content="""
+What I see in the log is git-annex is syncing files to/from the remote \"diskstation\". This remote is not encrypted at all, so git-annex is not using gpg. There is no mention of gpg in the log at all.
+
+So, I don't see any indication that whatever is causing too many gpg-agent processes to be spawned is git-annex. Can you share more information that would point toward git-annex being the cause of this problem?
+"""]]
diff --git a/doc/bugs/gpg-agent/comment_2_de6f9b8f20e44e93759b9bf0db695def._comment b/doc/bugs/gpg-agent/comment_2_de6f9b8f20e44e93759b9bf0db695def._comment
new file mode 100644
index 0000000000..8158500ae2
--- /dev/null
+++ b/doc/bugs/gpg-agent/comment_2_de6f9b8f20e44e93759b9bf0db695def._comment
@@ -0,0 +1,9 @@
+[[!comment format=mdwn
+ username="https://launchpad.net/~r0lf"
+ nickname="Rolf Leggewie"
+ avatar="http://cdn.libravatar.org/avatar/aa82122557e706df7ba83bd1983eb79ef1ba2e51350217850176d4f9a1bb2bc0"
+ subject="comment 2"
+ date="2016-12-24T17:56:13Z"
+ content="""
+time to trim the list of open bugs and close this ticket as unreproducible?
+"""]]
diff --git a/doc/bugs/gpg_goes_to_100__37___cpu_on_bad_input_data.mdwn b/doc/bugs/gpg_goes_to_100__37___cpu_on_bad_input_data.mdwn
new file mode 100644
index 0000000000..5f36a7cd0d
--- /dev/null
+++ b/doc/bugs/gpg_goes_to_100__37___cpu_on_bad_input_data.mdwn
@@ -0,0 +1,18 @@
+### Please describe the problem.
+Bad input data given to gpg will make it lock.
+
+### What steps will reproduce the problem?
+Trying to download gpg encrypted data from remote with bad/incorrect data
+
+### What version of git-annex are you using? On what operating system?
+4.20130501-gd9e288b, on ubuntu 13.04
+
+### Please provide any additional information below.
+Transcript
+http://paste.ubuntu.com/5626517/
+
+Note: The problem is most likely caused by bad data from either the nntp hooks program, or the nntp server itself.
+
+I also have one file with bad data that consistently is NOT a problem for gpg to handle(lines 24-25 of log).
+
+[[!tag moreinfo]]
diff --git a/doc/bugs/gpg_goes_to_100__37___cpu_on_bad_input_data/comment_1_889218fb7c0115b03d9bad0c07296097._comment b/doc/bugs/gpg_goes_to_100__37___cpu_on_bad_input_data/comment_1_889218fb7c0115b03d9bad0c07296097._comment
new file mode 100644
index 0000000000..43211bdd89
--- /dev/null
+++ b/doc/bugs/gpg_goes_to_100__37___cpu_on_bad_input_data/comment_1_889218fb7c0115b03d9bad0c07296097._comment
@@ -0,0 +1,36 @@
+[[!comment format=mdwn
+ username="http://joeyh.name/"
+ nickname="joey"
+ subject="comment 1"
+ date="2013-05-06T17:09:58Z"
+ content="""
+What version of gnupg do you have installed?
+
+
    +gpg: block_filter 0x946320: read error (size=12864,a->size=12864)
+gpg: [don't know]: invalid packet (ctb=68)
+gpg: [don't know]: invalid packet (ctb=21)
+gpg: mdc_packet with invalid encoding
+gpg: decryption failed: invalid packet
+gpg: [don't know]: invalid packet (ctb=00)
+gpg: block_filter: pending bytes!
+
    
+
+I don't see how git-annex can possibly guard against gpg behaving this way when given bad data. So the best thing to do would be to get a test case file that causes gpg to behave this way, and then I could file a bug on gpg and get it fixed to immediately exit.
+
+I tried to reproduce this by encrypting a 1 mb file with gpg. This yeilded a 1.1 mb file. I then truncated it back to 1 mb, and tried to decrypt it.
+
+[[!format sh \"\"\"
+joey@gnu:~>gpg --decrypt me2.gpg > x
+gpg: encrypted with 1 passphrase
+gpg: block_filter 0x9d6fda0: read error (size=15680,a->size=15680)
+gpg: Problem reading source (8570 bytes remaining)
+gpg: handle plaintext failed: file read error
+gpg: mdc_packet with invalid encoding
+gpg: decryption failed: invalid packet
+gpg: block_filter: pending bytes!
+zsh: exit 2     gpg --decrypt 
+\"\"\"]]
+
+gpg exited immediately on error, which is what it should do. So it doesn't seem likely I can guess at a test case file that causes gpg to behave this way. You will need to provide one for me to help.
+"""]]
diff --git a/doc/bugs/graft__47__graft_cleanup_commits_--_really_needed__63__.mdwn b/doc/bugs/graft__47__graft_cleanup_commits_--_really_needed__63__.mdwn
new file mode 100644
index 0000000000..50444ff5f6
--- /dev/null
+++ b/doc/bugs/graft__47__graft_cleanup_commits_--_really_needed__63__.mdwn
@@ -0,0 +1,61 @@
+### Please describe the problem.
+
+Playing with the new fancy "export" feature.  Thanks again!
+
+Somehow I see now commits in the git-annex branch in the quick succession establishing and then killing a "graft".
+Are they really needed/used by annex or could have been squashed/avoided?
+
+some details of the recent history of commands and how git-annex branch looks like are listed below
+
+
+### What version of git-annex are you using? On what operating system?
+
+6.20171018+gitgbb20b1ed3-1~ndall+1
+
+### Please provide any additional information below.
+
+[[!format sh """
+
+14883  tree /tmp/test-directory-export -a
+14887  git annex info
+14888  git annex export --to=directory-export
+14889  git annex export
+14890  git annex export --tracking master --to public-s3
+14891  git annex export --tracking master --to directory-export
+14892  git annex export
+14893  git annex export --to directory-export
+14894  git annex export
+14896  git annex sync --content
+14898  git log --stat git-annex
+
+
+$> git lg --stat git-annex | head -n 30
+* 4a03be8 - (git-annex) update (4 minutes ago) [Yaroslav Halchenko]| 
+|  export.log | 2 +-
+|  1 file changed, 1 insertion(+), 1 deletion(-)
+
+* b23ae9d - graft cleanup (4 minutes ago) [Yaroslav Halchenko]| 
+|  export.tree/.datalad/.gitattributes | 3 ---
+|  export.tree/.datalad/config         | 2 --
+|  export.tree/.gitattributes          | 1 -
+|  export.tree/123                     | 1 -
+|  export.tree/sub/dir/11              | 1 -
+|  5 files changed, 8 deletions(-)
+
+* 6a0bc5f - graft (4 minutes ago) [Yaroslav Halchenko]| 
+|  export.tree/.datalad/.gitattributes | 3 +++
+|  export.tree/.datalad/config         | 2 ++
+|  export.tree/.gitattributes          | 1 +
+|  export.tree/123                     | 1 +
+|  export.tree/sub/dir/11              | 1 +
+|  5 files changed, 8 insertions(+)
+
+* 919e345 - update (5 minutes ago) [Yaroslav Halchenko]| 
+|  export.log | 2 +-
+|  1 file changed, 1 insertion(+), 1 deletion(-)
+
+"""]]
+
+[[!meta author=yoh]]
+
+> Seems we're ok, so [[done]] --[[Joey]]
diff --git a/doc/bugs/graft__47__graft_cleanup_commits_--_really_needed__63__/comment_1_7bc271ddeafc92d36d2b90fcb1481891._comment b/doc/bugs/graft__47__graft_cleanup_commits_--_really_needed__63__/comment_1_7bc271ddeafc92d36d2b90fcb1481891._comment
new file mode 100644
index 0000000000..8de4c19158
--- /dev/null
+++ b/doc/bugs/graft__47__graft_cleanup_commits_--_really_needed__63__/comment_1_7bc271ddeafc92d36d2b90fcb1481891._comment
@@ -0,0 +1,21 @@
+[[!comment format=mdwn
+ username="joey"
+ subject="""comment 1"""
+ date="2017-10-25T18:56:49Z"
+ content="""
+They're really needed. Squashing them together would make the tree ref
+that's being grafted in not be reachable, so it would be subject to GC,
+which would lose data that the export feature needs to make sure is
+retained for later.
+
+It would be possible to squash the graft cleanup with the export.log update
+commit. However, I was unable to stage the graft
+into the git-annex branch index file using git update-index, due to
+possibly a bug in git, so I was not able to find a way to do that.
+See [[!commit 5483ea90eca33f61c799fb6a3c2675657caa9c75]]
+
+The diff makes it look somehow big and expensive, but the actual overhead
+on disk is two commit objects. The tree object being grafted in is the
+exported tree, so retaining it does not really add any overhead. So
+something in the area of 2 kb overhead, per export of a tree.
+"""]]
diff --git a/doc/bugs/graft__47__graft_cleanup_commits_--_really_needed__63__/comment_2_d0a7283aac4f3c231e20ab284a1f2253._comment b/doc/bugs/graft__47__graft_cleanup_commits_--_really_needed__63__/comment_2_d0a7283aac4f3c231e20ab284a1f2253._comment
new file mode 100644
index 0000000000..91eb4bcd93
--- /dev/null
+++ b/doc/bugs/graft__47__graft_cleanup_commits_--_really_needed__63__/comment_2_d0a7283aac4f3c231e20ab284a1f2253._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="yarikoptic"
+ avatar="http://cdn.libravatar.org/avatar/f11e9c84cb18d26a1748c33b48c924b4"
+ subject="got it"
+ date="2017-10-30T18:33:20Z"
+ content="""
+Thank you Joey!
+"""]]
diff --git a/doc/bugs/high_cpu_usage_in_cat-file_and_webapp.mdwn b/doc/bugs/high_cpu_usage_in_cat-file_and_webapp.mdwn
new file mode 100644
index 0000000000..d2dd2bbce9
--- /dev/null
+++ b/doc/bugs/high_cpu_usage_in_cat-file_and_webapp.mdwn
@@ -0,0 +1,314 @@
+### Please describe the problem.
+
+I've been running git-annex in the background for a few weeks now (my laptop is at home, occasionally gets suspended, but not for the past few days), and this morning (around 9:30am), I noticed I could hear its fan. Looking at CPU usage in htop, "git-annex webapp" was using ~18% and "git --git-dir=/home/flowblok/annex/.git --work-tree=/home/flowblok/annex -c core.bare=false cat-file --batch" was using ~46%.
+
+The repository is in direct mode.
+It has one remote which it copies to over the network with "annex-rsync-upload-options = --bwlimit 10".
+The last fsck ran at 4am, and finished (with one failure, which I can easily recover from) by 4:30am (I guess I should find out why it didn't notify me in any way).
+
+### What steps will reproduce the problem?
+
+Not a clue!
+Though hopefully I've given enough information below for you to figure out what went wrong: let me know if you need anything else.
+
+### What version of git-annex are you using? On what operating system?
+
+Debian Jessie
+
+$ uname -a
+Linux moash 3.16.0-4-amd64 #1 SMP Debian 3.16.7-ckt9-3~deb8u1 (2015-04-24) x86_64 GNU/Linux
+
+$ git-annex version
+git-annex version: 5.20141125
+build flags: Assistant Webapp Webapp-secure Pairing Testsuite S3 WebDAV Inotify DBus DesktopNotify XMPP DNS Feeds Quvi TDFA CryptoHash
+key/value backends: SHA256E SHA1E SHA512E SHA224E SHA384E SKEIN256E SKEIN512E SHA256 SHA1 SHA512 SHA224 SHA384 SKEIN256 SKEIN512 WORM URL
+remote types: git gcrypt S3 bup directory rsync web webdav tahoe glacier ddar hook external
+
+### Please provide any additional information below.
+
+[[!format sh """
+# strace -p $PIDOF_GIT_CATFILE
+open("/home/flowblok/annex/.git/objects/04/13fe78e4f9b37348b5fd03cbe5eabdbcaec068", O_RDONLY|O_NOATIME) = 3
+fstat(3, {st_mode=S_IFREG|0400, st_size=683, ...}) = 0
+mmap(NULL, 683, PROT_READ, MAP_PRIVATE, 3, 0) = 0x7f9ead5a0000
+close(3)                                = 0
+munmap(0x7f9ead5a0000, 683)             = 0
+open("/home/flowblok/annex/.git/objects/c0/4c0a1d5b43f82c50cf095e42d28d74f0cfc494", O_RDONLY|O_NOATIME) = 3
+fstat(3, {st_mode=S_IFREG|0400, st_size=132, ...}) = 0
+mmap(NULL, 132, PROT_READ, MAP_PRIVATE, 3, 0) = 0x7f9ead5a0000
+close(3)                                = 0
+munmap(0x7f9ead5a0000, 132)             = 0
+brk(0x1b5b000)                          = 0x1b5b000
+open("/home/flowblok/annex/.git/objects/25/21eb645b1fcac7b6b55551114fe2d69e095896", O_RDONLY|O_NOATIME) = 3
+fstat(3, {st_mode=S_IFREG|0400, st_size=70, ...}) = 0
+mmap(NULL, 70, PROT_READ, MAP_PRIVATE, 3, 0) = 0x7f9ead5a0000
+close(3)                                = 0
+munmap(0x7f9ead5a0000, 70)              = 0
+write(1, "2521eb645b1fcac7b6b55551114fe2d6"..., 49) = 49
+open("/home/flowblok/annex/.git/objects/25/21eb645b1fcac7b6b55551114fe2d69e095896", O_RDONLY|O_NOATIME) = 3
+fstat(3, {st_mode=S_IFREG|0400, st_size=70, ...}) = 0
+mmap(NULL, 70, PROT_READ, MAP_PRIVATE, 3, 0) = 0x7f9ead5a0000
+close(3)                                = 0
+munmap(0x7f9ead5a0000, 70)              = 0
+open("/home/flowblok/annex/.git/objects/25/21eb645b1fcac7b6b55551114fe2d69e095896", O_RDONLY|O_NOATIME) = 3
+fstat(3, {st_mode=S_IFREG|0400, st_size=70, ...}) = 0
+mmap(NULL, 70, PROT_READ, MAP_PRIVATE, 3, 0) = 0x7f9ead5a0000
+close(3)                                = 0
+write(1, "1435997202.486434s 0 5c33e13f-ed"..., 58) = 58
+munmap(0x7f9ead5a0000, 70)              = 0
+write(1, "\n", 1)                       = 1
+read(0, "refs/heads/git-annex:eec/e95/SHA"..., 4096) = 116
+lstat("/home/flowblok/annex/.git/refs/heads/git-annex", {st_mode=S_IFREG|0600, st_size=41, ...}) = 0
+open("/home/flowblok/annex/.git/refs/heads/git-annex", O_RDONLY) = 3
+read(3, "f9d2dc8d54151d01a27c7989a1fc7983"..., 255) = 41
+read(3, "", 214)                        = 0
+close(3)                                = 0
+lstat("/home/flowblok/annex/.git/refs/refs/heads/git-annex", 0x7ffc82e9e580) = -1 ENOENT (No such file or directory)
+stat("/home/flowblok/annex/.git/packed-refs", {st_mode=S_IFREG|0600, st_size=1049, ...}) = 0
+lstat("/home/flowblok/annex/.git/refs/tags/refs/heads/git-annex", 0x7ffc82e9e580) = -1 ENOENT (No such file or directory)
+stat("/home/flowblok/annex/.git/packed-refs", {st_mode=S_IFREG|0600, st_size=1049, ...}) = 0
+lstat("/home/flowblok/annex/.git/refs/heads/refs/heads/git-annex", 0x7ffc82e9e580) = -1 ENOENT (No such file or directory)
+stat("/home/flowblok/annex/.git/packed-refs", {st_mode=S_IFREG|0600, st_size=1049, ...}) = 0
+lstat("/home/flowblok/annex/.git/refs/remotes/refs/heads/git-annex", 0x7ffc82e9e580) = -1 ENOENT (No such file or directory)
+stat("/home/flowblok/annex/.git/packed-refs", {st_mode=S_IFREG|0600, st_size=1049, ...}) = 0
+lstat("/home/flowblok/annex/.git/refs/remotes/refs/heads/git-annex/HEAD", 0x7ffc82e9e580) = -1 ENOENT (No such file or directory)
+stat("/home/flowblok/annex/.git/packed-refs", {st_mode=S_IFREG|0600, st_size=1049, ...}) = 0
+open("/home/flowblok/annex/.git/objects/f9/d2dc8d54151d01a27c7989a1fc798320034a69", O_RDONLY|O_NOATIME) = 3
+fstat(3, {st_mode=S_IFREG|0400, st_size=207, ...}) = 0
+mmap(NULL, 207, PROT_READ, MAP_PRIVATE, 3, 0) = 0x7f9ead5a0000
+close(3)                                = 0
+munmap(0x7f9ead5a0000, 207)             = 0
+open("/home/flowblok/annex/.git/objects/65/dd2889161d7002f1d5f94587ab9bd0746d6f61", O_RDONLY|O_NOATIME) = 3
+fstat(3, {st_mode=S_IFREG|0400, st_size=98827, ...}) = 0
+mmap(NULL, 98827, PROT_READ, MAP_PRIVATE, 3, 0) = 0x7f9ea4c03000
+close(3)                                = 0
+brk(0x1b81000)                          = 0x1b81000
+munmap(0x7f9ea4c03000, 98827)           = 0
+open("/home/flowblok/annex/.git/objects/d7/bc8b27aad64c59effa77eae026778582761fae", O_RDONLY|O_NOATIME) = 3
+fstat(3, {st_mode=S_IFREG|0400, st_size=550, ...}) = 0
+mmap(NULL, 550, PROT_READ, MAP_PRIVATE, 3, 0) = 0x7f9ead5a0000
+close(3)                                = 0
+munmap(0x7f9ead5a0000, 550)             = 0
+open("/home/flowblok/annex/.git/objects/d8/00f4d59828022b9ee556b6f3758832da6bfb71", O_RDONLY|O_NOATIME) = 3
+fstat(3, {st_mode=S_IFREG|0400, st_size=131, ...}) = 0
+mmap(NULL, 131, PROT_READ, MAP_PRIVATE, 3, 0) = 0x7f9ead5a0000
+close(3)                                = 0
+munmap(0x7f9ead5a0000, 131)             = 0
+brk(0x1b5b000)                          = 0x1b5b000
+open("/home/flowblok/annex/.git/objects/ac/9ee326d36283143f9b3b38687870df6a888938", O_RDONLY|O_NOATIME) = 3
+fstat(3, {st_mode=S_IFREG|0400, st_size=71, ...}) = 0
+mmap(NULL, 71, PROT_READ, MAP_PRIVATE, 3, 0) = 0x7f9ead5a0000
+close(3)                                = 0
+munmap(0x7f9ead5a0000, 71)              = 0
+write(1, "ac9ee326d36283143f9b3b38687870df"..., 49) = 49
+open("/home/flowblok/annex/.git/objects/ac/9ee326d36283143f9b3b38687870df6a888938", O_RDONLY|O_NOATIME) = 3
+fstat(3, {st_mode=S_IFREG|0400, st_size=71, ...}) = 0
+mmap(NULL, 71, PROT_READ, MAP_PRIVATE, 3, 0) = 0x7f9ead5a0000
+close(3)                                = 0
+munmap(0x7f9ead5a0000, 71)              = 0
+open("/home/flowblok/annex/.git/objects/ac/9ee326d36283143f9b3b38687870df6a888938", O_RDONLY|O_NOATIME) = 3
+fstat(3, {st_mode=S_IFREG|0400, st_size=71, ...}) = 0
+mmap(NULL, 71, PROT_READ, MAP_PRIVATE, 3, 0) = 0x7f9ead5a0000
+close(3)                                = 0
+write(1, "1435997292.091244s 0 5c33e13f-ed"..., 58) = 58
+munmap(0x7f9ead5a0000, 71)              = 0
+write(1, "\n", 1)                       = 1
+read(0, "refs/heads/git-annex:c54/719/SHA"..., 4096) = 116
+lstat("/home/flowblok/annex/.git/refs/heads/git-annex", {st_mode=S_IFREG|0600, st_size=41, ...}) = 0
+open("/home/flowblok/annex/.git/refs/heads/git-annex", O_RDONLY) = 3
+read(3, "f9d2dc8d54151d01a27c7989a1fc7983"..., 255) = 41
+read(3, "", 214)                        = 0
+close(3)                                = 0
+lstat("/home/flowblok/annex/.git/refs/refs/heads/git-annex", 0x7ffc82e9e580) = -1 ENOENT (No such file or directory)
+stat("/home/flowblok/annex/.git/packed-refs", {st_mode=S_IFREG|0600, st_size=1049, ...}) = 0
+lstat("/home/flowblok/annex/.git/refs/tags/refs/heads/git-annex", 0x7ffc82e9e580) = -1 ENOENT (No such file or directory)
+stat("/home/flowblok/annex/.git/packed-refs", {st_mode=S_IFREG|0600, st_size=1049, ...}) = 0
+lstat("/home/flowblok/annex/.git/refs/heads/refs/heads/git-annex", 0x7ffc82e9e580) = -1 ENOENT (No such file or directory)
+stat("/home/flowblok/annex/.git/packed-refs", {st_mode=S_IFREG|0600, st_size=1049, ...}) = 0
+lstat("/home/flowblok/annex/.git/refs/remotes/refs/heads/git-annex", 0x7ffc82e9e580) = -1 ENOENT (No such file or directory)
+stat("/home/flowblok/annex/.git/packed-refs", {st_mode=S_IFREG|0600, st_size=1049, ...}) = 0
+lstat("/home/flowblok/annex/.git/refs/remotes/refs/heads/git-annex/HEAD", 0x7ffc82e9e580) = -1 ENOENT (No such file or directory)
+stat("/home/flowblok/annex/.git/packed-refs", {st_mode=S_IFREG|0600, st_size=1049, ...}) = 0
+open("/home/flowblok/annex/.git/objects/f9/d2dc8d54151d01a27c7989a1fc798320034a69", O_RDONLY|O_NOATIME) = 3
+fstat(3, {st_mode=S_IFREG|0400, st_size=207, ...}) = 0
+mmap(NULL, 207, PROT_READ, MAP_PRIVATE, 3, 0) = 0x7f9ead5a0000
+close(3)                                = 0
+munmap(0x7f9ead5a0000, 207)             = 0
+open("/home/flowblok/annex/.git/objects/65/dd2889161d7002f1d5f94587ab9bd0746d6f61", O_RDONLY|O_NOATIME) = 3
+fstat(3, {st_mode=S_IFREG|0400, st_size=98827, ...}) = 0
+mmap(NULL, 98827, PROT_READ, MAP_PRIVATE, 3, 0) = 0x7f9ea4c03000
+close(3)                                = 0
+brk(0x1b81000)                          = 0x1b81000
+munmap(0x7f9ea4c03000, 98827)           = 0
+open("/home/flowblok/annex/.git/objects/04/13fe78e4f9b37348b5fd03cbe5eabdbcaec068", O_RDONLY|O_NOATIME) = 3
+fstat(3, {st_mode=S_IFREG|0400, st_size=683, ...}) = 0
+mmap(NULL, 683, PROT_READ, MAP_PRIVATE, 3, 0) = 0x7f9ead5a0000
+close(3)                                = 0
+munmap(0x7f9ead5a0000, 683)             = 0
+open("/home/flowblok/annex/.git/objects/c0/4c0a1d5b43f82c50cf095e42d28d74f0cfc494", O_RDONLY|O_NOATIME) = 3
+fstat(3, {st_mode=S_IFREG|0400, st_size=132, ...}) = 0
+mmap(NULL, 132, PROT_READ, MAP_PRIVATE, 3, 0) = 0x7f9ead5a0000
+close(3)                                = 0
+munmap(0x7f9ead5a0000, 132)             = 0
+brk(0x1b5b000)                          = 0x1b5b000
+open("/home/flowblok/annex/.git/objects/25/21eb645b1fcac7b6b55551114fe2d69e095896", O_RDONLY|O_NOATIME) = 3
+fstat(3, {st_mode=S_IFREG|0400, st_size=70, ...}) = 0
+mmap(NULL, 70, PROT_READ, MAP_PRIVATE, 3, 0) = 0x7f9ead5a0000
+close(3)                                = 0
+munmap(0x7f9ead5a0000, 70)              = 0
+write(1, "2521eb645b1fcac7b6b55551114fe2d6"..., 49) = 49
+open("/home/flowblok/annex/.git/objects/25/21eb645b1fcac7b6b55551114fe2d69e095896", O_RDONLY|O_NOATIME) = 3
+fstat(3, {st_mode=S_IFREG|0400, st_size=70, ...}) = 0
+mmap(NULL, 70, PROT_READ, MAP_PRIVATE, 3, 0) = 0x7f9ead5a0000
+close(3)                                = 0
+munmap(0x7f9ead5a0000, 70)              = 0
+open("/home/flowblok/annex/.git/objects/25/21eb645b1fcac7b6b55551114fe2d69e095896", O_RDONLY|O_NOATIME) = 3
+fstat(3, {st_mode=S_IFREG|0400, st_size=70, ...}) = 0
+mmap(NULL, 70, PROT_READ, MAP_PRIVATE, 3, 0) = 0x7f9ead5a0000
+close(3)                                = 0
+write(1, "1435997202.486434s 0 5c33e13f-ed"..., 58) = 58
+munmap(0x7f9ead5a0000, 70)              = 0
+write(1, "\n", 1)                       = 1
+read(0, "refs/heads/git-annex:eec/e95/SHA"..., 4096) = 116
+lstat("/home/flowblok/annex/.git/refs/heads/git-annex", {st_mode=S_IFREG|0600, st_size=41, ...}) = 0
+open("/home/flowblok/annex/.git/refs/heads/git-annex", O_RDONLY) = 3
+read(3, "f9d2dc8d54151d01a27c7989a1fc7983"..., 255) = 41
+read(3, "", 214)                        = 0
+close(3)                                = 0
+lstat("/home/flowblok/annex/.git/refs/refs/heads/git-annex", 0x7ffc82e9e580) = -1 ENOENT (No such file or directory)
+stat("/home/flowblok/annex/.git/packed-refs", {st_mode=S_IFREG|0600, st_size=1049, ...}) = 0
+lstat("/home/flowblok/annex/.git/refs/tags/refs/heads/git-annex", 0x7ffc82e9e580) = -1 ENOENT (No such file or directory)
+stat("/home/flowblok/annex/.git/packed-refs", {st_mode=S_IFREG|0600, st_size=1049, ...}) = 0
+lstat("/home/flowblok/annex/.git/refs/heads/refs/heads/git-annex", 0x7ffc82e9e580) = -1 ENOENT (No such file or directory)
+stat("/home/flowblok/annex/.git/packed-refs", {st_mode=S_IFREG|0600, st_size=1049, ...}) = 0
+lstat("/home/flowblok/annex/.git/refs/remotes/refs/heads/git-annex", 0x7ffc82e9e580) = -1 ENOENT (No such file or directory)
+stat("/home/flowblok/annex/.git/packed-refs", {st_mode=S_IFREG|0600, st_size=1049, ...}) = 0
+lstat("/home/flowblok/annex/.git/refs/remotes/refs/heads/git-annex/HEAD", 0x7ffc82e9e580) = -1 ENOENT (No such file or directory)
+stat("/home/flowblok/annex/.git/packed-refs", {st_mode=S_IFREG|0600, st_size=1049, ...}) = 0
+open("/home/flowblok/annex/.git/objects/f9/d2dc8d54151d01a27c7989a1fc798320034a69", O_RDONLY|O_NOATIME) = 3
+fstat(3, {st_mode=S_IFREG|0400, st_size=207, ...}) = 0
+mmap(NULL, 207, PROT_READ, MAP_PRIVATE, 3, 0) = 0x7f9ead5a0000
+close(3)                                = 0
+munmap(0x7f9ead5a0000, 207)             = 0
+open("/home/flowblok/annex/.git/objects/65/dd2889161d7002f1d5f94587ab9bd0746d6f61", O_RDONLY|O_NOATIME) = 3
+fstat(3, {st_mode=S_IFREG|0400, st_size=98827, ...}) = 0
+mmap(NULL, 98827, PROT_READ, MAP_PRIVATE, 3, 0) = 0x7f9ea4c03000
+close(3)                                = 0
+brk(0x1b81000)                          = 0x1b81000
+munmap(0x7f9ea4c03000, 98827)           = 0
+open("/home/flowblok/annex/.git/objects/d7/bc8b27aad64c59effa77eae026778582761fae", O_RDONLY|O_NOATIME) = 3
+fstat(3, {st_mode=S_IFREG|0400, st_size=550, ...}) = 0
+mmap(NULL, 550, PROT_READ, MAP_PRIVATE, 3, 0) = 0x7f9ead5a0000
+close(3)                                = 0
+munmap(0x7f9ead5a0000, 550)             = 0
+open("/home/flowblok/annex/.git/objects/d8/00f4d59828022b9ee556b6f3758832da6bfb71", O_RDONLY|O_NOATIME) = 3
+fstat(3, {st_mode=S_IFREG|0400, st_size=131, ...}) = 0
+mmap(NULL, 131, PROT_READ, MAP_PRIVATE, 3, 0) = 0x7f9ead5a0000
+close(3)                                = 0
+munmap(0x7f9ead5a0000, 131)             = 0
+brk(0x1b5b000)                          = 0x1b5b000
+open("/home/flowblok/annex/.git/objects/ac/9ee326d36283143f9b3b38687870df6a888938", O_RDONLY|O_NOATIME) = 3
+fstat(3, {st_mode=S_IFREG|0400, st_size=71, ...}) = 0
+mmap(NULL, 71, PROT_READ, MAP_PRIVATE, 3, 0) = 0x7f9ead5a0000
+close(3)                                = 0
+munmap(0x7f9ead5a0000, 71)              = 0
+write(1, "ac9ee326d36283143f9b3b38687870df"..., 49) = 49
+open("/home/flowblok/annex/.git/objects/ac/9ee326d36283143f9b3b38687870df6a888938", O_RDONLY|O_NOATIME) = 3
+fstat(3, {st_mode=S_IFREG|0400, st_size=71, ...}) = 0
+mmap(NULL, 71, PROT_READ, MAP_PRIVATE, 3, 0) = 0x7f9ead5a0000
+close(3)                                = 0
+munmap(0x7f9ead5a0000, 71)              = 0
+open("/home/flowblok/annex/.git/objects/ac/9ee326d36283143f9b3b38687870df6a888938", O_RDONLY|O_NOATIME) = 3
+fstat(3, {st_mode=S_IFREG|0400, st_size=71, ...}) = 0
+mmap(NULL, 71, PROT_READ, MAP_PRIVATE, 3, 0) = 0x7f9ead5a0000
+close(3)                                = 0
+write(1, "1435997292.091244s 0 5c33e13f-ed"..., 58) = 58
+munmap(0x7f9ead5a0000, 71)              = 0
+write(1, "\n", 1)                       = 1
+read(0, "refs/heads/git-annex:c54/719/SHA"..., 4096) = 116
+lstat("/home/flowblok/annex/.git/refs/heads/git-annex", {st_mode=S_IFREG|0600, st_size=41, ...}) = 0
+open("/home/flowblok/annex/.git/refs/heads/git-annex", O_RDONLY) = 3
+read(3, "f9d2dc8d54151d01a27c7989a1fc7983"..., 255) = 41
+read(3, "", 214)                        = 0
+close(3)                                = 0
+lstat("/home/flowblok/annex/.git/refs/refs/heads/git-annex", 0x7ffc82e9e580) = -1 ENOENT (No such file or directory)
+stat("/home/flowblok/annex/.git/packed-refs", {st_mode=S_IFREG|0600, st_size=1049, ...}) = 0
+lstat("/home/flowblok/annex/.git/refs/tags/refs/heads/git-annex", 0x7ffc82e9e580) = -1 ENOENT (No such file or directory)
+stat("/home/flowblok/annex/.git/packed-refs", {st_mode=S_IFREG|0600, st_size=1049, ...}) = 0
+lstat("/home/flowblok/annex/.git/refs/heads/refs/heads/git-annex", 0x7ffc82e9e580) = -1 ENOENT (No such file or directory)
+stat("/home/flowblok/annex/.git/packed-refs", {st_mode=S_IFREG|0600, st_size=1049, ...}) = 0
+lstat("/home/flowblok/annex/.git/refs/remotes/refs/heads/git-annex", 0x7ffc82e9e580) = -1 ENOENT (No such file or directory)
+stat("/home/flowblok/annex/.git/packed-refs", {st_mode=S_IFREG|0600, st_size=1049, ...}) = 0
+lstat("/home/flowblok/annex/.git/refs/remotes/refs/heads/git-annex/HEAD", 0x7ffc82e9e580) = -1 ENOENT (No such file or directory)
+stat("/home/flowblok/annex/.git/packed-refs", {st_mode=S_IFREG|0600, st_size=1049, ...}) = 0
+open("/home/flowblok/annex/.git/objects/f9/d2dc8d54151d01a27c7989a1fc798320034a69", O_RDONLY|O_NOATIME) = 3
+fstat(3, {st_mode=S_IFREG|0400, st_size=207, ...}) = 0
+mmap(NULL, 207, PROT_READ, MAP_PRIVATE, 3, 0) = 0x7f9ead5a0000
+close(3)                                = 0
+munmap(0x7f9ead5a0000, 207)             = 0
+open("/home/flowblok/annex/.git/objects/65/dd2889161d7002f1d5f94587ab9bd0746d6f61", O_RDONLY|O_NOATIME) = 3
+fstat(3, {st_mode=S_IFREG|0400, st_size=98827, ...}) = 0
+mmap(NULL, 98827, PROT_READ, MAP_PRIVATE, 3, 0) = 0x7f9ea4c03000
+close(3)                                = 0
+brk(0x1b81000)                          = 0x1b81000
+munmap(0x7f9ea4c03000, 98827)           = 0
+# (truncated, since it just repeats over and over for a total of 613759 lines)
+
+# strace -p $PIDOF_GITANNEX_WEBAPP
+futex(0x417c1dc, FUTEX_WAIT_PRIVATE, 65, NULL) = ? ERESTARTSYS (To be restarted if SA_RESTART is set)
+--- SIGVTALRM {si_signo=SIGVTALRM, si_code=SI_TIMER, si_timerid=0, si_overrun=0, si_value={int=0, ptr=0}} ---
+rt_sigreturn()                          = 202
+futex(0x417c1dc, FUTEX_WAIT_PRIVATE, 65, NULL) = ? ERESTARTSYS (To be restarted if SA_RESTART is set)
+--- SIGVTALRM {si_signo=SIGVTALRM, si_code=SI_TIMER, si_timerid=0, si_overrun=0, si_value={int=0, ptr=0}} ---
+rt_sigreturn()                          = 202
+futex(0x417c1dc, FUTEX_WAIT_PRIVATE, 65, NULL) = ? ERESTARTSYS (To be restarted if SA_RESTART is set)
+--- SIGVTALRM {si_signo=SIGVTALRM, si_code=SI_TIMER, si_timerid=0, si_overrun=0, si_value={int=0, ptr=0}} ---
+rt_sigreturn()                          = 202
+futex(0x417c1dc, FUTEX_WAIT_PRIVATE, 65, NULL) = ? ERESTARTSYS (To be restarted if SA_RESTART is set)
+--- SIGVTALRM {si_signo=SIGVTALRM, si_code=SI_TIMER, si_timerid=0, si_overrun=0, si_value={int=0, ptr=0}} ---
+rt_sigreturn()                          = 202
+futex(0x417c1dc, FUTEX_WAIT_PRIVATE, 65, NULL) = ? ERESTARTSYS (To be restarted if SA_RESTART is set)
+--- SIGVTALRM {si_signo=SIGVTALRM, si_code=SI_TIMER, si_timerid=0, si_overrun=0, si_value={int=0, ptr=0}} ---
+rt_sigreturn()                          = 202
+futex(0x417c1dc, FUTEX_WAIT_PRIVATE, 65, NULL) = ? ERESTARTSYS (To be restarted if SA_RESTART is set)
+--- SIGVTALRM {si_signo=SIGVTALRM, si_code=SI_TIMER, si_timerid=0, si_overrun=0, si_value={int=0, ptr=0}} ---
+rt_sigreturn()                          = 202
+futex(0x417c1dc, FUTEX_WAIT_PRIVATE, 65, NULL) = ? ERESTARTSYS (To be restarted if SA_RESTART is set)
+--- SIGVTALRM {si_signo=SIGVTALRM, si_code=SI_TIMER, si_timerid=0, si_overrun=0, si_value={int=0, ptr=0}} ---
+rt_sigreturn()                          = 202
+futex(0x417c1dc, FUTEX_WAIT_PRIVATE, 65, NULL) = ? ERESTARTSYS (To be restarted if SA_RESTART is set)
+--- SIGVTALRM {si_signo=SIGVTALRM, si_code=SI_TIMER, si_timerid=0, si_overrun=0, si_value={int=0, ptr=0}} ---
+rt_sigreturn()                          = 202
+futex(0x417c1dc, FUTEX_WAIT_PRIVATE, 65, NULL) = ? ERESTARTSYS (To be restarted if SA_RESTART is set)
+--- SIGVTALRM {si_signo=SIGVTALRM, si_code=SI_TIMER, si_timerid=0, si_overrun=0, si_value={int=0, ptr=0}} ---
+rt_sigreturn()                          = 202
+futex(0x417c1dc, FUTEX_WAIT_PRIVATE, 65, NULL) = ? ERESTARTSYS (To be restarted if SA_RESTART is set)
+--- SIGVTALRM {si_signo=SIGVTALRM, si_code=SI_TIMER, si_timerid=0, si_overrun=0, si_value={int=0, ptr=0}} ---
+rt_sigreturn()                          = 202
+futex(0x417c1dc, FUTEX_WAIT_PRIVATE, 65, NULL) = ? ERESTARTSYS (To be restarted if SA_RESTART is set)
+futex(0x417c1dc, FUTEX_WAIT_PRIVATE, 65, NULL) = ? ERESTARTSYS (To be restarted if SA_RESTART is set)
+--- SIGVTALRM {si_signo=SIGVTALRM, si_code=SI_TIMER, si_timerid=0, si_overrun=0, si_value={int=0, ptr=0}} ---
+rt_sigreturn()                          = 202
+futex(0x417c1dc, FUTEX_WAIT_PRIVATE, 65, NULL) = ? ERESTARTSYS (To be restarted if SA_RESTART is set)
+--- SIGVTALRM {si_signo=SIGVTALRM, si_code=SI_TIMER, si_timerid=0, si_overrun=0, si_value={int=0, ptr=0}} ---
+rt_sigreturn()                          = 202
+futex(0x417c1dc, FUTEX_WAIT_PRIVATE, 65, NULL) = ? ERESTARTSYS (To be restarted if SA_RESTART is set)
+--- SIGVTALRM {si_signo=SIGVTALRM, si_code=SI_TIMER, si_timerid=0, si_overrun=0, si_value={int=0, ptr=0}} ---
+rt_sigreturn()                          = 202
+futex(0x417c1dc, FUTEX_WAIT_PRIVATE, 65, NULL) = ? ERESTARTSYS (To be restarted if SA_RESTART is set)
+--- SIGVTALRM {si_signo=SIGVTALRM, si_code=SI_TIMER, si_timerid=0, si_overrun=0, si_value={int=0, ptr=0}} ---
+rt_sigreturn()                          = 202
+futex(0x417c1dc, FUTEX_WAIT_PRIVATE, 65, NULL) = ? ERESTARTSYS (To be restarted if SA_RESTART is set)
+--- SIGVTALRM {si_signo=SIGVTALRM, si_code=SI_TIMER, si_timerid=0, si_overrun=0, si_value={int=0, ptr=0}} ---
+rt_sigreturn()                          = 202
+futex(0x417c1dc, FUTEX_WAIT_PRIVATE, 65, NULL) = ? ERESTARTSYS (To be restarted if SA_RESTART is set)
+--- SIGVTALRM {si_signo=SIGVTALRM, si_code=SI_TIMER, si_timerid=0, si_overrun=0, si_value={int=0, ptr=0}} ---
+rt_sigreturn()                          = 202
+futex(0x417c1dc, FUTEX_WAIT_PRIVATE, 65, NULL) = ? ERESTARTSYS (To be restarted if SA_RESTART is set)
+--- SIGVTALRM {si_signo=SIGVTALRM, si_code=SI_TIMER, si_timerid=0, si_overrun=0, si_value={int=0, ptr=0}} ---
+rt_sigreturn()                          = 202
+futex(0x417c1dc, FUTEX_WAIT_PRIVATE, 65, NULL) = ? ERESTARTSYS (To be restarted if SA_RESTART is set)
+--- SIGVTALRM {si_signo=SIGVTALRM, si_code=SI_TIMER, si_timerid=0, si_overrun=0, si_value={int=0, ptr=0}} ---
+rt_sigreturn()                          = 202
+# (ands lots more of the same)
+"""]]
diff --git a/doc/bugs/high_cpu_usage_in_cat-file_and_webapp/comment_1_7fee933eb54272a0447851279d31825b._comment b/doc/bugs/high_cpu_usage_in_cat-file_and_webapp/comment_1_7fee933eb54272a0447851279d31825b._comment
new file mode 100644
index 0000000000..d926d72d25
--- /dev/null
+++ b/doc/bugs/high_cpu_usage_in_cat-file_and_webapp/comment_1_7fee933eb54272a0447851279d31825b._comment
@@ -0,0 +1,16 @@
+[[!comment format=mdwn
+ username="joey"
+ subject="""comment 1"""
+ date="2015-07-16T17:36:07Z"
+ content="""
+The most likely reason for the assistant to use CPU like this is when it's running
+the expensive transfer scan to find files that need to be copied from/to
+remotes. That has to consider every file in your repository's work
+tree, and uses git cat-file to extract information from the git
+repository to see where the file is currently located. This is normally
+only done when the assistant starts up.
+
+.git/annex/daemon.log will show when the expensive transfer scan starts and
+finishes. Look for "starting scan" and "finished scan" lines. Or, open the
+webapp and look to see what activities it says it's performing.
+"""]]
diff --git a/doc/bugs/high_cpu_usage_in_cat-file_and_webapp/comment_2_f26b8846e1651c507d259b9891d26136._comment b/doc/bugs/high_cpu_usage_in_cat-file_and_webapp/comment_2_f26b8846e1651c507d259b9891d26136._comment
new file mode 100644
index 0000000000..f650709e58
--- /dev/null
+++ b/doc/bugs/high_cpu_usage_in_cat-file_and_webapp/comment_2_f26b8846e1651c507d259b9891d26136._comment
@@ -0,0 +1,21 @@
+[[!comment format=mdwn
+ username="jgoerzen"
+ subject="Still seeing this"
+ date="2016-06-08T18:03:51Z"
+ content="""
+This is on jessie with 5.20151208-1~bpo8.
+
+git-annex webapp and git --git-dir=.git --work-tree=. --literal-pathspecs -c core.bare=false cat-file --batch are using considerable CPU time.  The strace appears to be this:
+
+[[!format text \"\"\"
+[pid 29222] open(\".git/annex/journal/1e9_9e8_SHA256E-s109439--4df4c11870535f09bea79e309ac1d3de0e3ed5054900257c212277c0dc84290e.odt.log\", O_RDONLY|O_NOCTTY|O_NONBLOCK) = -1 ENOENT (No such file or directory)
+[pid 29222] write(15, \"refs/heads/git-annex:1e9/9e8/SHA\"..., 119) = 119
+[pid 29222] write(7, \"\1\0\0\0\0\0\0\0\", 8) = 8
+[pid 29161] read(7, \"\1\0\0\0\0\0\0\0\", 8) = 8
+[pid 29222] read(18, \"fb6b750de51cf247dae90279745c2478\"..., 8096) = 341
+\"\"\"]]
+
+What's more, the webapp shows that it's only synced to one of the two remotes, despite both being available.  The log has nothing about kicking off a sync run.
+
+I also wonder if this is and [[bugs/Assistant_having_a_child_git_cat-file_--batch_do_the_same_thing_over_and_over_and_using_a_lot_of_memory]] are the same issue.
+"""]]
diff --git a/doc/bugs/high_cpu_usage_in_cat-file_and_webapp/comment_3_89dbc16c9c1b74ccb3f46d3c558e0c8d._comment b/doc/bugs/high_cpu_usage_in_cat-file_and_webapp/comment_3_89dbc16c9c1b74ccb3f46d3c558e0c8d._comment
new file mode 100644
index 0000000000..46f8f4cbe1
--- /dev/null
+++ b/doc/bugs/high_cpu_usage_in_cat-file_and_webapp/comment_3_89dbc16c9c1b74ccb3f46d3c558e0c8d._comment
@@ -0,0 +1,7 @@
+[[!comment format=mdwn
+ username="jgoerzen"
+ subject="Seeing this everywhere"
+ date="2016-06-10T16:59:46Z"
+ content="""
+I am now seeing this pattern on every machine I use.
+"""]]
diff --git a/doc/bugs/how_to_copy_tp_multiple_remotes_simultaneously__63__.mdwn b/doc/bugs/how_to_copy_tp_multiple_remotes_simultaneously__63__.mdwn
new file mode 100644
index 0000000000..8727fd9d14
--- /dev/null
+++ b/doc/bugs/how_to_copy_tp_multiple_remotes_simultaneously__63__.mdwn
@@ -0,0 +1,42 @@
+### Please describe the problem.
+
+I would like to make multiple copies of precious data and I would like these copies to run simultaneously to minimise the elapsed time taken.  Ideally, I would like to specify multiple `--to` options to `git annex copy`.  I have tried running multiple processes simultaneously but these start to fail with lock contention on `.git/config`.
+
+### What steps will reproduce the problem?
+
+Running four simultaneous `git annex copy --to=` commands.
+
+### What version of git-annex are you using? On what operating system?
+
+OS is debian jessie, using `git-annex-standalone` from NeuroDebian.
+
+```
+git-annex version: 6.20160923+gitgd1dabb3-1~ndall+1
+build flags: Assistant Webapp Pairing Testsuite S3(multipartupload)(storageclasses) WebDAV Inotify DBus DesktopNotif
+y XMPP ConcurrentOutput TorrentParser MagicMime Feeds Quvi                                                         
+key/value backends: SHA256E SHA256 SHA512E SHA512 SHA224E SHA224 SHA384E SHA384 SHA3_256E SHA3_256 SHA3_512E SHA3_51
+2 SHA3_224E SHA3_224 SHA3_384E SHA3_384 SKEIN256E SKEIN256 SKEIN512E SKEIN512 SHA1E SHA1 MD5E MD5 WORM URL         
+remote types: git gcrypt S3 bup directory rsync web bittorrent webdav tahoe glacier ddar hook external
+local repository version: 6
+supported repository versions: 5 6
+upgrade supported from repository versions: 0 1 2 3 4 5
+operating system: linux x86_64
+```
+
+### Please provide any additional information below.
+
+What I have is a python script and its output which I can't imagine is that helpful, but if you want it I'll upload it.
+
+[[!format sh """
+# If you can, paste a complete transcript of the problem occurring here.
+# If the problem is with the git-annex assistant, paste in .git/annex/daemon.log
+
+
+# End of transcript or log.
+"""]]
+
+### Have you had any luck using git-annex before? (Sometimes we get tired of reading bug reports all day and a lil' positive end note does wonders)
+
+Usng it regularly and like it - many thanks.  This is a wishlist item, not a bug, as far as I'm concerned.
+
+
diff --git a/doc/bugs/how_to_copy_tp_multiple_remotes_simultaneously__63__/comment_1_311a8a43d360074abd522fff14bfcb45._comment b/doc/bugs/how_to_copy_tp_multiple_remotes_simultaneously__63__/comment_1_311a8a43d360074abd522fff14bfcb45._comment
new file mode 100644
index 0000000000..9661ac5f1f
--- /dev/null
+++ b/doc/bugs/how_to_copy_tp_multiple_remotes_simultaneously__63__/comment_1_311a8a43d360074abd522fff14bfcb45._comment
@@ -0,0 +1,15 @@
+[[!comment format=mdwn
+ username="joey"
+ subject="""comment 1"""
+ date="2016-10-26T18:02:43Z"
+ content="""
+You need to provide more detail about "lock contention on `.git/config`".
+Ideally an actual error message.
+
+Normally it's perfectly fine to run multiple `git annex copy`, or any
+other git-annex command for that matter.
+
+And the only thing I know of that locks .git/config is when the
+configuration is being changed, which doesn't normally happen when
+copying to a remote, unless perhaps this remote has never been used before.
+"""]]
diff --git a/doc/bugs/howto_guarantee_a_single_instance_of_a_special_remote__63__.mdwn b/doc/bugs/howto_guarantee_a_single_instance_of_a_special_remote__63__.mdwn
new file mode 100644
index 0000000000..489bae2415
--- /dev/null
+++ b/doc/bugs/howto_guarantee_a_single_instance_of_a_special_remote__63__.mdwn
@@ -0,0 +1,37 @@
+### Please describe the problem.
+
+Sorry if it doesn't really fit into the "bug" category, although may be I could have titled it "does not consider another instance of the same special remote in case one fails to initialize in parallel (-J) mode" may be?
+
+### What steps will reproduce the problem?
+
+In datalad-archives we probably should restrict to having just a single instance of that special remote running to avoid messing with tracking of which instance is taking care about which archive and stage (download, extraction, etc).  So I thought I would just implement that by assuring that there is a single instance of the special remote running per repository and all others just reporting 'PREPARE-FAILURE' and then git-annex would try another instance it has in the pool.
+
+But it seems to be not the case:
+
+
+[[!format sh """
+[2018-03-05 14:23:06.725630284] chat: /home/yoh/proj/datalad/datalad/venvs/dev/bin/git-annex-remote-datalad-archives []
+[2018-03-05 14:23:07.019016095] git-annex-remote-datalad-archives[2] --> VERSION 1
+[2018-03-05 14:23:07.019267772] git-annex-remote-datalad-archives[2] <-- EXTENSIONS INFO
+[2018-03-05 14:23:07.019461422] git-annex-remote-datalad-archives[2] --> UNSUPPORTED-REQUEST
+[2018-03-05 14:23:07.019596061] git-annex-remote-datalad-archives[2] <-- PREPARE
+[2018-03-05 14:23:07.019643377] git-annex-remote-datalad-archives[1] --> VERSION 1
+[2018-03-05 14:23:07.01973844] git-annex-remote-datalad-archives[1] <-- EXTENSIONS INFO
+[2018-03-05 14:23:07.019888855] git-annex-remote-datalad-archives[1] --> UNSUPPORTED-REQUEST
+[2018-03-05 14:23:07.019917528] git-annex-remote-datalad-archives[2] --> PREPARE-SUCCESS
+[2018-03-05 14:23:07.019972279] git-annex-remote-datalad-archives[1] <-- PREPARE
+[2018-03-05 14:23:07.02001778] git-annex-remote-datalad-archives[2] <-- TRANSFER RETRIEVE MD5E-s5238--ead47341c9363e7f49c1a50c895d170b.txt .git/annex/tmp/MD5E-s5238--ead47341c9363e7f49c1a50c895d170b.txt
+[2018-03-05 14:23:0detailed-methods.txt (from datalad-archives...) ] --> PREPARE-FAILURE Failed to prepare  due to Cannot lock repo /home/yoh/datalad__/crcn
+get docs/crcns-aa1-conditions.txt (from datalad-archives...) failed
+get docs/crcns-aa1-detailed-methods.txt (from datalad-archives...) 
+"""]]
+
+if one instance fails -- annex immediately reports that the url failed to download and doesn't try another instance it had to be ran in parallel.
+
+I wondered if there is an easy way to restrict a single instance for some special remotes (may be with recently added "FEATURES" to describe special remotes) or adjust parallel download logic to loop through available (not failed) instances thus reusing only one instance if all others fail?
+
+### What version of git-annex are you using? On what operating system?
+6.20180220+gitg811d0d313-1~ndall+1
+
+[[!meta author=yoh]]
+
diff --git a/doc/bugs/howto_guarantee_a_single_instance_of_a_special_remote__63__/comment_1_447c9cf5e9e137238fb2c662a54ee2d7._comment b/doc/bugs/howto_guarantee_a_single_instance_of_a_special_remote__63__/comment_1_447c9cf5e9e137238fb2c662a54ee2d7._comment
new file mode 100644
index 0000000000..1d5af193c9
--- /dev/null
+++ b/doc/bugs/howto_guarantee_a_single_instance_of_a_special_remote__63__/comment_1_447c9cf5e9e137238fb2c662a54ee2d7._comment
@@ -0,0 +1,19 @@
+[[!comment format=mdwn
+ username="joey"
+ subject="""comment 1"""
+ date="2018-03-06T16:08:23Z"
+ content="""
+When I try this with ssh remotes, which should act the same as external
+special remotes as far as git-annex get behavior is concerned,
+git-annex get moves on to the next remote that has the file when the first
+one fails.
+
+I think the difference might come down to the handling of the failed
+PREPARE. That throws an exception, so terminates the get action for that
+file.
+
+Indeed, there are quite a few `giveup` in Remote/External.hs, and while
+some of them are reasonable exceptions to throw, it needs an audit for ones
+that throw an exception when there's a better way to indicate failure like
+return False.
+"""]]
diff --git a/doc/bugs/howto_guarantee_a_single_instance_of_a_special_remote__63__/comment_2_8432a49e6af69efbfcb4d615e147f873._comment b/doc/bugs/howto_guarantee_a_single_instance_of_a_special_remote__63__/comment_2_8432a49e6af69efbfcb4d615e147f873._comment
new file mode 100644
index 0000000000..51c0c38be3
--- /dev/null
+++ b/doc/bugs/howto_guarantee_a_single_instance_of_a_special_remote__63__/comment_2_8432a49e6af69efbfcb4d615e147f873._comment
@@ -0,0 +1,38 @@
+[[!comment format=mdwn
+ username="joey"
+ subject="""comment 2"""
+ date="2018-03-06T16:33:41Z"
+ content="""
+Ok, audited all exceptions thrown in there, 
+and the only other one that stood out is 
+that TRANSFER-FAILURE throws an exception -- but that one
+is ok because it's in a Retriever, all of which exceptions are caught.
+
+But hmm, PREPARE-FAILURE throwing an exception when git-annex is preparing
+before retrieving a key is in the same Retriever, so that exception also
+should not be a problem. There might be other cases where PREPARE-FAILURE
+throwing an exception is not desirable, but this is not one.
+
+---
+
+Oh, I see, the bug report is really about some -J specific issue!
+
+I'm having difficulty reproducing it. I have two external special remotes
+that both send PREPARE-FAILURE, and when a file is present in both,
+`git annex get -J2` does try both of them:
+
+	[2018-03-06 12:45:21.123121045] chat: /home/joey/bin/git-annex-remote-directory []
+	[2018-03-06 12:45:21.125524418] git-annex-remote-directory[1] --> VERSION 1
+	[2018-03-06 12:45:21.125702152] git-annex-remote-directory[1] <-- EXTENSIONS INFO
+	[2018-03-06 12:45:21.126005492] git-annex-remote-directory[1] --> UNSUPPORTED-REQUEST
+	[2018-03-06 12:45:21.126186941] git-annex-remote-directory[1] <-- PREPARE
+	[2018-03-06 12:45:21.126518978] git-annex-remote-directory[1] --> PREPARE_FAILURE OOK
+	[2018-03-06 12:45:21.129247706] chat: /home/joey/bin/git-annex-remote-directory []
+	[2018-03-06 12:45:21.131475605] git-annex-remote-directory[1] --> VERSION 1
+	[2018-03-06 12:45:21.131636898] git-annex-remote-directory[1] <-- EXTENSIONS INFO
+	[2018-03-06 12:45:21.131848068] git-annex-remote-directory[1] --> UNSUPPORTED-REQUEST
+	[2018-03-06 12:45:21.131953342] git-annex-remote-directory[1] <-- PREPARE
+	[2018-03-06 12:45:21.132127177] git-annex-remote-directory[1] --> PREPARE_FAILURE OOK
+	get foo (from d1...) (from d2...)
+	  Unable to access these remotes: d1, d2
+"""]]
diff --git a/doc/bugs/howto_guarantee_a_single_instance_of_a_special_remote__63__/comment_3_f66844b716e48c5befbe99346e940962._comment b/doc/bugs/howto_guarantee_a_single_instance_of_a_special_remote__63__/comment_3_f66844b716e48c5befbe99346e940962._comment
new file mode 100644
index 0000000000..23d34f6331
--- /dev/null
+++ b/doc/bugs/howto_guarantee_a_single_instance_of_a_special_remote__63__/comment_3_f66844b716e48c5befbe99346e940962._comment
@@ -0,0 +1,10 @@
+[[!comment format=mdwn
+ username="yarikoptic"
+ avatar="http://cdn.libravatar.org/avatar/f11e9c84cb18d26a1748c33b48c924b4"
+ subject="comment 3"
+ date="2018-03-06T19:07:37Z"
+ content="""
+re reproduce:  may be it is relevant that in my case it is THE SAME (the same uuid, hardcoded similarly to web \"remote\" uuid) remote.
+
+re locking: have some ugly (almost) working fix -- but yet to troubleshoot why some files still manage to get \"failed\" status (may be it is somehow related to above)
+"""]]
diff --git a/doc/bugs/howto_guarantee_a_single_instance_of_a_special_remote__63__/comment_4_e90ba33f7506745582d18841ae9a9531._comment b/doc/bugs/howto_guarantee_a_single_instance_of_a_special_remote__63__/comment_4_e90ba33f7506745582d18841ae9a9531._comment
new file mode 100644
index 0000000000..5dcc4d0361
--- /dev/null
+++ b/doc/bugs/howto_guarantee_a_single_instance_of_a_special_remote__63__/comment_4_e90ba33f7506745582d18841ae9a9531._comment
@@ -0,0 +1,13 @@
+[[!comment format=mdwn
+ username="joey"
+ subject="""comment 4"""
+ date="2018-04-02T17:13:33Z"
+ content="""
+Ok so I misunderstood this, it's entirely about -J mode and external
+special remote processes. One is started per parallelism level, because
+otherwise nothing much would actually be done in parallel. 
+
+You wanted to start only one because reasons, which IIRC from talking
+with you were in the meantime fixed in your special remore program using
+locking.
+"""]]
diff --git a/doc/bugs/howto_guarantee_a_single_instance_of_a_special_remote__63__/comment_5_098b2e2c51f70d495fe1587220173f55._comment b/doc/bugs/howto_guarantee_a_single_instance_of_a_special_remote__63__/comment_5_098b2e2c51f70d495fe1587220173f55._comment
new file mode 100644
index 0000000000..49afab2c0a
--- /dev/null
+++ b/doc/bugs/howto_guarantee_a_single_instance_of_a_special_remote__63__/comment_5_098b2e2c51f70d495fe1587220173f55._comment
@@ -0,0 +1,30 @@
+[[!comment format=mdwn
+ username="joey"
+ subject="""comment 5"""
+ date="2018-04-03T19:17:32Z"
+ content="""
+PREPARE-FAILURE is documented as "the special remote cannot be used"
+and so I don't think it makes sense for git-annex to use the previously
+started instance of the program if a later one fails like that.
+
+It would need a new response to PREPARE. And some possibly not
+insignificant changes in Remote/External.hs to support it. In particular,
+Remote/External.hs currently delays sending PREPARE until the first time
+it uses a special remote, but this seems to need PREPARE to be sent
+earlier, when it starts up the special remote, so it can detect the new
+response and remember that it should not try to start up any more
+concurrent instances and instead use any already started instance.
+
+The best argument for doing it, I think, is if several different external
+special remote programs really only support a single instance running at a
+time, and if supporting that inside git-annex would be enough of a win,
+rather than making those programs do their own locking.
+
+Hmm, an external  special remote program can't just block its response to
+PREPARE when another instance is running, because it would never be able to
+un-block. So it seems they would have to use finer-grained locking when
+responding to eg TRANSFER. I don't know if anything other than datalad
+needs such locking (and IIUC datalad already got the necessary locking), 
+but it does seem like it would be worth adding an extra PREPARE response
+to avoid needing to so complicate external special remote programs.
+"""]]
diff --git a/doc/bugs/http_git_remote_uuid_discovery_repeated.mdwn b/doc/bugs/http_git_remote_uuid_discovery_repeated.mdwn
new file mode 100644
index 0000000000..7b3aa03c44
--- /dev/null
+++ b/doc/bugs/http_git_remote_uuid_discovery_repeated.mdwn
@@ -0,0 +1,5 @@
+When cloning eg , each time git-annex
+is run, it tries to get the uuid, fails, prints a warning. It should set
+annex-ignore instead, so that only happens once. --[[Joey]]
+
+> [[fixed|done]] --[[Joey]] 
diff --git a/doc/bugs/huge_multiple_copies_of___39__.nfs__42____39___and___39__.panfs__42____39___being_created.mdwn b/doc/bugs/huge_multiple_copies_of___39__.nfs__42____39___and___39__.panfs__42____39___being_created.mdwn
new file mode 100644
index 0000000000..4e62fa7583
--- /dev/null
+++ b/doc/bugs/huge_multiple_copies_of___39__.nfs__42____39___and___39__.panfs__42____39___being_created.mdwn
@@ -0,0 +1,32 @@
+### Please describe the problem.
+
+I have 2 indirect mode repos, both on network filesystems, that I have only used for adding
+data on one end, then syncing via `git annex sync` and `git annex get`.  The problem
+is that`.nfs` copies are being made for each git annex object data file, e.g:
+
+`./.git/annex/objects/34/2x/SHA256E-s4112535690--c5f0e5a8af7bf17dd4a8ca192c8ddfb01fe6ec10908c80cffa5ac64c00e28443.vtk.gz/.nfs0000000006d0018600002147`
+
+Reading up on .nfs files, they are generated when "an open file is removed but is still being accessed".
+
+### What steps will reproduce the problem?
+Clone a git annex repo on a network file system,  run 
+`git annex sync` ,
+`git annex drop` ,
+`git annex get` 
+
+### What version of git-annex are you using? On what operating system?
+* git-annex version: 5.20140818-g10bf03a
+* 2.6.34.9-69.fc13.x86_64 fedora 13
+* 2.6.32-279.22.1.el6.x86_64 centOS
+
+### Please provide any additional information below.
+
+[[!format sh """
+# If you can, paste a complete transcript of the problem occurring here.
+# If the problem is with the git-annex assistant, paste in .git/annex/daemon.log
+
+
+# End of transcript or log.
+"""]]
+
+[[!meta title="git-annex and NFS don't mix; could git annex init detect NFS and refuse to use it?"]]
diff --git a/doc/bugs/huge_multiple_copies_of___39__.nfs__42____39___and___39__.panfs__42____39___being_created/comment_1_833cf2db9881a401a3d5db6d4f3c450a._comment b/doc/bugs/huge_multiple_copies_of___39__.nfs__42____39___and___39__.panfs__42____39___being_created/comment_1_833cf2db9881a401a3d5db6d4f3c450a._comment
new file mode 100644
index 0000000000..430e26f5f0
--- /dev/null
+++ b/doc/bugs/huge_multiple_copies_of___39__.nfs__42____39___and___39__.panfs__42____39___being_created/comment_1_833cf2db9881a401a3d5db6d4f3c450a._comment
@@ -0,0 +1,19 @@
+[[!comment format=mdwn
+ username="joey"
+ subject="""comment 1"""
+ date="2015-02-04T19:31:32Z"
+ content="""
+git-annex uses posix file locking to avoid scenarios that could
+otherwise result in data loss when dropping a file. 
+
+This indeed involves keeping a file open and locked while removing it.
+
+Since NFS's posix file locking support ranges from poor to completely
+broken, it's not wise to use git-annex over NFS at all. It's much nicer
+to use it locally, with a git remote accessing the file server over
+eg, ssh.
+
+I can't see any changes to git-annex that could improve its handing
+of NFS, unless there's a good way to detect a NFS filesystem and make
+`git annex init` refuse to set up a repository there.
+"""]]
diff --git a/doc/bugs/huge_multiple_copies_of___39__.nfs__42____39___and___39__.panfs__42____39___being_created/comment_2_ae3d42be1a15dda2cfe97c99248ceacb._comment b/doc/bugs/huge_multiple_copies_of___39__.nfs__42____39___and___39__.panfs__42____39___being_created/comment_2_ae3d42be1a15dda2cfe97c99248ceacb._comment
new file mode 100644
index 0000000000..c9c5e758ae
--- /dev/null
+++ b/doc/bugs/huge_multiple_copies_of___39__.nfs__42____39___and___39__.panfs__42____39___being_created/comment_2_ae3d42be1a15dda2cfe97c99248ceacb._comment
@@ -0,0 +1,7 @@
+[[!comment format=mdwn
+ username="https://id.koumbit.net/anarcat"
+ subject="comment 2"
+ date="2015-06-10T18:30:21Z"
+ content="""
+i am not sure I understand how this is such a problem... those lockfiles are transient - they will eventually go away. do they take up too much space? or what is the problem exactly?
+"""]]
diff --git a/doc/bugs/huge_multiple_copies_of___39__.nfs__42____39___and___39__.panfs__42____39___being_created/comment_3_bea29c2596ddcc7f8cd7f02ff7bfd9ef._comment b/doc/bugs/huge_multiple_copies_of___39__.nfs__42____39___and___39__.panfs__42____39___being_created/comment_3_bea29c2596ddcc7f8cd7f02ff7bfd9ef._comment
new file mode 100644
index 0000000000..9822058c5a
--- /dev/null
+++ b/doc/bugs/huge_multiple_copies_of___39__.nfs__42____39___and___39__.panfs__42____39___being_created/comment_3_bea29c2596ddcc7f8cd7f02ff7bfd9ef._comment
@@ -0,0 +1,14 @@
+[[!comment format=mdwn
+ username="joey"
+ subject="""comment 3"""
+ date="2015-11-13T19:56:12Z"
+ content="""
+I've just added annex.pidlock to git-annex. Configure that to true, and it
+will use one top-level pid lock file, rather than these posix locks. Might
+work better on NFS.
+
+annex.pidlock is enabled automatically if probing detects that posix locks
+cannot be made at all. However, if the NFS lets posix locks be made, just
+with the poor behavior described here, the probing can't detect that and so
+you'd need to enable it manually.
+"""]]
diff --git a/doc/bugs/if_annex.genmetadata_is_true__44___modification_metadata_is_imported_from_older_file_versions_after_unlock+add.mdwn b/doc/bugs/if_annex.genmetadata_is_true__44___modification_metadata_is_imported_from_older_file_versions_after_unlock+add.mdwn
new file mode 100644
index 0000000000..f8a77f32dc
--- /dev/null
+++ b/doc/bugs/if_annex.genmetadata_is_true__44___modification_metadata_is_imported_from_older_file_versions_after_unlock+add.mdwn
@@ -0,0 +1,23 @@
+### Please describe the problem.
+I've enabled annex.genmetadata to track file modification time. When I unlock, edit and re-add a file, annex imports the modification data from its previous life instead of using the edited (or unedited) file's.
+
+Annex should always use the added file's mtime as modification time, even if there is conflicting pre-existing metadata. Even better would be if it stored modification time per-object, not worktree file.
+
+### What steps will reproduce the problem?
+```git init testannex;
+cd testannex;
+git annex init;
+git config annex.genmetadata true;
+touch bar;
+git annex add bar;
+git annex sync;
+git annex unlock bar;
+vim bar;
+git annex add bar```
+
+output: `Copied metadata from old version of bar to new version. If you don't want this copied metadata, run: git annex metadata --remove-all bar`
+
+### What version of git-annex are you using? On what operating system?
+6.20180227, various Linuxes.
+
+> [[fixed|done]] --[[Joey]]
diff --git a/doc/bugs/if_annex.genmetadata_is_true__44___modification_metadata_is_imported_from_older_file_versions_after_unlock+add/comment_1_b215c8ce0f2094a116672572951b19fc._comment b/doc/bugs/if_annex.genmetadata_is_true__44___modification_metadata_is_imported_from_older_file_versions_after_unlock+add/comment_1_b215c8ce0f2094a116672572951b19fc._comment
new file mode 100644
index 0000000000..d598c7afcb
--- /dev/null
+++ b/doc/bugs/if_annex.genmetadata_is_true__44___modification_metadata_is_imported_from_older_file_versions_after_unlock+add/comment_1_b215c8ce0f2094a116672572951b19fc._comment
@@ -0,0 +1,25 @@
+[[!comment format=mdwn
+ username="joey"
+ subject="""comment 1"""
+ date="2018-04-04T16:51:36Z"
+ content="""
+This happens in Annex.MetaData.genMetaData. First it copies
+metadata from the oldkey to the new key. Then it
+calls addMetaData on the dateMetaData of the file's mtime.
+
+In dateMetaData, there's a `filter isnew`, which makes
+it filter out any of the date fields that already exist
+in the metadata of the new key.
+
+This was done intentionally, see 
+[[!commit 8d5158fa3151be4c7fc698b96ed887b43ac48769]]
+But that's lacking an explanation of why it was done.
+
+Note that dateMetaData is also used in Command.ImportFeed
+to convert a itempubdate into year and month metadata.
+But changing its behavior to override old dates 
+would not change that code path.
+
+So, I don't see a problem with making this change, and have gone ahead and
+done it.
+"""]]
diff --git a/doc/bugs/importfeed_bad_request_without_User-Agent__58__.mdwn b/doc/bugs/importfeed_bad_request_without_User-Agent__58__.mdwn
new file mode 100644
index 0000000000..5c9b31713e
--- /dev/null
+++ b/doc/bugs/importfeed_bad_request_without_User-Agent__58__.mdwn
@@ -0,0 +1,228 @@
+### Please describe the problem.
+
+Since upgrading to `6.20180626-gdf91a5cff` (pre-built OS X binary on OS X 10.11) it appears `git annex importfeed` is unable to properly download a RSS feed from one particular site -- it gets back `400 Bad Request` from the website, and then unsurprisingly reports `warning: bad feed content; no enclosures to download`.  The relevant RSS feed appears sane when downloaded from the same URL with `wget` or `curl`.  The previous `git-annex` version I was using on this machine (before the security update), `6.20170320-g41c5d9d` was able to download this RSS feed (I realise that's a rather large window of changes!).
+
+It appears the older version (`6.20170320-g41c5d9d`) used `wget` to download the podcast RSS, and the newer version (`6.20180626-gdf91a5cff`) seems to fetch the RSS internally in a way that does not work with this RSS feed (nor does the newer version appear to provide debug output of what it is doing at that step that goes wrong, unlike the previous version which shows the explicit `wget` command run, even if multiple `--debug` and/or `--verbose` options are provided).
+
+All other podcast RSS feeds work on both versions.
+
+As best I can tell from packet captures and `telnet` debugging, the issue is that the new `git annex importfeed` does not send a `User-Agent:` header.  Possibly some anti-DoS/anti-spam front end is rejecting it as a result?
+
+Can a suitable `User-Agent:` header be added to the `git annex importfeed` HTTP requests?
+
+### What steps will reproduce the problem?
+
+Attempt to `git annex importfeed` the feed `'https://theythempodcast.com/episodes?format=RSS'`.  The problem also appears reproducible with `'http://theythempodcast.com/episodes?format=RSS'` (ie, unencrypted), which makes debugging a wee bit simpler (ie, packet captures can help).
+
+Failing, on `6.20180626-gdf91a5cff`:
+
+    ewen@ashram:~/Music/podcasts$ git annex --debug importfeed --template="${TEMPLATE}" 'https://theythempodcast.com/episodes?format=RSS' 2>&1 
+    importfeed checking known urls [2018-07-15 13:36:53.298188] read: git ["--git-dir=.git","--work-tree=.","--literal-pathspecs","show-ref","git-annex"]
+    [2018-07-15 13:36:53.306676] process done ExitSuccess
+    [2018-07-15 13:36:53.306768] read: git ["--git-dir=.git","--work-tree=.","--literal-pathspecs","show-ref","--hash","refs/heads/git-annex"]
+    [2018-07-15 13:36:53.314672] process done ExitSuccess
+    [2018-07-15 13:36:53.315353] read: git ["--git-dir=.git","--work-tree=.","--literal-pathspecs","ls-files","--stage","-z","--","."]
+    [2018-07-15 13:36:53.325249] chat: git ["--git-dir=.git","--work-tree=.","--literal-pathspecs","cat-file","--batch"]
+    [2018-07-15 13:36:53.32666] chat: git ["--git-dir=.git","--work-tree=.","--literal-pathspecs","cat-file","--batch-check=%(objectname) %(objecttype) %(objectsize)"]
+    [2018-07-15 13:36:53.768979] process done ExitSuccess
+    ok
+    importfeed https://theythempodcast.com/episodes?format=RSS download failed: 
+    
+    400 Bad Request
+    
+    
+    
+    
    400 Bad Request
    
+    
    e64OsAKG/PqPs52sZ @ Sun, 15 Jul 2018 01:36:57 GMT
    
+    
    SEC-43
    
+    
    
+    
+    
+
+      warning: bad feed content; no enclosures to download
+    ok
+    [2018-07-15 13:36:58.076606] process done ExitSuccess
+    [2018-07-15 13:36:58.077183] process done ExitSuccess
+    ewen@ashram:~/Music/podcasts$ 
+
+Working, on `6.20170320-g41c5d9d`:
+
+
    +ewen@ashram:~/Music/podcasts$ /Applications/OpenSource/git-annex-2017-03-20.app/Contents/MacOS/git-annex importfeed --template="${TEMPLATE}" 'https://theythempodcast.com/episodes?format=RSS'
+importfeed checking known urls ok
+importfeed https://theythempodcast.com/episodes?format=RSS 
+/var/folders/p1/gpj 100%[===================>]  32.62K  61.4KB/s    in 0.5s    
+2018-07-15 13:31:17 URL:https://theythempodcast.com/episodes?format=RSS [33404/33404] -> "/var/folders/p1/gpjb1g7s00zgp64p87w19pgm0000gn/T/feed16807282475249" [1]
+ok
+ewen@ashram:~/Music/podcasts$ /Applications/OpenSource/git-annex-2017-03-20.app/Contents/MacOS/git-annex --debug importfeed --template="${TEMPLATE}" 'https://theythempodcast.com/episodes?format=RSS' 2>&1 | grep -A 1 "format=RSS"
+importfeed https://theythempodcast.com/episodes?format=RSS 
+[2018-07-15 13:33:05.367548] call: wget ["-nv","--show-progress","--clobber","-c","-O","/var/folders/p1/gpjb1g7s00zgp64p87w19pgm0000gn/T/feed16807282475249","https://theythempodcast.com/episodes?format=RSS","--user-agent","git-annex/6.20170320-g41c5d9d"]
+
+     0K .......... .......... .......... ..                   100%  121K=0.3s2018-07-15 13:33:07 URL:https://theythempodcast.com/episodes?format=RSS [33404/33404] -> "/var/folders/p1/gpjb1g7s00zgp64p87w19pgm0000gn/T/feed16807282475249" [1]
+[2018-07-15 13:33:07.050443] process done ExitSuccess
+ewen@ashram:~/Music/podcasts$ 
+
    
+
+In both cases the `TEMPLATE` value is the same one I've used for years, and works with all the other podcast feeds:
+
+    ewen@ashram:~/Music/podcasts$ set | grep TEMPLATE
+    TEMPLATE='archive/${feedtitle}/${itemtitle}${extension}'
+    ewen@ashram:~/Music/podcasts$ 
+
+### What version of git-annex are you using? On what operating system?
+
+OS X 10.11 (El Capitan), using the pre-built OS X binaries downloaded just after the security release came out.
+
+
    +ewen@ashram:~/Music/podcasts$ git annex version
+git-annex version: 6.20180626-gdf91a5cff
+build flags: Assistant Webapp Pairing S3(multipartupload)(storageclasses) WebDAV FsEvents ConcurrentOutput TorrentParser MagicMime Feeds Testsuite
+dependency versions: aws-0.17.1 bloomfilter-2.0.1.0 cryptonite-0.23 DAV-1.3.1 feed-0.3.12.0 ghc-8.0.2 http-client-0.5.7.0 persistent-sqlite-2.6.2 torrent-10000.1.1 uuid-1.3.13 yesod-1.4.5
+key/value backends: SHA256E SHA256 SHA512E SHA512 SHA224E SHA224 SHA384E SHA384 SHA3_256E SHA3_256 SHA3_512E SHA3_512 SHA3_224E SHA3_224 SHA3_384E SHA3_384 SKEIN256E SKEIN256 SKEIN512E SKEIN512 BLAKE2B256E BLAKE2B256 BLAKE2B512E BLAKE2B512 BLAKE2B160E BLAKE2B160 BLAKE2B224E BLAKE2B224 BLAKE2B384E BLAKE2B384 BLAKE2S256E BLAKE2S256 BLAKE2S160E BLAKE2S160 BLAKE2S224E BLAKE2S224 BLAKE2SP256E BLAKE2SP256 BLAKE2SP224E BLAKE2SP224 SHA1E SHA1 MD5E MD5 WORM URL
+remote types: git gcrypt p2p S3 bup directory rsync web bittorrent webdav adb tahoe glacier ddar hook external
+operating system: darwin x86_64
+supported repository versions: 3 5 6
+upgrade supported from repository versions: 0 1 2 3 4 5
+local repository version: 5
+ewen@ashram:~/Music/podcasts$ 
+
    
+
+
+
+### Please provide any additional information below.
+
+Working request (created with `wget`, then replicated with `telnet`):
+
+
    +ewen@ashram:~$ telnet 198.185.159.144 80
+Trying 198.185.159.144...
+Connected to 198.185.159.144.
+Escape character is '^]'.
+GET /episodes?format=RSS HTTP/1.1
+User-Agent: Wget/1.19.5 (darwin15.6.0)
+Accept: */*
+Accept-Encoding: identity
+Host: theythempodcast.com
+Connection: Keep-Alive
+Range: bytes=0-
+
+HTTP/1.1 301 Moved Permanently
+Date: Sun, 15 Jul 2018 02:02:53 GMT
+X-ServedBy: web012
+Location: https://theythempodcast.com/episodes?format=RSS
+Transfer-Encoding: chunked
+x-contextid: ZBu72tcg/OJSeq1q9
+x-via: 1.1 echo017
+
+0
+
+^]
+telnet> quit
+Connection closed.
+ewen@ashram:~$ 
+
    
+
+Failing request (as sent by `git-annex importfeed`):
+
+    ewen@ashram:~$ telnet 198.185.159.144 80
+    Trying 198.185.159.144...
+    Connected to 198.185.159.144.
+    Escape character is '^]'.
+    GET /episodes?format=RSS HTTP/1.1
+    Host: theytheypodcast.com
+    Range: bytes=0-
+    Accept-Encoding: identity
+
+    HTTP/1.1 400 Bad Request
+    content-length: 378
+    x-synthetic: true
+    expires: Thu, 01 Jan 1970 00:00:00 UTC
+    pragma: no-cache
+    cache-control: no-cache, must-revalidate
+    content-type: text/html; charset=UTF-8
+    connection: close
+    date: Sun, 15 Jul 2018 02:01:14 UTC
+    x-contextid: 1WIJdGgE/Y1Tq8lJu
+    x-via: 1.1 echo008
+
+    
+    
+    400 Bad Request
+    
+    
+    
+    
    400 Bad Request
    
+    
    1WIJdGgE/Y1Tq8lJu @ Sun, 15 Jul 2018 02:01:14 GMT
    
+    
    SEC-43
    
+    
    
+    
+    Connection closed by foreign host.
+    ewen@ashram:~$ 
+
+Also failing request, with minimal HTTP/1.1 headers:
+
+    ewen@ashram:~$ telnet 198.185.159.144 80
+    Trying 198.185.159.144...
+    Connected to 198.185.159.144.
+    Escape character is '^]'.
+    GET /episodes?format=RSS HTTP/1.1
+    Host: theythempodcast.com
+
+    HTTP/1.1 400 Bad Request
+    content-length: 378
+    x-synthetic: true
+    expires: Thu, 01 Jan 1970 00:00:00 UTC
+    pragma: no-cache
+    cache-control: no-cache, must-revalidate
+    content-type: text/html; charset=UTF-8
+    connection: close
+    date: Sun, 15 Jul 2018 02:07:14 UTC
+    x-contextid: c05RnFlG/h78z0OqB
+    x-via: 1.1 echo034
+
+    
+    
+    400 Bad Request
+    
+    
+    
+    
    400 Bad Request
    
+    
    c05RnFlG/h78z0OqB @ Sun, 15 Jul 2018 02:07:14 GMT
    
+    
    SEC-43
    
+    
    
+    
+    Connection closed by foreign host.
+    ewen@ashram:~$ 
+
+Apparently minimal working request:
+
+    ewen@ashram:~$ telnet 198.185.159.144 80
+    Trying 198.185.159.144...
+    Connected to 198.185.159.144.
+    Escape character is '^]'.
+    GET /episodes?format=RSS HTTP/1.1
+    User-Agent: Wget/1.19.5 (darwin15.6.0)
+    Host: theythempodcast.com
+
+    HTTP/1.1 301 Moved Permanently
+    Date: Sun, 15 Jul 2018 02:09:09 GMT
+    X-ServedBy: web025
+    Location: https://theythempodcast.com/episodes?format=RSS
+    Transfer-Encoding: chunked
+    x-contextid: veJm4JC0/sT3ENh27
+    x-via: 1.1 echo017
+
+    0
+
+    ^]
+    telnet> quit
+    Connection closed.
+    ewen@ashram:~$ 
+
+Note that the only addition was adding a `User-Agent:` header (copied from the `wget` request).  The redirect to HTTPS is expected here, as we are initially requesting via HTTP for `telnet`s benefit (I did try to get `openssl s_client` working, but it appeared the TLS connection timed out before I could manually complete a request, and I had no luck piping the request into `openssl s_client`).
+
+### Have you had any luck using git-annex before? (Sometimes we get tired of reading bug reports all day and a lil' positive end note does wonders)
+
+Absolutely :-)  `git annex` has been my podcatcher, and media management tool for years.  Thanks for writing it!
+
+> [[fixed|done]] --[[Joey]]
diff --git a/doc/bugs/importfeed_bad_request_without_User-Agent__58__/comment_1_4412b2d9f4ed59ff078c8efd9ae6e13f._comment b/doc/bugs/importfeed_bad_request_without_User-Agent__58__/comment_1_4412b2d9f4ed59ff078c8efd9ae6e13f._comment
new file mode 100644
index 0000000000..322506b8a4
--- /dev/null
+++ b/doc/bugs/importfeed_bad_request_without_User-Agent__58__/comment_1_4412b2d9f4ed59ff078c8efd9ae6e13f._comment
@@ -0,0 +1,10 @@
+[[!comment format=mdwn
+ username="ewen"
+ avatar="http://cdn.libravatar.org/avatar/605b2981cb52b4af268455dee7a4f64e"
+ subject="User-Agent"
+ date="2018-07-15T02:21:47Z"
+ content="""
+In theory `User-Agent:` is \"optional\", but [clients `SHOULD` send it](https://www.w3.org/Protocols/rfc2616/rfc2616-sec14.html#sec14.43).  Looks like some servers are particularly fussy about it being sent... and off the top of my head I can't think of a reason *not* to send a `User-Agent:` header of some sort, as it is *very* widely sent.
+
+Ewen
+"""]]
diff --git a/doc/bugs/importfeed_bad_request_without_User-Agent__58__/comment_2_ae6eee51655aa6b1152ade3bdf7f700f._comment b/doc/bugs/importfeed_bad_request_without_User-Agent__58__/comment_2_ae6eee51655aa6b1152ade3bdf7f700f._comment
new file mode 100644
index 0000000000..eb9d012eed
--- /dev/null
+++ b/doc/bugs/importfeed_bad_request_without_User-Agent__58__/comment_2_ae6eee51655aa6b1152ade3bdf7f700f._comment
@@ -0,0 +1,11 @@
+[[!comment format=mdwn
+ username="joey"
+ subject="""comment 2"""
+ date="2018-07-16T15:48:08Z"
+ content="""
+Hmm, it should be sending a user agent. 
+
+Wow, it's not sending a user agent!!!
+
+Thank you very much for noticing this glaring omission.
+"""]]
diff --git a/doc/bugs/importfeed_does_not_work_with_socks_proxy.mdwn b/doc/bugs/importfeed_does_not_work_with_socks_proxy.mdwn
new file mode 100644
index 0000000000..4f892469dc
--- /dev/null
+++ b/doc/bugs/importfeed_does_not_work_with_socks_proxy.mdwn
@@ -0,0 +1,8 @@
+It appears that `git annex importfeed` can not be used in with socks proxies because it uses wget unconditionally (at least in the Debian build).
+
+For `addurl`, I could configure the system to use a socks proxy by setting `git config annex.web-download-command "curl --silent --preproxy socks4a://localhost:1080 %url -o %file"`; for `importfeed`, I found no option to override the command used to fetch the URL, and `wget` [lacks SOCKS support](https://savannah.gnu.org/bugs/?func=detailitem&item_id=43576).
+
+Please consider using `web-download-command` for `importfeed` too, introducing a dedicated option `web-get-command` (that would output to stdout rather than %file), or otherwise supporting operation behind a SOCKS proxy.
+
+> Closing this, since the http library git-annex uses supports socks
+> proxies via environment settings, as far as I know. [[done]] --[[Joey]] 
diff --git a/doc/bugs/importfeed_does_not_work_with_socks_proxy/comment_1_9797d8de55cf91288ecfc64a7e96645d._comment b/doc/bugs/importfeed_does_not_work_with_socks_proxy/comment_1_9797d8de55cf91288ecfc64a7e96645d._comment
new file mode 100644
index 0000000000..3db7a070da
--- /dev/null
+++ b/doc/bugs/importfeed_does_not_work_with_socks_proxy/comment_1_9797d8de55cf91288ecfc64a7e96645d._comment
@@ -0,0 +1,23 @@
+[[!comment format=mdwn
+ username="joey"
+ subject="""comment 1"""
+ date="2018-02-22T16:32:33Z"
+ content="""
+It's true that importfeed calls Url.download to download the feed file, 
+without bothering to support annex.web-download-command. That could easily
+be changed.
+
+However, it also uses Url.getUrlInfo, which does not and cannot use the
+annex.web-download-command interface, and which is too complicated an
+interface to make into a hook.
+
+Indeed, annex.web-download-command was never intended to cover all
+the ways git-annex uses http, but only uses of http to download
+large file contents. And importfeed does use it for such downloads,
+but not for its other http needs.
+
+To configure use of a proxy, you would probably be best served by using
+the `http_proxy` and `https_proxy` environment variables, which are
+supported by wget, curl, and by the haskell http library that git-annex
+also uses.
+"""]]
diff --git a/doc/bugs/impossible_to_login_to_the_website_at_times.mdwn b/doc/bugs/impossible_to_login_to_the_website_at_times.mdwn
new file mode 100644
index 0000000000..c732c23ebc
--- /dev/null
+++ b/doc/bugs/impossible_to_login_to_the_website_at_times.mdwn
@@ -0,0 +1,6 @@
+As reported before in private communication, remains the case.  Yesterday happened on both chromium and firefox.  Cookies are enabled and even present (see screenshot, couldn't attach), clearing cookie doesn't help.  Now managed to successfully login in firefox.
+
+
+[[!meta author=yoh]]
+
+[screenshot.png](http://www.onerussian.com/tmp/gkrellShoot_09-25-18_092905.png)
diff --git a/doc/bugs/impossible_to_login_to_the_website_at_times/comment_1_a5824d9099dc4035e09086166b13f133._comment b/doc/bugs/impossible_to_login_to_the_website_at_times/comment_1_a5824d9099dc4035e09086166b13f133._comment
new file mode 100644
index 0000000000..76f729959b
--- /dev/null
+++ b/doc/bugs/impossible_to_login_to_the_website_at_times/comment_1_a5824d9099dc4035e09086166b13f133._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="Ilya_Shlyakhter"
+ avatar="http://cdn.libravatar.org/avatar/1647044369aa7747829c38b9dcc84df0"
+ subject="comment 1"
+ date="2018-09-25T15:24:55Z"
+ content="""
+Same here.  Had to clear all my cookies, and even that doesn't always help.
+"""]]
diff --git a/doc/bugs/impossible_to_login_to_the_website_at_times/comment_2_fceb64e98fb7566f7cb999e9b37914f5._comment b/doc/bugs/impossible_to_login_to_the_website_at_times/comment_2_fceb64e98fb7566f7cb999e9b37914f5._comment
new file mode 100644
index 0000000000..1019bfce92
--- /dev/null
+++ b/doc/bugs/impossible_to_login_to_the_website_at_times/comment_2_fceb64e98fb7566f7cb999e9b37914f5._comment
@@ -0,0 +1,9 @@
+[[!comment format=mdwn
+ username="yarikoptic"
+ avatar="http://cdn.libravatar.org/avatar/f11e9c84cb18d26a1748c33b48c924b4"
+ subject="comment 2"
+ date="2018-09-25T15:27:13Z"
+ content="""
+happened to me again on firefox, where I first went to http://git-annex.branchable.com/bugs/ to file a new report, was directed to login, entered credentials, got error message.  I have then removed cookies, went in a new tab to the bugs, this bug, and then was able to login.
+So it is cookies related, but IMHO not a user fault 
+"""]]
diff --git a/doc/bugs/impossible_to_perform___34__read-only__34___git_annex_info_without_write_permissions.mdwn b/doc/bugs/impossible_to_perform___34__read-only__34___git_annex_info_without_write_permissions.mdwn
new file mode 100644
index 0000000000..6236174ce5
--- /dev/null
+++ b/doc/bugs/impossible_to_perform___34__read-only__34___git_annex_info_without_write_permissions.mdwn
@@ -0,0 +1,19 @@
+
+### What version of git-annex are you using? On what operating system?
+
+6.20171001+gitg542d0649f-1~ndall+1
+
+### What steps will reproduce the problem?
+
+[[!format sh """
+$> git annex info     
+
+git-annex: .git/annex/misctmp/mergedrefs.0: createDirectory: permission denied (Permission denied)
+failed
+git-annex: info: 1 failed
+
+"""]]
+
+unless there is really a need to have this operations performed within the same repository/the same filesystem, I do not understand why generic $TMPDIR could not be used for such operations so no write access has to be demanded
+
+[[!meta author=yoh]]
diff --git a/doc/bugs/impossible_to_perform___34__read-only__34___git_annex_info_without_write_permissions/comment_1_8b60c52d8fe41718377d6d15a25cae97._comment b/doc/bugs/impossible_to_perform___34__read-only__34___git_annex_info_without_write_permissions/comment_1_8b60c52d8fe41718377d6d15a25cae97._comment
new file mode 100644
index 0000000000..efef1a0e39
--- /dev/null
+++ b/doc/bugs/impossible_to_perform___34__read-only__34___git_annex_info_without_write_permissions/comment_1_8b60c52d8fe41718377d6d15a25cae97._comment
@@ -0,0 +1,20 @@
+[[!comment format=mdwn
+ username="joey"
+ subject="""comment 1"""
+ date="2018-02-22T16:17:35Z"
+ content="""
+The mergedrefs directory is used while building the commit to merge
+git-annex branches. So even if it was written someplace else, that commit
+would fail.
+
+I think this may be happening even when there are no
+git-annex refs to merge in, due to the transition code
+in Annex.Branch.updateTo that temporarily calls addMergedRefs
+in the "null tomerge" case. That was added in 2016, and is flagged as able
+to be safely removed. I've removed it.
+
+However, when there actually is a git-annex branch to merge, if a
+hypothetical readonly mode avoided doing so, it would necessarily see a
+different state of the git-annex branch than would be seen in non-readonly
+mode. That behavior difference could be fairly confusing potentially..
+"""]]
diff --git a/doc/bugs/impossible_to_perform___34__read-only__34___git_annex_info_without_write_permissions/comment_2_066d2f95ffab1b36b69aa84643d67765._comment b/doc/bugs/impossible_to_perform___34__read-only__34___git_annex_info_without_write_permissions/comment_2_066d2f95ffab1b36b69aa84643d67765._comment
new file mode 100644
index 0000000000..0863efa278
--- /dev/null
+++ b/doc/bugs/impossible_to_perform___34__read-only__34___git_annex_info_without_write_permissions/comment_2_066d2f95ffab1b36b69aa84643d67765._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="yarikoptic"
+ avatar="http://cdn.libravatar.org/avatar/f11e9c84cb18d26a1748c33b48c924b4"
+ subject="--read-only flag?"
+ date="2018-02-22T17:18:18Z"
+ content="""
+I hear you... so far though I was confused by the fact that what I thought would be a read-only operation was actually changing the state of the things (doing the annex merge).  Although I would have preferred just a warning like \"Cannot merge git-annex branch because of lacking permissions to do so, some information might be not up-to-date\", I wondered if then may be a generic resolution could be to add `--read-only` flag to such commands as `info` and `whereis`.  Then we (datalad) would be the one to explicitly check if there is write-permissions and if not -- issue the command in `--read-only` mode.  We might even make it a default mode of operation for some of our usecases where it would be confusing if things were changing in the background (e.g. with `ls` command)
+"""]]
diff --git a/doc/bugs/impossible_to_perform___34__read-only__34___git_annex_info_without_write_permissions/comment_3_05f7eec04634e5b4e200cf3dca1bb1b1._comment b/doc/bugs/impossible_to_perform___34__read-only__34___git_annex_info_without_write_permissions/comment_3_05f7eec04634e5b4e200cf3dca1bb1b1._comment
new file mode 100644
index 0000000000..3edb6654b5
--- /dev/null
+++ b/doc/bugs/impossible_to_perform___34__read-only__34___git_annex_info_without_write_permissions/comment_3_05f7eec04634e5b4e200cf3dca1bb1b1._comment
@@ -0,0 +1,33 @@
+[[!comment format=mdwn
+ username="joey"
+ subject="""comment 3"""
+ date="2018-02-22T17:27:06Z"
+ content="""
+There are situations where a git command that appears to be read-only,
+such as `git status` actually writes to the repository behind the scenes.
+It looks like git ignores write errors in at least some such cases. So
+there is precident for implicit read-only support, but I think not in cases
+where it would involve significant behavior changes, like it would for the
+git-annex branch auto-merging. In git's case the behavior change probably
+only involves repeated `git status` runs being slower than otherwise,
+or something like that.
+
+As well as populating the .git/annex/index file with information merged in from
+recently fetched git-annex branches, git-annex may need to write to other files
+in order to prepare caches needed to perform what appears to be "read-only"
+query operation, or to lock files in order to prevent someone who does have
+write access from dropping them in a situation where that will lose data.
+An example of the latter is running `git annex drop` in a repository you do have
+write access to, and it needing to exclusively lock files in origin,
+which requires write access to origin as well. Without write access,
+the drop may fail.
+
+The --read-only flag seems to be setting up a situation where git-annex handles
+some things being read-only, but then someone expects the flag to 
+make some other thing work read-only, which git-annex can't manage to support
+for whatever reason. 
+
+So I prefer a more specific name, like annex.merge-annex-branches=false.
+
+Implemented that.
+"""]]
diff --git a/doc/bugs/inAnnex_check_failed_repeatedly_for_present_content_v6.mdwn b/doc/bugs/inAnnex_check_failed_repeatedly_for_present_content_v6.mdwn
new file mode 100644
index 0000000000..60e145fa39
--- /dev/null
+++ b/doc/bugs/inAnnex_check_failed_repeatedly_for_present_content_v6.mdwn
@@ -0,0 +1,42 @@
+In a v6 repository, `git annex get` of a particular file re-downloaded it
+each time it was run. `git annex whereis` said the content was locally
+present. But, `git annex fsck` of the file said the content was
+missing, and removed it from the location log.
+
+The file was locked, and the repository was on ext4.
+
+Reported by gleachkr on IRC. Don't have enough information to reproduce
+the problem yet. --[[Joey]]
+
+> My initial analysis is that this must be a problem with
+> `Annex.Content.inAnnex`. Note that checks the cached inode for the
+> content and if it finds a mismatch, it behaves as if the content is not
+> present. That would be consistent with the problem as reported.
+> 
+> When I init a v6 repository and add some locked files, no inode cache is
+> recorded, which makes sense as they're locked.
+> 
+> Hypothesis: A cached inode for the key got into the keys database,
+> despite the file being locked, and that is messing up inAnnex.
+> 
+> Should inAnnex even be checking the inode cache for locked content?
+> This seems unncessary, and note that it's done for v4 mode as well as v6.
+> 
+> How could a cached inode for a locked file leak in? Perhaps the file
+> was earlier unlocked. Or perhaps another, unlocked file, had the same
+> content. I tried both these scenarios, and was able to get a cached
+> inode to be listed for a file, but in my tests at least, it also cached
+> the inode of the locked file, and I did not replicate the problem.
+> 
+> --[[Joey]]
+
+## more information needed
+
+If gleachkr comes back to IRC, it would be good to find out:
+
+* Was this file previously unlocked? `git log` on the file would probably
+  tell, unless it was briefly unlocked in between commits.
+* Run `git annex info` on the file to see what its key is.  
+  Then, run `sqlite3 .git/annex/keys/db` and .dump and see
+  what is recorded for that key, in both the "content" and "associated"
+  tables.
diff --git a/doc/bugs/inAnnex_check_failed_repeatedly_for_present_content_v6/comment_1_fa6b924d792613b60979a87deea2a66f._comment b/doc/bugs/inAnnex_check_failed_repeatedly_for_present_content_v6/comment_1_fa6b924d792613b60979a87deea2a66f._comment
new file mode 100644
index 0000000000..6ace37334d
--- /dev/null
+++ b/doc/bugs/inAnnex_check_failed_repeatedly_for_present_content_v6/comment_1_fa6b924d792613b60979a87deea2a66f._comment
@@ -0,0 +1,67 @@
+[[!comment format=mdwn
+ username="gleachkr@7c488e398809299a1100b93f8884de43dee83674"
+ nickname="gleachkr"
+ avatar="http://cdn.libravatar.org/avatar/c7ce6b5eae91547b25e9a05fc7c8cf22"
+ subject="More data points"
+ date="2017-09-16T01:16:59Z"
+ content="""
+the results of `git log` on one affected file are (the last commit being after I noticed the problem and tried to fix it)
+
+    commit 0e93d1c9c18cf7aead978e0ae453e66991d6e500
+    Author: Graham 
+    Date:   Tue Sep 12 19:20:33 2017 +0000
+
+        attempt to fix references for Whitney
+
+    commit 02bf95566eb2dd6947f417019e1d601abcfb55c1
+    Author: Graham 
+    Date:   Tue Aug 8 18:45:06 2017 -0500
+    
+        cleanup
+    
+    commit 655b0fd419945d0ea32d9d178c551af0a64e6afd
+    Author: Graham 
+    Date:   Sat Nov 19 19:21:10 2016 +0000
+    
+        database cleanup
+    
+    commit 54fa420bcaf33847eee77a30fbd9556beea28f77
+    Author: gleachkr 
+    Date:   Fri Sep 9 17:48:48 2016 -0500
+    
+        git-annex in graham@Descartes:~/music
+
+running `.dump` in sqlite3 yields the following two lines containing the key associated with one file suffering from the problem, with no lines containing both \"associated\" and the key (sorry, not a db expert):
+
+    INSERT INTO content VALUES(171,'SHA256E-s8350646--d9bbbd67402a1b7560ebc47bc7bafaf74115a99c628e7458ba4754d8a355908a.m4a','I \"237 8350646 1473527901\"');
+    INSERT INTO content VALUES(172,'SHA256E-s8350646--d9bbbd67402a1b7560ebc47bc7bafaf74115a99c628e7458ba4754d8a355908a.m4a','I \"1711652 8350646 1473527901\"');
+
+another affected file shows up on lines:
+
+    INSERT INTO associated VALUES(3,'SHA256E-s8789357--2bedeea689e7d7dda1e877989abd3e822f722c9bc45a14d1951d5dc104f4ad62.m4a','Car Seat Headrest/Teens of Denial/01 Fill in the Blank.m4a');
+    INSERT INTO content VALUES(31,'SHA256E-s8789357--2bedeea689e7d7dda1e877989abd3e822f722c9bc45a14d1951d5dc104f4ad62.m4a','I \"135750 8789357 1473525850\"');
+    INSERT INTO content VALUES(32,'SHA256E-s8789357--2bedeea689e7d7dda1e877989abd3e822f722c9bc45a14d1951d5dc104f4ad62.m4a','I \"1711014 8789357 1473525850\"');
+    INSERT INTO content VALUES(296,'SHA256E-s8789357--2bedeea689e7d7dda1e877989abd3e822f722c9bc45a14d1951d5dc104f4ad62.m4a','I \"1050021 8789357 1498915821\"');
+
+This second file has slightly different deviant behavior. It still registers as not present to `annex find` and `annex info`, is redownloaded by `annex get`, and shows up in `annex list`. But unlike the first, it does not register as not present when `annex fsck` is run. It just checks out, apparently.
+
+The second file gives `git log` results as follows:
+
+    commit 02bf95566eb2dd6947f417019e1d601abcfb55c1
+    Author: Graham 
+    Date:   Tue Aug 8 18:45:06 2017 -0500
+    
+        cleanup
+    
+    commit 655b0fd419945d0ea32d9d178c551af0a64e6afd
+    Author: Graham 
+    Date:   Sat Nov 19 19:21:10 2016 +0000
+    
+        database cleanup
+    
+    commit 54fa420bcaf33847eee77a30fbd9556beea28f77
+    Author: gleachkr 
+    Date:   Fri Sep 9 17:48:48 2016 -0500
+    
+        git-annex in graham@Descartes:~/music
+"""]]
diff --git a/doc/bugs/inAnnex_check_failed_repeatedly_for_present_content_v6/comment_2_5e37bcb145880fe6074995f17c5aa7c3._comment b/doc/bugs/inAnnex_check_failed_repeatedly_for_present_content_v6/comment_2_5e37bcb145880fe6074995f17c5aa7c3._comment
new file mode 100644
index 0000000000..94bada5999
--- /dev/null
+++ b/doc/bugs/inAnnex_check_failed_repeatedly_for_present_content_v6/comment_2_5e37bcb145880fe6074995f17c5aa7c3._comment
@@ -0,0 +1,32 @@
+[[!comment format=mdwn
+ username="joey"
+ subject="""comment 2"""
+ date="2017-09-16T14:54:49Z"
+ content="""
+The .dump that shows the file is in the "content" table but
+not the "associated" table seems to confirm my hypothesis.
+
+Aha -- I was able to replicate having "content" but no "associated" in the
+keys database, by first using `git annex add` on a file, then `git annex
+unlock`, then `git annex lock`. Any chance this is what you did? (Perhaps
+some of those commits that git log shows were the locking/unlocking; if you
+`git show` the commits and see that the mode of the file has changed, that
+would confirm it.)
+
+I've still not quite managed to replicate the problem, because the cached
+inodes were still right. Tried moving the file away to another repo, but
+it then removed the cached inodes and so avoided the problem.
+
+Very interesting about the second file with `git annex get` and `git annex
+fsck` behaving differently. Does the file 'Car Seat Headrest/Teens of Denial/01 Fill in the Blank.m4a'
+still exist in your git repository?
+
+Is annex.thin set in .git/config?
+
+----
+
+I probably have enough information to move on to getting your repository
+fixed so you can stop being bothered by the problem at least. I think you
+could probably move .git/annex/keys/db out of the way, and run `git annex
+lock` followed by `git annex fsck` to get into a non-broken state.
+"""]]
diff --git a/doc/bugs/inAnnex_check_failed_repeatedly_for_present_content_v6/comment_3_9efad60d2c43a902c7a0e2d6225be1c5._comment b/doc/bugs/inAnnex_check_failed_repeatedly_for_present_content_v6/comment_3_9efad60d2c43a902c7a0e2d6225be1c5._comment
new file mode 100644
index 0000000000..20d8320071
--- /dev/null
+++ b/doc/bugs/inAnnex_check_failed_repeatedly_for_present_content_v6/comment_3_9efad60d2c43a902c7a0e2d6225be1c5._comment
@@ -0,0 +1,23 @@
+[[!comment format=mdwn
+ username="gleachkr@7c488e398809299a1100b93f8884de43dee83674"
+ nickname="gleachkr"
+ avatar="http://cdn.libravatar.org/avatar/c7ce6b5eae91547b25e9a05fc7c8cf22"
+ subject="comment 3"
+ date="2017-09-16T16:02:41Z"
+ content="""
+The second file (`01 Fill in the Blank.m4a`) does still exist, although `annex get` always re-retrieves it. annx.thin is not set. Here's the git config, minus remotes:
+
+    [core]
+        repositoryformatversion = 0
+        filemode = true
+        bare = false
+        logallrefupdates = true
+    [annex]
+        uuid = e9731ab7-6a76-4eef-b337-2b8573380014
+        version = 6
+    [filter \"annex\"]
+        smudge = git-annex smudge %f
+        clean = git-annex smudge --clean %f
+
+Thanks for the fix. Just to make sure I understand before breaking anything futher, the idea would be to move `.git/annex/keys/db` somewhere safe, `git annex lock` all the affected files, and then `git annex fsck` the whole repository? or just the affected files?
+"""]]
diff --git a/doc/bugs/inAnnex_check_failed_repeatedly_for_present_content_v6/comment_4_90062420b67a7f364f5ece947408798f._comment b/doc/bugs/inAnnex_check_failed_repeatedly_for_present_content_v6/comment_4_90062420b67a7f364f5ece947408798f._comment
new file mode 100644
index 0000000000..af86bec35e
--- /dev/null
+++ b/doc/bugs/inAnnex_check_failed_repeatedly_for_present_content_v6/comment_4_90062420b67a7f364f5ece947408798f._comment
@@ -0,0 +1,12 @@
+[[!comment format=mdwn
+ username="joey"
+ subject="""comment 4"""
+ date="2017-09-16T16:04:24Z"
+ content="""
+Right, just back up the keys/db just in case, and you need to fsck at least
+any files that are not locked (to repopulate the keys/db for those), 
+so safest is to fsck the whole repository..
+
+Is it possible that you've run `git annex unlock` / `git annex lock` on
+some of the affected files in the past?
+"""]]
diff --git a/doc/bugs/inAnnex_check_failed_repeatedly_for_present_content_v6/comment_5_5f2e03340fb98bd146c4563e8e57dd30._comment b/doc/bugs/inAnnex_check_failed_repeatedly_for_present_content_v6/comment_5_5f2e03340fb98bd146c4563e8e57dd30._comment
new file mode 100644
index 0000000000..811e604271
--- /dev/null
+++ b/doc/bugs/inAnnex_check_failed_repeatedly_for_present_content_v6/comment_5_5f2e03340fb98bd146c4563e8e57dd30._comment
@@ -0,0 +1,12 @@
+[[!comment format=mdwn
+ username="joey"
+ subject="""comment 5"""
+ date="2017-09-16T16:07:38Z"
+ content="""
+Actually, you should `git annex lock` before moving the database out of
+the way, otherwise it might be confused. So:
+
+1. git annex lock
+2. move keys/db to a safe backup location
+3. git annex fsck
+"""]]
diff --git a/doc/bugs/inAnnex_check_failed_repeatedly_for_present_content_v6/comment_6_fce1ef0377e00fb9431d4c8b0215387b._comment b/doc/bugs/inAnnex_check_failed_repeatedly_for_present_content_v6/comment_6_fce1ef0377e00fb9431d4c8b0215387b._comment
new file mode 100644
index 0000000000..5cd5cbf454
--- /dev/null
+++ b/doc/bugs/inAnnex_check_failed_repeatedly_for_present_content_v6/comment_6_fce1ef0377e00fb9431d4c8b0215387b._comment
@@ -0,0 +1,11 @@
+[[!comment format=mdwn
+ username="gleachkr@7c488e398809299a1100b93f8884de43dee83674"
+ nickname="gleachkr"
+ avatar="http://cdn.libravatar.org/avatar/c7ce6b5eae91547b25e9a05fc7c8cf22"
+ subject="Thanks!"
+ date="2017-09-16T16:46:48Z"
+ content="""
+Removing the db and running `fsck` seems to have fixed the problem. I really appreciate it. I do think that I unlocked and then re-locked some of the affected files, but I think only after I noticed that they were behaving strangely---in particular, I think I only did this with the affected files that were not `fsck`ing correctly (and this was before I realized they were not `fsck`ing correctly), not with e.g. the Car Seat Headrest song.
+
+I didn't get the chance to fill in the \"Have you had any luck using git-annex before\" part of the standard bug report, so I thought I should add that here: Yes, I've used git-annex regularly for quite a while now. It's an inspiring piece of open-source software. I'm always finding new things it can do. Thank you for creating it, and thanks for your help with this issue!
+"""]]
diff --git a/doc/bugs/inAnnex_check_failed_repeatedly_for_present_content_v6/comment_7_448fddcc8f489030d92cfe9dbba1f07f._comment b/doc/bugs/inAnnex_check_failed_repeatedly_for_present_content_v6/comment_7_448fddcc8f489030d92cfe9dbba1f07f._comment
new file mode 100644
index 0000000000..6fab59e156
--- /dev/null
+++ b/doc/bugs/inAnnex_check_failed_repeatedly_for_present_content_v6/comment_7_448fddcc8f489030d92cfe9dbba1f07f._comment
@@ -0,0 +1,53 @@
+[[!comment format=mdwn
+ username="joey"
+ subject="""comment 7"""
+ date="2018-08-27T16:27:47Z"
+ content="""
+I was able to reproduce what I described in comment #2 with current
+git-annex.
+
+Also, I was able to reproduce the fsck failure, by touching the
+.git/annex/objects file. Even though the file is not modifiable, touching
+it updates its mtime, and so the inode cache is considered stale.
+This makes inAnnex think it's not present.
+
+And another way is, when annex.thin is enabled, to touch the unlocked file,
+which also touched the .git/annex/objects it's hardlinked to.
+git-annex lock then checksums it (because the inode cache is stale),
+concludes it's still good, and so proceeds to lock it, but the stale inode
+cache again causes inAnnex to think it's not present.
+
+I still don't know how to reproduce git-annex get redownloading a file that
+git-annex find lists.
+
+> Should inAnnex even be checking the inode cache for locked content? This seems unncessary, and note that it's done for v4 mode as well as v6.
+
+Unnecessary for v4 certianly, but in v6 with annex.thin,
+unlocked content is hard linked and could be modified, 
+so it does need to check that the inode cache is valid.
+
+Could remove the inode cache when locking a file, as long as there
+are no other associated files that are unlocked. That solves the problem
+for that case, and makes inAnnex avoid some unncessary work too.
+
+When the same content has a mix of locked and unlocked associated files,
+the inode cache needs to remain populated (to support annex.thin and so
+git-annex drop will drop the copies of the content from the locked files).
+But then if the inode cache for the locked content becomes stale, and
+the unlocked files get modified, inAnnex will again be wrong.
+
+So, it seems that inAnnex also needs to check if the annex object 
+has no hard links, and if so, treat it as present even when the inode cache
+does not match. That's cheaper than checking the inode cache, so it ought
+to be done first.
+
+Conclusion: Only check the inode cache for the annex object
+when annex.thin has made a hard link to it. If its link count is 1,
+inAnnex knows it's present and locked, so it can assume it's good.
+
+(Also, it's possible there could be a race when locking a file
+where the file gets modified after the inode cache is checked,
+so it's treated as unmodified but is really modified and as a hard link to
+the object file, the object file has the wrong content. Need to make sure
+this race can't occur.)
+"""]]
diff --git a/doc/bugs/inconsistent_output_upon_addurl_--batch_complicates_if_not_forbids_reliable_parsing_of_output.mdwn b/doc/bugs/inconsistent_output_upon_addurl_--batch_complicates_if_not_forbids_reliable_parsing_of_output.mdwn
new file mode 100644
index 0000000000..6601e9ba58
--- /dev/null
+++ b/doc/bugs/inconsistent_output_upon_addurl_--batch_complicates_if_not_forbids_reliable_parsing_of_output.mdwn
@@ -0,0 +1,24 @@
+### Please describe the problem.
+
+if using addurl in --batch mode, there is no --json formatting and text output is somewhat inconsistent.  It might include "downloading ..." with 'ok' on a separate line, possibly with a msg on recording state in git, or within the same line report 'ok', or 'failed' in upcoming lines.  This complicates parsing of such output to verify correct operation for a given addurl pair
+
+[[!format sh """
+
+$> echo 'http://127.0.0.1:33369/about.txt about.txt' | git -c receive.autogc=false annex addurl --with-files --batch --debug 2>/dev/null
+addurl about.txt (downloading http://127.0.0.1:33369/about.txt ...)                                                                     
+ok
+(recording state in git...)
+
+$> echo 'http://127.0.0.1:33369/about.txt about.txt' | git -c receive.autogc=false annex addurl --with-files --batch --debug 2>/dev/null
+addurl about.txt ok
+
+$> echo 'http://127.0.0.1:33369/about.stxt about.txt' | git -c receive.autogc=false annex addurl --with-files --batch --debug 2>/dev/null 
+addurl about.txt 
+failed
+
+"""]]
+
+[[!meta author=yoh]]
+
+> I've made --json work for addurl; I feel that's the way to go for parsing
+> its output. [[done]] --[[Joey]]
diff --git a/doc/bugs/indeterminite_preferred_content_state_for_duplicated_file.mdwn b/doc/bugs/indeterminite_preferred_content_state_for_duplicated_file.mdwn
new file mode 100644
index 0000000000..086c21ea80
--- /dev/null
+++ b/doc/bugs/indeterminite_preferred_content_state_for_duplicated_file.mdwn
@@ -0,0 +1,23 @@
+Consider if file `foo` uses key K, and file `archive/bar` uses the same key K.
+Using standard client preferred content settings, `git annex drop --auto`
+will want to drop `archive/bar`, but `git annex get --auto` will want to get
+`foo`. `git annex sync --content` will do both operations, getting and then
+dropping the key. Running these commands repeatedly churns unncessarily.
+
+Fixing this needs a map from key to files using it. Then, when checking
+preferred content of `archive/bar`, it can see that `foo` also uses the
+key. Since `foo` is wanted, it should not drop the key, even though
+`archive/bar` is not wanted.
+
+Such a map exists, in the keys database, but only in v6 mode repositories.
+So, this seems solvable in v6 repositories, but not in v5.
+Also, the associated files map may not be accurate at all times, so that's
+a wrinkle to using it for this. --[[Joey]]
+
+(See [[bugs/preferred_content_and_deduplication]] for a user bug report of
+the same thing, with some suggestions for workarounds too.)
+
+In the preferred content expressions for standard groups, the only
+place this bug can be triggered involves archive directories of
+repositories in the client group. A file both in the archive directory and
+in another directory has indeterminite status.
diff --git a/doc/bugs/info_--json_lists_backend_usage_stats_as_a_list_of_lists.mdwn b/doc/bugs/info_--json_lists_backend_usage_stats_as_a_list_of_lists.mdwn
new file mode 100644
index 0000000000..b84ab32e6a
--- /dev/null
+++ b/doc/bugs/info_--json_lists_backend_usage_stats_as_a_list_of_lists.mdwn
@@ -0,0 +1,34 @@
+### What version of git-annex are you using? On what operating system?
+
+5.20151116+gitg5416a1a-1~ndall+1
+
+### Please provide any additional information below.
+
+[[!format sh """
+    "backend usage": [
+        [
+            "MD5E",
+            2
+        ],
+        [
+            "SHA256E",
+            2
+        ]
+    ],
+"""]]
+
+instead of more logical
+
+[[!format sh """
+
+    "backend usage": {
+         "MD5E": 2,
+         "SHA256E": 2
+    }
+"""]]
+
+also it seems it just doubles them since I have only 2 files, 1 for each backend (as reported also by info "local annex keys": 2). 
+
+[[!meta author=yoh]]
+
+> [[fixed|done]] --[[Joey]]
diff --git a/doc/bugs/initremote_doesn__39__t_work_in_scripts.mdwn b/doc/bugs/initremote_doesn__39__t_work_in_scripts.mdwn
new file mode 100644
index 0000000000..75361a6c15
--- /dev/null
+++ b/doc/bugs/initremote_doesn__39__t_work_in_scripts.mdwn
@@ -0,0 +1,25 @@
+### Please describe the problem.
+
+
+### What steps will reproduce the problem?
+
+Run "git-annex initremote" from an environment (cron, CI, etc.) where /dev/tty doesn't exist.
+
+### What version of git-annex are you using? On what operating system?
+
+git-annex version: 6.20160613-g35dbe35 on debian jessie
+
+### Please provide any additional information below.
+
+Example of the error @ https://gitlab.com/DanielDent/git-annex-remote-rclone/builds/2166644
+
+Error:
+gpg: cannot open tty `/dev/tty': No such device or address
+
+failed
+
+Rumour on the internets is that the "--no-tty" parameter to GPG may fix the problem.
+
+### Have you had any luck using git-annex before? (Sometimes we get tired of reading bug reports all day and a lil' positive end note does wonders)
+
+Yes! It's great!
diff --git a/doc/bugs/initremote_doesn__39__t_work_in_scripts/comment_1_da93784aa66baad31f86231bf7cdce61._comment b/doc/bugs/initremote_doesn__39__t_work_in_scripts/comment_1_da93784aa66baad31f86231bf7cdce61._comment
new file mode 100644
index 0000000000..277a794995
--- /dev/null
+++ b/doc/bugs/initremote_doesn__39__t_work_in_scripts/comment_1_da93784aa66baad31f86231bf7cdce61._comment
@@ -0,0 +1,22 @@
+[[!comment format=mdwn
+ username="joey"
+ subject="""comment 1"""
+ date="2016-07-05T13:28:12Z"
+ content="""
+I tried running initremote without a tty, and it worked ok,
+using the following test case:
+
+	#!/bin/sh
+	set -e
+	cd /tmp/r
+	pwd
+	git annex initremote test type=directory directory=../d encryption=hybrid keyid=2512E3C7 --fast --debug >/dev/null
+
+So, I need more information to reproduce this.
+
+It seems likely that, if gpg is trying to read from the tty, it needs to
+prompt for your key's passphrase for some reason. In which case setting
+--batch would only make it fail differently. If you want to script
+something that needs access to your gpg secret key, you need to configure
+gpg to use a gpg agent.
+"""]]
diff --git a/doc/bugs/issue_with_syncing_between_v5_and_v6_repos.mdwn b/doc/bugs/issue_with_syncing_between_v5_and_v6_repos.mdwn
new file mode 100644
index 0000000000..f0ee4ba58c
--- /dev/null
+++ b/doc/bugs/issue_with_syncing_between_v5_and_v6_repos.mdwn
@@ -0,0 +1,112 @@
+### Please describe the problem.
+I was accidentally syncing between repos at v5 and v6, I didn't realize they were different versions until I started digging into this issue.
+
+I have two repos, one at v5 and one at v6. I add a file to v6 annex, unlock, then sync to my v5 repo. From my v5 repo, I then move the file into a sub-directory, in my v5 repo, and accidentally commit with git add; git commit, instead of git annex add; git commit, since with my v6 repo I am used to using git add for everything. The file content is now missing from annex on my v5 repo. I then sync from both repos. Now the file in v5 repo is broken and the content is not present in annex. The content is fine on the v6 repo. The only way to actually get annex to see the content, from my v5 repo, though is to do git annex upgrade, git annex sync, fsck repair don't help.
+
+### What steps will reproduce the problem?
+
+### What version of git-annex are you using? On what operating system?
+
+    $ git annex version
+    git-annex version: 6.20171128-g58b04cd2e
+    build flags: Assistant Webapp Pairing Testsuite S3(multipartupload)(storageclasses) WebDAV FsEvents ConcurrentOutput TorrentParser MagicMime Feeds Quvi
+    dependency versions: aws-0.17.1 bloomfilter-2.0.1.0 cryptonite-0.23 DAV-1.3.1 feed-0.3.12.0 ghc-8.0.2 http-client-0.5.7.0 persistent-sqlite-2.6.2 torrent-10000.1.1 uuid-1.3.13 yesod-1.4.5
+    key/value backends: SHA256E SHA256 SHA512E SHA512 SHA224E SHA224 SHA384E SHA384 SHA3_256E SHA3_256 SHA3_512E SHA3_512 SHA3_224E SHA3_224 SHA3_384E SHA3_384 SKEIN256E SKEIN256 SKEIN512E SKEIN512 SHA1E SHA1         MD5E MD5 WORM URL
+    remote types: git gcrypt p2p S3 bup directory rsync web bittorrent webdav tahoe glacier ddar hook external
+    local repository version: 6
+    supported repository versions: 3 5 6
+    upgrade supported from repository versions: 0 1 2 3 4 5
+    operating system: darwin x86_64
+
+### Please provide any additional information below.
+
+[[!format sh """
+# If you can, paste a complete transcript of the problem occurring here.
+
+#
+# Create a V6 repo and add an image to it
+#
+$ mkdir /tmp/repov6
+$ cd /tmp/repov6
+$ git init
+$ git annex init
+$ /tmp/repov6$ git annex upgrade
+upgrade . (v5 to v6...) ok
+
+$ git add image1.png 
+$ git commit -m "added image 1 to annex"
+$ /tmp/repov6$ git annex info image1.png 
+file: image1.png
+size: 17.69 kilobytes
+key: SHA256E-s17691--c98e6eae515fec6a094e2b4bdcd221ba089a5e6073f6c365f6d4c829243c8aa5.png
+present: true
+$ git annex unlock image1.png
+$ git commit -m "unlocked the image"
+
+#
+# Create a V5 repo
+# and sync with first repo
+#
+$ cd /tmp
+$ git clone /tmp/repov6/ repov5
+Cloning into 'repov5'...
+done.
+$ cd repov5
+$ git annex init "repo v5"
+$ git remote add repov6 /tmp/repov6
+$ git annex sync
+andrew@bumblebee /tmp/repov5$ ls -l image1.png 
+… image1.png -> .git/annex/objects/zJ/7J/SHA256E-s17691--c98e6eae515fec6a094e2b4bdcd221ba089a5e6073f6c365f6d4c829243c8aa5.png/SHA256E-s17691--c98e6eae515fec6a094e2b4bdcd221ba089a5e6073f6c365f6d4c829243c8aa5.png
+
+# this repo is still at version 5
+$ git annex version
+…
+local repository version: 5
+…
+
+# now lets move the unlocked not present image file
+$ mkdir imagesFolder
+$ mv image1.png imagesFolder/
+$ git add -A .
+$ git commit -m "moved file"
+
+## oops I accidentally committed my annexed file using git add from a v5 repo…
+## since I am used to using git add commands in my v6 repos
+
+#
+# Now lets sync
+#
+$ cd /tmp/repov6
+$ git remote add repov5 /tmp/repov5
+$ git annex sync
+$ ls -laL imagesFolder/image1.png 
+…  17691 Dec 10 21:26 imagesFolder/image1.png
+
+# 
+# Now lets try to get this to our V5 repo
+#
+$ cd /tmp/repov5
+$ git annex sync
+$ git annex info imagesFolder2/image1.png
+fatal: Not a valid object name imagesFolder2/image1.png
+info imagesFolder2/image1.png (not a directory or an annexed file or a treeish or a remote or a uuid) failed
+a
+
+# the following does not help
+$ git annex fsck
+$ git annex repair
+
+# the following resolves the issue
+$ git annex upgrade	
+
+
+# If the problem is with the git-annex assistant, paste in .git/annex/daemon.log
+
+
+# End of transcript or log.
+"""]]
+
+### Have you had any luck using git-annex before? (Sometimes we get tired of reading bug reports all day and a lil' positive end note does wonders)
+Yes
+
+> I don't consider this a bug, so [[done]] --[[Joey]]
diff --git a/doc/bugs/issue_with_syncing_between_v5_and_v6_repos/comment_1_4c3468a17c454cb344b3a944c0f71889._comment b/doc/bugs/issue_with_syncing_between_v5_and_v6_repos/comment_1_4c3468a17c454cb344b3a944c0f71889._comment
new file mode 100644
index 0000000000..9db9c27c4c
--- /dev/null
+++ b/doc/bugs/issue_with_syncing_between_v5_and_v6_repos/comment_1_4c3468a17c454cb344b3a944c0f71889._comment
@@ -0,0 +1,27 @@
+[[!comment format=mdwn
+ username="joey"
+ subject="""comment 1"""
+ date="2017-12-11T16:59:52Z"
+ content="""
+The problem is much simpler than that: Files that are unlocked in v6
+repositories are not treated as annexed files at all in v5 repositories.
+You cannot `git annex get` them or anything. 
+
+So, the simpler reproduction recipe is to `git add` a file to a v6
+repository and then in a v5 clone, `git annex get` and it won't say
+anything, because that's not an annexed file as far as it's concerned.
+
+It's also very easy to deal with this situation if you've gotten into it --
+all you need to do is `git annex lock` the files in a v6 repository and
+commit it, and then the v5 repository will be able to access them.
+
+A good way to avoid the problem while still using the features of v6
+in your v6 repository is to run `git annex adjust --unlock` in there.
+Then commits of unlocked files will automatically be translated to locked
+files that are compatible with v5.
+
+git-annex v5 can't possibly treat v6 unlocked files as annexed files.
+It could warn about the problem, but this would complicate it with needing
+to check if every non-symlink file is a v6 unlocked file. I don't feel
+this problem warrants that complication.
+"""]]
diff --git a/doc/bugs/issue_with_syncing_between_v5_and_v6_repos/comment_2_88a25974db9d544e10e6977056f49555._comment b/doc/bugs/issue_with_syncing_between_v5_and_v6_repos/comment_2_88a25974db9d544e10e6977056f49555._comment
new file mode 100644
index 0000000000..03d1214f14
--- /dev/null
+++ b/doc/bugs/issue_with_syncing_between_v5_and_v6_repos/comment_2_88a25974db9d544e10e6977056f49555._comment
@@ -0,0 +1,10 @@
+[[!comment format=mdwn
+ username="andrew"
+ avatar="http://cdn.libravatar.org/avatar/acc0ece1eedf07dd9631e7d7d343c435"
+ subject="makes sense"
+ date="2017-12-12T00:01:37Z"
+ content="""
+Thanks. Seems reasonable.
+
+Andrew
+"""]]
diff --git a/doc/bugs/jobs.mdwn b/doc/bugs/jobs.mdwn
new file mode 100644
index 0000000000..e32af703e9
--- /dev/null
+++ b/doc/bugs/jobs.mdwn
@@ -0,0 +1,83 @@
+### Please describe the problem.
+
+After many addeded files, a have error:
+Another git process seems to be running in this repository, e.g.
+an editor opened by 'git commit'. Please make sure all processes
+are terminated then try again. If it still fails, a git process
+may have crashed in this repository earlier:
+remove the file manually to continue.
+fatal: Unable to create '/home/syncer/updates/.git/index.lock': File exists.
+
+Another git process seems to be running in this repository, e.g.
+an editor opened by 'git commit'. Please make sure all processes
+are terminated then try again. If it still fails, a git process
+may have crashed in this repository earlier:
+remove the file manually to continue.
+fatal: Unable to create '/home/syncer/updates/.git/index.lock': File exists.
+
+Another git process seems to be running in this repository, e.g.
+an editor opened by 'git commit'. Please make sure all processes
+are terminated then try again. If it still fails, a git process
+may have crashed in this repository earlier:
+remove the file manually to continue.
+fatal: Unable to create '/home/syncer/updates/.git/index.lock': File exists.
+
+Another git process seems to be running in this repository, e.g.
+an editor opened by 'git commit'. Please make sure all processes
+are terminated then try again. If it still fails, a git process
+may have crashed in this repository earlier:
+remove the file manually to continue.
+
+git-annex: user error (xargs ["-0","git","--git-dir=.git","--work-tree=.","--literal-pathspecs","add","--"] exited 123)
+failed
+fatal: Unable to create '/home/syncer/updates/.git/index.lock': File exists.
+
+
+### What steps will reproduce the problem?
+cd /home/syncer/updates
+
+git init
+
+git annex init --version=6
+
+git clone ssh://git-annex.drweb.loc/home/syncer/updates
+
+echo "* annex.backend=WORM" > .gitattributes
+
+git add .gitattributes
+
+git commit -m "set attributes"
+
+git annex initremote myrsync type=rsync rsyncurl=git-annex.drweb.loc:/home/syncer/rsync_remote encryption=none
+
+git annex copy --all --to myrsync
+
+add and modify files
+
+git annex add -J8 .
+
+### What version of git-annex are you using? On what operating system?
+git-annex version: 6.20170101.1
+build flags: Assistant Webapp Pairing Testsuite S3(multipartupload)(storageclasses) WebDAV Inotify DBus DesktopNotify ConcurrentOutput TorrentParser MagicMime Feeds Quvi
+key/value backends: SHA256E SHA256 SHA512E SHA512 SHA224E SHA224 SHA384E SHA384 SHA3_256E SHA3_256 SHA3_512E SHA3_512 SHA3_224E SHA3_224 SHA3_384E SHA3_384 SKEIN256E SKEIN256 SKEIN512E SKEIN512 SHA1E SHA1 MD5E MD5 WORM URL
+remote types: git gcrypt p2p S3 bup directory rsync web bittorrent webdav tahoe glacier ddar hook external
+local repository version: 6
+supported repository versions: 3 5 6
+upgrade supported from repository versions: 0 1 2 3 4 5
+operating system: linux x86_64
+
+### Please provide any additional information below.
+
+[[!format sh """
+# If you can, paste a complete transcript of the problem occurring here.
+# If the problem is with the git-annex assistant, paste in .git/annex/daemon.log
+
+
+# End of transcript or log.
+"""]]
+
+### Have you had any luck using git-annex before? (Sometimes we get tired of reading bug reports all day and a lil' positive end note does wonders)
+
+[[!meta title="git annex add -J crash due to index somehow getting locked"]]
+
+> [[fixed|done]] --[[Joey]]
diff --git a/doc/bugs/jobs/comment_1_2f37448b1ab618cf485d5335d3b2977d._comment b/doc/bugs/jobs/comment_1_2f37448b1ab618cf485d5335d3b2977d._comment
new file mode 100644
index 0000000000..8f90a0bec9
--- /dev/null
+++ b/doc/bugs/jobs/comment_1_2f37448b1ab618cf485d5335d3b2977d._comment
@@ -0,0 +1,17 @@
+[[!comment format=mdwn
+ username="joey"
+ subject="""comment 1"""
+ date="2018-08-28T16:16:03Z"
+ content="""
+Reproduced by creating 10000 files and running 
+`git -c annex.queuesize=100 annex add -J8`
+
+Apparently it runs two concurrent git adds
+when flushing the queue in some circumstances.
+The smaller queue size must make it easier to reproduce;
+without it all 10000 files get added ok here.
+
+It's not specific to v6 at all.
+
+Two worker threads are flushing their queues at the same time.
+"""]]
diff --git a/doc/bugs/json_should_be_utf8_regardless_of_locale.mdwn b/doc/bugs/json_should_be_utf8_regardless_of_locale.mdwn
new file mode 100644
index 0000000000..ac7275a77a
--- /dev/null
+++ b/doc/bugs/json_should_be_utf8_regardless_of_locale.mdwn
@@ -0,0 +1,26 @@
+json is defined as always utf-8. However, when LANG=C,
+git-annex --json currently outputs "file":"���������"
+instead of "file":"äöü東" for that utf-8 filename. --[[Joey]]
+
+This can also affect keys when they contain some non-utf8 from eg the
+extension. And metadata keys and values can contain non-utf8 and also get
+converted to json with similar results.
+
+Note that git-annex can operate on non-utf8 filenames and keys; 
+it's not defined what the json contains then, and it currently contains
+similar garbage.
+
+This happens because aeson's instance of ToJSON for Char uses
+Text.singleton, and Text does not handle ghc's filesystem encoding
+for String. Instead it defaults to `\65533` for each byte encoded with the
+filesystem encoding.
+
+So, git-annex will need to convert filenames and keys and anything else
+that might use the filesystem encoding to Text itself in some
+way that does respect the filesystem encoding. Ie, use encodeBS to convert
+it to a ByteString and then Data.Text.Encoding.decodeUtf8. 
+
+> [[done]] that. --[[Joey]]
+
+What about git-annex commands that take json as input,
+when run in a non-utf8 locale? Tested that, it is handled ok. --[[Joey]]
diff --git a/doc/bugs/keeps_demanding_ssh_key_even_if_all_sync__39__s_turned_down.mdwn b/doc/bugs/keeps_demanding_ssh_key_even_if_all_sync__39__s_turned_down.mdwn
new file mode 100644
index 0000000000..adeb531e1a
--- /dev/null
+++ b/doc/bugs/keeps_demanding_ssh_key_even_if_all_sync__39__s_turned_down.mdwn
@@ -0,0 +1,60 @@
+### Please describe the problem.
+
+May be I was simply wrong to assume that if I disable all sync'ing for all the repos, assistant wouldn't try to contact those remote hosts. But it still does
+
+In the logs, since no time stamp per each line, hard to say either recent ones just continuation of previous entries, or new ones:
+
+[[
+
+[2015-05-18 10:29:39 EDT] Watcher: Performing startup scan
+(started...) ssh: connect to host vagus.cns.dartmouth.edu port 22: No route to host^M
+ssh: connect to host vagus.cns.dartmouth.edu port 22: No route to host^M
+ssh: connect to host vagus.cns.dartmouth.edu port 22: No route to host^M
+ssh: connect to host vagus.cns.dartmouth.edu port 22: No route to host^M
+ssh: connect to host vagus.cns.dartmouth.edu port 22: No route to host^M
+ssh: connect to host vagus.cns.dartmouth.edu port 22: No route to host^M
+...
+]]
+
+and in another
+
+[[
+fsck xppaut_6.11b+1.dfsg.orig.tar.gz ok
+fsck xserver-xorg-input-synaptics_1.6.2+git44-ge28575b.orig.tar.gz ok
+fsck xserver-xorg-video-intel_2.99.917.orig.tar.gz (checksum...)
+ok
+(recording state in git...)
+  ** No known copies exist of python-mne_0.8.3+dfsg.orig.tar.gz
+git-annex: fsck: 1 failed
+
+(process:3954): Gtk-WARNING **: Locale not supported by C library.
+    Using the fallback 'C' locale.
+Write failed: Broken pipe^M
+
+(process:20140): Gtk-WARNING **: Locale not supported by C library.
+    Using the fallback 'C' locale.
+
+(process:20178): Gtk-WARNING **: Locale not supported by C library.
+    Using the fallback 'C' locale.
+git-annex: Daemon is already running.
+
+(process:29035): Gtk-WARNING **: Locale not supported by C library.
+    Using the fallback 'C' locale.
+
+(process:29184): Gtk-WARNING **: Locale not supported by C library.
+    Using the fallback 'C' locale.
+
+(process:30127): Gtk-WARNING **: Locale not supported by C library.
+    Using the fallback 'C' locale.
+
+(process:30817): Gtk-WARNING **: Locale not supported by C library.
+    Using the fallback 'C' locale.
+Write failed: Broken pipe^M
+]]
+
+that host (vagus) is RIP (need to remove from syncs)
+btw which are falling into daemon.log.1 not .log
+
+### What version of git-annex are you using? On what operating system?
+
+5.20150509+gitga10e45d-1~ndall+1
diff --git a/doc/bugs/keeps_demanding_ssh_key_even_if_all_sync__39__s_turned_down/comment_1_e4b110f61502b5d37117ec65ec39896b._comment b/doc/bugs/keeps_demanding_ssh_key_even_if_all_sync__39__s_turned_down/comment_1_e4b110f61502b5d37117ec65ec39896b._comment
new file mode 100644
index 0000000000..faaa831ddc
--- /dev/null
+++ b/doc/bugs/keeps_demanding_ssh_key_even_if_all_sync__39__s_turned_down/comment_1_e4b110f61502b5d37117ec65ec39896b._comment
@@ -0,0 +1,12 @@
+[[!comment format=mdwn
+ username="joey"
+ subject="""comment 1"""
+ date="2015-07-06T20:03:32Z"
+ content="""
+I'm having a hard time following what you're trying to say here.
+
+What is the bad behavior you are seeing? How does the fsck output you
+posted relate the the problem? What git remotes are configured? What is the
+contents of your .git/config? How did you disable the vagus repository?
+Have you restarted the assistant after disabling it?
+"""]]
diff --git a/doc/bugs/linux_standalone_git_annex_won__39__t_work_in_a_directory_whose_path_contains___58___or___59__.mdwn b/doc/bugs/linux_standalone_git_annex_won__39__t_work_in_a_directory_whose_path_contains___58___or___59__.mdwn
new file mode 100644
index 0000000000..c660f065b8
--- /dev/null
+++ b/doc/bugs/linux_standalone_git_annex_won__39__t_work_in_a_directory_whose_path_contains___58___or___59__.mdwn
@@ -0,0 +1,48 @@
+### Please describe the problem.
+
+The shim for the standalone version of git annex invokes the dynamic linker with --library-path set appropriately.  glibc's parsing of library-path treats : and ; as separators and has no quoting syntax for those characters.  Also path items have to be absolute paths so you can't avoid having the whole path.
+
+As a result if you install the standalone version in a directory whose path includes either of those characters it will produce a library-path that's interpreted incorrectly leading to it looking for the libraries in the wrong place and potentially eventually using the system's installed versions if they are available.
+
+### What steps will reproduce the problem?
+
+1. Find a system with as few libraries installed as possible.  libnettle.so.6 is an easy one to avoid.
+2. create a directory called 'test;test'
+3. install git annex standalone in 'test;test'
+4. use the standalone git annex to do a 'git annex get'
+5. there will be an error saying it can't find libnettle.so.6
+
+### What version of git-annex are you using? On what operating system?
+
+This bug applies to any version.  with a ':' it will probably occur on any unix, with a ';' it will occur on at least Linux and Solaris.
+
+### Please provide any additional information below.
+
+This bug can't be fixed.  It's unlikely to trip people up but might with a bad combination of label-based drive mounting and odd drive labels.  The worst case would be incompatible libraries on the system install which could lead to data corruption.
+
+I recommend that the git annex shim should check for ':' or ';' in the path and exit non-zero if they are found.
+
+[[!format sh """
+# If you can, paste a complete transcript of the problem occurring here.
+# If the problem is with the git-annex assistant, paste in .git/annex/daemon.log
+
+jdamery-iabak@oklina:/mnt/iabak-ext;/IA.BAK/shard9$ PATH=../git-annex.linux/:$PATH git annex get prelinger_library/11annualreport00unitrich/11annualreport00unitrich_raw_jp2.zip
+get prelinger_library/11annualreport00unitrich/11annualreport00unitrich_raw_jp2.zip (from web...)
+/mnt/iabak-ext;/IA.BAK/git-annex.linux/shimmed/wget/wget: error while loading shared libraries: libnettle.so.6: cannot open shared object file: No such file or directory
+
+  Unable to access these remotes: web
+
+  Try making some of these repositories available:
+        00000000-0000-0000-0000-000000000001 -- web
+failed
+git-annex: get: 1 failed
+
+# End of transcript or log.
+"""]]
+
+> Actually, this bug is fixable. And I have! I made it detect the problem
+> path, and then fall back to making a directory in /tmp, symlinking the
+> git-annex.linux directory into it, and using that as its base directory.
+> 
+> Of course, that's a suboptimal configuration, but better than having some
+> configurations where it doesn't work. [[done]] --[[Joey]]
diff --git a/doc/bugs/linux_standalone_git_annex_won__39__t_work_in_a_directory_whose_path_contains___58___or___59__/comment_1_a98072047799a0c9ba8c5589471f6a7b._comment b/doc/bugs/linux_standalone_git_annex_won__39__t_work_in_a_directory_whose_path_contains___58___or___59__/comment_1_a98072047799a0c9ba8c5589471f6a7b._comment
new file mode 100644
index 0000000000..297a815213
--- /dev/null
+++ b/doc/bugs/linux_standalone_git_annex_won__39__t_work_in_a_directory_whose_path_contains___58___or___59__/comment_1_a98072047799a0c9ba8c5589471f6a7b._comment
@@ -0,0 +1,11 @@
+[[!comment format=mdwn
+ username="joey"
+ subject="""comment 1"""
+ date="2015-08-04T19:34:34Z"
+ content="""
+Nice analysis.
+
+I can't see any way to avoid this problem. So, seems like a "don't do that
+then" situation. And indeed, it makes sense to detect it and fail in a
+comprehansible way.
+"""]]
diff --git a/doc/bugs/lookupkey_started_to_spit_out___34__debug__34___messages_to_stdout.mdwn b/doc/bugs/lookupkey_started_to_spit_out___34__debug__34___messages_to_stdout.mdwn
new file mode 100644
index 0000000000..265005c9ed
--- /dev/null
+++ b/doc/bugs/lookupkey_started_to_spit_out___34__debug__34___messages_to_stdout.mdwn
@@ -0,0 +1,41 @@
+### Please describe the problem.
+
+with previous snapshot (6.20160119+gitgb2a2f5d-1~ndall+1) you just get a clean output
+
+[[!format sh """
+hopa:~/.tmp
+$> git clone /home/yoh/.tmp/datalad_temp_clone_url_RmXPyt 123
+Cloning into '123'...                    
+done.
+
+$> cd 123 
+INFO.txt  test-annex.dat@  test.dat
+
+$> git annex lookupkey test-annex.dat                                        
+SHA256E-s4--181210f8f9c779c26da1d9b2075bde0127302ee0e3fca38c9a83f5b1dd8e5d3b.dat
+
+"""]]
+
+with new one (6.20160126+gitg65f4442-1~ndall+1) you get it polluted in such a freshly cloned repo with
+
+[[!format sh """
+$> git annex lookupkey test-annex.dat                        
+(merging origin/git-annex into git-annex...)
+(recording state in git...)
+SHA256E-s4--181210f8f9c779c26da1d9b2075bde0127302ee0e3fca38c9a83f5b1dd8e5d3b.dat
+"""]]
+
+and since I am a "dude" who likes "devnulling" things for some reason (I hope reason is clear here ;)):
+
+[[!format sh """
+$> git annex lookupkey test-annex.dat 2>/dev/null            
+(merging origin/git-annex into git-annex...)     
+(recording state in git...)
+SHA256E-s4--181210f8f9c779c26da1d9b2075bde0127302ee0e3fca38c9a83f5b1dd8e5d3b.dat
+"""]]
+
+I guess I will just skip all lines starting with ( for now but thought to let you know about such changed behavior which might complicate pipelining etc
+
+[[!meta author=yoh]]
+
+> [[fixed|done]] --[[Joey]]
diff --git a/doc/bugs/make_install_doesn__39__t_create_git-annex-shell.mdwn b/doc/bugs/make_install_doesn__39__t_create_git-annex-shell.mdwn
new file mode 100644
index 0000000000..471c91e9d4
--- /dev/null
+++ b/doc/bugs/make_install_doesn__39__t_create_git-annex-shell.mdwn
@@ -0,0 +1,62 @@
+### Please describe the problem.
+There is no git-annex-shell command in path and thus sync with ssh server fails. If I symlink /usr/bin/git-annex-shell -> /usr/bin/git-annex, it complains that key is already present in annex and errors out. There is executable git-annex-shell in server's ~/.ssh/
+
+### What steps will reproduce the problem?
+Start webapp to sync local repo with ssh server.
+
+### What version of git-annex are you using? On what operating system?
+git-annex 4.20130627 on Gentoo
+
+### Please provide any additional information below.
+
+[[!format sh """
+# If you can, paste a complete transcript of the problem occurring here.
+# If the problem is with the git-annex assistant, paste in .git/annex/daemon.log
+
+[2013-07-08 17:28:48 EEST] main: starting assistant version 4.20130627
+[2013-07-08 17:28:49 EEST] TransferScanner: Syncing with netbook_Lit, sigma_Lit 
+
+  dbus failed; falling back to mtab polling (SocketError {socketErrorMessage = "connect: does not exist (Connection refused)", socketErrorFatal = True, socketErrorAddress = Just (Address "unix:abstract=/tmp/dbus-HBxh6EyMJ3,guid=adc3101676daede2a128013351daa535")})
+Already up-to-date.
+
+(scanning...) [2013-07-08 17:28:49 EEST] Watcher: Performing startup scan
+Already up-to-date.
+Already up-to-date.
+
+
+(started...) Everything up-to-date
+Everything up-to-date
+bash: git-annex-shell: command not found
+rsync: connection unexpectedly closed (0 bytes received so far) [sender]
+rsync error: remote command not found (code 127) at io.c(605) [sender=3.0.9]
+bash: git-annex-shell: command not found
+rsync: connection unexpectedly closed (0 bytes received so far) [sender]
+rsync error: remote command not found (code 127) at io.c(605) [sender=3.0.9]
+
+
+[2013-07-08 17:50:23 EEST] main: starting assistant version 4.20130627
+[2013-07-08 17:50:23 EEST] TransferScanner: Syncing with netbook_Lit, sigma_Lit 
+
+  dbus failed; falling back to mtab polling (SocketError {socketErrorMessage = "connect: does not exist (Connection refused)", socketErrorFatal = True, socketErrorAddress = Just (Address "unix:abstract=/tmp/dbus-HBxh6EyMJ3,guid=adc3101676daede2a128013351daa535")})
+Already up-to-date.
+
+(scanning...) [2013-07-08 17:50:23 EEST] Watcher: Performing startup scan
+Already up-to-date.
+Already up-to-date.
+
+
+(started...) Everything up-to-date
+Everything up-to-date
+git-annex-shell: key is already present in annex
+rsync: connection unexpectedly closed (0 bytes received so far) [sender]
+rsync error: error in rsync protocol data stream (code 12) at io.c(605) [sender=3.0.9]
+git-annex-shell: key is already present in annex
+rsync: connection unexpectedly closed (0 bytes received so far) [sender]
+rsync error: error in rsync protocol data stream (code 12) at io.c(605) [sender=3.0.9]
+
+
+
+# End of transcript or log.
+"""]]
+
+[[!tag done]]
diff --git a/doc/bugs/make_install_doesn__39__t_create_git-annex-shell/comment_1_8c20edd8c6483500f807528d616c6dfd._comment b/doc/bugs/make_install_doesn__39__t_create_git-annex-shell/comment_1_8c20edd8c6483500f807528d616c6dfd._comment
new file mode 100644
index 0000000000..ad32737237
--- /dev/null
+++ b/doc/bugs/make_install_doesn__39__t_create_git-annex-shell/comment_1_8c20edd8c6483500f807528d616c6dfd._comment
@@ -0,0 +1,14 @@
+[[!comment format=mdwn
+ username="http://joeyh.name/"
+ ip="4.153.254.222"
+ subject="comment 1"
+ date="2013-07-08T16:14:37Z"
+ content="""
+The Makefile contains, for the install target:
+
+[[!format sh \"\"\"
+        ln -sf git-annex $(DESTDIR)$(PREFIX)/bin/git-annex-shell
+\"\"\"]]
+
+How was git-annex installed on your server?
+"""]]
diff --git a/doc/bugs/make_install_doesn__39__t_create_git-annex-shell/comment_2_8b2cf0fe7219e0bc83fd326adbf26c8a._comment b/doc/bugs/make_install_doesn__39__t_create_git-annex-shell/comment_2_8b2cf0fe7219e0bc83fd326adbf26c8a._comment
new file mode 100644
index 0000000000..452b55d40e
--- /dev/null
+++ b/doc/bugs/make_install_doesn__39__t_create_git-annex-shell/comment_2_8b2cf0fe7219e0bc83fd326adbf26c8a._comment
@@ -0,0 +1,31 @@
+[[!comment format=mdwn
+ username="https://www.google.com/accounts/o8/id?id=AItOawm01ida6POv7vqyUYtOlymEbJTbrImAIzM"
+ nickname="Reinis"
+ subject="comment 2"
+ date="2013-07-08T18:09:50Z"
+ content="""
+That is Gentoo box and git-annex is installed using the following ebuild:
+
+[[https://github.com/gentoo-haskell/gentoo-haskell/blob/master/dev-vcs/git-annex/git-annex-4.20130627.ebuild]]
+
+which uses cabal to install it. It uses also the following eclass:
+
+[[http://sources.gentoo.org/cgi-bin/viewvc.cgi/gentoo-x86/eclass/haskell-cabal.eclass?revision=1.40&view=markup]]
+
+The main functions of interest there are cabal-bootstrap(), cabal-copy() and cabal_src_install(). I'm not familiar enough with haskell nor with cabal to follow trough in detail how the setup program is constructed and how it works, but the end result is that this is all that is installed. I don't know if normal cabal installs work as intended, maybe something there needs to be adjusted?
+
+[[!format sh \"\"\"
+ * Contents of dev-vcs/git-annex-4.20130627:
+/usr
+/usr/bin
+/usr/bin/git-annex
+/usr/lib64
+/usr/lib64/ghc-7.6.3
+/usr/lib64/ghc-7.6.3/gentoo
+/usr/lib64/ghc-7.6.3/gentoo/git-annex-4.20130627.conf
+/usr/share
+/usr/share/doc
+/usr/share/doc/git-annex-4.20130627
+/usr/share/doc/git-annex-4.20130627/COPYRIGHT.bz2
+\"\"\" ]]
+"""]]
diff --git a/doc/bugs/make_install_doesn__39__t_create_git-annex-shell/comment_3_25fe06eb127e59a4a07aeb52a5cfeabe._comment b/doc/bugs/make_install_doesn__39__t_create_git-annex-shell/comment_3_25fe06eb127e59a4a07aeb52a5cfeabe._comment
new file mode 100644
index 0000000000..fd97f6a21d
--- /dev/null
+++ b/doc/bugs/make_install_doesn__39__t_create_git-annex-shell/comment_3_25fe06eb127e59a4a07aeb52a5cfeabe._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="http://joeyh.name/"
+ ip="4.154.7.235"
+ subject="comment 3"
+ date="2013-07-08T18:22:18Z"
+ content="""
+`cabal install` also creates the git-annex-shell symlink. I have filed a bug on the ebuild: 
+"""]]
diff --git a/doc/bugs/make_install_doesn__39__t_create_git-annex-shell/comment_4_ec78032ba62d6918baa2c0b07ead5b50._comment b/doc/bugs/make_install_doesn__39__t_create_git-annex-shell/comment_4_ec78032ba62d6918baa2c0b07ead5b50._comment
new file mode 100644
index 0000000000..86f1b46f0b
--- /dev/null
+++ b/doc/bugs/make_install_doesn__39__t_create_git-annex-shell/comment_4_ec78032ba62d6918baa2c0b07ead5b50._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="https://www.google.com/accounts/o8/id?id=AItOawm01ida6POv7vqyUYtOlymEbJTbrImAIzM"
+ nickname="Reinis"
+ subject="comment 4"
+ date="2013-07-08T18:44:59Z"
+ content="""
+Thanks! That leaves the issue that even when manually creating the symlink it still errors out (second half of the log in original report), but that probably is another issue. Should I open separate report for it? There was something similar in [[http://git-annex.branchable.com/bugs/Rsync_remote_created_via_webapp_remains_empty/]], but the command= part was in the file on my local computer only and removing it didn't make any difference.
+"""]]
diff --git a/doc/bugs/man_page_for_command_misses_actual_command_in_the_synopsis_for_git-annex-checkpresentkey.mdwn b/doc/bugs/man_page_for_command_misses_actual_command_in_the_synopsis_for_git-annex-checkpresentkey.mdwn
new file mode 100644
index 0000000000..b5077f9109
--- /dev/null
+++ b/doc/bugs/man_page_for_command_misses_actual_command_in_the_synopsis_for_git-annex-checkpresentkey.mdwn
@@ -0,0 +1,18 @@
+[[!format sh """
+$> git annex version
+git-annex version: 6.20160126+gitg65f4442-1~ndall+1
+...
+$> man git-annex-checkpresentkey | head
+git-annex-checkpresentkey(1)               General Commands Manual               git-annex-checkpresentkey(1)
+
+NAME
+       git-annex-checkpresentkey - check if key is present in remote
+
+SYNOPSIS
+       git annex  key remote
+
+"""]]
+
+[[!meta author=yoh]]
+
+> [[fixed|done]] --[[Joey]]
diff --git a/doc/bugs/merge_causes_out_of_memory_on_large_repos.mdwn b/doc/bugs/merge_causes_out_of_memory_on_large_repos.mdwn
new file mode 100644
index 0000000000..c081f9799e
--- /dev/null
+++ b/doc/bugs/merge_causes_out_of_memory_on_large_repos.mdwn
@@ -0,0 +1,26 @@
+### Please describe the problem.
+
+`git annex merge` goes out of memory on a large repo.
+
+
+### What steps will reproduce the problem?
+
+    $ git annex merge
+    merge git-annex git-annex: out of memory (requested 1048576 bytes)
+
+### What version of git-annex are you using? On what operating system?
+
+    git-annex version: 4.20130827
+    build flags: Assistant Webapp Pairing Testsuite S3 WebDAV Inotify DBus XMPP DNS
+
+On debian wheezy i386
+
+### Please provide any additional information below.
+
+The repository contains a lot of files:
+
+    $ cd .git/objects
+    $ find . -type f | wc -l 
+    1091548
+
+[[!tag moreinfo]]
diff --git a/doc/bugs/merge_causes_out_of_memory_on_large_repos/comment_1_6d47485728ea65a9b555f8be7159dea5._comment b/doc/bugs/merge_causes_out_of_memory_on_large_repos/comment_1_6d47485728ea65a9b555f8be7159dea5._comment
new file mode 100644
index 0000000000..f87a9a84d1
--- /dev/null
+++ b/doc/bugs/merge_causes_out_of_memory_on_large_repos/comment_1_6d47485728ea65a9b555f8be7159dea5._comment
@@ -0,0 +1,10 @@
+[[!comment format=mdwn
+ username="https://www.google.com/accounts/o8/id?id=AItOawnxlx1UrzVhdy6_gFjzmF42x6QXxBUxg00"
+ nickname="Jakukyo"
+ subject="pre-commit hook of git annex"
+ date="2013-09-11T00:58:09Z"
+ content="""
+The pre-commit hook of git annex throws an 'out of memory' error, too.
+
+As a workaround, I've deleted ~/.git/hooks/pre-commit, and use `git merge` manually.
+"""]]
diff --git a/doc/bugs/merge_causes_out_of_memory_on_large_repos/comment_2_06723d13ecdaf87de5ff2b209e3c5198._comment b/doc/bugs/merge_causes_out_of_memory_on_large_repos/comment_2_06723d13ecdaf87de5ff2b209e3c5198._comment
new file mode 100644
index 0000000000..76a0d2fd48
--- /dev/null
+++ b/doc/bugs/merge_causes_out_of_memory_on_large_repos/comment_2_06723d13ecdaf87de5ff2b209e3c5198._comment
@@ -0,0 +1,10 @@
+[[!comment format=mdwn
+ username="http://joeyh.name/"
+ ip="4.154.2.134"
+ subject="comment 2"
+ date="2013-09-13T18:38:11Z"
+ content="""
+Can you run this so I can get a better idea of the size of this repository:
+
+git ls-tree -r git-annex | wc -l
+"""]]
diff --git a/doc/bugs/merge_causes_out_of_memory_on_large_repos/comment_3_9f83ef190547b291a715cda55b7977d4._comment b/doc/bugs/merge_causes_out_of_memory_on_large_repos/comment_3_9f83ef190547b291a715cda55b7977d4._comment
new file mode 100644
index 0000000000..11f03b39d9
--- /dev/null
+++ b/doc/bugs/merge_causes_out_of_memory_on_large_repos/comment_3_9f83ef190547b291a715cda55b7977d4._comment
@@ -0,0 +1,10 @@
+[[!comment format=mdwn
+ username="http://joeyh.name/"
+ ip="4.153.14.105"
+ subject="comment 3"
+ date="2013-09-19T18:53:14Z"
+ content="""
+I just fixed a memory leak that occurred when large files were checked directly into git (perhaps by accident by committing files manually when using direct mode). However, I don't think it affected `git annex merge`. 
+
+Still need information about how to reproduce this bug...
+"""]]
diff --git a/doc/bugs/merge_causes_out_of_memory_on_large_repos/comment_4_0e32ae0300472c56079cfbcd78a3e386._comment b/doc/bugs/merge_causes_out_of_memory_on_large_repos/comment_4_0e32ae0300472c56079cfbcd78a3e386._comment
new file mode 100644
index 0000000000..d1c447980c
--- /dev/null
+++ b/doc/bugs/merge_causes_out_of_memory_on_large_repos/comment_4_0e32ae0300472c56079cfbcd78a3e386._comment
@@ -0,0 +1,9 @@
+[[!comment format=mdwn
+ username="https://www.google.com/accounts/o8/id?id=AItOawnxlx1UrzVhdy6_gFjzmF42x6QXxBUxg00"
+ nickname="Jakukyo"
+ subject="size of repo"
+ date="2013-09-21T09:32:03Z"
+ content="""
+    $ git ls-tree -r git-annex | wc -l
+    29273
+"""]]
diff --git a/doc/bugs/merge_causes_out_of_memory_on_large_repos/comment_5_e8998716107e7ae8d0e8d332812517ad._comment b/doc/bugs/merge_causes_out_of_memory_on_large_repos/comment_5_e8998716107e7ae8d0e8d332812517ad._comment
new file mode 100644
index 0000000000..272dc2fd2d
--- /dev/null
+++ b/doc/bugs/merge_causes_out_of_memory_on_large_repos/comment_5_e8998716107e7ae8d0e8d332812517ad._comment
@@ -0,0 +1,14 @@
+[[!comment format=mdwn
+ username="http://joeyh.name/"
+ ip="4.152.108.220"
+ subject="comment 5"
+ date="2013-09-25T18:36:39Z"
+ content="""
+That doesn't look very big, I merge one 3x that large on a 128 mb machine.
+
+I think you will need to either email me privately so I can get a copy of your repository to investigate with ... or you can try to investigate on your own. 
+
+I think the first things I would try to debug this are to look over `git annex merge --debug` and see if I see anything unusual, and then I would probably `git checkout git-annex` in the repository, and wc -l on all the files and see if any file has a lot of lines, or is otherwise very large.
+
+If that found nothing, my next step would be to rebuild git-annex from source with memory profiling enabled, as explained in this book, and try to get a memory profiling graph that explained what was using up the memory. 
+"""]]
diff --git a/doc/bugs/metadata_view_does_not_vpop_to_original_view.mdwn b/doc/bugs/metadata_view_does_not_vpop_to_original_view.mdwn
new file mode 100644
index 0000000000..1d51181647
--- /dev/null
+++ b/doc/bugs/metadata_view_does_not_vpop_to_original_view.mdwn
@@ -0,0 +1,41 @@
+### Please describe the problem.
+
+I added some metadata to my music repository and tried a metadata-driven view. 
+
+    git annex view composer=* performer=
+
+When I wanted to return to the original list of directories and files, that fails. I now only see a list of directories that contain files which are actually on my laptop, and the generated views.
+
+When I did git annex vpop, I got the following:
+
+    frederik@freo:~/Music$ git annex vpop
+    vpop 1 
+    error: Your local changes to the following files would be overwritten by checkout:
+      Mozart, Wolfgang Amadeus/Academy of St. Martin in the Fields/01. Wolfgang Amadeus Mozart - Serenade No. 10 in B-flat major, KV 361 _Gran Partita_ - Ia. Largo_%Wolfgang Amadeus Mozart - Serenade No. 10 in B-flat major, KV 361 _Gran Partita_ (Academy of St. Martin in the Fields feat. conductor - Sir Neville Marriner)%.flac
+        Mozart, Wolfgang Amadeus/Academy of St. Martin in the Fields/02. Wolfgang Amadeus Mozart - Serenade No. 10 in B-flat major, KV 361 _Gran Partita_ - Ib. Allegro molto_%Wolfgang Amadeus Mozart - Serenade No. 10 in B-flat major, KV 361 _Gran Partita_ (Academy of St. Martin in the Fields feat. conductor - Sir Neville Marriner)%.flac
+
+I had not actually altered any of these files. I merely cd'd into the directory and did an ls to see what it looked like.
+
+The view is also not generated correctly, as re-running 
+
+    git annex view composer=* performer=
+
+gives me a list of directories with several directories that should not be present in the view.
+
+So far, I'm unable to go back to the original state.
+
+
+
+### What version of git-annex are you using? On what operating system?
+
+Debian/Jessie
+
+git-annex version: 5.20141125
+build flags: Assistant Webapp Webapp-secure Pairing Testsuite S3 WebDAV Inotify DBus DesktopNotify XMPP DNS Feeds Quvi TDFA CryptoHash
+key/value backends: SHA256E SHA1E SHA512E SHA224E SHA384E SKEIN256E SKEIN512E SHA256 SHA1 SHA512 SHA224 SHA384 SKEIN256 SKEIN512 WORM URL
+remote types: git gcrypt S3 bup directory rsync web webdav tahoe glacier ddar hook external
+local repository version: 5
+supported repository version: 5
+upgrade supported from repository versions: 0 1 2 4
+
+
diff --git a/doc/bugs/metadata_view_does_not_vpop_to_original_view/comment_1_91c69150533cb6c2147dcf2eb480ef39._comment b/doc/bugs/metadata_view_does_not_vpop_to_original_view/comment_1_91c69150533cb6c2147dcf2eb480ef39._comment
new file mode 100644
index 0000000000..bd66cb8e1b
--- /dev/null
+++ b/doc/bugs/metadata_view_does_not_vpop_to_original_view/comment_1_91c69150533cb6c2147dcf2eb480ef39._comment
@@ -0,0 +1,9 @@
+[[!comment format=mdwn
+ username="CandyAngel"
+ subject="comment 1"
+ date="2015-06-03T14:33:35Z"
+ content="""
+You should be able to get back to the original state (in terms of working directory) by checking out the 'master' branch (or whatever branch you started the view from):
+
+    git checkout master
+"""]]
diff --git a/doc/bugs/metadata_view_does_not_vpop_to_original_view/comment_2_ca1cc0b0e5c53daffd0b5ca78a6ac43f._comment b/doc/bugs/metadata_view_does_not_vpop_to_original_view/comment_2_ca1cc0b0e5c53daffd0b5ca78a6ac43f._comment
new file mode 100644
index 0000000000..a38b27f9e3
--- /dev/null
+++ b/doc/bugs/metadata_view_does_not_vpop_to_original_view/comment_2_ca1cc0b0e5c53daffd0b5ca78a6ac43f._comment
@@ -0,0 +1,11 @@
+[[!comment format=mdwn
+ username="CandyAngel"
+ subject="comment 2"
+ date="2015-06-03T14:40:12Z"
+ content="""
+Oh, you may also need to clean up directories files left over from the view with:
+
+    git clean -fd
+
+This will remove everything not ignored or tracked!
+"""]]
diff --git a/doc/bugs/metadata_view_does_not_vpop_to_original_view/comment_3_63bef9006a1e4ba2ebbfafbdf605a9e1._comment b/doc/bugs/metadata_view_does_not_vpop_to_original_view/comment_3_63bef9006a1e4ba2ebbfafbdf605a9e1._comment
new file mode 100644
index 0000000000..dffc6e2d5c
--- /dev/null
+++ b/doc/bugs/metadata_view_does_not_vpop_to_original_view/comment_3_63bef9006a1e4ba2ebbfafbdf605a9e1._comment
@@ -0,0 +1,13 @@
+[[!comment format=mdwn
+ username="joey"
+ subject="""comment 3"""
+ date="2015-06-03T17:25:28Z"
+ content="""
+The only way I can reproduce this is if I use `git annex edit` while inside
+the view.
+
+It might be that there's some kind of unclean git tree that can cause a
+problem when entering a view, but I was not able to find one.
+
+Please provide clear instructions how to reproduce this problem.
+"""]]
diff --git a/doc/bugs/metadata_view_does_not_vpop_to_original_view/comment_4_4873f0bc670b193cec3be327ee178a13._comment b/doc/bugs/metadata_view_does_not_vpop_to_original_view/comment_4_4873f0bc670b193cec3be327ee178a13._comment
new file mode 100644
index 0000000000..7dbdaf0fa9
--- /dev/null
+++ b/doc/bugs/metadata_view_does_not_vpop_to_original_view/comment_4_4873f0bc670b193cec3be327ee178a13._comment
@@ -0,0 +1,45 @@
+[[!comment format=mdwn
+ username="frederik@ffbea6a549cb3f460d110386c0f634c1ddc6a68a"
+ nickname="frederik"
+ subject="file name too long"
+ date="2015-06-04T04:49:23Z"
+ content="""
+OK, when I try git checkout master, it fails with some of these errors:
+
+    frederik@freo:~/Music$ git checkout master
+    error: cannot stat 'Händel, Georg Friedrich/Oregon Bach Festival Choir & Orchestra/09. George Frideric Handel - O Thou That Tellest Good Tidings to Zion--Air (Alto)_\%George Frideric Handel - The Messiah (feat. Oregon Bach Festival Choir & Orchestra -Rilling) (Disc 1 of 2)\%_%Händel, Georg Friedrich%Oregon Bach Festival Choir & Orchestra%.flac': File name too long
+
+These filenames have been created by git annex when creating the view. The original one is rather long too:
+
+    09. George Frideric Handel - O Thou That Tellest Good Tidings to Zion--Air (Alto).flac
+
+In directory:
+
+    Music/George Frideric Handel - The Messiah (feat. Oregon Bach Festival Choir & Orchestra -Rilling) (Disc 1 of 2)
+
+It looks like this has something to do with the problem.
+
+When I do git clean -fd, I only have the 2 directories left as created by the view. I still can't do git annex vpop anymore, as the reply is that I'm not in a view anymore.
+
+Some more info on that one file:
+
+    frederik@freo:~/Music/Händel, Georg Friedrich/Oregon Bach Festival Choir & Orchestra$ git annex metadata 09.\ George\ Frideric\ Handel\ -\ O\ Thou\ That\ Tellest\ Good\ Tidings\ to\ Zion--Air\ \(Alto\)_%George\ Frideric\ Handel\ -\ The\ Messiah\ \(feat.\ Oregon\ Bach\ Festival\ Choir\ \&\ Orchestra\ -Rilling\)\ \(Disc\ 1\ of\ 2\)%.flac
+    metadata 09. George Frideric Handel - O Thou That Tellest Good Tidings to Zion--Air (Alto)_%George Frideric Handel - The Messiah (feat. Oregon Bach Festival Choir & Orchestra -Rilling) (Disc 1 of 2)%.flac 
+      album=The Messiah
+      album-lastchanged=2015-06-03@13-27-45
+      codec=flac
+      codec-lastchanged=2015-06-03@13-27-45
+      composer=Händel, Georg Friedrich
+      composer-lastchanged=2015-06-03@13-27-45
+      genre=classical
+      genre-lastchanged=2015-06-03@13-27-45
+      lastchanged=2015-06-03@13-27-45
+      performer=Oregon Bach Festival Choir & Orchestra
+      performer-lastchanged=2015-06-03@13-27-45
+      tag=music
+      tag-lastchanged=2015-06-03@13-27-45
+    ok
+
+
+
+"""]]
diff --git a/doc/bugs/metadata_view_does_not_vpop_to_original_view/comment_5_bf63c9d3e0de468ffa5856dcf7c313b7._comment b/doc/bugs/metadata_view_does_not_vpop_to_original_view/comment_5_bf63c9d3e0de468ffa5856dcf7c313b7._comment
new file mode 100644
index 0000000000..2681d3aa87
--- /dev/null
+++ b/doc/bugs/metadata_view_does_not_vpop_to_original_view/comment_5_bf63c9d3e0de468ffa5856dcf7c313b7._comment
@@ -0,0 +1,22 @@
+[[!comment format=mdwn
+ username="CandyAngel"
+ subject="comment 5"
+ date="2015-06-04T09:05:18Z"
+ content="""
+I can at least reproduce the error.
+
+http://stackoverflow.com/questions/6571435/limit-on-file-name-length-in-bash
+
+    mkdir /tmp/pathtest
+    cd /tmp/pathtest
+    mkdir 'Händel, Georg Friedrich/Oregon Bach Festival Choir & Orchestra/'
+    mkdir: created directory ‘Händel, Georg Friedrich’
+    mkdir: created directory ‘Händel, Georg Friedrich/Oregon Bach Festival Choir & Orchestra/’
+    touch 'Händel, Georg Friedrich/Oregon Bach Festival Choir & Orchestra/09. George Frideric Handel - O Thou That Tellest Good Tidings to Zion--Air (Alto)_\%George Frideric Handel - The Messiah (feat. Oregon Bach Festival Choir & Orchestra -Rilling) (Disc 1 of 2)\%_%Händel, Georg Friedrich%Oregon Bach Festival Choir & Orchestra%.flac'
+    touch: cannot touch ‘Händel, Georg Friedrich/Oregon Bach Festival Choir & Orchestra/09. George Frideric Handel - O Thou That Tellest Good Tidings to Zion--Air (Alto)_\\%George Frideric Handel - The Messiah (feat. Oregon Bach Festival Choir & Orchestra -Rilling) (Disc 1 of 2)\\%_%Händel, Georg Friedrich%Oregon Bach Festival Choir & Orchestra%.flac’: File name too long
+
+In my case, I cannot create the file because the filename is 262 bytes long, and either ext3 or bash have a filename limit of 255 bytes.
+
+You may have to delete it using its inode or the \"rsync --delete with empty directory\" trick to get rid of it. Odd how git-annex managed to create it in the first place though..
+
+"""]]
diff --git a/doc/bugs/metadata_view_does_not_vpop_to_original_view/comment_6_83e389f5fdd58ffd62ead898ddbb5279._comment b/doc/bugs/metadata_view_does_not_vpop_to_original_view/comment_6_83e389f5fdd58ffd62ead898ddbb5279._comment
new file mode 100644
index 0000000000..ea507f001f
--- /dev/null
+++ b/doc/bugs/metadata_view_does_not_vpop_to_original_view/comment_6_83e389f5fdd58ffd62ead898ddbb5279._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="frederik@ffbea6a549cb3f460d110386c0f634c1ddc6a68a"
+ nickname="frederik"
+ subject="comment 6"
+ date="2015-06-04T11:54:20Z"
+ content="""
+I'm not really comfortable deleting anything at the moment, without knowing I'll be able to get the data back on my local annex. Having to transfer everything back from the remote will take a long while. I guess I could tar it up first.
+"""]]
diff --git a/doc/bugs/metadata_view_does_not_vpop_to_original_view/comment_7_a5511e21ee6c09d8edf2bca48482d848._comment b/doc/bugs/metadata_view_does_not_vpop_to_original_view/comment_7_a5511e21ee6c09d8edf2bca48482d848._comment
new file mode 100644
index 0000000000..cbfa90038d
--- /dev/null
+++ b/doc/bugs/metadata_view_does_not_vpop_to_original_view/comment_7_a5511e21ee6c09d8edf2bca48482d848._comment
@@ -0,0 +1,9 @@
+[[!comment format=mdwn
+ username="CandyAngel"
+ subject="comment 7"
+ date="2015-06-04T12:18:20Z"
+ content="""
+If I were in this situation, I would clone the broken repo, check the symlinks look correct in the clone, 'git annex reinit' it with the broken repos UUID, move .git/annex/objects to the clone, fsck and replace the broken one with the clone (just move the broken one aside if you want to keep it around).
+
+I don't think you can lose any data this way. Though any files you added while in the view (which you shouldn't do anyway) will be present in the annex, but have no symlink (they will show up in 'unused').
+"""]]
diff --git a/doc/bugs/metadata_view_does_not_vpop_to_original_view/comment_8_ef3e8e1c47984fc674e58646052205ef._comment b/doc/bugs/metadata_view_does_not_vpop_to_original_view/comment_8_ef3e8e1c47984fc674e58646052205ef._comment
new file mode 100644
index 0000000000..df5515ff14
--- /dev/null
+++ b/doc/bugs/metadata_view_does_not_vpop_to_original_view/comment_8_ef3e8e1c47984fc674e58646052205ef._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="frederik@ffbea6a549cb3f460d110386c0f634c1ddc6a68a"
+ nickname="frederik"
+ subject="Any more info needed"
+ date="2015-07-27T13:47:11Z"
+ content="""
+Hi Joey, while I was able to recover from the issue mentioned above, I have yet to try using the metadata views again out of uncertainty about the root cause of the problem I encountered. Is there anything more I could provide to determine what caused this problem and how to avoid it next time I try?
+"""]]
diff --git a/doc/bugs/migrate_and_move_duplicates_data.mdwn b/doc/bugs/migrate_and_move_duplicates_data.mdwn
new file mode 100644
index 0000000000..4f962cdb26
--- /dev/null
+++ b/doc/bugs/migrate_and_move_duplicates_data.mdwn
@@ -0,0 +1,36 @@
+### Please describe the problem.
+
+I have a main annex with ~2TB of data. In the past is was using SHA256 then I migrated to SHA256E . Recently it was becoming quite full so I took some spare HD and cloned it and moved data from the main to the spares. To my surprise, the main annex disk usage did not go down a bit.
+
+It took me some time to understand why . The problem is exemplified by the  shell script   .
+
+In short, if a annex is migrated to a new backend and afterwards files are moved, then the hardlinks are broken, and disk usage doubles.
+
+### What steps will reproduce the problem?
+
+run above script
+
+### What version of git-annex are you using? On what operating system?
+
+ 5.20141125  on Debian Jessie amd64
+
+### Please provide any additional information below.
+
+Of course a simple solution would be to drop all unused files. This is ugly , though, because it does not distinguish between
+(1) unused files that are previous copies of files I care about (2) unused files that are due to the problem described in the example, and that I do not care about.
+
+A more complex but more elegant solution would be:
+
+(a) when a file is migrated , the old and new objects in the annex are hardlinked; moreover two symlinks should be creates, so that git-annex knows at a glance which two files are hardlinked (see  for example)
+
+(b) when moving of copying files, all hardlinked versions whould be move/copied
+
+(c) when dropping , an option may be used to specify if all hardlinked versions should be dropped alltogether
+
+### bye
+
+and thanks, A.
+
+### ps
+
+I tried to attach two files to this bug report but failed
diff --git a/doc/bugs/migrate_and_move_duplicates_data/comment_1_ac3c5c141992c7b5d2cc36e085b0cba8._comment b/doc/bugs/migrate_and_move_duplicates_data/comment_1_ac3c5c141992c7b5d2cc36e085b0cba8._comment
new file mode 100644
index 0000000000..83af05970b
--- /dev/null
+++ b/doc/bugs/migrate_and_move_duplicates_data/comment_1_ac3c5c141992c7b5d2cc36e085b0cba8._comment
@@ -0,0 +1,18 @@
+[[!comment format=mdwn
+ username="joey"
+ subject="""comment 1"""
+ date="2015-07-17T15:41:53Z"
+ content="""
+I don't feel that migration is an important enough feature to complicate
+the rest of git-annex with special handling of multiple keys that point to
+the same content.
+
+You could have used `mv` in your use case to move the repo to the new drive
+while preseving hard links.
+
+What might be useful is for `git annex migrate` to write a list of the old keys
+someplace. These could then be dropped when the user wants to get rid of them,
+with mixing them up with other unused files. Although if you care about old
+versions of files and don't want to drop them as unused, it seems to me you'd
+also want to be able to access the old keys from before the migration.
+"""]]
diff --git a/doc/bugs/migrate_and_move_duplicates_data/comment_2_68a876f2eed5c446d92367aae4070408._comment b/doc/bugs/migrate_and_move_duplicates_data/comment_2_68a876f2eed5c446d92367aae4070408._comment
new file mode 100644
index 0000000000..8163bc212d
--- /dev/null
+++ b/doc/bugs/migrate_and_move_duplicates_data/comment_2_68a876f2eed5c446d92367aae4070408._comment
@@ -0,0 +1,16 @@
+[[!comment format=mdwn
+ username="mennucc1@6758d6a3817a0a6b29e8adbdb068c5ba35f6992b"
+ nickname="mennucc1"
+ subject="hack"
+ date="2015-07-18T09:16:03Z"
+ content="""
+hi  Joey, thanks for your interest;
+
+I created a script to address this problem
+
+
+
+it can relink keys that are not hardlinked anymore (-L option) ; it can use an unsed key to recreate a missin key (-R option) ; it can also drop redundant keys (-D option)
+
+a.
+"""]]
diff --git a/doc/bugs/migrated_files_not_showing_up_in_unused_list.mdwn b/doc/bugs/migrated_files_not_showing_up_in_unused_list.mdwn
new file mode 100644
index 0000000000..33776d13af
--- /dev/null
+++ b/doc/bugs/migrated_files_not_showing_up_in_unused_list.mdwn
@@ -0,0 +1,62 @@
+Word summary: After migrating from SHA256 to SHA256E I still have a ton of SHA256 files around that aren't being found by unused.
+
+Command outputs (see, specifically, the output of status showing number of SHA256 and SHA256E files):
+
+
+    greg@eeepc:/mnt/blackbox/Media/Pictures/Photos$ less .gitattributes
+    * annex.backend=SHA256E
+
+    greg@eeepc:/mnt/blackbox/Media/Pictures/Photos$ git-annex migrate .
+    (Recording state in git...)
+
+    greg@eeepc:/mnt/blackbox/Media/Pictures/Photos$ git-annex unused
+    unused . (checking for unused data...) (checking master...) (checking rose/master...) (checking x200s/master...)
+      Some partially transferred data exists in temporary files:
+        NUMBER  KEY
+        1       SHA256E-s15766010--8132a02a8b245eb9842e89c5e696df4e9c82d676f8dec3c6bb96892c19f99d51.jpg
+
+      To remove unwanted data: git-annex dropunused NUMBER
+
+    ok
+    (Recording state in git...)
+
+    greg@eeepc:/mnt/blackbox/Media/Pictures/Photos$ git-annex status
+    supported backends: SHA256E SHA1E SHA512E SHA224E SHA384E SHA256 SHA1 SHA512 SHA224 SHA384 WORM URL
+    supported remote types: git S3 bup directory rsync web hook
+    trusted repositories: 1
+            c0e4106e-2631-11e2-9749-1bfa37a61069 -- rose
+    semitrusted repositories: 3
+            00000000-0000-0000-0000-000000000001 -- web
+            9bd4077e-196c-11e2-9cc9-9faafb3e34ee -- x200s
+            c69d6fcc-18d1-11e2-9487-2fe6dbf0516b -- here (photos on eeepc)
+    untrusted repositories: 0
+    dead repositories: 1
+            3ebd5ac2-2092-11e2-856a-bb0203cce179 -- Photos on Rose
+    transfers in progress: 
+            downloading 2011/06/30/IMG_8180.jpg
+     from rose
+    available local disk space: 2 terabytes (+1 megabyte reserved)
+    temporary directory size: 9 megabytes (clean up with git-annex unused)
+    local annex keys: 36210
+    local annex size: 136 gigabytes
+    known annex keys: 23388
+    known annex size: 102 gigabytes
+    bloom filter size: 16 mebibytes (7.2% full)
+    backend usage: 
+            SHA256E: 37453
+            SHA256: 22145
+    (Recording state in git...)
+
+    greg@eeepc:/mnt/blackbox/Media/Pictures/Photos$ apt-cache policy git-annex
+    git-annex:
+      Installed: 3.20121017
+      Candidate: 3.20121017
+      Version table:
+     *** 3.20121017 0
+            600 http://ftp.us.debian.org/debian/ unstable/main i386 Packages
+            100 /var/lib/dpkg/status
+         3.20120629 0
+            650 http://ftp.us.debian.org/debian/ wheezy/main i386 Packages
+
+
+As Joey predicted, this took care of itself over time. Marking as [[bugs/done]].
diff --git a/doc/bugs/migrated_files_not_showing_up_in_unused_list/comment_1_2cfbf6693b051c758fe5efa5ee885829._comment b/doc/bugs/migrated_files_not_showing_up_in_unused_list/comment_1_2cfbf6693b051c758fe5efa5ee885829._comment
new file mode 100644
index 0000000000..ee5f056175
--- /dev/null
+++ b/doc/bugs/migrated_files_not_showing_up_in_unused_list/comment_1_2cfbf6693b051c758fe5efa5ee885829._comment
@@ -0,0 +1,16 @@
+[[!comment format=mdwn
+ username="http://joeyh.name/"
+ ip="4.152.108.27"
+ subject="comment 1"
+ date="2012-11-07T19:02:45Z"
+ content="""
+Notice where it says:
+
+> (checking master...) (checking rose/master...) (checking x200s/master...)
+
+AFAICS, every one of those branches still refers to the migrated keys. For one thing, `git annex migrate` makes changes, but it doesn't commit them to master. And then you need to wait until the other remote are updated to.
+
+Also, there's not really any point in worrying about getting rid of migrated keys, since `git-annex migrate` hard links the new key to the old key, so the old ones don't take up any additional disk space.
+
+In other words: This should take care of itself as you continue to use the repository.
+"""]]
diff --git a/doc/bugs/migrated_files_not_showing_up_in_unused_list/comment_2_acb1abeb32c3aba8ba65151afbea753c._comment b/doc/bugs/migrated_files_not_showing_up_in_unused_list/comment_2_acb1abeb32c3aba8ba65151afbea753c._comment
new file mode 100644
index 0000000000..ccb7115887
--- /dev/null
+++ b/doc/bugs/migrated_files_not_showing_up_in_unused_list/comment_2_acb1abeb32c3aba8ba65151afbea753c._comment
@@ -0,0 +1,10 @@
+[[!comment format=mdwn
+ username="http://grossmeier.net/"
+ nickname="greg"
+ subject="comment 2"
+ date="2012-11-15T04:11:55Z"
+ content="""
+You were right, this has taken care of itself.
+
+Thanks much for the solid piece of software.
+"""]]
diff --git a/doc/bugs/migrating_to_non-E_backend_still_adds_extensions.mdwn b/doc/bugs/migrating_to_non-E_backend_still_adds_extensions.mdwn
new file mode 100644
index 0000000000..cfe85cbfd5
--- /dev/null
+++ b/doc/bugs/migrating_to_non-E_backend_still_adds_extensions.mdwn
@@ -0,0 +1,49 @@
+### Please describe the problem.
+
+I'm migrating certain files from SHA256E to SHA256, to avoid problems caused by inconsistent-case extensions, and by weird filenames which git-annex misdetects as part of the extension. The obvious approach, `annex migrate --to SHA256`, does not work as expected – the new keys _still_ have the same extension appended. This kinda defeats the point.
+
+I can work around this by migrating to another hash first (e.g. to SHA1), then back.
+
+### What steps will reproduce the problem?
+
+1. Have a SHA256E or SHA1E file
+2. git annex migrate --backend=SHA256
+
+### What version of git-annex are you using? On what operating system?
+
+6.20180509-g0632c49c22, Linux
+
+### Please provide any additional information below.
+
+[[!format sh """
+# If you can, paste a complete transcript of the problem occurring here.
+# If the problem is with the git-annex assistant, paste in .git/annex/daemon.log
+
+$ ls -l
+lrwxrwxrwx 1 grawity grawity 211 May  2 07:58 EN_EXCH2003_ENT.ISO -> ../../../.git/annex/objects/41/Fq/SHA256E-s391118848--3b0a03ce8a821c98de7d3e67f9664f55b4bb7e13855721db993881bad501caf3.ISO/SHA256E-s391118848--3b0a03ce8a821c98de7d3e67f9664f55b4bb7e13855721db993881bad501caf3.ISO
+
+$ git annex migrate --backend=SHA256 --force
+migrate EN_EXCH2003_ENT.ISO (checksum...) ok
+(recording state in git...)
+
+$ ls -l
+lrwxrwxrwx 1 grawity grawity 209 May 22 08:15 EN_EXCH2003_ENT.ISO -> ../../../.git/annex/objects/09/3P/SHA256-s391118848--3b0a03ce8a821c98de7d3e67f9664f55b4bb7e13855721db993881bad501caf3.ISO/SHA256-s391118848--3b0a03ce8a821c98de7d3e67f9664f55b4bb7e13855721db993881bad501caf3.ISO
+
+$ git annex migrate --backend=SHA1
+migrate EN_EXCH2003_ENT.ISO (checksum...) (checksum...) ok
+(recording state in git...)
+
+$ git annex migrate --backend=SHA256
+migrate EN_EXCH2003_ENT.ISO (checksum...) (checksum...) ok
+(recording state in git...)
+
+$ ls -l
+lrwxrwxrwx 1 grawity grawity 201 May 22 08:17 EN_EXCH2003_ENT.ISO -> ../../../.git/annex/objects/xX/GW/SHA256-s391118848--3b0a03ce8a821c98de7d3e67f9664f55b4bb7e13855721db993881bad501caf3/SHA256-s391118848--3b0a03ce8a821c98de7d3e67f9664f55b4bb7e13855721db993881bad501caf3
+
+# End of transcript or log.
+"""]]
+
+### Have you had any luck using git-annex before? (Sometimes we get tired of reading bug reports all day and a lil' positive end note does wonders)
+
+
+> [[fixed|done]] --[[Joey]]
diff --git a/doc/bugs/migrating_to_non-E_backend_still_adds_extensions/comment_1_9fbdd6e150df22f26729b867ea3bd0bf._comment b/doc/bugs/migrating_to_non-E_backend_still_adds_extensions/comment_1_9fbdd6e150df22f26729b867ea3bd0bf._comment
new file mode 100644
index 0000000000..a3ab0e372f
--- /dev/null
+++ b/doc/bugs/migrating_to_non-E_backend_still_adds_extensions/comment_1_9fbdd6e150df22f26729b867ea3bd0bf._comment
@@ -0,0 +1,13 @@
+[[!comment format=mdwn
+ username="joey"
+ subject="""comment 1"""
+ date="2018-05-23T17:07:27Z"
+ content="""
+I suspect this is a bug in the fastMigrate code.
+
+Interestingly, the SHA256 key with an exctension works ok, eg fsck can
+verify the checksum.
+
+Also, a subsequent `git annex migrate --backend=SHA256E`
+results in a SHA256E key with no extension.
+"""]]
diff --git a/doc/bugs/migrating_to_non-E_backend_still_adds_extensions/comment_2_f9b73dc8344b5ca98257a0024e097535._comment b/doc/bugs/migrating_to_non-E_backend_still_adds_extensions/comment_2_f9b73dc8344b5ca98257a0024e097535._comment
new file mode 100644
index 0000000000..be48f61e40
--- /dev/null
+++ b/doc/bugs/migrating_to_non-E_backend_still_adds_extensions/comment_2_f9b73dc8344b5ca98257a0024e097535._comment
@@ -0,0 +1,27 @@
+[[!comment format=mdwn
+ username="joey"
+ subject="""comment 2"""
+ date="2018-05-23T17:18:35Z"
+ content="""
+The bug was that it got the test backwards for whether 
+it was supposed to be adding or removing the extension!
+
+Bug was introduced in [[!commit 9c4650358ca85a298b747bb897dbf4f8f891fa22]]
+over a year ago.
+
+The bogus SHA256 key with an extension tacked on at the end
+passes fsck because the code happens to always strip extensions from
+hashes, even if the key type is not supposed to include an extension.
+
+Fixed the bug. But this leaves the potential for these badly formed
+SHA256 keys with an extension on the end being in a repository and
+needing to keep code working for them. (The SHA256E keys without an
+extension that also result from the bug are technically not badly formed.)
+
+So, I also made migrate fix those badly formed keys. You have to specify
+--backend=SHA256, and then it will migrate the badly formed SHA256 key to
+a correctly formed SHA256 key.
+
+Also, git-annex fsck will now warn when it detects a key needing such a
+migration.
+"""]]
diff --git a/doc/bugs/minor_display_glitch_with_ssh_password_prompting_and_-J.mdwn b/doc/bugs/minor_display_glitch_with_ssh_password_prompting_and_-J.mdwn
new file mode 100644
index 0000000000..a8e0871a14
--- /dev/null
+++ b/doc/bugs/minor_display_glitch_with_ssh_password_prompting_and_-J.mdwn
@@ -0,0 +1,51 @@
+When using -J and there's a ssh password prompt (or other prompt eg ssh
+host key), the region-based display gets messed up by the ssh output. This
+is a minor display glitch; it's still fairly clear what git-annex is doing.
+
+The root problem is that the regional display code does not know the
+absolute cursor position. All cursor movement is relative. So when ssh
+display moves the cursor, all subsequent output goes to the wrong place.
+
+ansi-terminal has absolute cursor movement, but no way to query position.
+
+Some approaches to fix it:
+
+1. Allocate a slave pty and run ssh in there, forwarding IO from the slave
+   pty to the master pty. The ssh output is then added to the region that
+   it's prompting for the password for.
+   
+   Unix-specific and somewhat heavyweight solution.
+
+2. Set position to eg 0,0 when starting git-annex, and then the
+   absolute position can be calculated, and after ssh runs it can reset the
+   cursor to the previous position. 
+   
+   Would make -J take over the whole screen even if it's only transferring
+   1 file.
+
+3. Clear all regions before running the ssh command that can prompt,
+   (moving the cursor to the start of the first region),
+   and redraw them when it's done. So the ssh output would appear above the
+   redrawn regions.
+
+   This would cause some flicker in the common case where ssh does not have
+   any output. The N regions would display briefly, then be cleared, then
+   be redrawn. It might flicker multiple times, when multiple different
+   hosts are being accessed. 
+   
+   One way to avoid the flicker would be to first
+   try to ssh with password prompting disabled, and only if that fails do
+   regions need to be cleared for the ssh that will prompt. Also, since
+   we then know ssh will prompt, we can display the hostname as context for
+   the "Password:" prompt it uses.
+
+   Needs concurrent-output 1.8.0
+
+4. Find a way to add cursor position querying to ansi-terminal. Can it be
+   done portably?
+
+   See 
+
+--[[Joey]]
+
+> [[fixed|done]] using option #3. --[[Joey]]
diff --git a/doc/bugs/moreinfo.mdwn b/doc/bugs/moreinfo.mdwn
new file mode 100644
index 0000000000..579707dab1
--- /dev/null
+++ b/doc/bugs/moreinfo.mdwn
@@ -0,0 +1,2 @@
+This tags is for bugs needing more information from their submitter.
+Please respond to the bug and provide the requested information.
diff --git a/doc/bugs/move_violates_numcopies.mdwn b/doc/bugs/move_violates_numcopies.mdwn
new file mode 100644
index 0000000000..96598a7885
--- /dev/null
+++ b/doc/bugs/move_violates_numcopies.mdwn
@@ -0,0 +1,27 @@
+`git annex move` does not honor numcopies. It is the only
+git-annex command to not honor numcopies by default.
+
+This can be surprising, and it complicates git-annex's story about
+attempting to preserve numcopies since there's this exception on the side.
+
+Also, `git annex move --to untrusted-repo` drops the local copy even if the
+untrusted copy is the only remaining copy, which is another unique thing
+about move.
+
+Should `git annex move --safe` become the default, and `git annex move
+--unsafe` be needed to get the current behavior? 
+
+(Note that the `-u` short option makes that easy enough to type for those
+of us who have workflows using the current behavior.)
+
+Such a change would break workflows, and potentially quite a lot of
+examples in the documentation might need to be updated. Although with the
+default numcopies=1, the move behavior would not change (except when moving
+onto an untrusted repository, ), which will limit the imact some.
+
+There could be a transition period where move warns when run w/o
+--safe/--unsafe.
+
+--[[Joey]]
+
+> [[done]], using the "don't make it worse" approach. --[[Joey]]
diff --git a/doc/bugs/move_violates_numcopies/comment_1_36e22ecc569f097e4cc58c20d0fa4876._comment b/doc/bugs/move_violates_numcopies/comment_1_36e22ecc569f097e4cc58c20d0fa4876._comment
new file mode 100644
index 0000000000..d59c6ead8a
--- /dev/null
+++ b/doc/bugs/move_violates_numcopies/comment_1_36e22ecc569f097e4cc58c20d0fa4876._comment
@@ -0,0 +1,10 @@
+[[!comment format=mdwn
+ username="CandyAngel"
+ avatar="http://cdn.libravatar.org/avatar/15c0aade8bec5bf004f939dd73cf9ed8"
+ subject="comment 1"
+ date="2018-04-10T10:39:23Z"
+ content="""
+If git-annex knows it isn't going to do what I've instructed (move the file), I think it should just fail the command immediately, rather than copying but not dropping.
+
+You could make *--safe* fallback to copy (for those that want that behaviour) and *--unsafe* has the risky behaviour, but *move* should either do just that, move the file, or do nothing.
+"""]]
diff --git a/doc/bugs/move_violates_numcopies/comment_2_57299719a6e8f3615819da4fbc460da5._comment b/doc/bugs/move_violates_numcopies/comment_2_57299719a6e8f3615819da4fbc460da5._comment
new file mode 100644
index 0000000000..178164c811
--- /dev/null
+++ b/doc/bugs/move_violates_numcopies/comment_2_57299719a6e8f3615819da4fbc460da5._comment
@@ -0,0 +1,10 @@
+[[!comment format=mdwn
+ username="anarcat"
+ avatar="http://cdn.libravatar.org/avatar/4ad594c1e13211c1ad9edb81ce5110b7"
+ subject="force?"
+ date="2018-04-10T14:20:08Z"
+ content="""
+This definitely seems strange to me. i've been using git-annex for a long time, and I have never known `move` was unsafe by default. so I think that `--safe` should definitely be made defautl.
+
+but beyond that, I wouldn't use `--unsafe` as a new flag: we already have `--force` for `--drop` for example, shouldn't we reuse that here as well?
+"""]]
diff --git a/doc/bugs/move_violates_numcopies/comment_3_491d81f636a4da1deefd2d82063e2f05._comment b/doc/bugs/move_violates_numcopies/comment_3_491d81f636a4da1deefd2d82063e2f05._comment
new file mode 100644
index 0000000000..34abb9ff8b
--- /dev/null
+++ b/doc/bugs/move_violates_numcopies/comment_3_491d81f636a4da1deefd2d82063e2f05._comment
@@ -0,0 +1,13 @@
+[[!comment format=mdwn
+ username="richih@50508f31e0ee95720acd0120e16d6bdcad9d104b"
+ nickname="richih"
+ avatar="http://cdn.libravatar.org/avatar/499771047201f3eb29a462897b50a5f3"
+ subject="comment 3"
+ date="2018-04-10T15:49:35Z"
+ content="""
+I agree with anarcat.
+
+The core use case of git-annex is to maintain sets of known-good data. This function mainly relies on a directory structure, checksums, a defined minimum of copies, and tracking where they are. I would never have assumed that I would be able to get git-annex to go below the mincopies, at least not unless I was deep into the innards of git-annex and/or abusing --force or the like.
+
+-- RichiH
+"""]]
diff --git a/doc/bugs/move_violates_numcopies/comment_4_370ab3e7adbe6f1888eb5ebe542d18ad._comment b/doc/bugs/move_violates_numcopies/comment_4_370ab3e7adbe6f1888eb5ebe542d18ad._comment
new file mode 100644
index 0000000000..68fda3b6a9
--- /dev/null
+++ b/doc/bugs/move_violates_numcopies/comment_4_370ab3e7adbe6f1888eb5ebe542d18ad._comment
@@ -0,0 +1,21 @@
+[[!comment format=mdwn
+ username="joey"
+ subject="""comment 4"""
+ date="2018-04-10T17:04:14Z"
+ content="""
+Candyangel, thanks, failing is indeed the better thing to do
+than copying when it's not safe to move. That makes sense.
+
+Also, `git annex move` should honor required contents, and refuse to move
+content away from a repository that requires it.
+
+It would be ok to use --force instead of --unsafe, but it doesn't allow for
+a staged transition from --unsafe by default to --safe by default. But, if
+we decide a stanged transition is not needed, I would be inclined to use
+the --force.
+
+Move failing in these situations seems less likely to badly break existing
+workflows than move leaving both copies would; the caller will see
+that git-annex errored, rather than it silently leaving extra copies.
+So perhaps a staged transition could be skipped.
+"""]]
diff --git a/doc/bugs/move_violates_numcopies/comment_5_4ef4c6180c71326faa814bb096dab500._comment b/doc/bugs/move_violates_numcopies/comment_5_4ef4c6180c71326faa814bb096dab500._comment
new file mode 100644
index 0000000000..c60f554586
--- /dev/null
+++ b/doc/bugs/move_violates_numcopies/comment_5_4ef4c6180c71326faa814bb096dab500._comment
@@ -0,0 +1,32 @@
+[[!comment format=mdwn
+ username="joey"
+ subject="""comment 5"""
+ date="2018-04-10T22:39:30Z"
+ content="""
+Another way to approach this is: `move` should not make
+a situation worse, but is not required to make it better.
+
+That would allow moving a file from A to B when numcopies wants 2 copies
+but only one copy exists, because the file being on B is no worse than it
+being on A.
+
+But, if B already has a copy of the file, move would error rather than the
+current behavior of removing from A, when numcopies wants two copies.
+
+And, if B is untrusted (and A is not), or A has the file as required
+content, moving to B would also error, as in both situations it makes
+things worse.
+
+This seems better than the ideas above, because it keeps move a somewhat
+lowlevel operation, like it always has been, but no longer an unsafe one.
+It matches many of my uses of move, when perhaps I want more copies than I
+have, but can't currently spare the space (or am moving the file to a repo
+that will later let it get replicated elsewhere).
+
+It also means that after `git annex move --from`, the local repository will 
+always have the file present, rather than move sometimes failing before
+getting it due to numcopies. (And the converse with `--to`.)
+
+I think this is a small enough change from the current behavior of move
+that it can get away with not having a transition plan.
+"""]]
diff --git a/doc/bugs/move_violates_numcopies/comment_6_2ffb920ae8e5ebfacf9230a1a7c0c7c3._comment b/doc/bugs/move_violates_numcopies/comment_6_2ffb920ae8e5ebfacf9230a1a7c0c7c3._comment
new file mode 100644
index 0000000000..6bee1db8d1
--- /dev/null
+++ b/doc/bugs/move_violates_numcopies/comment_6_2ffb920ae8e5ebfacf9230a1a7c0c7c3._comment
@@ -0,0 +1,11 @@
+[[!comment format=mdwn
+ username="richih@50508f31e0ee95720acd0120e16d6bdcad9d104b"
+ nickname="richih"
+ avatar="http://cdn.libravatar.org/avatar/499771047201f3eb29a462897b50a5f3"
+ subject="comment 6"
+ date="2018-04-11T07:34:43Z"
+ content="""
+\"not making it worse\" sounds like acceptable middle ground to me. If you are not erroring out, at least printing a warning would be good. OTOH, this easily gets hidden in a flood of message if you're moving more than just a few files.
+
+Still, principle of least surprise would point towards not going against basic safety measures in any case.
+"""]]
diff --git a/doc/bugs/mysterious_dependency_of_git_annex_status_output_of_the_added_file.mdwn b/doc/bugs/mysterious_dependency_of_git_annex_status_output_of_the_added_file.mdwn
new file mode 100644
index 0000000000..a2c1b576a2
--- /dev/null
+++ b/doc/bugs/mysterious_dependency_of_git_annex_status_output_of_the_added_file.mdwn
@@ -0,0 +1,83 @@
+### Please describe the problem.
+
+in v6 mode -- Result depends on having a good sleep before running 'git annex add'.
+
+Without sleep, git annex manages first to stage file to be committed into git, but then also modifies it to be added into annex  (this is not shown above -- just inspect that repository obtained without having any sleep)
+
+I guess relates to http://git-annex.branchable.com/bugs/Too_difficult_if_not_impossible_to_explicitly_add__47__keep_file_under_git___40__not_annex__41___in_v6_without_employing_.gitattributes/
+
+### What steps will reproduce the problem?
+
+Run http://www.onerussian.com/tmp/ga-3.sh  twice:  once giving 0 secs to sleep, and then 1 (or about 0.3 might work as well)
+
+
+### What version of git-annex are you using? On what operating system?
+
+6.20170209+gitg16be7b5cc-1~ndall+1
+
+### Please provide any additional information below.
+
+if we just proceed with the script (init, add, status) without any delays -- git annex status would report it 
+[[!format sh """
+$> ./ga-3.sh 0
++ s=0
+++ mktemp -d
++ d=/home/yoh/.tmp/tmp.d6g0E7scxt
++ echo 'directory: /home/yoh/.tmp/tmp.d6g0E7scxt'
+directory: /home/yoh/.tmp/tmp.d6g0E7scxt
++ cd /home/yoh/.tmp/tmp.d6g0E7scxt
++ git init
+Initialized empty Git repository in /tmp/tmp.d6g0E7scxt/.git/
++ git annex init --version=6
+init  ok
+(recording state in git...)
++ sed -i -e 's,pre-commit ,pre-commit --debug ,g' .git/hooks/pre-commit
++ echo 'I: creating a file'
+I: creating a file
++ echo whatever
++ sleep 0
++ git -c annex.largefiles=nothing annex --debug add file5
+[2017-02-17 10:19:48.91932971] read: git ["--git-dir=.git","--work-tree=.","--literal-pathspecs","ls-files","--others","--exclude-standard","-z","--","file5"]
+add file5 (non-large file; adding content to git repository) ok
+[2017-02-17 10:19:48.923428344] read: git ["--git-dir=.git","--work-tree=.","--literal-pathspecs","ls-files","--modified","-z","--","file5"]
+(recording state in git...)
+[2017-02-17 10:19:48.927922289] feed: xargs ["-0","git","--git-dir=.git","--work-tree=.","--literal-pathspecs","add","--"]
+[2017-02-17 10:19:48.956812867] process done ExitSuccess
++ git annex status
+M file5
+"""]]
+
+And if we wait just a bit before running add -- we would get it reported added
+[[!format sh """
+hopa:~/.tmp
+$> ./ga-3.sh 1
++ s=1
+++ mktemp -d
++ d=/home/yoh/.tmp/tmp.4I7ym6dSx2
++ echo 'directory: /home/yoh/.tmp/tmp.4I7ym6dSx2'
+directory: /home/yoh/.tmp/tmp.4I7ym6dSx2
++ cd /home/yoh/.tmp/tmp.4I7ym6dSx2
++ git init
+Initialized empty Git repository in /tmp/tmp.4I7ym6dSx2/.git/
++ git annex init --version=6
+init  ok
+(recording state in git...)
++ sed -i -e 's,pre-commit ,pre-commit --debug ,g' .git/hooks/pre-commit
++ echo 'I: creating a file'
+I: creating a file
++ echo whatever
++ sleep 1
++ git -c annex.largefiles=nothing annex --debug add file5
+[2017-02-17 10:19:52.529445464] read: git ["--git-dir=.git","--work-tree=.","--literal-pathspecs","ls-files","--others","--exclude-standard","-z","--","file5"]
+add file5 (non-large file; adding content to git repository) ok
+[2017-02-17 10:19:52.533532166] read: git ["--git-dir=.git","--work-tree=.","--literal-pathspecs","ls-files","--modified","-z","--","file5"]
+(recording state in git...)
+[2017-02-17 10:19:52.537789158] feed: xargs ["-0","git","--git-dir=.git","--work-tree=.","--literal-pathspecs","add","--"]
+[2017-02-17 10:19:52.567222419] process done ExitSuccess
++ git annex status
+A file5
+"""]]
+
+[[!meta author=yoh]]
+
+[[done]]
diff --git a/doc/bugs/mysterious_dependency_of_git_annex_status_output_of_the_added_file/comment_1_4d7eb95c73eceb079a176db1957df25e._comment b/doc/bugs/mysterious_dependency_of_git_annex_status_output_of_the_added_file/comment_1_4d7eb95c73eceb079a176db1957df25e._comment
new file mode 100644
index 0000000000..80a863de50
--- /dev/null
+++ b/doc/bugs/mysterious_dependency_of_git_annex_status_output_of_the_added_file/comment_1_4d7eb95c73eceb079a176db1957df25e._comment
@@ -0,0 +1,13 @@
+[[!comment format=mdwn
+ username="benjamin.poldrack@d09ccff6d42dd20277610b59867cf7462927b8e3"
+ nickname="benjamin.poldrack"
+ avatar="http://cdn.libravatar.org/avatar/5c1a901caa7c2cfeeb7e17e786c5230d"
+ subject="comment 1"
+ date="2017-02-24T13:00:08Z"
+ content="""
+I just want to point out, that this is most likely related to this one: 
+
+Generally, we get into trouble with files in git in v6 repos everytime. Pretty much the same thing happens, if we clone an annex repo with a file directly in git and then `git annex init --version=6` that clone. In that case, I can't avoid it with such a delay. However, all of this doesn't happen if I do it manually via single commands in the shell, but does, if it's done in a script. In some way, that smudge filter seems run out of sync with the command call triggering it.
+
+[[ben]]
+"""]]
diff --git a/doc/bugs/mysterious_dependency_of_git_annex_status_output_of_the_added_file/comment_2_ef55541955a6a8840a63309aa403ecae._comment b/doc/bugs/mysterious_dependency_of_git_annex_status_output_of_the_added_file/comment_2_ef55541955a6a8840a63309aa403ecae._comment
new file mode 100644
index 0000000000..287e79a584
--- /dev/null
+++ b/doc/bugs/mysterious_dependency_of_git_annex_status_output_of_the_added_file/comment_2_ef55541955a6a8840a63309aa403ecae._comment
@@ -0,0 +1,9 @@
+[[!comment format=mdwn
+ username="kyle"
+ avatar="http://cdn.libravatar.org/avatar/7d6e85cde1422ad60607c87fa87c63f3"
+ subject="fixed"
+ date="2018-09-10T18:27:49Z"
+ content="""
+As far as I can tell, this was fixed by 10138056d (v6: avoid accidental conversion when annex.largefiles is not configured, 2018-08-27).
+
+"""]]
diff --git a/doc/bugs/new_git-annex-shell_protocol_hides_remote_error_messages.mdwn b/doc/bugs/new_git-annex-shell_protocol_hides_remote_error_messages.mdwn
new file mode 100644
index 0000000000..8c27d42961
--- /dev/null
+++ b/doc/bugs/new_git-annex-shell_protocol_hides_remote_error_messages.mdwn
@@ -0,0 +1,21 @@
+`git-annex-shell p2pstdio` hides error messages that were transported over
+ssh to display to the user before. For example, diskreserve problems or IO
+errors. --[[Joey]]
+
+git-annex discards stderr from the command because old
+versions of git-annex-shell don't support the command and error out.
+
+Simply letting stderr through seems like the best solution though,
+if a way can be found to do it.
+Otherwise, all errors would have to be trapped (easy), and all stderr
+output also trapped (hard!), to be sent over the protocol using ERROR.
+And, there'd be a problem with sending messages atomically; if a message is
+being sent and an exception is thrown, that message needs to somehow be
+ended before an ERROR message can be sent.
+
+Hmm, it negotiates the protocol version after opening the connection.
+Any error at that point would make it not use the p2p protocol, 
+so can be excluded. Then, after version negotiation is complete, it
+could start displaying stderr.
+
+> [[fixed|done]] --[[Joey]]
diff --git a/doc/bugs/new_whereis_--json_lost_information_about_web_urls_if_other_special_remotes_provide_them.mdwn b/doc/bugs/new_whereis_--json_lost_information_about_web_urls_if_other_special_remotes_provide_them.mdwn
new file mode 100644
index 0000000000..bd860e9e48
--- /dev/null
+++ b/doc/bugs/new_whereis_--json_lost_information_about_web_urls_if_other_special_remotes_provide_them.mdwn
@@ -0,0 +1,93 @@
+### Please describe the problem.
+
+With upgrade to 6.20160114+gitg6be9ee0-1~ndall+1 our (datalad) tests started to fail since apparently format of whereis --json has changed.  It changed probably for the best since now no need to parse out urls from the generic 'note' field.  BUT it seems that it has lost 1. ability to associate urls with remotes 2. doesn't list web remote urls if another remote provides urls as well
+
+### Please provide any additional information below.
+
+[[!format sh """
+$> git annex whereis  sub001/BOLD/task001_run001/bold.nii.gz                          
+whereis sub001/BOLD/task001_run001/bold.nii.gz (3 copies) 
+  	00000000-0000-0000-0000-000000000001 -- web
+   	70d0c2f3-0d57-485c-8802-f6e829503516 -- [datalad-archives]
+   	bb907499-dbea-4713-977f-0ccd209de415 -- yoh@hopa:~/datalad/crawl/openfmri/ds000005 [here]
+
+  web: http://www.onerussian.com/tmp/banner.png
+  web: http://www.onerussian.com/tmp/bold.nii.gz
+
+  datalad-archives: dl+archive:SHA256E-s110760--5c9a3b565944b84c7df381481b597d716061881cbfc85493317452e85ea9b391.tgz/ds005/sub001/BOLD/task001_run001/bold.nii.gz
+  datalad-archives: dl+archive:SHA256E-s111649--63c9168b53e033c29d188b97d4950e267fa93a4d991fc92f42a3bb9488013863.tgz/ds005/sub001/BOLD/task001_run001/bold.nii.gz
+  datalad-archives: dl+archive:SHA256E-s112069--f1afedb105994006cbf37c03e2a05b538397283701c5a9bee483287ca912d690.tgz/ds005/sub001/BOLD/task001_run001/bold.nii.gz
+  datalad-archives: dl+archive:SHA256E-s154612--8adda02a8bc1a88f864f2cff31766f5ad4fcefbb42afd7230f95edfb5e0dfcb1.tgz/ds005/sub001/BOLD/task001_run001/bold.nii.gz
+ok
+
+$> git annex whereis --json sub001/BOLD/task001_run001/bold.nii.gz | python -m json.tool                      
+{            
+    "command": "whereis",
+    "file": "sub001/BOLD/task001_run001/bold.nii.gz",
+    "note": "\t00000000-0000-0000-0000-000000000001 -- web\n \t70d0c2f3-0d57-485c-8802-f6e829503516 -- [datalad-archives]\n \tbb907499-dbea-4713-977f-0ccd209de415 -- yoh@hopa:~/datalad/crawl/openfmri/ds000005 [here]\n",
+    "success": true,
+    "untrusted": [],
+    "urls": [
+        "dl+archive:SHA256E-s110760--5c9a3b565944b84c7df381481b597d716061881cbfc85493317452e85ea9b391.tgz/ds005/sub001/BOLD/task001_run001/bold.nii.gz",
+        "dl+archive:SHA256E-s111649--63c9168b53e033c29d188b97d4950e267fa93a4d991fc92f42a3bb9488013863.tgz/ds005/sub001/BOLD/task001_run001/bold.nii.gz",
+        "dl+archive:SHA256E-s112069--f1afedb105994006cbf37c03e2a05b538397283701c5a9bee483287ca912d690.tgz/ds005/sub001/BOLD/task001_run001/bold.nii.gz",
+        "dl+archive:SHA256E-s154612--8adda02a8bc1a88f864f2cff31766f5ad4fcefbb42afd7230f95edfb5e0dfcb1.tgz/ds005/sub001/BOLD/task001_run001/bold.nii.gz"
+    ],
+    "whereis": [
+        {
+            "description": "web",
+            "here": false,
+            "uuid": "00000000-0000-0000-0000-000000000001"
+        },
+        {
+            "description": "[datalad-archives]",
+            "here": false,
+            "uuid": "70d0c2f3-0d57-485c-8802-f6e829503516"
+        },
+        {
+            "description": "yoh@hopa:~/datalad/crawl/openfmri/ds000005",
+            "here": true,
+            "uuid": "bb907499-dbea-4713-977f-0ccd209de415"
+        }
+    ]
+}
+
+"""]]
+
+as you can see -- only --json format is missing on web remote URLs.  I guess, ideally, "urls" should also be an associative array with keys corresponding to remote names and listing them under.  OR why not to list them within the remote record under 'whereis'?  Sounds like the most logical place for them to be at, e.g.
+
+[[!format sh """
+
+    "whereis": [
+        {
+            "description": "web",
+            "here": false,
+            "uuid": "00000000-0000-0000-0000-000000000001",
+            "urls": [ "http://www.onerussian.com/tmp/bold.nii.gz"]
+        },
+        {
+            "description": "[datalad-archives]",
+            "here": false,
+            "uuid": "70d0c2f3-0d57-485c-8802-f6e829503516"
+            "urls": [
+        "dl+archive:SHA256E-s110760--5c9a3b565944b84c7df381481b597d716061881cbfc85493317452e85ea9b391.tgz/ds005/sub001/BOLD/task001_run001/bold.nii.gz",
+        "dl+archive:SHA256E-s111649--63c9168b53e033c29d188b97d4950e267fa93a4d991fc92f42a3bb9488013863.tgz/ds005/sub001/BOLD/task001_run001/bold.nii.gz",
+        "dl+archive:SHA256E-s112069--f1afedb105994006cbf37c03e2a05b538397283701c5a9bee483287ca912d690.tgz/ds005/sub001/BOLD/task001_run001/bold.nii.gz",
+        "dl+archive:SHA256E-s154612--8adda02a8bc1a88f864f2cff31766f5ad4fcefbb42afd7230f95edfb5e0dfcb1.tgz/ds005/sub001/BOLD/task001_run001/bold.nii.gz"
+    ],
+
+
+        },
+        {
+            "description": "yoh@hopa:~/datalad/crawl/openfmri/ds000005",
+            "here": true,
+            "uuid": "bb907499-dbea-4713-977f-0ccd209de415"
+        }
+    ]
+"""]]
+
+what is the purpose of note in current output anyways since it just duplicates information in 'whereis' field?
+
+[[!meta author=yoh]]
+
+> [[fixed|done]] --[[Joey]]
diff --git a/doc/bugs/no_git-annex_shell_on_Windows.mdwn b/doc/bugs/no_git-annex_shell_on_Windows.mdwn
new file mode 100644
index 0000000000..60a67eae9e
--- /dev/null
+++ b/doc/bugs/no_git-annex_shell_on_Windows.mdwn
@@ -0,0 +1,28 @@
+### Please describe the problem.
+
+git annex installer on Windows only installs git-annex-licenses.txt  git-annex-uninstall.exe  git-annex.exe
+
+This makes git-annex unusable over ssh (Windows box runs cygwin sshd).
+
+### What steps will reproduce the problem?
+
+
+### What version of git-annex are you using? On what operating system?
+
+Windows 7 64bit
+
+git annex version
+git-annex version: 5.20140421-g78d6aa1
+build flags: Assistant Webapp Webapp-secure Pairing Testsuite S3 WebDAV DNS Feeds Quvi TDFA CryptoHash
+key/value backends: SHA256E SHA1E SHA512E SHA224E SHA384E SKEIN256E SKEIN512E SHA256 SHA1 SHA512 SHA224 SHA384 SKEIN256 SKEIN512 WORM URL
+remote types: git gcrypt S3 bup directory rsync web webdav tahoe glacier hook external
+
+### Please provide any additional information below.
+
+[[!format sh """
+# If you can, paste a complete transcript of the problem occurring here.
+# If the problem is with the git-annex assistant, paste in .git/annex/daemon.log
+
+
+# End of transcript or log.
+"""]]
diff --git a/doc/bugs/no_git-annex_shell_on_Windows/comment_10_72b2277c3728f02e4d158ebbd7db41b2._comment b/doc/bugs/no_git-annex_shell_on_Windows/comment_10_72b2277c3728f02e4d158ebbd7db41b2._comment
new file mode 100644
index 0000000000..93a27d7544
--- /dev/null
+++ b/doc/bugs/no_git-annex_shell_on_Windows/comment_10_72b2277c3728f02e4d158ebbd7db41b2._comment
@@ -0,0 +1,22 @@
+[[!comment format=mdwn
+ username="timotejs@7102fe834ef5514c095ceb7e09b525bafa7b1af4"
+ nickname="timotejs"
+ avatar="http://cdn.libravatar.org/avatar/3b2d892080bb891056a77a0be9ee8176"
+ subject="Did' nyone make it work?"
+ date="2018-02-06T08:15:06Z"
+ content="""
+I try to use SSH to connect to Win10 machine with annex repo.
+I am using the OpenSSH integrated into the Windows 10 (one you can install from Windows additional features thingy).
+
+it didn't know git-annex-shell so i created symlink
+
+    mklink \"C:\Program Files (x86)\Git\bin\git-annex-shell\" \"C:\Program Files (x86)\Git\bin\git-annex.exe\"
+
+and added ';.' to PATHEX
+
+Currently git-annex-shell is found but i get error:
+
+    Invalid argument `'configlist''
+
+when i try to sync remote on that W10 machine
+"""]]
diff --git a/doc/bugs/no_git-annex_shell_on_Windows/comment_11_b212c8b8cf361745f35a9f7ff6fee1f5._comment b/doc/bugs/no_git-annex_shell_on_Windows/comment_11_b212c8b8cf361745f35a9f7ff6fee1f5._comment
new file mode 100644
index 0000000000..c052520abc
--- /dev/null
+++ b/doc/bugs/no_git-annex_shell_on_Windows/comment_11_b212c8b8cf361745f35a9f7ff6fee1f5._comment
@@ -0,0 +1,11 @@
+[[!comment format=mdwn
+ username="joey"
+ subject="""comment 11"""
+ date="2018-02-06T17:04:51Z"
+ content="""
+If you copy git-annex.exe to git-annex-shell.exe, it should work.
+
+The installer doesn't do it, because doubling the installation size footprint
+for a feature that involves sshing into a Windows machine (not very common
+I suspect) didn't seem worth it.
+"""]]
diff --git a/doc/bugs/no_git-annex_shell_on_Windows/comment_12_9d445ddf84e15a59cdc2aa2b8dc30c44._comment b/doc/bugs/no_git-annex_shell_on_Windows/comment_12_9d445ddf84e15a59cdc2aa2b8dc30c44._comment
new file mode 100644
index 0000000000..cebd4b950d
--- /dev/null
+++ b/doc/bugs/no_git-annex_shell_on_Windows/comment_12_9d445ddf84e15a59cdc2aa2b8dc30c44._comment
@@ -0,0 +1,15 @@
+[[!comment format=mdwn
+ username="joey"
+ subject="""comment 12"""
+ date="2018-02-08T17:13:41Z"
+ content="""
+That error message seems to indicate that git-annex does not realize it's
+being run with the git-annex-shell program name. When I run "git-annex -c
+configlist /path/to/repo" I get that error message; when I run
+git-annex-shell with the same parameters, I get the expected repository
+config output.
+
+I'm guessing that the windows symlink thing you tried to use confuses the
+program path name lookup that it tries to do. I'm pretty sure that a copy
+of the file would work.
+"""]]
diff --git a/doc/bugs/no_git-annex_shell_on_Windows/comment_1_988768c3e4f30c751034a0b0390cdd88._comment b/doc/bugs/no_git-annex_shell_on_Windows/comment_1_988768c3e4f30c751034a0b0390cdd88._comment
new file mode 100644
index 0000000000..fc58c35dd2
--- /dev/null
+++ b/doc/bugs/no_git-annex_shell_on_Windows/comment_1_988768c3e4f30c751034a0b0390cdd88._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="http://joeyh.name/"
+ ip="108.236.230.124"
+ subject="comment 1"
+ date="2014-05-16T18:12:34Z"
+ content="""
+You should be able to copy git-annex.exe to git-annex-shell.exe and that will be usable as git-annex-shell. I could make the windows installer do this, but without the ability to easily hard or symlink the files, it would double the disk space for what seems an unusual use case..
+"""]]
diff --git a/doc/bugs/no_git-annex_shell_on_Windows/comment_2_70a6c9af57eed4f8ceb3de63cbf8a5e6._comment b/doc/bugs/no_git-annex_shell_on_Windows/comment_2_70a6c9af57eed4f8ceb3de63cbf8a5e6._comment
new file mode 100644
index 0000000000..dd8a5ef679
--- /dev/null
+++ b/doc/bugs/no_git-annex_shell_on_Windows/comment_2_70a6c9af57eed4f8ceb3de63cbf8a5e6._comment
@@ -0,0 +1,10 @@
+[[!comment format=mdwn
+ username="https://www.google.com/accounts/o8/id?id=AItOawln3ckqKx0x_xDZMYwa9Q1bn4I06oWjkog"
+ nickname="Michael"
+ subject="comment 2"
+ date="2014-05-16T18:58:13Z"
+ content="""
+Hmm, I don't understand the double disk space issue.
+If direct mode is used, files are stored once, right?
+
+"""]]
diff --git a/doc/bugs/no_git-annex_shell_on_Windows/comment_3_7f48959c6242c5cc52e9d0a79fd3f85d._comment b/doc/bugs/no_git-annex_shell_on_Windows/comment_3_7f48959c6242c5cc52e9d0a79fd3f85d._comment
new file mode 100644
index 0000000000..634c0d8b64
--- /dev/null
+++ b/doc/bugs/no_git-annex_shell_on_Windows/comment_3_7f48959c6242c5cc52e9d0a79fd3f85d._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="http://joeyh.name/"
+ ip="108.236.230.124"
+ subject="comment 3"
+ date="2014-05-16T20:36:02Z"
+ content="""
+I mean that copying git-annex.exe to git-annex-shell.exe doubles the disk space needed to install git-annex.
+"""]]
diff --git a/doc/bugs/no_git-annex_shell_on_Windows/comment_4_04b30dd9e510762ee547e7b45bdba660._comment b/doc/bugs/no_git-annex_shell_on_Windows/comment_4_04b30dd9e510762ee547e7b45bdba660._comment
new file mode 100644
index 0000000000..4a6fd402d3
--- /dev/null
+++ b/doc/bugs/no_git-annex_shell_on_Windows/comment_4_04b30dd9e510762ee547e7b45bdba660._comment
@@ -0,0 +1,12 @@
+[[!comment format=mdwn
+ username="https://www.google.com/accounts/o8/id?id=AItOawln3ckqKx0x_xDZMYwa9Q1bn4I06oWjkog"
+ nickname="Michael"
+ subject="comment 4"
+ date="2014-05-24T21:26:24Z"
+ content="""
+I tried copying git-annex.exe to git-annex-shell.exe, but it doesn't behave as git-annex-shell:
+
+git-annex-shell -c configlist
+git-annex-shell.exe: unknown command configlist
+
+"""]]
diff --git a/doc/bugs/no_git-annex_shell_on_Windows/comment_5_beebbd4a41cccfc7aac3c19d76c3b978._comment b/doc/bugs/no_git-annex_shell_on_Windows/comment_5_beebbd4a41cccfc7aac3c19d76c3b978._comment
new file mode 100644
index 0000000000..9d9acf54f3
--- /dev/null
+++ b/doc/bugs/no_git-annex_shell_on_Windows/comment_5_beebbd4a41cccfc7aac3c19d76c3b978._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="https://www.google.com/accounts/o8/id?id=AItOawln3ckqKx0x_xDZMYwa9Q1bn4I06oWjkog"
+ nickname="Michael"
+ subject="comment 5"
+ date="2014-05-24T21:45:55Z"
+ content="""
+Looks like .exe prefix confuses detection logic. If I remove extension, it does behave as git-annex-shell.
+"""]]
diff --git a/doc/bugs/no_git-annex_shell_on_Windows/comment_6_957c5da5f336c7ef59e4149602471888._comment b/doc/bugs/no_git-annex_shell_on_Windows/comment_6_957c5da5f336c7ef59e4149602471888._comment
new file mode 100644
index 0000000000..c9d4c3a6a5
--- /dev/null
+++ b/doc/bugs/no_git-annex_shell_on_Windows/comment_6_957c5da5f336c7ef59e4149602471888._comment
@@ -0,0 +1,15 @@
+[[!comment format=mdwn
+ username="https://www.google.com/accounts/o8/id?id=AItOawln3ckqKx0x_xDZMYwa9Q1bn4I06oWjkog"
+ nickname="Michael"
+ subject="comment 6"
+ date="2014-05-24T21:49:47Z"
+ content="""
+However, it's still not usable remotely over ssh:
+
+git annex sync kpf
+
+git-annex-shell: git: createProcess: invalid argument (Invalid argument)
+
+  Remote kpf does not have git-annex installed; setting annex-ignore
+
+"""]]
diff --git a/doc/bugs/no_git-annex_shell_on_Windows/comment_7_a8654cf656de55ccdcf40270a1ef4ca9._comment b/doc/bugs/no_git-annex_shell_on_Windows/comment_7_a8654cf656de55ccdcf40270a1ef4ca9._comment
new file mode 100644
index 0000000000..1733b29dd6
--- /dev/null
+++ b/doc/bugs/no_git-annex_shell_on_Windows/comment_7_a8654cf656de55ccdcf40270a1ef4ca9._comment
@@ -0,0 +1,35 @@
+[[!comment format=mdwn
+ username="https://www.google.com/accounts/o8/id?id=AItOawln3ckqKx0x_xDZMYwa9Q1bn4I06oWjkog"
+ nickname="Michael"
+ subject="comment 7"
+ date="2014-05-24T22:07:15Z"
+ content="""
+OK thought I'd use it with annex-ignore set, but:
+
+    git annex copy --to=kpf file.jpg
+    copy file.jpg (checking kpf...) git-annex-shell: git: createProcess: invalid argument (Invalid argument)
+(to kpf...) 
+    git-annex-shell: git: createProcess: invalid argument (Invalid argument)
+    rsync: connection unexpectedly closed (0 bytes received so far) [sender]
+    rsync error: error in rsync protocol data stream (code 12) at io.c(226) [sender=3.1.0]
+
+    rsync failed -- run git annex again to resume file transfer
+    failed
+git-annex: copy: 1 failed
+
+
+For reference, here's the .git/config section.
+
+I can either set receivepack/uploadpack like below or add /cygdrive/c/Program Files (x86)/Git/libexec/git-core to PATH in cygwin shell.
+
+Also, if I set url to kp:/cygdrive/f/backup/f, git complains that it's not a git repository (msysgit doesn't understand cygwin stuff?)
+
+    [remote \"kpf\"]
+            url = kp:f:/backup/ph
+            fetch = +refs/heads/*:refs/remotes/kpf/*
+            receivepack = git receive-pack
+            uploadpack = git upload-pack
+            annex-uuid = 99c498fc-e38e-11e3-8bb0-5368a291bd90
+            annex-ignore = true
+
+"""]]
diff --git a/doc/bugs/no_git-annex_shell_on_Windows/comment_8_3c8bb1b1d67ca1a6e568e381d24a2574._comment b/doc/bugs/no_git-annex_shell_on_Windows/comment_8_3c8bb1b1d67ca1a6e568e381d24a2574._comment
new file mode 100644
index 0000000000..ba7fadc23f
--- /dev/null
+++ b/doc/bugs/no_git-annex_shell_on_Windows/comment_8_3c8bb1b1d67ca1a6e568e381d24a2574._comment
@@ -0,0 +1,16 @@
+[[!comment format=mdwn
+ username="https://www.google.com/accounts/o8/id?id=AItOawln3ckqKx0x_xDZMYwa9Q1bn4I06oWjkog"
+ nickname="Michael"
+ subject="comment 8"
+ date="2014-05-24T22:58:16Z"
+ content="""
+Another data point.  If I install cygwin's git (so references like kp:/cygdrive/f would be understood) and remove annex-ignore setting, I get:
+
+    git annex sync kpf
+    commit  ok
+    pull kpf 
+    git-annex: Unknown command 'kp'
+
+Looks like hostname (kp) is taken as a command somehow.
+
+"""]]
diff --git a/doc/bugs/no_git-annex_shell_on_Windows/comment_9_03646c9df9d4f4ea27459660fe65a976._comment b/doc/bugs/no_git-annex_shell_on_Windows/comment_9_03646c9df9d4f4ea27459660fe65a976._comment
new file mode 100644
index 0000000000..7a5152b590
--- /dev/null
+++ b/doc/bugs/no_git-annex_shell_on_Windows/comment_9_03646c9df9d4f4ea27459660fe65a976._comment
@@ -0,0 +1,25 @@
+[[!comment format=mdwn
+ username="https://www.google.com/accounts/o8/id?id=AItOawnOSgFb3l7nL3Fs7Y9gPGJJjFiV7aJ1tek"
+ nickname="Phil"
+ subject="comment 9"
+ date="2014-07-05T22:29:01Z"
+ content="""
+Copying the `git-annex.exe` does work but you can also create a symlink and tell Windows to execute symlinks by doing the following.
+
+On the remote machine (the one you are connecting to)
+
+## Create a symlink:
+
+    mklink \"C:\Program Files (x86)\Git\bin\git-annex-shell\" \"C:\Program Files (x86)\Git\bin\git-annex.exe\"
+
+## Tell Windows to run files with no extension:
+
+* Open Environment Variables
+* At the end of `PATHEXT` add `;.`
+
+## A problem with rsync
+
+Unfortunately I now get `Access is denied` and `rsync: connection unexpectedly closed`. I think this is due to path issues on Windows. Trying rsync manually I need to use the `/cygdrive/c/rest/of/path` syntax otherwise I get `No such file or directory.`
+
+
+"""]]
diff --git a/doc/bugs/no_prebuilt_package_for_intel_64_architecture.mdwn b/doc/bugs/no_prebuilt_package_for_intel_64_architecture.mdwn
new file mode 100644
index 0000000000..9b3a8af24c
--- /dev/null
+++ b/doc/bugs/no_prebuilt_package_for_intel_64_architecture.mdwn
@@ -0,0 +1,25 @@
+### Please describe the problem.
+On the download page for linux, https://downloads.kitenet.net/git-annex/linux/current/ (from http://git-annex.branchable.com/install/) there are links to amd64 and Intel 386 versions, but no package for Intel 64 bit version.
+
+### What steps will reproduce the problem?
+Visit the site.
+
+### What version of git-annex are you using? On what operating system?
+
+
+### Please provide any additional information below.
+
+[[!format sh """
+# If you can, paste a complete transcript of the problem occurring here.
+# If the problem is with the git-annex assistant, paste in .git/annex/daemon.log
+
+
+# End of transcript or log.
+"""]]
+
+### Have you had any luck using git-annex before? (Sometimes we get tired of reading bug reports all day and a lil' positive end note does wonders)
+
+>  uses
+> "x86-64" to refer to those. The "amd64" in the name of the file is
+> basically an internal detail, I hope people reading the page will find it
+> clear enough. [[done]] --[[Joey]]
diff --git a/doc/bugs/no_prebuilt_package_for_intel_64_architecture/comment_1_366f4c56248099105f411c1df9b57733._comment b/doc/bugs/no_prebuilt_package_for_intel_64_architecture/comment_1_366f4c56248099105f411c1df9b57733._comment
new file mode 100644
index 0000000000..6ccdf20b02
--- /dev/null
+++ b/doc/bugs/no_prebuilt_package_for_intel_64_architecture/comment_1_366f4c56248099105f411c1df9b57733._comment
@@ -0,0 +1,10 @@
+[[!comment format=mdwn
+ username="olaf"
+ avatar="http://cdn.libravatar.org/avatar/4ae498d3d6ee558d6b65caa658f72572"
+ subject="comment 1"
+ date="2017-08-16T01:01:02Z"
+ content="""
+They're the same thing.
+
+For a much better explanation than I could write, see: [https://askubuntu.com/a/54298](https://askubuntu.com/a/54298)
+"""]]
diff --git a/doc/bugs/no_prebuilt_package_for_intel_64_architecture/comment_2_9255ebaad5239202c57d86467929321d._comment b/doc/bugs/no_prebuilt_package_for_intel_64_architecture/comment_2_9255ebaad5239202c57d86467929321d._comment
new file mode 100644
index 0000000000..5305ba4cae
--- /dev/null
+++ b/doc/bugs/no_prebuilt_package_for_intel_64_architecture/comment_2_9255ebaad5239202c57d86467929321d._comment
@@ -0,0 +1,13 @@
+[[!comment format=mdwn
+ username="metst13@1d16544ec52801db7efb2895d3dc7a4458b8eb45"
+ nickname="metst13"
+ avatar="http://cdn.libravatar.org/avatar/168d629704097ddc596f75ca32a687a3"
+ subject="links on page"
+ date="2017-08-22T05:10:11Z"
+ content="""
+Thanks, Olaf, for the link with some enlightenment. 
+
+Thanks, Joey. Maybe it would be better to leave only one link to the download place of sources? 
+
+I did't press \"Linux\", but the link on the right to that (which leads to a directory on server, not to the link you gave here).
+"""]]
diff --git a/doc/bugs/non-git-annex_symlinks_not_perserved_when_exporting_tree_via_rsync.mdwn b/doc/bugs/non-git-annex_symlinks_not_perserved_when_exporting_tree_via_rsync.mdwn
new file mode 100644
index 0000000000..793b2daea1
--- /dev/null
+++ b/doc/bugs/non-git-annex_symlinks_not_perserved_when_exporting_tree_via_rsync.mdwn
@@ -0,0 +1,15 @@
+### Please describe the problem.
+
+non-git-annex symlinks are not perserved when exporting tree via rsync. instead of symlinks on the remote destination, there are files that that contain text that describe the path of the local symlink.
+
+### What steps will reproduce the problem?
+
+add an ssh-based rsync remote with exporttree=yes and run `git annex export master --to test-rsync-remote`
+
+### What version of git-annex are you using? On what operating system?
+
+6.20180529 on Trisquel 8 GNU/Linux
+
+### Have you had any luck using git-annex before? (Sometimes we get tired of reading bug reports all day and a lil' positive end note does wonders)
+
+Yes, Git Annex has allowed us to transfer Gigabytes of data without a need for 7 GB of swap space and a git push command that takes hours to complete. :)
diff --git a/doc/bugs/non-git-annex_symlinks_not_perserved_when_exporting_tree_via_rsync/comment_1_b92c67f7d2f0424ce399ca4659e6c805._comment b/doc/bugs/non-git-annex_symlinks_not_perserved_when_exporting_tree_via_rsync/comment_1_b92c67f7d2f0424ce399ca4659e6c805._comment
new file mode 100644
index 0000000000..1ad0e16d5b
--- /dev/null
+++ b/doc/bugs/non-git-annex_symlinks_not_perserved_when_exporting_tree_via_rsync/comment_1_b92c67f7d2f0424ce399ca4659e6c805._comment
@@ -0,0 +1,9 @@
+[[!comment format=mdwn
+ username="joey"
+ subject="""comment 1"""
+ date="2018-06-12T16:46:43Z"
+ content="""
+Yes, neither are execute bits preserved. Most kinds of special remotes that
+an export could be done to don't support such things, and it seemed like it
+would unduely complicate the export interface to support them.
+"""]]
diff --git a/doc/bugs/non-repos_in_repositories_list___40__+_other_weird_output__41___from_git_annex_status.mdwn b/doc/bugs/non-repos_in_repositories_list___40__+_other_weird_output__41___from_git_annex_status.mdwn
new file mode 100644
index 0000000000..a66d8635a0
--- /dev/null
+++ b/doc/bugs/non-repos_in_repositories_list___40__+_other_weird_output__41___from_git_annex_status.mdwn
@@ -0,0 +1,89 @@
+### Please describe the problem.
+
+when running ```git annex status``` i get 'alien' content in the list of repositories (some timestamps, something looking like git status messages, and a long list of ```(recovery from race)``` messages. the actual content seems fine and the list of repositories in the assistant looks ok, but the CLI status messages show this unusual content (full dump below).
+
+### What steps will reproduce the problem?
+
+not sure how i would go about reproducing this: i only noticed this weird output just now and i'm pretty sure it was fine 24h ago - in the meanwhile the single non-ordinary operation was deleting via the assistant's webapp an SSH remote, rsync+encryption, then creating it again as bare repo without encryption.
+
+after the assistant finished removing all the content from the previous annex directory, it stalled reporting deletion in progress (the annex directory's parent on the remote server was not writeable for the user used for the ssh connection - perhaps that was the cause). even after chowning the parent dir and restarting the assistant on my local workstation, the assistant would still show the remote as pending deletion.
+
+after a while i could add the remote again, setting it to be a git bare repo, and content was copied over.
+
+from the assistant's webapp, i still see the correct list of remotes, but the output of ```git annex status``` is as below.
+
+```.git/config``` also looks ok, listing some core settings, some annex settings and my three real remotes for this annex.
+
+the part of logs relevant to the above change (deletion of remote, creation of new one + content sync) is unfortunately gone from the annex logs (including the rotated ones in ```.git/annex/daemon.log*```).
+
+### What version of git-annex are you using? On what operating system?
+
+4.20130709:amd64 on Debian testing (linux 3.9.8-1)
+
+
+### Please provide any additional information below.
+
+[[!format sh """
+$ git annex status
+supported backends: SHA256E SHA1E SHA512E SHA224E SHA384E SHA256 SHA1 SHA512 SHA224 SHA384 WORM URL
+supported remote types: git S3 bup directory rsync web webdav glacier hook
+repository mode: direct
+trusted repositories: 7
+        1359161911.638435s -- 1
+        1359161913.173474s -- 1
+        1359161913.288686s -- 1
+        1359332402.294867s -- 1
+        1359332406.839623s -- 1
+        1359332407.454511s -- 1
+        1359332411.932021s -- 1
+semitrusted repositories: 36
+        00000000-0000-0000-0000-000000000001 -- web
+        00511808-8d7c-11e2-a252-43e07a9aaa3c -- client
+        12527a3a-a21d-4cc7-a7ea-074e8fd0260f -- annex_on_rpi00 (standard)
+        12a7b66c-e681-4dcc-bf75-b995512e5e76 -- client
+        1359159154.357141s -- 1 2ce1be52-6745-11e2-83e5-e7d111798afd
+        1359159155.817851s -- 1 2ce1be52-6745-11e2-83e5-e7d111798afd
+        1359159155.931382s -- 1 e08336a0-6742-11e2-8a53-17c5ab00507f
+        1359161255.198519s -- 1 2ce1be52-6745-11e2-83e5-e7d111798afd
+        1359161256.72633s -- 1 2ce1be52-6745-11e2-83e5-e7d111798afd
+        1359161256.952372s -- 1 e08336a0-6742-11e2-8a53-17c5ab00507f
+        1359332437.573825s -- 1 2e71af50-674c-11e2-b542-6397738d1182
+        1359332440.782882s -- 0 2e71af50-674c-11e2-b542-6397738d1182
+        1359332441.153755s -- 1 42c358c4-6741-11e2-ba3f-8bdecec02757
+        1359332448.091981s -- 0 2ce1be52-6745-11e2-83e5-e7d111798afd
+        1359332507.354892s -- 1 2e71af50-674c-11e2-b542-6397738d1182
+        1359332512.553832s -- 1 2e71af50-674c-11e2-b542-6397738d1182
+        1359332512.666779s -- 1 42c358c4-6741-11e2-ba3f-8bdecec02757
+        1359332517.773994s -- 1 42c358c4-6741-11e2-ba3f-8bdecec02757
+        1359332523.112182s -- 0 2ce1be52-6745-11e2-83e5-e7d111798afd
+        1374091795.70517s -- 1 2e71af50-674c-11e2-b542-6397738d1182
+        1374507271.515001s -- 0 8d243fde-0c4c-4a19-8bce-ac1beb48eef0
+        1374507272.066961s -- 0 8d243fde-0c4c-4a19-8bce-ac1beb48eef0
+        1a76c11b-b359-4606-be1d-cc3b17dc684e -- client
+        2ce1be52-6745-11e2-83e5-e7d111798afd -- git.services__data_annex_ (transfer)
+        2e71af50-674c-11e2-b542-6397738d1182 -- here (client)
+        42c358c4-6741-11e2-ba3f-8bdecec02757 -- x11miel.local_data_annex (em@10.10.10.100:~/data/annex)
+        4eb75def-169c-45f8-b9b0-a0aa52017bd6 -- client
+        92aa538c-8d7b-11e2-9ae1-3fd2531928c1 -- client
+        author -- nnnn nnnn  1374509958 +0100
+        committer -- nnnn nnnn  1374509958 +0100
+        d83e4134-8d7a-11e2-8822-7f2218f2cc42 -- client
+        e08336a0-6742-11e2-8a53-17c5ab00507f -- backup
+        f7969bd0-d5dc-11e2-b362-cbb323f72415 -- transfer
+        merging -- 10.10.10.10__mnt_annex_00_annex/git-annex into git-annex (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race)
+        parent -- fcde6fd44ff2ad2aaca895bca1b32fb398d76545
+        tree -- b6d4b49d9b67b296b0a0d9a800b96f88d480f923
+untrusted repositories: 2
+        1359332417.999925s -- 0
+        1374507270.94356s -- 0
+transfers in progress: none
+available local disk space: 6 gigabytes (+1 megabyte reserved)
+local annex keys: 1984
+local annex size: 6 gigabytes
+known annex keys: 1989
+known annex size: 6 gigabytes
+bloom filter size: 16 mebibytes (0.3% full)
+backend usage:
+        SHA256E: 3973
+$ 
+"""]]
diff --git a/doc/bugs/non-repos_in_repositories_list___40__+_other_weird_output__41___from_git_annex_status/comment_1_fcd230cbb2ac363c469b98021042c011._comment b/doc/bugs/non-repos_in_repositories_list___40__+_other_weird_output__41___from_git_annex_status/comment_1_fcd230cbb2ac363c469b98021042c011._comment
new file mode 100644
index 0000000000..c9aa34806b
--- /dev/null
+++ b/doc/bugs/non-repos_in_repositories_list___40__+_other_weird_output__41___from_git_annex_status/comment_1_fcd230cbb2ac363c469b98021042c011._comment
@@ -0,0 +1,10 @@
+[[!comment format=mdwn
+ username="http://mey.vn/"
+ ip="46.65.14.106"
+ subject="git annex status output on connected nodes"
+ date="2013-07-25T16:42:04Z"
+ content="""
+i have now updated the git-annex package to the latest 4.20130723:amd64 still on Debian testing (linux 3.9.8-1) - i see the same issue (while files keep synchronising fine across the various connected nodes).
+
+i have noticed that the same output appears when i issue ```git annex status``` on the other nodes (an Ubuntu 13.04 with kernel 3.9.7 amd64 and a Debian stable with 3.10.1-1 amd64, all with latest Debian 4.20130723:amd64 package).
+"""]]
diff --git a/doc/bugs/non-repos_in_repositories_list___40__+_other_weird_output__41___from_git_annex_status/comment_2_23207ecabd4b41d9551d0491fa71e96b._comment b/doc/bugs/non-repos_in_repositories_list___40__+_other_weird_output__41___from_git_annex_status/comment_2_23207ecabd4b41d9551d0491fa71e96b._comment
new file mode 100644
index 0000000000..3c9d1f36f9
--- /dev/null
+++ b/doc/bugs/non-repos_in_repositories_list___40__+_other_weird_output__41___from_git_annex_status/comment_2_23207ecabd4b41d9551d0491fa71e96b._comment
@@ -0,0 +1,12 @@
+[[!comment format=mdwn
+ username="http://joeyh.name/"
+ ip="2001:4978:f:21a::2"
+ subject="comment 2"
+ date="2013-07-25T20:12:06Z"
+ content="""
+Please paste the output of: `git show git-annex:uuid.log`
+
+The file seems to have a lot of garbage in it. The \"merging -- 10.10.10.10__mnt_annex_00_annex/git-annex into git-annex (recovery from race)\" in particular is only ever passed to git as the commit message. It should not be possible for it to appear in a file unless data has somehow become corrupted.
+
+So, I think you should also run `git fsck`. You may also find that it helps to shut down any running git-annex, and move `.git/annex/index` away to a backup location. A corrupted index file would be kind of be the best scenario since git-annex can automatically rebuild it from the git repository..
+"""]]
diff --git a/doc/bugs/non-repos_in_repositories_list___40__+_other_weird_output__41___from_git_annex_status/comment_3_6ea92adfe955b6a5cd2a39fea78b3bf6._comment b/doc/bugs/non-repos_in_repositories_list___40__+_other_weird_output__41___from_git_annex_status/comment_3_6ea92adfe955b6a5cd2a39fea78b3bf6._comment
new file mode 100644
index 0000000000..56ae258a7c
--- /dev/null
+++ b/doc/bugs/non-repos_in_repositories_list___40__+_other_weird_output__41___from_git_annex_status/comment_3_6ea92adfe955b6a5cd2a39fea78b3bf6._comment
@@ -0,0 +1,119 @@
+[[!comment format=mdwn
+ username="http://mey.vn/"
+ ip="46.65.14.106"
+ subject="comment 3"
+ date="2013-07-26T14:54:23Z"
+ content="""
+thanks Joey.
+
+output from ```git show git-annex:uuid.log``` is below.
+
+```git fsck``` shows me 4201 lines of ```dangling blob ```.
+
+```git annex fsck``` instead checksums all the files and only reports a single error about no known copies of an individual file. i guess this can be relied upon to confirm that the actual data is fine?
+
+i have also tried shutting down any running git-annex and moving the index file away - it was rebuilt when i restarted the assistant, but i still get the same output from ```git annex status```.
+
+i should probably work around this by just copying the annex folder's contents to a new folder and create a new annex there (perhaps in indirect mode rather than direct as it is now, as this is mostly a document archive with no further changes to files except some occasional renaming) and create my remotes from scratch, but now i'm quite curious about trying to track down what happened and to understand if the annex is reliably storing and syncing content even though the status message contains alien data.
+
+
+(output from ```git show git-annex:uuid.log``` follows)
+
+    00511808-8d7c-11e2-a252-43e07a9aaa3c client timestamp=1363357452.108251s
+    12527a3a-a21d-4cc7-a7ea-074e8fd0260f pi@pi0:/mnt/annex/00/annex timestamp=1374508625.460693s
+    12527a3a-a21d-4cc7-a7ea-074e8fd0260f standard timestamp=1374508627.738835s
+    12527a3a-a21d-4cc7-a7ea-074e8fd0260f transfer timestamp=1374508627.731088s
+    12a7b66c-e681-4dcc-bf75-b995512e5e76 client timestamp=1363469702.545491s
+    1359159154.357141s 1 2ce1be52-6745-11e2-83e5-e7d111798afd
+    1359159155.817851s 1 2ce1be52-6745-11e2-83e5-e7d111798afd
+    1359159155.931382s 1 e08336a0-6742-11e2-8a53-17c5ab00507f
+    1359161255.198519s 1 2ce1be52-6745-11e2-83e5-e7d111798afd
+    1359161256.72633s 1 2ce1be52-6745-11e2-83e5-e7d111798afd
+    1359161256.952372s 1 e08336a0-6742-11e2-8a53-17c5ab00507f
+    1359161911.638435s 1
+    1359161913.173474s 1
+    1359161913.288686s 1
+    1359332402.294867s 1
+    1359332406.839623s 1
+    1359332407.454511s 1
+    1359332411.932021s 1
+    1359332417.999925s 0
+    1359332437.573825s 1 2e71af50-674c-11e2-b542-6397738d1182
+    1359332440.782882s 0 2e71af50-674c-11e2-b542-6397738d1182
+    1359332441.153755s 1 42c358c4-6741-11e2-ba3f-8bdecec02757
+    1359332448.091981s 0 2ce1be52-6745-11e2-83e5-e7d111798afd
+    1359332507.354892s 1 2e71af50-674c-11e2-b542-6397738d1182
+    1359332512.553832s 1 2e71af50-674c-11e2-b542-6397738d1182
+    1359332512.666779s 1 42c358c4-6741-11e2-ba3f-8bdecec02757
+    1359332517.773994s 1 42c358c4-6741-11e2-ba3f-8bdecec02757
+    1359332523.112182s 0 2ce1be52-6745-11e2-83e5-e7d111798afd
+    1374091795.70517s 1 2e71af50-674c-11e2-b542-6397738d1182
+    1374507270.94356s 0
+    1374507271.515001s 0 8d243fde-0c4c-4a19-8bce-ac1beb48eef0
+    1374507272.066961s 0 8d243fde-0c4c-4a19-8bce-ac1beb48eef0
+    1a76c11b-b359-4606-be1d-cc3b17dc684e client timestamp=1370536281.421365s
+    2ce1be52-6745-11e2-83e5-e7d111798afd  timestamp=1359155758.081281s
+    2ce1be52-6745-11e2-83e5-e7d111798afd transfer timestamp=1359155760.243544s
+    2e71af50-674c-11e2-b542-6397738d1182 em@x12yad:~/data/annex timestamp=1359158766.916677s
+    2e71af50-674c-11e2-b542-6397738d1182 client timestamp=1359158766.994574s
+    42c358c4-6741-11e2-ba3f-8bdecec02757 em@10.10.10.100:~/data/annex timestamp=1359314998.998417s
+    42c358c4-6741-11e2-ba3f-8bdecec02757 em@x11miel:~/data/annex timestamp=1359154076.554768s
+    42c358c4-6741-11e2-ba3f-8bdecec02757 client timestamp=1359154076.649004s
+    4eb75def-169c-45f8-b9b0-a0aa52017bd6 client timestamp=1363467301.827658s
+    8d243fde-0c4c-4a19-8bce-ac1beb48eef0 X timestamp=1374508357.215776s
+    8d243fde-0c4c-4a19-8bce-ac1beb48eef0 backup on rPi Zero timestamp=1374398146.433271s
+    8d243fde-0c4c-4a19-8bce-ac1beb48eef0 standard timestamp=1374508303.429835s
+    8d243fde-0c4c-4a19-8bce-ac1beb48eef0 unwanted timestamp=1374506712.447376s
+    8d243fde-0c4c-4a19-8bce-ac1beb48eef0 unwanted timestamp=1374508303.426075s
+    92aa538c-8d7b-11e2-9ae1-3fd2531928c1 client timestamp=1363357265.665495s
+    author nnnn nnnn  1374508641 +0100
+    author nnnn nnnn  1374508642 +0100
+    author nnnn nnnn  1374508643 +0100
+    author nnnn nnnn  1374508644 +0100
+    author nnnn nnnn  1374508645 +0100
+    author nnnn nnnn  1374509958 +0100
+    committer nnnn nnnn  1374508641 +0100
+    committer nnnn nnnn  1374508642 +0100
+    committer nnnn nnnn  1374508643 +0100
+    committer nnnn nnnn  1374508644 +0100
+    committer nnnn nnnn  1374508645 +0100
+    committer nnnn nnnn  1374509958 +0100
+    d83e4134-8d7a-11e2-8822-7f2218f2cc42 em@x11miel:~/data/photos timestamp=1363356952.82141s
+    d83e4134-8d7a-11e2-8822-7f2218f2cc42 client timestamp=1363356952.865265s
+    e08336a0-6742-11e2-8a53-17c5ab00507f backup timestamp=1359154795.94703s
+    f7969bd0-d5dc-11e2-b362-cbb323f72415  timestamp=1371315579.82712s
+    f7969bd0-d5dc-11e2-b362-cbb323f72415 transfer timestamp=1371315580.420388s
+    merging 10.10.10.10__mnt_annex_00_annex/git-annex into git-annex (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race)
+    merging 10.10.10.10__mnt_annex_00_annex/git-annex into git-annex (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race)
+    merging 10.10.10.10__mnt_annex_00_annex/git-annex into git-annex (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race)
+    merging 10.10.10.10__mnt_annex_00_annex/git-annex into git-annex (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race)
+    merging 10.10.10.10__mnt_annex_00_annex/git-annex into git-annex (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race)
+    merging 10.10.10.10__mnt_annex_00_annex/git-annex into git-annex (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race)
+    merging 10.10.10.10__mnt_annex_00_annex/git-annex into git-annex (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race)
+    merging 10.10.10.10__mnt_annex_00_annex/git-annex into git-annex (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race)
+    merging 10.10.10.10__mnt_annex_00_annex/git-annex into git-annex (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race)
+    merging 10.10.10.10__mnt_annex_00_annex/git-annex into git-annex (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race)
+    merging 10.10.10.10__mnt_annex_00_annex/git-annex into git-annex (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race)
+    merging 10.10.10.10__mnt_annex_00_annex/git-annex into git-annex (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race)
+    parent 02683b204119ba5668286ef6aa803bb52d3515d0
+    parent 07e2a9755790cefa6004496570bc8094aa757afa
+    parent 09199d3870ae53d6a602c26f4b9d59f62411236c
+    parent 0d436b9b3c3766dec276f32fdb80251c2159c154
+    parent 5aca77f1f93d2f34b521e7844b0608242a70f68f
+    parent 5e3331cc8f28603e4cb8ce890fbe221d3857717c
+    parent 92b9cacae1f40a23a2e8d5feea2b058e39806bd4
+    parent bc52e17b49503d598347bc7ddb1fa05a1ca6eba2
+    parent bfc242d6cfc27ec5fb7ec5fdb745ef8e08028fd9
+    parent d7867e1afaa32a829cb875589764e921a92bde5b
+    parent ecab3543a69ac98a1509933fa25600ac3f9c091b
+    parent fcde6fd44ff2ad2aaca895bca1b32fb398d76545
+    tree 14dc2bb1ff7682d7c2f4d14ae1e5c378b0b89d47
+    tree 59d730a282fa721332a4a2686061d6ee11024e72
+    tree 5a452c8e00eb1d1fb67ef63ed74381b8f80b2ae0
+    tree 80cc5887614c1d5dd200b2ea6a90ee6b881b966b
+    tree af2b572801e828df16c195d4dfd368a1f37db33f
+    tree b026659bb4b1bf02bf146c0b873244230c7cea06
+    tree b33eaa96e7f04c014c93b4e388d831d6ec7c3746
+    tree b6d4b49d9b67b296b0a0d9a800b96f88d480f923
+
+"""]]
diff --git a/doc/bugs/non-repos_in_repositories_list___40__+_other_weird_output__41___from_git_annex_status/comment_4_d0e55585f1612148163039d157253258._comment b/doc/bugs/non-repos_in_repositories_list___40__+_other_weird_output__41___from_git_annex_status/comment_4_d0e55585f1612148163039d157253258._comment
new file mode 100644
index 0000000000..1e2a8fe247
--- /dev/null
+++ b/doc/bugs/non-repos_in_repositories_list___40__+_other_weird_output__41___from_git_annex_status/comment_4_d0e55585f1612148163039d157253258._comment
@@ -0,0 +1,11 @@
+[[!comment format=mdwn
+ username="http://joeyh.name/"
+ ip="2001:4978:f:21a::2"
+ subject="comment 4"
+ date="2013-07-26T16:52:26Z"
+ content="""
+The 4201 dangling blobs is a little bit strange, although it could certainly happen in some normal scenarios.
+Overall, your repository seems to be ok except for this weird data in the one file. I do not anticipate the extra garbage causing any problems at all.
+
+To track this down, we need to find the commit that added the garbage. One way, assuming you're using indirect mode, is to `git checkout git-annex; git blame uuid.log` and `git show` the commit that added the garbage. If you're using direct mode, you should first `git clone` the repository and do that in the clone.
+"""]]
diff --git a/doc/bugs/non-repos_in_repositories_list___40__+_other_weird_output__41___from_git_annex_status/comment_6_5506dc1b08516677886da4aa97263864._comment b/doc/bugs/non-repos_in_repositories_list___40__+_other_weird_output__41___from_git_annex_status/comment_6_5506dc1b08516677886da4aa97263864._comment
new file mode 100644
index 0000000000..1aaeff404f
--- /dev/null
+++ b/doc/bugs/non-repos_in_repositories_list___40__+_other_weird_output__41___from_git_annex_status/comment_6_5506dc1b08516677886da4aa97263864._comment
@@ -0,0 +1,12 @@
+[[!comment format=mdwn
+ username="http://joeyh.name/"
+ ip="4.152.246.110"
+ subject="comment 6"
+ date="2013-07-26T20:59:42Z"
+ content="""
+I think that the whole `git show $sha` for each commit will provide useful context.
+
+I want to understand what happened here, because there are multiple weird things going on, and it's important to root cause this sort of bug.
+
+What version of git-annex is on the rPi and how was it installed? Is there a simple operation you can do that segfaults every time?
+"""]]
diff --git a/doc/bugs/non-repos_in_repositories_list___40__+_other_weird_output__41___from_git_annex_status/comment_7_073449cc2cb73efd2b2d3d778a5573de._comment b/doc/bugs/non-repos_in_repositories_list___40__+_other_weird_output__41___from_git_annex_status/comment_7_073449cc2cb73efd2b2d3d778a5573de._comment
new file mode 100644
index 0000000000..f7046a3496
--- /dev/null
+++ b/doc/bugs/non-repos_in_repositories_list___40__+_other_weird_output__41___from_git_annex_status/comment_7_073449cc2cb73efd2b2d3d778a5573de._comment
@@ -0,0 +1,14 @@
+[[!comment format=mdwn
+ username="http://mey.vn/"
+ ip="46.65.14.106"
+ subject="comment 7"
+ date="2013-07-26T22:31:15Z"
+ content="""
+thanks Joey. i'm sending the output of ```git show $sha``` separately as it contains some non-public data; according to apt/dpkg logs, the rPi was actually using git-annex:armhf 4.20130521 from Debian testing at the time these commits were done. i did in fact install 4.20130709 and then 4.20130723 a couple of days after the issue showed up, but only yesterday i realised that both versions were segfaulting all the time (just invoking ```git-annex``` would segfault); 4.20130521 works reliably, instead (i have just downgraded to test), so my previous question about an interrupted operation possibly causing the metadata corruption is not valid as the git-annex executable in use at the time of the issue was not segfaulting (at least not under every circumstance as the more recent versions *in my setup* - but i'll dig into this separately - probably some library issue or so).
+
+the only change i can't really trace in terms of timing is when i deleted the remote i had originally set up on my rPi (using encrypted rsync) via the assistant's web interface on my laptop, and set it up again as plain unencrypted ssh remote (bare repo). i'm pretty sure this operation was done the day the commits in the ```git blame``` for ```uuid.log``` were done, but i'm not sure about the time and don't have logs going back to that moment.
+
+i could disconnect all the remotes from my laptop and redo the setup on the rPi to see if i can reproduce the issue, but i would need some pointers on how to revert my main annex on my laptop to the state it was in before i first started using the rPi as remote: can i just ```git reset --hard ``` while on the ```git-annex``` branch? the annex is currently in direct mode (and as been so for a few months now).
+
+otherwise, i could easily copy all the data (or a subset, to start with) to a clean folder, set up a fresh annex there and pair the rPi again - if the corruption was due to some issue on the rPi it may not depend on the exact status the annex was at when the metadata corruption happened (although my full setup includes other remotes and i see some back-and-forth of merging going on throughout the history of the annex around the time the metadata issue started, so it may be hard to reproduce the exact same context).
+"""]]
diff --git a/doc/bugs/non-repos_in_repositories_list___40__+_other_weird_output__41___from_git_annex_status/comment_7_3516e71ba3b07427a10cbb4965712aa6._comment b/doc/bugs/non-repos_in_repositories_list___40__+_other_weird_output__41___from_git_annex_status/comment_7_3516e71ba3b07427a10cbb4965712aa6._comment
new file mode 100644
index 0000000000..d8e91fdd4b
--- /dev/null
+++ b/doc/bugs/non-repos_in_repositories_list___40__+_other_weird_output__41___from_git_annex_status/comment_7_3516e71ba3b07427a10cbb4965712aa6._comment
@@ -0,0 +1,24 @@
+[[!comment format=mdwn
+ username="http://mey.vn/"
+ ip="46.65.14.106"
+ subject="comment 7"
+ date="2013-07-26T18:11:25Z"
+ content="""
+thanks; the current ```uuid.log``` file carries content from 8 distinct commits :
+
+    335cb49c  2013-07-22 16:57:32 +0100
+    4ec397f7  2013-07-22 16:57:26 +0100
+    50542096  2013-07-22 17:19:21 +0100
+    621a1a03  2013-07-22 16:57:28 +0100
+    63dd4509  2013-07-22 16:57:26 +0100
+    8c2d3ff7  2013-07-22 16:57:19 +0100
+    9ba4166d  2013-07-22 18:38:29 +0100
+    e182ee60  2013-07-22 16:57:26 +0100
+
+would the whole diff for each commit be useful, or just the part related to uuid.log?
+
+all these commits were done by git-annex on my laptop; the earliest of these commits come seconds after two commits done on another node - a raspberry pi running armhf Raspbian, although i now see that ```git-annex``` and ```git-annex-shell``` on this rPi are constantly segfaulting (though i can't understand the reason from the strace output): perhaps a segfault while dealing with a git operation on the rPi started the metadata corruption somehow?
+
+btw - if this could be useful to track down a possible bug in git-annex (although only involving some metadata, not the actual data), i am happy to dig further to provide all the information needed to understand what might have happened, but if it's not something worth investigating further in the bigger scheme of things, from my point of view i'm ok with restarting a fresh annex with content moved from this annex, and pristine metadata :)
+
+"""]]
diff --git a/doc/bugs/non-repos_in_repositories_list___40__+_other_weird_output__41___from_git_annex_status/comment_8_ea2e4704adb2f304f9c11c61eb62e919._comment b/doc/bugs/non-repos_in_repositories_list___40__+_other_weird_output__41___from_git_annex_status/comment_8_ea2e4704adb2f304f9c11c61eb62e919._comment
new file mode 100644
index 0000000000..a87e4cab2b
--- /dev/null
+++ b/doc/bugs/non-repos_in_repositories_list___40__+_other_weird_output__41___from_git_annex_status/comment_8_ea2e4704adb2f304f9c11c61eb62e919._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="http://mey.vn/"
+ ip="46.65.14.106"
+ subject="comment 8"
+ date="2013-07-26T22:38:51Z"
+ content="""
+forgot to mention - git-annex:armhf 4.20130521 was installed on the rPi via apt-get after having added testing as extra source besides the base stable of Raspbian; the other two versions (the segfaulting ones) were installed via ```dpkg -i``` on the packages downloaded directly from packages.d.o.
+"""]]
diff --git a/doc/bugs/non-repos_in_repositories_list___40__+_other_weird_output__41___from_git_annex_status/comment_9_4d17fedead7977541371a3f2c192e030._comment b/doc/bugs/non-repos_in_repositories_list___40__+_other_weird_output__41___from_git_annex_status/comment_9_4d17fedead7977541371a3f2c192e030._comment
new file mode 100644
index 0000000000..f2d9f56d84
--- /dev/null
+++ b/doc/bugs/non-repos_in_repositories_list___40__+_other_weird_output__41___from_git_annex_status/comment_9_4d17fedead7977541371a3f2c192e030._comment
@@ -0,0 +1,46 @@
+[[!comment format=mdwn
+ username="http://joeyh.name/"
+ ip="4.152.246.110"
+ subject="comment 9"
+ date="2013-07-27T17:37:11Z"
+ content="""
+Here's an excerpt of the first commit where uuid.log got garbage into it. It seems that some other log files got garbage put in them before this point, and it will be interesting to see if every time new garbage gets in it's doing a \"(recovery from race)\" commit. If so, that would point pretty strongly at that code as a culprit.
+
+(Of course, it's intriguing that the race recovery commit itself seemed to encounter a race and this repeated dozens of times. An exceptionally unlikely thing to happen if everything is behaving correctly.)
+
+
    +commit 8c2d3ff75431873d99c512bcae007d68ff0b565e 
+Author: REDACTED AUTHOR
+Date:   Mon Jul 22 16:57:19 2013 +0100
+
+    merging 10.4.10.106__mnt_annex_00_annex/git-annex into git-annex (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race)
+
+diff --git a/uuid.log b/uuid.log
+index 7e7e0f9..0423e02 100644
+--- a/uuid.log
++++ b/uuid.log
+@@ -1,4 +1,6 @@
++
+ 00511808-8d7c-11e2-a252-43e07a9aaa3c client timestamp=1363357452.108251s
++12527a3a-a21d-4cc7-a7ea-074e8fd0260f pi@pi0:/mnt/annex/00/annex timestamp=1374508625.460693s
+ 12527a3a-a21d-4cc7-a7ea-074e8fd0260f transfer timestamp=1374508627.731088s
+ 12a7b66c-e681-4dcc-bf75-b995512e5e76 client timestamp=1363469702.545491s
+ 1359159154.357141s 1 2ce1be52-6745-11e2-83e5-e7d111798afd
+@@ -28,6 +30,13 @@
+ 8d243fde-0c4c-4a19-8bce-ac1beb48eef0 unwanted timestamp=1374506712.447376s
+ 8d243fde-0c4c-4a19-8bce-ac1beb48eef0 unwanted timestamp=1374508303.426075s
+ 92aa538c-8d7b-11e2-9ae1-3fd2531928c1 client timestamp=1363357265.665495s
++author REDACTED AUTHOR 1374508635 +0100
++committer REDACTED AUTHOR 1374508635 +0100
+ d83e4134-8d7a-11e2-8822-7f2218f2cc42 client timestamp=1363356952.865265s
+ e08336a0-6742-11e2-8a53-17c5ab00507f backup timestamp=1359154795.94703s
+ f7969bd0-d5dc-11e2-b362-cbb323f72415 transfer timestamp=1371315580.420388s
++merging 10.4.10.106__mnt_annex_00_annex/git-annex into git-annex (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race)
++merging 10.4.10.106__mnt_annex_00_annex/git-annex into git-annex (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race) (recovery from race)
++parent 059104588cc3c45a762586843e8ae375473993af
++parent 21a14d1ed6ea5afa11eb07c7f16fb2dd03f5d64c
++tree c4ebf18a3d1d7862484b1378124c8cb7d2c704da
+
    
+
+Looks a lot like a whole git commit object was union merged into the file, doesn't it?
+"""]]
diff --git a/doc/bugs/occasional_hang_with_p2pstdio.mdwn b/doc/bugs/occasional_hang_with_p2pstdio.mdwn
new file mode 100644
index 0000000000..ed7c4decd7
--- /dev/null
+++ b/doc/bugs/occasional_hang_with_p2pstdio.mdwn
@@ -0,0 +1,66 @@
+Using git annex get -J20 of 1000 files from ssh remote on localhost,
+I've thrice observed it to hang.
+
+	get 99 (from origin...) (checksum...) ok
+	get 992 (from origin...) (checksum...) ok
+	get 991 (from origin...) (checksum...) ok
+	get 993 (from origin...) (checksum...) ok
+	get 995 (from origin...) (checksum...) ok
+	get 1000 (from origin...) 
+	get 1 (from origin...) 
+	get 10 (from origin...) 
+	get 108 (from origin...) 
+	get 105 (from origin...) 
+	[some more]
+
+It seems it's trying to receive content of the last files listed, but has
+hung somehow in the P2P protocol and not received the data. 
+Those are the only files not present. --[[Joey]]
+
+The particular set of files it stalls on seems somewhat deterministic;
+the sets have been the same at least twice.
+
+Looking at --debug, it does not seem to get to the point of sending a P2P
+request for the keys of the files that it stalls on.
+
+So, a bug setting up the P2P ssh connection, it seems.
+
+Interestingly, the debug log shows it only ran git-annex-shell p2pstdio 
+6 times, despite the concurrency of 20. So, the other 14 must have stalled
+setting up the connection. Suggests the bug is in the connection pool
+code.
+
+> The hang does not involve the connection pool code itself; a call to
+> Annex.Ssh.sshCommand is hanging. So, this likely affected git-annex
+> before p2pstdio, although its timings of calls to sshCommand may be
+> exposing the problem.
+> 
+> The hang is in prepSocket; all the threads enter makeconnection near the
+> same time, and so all of them try to warm up the ssh connection at the
+> same time. And somehow many of those executions of ssh hang.
+> (Arguably there should be locking to prevent multiple threads doing
+> this, but the actual overhead of multiple threads doing it may be
+> smaller than the overhead of such added locking.)
+> 
+> Converted makeconnection to not use processTranscript,
+> and that does seem to avoid the hang.
+>
+> Why is makeconnection's use of processTranscript hanging?
+> processTranscriot tries to read the process's output (ssh has none),
+> and waiting for the output to get read is for some reason hanging
+> forever, despite the ssh process becoming a zombie. I actually
+> rewrote the part of processtranscript that reads the process's input,
+> to be a much simpler use of async, and that new implementation has the
+> same problem. It's hanging, not throwing an exception. Most puzzling!
+> 
+> Hmm.. Perhaps the write handle is staying open even after ssh exits?
+> processTranscript sets up a pipe for the process to write to, and
+> the ssh process may inherit the FDs for that pipe (other than as stdout and
+> stderr). If the write handle
+> remains open, reading from it would block. Since the ssh mux server is
+> involved, and the handle might be passed to it or something, that seems
+> at least possible as the cause. The windows version of createProcess
+> does not use that pipe, and switching to use it does avoid the hang.
+> Yep! Setting the pipe's FDs to close on exec did avoid the hang.
+>
+> [[done]] --[[Joey]]
diff --git a/doc/bugs/on_encfs__44___.fuse__95__hiddenXXXXX_files_left_behind.mdwn b/doc/bugs/on_encfs__44___.fuse__95__hiddenXXXXX_files_left_behind.mdwn
new file mode 100644
index 0000000000..6c8f73f5e9
--- /dev/null
+++ b/doc/bugs/on_encfs__44___.fuse__95__hiddenXXXXX_files_left_behind.mdwn
@@ -0,0 +1,52 @@
+### Please describe the problem.
+
+When git annex is used on a encfs mount, ``.fuse_hiddenXXXX`` files are left behind in ``.git/annex/objects/*/*/`` when I ``git annex move`` files off the encfs
+
+### What steps will reproduce the problem?
+
+    mkdir normal plain enc
+    encfs -o kernel_cache "$(realpath enc)" "$(realpath plain)"
+
+    cd normal
+    git init
+    git annex init normal
+
+    cd ../plain
+    git init
+    git annex init enc
+    dd if=/dev/urandom bs=1M count=11 of=11m
+    git annex add 11m
+    git remote add normal "$(realpath ../normal)"
+    du -hscL *
+        << 11MB-ish >>
+    du -hsc .git/annex/objects
+        << 11MB-ish >>
+    git annex move --to=normal 11m
+    du -hscL *
+        << 0 >>
+    du -hsc .git/annex/objects
+        << 11MB-ish but should be 0-ish >>
+    tree -a .git/annex/objects
+        << ??/??/SHA1-*/.fuse_hidden??????? >>
+    lsof .git/annex/objects/??/??/SHA*/.fuse_hidden*
+        << shows no programs have this file open >>
+
+
+### What version of git-annex are you using? On what operating system?
+
+5.20131221 on debian unstable (installed by aptitude from debian/main)
+
+### Please provide any additional information below.
+
+I did a couple quick internet searches on these .fuse_hidden files, and (if a random person on the internet is correct) they are created by enfs (or maybe fuse itself) when a file is unlinked while open. It is supposed to be deleted when the file is closed.
+
+Possibilities come to mind:
+
+*    There's a bug in encfs/fuse
+*    git annex fails to close the file, and whatever cleanup happens when the linux kernel closes git annex fails to trigger encfs/fuse's real unlinking/cleanup thing
+
+
+
+I'm happy to help with further testing, experimenting, etc. I'm good at git, the commandline, programming, etc but I don't know Haskell.
+
+- Jason 
diff --git a/doc/bugs/on_encfs__44___.fuse__95__hiddenXXXXX_files_left_behind/comment_1_b96ea3ece44d42e79e4d903ca66065ee._comment b/doc/bugs/on_encfs__44___.fuse__95__hiddenXXXXX_files_left_behind/comment_1_b96ea3ece44d42e79e4d903ca66065ee._comment
new file mode 100644
index 0000000000..bf7aaedeee
--- /dev/null
+++ b/doc/bugs/on_encfs__44___.fuse__95__hiddenXXXXX_files_left_behind/comment_1_b96ea3ece44d42e79e4d903ca66065ee._comment
@@ -0,0 +1,12 @@
+[[!comment format=mdwn
+ username="http://joeyh.name/"
+ ip="209.250.56.227"
+ subject="comment 1"
+ date="2013-12-31T19:35:20Z"
+ content="""
+When git-annex is deleting an object's content, it opens the object, locks it, unlinks it, and closes it. This is necessary in order to avoid various race conditions that could cause data loss (ie, another repository might be in the process of verifying that the object that is being deleted is present).
+
+I suspect that is what causes the problem with encfs. Since it is entirely a valid POSIX operation to do this, and since not doing it could cause data loss in some situations, I am very reluctant to change anything in git-annex in this area.
+
+It's unfortunate that encfs seems to cause so many problems for POSIX filesystem semantics. Luckily, there are other ways to encrypt filesystems that do not suffer from such problems.
+"""]]
diff --git a/doc/bugs/parallel_get_can_fail_some_downloads_and_require_re-getting_.mdwn b/doc/bugs/parallel_get_can_fail_some_downloads_and_require_re-getting_.mdwn
new file mode 100644
index 0000000000..10dbd55476
--- /dev/null
+++ b/doc/bugs/parallel_get_can_fail_some_downloads_and_require_re-getting_.mdwn
@@ -0,0 +1,33 @@
+### Please describe the problem.
+
+Well, I remember whining before on this issue.  It does cause some notable inconvenience when trying to automate use of annex -- I am left without a choice but try to reget files multiple times without knowing what was actual cause for it to fail to start with.  Originally I observed it while content had to be wget'ed, to which I thought "oh well -- might be some connection overload etc".  But now I have tried on a repository which is local to that drive -- there must be no problem accessing multiple files at once whatsoever.  But on a trial X with -J5 I have error: 613, ok: 255, on next call  error: 415,  ok: 198 and so on -- kinda converging but imho it must not be so difficult.  I do suspect some race condition in annex itself preventing correct operation.
+
+### What steps will reproduce the problem?
+
+git annex get -J10   on some well sized annex.  Possibly with original annex to fetch content from just right on the same drive
+
+### What version of git-annex are you using? On what operating system?
+
+6.20170408+gitg804f06baa-1~ndall+1
+
+### Please provide any additional information below.
+
+[[!format sh """
+$> git annex get -J5 --json | grep -v '"success":true' 2>&1 | head
+{"command":"get","wanted":[{"here":false,"uuid":"3db23446-8c40-441e-97ec-55ffc86b4fc0","description":"yoh@smaug:~/proj/datalad/datalad/.git/travis-ci/origin-annex [origin]"}],"note":"Try making some of these repositories available:\n\t3db23446-8c40-441e-97ec-55ffc86b4fc0 -- yoh@smaug:~/proj/datalad/datalad/.git/travis-ci/origin-annex [origin]\n","skipped":[],"success":false,"key":"SHA256E-s328--c2eb8088cdc71a0d4cbd660312bef5421a47ce7da3655efdb17712e7188be4a1.txt.gz","file":"3728/3728.9-None.txt.gz"}
+{"command":"get","wanted":[{"here":false,"uuid":"3db23446-8c40-441e-97ec-55ffc86b4fc0","description":"yoh@smaug:~/proj/datalad/datalad/.git/travis-ci/origin-annex [origin]"}],"note":"Try making some of these repositories available:\n\t3db23446-8c40-441e-97ec-55ffc86b4fc0 -- yoh@smaug:~/proj/datalad/datalad/.git/travis-ci/origin-annex [origin]\n","skipped":[],"success":false,"key":"SHA256E-s62840--c20189a229fac622bb781650af394cf40367b5563a833885480f
+825fdbf29b47.txt.gz","file":"3729/3729.1-0.txt.gz"}
+
+...
+$> git annex get --key SHA256E-s328--c2eb8088cdc71a0d4cbd660312bef5421a47ce7da3655efdb17712e7188be4a1.txt.gz
+get SHA256E-s328--c2eb8088cdc71a0d4cbd660312bef5421a47ce7da3655efdb17712e7188be4a1.txt.gz (from origin...)
+SHA256E-s328--c2eb8088cdc71a0d4cbd660312bef5421a47ce7da3655efdb17712e7188be4a1.txt.gz
+            328 100%    0.00kB/s    0:00:00 (xfr#1, to-chk=0/1)
+(checksum...) ok
+(recording state in git...)
+
+"""]]
+
+[[!meta author=yoh]]
+
+> [[fixed|done]] --[[Joey]]
diff --git a/doc/bugs/parallel_get_can_fail_some_downloads_and_require_re-getting_/comment_1_1173126bd91dcb7fd13c57f06fd16c2b._comment b/doc/bugs/parallel_get_can_fail_some_downloads_and_require_re-getting_/comment_1_1173126bd91dcb7fd13c57f06fd16c2b._comment
new file mode 100644
index 0000000000..469518202b
--- /dev/null
+++ b/doc/bugs/parallel_get_can_fail_some_downloads_and_require_re-getting_/comment_1_1173126bd91dcb7fd13c57f06fd16c2b._comment
@@ -0,0 +1,14 @@
+[[!comment format=mdwn
+ username="joey"
+ subject="""comment 1"""
+ date="2017-05-25T17:29:08Z"
+ content="""
+Let's not diagnose a concurrency problem prematurely.
+
+I've run many concurrent gets and never seen masses of failures, or indeed any
+failures without either an error message explaining why it failed, or something
+obviously wrong (like the drive not being mounted).
+
+The json output is making it harder than necessary to understand what's going on.
+It seems you should be easily able to replicate the same problem without --json.
+"""]]
diff --git a/doc/bugs/parallel_get_can_fail_some_downloads_and_require_re-getting_/comment_2_2ba8101ed0e91df7079744d1b37779fd._comment b/doc/bugs/parallel_get_can_fail_some_downloads_and_require_re-getting_/comment_2_2ba8101ed0e91df7079744d1b37779fd._comment
new file mode 100644
index 0000000000..c9a6f133ac
--- /dev/null
+++ b/doc/bugs/parallel_get_can_fail_some_downloads_and_require_re-getting_/comment_2_2ba8101ed0e91df7079744d1b37779fd._comment
@@ -0,0 +1,51 @@
+[[!comment format=mdwn
+ username="yarikoptic"
+ avatar="http://cdn.libravatar.org/avatar/f11e9c84cb18d26a1748c33b48c924b4"
+ subject="comment 2"
+ date="2017-05-25T18:39:03Z"
+ content="""
+didn't need to go far ;)
+
+[[!format sh \"\"\"
+$> git annex get -J5 
+(merging origin/git-annex into git-annex...)
+(recording state in git...)
+get 10/10.1-None.txt.gz get 1000/1000.2-0.txt.gz get 1000/1000.1-0.txt.gz get 1000/1000.3-0.txt.gz get 1000/1000.4-1.txt.gz error: could not lock config file .git/config: File exists
+(from origin...) (from origin...) (from origin...) 
+
+(transfer already in progress, or unable to take transfer lock) 
+  Unable to access these remotes: origin
+(from origin...) 
+
+  Try making some of these repositories available:
+  	3db23446-8c40-441e-97ec-55ffc86b4fc0 -- yoh@smaug:~/proj/datalad/datalad/.git/travis-ci/origin-annex [origin]
+failed
+SHA256E-s55194--48621840b3a869ac27dfdb2a201c202ee1382aa0114337563da2c38262a0c9e8.txt.gz
+         55,194 100%   21.39MB/s    0:00:00 (xfr#1, to-chk=0/1)
+SHA256E-s56291--48af9865017aca9c8931c3b91c4f0ff9954273d21007ff25de1a2c73ca0df95cSHA256E-s7813--7861750b475c620012f1a76635749d89b86b7777130352b6adb844753d3822c2.txt.gz
+.txt.gz
+         32,768  58%    0.00kB/s    0:00:00  (checksum...)    0:00:00 (xfr#1, to-chk=0/1)
+         56,291 100%   22.43MB/s    0:00:00 (xfr#1, to-chk=0/1)
+(checksum...) (checksum...) ok
+ok
+ok
+git-annex: git [Param \"config\",Param \"remote.origin.annex-uuid\",Param \"3db23446-8c40-441e-97ec-55ffc86b4fc0\"] failed
+CallStack (from HasCallStack):
+  error, called at ./Git/Command.hs:39:17 in main:Git.Command
+
+$> git annex get 2>&1 | head
+get 1000/1000.2-0.txt.gz (from origin...) 
+SHA256E-s56206--1fa38270b23b4c700c4a998712549c65782613d4567b8650b1ebaa3fcfc884f4.txt.gz
+         56,206 100%   22.35MB/s    0:00:00 (xfr#1, to-chk=0/1)
+(checksum...) ok
+get 1000/1000.3-0.txt.gz (from origin...) 
+SHA256E-s55880--dea4463f7e42c51463e236d29c4c40486b56754cbd8340c260e481bdd506115b.txt.gz
+         55,880 100%   22.04MB/s    0:00:00 (xfr#1, to-chk=0/1)
+(checksum...) ok
+get 1000/1000.5-1.txt.gz (from origin...) 
+SHA256E-s55028--f6f8d5f25bed2bb51d9ba17275a956a97276cdcc74c6538c7e590919426e97df.txt.gz
+
+\"\"\"]]
+
+FWIW, the repository in question is this one: http://datasets.datalad.org/devel/travis-buildlogs/.git/
+"""]]
diff --git a/doc/bugs/parallel_get_can_fail_some_downloads_and_require_re-getting_/comment_3_6674e4dbc7437ce941bcef6272c3433b._comment b/doc/bugs/parallel_get_can_fail_some_downloads_and_require_re-getting_/comment_3_6674e4dbc7437ce941bcef6272c3433b._comment
new file mode 100644
index 0000000000..a004b75b90
--- /dev/null
+++ b/doc/bugs/parallel_get_can_fail_some_downloads_and_require_re-getting_/comment_3_6674e4dbc7437ce941bcef6272c3433b._comment
@@ -0,0 +1,40 @@
+[[!comment format=mdwn
+ username="joey"
+ subject="""comment 3"""
+ date="2017-05-25T19:08:59Z"
+ content="""
+That looks like concurrent `git config` setting remote.origin.annex-uuid
+are failing.
+
+I have not reproduced the `.git/config` error, but with a local
+clone of a repository, I have been able to reproduce some intermittent
+"transfer already in progress, or unable to take transfer lock" failures
+with `git annex get -J5`, happening after remote.origin.annex-uuid has been
+cached.
+
+So, two distinct bugs I think..
+
+---
+
+Debugging, the lock it fails to take always seems to be the lock on
+the remote side, which points to the local clone being involved somehow.
+
+Debugging further, Utility.LockPool.STM.tryTakeLock is what's failing.
+That's supposed to only fail when another thread holds a conflicting lock,
+but as it's implemented with `orElse`, if the main STM
+transaction retries due to other STM activity on the same TVar,
+it will give up when it shouldn't.
+
+That's probably why this is happening under heavier concurrency loads;
+it makes that failure case much more likely. And with a local clone,
+twice as much locking is done.
+
+I've fixed this part of it! 
+
+---
+
+The concurrent `git config` part remains.
+Since git-annex can potentially have multiple threads doing different `git
+config` for their own reasons concurrently, it seems it will need to add
+its own locking around that.
+"""]]
diff --git a/doc/bugs/parallel_get_can_fail_some_downloads_and_require_re-getting_/comment_4_480df575c68bfb37b8bb4fb43737726f._comment b/doc/bugs/parallel_get_can_fail_some_downloads_and_require_re-getting_/comment_4_480df575c68bfb37b8bb4fb43737726f._comment
new file mode 100644
index 0000000000..3827077c2f
--- /dev/null
+++ b/doc/bugs/parallel_get_can_fail_some_downloads_and_require_re-getting_/comment_4_480df575c68bfb37b8bb4fb43737726f._comment
@@ -0,0 +1,29 @@
+[[!comment format=mdwn
+ username="joey"
+ subject="""comment 4"""
+ date="2017-05-25T21:52:37Z"
+ content="""
+The .git/config concurrent access happens because the remote
+list is only generated on demand, and nothing demands it when running with
+-J until all the threads are spun up and each thread has its own state
+then, so each generates the remote list.
+
+There don't look to be any other git-config settings that
+would cause problems for concurrency other than ones run while
+generating the remote list.
+
+So, generating the remote list before starting concurrent threads
+would avoid that problem, and also leads to a slightly faster startup
+since the remote git config only has to be read once, etc.
+
+The only risk in doing that would be if generating a Remote
+opens some kind of handle, which can't be used concurrently, or
+is less efficient if only opened once and then used by multiple threads.
+
+I've audited all the Remote.*.gen methods, and they all
+seem ok. For example, Remote.External.gen sets up a worker pool
+of external special remote processes, and new ones are allocated as needed.
+And Remote.P2P.chainGen sets up a connection pool.
+
+Ok, gone ahead with this fix.
+"""]]
diff --git a/doc/bugs/parallel_get_can_fail_some_downloads_and_require_re-getting_/comment_5_03e474b00b1cb2a5b8244d9161857b9b._comment b/doc/bugs/parallel_get_can_fail_some_downloads_and_require_re-getting_/comment_5_03e474b00b1cb2a5b8244d9161857b9b._comment
new file mode 100644
index 0000000000..abc1d700b8
--- /dev/null
+++ b/doc/bugs/parallel_get_can_fail_some_downloads_and_require_re-getting_/comment_5_03e474b00b1cb2a5b8244d9161857b9b._comment
@@ -0,0 +1,15 @@
+[[!comment format=mdwn
+ username="yarikoptic"
+ avatar="http://cdn.libravatar.org/avatar/f11e9c84cb18d26a1748c33b48c924b4"
+ subject="woohoo"
+ date="2017-05-26T02:03:57Z"
+ content="""
+Thank you Joey!  It seems to work very nice now!  Not a single one lost out of 1550!
+
+[[!format sh \"\"\"
+$> datalad get -J5 38*
+[INFO   ] Getting 100 items of dataset  ... 
+[INFO   ] Actually getting 1550 files 
+Tried to get 1550 files. Got 1550. 
+\"\"\"]]
+"""]]
diff --git a/doc/bugs/post-receive.mdwn b/doc/bugs/post-receive.mdwn
new file mode 100644
index 0000000000..e9422ce212
--- /dev/null
+++ b/doc/bugs/post-receive.mdwn
@@ -0,0 +1,69 @@
+### Please describe the problem.
+
+This is actually two (related) bugs.
+I discovered the second by means of troubleshooting the first.
+
+One issue I've had with a portable repo is when I init it from my arch machine it sets up everything as expected.
+When I try to use that drive with my raspberry pi, however, there are some new git hooks (or at least one) that the older version of git annex (still the latest available in the Pi's repos, 2016, which is really old :/) does not support. 
+
+What is necessary to get a newer version of git-annex available in the Raspbian repos for default users? I know I can just install the tarball (and I'm considering it) but for everyone else coming upon this issue...
+
+### What steps will reproduce the problem?
+See above. git-annex init on a newer system, then mount it as a drive on a Raspberry pi (with the older git-annex installed) and set up the pi as a remote. Then git-annex sync.
+
+Note: this is fixable by deleting the post-receive hook in the .git/hooks folder. I'm just not sure that's a great idea.
+
+### What version of git-annex are you using? On what operating system?
+Arch linux: 
+
+git-annex version: 6.20180913-g547d01fd0
+build flags: Assistant Webapp Pairing S3(multipartupload)(storageclasses) WebDAV Inotify DBus DesktopNotify ConcurrentOutput TorrentParser MagicMime Feeds Testsuite
+dependency versions: aws-0.20 bloomfilter-2.0.1.0 cryptonite-0.25 DAV-1.3.2 feed-1.0.0.0 ghc-8.4.3 http-client-0.5.13.1 persistent-sqlite-2.8.2 torrent-10000.1.1 uuid-1.3.13 yesod-1.6.0
+key/value backends: SHA256E SHA256 SHA512E SHA512 SHA224E SHA224 SHA384E SHA384 SHA3_256E SHA3_256 SHA3_512E SHA3_512 SHA3_224E SHA3_224 SHA3_384E SHA3_384 SKEIN256E SKEIN256 SKEIN512E SKEIN512 BLAKE2B256E BLAKE2B256 BLAKE2B512E BLAKE2B512 BLAKE2B160E BLAKE2B160 BLAKE2B224E BLAKE2B224 BLAKE2B384E BLAKE2B384 BLAKE2S256E BLAKE2S256 BLAKE2S160E BLAKE2S160 BLAKE2S224E BLAKE2S224 BLAKE2SP256E BLAKE2SP256 BLAKE2SP224E BLAKE2SP224 SHA1E SHA1 MD5E MD5 WORM URL
+remote types: git gcrypt p2p S3 bup directory rsync web bittorrent webdav adb tahoe glacier ddar hook external
+operating system: linux x86_64
+supported repository versions: 3 5 6
+upgrade supported from repository versions: 0 1 2 3 4 5
+local repository version: 5
+
+
+Raspberry Pi:
+
+git-annex version: 6.20160923
+build flags: Assistant Webapp Pairing Testsuite S3(multipartupload)(storageclasses) WebDAV Inotify DBus DesktopNotify XMPP ConcurrentOutput TorrentParser MagicMime Feeds Quvi
+key/value backends: SHA256E SHA256 SHA512E SHA512 SHA224E SHA224 SHA384E SHA384 SHA3_256E SHA3_256 SHA3_512E SHA3_512 SHA3_224E SHA3_224 SHA3_384E SHA3_384 SKEIN256E SKEIN256 SKEIN512E SKEIN512 SHA1E SHA1 MD5E MD5 WORM URL
+remote types: git gcrypt S3 bup directory rsync web bittorrent webdav tahoe glacier ddar hook external
+local repository version: 5
+supported repository versions: 5 6
+upgrade supported from repository versions: 0 1 2 3 4 5
+operating system: linux arm
+
+
+
+
+### Please provide any additional information below.
+
+I've also noticed a bug with tab complete on the latest git-annex when in a folder that is not git-annexed. I was looking into git-annex post-receive, and typed git-annex pos to get a listing of possible outputs. This was the result:
+
+
+
+[[!format sh """
+# If you can, paste a complete transcript of the problem occurring here.
+# If the problem is with the git-annex assistant, paste in .git/annex/daemon.log
+
+tai@trasa:~$ git-annex post-receive git-annex: Not in a git repository.
+git-annex: Not in a git repository.
+
+Display all 105 possibilities? (y or n)^C
+tai@trasa:~$
+
+
+
+# End of transcript or log.
+"""]]
+
+I did not hit enter, the script just failed on me during tab-complete and exited.
+
+Thanks, I look forward to any response from the community this might get.
+
+> [[fixed|done]] --[[Joey]]
diff --git a/doc/bugs/post-receive/comment_1_80c2251beb6a434b364346abc1b2fc02._comment b/doc/bugs/post-receive/comment_1_80c2251beb6a434b364346abc1b2fc02._comment
new file mode 100644
index 0000000000..589bee8067
--- /dev/null
+++ b/doc/bugs/post-receive/comment_1_80c2251beb6a434b364346abc1b2fc02._comment
@@ -0,0 +1,22 @@
+[[!comment format=mdwn
+ username="joey"
+ subject="""comment 1"""
+ date="2018-09-25T18:17:59Z"
+ content="""
+> When I try to use that drive with my raspberry pi, however, there are some new git hooks (or at least one) that the older version of git annex (still the latest available in the Pi's repos, 2016, which is really old :/).
+
+Ok, that sentance no verb. But I'll make a guess what you meant to say..
+The old post-receive hook installed by the new version of git-annex runs
+"git-annex post-receive", which fails on the old version of git-annex.
+
+Yes, it's fine to delete the hook in this situation.
+
+The fist version of git-annex to support that is 6.20170228.
+The latest raspbian release is tracking debian stable AFAICS, which has
+6.20170101, just slightly too old.
+
+I agree this is a backwards compatability problem that should have been avoided.
+I've made `git annex init` generate a better hook script that won't fail
+with an older git-annex version. You can re-run `git annex init` in
+your repository and it will update the hook script.
+"""]]
diff --git a/doc/bugs/post-receive/comment_2_1f4930040f00ecc021ad5d6a67f041d9._comment b/doc/bugs/post-receive/comment_2_1f4930040f00ecc021ad5d6a67f041d9._comment
new file mode 100644
index 0000000000..dc815dbf5c
--- /dev/null
+++ b/doc/bugs/post-receive/comment_2_1f4930040f00ecc021ad5d6a67f041d9._comment
@@ -0,0 +1,11 @@
+[[!comment format=mdwn
+ username="tai@83c0edf140a3f133031751b49c7131d9535a3fcc"
+ nickname="tai"
+ avatar="http://cdn.libravatar.org/avatar/443f3677ce7e3cabbe09cdb8ad648915"
+ subject="Thanks"
+ date="2018-09-25T19:52:03Z"
+ content="""
+Awesome, thanks so much. 
+
+Note: I the grammar in my sentence so it makes more sense now, but yes you guessed right.
+"""]]
diff --git a/doc/bugs/preferred_content__58___include_statement_does_not_allow_spaces_in_filenames.mdwn b/doc/bugs/preferred_content__58___include_statement_does_not_allow_spaces_in_filenames.mdwn
new file mode 100644
index 0000000000..56311bc70e
--- /dev/null
+++ b/doc/bugs/preferred_content__58___include_statement_does_not_allow_spaces_in_filenames.mdwn
@@ -0,0 +1,31 @@
+### Please describe the problem.
+Filenames for the *include* statement for [preferred content](http://git-annex.branchable.com/preferred_content/) can not contain spaces.
+
+### What steps will reproduce the problem?
+
+* Create an annex repo
+* Run `git annex vicfg`
+* Enter expression *include='pictures/dir with spaces'*
+* git annex complains: # ** Parse error in next line: Parse failure: near "with" Parse failure: near "spaces'"
+
+(The *'* is interpreted as part of the filepath.)
+
+### What version of git-annex are you using? On what operating system?
+
+I am using the current binaries from Debian stable amd64.
+
+
    +git-annex version: 5.20140320~bpo70+1
+build flags: Assistant Webapp Pairing S3 Inotify DBus XMPP Feeds Quvi TDFA
+key/value backends: SHA256E SHA1E SHA512E SHA224E SHA384E SHA256 SHA1 SHA512 SHA224 SHA384 WORM URL
+remote types: git gcrypt S3 bup directory rsync web tahoe glacier hook external
+local repository version: 5
+supported repository version: 5
+upgrade supported from repository versions: 0 1 2 4
+
    
+
+### Please provide any additional information below.
+
+The only workaround I found is to use a glob for the filepath which only works for the first space: *include='pictures/dir\*'*.
+
+[[!tag confirmed]]
diff --git a/doc/bugs/preferred_content__58___include_statement_does_not_allow_spaces_in_filenames/comment_1_ca10638d4b4b178cfd0de8736542c4dc._comment b/doc/bugs/preferred_content__58___include_statement_does_not_allow_spaces_in_filenames/comment_1_ca10638d4b4b178cfd0de8736542c4dc._comment
new file mode 100644
index 0000000000..a7438c559e
--- /dev/null
+++ b/doc/bugs/preferred_content__58___include_statement_does_not_allow_spaces_in_filenames/comment_1_ca10638d4b4b178cfd0de8736542c4dc._comment
@@ -0,0 +1,12 @@
+[[!comment format=mdwn
+ username="http://joeyh.name/"
+ ip="209.250.56.244"
+ subject="comment 1"
+ date="2014-04-02T18:45:36Z"
+ content="""
+A better workaround is:
+
+include='pictures/dir?with?spaces'
+
+Tokenizing text with embedded quotes is a bit of a PITA, certianly doable I suppose..
+"""]]
diff --git a/doc/bugs/preferred_content__58___include_statement_does_not_allow_spaces_in_filenames/comment_2_986a393a512229d35e529ba242b77b1e._comment b/doc/bugs/preferred_content__58___include_statement_does_not_allow_spaces_in_filenames/comment_2_986a393a512229d35e529ba242b77b1e._comment
new file mode 100644
index 0000000000..7dbeb3d088
--- /dev/null
+++ b/doc/bugs/preferred_content__58___include_statement_does_not_allow_spaces_in_filenames/comment_2_986a393a512229d35e529ba242b77b1e._comment
@@ -0,0 +1,10 @@
+[[!comment format=mdwn
+ username="http://ypid.wordpress.com/"
+ ip="213.153.84.215"
+ subject="comment 2"
+ date="2014-04-02T21:29:54Z"
+ content="""
+Thanks for your workaround ... Works for me after removing the single quote signs.
+
+With this workaround on hand one could include/match any filename. Works for me ;)
+"""]]
diff --git a/doc/bugs/prematurely___40__can__39__t_check_offline__41___marks_remote_as_annex-ignore.mdwn b/doc/bugs/prematurely___40__can__39__t_check_offline__41___marks_remote_as_annex-ignore.mdwn
new file mode 100644
index 0000000000..41e573dda2
--- /dev/null
+++ b/doc/bugs/prematurely___40__can__39__t_check_offline__41___marks_remote_as_annex-ignore.mdwn
@@ -0,0 +1,29 @@
+### Please describe the problem.
+
+I have cloned via http and then went offline... decided to get a file, annex immediately tried to fetch it from origin which it couldn't have access to atm and marked it as annex-ignore.  IMHO it is a bit premature action, and annex should mark a remote as ignore only if it managed to reach the remote location and discovered that it has no annex there, not on a mere connectivity fluke
+
+
+### What version of git-annex are you using? On what operating system?
+
+6.20160425+gitgffe2ea2-1~ndall+1
+
+### Please provide any additional information below.
+
+[[!format sh """
+$> git annex get cond001.txt
+(merging origin/git-annex into git-annex...)
+(recording state in git...)
+get cond001.txt 
+  Remote origin not usable by git-annex; setting annex-ignore
+(not available) 
+  Try making some of these repositories available:
+  	899f0347-0888-48ef-91b6-bac213ca8cef -- datalad-archives
+   	c8bd3d05-33d4-4b59-9d53-ca7efbdcdd13 -- yoh@smaug:/mnt/btrfs/datasets/datalad/crawl/openfmri/ds000001
+failed
+(recording state in git...)
+git-annex: get: 1 failed
+"""]]
+
+[[!meta author=yoh]]
+
+> [[fixed|done]] --[[Joey]]
diff --git a/doc/bugs/present_files__47__directories_are_dropped_after_a_sync.mdwn b/doc/bugs/present_files__47__directories_are_dropped_after_a_sync.mdwn
new file mode 100644
index 0000000000..69c029cd99
--- /dev/null
+++ b/doc/bugs/present_files__47__directories_are_dropped_after_a_sync.mdwn
@@ -0,0 +1,41 @@
+### Please describe the problem.
+
+This is a followup from the discussion on  where I unfortunately did not get a complete answer.
+I don't know if it is really a bug but at least it does not work as I would expect and the documentation provides no clear discussion on that.
+
+Now to the problem:
+My annex is in "manual" mode (or equivalently "exclude="*" and present" or an expression which contains "present".
+Then I get a file using "git annex get file".
+I would expect that this file is now synced because it is "present".
+But it is not. When I change the file it is synced to the remotes. This is what it should be.
+However, when a remote changes that file, the content is NOT synced, the file is silently dropped.
+
+Similarly, when I get a complete directory tree in manual mode, I would expect that it is synced. That means, when a remote adds a file or changes a file in that directory, it is also synced to the local machine. But it is not. If it is changed, it is silently dropped (as written above). If a file is added, only the metadata is added but the content is not synced.
+
+### What steps will reproduce the problem?
+
+ - Create a file 'file' on the server, git annex add/sync etc.
+ - On the client: git annex wanted here 'exclude="*" and present'
+ - On the client: git annex get file . The file is now present on the client
+ - Change the file on the server, git annex sync
+ - git annex sync --content on the client
+ - Result: File is dropped again on client
+
+Similarly for directories:
+
+ - Create a (sub-)directory 'subdir' with files and sync everything
+ - On the client: git annex get subdir . The subdirectory is now present, all files under it downloaded.
+ - On the server create a new file in 'subdir' and git annex add; git annex sync --content
+ - git annex sync --content on the client
+ - Result: Content of the files is not synced to client
+
+### What version of git-annex are you using? On what operating system?
+
+    git-annex version: 5.20140717-g5a7d4ff
+    build flags: Assistant Webapp Webapp-secure Pairing Testsuite S3 WebDAV DNS Feeds Quvi TDFA CryptoHash
+    key/value backends: SHA256E SHA1E SHA512E SHA224E SHA384E SKEIN256E SKEIN512E SHA256 SHA1 SHA512 SHA224 SHA384 SKEIN256 SKEIN512 WORM URL
+    remote types: git gcrypt S3 bup directory rsync web webdav tahoe glacier ddar hook external
+
+
+[[!meta title="manual mode preferred content expression does not want newer versions of present files"]]
+[[!tag confirmed]]
diff --git a/doc/bugs/present_files__47__directories_are_dropped_after_a_sync/comment_1_9d7591faf99ce48b1e5753c80306ae8b._comment b/doc/bugs/present_files__47__directories_are_dropped_after_a_sync/comment_1_9d7591faf99ce48b1e5753c80306ae8b._comment
new file mode 100644
index 0000000000..bb81333545
--- /dev/null
+++ b/doc/bugs/present_files__47__directories_are_dropped_after_a_sync/comment_1_9d7591faf99ce48b1e5753c80306ae8b._comment
@@ -0,0 +1,10 @@
+[[!comment format=mdwn
+ username="https://www.google.com/accounts/o8/id?id=AItOawlM_DRhi_5pJrTA0HbApHR25iAgy-NBXTY"
+ nickname="Tor Arne"
+ subject="comment 1"
+ date="2014-10-01T22:25:24Z"
+ content="""
+Have you found a solution for this? This seems useful if you're only interested in a subset of files/directories on your laptop, eg, but those that are fetched (present) that you are interested you'd want to keep up to date (in sync) with other computers?
+
+Btw, the link to the previous discussion didnt work for me.
+"""]]
diff --git a/doc/bugs/present_files__47__directories_are_dropped_after_a_sync/comment_2_7316cba69b9dc0415fea1389238edf25._comment b/doc/bugs/present_files__47__directories_are_dropped_after_a_sync/comment_2_7316cba69b9dc0415fea1389238edf25._comment
new file mode 100644
index 0000000000..194f7a3d9f
--- /dev/null
+++ b/doc/bugs/present_files__47__directories_are_dropped_after_a_sync/comment_2_7316cba69b9dc0415fea1389238edf25._comment
@@ -0,0 +1,14 @@
+[[!comment format=mdwn
+ username="http://joeyh.name/"
+ ip="209.250.56.96"
+ subject="comment 2"
+ date="2014-10-21T20:20:25Z"
+ content="""
+The problem is that there's no way for preferred content expressions to specify that a file is wanted just because some old version of the file is (or was) present.
+
+It's not clear to me how that could be added to the preferred content expressions in an efficient way.
+
+It might be possible to hack `git annex sync --content` and the assistant to look at incoming merges, and queue downloads of newer versions of files before merging.
+
+Also being discussed at .
+"""]]
diff --git a/doc/bugs/present_files__47__directories_are_dropped_after_a_sync/comment_3_0d53226a3991e3685d1311e4b19c4023._comment b/doc/bugs/present_files__47__directories_are_dropped_after_a_sync/comment_3_0d53226a3991e3685d1311e4b19c4023._comment
new file mode 100644
index 0000000000..a048d31b5d
--- /dev/null
+++ b/doc/bugs/present_files__47__directories_are_dropped_after_a_sync/comment_3_0d53226a3991e3685d1311e4b19c4023._comment
@@ -0,0 +1,55 @@
+[[!comment format=mdwn
+ username="joey"
+ subject="""complications"""
+ date="2015-09-22T18:34:43Z"
+ content="""
+* The "look at incoming merges, and queue downloads of newer versions of
+  present files" approach needs to do something about the case where
+  it's not able to successfully download a newer version immediately.
+  
+  If it let the merge proceed, the file would end up not being present
+  anymore, and so a later sync wouldn't know it had been present.
+  
+  Failing to get the contents of all changed files could just make the
+  sync fail before it merges, keeping the tree at the earlier version.
+  This might be desirable.
+
+  But, implementing that means changing sync to download file contents
+  before merging, rather than the current merge-first. I'm sure a lot of
+  people *won't* want that. (Ie, I certianly don't.) So, this seems to need
+  to be a new mode for syncing. 
+  
+  (Such a mode is probably generally useful, aside from this use case.)
+
+* If this was implemented, then when a file is modified, the content
+  of the new file would be present. git-annex already makes it so that,
+  when a file is moved, the content of the file is still present. But,
+  what if a file were first moved and then modified? If that happened in
+  multiple commits, they could be examined in turn (with additional
+  complication and slowdown) to conclude that the content is wanted. But if
+  that happened in a single commit, there's no way to tell that from
+  deleting the old file and adding a new file, whose content would not be
+  automatically wanted.
+
+* The bug report wants git-annex to somehow detect when the user has
+  manually gotten an entire directory tree and start getting new files in
+  that directory too, which seems pretty infeasible. How is git-annex
+  supposed to guess whether you want new files in a directory tree,
+  or just the files that are currently there? What if some files
+  are duplicated amoung 2 directory trees, and one tree ends up complete
+  while the other one doesn't? This seems like a request for mindreading
+  ponies.
+
+* There are many preferred content expressions that fully specify what
+  files are wanted, without using the "present" token. AFAICS,
+  there's no reason to do any of this work when the preferred content
+  expression doesn't include "present".
+
+TBH, I am not at all sure this is implementable anywhere near sanely.
+If I were you, I'd use preferred content to specify the files I want,
+which avoids these complexities and works great. Using metadata to tag
+files and making all tagged files be wanted in the preferred content
+expression is one nice way to go. And metadata is copied over when adding
+a new version of a file, so this tagging approach works across file
+modifications.
+"""]]
diff --git a/doc/bugs/present_files__47__directories_are_dropped_after_a_sync/comment_4_68bd50c6aeffc06d53eda7f38b06edee._comment b/doc/bugs/present_files__47__directories_are_dropped_after_a_sync/comment_4_68bd50c6aeffc06d53eda7f38b06edee._comment
new file mode 100644
index 0000000000..2b35b662d0
--- /dev/null
+++ b/doc/bugs/present_files__47__directories_are_dropped_after_a_sync/comment_4_68bd50c6aeffc06d53eda7f38b06edee._comment
@@ -0,0 +1,7 @@
+[[!comment format=mdwn
+ username="joey"
+ subject="""comment 4"""
+ date="2015-09-22T19:18:09Z"
+ content="""
+Broke part out into [[todo/deferred_update_mode]]
+"""]]
diff --git a/doc/bugs/problem_commit_normal_links.mdwn b/doc/bugs/problem_commit_normal_links.mdwn
new file mode 100644
index 0000000000..6dbd41fb4e
--- /dev/null
+++ b/doc/bugs/problem_commit_normal_links.mdwn
@@ -0,0 +1,59 @@
+Dear All,
+
+thank you for this wonderful tool!
+
+I am having an issue when I try to commit a normal link
+
+diokletian*194-> mkdir test
+
+diokletian*195-> cd test
+
+diokletian*196-> git init
+
+Initialized empty Git repository in /home/henrus/test/.git/
+
+diokletian*197-> git annex init new
+
+init new [master (root-commit) 49f5f91] git-annex setup
+
+ 1 files changed, 1 insertions(+), 0 deletions(-)
+
+ create mode 100644 .gitattributes
+
+[master 76496ff] git annex init
+
+ 1 files changed, 1 insertions(+), 0 deletions(-)
+
+ create mode 100644 .git-annex/uuid.log
+
+ok
+
+diokletian*198-> mkdir subdir
+
+diokletian*199-> ln -s subdir link
+
+diokletian*200-> git add link
+
+diokletian*201-> git commit -m "ok"
+
+[master f12f62d] ok
+
+ 1 files changed, 1 insertions(+), 0 deletions(-)
+
+ create mode 120000 link
+
+diokletian*202-> ln -s subdir/ link2
+
+diokletian*203-> git add link2
+
+diokletian*204-> git commit -m "not ok"
+
+git-annex: Prelude.head: empty list
+
+The trailing slash seems to make a difference!
+
+Best Regards,
+
+Henrik
+
+> Thanks for the bug report. This is fixed in 0.17. --[[Joey]]  [[!tag done]]
diff --git a/doc/bugs/problem_with_upgrade_v2_-__62___v3.mdwn b/doc/bugs/problem_with_upgrade_v2_-__62___v3.mdwn
new file mode 100644
index 0000000000..7f37668ad1
--- /dev/null
+++ b/doc/bugs/problem_with_upgrade_v2_-__62___v3.mdwn
@@ -0,0 +1,3 @@
+On several of my repos, the upgrade to v3 seemed to take forever. A Crl-C followed by another "git annex upgrade" "solved" the problem in some cases. Sometimes, I had to also delete the .git/annex/journal dir to have the upgrade. I didn't notice anything special about the non-working repos to help diagnose the problem.
+
+[[!tag done]]
diff --git a/doc/bugs/problem_with_upgrade_v2_-__62___v3/comment_1_5f60006c9bb095167d817f234a14d20b._comment b/doc/bugs/problem_with_upgrade_v2_-__62___v3/comment_1_5f60006c9bb095167d817f234a14d20b._comment
new file mode 100644
index 0000000000..0cf0ad4618
--- /dev/null
+++ b/doc/bugs/problem_with_upgrade_v2_-__62___v3/comment_1_5f60006c9bb095167d817f234a14d20b._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="http://joey.kitenet.net/"
+ nickname="joey"
+ subject="comment 1"
+ date="2011-07-04T22:58:46Z"
+ content="""
+Well if it happens again why don't you use `ps` or `strace` to see what it's doing.
+"""]]
diff --git a/doc/bugs/problem_with_upgrade_v2_-__62___v3/comment_2_cd0123392b16d89db41b45464165c247._comment b/doc/bugs/problem_with_upgrade_v2_-__62___v3/comment_2_cd0123392b16d89db41b45464165c247._comment
new file mode 100644
index 0000000000..4bef5f6454
--- /dev/null
+++ b/doc/bugs/problem_with_upgrade_v2_-__62___v3/comment_2_cd0123392b16d89db41b45464165c247._comment
@@ -0,0 +1,23 @@
+[[!comment format=mdwn
+ username="https://lithitux.org/openidserver/users/pavel"
+ nickname="pavel"
+ subject=""Me too""
+ date="2011-07-05T15:54:19Z"
+ content="""
+I've also seen this apparent hang during upgrade to v3. A few more details:
+
+The annex in question has just under 18k files (and hence that many log files), which can slow down directory operations when they're all in the same place (like, for example, .git/annex/journal).
+
+git-annex uses virtually no CPU time and disk IO when it's hanging like this; the first time it happened, 'ps' showed three defunct git processes, with two \"git-annex\" processes and three \"git\" procs:
+
+ * git --git-dir=/mnt/annex/.git --work-tree=/mnt/annex cat-file --batch
+ * git --git-dir=/mnt/annex/.git --work-tree=/mnt/annex hash-object -w --stdin-paths
+ * git --git-dir=/mnt/annex/.git --work-tree=/mnt/annex update-index -z --index-info
+
+I Ctrl+C'd that and tried again, but it hung again -- this time without the defunct gits.
+
+An strace of the process and its children at the time of hang can be found at http://pastebin.com/4kNh4zEJ . It showed somewhat weird behaviour: When I attached with strace, it would scroll through a whole bunch of syscalls making up the open-fstat-read-close-write loop on .git/annex/journal files, but then would block on a write (sorry, don't have that in my scrollback any more so can't give more details) until I Ctrl+C'd strace; when attaching again, it would again scroll through the syscalls for a second or so and then hang with no output.
+
+Ultimately I detached/reattached with strace about two dozen times and that caused it (?) to finish the upgrade; not really sure how to explain it, but it seems like too much of a timing coincidence.
+
+"""]]
diff --git a/doc/bugs/problem_with_upgrade_v2_-__62___v3/comment_3_86d9e7244ae492bcbe62720b8c4fc4a9._comment b/doc/bugs/problem_with_upgrade_v2_-__62___v3/comment_3_86d9e7244ae492bcbe62720b8c4fc4a9._comment
new file mode 100644
index 0000000000..e314e73fa0
--- /dev/null
+++ b/doc/bugs/problem_with_upgrade_v2_-__62___v3/comment_3_86d9e7244ae492bcbe62720b8c4fc4a9._comment
@@ -0,0 +1,16 @@
+[[!comment format=mdwn
+ username="http://joey.kitenet.net/"
+ nickname="joey"
+ subject="comment 3"
+ date="2011-07-05T17:31:22Z"
+ content="""
+I've seen this kind of piping stall that is unblocked by strace before. It can vary with versions of GHC, so it would be good to know what version built git-annex (and on what OS version). I filed a bug report upstream before at .
+
+I really need a full strace -f from the top, or at least a complete `strace -o log` of git-annex from one hang through to another hang. The strace you pastebinned does not seem complete. If I can work out which specific git command is being written to when it hangs I can lift the writing out into a separate thread or process to fix it.
+
+@pavel, you mentioned three defunct git processes, and then showed ps output for 3 git processes. Were there 6 git processes in total? And then when you ran it again you said there were no defunct gits -- where the other 3 git processes running once again?
+
+As best I can make out from the (apparently) running git processes, it seems like the journal files for the upgrade had all been written, and the hang occurred when staging them all into the index in preparation for a commit. I have committed a change that lifts the code that does that write out into a new process, which, if I am guessing right on the limited info I have, will avoid the hang.
+
+However, since I can't reproduce it, even when I put 200 thousand files in the journal and have git-annex process them, I can't be sure.
+"""]]
diff --git a/doc/bugs/problem_with_upgrade_v2_-__62___v3/comment_4_91439d4dbbf1461e281b276eb0003691._comment b/doc/bugs/problem_with_upgrade_v2_-__62___v3/comment_4_91439d4dbbf1461e281b276eb0003691._comment
new file mode 100644
index 0000000000..7bc32c259f
--- /dev/null
+++ b/doc/bugs/problem_with_upgrade_v2_-__62___v3/comment_4_91439d4dbbf1461e281b276eb0003691._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="http://joey.kitenet.net/"
+ nickname="joey"
+ subject="comment 4"
+ date="2011-07-05T18:37:21Z"
+ content="""
+I've managed to reproduce this and confirmed my fix works.
+"""]]
diff --git a/doc/bugs/problem_with_upgrade_v2_-__62___v3/comment_5_ca33a9ca0df33f7c1b58353d7ffb943d._comment b/doc/bugs/problem_with_upgrade_v2_-__62___v3/comment_5_ca33a9ca0df33f7c1b58353d7ffb943d._comment
new file mode 100644
index 0000000000..8649dc77a8
--- /dev/null
+++ b/doc/bugs/problem_with_upgrade_v2_-__62___v3/comment_5_ca33a9ca0df33f7c1b58353d7ffb943d._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="http://joey.kitenet.net/"
+ nickname="joey"
+ subject="comment 5"
+ date="2011-07-05T19:06:48Z"
+ content="""
+By the way, the original bug reporter mentioned deleting .git/annex/journal. This is not recommended, and doing it during an upgrade can result in git-annex losing location tracking information. You should probably run `git annex fsck` or reset to the old git tree (and `git config annex.version 2`) and upgrade again.
+"""]]
diff --git a/doc/bugs/problem_with_upgrade_v2_-__62___v3/comment_6_f360f0006bc9115bc5a3e2eb9fe58abd._comment b/doc/bugs/problem_with_upgrade_v2_-__62___v3/comment_6_f360f0006bc9115bc5a3e2eb9fe58abd._comment
new file mode 100644
index 0000000000..0852db0795
--- /dev/null
+++ b/doc/bugs/problem_with_upgrade_v2_-__62___v3/comment_6_f360f0006bc9115bc5a3e2eb9fe58abd._comment
@@ -0,0 +1,10 @@
+[[!comment format=mdwn
+ username="https://lithitux.org/openidserver/users/pavel"
+ nickname="pavel"
+ subject="comment 6"
+ date="2011-07-06T08:14:26Z"
+ content="""
+Ah, great, thanks very much for the quick fix!
+
+Yes, when I mentioned three defunct git processes, there were three processes shown as \"git [defunct]\", plus the three git processes I listed, plus two \"git-annex\" processes. Upon cancel/resume, there were no defunct git processes when I checked, but by the time I found the bug report on the forum and commented I'd already successfully upgraded by annex (by repeatedly attaching strace) and couldn't really easily get at either additional 'ps' info or a fuller strace than what I posted (that was just the log from one of the attach/detach cycles), so it's a relief you managed to pinpoint the problem.
+"""]]
diff --git a/doc/bugs/problems_with_android_and_gpg.mdwn b/doc/bugs/problems_with_android_and_gpg.mdwn
new file mode 100644
index 0000000000..86eb783632
--- /dev/null
+++ b/doc/bugs/problems_with_android_and_gpg.mdwn
@@ -0,0 +1,75 @@
+### Please describe the problem.
+When my android phone tries to decode files downloaded from a ssh remote using shared encryption, gpg errors occur.
+
+### What steps will reproduce the problem?
+Setup is very similar to my other bug report in .
+Only difference is the location of the annex on the android side.
+I have put it on the /data mount which uses ext4 to avoid the /sdcard fuse mount, which does not handle symlinks.
+
+1) setup git annex via webapp on laptop:
+
+* local annex
+
+* remote annex via ssh with shared encryption (tried two different servers, one with and one without git-annex installed)
+
+* share with my devices using jabber.me account
+
+2) setup git annex via webapp on android:
+
+* local annex in /data/data/ga.androidterm/annex (ext filesystem)
+
+* share with my devices using jabber.me account
+
+* ssh remote is automatically added via XMPP
+
+3) add file to annex on linux, which gets uploaded to the ssh remote
+
+4) symlink for file is created on phone and data downloaded, but never decrypted (see logs below)
+
+### What version of git-annex are you using? On what operating system?
+Ubunut Linux 12.04 with git-annex version:
+
+* 5.20140127.1 (from PPA)
+
+Android 4.2 (rooted) with git-annex version:
+
+* 5.20140211-g556cfeb (from autobuilds)
+
+### Please provide any additional information below.
+full logs:
+
+* linux: 
+
+* android: 
+
+most interesting parts:
+[[!format sh """
+#
+# android:
+#
+gpg: can't open `/usr/local/share/gnupg/options.skel': No such file or directory
+gpg: DBG: locking for `/sdcard/git-annex.home/.gnupg/secring.gpg.lock' done via O_EXCL
+gpg: DBG: locking for `/sdcard/git-annex.home/.gnupg/pubring.gpg.lock' done via O_EXCL
+gpg: decryption failed: bad key
+
+# followed by just the last line for all further attemps
+gpg: decryption failed: bad key
+
+# gpg it self seems to be fine
+root@android:/data/data/ga.androidterm # ./bin/gpg --version -v                
+gpg (GnuPG) 1.4.15
+Copyright (C) 2013 Free Software Foundation, Inc.
+License GPLv3+: GNU GPL version 3 or later 
+This is free software: you are free to change and redistribute it.
+There is NO WARRANTY, to the extent permitted by law.
+
+Home: ~/.gnupg
+Supported algorithms:
+Pubkey: RSA, RSA-E, RSA-S, ELG-E, DSA
+Cipher: IDEA, 3DES, CAST5, BLOWFISH, AES, AES192, AES256, TWOFISH,
+        CAMELLIA128, CAMELLIA192, CAMELLIA256
+Hash: MD5, SHA1, RIPEMD160, SHA256, SHA384, SHA512, SHA224
+Compression: Uncompressed, ZIP, ZLIB
+"""]]
+
+> Closing as this was a bug in the deprecated Android app. [[done]] --[[Joey]]
diff --git a/doc/bugs/problems_with_android_and_gpg/comment_1_526d8805cb1ae896e8b1920ac2aecc17._comment b/doc/bugs/problems_with_android_and_gpg/comment_1_526d8805cb1ae896e8b1920ac2aecc17._comment
new file mode 100644
index 0000000000..308d7ed8d7
--- /dev/null
+++ b/doc/bugs/problems_with_android_and_gpg/comment_1_526d8805cb1ae896e8b1920ac2aecc17._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="https://openid.stackexchange.com/user/3f8a1927-744c-4409-8425-48fb3c86672f"
+ nickname="kim"
+ subject="comment 1"
+ date="2014-02-14T19:16:02Z"
+ content="""
+I am experiencing the exaxact same issue on android 4.3 with the current git-annex version (5.20140211)
+"""]]
diff --git a/doc/bugs/problems_with_android_and_gpg/comment_2_2e1ae66bac4f55b74074b09e22ab270d._comment b/doc/bugs/problems_with_android_and_gpg/comment_2_2e1ae66bac4f55b74074b09e22ab270d._comment
new file mode 100644
index 0000000000..c7709f0fc9
--- /dev/null
+++ b/doc/bugs/problems_with_android_and_gpg/comment_2_2e1ae66bac4f55b74074b09e22ab270d._comment
@@ -0,0 +1,16 @@
+[[!comment format=mdwn
+ username="https://www.google.com/accounts/o8/id?id=AItOawm78jq1Uo-ZbyOPG3diJUWVvEiM0kyAcvk"
+ nickname="Dorian"
+ subject="any ideas or questions?"
+ date="2014-02-25T13:41:16Z"
+ content="""
+Hey Joey,
+
+I was wondering if you had any idea how we could fix this problem or if you need further information on this.
+Any response would be appreciated.
+
+Thanks for your great work on git-annex!
+
+Cheers,
+Dorian
+"""]]
diff --git a/doc/bugs/problems_with_android_and_gpg/comment_3_47510400e8e45a71a1581aed99a157a1._comment b/doc/bugs/problems_with_android_and_gpg/comment_3_47510400e8e45a71a1581aed99a157a1._comment
new file mode 100644
index 0000000000..2f663d9439
--- /dev/null
+++ b/doc/bugs/problems_with_android_and_gpg/comment_3_47510400e8e45a71a1581aed99a157a1._comment
@@ -0,0 +1,12 @@
+[[!comment format=mdwn
+ username="http://joeyh.name/"
+ ip="209.250.56.146"
+ subject="comment 3"
+ date="2014-02-26T18:34:18Z"
+ content="""
+AFAICS, the messages about locking are a red herring, since shared encryption is being used, the public and secret key rings are not used.
+
+This problem seems to be explained here: 
+
+It seems there must be a problem with the symmetric key used for shared encryption. Either the repository somehow has the wrong key in it, or the key is not extracted from the repository correctly somehow, or it's not fed into gpg correctly somehow.
+"""]]
diff --git a/doc/bugs/problems_with_android_and_gpg/comment_4_d28d773450d09e03160548d99f12256d._comment b/doc/bugs/problems_with_android_and_gpg/comment_4_d28d773450d09e03160548d99f12256d._comment
new file mode 100644
index 0000000000..8815f03c96
--- /dev/null
+++ b/doc/bugs/problems_with_android_and_gpg/comment_4_d28d773450d09e03160548d99f12256d._comment
@@ -0,0 +1,12 @@
+[[!comment format=mdwn
+ username="https://www.google.com/accounts/o8/id?id=AItOawm78jq1Uo-ZbyOPG3diJUWVvEiM0kyAcvk"
+ nickname="Dorian"
+ subject="comment 4"
+ date="2014-03-03T14:05:35Z"
+ content="""
+Is it correct, that the shared key is transfered through the git repo, meaning in this case XMPP?
+
+Then I guess this might simply be the problem of unreliable XMPP.
+Same problem as here: 
+
+"""]]
diff --git a/doc/bugs/problems_with_android_and_gpg/comment_5_74f1177d6dd42bab5ddfc040cbfb035e._comment b/doc/bugs/problems_with_android_and_gpg/comment_5_74f1177d6dd42bab5ddfc040cbfb035e._comment
new file mode 100644
index 0000000000..a9d24b97d0
--- /dev/null
+++ b/doc/bugs/problems_with_android_and_gpg/comment_5_74f1177d6dd42bab5ddfc040cbfb035e._comment
@@ -0,0 +1,10 @@
+[[!comment format=mdwn
+ username="http://joeyh.name/"
+ ip="209.250.56.146"
+ subject="comment 5"
+ date="2014-03-05T20:49:20Z"
+ content="""
+Yes, the key is in the git repository.
+
+While XMPP is an unreliable transport, it either manages to git pull over it, or it fails, git does not allow partial or corrupt pulls.
+"""]]
diff --git a/doc/bugs/problems_with_android_and_gpg/comment_6_8d5549e3facc6245ad09cc591ba83a4c._comment b/doc/bugs/problems_with_android_and_gpg/comment_6_8d5549e3facc6245ad09cc591ba83a4c._comment
new file mode 100644
index 0000000000..94b5056613
--- /dev/null
+++ b/doc/bugs/problems_with_android_and_gpg/comment_6_8d5549e3facc6245ad09cc591ba83a4c._comment
@@ -0,0 +1,15 @@
+[[!comment format=mdwn
+ username="joey"
+ subject="""comment 6"""
+ date="2018-05-08T18:57:20Z"
+ content="""
+I'm closing this bug report because the git-annex Android app that it
+was reported on is now deprecated. Instead, we have
+a way to run the regular git-annex build for linux on Android in termux:
+
+
+There were a lot of problems with the way the git-annex Android app was
+put together, and I hope this new approach avoids them. If you try it and
+still have the bug you reported, please followup and I'll reopen it.
+
+"""]]
diff --git a/doc/bugs/proposal_for_timestamp_semantics.mdwn b/doc/bugs/proposal_for_timestamp_semantics.mdwn
new file mode 100644
index 0000000000..51121719c2
--- /dev/null
+++ b/doc/bugs/proposal_for_timestamp_semantics.mdwn
@@ -0,0 +1,11 @@
+# High level overview:
+* This only concerns mtime (and its equivalents on other systems, if applicable). atime, ctime and permissions are out of scope.
+* An object added to an annex and later retrieved from it via `mv`, `cp -L`, `git annex unannex` and such should always keep its mtime, even if retrieved on an entirely different machine and/or from a backend that doesn't support timestamps natively.
+* If an added/reinjected object is already known to the annex, use the older mtime by default, since that's probably the version that's had its metadata preserved better.
+  * If that's too much of an assumption, provide a switch to use the older/newer/known/unknown mtime, or add a git-annex-touch command.
+* symlink and object file mtimes should reflect the mtime tracked by the annex.
+* ideally, directory mtimes would also be preserved, or failing that, `git-annex-fix`, `git-annex-add` and `git-checkout` should leave them untouched.
+
+# open questions/ideas:
+* What if the user `touch`es a file/symlink, bypassing git-annex? Should the data be reconciled or ignored?
+* preserving directory mtimes looks tricky, but could it maybe be done from a hook or two?
diff --git a/doc/bugs/proposal_for_timestamp_semantics/comment_1_a54fc1bda24e04b15490d0eaa1838d28._comment b/doc/bugs/proposal_for_timestamp_semantics/comment_1_a54fc1bda24e04b15490d0eaa1838d28._comment
new file mode 100644
index 0000000000..8fe2f552b3
--- /dev/null
+++ b/doc/bugs/proposal_for_timestamp_semantics/comment_1_a54fc1bda24e04b15490d0eaa1838d28._comment
@@ -0,0 +1,19 @@
+[[!comment format=mdwn
+ username="joey"
+ subject="""comment 1"""
+ date="2018-04-04T18:01:01Z"
+ content="""
+This is lacking a lot of detail about how this would be accomplished. 
+
+Assuming it's to be accomplished using git-annex metadata, it seems likely
+to signficantly slow down some git-annex operations (which would need to do
+an expensive git-annex branch lookup).
+
+There's also the complication of merging a git-annex
+branch that contains changes to the timestamp metadata. Would git-annex
+need to look over all the merged changes and go off and frob timestamps?
+
+[[todo/does_not_preserve_timestamps]] already contains discussion of this
+topic. I'm not sure that it's productive to discuss it in two different
+places. (Nor does this really seem like a bug report.)
+"""]]
diff --git a/doc/bugs/proposal_for_timestamp_semantics/comment_2_8ba9e0c35e7761b99feb55bcdd6065cf._comment b/doc/bugs/proposal_for_timestamp_semantics/comment_2_8ba9e0c35e7761b99feb55bcdd6065cf._comment
new file mode 100644
index 0000000000..b7edd53e40
--- /dev/null
+++ b/doc/bugs/proposal_for_timestamp_semantics/comment_2_8ba9e0c35e7761b99feb55bcdd6065cf._comment
@@ -0,0 +1,15 @@
+[[!comment format=mdwn
+ username="vrs+annex@ea5fa24dbb279be61a8e50adb638bf8366300717"
+ nickname="vrs+annex"
+ avatar="http://cdn.libravatar.org/avatar/74219abcec6eece8e2c9d4351c2c912c"
+ subject="comment 2"
+ date="2018-04-05T21:01:51Z"
+ content="""
+I have no opinion about what backend to use. If doing it via the metadata system significantly slows down things though and is generally awkward, why not build a separate subsystem?
+
+I don't know what you mean by \"look over all the merged changes and go off and frob timestamps\", but as long as n is on the order of [number of files changed in the commit], updating n files' timestamps sounds reasonable? There's the question of which timestamp has preference in a merge, but that sounds solvable.
+
+I made this a separate bug because it's a specific design proposal; I consider [[todo/does_not_preserve_timestamps]] a tracking bug/user story.
+
+proposal re [[/bugs/file_modification_time_should_be_stored_in_exactly_one_metadata_field/#comment-2ea94161228f0653917b91d4f999153f]]: File and symlink timestamps, after `git-annex-get` or `git-checkout`, are set to whatever's in the repo and then considered immutable. The user can of course change them with `touch`, but if the file is locked while that happens, that's considered a corruption like editing an object file and will be caught by `git-annex-fsck`.
+"""]]
diff --git a/doc/bugs/ran_once_then_stopped_running_opensuse_13.1.mdwn b/doc/bugs/ran_once_then_stopped_running_opensuse_13.1.mdwn
new file mode 100644
index 0000000000..271b429bf7
--- /dev/null
+++ b/doc/bugs/ran_once_then_stopped_running_opensuse_13.1.mdwn
@@ -0,0 +1,13 @@
+Installed stand-alone tarball amd64.
+I was able to launch webapp. (on laptop)
+Created a repository to local home directory.
+Then ceated another repository to invite local desktop pc.(this one had all the files i wanted to sync) 
+Went to dektop and accepted invitation.  But both machines never stopped synching?  and nothing really happened.
+so I removed repository on laptop to start fresh.
+But now webapp does nothing.  I removed and re-installed a few times but still nothing.
+Only as superuser will the webapp attempt to load but fails because it is super user running.
+As far as version of git-annex...  it prompted to upgrade and i think i saw a 5 in there somewhere.
+
+and since it won't load anymore i guess there is no log.
+
+[[!tag moreinfo]]
diff --git a/doc/bugs/ran_once_then_stopped_running_opensuse_13.1/comment_1_a9daf9e8f968b32f25e236f53ea4b845._comment b/doc/bugs/ran_once_then_stopped_running_opensuse_13.1/comment_1_a9daf9e8f968b32f25e236f53ea4b845._comment
new file mode 100644
index 0000000000..604da40227
--- /dev/null
+++ b/doc/bugs/ran_once_then_stopped_running_opensuse_13.1/comment_1_a9daf9e8f968b32f25e236f53ea4b845._comment
@@ -0,0 +1,14 @@
+[[!comment format=mdwn
+ username="http://joeyh.name/"
+ ip="71.80.94.56"
+ subject="comment 1"
+ date="2014-02-07T19:14:47Z"
+ content="""
+There's a bit of a dearth of information here. I think I need to know:
+
+* What the version is (output of `git annex version`)
+
+* What happens when you run `git annex webapp` (\"does nothing\" is very vague. Does it sit there forever without returning to the prompt? Print an error message? Open a web browser that's not usable somehow?)
+
+You might take a look at the file `~/.config/git-annex/autostart` and remove any repository you deleted from there, although normally git-annex will ignore entries that no longer exist.
+"""]]
diff --git a/doc/bugs/random_files_vanishing_when_assistant_gets_restarted.mdwn b/doc/bugs/random_files_vanishing_when_assistant_gets_restarted.mdwn
new file mode 100644
index 0000000000..aa34a8056a
--- /dev/null
+++ b/doc/bugs/random_files_vanishing_when_assistant_gets_restarted.mdwn
@@ -0,0 +1,34 @@
+#What steps will reproduce the problem?
+
+Running assistant in foreground on one repository that is paired to another repository, than killing it using CTRL-C and restarting it
+
+
+#What is the expected output? What do you see instead?
+
+I get messages like:
+
+    (Recording state in git...)
+    # Auf Zweig master
+    # Änderungen, die nicht zum Eintragen bereitgestellt sind:
+    #   (benutze "git add/rm ..." zum Bereitstellen)
+    #   (benutze "git checkout -- ..." um die Änderungen im Arbeitsverzeichnis zu verwerfen)
+    #
+    #       gelöscht:    "path/to/file"
+    #
+    keine Änderungen zum Eintragen hinzugefügt (benutze "git add" und/oder "git commit -a")
+    Total 0 (delta 0), reused 0 (delta 0)
+    To ssh://stormking@git-annex-volyova-stormking/data/repository/
+       4e2c631..911b80c  git-annex -> synced/git-annex
+    
+    Already up-to-date.
+
+Sorry for the german language, I'll try to reproduce it in english, later.
+After that, the symlinks for the file in the repository are gone. I can get
+them back by reverting the commit but things like that make me very nervous.
+
+
+#What version of git-annex are you using? On what operating system?
+
+3.20130102 on Arch Linux x64
+
+[[!tag /design/assistant moreinfo]]
diff --git a/doc/bugs/random_files_vanishing_when_assistant_gets_restarted/comment_1_53b4f388c47c1b3f6ffa4fc2155b30fc._comment b/doc/bugs/random_files_vanishing_when_assistant_gets_restarted/comment_1_53b4f388c47c1b3f6ffa4fc2155b30fc._comment
new file mode 100644
index 0000000000..aafd426499
--- /dev/null
+++ b/doc/bugs/random_files_vanishing_when_assistant_gets_restarted/comment_1_53b4f388c47c1b3f6ffa4fc2155b30fc._comment
@@ -0,0 +1,21 @@
+[[!comment format=mdwn
+ username="http://joeyh.name/"
+ ip="4.154.7.238"
+ subject="comment 1"
+ date="2013-01-17T18:30:06Z"
+ content="""
+AFAICS, the German message is the equivilant of `git status` when a file has been deleted:
+
+
    +# On branch master
+# Changes not staged for commit:
+#   (use \"git add/rm ...\" to update what will be committed)
+#   (use \"git checkout -- ...\" to discard changes in working directory)
+#
+#	deleted:    bigfile
+#
+no changes added to commit (use \"git add\" and/or \"git commit -a\")
+
    
+
+There's no indication here of when the file was deleted, or what deleted it. I need a way to reproduce this to be able to help. At this point it's not clear if the file is deleted by the old assistant that you ctrl-c'd, or if it was deleted while the assistant was not running, or if it somehow gets deleted when the new assistant is started up.
+"""]]
diff --git a/doc/bugs/random_files_vanishing_when_assistant_gets_restarted/comment_2_e66532b23b089c9ea61122d6664cddb9._comment b/doc/bugs/random_files_vanishing_when_assistant_gets_restarted/comment_2_e66532b23b089c9ea61122d6664cddb9._comment
new file mode 100644
index 0000000000..66c038f617
--- /dev/null
+++ b/doc/bugs/random_files_vanishing_when_assistant_gets_restarted/comment_2_e66532b23b089c9ea61122d6664cddb9._comment
@@ -0,0 +1,10 @@
+[[!comment format=mdwn
+ username="https://www.google.com/accounts/o8/id?id=AItOawnjjCyhVEcTRM5m4iIBqL3ZCooPx7ZYB_E"
+ nickname="Marcus"
+ subject="comment 2"
+ date="2013-01-18T09:08:05Z"
+ content="""
+>  At this point it's not clear if the file is deleted by the old assistant that you ctrl-c'd, or if it was deleted while the assistant was not running, or if it somehow gets deleted when the new assistant is started up.
+
+So how can I help to track this down? So far I can only rule out the file being deleted by some other process.
+"""]]
diff --git a/doc/bugs/random_files_vanishing_when_assistant_gets_restarted/comment_3_c9d692c867acc076f64f1213ea03ca11._comment b/doc/bugs/random_files_vanishing_when_assistant_gets_restarted/comment_3_c9d692c867acc076f64f1213ea03ca11._comment
new file mode 100644
index 0000000000..65dcb6b006
--- /dev/null
+++ b/doc/bugs/random_files_vanishing_when_assistant_gets_restarted/comment_3_c9d692c867acc076f64f1213ea03ca11._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="http://joeyh.name/"
+ ip="4.154.3.194"
+ subject="comment 3"
+ date="2013-01-18T20:18:36Z"
+ content="""
+If you can reproduce the problem, you should be able to see if the file is there when you've ctrl-c the assistant.
+"""]]
diff --git a/doc/bugs/regression_-_fails_to_drop_._Exit_code_11_wo_--debug__44___and_1_with_--debug.mdwn b/doc/bugs/regression_-_fails_to_drop_._Exit_code_11_wo_--debug__44___and_1_with_--debug.mdwn
new file mode 100644
index 0000000000..88dee9f9a0
--- /dev/null
+++ b/doc/bugs/regression_-_fails_to_drop_._Exit_code_11_wo_--debug__44___and_1_with_--debug.mdwn
@@ -0,0 +1,105 @@
+### Please describe the problem.
+
+DataLad unittest (datalad/crawler/pipelines/tests/test_simple_s3.py:test_drop) checking communication of git-annex with our special remote started to hang (should have crashed I guess, yet to check) and the attempt in independent execution showed that git-annex simply crashes.  Downgrade from 6.20180506+gitg6316072f0-1~ndall+1 to 6.20180416+gitg86b18966f-1~ndall+1 resolved the issue. Seems to boil down to change on interactions with the remote with new version crashing with `git-annex: readDirStream: invalid argument (Bad file descriptor)`?
+
+May be manages to stop/kill the custom special remote before stopping communicating with it?
+
+### Please provide any additional information below.
+
+[[!format sh """
+$> git annex whereis --all --debug
+[2018-05-07 21:49:18.991105877] read: git ["--git-dir=.git","--work-tree=.","--literal-pathspecs","show-ref","git-annex"]
+[2018-05-07 21:49:18.997322576] process done ExitSuccess
+[2018-05-07 21:49:18.997426863] read: git ["--git-dir=.git","--work-tree=.","--literal-pathspecs","show-ref","--hash","refs/heads/git-annex"]
+[2018-05-07 21:49:19.003137341] process done ExitSuccess
+[2018-05-07 21:49:19.003344009] read: git ["--git-dir=.git","--work-tree=.","--literal-pathspecs","log","refs/heads/git-annex..f9909eda54827d61b254605b5631482b3dc65468","--pretty=%H","-n1"]
+[2018-05-07 21:49:19.007972053] process done ExitSuccess
+[2018-05-07 21:49:19.008717245] chat: git ["--git-dir=.git","--work-tree=.","--literal-pathspecs","cat-file","--batch"]
+[2018-05-07 21:49:19.009717171] chat: git ["--git-dir=.git","--work-tree=.","--literal-pathspecs","cat-file","--batch-check=%(objectname) %(objecttype) %(objectsize)"]
+whereis MD5E-s8--966634ebf2fc135707d6753692bf4b1e.txt [2018-05-07 21:49:19.016115806] chat: /home/yoh/proj/datalad/datalad-master/venvs/dev/bin/git-annex-remote-datalad []
+[2018-05-07 21:49:19.832279118] git-annex-remote-datalad[1] --> VERSION 1
+[2018-05-07 21:49:19.832466638] git-annex-remote-datalad[1] <-- EXTENSIONS INFO
+[2018-05-07 21:49:19.832654467] git-annex-remote-datalad[1] --> UNSUPPORTED-REQUEST
+[2018-05-07 21:49:19.832898699] git-annex-remote-datalad[1] <-- PREPARE
+[2018-05-07 21:49:19.83308154] git-annex-remote-datalad[1] --> PREPARE-SUCCESS
+[2018-05-07 21:49:19.833153568] git-annex-remote-datalad[1] <-- WHEREIS MD5E-s8--966634ebf2fc135707d6753692bf4b1e.txt
+[2018-05-07 21:49:19.833276243] git-annex-remote-datalad[1] --> WHEREIS-FAILURE
+[2018-05-07 21:49:19.833536457] git-annex-remote-datalad[1] <-- CLAIMURL s3://datalad-test0-nonversioned/1version-nonversioned1.txt?versionId=null
+[2018-05-07 21:49:19.833847975] git-annex-remote-datalad[1] --> DEBUG Claiming url 's3://datalad-test0-nonversioned/1version-nonversioned1.txt?versionId=null'
+[2018-05-07 21:49:19.833975294] Claiming url 's3://datalad-test0-nonversioned/1version-nonversioned1.txt?versionId=null'
+[2018-05-07 21:49:19.834045759] git-annex-remote-datalad[1] --> CLAIMURL-SUCCESS
+(1 copy) 
+  	cf13d535-b47c-5df6-8590-0793cb08a90a -- [datalad]
+
+  datalad: s3://datalad-test0-nonversioned/1version-nonversioned1.txt?versionId=null
+ok
+git-annex: readDirStream: invalid argument (Bad file descriptor)
+
+(dev) 1 19899 ->1.....................................:Mon 07 May 2018 09:49:19 PM EDT:.
+
+# so -- it exited with exit code 1.  Downgrading now!
+
+$> sudo apt-get install git-annex-standalone=6.20180416+gitg86b18966f-1~ndall+1        
+...
+
+$> git annex whereis --all --debug                                                     
+[2018-05-07 21:50:33.064546504] read: git ["--git-dir=.git","--work-tree=.","--literal-pathspecs","show-ref","git-annex"]
+[2018-05-07 21:50:33.075890599] process done ExitSuccess
+[2018-05-07 21:50:33.076053279] read: git ["--git-dir=.git","--work-tree=.","--literal-pathspecs","show-ref","--hash","refs/heads/git-annex"]
+[2018-05-07 21:50:33.082610193] process done ExitSuccess
+[2018-05-07 21:50:33.082774661] read: git ["--git-dir=.git","--work-tree=.","--literal-pathspecs","log","refs/heads/git-annex..f9909eda54827d61b254605b5631482b3dc65468","--pretty=%H","-n1"]
+[2018-05-07 21:50:33.088684568] process done ExitSuccess
+[2018-05-07 21:50:33.089343824] chat: git ["--git-dir=.git","--work-tree=.","--literal-pathspecs","cat-file","--batch"]
+[2018-05-07 21:50:33.090037985] chat: git ["--git-dir=.git","--work-tree=.","--literal-pathspecs","cat-file","--batch-check=%(objectname) %(objecttype) %(objectsize)"]
+[2018-05-07 21:50:33.09515616] read: git ["--git-dir=.git","--work-tree=.","--literal-pathspecs","ls-tree","--full-tree","-z","-r","--name-only","--","refs/heads/git-annex"]
+whereis MD5E-s8--966634ebf2fc135707d6753692bf4b1e.txt [2018-05-07 21:50:33.101385386] chat: /home/yoh/proj/datalad/datalad-master/venvs/dev/bin/git-annex-remote-datalad []
+[2018-05-07 21:50:33.935964594] git-annex-remote-datalad[1] --> VERSION 1
+[2018-05-07 21:50:33.936100411] git-annex-remote-datalad[1] <-- EXTENSIONS INFO
+[2018-05-07 21:50:33.936325748] git-annex-remote-datalad[1] --> UNSUPPORTED-REQUEST
+[2018-05-07 21:50:33.936398107] git-annex-remote-datalad[1] <-- PREPARE
+[2018-05-07 21:50:33.936521005] git-annex-remote-datalad[1] --> PREPARE-SUCCESS
+[2018-05-07 21:50:33.936577067] git-annex-remote-datalad[1] <-- WHEREIS MD5E-s8--966634ebf2fc135707d6753692bf4b1e.txt
+[2018-05-07 21:50:33.936676987] git-annex-remote-datalad[1] --> WHEREIS-FAILURE
+[2018-05-07 21:50:33.937304604] git-annex-remote-datalad[1] <-- CLAIMURL s3://datalad-test0-nonversioned/1version-nonversioned1.txt?versionId=null
+[2018-05-07 21:50:33.937515713] git-annex-remote-datalad[1] --> DEBUG Claiming url 's3://datalad-test0-nonversioned/1version-nonversioned1.txt?versionId=null'
+[2018-05-07 21:50:33.937580054] Claiming url 's3://datalad-test0-nonversioned/1version-nonversioned1.txt?versionId=null'
+[2018-05-07 21:50:33.937645676] git-annex-remote-datalad[1] --> CLAIMURL-SUCCESS
+(1 copy) 
+  	cf13d535-b47c-5df6-8590-0793cb08a90a -- [datalad]
+
+  datalad: s3://datalad-test0-nonversioned/1version-nonversioned1.txt?versionId=null
+ok
+[2018-05-07 21:50:33.973870482] process done ExitSuccess
+[2018-05-07 21:50:33.974304561] process done ExitSuccess
+[2018-05-07 21:50:33.97461093] process done ExitSuccess
+
+"""]]
+
+all kosher.
+
+Original manifestation was (note that exit code was 1 for with --debug and 139 (128+11) without --debug):
+
+
+[[!format sh """
+(git-annex)hopa:~/.tmp/datalad_temp_test_drop2bWvne[master]git
+$> git annex drop --debug --json --all --force --
+[2018-05-07 21:43:12.673253137] read: git ["--git-dir=.git","--work-tree=.","--literal-pathspecs","show-ref","git-annex"]
+[2018-05-07 21:43:12.677379888] process done ExitSuccess
+[2018-05-07 21:43:12.677452681] read: git ["--git-dir=.git","--work-tree=.","--literal-pathspecs","show-ref","--hash","refs/heads/git-annex"]
+[2018-05-07 21:43:12.681153081] process done ExitSuccess
+[2018-05-07 21:43:12.681279588] read: git ["--git-dir=.git","--work-tree=.","--literal-pathspecs","log","refs/heads/git-annex..f9909eda54827d61b254605b5631482b3dc65468","--pretty=%H","-n1"]
+[2018-05-07 21:43:12.686123605] process done ExitSuccess
+[2018-05-07 21:43:12.686850715] chat: git ["--git-dir=.git","--work-tree=.","--literal-pathspecs","cat-file","--batch"]
+[2018-05-07 21:43:12.687525693] chat: git ["--git-dir=.git","--work-tree=.","--literal-pathspecs","cat-file","--batch-check=%(objectname) %(objecttype) %(objectsize)"]
+git-annex: readDirStream: invalid argument (Bad file descriptor)
+(dev) 1 19888 ->1.....................................:Mon 07 May 2018 09:43:12 PM EDT:.
+(git-annex)hopa:~/.tmp/datalad_temp_test_drop2bWvne[master]git
+$> git annex drop  --json --all --force --     
+error: git-annex died of signal 11
+(dev) 1 19889 ->139.....................................:Mon 07 May 2018 09:43:16 PM EDT:.
+"""]]
+which also resolves with downgrade of git-annex.
+
+[[!meta author=yoh]]
+
+> [[fixed|done]] --[[Joey]]
diff --git a/doc/bugs/regression_-_fails_to_drop_._Exit_code_11_wo_--debug__44___and_1_with_--debug/comment_1_3c2578d17ce519f80aa278a546df29cf._comment b/doc/bugs/regression_-_fails_to_drop_._Exit_code_11_wo_--debug__44___and_1_with_--debug/comment_1_3c2578d17ce519f80aa278a546df29cf._comment
new file mode 100644
index 0000000000..305eee8b38
--- /dev/null
+++ b/doc/bugs/regression_-_fails_to_drop_._Exit_code_11_wo_--debug__44___and_1_with_--debug/comment_1_3c2578d17ce519f80aa278a546df29cf._comment
@@ -0,0 +1,24 @@
+[[!comment format=mdwn
+ username="joey"
+ subject="""comment 1"""
+ date="2018-05-08T17:09:35Z"
+ content="""
+Neither git rev 6316072f0 nor 86b18966f is present in my git-annex git repo,
+which makes it kind of hard to look at the diff between whatever versions
+of git-annex you're running. Are you building out of a patched git
+repository?
+
+My guess is [[!commit bea0ad220a68138dc0a43d0c86bb2353ecf92d2c]]
+since it involves both directory reading and unsafeInterleaveIO.
+
+Indeed, it looks like that could defer a readDirStream due to
+unsafeInterleaveIO to outside the open/close bracket. And I suppose reading
+from a closed file handle might conceivably segfault.
+
+Looks like that would affect bare repositories, or using --all.
+I don't think it would affect those commands in a non-bare repository.
+Ah, I see youre whereis is indeed in a bare repository, and your drop
+used --all, so my analysis seems right.
+
+Fixed by backing out the problematic part of the commit.
+"""]]
diff --git a/doc/bugs/regression_-_yt__58___prefix_for___34__regular__34___urls.mdwn b/doc/bugs/regression_-_yt__58___prefix_for___34__regular__34___urls.mdwn
new file mode 100644
index 0000000000..9c74c6aeb7
--- /dev/null
+++ b/doc/bugs/regression_-_yt__58___prefix_for___34__regular__34___urls.mdwn
@@ -0,0 +1,31 @@
+### Please describe the problem.
+
+some tests started to fail in datalad when tried current master version of annex
+
+### What version of git-annex are you using? On what operating system?
+
+6.20171205+gitgcf05ebbeb-1~ndall+1
+
+### Please provide any additional information below.
+
+[[!format sh """
+(git-annex)hopa:/tmp/datalad_temp_test_AnnexRepo_web_remoteRusdWA[master]git
+$> git annex whereis --json 127.0.0.1_about.txt
+{"command":"whereis","note":"web: http://example.com/someurl\nweb: yt:http://127.0.0.1:33587/about.txt\n","success":true,"untrusted":[],"key":"SHA256E-s14--198775960d0d9827f134da4e219e5793949d24b62bfcb92aa9e2dc16d411b776.txt","whereis":[{"here":false,"uuid":"00000000-0000-0000-0000-000000000001","urls":["http://example.com/someurl","yt:http://127.0.0.1:33587/about.txt"],"description":"web"}],"file":"127.0.0.1_about.txt"}
+
+$> git annex whereis 127.0.0.1_about.txt      
+whereis 127.0.0.1_about.txt (1 copy) 
+  	00000000-0000-0000-0000-000000000001 -- web
+
+  web: http://example.com/someurl
+  web: yt:http://127.0.0.1:33587/about.txt
+
+"""]]
+
+This is a new behavior, this test was passing before, and I do not think we suggested that those test files are anyhow related to youtube before.
+
+Some tests also failed related to our datalad archives git annex special remote but I didn't look into them yet -- I guess though that it is related ;-)
+
+[[!meta author=yoh]]
+
+> [[fixed|done]] --[[Joey]]
diff --git a/doc/bugs/regression_-_yt__58___prefix_for___34__regular__34___urls/comment_1_56cd6e167c86f4419ac0971922f2e5d1._comment b/doc/bugs/regression_-_yt__58___prefix_for___34__regular__34___urls/comment_1_56cd6e167c86f4419ac0971922f2e5d1._comment
new file mode 100644
index 0000000000..ad2e84dc3c
--- /dev/null
+++ b/doc/bugs/regression_-_yt__58___prefix_for___34__regular__34___urls/comment_1_56cd6e167c86f4419ac0971922f2e5d1._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="joey"
+ subject="""comment 1"""
+ date="2017-12-06T15:51:03Z"
+ content="""
+Reporoduced by putting "Lots of abouts" in a http://localhost/about.txt file,
+and running git annex addurl --file with an existing file and that url.
+"""]]
diff --git a/doc/bugs/regression_-_yt__58___prefix_for___34__regular__34___urls/comment_2_1cebe1385d2b8a244afc259a78f72ada._comment b/doc/bugs/regression_-_yt__58___prefix_for___34__regular__34___urls/comment_2_1cebe1385d2b8a244afc259a78f72ada._comment
new file mode 100644
index 0000000000..624d094fd9
--- /dev/null
+++ b/doc/bugs/regression_-_yt__58___prefix_for___34__regular__34___urls/comment_2_1cebe1385d2b8a244afc259a78f72ada._comment
@@ -0,0 +1,6 @@
+[[!comment format=mdwn
+ username="joey"
+ subject="""comment 2"""
+ date="2017-12-06T16:02:55Z"
+ content="""
+"""]]
diff --git a/doc/bugs/regression_-_yt__58___prefix_for___34__regular__34___urls/comment_2_ca0bf1079acc89a4a79e55f8f58403c0._comment b/doc/bugs/regression_-_yt__58___prefix_for___34__regular__34___urls/comment_2_ca0bf1079acc89a4a79e55f8f58403c0._comment
new file mode 100644
index 0000000000..60e3d3bd3e
--- /dev/null
+++ b/doc/bugs/regression_-_yt__58___prefix_for___34__regular__34___urls/comment_2_ca0bf1079acc89a4a79e55f8f58403c0._comment
@@ -0,0 +1,26 @@
+[[!comment format=mdwn
+ username="yarikoptic"
+ avatar="http://cdn.libravatar.org/avatar/f11e9c84cb18d26a1748c33b48c924b4"
+ subject="comment 2"
+ date="2017-12-08T02:32:15Z"
+ content="""
+Thanks for the fix... seems to be a bit incomplete though -- in the --fast mode, url still has yt: prefix, reflected in the key as well:
+
+[[!format sh \"\"\"
+
+$> ls -l
+total 4
+lrwxrwxrwx 1 yoh yoh 110 Dec  7 21:27 1-copy.dat -> .git/annex/objects/gw/pw/URL--yt&chttp&c%%127.0.0.1&c34337%d1%1.dat/URL--yt&chttp&c%%127.0.0.1&c34337%d1%1.dat
+
+$> git annex whereis 1-copy.dat
+whereis 1-copy.dat (1 copy) 
+  	00000000-0000-0000-0000-000000000001 -- web
+
+  web: yt:http://127.0.0.1:34337/d1/1.dat
+ok
+
+$> git annex version
+git-annex version: 6.20171206+gitgc6e4bc0a2-1~ndall+1
+
+\"\"\"]]
+"""]]
diff --git a/doc/bugs/regression_-_yt__58___prefix_for___34__regular__34___urls/comment_4_f05e1047c5d22dddba21a9faa3397f0d._comment b/doc/bugs/regression_-_yt__58___prefix_for___34__regular__34___urls/comment_4_f05e1047c5d22dddba21a9faa3397f0d._comment
new file mode 100644
index 0000000000..b2aec7da15
--- /dev/null
+++ b/doc/bugs/regression_-_yt__58___prefix_for___34__regular__34___urls/comment_4_f05e1047c5d22dddba21a9faa3397f0d._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="joey"
+ subject="""comment 4"""
+ date="2017-12-08T18:48:46Z"
+ content="""
+That one happened with `git annex addurl --fast $url` so a different code
+path. Had to add a html page check to youtubeDlFileName to fix it.
+"""]]
diff --git a/doc/bugs/regression_-_yt__58___prefix_for___34__regular__34___urls/comment_5_811ba6ec07f6f30c701201178cdb00ac._comment b/doc/bugs/regression_-_yt__58___prefix_for___34__regular__34___urls/comment_5_811ba6ec07f6f30c701201178cdb00ac._comment
new file mode 100644
index 0000000000..aaf8143bd0
--- /dev/null
+++ b/doc/bugs/regression_-_yt__58___prefix_for___34__regular__34___urls/comment_5_811ba6ec07f6f30c701201178cdb00ac._comment
@@ -0,0 +1,9 @@
+[[!comment format=mdwn
+ username="yarikoptic"
+ avatar="http://cdn.libravatar.org/avatar/f11e9c84cb18d26a1748c33b48c924b4"
+ subject="comment 5"
+ date="2017-12-11T05:27:15Z"
+ content="""
+ok, tested with 6.20171208+gitg01f78e877-1~ndall+1 -- --fast mode issue is indeed resolved.  Thanks!
+Failing tests now only relate to our special remote (datalad-archives) which interfaces urls, so I thought it might relate... I will try to distill more info tomorrow unless you beat me to it figuring out where that regression could be (upon quick look didn't spot any yt: anywhere, so probably another issue) 
+"""]]
diff --git a/doc/bugs/regression_-_yt__58___prefix_for___34__regular__34___urls/comment_6_2cd156baab77dc3b3c248a2368b59040._comment b/doc/bugs/regression_-_yt__58___prefix_for___34__regular__34___urls/comment_6_2cd156baab77dc3b3c248a2368b59040._comment
new file mode 100644
index 0000000000..4dddd9af6b
--- /dev/null
+++ b/doc/bugs/regression_-_yt__58___prefix_for___34__regular__34___urls/comment_6_2cd156baab77dc3b3c248a2368b59040._comment
@@ -0,0 +1,105 @@
+[[!comment format=mdwn
+ username="yarikoptic"
+ avatar="http://cdn.libravatar.org/avatar/f11e9c84cb18d26a1748c33b48c924b4"
+ subject="comment 6"
+ date="2017-12-11T14:40:26Z"
+ content="""
+ok, first initial bit of information:  there is a difference in interactions with the special remote now. Here is the diff between old and new runs (sorry - content was also changing so there is difference in keys as well):
+
+[[!format sh \"\"\"
+$> diff -Nar -u6 /tmp/datalad_temp_tree_check_basic_scenarioVEOIhb/.git/bin/git-annex-remote-datalad-archive /tmp/datalad_temp_tree_check_basic_scenarioIfsLo7/.git/bin/git-annex-remote-datalad-archive
+--- /tmp/datalad_temp_tree_check_basic_scenarioVEOIhb/.git/bin/git-annex-remote-datalad-archive	2017-12-11 08:56:38.381298365 -0500
++++ /tmp/datalad_temp_tree_check_basic_scenarioIfsLo7/.git/bin/git-annex-remote-datalad-archive	2017-12-11 08:56:14.885677071 -0500
+@@ -48,66 +48,72 @@
+ ### ['git', '-c', 'receive.autogc=0', '-c', 'gc.auto=0', 'annex', 'add', '--debug', '--json', 'simple.txt']
+ 
+ ### ['git', '-c', 'receive.autogc=0', '-c', 'gc.auto=0', 'commit', '-m', 'Added the load file']
+ 
+ ### ['git', '-c', 'receive.autogc=0', '-c', 'gc.auto=0', 'annex', 'lookupkey', '--debug', 'a.tar.gz']
+ 
+-### ['git', '-c', 'receive.autogc=0', '-c', 'gc.auto=0', 'annex', 'addurl', '--debug', '--relaxed', '--file=simple.txt', 'dl+archive:SHA256E-s172--70cf6dd95738e5d3672a7139a2785b0a979f0f7955d0f6da0d94cc03c84a63b7.tar.gz#path=a/d/+%22%27%3Ba%26b%26cd+%60%7C+']
++### ['git', '-c', 'receive.autogc=0', '-c', 'gc.auto=0', 'annex', 'addurl', '--debug', '--relaxed', '--file=simple.txt', 'dl+archive:SHA256E-s173--db0a9680f8d15578de8e4a5b5c1e87f36d9372d6118fb24c1c60f390e71ad3c1.tar.gz#path=a/d/+%22%27%3Ba%26b%26cd+%60%7C+']
+ #send VERSION 1
+ #recv PREPARE
+ #send PREPARE-SUCCESS
+ #recv GETCOST
+ #send COST 500
+ #recv GETAVAILABILITY
+ #send AVAILABILITY LOCAL
+-#recv CLAIMURL dl+archive:SHA256E-s172--70cf6dd95738e5d3672a7139a2785b0a979f0f7955d0f6da0d94cc03c84a63b7.tar.gz#path=a/d/+%22%27%3Ba%26b%26cd+%60%7C+
+-#send DEBUG Claiming url 'dl+archive:SHA256E-s172--70cf6dd95738e5d3672a7139a2785b0a979f0f7955d0f6da0d94cc03c84a63b7.tar.gz#path=a/d/+%22%27%3Ba%26b%26cd+%60%7C+'
++#recv CLAIMURL dl+archive:SHA256E-s173--db0a9680f8d15578de8e4a5b5c1e87f36d9372d6118fb24c1c60f390e71ad3c1.tar.gz#path=a/d/+%22%27%3Ba%26b%26cd+%60%7C+
++#send DEBUG Claiming url 'dl+archive:SHA256E-s173--db0a9680f8d15578de8e4a5b5c1e87f36d9372d6118fb24c1c60f390e71ad3c1.tar.gz#path=a/d/+%22%27%3Ba%26b%26cd+%60%7C+'
+ #send CLAIMURL-SUCCESS
+-#recv CHECKURL dl+archive:SHA256E-s172--70cf6dd95738e5d3672a7139a2785b0a979f0f7955d0f6da0d94cc03c84a63b7.tar.gz#path=a/d/+%22%27%3Ba%26b%26cd+%60%7C+
++#recv CHECKURL dl+archive:SHA256E-s173--db0a9680f8d15578de8e4a5b5c1e87f36d9372d6118fb24c1c60f390e71ad3c1.tar.gz#path=a/d/+%22%27%3Ba%26b%26cd+%60%7C+
+ #send CHECKURL-CONTENTS UNKNOWN
+ #recv 
+ 
+ ### ['git', '-c', 'receive.autogc=0', '-c', 'gc.auto=0', 'annex', 'drop', '--debug', '--json', 'simple.txt']
+ #send VERSION 1
+ #recv PREPARE
+ #send PREPARE-SUCCESS
+ #recv CHECKPRESENT SHA256E-s3--a665a45920422f9d417e4867efdc4fb8a04a1f3fff1fa07e998e86f7f7a27ae3.txt
+ #send GETURLS SHA256E-s3--a665a45920422f9d417e4867efdc4fb8a04a1f3fff1fa07e998e86f7f7a27ae3.txt dl+archive:
+-#recv VALUE dl+archive:SHA256E-s172--70cf6dd95738e5d3672a7139a2785b0a979f0f7955d0f6da0d94cc03c84a63b7.tar.gz#path=a/d/+%22%27%3Ba%26b%26cd+%60%7C+
++#recv VALUE dl+archive:SHA256E-s173--db0a9680f8d15578de8e4a5b5c1e87f36d9372d6118fb24c1c60f390e71ad3c1.tar.gz#path=a/d/+%22%27%3Ba%26b%26cd+%60%7C+
+ #recv VALUE 
+ #send CHECKPRESENT-SUCCESS SHA256E-s3--a665a45920422f9d417e4867efdc4fb8a04a1f3fff1fa07e998e86f7f7a27ae3.txt
+ #recv 
+ 
+ ### ['git', '-c', 'receive.autogc=0', '-c', 'gc.auto=0', 'annex', 'whereis', '--debug', '--json', 'simple.txt']
+ #send VERSION 1
+ #recv PREPARE
+ #send PREPARE-SUCCESS
+ #recv WHEREIS SHA256E-s3--a665a45920422f9d417e4867efdc4fb8a04a1f3fff1fa07e998e86f7f7a27ae3.txt
+ #send WHEREIS-FAILURE
+-#recv CLAIMURL dl+archive:SHA256E-s172--70cf6dd95738e5d3672a7139a2785b0a979f0f7955d0f6da0d94cc03c84a63b7.tar.gz#path=a/d/+%22%27%3Ba%26b%26cd+%60%7C+
+-#send DEBUG Claiming url 'dl+archive:SHA256E-s172--70cf6dd95738e5d3672a7139a2785b0a979f0f7955d0f6da0d94cc03c84a63b7.tar.gz#path=a/d/+%22%27%3Ba%26b%26cd+%60%7C+'
+-#send CLAIMURL-SUCCESS
+ #recv 
+ 
+ ### ['git', '-c', 'receive.autogc=0', '-c', 'gc.auto=0', 'annex', 'find', '--debug', '--json', '--not', '--in', 'here', 'simple.txt']
+ 
+ ### ['git', '-c', 'receive.autogc=0', '-c', 'gc.auto=0', 'annex', 'get', '--debug', '--json', '--json-progress', 'simple.txt']
+ #send VERSION 1
+ #recv PREPARE
+ #send PREPARE-SUCCESS
+ #recv TRANSFER RETRIEVE SHA256E-s3--a665a45920422f9d417e4867efdc4fb8a04a1f3fff1fa07e998e86f7f7a27ae3.txt .git/annex/tmp/SHA256E-s3--a665a45920422f9d417e4867efdc4fb8a04a1f3fff1fa07e998e86f7f7a27ae3.txt
+ #send GETURLS SHA256E-s3--a665a45920422f9d417e4867efdc4fb8a04a1f3fff1fa07e998e86f7f7a27ae3.txt dl+archive:
+-#recv VALUE dl+archive:SHA256E-s172--70cf6dd95738e5d3672a7139a2785b0a979f0f7955d0f6da0d94cc03c84a63b7.tar.gz#path=a/d/+%22%27%3Ba%26b%26cd+%60%7C+
++#recv VALUE dl+archive:SHA256E-s173--db0a9680f8d15578de8e4a5b5c1e87f36d9372d6118fb24c1c60f390e71ad3c1.tar.gz#path=a/d/+%22%27%3Ba%26b%26cd+%60%7C+
+ #recv VALUE 
+ #send TRANSFER-SUCCESS RETRIEVE SHA256E-s3--a665a45920422f9d417e4867efdc4fb8a04a1f3fff1fa07e998e86f7f7a27ae3.txt
+ #recv 
+ 
+-### ['git', '-c', 'receive.autogc=0', '-c', 'gc.auto=0', 'annex', 'rmurl', '--debug', 'simple.txt', 'dl+archive:SHA256E-s172--70cf6dd95738e5d3672a7139a2785b0a979f0f7955d0f6da0d94cc03c84a63b7.tar.gz#path=a/d/+%22%27%3Ba%26b%26cd+%60%7C+']
++### ['git', '-c', 'receive.autogc=0', '-c', 'gc.auto=0', 'annex', 'rmurl', '--debug', 'simple.txt', 'dl+archive:SHA256E-s173--db0a9680f8d15578de8e4a5b5c1e87f36d9372d6118fb24c1c60f390e71ad3c1.tar.gz#path=a/d/+%22%27%3Ba%26b%26cd+%60%7C+']
++#send VERSION 1
++#recv PREPARE
++#send PREPARE-SUCCESS
++#recv CLAIMURL dl+archive:SHA256E-s173--db0a9680f8d15578de8e4a5b5c1e87f36d9372d6118fb24c1c60f390e71ad3c1.tar.gz#path=a/d/+%22%27%3Ba%26b%26cd+%60%7C+
++#send DEBUG Claiming url 'dl+archive:SHA256E-s173--db0a9680f8d15578de8e4a5b5c1e87f36d9372d6118fb24c1c60f390e71ad3c1.tar.gz#path=a/d/+%22%27%3Ba%26b%26cd+%60%7C+'
++#send CLAIMURL-SUCCESS
++#recv 
++
++### ['git', '-c', 'receive.autogc=0', '-c', 'gc.auto=0', 'annex', 'drop', '--debug', '--json', 'simple.txt']
+ send VERSION 1
+ recv PREPARE
+ send PREPARE-SUCCESS
+-recv CLAIMURL dl+archive:SHA256E-s172--70cf6dd95738e5d3672a7139a2785b0a979f0f7955d0f6da0d94cc03c84a63b7.tar.gz#path=a/d/+%22%27%3Ba%26b%26cd+%60%7C+
+-send DEBUG Claiming url 'dl+archive:SHA256E-s172--70cf6dd95738e5d3672a7139a2785b0a979f0f7955d0f6da0d94cc03c84a63b7.tar.gz#path=a/d/+%22%27%3Ba%26b%26cd+%60%7C+'
+-send CLAIMURL-SUCCESS
++recv CHECKPRESENT SHA256E-s3--a665a45920422f9d417e4867efdc4fb8a04a1f3fff1fa07e998e86f7f7a27ae3.txt
++send GETURLS SHA256E-s3--a665a45920422f9d417e4867efdc4fb8a04a1f3fff1fa07e998e86f7f7a27ae3.txt dl+archive:
++recv VALUE dl+archive:SHA256E-s173--db0a9680f8d15578de8e4a5b5c1e87f36d9372d6118fb24c1c60f390e71ad3c1.tar.gz#path=a/d/+%22%27%3Ba%26b%26cd+%60%7C+
++recv VALUE 
++send CHECKPRESENT-SUCCESS SHA256E-s3--a665a45920422f9d417e4867efdc4fb8a04a1f3fff1fa07e998e86f7f7a27ae3.txt
+ recv 
+-
+-### ['git', '-c', 'receive.autogc=0', '-c', 'gc.auto=0', 'annex', 'drop', '--debug', '--json', 'simple.txt']
+
+
+\"\"\"]]
+[our test](https://github.com/datalad/datalad/blob/master/datalad/customremotes/tests/test_archives.py#L86) verifies that annex refuses  to drop the content if we remove the dl+archive link for  the key, and now it fails
+"""]]
diff --git a/doc/bugs/regression_-_yt__58___prefix_for___34__regular__34___urls/comment_7_9f3f7c04524063960f1451c90948a41f._comment b/doc/bugs/regression_-_yt__58___prefix_for___34__regular__34___urls/comment_7_9f3f7c04524063960f1451c90948a41f._comment
new file mode 100644
index 0000000000..f3cd71433b
--- /dev/null
+++ b/doc/bugs/regression_-_yt__58___prefix_for___34__regular__34___urls/comment_7_9f3f7c04524063960f1451c90948a41f._comment
@@ -0,0 +1,33 @@
+[[!comment format=mdwn
+ username="joey"
+ subject="""comment 7"""
+ date="2017-12-11T17:12:41Z"
+ content="""
+The diff shows `git annex whereis` used to send CLAIMURL to the external,
+and no longer does.
+
+Reproduction recipe using git-annex-remote-ipfs:
+
+	git annex initremote ipfs type=external externaltype=ipfs encryption=none
+	date > somefile
+	git annex add somefile
+	git annex addurl --debug --relaxed ipfs:dummy --file somefile
+
+Which results in `git annex whereis somefile` saying it's present in ipfs,
+but not listing the ipfs url for it. And again, whereis does not sent
+CLAIMURL.
+
+And, in log.web, I see why:
+
+	+1513013502.312530881s 1 ipfs:dummy
+
+That is not an OtherDownloader url, it's lacking the ":" prefix.
+
+This seems to be particular to the addurl --relaxed --file code path;
+letting addurl add a new file does result in an OtherDownloader url
+being recorded.
+
+I guess the reason the test suite then fails is, the url it removes is not
+the one git-annex recorded, and so git-annex still thinks it's at the wrongly
+recorded url, and so dropping succeeds.
+"""]]
diff --git a/doc/bugs/regression_-_yt__58___prefix_for___34__regular__34___urls/comment_8_55b111079ecb9732a42d3486160e77b5._comment b/doc/bugs/regression_-_yt__58___prefix_for___34__regular__34___urls/comment_8_55b111079ecb9732a42d3486160e77b5._comment
new file mode 100644
index 0000000000..7e91d6208f
--- /dev/null
+++ b/doc/bugs/regression_-_yt__58___prefix_for___34__regular__34___urls/comment_8_55b111079ecb9732a42d3486160e77b5._comment
@@ -0,0 +1,9 @@
+[[!comment format=mdwn
+ username="joey"
+ subject="""comment 8"""
+ date="2017-12-11T17:43:30Z"
+ content="""
+Above problem fixed in [[!commit bd7f8be121a5cd310ffbc32c6020326ef437a151]
+
+Good thing you have such a good test suite.
+"""]]
diff --git a/doc/bugs/regression_with_S3_creds_not_used.mdwn b/doc/bugs/regression_with_S3_creds_not_used.mdwn
new file mode 100644
index 0000000000..b5ca1d181a
--- /dev/null
+++ b/doc/bugs/regression_with_S3_creds_not_used.mdwn
@@ -0,0 +1,59 @@
+### Please describe the problem.
+
+I am having a problem on Windows in which the AWS access credentials are stored but not used. When I have the ```AWS_ACCESS_KEY_ID``` and ```AWS_SECRET_ACCESS_KEY``` variables are set, everything works correctly. However if I open a new window that doesn't have envs set, git annex sync --content fails with a Forbidden message. Looking in the creds directory, I see plaintext creds stored.
+
+### What steps will reproduce the problem?
+
+```
+# set AWS_ACCESS_KEY_ID=thing1
+# set AWS_SECRET_ACCESS_KEY=thing2
+# git init test
+# cd test
+# git annex init
+# git annex initremote s3cloud type=S3 encryption=pubkey keyid=key embedcreds=yes
+ ... everything works, annex-uuid is created, etc ...
+
+# now open a new window, navigate to proper repo
+
+# git annex sync --content
+
+... forbidden messages
+
+# Reset the env vars
+... sync works as normal ...
+```
+
+Note that this fails *silently* when using the webapp. The app shows percentage of upload, and the uploads "complete" but the log shows:
+
+```
+  S3Error {s3StatusCode = Status {statusCode = 403, statusMessage = "Forbidden"}, s3ErrorCode = "InvalidAccessKeyId", s3ErrorMessage = "The AWS Access Key Id you provided does not exist in our records.", s3ErrorResource = Nothing, s3ErrorHostId = Just "/MfOhrvmoi1uTsuJDaV2c+3jxqt+/cAC8KBvGkbE+os1BAPAw/bLeNHiUF8kbSJ+2cMuNO+0mzQ=", s3ErrorAccessKeyId = Just "redacted", s3ErrorStringToSign = Nothing, s3ErrorBucket = Nothing, s3ErrorEndpointRaw = Nothing, s3ErrorEndpoint = Nothing}
+```
+
+### What version of git-annex are you using? On what operating system?
+
+I'm on Windows:
+
+```
+git-annex version: 5.20151019-gcc50c00
+build flags: Assistant Webapp Webapp-secure Pairing Testsuite S3 WebDAV DNS Feed
+s Quvi TDFA TorrentParser Database
+key/value backends: SHA256E SHA256 SHA512E SHA512 SHA224E SHA224 SHA384E SHA384
+SHA3_256E SHA3_256 SHA3_512E SHA3_512 SHA3_224E SHA3_224 SHA3_384E SHA3_384 SKEI
+N256E SKEIN256 SKEIN512E SKEIN512 SHA1E SHA1 MD5E MD5 WORM URL
+remote types: git gcrypt S3 bup directory rsync web bittorrent webdav tahoe glac
+ier ddar hook external
+```
+
+### Please provide any additional information below.
+
+[[!format sh """
+# If you can, paste a complete transcript of the problem occurring here.
+# If the problem is with the git-annex assistant, paste in .git/annex/daemon.log
+
+
+# End of transcript or log.
+"""]]
+
+### Have you had any luck using git-annex before? (Sometimes we get tired of reading bug reports all day and a lil' positive end note does wonders)
+
+I have successfully used git annex on linux and mac, and it is great!
diff --git a/doc/bugs/regression_with_S3_creds_not_used/comment_1_df5416ee63c8f026f3b3c77c74b67ce4._comment b/doc/bugs/regression_with_S3_creds_not_used/comment_1_df5416ee63c8f026f3b3c77c74b67ce4._comment
new file mode 100644
index 0000000000..dbfb7c0db1
--- /dev/null
+++ b/doc/bugs/regression_with_S3_creds_not_used/comment_1_df5416ee63c8f026f3b3c77c74b67ce4._comment
@@ -0,0 +1,20 @@
+[[!comment format=mdwn
+ username="joey"
+ subject="""comment 1"""
+ date="2015-12-02T17:11:47Z"
+ content="""
+I tried reproducing this with the current version of git-annex on windows,
+and didn't see the problem. The creds file was used when the AWS_*
+environment variables were not set.
+
+What was the version of git-annex that was working before this reversion?
+
+Does the creds files corresponding to the uuid of your S3 remote exist?
+Does it contain 2 lines, for the access key and secret key, respectively?
+
+The webapp failure message seems to indicate that git-annex is has found
+creds, but they somehow have the wrong values. Does the s3ErrorAccessKeyId
+that you redacted match the access key that you told git-annex to use?
+(Note that it should be ok to show the s3 access key as long as you don't
+also show what the secret key is AFAIK.)
+"""]]
diff --git a/doc/bugs/remote_files_appear_but_are_unreadable_because_their_symlink_targets_don__39__t_exist.mdwn b/doc/bugs/remote_files_appear_but_are_unreadable_because_their_symlink_targets_don__39__t_exist.mdwn
new file mode 100644
index 0000000000..3bfdf9e1d9
--- /dev/null
+++ b/doc/bugs/remote_files_appear_but_are_unreadable_because_their_symlink_targets_don__39__t_exist.mdwn
@@ -0,0 +1,67 @@
+### Please describe the problem.
+
+I created somelargefile on host1. A file with that name appears in the right place on host2 but I can't use it:
+
+
+    host2:~/annex% cat somelargefile
+    cat: somelargefile: No such file or directory
+
+
+### What steps will reproduce the problem?
+
+I am running http://downloads.kitenet.net/git-annex/linux/current/git-annex-standalone-amd64.tar.gz and I connected two hosts with the 'Local computer' workflow. I run assistant on both sides.
+
+I make a file on either side, wait a minute for it to sync, and then observe this on the remote side:
+
+    lrwxrwxrwx 1 drewp drewp 178 Jul  9 23:58 somelargefile -> .git/annex/objects/Pw/vw/SHA256E-s8--bf87165d313027621936809a01da1994f9bd20ff9580c1380e7636e2443fe4ed/SHA256E-s8--bf87165d313027621936809a01da1994f9bd20ff9580c1380e7636e2443fe4ed
+
+    host2:~/annex% ls -l .git/annex/objects/Pw/vw/SHA256E-s8--bf87165d313027621936809a01da1994f9bd20ff9580c1380e7636e2443fe4ed/              
+    total 4                                                                                                                            
+    -rw------- 1 drewp drewp 14 Jul  9 23:58 SHA256E-s8--bf87165d313027621936809a01da1994f9bd20ff9580c1380e7636e2443fe4ed.map
+
+My "large file" is in fact 8 bytes long. The .map file contains the string "somelargefile\n"
+
+
+### What version of git-annex are you using? On what operating system?
+
+git-annex version: 4.20130709-g339d1e0
+build flags: Assistant Webapp Pairing Testsuite S3 WebDAV Inotify DBus XMPP
+local repository version: 3
+default repository version: 3
+supported repository versions: 3 4
+upgrade supported from repository versions: 0 1 2
+
+ubuntu 12.10 64-bit
+
+
+### Please provide any additional information below.
+
+[[!format sh """
+# If you can, paste a complete transcript of the problem occurring here.
+# If the problem is with the git-annex assistant, paste in .git/annex/daemon.log
+
+New lines in .git/annex/daemon.log when I add another new file:
+
+
+    (merging synced/git-annex into git-annex...)
+
+    Updating bac8b65..0422d27
+    Fast-forward
+     newfile2 |    1 +
+     1 file changed, 1 insertion(+)
+     create mode 120000 newfile2
+    [2013-07-10 00:08:38 PDT] Committer: Committing changes to git
+    (Recording state in git...)
+    [2013-07-10 00:08:38 PDT] Pusher: Syncing with host1.local_annex 
+    To ssh://drewp@git-annex-host1.local-drewp_annex/~/annex/
+       a80b6f5..72c0865  git-annex -> synced/git-annex
+
+    Already up-to-date.
+
+
+# End of transcript or log.
+"""]]
+
+[[!meta title="local pairing git-annex-shell issue when using standalone tarball"]]
+
+[[!tag moreinfo]]
diff --git a/doc/bugs/remote_files_appear_but_are_unreadable_because_their_symlink_targets_don__39__t_exist/comment_10_8a1d16b2aaba224e94be3d9dcc036d91._comment b/doc/bugs/remote_files_appear_but_are_unreadable_because_their_symlink_targets_don__39__t_exist/comment_10_8a1d16b2aaba224e94be3d9dcc036d91._comment
new file mode 100644
index 0000000000..9cfc0ebeec
--- /dev/null
+++ b/doc/bugs/remote_files_appear_but_are_unreadable_because_their_symlink_targets_don__39__t_exist/comment_10_8a1d16b2aaba224e94be3d9dcc036d91._comment
@@ -0,0 +1,12 @@
+[[!comment format=mdwn
+ username="http://bigasterisk.com/"
+ nickname="Drew Perttula"
+ subject="comment 10"
+ date="2013-07-27T07:42:26Z"
+ content="""
+Sorry, I meant to get those logs earlier but I got distracted.
+
+One confounding thing in the previous runs is that I may have had accidental instances of git-annex assistant running on at least one of the machines. I killed them all for this next attempt, and named the dir annex2 just in case.
+
+daemon.log files are now at http://bigasterisk.com/post/git-annex/logs1/
+"""]]
diff --git a/doc/bugs/remote_files_appear_but_are_unreadable_because_their_symlink_targets_don__39__t_exist/comment_11_434ed328a22a6657dba3b2929a56e499._comment b/doc/bugs/remote_files_appear_but_are_unreadable_because_their_symlink_targets_don__39__t_exist/comment_11_434ed328a22a6657dba3b2929a56e499._comment
new file mode 100644
index 0000000000..13bfc4468b
--- /dev/null
+++ b/doc/bugs/remote_files_appear_but_are_unreadable_because_their_symlink_targets_don__39__t_exist/comment_11_434ed328a22a6657dba3b2929a56e499._comment
@@ -0,0 +1,18 @@
+[[!comment format=mdwn
+ username="http://joeyh.name/"
+ ip="4.152.246.110"
+ subject="comment 11"
+ date="2013-07-28T00:06:53Z"
+ content="""
+From the log on bang:
+
+
    +[2013-07-27 00:33:15 PDT] read: ssh [\"-S\",\"/home/drewp/annex2/.git/annex/ssh/drewp@git-annex-dash.local-drewp_annex2\",\"-o\",\"ControlMaster=auto\",\"-o\",\"ControlPersist=yes\",\"-T\",\"drewp@git-annex-dash.local-drewp_annex2\",\"git-annex-shell 'configlist' '/~/annex2/'\"]
+zsh:1: command not found: git-annex-shell
+Remote dash.local_annex2 does not have git-annex installed; setting remote.dash.local_annex2.annex-ignore
+
    
+
+You said you're running from the standalone tarball. I think that's the problem, probably if you get git-annex-shell into your path it'll just work.
+
+Can you paste the line git-annex added to \"dash\"'s .ssh/authorized_keys?
+"""]]
diff --git a/doc/bugs/remote_files_appear_but_are_unreadable_because_their_symlink_targets_don__39__t_exist/comment_12_1837b70ace42882db3ab82e001680934._comment b/doc/bugs/remote_files_appear_but_are_unreadable_because_their_symlink_targets_don__39__t_exist/comment_12_1837b70ace42882db3ab82e001680934._comment
new file mode 100644
index 0000000000..54ba310b54
--- /dev/null
+++ b/doc/bugs/remote_files_appear_but_are_unreadable_because_their_symlink_targets_don__39__t_exist/comment_12_1837b70ace42882db3ab82e001680934._comment
@@ -0,0 +1,29 @@
+[[!comment format=mdwn
+ username="http://bigasterisk.com/"
+ nickname="Drew Perttula"
+ subject="comment 12"
+ date="2013-07-28T05:36:39Z"
+ content="""
+Thanks for finding that. I added git-annex-shell to my path and ran again. The repo lists look good now. 
+
+For the record, here's what I am getting in authorized_keys on dash:
+
+    command=\"GIT_ANNEX_SHELL_DIRECTORY='/home/drewp/annex2' ~/.ssh/git-annex-shell\",no-agent-forwarding,no-port-forwarding,no-X11-forwarding ssh-rsa AAAAB3NzaC1y.......0c2ilnh drewp@bang
+
+
+When I went to the settings on each box, all 4 repository group select boxes were blank. I set them all to 'client'.
+
+How do we make this long ticket worth your time by improving the code so people don't hit the same trap in the future? I think I would have looked in the right direction if, instead of saying nothing and not having a uuid to make the links work, the web ui said something like this:
+
+    'command not found: git-annex-shell' on remote host. Sync disabled with this host.
+
+
+--------
+
+Next, I ran \"date > annex2/file1\" on bang. The web consoles said they synced, but no file appeared on dash. There was a ~/annex2/.git/annex/objects/../../SHA256.../SHA256... file on dash with the right contents, but no symlink showed up in dash:~/annex2. 
+
+I ran \"date > annex2/file2\" on dash, and then I got both file1 and file2 on both boxes.
+
+In the new logs at http://bigasterisk.com/post/git-annex/logs2/, the surprise is at [2013-07-27 22:10:49 PDT] where I thought dash would make a symlink called 'file1' but instead it reports no attempt or error. Later at [2013-07-27 22:12:08 PDT], while dealing with the upload of file2, dash finally makes the symlink for file1. 
+
+"""]]
diff --git a/doc/bugs/remote_files_appear_but_are_unreadable_because_their_symlink_targets_don__39__t_exist/comment_13_ca9c87a10f29e41572540edeb99652f2._comment b/doc/bugs/remote_files_appear_but_are_unreadable_because_their_symlink_targets_don__39__t_exist/comment_13_ca9c87a10f29e41572540edeb99652f2._comment
new file mode 100644
index 0000000000..07cf648653
--- /dev/null
+++ b/doc/bugs/remote_files_appear_but_are_unreadable_because_their_symlink_targets_don__39__t_exist/comment_13_ca9c87a10f29e41572540edeb99652f2._comment
@@ -0,0 +1,11 @@
+[[!comment format=mdwn
+ username="http://joeyh.name/"
+ ip="4.154.0.21"
+ subject="comment 13"
+ date="2013-07-30T18:51:38Z"
+ content="""
+I'm more interested in fixing core problems, like why it was not able to find git-annex-shell, than in fixing bugs in the web UI that follow on from those problems. 
+
+When you're using the standalone tarball, it's supposed to set up `~/.ssh/git-annex-shell` to point to wherever you've unpacked the tarball. The assistant does this the first time you run it, and any time it sees the file is out of date.
+I've tested this, and it appears to be working ok. I don't understand what happened in your case to prevent this from happening.
+"""]]
diff --git a/doc/bugs/remote_files_appear_but_are_unreadable_because_their_symlink_targets_don__39__t_exist/comment_1_69eafc4201e3014ef1b5d74fe319e462._comment b/doc/bugs/remote_files_appear_but_are_unreadable_because_their_symlink_targets_don__39__t_exist/comment_1_69eafc4201e3014ef1b5d74fe319e462._comment
new file mode 100644
index 0000000000..03d14c5860
--- /dev/null
+++ b/doc/bugs/remote_files_appear_but_are_unreadable_because_their_symlink_targets_don__39__t_exist/comment_1_69eafc4201e3014ef1b5d74fe319e462._comment
@@ -0,0 +1,10 @@
+[[!comment format=mdwn
+ username="http://joeyh.name/"
+ ip="4.154.7.235"
+ subject="comment 1"
+ date="2013-07-10T16:31:40Z"
+ content="""
+Everything you've described is expected behavior, when the content of the file has not yet been transferred to the repository.
+
+There should be something in daemon.log about the transfer of the file. If there's an error message for the transfer, it will certainly show up there. You can enable debugging to get more information in daemon.log about file transfers.
+"""]]
diff --git a/doc/bugs/remote_files_appear_but_are_unreadable_because_their_symlink_targets_don__39__t_exist/comment_2_b7a64db9abe006af8c30169ad849efe9._comment b/doc/bugs/remote_files_appear_but_are_unreadable_because_their_symlink_targets_don__39__t_exist/comment_2_b7a64db9abe006af8c30169ad849efe9._comment
new file mode 100644
index 0000000000..954d6e375f
--- /dev/null
+++ b/doc/bugs/remote_files_appear_but_are_unreadable_because_their_symlink_targets_don__39__t_exist/comment_2_b7a64db9abe006af8c30169ad849efe9._comment
@@ -0,0 +1,76 @@
+[[!comment format=mdwn
+ username="http://bigasterisk.com/"
+ nickname="Drew Perttula"
+ subject="debug log"
+ date="2013-07-16T04:02:40Z"
+ content="""
+receiver side logs again, with --debug mode on. I don't see anything weird. 
+
+
+    [2013-07-15 20:53:02 PDT] read: git [\"--git-dir=/home/drewp/annex/.git\",\"--work-tree=/home/drewp/annex\",\"show-ref\",\"git-annex\"]
+    [2013-07-15 20:53:02 PDT] read: git [\"--git-dir=/home/drewp/annex/.git\",\"--work-tree=/home/drewp/annex\",\"show-ref\",\"--hash\",\"refs/heads/git-annex\"]
+    [2013-07-15 20:53:02 PDT] read: git [\"--git-dir=/home/drewp/annex/.git\",\"--work-tree=/home/drewp/annex\",\"log\",\"refs/heads/git-annex..a7cb1b3f2a3f2a7d24827e10f9f3ac5848fd11d9\",\"--oneline\",\"-n1\"]
+    [2013-07-15 20:53:02 PDT] read: git [\"--git-dir=/home/drewp/annex/.git\",\"--work-tree=/home/drewp/annex\",\"log\",\"refs/heads/git-annex..4a2771e15cb5c7a0a8d70443e76b65c12115a8cd\",\"--oneline\",\"-n1\"]
+    [2013-07-15 20:53:02 PDT] read: git [\"--git-dir=/home/drewp/annex/.git\",\"--work-tree=/home/drewp/annex\",\"log\",\"refs/heads/git-annex..ddf6aa74d771521b11657edfae558858b60e8368\",\"--oneline\",\"-n1\"]
+    (merging synced/git-annex into git-annex...)
+    [2013-07-15 20:53:02 PDT] feed: git [\"--git-dir=/home/drewp/annex/.git\",\"--work-tree=/home/drewp/annex\",\"update-index\",\"-z\",\"--index-info\"]
+    [2013-07-15 20:53:02 PDT] read: git [\"--git-dir=/home/drewp/annex/.git\",\"--work-tree=/home/drewp/annex\",\"diff-index\",\"--raw\",\"-z\",\"-r\",\"--no-renames\",\"-l0\",\"--cached\",\"4a2771e15cb5c7a0a8d70443e76b65c12115a8cd\"]
+    [2013-07-15 20:53:02 PDT] read: git [\"--git-dir=/home/drewp/annex/.git\",\"--work-tree=/home/drewp/annex\",\"log\",\"4a2771e15cb5c7a0a8d70443e76b65c12115a8cd..refs/heads/git-annex\",\"--oneline\",\"-n1\"]
+    [2013-07-15 20:53:02 PDT] call: git [\"--git-dir=/home/drewp/annex/.git\",\"--work-tree=/home/drewp/annex\",\"update-ref\",\"refs/heads/git-annex\",\"4a2771e15cb5c7a0a8d70443e76b65c12115a8cd\"]
+    [2013-07-15 20:53:02 PDT] read: git [\"--git-dir=/home/drewp/annex/.git\",\"--work-tree=/home/drewp/annex\",\"symbolic-ref\",\"HEAD\"]
+    [2013-07-15 20:53:02 PDT] read: git [\"--git-dir=/home/drewp/annex/.git\",\"--work-tree=/home/drewp/annex\",\"show-ref\",\"refs/heads/master\"]
+    [2013-07-15 20:53:02 PDT] Merger: merging refs/heads/synced/master into refs/heads/master
+
+    [2013-07-15 20:53:02 PDT] read: git [\"--git-dir=/home/drewp/annex/.git\",\"--work-tree=/home/drewp/annex\",\"symbolic-ref\",\"HEAD\"]
+    [2013-07-15 20:53:02 PDT] read: git [\"--git-dir=/home/drewp/annex/.git\",\"--work-tree=/home/drewp/annex\",\"show-ref\",\"refs/heads/master\"]
+    [2013-07-15 20:53:02 PDT] read: git [\"--git-dir=/home/drewp/annex/.git\",\"--work-tree=/home/drewp/annex\",\"show-ref\",\"--hash\",\"refs/heads/master\"]
+    [2013-07-15 20:53:02 PDT] call: git [\"--git-dir=/home/drewp/annex/.git\",\"--work-tree=/home/drewp/annex/.git/annex/merge/\",\"merge\",\"--no-edit\",\"refs/heads/synced/master\"]
+    Updating 6f8cbe0..18cea18
+    Fast-forward
+     test-2013-07-15e |    1 +
+     1 file changed, 1 insertion(+)
+     create mode 120000 test-2013-07-15e
+    [2013-07-15 20:53:02 PDT] read: git [\"--git-dir=/home/drewp/annex/.git\",\"--work-tree=/home/drewp/annex\",\"show-ref\",\"--hash\",\"refs/heads/master\"]
+    [2013-07-15 20:53:02 PDT] read: git [\"--git-dir=/home/drewp/annex/.git\",\"--work-tree=/home/drewp/annex\",\"diff-tree\",\"-z\",\"--raw\",\"--no-renames\",\"-l0\",\"-r\",\"6f8cbe0a34d70e70a6365385bbd4338d97047d4b\",\"18cea18855143095a872a463eb0a7cf5dd81de4c\"]
+    [2013-07-15 20:53:02 PDT] read: git [\"--git-dir=/home/drewp/annex/.git\",\"--work-tree=/home/drewp/annex\",\"show-ref\",\"git-annex\"]
+    [2013-07-15 20:53:02 PDT] read: git [\"--git-dir=/home/drewp/annex/.git\",\"--work-tree=/home/drewp/annex\",\"show-ref\",\"--hash\",\"refs/heads/git-annex\"]
+    [2013-07-15 20:53:02 PDT] read: git [\"--git-dir=/home/drewp/annex/.git\",\"--work-tree=/home/drewp/annex\",\"log\",\"refs/heads/git-annex..4a2771e15cb5c7a0a8d70443e76b65c12115a8cd\",\"--oneline\",\"-n1\"]
+    [2013-07-15 20:53:02 PDT] read: git [\"--git-dir=/home/drewp/annex/.git\",\"--work-tree=/home/drewp/annex\",\"log\",\"refs/heads/git-annex..ddf6aa74d771521b11657edfae558858b60e8368\",\"--oneline\",\"-n1\"]
+    [2013-07-15 20:53:02 PDT] read: git [\"--git-dir=/home/drewp/annex/.git\",\"--work-tree=/home/drewp/annex\",\"log\",\"refs/heads/git-annex..a7cb1b3f2a3f2a7d24827e10f9f3ac5848fd11d9\",\"--oneline\",\"-n1\"]
+    [2013-07-15 20:53:02 PDT] feed: git [\"--git-dir=/home/drewp/annex/.git\",\"--work-tree=/home/drewp/annex\",\"update-index\",\"-z\",\"--index-info\"]
+    [2013-07-15 20:53:02 PDT] read: git [\"--git-dir=/home/drewp/annex/.git\",\"--work-tree=/home/drewp/annex\",\"diff-index\",\"--raw\",\"-z\",\"-r\",\"--no-renames\",\"-l0\",\"--cached\",\"refs/heads/git-annex\"]
+    [2013-07-15 20:53:02 PDT] Watcher: add symlink test-2013-07-15e
+    [2013-07-15 20:53:02 PDT] chat: git [\"--git-dir=/home/drewp/annex/.git\",\"--work-tree=/home/drewp/annex\",\"hash-object\",\"-t\",\"blob\",\"-w\",\"--stdin\",\"--no-filters\"]
+    [2013-07-15 20:53:02 PDT] Committer: committing 1 changes
+    [2013-07-15 20:53:02 PDT] Committer: Committing changes to git
+    (Recording state in git...)
+    [2013-07-15 20:53:02 PDT] feed: git [\"--git-dir=/home/drewp/annex/.git\",\"--work-tree=/home/drewp/annex\",\"update-index\",\"-z\",\"--index-info\"]
+    [2013-07-15 20:53:02 PDT] read: git [\"--git-dir=/home/drewp/annex/.git\",\"--work-tree=/home/drewp/annex\",\"commit\",\"--allow-empty-message\",\"--no-edit\",\"-m\",\"\",\"--quiet\",\"--no-verify\"]
+    [2013-07-15 20:53:02 PDT] Pusher: Syncing with host1.local_annex 
+    [2013-07-15 20:53:02 PDT] read: git [\"--git-dir=/home/drewp/annex/.git\",\"--work-tree=/home/drewp/annex\",\"symbolic-ref\",\"HEAD\"]
+    [2013-07-15 20:53:02 PDT] read: git [\"--git-dir=/home/drewp/annex/.git\",\"--work-tree=/home/drewp/annex\",\"show-ref\",\"refs/heads/master\"]
+    [2013-07-15 20:53:02 PDT] Pusher: pushing to [Remote { name =\"host1.local_annex\" }]
+    [2013-07-15 20:53:02 PDT] call: git [\"--git-dir=/home/drewp/annex/.git\",\"--work-tree=/home/drewp/annex\",\"branch\",\"-f\",\"synced/master\"]
+    [2013-07-15 20:53:02 PDT] call: git [\"--git-dir=/home/drewp/annex/.git\",\"--work-tree=/home/drewp/annex\",\"push\",\"host1.local_annex\",\"git-annex:synced/git-annex\",\"master:synced/master\"]
+    To ssh://drewp@git-annex-host1.local-drewp_annex/~/annex/
+       a7cb1b3..4a2771e  git-annex -> synced/git-annex
+    [2013-07-15 20:53:02 PDT] read: git [\"--git-dir=/home/drewp/annex/.git\",\"--work-tree=/home/drewp/annex\",\"show-ref\",\"git-annex\"]
+    [2013-07-15 20:53:02 PDT] read: git [\"--git-dir=/home/drewp/annex/.git\",\"--work-tree=/home/drewp/annex\",\"push\",\"host1.local_annex\",\"git-annex:synced/git-annex\",\"master\"]
+    [2013-07-15 20:53:02 PDT] read: git [\"--git-dir=/home/drewp/annex/.git\",\"--work-tree=/home/drewp/annex\",\"show-ref\",\"--hash\",\"refs/heads/git-annex\"]
+    [2013-07-15 20:53:02 PDT] read: git [\"--git-dir=/home/drewp/annex/.git\",\"--work-tree=/home/drewp/annex\",\"log\",\"refs/heads/git-annex..4a2771e15cb5c7a0a8d70443e76b65c12115a8cd\",\"--oneline\",\"-n1\"]
+    [2013-07-15 20:53:02 PDT] read: git [\"--git-dir=/home/drewp/annex/.git\",\"--work-tree=/home/drewp/annex\",\"log\",\"refs/heads/git-annex..ddf6aa74d771521b11657edfae558858b60e8368\",\"--oneline\",\"-n1\"]
+    [2013-07-15 20:53:02 PDT] read: git [\"--git-dir=/home/drewp/annex/.git\",\"--work-tree=/home/drewp/annex\",\"symbolic-ref\",\"HEAD\"]
+    [2013-07-15 20:53:02 PDT] read: git [\"--git-dir=/home/drewp/annex/.git\",\"--work-tree=/home/drewp/annex\",\"show-ref\",\"refs/heads/master\"]
+    [2013-07-15 20:53:02 PDT] Merger: merging refs/remotes/host1.local_annex/synced/master into refs/heads/master
+
+    [2013-07-15 20:53:02 PDT] read: git [\"--git-dir=/home/drewp/annex/.git\",\"--work-tree=/home/drewp/annex\",\"symbolic-ref\",\"HEAD\"]
+    [2013-07-15 20:53:02 PDT] read: git [\"--git-dir=/home/drewp/annex/.git\",\"--work-tree=/home/drewp/annex\",\"show-ref\",\"refs/heads/master\"]
+    [2013-07-15 20:53:02 PDT] read: git [\"--git-dir=/home/drewp/annex/.git\",\"--work-tree=/home/drewp/annex\",\"show-ref\",\"--hash\",\"refs/heads/master\"]
+    [2013-07-15 20:53:02 PDT] call: git [\"--git-dir=/home/drewp/annex/.git\",\"--work-tree=/home/drewp/annex/.git/annex/merge/\",\"merge\",\"--no-edit\",\"refs/remotes/host1.local_annex/synced/master\"]
+    Already up-to-date.
+    [2013-07-15 20:53:02 PDT] read: git [\"--git-dir=/home/drewp/annex/.git\",\"--work-tree=/home/drewp/annex\",\"show-ref\",\"--hash\",\"refs/heads/master\"]
+    [2013-07-15 20:53:02 PDT] read: git [\"--git-dir=/home/drewp/annex/.git\",\"--work-tree=/home/drewp/annex\",\"diff-tree\",\"-z\",\"--raw\",\"--no-renames\",\"-l0\",\"-r\",\"18cea18855143095a872a463eb0a7cf5dd81de4c\",\"18cea18855143095a872a463eb0a7cf5dd81de4c\"]
+    [2013-07-15 20:53:28 PDT] read: git [\"--git-dir=/home/drewp/annex/.git\",\"--work-tree=/home/drewp/annex\",\"ls-tree\",\"-z\",\"--\",\"refs/heads/git-annex\",\"uuid.log\",\"remote.log\",\"trust.log\",\"group.log\",\"preferred-content.log\"]
+
+I also noticed on host1 where the file was made, there is a git node in gitk that says 'Local uncommitted changes, not checked in to index' and this node appears to have all my file contents as diffs. So the problem is maybe not the transfer, but the failure of the originating side to commit the file contents properly?
+"""]]
diff --git a/doc/bugs/remote_files_appear_but_are_unreadable_because_their_symlink_targets_don__39__t_exist/comment_3_197ac6070f256131c6e18a07aa3834fa._comment b/doc/bugs/remote_files_appear_but_are_unreadable_because_their_symlink_targets_don__39__t_exist/comment_3_197ac6070f256131c6e18a07aa3834fa._comment
new file mode 100644
index 0000000000..d772de5af1
--- /dev/null
+++ b/doc/bugs/remote_files_appear_but_are_unreadable_because_their_symlink_targets_don__39__t_exist/comment_3_197ac6070f256131c6e18a07aa3834fa._comment
@@ -0,0 +1,14 @@
+[[!comment format=mdwn
+ username="http://joeyh.name/"
+ ip="4.154.4.90"
+ subject="comment 3"
+ date="2013-07-16T18:03:40Z"
+ content="""
+It's normal for git diff and gitk to look like that in a repository that's using direct mode.
+
+In your log, I see you added a file \"test-2013-07-15e\". And I don't see any indication that it tried to transfer it.
+
+There are a few reasons that could happen. If you've \"paused\" syncing with a remote in the webapp, it could do that.
+
+I think more likely would be if you've put the remote repository in a repository group that doesn't want the file. If it's configured to be in the Manual group or the Source group (or the Transfer, Public, or Small Archive group for that matter), the assistant won't try to send files to it. You can check this by editing the repository in the webapp. It should probably be set to be in the Client group.
+"""]]
diff --git a/doc/bugs/remote_files_appear_but_are_unreadable_because_their_symlink_targets_don__39__t_exist/comment_4_fe07832333b536c71b7dcb46a4a44bd0._comment b/doc/bugs/remote_files_appear_but_are_unreadable_because_their_symlink_targets_don__39__t_exist/comment_4_fe07832333b536c71b7dcb46a4a44bd0._comment
new file mode 100644
index 0000000000..2201db88e6
--- /dev/null
+++ b/doc/bugs/remote_files_appear_but_are_unreadable_because_their_symlink_targets_don__39__t_exist/comment_4_fe07832333b536c71b7dcb46a4a44bd0._comment
@@ -0,0 +1,19 @@
+[[!comment format=mdwn
+ username="http://bigasterisk.com/"
+ nickname="Drew Perttula"
+ subject="comment 4"
+ date="2013-07-25T05:02:16Z"
+ content="""
+I saw the 'syncing enabled'/'syncing disabled' switch in the webapp, which I think is what you're calling pause. That was always enabled.
+
+I never adjusted the groups, but I see in the webapp that my remote is busted in other ways. It has no title, and settings->edit makes an error page. The edit link is like this:
+
+http://127.0.0.1:38187/config/repository/edit/NoUUID?auth=4294a...333c
+
+and the error is 
+
+    Internal Server Error
+    Unknown UUID
+
+I'll try re-configuring that remote. I wish the webapp could tell me more about what is broken with the remote repo and where I might go to repair it. 
+"""]]
diff --git a/doc/bugs/remote_files_appear_but_are_unreadable_because_their_symlink_targets_don__39__t_exist/comment_5_540bca4e6fdfc10eeab875ecc0f2b3f3._comment b/doc/bugs/remote_files_appear_but_are_unreadable_because_their_symlink_targets_don__39__t_exist/comment_5_540bca4e6fdfc10eeab875ecc0f2b3f3._comment
new file mode 100644
index 0000000000..8e00198efa
--- /dev/null
+++ b/doc/bugs/remote_files_appear_but_are_unreadable_because_their_symlink_targets_don__39__t_exist/comment_5_540bca4e6fdfc10eeab875ecc0f2b3f3._comment
@@ -0,0 +1,10 @@
+[[!comment format=mdwn
+ username="http://bigasterisk.com/"
+ nickname="Drew Perttula"
+ subject="comment 5"
+ date="2013-07-25T05:14:12Z"
+ content="""
+BTW \"go enter it into the computer you want to pair with\" is a very frustrating instruction to read. Every time I try to do a pairing and I encounter that, it's telling me \"now it's time for you to guess what to do, so this is probably where you'll screw things up.\" I have no idea where to enter that secret phrase. I would also like to know if 'asdfasdf' is a good secret phrase or not, presuming the two computers are both on my internal network. Does the secret matter later or is it just used for a minute?
+
+So far, I keep getting blank repo names with the NoUUID edit url, or I fail to accomplish a pairing at all.
+"""]]
diff --git a/doc/bugs/remote_files_appear_but_are_unreadable_because_their_symlink_targets_don__39__t_exist/comment_6_3f236b35e9820cd88bb77fcd57d6975e._comment b/doc/bugs/remote_files_appear_but_are_unreadable_because_their_symlink_targets_don__39__t_exist/comment_6_3f236b35e9820cd88bb77fcd57d6975e._comment
new file mode 100644
index 0000000000..63261fb9dc
--- /dev/null
+++ b/doc/bugs/remote_files_appear_but_are_unreadable_because_their_symlink_targets_don__39__t_exist/comment_6_3f236b35e9820cd88bb77fcd57d6975e._comment
@@ -0,0 +1,43 @@
+[[!comment format=mdwn
+ username="http://bigasterisk.com/"
+ nickname="Drew Perttula"
+ subject="comment 6"
+ date="2013-07-25T05:36:07Z"
+ content="""
+Here's how little it takes for things to go wrong:  
+  
+----
+First host:  
+
+bang(pts/17):~/annex% git init  
+Initialized empty Git repository in /home/drewp/annex/.git/  
+
+bang(pts/17):~/annex% git-annex init  
+init  ok  
+(Recording state in git...)
+
+bang(pts/17):~/annex% git-annex webapp --listen 10.1.0.1:9999    
+http://10.1.0.1:9999/?auth=6f9a8b...
+
+
+----
+Second host:
+
+dash(pts/34):~/annex% git init  
+Initialized empty Git repository in /home/drewp/annex/.git/
+
+dash(pts/34):~/annex% git-annex init  
+init  ok  
+(Recording state in git...)
+
+dash(pts/34):~/annex% git-annex webapp --listen 10.1.0.229:9999  
+http://10.1.0.229:9999/?auth=f28bd56b456....
+
+
+Browse to first URI. Add another repo. Local computer. Enter 'qwerty'.   
+Browse to second URI. Respond to pairing request. Enter 'qwerty' again.  
+View both dashboards: http://bigasterisk.com/post/git-annex-bad-pair.png  
+
+
+
+"""]]
diff --git a/doc/bugs/remote_files_appear_but_are_unreadable_because_their_symlink_targets_don__39__t_exist/comment_7_3cc5dae0351201522711a7caeecd60d5._comment b/doc/bugs/remote_files_appear_but_are_unreadable_because_their_symlink_targets_don__39__t_exist/comment_7_3cc5dae0351201522711a7caeecd60d5._comment
new file mode 100644
index 0000000000..832f38fa5d
--- /dev/null
+++ b/doc/bugs/remote_files_appear_but_are_unreadable_because_their_symlink_targets_don__39__t_exist/comment_7_3cc5dae0351201522711a7caeecd60d5._comment
@@ -0,0 +1,10 @@
+[[!comment format=mdwn
+ username="http://joeyh.name/"
+ ip="2001:4978:f:21a::2"
+ subject="comment 7"
+ date="2013-07-25T20:02:18Z"
+ content="""
+What settings does `.git/config` have for the git remote created by the local pairing process?
+
+Since I can't reproduce any problems with local pairing -- it works great on this network -- you will need to enable debugging, reproduce the bug, and send a debug log for me to debug this any further.
+"""]]
diff --git a/doc/bugs/remote_files_appear_but_are_unreadable_because_their_symlink_targets_don__39__t_exist/comment_8_3c3883cb66d02a15d5de84d22aa113da._comment b/doc/bugs/remote_files_appear_but_are_unreadable_because_their_symlink_targets_don__39__t_exist/comment_8_3c3883cb66d02a15d5de84d22aa113da._comment
new file mode 100644
index 0000000000..bf14ad573e
--- /dev/null
+++ b/doc/bugs/remote_files_appear_but_are_unreadable_because_their_symlink_targets_don__39__t_exist/comment_8_3c3883cb66d02a15d5de84d22aa113da._comment
@@ -0,0 +1,38 @@
+[[!comment format=mdwn
+ username="http://bigasterisk.com/"
+ nickname="Drew Perttula"
+ subject="~/annex/.git/config files (after running the steps above)"
+ date="2013-07-26T13:20:29Z"
+ content="""
+    dash(pts/32):~/annex% cat .git/config 
+    [core]
+        repositoryformatversion = 0
+        filemode = true
+        bare = false
+        logallrefupdates = true
+    [annex]
+        uuid = ddbb5112-3cff-46d3-b42a-f3a6826827a5
+        version = 3
+    [remote \"bang.local_annex\"]
+        url = ssh://drewp@git-annex-bang.local-drewp_annex/~/annex/
+        fetch = +refs/heads/*:refs/remotes/bang.local_annex/*
+        annex-ignore = true
+        annex-cost = 175.0
+
+
+    bang(pts/16):~/annex% cat .git/config
+    [core]
+        repositoryformatversion = 0
+        filemode = true
+        bare = false
+        logallrefupdates = true
+    [annex]
+        uuid = c8a6c420-0567-4f33-8abb-e44b2012ad55
+        version = 3
+    [remote \"dash.local_annex\"]
+        url = ssh://drewp@git-annex-dash.local-drewp_annex/~/annex/
+        fetch = +refs/heads/*:refs/remotes/dash.local_annex/*
+        annex-ignore = true
+        annex-cost = 175.0
+        
+"""]]
diff --git a/doc/bugs/remote_files_appear_but_are_unreadable_because_their_symlink_targets_don__39__t_exist/comment_9_c8cece9559bd2dc6154cd28772369e48._comment b/doc/bugs/remote_files_appear_but_are_unreadable_because_their_symlink_targets_don__39__t_exist/comment_9_c8cece9559bd2dc6154cd28772369e48._comment
new file mode 100644
index 0000000000..0fd052a4cb
--- /dev/null
+++ b/doc/bugs/remote_files_appear_but_are_unreadable_because_their_symlink_targets_don__39__t_exist/comment_9_c8cece9559bd2dc6154cd28772369e48._comment
@@ -0,0 +1,10 @@
+[[!comment format=mdwn
+ username="http://joeyh.name/"
+ ip="2001:4978:f:21a::2"
+ subject="comment 9"
+ date="2013-07-26T16:55:08Z"
+ content="""
+Like I sort of expected, this is missing a remote.$foo.annex-uuid setting. But I don't know why yet. I need a debug log to figure this out.
+
+Can you enable debugging (run git-annex webapp --debug), reproduce the problem again, and send me the log?
+"""]]
diff --git a/doc/bugs/remote_not_showing_up_in_webapp.mdwn b/doc/bugs/remote_not_showing_up_in_webapp.mdwn
new file mode 100644
index 0000000000..f2f83f064f
--- /dev/null
+++ b/doc/bugs/remote_not_showing_up_in_webapp.mdwn
@@ -0,0 +1,100 @@
+### Please describe the problem.
+
+This is a followup on [[bugs/internal_server_error:_unknown_UUID_on_webapp]]. In that issue, webapps previous to 20130929 would crash with `internal server error: unknown UUID`. This was fixed at that date, but some problems remain, namely that the remote that is recognized on the commandline doesn't show up in the webapp.
+
+`markov` is able to push to `marcos`, but not the reverse because `markov` is hidden behind a NAT. `git annex sync` seems to do the right thing accordingly on both ends (which is: `marcos` doesn't try to push to `markov` but `markov` pushes to `marcos`).
+
+### What steps will reproduce the problem?
+
+See [[bugs/internal_server_error:_unknown_UUID_on_webapp]]. I didn't do any further changes other than upgrade `git-annex` on both ends.
+
+### What version of git-annex are you using? On what operating system?
+
+`marcos` is now running `Version: 4.20131105-g8efdc1a Build flags: Assistant Webapp Pairing Testsuite S3 WebDAV Inotify DBus XMPP DNS Feeds Quvi TDFA CryptoHash`
+
+`markov` is now running the wheezy backport, `4.20131002~bpo70+1`.
+
+### Please provide any additional information below.
+
+#### On `marcos`
+
+Here's the output of `git annex status` on `marcos`:
+
+[[!format sh """
+anarcat@marcos:books$ git annex status
+repository mode: direct
+trusted repositories: 0
+semitrusted repositories: 3
+        00000000-0000-0000-0000-000000000001 -- web
+        a75cbbf7-e055-423e-b375-443e0552c9e2 -- here (anarcat@marcos:/srv/books)
+        aa500f29-42d9-4777-ae02-4a2c3d47db44 -- anarcat@markov:~/books
+untrusted repositories: 0
+transfers in progress: none
+available local disk space: 7.04 gigabytes (+1 megabyte reserved)
+local annex keys: 736
+local annex size: 3.92 gigabytes
+annexed files in working tree: 721
+size of annexed files in working tree: 3.92 gigabytes
+bloom filter size: 16 mebibytes (0.1% full)
+backend usage:
+        SHA256E: 1457
+# End of transcript or log.
+"""]]
+
+Here's a screenshot of the idle webapp on marcos:
+
+
+
+You can clearly see that the webapp doesn't see the `markov` remote.
+
+When `markov` transfers stuff, `marcos` sees the transfers happening, but marks it as going to the `unknown` remote:
+
+
+
+Clicking on that link is what was previously triggering [[bugs/internal_server_error:_unknown_UUID_on_webapp]] but now yields a "Unknown remote" error.
+
+
+
+#### On `markov`:
+
+Here is a screenshot from `markov` that shows *it* knows about both repositories and seem to behave properly:
+
+
+
+And here's the output of `git annex status` on markov:
+
+[[!format sh """
+anarcat@desktop008:books$ git annex status
+repository mode: indirect
+trusted repositories: 0
+semitrusted repositories: 3
+        00000000-0000-0000-0000-000000000001 -- web
+        a75cbbf7-e055-423e-b375-443e0552c9e2 -- origin (anarcat@marcos:/srv/books)
+        aa500f29-42d9-4777-ae02-4a2c3d47db44 -- here (anarcat@markov:~/books)
+untrusted repositories: 0
+transfers in progress:
+        downloading Patrick K. O'Brien/Philip's Atlas of World History, Concise Edition (115)/Philip's Atlas of World History, Concise Edition - Patrick K. O'Brien.pdf from origin
+available local disk space: 93.25 gigabytes (+1 megabyte reserved)
+temporary directory size: 50.07 megabytes (clean up with git-annex unused)
+local annex keys: 708
+local annex size: 3.81 gigabytes
+known annex keys: 721
+known annex size: 3.92 gigabytes
+bloom filter size: 16 mebibytes (0.1% full)
+backend usage:
+        SHA256E: 1429
+"""]]
+
+Finally, note that you sometimes see `desktop008` above: it turns out I am running `git annex` from my workstation, which NFS-mounts the `/home` directory of `markov` into `/srv/musique`.  --[[anarcat]]
+
+Yesterday, I made the webapp register ssh remotes it creates. 
+They then show up in the webapp when run in a repository where that ssh
+remote is not enabled.
+
+You can also manually register a ssh remote. First set up the git remote
+as usual. Then run `git annex initremote type=git name=foo
+location=$url`.
+
+[[!meta title="webapp shows transfer from 'unknown' when no remote is configured for a system that is downloading files"]]
+
+[[!tag confirmed]]
diff --git a/doc/bugs/remote_not_showing_up_in_webapp/comment_1_2a269732fd528f505777542d3556437a._comment b/doc/bugs/remote_not_showing_up_in_webapp/comment_1_2a269732fd528f505777542d3556437a._comment
new file mode 100644
index 0000000000..4582a30715
--- /dev/null
+++ b/doc/bugs/remote_not_showing_up_in_webapp/comment_1_2a269732fd528f505777542d3556437a._comment
@@ -0,0 +1,17 @@
+[[!comment format=mdwn
+ username="http://joeyh.name/"
+ ip="209.250.56.47"
+ subject="comment 1"
+ date="2013-11-06T16:38:47Z"
+ content="""
+marcov does not show up in the webapp because there is no configured git remote for it.
+
+This is a slightly confusing corner of the webapp. The webapp will show repositories that do not have a configured remote, but it only does it for special remotes. ssh repos, being regular git remotes, don't currently show up in the webapp unless that repository is actually set up as a remote.
+
+It should certainly not show it as \"unknown\"; it would be much better to use the full repo description here, since it does not have a remote name.
+(Unless the description is really long!)
+
+I think you'll also get the \"unknown uuid\" screen even for a special remote that is not configured in the local repository. So that needs to be fixed.
+
+Finally, it would probably be good for the webapp to show ssh repos that don't have remotes as existing, and let the user enter a ssh address to configure them. The problem with trying to do this is it actually has no idea that this is a ssh repo. It could just as easily be a local directory. The UI to configure  it would be pretty elaborate.
+"""]]
diff --git a/doc/bugs/remote_not_showing_up_in_webapp/comment_2_10638e99e2e11460f99266f56adbc1db._comment b/doc/bugs/remote_not_showing_up_in_webapp/comment_2_10638e99e2e11460f99266f56adbc1db._comment
new file mode 100644
index 0000000000..fb2976657e
--- /dev/null
+++ b/doc/bugs/remote_not_showing_up_in_webapp/comment_2_10638e99e2e11460f99266f56adbc1db._comment
@@ -0,0 +1,10 @@
+[[!comment format=mdwn
+ username="https://id.koumbit.net/anarcat"
+ ip="72.0.72.144"
+ subject="comment 2"
+ date="2014-04-19T04:26:57Z"
+ content="""
+i hit that thing again, utterly confused. it happens all the time with my laptop, which has access to my workstation, but not the other way around. so from the POV of my laptop, things make sense somehow, but from the workstation, the laptop doesn't show up at all, and when there's a transfer, it says it's into the unknown.
+
+really confusing... surely there's a way to show the remote name at least...
+"""]]
diff --git a/doc/bugs/remote_not_showing_up_in_webapp/comment_3_4aa72acc0938f7f824ba10f3f102e8bc._comment b/doc/bugs/remote_not_showing_up_in_webapp/comment_3_4aa72acc0938f7f824ba10f3f102e8bc._comment
new file mode 100644
index 0000000000..c0280b48bc
--- /dev/null
+++ b/doc/bugs/remote_not_showing_up_in_webapp/comment_3_4aa72acc0938f7f824ba10f3f102e8bc._comment
@@ -0,0 +1,16 @@
+[[!comment format=mdwn
+ username="http://joeyh.name/"
+ ip="24.159.78.125"
+ subject="comment 3"
+ date="2014-05-23T14:57:43Z"
+ content="""
+Yesterday, I made the webapp register ssh remotes it creates.  
+They then show up in the webapp when run in a repository where that ssh
+remote is not enabled.
+ 
+You can also manually register a ssh remote. First set up the git remote
+as usual. Then run `git annex initremote type=git name=foo
+location=$url`.
+
+So, this bugs titular issue is fixed, but the issue of it displaying \"unknown\" rather than the remote description is not yet fixed. Will retitle this bug appropriately, and leave open until that gets fixed.
+"""]]
diff --git a/doc/bugs/remote_repository_must_be_version_6_as_well_to_have_copied_files_appear_correctly.mdwn b/doc/bugs/remote_repository_must_be_version_6_as_well_to_have_copied_files_appear_correctly.mdwn
new file mode 100644
index 0000000000..57a1744556
--- /dev/null
+++ b/doc/bugs/remote_repository_must_be_version_6_as_well_to_have_copied_files_appear_correctly.mdwn
@@ -0,0 +1,47 @@
+### Please describe the problem.
+
+Originally spotted/reported: https://github.com/datalad/datalad/issues/1020
+
+If that is mandatory, then I guess there should be some error message.
+
+### What steps will reproduce the problem?
+
+create a remote repo without specifying version, i.e. of version 5 and copy data into it
+
+### What version of git-annex are you using? On what operating system?
+
+6.20160923+gitgd1dabb3-1~ndall+1
+
+### Please provide any additional information below.
+
+Output from running http://www.onerussian.com/tmp/v6-push.sh  with argument which specifies annex version within remote 
+
+[[!format sh """
+
+$> ./v6-push.sh 5 2>&1 | grep '>>>' 
+>>> /annex/objects/SHA256E-s4--181210f8f9c779c26da1d9b2075bde0127302ee0e3fca38c9a83f5b1dd8e5d3b.dat
+>>> /annex/objects/SHA256E-s4--181210f8f9c779c26da1d9b2075bde0127302ee0e3fca38c9a83f5b1dd8e5d3b.dat
+
+$> ./v6-push.sh 6 2>&1 | grep '>>>' 
+>>> /annex/objects/SHA256E-s4--181210f8f9c779c26da1d9b2075bde0127302ee0e3fca38c9a83f5b1dd8e5d3b.dat
+>>> 123
+
+"""]]
+
+
+[[!meta author=yoh]]
+
+> The data loss bugs are fixed in [[!commit ee309d694161d0f416420db6c4efb834c813351e]].
+> 
+> I am not sure yet why the keys database lacked an entry for the file;
+> perhaps something to do with it being a v6 unlocked file in a v5
+> repository.
+> 
+> Ok.. Seems that cloning to a v5 repository, and then copying/getting
+> objects into it, and then upgrading to v6 reproduces the problem with the
+> keys database. The inode cache does not get populated for unlocked files
+> on upgrade. Also, unlocked files stay as pointer files even when their
+> content is present in annex/objects. Fixed the upgrade process to handle
+> this case.
+> 
+> [[fixed|done]]] --[[Joey]]
diff --git a/doc/bugs/remotes_disappeared.mdwn b/doc/bugs/remotes_disappeared.mdwn
new file mode 100644
index 0000000000..918dc04337
--- /dev/null
+++ b/doc/bugs/remotes_disappeared.mdwn
@@ -0,0 +1,300 @@
+### Please describe the problem.
+
+Some remotes disappeared from `git annex info` after synchronising with a new repo.
+
+### What steps will reproduce the problem?
+
+1. have a nice repository full of remotes and special remotes
+2. create a new repository with `git init; git annex init; git reinit `
+3. sync the two repositories
+
+Expected the result: restore the lost repository the its previous state.
+
+Actual result: previous state available, but lost track of other repositories.
+
+I suspect there may be a relation to an old "forget history" transition at play here, as the last commit on the git-annex branch is:
+
+```
+commit 266099a48af81eab71d27741b43776372aa519c4
+Merge: 13ed0a6 05681b9
+Author: Antoine Beaupré 
+Date:   Wed Mar 30 12:49:55 2016 -0400
+
+    continuing transition ["forget git history","forget dead remotes","forget git history","forget dead remotes","forget git history","forget git history","forget dead remotes","forget git history","forget dead remotes","forget git history","forget dead remotes","forget git history","forget dead remotes"]
+```
+
+So in short, some remotes that i never marked as dead seem to be caught in the whirlpool of `forget` transitions.
+
+### What version of git-annex are you using? On what operating system?
+
+ 5.20151208-1~bpo8+1 on debian jessie.
+
+### Please provide any additional information below.
+
+So here's what i could salvage from my terminal history. Note that window resizing may have truncated some lines. Apologies for the mess...
+
+    [1077]anarcat@angela:cb39412b-d221-4846-a9dd-cdbabd9958f1$ sudo mkdir Music
+    [1078]anarcat@angela:cb39412b-d221-4846-a9dd-cdbabd9958f1$ sudo chown anarcat Music
+    [1079]anarcat@angela:cb39412b-d221-4846-a9dd-cdbabd9958f1$ cd Music/
+    [1082]anarcat@angela:Music$ git init
+    Dépôt Git vide initialisé dans /media/anarcat/cb39412b-d221-4846-a9dd-cdbabd9958f1/Music/.gi
+    [1083]anarcat@angela:Music$ git annex init
+    init  ok
+    (recording state in git...)
+    [1084]anarcat@angela:Music130$ git annex reinit 6f812272-18c8-4346-b68a-f57ae50f657e
+    reinit 6f812272-18c8-4346-b68a-f57ae50f657e ok
+    [1086]anarcat@angela:Music$ git remote add origin ~/mp3
+    [1087]anarcat@angela:Music$ git remote update # 12:44
+    Récupération de origin
+    warning: no common commits
+    remote: Décompte des objets: 819312, fait.
+    remote: Compression des objets: 100% (368189/368189), fait.
+    Réception d'objets: 100% (819312/819312), 64.97 MiB | 5.19 MiB/s, fait.
+    remote: Total 819312 (delta 653024), reused 588679 (delta 449859)
+    Depuis /home/anarcat/mp3
+     * [nouvelle branche] git-annex  -> origin/git-annex
+     * [nouvelle branche] master     -> origin/master
+     * [nouvelle branche] synced/git-annex -> origin/synced/git-annex
+     * [nouvelle branche] synced/master -> origin/synced/master
+     * [nouvelle étiquette] bak        -> bak
+    [1089]anarcat@angela:Music$ git annex merge
+    merge git-annex (merging origin/git-annex origin/synced/git-annex into git-annex...)
+    (recording state in git...)
+    [1092]anarcat@angela:Music$ git co master
+    Extraction des fichiers: 100% (22187/22187), fait.
+    La branche master est paramétrée pour suivre la branche distante master depuis origin.
+    Déjà sur 'master'
+    [1099]anarcat@angela:Music130$ date; time git annex get --quiet --in here; date
+    mercredi 30 mars 2016, 12:53:29 (UTC-0400)
+    sha256sum: .git/annex/tmp/SHA256E-s4476433--bb954dfe81f3d0906a18e53d02040d1d8f8e78917552e0033b056bbf885710d9.mp3: Aucun fichier ou dossier de ce type
+      sha256sum failed
+    git-annex: .git/annex/tmp/SHA256E-s4476433--bb954dfe81f3d0906a18e53d02040d1d8f8e78917552e0033b056bbf885710d9.mp3: openBinaryFile: does not exist (No such file or directory)
+    git-annex: get: 1 failed
+    Command exited with non-zero status 1
+    156.09user 39.26system 36:44.62elapsed 8%CPU (0avgtext+0avgdata 44900maxresident)k
+    8775224inputs+5811728outputs (51major+281689minor)pagefaults 0swaps
+    mercredi 30 mars 2016, 13:30:36 (UTC-0400)
+
+... in the last step, the external device disappeared and the transfer failed.
+
+After reconnecting the device, I tried to synchronise its content with the content of another device, but that device is gone!
+
+[[!format sh """
+[1013]anarcat@angela:Music1$ git annex info --fast # first notice how the remotes are not the same on the two repositories
+repository mode: indirect
+trusted repositories: 0
+semitrusted repositories: 5
+        00000000-0000-0000-0000-000000000001 -- web
+        00000000-0000-0000-0000-000000000002 -- bittorrent
+        0f9185ea-8462-4230-8cae-462a1ad0df36 -- origin
+        45124790-dbb7-4e2e-bc0a-acfb618a01e0 -- anarcat@angela:/media/anarcat/cb39412b-d221-4846-a9dd-cdbabd9958f1/Music
+        6f812272-18c8-4346-b68a-f57ae50f657e -- here
+untrusted repositories: 0
+transfers in progress: none
+available local disk space: 4.68 gigabytes (+1 megabyte reserved)
+[1014]anarcat@angela:Music$ cd ~/mp3
+[1015]anarcat@angela:mp3$ git annex info --fast # here there is a lot more remotes!
+repository mode: indirect
+trusted repositories: 4
+        22921df6-ff75-491c-b5d9-5a2aab33a689 -- anarcat@marcos:/media/anarcat/79884590-6445-4a6f-ae12-050b9a7c1912/mp3
+        b7802161-c984-4c9f-8d05-787a29c41cfe -- anarcat@marcos:/srv/mp3 [marcos]
+        c2ca4a13-9a5f-461b-a44b-53255ed3e2f9 -- anarcat@desktop008:/srv/musique/anarcat/mp3 [markov]
+        f8818d12-9882-4ca5-bc0f-04e987888a8d -- anarcat@marcos:/media/anarcat/green_crypt/mp3/
+semitrusted repositories: 8
+        00000000-0000-0000-0000-000000000001 -- web
+        00000000-0000-0000-0000-000000000002 -- bittorrent
+        0f9185ea-8462-4230-8cae-462a1ad0df36 -- anarcat@angela:~/mp3 [here]
+        3f6d8082-6f4b-4faa-a3d9-bd5db1891077 -- anarcat@lab-sc.no-ip.org:mp3
+        4249a4ea-343a-43a8-9bba-457d2ff87c7d -- rachel@topcrapn:~/Musique/MUSIC/anarcat
+        487dda55-d164-4bf1-9d85-66caaa9c0743 -- 300GB hard drive labeled VHS [VHS]
+        6f812272-18c8-4346-b68a-f57ae50f657e -- htcones
+        f867da6f-78cb-49be-a0db-d1c8e5f53664 -- n900
+untrusted repositories: 0
+transfers in progress: none
+available local disk space: 13.51 gigabytes (+1 megabyte reserved)
+[1016]anarcat@angela:mp3$ cd -
+/media/anarcat/cb39412b-d221-4846-a9dd-cdbabd9958f1/Music
+[1017]anarcat@angela:Music$ git annex find --in f867da6f-78cb-49be-a0db-d1c8e5f53664 --not --in here # trying to sync with the n900 remote
+git-annex: there is no available git remote named "f867da6f-78cb-49be-a0db-d1c8e5f53664"
+[1018]anarcat@angela:Music1$ git annex sync # maybe some data is missing?
+commit  ok
+pull origin
+ok
+push origin
+Décompte des objets: 6368, fait.
+Delta compression using up to 2 threads.
+Compression des objets: 100% (6361/6361), fait.
+Écriture des objets: 100% (6368/6368), 693.21 KiB | 565.00 KiB/s, fait.
+Total 6368 (delta 5030), reused 11 (delta 3)
+To /home/anarcat/mp3
+   05681b9..266099a  git-annex -> synced/git-annex
+ok
+[1019]anarcat@angela:Music$ git annex find --in f867da6f-78cb-49be-a0db-d1c8e5f53664 --not --in here # trying again
+git-annex: there is no available git remote named "f867da6f-78cb-49be-a0db-d1c8e5f53664"
+[1020]anarcat@angela:Music1$ git annex find --in n900 --not --in here # of course, n900 is not a git remote locally
+git-annex: there is no available git remote named "n900"
+[1021]anarcat@angela:Music1$ git annex info --fast # and it's still not there
+repository mode: indirect
+trusted repositories: 0
+semitrusted repositories: 5
+        00000000-0000-0000-0000-000000000001 -- web
+        00000000-0000-0000-0000-000000000002 -- bittorrent
+        0f9185ea-8462-4230-8cae-462a1ad0df36 -- origin
+        45124790-dbb7-4e2e-bc0a-acfb618a01e0 -- anarcat@angela:/media/anarcat/cb39412b-d221-
+        6f812272-18c8-4346-b68a-f57ae50f657e -- here
+untrusted repositories: 0
+transfers in progress: none
+available local disk space: 4.68 gigabytes (+1 megabyte reserved)
+[1022]anarcat@angela:Music$ cd -
+/home/anarcat/mp3
+[1023]anarcat@angela:mp3$ git annex info --fast # worse: it's gone from my main repo!!
+repository mode: indirect
+trusted repositories: 2
+        b7802161-c984-4c9f-8d05-787a29c41cfe -- marcos
+        c2ca4a13-9a5f-461b-a44b-53255ed3e2f9 -- markov
+semitrusted repositories: 6
+        00000000-0000-0000-0000-000000000001 -- web
+        00000000-0000-0000-0000-000000000002 -- bittorrent
+        0f9185ea-8462-4230-8cae-462a1ad0df36 -- here
+        45124790-dbb7-4e2e-bc0a-acfb618a01e0 -- anarcat@angela:/media/anarcat/cb39412b-d221-4846-a9dd-cdbabd9958f1/Music
+        487dda55-d164-4bf1-9d85-66caaa9c0743 -- VHS
+        6f812272-18c8-4346-b68a-f57ae50f657e -- htcones
+untrusted repositories: 0
+transfers in progress: none
+available local disk space: 13.74 gigabytes (+1 megabyte rese
+"""]]
+
+In my opinion, git-annex shouldn't have lost the following repositories:
+
+        22921df6-ff75-491c-b5d9-5a2aab33a689 -- anarcat@marcos:/media/anarcat/79884590-6445-4a6f-ae12-050b9a7c1912/mp3
+        f8818d12-9882-4ca5-bc0f-04e987888a8d -- anarcat@marcos:/media/anarcat/green_crypt/mp3/
+        3f6d8082-6f4b-4faa-a3d9-bd5db1891077 -- anarcat@lab-sc.no-ip.org:mp3
+        4249a4ea-343a-43a8-9bba-457d2ff87c7d -- rachel@topcrapn:~/Musique/MUSIC/anarcat
+        f867da6f-78cb-49be-a0db-d1c8e5f53664 -- n900
+
+Those are repositories that are in the git-annex history, but that don't have git remotes associated with them, for various reasons. I do *not* believe I have marked any of those as "dead" except maybe 3f6d8082-6f4b-4faa-a3d9-bd5db1891077. f8818d12-9882-4ca5-bc0f-04e987888a8d was used during the weekend to do my backups, so it's definitely not dead.
+
+It is also interesting to note that even though `git annex info` doesn't know about the remotes, there is still tracking information about all of them (except the 3f one):
+
+[[!format txt """
+$ git cat-file -p git-annex:001/694/SHA256E-s6732474--e084001bc23a90bfd65d9a3fa20b7bf878be6a49fce7e5a9846efeeba8815516.mp3.log
+1376877225.866849s 1 b7802161-c984-4c9f-8d05-787a29c41cfe
+1378838314.653241s 1 c2ca4a13-9a5f-461b-a44b-53255ed3e2f9
+1379790798.215871s 1 0f9185ea-8462-4230-8cae-462a1ad0df36
+1391065040.28672s 1 22921df6-ff75-491c-b5d9-5a2aab33a689
+1397893686.079999s 1 487dda55-d164-4bf1-9d85-66caaa9c0743
+1398657510.376249s 1 4249a4ea-343a-43a8-9bba-457d2ff87c7d
+1407479149.838437s 1 f8818d12-9882-4ca5-bc0f-04e987888a8d
+1407479149.929843s 1 f8818d12-9882-4ca5-bc0f-04e987888a8d
+1407510928.457047s 1 487dda55-d164-4bf1-9d85-66caaa9c0743
+1424134721.290026s 1 b7802161-c984-4c9f-8d05-787a29c41cfe
+1424227570.153123s 1 487dda55-d164-4bf1-9d85-66caaa9c0743
+1445468844.056214s 1 f867da6f-78cb-49be-a0db-d1c8e5f53664
+1458775384.454193s 0 6f812272-18c8-4346-b68a-f57ae50f657e
+"""]]
+
+it's a mystery to me why that stuff disappeared from `git-annex info`. it's especially confusing since some commands seem to recognize there *was* a remote there:
+
+    $ git annex enableremote f867da6f-78cb-49be-a0db-d1c8e5f53664
+    git-annex: Unknown special remote.
+            f867da6f-78cb-49be-a0db-d1c8e5f53664 -- n900
+
+Notice how `n900` was shown here: that metadata obviously *is* somewhere! The uuid.log file is obviously damaged:
+
+    [1036]anarcat@angela:mp31$ git cat-file -p git-annex:uuid.log
+    45124790-dbb7-4e2e-bc0a-acfb618a01e0 anarcat@angela:/media/anarcat/cb39412b-d221-4846-a9dd-cdbabd9958f1/Music timestamp=1459356223.537195s
+
+... and it clearly looks like the file was damaged on that new repo:
+
+[[!format txt """
+*   266099a N (synced/git-annex, git-annex) continuing transition ["forget git history","forget dead remotes","forget git history","forget dead remotes","forget git history","forget git history","forget dead remotes","forget git history","forget dead remotes","forget git history","forget dead remotes","forget git history","forget dead remotes"] (il y a 75 minutes) 
+|\
+* | 298cc10 N (htcones/synced/git-annex) continuing transition ["forget git history","forget dead remotes","forget git history","forget dead remotes","forget git history","forget git history","forget dead remotes","forget git history","forget dead remotes","forget git history","forget dead remotes","forget git history","forget dead remotes"] (il y a 13 jours) | |
+| | diff --git a/uuid.log b/uuid.log
+| | index 656b369..a730ad0 100644
+| | --- a/uuid.log
+| | +++ b/uuid.log
+| | @@ -1,14 +1,9 @@
+| |  0f9185ea-8462-4230-8cae-462a1ad0df36 anarcat@angela:~/mp3 timestamp=1376882226.164478s
+| |  22921df6-ff75-491c-b5d9-5a2aab33a689 anarcat@marcos:/media/anarcat/79884590-6445-4a6f-ae12-050b9a7c1912/mp3 timestamp=1397741440.125973s
+| | -2fec390f-f21b-4293-be50-f219be10ea02 anarcat@marcos:/media/anarcat/Nokia N900/.sounds/mp3-test timestamp=1444709804.587901s
+| |  3f6d8082-6f4b-4faa-a3d9-bd5db1891077 anarcat@lab-sc.no-ip.org:mp3 timestamp=1397882243.602438s
+| | -3f6d8082-6f4b-4faa-a3d9-bd5db1891077 anarcat@serveur-maison:~/mp3 timestamp=1397870776.162703s
+| |  4249a4ea-343a-43a8-9bba-457d2ff87c7d rachel@topcrapn:~/Musique/MUSIC/anarcat timestamp=1398631775.545666s
+| |  487dda55-d164-4bf1-9d85-66caaa9c0743 300GB hard drive labeled VHS timestamp=1397880144.616515s
+| | -509d1cd7-ecd1-4e2f-803d-575d9c56a5bc anarcat@angela:/media/anarcat/Nokia N900/.sounds/mp3 timestamp=1444483581.960103s
+| |  b7802161-c984-4c9f-8d05-787a29c41cfe anarcat@marcos:/srv/mp3 timestamp=1376874943.951103s
+| |  c2ca4a13-9a5f-461b-a44b-53255ed3e2f9 anarcat@desktop008:/srv/musique/anarcat/mp3 timestamp=1410989161.237514s
+| | -c2ca4a13-9a5f-461b-a44b-53255ed3e2f9 anarcat@desktop008:/srv/musique/anarcat/musique/mp3 timestamp=1384884883.106166s
+| | -f641e18f-7cb1-49ba-abe1-7544b435b67f anarcat@marcos:/media/anarcat/Nokia N900/.sounds/mp3-init timestamp=1444708760.468268s
+| |  f867da6f-78cb-49be-a0db-d1c8e5f53664 n900 timestamp=1445438785.796624s
+| |  f8818d12-9882-4ca5-bc0f-04e987888a8d anarcat@marcos:/media/anarcat/green_crypt/mp3/ timestamp=1407470841.967597s
+"""]]
+
+So i guess i could manually edit that uuid.log file to restore my metadata, but i'm puzzled as to where my data went or how!
+
+### Workaround
+
+The workaround is to, of course, restore a known sane `uuid.log`:
+
+[[!format sh """
+[1048]anarcat@angela:~128$ git clone -b git-annex mp3 mp3.annex
+Clonage dans 'mp3.annex'...
+fait.
+Extraction des fichiers: 100% (32234/32234), fait.
+[1049]anarcat@angela:~$ cd mp3.annex/
+[1067]anarcat@angela:mp3.annex$ git co dfe1b77 uuid.log
+[1068]anarcat@angela:mp3.annex$ git status
+Sur la branche git-annex
+Votre branche est à jour avec 'origin/git-annex'.
+Modifications qui seront validées :
+  (utilisez "git reset HEAD ..." pour désindexer)
+
+        modifié :         uuid.log
+
+[1069]anarcat@angela:mp3.annex$ git commit -m"restore broken uuid.log"
+[git-annex 9628f3b] restore broken uuid.log
+ 1 file changed, 14 insertions(+), 1 deletion(-)
+ rewrite uuid.log (100%)
+[1070]anarcat@angela:mp3.annex$ git push
+Décompte des objets: 2, fait.
+Delta compression using up to 2 threads.
+Compression des objets: 100% (2/2), fait.
+Écriture des objets: 100% (2/2), 262 bytes | 0 bytes/s, fait.
+Total 2 (delta 1), reused 0 (delta 0)
+To /home/anarcat/mp3
+   266099a..9628f3b  git-annex -> git-annex
+[1071]anarcat@angela:mp3.annex$ cd -
+/home/anarcat
+[1072]anarcat@angela:~$ git ^C
+[1072]anarcat@angela:~130$ cd -
+/home/anarcat/mp3.annex
+[1072]anarcat@angela:mp3.annex$ cd ../mp3
+[1073]anarcat@angela:mp3$ git annex merge
+merge git-annex ok
+[1074]anarcat@angela:mp3$ git annex info --fast
+repository mode: indirect
+trusted repositories: 4
+        22921df6-ff75-491c-b5d9-5a2aab33a689 -- anarcat@marcos:/media/anarcat/79884590-6445-4a6f-ae12-050b9a7c1912/mp3
+        b7802161-c984-4c9f-8d05-787a29c41cfe -- anarcat@marcos:/srv/mp3 [marcos]
+        c2ca4a13-9a5f-461b-a44b-53255ed3e2f9 -- anarcat@desktop008:/srv/musique/anarcat/mp3 [markov]
+        f8818d12-9882-4ca5-bc0f-04e987888a8d -- anarcat@marcos:/media/anarcat/green_crypt/mp3/
+semitrusted repositories: 8
+        00000000-0000-0000-0000-000000000001 -- web
+        00000000-0000-0000-0000-000000000002 -- bittorrent
+        0f9185ea-8462-4230-8cae-462a1ad0df36 -- anarcat@angela:~/mp3 [here]
+        3f6d8082-6f4b-4faa-a3d9-bd5db1891077 -- anarcat@lab-sc.no-ip.org:mp3
+        4249a4ea-343a-43a8-9bba-457d2ff87c7d -- rachel@topcrapn:~/Musique/MUSIC/anarcat
+        487dda55-d164-4bf1-9d85-66caaa9c0743 -- 300GB hard drive labeled VHS [VHS]
+        6f812272-18c8-4346-b68a-f57ae50f657e -- htcones
+        f867da6f-78cb-49be-a0db-d1c8e5f53664 -- n900
+untrusted repositories: 0
+transfers in progress: none
+available local disk space: 13.44 gigabytes (+1 megabyte reserved)
+"""]]
+
+But it seems to me there is a weird interaction between transitions and fresh new git repo syncs... that could be fixed...
+
+### Have you had any luck using git-annex before? (Sometimes we get tired of reading bug reports all day and a lil' positive end note does wonders)
+
+Yes! Git-annex works generally well and, even though i often report bugs here, i am still quite happy with it. :) --[[anarcat]]
diff --git a/doc/bugs/remotes_disappeared/comment_1_7b7bfa0b2c9e66edd92c076e5c6ccdd7._comment b/doc/bugs/remotes_disappeared/comment_1_7b7bfa0b2c9e66edd92c076e5c6ccdd7._comment
new file mode 100644
index 0000000000..8252020d35
--- /dev/null
+++ b/doc/bugs/remotes_disappeared/comment_1_7b7bfa0b2c9e66edd92c076e5c6ccdd7._comment
@@ -0,0 +1,21 @@
+[[!comment format=mdwn
+ username="joey"
+ subject="""comment 1"""
+ date="2016-04-04T19:45:03Z"
+ content="""
+I see that you're using `reinit` .. so are you sure that the UUID you
+reused there or at some other time did not belong to a remote that had
+previously been marked dead? 
+
+I mean, reusing UUIDs of dead remotes (or just generally reusing UUIDS at
+all) seems like the most obvious way to shoot yourself in the foot and get
+into something resembling this sitation.
+
+Your transcript seems to show one git-annex info in the ~/mp3 repository
+listing the remotes, and a subsquent git-annex info in the ~/mp3 repository
+not listing the remotes, with no commands in between that touch that
+repository in any way. This makes me suspect something is being left out
+somehow. Are you using the assistant or something?
+
+You can see which remotes have been marked dead by `git show git-annex:trust.log | grep X`
+"""]]
diff --git a/doc/bugs/remotes_disappeared/comment_2_b9dae978aeeee5e5d246192fed813b3f._comment b/doc/bugs/remotes_disappeared/comment_2_b9dae978aeeee5e5d246192fed813b3f._comment
new file mode 100644
index 0000000000..2edc1a852a
--- /dev/null
+++ b/doc/bugs/remotes_disappeared/comment_2_b9dae978aeeee5e5d246192fed813b3f._comment
@@ -0,0 +1,13 @@
+[[!comment format=mdwn
+ username="joey"
+ subject="""comment 2"""
+ date="2016-04-04T19:56:42Z"
+ content="""
+I see no ecidence of a file being damanged. I see evidence of the
+dead remote removal code removing dead remotes from uuid.log, which
+is what it's supposed to do, if enabled by 
+`git annex forget --drop-dead` having been run at some point.
+
+Based on the number of transitions listed in the commit message,
+you've run it repeatedly, even.
+"""]]
diff --git a/doc/bugs/remotes_disappeared/comment_3_794372fb37621f37478313abc4ee92cf._comment b/doc/bugs/remotes_disappeared/comment_3_794372fb37621f37478313abc4ee92cf._comment
new file mode 100644
index 0000000000..2fa413cc02
--- /dev/null
+++ b/doc/bugs/remotes_disappeared/comment_3_794372fb37621f37478313abc4ee92cf._comment
@@ -0,0 +1,38 @@
+[[!comment format=mdwn
+ username="anarcat"
+ subject="""comment 3"""
+ date="2016-04-04T20:39:24Z"
+ content="""
+No trust information was ever assigned to that remote:
+
+    [1012]anarcat@angela:mp31$ git show git-annex:trust.log | grep 6f812272-18c8-4346-b68a-f57ae50f657e
+    [1013]anarcat@angela:mp31$ git log -p  git-annex -- trust.log | grep 6f812272-18c8-4346-b68a-f57ae50f657e
+    [1014]anarcat@angela:mp31$
+
+None of the repositories removed from the `git annex info` display
+were ever marked as dead, in fact, two of those were marked as
+`trusted`, from what I can see here:
+
+    [1018]anarcat@angela:mp31$ for repo in 22921df6-ff75-491c-b5d9-5a2aab33a689 f8818d12-9882-4ca5-bc0f-04e987888a8d 3f6d8082-6f4b-4faa-a3d9-bd5db1891077 4249a4ea-343a-43a8-9bba-457d2ff87c7d f867da6f-78cb-49be-a0db-d1c8e5f53664 ; do
+    >  git log -p  git-annex -- trust.log | grep $repo
+    > done
+     22921df6-ff75-491c-b5d9-5a2aab33a689 1 timestamp=1400247400.46351s
+    +22921df6-ff75-491c-b5d9-5a2aab33a689 1 timestamp=1400247400.46351s
+     f8818d12-9882-4ca5-bc0f-04e987888a8d 1 timestamp=1424140026.947448s
+     f8818d12-9882-4ca5-bc0f-04e987888a8d 1 timestamp=1424140026.947448s
+    +f8818d12-9882-4ca5-bc0f-04e987888a8d 1 timestamp=1424140026.947448s
+
+And yes, I did run a transition a while back. I don't remember if I
+ran it multiple times: this is one of my first git-annex repos, so a
+lot of stuff may have happened to it its 3 years of existence:
+
+    $ git log  | tail -5
+    commit 2400a4b46bc2b1a015e6881ef0c331c519016b64
+    Author: Antoine Beaupré 
+    Date:   Sun Aug 18 22:14:47 2013 -0400
+    
+        enter git annex
+
+It seems to me that the transition mechanism *may* be triggered before
+the `uuid.log` or `trust.log` files be fully merged - is that possible?
+"""]]
diff --git a/doc/bugs/reports_success_when_addurl_--batch__a_file_which_is_.gitignore__39__d.mdwn b/doc/bugs/reports_success_when_addurl_--batch__a_file_which_is_.gitignore__39__d.mdwn
new file mode 100644
index 0000000000..1fbd3bdaa4
--- /dev/null
+++ b/doc/bugs/reports_success_when_addurl_--batch__a_file_which_is_.gitignore__39__d.mdwn
@@ -0,0 +1,36 @@
+### Please describe the problem.
+
+Reports success although file is not added to git, since ignored due to .gitignore
+
+Although not sure yet if that is annex could actually take care about since probably annex stages those for 'add' command to git so wouldn't know right when addurl is called for a specific file?
+
+### What steps will reproduce the problem?
+
+see below
+
+### What version of git-annex are you using? On what operating system?
+
+6.20160728+gitg9a2fe62-1~ndall+1
+
+
+[[!format sh """
+
+$> rm -rf /tmp/123; mkdir /tmp/123; cd /tmp/123; git init; echo "*exclude*" >| .git/info/exclude; git annex init; { echo "http://www.onerussian.com/tmp/1exclude.txt 1exclude.txt"; echo "2nd one" >&2; echo "http://www.onerussian.com/tmp/2.txt 2.txt"\; } | git annex addurl -c annex.largefiles=exclude=*.txt --batch --json --with-files                                                                       
+Initialized empty Git repository in /tmp/123/.git/
+init  ok
+(recording state in git...)
+2nd one
+{"command":"addurl","key":null,"file":"1exclude.txt","note":"downloading http://www.onerussian.com/tmp/1exclude.txt ...","note":"non-large file; adding content to git repository","success":true}
+{"command":"addurl","key":null,"file":"2.txt;","note":"downloading http://www.onerussian.com/tmp/2.txt ...","key":"SHA256E-s2--53c234e5e8472b6ac51c1ae1cab3fe06fad053beb8ebfd8977b010655bfdd3c3.txt","success":true}
+The following paths are ignored by one of your .gitignore files:
+1exclude.txt
+Use -f if you really want to add them.
+git-annex: user error (xargs ["-0","git","--git-dir=.git","--work-tree=.","--literal-pathspecs","-c","annex.largefiles=exclude=*.txt","add","--"] exited 123)
+"""]]
+
+[[!meta author=yoh]]
+
+> And it leaves the unstaged symlink behind too.
+> 
+> [[fixed|done]] to check ignore status before creating the file.
+> --[[Joey]]
diff --git a/doc/bugs/rsync__58___protocol_version_mismatch.mdwn b/doc/bugs/rsync__58___protocol_version_mismatch.mdwn
new file mode 100644
index 0000000000..fb5500601a
--- /dev/null
+++ b/doc/bugs/rsync__58___protocol_version_mismatch.mdwn
@@ -0,0 +1,38 @@
+### Please describe the problem.
+This is a weird one. I'm getting rsync `protocol version mismatch -- is your shell clean?` on a particular file.
+
+I have tried to reproduce it but not been able.
+
+ * First time it happened, I went to the problem repo and did a `get`, which worked, destroying the evidence. "Luckily", this happened again a few days later.
+ * I thought maybe it was because there was a partial transfer in `.git/annex/tmp` with some specific characteristics. Nope, if I remove the file in `tmp` the problem remains.
+ * I made another clone think it was the problem repo, transferred the file to that repo, no problem. Dropped it, pointed everything back to the original bad repo. Still bad.
+ * Both sides are running `rsync  version 3.1.0  protocol version 31`.
+
+
+### What steps will reproduce the problem?
+
+Working on it. But I want to put the preliminaries here, in case someone else has seen this.
+
+### What version of git-annex are you using? On what operating system?
+
+Both sides are running `git-annex version: 5.20150508-g883d57f`, on Ubuntu 14.04.2 LTS.
+
+
+### Please provide any additional information below.
+
+[[!format sh """
+# If you can, paste a complete transcript of the problem occurring here.
+# If the problem is with the git-annex assistant, paste in .git/annex/daemon.log
+$ git annex copy --to acozed claes/colt/20150511_174818.jpg 
+copy claes/colt/20150511_174818.jpg (checking acozed...) (to acozed...) 
+protocol version mismatch -- is your shell clean?
+(see the rsync man page for an explanation)
+rsync error: protocol incompatibility (code 2) at compat.c(176) [sender=3.1.1]
+
+  rsync failed -- run git annex again to resume file transfer
+failed
+git-annex: copy: 1 failed
+
+
+# End of transcript or log.
+"""]]
diff --git a/doc/bugs/rsync__58___protocol_version_mismatch/comment_1_672d89a7d06d5ec336381b670a41a9c7._comment b/doc/bugs/rsync__58___protocol_version_mismatch/comment_1_672d89a7d06d5ec336381b670a41a9c7._comment
new file mode 100644
index 0000000000..8379a7d826
--- /dev/null
+++ b/doc/bugs/rsync__58___protocol_version_mismatch/comment_1_672d89a7d06d5ec336381b670a41a9c7._comment
@@ -0,0 +1,12 @@
+[[!comment format=mdwn
+ username="joey"
+ subject="""comment 1"""
+ date="2015-05-22T19:12:37Z"
+ content="""
+I think I've seen this where the shell was running some command
+at (non-interactive) login that output stuff and so screwed up the rsync
+protocol.
+
+rsync with sufficient -vvvv will print out a lot of debugging info about
+the protocol.
+"""]]
diff --git a/doc/bugs/rsync__58___protocol_version_mismatch/comment_2_531b85c911d390b1b93ee55a8cf5d47e._comment b/doc/bugs/rsync__58___protocol_version_mismatch/comment_2_531b85c911d390b1b93ee55a8cf5d47e._comment
new file mode 100644
index 0000000000..268f369936
--- /dev/null
+++ b/doc/bugs/rsync__58___protocol_version_mismatch/comment_2_531b85c911d390b1b93ee55a8cf5d47e._comment
@@ -0,0 +1,11 @@
+[[!comment format=mdwn
+ username="clacke"
+ subject="comment 2"
+ date="2015-05-22T21:08:26Z"
+ content="""
+It's weirder than that. I add a cow union mount over it, it works fine.
+
+I copy the file, I drop the file, I remove the union mount. Again it's back in the broken state. Rebooting does not help, so it's not some very insistent lock.
+
+Next I will try a copy of the repo, to see if that is able to carry over whatever strange state this thing is in.
+"""]]
diff --git a/doc/bugs/rsync__58___protocol_version_mismatch/comment_3_eff8f33134157635387ee681805ff7a8._comment b/doc/bugs/rsync__58___protocol_version_mismatch/comment_3_eff8f33134157635387ee681805ff7a8._comment
new file mode 100644
index 0000000000..b2a335a0bc
--- /dev/null
+++ b/doc/bugs/rsync__58___protocol_version_mismatch/comment_3_eff8f33134157635387ee681805ff7a8._comment
@@ -0,0 +1,7 @@
+[[!comment format=mdwn
+ username="clacke"
+ subject="comment 3"
+ date="2015-05-22T22:16:45Z"
+ content="""
+Note that a thousand files went over without a hitch, but this particular file consistently fails.
+"""]]
diff --git a/doc/bugs/rsync__58___protocol_version_mismatch/comment_4_77e59a25c859b6afec8b75f74885ef5e._comment b/doc/bugs/rsync__58___protocol_version_mismatch/comment_4_77e59a25c859b6afec8b75f74885ef5e._comment
new file mode 100644
index 0000000000..cad761d149
--- /dev/null
+++ b/doc/bugs/rsync__58___protocol_version_mismatch/comment_4_77e59a25c859b6afec8b75f74885ef5e._comment
@@ -0,0 +1,9 @@
+[[!comment format=mdwn
+ username="clacke"
+ subject="comment 4"
+ date="2015-05-24T17:41:17Z"
+ content="""
+Hm, not so sure that \"rebooted, did not help\" was actually true. I take that back.
+
+Now I saw a stray `git-annex-shell recv-key` process mentioning that file. I killed it and now everything seems fine. I will keep this in mind for next time, to see if I can verify that this was actually the cause of the message, but maybe it's a clue.
+"""]]
diff --git a/doc/bugs/rsync__58___protocol_version_mismatch/comment_5_7d819c3e4af2b8044a52fa6131f36187._comment b/doc/bugs/rsync__58___protocol_version_mismatch/comment_5_7d819c3e4af2b8044a52fa6131f36187._comment
new file mode 100644
index 0000000000..85af7d2215
--- /dev/null
+++ b/doc/bugs/rsync__58___protocol_version_mismatch/comment_5_7d819c3e4af2b8044a52fa6131f36187._comment
@@ -0,0 +1,22 @@
+[[!comment format=mdwn
+ username="joey"
+ subject="""comment 5"""
+ date="2015-05-26T14:52:11Z"
+ content="""
+The stray recv-key is a good clue. git-annex-shell only allows one upload
+of a given file to run at a time. So if you get a transfer stalled out,
+it will reject another transfer attempt. This can sometimes happen when eg,
+migrating between networks and restarting an upload when the old one is
+still running on the server.
+
+However, normally there is an error message "transfer already in progress".
+
+It may be that your rsync is not forwarding that stderr through to display
+it, for some reason.
+
+It would probably help if you can run the same git-annex shell command line
+on the server, verify that it fails with "transfer already in progress"
+when an recvkey of that key is already running. Then you could try sshing
+to the server noninteractively and having it run the same git-annex-shell
+command, and see if the error makes it through that.
+"""]]
diff --git a/doc/bugs/rsync_fails_with_sync_error__58___syntax_or_usage_error_.mdwn b/doc/bugs/rsync_fails_with_sync_error__58___syntax_or_usage_error_.mdwn
new file mode 100644
index 0000000000..09c79d78dd
--- /dev/null
+++ b/doc/bugs/rsync_fails_with_sync_error__58___syntax_or_usage_error_.mdwn
@@ -0,0 +1,85 @@
+### Please describe the problem.
+
+i tried to sync my home music annex to a server. Half the files worked nicely, but on the rest it fails with rsync errors.
+
+### What steps will reproduce the problem?
+
+git annex -d sync --content
+
+### What version of git-annex are you using? On what operating system?
+
+Client:
+
+ubuntu trusty
+Version: 5.20140517.4
+
+Server:
+
+ubuntu trusty
+Version: 5.20140412ubuntu1
+
+
+### Please provide any additional information below.
+
+[[!format sh """
+dirus-dom:/music$ git annex -d sync --content
+[2014-10-28 19:18:28 CET] read: git ["--git-dir=/exports/music/.git","--work-tree=/exports/music","show-ref","git-annex"]
+[2014-10-28 19:18:28 CET] read: git ["--git-dir=/exports/music/.git","--work-tree=/exports/music","show-ref","--hash","refs/heads/git-annex"]
+[2014-10-28 19:18:29 CET] read: git ["--git-dir=/exports/music/.git","--work-tree=/exports/music","log","refs/heads/git-annex..1599d29eba7a0ec50217c2b4a3f4cc1ecc8b2116","--oneline","-n1"]
+[2014-10-28 19:18:29 CET] read: git ["--git-dir=/exports/music/.git","--work-tree=/exports/music","log","refs/heads/git-annex..27b47301dcb5007705d1dcd5a414df964b840467","--oneline","-n1"]
+[2014-10-28 19:18:29 CET] read: git ["--git-dir=/exports/music/.git","--work-tree=/exports/music","log","refs/heads/git-annex..a95ca0282fefcd774ec8a00b6f33c11f08f789d8","--oneline","-n1"]
+[2014-10-28 19:18:29 CET] chat: git ["--git-dir=/exports/music/.git","--work-tree=/exports/music","cat-file","--batch"]
+commit  [2014-10-28 19:18:29 CET] read: git ["--git-dir=/exports/music/.git","--work-tree=/exports/music","commit","-a","-m","git-annex automatic sync"]
+ok
+[2014-10-28 19:18:31 CET] read: git ["--git-dir=/exports/music/.git","--work-tree=/exports/music","symbolic-ref","HEAD"]
+[2014-10-28 19:18:31 CET] read: git ["--git-dir=/exports/music/.git","--work-tree=/exports/music","show-ref","refs/heads/master"]
+[2014-10-28 19:18:31 CET] call: git ["--git-dir=/exports/music/.git","--work-tree=/exports/music","show-ref","--verify","-q","refs/heads/synced/master"]
+[2014-10-28 19:18:31 CET] read: git ["--git-dir=/exports/music/.git","--work-tree=/exports/music","log","refs/heads/master..refs/heads/synced/master","--oneline","-n1"]
+pull sync.poelzi.org__music
+[2014-10-28 19:18:31 CET] read: ssh ["-O","stop","-S","poelzigit@sync.poelzi.org","-o","ControlMaster=auto","-o","ControlPersist=yes","localhost"]
+[2014-10-28 19:18:31 CET] call: git ["--git-dir=/exports/music/.git","--work-tree=/exports/music","fetch","sync.poelzi.org__music"]
+[2014-10-28 19:18:33 CET] call: git ["--git-dir=/exports/music/.git","--work-tree=/exports/music","show-ref","--verify","-q","refs/remotes/sync.poelzi.org__music/master"]
+[2014-10-28 19:18:33 CET] read: git ["--git-dir=/exports/music/.git","--work-tree=/exports/music","log","refs/heads/master..refs/remotes/sync.poelzi.org__music/master","--oneline","-n1"]
+[2014-10-28 19:18:33 CET] call: git ["--git-dir=/exports/music/.git","--work-tree=/exports/music","show-ref","--verify","-q","refs/remotes/sync.poelzi.org__music/synced/master"]
+[2014-10-28 19:18:33 CET] read: git ["--git-dir=/exports/music/.git","--work-tree=/exports/music","log","refs/heads/synced/master..refs/remotes/sync.poelzi.org__music/synced/master","--oneline","-n1"]
+ok
+[2014-10-28 19:18:33 CET] read: git ["--git-dir=/exports/music/.git","--work-tree=/exports/music","show-ref","git-annex"]
+[2014-10-28 19:18:33 CET] read: git ["--git-dir=/exports/music/.git","--work-tree=/exports/music","show-ref","--hash","refs/heads/git-annex"]
+[2014-10-28 19:18:33 CET] read: git ["--git-dir=/exports/music/.git","--work-tree=/exports/music","log","refs/heads/git-annex..1599d29eba7a0ec50217c2b4a3f4cc1ecc8b2116","--oneline","-n1"]
+[2014-10-28 19:18:33 CET] read: git ["--git-dir=/exports/music/.git","--work-tree=/exports/music","log","refs/heads/git-annex..27b47301dcb5007705d1dcd5a414df964b840467","--oneline","-n1"]
+[2014-10-28 19:18:33 CET] read: git ["--git-dir=/exports/music/.git","--work-tree=/exports/music","log","refs/heads/git-annex..a95ca0282fefcd774ec8a00b6f33c11f08f789d8","--oneline","-n1"]
+[2014-10-28 19:18:33 CET] read: git ["--git-dir=/exports/music/.git","--work-tree=/exports/music","ls-files","--cached","-z","--"]
+[2014-10-28 19:18:33 CET] chat: git ["--git-dir=/exports/music/.git","--work-tree=/exports/music","check-attr","-z","--stdin","annex.backend","annex.numcopies","--"]
+copy Alan Parsons Project/Eye In The Sky/.07 - Psychobabble.mood copy Alan Parsons Project/Eye In The Sky/.07 - Psychobabble.mood (checking sync.poelzi.org__music...) [2014-10-28 19:18:43 CET] call: ssh ["-S",".git/annex/ssh/poelzigit@sync.poelzi.org","-o","ControlMaster=auto","-o","ControlPersist=yes","-T","poelzigit@sync.poelzi.org","git-annex-shell 'inannex' '/music/' 'SHA256E-s3000--da8a3336a484a171a438c99660260cc35cbd37c339dd2c18447cd025064bc00b.mood' --uuid 35a89672-4ff5-4d9a-9bf2-cedb272bb7cb"]
+(to sync.poelzi.org__music...)
+[2014-10-28 19:18:43 CET] read: rsync ["--progress","--inplace","--perms","-e","'ssh' '-S' '.git/annex/ssh/poelzigit@sync.poelzi.org' '-o' 'ControlMaster=auto' '-o' 'ControlPersist=yes' '-T' 'poelzigit@sync.poelzi.org' 'git-annex-shell ''recvkey'' ''/music/'' ''SHA256E-s3000--da8a3336a484a171a438c99660260cc35cbd37c339dd2c18447cd025064bc00b.mood'' --uuid 35a89672-4ff5-4d9a-9bf2-cedb272bb7cb ''--'' ''remoteuuid=97a3cd71-ee6c-4437-8740-253cde0d32ae'' ''direct='' ''associatedfile=Alan Parsons Project/Eye In The Sky/.07 - Psychobabble.mood'' ''--'''","--","/exports/music/.git/annex/objects/20/Z4/SHA256E-s3000--da8a3336a484a171a438c99660260cc35cbd37c339dd2c18447cd025064bc00b.mood/SHA256E-s3000--da8a3336a484a171a438c99660260cc35cbd37c339dd2c18447cd025064bc00b.mood","dummy:"]
+rsync error: syntax or usage error (code 1) at main.c(1183) [sender=3.1.1]
+
+  rsync failed -- run git annex again to resume file transfer
+failed
+copy Alessandro Scarlatti/Motets - Gérard Lesne, Veronique Gens (1993) [300]/06 - Infirmata, Vulnerata - VI Semper Gratus.ogg copy Alessandro Scarlatti/Motets - Gérard Lesne, Veronique Gens (1993) [300]/06 - Infirmata, Vulnerata - VI Semper Gratus.ogg (checking sync.poelzi.org__music...) [2014-10-28 19:18:48 CET] call: ssh ["-S",".git/annex/ssh/poelzigit@sync.poelzi.org","-o","ControlMaster=auto","-o","ControlPersist=yes","-T","poelzigit@sync.poelzi.org","git-annex-shell 'inannex' '/music/' 'SHA256E-s3847396--05c5498f08c727645ba84270cb8d82da69a3c9bede35520aa3128b938d003a3d.ogg' --uuid 35a89672-4ff5-4d9a-9bf2-cedb272bb7cb"]
+(to sync.poelzi.org__music...)
+[2014-10-28 19:18:48 CET] read: rsync ["--progress","--inplace","--perms","-e","'ssh' '-S' '.git/annex/ssh/poelzigit@sync.poelzi.org' '-o' 'ControlMaster=auto' '-o' 'ControlPersist=yes' '-T' 'poelzigit@sync.poelzi.org' 'git-annex-shell ''recvkey'' ''/music/'' ''SHA256E-s3847396--05c5498f08c727645ba84270cb8d82da69a3c9bede35520aa3128b938d003a3d.ogg'' --uuid 35a89672-4ff5-4d9a-9bf2-cedb272bb7cb ''--'' ''remoteuuid=97a3cd71-ee6c-4437-8740-253cde0d32ae'' ''direct='' ''associatedfile=Alessandro Scarlatti/Motets - G\233rard Lesne, Veronique Gens (1993) [300]/06 - Infirmata, Vulnerata - VI Semper Gratus.ogg'' ''--'''","--","/exports/music/.git/annex/objects/XJ/f9/SHA256E-s3847396--05c5498f08c727645ba84270cb8d82da69a3c9bede35520aa3128b938d003a3d.ogg/SHA256E-s3847396--05c5498f08c727645ba84270cb8d82da69a3c9bede35520aa3128b938d003a3d.ogg","dummy:"]
+rsync error: syntax or usage error (code 1) at main.c(1183) [sender=3.1.1]
+
+  rsync failed -- run git annex again to resume file transfer
+failed
+^C
+
+
+Calling this through python gives:
+
+In [5]: subprocess.call(["rsync", "--debug=all", "--progress","--inplace","--perms","-e","'ssh' '-S' '.git/annex/ssh/poelzigit@sync.poelzi.org' '-o' 'ControlMaster=auto' '-o' 'ControlPersist=yes' '-T' 'poelzigit@sync.poelzi.org' 'git-annex-shell ''recvkey'' ''/music/'' ''SHA256E-s3000--da8a3336a484a171a438c99660260cc35cbd37c339dd2c18447cd025064bc00b.mood'' --uuid 35a89672-4ff5-4d9a-9bf2-cedb272bb7cb ''--'' ''remoteuuid=97a3cd71-ee6c-4437-8740-253cde0d32ae'' ''direct='' ''associatedfile=Alan Parsons Project/Eye In The Sky/.07 - Psychobabble.mood'' ''--'''","--","/exports/music/.git/annex/objects/20/Z4/SHA256E-s3000--da8a3336a484a171a438c99660260cc35cbd37c339dd2c18447cd025064bc00b.mood/SHA256E-s3000--da8a3336a484a171a438c99660260cc35cbd37c339dd2c18447cd025064bc00b.mood","dummy:"])
+opening connection using: ssh -S .git/annex/ssh/poelzigit@sync.poelzi.org -o ControlMaster=auto -o ControlPersist=yes -T poelzigit@sync.poelzi.org "git-annex-shell 'recvkey' '/music/' 'SHA256E-s3000--da8a3336a484a171a438c99660260cc35cbd37c339dd2c18447cd025064bc00b.mood' --uuid 35a89672-4ff5-4d9a-9bf2-cedb272bb7cb '--' 'remoteuuid=97a3cd71-ee6c-4437-8740-253cde0d32ae' 'direct=' 'associatedfile=Alan Parsons Project/Eye In The Sky/.07 - Psychobabble.mood' '--'" dummy rsync --server -pe.Lsfx --log-format=X --debug=ALL --inplace . .  (19 args)
+(Client) Protocol versions: remote=31, negotiated=31
+[sender] change_dir(/exports/music/.git/annex/objects/20/Z4/SHA256E-s3000--da8a3336a484a171a438c99660260cc35cbd37c339dd2c18447cd025064bc00b.mood)
+send_files starting
+send_files phase=1
+send_files phase=2
+send files finished
+total: matches=0  hash_hits=0  false_alarms=0 data=0
+rsync error: syntax or usage error (code 1) at main.c(1183) [sender=3.1.1]
+[sender] _exit_cleanup(code=1, file=main.c, line=1183): about to call exit(1)
+
+
+"""]]
diff --git a/doc/bugs/rsync_fails_with_sync_error__58___syntax_or_usage_error_/comment_1_464d733de18f6ca438ebd84e88b8cee2._comment b/doc/bugs/rsync_fails_with_sync_error__58___syntax_or_usage_error_/comment_1_464d733de18f6ca438ebd84e88b8cee2._comment
new file mode 100644
index 0000000000..672a3a5600
--- /dev/null
+++ b/doc/bugs/rsync_fails_with_sync_error__58___syntax_or_usage_error_/comment_1_464d733de18f6ca438ebd84e88b8cee2._comment
@@ -0,0 +1,29 @@
+[[!comment format=mdwn
+ username="joey"
+ subject="""comment 1"""
+ date="2014-10-31T20:45:41Z"
+ content="""
+This is a rsync protocol level error; one side is sending something
+that the other side fails to deal with. We can see in your log that the
+two rsync are communicating successfully over the ssh connection
+at first.
+
+This could be something not clean about your ssh connection, or some incompatibility
+in the versions of rsync or git-annex between the client and the server.
+It probably wouldn't hurt to make sure client and server have the same rsync
+version, and perhaps upgrade them both to the newest git-annex in case this
+problem is somehow fixed there.
+
+Then, seems to me that the next step is to get git-annex-shell out of the
+picture and see if you can still reproduce the problem. You can find the rsync
+command that git-annex-shell runs by passing --debug to it. The just replace
+the git-annex-shell command in your python code with the rsync command it runs.
+
+Here's a shell command line I came up with by doing that. It will have
+different paths for your repo, and localhost will need to be changed to your
+server's name.
+
+
    +rsync --progress --inplace --perms --debug=all -e 'ssh  -4 -T localhost "rsync --server -t --inplace -e.Lsf . //home/joey/annex/.git/annex/tmp/SHA256E-s30--bdce956a335681853344fce6f1f940a5c8b7061007398661a3b14f2037843744" dummy rsync --server -pe.Lsfx --log-format=X --debug=ALL --inplace . .'  /tmp/annex/.git/annex/objects/Wx/Mf/SHA256E-s30--bdce956a335681853344fce6f1f940a5c8b7061007398661a3b14f2037843744/SHA256E-s30--bdce956a335681853344fce6f1f940a5c8b7061007398661a3b14f2037843744 dummy:
+
    
+"""]]
diff --git a/doc/bugs/s3_InternalIOException__63__.mdwn b/doc/bugs/s3_InternalIOException__63__.mdwn
new file mode 100644
index 0000000000..5a5674f663
--- /dev/null
+++ b/doc/bugs/s3_InternalIOException__63__.mdwn
@@ -0,0 +1,28 @@
+### Please describe the problem.
+
+I got the following errors from the assistant, in `.git/annex/daemon.log`:
+
+    InternalIOException send: resource vanished (Broken pipe)
+
+Then subsequent transfers seem to fail with:
+
+    InternalIOException send: resource vanished (Connection reset by peer)
+
+Workaround: restart the assistant.
+
+### What steps will reproduce the problem?
+
+It's unclear. The assistant is trying to sync a lot of stuff to S3 right now, as files are regularly added into the repository and the assistant migrates them all there. The repository is setup as a "source" repository to make sure it doesn't keep files and send them all to s3.
+
+### What version of git-annex are you using? On what operating system?
+
+standalone tarball `5.20150508-g883d57f` on Ubuntu Precise.
+
+### Please provide any additional information below.
+
+here are the two latest log files, but all previous logfiles available are similar...
+
+http://paste.ubuntu.com/11721025/
+http://paste.ubuntu.com/11721028/
+
+-- [[anarcat]]
diff --git a/doc/bugs/s3_InternalIOException__63__/comment_1_42bd2d1e364203f1b34efcc34ac795b8._comment b/doc/bugs/s3_InternalIOException__63__/comment_1_42bd2d1e364203f1b34efcc34ac795b8._comment
new file mode 100644
index 0000000000..544c8998f8
--- /dev/null
+++ b/doc/bugs/s3_InternalIOException__63__/comment_1_42bd2d1e364203f1b34efcc34ac795b8._comment
@@ -0,0 +1,14 @@
+[[!comment format=mdwn
+ username="joey"
+ subject="""comment 1"""
+ date="2015-06-15T18:57:07Z"
+ content="""
+Are you sure that it was subsequent transfers of other keys that failed,
+and not subsequent transfers of chunks of the same key?
+
+The latter seems more likely. There is a cached connection to S3 that is
+reused when uploading chunks of a key. But, each transfer of a key
+currently runs using its own individual connection. Difficulties with
+connections that close or are left running too long are why the S3 remote
+doesn't yet reuse connections.
+"""]]
diff --git a/doc/bugs/s3_InternalIOException__63__/comment_2_994dd3ebcf7eaacb0b9e06f1bc14a2d4._comment b/doc/bugs/s3_InternalIOException__63__/comment_2_994dd3ebcf7eaacb0b9e06f1bc14a2d4._comment
new file mode 100644
index 0000000000..4d3172ea30
--- /dev/null
+++ b/doc/bugs/s3_InternalIOException__63__/comment_2_994dd3ebcf7eaacb0b9e06f1bc14a2d4._comment
@@ -0,0 +1,7 @@
+[[!comment format=mdwn
+ username="anarcat"
+ subject="comment 2"
+ date="2015-06-15T19:15:12Z"
+ content="""
+i have no idea, really. i probably did *not* disable chunking explicitely on the repo, if that's any help..
+"""]]
diff --git a/doc/bugs/s3_InternalIOException__63__/comment_3_a69927ec705efa31aacb5941bf8d8f9d._comment b/doc/bugs/s3_InternalIOException__63__/comment_3_a69927ec705efa31aacb5941bf8d8f9d._comment
new file mode 100644
index 0000000000..c32621d6e3
--- /dev/null
+++ b/doc/bugs/s3_InternalIOException__63__/comment_3_a69927ec705efa31aacb5941bf8d8f9d._comment
@@ -0,0 +1,14 @@
+[[!comment format=mdwn
+ username="anarcat"
+ subject="comment 3"
+ date="2015-06-15T20:02:56Z"
+ content="""
+okay, after disabling chunking, it still doesn't work.
+
+it seems that it's completely stuck: i haven't seen the assistant transfer any new files yet. using `git annex move` seems to work:
+
+
    +www-data@ip-10-87-135-88:/persistent/media$ git annex move --to s3
+move video/mp4_sd/-_cineastas_indigenas_ntsc_-_eng_0.mov.mp4 (checking s3...) ok
+
    
+"""]]
diff --git a/doc/bugs/s3_InternalIOException__63__/comment_4_ee791ad24d5d2c0ad4f82ecf6fc434a9._comment b/doc/bugs/s3_InternalIOException__63__/comment_4_ee791ad24d5d2c0ad4f82ecf6fc434a9._comment
new file mode 100644
index 0000000000..fd65fa8f81
--- /dev/null
+++ b/doc/bugs/s3_InternalIOException__63__/comment_4_ee791ad24d5d2c0ad4f82ecf6fc434a9._comment
@@ -0,0 +1,28 @@
+[[!comment format=mdwn
+ username="anarcat"
+ subject="comment 4"
+ date="2015-06-15T20:05:13Z"
+ content="""
+nevermind that: `move` doesn't actually work either, it just skipped files that seemed to have already been transfered. the remaining files are still untransferable:
+
+
    +www-data@ip-10-87-135-88:/persistent/media$ git annex move --to s3
+move video/original/a_gente_luta_-_eng_0.mov (checking s3...) (to s3...)
+0%            0.0 B/s 0s
+  InternalIOException send: resource vanished (Broken pipe)
+failed
+move video/original/a_gente_luta_-_esp_0.mov (checking s3...) (to s3...)
+0%       255.9KB/s 9h23m
+  InternalIOException send: resource vanished (Broken pipe)
+failed
+move video/original/kinja_iakaha_-_dvcam_en.mov (checking s3...) (to s3...)
+0%            0.0 B/s 0s
+  InternalIOException send: resource vanished (Broken pipe)
+failed
+move video/original/quartet_for_deafblind_h264kbs18000_24.mov (checking s3...) (to s3...)
+0%            0.0 B/s 0s
+  InternalIOException send: resource vanished (Broken pipe)
+failed
+git-annex: move: 4 failed
+
    
+"""]]
diff --git a/doc/bugs/s3_InternalIOException__63__/comment_5_134d1d0ad9f59a2b7498d1ed335cf91a._comment b/doc/bugs/s3_InternalIOException__63__/comment_5_134d1d0ad9f59a2b7498d1ed335cf91a._comment
new file mode 100644
index 0000000000..59f6c1bd09
--- /dev/null
+++ b/doc/bugs/s3_InternalIOException__63__/comment_5_134d1d0ad9f59a2b7498d1ed335cf91a._comment
@@ -0,0 +1,9 @@
+[[!comment format=mdwn
+ username="anarcat"
+ subject="comment 5"
+ date="2015-07-23T19:44:10Z"
+ content="""
+any update of this?
+
+i still can't upload those files, while *other* files get uploaded to S3 fine. really strange. how can i reset the state of this?
+"""]]
diff --git a/doc/bugs/schedule.log_added_to_annex__63_____58____47____47____47__.mdwn b/doc/bugs/schedule.log_added_to_annex__63_____58____47____47____47__.mdwn
new file mode 100644
index 0000000000..9982135827
--- /dev/null
+++ b/doc/bugs/schedule.log_added_to_annex__63_____58____47____47____47__.mdwn
@@ -0,0 +1,55 @@
+### Please describe the problem.
+
+Look at this diff:
+
+```
+diff --git a/schedule.log b/schedule.log
+index 29c727f8c1..2f6d2f0858 100644
+--- a/schedule.log
++++ b/schedule.log
+@@ -1,19 +1 @@
+-0e6c5057-a323-4bec-b6d2-98bae312e7cd  timestamp=1518571009.042438126s
+-0e6c5057-a323-4bec-b6d2-98bae312e7cd fsck self 15m every day at any time timestamp=1503189066.20635041s
+-179b6e7f-0b2f-43b9-a95d-39df5b52d2fc  timestamp=1518565854.651788575s
+-179b6e7f-0b2f-43b9-a95d-39df5b52d2fc  timestamp=1518565869.769672252s
+-179b6e7f-0b2f-43b9-a95d-39df5b52d2fc fsck self 15m every day at any time timestamp=1503191937.325891749s
+-668bdeb3-a3e2-4a9f-ae1c-bae1880c62b8  timestamp=1516100332.848743619s
+-668bdeb3-a3e2-4a9f-ae1c-bae1880c62b8  timestamp=1518305729.698369069s
+-668bdeb3-a3e2-4a9f-ae1c-bae1880c62b8  timestamp=1518565670.560184121s
+-668bdeb3-a3e2-4a9f-ae1c-bae1880c62b8  timestamp=1518868059.554534192s
+-668bdeb3-a3e2-4a9f-ae1c-bae1880c62b8 fsck self 15m every day at any time timestamp=1504318394.78758178s
+-7f0bbee7-826f-4021-a774-0569ff58ed54 fsck self 10m every day at any time timestamp=1514213643.848668933s
+-7f0bbee7-826f-4021-a774-0569ff58ed54 fsck self 15m every day at any time timestamp=1504177427.472846327s
+-7f0bbee7-826f-4021-a774-0569ff58ed54 fsck self 1h every day at any time timestamp=1496609751.566017331s
+-83ffd5ab-40b3-4950-b437-87339242be57 fsck self 15m every day at any time timestamp=1503175558.988571178s
+-ba3a1e0a-9191-43b4-a324-1fee91a9bb8f  timestamp=1503184784.521086496s
+-ba3a1e0a-9191-43b4-a324-1fee91a9bb8f  timestamp=1518566966.391960267s
+-ba3a1e0a-9191-43b4-a324-1fee91a9bb8f fsck self 15m every day at any time timestamp=1503183823.810567346s
+-ba3a1e0a-9191-43b4-a324-1fee91a9bb8f fsck self 15m every day at any time timestamp=1503185213.589823679s
+-ba3a1e0a-9191-43b4-a324-1fee91a9bb8f fsck self 5m every day at any time timestamp=1514056713.473067585s
++/annex/objects/SHA256E-s1676--4e76f9f5e9c5eafe53367a6c493d7618366bcfea6ac8a859fcbaba55ed31f349.log
+```
+
+### What steps will reproduce the problem?
+
+Absolutely no idea.
+
+
+### What version of git-annex are you using? On what operating system?
+
+```
+git-annex version: 6.20170818
+operating system: linux x86_64
+```
+
+
+### Please provide any additional information below.
+
+None
+
+### Have you had any luck using git-annex before? (Sometimes we get tired of reading bug reports all day and a lil' positive end note does wonders)
+
+Considering my two other issues »WHICH LOSE DATA«, I can only conclude it’s too complicated, cf.
+
+* https://git-annex.branchable.com/bugs/Data_loss_when_copying_files_with_running_assistant/
+* https://git-annex.branchable.com/bugs/Infinite_loop_when_synchronizing_between_many_machines/
diff --git a/doc/bugs/security_hole_private_data_exposure_via_addurl.mdwn b/doc/bugs/security_hole_private_data_exposure_via_addurl.mdwn
new file mode 100644
index 0000000000..1452415da3
--- /dev/null
+++ b/doc/bugs/security_hole_private_data_exposure_via_addurl.mdwn
@@ -0,0 +1,214 @@
+CVE-2018-10857
+
+This is a security hole that allows exposure of
+private data in files located outside the git-annex repository.
+
+The attacker needs to have control over one of the remotes of the git-annex
+repository. For example, they may provide a public git-annex repository
+that the victim clones. Or the victim may have paired repositories with
+them. Or, equivilantly, the attacker could have read access to the victim's
+git-annex repository (eg on a server somewhere), and some channel to get
+commits into it (eg a pull requests).
+
+The attacker does `git-annex addurl --relaxed file:///etc/passwd` and commits
+this to the repository in some out of the way place. Then they wait for the
+victim to pull the change.
+
+The easiest exploit is when the victim is running the assistant, or is
+periodically doing `git annex sync --content`. The victim may also perform
+the equivilant actions manually, not noticing they're operating on the
+file. 
+
+In either case, git-annex gets the content of the annexed file, following
+the file:// url. Then git-annex transfers the content back to the
+attacker's repository.
+
+It may also be possible to exploit scp:// sftp:// smb:// etc urls to get at
+files on other computers that the user has access to as well as localhost.
+I was not able to get curl to download from scp:// or sftp:// on debian
+(unstable) but there may be configurations that allow it.
+
+If the url is attached to a key using a cryptographic hash, then the
+attacker would need to already know at least the hash of the content
+to exploit this.
+
+Sending that content back to them could be considered not a security
+hole. Except, I can guess what content some file on your system might have,
+and use this attack as an oracle to determine if I guessed right, and work
+toward penetrating your system using that information.
+
+So, best to not treat addurl with a hash any differently than 
+--relaxed and --fast when addressing this hole.
+
+----
+
+The fix must involve locking down the set of allowed in url schemes.
+Better to have a whitelist than a blacklist. http and https seems like the
+right default.
+
+Some users do rely on file:// urls, and this is fine in some use cases,
+eg when you're not merging changes from anyone else. 
+
+So this probably needs to be a git config of allowed url schemes,
+with an appropriatly scary name, like `annex.security.allowed-url-schemes`.
+
+Redirects from one url scheme to another could be usd to bypass such a
+whitelist. curl's `--proto` also affects redirects. http-conduit 
+is limited to only http and will probably remain that way.
+
+> done in [[!commit 28720c795ff57a55b48e56d15f9b6bcb977f48d9]] --[[Joey]]
+
+----
+
+The same kind of attack can be used to see the content of
+localhost urls and other non-globally-available urls.
+
+Redirects and DNS rebinding attacks mean that checking the IP address
+of the hostname in the url is not good enough. It needs to check the IP
+address that is actually connected to, for each http connection made,
+including redirects.
+
+There will need to be a config to relax checks, like
+with an appropriatly scary name, like
+`annex.security.allowed-http-addresses`. Users will want to enable
+support for specific subnets, or perhaps allow all addresses.
+
+When git-annex is configured to use curl, there seems to be no way
+to prevent curl from following urls that redirect to localhost, other than
+disabling redirections. And unless git-annex also pre-resolves the IP
+address and rewrites it into the url passed to curl, DNS rebinding attacks
+would still be possible. Also, one of the remaining reasons people enable
+curl is to use a netrc file with passwords, and the content of
+urls on those sites could also be exposed by this attack. So, it seems curl
+should not be enabled by default and should have a big security warning if
+it's supported at all. Probably ought to only enable it 
+when `annex.security.allowed-http-addresses=all`
+
+http-client does not have hooks that can be used to find out what IP
+address it's connecting to in a secure enough way.
+See 
+
+Seems that building my own http Manager is the only way to go. By building
+my own, I can do the IP address checks inside it when it's setting up
+connections. And, the same manager can be passed to the S3 and WebDav libraries.
+(The url scheme checks could also be moved into that Manager, to prevent
+S3 redirect to file: url scenarios..)
+
+> restricted http manager done and used in
+> [[!commit b54b2cdc0ef1373fc200c0d28fded3c04fd57212]];
+> curl also disabled by default
+
+http proxies are another problem. They could be on the local network,
+or even on localhost, and http-client does not provide a way to force
+a http proxy to connect to an particular IP address (is that even possible?)
+May have to disable use of http proxies unless
+`annex.security.allowed-http-addresses=all`
+Or better, find what http proxy is enabled and prevent using it if it's on
+an IP not allowed there.
+
+> done in [[!commit cc08135e659d3ca9ea157246433d8aa90de3baf7]]
+
+----
+
+The external special remote interface is another way to exploit this.
+Since it bypasses git-annex's usual url download code, whatever fixes are
+put in place there won't help external special remotes.
+
+External special remotes that use GETURLS, typically in conjunction with 
+CLAIMURL and CHECKURL, and then themselves download the content of urls
+in response to a TRANSFER RETRIEVE will have the same problems as
+git-annex's url downloading. 
+
+An external special remote might also make a simple http request to a
+key/value API to download a key, and follow a redirect to file:///
+or http://localhost/.
+
+If the key uses a cryptographic hash, git-annex verifies the content. 
+But, the attacker could have committed a key that doesn't
+use a hash. Also, the attacker could use the hash check as an oracle,
+to guess at the content of files.
+
+If the external special remote is encrypted, the http content is passed
+though gpg. So, whatever location an attacker redirects it to would also
+have to be encrypted. gpg is not told what encryption key the content is
+expected to be encrypted with. (Could it be told somehow for hybrid and
+shared encryption which key to use? pubkey encryption of course uses
+the regular gpg public key). 
+
+So if the attacker knows a file that the user has encrypted with any of
+their gpg keys, they can provide that file, and hope it will be decrypted.
+Note that this does not need a redirect to a local file or web server; the
+attacker can make their web server serve up a gpg encrypted file.
+This is a separate vulnerability and was assigned CVE-2018-10859.
+
+So, content downloaded from encrypted special remotes (both internal and
+external) must be rejected unless it can be verified with a hash. Then
+content using WORM and URL keys would not be able to be downloaded from
+them. Might as well also require a hash check for non-encrypted external
+special remotes, to block the redirection attack. There could be a config
+setting to say that the git-annex repository is not being shared with
+untrusted third parties, and relax that check.
+
+> done in [[!commit b657242f5d946efae4cc77e8aef95dd2a306cd6b]]
+
+Could also tighten down the gpg decryption to only allow decrypting with
+the provided symmetric key, as a further protection against CVE-2018-10859.
+If this can be done, then only remotes with encryption=pubkey will
+really need to reject WORM and URL keys, since encryption=shared
+and encryption=hybrid use a symetric key that's only used to encrypt data
+for that remote. Although opening those back up to WORM and URL would
+allow the remote sending other content stored on it, to get the wrong
+content decrypted. This seems unlikely to be a useful exploit in most
+cases, but perhaps not all cases, so probably best to not relax the
+rejection aven when doing this. It's still worth doing as a belt and braces
+fix.
+
+> AFAICS, gpg does not have a way to specify to decrypt with only a
+> symmetric encryption key. It could be done by running gpg in an
+> environment with an empty keyring, but gpg agent makes that difficult and
+> it would be added complexity. Decided not to do it.
+
+----
+
+Built-in special remotes that use protocols on top of http, eg S3 and WebDAV,
+don't use Utility.Url, could also be exploited, and will need to be fixed
+separately.
+
+> not affected for url schemes, because http-client only supports http,
+> not file:/
+
+> done for localhost/lan in [[!commit b54b2cdc0ef1373fc200c0d28fded3c04fd57212]]
+
+youtube-dl
+
+> already disabled file:/. Added a scheme check, but it won't block
+> redirects to other schemes. But youtube-dl goes off and does its own thing with other
+> protocols anyway, so that's fine.
+> 
+> The youtube-dl generic extractor will download media files (including 
+> videos and photos) if passed an direct url to them. It does not seem to
+> extract video etc from tags on html pages.
+
+> git-annex first checks if a web page
+> is html before pointing youtube-dl at it, to avoid using it to download
+> direct urls to media files. But that would not prevent a DNS rebinding
+> attack which made git-annex see the html page, and youtube-dl then see
+> a media file on localhost.
+> 
+> Also, the current code in htmlOnly
+> runs youtube-dl if the html page download test fails to download
+> anything.
+> 
+> Also, in the course of a download, youtube-dl could chain to other urls,
+> depending on how its extractor works. Those urls could redirect to
+> a localhost/lan web server.
+> 
+> So, youtube-dl needs to be disabled unless http address security checks
+> are turned off.
+> 
+> > done in [[!commit e62c4543c31a61186ebf2e4e0412df59fc8630c8]]
+
+
+----
+
+Both security holes are now fixed. [[done]] --[[Joey]]
diff --git a/doc/bugs/set_metadata_leaks_from_one___40__staged__41___key_to_another_during_rename_of_file.mdwn b/doc/bugs/set_metadata_leaks_from_one___40__staged__41___key_to_another_during_rename_of_file.mdwn
new file mode 100644
index 0000000000..31bf4cc041
--- /dev/null
+++ b/doc/bugs/set_metadata_leaks_from_one___40__staged__41___key_to_another_during_rename_of_file.mdwn
@@ -0,0 +1,45 @@
+### Please describe the problem.
+
+spent quite some time trying to figure out WTF I got some files not published due to tag being set...  managed to reproduce in minimal setup -- if I set metadata to a file, rename a file (mv, git add;  or even git mv), and add another file with different content but the same name -- it would obtain metadata of original key/file (unless I commit right after rename)
+
+### What steps will reproduce the problem?
+
+see below
+
+### What version of git-annex are you using? On what operating system?
+
+6.20170924+gitgd35053009-1~ndall+1
+
+### Please provide any additional information below.
+
+[[!format sh """
+$> sudo rm -rf /tmp/repo; mkdir /tmp/repo; cd /tmp/repo; git init; git annex init; echo 1 >| 1; git annex add 1; git commit -m 'added 1'; git annex metadata -s tag=value 1; git mv 1 2; git annex add 2; echo 2>1; git annex add 1; for f in 1 2; do echo "file $f"; ls -l $f; git annex metadata -g tag $f; done                                                                                                                          
+Initialized empty Git repository in /tmp/repo/.git/
+init  ok
+(recording state in git...)
+add 1 ok
+(recording state in git...)
+[master (root-commit) 750d619] added 1
+ 1 file changed, 1 insertion(+)
+ create mode 120000 1
+metadata 1 
+  lastchanged=2017-09-26@17-59-22
+  tag=value
+  tag-lastchanged=2017-09-26@17-59-22
+ok
+(recording state in git...)
+
+add 1 ok
+(recording state in git...)
+file 1
+lrwxrwxrwx 1 yoh yoh 178 Sep 26 13:59 1 -> .git/annex/objects/pX/ZJ/SHA256E-s0--e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855/SHA256E-s0--e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855
+value
+file 2
+lrwxrwxrwx 1 yoh yoh 178 Sep 26 13:59 2 -> .git/annex/objects/2W/V5/SHA256E-s2--4355a46b19d348dc2f57c046f8ef63d4538ebb936000f3c9ee954a27460dd865/SHA256E-s2--4355a46b19d348dc2f57c046f8ef63d4538ebb936000f3c9ee954a27460dd865
+value
+
+"""]]
+
+you can see in above taht 1 and 2 have different content/keys, but they both acquire the same tag=value.  If I commit 2 after it being renamed from 1, it is ok.
+
+[[!meta author=yoh]]
diff --git a/doc/bugs/set_metadata_leaks_from_one___40__staged__41___key_to_another_during_rename_of_file/comment_1_cd7043b66977b2a668bbd52f4d8e31ff._comment b/doc/bugs/set_metadata_leaks_from_one___40__staged__41___key_to_another_during_rename_of_file/comment_1_cd7043b66977b2a668bbd52f4d8e31ff._comment
new file mode 100644
index 0000000000..b4eef0dab8
--- /dev/null
+++ b/doc/bugs/set_metadata_leaks_from_one___40__staged__41___key_to_another_during_rename_of_file/comment_1_cd7043b66977b2a668bbd52f4d8e31ff._comment
@@ -0,0 +1,21 @@
+[[!comment format=mdwn
+ username="joey"
+ subject="""comment 1"""
+ date="2017-09-26T18:17:58Z"
+ content="""
+This is caused by an intentional feature. When `git annex add` is run
+on a modified file, the old metadata for the file (as committed to HEAD)
+is copied over. This prevents metadata being lost when modifying a file.
+
+The same would happen if the file were deleted and then new content added
+with the same filename. It seems like different things should be done in
+these cases, but the same thing can end up staged in several different
+cases, so the cases can't be distinguished.
+
+Committing the rename or deletion before adding a file back with the same
+name allows distinguishing from a modification of the file, and so that
+avoids the problem.
+
+I agree this led to confusing behavior here, but I'll bet it's also
+prevented confusing loss of metadata for people when editing a file..
+"""]]
diff --git a/doc/bugs/set_metadata_leaks_from_one___40__staged__41___key_to_another_during_rename_of_file/comment_2_e5b43f5733181fba9f79ae0035b354ef._comment b/doc/bugs/set_metadata_leaks_from_one___40__staged__41___key_to_another_during_rename_of_file/comment_2_e5b43f5733181fba9f79ae0035b354ef._comment
new file mode 100644
index 0000000000..7916814206
--- /dev/null
+++ b/doc/bugs/set_metadata_leaks_from_one___40__staged__41___key_to_another_during_rename_of_file/comment_2_e5b43f5733181fba9f79ae0035b354ef._comment
@@ -0,0 +1,10 @@
+[[!comment format=mdwn
+ username="joey"
+ subject="""comment 2"""
+ date="2017-09-26T18:36:48Z"
+ content="""
+Other than the always-popular "make it configurable",
+I wonder if it would suffice to simply output a note when copying metadata
+from the (presumed) old version of the file? Then there would be no
+confusion about why the metadata got set.
+"""]]
diff --git a/doc/bugs/set_metadata_leaks_from_one___40__staged__41___key_to_another_during_rename_of_file/comment_3_10d6135ad4b0c9706ff7b932323118d3._comment b/doc/bugs/set_metadata_leaks_from_one___40__staged__41___key_to_another_during_rename_of_file/comment_3_10d6135ad4b0c9706ff7b932323118d3._comment
new file mode 100644
index 0000000000..26d2ff01f3
--- /dev/null
+++ b/doc/bugs/set_metadata_leaks_from_one___40__staged__41___key_to_another_during_rename_of_file/comment_3_10d6135ad4b0c9706ff7b932323118d3._comment
@@ -0,0 +1,21 @@
+[[!comment format=mdwn
+ username="joey"
+ subject="""comment 3"""
+ date="2017-09-26T19:16:29Z"
+ content="""
+Hmm, if the file was not already in the index, that could be taken to
+indicate it was deleted/moved and replaced, rather than being modified,
+and so don't copy the metadata.
+
+But that would make these two sequences have different behavior:
+
+	git rm file; echo foo > file; git annex add file
+
+	echo foo > file; git annex add file
+
+As well as these two sequences:
+
+	git mv file other; echo foo > file; git annex add file
+
+	cp file other; echo foo > file; git annex add file other
+"""]]
diff --git a/doc/bugs/set_metadata_leaks_from_one___40__staged__41___key_to_another_during_rename_of_file/comment_4_fda77062452ac619a23298007bc0148a._comment b/doc/bugs/set_metadata_leaks_from_one___40__staged__41___key_to_another_during_rename_of_file/comment_4_fda77062452ac619a23298007bc0148a._comment
new file mode 100644
index 0000000000..7e79c4ca16
--- /dev/null
+++ b/doc/bugs/set_metadata_leaks_from_one___40__staged__41___key_to_another_during_rename_of_file/comment_4_fda77062452ac619a23298007bc0148a._comment
@@ -0,0 +1,10 @@
+[[!comment format=mdwn
+ username="yarikoptic"
+ avatar="http://cdn.libravatar.org/avatar/f11e9c84cb18d26a1748c33b48c924b4"
+ subject="comment 4"
+ date="2017-09-27T05:30:52Z"
+ content="""
+Thanks for the analysis/explaining.  My POV, I somewhat got used to the notion \"git annex cares about content/keys, and files are mere pointers\".  That is easy to explain, in particular why two files pointing to the same key would have the same meta-data. So I assumed that no meta-data would be copied over to the new content/key.  But I now do see how retaining the meta-data could be useful, e.g. if I unlock/modify/add a file where meta-data should be carried over (e.g. for music could be all the author/album/etc)... But if file was not unlocked, then may be such \"copying\" shouldn't happen?  not sure  what to suggest otherwise :-/.
+
+In my specific case in one case (newly assigned meta-data) I could simply workaround by first renaming and then assigning meta-data to the renamed file, but in the other (if meta-data is already assigned for the key), I would need to explicitly reset it then but it seems can't do it without committing since that \"copy metadata for the filename\" happens upon committing... so not sure on how to workaround without explicitly removing it from those files after committing... heh
+"""]]
diff --git a/doc/bugs/set_metadata_leaks_from_one___40__staged__41___key_to_another_during_rename_of_file/comment_5_fa6d5d0afdc00824da0e04d9bf23d6f4._comment b/doc/bugs/set_metadata_leaks_from_one___40__staged__41___key_to_another_during_rename_of_file/comment_5_fa6d5d0afdc00824da0e04d9bf23d6f4._comment
new file mode 100644
index 0000000000..2f7c8479ef
--- /dev/null
+++ b/doc/bugs/set_metadata_leaks_from_one___40__staged__41___key_to_another_during_rename_of_file/comment_5_fa6d5d0afdc00824da0e04d9bf23d6f4._comment
@@ -0,0 +1,20 @@
+[[!comment format=mdwn
+ username="joey"
+ subject="""comment 5"""
+ date="2017-09-28T16:03:28Z"
+ content="""
+Files are not unlocked before modifying in direct mode, and may be
+unlocked all the time in v6 mode. Also, in indirect mode it's of course
+fine to overwrite the symlink with a new version of a file. So detecting 
+if it's been unlocked doesn't seem to help with this.
+
+It may be that there are different sorts of metadata, some of which should
+be inherited by new versions of a file, and others not. If there was a way
+to tell git-annex which metadata was which, it could do the right thing.
+But it feels like stacking complications. Particularly since there might be
+some tags that should be inherited and others not, and tags are values..
+
+In the meantime, I've added the warning when it copies metadata. 
+I also added `git annex metadata --remove-all`, which the warning
+suggests running if you don't want the copied metadata.
+"""]]
diff --git a/doc/bugs/sharedpubkey_is_using_a_different_filename_encryption_method_than_shared.mdwn b/doc/bugs/sharedpubkey_is_using_a_different_filename_encryption_method_than_shared.mdwn
new file mode 100644
index 0000000000..24f535d238
--- /dev/null
+++ b/doc/bugs/sharedpubkey_is_using_a_different_filename_encryption_method_than_shared.mdwn
@@ -0,0 +1,67 @@
+### Please describe the problem.
+I am looking into [decrypting files in special remotes without git-annex](https://git-annex.branchable.com/forum/Shared_pubkeys__58___decrypting_files_in_special_remotes_without_git-annex/) also with comments in [disaster_recovery_with_an_encrypted_special_remote](http://git-annex.branchable.com/forum/Future_proofing___47___disaster_recovery_with_an_encrypted_special_remote/). User is trying to figure out filename encryption scheme on special remotes for sharedpubkey encryption scheme.
+
+`sharedpubkey` is using a different filename encryption method than `shared` on special remotes but [encryption](https://git-annex.branchable.com/encryption/) page implies they should be using the same method: “regular public key encryption with shared filename encryption (encryption=sharedpubkey)”.
+
+### What steps will reproduce the problem?
+	andrew@bumblebee /tmp$ mkdir repo1
+	andrew@bumblebee /tmp$ cd repo1/
+	andrew@bumblebee /tmp/repo1$ git init
+	Initialized empty Git repository in /private/tmp/repo1/.git/
+	andrew@bumblebee /tmp/repo1$ git annex init
+	init  ok
+	(recording state in git...)
+	andrew@bumblebee /tmp/repo1$ echo "hello a" > a.txt
+	andrew@bumblebee /tmp/repo1$ git annex add a.txt 
+	add a.txt ok
+	(recording state in git...)
+	andrew@bumblebee /tmp/repo1$ mkdir /tmp/remote1
+	andrew@bumblebee /tmp/repo1$ git annex initremote remote1 type=directory directory=/tmp/remote1 encryption=sharedpubkey keyid=0C5C0618
+	initremote remote1 (encryption setup) (to gpg keys: 222365310C5C0618) ok
+	(recording state in git...)
+	andrew@bumblebee /tmp/repo1$ git annex copy a.txt --to=remote1
+	copy a.txt (to remote1...) 
+	ok                              
+	(recording state in git...)
+	andrew@bumblebee /tmp/repo1$ mkdir /tmp/remote2
+	andrew@bumblebee /tmp/repo1$ git annex initremote remote2 type=directory directory=/tmp/remote2 encryption=shared
+	initremote remote2 (encryption setup) (encryption key stored in git repository) ok
+	(recording state in git...)
+	andrew@bumblebee /tmp/repo1$ git annex copy a.txt --to=remote2
+	copy a.txt (to remote2...) 
+	ok                              
+	(recording state in git...)
+	andrew@bumblebee /tmp/repo1$ ls -la /tmp/remote1/
+	432/ tmp/ 
+	andrew@bumblebee /tmp/repo1$ ls -la /tmp/remote1/432/d7f/GPGHMACSHA1--6e830539e4aac12d435b2abc1f693e0dfacf5e89/GPGHMACSHA1--6e830539e4aac12d435b2abc1f693e0dfacf5e89 
+	-r--r--r--  1 andrew  wheel  340 Jul 19 13:40 /tmp/remote1/432/d7f/GPGHMACSHA1--6e830539e4aac12d435b2abc1f693e0dfacf5e89/GPGHMACSHA1--6e830539e4aac12d435b2abc1f693e0dfacf5e89
+	andrew@bumblebee /tmp/repo1$ ls -la /tmp/remote2/
+	411/ tmp/ 
+	andrew@bumblebee /tmp/repo1$ ls -la /tmp/remote2/411/ad6/GPGHMACSHA1--0332a66cded87db8ec8427280c171133c958754f/GPGHMACSHA1--0332a66cded87db8ec8427280c171133c958754f 
+	-r--r--r--  1 andrew  wheel  78 Jul 19 13:40 /tmp/remote2/411/ad6/GPGHMACSHA1--0332a66cded87db8ec8427280c171133c958754f/GPGHMACSHA1--0332a66cded87db8ec8427280c171133c958754f
+	andrew@bumblebee /tmp/repo1$ ls -la a.txt 
+	lrwxr-xr-x  1 andrew  wheel  186 Jul 19 13:37 a.txt -> .git/annex/objects/8k/z6/SHA256E-s8--448a19594af45f888493a4cb8b6e12ed42fc773dab6db35a299370ebfc805270.txt/SHA256E-s8--448a19594af45f888493a4cb8b6e12ed42fc773dab6db35a299370ebfc805270.txt
+	andrew@bumblebee /tmp/repo1$ echo -n "SHA256E-s8--448a19594af45f888493a4cb8b6e12ed42fc773dab6db35a299370ebfc805270.txt" | openssl dgst -SHA1 -hmac "$(git show git-annex:remote.log | grep 'name='"remote2 " | sed -E 's/.*cipher=([^ ]+) .*/\1/' | base64 -D | tr -d '\n' | head  -c 256)"
+	0332a66cded87db8ec8427280c171133c958754f
+	andrew@bumblebee /tmp/repo1$ echo -n "SHA256E-s8--448a19594af45f888493a4cb8b6e12ed42fc773dab6db35a299370ebfc805270.txt" | openssl dgst -SHA1 -hmac "$(git show git-annex:remote.log | grep 'name='"remote1 " | sed -E 's/.*cipher=([^ ]+) .*/\1/' | base64 -D | tr -d '\n' | head  -c 256)"
+	bb3b1f4b27164c2edc81da32c43c64e8a4be75d8
+	andrew@bumblebee /tmp/repo1$ 
+
+### What version of git-annex are you using? On what operating system?
+	git-annex version: 6.20180626-gdf91a5cff
+	build flags: Assistant Webapp Pairing S3(multipartupload)(storageclasses) WebDAV FsEvents ConcurrentOutput TorrentParser MagicMime Feeds Testsuite
+	dependency versions: aws-0.17.1 bloomfilter-2.0.1.0 cryptonite-0.23 DAV-1.3.1 feed-0.3.12.0 ghc-8.0.2 http-client-0.5.7.0 persistent-sqlite-2.6.2 torrent-10000.1.1 uuid-1.3.13 yesod-1.4.5
+	key/value backends: SHA256E SHA256 SHA512E SHA512 SHA224E SHA224 SHA384E SHA384 SHA3_256E SHA3_256 SHA3_512E SHA3_512 SHA3_224E SHA3_224 SHA3_384E SHA3_384 SKEIN256E SKEIN256 SKEIN512E SKEIN512 BLAKE2B256E BLAKE2B256 BLAKE2B512E BLAKE2B512 BLAKE2B160E BLAKE2B160 BLAKE2B224E BLAKE2B224 BLAKE2B384E BLAKE2B384 BLAKE2S256E BLAKE2S256 BLAKE2S160E BLAKE2S160 BLAKE2S224E BLAKE2S224 BLAKE2SP256E BLAKE2SP256 BLAKE2SP224E BLAKE2SP224 SHA1E SHA1 MD5E MD5 WORM URL
+	remote types: git gcrypt p2p S3 bup directory rsync web bittorrent webdav adb tahoe glacier ddar hook external
+	operating system: darwin x86_64
+	supported repository versions: 3 5 6
+	upgrade supported from repository versions: 0 1 2 3 4 5
+	local repository version: 5
+
+
+You can see that I am able to generate correctly the special remote key used by `remote2` the `shared` remote, `0332a66cded87db8ec8427280c171133c958754f`. But using the same method for `remote1` the `sharedpubkey` remote does not yield a meaningful key `bb3b1f4b27164c2edc81da32c43c64e8a4be75d8`
+
+### Have you had any luck using git-annex before? (Sometimes we get tired of reading bug reports all day and a lil' positive end note does wonders)
+Yes. —[andrew](https://git-annex.branchable.com/users/andrew/)
+
+[[done]]
diff --git a/doc/bugs/sharedpubkey_is_using_a_different_filename_encryption_method_than_shared/comment_1_72dbc03edbd8559527968c504c4fe7af._comment b/doc/bugs/sharedpubkey_is_using_a_different_filename_encryption_method_than_shared/comment_1_72dbc03edbd8559527968c504c4fe7af._comment
new file mode 100644
index 0000000000..0164038c2b
--- /dev/null
+++ b/doc/bugs/sharedpubkey_is_using_a_different_filename_encryption_method_than_shared/comment_1_72dbc03edbd8559527968c504c4fe7af._comment
@@ -0,0 +1,28 @@
+[[!comment format=mdwn
+ username="joey"
+ subject="""comment 1"""
+ date="2018-07-19T20:38:12Z"
+ content="""
+Shared only uses the first 256 bytes of the cipher to
+encrypt filenames, while sharedpubkey uses the entire cipher.
+
+That difference makes sense, since shared uses the second half of the cipher to
+encrypt file contents, while sharedpubkey encrypts that to the gpg key(s).
+
+You are truncating the sharedpubkey cipher to 256 bytes; I suspect if you
+don't, it will work. If it still doesn't work, refer to the working code
+that I posted in the forum thread earlier.
+
+I don't think that doc/encryption.mdwn needs to go into detailed specifics.
+If someone would like to write a fully detailed explanation of how the
+encryption works it could go in doc/internals.mdwn or someplace like that.
+
+(Crypto.hs is also not hard to follow if you look at the types:
+cipherMac of a MacOnlyCipher uses the whole cipher, while
+cipherMac of a Cipher uses only the beginning of the cipher.
+And decryptCipher of a SharedPubKeyCipher creates a MacOnlyCipher.)
+
+This bug certianly does not warrent changing the behavior of git-annex,
+which would in any case only complicate the situation since it would still
+need to support the current data.
+"""]]
diff --git a/doc/bugs/sharedpubkey_is_using_a_different_filename_encryption_method_than_shared/comment_2_a327ce6e5fe169a3042e207ae7306245._comment b/doc/bugs/sharedpubkey_is_using_a_different_filename_encryption_method_than_shared/comment_2_a327ce6e5fe169a3042e207ae7306245._comment
new file mode 100644
index 0000000000..20a11e8b14
--- /dev/null
+++ b/doc/bugs/sharedpubkey_is_using_a_different_filename_encryption_method_than_shared/comment_2_a327ce6e5fe169a3042e207ae7306245._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="andrew"
+ avatar="http://cdn.libravatar.org/avatar/acc0ece1eedf07dd9631e7d7d343c435"
+ subject="comment 2"
+ date="2018-07-23T22:35:48Z"
+ content="""
+Thanks! That makes sense. Marking as done.
+"""]]
diff --git a/doc/bugs/should_be_possible_to_use_ssh_hosts_from_.ssh__47__config.mdwn b/doc/bugs/should_be_possible_to_use_ssh_hosts_from_.ssh__47__config.mdwn
new file mode 100644
index 0000000000..d016038bc1
--- /dev/null
+++ b/doc/bugs/should_be_possible_to_use_ssh_hosts_from_.ssh__47__config.mdwn
@@ -0,0 +1,26 @@
+### Please describe the problem.
+
+I am using a ssh-host configured in .ssh/config in a corporate-environment like this:
+
+[[!format sh """
+  Host home
+        Hostname 1.2.3.4
+        Port 443
+        ProxyCommand corkscrew proxy.fqdn 8080 %h %p ~/.authfile
+        # http://www.agroman.net/corkscrew/
+"""]]
+
+I now want to be able to add this host as a remote repository, but don't have the ability to add it. A tunnel to localhost would be possible but then I would have to setup this portforwarding for git-annex separately every time it is started.
+
+### What steps will reproduce the problem?
+
+Add a remote host that is only reachable through a proxy.
+
+### What version of git-annex are you using? On what operating system?
+
+gentoo
+git-annex-5.20141125 {+assistant +cryptohash +dbus +desktop-notify +dns +feed +inotify +pairing +production +quvi +s3 +tahoe +tdfa +testsuite +webapp +webapp-secure +webdav +xmpp}
+
+
+
+Is it possible to use this predefined host, or add the possibility to configure git-annex like above?
diff --git a/doc/bugs/should_be_possible_to_use_ssh_hosts_from_.ssh__47__config/comment_1_3dac095b7cf799e31925868d13888970._comment b/doc/bugs/should_be_possible_to_use_ssh_hosts_from_.ssh__47__config/comment_1_3dac095b7cf799e31925868d13888970._comment
new file mode 100644
index 0000000000..8fb2ae19cb
--- /dev/null
+++ b/doc/bugs/should_be_possible_to_use_ssh_hosts_from_.ssh__47__config/comment_1_3dac095b7cf799e31925868d13888970._comment
@@ -0,0 +1,7 @@
+[[!comment format=mdwn
+ username="gregor"
+ subject="workaround:"
+ date="2014-12-02T18:04:16Z"
+ content="""
+my workaround is to setup the remote with the local port-forward end then edit the .ssh/config to use the needed ProxyCommand
+"""]]
diff --git a/doc/bugs/should_be_possible_to_use_ssh_hosts_from_.ssh__47__config/comment_2_342b4913a063184650cdf4f541f7ea02._comment b/doc/bugs/should_be_possible_to_use_ssh_hosts_from_.ssh__47__config/comment_2_342b4913a063184650cdf4f541f7ea02._comment
new file mode 100644
index 0000000000..82108d0cda
--- /dev/null
+++ b/doc/bugs/should_be_possible_to_use_ssh_hosts_from_.ssh__47__config/comment_2_342b4913a063184650cdf4f541f7ea02._comment
@@ -0,0 +1,10 @@
+[[!comment format=mdwn
+ username="joey"
+ subject="""comment 2"""
+ date="2014-12-02T18:51:44Z"
+ content="""
+That will work fine as a git remote using that hostname. `git remote add home home:path`
+
+The webapp isn't currently aware of hostnames configured via .ssh/config,
+so it cannot be set up using the webapp.
+"""]]
diff --git a/doc/bugs/shouldn__39__t_keep_permissions_of_the_ssh_remote__63__.mdwn b/doc/bugs/shouldn__39__t_keep_permissions_of_the_ssh_remote__63__.mdwn
new file mode 100644
index 0000000000..fc9c3965ee
--- /dev/null
+++ b/doc/bugs/shouldn__39__t_keep_permissions_of_the_ssh_remote__63__.mdwn
@@ -0,0 +1,9 @@
+### Please describe the problem.
+
+Initially generated an annex while having a restrictive umask 077.  Then cloned that repository to another host for public consumption so directory had proper/good permissions set allowing group to access.  And everything is accessible but not the load which I 'annex get'ed.  Key directories were readable but not the content.  I guess there is somewhere 'preserve permissions' setting for rsync/scp which imho shouldn't be there and content should inherit local/environment settings
+
+### What version of git-annex are you using? On what operating system?
+
+6.20160208+gitg1ac9034-1~ndall+1
+
+[[!meta author=yoh]]
diff --git a/doc/bugs/shouldn__39__t_keep_permissions_of_the_ssh_remote__63__/comment_1_35eeb15c0031c18ba6d0922bb7ae8ed3._comment b/doc/bugs/shouldn__39__t_keep_permissions_of_the_ssh_remote__63__/comment_1_35eeb15c0031c18ba6d0922bb7ae8ed3._comment
new file mode 100644
index 0000000000..33047adf9f
--- /dev/null
+++ b/doc/bugs/shouldn__39__t_keep_permissions_of_the_ssh_remote__63__/comment_1_35eeb15c0031c18ba6d0922bb7ae8ed3._comment
@@ -0,0 +1,89 @@
+[[!comment format=mdwn
+ username="joey"
+ subject="""comment 1"""
+ date="2016-04-14T20:43:05Z"
+ content="""
+umask 077 means a file will be mode 400. The 400 mode is indeed retained
+when that object is transferred to another repository.
+
+git-annex does make sure that the file can be read by the current user.
+Which mode 400 does allow.
+
+Did your problem involve some other user needing to read the file? 
+If you configure core.sharedrepository group or world, then git-annex
+will adjust the mode of the received object to let the group or everyone
+read it.
+
+----
+
+The simplest way to make it use the umask would be this patch. But,
+with this patch, if I have a file that's mode 664 and my umask is 077, git
+annex add will change the file mode to 600.
+So, I don't like this patch; its impact is too broad.
+
+    diff --git a/Annex/Perms.hs b/Annex/Perms.hs
+    index 6444025..9b35bb8 100644
+    --- a/Annex/Perms.hs
+    +++ b/Annex/Perms.hs
+    @@ -5,6 +5,8 @@
+      - Licensed under the GNU GPL version 3 or higher.
+      -}
+     
+    +{-# LANGUAGE CPP #-}
+    +
+     module Annex.Perms (
+     	setAnnexFilePerm,
+     	setAnnexDirPerm,
+    @@ -28,6 +30,10 @@ import Git.SharedRepository
+     import qualified Annex
+     import Config
+     
+    +#ifndef mingw32_HOST_OS
+    +import Data.Bits
+    +#endif
+    +
+     withShared :: (SharedRepository -> Annex a) -> Annex a
+     withShared a = a =<< coreSharedRepository <$> Annex.getGitConfig
+     
+    @@ -82,7 +88,7 @@ createAnnexDirectory dir = walk dir [] =<< top
+     			setAnnexDirPerm p
+     
+     {- Normally, blocks writing to an annexed file, and modifies file
+    - - permissions to allow reading it.
+    + - permissions to not allow writing it and otherwise reflect the umask.
+      -
+      - When core.sharedRepository is set, the write bits are not removed from
+      - the file, but instead the appropriate group write bits are set. This is
+    @@ -98,9 +104,18 @@ freezeContent file = unlessM crippledFileSystem $
+     		addModes [ownerReadMode, groupReadMode, ownerWriteMode, groupWriteMode]
+     	go AllShared = liftIO $ void $ tryIO $ modifyFileMode file $
+     		addModes (readModes ++ writeModes)
+    -	go _ = liftIO $ modifyFileMode file $
+    -		removeModes writeModes .
+    -		addModes [ownerReadMode]
+    +#ifndef mingw32_HOST_OS
+    +	go _ = liftIO $ bracket
+    +		(liftIO $ setFileCreationMask 0o22) -- set umask to get it
+    +		(liftIO . setFileCreationMask) 
+    +		$ \umask -> modifyFileMode file $
+    +			removeModes writeModes .
+    +			(\mode -> mode .&. complement umask)
+    +#else
+    +	go _ = liftIO $ modifyFileMode file $ 
+    +			removeModes writeModes .
+    +			addModes [ ownerReadMode ]
+    +#endif
+     
+     isContentWritePermOk :: FilePath -> Annex Bool
+     isContentWritePermOk file = ifM crippledFileSystem
+
+The umask-based mode setting could be put into every remote
+that uses a transfer method that preserves permissions. (Including the
+external special remote I suppose, since it *might* do so.) Or put in
+`gitViaTmp` but there are a few places `getViaTmp` is used that do not
+involve transfers from remotes.
+
+Also this would add the overhead of umask()+stat()+chmod()+umask()
+to every file downloaded by git-annex, on top of the chmodding it already
+does.
+"""]]
diff --git a/doc/bugs/shouldn__39__t_keep_permissions_of_the_ssh_remote__63__/comment_2_65232a9c2bbba075d325ce00b7c5e13f._comment b/doc/bugs/shouldn__39__t_keep_permissions_of_the_ssh_remote__63__/comment_2_65232a9c2bbba075d325ce00b7c5e13f._comment
new file mode 100644
index 0000000000..7de7adc48c
--- /dev/null
+++ b/doc/bugs/shouldn__39__t_keep_permissions_of_the_ssh_remote__63__/comment_2_65232a9c2bbba075d325ce00b7c5e13f._comment
@@ -0,0 +1,7 @@
+[[!comment format=mdwn
+ username="https://me.yahoo.com/a/EbvxpTI_xP9Aod7Mg4cwGhgjrCrdM5s-#7c0f4"
+ subject="comment 2"
+ date="2016-04-18T14:16:31Z"
+ content="""
+heh -- it has been awhile and damn me didn't reveal originally the location of such a repository so I could not check if I set group settings for the repo \"correctly\"... don't know now -- would need to replicate at some point and follow up
+"""]]
diff --git a/doc/bugs/signal_weirdness.mdwn b/doc/bugs/signal_weirdness.mdwn
new file mode 100644
index 0000000000..5a00963343
--- /dev/null
+++ b/doc/bugs/signal_weirdness.mdwn
@@ -0,0 +1,50 @@
+For the record, there is a slight weirdness with how git-annex
+handles a signal like ctrl-c.
+
+For example:
+
+	joey@gnu:~/tmp/b>git annex copy a b --to origin
+	copy a (checking origin...) (to origin...) 
+	SHA256-s104857600--20492a4d0d84f8beb1767f6616229f85d44c2827b64bdbfb260ee12fa1109e0e
+	        3272   0%    0.00kB/s    0:00:00  ^C
+	zsh: interrupt  git annex copy a --to origin
+	joey@gnu:~/tmp/b>
+	rsync error: unexplained error (code 130) at rsync.c(549) [sender=3.0.9]
+
+Here git-annex exits before rsync has fully exited. Not a large problem
+but sorta weird.
+
+The culprit is `CmdLine.startup` in Utility.SafeCommand, which installs
+a default signal handler for SIGINT, which causes it to immediatly
+terminate git-annex. rsync, in turn, has its own SIGINT handler, which 
+prints the message, typically later.
+
+(Why it prints that message and not its more usual message about having
+received a signal, I'm not sure?)
+
+It's more usual for a `system` like thing to block SIGINT, letting the child
+catch it and exit, and then detecting the child's exit status and terminating. 
+However, since rsync *is* trapping SIGINT, and exiting nonzero explicitly,
+git-annex can't tell that rsync failed due to a SIGINT by examining the
+`waitpid` result.
+And, git-annex typically doesn't stop when a single child fails. In the
+example above, it would go on to copy `b` after a ctrl-c!
+
+A further complication is that git-annex is itself a child process
+of git, which does not block SIGINT either. So if git-annex blocks SIGINT,
+it will be left running in the background after git exits, and continuing
+with further actions too. (Perhaps its SIGINT handling is a bug in git.)
+
+Now, rsync does have a documented exit code it uses after a SIGINT.
+But other programs git-annex runs generally do not. So it would be possible
+to special case in support for rsync, blocking SIGINT while running it,
+noticing it exited with 20, and git-annex then stopping. But this is
+ugly and failure prone if rsync's code 20 changes. And it only
+would fix the rsync case, not helping with other commands like wget, unless
+it assumes they never trap SIGINT on their own.
+
+Which is why the current behavior of not blocking SIGINT was chosen,
+as a less bad alternative. Still, I'd like to find a better one.
+--[[Joey]] 
+
+[[!tag confirmed]]
diff --git a/doc/bugs/silently_failing_when_attempting_to_add_ignored_files.mdwn b/doc/bugs/silently_failing_when_attempting_to_add_ignored_files.mdwn
new file mode 100644
index 0000000000..d9b5f67486
--- /dev/null
+++ b/doc/bugs/silently_failing_when_attempting_to_add_ignored_files.mdwn
@@ -0,0 +1,31 @@
+### Please describe the problem.
+When attempting to add a file listed in .gitignore, git annex exits silently. I would expect to see an error message a la plain git: "The following paths are ignored by one of your .gitignore files: _that file_"
+
+### What steps will reproduce the problem?
+1. Add a file to .gitignore
+2. git annex add _that file_
+
+### What version of git-annex are you using? On what operating system?
+git-annex version: 6.20160613
+build flags: Assistant Webapp Pairing Testsuite S3(multipartupload)(storageclasses) WebDAV FsEvents XMPP ConcurrentOutput TorrentParser MagicMime Feeds Quvi
+key/value backends: SHA256E SHA256 SHA512E SHA512 SHA224E SHA224 SHA384E SHA384 SHA3_256E SHA3_256 SHA3_512E SHA3_512 SHA3_224E SHA3_224 SHA3_384E SHA3_384 SKEIN256E SKEIN256 SKEIN512E SKEIN512 SHA1E SHA1 MD5E MD5 WORM URL
+remote types: git gcrypt S3 bup directory rsync web bittorrent webdav tahoe glacier ddar hook external
+local repository version: 5
+supported repository versions: 5 6
+upgrade supported from repository versions: 0 1 2 3 4 5
+operating system: darwin x86_64
+
+
+### Please provide any additional information below.
+
+[[!format sh """
+# If you can, paste a complete transcript of the problem occurring here.
+# If the problem is with the git-annex assistant, paste in .git/annex/daemon.log
+
+
+# End of transcript or log.
+"""]]
+
+### Have you had any luck using git-annex before? (Sometimes we get tired of reading bug reports all day and a lil' positive end note does wonders)
+
+
diff --git a/doc/bugs/silently_failing_when_attempting_to_add_ignored_files/comment_1_f1208fda822d17a3a9e5457324fdd1df._comment b/doc/bugs/silently_failing_when_attempting_to_add_ignored_files/comment_1_f1208fda822d17a3a9e5457324fdd1df._comment
new file mode 100644
index 0000000000..10a9330a7f
--- /dev/null
+++ b/doc/bugs/silently_failing_when_attempting_to_add_ignored_files/comment_1_f1208fda822d17a3a9e5457324fdd1df._comment
@@ -0,0 +1,39 @@
+[[!comment format=mdwn
+ username="joey"
+ subject="""comment 1"""
+ date="2016-07-12T17:13:37Z"
+ content="""
+This is due to git-annex using `git ls-files --others --exclude-standard`
+to find files to add. Since that silently skips ignored files, so does
+git-annex.
+
+Now that git has `git check-ignore --stdin`, it would be possible for
+git-annex to not run `git ls-files` with `--exclude-standard`, and
+instead check each file it returns to see if it's ignored, and print out 
+a warning message.
+
+But, that would mean a round-trip through the pipe for each filename.
+When operating on a directory containing a lot of new 
+files, it would probably slow down the processing somewhat.
+
+Hmm, but, `git add somedir` does not warn if there are gitignored files in
+somedir; it silently skips them while adding the rest of the directory.
+The warning comes only when explicitly listing an ignored file.
+
+So, git-annex could do the same, only passing the filename through
+checkingnore when it's a normal file and not a directory. This does
+entail statting every command-line parameter though, and passing through
+checkignore would still slow things down. Particularly when `git annex add`
+is run with a huge list of files to add.
+
+I don't know if it's super-important for `git annex add` to mirror every
+behavior of `git add` anyway. Other differences include `git annex add`
+with no parameters defaulting to adding ".", and `git annex add`
+skipping dotfiles by default.
+
+In v6 mode, you can use `git add` to do the same thing as `git annex add`,
+only more slowly. So it could be argued that `git annex add` will remain
+separate from `git add` only because its different behavior can be more
+useful, and for performance reasons. Which makes slowing `git annex add`
+down in order to make it behave more like `git add` seem counterproductive.
+"""]]
diff --git a/doc/bugs/silently_failing_when_attempting_to_add_ignored_files/comment_2_9fe7fd6296db34ee8ee07fe68459e74b._comment b/doc/bugs/silently_failing_when_attempting_to_add_ignored_files/comment_2_9fe7fd6296db34ee8ee07fe68459e74b._comment
new file mode 100644
index 0000000000..fca560ce41
--- /dev/null
+++ b/doc/bugs/silently_failing_when_attempting_to_add_ignored_files/comment_2_9fe7fd6296db34ee8ee07fe68459e74b._comment
@@ -0,0 +1,17 @@
+[[!comment format=mdwn
+ username="dermungo@19d0cb1f22d4169b48363cfff60c9ede2c14fffa"
+ nickname="dermungo"
+ avatar="http://cdn.libravatar.org/avatar/4f4b91e2275a6673506e2990a4f96270"
+ subject="Work around"
+ date="2017-01-13T14:54:40Z"
+ content="""
+Perhaps other people knew this already, but I thought I would post it here in case someone else has trouble finding a solution to this issue.
+
+I use a global gitignore that ignores most of the files I want to have in my annex.
+The solution was to override my global .gitignore according to this answer at SO  by creating a .gitignore in the annexed repo with the pattern
+```
+!*
+```
+
+Now it automatically adds everything (except dotfiles) when running `git annex add $DIRPATH`.
+"""]]
diff --git a/doc/bugs/some_tests_fail_while_running_under_NFS.mdwn b/doc/bugs/some_tests_fail_while_running_under_NFS.mdwn
new file mode 100644
index 0000000000..2e87ff8a5b
--- /dev/null
+++ b/doc/bugs/some_tests_fail_while_running_under_NFS.mdwn
@@ -0,0 +1,54 @@
+### Please describe the problem.
+
+4 out of 269 tests failed (4468.00s)
+
+hard to assess how critical they are... 
+
+### What steps will reproduce the problem?
+
+run git annex test
+
+### What version of git-annex are you using? On what operating system?
+
+6.20160307+gitgb095561-1~ndall+1
+
+### Please provide any additional information below.
+
+[Full log](http://www.onerussian.com/tmp/git-annex-tests-6.20160307+gitgb095561-1~ndall+1.log)
+
+[[!format sh """
+smaug:/mnt/nfs/scrap/datalad/test_annex
+$> grep -B5 FAIL git-annex-tests-6.20160307+gitgb095561-1~ndall+1.log
+    crypto:                                               OK (50.57s)
+    preferred content:                                    OK (20.36s)
+    add subdirs:                                          OK (8.97s)
+    addurl:                                               .t/tmprepo73/.git/annex/keys: removeDirectoryRecursive: unsatisfied constraints (Directory not empty)
+sleeping 10 seconds and will retry directory cleanup
+FAIL
+--
+   293ed4c..c16b350  git-annex -> synced/git-annex
+   dd272eb..ffe1721  master -> synced/master
+OK (11.64s)
+    addurl:                                               .t/tmprepo73/.git/annex/keys/.nfs0000000009a305870000020e: removeDirectoryRecursive: resource busy (Device or resource busy)
+sleeping 10 seconds and will retry directory cleanup
+FAIL
+--
+   0993b09..c09ddc4  git-annex -> synced/git-annex
+   a823824..520f58c  master -> synced/master
+OK (13.67s)
+    addurl:                                               .t/tmprepo73/.git/annex/keys/.nfs0000000009a305870000020e: removeDirectoryRecursive: resource busy (Device or resource busy)
+sleeping 10 seconds and will retry directory cleanup
+FAIL
+--
+OK (14.59s)
+    addurl:                                               On branch master
+nothing to commit, working directory clean
+.t/tmprepo73/.git/annex/keys/.nfs0000000009a305870000020e: removeDirectoryRecursive: resource busy (Device or resource busy)
+sleeping 10 seconds and will retry directory cleanup
+FAIL
+
+
+# End of transcript or log.
+"""]]
+
+[[!meta author=yoh]]
diff --git a/doc/bugs/some_tests_fail_while_running_under_NFS/comment_1_8e0f72f633ec79bd373b009046ab66b7._comment b/doc/bugs/some_tests_fail_while_running_under_NFS/comment_1_8e0f72f633ec79bd373b009046ab66b7._comment
new file mode 100644
index 0000000000..f52704041d
--- /dev/null
+++ b/doc/bugs/some_tests_fail_while_running_under_NFS/comment_1_8e0f72f633ec79bd373b009046ab66b7._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="joey"
+ subject="""comment 1"""
+ date="2016-03-09T17:15:13Z"
+ content="""
+This is all down to those nfs lock files, so finding a way to probe for an
+NFS system and auto-enable pidlock is the best way to fix this.
+"""]]
diff --git a/doc/bugs/some_transfers_for_get_-J4_fail_over_ssh__47__rsync_and_require_retry.mdwn b/doc/bugs/some_transfers_for_get_-J4_fail_over_ssh__47__rsync_and_require_retry.mdwn
new file mode 100644
index 0000000000..fe9f4f2d5b
--- /dev/null
+++ b/doc/bugs/some_transfers_for_get_-J4_fail_over_ssh__47__rsync_and_require_retry.mdwn
@@ -0,0 +1,33 @@
+### Please describe the problem.
+
+I quickly searched but could not find exactly the same issue, so filing it new, but feel welcome to refer/close if it is already filed.
+
+Upon some transfers between local hosts, I am getting some files failing to be transferred upon initial run, and I need to rerun `get` to get them.  Could may be `annex get` internally retry a few times (I think the same is done for http already) upon transfer failure?
+
+[[!format sh """
+
+(git)smaug:/mnt/btrfs/datasets/datalad/crawl/dbic/QA[master]git
+$> git annex get -J4 --from=origin --all --json 2>&1 | tee /tmp/qa-get.log
+{"command":"get","note":"from origin...\nchecksum...","success":true,"key":"SHA256E-s9037046--92b3f24faf39b0c3710311fa57a614bdc24f998e5a4f7c8b33063d4ef01a0aaf.nii.gz","file":null}
+...
+git-annex: get: 1 failed
+git annex get -J4 --from=origin --all --json 2>&1  87.01s user 22.58s system 73% cpu 2:29.82 total
+tee /tmp/qa-get.log  0.00s user 0.02s system 0% cpu 2:29.82 total
+
+$> grep -v -e 'success":true' /tmp/qa-get.log 
+{"command":"get","note":"from origin...\nrsync failed -- run git annex again to resume file transfer","success":false,"key":"SHA256E-s44080906--a534bdaa5b4143fb64135aa83d86cf4d4f29d2c23b7f2e7e1d72a811835f2bb8.nii.gz","file":null}
+git-annex: get: 1 failed
+
+$> git annex get --key SHA256E-s44080906--a534bdaa5b4143fb64135aa83d86cf4d4f29d2c23b7f2e7e1d72a811835f2bb8.nii.gz --from origin
+get SHA256E-s44080906--a534bdaa5b4143fb64135aa83d86cf4d4f29d2c23b7f2e7e1d72a811835f2bb8.nii.gz (from origin...) 
+(checksum...) ok                  
+(recording state in git...)
+
+$> git annex version                  
+git-annex version: 6.20180807+git291-g549114118-1~ndall+1
+
+"""]]
+
+
+[[!meta author=yoh]]
+
diff --git a/doc/bugs/some_transfers_for_get_-J4_fail_over_ssh__47__rsync_and_require_retry/comment_1_b762aa4a31f2153c00988d185ce5efbe._comment b/doc/bugs/some_transfers_for_get_-J4_fail_over_ssh__47__rsync_and_require_retry/comment_1_b762aa4a31f2153c00988d185ce5efbe._comment
new file mode 100644
index 0000000000..40d8ae750e
--- /dev/null
+++ b/doc/bugs/some_transfers_for_get_-J4_fail_over_ssh__47__rsync_and_require_retry/comment_1_b762aa4a31f2153c00988d185ce5efbe._comment
@@ -0,0 +1,10 @@
+[[!comment format=mdwn
+ username="joey"
+ subject="""comment 1"""
+ date="2018-09-25T17:05:13Z"
+ content="""
+You neglected to show the actual error message from git-annex or rsync or
+whatever actually failed. I can't do anything with this.
+
+annex.retry affects all transfers, not only http.
+"""]]
diff --git a/doc/bugs/some_transfers_for_get_-J4_fail_over_ssh__47__rsync_and_require_retry/comment_2_7d5c57bf17659769d9cf4d56774ba67e._comment b/doc/bugs/some_transfers_for_get_-J4_fail_over_ssh__47__rsync_and_require_retry/comment_2_7d5c57bf17659769d9cf4d56774ba67e._comment
new file mode 100644
index 0000000000..1398291d87
--- /dev/null
+++ b/doc/bugs/some_transfers_for_get_-J4_fail_over_ssh__47__rsync_and_require_retry/comment_2_7d5c57bf17659769d9cf4d56774ba67e._comment
@@ -0,0 +1,36 @@
+[[!comment format=mdwn
+ username="yarikoptic"
+ avatar="http://cdn.libravatar.org/avatar/f11e9c84cb18d26a1748c33b48c924b4"
+ subject="comment 2"
+ date="2018-09-25T22:36:43Z"
+ content="""
+
+> You neglected to show the actual error message from git-annex or rsync or whatever actually failed. I can't do anything with this.
+
+oh lucky me still having that file where I dumped all the output and grepped in the original report to not flood the page: http://www.onerussian.com/tmp/qa-get.log
+but hey -- there were no actual error message as far as I see it:
+
+[[!format sh \"\"\"
+$> grep -v '^{' /tmp/qa-get.log
+git-annex: get: 1 failed
+\"\"\"]]
+
+so may be if annex included the message from git-annex/rsync in its record:
+
+[[!format sh \"\"\"
+$> grep rsync qa-get.log
+{\"command\":\"get\",\"note\":\"from origin...\nrsync failed -- run git annex again to resume file transfer\",\"success\":false,\"key\":\"SHA256E-s44080906--a534bdaa5b4143fb64135aa83d86cf4d4f29d2c23b7f2e7e1d72a811835f2bb8.nii.gz\",\"file\":null}
+\"\"\"]]
+
+we both would be happier...
+
+> annex.retry affects all transfers, not only http.
+
+that is great -- so why it didn't retry here?  is default `0`, i.e. no retries?
+
+[[!format sh \"\"\"
+$> grep a534bdaa5b4143fb64135aa83d86cf4d4f29d2c23b7f2e7e1d72a811835f2bb8 qa-get.log 
+{\"command\":\"get\",\"note\":\"from origin...\nrsync failed -- run git annex again to resume file transfer\",\"success\":false,\"key\":\"SHA256E-s44080906--a534bdaa5b4143fb64135aa83d86cf4d4f29d2c23b7f2e7e1d72a811835f2bb8.nii.gz\",\"file\":null}
+\"\"\"]]
+
+"""]]
diff --git a/doc/bugs/special_remote_protocol__58___adding_local_state_variables___40__not_under_git__41__.mdwn b/doc/bugs/special_remote_protocol__58___adding_local_state_variables___40__not_under_git__41__.mdwn
new file mode 100644
index 0000000000..bb7345123c
--- /dev/null
+++ b/doc/bugs/special_remote_protocol__58___adding_local_state_variables___40__not_under_git__41__.mdwn
@@ -0,0 +1,13 @@
+### Please describe the feature request.
+It would be useful in the special remote protocol to add commands to store or retrieve local key / value pairs (not stored under git, but only in local files). At the time of writing it seems to me that there are no ways for a special remote to store key / value pairs that are not so relevant to be stored in the git-annex branch but may be of great relevance for other uses (such as optimization).
+
+More on my specific use of the feature: Resumable uploads
+Use case: while uploading a file to a remote which allow resumable uploads (giving you a token to resume at each uploaded chunk), the network connection may drop. The special remote then may need to store the token value under a local file in order to try resume the upload at a later time.
+
+Since the token is not relevant to the story of the git-annex branch (not regarding the user or the repository itself) but of great relevance for optimization (and may also expire after some time) the addition of commands such as SETCONFIG and GETCONFIG (maybe with names like SETLOCALCONFIG and GETLOCALCONFIG) seems natural to me. They could be also useful to save other optimization values (settings that may need to vary from a repository to another one).
+
+### What version of git-annex are you using? On what operating system?
+git-annex 6.20170101 on Debian stretch (from debian archives)
+
+### Have you had any luck using git-annex before? (Sometimes we get tired of reading bug reports all day and a lil' positive end note does wonders)
+I'm using (and enjoying) it a lot :)
diff --git a/doc/bugs/special_remote_protocol__58___adding_local_state_variables___40__not_under_git__41__/comment_1_3d162dce2af8d84c5582cf63624e9310._comment b/doc/bugs/special_remote_protocol__58___adding_local_state_variables___40__not_under_git__41__/comment_1_3d162dce2af8d84c5582cf63624e9310._comment
new file mode 100644
index 0000000000..6fdd1e6ad9
--- /dev/null
+++ b/doc/bugs/special_remote_protocol__58___adding_local_state_variables___40__not_under_git__41__/comment_1_3d162dce2af8d84c5582cf63624e9310._comment
@@ -0,0 +1,19 @@
+[[!comment format=mdwn
+ username="joey"
+ subject="""comment 1"""
+ date="2017-02-07T16:49:40Z"
+ content="""
+A reasonable use case indeed.
+
+My initial feeling is, this can be handled well by storing the data
+yourself in the filesysystem. You only need to pick a place for your
+special remote to store the data. Presumably somewhere under `.git`
+or under `.git/annex`. (The protocol's GETGITDIR can be used to
+get the path to `.git`.)
+
+And you need to pick a filename that won't be used by something else.
+
+You can use the protocol's GETUUID to get the uuid of special remote,
+and using that in the path to the data pretty well guarantees nothing
+else is going to use that filename.
+"""]]
diff --git a/doc/bugs/standalone_build_is_slightly_out_of_date.mdwn b/doc/bugs/standalone_build_is_slightly_out_of_date.mdwn
new file mode 100644
index 0000000000..91a6415bd7
--- /dev/null
+++ b/doc/bugs/standalone_build_is_slightly_out_of_date.mdwn
@@ -0,0 +1,45 @@
+
+[[!format sh """
+hopa:/tmp
+$> wget -S https://downloads.kitenet.net/git-annex/linux/current/git-annex-standalone-amd64.tar.gz
+--2018-09-25 09:25:06--  https://downloads.kitenet.net/git-annex/linux/current/git-annex-standalone-amd64.tar.gz
+Resolving downloads.kitenet.net (downloads.kitenet.net)... 66.228.36.95, 2600:3c03::f03c:91ff:fe73:b0d2
+Connecting to downloads.kitenet.net (downloads.kitenet.net)|66.228.36.95|:443... connected.
+HTTP request sent, awaiting response... 
+  HTTP/1.1 200 OK
+  Date: Tue, 25 Sep 2018 13:25:07 GMT
+  Server: Apache/2.4.34 (Debian)
+  Last-Modified: Thu, 13 Sep 2018 20:29:04 GMT
+  ETag: "3e30e83-575c68ee3f927"
+  Accept-Ranges: bytes
+  Content-Length: 65212035
+  Keep-Alive: timeout=5, max=100
+  Connection: Keep-Alive
+  Content-Type: application/x-gzip
+Length: 65212035 (62M) [application/x-gzip]
+Saving to: ‘git-annex-standalone-amd64.tar.gz’
+
+git-annex-standalone-amd64. 100%[===========================================>]  62.19M  1.77MB/s    in 28s     
+
+2018-09-25 09:25:35 (2.20 MB/s) - ‘git-annex-standalone-amd64.tar.gz’ saved [65212035/65212035]
+
+
+$> tar -xzf git-annex-standalone-amd64.tar.gz                                                    
+
+$> git-annex.linux/git-annex version         
+git-annex version: 6.20180808-ga1327779a
+build flags: Assistant Webapp Pairing S3(multipartupload)(storageclasses) WebDAV Inotify DBus DesktopNotify ConcurrentOutput TorrentParser MagicMime Feeds Testsuite
+dependency versions: aws-0.19 bloomfilter-2.0.1.0 cryptonite-0.25 DAV-1.3.2 feed-1.0.0.0 ghc-8.2.2 http-client-0.5.13 persistent-sqlite-2.8.1.2 torrent-10000.1.1 uuid-1.3.13 yesod-1.6.0
+key/value backends: SHA256E SHA256 SHA512E SHA512 SHA224E SHA224 SHA384E SHA384 SHA3_256E SHA3_256 SHA3_512E SHA3_512 SHA3_224E SHA3_224 SHA3_384E SHA3_384 SKEIN256E SKEIN256 SKEIN512E SKEIN512 BLAKE2B256E BLAKE2B256 BLAKE2B512E BLAKE2B512 BLAKE2B160E BLAKE2B160 BLAKE2B224E BLAKE2B224 BLAKE2B384E BLAKE2B384 BLAKE2S256E BLAKE2S256 BLAKE2S160E BLAKE2S160 BLAKE2S224E BLAKE2S224 BLAKE2SP256E BLAKE2SP256 BLAKE2SP224E BLAKE2SP224 SHA1E SHA1 MD5E MD5 WORM URL
+remote types: git gcrypt p2p S3 bup directory rsync web bittorrent webdav adb tahoe glacier ddar hook external
+operating system: linux x86_64
+supported repository versions: 3 5 6
+upgrade supported from repository versions: 0 1 2 3 4 5
+
+$> git -C ~/proj/git-annex describe --contains 6.20180808-ga1327779a
+6.20180913~3^2~1
+
+"""]]
+
+[[!meta author=yoh]]
+
diff --git a/doc/bugs/standalone_build_is_slightly_out_of_date/comment_1_349f2d8f6ef0fddcfe576fc08b3cccb6._comment b/doc/bugs/standalone_build_is_slightly_out_of_date/comment_1_349f2d8f6ef0fddcfe576fc08b3cccb6._comment
new file mode 100644
index 0000000000..3993949b2b
--- /dev/null
+++ b/doc/bugs/standalone_build_is_slightly_out_of_date/comment_1_349f2d8f6ef0fddcfe576fc08b3cccb6._comment
@@ -0,0 +1,15 @@
+[[!comment format=mdwn
+ username="joey"
+ subject="""comment 1"""
+ date="2018-09-25T16:30:43Z"
+ content="""
+That's 6 commits back from the release tag for 6.20180913.
+The only code change missed is 
+[[!commit a7f0b99a3334fbc73e3cb53f37f5a065ee14b135]],
+which does not affect this build because this build bundles a newer version
+of git than the one that commit is dealing with.
+
+Without changing git-annex's build infrastructure to something that builds
+on released tags, a minor skew like this is to be expected, since I just
+ship the latest daily autobuilds with the release.
+"""]]
diff --git a/doc/bugs/standalone_builds_shouldn__39__t_pollute___126____47__.ssh_with_helpers_merely_upon_annex_init.mdwn b/doc/bugs/standalone_builds_shouldn__39__t_pollute___126____47__.ssh_with_helpers_merely_upon_annex_init.mdwn
new file mode 100644
index 0000000000..58c7d71fc6
--- /dev/null
+++ b/doc/bugs/standalone_builds_shouldn__39__t_pollute___126____47__.ssh_with_helpers_merely_upon_annex_init.mdwn
@@ -0,0 +1,18 @@
+### Please describe the problem.
+
+Currently if I using standalone annex builds mere 'annex init' immediately rushes to generate .ssh/git-annex-{shell,wrapper} helpers.  I would really prefer to keep ~/.ssh untouched by any magic from the tools which presumably unrelated to ssh.  Moreover it leads to failures such as 
+
+/usr/lib/git-annex.linux/runshell: 67: /usr/lib/git-annex.linux/runshell: cannot create /.ssh/git-annex-wrapper: Directory nonexistent
+
+happen if HOME was overridden (e.g. for testing etc) and I have no intent to use annex as an assistant etc.
+
+Originally described in http://git-annex.branchable.com/devblog/day_155__missing_bits/ as a solution to overcome problems with assistant's server operation from standalone builds.  Why those couldn't be installed alongside with git-annex wrapper? (actually there is also git-annex-shell at least in the Debian standalone packages)
+
+
+### What version of git-annex are you using? On what operating system?
+
+5.20150916+gitg79661ef-1~ndall+1
+
+[[!meta author=yoh]]
+
+Since packaged version suggestively (I think I did check) resolved the issue, marking it as [[done]]
diff --git a/doc/bugs/startup_scan_extremely_slow.mdwn b/doc/bugs/startup_scan_extremely_slow.mdwn
new file mode 100644
index 0000000000..449c88e94d
--- /dev/null
+++ b/doc/bugs/startup_scan_extremely_slow.mdwn
@@ -0,0 +1,69 @@
+### Please describe the problem.
+
+When the assistant starts it takes several hours to do the startup scan, even when there are no files to add.
+
+The repo contains many small files but it is configured to add the smaller ones via gitattributes. In particular there are: 91949 files added to git repo and 1029 annexed.
+This is my gitattributes
+
+    * annex.largefiles=(largerthan=500kb)
+
+annex.addunlocked is set to true
+
+### What steps will reproduce the problem?
+
+Create a repo with ~90000 files smaller than 500k and ~1000 files larger (in my case ranging from 500k to 32M). Set addunlocked to true and annex.largefiles to largerthan=500kb. Start the assistant and let it finish adding the files. Restart the assistant.
+
+### What version of git-annex are you using? On what operating system?
+
+git-annex version: 6.20160318
+build flags: Assistant Webapp Pairing Testsuite S3(multipartupload)(storageclasses) WebDAV Inotify DBus DesktopNotify XMPP ConcurrentOutput TorrentParser MagicMime Feeds Quvi
+key/value backends: SHA256E SHA256 SHA512E SHA512 SHA224E SHA224 SHA384E SHA384 SHA3_256E SHA3_256 SHA3_512E SHA3_512 SHA3_224E SHA3_224 SHA3_384E SHA3_384 SKEIN256E SKEIN256 SKEIN512E SKEIN512 SHA1E SHA1 MD5E MD5 WORM URL
+remote types: git gcrypt S3 bup directory rsync web bittorrent webdav tahoe glacier ddar hook external
+local repository version: 6
+
+I'm running it on Arch Linux (packaged version)
+
+### Please provide any additional information below.
+
+[[!format sh """
+
+[2016-03-29 22:08:26.356586] main: starting assistant version 6.20160318
+
+  No known network monitor available through dbus; falling back to polling
+(scanning...) [2016-03-29 22:08:41.426049] Watcher: Performing startup scan
+[2016-03-29 23:05:40.533113] Committer: Committing changes to git
+(recording state in git...)
+[2016-03-30 00:10:07.085051] Committer: Committing changes to git
+(recording state in git...)
+[2016-03-30 01:23:29.784236] Committer: Committing changes to git
+(recording state in git...)
+[2016-03-30 02:43:02.048312] Committer: Committing changes to git
+(recording state in git...)
+[2016-03-30 03:37:53.273057] Committer: Committing changes to git
+(recording state in git...)
+[2016-03-30 04:04:56.875573] Committer: Committing changes to git
+(recording state in git...)
+[2016-03-30 04:31:14.370618] Committer: Committing changes to git
+(recording state in git...)
+[2016-03-30 04:56:12.467889] Committer: Committing changes to git
+(recording state in git...)
+[2016-03-30 05:21:09.021728] Committer: Committing changes to git
+(recording state in git...)
+[2016-03-30 05:43:11.111616] Committer: Committing changes to git
+(recording state in git...)
+[2016-03-30 06:14:38.096425] Committer: Committing changes to git
+(recording state in git...)
+[2016-03-30 06:49:54.730879] Committer: Committing changes to git
+(recording state in git...)
+[2016-03-30 07:26:47.721929] Committer: Committing changes to git
+(recording state in git...)
+
+
+# End of transcript or log.
+"""]]
+
+At this point I stopped the assistant that was still doing the startup scan...
+
+### Have you had any luck using git-annex before?
+
+Sure!
diff --git a/doc/bugs/startup_scan_extremely_slow/comment_1_68f88f04c179800c0ce1ff6925d4c6ab._comment b/doc/bugs/startup_scan_extremely_slow/comment_1_68f88f04c179800c0ce1ff6925d4c6ab._comment
new file mode 100644
index 0000000000..ea3347db7f
--- /dev/null
+++ b/doc/bugs/startup_scan_extremely_slow/comment_1_68f88f04c179800c0ce1ff6925d4c6ab._comment
@@ -0,0 +1,7 @@
+[[!comment format=mdwn
+ username="zarel"
+ subject="comment 1"
+ date="2016-04-02T20:22:23Z"
+ content="""
+I suppose that the problem is strictly assistant related since both \"git annex status\" and \"git status\" give me the correct status when new files are present in a couple of seconds the first time and in a fraction of a second in the subsequent calls
+"""]]
diff --git a/doc/bugs/startup_scan_extremely_slow/comment_2_116e850299372027481c9d0a1667d5af._comment b/doc/bugs/startup_scan_extremely_slow/comment_2_116e850299372027481c9d0a1667d5af._comment
new file mode 100644
index 0000000000..03683dea1b
--- /dev/null
+++ b/doc/bugs/startup_scan_extremely_slow/comment_2_116e850299372027481c9d0a1667d5af._comment
@@ -0,0 +1,25 @@
+[[!comment format=mdwn
+ username="joey"
+ subject="""comment 2"""
+ date="2016-04-04T20:23:19Z"
+ content="""
+Note that v6 is still an experimental feature. I have not tested the
+assistant with it much.
+
+There is an issue documented on [[smudge]] where git can end up
+unncessarily running the smudge filter after git-annex eg, gets a file,
+or adds a file.
+
+This could be related to that; after the assistant added a lot of
+files here, the first `git status` run was quite slow as it ran the clean
+filter on every file. Subsequent `git status` runs then went fast.
+
+But, I don't know why this would make the startup scan slow; it
+doesn't seem to use any git commands that would need to smudge files.
+I tested by exporting `GIT_TRACE=1` and starting the assistant; the
+startup scan went fast and there was nothing in .git/annex/daemon.log
+about smudging.
+
+Also, what are these changes that are apparently being committed to git
+during your startup scan? I don't see such commits, either here.
+"""]]
diff --git a/doc/bugs/startup_scan_extremely_slow/comment_3_3eb4c2290e560451e7341b85b58212b0._comment b/doc/bugs/startup_scan_extremely_slow/comment_3_3eb4c2290e560451e7341b85b58212b0._comment
new file mode 100644
index 0000000000..a2ac70fae1
--- /dev/null
+++ b/doc/bugs/startup_scan_extremely_slow/comment_3_3eb4c2290e560451e7341b85b58212b0._comment
@@ -0,0 +1,14 @@
+[[!comment format=mdwn
+ username="zarel"
+ subject="comment 3"
+ date="2016-04-05T17:24:06Z"
+ content="""
+You're right, after the initial run there is no smudge filter overhead. I ran htop in tree view and got this result:
+
+    git-annex-assistant (55-60%)
+      |--> git-annex-assistant (six instances, a couple of those with load 20-40%, the others idling at 0%)
+      |--> git ls-files (mostly idle)
+      |--> git check-attr (mostly idle)
+      |--> git check-ignore (mostly idle)
+      |--> git cat-file (stuck at 40-60%)
+"""]]
diff --git a/doc/bugs/startup_scan_extremely_slow/comment_4_ade878000a050c87a345e8bc9914deba._comment b/doc/bugs/startup_scan_extremely_slow/comment_4_ade878000a050c87a345e8bc9914deba._comment
new file mode 100644
index 0000000000..76cf8faedf
--- /dev/null
+++ b/doc/bugs/startup_scan_extremely_slow/comment_4_ade878000a050c87a345e8bc9914deba._comment
@@ -0,0 +1,7 @@
+[[!comment format=mdwn
+ username="zarel"
+ subject="comment 4"
+ date="2016-04-22T15:27:20Z"
+ content="""
+git-annex updated to the last version. .git removed, repo recreated from scratch in v5 direct mode and it still happens. I'll try to export GIT_TRACE and understand why the \"cat-file\" is taking 50% of a cpu
+"""]]
diff --git a/doc/bugs/startup_scan_extremely_slow/comment_5_4ee924e70d5a8e35e706eaaff4cc76e1._comment b/doc/bugs/startup_scan_extremely_slow/comment_5_4ee924e70d5a8e35e706eaaff4cc76e1._comment
new file mode 100644
index 0000000000..d0fa0720ed
--- /dev/null
+++ b/doc/bugs/startup_scan_extremely_slow/comment_5_4ee924e70d5a8e35e706eaaff4cc76e1._comment
@@ -0,0 +1,28 @@
+[[!comment format=mdwn
+ username="zarel"
+ subject="comment 5"
+ date="2016-04-24T15:31:05Z"
+ content="""
+I exported GIT_TRACE and discovered that the assistant is re-adding to git the small files everytime I run it.
+Below there's a link to the trace (with filenames removed). All the files are not matching the annex.largefiles expression (which is okay), there are no duplicates (i.e. it's not a \"I'm adding the same files in the same run\" problem) and all of them were already added the first time I ran the assistant and are not new to git and git-annex as they are not shown in \"git status\" or in \"git annex status\".
+
+The daemon.log is pretty brief:
+
+    [2016-04-24 12:49:31.129582] main: starting assistant version 6.20160418
+    
+      No known network monitor available through dbus; falling back to polling
+    (scanning...) [2016-04-24 12:49:55.098167] Watcher: Performing startup scan
+    [2016-04-24 12:58:51.667861] Committer: Committing changes to git
+    (recording state in git...)
+    [2016-04-24 13:08:25.154713] Committer: Committing changes to git
+    (recording state in git...)
+
+You can find the trace here: 
+
+As you can see it almost takes ten minutes before reaching the first git-add.
+
+My git-annex version is now 6.20160418 and the git one is 2.8.0, what version of git did you use when tracing? I can try to reproduce in a different environment (e.g. Debian stable with backports) if it can be useful.
+
+Cheers,
+David
+"""]]
diff --git a/doc/bugs/strips___95___from_extensions_in_E_backends__63__.mdwn b/doc/bugs/strips___95___from_extensions_in_E_backends__63__.mdwn
new file mode 100644
index 0000000000..e0785b30b8
--- /dev/null
+++ b/doc/bugs/strips___95___from_extensions_in_E_backends__63__.mdwn
@@ -0,0 +1,17 @@
+in my obscure case filename is ds001_R1.1.0_raw.tgz  and resultant extension annex takes for the E backend is .0raw.tgz which is formed from .0_raw.tgz with _ removed.  IMHO if _ is not expected in the extensions, the target extension then should have been just .tgz.  If it does expect/allow for _ in extension -- should have been .0_raw.tgz
+
+[[!format sh """
+# If you can, paste a complete transcript of the problem occurring here.
+# If the problem is with the git-annex assistant, paste in .git/annex/daemon.log
+$> f=ds001_R1.1.0_raw.tgz; rm -rf /tmp/repo1; mkdir /tmp/repo1; cd /tmp/repo1; git init; git annex init ; echo 123>$f; git annex add --backend MD5E $f; ls -ld $f              
+Initialized empty Git repository in /tmp/repo1/.git/
+init  ok
+(recording state in git...)
+add ds001_R1.1.0_raw.tgz ok
+(recording state in git...)
+lrwxrwxrwx 1 yoh yoh 126 May 25 14:27 ds001_R1.1.0_raw.tgz -> .git/annex/objects/g5/jW/MD5E-s4--ba1f2511fc30423bdbb183fe33f3dd0f.0raw.tgz/MD5E-s4--ba1f2511fc30423bdbb183fe33f3dd0f.0raw.tgz
+"""]]
+
+[[!meta author=yoh]]
+
+> [[fixed|done]] --[[Joey]]
diff --git a/doc/bugs/suggests_to_enable_web_remote_even_when_there_is_no_web_urls_for_the_file.mdwn b/doc/bugs/suggests_to_enable_web_remote_even_when_there_is_no_web_urls_for_the_file.mdwn
new file mode 100644
index 0000000000..dd4566f918
--- /dev/null
+++ b/doc/bugs/suggests_to_enable_web_remote_even_when_there_is_no_web_urls_for_the_file.mdwn
@@ -0,0 +1,42 @@
+### Please describe the problem.
+
+Originally spotted in 
+https://neurostars.org/t/updating-datalad-datasets/1154/11 and thought that the
+guy is just offline and that is why can't get those "web" links, but apparently
+there were no web links for that file, so the message like
+
+[[!format sh """
+Try making some of these repositories available:
+    00000000-0000-0000-0000-000000000001 -- web
+...
+"""]]
+
+was a bit misleading.  Here is the full whereis  for that file (since
+then I've populated git-annex with urls, so if you would clone it you would get
+some http urls for web remote):
+
+[[!format sh """
+                              
+$> git annex whereis sub-10159/anat/sub-10159_T1w.nii.gz
+whereis sub-10159/anat/sub-10159_T1w.nii.gz (3 copies)
+    00000000-0000-0000-0000-000000000001 -- web
+    09ede57e-5ec2-484b-b6fb-8a632e5c7a4e -- [datalad-archives]
+    41f07c30-3cfc-4de3-9fbc-84383f5156e6 -- yoh@smaug:/mnt/btrfs/datasets/datalad/crawl/openfmri/ds000030
+                              
+  datalad-archives: dl+archive:MD5E-s3920586194--f5ecaf1365ea031dd6c20d0f958ed69b.tgz#path=ds030_R1.0.0/sub-10159/anat/sub-10159_T1w.nii.gz&size=11637742
+  datalad-archives: dl+archive:MD5E-s3920586194--f5ecaf1365ea031dd6c20d0f958ed69b.tgz/ds030_R1.0.0/sub-10159/anat/sub-10159_T1w.nii.gz#size=11637742
+  datalad-archives: dl+archive:MD5E-s4347673658--836cb09310fa22f7d2112c7f81e6258b.tgz#path=ds000030/sub-10159/anat/sub-10159_T1w.nii.gz&size=11637742
+  datalad-archives: dl+archive:MD5E-s4349211504--2fe25908e474d782e8963fd31d6fe4b5.zip#path=ds000030/sub-10159/anat/sub-10159_T1w.nii.gz&size=11637742
+  datalad-archives: dl+archive:MD5E-s4802398120--ce2d215f336e6dfa282d69cc35beb80d.tgz#path=sub-10159/anat/sub-10159_T1w.nii.gz&size=11637742
+ok
+"""]]
+
+
+### What version of git-annex are you using? On what operating system?
+
+6.20180807+git63-gbafc55c4a-1~ndall+1
+
+
+[[!meta name=yoh]]
+
+> closing as yoh says "nothing to fix" [[done]] --[[Joey]]
diff --git a/doc/bugs/suggests_to_enable_web_remote_even_when_there_is_no_web_urls_for_the_file/comment_1_fd32c6a7e5ac6b0ac721dd9edd824c29._comment b/doc/bugs/suggests_to_enable_web_remote_even_when_there_is_no_web_urls_for_the_file/comment_1_fd32c6a7e5ac6b0ac721dd9edd824c29._comment
new file mode 100644
index 0000000000..6c1a22ad35
--- /dev/null
+++ b/doc/bugs/suggests_to_enable_web_remote_even_when_there_is_no_web_urls_for_the_file/comment_1_fd32c6a7e5ac6b0ac721dd9edd824c29._comment
@@ -0,0 +1,28 @@
+[[!comment format=mdwn
+ username="joey"
+ subject="""comment 1"""
+ date="2018-08-21T15:38:51Z"
+ content="""
+Note that git-annex get from the web remote in this sitation says
+"no known url" before the message you quoted.
+
+Since location tracking and urls are separate peices of information, it's
+certianly possible for them to be inconsistent like this.
+
+It doesn't normally happen though. Eg, `git annex rmurl` of the last url
+of a remote will update location tracking to say it's missing.
+(Special remote SETURLMISSING does the same thing.)
+
+One way it can happen is using `git-annex setpresentkey` 
+to say a key is present in the web, and not setting an url.
+But that's plumbing and it's up the user to use it consistently.
+
+If you have some other way to get into this state that doesn't involve
+plumbing commands, that could be a bug.
+
+It might be possible to add something linking the data so that
+location tracking for the web is ignored if it doesn't have urls, which
+would also need to apply to some other special remotes that use urls, but
+not others that only might have an associated url. I'm doubtful of this
+idea tho.
+"""]]
diff --git a/doc/bugs/suggests_to_enable_web_remote_even_when_there_is_no_web_urls_for_the_file/comment_2_20e02b8cc70c602069b80086544a9fea._comment b/doc/bugs/suggests_to_enable_web_remote_even_when_there_is_no_web_urls_for_the_file/comment_2_20e02b8cc70c602069b80086544a9fea._comment
new file mode 100644
index 0000000000..ab8537d7a4
--- /dev/null
+++ b/doc/bugs/suggests_to_enable_web_remote_even_when_there_is_no_web_urls_for_the_file/comment_2_20e02b8cc70c602069b80086544a9fea._comment
@@ -0,0 +1,9 @@
+[[!comment format=mdwn
+ username="yarikoptic"
+ avatar="http://cdn.libravatar.org/avatar/f11e9c84cb18d26a1748c33b48c924b4"
+ subject="comment 2"
+ date="2018-08-21T16:23:05Z"
+ content="""
+Thanks for the analysis!
+Apparently we did have some web URLs available for that key awhile back, then bucket was redone so they were marked not available.  I wonder if may be for that batch fixup I just did it \"manually\" instead of using `rmurl` which would have taken care about it.  So I guess nothing to fix ;-)
+"""]]
diff --git a/doc/bugs/support_bare_git_repo__44___with_the_annex_directory_exposed_to_http.mdwn b/doc/bugs/support_bare_git_repo__44___with_the_annex_directory_exposed_to_http.mdwn
new file mode 100644
index 0000000000..ba7dcad300
--- /dev/null
+++ b/doc/bugs/support_bare_git_repo__44___with_the_annex_directory_exposed_to_http.mdwn
@@ -0,0 +1,20 @@
+Let's say that http://people.collabora.com/~alsuren/git/fate-suite.git/ is a bare git repo. It has been 'git update-server-info'd so that it can be served on a dumb http server.
+
+The repo is also a git annex remote, created using the following commands:
+
+* git remote add alsuren git+ssh://people.collabora.co.uk/user/alsuren/public_html/fate-suite.git
+* git push alsuren --all
+* git annex copy --to=alsuren
+
+so http://people.collabora.com/~alsuren/git/fate-suite.git/annex is a valid git annex (though listing dirs is forbidden, so you need to know the filenames ahead of time).
+
+I would like to be able to use the following commands to get a clone of the repo:
+
+* git clone http://people.collabora.com/~alsuren/git/fate-suite.git/
+* cd fate-suite
+* git annex get
+
+This would allow contributors to quickly get a copy of our upstream repo and start contributing with minimal bandwidth/effort.
+
+> This is now supported.. I look forward to seeing your project using it!
+> --[[Joey]] [[!tag done]]
diff --git a/doc/bugs/sync-git-annex_branch_not_syncing_in_the_assistant.mdwn b/doc/bugs/sync-git-annex_branch_not_syncing_in_the_assistant.mdwn
new file mode 100644
index 0000000000..d7c679e22b
--- /dev/null
+++ b/doc/bugs/sync-git-annex_branch_not_syncing_in_the_assistant.mdwn
@@ -0,0 +1,49 @@
+### Please describe the problem.
+
+It seems that the `synced/git-annex` branch (which I am a little confused about in the first place), doesn't get synced automatically by the assistant.
+
+I was expecting the assistant to regularly do the equivalent of `git annex sync`. However, after a client pushed changes to the `git-annex` branch, I had to manually do a `git annex sync` for the changes of the `synced/git-annex` branch to be merged down into the local `git-annex`.
+
+### What steps will reproduce the problem?
+
+I have 3 machines in this setup (to simplify: there are more, but those are sufficient). Let's call them foo, bar and quux. foo and quuex are connected to bar through password-less SSH connexions.
+
+foo commits a file to git-annex. the assistant syncs that to bar in the `synced/git-annex` branch.
+
+quux syncs with bar, and seems to ignore the `synced/git-annex` branch.
+
+git-annex sync on quux syncs the `synced/git-annex` branch into the local `git-annex`, working around the issue.
+
+### What version of git-annex are you using? On what operating system?
+
+foo is 5.20150610+gitg608172f-1~ndall+1 on Debian 7.8 (wheezy).
+
+bar and quux are 5.20150409+git126-ga29f683-1~ndall+1 and 5.20150610+gitg608172f-1~ndall+1 (respectively) on Ubuntu 12.04 (precise) .
+
+### Please provide any additional information below.
+
+I guess a more general question is how and how often do those branches get merged by the assistant... it's still unclear to me how this works.
+
+[[!format sh """
+$ sudo -u www-data -H git annex sync
+(merging origin/synced/git-annex into git-annex...)
+(recording state in git...)
+commit  ok
+pull origin
+Auto packing the repository in background for optimum performance.
+See "git help gc" for manual housekeeping.
+
+Already up-to-date.
+ok
+push origin
+Counting objects: 373637, done.
+# End of transcript or log.
+"""]]
+
+This was started at 20:23 UTC. Note that the sync had run previously under the assistant:
+
+
    +[2015-07-22 20:09:06 UTC] RemoteControl: Syncing with origin
+
    
+
+Available, as usual, for further debugging. :) -- [[anarcat]]
diff --git a/doc/bugs/sync-git-annex_branch_not_syncing_in_the_assistant/comment_1_b832288844a3e7e944fa08a6edb97d29._comment b/doc/bugs/sync-git-annex_branch_not_syncing_in_the_assistant/comment_1_b832288844a3e7e944fa08a6edb97d29._comment
new file mode 100644
index 0000000000..f06ebdaa77
--- /dev/null
+++ b/doc/bugs/sync-git-annex_branch_not_syncing_in_the_assistant/comment_1_b832288844a3e7e944fa08a6edb97d29._comment
@@ -0,0 +1,18 @@
+[[!comment format=mdwn
+ username="joey"
+ subject="""comment 1"""
+ date="2015-09-21T19:08:28Z"
+ content="""
+The assistant merges the synced/git-annex branch whenever it detects (via
+inotify) that a change has been made to that git ref. 
+
+I can't reproduce a problem from the information you've given. With
+`foo -> bar <- quux`, and assistants running on foo and quux, any file
+written to foo more or less immediately syncs to quux, and the git-annex
+branch is merged.
+
+Even when I stop the assistant running on foo, and manually run
+`git push bar git-annex:synced/git-annex`, quux immediately notices that
+bar has received that synced/git-annex, and fetches and merges it into its
+own git-annex branch.
+"""]]
diff --git a/doc/bugs/sync-git-annex_branch_not_syncing_in_the_assistant/comment_2_fbfc8929497f609916d9f3e5a0a9b531._comment b/doc/bugs/sync-git-annex_branch_not_syncing_in_the_assistant/comment_2_fbfc8929497f609916d9f3e5a0a9b531._comment
new file mode 100644
index 0000000000..d21e584a69
--- /dev/null
+++ b/doc/bugs/sync-git-annex_branch_not_syncing_in_the_assistant/comment_2_fbfc8929497f609916d9f3e5a0a9b531._comment
@@ -0,0 +1,18 @@
+[[!comment format=mdwn
+ username="joey"
+ subject="""comment 2"""
+ date="2015-09-21T19:21:30Z"
+ content="""
+Note that I'm assuming that bar is a remote of both foo and of quux.
+That seems like a reasonable reading of what you described, and is a
+reasonable configuration, but perhaps you were trying to describe some
+other configuration. 
+
+In [[forum/mesh_configurations]] you seem to instead
+describe a chain of remotes, where bar is a remote of foo, and quux is a
+remote of bar: `foo -> bar -> quux`. In this configuration, running the assistant on 
+foo and quux, but not on bar, would not result in changes made on quux
+ever getting to foo, because quux has no way to push objects to bar. So, I
+don't think that's the configuration you meant either. But lack of clarity
+in bug report makes it hard to know.
+"""]]
diff --git a/doc/bugs/sync_claims_data_loss_but_seems_to_just_lose_tracking.mdwn b/doc/bugs/sync_claims_data_loss_but_seems_to_just_lose_tracking.mdwn
new file mode 100644
index 0000000000..3223c61c05
--- /dev/null
+++ b/doc/bugs/sync_claims_data_loss_but_seems_to_just_lose_tracking.mdwn
@@ -0,0 +1,215 @@
+### Please describe the problem.
+Sync / Assistant between a `source` and `transfer` repo results in zero copies for whereis, but data is available.
+
+
+### What steps will reproduce the problem?
+See a script below...
+
+``` sh
+#!/bin/sh
+
+TMP=$(mktemp -d)
+
+
+##########################################
+# Set up the repos
+##########################################
+
+
+###
+# Create working directories
+cd ${TMP}
+mkdir source transfer
+
+
+###
+# Set up the source repo
+cd ${TMP}/source
+git init
+git annex init source
+git-annex group here source
+git annex wanted here standard
+
+
+###
+# Set up the transfer repo
+# N.B. It can see the source repo
+cd ${TMP}/transfer
+git init
+git remote add source ${TMP}/source
+git annex init transfer
+git-annex group here transfer
+git annex wanted here \
+  'not (inallgroup=client and copies=client:1) and ((include=* and ((exclude=*/archive/* and exclude=archive/*) or (not (copies=archive:1 or copies=smallarchive:1)))) or approxlackingcopies=1)'
+
+
+##########################################
+# Add some files and run the assistant
+##########################################
+
+###
+# Make sure it works by adding some files to the source...
+cd ${TMP}/source
+date > file.1
+date > file.2
+date > file.3
+
+# Run the assistant in the source...
+(cd ${TMP}/source && \
+    git annex assistant && \
+    sleep 30 && \
+    git annex assistant --stop && \
+    git annex whereis file.*)
+# PASS: All files are in source.
+
+# Now run it in the transfer group
+(cd ${TMP}/transfer && \
+    git annex assistant && \
+    sleep 30 && \
+    git annex assistant --stop && \
+    git annex whereis file.*)
+# PASS: All files are in transfer
+
+# Check the source
+(cd ${TMP}/source && \
+    git annex whereis file.*)
+# FAIL: Only file.1 was updated to be in source.
+
+# whereis file.1 (1 copy)
+#        64b193e0-86f0-4349-b073-70af919ce628 -- transfer
+# ok
+# whereis file.2 (0 copies) failed
+# whereis file.3 (0 copies) failed
+# git-annex: whereis: 2 failed
+
+
+##########################################
+# Start to scratch head and ask 'Why?'
+##########################################
+
+##########
+# What's happening???
+
+cd ${TMP}/source && git annex sync
+# Source thinks it is updated - working tree clean
+
+cd ${TMP}/transfer && git annex sync
+# Transfer pulls from source, so run again
+git annex sync
+# OK... working tree clean
+
+
+# Check again
+cd ${TMP}/source && git annex whereis file.*
+# whereis file.1 (1 copy)
+#         64b193e0-86f0-4349-b073-70af919ce628 -- transfer
+# ok
+# whereis file.2 (0 copies) failed
+# whereis file.3 (0 copies) failed
+# git-annex: whereis: 2 failed
+
+cd ${TMP}/transfer && git annex whereis file.*
+# whereis file.1 (1 copy)
+#         64b193e0-86f0-4349-b073-70af919ce628 -- transfer [here]
+# ok
+# whereis file.2 (0 copies) failed
+# whereis file.3 (0 copies) failed
+# git-annex: whereis: 2 failed
+
+
+# HOWEVER...  Files are definitely in the transfer repo
+
+cd ${TMP}/source && cat file.*
+# cat: file.1: No such file or directory
+# cat: file.2: No such file or directory
+# cat: file.3: No such file or directory
+
+
+cd ${TMP}/transfer && cat file.*
+# Sun May 14 21:22:02 PDT 2017
+# Sun May 14 21:22:03 PDT 2017
+# Sun May 14 21:22:06 PDT 2017
+```
+
+
+
+### What version of git-annex are you using? On what operating system?
+
+#### Version:
+```
+git-annex version: 6.20170510
+build flags: Assistant Webapp Pairing Testsuite S3(multipartupload)(storageclasses) WebDAV FsEvents ConcurrentOutput TorrentParser MagicMime Feeds Quvi
+dependency versions: aws-0.16 bloomfilter-2.0.1.0 cryptonite-0.23 DAV-1.3.1 feed-0.3.12.0 ghc-8.0.2 http-client-0.5.6.1 persistent-sqlite-2.6.2 torrent-10000.1.1 uuid-1.3.13 yesod-1.4.5
+key/value backends: SHA256E SHA256 SHA512E SHA512 SHA224E SHA224 SHA384E SHA384 SHA3_256E SHA3_256 SHA3_512E SHA3_512 SHA3_224E SHA3_224 SHA3_384E SHA3_384 SKEIN256E SKEIN256 SKEIN512E SKEIN512 SHA1E SHA1 MD5E MD5 WORM URL
+remote types: git gcrypt p2p S3 bup directory rsync web bittorrent webdav tahoe glacier ddar hook external
+local repository version: 5
+supported repository versions: 3 5 6
+upgrade supported from repository versions: 0 1 2 3 4 5
+operating system: darwin x86_64
+```
+
+#### OS:
+MacOS 10.12.4, but seems to apply to Debian Jessie too.
+
+
+### Please provide any additional information below.
+
+Annex assistant daemon.log for `source` repo:
+
+``` sh
+[2017-05-15 14:11:24.150202] main: starting assistant version 6.20170510
+[2017-05-15 14:11:24.196597] Cronner: You should enable consistency checking to protect your data.
+(scanning...) [2017-05-15 14:11:24.257123] Watcher: Performing startup scan
+(started...)
+[2017-05-15 14:11:45.505855] Committer: Adding file.1 file.2 file.3
+add file.1 ok
+add file.2 ok
+add file.3 ok
+[2017-05-15 14:11:45.544277] Committer: Committing changes to git
+(recording state in git...)
+(recording state in git...)
+```
+
+Annex assistant daemon.log for `transfer` repo:
+
+``` sh
+[2017-05-15 14:12:09.127406] main: starting assistant version 6.20170510
+[2017-05-15 14:12:09.157611] Cronner: You should enable consistency checking to protect your data.
+[2017-05-15 14:12:09.177452] TransferScanner: Syncing with source
+(scanning...) [2017-05-15 14:12:09.210605] Watcher: Performing startup scan
+(started...)
+warning: no common commits
+From /Users/olaf/tmp/git-annex/play/bug-play/source
+ * [new branch]      git-annex     -> source/git-annex
+ * [new branch]      master        -> source/master
+ * [new branch]      synced/master -> source/synced/master
+(merging source/git-annex into git-annex...)
+(recording state in git...)
+
+
+Already up-to-date.
+[2017-05-15 14:12:10.256195] Committer: Committing changes to git
+(recording state in git...)
+(checksum...) [2017-05-15 14:12:10.425494] Transferrer: Downloaded file.1
+[2017-05-15 14:12:11.17664] Pusher: Syncing with source
+(recording state in git...)
+[2017-05-15 14:12:11.356447] Committer: Committing changes to git
+(recording state in git...)
+To /Users/olaf/tmp/git-annex/play/bug-play/source
+ * [new branch]      git-annex -> synced/git-annex
+(checksum...) [2017-05-15 14:12:11.428676] Transferrer: Downloaded file.2
+(checksum...) [2017-05-15 14:12:11.451457] Transferrer: Downloaded file.3
+drop source file.1 ok
+drop source file.2 ok
+drop source file.3 ok
+[2017-05-15 14:12:13.477636] Pusher: Syncing with source
+(recording state in git...)
+To /Users/olaf/tmp/git-annex/play/bug-play/source
+   825d3dd..60c96cb  git-annex -> synced/git-annex
+```
+
+
+### Have you had any luck using git-annex before? (Sometimes we get tired of reading bug reports all day and a lil' positive end note does wonders)
+
+Yes.  Love it.  Donated.  Have been using it for years.  Recommend it and get(/force) my collaborators to use it.  ;-)
+
diff --git a/doc/bugs/sync_claims_data_loss_but_seems_to_just_lose_tracking/comment_1_562cf3ab3398c1ef874c0bfe5d89e47b._comment b/doc/bugs/sync_claims_data_loss_but_seems_to_just_lose_tracking/comment_1_562cf3ab3398c1ef874c0bfe5d89e47b._comment
new file mode 100644
index 0000000000..7511c36898
--- /dev/null
+++ b/doc/bugs/sync_claims_data_loss_but_seems_to_just_lose_tracking/comment_1_562cf3ab3398c1ef874c0bfe5d89e47b._comment
@@ -0,0 +1,10 @@
+[[!comment format=mdwn
+ username="leavingchicago@c04c893e78d1c4c76cb3e32b5c227cf42bbf7682"
+ nickname="leavingchicago"
+ avatar="http://cdn.libravatar.org/avatar/4ae498d3d6ee558d6b65caa658f72572"
+ subject="Related..."
+ date="2017-05-15T04:43:44Z"
+ content="""
+I think this bug is related to the problems that I'm having with [/forum/Get source group to automatically drop with assistant](https://git-annex.branchable.com/forum/Get___39__source__39___group_to_automatically_drop_with_assistant/)
+
+"""]]
diff --git a/doc/bugs/sync_claims_data_loss_but_seems_to_just_lose_tracking/comment_2_f686273c6be899d2a8bbaabd03a47fbf._comment b/doc/bugs/sync_claims_data_loss_but_seems_to_just_lose_tracking/comment_2_f686273c6be899d2a8bbaabd03a47fbf._comment
new file mode 100644
index 0000000000..34ed9f1d7d
--- /dev/null
+++ b/doc/bugs/sync_claims_data_loss_but_seems_to_just_lose_tracking/comment_2_f686273c6be899d2a8bbaabd03a47fbf._comment
@@ -0,0 +1,14 @@
+[[!comment format=mdwn
+ username="joey"
+ subject="""comment 2"""
+ date="2017-06-09T17:25:23Z"
+ content="""
+I have not had any luck with reproducing the problem, using
+your script, on Debian unstable. The transfer repo gets
+the file contents and both it and the source repo knows where they are.
+
+Do you see the same behavior if, rather than running the assistant
+for 30 seconds, you run `git annex sync --content` ?
+
+Are you using git-annex in direct mode and/or on an unusual filesystem?
+"""]]
diff --git a/doc/bugs/sync_claims_data_loss_but_seems_to_just_lose_tracking/comment_3_fd518d0601587dc5c718944d0b6f51fc._comment b/doc/bugs/sync_claims_data_loss_but_seems_to_just_lose_tracking/comment_3_fd518d0601587dc5c718944d0b6f51fc._comment
new file mode 100644
index 0000000000..79e388a4dc
--- /dev/null
+++ b/doc/bugs/sync_claims_data_loss_but_seems_to_just_lose_tracking/comment_3_fd518d0601587dc5c718944d0b6f51fc._comment
@@ -0,0 +1,10 @@
+[[!comment format=mdwn
+ username="joey"
+ subject="""comment 3"""
+ date="2017-06-09T17:33:17Z"
+ content="""
+I tried the script with `git annex direct` added at initialization,
+and didn't see the problem as described, but the assistant
+did fail to get the content into the transfer repo at all
+when using direct mode. `git annex sync --content` had no difficulty.
+"""]]
diff --git a/doc/bugs/sync_claims_data_loss_but_seems_to_just_lose_tracking/comment_4_a6074b68754b0d773385a1e406043952._comment b/doc/bugs/sync_claims_data_loss_but_seems_to_just_lose_tracking/comment_4_a6074b68754b0d773385a1e406043952._comment
new file mode 100644
index 0000000000..9eb06cf992
--- /dev/null
+++ b/doc/bugs/sync_claims_data_loss_but_seems_to_just_lose_tracking/comment_4_a6074b68754b0d773385a1e406043952._comment
@@ -0,0 +1,10 @@
+[[!comment format=mdwn
+ username="olaf"
+ avatar="http://cdn.libravatar.org/avatar/4ae498d3d6ee558d6b65caa658f72572"
+ subject="comment 4"
+ date="2017-06-22T04:55:35Z"
+ content="""
+So, you're suggesting manually running `git annex sync --content`?  I can `cron` that up, so I can live with it.
+
+Do you need any more info to identify the issue you found with the assistant?
+"""]]
diff --git a/doc/bugs/sync_deletes_files.mdwn b/doc/bugs/sync_deletes_files.mdwn
new file mode 100644
index 0000000000..da3afad5ed
--- /dev/null
+++ b/doc/bugs/sync_deletes_files.mdwn
@@ -0,0 +1,44 @@
+### Please describe the problem.
+
+Syncing two repositories causes many files that are on both systems to be deleted.
+
+My computers A and B have a directory with 1600 files taking 500MB.   I'm trying to sync A to B thus:
+
+git annex sync --no-commit --no-push
+
+This immediately deletes those 1600 files from B.   (Before I added -no-push it also deleted those files from A).
+
+Recovering from this with git reset --hard takes 13 minutes, so it's difficult to run experiments.   However I have spent about a full day on this over the last week.
+
+### What steps will reproduce the problem?
+
+This repository has evolved over time so I can't give a MWE.   However, I created the git repository before initing annex, and did some more work before adding the largefiles option.   It's probable that something inside annex is now badly confused.
+
+I don't expect you to solve my particular problem from this report.   However, sync has a major hidden problem. 
+
+### What version of git-annex are you using? On what operating system?
+
+20160511-1.
+the annex is v6.
+the OS is Ubuntu linux  4.4.0-38-generic 
+
+### Please provide any additional information below.
+
+[[!format sh """
+# If you can, paste a complete transcript of the problem occurring here.
+# If the problem is with the git-annex assistant, paste in .git/annex/daemon.log
+
+
+# End of transcript or log.
+"""]]
+
+.git/annex has no log files, even in subdirs.
+
+There have been occasional reports going back years from other people reporting that sync destroyed massive amounts of their repositories.    That this bug persists suggests that the problem is subtle.
+
+I upgraded my annexes to v6 because the concept of locked and unlocked looks good.   Then, when researching my problem before posting this bug report, I found a comment that upgrading to v6 is not (now?) recommended because of possible bugs.   It would have been nice if this warning were more prominent.
+
+### Have you had any luck using git-annex before? (Sometimes we get tired of reading bug reports all day and a lil' positive end note does wonders)
+
+The idea is great.  Preliminary tests were quite positive.   That's why I tried it on my big repositories.
+
diff --git a/doc/bugs/sync_on_a_repository_on_removable_media_should_not_leave_ssh_conns_open.mdwn b/doc/bugs/sync_on_a_repository_on_removable_media_should_not_leave_ssh_conns_open.mdwn
new file mode 100644
index 0000000000..b2be4426d3
--- /dev/null
+++ b/doc/bugs/sync_on_a_repository_on_removable_media_should_not_leave_ssh_conns_open.mdwn
@@ -0,0 +1 @@
+I have a client, non-bare, direct mode repository on a USB key.  I use this to work on my files on Windows terminals at work.  When I return to my own (Debian) workstation at home, I mount the drive, `cd /media/swhitton/SPWHITTON/annex && git annex sync --content`.  This sync the contents of the key with both my workstation's `/home` and a remote VPS over SSH.  When the sync is complete, Thunar can't unmount the key because git annex leaves ssh connections to my VPS open, with the working directory inside `/media/swhitton/SPWHITTON/annex`.  So I have to run `pkill ssh`.  It would be great if `git annex sync` could detect that a non-bare repository is on removable media, and not leave SSH connections open when it's finished the sync.  Thanks!  --spwhitton
diff --git a/doc/bugs/sync_on_a_repository_on_removable_media_should_not_leave_ssh_conns_open/comment_1_bf457dfa628f8d140e91bcd2f3b04c51._comment b/doc/bugs/sync_on_a_repository_on_removable_media_should_not_leave_ssh_conns_open/comment_1_bf457dfa628f8d140e91bcd2f3b04c51._comment
new file mode 100644
index 0000000000..91cd91756d
--- /dev/null
+++ b/doc/bugs/sync_on_a_repository_on_removable_media_should_not_leave_ssh_conns_open/comment_1_bf457dfa628f8d140e91bcd2f3b04c51._comment
@@ -0,0 +1,14 @@
+[[!comment format=mdwn
+ username="joey"
+ subject="""comment 1"""
+ date="2016-04-20T16:47:50Z"
+ content="""
+git-annex always registers a cleanup action when it starts using a ssh
+connection caching socket. As long as git-annex gets a change to shut down
+cleanly (ie is not interrupted), it should run the action to stop the ssh
+connection caching.
+
+For example, git-annex sync --debug here shows as its second to last line:
+
+	[2016-04-20 12:47:40.069633] read: ssh ["-O","stop","-S","server","-o","ControlMaster=auto","-o","ControlPersist=yes","server"]
+"""]]
diff --git a/doc/bugs/sync_on_a_repository_on_removable_media_should_not_leave_ssh_conns_open/comment_2_08c25e314bf86b8a67c1730a02957300._comment b/doc/bugs/sync_on_a_repository_on_removable_media_should_not_leave_ssh_conns_open/comment_2_08c25e314bf86b8a67c1730a02957300._comment
new file mode 100644
index 0000000000..861ff1270a
--- /dev/null
+++ b/doc/bugs/sync_on_a_repository_on_removable_media_should_not_leave_ssh_conns_open/comment_2_08c25e314bf86b8a67c1730a02957300._comment
@@ -0,0 +1,7 @@
+[[!comment format=mdwn
+ username="spwhitton"
+ subject="comment 2"
+ date="2016-04-20T19:01:21Z"
+ content="""
+Thanks for letting me know about that functionality.  I'll try to isolate why it sometimes fails on my machine.
+"""]]
diff --git a/doc/bugs/sync_with_clone_protocol_error.mdwn b/doc/bugs/sync_with_clone_protocol_error.mdwn
new file mode 100644
index 0000000000..2ae0cab420
--- /dev/null
+++ b/doc/bugs/sync_with_clone_protocol_error.mdwn
@@ -0,0 +1,15 @@
+	Failed to get annex.uuid configuration of repository clam
+
+	Instead, got: "(merging origin/git-annex origin/synced/git-annex into git-annex...)\n(recording state in git...)\nannex.uuid=$obscured\ncore.gcrypt-id=\n"
+
+Seen after cloning the repository to clam, not running git-annex init
+in it, adding clam as a remote, and git annex sync clam.
+Apparently git-annex-shell outputs some merging messages in this 
+case, which breaks the protocol. 
+
+Next git-annex sync clam worked ok and got the UUID.
+
+clam has git-annex 6.20170101 installed.
+--[[Joey]]
+
+> [[fixed|done]] --[[Joey]]
diff --git a/doc/bugs/tahoe_remote_has_no_repair.mdwn b/doc/bugs/tahoe_remote_has_no_repair.mdwn
new file mode 100644
index 0000000000..1f34767a8e
--- /dev/null
+++ b/doc/bugs/tahoe_remote_has_no_repair.mdwn
@@ -0,0 +1,27 @@
+### Please describe the problem.
+
+The tahoe-lafs remote has no built-in way to perform the repair operation.
+This results to data loss if expiration is enabled on the Tahoe grid.
+
+For the current tahoe-lafs release (1.10.0), the only way storage space is freed
+is via garbage collection. Garbage collection removes shares whose lease has expired.
+Data loss will occur if leases are not periodically renewed via
+"tahoe repair --add-lease WRITECAP".
+
+The current implementation of the Tahoe remote in git-annex does not offer a way to
+run lease renewal, and cannot be used on grids where GC is enabled. (GC is not enabled
+in the default configuration, but on private grids it is a sensible option.)
+
+One way renewal could be made easier to do is to add the uploaded files to a directory
+in Tahoe, so that the leases could be easily updated if the directory writecap is known,
+without needing to go through the full list of writecaps for each file stored.
+
+### What steps will reproduce the problem?
+
+1. Use tahoe remote on a tahoe grid where GC is enabled.
+
+2. After GC expiration period, data loss ensues.
+
+### What version of git-annex are you using? On what operating system?
+
+Seems to affect current git master (as of 2014-08-24).
diff --git a/doc/bugs/the_assistant_fail_to_correctly_add_or_sync_ssh_remote_with_non_ascii_letter_in_the_directory.mdwn b/doc/bugs/the_assistant_fail_to_correctly_add_or_sync_ssh_remote_with_non_ascii_letter_in_the_directory.mdwn
new file mode 100644
index 0000000000..179e353c8c
--- /dev/null
+++ b/doc/bugs/the_assistant_fail_to_correctly_add_or_sync_ssh_remote_with_non_ascii_letter_in_the_directory.mdwn
@@ -0,0 +1,40 @@
+### Please describe the problem.
+I've tried to setup a remote using assistant, but it failed to correctly sync apparently because there is a "è" in the directory
+
+### What steps will reproduce the problem?
+run the webapp
+use "Add another directory"
+then Remote Server
+add host name, user name password and directory as "Bibliothèque calibre" (to a repository that already exists)
+
+the remote is added as metadata only and the assistant will try to upload every file to it every time it is restarted.
+
+if a symlink is created named "calibre library", and is used instead, there is no problem.
+
+### What version of git-annex are you using? On what operating system?
+it is git-annex version: 5.20150916-1 on debian/sid
+
+### Please provide any additional information below.
+
+the log is full of:
+[[!format sh """
+# If you can, paste a complete transcript of the problem occurring here.
+# If the problem is with the git-annex assistant, paste in .git/annex/daemon.log
+ssh: Could not resolve hostname git-annex-toubib.lot-of-stuff.info-moi_22_biblioth%c3%a8que.20calibre: Name or service not known
+ssh: Could not resolve hostname git-annex-volesprit.lot-of-stuff.info-moi_22_biblioth%c3%a8que.20calibre: Name or service not known
+ssh: Could not resolve hostname git-annex-toubib.lot-of-stuff.info-moi_22_biblioth%c3%a8que.20calibre: Name or service not known
+ssh: Could not resolve hostname git-annex-volesprit.lot-of-stuff.info-moi_22_biblioth%c3%a8que.20calibre: Name or service not known
+ssh: Could not resolve hostname git-annex-toubib.lot-of-stuff.info-moi_22_biblioth%c3%a8que.20calibre: Name or service not known
+sssh: Could not resolve hostname git-annex-toubib.lot-of-stuff.info-moi_22_biblioth%c3%a8que.20calibre: Name or service not known
+ssh: Could not resolve hostname git-annex-volesprit.lot-of-stuff.info-moi_22_biblioth%c3%a8que.20calibre: Name or service not known
+ssh: Could not resolve hostname git-annex-toubib.lot-of-stuff.info-moi_22_biblioth%c3%a8que.20calibre: Name or service not known
+ssh: Could not resolve hostname git-annex-volesprit.lot-of-stuff.info-moi_22_biblioth%c3%a8que.20calibre: Name or service not known
+ssh: Could not resolve hostname git-annex-toubib.lot-of-stuff.info-moi_22_biblioth%c3%a8que.2ssh: Could not resolve hostname git-annex-volesprit.lot-of-stuff.info-m
+
+
+# End of transcript or log.
+"""]]
+
+### Have you had any luck using git-annex before? (Sometimes we get tired of reading bug reports all day and a lil' positive end note does wonders)
+
+Adding remote that use only ascii letter work realy well with the webapp. 
diff --git a/doc/bugs/thread_blocked_indefinitely_in_an_STM_transaction__while_moving_within__a_local_clone.mdwn b/doc/bugs/thread_blocked_indefinitely_in_an_STM_transaction__while_moving_within__a_local_clone.mdwn
new file mode 100644
index 0000000000..eaf79a862d
--- /dev/null
+++ b/doc/bugs/thread_blocked_indefinitely_in_an_STM_transaction__while_moving_within__a_local_clone.mdwn
@@ -0,0 +1,40 @@
+relates to having pidlock true
+
+[[!format sh """
+$> mkdir 123; cd 123; git init; git annex init; git config annex.pidlock true && echo "123" > 123.dat; git annex add 123.dat; git commit -m 'added';
+W: git-annex repositories not (yet) supported in the prompt
+Initialized empty Git repository in /tmp/123/.git/
+init  ok
+(recording state in git...)
+add 123.dat ok
+(recording state in git...)
+[master (root-commit) 9449f1b] added
+ 1 file changed, 1 insertion(+)
+ create mode 120000 123.dat
+
+$> git clone . ../123-clone && git remote add clone ../123-clone && git fetch clone && cd ../123-clone && git config annex.pidlock true && cd - && git annex move --to=clone .
+Cloning into '../123-clone'...                                                                                                                                         
+done.
+From ../123-clone
+ * [new branch]      master     -> clone/master
+move 123.dat git-annex: thread blocked indefinitely in an STM transaction
+
+$> echo $?
+1
+
+$> git annex version
+git-annex version: 6.20160226+gitg01f1de0-1~ndall+1
+build flags: Assistant Webapp Pairing Testsuite S3(multipartupload) WebDAV Inotify DBus DesktopNotify XMPP ConcurrentOutput TorrentParser MagicMime Feeds Quvi
+key/value backends: SHA256E SHA256 SHA512E SHA512 SHA224E SHA224 SHA384E SHA384 SHA3_256E SHA3_256 SHA3_512E SHA3_512 SHA3_224E SHA3_224 SHA3_384E SHA3_384 SKEIN256E SKEIN256 SKEIN512E SKEIN512 SHA1E SHA1 MD5E MD5 WORM URL
+remote types: git gcrypt S3 bup directory rsync web bittorrent webdav tahoe glacier ddar hook external
+local repository version: 5
+supported repository versions: 5 6
+upgrade supported from repository versions: 0 1 2 4 5
+
+"""]]
+
+and it works ok without pidlock enabled
+
+[[!meta author=yoh]]
+
+> [[fixed|done]] --[[Joey]]
diff --git a/doc/bugs/too_many_ssh_connections_during_sync_of_gcrypt_remotes.mdwn b/doc/bugs/too_many_ssh_connections_during_sync_of_gcrypt_remotes.mdwn
new file mode 100644
index 0000000000..079c9025a9
--- /dev/null
+++ b/doc/bugs/too_many_ssh_connections_during_sync_of_gcrypt_remotes.mdwn
@@ -0,0 +1,27 @@
+### Please describe the problem.
+`git-annex sync gcryptremote` makes too many ssh connections one after another for each sub-task involved, potentially triggering firewall rate-limits on the SSH server.
+
+### What steps will reproduce the problem?
+sync with gcrypt remote while watching sshd logs on the server (I was getting >=5 connections per single sync in quick succession)
+
+### What version of git-annex are you using? On what operating system?
+Fedora 19, installed through cabal (without s3 and webapp support as the deps were failing to build)
+
+    $ git annex version
+    git-annex version: 5.20150205
+    build flags: Assistant Pairing WebDAV Inotify DBus DesktopNotify XMPP DNS Feeds Quvi TDFA TorrentParser
+    key/value backends: SHA256E SHA1E SHA512E SHA224E SHA384E SKEIN256E SKEIN512E MD5E SHA256 SHA1 SHA512 SHA224 SHA384 SKEIN256 SKEIN512 MD5 WORM URL
+    remote types: git gcrypt bup directory rsync web bittorrent webdav tahoe glacier ddar hook external
+    local repository version: 5
+    supported repository version: 5
+    upgrade supported from repository versions: 0 1 2 4
+
+### Please provide any additional information below.
+
+[[!format sh """
+# If you can, paste a complete transcript of the problem occurring here.
+# If the problem is with the git-annex assistant, paste in .git/annex/daemon.log
+
+
+# End of transcript or log.
+"""]]
diff --git a/doc/bugs/too_many_ssh_connections_during_sync_of_gcrypt_remotes/comment_1_8522285987f6d231c851861418af633a._comment b/doc/bugs/too_many_ssh_connections_during_sync_of_gcrypt_remotes/comment_1_8522285987f6d231c851861418af633a._comment
new file mode 100644
index 0000000000..133b3ac8ea
--- /dev/null
+++ b/doc/bugs/too_many_ssh_connections_during_sync_of_gcrypt_remotes/comment_1_8522285987f6d231c851861418af633a._comment
@@ -0,0 +1,24 @@
+[[!comment format=mdwn
+ username="joey"
+ subject="""comment 1"""
+ date="2015-02-12T18:27:26Z"
+ content="""
+Are you sure it's git-annex that's making the connections, and not
+git-remote-gcrypt? What kind of uri are you using?
+
+I tried to reproduce such a problem, making this gcrypt special remote:
+
+git annex initremote gcrypt type=gcrypt gitrepo=ssh://localhost/tmp/r encryption=shared --fast
+
+When I `git annex sync gcrypt`, only 1 ssh connection is made:
+
+joey@darkstar:~/tmp/meep>systemctl status ssh.socket | grep Accepted  
+ Accepted: 80; Connected: 1  
+joey@darkstar:~/tmp/meep>git annex sync gcrypt >/dev/null 2>&1  
+joey@darkstar:~/tmp/meep>systemctl status ssh.socket | grep Accepted  
+ Accepted: 81; Connected: 1  
+
+git-annex's ssh connection caching code is working for this gcrypt remote,
+because git-annex's `GIT_SSH=` is being propigated through gcrypt to git
+push and pull.
+"""]]
diff --git a/doc/bugs/too_many_ssh_connections_during_sync_of_gcrypt_remotes/comment_2_f87b3ba05c0cdfadd4d5da894dea0abb._comment b/doc/bugs/too_many_ssh_connections_during_sync_of_gcrypt_remotes/comment_2_f87b3ba05c0cdfadd4d5da894dea0abb._comment
new file mode 100644
index 0000000000..ee52566a44
--- /dev/null
+++ b/doc/bugs/too_many_ssh_connections_during_sync_of_gcrypt_remotes/comment_2_f87b3ba05c0cdfadd4d5da894dea0abb._comment
@@ -0,0 +1,7 @@
+[[!comment format=mdwn
+ username="sairon"
+ subject="comment 2"
+ date="2015-02-17T15:04:55Z"
+ content="""
+looks like it was the assistant
+"""]]
diff --git a/doc/bugs/tor_setup_needs_a_unique_magic-wormhole_appid.mdwn b/doc/bugs/tor_setup_needs_a_unique_magic-wormhole_appid.mdwn
new file mode 100644
index 0000000000..962466392b
--- /dev/null
+++ b/doc/bugs/tor_setup_needs_a_unique_magic-wormhole_appid.mdwn
@@ -0,0 +1,24 @@
+### Please describe the problem.
+
+The new "tor onion server" setup function uses magic-wormhole to transfer the necessary keys and addresses (yay!). However it should really be using a distinct "app id". Magic-wormhole codes are scoped to a specific APPID, which means that someone using the normal CLI client (`wormhole send foo.jpg`) can get a wormhole code like "1-purple-kumquat", and someone doing a git-annex setup process can get a code like "1-green-mushroom", and they won't be competing with each other for the numeric channel-id.
+
+If magic-wormhole's file-transfer application got really popular, and there were thousands of simultaneous users, the file-transfer wormhole codes would grow to require three or four digits (e.g. "9134-purple-kumquat"), making them harder to type. But if git-annex uses a different APPID, then git-annex codes could continue to be short and easy, independent of contention for channel-ids on other applications.
+
+To tell magic-wormhole to use a different APPID, just do `wormhole --appid=$APPID send ...` or `wormhole --appid=$APPID receive ...`. APPIDs should be unique and scoped to an owner of some sort, so I'd recommend a DNS name or URL-shaped identifier, something like "git-annex.branchable.com/onion-setup". If you add a new distinct mode, one which doesn't interoperate with the current onion-setup mode, you can allocate a new APPID for that mode too (e.g. "git-annex.branchable.com/new-thing-setup").
+
+You'll have a compatibility hit when you land this: two applications using different APPIDs won't communicate, so e.g. git-annex before the patch won't do wormhole setups with git-annex after the patch.
+
+The --appid= feature was added in magic-wormhole 0.9.0, released 24-Dec-2016.
+
+### What steps will reproduce the problem?
+
+
+### What version of git-annex are you using? On what operating system?
+
+I haven't run git-annex's onion setup feature directly, but I'm reading through the source code from current git, cd8d905f3418b9c6a6c658a0c7256ae6f5066310, Utility/MagicWormhole.hs, and I don't see anything that adds an --appid argument. (I don't know Haskell at all, so I apologize if it's already doing --appid and I'm just not looking in the right place).
+
+### Have you had any luck using git-annex before? (Sometimes we get tired of reading bug reports all day and a lil' positive end note does wonders)
+
+It looks pretty amazing! Looking forward to using it in a Dropbox-like synchronization context soon.
+
+> I've gone with the 2021-12-31 flag day approach. [[done]] --[[Joey]]
diff --git a/doc/bugs/tor_setup_needs_a_unique_magic-wormhole_appid/comment_1_d3b23b5bba991ea7b9b7c24f812a9018._comment b/doc/bugs/tor_setup_needs_a_unique_magic-wormhole_appid/comment_1_d3b23b5bba991ea7b9b7c24f812a9018._comment
new file mode 100644
index 0000000000..46a54fb9f7
--- /dev/null
+++ b/doc/bugs/tor_setup_needs_a_unique_magic-wormhole_appid/comment_1_d3b23b5bba991ea7b9b7c24f812a9018._comment
@@ -0,0 +1,20 @@
+[[!comment format=mdwn
+ username="joey"
+ subject="""comment 1"""
+ date="2017-01-31T17:04:26Z"
+ content="""
+Werner, I appreciate you filing this bug report. And I see magic-wormhole
+0.9.1 is now available in testing so it will be in the next debian stable
+release probably.
+
+If I'm going to make this change, I need to do it ASAP. If git-annex
+ships in Debian stable not using a wormhole app-id, then it pretty
+much is going to need to continue not using an app-id forever.
+(Which might not be a total disaster really; I don't know that a ton
+of users are using this feature..)
+
+The timeline for this is very tight now, unfortunately. (Pity that I have
+been traveling and missed this until now.) I'm not sure
+if, I released a git-annex today with this change, it would reach testing
+in time for the freeze. I've pinged Richih to try to figure that out.
+"""]]
diff --git a/doc/bugs/tor_setup_needs_a_unique_magic-wormhole_appid/comment_2_3f4693f6d877e2fcf4e62bd2b38a4472._comment b/doc/bugs/tor_setup_needs_a_unique_magic-wormhole_appid/comment_2_3f4693f6d877e2fcf4e62bd2b38a4472._comment
new file mode 100644
index 0000000000..6c426001ef
--- /dev/null
+++ b/doc/bugs/tor_setup_needs_a_unique_magic-wormhole_appid/comment_2_3f4693f6d877e2fcf4e62bd2b38a4472._comment
@@ -0,0 +1,17 @@
+[[!comment format=mdwn
+ username="joey"
+ subject="""comment 2"""
+ date="2017-01-31T17:20:56Z"
+ content="""
+
+based on this, we may have missed the window; there are 5 days until
+Debian freezes, and it takes 10 days to get a change into testing.
+
+It's still probably possible to get a change in, but it would need manual
+approval. Guess it would come down to whether I could get the change
+pre-approved for migration into Debian before releasing a git-annex
+with that change.
+
+(AFAIK Debian is the only distribution containing git-annex that's getting
+ready to freeze it.)
+"""]]
diff --git a/doc/bugs/tor_setup_needs_a_unique_magic-wormhole_appid/comment_3_882d915354ccd653f784499662544f31._comment b/doc/bugs/tor_setup_needs_a_unique_magic-wormhole_appid/comment_3_882d915354ccd653f784499662544f31._comment
new file mode 100644
index 0000000000..bbe424cb8d
--- /dev/null
+++ b/doc/bugs/tor_setup_needs_a_unique_magic-wormhole_appid/comment_3_882d915354ccd653f784499662544f31._comment
@@ -0,0 +1,24 @@
+[[!comment format=mdwn
+ username="joey"
+ subject="""comment 3"""
+ date="2017-02-02T17:46:05Z"
+ content="""
+After talking it over with the Debian maintainer, conclusion is
+we unfortunately missed the window to make this change.
+
+One way to still make the change would be to modify git-annex now to use an
+appid, but only once the date is after 31 Dec 2021 or so. So the change
+then has plenty of time to get into the subseqent Debian stable release
+(assuming the current release happens sometime in the next 1.5 years, and
+the one after that takes around the current average of 2 years), and also
+plenty of time to get into other faster moving distributions.
+
+That leaves a few years in which there's a risk that popularity of tor
+wormhole pairing snowballs. Currently I do think that's a low risk. 
+Most users of git-annex with tor will likely only pair with well under 10
+other repositories (it's too tedious to pair with more, and it starts
+not scaling well to have links to too many repositories). Pairing takes
+maybe 10 minutes max to do. So, each user will run wormhole for 100
+minutes max. For there to be a constant load of 100 users running wormhole,
+needs 3600 new users every day, or 1.3 million per year.
+"""]]
diff --git a/doc/bugs/touch.hsc_has_problems_on_non-linux_based_systems.mdwn b/doc/bugs/touch.hsc_has_problems_on_non-linux_based_systems.mdwn
new file mode 100644
index 0000000000..116f1496f9
--- /dev/null
+++ b/doc/bugs/touch.hsc_has_problems_on_non-linux_based_systems.mdwn
@@ -0,0 +1,20 @@
+It seems that commit bc5c54c987f548505a3877e8a0e460abe0b2a081 introduced some linux specific things...
+
+
    +hsc2hs Touch.hsc
+Touch.hsc: In function ‘main’:
+Touch.hsc:46: error: ‘UTIME_OMIT’ undeclared (first use in this function)
+Touch.hsc:46: error: (Each undeclared identifier is reported only once
+Touch.hsc:46: error: for each function it appears in.)
+Touch.hsc:48: error: ‘UTIME_NOW’ undeclared (first use in this function)
+Touch.hsc:67: error: ‘AT_FDCWD’ undeclared (first use in this function)
+Touch.hsc:68: error: ‘AT_SYMLINK_NOFOLLOW’ undeclared (first use in this function)
+compiling Touch_hsc_make.c failed
+command was: /usr/bin/gcc -c -m32 -I/Library/Frameworks/GHC.framework/Versions/612/usr/lib/ghc-6.12.3/include/ Touch_hsc_make.c -o Touch_hsc_make.o
+make: *** [Touch.hs] Error 1
+
    
+
+I dug around the OSX documentation and fcntl.h header file and it seems that UTIME_OMIT, UTIME_NOW, AT_FDCWD and AT_SYMLINK_NOFOLLOW aren't defined (at least on OSX). I suspect the BSD's in general will have problems compiling git-annex.
+
+[[!meta title="annexed symlink mtime matching code is disabled on non-linux systems; needs testing"]]
+[[!tag confirmed]]
diff --git a/doc/bugs/touch.hsc_has_problems_on_non-linux_based_systems/comment_1_1d38283c9ea87174f3bbef9a58f5cb88._comment b/doc/bugs/touch.hsc_has_problems_on_non-linux_based_systems/comment_1_1d38283c9ea87174f3bbef9a58f5cb88._comment
new file mode 100644
index 0000000000..f26239c3e9
--- /dev/null
+++ b/doc/bugs/touch.hsc_has_problems_on_non-linux_based_systems/comment_1_1d38283c9ea87174f3bbef9a58f5cb88._comment
@@ -0,0 +1,10 @@
+[[!comment format=mdwn
+ username="http://joey.kitenet.net/"
+ nickname="joey"
+ subject="comment 1"
+ date="2011-03-16T16:07:26Z"
+ content="""
+Hmm.. is utimensat available at all?
+
+I've committed an update that may convince at least some compilers to expose this newer POSIX stuff. I don't know if it will help, please let me know.
+"""]]
diff --git a/doc/bugs/touch.hsc_has_problems_on_non-linux_based_systems/comment_2_bf112edd075fbebe4fc959a387946eb9._comment b/doc/bugs/touch.hsc_has_problems_on_non-linux_based_systems/comment_2_bf112edd075fbebe4fc959a387946eb9._comment
new file mode 100644
index 0000000000..0222e645b9
--- /dev/null
+++ b/doc/bugs/touch.hsc_has_problems_on_non-linux_based_systems/comment_2_bf112edd075fbebe4fc959a387946eb9._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="https://www.google.com/accounts/o8/id?id=AItOawkSq2FDpK2n66QRUxtqqdbyDuwgbQmUWus"
+ nickname="Jimmy"
+ subject="comment 2"
+ date="2011-03-16T16:49:18Z"
+ content="""
+Just pulled the changes, it still fails to build. utimensat doesn't seem to exist on OSX 10.6.6.
+"""]]
diff --git a/doc/bugs/touch.hsc_has_problems_on_non-linux_based_systems/comment_3_a46080fbe82adf0986c5dc045e382501._comment b/doc/bugs/touch.hsc_has_problems_on_non-linux_based_systems/comment_3_a46080fbe82adf0986c5dc045e382501._comment
new file mode 100644
index 0000000000..7e79dea881
--- /dev/null
+++ b/doc/bugs/touch.hsc_has_problems_on_non-linux_based_systems/comment_3_a46080fbe82adf0986c5dc045e382501._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="http://joey.kitenet.net/"
+ nickname="joey"
+ subject="comment 3"
+ date="2011-03-16T17:46:40Z"
+ content="""
+Alright, I've added #idefs and the symlink timestamp mirroring feature will be unavailable on OSX until I get a version that works there.
+"""]]
diff --git a/doc/bugs/touch.hsc_has_problems_on_non-linux_based_systems/comment_4_760437bf3ba972a775bb190fb4b38202._comment b/doc/bugs/touch.hsc_has_problems_on_non-linux_based_systems/comment_4_760437bf3ba972a775bb190fb4b38202._comment
new file mode 100644
index 0000000000..6b1e03b026
--- /dev/null
+++ b/doc/bugs/touch.hsc_has_problems_on_non-linux_based_systems/comment_4_760437bf3ba972a775bb190fb4b38202._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="https://www.google.com/accounts/o8/id?id=AItOawkSq2FDpK2n66QRUxtqqdbyDuwgbQmUWus"
+ nickname="Jimmy"
+ subject="comment 4"
+ date="2011-03-16T20:32:01Z"
+ content="""
+Just tried it out on my mac and it's working again. I guess this issue could be closed for now.
+"""]]
diff --git a/doc/bugs/touch.hsc_has_problems_on_non-linux_based_systems/comment_5_060ba5ea88dcab2f4a0c199f13ef4f67._comment b/doc/bugs/touch.hsc_has_problems_on_non-linux_based_systems/comment_5_060ba5ea88dcab2f4a0c199f13ef4f67._comment
new file mode 100644
index 0000000000..aeb576be37
--- /dev/null
+++ b/doc/bugs/touch.hsc_has_problems_on_non-linux_based_systems/comment_5_060ba5ea88dcab2f4a0c199f13ef4f67._comment
@@ -0,0 +1,10 @@
+[[!comment format=mdwn
+ username="http://joey.kitenet.net/"
+ nickname="joey"
+ subject="comment 5"
+ date="2011-03-20T18:12:59Z"
+ content="""
+I'm leaving this bug open because this feature, however minor is not available on OSX and BSD. 
+
+I have added a partial implementation using lutimes(3), which should be available on the BSDs. However, it's ifdefed out due to a casting problem: The TimeSpec uses a CTime, while lutimes uses a CLong. These data types may be internally the same on some or all platforms, so if you want this feature you can try changing the \"ifdef 0\" in Touch.hsc to 1 and try it, see if \"git annex add\" mirrors file modification time in created symlinks, and let me know.
+"""]]
diff --git a/doc/bugs/touch.hsc_has_problems_on_non-linux_based_systems/comment_6_548303d6ffb21a9370b6904f41ff49c1._comment b/doc/bugs/touch.hsc_has_problems_on_non-linux_based_systems/comment_6_548303d6ffb21a9370b6904f41ff49c1._comment
new file mode 100644
index 0000000000..cd116c232d
--- /dev/null
+++ b/doc/bugs/touch.hsc_has_problems_on_non-linux_based_systems/comment_6_548303d6ffb21a9370b6904f41ff49c1._comment
@@ -0,0 +1,42 @@
+[[!comment format=mdwn
+ username="https://www.google.com/accounts/o8/id?id=AItOawkSq2FDpK2n66QRUxtqqdbyDuwgbQmUWus"
+ nickname="Jimmy"
+ subject="comment 6"
+ date="2011-03-20T20:48:41Z"
+ content="""
+ok, pulling the latest master and building on OSX now does this...
+
+
    +ghc -O2 -Wall -ignore-package monads-fd --make git-annex
+[ 1 of 63] Compiling Touch            ( Touch.hs, Touch.o )
+
+Touch.hsc:24:0:
+    The type signature for `touchBoth' lacks an accompanying binding
+
+Touch.hsc:27:26: Not in scope: `touchBoth'
+make: *** [git-annex] Error 1
+
    
+
+changing the #if 0 to 1 gives this...
+
+
    +ghc -O2 -Wall -ignore-package monads-fd --make git-annex
+[ 1 of 63] Compiling Touch            ( Touch.hs, Touch.o )
+
+Touch.hsc:95:43:
+    Couldn't match expected type `CLong' against inferred type `CTime'
+    In the second argument of `(\ hsc_ptr
+                                    -> pokeByteOff hsc_ptr 0)', namely
+        `(sec :: CLong)'
+    In a stmt of a 'do' expression:
+        (\ hsc_ptr -> pokeByteOff hsc_ptr 0) ptr (sec :: CLong)
+    In the expression:
+        do { (\ hsc_ptr -> pokeByteOff hsc_ptr 0) ptr (sec :: CLong);
+             (\ hsc_ptr -> pokeByteOff hsc_ptr 4) ptr (0 :: CLong) }
+make: *** [git-annex] Error 1
+
    
+
+
+it seems that commit 6634b6a6b84a924f6f6059b5bea61f449d056eee has broken support for OSX.
+
+"""]]
diff --git a/doc/bugs/touch.hsc_has_problems_on_non-linux_based_systems/comment_7_7ca00527ab5db058aadec4fe813e51fd._comment b/doc/bugs/touch.hsc_has_problems_on_non-linux_based_systems/comment_7_7ca00527ab5db058aadec4fe813e51fd._comment
new file mode 100644
index 0000000000..e35dc8a827
--- /dev/null
+++ b/doc/bugs/touch.hsc_has_problems_on_non-linux_based_systems/comment_7_7ca00527ab5db058aadec4fe813e51fd._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="http://joey.kitenet.net/"
+ nickname="joey"
+ subject="comment 7"
+ date="2011-03-20T22:06:25Z"
+ content="""
+Fixed that, and removed the impossible cast so it can be built with #if 1
+"""]]
diff --git a/doc/bugs/touch.hsc_has_problems_on_non-linux_based_systems/comment_8_881aecb9ae671689453f6d5d780d844b._comment b/doc/bugs/touch.hsc_has_problems_on_non-linux_based_systems/comment_8_881aecb9ae671689453f6d5d780d844b._comment
new file mode 100644
index 0000000000..56a7eb360e
--- /dev/null
+++ b/doc/bugs/touch.hsc_has_problems_on_non-linux_based_systems/comment_8_881aecb9ae671689453f6d5d780d844b._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="https://www.google.com/accounts/o8/id?id=AItOawkSq2FDpK2n66QRUxtqqdbyDuwgbQmUWus"
+ nickname="Jimmy"
+ subject="comment 8"
+ date="2011-03-21T08:52:18Z"
+ content="""
+Just tried building both of the code paths, and they seem to build and somewhat function on OSX. I have yet to confirm the functionality is working correctly, but so far it's looking good. (I somewhat care less about the utimes/mtimes of my files since I care more about the content :) )
+"""]]
diff --git a/doc/bugs/treatment_of_largefiles_is_not_working_for_addurl_--fast___40__or_--relaxed__41__.mdwn b/doc/bugs/treatment_of_largefiles_is_not_working_for_addurl_--fast___40__or_--relaxed__41__.mdwn
new file mode 100644
index 0000000000..a77ee211d4
--- /dev/null
+++ b/doc/bugs/treatment_of_largefiles_is_not_working_for_addurl_--fast___40__or_--relaxed__41__.mdwn
@@ -0,0 +1,29 @@
+### Please describe the problem.
+
+As described [elsewhere](http://git-annex.branchable.com/todo/make_addurl_respect_annex.largefiles_option/#comment-817c4d5c215ddc0ec7bc5c4c05dff091) but since that one is closed -- making a dedicated one
+
+unfortunately addurl doesn't care about large files if --fast (or --relaxed, although that one I guess I can understand ):
+
+[[!format sh """
+
+$ chmod a+w -R /tmp/123; rm -rf /tmp/123; mkdir /tmp/123; cd /tmp/123; git init; git annex init;
+Initialized empty Git repository in /tmp/123/.git/
+init  ok
+(recording state in git...)
+
+$ git -c annex.largefiles=exclude=*.txt annex addurl --file=test.txt --fast \"http://www.onerussian.com/tmp/banner.png\"
+addurl test.txt ok
+(recording state in git...)
+
+$ ls -l test.txt
+lrwxrwxrwx 1 yoh yoh 132 Jan 13 18:44 test.txt -> .git/annex/objects/KW/kj/URL-s25319--http&c%%www.onerussian.com%tmp%banner.png/URL-s25319--http&c%%www.onerussian.com%tmp%banner.png
+
+"""]]
+
+it does consider largefiles for if addurl without  --fast, or --relaxed
+
+[[!meta author=yoh]]
+
+> [[done]]; this can't really be done, but I added `git annex
+> matchexpression` to allow scripting that checks expressions such as
+> annex.largefiles. --[[Joey]]
diff --git a/doc/bugs/unRAID_shares_treated_as_a_crippled_filesystem.mdwn b/doc/bugs/unRAID_shares_treated_as_a_crippled_filesystem.mdwn
new file mode 100644
index 0000000000..3105f3aeff
--- /dev/null
+++ b/doc/bugs/unRAID_shares_treated_as_a_crippled_filesystem.mdwn
@@ -0,0 +1,49 @@
+### Please describe the problem.
+
+Running `git annex init` on an [unRAID server](https://lime-technology.com/what-is-unraid/) results in an annex created with `crippledfilesystem = true` and `direct = true`. I understand from reading [this](https://git-annex.branchable.com/design/assistant/blog/day_188__crippled_filesystem_support/) that it occurs when `git annex init` performs a probe to determine if all of the following are supported:
+
+1. symlinks
+2. hard links
+3. unix permissions
+
+Although unRAID disks are formatted with xfs, and therefore support all three of the above, I'm assuming that unRAID's method of combining multiple disks into one "share" is the cause of the problem (hardlinks still work on a single disk, but not on shares that span multiple disks). Symlinks and unix permissions work normally in the unRAID-created shares.
+
+Is there any way to allow the use of 'indirect' mode with multi-disk shares? As I mentioned, symlinks and unix permissions work normally--it's only the hardlinks that won't work across the multi-disk shares.
+
+I can create a 'normal' annex as long as I `cd` to a single disk drive first--what would happen if the annex was later moved onto a multi-disk share? Would it still work? Would it fail gracefully? Would it cause data loss?
+
+### What steps will reproduce the problem?
+
+    cd /mnt/user/NameOfShare
+    git init
+    git annex init
+
+The following will result in the creation of a 'normal' indirect share:
+
+    cd /mnt/disk1
+    git init
+    git annex init
+
+### What version of git-annex are you using? On what operating system?
+
+    git-annex version: 6.20161211-gc3ab3c668
+    build flags: Assistant Webapp Pairing Testsuite S3(multipartupload)(storageclasses) WebDAV Inotify DBus DesktopNotify XMPP ConcurrentOutput TorrentParser MagicMime Feeds Quvi
+    key/value backends: SHA256E SHA256 SHA512E SHA512 SHA224E SHA224 SHA384E SHA384 SHA3_256E SHA3_256 SHA3_512E SHA3_512 SHA3_224E SHA3_224 SHA3_384E SHA3_384 SKEIN256E SKEIN256 SKEIN512E SKEIN512 SHA1E SHA1 MD5E MD5 WORM URL
+    remote types: git gcrypt p2p S3 bup directory rsync web bittorrent webdav tahoe glacier ddar hook external
+
+### Please provide any additional information below.
+
+[[!format sh """
+# If you can, paste a complete transcript of the problem occurring here.
+# If the problem is with the git-annex assistant, paste in .git/annex/daemon.log
+
+
+# End of transcript or log.
+"""]]
+
+### Have you had any luck using git-annex before? (Sometimes we get tired of reading bug reports all day and a lil' positive end note does wonders)
+
+Has been working great, so far, except for the above.
+
+> [[done]]; I think this just doesn't support permissions correcectly, and
+> there does not appear to be a git-annex bug. --[[Joey]]
diff --git a/doc/bugs/unRAID_shares_treated_as_a_crippled_filesystem/comment_1_3698dfec0b4b566b240da53d5e20ae5f._comment b/doc/bugs/unRAID_shares_treated_as_a_crippled_filesystem/comment_1_3698dfec0b4b566b240da53d5e20ae5f._comment
new file mode 100644
index 0000000000..723bde6bf3
--- /dev/null
+++ b/doc/bugs/unRAID_shares_treated_as_a_crippled_filesystem/comment_1_3698dfec0b4b566b240da53d5e20ae5f._comment
@@ -0,0 +1,31 @@
+[[!comment format=mdwn
+ username="joey"
+ subject="""comment 1"""
+ date="2017-06-06T19:28:18Z"
+ content="""
+Lack of support for hard links does not make git-annex enable 
+crippled filesystem mode. Lack of support for either symlinks
+or unix permissions are the only things that cause that.
+
+I assume that you've checked that you can create symlinks on the UnRaid.
+
+Unix permissions may *seem* to work, eg they can be set. However, git-annex
+checks if they *actually* work, by creating a file, removing the write bit,
+and trying to write to it. If the write succeeds, the filesystem is not
+actually honoring permissions.
+
+Using git-annex in indirect mode with such a filesystem can
+result in data loss. For example:
+
+	git annex add foo
+	git commit foo -m added
+	echo corrupt > foo
+
+Here the echo follows the symlink to the single copy of the file in
+.git/annex/objects/ and ignoring the permissions that don't allow writing
+it it, overwrites it with other data. `git annex fsck` will then tell you
+that you've lost the old content of the file.
+
+So, I don't recommend trying to bypass git-annex's check for crippled
+filesystems.
+"""]]
diff --git a/doc/bugs/unable_to_access_annexed_files_from_a_git_repo_website_serving_via___34__smart_HTTP__34__.mdwn b/doc/bugs/unable_to_access_annexed_files_from_a_git_repo_website_serving_via___34__smart_HTTP__34__.mdwn
new file mode 100644
index 0000000000..234a930eb6
--- /dev/null
+++ b/doc/bugs/unable_to_access_annexed_files_from_a_git_repo_website_serving_via___34__smart_HTTP__34__.mdwn
@@ -0,0 +1,120 @@
+### Please describe the problem.
+
+Our http://datasets.datalad.org has been providing git annex repos, some of which with the content, via a "dummy" HTTP support of git.  For various reasons (performance, progress reporting by git upon clone) we want to switch to use [Smart HTTP](https://git-scm.com/book/en/v2/Git-on-the-Server-Smart-HTTP) git-http-backend  backend.  Sample deployment is at http://datasets-dev.datalad.org/.
+I followed the docs to set it up and only added one more configuration tune up
+
+
+```
+ RewriteEngine On
+
+ RewriteCond  "%{HTTP_USER_AGENT}"  "(git)"
+
+ RewriteRule ^(.*)$ "/git/$1" [PT]
+```
+
+so that people could still browse the website in the browser, but whenever `git` tries to access it, we direct to the `git-http-backend` CGI serving under `/git/` prefix (`ScriptAlias /git/ /usr/lib/git-core/git-http-backend/`).
+
+Everything seems to work nicely on git side, BUT I am having difficulty to make git-annex being able to serve annexed files from it:
+ 
+### What version of git-annex are you using? On what operating system?
+
+6.20180913+git149-g23bd27773-1~ndall+1
+
+
+### Please provide any additional information below.
+
+[[!format sh """
+$> builtin cd /tmp/; rm -rf raiders; git clone http://datasets-dev.datalad.org/labs/haxby/raiders/ ; cd raiders; git annex get sub-rid000005/anat/sub-rid000005_run-01_T1w_defacemask.nii.gz                                                                                                                          Cloning into 'raiders'...                
+remote: Counting objects: 17926, done.
+remote: Compressing objects: 100% (7203/7203), done.
+remote: Total 17926 (delta 7356), reused 15517 (delta 6237)
+Receiving objects: 100% (17926/17926), 1.23 MiB | 6.53 MiB/s, done.
+Resolving deltas: 100% (7356/7356), done.
+README.md                 masks/       stimulus/       sub-rid000014/  sub-rid000028/  sub-rid000038/  task-raiders_bold.json
+dataset_description.json  scripts/     sub-rid000005/  sub-rid000015/  sub-rid000029/  sub-rid000042/
+derivatives/              sourcedata/  sub-rid000011/  sub-rid000020/  sub-rid000033/  sub-rid000043/
+(merging origin/git-annex into git-annex...)
+(recording state in git...)
+get sub-rid000005/anat/sub-rid000005_run-01_T1w_defacemask.nii.gz download failed: Not Found
+
+  Remote origin not usable by git-annex; setting annex-ignore
+(not available) 
+  Try making some of these repositories available:
+  	41e5039d-1750-43d2-8bea-89897d969326 -- /mnt/datasets/datalad/crawl/labs/haxby/raiders
+   	87d7db62-683d-43b2-b594-baeb420ae7a6 -- .
+   	afde6679-1f2f-41f2-935a-93e7e3d70274 -- nastase@head1:~/BIDS/haxby/raiders
+   	de53ce43-2c07-4971-8de8-0445c596f7dc -- datalad-public-ro
+
+  (Note that these git remotes have annex-ignore set: origin)
+failed
+(recording state in git...)
+git-annex: get: 1 failed
+"""]]
+
+fails because `config` file is under `.git/` subdirectory there and git-annex doesn't try to access it at all to deduce the uuid, thus marking origin as annex-ignore.
+
+But if I add that `.git` suffix to the url, then:
+
+[[!format sh """
+(git)hopa:/tmp/raiders[master]
+$> builtin cd /tmp/; rm -rf raiders; git clone http://datasets-dev.datalad.org/labs/haxby/raiders/.git/ ; cd raiders; git annex get sub-rid000005/anat/sub-rid000005_run-01_T1w_defacemask.nii.gz                                                                 
+Cloning into 'raiders'...
+remote: Counting objects: 17926, done.
+remote: Compressing objects: 100% (7203/7203), done.
+remote: Total 17926 (delta 7356), reused 15517 (delta 6237)
+Receiving objects: 100% (17926/17926), 1.23 MiB | 5.08 MiB/s, done.
+Resolving deltas: 100% (7356/7356), done.
+README.md                 masks/       stimulus/       sub-rid000014/  sub-rid000028/  sub-rid000038/  task-raiders_bold.json
+dataset_description.json  scripts/     sub-rid000005/  sub-rid000015/  sub-rid000029/  sub-rid000042/
+derivatives/              sourcedata/  sub-rid000011/  sub-rid000020/  sub-rid000033/  sub-rid000043/
+(merging origin/git-annex into git-annex...)
+(recording state in git...)
+get sub-rid000005/anat/sub-rid000005_run-01_T1w_defacemask.nii.gz (from origin...) 
+download failed: Not Found
+download failed: Not Found
+
+  Unable to access these remotes: origin
+
+  Try making some of these repositories available:
+  	41e5039d-1750-43d2-8bea-89897d969326 -- /mnt/datasets/datalad/crawl/labs/haxby/raiders
+   	87d7db62-683d-43b2-b594-baeb420ae7a6 -- .
+   	afde6679-1f2f-41f2-935a-93e7e3d70274 -- nastase@head1:~/BIDS/haxby/raiders
+   	de53ce43-2c07-4971-8de8-0445c596f7dc -- datalad-public-ro [origin]
+failed
+(recording state in git...)
+git-annex: get: 1 failed
+"""]]
+because it fails to find those two files under `.git/annex/objects`, here is apache log file
+
+
+```
+10.31.191.134 - - [01/Oct/2018:13:01:58 -0400] "GET /labs/haxby/raiders/.git//config HTTP/1.1" 206 501 "-" "-"
+
+10.31.191.134 - - [01/Oct/2018:13:01:58 -0400] "GET /labs/haxby/raiders/.git//annex/objects/Z8/f1/MD5E-s41438--06c245e709e7d40a90ed48c6c3b58295.nii.gz/MD5E-s41438--06c245e709e7d40a90ed48c6c3b58295.nii.gz HTTP/1.1" 404 243 "-" "git-annex/6.20180913+git149-g23bd27773-1~ndall+1"
+
+10.31.191.134 - - [01/Oct/2018:13:01:58 -0400] "GET /labs/haxby/raiders/.git//annex/objects/681/5d0/MD5E-s41438--06c245e709e7d40a90ed48c6c3b58295.nii.gz/MD5E-s41438--06c245e709e7d40a90ed48c6c3b58295.nii.gz HTTP/1.1" 404 243 "-" "git-annex/6.20180913+git149-g23bd27773-1~ndall+1"
+```
+
+where it seems to assume different layout:
+
+[[!format sh """
+$> ls -dl $webroot//labs/haxby/raiders/.git/annex/objects/*/*/MD5E-s41438--06c245e709e7d40a90ed48c6c3b58295.nii.gz
+drwxrwsr-x 1 yoh datalad 104 Sep 26  2016 /mnt/btrfs/manual-snapshots/srv-20180928/datasets.datalad.org/www///labs/haxby/raiders/.git/annex/objects/Z8/f1/MD5E-s41438--06c245e709e7d40a90ed48c6c3b58295.nii.gz/
+"""]]
+
+
+which git-annex assumes when working with the dummy HTTP:
+
+```
+10.31.191.134 - - [01/Oct/2018:13:09:53 -0400] "GET /labs/haxby/raiders/.git//config HTTP/1.1" 206 501 "-" "-"
+
+10.31.191.134 - - [01/Oct/2018:13:09:53 -0400] "GET /labs/haxby/raiders/.git//annex/objects/Z8/f1/MD5E-s41438--06c245e709e7d40a90ed48c6c3b58295.nii.gz/MD5E-s41438--06c245e709e7d40a90ed48c6c3b58295.nii.gz HTTP/1.1" 200 41679 "-" "git-annex/6.20180913+git149-g23bd27773-1~ndall+1"
+```
+
+So I wonder if I need to do something on my end in configuring apache2, or something could/should be done on git-annex side?  Ideally I would like to be able to just clone them without specifying `.git/` suffix to the url.
+
+But also note that `git-annex` seems to not even provide any agent value while trying to access `config` file:
+
+```
+10.31.191.134 - - [01/Oct/2018:13:12:45 -0400] "GET /labs/haxby/raiders/.git//config HTTP/1.1" 206 501 "-" "-"
+```
diff --git a/doc/bugs/unable_to_get_from_public_S3_remote_without_clear_reasoning_why.mdwn b/doc/bugs/unable_to_get_from_public_S3_remote_without_clear_reasoning_why.mdwn
new file mode 100644
index 0000000000..a310c9347f
--- /dev/null
+++ b/doc/bugs/unable_to_get_from_public_S3_remote_without_clear_reasoning_why.mdwn
@@ -0,0 +1,77 @@
+### Please describe the problem.
+
+As originally reported in the [comments](http://git-annex.branchable.com/devblog/day_518__S3_versioning_finishing_touches/?updated#comment-ecfa9f9e6075bf524a88f404bb19e086) and now wondering if it has to do with the awkwardness of that bucket -- its name has a period in its name, so https validation fails and we had to [workaround for that in datalad](https://github.com/datalad/datalad/commit/ea8977fddbd05faa3f299562b70d4a491a774cea) as well.  In either case - having at least some debug log message about the problem would be handy
+
+[[!format sh """
+test@hopa:/tmp$ mkdir testuser
+test@hopa:/tmp$ cd testuser
+test@hopa:/tmp/testuser$ git clone https://github.com/OpenNeuroDatasets/ds001506 ; cd ds001506; 
+Cloning into 'ds001506'...
+remote: Counting objects: 11017, done.
+remote: Compressing objects: 100% (6694/6694), done.
+remote: Total 11017 (delta 1856), reused 11007 (delta 1846), pack-reused 0
+Receiving objects: 100% (11017/11017), 1.17 MiB | 1.31 MiB/s, done.
+Resolving deltas: 100% (1856/1856), done.
+test@hopa:/tmp/testuser/ds001506$ git annex enableremote s3-PUBLIC public=yes
+(merging origin/git-annex into git-annex...)
+(recording state in git...)
+download failed: ConnectionFailure Network.Socket.connect: : timeout (Connection timed out)
+
+  Remote origin not usable by git-annex; setting annex-ignore
+enableremote s3-PUBLIC ok
+(recording state in git...)
+test@hopa:/tmp/testuser/ds001506$ git annex get --debug sub-01/ses-anatomy/anat
+[2018-09-11 13:00:24.89648131] read: git ["--git-dir=.git","--work-tree=.","--literal-pathspecs","ls-files","--cached","-z","--","sub-01/ses-anatomy/anat"]
+get sub-01/ses-anatomy/anat/sub-01_ses-anatomy_T1w.nii.gz [2018-09-11 13:00:24.900815381] read: git ["--git-dir=.git","--work-tree=.","--literal-pathspecs","show-ref","git-annex"]
+[2018-09-11 13:00:24.90536002] process done ExitSuccess
+[2018-09-11 13:00:24.905485937] read: git ["--git-dir=.git","--work-tree=.","--literal-pathspecs","show-ref","--hash","refs/heads/git-annex"]
+[2018-09-11 13:00:24.909586624] process done ExitSuccess
+[2018-09-11 13:00:24.909805571] read: git ["--git-dir=.git","--work-tree=.","--literal-pathspecs","log","refs/heads/git-annex..a152bc6cd4a6b1f06115d683e6a296374219e9f7","--pretty=%H","-n1"]
+[2018-09-11 13:00:24.914170223] process done ExitSuccess
+[2018-09-11 13:00:24.914585144] chat: git ["--git-dir=.git","--work-tree=.","--literal-pathspecs","cat-file","--batch"]
+[2018-09-11 13:00:24.91587481] chat: git ["--git-dir=.git","--work-tree=.","--literal-pathspecs","cat-file","--batch-check=%(objectname) %(objecttype) %(objectsize)"]
+(from s3-PUBLIC...) [2018-09-11 13:00:24.982444067] read: git ["--git-dir=.git","--work-tree=.","--literal-pathspecs","diff-tree","-z","--raw","--no-renames","-l0","-r","4b825dc642cb6eb9a060e54bf8d69288fbee4904","18df0d827a901db0a989999f765f2816b769bf49","--"]
+[2018-09-11 13:00:24.994539395] chat: git ["--git-dir=.git","--work-tree=.","--literal-pathspecs","cat-file","--batch"]
+[2018-09-11 13:00:24.995000749] chat: git ["--git-dir=.git","--work-tree=.","--literal-pathspecs","cat-file","--batch-check=%(objectname) %(objecttype) %(objectsize)"]
+[2018-09-11 13:00:25.433359795] process done ExitSuccess
+
+  Unable to access these remotes: s3-PUBLIC
+
+  Try making some of these repositories available:
+  	810b2f6b-fb98-4401-9130-6c84dd7ddc50 -- root@b3ba225d5547:/datalad/ds001506
+   	ca9b233b-7567-48b3-89c7-efe7f6a97d4a -- [s3-PUBLIC]
+
+  (Note that these git remotes have annex-ignore set: origin)
+failed
+[2018-09-11 13:00:25.657658586] process done ExitSuccess
+[2018-09-11 13:00:25.658108866] process done ExitSuccess
+[2018-09-11 13:00:25.658390098] process done ExitSuccess
+[2018-09-11 13:00:25.658635699] process done ExitSuccess
+git-annex: get: 1 failed
+test@hopa:/tmp/testuser/ds001506$ git annex version | head -n 1
+git-annex version: 6.20180807+git230-gaa291acfe-1~ndall+1
+test@hopa:/tmp/testuser/ds001506$ git annex info s3-PUBLIC
+uuid: ca9b233b-7567-48b3-89c7-efe7f6a97d4a
+description: [s3-PUBLIC]
+remote: s3-PUBLIC
+trust: untrusted
+cost: 200.0
+type: S3
+creds: not available
+bucket: openneuro.org
+endpoint: s3.amazonaws.com
+port: 80
+storage class: STANDARD
+partsize: 1.07 gigabytes
+public: yes
+versioning: no
+encryption: none
+chunking: none
+export: yes
+remote annex keys: 1287
+remote annex size: 103.28 gigabytes (+ 632 unknown size)
+
+"""]]
+
+
+[[!meta author=yoh]]
diff --git a/doc/bugs/unannex__58___Cannot_proceed_with_uncommitted_changes_staged_in_the_index.mdwn b/doc/bugs/unannex__58___Cannot_proceed_with_uncommitted_changes_staged_in_the_index.mdwn
new file mode 100644
index 0000000000..065ff3ce0c
--- /dev/null
+++ b/doc/bugs/unannex__58___Cannot_proceed_with_uncommitted_changes_staged_in_the_index.mdwn
@@ -0,0 +1,37 @@
+### Please describe the problem.
+
+I understand that `git annex unannex` is essentially there for undoing an accidental `git annex add`. Unfortunately it doesn't do that.
+
+If I have uncommitted changes, which is the case after a `git annex add`, it tells me:
+
+    git-annex: Cannot proceed with uncommitted changes staged in the index. Recommend you: git commit
+    CallStack (from HasCallStack):
+      error, called at ./Command/Unannex.hs:48:19 in main:Command.Unannex
+
+But I would expect it to `git reset` the file and then replace the symlink by the actual file content.
+
+### What steps will reproduce the problem?
+
+    > git init
+    Initialized empty Git repository in /somewhere/.git
+    > git annex init
+    init  ok
+    (recording state in git...)
+    > touch foo
+    > git annex add foo
+    add foo ok
+    (recording state in git...)
+    > git annex unannex foo
+    git-annex: Cannot proceed with uncommitted changes staged in the index. Recommend you: git commit
+    CallStack (from HasCallStack):
+      error, called at ./Command/Unannex.hs:48:19 in main:Command.Unannex
+
+### What version of git-annex are you using? On what operating system?
+
+git-annex version: 6.20160527-gf21a425
+
+Installed from the Arch Linux repository.
+
+### Have you had any luck using git-annex before? (Sometimes we get tired of reading bug reports all day and a lil' positive end note does wonders)
+
+Sure, I'm using it for photos, music and videos
diff --git a/doc/bugs/unannex__58___Cannot_proceed_with_uncommitted_changes_staged_in_the_index/comment_1_1c0cad1076d5d4d908b8297e7c13ea33._comment b/doc/bugs/unannex__58___Cannot_proceed_with_uncommitted_changes_staged_in_the_index/comment_1_1c0cad1076d5d4d908b8297e7c13ea33._comment
new file mode 100644
index 0000000000..4b54996eb4
--- /dev/null
+++ b/doc/bugs/unannex__58___Cannot_proceed_with_uncommitted_changes_staged_in_the_index/comment_1_1c0cad1076d5d4d908b8297e7c13ea33._comment
@@ -0,0 +1,32 @@
+[[!comment format=mdwn
+ username="joey"
+ subject="""comment 1"""
+ date="2016-07-12T16:23:51Z"
+ content="""
+In v5 mode, there is a complex interaction between unannex and the
+pre-commit hook. An unannexed file looks quite a lot like an unlocked file,
+so the pre-commit hook is prone to want to lock it, and so add it back to
+the annex as an annexed file.
+
+To avoid that problem, unannex needs to commit the unannexing of the
+files.
+
+However, if you have other staged changes, they'll also be included in that
+commit. Which would be a bug if it were allowed to happen. This is why
+it checks for a clean index first.
+
+It would be possible to improve the behavior by explicitly committing only
+the files that got unannexed, rather than all staged changes. Why didn't I
+do that?
+
+Well, `git commit $file` stages any changes to the file's content before
+committing. When the file has been unannexed, this stages the entire large
+file content into git, and adds it back. Not the desired behavior!
+
+Git in fact has no interface to make it commit only staged changes to
+only specific files. I can't get there from here. It would certianly
+be nice if git got the ability to do that, if someone wants a project to
+improve git.
+
+It's very nice that v6 mode avoids this problem entirely!
+"""]]
diff --git a/doc/bugs/unannex__58___Cannot_proceed_with_uncommitted_changes_staged_in_the_index/comment_2_4609f9a1545b08e08021bf786561f5e5._comment b/doc/bugs/unannex__58___Cannot_proceed_with_uncommitted_changes_staged_in_the_index/comment_2_4609f9a1545b08e08021bf786561f5e5._comment
new file mode 100644
index 0000000000..1c3169e70f
--- /dev/null
+++ b/doc/bugs/unannex__58___Cannot_proceed_with_uncommitted_changes_staged_in_the_index/comment_2_4609f9a1545b08e08021bf786561f5e5._comment
@@ -0,0 +1,9 @@
+[[!comment format=mdwn
+ username="tom.prince@6bf26c878bf6103667f28d70cf49d4fb74d33df7"
+ nickname="tom.prince"
+ avatar="http://cdn.libravatar.org/avatar/e81edff3af564b86f4c9d780a8023299"
+ subject="comment 2"
+ date="2017-07-21T18:22:28Z"
+ content="""
+As far as I can tell, from looking at the code, the pre-commit hook only looks at files in the index. Thus, if unannexing an uncommited file removed it from the index, the pre-commit will do the right thing. This would be nice to have, at least for the case where the files have never been committed.
+"""]]
diff --git a/doc/bugs/unannex__58___Cannot_proceed_with_uncommitted_changes_staged_in_the_index/comment_3_6c99a97d56b1a3b12092c15fafcf8761._comment b/doc/bugs/unannex__58___Cannot_proceed_with_uncommitted_changes_staged_in_the_index/comment_3_6c99a97d56b1a3b12092c15fafcf8761._comment
new file mode 100644
index 0000000000..47d417cec0
--- /dev/null
+++ b/doc/bugs/unannex__58___Cannot_proceed_with_uncommitted_changes_staged_in_the_index/comment_3_6c99a97d56b1a3b12092c15fafcf8761._comment
@@ -0,0 +1,35 @@
+[[!comment format=mdwn
+ username="https://launchpad.net/~stephane-gourichon-lpad"
+ nickname="stephane-gourichon-lpad"
+ avatar="http://cdn.libravatar.org/avatar/02d4a0af59175f9123720b4481d55a769ba954e20f6dd9b2792217d9fa0c6089"
+ subject="User expectations and what git annex unannex does."
+ date="2017-07-24T08:06:55Z"
+ content="""
+# Where we are
+
+@joey thank you for these explanations, more detailed than when I reported the same problem 8 months ago commenting https://git-annex.branchable.com/git-annex-unannex/ (@tom.prince had already written this page but I did not find it).
+
+Yet all this happens in a git world, where private history can be rewritten, so *there must be a simpler way*.
+
+# What people expect from \"undo accidental add command\"
+
+@tom.prince thanks for explaining what you expected `unannex` to do. Looks like I expected exactly the same behavior.
+
+IMHO current behavior of `git annex unannex` does not match what people expect of \"undo accidental add command\".
+
+# What current `git-annex unannex` actually does
+
+If behavior does not match words, perhaps behavior is interesting but should be matched with different words?
+
+Looking at what `git-annex unannex`, here are the words that came to mind:
+
+> git-annex unannex - turn a path which points to annexed content into a plain file that is store in regular git.
+
+Notice that:
+
+* `git-annex` retains history
+* other paths may still refer to the same content, so the annex may still contain a copy of the same data.  Else it becomes unused content subject to `git-annex dropunused`.
+
+Thank you for your attention.
+
+"""]]
diff --git a/doc/bugs/unclear_why_annex_fails_to_get_a_file_from_a_public_S3_bucket_where_it_exported_before.mdwn b/doc/bugs/unclear_why_annex_fails_to_get_a_file_from_a_public_S3_bucket_where_it_exported_before.mdwn
new file mode 100644
index 0000000000..304f585a0a
--- /dev/null
+++ b/doc/bugs/unclear_why_annex_fails_to_get_a_file_from_a_public_S3_bucket_where_it_exported_before.mdwn
@@ -0,0 +1,125 @@
+### Please describe the problem.
+
+Continuation of the prior brief troubleshooting in [devblog](http://git-annex.branchable.com/devblog/day_518__S3_versioning_finishing_touches/#comment-ced84e6126b523170638f36a73cdb4b1).
+
+Joey:
+
+> As to the problems you're having with that S3 remote, it looks like it does not have public=yes enabled in its configuration at all, which is why git-annex is failing to download from it. 
+
+I do not think that is the case.  s3-PUBLIC has `public=yes`.  But it is entirely not clear why git-annex fails to download - no diagnostic error message at --debug level.  Please try yourself:
+
+### What steps will reproduce the problem?
+
+[[!format sh """
+~datalad > git clone https://github.com/OpenNeuroDatasets/ds001419               
+Cloning into 'ds001419'...
+remote: Enumerating objects: 420, done.
+remote: Counting objects: 100% (420/420), done.
+remote: Compressing objects: 100% (232/232), done.
+remote: Total 420 (delta 99), reused 418 (delta 97), pack-reused 0
+Receiving objects: 100% (420/420), 37.43 KiB | 2.88 MiB/s, done.
+Resolving deltas: 100% (99/99), done.
+
+~datalad > cd ds001419 
+
+~datalad/providers/openneuro/ds001419 > git annex enableremote s3-PUBLIC
+(merging origin/git-annex into git-annex...)
+(recording state in git...)
+download failed: Not Found
+
+  Remote origin not usable by git-annex; setting annex-ignore
+enableremote s3-PUBLIC ok
+(recording state in git...)
+
+~datalad/providers/openneuro/ds001419 > git annex whereis sub-01/anat/sub-01_T1w.nii.gz                                                                                       
+whereis sub-01/anat/sub-01_T1w.nii.gz (2 copies)  
+  	303581cd-3c58-4a60-b685-42cae4314c68 -- root@1f69c4ed80cf:/datalad/ds001419
+   	c21131c5-9e22-442d-a4d6-22d225603bc9 -- s3-PRIVATE
+
+  The following untrusted locations may also have copies:
+  	336309c0-d7f8-430e-bd2b-4bdc61e587cb -- [s3-PUBLIC]
+ok
+
+~datalad/providers/openneuro/ds001419 > git annex get --from s3-PUBLIC --debug sub-01/anat/sub-01_T1w.nii.gz
+[2018-09-25 11:30:48.623186962] read: git ["--git-dir=.git","--work-tree=.","--literal-pathspecs","show-ref","git-annex"]
+[2018-09-25 11:30:48.6294298] process done ExitSuccess
+[2018-09-25 11:30:48.629531355] read: git ["--git-dir=.git","--work-tree=.","--literal-pathspecs","show-ref","--hash","refs/heads/git-annex"]
+[2018-09-25 11:30:48.63678286] process done ExitSuccess
+[2018-09-25 11:30:48.637035079] read: git ["--git-dir=.git","--work-tree=.","--literal-pathspecs","log","refs/heads/git-annex..a6e5981635fd80f9186e6aa840312a97153508a4","--pretty=%H","-n1"]
+[2018-09-25 11:30:48.648701597] process done ExitSuccess
+[2018-09-25 11:30:48.648965608] chat: git ["--git-dir=.git","--work-tree=.","--literal-pathspecs","cat-file","--batch"]
+[2018-09-25 11:30:48.649547391] chat: git ["--git-dir=.git","--work-tree=.","--literal-pathspecs","cat-file","--batch-check=%(objectname) %(objecttype) %(objectsize)"]
+[2018-09-25 11:30:48.655933893] read: git ["--git-dir=.git","--work-tree=.","--literal-pathspecs","ls-files","--cached","-z","--","sub-01/anat/sub-01_T1w.nii.gz"]
+get sub-01/anat/sub-01_T1w.nii.gz (from s3-PUBLIC...) [2018-09-25 11:30:48.662890927] read: git ["--git-dir=.git","--work-tree=.","--literal-pathspecs","diff-tree","-z","--raw","--no-renames","-l0","-r","4b825dc642cb6eb9a060e54bf8d69288fbee4904","3bbd58c6720ad7a764409c219397a4e15e4ed77d","--"]
+[2018-09-25 11:30:48.670609446] chat: git ["--git-dir=.git","--work-tree=.","--literal-pathspecs","cat-file","--batch"]
+[2018-09-25 11:30:48.672897086] chat: git ["--git-dir=.git","--work-tree=.","--literal-pathspecs","cat-file","--batch-check=%(objectname) %(objecttype) %(objectsize)"]
+[2018-09-25 11:30:48.681427258] process done ExitSuccess
+failed
+[2018-09-25 11:30:48.740042089] process done ExitSuccess
+[2018-09-25 11:30:48.740263179] process done ExitSuccess
+[2018-09-25 11:30:48.740549287] process done ExitSuccess
+[2018-09-25 11:30:48.740805353] process done ExitSuccess
+git-annex: get: 1 failed
+
+~datalad/providers/openneuro/ds001419 > git cat-file -p git-annex:remote.log
+336309c0-d7f8-430e-bd2b-4bdc61e587cb bucket=openneuro.org datacenter=US encryption=none exporttree=yes fileprefix=ds001419/ host=s3.amazonaws.com name=s3-PUBLIC partsize=1GiB port=80 public=yes publicurl=yes storageclass=STANDARD type=S3 timestamp=1537889429.294083309s
+c21131c5-9e22-442d-a4d6-22d225603bc9 bucket=openneuro-private datacenter=US encryption=none exporttree=yes fileprefix=ds001419/ host=s3.amazonaws.com name=s3-PRIVATE partsize=1GiB port=80 storageclass=STANDARD type=S3 timestamp=1531961539.924031958s
+
+~datalad/providers/openneuro/ds001419 > git annex info
+repository mode: indirect
+trusted repositories: 0
+semitrusted repositories: 5
+	00000000-0000-0000-0000-000000000001 -- web
+ 	00000000-0000-0000-0000-000000000002 -- bittorrent
+ 	303581cd-3c58-4a60-b685-42cae4314c68 -- root@1f69c4ed80cf:/datalad/ds001419
+ 	3cf65be0-250d-4456-bfee-8501390e28c7 -- yoh@hopa:~/proj/datalad/providers/openneuro/ds001419 [here]
+ 	c21131c5-9e22-442d-a4d6-22d225603bc9 -- s3-PRIVATE
+untrusted repositories: 1
+	336309c0-d7f8-430e-bd2b-4bdc61e587cb -- [s3-PUBLIC]
+transfers in progress: none
+available local disk space: 1.76 gigabytes (+1 megabyte reserved)
+local annex keys: 0
+local annex size: 0 bytes
+annexed files in working tree: 7
+size of annexed files in working tree: 183.03 megabytes
+bloom filter size: 32 mebibytes (0% full)
+backend usage: 
+	MD5E: 7
+
+~datalad/providers/openneuro/ds001419 > git annex info s3-PUBLIC
+uuid: 336309c0-d7f8-430e-bd2b-4bdc61e587cb
+description: [s3-PUBLIC]
+remote: s3-PUBLIC
+trust: untrusted
+cost: 200.0
+type: S3
+creds: not available
+bucket: openneuro.org
+endpoint: s3.amazonaws.com
+port: 80
+storage class: STANDARD
+partsize: 1.07 gigabytes
+public: yes
+versioning: no
+encryption: none
+chunking: none
+export: yes
+remote annex keys: 27
+remote annex size: 183.03 megabytes (+ 20 unknown size)
+
+"""]]
+
+
+may be it is somehow interaction with the fact that the bucket is versioned, although export happened with annex version which didn't support versioning?
+
+It would be nice, before introducing versioning information as you outlined in the [comment](http://git-annex.branchable.com/devblog/day_518__S3_versioning_finishing_touches/#comment-01400aba452e6596e2b6b2967a0b9bc1) to enable access to the bucket at least to download the most recent version(s) of the file(s).
+
+### What version of git-annex are you using? On what operating system?
+
+6.20180913+git52-gdb1644808-1~ndall+1
+
+
+[[!meta author=yoh]]
+
+> Added an error message when download of an invalid url is attempted
+> and also made the url be listed by --debug. [[done]] --[[Joey]]
diff --git a/doc/bugs/unclear_why_annex_fails_to_get_a_file_from_a_public_S3_bucket_where_it_exported_before/comment_1_08514c489b916b2ae8e4e0122051856f._comment b/doc/bugs/unclear_why_annex_fails_to_get_a_file_from_a_public_S3_bucket_where_it_exported_before/comment_1_08514c489b916b2ae8e4e0122051856f._comment
new file mode 100644
index 0000000000..769f0f3f59
--- /dev/null
+++ b/doc/bugs/unclear_why_annex_fails_to_get_a_file_from_a_public_S3_bucket_where_it_exported_before/comment_1_08514c489b916b2ae8e4e0122051856f._comment
@@ -0,0 +1,18 @@
+[[!comment format=mdwn
+ username="joey"
+ subject="""comment 1"""
+ date="2018-09-25T17:31:36Z"
+ content="""
+Misconfiguration:
+
+	joey@darkstar:~/tmp/ds001419>git show git-annex:remote.log | grep openneuro.org
+	336309c0-d7f8-430e-bd2b-4bdc61e587cb bucket=openneuro.org datacenter=US encryption=none exporttree=yes fileprefix=ds001419/ host=s3.amazonaws.com name=s3-PUBLIC partsize=1GiB port=80 public=yes publicurl=yes storageclass=STANDARD type=S3 timestamp=1537895418.565736858s
+
+publicurl=yes means that the bucket's url is "yes".
+
+Since constructing an URI object from that invalid url fails, the url
+downloader fails early and this avoids the debug output that would normally be
+shown too when downloading an url by recent versions of git-annex.
+
+This also explains probably why whereis doesn't list any urls.
+"""]]
diff --git a/doc/bugs/unclear_why_annex_fails_to_get_a_file_from_a_public_S3_bucket_where_it_exported_before/comment_2_1ae05c24612f50cfe03e2569b332a522._comment b/doc/bugs/unclear_why_annex_fails_to_get_a_file_from_a_public_S3_bucket_where_it_exported_before/comment_2_1ae05c24612f50cfe03e2569b332a522._comment
new file mode 100644
index 0000000000..2988e47778
--- /dev/null
+++ b/doc/bugs/unclear_why_annex_fails_to_get_a_file_from_a_public_S3_bucket_where_it_exported_before/comment_2_1ae05c24612f50cfe03e2569b332a522._comment
@@ -0,0 +1,51 @@
+[[!comment format=mdwn
+ username="yarikoptic"
+ avatar="http://cdn.libravatar.org/avatar/f11e9c84cb18d26a1748c33b48c924b4"
+ subject="resolved!"
+ date="2018-09-25T17:55:30Z"
+ content="""
+oh man, thanks!  indeed I should have noticed it
+
+FTR here is a protocol of manual changes:
+
+[[!format sh \"\"\"
+$> cd ds001419 
+CHANGES  README  dataset_description.json  participants.tsv  sub-01/
+1 5689.....................................:Tue 25 Sep 2018 01:52:13 PM EDT:.
+(git-annex)hopa:~datalad/providers/openneuro/ds001419[master]
+$> git co git-annex                                    
+Switched to branch 'git-annex'
+1 5690.....................................:Tue 25 Sep 2018 01:52:15 PM EDT:.
+(git-annex)hopa:~datalad/providers/openneuro/ds001419[git-annex]git
+$> vim remote.log 
+1 5691.....................................:Tue 25 Sep 2018 01:53:06 PM EDT:.
+(git-annex)hopa:~datalad/providers/openneuro/ds001419[git-annex]git
+$> git diff
+diff --git a/remote.log b/remote.log
+index 162f6f6..6864376 100644
+--- a/remote.log
++++ b/remote.log
+@@ -1,2 +1,2 @@
+-336309c0-d7f8-430e-bd2b-4bdc61e587cb bucket=openneuro.org datacenter=US encryption=none exporttree=yes fileprefix=ds001419/ host=s3.amazonaws.com name=s3-PUBLIC partsize=1GiB port=80 public=yes publicurl=yes storageclass=STANDARD type=S3 timestamp=1537889429.294083309s
++336309c0-d7f8-430e-bd2b-4bdc61e587cb bucket=openneuro.org datacenter=US encryption=none exporttree=yes fileprefix=ds001419/ host=s3.amazonaws.com name=s3-PUBLIC partsize=1GiB port=80 public=yes publicurl=http://openneuro.org.s3.amazonaws.com/ storageclass=STANDARD type=S3 timestamp=1537889429.294083309s
+ c21131c5-9e22-442d-a4d6-22d225603bc9 bucket=openneuro-private datacenter=US encryption=none exporttree=yes fileprefix=ds001419/ host=s3.amazonaws.com name=s3-PRIVATE partsize=1GiB port=80 storageclass=STANDARD type=S3 timestamp=1531961539.924031958s
+1 5695.....................................:Tue 25 Sep 2018 01:53:08 PM EDT:.
+(git-annex)hopa:~datalad/providers/openneuro/ds001419[git-annex]git
+$> git commit -m 'made public url to be http://openneuro.org.s3.amazonaws.com/' -a
+[git-annex 7f1a89e] made public url to be http://openneuro.org.s3.amazonaws.com/
+ 1 file changed, 1 insertion(+), 1 deletion(-)
+1 5696.....................................:Tue 25 Sep 2018 01:53:15 PM EDT:.
+(git-annex)hopa:~datalad/providers/openneuro/ds001419[git-annex]git
+$> git co master
+Switched to branch 'master'
+Your branch is up to date with 'origin/master'.
+1 5697.....................................:Tue 25 Sep 2018 01:53:17 PM EDT:.
+(git-annex)hopa:~datalad/providers/openneuro/ds001419[git-annex]git
+$> git annex get sub-01/anat/sub-01_T1w.nii.gz
+get sub-01/anat/sub-01_T1w.nii.gz (from s3-PUBLIC...) (checksum...) ok
+(recording state in git...)
+
+\"\"\"]]
+
+can be closed, thanks again
+"""]]
diff --git a/doc/bugs/unclear_why_annex_fails_to_get_a_file_from_a_public_S3_bucket_where_it_exported_before/comment_3_ae81a2ac4b2167d48c8a447811a492a2._comment b/doc/bugs/unclear_why_annex_fails_to_get_a_file_from_a_public_S3_bucket_where_it_exported_before/comment_3_ae81a2ac4b2167d48c8a447811a492a2._comment
new file mode 100644
index 0000000000..f9032fdcef
--- /dev/null
+++ b/doc/bugs/unclear_why_annex_fails_to_get_a_file_from_a_public_S3_bucket_where_it_exported_before/comment_3_ae81a2ac4b2167d48c8a447811a492a2._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="yarikoptic"
+ avatar="http://cdn.libravatar.org/avatar/f11e9c84cb18d26a1748c33b48c924b4"
+ subject="comment 3"
+ date="2018-09-25T17:56:58Z"
+ content="""
+although if annex would have puked some warning or debug message about incorrect url, it would have saved both of us some time ;)
+"""]]
diff --git a/doc/bugs/unclear_why_annex_fails_to_get_a_file_from_a_public_S3_bucket_where_it_exported_before/comment_4_c49e48ea12ed0a0edb675b532279a5a8._comment b/doc/bugs/unclear_why_annex_fails_to_get_a_file_from_a_public_S3_bucket_where_it_exported_before/comment_4_c49e48ea12ed0a0edb675b532279a5a8._comment
new file mode 100644
index 0000000000..88fe91a551
--- /dev/null
+++ b/doc/bugs/unclear_why_annex_fails_to_get_a_file_from_a_public_S3_bucket_where_it_exported_before/comment_4_c49e48ea12ed0a0edb675b532279a5a8._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="yarikoptic"
+ avatar="http://cdn.libravatar.org/avatar/f11e9c84cb18d26a1748c33b48c924b4"
+ subject="comment 4"
+ date="2018-09-25T17:58:25Z"
+ content="""
+and on the third hand, annex could've generated urls for amazon buckets without mandating to specify them? ;)
+"""]]
diff --git a/doc/bugs/unlock_should_warn_if_file_isn__39__t_in_repo.mdwn b/doc/bugs/unlock_should_warn_if_file_isn__39__t_in_repo.mdwn
new file mode 100644
index 0000000000..8ddbf85cba
--- /dev/null
+++ b/doc/bugs/unlock_should_warn_if_file_isn__39__t_in_repo.mdwn
@@ -0,0 +1,53 @@
+### Please describe the problem.
+
+I'm using git-annex with the [calibre e-book library](https://calibre-ebook.com/) software. Sometimes calibre will rename a directory. After a directory rename git annex unlock no longer works. It works once I've committed the changes to git.
+
+It would be nice if git-annex could give an error to explain why the unlock fails. I'd even be happy with an extra flag to unlock, like --debug or --verbose to show the message.
+
+### What steps will reproduce the problem?
+
+1. create a directory inside a repository
+2. save a file in the directory
+3. add the file to git annex and commit
+4. rename the directory
+5. try unlocking the file
+
+### What version of git-annex are you using? On what operating system?
+
+6.20161012 on Debian
+
+### Please provide any additional information below.
+
+[[!format sh """
+# If you can, paste a complete transcript of the problem occurring here.
+# If the problem is with the git-annex assistant, paste in .git/annex/daemon.log
+
+    ~/scratch$ mkdir annex
+    ~/scratch$ cd annex/
+    ~/scratch/annex$ git init
+    Initialized empty Git repository in /home/edward/scratch/annex/.git/
+    ~/scratch/annex (master)$ git annex init --version=6
+    init  ok
+    (recording state in git...)
+    ~/scratch/annex (master)$ mkdir foo
+    ~/scratch/annex (master)$ cd foo
+    ~/scratch/annex/foo (master)$ echo test > test
+    ~/scratch/annex/foo (master)$ git annex add test
+    add test ok
+    (recording state in git...)
+    ~/scratch/annex/foo (master)$ git commit -m 'test'
+    [master (root-commit) 6368036] test
+     1 file changed, 1 insertion(+)
+     create mode 120000 foo/test
+    ~/scratch/annex/foo (master)$ cd ..
+    ~/scratch/annex (master)$ mv foo bar
+    ~/scratch/annex (master)$ cd bar
+    ~/scratch/annex/bar (master)$ git annex unlock test
+    ~/scratch/annex/bar (master)$ 
+
+# End of transcript or log.
+"""]]
+
+### Have you had any luck using git-annex before? (Sometimes we get tired of reading bug reports all day and a lil' positive end note does wonders)
+
+git-annex is amazing, I use it all the time. Thanks!
diff --git a/doc/bugs/unlock_should_warn_if_file_isn__39__t_in_repo/comment_1_4fb6a9aa7cd3b141b2d2f1b4acd2092f._comment b/doc/bugs/unlock_should_warn_if_file_isn__39__t_in_repo/comment_1_4fb6a9aa7cd3b141b2d2f1b4acd2092f._comment
new file mode 100644
index 0000000000..7fe2096045
--- /dev/null
+++ b/doc/bugs/unlock_should_warn_if_file_isn__39__t_in_repo/comment_1_4fb6a9aa7cd3b141b2d2f1b4acd2092f._comment
@@ -0,0 +1,18 @@
+[[!comment format=mdwn
+ username="joey"
+ subject="""comment 1"""
+ date="2016-11-07T17:25:10Z"
+ content="""
+In general, git-annex silently skips files that are not known to git,
+or are not annexed files. That's what's happening here.
+
+This skipping behavior can sometimes be confusing, if you are not
+familiar with it. But it's also very convenient. For example `git annex
+unlock *` will unlock only annexed files and not complain about any other
+files; `git annex unlock .` will recursively unlock only annexed files and not
+complain about non-annexed files or files that are already unlocked.
+
+In your example, git-annex has no way to tell that the file has been
+renamed; a file that has never been added to git would look the same
+to it as that renamed file.
+"""]]
diff --git a/doc/bugs/unlock_should_warn_if_file_isn__39__t_in_repo/comment_2_2e878c5ae1892f16f5b4e515fbe8875b._comment b/doc/bugs/unlock_should_warn_if_file_isn__39__t_in_repo/comment_2_2e878c5ae1892f16f5b4e515fbe8875b._comment
new file mode 100644
index 0000000000..8f34197b6c
--- /dev/null
+++ b/doc/bugs/unlock_should_warn_if_file_isn__39__t_in_repo/comment_2_2e878c5ae1892f16f5b4e515fbe8875b._comment
@@ -0,0 +1,15 @@
+[[!comment format=mdwn
+ username="edward"
+ avatar="http://cdn.libravatar.org/avatar/1e64ab07e0faced09520a5c589deb70b"
+ subject="comment 2"
+ date="2016-11-07T17:43:31Z"
+ content="""
+I see your point, feel free to close this bug.
+
+There is a way for git-annex to tell that the file is probably meant to be part of the repo because it is a symlink, it doesn't look the same as a new file.
+
+    ~/scratch/annex/bar (master)$ ls -l
+    total 4
+    lrwxrwxrwx 1 edward edward 181 Nov  4 18:02 test -> ../.git/annex/objects/w8/pv/SHA256E-s5--f2ca1bb6c7e907d06dafe4687e579fce76b37e4e93b7605022da52e6ccc26fd2/SHA256E-s5--f2ca1bb6c7e907d06dafe4687e579fce76b37e4e93b7605022da52e6ccc26fd2
+    ~/scratch/annex/bar (master)$ 
+"""]]
diff --git a/doc/bugs/use_of_annex_in_submodule_replaces_.git_with_incorrect_symlink.mdwn b/doc/bugs/use_of_annex_in_submodule_replaces_.git_with_incorrect_symlink.mdwn
new file mode 100644
index 0000000000..43295e7a87
--- /dev/null
+++ b/doc/bugs/use_of_annex_in_submodule_replaces_.git_with_incorrect_symlink.mdwn
@@ -0,0 +1,49 @@
+### Please describe the problem.
+
+Created a temp repository with a submodule under 3rd/visloc referring to another annex via ssh
+
+[[!format sh """
+
+# I have removed some output/wrong cmd calls, hopefully without side-effects
+
+hopa:/tmp/test
+$> cat 3rd/visloc/.git
+gitdir: ../../.git/modules/3rd/visloc
+
+$> git commit -m 'RF: ...' -a                                
+[master (root-commit) fc37b4f] RF: ...
+ 2 files changed, 4 insertions(+)
+ create mode 100644 .gitmodules
+ create mode 160000 3rd/visloc
+
+$> ls -l 3rd/visloc/.git                                                                            
+-rw------- 1 yoh yoh 38 Feb  8 10:26 3rd/visloc/.git
+
+$> cat 3rd/visloc/.git
+gitdir: ../../.git/modules/3rd/visloc
+
+$> cd 3rd/visloc/sub-01/rois 
+
+$> git annex get *
+fatal: Not a git repository: '../../.git'
+git-annex: First run: git-annex init
+
+$> cd -
+/tmp/test
+
+$> ls -l 3rd/visloc/.git    
+lrwxrwxrwx 1 yoh yoh 41 Feb  8 10:43 3rd/visloc/.git -> ../../../../../../.git/modules/3rd/visloc
+
+$> git annex version
+git-annex version: 6.20160126+gitg65f4442-1~ndall+1
+build flags: Assistant Webapp Pairing Testsuite S3(multipartupload) WebDAV Inotify DBus DesktopNotify XMPP ConcurrentOutput DNS TDFA TorrentParser Feeds Quvi
+key/value backends: SHA256E SHA256 SHA512E SHA512 SHA224E SHA224 SHA384E SHA384 SHA3_256E SHA3_256 SHA3_512E SHA3_512 SHA3_224E SHA3_224 SHA3_384E SHA3_384 SKEIN256E SKEIN256 SKEIN512E SKEIN512 SHA1E SHA1 MD5E MD5 WORM URL
+remote types: git gcrypt S3 bup directory rsync web bittorrent webdav tahoe glacier ddar hook external
+local repository version: 5
+supported repository versions: 5 6
+upgrade supported from repository versions: 0 1 2 4 5
+"""]]
+
+[[!meta author=yoh]]
+
+> [[fixed|done]]
diff --git a/doc/bugs/using_old_remote_format_generates_irritating_output.mdwn b/doc/bugs/using_old_remote_format_generates_irritating_output.mdwn
new file mode 100644
index 0000000000..1809ade37e
--- /dev/null
+++ b/doc/bugs/using_old_remote_format_generates_irritating_output.mdwn
@@ -0,0 +1,30 @@
+a special remote (encrypted rsync) that got copied to long ago (not sure when, there are old files that already have sizes in their unencrypted file names) seems to use the aa/bb/GPGHMACSHA1-- format instead of aaa/bbb/GPGHMACSHA1-. ``git annex fsck`` over such files produces very irritating output:
+
+
+fsck L1100423.JPG (gpg) (checking …remote…...) 
+rsync: change_dir "…somewhere…/0a0/8cd/GPGHMACSHA1--91234b770b34eeff811d09c97ce94bb2398b3d72" failed: No such file or directory (2)
+
+sent 8 bytes  received 12 bytes  40.00 bytes/sec
+total size is 0  speedup is 0.00
+rsync error: some files/attrs were not transferred (see previous errors) (code 23) at main.c(1536) [Receiver=3.0.9]
+
+  rsync failed -- run git annex again to resume file transfer
+
+GPGHMACSHA1--91234b770b34eeff811d09c97ce94bb2398b3d72
+     3922730 100%  623.81kB/s    0:00:06 (xfer#1, to-check=0/1)
+
+sent 30 bytes  received 3923328 bytes  523114.40 bytes/sec
+total size is 3922730  speedup is 1.00
+(checksum...) ok
+
+
+(observed with debian's git-annex 3.20121017).
+
+while this does output an "ok" at th end and a zero exit status, having such messages in an fsck is highly irritating.
+
+i see two ways to enhance the situation:
+
+* silence the "not found" error when the file is found in another location
+* a way to rename the files in the remote (i guess the aaa/bbb part can be derived from the file name; in that case, that could even be done w/o network interaction).
+
+[[!tag confirmed]]
diff --git a/doc/bugs/using_old_remote_format_generates_irritating_output/comment_1_fceba878f1097e27f056580e8d6d5b31._comment b/doc/bugs/using_old_remote_format_generates_irritating_output/comment_1_fceba878f1097e27f056580e8d6d5b31._comment
new file mode 100644
index 0000000000..d2208b57b4
--- /dev/null
+++ b/doc/bugs/using_old_remote_format_generates_irritating_output/comment_1_fceba878f1097e27f056580e8d6d5b31._comment
@@ -0,0 +1,26 @@
+[[!comment format=mdwn
+ username="http://christian.amsuess.com/chrysn"
+ nickname="chrysn"
+ subject="also affects `git annex get`"
+ date="2012-11-12T23:11:36Z"
+ content="""
+the same problem also shows up with `git annex get`:
+
+    get …filename… (from prometheus...) 
+    rsync: change_dir \"/home/shared/photos/encrypted_storage/63e/50b/GPGHMACSHA1--b83e8aaf05918ae2fc81652368f9d4068f938625\" failed: No such file or directory (2)
+    
+    sent 8 bytes  received 12 bytes  8.00 bytes/sec
+    total size is 0  speedup is 0.00
+    rsync error: some files/attrs were not transferred (see previous errors) (code 23) at main.c(1536) [Receiver=3.0.9]
+    
+      rsync failed -- run git annex again to resume file transfer
+    
+    GPGHMACSHA1--b83e8aaf05918ae2fc81652368f9d4068f938625
+          513214 100%   95.68kB/s    0:00:05 (xfer#1, to-check=0/1)
+    
+    sent 30 bytes  received 513396 bytes  44645.74 bytes/sec
+    total size is 513214  speedup is 1.00
+    ok
+
+again, it says \"ok\", but the \"no such file or directory\" / \"rsync failed\" is visually more prominent.
+"""]]
diff --git a/doc/bugs/using_old_remote_format_generates_irritating_output/comment_2_416992874813f120721a56d88b2bef65._comment b/doc/bugs/using_old_remote_format_generates_irritating_output/comment_2_416992874813f120721a56d88b2bef65._comment
new file mode 100644
index 0000000000..1dfd0bde72
--- /dev/null
+++ b/doc/bugs/using_old_remote_format_generates_irritating_output/comment_2_416992874813f120721a56d88b2bef65._comment
@@ -0,0 +1,9 @@
+[[!comment format=mdwn
+ username="http://joeyh.name/"
+ ip="4.252.11.120"
+ subject="comment 2"
+ date="2012-11-13T17:27:31Z"
+ content="""
+So, I don't know how to suppress this message without causing worse problems, like suppressing real error messages, and even password prompts.
+
+"""]]
diff --git a/doc/bugs/using_old_remote_format_generates_irritating_output/comment_3_a20f470c5226ac5693eb15146a02b3f5._comment b/doc/bugs/using_old_remote_format_generates_irritating_output/comment_3_a20f470c5226ac5693eb15146a02b3f5._comment
new file mode 100644
index 0000000000..8ee00709d8
--- /dev/null
+++ b/doc/bugs/using_old_remote_format_generates_irritating_output/comment_3_a20f470c5226ac5693eb15146a02b3f5._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="http://christian.amsuess.com/chrysn"
+ nickname="chrysn"
+ subject="comment 3"
+ date="2012-11-14T07:31:11Z"
+ content="""
+how about renaming the stored files, them? if you give me a pointer on how the directory names are generated, i can write a script that does the migration (some hash of the file name?). i suppose that's just a relic from another naming scheme, isn't it?
+"""]]
diff --git a/doc/bugs/using_old_remote_format_generates_irritating_output/comment_4_a81f06191bc03a7aad5929af99f0634e._comment b/doc/bugs/using_old_remote_format_generates_irritating_output/comment_4_a81f06191bc03a7aad5929af99f0634e._comment
new file mode 100644
index 0000000000..9ea8047678
--- /dev/null
+++ b/doc/bugs/using_old_remote_format_generates_irritating_output/comment_4_a81f06191bc03a7aad5929af99f0634e._comment
@@ -0,0 +1,28 @@
+[[!comment format=mdwn
+ username="http://joeyh.name/"
+ ip="4.252.11.120"
+ subject="comment 4"
+ date="2012-11-14T17:31:38Z"
+ content="""
+The new hash directory tree is generated in a simple to explain way. Take the md5sum of the key and the first 3 characters are the first directory, and the next 3 characters are the second directory.
+
+The old hash directory tree is rather harder to explain. It takes the md5sum of the key, but rather than a string, represents it as 4 32bit words. Only the first word is used. It is converted into a string by the same mechanism that would be used to encode a normal md5sum value into a string, but where that would normally encode the bits using the 16 characters 0-9a-f, this instead uses the 32 characters \"0123456789zqjxkmvwgpfZQJXKMVWGPF\". The first 2 letters of the resulting string are the first directory, and the second 2 are the second directory.
+
+There's probably a 1:1 mapping between this special md5 encoding an a regular md5 encoding. But it's certainly easier just to use the existing Haskell implementation of the hash. The following program, which needs to be built inside a git-annex source tree, reads keys on stdin, and outputs their old hash directory tree values, and their new values on stdout.
+
+
    +import Locations
+import Types.Key
+import Utility.Misc
+
+main = interact $ \s -> case file2key $ firstLine s of
+        Nothing -> \"bad key\"
+        Just k -> hashDirMixed k ++ \" \" ++ hashDirLower k ++ \"\n\"
+
    
+
+
    +joey@gnu:~/src/git-annex>ghc --make convert.hs
+joey@gnu:~/src/git-annex>echo WORM--foo | ./ convert
+jq/8w/ 2b1/ba3/
+
    
+"""]]
diff --git a/doc/bugs/using_old_remote_format_generates_irritating_output/comment_5_7438caecf78b4fb5d21f9f31dff95cf2._comment b/doc/bugs/using_old_remote_format_generates_irritating_output/comment_5_7438caecf78b4fb5d21f9f31dff95cf2._comment
new file mode 100644
index 0000000000..707ea52cf9
--- /dev/null
+++ b/doc/bugs/using_old_remote_format_generates_irritating_output/comment_5_7438caecf78b4fb5d21f9f31dff95cf2._comment
@@ -0,0 +1,14 @@
+[[!comment format=mdwn
+ username="http://christian.amsuess.com/chrysn"
+ nickname="chrysn"
+ subject="single-line migration"
+ date="2012-11-22T03:14:45Z"
+ content="""
+i've successfully applied this monster to migrate my repository (as always with such expressions, use it only if you know what it does, and have a backup):
+
+    find . -path './??/??/*' -type d \( -exec python -c 'import sys, hashlib, os; hash = hashlib.md5(sys.argv[1][8:]).hexdigest(); h1 = hash[:3]; h2 = hash[3:6]; os.mkdir(h1) if not os.path.exists(h1) else None; os.mkdir(h1+\"/\"+h2) if not os.path.exists(h1+\"/\"+h2) else None; os.rename(sys.argv[1], h1+\"/\"+h2+\"/\"+sys.argv[1][8:])' '{}' ';' -o -print \) -prune
+
+when executed in an encrypted git annex object directory, it takes all two-letter directories, executes a python expression on them (in case of failure, printing the file name it failed on), and doesn't continue searching there (-prune avoids error messages about moved-away directories).
+
+the python expression itself generates the hash described above, generates the required directories (put awkwardly in an `a if b else c` expression to avoid ifs (which wouldn't fit in a single line) and because python still doesn't have a proper mkdir-p function), and moves the found object there. (nb: using the system's `mkdir -p` would trigger [[another bug|bugs/using_old_remote_format_generates_irritating_output]]).
+"""]]
diff --git a/doc/bugs/using_regular_magic_file__warning_pollutes_stderr.mdwn b/doc/bugs/using_regular_magic_file__warning_pollutes_stderr.mdwn
new file mode 100644
index 0000000000..c94a562477
--- /dev/null
+++ b/doc/bugs/using_regular_magic_file__warning_pollutes_stderr.mdwn
@@ -0,0 +1,25 @@
+### Please describe the problem.
+
+Although probably not an annex issue but thought to let you know since you might see
+a quick resolution on your end
+
+Here is the log from datalad:
+
+[[!format sh """
+2016-02-25 22:53:32,886 [DEBUG] Running: ['git', '-c', 'receive.autogc=false', '-c', 'annex.alwayscommit=false', 'annex', 'add', '--debug', '--json', '-c', 'annex.largefiles=exclude=CHANGES* and exclude=README* and exclude=*.[mc] and exclude=dataset*.json and (exclude=*.txt or include=*/*.txt)  and (exclude=*.json or include=*/*.json) and (exclude=*.tsv or include=*/*.tsv)', 'README.txt'] (cmd.py:351)
+2016-02-25 22:53:32,979 [ERROR] stderr| /etc/magic, 4: Warning: using regular magic file `/usr/share/misc/magic' (cmd.py:351)
+
+
+or outside (file is already under annex)
+
+$> git annex --debug add -c 'annex.largefiles=exclude=CHANGES* and exclude=README*' README.txt
+/etc/magic, 4: Warning: using regular magic file `/usr/share/misc/magic'
+
+"""]]
+
+annex is up to date: 6.20160225+gitg229db26-1~ndall+1
+
+edit1: that is happening on jessie with file 1:5.22+15-2+deb8u1 if that is relevant
+[[!meta author=yoh]]
+
+> [[fixed|done]] --[[Joey]]
diff --git a/doc/bugs/v6_-_under_subdir__58___git_add___34__whines__34____44___git_commit___34__blows__34__.mdwn b/doc/bugs/v6_-_under_subdir__58___git_add___34__whines__34____44___git_commit___34__blows__34__.mdwn
new file mode 100644
index 0000000000..3e29ab0689
--- /dev/null
+++ b/doc/bugs/v6_-_under_subdir__58___git_add___34__whines__34____44___git_commit___34__blows__34__.mdwn
@@ -0,0 +1,51 @@
+### Please describe the problem.
+
+
+### What steps will reproduce the problem?
+
+
+### What version of git-annex are you using? On what operating system?
+
+
+### Please provide any additional information below.
+
+[[!format sh """
+$> mkdir repo; cd repo; git init; git annex init; git annex upgrade; mkdir -p subdir; cd subdir; touch file; git add file
+Initialized empty Git repository in /tmp/repo/.git/
+init  ok
+(recording state in git...)
+upgrade (v5 to v6...) ok
+git-annex: git: createProcess: runInteractiveProcess: chdir: does not exist (No such file or directory)
+error: external filter 'git-annex smudge --clean %f' failed 1
+error: external filter 'git-annex smudge --clean %f' failed
+
+$> echo $?; git status
+0
+git-annex: git: createProcess: runInteractiveProcess: chdir: does not exist (No such file or directory)
+error: external filter 'git-annex smudge --clean %f' failed 1
+error: external filter 'git-annex smudge --clean %f' failed
+On branch master
+
+Initial commit
+
+Changes to be committed:
+  (use "git rm --cached ..." to unstage)
+
+	new file:   file
+
+$> git commit -m 'added file'
+git-annex: git: createProcess: runInteractiveProcess: chdir: does not exist (No such file or directory)
+
+$> echo $?
+1
+
+$> git annex version
+git-annex version: 6.20180807+git230-gaa291acfe-1~ndall+1
+
+"""]]
+
+also happens with 6.20180807-1  but not with 6.20170101-1+deb9u2, so it is a regression
+
+[[!meta author=yoh]]
+
+> [[fixed|done]] --[[Joey]]
diff --git a/doc/bugs/v6_-_under_subdir__58___git_add___34__whines__34____44___git_commit___34__blows__34__/comment_1_1c86937cee067b5980852fcb066b0bfb._comment b/doc/bugs/v6_-_under_subdir__58___git_add___34__whines__34____44___git_commit___34__blows__34__/comment_1_1c86937cee067b5980852fcb066b0bfb._comment
new file mode 100644
index 0000000000..dbf7debf7b
--- /dev/null
+++ b/doc/bugs/v6_-_under_subdir__58___git_add___34__whines__34____44___git_commit___34__blows__34__/comment_1_1c86937cee067b5980852fcb066b0bfb._comment
@@ -0,0 +1,15 @@
+[[!comment format=mdwn
+ username="yarikoptic"
+ avatar="http://cdn.libravatar.org/avatar/f11e9c84cb18d26a1748c33b48c924b4"
+ subject="the reason is system wide git version!"
+ date="2018-09-07T00:13:33Z"
+ content="""
+HA -- it is compatibility issue with git somewhere.  
+I had system wide 1:2.13.0~rc1+next.20170430-1~nd90+1  installation of git, and that was the reason.  Similar blows with 1:2.11.0-3+nd1~nd90+1, but works fine with 1:2.19.0~rc2-1 .
+At least that explains how we did not run into it while testing in DataLad:
+
+1. we use git bundled with git-annex when git annex standalone is installed
+2. We use \"git annex add\" and not \"git add\" directly even in v6 mode repos
+
+But it brings the point of possibly needing to test with a regular debian build of git annex more  which would use system-wide git. Also that may be git annex should test git version used and blow with informative message whenever git is too old?
+"""]]
diff --git a/doc/bugs/v6_-_under_subdir__58___git_add___34__whines__34____44___git_commit___34__blows__34__/comment_2_bd552525b7bb2ceaad72c8fa3bf41c6b._comment b/doc/bugs/v6_-_under_subdir__58___git_add___34__whines__34____44___git_commit___34__blows__34__/comment_2_bd552525b7bb2ceaad72c8fa3bf41c6b._comment
new file mode 100644
index 0000000000..1921f54b5a
--- /dev/null
+++ b/doc/bugs/v6_-_under_subdir__58___git_add___34__whines__34____44___git_commit___34__blows__34__/comment_2_bd552525b7bb2ceaad72c8fa3bf41c6b._comment
@@ -0,0 +1,28 @@
+[[!comment format=mdwn
+ username="joey"
+ subject="""comment 2"""
+ date="2018-09-11T17:39:05Z"
+ content="""
+There are a few places in git-annex where it runs git with the cwd
+overridden, eg to the top of a git repository.
+If it somehow comes up with a path that doesn't exist,
+that would explain this error. The most likely one is
+in `Git.Config.read'`, which git-annex always runs at startup.
+
+So it probably comes down to git running the clean filter with
+the wrong `GIT_DIR` setting or something like that, leading to this as a
+cascading failure. And I vaguely remember seeing something like that in the
+git commit log, or having dealt with such a problem before.
+
+We could make git-annex upgrade fail to run with too old a git (and it
+probably should at least when used with a git too old to support
+smudge/clean filters at all!), but it wouldn't help if the repository was
+upgraded in a different environment. git is going to run the git-annex via
+the clean filter before any check can happen and the most git-annex can
+then do is error out somehow.
+
+See also
+
+which seems to be the same problem without v6; they didn't say what git
+version.
+"""]]
diff --git a/doc/bugs/v6_-_under_subdir__58___git_add___34__whines__34____44___git_commit___34__blows__34__/comment_3_8cfa8fe790ff2586374f46ed10cc8fbe._comment b/doc/bugs/v6_-_under_subdir__58___git_add___34__whines__34____44___git_commit___34__blows__34__/comment_3_8cfa8fe790ff2586374f46ed10cc8fbe._comment
new file mode 100644
index 0000000000..0057472a71
--- /dev/null
+++ b/doc/bugs/v6_-_under_subdir__58___git_add___34__whines__34____44___git_commit___34__blows__34__/comment_3_8cfa8fe790ff2586374f46ed10cc8fbe._comment
@@ -0,0 +1,22 @@
+[[!comment format=mdwn
+ username="joey"
+ subject="""comment 3"""
+ date="2018-09-11T18:08:43Z"
+ content="""
+Ah yes, I was remembering this workaround in git-annex:
+
+	* v6: Work around git bug that runs smudge/clean filters at the top of the
+	  repository while passing them a relative GIT_WORK_TREE that may point
+	  outside of the repository, by using GIT_PREFIX to get back to the
+	  subdirectory where a relative GIT_WORK_TREE is valid.
+
+Perhaps that broke with some older versions of git.
+[[!commit e50ed4ba48f93cf0addb3638a4a9605a10f17976]]
+has the gory details, which includes a git bug.
+
+A good way to debug this is to set:
+
+	git config filter.annex.clean 'bash -c "set | grep GIT_ >&2; pwd >&2; git-annex smudge --clean %f"'
+
+Then when git runs the clean filter it will display the git environment variables.
+"""]]
diff --git a/doc/bugs/v6_-_under_subdir__58___git_add___34__whines__34____44___git_commit___34__blows__34__/comment_4_40e9a5b641200c92725df31a654b6a05._comment b/doc/bugs/v6_-_under_subdir__58___git_add___34__whines__34____44___git_commit___34__blows__34__/comment_4_40e9a5b641200c92725df31a654b6a05._comment
new file mode 100644
index 0000000000..c5813f3899
--- /dev/null
+++ b/doc/bugs/v6_-_under_subdir__58___git_add___34__whines__34____44___git_commit___34__blows__34__/comment_4_40e9a5b641200c92725df31a654b6a05._comment
@@ -0,0 +1,16 @@
+[[!comment format=mdwn
+ username="joey"
+ subject="""comment 4"""
+ date="2018-09-11T18:33:22Z"
+ content="""
+Reproduced in a debian stable chroot with current git-annex in it.
+
+	root@darkstar:/tmp# mkdir repo; cd repo; git init; git annex init; git annex upgrade; git config filter.annex.clean 'bash -c "set | grep GIT_ >&2; pwd >&2; git-annex smudge --clean %f"' ; mkdir -p subdir; cd subdir; touch file; git add file
+	...
+	GIT_DIR=.git
+	GIT_PREFIX=subdir/
+	/tmp/repo
+	git-annex: git: createProcess: runInteractiveProcess: chdir: does not exist (No such file or directory)
+
+Verifies my theory.
+"""]]
diff --git a/doc/bugs/v6_-_under_subdir__58___git_add___34__whines__34____44___git_commit___34__blows__34__/comment_5_eaafc970cce2c71ce29f136c90e04504._comment b/doc/bugs/v6_-_under_subdir__58___git_add___34__whines__34____44___git_commit___34__blows__34__/comment_5_eaafc970cce2c71ce29f136c90e04504._comment
new file mode 100644
index 0000000000..acb36873f3
--- /dev/null
+++ b/doc/bugs/v6_-_under_subdir__58___git_add___34__whines__34____44___git_commit___34__blows__34__/comment_5_eaafc970cce2c71ce29f136c90e04504._comment
@@ -0,0 +1,57 @@
+[[!comment format=mdwn
+ username="joey"
+ subject="""comment 5"""
+ date="2018-09-11T18:51:07Z"
+ content="""
+Unfortunately git 2.11.0 has the bug that I made git-annex look at
+`GIT_PREFIX` to avoid.
+
+	root@darkstar:/tmp/repo/foo/bar# git --work-tree=../.. ls-files --modified /tmp/repo/
+	BASH_EXECUTION_STRING='set | grep GIT_ >&2; pwd >&2; git-annex smudge --clean '\''subdir/file'\'''
+	GIT_DIR=/tmp/repo/.git
+	GIT_PREFIX=foo/bar/
+	GIT_WORK_TREE=../..
+	/tmp/repo
+
+So, it's not as simple as not looking at `GIT_PREFIX` at all with this
+version of git. When `GIT_WORK_TREE` is relative, and `GIT_PREFIX` is set,
+it needs to combine them to get the actual path to the work tree.
+
+Perhaps git-annex should only use `GIT_PREFIX` 
+for fixup of relative `GIT_WORK_TREE`, but not for `GIT_DIR`. I've so
+far only seen it be needed for `GIT_WORK_TREE`; I only made it also
+be used for `GIT_DIR` out of an assumption git would be consistent in its
+bugginess.
+
+Oh hell, here's another way it fails, with git 2.19:
+
+	joey@darkstar:/tmp/repo/subdir/foo> git --work-tree=../.. status
+	BASH_EXECUTION_STRING='set | grep GIT_ >&2; pwd >&2; git-annex smudge --clean '\''subdir/file'\'''
+	GIT_DIR=/tmp/repo/.git
+	GIT_EXEC_PATH=/usr/lib/git-core
+	GIT_MERGE_AUTOEDIT=no
+	GIT_PAGER=
+	GIT_PREFIX=subdir/foo/
+	GIT_WORK_TREE=.
+	/tmp/repo
+	git-annex: subdir/file: getFileStatus: does not exist (No such file or directory)
+	git-annex: smudge: 1 failed
+
+That one does not involve `GIT_DIR`, instead `GIT_WORK_TREE` is not relative to
+`GIT_PREFIX` so the workaround that assumes it is breaks. I guess that we
+could say that `GIT_DIR` is not relative to some directory in this case,
+so the `GIT_PREFIX` is irrelevant.
+
+Kind of feels like git's behavior is so inconsistent and ill-specified
+that trying to work around the bugs in it is likely to just be a neverending
+source of bugs.
+
+Unfortunately the git devs have so far ignored my bug report despite it
+having an easy test case needing nothing more than a simple shell script to
+reproduce. And there are plenty of differently broken git versions out there
+even if they eventually fix it, so it seems git-annex has to deal with this
+mess somehow.
+
+I'm going with not using `GIT_PREFIX` for `GIT_DIR` and not for
+`GIT_WORK_TREE=.` and hope that's enough.
+"""]]
diff --git a/doc/bugs/v6__58___git_status__47__add_after_unlock_is_linear_to_the_file_size.mdwn b/doc/bugs/v6__58___git_status__47__add_after_unlock_is_linear_to_the_file_size.mdwn
new file mode 100644
index 0000000000..3acf63831f
--- /dev/null
+++ b/doc/bugs/v6__58___git_status__47__add_after_unlock_is_linear_to_the_file_size.mdwn
@@ -0,0 +1,24 @@
+### Please describe the problem.
+
+After unlocking a repository, the next git status will take time linear to the file size. It seems to be highly inefficient (the I/O on my SSD is not anywhere near the limit).  
+For a 500Mb flac album it takes ~5–10s, for my 100GB local music archive it took well over 30min.
+
+### What steps will reproduce the problem?
+
+```
+git annex upgrade
+git annex unlock
+git status / git add -A
+```
+
+### What version of git-annex are you using? On what operating system?
+
+6.20160527  
+NixOS (master branch)
+
+### Have you had any luck using git-annex before? (Sometimes we get tired of reading bug reports all day and a lil' positive end note does wonders)
+
+Not yet. :)  
+But will use it to keep my music library and sync a smaller (ogg) version to my various devices (with beets handling metadata and conversion)
+
+> [[fixed|done]] --[[Joey]]
diff --git a/doc/bugs/v6__58___git_status__47__add_after_unlock_is_linear_to_the_file_size/comment_1_ea798d60cbdfa7534bad0ba47119379d._comment b/doc/bugs/v6__58___git_status__47__add_after_unlock_is_linear_to_the_file_size/comment_1_ea798d60cbdfa7534bad0ba47119379d._comment
new file mode 100644
index 0000000000..2ebcf375e3
--- /dev/null
+++ b/doc/bugs/v6__58___git_status__47__add_after_unlock_is_linear_to_the_file_size/comment_1_ea798d60cbdfa7534bad0ba47119379d._comment
@@ -0,0 +1,9 @@
+[[!comment format=mdwn
+ username="mail@4e627fd997ef5ca9f75e62ffc0aba5b27bd6aea1"
+ nickname="mail"
+ subject="comment 1"
+ date="2016-06-19T16:27:15Z"
+ content="""
+I noticed after filing the bug that the first `git commit` takes as much (or even more!) time as the `git add -A` before it.  
+So you pay overhead twice somehow.
+"""]]
diff --git a/doc/bugs/v6__58___git_status__47__add_after_unlock_is_linear_to_the_file_size/comment_2_7efc9b056b88fc7a75309a80445b5102._comment b/doc/bugs/v6__58___git_status__47__add_after_unlock_is_linear_to_the_file_size/comment_2_7efc9b056b88fc7a75309a80445b5102._comment
new file mode 100644
index 0000000000..43f9b37af5
--- /dev/null
+++ b/doc/bugs/v6__58___git_status__47__add_after_unlock_is_linear_to_the_file_size/comment_2_7efc9b056b88fc7a75309a80445b5102._comment
@@ -0,0 +1,10 @@
+[[!comment format=mdwn
+ username="joey"
+ subject="""comment 2"""
+ date="2016-07-06T18:56:04Z"
+ content="""
+This problem is dicussed in detail in [[todo/smudge]] and is the main
+reason that v6 mode is not production ready.
+
+I have patches to git that avoid this problem.
+"""]]
diff --git a/doc/bugs/v6__58___git_status__47__add_after_unlock_is_linear_to_the_file_size/comment_3_910399daee8c9ad9909068a23f0eb5c9._comment b/doc/bugs/v6__58___git_status__47__add_after_unlock_is_linear_to_the_file_size/comment_3_910399daee8c9ad9909068a23f0eb5c9._comment
new file mode 100644
index 0000000000..b98157be16
--- /dev/null
+++ b/doc/bugs/v6__58___git_status__47__add_after_unlock_is_linear_to_the_file_size/comment_3_910399daee8c9ad9909068a23f0eb5c9._comment
@@ -0,0 +1,12 @@
+[[!comment format=mdwn
+ username="ynikitenko"
+ avatar="http://cdn.libravatar.org/avatar/168d629704097ddc596f75ca32a687a3"
+ subject="same problem"
+ date="2017-10-24T09:31:06Z"
+ content="""
+When could that be fixed? 
+
+I face the same problem. My repo is only about 2Gb, but it syncs quite slow (like minutes or tens of minutes, but with every smallest change that I want to be synced). I don't want to use other than version 6 and thin to save space, but it takes much time. 
+
+For another repo of 100 thousand files (though 2.6Gb) git-annex commit did not even work (I killed that after about 20 hours of work - it loaded the machine, created a lot of files (twice the size of the repo), but did not stop in that time).
+"""]]
diff --git a/doc/bugs/v6__58___git_status__47__add_after_unlock_is_linear_to_the_file_size/comment_4_c005382360247da300d5e274d62bf19a._comment b/doc/bugs/v6__58___git_status__47__add_after_unlock_is_linear_to_the_file_size/comment_4_c005382360247da300d5e274d62bf19a._comment
new file mode 100644
index 0000000000..c81bf7a6c6
--- /dev/null
+++ b/doc/bugs/v6__58___git_status__47__add_after_unlock_is_linear_to_the_file_size/comment_4_c005382360247da300d5e274d62bf19a._comment
@@ -0,0 +1,10 @@
+[[!comment format=mdwn
+ username="ynikitenko"
+ avatar="http://cdn.libravatar.org/avatar/168d629704097ddc596f75ca32a687a3"
+ subject="not so slow"
+ date="2017-10-27T12:29:08Z"
+ content="""
+I'd like to add to my previous comment that now git-annex works pretty well with the same repository (I mean the one that I was able to build and which I use). 
+
+I don't know why that happened and why that was slow before. 
+"""]]
diff --git a/doc/bugs/v6__58___git_status__47__add_after_unlock_is_linear_to_the_file_size/comment_5_7c785f07f73ddbcf316fa6144b727375._comment b/doc/bugs/v6__58___git_status__47__add_after_unlock_is_linear_to_the_file_size/comment_5_7c785f07f73ddbcf316fa6144b727375._comment
new file mode 100644
index 0000000000..41c2b7588c
--- /dev/null
+++ b/doc/bugs/v6__58___git_status__47__add_after_unlock_is_linear_to_the_file_size/comment_5_7c785f07f73ddbcf316fa6144b727375._comment
@@ -0,0 +1,12 @@
+[[!comment format=mdwn
+ username="joey"
+ subject="""comment 5"""
+ date="2017-10-30T17:12:00Z"
+ content="""
+IIRC once git checks the work tree file one time and realizes it's not
+really changed, it then avoids the unncessarily expensive operation
+happening again.
+
+These inneficiencies can't be fixed without changes to git. It's all
+discussed in [[todo/smudge]] in detail.
+"""]]
diff --git a/doc/bugs/v6__58___write_permission_on_annex_object_if_unlocked__44___modified__44___and_readded.mdwn b/doc/bugs/v6__58___write_permission_on_annex_object_if_unlocked__44___modified__44___and_readded.mdwn
new file mode 100644
index 0000000000..a7c01972b2
--- /dev/null
+++ b/doc/bugs/v6__58___write_permission_on_annex_object_if_unlocked__44___modified__44___and_readded.mdwn
@@ -0,0 +1,35 @@
+### Please describe the problem.
+
+With a v6 repo, if I unlock an annex file, change it, and then add it back, the annex object is writeable.
+
+### What steps will reproduce the problem?
+
+    git init tmp-v6 && cd tmp-v6
+    git annex init --version=6
+    echo foo >foo && git annex add foo && git commit -m'add foo'
+    git annex unlock foo
+    echo more >>foo && git annex add foo && git commit -m'modify foo'
+    ls -l $(readlink foo)
+    # -rw-r--r-- 1 kyle kyle 9 Aug 29 17:18 .git/annex/objects/60/QW/SHA256E-s9--323409d9a706bc08d0b2c7f71309e21a757367c81cffb405a88e61749d79952d/SHA256E-s9--323409d9a706bc08d0b2c7f71309e21a757367c81cffb405a88e61749d79952d
+
+### What version of git-annex are you using? On what operating system?
+
+    git annex version
+
+    # git-annex version: 6.20180808-gdad627fa9
+    # build flags: Assistant Webapp Pairing S3(multipartupload)(storageclasses) WebDAV Inotify ConcurrentOutput TorrentParser Feeds Testsuite
+    # dependency versions: aws-0.17.1 bloomfilter-2.0.1.0 cryptonite-0.23 DAV-1.3.1 feed-0.3.12.0 ghc-8.0.2 http-client-0.5.7.0 persistent-sqlite-2.6.2 torrent-10000.1.1 uuid-1.3.13 yesod-1.4.5
+    # key/value backends: SHA256E SHA256 SHA512E SHA512 SHA224E SHA224 SHA384E SHA384 SHA3_256E SHA3_256 SHA3_512E SHA3_512 SHA3_224E SHA3_224 SHA3_384E SHA3_384 SKEIN256E SKEIN256 SKEIN512E SKEIN512 BLAKE2B256E BLAKE2B256 BLAKE2B512E BLAKE2B512 BLAKE2B160E BLAKE2B160 BLAKE2B224E BLAKE2B224 BLAKE2B384E BLAKE2B384 BLAKE2S256E BLAKE2S256 BLAKE2S160E BLAKE2S160 BLAKE2S224E BLAKE2S224 BLAKE2SP256E BLAKE2SP256 BLAKE2SP224E BLAKE2SP224 SHA1E SHA1 MD5E MD5 WORM URL
+    # remote types: git gcrypt p2p S3 bup directory rsync web bittorrent webdav adb tahoe glacier ddar hook external
+    # operating system: linux x86_64
+    # supported repository versions: 3 5 6
+    # upgrade supported from repository versions: 0 1 2 3 4 5
+    # local repository version: 6
+
+Debian Stretch, but building git-annex from source with `stack build`.
+
+### Have you had any luck using git-annex before? (Sometimes we get tired of reading bug reports all day and a lil' positive end note does wonders)
+
+Yes, everyday.  Thank you for git-annex :-)
+
+> [[fixed|done]] --[[Joey]]
diff --git a/doc/bugs/v6__58___write_permission_on_annex_object_if_unlocked__44___modified__44___and_readded/comment_1_663a9b9a1bd7846d79e35be8fb0be97b._comment b/doc/bugs/v6__58___write_permission_on_annex_object_if_unlocked__44___modified__44___and_readded/comment_1_663a9b9a1bd7846d79e35be8fb0be97b._comment
new file mode 100644
index 0000000000..29ecffdd9e
--- /dev/null
+++ b/doc/bugs/v6__58___write_permission_on_annex_object_if_unlocked__44___modified__44___and_readded/comment_1_663a9b9a1bd7846d79e35be8fb0be97b._comment
@@ -0,0 +1,14 @@
+[[!comment format=mdwn
+ username="joey"
+ subject="""comment 1"""
+ date="2018-08-30T15:00:39Z"
+ content="""
+Hmm, I don't reproduce this here.
+
+Also AFAIK there is no difference in the code path used here
+to move a file into the annex between v5 and v6. Does it happen with v5?
+
+Seems there is either something in your environment (umask, core.sharedRepository
+git config etc though I don't think any of those would explain this
+particular file mode) or the build on stretch is somehow behaving differently.
+"""]]
diff --git a/doc/bugs/v6__58___write_permission_on_annex_object_if_unlocked__44___modified__44___and_readded/comment_2_066cc775495fc4eb0b054b3737f20d5a._comment b/doc/bugs/v6__58___write_permission_on_annex_object_if_unlocked__44___modified__44___and_readded/comment_2_066cc775495fc4eb0b054b3737f20d5a._comment
new file mode 100644
index 0000000000..9412b4793d
--- /dev/null
+++ b/doc/bugs/v6__58___write_permission_on_annex_object_if_unlocked__44___modified__44___and_readded/comment_2_066cc775495fc4eb0b054b3737f20d5a._comment
@@ -0,0 +1,41 @@
+[[!comment format=mdwn
+ username="yarikoptic"
+ avatar="http://cdn.libravatar.org/avatar/f11e9c84cb18d26a1748c33b48c924b4"
+ subject="confirming"
+ date="2018-08-30T15:57:39Z"
+ content="""
+just wanted to say that I see the same (thanks Kyle for copy/pasteable example):
+
+[[!format sh \"\"\"
+$> git init tmp-v6 && cd tmp-v6
+git annex init --version=6     
+echo foo >foo && git annex add foo && git commit -m'add foo'
+git annex unlock foo                                        
+echo more >>foo && git annex add foo && git commit -m'modify foo'
+ls -l $(readlink foo)                                            
+Initialized empty Git repository in /tmp/tmp-v6/.git/
+init  ok
+(recording state in git...)
+add foo ok
+(recording state in git...)
+[master (root-commit) 19d9427] add foo
+ 1 file changed, 1 insertion(+)
+ create mode 120000 foo
+unlock foo ok
+(recording state in git...)
+add foo ok
+(recording state in git...)
+[master 9a2a079] modify foo
+ 1 file changed, 1 insertion(+), 1 deletion(-)
+-rw------- 1 yoh yoh 9 Aug 30 11:52 .git/annex/objects/60/QW/SHA256E-s9--323409d9a706bc08d0b2c7f71309e21a757367c81cffb405a88e61749d79952d/SHA256E-s9--323409d9a706bc08d0b2c7f71309e21a757367c81cffb405a88e61749d79952d
+
+$> umask
+077
+
+$> git annex version  
+git-annex version: 6.20180807+git142-g1a08a8613-1~ndall+1
+
+\"\"\"]]
+
+tried with umask 022 - the same
+"""]]
diff --git a/doc/bugs/v6__58___write_permission_on_annex_object_if_unlocked__44___modified__44___and_readded/comment_3_755c9123c3d1c627aca51932c121bdc5._comment b/doc/bugs/v6__58___write_permission_on_annex_object_if_unlocked__44___modified__44___and_readded/comment_3_755c9123c3d1c627aca51932c121bdc5._comment
new file mode 100644
index 0000000000..d4f9ac7c68
--- /dev/null
+++ b/doc/bugs/v6__58___write_permission_on_annex_object_if_unlocked__44___modified__44___and_readded/comment_3_755c9123c3d1c627aca51932c121bdc5._comment
@@ -0,0 +1,9 @@
+[[!comment format=mdwn
+ username="yarikoptic"
+ avatar="http://cdn.libravatar.org/avatar/f11e9c84cb18d26a1748c33b48c924b4"
+ subject="reproduced more"
+ date="2018-08-30T16:02:39Z"
+ content="""
+tried across a number of boxes, and other users, and older git-annex -- the same effect everywhere.
+The common denominator - I was using neurodebian build (will check next)
+"""]]
diff --git a/doc/bugs/v6__58___write_permission_on_annex_object_if_unlocked__44___modified__44___and_readded/comment_4_7e741ba5998cbc707951bf5dfc50479f._comment b/doc/bugs/v6__58___write_permission_on_annex_object_if_unlocked__44___modified__44___and_readded/comment_4_7e741ba5998cbc707951bf5dfc50479f._comment
new file mode 100644
index 0000000000..c4ddbfef4a
--- /dev/null
+++ b/doc/bugs/v6__58___write_permission_on_annex_object_if_unlocked__44___modified__44___and_readded/comment_4_7e741ba5998cbc707951bf5dfc50479f._comment
@@ -0,0 +1,12 @@
+[[!comment format=mdwn
+ username="kyle"
+ avatar="http://cdn.libravatar.org/avatar/7d6e85cde1422ad60607c87fa87c63f3"
+ subject="v5"
+ date="2018-08-30T16:05:01Z"
+ content="""
+> Also AFAIK there is no difference in the code path used here to move a file into the annex between v5 and v6. Does it happen with v5?
+
+No, it doesn't: 
+
+    -r--r--r-- 1 kyle kyle 9 Aug 29 17:02 .git/annex/objects/60/QW/SHA256E-s9--323409d9a706bc08d0b2c7f71309e21a757367c81cffb405a88e61749d79952d/SHA256E-s9--323409d9a706bc08d0b2c7f71309e21a757367c81cffb405a88e61749d79952d
+"""]]
diff --git a/doc/bugs/v6__58___write_permission_on_annex_object_if_unlocked__44___modified__44___and_readded/comment_5_d40df34ebcb2f3e1610c7b85bd3ef4e3._comment b/doc/bugs/v6__58___write_permission_on_annex_object_if_unlocked__44___modified__44___and_readded/comment_5_d40df34ebcb2f3e1610c7b85bd3ef4e3._comment
new file mode 100644
index 0000000000..a7d4f03114
--- /dev/null
+++ b/doc/bugs/v6__58___write_permission_on_annex_object_if_unlocked__44___modified__44___and_readded/comment_5_d40df34ebcb2f3e1610c7b85bd3ef4e3._comment
@@ -0,0 +1,26 @@
+[[!comment format=mdwn
+ username="yarikoptic"
+ avatar="http://cdn.libravatar.org/avatar/f11e9c84cb18d26a1748c33b48c924b4"
+ subject="clear docker for ubuntu 18:04 - the same"
+ date="2018-08-30T16:07:51Z"
+ content="""
+[[!format sh \"\"\"
+root@6f7d56840b4b:/tmp/tmp-v6# ls -l $(readlink foo)
+-rw-r--r-- 1 root root 9 Aug 30 16:06 .git/annex/objects/60/QW/SHA256E-s9--323409d9a706bc08d0b2c7f71309e21a757367c81cffb405a88e61749d79952d/SHA256E-s9--323409d9a706bc08d0b2c7f71309e21a757367c81cffb405a88e61749d79952d
+root@6f7d56840b4b:/tmp/tmp-v6# history
+    1  apt-get update -qqq
+    2  apt-get install -t eatmydata
+    3  apt-get install -y eatmydata
+    4  eatmydata apt-get install -y git-annex
+    5  cd /tmp
+    6  git init tmp-v6 && cd tmp-v6
+    7  git annex init --version=6
+    8  echo foo >foo && git annex add foo && git commit -m'add foo'
+    9  git annex unlock foo
+   10  echo more >>foo && git annex add foo && git commit -m'modify foo'
+   11  ls -l $(readlink foo)
+   12  history
+root@6f7d56840b4b:/tmp/tmp-v6# git annex version
+git-annex version: 6.20180227
+\"\"\"]]
+"""]]
diff --git a/doc/bugs/v6__58___write_permission_on_annex_object_if_unlocked__44___modified__44___and_readded/comment_6_2d3e926459c04c6a9a2c5a43ee9733e4._comment b/doc/bugs/v6__58___write_permission_on_annex_object_if_unlocked__44___modified__44___and_readded/comment_6_2d3e926459c04c6a9a2c5a43ee9733e4._comment
new file mode 100644
index 0000000000..3db37eee6b
--- /dev/null
+++ b/doc/bugs/v6__58___write_permission_on_annex_object_if_unlocked__44___modified__44___and_readded/comment_6_2d3e926459c04c6a9a2c5a43ee9733e4._comment
@@ -0,0 +1,23 @@
+[[!comment format=mdwn
+ username="joey"
+ subject="""comment 6"""
+ date="2018-09-05T18:26:33Z"
+ content="""
+Reproduced.
+
+Analysis: In v6, `git annex add` runs `git ls-files --modified`, 
+which runs the clean filter on the unlocked file as git sees it was
+modified. Which in ingests the file with lockingFile = False.
+So, the annex object doesn't get the write bit cleared at that point.
+Then when `git annex add` gets around to ingesting the file
+itself, since the annex object is already present it's used as-is.
+
+`git add` of a new file in v6 also puts content in the annex
+with the write bit set. If a different file with that same content is then passed
+to `git annex add`, the same thing happens with the symlink to unprotected
+content.
+
+So, linkToAnnex should freezeContent. That would solve all cases that lead
+to this. (But not when annex.thin has made the annex object a hard link,
+in that case it being writable is expected.)
+"""]]
diff --git a/doc/bugs/v6_appears_to_not_thin.mdwn b/doc/bugs/v6_appears_to_not_thin.mdwn
new file mode 100644
index 0000000000..7ccf06d026
--- /dev/null
+++ b/doc/bugs/v6_appears_to_not_thin.mdwn
@@ -0,0 +1,80 @@
+[[!meta title="v6 does not thin on hyperv"]]
+
+### Please describe the problem.
+
+First of all, I don't really believe the following is a good bug report, from my own career in SW development. I know what to do. However, this is a start to test the waters; I received no response in the general forum (not a complaint), but I am not sure if this is simply one of those issues for which the solution is so simple, that no one can bring themselves to reply.
+
+I can provide further detail, like console copies, but I figured that providing the config output to be a good first step. Apologies for the format of the config file: it was copied fine, but pasted on one line.
+
+
+It seems thinning is not doing what I expected:
+
+Which is: unlocking a file creates a hard link in the working directory, linked to the annex file, with the resulting dir size being the size of the one big file.
+
+Are my expectations correct?
+
+Is there something I am missing?
+
+
+
+### What steps will reproduce the problem?
+
+copied a big file to the working directory
+
+git add; git commit
+
+Locking the big file creates the symlink, with the file in the annex.
+
+Unlocking the big file creates a real file in the working dir, and the annex file stays there.
+
+The total dir is therefore twice the size of the big file.
+
+ls -li shows a unique inode for each file. Therefore it appears to not be a hard link.
+
+
+### What version of git-annex are you using? On what operating system?
+
+git-annex version: 6.20160524+gitg2b7b2c4-1~ndall+1
+
+Debian 8.5
+
+
+### Please provide any additional information below.
+
+repo config file:
+
+[core] 
+repositoryformatversion = 0 
+filemode = true 
+bare = false 
+logallrefupdates = true 
+
+[annex] 
+uuid = c5c203ec-9353-4213-8af5-6fcb8de36ca2 
+version = 6 
+thin = true 
+
+[filter "annex"] 
+smudge = git-annex smudge %f 
+clean = git-annex smudge --clean %f
+
+
+
+
+[[!format sh """
+# If you can, paste a complete transcript of the problem occurring here.
+# If the problem is with the git-annex assistant, paste in .git/annex/daemon.log
+
+
+# End of transcript or log.
+"""]]
+
+### Have you had any luck using git-annex before? (Sometimes we get tired of reading bug reports all day and a lil' positive end note does wonders)
+
+You bet. It is simply great. I am in the process of migrating all my big data. I have roughly 6TB, and it is taking me many weeks.
+
+I am creating a typical duplicate topology. 2 remotes, 2 backup nodes. Multiple clients, each connected to the 2 remotes. Each client will only clone the repos or parts thereof that are relevant to that client.
+
+I have multiple clients on multiple VMs. I generally separate functions to VMs. I.e. postgresql to a pair of VMs; etc. Poorly behaving apps like iTunes to its very own penalty box win VM. HA to its own VM. etc.
+
+
diff --git a/doc/bugs/v6_appears_to_not_thin/comment_1_d62791da504f6cf9bb26fa588756234c._comment b/doc/bugs/v6_appears_to_not_thin/comment_1_d62791da504f6cf9bb26fa588756234c._comment
new file mode 100644
index 0000000000..c2d5a804e1
--- /dev/null
+++ b/doc/bugs/v6_appears_to_not_thin/comment_1_d62791da504f6cf9bb26fa588756234c._comment
@@ -0,0 +1,36 @@
+[[!comment format=mdwn
+ username="joey"
+ subject="""comment 1"""
+ date="2016-07-06T18:42:47Z"
+ content="""
+annex.thin can only work on filesystems that support hard links,
+so the most likely reason, from the information you have given,
+is that the repository is on a FAT or other non-hard-link supporting
+filesystem.
+
+	joey@darkstar:~/tmp/xxx>date > bigfile
+	joey@darkstar:~/tmp/xxx>git annex add bigfile
+	add bigfile ok
+	(recording state in git...)
+	joey@darkstar:~/tmp/xxx>git commit -m add
+	[master 39d19a4] add
+	 1 file changed, 1 insertion(+), 1 deletion(-)
+	 rewrite bigfile (100%)
+	 mode change 100644 => 120000
+	joey@darkstar:~/tmp/xxx>git annex unlock bigfile
+	unlock bigfile ok
+	(recording state in git...)
+	joey@darkstar:~/tmp/xxx>ls -l bigfile
+	-rw-r--r-- 2 joey joey 30 Jul  6 14:46 bigfile
+
+It's working fine here as seen by the fact that bigfile
+has a link count of 2 after being unlocked.
+
+Other possibilies:
+
+* If the repository was cloned using `git clone --shared`,
+  annex.thin is not supported.
+* If there are multiple files in the working tree that have the same
+  content, annex.thin will only hard link one of them; the others
+  will be non-thin copies.
+"""]]
diff --git a/doc/bugs/v6_appears_to_not_thin/comment_2_82ff17c4063791d91a4fbe4e31802595._comment b/doc/bugs/v6_appears_to_not_thin/comment_2_82ff17c4063791d91a4fbe4e31802595._comment
new file mode 100644
index 0000000000..e6093c2159
--- /dev/null
+++ b/doc/bugs/v6_appears_to_not_thin/comment_2_82ff17c4063791d91a4fbe4e31802595._comment
@@ -0,0 +1,82 @@
+[[!comment format=mdwn
+ username="Stan"
+ subject="Success / failure scenarios"
+ date="2016-07-07T23:29:19Z"
+ content="""
+Thanks again Joey. My progress is detailed below. I can run/collect more detailed scenarios if it will help.
+
+Summary:
+
+Success with a VM on qemu.
+Failure with VMs on hyperv.
+
+* Hardlinks are not created for the VMs on hyperv, in spite of the VM filesystems being ext4.
+* This may only be associative, and not causal, but it is somewhat surprising. Yet, I am no hardlink expert.
+* I can however make it work if I use a VM on qemu, so far; so that is very good.
+
+Setup:
+
+* mkdir gitx1
+* git init
+* git annex init --version=6
+* git config annex.thin true
+* git annex fix
+* copy bigfile to gitx1
+* git annex add bigfile
+* git commit -m add
+* ls -li
+
+Test Procedure:
+
+* unlock bigfile
+* ls -li
+
+Pass criteria:
+
+* ls -li shows 2 links and the inode matches the SHA file inode in annex/objects/
+* gitx1 dir size is equal to one bigfile
+
+
+Fail criteria:
+
+* ls -li shows 1 link and the inode numbers are different: working dir compared to annex/objects
+* gitx1 dir size is equal to 2 bigfiles.
+
+
+Scenario 1:
+
+* VM arch linux
+* git-annex: 6.20160613-g1e4e6f4
+* Host: Centos, qemu
+* gnome-disk-utility: QEMU HARDDISK (1.5.3)
+* partition: MBR
+* file system: ext4
+* Result: Pass
+
+
+Scenario 2:
+
+* VM arch linux
+* git-annex: 6.20160613-g1e4e6f4
+* Host: Win10, hyperv
+* gnome-disk-utility: Msft Virtual Disk (1.0)
+* partition: MBR
+* file system: ext4
+* Result: Fail
+
+
+Scenario 3:
+
+* VM Debian 8.5
+* git-annex: 6.20160524+gitg2b7b2c4-1~ndall+1
+* Host: Win10, hyperv
+* gnome-disk-utility: /dev/...vg/root
+* partition: lvm
+* file system: ext4
+* Result: Fail
+
+
+
+
+
+"""]]
diff --git a/doc/bugs/v6_appears_to_not_thin/comment_3_69afd9bc3f250418ba33cb1abeb2078d._comment b/doc/bugs/v6_appears_to_not_thin/comment_3_69afd9bc3f250418ba33cb1abeb2078d._comment
new file mode 100644
index 0000000000..2c95022290
--- /dev/null
+++ b/doc/bugs/v6_appears_to_not_thin/comment_3_69afd9bc3f250418ba33cb1abeb2078d._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="Stan"
+ subject="Hardlinks on hyperv VM"
+ date="2016-07-08T00:27:11Z"
+ content="""
+I am actually able to manually create hardlinks on all of the VMs in the tests.
+I did so with files I created outside any git dirs.
+"""]]
diff --git a/doc/bugs/v6_appears_to_not_thin/comment_4_bba7ba9c5b93fa33fc728daa0bb89497._comment b/doc/bugs/v6_appears_to_not_thin/comment_4_bba7ba9c5b93fa33fc728daa0bb89497._comment
new file mode 100644
index 0000000000..c3d4833c62
--- /dev/null
+++ b/doc/bugs/v6_appears_to_not_thin/comment_4_bba7ba9c5b93fa33fc728daa0bb89497._comment
@@ -0,0 +1,31 @@
+[[!comment format=mdwn
+ username="joey"
+ subject="""comment 4"""
+ date="2016-07-11T16:23:57Z"
+ content="""
+Well, certinaly seems that hyperv is doing something to the environment
+that causes the problem. 
+
+I don't know anything about hyperv, so cannot venture an educated guess.
+But there are only three reasons that git-annex might not make hard links,
+assuming it's seeing the annex.thin setting:
+
+1. It thinks that the file already has a link count of more than 1.
+2. It tries to create a hard link, but that fails for some reason,
+   and so it falls back to a copy.
+3. It successfully creates a hard link, but code that tries to preserve
+   the mode of the file fails for some reason, and so it falls back to a
+   copy.
+
+Getting a `strace -v` might help narrow it down. Here for reference is a
+strace of the hard link successfully being made:
+
+	32624 stat(".git/annex/objects/P7/mF/SHA256E-s30--c5e0f3090f3658e908aa0a02bfc7eeb3e203c67c8ecd86c7acd501a32454ed6a/SHA256E-s30--c5e0f3090f3658e908aa0a02bfc7eeb3e203c67c8ecd86c7acd501a32454ed6a", {st_dev=makedev(8, 3), st_ino=3675012, st_mode=S_IFREG|0444, st_nlink=1, st_uid=1000, st_gid=1000, st_blksize=4096, st_blocks=8, st_size=30, st_atime=2016/07/11-12:30:25.203217806, st_mtime=2016/07/11-12:27:58.523091486, st_ctime=2016/07/11-12:30:25.207217809}) = 0
+	32624 link(".git/annex/objects/P7/mF/SHA256E-s30--c5e0f3090f3658e908aa0a02bfc7eeb3e203c67c8ecd86c7acd501a32454ed6a/SHA256E-s30--c5e0f3090f3658e908aa0a02bfc7eeb3e203c67c8ecd86c7acd501a32454ed6a", ".git/annex/misctmp/foo.0/foo") = 0
+	32624 stat(".git/annex/misctmp/foo.0/foo", {st_dev=makedev(8, 3), st_ino=3675012, st_mode=S_IFREG|0444, st_nlink=2, st_uid=1000, st_gid=1000, st_blksize=4096, st_blocks=8, st_size=30, st_atime=2016/07/11-12:30:25.203217806, st_mtime=2016/07/11-12:27:58.523091486, st_ctime=2016/07/11-12:30:49.819239094}) = 0
+	32624 stat(".git/annex/misctmp/foo.0/foo", {st_dev=makedev(8, 3), st_ino=3675012, st_mode=S_IFREG|0444, st_nlink=2, st_uid=1000, st_gid=1000, st_blksize=4096, st_blocks=8, st_size=30, st_atime=2016/07/11-12:30:25.203217806, st_mtime=2016/07/11-12:27:58.523091486, st_ctime=2016/07/11-12:30:49.819239094}) = 0
+	32624 chmod(".git/annex/misctmp/foo.0/foo", 0100644) = 0
+	32624 stat(".git/annex/objects/P7/mF/SHA256E-s30--c5e0f3090f3658e908aa0a02bfc7eeb3e203c67c8ecd86c7acd501a32454ed6a/SHA256E-s30--c5e0f3090f3658e908aa0a02bfc7eeb3e203c67c8ecd86c7acd501a32454ed6a", {st_dev=makedev(8, 3), st_ino=3675012, st_mode=S_IFREG|0644, st_nlink=2, st_uid=1000, st_gid=1000, st_blksize=4096, st_blocks=8, st_size=30, st_atime=2016/07/11-12:30:25.203217806, st_mtime=2016/07/11-12:27:58.523091486, st_ctime=2016/07/11-12:30:49.819239094}) = 0
+	32624 stat(".git/annex/misctmp/foo.0/foo", {st_dev=makedev(8, 3), st_ino=3675012, st_mode=S_IFREG|0644, st_nlink=2, st_uid=1000, st_gid=1000, st_blksize=4096, st_blocks=8, st_size=30, st_atime=2016/07/11-12:30:25.203217806, st_mtime=2016/07/11-12:27:58.523091486, st_ctime=2016/07/11-12:30:49.819239094}) = 0
+	32624 rename(".git/annex/misctmp/foo.0/foo", "foo") = 0
+"""]]
diff --git a/doc/bugs/vicfg_and_description_often_not_propagated.mdwn b/doc/bugs/vicfg_and_description_often_not_propagated.mdwn
new file mode 100644
index 0000000000..7e6dd9269e
--- /dev/null
+++ b/doc/bugs/vicfg_and_description_often_not_propagated.mdwn
@@ -0,0 +1,154 @@
+### Please describe the problem.
+
+I can change the settings in one repo and sync it everywhere. Just to be surprised that one repo starts syncing to the transfer, every time it turns out that this repo lost its vicfg settings. Especially the Repository preferred contents are all back on standard. It was even once that it had the current settings and after the change and sync it goes back to some older state instead of the new one.
+
+### What steps will reproduce the problem?
+
+Well that is very hard. I have 8 repos and it happens randomly to some of them. I recreated all of them recently because I thought they are corrupt, that didn't help, just took me one week of time. It is also very hard to find a way to reproduce this because every vicfg causes a merge which takes minutes to hours.
+
+### What version of git-annex are you using? On what operating system?
+
+Linux: git-annex version: 5.20140412ubuntu1
+
+Mac OS: git-annex version: 5.20140717
+
+### Please provide any additional information below.
+
+Layout:
+
+transfer on rsync.net, conntented to that:
+
+ - Two OS X Clients
+
+ - Two Linux Archives
+
+My settings:
+
+
+
+[[!format sh """
+# If you can, paste a complete transcript of the problem occurring here.
+# If the problem is with the git-annex assistant, paste in .git/annex/daemon.log
+
+# git-annex configuration
+# 
+# Changes saved to this file will be recorded in the git-annex branch.
+# 
+# Lines in this file have the format:
+#   setting field = value
+
+# Repository trust configuration
+# (Valid trust levels: trusted semitrusted untrusted dead)
+# (for Music bei Pirmin)
+trust 0734498b-817c-419f-a0c0-660854dc7cbe = trusted
+# (for Music bei Jean (Willikins) [willikins])
+trust 20e9d2e5-9563-4507-82d5-bf8e23be29a5 = trusted
+# (for Music bei Jean (Willikins Clone))
+trust 6e3431e9-8ec2-404a-9c35-b967db63147d = trusted
+# (for Music bei Jean (Watson))
+trust a6febfa0-9fe5-4a65-95bb-dc255d87c2e2 = trusted
+# (for )
+trust dafe9a64-2480-40e2-9688-9f783577ef72 = dead
+# (for web)
+#trust 00000000-0000-0000-0000-000000000001 = semitrusted
+# (for music transfer via rsync.net [music_rsync])
+#trust 83c42610-42ad-459d-92a4-1aca2dfb97e1 = semitrusted
+
+# Repository groups
+# (Standard groups: client transfer backup incrementalbackup smallarchive archive source manual public unwanted)
+# (Separate group names with spaces)
+# (for Music bei Jean (Willikins) [willikins])
+group 20e9d2e5-9563-4507-82d5-bf8e23be29a5 = archive
+# (for Music bei Jean (Willikins Clone))
+group 6e3431e9-8ec2-404a-9c35-b967db63147d = archive
+# (for )
+group 26d38f31-cb6c-412c-84ef-597d7959a680 = backup
+# (for Music bei Pirmin)
+group 0734498b-817c-419f-a0c0-660854dc7cbe = client
+# (for Music bei Jean (Watson))
+group a6febfa0-9fe5-4a65-95bb-dc255d87c2e2 = client
+# (for music transfer via rsync.net [music_rsync])
+group 83c42610-42ad-459d-92a4-1aca2dfb97e1 = transfer
+# (for )
+group dafe9a64-2480-40e2-9688-9f783577ef72 = unwanted
+# (for web)
+#group 00000000-0000-0000-0000-000000000001 = 
+
+# Repository preferred contents
+# (Set to "standard" to use a repository's group's preferred contents)
+# (for Music bei Jean (Willikins) [willikins])
+wanted 20e9d2e5-9563-4507-82d5-bf8e23be29a5 = (not (copies=archive:2 or copies=smallarchive:2)) or approxlackingcopies=2
+# (for Music bei Jean (Willikins Clone))
+wanted 6e3431e9-8ec2-404a-9c35-b967db63147d = (not (copies=archive:2 or copies=smallarchive:2)) or approxlackingcopies=2
+# (for music transfer via rsync.net [music_rsync])
+wanted 83c42610-42ad-459d-92a4-1aca2dfb97e1 = not (inallgroup=client and copies=archive:2 and copies=client:2) and ((((exclude=*/archive/* and exclude=archive/*) or (not (copies=archive:1 or copies=smallarchive:1))) and not unused) or approxlackingcopies=1)
+# (for Music bei Pirmin)
+wanted 0734498b-817c-419f-a0c0-660854dc7cbe = standard
+# (for )
+wanted 26d38f31-cb6c-412c-84ef-597d7959a680 = standard
+# (for )
+wanted dafe9a64-2480-40e2-9688-9f783577ef72 = standard
+# (for web)
+#wanted 00000000-0000-0000-0000-000000000001 = 
+# (for Music bei Jean (Watson))
+wanted a6febfa0-9fe5-4a65-95bb-dc255d87c2e2 = standard
+
+# Group preferred contents
+# (Used by repositories with "groupwanted" in their preferred contents)
+#groupwanted archive = 
+#groupwanted backup = 
+#groupwanted client = 
+#groupwanted incrementalbackup = 
+#groupwanted manual = 
+#groupwanted public = 
+#groupwanted smallarchive = 
+#groupwanted source = 
+#groupwanted transfer = 
+#groupwanted unwanted = 
+
+# Standard preferred contents
+# (Used by wanted or groupwanted expressions containing "standard")
+# (For reference only; built-in and cannot be changed!)
+# standard client = (((exclude=*/archive/* and exclude=archive/*) or (not (copies=archive:1 or copies=smallarchive:1))) and not unused) or approxlackingcopies=1
+# standard transfer = (not (inallgroup=client and copies=client:2) and ((((exclude=*/archive/* and exclude=archive/*) or (not (copies=archive:1 or copies=smallarchive:1))) and not unused) or approxlackingcopies=1)) or approxlackingcopies=1
+# standard backup = include=* or unused
+# standard incrementalbackup = ((include=* or unused) and (not copies=incrementalbackup:1)) or approxlackingcopies=1
+# standard smallarchive = ((include=*/archive/* or include=archive/*) and ((not (copies=archive:1 or copies=smallarchive:1)) or approxlackingcopies=1)) or approxlackingcopies=1
+# standard archive = (not (copies=archive:1 or copies=smallarchive:1)) or approxlackingcopies=1
+# standard source = not (copies=1)
+# standard manual = present and ((((exclude=*/archive/* and exclude=archive/*) or (not (copies=archive:1 or copies=smallarchive:1))) and not unused) or approxlackingcopies=1)
+# standard public = inpreferreddir
+# standard unwanted = exclude=*
+
+# Repository required contents
+# (for web)
+#required 00000000-0000-0000-0000-000000000001 = 
+# (for Music bei Pirmin)
+#required 0734498b-817c-419f-a0c0-660854dc7cbe = 
+# (for Music bei Jean (Willikins) [willikins])
+#required 20e9d2e5-9563-4507-82d5-bf8e23be29a5 = 
+# (for Music bei Jean (Willikins Clone))
+#required 6e3431e9-8ec2-404a-9c35-b967db63147d = 
+# (for music transfer via rsync.net [music_rsync])
+#required 83c42610-42ad-459d-92a4-1aca2dfb97e1 = 
+# (for Music bei Jean (Watson))
+#required a6febfa0-9fe5-4a65-95bb-dc255d87c2e2 = 
+
+# Scheduled activities
+# (Separate multiple activities with "; ")
+# (for web)
+#schedule 00000000-0000-0000-0000-000000000001 = 
+# (for Music bei Pirmin)
+#schedule 0734498b-817c-419f-a0c0-660854dc7cbe = 
+# (for Music bei Jean (Willikins) [willikins])
+#schedule 20e9d2e5-9563-4507-82d5-bf8e23be29a5 = 
+# (for Music bei Jean (Willikins Clone))
+#schedule 6e3431e9-8ec2-404a-9c35-b967db63147d = 
+# (for music transfer via rsync.net [music_rsync])
+#schedule 83c42610-42ad-459d-92a4-1aca2dfb97e1 = 
+# (for Music bei Jean (Watson))
+#schedule a6febfa0-9fe5-4a65-95bb-dc255d87c2e2 =
+# End of transcript or log.
+"""]]
+
+[[!tag moreinfo]]
diff --git a/doc/bugs/vicfg_and_description_often_not_propagated/comment_1_60c5ba2de0c8b9b4538a4abd146007f6._comment b/doc/bugs/vicfg_and_description_often_not_propagated/comment_1_60c5ba2de0c8b9b4538a4abd146007f6._comment
new file mode 100644
index 0000000000..47c2de6f14
--- /dev/null
+++ b/doc/bugs/vicfg_and_description_often_not_propagated/comment_1_60c5ba2de0c8b9b4538a4abd146007f6._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="http://joeyh.name/"
+ ip="209.250.56.132"
+ subject="comment 1"
+ date="2014-09-11T18:11:43Z"
+ content="""
+vicfg makes changes to files in the git-annex branch, so you can use regular git commands to examine those changes, and look through the history to find any changes that are contrary to the settings you want, and see when and where they were committed. That should provide a strong pointer to what is causing this to happen.
+"""]]
diff --git a/doc/bugs/vicfg_and_description_often_not_propagated/comment_2_d56aed617e0791aa17d9f37c8d3fd317._comment b/doc/bugs/vicfg_and_description_often_not_propagated/comment_2_d56aed617e0791aa17d9f37c8d3fd317._comment
new file mode 100644
index 0000000000..cb485362bf
--- /dev/null
+++ b/doc/bugs/vicfg_and_description_often_not_propagated/comment_2_d56aed617e0791aa17d9f37c8d3fd317._comment
@@ -0,0 +1,12 @@
+[[!comment format=mdwn
+ username="http://joeyh.name/"
+ ip="209.250.56.96"
+ subject="comment 2"
+ date="2014-10-21T20:02:40Z"
+ content="""
+Can you please provide more information, like showing the commits made to the git-annex branch when the configuration was reverted?
+
+Also, might some of the clocks of computers where you're using git-annex be set wrong?
+
+I have tagged this report moreinfo because I don't have enough information to do anything else.
+"""]]
diff --git a/doc/bugs/webapp__58___difficult_to_abort_adding_a_repository.mdwn b/doc/bugs/webapp__58___difficult_to_abort_adding_a_repository.mdwn
new file mode 100644
index 0000000000..41007dbb45
--- /dev/null
+++ b/doc/bugs/webapp__58___difficult_to_abort_adding_a_repository.mdwn
@@ -0,0 +1,24 @@
+### Please describe the problem.
+I could not find a way to abort the addition of a new remote repository.
+
+### What steps will reproduce the problem?
+- start adding a remote repository (unencrypted, with git-annex installed);
+- forget to create the folder on the remote host;
+- navigate away from the repository page;
+- the dashboard says the repository is partially set-up, and the only thing one can do is look at the log (which says the folder is missing).
+
+I was able to solve it by creating another repository with the exact same data.
+
+### What version of git-annex are you using? On what operating system?
+
+Version: 4.20131002-gf25991c on OS X 10.8.5
+
+### Please provide any additional information below.
+
+[[!format sh """
+# If you can, paste a complete transcript of the problem occurring here.
+# If the problem is with the git-annex assistant, paste in .git/annex/daemon.log
+
+
+# End of transcript or log.
+"""]]
diff --git a/doc/bugs/webdav_fails___40__server__58___radicale_0.9-1__41__.mdwn b/doc/bugs/webdav_fails___40__server__58___radicale_0.9-1__41__.mdwn
new file mode 100644
index 0000000000..8da66a810a
--- /dev/null
+++ b/doc/bugs/webdav_fails___40__server__58___radicale_0.9-1__41__.mdwn
@@ -0,0 +1,45 @@
+The current command fails (git-annex 5.20141125, debian sid, DAV server: radicale 0.9-1):
+
+    $ WEBDAV_USERNAME=user WEBDAV_PASSWORD=password git annex --debug initremote dav.example.com type=webdav url=https://dav.example.com encryption=none
+    [2014-11-28 16:53:29 CET] read: git ["--git-dir=/home/user/test-annex/.git","--work-tree=/home/user/test-annex","show-ref","git-annex"]
+    [2014-11-28 16:53:29 CET] read: git ["--git-dir=/home/user/test-annex/.git","--work-tree=/home/user/test-annex","show-ref","--hash","refs/heads/git-annex"]
+    [2014-11-28 16:53:29 CET] read: git ["--git-dir=/home/user/test-annex/.git","--work-tree=/home/user/test-annex","log","refs/heads/git-annex..f60365a0d5333e1aefe80a3fe747e2cb3d5c8162","-n1","--pretty=%H"]
+    [2014-11-28 16:53:29 CET] chat: git ["--git-dir=/home/user/test-annex/.git","--work-tree=/home/user/test-annex","cat-file","--batch"]
+    initremote dav.example.com (testing WebDAV server...)
+
+    git-annex: WebDAV test failed: StatusCodeException (Status {statusCode = 500, statusMessage = "Internal Server Error"}) [("Server","nginx/1.2.1"),("Date","Fri, 28 Nov 2014 15:53:37 GMT"),("Content-Type","text/plain"),("Content-Length","59"),("Connection","keep-alive"),("Keep-Alive","timeout=75"),("X-Response-Body-Start","A server error occurred. Please contact the administrator."),("X-Request-URL","MKCOL https://dav.example.com:443/tmp")] (CJ {expose = []}): user error
+    failed
+    git-annex: initremote: 1 failed
+
+server log:
+
+    ::ffff:192.168.1.6 - user [28/Nov/2014:16:53:17 +0100] "MKCOL / HTTP/1.1" 500 59 "-" "hDav-using application"
+    ::ffff:192.168.1.6 - user [28/Nov/2014:16:53:18 +0100] "PROPFIND /tmp HTTP/1.1" 500 59 "-" "hDav-using application"
+    ::ffff:192.168.1.6 - user [28/Nov/2014:16:53:21 +0100] "MKCOL /tmp HTTP/1.1" 500 59 "-" "hDav-using application"
+    ::ffff:192.168.1.6 - user [28/Nov/2014:16:53:32 +0100] "MKCOL / HTTP/1.1" 500 59 "-" "hDav-using application"
+    ::ffff:192.168.1.6 - user [28/Nov/2014:16:53:34 +0100] "PROPFIND /tmp HTTP/1.1" 500 59 "-" "hDav-using application"
+    ::ffff:192.168.1.6 - user [28/Nov/2014:16:53:37 +0100] "MKCOL /tmp HTTP/1.1" 500 59 "-" "hDav-using application"
+    ::ffff:192.168.1.6 - user [28/Nov/2014:16:58:31 +0100] "PROPFIND / HTTP/1.1" 207 1035 "-" "cadaver/0.23.3 neon/0.30.1"
+    ::ffff:192.168.1.6 - user [28/Nov/2014:16:59:19 +0100] "MKCOL / HTTP/1.1" 500 59 "-" "hDav-using application"
+    ::ffff:192.168.1.6 - user [28/Nov/2014:16:59:21 +0100] "PROPFIND /tmp HTTP/1.1" 500 59 "-" "hDav-using application"
+    ::ffff:192.168.1.6 - user [28/Nov/2014:16:59:23 +0100] "MKCOL /tmp HTTP/1.1" 500 59 "-" "hDav-using application"
+
+
+cadaver works:
+
+    $ cadaver https://dav.example.com
+    Authentication required for Radicale - Password Required on server `dav.example.com':
+    Username: user
+    Password: 
+    dav:/> ls
+    Listing collection `/': succeeded.
+    Coll:   user                                  0  Jan  1  1970
+
+server log:
+
+    ::ffff:192.168.1.6 - - [28/Nov/2014:17:01:31 +0100] "OPTIONS / HTTP/1.1" 200 0 "-" "cadaver/0.23.3 neon/0.30.1"
+    ::ffff:192.168.1.6 - - [28/Nov/2014:17:01:31 +0100] "PROPFIND / HTTP/1.1" 401 0 "-" "cadaver/0.23.3 neon/0.30.1"
+    ::ffff:192.168.1.6 - user [28/Nov/2014:17:01:42 +0100] "PROPFIND / HTTP/1.1" 207 579 "-" "cadaver/0.23.3 neon/0.30.1"
+    ::ffff:192.168.1.6 - user [28/Nov/2014:17:01:42 +0100] "PROPFIND / HTTP/1.1" 207 579 "-" "cadaver/0.23.3 neon/0.30.1"
+    ::ffff:192.168.1.6 - user [28/Nov/2014:17:02:01 +0100] "PROPFIND / HTTP/1.1" 207 1035 "-" "cadaver/0.23.3 neon/0.30.1"
+
diff --git a/doc/bugs/webdav_fails___40__server__58___radicale_0.9-1__41__/comment_1_487bf264afeef2a3b0a61d306ebfc3c5._comment b/doc/bugs/webdav_fails___40__server__58___radicale_0.9-1__41__/comment_1_487bf264afeef2a3b0a61d306ebfc3c5._comment
new file mode 100644
index 0000000000..56e34a09d3
--- /dev/null
+++ b/doc/bugs/webdav_fails___40__server__58___radicale_0.9-1__41__/comment_1_487bf264afeef2a3b0a61d306ebfc3c5._comment
@@ -0,0 +1,12 @@
+[[!comment format=mdwn
+ username="joey"
+ subject="""comment 1"""
+ date="2014-12-02T19:02:10Z"
+ content="""
+The webdav server seems to have rejected the MKCOL command,
+which makes a collection (subdirectory) on the server.
+
+Is radicale intended to be a general-purpose webdav server?
+The description of the debian package says it's a CalDAV and CardDAV
+server. Perhaps it simply doesn't support generic top-level collections.
+"""]]
diff --git a/doc/bugs/weird_bug_with_annex_unlock_with_annex.thin_true_about_hard_links.mdwn b/doc/bugs/weird_bug_with_annex_unlock_with_annex.thin_true_about_hard_links.mdwn
new file mode 100644
index 0000000000..c5fe398bfc
--- /dev/null
+++ b/doc/bugs/weird_bug_with_annex_unlock_with_annex.thin_true_about_hard_links.mdwn
@@ -0,0 +1,76 @@
+### Please describe the problem.
+
+when unlocking multiple files, files are not hard linked, but they are if unlocked individually
+
+
+### What steps will reproduce the problem?
+
+I don't really know. 
+Currently I have a repository that exhibits the problem:
+
+    %git status --> all clean
+    
+    # this works as expected
+    %git annex unlock bucket/tata
+    %ls -li bucket/tata
+    33292697 -rw-rw-r-- 2 karl qbstaff 102400000 Dec  7 19:02 bucket/tata
+    %find . -inum 33292697
+    ./bucket/tata
+    ./.git/annex/objects/V0/3Z/SHA256E-s102400000--c50bbc52a81112507134d764ec570ab373be5c4a3b1dd1d87ce609d14d031a17/SHA256E-s102400000--c50bbc52a81112507134d764ec570ab373be5c4a3b1dd1d87ce609d14d031a17
+    git annex lock bucket/tata
+
+
+
+This does not:
+
+    %git annex unlock bucket/
+    %ls -li bucket/tata 
+    33292708 -rw-rw-r-- 1 karl qbstaff 102400000 Dec  7 19:02 bucket/tata
+    %find . -inum 33292708
+    ./bucket/tata
+
+As you can see, now the inode is wrong (different from the annex object file), and the unlocked file is not a hard link. The only difference is the argument to git annex unlock
+This is really annoying, I switched to V6 for this particular feature, I have a large repository to update and it takes ages since files are copied instead of hard linked.
+
+
+### What version of git-annex are you using? On what operating system?
+
+    %git config annex.thin 
+    true
+
+    %git annex version
+    git-annex version: 6.20171018+gitgbb20b1ed3-1~ndall+1
+    build flags: Assistant Webapp Pairing Testsuite S3(multipartupload)(storageclasses) WebDAV Inotify DBus DesktopNotify ConcurrentOutput TorrentParser MagicMime Feeds Quvi
+    dependency versions: aws-0.14.1 bloomfilter-2.0.1.0 cryptonite-0.20 DAV-1.3.1 feed-0.3.11.1 ghc-8.0.1 http-client-0.4.31.1 persistent-sqlite-2.6 torrent-10000.0.0 uuid-1.3.12 yesod-1.4.3
+    key/value backends: SHA256E SHA256 SHA512E SHA512 SHA224E SHA224 SHA384E SHA384 SHA3_256E SHA3_256 SHA3_512E SHA3_512 SHA3_224E SHA3_224 SHA3_384E SHA3_384 SKEIN256E SKEIN256 SKEIN512E SKEIN512 SHA1E SHA1 MD5E MD5 WORM URL
+    remote types: git gcrypt p2p S3 bup directory rsync web bittorrent webdav tahoe glacier ddar hook external
+    local repository version: 6
+    supported repository versions: 3 5 6
+    upgrade supported from repository versions: 0 1 2 3 4 5
+    operating system: linux x86_64
+
+    %lsb_release -a
+    No LSB modules are available.
+    Distributor ID:	Ubuntu
+    Description:	Ubuntu 14.04.5 LTS
+    Release:	14.04
+    Codename:	trusty
+
+
+
+### Please provide any additional information below.
+
+[[!format sh """
+# If you can, paste a complete transcript of the problem occurring here.
+# If the problem is with the git-annex assistant, paste in .git/annex/daemon.log
+
+
+# End of transcript or log.
+"""]]
+
+### Have you had any luck using git-annex before? (Sometimes we get tired of reading bug reports all day and a lil' positive end note does wonders)
+
+Yes, we use it on a daily basis and love it.
+Note that we will have to switch to LFS for a lot of use cases because gitlab no longer supports git annex from v9 release.
+
+
diff --git a/doc/bugs/weird_bug_with_annex_unlock_with_annex.thin_true_about_hard_links/comment_1_cb6c6e7bcbbe3006dd80e89fa545016d._comment b/doc/bugs/weird_bug_with_annex_unlock_with_annex.thin_true_about_hard_links/comment_1_cb6c6e7bcbbe3006dd80e89fa545016d._comment
new file mode 100644
index 0000000000..c60dbbfa9d
--- /dev/null
+++ b/doc/bugs/weird_bug_with_annex_unlock_with_annex.thin_true_about_hard_links/comment_1_cb6c6e7bcbbe3006dd80e89fa545016d._comment
@@ -0,0 +1,25 @@
+[[!comment format=mdwn
+ username="https://me.yahoo.com/a/x4ddlZwKw9O9IsCHmNZz7tSJn2h0H3vX#23f03"
+ nickname="Karl"
+ avatar="http://cdn.libravatar.org/avatar/7bc6190890ad7485f9913c49aa13165cecf2d98a54bc7934749771a1b91290f6"
+ subject="additional information"
+ date="2017-12-08T10:17:40Z"
+ content="""
+in the same git repository, some files will not be hardlinked even if unlocked individually, e.g.
+
+    ll -i bucket/dev/nkt-16-358-01/data/nkt-16-358-01_flow_annotation.csv
+    33292708 lrwxrwxrwx 1 karl qbstaff 208 Nov 29 14:19 bucket/dev/nkt-16-358-01/data/nkt-16-358-01_flow_annotation.csv -> ../../../../.git/annex/objects/74/mK/SHA256E-s798268--3ae6c93d5b03cae3b7fb3d9920a4728e3ba572419f8918a938ef50efccdcfc1b.csv/SHA256E-   s798268--3ae6c93d5b03cae3b7fb3d9920a4728e3ba572419f8918a938ef50efccdcfc1b.csv*
+    
+    ll -i .git/annex/objects/74/mK/SHA256E-s798268--3ae6c93d5b03cae3b7fb3d9920a4728e3ba572419f8918a938ef50efccdcfc1b.csv/SHA256E-s798268--3ae6c93d5b03cae3b7fb3d9920a4728e3ba572419f8918a938ef50efccdcfc1b.csv
+    33161645 -rwxrw---- 1 karl qbstaff 798268 Nov 29 14:19 .git/annex/objects/74/mK/SHA256E-s798268--3ae6c93d5b03cae3b7fb3d9920a4728e3ba572419f8918a938ef50efccdcfc1b.csv/SHA256E-s798268--3ae6c93d5b03cae3b7fb3d9920a4728e3ba572419f8918a938ef50efccdcfc1b.csv*
+    # so the inode of the object is 33161645
+    # unlock it
+    git annex unlock bucket/dev/nkt-16-358-01/data/nkt-16-358-01_flow_annotation.csv
+    # check the inum of the unlocked file
+    # ll -i bucket/dev/nkt-16-358-01/data/nkt-16-358-01_flow_annotation.csv
+    33292719 -rwxrwx--x 1 karl qbstaff 798268 Nov 29 14:19 bucket/dev/nkt-16-358-01/data/nkt-16-358-01_flow_annotation.csv
+    # inodes do not match
+
+
+
+"""]]
diff --git a/doc/bugs/weird_bug_with_annex_unlock_with_annex.thin_true_about_hard_links/comment_2_146ed4bf42ce1979815db82bbf9a5b86._comment b/doc/bugs/weird_bug_with_annex_unlock_with_annex.thin_true_about_hard_links/comment_2_146ed4bf42ce1979815db82bbf9a5b86._comment
new file mode 100644
index 0000000000..dd059a2b78
--- /dev/null
+++ b/doc/bugs/weird_bug_with_annex_unlock_with_annex.thin_true_about_hard_links/comment_2_146ed4bf42ce1979815db82bbf9a5b86._comment
@@ -0,0 +1,10 @@
+[[!comment format=mdwn
+ username="andrey_utkin@49e37627b3060c40292113d73d7ffbf317233e62"
+ nickname="andrey_utkin"
+ avatar="http://cdn.libravatar.org/avatar/95bb7f4f7647cc24c1cf635b61578842"
+ subject="comment 2"
+ date="2018-07-21T23:41:34Z"
+ content="""
+Hit this, too. dev-vcs/git-annex-6.20180529.
+All files which are successfully hardlinked have executability bit not set. `chmod a-x *` eventually fixes hardlinking for almost all files, but interestingly, not for all. Also as I see executability permissions bit does not propagate through git commits with modechange.
+"""]]
diff --git a/doc/bugs/weird_unicode_bug_on_windows.mdwn b/doc/bugs/weird_unicode_bug_on_windows.mdwn
new file mode 100644
index 0000000000..f1f0377f89
--- /dev/null
+++ b/doc/bugs/weird_unicode_bug_on_windows.mdwn
@@ -0,0 +1,17 @@
+### Please describe the problem.
+
+My repo contains more than 1000 files, many are unicode filenames. Now that [this bug](http://git-annex.branchable.com/bugs/fails_to_get_content_from_bare_repo_on_windows/) is resolved, almost all files are fetched, except one file named '移动硬盘 1T Buffalo USB3.0 白色.rtf'. 
+
+### What steps will reproduce the problem?
+
+I reproduced this problem by creating a repo containing only this file with no content. If anyone wants to reproduce this, just create an empty file by copy and paste the above filename in the quote.
+
+I created the repo on mac, synced with a usb drive, then on windows machine try to sync it back.
+
+* When run git-annex webapp in the newly created windows repo, after the usb drive is added as a remote, an error popups up: http://imgur.com/5ZfIeGQ  although the remote is added successfully (http://imgur.com/04O8kaC)
+* On the command line, git annex sync runs successfully, but git annex get . failed: http://imgur.com/bIVrbe2
+* The file is there (http://imgur.com/URGwWWt) with correct filename, just that the content is not there. 
+
+### What version of git-annex are you using? On what operating system?
+
+Latest release of git-annex on both mac and windows. The initial repo on mac is indirect repo, the one on usb drive is a bare repo, the one on windows is direct repo.
diff --git a/doc/bugs/weird_unicode_bug_on_windows/comment_1_69af9bd8c7898fccc2219edd860d547b._comment b/doc/bugs/weird_unicode_bug_on_windows/comment_1_69af9bd8c7898fccc2219edd860d547b._comment
new file mode 100644
index 0000000000..b16155c5f0
--- /dev/null
+++ b/doc/bugs/weird_unicode_bug_on_windows/comment_1_69af9bd8c7898fccc2219edd860d547b._comment
@@ -0,0 +1,12 @@
+[[!comment format=mdwn
+ username="http://joeyh.name/"
+ ip="209.250.56.203"
+ subject="comment 1"
+ date="2014-06-19T22:39:46Z"
+ content="""
+I don't quite reproduce this on windows. Instead of the file showing up and `git annex get` not getting it, the file never shows up at all. It seems that the direct mode merge code fails to create the file in the work tree. This also means that another sync will commit the deletion of the file.
+
+Based on the screenshot, I do have the identical filename checked into git, which git ls-files etc represents as \"\347\247\273\345\212\250\347\241\254\347\233\230 1T Buffalo USB3.0 \347\231\275\350\211\262.rtf\"
+
+Anyway, the root cause is probably the same. See [[todo/windows_support]] for a discussion of several issues with filename encodings and windows.
+"""]]
diff --git a/doc/bugs/when_you_get_a_file_but_don__39__t_actually_have_enough_space_for_it__44___the_error_message_makes_useless_suggestions.mdwn b/doc/bugs/when_you_get_a_file_but_don__39__t_actually_have_enough_space_for_it__44___the_error_message_makes_useless_suggestions.mdwn
new file mode 100644
index 0000000000..19e839263e
--- /dev/null
+++ b/doc/bugs/when_you_get_a_file_but_don__39__t_actually_have_enough_space_for_it__44___the_error_message_makes_useless_suggestions.mdwn
@@ -0,0 +1,21 @@
+The suggestion to make remotes available isn't really applicable, since the error was local.
+
+This is with git annex 6.20161110-gd48f4ca.
+
+[[!format sh """
+  ../git-annex.linux/git-annex get archiveteam-fire/metro.co.uk-urls-2007-04-12-20150627/metro.co.uk-urls-2007-04-12-20150627_meta.xml
+get archiveteam-fire/metro.co.uk-urls-2007-04-12-20150627/metro.co.uk-urls-2007-04-12-20150627_meta.xml 
+  not enough free space, need 98.82 GB more (use --force to override this check or adjust annex.diskreserve)
+
+  Unable to access these remotes: web
+
+  Try making some of these repositories available:
+        00000000-0000-0000-0000-000000000001 -- web
+        9f8218c0-763f-463d-9152-ecdc56d4452c -- iabak@redwyne.jwintjen.de:~/IA.BAK/shard12
+failed
+git-annex: get: 1 failed
+"""]]
+
+### Have you had any luck using git-annex before? (Sometimes we get tired of reading bug reports all day and a lil' positive end note does wonders)
+
+mixed success
diff --git a/doc/bugs/whereis_claims_file_is_not_here__44___but_it_is_available_both_here_and_in_another_remote.mdwn b/doc/bugs/whereis_claims_file_is_not_here__44___but_it_is_available_both_here_and_in_another_remote.mdwn
new file mode 100644
index 0000000000..98ee21d161
--- /dev/null
+++ b/doc/bugs/whereis_claims_file_is_not_here__44___but_it_is_available_both_here_and_in_another_remote.mdwn
@@ -0,0 +1,30 @@
+`git annex whereis` claims that I have zero copies of two particular files. However, it is wrong; both files are present:
+
+[[!format txt """
+[jkt@vorvan fotky]$ git annex whereis '2011-08-13 Svatba Anička Fellnerová a výlet s Julií/IMG_4301'*
+(merging synced/git-annex into git-annex...)
+whereis 2011-08-13 Svatba Anička Fellnerová a výlet s Julií/IMG_4301.CR2 (0 copies) failed
+whereis 2011-08-13 Svatba Anička Fellnerová a výlet s Julií/IMG_4301.JPG (0 copies) failed
+git-annex: whereis: 2 failed
+[jkt@vorvan fotky]$ ll 2011-08-13\ Svatba\ Anička\ Fellnerová\ a\ výlet\ s\ Julií/IMG_4301*
+lrwxrwxrwx. 1 jkt jkt 331 Aug 13  2011 2011-08-13 Svatba Anička Fellnerová a výlet s Julií/IMG_4301.CR2 -> ../.git/annex/objects/mg/qq/SHA512E-s19424303--9bdd6917fe85a450b50233a444c1814d407d41b550b66661272728aaf139576cc7c09de158f47a33ff263d7d25253cec193d0e44ed5fb4a1581c28a9a05e75e1.CR2/SHA512E-s19424303--9bdd6917fe85a450b50233a444c1814d407d41b550b66661272728aaf139576cc7c09de158f47a33ff263d7d25253cec193d0e44ed5fb4a1581c28a9a05e75e1.CR2
+lrwxrwxrwx. 1 jkt jkt 329 Aug 13  2011 2011-08-13 Svatba Anička Fellnerová a výlet s Julií/IMG_4301.JPG -> ../.git/annex/objects/zG/xq/SHA512E-s4744850--da9b1e19f0ecdf08d2f5564232a9c7d4b5ec9fbc9c67033121e5c90468e09e5573ddac42469cfd51fb0d8328dff925830e2d753022365df6d92b115b77831228.JPG/SHA512E-s4744850--da9b1e19f0ecdf08d2f5564232a9c7d4b5ec9fbc9c67033121e5c90468e09e5573ddac42469cfd51fb0d8328dff925830e2d753022365df6d92b115b77831228.JPG
+"""]]
+
+Looking at another repo, the situation is the same:
+
+[[!format txt """
+jkt@svist /mnt/storage5/fotky[master] $ git annex whereis '2011-08-13 Svatba Anička Fellnerová a výlet s Julií/IMG_4301'*
+whereis 2011-08-13 Svatba Anička Fellnerová a výlet s Julií/IMG_4301.CR2 (0 copies) failed
+whereis 2011-08-13 Svatba Anička Fellnerová a výlet s Julií/IMG_4301.JPG (0 copies) failed
+git-annex: whereis: 2 failed
+jkt@svist /mnt/storage5/fotky[master] $ ll '2011-08-13 Svatba Anička Fellnerová a výlet s Julií/IMG_4301'*
+lrwxrwxrwx 1 jkt jkt 331 Mar  3 02:08 2011-08-13 Svatba Anička Fellnerová a výlet s Julií/IMG_4301.CR2 -> ../.git/annex/objects/mg/qq/SHA512E-s19424303--9bdd6917fe85a450b50233a444c1814d407d41b550b66661272728aaf139576cc7c09de158f47a33ff263d7d25253cec193d0e44ed5fb4a1581c28a9a05e75e1.CR2/SHA512E-s19424303--9bdd6917fe85a450b50233a444c1814d407d41b550b66661272728aaf139576cc7c09de158f47a33ff263d7d25253cec193d0e44ed5fb4a1581c28a9a05e75e1.CR2
+lrwxrwxrwx 1 jkt jkt 329 Mar  3 02:08 2011-08-13 Svatba Anička Fellnerová a výlet s Julií/IMG_4301.JPG -> ../.git/annex/objects/zG/xq/SHA512E-s4744850--da9b1e19f0ecdf08d2f5564232a9c7d4b5ec9fbc9c67033121e5c90468e09e5573ddac42469cfd51fb0d8328dff925830e2d753022365df6d92b115b77831228.JPG/SHA512E-s4744850--da9b1e19f0ecdf08d2f5564232a9c7d4b5ec9fbc9c67033121e5c90468e09e5573ddac42469cfd51fb0d8328dff925830e2d753022365df6d92b115b77831228.JPG
+"""]]
+
+The directory names are valid UTF-8. These are very common on my machine and there is a ton of directories with these funny names here -- all working without any real trouble.
+
+As far as I know, the file which the links point to is absolutely correct and not corrupted. Looking at the files in the directory chronologically, it also appears that the symlinks point to a correct file.
+
+[[!tag moreinfo]]
diff --git a/doc/bugs/whereis_claims_file_is_not_here__44___but_it_is_available_both_here_and_in_another_remote/comment_1_d823b7ee32183fbadd4a49f65e1a3a8b._comment b/doc/bugs/whereis_claims_file_is_not_here__44___but_it_is_available_both_here_and_in_another_remote/comment_1_d823b7ee32183fbadd4a49f65e1a3a8b._comment
new file mode 100644
index 0000000000..11c55bb5b1
--- /dev/null
+++ b/doc/bugs/whereis_claims_file_is_not_here__44___but_it_is_available_both_here_and_in_another_remote/comment_1_d823b7ee32183fbadd4a49f65e1a3a8b._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="http://joeyh.name/"
+ ip="209.250.56.146"
+ subject="comment 1"
+ date="2014-03-05T03:14:09Z"
+ content="""
+Have you tried running `git annex fsck`?
+"""]]
diff --git a/doc/bugs/whereis_claims_file_is_not_here__44___but_it_is_available_both_here_and_in_another_remote/comment_2_f430538101f0ef6114b5e953248fa599._comment b/doc/bugs/whereis_claims_file_is_not_here__44___but_it_is_available_both_here_and_in_another_remote/comment_2_f430538101f0ef6114b5e953248fa599._comment
new file mode 100644
index 0000000000..e78971a90e
--- /dev/null
+++ b/doc/bugs/whereis_claims_file_is_not_here__44___but_it_is_available_both_here_and_in_another_remote/comment_2_f430538101f0ef6114b5e953248fa599._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="http://joeyh.name/"
+ ip="209.250.56.102"
+ subject="ping?"
+ date="2014-03-19T20:24:23Z"
+ content="""
+Have you tried running git-annex fsck?
+"""]]
diff --git a/doc/bugs/whereis_claims_file_is_not_here__44___but_it_is_available_both_here_and_in_another_remote/comment_3_d96ded3c27c705789e5689f19dbcabcb._comment b/doc/bugs/whereis_claims_file_is_not_here__44___but_it_is_available_both_here_and_in_another_remote/comment_3_d96ded3c27c705789e5689f19dbcabcb._comment
new file mode 100644
index 0000000000..3f38426f68
--- /dev/null
+++ b/doc/bugs/whereis_claims_file_is_not_here__44___but_it_is_available_both_here_and_in_another_remote/comment_3_d96ded3c27c705789e5689f19dbcabcb._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="jhannwong@c9c7a67b5632a4bbc0c959cfeb3d340e02f28565"
+ nickname="jhannwong"
+ subject="git annex fsck didn't solve this problem"
+ date="2015-11-20T01:49:03Z"
+ content="""
+Same problem here.
+"""]]
diff --git a/doc/bugs/whereis_claims_file_is_not_here__44___but_it_is_available_both_here_and_in_another_remote/comment_4_b70e83a44a85b006c2c6105660cd6a7a._comment b/doc/bugs/whereis_claims_file_is_not_here__44___but_it_is_available_both_here_and_in_another_remote/comment_4_b70e83a44a85b006c2c6105660cd6a7a._comment
new file mode 100644
index 0000000000..68e414e53d
--- /dev/null
+++ b/doc/bugs/whereis_claims_file_is_not_here__44___but_it_is_available_both_here_and_in_another_remote/comment_4_b70e83a44a85b006c2c6105660cd6a7a._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="joey"
+ subject="""comment 4"""
+ date="2015-11-30T20:08:10Z"
+ content="""
+@jhannwong you had [2 repos with the same annex.uuid](http://git-annex.branchable.com/bugs/remote_repo_marked_as___34__here__34__)
+so if you had the same problem, it was a misconfiguration on your part.
+"""]]
diff --git a/doc/bugs/whereis_outputs_no_informaiton_for_unlocked_files.mdwn b/doc/bugs/whereis_outputs_no_informaiton_for_unlocked_files.mdwn
new file mode 100644
index 0000000000..d4d4826ff5
--- /dev/null
+++ b/doc/bugs/whereis_outputs_no_informaiton_for_unlocked_files.mdwn
@@ -0,0 +1,46 @@
+What steps will reproduce the problem?
+
+    ...:/tmp$ mkdir repro
+    ...:/tmp$ cd repro/
+    ...:/tmp/repro$ git init
+    Initialized empty Git repository in /tmp/repro/.git/
+    ...:/tmp/repro$ git annex init test
+    init test ok
+    ...:/tmp/repro$ echo "A" > a.txt
+    ...:/tmp/repro$ git annex add a.txt
+    add a.txt (checksum...) ok
+    (Recording state in git...)
+    ...:/tmp/repro$ git commit -m "add file"
+    [master (root-commit) bf53ce2] add file
+     1 file changed, 1 insertion(+)
+     create mode 120000 a.txt
+    ...:/tmp/repro$ git annex whereis a.txt
+    whereis a.txt (1 copy) 
+      	5c028c6a-2c5e-11e2-bb9c-17bd7ce81377 -- here (test)
+    ok
+    ...:/tmp/repro$ git annex unlock a.txt
+    unlock a.txt (copying...) ok
+    ...:/tmp/repro$ git annex whereis a.txt
+
+What is the expected output? What do you see instead?
+
+    I'd expect that whereis executed on an unlocked file would behave like whereis executed on a locked file.
+
+What version of git-annex are you using? On what operating system?
+
+    $ cat /etc/issue
+    Ubuntu 12.04.1 LTS \n \l
+
+    $ git-annex version
+    git-annex version: 3.20120406
+    local repository version: 3
+    default repository version: 3
+    supported repository versions: 3
+    upgrade supported from repository versions: 0 1 2
+
+    $ uname -a
+    Linux ... 3.2.0-31-generic #50-Ubuntu SMP Fri Sep 7 16:16:45 UTC 2012 x86_64 x86_64 x86_64 GNU/Linux
+
+Please provide any additional information below.
+
+[[!tag confirmed]]
diff --git a/doc/bugs/whereis_outputs_no_informaiton_for_unlocked_files/comment_1_47bd0fc8d1c65f8a868d9722e66c71db._comment b/doc/bugs/whereis_outputs_no_informaiton_for_unlocked_files/comment_1_47bd0fc8d1c65f8a868d9722e66c71db._comment
new file mode 100644
index 0000000000..e92ec33f99
--- /dev/null
+++ b/doc/bugs/whereis_outputs_no_informaiton_for_unlocked_files/comment_1_47bd0fc8d1c65f8a868d9722e66c71db._comment
@@ -0,0 +1,10 @@
+[[!comment format=mdwn
+ username="http://joeyh.name/"
+ ip="209.250.56.102"
+ subject="comment 1"
+ date="2014-03-19T20:52:08Z"
+ content="""
+The reason this doesn't work is that, in indirect mode, git-annex looks at the current state of the symlink in the work tree to know what key is associated with a file. And an unlocked file has no symlink.
+
+Direct mode avoids this problem, but at the expense of being less flexible and well, doing more work.
+"""]]
diff --git a/doc/bugs/whereis_shouldn__39__t_just_crash_if_one_of_the_special_remotes___34__drivers__34___is_not_available.mdwn b/doc/bugs/whereis_shouldn__39__t_just_crash_if_one_of_the_special_remotes___34__drivers__34___is_not_available.mdwn
new file mode 100644
index 0000000000..0b4611e483
--- /dev/null
+++ b/doc/bugs/whereis_shouldn__39__t_just_crash_if_one_of_the_special_remotes___34__drivers__34___is_not_available.mdwn
@@ -0,0 +1,38 @@
+[[!format sh """
+(git-annex)hopa:~/datalad/neurovault/snapshots[master]
+$> git annex whereis 1003/13873.nii.gz
+whereis 1003/13873.nii.gz git-annex: Cannot run git-annex-remote-datalad-archives -- It is not installed in PATH (/usr/lib/git-annex.linux/bin:/usr/lib/git-core:/home/yoh/gocode/bin:/home/yoh/gocode/bin:/home/yoh/bin:/home/yoh/.local/bin:/usr/local/bin:/usr/bin:/bin:/usr/local/games:/usr/games:/snap/bin:/sbin:/usr/sbin:/usr/local/sbin:/usr/lib/git-annex.linux/extra)
+
+
+$> source ~datalad/datalad-master/venvs/dev/bin/activate
+
+$> git annex whereis 1003/13873.nii.gz                  
+whereis 1003/13873.nii.gz (3 copies) 
+  	123c73e5-a8dc-4cff-8ffc-679c7ea67f94 -- yoh@smaug:/mnt/datasets/datalad/crawl/neurovault
+   	48c1556f-6241-45de-9497-338d437fcb62 -- yoh@falkor:/srv/datasets.datalad.org/www/neurovault/snapshots [origin]
+   	af2785da-2538-4346-a6f6-f2f30fc3f025 -- [datalad-archives]
+
+  datalad-archives: dl+archive:SHA256E-s6460020224--710cc05117e2290e2f793271d11e26452cdc111121e09a937dbf5a34b3cc0107.tar/neurovault_snapshot/1003/13873.nii.gz#size=23262
+
+$> git annex version
+git-annex version: 6.20180807+git230-gaa291acfe-1~ndall+1
+"""]]
+
+IMHO output should be like (since all information is actually available)
+
+[[!format sh """
+$> git annex whereis 1003/13873.nii.gz
+git-annex: ERROR - Cannot run git-annex-remote-datalad-archives
+whereis 1003/13873.nii.gz (3 copies)
+  	123c73e5-a8dc-4cff-8ffc-679c7ea67f94 -- yoh@smaug:/mnt/datasets/datalad/crawl/neurovault
+   	48c1556f-6241-45de-9497-338d437fcb62 -- yoh@falkor:/srv/datasets.datalad.org/www/neurovault/snapshots [origin]
+   	af2785da-2538-4346-a6f6-f2f30fc3f025 -- datalad-archives {ERROR}
+
+  datalad-archives: dl+archive:SHA256E-s6460020224--710cc05117e2290e2f793271d11e26452cdc111121e09a937dbf5a34b3cc0107.tar/neurovault_snapshot/1003/13873.nii.gz#size=23262
+"""]]
+
+and indeed with error code exit may be.
+
+[[!meta author=yoh]]
+
+
diff --git a/doc/bugs/whereis_shouldn__39__t_just_crash_if_one_of_the_special_remotes___34__drivers__34___is_not_available/comment_1_3f8100a4d9e6ab4302c89eeafd0f843e._comment b/doc/bugs/whereis_shouldn__39__t_just_crash_if_one_of_the_special_remotes___34__drivers__34___is_not_available/comment_1_3f8100a4d9e6ab4302c89eeafd0f843e._comment
new file mode 100644
index 0000000000..40f5b4cba3
--- /dev/null
+++ b/doc/bugs/whereis_shouldn__39__t_just_crash_if_one_of_the_special_remotes___34__drivers__34___is_not_available/comment_1_3f8100a4d9e6ab4302c89eeafd0f843e._comment
@@ -0,0 +1,13 @@
+[[!comment format=mdwn
+ username="joey"
+ subject="""comment 1"""
+ date="2018-09-12T18:16:49Z"
+ content="""
+Hmm, but the special remote can add urls to the whereis output, indeed it
+is doing so in your example. So if the repository has the special remote
+enabled it seems it ought to need it to be in working order.
+
+External special remote programs are only spun up on demand, so the only
+way to get to this crash is when it's actually being used for something,
+AFAICS.
+"""]]
diff --git a/doc/bugs/windows_annex_link_wrong.mdwn b/doc/bugs/windows_annex_link_wrong.mdwn
new file mode 100644
index 0000000000..8a1b29280b
--- /dev/null
+++ b/doc/bugs/windows_annex_link_wrong.mdwn
@@ -0,0 +1,6 @@
+On windows, the links to .git/annex/objects are no longer relative; include
+drive letter and full path.
+
+This used to not be the case; it must have gotten broken. --[[Joey]]
+
+> [[fixed|done]]
diff --git a/doc/bugs/wrong_permissions_of_unused__44___badunused_and_tmpunused__63__.mdwn b/doc/bugs/wrong_permissions_of_unused__44___badunused_and_tmpunused__63__.mdwn
new file mode 100644
index 0000000000..e03ea396a8
--- /dev/null
+++ b/doc/bugs/wrong_permissions_of_unused__44___badunused_and_tmpunused__63__.mdwn
@@ -0,0 +1,73 @@
+The files `unused`, `badunused` and `tmpunused` in `.git/annex/` do not have the correct permissions when the git repository is set to `--shared=group`. Such files are `600`, while they should be `660`, as other files in `.git/annex`.
+
+For this reason, when those files are created, they are accessibile only to one user (the owner), triggering errors when other users in the group attempt things like `git annex unused` or `git annex dropunused`. At least this occurs with git-annex 6.20171018+gitgbb20b1ed3-1~ndall+1 .
+
+To reproduce the problem:
+
+    git init --shared=group
+    git annex init
+    echo test > test
+    git annex add test
+    git commit -m test
+    git rm test
+    git commit -m removed
+    git annex unused
+
+you get:
+
+    unused . (checking for unused data...) (checking master...) 
+      Some annexed data is no longer used by any files:
+        NUMBER  KEY
+        1       SHA256E-s5--f2ca1bb6c7e907d06dafe4687e579fce76b37e4e93b7605022da52e6ccc26fd2
+      (To see where data was previously used, try: git log --stat -S'KEY')
+      
+      To remove unwanted data: git-annex dropunused NUMBER
+      
+    ok
+
+
+Conversely, from another user of the group you get:
+
+    > git annex unused
+    unused . (checking for unused data...) (checking master...) 
+      Some annexed data is no longer used by any files:
+        NUMBER  KEY
+        1       SHA256E-s5--f2ca1bb6c7e907d06dafe4687e579fce76b37e4e93b7605022da52e6ccc26fd2
+      (To see where data was previously used, try: git log --stat -S'KEY')
+      
+      To remove unwanted data: git-annex dropunused NUMBER
+      
+    
+    git-annex: .git/annex/unused: openFile: permission denied (Permission denied)
+    failed
+    git-annex: unused: 1 failed
+
+Moreover:
+
+    > git annex dropunused 1
+    git-annex: .git/annex/misctmp/mergedrefs.0: createDirectory: permission denied (Permission denied)
+
+
+This is somewhat expected, because the permissions of `unused`, `badunused` and `tmpunused` are `600`:
+
+    > ll .git/annex/
+    total 40
+    drwxrwsr-x 5 ele testgroup 4096 dic 19 14:50 ./
+    drwxrwsr-x 9 ele testgroup 4096 dic 19 14:50 ../
+    -rw------- 1 ele testgroup    0 dic 19 14:50 badunused
+    -rw-rw-r-- 1 ele testgroup  345 dic 19 14:49 index
+    -rw-rw-r-- 1 ele testgroup   41 dic 19 14:49 index.lck
+    drwxrwsr-x 2 ele testgroup 4096 dic 19 14:49 journal/
+    -rw-rw---- 1 ele testgroup    0 dic 19 14:49 journal.lck
+    -rw-rw-r-- 1 ele testgroup   62 dic 19 14:49 mergedrefs
+    drwxrwsr-x 2 ele testgroup 4096 dic 19 14:49 misctmp/
+    drwxrwsr-x 3 ele testgroup 4096 dic 19 14:49 objects/
+    -rw-rw---- 1 ele testgroup    0 dic 19 14:49 precommit.lck
+    -rw-rw-r-- 1 ele testgroup    0 dic 19 14:49 sentinal
+    -rw-rw-r-- 1 ele testgroup   21 dic 19 14:49 sentinal.cache
+    -rw------- 1 ele testgroup    0 dic 19 14:50 tmpunused
+    -rw------- 1 ele testgroup  101 dic 19 14:50 unused
+
+If this is the intended behavior, could you please explain me how to use `git annex unused` and `dropunused` in a shared repository?
+
+> Fixed all of this I was able to reproduce. [[done]] --[[Joey]]
diff --git a/doc/bugs/wrong_permissions_of_unused__44___badunused_and_tmpunused__63__/comment_1_ca6b7e3d5d83d2dd6532533ecdb965c3._comment b/doc/bugs/wrong_permissions_of_unused__44___badunused_and_tmpunused__63__/comment_1_ca6b7e3d5d83d2dd6532533ecdb965c3._comment
new file mode 100644
index 0000000000..c573945075
--- /dev/null
+++ b/doc/bugs/wrong_permissions_of_unused__44___badunused_and_tmpunused__63__/comment_1_ca6b7e3d5d83d2dd6532533ecdb965c3._comment
@@ -0,0 +1,9 @@
+[[!comment format=mdwn
+ username="emanuele"
+ nickname="emanuele.olivetti"
+ avatar="http://cdn.libravatar.org/avatar/f51cc5c6c3a0eb28faa6491c3cbcfcce"
+ subject="comment 1"
+ date="2017-12-19T14:19:14Z"
+ content="""
+Notice that, if I manually change the permsissions of `unused`, `badunused` and `tmpunused` to `660`, then other another user of the group can do `git annex unused` and `git annex dropunused 1` without errors. Unfortunately, after `git annex unused`, the ownership of `unused`, `badunused` and `tmpunused` changes from the initial user to the new user, and permissions are reset to `600`, which re-creates the initial problem.
+"""]]
diff --git a/doc/bugs/wrong_permissions_of_unused__44___badunused_and_tmpunused__63__/comment_2_252800c3f4df5a5219e7f13c2b6ef841._comment b/doc/bugs/wrong_permissions_of_unused__44___badunused_and_tmpunused__63__/comment_2_252800c3f4df5a5219e7f13c2b6ef841._comment
new file mode 100644
index 0000000000..ef6ede63ed
--- /dev/null
+++ b/doc/bugs/wrong_permissions_of_unused__44___badunused_and_tmpunused__63__/comment_2_252800c3f4df5a5219e7f13c2b6ef841._comment
@@ -0,0 +1,9 @@
+[[!comment format=mdwn
+ username="joey"
+ subject="""comment 2"""
+ date="2018-01-02T20:17:03Z"
+ content="""
+Fixed, but I was not able to reproduce the part where dropunused failed
+with a permissions error involving .git/annex/misctmp, despite having it
+set up the same as you did.
+"""]]
diff --git a/doc/bugs/wrong_permissions_of_unused__44___badunused_and_tmpunused__63__/comment_3_608fc6f82962c70e55259923c0522f6f._comment b/doc/bugs/wrong_permissions_of_unused__44___badunused_and_tmpunused__63__/comment_3_608fc6f82962c70e55259923c0522f6f._comment
new file mode 100644
index 0000000000..edc531049c
--- /dev/null
+++ b/doc/bugs/wrong_permissions_of_unused__44___badunused_and_tmpunused__63__/comment_3_608fc6f82962c70e55259923c0522f6f._comment
@@ -0,0 +1,12 @@
+[[!comment format=mdwn
+ username="emanuele"
+ avatar="http://cdn.libravatar.org/avatar/f51cc5c6c3a0eb28faa6491c3cbcfcce"
+ subject="Thanks!"
+ date="2018-01-03T10:51:45Z"
+ content="""
+Joey, thanks for fixing the bug and for the larger commit (24df95f0f6ab474119aff3bbd942251373754ab2) to properly handle core.sharedRepository!
+
+I see you couldn't reproduce the permissions error involving .git/annex/misctmp. Just to clarify: it occurs to me only with the user for which `git annex unused` fails, and not to the initial user that did `git rm`, for which `git annex unused` works. I can still reproduce the issue in this very moment with git-annex 6.20171211+gitgba511f4de-1~ndall+1. Nevertheless, I'll try the updated code as soon as possible.
+
+And thanks to the anonymous bitcoin donor for supporting the fix of this bug!
+"""]]
diff --git a/doc/bugs/wrong_permissions_of_unused__44___badunused_and_tmpunused__63__/comment_4_bb6c06fb6d0fb7053c4e3ade80b5f9da._comment b/doc/bugs/wrong_permissions_of_unused__44___badunused_and_tmpunused__63__/comment_4_bb6c06fb6d0fb7053c4e3ade80b5f9da._comment
new file mode 100644
index 0000000000..c6f4e7338b
--- /dev/null
+++ b/doc/bugs/wrong_permissions_of_unused__44___badunused_and_tmpunused__63__/comment_4_bb6c06fb6d0fb7053c4e3ade80b5f9da._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="joey"
+ subject="""comment 4"""
+ date="2018-01-04T18:28:20Z"
+ content="""
+Yes, I was running dropunused as other user, and still see no permissions
+problem there. .git/annex/misctmp is writable by the group.
+"""]]
diff --git a/doc/bugs/wrong_permissions_of_unused__44___badunused_and_tmpunused__63__/comment_5_4fdbb304d67b198f32aa55b1672b0c78._comment b/doc/bugs/wrong_permissions_of_unused__44___badunused_and_tmpunused__63__/comment_5_4fdbb304d67b198f32aa55b1672b0c78._comment
new file mode 100644
index 0000000000..77d25eaa27
--- /dev/null
+++ b/doc/bugs/wrong_permissions_of_unused__44___badunused_and_tmpunused__63__/comment_5_4fdbb304d67b198f32aa55b1672b0c78._comment
@@ -0,0 +1,10 @@
+[[!comment format=mdwn
+ username="emanuele"
+ avatar="http://cdn.libravatar.org/avatar/f51cc5c6c3a0eb28faa6491c3cbcfcce"
+ subject="comment 5"
+ date="2018-01-09T08:46:44Z"
+ content="""
+I can confirm that the permissions I see are set as you say: .git/annex/misctmp is writable by the group. Still, the error message appears here, using git-annex 6.20171211. I will look more in detail into this one. If something interesting pops out, I'll write here.
+
+Thanks again for your work.
+"""]]
diff --git a/doc/bugs/yesod-default_appears_no_longer_necessary.mdwn b/doc/bugs/yesod-default_appears_no_longer_necessary.mdwn
new file mode 100644
index 0000000000..50c439ef82
--- /dev/null
+++ b/doc/bugs/yesod-default_appears_no_longer_necessary.mdwn
@@ -0,0 +1,23 @@
+### Please describe the problem.
+
+The description for yesod-default is:
+
+> Since version 1.2 of Yesod, this functionality is provided by the yesod package. This package is no longer needed.
+
+git-annex depends on yesod-default >=1.2.0 and yesod >=1.2.6 (and many other yesod-*>1.2).
+
+Thus git-annex should be able to drop yesod-default from the cabal file.
+
+### What steps will reproduce the problem?
+
+Just build from source.
+
+### What version of git-annex are you using? On what operating system?
+
+6.20171109, source tarball.
+
+### Have you had any luck using git-annex before? (Sometimes we get tired of reading bug reports all day and a lil' positive end note does wonders)
+
+Sure, this is not really a "problem" anyway.
+
+> Thanks for noticing this, I've made the changes. [[done]] --[[Joey]]
diff --git a/doc/builds.mdwn b/doc/builds.mdwn
new file mode 100644
index 0000000000..765e87b09f
--- /dev/null
+++ b/doc/builds.mdwn
@@ -0,0 +1,63 @@
+[[!meta title="git-annex autobuild overview"]]
+
+[[!sidebar content="""
+# last uploaded build-versions
+
+
    Linux i386
    
+
+
    Linux amd64
    
+
+
    Linux armel
    
+
+
    Linux i386-ancient
    
+
+
    Linux armel-ancient
    
+
+
    Android
    
+
+
    OSX Yosemite
    
+
+
    Windows
    
+
+"""]]
+
+# build logs
+
+
    Linux i386
    
+
+
    Linux amd64
    
+
+
    Linux armel
    
+
+
    Linux i386-ancient
    
+
+
    Linux armel-ancient
    
+
+
    Android
    
+
+
    OSX Yosemite
    
+
+
+
    Windows
    
+here
+
    Debian
    
+
+
diff --git a/doc/chunking.mdwn b/doc/chunking.mdwn
new file mode 100644
index 0000000000..43e604f232
--- /dev/null
+++ b/doc/chunking.mdwn
@@ -0,0 +1,43 @@
+Most [[special_remotes]] have support for breaking large files up into
+chunks that are stored on the remote.
+
+This can be useful to work around limitations on the size of files
+on the remote.
+
+Chunking also allows for resuming interrupted downloads and uploads.
+
+Note that git-annex has to buffer chunks in memory before they are sent to
+a remote. So, using a large chunk size will make it use more memory.
+
+To enable chunking, pass a `chunk=nnMiB` parameter to `git annex
+initremote`, specifying the chunk size. 
+
+Good chunk sizes will depend on the remote, but a good starting place
+is probably `1MiB`. Very large chunks are problematic, both because
+git-annex needs to buffer one chunk in memory when uploading, and because
+a larger chunk will make resuming interrupted transfers less efficient.
+On the other hand, when a file is split into a great many chunks,
+there can be increased overhead of making many requests to the remote.
+
+To disable chunking of a remote that was using chunking,
+pass `chunk=0` to `git annex enableremote`. Any content already stored on
+the remote using chunks will continue to be accessed via chunks, this
+just prevents using chunks when storing new content.
+
+To change the chunk size, pass a `chunk=nnMiB` parameter to
+`git annex enableremote`. This only affects the chunk sized used when
+storing new content.
+
+# old-style chunking
+
+Note that older versions of git-annex used a different chunk method, which
+was configured by passing `chunksize=nnMib` when initializing a remote.
+
+The old-style chunking had a number of problems, including being less
+efficient, and not allowing resumes of encrypted uploads.
+
+It's not possible to change a remote using that old chunking method to the
+new one, but git-annex continues to support the old-style chunking to
+support such remotes.
+
+See also: [[design document|design/assistant/chunks]]
diff --git a/doc/coding_style.mdwn b/doc/coding_style.mdwn
new file mode 100644
index 0000000000..425665e377
--- /dev/null
+++ b/doc/coding_style.mdwn
@@ -0,0 +1,113 @@
+If you do nothing else, avoid use of partial functions from the Prelude!
+`import Utility.PartialPrelude` helps avoid this by defining conflicting
+functions for all the common ones. Also avoid `!!`, it's partial too.
+
+Use tabs for indentation. The one exception to this rule are
+the Hamlet format files in `templates/*`. Hamlet, infuriatingly, refuses
+to allow tabs to be used for indentation.
+
+Code should make sense with any tab stop setting, but 8 space tabs are
+the default. With 8 space tabs, code should not exceed 80 characters
+per line. (With larger tabs, it may of course.)
+
+Use spaces for layout. For example, here spaces (indicated with `.`)
+are used after the initial tab to make the third test line up with
+the others.
+
+		when (foo_test || bar_test ||
+		......some_other_long_test)
+			print "hi"
+
+As a special Haskell-specific rule, "where" clauses are indented with two
+spaces, rather than a tab. This makes them stand out from the main body
+of the function, and avoids excessive indentation of the where cause content.
+The definitions within the where clause should be put on separate lines,
+each indented with a tab.
+
+	main = do
+		foo
+		bar
+		foo
+	  where
+		foo = ...
+		bar = ...
+
+Where clauses for instance definitions and modules tend to appear at the end
+of a line, rather than on a separate line.
+
+	module Foo (Foo, mkFoo, unFoo) where
+	instance MonadBaseControl IO Annex where
+
+When a function's type signature needs to be wrapped to another line,
+it's typical to switch to displaying one parameter per line.
+
+	foo :: Bar -> Baz -> (Bar -> Baz) -> IO Baz
+
+	foo'
+		:: Bar
+		-> Baz
+		-> (Bar -> Baz)
+		-> IO Baz
+ 
+Note that the "::" then starts its own line. It is not put on the same
+line as the function name because then it would not be guaranteed to line
+up with the "->" at all tab width settings. Similarly, guards are put
+on their own lines:
+
+	splat i
+		| odd i = error "splat!"
+		| otherwise = i
+
+Multiline lists and record syntax are written with leading commas,
+that line up with the open and close punctuation.
+
+	list =
+		[ item1
+		, item2
+		, item3
+		]
+
+	foo = DataStructure
+		{ name = "bar"
+		, address = "baz"
+		}
+
+Similarly, data structures line up the leading `=` with the following `|`
+
+	data Foo
+		= Bar
+		| Baz
+		| Quux Foo
+		deriving (Eq, Ord)
+
+Module imports are separated into two blocks, one for third-party modules,
+and one for modules that are part of git-annex. (Additional blocks can be used
+if it makes sense.)
+
+Using tabs for indentation makes use of `let .. in` particularly tricky.
+There's no really good way to bind multiple names in a let clause with
+tab indentation. Instead, a where clause is typically used. To bind a single
+name in a let clause, this is sometimes used:
+
+	foo = let x = 42
+		in x + (x-1) + x
+
+-----
+
+If you feel that this coding style leads to excessive amounts of horizontal
+or vertical whitespace around your code, making it hard to fit enough of it
+on the screen, consider finding a better abstraction, so the code that
+does fit on the screen is easily understandable. ;)
+
+-----
+
+Note for emacs users: You can put the following snippet into a file called 
+`.dir-locals.el` at root of git-annex's source tree to use tabs for indentation:
+
+	((nil . ((indent-tabs-mode . t)
+	         (tab-width . 8)
+	         (fill-column . 80)))
+	 ;; Warn about spaces used for indentation:
+	 (haskell-mode . ((eval . (highlight-regexp "^ *")))))
+
+Also consider [haskell-tab-indent-mode](https://spwhitton.name/tech/code/haskell-tab-indent/).  The standard indentation modes that come with haskell-mode do not work well with tabs for indentation.  This mode works well for hacking on git-annex.
diff --git a/doc/coding_style/comment_1_70521cf79ad06832b1d73fc2c20c68ec._comment b/doc/coding_style/comment_1_70521cf79ad06832b1d73fc2c20c68ec._comment
new file mode 100644
index 0000000000..a8200f856c
--- /dev/null
+++ b/doc/coding_style/comment_1_70521cf79ad06832b1d73fc2c20c68ec._comment
@@ -0,0 +1,20 @@
+[[!comment format=mdwn
+ username="https://www.google.com/accounts/o8/id?id=AItOawnq-RfkVpFN15SWvQ2lpSGAi0XpNQuLxKM"
+ nickname="Yuval"
+ subject="What about safe?"
+ date="2014-07-06T10:45:59Z"
+ content="""
+https://hackage.haskell.org/package/safe
+
+> A library wrapping Prelude/Data.List functions that can throw exceptions, such as head and !!. Each unsafe function has up to four variants, e.g. with tail:
+> 
+>     tail :: [a] -> [a], raises an error on tail [].
+
+>     tailMay :: [a] -> Maybe [a], turns errors into Nothing.
+
+>     tailDef :: [a] -> [a] -> [a], takes a default to return on errors.
+
+>     tailNote :: String -> [a] -> [a], takes an extra argument which supplements the error message.
+
+>     tailSafe :: [a] -> [a], returns some sensible default if possible, [] in the case of tail.
+"""]]
diff --git a/doc/coding_style/comment_2_a820b7c8ae7c2290eb000f61bdb5c514._comment b/doc/coding_style/comment_2_a820b7c8ae7c2290eb000f61bdb5c514._comment
new file mode 100644
index 0000000000..9f9d961962
--- /dev/null
+++ b/doc/coding_style/comment_2_a820b7c8ae7c2290eb000f61bdb5c514._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="http://joeyh.name/"
+ ip="209.250.56.2"
+ subject="comment 2"
+ date="2014-07-10T20:24:03Z"
+ content="""
+safe does not prevent using the unsafe prelude functions. Utility.PartialPrelude does, and provides a few safe wrappers like lastMaybe. 
+"""]]
diff --git a/doc/coding_style/comment_3_61c178b997dd044f3531303913f8fa91._comment b/doc/coding_style/comment_3_61c178b997dd044f3531303913f8fa91._comment
new file mode 100644
index 0000000000..c0a5661587
--- /dev/null
+++ b/doc/coding_style/comment_3_61c178b997dd044f3531303913f8fa91._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="Don"
+ subject="Style for man pages?"
+ date="2015-09-28T10:00:45Z"
+ content="""
+Could you add some suggestions for editing the .mdwn files?  I'd like to help clean up a few (really minor) things I noticed, but I don't want to make more work for you with the changes I make.  One thing I'm not sure about is where to set my fill column when editing--I can't seem to pick any single value (79? 80?) that doesn't make some paragraphs change when I rewrap them.  Also, it might be helpful to have a quick guide for when you surround commands and other computer-y words with quotes, backticks, or nothing at all.
+   
+"""]]
diff --git a/doc/coding_style/comment_4_433b7ba5e51e746f6b033ebb5c0e50a2._comment b/doc/coding_style/comment_4_433b7ba5e51e746f6b033ebb5c0e50a2._comment
new file mode 100644
index 0000000000..118a20caf2
--- /dev/null
+++ b/doc/coding_style/comment_4_433b7ba5e51e746f6b033ebb5c0e50a2._comment
@@ -0,0 +1,22 @@
+[[!comment format=mdwn
+ username="joey"
+ subject="""man page style"""
+ date="2015-09-29T15:20:10Z"
+ content="""
+Well, I don't always word-wrap paras when re-editing them, and several
+people have had their hands on the man page and may have different wrapping
+settings. 
+
+My suggestion would be, if you need to re-wrap, or generally make
+whitespace changes, do it in a separate commit from other changes. 
+Similarly, make spelling corrections in a separate commit from any
+substance changes.
+
+Note that you can push man page commits directly to this site via git://
+protocol, so it's not going to use my time as long as there's a reasonable
+commit message.
+
+Use backticks around any command name or fragment of a command line. 
+Use quotes around program output, or other values that are not command
+names or switch names.
+"""]]
diff --git a/doc/comments.mdwn b/doc/comments.mdwn
new file mode 100644
index 0000000000..e19962b92a
--- /dev/null
+++ b/doc/comments.mdwn
@@ -0,0 +1,9 @@
+[[!sidebar content="""
+[[!inline pages="comment_pending(*)" feedfile=pendingmoderation
+description="comments pending moderation" show=-1]]
+Comments in the [[!commentmoderation desc="moderation queue"]]:
+[[!pagecount pages="comment_pending(*)"]]
+"""]]
+
+Recent comments posted to this site:
+[[!inline pages="comment(*)" template="comment"]]
diff --git a/doc/contact.mdwn b/doc/contact.mdwn
new file mode 100644
index 0000000000..db9914d50f
--- /dev/null
+++ b/doc/contact.mdwn
@@ -0,0 +1,11 @@
+Joey Hess  is the author of git-annex. If you need to
+talk about something privately, email me.
+
+The [[forum]] is the best place to discuss git-annex.
+
+For realtime chat, use the `#git-annex` channel on irc.oftc.net.
+You can also watch incoming commits there.
+
+The [VCS-home mailing list](http://lists.madduck.net/listinfo/vcs-home)
+is a good mailing list for users who want to use git-annex in the context
+of managing their large personal files.
diff --git a/doc/contribute.mdwn b/doc/contribute.mdwn
new file mode 100644
index 0000000000..08b7bb4734
--- /dev/null
+++ b/doc/contribute.mdwn
@@ -0,0 +1,60 @@
+Help make git-annex better!
+
+## wiki gardening
+
+This website is a wiki, so you can edit and improve any page.
+
+Or, write a [[new_tip|tips]] explaining how to accomplish something with
+git-annex.
+
+## advanced wiki editing
+
+To make larger changes to the website, you can 
+`git clone git://git-annex.branchable.com/ git-annex` and will find the
+whole website source in `doc/`.
+
+You can even `git push` doc-only changes back without authentication,
+since it is a wiki!
+
+## bug triage
+
+People often file [[bugs]] on git-annex that are easily resolved by helping
+them with a problem, and perhaps writing some documentation.
+
+Many bugs are also filed without enough information
+to reproduce the problem, and need to be tagged moreinfo and a comment
+posted asking the submitter for details.
+
+Joey spends a lot of time dealing with this kind of bug triage. If you can
+take the time to pick a bug that is not marked as "confirmed" or "moreinfo"
+from the list of [[bugs]], try to reproduce it and follow up either
+confirming that the problem exists, or asking the submitter for more info,
+you'll make Joey more productive!
+
+## code contributions
+
+[[download]] the source code, [[build|install/fromsource]] it
+and send patches!
+
+If you know Haskell, git-annex has lots of Haskell code that
+could be improved. See the [[coding_style]] and have at it.
+
+If you don't know Haskell, git-annex has many other coding opportunities.
+You could work to improve the Android port (Java etc) or improve the
+Javascript and CSS of the git-annex webapp, or work on porting libraries
+needed by the Windows port.
+
+To send patches, either include the patch in a [[bug|bugs]] report (small
+patch) or put up a branch in a git repository containing your changes.
+
+## learning some Haskell
+
+Want to learn some Haskell to get hacking on git-annex?
+
+As Haskell programs go, git-annex does not use too many advanced features.
+[Learn You A Haskell](http://learnyouahaskell.com/) will teach you enough
+to get started.
+
+Of course git-annex does use monads, and particularly the `Annex``monad
+which gives access to its state about the git repository as well as
+lower-level IO.
diff --git a/doc/contribute/comment_1_c61530e14bdbd7c5ec8b0137c0da92f8._comment b/doc/contribute/comment_1_c61530e14bdbd7c5ec8b0137c0da92f8._comment
new file mode 100644
index 0000000000..0b43b3f0ae
--- /dev/null
+++ b/doc/contribute/comment_1_c61530e14bdbd7c5ec8b0137c0da92f8._comment
@@ -0,0 +1,19 @@
+[[!comment format=mdwn
+ username="ggreif@8132a868199b4ffec14150c87f538dc06a538220"
+ nickname="ggreif"
+ subject="GHC HEAD warnings"
+ date="2016-05-27T14:21:11Z"
+ content="""
+I have a patch:
+
+https://github.com/ggreif/git-annex/tree/patch-1
+
+It heals e.g.
+
+    Assistant/WebApp/Form.hs:52:1: warning: [-Wredundant-constraints]
+        ? Redundant constraint: Monad m
+        ? In the type signature for:
+               withNote :: (Monad m, ToWidget (HandlerSite m) a) =>
+                           Field m v -> a -> Field m v
+
+"""]]
diff --git a/doc/contribute/comment_2_637b05cfd48d2e337c3192989356d183._comment b/doc/contribute/comment_2_637b05cfd48d2e337c3192989356d183._comment
new file mode 100644
index 0000000000..c57755a687
--- /dev/null
+++ b/doc/contribute/comment_2_637b05cfd48d2e337c3192989356d183._comment
@@ -0,0 +1,10 @@
+[[!comment format=mdwn
+ username="joey"
+ subject="""comment 2"""
+ date="2016-05-27T15:45:49Z"
+ content="""
+Please open [[todo]] items for patches, don't send them as comments here.
+
+(I suspect that the patch as provided might break compilation with old
+versions of ghc, or old versions of yesod..)
+"""]]
diff --git a/doc/copies.mdwn b/doc/copies.mdwn
new file mode 100644
index 0000000000..3e0ebc6c69
--- /dev/null
+++ b/doc/copies.mdwn
@@ -0,0 +1,35 @@
+Annexed data is stored inside  your git repository's `.git/annex` directory.
+Some [[special_remotes]] can store annexed data elsewhere.
+
+It's important that data not get lost by an ill-considered `git annex drop`
+command.  So, git-annex can be configured to try
+to keep N copies of a file's content available across all repositories. 
+(Although [[untrusted_repositories|trust]] don't count toward this total.)
+
+By default, N is 1; it is configured by running `git annex numcopies N`.
+This default can be overridden on a per-file-type basis by the annex.numcopies
+setting in `.gitattributes` files. The --numcopies switch allows
+temporarily using a different value.
+
+`git annex drop` attempts to check with other git remotes, to check that N
+copies of the file exist. If enough repositories cannot be verified to have
+it, it will retain the file content to avoid data loss. Note that
+[[trusted_repositories|trust]] are not explicitly checked.
+
+For example, consider three repositories: Server, Laptop, and USB. Both Server
+and USB have a copy of a file, and N=1. If on Laptop, you `git annex get
+$file`, this will transfer it from either Server or USB (depending on which
+is available), and there are now 3 copies of the file.
+
+Suppose you want to free up space on Laptop again, and you `git annex drop` the file
+there. If USB is connected, or Server can be contacted, git-annex can check
+that it still has a copy of the file, and the content is removed from
+Laptop. But if USB is currently disconnected, and Server also cannot be
+contacted, it can't verify that it is safe to drop the file, and will
+refuse to do so.
+
+With N=2, in order to drop the file content from Laptop, it would need access
+to both USB and Server.
+
+For more complicated requirements about which repositories contain which
+content, see [[required_content]].
diff --git a/doc/copies/comment_1_af9bee33777fb8a187b714fc8c5fb11d._comment b/doc/copies/comment_1_af9bee33777fb8a187b714fc8c5fb11d._comment
new file mode 100644
index 0000000000..45e5722604
--- /dev/null
+++ b/doc/copies/comment_1_af9bee33777fb8a187b714fc8c5fb11d._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="http://olivier.berger.myopenid.com/"
+ nickname="obergix"
+ subject="See also section on backups in walthroug"
+ date="2013-08-25T05:37:04Z"
+ content="""
+See also [walthrough/backups](/walkthrough/backups/) for some details about copies. 
+"""]]
diff --git a/doc/design.mdwn b/doc/design.mdwn
new file mode 100644
index 0000000000..e4f0e5c2a9
--- /dev/null
+++ b/doc/design.mdwn
@@ -0,0 +1,6 @@
+git-annex's high-level design is mostly inherent in the data that it
+stores in git, and alongside git. See [[internals]] for details.
+
+Here's the other design documents we have:
+
+[[!map pages="page(design/*) and !design/*/*"]]
diff --git a/doc/design/adjusted_branches.mdwn b/doc/design/adjusted_branches.mdwn
new file mode 100644
index 0000000000..608c04c9a7
--- /dev/null
+++ b/doc/design/adjusted_branches.mdwn
@@ -0,0 +1,316 @@
+Consider two use cases:
+
+1. Using a v6 repo with locked files on a crippled filesystem not
+   supporting symlinks. For the files to be usable, they need to be
+   unlocked. But, the user may not want to unlock the files everywhere,
+   just on this one crippled system.
+2. [[todo/hide_missing_files]]
+
+Both of these could be met by making `git-annex sync` maintain an adjusted
+version of the original branch, eg `adjusted/master`.
+
+There would be a adjustment function. For #1 above it would simply convert all
+annex symlinks to annex file pointers. For #2 above it would omit files
+whose content is not currently in the annex. Sometimes, both #1 and #2 would
+be wanted. The function is currently implemented as the [[git-annex-adjust]] command.
+
+[Alternatively, it could stay on the master branch, and only adjust the
+work tree and index. See WORKTREE notes below for how this choice would
+play out.]
+
+[[!toc]]
+
+## adjusting
+
+	master           adjusted/master
+	A
+	|--------------->A'
+	|                |
+
+When generating commit A', reuse the date of A and use a standard author,
+committer, and message. This means that two users with the adjusted branch
+checked out and using the same adjustments will get identical shas for A', and
+so can collaborate on them.
+
+## commit
+
+When committing changes, a commit is made as usual to the adjusted branch.
+So, the user can `git commit` as usual. This does not touch the
+original branch yet. 
+
+Then we need to get from that commit to one with the adjustments reversed,
+which should be the same as if the adjusted branch had not been used.
+This commit gets added onto the original branch.
+
+So, the branches would look like this:
+
+	master           adjusted/master
+	A
+	|--------------->A'
+	|                |
+	|                C (new commit)
+	B < - - - - - - -
+	|                
+	|--------------->B'
+	|                |
+
+Note particularly that B does not have A' or C in its history;
+the adjusted branch is not evident from outside.
+
+Also note that B gets adjusted and the adjusted branch is rebased on top of
+it, so C does not remain in the adjusted branch history either. This will
+make other checkouts that are in the same adjusted branch end up with the
+same B' commit when they pull B.
+
+There may be multiple commits made to the adjusted branch before any get
+applied back to the original branch. This is handled by reverse adjusting
+commits one at a time and rebasing the others on top.
+
+	master           adjusted/master
+	A
+	|--------------->A'
+	|                |
+	|                C1
+	|                |
+	|                C2
+
+
+	master           adjusted/master
+	A
+	|--------------->A'
+	|                |
+	|                C1
+	B1< - - - - - - -
+	|
+	|--------------->B1'
+	|                |
+	|                C2'
+	B2< - - - - - - -
+	|
+	|--------------->B2'
+
+
+[WORKTREE: A pre-commit hook would be needed to update the staged changes, 
+reversing the adjustment before the commit is made. All the other complications
+above are avoided.]
+
+## merge
+
+This would be done by `git annex merge` and `git annex sync`, with the goal
+of merging origin/master into master, and updating adjusted/master.
+
+Note that the adjusted files db needs to be updated to reflect the changes
+that are merged in, for object add/remove to work as described below.
+
+When merging, there should never be any commits present on the
+adjusted/master branch that have not yet been propigated back to the master
+branch. If there are any such commits, just propigate them into master
+before beginning the merge. There may be staged changes, or changes in the
+work tree.
+
+First, merge origin/master into master. This is done in a temp work
+tree and with a temp index, so does not affect the checked out adjusted
+branch.
+
+(Note that the reason this is done, rather than adjusting origin/master
+and merging it into the work tree, is that merge conflicts would be very
+common with the naive approach, because the adjusted branch often changes
+files, and origin/master may change the same files.)
+
+	origin/master  master   adjusted/master
+	A------------->A- - - ->A'
+	|              |
+	B------------->C
+
+While a fast-forward merge is shown here, other merges work the same way.
+There may be merge conflicts; if so they're auto-resolved.
+
+Then, adjust merge commit C, and merge that into adjusted/master.
+	
+	origin/master  master   adjusted/master
+	A------------->A- - - ->A'
+	|              |        |
+	B------------->C- - C'->D'
+
+This merge is done in-worktree, so the work tree gets updated.
+There may be more merge conflicts here; they're also auto-resolved.
+
+Now, D' is a merge commit, between A' and C'. 
+To finish, change that commit so it does not have A' as its parent.
+
+This can be accomplished by propigating the reverse-adjusted D'
+back to master, and then adjusting master to yield the final
+adjusted/master.
+	
+	origin/master  master   adjusted/master
+	A------------->A
+	|              |         
+	B------------->C         
+	               |         
+		       D - - -> D'
+
+Notice how similar this is to the commit graph. Indeed, "fast-forward" 
+merging the same B commit from origin/master will lead to an identical
+sha for B' as the original committer got!
+
+Since the adjusted/master branch is not present on the remote, if the user
+does a `git pull`, it won't merge in changes from origin/master. Which is
+good because the adjustment needs to be applied first.
+
+However, if the user does `git merge origin/master`, they'll get into a
+state where the adjustment has not been applied. The post-merge hook could be
+used to clean up after that. Or, let the user foot-shoot this way; they can
+always reset back once they notice the mistake.
+
+[WORKTREE: `git pull` would update the work tree, and may lead to conflicts
+between the adjusted work tree and pulled changes. A post-merge hook would
+be needed to re-adjust the work tree, and there would be a window where eg,
+not present files would appear in the work tree.]
+
+## annex object add/remove
+
+When objects are added/removed from the annex, the associated file has to
+be looked up, and the adjustment applied to it. So, dropping a file with the
+missing file adjustment would cause it to be removed from the adjusted branch,
+and receiving a file's content would cause it to appear in the adjusted
+branch. TODO
+
+These changes would need to be committed to the adjusted branch, otherwise
+`git diff` would show them.
+
+How to avoid making a new commit each time a single object is
+added/removed? That seems too expensive in both CPU and dangling git
+objects for old versions of the adjusted branch. It would be fine if
+`git annex get` and `git annex drop` only re-adjusted the branch one time
+at the end. OTOH, when should the assistant re-adjust the branch?
+
+Maybe instead of re-adjusting the branch after each file, stage the
+worktree change, and hold off on committing. Then when a commit is
+eventually made, the reverse adjusting to propigate it to master would need
+to make sure to not remove files that were deleted as part of the commit,
+if their content is not present.
+
+[WORKTREE: Simply adjust the work tree (and index) per the adjustment.]
+
+## reverse adjusting commits
+
+A user's commits on the adjusted branch have to be reverse adjusted
+to get changes to apply to the master branch.
+
+This reversal of one adjustment can be done as just another adjustment.
+Since only files touched by the commit will be reverse adjusted, it doesn't
+need to reverse all changes made by the original adjustment.
+
+For example, reversing the unlock adjustment might lock the file. Or, it might
+do nothing, which would make all committed files remain unlocked.
+
+## push
+
+The new master branch can then be pushed out to remotes. The
+adjusted/master branch is not pushed to remotes. `git-annex sync` should
+automatically push master when adjusted/master is checked out.
+
+When push.default is "simple" (the new default), running `git push` when in
+adjusted/master won't push anything. It would with "matching". Pity. (I
+continue to feel git picked the wrong default here.) Users may find that
+surprising. Users of `git-annex sync` won't need to worry about it though.
+
+[WORKTREE: push works as usual]
+
+## acting on filtered-out files
+
+If a file is filtered out due to not existing, there should be a way
+for `git annex get` to get it. Since the filtered out file is not in the
+index, that would not normally work. What to do?
+
+Maybe instead of making a branch where the file is deleted, it would be
+better to delete it from the work tree, but keep the branch as-is. Then
+`git annex get` would see the file, as it's in the index. 
+
+But, not maintaining an adjusted branch complicates other things. See
+WORKTREE notes throughout this page. Overall, the WORKTREE approach seems
+too problematic.
+
+Ah, but we know that when adjustment #2 is in place, any file that `git annex
+get` could act on is not in the index. So, it could look at the master branch
+instead. (Same for `git annex move --from` and `git annex copy --from` and
+the assistant.)
+
+OTOH, if adjustment #1 is in place and not #2, a file might be renamed in the
+index, and `git annex get $newname` should work. So, it should look at the
+index in that case.
+
+## problems
+
+Using `git checkout` when in an adjusted branch is problematic, because a
+non-adjusted branch would then be checked out. But, we can just say, if
+you want to get into an adjusted branch, you have to run git annex adjust
+Or, could make a post-checkout hook. This is would mostly be confusing when
+git-annex init switched into the adjusted branch due to lack of symlink
+support.
+
+After a commit to an adjusted branch, `git push` won't do anything. The
+user has to know to git-annex sync. (Even if a pre-commit hook propigated
+the commit back to the master branch, `git push` wouldn't push it with the
+default "matching" push strategy.)
+
+Tags are bit of a problem. If the user tags an ajusted branch, the tag
+includes the local adjustments.  
+[WORKTREE: not a problem]
+
+If the user refers to commit shas (in, eg commit messages), those won't be
+visible to anyone else.  
+[WORKTREE: not a problem]
+
+When a pull modifies a file, its content won't be available, and so it
+would be hidden temporarily by adjustment #2. So the file would seem to vanish,
+and come back later, which could be confusing. Could be fixed as discussed
+in [[todo/deferred_update_mode]]. Arguably, it's just as confusing for the
+file to remain visible but have its content temporarily replaced with a
+annex pointer.
+
+### master push overwrite race (fixed)
+
+There are potentially races in code that assumes a branch like
+master is not being changed by someone else. 
+  
+In particular, if propigateAdjustedCommits rebases the adjusted branch on
+top of master. That is called by sync. The assumption is that any changes
+in master have already been handled by updateAdjustedBranch. But, if
+another remote pushed a new master at just the right time, the adjusted
+branch could be rebased on top of a master that it doesn't incorporate,
+which is wrong.
+
+Best fix seems to be to maintain a basis ref, that is not a branch,
+like refs/adjusted/master(unlocked). Copy master's ref to it when
+entering the view branch. Then, make all adjustments via the basis
+ref, and propigate back to refs/heads/master.
+
+It's fine to overwrite changes that were pushed to master when
+propigating from the adjusted branch. Synced changes also go to
+synced/master so won't be lost. Pushes not made using git-annex sync
+of master are not really desired, just a possibility.
+
+## integration with view branches
+
+Entering a view from an adjusted branch should probably carry the adjusting
+over into the creation/updating of the view branch.
+
+Could go a step further, and implement view branches as another branch
+adjustment, albeit an extreme one. This might improve view branches.
+For example, it's not currently possible to update a view branch with
+changes fetched from a remote, and this could get us there.
+
+This would need the reverse adjust to be able to change metadata,
+so that a commit that moved files in the view updates their metadata.
+
+[WORKTREE: Wouldn't be able to integrate, unless view branches are changed
+into adjusted view worktrees.]
+
+## TODOs
+
+* Interface in webapp to enable adjustments.
+* Honor annex.thin when entering an adjusted branch. git checkout
+  will make copies of the content of annexed files, so this would need
+  to checkout the adjusted branch some other way. Maybe generalize so this
+  more efficient checkout is available as a git-annex command?
diff --git a/doc/design/adjusted_branches/comment_1_5bdbf54ae75f77c69c8f557f4dbac13d._comment b/doc/design/adjusted_branches/comment_1_5bdbf54ae75f77c69c8f557f4dbac13d._comment
new file mode 100644
index 0000000000..36f2db3065
--- /dev/null
+++ b/doc/design/adjusted_branches/comment_1_5bdbf54ae75f77c69c8f557f4dbac13d._comment
@@ -0,0 +1,10 @@
+[[!comment format=mdwn
+ username="https://me.yahoo.com/a/EbvxpTI_xP9Aod7Mg4cwGhgjrCrdM5s-#7c0f4"
+ subject="would it be possible to get a branch only with annexed content?"
+ date="2016-02-25T03:04:26Z"
+ content="""
+I am yet to grasp all the glorious plans here, but wondered to ask:  it seems like it then should be possible to establish a branch only with annexed archive? may be matching some preferred content expression?
+Usecase:  a relatively large git/annex repository mixing in code, large data files, and some pre-built binaries.  If I could seamlessly and reproducibly (with the progress of their master) tease those apart (for separate Debian packaging ;)), that would be really handy
+
+[[!meta author=yoh]]
+"""]]
diff --git a/doc/design/adjusted_branches/comment_2_ff7cbdf93f0be5886f212bdd411efb5b._comment b/doc/design/adjusted_branches/comment_2_ff7cbdf93f0be5886f212bdd411efb5b._comment
new file mode 100644
index 0000000000..6a3502997e
--- /dev/null
+++ b/doc/design/adjusted_branches/comment_2_ff7cbdf93f0be5886f212bdd411efb5b._comment
@@ -0,0 +1,9 @@
+[[!comment format=mdwn
+ username="joey"
+ subject="""comment 2"""
+ date="2016-03-14T20:38:09Z"
+ content="""
+Yes, lots of things are possible with this. Filtering to only annexed files
+probably. May be some hairyness around updating the branch when files are
+got or dropped. Matching a preferred content expression, maybe..
+"""]]
diff --git a/doc/design/adjusted_branches/comment_3_e2a4b1719ba9860da711b802de64daf5._comment b/doc/design/adjusted_branches/comment_3_e2a4b1719ba9860da711b802de64daf5._comment
new file mode 100644
index 0000000000..a30f4c50c7
--- /dev/null
+++ b/doc/design/adjusted_branches/comment_3_e2a4b1719ba9860da711b802de64daf5._comment
@@ -0,0 +1,10 @@
+[[!comment format=mdwn
+ username="thowz"
+ subject="adjusting symlink paths"
+ date="2016-05-13T22:32:33Z"
+ content="""
+Another good use case for this would be adjusting the target paths of symlinks for non-standard locations of the .git/annex/objects folder, such as when cloning into a repo where the GIT_DIR is outside the worktree folder, or when someone is using multiple worktrees and the links in other worktrees need to point to the GIT_COMMON_DIR.
+
+[[https://git-scm.com/docs/git-worktree]]
    
+[[https://stacktoheap.com/blog/2016/01/19/using-multiple-worktrees-with-git/]]
+"""]]
diff --git a/doc/design/adjusted_branches/comment_4_adfcbfa7a9193318cba459766a60c072._comment b/doc/design/adjusted_branches/comment_4_adfcbfa7a9193318cba459766a60c072._comment
new file mode 100644
index 0000000000..2037a5f32b
--- /dev/null
+++ b/doc/design/adjusted_branches/comment_4_adfcbfa7a9193318cba459766a60c072._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="joey"
+ subject="""comment 4"""
+ date="2016-05-16T20:34:28Z"
+ content="""
+@thowz great idea! And not hard at all to implement! I've done so:
+`git annex adjust --fix`
+"""]]
diff --git a/doc/design/adjusted_branches/comment_5_882520747d41500d0ea8a18ab817121c._comment b/doc/design/adjusted_branches/comment_5_882520747d41500d0ea8a18ab817121c._comment
new file mode 100644
index 0000000000..91f5d824c1
--- /dev/null
+++ b/doc/design/adjusted_branches/comment_5_882520747d41500d0ea8a18ab817121c._comment
@@ -0,0 +1,22 @@
+[[!comment format=mdwn
+ username="zack"
+ subject="adjusted branche to "focus" on a specific subtree"
+ date="2016-08-22T14:19:57Z"
+ content="""
+I'm having hard time supporting the following use case with git-annex:
+
+* using git annex so that \"git annex sync --content\" import in batch pictures from a camera SD card
+* destination of the import is a \"pictures/INCOMING\" subdir of a larger \"multimedia archive\" annex
+* I do not want to have all the other folders of the annex (parent and siblings of pictures/INCOMING) present on the SD card
+
+I wonder if adjusted branches might be a solution to this.
+Do I get it right that \"unlocked\" is currently the only supported kind of adjustment?
+
+If so, I'm not sure what would be needed to make the above use case feasible.
+At first sight though, commits int the adjusted branch that \"mounts\" pictures/INCOMING/ are conceptually easy to translate to the main branch: the changes would be the same, only they'll have to be applied in a specific subtree.
+They won't merge cleanly though.
+
+Is this an interesting/worthwhile use case for adjusted branches, or am I looking into the wrong part of git-annex design?
+
+Thanks for this amazing tool!
+"""]]
diff --git a/doc/design/assistant.mdwn b/doc/design/assistant.mdwn
new file mode 100644
index 0000000000..052c536782
--- /dev/null
+++ b/doc/design/assistant.mdwn
@@ -0,0 +1,45 @@
+These are the design pages for the git-annex [[/assistant]].
+
+Parts of the design is still being fleshed out, still many ideas
+and use cases to add. Feel free to chip in with comments! --[[Joey]]
+
+See [[roadmap]] for current plans, as this list was mostly completed.
+
+## initial development kickstarter year overview (2012-2013)
+
+* Month 1 "like dropbox": [[!traillink inotify]] [[!traillink syncing]]
+* Month 2 "shiny webapp": [[!traillink webapp]] [[!traillink progressbars]]
+* Month 3 "easy setup": [[!traillink configurators]] [[!traillink pairing]]
+* Month 4 "cloud": [[!traillink cloud]] [[!traillink transfer_control]]
+* Month 5 "cloud continued": [[!traillink xmpp]] [[!traillink more_cloud_providers]]
+* Month 6 "9k bonus round": [[!traillink desymlink]]
+* Month 7: user-driven features and polishing;
+  [presentation at LCA2013](http://mirror.linux.org.au/linux.conf.au/2013/mp4/gitannex.mp4)
+* Month 8: [[!traillink Android]]
+* Month 9: [[screencasts|videos]] and polishing
+* Month 10: bugfixing, [[Android]] webapp
+* Month 11: [[!traillink Windows]] porting, finishing touches
+* Month 12: [presentation at SELF2013](http://www.southeastlinuxfest.org/), finishing touches
+
+## porting
+
+* [[OSX]] port is in fairly good shape, but still has some room for improvement
+* [[android]] port is zooming along
+* [[Windows]] port is barely getting started
+
+## not yet on the map:
+
+* [[rate_limiting]]
+* [[partial_content]]
+* [[deltas]]
+* [[leftovers]]
+* [[other todo items|todo]]
+
+## polls
+
+I post [[polls]] occasionally to make decisions. You can vote!
+
+## blog
+
+I'm blogging about my progress in the [[devblog]] on a semi-daily basis.
+Follow along!
diff --git a/doc/design/assistant/OSX.mdwn b/doc/design/assistant/OSX.mdwn
new file mode 100644
index 0000000000..400e517bf4
--- /dev/null
+++ b/doc/design/assistant/OSX.mdwn
@@ -0,0 +1,13 @@
+Misc OSX porting things:
+
+* autostart the assistant on OSX, using launchd **done**
+* icon to start webapp **done**
+* use FSEvents to detect file changes (better than kqueue) **done**
+* Use OSX's "network reachability functionality" to detect when on a network
+  
+* Switch from gpg to . According to a user,
+  this is better because it can show a dialog window for password prompts.
+
+Bugs:
+
+[[!inline pages="tagged(design/assistant/OSX) and !link(bugs/done)" show=0 archive=yes]]
diff --git a/doc/design/assistant/OSX/comment_1_9290f6e6f265e906b08631224392b7bf._comment b/doc/design/assistant/OSX/comment_1_9290f6e6f265e906b08631224392b7bf._comment
new file mode 100644
index 0000000000..633dd01b45
--- /dev/null
+++ b/doc/design/assistant/OSX/comment_1_9290f6e6f265e906b08631224392b7bf._comment
@@ -0,0 +1,14 @@
+[[!comment format=mdwn
+ username="https://www.google.com/accounts/o8/id?id=AItOawlu-fdXIt_RF9ggvg4zP0yBbtjWQwHAMS4"
+ nickname="Jörn"
+ subject="Mount detection"
+ date="2012-09-21T09:23:34Z"
+ content="""
+regarding the current mount polling on OSX: why not use the NSNotificationCenter for being notified on mount events on OSX?
+
+Details see:
+
+1. 
+2. 
+3. 
+"""]]
diff --git a/doc/design/assistant/android.mdwn b/doc/design/assistant/android.mdwn
new file mode 100644
index 0000000000..8beaffabc3
--- /dev/null
+++ b/doc/design/assistant/android.mdwn
@@ -0,0 +1,42 @@
+The Android port is just about usable. Still, we have some fun todo items
+to improve it.
+
+## high-priority TODO
+
+* [[bugs/Android_app_permission_denial_on_startup]]
+* S3 doesn't work (at least to Internet Archive: 
+  "connect: does not exist (connection refused)")
+* Get app into Google Play and/or FDroid
+
+## TODO
+
+* Don't make app initially open terminal + webapp, but go to a page that
+  allows opening the webapp or terminal.  
+  Possibly, switch from running inside terminal app to real standalone app.
+  See 
+  and .
+
+* I have seen an assistant thread crash with an interrupted system call
+  when the device went to sleep while it was running. Auto-detect and deal with
+  that somehow.
+* Make git stop complaining that "warning: no threads uspport, ignoring --threads"
+* git does not support http remotes. To fix, need to port libcurl and
+  allow git to link to it.
+* getEnvironment is broken on Android 
+  and a few places use it. I have some horrible workarounds in place.
+* Get local pairing to work. network-multicast and network-info don't
+  currently install.
+* Get test suite to pass. `git clone` of a local repo fails on android
+  for some reason.
+* Make app autostart on boot, optionally. 
+* The app should be aware of power status, and avoid expensive background
+  jobs when low on battery or run flat out when plugged in.
+* The app should be aware of network status, and avoid expensive data
+  transfers when not on wifi. This may need to be configurable.
+* glacier and local pairing are not yet enabled for Android.
+* The "Files" link doesn't start a file browser. Should be possible to do
+  on Android via intents, I suppose?
+* Adding removable drives would work, but the android app is not in the 
+  appropriate group to write to them. `WRITE_MEDIA_STORAGE` permission
+  needed. Added to AndroidManifest, but did not seem to be used.
+  Googleing for it will find a workaround that needs a rooted device.
diff --git a/doc/design/assistant/android/comment_10_316bde8d22628e5e9d4f8dabce1d2ad4._comment b/doc/design/assistant/android/comment_10_316bde8d22628e5e9d4f8dabce1d2ad4._comment
new file mode 100644
index 0000000000..89f23d72ca
--- /dev/null
+++ b/doc/design/assistant/android/comment_10_316bde8d22628e5e9d4f8dabce1d2ad4._comment
@@ -0,0 +1,14 @@
+[[!comment format=mdwn
+ username="Xyem"
+ ip="87.194.19.134"
+ subject="comment 10"
+ date="2013-05-06T12:18:39Z"
+ content="""
+One of the points is:
+
+* The app should be aware of power status, and avoid expensive background jobs when low on battery or run flat out when plugged in.
+
+This isn't good for situations where the device is plugged in and charging, but will only be so for a limited time (such as charging during a car journey) as the device would charge slower.
+
+Could this behaviour be configurable too?
+"""]]
diff --git a/doc/design/assistant/android/comment_11_cf801ce1f49f0201681f0c9f69a62aae._comment b/doc/design/assistant/android/comment_11_cf801ce1f49f0201681f0c9f69a62aae._comment
new file mode 100644
index 0000000000..7098514441
--- /dev/null
+++ b/doc/design/assistant/android/comment_11_cf801ce1f49f0201681f0c9f69a62aae._comment
@@ -0,0 +1,14 @@
+[[!comment format=mdwn
+ username="pweemeeuw@86491f921da15d6a4dc6e1878fd42750b33f6963"
+ nickname="pweemeeuw"
+ subject="Is android 6.0 supported?"
+ date="2015-11-30T16:05:13Z"
+ content="""
+I tried to install the version for android 5 on my nexus 5 with the most recent firmware. The initial sync (git clone) succeeds, the git annex stuff fails with \"remote origin does not have git-annex installed; setting annex-ignore\".
+
+I have the android 4 version working on my tablet for the same ssh remote, so I think the remote is OK.
+
+I can provide full logs of the terminal session if needed (I didn't use the assistant front end).
+
+BTW: this is great software, thanks you so very much! I hope to have it up and running on all my devices soon as a replacement for Google Drive. I happen to care about privacy.
+"""]]
diff --git a/doc/design/assistant/android/comment_1_8be9a74e5fc4641c2bf2e1bb7673dd59._comment b/doc/design/assistant/android/comment_1_8be9a74e5fc4641c2bf2e1bb7673dd59._comment
new file mode 100644
index 0000000000..389eac026d
--- /dev/null
+++ b/doc/design/assistant/android/comment_1_8be9a74e5fc4641c2bf2e1bb7673dd59._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="https://launchpad.net/~gdr-go2"
+ nickname="gdr-go2"
+ subject="FAT symlinks"
+ date="2012-05-28T18:12:10Z"
+ content="""
+It's a linux kernel so perhaps another option would be to create a big file and mount -o loop
+"""]]
diff --git a/doc/design/assistant/android/comment_2_3dd386ac1b757c73d14f14377b9eedd4._comment b/doc/design/assistant/android/comment_2_3dd386ac1b757c73d14f14377b9eedd4._comment
new file mode 100644
index 0000000000..3ff46ca060
--- /dev/null
+++ b/doc/design/assistant/android/comment_2_3dd386ac1b757c73d14f14377b9eedd4._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="https://www.google.com/accounts/o8/id?id=AItOawlhIMPGF1E0XEJKV6j6-PFzAxA1-nIlydo"
+ nickname="Bernhard"
+ subject="re @  gdr-go2 "
+ date="2012-09-10T19:34:00Z"
+ content="""
+`mount` requires root and you'll have still the 4gb limit for your image by FAT. some phones (e.g. galaxy nexus) already use `ext4` for `/sdcard` though.
+"""]]
diff --git a/doc/design/assistant/android/comment_3_5dca47a4599d6e88d19193701c5a571b._comment b/doc/design/assistant/android/comment_3_5dca47a4599d6e88d19193701c5a571b._comment
new file mode 100644
index 0000000000..8f60efdb06
--- /dev/null
+++ b/doc/design/assistant/android/comment_3_5dca47a4599d6e88d19193701c5a571b._comment
@@ -0,0 +1,46 @@
+[[!comment format=mdwn
+ username="https://www.google.com/accounts/o8/id?id=AItOawlhIMPGF1E0XEJKV6j6-PFzAxA1-nIlydo"
+ nickname="Bernhard"
+ subject="GHC and Android"
+ date="2012-09-10T21:58:28Z"
+ content="""
+I played around a bit with GHC and Android today. It isn't really a result, but maybe useful for someone out there.
+
+I have a Debian `chroot` environment on my Android device (howto: ). In the Debian box:
+
+    $ cat arm.hs
+    main = do
+      putStrLn \"Hello ARM\"
+    $ ghc -static --make arm.hs
+    Linking arm ...
+    $ ldd arm
+	libgmp.so.3 => /usr/lib/libgmp.so.3 (0x40233000)
+	libm.so.6 => /lib/libm.so.6 (0x400c8000)
+	libffi.so.5 => /usr/lib/libffi.so.5 (0x401b1000)
+	librt.so.1 => /lib/librt.so.1 (0x40171000)
+	libdl.so.2 => /lib/libdl.so.2 (0x40180000)
+	libgcc_s.so.1 => /lib/libgcc_s.so.1 (0x4018b000)
+	libc.so.6 => /lib/libc.so.6 (0x40282000)
+	/lib/ld-linux.so.3 (0x400a2000)
+	libpthread.so.0 => /lib/libpthread.so.0 (0x4007e000)
+
+well, that isn't really static. tell the linker to build a static binary (those are arguments to `ld`):
+
+    $ ghc --make arm.hs -optl-static -optl-pthread
+    [1 of 1] Compiling Main             ( arm.hs, arm.o )
+    Linking arm ...
+    $ file arm
+    arm: ELF 32-bit LSB executable, ARM, version 1 (SYSV), statically linked, for GNU/Linux 2.6.18, not stripped
+    $ ldd arm
+    	not a dynamic executable
+    $ ./arm
+    Hello ARM
+
+now, get this (quite big) binary into the normal android environment, using `adb`, `SSHDroid` or whatever:
+
+    % cd /data/local/tmp # assuming destination of file transfer
+    % ./arm
+    arm: mkTextEncoding: invalid argument (Invalid argument)
+
+looking in the source of `System.IO` it seems like an `iconv` issue. So, there's still some dynamic dependency in there... *sigh*
+"""]]
diff --git a/doc/design/assistant/android/comment_4_054f06311e2b51d73be569f181eb004f._comment b/doc/design/assistant/android/comment_4_054f06311e2b51d73be569f181eb004f._comment
new file mode 100644
index 0000000000..76c32908a6
--- /dev/null
+++ b/doc/design/assistant/android/comment_4_054f06311e2b51d73be569f181eb004f._comment
@@ -0,0 +1,10 @@
+[[!comment format=mdwn
+ username="http://joeyh.name/"
+ ip="4.153.8.30"
+ subject="comment 4"
+ date="2012-09-11T02:12:45Z"
+ content="""
+Thanks Bernard, that's really massively useful. It makes sense -- statically building with libc should work, the Android kernel is still Linux after all. 
+
+To get past the iconv problem, I think all you need is part of the `locales` package from your linux system installed on the Android. Probably just a few of the data files from /usr/share/i18n/charmaps/
+"""]]
diff --git a/doc/design/assistant/android/comment_5_bb3d36e9d29f2fa77bee6d47ef9917fe._comment b/doc/design/assistant/android/comment_5_bb3d36e9d29f2fa77bee6d47ef9917fe._comment
new file mode 100644
index 0000000000..be1e3e9e81
--- /dev/null
+++ b/doc/design/assistant/android/comment_5_bb3d36e9d29f2fa77bee6d47ef9917fe._comment
@@ -0,0 +1,21 @@
+[[!comment format=mdwn
+ username="https://www.google.com/accounts/o8/id?id=AItOawlhIMPGF1E0XEJKV6j6-PFzAxA1-nIlydo"
+ nickname="Bernhard"
+ subject="comment 5"
+ date="2012-09-11T09:34:42Z"
+ content="""
+`/usr/share/i18n/` does not exists on my Debian ARM system :/
+
+however, `strace ./arm` in the debian chroot reveals that some files from `/usr/lib/gconv/` are loaded:
+
+    [...]
+    open(\"/usr/lib/gconv/UTF-32.so\", O_RDONLY) = 3
+    read(3, \"\177ELF\1\1\1\0\0\0\0\0\0\0\0\0\3\0(\0\1\0\0\0\4\4\0\0004\0\0\0\"..., 512) = 512
+    [...]
+
+full log: .  Unfortunately, I don't have `strace` in the android userland for comparison.
+
+Just copying the related `gconv` files didn't work. I don't have so much time at the moment, I'll investigate further in some days or so.
+
+At least, output using `error :: String -> a` does work :-)
+"""]]
diff --git a/doc/design/assistant/android/comment_6_fee32a831eeb5736fe1dce52e30320c8._comment b/doc/design/assistant/android/comment_6_fee32a831eeb5736fe1dce52e30320c8._comment
new file mode 100644
index 0000000000..9dfe5670de
--- /dev/null
+++ b/doc/design/assistant/android/comment_6_fee32a831eeb5736fe1dce52e30320c8._comment
@@ -0,0 +1,16 @@
+[[!comment format=mdwn
+ username="https://www.google.com/accounts/o8/id?id=AItOawl9sYlePmv1xK-VvjBdN-5doOa_Xw-jH4U"
+ nickname="Richard"
+ subject="comment 6"
+ date="2012-10-30T08:44:32Z"
+ content="""
+Just so this does not get lost: For better or for worse, the vanilla Android devices stopped shipping with micro-SD support in 2011 (or 2010 if the Nexus S does not support them either; on sketchy GPRS so not googling around). Most higher-end Android devices ship with at least 8 GiB of on-board Flash storage, some even go up to 64 GiB.
+
+IMHO, this would make it viable to first get git-annex working on Android without regard for FAT.
+
+The obvious advantage is that porting should be easier and quicker.
+
+The obvious downside is that this may mean revisiting some parts of the code later.
+
+-- Richard
+"""]]
diff --git a/doc/design/assistant/android/comment_7_d8e9b0a5287fc96b19dc2cb9da3586ce._comment b/doc/design/assistant/android/comment_7_d8e9b0a5287fc96b19dc2cb9da3586ce._comment
new file mode 100644
index 0000000000..0419b1d1b0
--- /dev/null
+++ b/doc/design/assistant/android/comment_7_d8e9b0a5287fc96b19dc2cb9da3586ce._comment
@@ -0,0 +1,17 @@
+[[!comment format=mdwn
+ username="https://www.google.com/accounts/o8/id?id=AItOawl9sYlePmv1xK-VvjBdN-5doOa_Xw-jH4U"
+ nickname="Richard"
+ subject="comment 7"
+ date="2012-10-30T08:51:17Z"
+ content="""
+Actually, this is something that would be ideal for a poll:
+
+Should FAT-based Android repos be implemented:
+
+* Immediately
+* After the initial release of the Android app is working
+* Before the kickstarter ends
+* Not at all
+
+Also, as another data point, the FAT-based SD card can be mounted as USB storage by any computer an Android device is connected to whereas the EXT4-based root FS can only be accessed via MTP.
+"""]]
diff --git a/doc/design/assistant/android/comment_8_79a7b5bb5f4aaeea4a4e8ced0561701a._comment b/doc/design/assistant/android/comment_8_79a7b5bb5f4aaeea4a4e8ced0561701a._comment
new file mode 100644
index 0000000000..91db74ec65
--- /dev/null
+++ b/doc/design/assistant/android/comment_8_79a7b5bb5f4aaeea4a4e8ced0561701a._comment
@@ -0,0 +1,11 @@
+[[!comment format=mdwn
+ username="https://www.google.com/accounts/o8/id?id=AItOawnZEanlyzay_QlEAL0CWpyZcRTyN7vay8U"
+ nickname="Carlo"
+ subject="Terminal IDE"
+ date="2012-11-21T21:46:19Z"
+ content="""
+I use Terminal IDE on android, which is 'sort of a shell environment'.
+
+It has bash, git, dropbear ssh, and vim. It is a bit limited, but it does feel like a linux shell.
+
+"""]]
diff --git a/doc/design/assistant/android/comment_9_55ea70a6929523d26248ff6409b04a6e._comment b/doc/design/assistant/android/comment_9_55ea70a6929523d26248ff6409b04a6e._comment
new file mode 100644
index 0000000000..d371c5fedb
--- /dev/null
+++ b/doc/design/assistant/android/comment_9_55ea70a6929523d26248ff6409b04a6e._comment
@@ -0,0 +1,10 @@
+[[!comment format=mdwn
+ username="https://www.google.com/accounts/o8/id?id=AItOawl7ciFZWmffuw6sLRww3CcL_F5ItsttL9w"
+ nickname="Aaron"
+ subject="Feature request: Events triggered on wifi SSID or connection state"
+ date="2013-01-08T13:04:05Z"
+ content="""
+I'd like to have my phone sync a bunch of files when I'm at home and when I get to work, have them synced to my work machine, deleted off my phone and when I get home, also deleted off my home machine.
+
+This will allow incremental backups to a local folder at home and I can then \"mule\" my files, sneaker-net style, to work with zero user intervention. It only works if events can be triggered or I can combine a triggering app with a special android intent for both ends of my trip.
+"""]]
diff --git a/doc/design/assistant/blog.mdwn b/doc/design/assistant/blog.mdwn
new file mode 100644
index 0000000000..561173d27b
--- /dev/null
+++ b/doc/design/assistant/blog.mdwn
@@ -0,0 +1,11 @@
+This blog was updated on a semi-daily basis by Joey during the year of work
+concentrating on the git-annex assistant that was funded by his kickstarter
+campaign.
+
+Post-kickstarter work will instead appear on the [[devblog]]. However,
+this page's RSS feed will continue to work, so you don't have to migrate
+your RSS reader.
+
+[[!inline pages="page(devblog/*)" show=-1 feedshow=7]]
+
+[[!inline pages="page(design/assistant/blog/*)" show=0 feeds=no]]
diff --git a/doc/design/assistant/blog/day_100__cursed_clouds.mdwn b/doc/design/assistant/blog/day_100__cursed_clouds.mdwn
new file mode 100644
index 0000000000..7ac38f4634
--- /dev/null
+++ b/doc/design/assistant/blog/day_100__cursed_clouds.mdwn
@@ -0,0 +1,19 @@
+Preferred content control is wired up to `--auto` and working for `get`,
+`copy`, and `drop`. Note that `drop --from remote --auto` drops files that
+the remote's preferred content settings indicate it doesn't want;
+likewise `copy --to remote --auto` sends content that the remote does want.
+
+Also implemented `smallerthan`, `largerthan`, and `ingroup` limits,
+which should be everything needed for the scenarios described in
+[[transfer_control]].
+
+Dying to hook this up to the assistant, but a cloudy day is forcing me to
+curtail further computer use.
+
+----
+
+Also, last night I developed a patch for the hS3 library, that should let
+git-annex upload large files to S3 without buffering their whole content in
+memory. I have a `s3-memory-leak` in git-annex that uses the new API I
+developed. Hopefully hS3's maintainer will release a new version with that
+soon.
diff --git a/doc/design/assistant/blog/day_102__very_high_level_programming.mdwn b/doc/design/assistant/blog/day_102__very_high_level_programming.mdwn
new file mode 100644
index 0000000000..4e29cc65df
--- /dev/null
+++ b/doc/design/assistant/blog/day_102__very_high_level_programming.mdwn
@@ -0,0 +1,37 @@
+## today
+
+Came up with four groups of repositories that it makes sense to
+define standard preferred content expressions for.
+
+[[!format haskell """
+	preferredContent :: StandardGroup -> String
+	preferredContent ClientGroup = "exclude=*/archive/*"
+	preferredContent TransferGroup = "not inallgroup=client and " ++ preferredContent ClientGroup
+	preferredContent ArchiveGroup = "not copies=archive:1"
+	preferredContent BackupGroup = "" -- all content is preferred
+"""]]
+
+[[preferred_content]] has the details about these groups, but
+as I was writing those three preferred content expressions,
+I realized they are some of the highest level programming I've ever done,
+in a way.
+
+Anyway, these make for a very simple repository configuration UI:
+
+[[!img /assistant/repogroups.png alt="form with simple select box"]]
+
+## yesterday (forgot to post this)
+
+Got the assistant honoring preferred content settings. Although so far that
+only determines what it transfers. Additional work will be needed to make
+content be dropped when it stops being preferred.
+
+----
+
+Added a "configure" link next to each repository on the repository config
+page. This will go to a form to allow setting things like repository
+descriptions, groups, and preferred content settings.
+
+----
+
+Cut a release.
diff --git a/doc/design/assistant/blog/day_102__very_high_level_programming/comment_1_c028b403261dd66bcf83e6ffd134b80b._comment b/doc/design/assistant/blog/day_102__very_high_level_programming/comment_1_c028b403261dd66bcf83e6ffd134b80b._comment
new file mode 100644
index 0000000000..d9be239bd5
--- /dev/null
+++ b/doc/design/assistant/blog/day_102__very_high_level_programming/comment_1_c028b403261dd66bcf83e6ffd134b80b._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="https://www.google.com/accounts/o8/id?id=AItOawlVwbp82f-7rNOyEpvqc9A8FEfn7wl2Akw"
+ nickname="Jos"
+ subject="Works for me"
+ date="2012-10-11T13:40:08Z"
+ content="""
+These categories make it much clearer to me how I will be able to use git annex assistant
+"""]]
diff --git a/doc/design/assistant/blog/day_103__bugfix_day.mdwn b/doc/design/assistant/blog/day_103__bugfix_day.mdwn
new file mode 100644
index 0000000000..7c08a2065e
--- /dev/null
+++ b/doc/design/assistant/blog/day_103__bugfix_day.mdwn
@@ -0,0 +1,25 @@
+Bugfixes all day.
+
+The most amusing bug, which I just stumbled over randomly on my own,
+after someone on IRC yesterday was possibly encountering the same issue,
+made `git annex webapp` go into an infinite memory-consuming loop on
+startup if the repository it had been using was no longer a valid git
+repository.
+
+Then there was the place where HOME got unset, with also sometimes amusing
+results.
+
+Also fixed several build problems, including a threaded runtime hang
+in the test suite. Hopefully the next release will build on all Debian
+architectures again.
+
+I'll be cutting that release tomorrow. I also updated the linux
+prebuilt tarballs today.
+
+----
+
+Hmm, not entirely bugfixes after all. Had time (and power) to work
+on the repository configuration form too, and added a check box to it that
+can be unchecked to disable syncing with a repository. 
+Also, made that form be displayed after the webapp creates a new
+repository.
diff --git a/doc/design/assistant/blog/day_104__misc.mdwn b/doc/design/assistant/blog/day_104__misc.mdwn
new file mode 100644
index 0000000000..bda802bfe5
--- /dev/null
+++ b/doc/design/assistant/blog/day_104__misc.mdwn
@@ -0,0 +1,18 @@
+Switched the OSX standalone app to use `DYLD_ROOT_PATH`.
+This is the third `DYLD_*` variable I've tried; neither
+of the other two worked in all situations. This one may do better.
+If not, I may be stuck modifying the library names in each executable
+using `install_name_tool`
+([good reference for doing that](http://www.mikeash.com/pyblog/friday-qa-2009-11-06-linking-and-install-names.html)).
+As far as I know, every existing dynamic library lookup system is broken
+in some way other other; nothing I've seen about OSX's so far
+disproves that rule.
+
+Fixed a nasty utf-8 encoding crash that could occur when merging the
+git-annex branch. I hope I'm almost done with those.
+
+Made git-annex auto-detect when a git remote is on a sever like github
+that doesn't support git-annex, and automatically set annex-ignore.
+
+Finished the UI for pausing syncing of a remote. Making the syncing
+actually stop still has some glitches to resolve.
diff --git a/doc/design/assistant/blog/day_104__misc/comment_1_13d7fad2d3f8eab10314784c035e2a16._comment b/doc/design/assistant/blog/day_104__misc/comment_1_13d7fad2d3f8eab10314784c035e2a16._comment
new file mode 100644
index 0000000000..67e342a4ba
--- /dev/null
+++ b/doc/design/assistant/blog/day_104__misc/comment_1_13d7fad2d3f8eab10314784c035e2a16._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="https://www.google.com/accounts/o8/id?id=AItOawmubB1Sj2rwFoVdZYvGV0ACaQUJQyiJXJI"
+ nickname="Paul"
+ subject="Thank you for stickers"
+ date="2012-10-14T13:59:55Z"
+ content="""
+Mail received in Finland.
+"""]]
diff --git a/doc/design/assistant/blog/day_105__lazy_Sunday.mdwn b/doc/design/assistant/blog/day_105__lazy_Sunday.mdwn
new file mode 100644
index 0000000000..eb25ddacef
--- /dev/null
+++ b/doc/design/assistant/blog/day_105__lazy_Sunday.mdwn
@@ -0,0 +1,43 @@
+Did a fair amount of testing and bug fixing today. 
+
+There is still some buggy behavior around pausing syncing to a remote,
+where transfers still happen to it, but I fixed the worst bug there.
+
+Noticed that if a non-bare repo is set up on a removable drive,
+its file tree will not normally be updated as syncs come in -- because the
+assistant is not running on that repo, and so incoming syncs are not
+merged into the local master branch. For now I made it always use bare
+repos on removable drives, but I may want to revisit this.
+
+The repository edit form now has a field for the name of the repo,
+so the ugly names that the assistant comes up with for ssh remotes
+can be edited as you like. `git remote rename` is a very nice thing.
+
+Changed the preferred content expression for transfer repos to this:
+"not (inallgroup=client **and copies=client:2)**". This way, when there's
+just one client, files on it will be synced to transfer repos, even
+though those repos have no other clients to transfer them to. Presumably,
+if a transfer repo is set up, more clients are coming soon, so this avoids
+a wait. Particularly useful with removable drives, as the drive will start
+being filled as soon as it's added, and can then be brought to a client
+elsewhere. The "2" does mean that, once another client is found,
+the data on the transfer repo will be dropped, and so if it's brought
+to yet another new client, it won't have data for it right away.
+I can't see way to generalize this workaround to more than 2 clients;
+the transfer repo has to start dropping apparently unwanted content at
+some point. Still, this will avoid a potentially very confusing behavior
+when getting started.
+
+----
+
+I need to get that dropping of non-preferred content to happen still.
+Yesterday, I did some analysis of all the events that can cause previously
+preferred content to no longer be preferred, so I know all the places
+I have to deal with this.
+
+The one that's giving me some trouble is checking in the transfer scan. If it
+checks for content to drop at the same time as content to transfer, it could
+end up doing a lot of transfers before dropping anything. It'd be nicer to
+first drop as much as it can, before getting more data, so that transfer
+remotes stay as small as possible. But the scan is expensive, and it'd also
+be nice not to need two passes.
diff --git a/doc/design/assistant/blog/day_106__lazy_Monday.mdwn b/doc/design/assistant/blog/day_106__lazy_Monday.mdwn
new file mode 100644
index 0000000000..84b96f6350
--- /dev/null
+++ b/doc/design/assistant/blog/day_106__lazy_Monday.mdwn
@@ -0,0 +1,10 @@
+I was mostly working on other things today, but I did do some bug fixing.
+The worst of these is a bug introduced in 3.20121009 that breaks
+`git-annex-shell configlist`. That's pretty bad for using git-annex
+on servers, although you mostly won't notice unless you're just getting
+started using a ssh remote, since that's when it calls configlist.
+I will be releasing a new version as soon as I have bandwidth (tomorrow).
+
+Also made the standalone Linux and OSX binaries build with ssh connection
+caching disabled, since they don't bundle their own ssh and need to work
+with whatever ssh is installed.
diff --git a/doc/design/assistant/blog/day_107__memory_leak.mdwn b/doc/design/assistant/blog/day_107__memory_leak.mdwn
new file mode 100644
index 0000000000..e33008f97e
--- /dev/null
+++ b/doc/design/assistant/blog/day_107__memory_leak.mdwn
@@ -0,0 +1,11 @@
+More bugfixes today. The assistant now seems to have enough users that
+they're turning up interesting bugs, which is good. But does keep me too
+busy to add many more bugs\^Wcode.
+
+The fun one today made it bloat to eat all memory when logging out of a
+Linux desktop. I tracked that back to a bug in the Haskell DBUS library
+when a session connection is open and the session goes away. Developed a
+test case, and even profiled it, and sent it all of to the library's
+author. Hopefully there will be a quick fix, in the meantime today's
+release has DBUS turned off. Which is ok, it just makes it a little bit
+slower to notice some events.
diff --git a/doc/design/assistant/blog/day_108__another_zombie_outbreak.mdwn b/doc/design/assistant/blog/day_108__another_zombie_outbreak.mdwn
new file mode 100644
index 0000000000..ee46073bd2
--- /dev/null
+++ b/doc/design/assistant/blog/day_108__another_zombie_outbreak.mdwn
@@ -0,0 +1,33 @@
+I released git-annex an unprecidented two times yesterday, because just
+after the first release, I learned of a another zombie problem. Turns out
+this zombie had existed for a while, but it was masked by zombie reaping
+code that I removed recently, after fixing most of the other zombie
+problems. This one, though, is not directly caused by git-annex. When rsync
+runs ssh, it seems to run two copies, and one is left unwaited on as a
+zombie. Oddly, this only happens when rsync's stdout is piped into
+git-annex, for progress bar handling. I have not source-dived rsync's code
+to get to the bottom of this, but I put in a workaround.
+
+I did get to the bottom of yesterday's runaway dbus library. Got lucky and
+found the cause of the memory leak in that library on the first try, which
+is nice since each try involved logging out of X. I've been corresponding
+with its author, and a fix will be available soon, and then git-annex will
+need some changes to handle dbus reconnection.
+
+-----
+
+For the first time, I'm starting to use the assistant on my own personal
+git-annex repo. The preferred content and group settings let me configure it
+use the complex system of partial syncing I need. For example, I have this
+configured for my sound files, keeping new podcasts on a server until they land
+somewhere near me. And keeping any sound files that I've manually put on my
+laptop, and syncing new podcasts, but not other stuff.
+
+	# (for my server)
+	preferred-content 87e06c7a-7388-11e0-ba07-03cdf300bd87 = include=podcasts/* and (not copies=nearjoey:1)
+	# (for my laptop)
+	preferred-content 0c443de8-e644-11df-acbf-f7cd7ca6210d = exclude=*/out/* and (in=here or (include=podcasts/*))
+
+Found and fixed a bug in the preferred content matching code, where
+if the assistant was run in a subdirectory of the repo, it failed to
+match files correctly.
diff --git a/doc/design/assistant/blog/day_108__another_zombie_outbreak/comment_1_194c48d65993462f809a2cfaa774a3e2._comment b/doc/design/assistant/blog/day_108__another_zombie_outbreak/comment_1_194c48d65993462f809a2cfaa774a3e2._comment
new file mode 100644
index 0000000000..690d59d17c
--- /dev/null
+++ b/doc/design/assistant/blog/day_108__another_zombie_outbreak/comment_1_194c48d65993462f809a2cfaa774a3e2._comment
@@ -0,0 +1,11 @@
+[[!comment format=mdwn
+ username="http://a-or-b.myopenid.com/"
+ ip="203.45.2.230"
+ subject="comment 1"
+ date="2012-10-19T01:21:42Z"
+ content="""
+Can you elaborate on the \"nearjoey\" option below?
+> (not copies=nearjoey:1)
+
+Are you able to assign a \"nearjoey\" flag to multiple annexes?
+"""]]
diff --git a/doc/design/assistant/blog/day_108__another_zombie_outbreak/comment_2_ef5ee5933fcadcb81cc81b816db14bda._comment b/doc/design/assistant/blog/day_108__another_zombie_outbreak/comment_2_ef5ee5933fcadcb81cc81b816db14bda._comment
new file mode 100644
index 0000000000..0da13b7351
--- /dev/null
+++ b/doc/design/assistant/blog/day_108__another_zombie_outbreak/comment_2_ef5ee5933fcadcb81cc81b816db14bda._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="http://joeyh.name/"
+ ip="4.154.0.23"
+ subject="comment 2"
+ date="2012-10-19T17:07:08Z"
+ content="""
+Yes, using the group command.
+"""]]
diff --git a/doc/design/assistant/blog/day_109__dropping.mdwn b/doc/design/assistant/blog/day_109__dropping.mdwn
new file mode 100644
index 0000000000..210f71f553
--- /dev/null
+++ b/doc/design/assistant/blog/day_109__dropping.mdwn
@@ -0,0 +1,16 @@
+Got unwanted content to be dropped from the local repo, as well as remotes
+when doing the expensive scan. I kept the scan a single pass for now,
+need to revisit that later to drop content before transferring more.
+Also, when content is downloaded or uploaded, this can result in it
+needing to be dropped from somewhere, and the assistant handles that too.
+
+There are some edge cases with hypothetical, very weird preferred
+content expressions, where the assistant won't drop content right away.
+(But will later in the expensive scan.) Other than those, I think I have
+nearly all content dropping sorted out. The only common case I know of where
+unwanted content is not dropped by the assistant right away is when a file
+is renamed (eg, put in a "Trash" directory).
+
+In other words, repositories put into the transfer group will now work as
+described, only retaining content as long as is needed to distribute it to
+clients. Big milestone!
diff --git a/doc/design/assistant/blog/day_10__lsof.mdwn b/doc/design/assistant/blog/day_10__lsof.mdwn
new file mode 100644
index 0000000000..d4217677f0
--- /dev/null
+++ b/doc/design/assistant/blog/day_10__lsof.mdwn
@@ -0,0 +1,54 @@
+A rather frustrating and long day coding went like this:
+
+## 1-3 pm
+
+Wrote a single function, of which all any Haskell programmer needs to know
+is its type signature:
+
+	Lsof.queryDir :: FilePath -> IO [(FilePath, LsofOpenMode, ProcessInfo)]
+
+When I'm spending another hour or two taking a unix utility like lsof and
+parsing its output, which in this case is in a rather complicated
+machine-parsable output format, I often wish unix streams were strongly
+typed, which would avoid this bother.
+
+## 3-9 pm
+
+Six hours spent making it defer annexing files until the commit thread
+wakes up and is about to make a commit. Why did it take so horribly long?
+Well, there were a number of complications, and some really bad bugs
+involving races that were hard to reproduce reliably enough to deal with.
+
+In other words, I was lost in the weeds for a lot of those hours...
+
+At one point, something glorious happened, and it was always making exactly
+one commit for batch mode modifications of a lot of files (like untarring
+them). Unfortunately, I had to lose that gloriousness due to another
+potential race, which, while unlikely, would have made the program deadlock
+if it happened. 
+
+So, it's back to making 2 or 3 commits per batch mode change. I also have a
+buglet that causes sometimes a second empty commit after a file is added.
+I know why (the inotify event for the symlink gets in late,
+after the commit); will try to improve commit frequency later.
+
+## 9-11 pm
+
+Put the capstone on the day's work, by calling lsof on a directory full
+of hardlinks to the files that are about to be annexed, to check if any
+are still open for write.
+
+This works great! Starting up `git annex watch` when processes have files
+open is no longer a problem, and even if you're evil enough to try having
+multiple processes open the same file, it will complain and not annex it
+until all the writers close it.
+
+(Well, someone really evil could turn the write bit back on after git annex
+clears it, and open the file again, but then really evil people can do
+that to files in `.git/annex/objects` too, and they'll get their just
+deserts when `git annex fsck` runs. So, that's ok..)
+
+----
+
+Anyway, will beat on it more tomorrow, and if all is well, this will finally
+go out to the beta testers.
diff --git a/doc/design/assistant/blog/day_10__lsof/comment_1_9b8c28c85c979f32e5c295b6a03c048e._comment b/doc/design/assistant/blog/day_10__lsof/comment_1_9b8c28c85c979f32e5c295b6a03c048e._comment
new file mode 100644
index 0000000000..9d970da22e
--- /dev/null
+++ b/doc/design/assistant/blog/day_10__lsof/comment_1_9b8c28c85c979f32e5c295b6a03c048e._comment
@@ -0,0 +1,9 @@
+[[!comment format=mdwn
+ username="http://dieter-be.myopenid.com/"
+ nickname="dieter"
+ subject="comment 1"
+ date="2012-06-16T09:14:26Z"
+ content="""
+maybe at some point, your tool could show \"warning, the following files are still open and are hence not being annexed\"
+to avoid any nasty surprises of a file not being annexed and the user not realizing it.
+"""]]
diff --git a/doc/design/assistant/blog/day_110__more_dropping.mdwn b/doc/design/assistant/blog/day_110__more_dropping.mdwn
new file mode 100644
index 0000000000..71fb01ab12
--- /dev/null
+++ b/doc/design/assistant/blog/day_110__more_dropping.mdwn
@@ -0,0 +1,55 @@
+Got preferred content checked when files are moved around.
+So, in repositories in the default client group, if you make a
+"archive" directory and move files to it, the assistant will drop
+their content (when possible, ie when it's reached an archive or backup).
+Move a file out of an archive directory, and the assistant will get its
+content again. Magic.
+
+Found an intractable bug, obvious in retrospect, with the git-annex branch
+read cache, and had to remove that cache. I have not fully determined
+if this will slow down git-annex in some use cases; might need to add more
+higher-level caching. It was a very minimal cache anyway, just of one file.
+
+Removed support for "in=" from preferred content expressions. That was
+problematic in two ways. First, it referred to a remote by name, but
+preferred content expressions can be evaluated elsewhere, where that remote
+doesn't exist, or a different remote has the same name. This name lookup
+code could error out at runtime. Secondly, "in=" seemed pretty useless, and
+indeed counterintuitive in preferred content expressions. "in=here" did not
+cause content to be gotten, but it did let present content be dropped.
+Other uses of "in=" are better handled by using groups.
+
+In place of "in=here", preferred content expressions can now use "present",
+which is useful if you want to disable automatic getting or dropping of
+content in some part of a repository. Had to document that "not present"
+is not a good thing to use -- it's unstable. Still, I find "present" handy
+enough to put up with that wart.
+
+Realized last night that the code I added to the TransferWatcher
+to check preferred content once a transfer is done is subject to a race;
+it will often run before the location log gets updated. Haven't found a good
+solution yet, but this is something I want working now, so I did put in a
+quick delay hack to avoid the race. Delays to avoid races are never a real
+solution, but sometimes you have to TODO it for later.
+
+----
+
+Been thinking about how to make the assistant notice changes to configuration
+in the git-annex branch that are merged in from elsewhere while it's running.
+I'd like to avoid re-reading unchanged configuration files after each merge
+of the branch.
+
+The most efficient way would be to reorganise the git-annex branch, moving
+config files into a configs directory, and logs into a logs directory. Then it
+could `git ls-tree git-annex configs` and check if the sha of the configs
+directory had changed, with git doing minimal work 
+(benchmarked at 0.011 seconds).
+
+Less efficiently, keep the current git-annex branch layout, and 
+use: `git ls-tree git-annex uuid.log remote.log preferred-content.log group.log trust.log`
+(benchmarked at 0.015 seconds)
+
+Leaning toward the less efficient option, with a rate limiter so it
+doesn't try more often than once every minute. Seems reasonable for it to
+take a minute for config changes take effect on remote repos, even
+if the assistant syncs file changes to them more quickly.
diff --git a/doc/design/assistant/blog/day_111__config_monitor.mdwn b/doc/design/assistant/blog/day_111__config_monitor.mdwn
new file mode 100644
index 0000000000..8addf54ae8
--- /dev/null
+++ b/doc/design/assistant/blog/day_111__config_monitor.mdwn
@@ -0,0 +1,18 @@
+Added yet another thread, the ConfigMonitor. Since that thread needs to run
+code to reload cached config values from the git-annex branch when files
+there change, writing it also let me review where config files are cached,
+and I found that every single config file in the git-annex branch does
+get cached, with the exception of the uuid.log. So, added a cache for that,
+and now I'm more sanguine about yesterday's removal of the lower-level
+cache, because the only thing not being cached is location log information.
+
+The ConfigMonitor thread seems to work, though I have not tested it
+extensively. The assistant should notice and apply config changes
+made locally, as well as any config changes pushed in from remotes.
+So, for example, if you add a S3 repo in the webapp, and are paired with
+another computer, that one's webapp will shortly include the new repo in
+its list. And all the preferred content, groups, etc settings will
+propigate over and be used as well.
+
+Well ... almost. Seems nothing causes git-annex branch changes to be
+pushed, until there's some file change to sync out.
diff --git a/doc/design/assistant/blog/day_112__and_now_for_something_completely_different.mdwn b/doc/design/assistant/blog/day_112__and_now_for_something_completely_different.mdwn
new file mode 100644
index 0000000000..b7d72a2759
--- /dev/null
+++ b/doc/design/assistant/blog/day_112__and_now_for_something_completely_different.mdwn
@@ -0,0 +1,50 @@
+Time to solve the assistant's [[cloud]] notification problem. This is
+really the last big road-bump to making it be able to sync computers
+across the big, bad internet.
+
+So, IRC still seems a possibility, but I'm going to try XMPP first. Since
+Google Talk uses XMPP, it should be easy for users to get an account, and
+it's also easy to run your own XMPP server.
+
+Played around with the Haskell XMPP library. Clint helpfully showed me an
+example of a simple client, which helped cut through that library's thicket
+of data types. In short order I had some clients that were able to see each
+other connecting to my personal XMPP server. On to some design..
+
+1. I want to avoid user-visible messages.
+   (dvcs-autosync also uses XMPP, but I checked the code and it
+   seems to send user-visible messages, so I diverge from its lead here.)
+   This seems very possible, only a matter of finding the right
+   way to use the protocol, or an appropriate and widely deployed extension.
+   The only message I need to send with XMPP, really, is "I have pushed to our
+   git repo". One bit of data would do; being able to send a UUID of the repo
+   that did the update would be nice.
+
+2. I'd also like to broadcast my notification to a user's buddies.
+   dvcs-autosync sends only self-messages, but that requires every node
+   have the same XMPP account configured. While I want to be able to run
+   in that mode, I also want to support pairs of users who have their own XMPP
+   accounts, that are buddied up in XMPP.
+
+3. To add to the fun, the assistant's use of XMPP should not make that XMPP
+   account appear active to its buddies. Users should not need a dedicated
+   XMPP account for git-annex, and always seeming to be available when
+   git-annex is running would not be nice.
+
+The first method I'm trying out is to encode the notification
+data inside a XMPP presence broadcast. This should meet all three
+requirements. The plan is two send two
+presence messages, the first marks the client as available, and the second
+as unavailable again.
+The "id" attribute will be set to some
+value generated by the assistant. That attribute is allowed on presence
+messages, and servers are [required to preserve it](http://xmpp.org/rfcs/rfc6121.html#presence-probe-inbound-id)
+while the client is connected.
+(I'd only send unavailable messages, but while
+that worked when I tested it using the prosody server, with google talk,
+repeated unavailable messages were suppressed. Also, google talk does not
+preserve the "id" attribute of unavailable presence messages.)
+
+If this presence hackery doesn't work out, I could try
+[XEP-0163: Personal Eventing Protocol](http://xmpp.org/extensions/xep-0163.html).
+But I like not relying on any extensions.
diff --git a/doc/design/assistant/blog/day_112__and_now_for_something_completely_different/comment_1_5e4fe1538d9ae1c450b0a6602fc6d29b._comment b/doc/design/assistant/blog/day_112__and_now_for_something_completely_different/comment_1_5e4fe1538d9ae1c450b0a6602fc6d29b._comment
new file mode 100644
index 0000000000..26514601b5
--- /dev/null
+++ b/doc/design/assistant/blog/day_112__and_now_for_something_completely_different/comment_1_5e4fe1538d9ae1c450b0a6602fc6d29b._comment
@@ -0,0 +1,10 @@
+[[!comment format=mdwn
+ username="https://www.google.com/accounts/o8/id?id=AItOawmQ4Oe5-qOANRuZel9kDvtBfQG1zlEcIzw"
+ nickname="Dominik"
+ subject="PEP is ok"
+ date="2012-10-24T05:15:38Z"
+ content="""
+PEP and pubsub are really widely used. You will have a hard time finding a server that doesn't support it.
+
+Also note XMPPs ability to send status to only selected buddies and even resources. You can globally appear as offline while sending availability to myownjid@example.com/thatannexthere.
+"""]]
diff --git a/doc/design/assistant/blog/day_112__and_now_for_something_completely_different/comment_2_c5a734f611ecc95729904e645583ee43._comment b/doc/design/assistant/blog/day_112__and_now_for_something_completely_different/comment_2_c5a734f611ecc95729904e645583ee43._comment
new file mode 100644
index 0000000000..2b85707a91
--- /dev/null
+++ b/doc/design/assistant/blog/day_112__and_now_for_something_completely_different/comment_2_c5a734f611ecc95729904e645583ee43._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="https://www.google.com/accounts/o8/id?id=AItOawmQ4Oe5-qOANRuZel9kDvtBfQG1zlEcIzw"
+ nickname="Dominik"
+ subject="Remote control clients"
+ date="2012-10-24T05:18:02Z"
+ content="""
+Then of course there is XEP-0146, which has some advantages to it, like controlling annex from the context menu of another Jabber client. 
+"""]]
diff --git a/doc/design/assistant/blog/day_112__and_now_for_something_completely_different/comment_3_46b16dcd0fce07036cd8ed6ed9d2b055._comment b/doc/design/assistant/blog/day_112__and_now_for_something_completely_different/comment_3_46b16dcd0fce07036cd8ed6ed9d2b055._comment
new file mode 100644
index 0000000000..5c6c161ffc
--- /dev/null
+++ b/doc/design/assistant/blog/day_112__and_now_for_something_completely_different/comment_3_46b16dcd0fce07036cd8ed6ed9d2b055._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="http://joeyh.name/"
+ ip="4.154.0.23"
+ subject="comment 3"
+ date="2012-10-24T22:44:41Z"
+ content="""
+Google talk does not seem to support PEP. At least, I'm sending something that prosody accepts and that looks like the example at , and it replies with an IQ Error.
+"""]]
diff --git a/doc/design/assistant/blog/day_112__and_now_for_something_completely_different/comment_4_1fe036e4c65fb4211aa2c394f535344a._comment b/doc/design/assistant/blog/day_112__and_now_for_something_completely_different/comment_4_1fe036e4c65fb4211aa2c394f535344a._comment
new file mode 100644
index 0000000000..d19b70189d
--- /dev/null
+++ b/doc/design/assistant/blog/day_112__and_now_for_something_completely_different/comment_4_1fe036e4c65fb4211aa2c394f535344a._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="https://www.google.com/accounts/o8/id?id=AItOawmaM3-vbAh5B8tY_IL3yh31ik6cAVsgoPw"
+ nickname="IC"
+ subject="Haskell XMPP"
+ date="2012-10-27T05:54:53Z"
+ content="""
+Which xmpp library you've decided to use?
+"""]]
diff --git a/doc/design/assistant/blog/day_112__and_now_for_something_completely_different/comment_5_e4ba3568c4efd98f212dd47427a1cf47._comment b/doc/design/assistant/blog/day_112__and_now_for_something_completely_different/comment_5_e4ba3568c4efd98f212dd47427a1cf47._comment
new file mode 100644
index 0000000000..272970caa2
--- /dev/null
+++ b/doc/design/assistant/blog/day_112__and_now_for_something_completely_different/comment_5_e4ba3568c4efd98f212dd47427a1cf47._comment
@@ -0,0 +1,10 @@
+[[!comment format=mdwn
+ username="http://joeyh.name/"
+ ip="2001:4978:f:21a::2"
+ subject="comment 5"
+ date="2012-10-27T06:08:44Z"
+ content="""
+I'm using http://hackage.haskell.org/package/network-protocol-xmpp
+
+While its author thinks \"its current design (session in an XMPP monad) is a failed mistake\", I've found it pretty good. Aside from the segfault problem, which we got fixed today.
+"""]]
diff --git a/doc/design/assistant/blog/day_113__notifier_work.mdwn b/doc/design/assistant/blog/day_113__notifier_work.mdwn
new file mode 100644
index 0000000000..920b48dfeb
--- /dev/null
+++ b/doc/design/assistant/blog/day_113__notifier_work.mdwn
@@ -0,0 +1,22 @@
+Built out the XMPP push notifier; around 200 lines of code.
+Haven't tested it yet, but it just might work. It's in the `xmpp` branch
+for now.
+
+I decided to send the UUID of the repo that was pushed to, otherwise
+peers would have to speculatively pull from every repo. A wrinkle in this
+is that not all git repos have a git-annex UUID. So it might notify that
+a push was sent to an unidentified repo, and then peers need to pull from
+every such repo. In the common case, there will only be one or a few such
+repos, at someplace like at github that doesn't support git-annex. I could
+send the URL, but there's no guarantee different clients have the same
+URLs for a git remote, and also sending the URL leaks rather more data than
+does a random UUID.
+
+Had a bit of a scare where it looked like I couldn't use the haskell
+`network-protocol-xmpp` package together with the `mtl` package that
+git-annex already depends on. With help from #haskell I found the way
+to get them co-existing, by using the PackageImports extension. Whew!
+
+Need to add configuration of the XMPP server to use in the webapp, and
+perhaps also a way to create `.git/annex/creds/notify-xmpp` from the
+command line.
diff --git a/doc/design/assistant/blog/day_114__xmpp.mdwn b/doc/design/assistant/blog/day_114__xmpp.mdwn
new file mode 100644
index 0000000000..617824d48e
--- /dev/null
+++ b/doc/design/assistant/blog/day_114__xmpp.mdwn
@@ -0,0 +1,56 @@
+Had to toss out my XMPP presence hack. Turns out that, at least in Google
+Talk, presence info is not sent to clients that have marked themselves
+unavailable, and that means the assistant would not see notifications, as it
+was nearly always marked unavailable as part of the hack.
+
+I tried writing a test program that uses XMPP personal eventing, only
+to find that Google Talk rejected my messages. I'm not 100% sure my
+messages were right, but I was directly copying the example in the RFC,
+and prosody accepted them. I could not seem to get a list of extensions out
+of Google Talk either, so I don't know if it doesn't support personal
+eventing, or perhaps only supports certian specific types of events.
+
+So, plan C... using XMPP [presence extended content](http://xmpp.org/rfcs/rfc6121.html#presence-extended).
+The assistant generates a presence message tagged "xa" (Extended Away),
+which hopefully will make it not seem present to clients.
+And to that presence message, I add my own XML element:
+
+	
+
+This is all entirely legal, and not at all a hack.
+(Aside from this not really being presence info.) Isn't XML fun?
+
+And plan C works, with Google Talk, and prosody. I've successfully gotten
+push notifications flowing over XMPP!
+
+----
+
+Spent some hours dealing with an unusual probolem: git-annex started
+segfaulting intermittently on startup with the new XMPP code. 
+
+Haskell code is not supposed to segfault..
+
+I think this was probably due to not using a bound thread for XMPP,
+so if haskell's runtime system recheduled its green thread onto a different
+OS thread during startup, when it's setting up TLS, it'd make gnuTLS very
+unhappy.
+
+So, fixed it to use a bound thread. Will wait and see if the crash is gone.
+
+----
+
+Re-enabled DBUS support, using a new version of the library that avoids the
+memory leak. Will need further changes to the library to support
+reconnecting to dbus.
+
+----
+
+Next will be a webapp configuration UI for XMPP. Various parts of the
+webapp will direct the user to set up XMPP, when appropriate, especially
+when the user sets up a cloud remote.
+
+To make XMPP sufficiently easy to configure, I need to check SRV records to
+find the XMPP server, which is an unexpected PITA because `getaddrinfo`
+can't do that. There are several haskell DNS libraries that I could use for
+SRV, or I could use the `host` command: 
+`host -t SRV _xmpp-client._tcp.gmail.com`
diff --git a/doc/design/assistant/blog/day_114__xmpp/comment_1_c2b0617a2fc3dc4f19a6be6947913842._comment b/doc/design/assistant/blog/day_114__xmpp/comment_1_c2b0617a2fc3dc4f19a6be6947913842._comment
new file mode 100644
index 0000000000..a391435c4d
--- /dev/null
+++ b/doc/design/assistant/blog/day_114__xmpp/comment_1_c2b0617a2fc3dc4f19a6be6947913842._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="http://phil.0x539.de/"
+ nickname="Philipp Kern"
+ subject="How to deal with offline messages being eaten?"
+ date="2012-11-09T14:05:30Z"
+ content="""
+How do you avoid eating up normal text messages when git-annex is the only client left? Granted, Google Talk does some additional magic so that it shows you the history when connecting with the web page client. But with other servers it would be inconvenient if new messages weren't saved as offline messages but sent to the dæmon instead.
+"""]]
diff --git a/doc/design/assistant/blog/day_114__xmpp/comment_2_d14375dfb5791615802dab3c5438f8e2._comment b/doc/design/assistant/blog/day_114__xmpp/comment_2_d14375dfb5791615802dab3c5438f8e2._comment
new file mode 100644
index 0000000000..716f7d99ad
--- /dev/null
+++ b/doc/design/assistant/blog/day_114__xmpp/comment_2_d14375dfb5791615802dab3c5438f8e2._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="http://joeyh.name/"
+ ip="4.152.108.27"
+ subject="comment 2"
+ date="2012-11-09T21:55:42Z"
+ content="""
+It does try to mark itself as extended away, but yes, I think this is a potential concern. If you find the assistant interacts badly with your other clients, you can certainly give it its own XMPP account.
+"""]]
diff --git a/doc/design/assistant/blog/day_114__xmpp/comment_3_6d72ea32c111e605be30ad2153fc71c9._comment b/doc/design/assistant/blog/day_114__xmpp/comment_3_6d72ea32c111e605be30ad2153fc71c9._comment
new file mode 100644
index 0000000000..6d2147d375
--- /dev/null
+++ b/doc/design/assistant/blog/day_114__xmpp/comment_3_6d72ea32c111e605be30ad2153fc71c9._comment
@@ -0,0 +1,10 @@
+[[!comment format=mdwn
+ username="http://phil.0x539.de/"
+ nickname="Philipp Kern"
+ subject="comment 3"
+ date="2012-11-10T18:50:46Z"
+ content="""
+Sure, I could, as I'm operating my own server anyway. Others might not be willing to go to some random server and create another account, though.
+
+Reviewing RFC 6121: Did you try negative priorities for your resources already? It's possible that Gtalk does something weird but in theory they should be ignored for messages directed to non-qualified JIDs (i.e. without an explicit resource setting). Setting \"xa\" alone won't help you anything, it will only cause the others to see you as \"xa\" but that's a perfectly valid common chat status where you expect messages to be queued in the client until the user returns.
+"""]]
diff --git a/doc/design/assistant/blog/day_114__xmpp/comment_4_e51d6f854db5f9e74a1aa58bd8923795._comment b/doc/design/assistant/blog/day_114__xmpp/comment_4_e51d6f854db5f9e74a1aa58bd8923795._comment
new file mode 100644
index 0000000000..7320d03654
--- /dev/null
+++ b/doc/design/assistant/blog/day_114__xmpp/comment_4_e51d6f854db5f9e74a1aa58bd8923795._comment
@@ -0,0 +1,12 @@
+[[!comment format=mdwn
+ username="http://joeyh.name/"
+ ip="4.152.108.27"
+ subject="comment 4"
+ date="2012-11-10T19:48:01Z"
+ content="""
+I hadn't tried it, but it looks like a very good idea. I will have to change my protocol a little bit as I currently sometimes send chat messages not directed to a specific client, in order to reach any clients using an account. To reach clients with a negative priority, chat messages have to be directed at the client.
+
+Ok, done; priority is set to -1 and all the XMPP messaging in git-annex still seems to work!
+
+I don't know though, that this entirely solves the problem. In particular, a regular xmpp client may decide to direct a chat message at a git-annex client, due to that being the only client connected. If this does become a problem, I suppose git-annex could buffer and re-send such messages when it sees a regular client connect. Or it could show them in the webapp, but that seems feature creep.
+"""]]
diff --git a/doc/design/assistant/blog/day_115__my_new_form.mdwn b/doc/design/assistant/blog/day_115__my_new_form.mdwn
new file mode 100644
index 0000000000..d44565084a
--- /dev/null
+++ b/doc/design/assistant/blog/day_115__my_new_form.mdwn
@@ -0,0 +1,17 @@
+Built a SRV lookup library that can use either `host` or ADNS.
+
+Worked on DBUS reconnection some more; found a FD leak in the dbus library,
+and wrote its long-suffering author, John Millikin (also the XMPP library
+author, so I've been bothering him a lot lately), who once again came
+through with a quick fix.
+
+Built a XMPP configuration form, that tests the connection to the server.
+Getting the wording right on this was hard, and it's probably still not
+100% right.
+
+[[!img /assistant/xmpp.png]]
+ 
+Pairing over XMPP is something I'm still thinking about. It's
+contingent on tunneling git over XMPP (actually not too hard),
+and getting a really secure XMPP connection (needs library improvements,
+as the library currently accepts any SSL certificate).
diff --git a/doc/design/assistant/blog/day_116__the_segfault.mdwn b/doc/design/assistant/blog/day_116__the_segfault.mdwn
new file mode 100644
index 0000000000..47633454a1
--- /dev/null
+++ b/doc/design/assistant/blog/day_116__the_segfault.mdwn
@@ -0,0 +1,25 @@
+Continuing to flail away at this XMPP segfault, which turned out not to be
+fixed by bound threads. I managed to make a fairly self-contained and small
+reproducible test case for it that does not depend on the network.
+Seems the bug is gonna be either in the Haskell binding for GNUTLS,
+or possibly in GNUTLS itself.
+
+Update: John was able to fix it using my testcase! It was a GNUTLS
+credentials object that went out of scope and got garbage collected.
+I think I was seeing the crash only with the threaded runtime because
+it has a separate garbage collection thread.
+
+----
+
+Arranged for the XMPP thread to restart when network connections
+change, as well as when the webapp configures it.
+
+Added an alert to nudge users to enable XMPP. It's displayed after adding a
+remote in the cloud.
+
+[[!img /assistant/xmppnudge.png]]
+
+----
+
+So, the first stage of XMPP is done. But so far all it does is push
+notification. Much more work to do here.
diff --git a/doc/design/assistant/blog/day_117__new_topologies.mdwn b/doc/design/assistant/blog/day_117__new_topologies.mdwn
new file mode 100644
index 0000000000..f62da3c5bb
--- /dev/null
+++ b/doc/design/assistant/blog/day_117__new_topologies.mdwn
@@ -0,0 +1,41 @@
+Back in [[day_85__more_foundation_work]], I wrote:
+
+> I suspect, but have not proven,
+> that the assistant is able to keep repos arranged in any shape of graph in
+> sync, as long as it's connected (of course) and each connection is
+> bi-directional. [And each node is running the assistant.]
+
+After today's work, many more graph topologies can be kept in sync -- the
+assistant now can keep repos in sync that are not directly connected, but
+must go through a central transfer point, which does not run the assistant
+at all. Major milestone!
+
+To get that working, as well as using XMPP push notifications, it turned
+out to need to be more agressive about pushing out changed location log
+information. And, it seems, that was the last piece that was missing.
+Although I narrowly avoided going down a blind alley involving sending
+transfer notifications over XMPP. Luckily, I came to my senses.
+
+----
+
+This month's focus was the cloud, and the month is almost done. And now
+the assistant can, indeed be used to sync over the cloud! I would have
+liked to have gotten on to implementing Amazon Glacier or Google Drive
+support, but at least the cloud fundamentals are there.
+
+Now that I have XMPP support, I'm tending toward going ahead and adding
+XMPP pairing, and git push over XMPP. This will open up lots of excellent
+use cases.
+
+So, how to tunnel git pushes over XMPP? Well, `GIT_SHELL` can be set to
+something that intercepts the output of `git-send-pack` and
+`git-receive-pack`, and that data can be tunneled through XMPP to connect
+them. Probably using XMPP ping.
+(XEP-0047: In-Band Bytestreams would be the right way ...
+but of course Google Talk doesn't support that extension.)
+
+XMPP requires ugly encoding that will bloat the data, but the data
+quantities are fairly small to sync up a few added or moved files
+(of course, we'll not be sending file contents over XMPP). Pairing with
+an large git repository over XMPP will need rather more bandwidth,
+of course.
diff --git a/doc/design/assistant/blog/day_118__monadic_discontinuity.mdwn b/doc/design/assistant/blog/day_118__monadic_discontinuity.mdwn
new file mode 100644
index 0000000000..758b26974a
--- /dev/null
+++ b/doc/design/assistant/blog/day_118__monadic_discontinuity.mdwn
@@ -0,0 +1,15 @@
+Spent most of the past day moving the assistant into a monad of its own
+that encapsulates all the communications channels for its threads. This
+involved modifiying nearly every line of code in the whole assistant.
+
+Typical change:
+
+[[!format haskell """
+handleConnection threadname st dstatus scanremotes pushnotifier = do
+        reconnectRemotes threadname st dstatus scanremotes (Just pushnotifier)
+                =<< networkRemotes st
+
+handleConnection = reconnectRemotes True =<< networkRemotes
+"""]]
+
+So, it's getting more readable..
diff --git a/doc/design/assistant/blog/day_119__time_for_testing.mdwn b/doc/design/assistant/blog/day_119__time_for_testing.mdwn
new file mode 100644
index 0000000000..adf1d0d411
--- /dev/null
+++ b/doc/design/assistant/blog/day_119__time_for_testing.mdwn
@@ -0,0 +1,12 @@
+Finished working the new assistant monad into all the assistant's code.
+I've changed 1870 lines of code in the past two days. It feels like more.
+While the total number of lines of code has gone up by around 100, the
+actual code size has gone *down*; the monad allowed dropping 3.4 kilobytes
+of manual variable threading complications. Or around 1% of a novel edited
+away, in other words.
+
+I don't seem to have broken anything, but I'm started an extensive test
+of all the assistant and webapp. So far, the bugs I've found were not
+introduced by my monadic changes. Fixed several bugs around adding
+removable drives, and a few other minor bugs. Plan to continue testing
+tomorrow.
diff --git a/doc/design/assistant/blog/day_11__freebsd.mdwn b/doc/design/assistant/blog/day_11__freebsd.mdwn
new file mode 100644
index 0000000000..92a3ef2896
--- /dev/null
+++ b/doc/design/assistant/blog/day_11__freebsd.mdwn
@@ -0,0 +1,50 @@
+I've been investigating how to make `git annex watch` work on
+FreeBSD, and by extension, OSX.
+
+One option is kqueue, which works on both operating systems, and allows
+very basic monitoring of file changes. There's also an OSX specific
+hfsevents interface.
+
+Kqueue is far from optimal for `git annex watch`, because it provides even
+less information than inotify (which didn't really provide everything I
+needed, thus the lsof hack). Kqueue doesn't have events for files being
+closed, only an event when a file is created. So it will be difficult for
+`git annex watch` to know when a file is done being written to and can be
+annexed. git annex will probably need to run lsof periodically to check when
+recently added files are complete. (hsevents shares this limitation)
+
+Kqueue also doesn't provide specific events when a file or directory is
+moved. Indeed, it doesn't provide specific events about what changed at
+all. All you get with kqueue is a generic "oh hey, the directory you're
+watching changed in some way", and it's up to you to scan it to work out
+how. So git annex will probably need to run `git ls-tree --others`
+to find changes in the directory tree. This could be expensive with large
+trees. (hsevents has per-file events on current versions of OSX)
+
+Despite these warts, I want to try kqueue first, since it's more portable
+than hfsevents, and will surely be easier for me to develop support for,
+since I don't have direct access to OSX.
+
+So I went to a handy Debian kFreeBSD porter box, and tried some kqueue
+stuff to get a feel for it. I got a python program that does basic
+directory monitoring with kqueue to work, so I know it's usable there.
+
+Next step was getting kqueue working from Haskell. Should be easy, there's
+a Haskell library already. I spent a while trying to get it to work on
+Debian kFreeBSD, but ran into a
+[problem](https://github.com/hesselink/kqueue/issues/1) that could be
+caused by the Debian kFreeBSD being different, or just a bug in the Haskell
+library. I didn't want to spend too long shaving this yak; I might install
+"real" FreeBSD on a spare laptop and try to get it working there instead.
+
+But for now, I've dropped down to C instead, and have a simple C program
+that can monitor a directory with kqueue. Next I'll turn it into a simple
+library, which can easily be linked into my Haskell code. The Haskell code
+will pass it a set of open directory descriptors, and it'll return the
+one that it gets an event on. This is necessary because kqueue doesn't
+recurse into subdirectories on its own.
+
+I've generally had good luck with this approach to adding stuff in Haskell;
+rather than writing a bit-banging and structure packing low level interface
+in Haskell, write it in C, with a simpler interface between C and
+Haskell.
diff --git a/doc/design/assistant/blog/day_120__test_day.mdwn b/doc/design/assistant/blog/day_120__test_day.mdwn
new file mode 100644
index 0000000000..548222989d
--- /dev/null
+++ b/doc/design/assistant/blog/day_120__test_day.mdwn
@@ -0,0 +1,2 @@
+Did a lot of testing, found and fixed 4 bugs with repository setup
+configurators. None of them were caused by the recent code reworking.
diff --git a/doc/design/assistant/blog/day_121__buddy_list.mdwn b/doc/design/assistant/blog/day_121__buddy_list.mdwn
new file mode 100644
index 0000000000..48ea1ffda3
--- /dev/null
+++ b/doc/design/assistant/blog/day_121__buddy_list.mdwn
@@ -0,0 +1,10 @@
+Got the XMPP client maintaining a list of buddies, including tracking which
+clients are present and away, and which clients are recognised as other
+git-annex assistant clients. This was fun, it is almost all pure
+functional code, which always makes me happy.
+
+Started building UI for XMPP pairing. So far, I have it showing a list of
+buddies who are also running git-annex (or not). The list even refreshes
+in real time as new buddies come online.
+
+[[!img /assistant/buddylist.png]]
diff --git a/doc/design/assistant/blog/day_122__xmpp_pairing.mdwn b/doc/design/assistant/blog/day_122__xmpp_pairing.mdwn
new file mode 100644
index 0000000000..8683827255
--- /dev/null
+++ b/doc/design/assistant/blog/day_122__xmpp_pairing.mdwn
@@ -0,0 +1,29 @@
+Reworked my XMPP code, which was still specific to push notification, into
+a more generic XMPP client, that's based on a very generic NetMessager
+class, that the rest of the assistant can access without knowing anything
+about XMPP.
+
+Got pair requests flowing via XMPP ping, over Google Talk! And when the
+webapp receives a pair request, it'll pop up an alert to respond. The rest
+of XMPP pairing should be easy to fill in from here.
+
+To finish XMPP pairing, I'll need git pull over XMPP, which is nontrivial,
+but I think I know basically how to do. And I'll need some way to represent
+an XMPP buddy as a git remote, which is all that XMPP pairing will really
+set up.
+
+It could be a git remote using an `xmpp:user@host` URI for the git url, but
+that would confuse regular git to no end (it'd think it was a ssh host),
+and probably need lots of special casing in the parts of git-annex that
+handle git urls too. Or it could be a git remote without an url set, and
+use another config field to represent the XMPP data. But then git wouldn't
+think it was a remote at all, which would prevent using "git pull
+xmppremote" at all, which I'd like to be able to use when implementing git
+pull over XMPP.
+
+Aha! The trick seems to be to leave the url unset in git config,
+but temporarily set it when pulling:
+
+	GIT_SSH=git-annex git git -c remote.xmppremote.url=xmpp:client pull xmppremote
+
+Runs git-annex with "xmpp git-upload-pack 'client'".. Just what I need.
diff --git a/doc/design/assistant/blog/day_122__xmpp_pairing/comment_1_e95efb23eb2e67e3f11a5c7de56424a7._comment b/doc/design/assistant/blog/day_122__xmpp_pairing/comment_1_e95efb23eb2e67e3f11a5c7de56424a7._comment
new file mode 100644
index 0000000000..990fa816ee
--- /dev/null
+++ b/doc/design/assistant/blog/day_122__xmpp_pairing/comment_1_e95efb23eb2e67e3f11a5c7de56424a7._comment
@@ -0,0 +1,10 @@
+[[!comment format=mdwn
+ username="http://meep.pl/"
+ ip="193.219.28.34"
+ subject="git-remote-helpers"
+ date="2012-11-09T08:54:11Z"
+ content="""
+It seems that xmpp::user@host (with double colon) would cause git to try to use a remote-xmpp helper.
+Wouldn't it be a way to have the remote address (if not really URL) set and not be treated as ssh?
+
+"""]]
diff --git a/doc/design/assistant/blog/day_122__xmpp_pairing/comment_2_30e251e73146512bde8b2f69eddeef2e._comment b/doc/design/assistant/blog/day_122__xmpp_pairing/comment_2_30e251e73146512bde8b2f69eddeef2e._comment
new file mode 100644
index 0000000000..256b912426
--- /dev/null
+++ b/doc/design/assistant/blog/day_122__xmpp_pairing/comment_2_30e251e73146512bde8b2f69eddeef2e._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="http://joeyh.name/"
+ ip="4.152.108.27"
+ subject="comment 2"
+ date="2012-11-09T17:41:08Z"
+ content="""
+Thanks, I think that's exactly what I should do.
+"""]]
diff --git a/doc/design/assistant/blog/day_123__xmpp_insanity.mdwn b/doc/design/assistant/blog/day_123__xmpp_insanity.mdwn
new file mode 100644
index 0000000000..bfbc306690
--- /dev/null
+++ b/doc/design/assistant/blog/day_123__xmpp_insanity.mdwn
@@ -0,0 +1,49 @@
+Spent about 5 hours the other night in XMPP hell. At every turn Google Talk
+exhibited behavior that may meet the letter of the XMPP spec (or not), but
+varies between highly annoying and insane.
+
+By "insane", I mean this: If a presence message is directed from one client
+to another client belonging to that same user, randomly leaking that message
+out to other users who are subscribed is just a security hole waiting to
+happen. 
+
+Anyway, I came out of that with a collection of hacks that worked, but I
+didn't like. I was using directed presence for buddy-to-buddy pairing, and
+an IQ message hack for client-to-client pairing. 
+
+Today I got chat messages working instead, for both sorts of pairing. These
+chat messages have an empty body, which *should* prevent clients from
+displaying them, but they're sent directed to only git-annex clients
+anyway.
+
+----
+
+And XMPP pairing 100% works now! Of course, it doesn't know how to git pull
+over XMPP yet, but everything else works.
+
+Here's a real `.git/config` generated by the assistant after XMPP pairing.
+
+	[remote "joey"]
+	        url = 
+	        fetch = +refs/heads/*:refs/remotes/joey/*
+	        annex-uuid = 14f5e93e-1ed0-11e2-aa1c-f7a45e662d39
+	        annex-xmppaddress = joey@kitenet.net 
+
+----
+
+Fixed a typo that led to an infinite loop when adding a ssh git repo with the
+assistant. Only occurred when an absolute directory was specified, which 
+is why I didn't notice it before.
+
+----
+
+Security fix: Added a `GIT_ANNEX_SHELL_DIRECTORY` environment variable that
+locks down git-annex-shell to operating in only a single directory. The
+assistant sets that in ssh `authorized_keys` lines it creates. This
+prevents someone you pair with from being able to access any other git or
+git-annex repositories you may have.
+
+----
+
+Next up, more craziness. But tomorrow is Nov 6th, so you in the US already
+knew that..
diff --git a/doc/design/assistant/blog/day_124__git_push_over_xmpp_groundwork.mdwn b/doc/design/assistant/blog/day_124__git_push_over_xmpp_groundwork.mdwn
new file mode 100644
index 0000000000..ecb6023f20
--- /dev/null
+++ b/doc/design/assistant/blog/day_124__git_push_over_xmpp_groundwork.mdwn
@@ -0,0 +1,28 @@
+Laying the groundwork for git push over XMPP. BTW, the motivation for doing
+this now is that if the assistant can push git data peer-to-peer, users
+who are entirely using the cloud don't need to set up a git repo in the
+cloud somewhere. Instead, a single special remote in the cloud will be all
+they need. So this is a keystone in the assistant's cloud support.
+
+I'm building the following pipeline:
+
+	git push <--> git-annex xmppgit <--> xmppPush <-------> xmpp
+	                                                         |
+	git receive-pack <--> xmppReceivePack <---------------> xmpp
+
+A tricky part of this is `git-annex xmppgit`, which is run by `git push`
+rather than the usual `ssh`. Rather than speak XMPP itself, that feeds the
+data through the assistant daemon, using some special FDs that are set
+up by the assistant when it runs `git push`, and communicated via
+environment variables. I hoped to set up a pipe and not need it to do any
+work on its own, but short of using the linux-specific `splice(2)`, that
+doesn't seem possible. It also will receive the exit status of
+`git receive-pack` and propigate it to `git push`.
+
+Also built the IO sides of `xmppPush` and `xmppReceivePack` although these
+are not tested. The XMPP sides of them come next.
+
+----
+
+Stuffing lots of git-annex branded USB keys into envelopes tonight, while
+watching the election coverage.
diff --git a/doc/design/assistant/blog/day_125__xmpp_push_continues.mdwn b/doc/design/assistant/blog/day_125__xmpp_push_continues.mdwn
new file mode 100644
index 0000000000..010ab14dad
--- /dev/null
+++ b/doc/design/assistant/blog/day_125__xmpp_push_continues.mdwn
@@ -0,0 +1,15 @@
+I've finished building the XMMP side of git push over XMPP. Now I only
+have to add code to trigger these pushes. And of course, fix all the bugs,
+since none of this has been tested at all.
+
+Had to deal with some complications, like handling multiple clients that
+all want to push at the same time. Only one push is handled at a time;
+messages for others are queued. Another complication I don't deal with yet
+is what to do if a client stops responding in the middle of a push. It
+currently will wait forever for a message from the client; instead it
+should time out.
+
+----
+
+Jimmy got the OSX builder working again, despite my best attempts to add
+dependencies and break it.
diff --git a/doc/design/assistant/blog/day_126__mr_watson_come_here.mdwn b/doc/design/assistant/blog/day_126__mr_watson_come_here.mdwn
new file mode 100644
index 0000000000..0dd600cc89
--- /dev/null
+++ b/doc/design/assistant/blog/day_126__mr_watson_come_here.mdwn
@@ -0,0 +1,52 @@
+I'm stunned and stoked to have gotten git push over XMPP working today.
+And am nearly out of steam, it was a wild ride..
+
+	To xmpp::joey@kitenet.net
+	  * [new branch]      master -> refs/xmpp/newmaster
+
+The surprising part is how close my initial implementation came to just
+working on the first try. It had around 3 bugs, which took hours of staring
+at debugging output to find:
+
+1. The git push action was run in the same thread as the XMPP
+   client, which prevented the client from continuing to run and relaying
+   messages.
+2. The git-receive-pack side waited on the wrong thread, so didn't
+   notice when the program was done.
+3. I accidentally used the wrong attribute name when sending a ReceivePackDone
+   message.
+
+But all in all, it just worked.
+
+Here's a sample of the actual data sent when one file is added to the
+repository (also includes the corresponding update to the git-annex branch):
+
+	MDA4NjhhMmNmOGZjMWE3MTlkOGVjOWVmOWZiMGZiNjVlODc2NjQ1NDAyMTAgODIwNTZjMDM4
+	ZjU2YzE1ODdjYzllOWRhNzQzMzU0YjE4NzNjZWJlOSByZWZzL3htcHAvbmV3bWFzdGVyACBy
+	ZXBvcnQtc3RhdHVzIHNpZGUtYmFuZC02NGswMDAw
+
+	UEFDSwAAAAIAAAADnAx4nJXLTQ4CIQxA4T2n4AKaAqVAYoxL4y2gU+Jo5iczdeHtnSu4eMm3
+	ebqJ2NwgSCLmNkTBlKFCYwwhoHOtQ+scqZCwWesms9pcPffc2dXkypCFi/TSG/RGUXIiwojg
+	HZj60eey2cciX3uXfbeX18Hbe1SZRc9HV+tC9FgyJW9PgACGl2kaVeXfz/wArHQ81qMGeJwz
+	NDIAAoVUI4ZZB9RW1E8NtXp5t77/fn3hw41cl2MNIbIZqTk5+Qwerw+aJX2INjsffYndtdCz
+	5mZWLDdUQV5qeVpmDtCQnx/3/6s40+Q4P/7O+Y4ShS+1Ad83AwC6CirftAt4nK3MsRGDMAwF
+	0IkcSVgSdpkidzRUmcDWBy4pSAEFl+mzRN4A77a9Tmr7vlz06e8lzoPmmb5Mz+k+mD/SkTkl
+	eFHPq9eqQ+nSzFsWaDFnFmCMCEOvHgLrCrQxS7AWdvUVhv9uPwHxMbfumlvWdco1RLL4wSQF
+	g0uFFOKu3Q==
+
+Git said this push took 385 bytes; after base64 encoding to transport it over
+XMPP as shown above, it needs 701 bytes, and the XMPP envelope and encryption
+adds more overhead (although the XMPP connection may also be compressed?)
+
+Not the most efficient git transport, but still a practical one!
+
+----
+
+Big thanks by the way to meep, who posted a comment reminding me about
+`git-remote-helpers`. This was the right thing to use for XMPP over git,
+it lets the git remote be configured with `url = xmpp::user@host`.
+
+----
+
+Next, I need to get the assistant to use this for syncing. Currently, it only
+pushes a test branch.
diff --git a/doc/design/assistant/blog/day_126__mr_watson_come_here/comment_1_ee1361e6b235f4e1c00596ba516b519a._comment b/doc/design/assistant/blog/day_126__mr_watson_come_here/comment_1_ee1361e6b235f4e1c00596ba516b519a._comment
new file mode 100644
index 0000000000..47766dc778
--- /dev/null
+++ b/doc/design/assistant/blog/day_126__mr_watson_come_here/comment_1_ee1361e6b235f4e1c00596ba516b519a._comment
@@ -0,0 +1,10 @@
+[[!comment format=mdwn
+ username="https://www.google.com/accounts/o8/id?id=AItOawm5vDem1yeIu6uith5pxfb4mdKdIWVJpCs"
+ nickname="Louis"
+ subject="Base64 vs. yEnc"
+ date="2012-11-12T02:02:17Z"
+ content="""
+Would yEnc be a suitable alternative to base64 for encoding the binary transfers over XMPP?  Or why not?
+
+Thanks!
+"""]]
diff --git a/doc/design/assistant/blog/day_126__mr_watson_come_here/comment_2_8eb366ae7efb347bd3bbd9a98e0821b3._comment b/doc/design/assistant/blog/day_126__mr_watson_come_here/comment_2_8eb366ae7efb347bd3bbd9a98e0821b3._comment
new file mode 100644
index 0000000000..6f76ec4c8e
--- /dev/null
+++ b/doc/design/assistant/blog/day_126__mr_watson_come_here/comment_2_8eb366ae7efb347bd3bbd9a98e0821b3._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="http://joeyh.name/"
+ ip="4.152.108.27"
+ subject="comment 2"
+ date="2012-11-12T02:39:19Z"
+ content="""
+AFAIK, yenc can use xml characters like < and > in its encoding.
+"""]]
diff --git a/doc/design/assistant/blog/day_127__xmpp_syncs.mdwn b/doc/design/assistant/blog/day_127__xmpp_syncs.mdwn
new file mode 100644
index 0000000000..7f95f87e39
--- /dev/null
+++ b/doc/design/assistant/blog/day_127__xmpp_syncs.mdwn
@@ -0,0 +1,35 @@
+I got full-on git-annex assistant syncing going over XMPP today!
+
+How well does it work? Well, I'm at the cabin behind a dialup modem. I have
+two repos that can only communicate over XMPP. One uses my own XMPP server,
+and the other uses a Google Talk account. I make a file in one repo, and
+switch windows to the other, and type `ls`, and the file (not its content
+tho..) has often already shown up. So, it's about as fast as syncing over
+ssh, although YMMV.
+
+----
+
+Refactored the git push over XMPP code rather severely. It's quite a
+lot cleaner now.
+
+----
+
+Set XMPP presence priority to a negative value, which will hopefully
+prevent git-annex clients that share a XMPP account with other clients from
+intercepting chat messages. Had to change my XMPP protocol some to deal
+with this.
+
+----
+
+Some webapp UI work. When showing the buddy list, indicate which buddies
+are already paired with. 
+
+After XMPP pairing, it now encourages setting up a shared cloud repository.
+
+[[!img /assistant/xmpppairingend.png]]
+
+I still need to do more with the UI after XMPP pairing, to help the paired
+users configure a shared cloud transfer remote. Perhaps the thing to do is
+for the ConfigMonitor to notice when a git push adds a new remote,
+and pop up an alert suggesting the user enable it. Then one user
+can create the repository, and the other one enable it.
diff --git a/doc/design/assistant/blog/day_128__last_xmpp_day.mdwn b/doc/design/assistant/blog/day_128__last_xmpp_day.mdwn
new file mode 100644
index 0000000000..2c7d70accc
--- /dev/null
+++ b/doc/design/assistant/blog/day_128__last_xmpp_day.mdwn
@@ -0,0 +1,49 @@
+I hope I'm nearly at the end of this XMPP stuff after today. Planning a new
+release tomorrow.
+
+----
+
+Split up the local pairing and XMPP pairing UIs, and wrote a
+[[/assistant/share_with_a_friend_walkthrough]].
+
+----
+
+Got the XMPP push code to time out if expected data doesn't arrive within
+2 minutes, rather than potentially blocking other XMPP push forever if 
+the other end went away. 
+
+I pulled in the Haskell
+[async](http://hackage.haskell.org/package/async) library for this,
+which is yes, yet another library, but one that's now in the haskell platform.
+It's worth it, because of how nicely it let me implement IO actions that
+time out.
+
+[[!format haskell """
+runTimeout :: Seconds -> IO a -> IO (Either SomeException a)
+runTimeout secs a = do
+        runner <- async a
+        controller <- async $ do
+                threadDelaySeconds secs
+                cancel runner
+        cancel controller `after` waitCatch runner
+"""]]
+
+This would have been 20-50 lines of gnarly code without async, and I'm sure
+I'll find more uses for async in the future.
+
+----
+
+Discovered that the XMPP push code could deadlock, if both clients started
+a push to the other at the same time. I decided to fix this by allowing
+each client to run both one push and one receive-pack over XMPP at the same
+time.
+
+----
+
+Prevented the transfer scanner from trying to queue transfers to XMPP remotes.
+
+----
+
+Made XMPP pair requests that come from the same account we've already
+paired with be automatically accepted. So once you pair with one device,
+you can easily add more.
diff --git a/doc/design/assistant/blog/day_128__last_xmpp_day/comment_1_fd8c1d6358cb50f4dad8ba11d33d861f._comment b/doc/design/assistant/blog/day_128__last_xmpp_day/comment_1_fd8c1d6358cb50f4dad8ba11d33d861f._comment
new file mode 100644
index 0000000000..add2cf313a
--- /dev/null
+++ b/doc/design/assistant/blog/day_128__last_xmpp_day/comment_1_fd8c1d6358cb50f4dad8ba11d33d861f._comment
@@ -0,0 +1,10 @@
+[[!comment format=mdwn
+ username="https://www.google.com/accounts/o8/id?id=AItOawmqVi9eQjkZt8EC-byTXJ8TnY7VyOHzW2s"
+ nickname="Zahary"
+ subject="Pairing on the local network"
+ date="2012-11-12T11:39:17Z"
+ content="""
+Joey, How does pairing on the local network work btw?
+Now that you have experience intercepting and relaying the git pack output maybe you can use UDP broadcast on the local network for a truly fast sync.
+With dropbox, I'm often syncing working copies of software projects between several VMs so I can test the code on all platforms before committing. I've found that to be faster (for builds) then using a VM shared folders, network shares, etc.
+"""]]
diff --git a/doc/design/assistant/blog/day_128__last_xmpp_day/comment_2_43664b73c71c41d71bc95e665f128106._comment b/doc/design/assistant/blog/day_128__last_xmpp_day/comment_2_43664b73c71c41d71bc95e665f128106._comment
new file mode 100644
index 0000000000..3a83186184
--- /dev/null
+++ b/doc/design/assistant/blog/day_128__last_xmpp_day/comment_2_43664b73c71c41d71bc95e665f128106._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="http://phil.0x539.de/"
+ nickname="Philipp Kern"
+ subject="comment 2"
+ date="2012-11-12T11:50:42Z"
+ content="""
+Developing something new based on IPv4 UDP broadcast seems to be insane. IPv6 link-local multicast should be available virtually anywhere. XMPP relies on TCP and a central server to guarantee that a single packet is not split up, which would need quite some protocol engineering to get right over lossy UDP (e.g. segmentation, flow control, congestion avoidance). But TCP as discovered using mDNS might work… Still needs some kind of authentication / encryption, though.
+"""]]
diff --git a/doc/design/assistant/blog/day_128__last_xmpp_day/comment_3_d369b04f686009a9dbb57b999107a55e._comment b/doc/design/assistant/blog/day_128__last_xmpp_day/comment_3_d369b04f686009a9dbb57b999107a55e._comment
new file mode 100644
index 0000000000..951cc7e057
--- /dev/null
+++ b/doc/design/assistant/blog/day_128__last_xmpp_day/comment_3_d369b04f686009a9dbb57b999107a55e._comment
@@ -0,0 +1,11 @@
+[[!comment format=mdwn
+ username="https://www.google.com/accounts/o8/id?id=AItOawmqVi9eQjkZt8EC-byTXJ8TnY7VyOHzW2s"
+ nickname="Zahary"
+ subject="Jingle"
+ date="2012-11-12T12:00:50Z"
+ content="""
+Also, you probably know about Jingle:
+http://en.wikipedia.org/wiki/Jingle_(protocol)
+
+What are you thoughts about it? I guess it would have been harder to use the Google C++ library from Haskell and full mesh peer-to-peer is certainly more complicated than using the XMPP server to effectively multicast to all paired clients.
+"""]]
diff --git a/doc/design/assistant/blog/day_128__last_xmpp_day/comment_4_095855d301e7ccd3689ffe507cfb63ee._comment b/doc/design/assistant/blog/day_128__last_xmpp_day/comment_4_095855d301e7ccd3689ffe507cfb63ee._comment
new file mode 100644
index 0000000000..0e5295ad2e
--- /dev/null
+++ b/doc/design/assistant/blog/day_128__last_xmpp_day/comment_4_095855d301e7ccd3689ffe507cfb63ee._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="https://www.google.com/accounts/o8/id?id=AItOawmqVi9eQjkZt8EC-byTXJ8TnY7VyOHzW2s"
+ nickname="Zahary"
+ subject="@Philipp Kern"
+ date="2012-11-12T12:13:33Z"
+ content="""
+Alright, I stand corrected, although I'm aware of some libraries that have already solved the problem so you don't really have to implement reliable UDP from scratch: http://code.google.com/p/openpgm/
+"""]]
diff --git a/doc/design/assistant/blog/day_128__last_xmpp_day/comment_5_da7b0586b0b28e1e0fe4126f6543a7bc._comment b/doc/design/assistant/blog/day_128__last_xmpp_day/comment_5_da7b0586b0b28e1e0fe4126f6543a7bc._comment
new file mode 100644
index 0000000000..f10326a3c9
--- /dev/null
+++ b/doc/design/assistant/blog/day_128__last_xmpp_day/comment_5_da7b0586b0b28e1e0fe4126f6543a7bc._comment
@@ -0,0 +1,9 @@
+[[!comment format=mdwn
+ username="http://phil.0x539.de/"
+ nickname="Philipp Kern"
+ subject="comment 5"
+ date="2012-11-12T12:50:26Z"
+ content="""
+To quote the website (emphasis mine):
+> PGM is appropriate for applications that require duplicate-free multicast data delivery from multiple sources to multiple receivers. *PGM does not support acknowledged delivery, nor does it guarantee ordering of packets from multiple senders.*
+"""]]
diff --git a/doc/design/assistant/blog/day_128__last_xmpp_day/comment_6_2f9ba367e19d77bf52f372b6f0f5938a._comment b/doc/design/assistant/blog/day_128__last_xmpp_day/comment_6_2f9ba367e19d77bf52f372b6f0f5938a._comment
new file mode 100644
index 0000000000..e73c4c071d
--- /dev/null
+++ b/doc/design/assistant/blog/day_128__last_xmpp_day/comment_6_2f9ba367e19d77bf52f372b6f0f5938a._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="http://joeyh.name/"
+ ip="4.152.108.27"
+ subject="comment 6"
+ date="2012-11-12T14:43:32Z"
+ content="""
+For local pairing, multicast UDP is used for discovery and authentication, and then all data transfers take place over ssh.
+"""]]
diff --git a/doc/design/assistant/blog/day_129__release.mdwn b/doc/design/assistant/blog/day_129__release.mdwn
new file mode 100644
index 0000000000..553b0e91f3
--- /dev/null
+++ b/doc/design/assistant/blog/day_129__release.mdwn
@@ -0,0 +1,4 @@
+Cut a new release today. It's been nearly a month since the last one, and a
+large number of improvements.. Be sure to read the
+[[/assistant/release_notes]] if upgrading. All the standalone builds are
+updated already.
diff --git a/doc/design/assistant/blog/day_12__freebsd_redux.mdwn b/doc/design/assistant/blog/day_12__freebsd_redux.mdwn
new file mode 100644
index 0000000000..5ec446c9de
--- /dev/null
+++ b/doc/design/assistant/blog/day_12__freebsd_redux.mdwn
@@ -0,0 +1,23 @@
+Followed my plan from yesterday, and wrote a simple C library to interface
+to `kqueue`, and Haskell code to use that library. By now I think I
+understand kqueue fairly well -- there are some very tricky parts to the
+interface.
+
+But... it still didn't work. After building all this, my code was
+failing the same way that the
+[haskell kqueue library failed](https://github.com/hesselink/kqueue/issues/1)
+yesterday. I filed a [bug report with a testcase]().
+
+Then I thought to ask on #haskell. Got sorted out in quick order! The
+problem turns out to be that haskell's runtime has a periodic SIGALARM,
+that is interrupting my kevent call. It can be worked around with `+RTS -V0`,
+but I put in a fix to retry to kevent when it's interrupted.
+
+And now `git-annex watch` can detect changes to directories on BSD and OSX!
+
+Note: I said "detect", not "do something useful in response to". Getting
+from the limited kqueue events to actually staging changes in the git repo
+is going to be another day's work. Still, brave FreeBSD or OSX users
+might want to check out the `watch` branch from git and see if 
+`git annex watch` will at least *say* it sees changes you make to your
+repository.
diff --git a/doc/design/assistant/blog/day_12__freebsd_redux/comment_1_5da32cf53f1de27bfe6cec2d294db3e1._comment b/doc/design/assistant/blog/day_12__freebsd_redux/comment_1_5da32cf53f1de27bfe6cec2d294db3e1._comment
new file mode 100644
index 0000000000..253af9e7c5
--- /dev/null
+++ b/doc/design/assistant/blog/day_12__freebsd_redux/comment_1_5da32cf53f1de27bfe6cec2d294db3e1._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="https://www.google.com/accounts/o8/id?id=AItOawkSq2FDpK2n66QRUxtqqdbyDuwgbQmUWus"
+ nickname="Jimmy"
+ subject="comment 1"
+ date="2012-06-19T06:53:26Z"
+ content="""
+heh, yea, it's detecting changes on OSX ;)
+"""]]
diff --git a/doc/design/assistant/blog/day_12__freebsd_redux/comment_2_696d6e22034acf5bb60d80124b72ef2f._comment b/doc/design/assistant/blog/day_12__freebsd_redux/comment_2_696d6e22034acf5bb60d80124b72ef2f._comment
new file mode 100644
index 0000000000..9f3e34adb9
--- /dev/null
+++ b/doc/design/assistant/blog/day_12__freebsd_redux/comment_2_696d6e22034acf5bb60d80124b72ef2f._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="https://www.google.com/accounts/o8/id?id=AItOawkSq2FDpK2n66QRUxtqqdbyDuwgbQmUWus"
+ nickname="Jimmy"
+ subject="comment 1"
+ date="2012-06-19T07:01:26Z"
+ content="""
+issues with the watch command on OSX, it seems that there is a race condition somewhere. I dumped a few iso's into an annex and it only annexed the smaller files (checksums) and the bigger ones (the iso's) just got made read only. also do you want these bugs to be logged here or in the bugs section?
+"""]]
diff --git a/doc/design/assistant/blog/day_12__freebsd_redux/comment_3_5ab7808595e3b51ca4141d15fdd44743._comment b/doc/design/assistant/blog/day_12__freebsd_redux/comment_3_5ab7808595e3b51ca4141d15fdd44743._comment
new file mode 100644
index 0000000000..6bcba4d3e9
--- /dev/null
+++ b/doc/design/assistant/blog/day_12__freebsd_redux/comment_3_5ab7808595e3b51ca4141d15fdd44743._comment
@@ -0,0 +1,7 @@
+[[!comment format=mdwn
+ username="chris"
+ subject="Thanks"
+ date="2014-12-25T11:58:55Z"
+ content="""
+Thanks
+"""]]
diff --git a/doc/design/assistant/blog/day_130__what_now.mdwn b/doc/design/assistant/blog/day_130__what_now.mdwn
new file mode 100644
index 0000000000..4fc3fe53ca
--- /dev/null
+++ b/doc/design/assistant/blog/day_130__what_now.mdwn
@@ -0,0 +1,36 @@
+Dealt with post-release feedback deluge. There are a couple weird bugs that
+I don't understand yet. OSX app is still not working everywhere.
+
+----
+
+Got the list of repositories in the webapp to update automatically when
+repositories are added, including when syncing with a remote causes
+repositories to be discovered.
+
+----
+
+I need a plan for the rest of the month. It feels right to focus on more
+cloud storage support. Particularly because all the cloud providers
+supported so far are ones that, while often being best of breed, also cost
+money. To finish up the cloud story, need support for some free ones.
+
+Looking at the results of the [[polls/prioritizing_special_remotes]]
+poll, I suspect that free storage is a large part of why Google Drive got
+so many votes. Soo, since there is not yet a Haskell library for Google
+Drive, rather than spending a large chunk of time writing one, I hope to
+use a [Haskell WebDAV library](http://hackage.haskell.org/package/DAV)
+that my friend Clint recently wrote. A generic
+WebDAV special remote in git-annex will provide much better support for
+box.com (which has 5 to 50 gb free storage), as well as all the 
+[OwnCloud providers](http://owncloud.org/providers/), at least one of which
+provides 5 gb free storage.
+
+If I have time left this month after doing that, I'd be inclined to do
+Amazon Glacier. People have already gotten that working with git-annex, but
+a proper special remote should be better/easier, and will allow integrating
+support for it into the assistant, which should help deal with its long
+retrieval delays. And since, if you have a lot of data
+archived in Glacier, you will only want to pull out a few files at a time,
+this is another place besides mobile phones where a [[partial_content]]
+retrieval UI is needed. Which is on the roadmap to be worked on next
+month-ish. Synergy, here I come. I hope.
diff --git a/doc/design/assistant/blog/day_130__what_now/comment_1_402f00cc034351d8253a797dd4de55bf._comment b/doc/design/assistant/blog/day_130__what_now/comment_1_402f00cc034351d8253a797dd4de55bf._comment
new file mode 100644
index 0000000000..ec2158c226
--- /dev/null
+++ b/doc/design/assistant/blog/day_130__what_now/comment_1_402f00cc034351d8253a797dd4de55bf._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="https://www.google.com/accounts/o8/id?id=AItOawm7AuSfii_tCkLyspL6Mr0ATlO6OxLNYOo"
+ nickname="Georg"
+ subject="definitely glacier would be cool"
+ date="2012-11-14T08:51:40Z"
+ content="""
+Because having it available as an archival system to move stuff over that you don't expect to constantly need available. But what would be needed is a way to prevent the assistant to pull all the data from there back on your clients. This probably allready is doable with preferred content, but probably needs an easier way to set up - when I tried the new version it immediately started to pull all the stuff from my server repository that I moved there to get my disk space free. So my scenario would be \"move stuff to archives and when I drop it afterwards, keep it dropped locally if at least one archival server has it\".
+"""]]
diff --git a/doc/design/assistant/blog/day_131__webdav_groundwork.mdwn b/doc/design/assistant/blog/day_131__webdav_groundwork.mdwn
new file mode 100644
index 0000000000..795acfea10
--- /dev/null
+++ b/doc/design/assistant/blog/day_131__webdav_groundwork.mdwn
@@ -0,0 +1,28 @@
+Read up on WebDAV, and got the haskell library working. Several hours
+were wasted by stumbling over a bug in the library, that requires a
+carefully crafted XML document to prevent. Such a pity about things
+like DAV (and XMPP) being designed back when people were gung-ho about
+XML.. but we're stuck with them now.
+
+Now I'm able to send and receive files to box.com using the library. Trying to
+use an OwnCloud server, though, I get a most strange error message, which
+looks to be coming from deep in the HTTPS library stack: "invalid IV length"
+
+The haskell DAV library didn't have a way to delete files. I've added one
+and sent off a patch.
+
+Roughed in a skeleton of a webdav special remote. Doesn't do anything yet.
+Will soon.
+
+Factored out a Creds module from parts of the S3 special remote and XMPP
+support, that all has to do with credentials storage. Using this for webdav
+creds storage too.
+
+Will also need to factor out the code that's currently in the directory
+special remote, for chunking of files.
+
+----
+
+PS: WebDAV, for all its monstrously complicated feature set, lacks one obvious
+feature: The ability to check how much free space is available to store
+files. Eyeroll.
diff --git a/doc/design/assistant/blog/day_132__webdav_continued.mdwn b/doc/design/assistant/blog/day_132__webdav_continued.mdwn
new file mode 100644
index 0000000000..075a5d8c41
--- /dev/null
+++ b/doc/design/assistant/blog/day_132__webdav_continued.mdwn
@@ -0,0 +1,22 @@
+Two releases of the Haskell DAV library today. First release had my changes
+from yesterday. Then I realized I'd also need support for making WebDAV
+"collections" (subdirectories), and sent Clint a patch for that too, as
+well as a patch for querying DAV properties, and that was 0.2.
+Got it into Debian unstable as well. Should have everything I'll need now.
+
+The webdav special remote is now working! Still todo:
+Encryption support, progress bars, large file chunking, and webapp
+configurators. But already, it's a lot nicer than the old approach of using
+davfs2, which was really flakey and slow with large data volumes.
+
+I did notice, though, that uploading a 100 mb file made the process use 100
+mb of memory. This is a problem I've struggled with earlier with S3, the
+Haskell http libraries are prevented from streaming data by several parts
+of the protocol that cause the data to be accessed more than once. I guess
+this won't be a problem for DAV though, since it'll probably be chunking
+files anyway.
+
+---
+
+Mailed out all my Kickstarter USB key rewards, and ordered
+the T-shirts too.
diff --git a/doc/design/assistant/blog/day_133__webdav_working.mdwn b/doc/design/assistant/blog/day_133__webdav_working.mdwn
new file mode 100644
index 0000000000..d71f258520
--- /dev/null
+++ b/doc/design/assistant/blog/day_133__webdav_working.mdwn
@@ -0,0 +1,31 @@
+Worked on webdav special remotes all day. 
+
+* Got encryption working,
+  after fixing an amusing typo that made `initremote` for webdav throw away the
+  encryption configuration and store files unencrypted.
+* Factored out parts of the directory special remote that had to do with file
+  chunking, and am using that for webdav. This refactoring was painful.
+
+At this point, I feel the webdav special remote works better than the old
+davfs2 + directory special remote hack. While webdav doesn't yet have
+progress info for uploads, that info was pretty busted anyway with
+davfs2 due to how it buffers files. So ... I've merged webdav into master!
+
+-----
+
+Tomorrow, webapp configurators for Box.com and any other webdav supporting
+sites I can turn up and get to work..
+
+-----
+
+A while ago I made git-annex not store login credentials in git for special
+remotes, when it's only encrypting them with a shared cipher. The
+rationalle was that you don't want to give everyone who gets ahold of your
+git repo (which includes the encryption key) access to your passwords,
+Amazon S3 account, or whatever. I'm now considering adding a checkbox (or
+command-line flag) that allows storing the login credentials in git,
+if the user wants to. While using public key crypto is the real solution
+(and is fully supported by git-annex (but not yet configurable in the
+webapp)), this seems like a reasonable thing to do in some circumstances,
+like when you have a Box.com account you really do want to share with
+the people who use the git repo.
diff --git a/doc/design/assistant/blog/day_134__box.com_configurator.mdwn b/doc/design/assistant/blog/day_134__box.com_configurator.mdwn
new file mode 100644
index 0000000000..1156336404
--- /dev/null
+++ b/doc/design/assistant/blog/day_134__box.com_configurator.mdwn
@@ -0,0 +1,8 @@
+I needed an easy day, and I got one. Configurator in the webapp for Box.com
+came together really quickly and easily, and worked on the first try.
+
+Also filed a [bug](https://github.com/vincenthz/hs-cryptocipher/issues/21)
+on the Haskell library that is failing on portknox.com's SSL certificate.
+That site is the only OwnCloud provider currently offering free WebDAV
+storage. Will hold off on adding OwnCloud to the webapp's cloud provider lists
+until that's fixed.
diff --git a/doc/design/assistant/blog/day_135__progress_revisited.mdwn b/doc/design/assistant/blog/day_135__progress_revisited.mdwn
new file mode 100644
index 0000000000..63e3c4bf24
--- /dev/null
+++ b/doc/design/assistant/blog/day_135__progress_revisited.mdwn
@@ -0,0 +1,37 @@
+Unexpectedly today, I got progress displays working for uploads via WebDAV.
+
+The roadblock had been that the interface of for uploading to S3 and WebDAV
+is something like `ByteString -> IO a`. Which doesn't provide any hook to
+update a progress display as the ByteString is consumed.
+
+My solution to this was to create a `hGetContentsObserved`, that's similar
+to `hGetContents`, but after reading each 64kb chunk of data from the
+Handle to populate the ByteString, it runs some observing action. So when
+the ByteString is streamed, as each chunk is consumed, the observer
+runs. I was able to get this to run in constant space, despite not having
+access to some of the ByteString internals that `hGetContents` is built
+with.
+
+So, a little scary, but nice. I am curious if there's not a better way
+to solve this problem hidden in a library somewhere. Perhaps it's another
+thing that conduit solves somehow? Because if there's not, my solution
+probably deserves to be put into a library. Any Haskell folk know?
+
+----
+
+Used above to do progress displays for uploads to S3. Also did progress
+display to console on download from S3. Now very close to being done
+with [[progressbars]]. Finally. Only bup and hook remotes need progress
+work.
+
+----
+
+Reworked the core crypto interface, to better support streaming data through
+gpg. This allowed fixing both the directory and webdav special remotes to
+not buffer whole files in memory when retrieving them as chunks from the
+remote.
+
+-----
+
+Spent some time dealing with API changes in Yesod and Conduit. Some of them
+annoyingly gratuitous.
diff --git a/doc/design/assistant/blog/day_136__misc.mdwn b/doc/design/assistant/blog/day_136__misc.mdwn
new file mode 100644
index 0000000000..5a14156d58
--- /dev/null
+++ b/doc/design/assistant/blog/day_136__misc.mdwn
@@ -0,0 +1,14 @@
+Changed how the directory and webdav special remotes store content.
+The new method uses less system calls, is more robust, and leaves any
+partially transferred key content in a single tmp directory, which
+will make it easier to clean that out later.
+
+Also found & fixed a cute bug in the directory special remote when the
+chunksize is set to a smaller value than the ByteString chunk size, that
+caused it to loop forever creating empty files.
+
+----
+
+Added an embedcreds=yes option when setting up special remotes. 
+Will put UI for it into the webapp later, but rather than work on that
+tomorrow, I plan to work on glacier.
diff --git a/doc/design/assistant/blog/day_137__Glacier.mdwn b/doc/design/assistant/blog/day_137__Glacier.mdwn
new file mode 100644
index 0000000000..4e6787eb5c
--- /dev/null
+++ b/doc/design/assistant/blog/day_137__Glacier.mdwn
@@ -0,0 +1,30 @@
+Got Amazon Glacier working as a full-fledged special remote.
+
+(Well, I think it works... Since it takes 4 hours to get data out,
+which is longer than the time it took me to sign up for Glacier and
+write the special remote ... I've yet to fully test it!)
+
+Thanks to Robie Basak for writing glacier-cli, and developing the intial
+hook remote support. Also thanks to Peter Todd for pointing out that
+Glacier cannot store empty files, which had to be worked around in the
+special remote.
+
+Of course the 4 hour delay on retreval makes Glacier interesting. For now,
+you have to run "git annex get" twice, once to queue the retrieval, and a
+second time in 4 hours to get the file(s). There is a helpful example in
+[[tips/using_Amazon_Glacier]].
+
+The real complication though, is that Glacier's inventories take a long
+time to get, and can be out of date. So glacier-cli caches inventory info.
+I didn't feel comfortable making git-annex trust that information,
+so it'll refuse to trust that Glacier has a copy of a file when dropping
+it. There's a `--trust-glacier` switch to override this default paranoid
+behavior when dropping files.
+
+----
+
+Tomorrow ... er, tomorrow is Thanksgiving trip start.
+
+Next weekend: Webapp configurator for glacier, and maybe something
+to get the assistant to detect when jobs are complete and finish
+retrievals from Glacier, automatically.
diff --git a/doc/design/assistant/blog/day_138__back.mdwn b/doc/design/assistant/blog/day_138__back.mdwn
new file mode 100644
index 0000000000..7c2b4ec459
--- /dev/null
+++ b/doc/design/assistant/blog/day_138__back.mdwn
@@ -0,0 +1,25 @@
+Added a configurator for Glacier repositories to the webapp. That was the last
+cloud repository configurator that was listed in the webapp and wasn't
+done. Indeed, just two more repository configurators remain to be filled in:
+phone and NAS.
+
+By default, Glacier repositories are put in a new "small archive" group.
+This makes only files placed in "archive" directories be sent to Glacier
+(as well as being dropped from clients), unlike the full archive group
+which archives all files. Of course you can change this setting, but
+avoiding syncing all data to Glacier seemed like a good default, especially
+since some are still worried about Glacier's pricing model.
+
+Fixed several bugs in the handling of archive directories, and 
+the webapp makes a toplevel archive directory when an archive remote is
+created, so the user can get on with using it.
+
+Made the assistant able to drop local files immediately after transferring
+them to glacier, despite not being able to trust glacier's inventory.
+This was accomplished by making the transferrer, after a successful upload,
+indicate that it trusts the remote it just uploaded to has the file,
+when it checks if the file should be dropped.
+
+Only thing left to do for glacier is to make the assistant retry failed
+downloads from it after 4 hours, or better, as soon as they become
+available.
diff --git a/doc/design/assistant/blog/day_138__back/comment_1_65a8499b284bed38d2bde1886a54a311._comment b/doc/design/assistant/blog/day_138__back/comment_1_65a8499b284bed38d2bde1886a54a311._comment
new file mode 100644
index 0000000000..5bd94d13eb
--- /dev/null
+++ b/doc/design/assistant/blog/day_138__back/comment_1_65a8499b284bed38d2bde1886a54a311._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="https://www.google.com/accounts/o8/id?id=AItOawmnG4EuvZWse5hvgrl0XAK-U61e-0iGaao"
+ nickname="David"
+ subject="NAS?"
+ date="2012-11-25T12:27:50Z"
+ content="""
+What are you planning to support with the NAS repository type?  Will the assistant sync over NFS? I wanted to read more about this but I couldn't find anything on the design pages.
+"""]]
diff --git a/doc/design/assistant/blog/day_139__catch_up.mdwn b/doc/design/assistant/blog/day_139__catch_up.mdwn
new file mode 100644
index 0000000000..fc1e572b02
--- /dev/null
+++ b/doc/design/assistant/blog/day_139__catch_up.mdwn
@@ -0,0 +1,11 @@
+Got progress bars working for glacier. This needed some glacier-cli
+changes, which Robie helpfully made earlier.
+
+Spent some hours getting caught up and responding to bug reports, etc.
+
+Spent a while trying to make git-annex commands that fail to find
+any matching files to act on print a useful warning message,
+rather than the current nothing. Concluded this will be surprisingly
+hard to do, due to the multiple seek passes some commands perform. Update:
+Thought of a non-invasive and not too ugly way to do this while on my
+evening walk, and this wart is gone.
diff --git a/doc/design/assistant/blog/day_13__kqueue_continued.mdwn b/doc/design/assistant/blog/day_13__kqueue_continued.mdwn
new file mode 100644
index 0000000000..fd0cbb372b
--- /dev/null
+++ b/doc/design/assistant/blog/day_13__kqueue_continued.mdwn
@@ -0,0 +1,34 @@
+Good news! My beta testers report that the new kqueue code works on OSX.
+At least "works" as well as it does on Debian kFreeBSD. My crazy
+development strategy of developing on Debian kFreeBSD while targeting Mac
+OSX is vindicated. ;-)
+
+So, I've been beating the kqueue code into shape for the last 12 hours,
+minus a few hours sleep.
+
+First, I noticed it was seeming to starve the other threads. I'm using
+Haskell's non-threaded runtime, which does cooperative multitasking between
+threads, and my C code was never returning to let the other threads run.
+Changed that around, so the C code runs until SIGALARMed, and then that
+thread calls `yield` before looping back into the C code. Wow, cooperative
+multitasking.. I last dealt with that when programming for Windows 3.1!
+(Should try to use Haskell's -threaded runtime sometime, but git-annex
+doesn't work under it, and I have not tried to figure out why not.)
+
+Then I made a [single commit](http://source.git-annex.branchable.com/?p=source.git;a=commitdiff;h=2bfcc0b09c5dd37c5e0ab65cb089232bfcc31934),
+with no testing, in which I made the kqueue code maintain a cache of what
+it expects in the directory tree, and use that to determine what files
+changed how when a change is detected. Serious code. It worked on the
+first go. If you were wondering why I'm writing in Haskell ... yeah,
+that's why.
+
+And I've continued to hammer on the kqueue code, making lots of little
+fixes, and at this point it seems *almost* able to handle the changes I
+throw at it. It does have one big remaining problem; kqueue doesn't tell me
+when a writer closes a file, so it will sometimes miss adding files. To fix
+this, I'm going to need to make it maintain a queue of new files, and
+periodically check them, with `lsof`, to see when they're done being
+written to, and add them to the annex. So while a file is being written
+to, `git annex watch` will have to wake up every second or so, and run
+`lsof` ... and it'll take it at least 1 second to notice a file's complete.
+Not ideal, but the best that can be managed with kqueue.
diff --git a/doc/design/assistant/blog/day_140__release_monday.mdwn b/doc/design/assistant/blog/day_140__release_monday.mdwn
new file mode 100644
index 0000000000..9c9ccd166f
--- /dev/null
+++ b/doc/design/assistant/blog/day_140__release_monday.mdwn
@@ -0,0 +1,25 @@
+New release today, the main improvements in this one being WebDAV,
+Box.com, and Amazon glacier support. [[/assistant/release_notes]]
+
+Collected together all the OSX problem reports into one place
+at [[/assistant/OSX]], to make it easier to get an overview of them.
+
+Did some testing of the OSX standalone app and found that it was missing
+some libraries. It seems some new libraries it's using themselves depend on
+other libraries, and `otool -L` doesn't recursively resolve this. 
+
+So I converted the simplistic shell script it was using to install
+libraries into a haskell progream that recursively adds libraries until
+there are no more to add. It's pulling in quite a lot more libraries now.
+This may fix some of the problems that have been reported with the
+standalone app; I don't really know since I can only do very limited
+testing on OSX.
+
+Still working on getting the standalone builds for this release done,
+should be done by the end of today.
+
+Also found a real stinker of a bug in `dirContentsRecursive`, which was
+just completely broken, apparently since day 1. Fixing that has certainly
+fixed buggy behavior of `git annex import`. It seems that the other
+user of it, the transfer log code, luckily avoided the deep directory
+trees that triggered the bug.
diff --git a/doc/design/assistant/blog/day_141__release_tuesday.mdwn b/doc/design/assistant/blog/day_141__release_tuesday.mdwn
new file mode 100644
index 0000000000..41c323f01e
--- /dev/null
+++ b/doc/design/assistant/blog/day_141__release_tuesday.mdwn
@@ -0,0 +1,6 @@
+I had planned to do nothing today; I can't remember the last time I did
+that. Twas not to be; instead I had to make a new release to fix
+a utterly stupid typo in the rsync special remote. I'm also seeing
+some slightly encouraging signs of the OSX app being closer to working
+and this release has a further fix toward that end; unsetting all the
+environment variables before running the system's web browser.
diff --git a/doc/design/assistant/blog/day_141__release_tuesday/comment_1_a5adea7a726df12f9121c744a036f08d._comment b/doc/design/assistant/blog/day_141__release_tuesday/comment_1_a5adea7a726df12f9121c744a036f08d._comment
new file mode 100644
index 0000000000..69fb647781
--- /dev/null
+++ b/doc/design/assistant/blog/day_141__release_tuesday/comment_1_a5adea7a726df12f9121c744a036f08d._comment
@@ -0,0 +1,10 @@
+[[!comment format=mdwn
+ username="https://www.google.com/accounts/o8/id?id=AItOawnbBRfl5F8gKRr1ko8Ai6FbEZStXXNF1S4"
+ nickname="Áron"
+ subject="deb package for amd64"
+ date="2012-12-04T08:51:55Z"
+ content="""
+Hey, great work!
+Will we also have 64bit binaries in debian unstable soon?
+
+"""]]
diff --git a/doc/design/assistant/blog/day_142__filling_in.mdwn b/doc/design/assistant/blog/day_142__filling_in.mdwn
new file mode 100644
index 0000000000..d7f5c84bf0
--- /dev/null
+++ b/doc/design/assistant/blog/day_142__filling_in.mdwn
@@ -0,0 +1,9 @@
+Just filling in a few remaining bits and pieces from this month's work.
+
+* Made the assistant periodically check glacier-cli for archives that are
+  ready, and queue downloads of them.
+* The box.com configurator defaults to embedding the account info, allowing
+  one-click enabling of the repository. There's a check box to control this.
+* Fix some bugs with how the standalone images run git-annex.
+* Included ssh in the standalone images.
+* Various other bug fies.
diff --git a/doc/design/assistant/blog/day_143__what_next.mdwn b/doc/design/assistant/blog/day_143__what_next.mdwn
new file mode 100644
index 0000000000..6c277f7a26
--- /dev/null
+++ b/doc/design/assistant/blog/day_143__what_next.mdwn
@@ -0,0 +1,22 @@
+Yesterday, I woke up and realized I didn't know what I needed to work on in
+git-annex. Added a poll, [[polls/Android]] to help me decide what major
+thing to work on next.
+
+----
+
+More flailing at the OSX monster. (A species of gelatinous cube?)
+Current fun seems to involve git processes spinning if git-annex
+was started without a controlling TTY. I'm befuddled by this.
+
+----
+
+Made the S3 and Glacier configurators have a list of regions, rather than
+requiring a the region's code be entered correctly. I could not find a list of
+regions, or better, an API to get a list, so I'll have to keep updating as
+Amazon adds services in new regions.
+
+----
+
+Spent some time trying to get WebDAV to work with livedrive.com.
+It doesn't like empty PROPPATCH. I've developed a change to the haskell DAV
+library that will let me avoid this problem.
diff --git a/doc/design/assistant/blog/day_143__what_next/comment_1_40cf25a2ebdd43d8974a28e180e100e5._comment b/doc/design/assistant/blog/day_143__what_next/comment_1_40cf25a2ebdd43d8974a28e180e100e5._comment
new file mode 100644
index 0000000000..28ef42975d
--- /dev/null
+++ b/doc/design/assistant/blog/day_143__what_next/comment_1_40cf25a2ebdd43d8974a28e180e100e5._comment
@@ -0,0 +1,13 @@
+[[!comment format=mdwn
+ username="http://1id.com/contact/=tp"
+ nickname="Tom"
+ subject="In case it might be helpful"
+ date="2012-12-04T14:25:14Z"
+ content="""
+S3 and Glacier regions.  
+  
+
+
+(EC2) Region list API - likely a superset of the other service regions, could be used for sanity checks.  
+
+"""]]
diff --git a/doc/design/assistant/blog/day_143__what_next/comment_2_af9ccbbc5131e6333c029415141bdb51._comment b/doc/design/assistant/blog/day_143__what_next/comment_2_af9ccbbc5131e6333c029415141bdb51._comment
new file mode 100644
index 0000000000..b9df466cf7
--- /dev/null
+++ b/doc/design/assistant/blog/day_143__what_next/comment_2_af9ccbbc5131e6333c029415141bdb51._comment
@@ -0,0 +1,10 @@
+[[!comment format=mdwn
+ username="https://www.google.com/accounts/o8/id?id=AItOawm5vDem1yeIu6uith5pxfb4mdKdIWVJpCs"
+ nickname="Louis"
+ subject="AWS regions in JSON"
+ date="2012-12-05T03:27:52Z"
+ content="""
+Mitch Garnaat maintains a json data file with AWS service regions & other useful data...
+
+https://github.com/garnaat/missingcloud/blob/master/aws.json
+"""]]
diff --git a/doc/design/assistant/blog/day_144__webapp_work.mdwn b/doc/design/assistant/blog/day_144__webapp_work.mdwn
new file mode 100644
index 0000000000..4e0c8287fa
--- /dev/null
+++ b/doc/design/assistant/blog/day_144__webapp_work.mdwn
@@ -0,0 +1,8 @@
+Made the webapp show runtime errors on a prettified page that includes version
+info, a bug reporting link, etc.
+
+Dealt with a bad interaction between required fields and the bootstrap modals
+displayed when submitting some configuration forms. This was long, complex,
+and had lots of blind alleys. In the end, I had to derive new password and
+text fields in yesod that don't set the required attribute in the generated
+html.
diff --git a/doc/design/assistant/blog/day_145__more_webapp_work.mdwn b/doc/design/assistant/blog/day_145__more_webapp_work.mdwn
new file mode 100644
index 0000000000..f9649be39d
--- /dev/null
+++ b/doc/design/assistant/blog/day_145__more_webapp_work.mdwn
@@ -0,0 +1,12 @@
+One problem with the current configurators for remotes is they have a lot
+of notes and text to read at the top. I've worked on cut that down somewhat,
+mostly by adding links and help buttons next to fields in the form.
+
+I also made each form have a check box controlling whether encryption is
+enabled. Mostly as a way to declutter the text at the top, which always had
+to say encryption is enabled.
+
+---
+
+I have a fairly well worked out design for [[desymlink]]. Will wait a few
+days to work on it to let it jell.
diff --git a/doc/design/assistant/blog/day_146__meanwhile.mdwn b/doc/design/assistant/blog/day_146__meanwhile.mdwn
new file mode 100644
index 0000000000..8ff8b1d723
--- /dev/null
+++ b/doc/design/assistant/blog/day_146__meanwhile.mdwn
@@ -0,0 +1,22 @@
+Biding my time while [[desymlink]] gells in my head..
+
+Fixed a bug in the assistant's local pairing that rejected ssh keys with a
+period in the comment.
+
+Fixed a bug in the assistant that made it try to drop content from remotes
+that didn't have it, and avoided a drop failure crashing a whole assistant
+thread.
+
+Made --auto behave better when preferred content is set.
+
+Looked into making the transfer queue allow running multiple transfers at
+the same time, ie, one per remote. This seems to essentially mean splitting
+the queue into per remote queues. There are some complexities, and I
+decided not to dive into working through it right now, since it'd be
+a distraction from thinking about [[desymlink]]. Will revisit it later.
+
+Allow specifying a port when setting up a ssh remote.
+
+While doing that, noticed that the assistant fails to transfer files to
+sync to a ssh remote that was just added. That got broken while optimising
+reconnecting with a remote; fixed it.
diff --git a/doc/design/assistant/blog/day_147__direct_mode.mdwn b/doc/design/assistant/blog/day_147__direct_mode.mdwn
new file mode 100644
index 0000000000..1a7523785b
--- /dev/null
+++ b/doc/design/assistant/blog/day_147__direct_mode.mdwn
@@ -0,0 +1,36 @@
+Started laying the groundwork for [[desymlink]]'s direct mode.
+I got rather far!
+
+A git repo can be configured with `annex.direct` and all actions that
+transfer objects to it will replace the symlinks with regular files.
+Removing objects also works (and puts back a broken symlink),
+as does checking if an object is present, which even detects if a file
+has been modified.
+
+So far, this works best when such a direct mode repository is used as a git
+remote of another repository. It is also possible to run a few git-annex
+commands, like "get" in a direct mode repository, though others, like
+"drop" won't work because they rely on the symlink to map back to the key.
+
+Direct mode relies on map files existing for each key in the repository, that tell
+what file(s) use it. It also relies on cache files, that contain the last
+known mtime, size, and inode of the file. So far, I've been setting these
+files up by hand.
+
+The main thing that's missing is support for transferring objects from
+direct mode repositories. There's no single place I can modify to support
+that (like I could for the stuff mentioned above), and also it's difficult
+to do safely, since files could be modified at any time. 
+
+So it'll need to quarantine files, to prevent a modified version from
+getting sent out. I could either do this by copying the file, or by
+temporarily `git annex lock`ing it. Unsure which choice would be less
+annoying..
+
+----
+
+Also did some investigation with Jimmy of the OSX app git-config hang.
+Seems to be some kind of imcompatability between the 10.7 autobuilder and
+10.8. Next step is probably to try to build on 10.8. Might also
+be worth trying , although my
+own scripts do more or less the same thing to build the app.
diff --git a/doc/design/assistant/blog/day_147__direct_mode/comment_1_0bd69532afce9dc04e3d88bfd0aed4b2._comment b/doc/design/assistant/blog/day_147__direct_mode/comment_1_0bd69532afce9dc04e3d88bfd0aed4b2._comment
new file mode 100644
index 0000000000..567c094c9f
--- /dev/null
+++ b/doc/design/assistant/blog/day_147__direct_mode/comment_1_0bd69532afce9dc04e3d88bfd0aed4b2._comment
@@ -0,0 +1,16 @@
+[[!comment format=mdwn
+ username="http://lj.rossia.org/users/imz/"
+ ip="79.165.59.119"
+ subject=""removing"  vs drop"
+ date="2012-12-12T13:20:42Z"
+ content="""
+I don't understand the difference behind:
+
+> Removing objects also works (and puts back a broken symlink)
+
+and
+
+> \"drop\" won't work because they rely on the symlink to map back to the key.
+
+If a file is removed (its content, which is replaced by a symlink), then it's not present there, so effectively it should be counted as \"dropped\" at this place. So, removing a file without counting it as dropped is something inconsistent, isn't it? Do I misunderstand something?
+"""]]
diff --git a/doc/design/assistant/blog/day_147__direct_mode/comment_2_3b26f0d081c3bf1037bb872d529ce825._comment b/doc/design/assistant/blog/day_147__direct_mode/comment_2_3b26f0d081c3bf1037bb872d529ce825._comment
new file mode 100644
index 0000000000..b29d453909
--- /dev/null
+++ b/doc/design/assistant/blog/day_147__direct_mode/comment_2_3b26f0d081c3bf1037bb872d529ce825._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="http://joeyh.name/"
+ ip="4.153.8.117"
+ subject="comment 2"
+ date="2012-12-12T23:45:42Z"
+ content="""
+`git annex drop` is a user-level operation built on top of lower-level object removal functions that are also used by other things.
+"""]]
diff --git a/doc/design/assistant/blog/day_148__direct_mode.mdwn b/doc/design/assistant/blog/day_148__direct_mode.mdwn
new file mode 100644
index 0000000000..14f1f889f8
--- /dev/null
+++ b/doc/design/assistant/blog/day_148__direct_mode.mdwn
@@ -0,0 +1,42 @@
+Got object sending working in direct mode. However, I don't yet have a
+reliable way to deal with files being modified while they're being
+transferred. I have code that detects it on the sending side, but the
+receiver is still free to move the wrong content into its annex, and record
+that it has the content. So that's not acceptable, and I'll need to work
+on it some more. However, at this point I can use a direct mode repository
+as a remote and transfer files from and to it.
+
+----
+
+Automated updating of the cached mtime, etc data. Next I need to automate
+generation of the key to filename mapping files. I'm thinking that I'll make
+`git annex sync` do it. Then, once I get committing and 
+merging working in direct mode repositories (which is likely to be a
+good week's worth of work), the workflow for using these repositories
+will be something like this:
+
+	git config annex.direct true
+	git annex sync # pulls any changes, merges, updates maps and caches
+	git annex get
+	# modify files
+	git annex sync # commits and pushes changes
+
+And once I get direct mode repositories working to this degree at the
+command line, I can get on with adding support to the assistant.
+
+----
+
+Also did some more work today on the OSX app. Am in the middle of getting
+it to modify the binaries in the app to change the paths to the libraries they
+depend on. This will avoid the hacky environment variable it is currently
+using, and make runshell a much more usable environment. It's the right way
+to do it. (I can't believe I just said RPATH was the right way to do
+anything.)
+
+In the middle of this, I discovered
+, which does the same
+type of thing.
+
+Anyway, I have to do some crazy hacks to work around short library name
+fields in executables that I don't want to have to be specially rebuilt in
+order to build the webapp. Like git.
diff --git a/doc/design/assistant/blog/day_149__rainy_day.mdwn b/doc/design/assistant/blog/day_149__rainy_day.mdwn
new file mode 100644
index 0000000000..eb42108843
--- /dev/null
+++ b/doc/design/assistant/blog/day_149__rainy_day.mdwn
@@ -0,0 +1,15 @@
+Made `git annex sync` update the file mappings in direct mode.
+To do this efficiently, it uses `git diff-tree` to find files that are
+changed by the sync, and only updates those mappings. I'm rather happy
+with this, as a first step to fully supporting sync in direct mode.
+
+Finished the overhaul of the OSX app's library handling. It seems to work
+well, and will fix a whole class of ways the OSX app could break.
+
+Fixed a bug in the preferred content settings for backup repositories,
+introduced by some changes I made to preferred content handling 4 days ago.
+
+Fixed the Debian package to build with WebDAV support, which I forgot to
+turn on before.
+
+Planning a release tomorrow.
diff --git a/doc/design/assistant/blog/day_14__kqueue_kqueue_kqueue.mdwn b/doc/design/assistant/blog/day_14__kqueue_kqueue_kqueue.mdwn
new file mode 100644
index 0000000000..d7dab36117
--- /dev/null
+++ b/doc/design/assistant/blog/day_14__kqueue_kqueue_kqueue.mdwn
@@ -0,0 +1,23 @@
+... I'm getting tired of kqueue.
+
+But the end of the tunnel is in sight. Today I made git-annex handle files
+that are still open for write after a kqueue creation event is received.
+Unlike with inotify, which has a new event each time a file is closed,
+kqueue only gets one event when a file is first created, and so git-annex
+needs to retry adding files until there are no writers left.
+
+Eventually I found an elegant way to do that. The committer thread already
+wakes up every second as long as there's a pending change to commit. So for
+adds that need to be retried, it can just push them back onto the change
+queue, and the committer thread will wait one second and retry the add. One
+second might be too frequent to check, but it will do for now.
+
+This means that `git annex watch` should now be usable on OSX, FreeBSD, and
+NetBSD! (It'll also work on Debian kFreeBSD once [lsof is ported to it](http://bugs.debian.org/589103).)
+I've meged kqueue support to `master`.
+
+I also think I've squashed the empty commits that were sometimes made.
+
+Incidentally, I'm 50% through my first month, and finishing [[inotify]]
+was the first half of my roadmap for this month. Seem to be right on
+schedule.. Now I need to start thinking about [[syncing]].
diff --git a/doc/design/assistant/blog/day_14__thinking_about_syncing.mdwn b/doc/design/assistant/blog/day_14__thinking_about_syncing.mdwn
new file mode 100644
index 0000000000..4173fbf777
--- /dev/null
+++ b/doc/design/assistant/blog/day_14__thinking_about_syncing.mdwn
@@ -0,0 +1,44 @@
+Pondering [[syncing]] today. I will be doing syncing of the git repository
+first, and working on syncing of file data later.
+
+The former seems straightforward enough, since we just want to push all
+changes to everywhere. Indeed, git-annex already has a [[sync]] command
+that uses a smart technique to allow syncing between clones without a
+central bare repository. (Props to Joachim Breitner for that.)
+
+But it's not all easy. Syncing should happen as fast as possible, so
+changes show up without delay. Eventually it'll need to support syncing
+between nodes that cannot directly contact one-another. Syncing needs to
+deal with nodes coming and going; one example of that is a USB drive being
+plugged in, which should immediately be synced, but network can also come
+and go, so it should periodically retry nodes it failed to sync with. To
+start with, I'll be focusing on fast syncing between directly connected
+nodes, but I have to keep this wider problem space in mind.
+
+One problem with `git annex sync` is that it has to be run in both clones
+in order for changes to fully propagate. This is because git doesn't allow
+pushing changes into a non-bare repository; so instead it drops off a new
+branch in `.git/refs/remotes/$foo/synced/master`. Then when it's run locally
+it merges that new branch into `master`. 
+
+So, how to trigger a clone to run `git annex sync` when syncing to it?
+Well, I just realized I have spent two weeks developing something that can
+be repurposed to do that! [[Inotify]] can watch for changes to
+`.git/refs/remotes`, and the instant a change is made, the local sync
+process can be started. This avoids needing to make another ssh connection
+to trigger the sync, so is faster and allows the data to be transferred
+over another protocol than ssh, which may come in handy later.
+
+So, in summary, here's what will happen when a new file is created:
+
+1. inotify event causes the file to be added to the annex, and
+   immediately committed.
+2. new branch is pushed to remotes (probably in parallel)
+3. remotes notice new sync branch and merge it
+4. (data sync, TBD later)
+5. file is fully synced and available
+
+Steps 1, 2, and 3 should all be able to be accomplished in under a second.
+The speed of `git push` making a ssh connection will be the main limit
+to making it fast. (Perhaps I should also reuse git-annex's existing ssh
+connection caching code?)
diff --git a/doc/design/assistant/blog/day_150__12__58__12.mdwn b/doc/design/assistant/blog/day_150__12__58__12.mdwn
new file mode 100644
index 0000000000..8e1f192dba
--- /dev/null
+++ b/doc/design/assistant/blog/day_150__12__58__12.mdwn
@@ -0,0 +1,53 @@
+Yesterday I cut another release. However, getting an OSX build took until
+12:12 pm today because of a confusion about the location of lsof on OSX. The
+OSX build is now available, and I'm looking forward to hearing if it's working!
+
+----
+
+Today I've been working on making `git annex sync` commit in direct mode.
+
+For this I needed to find all new, modified, and deleted files, and I also
+need the git SHA from the index for all non-new files. There's not really
+an ideal git command to use to query this. For now I'm using
+`git ls-files --others --stage`, which works but lists more files than I
+really need to look at. It might be worth using one of the Haskell libraries
+that can directly read git's index.. but for now I'll stick with `ls-files`.
+
+It has to check all direct mode files whose content is present, which means
+one stat per file (on top of the stat that git already does), as well as one
+retrieval of the key per file (using the single `git cat-file` process that
+git-annex talks to).
+
+This is about as efficient as I can make it, except that unmodified
+annexed files whose content is not present are listed due to --stage,
+and so it has to stat those too, and currently also feeds them into `git add`.
+
+The assistant will be able to avoid all this work, except once at startup.
+
+Anyway, direct mode committing is working!
+
+For now, `git annex sync` in direct mode also adds new files. This because
+`git annex add` doesn't work yet in direct mode.
+
+It's possible for a direct mode file to be changed during a commit,
+which would be a problem since committing involves things like calculating
+the key and caching the mtime/etc, that would be screwed up. I took
+care to handle that case; it checks the mtime/etc cache before and after
+generating a key for the file, and if it detects the file has changed,
+avoids committing anything. It could retry, but if the file is a VM disk
+image or something else that's constantly modified, commit retrying forever
+would not be good.
+
+----
+
+For `git annex sync` to be usable in direct mode, it still needs
+to handle merging. It looks like I may be able to just enhance the automatic
+conflict resolution code to know about typechanged direct mode files.
+
+The other missing piece before this can really be used is that currently
+the key to file mapping is only maintained for files added locally, or
+that come in via `git annex sync`. Something needs to set up that mapping
+for files present when the repo is initally cloned. Maybe the thing
+to do is to have a `git annex directmode` command that enables/disables
+direct mode and can setup the the mapping, as well as any necessary unlocks
+and setting the trust level to untrusted.
diff --git a/doc/design/assistant/blog/day_151__direct_mode_toggle.mdwn b/doc/design/assistant/blog/day_151__direct_mode_toggle.mdwn
new file mode 100644
index 0000000000..c0b3f32459
--- /dev/null
+++ b/doc/design/assistant/blog/day_151__direct_mode_toggle.mdwn
@@ -0,0 +1,59 @@
+Built `git annex direct` and `git annex indirect` to toggle back and forth
+between direct mode. Made `git annex status` show if the repository is in
+direct mode. Now *only* merging is needed for direct mode to be basically
+usable. 
+
+I can do a little demo now. Pay attention to the "@" ls shows at the end
+of symlinks.
+
+	joey@gnu:~/tmp/bench/rdirect>ls
+	myfile@  otherfile@
+	joey@gnu:~/tmp/bench/rdirect>git annex find
+	otherfile
+	# So, two files, only one present in this repo.
+
+	joey@gnu:~/tmp/bench/rdirect>git annex direct
+	commit  
+	# On branch master
+	# Your branch is ahead of 'origin/master' by 7 commits.
+	#
+	nothing to commit (working directory clean)
+	ok
+	direct myfile ok
+	direct otherfile ok
+	direct  ok
+
+	joey@gnu:~/tmp/bench/rdirect>ls
+	myfile@  otherfile
+	# myfile is still a broken symlink because we don't have its content
+	joey@gnu:~/tmp/bench/rdirect>git annex get myfile
+	get myfile (from origin...) ok
+	(Recording state in git...)
+	joey@gnu:~/tmp/bench/rdirect>ls
+	myfile  otherfile
+
+	joey@gnu:~/tmp/bench/rdirect>echo "look mom, no symlinks" >> myfile
+	joey@gnu:~/tmp/bench/rdirect>git annex sync
+	add myfile (checksum...) ok
+	commit  
+	(Recording state in git...)
+	[master 0e8de9b] git-annex automatic sync
+	...
+	ok
+
+	joey@gnu:~/tmp/bench/rdirect>git annex indirect
+	commit  ok
+	indirect myfile ok
+	indirect otherfile ok
+	indirect  ok
+	joey@gnu:~/tmp/bench/rdirect>ls
+	myfile@  otherfile@
+
+I'd like `git annex direct` to set the repository to untrusted, but
+I didn't do it. Partly because having `git annex indirect` set it back to
+semitrusted seems possibly wrong -- the user might not trust a repo even in
+indirect mode. Or might fully trust it. The docs will encourage users to
+set direct mode repos to untrusted -- in direct mode you're operating
+without large swathes of git-annex's carefully constructed safety net.
+(When the assistant later uses direct mode, it'll untrust the repository
+automatically.)
diff --git a/doc/design/assistant/blog/day_152__bugfixes.mdwn b/doc/design/assistant/blog/day_152__bugfixes.mdwn
new file mode 100644
index 0000000000..da70fd1565
--- /dev/null
+++ b/doc/design/assistant/blog/day_152__bugfixes.mdwn
@@ -0,0 +1,18 @@
+Fixed a bug in the kqueue code that made the assistant not notice when a
+file was renamed into a subdirectory. This turned out to be because the
+symlink got broken, and it was using `stat` on the file. Switching to
+`lstat` fixed that.
+
+Improved installation of programs into standalone bundles. Now it uses
+the programs detected by configure, rather than a separate hardcoded list.
+Also improved handling of lsof, which is not always in PATH.
+
+Made a OSX 10.8.2 build of the app, which is nearly my last gasp attempt
+at finding a way around this crazy `git init` spinning problem with Jimmy's
+daily builds are used with newer OSX versions. Try it here:
+
+
+----
+
+Mailed out the Kickstarter T-shirt rewards today, to people in the US.
+Have to fill out a bunch of forms before I can mail the non-US ones.
diff --git a/doc/design/assistant/blog/day_152__bugfixes/comment_1_46863a875f9daa6f2c9248b66ff91929._comment b/doc/design/assistant/blog/day_152__bugfixes/comment_1_46863a875f9daa6f2c9248b66ff91929._comment
new file mode 100644
index 0000000000..9199f3768e
--- /dev/null
+++ b/doc/design/assistant/blog/day_152__bugfixes/comment_1_46863a875f9daa6f2c9248b66ff91929._comment
@@ -0,0 +1,9 @@
+[[!comment format=mdwn
+ username="https://www.google.com/accounts/o8/id?id=AItOawmYiJgOvC4IDYkr2KIjMlfVD9r_1Sij_jY"
+ nickname="Douglas"
+ subject="OSX troubles"
+ date="2012-12-17T16:07:41Z"
+ content="""
+Have you considered making a homebrew installable version of git-annex? It may make things easier for you when it comes to build dependencies. 
+
+"""]]
diff --git a/doc/design/assistant/blog/day_152__bugfixes/comment_2_a586e617bc024c8a9ff60f1b8345d74d._comment b/doc/design/assistant/blog/day_152__bugfixes/comment_2_a586e617bc024c8a9ff60f1b8345d74d._comment
new file mode 100644
index 0000000000..af7a901a9b
--- /dev/null
+++ b/doc/design/assistant/blog/day_152__bugfixes/comment_2_a586e617bc024c8a9ff60f1b8345d74d._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="http://joeyh.name/"
+ ip="4.153.8.117"
+ subject="comment 2"
+ date="2012-12-17T16:16:47Z"
+ content="""
+I would *love* *love* *love* for there to be a homebrew for git-annex. But I know nothing about homebrew so am not in a very good position to do it.
+"""]]
diff --git a/doc/design/assistant/blog/day_153__hibernation.mdwn b/doc/design/assistant/blog/day_153__hibernation.mdwn
new file mode 100644
index 0000000000..5406fcb868
--- /dev/null
+++ b/doc/design/assistant/blog/day_153__hibernation.mdwn
@@ -0,0 +1,26 @@
+As winter clouds set in, I have to ration my solar power and have been less
+active than usual.
+
+It seems that the OSX 10.8.2 `git init` hanging issue has indeed been
+resolved, by building the app on 10.8.2. Very good news!  Autobuilder setup is
+in progress.
+
+----
+
+Finally getting stuck in to direct mode git-merge handling. It's
+not possible to run `git merge` in a direct mode tree, because it'll
+see typechanged files and refuse to do anything.
+
+So the only way to use `git merge`, rather than writing my own merge engine,
+is to use `--work-tree` to make it operate in a temporary work tree directory
+rather than the real one.
+
+When it's run this way, any new, modified, or renamed files will be added
+to the temp dir, and will need to be moved to the real work tree.
+To detect deleted files, need to use `git ls-files --others`, and
+look at the old tree to see if the listed files were in it.
+
+When a merge conflict occurs, the new version of the file will be in the temp
+directory, and the old one in the work tree. The normal automatic merge
+conflict resolution machinery should work, with just some tweaks to handle
+direct mode.
diff --git a/doc/design/assistant/blog/day_154__direct_mode_merging.mdwn b/doc/design/assistant/blog/day_154__direct_mode_merging.mdwn
new file mode 100644
index 0000000000..c3233c36dd
--- /dev/null
+++ b/doc/design/assistant/blog/day_154__direct_mode_merging.mdwn
@@ -0,0 +1,21 @@
+Got merging working in direct mode!
+
+Basically works as outlined yesterday, although slightly less clumsily.
+Since there was already code that ran `git diff-tree` to update the
+associated files mappings after a merge, I was able to adapt that same code
+to also update the working tree.
+
+An important invariant for direct mode merges is that they should never
+cause annexed objects to be dropped. So if a file is deleted by a merge,
+and was a direct mode file that was the only place in the working copy
+where an object was stored, the object is moved into `.git/annex/objects`.
+This avoids data loss and any need to re-transfer objects after a merge.
+It also makes renames and other move complex tree manipulations always end
+up with direct mode files, when their content was present.
+
+Automatic merge conflict resoltion doesn't quite work right yet in direct
+mode.
+
+Direct mode has landed in the `master` branch, but I still consider it
+experimental, and of course the assistant still needs to be updated to
+support it.
diff --git a/doc/design/assistant/blog/day_155__bugfixes.mdwn b/doc/design/assistant/blog/day_155__bugfixes.mdwn
new file mode 100644
index 0000000000..42a95cc5a8
--- /dev/null
+++ b/doc/design/assistant/blog/day_155__bugfixes.mdwn
@@ -0,0 +1,15 @@
+Finished getting automatic merge conflict resolution working in direct
+mode. Turned out I was almost there yesterday, just a bug in a filename
+comparison needed to be fixed.
+
+Fixed a bug where the assistant dropped a file after transferring it,
+despite the preferred content settings saying it should keep its copy of
+the file. This turned out to be due to it reading the transfer info
+incorrectly, and adding a "\n" to the end of the filename, which caused the
+preferred content check to think it wasn't wanted after all. (Probably
+because it thought 0 copies of the file were wanted, but I didn't look into
+this in detail.)
+
+Worked on my test suite, particularly more QuickCheck tests. I need to
+use QuickCheck more, particularly when I've pairs of functions, like encode
+and decode, that make for easy QuickCheck properties.
diff --git a/doc/design/assistant/blog/day_156_and_157__direct_mode_assistant.mdwn b/doc/design/assistant/blog/day_156_and_157__direct_mode_assistant.mdwn
new file mode 100644
index 0000000000..f49d2914f6
--- /dev/null
+++ b/doc/design/assistant/blog/day_156_and_157__direct_mode_assistant.mdwn
@@ -0,0 +1,45 @@
+Over Christmas, I'm working on making the assistant support direct
+mode. I like to have a fairly detailed plan before starting this kind of
+job, but in this case, I don't. (Also I have a cold so planning? Meh.)
+This is a process of seeing what's broken in direct mode and fixing it.
+I don't know if it'll be easy or hard. Let's find out..
+
+* First, got direct mode adding of new files working. This was not hard, all the
+  pieces I needed were there. For now, it uses the same method as in
+  indirect mode to make sure nothing can modify the file while it's being
+  added.
+
+* An unexpected problem is that in its startup scan, the assistant runs
+  `git add --update` to detect and stage any deletions that happened
+  while it was not running. But in direct mode that also stages the full file
+  contents, so can't be used. Had to switch to using git plumbing to only
+  stage deleted files. Happily this also led to fixing a bug; deletions
+  were not always committed at startup when using the old method; with the
+  new method it can tell when there are deletions and trigger a commit.
+
+* Next, got it to commit when direct mode files are modified. The Watcher
+  thread gets a inotify event when this happens, so that was easy. (Although
+  in doing that I had to disable a guard in direct mode that made annexed
+  files co-exist with regular in-git files, so such mixed repositories
+  probably won't work in direct mode yet.)
+
+  However, naughty kqueue is another story, there are no kqueue events for
+  file modifications. So this won't work on OSX or the BSDs yet. I tried
+  setting some more kqueue flags in hope that one would make such events
+  appear, but no luck. Seems I will need to find some other method to detect
+  file modifications, possibly an OSX-specific API.
+
+* Another unexpected problem: When an assistant receives new files from one
+  of its remotes, in direct mode it still sets up symlinks to the content.
+  This was because the Merger thread didn't use the `sync` command's direct
+  mode aware merge code.. so fixed that.
+
+* Finally there was some direct mode bookeeping the assistant has
+  to get right. For example, when a file is modified, the old object has
+  to be looked up, and be marked as not locally present any longer. That
+  lookup relies on the already running `git cat-file --batch`, so it's
+  not as fast as it could be, if I kept a local cache of the mapping
+  between files and objects. But it seems fast enough for now.
+
+At this point the assistant seems to work in direct mode on Linux!
+Needs more testing..
diff --git a/doc/design/assistant/blog/day_158__fsevents.mdwn b/doc/design/assistant/blog/day_158__fsevents.mdwn
new file mode 100644
index 0000000000..472b6c2be5
--- /dev/null
+++ b/doc/design/assistant/blog/day_158__fsevents.mdwn
@@ -0,0 +1,20 @@
+Investigated using the OSX fsevents API to detect when files are modified,
+so they can be committed when using direct mode. There's a
+[haskell library](http://hackage.haskell.org/package/hfsevents-0.1.3)
+and even a [sample directory watching program](http://hackage.haskell.org/package/hobbes).
+Initial tests look good...
+
+Using fsevents will avoid kqueue's problems with needing enough file
+descriptors to open every subdirectory. kqueue is a rather poor match for
+git-annex's needs, really. It does not seem to provide events for file
+modifications at all, unless every *file* is individually opened. While I
+dislike leaving the BSD's out, they need a better interface to be perfectly
+supported by git-annex, and kqueue will still work for indirect mode
+repositories.
+
+----
+
+Got the assistant to use fsevents. It seems to work well!
+
+The only problem I know of is that it doesn't yet handle whole directory
+renames. That should be easy to fix later.
diff --git a/doc/design/assistant/blog/day_158__fsevents/comment_1_b278372ac6399f64d5fa9da178278a6d._comment b/doc/design/assistant/blog/day_158__fsevents/comment_1_b278372ac6399f64d5fa9da178278a6d._comment
new file mode 100644
index 0000000000..a13936ed55
--- /dev/null
+++ b/doc/design/assistant/blog/day_158__fsevents/comment_1_b278372ac6399f64d5fa9da178278a6d._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="https://www.google.com/accounts/o8/id?id=AItOawkey8WuXUh_x5JC2c9_it1CYRnVTgdGu1M"
+ nickname="Dustin"
+ subject="Linux Support"
+ date="2012-12-27T21:37:45Z"
+ content="""
+Does this mean Linux will not support Direct mode?
+"""]]
diff --git a/doc/design/assistant/blog/day_158__fsevents/comment_2_2d5ce9b2807068c3517e185945662bd2._comment b/doc/design/assistant/blog/day_158__fsevents/comment_2_2d5ce9b2807068c3517e185945662bd2._comment
new file mode 100644
index 0000000000..edd63b1251
--- /dev/null
+++ b/doc/design/assistant/blog/day_158__fsevents/comment_2_2d5ce9b2807068c3517e185945662bd2._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="http://joeyh.name/"
+ ip="4.154.0.127"
+ subject="comment 2"
+ date="2012-12-28T02:46:23Z"
+ content="""
+No, linux's inotify has everything I need for direct mode.
+"""]]
diff --git a/doc/design/assistant/blog/day_159__fsevents_and_assistant.mdwn b/doc/design/assistant/blog/day_159__fsevents_and_assistant.mdwn
new file mode 100644
index 0000000000..754e2c91fb
--- /dev/null
+++ b/doc/design/assistant/blog/day_159__fsevents_and_assistant.mdwn
@@ -0,0 +1,16 @@
+Short day today, but I spent it all on testing the new FSEvents code,
+getting it working with the assistant in direct mode. This included fixing
+its handling of renaming, and various other bugs.
+
+The assistant in direct mode now seems to work well on OSX. So I made
+the assistant *default* to making direct mode repositories on OSX.
+
+That'll presumably flush out any bugs. :) More importantly,
+it let me close several OSX-specific bugs to do with interactions between
+git-annex's symlinks and OSX programs that were apparently written under the
+misprehension that it's a user-mode program's job to manually follow symlinks.
+
+Of course, defaulting to direct mode also means users can just modify files
+as they like and the assistant will commit and sync the changed files.
+I'm waiting to see if direct mode becomes popular enough to make it the
+default on all OS's.
diff --git a/doc/design/assistant/blog/day_159__fsevents_and_assistant/comment_1_b85f446c3fa8d703a2a8882825c6f33f._comment b/doc/design/assistant/blog/day_159__fsevents_and_assistant/comment_1_b85f446c3fa8d703a2a8882825c6f33f._comment
new file mode 100644
index 0000000000..8bc089857c
--- /dev/null
+++ b/doc/design/assistant/blog/day_159__fsevents_and_assistant/comment_1_b85f446c3fa8d703a2a8882825c6f33f._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="https://www.google.com/accounts/o8/id?id=AItOawm5iosFbL2By7UFeViqkc6v-hoAtqILeDA"
+ nickname="Laszlo"
+ subject="comment 1"
+ date="2012-12-28T23:53:22Z"
+ content="""
+Make a release please, Im eager to try it out!:)
+"""]]
diff --git a/doc/design/assistant/blog/day_159__fsevents_and_assistant/comment_2_a150b404e0c39e0bb2f7dd00cda63cdc._comment b/doc/design/assistant/blog/day_159__fsevents_and_assistant/comment_2_a150b404e0c39e0bb2f7dd00cda63cdc._comment
new file mode 100644
index 0000000000..bc9c7031b0
--- /dev/null
+++ b/doc/design/assistant/blog/day_159__fsevents_and_assistant/comment_2_a150b404e0c39e0bb2f7dd00cda63cdc._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="https://www.google.com/accounts/o8/id?id=AItOawnFhHPFP7j3wGNgBxEJoA8LcBJ4Xd1tTMY"
+ nickname="Joe"
+ subject="Release + Version Number"
+ date="2012-12-29T11:16:05Z"
+ content="""
+I second Laszlo's request for a release! Also, could you include a version number of some sort on the download (and binary if there isn't one there) to make it easy to tell what is available for download vs. what is installed?
+"""]]
diff --git a/doc/design/assistant/blog/day_159__fsevents_and_assistant/comment_3_37abc41bae23a1d7de0d19d952aec492._comment b/doc/design/assistant/blog/day_159__fsevents_and_assistant/comment_3_37abc41bae23a1d7de0d19d952aec492._comment
new file mode 100644
index 0000000000..b08636fd8e
--- /dev/null
+++ b/doc/design/assistant/blog/day_159__fsevents_and_assistant/comment_3_37abc41bae23a1d7de0d19d952aec492._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="https://www.google.com/accounts/o8/id?id=AItOawkSq2FDpK2n66QRUxtqqdbyDuwgbQmUWus"
+ nickname="Jimmy"
+ subject="comment 3"
+ date="2012-12-29T13:26:30Z"
+ content="""
+There are daily/hourly builds at the [[install/OSX]] install page, try out the new features from there
+"""]]
diff --git a/doc/design/assistant/blog/day_15__its_aliiive.mdwn b/doc/design/assistant/blog/day_15__its_aliiive.mdwn
new file mode 100644
index 0000000000..10ef4ffe56
--- /dev/null
+++ b/doc/design/assistant/blog/day_15__its_aliiive.mdwn
@@ -0,0 +1,33 @@
+Syncing works! I have two clones, and any file I create in the first
+is immediately visible in the second. Delete that file from the second, and
+it's immediately removed from the first.
+
+Most of my work today felt like stitching existing limbs onto a pre-existing
+monster. Took the committer thread, that waits for changes and commits them,
+and refashioned it into a pusher thread, that waits for commits and pushes
+them. Took the watcher thread, that watches for files being made,
+and refashioned it into a merger thread, that watches for git refs being
+updated. Pulled in bits of the `git annex sync` command to reanimate this.
+
+It may be a shambling hulk, but it works.
+
+Actually, it's not much of a shambling hulk; I refactored my code after
+copying it. ;)
+
+I think I'm up to 11 threads now in the new
+`git annex assistant` command, each with its own job, and each needing
+to avoid stepping on the other's toes. I did see one MVar deadlock error
+today, which I have not managed to reproduce after some changes. I think
+the committer thread was triggering the merger thread, which probably
+then waited on the Annex state MVar the committer thread had held.
+
+Anyway, it even pushes to remotes in parallel, and keeps track of remotes
+it failed to push to, although as of yet it doesn't do any attempt at
+periodically retrying.
+
+One bug I need to deal with is that the push code assumes any change
+made to the remote has already been pushed back to it. When it hasn't, 
+the push will fail due to not being a fast-forward. I need to make it
+detect this case and pull before pushing.
+
+(I've pushed this work out in a new `assistant branch`.)
diff --git a/doc/design/assistant/blog/day_160__finishing_up_direct_mode.mdwn b/doc/design/assistant/blog/day_160__finishing_up_direct_mode.mdwn
new file mode 100644
index 0000000000..c789ca2f61
--- /dev/null
+++ b/doc/design/assistant/blog/day_160__finishing_up_direct_mode.mdwn
@@ -0,0 +1,10 @@
+A few final bits and pieces of direct mode. Fixed a few more bugs in the
+assistant. Made all git-annex commands that don't work at
+all, or only partially work in direct mode, refuse to run at all. Also,
+some optimisations.
+
+I'll surely need to revisit direct mode later and make more commands
+support it; `fsck` and `add` especially.
+But the only thing I'd like to deal with before I make a release with direct
+mode is the problem of files being able to be modified while they're
+being transferred, which can result in data loss.
diff --git a/doc/design/assistant/blog/day_161__release_day.mdwn b/doc/design/assistant/blog/day_161__release_day.mdwn
new file mode 100644
index 0000000000..5e3a621683
--- /dev/null
+++ b/doc/design/assistant/blog/day_161__release_day.mdwn
@@ -0,0 +1,8 @@
+Released the first git-annex with direct mode today. Notably, the assistant
+enables direct mode in repositories it creates. All builds are updated to
+3.20130102 now.
+
+My plan for this month is to fix whatever things currently might be
+preventing you from using the git-annex assistant. So bugfixes and
+whatever other important gaps need to be filled, but no major new
+feature developments.
diff --git a/doc/design/assistant/blog/day_161__release_day/comment_1_e82c67f3ce216618149537bba1e0b850._comment b/doc/design/assistant/blog/day_161__release_day/comment_1_e82c67f3ce216618149537bba1e0b850._comment
new file mode 100644
index 0000000000..f697d12dcc
--- /dev/null
+++ b/doc/design/assistant/blog/day_161__release_day/comment_1_e82c67f3ce216618149537bba1e0b850._comment
@@ -0,0 +1,19 @@
+[[!comment format=mdwn
+ username="https://www.google.com/accounts/o8/id?id=AItOawm5iosFbL2By7UFeViqkc6v-hoAtqILeDA"
+ nickname="Laszlo"
+ subject="comment 1"
+ date="2013-01-02T21:26:32Z"
+ content="""
+1. Im not able to set direct mode (linux build, Version: 3.20130102 in about page).
+Is there something specific needed? I started fresh. I added a new directory (empty), once 
+git annex assistant started, I copied files to that directory, and the assistant 
+replaced all files with symlinks, just as before.
+What am I missing here?
+
+2. It is just minor usability remark:
+ I would like to see in the gui somewhere shown where the actual config is stored (~/.config/git-annex), 
+and an option to wipe it out and start fresh. Also a restart button.
+
+Best, Laszlo
+
+"""]]
diff --git a/doc/design/assistant/blog/day_161__release_day/comment_2_b1fe96fd818935c0497b78bb8ad32ffa._comment b/doc/design/assistant/blog/day_161__release_day/comment_2_b1fe96fd818935c0497b78bb8ad32ffa._comment
new file mode 100644
index 0000000000..1e44738a15
--- /dev/null
+++ b/doc/design/assistant/blog/day_161__release_day/comment_2_b1fe96fd818935c0497b78bb8ad32ffa._comment
@@ -0,0 +1,14 @@
+[[!comment format=mdwn
+ username="http://joeyh.name/"
+ ip="4.152.108.211"
+ subject="comment 2"
+ date="2013-01-03T17:59:22Z"
+ content="""
+\"I added a new directory (empty)\"
+
+Sounds like you set up the repository yourself, rather than letting the assistant do it. If so, it won't be in direct mode. You can `git annex assistant--stop`, and then `git annex direct` to enable direct mode.
+
+Restart/shutdown in the UI coming soon, I think.
+
+The entire git-annex configuration, include that of the assistant, is stored in the `.git/config` of your reposository, and on its `git-annex` branch. Deleting the repository is thus all that's needed to start over.
+"""]]
diff --git a/doc/design/assistant/blog/day_161__release_day/comment_3_40bac0e1756aa77bb966c4654857141c._comment b/doc/design/assistant/blog/day_161__release_day/comment_3_40bac0e1756aa77bb966c4654857141c._comment
new file mode 100644
index 0000000000..5bb35353b3
--- /dev/null
+++ b/doc/design/assistant/blog/day_161__release_day/comment_3_40bac0e1756aa77bb966c4654857141c._comment
@@ -0,0 +1,44 @@
+[[!comment format=mdwn
+ username="https://www.google.com/accounts/o8/id?id=AItOawm5iosFbL2By7UFeViqkc6v-hoAtqILeDA"
+ nickname="Laszlo"
+ subject="comment 3"
+ date="2013-01-03T22:23:36Z"
+ content="""
+>> I added a new directory (empty)
+
+> Sounds like you set up the repository yourself, rather than letting the assistant do it. 
+> If so, it won't be in direct mode. You can git annex assistant--stop, 
+> and then git annex direct to enable direct mode.
+
+I added like this:
+0. no git-annex or git-annex-assistant is running
+1. create directory (/mnt/dxd/annex)
+2. launch git-annex-assistant
+3. add repository via the webbrowser
+4. copy a file over this(/mnt/dxd/annex) directory (via command line or nautilus)
+
+The file is replaced with symlink.
+
+This fix the issue:
+0. stop git-annex-assistant
+1. Go in the shell to that directory (cd /mnt/dxd/annex)
+2. switch to direct mode (/home/looser/Desktop/download/git-annex.linux/git-annex direct)
+3. start git-annex-assistant
+4. create a new repository(/mnt/dxd/annex2) via the webbrowser (add repository in top right corner)
+5. Now:
+/mnt/dxd/annex - NOT in direct mode (symlinks created instantly)
+/mnt/dxd/annex2 - in direct mode
+
+Im really happy I have finally direct mode running. 
+I will surely stress-test git-annex in the coming weeks.
+
+> Restart/shutdown in the UI coming soon, I think.
+
+Awesome.
+
+> The entire git-annex configuration, include that of the assistant, 
+> is stored in the .git/config of your reposository, and on its git-annex branch. 
+> Deleting the repository is thus all that's needed to start over.
+
+Thank you for the answer.
+"""]]
diff --git a/doc/design/assistant/blog/day_161__release_day/comment_4_af65656b0d1179636937595868bb97b0._comment b/doc/design/assistant/blog/day_161__release_day/comment_4_af65656b0d1179636937595868bb97b0._comment
new file mode 100644
index 0000000000..178e2ead4e
--- /dev/null
+++ b/doc/design/assistant/blog/day_161__release_day/comment_4_af65656b0d1179636937595868bb97b0._comment
@@ -0,0 +1,30 @@
+[[!comment format=mdwn
+ username="https://www.google.com/accounts/o8/id?id=AItOawm5iosFbL2By7UFeViqkc6v-hoAtqILeDA"
+ nickname="Laszlo"
+ subject="comment 4"
+ date="2013-01-03T22:33:54Z"
+ content="""
+Usecase:
+Keep two directories in sync on the same computer.
+
+\"Removable drive\" almost covers this, except you need an usb thumbdrive 
+permanently plugged into the computer.
+
+Would be nice, if I could select a \"sync to this local folder\" option 
+alongside the \"Select drive:\" option.
+
+So it is my choice if I want to sync to a partition or to a folder.
+
+Bug(?):
+I created two repositories:
+/mnt/dxd/annex 
+/mnt/dxd/annex2
+
+And I added a \"Removable drive\" syncing option to /mnt/dxd/annex2, and I selected the \"/mnt/dxd\" drive as removable drive.
+The two repositories got mixed together 
+(ie. broken symlinks pointing to nonexistent .git/annex/objects/.. file)
+
+Best, 
+ Laszlo
+
+"""]]
diff --git a/doc/design/assistant/blog/day_161__release_day/comment_5_0c05caaaf9588e124585041bf5f45d75._comment b/doc/design/assistant/blog/day_161__release_day/comment_5_0c05caaaf9588e124585041bf5f45d75._comment
new file mode 100644
index 0000000000..e461bfdc4e
--- /dev/null
+++ b/doc/design/assistant/blog/day_161__release_day/comment_5_0c05caaaf9588e124585041bf5f45d75._comment
@@ -0,0 +1,20 @@
+[[!comment format=c
+ username="https://www.google.com/accounts/o8/id?id=AItOawm5iosFbL2By7UFeViqkc6v-hoAtqILeDA"
+ nickname="Laszlo"
+ subject="comment 5"
+ date="2013-01-03T22:49:41Z"
+ content="""
+Ok, I think I figured it out. I have quit from git-annex-assistant (Ctrl-C in terminal window).
+
+And launched this command:
+$ ps -e |grep git-ann
+ 1782 ?        00:01:00 git-annex
+20144 pts/0    00:00:00 git-annex-webap
+20148 pts/0    00:00:00 git-annex
+20178 pts/0    00:00:00 git-annex 
+20508 pts/0    00:00:00 git-annex 
+20528 ?        00:00:02 git-annex
+21132 ?        00:00:00 git-annex 
+
+I will be more cautious in future.
+"""]]
diff --git a/doc/design/assistant/blog/day_161__release_day/comment_6_5dfb5f428633d6062925f61af2b8829b._comment b/doc/design/assistant/blog/day_161__release_day/comment_6_5dfb5f428633d6062925f61af2b8829b._comment
new file mode 100644
index 0000000000..f9683c820c
--- /dev/null
+++ b/doc/design/assistant/blog/day_161__release_day/comment_6_5dfb5f428633d6062925f61af2b8829b._comment
@@ -0,0 +1,23 @@
+[[!comment format=c
+ username="https://www.google.com/accounts/o8/id?id=AItOawm5iosFbL2By7UFeViqkc6v-hoAtqILeDA"
+ nickname="Laszlo"
+ subject="comment 6"
+ date="2013-01-03T22:55:41Z"
+ content="""
+another problem:
+1. If you add a removable drive via the gui, you can not remove it anymore (from the gui)
+2. If you delete the directory on the removable drive, git-annex assistant silently fails
+to recreate the directory and start fresh.
+2/a. on the gui, there is no indication something is wrong with the removable drive repository
+2/b on the terminal it shows only this: 
+fatal: Not a git repository: '/mnt/dxd/annex/.git'
+
+On the main page it shows the two repositories, but on the upper right corner, the dropdown menu
+contains only the original repository and not the removeable drive repository.
+
+I stop spaming now...
+
+Laszlo
+
+
+"""]]
diff --git a/doc/design/assistant/blog/day_161__release_day/comment_7_ac4effb381b08d94d4a2d2482e92c89a._comment b/doc/design/assistant/blog/day_161__release_day/comment_7_ac4effb381b08d94d4a2d2482e92c89a._comment
new file mode 100644
index 0000000000..359947a5a9
--- /dev/null
+++ b/doc/design/assistant/blog/day_161__release_day/comment_7_ac4effb381b08d94d4a2d2482e92c89a._comment
@@ -0,0 +1,13 @@
+[[!comment format=c
+ username="https://www.google.com/accounts/o8/id?id=AItOawm5iosFbL2By7UFeViqkc6v-hoAtqILeDA"
+ nickname="Laszlo"
+ subject="comment 7"
+ date="2013-01-03T23:07:27Z"
+ content="""
+Can not withstand myself.
+
+1. If you add a Removable drive repository, only the *new* files are copied over to the removable drive.
+2. Removable drive repository is not in direct mode. 
+Would be nice if it could be kept the same mode as the original repository.
+It is some kind of special repsoitory, I do not see any seemlinks either.
+"""]]
diff --git a/doc/design/assistant/blog/day_161__release_day/comment_8_32600e89e3098e52a1280895e03b3f86._comment b/doc/design/assistant/blog/day_161__release_day/comment_8_32600e89e3098e52a1280895e03b3f86._comment
new file mode 100644
index 0000000000..c220fc826b
--- /dev/null
+++ b/doc/design/assistant/blog/day_161__release_day/comment_8_32600e89e3098e52a1280895e03b3f86._comment
@@ -0,0 +1,13 @@
+[[!comment format=mdwn
+ username="http://csights.myopenid.com/"
+ ip="70.226.161.163"
+ subject="walkthroughs?"
+ date="2013-01-04T02:39:59Z"
+ content="""
+Could you post some walk-throughs?
+Is it possible to set up a direct mode repository using git-annex webapp ?
+
+Thanks for your work!  Sad to hear you had to get a generator.  :)
+C.
+
+"""]]
diff --git a/doc/design/assistant/blog/day_161__release_day/comment_9_07e5d0c3cad0ce2bd4943e53b61f1767._comment b/doc/design/assistant/blog/day_161__release_day/comment_9_07e5d0c3cad0ce2bd4943e53b61f1767._comment
new file mode 100644
index 0000000000..fc8c9c4b1b
--- /dev/null
+++ b/doc/design/assistant/blog/day_161__release_day/comment_9_07e5d0c3cad0ce2bd4943e53b61f1767._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="spwhitton"
+ ip="163.1.166.255"
+ subject="comment 9"
+ date="2013-01-04T20:26:46Z"
+ content="""
+You noted in a recent blog post that direct mode won't work with mixed repositories, that is, those that have files checked into git alongside annexed ones.  Is this still the case?  Thanks.
+"""]]
diff --git a/doc/design/assistant/blog/day_162__UI.mdwn b/doc/design/assistant/blog/day_162__UI.mdwn
new file mode 100644
index 0000000000..8994835c62
--- /dev/null
+++ b/doc/design/assistant/blog/day_162__UI.mdwn
@@ -0,0 +1,17 @@
+[Installed a generator](http://joeyh.name/blog/entry/overcast/),
+so I'll have more power and less hibernation.
+
+Added UI in the webapp to shut down the daemon. Would like to also have
+restart UI, but that's rather harder to do, seems it'd need to start
+another copy of the webapp (and so, of the assistant), and redirect the
+browser to its new url. ... But running two assistants in the same repo at
+the same time isn't good. Anyway, users can now use the UI to shut it down,
+and then use their native desktop UI to start it back up.
+
+[[!img /assistant/controlmenu.png]]
+
+Spiffed up the control menu. Had to stop listing other local repositories
+in the menu, because there was no way to notice when a new one was added
+(without checking a file on every page load, which is way overkill for this
+minor feature). Instead added a new page that lists local repositories it
+can switch to.
diff --git a/doc/design/assistant/blog/day_163__free_features.mdwn b/doc/design/assistant/blog/day_163__free_features.mdwn
new file mode 100644
index 0000000000..444b6cc30c
--- /dev/null
+++ b/doc/design/assistant/blog/day_163__free_features.mdwn
@@ -0,0 +1,32 @@
+There was a typo in cabal file that broke building the assistant on OSX.
+This didn't affect the autobuilds of the app, but several users building by
+hand reported problems. I made a new minor release fixing that typo, and
+also a resouce leak bug.
+
+Got a restart UI working after all. It's a hack though. It
+opens a new tab for the new assistant instance, and as most web browsers
+don't allow javascript to close tabs, the old tab is left open. At some
+point I need to add a proper thread manager to the assistant, which the
+restart code could use to kill the watcher and committer threads, and then
+I could do a clean restart, bringing up the new daemon and redirecting the
+browser to it.
+
+Found a bug in the assistant in direct mode -- the expensive transfer scan
+didn't queue uploads needed to sync to other repos in direct mode, although
+it did queue downloads. Fixing this laid some very useful groundwork for
+making more commands support direct mode, too. Got stuck for a long time
+dealing with some very strange `git-cat-file` behavior while making this
+work. Ended up putting in a workaround.
+
+After that, I found that these commands work in direct mode, without
+needing any futher changes!
+
+* `git annex find`
+* `git annex whereis`
+* `git annex copy`
+* `git annex move`
+* `git annex drop`
+* `git annex log`
+
+Enjoy! The only commands I'd like to add to this are `fsck`, `add`,
+and `addurl`...
diff --git a/doc/design/assistant/blog/day_164__bugfixes.mdwn b/doc/design/assistant/blog/day_164__bugfixes.mdwn
new file mode 100644
index 0000000000..1edca3d19d
--- /dev/null
+++ b/doc/design/assistant/blog/day_164__bugfixes.mdwn
@@ -0,0 +1,17 @@
+Several bugfixes from user feedback today.
+
+Made the assistant detect misconfigured systems where git will fail to
+commit because it cannot determine the user's name or email address, and
+dummy up enough info to get git working. It makes sense for git and
+git-annex to fail at the command line on such a misconfigured system, so
+the user can fix it, but for the assistant it makes sense to plow on and just
+work.
+
+I found a big gap in direct mode -- all the special remotes expected to find
+content in the indirect mode location when transferring to the remote. It
+was easy to fix once I noticed the problem. This is a big enough bug that
+I will be making a new release in a day or so.
+
+Also, got fsck working in direct mode. It doesn't check as many things
+as in indirect mode, because direct mode files can be modified at any time.
+Still, it's usable, and useful.
diff --git a/doc/design/assistant/blog/day_165__release_day.mdwn b/doc/design/assistant/blog/day_165__release_day.mdwn
new file mode 100644
index 0000000000..f29c6e589a
--- /dev/null
+++ b/doc/design/assistant/blog/day_165__release_day.mdwn
@@ -0,0 +1,16 @@
+(Posted a day late.) [[!meta date="Mon Jan  7 16:05:13 JEST 2013"]]
+
+Got `git annex add` (and `addurl`) working in direct mode. This allowed me
+to make `git annex sync` in direct mode no longer automatically add new
+files.
+
+It's also now safe to mix direct mode annexed files with regular files in
+git, in the same repository. Might have been safe all along, but I've
+tested it, and it certainly works now. You just have to be careful to not
+use `git commit -a` to commit changes to such files, since that'll also 
+stage the entire content of direct mode files.
+
+Made a minor release for these recent changes and bugfixes. Recommended if
+you're using direct mode. Had to chase down a stupid typo I made yesterday
+that caused fsck to infinite loop if it found a corrupted file. Thank
+goodness for test suites.
diff --git a/doc/design/assistant/blog/day_166__a_short_long_day.mdwn b/doc/design/assistant/blog/day_166__a_short_long_day.mdwn
new file mode 100644
index 0000000000..5f03fd1cf5
--- /dev/null
+++ b/doc/design/assistant/blog/day_166__a_short_long_day.mdwn
@@ -0,0 +1,25 @@
+I was up at the crack of dawn wrestling 100 pound batteries around for 3
+hours and rewiring most of my battery bank, so today is a partial day...
+but a day with power, which is always nice.
+
+Did some design work on finally making transfers of files from direct mode
+repositories safe, even if a file is modified as it's being uploaded.
+This seems easily doable for special remotes; git to git repository
+transfers are harder, but I think I see how to do it without breaking
+backwards compatability. 
+
+(An unresolved problem is that a temp file would be left behind when a
+transfer failed due to a file being changed. What would really be nice to
+do is to use that temp file as the rsync basis when transferring the new
+version of a file. Although this really goes beyond direct mode, and into
+[[deltas]] territory.)
+
+Made fsck work better in direct mode repositories. While it's expected for
+files to change at any time in direct mode, and so fsck cannot complain
+every time there's a checksum mismatch, it is possible for it to detect
+when a file does not *seem* to have changed, then check its checksum,
+and so detect disk corruption or other data problems.
+
+Also dealt with several bug reports. One really weird one involves `git
+cat-file` failing due to some kind of gpg signed data in the git-annex
+branch. I don't understand that at all yet.
diff --git a/doc/design/assistant/blog/day_167__safe_direct_mode_transfers.mdwn b/doc/design/assistant/blog/day_167__safe_direct_mode_transfers.mdwn
new file mode 100644
index 0000000000..8aa15fb13a
--- /dev/null
+++ b/doc/design/assistant/blog/day_167__safe_direct_mode_transfers.mdwn
@@ -0,0 +1,12 @@
+Well underway on making direct mode transfers roll back when the file is
+modified while it's transferred.
+
+As expected, it was easy to do for all the special remotes ... Except for
+bup, which does not allow deleting content. For bup it just removes the git
+ref for the bad content, and relies on bup's use of git delta compression
+to keep space use sane.
+
+The problem is also handled by `git-annex-shell sendkey`.
+But not yet for downloads from other git repositories. Bit stuck on that.
+
+Also: A few minor bug fixes.
diff --git a/doc/design/assistant/blog/day_167__safe_direct_mode_transfers/comment_1_f1aa64fe803d8c14b250a4e98b88142a._comment b/doc/design/assistant/blog/day_167__safe_direct_mode_transfers/comment_1_f1aa64fe803d8c14b250a4e98b88142a._comment
new file mode 100644
index 0000000000..4b5092036c
--- /dev/null
+++ b/doc/design/assistant/blog/day_167__safe_direct_mode_transfers/comment_1_f1aa64fe803d8c14b250a4e98b88142a._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="https://www.google.com/accounts/o8/id?id=AItOawm7AuSfii_tCkLyspL6Mr0ATlO6OxLNYOo"
+ nickname="Georg"
+ subject="comment 1"
+ date="2013-01-10T08:49:20Z"
+ content="""
+hmm, wouldn't it make sense to do the downloads into the annex object space, just like with indirect annexes? Then when the download is complete, with indirect annexes you create a soft link and with direct annexes you move the file (or create a hardlink and remove the directory entry in the annex object space)? Actually when you started to talk about direct I first envisioned it to be exactly like indirect, only with hard links instead of soft links.
+"""]]
diff --git a/doc/design/assistant/blog/day_167__safe_direct_mode_transfers/comment_2_5ce1db84c9ead713f1272c4975645b93._comment b/doc/design/assistant/blog/day_167__safe_direct_mode_transfers/comment_2_5ce1db84c9ead713f1272c4975645b93._comment
new file mode 100644
index 0000000000..c1b2724cb0
--- /dev/null
+++ b/doc/design/assistant/blog/day_167__safe_direct_mode_transfers/comment_2_5ce1db84c9ead713f1272c4975645b93._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="http://joeyh.name/"
+ ip="4.154.7.238"
+ subject="comment 2"
+ date="2013-01-10T16:06:28Z"
+ content="""
+That is how direct mode works. (Although without the hard links, since maintaining them doesn't have any benefits.) And that's how downloading into a direct mode repo works. The issue I'm dealing with, though, is uploading from a direct mode repo, where the file that's being uploaded can be modified at the same time.
+"""]]
diff --git a/doc/design/assistant/blog/day_168__back_to_theme.mdwn b/doc/design/assistant/blog/day_168__back_to_theme.mdwn
new file mode 100644
index 0000000000..ff230e7642
--- /dev/null
+++ b/doc/design/assistant/blog/day_168__back_to_theme.mdwn
@@ -0,0 +1,18 @@
+This month's theme is supposed to be fixing up whatever might prevent users
+from using the assistant. To that end, I've posted an open-ended poll,
+[[polls/what is preventing me from using git-annex assistant]]. Please go
+fill it out so I can get an idea of how many people are using the
+assistant, and what needs to be done to get the rest of you, and your
+friends and family able to use it.
+
+In the meantime, today I fixed several bugs that were recently reported in
+the webapp and assistant. Getting it working as widely as possible, even on
+strange IPv6 only systems, and with browsers that didn't like my generated
+javascript code is important, and fits right into this month's theme. I'm
+happy to see lots of bugs being filed, since it means more users are trying
+the assistant out.
+
+Also continued work on making direct mode transfers safe. All transfers to
+local git remotes (wish I had a better phrase!) are now safe in direct mode.
+Only uploading from a direct mode repository over ssh to another git
+repository is still potentially unsafe.
diff --git a/doc/design/assistant/blog/day_168__back_to_theme/comment_1_f248780bfcbd0384d9d72c2633a4ea46._comment b/doc/design/assistant/blog/day_168__back_to_theme/comment_1_f248780bfcbd0384d9d72c2633a4ea46._comment
new file mode 100644
index 0000000000..eaa098d692
--- /dev/null
+++ b/doc/design/assistant/blog/day_168__back_to_theme/comment_1_f248780bfcbd0384d9d72c2633a4ea46._comment
@@ -0,0 +1,12 @@
+[[!comment format=mdwn
+ username="http://jasonwoof.com/"
+ nickname="JasonWoof"
+ subject="phrase suggestion"
+ date="2013-01-11T00:24:39Z"
+ content="""
+I suggest using the term \"repository\" not \"remote\". Two reasons: 1: it's a noun in *English. 2: it works for both local and remote repositories.
+
+Both get +1 because they match git terminology. (I have a secret dream that the assistant might bring people to git, maybe that's silly.)
+
+\* OK, so \"remote\" is also a noun in English, but it means the widget with buttons that controls the tv/whatever.
+"""]]
diff --git a/doc/design/assistant/blog/day_168__back_to_theme/comment_2_5beba073373b8e75a32d1fcfdc1a0782._comment b/doc/design/assistant/blog/day_168__back_to_theme/comment_2_5beba073373b8e75a32d1fcfdc1a0782._comment
new file mode 100644
index 0000000000..6308920256
--- /dev/null
+++ b/doc/design/assistant/blog/day_168__back_to_theme/comment_2_5beba073373b8e75a32d1fcfdc1a0782._comment
@@ -0,0 +1,13 @@
+[[!comment format=mdwn
+ username="http://csights.myopenid.com/"
+ ip="70.226.161.163"
+ subject=""local git remotes""
+ date="2013-01-13T02:36:48Z"
+ content="""
+I am not quite sure how git-annex works and I do not know what a \"local git remote\" functionality is.
+
+How about this: call a \"git remote\" a \"git-annex store\" or just \"store\".  Then instead of a remote git remote it would be a remote git-annex store or just remote store.  :)
+
+
+
+"""]]
diff --git a/doc/design/assistant/blog/day_169__direct_mode_is_safe.mdwn b/doc/design/assistant/blog/day_169__direct_mode_is_safe.mdwn
new file mode 100644
index 0000000000..c3f569bd8d
--- /dev/null
+++ b/doc/design/assistant/blog/day_169__direct_mode_is_safe.mdwn
@@ -0,0 +1,24 @@
+I've finished making direct mode file transfers safe. The last piece of the
+puzzle was making `git-annex-shell recv-key` check data it's received from
+direct mode repositories. This is a bit expensive, but avoids adding
+another round-trip to the protocol. I might revisit this later, this was
+just a quick fix.
+
+---
+
+The [[poll|polls/what is preventing me from using git-annex assistant]] was 
+quite useful. Some interesting points:
+
+* 14% have been reading this blog, and rightly don't trust direct mode to
+  be safe. Which is why I went ahead with a quick fix to make it safe.
+* 6% want an Ubuntu PPA. I don't anticipate doing this myself, but
+  if anyone who develops for Ubuntu wants to put together a PPA with a
+  newer version, I can help you pick the newer haskell packages you'll
+  need from Debian, etc.
+* 9% just need me to update the amd64 build in Debian sid. I forgot to
+  include it in the last release, and the Debian buildds cannot currently
+  autobuild git-annex due to some breakage in the versions of haskell
+  libraries in unstable. Hopefully I'll remember to include an amd64 build
+  in my next release.
+
+And lots of other interesting stuff, I have a nice new TODO list now. :)
diff --git a/doc/design/assistant/blog/day_169__direct_mode_is_safe/comment_1_65f87656c4e6bc7cdb614f53961341c9._comment b/doc/design/assistant/blog/day_169__direct_mode_is_safe/comment_1_65f87656c4e6bc7cdb614f53961341c9._comment
new file mode 100644
index 0000000000..b51153cad5
--- /dev/null
+++ b/doc/design/assistant/blog/day_169__direct_mode_is_safe/comment_1_65f87656c4e6bc7cdb614f53961341c9._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="spwhitton"
+ ip="163.1.166.255"
+ subject="comment 1"
+ date="2013-01-11T20:25:48Z"
+ content="""
+If you have time, an updated sid package for armel would be much appreciated.
+"""]]
diff --git a/doc/design/assistant/blog/day_169__direct_mode_is_safe/comment_2_a116a402a126c62be54c06afd82439ab._comment b/doc/design/assistant/blog/day_169__direct_mode_is_safe/comment_2_a116a402a126c62be54c06afd82439ab._comment
new file mode 100644
index 0000000000..7afd951e17
--- /dev/null
+++ b/doc/design/assistant/blog/day_169__direct_mode_is_safe/comment_2_a116a402a126c62be54c06afd82439ab._comment
@@ -0,0 +1,19 @@
+[[!comment format=mdwn
+ username="http://csights.myopenid.com/"
+ ip="70.226.161.163"
+ subject="use debian repos instead of Ubuntu PPA?"
+ date="2013-01-12T13:43:44Z"
+ content="""
+Maybe Ubuntu users would be satisfied with tweaks to their sources.list and preferences?
+
+/etc/apt/sources.list.d/git-annex.sources.list
+deb http://ftp.us.debian.org/debian unstable main
+
+/etc/apt/preferences.d/git-annex.preferences
+Package: *
+Pin: release a=unstable,o=Debian
+Pin-Priority: 400
+
+
+Not sure how that will work out in Ubuntu package manager.  In aptitude you'll get warning about untrusted repos (maybe install Debian's gpg keys) and will probably have to manually select dependencies from Debian repos.
+"""]]
diff --git a/doc/design/assistant/blog/day_16__more_robust_syncing.mdwn b/doc/design/assistant/blog/day_16__more_robust_syncing.mdwn
new file mode 100644
index 0000000000..e9343f846d
--- /dev/null
+++ b/doc/design/assistant/blog/day_16__more_robust_syncing.mdwn
@@ -0,0 +1,44 @@
+I released a version of git-annex over the weekend that includes the `git
+annex watch` command. There's a minor issue installing it from cabal on
+OSX, which I've fixed in my tree. Nice timing: At least the watch command
+should be shipped in the next Debian release, which freezes at the end of
+the month.
+
+Jimmy found out how kqueue [[blows
+up|bugs/Issue_on_OSX_with_some_system_limits]] when there are too many
+directories to keep all open. I'm not surprised this happens, but it's nice
+to see exactly how. Odd that it happened to him at just 512 directories;
+I'd have guessed more. I have plans to fork watcher programs that each
+watch 512 directories (or whatever the ulimit is), to deal with this. What
+a pitiful interface is kqueue.. I have not thought yet about how the watcher
+programs would communicate back to the main program.
+
+----
+
+Back on the assistant front, I've worked today on making git syncing more
+robust. Now when a push fails, it tries a pull, and a merge, and repushes.
+That ensures that the push is, almost always, a fast-forward. Unless
+something else gets in a push first, anyway!
+
+If a push still fails, there's Yet Another Thread, added today, that will
+wake up after 30 minutes and retry the push. It currently keeps retrying
+every 30 minutes until the push finally gets though. This will deal, to
+some degree, with those situations where a remote is only sometimes
+available.
+
+I need to refine the code a bit, to avoid it keeping an ever-growing queue
+of failed pushes, if a remote is just dead. And to clear old failed pushes
+from the queue when a later push succeeds.
+
+I also need to write a git merge driver that handles conflicts in the tree.
+If two conflicting versions of a file `foo` are saved, this would merge
+them, renaming them to `foo.X` and `foo.Y`. Probably X and Y are the
+git-annex keys for the content of the files; this way all clones will
+resolve the conflict in a way that leads to the same tree. It's also
+possible to get a conflict by one repo deleting a file, and another
+modifying it. In this case, renaming the deleted file to `foo.Y` may
+be the right approach, I am not sure.
+
+I glanced through some Haskell dbus bindings today. I belive there are dbus
+events available to detect when drives are mounted, and on Linux this would
+let git-annex notice and sync to usb drives, etc.
diff --git a/doc/design/assistant/blog/day_16__more_robust_syncing/comment_1_23e7a90429e4431f90787cd016ebe188._comment b/doc/design/assistant/blog/day_16__more_robust_syncing/comment_1_23e7a90429e4431f90787cd016ebe188._comment
new file mode 100644
index 0000000000..fece5c9aff
--- /dev/null
+++ b/doc/design/assistant/blog/day_16__more_robust_syncing/comment_1_23e7a90429e4431f90787cd016ebe188._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="https://www.google.com/accounts/o8/id?id=AItOawmBUR4O9mofxVbpb8JV9mEbVfIYv670uJo"
+ nickname="Justin"
+ subject="comment 1"
+ date="2012-06-27T12:46:31Z"
+ content="""
+can X and Y be the names of the git-annex remotes?
+"""]]
diff --git a/doc/design/assistant/blog/day_16__more_robust_syncing/comment_2_8e7e7cd27791bb47625e60a284e9c802._comment b/doc/design/assistant/blog/day_16__more_robust_syncing/comment_2_8e7e7cd27791bb47625e60a284e9c802._comment
new file mode 100644
index 0000000000..09613cc906
--- /dev/null
+++ b/doc/design/assistant/blog/day_16__more_robust_syncing/comment_2_8e7e7cd27791bb47625e60a284e9c802._comment
@@ -0,0 +1,10 @@
+[[!comment format=mdwn
+ username="http://joeyh.name/"
+ ip="4.153.2.189"
+ subject="comment 2"
+ date="2012-07-26T17:27:39Z"
+ content="""
+That's a good question. Unfortunatly they cannot; X and Y need to be stable across repositories, and git remotes can have different names in different repositories.
+
+Even using the description that git-annex stores for each repository for X and Y is problematic, since that description can change, and so could be different in two repos that are each trying to resolve the same merge conflict.
+"""]]
diff --git a/doc/design/assistant/blog/day_170__bugfixes_and_release.mdwn b/doc/design/assistant/blog/day_170__bugfixes_and_release.mdwn
new file mode 100644
index 0000000000..251ee7636d
--- /dev/null
+++ b/doc/design/assistant/blog/day_170__bugfixes_and_release.mdwn
@@ -0,0 +1,8 @@
+Fixed a goodly amount of bugs today.
+
+The most interesting change was that in direct mode, files using the same
+key are no longer hardlinked, as that could cause a surprising behavior if
+modifying one, where the other would also change.
+
+Made a release, which is almost entirely bug fixes. Debian amd64 build
+included this time. :)
diff --git a/doc/design/assistant/blog/day_171__logs.mdwn b/doc/design/assistant/blog/day_171__logs.mdwn
new file mode 100644
index 0000000000..7c7a4c821d
--- /dev/null
+++ b/doc/design/assistant/blog/day_171__logs.mdwn
@@ -0,0 +1,23 @@
+I've noticed people have some problems getting me logs when there'a a bug,
+so I worked on improving the logging of the assistant.
+
+While the assistant logged to `.git/annex/daemon.log` when started as a
+daemon, when the webapp ran it didn't log there. It's somewhat tricky to
+make the webapp redirect messages to the log, because it may start a web
+browser that wants to write to the console. Took some file descriptor
+juggling, but I made this work. Now the log is even used when the assistant
+is started for the first time in a newly created repository. So, we have
+total log coverage.
+
+Next, I made a simple page in the webapp to display the accumulated logs.
+It does not currently refresh as new things are logged. But it's easier
+for me to tell users to click on `Current Repository -> View log` than
+ask for them to look for the daemon.log file.
+
+Finally, I made all the webapp's alerts also be written to the log.
+
+---
+
+Also did the requisite bug fixes.
+
+[[!img /assistant/logs.png alt="screenshot of logs page"]]
diff --git a/doc/design/assistant/blog/day_172__short_day.mdwn b/doc/design/assistant/blog/day_172__short_day.mdwn
new file mode 100644
index 0000000000..80a32d87ec
--- /dev/null
+++ b/doc/design/assistant/blog/day_172__short_day.mdwn
@@ -0,0 +1,22 @@
+Only one bug fix today, but it was a doozie. It seems that gpg2 has an
+incompatibility with the gpg 1.x that git-annex was written for, that
+causes large numbers of excess passphrase prompts, when it's supposed to be
+using a remote's symmetric encryption key. Adding the --batch parameter
+fixed this.
+
+I also put together a page listing [[related_software]] to git-annex.
+
+I've also updated [[direct_mode]]'s documentation, about when it's safe to
+use direct mode. The intuition I've developed about direct mode is that if
+you don't need full versioning of files (with the ability to get back old
+versions), direct mode is fine and safe to use. If you want full
+versioning, it's best to not use direct mode. Or a reasonable compromise is
+to `git annex untrust` the direct mode repository and set up a backup remote.
+With this compromise, only if you edit a file twice in a row might the old
+version get lost before it can be backed up.
+
+Of course, it would be possible to make direct mode fully version
+preserving, but it'd have to back up every file in the repository locally
+to do so. Going back to two local copies of every file, which is part of
+git that git-annex happily avoids. Hmm, it might be possible to only back
+up a file locally until it reaches the backup remote..
diff --git a/doc/design/assistant/blog/day_172__short_day/comment_1_b75e26b77a23a45da1c4c3bca1399246._comment b/doc/design/assistant/blog/day_172__short_day/comment_1_b75e26b77a23a45da1c4c3bca1399246._comment
new file mode 100644
index 0000000000..c7d4c74ca2
--- /dev/null
+++ b/doc/design/assistant/blog/day_172__short_day/comment_1_b75e26b77a23a45da1c4c3bca1399246._comment
@@ -0,0 +1,12 @@
+[[!comment format=mdwn
+ username="https://launchpad.net/~rubiojr"
+ nickname="rubiojr"
+ subject="comment 1"
+ date="2013-01-18T18:42:35Z"
+ content="""
+Awesome @joey, crisp clear. 
+
+I guess that most people used to the Dropbox way of doing things will not miss versioning that much, which is not very useful for medium/large binary files in most cases I'd say. 
+
+Being able to choose the mode for power users is pretty cool though.
+"""]]
diff --git a/doc/design/assistant/blog/day_173__snow_day.mdwn b/doc/design/assistant/blog/day_173__snow_day.mdwn
new file mode 100644
index 0000000000..9b07a8c5f7
--- /dev/null
+++ b/doc/design/assistant/blog/day_173__snow_day.mdwn
@@ -0,0 +1,22 @@
+When I wasn't dealing with the snowstorm today, I was fixing more bugs.
+Rather serious bugs.
+
+One actually involved corruption to git-annex's location tracking info, due
+to a busted three-way merge. Takes an unusual set of circumstances for that
+bug to be triggered, which is why it was not noticed before now. Also,
+since git-annex is designed to not trust its location tracking info, and
+recover from it becoming inconsistent, someone could have experienced the
+bug and not really noticed it. Still it's a serious problem and I'm in debt
+to user a-or-b for developing a good test case that let me reproduce it and
+fix it. (Also added to the test suite.)
+[[This is how to make a perfect bug report|bugs/Annex_thinks_file_exists_afer_being_dropped]]
+
+Another bug made `git add; git commit` cause data loss in direct mode.
+I was able to make that not lose data, although it still does something
+that's unlikely to be desired, unless the goal is to move a file from an
+annexed direct mode file to having its entire contents stored in git.
+
+Also found a bug with sync's automatic resolution of git conflicts. It
+failed when two repositories both renamed a file to different names.
+I had neglected to explicitly `git rm` the old file name, which is
+necessary to resolve such a conflict.
diff --git a/doc/design/assistant/blog/day_174__last_weekend_before_AU.mdwn b/doc/design/assistant/blog/day_174__last_weekend_before_AU.mdwn
new file mode 100644
index 0000000000..b86a39d660
--- /dev/null
+++ b/doc/design/assistant/blog/day_174__last_weekend_before_AU.mdwn
@@ -0,0 +1,25 @@
+On Friday, worked on several bugs in direct mode mapping code. Fixed it
+to not crash on invalid unicode in filenames. Dealt with some bugs when
+mappings were updated in subdirectories of the repository.
+
+Those bugs could result in inconsistent mapping files, so today I
+made `fsck` check mapping files for consistency.
+
+Leaving for Australia tomorrow, but I also hope to get another bugfix
+release out before my flight leaves. Then will be on vacation for several
+days, more or less. Then at Linux Conf Australia, where there will
+be a git-annex presentation on February 1st.
+
+----
+
+BTW, I've lined up my Android development hardware for next month. I will
+be using an [Asus Transformer](http://en.wikipedia.org/wiki/ASUS_Eee_Pad_Transformer),
+kindly lent to me by Mark H. This has the advantage
+of having a real keyboard, and running the (currently) second most widely used
+version of Android, 4.0.x. I have already experienced frustration getting photos
+off the thing and into my photo annex; the file manager is the worst I've
+seen since the 80's. I understand why so many want an Android port!
+
+Interestingly, its main user filesystem is a FUSE mount point on `/sdcard`
+backed by an ext4 filesystem on `/data` that a regular user is not allowed
+to access. Whatever craziness this entails does not support symlinks.
diff --git a/doc/design/assistant/blog/day_174__last_weekend_before_AU/comment_1_05a8fd47f54373331741cc869a53b0c3._comment b/doc/design/assistant/blog/day_174__last_weekend_before_AU/comment_1_05a8fd47f54373331741cc869a53b0c3._comment
new file mode 100644
index 0000000000..f2a51310d0
--- /dev/null
+++ b/doc/design/assistant/blog/day_174__last_weekend_before_AU/comment_1_05a8fd47f54373331741cc869a53b0c3._comment
@@ -0,0 +1,10 @@
+[[!comment format=mdwn
+ username="https://www.google.com/accounts/o8/id?id=AItOawl9sYlePmv1xK-VvjBdN-5doOa_Xw-jH4U"
+ nickname="Richard"
+ subject="comment 1"
+ date="2013-01-19T20:40:52Z"
+ content="""
+The Nexus devices all come without support for SD cards and without FAT anywhere to be seen. Indirect mode should work nicely on those.
+
+-- Richard
+"""]]
diff --git a/doc/design/assistant/blog/day_174__last_weekend_before_AU/comment_2_fc8e65eef954c4caa8321c2fe8b711b7._comment b/doc/design/assistant/blog/day_174__last_weekend_before_AU/comment_2_fc8e65eef954c4caa8321c2fe8b711b7._comment
new file mode 100644
index 0000000000..b84ad4e3e0
--- /dev/null
+++ b/doc/design/assistant/blog/day_174__last_weekend_before_AU/comment_2_fc8e65eef954c4caa8321c2fe8b711b7._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="EskildHustvedt"
+ ip="84.48.83.221"
+ subject="Presentation video"
+ date="2013-01-19T22:09:50Z"
+ content="""
+Any chance your presentation will be filmed and made available?
+"""]]
diff --git a/doc/design/assistant/blog/day_174__last_weekend_before_AU/comment_3_399534f540d85cac067fbb7be9d373b4._comment b/doc/design/assistant/blog/day_174__last_weekend_before_AU/comment_3_399534f540d85cac067fbb7be9d373b4._comment
new file mode 100644
index 0000000000..ebce958a75
--- /dev/null
+++ b/doc/design/assistant/blog/day_174__last_weekend_before_AU/comment_3_399534f540d85cac067fbb7be9d373b4._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="hamish"
+ ip="203.0.139.23"
+ subject="AU"
+ date="2013-01-24T12:50:06Z"
+ content="""
+I dont know how busy your LCA week is looking, but I'd love to shout you a beer one evening - perhaps there would also be other git-annex people hanging around the conference and interested too
+"""]]
diff --git a/doc/design/assistant/blog/day_175__pacific_features.mdwn b/doc/design/assistant/blog/day_175__pacific_features.mdwn
new file mode 100644
index 0000000000..403088ec84
--- /dev/null
+++ b/doc/design/assistant/blog/day_175__pacific_features.mdwn
@@ -0,0 +1,15 @@
+15 hours in a plane with in-seat power. Ok, time for some new features!
+
+Added two new repository groups. 
+
+"manual" can be used to avoid the assistant downloading any file contents
+on its own. It'll still upload and otherwise sync data. To download files,
+you can use `git annex get` while the assistant is running. You can also
+drop files using the command line.
+
+"source" is for repositories that are the source of new files, but don't
+need to retain a copy once the file has been moved to another repository.
+A camera would be a good example.
+
+Ok, those were easy features to code; I suck at being productive on planes.
+Release coming up with those, once I find enough bandwidth here in AU.
diff --git a/doc/design/assistant/blog/day_175__pacific_features/comment_1_c3ee4386f872b7c76aaecfa638b368cb._comment b/doc/design/assistant/blog/day_175__pacific_features/comment_1_c3ee4386f872b7c76aaecfa638b368cb._comment
new file mode 100644
index 0000000000..0f2708005c
--- /dev/null
+++ b/doc/design/assistant/blog/day_175__pacific_features/comment_1_c3ee4386f872b7c76aaecfa638b368cb._comment
@@ -0,0 +1,9 @@
+[[!comment format=mdwn
+ username="https://www.google.com/accounts/o8/id?id=AItOawlyFDd2xkx0uj738OArDj10CqEkVfWu_mQ"
+ nickname="Johann"
+ subject="comment 1"
+ date="2013-01-24T15:36:34Z"
+ content="""
+Regarding 'manual': Maybe it is necessary to adjust the other standard settings for preferred content then.
+For the 'transfer' for example it would still make sense to treat a manual repository as a client.
+"""]]
diff --git a/doc/design/assistant/blog/day_176__thread_management.mdwn b/doc/design/assistant/blog/day_176__thread_management.mdwn
new file mode 100644
index 0000000000..38f043fcfb
--- /dev/null
+++ b/doc/design/assistant/blog/day_176__thread_management.mdwn
@@ -0,0 +1,13 @@
+Got back to hacking today, and did something I've wanted to do for some
+time. Made all the assistant's threads be managed by a thread manager. This
+allows restarting threads if they crash, by clicking a button in the
+webapp. It also will allow for other features later, like stopping and
+starting the watcher thread, to pause the assistant adding local files.
+
+[[!img /assistant/crashrecovery.png]]
+
+I added the haskell async library as a dependency, which made this pretty
+easy to implement. The only hitch is that async's documentation is not
+clear about how it handles asyncronous exceptions. It took me quite a while
+to work out why the errors I'd inserted into threads to test were crashing
+the whole program rather than being caught!
diff --git a/doc/design/assistant/blog/day_178__bus_hacking.mdwn b/doc/design/assistant/blog/day_178__bus_hacking.mdwn
new file mode 100644
index 0000000000..738afd2c9e
--- /dev/null
+++ b/doc/design/assistant/blog/day_178__bus_hacking.mdwn
@@ -0,0 +1,10 @@
+Hacking on a bus to Canberra for [LCA2013](https://lca2013.linux.org.au/),
+I made the webapp's UI for pausing syncing to a repository also work for
+the local repository. This pauses the watcher thread. (There's also an
+annex.autocommit config setting for this.)
+
+Ironically, this didn't turn out to the use the thread manager I built
+yesterday. I am not sure that a ThreadKilled exception would never be
+masked in the watcher thread. (There is some overly broad exception
+handling in git-annex that dates to back before I quite understood haskell
+exceptions.)
diff --git a/doc/design/assistant/blog/day_179__brief_updates.mdwn b/doc/design/assistant/blog/day_179__brief_updates.mdwn
new file mode 100644
index 0000000000..e755ca8aea
--- /dev/null
+++ b/doc/design/assistant/blog/day_179__brief_updates.mdwn
@@ -0,0 +1,19 @@
+Not doing significant coding here at LCA2013, but stuff is still happening:
+
+* I'll be giving a talk and demo of git-annex and the assistant tomorrow.
+  Right after a keynote by Tim Berners-Lee! There's no streaming, but
+  a recording will be available later.
+* I've met numerous git-annex users and git-annex curious folk from down
+  under.
+* I had a suggestion that direct mode rename the `.git` directory to
+  something else, to prevent foot-shooting git commands being used.
+  A wrapper around git could be used to run git commands, and limit
+  to safe ones. Under consideration.
+* I finally updated the OSX 10.8.2 build to last week's release.
+  Been having some problems with the autobuilder, but it finally spat out
+  a build. Hopefully this build is good, and it should fix the javascript
+  issues with Safari and the webapp.
+* Ulrik Sverdrup has written ,
+  which allows using gpg encrypted ssh remotes with git. The same idea
+  could be expanded to other types of remotes, like S3. I'm excited
+  about adding encrypted git remote support to the assistant!
diff --git a/doc/design/assistant/blog/day_179__brief_updates/comment_1_920a84457d40358507a3eb817a4568d9._comment b/doc/design/assistant/blog/day_179__brief_updates/comment_1_920a84457d40358507a3eb817a4568d9._comment
new file mode 100644
index 0000000000..c58529cf1f
--- /dev/null
+++ b/doc/design/assistant/blog/day_179__brief_updates/comment_1_920a84457d40358507a3eb817a4568d9._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="https://www.google.com/accounts/o8/id?id=AItOawmnG4EuvZWse5hvgrl0XAK-U61e-0iGaao"
+ nickname="David"
+ subject="git commit in direct mode"
+ date="2013-01-31T08:30:19Z"
+ content="""
+How about creating a .git/hooks/pre-commit script that warns the user or checks the commit before anything goes wrong? 
+"""]]
diff --git a/doc/design/assistant/blog/day_17__push_queue_prune.mdwn b/doc/design/assistant/blog/day_17__push_queue_prune.mdwn
new file mode 100644
index 0000000000..54ee75fb8d
--- /dev/null
+++ b/doc/design/assistant/blog/day_17__push_queue_prune.mdwn
@@ -0,0 +1,19 @@
+Not much available time today, only a few hours.
+
+Main thing I did was fixed up the failed push tracking to use a better data
+structure. No need for a queue of failed pushes, all it needs is a map of
+remotes that have an outstanding failed push, and a timestamp. Now it
+won't grow in memory use forever anymore. :)
+
+Finding the right thread mutex type for this turned out to be a bit of a
+challenge. I ended up with a STM TMVar, which is left empty when there are
+no pushes to retry, so the thread using it blocks until there are some. And,
+it can be updated transactionally, without races.
+
+I also fixed a bug outside the git-annex assistant code. It was possible to
+crash git-annex if a local git repository was configured as a remote, and
+the repository was not available on startup. git-annex now ignores such
+remotes. This does impact the assistant, since it is a long running process
+and git repositories will come and go. Now it ignores any that
+were not available when it started up. This will need to be dealt with when
+making it support removable drives.
diff --git a/doc/design/assistant/blog/day_180__back.mdwn b/doc/design/assistant/blog/day_180__back.mdwn
new file mode 100644
index 0000000000..adb336244b
--- /dev/null
+++ b/doc/design/assistant/blog/day_180__back.mdwn
@@ -0,0 +1,7 @@
+Back from Australia. Either later today or tomorrow I'll dig into the
+messages I was not able to get to while traveling, and then the plan is to
+get into the Android port.
+
+Video of my LCA2013 [git-annex talk](http://mirror.linux.org.au/linux.conf.au/2013/mp4/gitannex.mp4)
+is now available. I have not watched it yet, hope it turned out ok despite
+some technical difficulties!
diff --git a/doc/design/assistant/blog/day_181__triage.mdwn b/doc/design/assistant/blog/day_181__triage.mdwn
new file mode 100644
index 0000000000..9c5e92280c
--- /dev/null
+++ b/doc/design/assistant/blog/day_181__triage.mdwn
@@ -0,0 +1,23 @@
+Got fairly far along in my triage of my backlog, looking through everything
+that happened after January 23rd. Still 39 or so items to look at.
+
+There have been several reports of problems with ssh password prompts.
+I'm beginning to think the assistant may need to prompt for the password
+when setting up a ssh remote. This should be handled by `ssh-askpass` or
+similar, but some Linux users probably don't have it installed, and there
+seems to be no widely used OSX equivalent.
+
+---
+
+Fixed several bugs today, all involving (gasp) direct mode.
+
+The tricky one involved renaming or deleting files in direct mode.
+Currently nothing removes the old filename from the direct mode
+mapping, and this could result in the renamed or deleted file
+unexpectedly being put back into the tree when content is downloaded.
+
+To deal with this, it now assumes that direct mode mappings may be out of
+date compared to the tree, and does additional checks to catch
+inconsistencies. While that works well enough for the assistant,
+I'd also like to make the `pre-commit` hook update the mappings for files
+that are changed. That's needed to handle things like `git mv`.
diff --git a/doc/design/assistant/blog/day_182__it_begins.mdwn b/doc/design/assistant/blog/day_182__it_begins.mdwn
new file mode 100644
index 0000000000..48daea78fc
--- /dev/null
+++ b/doc/design/assistant/blog/day_182__it_begins.mdwn
@@ -0,0 +1,50 @@
+I need an Android development environment. I briefly looked into rooting
+the Asus Transformer so I could put a Debian chroot on it and build
+git-annex in there, but this quickly devolved to the typical maze of
+forum posts all containing poor instructions and dead links. Not worth it.
+
+Instead, I'm doing builds on my Sheevaplug, and once I have a static armel
+binary, will see what I need to do to get it running on Android.
+
+Fixed building with the webapp disabled, was broken by recent improvements.
+I'll be building without the webapp on arm initially, because ghci/template
+haskell on arm is still getting sorted out. (I tried ghc 7.6.2 and ghci is
+available, but doesn't quite work.)
+
+From there, I got a binary built pretty quickly (well, it's arm, so not *too*
+quickly). Then tried to make it static by appending 
+`-optl-static -optl-pthread` to the ghc command line.
+This failed with a bunch of errors:
+
+
    +/usr/lib/gcc/arm-linux-gnueabi/4.6/../../../arm-linux-gnueabi/libxml2.a(nanohttp.o): In function `xmlNanoHTTPMethodRedir': (.text+0x2128): undefined reference to `inflateInit2_'
+/usr/lib/gcc/arm-linux-gnueabi/4.6/../../../arm-linux-gnueabi/libxml2.a(xzlib.o): In function `xz_decomp': (.text+0x36c): undefined reference to `lzma_code'
+...
+
    
+
+Disabling DBUS and (temporarily) XMPP got around that.
+
+Result!
+
+
    +joey@leech:~/git-annex>ldd tmp/git-annex 
+        not a dynamic executable
+joey@leech:~/git-annex>ls -lha tmp/git-annex 
+-rwxr-xr-x 1 joey joey 18M Feb  6 16:23 tmp/git-annex*
+
    
+
+Next: Copy binary to Android device, and watch it fail in some interesting way.  
+Repeat.
+
+---
+
+Also more bug triage this morning...
+
+Got the pre-commit hook to update direct mode mappings.
+Uses `git diff-index HEAD` to find out what's changed. The only
+tricky part was detecting when `HEAD` doesn't exist yet. Git's
+plumbing is deficient in this area. Anyway, the mappings get updated
+much better now.
+
+Fixed a wacky bug where `git annex uninit` behaved badly on a filesystem
+that does not support hardlinks.
diff --git a/doc/design/assistant/blog/day_183__plan_b.mdwn b/doc/design/assistant/blog/day_183__plan_b.mdwn
new file mode 100644
index 0000000000..b310d0412e
--- /dev/null
+++ b/doc/design/assistant/blog/day_183__plan_b.mdwn
@@ -0,0 +1,19 @@
+Have not tried to run my static binary on Android yet, but I'm already
+working on a plan B in case that doesn't work. Yesterday I stumbled upon
+, a ghc cross-compiler for
+Android that uses the Android native development kit. 
+It first appeared on February 4th. Good timing!
+
+I've gotten it to build and it emits arm executables, that seem to use the
+Android linker. So that's very promising indeed. 
+
+I've also gotten cabal working with it, and have it chewing through
+installing git-annex's build dependencies.
+
+----
+
+Also made a release today, this is another release that's mostly bugfixes,
+and a few minor features. Including one bug fixed at 6 am this morning, urk.
+
+I think I will probably split my days between working on Android porting
+and other git-annex development.
diff --git a/doc/design/assistant/blog/day_184__just_wanna_run_something.mdwn b/doc/design/assistant/blog/day_184__just_wanna_run_something.mdwn
new file mode 100644
index 0000000000..ac34293c3f
--- /dev/null
+++ b/doc/design/assistant/blog/day_184__just_wanna_run_something.mdwn
@@ -0,0 +1,46 @@
+Have been working on getting all the haskell libraries git-annex uses
+built with the android cross compiler. Difficulties so far are
+libraries that need tweaks to work with the new version of ghc, and some
+that use cabal in ways that break cross compilation. Haskell's network
+library was the last and most challenging of those.
+
+At this point, I'm able to start trying to build git-annex for android.
+Here's the first try!
+
+
    +joey@gnu:~/src/git-annex>cabal install -w $HOME/.ghc-android-14-arm-linux-androideabi-4.7/bin/arm-unknown-linux-androideabi-ghc --with-ghc-pkg=$HOME/.ghc-android-14-arm-linux-androideabi-4.7/bin/arm-unknown-linux-androideabi-ghc-pkg --with-ld=$HOME/.ghc-android-14-arm-linux-androideabi-4.7/bin/arm-linux-androideabi-ld --flags="-Webapp -WebDAV -XMPP -S3 -Dbus"
+Resolving dependencies...
+Configuring git-annex-3.20130207...
+Building git-annex-3.20130207...
+Preprocessing executable 'git-annex' for git-annex-3.20130207...
+on the commandline: Warning:
+    -package-conf is deprecated: Use -package-db instead
+
+Utility/libdiskfree.c:28:26:
+     fatal error: sys/statvfs.h: No such file or directory
+compilation terminated.
+
    
+
+Should not be far from a first android build now..
+
+----
+
+While I already have Android "hello world" executables to try, I have not yet
+been able to run them. Can't seem to find a directory I can write to on the
+Asus Transformer, with a filesystem that supports the +x bit. Do you really
+have to root Android just to run simple binaries? I'm crying inside.
+
+It seems that the blessed Android NDK way would involve making a Java app,
+that pulls in a shared library that contains the native code. For haskell,
+the library will need to contain a C shim that, probably, calls an entry
+point to the Haskell runtime system. Once running, it can use the FFI to
+communicate back to the Java side, probably. The good news is that CJ van
+den Berg, who already saved my bacon once by developing ghc-android, tells
+me he's hard at work on that very thing.
+
+----
+
+In the meantime, downloaded the Android SDK. Have gotten it to build a
+`.apk` package from just javascript code, and managed to do it without
+using eclipse (thank god). Will need this later, but for now want to wash
+my brain out with soap after using it.
diff --git a/doc/design/assistant/blog/day_184__just_wanna_run_something/comment_1_689adac7e26cb0b0a4e7ecc787cfd716._comment b/doc/design/assistant/blog/day_184__just_wanna_run_something/comment_1_689adac7e26cb0b0a4e7ecc787cfd716._comment
new file mode 100644
index 0000000000..b6ad3ac7e0
--- /dev/null
+++ b/doc/design/assistant/blog/day_184__just_wanna_run_something/comment_1_689adac7e26cb0b0a4e7ecc787cfd716._comment
@@ -0,0 +1,16 @@
+[[!comment format=mdwn
+ username="hhm"
+ ip="108.17.80.177"
+ subject="Write and exec on unrooted Android."
+ date="2013-02-10T05:43:41Z"
+ content="""
+See  for info on where in the android filesystem you have write, exec ability.
+
+Basically you have these abilities in `/data/local` from `adb shell` (and in debuggable app's folders using run-as with `adb shell`), and in `/data/data//` for each app (for example the terminal emulator's data dir when using the terminal emulator).
+
+It might be possible to just have temporary root using an old exploit and writing up a bit of code to just setuid on the process and exec sh (source for nachoroot exploit ).
+
+It *is* possible to install a distro without root using a chroot faker.
+
+I am working here  on a way to install the OpenWrt distro on Android. This is working on my phone already for quite a while! So far only sketchy notes are up, but more to come, G-d willing.
+"""]]
diff --git a/doc/design/assistant/blog/day_185__android_liftoff.mdwn b/doc/design/assistant/blog/day_185__android_liftoff.mdwn
new file mode 100644
index 0000000000..88fc5635ca
--- /dev/null
+++ b/doc/design/assistant/blog/day_185__android_liftoff.mdwn
@@ -0,0 +1,20 @@
+Thanks to hhm, who pointed me at [KBOX](http://kevinboone.net/kbox.html),
+I have verified that I can build haskell programs that work on Android.
+
+After hacking on it all day, I've succeeded in making an initial build of
+git-annex for Android. It links! It runs!
+
+Which is not to say it's usable yet; for one thing I need to get a port
+of git before it can do anything useful. (Some of the other things git-annex
+needs, like ssh and sha256sum, are helpfully provided by KBOX.)
+
+Next step will be to find or built a git port for Android. I know there's
+one in the "Terminal IDE" app. Once I can use git-annex at the command line
+on Android, I'll be able to test it out some (I can also port the test
+suite program and run it on Android), and get a feeling for what is needed
+to get the port to a usable command-line state.
+
+And then on to the webapp, and an Android app, I suppose. So far, the port
+doesn't include the webapp, but does include the assistant. The webapp
+needs ghci/template haskell for arm. A few people have been reporting they
+have that working, but I don't yet.
diff --git a/doc/design/assistant/blog/day_185__android_liftoff/comment_1_b7d28010a72619a7e9a5ad4f2a0d6c07._comment b/doc/design/assistant/blog/day_185__android_liftoff/comment_1_b7d28010a72619a7e9a5ad4f2a0d6c07._comment
new file mode 100644
index 0000000000..c0d336b115
--- /dev/null
+++ b/doc/design/assistant/blog/day_185__android_liftoff/comment_1_b7d28010a72619a7e9a5ad4f2a0d6c07._comment
@@ -0,0 +1,9 @@
+[[!comment format=mdwn
+ username="http://mlinden.myopenid.com/"
+ ip="2001:470:d1c3:0:5c01:5562:e242:37b9"
+ subject=""Build native GNU/Linux applications: the easy way""
+ date="2013-02-10T22:02:19Z"
+ content="""
+Maybe this helps:
+[[http://forum.xda-developers.com/showthread.php?t=1444792]]
+"""]]
diff --git a/doc/design/assistant/blog/day_185__android_liftoff/comment_2_ddeb24e86fafb7dae93142cc02767aad._comment b/doc/design/assistant/blog/day_185__android_liftoff/comment_2_ddeb24e86fafb7dae93142cc02767aad._comment
new file mode 100644
index 0000000000..ed0c3dc317
--- /dev/null
+++ b/doc/design/assistant/blog/day_185__android_liftoff/comment_2_ddeb24e86fafb7dae93142cc02767aad._comment
@@ -0,0 +1,10 @@
+[[!comment format=mdwn
+ username="https://www.google.com/accounts/o8/id?id=AItOawnbBRfl5F8gKRr1ko8Ai6FbEZStXXNF1S4"
+ nickname="Áron"
+ subject="another android git app"
+ date="2013-02-11T08:20:42Z"
+ content="""
+https://play.google.com/store/apps/details?id=com.madgag.agit&hl=en
+
+I used it and its decent and certainly runs git as a backend (although I havent seen its internals)
+"""]]
diff --git a/doc/design/assistant/blog/day_186__Android_success.mdwn b/doc/design/assistant/blog/day_186__Android_success.mdwn
new file mode 100644
index 0000000000..8ede7f29a5
--- /dev/null
+++ b/doc/design/assistant/blog/day_186__Android_success.mdwn
@@ -0,0 +1,33 @@
+I'm now successfully using git-annex at the command line on Android.
+`git annex watch` works too.
+
+For now, I'm using a git repository under `/data`, which is on a real,
+non-cripped filesystem, so symlinks work there.
+
+There's still the issue of running without any symlinks on `/mnt/sdcard`.
+While direct mode gets most of the way, it still uses symlinks in a few
+places, so some more work will be needed there. Also, git-annex uses hard
+links to lock down files, which won't work on cripped filesystems.
+
+Besides that, there's lots of minor porting, but no big show-stoppers
+currently.. Some of today's porting work:
+
+* Cross-compiled git for Android. While the Terminal IDE app has some git
+  stuff, it's not complete and misses a lot of plumbing commands git-annex
+  uses. My git build needs some tweaks to be relocatable without setting
+  `GIT_EXEC_PATH`, but it works.
+
+* Switched git-annex to use the Haskell glob library, rather than PCRE. This
+  avoids needing libpcre, which simplifies installation on several platforms
+  (including Android).
+
+* Made git-annex's `configure` hardcode some settings when cross-compiling
+  for Android, rather than probing the build system.
+
+* Android's built-in `lsof` doesn't support the -F option to use a
+  machine-readable output format. So wrote a separate lsof output parser for
+  the standard lsof output format. Unfortunatly, Android's lsof does not
+  provide any information about where a file is open for read or write, so
+  for safety, git-annex has to assume any file that's open might be written
+  to, and avoid annexing it. It might be better to provide my own lsof
+  eventually.
diff --git a/doc/design/assistant/blog/day_186__Android_success/comment_1_1629da240ca7db5f8a32059f561fd435._comment b/doc/design/assistant/blog/day_186__Android_success/comment_1_1629da240ca7db5f8a32059f561fd435._comment
new file mode 100644
index 0000000000..07ef87bb4f
--- /dev/null
+++ b/doc/design/assistant/blog/day_186__Android_success/comment_1_1629da240ca7db5f8a32059f561fd435._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="https://www.google.com/accounts/o8/id?id=AItOawkSq2FDpK2n66QRUxtqqdbyDuwgbQmUWus"
+ nickname="Jimmy"
+ subject="comment 1"
+ date="2013-02-12T10:24:50Z"
+ content="""
+heh, +1 for removing the need for pcre
+"""]]
diff --git a/doc/design/assistant/blog/day_187__porting_utilities.mdwn b/doc/design/assistant/blog/day_187__porting_utilities.mdwn
new file mode 100644
index 0000000000..496c74a1f5
--- /dev/null
+++ b/doc/design/assistant/blog/day_187__porting_utilities.mdwn
@@ -0,0 +1,22 @@
+Ported all the utilities git-annex needs to run on Android: 
+git, rsync, gnupg, dropbear (ssh client), busybox. Built a
+Makefile that can download, patch, and cross build these from source.
+
+While all the utilities work, dropbear doesn't allow git-annex to use ssh
+connection caching, which is rather annoying especially since these systems
+tend to be rather slow and take a while to start up ssh connections.
+I'd sort of like to try to get openssh's client working on Android instead.
+Don't know how realistic that is.
+
+Dealt with several parts of git-annex that assumed `/bin/sh` exists,
+so it instead uses `/system/bin/sh` on Android. Also adapted `runshell`
+for Android.
+
+Now I have a 8 mb compressed tarball for Android.
+Uncompressed it's 25 mb. This includes a lot of git and busybox
+commands that won't be used, so it could be trimmed down further.
+16 mb of it is git-annex itself.
+
+[[Instructions for using the Android tarball|install/Android]]  
+This is for users who are rather brave, not afraid of command line and
+keyboard usage. Good first step.
diff --git a/doc/design/assistant/blog/day_187__porting_utilities/comment_1_0e6a3f4fe8e09f247fa04156bc60f8c7._comment b/doc/design/assistant/blog/day_187__porting_utilities/comment_1_0e6a3f4fe8e09f247fa04156bc60f8c7._comment
new file mode 100644
index 0000000000..e25ff1c15c
--- /dev/null
+++ b/doc/design/assistant/blog/day_187__porting_utilities/comment_1_0e6a3f4fe8e09f247fa04156bc60f8c7._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="http://a-or-b.myopenid.com/"
+ ip="203.45.2.230"
+ subject="comment 1"
+ date="2013-02-13T22:16:17Z"
+ content="""
+In addition to dropbear, there is ConnectBot  which might make the ssh side of the equation simpler.  I prefer it because it is half the size of dropbear.
+"""]]
diff --git a/doc/design/assistant/blog/day_188__crippled_filesystem_support.mdwn b/doc/design/assistant/blog/day_188__crippled_filesystem_support.mdwn
new file mode 100644
index 0000000000..48f6069d45
--- /dev/null
+++ b/doc/design/assistant/blog/day_188__crippled_filesystem_support.mdwn
@@ -0,0 +1,37 @@
+There are at least three problems with using git-annex
+on `/sdcard` on Android, or on a FAT filesystem, or on (to a first
+approximation) Windows:
+
+1. symlinks
+2. hard links
+3. unix permissions
+
+So, I've added an `annex.crippledfilesystem` setting. `git annex init` now
+probes to see if all three are supported, and if not, enables that, as well
+as direct mode.
+
+In crippled filesystem mode, all the permissions settings are skipped.
+Most of them are only used to lock down content in the annex in indirect
+mode anyway, so no great loss.
+
+There are several uses of hard links, most of which can be dealt with by
+making copies. The one use of permissions and hard links I absolutely
+needed to deal with was that they're used to lock down a file as it's being
+ingested into the annex. That can't be done on crippled filesystems, so I
+made it instead check the metadata of the file before and after to detect
+if it changed, the same way direct mode detects when files are modified.
+This is likely better than the old method anyway.
+
+The other reason files are hardlinked while they're being ingested is that
+this allows running lsof on a single directory of files that are in the
+process of being added, to detect if anything has them open for write.
+I still need to adjust the lsof check to work in crippled filesystem mode.
+It seems this won't make it much slower to run lsof on the whole repository.
+
+At this point, I can use git-annex with a repository on `/sdcard` or a FAT
+filesystem, and at least `git annex add` works. 
+
+Still several things on the TODO list before crippled filesystem mode is
+complete. The only one I'm scared about is making `git merge` do something
+sane when it wants to merge two trees full of symlinks, and the filesystem
+doesn't let it create a symlink..
diff --git a/doc/design/assistant/blog/day_188__crippled_filesystem_support/comment_1_32a296fce23ae4b1e18bd5a9964bf619._comment b/doc/design/assistant/blog/day_188__crippled_filesystem_support/comment_1_32a296fce23ae4b1e18bd5a9964bf619._comment
new file mode 100644
index 0000000000..2796964e0e
--- /dev/null
+++ b/doc/design/assistant/blog/day_188__crippled_filesystem_support/comment_1_32a296fce23ae4b1e18bd5a9964bf619._comment
@@ -0,0 +1,14 @@
+[[!comment format=mdwn
+ username="http://nullroute.eu.org/~grawity/"
+ nickname="Mantas"
+ subject="comment 1"
+ date="2013-02-15T00:38:00Z"
+ content="""
+Windows supports hardlinks on NTFS filesystems. Not sure about the privileges needed; I think all users can create them.
+
+Recent versions also support symlinks, but that's limited to Administrators by default, so Cygwin-Git (like the rest of Cygwin) uses special files with the \"system\" flag.
+
+As far as I know, Git does not try to dereference symlinks when merging – it only tracks the paths they point to. Where symlinks are not supported (e.g. vfat filesystems in general, or msysGit on Windows) it simply creates regular files containing the target path (i.e. the raw contents of the respective blobs).
+
+Regarding the use of Unix permissions: The \"open file\" syscall on Windows has a \"share mode\" flag, which can be used to disallow other programs from opening the file for writing (and/or even reading) as long as git-annex keeps it open. (I guess this is similar to mandatory locks in Linux.)
+"""]]
diff --git a/doc/design/assistant/blog/day_189__more_crippling.mdwn b/doc/design/assistant/blog/day_189__more_crippling.mdwn
new file mode 100644
index 0000000000..f3ddad2667
--- /dev/null
+++ b/doc/design/assistant/blog/day_189__more_crippling.mdwn
@@ -0,0 +1,44 @@
+Finished crippled filesystem support, except for symlink handling.
+This was straightforward, just got lsof working in that mode, made
+`migrate` copy key contents, and adapted the rsync special remote to
+support it. Encrypted rsync special remotes have no more overhead on
+crippled filesystems than normally. Un-encrypted rsync special remotes
+have some added overhead, but less than encrypted remotes. Acceptable
+for now.
+
+I've now successfully run the assistant on a FAT filesystem.
+
+----
+
+Git handles symlinks on crippled filesystems by setting
+`core.symlinks=false` and checking them out as files containing the link
+text. So to finish up crippled filesystem support, git-annex needs to
+do the same whenever it creates a symlink, and needs to read file contents
+when it normally reads a symlink target.
+
+There are rather a lot of calls to `createSymbolicLink`,
+`readSymbolicLink`, `getSymbolicLinkStatus`, `isSymbolicLink`, and `isSymLink`
+in the tree; only ones that are used in direct mode
+need to be converted. This will take a while.
+
+Checking whether something is a symlink, or where it points is especially
+tricky. How to tell if a small file in a git working tree is intended to be
+a symlink or not? Well, it can look at the content and see if it makes
+sense as a link text pointing at a git-annex key. As long as the
+possibility of false positives is ok. It might be possible, in some cases,
+to query git to verify if the object stored for that file is really a
+symlink, but that won't work if the file has been renamed, for example.
+
+Converted some of the most commonly used symlink code to handle this.
+Much more to do, but it basically works; I can `git annex get` and `git
+annex drop` on FAT, and it works.
+
+-----
+
+Unfortunately, got side-tracked when I discovered that the last release
+introduced a bug in direct mode. Due to the bug, "git annex get file; git annex
+drop file; git annex get file" would end up with the file being an indirect
+mode symlink to the content, rather than a direct mode file. No data loss,
+but not right. So, spent several hours fixing that reversion, which was
+caused by me stupidly fixing another bug at 5 am in the morning last week.. 
+and I'll probably be pushing out another release tomorrow with the fix.
diff --git a/doc/design/assistant/blog/day_18__merging.mdwn b/doc/design/assistant/blog/day_18__merging.mdwn
new file mode 100644
index 0000000000..f963cf85dd
--- /dev/null
+++ b/doc/design/assistant/blog/day_18__merging.mdwn
@@ -0,0 +1,82 @@
+Worked on automatic merge conflict resolution today. I had expected to be
+able to use git's merge driver interface for this, but that interface is
+not sufficient. There are two problems with it:
+
+1. The merge program is run when git is in the middle of an operation
+   that locks the index. So it cannot delete or stage files. I need to
+   do both as part of my conflict resolution strategy.
+2. The merge program is not run at all when the merge conflict is caused
+   by one side deleting a file, and the other side modifying it. This is
+   an important case to handle.
+
+So, instead, git-annex will use a regular `git merge`, and if it fails, it
+will fix up the conflicts.
+
+That presented its own difficulty, of finding which files in the tree
+conflict. `git ls-files --unmerged` is the way to do that, but its output
+is a quite raw form:
+
+	120000 3594e94c04db171e2767224db355f514b13715c5 1	foo
+	120000 35ec3b9d7586b46c0fd3450ba21e30ef666cfcd6 3	foo
+	100644 1eabec834c255a127e2e835dadc2d7733742ed9a 2	bar
+	100644 36902d4d842a114e8b8912c02d239b2d7059c02b 3	bar
+
+I had to stare at the rather impenetrable documentation for hours and
+write a lot of parsing and processing code to get from that to these mostly
+self explanatory data types:
+
+	data Conflicting v = Conflicting
+	        { valUs :: Maybe v
+	        , valThem :: Maybe v
+	        } deriving (Show)
+
+	data Unmerged = Unmerged
+	        { unmergedFile :: FilePath
+	        , unmergedBlobType :: Conflicting BlobType
+	        , unmergedSha :: Conflicting Sha
+	        } deriving (Show)
+
+Not the first time I've whined here about time spent parsing unix command
+output, is it? :)
+
+From there, it was relatively easy to write the actual conflict cleanup
+code, and make `git annex sync` use it. Here's how it looks:
+
+	$ ls -1
+	foo.png
+	bar.png
+	$ git annex sync
+	commit  
+	# On branch master
+	nothing to commit (working directory clean)
+	ok
+	merge synced/master 
+	CONFLICT (modify/delete): bar.png deleted in refs/heads/synced/master and modified in HEAD. Version HEAD of bar.png left in tree.
+	Automatic merge failed; fix conflicts and then commit the result.
+	bar.png: needs merge
+	(Recording state in git...)
+	[master 0354a67] git-annex automatic merge conflict fix
+	ok
+	$ ls -1
+	foo.png
+	bar.variant-a1fe.png
+	bar.variant-93a1.png
+
+There are very few options for ways for the conflict resolution code to
+name conflicting variants of files. The conflict resolver can only use data
+present in git to generate the names, because the same conflict needs to 
+be resolved the same everywhere.
+
+So I had to choose between using the full key name in the filenames produced
+when resolving a merge, and using a shorter checksum of the key, that would be
+more user-friendly, but could theoretically collide with another key. 
+I chose the checksum, and weakened it horribly by only using 32 bits of it!
+
+Surprisingly, I think this is a safe choice. The worst that can
+happens if such a collision happens is another conflict, and the conflict
+resolution code will work on conflicts produced by the conflict resolution
+code! In such a case, it does fall back to putting the whole key in
+the filename:
+"bar.variant-SHA256-s2550--2c09deac21fa93607be0844fefa870b2878a304a7714684c4cc8f800fda5e16b.png"
+
+Still need to hook this code into `git annex assistant`.
diff --git a/doc/design/assistant/blog/day_18__merging/comment_1_7a553ad559519f3b3e3cd20115b4c44e._comment b/doc/design/assistant/blog/day_18__merging/comment_1_7a553ad559519f3b3e3cd20115b4c44e._comment
new file mode 100644
index 0000000000..94261f81eb
--- /dev/null
+++ b/doc/design/assistant/blog/day_18__merging/comment_1_7a553ad559519f3b3e3cd20115b4c44e._comment
@@ -0,0 +1,15 @@
+[[!comment format=mdwn
+ username="http://droggl.myopenid.com/"
+ ip="2001:638:602:1181:a6ba:dbff:fedd:8041"
+ subject="comment 1"
+ date="2013-09-11T11:55:27Z"
+ content="""
+Nice!
+However I didnt find any documentation yet on how to actually resolve the situation as a user (sorry if I'm being blind).
+To be more specific: Say a conflict occurs on foo.txt and i now have two files foo.variant-dead.txt and foo.variant-beef.txt.
+Now what? Say I used my favorite merge diffmerge tool to create a merged version foo-merged.txt, should I rename that to foo.txt?
+Should I delete the variant files? Or drop them? Or would that lead to another merge for systems that still have the old foo.txt?
+(You see I'm kind of confused of what happens here ;))
+
+TIA for any helpers
+"""]]
diff --git a/doc/design/assistant/blog/day_190-191__weekend.mdwn b/doc/design/assistant/blog/day_190-191__weekend.mdwn
new file mode 100644
index 0000000000..21e46529a7
--- /dev/null
+++ b/doc/design/assistant/blog/day_190-191__weekend.mdwn
@@ -0,0 +1,28 @@
+Pushed out a release yesterday mostly for a bug fix. I have to build
+git-annex 5 times now when releasing. Am wondering if I could get rid of
+the Linux 64 bit standalone build. The 32 bit build should run ok on 64 bit
+Linux systems, since it has all its own 32 bit libraries. What I really
+need to do is set up autobuilders for Linux and Android, like we have for OSX.
+
+Today, dealt with all code that creates or looks at symlinks. Audited every
+bit of it, and converted all relevant parts to use a new abstraction layer
+that handles the pseudolink files git uses when core.symlinks=false.
+This is untested, but I'm quite happy with how it turned out.
+
+----
+
+Where next for Android? I want to spend a while testing command-line
+git-annex. After I'm sure it's really solid, I should try to get the webapp
+working, if possible.
+
+I've heard rumors that Ubuntu's version of ghc somehow supports template
+haskell on arm, so I need to investigate that. If I am unable to get
+template haskell on arm, I would need to either wait for further
+developments, or try to expand yesod's template haskell to regular haskell
+and then build it on arm, or I could of course switch away from hamlet
+(using blaze-html instead is appealing in some ways) and
+use yesod in non-template-haskell mode entirely. One of these will work,
+for sure, only question is how much pain.
+
+After getting the webapp working, there's still the issue of bundling it
+all up in an Android app that regular users can install.
diff --git a/doc/design/assistant/blog/day_190-191__weekend/comment_1_dbd692d12c14d08acd7d73a655b34e8b._comment b/doc/design/assistant/blog/day_190-191__weekend/comment_1_dbd692d12c14d08acd7d73a655b34e8b._comment
new file mode 100644
index 0000000000..965a762e6a
--- /dev/null
+++ b/doc/design/assistant/blog/day_190-191__weekend/comment_1_dbd692d12c14d08acd7d73a655b34e8b._comment
@@ -0,0 +1,10 @@
+[[!comment format=mdwn
+ username="http://jasonwoof.com/"
+ nickname="JasonWoof"
+ subject="phonegap might help"
+ date="2013-02-17T22:43:06Z"
+ content="""
+In your research to see how to bundle up a graphical android app, I suggest you read up on phonegap: http://phonegap.com/
+
+Essentially they bundle webkit so you can turn a web app into an android app. Not sure if they help you ship native/binary stuff along side the html/javascript/css, but phonegap should at least be useful as a reference for how to do some of the bundling-related stuff.
+"""]]
diff --git a/doc/design/assistant/blog/day_190-191__weekend/comment_2_c813830e53471a9732e010a748d574fc._comment b/doc/design/assistant/blog/day_190-191__weekend/comment_2_c813830e53471a9732e010a748d574fc._comment
new file mode 100644
index 0000000000..bdda398b06
--- /dev/null
+++ b/doc/design/assistant/blog/day_190-191__weekend/comment_2_c813830e53471a9732e010a748d574fc._comment
@@ -0,0 +1,28 @@
+[[!comment format=c
+ username="https://www.google.com/accounts/o8/id?id=AItOawm5iosFbL2By7UFeViqkc6v-hoAtqILeDA"
+ nickname="Laszlo"
+ subject="Two direct mode repository on the same computer"
+ date="2013-02-19T09:24:21Z"
+ content="""
+Hi,
+
+I just tried the newest git annex webapp (20130216), 
+I am still not able to configure two direct mode repositories (one normal, one backup) on the same computer.
+
+It does recognize my other partition (/mnt/the) as a removable drive, but when I add it 
+it gives me internal server error:
+\"\"\"
+Internal Server Error
+
+there is no available git remote named \"the\"
+git-annex version 3.20130216
+\"\"\"
+
+Could you please (pretty please) add a possibility to specify the removable drive as a simple folder?
+So I could choose arbitrary folder(/mnt/the/annex-backup) on my computer and 
+it would treat it as a direct mode repository.
+
+Best, 
+ Laszlo
+
+"""]]
diff --git a/doc/design/assistant/blog/day_192_193__more_porting.mdwn b/doc/design/assistant/blog/day_192_193__more_porting.mdwn
new file mode 100644
index 0000000000..59f7cf8369
--- /dev/null
+++ b/doc/design/assistant/blog/day_192_193__more_porting.mdwn
@@ -0,0 +1,44 @@
+Felt spread thin yesterday, as I was working on multiple things
+concurrently & bouncing around as compiles finished. Been working to get
+openssh to build for Android, which is quite a pain, starting with getting
+openssl to build and then dealing with the Cyanogenmod patches, some of
+which are necessary to build on Android and some of which break builds
+outside Cyanogenmod. At the same time was testing git-annex on Android.
+Found and fixed several more portability bugs while doing that. And on the
+back burner I was making some changes to the webapp..
+
+(Forgot to commit my blog post yesterday too..)
+
+Today, that all came together.
+
+* When adding another local repository in the webapp, 
+  it now allows you to choose whether it should be combined with
+  your current repository, or kept separate. Several people had requested
+  a way to add local clones with the webapp, for various reasons, like
+  wanting a backup repository, or wanting to make a repository on a NFS
+  server, and this allows doing that.
+
+[[!img /assistant/combinerepos.png]]
+
+* More porting fun. FAT filesystems and other things used on Android can
+  get all new inode numbers each time mounted. Made git-annex use a
+  sentinal file to detect when this has happened, since in direct mode
+  it compares inodes. (As a bonus this also makes copying direct mode
+  repositories between filesystems work.)
+
+* Got openssh building for Android. Changed it to use $HOME/.ssh rather
+  than trusting pwent.
+
+* Got git-annex's ssh connection caching working on Android. That needs
+  a place where it can create a socket. When the
+  repository is on a crippled filesystem, it instead puts the socket
+  in a temporary directory set up on the filesystem where the git-annex
+  program resides.
+
+With ssh connection caching, transferring multiple files off my Android
+tablet *screams*! I was seeing 6.5 megabytes transferred per second,
+sustained over a whole month's worth of photos.
+
+Next problem: `git annex assistant` on Android is for some reason crashing
+with a segfault on startup. Especially odd since `git annex watch` works.
+I'm so close to snap-photo-and-it-syncs-nirvana, but still so far away...
diff --git a/doc/design/assistant/blog/day_194__nice_moment.mdwn b/doc/design/assistant/blog/day_194__nice_moment.mdwn
new file mode 100644
index 0000000000..1a5ee9b1ea
--- /dev/null
+++ b/doc/design/assistant/blog/day_194__nice_moment.mdwn
@@ -0,0 +1,37 @@
+
+video
+
+Today's work:
+
+* Fixed `git annex add` of a modified file in direct mode.
+* Fixed bugs in the inode sentinal file code added yesterday.
+* With some help from Kevin Boone, I now understand how KBOX works and
+  how to use similar techniques to build my own standalone Android app
+  that includes git-annex.
+
+  Kevin is using a cute hack; he ships a tarball and some other stuff
+  as (pseudo-)library files (`libfoo.so`), which are the only files
+  the Android package manager deigns to install. Then the app runs one
+  of these, which installs the programs.
+
+  This avoids needing to write Java code that extracts the programs from
+  one of its assets and writes it to an executable file, which is the
+  canonical way to do this sort of thing. But I noticed it has a benefit too
+  (which KBOX does not yet exploit). Since the pseudo-library file is installed
+  with the X bit set, if it's really a program, such as busybox or git-annex,
+  that program can be run without needing to copy it to an executable file.
+  This can save a lot of disk space. So, I'm planning to include all
+  the binaries needed by git-annex as these pseudo-libraries.
+* Got the Android Terminal Emulator to build. I will be basing my first
+  git-annex Android app on this, since a terminal is needed until there's 
+  a webapp.
+* Wasted several hours fighting with `Android.mk` files to include
+  my pseudo shared library. This abuse of Makefiles by the NDK is what CDBS
+  wants to grow up to be.. or is it the other way around? Anyway, it
+  sucks horribly, and I finally found a way to do it without
+  modifying the file at all. Ugh.
+* At this point, I can build a `git-annex.apk` file containing a
+  `libgit-annex.so`, and a `libbusybox.so`, that can both be directly
+  run. The plan from here is to give git-annex the ability to
+  auto-install itself, and the other pseudo-libraries, when it's run as
+  `libgit-annex.so`.
diff --git a/doc/design/assistant/blog/day_195__real_android_app.mdwn b/doc/design/assistant/blog/day_195__real_android_app.mdwn
new file mode 100644
index 0000000000..f6a4eeebe3
--- /dev/null
+++ b/doc/design/assistant/blog/day_195__real_android_app.mdwn
@@ -0,0 +1,32 @@
+Well, it's built. [Real Android app for git-annex](http://downloads.kitenet.net/git-annex/android/current/).
+
+[[!img /android/appinstalled.png]]
+
+When installed, this will open a terminal in which you have access to
+git-annex and all the git commands and busybox commands as well. No webapp
+yet, but command line users should feel right at home.
+
+[[!img /android/terminal.png]]
+
+Please test it out, at least as far as installing it, opening the terminal,
+and checking that you can run `git annex`; I've only been able to test on
+one Android device so far. I'm especially keen to know if it works with
+newer versions of Android than 4.0.3. (I know it only supports arm based
+Android, no x86 etc.) Please comment if you tried it.
+
+----
+
+Building this went mostly as planned, although I had about 12 builds of
+the app in the middle which crashed on startup with no error message ora
+logs. Still, it took only one day to put it all together,
+ and I even had time to gimp up a quick icon. (Better icons welcome.)
+
+Kevin thinks that my space-saving hack won't work on all Androiden, and he
+may be right. If the `lib` directory is on a different filesystem on some
+devices, it will fail. But I used it for now anyhow. Thanks to the hack,
+the 7.8 mb compressed .apk file installs to use around 23 mb of disk space.
+
+----
+
+Tomorrow: Why does `git-annex assistant` on Android re-add all existing
+files on startup?
diff --git a/doc/design/assistant/blog/day_195__real_android_app/comment_10_0112007552b30cd9bfeac614a1e399c4._comment b/doc/design/assistant/blog/day_195__real_android_app/comment_10_0112007552b30cd9bfeac614a1e399c4._comment
new file mode 100644
index 0000000000..cc24c08c34
--- /dev/null
+++ b/doc/design/assistant/blog/day_195__real_android_app/comment_10_0112007552b30cd9bfeac614a1e399c4._comment
@@ -0,0 +1,10 @@
+[[!comment format=mdwn
+ username="http://joeyh.name/"
+ ip="4.153.253.75"
+ subject="comment 10"
+ date="2013-02-22T17:30:59Z"
+ content="""
+@rillian could you try to install KBOX  and see if its terminal accepts input? It's the same terminal app, so will be good to know if I broke it or this is a wider problem.
+
+Also, did you try the soft keyboard? There's a menu item to enable it.
+"""]]
diff --git a/doc/design/assistant/blog/day_195__real_android_app/comment_11_230d3c169c713f613b9d607d84ce5092._comment b/doc/design/assistant/blog/day_195__real_android_app/comment_11_230d3c169c713f613b9d607d84ce5092._comment
new file mode 100644
index 0000000000..a984dd339a
--- /dev/null
+++ b/doc/design/assistant/blog/day_195__real_android_app/comment_11_230d3c169c713f613b9d607d84ce5092._comment
@@ -0,0 +1,12 @@
+[[!comment format=mdwn
+ username="rillian"
+ ip="64.213.70.194"
+ subject="Re: no keybourd events on Galaxy Tab 10.1"
+ date="2013-02-22T18:28:11Z"
+ content="""
+KBOX has the same input problem so you didn't break anything with your changes.
+
+I was testing with the soft keyboard. Turns out the spacebar and number keys on the soft keyboard work, as well as return, just not the alphabetic keys.
+
+Trying on a hardware keyboard (Samsung docking station) is similar. Space, Enter and numbers work, but not letters. There's some additional confusion about shift state, where it gets confused and reports shifted digits from the hardware keys when the soft keyboard is in some shift states. E.g. switching the soft keyboard to numeric mode results in shift-digits (punctuation) from the hardware numeric keys.
+"""]]
diff --git a/doc/design/assistant/blog/day_195__real_android_app/comment_12_8d74ad2a61c02272758d157282ad56ec._comment b/doc/design/assistant/blog/day_195__real_android_app/comment_12_8d74ad2a61c02272758d157282ad56ec._comment
new file mode 100644
index 0000000000..76aee55fd6
--- /dev/null
+++ b/doc/design/assistant/blog/day_195__real_android_app/comment_12_8d74ad2a61c02272758d157282ad56ec._comment
@@ -0,0 +1,10 @@
+[[!comment format=mdwn
+ username="http://joeyh.name/"
+ ip="4.153.253.75"
+ subject="comment 12"
+ date="2013-02-22T18:35:35Z"
+ content="""
+@rillian the upstream for this terminal is 
+and I found a bug that looks similar. 
+You might add your info there, and try the workaround documented there.
+"""]]
diff --git a/doc/design/assistant/blog/day_195__real_android_app/comment_13_4f6bc0680f2debd638933968a26975e0._comment b/doc/design/assistant/blog/day_195__real_android_app/comment_13_4f6bc0680f2debd638933968a26975e0._comment
new file mode 100644
index 0000000000..455029a411
--- /dev/null
+++ b/doc/design/assistant/blog/day_195__real_android_app/comment_13_4f6bc0680f2debd638933968a26975e0._comment
@@ -0,0 +1,10 @@
+[[!comment format=mdwn
+ username="http://joeyh.name/"
+ ip="4.153.253.75"
+ subject="comment 13"
+ date="2013-02-22T19:39:15Z"
+ content="""
+@Karsten does KBOX work on this version of Android? 
+
+It may well be that this is just too old and incompatible for the terminal to work.
+"""]]
diff --git a/doc/design/assistant/blog/day_195__real_android_app/comment_14_71539c62608866464e8faa76bc522a55._comment b/doc/design/assistant/blog/day_195__real_android_app/comment_14_71539c62608866464e8faa76bc522a55._comment
new file mode 100644
index 0000000000..bfb12f24a6
--- /dev/null
+++ b/doc/design/assistant/blog/day_195__real_android_app/comment_14_71539c62608866464e8faa76bc522a55._comment
@@ -0,0 +1,9 @@
+[[!comment format=mdwn
+ username="http://meep.pl/"
+ ip="193.23.174.18"
+ subject="comment 14"
+ date="2013-02-22T21:45:54Z"
+ content="""
+On Android 2.3.3 (Samsung Galaxy S+) KBOX works, git-annex displays the \"Terminal session is running\" notification and dies immediately.
+
+"""]]
diff --git a/doc/design/assistant/blog/day_195__real_android_app/comment_15_e1b205289721ae79ac7fbed2f44018b2._comment b/doc/design/assistant/blog/day_195__real_android_app/comment_15_e1b205289721ae79ac7fbed2f44018b2._comment
new file mode 100644
index 0000000000..e48ee71349
--- /dev/null
+++ b/doc/design/assistant/blog/day_195__real_android_app/comment_15_e1b205289721ae79ac7fbed2f44018b2._comment
@@ -0,0 +1,10 @@
+[[!comment format=mdwn
+ username="http://joeyh.name/"
+ ip="4.153.253.75"
+ subject="comment 15"
+ date="2013-02-22T22:26:22Z"
+ content="""
+I'm seeing a pattern with Android 2.3 failing.. 
+
+This failure mode suggests that the bug is in the installation script. The next build will make the terminal not exit immediately if that script fails, so you can go in to the shell and run it by hand to see what's going wrong.
+"""]]
diff --git a/doc/design/assistant/blog/day_195__real_android_app/comment_1_4bc0aeae4fa1116944644c64feaf9697._comment b/doc/design/assistant/blog/day_195__real_android_app/comment_1_4bc0aeae4fa1116944644c64feaf9697._comment
new file mode 100644
index 0000000000..05e3d298b5
--- /dev/null
+++ b/doc/design/assistant/blog/day_195__real_android_app/comment_1_4bc0aeae4fa1116944644c64feaf9697._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="http://andrew.aylett.co.uk/"
+ nickname="andrew"
+ subject="Works on Samsung Galaxy S3"
+ date="2013-02-21T23:23:08Z"
+ content="""
+Installed, executed and managed to get the help text on a Samsung Galaxy S3 running 4.1.2.  Not tried anything else yet, but looks good so far, thank you very much :).
+"""]]
diff --git a/doc/design/assistant/blog/day_195__real_android_app/comment_2_17bb6e7565d4c757f6c1e3514c22f47d._comment b/doc/design/assistant/blog/day_195__real_android_app/comment_2_17bb6e7565d4c757f6c1e3514c22f47d._comment
new file mode 100644
index 0000000000..138518bdb8
--- /dev/null
+++ b/doc/design/assistant/blog/day_195__real_android_app/comment_2_17bb6e7565d4c757f6c1e3514c22f47d._comment
@@ -0,0 +1,10 @@
+[[!comment format=mdwn
+ username="https://www.google.com/accounts/o8/id?id=AItOawlmNbexnCgLywD0MKyT9mG9mWFb_qVj-8I"
+ nickname="Joshua"
+ subject="Toshiba Excite 7.7 (Android 4.1.1)"
+ date="2013-02-22T00:15:46Z"
+ content="""
+Seems to run fine. I managed to create a repository, git-annex init it (had to set email and name per repository). 
+
+The first time I tried git-annex init it said \"Detected a crippled file system.\" \"Enabling direct mode.\" The final try that actually worked only mentioned the crippled filesystem and not direct mode.
+"""]]
diff --git a/doc/design/assistant/blog/day_195__real_android_app/comment_3_cd8a6bec0f7c6843dd11d3266f25f864._comment b/doc/design/assistant/blog/day_195__real_android_app/comment_3_cd8a6bec0f7c6843dd11d3266f25f864._comment
new file mode 100644
index 0000000000..643605d5b6
--- /dev/null
+++ b/doc/design/assistant/blog/day_195__real_android_app/comment_3_cd8a6bec0f7c6843dd11d3266f25f864._comment
@@ -0,0 +1,44 @@
+[[!comment format=mdwn
+ username="https://www.google.com/accounts/o8/id?id=AItOawnBJ6Dv1glxzzi4qIzGFNa6F-mfHIvv9Ck"
+ nickname="Jim"
+ subject="Installs and runs"
+ date="2013-02-22T00:17:30Z"
+ content="""
+No problem installing on a Galaxy Nexus with 4.2.2
+
+    1|u0_a162@android:/sdcard $ git clone https://git.jim.sh/jim/annex.git
+    Cloning into 'annex'...
+    fatal: Unable to find remote helper for 'https'
+
+Too much to hope for at this stage, I suppose :)
+
+    128|u0_a162@android:/sdcard $ git annex assistant
+    git-annex: Not in a git repository.
+    1|u0_a162@android:/sdcard $ git init test
+    Initialized empty Git repository in /storage/emulated/legacy/test/.git/
+    u0_a162@android:/sdcard $ cd test
+    u0_a162@android:/sdcard/test $ git annex init
+    init
+      Detected a crippled filesystem.
+    
+      Enabling direct mode.
+    
+    *** Please tell me who you are.
+    
+    Run
+    
+      git config --global user.email \"you@example.com\"
+      git config --global user.name \"Your Name\"
+    
+    to set your account's default identity.
+    Omit --global to set the identity only in this repository.
+    
+    fatal: unable to auto-detect email address (got 'u0_a162@localhost.(none)')
+    
+    git-annex: user error (git [\"--git-dir=/storage/emulated/legacy/test/.git\",\"--work-tree=/storage/emulated/legacy/test\",\"commit-tree\",\"4b825dc642cb6eb9a060e54bf8d69288fbee4904\"] exited 128)
+    failed
+    git-annex: init: 1 failed
+    1|u0_a162@android:/sdcard/test $
+
+However, I get the same error even after setting `user.email` and `user.name` (which seems to work fine)
+"""]]
diff --git a/doc/design/assistant/blog/day_195__real_android_app/comment_4_2d2eee4bcbbd1d069a80bff5edc90c3c._comment b/doc/design/assistant/blog/day_195__real_android_app/comment_4_2d2eee4bcbbd1d069a80bff5edc90c3c._comment
new file mode 100644
index 0000000000..acb022e132
--- /dev/null
+++ b/doc/design/assistant/blog/day_195__real_android_app/comment_4_2d2eee4bcbbd1d069a80bff5edc90c3c._comment
@@ -0,0 +1,13 @@
+[[!comment format=mdwn
+ username="http://joeyh.name/"
+ ip="4.153.253.75"
+ subject="comment 4"
+ date="2013-02-22T00:54:56Z"
+ content="""
+@jim git http support is disabled to save space
+
+What seems to work is: git config user.email \"you@example.com\" 
+
+It's probably looking in the wrong place for the global .gitconfig so --global doesn't work. The assistant
+also automates dealing with this problem BTW.
+"""]]
diff --git a/doc/design/assistant/blog/day_195__real_android_app/comment_6_3d96568c469a8c53a982f304eae5e7d4._comment b/doc/design/assistant/blog/day_195__real_android_app/comment_6_3d96568c469a8c53a982f304eae5e7d4._comment
new file mode 100644
index 0000000000..0fc0a5cecf
--- /dev/null
+++ b/doc/design/assistant/blog/day_195__real_android_app/comment_6_3d96568c469a8c53a982f304eae5e7d4._comment
@@ -0,0 +1,10 @@
+[[!comment format=mdwn
+ username="rillian"
+ ip="174.6.6.167"
+ subject="no keybourd events on Galaxy Tab 10.1"
+ date="2013-02-22T06:54:54Z"
+ content="""
+Hey, thanks for working on this.
+
+I tried the Feb 21 apk on my Samsung galaxy Tab 10.1 with Android 4.0.4. It installed and launched fine, bringing up a terminal window, keyboard, etc, but only the return key works. It's not possible to type znything. The menus, new window button etc all seem to be functional.
+"""]]
diff --git a/doc/design/assistant/blog/day_195__real_android_app/comment_6_e8667c47d07fc842cf0fe2ebbfbc1c58._comment b/doc/design/assistant/blog/day_195__real_android_app/comment_6_e8667c47d07fc842cf0fe2ebbfbc1c58._comment
new file mode 100644
index 0000000000..8f1c71bad6
--- /dev/null
+++ b/doc/design/assistant/blog/day_195__real_android_app/comment_6_e8667c47d07fc842cf0fe2ebbfbc1c58._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="https://www.google.com/accounts/o8/id?id=AItOawn7gQ1zZDdWhXy9H51W2krZYShNmKL3qfM"
+ nickname="Karsten"
+ subject="comment 6"
+ date="2013-02-22T07:00:41Z"
+ content="""
+Installed on Cyanogenmod 7.2 (Android 2.3.7) on my HTC Desire. When starting the App \"Terminalsitzung wird ausgeführt\" (Terminalsession is started) shows in the statusline, then the app immediately closes.
+"""]]
diff --git a/doc/design/assistant/blog/day_195__real_android_app/comment_7_cf8da7720ddc20b05955ee671ca4acd5._comment b/doc/design/assistant/blog/day_195__real_android_app/comment_7_cf8da7720ddc20b05955ee671ca4acd5._comment
new file mode 100644
index 0000000000..b583d675bf
--- /dev/null
+++ b/doc/design/assistant/blog/day_195__real_android_app/comment_7_cf8da7720ddc20b05955ee671ca4acd5._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="https://www.google.com/accounts/o8/id?id=AItOawm7AuSfii_tCkLyspL6Mr0ATlO6OxLNYOo"
+ nickname="Georg"
+ subject="installed successfully on galaxy nexus stock 4.2.2 rom"
+ date="2013-02-22T08:28:36Z"
+ content="""
+could run git annex and get the help screen. didn't do much else after that since I am at work currently.
+"""]]
diff --git a/doc/design/assistant/blog/day_195__real_android_app/comment_8_f4709bdbc739182819b648fd6aa00531._comment b/doc/design/assistant/blog/day_195__real_android_app/comment_8_f4709bdbc739182819b648fd6aa00531._comment
new file mode 100644
index 0000000000..367e791ff2
--- /dev/null
+++ b/doc/design/assistant/blog/day_195__real_android_app/comment_8_f4709bdbc739182819b648fd6aa00531._comment
@@ -0,0 +1,36 @@
+[[!comment format=mdwn
+ username="https://www.google.com/accounts/o8/id?id=AItOawl9sYlePmv1xK-VvjBdN-5doOa_Xw-jH4U"
+ nickname="Richard"
+ subject="nexus 4, android 4.2.2"
+ date="2013-02-22T11:59:13Z"
+ content="""
+Installs but can not detect email even when set:
+
+    --global user.name \"Your Name\"                               <
+    --global user.email \"you@example.com\"                         <
+    u0_a135@android:/sdcard/git-annex.home $ git annex init
+    init
+      Detected a crippled filesystem.
+
+    *** Please tell me who you are.
+    
+    Run
+    
+      git config --global user.email \"you@example.com\"
+      git config --global user.name \"Your Name\"
+    
+    to set your account's default identity.
+    Omit --global to set the identity only in this repository.
+    
+    fatal: unable to auto-detect email address (got 'u0_a135@localhost.(none)')
+    
+    git-annex: user error (git [\"--git-  dir=/storage/emulated/legacy/git-annex.home/.git\",\"--work-tree=/storage/emulated/legacy/git-annex.home\",\"commit-tree\",\"4b825dc642cb6eb9a060e54bf8d69288fbee4904\"] exited 128)
+failed
+    git-annex: init: 1 failed
+    1|u0_a135@android:/sdcard/git-annex.home $
+
+
+Sorry for bad copy paste... On tour bus in Stockholm and only have phone, no laptop...
+
+RichiH
+"""]]
diff --git a/doc/design/assistant/blog/day_195__real_android_app/comment_9_e66af12c7eca0d457b8406e9fb4b69be._comment b/doc/design/assistant/blog/day_195__real_android_app/comment_9_e66af12c7eca0d457b8406e9fb4b69be._comment
new file mode 100644
index 0000000000..46a9eff3de
--- /dev/null
+++ b/doc/design/assistant/blog/day_195__real_android_app/comment_9_e66af12c7eca0d457b8406e9fb4b69be._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="http://ertai.myopenid.com/"
+ nickname="npouillard"
+ subject="comment 9"
+ date="2013-02-22T15:19:25Z"
+ content="""
+Same trouble with --global config, however local config works.
+"""]]
diff --git a/doc/design/assistant/blog/day_196__android_bugfixes.mdwn b/doc/design/assistant/blog/day_196__android_bugfixes.mdwn
new file mode 100644
index 0000000000..a78853f93e
--- /dev/null
+++ b/doc/design/assistant/blog/day_196__android_bugfixes.mdwn
@@ -0,0 +1,26 @@
+So it seems the Android app works pretty well on a variety of systems.
+Only report of 100% failure so far is on Cyanogenmod 7.2 (Android 2.3.7).
+
+Worked today on some of the obvious bugs.
+
+* Turns out that getEnvironment is broken on Android, returning no
+  environment, which explains the weird git behavior where it complains
+  that it cannot determine the username and email (because it sees no USER
+  or HOST), and suggests setting them in the global git config (which it
+  ignores, because it sees no HOME). Put in a work around for this
+  that makes `git annex init` more pleasant, and opened a bug report on
+  ghc-android.
+* Made the Android app detect when it's been upgraded, and re-link all
+  the commands, etc.
+* Fixed the bug that made `git-annex assistant` on Android re-add all
+  existing files on startup.
+* Enabled a few useful things in busybox. Including vi.
+* Replaced the service notification icon with one with the git-annex logo.
+* Made the terminal not close immediately when the shell exits, which
+  should aid in debugging of certain types of crashes.
+
+I want to set up an autobuilder for Android, but to do that I need to
+install all the haskell libraries on my server. Since getting them built
+for Android involved several days of hacking the first time, this will
+be an opportunity to make sure I can replicate that. Hopefully in less time.
+;)
diff --git a/doc/design/assistant/blog/day_197__template_haskell.mdwn b/doc/design/assistant/blog/day_197__template_haskell.mdwn
new file mode 100644
index 0000000000..cb42b08038
--- /dev/null
+++ b/doc/design/assistant/blog/day_197__template_haskell.mdwn
@@ -0,0 +1,36 @@
+Set up an autobuilder for the linux standalone binaries.
+Did not get an Android autobuilder set up yet, but I did update
+the Android app with recent improvements, so [[upgrade|install/Android]].
+
+----
+
+Investigated further down paths to getting the webapp built for Android.
+
+* Since recent ghc versions support ghci and thus template haskell on arm,
+  at least some of the time, I wonder what's keeping the ghc-android build
+  from doing so? It might be due to it being a cross compiler. I tried
+  recompiling it with the stage 2, native compiler enabled. While I was
+  able to use that ghc binary on Android, it refused to run --interactive,
+  claiming it was not built with that enabled. Don't really understand
+  the ghc build system, so might have missed something.
+
+  Maybe I need to recompile ghc using the native ghc running on Android.
+  But that would involve porting gcc and a lot of libraries and toolchain
+  stuff to Android.
+
+* [yesod-pure](http://hackage.haskell.org/package/yesod-pure) is an option,
+  and I would not mind making all the code changes to use it, getting
+  rid of template haskell entirely. (Probably around 1 thousand lines of
+  code would need to be written, but most of it would be trivial
+  conversion of hamlet templates.)
+  
+  Question is, will yesod install at all without template haskell? Not
+  easily. `vector`, `monad-logger`, `aeson`, `shakespeare`,
+  `shakespeare-css`, `shakespeare-js`, `shakespeare-i18n`, `hamlet`
+  all use TH at build time. Hacked them all to just remove the TH parts.
+
+  The hack job on `yesod-core` was especially rough, involving things like
+  404 handlers. Did get it to build tho!
+
+  Still a dozen packages before I can build yesod, and then will try 
+  building [this yesod-pure demo](https://gist.github.com/snoyberg/3870834/raw/212f0164de36524291df3ab35788e2b72d8d1e75/fib.hs).
diff --git a/doc/design/assistant/blog/day_197__template_haskell/comment_1_82d9f9508929d84abf7b718c59436ae8._comment b/doc/design/assistant/blog/day_197__template_haskell/comment_1_82d9f9508929d84abf7b718c59436ae8._comment
new file mode 100644
index 0000000000..95fcde0d6a
--- /dev/null
+++ b/doc/design/assistant/blog/day_197__template_haskell/comment_1_82d9f9508929d84abf7b718c59436ae8._comment
@@ -0,0 +1,18 @@
+[[!comment format=mdwn
+ username="http://meep.pl/"
+ ip="193.23.174.18"
+ subject="Android 2.3"
+ date="2013-02-24T15:16:51Z"
+ content="""
+The updated version lets me see the failure(s), and it looks like this:
+
+    cd: can't cd to /data/data/ga.androidterm/lib/lib.runshell.so/../..
+    [: not found
+    [: not found
+    could not open //installed-version, No such file or directory
+    /data/data/ga.androidterm/lib/lib.runshell.so: cannot create
+    /sdcard/git-annex.home/git-annex-install.log: directory nonexistent
+
+    [Terminal session finished]
+
+"""]]
diff --git a/doc/design/assistant/blog/day_198__bugfixes.mdwn b/doc/design/assistant/blog/day_198__bugfixes.mdwn
new file mode 100644
index 0000000000..ec8535ad1c
--- /dev/null
+++ b/doc/design/assistant/blog/day_198__bugfixes.mdwn
@@ -0,0 +1,11 @@
+Wrote a C shim to get the Android app started. This avoids it relying on
+the Android /system/bin/sh to run its shell script, or indeed relying on
+any unix utilities from Android at all, which may help on some
+systems. Pushed a new build of the Android app.
+
+Tracked down a failure a lot of people are reporting with WebDAV support
+to a backported security fix in the TLS library, and filed an upstream bug
+about it.
+
+Various other misc fixing and stuff.
+My queue of bug reports and stuff only has 47 items in it now. Urk..
diff --git a/doc/design/assistant/blog/day_198__bugfixes/comment_1_5a15b5bad0f9ba2423d2aebe440ac0ea._comment b/doc/design/assistant/blog/day_198__bugfixes/comment_1_5a15b5bad0f9ba2423d2aebe440ac0ea._comment
new file mode 100644
index 0000000000..279a53f044
--- /dev/null
+++ b/doc/design/assistant/blog/day_198__bugfixes/comment_1_5a15b5bad0f9ba2423d2aebe440ac0ea._comment
@@ -0,0 +1,19 @@
+[[!comment format=mdwn
+ username="http://meep.pl/"
+ ip="193.23.174.18"
+ subject="Android 2.3 again"
+ date="2013-02-24T23:07:28Z"
+ content="""
+I got _a lot_ further with the latest upgrade.
+
+* installation worked
+* `git init` worked
+* first `git annex init` asked to set user.{email,name}
+* `git config --global ...` worked
+* second `git annex init` segfaulted, but not immediately:
+
+      init
+        Detected a crippled filesystem.
+      error: git-annex died of signal 11
+
+"""]]
diff --git a/doc/design/assistant/blog/day_198__bugfixes/comment_2_36d94b838e5e65c85e7afaabe8a578f1._comment b/doc/design/assistant/blog/day_198__bugfixes/comment_2_36d94b838e5e65c85e7afaabe8a578f1._comment
new file mode 100644
index 0000000000..0a34148647
--- /dev/null
+++ b/doc/design/assistant/blog/day_198__bugfixes/comment_2_36d94b838e5e65c85e7afaabe8a578f1._comment
@@ -0,0 +1,12 @@
+[[!comment format=mdwn
+ username="https://www.google.com/accounts/o8/id?id=AItOawm5iosFbL2By7UFeViqkc6v-hoAtqILeDA"
+ nickname="Laszlo"
+ subject="comment 2"
+ date="2013-02-25T11:22:53Z"
+ content="""
+Is the autiobuilder for linux available already?
+I can't find it, and I'm eager to try the thumbdrive related fixes (where I can specify a simple folder).
+
+Best, 
+ Laszlo
+"""]]
diff --git a/doc/design/assistant/blog/day_198__bugfixes/comment_3_ae9b74341a3bc6e1e84d2c0ca4c5f612._comment b/doc/design/assistant/blog/day_198__bugfixes/comment_3_ae9b74341a3bc6e1e84d2c0ca4c5f612._comment
new file mode 100644
index 0000000000..39e017cebb
--- /dev/null
+++ b/doc/design/assistant/blog/day_198__bugfixes/comment_3_ae9b74341a3bc6e1e84d2c0ca4c5f612._comment
@@ -0,0 +1,10 @@
+[[!comment format=mdwn
+ username="http://joeyh.name/"
+ ip="4.152.108.210"
+ subject="comment 3"
+ date="2013-02-25T15:53:30Z"
+ content="""
+@meep try running it with --debug for some more hint of where it's crashing
+
+@Laszlo see [[install/Linux_standalone]]
+"""]]
diff --git a/doc/design/assistant/blog/day_198__bugfixes/comment_4_5a4827227c03bcff3b1e4c44b531f816._comment b/doc/design/assistant/blog/day_198__bugfixes/comment_4_5a4827227c03bcff3b1e4c44b531f816._comment
new file mode 100644
index 0000000000..fafb8fd274
--- /dev/null
+++ b/doc/design/assistant/blog/day_198__bugfixes/comment_4_5a4827227c03bcff3b1e4c44b531f816._comment
@@ -0,0 +1,12 @@
+[[!comment format=mdwn
+ username="https://www.google.com/accounts/o8/id?id=AItOawm5iosFbL2By7UFeViqkc6v-hoAtqILeDA"
+ nickname="Laszlo"
+ subject="comment 4"
+ date="2013-02-25T18:28:09Z"
+ content="""
+joeyh: Is the 20130216 is the newest? Because you just wrote about two days ago of direct mode fixes, I'm interested in.
+I tried 20130216 already.
+
+Laszlo
+
+"""]]
diff --git a/doc/design/assistant/blog/day_198__bugfixes/comment_5_9c5f4c85217e898be4c57c615e53c36f._comment b/doc/design/assistant/blog/day_198__bugfixes/comment_5_9c5f4c85217e898be4c57c615e53c36f._comment
new file mode 100644
index 0000000000..221e28d25a
--- /dev/null
+++ b/doc/design/assistant/blog/day_198__bugfixes/comment_5_9c5f4c85217e898be4c57c615e53c36f._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="http://joeyh.name/"
+ ip="4.152.108.210"
+ subject="comment 5"
+ date="2013-02-26T16:33:11Z"
+ content="""
+I have not made a release with those fixes yet, but they are available in the autobuilds.
+"""]]
diff --git a/doc/design/assistant/blog/day_198__bugfixes/comment_6_bccf1abfb7f56d97673158f3ccfce511._comment b/doc/design/assistant/blog/day_198__bugfixes/comment_6_bccf1abfb7f56d97673158f3ccfce511._comment
new file mode 100644
index 0000000000..22012c0cdf
--- /dev/null
+++ b/doc/design/assistant/blog/day_198__bugfixes/comment_6_bccf1abfb7f56d97673158f3ccfce511._comment
@@ -0,0 +1,9 @@
+[[!comment format=mdwn
+ username="http://meep.pl/"
+ ip="193.23.174.18"
+ subject="git annex init --debug"
+ date="2013-02-27T06:39:19Z"
+ content="""
+The last thing before the signal 11 error that I can see with `--debug` is `read: uname [\"-n\"]` after reports of running 5 git commands.
+
+"""]]
diff --git a/doc/design/assistant/blog/day_198__bugfixes/comment_7_6f1b51b002cc5d2b505d80e3e04bf6f3._comment b/doc/design/assistant/blog/day_198__bugfixes/comment_7_6f1b51b002cc5d2b505d80e3e04bf6f3._comment
new file mode 100644
index 0000000000..8adcd394d2
--- /dev/null
+++ b/doc/design/assistant/blog/day_198__bugfixes/comment_7_6f1b51b002cc5d2b505d80e3e04bf6f3._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="http://meep.pl/"
+ ip="193.23.174.18"
+ subject="comment 7"
+ date="2013-02-27T06:53:15Z"
+ content="""
+But the error does not happen immediately after that. All the 6 log messages have the same timestamp and `git annex init --debug; date` prints a timestamp that is 3-4 seconds later.
+"""]]
diff --git a/doc/design/assistant/blog/day_198__bugfixes/comment_8_8a3542437663028b17442818eba3f7c5._comment b/doc/design/assistant/blog/day_198__bugfixes/comment_8_8a3542437663028b17442818eba3f7c5._comment
new file mode 100644
index 0000000000..f9a50b94e7
--- /dev/null
+++ b/doc/design/assistant/blog/day_198__bugfixes/comment_8_8a3542437663028b17442818eba3f7c5._comment
@@ -0,0 +1,9 @@
+[[!comment format=mdwn
+ username="http://meep.pl/"
+ ip="193.23.174.18"
+ subject="comment 8"
+ date="2013-02-27T06:55:24Z"
+ content="""
+Also, this was after a reinstall. The next time I start the app the terminal opens but prompt does not appear (at least not for a very long time). Then after uninstall and reinstall I get to the prompt again.
+
+"""]]
diff --git a/doc/design/assistant/blog/day_199__wrapping_up_Android_for_now.mdwn b/doc/design/assistant/blog/day_199__wrapping_up_Android_for_now.mdwn
new file mode 100644
index 0000000000..7d37e8f948
--- /dev/null
+++ b/doc/design/assistant/blog/day_199__wrapping_up_Android_for_now.mdwn
@@ -0,0 +1,26 @@
+An Android autobuilder is now set up to run nightly. At this point
+I don't see an immediate way to getting the webapp working on Android, so
+it's best to wait a month or two and see how things develop in Haskell land.
+So I'm moving on to other things.
+
+Today:
+
+* Fixed a nasty regression that made `*` not match files in subdirectories.
+  That broke the preferred content handling, among other things. I will
+  be pushing out a new release soon.
+* As a last Android thing (for now), made the Android app automatically
+  run `git annex assistant --autostart` , so you can manually set up
+  an assistant-driven repository on Android, listing the repository in
+  `.config/git-annex/autostart`
+* Made the webapp display any error message from `git init` if it fails.
+  This was the one remaining gap in the logging.
+  One reason it could fail is if the system has a newer git in use, and
+  `~/.gitconfig` is configured with some options the older git bundled
+  with git-annex doesn't like.
+* Bumped the major version to 4, and annex.version will be set to 4 in
+  new direct mode repositories. (But version 3 is otherwise still used, to
+  avoid any upgrade pain.) This is to prevent old versions that don't
+  understand direct mode from getting confused. I hope direct mode is
+  finally complete, too, after the work to make it work on crippled
+  filesystems this month.
+* Misc other bugfixes etc. Backlog down to 43.
diff --git a/doc/design/assistant/blog/day_199__wrapping_up_Android_for_now/comment_1_ec57358afc7e78d2860aa4237793832d._comment b/doc/design/assistant/blog/day_199__wrapping_up_Android_for_now/comment_1_ec57358afc7e78d2860aa4237793832d._comment
new file mode 100644
index 0000000000..1d79237395
--- /dev/null
+++ b/doc/design/assistant/blog/day_199__wrapping_up_Android_for_now/comment_1_ec57358afc7e78d2860aa4237793832d._comment
@@ -0,0 +1,11 @@
+[[!comment format=mdwn
+ username="https://www.google.com/accounts/o8/id?id=AItOawl9sYlePmv1xK-VvjBdN-5doOa_Xw-jH4U"
+ nickname="Richard"
+ subject="Automagic photo backups"
+ date="2013-02-26T21:56:44Z"
+ content="""
+Did you investigate how hard it would be to only commit & upload once the phone is being charged _and_ on WiFi? That does not rely on the webapp, but it would make using git-annex on Android a lot more useful until you restart your work on this.
+
+
+RichiH
+"""]]
diff --git a/doc/design/assistant/blog/day_19__random_improvements.mdwn b/doc/design/assistant/blog/day_19__random_improvements.mdwn
new file mode 100644
index 0000000000..acb30bf934
--- /dev/null
+++ b/doc/design/assistant/blog/day_19__random_improvements.mdwn
@@ -0,0 +1,50 @@
+Random improvements day..
+
+Got the merge conflict resolution code working in `git annex assistant`.
+
+Did some more fixes to the pushing and pulling code, covering some cases
+I missed earlier. 
+
+Git syncing seems to work well for me now; I've seen it recover
+from a variety of error conditions, including merge conflicts and repos
+that were temporarily unavailable.
+
+----
+
+There is definitely a MVar deadlock if the merger thread's inotify event
+handler tries to run code in the Annex monad. Luckily, it doesn't
+currently seem to need to do that, so I have put off debugging what's going
+on there.
+
+Reworked how the inotify thread runs, to avoid the two inotify threads
+in the assistant now from both needing to wait for program termination,
+in a possibly conflicting manner.
+
+Hmm, that *seems* to have fixed the MVar deadlock problem.
+
+----
+
+Been thinking about how to fix [[bugs/watcher_commits_unlocked_files]].
+Posted some thoughts there.
+
+It's about time to move on to data [[syncing]]. While eventually that will
+need to build a map of the repo network to efficiently sync data over the
+fastest paths, I'm thinking that I'll first write a dumb version. So, two
+more threads:
+
+1. Uploads new data to every configured remote. Triggered by the watcher
+   thread when it adds content. Easy; just use a `TSet` of Keys to send.
+
+2. Downloads new data from the cheapest remote that has it. Could be 
+   triggered by the
+   merger thread, after it merges in a git sync. Rather hard; how does it
+   work out what new keys are in the tree without scanning it all? Scan
+   through the git history to find newly created files? Maybe the watcher
+   triggers this thread instead, when it sees a new symlink, without data,
+   appear.
+
+Both threads will need to be able to be stopped, and restarted, as needed
+to control the data transfer. And a lot of other control smarts will
+eventually be needed, but my first pass will be to do a straightforward
+implementation. Once it's done, the git annex assistant will be basically
+usable.
diff --git a/doc/design/assistant/blog/day_1__inotify.mdwn b/doc/design/assistant/blog/day_1__inotify.mdwn
new file mode 100644
index 0000000000..ca27a45861
--- /dev/null
+++ b/doc/design/assistant/blog/day_1__inotify.mdwn
@@ -0,0 +1,57 @@
+First day of [Kickstarter funded work](http://www.kickstarter.com/projects/joeyh/git-annex-assistant-like-dropbox-but-with-your-own/)! 
+
+Worked on [[inotify]] today. The `watch` branch in git now does a pretty
+good job of following changes made to the directory, annexing files
+as they're added and staging other changes into git. Here's a quick
+transcript of it in action:
+
+	joey@gnu:~/tmp>mkdir demo
+	joey@gnu:~/tmp>cd demo
+	joey@gnu:~/tmp/demo>git init
+	Initialized empty Git repository in /home/joey/tmp/demo/.git/
+	joey@gnu:~/tmp/demo>git annex init demo
+	init demo ok
+	(Recording state in git...)
+	joey@gnu:~/tmp/demo>git annex watch &
+	[1] 3284
+	watch . (scanning...) (started)
+	joey@gnu:~/tmp/demo>dd if=/dev/urandom of=bigfile bs=1M count=2
+	add ./bigfile 2+0 records in
+	2+0 records out
+	2097152 bytes (2.1 MB) copied, 0.835976 s, 2.5 MB/s
+	(checksum...) ok
+	(Recording state in git...)
+	joey@gnu:~/tmp/demo>ls -la bigfile
+	lrwxrwxrwx 1 joey joey 188 Jun  4 15:36 bigfile -> .git/annex/objects/Wx/KQ/SHA256-s2097152--e5ced5836a3f9be782e6da14446794a1d22d9694f5c85f3ad7220b035a4b82ee/SHA256-s2097152--e5ced5836a3f9be782e6da14446794a1d22d9694f5c85f3ad7220b035a4b82ee
+	joey@gnu:~/tmp/demo>git status -s
+	A  bigfile
+	joey@gnu:~/tmp/demo>mkdir foo
+	joey@gnu:~/tmp/demo>mv bigfile foo
+	"del ./bigfile"
+	joey@gnu:~/tmp/demo>git status -s
+	AD bigfile
+	A  foo/bigfile
+
+Due to Linux's inotify interface, this is surely some of the most subtle,
+race-heavy code that I'll need to deal with while developing the git annex
+assistant. But I can't start wading, need to jump off the deep end to make
+progress!
+
+The hardest problem today involved the case where a directory is moved
+outside of the tree that's being watched. Inotify will still send events
+for such directories, but it doesn't make sense to continue to handle them.
+
+Ideally I'd stop inotify watching such directories, but a lot of state
+would need to be maintained to know which inotify handle to stop watching.
+(Seems like Haskell's inotify API makes this harder than it needs to be...)
+
+Instead, I put in a hack that will make it detect inotify events from
+directories moved away, and ignore them. This is probably acceptable,
+since this is an unusual edge case.
+
+----
+
+The notable omission in the inotify code, which I'll work on next, is
+staging deleting of files. This is tricky because adding a file to the
+annex happens to cause a deletion event. I need to make sure there are no
+races where that deletion event causes data loss.
diff --git a/doc/design/assistant/blog/day_200__release_day.mdwn b/doc/design/assistant/blog/day_200__release_day.mdwn
new file mode 100644
index 0000000000..99aebcf700
--- /dev/null
+++ b/doc/design/assistant/blog/day_200__release_day.mdwn
@@ -0,0 +1,19 @@
+As well as making a new release, I rewrote most of the Makefile, so that it
+uses cabal to build git-annex. This avoids some duplication, and most
+importantly, means that the Makefile can auto-detect available libraries
+rather than needing to juggle build flags manually. Which was becoming a
+real pain.
+
+I had avoided doing this before because cabal is slow for me on my little
+netbook. Adding ten seconds to every rebuild really does matter! But I came
+up with a hack to let me do incremental development builds without the
+cabal overhead, by intercepting and reusing the ghc command that cabal
+runs.
+
+There was also cabal "fun" to get the Android build working with cabal.
+And more fun involving building the test suite. For various reasons, I
+decided to move the test suite into the git-annex binary. So you can run
+`git annex test` at any time, any place, and it self-tests. That's a neat
+trick I've seen one or two other programs do, and probably the nicest thing
+to come out of what was otherwise a pretty yak shaving change that involved
+babysitting builds all day.
diff --git a/doc/design/assistant/blog/day_200__release_day/comment_10_40cfe9bfd9e611fd734dbb5aad348aa3._comment b/doc/design/assistant/blog/day_200__release_day/comment_10_40cfe9bfd9e611fd734dbb5aad348aa3._comment
new file mode 100644
index 0000000000..dfd4f32fed
--- /dev/null
+++ b/doc/design/assistant/blog/day_200__release_day/comment_10_40cfe9bfd9e611fd734dbb5aad348aa3._comment
@@ -0,0 +1,10 @@
+[[!comment format=mdwn
+ username="http://joeyh.name/"
+ nickname="joey"
+ subject="comment 10"
+ date="2013-03-01T04:23:48Z"
+ content="""
+@Brian, I follow your logic, it does seem that start.c is not finding itself. But, you checked the pid of the terminal, while start.c is the program it (usually) starts.
+
+Anyway, I have tried making start.c fall back to a hard coded path if it cannot find busybox. It'll also say what path it detected, which may help debug the underlying problem. You can pick up an app with this change from the autobuilder.
+"""]]
diff --git a/doc/design/assistant/blog/day_200__release_day/comment_11_b26890fdae575d42170988073fb2e45d._comment b/doc/design/assistant/blog/day_200__release_day/comment_11_b26890fdae575d42170988073fb2e45d._comment
new file mode 100644
index 0000000000..164dce53fc
--- /dev/null
+++ b/doc/design/assistant/blog/day_200__release_day/comment_11_b26890fdae575d42170988073fb2e45d._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="https://www.google.com/accounts/o8/id?id=AItOawmmLfBDEDFfEOba8Ra46nsnTmoNHFsLObo"
+ nickname="Brian"
+ subject="comment 11"
+ date="2013-03-01T12:02:09Z"
+ content="""
+Good point--it didn't occur to me start.c runs as a child process. Makes sense. How do I download the output of the autobuild? Size-wise and behavior-wise, http://downloads.kitenet.net/git-annex/android/current/git-annex.apk looks like the same file I already had.
+"""]]
diff --git a/doc/design/assistant/blog/day_200__release_day/comment_13_710a30c5d31bf549833ecfe9a0997c94._comment b/doc/design/assistant/blog/day_200__release_day/comment_13_710a30c5d31bf549833ecfe9a0997c94._comment
new file mode 100644
index 0000000000..50b99a9658
--- /dev/null
+++ b/doc/design/assistant/blog/day_200__release_day/comment_13_710a30c5d31bf549833ecfe9a0997c94._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="http://joeyh.name/"
+ nickname="joey"
+ subject="comment 13"
+ date="2013-03-01T16:36:58Z"
+ content="""
+@Brian the autobuild is linked to on [[install/Android]]
+"""]]
diff --git a/doc/design/assistant/blog/day_200__release_day/comment_13_b6f62ab7e810ba6d3a43f0ead370c79a._comment b/doc/design/assistant/blog/day_200__release_day/comment_13_b6f62ab7e810ba6d3a43f0ead370c79a._comment
new file mode 100644
index 0000000000..efd38879fe
--- /dev/null
+++ b/doc/design/assistant/blog/day_200__release_day/comment_13_b6f62ab7e810ba6d3a43f0ead370c79a._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="https://www.google.com/accounts/o8/id?id=AItOawmmLfBDEDFfEOba8Ra46nsnTmoNHFsLObo"
+ nickname="Brian"
+ subject="comment 13"
+ date="2013-03-01T23:52:53Z"
+ content="""
+I got the latest autobuild version installed and the fallback works for me--thanks. FYI, the fallback error message says: \"cannot find expected files in /data/app-lib\"
+"""]]
diff --git a/doc/design/assistant/blog/day_200__release_day/comment_1_a68e1ed7829b49086c567d97ddc09912._comment b/doc/design/assistant/blog/day_200__release_day/comment_1_a68e1ed7829b49086c567d97ddc09912._comment
new file mode 100644
index 0000000000..51247f6b70
--- /dev/null
+++ b/doc/design/assistant/blog/day_200__release_day/comment_1_a68e1ed7829b49086c567d97ddc09912._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="https://www.google.com/accounts/o8/id?id=AItOawnxp2XU8gIribhhGhGuYtU6eMMwHv5gUGI"
+ nickname="Amitai"
+ subject="comment 1"
+ date="2013-02-28T02:53:53Z"
+ content="""
+That _is_ a neat trick, especially since you're deploying to odd platforms where you'll probably be even gladder of the in-app tests far in the future than you are now!
+"""]]
diff --git a/doc/design/assistant/blog/day_200__release_day/comment_2_39d3ad0a029fe56e96f97d28d17fbbd2._comment b/doc/design/assistant/blog/day_200__release_day/comment_2_39d3ad0a029fe56e96f97d28d17fbbd2._comment
new file mode 100644
index 0000000000..f6aeb306de
--- /dev/null
+++ b/doc/design/assistant/blog/day_200__release_day/comment_2_39d3ad0a029fe56e96f97d28d17fbbd2._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="http://meep.pl/"
+ ip="193.23.174.18"
+ subject="comment 2"
+ date="2013-02-28T06:13:21Z"
+ content="""
+Maybe it's the build changes? Android apk disappeared from 
+"""]]
diff --git a/doc/design/assistant/blog/day_200__release_day/comment_3_5b752d6a8d74e61190f09384b6108206._comment b/doc/design/assistant/blog/day_200__release_day/comment_3_5b752d6a8d74e61190f09384b6108206._comment
new file mode 100644
index 0000000000..066abd19ab
--- /dev/null
+++ b/doc/design/assistant/blog/day_200__release_day/comment_3_5b752d6a8d74e61190f09384b6108206._comment
@@ -0,0 +1,31 @@
+[[!comment format=c
+ username="https://www.google.com/accounts/o8/id?id=AItOawm5iosFbL2By7UFeViqkc6v-hoAtqILeDA"
+ nickname="Laszlo"
+ subject="comment 3"
+ date="2013-02-28T08:40:46Z"
+ content="""
+Two suggestion:
+1. Possibility to restrict CPU usage (a 10-20% would be nice, or a slider)
+2. Pager on Dashboard
+
+I have a directory, it is 741MB, and contains 23381 files. 
+This directory is pretty much everything needed for my daily job.
+
+I added a copy of this directory to annex, then I created a backup dir of annex, 
+and my laptop is using almost 100%CPU al the time, it is running since 1 hour already, and 
+only 140MB was copied over to the backup directory.
+
+Don't get me wrong, this application just blows my mind, and it seems to *work* for the first time for real.
+
+The suggestion is simply a slider in resource manager (preferably inside dashboard):
+Restrict CPU usage [0---X---100]34%
+Restrict upload bandwith [0-X----800]9kB/s
+Restrict download bandwith [0-X----800]80kB/s
+
+Also the pager is musthave in dashboard, because it renders firefox unusable with that many entries (21000).
+
+Nice work, thank you for the release!
+
+Laszlo
+
+"""]]
diff --git a/doc/design/assistant/blog/day_200__release_day/comment_4_881274ae0d6230bb4cafa4151ad72b49._comment b/doc/design/assistant/blog/day_200__release_day/comment_4_881274ae0d6230bb4cafa4151ad72b49._comment
new file mode 100644
index 0000000000..75d54e04e2
--- /dev/null
+++ b/doc/design/assistant/blog/day_200__release_day/comment_4_881274ae0d6230bb4cafa4151ad72b49._comment
@@ -0,0 +1,12 @@
+[[!comment format=mdwn
+ username="https://www.google.com/accounts/o8/id?id=AItOawm5iosFbL2By7UFeViqkc6v-hoAtqILeDA"
+ nickname="Laszlo"
+ subject="lot of stuck git processes"
+ date="2013-02-28T09:01:25Z"
+ content="""
+Oh, and here is how it looks 
+ps -e |grep git 
+command after git-annex has been stopped (from the gui):
+(54 process, from which 50 process )
+http://pastebin.com/5u74qTNU
+"""]]
diff --git a/doc/design/assistant/blog/day_200__release_day/comment_5_e220059be77cf0ef396f37a4f9ccf9b5._comment b/doc/design/assistant/blog/day_200__release_day/comment_5_e220059be77cf0ef396f37a4f9ccf9b5._comment
new file mode 100644
index 0000000000..bd518cc0a7
--- /dev/null
+++ b/doc/design/assistant/blog/day_200__release_day/comment_5_e220059be77cf0ef396f37a4f9ccf9b5._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="http://joeyh.name/"
+ nickname="joey"
+ subject="comment 5"
+ date="2013-02-28T19:42:09Z"
+ content="""
+@meep That's not the autobuild. I put the wrong file there! Fixed.
+"""]]
diff --git a/doc/design/assistant/blog/day_200__release_day/comment_6_ec2152151188dd252cdb61c68cfc12e4._comment b/doc/design/assistant/blog/day_200__release_day/comment_6_ec2152151188dd252cdb61c68cfc12e4._comment
new file mode 100644
index 0000000000..43e676f609
--- /dev/null
+++ b/doc/design/assistant/blog/day_200__release_day/comment_6_ec2152151188dd252cdb61c68cfc12e4._comment
@@ -0,0 +1,10 @@
+[[!comment format=mdwn
+ username="http://joeyh.name/"
+ nickname="joey"
+ subject="comment 6"
+ date="2013-02-28T19:52:59Z"
+ content="""
+Lazlo, could you please file a proper [[bug|bugs]] report and include details like what version of git-annex you're using there when you get the zombies.
+
+I have modified the webapp to only ever show 10 queued transfers on the dashboard.
+"""]]
diff --git a/doc/design/assistant/blog/day_200__release_day/comment_7_42572411617c287368482bb9dcf94324._comment b/doc/design/assistant/blog/day_200__release_day/comment_7_42572411617c287368482bb9dcf94324._comment
new file mode 100644
index 0000000000..409ade91db
--- /dev/null
+++ b/doc/design/assistant/blog/day_200__release_day/comment_7_42572411617c287368482bb9dcf94324._comment
@@ -0,0 +1,18 @@
+[[!comment format=mdwn
+ username="https://www.google.com/accounts/o8/id?id=AItOawmmLfBDEDFfEOba8Ra46nsnTmoNHFsLObo"
+ nickname="Brian"
+ subject="link busybox: No such file or directory"
+ date="2013-03-01T02:32:32Z"
+ content="""
+When I run the 28-Feb-2013 android version of git annex, I'm getting the error:
+
+    link busybox: No such file or directory
+
+Maybe this code is not finding the busybox.so file:
+
+    link(\"lib/lib.busybox.so\", \"busybox\")
+
+Any ideas how I can troubleshoot?
+
+I'm running Android 4.2.2.
+"""]]
diff --git a/doc/design/assistant/blog/day_200__release_day/comment_8_6b69aa81a9ba4e07e547ed1869946d51._comment b/doc/design/assistant/blog/day_200__release_day/comment_8_6b69aa81a9ba4e07e547ed1869946d51._comment
new file mode 100644
index 0000000000..c375f3b844
--- /dev/null
+++ b/doc/design/assistant/blog/day_200__release_day/comment_8_6b69aa81a9ba4e07e547ed1869946d51._comment
@@ -0,0 +1,15 @@
+[[!comment format=mdwn
+ username="http://joeyh.name/"
+ nickname="joey"
+ subject="comment 8"
+ date="2013-03-01T03:10:35Z"
+ content="""
+@Brian what you can do is go to the terminal's preferences menu and change the shell it runs to `/system/bin/sh` . This should let you open a new tab with a shell prompt. Then you can try to cd to `/data/data/ga.androidterm`, and see if both `lib/lib.start.so` and `lib/lib.busybox.so` exist.
+
+You could then try to do the same thing it does to set up the system:
+
+
    +ln lib/lib.busybox.so busybox
+./busybox sh lib/lib.runshell.so
+
    
+"""]]
diff --git a/doc/design/assistant/blog/day_200__release_day/comment_9_b070a2e4151d9fbf43d7906efa78515f._comment b/doc/design/assistant/blog/day_200__release_day/comment_9_b070a2e4151d9fbf43d7906efa78515f._comment
new file mode 100644
index 0000000000..4acf003c86
--- /dev/null
+++ b/doc/design/assistant/blog/day_200__release_day/comment_9_b070a2e4151d9fbf43d7906efa78515f._comment
@@ -0,0 +1,12 @@
+[[!comment format=mdwn
+ username="https://www.google.com/accounts/o8/id?id=AItOawmmLfBDEDFfEOba8Ra46nsnTmoNHFsLObo"
+ nickname="Brian"
+ subject="comment 9"
+ date="2013-03-01T03:55:46Z"
+ content="""
+@joey Both .so files were present and I was able to run the ln and busybox commands. And after that 'git annex' displayed the help screen. Thanks!
+
+Then I ran 'ps | grep ga.androidterm' to find the pid and then 'ls -l /proc/<pid>/exe' and it showed the symlink points at '/system/bin/app_process'.
+
+So the 'int n=readlink(\"/proc/self/exe\", buf, 1023);' in start.c isn't very useful in this case.
+"""]]
diff --git a/doc/design/assistant/blog/day_201__real_Android_wrapup.mdwn b/doc/design/assistant/blog/day_201__real_Android_wrapup.mdwn
new file mode 100644
index 0000000000..8ff743f68c
--- /dev/null
+++ b/doc/design/assistant/blog/day_201__real_Android_wrapup.mdwn
@@ -0,0 +1,38 @@
+[[!meta title="day 201  real Android wrapup"]]
+
+I got yesod-pure fully working on Android...
+
+[[!img fib.png size=400x]]
+
+As expected, this involved manually splicing some template haskell. I'm now
+confident I can port the git-annex webapp to Android this way, and that it
+will take about a week. Probably will start on that in a month or so. If
+anyone has some spare Android hardware they'd like to loan me, possibly
+sooner. (Returning loaner Asus Transformer tomorrow; thanks Mark.) Although
+I'm inclined to let the situation develop; we may just get a ghc-android
+that supports TH..
+
+Also:
+
+* Fixed several bugs in the Android installation process.
+* Committed patches for all Haskell libraries I've modified to
+  the git-annex git repo.
+* Ran the test suite on Android. It found a problem; seems `git clone`
+  of a local repository is broken in the Android environment.
+
+Non-Android:
+
+* Made the assistant check every hour if logs have grown larger than a
+  megabyte, and rotate them to avoid using too much disk space.
+* Avoided noise in log about typechanged objects when running
+  git commit in direct mode repositories. Seems `git commit`
+  has no way to shut that up, so I had to /dev/null it.
+* When run with `--debug`, the assistant now logs more information
+  about why it transfers or drops objects.
+* Found and fixed a case where moving a file to an archive directory would
+  not cause its content to be dropped.
+* Working on a bug with the assistant where moving a file out of an 
+  archive directory in direct mode sometimes ends up with a symlink
+  rather than a proper direct mode file. Have not gotten to the bottom
+  of it entirely, but it's a race, and I think the race is between
+  the direct mode mapping being updated, and the file being transferred.
diff --git a/doc/design/assistant/blog/day_201__real_Android_wrapup/comment_1_88b9950c51324f0bb89c5646b3170952._comment b/doc/design/assistant/blog/day_201__real_Android_wrapup/comment_1_88b9950c51324f0bb89c5646b3170952._comment
new file mode 100644
index 0000000000..b9edfc4b61
--- /dev/null
+++ b/doc/design/assistant/blog/day_201__real_Android_wrapup/comment_1_88b9950c51324f0bb89c5646b3170952._comment
@@ -0,0 +1,19 @@
+[[!comment format=mdwn
+ username="http://ao2.it/"
+ nickname="ao2"
+ subject="Using the Emulator?"
+ date="2013-03-04T13:58:18Z"
+ content="""
+Hi Joey,
+
+while you wait for some actual Android device have you tried the emulator?
+
+I've recently started playing with Android without any physical device, and the emulator is pretty responsive with hardware acceleration on (you need to run an x86 AVD).
+
+Here are some instructions: http://git.ao2.it/android/android-app-development-getting-started.git/blob/HEAD:/android-app-development-getting-started.sh
+
+Also, are you going to use a WebView connecting to the haskell web server for the WebApp?
+
+Ciao,
+   Antonio
+"""]]
diff --git a/doc/design/assistant/blog/day_201__real_Android_wrapup/fib.png b/doc/design/assistant/blog/day_201__real_Android_wrapup/fib.png
new file mode 100644
index 0000000000..f9b8c27fd4
Binary files /dev/null and b/doc/design/assistant/blog/day_201__real_Android_wrapup/fib.png differ
diff --git a/doc/design/assistant/blog/day_201__working_web_server.mdwn b/doc/design/assistant/blog/day_201__working_web_server.mdwn
new file mode 100644
index 0000000000..c9e959a334
--- /dev/null
+++ b/doc/design/assistant/blog/day_201__working_web_server.mdwn
@@ -0,0 +1,31 @@
+Seems I am not done with the Android porting just yet after all. One more
+porting day..
+
+Last night I managed to get all of Yesod to build for Android.
+I even successfully expanded some Template Haskell used in yesod-form. And
+am fairly confident I could manually expand all the TH in there, so it's
+actually useable without TH. Most of the TH is just commented out for now.
+
+However, programs using Yesod didn't link; lots of missing symbols. I have
+been fighting to fix those all day today.
+
+Finally, I managed to build [the yesod-pure demo server](https://gist.github.com/snoyberg/3870834/raw/212f0164de36524291df3ab35788e2b72d8d1e75/fib.hs),
+and I have a working web server on Android! It listens for requests, it logs
+them correctly, and it replies to requests. I did cripple yesod's routing
+code in my hack-n-slash port of it, so it fails to *display* any pages,
+but never has "Internal Server Error" in a web browser been such a sweet
+sight. ;-)
+
+At this point, I estimate about 1 or 2 weeks work to get to an Android
+webapp. I'd need to:
+
+1. More carefully port Yesod, manually expanding all Template Haskell
+   as I went, rather than commenting it all out like I did this time.
+2. Either develop a tool to automatically expand Hamlet TH splices
+   (preferred; seems doable), or convert all the webapp's templates
+   to not use Hamlet.
+
+-----
+
+I've modified 38 Haskell libraries so far to port them to Android. Mostly
+small hacks, but eep this is a lot of stuff to keep straight.
diff --git a/doc/design/assistant/blog/day_203__procrastination.mdwn b/doc/design/assistant/blog/day_203__procrastination.mdwn
new file mode 100644
index 0000000000..b6eb262e96
--- /dev/null
+++ b/doc/design/assistant/blog/day_203__procrastination.mdwn
@@ -0,0 +1,25 @@
+Stuck on a bug or two, I instead built a new Preferences page:
+
+[[!img /assistant/preferences.png]]
+
+The main reason I wanted that was to allow enabling debug logging at
+runtime. But I've also wanted to expose annex.diskreserve and
+annex.numcopies settings to the webapp user. Might as well let them control
+whether it auto-starts too.
+
+Had some difficulty deciding where to put this. It could be considered
+additional configuration for the local repository, and so go in the
+local repository edit form. However, this stuff can only be configured for
+local repositories, and not remotes, and that same form is used to edit 
+remotes, which would lead to inconsistent UI and complicate the code.
+Also, it might grow to include things not tied to any repository,
+like choice of key/value backends. So, I put the preferences on their own
+page.
+
+---
+
+Also, based on some useful feedback from testing the assistant with a large
+number of files, I made the assistant disable git-gc auto packing in
+repositories it sets up. (Like fsck, git-gc always seems to run exactly
+when you are in a hurry.) Instead, it'll pack at most once a day, and with
+a rather higher threshold for the number of loose objects.
diff --git a/doc/design/assistant/blog/day_204__deprocrastination.mdwn b/doc/design/assistant/blog/day_204__deprocrastination.mdwn
new file mode 100644
index 0000000000..6299140e35
--- /dev/null
+++ b/doc/design/assistant/blog/day_204__deprocrastination.mdwn
@@ -0,0 +1,62 @@
+Tracked down the bug that's been eluding me for days. It was indeed a race, and
+could result in a file being transferred into a direct mode repository and
+ending up in indirect mode. Was easy to fix once understood, just needed to
+update the direct mode mapping before starting the transfer.
+
+While I was in there, I noticed another potential race, also in direct
+mode, where the watcher could decide to rewrite a symlink to fix its
+target, and at just the wrong time direct mode content could arrive in its
+place, and so get deleted. Fixed that too.
+
+Seems likely there are some other direct mode races. I spent quite a while
+hammering on dealing with the indirect mode races with the assistant 
+originally.
+
+-----
+
+Next on my list is revisiting XMPP.
+
+Verified that git push over XMPP works between multiple repositories that
+are sharing the same XMPP account. It does.
+
+Seeing the XMPP setup process with fresh eyes, I found several places
+wording could be improved. Also, when the user goes in and configures 
+(or reconfigures) an XMPP account, the next step is to do pairing,
+so it now redirects directly to there.
+
+Next I need to make XMPP get back into sync after a network disconnection
+or when the assistant is restarted. This currently doesn't happen until
+a XMPP push is received due to a new change being made.
+
+### back burner: yesod-pure
+
+Last night I made a yesod-pure branch, and did some exploratory conversion
+away from using Hamlet, of the Preferences page I built yesterday.
+
+I was actually finding writing pure Blaze worked *better* than Hamlet,
+at first. Was able to refactor several things into functions that in Hamlet
+are duplicated over and over in my templates, and built some stuff that makes
+rendering type safe urls in pure Blaze not particularly ungainly. For example,
+this makes a submit button and a cancel button that redirects to another page:
+
+[[!format haskell """
+        buttons = toWidget $ \redir ->
+               "Save Preferences" <>|<> redir ConfigurationR []
+"""]]
+
+The catch is how to deal with widgets that need to be nested inside other
+html. It's not possible to do this both cleanly and maximally
+efficiently, with Blaze. For max efficiency, all the html before the widget
+should be emitted, and then the widget run, and then all html after it be
+emitted. To use Blaze, it would have to instead generate the full html,
+then split it around the widget, and then emit the parts, which is less
+efficient, doesn't stream, etc.
+
+I guess that's the core of what Hamlet does; it allows a clean
+representation and due to running TH at build time, can convert this into
+an efficient (but ugly) html emitter.
+
+So, I may give up on this experiment. Or I may make the webapp less than
+maximally fast at generating html and go on with it. After all, these
+sorts of optimisations are mostly aimed at high-volume websites, not local
+webapps.
diff --git a/doc/design/assistant/blog/day_205_206__rainy_day__snow_day.mdwn b/doc/design/assistant/blog/day_205_206__rainy_day__snow_day.mdwn
new file mode 100644
index 0000000000..e1d07c8e42
--- /dev/null
+++ b/doc/design/assistant/blog/day_205_206__rainy_day__snow_day.mdwn
@@ -0,0 +1,12 @@
+Yesterday was all bug fixes, nothing to write about really.
+
+Today I've been working on getting XMPP remotes to sync more reliably.
+I left some big holes when I stopped work on it in November:
+
+1. The assistant did not sync with XMPP remotes when it started up.
+2. .. Or when it detected a network reconnection.
+3. There was no way to trigger a full scan for transfers
+   after receiving a push from an XMPP remote.
+
+The asynchronous nature of git push over XMPP complicated doing this, but
+I've solved all 3 issues today.
diff --git a/doc/design/assistant/blog/day_207__XMPP.mdwn b/doc/design/assistant/blog/day_207__XMPP.mdwn
new file mode 100644
index 0000000000..8419905f16
--- /dev/null
+++ b/doc/design/assistant/blog/day_207__XMPP.mdwn
@@ -0,0 +1,7 @@
+More XMPP fixes. The most important change is that it now stores important
+messages, like push requests, and (re)sends them when a buddy's client
+sends XMPP presence. This makes XMPP syncing much more robust, all the
+clients do not need to already be connected when messages are initially
+sent, but can come and go. Also fixed a bug preventing syncing from working
+immediately after XMPP pairing. XMPP seems to be working well now; I only
+know of one minor bug.
diff --git a/doc/design/assistant/blog/day_208__bugfixes.mdwn b/doc/design/assistant/blog/day_208__bugfixes.mdwn
new file mode 100644
index 0000000000..41874b9184
--- /dev/null
+++ b/doc/design/assistant/blog/day_208__bugfixes.mdwn
@@ -0,0 +1,17 @@
+Fixed the last XMPP bug I know of. Turns out it was not specific to XMPP at
+all; the assistant could forget to sync with any repository on startup
+under certain conditions.
+
+Also fixed bugs in `git annex add` and in the glob matching, and some more.
+
+I've been working on some screencasts. More on them later.. But while doing
+them I found a perfect way to reliably reproduce the webapp hang that
+I've been chasing for half a year, and last saw at my presentation in
+Australia. Seems the old joke about bugs only reproducible during
+presentations is literally true here!
+
+I have given this bug its [[own page|bugs/webapp_hang]] at last, and have a
+tcpdump of it happening and everything. Am working on an hypotheses that it
+might be caused by Warp's [slowloris](http://ha.ckers.org/slowloris/)
+attack prevention code being falsely triggered by the repeated hits the web
+browser makes as the webapp's display is updated.
diff --git a/doc/design/assistant/blog/day_209__The_Bug.mdwn b/doc/design/assistant/blog/day_209__The_Bug.mdwn
new file mode 100644
index 0000000000..b2e87fd8fd
--- /dev/null
+++ b/doc/design/assistant/blog/day_209__The_Bug.mdwn
@@ -0,0 +1,23 @@
+> And so we waited. Tick-tock, blink-blink, thirty seconds stretched
+> themselves out one by one, a hole in human experience. -- The Bug
+
+I *think* I've managed to fully track down the [[webapp_hang]]. It is,
+apparently, a bug in the Warp web server's code intended to protect against
+the [Slowloris](http://ha.ckers.org/slowloris/) attack. It assumes,
+incorrectly, that a web browser won't reuse a connection it's left idle for
+30 seconds. Some bad error handling keeps a connection open with no thread
+to service it, leading to the hang.
+
+
+Have put a 30 minute timeout into place as a workaround, and, unless
+a web browser sits on an idle connection for a full 30 minutes and then
+tries to reuse it, this should be sufficient.
+
+I was chasing that bug, quietly, for 6 months. Would see it now and
+then, but not be able to reproduce it or get anywhere with analysis.
+I had nearly given up. If you enjoy stories like that, read Ellen
+Ullman's excellent book The Bug.
+
+> To discover that between the blinks of the machine’s shuttered eye—going
+> on without pause or cease; simulated, imagined, but still not caught—was
+> life.
diff --git a/doc/design/assistant/blog/day_20__data_transfer_design.mdwn b/doc/design/assistant/blog/day_20__data_transfer_design.mdwn
new file mode 100644
index 0000000000..4f47ae63c4
--- /dev/null
+++ b/doc/design/assistant/blog/day_20__data_transfer_design.mdwn
@@ -0,0 +1,22 @@
+Today is a planning day. I have only a few days left before I'm off to
+Nicaragua for [DebConf](http://debconf12.debconf.org/), where I'll only
+have smaller chunks of time without interruptions. So it's important to get
+some well-defined smallish chunks designed that I can work on later. See
+bulleted action items below (now moved to [[syncing]]. Each
+should be around 1-2 hours unless it turns out to be 8 hours... :)
+
+First, worked on writing down a design, and some data types, for data transfer
+tracking (see [[syncing]] page). Found that writing down these simple data
+types before I started slinging code has clarified things a lot for me.
+
+Most importantly, I realized that I will need to modify `git-annex-shell`
+to record on disk what transfers it's doing, so the assistant can get that
+information and use it to both avoid redundant transfers (potentially a big
+problem!), and later to allow the user to control them using the web app.
+
+While eventually the user will be able to use the web app to prioritize
+transfers, stop and start, throttle, etc, it's important to get the default
+behavior right. So I'm thinking about things like how to prioritize uploads
+vs downloads, when it's appropriate to have multiple downloads running at
+once, etc.
+
diff --git a/doc/design/assistant/blog/day_210__spring.mdwn b/doc/design/assistant/blog/day_210__spring.mdwn
new file mode 100644
index 0000000000..ff34cfbfe0
--- /dev/null
+++ b/doc/design/assistant/blog/day_210__spring.mdwn
@@ -0,0 +1,29 @@
+Trying to record screencasts demoing the assistant is really helping me
+see things that need to be fixed.
+
+Got the version of the haskell TLS library in Debian fixed, backporting
+some changes to fix a botched security fix that made it reject all
+certificates. So WebDAV special remotes will work again on the next release.
+
+Fixed some more problems around content being dropped when files are
+moved to archive directories, and gotten again when files are
+moved out.
+
+Fixed some problems around USB drives. One was a real jaw-dropping
+bug: "git annex drop --from usbdrive" when the drive was not
+connected still updated the location log to indicate it did not have
+the file anymore! (Thank goodness for fsck..)
+
+I've noticed that moving around files in direct mode repos is inneficient,
+because the assistant re-checksums the "new" file. One way to avoid
+that would be to have a lookup table from (inode, size, mtime) to
+key, but I don't have one, and would like to avoid adding one.
+
+Instead, I have a cunning plan to deal with this heuristically. If the
+assistant can notice a file was removed and another file added at the same
+time, it can compare the (inode, size, mtime) to see if it's a rename, and
+avoid the checksum overhead.
+
+The first step to getting there was to make the assistant better at
+batching together delete+add events into a single rename commit. I'm happy
+to say I've accomplished that, with no perceptable delay to commits.
diff --git a/doc/design/assistant/blog/day_211__zooming_along.mdwn b/doc/design/assistant/blog/day_211__zooming_along.mdwn
new file mode 100644
index 0000000000..fb87077bef
--- /dev/null
+++ b/doc/design/assistant/blog/day_211__zooming_along.mdwn
@@ -0,0 +1,24 @@
+Got renaming fully optimised in the assistent in direct mode. I even got it
+to work for whole directory renames. I can drag files around all day in the
+file manager and the assistant often finishes committing the rename before
+the file manager updates. So much better than checksumming every single
+renamed file! Also, this means the assistant makes just 1 commit when a
+whole directory is renamed.
+
+Last night I added a feature to `git annex status`. It can now be asked to
+only show the status of a single directory, rather than the whole annex.
+All the regular file filtering switches work, so some neat commands
+are possible. I like `git annex status . --in foo --not --in bar` to see
+how much data is in one remote but not another.
+
+This morning, an important thought about [[bugs/smarter_flood_filling]],
+that will avoid unnecessary uploads to transfer remotes when all that's
+needed to get the file to its destination is a transfer over the LAN.
+I found an easy way to make that work, at least in simple cases.
+Hoping to implement it soon.
+
+Less fun, direct mode turns out to be somewhat buggy when files with
+duplicate content are in the repository. Nothing fails, but `git annex
+sync` will re-checksum files each time it's run in this situation, and the
+assistant will re-checksum files in certian cases. Need to work on this
+soon too.
diff --git a/doc/design/assistant/blog/day_212__accidental_all_nighter.mdwn b/doc/design/assistant/blog/day_212__accidental_all_nighter.mdwn
new file mode 100644
index 0000000000..600522d14a
--- /dev/null
+++ b/doc/design/assistant/blog/day_212__accidental_all_nighter.mdwn
@@ -0,0 +1,24 @@
+Last night, revamped the web site, including making a [[/videos]]
+page, which includes a new screencast introducing the git-annex assistant.
+
+Worked on improving my Haskell development environment in vim.
+hdevtools is an excellent but tricky thing to get working. Where before
+it took around 30 seconds per compile for me to see type errors,
+I now see them in under a second each time I save, and can also look up
+types of any expression in the file. Since programming in Haskell is
+mostly driven by reacting to type errors ;) this should speed me up a lot,
+although it's not perfect. Unfortunatly, I got really caught up in tuning
+my setup, and only finished doing that at 5:48 am.
+
+Disasterously late this morning, fixed the assistant's
+`~/.ssh/git-annex-shell` wrapper so it will work when the ssh key does
+not force a command to be run. Also made the webapp behave better
+when it's told to create a git repository that already exists.
+
+After entirely too little sleep, I found a puzzling bug where copying files
+to a local repo fails once the inode cache has been invalidated. This
+turned out to involve running a check in the state monad of the wrong
+repository. A failure mode I'd never encountered before.
+
+Only thing I had brains left to do today was to record another screencast,
+which is rendering now...
diff --git a/doc/design/assistant/blog/day_212__accidental_all_nighter/comment_1_6ee1f8056eedb6eb18013faf8f5ec212._comment b/doc/design/assistant/blog/day_212__accidental_all_nighter/comment_1_6ee1f8056eedb6eb18013faf8f5ec212._comment
new file mode 100644
index 0000000000..6ad0ab2354
--- /dev/null
+++ b/doc/design/assistant/blog/day_212__accidental_all_nighter/comment_1_6ee1f8056eedb6eb18013faf8f5ec212._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="https://www.google.com/accounts/o8/id?id=AItOawl1D_4vD5ueaDw8gRsIYPO3UHRKEpFfg9I"
+ nickname="Дмитрий"
+ subject="About development in Vim"
+ date="2013-03-13T02:18:20Z"
+ content="""
+Could you describe your vim configuration for haskell development, what plugins do you use, what is in you .vimrc file and maybe even record a screencast of typical \"session\" of work?
+"""]]
diff --git a/doc/design/assistant/blog/day_212__accidental_all_nighter/comment_2_07c83d75bb105bb77ada07359ed0ea7a._comment b/doc/design/assistant/blog/day_212__accidental_all_nighter/comment_2_07c83d75bb105bb77ada07359ed0ea7a._comment
new file mode 100644
index 0000000000..06aae6e1b7
--- /dev/null
+++ b/doc/design/assistant/blog/day_212__accidental_all_nighter/comment_2_07c83d75bb105bb77ada07359ed0ea7a._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="https://www.google.com/accounts/o8/id?id=AItOawmU_2tE75oyG0h2ZPN4lcroIKEMC8G-otE"
+ nickname="Michael"
+ subject="Global preferences?"
+ date="2013-03-13T08:00:55Z"
+ content="""
+I just did a \"cabal install git-annex --bindir=$HOME/bin\" on Ubuntu 12.10 and I don't see the global preferences (where you configure the autostart etc.) from the screencast. How can I enable this?
+"""]]
diff --git a/doc/design/assistant/blog/day_212__accidental_all_nighter/comment_3_2c904d33f4f14807fbe718a01e98800a._comment b/doc/design/assistant/blog/day_212__accidental_all_nighter/comment_3_2c904d33f4f14807fbe718a01e98800a._comment
new file mode 100644
index 0000000000..943bd52778
--- /dev/null
+++ b/doc/design/assistant/blog/day_212__accidental_all_nighter/comment_3_2c904d33f4f14807fbe718a01e98800a._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="http://joeyh.name/"
+ nickname="joey"
+ subject="comment 3"
+ date="2013-03-13T16:08:35Z"
+ content="""
+The global preferences page is a new feature that will be in the next release. You can use a daily build or build from source to try it now.
+"""]]
diff --git a/doc/design/assistant/blog/day_212__accidental_all_nighter/comment_4_59ec5c1cab75df87293800a7a03fe9c6._comment b/doc/design/assistant/blog/day_212__accidental_all_nighter/comment_4_59ec5c1cab75df87293800a7a03fe9c6._comment
new file mode 100644
index 0000000000..c3e8f83515
--- /dev/null
+++ b/doc/design/assistant/blog/day_212__accidental_all_nighter/comment_4_59ec5c1cab75df87293800a7a03fe9c6._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="https://www.google.com/accounts/o8/id?id=AItOawmU_2tE75oyG0h2ZPN4lcroIKEMC8G-otE"
+ nickname="Michael"
+ subject="comment 4"
+ date="2013-03-13T16:34:46Z"
+ content="""
+Build from source is too hard for a non-haskell developer, I run into various errors I can not resolve. The daily build gives me \"git-annex: error while loading shared libraries: libyaml-0.so.2: cannot open shared object file: No such file or directory\". For Ubuntu a PPA with daily build would be handy.
+"""]]
diff --git a/doc/design/assistant/blog/day_212__accidental_all_nighter/comment_5_13893f106e835dcc52e03c7c6740c35b._comment b/doc/design/assistant/blog/day_212__accidental_all_nighter/comment_5_13893f106e835dcc52e03c7c6740c35b._comment
new file mode 100644
index 0000000000..8c6419b1c3
--- /dev/null
+++ b/doc/design/assistant/blog/day_212__accidental_all_nighter/comment_5_13893f106e835dcc52e03c7c6740c35b._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="http://joeyh.name/"
+ nickname="joey"
+ subject="comment 5"
+ date="2013-03-13T16:58:04Z"
+ content="""
+In the standalone linux daily build, you need to use the \"runshell\" script to start a shell environment in which you can run git-annex.
+"""]]
diff --git a/doc/design/assistant/blog/day_213__costs.mdwn b/doc/design/assistant/blog/day_213__costs.mdwn
new file mode 100644
index 0000000000..41517a7ab1
--- /dev/null
+++ b/doc/design/assistant/blog/day_213__costs.mdwn
@@ -0,0 +1,34 @@
+Got the assistant to check again, just before starting a transfer, if
+the remote still wants the object. This should be all that's needed to
+handle the case where there is a transfer remote on the internet somewhere,
+and a locally paired client on the LAN. As long as the paired repository
+has a lower cost value, it will be sent any new file first, and if that
+is the only client, the file will not be sent to the transfer remote at
+all.
+
+But.. locally paired repos did not have a lower cost set, at all.
+So I made their cost be set to 175 when they're created. Anyone
+who already did local pairing should make sure the Repositories
+list shows locally paired repositories above transfer remotes.
+
+Which brought me to needing an easy way to reorder that list of remotes,
+which I plan to do by letting the user drag and drop remotes around,
+which will change their cost accordingly. Implementing that has two
+pain points:
+
+1. Often a lot of remotes will have the same default cost value.
+   So how to insert a remote in between two that have cost 100?
+   This would be easy if git-annex didn't have these cost numbers,
+   and instead just had an ordered list of remotes.. but it doesn't.
+   Instead, dragging remotes in the list will sometimes need to change
+   the costs of others, to make room to insert them in. It's BASIC
+   renumbering all over again. So I wrote some code to do this with as
+   little bother as possible.
+
+2. Drag and drop means javascript. I got the basics going quickly with
+   jquery-ui, only to get stuck for over an hour on some CSS issue
+   that made lines from the list display all weird while being dragged.
+   It is always something like this with javascript..
+
+So I've got these 2 peices working, and even have the AJAX call
+firing, but it's not quite wired up just yet. Tomorrow.
diff --git a/doc/design/assistant/blog/day_214__release_day.mdwn b/doc/design/assistant/blog/day_214__release_day.mdwn
new file mode 100644
index 0000000000..2ada213086
--- /dev/null
+++ b/doc/design/assistant/blog/day_214__release_day.mdwn
@@ -0,0 +1,5 @@
+Fighting with javascript all day and racing to get a release out. Unstuck
+the OSX and Android autobuilders. Got drag and drop repository list
+reordering working great. Tons of changes in this release!
+
+Also put up a new podcast.
diff --git a/doc/design/assistant/blog/day_215__dashboard_UI_refresh.mdwn b/doc/design/assistant/blog/day_215__dashboard_UI_refresh.mdwn
new file mode 100644
index 0000000000..9381ebee20
--- /dev/null
+++ b/doc/design/assistant/blog/day_215__dashboard_UI_refresh.mdwn
@@ -0,0 +1,25 @@
+I've reworked the UI of the webapp's dashboard. Now the repository list is
+included, above the transfers. I found I was spending a lot of time
+switching between the dashboard and repository list, so might as well
+combine them into a single screen. Yesod's type safe urls and widgets
+made this quite easy to do, despite it being a thousand line commit.
+Liking the result ... Even though it does make all my screencasts dated.
+
+[[!img /assistant/dashboard.png]]
+
+----
+
+Rest of my time was spent on XMPP pairing UI. Using the same pages for both
+pairing with a friend and for self-pairing was confusing, so now the two
+options are split.
+
+Now every time an XMPP git push is received or sent, it checks if there's
+a cloud repository configured, which is needed to send the contents of
+files. If not, it'll display this alert. Hopefully this
+will be enough to get users fully set up.
+
+[[!img /assistant/cloudnudge.png]]
+
+At this point I'm finally happy enough with the XMPP pairing + cloud
+repository setup process to film a screencast of it. As soon as I have
+some time & bandwidth someplace quiet. Expect one by the end of the month.
diff --git a/doc/design/assistant/blog/day_216__more_bugfixes.mdwn b/doc/design/assistant/blog/day_216__more_bugfixes.mdwn
new file mode 100644
index 0000000000..a988643ff2
--- /dev/null
+++ b/doc/design/assistant/blog/day_216__more_bugfixes.mdwn
@@ -0,0 +1,42 @@
+A long time ago I made Remote be an instance of the Ord typeclass, with an
+implementation that compared the costs of Remotes. That seemed like a good
+idea at the time, as it saved typing.. But at the time I was still making
+custom Read and Show instances too. I've since learned that this is *not* a
+good idea, and neither is making custom Ord instances, without deep thought
+about the possible sets of values in a type.
+
+This Ord instance came around and bit me when I put Remotes into a Set,
+because now remotes with the same cost appeared to be in the Set even if 
+they were not. Also affected putting Remotes into a Map. I noticed this
+when the webapp got confused about which Remotes were paused.
+
+Rarely does a bug go this deep. I've fixed it comprehensively, first
+removing the Ord instance entirely, and fixing the places that wanted to
+order remotes by cost to do it explicitly. Then adding back an Ord instance
+that is much more sane. Also by checking the rest of the Ord (and Eq) 
+instances in the code base (which were all ok).
+
+While doing that, I found lots of places that kept remotes in Maps and
+Sets. All of it was probably subtly broken in one way or another before
+this fix, but it would be hard to say exactly how the bugs would
+manifest.
+
+-----
+
+Also fought some with Google Talk today. Seem to be missing presence
+messages sometimes. Ugh. May have fixed it, but I've thought that before..
+
+Made --debug include a sanitized dump of the XMPP protocol.
+
+Made UI changes to encourage user to install git-annex on the server when
+adding a ssh server, rather than just funneling them through to rsync.
+
+Fixed UI glitches in XMPP username/password prompt.
+
+Switched all forms in the webapp to use POST, to avoid sensitive
+information leaking on the url bar.
+
+----
+
+Added an incremental backup group. Repositories in this group only want
+files that have not been backed up somewhere else yet.
diff --git a/doc/design/assistant/blog/day_216__more_bugfixes/comment_1_299462bcdd0e4f6cd7895b5f40ca00ad._comment b/doc/design/assistant/blog/day_216__more_bugfixes/comment_1_299462bcdd0e4f6cd7895b5f40ca00ad._comment
new file mode 100644
index 0000000000..7d28cfdbf8
--- /dev/null
+++ b/doc/design/assistant/blog/day_216__more_bugfixes/comment_1_299462bcdd0e4f6cd7895b5f40ca00ad._comment
@@ -0,0 +1,10 @@
+[[!comment format=mdwn
+ username="https://www.google.com/accounts/o8/id?id=AItOawnR7hb8IaKB3IKZptRukje0yahmhfLOO98"
+ nickname="Adam"
+ subject="comment 1"
+ date="2013-03-17T00:30:26Z"
+ content="""
+> Made UI changes to encourage user to install git-annex on the server when adding a ssh server, rather than just funneling them through to rsync.
+
+What's the difference between having git-annex available versus not?
+"""]]
diff --git a/doc/design/assistant/blog/day_216__more_bugfixes/comment_2_1913d65dfe4ba08379d82a4a2ca91c40._comment b/doc/design/assistant/blog/day_216__more_bugfixes/comment_2_1913d65dfe4ba08379d82a4a2ca91c40._comment
new file mode 100644
index 0000000000..200846545c
--- /dev/null
+++ b/doc/design/assistant/blog/day_216__more_bugfixes/comment_2_1913d65dfe4ba08379d82a4a2ca91c40._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="https://www.google.com/accounts/o8/id?id=AItOawnxp2XU8gIribhhGhGuYtU6eMMwHv5gUGI"
+ nickname="Amitai"
+ subject="Google Talk silently drops non-Google invites (at least)"
+ date="2013-03-17T13:44:18Z"
+ content="""
+Have you seen ? Maybe they're also doing other funny stuff that gets in your way.
+"""]]
diff --git a/doc/design/assistant/blog/day_216__more_bugfixes/comment_3_92c774599a78540ad398afcd1d05f7ce._comment b/doc/design/assistant/blog/day_216__more_bugfixes/comment_3_92c774599a78540ad398afcd1d05f7ce._comment
new file mode 100644
index 0000000000..05847a73b7
--- /dev/null
+++ b/doc/design/assistant/blog/day_216__more_bugfixes/comment_3_92c774599a78540ad398afcd1d05f7ce._comment
@@ -0,0 +1,20 @@
+[[!comment format=mdwn
+ username="https://www.google.com/accounts/o8/id?id=AItOawnR7hb8IaKB3IKZptRukje0yahmhfLOO98"
+ nickname="Adam"
+ subject="comment 3"
+ date="2013-03-25T01:58:23Z"
+ content="""
+About the UI prompt for the webapp:
+
+> What's the difference between having git-annex available versus not?
+
+I can see from the UI that if you make it be a git repository, you get this:
+
+> All your data will be uploaded to the server, including the full git repository. This is a great choice if you want to set up other devices to use the same server, or share the repository with others.
+
+Versus an rsync repo:
+
+> The contents of your files will be stored, fully encrypted, on the server. The server will not store other information about your git repository. This is the best choice if you don't run the server yourself, or have sensitive data.
+
+Unfortunately I'm looking for a shared server (as per the git repo description), but **also fully encrypted** because I'll be storing sensitive data.
+"""]]
diff --git a/doc/design/assistant/blog/day_217__nothing.mdwn b/doc/design/assistant/blog/day_217__nothing.mdwn
new file mode 100644
index 0000000000..95deabd238
--- /dev/null
+++ b/doc/design/assistant/blog/day_217__nothing.mdwn
@@ -0,0 +1,2 @@
+Is what I planned to do on git-annex today. Instead I fixed several bugs,
+but I'm drawing the line at blogging. Oops.
diff --git a/doc/design/assistant/blog/day_219__bug_triage.mdwn b/doc/design/assistant/blog/day_219__bug_triage.mdwn
new file mode 100644
index 0000000000..82494ceca8
--- /dev/null
+++ b/doc/design/assistant/blog/day_219__bug_triage.mdwn
@@ -0,0 +1,14 @@
+Triaged some of the older bugs and was able to close a lot of them.
+
+-----
+
+Should mention that I will be in Boston this weekend, attending
+[LibrePlanet 2013](http://libreplanet.org/wiki/LibrePlanet:Conference/2013).
+Drop by and find me, I'll have git-annex stickers! ;)
+
+-----
+
+Did some UI work on the webapp. Minor stuff, but stuff that needed to be
+fixed up. Like inserting zero-width spaces into filenames displayed in it
+so very long filenames always get reasonably wrapped by the browser.
+(Perhaps there's a better way to do that with CSS?)
diff --git a/doc/design/assistant/blog/day_219__bug_triage/comment_1_c6b977a969cacdce62987a439b7686f5._comment b/doc/design/assistant/blog/day_219__bug_triage/comment_1_c6b977a969cacdce62987a439b7686f5._comment
new file mode 100644
index 0000000000..5722f7e19d
--- /dev/null
+++ b/doc/design/assistant/blog/day_219__bug_triage/comment_1_c6b977a969cacdce62987a439b7686f5._comment
@@ -0,0 +1,16 @@
+[[!comment format=mdwn
+ username="http://jasonwoof.com/"
+ nickname="JasonWoof"
+ subject="wrapping css"
+ date="2013-03-20T20:28:04Z"
+ content="""
+You can do this in css:
+
+    word-wrap: break-word;
+
+That should make the text not leave the box. Not sure if this makes it break \"reasonably\" though, eg a two letter word followed by an 200 letter \"word\" would probably leave the two letter word on a line by itself.
+
+Worth a try though.
+
+Seems to be well supported on browsers: [http://caniuse.com/#search=wrap](http://caniuse.com/#search=wrap)
+"""]]
diff --git a/doc/design/assistant/blog/day_21__transfer_tracking.mdwn b/doc/design/assistant/blog/day_21__transfer_tracking.mdwn
new file mode 100644
index 0000000000..79c0b64387
--- /dev/null
+++ b/doc/design/assistant/blog/day_21__transfer_tracking.mdwn
@@ -0,0 +1,28 @@
+Worked today on two action items from my last blog post:
+
+* on-disk transfers in progress information files (read/write/enumerate)
+* locking for the files, so redundant transfer races can be detected,
+  and failed transfers noticed
+
+That's all done, and used by the `get`, `copy`, and `move` subcommands.
+
+Also, I made `git-annex status` use that information to display any
+file transfers that are currently in progress:
+
+	joey@gnu:~/lib/sound/misc>git annex status
+	[...]
+	transfers in progress: 
+		downloading Vic-303.mp3 from leech
+
+(Webapp, here we come!)
+
+However... Files being sent or received by `git-annex-shell` don't yet
+have this transfer info recorded. The problem is that to do so,
+`git-annex-shell` will need to be run with a `--remote=` parameter. But
+old versions will of course fail when run with such an unknown parameter.
+
+This is a problem I last faced in December 2011 when adding the `--uuid=`
+parameter. That time I punted and required the remote `git-annex-shell` be
+updated to a new enough version to accept it. But as git-annex gets more widely
+used and packaged, that's becoming less an option. I need to find a real
+solution to this problem.
diff --git a/doc/design/assistant/blog/day_220__performance.mdwn b/doc/design/assistant/blog/day_220__performance.mdwn
new file mode 100644
index 0000000000..a1b7b329ac
--- /dev/null
+++ b/doc/design/assistant/blog/day_220__performance.mdwn
@@ -0,0 +1,40 @@
+I've been running some large transfers with the assistant, and looking at
+ways to improve performance. (I also found and fixed a zombie process
+leak.)
+
+----
+
+One thing I noticed is that the assistant pushes changes to the git-annex
+location log quite frequently during a batch transfer. If the files being
+transferred are reasonably sized, it'll be pushing once per file transfer. 
+It would be good to reduce the number of pushes, but the pushes are
+important in some network topologies to inform other nodes
+when a file gets near to them, so they can get the file too.
+
+Need to see if I can find a smart way to avoid some of the pushes.
+For example, if we've just downloaded a file, and are queuing uploads
+of the file to a remote, we probably don't need to push the git-annex
+branch to the remote.
+
+----
+
+Another performance problem is that having the webapp open while transfers
+are running uses significant CPU just for the browser to update the progress
+bar. Unsurprising, since the webapp is sending the browser a new `
    `
+each time. Updating the DOM instead from javascript would avoid that;
+the webapp just needs to send the javascript either a full `
    ` or a
+changed percentage and quantity complete to update a single progress bar.
+
+I'd prefer to wait on doing that until I'm able to use Fay to generate
+Javascript from Haskell, because it would be much more pleasant.. will see.
+
+----
+
+Also a performance problem when performing lots of transfers, particularly
+of small files, is that the assistant forks off a `git annex transferkey`
+for each transfer, and that has to in turn start up several git commands.
+
+Today I have been working to change that, so the assistant maintains a
+pool of transfer processes, and dispatches each transfer it wants to make
+to a process from the pool. I just got all that to build, although untested
+so far, in the `transferpools` branch.
diff --git a/doc/design/assistant/blog/day_221__this_and_that.mdwn b/doc/design/assistant/blog/day_221__this_and_that.mdwn
new file mode 100644
index 0000000000..87406547e5
--- /dev/null
+++ b/doc/design/assistant/blog/day_221__this_and_that.mdwn
@@ -0,0 +1,28 @@
+Was unsure yesterday if my transferrer pools code would just work, or
+would be horribly broken and need a lot of work to get going. It was a
+complex change involving both high-level STM code and low-level pipes and
+fds. Well, it almost worked 100% first time, I just had a minor issue in
+my fd setup to fix. Everything else seems to work perfectly.
+Very happy how that went!
+
+----
+
+Improved support and documentation for using the OSX app and Linux
+standalone tarball at the command line. Now it's sufficient to just put
+their directory into `PATH`, rather than using `runshell`.
+
+----
+
+The webapp's form for adding a removable drive now allows specifying the
+directory to use within the drive (default "annex").
+
+When the drive's repository already exists, and it's not a repository
+that git-annex knows about, it confirms that the user wants
+to combine its contents into their repository.
+
+
+(Should probably implement this same check when adding a ssh remote.)
+
+----
+
+Off to Boston!
diff --git a/doc/design/assistant/blog/day_222__back.mdwn b/doc/design/assistant/blog/day_222__back.mdwn
new file mode 100644
index 0000000000..9c3c17094e
--- /dev/null
+++ b/doc/design/assistant/blog/day_222__back.mdwn
@@ -0,0 +1,16 @@
+Back from my trip. Spent today getting caught up.
+
+Didn't do much while I was away. Pushed out a new release on Saturday.
+Made `git annex` usage display nicer.
+
+Fixed some minor webapp bugs today. The interesting bug was a
+race that sometimes caused alerts or other notifications to be
+missed and not be immediately displayed if they occurred while
+a page was loading. You sometimes had to hit reload to see them,
+but not anymore!
+
+Checked if the `push.default=simple` change in upcoming git release will
+affect git-annex. It shouldn't affect the assistant, or `git annex sync`,
+since they always list all branches to push explicitly. But if you `git push`
+manually, when the default changes that won't include the git-annex branch
+in the push any longer.
diff --git a/doc/design/assistant/blog/day_222__back/comment_1_f05b48231a1ee0cffba7d66e112e5551._comment b/doc/design/assistant/blog/day_222__back/comment_1_f05b48231a1ee0cffba7d66e112e5551._comment
new file mode 100644
index 0000000000..f788cc283f
--- /dev/null
+++ b/doc/design/assistant/blog/day_222__back/comment_1_f05b48231a1ee0cffba7d66e112e5551._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="https://www.google.com/accounts/o8/id?id=AItOawm7AuSfii_tCkLyspL6Mr0ATlO6OxLNYOo"
+ nickname="Georg"
+ subject="something for the wishlist maybe?"
+ date="2013-03-28T08:30:00Z"
+ content="""
+Something I was wondering with the assistant now getting really smart: would it be possible to have a cli-mode for the assistant that runs in the forground and only until there is nothing to do anymore? Something like a single-run-assistant`? Reason being that in my usecases most often I know exactly when the right moment for a run is - for example after processing a bunch of photographs - and I most definitely don't want a background process to work on those photographs while I do work on them. Or with archival media, I know when I will hook up the archive drives. In allmost all of my scenarios it would be much easier to not run the assistant in the background, but to fire it from the cli at the right moments, letting it work over night and have it shut down automatically when all is synced.
+"""]]
diff --git a/doc/design/assistant/blog/day_222__back/comment_2_4d5f003ccd81580017ebf0dc31bc9cda._comment b/doc/design/assistant/blog/day_222__back/comment_2_4d5f003ccd81580017ebf0dc31bc9cda._comment
new file mode 100644
index 0000000000..cc28d56086
--- /dev/null
+++ b/doc/design/assistant/blog/day_222__back/comment_2_4d5f003ccd81580017ebf0dc31bc9cda._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="http://joeyh.name/"
+ nickname="joey"
+ subject="comment 2"
+ date="2013-03-28T15:50:47Z"
+ content="""
+That's an interesting thought. I think it would be hard to find a clear condition when it was done and could shut down. While there is easy visibility into the transfer queue to tell when file transfers are done, it's harder to tell when it's actively syncing git repositories or scanning for transfers to do, or finding new files to commit. Each thread knows what it's doing, but none of the other threads do. Maybe active threads could hold a lock to indicate they're busy, and this combined with visibility into the various work queues could do the job.
+"""]]
diff --git a/doc/design/assistant/blog/day_223__progress_revisited.mdwn b/doc/design/assistant/blog/day_223__progress_revisited.mdwn
new file mode 100644
index 0000000000..9f899899db
--- /dev/null
+++ b/doc/design/assistant/blog/day_223__progress_revisited.mdwn
@@ -0,0 +1,24 @@
+Went out and tried for the second time to record a screencast demoing
+setting up syncing between two computers using just Jabber and a cloud
+remote. I can't record this one at home, or viewers would think git-annex
+was crazy slow, when it's just my dialup. ;) But once again I encountered
+bugs, and so I found myself working on progress bars today, unexpectedly.
+
+Seems there was confusion in different parts of the progress bar code
+about whether an update contained the total number of bytes transferred, or
+the delta of bytes transferred since the last update. One way this bug
+showed up was progress bars that seemed to stick at 0% for a long time.
+Happened for most special remotes, although not for rsync or git remotes.
+In order to fix it comprehensively, I added a new BytesProcessed data type,
+that is explicitly a total quantity of bytes, not a delta. And checked and
+fixed all the places that used a delta as that type was knitted into
+the code.
+
+(Note that this doesn't necessarily fix every problem with progress bars.
+Particularly, buffering can now cause progress bars to seem to run ahead
+of transfers, reaching 100% when data is still being uploaded. Still,
+they should be a lot better than before.)
+
+I've just successfully run through the Jabber + Cloud remote setup process
+again, and it seems to be working great now. Maybe I'll still get the
+screencast recorded by the end of March.
diff --git a/doc/design/assistant/blog/day_224__annex.largefiles.mdwn b/doc/design/assistant/blog/day_224__annex.largefiles.mdwn
new file mode 100644
index 0000000000..4fab93fb65
--- /dev/null
+++ b/doc/design/assistant/blog/day_224__annex.largefiles.mdwn
@@ -0,0 +1,23 @@
+Built a feature for power users today. `annex.largefiles` can be
+configured to specify what files `git annex add` and the assistant should
+put into the annex. It uses the same syntax as [[/preferred_content]],
+so arbitrarily complex expressions can be built.
+
+For example, a game written in C with some large data files could
+include only 100kb or larger files, that are not C code:
+
+	annex.largefiles = largerthan=100kb and not (include=*.c or include=*.h) 
+
+The assistant will commit small files to git directly!
+`git annex add`, being a lower level tool, skips small files
+and leaves it up to you to `git add` them as desired.
+
+It's even possible to tell the assistant that no file is too large to be
+committed directly to git. `git config annex.largefiles 'exclude=*'`
+The result should be much like using SparkleShare or dvcs-autosync.
+
+-----
+
+Also today, made the remote ssh server checking code in the webapp
+deal with servers where the default shell is csh or some other non-POSIX
+shell.
diff --git a/doc/design/assistant/blog/day_224__annex.largefiles/comment_1_408e4021b18f7ff5548d2d19ab558922._comment b/doc/design/assistant/blog/day_224__annex.largefiles/comment_1_408e4021b18f7ff5548d2d19ab558922._comment
new file mode 100644
index 0000000000..370f599c27
--- /dev/null
+++ b/doc/design/assistant/blog/day_224__annex.largefiles/comment_1_408e4021b18f7ff5548d2d19ab558922._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="https://www.google.com/accounts/o8/id?id=AItOawm7AuSfii_tCkLyspL6Mr0ATlO6OxLNYOo"
+ nickname="Georg"
+ subject="Great idea! "
+ date="2013-03-30T08:00:16Z"
+ content="""
+This will be perfect for my raw image pool! I use Lightroom, so my photos are only imported once and only in case of DNG are they actually changed, in all other formats LR creates little sidecar XML files with processing instructions. So with this change the side car files can actually be fully Versionen and lage files are kept out of the repo and handled by archiving with annex. Cool! 
+"""]]
diff --git a/doc/design/assistant/blog/day_224__annex.largefiles/comment_2_b24d1da2bc4307ded0216daffb8f9336._comment b/doc/design/assistant/blog/day_224__annex.largefiles/comment_2_b24d1da2bc4307ded0216daffb8f9336._comment
new file mode 100644
index 0000000000..14fdfed5ce
--- /dev/null
+++ b/doc/design/assistant/blog/day_224__annex.largefiles/comment_2_b24d1da2bc4307ded0216daffb8f9336._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="https://www.google.com/accounts/o8/id?id=AItOawmWBvsZvSsAL8P2ye3F0OBStjFCVnOImzM"
+ nickname="Jarno"
+ subject="Making "git-annex add" commit largefiles, too?"
+ date="2013-10-31T00:26:21Z"
+ content="""
+Is it possible to make `git annex add` also *not* skip largefiles but do an automatic `git add` instead? I'd like to keep .txt, .sh and the like in git directly (for easier diffing) but having to add them separately sounds like trouble.
+"""]]
diff --git a/doc/design/assistant/blog/day_225__back_from_the_dead.mdwn b/doc/design/assistant/blog/day_225__back_from_the_dead.mdwn
new file mode 100644
index 0000000000..e550c6b8b8
--- /dev/null
+++ b/doc/design/assistant/blog/day_225__back_from_the_dead.mdwn
@@ -0,0 +1,47 @@
+I've posted a poll: [[polls/goals_for_April]]
+
+----
+
+Today added UI to the webapp to delete repositories, which many
+users have requested. It can delete the local repository,
+with appropriate cautions and sanity checks:
+
+[[!img /assistant/deleterepository.png]]
+
+More likely, you'll use it to remove a remote, which is done with no muss
+and no fuss, since that doesn't delete any data and the remote can always
+be added back if you change your mind.
+
+It also has an option to fully delete the data on a remote. This doesn't
+actually delete the remote right away. All it does is marks the remote
+as untrusted[1], and configures it to not want any content.
+This causes all the content on it to be sucked off to whatever
+other repositories can hold it.
+
+I had to adjust the preferred content expressions to make that work. For
+example, when deleting an archive drive, your local (client) repository
+does not normally want to hold all the data it has in "archive"
+directories. With the adjusted preferred content expressions, any data on
+an untrusted or dead repository is wanted. An interesting result is that
+once a client repository has moved content from an untrusted remote, it
+will decide it doesn't want it anymore, and shove it out to any other
+remote that will accept it. Which is just the behavior we want. All it took
+to get all this behavior is adding "or (not copies=semitrusted:1)" to the
+preferred content expressions!
+
+For most special remotes, just sucking the data from them is sufficient to
+pretty well delete them. You'd want to delete an Amazon bucket or glacier
+once it's empty, and git repositories need to be fully deleted. Since this
+would need unique code for each type of special remote, and it would be
+code that a) deletes possibly large quantities of data with no real way to
+sanity check it and b) doesn't get run and tested very often; it's not
+something I'm thrilled about fully automating. However, I would like to
+make the assistant detect when all the content has been sucked out of a
+remote, and pop up at least a message prompting to finish the deletion.
+Future work.
+
+-----
+
+[1] I really, really wanted to mark it dead, but letting puns drive code
+is probably a bad idea. I had no idea I'd get here when I started
+developing this feature this morning.. Honest!
diff --git a/doc/design/assistant/blog/day_225__back_from_the_dead/comment_1_9ac37c3b5c4c72ec8a39dce00bcbe420._comment b/doc/design/assistant/blog/day_225__back_from_the_dead/comment_1_9ac37c3b5c4c72ec8a39dce00bcbe420._comment
new file mode 100644
index 0000000000..325eff30ce
--- /dev/null
+++ b/doc/design/assistant/blog/day_225__back_from_the_dead/comment_1_9ac37c3b5c4c72ec8a39dce00bcbe420._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="andy"
+ ip="99.65.214.16"
+ subject="Just wondering what I'm missing..."
+ date="2013-04-02T21:43:33Z"
+ content="""
+I haven't played around with the preferred content expressions, but what is is that keeps \"or (not copies=semitrusted:1)\" from grabbing all the content from trusted repositories, as well as untrusted or dead ones?
+"""]]
diff --git a/doc/design/assistant/blog/day_225__back_from_the_dead/comment_2_26125dd9ef2bd10b597d14b2c6180952._comment b/doc/design/assistant/blog/day_225__back_from_the_dead/comment_2_26125dd9ef2bd10b597d14b2c6180952._comment
new file mode 100644
index 0000000000..384d20f315
--- /dev/null
+++ b/doc/design/assistant/blog/day_225__back_from_the_dead/comment_2_26125dd9ef2bd10b597d14b2c6180952._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="http://joeyh.name/"
+ nickname="joey"
+ subject="comment 2"
+ date="2013-04-03T03:42:40Z"
+ content="""
+That's some seriously sharp eyes, Andy. Nothing. I forgot it didn't work that way. Now it uses semitrusted+:1 which works the right way.
+"""]]
diff --git a/doc/design/assistant/blog/day_226__poll_results.mdwn b/doc/design/assistant/blog/day_226__poll_results.mdwn
new file mode 100644
index 0000000000..e79ff25859
--- /dev/null
+++ b/doc/design/assistant/blog/day_226__poll_results.mdwn
@@ -0,0 +1,28 @@
+Both the assistant and `git annex drop --auto` refused to drop files from
+untrusted repositories. Got that fixed.
+
+Finally recorded the xmpp pairing screencast. In one perfect take, which
+somehow `recordmydesktop` lost the last 3 minutes of.
+Argh! Anyway I'm editing it now, so, look for that screencast soon.
+
+The [[polls/goals_for_April]] poll results are in. 
+
+* There have been no votes at all for working on
+  cloud remotes. Seems that git-annex supports enough cloud remotes already.
+* A lot of people want the Android webapp port to be done, so I will
+  probably spend some time on that this month.
+* Interest in other various features is split. I am surprised how many 
+  want git-remote-gcrypt, compared to the features that would make
+  syncing use less bandwidth. Doesn't git push over xmpp cover most
+  of the use cases where git-remote-gcrypt would need to be used with the
+  assistant?
+* Nearly as many people as want features, want me to work on bug
+  fixing and polishing what's already there.
+  So I should probably continue to make screencasts, since they often force
+  me to look at things with fresh eyes and see and fix problems. And of course,
+  continue working on bugs as they're reported.
+* I'm not sure what to make of the 10% who want me to add direct mode support.
+  Since direct mode is already used by default, perhaps they want
+  me to take time off? :) (I certainly need to fix the
+  [[bugs/Direct_mode_keeps_re-checksuming_duplicated_files]] bug, and one other
+  direct mode bug I discovered yesterday.)
diff --git a/doc/design/assistant/blog/day_226__poll_results/comment_1_1ed980472214be6d0a8cf55f37797fda._comment b/doc/design/assistant/blog/day_226__poll_results/comment_1_1ed980472214be6d0a8cf55f37797fda._comment
new file mode 100644
index 0000000000..f936e68ed0
--- /dev/null
+++ b/doc/design/assistant/blog/day_226__poll_results/comment_1_1ed980472214be6d0a8cf55f37797fda._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="https://dzsino.myopenid.com/"
+ nickname="dzsino"
+ subject="I've voted for git-remote-gcrypt because.."
+ date="2013-04-02T09:07:23Z"
+ content="""
+... as important a Windows port seems for git-annex to really take off (IMO), I would welcome encrypted repos for storing stuff on external drives: directory special remotes feel kinda cumbersome for me, I would rather use full blown repos. 
+"""]]
diff --git a/doc/design/assistant/blog/day_226__poll_results/comment_2_6823b0a9a8037f1a5214db4db98fb16e._comment b/doc/design/assistant/blog/day_226__poll_results/comment_2_6823b0a9a8037f1a5214db4db98fb16e._comment
new file mode 100644
index 0000000000..c6e65adf2a
--- /dev/null
+++ b/doc/design/assistant/blog/day_226__poll_results/comment_2_6823b0a9a8037f1a5214db4db98fb16e._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="http://joeyh.name/"
+ nickname="joey"
+ subject="comment 2"
+ date="2013-04-02T21:36:50Z"
+ content="""
+Thanks for reminding me about that use case. I have created a page [[encrypted_git_remotes]] for this.
+"""]]
diff --git a/doc/design/assistant/blog/day_227__bigfixing_all_day_today.mdwn b/doc/design/assistant/blog/day_227__bigfixing_all_day_today.mdwn
new file mode 100644
index 0000000000..74e9fd82f7
--- /dev/null
+++ b/doc/design/assistant/blog/day_227__bigfixing_all_day_today.mdwn
@@ -0,0 +1,21 @@
+The [[xmpp screencast|videos/git-annex_assistant_remote_sharing]]
+is at long last done!
+
+----
+
+Fixed a bug that could cause the assistant to unstage files
+from git sometimes. This happened because of a bad optimisation; adding a
+file when it's already present and unchanged was optimised to do nothing.
+But if the file had just been removed, and was put back, this resulted
+in the removal being staged, and the add not being staged. Ugly bug,
+although the assistant's daily sanity check automatically restaged the
+files.
+
+Underlying that bug was a more important problem: git-annex does not always
+update working tree files atomically. So a crash at just the wrong instant
+could cause a file to be deleted from the working tree. I fixed that too;
+all changes to files in the working tree should now be staged in a temp
+file, which is renamed into place atomically.
+
+Also made a bunch of improvements to the dashboard's transfer display, and
+to the handling of the underlying transfer queue.
diff --git a/doc/design/assistant/blog/day_228__more_work_on_repository_removals.mdwn b/doc/design/assistant/blog/day_228__more_work_on_repository_removals.mdwn
new file mode 100644
index 0000000000..f8b4502169
--- /dev/null
+++ b/doc/design/assistant/blog/day_228__more_work_on_repository_removals.mdwn
@@ -0,0 +1,27 @@
+Getting back to the repository removal handling from Sunday, I made the
+assistant detect when a repository that has been marked as unwanted becomes
+empty, and finish the removal process.
+
+I was able to add this to the expensive transfer scan without making it any
+more expensive than it already was, since that scan already looks at the
+location of all keys. Although when a remote is detected as empty, it then
+does one more check, equivilant to `git annex unused`, to find any
+remaining objects on the remote, and force them off.
+
+I think this should work pretty well, but it needs some testing and
+probably some UI work.
+
+----
+
+Andy spotted a bug in the preferred content expressions I was using to
+handle untrusted remotes. So he saved me several hours dealing with an ugly
+bug at some point down the line. I had misread my own preferred content
+expression documentation, and `copies=semitrusted:1` was not doing what I
+thought it was. Added a new syntax that does what I need,
+`copies=semitrusted+:1`
+
+----
+
+The 64 bit linux standalone builds are back. Apparently the 32 bit builds
+have stopped working on recent Fedora, for reasons that are unclear. I set
+up an autobuilder to produce the 64 bit builds.
diff --git a/doc/design/assistant/blog/day_229__rainy_day_bugfixes.mdwn b/doc/design/assistant/blog/day_229__rainy_day_bugfixes.mdwn
new file mode 100644
index 0000000000..87611a97d1
--- /dev/null
+++ b/doc/design/assistant/blog/day_229__rainy_day_bugfixes.mdwn
@@ -0,0 +1,17 @@
+Got caught up on bug reports and made some bug fixes.
+
+The one bug I was really worried about, a strange file corruption problem
+on Android, turned out not to be a bug in git-annex. (Nor is it a bug that
+will affect regular users.)
+
+The only interesting bug fixed was a mixed case hash directory name
+collision when a repository is put on a VFAT filesystem (or other
+filesystem with similar semantics). I was able to fix that nicely; since
+such a repository will be in crippled filesystem mode due to other
+limitations of the filesystem, and so won't be using symlinks,
+it doesn't need to use the mixed case hash directory names.
+
+Last night, finished up the repository removal code, and associated UI
+tweaks. It works very well.
+
+Will probably make a release tomorrow.
diff --git a/doc/design/assistant/blog/day_22__horrible_option_parsing_hack.mdwn b/doc/design/assistant/blog/day_22__horrible_option_parsing_hack.mdwn
new file mode 100644
index 0000000000..8f6708e597
--- /dev/null
+++ b/doc/design/assistant/blog/day_22__horrible_option_parsing_hack.mdwn
@@ -0,0 +1,34 @@
+Well, sometimes you just have to go for the hack. Trying to find a way
+to add additional options to git-annex-shell without breaking backwards
+compatibility, I noticed that it ignores all options after `--`, because
+those tend to be random rsync options due to the way rsync runs it.
+
+So, I've added a new class of options, that come in between, like
+`-- opt=val opt=val ... --`
+
+The parser for these will not choke on unknown options, unlike normal
+getopt. So this let me add the additional info I needed to
+pass to git-annex-shell to make it record transfer information. And
+if I need to pass more info in the future, that's covered too.
+
+It's ugly, but since only git-annex runs git-annex-shell, this is an
+ugliness only I (and now you, dear reader) have to put up with.
+
+Note to self: Command-line programs are sometimes an API, particularly
+if designed to be called remotely, and so it makes sense consider
+whether they are, and design expandability into them from day 1.
+
+---
+
+Anyway, we now have full transfer tracking in git-annex! Both sides of
+a transfer know what's being transferred, and from where, and have
+the info necessary to interrupt the transfer.
+
+---
+
+Also did some basic groundwork, adding a queue of transfers to perform,
+and adding to the daemon's status information a map of currently running
+transfers. 
+
+Next up: The daemon will use inotify to notice new and deleted transfer
+info files, and update its status info.
diff --git a/doc/design/assistant/blog/day_230__Mom.mdwn b/doc/design/assistant/blog/day_230__Mom.mdwn
new file mode 100644
index 0000000000..66a0c864d1
--- /dev/null
+++ b/doc/design/assistant/blog/day_230__Mom.mdwn
@@ -0,0 +1,35 @@
+Made a release today. Releasing has sure gotten easier with all the
+autobuilds to use!
+
+I am now using git-annex to share files with my mom. Here's how the webapp
+looks for our family's repository. Soon several of us will be using this
+repository.
+
+[[!img assistant/example.png]]
+
+We're using XMPP and rsync.net, so pretty standard setup much like
+shown in my last screencast.
+
+Real-world deployments help find bugs, and I found a few:
+
+* If you're running the webapp in `w3m` on a remote computer to set it up,
+  some forms are lacking submit buttons. This must be a issue with
+  Bootstrap, or HTML5, I guess. I switched to `lynx` and it offers a
+  way to submit forms that lack an explicit button.
+
+* Progress bars for downloads from encrypted rsync repos don't update
+  during the actual download, but only when gpg is decrypting the
+  downloaded file.
+
+* XMPP pushes sometimes fail still. Especially when your mom's computer
+  is saturating its limited outgoing network connection uploading hundreds of
+  photos. I have not yet determined if this is a packet loss/corruption issue,
+  or if the XMPP messages are getting out of order. My gut feeling is it's
+  the latter, in which can I can fix this pretty easily by adding sequence
+  numbers and some buffering for out of order packets. Or perhaps just
+  make it retry failed pushes when this happens.
+  
+  Anyway, I found it was useful to set up a regular git repository on a
+  server to suppliment the git pushes over XMPP. It's helpful to have
+  such a git repository anyway, so that clients can push to there when the
+  other client(s) are not online to be pushed to directly over XMPP.
diff --git a/doc/design/assistant/blog/day_230__Mom/comment_1_696bba2246c8a9e6ce4aed3071bcc96c._comment b/doc/design/assistant/blog/day_230__Mom/comment_1_696bba2246c8a9e6ce4aed3071bcc96c._comment
new file mode 100644
index 0000000000..5b8b8187f1
--- /dev/null
+++ b/doc/design/assistant/blog/day_230__Mom/comment_1_696bba2246c8a9e6ce4aed3071bcc96c._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="http://edheil.wordpress.com/"
+ ip="68.35.120.66"
+ subject="comment 1"
+ date="2013-04-05T23:50:14Z"
+ content="""
+This is probably a Dumb Git Question, but is there a tag in the repo for the new release?  I go into my clone and type \"git tag\" and the latest one I see is 4.20130323.
+"""]]
diff --git a/doc/design/assistant/blog/day_230__Mom/comment_2_2fa295ab6db0828cb725cfcfb6777822._comment b/doc/design/assistant/blog/day_230__Mom/comment_2_2fa295ab6db0828cb725cfcfb6777822._comment
new file mode 100644
index 0000000000..3f1666d891
--- /dev/null
+++ b/doc/design/assistant/blog/day_230__Mom/comment_2_2fa295ab6db0828cb725cfcfb6777822._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="https://www.google.com/accounts/o8/id?id=AItOawln3ckqKx0x_xDZMYwa9Q1bn4I06oWjkog"
+ nickname="Michael"
+ subject="deletions by mistake?"
+ date="2013-04-06T04:58:09Z"
+ content="""
+Let's say a family member deletes a file by mistake in a synced diretory?  Looks like this kind of error is easy to go unnoticed.  What would be a good way to handle this?
+"""]]
diff --git a/doc/design/assistant/blog/day_230__Mom/comment_3_fafd7abec629290418334ddb015bf62c._comment b/doc/design/assistant/blog/day_230__Mom/comment_3_fafd7abec629290418334ddb015bf62c._comment
new file mode 100644
index 0000000000..9cfa45acaa
--- /dev/null
+++ b/doc/design/assistant/blog/day_230__Mom/comment_3_fafd7abec629290418334ddb015bf62c._comment
@@ -0,0 +1,10 @@
+[[!comment format=mdwn
+ username="http://joeyh.name/"
+ nickname="joey"
+ subject="comment 3"
+ date="2013-04-06T16:26:57Z"
+ content="""
+Weird, I don't know where the tag went. I've pushed a new one.
+
+If a file is deleted, it's still there in git, and git-annex hangs onto its contents. You need a git history browser to get it back of course.
+"""]]
diff --git a/doc/design/assistant/blog/day_230__Mom/comment_4_450cac0f2e82c94fd34b527ae05ef1b8._comment b/doc/design/assistant/blog/day_230__Mom/comment_4_450cac0f2e82c94fd34b527ae05ef1b8._comment
new file mode 100644
index 0000000000..56c8d3204e
--- /dev/null
+++ b/doc/design/assistant/blog/day_230__Mom/comment_4_450cac0f2e82c94fd34b527ae05ef1b8._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="https://www.google.com/accounts/o8/id?id=AItOawlUbH3eytydcwlWqv8oauE2Jg4NwcV9uA0"
+ nickname="Anna"
+ subject="me!"
+ date="2013-04-06T18:22:30Z"
+ content="""
+I want in on the family repository! :-)
+"""]]
diff --git a/doc/design/assistant/blog/day_231__insert_title.mdwn b/doc/design/assistant/blog/day_231__insert_title.mdwn
new file mode 100644
index 0000000000..6f91b2d363
--- /dev/null
+++ b/doc/design/assistant/blog/day_231__insert_title.mdwn
@@ -0,0 +1,26 @@
+Finally fixed the bug causing repeated checksumming when a direct mode file
+contains duplicate files. I may need to add some cleaning of stale inode
+caches eventually.
+
+Meanwhile, Guilhem made `git annex initremote` use higher quality entropy,
+with `--fast` getting back to the old behavior of urandom quality entropy.
+The assistant doesn't use high quality entropy since I have no way to
+prompt when the user would need to generate more. I did have a fun idea to
+deal with this: Make a javascript game, that the user can play while
+waiting, which would generate enropy nicely. Maybe one day.. ;)
+
+Also made a small but significant change to [[archive directory handling|todo/assistant_smarter_archive_directory_handling]].
+Now the assistant syncs files that are in `archive` directories like any
+other file, until they reach an archive repository. Then they get dropped
+from all the clients. This way, users who don't set up archive repositories
+don't need to know about this special case, and users who do want to use
+them can, with no extra configuration.
+
+After recent changes, the preferred content expression for transfer
+repositories is becoming a bit unweidly, at 212 characters. Probably
+time to add support for macros..
+
+`(not (inallgroup=client and copies=client:2) and (((exclude=*/archive/* and exclude=archive/*) or (not (copies=archive:1 or copies=smallarchive:1))) or (not copies=semitrusted+:1))) or (not copies=semitrusted+:1)`
+
+Still, it's pretty great how much this little language lets me express, so
+easily.
diff --git a/doc/design/assistant/blog/day_232__headless_webapp.mdwn b/doc/design/assistant/blog/day_232__headless_webapp.mdwn
new file mode 100644
index 0000000000..a60ab383db
--- /dev/null
+++ b/doc/design/assistant/blog/day_232__headless_webapp.mdwn
@@ -0,0 +1,22 @@
+Developed a way to run the webapp on a remote or headless computer.
+
+The webapp can now be started on a remote or headless computer, just
+specify `--listen=address` to make it listen on an address other than
+localhost. It'll print out the URL to open to access it.
+
+This doesn't use HTTPS yet, because it'd need to generate a
+certificate, and even if it generated a self-signed SSL certificate,
+there'd be no easy way for the browser to verify it and avoid a MITM.
+
+So `--listen` is a less secure but easier option; using ssh to forward
+the webapp's port to the remote computer is more secure.
+
+(I do have an idea for a way to do this entirely securely, making
+the webapp set up the ssh port forwarding, which I have written down
+in [[webapp]].. but it would be rather complicated to implement.)
+
+----
+
+Made the webapp rescan for transfers after it's been used to change a
+repository's group. Would have been easy, but I had to chase down a
+cache invalidation bug.
diff --git a/doc/design/assistant/blog/day_232__headless_webapp/comment_1_0fdd77d143ecba6fdb9f75cb6fc37bfb._comment b/doc/design/assistant/blog/day_232__headless_webapp/comment_1_0fdd77d143ecba6fdb9f75cb6fc37bfb._comment
new file mode 100644
index 0000000000..1b36a9ad89
--- /dev/null
+++ b/doc/design/assistant/blog/day_232__headless_webapp/comment_1_0fdd77d143ecba6fdb9f75cb6fc37bfb._comment
@@ -0,0 +1,16 @@
+[[!comment format=mdwn
+ username="https://www.google.com/accounts/o8/id?id=AItOawl9sYlePmv1xK-VvjBdN-5doOa_Xw-jH4U"
+ nickname="Richard"
+ subject="Certificate storage"
+ date="2013-04-09T00:43:12Z"
+ content="""
+Why not store the certificate within the repository? Either a common one in, e.g.
+
+    .git-annex.key-cert.pem
+
+or host-specific as per
+
+    .git-annex.key-cert.pem.$HOST
+
+-- Richard
+"""]]
diff --git a/doc/design/assistant/blog/day_232__headless_webapp/comment_2_0784a2a73c3e2945f3d3f2577b3b9c9c._comment b/doc/design/assistant/blog/day_232__headless_webapp/comment_2_0784a2a73c3e2945f3d3f2577b3b9c9c._comment
new file mode 100644
index 0000000000..67e7972434
--- /dev/null
+++ b/doc/design/assistant/blog/day_232__headless_webapp/comment_2_0784a2a73c3e2945f3d3f2577b3b9c9c._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="https://www.google.com/accounts/o8/id?id=AItOawmU_2tE75oyG0h2ZPN4lcroIKEMC8G-otE"
+ nickname="Michael"
+ subject="comment 2"
+ date="2013-04-09T07:27:53Z"
+ content="""
+Can you also set the port via listen= ?
+"""]]
diff --git a/doc/design/assistant/blog/day_232__headless_webapp/comment_3_ccb9fa22422fb913b6a496ebe65c49fb._comment b/doc/design/assistant/blog/day_232__headless_webapp/comment_3_ccb9fa22422fb913b6a496ebe65c49fb._comment
new file mode 100644
index 0000000000..e2c00c59e3
--- /dev/null
+++ b/doc/design/assistant/blog/day_232__headless_webapp/comment_3_ccb9fa22422fb913b6a496ebe65c49fb._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="http://joeyh.name/"
+ nickname="joey"
+ subject="comment 3"
+ date="2013-04-09T18:58:40Z"
+ content="""
+@Richard besides the issues with where to store the private key part, that doesn't help with probably the most common use case for a headless webapp: Setting up a new repository on the remote system, which then might get joined into an existing repository network.
+"""]]
diff --git a/doc/design/assistant/blog/day_232__headless_webapp/comment_4_ceba4468760a2525960327698431cee2._comment b/doc/design/assistant/blog/day_232__headless_webapp/comment_4_ceba4468760a2525960327698431cee2._comment
new file mode 100644
index 0000000000..e953233ebe
--- /dev/null
+++ b/doc/design/assistant/blog/day_232__headless_webapp/comment_4_ceba4468760a2525960327698431cee2._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="http://joeyh.name/"
+ nickname="joey"
+ subject="comment 4"
+ date="2013-04-09T19:18:24Z"
+ content="""
+@Michael you can now! --listen=address:port
+"""]]
diff --git a/doc/design/assistant/blog/day_232__headless_webapp/comment_5_7e51a197ff9970ae50cf47bd3a922257._comment b/doc/design/assistant/blog/day_232__headless_webapp/comment_5_7e51a197ff9970ae50cf47bd3a922257._comment
new file mode 100644
index 0000000000..de4be2556b
--- /dev/null
+++ b/doc/design/assistant/blog/day_232__headless_webapp/comment_5_7e51a197ff9970ae50cf47bd3a922257._comment
@@ -0,0 +1,18 @@
+[[!comment format=mdwn
+ username="http://joeyh.name/"
+ ip="209.250.56.22"
+ subject="comment 5"
+ date="2014-09-16T18:36:18Z"
+ content="""
+Note that --listen=address:port had to be removed.
+
+OTOH, the webapp can be run with a https certificate now, which makes remote access much more secure.
+
+The webapp will use HTTPS if it finds
+a .git/annex/privkey.pem and .git/annex/certificate.pem. Here's
+one way to generate those files, using a self-signed certificate:
+
+    openssl genrsa -out .git/annex/privkey.pem 4096
+    openssl req -new -x509 -key .git/annex/privkey.pem > .git/annex/certificate.pem
+
+"""]]
diff --git a/doc/design/assistant/blog/day_233__taxes.mdwn b/doc/design/assistant/blog/day_233__taxes.mdwn
new file mode 100644
index 0000000000..55273dd4b5
--- /dev/null
+++ b/doc/design/assistant/blog/day_233__taxes.mdwn
@@ -0,0 +1,11 @@
+Did my taxes today. Not very pretty. Planning to run them by a professional.
+
+Reproduced a bug that prevents git-annex from authenticating to the ejabberd
+server, and passed the buck upstream with a test case to the author of the
+haskell XMPP library.
+
+Added some animations to the webapp to show when it's busy doing things.
+
+Made `git annex webapp --listen=address:port` work.
+
+Added a `annex.web-download-command` setting.
diff --git a/doc/design/assistant/blog/day_233__taxes/comment_1_9473ffdc42595af9c293fbcd5a1cdb54._comment b/doc/design/assistant/blog/day_233__taxes/comment_1_9473ffdc42595af9c293fbcd5a1cdb54._comment
new file mode 100644
index 0000000000..a35e738c14
--- /dev/null
+++ b/doc/design/assistant/blog/day_233__taxes/comment_1_9473ffdc42595af9c293fbcd5a1cdb54._comment
@@ -0,0 +1,14 @@
+[[!comment format=mdwn
+ username="https://www.google.com/accounts/o8/id?id=AItOawm5iosFbL2By7UFeViqkc6v-hoAtqILeDA"
+ nickname="Laszlo"
+ subject="comment 1"
+ date="2013-04-10T13:35:30Z"
+ content="""
+I don't really want to nagging you, but I'm impatiently waiting 
+for a new build (for linux) to be able to try out your duplicated file fix.
+
+Please don't delay your next release too much:)
+
+Laszlo
+
+"""]]
diff --git a/doc/design/assistant/blog/day_233__taxes/comment_2_5feed8d7053ba03812ccda8c61fd9775._comment b/doc/design/assistant/blog/day_233__taxes/comment_2_5feed8d7053ba03812ccda8c61fd9775._comment
new file mode 100644
index 0000000000..32e4f077fb
--- /dev/null
+++ b/doc/design/assistant/blog/day_233__taxes/comment_2_5feed8d7053ba03812ccda8c61fd9775._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="http://joeyh.name/"
+ nickname="joey"
+ subject="comment 2"
+ date="2013-04-11T14:57:24Z"
+ content="""
+You can already try that fix out using the [[install/Linux_standalone]] autobuild.
+"""]]
diff --git a/doc/design/assistant/blog/day_234__clean_shutdown.mdwn b/doc/design/assistant/blog/day_234__clean_shutdown.mdwn
new file mode 100644
index 0000000000..61b765b593
--- /dev/null
+++ b/doc/design/assistant/blog/day_234__clean_shutdown.mdwn
@@ -0,0 +1,29 @@
+Short day because I spent 3 hours this morning explaining free software
+and kickstarter to an accountant. And was away until 3 pm, so how did I get
+all this done‽
+
+Eliot pointed out that shutting down the assistant could leave transfers
+running. This happened because `git annex transferkeys` is a separate
+process, and so it was left to finish up any transfer that was in
+process. I've made shutdown stop all transfers that the
+assistant started. (Other paired computers could still be connecting to
+make transfers even when the assistant is not running, and those are not
+affected.)
+
+Added sequence numbers to the XMPP messages used for git pushes. While
+these numbers are not used yet, they're available for debugging, and will
+help me determine if packets are lost or come out of order. So if you have
+experienced problems with XMPP syncing sometimes failing, run tonight's
+build of the assistant with `--debug` (or turn on debugging in the webapp
+configuration screen), and send me a log by email to
+
+
+Changed the way that autobuilds and manual builds report their version
+number. It now includes the date of the last commit, and the abbreviated
+commit ID, rather than being some random date after the last release.
+
+Frederik found a bug using the assistant on a FAT filesystem. It didn't
+properly handle the files that git uses to stand-in for symlinks in that
+situation, and annexed those files. I've fixed this, and even moving
+around symlink stand-in files on a FAT filesystem now results in correct
+changes to symlinks being committed.
diff --git a/doc/design/assistant/blog/day_235__birthday.mdwn b/doc/design/assistant/blog/day_235__birthday.mdwn
new file mode 100644
index 0000000000..2bd2b0d225
--- /dev/null
+++ b/doc/design/assistant/blog/day_235__birthday.mdwn
@@ -0,0 +1,31 @@
+Felt like spending my birthday working on git-annex. Thanks again to
+everyone who makes it possible for me to work on something I care about
+every day.
+
+----
+
+Did some work on `git annex addurl` today. It had gotten broken in direct
+mode (I think by an otherwise good and important bugfix). After fixing
+that, I made it interoperate with the webapp. So if you have the webapp
+open, it will display progress bars for downloads being run by `git annex
+addurl`.
+
+This enhancement meshes nicely with a FlashGot script Andy
+contributed, which lets you queue up downloads into your annex from a web
+browser. Andy described how to set it up in
+[[this_tip|tips/Using_Git-annex_as_a_web_browsing_assistant]].
+
+(I also looked briefly into ways to intercept a drag and drop of a link
+into the webapp and make it lauch a download for you. It doesn't seem that
+browsers allow javascript to override their standard behavior of loading
+links that are dropped into them. Probably good to prevent misuse, but it
+would be nice here...)
+
+----
+
+Also, I think I have fixed the progress bars displayed when downloading
+a file from an encrypted remote. I did this by hooking up existing
+download progress metering (that was only being used to display
+a download percentage in the console) into the location log, so the
+webapp can use it. So that was a *lot* easier than it could have been,
+but still a pretty large patch (500+ lines). Haven't tested this; should work.
diff --git a/doc/design/assistant/blog/day_235__birthday/comment_1_db558b071067c1e63cde05cca0551094._comment b/doc/design/assistant/blog/day_235__birthday/comment_1_db558b071067c1e63cde05cca0551094._comment
new file mode 100644
index 0000000000..32c2c5d50b
--- /dev/null
+++ b/doc/design/assistant/blog/day_235__birthday/comment_1_db558b071067c1e63cde05cca0551094._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="https://www.google.com/accounts/o8/id?id=AItOawksoT5jmbiZ9UghROilchqy3kNJ1XGqCAo"
+ nickname="Rob"
+ subject="Drag and Drop"
+ date="2013-04-12T03:37:53Z"
+ content="""
+Is this along the lines of what you are looking for? http://www.dropzonejs.com/
+"""]]
diff --git a/doc/design/assistant/blog/day_235__birthday/comment_2_d1a2c1124781118267599457ae9e0512._comment b/doc/design/assistant/blog/day_235__birthday/comment_2_d1a2c1124781118267599457ae9e0512._comment
new file mode 100644
index 0000000000..e41c083ce5
--- /dev/null
+++ b/doc/design/assistant/blog/day_235__birthday/comment_2_d1a2c1124781118267599457ae9e0512._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="marvin"
+ ip="91.152.75.65"
+ subject="Happy Birthday"
+ date="2013-04-12T17:52:43Z"
+ content="""
+Great work!
+"""]]
diff --git a/doc/design/assistant/blog/day_235__birthday/comment_3_b853508d1d15234958b9f4a39277e45c._comment b/doc/design/assistant/blog/day_235__birthday/comment_3_b853508d1d15234958b9f4a39277e45c._comment
new file mode 100644
index 0000000000..350ac2bc2f
--- /dev/null
+++ b/doc/design/assistant/blog/day_235__birthday/comment_3_b853508d1d15234958b9f4a39277e45c._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="robconnolly"
+ ip="118.90.78.117"
+ subject="comment 3"
+ date="2013-04-13T03:12:14Z"
+ content="""
+Thanks for fixing the addurl bug and Happy Birthday!
+"""]]
diff --git a/doc/design/assistant/blog/day_235__birthday/comment_5_73aad3398a43bc4d28bca9bf635fa757._comment b/doc/design/assistant/blog/day_235__birthday/comment_5_73aad3398a43bc4d28bca9bf635fa757._comment
new file mode 100644
index 0000000000..9692539714
--- /dev/null
+++ b/doc/design/assistant/blog/day_235__birthday/comment_5_73aad3398a43bc4d28bca9bf635fa757._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="andy"
+ ip="99.48.75.171"
+ subject="Just in case it might be helpful..."
+ date="2013-04-14T05:56:47Z"
+ content="""
+For whatever reason, dragging links onto the page text in the \"Using drag and drop for selecting\" section of  does not trigger the usual page change. I don't really know why--I couldn't get it to work on my own with Firebug. And thanks for git-annex--love it!
+"""]]
diff --git a/doc/design/assistant/blog/day_236__evil_splicer.mdwn b/doc/design/assistant/blog/day_236__evil_splicer.mdwn
new file mode 100644
index 0000000000..72c8f51a99
--- /dev/null
+++ b/doc/design/assistant/blog/day_236__evil_splicer.mdwn
@@ -0,0 +1,29 @@
+Spent today bulding the Evil Splicer, a program that parses `ghc -ddump-splices`
+output, and uses it to expand Template Haskell splices in source code.
+I hope to use this crazy hack to get the webapp working on Android.
+
+This was a good opportunity to use the
+[Parsec](http://hackage.haskell.org/package/parsec) library for parsing the
+ghc output. I've never really used it before, but found it quite nice to
+work with. The learning curve, if you already know monads and applicatives,
+is about 5 minutes. And instead of ugly regular expressions, you can work
+with nice code that's easily composable and refactorable. Even the ugly
+bits come out well:
+
+[[!format haskell """
+	{- All lines of the splice result will start with the same
+	- indent, which is stripped. Any other indentation is preserved. -}
+	i <- lookAhead indent
+	result <- unlines <$> many1 (string i >> restOfLine)
+"""]]
+
+Anyway, it works.. sorta. The parser works great! The splices that ghc
+outputs are put into the right places in the source files, and formatted in
+a way that ghc is able to build. Often though, they contain code that
+doesn't actually build as-is. I'm working to fix up the code to get closer
+to buildable.
+
+----
+
+Meanwhile, guilhem has made ssh connection caching work for rsync special
+remotes! It's very nice to have another developer working on git-annex. :)
diff --git a/doc/design/assistant/blog/day_237__gnome-keyring_craziness.mdwn b/doc/design/assistant/blog/day_237__gnome-keyring_craziness.mdwn
new file mode 100644
index 0000000000..01b8195b69
--- /dev/null
+++ b/doc/design/assistant/blog/day_237__gnome-keyring_craziness.mdwn
@@ -0,0 +1,29 @@
+Fixed a bug where the locked down ssh key that the assistant sets up to
+access the annex on a remote server was being used by ssh *by default* for
+all logins to that server.
+
+That should not have happened. The locked down key is written to a filename
+that ssh won't use at all, by default. But, I found code in gnome-keyring
+that watches for `~/.ssh/*.pub` to appear, and automatically adds all such
+keys to the keyring. In at least some cases, probably when it has no other
+key, it then tells ssh to go ahead and use that key. Astounding.
+
+To avoid this, the assistant will store its keys in `~/.ssh/annex/`
+`~/.ssh/git-annex/` instead. gnome-keyring does not look there (verified in
+the source). If you use gnome-keyring and have set up a repository on a
+remote server with the assistant, I'd recommend moving the keys it set up
+and editing `~/.ssh/config` to point to their new location.
+
+gnome-keyring is not the only piece of software that has a bad
+interaction with git-annex. I've been working on a bug that makes git-annex
+fail to authenticate to ejabberd. ejabberd 2.1.10 got support for
+SCRAM-SHA-1, but its code violates the RFC, and chokes on an address
+attribute that the haskell XMPP library provides. I hope to get this fixed
+in ejabberd.
+
+
+Also did some more work on the Evil Splicer today, integrating it into the
+build of the Android app, and making it support incremental building.
+Improved its code generation, and am at the milestone where it creates
+valid haskell code for the entire `Assistant/WebApp/Types.hs` file,
+where Template Haskell expands 2 lines into 2300 lines of code!
diff --git a/doc/design/assistant/blog/day_237__gnome-keyring_craziness/comment_1_0cb088b732881d1fa92493aa1fd93d43._comment b/doc/design/assistant/blog/day_237__gnome-keyring_craziness/comment_1_0cb088b732881d1fa92493aa1fd93d43._comment
new file mode 100644
index 0000000000..e8cc457821
--- /dev/null
+++ b/doc/design/assistant/blog/day_237__gnome-keyring_craziness/comment_1_0cb088b732881d1fa92493aa1fd93d43._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="https://www.google.com/accounts/o8/id?id=AItOawl9sYlePmv1xK-VvjBdN-5doOa_Xw-jH4U"
+ nickname="Richard"
+ subject="Confusing name for storage directory"
+ date="2013-04-14T23:55:03Z"
+ content="""
+`~/.ssh/annex/` would make sense for ssh-annex. I would suggest using `~/.ssh/git-annex/` or similar, instead.
+"""]]
diff --git a/doc/design/assistant/blog/day_237__gnome-keyring_craziness/comment_2_b855fd710954beebaafe6d2bd03eb368._comment b/doc/design/assistant/blog/day_237__gnome-keyring_craziness/comment_2_b855fd710954beebaafe6d2bd03eb368._comment
new file mode 100644
index 0000000000..cf3e8fbbf2
--- /dev/null
+++ b/doc/design/assistant/blog/day_237__gnome-keyring_craziness/comment_2_b855fd710954beebaafe6d2bd03eb368._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="http://joeyh.name/"
+ nickname="joey"
+ subject="comment 2"
+ date="2013-04-14T23:59:44Z"
+ content="""
+I agree, change made
+"""]]
diff --git a/doc/design/assistant/blog/day_238__back_to_Android.mdwn b/doc/design/assistant/blog/day_238__back_to_Android.mdwn
new file mode 100644
index 0000000000..b07d784fa3
--- /dev/null
+++ b/doc/design/assistant/blog/day_238__back_to_Android.mdwn
@@ -0,0 +1,11 @@
+Back to really working toward an Android webapp now. I have been improving
+the EvilSplicer, and the build machinery, and build environment all day.
+Slow but steady progress.
+
+First milestone of the day was when I got `yesod-form` to build with all
+Template Haskell automatically expanded by the EvilSplicer. (With a few
+manual fixups where it's buggy.)
+
+At this point the Android build with the webapp enabled successfully
+builds several files containing Yesod code.. And I suspect I am very close
+to getting a first webapp build for Android.
diff --git a/doc/design/assistant/blog/day_239__bugfixes_and_frustration.mdwn b/doc/design/assistant/blog/day_239__bugfixes_and_frustration.mdwn
new file mode 100644
index 0000000000..2ca62cfe67
--- /dev/null
+++ b/doc/design/assistant/blog/day_239__bugfixes_and_frustration.mdwn
@@ -0,0 +1,28 @@
+Several bug fixes today, and got mostly caught up on recent messages.
+Still have a backlog of two known bugs that I cannot reproduce well enough
+to have worked on, but I am thinking I will make a release tomorrow. There
+have been a lot of changes in the 10 days since the last release.
+
+----
+
+I am, frustratingly, stuck building the webapp on Android with no forward
+progress today (and last night) after such a productive day yesterday.
+
+The expanded Template Haskell code of the webapp fails to compile,
+whereever type safe urls are used.
+
+
    +Assistant/WebApp/Types.hs:95:63:
+    Couldn't match expected type `Route WebApp -> t2'
+                with actual type `Text'
+    The function `urender_a1qcK' is applied to three arguments,
+    but its type `Route WebApp -> [(Text, Text)] -> Text' has only two
+    In the expression: urender_a1qcK u_a1qcL [] LogR
+    In the first argument of `toHtml', namely
+      `(\ u_a1qcL -> urender_a1qcK u_a1qcL [] LogR)'
+
    
+
+My best guess is this is a mismatch between the versions of yesod (or other
+libraries) used for the native and cross compiled ghc's. So I've been
+slowly trying to get a fully matched set of versions in between working on
+bugs.
diff --git a/doc/design/assistant/blog/day_23__transfer_watching.mdwn b/doc/design/assistant/blog/day_23__transfer_watching.mdwn
new file mode 100644
index 0000000000..3e4e27d479
--- /dev/null
+++ b/doc/design/assistant/blog/day_23__transfer_watching.mdwn
@@ -0,0 +1,25 @@
+Starting to travel, so limited time today.
+
+Yet Another Thread added to the assistant, all it does is watch for changes
+to transfer information files, and update the assistant's map of transfers
+currently in progress. Now the assistant will know if some other repository
+has connected to the local repo and is sending or receiving a file's
+content.
+
+This seemed really simple to write, it's just 78 lines of code. It worked
+100% correctly the first time. :) But it's only so easy because I've got
+this shiny new inotify hammer that I keep finding places to use in the
+assistant.
+
+Also, the new thread does some things that caused a similar thread (the
+merger thread) to go into a MVar deadlock. Luckily, I spent much of 
+[day 19](day_19__random_improvements) investigating and fixing that
+deadlock, even though it was not a problem at the time.
+
+So, good.. I'm doing things right and getting to a place where rather
+nontrivial features can be added easily.
+
+--
+
+Next up: Enough nonsense with tracking transfers... Time to start actually
+transferring content around!
diff --git a/doc/design/assistant/blog/day_240__it_builds.mdwn b/doc/design/assistant/blog/day_240__it_builds.mdwn
new file mode 100644
index 0000000000..dc975af817
--- /dev/null
+++ b/doc/design/assistant/blog/day_240__it_builds.mdwn
@@ -0,0 +1,37 @@
+Late last night, I successfully built the full webapp for Android!
+
+That was with several manual modifications to the generated code, which I
+still need to automate. And I need to set up the autobuilder properly
+still. And I need to find a way to make the webapp open Android's web browser
+to URL. So it'll be a while yet until a package is available
+to try. But what a milestone!
+
+The point I was stuck on all day yesterday was generated code that
+looked like this:
+
+[[!format haskell """
+(toHtml
+  (\ u_a2ehE -> urender_a2ehD u_a2ehE []
+    (CloseAlert aid)))));
+"""]]
+
+That just couldn't type check at all. Most puzzling. My best guess is that
+`u_a2ehE` is the dictionary GHC passes internally to make a typeclass work,
+which somehow leaked out and became visible. Although
+I can't rule out that I may have messed something up in my build environment.
+The EvilSplicer has a hack in it that finds such code and converts it to
+something like this:
+
+[[!format haskell """
+(toHtml
+  (flip urender_a2ehD []
+    (CloseAlert aid)))));
+"""]]
+
+I wrote some more about the process of the Android port in my personal blog:
+[Template Haskell on impossible architectures](http://joeyh.name/blog/entry/Template_Haskell_on_impossible_architectures/)
+
+----
+
+Release day today. The OSX builds are both not available yet for this
+release, hopefully will come out soon.
diff --git a/doc/design/assistant/blog/day_240__it_builds/comment_1_151840ae0020ea63b2f041488c905386._comment b/doc/design/assistant/blog/day_240__it_builds/comment_1_151840ae0020ea63b2f041488c905386._comment
new file mode 100644
index 0000000000..a8e4895827
--- /dev/null
+++ b/doc/design/assistant/blog/day_240__it_builds/comment_1_151840ae0020ea63b2f041488c905386._comment
@@ -0,0 +1,25 @@
+[[!comment format=mdwn
+ username="https://www.google.com/accounts/o8/id?id=AItOawm5iosFbL2By7UFeViqkc6v-hoAtqILeDA"
+ nickname="Laszlo"
+ subject="Duplicated files resolved"
+ date="2013-04-18T05:52:35Z"
+ content="""
+Hi,
+
+Seems like the duplicated file issue finally resolved in this release, 
+also the logging is not so detailed, so it does not explode the disk space either.
+
+I think a new bug introduced in this release, 
+namely if you create a repository and a backup repository of it (pendrive), then this repository is not in direct mode by default.
+(and there is a missing direct = True variable in that repository (/mnt/dat/annex_backup/.git/annex/config)).
+
+Also I run into resource exhausted bug, and every thread is crashed.
+Can git-annex only watch a directory, and not every file in it?
+
+I will create a bugreport for resource exhausted.
+
+Best, 
+ Laszlo
+
+
+"""]]
diff --git a/doc/design/assistant/blog/day_241__cleanup.mdwn b/doc/design/assistant/blog/day_241__cleanup.mdwn
new file mode 100644
index 0000000000..077922695e
--- /dev/null
+++ b/doc/design/assistant/blog/day_241__cleanup.mdwn
@@ -0,0 +1,14 @@
+Finished the last EvilSplicer tweak and other fixes to make the 
+Android webapp build without any hand-holding.
+
+Currently setting up the Android autobuilder to include the webapp in its
+builds. To make this work I had to set up a new chroot with all the right
+stuff installed.
+
+Investigated how to make the Android webapp open a web browser when run.
+As far as I can tell (without access to an Android device right now), 
+`am start -a android.intent.action.VIEW -d http://localhost/etc` should do
+it.
+
+Seems that git 1.8.2 broke the assistant. I've put in a fix but have not
+yet tested it.
diff --git a/doc/design/assistant/blog/day_241__cleanup/comment_1_0e283cdf66a25b3cc9423fe651084cb9._comment b/doc/design/assistant/blog/day_241__cleanup/comment_1_0e283cdf66a25b3cc9423fe651084cb9._comment
new file mode 100644
index 0000000000..d9e468f324
--- /dev/null
+++ b/doc/design/assistant/blog/day_241__cleanup/comment_1_0e283cdf66a25b3cc9423fe651084cb9._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="https://www.google.com/accounts/o8/id?id=AItOawlJ2utMQgMEYAOs3Dfc6eZRyUzt4acNXUU"
+ nickname="David"
+ subject="am start -a android.intent.action.VIEW -d http://localhost/etc"
+ date="2013-04-18T22:39:50Z"
+ content="""
+Just tested the command on an Android device, it works well.
+"""]]
diff --git a/doc/design/assistant/blog/day_242__more_porting.mdwn b/doc/design/assistant/blog/day_242__more_porting.mdwn
new file mode 100644
index 0000000000..89fbfea0f1
--- /dev/null
+++ b/doc/design/assistant/blog/day_242__more_porting.mdwn
@@ -0,0 +1,4 @@
+Got WebDAV enabled in the Android build. Had to deal with some system calls
+not available in Android's libc.
+
+New poll: [[polls/Android_default_directory]]
diff --git a/doc/design/assistant/blog/day_243__in_the_field.mdwn b/doc/design/assistant/blog/day_243__in_the_field.mdwn
new file mode 100644
index 0000000000..b05ccac9e9
--- /dev/null
+++ b/doc/design/assistant/blog/day_243__in_the_field.mdwn
@@ -0,0 +1,21 @@
+Today was not a work day for me, but I did get a chance to install
+git-annex in real life while visiting. Was happy to download the standalone
+Linux tarball and see that it could be unpacked, and git-annex webapp
+started just by clicking around in the GUI. And in very short order got it
+set up.
+
+I was especially pleased to see my laptop noticed this new repository
+had appeared on the network via XMPP push, and started immediately
+uploading files to my rsync.net transfer repository so the new
+repository could get them.
+
+Did notice that the standalone tarball neglected to install a
+FDO menu file. Fixed that, and some other minor issues I noticed.
+
+
+I also got a brief chance to try the Android webapp. It fails to start;
+apparently `getaddrinfo` doesn't like the flags passed to it and is
+failing. As failure modes go, this isn't at all bad. I can certainly work
+around it with some hardcoded port numbers, but I want to fix it the right
+way. Have ordered a replacement battery for my dead phone so I can use it
+for Android testing.
diff --git a/doc/design/assistant/blog/day_244__android_porting.mdwn b/doc/design/assistant/blog/day_244__android_porting.mdwn
new file mode 100644
index 0000000000..e535c1054b
--- /dev/null
+++ b/doc/design/assistant/blog/day_244__android_porting.mdwn
@@ -0,0 +1,6 @@
+Ported all the C libraries needed for XMPP to Android. (gnutls,
+libgcrypt, libgpg-error, nettle, xml2, gsasl, etc). Finally
+got it all to link. What a pain.
+
+Bonus: Local pairing support builds for Android now, seems recent changes to
+the network library for WebDAV also fixed it.
diff --git a/doc/design/assistant/blog/day_245__misc.mdwn b/doc/design/assistant/blog/day_245__misc.mdwn
new file mode 100644
index 0000000000..8559ac4f10
--- /dev/null
+++ b/doc/design/assistant/blog/day_245__misc.mdwn
@@ -0,0 +1,15 @@
+Got the OSX autobuilder back running, and finally got a OSX build up for
+the 4.20130417 release. Also fixed the OSX app build machinery to handle
+rpath.
+
+Made the assistant (and `git annex sync`) sync with git remotes that have
+`annex-ignore` set. So, `annex-ignore` is only used to prevent using
+the annex of a remote, not syncing with it. The benefit of this change
+is that it'll make the assistant sync the local git repository with
+a git remote that is on a server that does not have git-annex installed.
+It can even sync to github.
+
+Worked around more breakage on misconfigured systems that don't have GECOS
+information.
+
+... And other bug fixes and bug triage.
diff --git a/doc/design/assistant/blog/day_245__misc/comment_1_3a2976617bb0cdc206fb1397a2ef1177._comment b/doc/design/assistant/blog/day_245__misc/comment_1_3a2976617bb0cdc206fb1397a2ef1177._comment
new file mode 100644
index 0000000000..4ee7612f11
--- /dev/null
+++ b/doc/design/assistant/blog/day_245__misc/comment_1_3a2976617bb0cdc206fb1397a2ef1177._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="spwhitton"
+ ip="192.76.7.217"
+ subject="comment 1"
+ date="2013-04-23T17:11:46Z"
+ content="""
+If I set my annex-ignore'd respository to `annex-sync = false` will this override the new behaviour?  My remote requires mounting a fuse mount, so this change will just result in my seeing lots of sync failure messages in the webapp.
+"""]]
diff --git a/doc/design/assistant/blog/day_245__misc/comment_2_e0f9704e91fedca8ff26356f354cc1c3._comment b/doc/design/assistant/blog/day_245__misc/comment_2_e0f9704e91fedca8ff26356f354cc1c3._comment
new file mode 100644
index 0000000000..545da72cdb
--- /dev/null
+++ b/doc/design/assistant/blog/day_245__misc/comment_2_e0f9704e91fedca8ff26356f354cc1c3._comment
@@ -0,0 +1,10 @@
+[[!comment format=mdwn
+ username="http://joeyh.name/"
+ nickname="joey"
+ subject="comment 2"
+ date="2013-04-23T20:16:26Z"
+ content="""
+Yes, setting that, or just clicking to pause syncing to a remote in the webapp will still work.
+
+Although I don't think you need to.. If the remote is on a filesystem mount point, the assistant will just ignore it when it's not mounted. And will automatically detect when it gets mounted and sync with it, too. Same as USB drives are handled.
+"""]]
diff --git a/doc/design/assistant/blog/day_245__misc/comment_3_93003a0d0983efbdc046d7459be194b0._comment b/doc/design/assistant/blog/day_245__misc/comment_3_93003a0d0983efbdc046d7459be194b0._comment
new file mode 100644
index 0000000000..45a3b6adc8
--- /dev/null
+++ b/doc/design/assistant/blog/day_245__misc/comment_3_93003a0d0983efbdc046d7459be194b0._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="spwhitton"
+ ip="163.1.167.50"
+ subject="comment 3"
+ date="2013-04-24T17:09:50Z"
+ content="""
+It's a userspace fuse mount so the assistant won't know it's a mountpoint unless mounted.  Also I don't want the automatic sync since I like to keep this respository manual as a kind of backup.  Thanks for the info.
+"""]]
diff --git a/doc/design/assistant/blog/day_246__bug_treadmill.mdwn b/doc/design/assistant/blog/day_246__bug_treadmill.mdwn
new file mode 100644
index 0000000000..36ee0c6174
--- /dev/null
+++ b/doc/design/assistant/blog/day_246__bug_treadmill.mdwn
@@ -0,0 +1,18 @@
+There seem to be a steady state of enough bug reports coming in that I can
+work on them whenever I'm not working on anything else. As I did all day
+today.
+
+This doesn't bother me if the bug reports are of real bugs that I can
+reproduce and fix, but I'm spending a lot of time currently following up to
+messages and asking simple questions like "what version number" and "can I
+please see the whole log file". And just trying to guess what a vague
+problem report means and read people's minds to get to a definite bug
+with a test case that I can then fix.
+
+I've noticed the overall quality of bug reports nosedive over the past
+several months. My guess is this means that git-annex has found a less
+technical audience. I need to find something to do about this.
+
+With that whining out of the way ...
+I fixed a pretty ugly bug on FAT/Android today, and
+I am 100% caught up on messages right now!
diff --git a/doc/design/assistant/blog/day_246__bug_treadmill/comment_1_f76f653364fe2b97e85e8356c93b0fce._comment b/doc/design/assistant/blog/day_246__bug_treadmill/comment_1_f76f653364fe2b97e85e8356c93b0fce._comment
new file mode 100644
index 0000000000..33746ed4aa
--- /dev/null
+++ b/doc/design/assistant/blog/day_246__bug_treadmill/comment_1_f76f653364fe2b97e85e8356c93b0fce._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="https://www.google.com/accounts/o8/id?id=AItOawkQafKy7hNSEolLs6TvbgUnkklTctUY9LI"
+ nickname="Zellyn"
+ subject="Bugs..."
+ date="2013-04-23T23:30:47Z"
+ content="""
+Perhaps if you made a bug report form that asked for the version number (with helpful command to get it printed below), and had a checkbox for \"full log attached\", etc. you might be able to weed out low-value bugs...  Not that adding UI to force good bugs really works, but there might be a couple of low-hanging fruit before the diminishing returns kick in.
+"""]]
diff --git a/doc/design/assistant/blog/day_247__performance_tuning.mdwn b/doc/design/assistant/blog/day_247__performance_tuning.mdwn
new file mode 100644
index 0000000000..964710b8ec
--- /dev/null
+++ b/doc/design/assistant/blog/day_247__performance_tuning.mdwn
@@ -0,0 +1,16 @@
+Working on assistant's performance when it has to add a whole lot of files
+(10k to 100k). 
+
+Improved behavior in several ways, including fixing display
+of the alert in the webapp when the default inotify limit of 8192
+directories is exceeded.
+
+Created a new TList data type, a transactional DList. Much nicer
+implementation than the TChan based thing it was using to keep track of the
+changes, although it only improved runtime and memory usage a little bit.
+The way that this is internally storing a function in STM and modifying
+that function to add items to the list is way cool.
+
+Other tuning seems to have decreased the time it would take to import 100k
+files from somewhere in the range of a full day (too long to wait to see),
+to around 3.5 hours. I don't know if that's good, but it's certainly better.
diff --git a/doc/design/assistant/blog/day_248__Internet_Archive.mdwn b/doc/design/assistant/blog/day_248__Internet_Archive.mdwn
new file mode 100644
index 0000000000..f002939f13
--- /dev/null
+++ b/doc/design/assistant/blog/day_248__Internet_Archive.mdwn
@@ -0,0 +1,28 @@
+Very productive & long day today, spent adding a new feature to the 
+webapp: Internet Archive support!
+
+[[!img /assistant/iaitem.png]]
+
+git-annex already supported using archive.org via its S3 special remotes,
+so this is just a nice UI around that.
+
+How does it decide which files to publish on archive.org? Well,
+the item has a unique name, which is based on the description
+field. Any files located in a directory with that name will be uploaded
+to that item. (This is done via a new preferred content expression I added.)
+
+So, you can have one repository with multiple IA items attached, and
+sort files between them however you like.
+I plan to make a screencast eventually demoing that. 
+
+Another interesting use case, once the Android webapp is done, would be add
+a repository on the DCIM directory, set the archive.org repository to
+prefer all content, and *bam*, you have a phone or tablet that
+auto-publishes and archives every picture it takes.
+
+Another nice little feature added today is that whenever a file is uploaded
+to the Internet Archive, its public url is automatically recorded, same
+as if you'd ran `git annex addurl`. So any users who can clone your
+repository can download the files from archive.org, without needing any
+login or password info. This makes the Internet Archive a nice way to
+publish the large files associated with a public git repository.
diff --git a/doc/design/assistant/blog/day_249__quiet_day.mdwn b/doc/design/assistant/blog/day_249__quiet_day.mdwn
new file mode 100644
index 0000000000..28c1e6f3d7
--- /dev/null
+++ b/doc/design/assistant/blog/day_249__quiet_day.mdwn
@@ -0,0 +1,7 @@
+Quiet day. Only did minor things, like adding webapp UI for changing the
+directory used by Internet Archive remotes, and splitting out an
+`enableremote` command from `initremote`.
+
+My Android development environment is set up and ready to go on my Motorola
+Droid. The current Android build of git-annex fails to link at run time, so
+my work is cut out for me. Probably broke something while enabling XMPP?
diff --git a/doc/design/assistant/blog/day_24__airport_digressions.mdwn b/doc/design/assistant/blog/day_24__airport_digressions.mdwn
new file mode 100644
index 0000000000..6952969748
--- /dev/null
+++ b/doc/design/assistant/blog/day_24__airport_digressions.mdwn
@@ -0,0 +1,99 @@
+In a series of airport layovers all day. Since I woke up at 3:45 am, 
+didn't feel up to doing serious new work, so instead I worked through some
+OSX support backlog.
+
+git-annex will now use Haskell's SHA library if the `sha256sum`
+command is not available. That library is slow, but it's guaranteed to be
+available; git-annex already depended on it to calculate HMACs.
+
+Then I decided to see if it makes sense to use the SHA library
+when adding smaller files. At some point, its slower implementation should
+win over needing to fork and parse the output of `sha256sum`. This was 
+the first time I tried out Haskell's 
+[Criterion](http://hackage.haskell.org/package/criterion) benchmarker,
+and I built this simple benchmark in short order.
+
+[[!format haskell """
+import Data.Digest.Pure.SHA
+import Data.ByteString.Lazy as L
+import Criterion.Main
+import Common
+
+testfile :: FilePath
+testfile = "/tmp/bar" -- on ram disk
+
+main = defaultMain
+        [ bgroup "sha256"
+                [ bench "internal" $ whnfIO internal
+                , bench "external" $ whnfIO external
+                ]
+        ]
+
+internal :: IO String
+internal = showDigest . sha256 <$> L.readFile testfile
+
+external :: IO String
+external = pOpen ReadFromPipe "sha256sum" [testfile] $ \h ->
+        fst . separate (== ' ') <$> hGetLine h
+"""]]
+
+The nice thing about benchmarking in Airports is when you're running a
+benchmark locally, you don't want to do anything else with the computer,
+so can alternate people watching, spacing out, and analizing results.
+
+100 kb file:
+
+	benchmarking sha256/internal
+	mean: 15.64729 ms, lb 15.29590 ms, ub 16.10119 ms, ci 0.950
+	std dev: 2.032476 ms, lb 1.638016 ms, ub 2.527089 ms, ci 0.950
+
+	benchmarking sha256/external
+	mean: 8.217700 ms, lb 7.931324 ms, ub 8.568805 ms, ci 0.950
+	std dev: 1.614786 ms, lb 1.357791 ms, ub 2.009682 ms, ci 0.950
+
+75 kb file:
+
+	benchmarking sha256/internal
+	mean: 12.16099 ms, lb 11.89566 ms, ub 12.50317 ms, ci 0.950
+	std dev: 1.531108 ms, lb 1.232353 ms, ub 1.929141 ms, ci 0.950
+
+	benchmarking sha256/external
+	mean: 8.818731 ms, lb 8.425744 ms, ub 9.269550 ms, ci 0.950
+	std dev: 2.158530 ms, lb 1.916067 ms, ub 2.487242 ms, ci 0.950
+
+50 kb file:
+
+	benchmarking sha256/internal
+	mean: 7.699274 ms, lb 7.560254 ms, ub 7.876605 ms, ci 0.950
+	std dev: 801.5292 us, lb 655.3344 us, ub 990.4117 us, ci 0.950
+
+	benchmarking sha256/external
+	mean: 8.715779 ms, lb 8.330540 ms, ub 9.102232 ms, ci 0.950
+	std dev: 1.988089 ms, lb 1.821582 ms, ub 2.181676 ms, ci 0.950
+
+10 kb file:
+
+	benchmarking sha256/internal
+	mean: 1.586105 ms, lb 1.574512 ms, ub 1.604922 ms, ci 0.950
+	std dev: 74.07235 us, lb 51.71688 us, ub 108.1348 us, ci 0.950
+
+	benchmarking sha256/external
+	mean: 6.873742 ms, lb 6.582765 ms, ub 7.252911 ms, ci 0.950
+	std dev: 1.689662 ms, lb 1.346310 ms, ub 2.640399 ms, ci 0.950
+
+It's possible to get nice graphical reports out of Criterion, but 
+this is clear enough, so I stopped here. 50 kb seems a reasonable
+cutoff point.
+
+I also used this to benchmark the SHA256 in Haskell's Crypto package.
+Surprisingly, it's a *lot* slower than even the Pure.SHA code.
+On a 50 kb file:
+
+	benchmarking sha256/Crypto
+	collecting 100 samples, 1 iterations each, in estimated 6.073809 s
+	mean: 69.89037 ms, lb 69.15831 ms, ub 70.71845 ms, ci 0.950
+	std dev: 3.995397 ms, lb 3.435775 ms, ub 4.721952 ms, ci 0.950
+
+
+There's another Haskell library, [SHA2](http://hackage.haskell.org/package/SHA2),
+which I should try some time.
diff --git a/doc/design/assistant/blog/day_250__stymied.mdwn b/doc/design/assistant/blog/day_250__stymied.mdwn
new file mode 100644
index 0000000000..f9d50b48c9
--- /dev/null
+++ b/doc/design/assistant/blog/day_250__stymied.mdwn
@@ -0,0 +1,23 @@
+Turns out my old Droid has such an old version of Android (2.2) that
+it doesn't work with any binaries produced by my haskell cross-compiler. I
+think it's using a symbol not in its version of libc. Since upgrading this
+particular phone is a ugly process and the hardware is dying anyway (bad
+USB power connecter), I have given up on using it, and ordered an Android
+tablet instead to use for testing. Until that arrives, no Android. Bah.
+Wanted to get the Android app working in April.
+
+Instead, today I worked on making the webapp require less redundant
+password entry when adding multiple repositories using the same cloud
+provider. This is especially needed for the Internet Archive, since users
+will often want to have quite a few repositories, for different IA items.
+Implemented it for box.com, and Amazon too.
+
+Francois Marier has built an Ubuntu PPA for git-annex, containing the
+current version, with the assistant and webapp. It's targeted at Precise,
+but I think will probably also work with newer releases.
+
+
+Probably while I'm waiting to work on Android again, I will try to
+improve the situation with using a single XMPP account for multiple
+repositories. Spent a while today thinking through ways to improve the
+design, and have some ideas.
diff --git a/doc/design/assistant/blog/day_250__stymied/comment_1_330a10d447ccc3db03fcbfe571dbb404._comment b/doc/design/assistant/blog/day_250__stymied/comment_1_330a10d447ccc3db03fcbfe571dbb404._comment
new file mode 100644
index 0000000000..26c7d7c7ff
--- /dev/null
+++ b/doc/design/assistant/blog/day_250__stymied/comment_1_330a10d447ccc3db03fcbfe571dbb404._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="https://www.google.com/accounts/o8/id?id=AItOawnLgNOfkl8c6oQu9b42U0Pm_uC4n3Gkgiw"
+ nickname="Mark"
+ subject="Thanks!"
+ date="2013-04-29T01:24:55Z"
+ content="""
+I was not looking forward to figuring out how to come up with all of those haskell build deps on Precise, so thanks to François Marier for putting this together.  (Does Ubuntu/launchpad autobuild all of the architectures, or do they need to be contributed? I see that `armhf` isn't there at the moment...)
+"""]]
diff --git a/doc/design/assistant/blog/day_251__xmpp_improvements.mdwn b/doc/design/assistant/blog/day_251__xmpp_improvements.mdwn
new file mode 100644
index 0000000000..7007d7c0d0
--- /dev/null
+++ b/doc/design/assistant/blog/day_251__xmpp_improvements.mdwn
@@ -0,0 +1,34 @@
+Took 2 days in a row off, because I noticed I have forgotten to do that
+since February, or possibly earlier, not counting trips. Whoops!
+
+Also, I was feeling overwhelmed with the complexity of fixing XMPP to not
+be buggy when there are multiple separate repos using the same XMPP
+account. Let my subconscious work on that, and last night it served up the
+solution, in detail. Built it today.
+
+It's only a partial solution, really. If you want to use the same XMPP
+account for multiple separate repositories, you cannot use the "Share with
+your other devices" option to pair your devices. That's because XMPP
+pairing assumes all your devices are using the same XMPP account, in order
+to avoid needing to confirm on every device each time you add a new device.
+The UI is clear about that, and it avoids complexity, so I'm ok with that.
+
+But, if you want to instead use "Share with a friend", you now can use the
+same XMPP account for as many separate repositories as you like. The
+assistant now ignores pushes from repositories it doesn't know about.
+Before, it would merge them all together without warning.
+
+----
+
+While I was testing that, I think I found out the real reason why XMPP
+pushes have seemed a little unreliable. It turns out to not be an XMPP
+issue at all! Instead, the merger was simply not always
+noticing when `git receive-pack` updated a ref, and not merging it into
+master. That was easily fixed.
+
+----
+
+Adam Spiers has been getting a `.gitignore` query interface suitable for
+the assistant to use into `git`, and he tells me it's landed in `next`.
+I should soon check that out and get the assistant using it. But first,
+Android app!
diff --git a/doc/design/assistant/blog/day_252__release_day.mdwn b/doc/design/assistant/blog/day_252__release_day.mdwn
new file mode 100644
index 0000000000..70c99760a8
--- /dev/null
+++ b/doc/design/assistant/blog/day_252__release_day.mdwn
@@ -0,0 +1,6 @@
+Pushed out a release today. Looking back over April, I'm happy with it as a
+bug fix and stabilization month. Wish I'd finished the Android app in April,
+but let's see what happens tomorrow.
+
+Recorded part of a screencast on using Archive.org, but `recordmydesktop`
+lost the second part. Grr. Will work on that later.
diff --git a/doc/design/assistant/blog/day_253__OMG.mdwn b/doc/design/assistant/blog/day_253__OMG.mdwn
new file mode 100644
index 0000000000..b1cc89fffd
--- /dev/null
+++ b/doc/design/assistant/blog/day_253__OMG.mdwn
@@ -0,0 +1,22 @@
+[[!img /android/webapp.png alt="git-annex webapp on Android"]]
+
+I fixed what I thought was keeping the webapp from working on Android, but
+then it started segfaulting every time it was started. Eventually I
+determined this segfault happened whenever haskell code called
+`getaddrinfo`. I don't know why. This is particularly weird since I had
+a demo web server that used `getaddrinfo` working way back in
+[[day_201__real_Android_wrapup]]. Anyway, I worked around it by not using
+`getaddrinfo` on Android.
+
+Then I spent 3 hours stuck, because the webapp seemed to run, but
+nothing could connect to the port it was on. Was it a firewall? Was
+the Haskell threaded runtime's use of `accept()` broken? I went all the way
+down to the raw system calls, and back, only to finally notice I had `netstat`
+available on my Android. Which showed it was not listening to the port I
+thought it was!
+
+Seems that `ntohs` and `htons` are broken somehow. To get the
+screenshot, I fixed up the port manually. Have a build running that
+should work around the issue.
+
+Anyway, the webapp works on Android!
diff --git a/doc/design/assistant/blog/day_253__OMG/comment_1_bbdc61092771163e65a90a4755a807d8._comment b/doc/design/assistant/blog/day_253__OMG/comment_1_bbdc61092771163e65a90a4755a807d8._comment
new file mode 100644
index 0000000000..0fa06b32ff
--- /dev/null
+++ b/doc/design/assistant/blog/day_253__OMG/comment_1_bbdc61092771163e65a90a4755a807d8._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="https://www.google.com/accounts/o8/id?id=AItOawnxp2XU8gIribhhGhGuYtU6eMMwHv5gUGI"
+ nickname="Amitai"
+ subject="Woohoo!"
+ date="2013-05-03T03:13:25Z"
+ content="""
+(See above)
+"""]]
diff --git a/doc/design/assistant/blog/day_254__Android_app_polishing.mdwn b/doc/design/assistant/blog/day_254__Android_app_polishing.mdwn
new file mode 100644
index 0000000000..cbb6f7f15c
--- /dev/null
+++ b/doc/design/assistant/blog/day_254__Android_app_polishing.mdwn
@@ -0,0 +1,35 @@
+There's a new page [[/Android]] that documents using git-annex on Android
+in detail.
+
+The Android app now opens the webapp when a terminal window is opened.
+This is good enough for trying it out easily, but far from ideal.
+
+Fixed an EvilSplicer bug that corrupted newlines in 
+the static files served by the webapp. Now the icons in the webapp
+display properly, and the javascript works.
+
+Made the startup screen default to `/sdcard/annex` for the repository
+location, and also have a button to set up a camera repository. The camera
+repository is put in the "source" preferred content group, so it will only
+hang onto photos and videos until they're uploaded off the Android device.
+
+Quite a lot of other small fixes on Android. At this point I've tested the
+following works:
+
+* Starting webapp.
+* Making a repository, adding files.
+* All the basic webapp UI.
+
+However, I was not able to add any remote repository using only the webapp,
+due to some more problems with the network stack.
+
+* Jabber and Webdav don't quite work ("getProtocolByname: does not exist (no
+  such protocol name: tcp)").
+* SSH server fails.
+  ("Network/Socket/Types.hsc:(881,3)-(897,61): Non-exhaustive patterns in case")
+  I suspect it will work if I disable the DNS expansion code.
+
+So, that's the next thing that needs to be tackled.
+
+If you'd like to play with it in its current state, I've updated the
+Android builds to incorporate all my work so far.
diff --git a/doc/design/assistant/blog/day_254__Android_app_polishing/comment_1_37f4ff5227566ce4b3fa69fc32568841._comment b/doc/design/assistant/blog/day_254__Android_app_polishing/comment_1_37f4ff5227566ce4b3fa69fc32568841._comment
new file mode 100644
index 0000000000..7ba9f94205
--- /dev/null
+++ b/doc/design/assistant/blog/day_254__Android_app_polishing/comment_1_37f4ff5227566ce4b3fa69fc32568841._comment
@@ -0,0 +1,14 @@
+[[!comment format=mdwn
+ username="https://www.google.com/accounts/o8/id?id=AItOawkGCmVc5qIJaQQgG82Hc5zzBdAVdhe2JEM"
+ nickname="Bruno"
+ subject="comment 1"
+ date="2013-05-04T05:30:26Z"
+ content="""
+> The camera repository is put in the \"source\" preferred content group, so it will only hang onto photos and videos until they're uploaded off the Android device.
+
+This is very nice but I'm wondering if I take a picture that I want to share using an Android app like Facebook, would the picture get deleted before I could share it?
+
+Would there be a way to access old pictures? For example, to show them to people you meet.
+
+
+"""]]
diff --git a/doc/design/assistant/blog/day_254__Android_app_polishing/comment_2_58bbb105bdbb72bba85c3622195f43b9._comment b/doc/design/assistant/blog/day_254__Android_app_polishing/comment_2_58bbb105bdbb72bba85c3622195f43b9._comment
new file mode 100644
index 0000000000..7311b72e0f
--- /dev/null
+++ b/doc/design/assistant/blog/day_254__Android_app_polishing/comment_2_58bbb105bdbb72bba85c3622195f43b9._comment
@@ -0,0 +1,12 @@
+[[!comment format=mdwn
+ username="http://joeyh.name/"
+ nickname="joey"
+ subject="comment 2"
+ date="2013-05-06T15:08:19Z"
+ content="""
+As it stands, using the \"source\" repository type will tend to remove the picture from that repository pretty quickly, assuming a fast network connection. I think if you want to do things on your phone with pictures you take, it's best to use \"client\" instead. Perhaps with an archive repository set up, so you can move stuff to an \"archive\" folder when done with it to get it removed from the phone.
+
+Whether \"source\" is a good default in this case remains to be seen...
+
+It might also be possible to change \"source\" in some way to make it more useful. For example, if it held onto new files for a day after they were created, you'd have plenty of time to use them before they expire off your phone. I have posted some thoughts about this to [[partial_content]].
+"""]]
diff --git a/doc/design/assistant/blog/day_255__Debian_release_day.mdwn b/doc/design/assistant/blog/day_255__Debian_release_day.mdwn
new file mode 100644
index 0000000000..9c41ab2b2d
--- /dev/null
+++ b/doc/design/assistant/blog/day_255__Debian_release_day.mdwn
@@ -0,0 +1,26 @@
+Created a backport of the latest git-annex release for Debian 7.0 wheezy.
+Needed to backport a dozen haskell dependencies, but not too bad.
+This will be available in the backports repository once Debian
+starts accepting new packages again. I plan to keep the backport up-to-date
+as I make new releases.
+
+The cheap Android tablet I bought to do this last Android push with came
+pre-rooted from the factory. This may be why I have not seen this bug:
+[[bugs/Android_app_permission_denial_on_startup]]. If you have Android
+4.2.2 or a similar version, your testing would be helpful for me to know if
+this is a widespread problem. I have an idea about a way to work around the
+problem, but it involves writing Java code, and probably polling a file,
+ugh.
+
+Got S3 support to build for Android. Probably fails to work 
+due to the same network stack problems affecting WebDAV and Jabber.
+
+Got removable media mount detection working on Android. Bionic has an
+amusing stub for `getmntent` that prints out "FIX ME! implement
+getmntent()". But, `/proc/mounts` is there, so I just parse it.
+Also enabled the app's `WRITE_MEDIA_STORAGE` permission to allow
+access to removable media. However, this didn't seem to do anything. :(
+
+Several fixes to make the Android webapp be able to set up repositories on
+remote ssh servers. However, it fails at the last hurdle with what
+looks like a `git-annex-shell` communication problem. Almost there..
diff --git a/doc/design/assistant/blog/day_256__8bit.mdwn b/doc/design/assistant/blog/day_256__8bit.mdwn
new file mode 100644
index 0000000000..1155fd2f8c
--- /dev/null
+++ b/doc/design/assistant/blog/day_256__8bit.mdwn
@@ -0,0 +1,27 @@
+This seems a very auspicious day to have finally gotten the Android app
+doing something useful! I've fixed the last bugs with using it to set up a
+remote ssh server, which is all I need to make my Android tablet
+sync photos I take with a repository on my laptop.
+
+[[!img /android/DCIM.png alt="git-annex webapp running on Android"]]
+
+I set this up entirely in the GUI, except for needing to switch to the
+terminal twice to enter my laptop's password.
+
+How fast is it? Even several minute long videos transfer before I can
+switch from the camera app to the webapp. To get this screenshot with it in
+the process of syncing, I had to take a dozen pictures in a minute. Nice
+problem to have. ;)
+
+Have fun trying this out for yourself after tonight's autobuilds. But a
+warning: One of the bugs I fixed today had to be fixed in `git-annex-shell`,
+as run on the ssh server that the Android connects to. So the Android app
+will only work with ssh servers running a new enough version of git-annex.
+
+----
+
+Worked on geting git-annex into Debian testing, which is needed before
+the wheezy backport can go in. Think I've worked around most of the issues
+that were keeping it from building on various architectures.
+
+Caught up on some bug reports and fixed some of them.
diff --git a/doc/design/assistant/blog/day_256__8bit/comment_1_f9b50263e3997d4c5b9836a2e0a346d7._comment b/doc/design/assistant/blog/day_256__8bit/comment_1_f9b50263e3997d4c5b9836a2e0a346d7._comment
new file mode 100644
index 0000000000..b7c54ed58f
--- /dev/null
+++ b/doc/design/assistant/blog/day_256__8bit/comment_1_f9b50263e3997d4c5b9836a2e0a346d7._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="https://www.google.com/accounts/o8/id?id=AItOawm8wY171R5c4u_jPmB6LU6n6Px2xePM4sE"
+ nickname="Efraim"
+ subject="comment 1"
+ date="2013-05-07T06:20:35Z"
+ content="""
+that's amazing.  I saw in one of the conference videos that you said you didn't use dropbox, and I wanted to let you know (in case you were curious) that this is one of the features that they have on their android app, that pictures and videos are uploaded to one's dropbox.  Two questions, is there a toggle or option somewhere to upload over wifi only? and did you set up the tablet's camera folder as a source directory?
+"""]]
diff --git a/doc/design/assistant/blog/day_257__rainy_day.mdwn b/doc/design/assistant/blog/day_257__rainy_day.mdwn
new file mode 100644
index 0000000000..871e5b8b56
--- /dev/null
+++ b/doc/design/assistant/blog/day_257__rainy_day.mdwn
@@ -0,0 +1,6 @@
+Put in a fix for `getprotobyname` apparently not returning anything for
+"tcp" on Android. This might fix all the special remotes there, but I don't
+know yet, because I have to rebuild a lot of Haskell libraries to try it.
+
+So, I spent most of today writing a script to build all the Haskell
+libraries for Android from scratch, with all my patches.
diff --git a/doc/design/assistant/blog/day_258__beginning_of_the_end.mdwn b/doc/design/assistant/blog/day_258__beginning_of_the_end.mdwn
new file mode 100644
index 0000000000..e2e544399f
--- /dev/null
+++ b/doc/design/assistant/blog/day_258__beginning_of_the_end.mdwn
@@ -0,0 +1,24 @@
+Fixed a nasty bug that affects at least some FreeBSD systems. It misparsed
+the output of `sha256`, and thought every file had a SHA256 of "SHA256".
+Added multiple layers of protection against checksum programs not having
+the expected output format.
+
+Lots more building and rebuilding today of Android libs than I wanted to do.
+Finally have a completly clean build, which might be able to open TCP
+connections. Will test tomorrow.
+
+In the meantime, I fired up the evil twin of my development laptop.
+It's identical, except it runs Windows.
+
+I installed the Haskell Platform for Windows on it, and removed
+some of the bloatware to free up disk space and memory for development.
+While a rather disgusting experience, I certainly have a usable Haskell
+development environment on this OS a lot faster than I did on Android!
+Cabal is happily installing some stuff, and other stuff wants me to install
+Cygwin.
+
+So, the clock on my month of working on a Windows port starts now. Since
+I've already done rather a lot of ground work that was necessary for a
+Windows port (direct mode, crippled filesystem support), and for general
+sanity and to keep everything else from screeching to a halt, I plan to
+only spend half my time messing with Windows over the next 30 days.
diff --git a/doc/design/assistant/blog/day_259__Android_dominos_toppling.mdwn b/doc/design/assistant/blog/day_259__Android_dominos_toppling.mdwn
new file mode 100644
index 0000000000..3f96b1fb0b
--- /dev/null
+++ b/doc/design/assistant/blog/day_259__Android_dominos_toppling.mdwn
@@ -0,0 +1,15 @@
+It all came together for Android today. Went from a sort of working app
+to a fully working app!
+
+* rsync.net works.
+* Box.com appears to work -- at least it's failing with the same
+  timeout I get on my linux box here behind the firewall of dialup doom.
+* XMPP is working too!
+
+These all needed various little fixes. Like loading TLS certificates from
+where they're stored on Android, and ensuring that totally crazy file
+permissions from Android (----rwxr-x for files?!) don't leak out into rsync
+repositories. Mostly though, it all just fell into place today.
+Wonderful..
+
+The Android autobuild is updated with all of today's work, so try it out.
diff --git a/doc/design/assistant/blog/day_259__Android_dominos_toppling/comment_1_0b4a6e4893b0157e4768b46468dbbb87._comment b/doc/design/assistant/blog/day_259__Android_dominos_toppling/comment_1_0b4a6e4893b0157e4768b46468dbbb87._comment
new file mode 100644
index 0000000000..9e1f666aa1
--- /dev/null
+++ b/doc/design/assistant/blog/day_259__Android_dominos_toppling/comment_1_0b4a6e4893b0157e4768b46468dbbb87._comment
@@ -0,0 +1,10 @@
+[[!comment format=mdwn
+ username="https://www.google.com/accounts/o8/id?id=AItOawkGCmVc5qIJaQQgG82Hc5zzBdAVdhe2JEM"
+ nickname="Bruno"
+ subject="comment 1"
+ date="2013-05-10T23:52:43Z"
+ content="""
+I have a 'Connection timed out' when I try to set-up my Gtalk account on my Nexus 4 (4.2.2). I tested it at home without a web proxy.
+
+Here's a screen capture: [http://i.imgur.com/RBcNjSv.png](http://i.imgur.com/RBcNjSv.png).
+"""]]
diff --git a/doc/design/assistant/blog/day_259__Android_dominos_toppling/comment_2_1ebc5aff5d217e1392cb7c8bb6c5156b._comment b/doc/design/assistant/blog/day_259__Android_dominos_toppling/comment_2_1ebc5aff5d217e1392cb7c8bb6c5156b._comment
new file mode 100644
index 0000000000..cfe1347bdb
--- /dev/null
+++ b/doc/design/assistant/blog/day_259__Android_dominos_toppling/comment_2_1ebc5aff5d217e1392cb7c8bb6c5156b._comment
@@ -0,0 +1,14 @@
+[[!comment format=mdwn
+ username="https://www.google.com/accounts/o8/id?id=AItOawmmLfBDEDFfEOba8Ra46nsnTmoNHFsLObo"
+ nickname="Brian"
+ subject="I also see the connection timed out"
+ date="2013-05-15T00:19:34Z"
+ content="""
+Same Android device and same behavior as the above comment. While the connection attempt was being made, I switched to the terminal window and saw a connection to 173.194.46.86:5222 in the SYN_SENT state (and that connection wasn't there prior to the jabber configuration attempt).
+
+Even from my desktop, manual connection attempts to 173.194.46.86:5222 timed out.
+
+Then I ran the same jabber configuration from my desktop and it worked fin. Using netstat, I could see it connected to 74.125.133.125:5222.
+
+So is the Android version using the wrong address?
+"""]]
diff --git a/doc/design/assistant/blog/day_259__Android_dominos_toppling/comment_3_eed7792f6142f3fc74d3c384bb16559b._comment b/doc/design/assistant/blog/day_259__Android_dominos_toppling/comment_3_eed7792f6142f3fc74d3c384bb16559b._comment
new file mode 100644
index 0000000000..245ae7a06c
--- /dev/null
+++ b/doc/design/assistant/blog/day_259__Android_dominos_toppling/comment_3_eed7792f6142f3fc74d3c384bb16559b._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="http://joeyh.name/"
+ nickname="joey"
+ subject="comment 3"
+ date="2013-05-19T20:56:25Z"
+ content="""
+For the record, there was a bug with XMPP and SRV records. This got fixed in time for the last release.
+"""]]
diff --git a/doc/design/assistant/blog/day_25__transfer_queueing.mdwn b/doc/design/assistant/blog/day_25__transfer_queueing.mdwn
new file mode 100644
index 0000000000..b07e4592e9
--- /dev/null
+++ b/doc/design/assistant/blog/day_25__transfer_queueing.mdwn
@@ -0,0 +1,41 @@
+So as not to bury the lead, I've been hard at work on my first day in
+Nicaragua, and ** the git-annex assistant fully syncs files (including
+their contents) between remotes now !! **
+
+Details follow..
+
+Made the committer thread queue Upload Transfers when new files
+are added to the annex. Currently it tries to transfer the new content
+to *every* remote; this inefficiency needs to be addressed later.
+
+Made the watcher thread queue Download Transfers when new symlinks
+appear that point to content we don't have. Typically, that will happen
+after an automatic merge from a remote. This needs to be improved as it
+currently adds Transfers from every remote, not just those that have the
+content.
+
+This was the second place that needed an ordered list of remotes
+to talk to. So I cached such a list in the DaemonStatus state info.
+This will also be handy later on, when the webapp is used to add new
+remotes, so the assistant can know about them immediately.
+
+Added YAT (Yet Another Thread), number 15 or so, the transferrer thread
+that waits for transfers to be queued and runs them. Currently a naive
+implementation, it runs one transfer at a time, and does not do anything
+to recover when a transfer fails.
+
+Actually transferring content requires YAT, so that the transfer
+action can run in a copy of the Annex monad, without blocking
+all the assistant's other threads from entering that monad while a transfer
+is running. This is also necessary to allow multiple concurrent transfers
+to run in the future.
+
+This is a very tricky piece of code, because that thread will modify the
+git-annex branch, and its parent thread has to invalidate its cache in
+order to see any changes the child thread made. Hopefully that's the extent
+of the complication of doing this. The only reason this was possible at all
+is that git-annex already support multiple concurrent processes running
+and all making independent changes to the git-annex branch, etc.
+
+After all my groundwork this week, file content transferring is now
+fully working!
diff --git a/doc/design/assistant/blog/day_25__transfer_queueing/comment_1_59fd4f1ffe96c412f613dc86276e7dbd._comment b/doc/design/assistant/blog/day_25__transfer_queueing/comment_1_59fd4f1ffe96c412f613dc86276e7dbd._comment
new file mode 100644
index 0000000000..0a5b6c0991
--- /dev/null
+++ b/doc/design/assistant/blog/day_25__transfer_queueing/comment_1_59fd4f1ffe96c412f613dc86276e7dbd._comment
@@ -0,0 +1,10 @@
+[[!comment format=mdwn
+ username="https://www.google.com/accounts/o8/id?id=AItOawldKnauegZulM7X6JoHJs7Gd5PnDjcgx-E"
+ nickname="Matt"
+ subject="Source code"
+ date="2012-07-06T00:12:15Z"
+ content="""
+Hi Joey,
+
+Is the source code for git-annex assistant available somewhere?
+"""]]
diff --git a/doc/design/assistant/blog/day_25__transfer_queueing/comment_2_93bf768a67117e873af5732ecf08dc78._comment b/doc/design/assistant/blog/day_25__transfer_queueing/comment_2_93bf768a67117e873af5732ecf08dc78._comment
new file mode 100644
index 0000000000..6c0ca0781d
--- /dev/null
+++ b/doc/design/assistant/blog/day_25__transfer_queueing/comment_2_93bf768a67117e873af5732ecf08dc78._comment
@@ -0,0 +1,7 @@
+[[!comment format=mdwn
+ username="http://joeyh.name/"
+ subject="comment 2"
+ date="2012-07-06T00:21:43Z"
+ content="""
+It's in the `assistant` branch of git://git-annex.branchable.com/
+"""]]
diff --git a/doc/design/assistant/blog/day_260__Windows_dev_environment.mdwn b/doc/design/assistant/blog/day_260__Windows_dev_environment.mdwn
new file mode 100644
index 0000000000..4fa2ccf5a6
--- /dev/null
+++ b/doc/design/assistant/blog/day_260__Windows_dev_environment.mdwn
@@ -0,0 +1,46 @@
+Set up my Windows development environment. For future reference, I've
+installed:
+
+* haskell platform for windows
+* cygwin
+* gcc and a full C toolchain in cygwin
+* git from upstream (probably git-annex will use this)
+* git in cygwin (the other git was not visible inside cygwin)
+* vim in cygwin
+* vim from upstream, as the cygwin vim is not very pleasant to use
+* openssh in cygwin (seems to be missing a ssh server)
+* rsync in cygwin
+* Everything that `cabal install git-annex` is able to install successfully.  
+  This includes all the libraries needed to build regular git-annex,
+  but not the webapp. Good start though.
+
+Result basically feels like a linux system that can't decide which way
+slashes in paths go. :P I've never used Cygwin before (I last used a
+Windows machine in 2003 for that matter), and it's a fairly impressive hack.
+
+----
+
+Fixed up git-annex's configure program to run on Windows (or, at least, in
+Cygwin), and have started getting git-annex to build.
+
+For now, I'm mostly stubbing out functions that use unix stuff. Gotten the
+first 44 of 300 source files to build this way.
+
+Once I get it to build, if only with stubs, I'll have a good
+idea about all the things I need to find Windows equivilants of.
+Hopefully most of it will be provided by
+.
+
+----
+
+So that's the plan. There is a possible shortcut, rather than doing a full
+port. It seems like it would probably not be too hard to rebuild ghc inside
+Cygwin, and the resulting ghc would probably have a full POSIX emulation
+layer going through cygwin. From ghc's documentation, it looks like that's
+how ghc used to be built at some point in the past, so it would probably
+not be too hard to build it that way. With such a cygwin ghc, git-annex
+would probably build with little or no changes. However, it would be a
+git-annex targeting Cygwin, and not really a native Windows port. So
+it'd see Cygwin's emulated POSIX filesystem paths, etc. That
+seems probably not ideal for most windows users.. but if I get really stuck
+I may go back and try this method.
diff --git a/doc/design/assistant/blog/day_261__Windows_first_stage_complete.mdwn b/doc/design/assistant/blog/day_261__Windows_first_stage_complete.mdwn
new file mode 100644
index 0000000000..4d7c9582e1
--- /dev/null
+++ b/doc/design/assistant/blog/day_261__Windows_first_stage_complete.mdwn
@@ -0,0 +1,29 @@
+After working on it all day, git-annex now builds on Windows!
+
+Even better, `git annex init` works. So does `git annex status`, and
+probably more. Not `git annex add` yet, so I wasn't able to try much more.
+
+I didn't have to add many stubs today, either. Many of the missing Windows
+features were only used in code paths that made git-annex faster, but I
+could fall back to a slower code path on Windows.
+
+The things that are most problematic so far:
+
+* POSIX file locking. This is used in git-annex in several places to 
+  make it safe when multiple git-annex processes are running. I put in
+  really horrible dotfile type locking in the Windows code paths, but I
+  don't trust it at all of course.
+* There is, apparently, no way to set an environment variable in Windows
+  from Haskell. It is only possible to set up a new process' environment
+  before starting it. Luckily most of the really crucial environment
+  variable stuff in git-annex is of this latter sort, but there were
+  a few places I had to stub out code that tries to manipulate git-annex's
+  own environment.
+
+The `windows` branch has a diff of 2089 lines. It add 88 ifdefs to the code
+base. Only 12 functions are stubbed out on Windows. This could be so much
+worse.
+
+Next step: Get the test suite to build. Currently ifdefed out because it
+uses some stuff like `setEnv` and `changeWorkingDirectory` that I don't know
+how to do in Windows yet.
diff --git a/doc/design/assistant/blog/day_262__DOS_path_separators.mdwn b/doc/design/assistant/blog/day_262__DOS_path_separators.mdwn
new file mode 100644
index 0000000000..a7f6c07cca
--- /dev/null
+++ b/doc/design/assistant/blog/day_262__DOS_path_separators.mdwn
@@ -0,0 +1,14 @@
+It's remarkable that a bad decision made in 1982 can cause me to waste an
+entire day in 2013. Yes, `/` vs `\` fun time. Even though I long ago
+converted git-annex to use the haskell `` operator wherever it builds
+up paths (which transparently handles either type of separator), I still
+spent most of today dealing with it. Including some libraries I use that
+get it wrong. Adding to the fun is that git uses `/` internally, even on
+Windows, so Windows separated paths have to be converted when being fed
+into git.
+
+Anyway, `git annex add` now works on Windows. So does `git annex find`,
+and `git annex whereis`, and probably most query stuff.
+
+Today was very un-fun and left me with a splitting headache, so I will
+certainly *not* be working on the Windows port tomorrow.
diff --git a/doc/design/assistant/blog/day_262__DOS_path_separators/comment_1_45ecae90b22e31202c21083980d6b567._comment b/doc/design/assistant/blog/day_262__DOS_path_separators/comment_1_45ecae90b22e31202c21083980d6b567._comment
new file mode 100644
index 0000000000..a024103a77
--- /dev/null
+++ b/doc/design/assistant/blog/day_262__DOS_path_separators/comment_1_45ecae90b22e31202c21083980d6b567._comment
@@ -0,0 +1,10 @@
+[[!comment format=mdwn
+ username="http://jamesjustjames.wordpress.com/"
+ nickname="purpleidea"
+ subject="Isn't windows deprecated?"
+ date="2013-05-13T05:51:20Z"
+ content="""
+I feel bad for you, however I respect you for keeping your promise to try and hack on Windows. I had to port over some code than ran beautifully on GNU/Linux and it was more trouble than it was worth. In the end it was never used :P
+
+Cheers!
+"""]]
diff --git a/doc/design/assistant/blog/day_263_catching_up.mdwn b/doc/design/assistant/blog/day_263_catching_up.mdwn
new file mode 100644
index 0000000000..b8eadbc965
--- /dev/null
+++ b/doc/design/assistant/blog/day_263_catching_up.mdwn
@@ -0,0 +1,11 @@
+Spent some time today to get caught up on bug reports and website traffic.
+Fixed a few things.
+
+Did end up working on Windows for a while too. I got `git annex drop`
+working. But nothing that moves content quite works yet..
+
+I've run into a stumbling block with `rsync`. It thinks that
+`C:\repo` is a path on a ssh server named "C". Seems I will need to translate
+native windows paths to unix-style paths when running rsync.
+
+[[!meta date="13 May 2013"]]
diff --git a/doc/design/assistant/blog/day_263_catching_up/comment_1_9023da0573dfc81644d68128adb331a7._comment b/doc/design/assistant/blog/day_263_catching_up/comment_1_9023da0573dfc81644d68128adb331a7._comment
new file mode 100644
index 0000000000..83e7523ff9
--- /dev/null
+++ b/doc/design/assistant/blog/day_263_catching_up/comment_1_9023da0573dfc81644d68128adb331a7._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="https://www.google.com/accounts/o8/id?id=AItOawkHscTHMCNvjJ6nLI1VpsBrJFI5FTwhUT4"
+ nickname="David"
+ subject="Just use Unix Paths like git does?"
+ date="2013-05-15T20:43:34Z"
+ content="""
+Can't you just use unix-style paths for your internal data structures, and wrap/patch the filesystem I/O code to convert between the two?
+"""]]
diff --git a/doc/design/assistant/blog/day_264__Windows_second_stage_complete.mdwn b/doc/design/assistant/blog/day_264__Windows_second_stage_complete.mdwn
new file mode 100644
index 0000000000..f34e8b15ed
--- /dev/null
+++ b/doc/design/assistant/blog/day_264__Windows_second_stage_complete.mdwn
@@ -0,0 +1,21 @@
+The Windows port can now do everything in the [[walkthrough]]. It can use
+both local and remote git repositories. Some special remotes work
+(directory at least; probably rsync; likely any other special remote that
+can have its dependencies built). Missing features include most special
+remotes, gpg encryption, and of course, the assistant.
+
+Also built a NullSoft installer for git-annex today. This was made very
+easy when I found the Haskell ncis library, which provides a DSL embedding
+the language used to write NullSoft installers into Haskell. So I didn't
+need to learn a new language, yay! And could pull in all my helpful
+Haskell utility libraries in the program that builds the installer.
+
+The only tricky part was: How to get git-annex onto PATH? The standard way
+to do this seems to be to use a multiple-hundred line include file. Of
+course, that file does not have any declared license..  Instead of that,
+I used a hack. The git installer for Windows adds itself to PATH, and is
+a pre-requisite for git-annex. So the git-annex installer just installs
+it into the same directory as git.
+
+So.. I'll be including this first stage Windows port, with installer in
+the next release. Anyone want to run a Windows autobuilder?
diff --git a/doc/design/assistant/blog/day_264__Windows_second_stage_complete/comment_1_42a7502d6ece75520eb59a76fdb1e2f0._comment b/doc/design/assistant/blog/day_264__Windows_second_stage_complete/comment_1_42a7502d6ece75520eb59a76fdb1e2f0._comment
new file mode 100644
index 0000000000..e7846d5fb0
--- /dev/null
+++ b/doc/design/assistant/blog/day_264__Windows_second_stage_complete/comment_1_42a7502d6ece75520eb59a76fdb1e2f0._comment
@@ -0,0 +1,9 @@
+[[!comment format=mdwn
+ username="https://www.google.com/accounts/o8/id?id=AItOawlJ2utMQgMEYAOs3Dfc6eZRyUzt4acNXUU"
+ nickname="David"
+ subject="git on windows"
+ date="2013-05-15T04:26:48Z"
+ content="""
+Adding git to the PATH variable is an optional step in the msysgit installer, fyi.
+
+"""]]
diff --git a/doc/design/assistant/blog/day_264__Windows_second_stage_complete/comment_2_f2b11322ac87e2a36cddc035b2c3c1ea._comment b/doc/design/assistant/blog/day_264__Windows_second_stage_complete/comment_2_f2b11322ac87e2a36cddc035b2c3c1ea._comment
new file mode 100644
index 0000000000..1ff6bd542a
--- /dev/null
+++ b/doc/design/assistant/blog/day_264__Windows_second_stage_complete/comment_2_f2b11322ac87e2a36cddc035b2c3c1ea._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="https://www.google.com/accounts/o8/id?id=AItOawkSq2FDpK2n66QRUxtqqdbyDuwgbQmUWus"
+ nickname="Jimmy"
+ subject="comment 2"
+ date="2013-05-15T22:30:05Z"
+ content="""
+I was thinking maybe migrating away from gitbuilder to using something like buildbot as it seems to be a better fit for this project than gitbuilder.
+"""]]
diff --git a/doc/design/assistant/blog/day_264__Windows_second_stage_complete/comment_3_ea6ee05acb946fc7e8d95e62647cfa2a._comment b/doc/design/assistant/blog/day_264__Windows_second_stage_complete/comment_3_ea6ee05acb946fc7e8d95e62647cfa2a._comment
new file mode 100644
index 0000000000..39d2a837d0
--- /dev/null
+++ b/doc/design/assistant/blog/day_264__Windows_second_stage_complete/comment_3_ea6ee05acb946fc7e8d95e62647cfa2a._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="http://joeyh.name/"
+ nickname="joey"
+ subject="comment 3"
+ date="2013-05-15T22:56:27Z"
+ content="""
+@Jimmy, would be fine with me if it works better
+"""]]
diff --git a/doc/design/assistant/blog/day_264__Windows_second_stage_complete/comment_4_9ce106baf28b7f75f7f6febd7bfcea70._comment b/doc/design/assistant/blog/day_264__Windows_second_stage_complete/comment_4_9ce106baf28b7f75f7f6febd7bfcea70._comment
new file mode 100644
index 0000000000..d3ac0df12f
--- /dev/null
+++ b/doc/design/assistant/blog/day_264__Windows_second_stage_complete/comment_4_9ce106baf28b7f75f7f6febd7bfcea70._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="https://www.google.com/accounts/o8/id?id=AItOawkSq2FDpK2n66QRUxtqqdbyDuwgbQmUWus"
+ nickname="Jimmy"
+ subject="comment 4"
+ date="2013-05-16T17:31:40Z"
+ content="""
+joey, I'll poke at it on the weekend, been meaning to look for an excuse to learn a bit more about buildbot.
+"""]]
diff --git a/doc/design/assistant/blog/day_265__correctness.mdwn b/doc/design/assistant/blog/day_265__correctness.mdwn
new file mode 100644
index 0000000000..a4415b4446
--- /dev/null
+++ b/doc/design/assistant/blog/day_265__correctness.mdwn
@@ -0,0 +1,23 @@
+Laid some groundwork for porting the test suite to Windows, and getting it
+working in direct mode. That's not complete, but even starting to run the
+test suite in direct mode and looking at all the failures (many of them
+benign, like files not being symlinks) highlighted something
+I have been meaning to look into for quite a while: Why, in direct mode,
+`git-annex` doesn't operate on data staged in the index, but requires you
+commit changes to files before it'll see them. That's an annoying
+difference between direct and indirect modes.
+
+It turned out that I introduced this behavior back on
+[[January 5th|day_163__free_features]], working around a nasty
+bug I didn't understand. Bad Joey, should have root caused the bug at the
+time! But the commit says I was stuck on it for hours, and it was
+presenting as if it was a bug in `git cat-file` itself, so ok. Anyway,
+I quickly got to the bottom of it today, fixed the underlying bug (which 
+was in git-annex, not git itself), and got rid of the workaround and its
+undesired consequences. Much better.
+
+The test suite is turning up some other minor problems with direct mode.
+Should have found time to port it earlier.
+
+Also, may have fixed the issue that was preventing GTalk from working on
+Android. (Missing DNS library so it didn't do SRV lookups right.)
diff --git a/doc/design/assistant/blog/day_265__correctness/comment_1_e8959a6df87eb92310947e66c7471e97._comment b/doc/design/assistant/blog/day_265__correctness/comment_1_e8959a6df87eb92310947e66c7471e97._comment
new file mode 100644
index 0000000000..96731a43e3
--- /dev/null
+++ b/doc/design/assistant/blog/day_265__correctness/comment_1_e8959a6df87eb92310947e66c7471e97._comment
@@ -0,0 +1,12 @@
+[[!comment format=mdwn
+ username="https://www.google.com/accounts/o8/id?id=AItOawkF8_uQjLYm5Mf5F_JuVW-BxlvzpWjvR_o"
+ nickname="Andrew"
+ subject="Android success"
+ date="2013-05-16T04:14:09Z"
+ content="""
+I now have my android phone and laptop talking with GTalk and used a remote ssh server to synchronise files.
+
+Photos are flowing from my phone to my laptop quite happily. Magic.
+
+Would help to exclude .thumbnails from the DCIM directory on the phone though...
+"""]]
diff --git a/doc/design/assistant/blog/day_265__correctness/comment_2_0cb953fcc085eedb34e65c227309ede7._comment b/doc/design/assistant/blog/day_265__correctness/comment_2_0cb953fcc085eedb34e65c227309ede7._comment
new file mode 100644
index 0000000000..cf936cf2aa
--- /dev/null
+++ b/doc/design/assistant/blog/day_265__correctness/comment_2_0cb953fcc085eedb34e65c227309ede7._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="https://www.google.com/accounts/o8/id?id=AItOawmubB1Sj2rwFoVdZYvGV0ACaQUJQyiJXJI"
+ nickname="Paul"
+ subject="GTalk/XMPP"
+ date="2013-05-19T09:58:51Z"
+ content="""
+Google seems to be deprecating XMPP and GTalk. Can git-annex assistant use other XMPP services? And does it handle federation correctly?
+"""]]
diff --git a/doc/design/assistant/blog/day_265__correctness/comment_3_df57628a8969af2995732e7ea2a0fae3._comment b/doc/design/assistant/blog/day_265__correctness/comment_3_df57628a8969af2995732e7ea2a0fae3._comment
new file mode 100644
index 0000000000..27bee6171b
--- /dev/null
+++ b/doc/design/assistant/blog/day_265__correctness/comment_3_df57628a8969af2995732e7ea2a0fae3._comment
@@ -0,0 +1,10 @@
+[[!comment format=mdwn
+ username="http://joeyh.name/"
+ nickname="joey"
+ subject="comment 3"
+ date="2013-05-19T19:12:59Z"
+ content="""
+git-annex can use any xmpp server.  The only thing that would need to be changed if google talk went away tomorrow is changing some text that suggests entering one's gmail address, to some other text that suggests some other xmpp server to use.
+
+However, I am skeptical about echo chamber conclusions (and/or tech press) about what google is, or is not doing, or is or is not planning to do.
+"""]]
diff --git a/doc/design/assistant/blog/day_266__release_day.mdwn b/doc/design/assistant/blog/day_266__release_day.mdwn
new file mode 100644
index 0000000000..8e373c94c5
--- /dev/null
+++ b/doc/design/assistant/blog/day_266__release_day.mdwn
@@ -0,0 +1,6 @@
+Made a release.
+
+I am firming up some ideas for post-kickstarter. More on that later.
+
+In the process of setting up a Windows autobuilder, using the same
+jenkins installation that is used to autobuild msysgit.
diff --git a/doc/design/assistant/blog/day_266__release_day/comment_1_92c8d1d9216b46b07dfe69bbc77a923e._comment b/doc/design/assistant/blog/day_266__release_day/comment_1_92c8d1d9216b46b07dfe69bbc77a923e._comment
new file mode 100644
index 0000000000..39ff885413
--- /dev/null
+++ b/doc/design/assistant/blog/day_266__release_day/comment_1_92c8d1d9216b46b07dfe69bbc77a923e._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="https://www.google.com/accounts/o8/id?id=AItOawmubB1Sj2rwFoVdZYvGV0ACaQUJQyiJXJI"
+ nickname="Paul"
+ subject="Continued work"
+ date="2013-05-17T07:44:38Z"
+ content="""
+Can we continue to fund your work in some way?
+"""]]
diff --git a/doc/design/assistant/blog/day_267__windows_autobuilder.mdwn b/doc/design/assistant/blog/day_267__windows_autobuilder.mdwn
new file mode 100644
index 0000000000..b9e28438de
--- /dev/null
+++ b/doc/design/assistant/blog/day_267__windows_autobuilder.mdwn
@@ -0,0 +1,9 @@
+git-annex is now autobuilt for Windows on the same Jenkins farm that
+builds msysgit. Thanks for Yury V. Zaytsev for providing that! Spent about
+half of today setting up the build.
+
+Got the test suite to pass in direct mode, and indeed in direct mode
+on a FAT file system. Had to fix one corner case in direct mode `git annex
+add`. Unfortunately it still doesn't work on Android; somehow `git clone` 
+of a local repository is broken there. Also got the test suite to build,
+and run on Windows, though it fails pretty miserably.
diff --git a/doc/design/assistant/blog/day_267__windows_autobuilder/comment_1_978b584d86395f2f621b0e1f7c5e70d7._comment b/doc/design/assistant/blog/day_267__windows_autobuilder/comment_1_978b584d86395f2f621b0e1f7c5e70d7._comment
new file mode 100644
index 0000000000..ccda628ae8
--- /dev/null
+++ b/doc/design/assistant/blog/day_267__windows_autobuilder/comment_1_978b584d86395f2f621b0e1f7c5e70d7._comment
@@ -0,0 +1,21 @@
+[[!comment format=mdwn
+ username="https://www.google.com/accounts/o8/id?id=AItOawkDb5BRI8VzliQuKRVM7NwMdudvxiUNPHI"
+ nickname="Richard"
+ subject="git annex add on Win7 NTFS"
+ date="2013-06-14T09:26:24Z"
+ content="""
+Hello I took latest build from download #243.
+
+running in CMD or in GIT bash as admin user.
+
+git annex add, fails on all files.
+
+Example:
+add IS Breast/Changing IP Address of The Server/flowplayer-3.2.4.min.js (checksum...)
+git-annex: C:\development storage\.git\annex\objects\e4b\168\SHA256E-s15723--f6950
+bd995674741c600d0465a333f5491d5713e8ac2e3fc57d61cccadba522d.min.js\SHA256E-s15723--f6950bd995674741c600d0465a333f5491d57
+13e8ac2e3fc57d61cccadba522d.min.js.cache: openFile: does not exist (No such file or directory)
+
+Am I doing something wrong? I somehow understood that this should be ok now.
+Thanks for your kind answer
+"""]]
diff --git a/doc/design/assistant/blog/day_267__windows_autobuilder/comment_2_8f978d2811c8fbf11e3d12f245bdb52b._comment b/doc/design/assistant/blog/day_267__windows_autobuilder/comment_2_8f978d2811c8fbf11e3d12f245bdb52b._comment
new file mode 100644
index 0000000000..796dea21e7
--- /dev/null
+++ b/doc/design/assistant/blog/day_267__windows_autobuilder/comment_2_8f978d2811c8fbf11e3d12f245bdb52b._comment
@@ -0,0 +1,10 @@
+[[!comment format=mdwn
+ username="http://joeyh.name/"
+ ip="4.152.108.193"
+ subject="comment 2"
+ date="2013-06-25T17:12:17Z"
+ content="""
+That is the same problem that is currently making the test suite fail on the windows autobuilder.
+
+I don't know what is causing this problem, and have not been able to reproduce it locally in order to debug it.
+"""]]
diff --git a/doc/design/assistant/blog/day_268__core_monad_change.mdwn b/doc/design/assistant/blog/day_268__core_monad_change.mdwn
new file mode 100644
index 0000000000..2704bb090e
--- /dev/null
+++ b/doc/design/assistant/blog/day_268__core_monad_change.mdwn
@@ -0,0 +1,9 @@
+Today I had to change the implementation of the Annex monad. The old one
+turned out to be buggy around exception handling -- changes to state
+recorded by code that ran in an exception handler were discarded when it
+threw an exception. Changed from a StateT monad to a ReaderT with
+a MVar. Really deep-level change, but it went off without a
+hitch!
+
+Other than that it was a bug catch up day. Almost entirely caught up once
+more.
diff --git a/doc/design/assistant/blog/day_269__bugfixes.mdwn b/doc/design/assistant/blog/day_269__bugfixes.mdwn
new file mode 100644
index 0000000000..7cbdb58633
--- /dev/null
+++ b/doc/design/assistant/blog/day_269__bugfixes.mdwn
@@ -0,0 +1,14 @@
+Worked on several important bug fixes today. One affects automatic merge
+confict resolution, and can case data loss in direct mode, so I will be
+making a release with the fix tomorrow.
+
+Practiced TDD today, and good thing too. The new improved test suite
+turned up a really subtle bug involving the git-annex branch vector
+clocks-ish code, which I also fixed.
+
+Also, fixes to the OSX autobuilds. One of them had a broken gpg, which is
+now fixed. The other one is successfully building again. And, I'm switching
+the Linux autobuilds to build against Debian stable, since testing has a
+new version of libc now, which would make the autobuilds not work on older
+systems. Getting an amd64 chroot into shape is needing rather a lot
+of backporting of build dependencies, which I already did for i386.
diff --git a/doc/design/assistant/blog/day_26__dying_drives.mdwn b/doc/design/assistant/blog/day_26__dying_drives.mdwn
new file mode 100644
index 0000000000..109ceb19ec
--- /dev/null
+++ b/doc/design/assistant/blog/day_26__dying_drives.mdwn
@@ -0,0 +1,28 @@
+My laptop's SSD died this morning. I had some work from yesterday
+committed to the git repo on it, but not pushed as it didn't build.
+Luckily I was able to get that off the SSD, which is now a read-only
+drive -- even mounting it fails with fsck write errors.
+
+Wish I'd realized the SSD was dying before the day before my trip to
+Nicaragua..
+Getting back to a useful laptop used most of my time and energy today.
+
+I did manage to fix transfers to not block the rest of the assistant's
+threads. Problem was that, without Haskell's threaded runtime, waiting
+on something like a rsync command blocks all threads. To fix this,
+transfers now are run in separate processes.
+
+Also added code to allow multiple transfers to run at once. Each transfer
+takes up a slot, with the number of free slots tracked by a `QSemN`.
+This allows the transfer starting thread to block until a slot frees up,
+and then run the transfer.
+
+This needs to be extended to be aware of transfers initiated by remotes.
+The transfer watcher thread should detect those starting and stopping 
+and update the `QSemN` accordingly. It would also be nice if transfers
+initiated by remotes would be delayed when there are no free slots for them
+... but I have not thought of a good way to do that.
+
+There's a bug somewhere in the new transfer code, when two transfers are
+queued close together, the second one is lost and doesn't happen.
+Would debug this, but I'm spent for the day.
diff --git a/doc/design/assistant/blog/day_270__release_and_xmpp.mdwn b/doc/design/assistant/blog/day_270__release_and_xmpp.mdwn
new file mode 100644
index 0000000000..eb28f97bde
--- /dev/null
+++ b/doc/design/assistant/blog/day_270__release_and_xmpp.mdwn
@@ -0,0 +1,39 @@
+Got the bugfix release out.
+
+Tobias contributed [[tips/megaannex]], which allows using mega.co.nz as a
+special remote. Someone should do this with Flickr, using
+[filr](https://github.com/ricardobeat/filr). I have improved the
+[[special_remotes/hook]] special remote to make it easier to create
+and use reusable programs like megaannex.
+
+But, I am too busy rewriting lots of the XMPP code to join in the
+special remote fun. Spent all last night staring at protocol traces and
+tests, and came to the conclusion that it's working well at the basic
+communication level, but there are a lot of bugs above that level. This
+mostly shows up as one side refusing to push changes made to its tree,
+although it will happily merge in changes sent from the other side.
+
+The NetMessanger code, which handles routing messages to git commands and
+queuing other messages, seems to be just wrong. This is code I wrote in the
+fall, and have basically not touched since. And it shows. Spent 4 hours
+this morning rewriting it. Went all Erlang and implemented message inboxes
+using STM. I'm much more confident it won't drop messages on the
+floor, which the old code certainly did do sometimes.
+
+Added a check to avoid unnecessary pushes over XMPP. Unfortunately, this
+required changing the protocol in a way that will make previous versions of
+git-annex refuse to accept any pushes advertised by this version. Could not
+find a way around that, but there were so many unnecessary pushes happening
+(and possibly contributing to other problems) that it seemed worth the
+upgrade pain.
+
+Will be beating on XMPP a bit more. There is one problem I was seeing
+last night that I cannot reproduce now. It may have been masked or even
+fixed by these changes, but I need to verify that, or put in a workaround.
+It seemed that sometimes this code in `runPush` would run the setup
+and the action, but either the action blocked forever, or an exception
+got through and caused the cleanup not to be run.
+
+[[!format haskell """
+        r <- E.bracket_ setup cleanup <~> a
+"""]]
diff --git a/doc/design/assistant/blog/day_271__more_xmpp.mdwn b/doc/design/assistant/blog/day_271__more_xmpp.mdwn
new file mode 100644
index 0000000000..14e734a2d3
--- /dev/null
+++ b/doc/design/assistant/blog/day_271__more_xmpp.mdwn
@@ -0,0 +1,31 @@
+Tobias has been busy again today, creating a [[/tips/flickrannex]]
+special remote! Meanwhile, I'm thinking about providing a
+[[more complete interface|/todo/support_for_writing_external_special_remotes]]
+so that special remote programs not written in Haskell can do some of the
+things the hook special remote's simplicity doesn't allow.
+
+Finally realized last night that the main problem with the XMPP push code
+was an inversion of control. Reworked it so now there are two new threads,
+XMPPSendpack and XMPPReceivePack, each with their own queue of push
+initiation requests, that run the pushes. This is a lot easier to
+understand, probably less buggy, and lets it apply some smarts to squash
+duplicate actions and pick the best request to handle next.
+
+Also made the XMPP client send pings to detect when it has been disconnected
+from the server. Currently every 120 seconds, though that may change. Testing
+showed that without this, it did not notice (for at least 20 minutes) when 
+it lost routing to the server. Not sure why -- I'd think the TCP connections
+should break and this throw an error -- but this will also handle any idle
+disconnection problems that some XMPP servers might have.
+
+While writing that, I found myself writing this jem using
+[async](http://hackage.haskell.org/package/async), which has a comment
+much longer than the code, but basically we get 4 threads that are all
+linked, so when any dies, all do.
+
+[[!format haskell """
+pinger `concurrently` sender `concurrently` receiver
+"""]]
+
+Anyway, I need to run some long-running XMPP push tests to see if I've
+really ironed out all the bugs.
diff --git a/doc/design/assistant/blog/day_272__fuzz_tester.mdwn b/doc/design/assistant/blog/day_272__fuzz_tester.mdwn
new file mode 100644
index 0000000000..9d352f70a3
--- /dev/null
+++ b/doc/design/assistant/blog/day_272__fuzz_tester.mdwn
@@ -0,0 +1,37 @@
+The Android app should work on some more devices now, where hard linking to
+busybox didn't work. Now it installs itself using symlinks.
+
+Pushed a point release so `cabal install git-annex` works again. And,
+I'm really happy to see that the 4.20130521 release has autobuilt on all
+Debian architectures, and will soon be replacing the old 3.20120629 version
+in testing. (Well, once a libffi transition completes..)
+
+TobiasTheMachine has done it again: [[tips/googledriveannex]]  
+
+-----
+
+I spent most of today building a fuzz tester for the assistant. `git annex
+fuzztest` will (once you find the special runes to allow it to run) create
+random files in the repository, move them around, delete them, move
+directory trees around, etc. The plan is to use this to run some long
+duration tests with eg, XMPP, to make sure the assistant keeps things
+in shape after a lot of activity. It logs in machine-readable format, 
+so if it turns up a bug I may even be able to use it to reproduce the same
+bug (fingers crossed).
+
+I was able to use QuickCheck to generate random data for some parts of the fuzz
+tester. (Though the actual file names it uses are not generated using
+QuickCheck.) Liked this part:
+
+[[!format haskell """
+instance Arbitrary FuzzAction where
+        arbitrary = frequency
+                [ (100, FuzzAdd <$> arbitrary)
+                , (10, FuzzDelete <$> arbitrary)
+                , (10, FuzzMove <$> arbitrary <*> arbitrary)
+                , (10, FuzzModify <$> arbitrary)
+                , (10, FuzzDeleteDir <$> arbitrary)
+                , (10, FuzzMoveDir <$> arbitrary <*> arbitrary)
+                , (10, FuzzPause <$> arbitrary)
+                ]
+"""]]
diff --git a/doc/design/assistant/blog/day_273-274__fun.mdwn b/doc/design/assistant/blog/day_273-274__fun.mdwn
new file mode 100644
index 0000000000..2fa12003a5
--- /dev/null
+++ b/doc/design/assistant/blog/day_273-274__fun.mdwn
@@ -0,0 +1,19 @@
+Got caught up on some bug reports yesterday. The main one was odd behavior
+of the assistant when the repository was in manual mode. A recent change to
+the preferred content expression caused it. But the expression was not
+broken. The problem was in the parser, which got the parentheses wrong
+in this case. I had to mostly rewrite the parser, unfortunately. I've tested
+the new one fairly extensively -- on the other hand this bug lurked in the
+old parser for several years (this same code is used for matching files
+with command-line parameters).
+
+Just as I finished with that, I noticed another bug. Turns out git-cat-file
+doesn't reload the index after it's started. So last week's changes to make
+git-annex check the state of files in the index won't work when using the
+assistant. Luckily there was an easy workaround for this.
+
+Today I finished up some robustness fixes, and added to the test suite
+checks for preferred content expressions, manual mode, etc.
+
+I've started a stress test, syncing 2 repositories over XMPP, with the fuzz
+tester running in each to create lots of changes to keep in sync.
diff --git a/doc/design/assistant/blog/day_275__working_hard_or.mdwn b/doc/design/assistant/blog/day_275__working_hard_or.mdwn
new file mode 100644
index 0000000000..b667ba08a1
--- /dev/null
+++ b/doc/design/assistant/blog/day_275__working_hard_or.mdwn
@@ -0,0 +1,12 @@
+Fuzz tester has found several interesting bugs that I've now fixed. It's
+even found a bug in my fixes. Most of the problems the fuzz testing has
+found have had to do with direct mode merges, and automatic merge conflict
+resoltion. Turns out the second level of automatic merge conflict
+resolution (where the changes made to resolve a merge conflict themselves
+turn out conflict in a later merge) was buggy, for example.
+
+So, didn't really work a lot today -- was not intending to work at all
+actually -- but have still accomplished a lot.
+
+(Also, Tobias contributed [[tips/dropboxannex]] .. I'll be curious to see
+what the use case for that is, if any!)
diff --git a/doc/design/assistant/blog/day_276__fuzzing_continues.mdwn b/doc/design/assistant/blog/day_276__fuzzing_continues.mdwn
new file mode 100644
index 0000000000..d6fc88b057
--- /dev/null
+++ b/doc/design/assistant/blog/day_276__fuzzing_continues.mdwn
@@ -0,0 +1,12 @@
+The fuzz testing found a file descriptor leak in the XMPP git push code.
+The assistant seems to hold up under fuzzing for quite a while now.
+
+Have started trying to work around some versions of Android not letting 
+the `am` command be used by regular users to open a web browser on an URL.
+Here is my current crazy plan: Hack the terminal emulator's title setting
+code, to get a new escape sequence that requests an URL be opened. This
+assumes I can just use `startActivity()` from inside the app and it will
+work. This may sound a little weird, but it avoids me needing to set up a
+new communications channel from the assistant to the Java app. Best of all,
+I have to write very little Java code. I last wrote Java code in 1995, so
+writing much more is probably a good thing to avoid.
diff --git a/doc/design/assistant/blog/day_276__fuzzing_continues/comment_1_f5dd0658511a1063c2eb025b0fe98426._comment b/doc/design/assistant/blog/day_276__fuzzing_continues/comment_1_f5dd0658511a1063c2eb025b0fe98426._comment
new file mode 100644
index 0000000000..4a54fa1880
--- /dev/null
+++ b/doc/design/assistant/blog/day_276__fuzzing_continues/comment_1_f5dd0658511a1063c2eb025b0fe98426._comment
@@ -0,0 +1,14 @@
+[[!comment format=mdwn
+ username="https://www.google.com/accounts/o8/id?id=AItOawkGCmVc5qIJaQQgG82Hc5zzBdAVdhe2JEM"
+ nickname="Bruno"
+ subject="comment 1"
+ date="2013-05-28T03:23:50Z"
+ content="""
+Does the Android application use a [WakeLock](https://developer.android.com/reference/android/os/PowerManager.WakeLock.html) to ensure syncing happens even if the device screen goes off?
+
+I'm under the impression that it doesn't since it took a very long time to sync the first time (before I used the terminal's options to prevent the phone from sleeping).
+
+I think that option shouldn't be left on since it would waste the battery and I think the application should block sleep mode only when syncing.
+
+I might be wrong. I never had to use a WakeLock on Android yet.
+"""]]
diff --git a/doc/design/assistant/blog/day_276__fuzzing_continues/comment_2_a56c4c26a9e7bb8cfe3f598dbeed0813._comment b/doc/design/assistant/blog/day_276__fuzzing_continues/comment_2_a56c4c26a9e7bb8cfe3f598dbeed0813._comment
new file mode 100644
index 0000000000..3a98c8a092
--- /dev/null
+++ b/doc/design/assistant/blog/day_276__fuzzing_continues/comment_2_a56c4c26a9e7bb8cfe3f598dbeed0813._comment
@@ -0,0 +1,10 @@
+[[!comment format=mdwn
+ username="http://joeyh.name/"
+ nickname="joey"
+ subject="comment 2"
+ date="2013-05-29T16:30:20Z"
+ content="""
+The terminal has options to take the wake lock, and also the wifi lock, in its menu.
+
+Currently this has to be managed manually. The assistant does not try to deal with this.
+"""]]
diff --git a/doc/design/assistant/blog/day_277__private_static_protected_void.mdwn b/doc/design/assistant/blog/day_277__private_static_protected_void.mdwn
new file mode 100644
index 0000000000..dbe2244a02
--- /dev/null
+++ b/doc/design/assistant/blog/day_277__private_static_protected_void.mdwn
@@ -0,0 +1,19 @@
+Yeah, Java hacking today. I have something that I think should deal with
+the [[bugs/Android_app_permission_denial_on_startup]] problem. Added a "Open
+WebApp" item to the terminal's menu, which should behave as advertised.
+This is available in the Android daily build now, if your device has that
+problem.
+
+I was not able to get the escape sequence hack to work. I had no difficulty
+modifying the terminal to send an intent to open an url when it received a
+custom escape sequence. But sending the intent just seemed to lock up the
+terminal for a minute without doing anything. No idea why. I had to propigate a
+context object in to the terminal emulator through several layers of objects.
+Perhaps that doesn't really work despite what I [read on stackoverflow](http://stackoverflow.com/questions/9051849/opening-a-link-in-the-browser/9052208#9052208).
+
+Anyway, that's all I have time to do. It would be nice if I, or some other
+interested developer who is more comfortable with Java, could write a custom
+Android frontend app, that embedded a web browser widget for the webapp,
+rather than abusing the terminal this way. OTOH, this way does provide the
+bonus of a pretty good terminal and git shell environment for Android to go
+with git-annex.
diff --git a/doc/design/assistant/blog/day_278__winding_down.mdwn b/doc/design/assistant/blog/day_278__winding_down.mdwn
new file mode 100644
index 0000000000..824feceaf5
--- /dev/null
+++ b/doc/design/assistant/blog/day_278__winding_down.mdwn
@@ -0,0 +1,11 @@
+Winding down work for now, as I prepare for a week at the beach starting in
+2 days. That will be followed by a talk about git-annex at
+[SELF2013](http://www.southeastlinuxfest.org/) in Charlotte NC on June 9th.
+
+Bits & pieces today.
+
+Want to get a release out RSN, but I'm waiting for the previous release
+to finally reach Debian testing, which should happen on Saturday. Luckily I
+hear the beach house has wifi, so I will probably end up cutting the
+release from there. Only other thing I might work on next week is updating
+to yesod 1.2.
diff --git a/doc/design/assistant/blog/day_279__final_release_prep.mdwn b/doc/design/assistant/blog/day_279__final_release_prep.mdwn
new file mode 100644
index 0000000000..f209c1ccb6
--- /dev/null
+++ b/doc/design/assistant/blog/day_279__final_release_prep.mdwn
@@ -0,0 +1,14 @@
+Landed two final changes before the release..
+
+First, made git-annex detect if any of the several long-running git process
+it talks to have died, and, if yes, restart them. My stress test is reliably
+able to get at least `git cat-file` to crash, and while I don't know why (and
+obviously should follow up by getting a core dump and stack trace of it),
+the assistant needs to deal with this to be robust.
+
+Secondly, wrote rather a lot of Java code to better open the web browser
+when the Android app is started. A thread listens for URLs to be written to
+a FIFO. Creating a FIFO from fortran^Wjava code is .. interesting. Glad to
+see the back of the `am` command; it did me no favors.
+
+AFK
diff --git a/doc/design/assistant/blog/day_27__robust_transfers.mdwn b/doc/design/assistant/blog/day_27__robust_transfers.mdwn
new file mode 100644
index 0000000000..49ace417b7
--- /dev/null
+++ b/doc/design/assistant/blog/day_27__robust_transfers.mdwn
@@ -0,0 +1,31 @@
+Spent most of the day making file content transfers robust. There were lots
+of bugs, hopefully I've fixed most of them. It seems to work well now,
+even when I throw a lot of files at it.
+
+One of the changes also sped up transfers; it no longer roundtrips to the
+remote to verify it has a file. The idea here is that when the assistant is
+running, repos should typically be fairly tightly synced to their remotes
+by it, so some of the extra checks that the `move` command does are
+unnecessary.
+
+Also spent some time trying to use ghc's threaded runtime, but continue to
+be baffled by the random hangs when using it. This needs fixing eventually;
+all the assistant's threads can potentially be blocked when it's waiting on
+an external command it has run.
+
+Also changed how transfer info files are locked. The lock file is now
+separate from the info file, which allows the TransferWatcher thread to
+notice when an info file is created, and thus actually track transfers
+initiated by remotes.
+
+---
+
+I'm fairly close now to merging the `assistant` branch into `master`.
+The data syncing code is very brute-force, but it will work well enough
+for a first cut.
+
+Next I can either add some repository network mapping, and use graph
+analysis to reduce the number of data transfers, or I can move on to the
+[[webapp]]. Not sure yet which I'll do. It's likely that since DebConf
+begins tomorrow I'll put off either of those big things until after the
+conference.
diff --git a/doc/design/assistant/blog/day_28-35__threaded_runtime_tarpit.mdwn b/doc/design/assistant/blog/day_28-35__threaded_runtime_tarpit.mdwn
new file mode 100644
index 0000000000..612dfc514c
--- /dev/null
+++ b/doc/design/assistant/blog/day_28-35__threaded_runtime_tarpit.mdwn
@@ -0,0 +1,17 @@
+I didn't plan to work on git-annex much while at DebConf, because the conference
+always prevents the kind of concentration I need. But I unexpectedly also had to deal
+with [three dead drives](http://joeyh.name/blog/entry/I_am_become_Joey_destroyer_of_drives/)
+and illness this week. 
+
+That said, I have been trying to debug a problem with git-annex and Haskell's threaded
+runtime all week. It just hangs, randomly. No luck so far isolating why, although I now
+have a branch that hangs fairly reliably, and in which I am trying to whittle the entire
+git-annex code base (all 18 thousand lines!) into a nice test case.
+
+This threaded runtime problem doesn't affect the assistant yet, but if I want to use
+Yesod in developing the webapp, I'll need the threaded runtime, and using the threaded
+runtime in the assistant generally would make it more responsive and less hacky.
+
+Since this is a task I can work on without much concentration, I'll probably keep beating
+on it until I return home. Then I need to spend some quality thinking time on where
+to go next in the assistant.
diff --git a/doc/design/assistant/blog/day_280__yesod.mdwn b/doc/design/assistant/blog/day_280__yesod.mdwn
new file mode 100644
index 0000000000..ff02fd02ba
--- /dev/null
+++ b/doc/design/assistant/blog/day_280__yesod.mdwn
@@ -0,0 +1,7 @@
+Today marks 1 year since I started working on the git-annex assistant. 280
+solid days of work!
+
+As a background task here at the beach I've been porting git-annex to yesod
+1.2. Finished it today, earlier than expected, and also managed to keep it
+building with older versions. Some tricks kept the number of
+ifdefs reasonably low.
diff --git a/doc/design/assistant/blog/day_280__yesod/comment_1_a42213a8cef71f2b54db18606028136d._comment b/doc/design/assistant/blog/day_280__yesod/comment_1_a42213a8cef71f2b54db18606028136d._comment
new file mode 100644
index 0000000000..3a99c22108
--- /dev/null
+++ b/doc/design/assistant/blog/day_280__yesod/comment_1_a42213a8cef71f2b54db18606028136d._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="https://www.google.com/accounts/o8/id?id=AItOawkGCmVc5qIJaQQgG82Hc5zzBdAVdhe2JEM"
+ nickname="Bruno"
+ subject="comment 1"
+ date="2013-06-04T12:51:20Z"
+ content="""
+Congrats on the first year Joey!
+"""]]
diff --git a/doc/design/assistant/blog/day_281__back.mdwn b/doc/design/assistant/blog/day_281__back.mdwn
new file mode 100644
index 0000000000..03e42c0a59
--- /dev/null
+++ b/doc/design/assistant/blog/day_281__back.mdwn
@@ -0,0 +1,37 @@
+Slowly getting through the bugs that were opened while I was on vacation and
+then I'll try to get to all the comments. 60+ messages to go.
+
+Got git-annex working better on encfs, which does not support hard links in
+paranoid mode. Now git-annex can be used in indirect mode, it doesn't force
+direct mode when hard links are not supported.
+
+Made the Android repository setup special case generate a .gitignore file
+to ignore thumbnails. Which will only start working once the assistant
+gets .gitignore support.
+
+-----
+
+Been thinking today about encrypting XMPP traffic, particularly git push
+data. Of course, XMPP is already encrypted, but that doesn't hide it from
+those entities who have access to the XMPP server or its encryption key.
+So adding client-to-client encryption has been on the TODO list all along.
+
+OTR would be a great way to do it. But I worry that the confirmation steps
+OTR uses to authenticate the recipient would make the XMPP pairing UI harder
+to get through.
+
+Particularly when pairing your own devices over XMPP, with several devices
+involved, you'd need to do a lot of cross-confirmations. It would be better
+there, I think, to just use a shared secret for authentication. (The need
+to enter such a secret on each of your devices before pairing them would
+also provide a way to use different repositories with the same XMPP
+account, so 2birds1stone.)
+
+Maybe OTR confirmations would be ok when setting up sharing with a friend.
+If OTR was not used there, and it just did a Diffie-Hellman key exchange
+during the pairing process, it could be attacked by an active MITM spoofing
+attack. The attacker would then know the keys, and could decrypt future
+pushes. How likely is such an attack? This goes far beyond what we're
+hearing about. Might be best to put in some basic encryption now, so
+we don't have to worry about pushes being passively recorded on the
+server. Comments appreciated.
diff --git a/doc/design/assistant/blog/day_281__back/comment_1_128809c5a2a9f5cc345a10fdbf55be01._comment b/doc/design/assistant/blog/day_281__back/comment_1_128809c5a2a9f5cc345a10fdbf55be01._comment
new file mode 100644
index 0000000000..a120cc2d32
--- /dev/null
+++ b/doc/design/assistant/blog/day_281__back/comment_1_128809c5a2a9f5cc345a10fdbf55be01._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="https://www.google.com/accounts/o8/id?id=AItOawlJ2utMQgMEYAOs3Dfc6eZRyUzt4acNXUU"
+ nickname="David"
+ subject="comment 1"
+ date="2013-06-10T23:42:20Z"
+ content="""
+If you do a D-H key exchange and display the key fingerprint on both devices, that would be a pretty strong disincentive against XMPP servers doing a MitM. You only need one victim to casually notice a discrepancy, and it leaves behind a strong proof of tampering.
+"""]]
diff --git a/doc/design/assistant/blog/day_281__back/comment_2_6d0bbdf6ebaff9da399804570f0e606d._comment b/doc/design/assistant/blog/day_281__back/comment_2_6d0bbdf6ebaff9da399804570f0e606d._comment
new file mode 100644
index 0000000000..884b2484ef
--- /dev/null
+++ b/doc/design/assistant/blog/day_281__back/comment_2_6d0bbdf6ebaff9da399804570f0e606d._comment
@@ -0,0 +1,10 @@
+[[!comment format=mdwn
+ username="http://joeyh.name/"
+ nickname="joey"
+ subject="comment 2"
+ date="2013-06-10T23:50:27Z"
+ content="""
+David, along those lines, if a value derived from the D-H key were committed to the git repo, then the MITM would need to maintain two distinct git trees, and translate between them on an ongoing basis. Which is harder. And then if both clients at any point communicated via another channel (ie, local pairing), the forgery would be very noticeable.
+
+Which is not to say that this is necessarily good enough..
+"""]]
diff --git a/doc/design/assistant/blog/day_282-283__caught_up.mdwn b/doc/design/assistant/blog/day_282-283__caught_up.mdwn
new file mode 100644
index 0000000000..b097e8d248
--- /dev/null
+++ b/doc/design/assistant/blog/day_282-283__caught_up.mdwn
@@ -0,0 +1,18 @@
+Got caught up on my backlog yesterday.
+
+Part of adding files in direct mode involved removing write permission from
+them temporarily. That turned out to cause problems with some programs that
+open a file repeatedly, and was generally against the principle that direct
+mode files are always directly available. Happily, I was able to get rid of
+that without sacrificing any safety.
+
+Improved syncing to bare repositories. Normally syncing pushes to a
+synced/master branch, which is good for non-bare repositories since git
+does not allow pushing to the currently checked out branch. But for bare
+repositories, this could leave them without a master branch, so cloning
+from them wouldn't work. A funny thing is that git does not really have any
+way to tell if a remote repository is bare or not. Anyway, I did put in a
+fix, at the expense of pushing twice (but the git data should only be
+transferred once anyway).
+
+
diff --git a/doc/design/assistant/blog/day_284__porting.mdwn b/doc/design/assistant/blog/day_284__porting.mdwn
new file mode 100644
index 0000000000..f0648b0e1a
--- /dev/null
+++ b/doc/design/assistant/blog/day_284__porting.mdwn
@@ -0,0 +1,13 @@
+Today I got to deal with bugs on Android (busted use of `cp` among other
+problems), Windows (fixed a strange hang when adding several files), and
+Linux (`.desktop` files suck and Wine ships a particularly nasty one).
+Pretty diverse!
+
+Did quite a lot of research and thinking on XMPP encryption yesterday, but
+have not run any code yet (except for trying out a D-H exchange in `ghci`).
+I have listed several options on the [[XMPP]] page.
+
+Planning to take a look at
+[[bugs/Handling_of_files_inside_and_outside_archive_directory_at_the_same_time]]
+tomorrow; maybe I can come up with a workaround to avoid it behaving so
+badly in that case.
diff --git a/doc/design/assistant/blog/day_285__fixed_the_archive_directory_loop.mdwn b/doc/design/assistant/blog/day_285__fixed_the_archive_directory_loop.mdwn
new file mode 100644
index 0000000000..252dc367f7
--- /dev/null
+++ b/doc/design/assistant/blog/day_285__fixed_the_archive_directory_loop.mdwn
@@ -0,0 +1,23 @@
+Yay, I fixed the
+[[bugs/Handling_of_files_inside_and_outside_archive_directory_at_the_same_time]]
+bug!
+At least in direct mode, which thanks to its associated files tracking
+knows when a given file has another file in the repository with the same
+content. Had not realized the behavior in direct mode was so bad, or the
+fix so relatively easy. Pity I can't do the same for indirect mode, but
+the problem is much less serious there.
+
+That was this weekend. Today, I nearly put out a new release (been 2 weeks
+since the last one..), but ran out of time in the end, and need to get the
+OSX autobuilder fixed first, so have deferred it until Friday.
+
+However, I did make some improvements today.
+Added an `annex.debug` git config setting, so debugging can
+be turned on persistently. People seem to expect that to happen when
+checking the checkbox in the webapp, so now it does.
+
+Fixed 3 or 4 bugs in the Windows port. Which actually, has users now, or
+at least one user. It's very handy to actually get real world testing of
+that port.
+
+[[!meta date="Mon, 17 Jun 2013 17:14:04 -0400"]]
diff --git a/doc/design/assistant/blog/day_285__fixed_the_archive_directory_loop/comment_1_1065e756dc6d66aefd214eb8ac5ebe1d._comment b/doc/design/assistant/blog/day_285__fixed_the_archive_directory_loop/comment_1_1065e756dc6d66aefd214eb8ac5ebe1d._comment
new file mode 100644
index 0000000000..41494c02f3
--- /dev/null
+++ b/doc/design/assistant/blog/day_285__fixed_the_archive_directory_loop/comment_1_1065e756dc6d66aefd214eb8ac5ebe1d._comment
@@ -0,0 +1,25 @@
+[[!comment format=mdwn
+ username="https://www.google.com/accounts/o8/id?id=AItOawkSq2FDpK2n66QRUxtqqdbyDuwgbQmUWus"
+ nickname="Jimmy"
+ subject="comment 1"
+ date="2013-06-26T08:36:33Z"
+ content="""
+Been pretty bad with keeping the osx builder going, but this should fix it...
+
+
+
    +[jtang@x00 build ((cfb577d...))]$ git diff
+diff --git a/.gitignore b/.gitignore
+index 93bd49e..717b58a 100644
+--- a/.gitignore
++++ b/.gitignore
+@@ -24,3 +24,5 @@ cabal-dev
+ .virthualenv
+ tags
+ Setup
++*.hi
++*.o
+
    
+
+The build on OSX leaves some stray files and it complains about it.
+"""]]
diff --git a/doc/design/assistant/blog/day_286__Windows_test_suite.mdwn b/doc/design/assistant/blog/day_286__Windows_test_suite.mdwn
new file mode 100644
index 0000000000..f62aa8eb26
--- /dev/null
+++ b/doc/design/assistant/blog/day_286__Windows_test_suite.mdwn
@@ -0,0 +1,19 @@
+One of my Windows fixes yesterday got the test suite close to sort of
+working on Windows, and I spent all day today pounding on it. Fixed
+numerous bugs, and worked around some weird Windows behaviors -- like
+recursively deleting a directory sometimes fails with a permission denied
+error about a file in it, and leaves behind an empty directory. (What!?)
+The most important bug I fixed caused CR to leak into files in the
+git-annex branch from Windows, during a union merge, which was not a good
+thing at all.
+
+At the end of the day, I only have 6 remaining failing test cases on
+Windows. Half of them are some problem where running `git annex sync`
+from the test suite stomps on PATH somehow and prevents xargs from working.
+The rest are probably real bugs in the directory (again something to do
+with recursive directory deletion, hmmm..), hook, and rsync
+special remotes on Windows. I'm punting on those 6 for now, they'll be
+skipped on Windows.
+
+Should be worth today's pain to know in the future when I break
+something that I've oh-so-painfully gotten working on Windows.
diff --git a/doc/design/assistant/blog/day_287__niceness.mdwn b/doc/design/assistant/blog/day_287__niceness.mdwn
new file mode 100644
index 0000000000..5fc193353d
--- /dev/null
+++ b/doc/design/assistant/blog/day_287__niceness.mdwn
@@ -0,0 +1,13 @@
+Pushed out a release today. While I've somewhat ramped down activity this
+month with the Kickstarter period over and summer trips and events ongoing,
+looking over the changelog I still see a ton of improvements in the 20 days
+since the last release.
+
+Been doing some work to make the assistant daemon be more `nice`. I don't
+want to nice the whole program, because that could make the web interface
+unresponsive. What I am able to do, thanks to Linux violating POSIX, is to
+`nice` certain expensive operations, including the startup scan and the daily
+sanity check. Also, I put in a call to `ionice` (when it's available) 
+when `git annex assistant --autostart` is run, so the daemon's
+disk IO will be prioritized below other IO. Hope this keeps it out of your
+way while it does its job.
diff --git a/doc/design/assistant/blog/day_288__success_stories.mdwn b/doc/design/assistant/blog/day_288__success_stories.mdwn
new file mode 100644
index 0000000000..c19f64e1ff
--- /dev/null
+++ b/doc/design/assistant/blog/day_288__success_stories.mdwn
@@ -0,0 +1,32 @@
+Got caught up on a big queue of messages today. Mostly I hear from people
+when git-annex is not working for them, or they have a question about using
+it. From time to time someone does mention that it's working for them.
+
+> We have 4 or so machines all synching with each other via the local
+> network thing. I'm always amazed when it doesn't just explode :)
+
+Due to the nature of git-annex, a lot of people can be using it without
+anyone knowing about it. Which is great. But these little success stories
+can make all the difference. It motivates me to keep pounding out the
+development hours, it encourages other people to try it, and it'd be a good
+thing to be able to point at if I tried to raise more funding now that I'm
+out of Kickstarter money.
+
+I'm posting my own success story to my main blog:
+[git annex and my mom](http://joeyh.name/blog/entry/git_annex_and_my_mom)
+
+If you have a success story to share, why not blog about it, microblog it,
+or just post a comment here, or even send me a private message. Just a
+quick note is fine. Thanks!
+
+----
+
+Going through the bug reports and questions today, I ended up fixing
+three separate bugs that could break setting up a repo on a remote ssh
+server from the webapp.
+
+Also developed a minimal test case for some gnucash behavior that prevents
+the watcher from seeing files it writes out. I understand the problem,
+but don't have a fix for that yet. Will have to think about it. (A year ago
+today, my blog featured the
+[[first_release_of_the_watcher|day_16__more_robust_syncing]].)
diff --git a/doc/design/assistant/blog/day_288__success_stories/comment_10_9ddf57b8ae0241268bb33bec1b169e4c._comment b/doc/design/assistant/blog/day_288__success_stories/comment_10_9ddf57b8ae0241268bb33bec1b169e4c._comment
new file mode 100644
index 0000000000..f53f4b654d
--- /dev/null
+++ b/doc/design/assistant/blog/day_288__success_stories/comment_10_9ddf57b8ae0241268bb33bec1b169e4c._comment
@@ -0,0 +1,20 @@
+[[!comment format=mdwn
+ username="https://www.google.com/accounts/o8/id?id=AItOawmnG4EuvZWse5hvgrl0XAK-U61e-0iGaao"
+ nickname="David"
+ subject="Nothing fancy but..."
+ date="2013-06-27T16:52:12Z"
+ content="""
+Hi Joey,
+
+First, thanks very much for your effort put into git-annex.  I store my photos in git-annex and I like it because:
+
+* I want to store multiple copies of the pictures and I can see where they are,
+* I can have a global view of the collection on my laptop without carrying all the files.
+* the indirect mode makes it difficult to me to do stupid things.
+
+So there's nothing fancy, but it works for me very well.  From time to time I try the assistant's new features and I'm planning to replace Dropbox eventually.
+
+Secondly, thanks very much for the dev blog!  I was looking forward every morning to see what's coming next, what problems you are facing and how you're planning to solve them.  It is simply amazing what you've achieved in the last year and your posts gave me a lot of motivation too (in finishing my thesis  :-))!
+
+Thanks!
+"""]]
diff --git a/doc/design/assistant/blog/day_288__success_stories/comment_11_50b8a597bd8677608f2ef176443f23f3._comment b/doc/design/assistant/blog/day_288__success_stories/comment_11_50b8a597bd8677608f2ef176443f23f3._comment
new file mode 100644
index 0000000000..219e1e651d
--- /dev/null
+++ b/doc/design/assistant/blog/day_288__success_stories/comment_11_50b8a597bd8677608f2ef176443f23f3._comment
@@ -0,0 +1,10 @@
+[[!comment format=mdwn
+ username="http://joeyh.name/"
+ ip="4.152.108.193"
+ subject="comment 11"
+ date="2013-06-27T18:11:03Z"
+ content="""
+Excerpt from a [longer comment](http://git-annex.branchable.com/bugs/Problems_with_syncing_gnucash/#comment-7d70d822a0fdf672ba09535f943af7c9) by Florian:
+
+I know that my expectations in this project are a little bit high and I see how much work it was to get where we are now. Once again thank you for the excellent work done so far. I don't fear to synchronize important data with git-annex. I even use git-annex to backup and synchronize the member database for our club (wannabe hackerspace).
+"""]]
diff --git a/doc/design/assistant/blog/day_288__success_stories/comment_12_f2df427cf3608377e9a52d8bdeadb26f._comment b/doc/design/assistant/blog/day_288__success_stories/comment_12_f2df427cf3608377e9a52d8bdeadb26f._comment
new file mode 100644
index 0000000000..10699c9593
--- /dev/null
+++ b/doc/design/assistant/blog/day_288__success_stories/comment_12_f2df427cf3608377e9a52d8bdeadb26f._comment
@@ -0,0 +1,21 @@
+[[!comment format=mdwn
+ username="http://joeyh.name/"
+ ip="4.152.108.193"
+ subject="comment 12"
+ date="2013-06-27T18:11:45Z"
+ content="""
+From a mail I received: --[[Joey]]
+
+Git-annex makes it possible for me to have lots of data.  Before I
+started using it I had a music library and a few massive tar files,
+none of it properly backed up, and occasionally there would be random
+corruption (I still have broken mp3s which jump and stutter in places)
+because I would carelessly cp them from HDD to HDD.  So I'd avoid
+actually letting data accumulate; this was inconvenient in itself.
+Now git-annex does everything for me.
+
+Following your development blog, as a novice programmer, has been
+really enjoyable and interesting.  My only experience of professional
+IT has put me of but it's good to see you working through things
+methodically and properly.
+"""]]
diff --git a/doc/design/assistant/blog/day_288__success_stories/comment_13_8762efed97f21eeba8f0a7be45bd924a._comment b/doc/design/assistant/blog/day_288__success_stories/comment_13_8762efed97f21eeba8f0a7be45bd924a._comment
new file mode 100644
index 0000000000..3b18645e4c
--- /dev/null
+++ b/doc/design/assistant/blog/day_288__success_stories/comment_13_8762efed97f21eeba8f0a7be45bd924a._comment
@@ -0,0 +1,35 @@
+[[!comment format=mdwn
+ username="http://julien.lefrique.name/"
+ nickname="jlefrique"
+ subject="Many thanks!"
+ date="2013-06-27T19:59:47Z"
+ content="""
+Hi Joey,
+
+More than ten years ago, I was looking for a software to index the content of all my
+CD-ROMs in order to create a kind of \"table of content\". Today there is
+git-annex!
+
+I discovered git-annex few months ago and it really helps me to keep my file
+well organized. I missed the first crowd funding campaign but would be happy
+to contribute to the next one :D
+
+I mainly use four different annexes for:
+
+* my private photos and videos,
+* my music collection,
+* my other multimedia files (movies...),
+* my other files that are too large to be commited into git directly (pdf, binaries...).
+
+I use git-annex manually via the command line. I started to play with the
+assistant on my Android tablet and I plan to install it on my wife's
+Macbook to synchronize her data with the NAS.
+
+A standalone build for arm to run git-annex on my NAS (QNAP 219 PII) would be perfect :-P
+
+Thank you very much for all your great softwares (yes I also use etckeeper,
+some moreutils tools and more recently ikiwiki).
+
+-Julien
+
+"""]]
diff --git a/doc/design/assistant/blog/day_288__success_stories/comment_14_55e1bb15c3a93d582d110f8173ceefc2._comment b/doc/design/assistant/blog/day_288__success_stories/comment_14_55e1bb15c3a93d582d110f8173ceefc2._comment
new file mode 100644
index 0000000000..f63dc7b9e0
--- /dev/null
+++ b/doc/design/assistant/blog/day_288__success_stories/comment_14_55e1bb15c3a93d582d110f8173ceefc2._comment
@@ -0,0 +1,9 @@
+[[!comment format=mdwn
+ username="EvanDeaubl"
+ ip="67.128.198.190"
+ subject="comment 14"
+ date="2013-06-28T18:35:55Z"
+ content="""
+git-annex has been working brilliantly for me. I have been slowly moving content out of the various places I've had it stored (file servers, folders hidden away on all my machines, Dropbox, Google Drive, etc.) into a single git-annex repo. The idea has been to mimic Dropbox's single folder, but with much more flexibility: I sync my working machines, my home fileserver, my VPS (for file access on the go), Android devices, and even a couple of dumb MP3 players from git-annex. Best of all, I have one place to go for all of my files, and if they're not on the device at the moment I need them, they're a download away. I didn't find git-annex until after the first crowdfunding campaign had completed, but if there's another, I would be a happy supporter.
+
+"""]]
diff --git a/doc/design/assistant/blog/day_288__success_stories/comment_15_5749aef8b585b293385b20b75c40f9d8._comment b/doc/design/assistant/blog/day_288__success_stories/comment_15_5749aef8b585b293385b20b75c40f9d8._comment
new file mode 100644
index 0000000000..cefc5045c6
--- /dev/null
+++ b/doc/design/assistant/blog/day_288__success_stories/comment_15_5749aef8b585b293385b20b75c40f9d8._comment
@@ -0,0 +1,31 @@
+[[!comment format=mdwn
+ username="http://jasonwoof.com/"
+ nickname="JasonWoof"
+ subject="It's been great!"
+ date="2013-06-29T03:48:08Z"
+ content="""
+Looks like I started using git-annex in November 2010.
+
+I'd already been tracking lots of small files in my home directory (settings, scripts, notes, todo list, timesheets, invoices, etc) with git.
+
+So awesome to get backups, syncing and manual merge conflict resolution.
+
+git-annex extended that to handle my larger files. git-annex's ability to be selective about what is stored where, allowed my to still be able to git clone my home directory onto small devices (I've got a debian chroot with only 6GB or so, and I installed debian on my new chromebook, that's only got 16GB of disk space (for the whole system.))
+
+I love having the full catalog visible on my filesystem with symlinks, and I can just request a file, and git-annex will figure out where to get it from, even if I'm away from home on my laptop.
+
+I just got a 2nd external hard drive, and I'm delighted with how easy it was to get a backup of everything on there.
+
+I was able to quickly and safely clear up space on my desktop computer! I'm confident that my data is safe because I've got git-annex configured to make sure there's at least two copies of my videos, and 3 copies of everything else.
+
+I've finally gone from either keeping hard drives from my past computers or keeping a full copy of the data, to having a proper archiving and backup system for all files I care about.
+
+I've got over 12,000 files stored on my main annex, and it's perfectly responsive.
+
+Joey does a great job, and has been very responsive on the rare occasions where I needed help.
+
+Three cheers!
+
+-- 
+Jason
+"""]]
diff --git a/doc/design/assistant/blog/day_288__success_stories/comment_16_911c6d2764906cad7d6324835441ed34._comment b/doc/design/assistant/blog/day_288__success_stories/comment_16_911c6d2764906cad7d6324835441ed34._comment
new file mode 100644
index 0000000000..64d17f8721
--- /dev/null
+++ b/doc/design/assistant/blog/day_288__success_stories/comment_16_911c6d2764906cad7d6324835441ed34._comment
@@ -0,0 +1,12 @@
+[[!comment format=mdwn
+ username="https://dzsino.myopenid.com/"
+ nickname="dzsino"
+ subject="thanks!"
+ date="2013-06-30T13:10:35Z"
+ content="""
+I'm using git-annex to consolidate all the stuff that accumulated for years on various drives I have. Music I downloaded and bought (don't wanna loose those), photos from 2003 and on, archives of school and work stuff etc. Using it for 6-7 months now, I came to trust git-annex, I very much like the command line robustness it has and I really like that all my files are accessible in a single tree. It would be cool to have some metadata storage (for file dates, tags and such) and maybe spotlight-like search (elastic-search comes to mind).. 
+Still, without these, it has a great problem-solution fit for me!
+
+Thanks Joey for all the great work, I really hope you can continue improving it, I would gladly join to a second kickstarter funding!
+
+"""]]
diff --git a/doc/design/assistant/blog/day_288__success_stories/comment_17_eb6aa8af5aa70877255a11d132d51aba._comment b/doc/design/assistant/blog/day_288__success_stories/comment_17_eb6aa8af5aa70877255a11d132d51aba._comment
new file mode 100644
index 0000000000..0586afab57
--- /dev/null
+++ b/doc/design/assistant/blog/day_288__success_stories/comment_17_eb6aa8af5aa70877255a11d132d51aba._comment
@@ -0,0 +1,10 @@
+[[!comment format=mdwn
+ username="GLITTAH"
+ ip="50.23.191.90"
+ subject="Several annexes managing >3TB. No problems!"
+ date="2013-07-02T00:12:27Z"
+ content="""
+I've got an annex with all my DVD ISOs that's nearly 3TB alone, uses the SHA512 hash and gets split over three computers, 6 external SATA drives and 3 USB drives.  No errors so far, and the issues with git-annex I've had have all been addressed very quickly (thanks Joey!) and in a non-destructive manner so I don't have to re-init a repo.  I also use it to manage my music library, which allows to me to sleep at night without worrying about a corrupted file spreading through all my backups over time since git-annex has a fsck and hashing utility.  I can't imagine using anything else for managing collections of files: it offloads the task of keeping track of, getting, verifying, and general management of files so I don't even need to worry about the boring stuff, I just get around to watching/listening to my media collection.
+
+Thanks for all the work, it's changed my workflow and file management (I get more sleep too)!
+"""]]
diff --git a/doc/design/assistant/blog/day_288__success_stories/comment_18_9a57de4cea407a73b2d023d85afdccc6._comment b/doc/design/assistant/blog/day_288__success_stories/comment_18_9a57de4cea407a73b2d023d85afdccc6._comment
new file mode 100644
index 0000000000..1e42772772
--- /dev/null
+++ b/doc/design/assistant/blog/day_288__success_stories/comment_18_9a57de4cea407a73b2d023d85afdccc6._comment
@@ -0,0 +1,12 @@
+[[!comment format=mdwn
+ username="http://joeyh.name/"
+ ip="4.154.4.193"
+ subject="comment 18"
+ date="2013-07-03T15:51:48Z"
+ content="""
+Another good tweet: --[[Joey]]
+
+My google Reader replacement: newsbeuter with the data directory stored in an autosynced git-annex repo.
+
+
+"""]]
diff --git a/doc/design/assistant/blog/day_288__success_stories/comment_19_1767c86067bee35941004282b96b8e95._comment b/doc/design/assistant/blog/day_288__success_stories/comment_19_1767c86067bee35941004282b96b8e95._comment
new file mode 100644
index 0000000000..93bfb03e6f
--- /dev/null
+++ b/doc/design/assistant/blog/day_288__success_stories/comment_19_1767c86067bee35941004282b96b8e95._comment
@@ -0,0 +1,10 @@
+[[!comment format=mdwn
+ username="GLITTAH"
+ ip="176.31.181.25"
+ subject="Forgot to add..."
+ date="2013-07-03T19:28:42Z"
+ content="""
+Forgot to add: I'm horrible at keeping things organized.  Git-annex is a huge help with that -- it keeps track of files and the number of their copies.  When I first started using it, I had old songs duplicated across 4 drives, while 2 of those drives had old tags and were obsolete, and one drive had the correct tags, but not the correct filenames.  Every few years I'd have to go through several TB of media files, hand checking each of them to see what was old, what was unwanted, etc.  Once my media libraries got larger than consumer-grade HDD's, I had to find a better solution.  Git-annex removes that burden, and keeps things organized for me.  Once I get an album or movie I tag/rename it the way I want, throw it in the annex and let my computer do the boring and tedious parts like automatically verifying and making sure there's enough copies.  If I change a file, that change propagates through all my repo clones.  It wasn't until after I started using git-annex that I realized how stressed out I was getting over simply maintaining a media library.
+
+If you do another fundraiser you've got my support!
+"""]]
diff --git a/doc/design/assistant/blog/day_288__success_stories/comment_1_22b28ca3d4d3283ad8c21ae052fb9752._comment b/doc/design/assistant/blog/day_288__success_stories/comment_1_22b28ca3d4d3283ad8c21ae052fb9752._comment
new file mode 100644
index 0000000000..494cb798a5
--- /dev/null
+++ b/doc/design/assistant/blog/day_288__success_stories/comment_1_22b28ca3d4d3283ad8c21ae052fb9752._comment
@@ -0,0 +1,11 @@
+[[!comment format=mdwn
+ username="http://edheil.wordpress.com/"
+ ip="99.54.57.201"
+ subject="comment 1"
+ date="2013-06-25T22:26:27Z"
+ content="""
+I've been real quiet cause everything's been working great. :)
+
+One thing I have not messed with yet is the whole XMPP thing.  I mostly don't share between two machines that are online at the same time, I just use git annex for backups and to have content \"optionally\" available.  I should give XMPP a try sometime.
+
+"""]]
diff --git a/doc/design/assistant/blog/day_288__success_stories/comment_20_1d47f3e1b9f0081649cedae4288bac83._comment b/doc/design/assistant/blog/day_288__success_stories/comment_20_1d47f3e1b9f0081649cedae4288bac83._comment
new file mode 100644
index 0000000000..501cf561f8
--- /dev/null
+++ b/doc/design/assistant/blog/day_288__success_stories/comment_20_1d47f3e1b9f0081649cedae4288bac83._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="https://www.google.com/accounts/o8/id?id=AItOawmhfodZquCI_EEl-f3h7HkROTszlsQL6yA"
+ nickname="Joe"
+ subject="comment 20"
+ date="2013-07-04T12:41:35Z"
+ content="""
+I'm using git-annex to make videos, pictures and music available on all my devices (windows, android, linux). It has saved me on trips with my 2yr daughter when she's had meltdowns; I have her favorite movie ready. I quickly exceeded the free dropbox quota and git-annex has replaced the need and gives me greater confidence and control over my data. Thank you!
+"""]]
diff --git a/doc/design/assistant/blog/day_288__success_stories/comment_21_31d3f58cad83cb1ecc4821a15ca258d8._comment b/doc/design/assistant/blog/day_288__success_stories/comment_21_31d3f58cad83cb1ecc4821a15ca258d8._comment
new file mode 100644
index 0000000000..99c6c14af4
--- /dev/null
+++ b/doc/design/assistant/blog/day_288__success_stories/comment_21_31d3f58cad83cb1ecc4821a15ca258d8._comment
@@ -0,0 +1,14 @@
+[[!comment format=mdwn
+ username="https://www.google.com/accounts/o8/id?id=AItOawm0sGxsiJ7yj5iQsF-A5tEl6XKOGQieqEo"
+ nickname="Matthew"
+ subject="Writing papers"
+ date="2013-07-04T15:32:24Z"
+ content="""
+I'm a graduate student, so I write a lot.  I started using git to manage my latex sources a while ago, but I didn't like gumming up my repository with a bunch of figures (large binaries). I was constantly forgetting to scp the latest versions of my figures from here to there (as I do my work on a number of different computers.)  Git annex gives me a good way to version the figures and manage them with the same tool I'm using for the text.  
+
+My use case seems pretty far from typical, judging from the other comments here, so I thought I should add my two cents!  I'm constantly trying to sell this approach to my fellow students.
+
+Also, I'm going to slip in a plug for another great Haskell project here -- Pandoc. I work with people who suggest changes to my drafts, but only using Word's \"Track Changes\".  So now I'm generating Word docs from Pandoc markdown for my collaborators to edit.  When it comes time to publish, I can convert my markdown to latex and clean up the formatting.  
+
+Why are all the really cool projects written in Haskell? :)  
+"""]]
diff --git a/doc/design/assistant/blog/day_288__success_stories/comment_22_b512bd2bf29dfaab6b36bf204518fdb6._comment b/doc/design/assistant/blog/day_288__success_stories/comment_22_b512bd2bf29dfaab6b36bf204518fdb6._comment
new file mode 100644
index 0000000000..e7a62fd019
--- /dev/null
+++ b/doc/design/assistant/blog/day_288__success_stories/comment_22_b512bd2bf29dfaab6b36bf204518fdb6._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="http://sunny256.sunbase.org/"
+ nickname="sunny256"
+ subject="git-annex is teh awesome"
+ date="2013-07-15T12:37:54Z"
+ content="""
+Just like Git, git-annex is teh awesome. I've used it for almost a year now, and I've never had more control of all my stuff. I store all my valuable (and also not so valuable) files in it, more than a terabyte, and it's rapidly growing. Movies, pictures, recordings, software, podcasts, DVDs and CDs, you name it. It's rock solid with a really good design. Joey says it isn't a backup solution, but I strongly disagree. My data have never been safer. git-annex ensures that I have several copies in different locations, I always know where all the copies are, and the disk space is used way more efficiently now that I don't have unnecessary duplicates everywhere anymore.
+"""]]
diff --git a/doc/design/assistant/blog/day_288__success_stories/comment_2_343333356de20e170edb8020faa7400d._comment b/doc/design/assistant/blog/day_288__success_stories/comment_2_343333356de20e170edb8020faa7400d._comment
new file mode 100644
index 0000000000..768469e3e0
--- /dev/null
+++ b/doc/design/assistant/blog/day_288__success_stories/comment_2_343333356de20e170edb8020faa7400d._comment
@@ -0,0 +1,10 @@
+[[!comment format=mdwn
+ username="https://launchpad.net/~subito"
+ nickname="subito"
+ subject="Big success!"
+ date="2013-06-26T07:00:15Z"
+ content="""
+Annex is working great for me since 8/2012 or something... Using it a lot and the Assistant is awesome!
+
+If you would raise some more funding, I'd be more than happy to support you once more - really great software!
+"""]]
diff --git a/doc/design/assistant/blog/day_288__success_stories/comment_3_4e4034bec789543b562ac263df3e21dd._comment b/doc/design/assistant/blog/day_288__success_stories/comment_3_4e4034bec789543b562ac263df3e21dd._comment
new file mode 100644
index 0000000000..c8ae3d7850
--- /dev/null
+++ b/doc/design/assistant/blog/day_288__success_stories/comment_3_4e4034bec789543b562ac263df3e21dd._comment
@@ -0,0 +1,15 @@
+[[!comment format=mdwn
+ username="https://www.google.com/accounts/o8/id?id=AItOawl3A5oeZFLreGhDTFVMOJYIy1auKiTL_ZY"
+ nickname="Maggie"
+ subject=":)"
+ date="2013-06-26T15:04:45Z"
+ content="""
+Poets aren't humble people.
+We revel in our own narcissism.
+So being mentioned in my brother's blog
+was great for me.
+
+I am proud of the success of Git Annex.
+
+(My poems tend to be _much_ better than this.)
+"""]]
diff --git a/doc/design/assistant/blog/day_288__success_stories/comment_4_0c52794c77a9b7afc5112f5edf9cb793._comment b/doc/design/assistant/blog/day_288__success_stories/comment_4_0c52794c77a9b7afc5112f5edf9cb793._comment
new file mode 100644
index 0000000000..f285863f51
--- /dev/null
+++ b/doc/design/assistant/blog/day_288__success_stories/comment_4_0c52794c77a9b7afc5112f5edf9cb793._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="http://nicolas-schodet.myopenid.com/"
+ ip="2a01:e35:8ae6:f130:1e4b:d6ff:fe78:1ddb"
+ subject="Thank you for this great tool!"
+ date="2013-06-26T18:10:05Z"
+ content="""
+I use git-annex for my picture collection. I can add pictures from my computers, or my FTP server when someone wants to send me their pictures. Then I push them to my online gallery. Git annex is great to keep all my repository synchronized while not requesting that all my pictures are present on my laptop.  Thank you!
+"""]]
diff --git a/doc/design/assistant/blog/day_288__success_stories/comment_5_7ca419aa3a187857b19268572d5df297._comment b/doc/design/assistant/blog/day_288__success_stories/comment_5_7ca419aa3a187857b19268572d5df297._comment
new file mode 100644
index 0000000000..1dd6fa4a98
--- /dev/null
+++ b/doc/design/assistant/blog/day_288__success_stories/comment_5_7ca419aa3a187857b19268572d5df297._comment
@@ -0,0 +1,18 @@
+[[!comment format=mdwn
+ username="http://joeyh.name/"
+ ip="4.152.108.193"
+ subject="comment 5"
+ date="2013-06-26T19:39:15Z"
+ content="""
+Excerpt from an email I received: --[[Joey]]
+
+Another story, but without the assistant this time: I've \"designed\" a
+small f2f file sharing network with it, I'm using it with a friend to
+share accross several computer. This is basically a workflow
+with git annex and gitoline, the flexibility of git and the capacities
+of git annex just gave us this out of the box. Some specs are here if
+you are curious
+http://worlddomination.be/ideas/2012/idea-n-21-build-a-f2f-sharing-network-using
+-git-annex.html
+
+"""]]
diff --git a/doc/design/assistant/blog/day_288__success_stories/comment_6_3edd56b3b04f19faba8d75cca285a662._comment b/doc/design/assistant/blog/day_288__success_stories/comment_6_3edd56b3b04f19faba8d75cca285a662._comment
new file mode 100644
index 0000000000..f08fce4b3a
--- /dev/null
+++ b/doc/design/assistant/blog/day_288__success_stories/comment_6_3edd56b3b04f19faba8d75cca285a662._comment
@@ -0,0 +1,10 @@
+[[!comment format=mdwn
+ username="andy"
+ ip="99.127.140.99"
+ subject="Another success story"
+ date="2013-06-27T02:03:00Z"
+ content="""
+I'm using Ubuntu, and I do a clean install about every 6 months when the new versions come out. I also have 2-3 systems I need to keep in sync. I use git annex to store files I want to keep between OS reinstalls, and to sync the various systems I need data on. I also use it to backup data to the cloud and to bup. I have 6+ repos, 235 GB of data, and 31,029 known annex keys (thanks `git annex map` and `git annex status`!); nevertheless git annex is generally pretty cooperative. I'm not using the assistant yet, but I'd like to eventually. I almost don't have to think about the stuff annex handles. It's really fun to install a fresh OS, clone the git annex repo, and have my entire annex directory tree available immediately.
+
+Thanks Joey! You've done a great job!
+"""]]
diff --git a/doc/design/assistant/blog/day_288__success_stories/comment_7_146331ae2de25a6dc3595dffab9514de._comment b/doc/design/assistant/blog/day_288__success_stories/comment_7_146331ae2de25a6dc3595dffab9514de._comment
new file mode 100644
index 0000000000..61d581380d
--- /dev/null
+++ b/doc/design/assistant/blog/day_288__success_stories/comment_7_146331ae2de25a6dc3595dffab9514de._comment
@@ -0,0 +1,12 @@
+[[!comment format=mdwn
+ username="http://joeyh.name/"
+ ip="4.152.108.193"
+ subject="comment 7"
+ date="2013-06-27T06:31:29Z"
+ content="""
+Amazing tweet I found! --[[Joey]]
+
+Git-annex now auto-syncing photos from my android phone to a Tor hidden SSH service I control (via @guardianproject's Orbot) #prismbreak
+
+
+"""]]
diff --git a/doc/design/assistant/blog/day_288__success_stories/comment_8_72be9307e75eb120451f3d6ab7c8165e._comment b/doc/design/assistant/blog/day_288__success_stories/comment_8_72be9307e75eb120451f3d6ab7c8165e._comment
new file mode 100644
index 0000000000..ab482d179f
--- /dev/null
+++ b/doc/design/assistant/blog/day_288__success_stories/comment_8_72be9307e75eb120451f3d6ab7c8165e._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="https://www.google.com/accounts/o8/id?id=AItOawnAvbXOnK57sqgvZvxkbG74NUKBDwKDcuk"
+ nickname="Tim"
+ subject="Backups"
+ date="2013-06-27T13:14:43Z"
+ content="""
+We use rsync and tar to backup to a server on the local network, and since we discoverd git-annex, we use this mechanism to automatically put all this data in both a local bup repo and a remote one. This gives us a daily full backup, with history available, both locally and off-site. Thanks to bup our 4T of data gets deduplicated to about 270G, only a couple of megs get transferred every night, and thanks to git-annex everything just works, like it should!
+"""]]
diff --git a/doc/design/assistant/blog/day_288__success_stories/comment_9_c27eb0a4181e85a3eed41130402350bf._comment b/doc/design/assistant/blog/day_288__success_stories/comment_9_c27eb0a4181e85a3eed41130402350bf._comment
new file mode 100644
index 0000000000..cf77845b53
--- /dev/null
+++ b/doc/design/assistant/blog/day_288__success_stories/comment_9_c27eb0a4181e85a3eed41130402350bf._comment
@@ -0,0 +1,12 @@
+[[!comment format=mdwn
+ username="https://www.google.com/accounts/o8/id?id=AItOawlOc-EOD5ZyggsAp6lOnU7x5MxizwLtUXA"
+ nickname="Hendrik"
+ subject="more success"
+ date="2013-06-27T14:09:47Z"
+ content="""
+Hi,
+
+I have moved all my photographs to an annex running on a Ubuntu file server while doing photo editing, presenting etc. on my MacBook. The annex is great in selectively moving files between these machines for editing and presentation purposes. I started using it around christmas season last year and added a glacier \"backup\" recently. My statistics: 6 repositories in total, more than 300GB (dedup'd from 380GB), 60.000 local keys.
+
+What is missing from my point of view (and I would happily join crowd funding): stable windows and android versions, managing selective transfers from the assistant (select folders to be transferred from/to a specific repository), options for running the annex as a server daemon with web GUI (authentication to GUI + fixed URLs, status reports on GUI, etc.). All these would dramatically increase the WAF.
+"""]]
diff --git a/doc/design/assistant/blog/day_289__back_in_the_swing.mdwn b/doc/design/assistant/blog/day_289__back_in_the_swing.mdwn
new file mode 100644
index 0000000000..75923f8978
--- /dev/null
+++ b/doc/design/assistant/blog/day_289__back_in_the_swing.mdwn
@@ -0,0 +1,16 @@
+Came up with a fix for the gnucash hard linked file problem that makes the
+assistant notice the files gnucash writes. This is not full hard link support;
+hard linked files still don't cleanly sync around. But new hard links to
+files are noticed and added, which is enough to support gnucash.
+
+Spent around 4 hours on reproducing and trying to debug
+[[bugs/Hanging_on_install_on_Mountain_lion]]. It seems that recent upgrades
+of the OSX build machine led to this problem. And indeed, building with an
+older version of Yesod and Warp seems to have worked around the problem. So
+I updated the OSX build for the last release. I will have to re-install the
+new Yesod on my laptop and investigate further -- is this an OSX specific
+problem, or does it affect Linux? Urgh, this is the second hang I've
+encountered involving Warp..
+
+Got several nice [[success stories|day_288__success_stories]], but I don't
+think I've seen *yours* yet. ;) Please post!
diff --git a/doc/design/assistant/blog/day_290__https_release.mdwn b/doc/design/assistant/blog/day_290__https_release.mdwn
new file mode 100644
index 0000000000..ea017d2173
--- /dev/null
+++ b/doc/design/assistant/blog/day_290__https_release.mdwn
@@ -0,0 +1,17 @@
+Spent too many hours last night tracking down a bug that caused the webapp
+to hang when it got built with the new yesod 1.2 release. Much of that time
+was spent figuring out that yesod 1.2 was causing the problem. It turned out to
+be a stupid typo in my yesod compatability layer. `liftH = liftH` in
+Haskell is an infinite loop, not the stack overflow you get in most
+languages. ;)
+
+Even though it's only been a week since the last release,
+that was worth pushing a release out for, which I've just done.
+This release is essentially all bug fixes (aside from the automatic 
+ionice and nicing of the daemon).
+
+This website is now available over https. Perhaps more importantly, all the
+links to download git-annex builds are https by default.
+
+The [[success_stories|day_288__success_stories]] list is getting really
+nice. Only way it could possibly be nicer is if you added your story! Hint. ;)
diff --git a/doc/design/assistant/blog/day_291__--all.mdwn b/doc/design/assistant/blog/day_291__--all.mdwn
new file mode 100644
index 0000000000..0a5584182d
--- /dev/null
+++ b/doc/design/assistant/blog/day_291__--all.mdwn
@@ -0,0 +1,32 @@
+I've felt for a while that git-annex needed better support for managing
+the contents of past versions of files that are stored in the annex. I know
+some people get confused about whether git-annex even supports old versions
+of files (it does, but you should use indirect mode; direct mode doesn't
+guarantee old versions of files will be preserved).
+
+So today I've worked on adding command-line power for managing past
+versions: a new `--all` option.
+
+So, if you want to copy every version of every file in your repository to
+an archive, you can run `git annex copy --all --to archive`.
+Or if you've got a repository on a drive that's dying, you can run
+`git annex copy --all --to newdrive`, and then on the new drive, run `git
+annex fsck --all` to check all the data.
+
+In a bare repository, `--all` is default, so you can run `git annex get`
+inside a bare repository and it will try to get every version of every file
+that it can from the remotes.
+
+The tricky thing about `--all` is that since it's operating on objects and
+not files, it can't check `.gitattributes` settings, which are tied to the
+file name. I worried for a long time that adding `--all` would make
+annex.numcopies settings in those files not be honored, and that this would
+be a Bad Thing. The solution turns out to be simple: I just didn't
+implement `git annex drop --all`! Dropping is the only action that needs to
+check numcopies (move can also reduce the number of copies, but explicitly
+bypasses numcopies settings).
+
+I also added an `--unused` option. So if you have a repository that has
+been accumulating history, and you'd like to move all file contents not
+currently in use to a central server, you can run `git annex unused; git
+annex move --unused --to origin`
diff --git a/doc/design/assistant/blog/day_291__--all/comment_1_eaa9fef19a035bef9c439e87d47c834b._comment b/doc/design/assistant/blog/day_291__--all/comment_1_eaa9fef19a035bef9c439e87d47c834b._comment
new file mode 100644
index 0000000000..60cb4abb0d
--- /dev/null
+++ b/doc/design/assistant/blog/day_291__--all/comment_1_eaa9fef19a035bef9c439e87d47c834b._comment
@@ -0,0 +1,17 @@
+[[!comment format=mdwn
+ username="https://me.yahoo.com/a/2grhJvAC049fJnvALDXek.6MRZMTlg--#eec89"
+ nickname="John"
+ subject="comment 1"
+ date="2013-07-03T22:16:32Z"
+ content="""
+Excellent stuff!  Another thing I'd like to see is that I find myself doing the following all the time:
+
+    git annex copy --to titan --not --in titan
+
+Meaning: copy everything to titan that it doesn't already have.  This operation is very quick.  But I write just:
+
+    git annex copy --to titan
+
+This operation is extremely slow, since it appears to scan through every file in the repository.  Is there a way to have the latter imply the former?  I wouldn't ever want to copy a file that it already has anyway...
+
+"""]]
diff --git a/doc/design/assistant/blog/day_291__--all/comment_2_90bbc26bf92048de7cbaf5fb719c9593._comment b/doc/design/assistant/blog/day_291__--all/comment_2_90bbc26bf92048de7cbaf5fb719c9593._comment
new file mode 100644
index 0000000000..6838ac0700
--- /dev/null
+++ b/doc/design/assistant/blog/day_291__--all/comment_2_90bbc26bf92048de7cbaf5fb719c9593._comment
@@ -0,0 +1,11 @@
+[[!comment format=mdwn
+ username="sfs"
+ ip="65.204.1.102"
+ subject="See history of a file"
+ date="2013-07-07T00:49:35Z"
+ content="""
+It's great that you are addressing the management of older versions. I am new to git-annex but couldn't find much information about this topic. Is it also possible to make git-annex show you the history of files (with date, size, ...) and where (which repositories) they are stored? Or to retrieve a specific version from a remote repository?
+
+Thanks for this amazing peace of software!
+Stephan
+"""]]
diff --git a/doc/design/assistant/blog/day_291__--all/comment_3_75006e9909425dcbf86415a9f7c90372._comment b/doc/design/assistant/blog/day_291__--all/comment_3_75006e9909425dcbf86415a9f7c90372._comment
new file mode 100644
index 0000000000..748db49f38
--- /dev/null
+++ b/doc/design/assistant/blog/day_291__--all/comment_3_75006e9909425dcbf86415a9f7c90372._comment
@@ -0,0 +1,10 @@
+[[!comment format=mdwn
+ username="http://joeyh.name/"
+ ip="4.153.254.222"
+ subject="comment 3"
+ date="2013-07-07T17:20:05Z"
+ content="""
+@john I have considered making copy trust the location log for the remote (which is what your --not --in titan does), but this does change its behavior in a subtly way, and IIRC there were scenarios where this is not desirable.
+
+@sfs the best way to look at older versions of files is to use `git checkout` to check out an older version of the repository. You can then use `git annex get`, `git annex whereis`, etc like you usually would, on the old version of files.
+"""]]
diff --git a/doc/design/assistant/blog/day_291__--all/comment_4_5440449bbc5a353f7430f72e19c35e92._comment b/doc/design/assistant/blog/day_291__--all/comment_4_5440449bbc5a353f7430f72e19c35e92._comment
new file mode 100644
index 0000000000..a4739c5e7d
--- /dev/null
+++ b/doc/design/assistant/blog/day_291__--all/comment_4_5440449bbc5a353f7430f72e19c35e92._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="https://www.google.com/accounts/o8/id?id=AItOawl9sYlePmv1xK-VvjBdN-5doOa_Xw-jH4U"
+ nickname="Richard"
+ subject="comment 4"
+ date="2013-07-14T16:03:39Z"
+ content="""
+@john, @joeyh: I like the fact that normal copy takes some time, but that it _verifies_. Default to save, optimize to quick, imo.
+"""]]
diff --git a/doc/design/assistant/blog/day_292__bugfixes.mdwn b/doc/design/assistant/blog/day_292__bugfixes.mdwn
new file mode 100644
index 0000000000..a514afd0e0
--- /dev/null
+++ b/doc/design/assistant/blog/day_292__bugfixes.mdwn
@@ -0,0 +1,24 @@
+Actually spread out over several days..
+
+I think I have finally comprehensively dealt with all the wacky system
+misconfigurations that can make `git commit` complain and refuse to commit.
+The last of these is [a system with a FQDN that doesn't have a dot in it](http://git-annex.branchable.com/bugs/view_logs_fails:_Internal_Server_Error__internal_liftAnnex/). 
+I personally think git should just use the hostname as-is in the email
+address for commits here -- it's better to be robust. Indeed, I think it
+would make more sense if `git commit` never failed, unless it ran out of
+disk or the repo is corrupt. But anyway, `git-annex
+init` will now detect when the commit fails because of this and put a
+workaround in place.
+
+Fixed a bug in `git annex addurl --pathdepth` when the url's path was
+shorter than the amount requested to remove from it.
+
+Tracked down a bug that prevents git-annex from working on a system with an
+old linux kernel. Probably the root cause is that the kernel was built
+without EVENTFD support. Found a workaround to get a usable git-annex on
+such a system is to build it without the webapp, since that disables the
+threaded runtime which triggered the problem.
+
+Dealt with a lot of Windows bugs. Very happy that it's working well enough
+that some users are reporting bugs on it in Windows, and with enough detail
+that I have not needed to boot Windows to fix them so far. ;)
diff --git a/doc/design/assistant/blog/day_292__bugfixes/comment_1_bbac3878d80f7540d229183c56664784._comment b/doc/design/assistant/blog/day_292__bugfixes/comment_1_bbac3878d80f7540d229183c56664784._comment
new file mode 100644
index 0000000000..5aafab12d5
--- /dev/null
+++ b/doc/design/assistant/blog/day_292__bugfixes/comment_1_bbac3878d80f7540d229183c56664784._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="http://cweiske.de/"
+ nickname="cweiske"
+ subject="comment 1"
+ date="2013-07-09T06:52:11Z"
+ content="""
+Thanks for the quick bugfix!
+"""]]
diff --git a/doc/design/assistant/blog/day_292__bugfixes/comment_2_8c9e5291ceb257f3a938af0ad967c5d7._comment b/doc/design/assistant/blog/day_292__bugfixes/comment_2_8c9e5291ceb257f3a938af0ad967c5d7._comment
new file mode 100644
index 0000000000..adb161fd09
--- /dev/null
+++ b/doc/design/assistant/blog/day_292__bugfixes/comment_2_8c9e5291ceb257f3a938af0ad967c5d7._comment
@@ -0,0 +1,10 @@
+[[!comment format=mdwn
+ username="http://nullroute.eu.org/~grawity/"
+ nickname="Mantas"
+ subject="comment 2"
+ date="2013-07-09T13:47:20Z"
+ content="""
+git **used to** use the hostname as-is, but that didn't work out well – many people just kept on committing as `joeyh@kremvax.(none)`, which doesn't make for a good email address or a globally-unique identifier (sometimes by being lazy; sometimes by forgetting to carry their Git config over to a new machine).
+
+The stricter check was implemented in [8c5b1ae1](https://github.com/gitster/git/commit/8c5b1ae1b26a7512eb29e75391b8b24c0d0439e7) for git v1.8.3. Setting `$EMAIL` or running `git -c user.email=foo@bar` should override it.
+"""]]
diff --git a/doc/design/assistant/blog/day_292__bugfixes/comment_3_02f875e8edd30f47939249f16d92712b._comment b/doc/design/assistant/blog/day_292__bugfixes/comment_3_02f875e8edd30f47939249f16d92712b._comment
new file mode 100644
index 0000000000..3d02b447bd
--- /dev/null
+++ b/doc/design/assistant/blog/day_292__bugfixes/comment_3_02f875e8edd30f47939249f16d92712b._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="http://nullroute.eu.org/~grawity/"
+ nickname="Mantas"
+ subject="comment 3"
+ date="2013-07-09T13:48:49Z"
+ content="""
+(I was wrong about the version; it appears to be v1.7.11.)
+"""]]
diff --git a/doc/design/assistant/blog/day_293__gpg_builds.mdwn b/doc/design/assistant/blog/day_293__gpg_builds.mdwn
new file mode 100644
index 0000000000..eb3ccbd25c
--- /dev/null
+++ b/doc/design/assistant/blog/day_293__gpg_builds.mdwn
@@ -0,0 +1,32 @@
+Two gpg fixes today. The OSX Mtn Lion builds were pulling in a build of gpg
+that wanted a gpg-agent to be installed in /usr/local or it wouldn't work.
+I had to build my own gpg on OSX to work around this. I am pondering making
+the OSX dmg builds pull down the gpg source and build their own binary,
+so issues on the build system can't affect it. But would really rather not,
+since maintaining your own version of every dependency on every
+OS is hard (pity about there still being so many OS's without sane package
+management).
+
+On Android, which I have not needed to touch for a month, gpg was built
+with --enable-minimal, which turned out to not be necessary and was
+limiting the encryption algorythms included, and led to
+interoperability problems for some. Fixed that gpg build too.
+
+Also fixed an ugly bug in the webapp when setting up a rsync repository.
+It would configure `~/.ssh/authorized_keys` on the server to force
+git-annex-shell to be run. Which doesn't work for rsync. I didn't notice
+this before because it doesn't affect ssh servers that already have a ssh
+setup that allows accessing them w/o a password.
+
+Spent a while working on a bug that can occur in a non-utf8 locale
+when using special characters in the directory name of a ssh remote.
+I was able to reproduce it, but have not worked out how to fix it; encoding
+issues like this are always tricky.
+
+Added something to the walkthrough to help convince people that yes, you
+can use tags and branches with git-annex just like with regular git. One of
+those things that is so obvious to the developer writing the docs 
+that it's hard to realize it will be a point of concern.
+
+Seems like there is a release worth of changes already, so I plan to push
+it out tomorrow.
diff --git a/doc/design/assistant/blog/day_293__gpg_builds/comment_1_4f152de8ea5aca4ec381d439e2a821f7._comment b/doc/design/assistant/blog/day_293__gpg_builds/comment_1_4f152de8ea5aca4ec381d439e2a821f7._comment
new file mode 100644
index 0000000000..82563a41d3
--- /dev/null
+++ b/doc/design/assistant/blog/day_293__gpg_builds/comment_1_4f152de8ea5aca4ec381d439e2a821f7._comment
@@ -0,0 +1,12 @@
+[[!comment format=mdwn
+ username="https://www.google.com/accounts/o8/id?id=AItOawnXybLxkPMYpP3yw4b_I6IdC3cKTD-xEdU"
+ nickname="Matt"
+ subject="comment 1"
+ date="2013-07-09T07:18:09Z"
+ content="""
+Hi, glad you added the git history and branch info to the walk through.  With a 40,000 foot view some of the wording in the docs (whilst correct) may lead to the wrong impression.  For example, in the unused data section, there is a sense that it seems bad that data can accumulate and doing a \"git rm\" rather than doing a \"git annex drop\" is a naughty thing to do (I know you don't actually say this).
+
+I think it needs to made clear that \"removing\" files is more subtle than in a traditional file-system: that there is a choice of deleting from the working tree so that contents remain and can be recovered or removing the file and contents labeled together (a.k.a \"dropping\").  With dropping so pervasive in the docs its easy to think this \"right\" (and only) way.  
+
+I wonder if a \"git annex rm\" which is simply an alias for \"git rm\" might be a good idea for a UI perspective...
+"""]]
diff --git a/doc/design/assistant/blog/day_293__gpg_builds/comment_2_42f625638638bc875379f6c604d6f673._comment b/doc/design/assistant/blog/day_293__gpg_builds/comment_2_42f625638638bc875379f6c604d6f673._comment
new file mode 100644
index 0000000000..6f00552e25
--- /dev/null
+++ b/doc/design/assistant/blog/day_293__gpg_builds/comment_2_42f625638638bc875379f6c604d6f673._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="LP"
+ ip="72.227.131.46"
+ subject="Just to say.."
+ date="2013-07-10T23:01:56Z"
+ content="""
+These updates are fantastic. They're great demonstrations of transparency in work, fun to read as a kstarter contributor, and interesting as a copypasta coder. Bravo.
+"""]]
diff --git a/doc/design/assistant/blog/day_294__release_day.mdwn b/doc/design/assistant/blog/day_294__release_day.mdwn
new file mode 100644
index 0000000000..b64b91ae50
--- /dev/null
+++ b/doc/design/assistant/blog/day_294__release_day.mdwn
@@ -0,0 +1,7 @@
+Got the release out, after fixing test suite and windows build breakage.
+This release has all the features on the command line side (--all,
+--unused, etc), but several bugfixes on the assistant side, and a lot
+of Windows bug fixes.
+
+I've spent this evening adding icons to git-annex on Linux.
+Even got the Linux standalone tarball to automatically install icons.
diff --git a/doc/design/assistant/blog/day_295__balls_in_the_air.mdwn b/doc/design/assistant/blog/day_295__balls_in_the_air.mdwn
new file mode 100644
index 0000000000..ebbad2973a
--- /dev/null
+++ b/doc/design/assistant/blog/day_295__balls_in_the_air.mdwn
@@ -0,0 +1,13 @@
+Been keeping several non-coding balls in the air recently, two of which
+landed today.
+
+First, Rsync.net is [offering a discount to all git-annex users](http://www.rsync.net/products/git-annex-pricing.html),
+at one third their normal price.
+"People using git-annex are clueful and won't be a big support burden for us,
+so it's a win-win."
+The web app will be updated offer the discount when setting up a rsync.net
+repository.
+
+Secondly, I've recorded an interview today for the Git Minutes podcast,
+about git-annex. Went well, looking forward to it going up, probably on
+Monday.
diff --git a/doc/design/assistant/blog/day_296__new_crowdfunding_campaign.mdwn b/doc/design/assistant/blog/day_296__new_crowdfunding_campaign.mdwn
new file mode 100644
index 0000000000..759bde12f0
--- /dev/null
+++ b/doc/design/assistant/blog/day_296__new_crowdfunding_campaign.mdwn
@@ -0,0 +1,41 @@
+Surprise! I'm running a new crowdfunding campaign, which I hope will fund
+several more months of git-annex development.
+
+
+
+Please don't feel you have to give, but if you do decide to, give
+generously. ;) I'm accepting both Paypal and Bitcoin (via CoinBase.com),
+and have some rewards that you might enjoy.
+
+----
+
+I came up with two lists of things I hope this campaign will fund.
+These are by no means complete lists. First, some general features and
+development things:
+
+* Integrate better with Android.
+* Get the assistant and webapp ported to Windows.
+* Refine the automated stress testing tools to find and fix more problems
+  before users ever see them.
+* Automatic recovery. Cosmic ray flipped a bit in a file? 
+  USB drive corrupted itself? The assistant should notice these problems,
+  and fix them.
+* Encourage more contributions from others. For example, improve the
+  special remote plugin interface so it can do everything the native Haskell
+  interface can do. Eight new cloud storage services were added this year
+  as plugins, but we can do better!
+* Use deltas to reduce bandwidth needed to transfer modified versions of files.
+
+Secondly, some things to improve security:
+
+* Add easy support for encrypted git repositories
+  using [git-remote-gcrypt](https://github.com/blake2-ppc/git-remote-gcrypt),
+  so you can safely push to a repository on a server you don't control.
+* Add support for setting up and using GPG keys in the webapp.
+* Add protection to the XMPP protocol to guard against man in the middle
+  attacks if the XMPP server is compromised. Ie, Google should not be able to
+  learn about your git-annex repository even if you're using their servers.
+* To avoid leaking even the size of your encrypted files to
+  cloud storage providers, add a mode that stores fixed size chunks.
+
+It will also, of course, fund ongoing bugfixing, support, etc.
diff --git a/doc/design/assistant/blog/day_296__new_crowdfunding_campaign/comment_1_cccad1a5103c504d21d0f8e69bb39e1b._comment b/doc/design/assistant/blog/day_296__new_crowdfunding_campaign/comment_1_cccad1a5103c504d21d0f8e69bb39e1b._comment
new file mode 100644
index 0000000000..71cfd60c4e
--- /dev/null
+++ b/doc/design/assistant/blog/day_296__new_crowdfunding_campaign/comment_1_cccad1a5103c504d21d0f8e69bb39e1b._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="EskildHustvedt"
+ ip="80.202.103.73"
+ subject="Shipping costs"
+ date="2013-07-15T05:07:07Z"
+ content="""
+Hi, it doesn't mention anything about offsetting shipping costs for people located outside of the US (like I am). Any chance you could add that?
+"""]]
diff --git a/doc/design/assistant/blog/day_296__new_crowdfunding_campaign/comment_2_4fef7bd9c8e15cd57df365fadb95717f._comment b/doc/design/assistant/blog/day_296__new_crowdfunding_campaign/comment_2_4fef7bd9c8e15cd57df365fadb95717f._comment
new file mode 100644
index 0000000000..57df8f9956
--- /dev/null
+++ b/doc/design/assistant/blog/day_296__new_crowdfunding_campaign/comment_2_4fef7bd9c8e15cd57df365fadb95717f._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="http://joeyh.name/"
+ ip="4.152.246.187"
+ subject="comment 2"
+ date="2013-07-15T05:26:54Z"
+ content="""
+Shipping is minimal, either $1 or $1.50 outside the US. It's factored into the reward levels, basically.
+"""]]
diff --git a/doc/design/assistant/blog/day_296__new_crowdfunding_campaign/comment_3_0b9258a1f5079e53c60138f06d0c63b1._comment b/doc/design/assistant/blog/day_296__new_crowdfunding_campaign/comment_3_0b9258a1f5079e53c60138f06d0c63b1._comment
new file mode 100644
index 0000000000..fc8754502f
--- /dev/null
+++ b/doc/design/assistant/blog/day_296__new_crowdfunding_campaign/comment_3_0b9258a1f5079e53c60138f06d0c63b1._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="https://www.google.com/accounts/o8/id?id=AItOawm3RqHNOKdnMCRF5V4306ON55XyJtQNlpY"
+ nickname="Marco"
+ subject="BitSync"
+ date="2013-07-15T13:11:03Z"
+ content="""
+How about using torrent to retrive files from a annex, something like BitSync would be great. Nice job Joey! Thank you.
+"""]]
diff --git a/doc/design/assistant/blog/day_296__new_crowdfunding_campaign/comment_4_46183b97ca904bc06e46569c30db2edc._comment b/doc/design/assistant/blog/day_296__new_crowdfunding_campaign/comment_4_46183b97ca904bc06e46569c30db2edc._comment
new file mode 100644
index 0000000000..3febd091ec
--- /dev/null
+++ b/doc/design/assistant/blog/day_296__new_crowdfunding_campaign/comment_4_46183b97ca904bc06e46569c30db2edc._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="https://dzsino.myopenid.com/"
+ nickname="dzsino"
+ subject="mo' stickers, less problems :)"
+ date="2013-07-16T09:56:03Z"
+ content="""
+Hi Joey, since shipping would burn 10-15% of my funding, I added another $10, so you can just batch the stickers together. I hope it's cheaper this way.. Thanks!
+"""]]
diff --git a/doc/design/assistant/blog/day_297__back_to_work.mdwn b/doc/design/assistant/blog/day_297__back_to_work.mdwn
new file mode 100644
index 0000000000..c8dbedc45f
--- /dev/null
+++ b/doc/design/assistant/blog/day_297__back_to_work.mdwn
@@ -0,0 +1,16 @@
+It looks like I'm funded for at least the next 9 months! It would still be
+nice to get to a year. ;) 
+
+Working to get caught up on recent bug reports..
+
+Made `git annex uninit` not nuke anything that's left over in
+`.git/annex/objects` after unannexing all the files. After all, that could
+be important old versions of files or deleted file, and just because the
+user wants to stop using git-annex, doesn't mean git-annex shouldn't try to
+protect that data with its dying breath. So it prints out some suggestions
+in this case, and leaves it up to the user to decide what to do with the
+data.
+
+Fixed the Android autobuilder, which had stopped including the webapp.
+
+Looks like another autobuilder will be needed for OSX 10.9.
diff --git a/doc/design/assistant/blog/day_297__back_to_work/comment_1_e300feb821bfe7b76b2cec4376d16ffa._comment b/doc/design/assistant/blog/day_297__back_to_work/comment_1_e300feb821bfe7b76b2cec4376d16ffa._comment
new file mode 100644
index 0000000000..4e0042ed53
--- /dev/null
+++ b/doc/design/assistant/blog/day_297__back_to_work/comment_1_e300feb821bfe7b76b2cec4376d16ffa._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="http://grossmeier.net/"
+ nickname="greg"
+ subject="uninit"
+ date="2013-07-16T23:43:38Z"
+ content="""
+Hah, that bit me in the butt when I first started experimenting with git-annex. I didn't lose anything, but it taught me a very important lesson (that I've had to learn many a time).
+"""]]
diff --git a/doc/design/assistant/blog/day_298__exceptional.mdwn b/doc/design/assistant/blog/day_298__exceptional.mdwn
new file mode 100644
index 0000000000..6be34821e6
--- /dev/null
+++ b/doc/design/assistant/blog/day_298__exceptional.mdwn
@@ -0,0 +1,21 @@
+Theme today seems to be fun with exceptions. 
+
+Fixed an uncaught exception that could crash the assistant's Watcher thread
+if just the right race occurred.
+
+Also fixed it to not throw an exception if another process is
+already transferring a file. What this means is that if you run multiple 
+`git annex get` processes on the same files, they'll cooperate in each
+picking their own files to get and download in parallel. (Also works for
+copy, etc.) Especially useful when downloading from an encrypted remote,
+since often one process will be decrypting a file while the other is
+downloading the next file. There is still room for improvement here; 
+a -jN option could better handle ensuring N downloads ran concurrently, and
+decouple decryption from downloading. But it would need the output layer to
+be redone to avoid scrambled output. (All the other stuff to make parallel
+git-annex transfers etc work was already in place for a long time.)
+
+----
+
+Campaign update: Now funded for nearly 10 months, and aiming for a year.
+
diff --git a/doc/design/assistant/blog/day_299__bugfixing.mdwn b/doc/design/assistant/blog/day_299__bugfixing.mdwn
new file mode 100644
index 0000000000..2746151914
--- /dev/null
+++ b/doc/design/assistant/blog/day_299__bugfixing.mdwn
@@ -0,0 +1,8 @@
+Succeeded fixing a few bugs today, and followed up on a lot of other ones..
+
+Fixed checking when content is present in a non-bare repository accessed via
+http.
+
+My changes a few days ago turned out to make uninit leave hard links behind
+in .git/annex. Luckily the test suite caught this bug, and it was easily
+fixed by making uninit delete objects with 2 or more hard links at the end.
diff --git a/doc/design/assistant/blog/day_2__races.mdwn b/doc/design/assistant/blog/day_2__races.mdwn
new file mode 100644
index 0000000000..19f868a712
--- /dev/null
+++ b/doc/design/assistant/blog/day_2__races.mdwn
@@ -0,0 +1,45 @@
+Last night I got `git annex watch` to also handle deletion of files.
+This was not as tricky as feared; the key is using `git rm --ignore-unmatch`,
+which avoids most problematic situations (such as a just deleted file
+being added back before git is run).
+
+Also fixed some races when `git annex watch` is doing its startup scan of
+the tree, which might be changed as it's being traversed. Now only one
+thread performs actions at a time, so inotify events are queued up during
+the scan, and dealt with once it completes. It's worth noting that inotify
+can only buffer so many events .. Which might have been a problem except
+for a very nice feature of Haskell's inotify interface: It has a thread
+that drains the limited inotify buffer and does its own buffering.
+
+----
+
+Right now, `git annex watch` is not as fast as it could be when doing
+something like adding a lot of files, or deleting a lot of files.
+For each file, it currently runs a git command that updates the index.
+I did some work toward coalescing these into one command (which `git annex`
+already does normally). It's not quite ready to be turned on yet,
+because of some races involving `git add` that become much worse
+if it's delayed by event coalescing.
+
+----
+
+And races were the theme of today. Spent most of the day really
+getting to grips with all the fun races that can occur between
+modification happening to files, and `git annex watch`. The [[inotify]]
+page now has a long list of known races, some benign, and several,
+all involving adding files, that are quite nasty.
+
+I fixed one of those races this evening. The rest will probably involve
+moving away from using `git add`, which necessarily examines the file
+on disk, to directly shoving the symlink into git's index.
+
+BTW, it turns out that `dvcs-autosync` has grappled with some of these same
+races: 
+I hope that `git annex watch` will be in a better place to deal with them,
+since it's only dealing with git, and with a restricted portion of it
+relevant to git-annex.
+
+It's important that `git annex watch` be rock solid. It's the foundation
+of the git annex assistant. Users should not need to worry about races
+when using it. Most users won't know what race conditions are. If only I
+could be so lucky!
diff --git a/doc/design/assistant/blog/day_300__new_logo.mdwn b/doc/design/assistant/blog/day_300__new_logo.mdwn
new file mode 100644
index 0000000000..9e13c4b131
--- /dev/null
+++ b/doc/design/assistant/blog/day_300__new_logo.mdwn
@@ -0,0 +1,36 @@
+git-annex has a new nicer versions of its [[logo]], thanks to John Lawrence.
+
+Finally tracked down a week-old bug about the watcher crashing. It turned
+out to crash when it encountered a directory containing a character that's
+invalid in the current locale. I've noticed that 'ü' is often the character I
+get bug reports about. After reproducing the bug I quickly tracked it down
+to code in the haskell hinotify library, and sent in a patch.
+
+Also uploaded a fixed hinotify to Debian, and deployed it to all 3 of the
+autobuilder chroots. That took much more time than actually fixing the bug.
+Quite a lot of yak shaving went on actually. Oh well. The Linux
+autobuilders are updated to use Debian unstable again, which is nice.
+
+Fixed a bug that prevented annex.diskreserve to be honored when storing
+files encrypted in a directory special remote.
+
+Taught the webapp the difference between initializing a new special remote
+and enabling an existing special remote, which fixed some bad behavior when
+it got confused.
+
+----
+
+And then for the really fun bug of the day! A user sent me a large file
+which badly breaks git annex add. Adding the file causes a symlink to be
+set up, but the file's content is not stored in the annex. Indeed, it's
+deleted. This is the first data loss bug since January 2012.
+
+Turns out it was caused by the code that handles the dummy files git uses
+in place of symlinks on FAT etc filesystems. Code that had no business
+running when `core.symlinks=true`. Code that was prone to false positives
+when looking at a tarball of a git-annex repository. So I put in multiple
+fixes for this bug. I'll be making a release on Monday.
+
+----
+
+Today's work was sponsored by Mikhail Barabanov. Thanks, Mikhail!
diff --git a/doc/design/assistant/blog/day_300__new_logo/comment_1_9fc64e33863b9fce00f6a03417a91e36._comment b/doc/design/assistant/blog/day_300__new_logo/comment_1_9fc64e33863b9fce00f6a03417a91e36._comment
new file mode 100644
index 0000000000..714fbc8254
--- /dev/null
+++ b/doc/design/assistant/blog/day_300__new_logo/comment_1_9fc64e33863b9fce00f6a03417a91e36._comment
@@ -0,0 +1,9 @@
+[[!comment format=mdwn
+ username="https://launchpad.net/~mikapflueger"
+ nickname="mikapflueger"
+ subject="The dreaded "ü""
+ date="2013-07-21T20:56:30Z"
+ content="""
+Yeah, with the last name of \"Pflüger\" I also usually find bugs having to do with the letter \"ü\". (-;
+I guess it is simply one of the more frequent german umlauts, especially in names (the classic \"Müller\", which is one of the most common last names in Germany). And apparently the German cabal strikes here again.
+"""]]
diff --git a/doc/design/assistant/blog/day_300__new_logo/comment_2_e8aac0298f90004e81492d2c7f85eda0._comment b/doc/design/assistant/blog/day_300__new_logo/comment_2_e8aac0298f90004e81492d2c7f85eda0._comment
new file mode 100644
index 0000000000..b4bd78e3c9
--- /dev/null
+++ b/doc/design/assistant/blog/day_300__new_logo/comment_2_e8aac0298f90004e81492d2c7f85eda0._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="https://www.google.com/accounts/o8/id?id=AItOawmNu4V5fvpLlBhaCUfXXOB0MI5NXwh8SkU"
+ nickname="Adam"
+ subject="Horns?"
+ date="2013-07-31T19:53:51Z"
+ content="""
+I don't necessarily mean this as a criticism, just an observation. I wasn't a big fan of the old logo, but this new one looks like it has horns. Or a bit like a pitchfork. 
+"""]]
diff --git a/doc/design/assistant/blog/day_300__new_logo/comment_3_6308c767f6e4bf090102191c91520d04._comment b/doc/design/assistant/blog/day_300__new_logo/comment_3_6308c767f6e4bf090102191c91520d04._comment
new file mode 100644
index 0000000000..beb5e281e0
--- /dev/null
+++ b/doc/design/assistant/blog/day_300__new_logo/comment_3_6308c767f6e4bf090102191c91520d04._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="http://joeyh.name/"
+ ip="4.154.0.21"
+ subject="comment 3"
+ date="2013-07-31T20:03:13Z"
+ content="""
+I think the arrows are a bit too pointy. If someone wants to adjust the svg to make the arrows have a bit more of a rounded feel like in the old logo, that'd be fine by me.
+"""]]
diff --git a/doc/design/assistant/blog/day_301__direct_unannex.mdwn b/doc/design/assistant/blog/day_301__direct_unannex.mdwn
new file mode 100644
index 0000000000..c227607d66
--- /dev/null
+++ b/doc/design/assistant/blog/day_301__direct_unannex.mdwn
@@ -0,0 +1,21 @@
+No release today after all. Unexpected bandwidth failure. Maybe in a few
+days..
+
+Got unannex and uninit working in direct mode. This is one of the more
+subtle parts of git-annex, and took some doing to get it right.
+Surprisingly, unannex in direct mode actually turns out to be faster than
+in indirect mode. In direct mode it doesn't have to immediately commit the
+unannexing, it can just stage it to be committed later.
+
+Also worked on the ssh connection caching code. The perrennial problem with
+that code is that the fifo used to communicate with ssh has a small limit
+on its path, somewhere around 100 characters. This had caused problems when
+the hostname was rather long. I found a way to avoid needing to be able to
+reverse back from the fifo name to the hostname, and this let me take the
+md5sum of long hostnames, and use that shorter string for the fifo.
+
+Also various other bug followups.
+
+----
+
+[Campaign](https://campaign.joeyh.name/) is almost to 1 year!
diff --git a/doc/design/assistant/blog/day_302_release_day.mdwn b/doc/design/assistant/blog/day_302_release_day.mdwn
new file mode 100644
index 0000000000..d9dd9ec559
--- /dev/null
+++ b/doc/design/assistant/blog/day_302_release_day.mdwn
@@ -0,0 +1,6 @@
+Got the release out.
+
+I've been working on fleshing out 
+the [[timeline|/design/assistant]] for the next year.
+Including a fairly detailed set of things I want to do around
+[[/design/assistant/disaster_recovery]] in the assistant.
diff --git a/doc/design/assistant/blog/day_302_release_day/comment_1_fe6e572ba706e95188463d9f3e004d03._comment b/doc/design/assistant/blog/day_302_release_day/comment_1_fe6e572ba706e95188463d9f3e004d03._comment
new file mode 100644
index 0000000000..86984c478b
--- /dev/null
+++ b/doc/design/assistant/blog/day_302_release_day/comment_1_fe6e572ba706e95188463d9f3e004d03._comment
@@ -0,0 +1,17 @@
+[[!comment format=mdwn
+ username="https://www.google.com/accounts/o8/id?id=AItOawmnG4EuvZWse5hvgrl0XAK-U61e-0iGaao"
+ nickname="David"
+ subject="Broken dependencies on Debian wheezy "
+ date="2013-07-24T07:21:18Z"
+ content="""
+Hi Joey,
+Having installed this release (x86_64) on Debian wheezy I get the following error when I try to run git-annex:
+    
+    git-annex: /lib/x86_64-linux-gnu/libc.so.6: version `GLIBC_2.14' not found (required by git-annex)
+    git-annex: /lib/x86_64-linux-gnu/libc.so.6: version `GLIBC_2.14' not found (required by $HOME/.local/share/git-annex//usr/lib/x86_64-linux-gnu/libidn.so.11)
+    
+etc. for at least a dozen libraries. Different libs try to find different glibc version (2.14, 2.15, 2.16, 2.17). Previous releases were linked with GLIBC_2.13.
+
+
+
+"""]]
diff --git a/doc/design/assistant/blog/day_303__oops.mdwn b/doc/design/assistant/blog/day_303__oops.mdwn
new file mode 100644
index 0000000000..639e96de8f
--- /dev/null
+++ b/doc/design/assistant/blog/day_303__oops.mdwn
@@ -0,0 +1,8 @@
+Seems I forgot why I was using debian stable chroots to make the
+autobuilds: Lots of people still using old glibc version. Had to rebuild
+the stable chroots that I had upgraded to unstable. Wasted several hours..
+I was able to catch up on recent traffic in between.
+
+Was able to reproduce a bug where `git annex initremote` hung with some
+encrypted special remotes. Turns out to be a deadlock when it's not built
+with the threaded GHC runtime. So I've forced that runtime to be used.
diff --git a/doc/design/assistant/blog/day_304__dropunused_safety.mdwn b/doc/design/assistant/blog/day_304__dropunused_safety.mdwn
new file mode 100644
index 0000000000..28f5dc13c2
--- /dev/null
+++ b/doc/design/assistant/blog/day_304__dropunused_safety.mdwn
@@ -0,0 +1,28 @@
+The big news: Important behavior change in `git annex dropunused`. Now it
+checks, just like `git annex drop`, that it's not dropping the last copy of
+the file. So to lose data, you have to use `--force`. This continues the
+recent theme of making git-annex hold on more tenaciously to old data, and
+AFAIK it was the last place data could be removed without `--force`.
+
+Also a nice little fix to `git annex unused` so it doesn't identify
+temporary files as unused if they're being used to download a file.
+Fixing it was easy thanks to all the transfer logs and locking
+infrastucture built for the assistant.
+
+Fixed a bug in the assistant where even though syncing to a network
+remote was disabled, it would still sync with it every hour, or whenever
+a network connection was detected.
+
+Working on some direct mode scalability problems when thousands of the
+identical files are added. Fixing this may involvie replacing the current
+simple map files with something more scalable like a sqllite database.
+
+While tracking that down, I also found a bug with adding a ton of files
+in indirect mode, that could make the assistant stall.
+Turned out to be a laziness problem. (Worst kind of Haskell bug.) Fixed.
+
+----
+
+Today's sponsor is my sister, Anna Hess, who incidentially just put
+the manuscript of her latest ebook in the family's annex prior to its
+publication on Amazon this weekend.
diff --git a/doc/design/assistant/blog/day_304__dropunused_safety/comment_1_1bbcf6c74b6437c44ff8604401fb1432._comment b/doc/design/assistant/blog/day_304__dropunused_safety/comment_1_1bbcf6c74b6437c44ff8604401fb1432._comment
new file mode 100644
index 0000000000..d697770903
--- /dev/null
+++ b/doc/design/assistant/blog/day_304__dropunused_safety/comment_1_1bbcf6c74b6437c44ff8604401fb1432._comment
@@ -0,0 +1,10 @@
+[[!comment format=mdwn
+ username="https://www.google.com/accounts/o8/id?id=AItOawlUbH3eytydcwlWqv8oauE2Jg4NwcV9uA0"
+ nickname="Anna"
+ subject="A good day to be the sponsor!"
+ date="2013-07-27T12:31:27Z"
+ content="""
+Thanks for mentioning me!  I like being responsible for not allowing things to be deleted accidentally --- just my kind of feature. :-)
+
+As a shameless plug, here's the ebook mentioned above for those of you not privy to the family annex: http://www.amazon.com/Root-Cellar-Vegetables-Simplicity-ebook/dp/B00E6EGS0M/
+"""]]
diff --git a/doc/design/assistant/blog/day_305__interesting_bugs.mdwn b/doc/design/assistant/blog/day_305__interesting_bugs.mdwn
new file mode 100644
index 0000000000..8b76c929d9
--- /dev/null
+++ b/doc/design/assistant/blog/day_305__interesting_bugs.mdwn
@@ -0,0 +1,21 @@
+Worked on 3 interesting bugs today. One I noticed myself while doing tests
+with adding many thousands of files yesterday. Assistant was delaying
+making a last commit of the batch of files, and would only wake up and
+commit them after a further change was made. Turns out this bug was
+introduced in April while improving commit batching when making very large
+commits. I seem to remember someone mentioning this problem at some point,
+but I have not been able to find a bug report to close.
+
+Also tried to reproduce [[this_bug|bugs/utf8]]. Frustrating, because I'm
+quite sure I have made changes that will avoid it happening again,
+but since I still don't know what the root cause was, I can't let it go.
+
+The last bug is
+[[non-repos_in_repositories_list___40__+_other_weird_output__41___from_git_annex_status]]
+and is a most strange thing. Still trying to get a handle on multiple
+aspects of it.
+
+Also various other bug triage. Down to only 10 messages in my git-annex
+folder.  That included merging about a dozen bugs about
+local pairing, that all seem to involve git-annex-shell not being found in
+path. Something is up with that..
diff --git a/doc/design/assistant/blog/day_306__offtopic.mdwn b/doc/design/assistant/blog/day_306__offtopic.mdwn
new file mode 100644
index 0000000000..cf96401d57
--- /dev/null
+++ b/doc/design/assistant/blog/day_306__offtopic.mdwn
@@ -0,0 +1,2 @@
+Technically offtopic, but did a fun side project today:
+
diff --git a/doc/design/assistant/blog/day_307__buuuugs.mdwn b/doc/design/assistant/blog/day_307__buuuugs.mdwn
new file mode 100644
index 0000000000..9cc4cae587
--- /dev/null
+++ b/doc/design/assistant/blog/day_307__buuuugs.mdwn
@@ -0,0 +1,31 @@
+Back to bug squashing. Fixed several, including a long-standing problem on
+OSX that made the app icon seem to "bounce" or not work. Followed up on a
+bunch more.
+
+The 4.20130723 git-annex release turns out to have broken support for
+running on crippled filesystems (Android, Windows). `git annex sync` will
+add dummy symlinks to the annex as if they were regular files, which is
+not good!
+[Recovery instructions](http://git-annex.branchable.com/bugs/regression_in_direct_mode_on_windows_:_weird___96__git_annex_sync__96___behavior/#comment-5d80649f9da85ac2fb505445a41207f5)
+I've updated the Android and Windows builds and recommend an immediate upgrade.
+Will make a formal release on Friday. 
+
+Spent some time improving the test suite on Windows, to catch this bug,
+and fix a bug that was preventing it from testing `git annex sync` on
+Windows.
+
+----
+
+I am getting very frustrated with this "unknown UUID" problem that a dozen
+people have reported. So far nobody has given me enough information to
+reproduce the problem. It seems to have something to do with
+`git-annex-shell` not being found on the remote system that has been either
+local paired with or is being used as a ssh server, but I don't yet
+understand what. I have spent hours today trying various scenarios to break
+git-annex and get this problem to happen.
+
+I certainly can improve the webapp's behavior when a repository's UUID is
+not known. The easiest fix would be to simply not display such
+repositories. Or there could be a UI to try to get the UUID.
+But I'm more interested in fixing the core problem than putting
+in a UI bandaid.
diff --git a/doc/design/assistant/blog/day_308__ssh-agent.mdwn b/doc/design/assistant/blog/day_308__ssh-agent.mdwn
new file mode 100644
index 0000000000..e18ff2ec15
--- /dev/null
+++ b/doc/design/assistant/blog/day_308__ssh-agent.mdwn
@@ -0,0 +1,16 @@
+Turns out ssh-agent is the cause of the unknown UUID bug! I got a tip
+about this from a user, and was quickly able to reproduce the bug that had
+eluded me so long. Anyone who has run `ssh-add` and is using ssh-agent
+would see the bug.
+
+It was easy enough to fix as it turns out. Just need to set IdentitiesOnly
+in .ssh/config where git-annex has set up its own IdentityFile to ensure
+that its special purpose ssh key is used rather than whatever key the
+ssh-agent has loaded into it. I do wonder why ssh behaves this way -- why
+would I set an IdentityFile for a host if I didn't want ssh to use it?
+
+Spent the rest of the day cleaning up after the bug. Since this affects so
+many people, I automated the clean up process. The webapp will
+detect repositories with this problem, and the user just has to click to
+clean up. It'll then correct their .ssh/config and re-enable the
+repository.
diff --git a/doc/design/assistant/blog/day_308__ssh-agent/comment_1_5f0fc810cf1e1cd9b3ddba3cd19bb19d._comment b/doc/design/assistant/blog/day_308__ssh-agent/comment_1_5f0fc810cf1e1cd9b3ddba3cd19bb19d._comment
new file mode 100644
index 0000000000..a40921ef69
--- /dev/null
+++ b/doc/design/assistant/blog/day_308__ssh-agent/comment_1_5f0fc810cf1e1cd9b3ddba3cd19bb19d._comment
@@ -0,0 +1,12 @@
+[[!comment format=mdwn
+ username="https://launchpad.net/~mikapflueger"
+ nickname="mikapflueger"
+ subject="Thank you so much!"
+ date="2013-08-01T01:13:51Z"
+ content="""
+This behaviour of ssh has bugged me for years now (I even think it is an information leak – ssh offers basically all my public ssh keys to every server I try to connect to, even though that public key might be confidential). Although it had nothing to do with git-annex, you helped me a _lot_ by pointing me towards IdentitiesOnly. Really a nice setting!
+
+Cheers,
+
+Mika
+"""]]
diff --git a/doc/design/assistant/blog/day_309__filenames.mdwn b/doc/design/assistant/blog/day_309__filenames.mdwn
new file mode 100644
index 0000000000..30c9b516cb
--- /dev/null
+++ b/doc/design/assistant/blog/day_309__filenames.mdwn
@@ -0,0 +1,17 @@
+Today was a nice reminder that there are no end of bugs lurking in filename
+handling code. 
+
+First, fixed a bug that prevented git-annex from adding
+filenames starting with ":", because that is a special character to git.
+
+Second, discovered that git 1.8.4 rc0 has changed `git-cat-file --batch` in
+a way that makes it impossible to operate on filenames containing spaces.
+This is, IMHO, a reversion, so hopefully my
+[bug report](http://bugs.debian.org/718517) will get it fixed.
+
+Put in a workaround for that, although using the broken version of git
+with a direct mode repository with lots of spaces in file or directory
+names is going to really slow down git-annex, since it often has to fork a
+new git cat-file process for each file.
+
+Release day tomorrow..
diff --git a/doc/design/assistant/blog/day_310__release_day.mdwn b/doc/design/assistant/blog/day_310__release_day.mdwn
new file mode 100644
index 0000000000..1a8a84a662
--- /dev/null
+++ b/doc/design/assistant/blog/day_310__release_day.mdwn
@@ -0,0 +1,18 @@
+Got the release out, with rather a lot of fiddling to fix broken builds on
+various platforms.
+
+Also released a backport to Debian stable. This backport has the assistant,
+although without WebDAV support. Unfortunately it's an old version from
+May, since ghc transitions and issues have kept newer versions out of
+testing so far. Hope that will clear up soon (probably by dropping haskell
+support for s390x), and I can update it to a newer version. 
+If nothing else it allows using direct mode with Debian stable.
+
+Pleased that the git-cat-files bug was quickly fixed by Peff and has
+already been pulled into Junio's release tree!
+
+----
+
+This evening, I've added an interface around the new improved 
+`git check-ignore` in git 1.8.4. The assistant can finally honor `.gitignore`
+files!
diff --git a/doc/design/assistant/blog/day_310__release_day/comment_1_1e008583cebd8e373e83729529914db7._comment b/doc/design/assistant/blog/day_310__release_day/comment_1_1e008583cebd8e373e83729529914db7._comment
new file mode 100644
index 0000000000..749f77d9d2
--- /dev/null
+++ b/doc/design/assistant/blog/day_310__release_day/comment_1_1e008583cebd8e373e83729529914db7._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="http://adamspiers.myopenid.com/"
+ nickname="Adam"
+ subject="This makes me happy :-)"
+ date="2013-08-04T00:46:25Z"
+ content="""
+Glad my work was worth the effort. Thanks for that.
+"""]]
diff --git a/doc/design/assistant/blog/day_311__Windows_porting.mdwn b/doc/design/assistant/blog/day_311__Windows_porting.mdwn
new file mode 100644
index 0000000000..fe14a6e4b6
--- /dev/null
+++ b/doc/design/assistant/blog/day_311__Windows_porting.mdwn
@@ -0,0 +1,10 @@
+Made two big improvements to the Windows port, in just a few hours.
+First, got gpg working, and encrypted special remotes work on Windows.
+Next, fixed a permissions problem that was breaking removing files
+from directory special remotes on Windows.
+(Also cleaned up a lot of compiler warnings on Windows.)
+
+I think I'm almost ready to move the Windows port from alpha to beta
+status. The only really bad problem that I know of with using it is that
+due to a lack of locking, it's not safe to run multiple git-annex
+commands at the same time in Windows.
diff --git a/doc/design/assistant/blog/day_311__Windows_porting/comment_1_8e738f54a72557bee1e19970472b925c._comment b/doc/design/assistant/blog/day_311__Windows_porting/comment_1_8e738f54a72557bee1e19970472b925c._comment
new file mode 100644
index 0000000000..9f4bf2284a
--- /dev/null
+++ b/doc/design/assistant/blog/day_311__Windows_porting/comment_1_8e738f54a72557bee1e19970472b925c._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="https://www.google.com/accounts/o8/id?id=AItOawlpSOjMH7Iaz56v6Pr9KCFSpbvMXvg-y9o"
+ nickname="Dominik"
+ subject="That's amazing..."
+ date="2013-08-17T11:18:05Z"
+ content="""
+...so pretty soon my bi-directional Mac <-> Win Git-Annex-Assitant sync via rsync.net will work without a VM on Windows :-)
+"""]]
diff --git a/doc/design/assistant/blog/day_312__DebConf_midpoint.mdwn b/doc/design/assistant/blog/day_312__DebConf_midpoint.mdwn
new file mode 100644
index 0000000000..40c38aed4f
--- /dev/null
+++ b/doc/design/assistant/blog/day_312__DebConf_midpoint.mdwn
@@ -0,0 +1,30 @@
+Wow, 11 days off! I was busy with first dentistry and then DebConf.
+
+Yesterday I [visited CERN](http://joeyh.name/blog/entry/words_fail_me/) and
+got to talk with some of their IT guys about how they manage their tens of
+petabytes of data. Interested to hear they also have the equivilant of a
+per-subdirectory annex.numcopies setting. OTOH, they have half a billion
+more files than git's index file is likely to be able to scale to support. ;)
+
+Pushed a release out today despite not having many queued changes. 
+Also, I got git-annex migrated to Debian testing, and so was also
+able to update the wheezy backport to a just 2 week old version.
+
+Today is also the last day of the [campaign](https://campaign.joeyh.name/)!
+
+----
+
+There has been a ton of discussion about git-annex here at DebConf,
+including 3 BoF sessions that mostly focused on it, among other git stuff.
+Also, RichiH will be presenting his
+"[Gitify Your Life](http://penta.debconf.org/dc13_schedule/events/1025.en.html)"
+talk on Friday; you can catch it on the [live stream](http://blog.debconf.org/blog/2013/08/14#hl_dc13_recordings).
+
+I've also had a continual stream of in-person bug and feature requests.
+(Mostly features.)
+These have been added to the wiki and I look forward to working on that
+backlog when I get home.
+
+As for coding, I am doing little here, but I do have a branch cooking that
+adds some options to `git annex import` to control handling of duplicate
+files.
diff --git a/doc/design/assistant/blog/day_313__back.mdwn b/doc/design/assistant/blog/day_313__back.mdwn
new file mode 100644
index 0000000000..28c7f971d4
--- /dev/null
+++ b/doc/design/assistant/blog/day_313__back.mdwn
@@ -0,0 +1,34 @@
+Back home. I have some 170 messages of backlog to attend to. Rather than
+digging into that on my first day back, I spent some time implementing some
+new features.
+
+`git annex import` has grown three options that help managing importing of
+duplicate files in different ways. I started work on that last week, but
+didn't have time to find a way to avoid the `--deduplicate` option
+checksumming each imported file twice. Unfortunately, I have still not
+found a way I'm happy with, so it works but is not as efficient as it could
+be.
+
+`git annex mirror` is a new command suggested to me by someone at DebConf
+(they don't seem to have filed the requested todo). It arranges for two
+repositories to contain the same set of files, as much as possible (when
+numcopies allows). So for example, `git annex mirror --to otherdrive`
+will make the otherdrive remote have the same files present and not present
+as the local repository.
+
+I am thinking about expanding `git annex sync` with an option to also sync
+data. I know some find it confusing that it only syncs the git metadata
+and not the file contents. That still seems to me to be the best and most
+flexible behavior, and not one I want to change in any case since
+it would be most unexpected if `git annex sync` downloaded a lot of stuff
+you don't want. But I can see making `git annex sync --data` download
+all the file contents it can, as well as uploading all available file
+contents to each remote it syncs with. And `git annex sync --data --auto`
+limit that to only the preferred content. Although perhaps
+these command lines are too long to be usable?
+
+----
+
+With the campaign more or less over, I only have a little over a week
+before it's time to dive into the first big item on the roadmap. Hope
+to be through the backlog by then.
diff --git a/doc/design/assistant/blog/day_313__back/comment_1_fbf3fdf9688c18156753d446facd942d._comment b/doc/design/assistant/blog/day_313__back/comment_1_fbf3fdf9688c18156753d446facd942d._comment
new file mode 100644
index 0000000000..7230a15682
--- /dev/null
+++ b/doc/design/assistant/blog/day_313__back/comment_1_fbf3fdf9688c18156753d446facd942d._comment
@@ -0,0 +1,10 @@
+[[!comment format=mdwn
+ username="GLITTAH"
+ ip="77.247.181.162"
+ subject="comment 1"
+ date="2013-08-20T22:42:46Z"
+ content="""
+I wouldn't worry about the commands being too long.  If they're used often enough, they'll get aliased.
+
+alias GASDA='git annex sync --data --auto'
+"""]]
diff --git a/doc/design/assistant/blog/day_314__quvi.mdwn b/doc/design/assistant/blog/day_314__quvi.mdwn
new file mode 100644
index 0000000000..3c38e427d8
--- /dev/null
+++ b/doc/design/assistant/blog/day_314__quvi.mdwn
@@ -0,0 +1,27 @@
+Made some good progress on the backlog today. Fixed some bugs, applied some
+patches. Noticing that without me around, things still get followed up
+on, to a point, for example incomplete test cases for bugs get corrected so
+they work. This is a very good thing. Community!
+
+I had to stop going through the backlog when I got to one message from
+Anarcat mentioning [quvi](http://quvi.sourceforge.net/). That turns
+out to be just what is needed to implement the often-requested feature
+of `git-annex addurl` supporting YouTube and other similar sites. So I
+spent the rest of the day making that work. For example:
+
+
    +% git annex addurl --fast 'http://www.youtube.com/watch?v=1mxPFHBCfuU&list=PL4F80C7D2DC8D9B6C&index=1'
+addurl Star_Wars_X_Wing__Seth_Green__Clare_Grant__and_Mike_Lamond_Join_Wil_on_TableTop_SE2E09.webm ok
+
    
+
+Yes, that got the video title and used it as the filename, and yes,
+I can commit this file and run `git annex get` later, and it will be
+able to go download the video! I can even use `git annex fsck --fast`
+to make sure YouTube still has my videos. Awesome.
+
+The great thing about quvi is it takes the url to a video webpage, and
+returns an url that can be used to download the actual video file. So it
+simplifies ugly flash videos as far out of existence as is possible.
+However, since the direct url to the video file may not keep working for long.
+addurl actually records the page's url, with an added indication that quvi
+should be used to get it.
diff --git a/doc/design/assistant/blog/day_314__quvi/comment_1_3fdfb0742cb5422530ddd97b904f2a42._comment b/doc/design/assistant/blog/day_314__quvi/comment_1_3fdfb0742cb5422530ddd97b904f2a42._comment
new file mode 100644
index 0000000000..797d25101d
--- /dev/null
+++ b/doc/design/assistant/blog/day_314__quvi/comment_1_3fdfb0742cb5422530ddd97b904f2a42._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="https://www.google.com/accounts/o8/id?id=AItOawmNu4V5fvpLlBhaCUfXXOB0MI5NXwh8SkU"
+ nickname="Adam"
+ subject="New YouTube problems"
+ date="2013-09-05T03:45:16Z"
+ content="""
+Joey, you may notice eventually that YouTube is moving some videos, especially long ones, to a new format called DASH, which splits them up into smaller parts; unfortunately, it also delivers the audio as a separate file, requiring something like ffmpeg to merge it with the video.  I don't know anything about quvi, but youtube-dl recently added support for DASH.  It's a Python program that's packaged in Debian, but it can also update itself if you install it locally, which is helpful if Debian is frozen...  Anyway, if you need it in the future, youtube-dl might be helpful.
+"""]]
diff --git a/doc/design/assistant/blog/day_315__backlog.mdwn b/doc/design/assistant/blog/day_315__backlog.mdwn
new file mode 100644
index 0000000000..ec0ee33be3
--- /dev/null
+++ b/doc/design/assistant/blog/day_315__backlog.mdwn
@@ -0,0 +1,12 @@
+After a couple days plowing through it, my backlog is down to 30 messages
+from 150. And most of what's left is legitimate bugs and todo items.
+
+Spent a while today on an ugly file descriptor leak in the assistant's
+local pairing listener. This was an upstream bug in the network-multicast
+library, so while I've written a patch to fix it, the fix isn't quite 
+deployed yet. The file descriptor leak happens when the assistant is
+running and there is no network interface that supports multicast.
+I was able to reproduce it by just disconnecting from wifi.
+
+Meanwhile, guilhem has been working on patches that promise to massively
+speed up `git annex unused`! I will be reviewing them tonight.
diff --git a/doc/design/assistant/blog/day_316__day_off.mdwn b/doc/design/assistant/blog/day_316__day_off.mdwn
new file mode 100644
index 0000000000..b26118b17b
--- /dev/null
+++ b/doc/design/assistant/blog/day_316__day_off.mdwn
@@ -0,0 +1,6 @@
+Today was a day off, really. However, I have a job running to try to
+build get a version of ghc-android that works on newer Android releases.
+
+Also, guilhem's `git annex unused` speedup patch landed. The results are
+extrordinary -- speedups on the order of 50 to 100 times faster should
+not be uncommon. Best of all (for me), it still runs in constant memory!
diff --git a/doc/design/assistant/blog/day_317__misc.mdwn b/doc/design/assistant/blog/day_317__misc.mdwn
new file mode 100644
index 0000000000..e2acf21684
--- /dev/null
+++ b/doc/design/assistant/blog/day_317__misc.mdwn
@@ -0,0 +1,17 @@
+Spent a while tracking down a bug that causes a crash on OSX when setting
+up an XMPP account. I managed to find a small test case that reliably
+crashes, and sent it off to the author of the haskell-gnutls bindings,
+which had one similar segfault bug fixed before with a similar test case.
+Fingers crossed..
+
+Just finished tracking down a bug in the Android app that caused its
+terminal to spin and consume most CPU (and presumably a lot of battery).
+I introduced this bug when adding the code to open urls written to a fifo,
+due to misunderstanding how java objects are created, basically. This bug
+is bad enough to do a semi-immediate release for; luckily it's just about
+time for a release anyway with other improvements, so in the next few
+days..
+
+Have not managed to get a recent ghc-android to build so far.
+
+Guilhem fixed some bugs in `git annex unused`.
diff --git a/doc/design/assistant/blog/day_36__minimal_test_case.mdwn b/doc/design/assistant/blog/day_36__minimal_test_case.mdwn
new file mode 100644
index 0000000000..b77da14ddf
--- /dev/null
+++ b/doc/design/assistant/blog/day_36__minimal_test_case.mdwn
@@ -0,0 +1,9 @@
+Managed to find a minimal, 20 line test case for at least one of the ways
+git-annex was hanging with GHC's threaded runtime. Sent it off to
+haskell-cafe for analysis.
+[thread](http://thread.gmane.org/gmane.comp.lang.haskell.cafe/99334)
+
+Further managed to narrow the bug down to MissingH's use of logging code,
+that git-annex doesn't use. [bug report](http://bugs.debian.org/681621).
+So, I can at least get around this problem with a modified version of
+MissingH. Hopefully that was the only thing causing the hangs I was seeing!
diff --git a/doc/design/assistant/blog/day_37__back.mdwn b/doc/design/assistant/blog/day_37__back.mdwn
new file mode 100644
index 0000000000..7aef0b681a
--- /dev/null
+++ b/doc/design/assistant/blog/day_37__back.mdwn
@@ -0,0 +1,64 @@
+Back home and laptop is fixed.. back to work.
+
+Warmup exercises:
+
+* Went in to make it queue transfers when a broken symlink is received,
+  only to find I'd already written code to do that, and forgotten about it.
+  Heh. Did check that the git-annex branch is always sent first,
+  which will ensure that code always knows where to transfer a key from.
+  I had probably not considered this wrinkle when first writing the code;
+  it worked by accident.
+
+* Made the assistant check that a remote is known to have a key before
+  queueing a download from it.
+
+* Fixed a bad interaction between the `git annex map` command and the
+  assistant.
+
+----
+
+Tried using a modified version of `MissingH` that doesn't use `HSLogger`
+to make git-annex work with the threaded GHC runtime. Unfortunatly,
+I am still seeing hangs in at least 3 separate code paths when
+running the test suite. I may have managed to fix one of the hangs,
+but have not grokked what's causing the others.
+
+----
+
+I now have access to a Mac OSX system, thanks to Kevin M. I've fixed
+some portability problems in git-annex with it before, but today I tested
+the assistant on it:
+
+* Found a problem with the kqueue code that prevents incoming pushes from
+  being noticed.
+
+  The problem was that the newly added git ref file does not trigger an add
+  event. The kqueue code saw a generic change event for the refs directory,
+  but since the old file was being deleted and replaced by the new file,
+  the kqueue code, which already had the old file in its cache, did not notice
+  the file had been replaced.
+
+  I fixed that by making the kqueue code also track the inode of each file.
+  Currently that adds the overhead of a stat of each file, which could be
+  avoided if haskell exposed the inode returned by `readdir`. Room to
+  optimise this later...
+
+* Also noticed that the kqueue code was not separating out file deletions
+  from directory deletions. IIRC Jimmy had once mentioned a problem with file
+  deletions not being noticed by the assistant, and this could be responsible
+  for that, although the directory deletion code seems to handle them ok
+  normally. It was making the transfer watching thread not notice when
+  any transfers finished, for sure. I fixed this oversight, looking in the
+  cache to see if there used to be a file or a directory, and running the
+  appropriate hook.
+
+Even with these fixes, the assistant does not yet reliably transfer file
+contents on OSX. I think the problem is that with kqueue we're not
+guaranteed to get an add event, and a deletion event for a transfer
+info file -- if it's created and quickly deleted, the code that
+synthensizes those events doesn't run in time to know it existed.
+Since the transfer code relies on deletion events to tell when transfers
+are complete, it stops sending files after the first transfer, if the
+transfer ran so quickly it doesn't get the expected events. 
+
+So, will need to work on OSX support some more...
diff --git a/doc/design/assistant/blog/day_39__twice_is_enemy_action.mdwn b/doc/design/assistant/blog/day_39__twice_is_enemy_action.mdwn
new file mode 100644
index 0000000000..14896fcb1b
--- /dev/null
+++ b/doc/design/assistant/blog/day_39__twice_is_enemy_action.mdwn
@@ -0,0 +1,66 @@
+Beating my head against the threaded runtime some more. I can reproduce
+one of the hangs consistently by running 1000 git annex add commands
+in a loop. It hangs around 1% of the time, reading from `git cat-file`.
+
+Interestingly, `git cat-file` is not yet running at this point -- 
+git-annex has forked a child process, but the child has not yet exec'd it.
+Stracing the child git-annex, I see it stuck in a futex. Adding tracing,
+I see the child never manages to run any code at all.
+
+This really looks like the problem is once again in MissingH, which uses
+`forkProcess`. Which happens to come with a big warning about being very
+unsafe, in very subtle ways. Looking at the C code that the newer `process`
+library uses when sparning a pipe to a process, it messes around with lots of
+things; blocking signals, stopping a timer, etc. Hundreds of lines of C
+code to safely start a child process, all doing things that MissingH omits.
+
+That's the second time I've seemingly isolated a hang in the GHC threaded
+runtime to MissingH.
+
+And so I've started converting git-annex to use the new `process` library,
+for running all its external commands. John Goerzen had mentioned `process`
+to me once before when I found a nasty bug in MissingH, as the cool new
+thing that would probably eliminate the `System.Cmd.Utils` part of MissingH,
+but I'd not otherwise heard much about it. (It also seems to have the
+benefit of supporting Windows.)
+
+This is a big change and it's early days, but each time I see a hang, I'm
+converting the code to use `process`, and so far the hangs have just gone
+away when I do that.
+
+---
+
+Hours later... I've converted *all* of git-annex to use `process`.
+
+In the er, process, the `--debug` switch stopped printing all the commands
+it runs. I may try to restore that later.
+
+I've not tested everything, but the test suite passes, even when
+using the threaded runtime. **MILESTONE**
+
+Looking forward to getting out of these weeds and back to useful work..
+
+---
+
+Hours later yet.... The `assistant` branch in git now uses the threaded
+runtime. It works beautifully, using proper threads to run file transfers
+in.
+
+That should fix the problem I was seeing on OSX yesterday. Too tired to
+test it now.
+
+--
+
+Amazingly, all the assistant's own dozen or so threads and thread
+synch variables etc all work great under the threaded runtime. I had
+assumed I'd see yet more concurrency problems there when switching to it,
+but it all looks good. (Or whatever problems there are are subtle ones?)
+
+I'm very relieved. The threaded logjam is broken! I had been getting
+increasingly worried that not having the threaded runtime available would
+make it very difficult to make the assistant perform really well, and cause
+problems with the webapp, perhaps preventing me from using Yesod.
+
+Now it looks like smooth sailing ahead. Still some hard problems, but
+it feels like with inotify and kqueue and the threaded runtime all
+dealt with, the really hard infrastructure-level problems are behind me.
diff --git a/doc/design/assistant/blog/day_3__more_races.mdwn b/doc/design/assistant/blog/day_3__more_races.mdwn
new file mode 100644
index 0000000000..9c11828420
--- /dev/null
+++ b/doc/design/assistant/blog/day_3__more_races.mdwn
@@ -0,0 +1,26 @@
+Today I worked on the race conditions, and fixed two of them. Both
+were fixed by avoiding using `git add`, which looks at the files currently
+on disk. Instead, `git annex watch` injects symlinks directly into git's
+index, using `git update-index`.
+
+There is one bad race condition remaining. If multiple processes have a
+file open for write, one can close it, and it will be added to the annex.
+But then the other can still write to it.
+
+----
+
+Getting away from race conditions for a while, I made `git annex watch`
+not annex `.gitignore` and `.gitattributes` files.
+
+And, I made it handle running out of inotify descriptors. By default,
+`/proc/sys/fs/inotify/max_user_watches` is 8192, and that's how many
+directories inotify can watch. Now when it needs more, it will print
+a nice message showing how to increase it with `sysctl`.
+
+FWIW, DropBox also uses inotify and has the same limit. It seems to not
+tell the user how to fix it when it goes over. Here's what `git annex
+watch` will say:
+
+	Too many directories to watch! (Not watching ./dir4299)
+	Increase the limit by running:
+	  echo fs.inotify.max_user_watches=81920 | sudo tee -a /etc/sysctl.conf; sudo sysctl -p
diff --git a/doc/design/assistant/blog/day_3__more_races/comment_1_d6015338f602b574a3805de5481fc45e._comment b/doc/design/assistant/blog/day_3__more_races/comment_1_d6015338f602b574a3805de5481fc45e._comment
new file mode 100644
index 0000000000..2d330f3327
--- /dev/null
+++ b/doc/design/assistant/blog/day_3__more_races/comment_1_d6015338f602b574a3805de5481fc45e._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="https://www.google.com/accounts/o8/id?id=AItOawkmtR6oVColYKoU0SjBORLDGrwR10G-mKo"
+ nickname="Jo-Herman"
+ subject="Dropbox Inotify"
+ date="2012-06-06T22:03:29Z"
+ content="""
+Actually, Dropbox giver you a warning via libnotify inotify. It tends to go away too quickly to properly read though, much less actually copy down the command...
+"""]]
diff --git a/doc/design/assistant/blog/day_3__more_races/comment_2_4d6b23fc6442e0ee0303523cb69d0fba._comment b/doc/design/assistant/blog/day_3__more_races/comment_2_4d6b23fc6442e0ee0303523cb69d0fba._comment
new file mode 100644
index 0000000000..523e6d85ff
--- /dev/null
+++ b/doc/design/assistant/blog/day_3__more_races/comment_2_4d6b23fc6442e0ee0303523cb69d0fba._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="http://joeyh.name/"
+ ip="4.252.8.36"
+ subject="comment 2"
+ date="2012-06-06T23:25:57Z"
+ content="""
+When I work on the [[webapp]], I'm planning to make it display this warning, and any other similar warning messages that might come up.
+"""]]
diff --git a/doc/design/assistant/blog/day_3__more_races/comment_3_03f5b2344c2a47dea60086f217d60f9b._comment b/doc/design/assistant/blog/day_3__more_races/comment_3_03f5b2344c2a47dea60086f217d60f9b._comment
new file mode 100644
index 0000000000..92f5dcbd62
--- /dev/null
+++ b/doc/design/assistant/blog/day_3__more_races/comment_3_03f5b2344c2a47dea60086f217d60f9b._comment
@@ -0,0 +1,14 @@
+[[!comment format=mdwn
+ username="https://www.google.com/accounts/o8/id?id=AItOawnBJ6Dv1glxzzi4qIzGFNa6F-mfHIvv9Ck"
+ nickname="Jim"
+ subject="Wording"
+ date="2012-06-07T03:43:19Z"
+ content="""
+For the unfamiliar, it's hard to tell if a command like that would persist.  I'd suggest being as clear as possible, e.g.:
+
+    Increase the limit for now by running:
+      sudo sysctl fs.inotify.max_user_watches=81920
+    Increase the limit now and automatically at every boot by running:
+      echo fs.inotify.max_user_watches=81920 | sudo tee -a /etc/sysctl.conf; sudo sysctl -p
+
+"""]]
diff --git a/doc/design/assistant/blog/day_3__more_races/comment_4_860e90e989ec022100001c65e353a91e._comment b/doc/design/assistant/blog/day_3__more_races/comment_4_860e90e989ec022100001c65e353a91e._comment
new file mode 100644
index 0000000000..05b601eafe
--- /dev/null
+++ b/doc/design/assistant/blog/day_3__more_races/comment_4_860e90e989ec022100001c65e353a91e._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="http://joeyh.name/"
+ ip="4.252.8.36"
+ subject="comment 4"
+ date="2012-06-07T04:48:15Z"
+ content="""
+Good thought Jim. I've done something like that.
+"""]]
diff --git a/doc/design/assistant/blog/day_40__dbus.mdwn b/doc/design/assistant/blog/day_40__dbus.mdwn
new file mode 100644
index 0000000000..049c3ffe0d
--- /dev/null
+++ b/doc/design/assistant/blog/day_40__dbus.mdwn
@@ -0,0 +1,100 @@
+Really productive day today, now that I'm out of the threaded runtime
+tarpit!
+
+First, brought back `--debug` logging, better than before! As part of that, I
+wrote some 250 lines of code to provide a IMHO more pleasant interface to
+`System.Process` (itself only 650 lines of code) that avoids all the
+low-level setup, cleanup, and tuple unpacking. Now I can do things like
+write to a pipe to a process, and ensure it exits nonzero, this easily:
+
+	withHandle StdinHandle createProcessSuccess (proc "git" ["hash-object", "--stdin"]) $ \h ->
+		hHutStr h objectdata
+
+My interface also makes it easy to run nasty background processes,
+reading their output lazily.
+
+	lazystring <- withHandle StdoutHandle createBackgroundProcess (proc "find" ["/"]) hGetContents
+
+Any true Haskellers are shuddering here, I really should be using
+conduits or pipes, or something. One day..
+
+----
+
+The assistant needs to detect when removable drives are attached, and
+sync with them. This is a reasonable thing to be working on at this point,
+because it'll make the currently incomplete data transfer code fully usable
+for the sneakernet use case, and firming that up will probably be a good
+step toward handing other use cases involving data transfer over the
+network, including cases where network remotes are transientely available.
+
+So I've been playing with using dbus to detect mount events.
+There's a very nice Haskell library to use dbus.
+
+This simple program will detect removable drives being mounted, and
+works on Xfce (as long as you have automounting enabled in its
+configuration), and should also work on Gnome, and, probably, KDE:
+
+[[!format haskell """
+{-# LANGUAGE OverloadedStrings #-}
+
+import Data.List (sort)
+import DBus
+import DBus.Client
+import Control.Monad
+
+main = do
+	client <- connectSession
+    
+	listen client mountadded $ \s ->
+		putStrLn (show s)
+
+	forever $ getLine -- let listener thread run forever
+
+	where
+		mountadded = matchAny
+			{ matchInterface = Just "org.gtk.Private.RemoteVolumeMonitor"
+			, matchMember = Just "MountAdded"
+			}
+"""]]
+
+(Yeah... "org.gtk.Private.RemoteVolumeMonitor". There are so
+many things wrong with that string. What does gtk have to do with
+mounting a drive? Why is it Private? Bleagh. Should I only match
+the "MountAdded" member and not the interface? Seems everyone who does
+this relies on google to find other people who have cargo-culted it,
+or just runs `dbus-monitor` and picks out things.
+There seems to be no canonical list of events. Bleagh.)
+
+----
+
+Spent a while shaving a yak of needing a `getmntent` interface in Haskell.
+Found one in a hsshellscript library; since that library is not packaged
+in Debian, and I don't really want to depend on it, I extracted just
+the mtab and fstab parts of it into a little library in git-annex.
+
+----
+
+I've started putting together a MountWatcher thread. On systems without
+dbus (do OSX or the BSDs have dbus?), or if dbus is not running, it polls
+`/etc/mtab` every 10 seconds for new mounts. When dbus is available,
+it doesn't need the polling, and should notice mounts more quickly.
+
+Open question: Should it still poll even when dbus is available? Some of us
+like to mount our own drives, by hand and may have automounting disabled. It'd
+be good if the assistant supported that. This might need a
+`annex.no-dbus` setting, but I'd rather avoid needing such manual
+configuration.
+
+One idea is to do polling in addition to dbus, if `/etc/fstab` contains
+mount points that seem to be removable drives, on which git remotes lives.
+Or it could always do polling in addition to dbus, which is just some extra
+work. Or, it could try to introspect dbus to see if mount events will
+be generated.
+
+The MountWatcher so far only detects new mounts and prints out what
+happened. Next up: Do something in response to them. 
+
+This will involve manipulating the Annex state to belatedly add the Remote
+on the mount point.. tricky. And then, for Git Remotes, it should pull/push
+the Remote to sync git data. Finally, for all remotes, it will need to
+queue Transfers of file contents from/to the newly available Remote.
diff --git a/doc/design/assistant/blog/day_40__dbus/comment_1_43ed2a79629868b018ec9f54a32bcacc._comment b/doc/design/assistant/blog/day_40__dbus/comment_1_43ed2a79629868b018ec9f54a32bcacc._comment
new file mode 100644
index 0000000000..3670e7dd55
--- /dev/null
+++ b/doc/design/assistant/blog/day_40__dbus/comment_1_43ed2a79629868b018ec9f54a32bcacc._comment
@@ -0,0 +1,12 @@
+[[!comment format=mdwn
+ username="https://www.google.com/accounts/o8/id?id=AItOawl9sYlePmv1xK-VvjBdN-5doOa_Xw-jH4U"
+ nickname="Richard"
+ subject="dbus vs polling"
+ date="2012-07-19T17:38:40Z"
+ content="""
+I am running KDE, dbus, and I like to mount drives by hand; all that without bothering to set up /etc/fstab... Often from CLI so I don't even see the notifications for new drives.
+
+Especially while on the road, I will use no KDE, but just a tty or three. Still, dbus will be running.
+
+Long story short, I would need polling for the assistant to work flawlessly in all use cases.
+"""]]
diff --git a/doc/design/assistant/blog/day_40__dbus/comment_2_6799f2baf6a6ce14b1fa76a8402840c0._comment b/doc/design/assistant/blog/day_40__dbus/comment_2_6799f2baf6a6ce14b1fa76a8402840c0._comment
new file mode 100644
index 0000000000..832be854a3
--- /dev/null
+++ b/doc/design/assistant/blog/day_40__dbus/comment_2_6799f2baf6a6ce14b1fa76a8402840c0._comment
@@ -0,0 +1,10 @@
+[[!comment format=mdwn
+ username="hamish"
+ ip="203.0.139.24"
+ subject="dbus vs polling "
+ date="2012-07-22T07:13:37Z"
+ content="""
+I, too, am running a dbus but like to hand mount my filesystems.  However, I'd imagine that I am both a minority and that my minority could like the extra control, so perhaps even a \"re-read the mtab /now/\" command that can be manually run after something is  manually mounted would suffice
+
+Is it not possible to use inotify on the mtab?
+"""]]
diff --git a/doc/design/assistant/blog/day_40__dbus/comment_3_fa1d7444bdafcb990cacf2ace7ee6ef1._comment b/doc/design/assistant/blog/day_40__dbus/comment_3_fa1d7444bdafcb990cacf2ace7ee6ef1._comment
new file mode 100644
index 0000000000..a372670b86
--- /dev/null
+++ b/doc/design/assistant/blog/day_40__dbus/comment_3_fa1d7444bdafcb990cacf2ace7ee6ef1._comment
@@ -0,0 +1,10 @@
+[[!comment format=mdwn
+ username="http://joeyh.name/"
+ ip="4.154.4.169"
+ subject="comment 3"
+ date="2012-07-22T16:03:52Z"
+ content="""
+How did I not think about using my favorite hammer on this problem too? But, no, /proc/mounts cannot be watched with inotify it seems, and of course the BSDs don't seem to have a file at all.
+
+I think the dbus stuff is sorted out for manual users, see later blog entries.
+"""]]
diff --git a/doc/design/assistant/blog/day_40__dbus/comment_4_3399ddad951c1a950281bb6941fc3f6f._comment b/doc/design/assistant/blog/day_40__dbus/comment_4_3399ddad951c1a950281bb6941fc3f6f._comment
new file mode 100644
index 0000000000..9007850e46
--- /dev/null
+++ b/doc/design/assistant/blog/day_40__dbus/comment_4_3399ddad951c1a950281bb6941fc3f6f._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="https://www.google.com/accounts/o8/id?id=AItOawkSq2FDpK2n66QRUxtqqdbyDuwgbQmUWus"
+ nickname="Jimmy"
+ subject="comment 2"
+ date="2012-07-19T18:43:30Z"
+ content="""
+Joey, yes dbus is available from macports and homebrew, it's not installed by default (or as a dependancy) for most packages in macports.
+"""]]
diff --git a/doc/design/assistant/blog/day_40__dbus/comment_5_40b6b9d741d3081203f0cc94eb8dc3ea._comment b/doc/design/assistant/blog/day_40__dbus/comment_5_40b6b9d741d3081203f0cc94eb8dc3ea._comment
new file mode 100644
index 0000000000..38916dd8c2
--- /dev/null
+++ b/doc/design/assistant/blog/day_40__dbus/comment_5_40b6b9d741d3081203f0cc94eb8dc3ea._comment
@@ -0,0 +1,12 @@
+[[!comment format=mdwn
+ username="http://smcv.pseudorandom.co.uk/"
+ nickname="smcv"
+ subject="comment 5"
+ date="2012-07-31T09:58:13Z"
+ content="""
+It's `org.gtk` because gvfs, GLib and Gtk are all products of gtk.org (there is no separate glib.org).
+
+It's `Private` because the gvfs developers don't consider its D-Bus interface to be stable API, I believe. The official API for it is the C API in GIO, part of GLib.
+
+To poll for changes to /proc/mounts on Linux, you open it for reading and poll() for POLLERR (not sure why it gets to be different, but it is). See gio/gunixmounts.c in recent GLib.
+"""]]
diff --git a/doc/design/assistant/blog/day_41__foo.mdwn b/doc/design/assistant/blog/day_41__foo.mdwn
new file mode 100644
index 0000000000..99a977f602
--- /dev/null
+++ b/doc/design/assistant/blog/day_41__foo.mdwn
@@ -0,0 +1,46 @@
+I made the MountWatcher only use dbus if it sees a client connected to dbus
+that it knows will send mount events, or if it can start up such a client
+via dbus. (Fancy!) Otherwise it falls back to polling. This should be enough
+to support users who manually mount things -- if they have gvfs
+installed, it'll be used to detect their manual mounts, even when a desktop
+is not running, and if they don't have gvfs, they get polling.
+
+Also, I got the MountWatcher to work with KDE. Found a dbus event that's
+emitted when KDE mounts a drive, and this is also used. If anyone with
+some other desktop environment wants me to add support for it, and it uses
+dbus, it should be easy: Run `dbus-monitor`, plug in a drive, get
+it mounted, and send me a transcript.
+
+Of course, it'd also be nice to support anything similar on OSX that can
+provide mount event notifications. Not a priority though, since the polling
+code will work.
+
+---
+
+Some OS X fixes today..
+
+* Jimmy pointed out that my `getmntent` code broke the build on OSX again.
+  Sorry about that.. I keep thinking Unix portability nightmares are a 80's
+  thing, not a 2010's thing. Anyway, adapted a lot of hackish C code
+  to emulate `getmntent` on BSD systems, and it seems to work. (I actually
+  think the BSD interface to this is saner than Linux's, but I'd rather have
+  either one than both, sigh..)
+* Kqueue was blocking all the threads on OSX. This is fixed, and the
+  assistant seems to be working on OSX again.
+
+----
+
+I put together a preliminary page thanking everyone who contributed to the
+git-annex Kickstarter. [[/assistant/thanks]] The wall-o-names is scary crazy humbling.
+
+----
+
+Improved `--debug` mode for the assistant, now every thread says whenever
+it's doing anything interesting, and also there are timestamps.
+
+----
+
+Had been meaning to get on with syncing to drives when they're mounted, but
+got sidetracked with the above. Maybe tomorrow. I did think through it
+in some detail as I was waking up this morning, and think I have a pretty
+good handle on it.
diff --git a/doc/design/assistant/blog/day_41__foo/comment_1_ace21fa257a4c2fd412b6ff2944a23e8._comment b/doc/design/assistant/blog/day_41__foo/comment_1_ace21fa257a4c2fd412b6ff2944a23e8._comment
new file mode 100644
index 0000000000..e27f7904ed
--- /dev/null
+++ b/doc/design/assistant/blog/day_41__foo/comment_1_ace21fa257a4c2fd412b6ff2944a23e8._comment
@@ -0,0 +1,10 @@
+[[!comment format=mdwn
+ username="https://www.google.com/accounts/o8/id?id=AItOawnYD2ZzaOz-0anQDrN-Hg8Tvh5_C7wtStk"
+ nickname="roucaries"
+ subject="Portability"
+ date="2012-07-21T20:31:32Z"
+ content="""
+For portability why not using gnulib ? It will ease porting to windows BTW.
+
+Bastien
+"""]]
diff --git a/doc/design/assistant/blog/day_42__the_answer.mdwn b/doc/design/assistant/blog/day_42__the_answer.mdwn
new file mode 100644
index 0000000000..fd7c4ebb5a
--- /dev/null
+++ b/doc/design/assistant/blog/day_42__the_answer.mdwn
@@ -0,0 +1,27 @@
+Made the MountWatcher update state for remotes located in a drive that
+gets mounted. This was tricky code. First I had to make remotes declare
+when they're located in a local directory. Then it has to rescan git
+configs of git remotes (because the git repo mounted at a mount point may
+change), and update all the state that a newly available remote can affect.
+
+And it works: I plug in a drive containing one of my git remotes, and the
+assistant automatically notices it and syncs the git repositories.
+
+---
+
+But, data isn't transferred yet. When a disconnected remote becomes
+connected, keys should be transferred in both directions to get back into
+sync.
+
+To that end, added Yet Another Thread; the TransferScanner thread
+will scan newly available remotes to find keys, and queue low priority
+transfers to get them fully in sync.
+
+(Later, this will probably also be used for network remotes that become
+available when moving between networks. I think network-manager sends
+dbus events it could use..)
+
+This new thread is missing a crucial peice, it doesn't yet have a way to
+find the keys that need to be transferred. Doing that efficiently (without
+scanning the whole git working copy) is Hard. I'm considering design
+possibilities..
diff --git a/doc/design/assistant/blog/day_43__simple_scanner.mdwn b/doc/design/assistant/blog/day_43__simple_scanner.mdwn
new file mode 100644
index 0000000000..11ee3cca49
--- /dev/null
+++ b/doc/design/assistant/blog/day_43__simple_scanner.mdwn
@@ -0,0 +1,37 @@
+Milestone: I can run `git annex assistant`, plug in a USB drive, and it
+automatically transfers files to get the USB drive and current repo back in
+sync.
+
+I decided to implement the naive scan, to find files needing to be
+transferred. So it walks through `git ls-files` and checks each file
+in turn. I've deferred less expensive, more sophisticated approaches to later.
+
+I did some work on the TransferQueue, which now keeps track of the length
+of the queue, and can block attempts to add Transfers to it if it gets too
+long. This was a nice use of STM, which let me implement that without using
+any locking.
+
+[[!format haskell """
+atomically $ do
+        sz <- readTVar (queuesize q)
+        if sz <= wantsz
+                then enqueue schedule q t (stubInfo f remote)
+                else retry -- blocks until queuesize changes
+"""]]
+
+Anyway, the point was that, as the scan finds Transfers to do,
+it doesn't build up a really long TransferQueue, but instead is blocked
+from running further until some of the files get transferred. The resulting
+interleaving of the scan thread with transfer threads means that transfers
+start fairly quickly upon a USB drive being plugged in, and kind of hides
+the innefficiencies of the scanner, which will most of the time be
+swamped out by the IO bound large data transfers.
+
+---
+
+At this point, the assistant should do a good job of keeping repositories
+in sync, as long as they're all interconnected, or on removable media
+like USB drives. There's lots more work to be done to handle use cases
+where repositories are not well-connected, but since the assistant's
+[[syncing]] now covers at least a couple of use cases, I'm ready to move
+on to the next phase. [[Webapp]], here we come!
diff --git a/doc/design/assistant/blog/day_44__webapp_basics.mdwn b/doc/design/assistant/blog/day_44__webapp_basics.mdwn
new file mode 100644
index 0000000000..a8fdb131af
--- /dev/null
+++ b/doc/design/assistant/blog/day_44__webapp_basics.mdwn
@@ -0,0 +1,83 @@
+After an all-nighter, I have `git annex webapp` launching a WebApp!
+
+It doesn't do anything useful yet, just uses Yesod to display a couple of
+hyperlinked pages and a favicon, securely.
+
+The binary size grew rather alarmingly, BTW. :) Indeed, it's been growing
+for months..
+
+	-rwxr-xr-x 1 root root 9.4M Jul 21 16:59 git-annex-no-assistant-stripped
+	-rwxr-xr-x 1 joey joey  12M Jul 25 20:54 git-annex-no-webapp-stripped
+	-rwxr-xr-x 1 joey joey  17M Jul 25 20:52 git-annex-with-webapp-stripped
+
+----
+
+Along the way, some Not Invented Here occurred:
+
+I didn't use the yesod scaffolded site, because it's a lot of what
+seems mostly to be cruft in this use case. And because I don't like
+code generated from templates that people are then expected to edit. Ugh.
+That's my least favorite part of Yesod. This added some pain, since
+I had to do everything the hard way.
+
+I didn't use [wai-handler-launch](http://hackage.haskell.org/package/wai-handler-launch)
+because:
+
+  * It seems broken on IPv6 capable machines (it always opens
+    `http://127.0.0.1:port/` even though it apparently doesn't always
+    listen there.. I think it was listening on my machine's ipv6 address
+    instead. I know, I know; I should file a bug about this..)
+  * It always uses port 4587, which is **insane**. What if you have two
+    webapps?
+  * It requires javascript in the web browser, which
+    is used to ping the server, and shut it down when the web browser closes
+    (which behavior is wrong for git-annex anyway, since the daemon should
+    stay running across browser closes).
+  * It opens the webapp on web server startup, which is wrong for git-annex; 
+    instead the command `git annex webapp` will open the webapp,
+    after `git annex assistant` started the web server.
+
+Instead, I rolled my own WAI webapp laucher, that binds to any free port
+on localhost, It does use `xdg-open` to launch the web browser, 
+like wai-handler-launch (or just `open` on OS X).
+
+Also, I wrote my own WAI logger, which logs using System.Log.Logger,
+instead of to stdout, like `runDebug` does.
+
+----
+
+The webapp only listens for connections from localhost, but that's
+not sufficient "security". Instead, I added a secret token to
+every url in the webapp, that only `git annex webapp` knows about.
+
+But, if that token is passed to `xdg-open` on its command line,
+it will be briefly visible to local attackers in the parameters of
+`xdg-open`.. And if the web browser's not already running, it'll run
+with it as a parameter, and be *very* visible.
+
+So instead, I used a nasty hack. On startup, the assistant
+will create a html file, readably only by the user, that redirects
+the user to the real site url. Then `git annex webapp` will run
+xdg-open on that file.
+
+----
+
+Making Yesod check the `auth=` parameter (to verify that the secret token
+is right) is when using Yesod started to pay off. Yesod has a simple
+`isAuthorized` method that can be overridden to do your own authentication
+like this.
+
+But Yesod really started to shine when I went to add the `auth=` parameter
+to every url in the webapp. There's a `joinPath` method can can be used
+to override the default url builder. And every type-safe url in the
+application goes through there, so it's perfect for this.
+
+I just had to be careful to make it not add `auth=` to the url for the
+favicon, which is included in the "Permission Denied" error page. That'd be
+an amusing security hole..
+
+----
+
+Next up: Doing some AJAX to get a dynamic view of the state of the daemon,
+including currently running transfers, in the webapp. AKA stuff I've never
+done before, and that, unlike all this heavy Haskell Yesod, scares me. :)
diff --git a/doc/design/assistant/blog/day_44__webapp_basics/comment_1_d5fb67f373038e9f583cb2e1992bef67._comment b/doc/design/assistant/blog/day_44__webapp_basics/comment_1_d5fb67f373038e9f583cb2e1992bef67._comment
new file mode 100644
index 0000000000..bc1f259e85
--- /dev/null
+++ b/doc/design/assistant/blog/day_44__webapp_basics/comment_1_d5fb67f373038e9f583cb2e1992bef67._comment
@@ -0,0 +1,18 @@
+[[!comment format=mdwn
+ username="http://jasonwoof.com/"
+ nickname="JasonWoof"
+ subject="Your security solution is fine"
+ date="2012-07-27T06:37:03Z"
+ content="""
+I think making the html file redirect is just fine. desktopcouch does something like this. They talk about it being there to help you find the port number, but it contains the security token too.
+
+They also have some javascript to delay the redirect for a while so you can bookmark the redirect file. You can click to set a cookie to skip the delay in future.
+
+It's been a while since I used desktopcouch, but I found an old redirect file laying around. Here it is:
+
+http://jasonwoof.com/downloads/desktopcouch-redirect.txt
+
+The code is pretty short, you might find something useful there.
+
+      - Jason
+"""]]
diff --git a/doc/design/assistant/blog/day_45__long_polling.mdwn b/doc/design/assistant/blog/day_45__long_polling.mdwn
new file mode 100644
index 0000000000..380c55c127
--- /dev/null
+++ b/doc/design/assistant/blog/day_45__long_polling.mdwn
@@ -0,0 +1,66 @@
+The webapp now displays actual progress bars, for the actual transfers
+that the assistant is making! And it's seriously shiny.
+
+[[!img full.png]]
+
+Yes, I used Bootstrap. I can see why so many people are using it,
+that the common complaint is everything looks the same. I spent a few hours
+mocking up the transfer display part of the WebApp using Bootstrap, and
+arrived at something that doesn't entirely suck remarkably quickly.
+
+The really sweet thing about Bootstrap is that when I resized my browser to
+the shape of a cell phone, it magically redrew the WebApp like so:
+
+[[!img phone.png]]
+
+---
+
+To update the display, the WebApp uses two techniques. On noscript
+browsers, it just uses a meta refresh, which is about the best I can do. I
+welcome feedback; it might be better to just have an "Update" button in
+this case.
+
+With javascript enabled, it uses long polling, done over AJAX. There are
+some other options I considered, including websockets, and server-sent
+events. Websockets seem too new, and while there's a WAI module supporting
+server-sent events, and even an example of them in the Yesod book, the
+module is not packaged for Debian yet. Anyway, long polling is the most
+widely supported, so a good starting place. It seems to work fine too, I
+don't really anticipate needing the more sophisticated methods.
+
+(Incidentially, this's the first time I've ever written code that uses AJAX.)
+
+Currently the status display is rendered in html by the web server, and
+just updated into place by javascript. I like this approach since it
+keeps the javascript code to a minimum and the pure haskell code to a
+maximum. But who knows, I may have to switch to JSON that gets rendered by
+javascript, for some reason, later on.
+
+---
+
+I was very happy with Yesod when I managed to factor out a
+general purpose widget that adds long-polling and meta-refresh to any
+other widget. I was less happy with Yesod when I tried to include
+jquery on my static site and it kept serving up a truncated version of it.
+Eventually worked around what's seemingly a bug in the default WAI
+middleware, by disabling that middleware.
+
+----
+
+Also yesterday I realized there were about 30 comments stuck in moderation on
+this website. I thought I had a feed of those, but obviously I didn't. I've
+posted them all, and also read them all.
+
+----
+
+Next up is probably some cleanup of bugs and minor todos. Including
+figuring out why `watch` has started to segfault on OSX when it was
+working fine before.
+
+After that, I need to build a way to block the long polling request
+until the DaemonStatus and/or TransferQueue change from the version
+previously displayed by the WebApp. An interesting concurrency problem..
+
+Once I have that working, I can reduce the current 3 second delay between
+refreshes to a very short delay, and the WebApp will update in
+near-realtime as changes come in.
diff --git a/doc/design/assistant/blog/day_45__long_polling/comment_1_994bec0978324e268666073e8ff4f6ae._comment b/doc/design/assistant/blog/day_45__long_polling/comment_1_994bec0978324e268666073e8ff4f6ae._comment
new file mode 100644
index 0000000000..20d3a64e65
--- /dev/null
+++ b/doc/design/assistant/blog/day_45__long_polling/comment_1_994bec0978324e268666073e8ff4f6ae._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="http://jasonwoof.com/"
+ nickname="JasonWoof"
+ subject="auth token length"
+ date="2012-07-27T18:52:19Z"
+ content="""
+Your auth token looks a little short. Aren't you worried about people brute-forcing it? ;)
+"""]]
diff --git a/doc/design/assistant/blog/day_45__long_polling/comment_2_dfa164c86290899139491acccddd8b2b._comment b/doc/design/assistant/blog/day_45__long_polling/comment_2_dfa164c86290899139491acccddd8b2b._comment
new file mode 100644
index 0000000000..836fe89e9a
--- /dev/null
+++ b/doc/design/assistant/blog/day_45__long_polling/comment_2_dfa164c86290899139491acccddd8b2b._comment
@@ -0,0 +1,10 @@
+[[!comment format=mdwn
+ username="http://joeyh.name/"
+ ip="4.153.2.20"
+ subject="comment 2"
+ date="2012-07-27T18:55:51Z"
+ content="""
+Heh, I consider it overflowing most address fields a bonus, as you don't have to worry when making screenshots. :)
+
+Of course, it changes each app run too..
+"""]]
diff --git a/doc/design/assistant/blog/day_45__long_polling/comment_3_5526c9dd4fd87da56cb8456083169f55._comment b/doc/design/assistant/blog/day_45__long_polling/comment_3_5526c9dd4fd87da56cb8456083169f55._comment
new file mode 100644
index 0000000000..7881bcda56
--- /dev/null
+++ b/doc/design/assistant/blog/day_45__long_polling/comment_3_5526c9dd4fd87da56cb8456083169f55._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="https://www.google.com/accounts/o8/id?id=AItOawmz8d2M0lQDYWLSbDQSjYRHfrQkWKgPu60"
+ nickname="Alex"
+ subject="long polling in Widget form"
+ date="2014-01-16T00:08:12Z"
+ content="""
+I've been hacking around for a couple days trying to get long polling into a Yesod web app, but I haven't found a composable solution yet. It seems like a thing worthy of generalization. Can you perhaps point me to source in git-annex that would help me take a shot at building a long polling library?
+"""]]
diff --git a/doc/design/assistant/blog/day_45__long_polling/comment_4_91630f5bf162dfd4fbb3920f1318535b._comment b/doc/design/assistant/blog/day_45__long_polling/comment_4_91630f5bf162dfd4fbb3920f1318535b._comment
new file mode 100644
index 0000000000..53b9f40983
--- /dev/null
+++ b/doc/design/assistant/blog/day_45__long_polling/comment_4_91630f5bf162dfd4fbb3920f1318535b._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="http://joeyh.name/"
+ ip="209.250.56.43"
+ subject="comment 4"
+ date="2014-01-16T03:03:48Z"
+ content="""
+Alex, the long polling is done almost entirely on the javascript side; the yesod handler only needs to block until the next change it wants to display. So I doubt any of what I have would make a useful library. (Although it would be very useful to have a library!)
+"""]]
diff --git a/doc/design/assistant/blog/day_45__long_polling/comment_5_b3e41ba77f21e93a4e086483793bf5ce._comment b/doc/design/assistant/blog/day_45__long_polling/comment_5_b3e41ba77f21e93a4e086483793bf5ce._comment
new file mode 100644
index 0000000000..d9886216c9
--- /dev/null
+++ b/doc/design/assistant/blog/day_45__long_polling/comment_5_b3e41ba77f21e93a4e086483793bf5ce._comment
@@ -0,0 +1,10 @@
+[[!comment format=mdwn
+ username="https://www.google.com/accounts/o8/id?id=AItOawmz8d2M0lQDYWLSbDQSjYRHfrQkWKgPu60"
+ nickname="Alex"
+ subject="re: long polling"
+ date="2014-01-17T20:27:23Z"
+ content="""
+After poking through git-annex, I had the same realization. I hadn't realized that Yesod (or perhaps more accurately, Warp) is asynchronous above the Application level. I had feared I'd need to write some WAI middleware to accommodate the blocking nature of long polling requests. In retrospect it seems kind of silly: what kind of high-performance webserver doesn't handle requests concurrently?
+
+I do still think there's a pattern worth encapsulating, even if it won't be long until WebSockets and SSE are widespread enough to be developed against without fallbacks.
+"""]]
diff --git a/doc/design/assistant/blog/day_45__long_polling/full.png b/doc/design/assistant/blog/day_45__long_polling/full.png
new file mode 100644
index 0000000000..3963ae1dcb
Binary files /dev/null and b/doc/design/assistant/blog/day_45__long_polling/full.png differ
diff --git a/doc/design/assistant/blog/day_45__long_polling/phone.png b/doc/design/assistant/blog/day_45__long_polling/phone.png
new file mode 100644
index 0000000000..389334d95a
Binary files /dev/null and b/doc/design/assistant/blog/day_45__long_polling/phone.png differ
diff --git a/doc/design/assistant/blog/day_46__notification_pools.mdwn b/doc/design/assistant/blog/day_46__notification_pools.mdwn
new file mode 100644
index 0000000000..483ad95b1c
--- /dev/null
+++ b/doc/design/assistant/blog/day_46__notification_pools.mdwn
@@ -0,0 +1,68 @@
+Focus today was writing a notification broadcaster library. This is a way to
+send a notification to a set of clients, any of which can be blocked
+waiting for a new notification to arrive. A complication is that any number
+of clients may be be dead, and we don't want stale notifications for those
+clients to pile up and leak memory.
+
+It took me 3 tries to find the solution, which turns out to be head-smackingly
+simple: An array of SampleVars, one per client.
+
+Using SampleVars means that clients only see the most recent notification,
+but when the notification is just "the assistant's state changed somehow;
+display a refreshed rendering of it", that's sufficient.
+
+----
+
+First use of that was to make the thread that woke up every 10 minutes
+and checkpointed the daemon status to disk also wait for a notification
+that it changed. So that'll be more current, and use less IO.
+
+----
+
+Second use, of course, was to make the WebApp block long polling clients
+until there is really a change since the last time the client polled.
+
+To do that, I made one change to my Yesod routes:
+
+[[!format diff """
+	-/status StatusR GET
+	+/status/#NotificationId StatusR GET
+"""]]
+
+Now I find another reason to love Yesod, because after doing that,
+I hit "make".. and fixed the type error. And hit make.. and fixed
+the type error. And then it just freaking worked! Html was generated with
+all urls to /status including a `NotificationId`, and the handler for
+that route got it and was able to use it:
+
+[[!format haskell """
+	{- Block until there is an updated status to display. -}
+	b <- liftIO $ getNotificationBroadcaster webapp
+	liftIO $ waitNotification $ notificationHandleFromId b nid
+"""]]
+
+And now the WebApp is able to display transfers in realtime!
+When I have both the WebApp and `git annex get` running on the same screen,
+the WebApp displays files that git-annex is transferring about as fast
+as the terminal updates.
+
+The [[progressbars]] still need to be sorted out, but otherwise
+the WebApp is a nice live view of file transfers.
+
+---
+
+I also had some fun with Software Transactional Memory. Now when the
+assistant moves a transfer from its queue of transfers to do, to its map of
+transfers that are currently running, it does so in an atomic transaction.
+This will avoid the transfer seeming to go missing (or be listed twice) if
+the webapp refreshes at just the wrong point in time. I'm really starting
+to get into STM.
+
+----
+
+Next up, I will be making the WebApp maintain a list of notices, displayed
+on its sidebar, scrolling new notices into view, and removing ones the user
+closes, and ones that expire. This will be used for displaying errors, as
+well as other communication with the user (such as displaying a notice
+while a git sync is in progress with a remote, etc). Seems worth doing now,
+so the basic UI of the WebApp is complete with no placeholders.
diff --git a/doc/design/assistant/blog/day_47__alert_messages.mdwn b/doc/design/assistant/blog/day_47__alert_messages.mdwn
new file mode 100644
index 0000000000..81551fa95b
--- /dev/null
+++ b/doc/design/assistant/blog/day_47__alert_messages.mdwn
@@ -0,0 +1,14 @@
+Some days I spend 2 hours chasing red herrings (like "perhaps my JSON ajax
+calls arn't running asynchronoously?") that turn out to be a simple
+one-word typo. This was one of them.
+
+However, I did get the sidebar displaying alert messages, which can be
+easily sent to the user from any part of the assistant. This includes
+transient alerts of things it's doing, which disappear once the action
+finishes, and long-term alerts that are displayed until the user closes
+them. It even supports rendering arbitrary Yesod widgets as alerts, so
+they can also be used for asking questions, etc.
+
+Time for a screencast!
+
+
diff --git a/doc/design/assistant/blog/day_48__intro.mdwn b/doc/design/assistant/blog/day_48__intro.mdwn
new file mode 100644
index 0000000000..2cac85d4d1
--- /dev/null
+++ b/doc/design/assistant/blog/day_48__intro.mdwn
@@ -0,0 +1,8 @@
+Lots of WebApp UI improvements, mostly around the behavior when
+displaying alert messages. Trying to make the alerts informative
+without being intrusively annoying, think I've mostly succeeded now.
+
+Also, added an intro display. Shown is the display with only one repo;
+if there are more repos it also lists them all.
+
+[[!img screenshot/intro.png]]
diff --git a/doc/design/assistant/blog/day_49__first_run_experience.mdwn b/doc/design/assistant/blog/day_49__first_run_experience.mdwn
new file mode 100644
index 0000000000..437046d34d
--- /dev/null
+++ b/doc/design/assistant/blog/day_49__first_run_experience.mdwn
@@ -0,0 +1,39 @@
+Started work on the interface displayed when the webapp is started 
+with no existing git-annex repository. All this needs to do is walk the user
+through setting up a repository, as simply as possible.
+
+A tricky part of this is that most of git-annex runs in the Annex monad,
+which requires a git-annex repository. Luckily, much of the webapp
+does not run in Annex, and it was pretty easy to work around the parts that
+do. Dodged a bullet there.
+
+There will, however, be a tricky transition from this first run webapp,
+to a normally fully running git-annex assistant and webapp. I think the
+first webapp will have to start up all the normal threads once it makes the
+repository, and then redirect the user's web browser to the full webapp.
+
+Anyway, the UI I've made is very simple: A single prompt, for the
+directory where the repository should go. With, eventually, tab completion,
+sanity checking (putting the repository in "/" is not good, and making it
+all of "$HOME" is probably unwise).
+
+[[!img screenshot/firstrun.png]]
+
+Ideally most users will accept the default, which will be something
+like `/home/username/Desktop/Annex`, and be through this step in seconds.
+
+Suggestions for a good default directory name appreciated.. Putting it on a
+folder that will appear on the desktop seems like a good idea, when there's
+a Desktop directory. I'm unsure if I should name it something specific like
+"GitAnnex", or something generic like "Synced".
+
+Time for the first of probably many polls!
+
+What should the default directory name used by the git-annex assistant be?
+
+[[!poll open=no 19 "Annex" 7 "GitAnnex" 1 "~/git-annex/" 10 "Synced" 0 "AutoSynced" 1 "Shared" 10 "something lowercase!" 1 "CowboyNeal" 1 "Annexbox"]]
+
+(Note: This is a wiki. You can edit this page to add your own
+[[ikiwiki/directive/poll]] options!)
+
+[[!tag polls]]
diff --git a/doc/design/assistant/blog/day_49__first_run_experience/comment_1_e146cf06c8dd6303dd6a991f152a73fe._comment b/doc/design/assistant/blog/day_49__first_run_experience/comment_1_e146cf06c8dd6303dd6a991f152a73fe._comment
new file mode 100644
index 0000000000..9f1d74e677
--- /dev/null
+++ b/doc/design/assistant/blog/day_49__first_run_experience/comment_1_e146cf06c8dd6303dd6a991f152a73fe._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="http://nico.kaiser.me/"
+ nickname="Nico Kaiser"
+ subject="Directory location"
+ date="2012-08-01T07:51:36Z"
+ content="""
+I don't think the Annex directory (or whatever it will be called – \"Annex\" seems like a good choice in my opinion) should be inside the \"Desktop\" folder by default. All other xdg-user-dirs (Documents, Downloads, etc.) are in $HOME, and i think Desktop should not be cluttered with folders.
+"""]]
diff --git a/doc/design/assistant/blog/day_49__first_run_experience/comment_2_5d6adcf6782c02283bef6189582ee467._comment b/doc/design/assistant/blog/day_49__first_run_experience/comment_2_5d6adcf6782c02283bef6189582ee467._comment
new file mode 100644
index 0000000000..40f75ca0bd
--- /dev/null
+++ b/doc/design/assistant/blog/day_49__first_run_experience/comment_2_5d6adcf6782c02283bef6189582ee467._comment
@@ -0,0 +1,12 @@
+[[!comment format=mdwn
+ username="http://joeyh.name/"
+ ip="4.154.3.236"
+ subject="comment 2"
+ date="2012-08-01T14:04:08Z"
+ content="""
+Hmm, my thought on putting it under Desktop is that'll let me install git-annex and tell my mom (really!) that files dragged to there are shared. Don't the other xdg directories get used as default save/open locations for things like word processors, so not need to be on the desktop?
+
+Although that may not be necessary. I've realized I can, at least on linux, easily make a button on the webapp that pops open the directory in the desktop's file manager.
+
+I guess another way of looking at the question is: Where does dropbox put its folder?
+"""]]
diff --git a/doc/design/assistant/blog/day_49__first_run_experience/comment_3_7ac2e34c2a7bc9b57488ca0c91307d32._comment b/doc/design/assistant/blog/day_49__first_run_experience/comment_3_7ac2e34c2a7bc9b57488ca0c91307d32._comment
new file mode 100644
index 0000000000..22eff96b65
--- /dev/null
+++ b/doc/design/assistant/blog/day_49__first_run_experience/comment_3_7ac2e34c2a7bc9b57488ca0c91307d32._comment
@@ -0,0 +1,14 @@
+[[!comment format=mdwn
+ username="https://www.google.com/accounts/o8/id?id=AItOawllHWUwOT-UVe5bGkk8G87bpeRAzy-5T7w"
+ nickname="Rick"
+ subject="Folder location"
+ date="2012-08-01T14:18:21Z"
+ content="""
+I'm of two minds about where the Annex folder should be.
+
+For beginners, the desktop is ideal. No doubt about it. And more experienced users can override the default path during the install.
+
+On the other hand, many of the other 'synced folder' services place their folder in the user's home. Both Dropbox and Skydrive do this. It's where I'd put it for consistency, and to keep my desktop free of clutter.
+
+It's a close call, but if push came to shove, I'd probably go with the user home folder.
+"""]]
diff --git a/doc/design/assistant/blog/day_49__first_run_experience/comment_4_549b07bb02c07a5b1b95445b01758db2._comment b/doc/design/assistant/blog/day_49__first_run_experience/comment_4_549b07bb02c07a5b1b95445b01758db2._comment
new file mode 100644
index 0000000000..f261d1c238
--- /dev/null
+++ b/doc/design/assistant/blog/day_49__first_run_experience/comment_4_549b07bb02c07a5b1b95445b01758db2._comment
@@ -0,0 +1,14 @@
+[[!comment format=mdwn
+ username="dhess"
+ ip="173.247.200.7"
+ subject="Choose a friendly/unintimidating name"
+ date="2012-09-13T00:32:15Z"
+ content="""
+You've already decided to accommodate mom by choosing ~/Desktop as the default location, so you should be consistent: choose a name that means something to people. \"Annex\" and \"GitAnnex\" are good branding, but not very user-friendly. (Contrast with \"Dropbox\", the default folder name for Dropbox -- good branding *and* reasonably meaningful.)
+
+\"Shared\" is friendly, but implies that you're sharing with other people, which isn't necessarily the case. (You should reserve the name \"Shared\" for a sub-directory of the default directory, anyway, if/when the time comes to implement sharing URLs to your g-a-a objects with other users.)
+
+I think that \"Synced\" is fine. Most English-speakers will know what it means, and it's a good description of what g-a-a does.
+
+Don't worry whether we like it or not. Nobody who comments here, nor, likely, anyone who votes, will use the default name anyway ;)
+"""]]
diff --git a/doc/design/assistant/blog/day_4__speed.mdwn b/doc/design/assistant/blog/day_4__speed.mdwn
new file mode 100644
index 0000000000..085d9547b1
--- /dev/null
+++ b/doc/design/assistant/blog/day_4__speed.mdwn
@@ -0,0 +1,47 @@
+Only had a few hours to work today, but my current focus is speed, and I
+have indeed sped up parts of `git annex watch`.
+
+One thing folks don't realize about git is that despite a rep for being
+fast, it can be rather slow in one area: Writing the index. You don't
+notice it until you have a lot of files, and the index gets big. So I've
+put a lot of effort into git-annex in the past to avoid writing the index
+repeatedly, and queue up big index changes that can happen all at once. The
+new `git annex watch` was not able to use that queue. Today I reworked the
+queue machinery to support the types of direct index writes it needs, and
+now repeated index writes are eliminated.
+
+... Eliminated too far, it turns out, since it doesn't yet *ever* flush
+that queue until shutdown! So the next step here will be to have a worker
+thread that wakes up periodically, flushes the queue, and autocommits.
+(This will, in fact, be the start of the [[syncing]] phase of my roadmap!)
+There's lots of room here for smart behavior. Like, if a lot of changes are
+being made close together, wait for them to die down before committing. Or,
+if it's been idle and a single file appears, commit it immediately, since
+this is probably something the user wants synced out right away. I'll start
+with something stupid and then add the smarts.
+
+(BTW, in all my years of programming, I have avoided threads like the nasty
+bug-prone plague they are. Here I already have three threads, and am going to
+add probably 4 or 5 more before I'm done with the git annex assistant. So
+far, it's working well -- I give credit to Haskell for making it easy to
+manage state in ways that make it possible to reason about how the threads
+will interact.)
+
+What about the races I've been stressing over? Well, I have an ulterior
+motive in speeding up `git annex watch`, and that's to also be able to
+**slow it down**. Running in slow-mo makes it easy to try things that might
+cause a race and watch how it reacts. I'll be using this technique when
+I circle back around to dealing with the races.
+
+Another tricky speed problem came up today that I also need to fix. On
+startup, `git annex watch` scans the whole tree to find files that have
+been added or moved etc while it was not running, and take care of them.
+Currently, this scan involves re-staging every symlink in the tree. That's
+slow! I need to find a way to avoid re-staging symlinks; I may use `git
+cat-file` to check if the currently staged symlink is correct, or I may
+come up with some better and faster solution. Sleeping on this problem.
+
+----
+
+Oh yeah, I also found one more race bug today. It only happens at startup
+and could only make it miss staging file deletions.
diff --git a/doc/design/assistant/blog/day_4__speed/comment_1_bf3c9c33cc0dea5eaeb6f2af110b924b._comment b/doc/design/assistant/blog/day_4__speed/comment_1_bf3c9c33cc0dea5eaeb6f2af110b924b._comment
new file mode 100644
index 0000000000..fb5b95490f
--- /dev/null
+++ b/doc/design/assistant/blog/day_4__speed/comment_1_bf3c9c33cc0dea5eaeb6f2af110b924b._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="https://www.google.com/accounts/o8/id?id=AItOawldKnauegZulM7X6JoHJs7Gd5PnDjcgx-E"
+ nickname="Matt"
+ subject="open source?"
+ date="2012-06-09T22:34:30Z"
+ content="""
+Are you publishing the source code for git-annex assistant somewhere?
+"""]]
diff --git a/doc/design/assistant/blog/day_4__speed/comment_2_33aba4c9abaa3e6a05a2c87ab7df9d0e._comment b/doc/design/assistant/blog/day_4__speed/comment_2_33aba4c9abaa3e6a05a2c87ab7df9d0e._comment
new file mode 100644
index 0000000000..1fcc197ab7
--- /dev/null
+++ b/doc/design/assistant/blog/day_4__speed/comment_2_33aba4c9abaa3e6a05a2c87ab7df9d0e._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="http://joeyh.name/"
+ ip="4.153.8.126"
+ subject="comment 2"
+ date="2012-06-09T23:01:29Z"
+ content="""
+Yes, it's in [[git|download]] with the rest of git-annex. Currently in the `watch` branch.
+"""]]
diff --git a/doc/design/assistant/blog/day_50__directory_name.mdwn b/doc/design/assistant/blog/day_50__directory_name.mdwn
new file mode 100644
index 0000000000..a0e57cae68
--- /dev/null
+++ b/doc/design/assistant/blog/day_50__directory_name.mdwn
@@ -0,0 +1,20 @@
+Based on the results of yesterday's poll, the WebApp defaults to
+`~/Desktop/annex` when run in the home directory. If there's no `Desktop`
+directory, it uses just `~/annex`. And if run from some other place than
+the home directory, it assumes you want to use cwd. Of course, you can
+change this default, but I think it's a good one for most use cases.
+
+----
+
+My work today has all been on making **one second** of the total lifetime
+of the WebApp work. It's the very tricky second in between clicking on
+"Make repository" and being redirected to a WebApp running in your new
+repository. The trickiness involves threads, and MVars, and
+multiple web servers, and I don't want to go into details here.
+I'd rather forget. ;-)
+
+Anyway, it works; you can run "git annex webapp" and be walked right
+through to having a usable repository! Now I need to see about adding
+that to the desktop menus, and making "git annex webapp", when run a second
+time, remembering where your repository is. I'll use
+`~/.config/git-annex/repository` for storing that.
diff --git a/doc/design/assistant/blog/day_50__directory_name/comment_1_782cec95a8558a05b2b38a2d2302214d._comment b/doc/design/assistant/blog/day_50__directory_name/comment_1_782cec95a8558a05b2b38a2d2302214d._comment
new file mode 100644
index 0000000000..be55932f13
--- /dev/null
+++ b/doc/design/assistant/blog/day_50__directory_name/comment_1_782cec95a8558a05b2b38a2d2302214d._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="https://www.google.com/accounts/o8/id?id=AItOawl9sYlePmv1xK-VvjBdN-5doOa_Xw-jH4U"
+ nickname="Richard"
+ subject="comment 1"
+ date="2012-08-01T21:19:03Z"
+ content="""
+Please use `$XDG_CONFIG_HOME` and fall back to `~/.config` if that's not defined.
+"""]]
diff --git a/doc/design/assistant/blog/day_50__directory_name/comment_2_2b8ceb0a26f25e8ed2711bcbe7225a58._comment b/doc/design/assistant/blog/day_50__directory_name/comment_2_2b8ceb0a26f25e8ed2711bcbe7225a58._comment
new file mode 100644
index 0000000000..a09b721be9
--- /dev/null
+++ b/doc/design/assistant/blog/day_50__directory_name/comment_2_2b8ceb0a26f25e8ed2711bcbe7225a58._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="http://piotr.ozarowski.pl/"
+ nickname="POX"
+ subject="XDG"
+ date="2012-08-02T10:30:55Z"
+ content="""
+same for XDG_DESKTOP_DIR (for pl_PL ~/.config/user-dirs.dirs file contains XDG_DESKTOP_DIR=\"$HOME/Pulpit\" instead of Desktop)
+"""]]
diff --git a/doc/design/assistant/blog/day_51__desktop.mdwn b/doc/design/assistant/blog/day_51__desktop.mdwn
new file mode 100644
index 0000000000..c079a9a912
--- /dev/null
+++ b/doc/design/assistant/blog/day_51__desktop.mdwn
@@ -0,0 +1,34 @@
+Now installing git-annex automatically generates a freedesktop.org .desktop
+file, and installs it, either system-wide (root) or locally (user). So
+`Menu -> Internet -> Git Annex` will start up the web app.
+
+(I don't entirely like putting it on the Internet menu, but the
+Accessories menu is not any better (and much more crowded here), 
+and there's really no menu where it entirely fits.)
+
+I generated that file by writing a generic library to deal with
+freedesktop.org desktop files and locations. Which seemed like overkill at
+the time, but then I found myself continuing to use that library. Funny how
+that happens.
+
+So, there's also another .desktop file that's used to autostart the
+`git-annex assistant` daemon when the user logs into the desktop.
+
+This even works when git-annex is installed to the ugly non-PATH location
+`.cabal/bin/git-annex` by Cabal! To make that work, it records the path
+the binary is at to a freedesktop.org data file, at install time.
+
+---
+
+That should all work in Gnome, KDE, XFCE, etc. Not Mac OSX I'm guessing...
+
+---
+
+Also today, I added a sidebar notification when the assistant notices new
+files. To make that work well, I implemented merging of related sidebar
+action notifications, so the effect is that there's one notification that
+collectes a list of recently added files, and transient notifications that
+show up if a really big file is taking a while to checksum.
+
+I'm pleased that the notification interface is at a point where I was able
+to  implement all that, entirely in pure functional code.
diff --git a/doc/design/assistant/blog/day_52__file_browser.mdwn b/doc/design/assistant/blog/day_52__file_browser.mdwn
new file mode 100644
index 0000000000..a9762cc096
--- /dev/null
+++ b/doc/design/assistant/blog/day_52__file_browser.mdwn
@@ -0,0 +1,21 @@
+Today I added a "Files" link in the navbar of the WebApp. It looks like a
+regular hyperlink, but clicking on it opens up your desktop's native file
+manager, to manage the files in the repository!
+
+Quite fun to be able to do this kind of thing from a web page. :)
+
+---
+
+Made `git annex init` (and the WebApp) automatically generate a description
+of the repo when none is provided.
+
+---
+
+Also worked on the configuration pages some. I don't want to get ahead
+of myself by diving into the full configuration stage yet, but I am at
+least going to add a configuration screen to clone the repo to a removable
+drive.
+
+After that, the list of transfers on the dashboard needs some love.
+I'll probably start by adding UI to cancel running transfers, and then
+try to get drag and drop reordering of transfers working.
diff --git a/doc/design/assistant/blog/day_52__file_browser/comment_1_cd000c2d56b60cc1f17b221322a32aa7._comment b/doc/design/assistant/blog/day_52__file_browser/comment_1_cd000c2d56b60cc1f17b221322a32aa7._comment
new file mode 100644
index 0000000000..206b0c70e4
--- /dev/null
+++ b/doc/design/assistant/blog/day_52__file_browser/comment_1_cd000c2d56b60cc1f17b221322a32aa7._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="http://cweiske.de/"
+ nickname="cweiske"
+ subject="comment 1"
+ date="2012-08-15T22:02:14Z"
+ content="""
+I thought that most (if not all) browsers prevent opening local files/directories via web pages. How did you solve that problem?
+"""]]
diff --git a/doc/design/assistant/blog/day_52__file_browser/comment_2_21d1da67cf9105a545583ba2302c10fb._comment b/doc/design/assistant/blog/day_52__file_browser/comment_2_21d1da67cf9105a545583ba2302c10fb._comment
new file mode 100644
index 0000000000..9ba5104e2a
--- /dev/null
+++ b/doc/design/assistant/blog/day_52__file_browser/comment_2_21d1da67cf9105a545583ba2302c10fb._comment
@@ -0,0 +1,7 @@
+[[!comment format=mdwn
+ username="http://joeyh.name/"
+ subject="comment 2"
+ date="2012-08-16T23:34:12Z"
+ content="""
+Yes, it's surprising to me when I see it work, and I wrote it. :) Since git-annex is running locally on the same system as the browser, it can open the file manager when the user clicks on a link in the web browser. To the web browser, this is just an AJAX request.
+"""]]
diff --git a/doc/design/assistant/blog/day_54__adding_removable_drives.mdwn b/doc/design/assistant/blog/day_54__adding_removable_drives.mdwn
new file mode 100644
index 0000000000..8d57518bb5
--- /dev/null
+++ b/doc/design/assistant/blog/day_54__adding_removable_drives.mdwn
@@ -0,0 +1,99 @@
+Spent yesterday and today making the WebApp handle adding removable drives.
+
+While it needs more testing, I think that it's now possible to use the WebApp
+for a complete sneakernet usage scenario.
+
+* Start up the webapp, let it make a local repo.
+* Add some files, by clicking to open the file manager, and dragging them in.
+* Plug in a drive, and tell the webapp to add it.
+* Wait while files sync..
+* Take the drive to another computer, and repeat the process there.
+
+No command-line needed, and files will automatically be synced between
+two or more computers using the drive.
+
+Sneakernet is only one usage scenario for the git-annex assistant, but I'm
+really happy to have one scenario 100% working!
+
+Indeed, since the assistant and webapp can now actually do something
+useful, I'll probably be merging them into `master` soon.
+
+Details follow..
+
+---
+
+So, yesterday's part of this was building the configuration page to add
+a removable drive. That needs to be as simple as possible, and it currently
+consists of a list of things git-annex thinks might be mount points of
+removable drives, along with how much free space they have. Pick a drive,
+click the pretty button, and away it goes..
+
+(I decided to make the page so simple it doesn't even ask where you want
+to put the directory on the removable drive. It always puts it in
+a "annex" directory. I might add an expert screen later, but experts can
+always set this up themselves at the command line too.)
+
+I also fought with Yesod and Bootstrap rather a lot to make the form look good.
+Didn't entirely succeed, and had to file a bug on Yesod about its handling of
+check boxes. (Bootstrap also has a bug, IMHO; its drop down lists are not
+always sized wide enough for their contents.)
+
+Ideally this configuration page would listen for mount events, and refresh
+its list. I may add that eventually; I didn't have a handy channel it
+could use to do that, so defferred it. Another idea is to have the mount
+event listener detect removable drives that don't have an annex on them yet,
+and pop up an alert with a link to this configuration page.
+
+----
+
+Making the form led to a somewhat interesting problem: How to tell if a mounted
+filesystem is a removable drive, or some random thing like `/proc` or
+a fuse filesystem. My answer, besides checking that the user can
+write to it, was various heuristics, which seem to work ok, at least here..
+
+[[!format haskell """
+               sane Mntent { mnt_dir = dir, mnt_fsname = dev }
+                        {- We want real disks like /dev/foo, not
+                         - dummy mount points like proc or tmpfs or
+                         - gvfs-fuse-daemon. -}
+                        | not ('/' `elem` dev) = False
+                        {- Just in case: These mount points are surely not
+                         - removable disks. -}
+                        | dir == "/" = False
+                        | dir == "/tmp" = False
+                        | dir == "/run/shm" = False
+                        | dir == "/run/lock" = False
+"""]]
+
+----
+
+Today I did all the gritty coding to make it create a git repository on the
+removable drive, and tell the Annex monad about it, and ensure it gets synced.
+
+As part of that, it detects when the removable drive's filesystem doesn't
+support symlinks, and makes a bare repository in that case. Another expert
+level config option that's left out for now is to always make a bare
+repository, or even to make a directory special remote rather than a git
+repository at all. (But directory special remotes cannot support the
+sneakernet use case by themselves...)
+
+----
+
+Another somewhat interesting problem was what to call the git remotes
+that it sets up on the removable drive and the local repository.
+Again this could have an expert-level configuration, but the defaults
+I chose are to use the hostname as the remote name on the removable drive,
+and to use the basename of the mount point of the removable drive as the 
+remote name in the local annex.
+
+----
+
+Originally, I had thought of this as cloning the repository to the drive.
+But, partly due to luck, I started out just doing a `git init` to make
+the repository (I had a function lying around to do that..).
+
+And as I worked on it some more, I realized this is not as simple as a
+clone. It's a bi-directional sync/merge, and indeed the removable drive may
+have all the data already in it, and the local repository have just been
+created. Handling all the edge cases of that (like, the local repository
+may not have a "master" branch yet..) was fun!
diff --git a/doc/design/assistant/blog/day_54__adding_removable_drives/comment_1_5de4f220a3534f55b1f2208d1d812d63._comment b/doc/design/assistant/blog/day_54__adding_removable_drives/comment_1_5de4f220a3534f55b1f2208d1d812d63._comment
new file mode 100644
index 0000000000..56f513a38c
--- /dev/null
+++ b/doc/design/assistant/blog/day_54__adding_removable_drives/comment_1_5de4f220a3534f55b1f2208d1d812d63._comment
@@ -0,0 +1,10 @@
+[[!comment format=mdwn
+ username="https://www.google.com/accounts/o8/id?id=AItOawl9sYlePmv1xK-VvjBdN-5doOa_Xw-jH4U"
+ nickname="Richard"
+ subject="comment 1"
+ date="2012-08-05T23:50:57Z"
+ content="""
+Using `annex` as default is fine, but even non-technical users will need a way to create different annexes for different usage. Or if two users share one thumb drive and each has their own annex. Long story short, I don't think this is an expert option, but something every user should be able to change immediately.
+
+A pre-populated text area makes most sense, imo.
+"""]]
diff --git a/doc/design/assistant/blog/day_54__adding_removable_drives/comment_2_8dae1ed0a70acf9628b88692dc32ac5f._comment b/doc/design/assistant/blog/day_54__adding_removable_drives/comment_2_8dae1ed0a70acf9628b88692dc32ac5f._comment
new file mode 100644
index 0000000000..b26d9b85f7
--- /dev/null
+++ b/doc/design/assistant/blog/day_54__adding_removable_drives/comment_2_8dae1ed0a70acf9628b88692dc32ac5f._comment
@@ -0,0 +1,10 @@
+[[!comment format=mdwn
+ username="http://dieter-be.myopenid.com/"
+ nickname="dieter"
+ subject="comment 2"
+ date="2012-08-06T08:30:47Z"
+ content="""
++1 on what Richard said
+about the correctly figuring out what are valid mountpoints / devices that can be used, your current code doesn't seem very robust.
+check this out https://github.com/Dieterbe/aif/blob/develop/src/core/libs/lib-blockdevices-filesystems.sh#L213
+"""]]
diff --git a/doc/design/assistant/blog/day_55__alerts.mdwn b/doc/design/assistant/blog/day_55__alerts.mdwn
new file mode 100644
index 0000000000..95e52e8ad6
--- /dev/null
+++ b/doc/design/assistant/blog/day_55__alerts.mdwn
@@ -0,0 +1,10 @@
+Nothing flashy today; I was up all night trying to download photos taken 
+by a robot lowered onto Mars by a skycrane.
+
+Some work on alerts. Added an alert when a file transfer succeeds or fails.
+Improved the alert combining code so it handles those alerts, and
+simplified it a lot, and made it more efficient.
+
+Also made the text of action alerts change from present to past tense when
+the action finishes. To support that I wrote a fun data type, a `TenseString`
+that can be rendered in either tense.
diff --git a/doc/design/assistant/blog/day_55__alerts/comment_1_6319045500a8a5e049304fdec5ff4cf4._comment b/doc/design/assistant/blog/day_55__alerts/comment_1_6319045500a8a5e049304fdec5ff4cf4._comment
new file mode 100644
index 0000000000..67259bf1e5
--- /dev/null
+++ b/doc/design/assistant/blog/day_55__alerts/comment_1_6319045500a8a5e049304fdec5ff4cf4._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="https://www.google.com/accounts/o8/id?id=AItOawlJ2utMQgMEYAOs3Dfc6eZRyUzt4acNXUU"
+ nickname="David"
+ subject="comment 1"
+ date="2012-08-06T23:48:28Z"
+ content="""
+I see you are a regular XKCD reader
+"""]]
diff --git a/doc/design/assistant/blog/day_56__transfer_control.mdwn b/doc/design/assistant/blog/day_56__transfer_control.mdwn
new file mode 100644
index 0000000000..1ce9268974
--- /dev/null
+++ b/doc/design/assistant/blog/day_56__transfer_control.mdwn
@@ -0,0 +1,8 @@
+A bit under the weather, but got into building buttons to control running
+and queued transfers today. The html and javascript side is done, with
+each transfer now having a cancel button, as well as a pause/start button.
+
+Canceling queued transfers works. Canceling running transfers will
+need some more work, because killing a thread doesn't kill the processes
+being run by that thread. So I'll have to make the assistant run separate
+git-annex processes for transfers, that can be individually sent signals.
diff --git a/doc/design/assistant/blog/day_57__afk.mdwn b/doc/design/assistant/blog/day_57__afk.mdwn
new file mode 100644
index 0000000000..c72849ce73
--- /dev/null
+++ b/doc/design/assistant/blog/day_57__afk.mdwn
@@ -0,0 +1,40 @@
+Probably won't be doing any big coding on the git-annex assistant in the
+upcoming week, as I'll be traveling and/or slightly ill enough that I can't
+fully get into flow.
+
+---
+
+There was a new Yesod release this week, which required minor changes to
+make the webapp build with it. I managed to keep the old version of Yesod
+also supported, and plan to keep that working so it can be built with the
+version of Yesod available in, eg, Linux distributions. TBD how much pain
+that will involve going forward.
+
+---
+
+I'm mulling over how to support stopping/pausing transfers. The problem
+is that if the assistant is running a transfer in one thread, and the
+webapp is used to cancel it, killing that thread won't necessarily stop the
+transfer, because, at least in Haskell's thread model, killing a thread
+does not kill processes started by the thread (like rsync).
+
+So one option is to have the transfer thread run a separate git-annex
+process, which will run the actual transfer. And killing that process will
+stop the transfer nicely. However, using a separate git-annex process means
+a little startup overhead for each file transferred (I don't know if it'd
+be enough to matter). Also, there's the problem that git-annex is sometimes
+not installed in PATH (wish I understood why cabal does that), which
+makes it kind of hard for it to run itself. (It can't simply fork, sadly.
+See past horrible pain with forking and threads.)
+
+The other option is to change the API for git-annex remotes, so that
+their `storeKey` and `retrieveKeyFile` methods return a pid of the program
+that they run. When they *do* run a program.. not all remotes do. This
+seems like it'd make the code in the remotes hairier, and it is also asking
+for bugs, when a remote's implementation changes. Or I could go
+lower-level, and make every place in the utility libraries that forks a
+process record its pid in a per-thread MVar. Still seems to be asking for
+bugs.
+
+Oh well, at least git-annex is already crash-safe, so once I figure out
+how to kill a transfer process, I can kill it safely. :)
diff --git a/doc/design/assistant/blog/day_57__afk/comment_1_70e1c9f925f040c1700d3e26bab373d5._comment b/doc/design/assistant/blog/day_57__afk/comment_1_70e1c9f925f040c1700d3e26bab373d5._comment
new file mode 100644
index 0000000000..c83403620a
--- /dev/null
+++ b/doc/design/assistant/blog/day_57__afk/comment_1_70e1c9f925f040c1700d3e26bab373d5._comment
@@ -0,0 +1,9 @@
+[[!comment format=mdwn
+ username="http://claimid.com/strager"
+ nickname="strager"
+ subject="comment 1"
+ date="2012-08-11T04:50:52Z"
+ content="""
+What if `storeKey`, `retrieveKeyFile`, etc. return an `IO ()` which cancels the operation, if possible?  The implementation can be canceled regardless if it uses separate processes or Haskell threads.
+
+"""]]
diff --git a/doc/design/assistant/blog/day_57__afk/comment_2_c70d3faccfcebf47deb25e270498cb56._comment b/doc/design/assistant/blog/day_57__afk/comment_2_c70d3faccfcebf47deb25e270498cb56._comment
new file mode 100644
index 0000000000..7539ea2def
--- /dev/null
+++ b/doc/design/assistant/blog/day_57__afk/comment_2_c70d3faccfcebf47deb25e270498cb56._comment
@@ -0,0 +1,18 @@
+[[!comment format=mdwn
+ username="http://claimid.com/strager"
+ nickname="strager"
+ subject="comment 2"
+ date="2012-08-11T04:55:13Z"
+ content="""
+In fact, making a dedicated data type or some typeclasses may be more appropriate:
+
+    class Cancelable a where cancel :: a -> IO ()
+    class Pauseable a where pause :: a -> IO ()
+
+    -- Alternatively:
+
+    data Transfer = Transfer { cancel :: IO (), pause :: IO () }
+
+    -- Or both!
+
+"""]]
diff --git a/doc/design/assistant/blog/day_57__afk/comment_3_89020ebc6d31485339bdea41a872df3c._comment b/doc/design/assistant/blog/day_57__afk/comment_3_89020ebc6d31485339bdea41a872df3c._comment
new file mode 100644
index 0000000000..ed02b9dcee
--- /dev/null
+++ b/doc/design/assistant/blog/day_57__afk/comment_3_89020ebc6d31485339bdea41a872df3c._comment
@@ -0,0 +1,11 @@
+[[!comment format=mdwn
+ username="http://joeyh.name/"
+ subject="comment 3"
+ date="2012-08-11T14:41:51Z"
+ content="""
+That's the lines I was thinking along, and I even made a throwaway branch with some types. But the problem is reworking all the code to do that. Particularly since lots of the code uses generic utility functions that are reused in other, unrelated places and would have to be modified to pass back cancel actions.
+
+The first case the type checker landed me on when I changed the types was code that downloads an url from the web. Naturally that uses a Utility.Url.download. How to cancel `download`? Depends on its implementation -- it happens to currently shell out to curl, so you have to kill curl, but it could just as easily have used libcurl (other parts of my Utility.Url library do), and then it would need to fork its own thread. So it's an abstraction layer violation problem.
+
+If I had a month to devote to this one problem, I might manage to come up with some clean solution involving monads, or maybe convert all my code to use conduit or something that might allow managing these effects better. Just a guess..
+"""]]
diff --git a/doc/design/assistant/blog/day_57__afk/comment_4_8b1f65f141ffd9813e7f5a3380f7f520._comment b/doc/design/assistant/blog/day_57__afk/comment_4_8b1f65f141ffd9813e7f5a3380f7f520._comment
new file mode 100644
index 0000000000..c87e447d1c
--- /dev/null
+++ b/doc/design/assistant/blog/day_57__afk/comment_4_8b1f65f141ffd9813e7f5a3380f7f520._comment
@@ -0,0 +1,27 @@
+[[!comment format=mdwn
+ username="http://claimid.com/strager"
+ ip="173.228.13.253"
+ subject="comment 4"
+ date="2012-08-11T16:08:47Z"
+ content="""
+> How to cancel download?  Depends on its implementation ....  So it's an abstraction layer violation problem.
+
+Precisely why I suggested returning something as generic as `IO ()`:
+
+    -- Current
+    download :: URLString -> Headers -> [CommandParam] -> FilePath -> IO Bool
+
+    -- Suggestion
+    data Transfer a = Transfer { run :: IO a, cancel :: IO () }
+    download :: URLString -> Headers -> [CommandParam] -> FilePath -> Transfer
+
+    transfer <- download ...
+    -- You can pass `cancel transfer` to another thread
+    -- which you want to be able to cancel the transfer.
+    run transfer  -- blocking
+
+I realized while writing this that you may not get any result from e.g. a download while it is occurring (because the function is blocking).  Maybe that's where a misunderstanding occurred.  I separated the concepts of creating a transfer and starting/canceling it.
+
+(My idea is starting to feel a bit object-oriented...  ;P)
+
+"""]]
diff --git a/doc/design/assistant/blog/day_58__more_transfer_control.mdwn b/doc/design/assistant/blog/day_58__more_transfer_control.mdwn
new file mode 100644
index 0000000000..08b69a75d5
--- /dev/null
+++ b/doc/design/assistant/blog/day_58__more_transfer_control.mdwn
@@ -0,0 +1,26 @@
+Unexpectedly managed a mostly productive day today. 
+
+Went ahead with making the assistant run separate `git-annex` processes for
+transfers. This will currently fail if git-annex is not installed in PATH.
+(Deferred dealing with that.)
+
+To stop a transfer, the webapp needs to signal not just the git-annex
+process, but all its children. I'm using process groups for this, which is
+working, but I'm not extremely happy with.
+
+Anyway, the webapp's UI can now be used for stopping transfers, and it
+wasn't far from there to also implementing pausing of transfers. 
+
+Pausing a transfer is actually the same as stopping it, except a special
+signal is sent to the transfer control thread, which keeps running, despite
+the git-annex process having been killed, waits for a special resume
+signal, and restarts the transfer. This way a paused transfer continues to
+occupy a transfer slot, which prevents other queued transfers from running.
+This seems to be the behavior that makes sense.
+
+Still need to wire up the webapp's button for starting a transfer. For a
+paused transfer, that will just need to resume it. I have not decided what
+the button should do when used on a transfer that is queued but not running
+yet. Maybe it forces it to run even if all transfer slots are already in
+use? Maybe it stops one of the currently running transfers to free up a
+slot?
diff --git a/doc/design/assistant/blog/day_59__dinner.mdwn b/doc/design/assistant/blog/day_59__dinner.mdwn
new file mode 100644
index 0000000000..545125ac66
--- /dev/null
+++ b/doc/design/assistant/blog/day_59__dinner.mdwn
@@ -0,0 +1,10 @@
+Actually did do some work on the webapp today, just fixing a bug I noticed
+in a spare moment. Also managed a bit in the plane earlier this week,
+implementing resuming of paused transfers. (Still need to test that.)
+
+But the big thing today was dinner with one of my major Kickstarter
+backers, and as it turned out, "half the Haskell community of San
+Francisco" (3 people). Enjoyed talking about git-annex and haskell with
+them.
+
+I'm looking forward to getting back home and back to work on Monday..
diff --git a/doc/design/assistant/blog/day_59__dinner/comment_1_0c1e2d69496473e7e4a2956a2814f5dd._comment b/doc/design/assistant/blog/day_59__dinner/comment_1_0c1e2d69496473e7e4a2956a2814f5dd._comment
new file mode 100644
index 0000000000..1e10549d37
--- /dev/null
+++ b/doc/design/assistant/blog/day_59__dinner/comment_1_0c1e2d69496473e7e4a2956a2814f5dd._comment
@@ -0,0 +1,9 @@
+[[!comment format=mdwn
+ username="http://claimid.com/strager"
+ ip="173.228.13.253"
+ subject="comment 1"
+ date="2012-08-17T15:59:49Z"
+ content="""
+It was fun to have you here!  =]
+
+"""]]
diff --git a/doc/design/assistant/blog/day_5__committing.mdwn b/doc/design/assistant/blog/day_5__committing.mdwn
new file mode 100644
index 0000000000..5623873805
--- /dev/null
+++ b/doc/design/assistant/blog/day_5__committing.mdwn
@@ -0,0 +1,57 @@
+After a few days otherwise engaged, back to work today.
+
+My focus was on adding the committing thread mentioned in [[day_4__speed]].
+I got rather further than expected!
+
+First, I implemented a really dumb thread, that woke up once per second,
+checked if any changes had been made, and committed them. Of course, this
+rather sucked. In the middle of a large operation like untarring a tarball,
+or `rm -r` of a large directory tree, it made lots of commits and made
+things slow and ugly. This was not unexpected.
+
+So next, I added some smarts to it. First, I wanted to stop it waking up
+every second when there was nothing to do, and instead blocking wait on a
+change occurring. Secondly, I wanted it to know when past changes happened,
+so it could detect batch mode scenarios, and avoid committing too
+frequently. 
+
+I played around with combinations of various Haskell thread communications
+tools to get that information to the committer thread: `MVar`, `Chan`,
+`QSem`, `QSemN`. Eventually, I realized all I needed was a simple channel
+through which the timestamps of changes could be sent. However, `Chan`
+wasn't quite suitable, and I had to add a dependency on 
+[Software Transactional Memory](http://en.wikipedia.org/wiki/Software_Transactional_Memory),
+and use a `TChan`. Now I'm cooking with gas!
+
+With that data channel available to the committer thread, it quickly got
+some very nice smart behavior. Playing around with it, I find it commits
+*instantly* when I'm making some random change that I'd want the
+git-annex assistant to sync out instantly; and that its batch job detection
+works pretty well too.
+
+There's surely room for improvement, and I made this part of the code
+be an entirely pure function, so it's really easy to change the strategy.
+This part of the committer thread is so nice and clean, that here's the
+current code, for your viewing pleasure:
+
+[[!format haskell """
+{- Decide if now is a good time to make a commit.
+ - Note that the list of change times has an undefined order.
+ -
+ - Current strategy: If there have been 10 commits within the past second,
+ - a batch activity is taking place, so wait for later.
+ -}
+shouldCommit :: UTCTime -> [UTCTime] -> Bool
+shouldCommit now changetimes
+       | len == 0 = False
+       | len > 4096 = True -- avoid bloating queue too much
+       | length (filter thisSecond changetimes) < 10 = True
+       | otherwise = False -- batch activity
+       where
+               len = length changetimes
+               thisSecond t = now `diffUTCTime` t <= 1
+"""]]
+
+Still some polishing to do to eliminate minor inefficiencies and deal
+with more races, but this part of the git-annex assistant is now very usable,
+and will be going out to my beta testers soon!
diff --git a/doc/design/assistant/blog/day_60__taking_stock.mdwn b/doc/design/assistant/blog/day_60__taking_stock.mdwn
new file mode 100644
index 0000000000..040d0cd7c1
--- /dev/null
+++ b/doc/design/assistant/blog/day_60__taking_stock.mdwn
@@ -0,0 +1,40 @@
+As I prepare to dive back into development, now is a good time to review
+what I've built so far, and how well I'm keeping up with my planned
+[[roadmap|assistant]].
+
+I started working two and a half months ago, so am nearing the end of
+the three months I originally asked to be funded for on Kickstarter.
+
+I've built much of what I planned to build in the first three months --
+[[inotify]] is done (and kqueue is basically working, but needs scalability
+work), local [[syncing]] is done, the [[webapp]] works, and I've built some
+of the first [[configurators]]. It's all functional in a narrow use case
+involving syncing to removable drives. 
+
+[[progressbars]] still need to be dealt with, and network syncing needs to
+be revisited soon, so that I can start building easy configurators for
+further use cases, like using the cloud, or another machine on the local
+network.
+
+I think I'm a little behind my original schedule, but not too bad,
+and at the same time, I think I've built things rather more solidly than I
+expected them to be at this point. I'm particularly happy with how well
+the inotify code works, no matter what is thrown at it, and how nice
+the UI in the webapp is shaping up to be.
+
+----
+
+I also need to get started on fulfilling my Kickstarter rewards, and
+I was happy to spend some time in the airport working on the main
+blocker toward that, a lack of a scalable git-annex logo, which is needed
+for printing on swag.
+
+Turns out that inkscape has some amazing bitmap tracing capabilities.
+I was able to come up with this scalable logo in short order, it 
+actually took longer to add back the colors, as the tracer generated a
+black and white version.
+
+With that roadblock out of the way, I am moving toward ordering large
+quantities of usb drives, etc.
+
+[[logo.svg]]
diff --git a/doc/design/assistant/blog/day_60__taking_stock/comment_1_6722f81ee084f1ea9e8fe47f34576397._comment b/doc/design/assistant/blog/day_60__taking_stock/comment_1_6722f81ee084f1ea9e8fe47f34576397._comment
new file mode 100644
index 0000000000..cfe4fa300b
--- /dev/null
+++ b/doc/design/assistant/blog/day_60__taking_stock/comment_1_6722f81ee084f1ea9e8fe47f34576397._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="http://dieter-be.myopenid.com/"
+ nickname="dieter"
+ subject="better logo?"
+ date="2012-08-20T21:22:25Z"
+ content="""
+Maybe you should contact designers and see what kind of logo they can come up with.  Probably something better.
+"""]]
diff --git a/doc/design/assistant/blog/day_61__network_connection_detection.mdwn b/doc/design/assistant/blog/day_61__network_connection_detection.mdwn
new file mode 100644
index 0000000000..8ab40f5162
--- /dev/null
+++ b/doc/design/assistant/blog/day_61__network_connection_detection.mdwn
@@ -0,0 +1,36 @@
+Today, added a thread that deals with recovering when there's been a loss
+of network connectivity. When the network's down, the normal immediate
+syncing of changes of course doesn't work. So this thread detects when the
+network comes back up, and does a pull+push to network remotes, and
+triggers scanning for file content that needs to be transferred.
+
+I used dbus again, to detect events generated by both network-manager and
+wicd when they've sucessfully brought an interface up. Or, if they're not
+available, it polls every 30 minutes.
+
+When the network comes up, in addition to the git pull+push, it also
+currently does a full scan of the repo to find files whose contents
+need to be transferred to get fully back into sync.
+
+I think it'll be ok for some git pulls and pushes to happen when
+moving to a new network, or resuming a laptop (or every 30 minutes when
+resorting to polling). But the transfer scan is currently really too heavy
+to be appropriate to do every time in those situations. I have an idea for
+avoiding that scan when the remote's git-annex branch has not changed. But
+I need to refine it, to handle cases like this:
+
+1. a new remote is added
+2. file contents start being transferred to (or from it)
+3. the network is taken down
+4. all the queued transfers fail
+5. the network comes back up
+6. the transfer scan needs to know the remote was not all in sync
+   before #3, and so should do a full scan despite the git-annex branch
+   not having changed
+
+---
+
+Doubled the ram in my netbook, which I use for all development. Yesod needs
+rather a lot of ram to compile and link, and this should make me quite a
+lot more productive. I was struggling with OOM killing bits of chromium
+during my last week of development.
diff --git a/doc/design/assistant/blog/day_61__network_connection_detection/comment_1_09b58f41a8d48f218619711ee19511ac._comment b/doc/design/assistant/blog/day_61__network_connection_detection/comment_1_09b58f41a8d48f218619711ee19511ac._comment
new file mode 100644
index 0000000000..029aec7832
--- /dev/null
+++ b/doc/design/assistant/blog/day_61__network_connection_detection/comment_1_09b58f41a8d48f218619711ee19511ac._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="https://www.google.com/accounts/o8/id?id=AItOawmubB1Sj2rwFoVdZYvGV0ACaQUJQyiJXJI"
+ nickname="Paul"
+ subject="Amazon Glacier"
+ date="2012-08-23T06:32:24Z"
+ content="""
+Do you think git-annex could support [Amazon Glacier](http://aws.amazon.com/glacier/) as a backend?
+"""]]
diff --git a/doc/design/assistant/blog/day_62__smarter_syncing.mdwn b/doc/design/assistant/blog/day_62__smarter_syncing.mdwn
new file mode 100644
index 0000000000..28fa892d38
--- /dev/null
+++ b/doc/design/assistant/blog/day_62__smarter_syncing.mdwn
@@ -0,0 +1,21 @@
+Woke up this morning with most of the design for a smarter approach to
+[[syncing]] in my head. (This is why I sometimes slip up and tell people I
+work on this project 12 hours a day..)
+
+To keep the current `assistant` branch working while I make changes
+that break use cases that are working, I've started 
+developing in a new branch, `assistant-wip`.
+
+In it, I've started getting rid of unnecessary expensive transfer scans.
+
+First optimisation I've done is to detect when a remote that was
+disconnected has diverged its `git-annex` branch from the local branch.
+Only when that's the case does a new transfer scan need to be done, to find
+out what new stuff might be available on that remote, to have caused the
+change to its branch, while it was disconnected.
+
+That broke a lot of stuff. I have a plan to fix it written down in
+[[syncing]]. It'll involve keeping track of whether a transfer scan has
+ever been done (if not, one should be run), and recording logs when
+transfers failed, so those failed transfers can be retried when the
+remote gets reconnected.
diff --git a/doc/design/assistant/blog/day_63__transfer_retries.mdwn b/doc/design/assistant/blog/day_63__transfer_retries.mdwn
new file mode 100644
index 0000000000..d668f507ba
--- /dev/null
+++ b/doc/design/assistant/blog/day_63__transfer_retries.mdwn
@@ -0,0 +1,26 @@
+Implemented everything I planned out yesterday: Expensive scans are only
+done once per remote (unless the remote changed while it was disconnected),
+and failed transfers are logged so they can be retried later.
+
+Changed the TransferScanner to prefer to scan low cost remotes first,
+as a crude form of scheduling lower-cost transfers first.
+
+A whole bunch of interesting syncing scenarios should work now. I have not
+tested them all in detail, but to the best of my knowledge, all these
+should work:
+
+* Connect to the network. It starts syncing with a networked remote.
+  Disconnect the network. Reconnect, and it resumes where it left off.
+* Migrate between networks (ie, home to cafe to work). Any transfers
+  that can only happen on one LAN are retried on each new network you
+  visit, until they succeed.
+
+One that is not working, but is soooo close:
+
+* Plug in a removable drive. Some transfers start. Yank the plug.
+  Plug it back in. All necessary transfers resume, and it ends up
+  fully in sync, no matter how many times you yank that cable.
+
+That's not working because of an infelicity in the MountWatcher.
+It doesn't notice when the drive gets unmounted, so it ignores
+the new mount event.
diff --git a/doc/design/assistant/blog/day_63__transfer_retries/comment_1_990d4eb6066c4e2b9ddb3cabef32e4b9._comment b/doc/design/assistant/blog/day_63__transfer_retries/comment_1_990d4eb6066c4e2b9ddb3cabef32e4b9._comment
new file mode 100644
index 0000000000..119aee2c91
--- /dev/null
+++ b/doc/design/assistant/blog/day_63__transfer_retries/comment_1_990d4eb6066c4e2b9ddb3cabef32e4b9._comment
@@ -0,0 +1,10 @@
+[[!comment format=mdwn
+ username="https://www.google.com/accounts/o8/id?id=AItOawmBUR4O9mofxVbpb8JV9mEbVfIYv670uJo"
+ nickname="Justin"
+ subject="comment 1"
+ date="2012-08-23T21:25:48Z"
+ content="""
+Do encrypted rsync remotes resume quickly as well?
+
+One thing I noticed was that if a copy --to an encrypted rsync remote gets interrupted it will remove the tmp file and re-encrypt the whole file before resuming rsync.
+"""]]
diff --git a/doc/design/assistant/blog/day_64__syncing_robustly.mdwn b/doc/design/assistant/blog/day_64__syncing_robustly.mdwn
new file mode 100644
index 0000000000..ab0090b92d
--- /dev/null
+++ b/doc/design/assistant/blog/day_64__syncing_robustly.mdwn
@@ -0,0 +1,33 @@
+Working toward getting the data syncing to happen robustly,
+so a bunch of improvements.
+
+* Got unmount events to be noticed, so unplugging and replugging
+  a removable drive will resume the syncing to it. There's really no
+  good unmount event available on dbus in kde, so it uses a heuristic
+  there.
+* Avoid requeuing a download from a remote that no longer has a key.
+* Run a full scan on startup, for multiple reasons, including dealing with
+  crashes.
+
+Ran into a strange issue: Occasionally the assistant will run `git-annex
+copy` and it will not transfer the requested file. It seems that
+when the copy command runs `git ls-files`, it does not see the file
+it's supposed to act on in its output.
+
+Eventually I figured out what's going on: When updating the git-annex
+branch, it sets `GIT_INDEX_FILE`, and of course environment settings are
+not thread-safe! So there's a race between threads that access
+the git-annex branch, and the Transferrer thread, or any other thread
+that might expect to look at the normal git index.
+
+Unfortunatly, I don't have a fix for this yet.. Git's only interface for
+using a different index file is `GIT_INDEX_FILE`. It seems I have a lot of
+code to tear apart, to push back the setenv until after forking every git
+command. :(
+
+Before I figured out the root problem, I developed a workaround for the
+symptom I was seeing. I added a `git-annex transferkey`, which is
+optimised to be run by the assistant, and avoids running `git ls-files`, so
+avoids the problem. While I plan to fix this environment variable problem
+properly, `transferkey` turns out to be so much faster than how it was
+using `copy` that I'm going to keep it.
diff --git a/doc/design/assistant/blog/day_65__transfer_polish.mdwn b/doc/design/assistant/blog/day_65__transfer_polish.mdwn
new file mode 100644
index 0000000000..af1c691621
--- /dev/null
+++ b/doc/design/assistant/blog/day_65__transfer_polish.mdwn
@@ -0,0 +1,33 @@
+Almost done with the data transfer code.. Today I filled in some bits and
+peices.
+
+Made the expensive transfer scan handle multiple remotes in one pass.
+So on startup, it only runs once, not N times. And when reconnecting to the
+network, when a remote has changed, it scans all network remotes in one
+pass, rather than making M redundant passes.
+
+Got syncing with special remotes all working. Pretty easy actually. Just
+had to avoid doing any git repo push/pull with them, while still queueing
+data transfers.
+
+It'll even download anything it can from the web special remote. To support
+that, I added generic support for readonly remotes; it'll only download from
+those and not try to upload to them.
+
+(Oh, and I properly fixed the nasty `GIT_INDEX_FILE` environment variable
+problem I had the other day.)
+
+I feel I'm very close to being able to merge the assistant branch into
+master now. I'm reasonably confident the data transfer code will work
+well now, and manage to get things in sync eventually in all circumstances.
+(Unless there are bugs.) All the other core functionality of the assistant
+and webapp is working. The only think I might delay because of is the
+missing [[progressbars]] in the webapp .. but that's a silly thing to
+block the merge on. 
+
+Still, I might spend a day and get a dumb implementation of progress bars
+for downloads working first (progress bars for uploads are probably rather
+harder). I'd spend longer on progress bars, but there are so many more
+exciting things I'm now ready to develop, like automatic configurators
+for using your git annex with Amazon S3, rsync.net, and the computer across
+the room..!
diff --git a/doc/design/assistant/blog/day_66__the_merge.mdwn b/doc/design/assistant/blog/day_66__the_merge.mdwn
new file mode 100644
index 0000000000..0442865ba3
--- /dev/null
+++ b/doc/design/assistant/blog/day_66__the_merge.mdwn
@@ -0,0 +1,19 @@
+It's done! The assistant branch is merged into master.
+
+Updated the [[assistant]] page with some screenshots and instructions for
+using it.
+
+Made some cosmetic fixes to the webapp.
+
+Fixed the transferrer to use `~/.config/git-annex/program`
+to find the path to git-annex when running it. (There are ways to find the
+path of the currently running program in unux, but they all suck, so I'm
+avoiding them this way.)
+
+Read some OSX launchd documentation, and it seems it'd be pretty easy to
+get the assistant to autostart on login on OSX. If someone would like to
+test launchd files for me, get in touch.
+
+-----
+
+AKA: Procrastinating really hard on those progress bars. ;)
diff --git a/doc/design/assistant/blog/day_66__the_merge/comment_10_eeccf4e73cc321542a1fe4780805a81e._comment b/doc/design/assistant/blog/day_66__the_merge/comment_10_eeccf4e73cc321542a1fe4780805a81e._comment
new file mode 100644
index 0000000000..2e787076d8
--- /dev/null
+++ b/doc/design/assistant/blog/day_66__the_merge/comment_10_eeccf4e73cc321542a1fe4780805a81e._comment
@@ -0,0 +1,12 @@
+[[!comment format=mdwn
+ username="http://kevwalke.myopenid.com/"
+ ip="94.175.38.180"
+ subject="confirmed!"
+ date="2012-09-02T07:17:43Z"
+ content="""
+Got it built and running. Thanks for this.
+I can see already that this is going to be a fantastic application. 
+
+Good Work!
+-Kev
+"""]]
diff --git a/doc/design/assistant/blog/day_66__the_merge/comment_1_a34e89316d1662826848f31061c4e46b._comment b/doc/design/assistant/blog/day_66__the_merge/comment_1_a34e89316d1662826848f31061c4e46b._comment
new file mode 100644
index 0000000000..be612801e3
--- /dev/null
+++ b/doc/design/assistant/blog/day_66__the_merge/comment_1_a34e89316d1662826848f31061c4e46b._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="http://kevwalke.myopenid.com/"
+ ip="94.175.38.180"
+ subject="well done!"
+ date="2012-08-27T22:08:23Z"
+ content="""
+absolutely fantastic news! I am itching to try this out now. Thank you!
+"""]]
diff --git a/doc/design/assistant/blog/day_66__the_merge/comment_2_09e244d23d05052fa2b11a7181888366._comment b/doc/design/assistant/blog/day_66__the_merge/comment_2_09e244d23d05052fa2b11a7181888366._comment
new file mode 100644
index 0000000000..874aa5ea6a
--- /dev/null
+++ b/doc/design/assistant/blog/day_66__the_merge/comment_2_09e244d23d05052fa2b11a7181888366._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="http://claimid.com/strager"
+ ip="173.228.13.253"
+ subject="comment 2"
+ date="2012-08-28T16:29:56Z"
+ content="""
+I have a few OS X machines I can test on.  Send me details at: strager.nds@gmail.com
+"""]]
diff --git a/doc/design/assistant/blog/day_66__the_merge/comment_3_3961a03e167903959b96b054835613f6._comment b/doc/design/assistant/blog/day_66__the_merge/comment_3_3961a03e167903959b96b054835613f6._comment
new file mode 100644
index 0000000000..4a4d0df409
--- /dev/null
+++ b/doc/design/assistant/blog/day_66__the_merge/comment_3_3961a03e167903959b96b054835613f6._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="https://www.google.com/accounts/o8/id?id=AItOawkSq2FDpK2n66QRUxtqqdbyDuwgbQmUWus"
+ nickname="Jimmy"
+ subject="comment 3"
+ date="2012-08-28T18:44:28Z"
+ content="""
+As per usual I'd volunteer to test on OSX as well.
+"""]]
diff --git a/doc/design/assistant/blog/day_66__the_merge/comment_4_12a57af9f580918818b4a9f68396d5c4._comment b/doc/design/assistant/blog/day_66__the_merge/comment_4_12a57af9f580918818b4a9f68396d5c4._comment
new file mode 100644
index 0000000000..6852a805a7
--- /dev/null
+++ b/doc/design/assistant/blog/day_66__the_merge/comment_4_12a57af9f580918818b4a9f68396d5c4._comment
@@ -0,0 +1,23 @@
+[[!comment format=mdwn
+ username="http://kevwalke.myopenid.com/"
+ ip="77.86.30.139"
+ subject="building and installing with cabal"
+ date="2012-08-29T18:15:07Z"
+ content="""
+So I got round to trying it out. I cloned the latest from the repository@ git://git-annex.branchable.com/ and then followed the instructions on the install page: http://git-annex.branchable.com/install/cabal/
+However I get a dependency problem:
+
+cabal install --only-dependencies
+Resolving dependencies...
+cabal: Could not resolve dependencies:
+trying: git-annex-3.20120825 (user goal)
+trying: git-annex-3.20120825:+oldyesod
+trying: git-annex-3.20120825:+currentyesod
+next goal: yesod-default (dependency of git-annex-3.20120825:+oldyesod)
+rejecting: yesod-default-1.1.0 (conflict: git-annex-3.20120825:oldyesod =>
+yesod-default(<=1.0.1.1))
+rejecting: yesod-default-1.0.1.1, 1.0.1, 1.0.0, 0.6.1, 0.5.0, 0.4.1, 0.4.0,
+0.3.1 (conflict: git-annex-3.20120825:currentyesod => yesod-default(>=1.1.0))
+
+I'm not sure how to proceed from here and would be grateful of any pointers to help get this built
+"""]]
diff --git a/doc/design/assistant/blog/day_66__the_merge/comment_5_8ce638960012367c888e018a5f05db19._comment b/doc/design/assistant/blog/day_66__the_merge/comment_5_8ce638960012367c888e018a5f05db19._comment
new file mode 100644
index 0000000000..f51f5ef528
--- /dev/null
+++ b/doc/design/assistant/blog/day_66__the_merge/comment_5_8ce638960012367c888e018a5f05db19._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="http://joeyh.name/"
+ ip="4.152.246.16"
+ subject="comment 5"
+ date="2012-08-29T22:17:01Z"
+ content="""
+@kevwalke: I fixed that problem today.
+"""]]
diff --git a/doc/design/assistant/blog/day_66__the_merge/comment_6_f461b856b940e6914bcd2b681cf9505f._comment b/doc/design/assistant/blog/day_66__the_merge/comment_6_f461b856b940e6914bcd2b681cf9505f._comment
new file mode 100644
index 0000000000..2232c426b8
--- /dev/null
+++ b/doc/design/assistant/blog/day_66__the_merge/comment_6_f461b856b940e6914bcd2b681cf9505f._comment
@@ -0,0 +1,13 @@
+[[!comment format=mdwn
+ username="http://kevwalke.myopenid.com/"
+ ip="77.86.30.139"
+ subject="getting new error now."
+ date="2012-08-30T11:55:49Z"
+ content="""
+Seems there is still an issue here on fresh install...
+
+cabal: Error: some packages failed to install:
+dbus-0.10 depends on libxml-sax-0.7.2 which failed to install.
+libxml-sax-0.7.2 failed during the configure step. The exception was:
+ExitFailure 
+"""]]
diff --git a/doc/design/assistant/blog/day_66__the_merge/comment_7_6e73aca1fc1747d0e742e054b88b5d78._comment b/doc/design/assistant/blog/day_66__the_merge/comment_7_6e73aca1fc1747d0e742e054b88b5d78._comment
new file mode 100644
index 0000000000..0cb970a4cd
--- /dev/null
+++ b/doc/design/assistant/blog/day_66__the_merge/comment_7_6e73aca1fc1747d0e742e054b88b5d78._comment
@@ -0,0 +1,12 @@
+[[!comment format=mdwn
+ username="http://joeyh.name/"
+ ip="4.152.246.16"
+ subject="comment 7"
+ date="2012-08-30T17:15:58Z"
+ content="""
+It's hard to tell from that error message what's going on. I'd hope that there was actually a real error message printed earlier than that that.
+
+For now, I can only guess.. In order to install the haskell libxml-sax library, you need to have the GNOME XML2 library installed, including development headers. On Debian and derivatives, that is `apt-get install libxml2-dev`. 
+
+Alternatively, you can disable use of dbus by passing this to your cabal install: `--flags=-Dbus`
+"""]]
diff --git a/doc/design/assistant/blog/day_66__the_merge/comment_8_d85f1ce23ae16d5a8eb88d2c3999acb7._comment b/doc/design/assistant/blog/day_66__the_merge/comment_8_d85f1ce23ae16d5a8eb88d2c3999acb7._comment
new file mode 100644
index 0000000000..f2068146d0
--- /dev/null
+++ b/doc/design/assistant/blog/day_66__the_merge/comment_8_d85f1ce23ae16d5a8eb88d2c3999acb7._comment
@@ -0,0 +1,19 @@
+[[!comment format=mdwn
+ username="http://kevwalke.myopenid.com/"
+ ip="94.175.38.180"
+ subject="thanks"
+ date="2012-09-01T14:44:58Z"
+ content="""
+That got rid of that error. However now I get the following at the last step:
+
+Linking dist/build/git-annex/git-annex ...
+Installing executable(s) in /home/kev/bin
+setup: git-annex.1: does not exist
+cabal: Error: some packages failed to install:
+git-annex-3.20120826 failed during the final install step. The exception was:
+ExitFailure 1
+
+There doesn't seem to be any failures further up the output just some warnings. All the other steps complete fine
+Thanks for any advice/pointers here
+-Kevin
+"""]]
diff --git a/doc/design/assistant/blog/day_66__the_merge/comment_9_c06dab4d78122c85beeaf300ffc3e376._comment b/doc/design/assistant/blog/day_66__the_merge/comment_9_c06dab4d78122c85beeaf300ffc3e376._comment
new file mode 100644
index 0000000000..5a2b3c8d19
--- /dev/null
+++ b/doc/design/assistant/blog/day_66__the_merge/comment_9_c06dab4d78122c85beeaf300ffc3e376._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="http://joeyh.name/"
+ ip="4.152.246.16"
+ subject="comment 9"
+ date="2012-09-01T19:32:02Z"
+ content="""
+I think I've fixed that.
+"""]]
diff --git a/doc/design/assistant/blog/day_67__progress_bars.mdwn b/doc/design/assistant/blog/day_67__progress_bars.mdwn
new file mode 100644
index 0000000000..1110a3c3f5
--- /dev/null
+++ b/doc/design/assistant/blog/day_67__progress_bars.mdwn
@@ -0,0 +1,10 @@
+Got the webapp's progress bars updating for downloads. Updated
+[[progressbars]] with all the options for ways to get progress info. For
+downloads, it currently uses the easy, and not very expensive, approach of
+periodically polling the sizes of files that are being downloaded.
+
+For uploads, something more sophisticated will be called for..
+
+---
+
+The webapp really feels alive now that it has progress bars!
diff --git a/doc/design/assistant/blog/day_68__transfers.mdwn b/doc/design/assistant/blog/day_68__transfers.mdwn
new file mode 100644
index 0000000000..498f0eae4c
--- /dev/null
+++ b/doc/design/assistant/blog/day_68__transfers.mdwn
@@ -0,0 +1,15 @@
+More work on the display and control of transfers.
+
+* Hide redundant downloads from the transfer display. It seemed simplest
+  to keep the behavior of queuing downloads from every remote that has a
+  file, rather than going to some other data structure, but it's clutter
+  to display those to the user, especially when you often have 7 copies
+  of each file, like I do.
+* When canceling a download, cancel all other queued downloads of that
+  key too.
+* Fixed unsettting of the paused flag when resuming a paused transfer.
+* Implemented starting queued transfers by clicking on the start button.
+* Spent a long time debugging why pausing, then resuming, and then pausing
+  a transfer doesn't successfully pause it the second time. I see where
+  the code is seemingly locking up in a `throwTo`, but I don't understand
+  why that blocks forever. Urgh..
diff --git a/doc/design/assistant/blog/day_68__transfers/comment_1_5282960c0b553fbc0f411345b9745324._comment b/doc/design/assistant/blog/day_68__transfers/comment_1_5282960c0b553fbc0f411345b9745324._comment
new file mode 100644
index 0000000000..0921ae8559
--- /dev/null
+++ b/doc/design/assistant/blog/day_68__transfers/comment_1_5282960c0b553fbc0f411345b9745324._comment
@@ -0,0 +1,14 @@
+[[!comment format=mdwn
+ username="https://www.google.com/accounts/o8/id?id=AItOawl9sYlePmv1xK-VvjBdN-5doOa_Xw-jH4U"
+ nickname="Richard"
+ subject="comment 1"
+ date="2012-08-29T21:24:37Z"
+ content="""
+It seems to be a bad idea to cancel everything when I cancel a single transfer.
+
+I may not want to transmit videos and RAW files at the moment, but would still like to know my JPGs are stored in a second location.
+
+Two obvious approaches, ideally both offered at the same time would be to allow the user to cancel, cancel all to this remote, and cancel all plus allowing them to send transfers to the end of the queue.
+
+If this is implemented already then sorry; I did _not_ test if this is available before writing this comment.
+"""]]
diff --git a/doc/design/assistant/blog/day_69__build_fixes.mdwn b/doc/design/assistant/blog/day_69__build_fixes.mdwn
new file mode 100644
index 0000000000..c3ee34e27d
--- /dev/null
+++ b/doc/design/assistant/blog/day_69__build_fixes.mdwn
@@ -0,0 +1,7 @@
+Short day today.
+
+* Worked on fixing a number of build failures people reported.
+* Solved the problem that was making transfer pause/resume not always work.
+  Although there is another bug where pausing a transfer sometimes lets
+  another queued transfer start running. 
+* Worked on getting the assistant to start on login on OSX.
diff --git a/doc/design/assistant/blog/day_6__polish.mdwn b/doc/design/assistant/blog/day_6__polish.mdwn
new file mode 100644
index 0000000000..ebe8068c33
--- /dev/null
+++ b/doc/design/assistant/blog/day_6__polish.mdwn
@@ -0,0 +1,50 @@
+Since my last blog, I've been polishing the `git annex watch` command.
+
+First, I fixed the double commits problem. There's still some extra
+committing going on in the `git-annex` branch that I don't understand. It
+seems like a shutdown event is somehow being triggered whenever
+a git command is run by the commit thread.
+
+I also made `git annex watch` run as a proper daemon, with locking to
+prevent multiple copies running, and a pid file, and everything.
+I made `git annex watch --stop` stop it.
+
+---
+
+Then I managed to greatly increase its startup speed. At startup, it
+generates "add" events for every symlink in the tree. This is necessary
+because it doesn't really know if a symlink is already added, or was
+manually added before it starter, or indeed was added while it started up.
+Problem was that these events were causing a lot of work staging the
+symlinks -- most of which were already correctly staged.
+
+You'd think it could just check if the same symlink was in the index.
+But it can't, because the index is in a constant state of flux. The
+symlinks might have just been deleted and re-added, or changed, and
+the index still have the old value.
+
+Instead, I got creative. :) We can't trust what the index says about the
+symlink, but if the index happens to contain a symlink that looks right,
+we can trust that the SHA1 of its blob is the right SHA1, and reuse it
+when re-staging the symlink. Wham! Massive speedup!
+
+---
+
+Then I started running `git annex watch` on my own real git annex repos,
+and noticed some problems.. Like it turns normal files already checked into
+git into symlinks. And it leaks memory scanning a big tree. Oops..
+
+---
+
+I put together a quick screencast demoing `git annex watch`.
+
+
+
+While making the screencast, I noticed that `git-annex watch` was spinning
+in strace, which is bad news for powertop and battery usage. This seems to
+be a [GHC bug](http://bugs.debian.org/677096) also affecting Xmonad. I
+tried switching to GHC's threaded runtime, which solves that problem, but
+causes git-annex to hang under heavy load. Tried to debug that for quite a
+while, but didn't get far. Will need to investigate this further.. 
+Am seeing indications that this problem only affects ghc 7.4.1; in
+particular 7.4.2 does not seem to have the problem.
diff --git a/doc/design/assistant/blog/day_70__adding_ssh_remotes.mdwn b/doc/design/assistant/blog/day_70__adding_ssh_remotes.mdwn
new file mode 100644
index 0000000000..be83daa31a
--- /dev/null
+++ b/doc/design/assistant/blog/day_70__adding_ssh_remotes.mdwn
@@ -0,0 +1,66 @@
+Today I built the UI in the webapp to set up a ssh or rsync remote.
+
+This is the most generic type of remote, and so it's surely got the most
+complex description. I've tried to word it as clearly as I can; suggestions
+most appreciated. Perhaps I should put in a diagram?
+
+[[!img /assistant/addsshserver.png]]
+
+The idea is that this will probe the server, using ssh. If `git-annex-shell`
+is available there, it'll go on to set up a full git remote. If not, it'll
+fall back to setting up a rsync special remote. It'll even fall all the way
+back to using `rsync://` protocol if it can't connect by ssh. So the user
+can just point it at a server and let it take care of the details,
+generally.
+
+The trickiest part of this will be authentication, of course. I'm relying
+on ssh using `ssh-askpass` to prompt for any passwords, etc, when there's
+no controlling terminal. But beyond passwords, this has to deal with ssh
+keys.
+
+I'm planning to make it check if you have a ssh key configured already. If
+you do, it doesn't touch your ssh configuration. I don't want to get in the
+way of people who have a manual configuration or are using MonkeySphere.
+
+But for a user who has never set up a ssh key, it will prompt asking if
+they'd like a key to be set up. If so, it'll generate a key and configure
+ssh to only use it with the server.. and as part of its ssh probe, that key
+will be added to `authorized_keys`.
+
+(Obviously, advanced users can skip this entirely; `git remote add
+ssh://...` still works..)
+
+----
+
+Also today, fixed more UI glitches in the transfer display. I think
+I have them all fixed now, except for the one that needs lots of javascript
+to be written to fix it.
+
+Amusingly, while I was working on UI glitches, it turned out that all the
+fixes involved 100% pure code that has nothing to do with UI. The UI was
+actually just exposing bugs.
+
+For example, closing a running transfer
+had a bug that weirdly reordered the queue. This turned out to be
+due to the transfer queue actually maintaining two versions of the queue,
+one in a TChan and one a list. Some unknown bugs caused these to get out of
+sync. That was fixed very handily by deleting the TChan, so there's only
+one copy of the data.
+
+I had only been using that TChan because I wanted a way to block while the
+queue was empty. But now that I'm more comfortable with STM, I know how
+to do that easily using a list:
+
+[[!format haskell """
+	getQueuedTransfer q = atomically $ do
+		sz <- readTVar (queuesize q)
+		if sz < 1
+			then retry -- blocks until size changes
+			else ...
+"""]]
+
+Ah, the times before [STM](http://en.wikipedia.org/wiki/Software_transactional_memory)
+were dark times indeed. I'm writing more and more STM code lately, building
+up more and more complicated and useful transactions. If you use threads and
+don't know about STM, it's a great thing to learn, to get out of the dark ages
+of dealing with priority inversions, deadlocks, and races.
diff --git a/doc/design/assistant/blog/day_70__adding_ssh_remotes/comment_10_2fac85357ac8feccff82beabd3791439._comment b/doc/design/assistant/blog/day_70__adding_ssh_remotes/comment_10_2fac85357ac8feccff82beabd3791439._comment
new file mode 100644
index 0000000000..1b4833b125
--- /dev/null
+++ b/doc/design/assistant/blog/day_70__adding_ssh_remotes/comment_10_2fac85357ac8feccff82beabd3791439._comment
@@ -0,0 +1,13 @@
+[[!comment format=mdwn
+ username="https://www.google.com/accounts/o8/id?id=AItOawmLB39PC89rfGaA8SwrsnB6tbumezj-aC0"
+ nickname="Tobias"
+ subject="comment 10"
+ date="2012-09-09T16:58:55Z"
+ content="""
+That's great news.
+
+I suggest we disregard this comment thread, i'll just follow the proper bug report(which i guess i should have made to begin with).
+
+Thanks for your good work.
+
+"""]]
diff --git a/doc/design/assistant/blog/day_70__adding_ssh_remotes/comment_11_e9e496005fd1bf5a10c9e286b83e51fa._comment b/doc/design/assistant/blog/day_70__adding_ssh_remotes/comment_11_e9e496005fd1bf5a10c9e286b83e51fa._comment
new file mode 100644
index 0000000000..1477a71db7
--- /dev/null
+++ b/doc/design/assistant/blog/day_70__adding_ssh_remotes/comment_11_e9e496005fd1bf5a10c9e286b83e51fa._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="http://joeyh.name/"
+ ip="4.153.8.30"
+ subject="comment 11"
+ date="2012-09-09T17:12:55Z"
+ content="""
+That bug should be fixed now.
+"""]]
diff --git a/doc/design/assistant/blog/day_70__adding_ssh_remotes/comment_1_913e6ae7c8f7db90b9767ec35fc84205._comment b/doc/design/assistant/blog/day_70__adding_ssh_remotes/comment_1_913e6ae7c8f7db90b9767ec35fc84205._comment
new file mode 100644
index 0000000000..f797a0249b
--- /dev/null
+++ b/doc/design/assistant/blog/day_70__adding_ssh_remotes/comment_1_913e6ae7c8f7db90b9767ec35fc84205._comment
@@ -0,0 +1,23 @@
+[[!comment format=mdwn
+ username="https://www.google.com/accounts/o8/id?id=AItOawmLB39PC89rfGaA8SwrsnB6tbumezj-aC0"
+ nickname="Tobias"
+ subject="gpg encryption on ssh remotes"
+ date="2012-09-01T08:25:17Z"
+ content="""
+I've been playing around with the Assistant in the last few days.
+
+And besides for gpg encryption it works great.
+
+But whenever i set up a special remote(done it manually since you hadn't made this yet) it will break, bad.
+
+Some files will transfer fine, but at some point it will still(the stdout from git-annex webapp shows gpg being stupid).
+
+I get a huge amount of [git]  in my ps auxw. And two git-annex transferkey  --to host --file 
+
+It seems to consistently fall on a 9mb file i have. I have found no way to recover from this(killing and restarting git-annex will say 'transfer already in progress'. Deleting the file was to no avail, it still tries to transfer).
+
+I have done a lot of testing and i've only seen this on special remotes when gpg encryption is enabled(i haven't tried encryption=shared)
+
+Besides for this issue, the Assistant is looking great.
+
+"""]]
diff --git a/doc/design/assistant/blog/day_70__adding_ssh_remotes/comment_2_634ca3c236e2062289e7df5f0d77a3c5._comment b/doc/design/assistant/blog/day_70__adding_ssh_remotes/comment_2_634ca3c236e2062289e7df5f0d77a3c5._comment
new file mode 100644
index 0000000000..84e780b4bf
--- /dev/null
+++ b/doc/design/assistant/blog/day_70__adding_ssh_remotes/comment_2_634ca3c236e2062289e7df5f0d77a3c5._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="http://joeyh.name/"
+ ip="4.152.108.236"
+ subject="comment 2"
+ date="2012-09-02T21:40:17Z"
+ content="""
+@tobias, when you say \"gpg being stupid\", what exactly do you see at that point? Please paste it.
+"""]]
diff --git a/doc/design/assistant/blog/day_70__adding_ssh_remotes/comment_3_e365bbcbb7f66ce2b35fcd5b969ab315._comment b/doc/design/assistant/blog/day_70__adding_ssh_remotes/comment_3_e365bbcbb7f66ce2b35fcd5b969ab315._comment
new file mode 100644
index 0000000000..741cdfc63e
--- /dev/null
+++ b/doc/design/assistant/blog/day_70__adding_ssh_remotes/comment_3_e365bbcbb7f66ce2b35fcd5b969ab315._comment
@@ -0,0 +1,16 @@
+[[!comment format=mdwn
+ username="https://www.google.com/accounts/o8/id?id=AItOawmLB39PC89rfGaA8SwrsnB6tbumezj-aC0"
+ nickname="Tobias"
+ subject="comment 3"
+ date="2012-09-02T23:49:17Z"
+ content="""
+Updated to the latest trunk, and did some more testing.
+
+I'll retract the \"gpg being stupid\" comment. it just gives the normal \"(gpg)\".
+
+There really isn't anything interesting in the stdout(but a lot of defunct git's in ps auxw), but i still can't get any files larger than ~710kbyte to transfer. They just stall.
+
+http://pastebin.com/ZA8dzxZD <- stdout/stderr
+http://pastebin.com/f1J1T79E <- ps auxw | grep git
+
+"""]]
diff --git a/doc/design/assistant/blog/day_70__adding_ssh_remotes/comment_4_b15499722a655489f9ea60ff9d4c47c6._comment b/doc/design/assistant/blog/day_70__adding_ssh_remotes/comment_4_b15499722a655489f9ea60ff9d4c47c6._comment
new file mode 100644
index 0000000000..d78f3d268d
--- /dev/null
+++ b/doc/design/assistant/blog/day_70__adding_ssh_remotes/comment_4_b15499722a655489f9ea60ff9d4c47c6._comment
@@ -0,0 +1,12 @@
+[[!comment format=mdwn
+ username="https://www.google.com/accounts/o8/id?id=AItOawmLB39PC89rfGaA8SwrsnB6tbumezj-aC0"
+ nickname="Tobias"
+ subject="comment 4"
+ date="2012-09-03T08:12:39Z"
+ content="""
+So, i've let the assistant run since i posted my last message, while i slept.
+
+But 8 hours later, it still hasn't made any progress. :(
+
+Again, only see this with gpg encryption enabled, and on files >700kbyte.
+"""]]
diff --git a/doc/design/assistant/blog/day_70__adding_ssh_remotes/comment_5_8ea48276f060e75d9f40617d2a1ccd08._comment b/doc/design/assistant/blog/day_70__adding_ssh_remotes/comment_5_8ea48276f060e75d9f40617d2a1ccd08._comment
new file mode 100644
index 0000000000..0cbf81e413
--- /dev/null
+++ b/doc/design/assistant/blog/day_70__adding_ssh_remotes/comment_5_8ea48276f060e75d9f40617d2a1ccd08._comment
@@ -0,0 +1,12 @@
+[[!comment format=mdwn
+ username="http://joeyh.name/"
+ ip="4.152.108.236"
+ subject="comment 5"
+ date="2012-09-04T20:06:17Z"
+ content="""
+Can you try running the transfer command by hand? The command line is:
+
+`/home/alansmithee/bin/git-annex transferkey SHA256-s806100--ad2c33a88a665c56d97393fad99b37529afc5947694eab7d9c27f4d178d182f4 --to host1free --file 787kbyte.txt`
+
+Paste me the full output until it seems to hang (if it hangs this way), and if it does hang you may also need to strace it and send me that..
+"""]]
diff --git a/doc/design/assistant/blog/day_70__adding_ssh_remotes/comment_6_9b8bf7e9fa715977fbeb98087deefd1a._comment b/doc/design/assistant/blog/day_70__adding_ssh_remotes/comment_6_9b8bf7e9fa715977fbeb98087deefd1a._comment
new file mode 100644
index 0000000000..c5c2aaa8a4
--- /dev/null
+++ b/doc/design/assistant/blog/day_70__adding_ssh_remotes/comment_6_9b8bf7e9fa715977fbeb98087deefd1a._comment
@@ -0,0 +1,10 @@
+[[!comment format=mdwn
+ username="https://www.google.com/accounts/o8/id?id=AItOawmLB39PC89rfGaA8SwrsnB6tbumezj-aC0"
+ nickname="Tobias"
+ subject="comment 6"
+ date="2012-09-04T21:57:05Z"
+ content="""
+No joy.
+
+strace: http://pastebin.com/UYSrZZAg
+"""]]
diff --git a/doc/design/assistant/blog/day_70__adding_ssh_remotes/comment_7_42e09eacdc10c7cf579bfc6470b5117c._comment b/doc/design/assistant/blog/day_70__adding_ssh_remotes/comment_7_42e09eacdc10c7cf579bfc6470b5117c._comment
new file mode 100644
index 0000000000..5a8e2f9f8d
--- /dev/null
+++ b/doc/design/assistant/blog/day_70__adding_ssh_remotes/comment_7_42e09eacdc10c7cf579bfc6470b5117c._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="http://joeyh.name/"
+ ip="4.152.108.236"
+ subject="comment 7"
+ date="2012-09-06T16:20:49Z"
+ content="""
+From the strace, I can see it reading the file and feeding it to gpg. Either gpg is stalling after a while, or perhaps it is not reading the output back from gpg. What type of special remote is this (rsync/directory/S3)? Can you provide `strace -f` output for the same command you straced before?
+"""]]
diff --git a/doc/design/assistant/blog/day_70__adding_ssh_remotes/comment_8_6c02f31063b3d399d1b4f823bd6543ce._comment b/doc/design/assistant/blog/day_70__adding_ssh_remotes/comment_8_6c02f31063b3d399d1b4f823bd6543ce._comment
new file mode 100644
index 0000000000..391ec44379
--- /dev/null
+++ b/doc/design/assistant/blog/day_70__adding_ssh_remotes/comment_8_6c02f31063b3d399d1b4f823bd6543ce._comment
@@ -0,0 +1,16 @@
+[[!comment format=mdwn
+ username="https://www.google.com/accounts/o8/id?id=AItOawmLB39PC89rfGaA8SwrsnB6tbumezj-aC0"
+ nickname="Tobias"
+ subject="comment 8"
+ date="2012-09-06T18:41:47Z"
+ content="""
+I've tried with rsync and directory(real, mountpoint, and fuse mount), no S3. Same result on all of them.
+
+Here is an strace -f http://pastebin.com/YqCbi7Ue
+
+Funny thing is that now i hitting the failure on smaller files(<500kbyte).
+
+I even did a restart because of that. To see if a clean boot would increate it back to the ~700kbyte, but it made no difference.
+
+The annex hasn't been idle since last time, but i haven't done anything major in it. In total the annex is 204mb(up from 185mb at the time of my first post about this)
+"""]]
diff --git a/doc/design/assistant/blog/day_70__adding_ssh_remotes/comment_9_dd0447cb3b39d3a8c1a7cc00f17d8bc2._comment b/doc/design/assistant/blog/day_70__adding_ssh_remotes/comment_9_dd0447cb3b39d3a8c1a7cc00f17d8bc2._comment
new file mode 100644
index 0000000000..cd879bcfd4
--- /dev/null
+++ b/doc/design/assistant/blog/day_70__adding_ssh_remotes/comment_9_dd0447cb3b39d3a8c1a7cc00f17d8bc2._comment
@@ -0,0 +1,10 @@
+[[!comment format=mdwn
+ username="http://joeyh.name/"
+ ip="4.153.8.30"
+ subject="comment 9"
+ date="2012-09-09T16:52:33Z"
+ content="""
+Someone else reported what looks like the same problem, here: [[bugs/transferkey_fails_due_to_gpg]]
+
+I'mm be following up to that bug report. In fact, I think I just managed to replicate the bug!
+"""]]
diff --git a/doc/design/assistant/blog/day_71__ssh_probing.mdwn b/doc/design/assistant/blog/day_71__ssh_probing.mdwn
new file mode 100644
index 0000000000..ac5a473800
--- /dev/null
+++ b/doc/design/assistant/blog/day_71__ssh_probing.mdwn
@@ -0,0 +1,26 @@
+Got ssh probing implemented. It checks if it can connect to the server, and
+probes the server to see how it should be used. 
+
+Turned out to need two ssh probes. The first uses the system's existing ssh
+configuration, but disables password prompts. If that's able to get in
+without prompting for a password, then the user must have set that up,
+and doesn't want to be bothered with password prompts, and it'll respect
+that configuration.
+
+Otherwise, it sets up a per-host ssh key, and configures a hostname alias
+in `~/.ssh/config` to use that key, and probes using that. 
+Configuring ssh this way is nice because it avoids changing ssh's
+behavior except when git-annex uses it, and it does not open up the server
+to arbitrary commands being run without password.
+
+--
+
+Next up will be creating the repositories. When there's a per-host key,
+this will also involve setting up `authorized_keys`, locking down the ssh
+key to only allow running git-annex-shell or rsync.
+
+I decided to keep that separate from the ssh probing, even though it means
+the user will be prompted twice for their ssh password. It's cleaner and
+allows the probing to do other checks -- maybe it'll later check the amount
+of free disk space -- and the user should be able to decide after the probe
+whether or not to proceed with making the repository.
diff --git a/doc/design/assistant/blog/day_71__ssh_probing/comment_1_56a0c29f7454cfca5cc30b2849e6e942._comment b/doc/design/assistant/blog/day_71__ssh_probing/comment_1_56a0c29f7454cfca5cc30b2849e6e942._comment
new file mode 100644
index 0000000000..606037372d
--- /dev/null
+++ b/doc/design/assistant/blog/day_71__ssh_probing/comment_1_56a0c29f7454cfca5cc30b2849e6e942._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="https://www.google.com/accounts/o8/id?id=AItOawmRFKwny4rArBaz-36xTcsJYqKIgdDaw5Q"
+ nickname="Andrew"
+ subject="comment 1"
+ date="2013-01-19T07:23:38Z"
+ content="""
+I'm finding a few circumstances where the per-host key never gets setup. Is there a way to force this probing or key setup to happen?
+"""]]
diff --git a/doc/design/assistant/blog/day_71__ssh_probing/comment_2_f3bd3e366c92c833c7e217da125481b8._comment b/doc/design/assistant/blog/day_71__ssh_probing/comment_2_f3bd3e366c92c833c7e217da125481b8._comment
new file mode 100644
index 0000000000..3ce31969db
--- /dev/null
+++ b/doc/design/assistant/blog/day_71__ssh_probing/comment_2_f3bd3e366c92c833c7e217da125481b8._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="http://joeyh.name/"
+ ip="4.154.3.194"
+ subject="comment 2"
+ date="2013-01-19T16:09:15Z"
+ content="""
+It'd be best to file a bug report with details.
+"""]]
diff --git a/doc/design/assistant/blog/day_72__remote_ssh_server_configurator_finished.mdwn b/doc/design/assistant/blog/day_72__remote_ssh_server_configurator_finished.mdwn
new file mode 100644
index 0000000000..4dfc297b2c
--- /dev/null
+++ b/doc/design/assistant/blog/day_72__remote_ssh_server_configurator_finished.mdwn
@@ -0,0 +1,34 @@
+Decided to only make bare git repos on remote ssh servers. This
+configurator is aimed at using a server somewhere, which is probably not
+going to be running the assistant. So it doesn't need a non-bare repo, and
+there's nothing to keep the checked out branch in a non-bare repo
+up-to-date on such a server, anyway. For non-bare repos on locally
+accessible boxes, the [[pairing]] configurator will be the thing
+to use, instead of this one.
+
+Note: While the remote ssh configurator works great, and you could even have the
+assistant running on multiple computers and use it to point them all at the
+same repo on a server, the assistant does not yet support keeping such a
+network topology in sync. That needs some of the ideas in [[cloud]] to
+happen, so clients can somehow inform each other when there are changes.
+Until that happens, the assistant polls only every 30 minutes, so it'll
+keep in sync with a 30 minute delay.
+
+---
+
+This configurator can also set up encryped rsync special remotes. Currently
+it always encrypts them, using the shared cipher mode of git-annex's
+encryption. That avoids issues with gpg key generation and distribution,
+and was easy to get working.
+
+---
+
+I feel I'm in a good place now WRT adding repository configurator wizards
+to the webapp. This one took about 2.5 days, and involved laying some
+groundwork that will be useful for other repository configurators. And it
+was probably one of the more complex ones. 
+
+Now I should be able to crank out configurators for things like Amazon S3,
+Bup, Rsync.net, etc fairly quickly. First, I need to do a beta release of
+the assistant, and start getting feedback from my backers to prioritize
+what to work on.
diff --git a/doc/design/assistant/blog/day_73__rsync.net_configurator.mdwn b/doc/design/assistant/blog/day_73__rsync.net_configurator.mdwn
new file mode 100644
index 0000000000..9000636a73
--- /dev/null
+++ b/doc/design/assistant/blog/day_73__rsync.net_configurator.mdwn
@@ -0,0 +1,17 @@
+Now finished building a special configurator for rsync.net. While
+this is just a rsync remote to git-annex, there are some tricky bits to
+setting up the ssh key using rsync.net's restricted shell. The configurator
+automates that nicely. It took about 3 hours of work, and 49 lines of
+rsync.net specific code to build this.
+
+[[!img /assistant/rsync.net.png]]
+
+Thanks to rsync.net who heard of my Kickstarter and gave me a really
+nice free lifetime account. BTW guys, I wish your restricted shell
+supported '&&' in between commands, and returned a nonzero exit status when
+the command fails. This would make my error handling work better.
+
+I've also reworked the repository management page. Nice to see those
+configurators start to fill in!
+
+[[!img /assistant/repositories.png]]
diff --git a/doc/design/assistant/blog/day_74__bits_and_peices.mdwn b/doc/design/assistant/blog/day_74__bits_and_peices.mdwn
new file mode 100644
index 0000000000..c3a71d7964
--- /dev/null
+++ b/doc/design/assistant/blog/day_74__bits_and_peices.mdwn
@@ -0,0 +1,7 @@
+* On OSX, install a launcher plist file, to run the assistant on login,
+  and a `git-annex-webapp.command` file in the desktop. This is not tested
+  yet.
+* Made the webapp display alerts when the inotify/kqueue layer has a
+  warning message.
+* Handle any crashes of each of the 15 or so named threads by displaying
+  an alert. (Of course, this should never happen.)
diff --git a/doc/design/assistant/blog/day_75__zeromq_and_pairing.mdwn b/doc/design/assistant/blog/day_75__zeromq_and_pairing.mdwn
new file mode 100644
index 0000000000..9f2ba48169
--- /dev/null
+++ b/doc/design/assistant/blog/day_75__zeromq_and_pairing.mdwn
@@ -0,0 +1,50 @@
+Started reading about ZeroMQ with the hope that it could do some firewall
+traversal thing, to connect mutually-unroutable nodes. Well, it could, but
+it'd need a proxy to run on a server both can contact, and lots of
+users won't have a server to run that on. The XMPP approach used by
+dvcs-autosync is looking like the likeliest way for git-annex to handle
+that use case.
+
+However, ZeroMQ did point in promising directions to handle another use
+case I need to support: Local [[pairing]]. In fairly short order, I got
+ZeroMQ working over IP Multicast (PGM), with multiple publishers sending
+messages that were all seen by multiple clients on the LAN (actually the
+WAN; works over OpenVPN too). I had been thinking about using
+Avahi/ZeroConf for discovery of systems to pair with, but ZeroMQ is rather
+more portable and easy to work with.
+
+Unfortunatly, I wasn't able to get ZeroMQ to behave reliably enough.
+It seems to have some timeout issues the way I'm trying to use it,
+or perhaps its haskell bindings are buggy? Anyway, it's really overkill
+to use PGM when all I need for git-annex pairing discovery is lossy
+UDP Multicast. Haskell has a simple `network-multicast` library for that,
+and it works great.
+
+With discovery out of the way (theoretically), the hard part about
+[[pairing]] is going to be verifying that the desired repository is being
+paired with, and not some imposter. My plan to deal with this involves a
+shared secret, that can be communicated out of band, and HMAC. The webapp
+will prompt both parties to enter the same agreed upon secret (which could
+be any phrase, ideally with 64 bytes of entropy), and will then use it as
+the key for HMAC on the ssh public key. The digest will be sent over the
+wire, along with the ssh public key, and the other side can use the shared
+secret to verifiy the key is correct.
+
+The other hard part about [[pairing]] will be finding the best address to
+use for git, etc to connect to the other host. If MDNS is available, it's
+ideal, but if not the pair may have to rely on local DNS, or even
+hard-coded IPs, which will be much less robust. Or, the assistant could
+broadcast queries for a peer's current IP address itself, as a poor man's
+MDNS.
+
+All right then! That looks like a good week's worth of work to embark on.
+
+---
+
+Slight detour to package the haskell network-multicast library and upload
+to Debian unstable.
+
+Roughed out a data type that models the whole pairing conversation,
+and can be serialized to implement it. And a state machine to run
+that conversation. Not yet hooked up to any transport such as multicast
+UDP.
diff --git a/doc/design/assistant/blog/day_76__pairing.mdwn b/doc/design/assistant/blog/day_76__pairing.mdwn
new file mode 100644
index 0000000000..0ebf55090f
--- /dev/null
+++ b/doc/design/assistant/blog/day_76__pairing.mdwn
@@ -0,0 +1,16 @@
+About half way done with implementing [[pairing]]. The webapp's interface
+to prompt for a secret and start pairing is done; the protocol is
+implemented; broadcasting of pairing requests is working; added Yet Another
+Thread to listen for incoming pairing traffic.
+
+Very happy with how this came together; starting with defining the protocol
+with data types let me rapidly iterate until I had designed a simple, clean,
+robust protocol. The implementation works well too; it's even possible to
+start pairing, and only then bring up the network interface to the machine
+you intended to pair with, and it'll detect the new interface and start
+sending requests to it.
+
+Next, I need to make alerts have a button that performs a stored
+IO action. So that the incoming pair request alert can have a button to
+respond to the pair request. And then I need to write the code to actually
+perform the pairing, including ssh key setup.
diff --git a/doc/design/assistant/blog/day_76__pairing/comment_1_09665f269343422cd18051fad1a8c19e._comment b/doc/design/assistant/blog/day_76__pairing/comment_1_09665f269343422cd18051fad1a8c19e._comment
new file mode 100644
index 0000000000..16956ead21
--- /dev/null
+++ b/doc/design/assistant/blog/day_76__pairing/comment_1_09665f269343422cd18051fad1a8c19e._comment
@@ -0,0 +1,24 @@
+[[!comment format=mdwn
+ username="https://www.google.com/accounts/o8/id?id=AItOawmubB1Sj2rwFoVdZYvGV0ACaQUJQyiJXJI"
+ nickname="Paul"
+ subject="Hogging memory"
+ date="2012-09-09T15:51:40Z"
+ content="""
+How do you prefer bugs to be reported?
+
+I'm having a problem where git-annex (from git, 1e41c0d85ecc24e8656bff79b2fba46c3663a054) is taking over 20GB of RAM after adding a single file.
+
+To reproduce:
+
+1. Create a new annex (I used the web app, created an annex at ~/annex)
+2. Add box.com as a remote, using encryption (followed , basically `cd ~/annex; git annex initremote box.com type=directory directory=/media/box.com encryption=$GPGID`)
+3. copied a file into the annex (ok, I admit, the file was a bit large, 350MB, but still)
+4. refreshed the web app, nothing happened
+5. closed the web app
+6. started git annex assistant inside ~/annex
+7. git annex webapp
+8. noted that git annex noticed the file, and started transferring
+9. wait
+10. memory usage for the git-annex process goes beyond 21GB RAM
+11. oomkiller kills git-annex
+"""]]
diff --git a/doc/design/assistant/blog/day_76__pairing/comment_2_8e1b2233579bc26bfd758bbf6b3bdc07._comment b/doc/design/assistant/blog/day_76__pairing/comment_2_8e1b2233579bc26bfd758bbf6b3bdc07._comment
new file mode 100644
index 0000000000..a2e64671fe
--- /dev/null
+++ b/doc/design/assistant/blog/day_76__pairing/comment_2_8e1b2233579bc26bfd758bbf6b3bdc07._comment
@@ -0,0 +1,10 @@
+[[!comment format=mdwn
+ username="http://joeyh.name/"
+ ip="4.153.8.30"
+ subject="comment 2"
+ date="2012-09-09T16:25:52Z"
+ content="""
+The [[bugs]] page is the place to put bug reports like this so I won't forget them.
+
+This should certainly not be happening. There are actually two git-annex processes running in the situation you describe; I'd be most curious to know whether the `git annex transfer` process was the one that blew up, or if the `git annex assistant` blew up. Also, it's not clear to me if you enabled the chunksize parameter when setting up the special remote, which could well be a significant detail.
+"""]]
diff --git a/doc/design/assistant/blog/day_76__pairing/comment_3_a8b6a8432da20c468c633da8e7cbc2f3._comment b/doc/design/assistant/blog/day_76__pairing/comment_3_a8b6a8432da20c468c633da8e7cbc2f3._comment
new file mode 100644
index 0000000000..28439b34fc
--- /dev/null
+++ b/doc/design/assistant/blog/day_76__pairing/comment_3_a8b6a8432da20c468c633da8e7cbc2f3._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="https://www.google.com/accounts/o8/id?id=AItOawmubB1Sj2rwFoVdZYvGV0ACaQUJQyiJXJI"
+ nickname="Paul"
+ subject="comment 3"
+ date="2012-09-09T17:34:07Z"
+ content="""
+I'm almost certain that I specified chunksize like in the tip. How do I check?
+"""]]
diff --git a/doc/design/assistant/blog/day_76__pairing/comment_4_36a428a2e1803f4391b821d1892f0cd7._comment b/doc/design/assistant/blog/day_76__pairing/comment_4_36a428a2e1803f4391b821d1892f0cd7._comment
new file mode 100644
index 0000000000..e462402652
--- /dev/null
+++ b/doc/design/assistant/blog/day_76__pairing/comment_4_36a428a2e1803f4391b821d1892f0cd7._comment
@@ -0,0 +1,10 @@
+[[!comment format=mdwn
+ username="http://joeyh.name/"
+ ip="4.153.8.30"
+ subject="comment 4"
+ date="2012-09-09T18:00:48Z"
+ content="""
+You can see the full configuration with: git show git-annex:remote.log
+
+(You probably do not want to paste the cipher= part of that here, although it *is* encrypted with your gpg key, unless you are using the shared encryption mode.)
+"""]]
diff --git a/doc/design/assistant/blog/day_76__pairing/comment_5_11f332fe2050d8c1416e71f9e85ba280._comment b/doc/design/assistant/blog/day_76__pairing/comment_5_11f332fe2050d8c1416e71f9e85ba280._comment
new file mode 100644
index 0000000000..5c93d3d3e8
--- /dev/null
+++ b/doc/design/assistant/blog/day_76__pairing/comment_5_11f332fe2050d8c1416e71f9e85ba280._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="https://www.google.com/accounts/o8/id?id=AItOawmubB1Sj2rwFoVdZYvGV0ACaQUJQyiJXJI"
+ nickname="Paul"
+ subject="comment 5"
+ date="2012-09-10T14:54:17Z"
+ content="""
+chunksize=2m was defined. I tried again, and the transfers again didn't seem to start. Then I did `killall git-annex`, cleaned the files from the box.com account, and tried again. This time it seems to be working, but the webapp isn't showing progress. Memory usage also stays reasonable, and doesn't seem to grow. Maybe it was just bad handling of unexpected repository state?
+"""]]
diff --git a/doc/design/assistant/blog/day_76__pairing/comment_6_973aeb656b78eca97474ea1a3f5b57b7._comment b/doc/design/assistant/blog/day_76__pairing/comment_6_973aeb656b78eca97474ea1a3f5b57b7._comment
new file mode 100644
index 0000000000..120a87845e
--- /dev/null
+++ b/doc/design/assistant/blog/day_76__pairing/comment_6_973aeb656b78eca97474ea1a3f5b57b7._comment
@@ -0,0 +1,12 @@
+[[!comment format=mdwn
+ username="http://joeyh.name/"
+ ip="4.153.8.30"
+ subject="comment 6"
+ date="2012-09-10T17:31:20Z"
+ content="""
+Re transfer progress not being shown, that's expected as upload progress displays are not yet implemented in the webapp.
+
+I can't think of a way git-annex would care what was in your box.com repo. It just makes directories as needed, and will overwrite existing files in the rare case they already exist. And ignore any non-git-annex files.
+
+Are you sure it was git-annex's memory use blowing up, and not the memory use of the davfs2 daemon?
+"""]]
diff --git a/doc/design/assistant/blog/day_76__pairing/comment_7_03d2b3343f34377a4d6171e06b7609f6._comment b/doc/design/assistant/blog/day_76__pairing/comment_7_03d2b3343f34377a4d6171e06b7609f6._comment
new file mode 100644
index 0000000000..5aea18576c
--- /dev/null
+++ b/doc/design/assistant/blog/day_76__pairing/comment_7_03d2b3343f34377a4d6171e06b7609f6._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="https://www.google.com/accounts/o8/id?id=AItOawmubB1Sj2rwFoVdZYvGV0ACaQUJQyiJXJI"
+ nickname="Paul"
+ subject="comment 7"
+ date="2012-09-10T17:35:48Z"
+ content="""
+Positive. I was grepping for git-annex from ps for the mem usage.
+"""]]
diff --git a/doc/design/assistant/blog/day_77_alert_buttons.mdwn b/doc/design/assistant/blog/day_77_alert_buttons.mdwn
new file mode 100644
index 0000000000..2c05b495b5
--- /dev/null
+++ b/doc/design/assistant/blog/day_77_alert_buttons.mdwn
@@ -0,0 +1,21 @@
+Alerts can now have buttons, that go to some url when clicked. Yay.
+
+Implementing that was a PITA, because Yesod really only wants its type-safe
+urls to be rendered from within its Handler monad. Which most things that
+create alerts are not. I managed to work around Yesod's insistence on this
+only by using a MVar to store the pure function that Yesod uses internally.
+That function can only be obtained once the webapp is running.
+
+----
+
+Fixed a nasty bug where using gpg would cause hangs. I introduced this back
+when I was reworking all the code in git-annex that runs processes, so it
+would work with threading. In the process a place that had forked a process
+to feed input to gpg was lost. Fixed it by spawning a thread to feed gpg.
+Luckily I have never released a version of git-annex with that bug, but
+the many users who are building from the master branch should update.
+
+----
+
+Made alerts be displayed while pairing is going on, with buttons to cancel
+pairing or respond to a pairing request.
diff --git a/doc/design/assistant/blog/day_78__pairing_continued.mdwn b/doc/design/assistant/blog/day_78__pairing_continued.mdwn
new file mode 100644
index 0000000000..74f5107150
--- /dev/null
+++ b/doc/design/assistant/blog/day_78__pairing_continued.mdwn
@@ -0,0 +1,8 @@
+Worked on [[pairing]] all day. It's complicated and I was close to being in
+the weeds at times. I think it probably works now, but I have not tested it
+at all. Tomorrow, testing, and cleaning up known problems.
+
+----
+
+Also ordered 1.5 terabytes of USB keys and a thousand git-annex stickers
+today.
diff --git a/doc/design/assistant/blog/day_79__pairing_finished.mdwn b/doc/design/assistant/blog/day_79__pairing_finished.mdwn
new file mode 100644
index 0000000000..ca4bbc1257
--- /dev/null
+++ b/doc/design/assistant/blog/day_79__pairing_finished.mdwn
@@ -0,0 +1,33 @@
+Tons of pairing work, which culminated today in pairing fully working for
+the very first time. And it works great! Type something like "my
+hovercraft is full of eels" into two git annex webapps on the same LAN
+and the two will find each other, automatically set up ssh keys, and sync
+up, like magic. Magic based on math.
+
+* Revert changes made to `authorized_keys` when the user cancels
+  a pairing response. Which could happen if the machine that sent the
+  pairing request originally is no longer on the network.
+* Some fixes to handle lossy UDP better. Particularly tricky at the end
+  of the conversation -- how do both sides reliably know when a
+  conversation is over when it's over a lossy wire? My solution is just
+  to remember some conversatons we think are over, and keep saying
+  "this conversation is over" if we see messages in that conversation.
+  Works.
+* Added a UUID that must be the same in related pairing messages.
+  This has a nice security feature: It allows detection of brute-force
+  attacks to guess the shared secret, after the first wrong guess! 
+  In which case the pairing is canceled and a warning printed.
+* That led to a thorough security overview, which I've added to
+  the [[pairing]] page. Added some guards against unusual attacks,
+  like console poisioning attacks. I feel happy with the security
+  of pairing now, with the caveats that only I have reviewed it (and
+  reviewing your own security designs is never ideal), and that the
+  out-of-band shared secret communication between users is only as good
+  as they make it.
+* Found [a bug](https://github.com/yesodweb/yesod/issues/421) 
+  in Yesod's type safe urls. At least, I think it's a bug. Worked around it.
+* Got very stuck trying to close the sockets that are opened to send
+  multicast pairing messages. Nothing works, down to and including calling
+  C `close()`. At the moment I have a socket leak. :(
+  I need to understand the details of multicast sockets better to fix this.
+  Emailed the author of the library I'm using for help.
diff --git a/doc/design/assistant/blog/day_7__bugfixes.mdwn b/doc/design/assistant/blog/day_7__bugfixes.mdwn
new file mode 100644
index 0000000000..79f36fe98d
--- /dev/null
+++ b/doc/design/assistant/blog/day_7__bugfixes.mdwn
@@ -0,0 +1,45 @@
+Kickstarter is over. Yay!
+
+Today I worked on the bug where `git annex watch` turned regular files
+that were already checked into git into symlinks. So I made it check
+if a file is already in git before trying to add it to the annex.
+
+The tricky part was doing this check quickly. Unless I want to write my
+own git index parser (or use one from Hackage), this check requires running
+`git ls-files`, once per file to be added. That won't fly if a huge
+tree of files is being moved or unpacked into the watched directory.
+
+Instead, I made it only do the check during `git annex watch`'s initial
+scan of the tree. This should be OK, because once it's running, you
+won't be adding new  files to git anyway, since it'll automatically annex
+new files. This is good enough for now, but there are at least two problems
+with it:
+
+* Someone might `git merge` in a branch that has some regular files,
+  and it would add the merged in files to the annex.
+* Once `git annex watch` is running, if you modify a file that was
+  checked into git as a regular file, the new version will be added
+  to the annex.
+
+I'll probably come back to this issue, and may well find myself directly
+querying git's index.
+
+---
+
+I've started work to fix the memory leak I see when running `git annex
+watch` in a large repository (40 thousand files). As always with a Haskell
+memory leak, I crack open [Real World Haskell's chapter on profiling](http://book.realworldhaskell.org/read/profiling-and-optimization.html).
+
+Eventually this yields a nice graph of the problem:
+
+[[!img profile.png alt="memory profile"]]
+
+So, looks like a few minor memory leaks, and one huge leak. Stared
+at this for a while and trying a few things, and got a much better result:
+
+[[!img profile2.png alt="memory profile"]]
+
+I may come back later and try to improve this further, but it's not bad memory
+usage. But, it's still rather slow to start up in such a large repository,
+and its initial scan is still doing too much work. I need to optimize
+more..
diff --git a/doc/design/assistant/blog/day_7__bugfixes/profile.png b/doc/design/assistant/blog/day_7__bugfixes/profile.png
new file mode 100644
index 0000000000..702af1ca00
Binary files /dev/null and b/doc/design/assistant/blog/day_7__bugfixes/profile.png differ
diff --git a/doc/design/assistant/blog/day_7__bugfixes/profile2.png b/doc/design/assistant/blog/day_7__bugfixes/profile2.png
new file mode 100644
index 0000000000..4d487d02a2
Binary files /dev/null and b/doc/design/assistant/blog/day_7__bugfixes/profile2.png differ
diff --git a/doc/design/assistant/blog/day_80__default_backend.mdwn b/doc/design/assistant/blog/day_80__default_backend.mdwn
new file mode 100644
index 0000000000..5b6aa7f981
--- /dev/null
+++ b/doc/design/assistant/blog/day_80__default_backend.mdwn
@@ -0,0 +1,14 @@
+I've changed the default backend used by git-annex from SHA256 to SHA256E.
+Including the filename extension in the key is known to make repositories
+more usable on things like MP3 players, and I've recently learned it also
+avoids [[forum/Weird_behavior_with_OS_X_Finder_and_Preview.app]]. 
+
+I thought about only changing the default in repositories set up by the
+assistant, but it seemed simpler to change the main default. The old
+backend is really only better if you might have multiple copies of files
+with the same content that have different extensions.
+
+Fixed the socket leak in pairing that eluded me earlier.
+
+I've made a new [[polls]] page, and posted a poll:
+[[polls/prioritizing_special_remotes]]
diff --git a/doc/design/assistant/blog/day_81__enabling_pre-existing_special_remotes.mdwn b/doc/design/assistant/blog/day_81__enabling_pre-existing_special_remotes.mdwn
new file mode 100644
index 0000000000..45a7e9f20b
--- /dev/null
+++ b/doc/design/assistant/blog/day_81__enabling_pre-existing_special_remotes.mdwn
@@ -0,0 +1,34 @@
+It's possible for one git annex repository to configure a special remote
+that it makes sense for other repositories to also be able to use. Today I
+added the UI to support that; in the list of repositories, such
+repositories have a "enable" link.
+
+To enable pre-existing rsync special remotes, the webapp has to do the same
+probing and ssh key setup that it does when initially creating them.
+Rsync.net is also handled as a special case in that code. There was one
+ugly part to this.. When a rsync remote is configured in the webapp,
+it uses a mangled hostname like "git-annex-example.com-user", to
+make ssh use the key it sets up. That gets stored in the `remote.log`, and so
+the enabling code has to unmangle it to get back to the real hostname.
+
+---
+
+Based on the still-running [[prioritizing_special_remotes]] poll, a lot
+of people want special remote support for their phone or mp3 player.
+(As opposed to running git-annex on an Android phone, which comes later..)
+It'd be easy enough to make the webapp set up a directory special remote
+on such a device, but that makes consuming some types of content on the
+phone difficult (mp3 players seem to handle them ok based on what people tell
+me). I need to think more about some of the ideas mentioned in [[android]]
+for more suitable ways of storing files.
+
+One thing's for sure: You won't want the assistant to sync all your files
+to your phone! So I also need to start coming up with partial syncing
+controls. One idea is for each remote to have a configurable matcher for files
+it likes to receive. That could be only mp3 files, or all files inside a
+given subdirectory, or all files *not* in a given subdirectory. That means
+that when the assistant detects a file has been moved, it'll need to add
+(or remove) a queued transfer. Lots of other things could be matched on,
+like file size, number of copies, etc. Oh look, I have a
+[beautiful library I wrote earlier](http://joeyh.name/blog/entry/happy_haskell_hacker)
+that I can reuse!
diff --git a/doc/design/assistant/blog/day_82__git-annex_branch_work.mdwn b/doc/design/assistant/blog/day_82__git-annex_branch_work.mdwn
new file mode 100644
index 0000000000..ed2e4e6029
--- /dev/null
+++ b/doc/design/assistant/blog/day_82__git-annex_branch_work.mdwn
@@ -0,0 +1,26 @@
+Started today doing testing of [[syncing]], and found some bugs and things
+it needs to do better. But was quickly sidetracked when I noticed that
+`transferkey` was making a commit to the git-annex branch for every file it
+transferred, which is too slow and bloats history too much.
+
+To fix that actually involved fixing a long-standing annoyance; that
+read-only git-annex commands like `whereis` sometimes start off with
+"(Recording state in git)", when the journal contains some not yet
+committed changes to the git-annex branch. I had to carefully think
+through the cases to avoid those commits.
+
+As I was working on that, I found a real nasty lurking bug in the git-annex
+branch handling. It's unlikely to happen unless `annex.autocommit=false` is
+set, but it could occur when two git-annex processes race one another just
+right too. The root of the bug is that `git cat-file --batch` does not
+always show changes made to the index after it started. I think it does
+in enough cases to have tricked me before, but in general it can't be
+trusted to report the current state of the index, but only some past state.
+
+I was able to fix the bug, by ensuring that changes being made to the
+branch are always visible in either the journal or the branch -- never in
+the index alone.
+
+----
+
+Hopefully something less low-level tomorrow..!
diff --git a/doc/design/assistant/blog/day_83__3-way.mdwn b/doc/design/assistant/blog/day_83__3-way.mdwn
new file mode 100644
index 0000000000..9411c4c1c2
--- /dev/null
+++ b/doc/design/assistant/blog/day_83__3-way.mdwn
@@ -0,0 +1,73 @@
+Syncing works well when the graph of repositories is strongly connected.
+Now I'm working on making it work reliably with less connected graphs.
+
+I've been focusing on and testing a doubly-connected list of repositories,
+such as: `A <-> B <-> C`
+
+----
+
+I was seeing a lot of git-annex branch push failures occuring in
+this line of repositories topology. Sometimes was is able to recover from
+these, but when two repositories were trying to push to one-another at the
+same time, and both failed, both would pull and merge, which actually keeps
+the git-annex branch still diverged. (The two merge commits differ.)
+
+A large part of the problem was that it pushed directly into the git-annex
+branch on the remote; the same branch the remote modifies. I changed it to
+push to `synced/git-annex` on the remote, which avoids most push failures.
+Only when A and C are both trying to push into `B/synced/git-annex` at the
+same time would one fail, and need to pull, merge, and retry.
+
+-----
+
+With that change, git syncing always succeeded in my tests, and without
+needing any retries. But with more complex sets of repositories, or more
+traffic, it could still fail.
+
+I want to avoid repeated retries, exponential backoffs, and that kind of
+thing. It'd probably be good enough, but I'm not happy with it because
+it could take arbitrarily long to get git in sync.
+
+I've settled on letting it retry once to push to the synced/git-annex
+and synced/master branches. If the retry fails, it enters a fallback mode,
+which is guaranteed to succeed, as long as the remote is accessible.
+
+The problem with the fallback mode is it uses really ugly branch names.
+Which is why Joachim Breitner and I originally decided on making `git annex
+sync` use the single `synced/master` branch, despite the potential for
+failed syncs. But in the assistant, the requirements are different,
+and I'm ok with the uglier names.
+
+It does seem to make sense to only use the uglier names as a fallback,
+rather than by default. This preserves compatability with `git annex sync`,
+and it allows the assistant to delete fallback sync branches after it's
+merged them, so the ugliness is temporary.
+
+---
+
+Also worked some today on a bug that prevents C from receiving files
+added to A.
+
+The problem is that file contents and git metadata sync independantly. So C
+will probably receive the git metadata from B before B has finished
+downloading the file from A. C would normally queue a download of the
+content when it sees the file appear, but at this point it has nowhere to
+get it from.
+
+My first stab at this was a failure. I made each download of a file result
+in uploads of the file being queued to every remote that doesn't have it
+yet. So rather than C downloading from B, B uploads to C. Which works fine,
+but then C sees this download from B has finished, and proceeds to try to
+re-upload to B. Which rejects it, but notices that this download has
+finished, so re-uploads it to C...
+
+The problem with that approach is that I don't have an event when a download
+succeeds, just an event when a download ends.  Of course, C could skip
+uploading back to the same place it just downloaded from, but loops are
+still possible with other network topologies (ie, if D is connected to both
+B and C, there would be an upload loop 'B -> C -> D -> B`). So unless I can
+find a better event to hook into, this idea is doomed.
+
+I do have another idea to fix the same problem. C could certainly remember
+that it saw a file and didn't know where to get the content from, and then
+when it receives a git push of a git-annex branch, try again.
diff --git a/doc/design/assistant/blog/day_84__deferred_downloads.mdwn b/doc/design/assistant/blog/day_84__deferred_downloads.mdwn
new file mode 100644
index 0000000000..0ae684e476
--- /dev/null
+++ b/doc/design/assistant/blog/day_84__deferred_downloads.mdwn
@@ -0,0 +1,33 @@
+Implemented deferred downloads. So my example from yesterday of three
+repositories in a line keep fully in sync now!
+
+I punted on one problem while doing it. It might be possible to get a really
+big list of deferred downloads in some situation. That all lives in memory.
+I aim for git-annex to always have a constant upper bound on memory use,
+so that's not really acceptable. I have TODOed a reminder to do something
+about limiting the size of this list.
+
+----
+
+I also ran into a nasty crash while implementing this, where two threads
+were trying to do things to git HEAD at the same time, and so one crashed,
+and in a way I don't entirely understand, that crash took down another
+thread with a BlockedIndefinitelyOnSTM exception. I think I've fixed
+this, but it's bothersome that this is the second time that modifications
+to the Merger thread have led to a concurrency related crash that I
+have not fully understood. 
+
+My guess is that STM can get confused when it's
+retrying, and the thread that was preventing it from completing a
+transaction crashes, because it suddenly does not see any other
+references to the TVar(s) involved in the transaction. Any GHC STM gurus
+out there?
+
+---
+
+Still work to be done on making data transfers to keep fully in sync in all
+circumstances. One case I've realized needs work occurs when a USB drive is
+plugged in. Files are downloaded from it to keep the repo in sync, but the
+repo neglects to queue uploads of those files it just got out to other
+repositories it's in contact with. Seems I still need to do something to
+detecting when a successful download is done, and queue uploads.
diff --git a/doc/design/assistant/blog/day_85__more_foundation_work.mdwn b/doc/design/assistant/blog/day_85__more_foundation_work.mdwn
new file mode 100644
index 0000000000..25bad37488
--- /dev/null
+++ b/doc/design/assistant/blog/day_85__more_foundation_work.mdwn
@@ -0,0 +1,17 @@
+Turns out I was able to easily avoid the potential upload loops that would
+occur if each time a repo receives a download, it queues uploads to the
+repos it's connected to. With that done. I suspect, but have not proven,
+that the assistant is able to keep repos arranged in any shape of graph in
+sync, as long as it's connected (of course) and each connection is
+bi-directional. That's a good start .. or at least a nice improvement from
+only strongly connected graphs being kept in sync.
+
+Eliminated some empty commits that would be made sometimes, which is a nice
+optimisation.
+
+------
+
+I wanted to get back to some UI work after this week's deep dive into the
+internals. So I filled in a missing piece, the repository switcher in the
+upper right corner. Now the webapp's UI allows setting up different
+repositories for different purposes, and switching between them.
diff --git a/doc/design/assistant/blog/day_86__towards_the_beta.mdwn b/doc/design/assistant/blog/day_86__towards_the_beta.mdwn
new file mode 100644
index 0000000000..b11dd9bf90
--- /dev/null
+++ b/doc/design/assistant/blog/day_86__towards_the_beta.mdwn
@@ -0,0 +1,33 @@
+Putting together a shortlist of things I want to sort out before the beta.
+
+* [[Progress bars|progressbars]] for file uploads.
+* No mocked up parts in the webapp's UI. Think I implemented the last of
+  those yesterday, although there are some unlinked repository configuration
+  options.
+* The basic watching functionality, should work reliably.
+  There are some known scalability issues with eg,
+  [[kqueue on OSX|bugs/Issue_on_OSX_with_some_system_limits]] that
+  need to be dealt with, but needn't block a beta.
+* Should keep any configuration of repositories that can be set up using
+  the webapp in sync whenever it's possible to do so. I think that'll work
+  after the past few days work.
+* Should be easy to install and get running. Of course part of the point
+  of the beta release is to get it out there, on Hackage, in Debian
+  unstable, and in the other places that git-annex packagers put it.
+  As to getting it running, the autostart files and menu items look good
+  on Linux. The OSX equivilants still need work and testing.
+* No howlingly bad bugs. [[This bug|bugs/pasting_into_annex_on_OSX]] is
+  the one I'm most concerned with currently. OTOH, 
+  [[bugs/watcher_commits_unlocked_files]] can be listed in the errata.
+
+----
+
+So I worked on progress bars for uploads today. Wrote a nice little
+parser for rsync's progress output, that parses arbitrary size chunks,
+returning any unparsable part. Added a ProgressCallback parameter to
+all the backends' upload methods. Wrote a nasty thing that intercepts
+rsync's output, currently a character at a time (horrible, but rsync
+doesn't output that much, so surprisingly acceptable), and outputs it and
+parses it. Hooked all this up, and got it working for uploads to
+git remotes. That's 1/10th of the total ways uploads can happen that
+have working progress bars. It'll take a while to fill in the rest..
diff --git a/doc/design/assistant/blog/day_87__more_progress_progress.mdwn b/doc/design/assistant/blog/day_87__more_progress_progress.mdwn
new file mode 100644
index 0000000000..c2b266d6d9
--- /dev/null
+++ b/doc/design/assistant/blog/day_87__more_progress_progress.mdwn
@@ -0,0 +1,28 @@
+Worked more on upload progress tracking. I'm fairly happy with its state
+now:
+
+* It's fully implemented for rsync special remotes.
+
+* Git remotes also fully support it, with the
+  notable exception of file uploads run by `git-annex-shell recvkey`. That
+  runs `rsync --server --sender`, and in that mode, rsync refuses to output
+  progress info. Not sure what to do about this case. Maybe I should
+  write a parser for the rsync wire protocol that can tell what chunk of the
+  file is being sent, and shim it in front of the rsync server? That's
+  rather hardcore, but it seems the best of a bad grab bag of options that
+  include things like `LD_PRELOAD` hacks.
+
+* Also optimised the rsync progress bar reader to read whole
+  chunks of data rather than one byte at a time.
+
+* Also got progress bars to actually update in the webapp for uploads.
+
+  This turned out to be tricky because kqueue cannot be used to detect when
+  existing files have been modified. (One of kqueue's worst shortcomings vs
+  inotify.) Currently on kqueue systems it has to poll.
+
+I will probably upload add progress tracking to the directory special remote,
+which should be very easy (it already implements its own progress bars),
+and leave the other special remotes for later. I can add upload progress
+tracking to each special remote when I add support for configuring it in
+the webapp.
diff --git a/doc/design/assistant/blog/day_88__progressbars_still_progressing.mdwn b/doc/design/assistant/blog/day_88__progressbars_still_progressing.mdwn
new file mode 100644
index 0000000000..cd3c493c15
--- /dev/null
+++ b/doc/design/assistant/blog/day_88__progressbars_still_progressing.mdwn
@@ -0,0 +1,18 @@
+Short day today, but I again worked only on progress bars.
+
+* Added upload progress tracking for the directory special remote.
+* Some optimisations.
+* Added a `git annex-shell transferkey` command. This isn't used yet,
+  but the plan is to use it to feed back information about how much
+  of a file has been sent when downloading it. So that the uploader
+  can display a progress bar. This method avoids needing to parse the rsync
+  protocol, which is approximately impossible without copying half of rsync.
+  Happily, git-annex's automatic ssh connection caching will make the small
+  amount of data this needs to send be efficiently pipelined over the same
+  ssh connection that rsync is using.
+
+I probably have less than 10 lines of code to write to finish up
+[[progressbars]] for now. Looking forward to getting that behind me, and on
+to something more interesting. Even doing mail merge to print labels to
+mail out Kickstarter rewards is more interesting than progress bars at this
+point. :)
diff --git a/doc/design/assistant/blog/day_89__final_polish.mdwn b/doc/design/assistant/blog/day_89__final_polish.mdwn
new file mode 100644
index 0000000000..4a72a7ff23
--- /dev/null
+++ b/doc/design/assistant/blog/day_89__final_polish.mdwn
@@ -0,0 +1,24 @@
+Finally wrapped up progress bars; upload progress is now reported in all
+situations. 
+
+After all that, I was pleased to find a use for the progress info, beyond
+displaying it to the user. Now the assistant uses it to decide whether it
+makes sense to immediately retry a failed transfer. This should make it
+work nicely, or at least better, with flaky network or drives.
+
+The webapp crashed on startup when there was no `~/.gitconfig`.
+Guess all of us who have tried it so far are actual git users,
+but I'm glad I caught this before releasing the beta.
+
+Jimmy Tang kindly took on making a OS X .app directory for git-annex.
+So it now has an icon that will launch the webapp. 
+[[!img /assistant/osx-app.png]]
+
+I'm getting lots of contributors to git-annex all of a sudden. I've had 3
+patches this weekend, and 2 of them have been to Haskell code.
+Justin Azoff is working on [[todo/incremental_fsck]], and Robie Basak
+has [gotten Amazon Glacier working](https://github.com/basak/glacier-cli)
+using the hook special remote.
+
+Started doing some design for [[transfer_control]]. I will start
+work on this after releasing the first beta.
diff --git a/doc/design/assistant/blog/day_8__speed.mdwn b/doc/design/assistant/blog/day_8__speed.mdwn
new file mode 100644
index 0000000000..d99add97a7
--- /dev/null
+++ b/doc/design/assistant/blog/day_8__speed.mdwn
@@ -0,0 +1,67 @@
+Since last post, I've worked on speeding up `git annex watch`'s startup time
+in a large repository.
+
+The problem was that its initial scan was naively staging every symlink in
+the repository, even though most of them are, presumably, staged correctly
+already. This was done in case the user copied or moved some symlinks
+around while `git annex watch` was not running -- we want to notice and
+commit such changes at startup.
+
+Since I already had the `stat` info for the symlink, it can look at the
+`ctime` to see if the symlink was made recently, and only stage it if so.
+This sped up startup in my big repo from longer than I cared to wait (10+
+minutes, or half an hour while profiling) to a minute or so. Of course,
+inotify events are already serviced during startup, so making it scan
+quickly is really only important so people don't think it's a resource hog.
+First impressions are important. :)
+
+But what does "made recently" mean exactly? Well, my answer is possibly
+over engineered, but most of it is really groundwork for things I'll need
+later anyway. I added a new data structure for tracking the status of the
+daemon, which is periodically written to disk by another thread (thread #6!)
+to `.git/annex/daemon.status` Currently it looks like this; I anticipate
+adding lots more info as I move into the [[syncing]] stage:
+
+	lastRunning:1339610482.47928s
+	scanComplete:True
+
+So, only symlinks made after the daemon was last running need to be
+expensively staged on startup. Although, as RichiH pointed out,
+this fails if the clock is changed. But I have been planning to have a
+cleanup thread anyway, that will handle this, and other
+potential problems, so I think that's ok.
+
+Stracing its startup scan, it's fairly tight now. There are some repeated 
+`getcwd` syscalls that could be optimised out for a minor speedup.
+
+----
+
+Added the sanity check thread. Thread #7! It currently only does one sanity
+check per day, but the sanity check is a fairly lightweight job,
+so I may make it run more frequently. OTOH, it may never ever find a
+problem, so once per day seems a good compromise. 
+
+Currently it's only checking that all files in the tree are properly staged
+in git. I might make it `git annex fsck` later, but fscking the whole tree
+once per day is a bit much. Perhaps it should only fsck a few files per
+day? TBD
+
+Currently any problems found in the sanity check are just fixed and logged.
+It would be good to do something about getting problems that might indicate
+bugs fed back to me, in a privacy-respecting way. TBD
+
+----
+
+I also refactored the code, which was getting far too large to all be in
+one module. 
+
+I have been thinking about renaming `git annex watch` to `git annex assistant`,
+but I think I'll leave the command name as-is. Some users might
+want a simple watcher and stager, without the assistant's other features
+like syncing and the webapp. So the next stage of the
+[[roadmap|design/assistant]] will be a different command that also runs
+`watch`.
+
+At this point, I feel I'm done with the first phase of [[inotify]].
+It has a couple known bugs, but it's ready for brave beta testers to try.
+I trust it enough to be running it on my live data.
diff --git a/doc/design/assistant/blog/day_8__speed/comment_1_a3dba537b276d5737abc8cb93f1965f4._comment b/doc/design/assistant/blog/day_8__speed/comment_1_a3dba537b276d5737abc8cb93f1965f4._comment
new file mode 100644
index 0000000000..d0a207c82a
--- /dev/null
+++ b/doc/design/assistant/blog/day_8__speed/comment_1_a3dba537b276d5737abc8cb93f1965f4._comment
@@ -0,0 +1,10 @@
+[[!comment format=mdwn
+ username="https://www.google.com/accounts/o8/id?id=AItOawl9sYlePmv1xK-VvjBdN-5doOa_Xw-jH4U"
+ nickname="Richard"
+ subject="Battery usage"
+ date="2012-06-15T09:57:33Z"
+ content="""
+Complete fsck is good, but once a week probably enough.
+
+But please see if you can make fsck optional depending on if the machine is running on battery.
+"""]]
diff --git a/doc/design/assistant/blog/day_90__beta.mdwn b/doc/design/assistant/blog/day_90__beta.mdwn
new file mode 100644
index 0000000000..fc9bf4a41d
--- /dev/null
+++ b/doc/design/assistant/blog/day_90__beta.mdwn
@@ -0,0 +1,16 @@
+Just released git-annex 3.20120924, which includes beta versions of
+the assistant and webapp. Read the [[/assistant/errata]], then give it a
+try!
+
+I've uploaded it to Haskell's cabal, and to Debian unstable, and hope my
+helpers for other distributions will update them soon. (Although the
+additional dependencies to build the webapp may take a while on some.)
+I also hope something can be done to make a prebuilt version available on
+OSX soonish.
+
+I've decided to license the webapp under the
+[AGPL](http://www.gnu.org/licenses/agpl-3.0.html). This should not impact
+normal users of it, and git-annex can be built without the webapp as a pure
+GPL licensed program. This is just insurance to prevent someone turning the
+webapp into a propritary web-only service, by requiring that anyone who
+does so provide the source of the webapp.
diff --git a/doc/design/assistant/blog/day_90__beta/comment_1_5f2a3b18ad7558abe04f51534a29ff13._comment b/doc/design/assistant/blog/day_90__beta/comment_1_5f2a3b18ad7558abe04f51534a29ff13._comment
new file mode 100644
index 0000000000..6b95e317dd
--- /dev/null
+++ b/doc/design/assistant/blog/day_90__beta/comment_1_5f2a3b18ad7558abe04f51534a29ff13._comment
@@ -0,0 +1,9 @@
+[[!comment format=mdwn
+ username="https://www.google.com/accounts/o8/id?id=AItOawlJEI45rGczFAnuM7gRSj4C6s9AS9yPZDc"
+ nickname="Kevin"
+ subject="Haskell on MacPorts"
+ date="2012-09-24T21:09:39Z"
+ content="""
+FYI, I just came across a [MacPorts ticket](https://trac.macports.org/ticket/35813) to update Haskell to 2012.2.0.0.  It would be nice to install git-annex from MacPorts.
+
+"""]]
diff --git a/doc/design/assistant/blog/day_90__beta/comment_2_961c4eba97f4eac75174244d6b2b00c0._comment b/doc/design/assistant/blog/day_90__beta/comment_2_961c4eba97f4eac75174244d6b2b00c0._comment
new file mode 100644
index 0000000000..ed6c8e2aec
--- /dev/null
+++ b/doc/design/assistant/blog/day_90__beta/comment_2_961c4eba97f4eac75174244d6b2b00c0._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="https://www.google.com/accounts/o8/id?id=AItOawkSq2FDpK2n66QRUxtqqdbyDuwgbQmUWus"
+ nickname="Jimmy"
+ subject="comment 2"
+ date="2012-09-24T21:21:57Z"
+ content="""
+cool, I haven't been on top of things recently. If haskell-platform gets updated in macports and assuming its easy to package up any dependancies (if any) as macport packages, git-annex should be relatively straightforward to package up
+"""]]
diff --git a/doc/design/assistant/blog/day_90__beta/comment_3_c76675a4633cbbe347ed42c222918d38._comment b/doc/design/assistant/blog/day_90__beta/comment_3_c76675a4633cbbe347ed42c222918d38._comment
new file mode 100644
index 0000000000..151a9c2578
--- /dev/null
+++ b/doc/design/assistant/blog/day_90__beta/comment_3_c76675a4633cbbe347ed42c222918d38._comment
@@ -0,0 +1,24 @@
+[[!comment format=mdwn
+ username="https://www.google.com/accounts/o8/id?id=AItOawkurjhi0CRJvgm7QNaZDWS9hitBtavqIpc"
+ nickname="Bret"
+ subject="A few OS X bugs"
+ date="2012-09-25T00:50:56Z"
+ content="""
+Cabal update
+
+Cabal install git-annex 
+Seemed to install correctly on OS X 10.7
+
+After the installation, an application for git-annex showed up on my desktop.  This is the wrong place to put applications in OS X.  It should belong in the /Applications Folder.  
+Launching the git-annex app, the web-browser is opened, and it tells me to make a repository.  I entered /Users/[my home folder]/Annex and continued.  I then got an internal server error and nothing else happened.  The Error:
+
+uuid [\"-m\"] exited 127
+
+Annex was created, and there is a .git folder in it.
+
+Checking the annex status:
+Bret-Air:Annex bret$ git annex status
+git-annex: First run: git-annex init
+
+I'll try it on 10.6 here in just a moment.  Let me know if I can provide any more useful information.  
+"""]]
diff --git a/doc/design/assistant/blog/day_90__beta/comment_4_f0b8f77cb691e747fe35bcf2f51b5baa._comment b/doc/design/assistant/blog/day_90__beta/comment_4_f0b8f77cb691e747fe35bcf2f51b5baa._comment
new file mode 100644
index 0000000000..f6889fe049
--- /dev/null
+++ b/doc/design/assistant/blog/day_90__beta/comment_4_f0b8f77cb691e747fe35bcf2f51b5baa._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="https://www.google.com/accounts/o8/id?id=AItOawkurjhi0CRJvgm7QNaZDWS9hitBtavqIpc"
+ nickname="Bret"
+ subject="comment 4"
+ date="2012-09-25T00:59:51Z"
+ content="""
+OS X 10.6 is doing the same thing as on 10.7.
+"""]]
diff --git a/doc/design/assistant/blog/day_90__beta/comment_5_99fbc9feac62e66a12b0d357cf86ccc1._comment b/doc/design/assistant/blog/day_90__beta/comment_5_99fbc9feac62e66a12b0d357cf86ccc1._comment
new file mode 100644
index 0000000000..d886d56bb5
--- /dev/null
+++ b/doc/design/assistant/blog/day_90__beta/comment_5_99fbc9feac62e66a12b0d357cf86ccc1._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="http://joeyh.name/"
+ ip="4.153.14.141"
+ subject="comment 5"
+ date="2012-09-25T14:49:31Z"
+ content="""
+@Bret my best guess (without any error message from the uuid command) is that your uuid doesn't support the -m parameter. I've made the configure step check for this and it'll fall back to uuid without the parameter.
+"""]]
diff --git a/doc/design/assistant/blog/day_91__break.mdwn b/doc/design/assistant/blog/day_91__break.mdwn
new file mode 100644
index 0000000000..f2102746da
--- /dev/null
+++ b/doc/design/assistant/blog/day_91__break.mdwn
@@ -0,0 +1,7 @@
+Mostly took a break from working on the assistant today. Instead worked
+on adding incremental fsck to git-annex. Well, that will be something
+that assistant will use, eventually, probably.
+
+Jimmy and I have been working on a self-contained OSX app for using the
+assistant, that doesn't depend on installing git, etc. More on that
+once we have something that works.
diff --git a/doc/design/assistant/blog/day_92__S3.mdwn b/doc/design/assistant/blog/day_92__S3.mdwn
new file mode 100644
index 0000000000..c3f275a865
--- /dev/null
+++ b/doc/design/assistant/blog/day_92__S3.mdwn
@@ -0,0 +1,23 @@
+Amazon S3 was the second most popular choice in the
+[[polls/prioritizing_special_remotes]] poll, and since I'm not sure how
+I want to support phone/mp3 players, I did it first.
+
+So I added a configurator today to easily set up an Amazon S3 repository.
+That was straightforward and didn't take long since git-annex already
+supported S3.
+
+The hard part, of course, is key distribution. Since the webapp so far
+can only configure the shared encryption method, and not fullblown gpg keys,
+I didn't feel it would be secure to store the S3 keys in the git repository.
+Anyone with access to that git repo would have full access to S3 ... just not
+acceptable. Instead, the webapp stores the keys in a 600 mode file locally,
+and they're not distributed at all.
+
+When the same S3 repository is enabled on another computer, it prompts for
+keys then too. I did add a hint about using the IAM Management Console in
+this case -- it should be possible to set up users in IAM who can only
+access a single bucket, although I have not tried to set that up.
+
+---
+
+Also, more work on the standalone OSX app.
diff --git a/doc/design/assistant/blog/day_92__S3/comment_1_eda656247d11cea7fbed2e33137a39e5._comment b/doc/design/assistant/blog/day_92__S3/comment_1_eda656247d11cea7fbed2e33137a39e5._comment
new file mode 100644
index 0000000000..af610bdb73
--- /dev/null
+++ b/doc/design/assistant/blog/day_92__S3/comment_1_eda656247d11cea7fbed2e33137a39e5._comment
@@ -0,0 +1,10 @@
+[[!comment format=mdwn
+ username="https://www.google.com/accounts/o8/id?id=AItOawl9sYlePmv1xK-VvjBdN-5doOa_Xw-jH4U"
+ nickname="Richard"
+ subject="comment 1"
+ date="2012-09-26T23:52:50Z"
+ content="""
+I think a lot of people misinterpreted \"phone/mp3 player\" as \"Android\"; I know I did.
+
+As you already support directories, MTP would be needed to support the other half of pretty much all phones and media players (other than those made by Apple). MTP is a valley of pain, though.
+"""]]
diff --git a/doc/design/assistant/blog/day_92__S3/comment_2_8249d2d9521e44c674da3fda74be077a._comment b/doc/design/assistant/blog/day_92__S3/comment_2_8249d2d9521e44c674da3fda74be077a._comment
new file mode 100644
index 0000000000..fb83b23e4a
--- /dev/null
+++ b/doc/design/assistant/blog/day_92__S3/comment_2_8249d2d9521e44c674da3fda74be077a._comment
@@ -0,0 +1,10 @@
+[[!comment format=mdwn
+ username="https://www.google.com/accounts/o8/id?id=AItOawlFd2EbAmGD0FjHmuoerXfT0GO_jPcgvQA"
+ nickname="Gert"
+ subject="Single bucket for S3"
+ date="2012-09-27T09:17:24Z"
+ content="""
+You can easily do that with IAM, checkout .
+
+It boils down to creating a policy for each user/bucket. But allow read access to the full bucket list (otherwise the AWS Console won't work).
+"""]]
diff --git a/doc/design/assistant/blog/day_93__OSX_standalone_app.mdwn b/doc/design/assistant/blog/day_93__OSX_standalone_app.mdwn
new file mode 100644
index 0000000000..711710fc7c
--- /dev/null
+++ b/doc/design/assistant/blog/day_93__OSX_standalone_app.mdwn
@@ -0,0 +1,23 @@
+Various bug fixes, and work on the OSX app today:
+
+* Avoid crashing when ssh-keygen fails due to not being able to parse
+  `authorized_keys`.. seems a lot of people have crufty unparsable
+  `authorized_keys` files.
+* On OSX, for some reason the webapp was failing to start sometimes due
+  to bind failing with EINVAL. I don't understand why, as that should
+  only happen if the socket is already bound, which it should not as
+  it's just been created. I was able to work around this by retrying
+  with a new socket when bind fails.
+* When setting up `authorized_keys` to let `git-annex-shell` be run,
+  it had been inserting a perl oneliner into it. I changed that
+  to instead call a `~/.ssh/git-annex-shell` wrapper script that it sets
+  up. The benefits are it no longer needs perl, and it's less ugly,
+  and the standalone OSX app can modify the wrapper script to point to
+  wherever it's installed today (people like to move these things around I
+  guess).
+* Made the standalone OSX app set up autostarting when it's first run.
+* Spent rather a long time collecting the licenses of all the software that
+  will be bundled with the standalone OSX app. Ended up with a file
+  containing 3954 lines of legalese. Happily, all the software appears
+  redistributable, and free software; even the couple of OSX system libraries
+  we're bundling are licensed under the APSL.
diff --git a/doc/design/assistant/blog/day_93__easy_install.mdwn b/doc/design/assistant/blog/day_93__easy_install.mdwn
new file mode 100644
index 0000000000..013b872e12
--- /dev/null
+++ b/doc/design/assistant/blog/day_93__easy_install.mdwn
@@ -0,0 +1,34 @@
+I hear that people want the git-annex assistant to be easy to install
+without messing about building it from source..
+
+## on OSX
+
+So Jimmy and I have been working all week on making an easily installed OSX
+app of the assistant. This is a .dmz file that bundles all the dependencies
+(git, etc) in, so it can be installed with one click.
+
+It seems to basically work. You can get it [[here|install/OSX]].
+
+Unfortunatly, the [[bugs/pasting_into_annex_on_OSX]] bug resurfaced while
+testing this.. So I can't really recommend using it on real data yet.
+
+Still, any testing you can do is gonna be really helpful. I'm squashing OSX
+bugs right and left.
+
+## on Linux
+
+First of all, the git-annex assistant is now available in Debian unstable,
+and in Arch Linux's AUR. Proper packages.
+
+For all the other Linux distributions, I have a workaround. It's
+a big hack, but it seems to work.. at least on Debian stable.
+
+I've just put up a [[install/linux_standalone]] tarball, which has **no
+library dependencies** apart from glibc, and doesn't even need git to be
+installed on your system.
+
+## on FreeBSD
+
+The FreeBSD port has been updated to include the git-annex assistant too..
+
+[[!meta title="day 94 easy install"]]
diff --git a/doc/design/assistant/blog/day_93__easy_install/comment_1_d4f7de723c98577ef28d89ee6b87fd13._comment b/doc/design/assistant/blog/day_93__easy_install/comment_1_d4f7de723c98577ef28d89ee6b87fd13._comment
new file mode 100644
index 0000000000..57a0c78d7a
--- /dev/null
+++ b/doc/design/assistant/blog/day_93__easy_install/comment_1_d4f7de723c98577ef28d89ee6b87fd13._comment
@@ -0,0 +1,10 @@
+[[!comment format=mdwn
+ username="http://rjc.pip.verisignlabs.com/"
+ ip="86.22.66.200"
+ subject="assistant in Debian"
+ date="2012-09-30T09:10:17Z"
+ content="""
+The same version of ghc in Debian repository on i386 and amd64 privides ghc-ghci virtual package while on all the other architectures it doesn't. Is that normal or shall I file a bug?
+
+What it basically means is that the packages, both git-annex (with assistant) and its build dependencies, for all the other archs cannot be build on Debian porting machines until that is fixed. It already is in the experimental version but until that one gets to unstable there won't be any official debs for all non-Intel architecures.
+"""]]
diff --git a/doc/design/assistant/blog/day_93__easy_install/comment_2_6337b341c1cfb2132b59704394e57b36._comment b/doc/design/assistant/blog/day_93__easy_install/comment_2_6337b341c1cfb2132b59704394e57b36._comment
new file mode 100644
index 0000000000..ba027e9217
--- /dev/null
+++ b/doc/design/assistant/blog/day_93__easy_install/comment_2_6337b341c1cfb2132b59704394e57b36._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="http://joeyh.name/"
+ ip="4.154.0.149"
+ subject="comment 2"
+ date="2012-09-30T17:11:13Z"
+ content="""
+The next upload to debian will enable building the assistant on architectures that don't have template haskell and so cannot build the webapp.
+"""]]
diff --git a/doc/design/assistant/blog/day_95__repository_groups.mdwn b/doc/design/assistant/blog/day_95__repository_groups.mdwn
new file mode 100644
index 0000000000..5e4bc6d209
--- /dev/null
+++ b/doc/design/assistant/blog/day_95__repository_groups.mdwn
@@ -0,0 +1,21 @@
+Spent a lot of time this weekend thinking about/stuck on the [[cloud]]
+notification problem. Currently IRC is looking like the best way for
+repositories to notify one-another when changes are made, but I'm not sure
+about using that, and not ready to start on it.
+
+Instead, laid some groundwork for [[transfer_control]] today. Added 
+some simple commands to manage groups of repositories, and find files
+that are present in repositories in a group. I'm not completely happy
+with the syntax for that, and need to think up some good syntax to specify
+files that are present in *all* repositories in a group.
+
+The plan is to have the assistant automatically guess at groups to put new
+repositories it makes in (it should be able to make good guesses),
+as well as have an interface to change them, and an interface to configure
+transfer control using these groups (and other ways of matching files).
+And, probably, some canned transfer control recipes for common setups.
+
+---
+
+Collected up the past week's work and made a release today. I'm probably
+back to making regular releases every week or two.
diff --git a/doc/design/assistant/blog/day_96__revisiting_file_adds.mdwn b/doc/design/assistant/blog/day_96__revisiting_file_adds.mdwn
new file mode 100644
index 0000000000..a9607bb804
--- /dev/null
+++ b/doc/design/assistant/blog/day_96__revisiting_file_adds.mdwn
@@ -0,0 +1,24 @@
+Today I revisited something from way back in [[day_7__bugfixes]].
+Back then, it wasn't practical to run `git ls-files` on every
+file the watcher noticed, to check if it was already in git. Revisiting
+this, I found I could efficiently do that check at the same point it checks
+`lsof`. When there's a lot of files being added, they're batched up at that
+point, so it won't be calling `git ls-files` repeatedly.
+
+Result: It's safe to mix use of the assistant with files stored in git
+in the normal way. And it's safe to mix use of `git annex unlock` with
+the assistant; it won't immediately re-lock files. Yay!
+
+----
+
+Also fixed a crash in the committer, and made `git annex status` display
+repository groups.
+
+----
+
+Been thinking through where to store the [[transfer_control]] expressions.
+Since repositories need to know about the transfer controls of other
+remotes, storing them in `.git/config` isn't right. I thought it might be
+nice to configure the expressions in `.gitattributes`, but it seems the
+file format doesn't allow complicated multi-word attributes. Instead,
+they'll be stored in the git-annex branch.
diff --git a/doc/design/assistant/blog/day_96__revisiting_file_adds/comment_1_da3ca47041168b6c82aeb2c18acc5017._comment b/doc/design/assistant/blog/day_96__revisiting_file_adds/comment_1_da3ca47041168b6c82aeb2c18acc5017._comment
new file mode 100644
index 0000000000..298a953710
--- /dev/null
+++ b/doc/design/assistant/blog/day_96__revisiting_file_adds/comment_1_da3ca47041168b6c82aeb2c18acc5017._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="http://wiggy.net/"
+ nickname="Wichert"
+ subject="symlinks?"
+ date="2012-10-03T07:15:21Z"
+ content="""
+Does that mean that git-annex will no longer need to turn everything into symlinks? I suspect that that will improve behaviour on at least OSX quite a bit.
+"""]]
diff --git a/doc/design/assistant/blog/day_97__stuffing.mdwn b/doc/design/assistant/blog/day_97__stuffing.mdwn
new file mode 100644
index 0000000000..595aca1e0c
--- /dev/null
+++ b/doc/design/assistant/blog/day_97__stuffing.mdwn
@@ -0,0 +1,14 @@
+Not a lot of programming today; I spent most of the day stuffing hundreds
+of envelopes for this Kickstarter thing you may have heard of. Some post
+office is going to be very surprised with all the international mail soon.
+
+----
+
+That said, I did write 184 lines of code. (Actually rather a lot, but it
+was mostly pure functional code, so easy to write.) That
+pops up your text editor on a file with the the trust and group
+configurations of repositories, that's stored in the git-annex branch.
+Handy for both viewing that stuff all in one place, and changing it.
+
+The real reason for doing that is to provide a nice interface for editing
+transfer control expressions, which I'll be adding next.
diff --git a/doc/design/assistant/blog/day_98__preferred_content.mdwn b/doc/design/assistant/blog/day_98__preferred_content.mdwn
new file mode 100644
index 0000000000..d1324f69eb
--- /dev/null
+++ b/doc/design/assistant/blog/day_98__preferred_content.mdwn
@@ -0,0 +1,44 @@
+Started implementing [[transfer_control]]. Although I'm currently calling
+the configuration for it "preferred content expressions". (What a mouthful!)
+
+I was mostly able to reuse the Limit code (used to handle parameters like
+--not --in otherrepo), so it can already build Matchers for preferred content
+expressions in my little Domain Specific Language.
+
+Preferred content expressions can be edited with `git annex vicfg`, which
+checks that they parse properly. 
+
+The plan is that the first place to use them is not going to be inside the
+assistant, but in commands that use the `--auto` parameter, which will use
+them as an additional constraint, in addition to the numcopies setting
+already used. Once I get it working there, I'll add it to the assistant.
+
+Let's say a repo has a preferred content setting of
+"(not copies=trusted:2) and (not in=usbdrive)"
+
+* `git annex get --auto` will get files that have less than 2 trusted
+  copies, and are not in the usb drive.
+* `git annex drop --auto` will drop files that have 2 or more trusted
+  copies, and are not in the usb drive (assuming numcopies allows dropping
+  them of course).
+* `git annex copy --auto --to thatrepo` run from another repo
+  will only copy files that have less than 2 trusted copies. (And if that
+  was run on the usb drive, it'd never copy anything!)
+
+There is a complication here.. What if the repo with that preferred content
+setting is itself trusted? Then when it gets a file, its number of
+trusted copies increases, which will make it be dropped again. :-/
+
+This is a nuance that the numcopies code already deals with, but it's
+much harder to deal with it in these complicated expressions. I need to think
+about this; the three ideas I'm working on are:
+
+1. Leave it to whoever/whatever writes these expressions to write ones
+   that avoid such problems. Which is ok if I'm the only one writing
+   pre-canned ones, in practice..
+2. Transform expressions into ones that avoid such problems. (For example,
+   replace "not copies=trusted:2" with "not (copies=trusted:2 or (in=here and
+   trusted=here and copies=trusted:3))"
+3. Have some of the commands (mostly drop I think) pretend the drop
+   has already happened, and check if it'd then want to get the file back
+   again.
diff --git a/doc/design/assistant/blog/day_98__preferred_content/comment_1_2136618e3515d0ac6369a41f1934ec2a._comment b/doc/design/assistant/blog/day_98__preferred_content/comment_1_2136618e3515d0ac6369a41f1934ec2a._comment
new file mode 100644
index 0000000000..b0fcd8c798
--- /dev/null
+++ b/doc/design/assistant/blog/day_98__preferred_content/comment_1_2136618e3515d0ac6369a41f1934ec2a._comment
@@ -0,0 +1,17 @@
+[[!comment format=mdwn
+ username="http://meep.pl/"
+ ip="193.23.174.18"
+ subject="Ah, the barber paradox"
+ date="2012-10-05T06:51:11Z"
+ content="""
+Nice. Would (not in=here) be the simplest paradoxical expression?
+
+Is just disregarding the target repo completely during checks a possibility? This would interpret (not copies=trusted:X) as \"not in X *other* trusted repositories, no matter whether we are trusted or not\", and (not in=here) just as \"true\". I think this should generally arrive at the same results as the option 2., but by definition of the expression meaing, not by rewriting.
+
+Alternative 3 (or is my wording different enough to be 3a?) - check that the invariant \"we have all the known files matching our PCE and only these files\" would hold after an operation before actually performing it - could be bistable if done both for gets and drops:
+
+* (not in=here) and we do not have the file -> get thinks \"if we get it, we have a file not matching the PCE\" -> get does not get it;
+* (not in=here) and we do have the file -> drop thinks \"if we drop it, there exists a file matching the PCE which we miss\" -> drop does not drop it.
+
+This is not necessarily bad. Checking just for drops should be monostable, I guess, but doesn't it look a bit arbitrary? (Though it would be again equivalent to option 2, wouldn't it? So maybe not that arbitrary.)
+"""]]
diff --git a/doc/design/assistant/blog/day_98__preferred_content/comment_2_5f6db00e69628bf2f72b0e6f2981a49b._comment b/doc/design/assistant/blog/day_98__preferred_content/comment_2_5f6db00e69628bf2f72b0e6f2981a49b._comment
new file mode 100644
index 0000000000..fa1ce32b83
--- /dev/null
+++ b/doc/design/assistant/blog/day_98__preferred_content/comment_2_5f6db00e69628bf2f72b0e6f2981a49b._comment
@@ -0,0 +1,14 @@
+[[!comment format=mdwn
+ username="http://joeyh.name/"
+ ip="4.154.0.149"
+ subject="comment 2"
+ date="2012-10-05T15:08:26Z"
+ content="""
+Yes, I think checking the future only for drops is both stable and equivilant to the other choices. 
+
+Disregarding the target solves the problem for the current set of expressions. There may be future expressions or operations where that does not hold. For example, if move supported --auto (which it does not), you'd need to disregard both sides.
+
+That method would make it impossible to do some possibly useful things. \"in=here or (not copies=3)\"
+
+The real problem with it is that existing options like --copies and --in already take all repos into account, so this would potentally lead to two divergent DSLs being used by git-annex, which would probably be confusing.
+"""]]
diff --git a/doc/design/assistant/blog/day_99_shotgun.mdwn b/doc/design/assistant/blog/day_99_shotgun.mdwn
new file mode 100644
index 0000000000..77a08f3cb3
--- /dev/null
+++ b/doc/design/assistant/blog/day_99_shotgun.mdwn
@@ -0,0 +1,70 @@
+Fixed the assistant to wait on all the zombie processes that would sometimes
+pile up. I didn't realize this was as bad as it was.
+
+Zombies and git-annex have been a problem since I started developing it,
+because back then I made some rather poor choices, due to barely knowing
+how to write Haskell. So parts of the code that stream input from git commands
+don't clean up after them properly. Not normally a problem, because
+git-annex reaps the zombies after each file it processes. But this reaping
+is not thread-safe; it cannot be used in the assistant.
+
+If I were starting git-annex today, I'd use one of the new Haskell things like
+Conduits, that allow for very clean control over finalization of resources.
+But switching it to Conduits now would probably take weeks of work; I've not
+yet felt it was worthwhile. (Also it's not clear Conduits are the last,
+best thing.)
+
+For now, it keeps track of the pids it needs to wait on, and all the code
+run by the assistant is zombie-free. However, some code for fsck and unused
+that I anticipate the assistant using eventually still has some lurking
+zombies.
+
+----
+
+Solved the issue with preferred content expressions and dropping that
+I mentioned yesterday. My solution was to add a parameter to specify a set
+of repositories where content should be assumed not to be present. When
+deciding whether to drop, it can put the current repository in, and then
+if the expression fails to match, the content can be dropped.
+
+Using yesterday's example "(not copies=trusted:2) and (not in=usbdrive)",
+when the local repo is one of the 2 trusted copies, the drop check will
+see only 1 trusted copy, so the expression matches, and so the content will
+not be dropped.
+
+I've not tested my solution, but it type checks. :P I'll wire it up to
+`get/drop/move --auto` tomorrow and see how it performs.
+
+----
+
+Would preferred content expressions be more readble if they were inverted
+(becoming content filtering expressions)?
+
+1. "(not copies=trusted:2) and (not in=usbdrive)" becomes
+   "copies=trusted:2 or in=usbdrive"
+2. "smallerthan=10mb and include=*.mp3 and exclude=junk/*" becomes
+   "largerthan=10mb or exclude=*.mp3" or include=junk/*"
+3. "(not group=archival) and (not copies=archival:1)" becomes
+   "group=archival or copies=archival:1"
+
+1 and 3 are improved, but 2, less so. It's a trifle weird for "include"
+to mean "include in excluded content".
+
+The other reason not to do this is that currently the expressions
+can be fed into `git annex find` on the command line, and it'll come
+back with the files that would be kept.
+
+Perhaps a middle groud is to make "dontwant" be an alias for "not".
+Then we can write "dontwant (copies=trusted:2 or in=usbdrive)"
+
+----
+
+A user told me this:
+
+> I can confirm that the assistant does what it is supposed to do really well. I
+> just hooked up my notebook to the network and it starts syncing from notebook to
+> fileserver and the assistant on the fileserver also immediately starts syncing
+> to the [..] backup
+
+That makes me happy, it's the first quite so real-world success report I've
+heard.
diff --git a/doc/design/assistant/blog/day_99_shotgun/comment_1_12bb8f54bb13ea20ac4187a2301d77ca._comment b/doc/design/assistant/blog/day_99_shotgun/comment_1_12bb8f54bb13ea20ac4187a2301d77ca._comment
new file mode 100644
index 0000000000..eeaefef421
--- /dev/null
+++ b/doc/design/assistant/blog/day_99_shotgun/comment_1_12bb8f54bb13ea20ac4187a2301d77ca._comment
@@ -0,0 +1,10 @@
+[[!comment format=mdwn
+ username="http://meep.pl/"
+ ip="193.23.174.18"
+ subject="Include/exclude"
+ date="2012-10-06T08:40:23Z"
+ content="""
+It seems that example 2 reads weird for filtering because exclude/include say *what to do* while copies/in say *what to test*. An alias (maybe \"glob\") for include  should be acceptable both ways (largerthan=10mb or glob=junk/\* or not glob=\*.mp3).
+
+That said, I would vote for keeping PCEs. And actually the simple \"not (copies=trusted:2 or in=usbdrive)\" does not read significantly worse than with \"dontwant\". (In *my bikeshed* we would have \"all\" == \"glob=\*\" and \"except\" == \"and not\").
+"""]]
diff --git a/doc/design/assistant/blog/day_9__correctness.mdwn b/doc/design/assistant/blog/day_9__correctness.mdwn
new file mode 100644
index 0000000000..1fa4c09d0d
--- /dev/null
+++ b/doc/design/assistant/blog/day_9__correctness.mdwn
@@ -0,0 +1,30 @@
+	git merge watch_
+
+My cursor has been mentally poised here all day, but I've been reluctant to
+merge watch into master. It seems solid, but is it correct? I was able to
+think up a lot of races it'd be subject to, and deal with them, but did I
+find them all?
+
+Perhaps I need to do some automated fuzz testing to reassure myself.
+I looked into using [genbackupdata](http://liw.fi/genbackupdata/) to that
+end. It's not quite what I need, but could be
+[moved in that direction](http://bugs.debian.org/677542). Or I could write
+my own fuzz tester, but it seems better to use someone else's, because 
+a) laziness and b) they're less likely to have the same blind spots I do.
+
+My reluctance to merge isn't helped by the known bugs with files that are
+either already open before `git annex watch` starts, or are opened by two
+processes at once, and confuse it into annexing the still-open file when one
+process closes it.
+
+I've been thinking about just running `lsof` on every file as it's being
+annexed to check for that, but in the end, `lsof` is too slow. Since its
+check involves trawling through all of /proc, it takes it a good half a
+second to check a file, and adding 25 seconds to the time it takes to
+process 100 files is just not acceptable.
+
+But an option that could work is to run `lsof` after a bunch of new files
+have been annexed. It can check a lot of files nearly as fast as a single
+one. In the rare case that an annexed file is indeed still open, it could
+be moved back out of the annex. Then when its remaining writer finally
+closes it, another inotify event would re-annex it.
diff --git a/doc/design/assistant/blog/day_9__correctness/comment_1_564a39cb976767e2c0a9c74fabe10be4._comment b/doc/design/assistant/blog/day_9__correctness/comment_1_564a39cb976767e2c0a9c74fabe10be4._comment
new file mode 100644
index 0000000000..8236002ccd
--- /dev/null
+++ b/doc/design/assistant/blog/day_9__correctness/comment_1_564a39cb976767e2c0a9c74fabe10be4._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="http://wiggy.net/"
+ nickname="Wichert"
+ subject="os compatibility"
+ date="2012-06-15T07:19:23Z"
+ content="""
+A downside of relying on lsof is that you might be painting yourself into a linux corner: other operating systems might not have a lsof or alternative you can rely on. Especially for Windows this might be a worry. 
+"""]]
diff --git a/doc/design/assistant/blog/day_9__correctness/comment_2_77924e9d50b40f05e792e427a25849a6._comment b/doc/design/assistant/blog/day_9__correctness/comment_2_77924e9d50b40f05e792e427a25849a6._comment
new file mode 100644
index 0000000000..8744882c97
--- /dev/null
+++ b/doc/design/assistant/blog/day_9__correctness/comment_2_77924e9d50b40f05e792e427a25849a6._comment
@@ -0,0 +1,9 @@
+[[!comment format=mdwn
+ username="http://dieter-be.myopenid.com/"
+ nickname="dieter"
+ subject="filesystem number of open file handles on a file"
+ date="2012-06-15T08:21:37Z"
+ content="""
+wasn't there some filesystem functionality that could tell you the amount of open file handles on a certain file?  I thought this was tracked per-file too.
+Or maybe i'm just confusing it with the number of hard links (which stat can tell you), anyway something to look into.
+"""]]
diff --git a/doc/design/assistant/blog/day_9__correctness/comment_3_92bd86cd06d579e23800af2e5c66a291._comment b/doc/design/assistant/blog/day_9__correctness/comment_3_92bd86cd06d579e23800af2e5c66a291._comment
new file mode 100644
index 0000000000..680f16ecc1
--- /dev/null
+++ b/doc/design/assistant/blog/day_9__correctness/comment_3_92bd86cd06d579e23800af2e5c66a291._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="https://www.google.com/accounts/o8/id?id=AItOawkSq2FDpK2n66QRUxtqqdbyDuwgbQmUWus"
+ nickname="Jimmy"
+ subject="comment 3"
+ date="2012-06-15T08:58:17Z"
+ content="""
+I would also be reluctant to use lsof for the sake of non-linux systems or systems that don't have lsof. I've only been playing around with the watch branch of my \"other\" laptop under archlinux. It looks usable, however I would prefer support for OSX before the watch branch gets merged to master ;)
+"""]]
diff --git a/doc/design/assistant/blog/day_9__correctness/comment_4_0d12b51ccdfc2a94d3e59a5628521e0a._comment b/doc/design/assistant/blog/day_9__correctness/comment_4_0d12b51ccdfc2a94d3e59a5628521e0a._comment
new file mode 100644
index 0000000000..7218754916
--- /dev/null
+++ b/doc/design/assistant/blog/day_9__correctness/comment_4_0d12b51ccdfc2a94d3e59a5628521e0a._comment
@@ -0,0 +1,10 @@
+[[!comment format=mdwn
+ username="https://www.google.com/accounts/o8/id?id=AItOawl9sYlePmv1xK-VvjBdN-5doOa_Xw-jH4U"
+ nickname="Richard"
+ subject="comment 4"
+ date="2012-06-15T10:21:17Z"
+ content="""
+Corner case, but if the other program finishes writing while you are annexing and your check shows no open files, you are left with bad checksum on a correct file. This \"broken\" file with propagate and the next round of fsck will show that all copies are \"bad\".
+
+Without verifying if this is viable, could you set the file RO and thus block future writes before starting to annex?
+"""]]
diff --git a/doc/design/assistant/blog/day_9__correctness/comment_5_208f9dd3e1d92555b05c29159538a901._comment b/doc/design/assistant/blog/day_9__correctness/comment_5_208f9dd3e1d92555b05c29159538a901._comment
new file mode 100644
index 0000000000..b7fbecc398
--- /dev/null
+++ b/doc/design/assistant/blog/day_9__correctness/comment_5_208f9dd3e1d92555b05c29159538a901._comment
@@ -0,0 +1,14 @@
+[[!comment format=mdwn
+ username="http://joeyh.name/"
+ ip="4.154.6.135"
+ subject="comment 5"
+ date="2012-06-15T15:14:52Z"
+ content="""
+@wichert All this inotify stuff is entirely linux specific AFAIK anyway, so it's find for workarounds to limitations in inotify functionality to also be linux specific.
+
+@dieter I think you're thinking of hard links, filesystems don't track number of open file handles afaik.
+
+@Jimmy, I'm planning to get watch going on freebsd (and hopefully that will also cover OSX), after merging it :)
+
+@Richard, the file is set RO while it's being annexed, so any lsof would come after that point.
+"""]]
diff --git a/doc/design/assistant/blog/day_9__correctness/comment_6_90cc6b60718896fb175919417600fdf9._comment b/doc/design/assistant/blog/day_9__correctness/comment_6_90cc6b60718896fb175919417600fdf9._comment
new file mode 100644
index 0000000000..622b141fdb
--- /dev/null
+++ b/doc/design/assistant/blog/day_9__correctness/comment_6_90cc6b60718896fb175919417600fdf9._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="http://joeyh.name/"
+ ip="4.154.6.135"
+ subject="comment 6"
+ date="2012-06-15T15:23:21Z"
+ content="""
+But Rich is right, and I was thinking the same thing earlier this morning, that delaying the lsof allows the writer to change the file and exit, and only fsck can detect the problem then. Setting file permissions doesn't help once a process already has it open for write. Which has put me off the delayed lsof idea unfortunately. lsof *could* be run safely during the intial annexing.
+"""]]
diff --git a/doc/design/assistant/chunks.mdwn b/doc/design/assistant/chunks.mdwn
new file mode 100644
index 0000000000..0aa389899a
--- /dev/null
+++ b/doc/design/assistant/chunks.mdwn
@@ -0,0 +1,271 @@
+To avoid leaking even the size of your encrypted files to cloud storage
+providers, add a mode that stores fixed size chunks.
+
+May be a useful starting point for [[deltas]].
+
+May also allow for downloading different chunks of a file concurrently from
+multiple remotes.
+
+Also, can allow resuming of interrupted uploads and downloads.
+
+# legacy chunking
+
+Supported by only the webdav and directory special remotes.
+
+Filenames are used for the chunks that make it easy to see which chunks
+belong together, even when encryption is used. There is also a chunkcount
+file, that similarly leaks information.
+
+It is not possible to enable chunking on a non-chunked remote.
+
+Problem: Two uploads of the same key from repos with different chunk sizes
+could lead to data loss. For example, suppose A is 10 mb chunksize, and B
+is 20 mb, and the upload speed is the same. If B starts first, when A will
+overwrite the file it is uploading for the 1st chunk. Then A uploads the
+second chunk, and once A is done, B finishes the 1st chunk and uploads its
+second. We now have [chunk 1(from A), chunk 2(from B)].
+
+# new requirements
+
+Every special remote should support chunking. (It does not make sense
+to support it for git remotes, but gcrypt remotes should support it.)
+
+S3 remotes should chunk by default, because the current S3 backend fails
+for files past a certian size. See [[bugs/Truncated_file_transferred_via_S3]].
+
+The size of chunks, as well as whether any chunking is done, should be
+configurable on the fly without invalidating data already stored in the
+remote. This seems important for usability (eg, so users can turn chunking
+on in the webapp when configuring an existing remote).
+
+Two concurrent uploaders of the same object to a remote should be safe,
+even if they're using different chunk sizes.
+
+The legacy chunk method needs to be supported for back-compat, so
+keep the chunksize= setting to enable that mode, and add a new chunk=
+setting for the new mode.
+
+# obscuring file sizes
+
+To hide from a remote any information about the sizes of files could be
+another goal of chunking. At least two things are needed for this:
+
+1. The filenames used on the remote don't indicate which chunks belong
+   together.
+
+2. The final short chunk needs to be padded with random data,
+   so that a remote sees only encrypted files with uniform sizes
+   and cannot make guesses about the kinds of data being stored.
+
+Note that padding cannot completely hide all information from an attacker
+who is logging puts or gets. An attacker could, for example, look at the
+times of puts, and guess at when git-annex has moved on to
+encrypting/decrypting the next object, and so guess at the approximate
+sizes of objects. (Concurrent uploads/downloads or random delays could be
+added to prevent these kinds of attacks.)
+
+And, obviously, if someone stores 10 tb of data in a remote, they probably
+have around 10 tb of files, so it's probably not a collection of recipes..
+
+Given its inneficiencies and lack of fully obscuring file sizes,
+padding may not be worth adding, but is considered in the designs below.
+
+# design 1
+
+Add an optional chunk field to Key. It is only present for chunk
+2 and above. Ie, SHA256-s12345--xxxxxxx is the first chunk (or whole
+object), while SHA256-s12345-c2--xxxxxxx is the second chunk.
+
+On an encrypted remote, Keys are generated with the chunk field, and then
+HMAC enrypted.
+
+Note that only using it for chunks 2+ means that git-annex can start by
+requesting the regular key, so an observer sees the same request whether
+chunked or not, and does not see eg, a pattern of failed requests for 
+a non-chunked key, followed by successful requests for the chunked keys.
+(Both more efficient and perhaps more secure.)
+
+Problem: This makes putting chunks easy. But there is a problem when getting 
+an object that has been chunked. If the key size is not known, we
+cannot tell when we've gotten the last chunk. (Also, we cannot strip
+padding.) Note that `addurl` sometimes generates keys w/o size info
+(particularly, it does so by design when using quvi).
+
+Problem: Also, this makes `checkPresent` hard to implement: How can it know if
+all the chunks are present, if the key size is not known?
+
+Problem: Also, this makes it difficult to download encrypted keys, because
+we only know the decrypted size, not the encrypted size, so we can't
+be sure how many chunks to get, and all chunks need to be downloaded before
+we can decrypt any of them. (Assuming we encrypt first; chunking first
+avoids this problem.)
+
+Problem: Does not solve concurrent uploads with different chunk sizes.
+
+# design 2
+
+When chunking is enabled, always put a chunk number in the Key,
+along with the chunk size.
+So, SHA256-1048576-c1--xxxxxxx for the first chunk of 1 megabyte.
+
+Before any chunks are stored, write a chunkcount file, eg
+SHA256-s12345-c0--xxxxxxx. Note that this key is the same as the original
+object's key, except with chunk number set to 0. This file contains both
+the number of chunks, and also the chunk size used. `checkPresent` downloads this
+file, and then verifies that each chunk is present, looking for keys with
+the expected chunk numbers and chunk size.
+
+This avoids problems with multiple writers using different chunk sizes,
+since they will be uploading to different files.
+
+Problem: In such a situation, some duplicate data might be stored, not
+referenced by the last chunkcount file to be written. It would not be
+dropped when the key was removed from the remote.
+
+Note: This design lets an attacker with logs tell the (appoximate) size of
+objects, by finding the small files that contain a chunk count, and
+correlating when that is written/read and when other files are
+written/read. That could be solved by padding the chunkcount key up to the
+size of the rest of the keys, but that's very innefficient; `checkPresent` is not
+designed to need to download large files.
+
+# design 3
+
+Like design 1, but add an encrypted chunk count prefix to the first object.
+This needs to be done in a way that does not let an attacker tell if the
+object has an encrypted chunk count prefix or not. 
+
+This seems difficult; attacker could probably tell where the first encrypted
+part stops and the next encrypted part starts by looking for gpg headers,
+and so tell which files are the first chunks.
+
+Also, `checkPresent` would need to download some or all of the first file.
+If all, that's a lot more expensive. If only some is downloaded, an
+attacker can guess that the file that was partially downloaded is the
+first chunk in a series, and wait for a time when it's fully downloaded to
+determine which are the other chunks.
+
+Problem: Two uploads of the same key from repos with different chunk sizes
+could lead to data loss. (Same as in design 2.)
+
+# design 4
+
+Use key SHA256-s12345-S1048576-C1--xxxxxxx for the first chunk of 1 megabyte.
+
+Note that keeping the 's'ize field unchanged is necessary because it 
+disambiguates eg, WORM keys. So a 'S'ize field is used to hold the chunk
+size.
+
+Instead of storing the chunk count in the special remote, store it in 
+the git-annex branch. 
+
+The location log does not record locations of individual chunk keys
+(too space-inneficient). Instead, look at a chunk log in the
+git-annex branch to get the chunk count and size for a key.
+
+`checkPresent` would check if any of the logged sets of chunks is 
+present on the remote. It would also check if the non-chunked key is
+present, as a fallback.
+
+When dropping a key from the remote, drop all logged chunk sizes.
+(Also drop any non-chunked key.)
+
+As long as the location log and the chunk log are committed atomically,
+this guarantees that no orphaned chunks end up on a remote
+(except any that might be left by interrupted uploads).
+
+This has the best security of the designs so far, because the special 
+remote doesn't know anything about chunk sizes. It uses a little more
+data in the git-annex branch, although with care (using the same timestamp
+as the location log), it can compress pretty well.
+
+## chunk log
+
+Stored in the git-annex branch, this provides a mapping `Key -> [[Key]]`.
+
+Note that a given remote uuid might have multiple sets of chunks (with
+different sizes) logged, if a key was stored on it twice using different
+chunk sizes. Also note that even when the log indicates a key is chunked,
+the object may be stored non-chunked on the remote too.
+
+For fixed size chunks, there's no need to store the list of chunk keys,
+instead the log only records the number of chunks (needed because the size
+of the parent Key may not be known), and the chunk size.
+
+Example:
+
+	1287290776.765152s e605dca6-446a-11e0-8b2a-002170d25c55:10240 9
+
+Later, might want to support other kinds of chunks, for example ones made
+using a rsync-style rolling checksum. It would probably not make sense to
+store the full [Key] list for such chunks in the log. Instead, it might be
+stored in a file on the remote.
+
+To support such future developments, when updating the chunk log, 
+git-annex should preserve unparsable values (the part after the colon).
+
+## chunk then encrypt
+
+Rather than encrypting the whole object 1st and then chunking, chunk and
+then encrypt.
+
+Reasons:
+
+1. If 2 repos are uploading the same key to a remote concurrently,
+   this allows some chunks to come from one and some from another,
+   and be reassembled without problems.
+
+2. Also allows chunks of the same object to be downloaded from different
+   remotes, perhaps concurrently, and again be reassembled without
+   problems.
+
+3. Prevents an attacker from re-assembling the chunked file using details
+   of the gpg output. Which would expose approximate
+   file size even if padding is being used to obscure it.
+
+Note that this means that the chunks won't exactly match the configured
+chunk size. gpg does compression, which might make them a
+lot smaller. Or gpg overhead could make them slightly larger. So `checkPresent`
+cannot check exact file sizes.
+
+If padding is enabled, gpg compression should be disabled, to not leak
+clues about how well the files compress and so what kind of file it is.
+
+## chunk key hashing
+
+A chunk key should hash into the same directory structure as its parent
+key. This will avoid lots of extra hash directories when using chunking
+with non-encrypted keys.
+
+Won't happen when the key is encrypted, but that is good; hashing to the
+same bucket then would allow statistical correlation.
+
+## resuming interupted transfers
+
+Resuming interrupted downloads, and uploads are both possible.
+
+Downloads: If the tmp file for a key exists, round it to the chunk size,
+and skip forward to the next needed chunk. Easy.
+
+Uploads: Check if the 1st chunk is present. If so, check the second chunk,
+etc. Once the first missing chunk is found, start uploading from there.
+
+That adds one extra checkPresent call per upload. Probably a win in most cases.
+Can be improved by making special remotes open a persistent
+connection that is used for transferring all chunks, as well as for
+checking checkPresent.
+
+Note that this is safe to do only as long as the Key being transferred
+cannot possibly have 2 different contents in different repos. Notably not
+necessarily the case for the URL keys generated for quvi.
+
+Both **done**.
+
+## parallel
+
+If 2 remotes both support chunking, uploading could upload different chunks
+to them in parallel. However, the chunk log does not currently allow
+representing the state where some chunks are on one remote and others on
+another remote.
+
+Parallel downloading of chunks from different remotes is a bit more doable.
diff --git a/doc/design/assistant/cloud.mdwn b/doc/design/assistant/cloud.mdwn
new file mode 100644
index 0000000000..f0452df2a9
--- /dev/null
+++ b/doc/design/assistant/cloud.mdwn
@@ -0,0 +1,45 @@
+The [[syncing]] design assumes the network is connected. But it's often
+not in these pre-IPV6 days, so the cloud needs to be used to bridge between
+LANS.
+
+## The cloud notification problem (**done**)
+
+Alice and Bob have repos, and there is a cloud remote they both share.
+Alice adds a file; the assistant transfers it to the cloud remote.
+How does Bob find out about it?
+
+There are two parts to this problem. Bob needs to find out that there's
+been a change to Alice's git repo. Then he needs to pull from Alice's git repo,
+or some other repo in the cloud she pushed to. Once both steps are done,
+the assistant will transfer the file from the cloud to Bob.
+
+* dvcs-autosync uses xmppp; all repos need to have the same xmpp account
+  configured, and send self-messages. An alternative would be to have
+  different accounts that join a channel or message each other. Still needs
+  account configuration.
+* irc could be used. With a default irc network, and an agreed-upon channel,
+  no configuration should be needed. IRC might be harder to get through
+  some firewalls, and is prone to netsplits, etc. IRC networks have reasons
+  to be wary of bots using them. Only basic notifications could be done over
+  irc, as it has little security.
+* When there's a ssh server involved, code could be run on it to notify
+  logged-in clients. But this is not a general solution to this problem.
+* pubsubhubbub does not seem like an option; its hubs want to pull down
+  a feed over http.
+
+See [[xmpp]] for design of git-annex's use of xmpp for push notifications.
+
+## storing git repos in the cloud **done for XMPP**
+
+Of course, one option is to just use github etc to store the git repo.
+
+Two things can store git repos in Amazon S3:
+* 
+* 
+
+Another option is to not store the git repo in the cloud, but push/pull
+peer-to-peer. When peers cannot directly talk to one-another, this could be
+bounced through something like XMPP. This is **done** for [[xmpp]]!
+
+Another option: Use  to store
+git repo encrypted on cloud storage.
diff --git a/doc/design/assistant/cloud/comment_1_4997778abc171999499487b71b31c9ba._comment b/doc/design/assistant/cloud/comment_1_4997778abc171999499487b71b31c9ba._comment
new file mode 100644
index 0000000000..1a01afaa33
--- /dev/null
+++ b/doc/design/assistant/cloud/comment_1_4997778abc171999499487b71b31c9ba._comment
@@ -0,0 +1,16 @@
+[[!comment format=mdwn
+ username="https://www.google.com/accounts/o8/id?id=AItOawkq0-zRhubO6kR9f85-5kALszIzxIokTUw"
+ nickname="James"
+ subject="Cloud Service Limitations"
+ date="2012-06-11T02:15:04Z"
+ content="""
+Hey Joey!
+
+I'm not very tech savvy, but here is my question. 
+I think for all cloud service providers, there is an upload limitation on how big one file may be.
+For example, I can't upload a file bigger than 100 MB on box.net. 
+Does this affect git-annex at all? Will git-annex automatically split the file depending on the cloud provider or will I have to create small RAR archives of one large file to upload them?
+
+Thanks!
+James
+"""]]
diff --git a/doc/design/assistant/cloud/comment_2_08da8bc74a4845e354dca99184cffd70._comment b/doc/design/assistant/cloud/comment_2_08da8bc74a4845e354dca99184cffd70._comment
new file mode 100644
index 0000000000..a9b377ea5f
--- /dev/null
+++ b/doc/design/assistant/cloud/comment_2_08da8bc74a4845e354dca99184cffd70._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="http://joeyh.name/"
+ ip="4.153.8.126"
+ subject="re: cloud"
+ date="2012-06-11T04:48:08Z"
+ content="""
+Yes, git-annex has to split files for certian providers. I already added support for this as part of my first pass at supporting box.com, see [[tips/using_box.com_as_a_special_remote]].
+"""]]
diff --git a/doc/design/assistant/cloud/comment_3_faafd1266301997b1822d215ec8e8d8c._comment b/doc/design/assistant/cloud/comment_3_faafd1266301997b1822d215ec8e8d8c._comment
new file mode 100644
index 0000000000..074a3a82c8
--- /dev/null
+++ b/doc/design/assistant/cloud/comment_3_faafd1266301997b1822d215ec8e8d8c._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="https://www.google.com/accounts/o8/id?id=AItOawn7Oyqusvn0oONFtVhCx5gRAcvPjyRMcBI"
+ nickname="Michaël"
+ subject="is ftp an option?"
+ date="2012-05-30T10:44:12Z"
+ content="""
+for people only having ftp-access to there storage.
+"""]]
diff --git a/doc/design/assistant/cloud/comment_4_3eb557d5439831f6e0032944d12c02cf._comment b/doc/design/assistant/cloud/comment_4_3eb557d5439831f6e0032944d12c02cf._comment
new file mode 100644
index 0000000000..e3879a1142
--- /dev/null
+++ b/doc/design/assistant/cloud/comment_4_3eb557d5439831f6e0032944d12c02cf._comment
@@ -0,0 +1,12 @@
+[[!comment format=mdwn
+ username="https://www.google.com/accounts/o8/id?id=AItOawk9XEh8pxrJxZxIkyK7lWaA7QG1UWt9lgU"
+ nickname="Gugelplus"
+ subject="OwnCloud"
+ date="2012-08-27T20:43:19Z"
+ content="""
+“Google drive (attractive because it's free, only 5 gb tho)”
+
+Just in case somebody also wants their 5GB of disk space in the cloud, consider using some of the [owncloud providers](http://owncloud.org/providers/). They also offer that amount, but they use free software for everything, using standard protocols (WebDav mostly, because is well supported in all OS).
+
+Git Annex works with them through davfs2, but it would be great if it could support this other program/protocol (OwnCloud/WebDAV) in a more integrated way.
+"""]]
diff --git a/doc/design/assistant/comment_10_f2233fad55c20686cf299bf6788f1f23._comment b/doc/design/assistant/comment_10_f2233fad55c20686cf299bf6788f1f23._comment
new file mode 100644
index 0000000000..f24357fb64
--- /dev/null
+++ b/doc/design/assistant/comment_10_f2233fad55c20686cf299bf6788f1f23._comment
@@ -0,0 +1,10 @@
+[[!comment format=mdwn
+ username="http://www.klomp.eu/"
+ ip="95.91.241.82"
+ subject="Watch also possible with git?"
+ date="2012-06-15T17:25:30Z"
+ content="""
+Hi,
+
+it seems that you put a lot of efforts in handling race conditions. Thats great. I wonder if the watch can also be used with git (i.e. changes are commited into git and not as annex)? I know that other projects follow this idea but why using different tools if the git-annex assistant could handle both...
+"""]]
diff --git a/doc/design/assistant/comment_11_a38f0f21c2346e65b786d791b6829f9b._comment b/doc/design/assistant/comment_11_a38f0f21c2346e65b786d791b6829f9b._comment
new file mode 100644
index 0000000000..71a9f155d3
--- /dev/null
+++ b/doc/design/assistant/comment_11_a38f0f21c2346e65b786d791b6829f9b._comment
@@ -0,0 +1,12 @@
+[[!comment format=mdwn
+ username="https://www.google.com/accounts/o8/id?id=AItOawneJXwhacIb0YvvdYFxhlNVpz6Wpg6V7AA"
+ nickname="Shayne"
+ subject="comment 11"
+ date="2012-08-13T00:37:35Z"
+ content="""
+Yeah definately go with homebrew rather than macports if possible. macports and fink, whilst great systems, have a tendency to sort of create their own alternative-dimension of files within the system that just dont always feel particularly well integrated. As a result \"brew\" has become increasingly more popular to the point its almost ubuquitous now.
+
+Plus its brew-doctor thing is awesome. 
+
+The best approach though thats agnostic to distro systems is to simply go for a generic installer.
+"""]]
diff --git a/doc/design/assistant/comment_12_5e991177d6577384f39a36ae02f5f574._comment b/doc/design/assistant/comment_12_5e991177d6577384f39a36ae02f5f574._comment
new file mode 100644
index 0000000000..b2de673e3c
--- /dev/null
+++ b/doc/design/assistant/comment_12_5e991177d6577384f39a36ae02f5f574._comment
@@ -0,0 +1,13 @@
+[[!comment format=mdwn
+ username="https://www.google.com/accounts/o8/id?id=AItOawlu-fdXIt_RF9ggvg4zP0yBbtjWQwHAMS4"
+ nickname="Jörn"
+ subject="Multiple annexes?"
+ date="2012-09-20T16:10:29Z"
+ content="""
+Thank you for this great piece of software which is now becoming even better with the assistant.
+
+Just one question: will one instance of the assistant be able to track multiple git annex repositories each building up their own network of annexes OR would I need to run multiple instances of the assistant?
+
+Thanks,
+Jörn
+"""]]
diff --git a/doc/design/assistant/comment_13_f8625c6f43b58847840df338a73b7972._comment b/doc/design/assistant/comment_13_f8625c6f43b58847840df338a73b7972._comment
new file mode 100644
index 0000000000..c121eaf7f1
--- /dev/null
+++ b/doc/design/assistant/comment_13_f8625c6f43b58847840df338a73b7972._comment
@@ -0,0 +1,7 @@
+[[!comment format=mdwn
+ username="http://joeyh.name/"
+ subject="comment 13"
+ date="2012-09-20T16:21:11Z"
+ content="""
+@Jörn, two days ago I added support in the webapp for multiple independent repositories. So you can create independent repos using it, and switch between them from a menu. Under the hood there are multiple git-annex assistant daemons running, but this is an implementation detail; it's not like these use any more memory or other resources than would a single daemon that managed multiple repositories.
+"""]]
diff --git a/doc/design/assistant/comment_14_c37ef5931b0f5c1f808083e0d636a208._comment b/doc/design/assistant/comment_14_c37ef5931b0f5c1f808083e0d636a208._comment
new file mode 100644
index 0000000000..b43741febe
--- /dev/null
+++ b/doc/design/assistant/comment_14_c37ef5931b0f5c1f808083e0d636a208._comment
@@ -0,0 +1,11 @@
+[[!comment format=mdwn
+ username="https://id.koumbit.net/anarcat"
+ subject="you rock! & roadmap update?"
+ date="2012-09-21T04:25:58Z"
+ content="""
+joey, you rock. I just want to push on that point - it doesn't seem like there's that many tools that do what git-annex is trying to do out there, and you seem to be doing an incredible job at doing it, so this is great, keep going!
+
+i was wondering - i am glad to see the progress, but it is unclear to me where you actually are in the roadmap. are things going according to plan? are we at month 3? 4? or 1-4? :) just little updates on that roadmap section above would be quite useful!
+
+thanks again!
+"""]]
diff --git a/doc/design/assistant/comment_15_68c98a27083567f20c2e6bc2a760991b._comment b/doc/design/assistant/comment_15_68c98a27083567f20c2e6bc2a760991b._comment
new file mode 100644
index 0000000000..10b55bbe30
--- /dev/null
+++ b/doc/design/assistant/comment_15_68c98a27083567f20c2e6bc2a760991b._comment
@@ -0,0 +1,9 @@
+[[!comment format=mdwn
+ username="http://joeyh.name/"
+ subject="comment 15"
+ date="2012-09-21T05:25:53Z"
+ content="""
+We're on month 4 of work, and most of months 1-3 is done well enough for a first pass. Very little of what's listed in month 4 has happened yet, due to my being maybe 2 weeks behind schedule, but bits of [[cloud]] are being planned.
+
+I've made a small adjustment, I think it'll make sense to spend a month on user-driven features before getting into Android.
+"""]]
diff --git a/doc/design/assistant/comment_16_8e6788c817c60371d2a2f158e1a65f87._comment b/doc/design/assistant/comment_16_8e6788c817c60371d2a2f158e1a65f87._comment
new file mode 100644
index 0000000000..de5e5b0785
--- /dev/null
+++ b/doc/design/assistant/comment_16_8e6788c817c60371d2a2f158e1a65f87._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="https://launchpad.net/~gdr-go2"
+ nickname="gdr-go2"
+ subject="Maybe a DEB?2"
+ date="2012-09-27T09:44:14Z"
+ content="""
+Month 3 was all about easy setup, so I kind of expected to download a deb package and just install it, not to download a whole bunch of haskell libraries. Is there a chance that you will release some packages?
+"""]]
diff --git a/doc/design/assistant/comment_17_97bdfacac5ac492281c9454ee4c0228e._comment b/doc/design/assistant/comment_17_97bdfacac5ac492281c9454ee4c0228e._comment
new file mode 100644
index 0000000000..ea3f50e611
--- /dev/null
+++ b/doc/design/assistant/comment_17_97bdfacac5ac492281c9454ee4c0228e._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="http://joeyh.name/"
+ ip="4.153.14.141"
+ subject="comment 17"
+ date="2012-09-27T18:44:11Z"
+ content="""
+@gdr A package with the assistant is available in Debian unstable.
+"""]]
diff --git a/doc/design/assistant/comment_18_53137b2df4913496c0afb2d895aa4ee2._comment b/doc/design/assistant/comment_18_53137b2df4913496c0afb2d895aa4ee2._comment
new file mode 100644
index 0000000000..672c2a9d75
--- /dev/null
+++ b/doc/design/assistant/comment_18_53137b2df4913496c0afb2d895aa4ee2._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="https://www.google.com/accounts/o8/id?id=AItOawm7AuSfii_tCkLyspL6Mr0ATlO6OxLNYOo"
+ nickname="Georg"
+ subject="for OSX, package managers (homebrew and macports) are really second-class"
+ date="2013-07-24T07:24:49Z"
+ content="""
+at least with regards to wide spread use. I agree with Wichert, the average OSX user won't use them. I myself would go with homebrew (because it is far less of a pain than macports), but actually static built binaries that are just dragged from a DMG to your programs folder is the way to go if you can't go osx app store. It's just how most Apple users are wired. PKG would be fine, too, since most will know them, but those are really only needed if you need to place stuff in /Library/Frameworks or things like that. If it can be done as a all-inclusive OSX .app, do it that way and just pack it up in a .DMG.
+"""]]
diff --git a/doc/design/assistant/comment_19_ff1b0ba57e22ed757ec3fc5400b5e43e._comment b/doc/design/assistant/comment_19_ff1b0ba57e22ed757ec3fc5400b5e43e._comment
new file mode 100644
index 0000000000..1ef6136b90
--- /dev/null
+++ b/doc/design/assistant/comment_19_ff1b0ba57e22ed757ec3fc5400b5e43e._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="https://www.google.com/accounts/o8/id?id=AItOawlnoH5btjn_BLib3_IhES5uMhrzuOiwCYo"
+ nickname="András"
+ subject="windows port"
+ date="2013-07-25T07:50:19Z"
+ content="""
+I'd love to use the Windows port of the assistant by Christmas. Keep up the good work! :)
+"""]]
diff --git a/doc/design/assistant/comment_1_a48fcfbf97f0a373ea375cd8f07f0fc8._comment b/doc/design/assistant/comment_1_a48fcfbf97f0a373ea375cd8f07f0fc8._comment
new file mode 100644
index 0000000000..646a03398a
--- /dev/null
+++ b/doc/design/assistant/comment_1_a48fcfbf97f0a373ea375cd8f07f0fc8._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="https://www.google.com/accounts/o8/id?id=AItOawkSq2FDpK2n66QRUxtqqdbyDuwgbQmUWus"
+ nickname="Jimmy"
+ subject="comment 1"
+ date="2012-06-02T12:06:37Z"
+ content="""
+Will statically linked binaries be provided for say Linux, OSX and *BSD?  I think having some statically linked binaries will certainly help and appeal to a lot of users.
+"""]]
diff --git a/doc/design/assistant/comment_20_099da245e3276fa84f5e14312d186621._comment b/doc/design/assistant/comment_20_099da245e3276fa84f5e14312d186621._comment
new file mode 100644
index 0000000000..f423c3e10d
--- /dev/null
+++ b/doc/design/assistant/comment_20_099da245e3276fa84f5e14312d186621._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="http://joeyh.name/"
+ ip="2001:4978:f:21a::2"
+ subject="comment 20"
+ date="2013-07-25T18:22:51Z"
+ content="""
+@Georg, DMG have been built for OSX for a long time. See [[install/OSX]]
+"""]]
diff --git a/doc/design/assistant/comment_2_6d3552414fdcc2ed3244567e6c67989d._comment b/doc/design/assistant/comment_2_6d3552414fdcc2ed3244567e6c67989d._comment
new file mode 100644
index 0000000000..8056eec168
--- /dev/null
+++ b/doc/design/assistant/comment_2_6d3552414fdcc2ed3244567e6c67989d._comment
@@ -0,0 +1,7 @@
+[[!comment format=mdwn
+ username="http://joeyh.name/"
+ subject="comment 2"
+ date="2012-06-04T19:45:00Z"
+ content="""
+Jimmy, I hope to make it as easy as possible to install. I've been focusing on getting it directly into popular Linux distributions, rather than shipping my own binary. The OSX binary is static, and while I lack a OSX machine, I would like to get it easier to distribute to OSX users.
+"""]]
diff --git a/doc/design/assistant/comment_3_05223be50c889b2ed6bc4abf74116450._comment b/doc/design/assistant/comment_3_05223be50c889b2ed6bc4abf74116450._comment
new file mode 100644
index 0000000000..a78fa33439
--- /dev/null
+++ b/doc/design/assistant/comment_3_05223be50c889b2ed6bc4abf74116450._comment
@@ -0,0 +1,9 @@
+[[!comment format=mdwn
+ username="https://www.google.com/accounts/o8/id?id=AItOawkSq2FDpK2n66QRUxtqqdbyDuwgbQmUWus"
+ nickname="Jimmy"
+ subject="comment 3"
+ date="2012-06-07T20:22:55Z"
+ content="""
+I'd agree getting it into the main distros is the way to go, if you need OSX binaries, I could volunteer to setup an autobuilder to generate binaries for OSX users, however it would rely on users to have macports with the correct ports installed to use it (things like coreutils etc...)
+
+"""]]
diff --git a/doc/design/assistant/comment_4_fbbd93b55803ae21e6ba4b6568c2fafd._comment b/doc/design/assistant/comment_4_fbbd93b55803ae21e6ba4b6568c2fafd._comment
new file mode 100644
index 0000000000..cd3b5aaef7
--- /dev/null
+++ b/doc/design/assistant/comment_4_fbbd93b55803ae21e6ba4b6568c2fafd._comment
@@ -0,0 +1,9 @@
+[[!comment format=mdwn
+ username="http://joeyh.name/"
+ subject="comment 4"
+ date="2012-06-08T01:56:52Z"
+ content="""
+I always appreciate your OSX work Jimmy...
+
+Could it be put into macports?
+"""]]
diff --git a/doc/design/assistant/comment_5_f4e9af3fed6c27e8ff39badb9794064d._comment b/doc/design/assistant/comment_5_f4e9af3fed6c27e8ff39badb9794064d._comment
new file mode 100644
index 0000000000..bf8d9709e8
--- /dev/null
+++ b/doc/design/assistant/comment_5_f4e9af3fed6c27e8ff39badb9794064d._comment
@@ -0,0 +1,12 @@
+[[!comment format=mdwn
+ username="https://www.google.com/accounts/o8/id?id=AItOawkSq2FDpK2n66QRUxtqqdbyDuwgbQmUWus"
+ nickname="Jimmy"
+ subject="comment 5"
+ date="2012-06-08T07:22:34Z"
+ content="""
+In relation to macports, I often found that haskell in macports are often behind other distros, and I'm not willing to put much effort into maintaining or updating those ports. I found that to build git-annex, installing macports manually and then installing haskell-platform from the upstream to be the best way to get the most up to date dependancies for git-annex.
+
+fyi in macports ghc is at version 6.10.4 and haskell platform is at version 2009.2, so there are a significant number of ports to update.
+
+I was thinking about this a bit more and I reckon it might be easier to try and build a self contained .pkg package and have all the needed binaries in a .app styled package, that would work well when the webapp comes along. I will take a look at it in a week or two (currently moving house so I dont have much time)
+"""]]
diff --git a/doc/design/assistant/comment_6_c7ad07cade1f44f9a8b61f92225bb9c5._comment b/doc/design/assistant/comment_6_c7ad07cade1f44f9a8b61f92225bb9c5._comment
new file mode 100644
index 0000000000..9fa66d6d31
--- /dev/null
+++ b/doc/design/assistant/comment_6_c7ad07cade1f44f9a8b61f92225bb9c5._comment
@@ -0,0 +1,10 @@
+[[!comment format=mdwn
+ username="https://www.google.com/accounts/o8/id?id=AItOawkSq2FDpK2n66QRUxtqqdbyDuwgbQmUWus"
+ nickname="Jimmy"
+ subject="comment 6"
+ date="2012-06-08T15:21:18Z"
+ content="""
+It's not much for now... but see  I'm ignoring the debian-stable and pristine-tar branches for now, as I am just building and testing on osx 10.7.
+
+Hope the autobuilder will help you develop the OSX side of things without having direct access to an osx machine! I will try and get gitbuilder to spit out appropriately named tarballs of the compiled binaries in a few days when I have more time.
+"""]]
diff --git a/doc/design/assistant/comment_7_609d38e993267195a80fecd84c93d1e2._comment b/doc/design/assistant/comment_7_609d38e993267195a80fecd84c93d1e2._comment
new file mode 100644
index 0000000000..6685c6548e
--- /dev/null
+++ b/doc/design/assistant/comment_7_609d38e993267195a80fecd84c93d1e2._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="http://joeyh.name/"
+ ip="4.153.8.126"
+ subject="comment 7"
+ date="2012-06-09T18:07:51Z"
+ content="""
+Thanks, that's already been useful to me. You might as well skip the debian-specific \"bpo\" tags too.
+"""]]
diff --git a/doc/design/assistant/comment_8_22b818e1a2a825efb78139271a14f944._comment b/doc/design/assistant/comment_8_22b818e1a2a825efb78139271a14f944._comment
new file mode 100644
index 0000000000..57f354e494
--- /dev/null
+++ b/doc/design/assistant/comment_8_22b818e1a2a825efb78139271a14f944._comment
@@ -0,0 +1,10 @@
+[[!comment format=mdwn
+ username="https://www.google.com/accounts/o8/id?id=AItOawldKnauegZulM7X6JoHJs7Gd5PnDjcgx-E"
+ nickname="Matt"
+ subject="Homebrew instead of MacPorts"
+ date="2012-06-22T04:26:02Z"
+ content="""
+[Homebrew] is a much better package manager than MacPorts IMO.
+
+[Homebrew]: http://mxcl.github.com/homebrew/
+"""]]
diff --git a/doc/design/assistant/comment_9_d052e2142da8b4838fb1edf791ea23ae._comment b/doc/design/assistant/comment_9_d052e2142da8b4838fb1edf791ea23ae._comment
new file mode 100644
index 0000000000..5e955c2b6c
--- /dev/null
+++ b/doc/design/assistant/comment_9_d052e2142da8b4838fb1edf791ea23ae._comment
@@ -0,0 +1,10 @@
+[[!comment format=mdwn
+ username="http://wiggy.net/"
+ nickname="Wichert"
+ subject="macports"
+ date="2012-06-12T13:00:34Z"
+ content="""
+The average OSX user has a) no idea what macports is, and b) will not be able to install it. Anything that requires a user to do anything with a commandline (or really anything other than using a GUI installer) is effectively a dealbreaker. For our use cases OSX is definitely a requirement, but it must only use standard OSX installation methods in order to be usable. Being in the appstore would be ideal, but standard dmg/pkg installers are still common enough that they are also acceptable.
+
+FWIW this is the same reason many git GUIs were not usable for our OSX users: they required separate installation of the git commandline tools.
+"""]]
diff --git a/doc/design/assistant/configurators.mdwn b/doc/design/assistant/configurators.mdwn
new file mode 100644
index 0000000000..4d08ac54a9
--- /dev/null
+++ b/doc/design/assistant/configurators.mdwn
@@ -0,0 +1,20 @@
+Add to the [[webapp]] some configuration of git-annex.
+
+There are some basic settings that pass through to `git config`, things
+like how much disk space to leave free, how many copies to ensure are kept
+of files, etc.
+
+The meat of the configuration will be in configuration assistants that walk
+through setting up common use cases.
+
+* Create a repository (run when the web app is started without a configured
+  repository too). **done**
+* Clone this repo to a USB drive or other removable drive. **done**
+* Make a bare repo on a remote ssh server **done**
+* Clone this repo to another host. (Needs [[pairing]]) **done**
+* Set up Amazon S3. **done**
+* Set up encrypted rsync remote. **done**
+* Rsync.net special case **done**
+* Set up gpg encryption key; gpg key distribution.
+* I lost my USB drive!
+* etc -- many more possibilities
diff --git a/doc/design/assistant/deltas.mdwn b/doc/design/assistant/deltas.mdwn
new file mode 100644
index 0000000000..0f7d308b81
--- /dev/null
+++ b/doc/design/assistant/deltas.mdwn
@@ -0,0 +1,27 @@
+Speed up syncing of modified versions of existing files. 
+
+One simple way is to find the key of the old version of a file that's
+being transferred, so it can be used as the basis for rsync, or any
+other similar transfer protocol.
+
+For remotes that don't use rsync, use a rolling checksum based chunker,
+such as BuzHash. This will produce [[chunks]], which can be stored on the
+remote as regular Keys -- where unlike the fixed size chunk keys, the
+SHA256 part of these keys is the checksum of the chunk they contain.
+
+Once that's done, it's easy to avoid uploading chunks that have been sent
+to the remote before.
+
+When retriving a new version of a file, there would need to be a way to get
+the list of chunk keys that constitute the new version. Probably best to
+store this list on the remote. Then there needs to be a way to find which
+of those chunks are available in locally present files, so that the locally
+available chunks can be extracted, and combined with the chunks that need
+to be downloaded, to reconstitute the file.
+
+To find which chucks are locally available, here are 2 ideas:
+
+1. Use a single basis file, eg an old version of the file. Re-chunk it, and
+   use its chunks.  Slow, but simple.
+2. Some kind of database of locally available chunks. Would need to be kept
+   up-to-date as files are added, and as files are downloaded.
diff --git a/doc/design/assistant/deltas/comment_1_bdb477af913c9782c0e8509e6b294b6e._comment b/doc/design/assistant/deltas/comment_1_bdb477af913c9782c0e8509e6b294b6e._comment
new file mode 100644
index 0000000000..3361de4bf1
--- /dev/null
+++ b/doc/design/assistant/deltas/comment_1_bdb477af913c9782c0e8509e6b294b6e._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="https://www.google.com/accounts/o8/id?id=AItOawkfHTPsiAcHEEN7Xl7WxiZmYq-vX7azxFY"
+ nickname="Vincent"
+ subject="zsync?"
+ date="2012-09-10T12:35:45Z"
+ content="""
+zsync.moria.org.uk may have broken some of the ground here.
+"""]]
diff --git a/doc/design/assistant/deltas/comment_2_71889d15ba20ebb0fe13080c68162a5b._comment b/doc/design/assistant/deltas/comment_2_71889d15ba20ebb0fe13080c68162a5b._comment
new file mode 100644
index 0000000000..3ef911b776
--- /dev/null
+++ b/doc/design/assistant/deltas/comment_2_71889d15ba20ebb0fe13080c68162a5b._comment
@@ -0,0 +1,11 @@
+[[!comment format=mdwn
+ username="https://www.google.com/accounts/o8/id?id=AItOawm72W-CH7bzZ8uHvaw1KJGrToPSQDNBnIA"
+ nickname="Beni"
+ subject="bup splitting"
+ date="2013-08-05T17:13:14Z"
+ content="""
+bup splits files into chunks based on rolling sums.
+https://github.com/apenwarr/bup/blob/master/DESIGN
+For small changes to big files, this would improve not only transfer speed but also repository size.
+And git-annex already supports bup remotes.  What's missing?
+"""]]
diff --git a/doc/design/assistant/desymlink.mdwn b/doc/design/assistant/desymlink.mdwn
new file mode 100644
index 0000000000..ffa37399dd
--- /dev/null
+++ b/doc/design/assistant/desymlink.mdwn
@@ -0,0 +1,145 @@
+While dropbox allows modifying files in the folder, git-annex freezes
+them upon creation, using symlinks.
+
+This is a core design simplification of git-annex.
+But it is problematic too:
+
+* To allow directly editing files in its folder, something like [[todo/smudge]] is
+  needed, to get rid of the symlinks that stand in for the files.
+* OSX seems to have a [[lot_of_problems|bugs/OSX_alias_permissions_and_versions_problem]]
+  with stupid programs that follow symlinks and present the git-annex
+  hash filename to the user.
+* FAT sucks and doesn't support symlinks at all, so [[Android]] can't
+  have regular repos on it.
+
+One approach for this would be to hide the git repo away somewhere,
+and have the git-annex assistant watch a regular directory, with
+regular files.
+
+There would have to be a mapping from files to git-annex objects.
+And some intelligent way to determine when a file has been changed
+and no longer corresponds to its object. (Not expensive hashing every time,
+plz.)
+
+Since this leaves every file open to modification, any such repository
+probably needs to be considered untrusted by git-annex. So it doesn't
+leave its only copy of a file in such a repository, but instead
+syncs it to a proper git-annex repository.
+
+The assistant would make git commits still, of symlinks. It can already do
+that with without actual symlinks existing on disk. More difficult is
+handling merging; git merge wants a real repository with files it can
+really operate on. The assistant would need to calculate merges on its own,
+and update the regular directory to reflect changes made in the merge.
+
+Another massive problem with this idea is that it doesn't allow for
+[[partial_content]]. The symlinks that everyone loves to hate on are what
+make it possible for the contents of some files to not be present on
+disk, while the files are still in git and can be retreived as desired.
+With real files, some other standin for a missing file would be needed.
+Perhaps a 0 length, unreadable, unwritable file? On systems that
+support symlinks it could be a broken symlink like is used now, that
+is converted to a real file when it becomes present.
+
+## concrete design
+
+* Enable with annex.direct
+* Use .git/ for the git repo, but `.git/annex/objects` won't be used
+  for object storage.
+* `git status` and similar will show all files as type changed, and
+  `git commit` would be a very bad idea. Just don't support users running
+  git commands that affect the repository in this mode. Probably.
+* However, `git status` and similar also will show deleted and new files,
+  which will be helpful for the assistant to use when starting up.
+* Cache the mtime, size etc of files, and use this to detect when they've been
+  modified when the assistant was not running. This would only need to be
+  checked at startup, probably.
+* Use dangling symlinks for standins for missing content, as now.
+  This allows just cloning one of these repositories normally, and then
+  as the files are synced in, they become real files.
+* Maintain a local mapping from keys to files in the tree. This is needed
+  when sending/receiving/dropping keys to know what file to access.
+  Note that a key can map to multiple files. And that when a file is
+  deleted or moved, the mapping needs to be updated.
+* May need a reverse mapping, from files in the tree to keys? TBD
+  (Currently, getting by looking up symlinks using `git cat-file`)
+  (Needed to make things like `git annex drop` that want to map from the
+  file back to the key work.)
+* The existing watch code detects when a file gets closed, and in this
+  mode, it could be a new file, or a modified file, or an unchanged file.
+  For a modified file, can compare mtime, size, etc, to see if it needs
+  to be re-added.
+* The inotify/kqueue interface does not tell us when a file is renamed.
+  So a rename has to be treated as a delete and an add, so can have a lot
+  of overhead, to re-hash the file.
+* Note that this could be used without the assistant, as a git remote
+  that content is transferred to and from. Without the assistant, changes
+  to files in this remote would not be noticed and committed, unless
+  a git-annex command were added to do so.
+  Getting it basically working as a remote would be a good 1st step.
+* It could also be used without the assistant as a repository that
+  the user uses directly. Would need some git-annex commands
+  to merge changes into the repo, update caches, and commit changes.
+  This could all be done by "git annex sync".
+
+## TODO
+
+* kqueue does not deliver an event when an existing file is modified.
+  This doesn't affect OSX, which uses FSEvents now, but it makes direct
+  mode assistant not 100% on other BSD's.
+
+## done
+
+* `git annex sync` updates the key to files mappings for files changed,
+  but needs much other work to handle direct mode:
+  * Generate git commit, without running `git commit`, because it will
+    want to stage the full files. **done**
+  * Update location logs for any files deleted by a commit. **done**
+  * Generate a git merge, without running `git merge` (or possibly running
+    it in a scratch repo?), because it will stumble over the direct files.
+    **done**
+  * Drop contents of files deleted by a merge (including updating the
+    location log), or if we cannot drop,
+    move their contents to `.git/annex/objects/`.  **no**  .. instead, 
+    avoid ever losing file contents in a direct mode merge. If the file is
+    deleted, its content is moved back to .git/annex/objects, if necessary.
+  * When a merge adds a symlink pointing at a key that is present in the
+    repo, replace the symlink with the direct file (either moving out
+    of `.git/annex/objects/` or hard-linking if the same key is present
+    elsewhere in the tree. **done**
+  * handle merge conflicts on direct mode files **done**
+* support direct mode in the assistant (many little fixes)
+
+* Deal with files changing as they're being transferred from a direct mode
+  repository to another git repository. The remote repo currently will 
+  accept the bad data and update the location log to say it has the key.
+
+  This affects both special remotes and git remotes. 
+
+  For special remotes,
+  it seems the best that could be done is to have an error unwind action
+  passed to `sendAnnex` that is called if the file is modified as it's
+  transferred. That would then remove the probably corrupted file from the
+  remote. (The full transfer would still run, unless there was also a way
+  to cancel an in progress transfer.) **done**
+
+  (With the above, there is some potential for the bad content being
+  downloaded from the special remote into another repo. This would only
+  happen if the other repo for some reason thinks the special remote 
+  has the content. Since the location log would not be updated until the
+  transfer is successful, this should not happen.)
+
+  For local git remotes, need to check the direct mode file after it's
+  copied and before it's put into place as a key's content. **done**
+  (untested)
+
+  `git-annex-shell sendkey` needs to do something if it sent bad
+  data. This seems to not need protocol changes; it can just detect 
+  the problem and exit nonzero. Would need to do something to clean up
+  the temp file, which is probably corrupt. (Could in future use it as a
+  basis for transferring the new key..) **done**
+
+  For git remotes, added a flag to `git-annex-shell recvkey` (using a field
+  after the "--" to remain back-compat). With this flag, after receiving
+  the data, the remote fscks the data. This is not optimal, but avoids
+  needing another round-trip, or a protocol change.
diff --git a/doc/design/assistant/desymlink/comment_1_f1bfe250b7f872359f7075998b6e42e3._comment b/doc/design/assistant/desymlink/comment_1_f1bfe250b7f872359f7075998b6e42e3._comment
new file mode 100644
index 0000000000..5e60ff29e0
--- /dev/null
+++ b/doc/design/assistant/desymlink/comment_1_f1bfe250b7f872359f7075998b6e42e3._comment
@@ -0,0 +1,11 @@
+[[!comment format=mdwn
+ username="https://www.google.com/accounts/o8/id?id=AItOawn-KDr_Z4CMkjS0v_TxQ08SzAB5ecHG3K0"
+ nickname="Glen"
+ subject="Modifying files in a folder watched by assistant"
+ date="2012-09-19T14:18:54Z"
+ content="""
+I think this feature is really important.  For my use cases, if I can't modify files that are in the folder that is kept in sync, then I would just use rsync or manually keep copies on multiple systems..
+
+I might be missing something, but tried a git annex unlock on the file, but if the assistant is running, it seems to revert this back.
+
+"""]]
diff --git a/doc/design/assistant/desymlink/comment_2_5e876edfe9853645f761b5ed9b5021aa._comment b/doc/design/assistant/desymlink/comment_2_5e876edfe9853645f761b5ed9b5021aa._comment
new file mode 100644
index 0000000000..b79ae4781a
--- /dev/null
+++ b/doc/design/assistant/desymlink/comment_2_5e876edfe9853645f761b5ed9b5021aa._comment
@@ -0,0 +1,9 @@
+[[!comment format=mdwn
+ username="https://www.google.com/accounts/o8/id?id=AItOawknpbpx4uDElXRzJ4kPXuJU1KdcslOI6go"
+ nickname="Rodrigo"
+ subject="I strongly agree with Glen"
+ date="2012-10-03T14:07:40Z"
+ content="""
+And if your goal is to build an app \"like dropbox\" it is a must have feature.
+Thanks for your work. You are doing great!
+"""]]
diff --git a/doc/design/assistant/desymlink/comment_3_538561d74371e53c2f8df7f5ebdf58a8._comment b/doc/design/assistant/desymlink/comment_3_538561d74371e53c2f8df7f5ebdf58a8._comment
new file mode 100644
index 0000000000..40067b25bb
--- /dev/null
+++ b/doc/design/assistant/desymlink/comment_3_538561d74371e53c2f8df7f5ebdf58a8._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="http://joeyh.name/"
+ ip="4.154.0.149"
+ subject="comment 3"
+ date="2012-10-05T00:43:11Z"
+ content="""
+@Glen, for what it's worth, you can now manually \"git annex unlock\" a file and it'll stay unlocked until you manually \"git annex add\" the new version.
+"""]]
diff --git a/doc/design/assistant/desymlink/comment_4_586ecaa800e6c162377c937da5e65440._comment b/doc/design/assistant/desymlink/comment_4_586ecaa800e6c162377c937da5e65440._comment
new file mode 100644
index 0000000000..aeb1587edc
--- /dev/null
+++ b/doc/design/assistant/desymlink/comment_4_586ecaa800e6c162377c937da5e65440._comment
@@ -0,0 +1,12 @@
+[[!comment format=mdwn
+ username="https://www.google.com/accounts/o8/id?id=AItOawlJEI45rGczFAnuM7gRSj4C6s9AS9yPZDc"
+ nickname="Kevin"
+ subject="unlock not behaving as you describe"
+ date="2012-11-05T22:35:35Z"
+ content="""
+@joeyh,
+
+I'm not seeing the behavior you describe re \"git annex unlock\" when using the assistant.  After unlocking and writing the file, git annex assistant commits the changes and recreates the symlink from under me which is annoying.
+
+I'm running OS X 10.7.5 using the bundled git-annex.app version 3.20121017.
+"""]]
diff --git a/doc/design/assistant/desymlink/comment_5_8fc703de67814cf2aec2a908852298a4._comment b/doc/design/assistant/desymlink/comment_5_8fc703de67814cf2aec2a908852298a4._comment
new file mode 100644
index 0000000000..ca43615326
--- /dev/null
+++ b/doc/design/assistant/desymlink/comment_5_8fc703de67814cf2aec2a908852298a4._comment
@@ -0,0 +1,10 @@
+[[!comment format=mdwn
+ username="http://joeyh.name/"
+ ip="4.152.108.27"
+ subject="comment 5"
+ date="2012-11-05T23:40:50Z"
+ content="""
+Well, you're able to unlock and modify the file -- that's what I fixed.
+
+Committing changes to files is what the assistant does..
+"""]]
diff --git a/doc/design/assistant/desymlink/comment_6_1b473ad89494afb82250af4b6df5f5c9._comment b/doc/design/assistant/desymlink/comment_6_1b473ad89494afb82250af4b6df5f5c9._comment
new file mode 100644
index 0000000000..fad9cbe984
--- /dev/null
+++ b/doc/design/assistant/desymlink/comment_6_1b473ad89494afb82250af4b6df5f5c9._comment
@@ -0,0 +1,22 @@
+[[!comment format=mdwn
+ username="https://www.google.com/accounts/o8/id?id=AItOawm5iosFbL2By7UFeViqkc6v-hoAtqILeDA"
+ nickname="Laszlo"
+ subject="comment 6"
+ date="2012-12-03T05:33:44Z"
+ content="""
+> One approach for this would be to hide the git repo away somewhere, and have the git-annex assistant watch a regular directory, with regular files.
+
+I think this would be the single most important feature added to git-annex.
+1. It could be used to backup regularly a smb share over the network.
+2. For those who dont trust a program who modifies their files directly. 
+(I had a 3GB permanent data loss, when trying to push a 10GB directory into git annex.)
+3. Android 
+
+I do hope the above method will be implemented for Android too, so its like two birds with one stone. 
+(Im really for option 2 here. I dont like the very idea of modifying my files directly. 
+Git does not do it either, I have disk space (if not I buy more disks), rather then risquing losing important datas)
+
+Best, 
+ Laszlo
+
+"""]]
diff --git a/doc/design/assistant/disaster_recovery.mdwn b/doc/design/assistant/disaster_recovery.mdwn
new file mode 100644
index 0000000000..6fcf955195
--- /dev/null
+++ b/doc/design/assistant/disaster_recovery.mdwn
@@ -0,0 +1,185 @@
+The assistant should help the user recover their repository when things go
+wrong.
+
+[[!toc ]]
+
+## dangling lock files
+
+There are a few ways a git repository can get broken that are easily fixed.
+One is left over index.lck files. When a commit to a repository fails,
+check that nothing else is using it, fix the problem, and redo the commit.
+
+* **done** for .git/annex/index.lock, can be handled safely and automatically.
+* **done** for .git/index.lock, only when the assistant is starting up.
+* What about local remotes, eg removable drives? git-annex does attempt
+  to commit to the git-annex branch of those. It will use the automatic
+  fix if any are dangling. It does not commit to the master branch; indeed
+  a removable drive typically has a bare repository.
+  However, it does a scan for broken locks anyway if there's a problem
+  syncing. **done**
+* What about git-annex-shell? If the ssh remote has the assistant running,
+  it can take care of it, and if not, it's a server, and perhaps the user
+  should be required to fix up if it crashes during a commit. This should
+  not affect the assistant anyway.
+* **done** Seems that refs can also have stale lock files, for example
+  '/storage/emulated/legacy/DCIM/.git/refs/remotes/flick_phonecamera/synced/git-annex.lock'
+  All git lock files are now handled (except gc lock files).
+
+## incremental fsck
+
+Add webapp UI to enable incremental fsck **done**
+
+Of course, incremental fsck will run as an niced (and ioniced) background
+job. There will need to be a button in the webapp to stop it, in case it's
+annoying. **done**
+
+When fsck finds a damanged file, queue a download of the file from a
+remote. **done**
+
+Detect when a removable drive is connected in the Cronner, and check
+and try to run its remote fsck jobs. **done** (Same mechanism will work for
+network remotes becoming connected.)
+
+TODO: If no accessible remote has a file that fsck reported missing,
+prompt the user to eg, connect a drive containing it. Or perhaps this is a
+special case of a general problem, and the webapp should prompt the user
+when any desired file is available on a remote that's not mounted?
+
+## git-annex-shell remote fsck
+
+TODO: git-annex-shell fsck support, which would allow cheap fast fscks
+of ssh remotes.
+
+Would be nice; otherwise remote fsck is too expensive (downloads
+everything) to have the assistant do.
+
+Note that Remote.Git already tries to use this, but the assistant does not
+call it for non-local remotes.
+
+## git fsck and repair
+
+Add git fsck to scheduled self fsck **done**
+
+TODO: git fsck on ssh remotes? Probably not worth the complexity..
+
+TODO: If committing to the repository fails, after resolving any dangling
+lock files (see above), it should git fsck. This is difficult, because
+git commit will also fail if the commit turns out to be empty, or due to
+other transient problems.. So commit failures are currently ignored by the
+assistant.
+
+If git fsck finds problems, launch git repository repair. **done**
+
+git annex fsck --fast at end of repository repair to ensure
+git-annex branch is accurate. **done**
+
+If syncing with a local repository fails, try to repair it. **done**
+
+TODO: "Repair" gcrypt remotes, by removing all refs and objects,
+and re-pushing. (Since the objects are encrypted data, there is no way
+to pull missing ones from anywhere..) 
+Need to preserve gcrypt-id while doing this!
+
+TODO: along with displaying alert when there is a problem detected
+by consistency check, send an email alert. (Using system MTA?)
+
+## nudge user to schedule fscks
+
+Make the webapp encourage users to schedule fscks of their
+local repositories. The goal here was that it should not be obnoxious about
+repeatedly pestering the user to set that up, but should still encourage
+anyone who cares to set it up.
+
+Maybe: Display a message only once per week, and only after the repository
+has existed for at least one full day. But, this will require storing
+quite a lot of state.
+
+Or: Display a message whenever a removable drive is detected to have been
+connected. I like this, but what about nudging the main repo? Could do it
+every webapp startup, perhaps? **done**
+
+There should be a "No thanks" button that prevents it nudging again for a
+repo. **done**
+
+## git repository repair
+
+There are several ways git repositories can get damanged. 
+
+The most common is empty files in .git/annex/objects and commits that refer
+to those objects. When the objects have not yet been pushed anywhere.
+I've several times recovered from this manually by
+removing the bad files and resetting to before the commits that referred to
+them. Then re-staging any divergence in the working tree. This could
+perhaps be automated.
+
+As long as the git repository has at least one remote, another method is to
+clone the remote, sync from all other remotes, move over .git/config and
+.git/annex/objects, and tar up the old broken git repo and `git annex add`
+it. This should be automatable and get the user back on their feet. User
+could just click a button and have this be done.
+
+This is useful outside git-annex as well, so make it a
+git-recover-repository command.
+
+### detailed design
+
+Run `git fsck` and parse output to find bad objects. **done** Note that
+fsck may fall over and fail to print out all bad objects, when
+files are corrupt. So if the fsck exits nonzero, need to collect all
+bad objects it did find, and:
+
+1. If the local repository contains packs, the packs may be corrupt.
+   So, start by using `git unpack-objects` to unpack all
+   packs it can handle (which may include parts of corrupt packs) 
+   back to loose objects. And delete all packs. **done**
+2. Delete all loose corrupt objects. **done**
+
+Repeat until fsck finds no new problems. **done**
+
+Check if there's a remote. If so, and if the bad objects are all
+present on it, can simply get all bad objects from the remote,
+and inject them back into .git/objects to recover:
+
+3. Make a new (bare) clone from the remote.
+   (Note: git does not seem to provide a way to fetch specific missing
+   objects from the remote. Also, cannot use `--reference` against
+   a repository with missing refs. So this seems unavoidably
+   network-expensive.) **done**
+5. Rsync objects over. (Turned out to work better than git-cat-file, 
+   because we don't have to walk the graph to add missing objects.)
+   **done**
+6. If each bad object was able to be repaired this way, we're done!
+   (If not, can reuse the clone for getting objects from the next remote.)
+   **done**
+
+If some missing objects cannot be recovered from remotes, find commits in each
+local branch that are broken by all remaining missing objects. Some of this can
+be parsed from git fsck output, but for eg blobs, the commits need to
+be walked to walk the trees, to find trees that refer to the blobs. **done**
+
+For each branch that is affected, look in the reflog and/or `git log
+$branch` to find the last good commit that predates all broken commits. (If
+the head commit of a branch is broken, git log is not going to show
+anything useful, but the reflog can be used to find past refs for the
+branch -- have to first delete the .git/HEAD file if it points to the
+broken ref.) **done**
+
+The basic idea then is to reset the branch to the last good commit
+that was found for it.
+
+* For the HEAD branch, can just reset it. (If no last good commit was found
+  for the HEAD branch, reset it to a dummy empty commit.) This will
+  leave git showing any changes made since then as staged in the index and
+  uncommitted. Or if the index is missing/corrupt, any files in the tree will
+  show as modified and uncommitted. User (or git-annex assistant) can then
+  commit as appropriate. Print appropriate warning message. **done**
+* Special handling for git-annex branch and index. **done**
+* Remote tracking branches can just be removed, and then `git fetch`
+  from the remote, which will re-download missing objects from it and
+  reinstate the tracking branch. **done**
+* For other branches, reset them to last good commit, or delete
+  if none was found. **done**
+* (Decided not to touch tags.)
+
+The index file can still refer to objects that were missing.
+Rewrite to remove them. **done**
diff --git a/doc/design/assistant/disaster_recovery/comment_1_955dc807196863da23aa8dbd15e04364._comment b/doc/design/assistant/disaster_recovery/comment_1_955dc807196863da23aa8dbd15e04364._comment
new file mode 100644
index 0000000000..63c7e942d9
--- /dev/null
+++ b/doc/design/assistant/disaster_recovery/comment_1_955dc807196863da23aa8dbd15e04364._comment
@@ -0,0 +1,10 @@
+[[!comment format=mdwn
+ username="http://nullroute.eu.org/~grawity/"
+ nickname="Mantas"
+ subject="comment 1"
+ date="2013-10-19T15:50:47Z"
+ content="""
+The restriction on fetching over the Git protocol is, partly, for security reasons – e.g. if one accidentally pushes a commit with private data, and then `push --force`'s a cleaned-up version, Git needs to prevent anyone from downloading the old commit by just giving its SHA1 (e.g. obtained from an IRC/email push notification). So it restricts fetching to the tips of any ref. (I've been told that it could check if the given object is merely *reachable* from any ref, but it doesn't do so for performance reasons.)
+
+git 1.8 has a minor way to relax this requirement – it allows giving a SHA1 to `git fetch` (although I think the protocol already worked this way), and it allows refs to be hidden server-side but still remain fetchable, so in theory there could be a (hidden) ref for every object, for easy fetching...
+"""]]
diff --git a/doc/design/assistant/encrypted_git_remotes.mdwn b/doc/design/assistant/encrypted_git_remotes.mdwn
new file mode 100644
index 0000000000..b7e817cff4
--- /dev/null
+++ b/doc/design/assistant/encrypted_git_remotes.mdwn
@@ -0,0 +1,22 @@
+Encrypted git remotes are now possible 
+using [git-remote-gcrypt](https://spwhitton.name/tech/code/git-remote-gcrypt/).
+
+There are at least two use cases for this in the assistant:
+
+* Storing an encrypted git repository on a local drive. **done**
+* Or on a remote server. This could even allow using github. But more
+  likely would be a shell server that has git-annex-shell on it so can
+  also store file contents, and which is not trusted with unencrypted data.
+  **done**
+
+git-remote-gcrypt is already usable with git-annex. What's needed is
+to make sure it's installed (ie, get it packaged into distros or embedded
+into git-annex), and make it easy to set up from the webapp. **done**
+
+Hmm, this will need gpg key creation, so would also be a good opportunity
+to make the webapp allow using that for special remotes too.
+
+One change is needed in git-annex core.. It currently does not support
+storing encrypted files on git remotes, only on special remotes. Perhaps
+the way to deal with this is to make it consider git-remote-grypt remotes
+to be a special remote type? **done**
diff --git a/doc/design/assistant/gpgkeys.mdwn b/doc/design/assistant/gpgkeys.mdwn
new file mode 100644
index 0000000000..9d1254eb93
--- /dev/null
+++ b/doc/design/assistant/gpgkeys.mdwn
@@ -0,0 +1,40 @@
+Currently the assistant sets up a shared encryption key, which is checked
+into git, so anyone who gets the repository can decrypt files that are
+stored encrypted on special remotes.
+
+To support using gpg keys in the assistant, we need some things:
+
+1. Help user set up a gpg key if they don't have one. This could be a
+   special-purpose key dedicated to being used by git-annex. It might be
+   nice to leave the user with a securely set up general purpose key,
+   but that would certainly preclude prompting for its password in the
+   webapp. Indeed, the password prompt is the main problem here.
+   Best solution would be to get gpg agent working on all supported 
+   platforms.
+
+   Update: For now, git-annex only assists in generating gpg keys that are
+   intended to only be used to encrypt a repo.
+
+2. After generating a gpg key, back it up. It might be the only way
+   some data is accessible.
+
+   One way I'm considering is generating a QR code
+   of the key, which could be printed to paper. Preliminary results
+   are good; a 4096 bit secret key fits in a QR code (a secret key
+   with many subkeys may not). Debian has command-line utilities that
+   can generate and read such a QR code.
+
+   Another way would be to use shamir secret sharing to split the key into
+   N peices and send each one to one of the user's repos.
+
+3. Help user learn the gpg keys of people they want to share their repo
+   with, and give them access. If the public key was recorded in the git-annex
+   branch, this could be easily determined when sharing repositories with
+   friends. Or, use MonkeySphere, or Monkeysign..
+
+-----
+
+Another gpg key security thing is that currently git-annex stores
+crypto creds in memory while it's running. Should use locked memory. See
+ and
+
diff --git a/doc/design/assistant/gpgkeys/comment_1_a14427f88c9fd8e25ad8708146bb4bff._comment b/doc/design/assistant/gpgkeys/comment_1_a14427f88c9fd8e25ad8708146bb4bff._comment
new file mode 100644
index 0000000000..610f9bbb67
--- /dev/null
+++ b/doc/design/assistant/gpgkeys/comment_1_a14427f88c9fd8e25ad8708146bb4bff._comment
@@ -0,0 +1,15 @@
+[[!comment format=mdwn
+ username="gernot"
+ ip="85.197.15.125"
+ subject="Paperkey"
+ date="2013-12-05T11:37:02Z"
+ content="""
+Regarding backups, have you considered using [paperkey](http://www.jabberwocky.com/software/paperkey/) (also [in Debian](http://packages.debian.org/search?keywords=paperkey))?
+It only stores a minimal amount of key data and formats it in a human-readable way.
+
+The result is mainly meant to be printed but `paperkey` would probably also be a good way to keep the size of QR codes down.
+
+I've actually recovered a key from a such a printed backup (using OCR) and it worked great thanks to line-wise checksums.
+Maybe you could create paperkey PDF files with a proper OCR font through the web app?
+
+"""]]
diff --git a/doc/design/assistant/inotify.mdwn b/doc/design/assistant/inotify.mdwn
new file mode 100644
index 0000000000..c951f4fe2a
--- /dev/null
+++ b/doc/design/assistant/inotify.mdwn
@@ -0,0 +1,234 @@
+"git annex watch" command, which runs, in the background, watching via
+inotify for changes, and automatically annexing new files, etc. Now
+available!
+
+[[!toc]]
+
+## known bugs
+
+* Kqueue has to open every directory it watches, so too many directories
+  will run it out of the max number of open files (typically 1024), and fail.
+  I may need to fork off multiple watcher processes to handle this.
+  See [[bug|bugs/Issue_on_OSX_with_some_system_limits]]. (Does not affect
+  OSX any longer, only other BSDs).
+
+## beyond Linux
+
+I'd also like to support OSX and if possible the BSDs.
+
+* kqueue ([haskell bindings](http://hackage.haskell.org/package/kqueue))
+  is supported by FreeBSD, OSX, and other BSDs.
+
+  In kqueue, to watch for changes to a file, you have to have an open file
+  descriptor to the file. This wouldn't scale. 
+
+  Apparently, a directory can be watched, and events are generated when
+  files are added/removed from it. You then have to scan to find which
+  files changed. [example](https://developer.apple.com/library/mac/#samplecode/FileNotification/Listings/Main_c.html#//apple_ref/doc/uid/DTS10003143-Main_c-DontLinkElementID_3)
+
+  Gamin does the best it can with just kqueue, supplimented by polling.
+  The source file `server/gam_kqueue.c` makes for interesting reading.
+  Using gamin to do the heavy lifting is one option. 
+  ([haskell bindings](http://hackage.haskell.org/package/hlibfam) for FAM;
+  gamin shares the API)
+
+  kqueue does not seem to provide a way to tell when a file gets closed,
+  only when it's initially created. Poses problems..
+
+  * [man page](http://www.freebsd.org/cgi/man.cgi?query=kqueue&apropos=0&sektion=0&format=html)
+  *  (good example program)
+
+  *kqueue is now supported*
+
+* hfsevents ([haskell bindings](http://hackage.haskell.org/package/hfsevents))
+  is OSX specific.
+
+  Originally it was only directory level, and you were only told a
+  directory had changed and not which file. Based on the haskell
+  binding's code, from OSX 10.7.0, file level events were added.
+
+  This will be harder for me to develop for, since I don't have access to
+  OSX machines..
+
+  hfsevents does not seem to provide a way to tell when a file gets closed,
+  only when it's initially created. Poses problems..
+
+  * 
+  *  (good example program)
+  *  (good example program)
+
+  *hfsevents is now supported*
+
+* Windows has a Win32 ReadDirectoryChangesW, and perhaps other things.
+
+  It was easy to get watching to work in windows. But there is no lsof,
+  to check if a file can safely be added. So, need to carefully consider
+  how to make adding a file safe in windows.
+
+  Without lsof, an InodeCache is generated in "lockdown" (which doesn't
+  do anything to prevent new writers), and is compared with the stat of the
+  file after it's ingested (and checksummed). This will detect many changes
+  to files, which change the size or mtime.
+
+  So, we have 2 cases to worry about. 
+
+  1. A process has the file open for write as it's added, does not change
+     it until the add is done.
+
+     As long as an event is generated once the file does get closed, this
+     is fine -- the modified version will be re-added. And such events are
+     indeed generated on windows.
+
+  2. A process has the file open for write as it's added, and changes it 
+     in some way that does not affect size or mtime.
+
+     If an event is generated when the file does get closed, this is the
+     same as a scenario where a process opens the file after it's added,
+     makes such a change, and closes it. In either case, a file closed
+     event is generated, and the Watcher will not detect any change
+     using the inode cache, so will not re-add the file.
+
+     So, this scenario is a potential problem, but it seems at least
+     unlikely that a program would modify a file without affecting its
+     mtime. Note that this same scenario can happen even with lsof, and
+     even on linux (although on linux the InodeCache includes an actual
+     inode, which might detect the change too).
+
+  Conclusion: It's probably ok to run without lsof on Windows.
+
+  Corrolary: lsof might not generally be needed in direct mode, on
+  systems that do generate file close events (but not when
+  eventsCoalesce).
+  The same arguments given above seem to apply to !Windows. Note that lsof
+  is needed in indirect mode, as discussed below.
+
+  **windows is now supported**
+
+## the races
+
+Many races need to be dealt with by this code. Here are some of them.
+
+* File is added and then removed before the add event starts.
+
+  Not a problem; The add event does nothing since the file is not present.
+
+* File is added and then removed before the add event has finished
+  processing it.
+  
+  **Minor problem**; When the add's processing of the file (checksum and so
+  on) fails due to it going away, there is an ugly error message, but
+  things are otherwise ok.
+
+* File is added and then replaced with another file before the annex add
+  moves its content into the annex.
+
+  Fixed this problem; Now it hard links the file to a temp directory and
+  operates on the hard link, which is also made unwritable.
+
+* File is added and then replaced with another file before the annex add
+  makes its symlink.
+
+  **Minor problem**; The annex add will fail creating its symlink since
+  the file exists. There is an ugly error message, but the second add
+  event will add the new file.
+
+* File is added and then replaced with another file before the annex add
+  stages the symlink in git.
+
+  Now fixed; `git annex watch` avoids running `git add` because of this
+  race. Instead, it stages symlinks directly into the index, without
+  looking at what's currently on disk.
+
+* Link is moved, fixed link is written by fix event, but then that is
+  removed by the user and replaced with a file before the event finishes.
+
+  Now fixed; same fix as previous race above.
+
+* File is removed and then re-added before the removal event starts.
+
+  Not a problem; The removal event does nothing since the file exists,
+  and the add event replaces it in git with the new one.
+
+* File is removed and then re-added before the removal event finishes.
+
+  Not a problem; The removal event removes the old file from the index, and
+  the add event adds the new one.
+
+* Symlink appears, but is then deleted before it can be processed.
+
+  Leads to an ugly message, otherwise no problem:
+
+	./me: readSymbolicLink: does not exist (No such file or directory)
+
+  Here `me` is a file that was in a conflicted merge, which got
+  removed as part of the resolution. This is probably coming from the watcher
+  thread, which sees the newly added symlink (created by the git merge),
+  but finds it deleted (by the conflict resolver) by the time it processes it.
+
+## done
+
+- on startup, add any files that have appeared since last run **done**
+- on startup, fix the symlinks for any renamed links **done**
+- on startup, stage any files that have been deleted since last run
+  (seems to require a `git commit -a` on startup, or at least a
+  `git add --update`, which will notice deleted files) **done**
+- notice new files, and git annex add **done**
+- notice renamed files, auto-fix the symlink, and stage the new file location
+  **done**
+- handle cases where directories are moved outside the repo, and stop
+  watching them **done**
+- when a whole directory is deleted or moved, stage removal of its
+  contents from the index **done**
+- notice deleted files and stage the deletion
+  (tricky; there's a race with add since it replaces the file with a symlink..)
+  **done**
+- Gracefully handle when the default limit of 8192 inotified directories
+  is exceeded. This can be tuned by root, so help the user fix it.
+  **done**
+- periodically auto-commit staged changes (avoid autocommitting when
+  lots of changes are coming in) **done**
+- coleasce related add/rm events for speed and less disk IO **done**
+- don't annex `.gitignore` and `.gitattributes` files **done**
+- run as a daemon **done**
+- A process has a file open for write, another one closes it,
+  and so it's added. Then the first process modifies it.
+
+  Or, a process has a file open for write when `git annex watch` starts
+  up, it will be added to the annex. If the process later continues
+  writing, it will change content in the annex.
+
+  This changes content in the annex, and fsck will later catch
+  the inconsistency.
+
+  Possible fixes: 
+
+  * Somehow track or detect if a file is open for write by any processes.
+    `lsof` could be used, although it would be a little slow.
+
+    Here's one way to avoid the slowdown: When a file is being added,
+    set it read-only, and hard-link it into a quarantine directory,
+    remembering both filenames.
+    Then use the batch change mode code to detect batch adds and bundle
+    them together.
+    Just before committing, lsof the quarantine directory. Any files in
+    it that are still open for write can just have their write bit turned
+    back on and be deleted from quarantine, to be handled when their writer
+    closes. Files that pass quarantine get added as usual. This avoids
+    repeated lsof calls slowing down adds, but does add a constant factor
+    overhead (0.25 seconds lsof call) before any add gets committed. **done**
+
+  * Or, when possible, making a copy on write copy before adding the file
+    would avoid this.
+  * Or, as a last resort, make an expensive copy of the file and add that.
+  * Tracking file opens and closes with inotify could tell if any other
+    processes have the file open. But there are problems.. It doesn't
+    seem to differentiate between files opened for read and for write.
+    And there would still be a race after the last close and before it's
+    injected into the annex, where it could be opened for write again.
+    Would need to detect that and undo the annex injection or something.
+
+- If a file is checked into git as a normal file and gets modified
+  (or merged, etc), it will be converted into an annexed file.
+  See [[blog/day_7__bugfixes]]. **done**; we always check ls-files now
+- When you `git annex unlock` a file, it will immediately be re-locked.
+  See [[bugs/watcher_commits_unlocked_files]]. Seems fixed now?
diff --git a/doc/design/assistant/inotify/comment_1_3d3ff74447452d65c10ccc3dbfc323cd._comment b/doc/design/assistant/inotify/comment_1_3d3ff74447452d65c10ccc3dbfc323cd._comment
new file mode 100644
index 0000000000..d042806a17
--- /dev/null
+++ b/doc/design/assistant/inotify/comment_1_3d3ff74447452d65c10ccc3dbfc323cd._comment
@@ -0,0 +1,7 @@
+[[!comment format=mdwn
+ username="http://ciffer.net/~svend/"
+ subject="comment 1"
+ date="2012-06-04T19:42:07Z"
+ content="""
+I would find it useful if the watch command could 'git add' new files (instead of 'git annex add') for certain repositories.
+"""]]
diff --git a/doc/design/assistant/inotify/comment_2_a3c0fa6d97397c508b4b8aafdcee8f6f._comment b/doc/design/assistant/inotify/comment_2_a3c0fa6d97397c508b4b8aafdcee8f6f._comment
new file mode 100644
index 0000000000..13ee1523c3
--- /dev/null
+++ b/doc/design/assistant/inotify/comment_2_a3c0fa6d97397c508b4b8aafdcee8f6f._comment
@@ -0,0 +1,7 @@
+[[!comment format=mdwn
+ username="http://joeyh.name/"
+ subject="comment 2"
+ date="2012-06-04T19:46:03Z"
+ content="""
+I think it's already on the list: \"configurable option to only annex files meeting certian size or filename criteria\" -- files not meeting those criteria would just be git added.
+"""]]
diff --git a/doc/design/assistant/inotify/comment_3_b346e870c1cd80e4b0a313c3a9fed6b3._comment b/doc/design/assistant/inotify/comment_3_b346e870c1cd80e4b0a313c3a9fed6b3._comment
new file mode 100644
index 0000000000..c1f22c4b8a
--- /dev/null
+++ b/doc/design/assistant/inotify/comment_3_b346e870c1cd80e4b0a313c3a9fed6b3._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="https://www.google.com/accounts/o8/id?id=AItOawkSq2FDpK2n66QRUxtqqdbyDuwgbQmUWus"
+ nickname="Jimmy"
+ subject="comment 3"
+ date="2012-06-17T08:52:32Z"
+ content="""
+In relation to OSX support, hfsevents (or supporting hfs is probably a bad idea), its very osx specific and users who are moving usb keys and disks between systems will probably end up using fat32/exfat/vfat disks around. Also if you want I can lower the turn around time for the OSX auto-builder that I have setup to every 1 or 2mins? would that help?
+"""]]
diff --git a/doc/design/assistant/inotify/comment_4_32be58b4c3b17a4ea539690d2fb45159._comment b/doc/design/assistant/inotify/comment_4_32be58b4c3b17a4ea539690d2fb45159._comment
new file mode 100644
index 0000000000..d854d91c2b
--- /dev/null
+++ b/doc/design/assistant/inotify/comment_4_32be58b4c3b17a4ea539690d2fb45159._comment
@@ -0,0 +1,12 @@
+[[!comment format=mdwn
+ username="http://joeyh.name/"
+ ip="4.154.2.147"
+ subject="comment 4"
+ date="2012-06-17T16:39:43Z"
+ content="""
+hfsevents seems usable, git-annex does not need to watch for file changes on remotes on other media.
+
+But, trying kqueue first.
+
+You could perhaps run the autobuilder on a per-commit basis..
+"""]]
diff --git a/doc/design/assistant/inotify/comment_5_0cdd3046d90ad2012025d846ece0731e._comment b/doc/design/assistant/inotify/comment_5_0cdd3046d90ad2012025d846ece0731e._comment
new file mode 100644
index 0000000000..8b075c36f0
--- /dev/null
+++ b/doc/design/assistant/inotify/comment_5_0cdd3046d90ad2012025d846ece0731e._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="https://www.google.com/accounts/o8/id?id=AItOawkSq2FDpK2n66QRUxtqqdbyDuwgbQmUWus"
+ nickname="Jimmy"
+ subject="comment 5"
+ date="2012-06-17T21:42:59Z"
+ content="""
+okay, I've gotten gitbuilder to poll the git repo every minute for changes, gitbuilder doesn't build every commit. It doesn't work like that, it checks out the master and builds that. If there is a failure it automatically bisects to find out where the problem first got introduced. Hope the change to the builder helps!
+"""]]
diff --git a/doc/design/assistant/inotify/comment_6_e197d5d0d853572ec1f2e5985762e60d._comment b/doc/design/assistant/inotify/comment_6_e197d5d0d853572ec1f2e5985762e60d._comment
new file mode 100644
index 0000000000..76716ddda5
--- /dev/null
+++ b/doc/design/assistant/inotify/comment_6_e197d5d0d853572ec1f2e5985762e60d._comment
@@ -0,0 +1,9 @@
+[[!comment format=mdwn
+ username="https://www.google.com/accounts/o8/id?id=AItOawnV2c63kDc6X21a1H81me1mIenUCScd2Gs"
+ nickname="Emanuele"
+ subject="watch branch?"
+ date="2012-06-01T19:19:17Z"
+ content="""
+Hello there? Where can I find more info about this git watch branch? 
+Keep up the good work!
+"""]]
diff --git a/doc/design/assistant/inotify/comment_7_00809aaad6b68f189a9cc42af810a0a6._comment b/doc/design/assistant/inotify/comment_7_00809aaad6b68f189a9cc42af810a0a6._comment
new file mode 100644
index 0000000000..7e1ad7db5c
--- /dev/null
+++ b/doc/design/assistant/inotify/comment_7_00809aaad6b68f189a9cc42af810a0a6._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="https://www.google.com/accounts/o8/id?id=AItOawnwfqF4wL6l_O26RyzoBowUMvQ_955Vpao"
+ nickname="Markus"
+ subject="comment 7"
+ date="2014-06-14T06:29:55Z"
+ content="""
+FWIW: Removing the [raspi-copies-and-fills](https://github.com/simonjhall/copies-and-fills) package ('apt-get purge raspi-copies-and-fills') stops annex from printing the message over and over again. According to the package's description though, you suffer a performance penalty without the package.
+"""]]
diff --git a/doc/design/assistant/leftovers.mdwn b/doc/design/assistant/leftovers.mdwn
new file mode 100644
index 0000000000..b8c0f456b6
--- /dev/null
+++ b/doc/design/assistant/leftovers.mdwn
@@ -0,0 +1,17 @@
+Things that don't fit anywhere else:
+
+* Automatically start daemon on boot or when user logs in, using
+  freedesktop autostart file. **done**
+* Somehow get content that is unavailable. This is problematic with inotify,
+  since we only get an event once the user has tried (and failed) to read
+  from the file. This is only needed if all the files in the directory 
+  are not kept synced, but in some situations (ie, low disk space phones),
+  that is likely.
+* Drop files that have not been used lately, or meet some other criteria
+  (as long as there's a copy elsewhere). **done** (via preferred content;
+  eg archive directories)
+* Perhaps automatically dropunused files that have been deleted,
+  although I cannot see a way to do that, since by the time the inotify
+  deletion event arrives, the file is deleted, and we cannot see what
+  its symlink pointed to! Alternatively, perhaps automatically
+  do an expensive unused/dropunused cleanup process.
diff --git a/doc/design/assistant/leftovers/comment_1_b20c88bb3c583a32023c1f6b6dc9486d._comment b/doc/design/assistant/leftovers/comment_1_b20c88bb3c583a32023c1f6b6dc9486d._comment
new file mode 100644
index 0000000000..3c7e1820db
--- /dev/null
+++ b/doc/design/assistant/leftovers/comment_1_b20c88bb3c583a32023c1f6b6dc9486d._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="https://www.google.com/accounts/o8/id?id=AItOawl9sYlePmv1xK-VvjBdN-5doOa_Xw-jH4U"
+ nickname="Richard"
+ subject="comment 1"
+ date="2012-10-30T09:07:37Z"
+ content="""
+I think the automatic dropunused part is handled by the assistant already anyway, but in case it's not: The git repo still knows where the symlink pointed to so it should not be a problem to get that info from the last commit that was 'actively' locally and which contained said symlink.
+"""]]
diff --git a/doc/design/assistant/more_cloud_providers.mdwn b/doc/design/assistant/more_cloud_providers.mdwn
new file mode 100644
index 0000000000..16e7276570
--- /dev/null
+++ b/doc/design/assistant/more_cloud_providers.mdwn
@@ -0,0 +1,24 @@
+Git-annex already supports storing large files in 
+several cloud providers via [[special_remotes]].
+More should be added, such as:
+
+* Google drive (attractive because it's free, only 5 gb tho)
+* Owncloud (has several [providers](http://owncloud.org/providers/);
+  at least one provides 5 gb free; open DAV based API)
+* OpenStack Swift (teh future)
+* Box.com (it's free, and current method is hard to set up and a sorta
+  shakey; a better method would be to use its API) **done**
+* Dropbox? That would be ironic.. Via its API, presumably.
+* [[Amazon Glacier|todo/special_remote_for_amazon_glacier]] **done**
+* Internet Archive **done**
+* [nimbus.io](https://nimbus.io/) Fairly low prices ($0.06/GB);
+  REST API; free software
+* Mediafire provides 50gb free and has a REST API.
+* Flickr provides 1 tb (!!!!) to free accounts, and can store at least
+  photos and videos.  is a hack
+  to allow storing any type of file on Flickr.
+* mega.co.nz. Already supported via [[tips/megaannex]], would just need
+  webapp modifications to configure it. May want to use megaannex as-is to
+  build a non-hook special remote in haskell.
+
+See poll at [[polls/prioritizing_special_remotes]].
diff --git a/doc/design/assistant/pairing.mdwn b/doc/design/assistant/pairing.mdwn
new file mode 100644
index 0000000000..d09c644ee7
--- /dev/null
+++ b/doc/design/assistant/pairing.mdwn
@@ -0,0 +1,83 @@
+For git-annex to be able to clone its repo to another host, it'd be good to
+have some way of pairing devices.
+
+## security
+
+Pairing uses its own network protocol, built on top of multicast UDP.
+
+It's important that pairing securely verifies that the right host is being
+paired with. This is accomplied by having a shared secret be entered on
+both the hosts that will be paired. Hopefully that secret is communicated
+securely out of band. 
+
+(In practice, the security of that communication will vary. To guard against
+interception, each pairing session pairs exactly two hosts and then forgets
+the shared secret. So an attacker who tries to reuse an intercepted secret
+will not succeed in pairing. This does not guard against an intercepted
+secret that is used before the legitimate parties finish pairing.)
+
+Each host can construct messages that the other host can verify using the
+shared secret, and so know that, for example, the ssh public key it
+received belongs to the right host and has not been altered by a man in the
+middle.
+
+The verification works like this: Take a HMAC SHA1 checksum of the message,
+using the shared secret as the HMAC key. Include this checksum after the
+message. The other host can then do the same calculation and verify the
+checksum.
+
+Additionally, a UUID is included in the message. Messages that are part of
+the same pairing session all share a UUID. And all such messages should
+be verifiable as described above. If a message has the same UUID but is
+not verifiable, then someone on the network is up to no good. Perhaps
+they are trying to brute-force the shared secret. When this is detected,
+the pairing session is shut down. (Which would still let an attacker
+DOS pairing, but that's not a very interesting attack.)
+
+The protocol used for pairing consists of 3 messages, a PairReq, and
+PairAck, and a PairDone. Let's consider what an attacker could accomplish
+by replaying these:
+
+* PairReq: This would make the webapp pop up an alert about an incoming
+  pair request. If the user thought it was real and for some reason
+  entered the right shared secret used in the real one earlier, the
+  ssh key inside the PairReq would be added to `authorized_keys`. Which
+  allows the host that originally sent the PairReq to access its git
+  repository, but doesn't seem to do the attacker any good.
+* PairAck:  If the host that originally sent
+  the PairReq is still pairing, it'll add the ssh key from the PairAck,
+  and start syncing, which again does the attacker no good.
+* PairDone: If the host that sent the PairAck is still syncing, it'll
+  add the ssh key from the PairDone, and start syncing, and stop
+  sending PairAcks. But probably, it's not syncing, because it would have
+  seen the original PairDone.. and anyway, this seems to do the attacker no
+  good.
+
+So replay attacks don't seem to be a problem.
+
+So far I've considered security from a third-party attacker, but either the
+first or second parties in pairing could also be attackers. Presumably they
+trust each other with access to their files as mediated by
+[[git-annex-shell]]. However, one could try to get shell access to the
+other's computer by sending malicious data in a pairing message. So the
+pairing code always checks every data field's content, for example the ssh
+public key is rejected if it looks at all unusual. Any control characters
+in the pairing message cause it to be rejected, to guard against console
+poisoning attacks. Furthermore, git-annex is careful not to expose data to
+the shell, and the webapp uses Yesod's type safety to ensure all user input
+is escaped before going to the browser.
+
+## TODO
+
+* pairing over IPV6 only networks does not work. Haskell's
+  `network-multicast` library complains "inet_addr: Malformed address: ff02::1"
+  .. seems it just doesn't support IPv6. The pairing code in git-annex
+  does support ipv6, apart from this, it's just broadcasting the messages
+  that fails. (Pairing over mixed networks is fine.)
+* If there are three assistants on the network, and 2 pair, the third is
+  left displaying a "Pair request from foo" alert, until it's close. 
+  Or, if the user clicks the button to pair, it'll get to the 
+  "Pairing in progress" alert, which will show forever (until canceled).
+
+  It should be possible for third parties to tell when pairing is done,
+  but it's actually rather hard since they don't necessarily share the secret.
diff --git a/doc/design/assistant/partial_content.mdwn b/doc/design/assistant/partial_content.mdwn
new file mode 100644
index 0000000000..cbfbfcea83
--- /dev/null
+++ b/doc/design/assistant/partial_content.mdwn
@@ -0,0 +1,36 @@
+On a regular system, a reasonable simplifying assumption is that all the
+files in the folder will be synced to the system. A user might want to
+disable syncing of some subdirectories, for eg, archived files. But in
+general, things are simpler to understand and implement if all files sync.
+
+But, an Android gadget probably cannot hold all a user's files. Indeed,
+it's likely that old files will be aggressively dropped from the Android
+after syncing to elsewhere, in order to keep enough free space on it for
+new files. 
+
+There needs to be a way for the user to browse files not on the gadget and
+request they be transferred to it. This could be done as a browser in the
+web app, or using a subdirectory full of placeholder files (not symlinks;
+see [[Android]]) that start transfer of the real file when accessed.
+
+----
+
+Currently, Android uses the "source" repository type in some
+configurations. This makes files be removed as soon as they are sent
+somewhere else.
+
+A compromise that avoids needing UI might be to change "source" so it
+retained files for a while after they were created, even after they were
+uploaded elsewhere. For example, it could hold onto them for a day. This
+would allow the user time to do things with new files before they are
+removed from the android device.
+
+One way to implement that would be a new preferred content expression like
+"age(1 day)". But this would need at least a daily full transfer scan to be
+run. 
+
+Another way would be to have a way to make drops of files be deferred
+for a period of time. This approach would not need to be specific to the
+"source" repository type. And seems easy enough to do, just have a
+configuration setting for the time interval, and an ordered drop queue
+and a thread that waits as needed before dropping.
diff --git a/doc/design/assistant/partial_content/comment_1_58c4faa321a5bb71adf9fdee079849f4._comment b/doc/design/assistant/partial_content/comment_1_58c4faa321a5bb71adf9fdee079849f4._comment
new file mode 100644
index 0000000000..aa65cd6b9a
--- /dev/null
+++ b/doc/design/assistant/partial_content/comment_1_58c4faa321a5bb71adf9fdee079849f4._comment
@@ -0,0 +1,18 @@
+[[!comment format=mdwn
+ username="https://www.google.com/accounts/o8/id?id=AItOawkpEY8WTFDhjIVTWG38Ph7ppmuXUTJAHAg"
+ nickname="Justin"
+ subject="selective sync"
+ date="2012-07-28T04:08:00Z"
+ content="""
+hey joey
+
+great work!!
+
+will partial content work like selective sync in dropbox
+
+use case: on desktop i have photos/mp3s/docs, but would only want to sync the docs to my netbook
+
+cheers
+
+justin
+"""]]
diff --git a/doc/design/assistant/polls.mdwn b/doc/design/assistant/polls.mdwn
new file mode 100644
index 0000000000..3457fd2453
--- /dev/null
+++ b/doc/design/assistant/polls.mdwn
@@ -0,0 +1 @@
+[[!inline pages="(page(design/assistant/blog/*) and tagged(design/assistant/polls)) or page(design/assistant/polls/*)" show=0]]
diff --git a/doc/design/assistant/polls/2013_user_survey.mdwn b/doc/design/assistant/polls/2013_user_survey.mdwn
new file mode 100644
index 0000000000..d1bac0ef8a
--- /dev/null
+++ b/doc/design/assistant/polls/2013_user_survey.mdwn
@@ -0,0 +1 @@
+The 2013 git-annex user survey is in progress [[here|/polls/2013]].
diff --git a/doc/design/assistant/polls/Android.mdwn b/doc/design/assistant/polls/Android.mdwn
new file mode 100644
index 0000000000..78806eac29
--- /dev/null
+++ b/doc/design/assistant/polls/Android.mdwn
@@ -0,0 +1,18 @@
+Help me choose a goal for the month of December. The last poll showed 
+a lot of interest in using the git-annex assistant with phones, etc.
+
+Background: git-annex uses symbolic links in its repositories. This makes it
+hard to use with filesystems, such as FAT, that do not support symbolic links.
+FAT filesystems are the main storage available on some Android devices that
+have a micro-SD card. Other, newer Android devices don't have a SD card and so
+avoid this problem.
+
+I can either work on the idea described in
+[[design/assistant/desymlink]], which could solve the symlink problem and
+also could lead to a nicer workflow to editing files that are stored in
+git-annex.
+
+Or, I can work on [[Android_porting|design/assistant/android]], and try to
+get the assistant working on Android's built-in storage.
+
+[[!poll open=no 81 "solve the symlink problem first" 17 "port to Android first" 1 "other"]]
diff --git a/doc/design/assistant/polls/Android/comment_1_fa6c409833f28c67da105d25f4a440e0._comment b/doc/design/assistant/polls/Android/comment_1_fa6c409833f28c67da105d25f4a440e0._comment
new file mode 100644
index 0000000000..49b7796073
--- /dev/null
+++ b/doc/design/assistant/polls/Android/comment_1_fa6c409833f28c67da105d25f4a440e0._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="https://www.google.com/accounts/o8/id?id=AItOawn4yApngiiAJE5iX1MQ0VO_LXjLt4LgdHE"
+ nickname="Jim"
+ subject="No mass storage"
+ date="2012-12-02T03:33:06Z"
+ content="""
+I would have asked for better FAT support, but my Galaxy Nexus doesn't support mass storage mode, and MTP is not as easy to use nor reliably supported everywhere.  So I'd lean towards functionality that stays on the phone as much as possible.
+"""]]
diff --git a/doc/design/assistant/polls/Android_default_directory.mdwn b/doc/design/assistant/polls/Android_default_directory.mdwn
new file mode 100644
index 0000000000..8aa8cfefa3
--- /dev/null
+++ b/doc/design/assistant/polls/Android_default_directory.mdwn
@@ -0,0 +1,7 @@
+What directory should the Android webapp default to creating an annex in?
+
+Same as the desktop webapp, users will be able to enter a directory they
+want the first time they run it, but to save typing on android, anything
+that gets enough votes will be included in a list of choices as well.
+
+[[!poll open=no expandable=yes 79 "/sdcard/annex" 6 "Whole /sdcard" 9 "DCIM directory (photos and videos only)" 3 "Same as for regular git-annex. ~/annex/"]]
diff --git a/doc/design/assistant/polls/Android_default_directory/comment_1_d39655091ac3ed51a9d4325d86b23ad7._comment b/doc/design/assistant/polls/Android_default_directory/comment_1_d39655091ac3ed51a9d4325d86b23ad7._comment
new file mode 100644
index 0000000000..30b73d184c
--- /dev/null
+++ b/doc/design/assistant/polls/Android_default_directory/comment_1_d39655091ac3ed51a9d4325d86b23ad7._comment
@@ -0,0 +1,10 @@
+[[!comment format=mdwn
+ username="https://www.google.com/accounts/o8/id?id=AItOawl9sYlePmv1xK-VvjBdN-5doOa_Xw-jH4U"
+ nickname="Richard"
+ subject="comment 1"
+ date="2013-04-19T06:29:14Z"
+ content="""
+While my main use case is a photo repo, I don't think it's wise to default to there.
+
+Similarly, spamming what amounts to the root dir of \"external\" storage with repos seems like a bad idea.
+"""]]
diff --git a/doc/design/assistant/polls/Android_default_directory/comment_2_2f1eaae95075db26488517720afd1c63._comment b/doc/design/assistant/polls/Android_default_directory/comment_2_2f1eaae95075db26488517720afd1c63._comment
new file mode 100644
index 0000000000..b2df3b0a27
--- /dev/null
+++ b/doc/design/assistant/polls/Android_default_directory/comment_2_2f1eaae95075db26488517720afd1c63._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="https://www.google.com/accounts/o8/id?id=AItOawnE6kFAbud1LWrQuyX76yMYnUjHt9tR-A8"
+ nickname="Leonardo"
+ subject="Direct mode"
+ date="2013-04-20T20:55:09Z"
+ content="""
+I think direct mode should be the default on Android
+"""]]
diff --git a/doc/design/assistant/polls/Android_default_directory/comment_3_b484012f60789be73d7d5b338cff6203._comment b/doc/design/assistant/polls/Android_default_directory/comment_3_b484012f60789be73d7d5b338cff6203._comment
new file mode 100644
index 0000000000..7474d774df
--- /dev/null
+++ b/doc/design/assistant/polls/Android_default_directory/comment_3_b484012f60789be73d7d5b338cff6203._comment
@@ -0,0 +1,10 @@
+[[!comment format=mdwn
+ username="http://joeyh.name/"
+ nickname="joey"
+ subject="comment 3"
+ date="2013-04-20T23:38:47Z"
+ content="""
+@Richard, including all of /sdcard seems a reasonable thing to do if you want to back it all up. I don't know how likely anyone would be to want to sync the whole contents though.
+
+@Leonardo direct mode is the default for all respositories created by the webapp, as well as all repositories on crippled filesystems that cannot support indirect mode. Android webapp thus defaults to direct mode twice over!
+"""]]
diff --git a/doc/design/assistant/polls/goals_for_April.mdwn b/doc/design/assistant/polls/goals_for_April.mdwn
new file mode 100644
index 0000000000..53132dd441
--- /dev/null
+++ b/doc/design/assistant/polls/goals_for_April.mdwn
@@ -0,0 +1,17 @@
+What should I work on in April? I expect I could get perhaps two of these
+features done in a month if I'm lucky. I have only 3 more funded months,
+and parts of one will be spent working on porting to Windows, so choose wisely!
+--[[Joey]]
+
+[[!poll open=yes expandable=yes 4 "upload and download rate limiting" 15 "get webapp working on Android" 5 "deltas: speed up syncing modified versions of existing files" 8 "encrypted git remotes using git-remote-gcrypt" 0 "add support for more cloud storage remotes" 19 "don't work on features, work on making it easier to install and use" 2 "Handle duplicate files" 6 "direct mode (aka real files instead of symlinks) [already done --joey]" 3 "start windows port now"]]
+
+References:
+
+* [[rate_limiting]]
+* [[Android]]
+* [[deltas]] to speed up syncing modified files (at least for remotes using rsync)
+* [[encrypted_git_remotes]]
+* [[more_cloud_providers]] (OpenStack Swift, Owncloud, Google drive,
+  Dropbox, Mediafire, nimbus.io, Mega, etc.)
+* [[old poll on "what is preventing me from using git-annex assistant"|what_is_preventing_me_from_using_git-annex_assistant]]
+  (many of the items on it should be fixed now, but I have plenty of bug reports to chew on still)
diff --git a/doc/design/assistant/polls/goals_for_April/comment_1_9f81fa96db5970a4be0828c74a6d2d55._comment b/doc/design/assistant/polls/goals_for_April/comment_1_9f81fa96db5970a4be0828c74a6d2d55._comment
new file mode 100644
index 0000000000..f63460543b
--- /dev/null
+++ b/doc/design/assistant/polls/goals_for_April/comment_1_9f81fa96db5970a4be0828c74a6d2d55._comment
@@ -0,0 +1,22 @@
+[[!comment format=mdwn
+ username="https://www.google.com/accounts/o8/id?id=AItOawm5iosFbL2By7UFeViqkc6v-hoAtqILeDA"
+ nickname="Laszlo"
+ subject="Duplicate files"
+ date="2013-04-01T07:58:51Z"
+ content="""
+Having duplicate files is fairly easy. 
+a) Just a backup of your files, to not have accidental deleting.
+
+b) Some programs implements UNDO command (Ctrl-Z) as simply copying over the working text file
+to a backup directory. 
+
+c) If you tests precompiled programs, like git annex itself, it has identical files across releases.
+
+
+For more info about duplicate files, read hear these two bugreports:
+
+http://git-annex.branchable.com/bugs/When_syncing_two_repositories__44___git_annex_uses_9x_times_diskspace/
+(last comment)
+
+http://git-annex.branchable.com/bugs/Direct_mode_keeps_re-checksuming_duplicated_files/
+"""]]
diff --git a/doc/design/assistant/polls/goals_for_April/comment_2_d8956d220ccacff3d2f6cbeb15718459._comment b/doc/design/assistant/polls/goals_for_April/comment_2_d8956d220ccacff3d2f6cbeb15718459._comment
new file mode 100644
index 0000000000..8fed65fb0c
--- /dev/null
+++ b/doc/design/assistant/polls/goals_for_April/comment_2_d8956d220ccacff3d2f6cbeb15718459._comment
@@ -0,0 +1,28 @@
+[[!comment format=mdwn
+ username="https://www.google.com/accounts/o8/id?id=AItOawkWG4T4SGZxY-q4Wo8Jbxwb67g4J-WYmQM"
+ nickname="Mark"
+ subject="don't work on features option"
+ date="2013-04-01T14:03:35Z"
+ content="""
+The \"don't work on features...\" poll entry is a bit vague, so I figured stating my interpretation of it and why I choose it might be a good idea.
+I've had git-annex installed for the last few months during which it has been steadily improving, but IMO it still lacks polish.
+From time to time I see transient issues: (even on the current version)
+
+- random download stalling when I know the remote is accessible
+- files disappearing then reappearing without an obvious cause
+- xmmp hanging
+
+I do not have any means of replicating these issues (though IIRC some of the recently worked on bugs related to these issues).
+In my past experience this indicates that there are all sorts of 'fun' bugs hiding in the source which you seem to be chasing down.
+Heck, I could have a simple configuration error from when I set things up on my remote server.
+There was little documentation available when I setup my remote server for this and unless I have missed something in the RecentChanges feed, there still is relatively little.
+So, some issues with my xmmp daemon, local ssh keys via ssh-agent, or bad $PATH stuff could be causing things to subtlety malfunction at no fault of git-annex.
+
+Falling back to the command line only tends to be a good response, but outside of the assistant there does not seem to be any manual way to handle the special remotes.
+Fair enough, but it would seem logical for error handling to recognize that these are assistant only urls rather than some generic \"bad url\".
+Just getting the right error messages while a small touch would be a sign of some polish even though it is a nitpick.
+
+Perhaps some simple test code run repeatedly to form a stress test could reveal some odd behavior, but I'm not sure myself.
+
+With all of that said, I like what has been done so far and I'm hoping to see all the nooks of git-annex get the polishing that they deserve.
+"""]]
diff --git a/doc/design/assistant/polls/goals_for_April/comment_3_aadad6dfd56d068d2e377606910c006f._comment b/doc/design/assistant/polls/goals_for_April/comment_3_aadad6dfd56d068d2e377606910c006f._comment
new file mode 100644
index 0000000000..f3cea47c83
--- /dev/null
+++ b/doc/design/assistant/polls/goals_for_April/comment_3_aadad6dfd56d068d2e377606910c006f._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="http://adamspiers.myopenid.com/"
+ nickname="Adam"
+ subject="Ask for more money :-)"
+ date="2013-04-01T19:01:49Z"
+ content="""
+This is slightly off topic, but from my perspective this Kickstarter project has been a riotous success, and due to the credibility you have built up from 9 months of consistently high quality work produced in a very transparent and community-oriented fashion, I'd be surprised if you weren't able to raise more money to continue development after the 12 months is up.  Assuming you want to of course :)  Maybe you need a break or change of scene.  But if not, it would be a shame not to continue if the userbase wants it.
+"""]]
diff --git a/doc/design/assistant/polls/prioritizing_special_remotes.mdwn b/doc/design/assistant/polls/prioritizing_special_remotes.mdwn
new file mode 100644
index 0000000000..c6c2244b25
--- /dev/null
+++ b/doc/design/assistant/polls/prioritizing_special_remotes.mdwn
@@ -0,0 +1,16 @@
+Background: git-annex supports storing data in various [[special remotes]].
+The git-annex assistant will make it easy to configure these, and easy
+configurators have already been built for a few: removable drives, rsync.net,
+locally paired systems, and remote servers with rsync.
+
+Help me prioritize my work: What special remote would you most like
+to use with the git-annex assistant?
+
+[[!poll open=yes 18 "Amazon S3 (done)" 13 "Amazon Glacier (done)" 10 "Box.com (done)" 77 "My phone (or MP3 player)" 29 "Tahoe-LAFS" 17 "OpenStack SWIFT" 37 "Google Drive"]]
+
+This poll is ordered with the options I consider easiest to build
+listed first. Mostly because git-annex already supports them and they
+only need an easy configurator. The ones at the bottom are likely to need
+significant work. See [[cloud]] for detailed discussion.
+
+Have another idea? Absolutely need two or more? Post comments..
diff --git a/doc/design/assistant/polls/prioritizing_special_remotes/comment_1_dd9280df27848a7ff132f5809dab0a79._comment b/doc/design/assistant/polls/prioritizing_special_remotes/comment_1_dd9280df27848a7ff132f5809dab0a79._comment
new file mode 100644
index 0000000000..0214d4a12b
--- /dev/null
+++ b/doc/design/assistant/polls/prioritizing_special_remotes/comment_1_dd9280df27848a7ff132f5809dab0a79._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="https://www.google.com/accounts/o8/id?id=AItOawkSq2FDpK2n66QRUxtqqdbyDuwgbQmUWus"
+ nickname="Jimmy"
+ subject="comment 1"
+ date="2012-09-13T08:39:59Z"
+ content="""
+I've been looking at ceph for various reasons in work, it supports a swift interface as well as it's own restful api. so +1 for swift (and any s3 compatible api).
+"""]]
diff --git a/doc/design/assistant/polls/prioritizing_special_remotes/comment_2_370e0b9c43486ee96c825f9155eebde4._comment b/doc/design/assistant/polls/prioritizing_special_remotes/comment_2_370e0b9c43486ee96c825f9155eebde4._comment
new file mode 100644
index 0000000000..4d00922a48
--- /dev/null
+++ b/doc/design/assistant/polls/prioritizing_special_remotes/comment_2_370e0b9c43486ee96c825f9155eebde4._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="https://www.google.com/accounts/o8/id?id=AItOawl9sYlePmv1xK-VvjBdN-5doOa_Xw-jH4U"
+ nickname="Richard"
+ subject="comment 2"
+ date="2012-09-13T09:07:02Z"
+ content="""
+Swift has its own API but offers a S3 compatibility layer. Last I tried that layer, it did not work.
+"""]]
diff --git a/doc/design/assistant/polls/prioritizing_special_remotes/comment_3_883a003b9c552b89f191135c582f99aa._comment b/doc/design/assistant/polls/prioritizing_special_remotes/comment_3_883a003b9c552b89f191135c582f99aa._comment
new file mode 100644
index 0000000000..83844cd351
--- /dev/null
+++ b/doc/design/assistant/polls/prioritizing_special_remotes/comment_3_883a003b9c552b89f191135c582f99aa._comment
@@ -0,0 +1,14 @@
+[[!comment format=mdwn
+ username="https://www.google.com/accounts/o8/id?id=AItOawmsz4weoPXV2oEtv3zpo9dOxn_SEPz-7Iw"
+ nickname="Zooko"
+ subject="reasons to like Tahoe-LAFS as a special remote"
+ date="2012-10-12T18:17:42Z"
+ content="""
+Here are a couple of things which are (I think) unique about the Tahoe-LAFS special remote:
+
+1. encryption ; All of the data is encrypted before leaving your local system and heading for the server (or for the clouds). This is true even though you don't (I think) have to enter an encryption key into git-annex to access your data.
+
+(Note: the above implies that you're in danger of permanently losing access to your data, by losing the last copy of the encryption key, if your local git-annex state is lost. This deserves careful consideration.)
+
+2. erasure-coding ; You can configure Tahoe-LAFS to spread the data out in a RAID-like way across multiple remote storage servers, where each server holds only, say, 1/3 of the data, but there are, say, 10 different servers, where any 3 of them are sufficient to give you full access to your data. Does that make sense it uses less bandwidth and storage space than replication (i.e. putting a complete replica of your data on each of 4 or 5 or 10 different storage servers), but it is more robust than sharding (i.e. putting 1/3 of your data on each of three different servers so that if any one of them goes down you lose 1/3 of your data).
+"""]]
diff --git a/doc/design/assistant/polls/prioritizing_special_remotes/comment_4_746006c3fffc7f917c4526fd688051f7._comment b/doc/design/assistant/polls/prioritizing_special_remotes/comment_4_746006c3fffc7f917c4526fd688051f7._comment
new file mode 100644
index 0000000000..713d1d7b59
--- /dev/null
+++ b/doc/design/assistant/polls/prioritizing_special_remotes/comment_4_746006c3fffc7f917c4526fd688051f7._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="https://www.google.com/accounts/o8/id?id=AItOawnjrMQEhzd8xI81V9BL2jsKlNgVJLD7PKs"
+ nickname="Pankaj"
+ subject="Google Music as remote?"
+ date="2012-10-31T06:07:32Z"
+ content="""
+A dropbox like folder which syncs with Google Music. Google Music allows uploading upto 20K songs. Also using git-annex , I can ensure that I dont need to store all the duplicate mp3 on my local drive. Only one copy of \"older music\" is \"arvhived\" on GMusic, whereas more recent songs are on my local drive.
+"""]]
diff --git a/doc/design/assistant/polls/what_is_preventing_me_from_using_git-annex_assistant.mdwn b/doc/design/assistant/polls/what_is_preventing_me_from_using_git-annex_assistant.mdwn
new file mode 100644
index 0000000000..4b4778c1f4
--- /dev/null
+++ b/doc/design/assistant/polls/what_is_preventing_me_from_using_git-annex_assistant.mdwn
@@ -0,0 +1,16 @@
+My goal for this month is to get more people using the git-annex assistant,
+and fix issues that might be blocking you from using it. To do this,
+I'd like to get an idea about whether you're already using it,
+or what's keeping you from using it.
+
+If you use `git-annex` at the command line and have no reason to use the
+assistant, please instead fill in this poll on behalf of less technically
+adept friends or family -- what's preventing you from introducing them to
+the assistant?
+
+[[!poll open=no expandable=yes 8 "I'm using the assistant!" 28 "I need a Windows port" 30 "I need an Android port" 3 "I need an IPhone port (not holding my breath)" 2 "Well, it's still in beta..." 11 "I want to, but have not had the time to try it" 5 "Just inertia. I've got this dropbox/whatever that already works.." 3 "It's too hard to install (please say why in comments)" 2 "Perceived recent increase of bug reports and thus sitting it out." 25 "Initially the lack of direct-mode. Now concerns about the safety of direct mode. Perhaps after the next release." 10 "I haven't always well understood the differences between commandline operation & the assistant, so the differences would confuse me, and I found the command line more understandable & less scary.  Now trying to learn to like & trust the assistant. :)" 21 "An Ubuntu PPA would be supercool! Thanks for your great work!!" 18 "Not yet in Debian sid amd64" 6 "Waiting for Fedora/CentOS rpm repository." 2 "throttling transfers, it upsets people when I saturate the connection" 2 "partial content" 1 "Not yet available in macports" 4 "No build yet for Nokia N9" 3 "Using only git-annex webapp to config does not seem to work: Create walkthough?" 5 "No build for OSX 10.6" 5 "Needs more focus on the UI." 1 "Just inertia. I don't have a Dropbox/whatever." 4 "Replaces files with a symlink mess." 2 "configurable option to only annex files meeting certian size or filename criteria" 4 "I'm really confused about how to make it sync with a remote NON-bare repository. I'm even afraid to try `git remote add`, since there is no clear method to completely forget a git-annex remote..." 5 "A build for te raspberry pi would be supercol!" 1 "Would be nice to exclude subfolders from the gui or through a config file" 1 "I wish I had transparently encrypted git repos in the cloud available, like jgit." 1 "too many inodes used in direct mode. maybe it's possible to keep more info as git objects instead?" 2 "I need to be able to restrict in which repo dirs changes get auto-committed" 1 "Provide .deb package" 1 "Better documentation/walkthroughs on using git-annex within an existing git repo. AKA mixed use" 1 "Union mounts to have a single view of file collection on the network" 1 "Ubuntu PPA does not build with webapp" 1 "I set it up, but am confused about what I set up! It would be great to be able to start from scratch." 1 "I need to be able to restrict in which repo dirs changes get auto-committed using a syntax similar to gitignore"]]
+
+Feel free to write in your own reasons, or add a comment to give me more info.
+
+Note: Poll is now closed. Nearly all these issues have been dealt with.
+Please file bug reports if any of these issues still affect you.
diff --git a/doc/design/assistant/polls/what_is_preventing_me_from_using_git-annex_assistant/comment_10_10a4839a05be39ced54ffbe880a588bb._comment b/doc/design/assistant/polls/what_is_preventing_me_from_using_git-annex_assistant/comment_10_10a4839a05be39ced54ffbe880a588bb._comment
new file mode 100644
index 0000000000..b501be3b8b
--- /dev/null
+++ b/doc/design/assistant/polls/what_is_preventing_me_from_using_git-annex_assistant/comment_10_10a4839a05be39ced54ffbe880a588bb._comment
@@ -0,0 +1,25 @@
+[[!comment format=mdwn
+ username="https://www.google.com/accounts/o8/id?id=AItOawm5iosFbL2By7UFeViqkc6v-hoAtqILeDA"
+ nickname="Laszlo"
+ subject="comment 10"
+ date="2013-01-11T21:57:29Z"
+ content="""
+@joey: My usecase is fairly simple. 
+I have a working directory where I do work 
+(not programing, but text editing with homecooked spreadsheet/texteditor).
+
+I work/type on spot/on site. So I'm walking while typing.
+
+Its easy to do stupid things while you work, 
+so would be really handy to have a backup automagically on the same computer to prevent mistakes, user errors.
+(The last mistake, I did is I copied all my daily files into a \"tmp\" directory, 
+then I launched my homebrew program which also created a tmp dir then deleted the whole content of it. 
+I fixed it, but my datas was lost. It is a simple user mistake which could be prevented if I would have a 
+live backup all the time. Creating a git repo for daily typing is just too much overhead. 
+It is like you create a git repo for your text messages...)
+
+So my usecase is a live backup. Or the broken file deleting thing on unix/linux since forever.
+
+Laszlo
+
+"""]]
diff --git a/doc/design/assistant/polls/what_is_preventing_me_from_using_git-annex_assistant/comment_11_ac91d866f11c66dd8c86e2cd1a368c85._comment b/doc/design/assistant/polls/what_is_preventing_me_from_using_git-annex_assistant/comment_11_ac91d866f11c66dd8c86e2cd1a368c85._comment
new file mode 100644
index 0000000000..1657d11ad0
--- /dev/null
+++ b/doc/design/assistant/polls/what_is_preventing_me_from_using_git-annex_assistant/comment_11_ac91d866f11c66dd8c86e2cd1a368c85._comment
@@ -0,0 +1,10 @@
+[[!comment format=mdwn
+ username="https://www.google.com/accounts/o8/id?id=AItOawl9sYlePmv1xK-VvjBdN-5doOa_Xw-jH4U"
+ nickname="Richard"
+ subject="comment 11"
+ date="2013-01-11T22:17:07Z"
+ content="""
+@dzsino: You may want to have a look at [metamonger](https://github.com/RichiH/metamonger) which tries to solve exactly this problem. It's not finished yet, though.
+
+-- Richard
+"""]]
diff --git a/doc/design/assistant/polls/what_is_preventing_me_from_using_git-annex_assistant/comment_12_e244c1bf334b1cc9ad0cc760bf8fe5de._comment b/doc/design/assistant/polls/what_is_preventing_me_from_using_git-annex_assistant/comment_12_e244c1bf334b1cc9ad0cc760bf8fe5de._comment
new file mode 100644
index 0000000000..fc2b03baca
--- /dev/null
+++ b/doc/design/assistant/polls/what_is_preventing_me_from_using_git-annex_assistant/comment_12_e244c1bf334b1cc9ad0cc760bf8fe5de._comment
@@ -0,0 +1,29 @@
+[[!comment format=mdwn
+ username="https://www.google.com/accounts/o8/id?id=AItOawkx5V3MTbzCXS3J7Mn9FEq8M9bPPYMkAHY"
+ nickname="Pedro"
+ subject="It fails silently and leaves symlinks behind"
+ date="2013-01-11T23:49:51Z"
+ content="""
+I dedicated a day to looking into using the assistant and although it's clearly an exciting piece of software it's still pretty hard to get working. To get two machines to sync with one another the options seem to be:
+
+1. Connect both of them to a XMPP server: That makes metadata sync flawlessly and yet leaves a symlink mess behind. As it turns out that's because XMPP syncs only metadata but nowhere was that stated on the webapp (that I could find) and no error message given
+2. So then I tried the sync method with SSH. 
+
+For that you'll need to:
+
+* Have SSH servers installed on both endpoints (for me that was easy to do but I'm sure most users will have a harder time)
+* Make sure both computers are actually on the same broadcast network otherwise they won't see each other (I was testing with a VM behind virtualbox's NAT so initially it didn't work)
+* Install the assistant on both sides and make sure both the base program and the shell wrapper are in the PATH. (this was harder to do than needed as in the standalone bundle they're not on the same folder and the shell wrapper needs to be run by the runshell script, so even if you get it in the path it will still fail with missing libraries. An Ubuntu PPA would have solved this I'm sure.
+
+For some reason I couldn't figure out they still couldn't sync and symlink mess ensued.
+
+I think there are a few usability issues here:
+
+1. Direct mode should never overwrite files in a way you can no longer use them just because metadata updates came before content updates. I'd rather have an older version of the file than lose access to it.
+2. Error reporting needs to be a lot more explicit to be able to debug these issues
+3. At the best of times SSH sync is going to be a problem. In my opinion to be really dropbox-like the sync method for normal users should be to connect all devices to the same XMPP account (this works great now) and then have those assistants automatically reach each other by a direct connection when possible and by something like STUN if NAT punching is needed. The SSH/bup/etc remotes all have their place for advanced setups but being able to tell the user to just connect to XMPP and be done would turn this from \"something I'll spend an afternoon writing puppet manifests to deploy in my personal servers/computers\" to \"something I can explain to a non-technical friend how to use over a phone call and then share files with him\".
+
+On the XMPP side though, it would work best if you could authenticate a Google XMPP acount through an OAuth workflow instead of asking for the password. I wouldn't give any program my gmail password but I'd easily give it permission to use gtalk on my behalf.
+
+As I started by saying the assistant (and git-annex in general) is an extremely impressive piece of software that I'm very excited about. I hope it keeps improving at the current pace as there are still a few features I'd love to have (partial content particularly) but by far the thing that's keeping me most from using it is the ability to easily get a reliable sync going that won't leave me with a folder full of symlinks that I don't know what to do about. One of the great things about dropbox is that whenever it isn't running or can't sync for any reason your folder degenerates into a folder like any other on your disk. The right way of thinking about direct mode should be that one, \"a folder like any other that we stream changes from and to on multiple machines\". Right now, with the symlink replacement when sync isn't possible and to a lesser extent the adding of .git folders it doesn't live up to that guarantee.
+"""]]
diff --git a/doc/design/assistant/polls/what_is_preventing_me_from_using_git-annex_assistant/comment_13_1a0faf4bdc78741937e8a2f5cb5bbec6._comment b/doc/design/assistant/polls/what_is_preventing_me_from_using_git-annex_assistant/comment_13_1a0faf4bdc78741937e8a2f5cb5bbec6._comment
new file mode 100644
index 0000000000..066859b32b
--- /dev/null
+++ b/doc/design/assistant/polls/what_is_preventing_me_from_using_git-annex_assistant/comment_13_1a0faf4bdc78741937e8a2f5cb5bbec6._comment
@@ -0,0 +1,12 @@
+[[!comment format=mdwn
+ username="https://www.google.com/accounts/o8/id?id=AItOawm5iosFbL2By7UFeViqkc6v-hoAtqILeDA"
+ nickname="Laszlo"
+ subject="comment 13"
+ date="2013-01-12T08:16:48Z"
+ content="""
+@Pedro: That is an excellent test, and usecase!
+
+Indeed, the gui needs a lot more info on it. 
+Also file integratity is a real fear with this kind of application.
+
+"""]]
diff --git a/doc/design/assistant/polls/what_is_preventing_me_from_using_git-annex_assistant/comment_14_8d8a11dbfae7a7bc574bdf37f87e0684._comment b/doc/design/assistant/polls/what_is_preventing_me_from_using_git-annex_assistant/comment_14_8d8a11dbfae7a7bc574bdf37f87e0684._comment
new file mode 100644
index 0000000000..1ecaff59f0
--- /dev/null
+++ b/doc/design/assistant/polls/what_is_preventing_me_from_using_git-annex_assistant/comment_14_8d8a11dbfae7a7bc574bdf37f87e0684._comment
@@ -0,0 +1,16 @@
+[[!comment format=mdwn
+ username="https://launchpad.net/~rubiojr"
+ nickname="rubiojr"
+ subject="git-annex PPA"
+ date="2013-01-15T14:16:35Z"
+ content="""
+Hey Joey,
+
+I do maintain a PPA with a more up2date version in fact:
+
+https://launchpad.net/~rubiojr/+archive/git-annex
+
+Now that the direct mode is in place, I guess it's probably a good time to backport again.
+
+Thanks for such a great piece of software.
+"""]]
diff --git a/doc/design/assistant/polls/what_is_preventing_me_from_using_git-annex_assistant/comment_15_c437adeaccf0b3d134e0f81c64e25b9f._comment b/doc/design/assistant/polls/what_is_preventing_me_from_using_git-annex_assistant/comment_15_c437adeaccf0b3d134e0f81c64e25b9f._comment
new file mode 100644
index 0000000000..e6e4af37e2
--- /dev/null
+++ b/doc/design/assistant/polls/what_is_preventing_me_from_using_git-annex_assistant/comment_15_c437adeaccf0b3d134e0f81c64e25b9f._comment
@@ -0,0 +1,14 @@
+[[!comment format=mdwn
+ username="https://launchpad.net/~rubiojr"
+ nickname="rubiojr"
+ subject="direct mode still 'untrusted'?"
+ date="2013-01-15T14:23:22Z"
+ content="""
+BTW,
+
+Does this still apply?
+
+\"I'd like git annex direct to set the repository to untrusted, but I didn't do it. Partly because having git annex indirect set it back to semitrusted seems possibly wrong -- the user might not trust a repo even in indirect mode. Or might fully trust it. The docs will encourage users to set direct mode repos to untrusted -- in direct mode you're operating without large swathes of git-annex's carefully constructed safety net. (When the assistant later uses direct mode, it'll untrust the repository automatically.)\"
+
+http://git-annex.branchable.com/design/assistant/blog/day_151__direct_mode_toggle/
+"""]]
diff --git a/doc/design/assistant/polls/what_is_preventing_me_from_using_git-annex_assistant/comment_16_6e3fce3a32ab346dc3d0fd4b69967536._comment b/doc/design/assistant/polls/what_is_preventing_me_from_using_git-annex_assistant/comment_16_6e3fce3a32ab346dc3d0fd4b69967536._comment
new file mode 100644
index 0000000000..afda489a13
--- /dev/null
+++ b/doc/design/assistant/polls/what_is_preventing_me_from_using_git-annex_assistant/comment_16_6e3fce3a32ab346dc3d0fd4b69967536._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="http://adamspiers.myopenid.com/"
+ nickname="Adam"
+ subject="just one thing"
+ date="2013-01-16T01:45:54Z"
+ content="""
+http://git-annex.branchable.com/todo/wishlist:_disable_automatic_commits/ is the only thing I'm waiting for.  Will become a very keen beta-tester (maybe even coder) when that happens.
+"""]]
diff --git a/doc/design/assistant/polls/what_is_preventing_me_from_using_git-annex_assistant/comment_17_1b7233d88593d0d99b26ea3e7af20d9c._comment b/doc/design/assistant/polls/what_is_preventing_me_from_using_git-annex_assistant/comment_17_1b7233d88593d0d99b26ea3e7af20d9c._comment
new file mode 100644
index 0000000000..278181ad51
--- /dev/null
+++ b/doc/design/assistant/polls/what_is_preventing_me_from_using_git-annex_assistant/comment_17_1b7233d88593d0d99b26ea3e7af20d9c._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="http://joeyh.name/"
+ ip="4.154.7.238"
+ subject="comment 17"
+ date="2013-01-16T19:38:36Z"
+ content="""
+@rubiojr I recommend configuring direct mode repos as untrusted if you care about accessing old versions of your files as stored in git. If that is not a concern, direct mode is safe, it's just that it allows editing/deleting any file at any time, even if that's the only copy of the file.
+"""]]
diff --git a/doc/design/assistant/polls/what_is_preventing_me_from_using_git-annex_assistant/comment_18_a23d5a0e2718b8e486f036fe8a413b36._comment b/doc/design/assistant/polls/what_is_preventing_me_from_using_git-annex_assistant/comment_18_a23d5a0e2718b8e486f036fe8a413b36._comment
new file mode 100644
index 0000000000..5f82d7ebb6
--- /dev/null
+++ b/doc/design/assistant/polls/what_is_preventing_me_from_using_git-annex_assistant/comment_18_a23d5a0e2718b8e486f036fe8a413b36._comment
@@ -0,0 +1,10 @@
+[[!comment format=mdwn
+ username="http://christian.amsuess.com/chrysn"
+ nickname="chrysn"
+ subject="comment 18"
+ date="2013-01-17T12:04:34Z"
+ content="""
+the \"I haven't always well understood the differences between commandline operation & the assistant, so the differences would confuse me, and I found the command line more understandable & less scary. Now trying to learn to like & trust the assistant.\" fits pretty well for me -- i'd like to use the webapp at least for viewing, but i'm worried that some auto-magic would kick in, and i regularly have uncommitted stuff in at least one git-annex repo.
+
+a --dry-run flag or similar on the `git annex webapp` would be nice, or a description of what would happen or how to turn that off / make it create other branches / even commit or push something.
+"""]]
diff --git a/doc/design/assistant/polls/what_is_preventing_me_from_using_git-annex_assistant/comment_19_f4c84a9d701d52cf2f2e45f3d764a90c._comment b/doc/design/assistant/polls/what_is_preventing_me_from_using_git-annex_assistant/comment_19_f4c84a9d701d52cf2f2e45f3d764a90c._comment
new file mode 100644
index 0000000000..35def93746
--- /dev/null
+++ b/doc/design/assistant/polls/what_is_preventing_me_from_using_git-annex_assistant/comment_19_f4c84a9d701d52cf2f2e45f3d764a90c._comment
@@ -0,0 +1,18 @@
+[[!comment format=mdwn
+ username="https://www.google.com/accounts/o8/id?id=AItOawnmvVKc1ECzZx7EhtLHBP6RWPZewq4x_9I"
+ nickname="Daniele"
+ subject="Windows + git ignore like syntax for files to sync"
+ date="2013-04-19T10:47:36Z"
+ content="""
+Hi,
+
+my company would need this for some project.
+
+But to adopt it I would need:
+* windows support (I don't care if now symlink support for now)
+* being able to define which files are automatically synced using a syntax similar to the gitignore one
+* being able to disable auto-commit (just synching, user should be able to commit on their own)
+
+thanks
+
+"""]]
diff --git a/doc/design/assistant/polls/what_is_preventing_me_from_using_git-annex_assistant/comment_1_00a0de8190d946caaeeca3b44646146f._comment b/doc/design/assistant/polls/what_is_preventing_me_from_using_git-annex_assistant/comment_1_00a0de8190d946caaeeca3b44646146f._comment
new file mode 100644
index 0000000000..4bebe748d1
--- /dev/null
+++ b/doc/design/assistant/polls/what_is_preventing_me_from_using_git-annex_assistant/comment_1_00a0de8190d946caaeeca3b44646146f._comment
@@ -0,0 +1,16 @@
+[[!comment format=mdwn
+ username="https://www.google.com/accounts/o8/id?id=AItOawl9sYlePmv1xK-VvjBdN-5doOa_Xw-jH4U"
+ nickname="Richard"
+ subject="Two reasons"
+ date="2013-01-10T18:00:05Z"
+ content="""
+As of right now, the assistant has only secondary benefit over plain git-annex inasmuch there's a lot more code activity.
+
+As soon as the assistant supports Android, I will use for syncing photos off of my phone and may start to use the assistant on my usual repos as a natural consequence.
+
+Additionally, there's a subjective feeling of more bugs being reported. That may or may not be true, but as long as there's no Android port, I don't have an actual reason to \"risk\" it.
+
+-- Richard
+
+PS: This seems to be the first poll where you can place only one vote while it's the first where I really wanted to vote on two separate items.
+"""]]
diff --git a/doc/design/assistant/polls/what_is_preventing_me_from_using_git-annex_assistant/comment_20_199c9807499470771af6cbca6d034cfa._comment b/doc/design/assistant/polls/what_is_preventing_me_from_using_git-annex_assistant/comment_20_199c9807499470771af6cbca6d034cfa._comment
new file mode 100644
index 0000000000..51e1bc4ea0
--- /dev/null
+++ b/doc/design/assistant/polls/what_is_preventing_me_from_using_git-annex_assistant/comment_20_199c9807499470771af6cbca6d034cfa._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="http://joeyh.name/"
+ nickname="joey"
+ subject="comment 20"
+ date="2013-04-19T18:01:44Z"
+ content="""
+You can turn off automatic commits in the webapp by pausing syncing for the local repository. 
+"""]]
diff --git a/doc/design/assistant/polls/what_is_preventing_me_from_using_git-annex_assistant/comment_21_9185b0e05b1b1997533694da1de83073._comment b/doc/design/assistant/polls/what_is_preventing_me_from_using_git-annex_assistant/comment_21_9185b0e05b1b1997533694da1de83073._comment
new file mode 100644
index 0000000000..5ec65ad98a
--- /dev/null
+++ b/doc/design/assistant/polls/what_is_preventing_me_from_using_git-annex_assistant/comment_21_9185b0e05b1b1997533694da1de83073._comment
@@ -0,0 +1,22 @@
+[[!comment format=mdwn
+ username="https://www.google.com/accounts/o8/id?id=AItOawmH-n1yD04qmSeXKKzYaXFTN1ciFWVb5As"
+ nickname="Martyn"
+ subject="Can't seem to get the configuration I am looking for from the webapp"
+ date="2014-01-02T23:41:44Z"
+ content="""
+Currently, I have a local server, desktop and laptop (all on the same network). My laptop leaves the house occasionally where I work on it...
+Quite simply, I want make the local server the centralised repository and have the desktop + laptop sync up and down to and from the server. The server would act as a backup and file change history between all local repositories on desktop and laptop. So I would sync between trips away.
+
+Sadly every combination with git-annex webapp fails for a number of reasons, it could be just me but here is what I've experienced:
+
+a) I want to use direct mode ALL the time or at least be able to configure it without the command line. Currently server side set ups are indirect depending on the repository group. So no symlinks basically. I think \"full backup\" and \"client\" are the only two that do what I was hoping for here.
+b) The list of repository group types miss something I'm looking for, I want the \"full backup\" with \"transfer\" for the remote server - I guess? 
+c) For remote cloud solutions, I would like the files to exist, not just the raw .git/ directory to exist on the server. When I tested it, I couldn't see a way to test on the server itself the integrity of the files (could be my lack of git-annex knowledge :)
+d) I would like it if the assistant made it simpler to configure remotes, i.e. right now if I have 2 directories I believe I would need 4 configurations to push/pull locally and remotely to the 1 server with each machine. I've had this working and it's the closest I've got so far, but tedious if I want to do it for 5 directories on 2 machines (that's 20 configurations to add in all unless there is an easier way?)
+e) The \"share with other devices\" option seems one of the best right now, but it suffers with the issues of c) AFAICT. Why not have the server just be a remote when unencrypted?
+
+From some reading up on the command line tools, it seems like it would be easier to set up using those for what I want than using the webapp.
+I currently love the webapp, it's just missing some polish and options to do what I need.
+
+Thanks for the great work so far! :)
+"""]]
diff --git a/doc/design/assistant/polls/what_is_preventing_me_from_using_git-annex_assistant/comment_2_35f6f121e54260cb960211a6e2e51e8e._comment b/doc/design/assistant/polls/what_is_preventing_me_from_using_git-annex_assistant/comment_2_35f6f121e54260cb960211a6e2e51e8e._comment
new file mode 100644
index 0000000000..bf52b9c1c2
--- /dev/null
+++ b/doc/design/assistant/polls/what_is_preventing_me_from_using_git-annex_assistant/comment_2_35f6f121e54260cb960211a6e2e51e8e._comment
@@ -0,0 +1,14 @@
+[[!comment format=mdwn
+ username="https://www.google.com/accounts/o8/id?id=AItOawniCRkhl_W87gOK5eElfsef3FoUsUFpAr4"
+ nickname="Alexandre"
+ subject="Two assistants with shared transfer repository"
+ date="2013-01-11T07:17:12Z"
+ content="""
+I voted for the Windows port, which unfortunately I must use at work.
+
+But another thing that Dropbox gets right is that you can [sync two clients with one shared repository](http://git-annex.branchable.com/forum/How_to_set_up_two_assistants_with_one_shared_transfer_repository__63__) that works without any configuration whether you are home or somewhere else. This use case is covered by sync over XMPP, but there are two small issues with this :
+- if I use Google XMPP servers to sync two assistants in the same network, everything goes through the slower Internet link
+- if I setup my own XMPP server, well, this is more setup, more open ports, etc.
+
+It would be nice to have repository pushs on a shared repository notified to the connected clients in the SSH connection.
+"""]]
diff --git a/doc/design/assistant/polls/what_is_preventing_me_from_using_git-annex_assistant/comment_3_acbe4f63b5d552ac5ae5a12c6f42dc18._comment b/doc/design/assistant/polls/what_is_preventing_me_from_using_git-annex_assistant/comment_3_acbe4f63b5d552ac5ae5a12c6f42dc18._comment
new file mode 100644
index 0000000000..0d971c29c1
--- /dev/null
+++ b/doc/design/assistant/polls/what_is_preventing_me_from_using_git-annex_assistant/comment_3_acbe4f63b5d552ac5ae5a12c6f42dc18._comment
@@ -0,0 +1,25 @@
+[[!comment format=mdwn
+ username="http://a-or-b.myopenid.com/"
+ ip="220.244.41.108"
+ subject="comment 3"
+ date="2013-01-11T09:32:33Z"
+ content="""
+I've got a few wishes for git-annex…  Referring to the poll options:
+
+- I need an Android port
+
+I think this is the biggest issue for getting a high WAF.  (I want to put our photos in an annex, and getting them auto-magically off her phone would get me massive brownie points.)
+
+- Initially the lack of direct-mode. Now concerns about the safety of direct mode. Perhaps after the next release.
+
+The \"potentially unsafe\" sort of comments in the blog make me worry about trusting my data to the direct mode.  Saying that I *really* appreciate the honesty of the comments.  That is why I want to keep using, and recommend to others, git-annex.
+
+- I haven't always well understood the differences between commandline operation & the assistant, so the differences would confuse me, and I found the command line more understandable & less scary. Now trying to learn to like & trust the assistant.
+
+I also have worried about the automatic adding by the assistant - prior to direct mode.
+
+- throttling transfers, it upsets people when I saturate the connection
+
+I think a bittorrent option would be ideal.  Not public torrents, but torrents between my annexes.
+
+"""]]
diff --git a/doc/design/assistant/polls/what_is_preventing_me_from_using_git-annex_assistant/comment_4_0d988280865caae498a3b693b6342e37._comment b/doc/design/assistant/polls/what_is_preventing_me_from_using_git-annex_assistant/comment_4_0d988280865caae498a3b693b6342e37._comment
new file mode 100644
index 0000000000..b66cd12f7e
--- /dev/null
+++ b/doc/design/assistant/polls/what_is_preventing_me_from_using_git-annex_assistant/comment_4_0d988280865caae498a3b693b6342e37._comment
@@ -0,0 +1,16 @@
+[[!comment format=mdwn
+ username="https://www.google.com/accounts/o8/id?id=AItOawm5iosFbL2By7UFeViqkc6v-hoAtqILeDA"
+ nickname="Laszlo"
+ subject="Two local repositories"
+ date="2013-01-11T10:31:54Z"
+ content="""
+Here is my usecase:
+I want two local (direct mode) repositories. So basically a working directory and a backup directory to the working one.
+
+The removable media repository *almost* fulfill this usecase, except you can only select drive and not a directory.
+I think it would be relatively simple to implement. (A drive is a directory under linux after all).
+
+Best, 
+ Laszlo
+
+"""]]
diff --git a/doc/design/assistant/polls/what_is_preventing_me_from_using_git-annex_assistant/comment_5_ac8fe3768c30dd7999c183500f8567bb._comment b/doc/design/assistant/polls/what_is_preventing_me_from_using_git-annex_assistant/comment_5_ac8fe3768c30dd7999c183500f8567bb._comment
new file mode 100644
index 0000000000..713e8ef7d9
--- /dev/null
+++ b/doc/design/assistant/polls/what_is_preventing_me_from_using_git-annex_assistant/comment_5_ac8fe3768c30dd7999c183500f8567bb._comment
@@ -0,0 +1,19 @@
+[[!comment format=mdwn
+ username="marvin"
+ ip="91.154.225.32"
+ subject="SSH keys"
+ date="2013-01-11T12:02:50Z"
+ content="""
+Really looking forward to seeing this project go forward and impressed by what you have accomplished.
+
+I have my own server which I want to use as a repository, so I'll choose \"Remote server\". I want to use direct mode.
+Now I can only create an encrypted remote which is fine for me, but I don't know how to sync that with another client using this remote.
+
+Also with the generated ssh keys I no longer can log into my server normally as it tries to use the generated keys.
+The generated keys should also be restricted to only be able to change the repository and nothing else.
+
+Maybe webdav could be used to create something with better access control and repositories for groups etc.
+Also with a simple web upload/download page, similar to dropbox.
+
+Sorry if my post is long and idiotic, just simple (hopefully constructive) feedback from a simple user :)
+"""]]
diff --git a/doc/design/assistant/polls/what_is_preventing_me_from_using_git-annex_assistant/comment_6_36832de705a2bebf8dc6e65dcd661731._comment b/doc/design/assistant/polls/what_is_preventing_me_from_using_git-annex_assistant/comment_6_36832de705a2bebf8dc6e65dcd661731._comment
new file mode 100644
index 0000000000..525a325f03
--- /dev/null
+++ b/doc/design/assistant/polls/what_is_preventing_me_from_using_git-annex_assistant/comment_6_36832de705a2bebf8dc6e65dcd661731._comment
@@ -0,0 +1,15 @@
+[[!comment format=mdwn
+ username="http://edheil.wordpress.com/"
+ ip="173.162.44.162"
+ subject="comment 6"
+ date="2013-01-11T15:06:47Z"
+ content="""
+Laszlo's concern about \"you can only select a drive and not a directory\" relates to a problem I had.
+
+I have two local directories, ~/annex indirect my main annex, and ~/directannex for playing with direct mode.
+
+I had hoped to back each up to the same shared drive, but didn't realize there was no option to select a directory within a shared drive, so when I tried to do that through the assistant I inadvertently connected the two by backing them up to the same directory on the shared drive!
+
+This is probably a \"power user problem\" which has a \"power user solution\" of setting up the separate repository manually.  But it did take me by surprise.
+
+"""]]
diff --git a/doc/design/assistant/polls/what_is_preventing_me_from_using_git-annex_assistant/comment_7_3618067e473577a112e36970ca71e0ab._comment b/doc/design/assistant/polls/what_is_preventing_me_from_using_git-annex_assistant/comment_7_3618067e473577a112e36970ca71e0ab._comment
new file mode 100644
index 0000000000..bfb1b949ce
--- /dev/null
+++ b/doc/design/assistant/polls/what_is_preventing_me_from_using_git-annex_assistant/comment_7_3618067e473577a112e36970ca71e0ab._comment
@@ -0,0 +1,12 @@
+[[!comment format=mdwn
+ username="https://dzsino.myopenid.com/"
+ nickname="dzsino"
+ subject="Recording file creation dates?"
+ date="2013-01-11T16:35:27Z"
+ content="""
+I've just started using git annex, command line feels safer for the time being. One issue I've got with assistant that I had to kill it couple of times when it got confused over something (could be that I used command line git annex at the same time?),  shutdown from the UI is welcome.
+
+More of a git annex issue for me that I lost file creation dates for annexed files, which i don't really mind for most of my content, except really old photos without proper EXIF tags.
+I think of git annex as more of a DVR for my digital life, rather than a mere sync tool, so I would welcome some basic metadata facility. Would you consider adding this?
+(Really getting carried away, I would want to full text search on file names, ID3 tags, document metadata and text etc. and requesting files based on search hits, spotlight style..)
+"""]]
diff --git a/doc/design/assistant/polls/what_is_preventing_me_from_using_git-annex_assistant/comment_8_07a13b6f000ddc0ac4472b863d8b50bd._comment b/doc/design/assistant/polls/what_is_preventing_me_from_using_git-annex_assistant/comment_8_07a13b6f000ddc0ac4472b863d8b50bd._comment
new file mode 100644
index 0000000000..8bbd57b5d2
--- /dev/null
+++ b/doc/design/assistant/polls/what_is_preventing_me_from_using_git-annex_assistant/comment_8_07a13b6f000ddc0ac4472b863d8b50bd._comment
@@ -0,0 +1,14 @@
+[[!comment format=mdwn
+ username="http://joeyh.name/"
+ ip="4.154.7.238"
+ subject="comment 8"
+ date="2013-01-11T20:02:39Z"
+ content="""
+@Alexandre, what you want can be achieved by setting up XMPP pairing, and also [[local pairing|assistant/local_pairing_walkthrough]]. Then when computers are on the same network, transfers are done via the LAN.
+
+@Laszlo, this can be set up without using the webapp. Just make the two repositories, using the webapp. Then go into each and run \"git remote add myotherrepo $path_to_other_repo\". Then assistant will automatically sync them. I *have* thought about adding a configurator for this to the webapp. It would help to know what use cases you're getting at with this.
+
+@marvin The generated ssh keys are configured to only allow running `git-annex-shell`, which limits it to only acting on the repository. The keys are also configured, in `.ssh/config` to only be used when a particular host alias is used. They should not be used when you just ssh normally to the host. If this is happening to you, please file a bug report with details.
+
+@dzsino you can get file creation dates out of git using git log :)
+"""]]
diff --git a/doc/design/assistant/polls/what_is_preventing_me_from_using_git-annex_assistant/comment_9_e15eb407d988fda363296c8b566cc8fb._comment b/doc/design/assistant/polls/what_is_preventing_me_from_using_git-annex_assistant/comment_9_e15eb407d988fda363296c8b566cc8fb._comment
new file mode 100644
index 0000000000..f7dd3eb884
--- /dev/null
+++ b/doc/design/assistant/polls/what_is_preventing_me_from_using_git-annex_assistant/comment_9_e15eb407d988fda363296c8b566cc8fb._comment
@@ -0,0 +1,12 @@
+[[!comment format=mdwn
+ username="https://www.google.com/accounts/o8/id?id=AItOawkQafKy7hNSEolLs6TvbgUnkklTctUY9LI"
+ nickname="Zellyn"
+ subject="Question about git-annex"
+ date="2013-01-11T21:05:55Z"
+ content="""
+Not sure where the correct place to ask this is...
+
+My use case for git-annex is to back up important files (pictures, videos, etc.) on a hard disk at work and one at home. Is there an easy way to do that? I'd like to be able to get git-annex to fill up my phone/laptop with files that it thinks are backed up in <2 of the locations, and ferry them back and forth. They can be deleted to free up space after that, if necessary.
+
+Thanks for all your work, by the way. I missed the kickstarter, but if there's a simple and direct way to donate (dwolla?, gittip?), I'd love to chip in.
+"""]]
diff --git a/doc/design/assistant/progressbars.mdwn b/doc/design/assistant/progressbars.mdwn
new file mode 100644
index 0000000000..7de70452d0
--- /dev/null
+++ b/doc/design/assistant/progressbars.mdwn
@@ -0,0 +1,43 @@
+Currently, git-annex takes a very lazy approch to displaying
+progress into. It just lets rsync or whatever display the progress
+for it, in the terminal.
+
+Something better is needed for the [[webapp]]. There needs to be a
+way for the web app to know what the current progress is of all transfers.
+
+This is one of those potentially hidden but time consuming problems.
+
+## downloads
+
+* Watch temp file as it's coming in and use its size.
+  Can either poll every .5 seconds or so to check file size, or
+  could use inotify. **done**
+* When easily available, remotes call the MeterUpdate callback as downloads
+  progress. **done**
+* S3: TODO  
+  While it has a download progress bar, `getObject` probably buffers the whole
+  download in memory before returning. Leaving the progress bar to only
+  display progress for writing the file out of memory. Fixing this would
+  involve making hS3 stream better (also avoids it wasting memory).
+
+## uploads
+
+Each individual remote type needs to implement its own support for calling
+the MeterUpdate callback as the upload progresses.
+
+* git: **done**
+* rsync: **done**
+* directory: **done**
+* web: Not applicable; does not upload 
+* webdav: **done**
+* S3: **done**
+* glacier: **done**
+* bup: **done**
+* hook: Would require the hook interface to somehow do this, which seems
+  too complicated. So skipping.
+
+## communication
+
+It may be worth using a better communication channel than files on disk for
+the transfer progress. Shared memory could be used, or http posts to the
+webapp.
diff --git a/doc/design/assistant/progressbars/comment_1_3ea263b1f334e8e38e14f00a96202988._comment b/doc/design/assistant/progressbars/comment_1_3ea263b1f334e8e38e14f00a96202988._comment
new file mode 100644
index 0000000000..4a011f61be
--- /dev/null
+++ b/doc/design/assistant/progressbars/comment_1_3ea263b1f334e8e38e14f00a96202988._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="http://abhidg.myopenid.com/"
+ ip="129.67.132.87"
+ subject="librsync"
+ date="2012-06-13T02:14:29Z"
+ content="""
+There's librsync which might support reporting the progress through its API, but it seems to be in beta.
+"""]]
diff --git a/doc/design/assistant/rate_limiting.mdwn b/doc/design/assistant/rate_limiting.mdwn
new file mode 100644
index 0000000000..3ab804329a
--- /dev/null
+++ b/doc/design/assistant/rate_limiting.mdwn
@@ -0,0 +1,57 @@
+Webapp needs a simple speed control knob, especially to avoid saturating 
+bandwidth on uploads.
+
+We have basic throttling support in git-annex for rsync,
+but none for any special remotes. A good first step would be to expose
+this in the webapp, and ensure that `git-annex-shell` also honors it when
+sending/receiving data.
+
+We actually need two speed controls, one for upload and one for download.
+
+It is probably not necessary to throttle git push/pull operations, as the
+data transfers tend to be small. Only throttling file transfers is
+essential.
+
+## possibility: trickle
+
+Since `git-annex transferkeys` is a separate process, one easy option would
+be to run it inside `trickle`. If the user changes the bandwidth limits,
+it could kill the transfer and restart it with different trickle options.
+
+Problem: Not all special remotes support resuming transfers, so this is
+suboptimal. (So too are the pause/resume buttons, when using those
+remotes!)
+
+`trickle` is available for OSX as well as Linux and BSDs.
+
+
+It is probably not easily available for Android, as it uses `LD_PRELOAD`.
+
+## possibility: built in IO limiting
+
+A cleaner method would be to do the limiting inside git-annex. We already
+have metered file IO. It should be possible to make the meter not only report
+on the transfer speed, but detect when it's going too fast, and delay. This
+will delay the file streaming through the special remote's transfer code,
+so should work for a variety of special remotes. (Not for rsync or bup
+or git-annex-shell though, so those need to be handled separately.)
+
+Should work well for uploads at least. I don't know how well it would work
+for throttling downloads; the sender may just keep sending data and the
+data buffer before it gets to the IO meter. Maybe once the buffers fill the
+OS would have the TCP throttled down. Needs investigation; trickle claims
+to throttle downloads.
+
+## communications channels
+
+There would need to be a communication channel for the assistant to tell
+`git annex transferkeys` when the rate limit has changed. It could for
+example send it a SIGUSR1, and then leave it up to the process to reload
+the git config. Inside the IO meter, we could have an MVar that contains
+the current throttle value, so the IO meter could check it each time it's
+called and adjust its throttling appropriately.
+
+Ideally, the assistant could also communicate in the same way with
+`git-annex-shell` to tell it when the limit has changed. Since
+`git-annex-shell` uses rsync, it would need to abort the transfer, and rely
+on the other side retrying to start it up with the new limit.
diff --git a/doc/design/assistant/screenshot/firstrun.png b/doc/design/assistant/screenshot/firstrun.png
new file mode 100644
index 0000000000..7e9d505b40
Binary files /dev/null and b/doc/design/assistant/screenshot/firstrun.png differ
diff --git a/doc/design/assistant/screenshot/intro.png b/doc/design/assistant/screenshot/intro.png
new file mode 100644
index 0000000000..23ed49d677
Binary files /dev/null and b/doc/design/assistant/screenshot/intro.png differ
diff --git a/doc/design/assistant/sshpassword.mdwn b/doc/design/assistant/sshpassword.mdwn
new file mode 100644
index 0000000000..59f981bb16
--- /dev/null
+++ b/doc/design/assistant/sshpassword.mdwn
@@ -0,0 +1,65 @@
+Currently the assistant sets up dedicated ssh keys, that can just use
+git-annex. This is ok. The problem is that the initial 2 connections to the
+ssh server when setting up these keys involve a password prompt, which is
+done at the console unless the system happens to have a working ssh agent
+that can pop up a dialog. That can be confusing.
+
+It would be nice to have the webapp prompt for the password. Can it be done
+securely?
+
+This might come down to a simple change to the webapp to prompt for the
+password, and then rather a lot of pain to make the webapp use HTTPS so we
+can be pretty sure noone is sniffing the (localhost) connection.
+
+## ssh-askpass approach
+
+* If ssh-askpass is in PATH, or `SSH_ASKPASS` is set, do nothing.
+  (Unless webapp is run remotely.)  
+  XXX not currently done; the UI would need to omit the password entry
+  fields in this case.
+* Otherwise, have the assistant set `SSH_ASKPASS` to a command that will
+  cause the webapp to read the password and forward it on. Also, set
+  DISPLAY to ensure that ssh runs the program. **done**
+
+Looking at ssh.exe, I think this will even work on Windows; it contains the
+code to run ssh-askpass. (It does work on Windows!)
+
+### securely handling the password
+
+* Maybe force upgrade webapp to https? Locally, the risk would be that
+  root could tcpdump and read password, so not large risk. If webapp
+  is being accessed remotely, absolutely: require https.
+* Use hs-securemem to store password.
+* Avoid storing password for long. Erase it after webapp setup of remote
+  is complete. Time out after 10 minutes and erase it. **done**
+* If the user is slow, the cached ssh key can exire before they finish.
+  This results in ssh being given no password, and failing. The UI
+  now detects this and suggests the user retry. **done**
+* Prompt using a html field name that does not trigger web browser password
+  saving if possible.
+
+### ssh-askpass shim, and password forwarding
+
+`SSH_ASKPASS` needs to be set to a program (probably git-annex)
+which gets the password from the webapp, and outputs it to stdout. **done**
+
+Seems to call for the webapp and program to communicate over a local
+socket (locked down so only user can access) or environment.
+Environment is not as secure (easily snooped by root).
+Local socket probably won't work on Windows. Could just use a temp file.
+
+(Currently uses a temp file with locked down perms that it's careful
+to clean up after use.)
+
+Note that the webapp can probe to see if ssh needs a password, and can
+prompt the user for it before running ssh and the ssh-askpass shim.
+This avoids some complexity, and perhaps some attack vectors,
+if the shim cannot requst an arbitrary password prompt.
+(This complexity not needed with the temp file approach..)
+
+### TODO
+
+* test on OSX
+* test on Android
+* remove the vestigial terminal on Windows and Android, since this was the
+  last thing actually using it (not easy!)
diff --git a/doc/design/assistant/sshpassword/comment_1_24399abe0a0c1de271490ee15e064760._comment b/doc/design/assistant/sshpassword/comment_1_24399abe0a0c1de271490ee15e064760._comment
new file mode 100644
index 0000000000..e71806fa29
--- /dev/null
+++ b/doc/design/assistant/sshpassword/comment_1_24399abe0a0c1de271490ee15e064760._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="https://www.google.com/accounts/o8/id?id=AItOawnXybLxkPMYpP3yw4b_I6IdC3cKTD-xEdU"
+ nickname="Matt"
+ subject="comment 1"
+ date="2014-04-29T10:10:04Z"
+ content="""
+Can you do something similar for gpg-agent?  Or do it instead with a ssh supported gpg-agent?
+"""]]
diff --git a/doc/design/assistant/sshpassword/comment_2_36a811bca209c7ac8a44d64bf8bc5bf3._comment b/doc/design/assistant/sshpassword/comment_2_36a811bca209c7ac8a44d64bf8bc5bf3._comment
new file mode 100644
index 0000000000..8d9925af2e
--- /dev/null
+++ b/doc/design/assistant/sshpassword/comment_2_36a811bca209c7ac8a44d64bf8bc5bf3._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="http://joeyh.name/"
+ ip="216.145.95.162"
+ subject="comment 2"
+ date="2014-05-19T15:47:19Z"
+ content="""
+I don't feel comfortable prompting for gpg passphrases in the webapp. See [[gpgkeys]] for thoughts on this.
+"""]]
diff --git a/doc/design/assistant/syncing.mdwn b/doc/design/assistant/syncing.mdwn
new file mode 100644
index 0000000000..df9a771b13
--- /dev/null
+++ b/doc/design/assistant/syncing.mdwn
@@ -0,0 +1,220 @@
+Once files are added (or removed or moved), need to send those changes to
+all the other git clones, at both the git level and the key/value level.
+
+## misc TODO
+
+* Test MountWatcher on LXDE.
+* Add a hook, so when there's a change to sync, a program can be run
+   and do its own signaling.
+* --debug will show often unnecessary work being done. Optimise.
+* Configurablity, including only enabling git syncing but not data transfer;
+  only uploading new files but not downloading, and only downloading
+  files in some directories and not others. See for use cases:
+  [[forum/Wishlist:_options_for_syncing_meta-data_and_data]]
+* speed up git syncing by using the cached ssh connection for it too
+  Will need to use `GIT_SSH`, which needs to point to a command to run,
+  not a shell command line. Beware that the network connection may have
+  bounced and the cached ssh connection not be usable.
+* Map the network of git repos, and use that map to calculate
+  optimal transfers to keep the data in sync. Currently a naive flood fill
+  is done instead. Maybe use XMPP as a side channel to learn about the
+  network topology?
+* Find a more efficient way for the TransferScanner to find the transfers
+  that need to be done to sync with a remote. Currently it walks the git
+  working copy and checks each file. That probably needs to be done once,
+  but further calls to the TransferScanner could eg, look at the delta
+  between the last scan and the current one in the git-annex branch.
+* [[use multiple transfer slots|todo/Slow_transfer_for_a_lot_of_small_files.]]
+* The TransferQueue's list of deferred downloads could theoretically
+  grow without bounds in memory. Limit it to a given number of entries,
+  and fall back to some other method -- either storing deferred downloads
+  on disk, or perhaps scheduling a TransferScanner run to get back into sync.
+
+## More efficient syncing
+
+See [[syncing/efficiency]]
+
+## TransferScanner efficiency
+
+The TransferScanner thread needs to find keys that need to be Uploaded
+to a remote, or Downloaded from it.
+
+How to find the keys to transfer? I'd like to avoid potentially
+expensive traversals of the whole git working copy if I can.
+(Currently, the TransferScanner does do the naive and possibly expensive
+scan of the git working copy.)
+
+One way would be to do a git diff between the (unmerged) git-annex branches
+of the git repo, and its remote. Parse that for lines that add a key to
+either, and queue transfers. That should work fairly efficiently when the
+remote is a git repository. Indeed, git-annex already does such a diff
+when it's doing a union merge of data into the git-annex branch. It
+might even be possible to have the union merge and scan use the same
+git diff data.
+
+But that approach has several problems:
+
+1. The list of keys it would generate wouldn't have associated git
+   filenames, so the UI couldn't show the user what files were being
+   transferred.
+2. Worse, without filenames, any later features to exclude
+   files/directories from being transferred wouldn't work.
+3. Looking at a git diff of the git-annex branches would find keys
+   that were added to either side while the two repos were disconnected.
+   But if the two repos' keys were not fully in sync before they
+   disconnected (which is quite possible; transfers could be incomplete),
+   the diff would not show those older out of sync keys.
+
+The remote could also be a special remote. In this case, I have to either
+traverse the git working copy, or perhaps traverse the whole git-annex
+branch (which would have the same problems with filesnames not being
+available).
+
+If a traversal is done, should check all remotes, not just
+one. Probably worth handling the case where a remote is connected
+while in the middle of such a scan, so part of the scan needs to be
+redone to check it.
+
+## done
+
+1. Can use `git annex sync`, which already handles bidirectional syncing.
+   When a change is committed, launch the part of `git annex sync` that pushes
+   out changes. **done**; changes are pushed out to all remotes in parallel
+1. Watch `.git/refs/remotes/` for changes (which would be pushed in from
+   another node via `git annex sync`), and run the part of `git annex sync`
+   that merges in received changes, and follow it by the part that pushes out
+   changes (sending them to any other remotes).
+   [The watching can be done with the existing inotify code! This avoids needing
+   any special mechanism to notify a remote that it's been synced to.]  
+   **done**
+1. Periodically retry pushes that failed.  **done** (every half an hour)
+1. Also, detect if a push failed due to not being up-to-date, pull,
+   and repush. **done**
+2. Use a git merge driver that adds both conflicting files,
+   so conflicts never break a sync. **done**
+
+* on-disk transfers in progress information files (read/write/enumerate)
+  **done**
+* locking for the files, so redundant transfer races can be detected,
+  and failed transfers noticed **done**
+* transfer info for git-annex-shell **done**
+* update files as transfers proceed. See [[progressbars]]
+  (updating for downloads is easy; for uploads is hard)
+* add Transfer queue TChan **done**
+* add TransferInfo Map to DaemonStatus for tracking transfers in progress.
+  **done**
+* Poll transfer in progress info files for changes (use inotify again!
+  wow! hammer, meet nail..), and update the TransferInfo Map **done**
+* enqueue Transfers (Uploads) as new files are added to the annex by
+  Watcher. **done**
+* enqueue Tranferrs (Downloads) as new dangling symlinks are noticed by
+  Watcher. **done**
+  (Note: Needs git-annex branch to be merged before the tree is merged,
+  so it knows where to download from. Checked and this is the case.)
+* Write basic Transfer handling thread. Multiple such threads need to be
+  able to be run at once. Each will need its own independant copy of the 
+  Annex state monad. **done**
+* Write transfer control thread, which decides when to launch transfers.
+  **done**
+* Transfer watching has a race on kqueue systems, which makes finished
+  fast transfers not be noticed by the TransferWatcher. Which in turn
+  prevents the transfer slot being freed and any further transfers
+  from happening. So, this approach is too fragile to rely on for
+  maintaining the TransferSlots. Instead, need [[todo/assistant_threaded_runtime]],
+  which would allow running something for sure when a transfer thread
+  finishes. **done**
+* Test MountWatcher on KDE, and add whatever dbus events KDE emits when
+  drives are mounted. **done**
+* It would be nice if, when a USB drive is connected, 
+  syncing starts automatically. Use dbus on Linux? **done**
+* Optimisations in 5c3e14649ee7c404f86a1b82b648d896762cbbc2 temporarily
+  broke content syncing in some situations, which need to be added back.
+  **done**
+
+  Now syncing a disconnected remote only starts a transfer scan if the
+  remote's git-annex branch has diverged, which indicates it probably has
+  new files. But that leaves open the cases where the local repo has
+  new files; and where the two repos git branches are in sync, but the
+  content transfers are lagging behind; and where the transfer scan has
+  never been run.
+
+  Need to track locally whether we're believed to be in sync with a remote.
+  This includes:
+  * All local content has been transferred to it successfully.
+  * The remote has been scanned once for data to transfer from it, and all
+    transfers initiated by that scan succeeded.
+
+  Note the complication that, if it's initiated a transfer, our queued
+  transfer will be thrown out as unnecessary. But if its transfer then
+  fails, that needs to be noticed.
+
+  If we're going to track failed transfers, we could just set a flag,
+  and use that flag later to initiate a new transfer scan. We need a flag
+  in any case, to ensure that a transfer scan is run for each new remote.
+  The flag could be `.git/annex/transfer/scanned/uuid`.
+
+  But, if failed transfers are tracked, we could also record them, in 
+  order to retry them later, without the scan. I'm thinking about a
+  directory like `.git/annex/transfer/failed/{upload,download}/uuid/`,
+  which failed transfer log files could be moved to.
+* A remote may lose content it had before, so when requeuing
+  a failed download, check the location log to see if the remote still has
+  the content, and if not, queue a download from elsewhere. (And, a remote
+  may get content we were uploading from elsewhere, so check the location
+  log when queuing a failed Upload too.) **done**
+* Fix MountWatcher to notice umounts and remounts of drives. **done**
+* Run transfer scan on startup. **done**
+* Often several remotes will be queued for full TransferScanner scans,
+  and the scan does the same thing for each .. so it would be better to
+  combine them into one scan in such a case. **done**
+* The syncing code currently doesn't run for special remotes. While
+  transfering the git info about special remotes could be a complication,
+  if we assume that's synced between existing git remotes, it should be
+  possible for them to do file transfers to/from special remotes.
+  **done**
+
+* The transfer code doesn't always manage to transfer file contents.
+
+  Besides reconnection events, there are two places where transfers get queued:
+
+  1. When the committer commits a file, it queues uploads.
+  2. When the watcher sees a broken symlink be created, it queues downloads.
+
+  Consider a doubly-linked chain of three repositories, A B and C.
+  (C and A do not directly communicate.)
+  
+  * File is added to A.
+  * A uploads its content to B.
+  * At the same time, A git syncs to B.
+  * Once B gets the git sync, it git syncs to C.
+  * When C's watcher sees the file appear, it tries to download it. But if
+    B had not finished receiving the file from A, C doesn't know B has it,
+    and cannot download it from anywhere.
+
+  Possible solution: After B receives content, it could queue uploads of it 
+  to all remotes that it doesn't know have it yet, which would include C. 
+  **done**
+  
+  In practice, this had the problem that when C receives the content,
+  it will queue uploads of it, which can send back to B (or to some other repo
+  that already has the content) and loop, until the git-annex branches catch
+  up and break the cycle.
+
+  To avoid that problem, incoming uploads should not result in a transfer
+  info file being written when the key is already present. **done**
+
+  Possible solution: C could record a deferred download. (Similar to a failed
+  download, but with an unknown source.) When C next receives a git-annex
+  branch push, it could try to queue deferred downloads. **done**
+
+  Note that this solution won't cover use cases the other does. For example,
+  connect a USB drive A; B syncs files from it, and then should pass them to C.
+  If the files are not new, C won't immediatly request them from B.
+
+* Running the assistant in a fresh clone of a repository, it sometimes
+  skips downloading a file, while successfully downloading all the rest.
+  There does not seem to be an error message. This will sometimes reproduce
+  (in a fresh clone each time) several times in a row, but then stops happening,
+  which has prevented me from debugging it.
+  This could possibly have been caused by the bug fixed in 750c4ac6c282d14d19f79e0711f858367da145e4.
+  Provisionally closed.
diff --git a/doc/design/assistant/syncing/comment_1_c70156174ff19b503978d623bd2df36f._comment b/doc/design/assistant/syncing/comment_1_c70156174ff19b503978d623bd2df36f._comment
new file mode 100644
index 0000000000..019490e619
--- /dev/null
+++ b/doc/design/assistant/syncing/comment_1_c70156174ff19b503978d623bd2df36f._comment
@@ -0,0 +1,19 @@
+[[!comment format=mdwn
+ username="https://www.google.com/accounts/o8/id?id=AItOawk4YX0PWICfWGRLuncCPufMPDctT7KAYJA"
+ nickname="betabrain"
+ subject="selective data syncing"
+ date="2012-07-24T15:27:08Z"
+ content="""
+How will the assistant know which files' data to distribute between the repos?
+
+I'm using git-annex and it's numcopies attribute to maintain a redundant archive spread over different computers and usb drives. Not all drives should get a copy of everything, e.g. the usb drive I take to work should not automatically get a copy of family pictures.
+
+How about .gitattributes?
+
+* \* annex.auto-sync-data = false # don't automatically sync the data
+* archive/ annex.auto-push-repos = NAS # everything added to archive/ in any repo goes automatically to the NAS remote.
+* work/ annex.auto-synced-repos = LAPTOP WORKUSB # everything added to work/ in LAPTOP or WORKUSB gets synced to WORKUSB and LAPTOP
+* work/ annex.auto-push-repos = LAPTOP WORKUSB # stuff added to work/ anywhere gets synced to LAPTOP and WORKUSB
+* important/ annex.auto-sync-data = true # push data to all repos
+* webserver_logs/ annex.remote.WEBSERVER.auto-push-repos = S3 # only the assistant running in WEBSERVER pushes webserver_logs/ to S3 remote
+"""]]
diff --git a/doc/design/assistant/syncing/comment_2_eb992b5b2c7a5ce23443e2a6007e5ff9._comment b/doc/design/assistant/syncing/comment_2_eb992b5b2c7a5ce23443e2a6007e5ff9._comment
new file mode 100644
index 0000000000..a4609d7e19
--- /dev/null
+++ b/doc/design/assistant/syncing/comment_2_eb992b5b2c7a5ce23443e2a6007e5ff9._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="https://www.google.com/accounts/o8/id?id=AItOawnBl7cA6wLDxVNUyLIHvAyCkf8ir3alYpk"
+ nickname="Tyson"
+ subject="Bridging LANs"
+ date="2012-07-10T10:20:59Z"
+ content="""
+Why rely on the cloud when you can instead use XMPP and jingle to perform NAT traversal for you? AFAIKT, it also means that traffic won't leave your router if the two endpoints are behind the same router.
+"""]]
diff --git a/doc/design/assistant/syncing/comment_3_e1b5e8a24556de16d1cacd27ee0c1bd1._comment b/doc/design/assistant/syncing/comment_3_e1b5e8a24556de16d1cacd27ee0c1bd1._comment
new file mode 100644
index 0000000000..c9118595c9
--- /dev/null
+++ b/doc/design/assistant/syncing/comment_3_e1b5e8a24556de16d1cacd27ee0c1bd1._comment
@@ -0,0 +1,80 @@
+[[!comment format=mdwn
+ username="https://www.google.com/accounts/o8/id?id=AItOawkSq2FDpK2n66QRUxtqqdbyDuwgbQmUWus"
+ nickname="Jimmy"
+ subject="comment 1"
+ date="2012-07-03T08:26:43Z"
+ content="""
+On \"git syncing\" point number 9, on OSX you could potentially do this on a semi-regular basis
+
+
    +system_profiler SPNetworkVolumeDataType
+Volumes:
+
+    net:
+
+      Type: autofs
+      Mount Point: /net
+      Mounted From: map -hosts
+      Automounted: Yes
+
+    home:
+
+      Type: autofs
+      Mount Point: /home
+      Mounted From: map auto_home
+      Automounted: Yes
+
    
+
+and
+
+
    +x00:~ jtang$ system_profiler SPUSBDataType
+USB:
+
+    USB High-Speed Bus:
+
+      Host Controller Location: Built-in USB
+      Host Controller Driver: AppleUSBEHCI
+      PCI Device ID: 0x0aa9 
+      PCI Revision ID: 0x00b1 
+      PCI Vendor ID: 0x10de 
+      Bus Number: 0x26 
+
+        Hub:
+
+          Product ID: 0x2504
+          Vendor ID: 0x0424  (SMSC)
+          Version: 0.01
+          Speed: Up to 480 Mb/sec
+          Location ID: 0x26200000 / 3
+          Current Available (mA): 500
+          Current Required (mA): 2
+
+            USB to ATA/ATAPI Bridge:
+
+              Capacity: 750.16 GB (750,156,374,016 bytes)
+              Removable Media: Yes
+              Detachable Drive: Yes
+              BSD Name: disk1
+              Product ID: 0x2338
+              Vendor ID: 0x152d  (JMicron Technology Corp.)
+              Version: 1.00
+              Serial Number: 313541813001
+              Speed: Up to 480 Mb/sec
+              Manufacturer: JMicron
+              Location ID: 0x26240000 / 5
+              Current Available (mA): 500
+              Current Required (mA): 2
+              Partition Map Type: MBR (Master Boot Record)
+              S.M.A.R.T. status: Not Supported
+              Volumes:
+                Porta-Disk:
+                  Capacity: 750.16 GB (750,156,341,760 bytes)
+                  Available: 668.42 GB (668,424,208,384 bytes)
+                  Writable: Yes
+                  File System: ExFAT
+....
+
    
+
+I think its possible to programatically get this information either from the CLI (it dumps out XML output if required) or some development library. There is also DBUS in macports, but I have never had much interaction with it, so I don't know if its good or bad on OSX.
+"""]]
diff --git a/doc/design/assistant/syncing/efficiency.mdwn b/doc/design/assistant/syncing/efficiency.mdwn
new file mode 100644
index 0000000000..09bcc11b69
--- /dev/null
+++ b/doc/design/assistant/syncing/efficiency.mdwn
@@ -0,0 +1,77 @@
+Currently, the git-annex assistant syncs with remotes in a way that is
+dumb, and potentially inneficient:
+
+1. Files are transferred to each reachable remote whose
+   [[preferred_content]] setting indicates it wants the file.
+
+2. After each file transfer (upload or download), a git sync
+   is done to all the remotes, to update location log information.
+
+## unncessary transfers
+
+There are network toplogies where #1 is massively inneficient.
+For example:
+
+
    +  laptopA-----laptopB-----laptopC
+      \         |             /
+       \---cloud based repo--/
+
    
+
+When laptopA has a new file, it will first send it to laptopB. It will then
+check if the cloud based transfer repository wants a copy. It will, because
+laptopC has not yet gotten a copy. So laptopA will proceed with a slow
+upload to the cloud, while meanwhile laptopB is sending the file over fast
+LAN to laptopC.
+
+(The more common case with no laptopC happens to work efficiently.
+So does the case where laptopA is paired with laptopC.)
+
+## unncessary syncing
+
+Less importantly, the constant git syncing after each transfer is rather a
+lot of work, and prevents collecting multiple presence changes to the git-annex 
+branch into larger commits, which would save disk space over time.
+
+In many cases, this sync is necessary. For example, when a file is uploaded
+to a transfer remote, the location change needs to be synced out so that
+other clients know to grab it.
+
+Or, when downloading a file from a drive, the sync lets other locally
+paired repositories know we got it, so they can download it from us. 
+OTOH, this is also a case where a sync is sometimes unnecessary, since
+if we're going to upload the file to them after getting it, the sync
+only perhaps lets them start downloading it before our transfer queue
+reaches a point where we'd upload it.
+
+It would be good to find a way to detect when syncing is not immediately
+necessary, and defer it.
+
+## mapping
+
+Mapping the repository network has the potential to get git-annex the
+information it needs to avoid unnecessary transfers and/or unncessary
+syncing.
+
+Mapping the network can reuse code in `git annex map`. Once the map is
+built, we want to find paths through the network that reach all nodes
+eventually, with the least cost. This is a minimum spanning tree problem,
+except with a directed graph, so really a Arborescence problem.
+
+A significant problem in mapping is that nodes are mobile, they can move
+between networks over time. This breaks LAN based paths through the
+network. Mapping would need a way to detect this. Note that individual
+git-annex assistants can tell when they've switched networks by using the
+`networkConnectedNotifier`.
+
+## P2P signaling
+
+Another approach that might help with these problems is if git-annex
+repositories have a non-git out of band signaling mechanism. This could,
+for example, be used by laptopB to tell laptopA that it's trying to send 
+a file directly to laptopC. laptopA could then defer the upload to the
+cloud for a while.
+
+## syncing only requested content
+
+See [[adhoc_routing]]
diff --git a/doc/design/assistant/telehash.mdwn b/doc/design/assistant/telehash.mdwn
new file mode 100644
index 0000000000..788074f961
--- /dev/null
+++ b/doc/design/assistant/telehash.mdwn
@@ -0,0 +1,136 @@
+[Telehash](http://telehash.org/) for secure P2P communication between
+git-annex (assistant) repositories.
+
+Or something similar like [Snow](http://www.trustiosity.com/snow/)
+or [cjdns](https://github.com/cjdelisle/cjdns) or tor or i2p or [magic wormhole](http://magic-wormhole.io/).
+
+## telehash implementation status
+
+* node.js version seems almost complete
+* C version currently lacks channel support and seems buggy (13 Jan 2014)
+* No pure haskell implementation of telehash v2. There was one of
+  telehash v1 (even that seems incomplete). I have pinged its author
+  to see if he anticipates updating it.
+* Rapid development, situation may change in a month or 2. (2014)  
+  Not seeing many commits now (2015)
+* Is it secure? A security review should be done by competant people
+  (not Joey). See 
+* **Haskell version** 
+  
+  May support v2; v3 support seems not started yet, and not in active
+  development at the moment, although there has been work on it a year ago.
+* Not very convinced this is going to be usable anytime soon, so would like
+  to find something that is. (2015)
+
+## snow status
+
+* Seems ready to use?
+* NAT punching works per docs; relies on a DHT network for hole punching,
+  and the reliabilty of that network is not known. I notice it has only 1
+  pre-seeded peer in the source tree for the DHT, and that peer was not up
+  when I tried it.
+* Only provides network-layer transport, still need to implement some 
+  file transfer protocol on top.
+
+## cjdns status
+
+* Has a network with "hundreds of active nodes"
+* Is not pure P2P; there's a network that does routing
+  of packets. This may be a good thing, or not.
+* Seems to require manual configuration of a "friend"
+  node that's already on the network, with address and password to connect
+  to it, so if you can't find someone you know to connect to their node,
+  you can't use it. Urk.
+
+## tor status
+
+* Awesome.
+* Easy to install, use; very well known.
+* Supported in git-annex now!
+
+## i2p status
+
+## magic wormhole
+
+* simple file transfer protocol with out of band shared secret
+* handles NAT transversal
+* easy to use
+* doesn't require a running daemon
+* can transfer arbitrary blobs (strings, directories, files)
+
+## general design
+
+* There is a generic P2P protocol, which should be usable with any P2P
+  system that can send messages between peers.
+* A p2p remote has an url like tor-annex::fijdksajdksjfkj, which connects
+  to a specific peer. The peer's address may be kept private, but 
+  the design allows the address to be public without giving access to
+  the peer.
+* An authtoken also needs to be presented when connecting with a peer.
+  This is stored in local creds storage and must be kept private.
+* The remotedaemon runs, and/or communicates with the program implementing
+  the P2P network. For example for tor, the remotedaemon runs the
+  hidden service.
+* The remotedaemon handles both sides of git push over the transport.
+* The remotedaemon may also support sending objects over the transport,
+  depending on the transport.
+
+## address exchange
+
+The address is a public key, and the authtoken is some large chunk of data,
+so won't want to type that in. Need discovery or exchange for peering.
+
+* Easy way is any set of repos that are already connected can communicate
+  them via address.log.
+* Address and authtoken can be communicated out of band (eg,
+  via an OTR session or gpg encrypted mail or phone call),
+  and pasted into the webapp.
+* Use eg, electrum-mnemonic to encode the address+authtoken so that
+  it can be read over the phone.
+* Users may not have a way to communicate with perfect forward secrecy.
+  So it would be good to have a address+authtoken that can only be used
+  one time during pairing.
+* Check out [PAKE](https://en.wikipedia.org/wiki/Password-authenticated_key_agreement)
+  for MITM resistance.
+* Possibly use magic wormhole to exchange the address, which avoids
+  the users needing to exchange so much data. The magic wormhole code
+  is just 3 words, and it uses PAKE.
+  
+  I tried it, and opened a couple of bug reports that would be useful in
+  integrating it with git-annex:
+
+  - [option to receive to a specific file](https://github.com/warner/magic-wormhole/issues/101)
+  - [machine readable wormhole code ](https://github.com/warner/magic-wormhole/issues/104])
+
+## local lan detection
+
+At connection time, after authentication, the remote can send
+(ip address, ssh host key). Try sshing to the ip address to check if
+the host key matches. If so, can enable a ssh remote, which will
+be cheaper than using the transport. Send the ssh public key back to the
+remote to get it authorized.
+
+## remotedaemon
+
+See [[git-remote-daemon]] for its design.
+
+Advantages:
+
+* `git annex sync` could also use the running daemon
+* `git-remote-$transport` could use the running daemon
+* c-telehash might end up linked to openssl, which has licence combination
+  problems with git-annex. A separate process not using git-annex's code
+  would avoid this.
+* Allows the daemon to be written in some other language if necessary
+  (for example, if c-telehash development stalls and the nodejs version is
+  already usable)
+* Potentially could be generalized to handle other similar protocols.
+  Or even the xmpp code moved into it. There could even be git-annex native
+  exchange protocols implemented in such a daemon to allow SSH-less
+  transfers.
+* Security holes in telehash would not need to compromise the entire
+  git-annex. daemon could be sandboxed in one way or another.
+
+Disadvantages:
+
+* Adds some complexity.
diff --git a/doc/design/assistant/thanks/comment_1_8b08b5c30e5aea3fc4599f856fd25df5._comment b/doc/design/assistant/thanks/comment_1_8b08b5c30e5aea3fc4599f856fd25df5._comment
new file mode 100644
index 0000000000..77a0873ee4
--- /dev/null
+++ b/doc/design/assistant/thanks/comment_1_8b08b5c30e5aea3fc4599f856fd25df5._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="https://www.google.com/accounts/o8/id?id=AItOawlUbH3eytydcwlWqv8oauE2Jg4NwcV9uA0"
+ nickname="Anna"
+ subject="Special"
+ date="2012-07-20T23:45:15Z"
+ content="""
+I feel pretty special getting an individualized thank you!  Btw, the good news is that your video finally explained what you were working on so that I understood it. :-)
+"""]]
diff --git a/doc/design/assistant/todo.mdwn b/doc/design/assistant/todo.mdwn
new file mode 100644
index 0000000000..2b369a2e2e
--- /dev/null
+++ b/doc/design/assistant/todo.mdwn
@@ -0,0 +1,4 @@
+This is a subset of [[/todo]] and [[/bugs]] 
+for items tagged for the assistant.
+
+[[!inline pages="tagged(design/assistant) and !link(bugs/done) and !link(bugs/moreinfo)" show=0 archive=yes]]
diff --git a/doc/design/assistant/transfer_control.mdwn b/doc/design/assistant/transfer_control.mdwn
new file mode 100644
index 0000000000..b0a14ed2b7
--- /dev/null
+++ b/doc/design/assistant/transfer_control.mdwn
@@ -0,0 +1,123 @@
+Some remotes are too small to sync everything to them.
+
+The case of a small remote on a gadget that the user interacts with,
+such as a phone, where they may want to request it get content
+it doesn't currently have, is covered by the [[partial_content]] page.
+
+But often the remote is just a removable drive or a cloud remote,
+that has a limited size. This page is about making the assistant do
+something smart with such remotes.
+
+## TODO
+
+* The expensive scan currently makes one pass, dropping content at the same
+  time more uploads and downloads are queued. It would be better to drop as
+  much content as possible upfront, to keep the total annex size as small
+  as possible. How to do that without making two expensive scans?
+* The TransferWatcher's finishedTransfer function relies on the location
+  log having been updated after a transfer. But there's a race; if the
+  log is not updated in time, it will fail to drop unwanted content.
+  (There's a 10 second sleep there now to avoid the race, but that's hardly
+  a fix.)
+
+### dropping no longer preferred content
+
+When a file is renamed, it might stop being preferred, so
+could be checked and dropped. (If there's multiple links to
+the same content, this gets tricky. Let's assume there are not.)
+
+### analysis of changes that can result in content no longer being preferred
+
+1. The preferred content expression can change, or a new repo is added, or
+   groups change. Generally, some change to global annex state. Only way to deal
+   with this is an expensive scan. (The rest of the items below come from
+   analizing the terminals used in preferred content expressions.) **done**
+2. renaming of a file (ie, moved to `archive/`) **done**
+   (note also that renaming a file can also make it become preferred content
+   again, and should cause it to be transferred in that case) **done**
+3. we get a file (`in`, `copies`) **done**
+4. we sent a file (`in`, `copies`) **done**
+5. some other repository drops the file (`in`, `copies` .. However, it's
+   unlikely that an expression would prefer content when *more* copies
+   exisited, and want to drop it when less do. That's nearly a pathological
+   case.)
+6. `migrate` is used to change a backend (`inbackend`; unlikely)
+
+That's all! Of these, 1-4 are by far the most important.
+
+## specifying what data a remote prefers to contain (**done**)
+
+Imagine a per-remote preferred content setting, that matches things that
+should be stored on the remote.
+
+For example, a MP3 player might use:
+`smallerthan(10mb) and filename(*.mp3) and (not filename(junk/*))`
+
+Adding that as a filter to files sent to a remote should be
+straightforward.
+
+A USB drive that is carried between three laptops and used to sync data
+between them might use: `not (in=laptop1 and in=laptop2 and in=laptop3)`
+
+In this case, transferring data from the usb repo should
+check if preferred content settings rejects the data, and if so, drop it
+from the repo. So once all three laptops have the data, it is
+pruned from the transfer drive.
+
+## repo groups (**done**)
+
+Seems like git-annex needs a way to know the groups of repos. Some
+groups:
+
+* enduser: The user interacts with this repo directly.
+* archival: This repo accumulates stuff, and once it's in enough archives,
+  it tends to get removed from other places.
+* transfer: This repo is used to transfer data between enduser repos,
+  it does not hold data for long periods of time, and tends to have a
+  limited size.
+
+Add a group.log that can assign repos to these or other groups. (**done**)
+
+Some examples of using groups:
+
+* Want to remove content from a repo, if it's not an archival repo,
+  and the content has reached at least one archival repo:
+
+  `(not group=archival) and (not copies=archival:1)`
+
+  That would make send to configure on all repos, or even set
+  a global `annex.accept` to it. **done**
+
+* Make a cloud repo only hold data until all known clients have a copy:
+
+  `not ingroup(enduser)`
+
+## configuration
+
+The above is all well and good for those who enjoy boolean algebra, but
+how to configure these sorts of expressions in the webapp?
+
+Currently, we have a simple drop down list to select between a few
+predefined groups with pre-defined preferred content recipes. Is this good
+enough?
+
+I think so; useful recipes can be developed on the wiki and included in
+git-annex.
+
+## the state change problem (**done**)
+
+Imagine that a trusted repo has setting like `not copies=trusted:2`
+This means that `git annex get --auto` should get files not in 2 trusted
+repos. But once it has, the file is in 3 trusted repos, and so `git annex
+drop --auto` should drop it again!
+
+How to fix? Can it even be fixed? Maybe care has to be taken when
+writing expressions, to avoid this problem. One that avoids it:
+`not (copies=trusted:2 or (in=here and trusted=here and copies=trusted:3))`
+
+Or, expressions could be automatically rewritten to avoid the problem.
+
+Or, perhaps simulation could be used to detect the problem. Before
+dropping, check the expression. Then simulate that the drop has happened.
+Does the expression now make it want to add it? Then don't drop it!
+**done**.. effectively using this approach.
diff --git a/doc/design/assistant/transfer_control/comment_1_d5adaef4712913dc0263d4ebafb79320._comment b/doc/design/assistant/transfer_control/comment_1_d5adaef4712913dc0263d4ebafb79320._comment
new file mode 100644
index 0000000000..54e6a59cc2
--- /dev/null
+++ b/doc/design/assistant/transfer_control/comment_1_d5adaef4712913dc0263d4ebafb79320._comment
@@ -0,0 +1,15 @@
+[[!comment format=mdwn
+ username="https://www.google.com/accounts/o8/id?id=AItOawl9sYlePmv1xK-VvjBdN-5doOa_Xw-jH4U"
+ nickname="Richard"
+ subject="comment 1"
+ date="2012-09-23T21:58:08Z"
+ content="""
+You could try to do it similar to [RT](http://bestpractical.com/rt/):
+
+* Implement saved statements, i.e. offer common use cases
+* Allow those statements (or aliases? stanzas?) to be loaded in a relatively simple editor with a basic wizard to support the building of new rules
+* Offer a free-text input for advanced users. It should be able to parse that and load it into the simple editor.
+* Users should be able to save, export, and import those statements.
+* Optionally, allow users to dry run the rules by showing them what git-annex needs to do to fulfill the requirements set by the statements.
+
+"""]]
diff --git a/doc/design/assistant/transfer_control/comment_2_3b51474fefa6c0d19055046e06af196d._comment b/doc/design/assistant/transfer_control/comment_2_3b51474fefa6c0d19055046e06af196d._comment
new file mode 100644
index 0000000000..ec53f7aacd
--- /dev/null
+++ b/doc/design/assistant/transfer_control/comment_2_3b51474fefa6c0d19055046e06af196d._comment
@@ -0,0 +1,14 @@
+[[!comment format=mdwn
+ username="https://www.google.com/accounts/o8/id?id=AItOawnRRUUZo3W7pAoRoST8P_l0PtUBhvYuzDg"
+ nickname="Lyle"
+ subject="exactly this use case: cloud remotes too small"
+ date="2014-01-06T00:31:34Z"
+ content="""
+I want to store a lot of scientific data in git-annex and have only a specific subset of my often used data in a box.com account so it is quickly synced between computers (my home internet has a limited upload speed). My home server will store all the files as it has effectively unlimited space. 
+
+I basically want the opposite of archive for git-annex-assistant, I'd like to mark the box.com special remote to store every *except* a certain special directory called less used files or whatever, or vise-versa, only store a folder of often used files, the local clients would store everything and my home server would also store everything. 
+
+It seems like transfer control and the groups.log can do this but I'm just not sure how to actually set it up?
+
+Thanks!
+"""]]
diff --git a/doc/design/assistant/transfer_control/comment_3_44a1a6d2db9097de9ae68ea1ff1b08a2._comment b/doc/design/assistant/transfer_control/comment_3_44a1a6d2db9097de9ae68ea1ff1b08a2._comment
new file mode 100644
index 0000000000..33ab5565bd
--- /dev/null
+++ b/doc/design/assistant/transfer_control/comment_3_44a1a6d2db9097de9ae68ea1ff1b08a2._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="http://joeyh.name/"
+ ip="209.250.56.35"
+ subject="comment 3"
+ date="2014-01-06T15:04:33Z"
+ content="""
+Just put the box.com remote in the transfer group. There is no need for special subdirectories, the transfer group makes the remote only want files that have not yet reached all the known clients.
+"""]]
diff --git a/doc/design/assistant/upgrading.mdwn b/doc/design/assistant/upgrading.mdwn
new file mode 100644
index 0000000000..1395d0cd85
--- /dev/null
+++ b/doc/design/assistant/upgrading.mdwn
@@ -0,0 +1,52 @@
+The assistant should support upgrading itself.
+
+## non-distro upgrades
+
+When git-annex was installed from this website, the assistant should poll
+periodically (once a day or so) to see if there is a new version.
+It downloads, over https, a .info file, which contains a serialized data
+type containing upgrade information. The url it's downloaded from is
+configured by setting `UPGRADE_LOCATION` when building git-annex on the
+autobuilders.
+
+When a new version is found, the webapp prompts the user to start the
+upgrade. (annex.autoupgrade can be set to true to upgrade w/o prompting.)
+
+The upgrade process is automatic, and rather tricky. The file is downloaded
+using git-annex (as a regular key!), and is then unpacked into a new
+directory, and the programfile updated to point to it. Then git-annex
+restarts itself.
+
+### manifest files
+
+To clean up the old installation, a git-annex.MANIFEST file is looked for
+in it, and the files listed, as well as empty directories, are deleted.
+I don't want to accidentally delete something I didn't ship!
+
+## restart on upgrade
+
+When git-annex is installed from a proper distribution package, there is no
+need for the above. But, the assistant still needs to notice when git-annex
+get upgraded, and offer to restart (or automatically restart when
+annex.autoupgrade is set).
+
+This is done using the DirWatcher, watching the directory containing the
+git-annex binary. Or, in the case of a non-distro install, watching the
+directory where eg git-annex.linux/ was unpacked.
+
+When an change is detected, restart.
+
+## multi-daemon upgrades
+
+A single system may have multiple assistant daemons running in different
+repositories.
+
+In this case, one daemon should do the non-distro upgrade, and the rest
+should notice the upgrade and restart.
+
+I don't want every daemon trying to download the file at once..
+
+Approach: The first new version is installed into a stable directory, based
+on its version. So, start the upgrade by making this directory. If upgrade
+is already in progress, the directory will already exist. (Remove directory
+if upgrade fails.)
diff --git a/doc/design/assistant/webapp.mdwn b/doc/design/assistant/webapp.mdwn
new file mode 100644
index 0000000000..797c9ad301
--- /dev/null
+++ b/doc/design/assistant/webapp.mdwn
@@ -0,0 +1,65 @@
+The webapp is a web server that displays a shiny interface.
+
+## performance
+
+Having the webapp open while transfers are
+running uses significant CPU just for the browser to update the progress 
+bar. Unsurprising, since the webapp is sending the browser a new `
    `
+each time. Updating the DOM instead from javascript would avoid that;
+the webapp just needs to send the javascript either a full `
    ` or a 
+changed percentage and quantity complete to update a single progress bar.
+
+(Another reason to do this is it'll cut down on the refreshes, which 
+sometimes make browsers ignore clicks on UI elements like the pause button, 
+if the transfer display refreshes just as the click is made.)
+
+## other features
+
+* there could be a UI to export a file, which would make it be served up
+  over http by the web app
+* there could be a UI (some javascript thing) in the web browser to
+  submit urls to the web app to be added to the annex and downloaded.
+  See: [[todo/wishlist:_an_"assistant"_for_web-browsing_--_tracking_the_sources_of_the_downloads]]
+* Display the `inotify max_user_watches` exceeded message. **done**
+* Display something sane when kqueue runs out of file descriptors.
+* allow removing git remotes **done**
+* allow disabling syncing to here, which should temporarily disable all
+  local syncing. **done**
+
+## better headless support
+
+`--listen` is insecure, and using HTTPS would still not make it 100% secure
+as there would be no way for the browser to verify its certificate.
+
+I do have a better idea, but it'd be hard to implement.
+`git annex webapp --remote user@host:dir` could ssh to the remote host,
+run the webapp there, listening only on localhost, and then send the
+port the webapp chose back over the ssh connection. Then the same
+ssh connection could be reused (using ssh connection caching) to set up
+port forwarding from a port on the local host to the remote webapp.
+
+This would need to handle the first run case too, which would require
+forwarding a second port once the webapp made the repository and
+the second webapp started up.
+
+## first start **done**
+
+* make git repo **done**
+* generate a nice description like "joey@hostname Desktop/annex" **done**
+* record repository that was made, and use it next time run **done**
+* write a pid file, to prevent more than one first-start process running
+  at once **done**
+
+## security **acceptable/done**
+
+* Listen only to localhost. **done**
+* Instruct the user's web browser to open an url that contains a secret
+  token. This guards against other users on the same system. **done**
+  (I would like to avoid passwords or other authentication methods,
+  it's your local system.)
+* Don't pass the url with secret token directly to the web browser,
+  as that exposes it to `ps`. Instead, write a html file only the user can read,
+  that redirects to the webapp. **done**
+* Alternative for Linux at least would be to write a small program using
+  GTK+ Webkit, that runs the webapp, and can know what user ran it, avoiding
+  needing authentication.
diff --git a/doc/design/assistant/webapp/comment_1_bab6f6fa720273c0f9700a3765150189._comment b/doc/design/assistant/webapp/comment_1_bab6f6fa720273c0f9700a3765150189._comment
new file mode 100644
index 0000000000..3e1330f96d
--- /dev/null
+++ b/doc/design/assistant/webapp/comment_1_bab6f6fa720273c0f9700a3765150189._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="https://www.google.com/accounts/o8/id?id=AItOawlup4hyZo4eCjF8T85vfRXMKBxGj9bMdl0"
+ nickname="Ben"
+ subject="ARM support"
+ date="2012-07-13T16:51:15Z"
+ content="""
+The closure of [this](http://hackage.haskell.org/trac/ghc/ticket/5839) ticket hopefully marks the end of TH issues on ARM. As of 7.4.2, GHC's linker has enough ARM support to allow a selection of common packages compile on my PandaBoard. That being said, it hasn't had a whole lot of testing so it's possible I still need to implement a few relocation types.
+"""]]
diff --git a/doc/design/assistant/webapp/comment_2_3cf0cf460c7869d0cc22940fcc84aec4._comment b/doc/design/assistant/webapp/comment_2_3cf0cf460c7869d0cc22940fcc84aec4._comment
new file mode 100644
index 0000000000..60d678aba4
--- /dev/null
+++ b/doc/design/assistant/webapp/comment_2_3cf0cf460c7869d0cc22940fcc84aec4._comment
@@ -0,0 +1,10 @@
+[[!comment format=mdwn
+ username="yatesa"
+ ip="171.25.193.21"
+ subject="Secret URL token"
+ date="2012-06-19T03:41:16Z"
+ content="""
+> Instruct the user's web browser to open an url that contains a secret token. This guards against other users on the same system.
+
+How will you implement that? Running \"sensible-browser URL\" would be the obvious way, but the secret URL would show up in a well timed ps listing. (And depending on the browser, ps may show the URL the entire time it's running.)
+"""]]
diff --git a/doc/design/assistant/webapp/comment_3_428e153135f7a64215730719207d82c4._comment b/doc/design/assistant/webapp/comment_3_428e153135f7a64215730719207d82c4._comment
new file mode 100644
index 0000000000..7a73799aa4
--- /dev/null
+++ b/doc/design/assistant/webapp/comment_3_428e153135f7a64215730719207d82c4._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="jtang"
+ ip="79.97.135.214"
+ subject="comment 3"
+ date="2012-07-26T17:35:18Z"
+ content="""
+Using twitter-bootstrap for the webapp - this might be a wishlist item, but would it be possible to ensure that the webapp's css uses twitter-bootstrap classes. It would make theming much easier in the long run and it would give you a nice modern look with a low amount of effort.
+"""]]
diff --git a/doc/design/assistant/webapp/comment_4_f4068a7abbb77ba6a3297cbcf1e503e9._comment b/doc/design/assistant/webapp/comment_4_f4068a7abbb77ba6a3297cbcf1e503e9._comment
new file mode 100644
index 0000000000..d713e3e8fc
--- /dev/null
+++ b/doc/design/assistant/webapp/comment_4_f4068a7abbb77ba6a3297cbcf1e503e9._comment
@@ -0,0 +1,10 @@
+[[!comment format=mdwn
+ username="http://joeyh.name/"
+ ip="4.153.2.189"
+ subject="comment 4"
+ date="2012-07-26T17:45:28Z"
+ content="""
+So, Yesod's scaffolded site actually does use bootstrap, but I didn't use the scaffolded site so don't have it. I am not quite to the point of doing any theming of the webapp, but I do have this nice example of how to put in bootstrap right here..
+
+By the way, if anyone would like to play with the html templates for the webapp, the main html template is `templates/default-layout.hamlet`. Uses a slightly weird template markup, but plain html will also work. And there's also the `static/` directory; every file in there will be compiled directly into the git-annex binary, and is available at `http://localhost:port/static/$file` in the webapp. See the favicon link in `default-layout.hamlet` of how to construct a type-safe link to a static file: `href=@{StaticR favicon_ico}`. That's all you really need to theme the webapp, without doing any real programming!
+"""]]
diff --git a/doc/design/assistant/webapp/comment_5_2bdb436f35c659316193b89ee6e52fcb._comment b/doc/design/assistant/webapp/comment_5_2bdb436f35c659316193b89ee6e52fcb._comment
new file mode 100644
index 0000000000..b1592af20b
--- /dev/null
+++ b/doc/design/assistant/webapp/comment_5_2bdb436f35c659316193b89ee6e52fcb._comment
@@ -0,0 +1,11 @@
+[[!comment format=mdwn
+ username="dirk.schmidt@045f26624b94d351628bbb315f94150d627fb18b"
+ nickname="dirk.schmidt"
+ subject="Webapp don open"
+ date="2015-07-08T12:49:01Z"
+ content="""
+Hey there,
+i am using Ubuntu 14.04 
+I have disabled all my repositories and now i cannot open the webapp any more.
+Any suggestions ?
+"""]]
diff --git a/doc/design/assistant/windows.mdwn b/doc/design/assistant/windows.mdwn
new file mode 100644
index 0000000000..78ab69febb
--- /dev/null
+++ b/doc/design/assistant/windows.mdwn
@@ -0,0 +1,33 @@
+See [[todo/windows_support]]..
+
+## symlinks
+
+Apparently new versions of Windows have something very like symlinks.
+(Or really, 3 or so things not entirely unlike symlinks and all different.)
+Stackoverflow has some details.
+
+NTFS supports symbolic links two different ways: an [[!wikipedia NTFS symbolic link]] and an [[!wikipedia NTFS_junction_point]].  The former seems like the closest analogue to POSIX symlinks.
+
+The windows port will not use symlinks. It will only support direct mode.
+
+## POSIX
+
+Lots of ifdefs and pain to deal with POSIX calls in the code base.
+
+Or I could try to use Cygwin.
+
+## Deeper system integration
+
+[NTFS Reparse Points](http://msdn.microsoft.com/en-us/library/aa365503%28v=VS.85%29.aspx) allow a program to define how the OS will interpret a file or directory in arbitrary ways.  This requires writing a file system filter.
+
+## Developement environment
+
+Someone wrote in to say:
+
+> For Windows Development you can easily qualify
+> for Bizspark - http://www.microsoft.com/bizspark/
+> 
+> This will get you 100% free Windows OS licenses and
+> Dev tools, plus a free Azure account for cloud testing.
+> (You can also now deploy Linux VMs to Azure as well)
+> No money required at all.
diff --git a/doc/design/assistant/windows/comment_1_f4b829318b182e1cec29f13babb6498e._comment b/doc/design/assistant/windows/comment_1_f4b829318b182e1cec29f13babb6498e._comment
new file mode 100644
index 0000000000..0bccc3a354
--- /dev/null
+++ b/doc/design/assistant/windows/comment_1_f4b829318b182e1cec29f13babb6498e._comment
@@ -0,0 +1,10 @@
+[[!comment format=mdwn
+ username="https://www.google.com/accounts/o8/id?id=AItOawlCGROoy62svBUy6P24x1KoGoDWrBq2ErA"
+ nickname="Steve"
+ subject="comment 1"
+ date="2012-08-07T04:15:43Z"
+ content="""
+NTFS symbolic links should do exactly what you would expect them to do. They can point to files or directories. Junction points are legacy NTFS functionality and reparse points are more like the POSIX mount functionality.
+
+NTFS symbolic links should work for you, junction point should be avoided, and reparse points would be like using a nuke to kill a fly. The only hang up you might have is that I think all three features require elevated privileges to manage.
+"""]]
diff --git a/doc/design/assistant/xmpp.mdwn b/doc/design/assistant/xmpp.mdwn
new file mode 100644
index 0000000000..4bdc58874f
--- /dev/null
+++ b/doc/design/assistant/xmpp.mdwn
@@ -0,0 +1,136 @@
+The git-annex assistant uses XMPP to communicate between peers that
+cannot directly talk to one-another. A typical scenario is two users
+who share a repository, that is stored in the [[cloud]].
+
+### TODO
+
+* Do git-annex clients sharing an account with regular clients cause confusing
+  things to happen? 
+  See 
+* Support registering with XMPP provider from within the webapp, 
+  as clients like pidgin are able to do.
+* At least some XMPP servers are lossy (does XMPP guarantee delivery)?
+  I have seen a log where a push's packet 1 and 3 arrived, but 2 did not.
+  To deal with this, need at least a 1 packet buffer and ACK or resend
+  request implemented over top of XMPP. Essentially, TCP over XMPP. :(
+
+## design goals
+
+1. Avoid user-visible messages. dvcs-autosync uses XMPP similarly, but
+   sends user-visible messages. Avoiding user-visible messages lets
+   the user configure git-annex to use his existing XMPP account
+   (eg, Google Talk).
+
+2. Send notifications to buddies. dvcs-autosync sends only self-messages,
+   but that requires every node have the same XMPP account configured.
+   git-annex should support that mode, but it should also send notifications
+   to a user's buddies. (This will also allow for using XMPP for pairing
+   in the future.)
+
+3. Don't make account appear active. Just because git-annex is being an XMPP
+   client, it doesn't mean that it wants to get chat messages, or make the
+   user appear active when he's not using his chat program.
+
+## protocol
+
+To avoid relying on XMPP extensions, git-annex communicates
+using presence messages, and chat messages (with empty body tags,
+so clients don't display them).
+
+git-annex sets a negative presence priority, to avoid any regular messages
+getting eaten by its clients. It also sets itself extended away.
+Note that this means that chat messages always have to be directed at
+specific git-annex clients.
+
+To the presence and chat messages, it adds its own tag as
+[extended content](http://xmpp.org/rfcs/rfc6121.html#presence-extended).
+The xml namespace is "git-annex" (not an URL because I hate wasting bandwidth).
+
+To indicate it's pushed changes to a git repo with a given UUID, a message
+that is sent to all buddies and other clients using the account (no
+explicit pairing needed), it uses a broadcast presence message containing:
+
+	
+
+Multiple UUIDs can be listed when multiple clients were pushed. If the
+git repo does not have a git-annex UUID, an empty string is used.
+
+To query if other git-annex clients are around, a presence message is used,
+containing:
+
+	
+
+For pairing, a chat message is sent to every known git-annex client,
+containing:
+
+	
+
+### git push over XMPP
+
+To indicate that we could push over XMPP, a chat message is sent,
+to each known client of each XMPP remote.
+
+	
+
+The shas are omitted by old clients. If present, they are the git shas of
+the head and git-annex branches that are available to be pushed. This lets
+the receiver check if it's already got them.
+
+To request that a remote push to us, a chat message can be sent.
+
+	
+
+When replying to an canpush message, this is directed at the specific
+client that indicated it could push. To solicit pushes from all clients,
+the message has to be sent directed individually to each client.
+
+When a peer is ready to send a git push, it sends:
+
+	
+
+The receiver runs `git receive-pack`, and sends back its output in
+one or more chat messages, directed to the client that is pushing:
+
+	
+	007b27ca394d26a05d9b6beefa1b07da456caa2157d7 refs/heads/git-annex report-status delete-refs side-band-64k quiet ofs-delta
+	
+
+The sender replies with the data from `git push`, in 
+one or more chat messages, directed to the receiver:
+
+	
+	data
+	
+
+The value of rp and sp used to be empty, but now it's a sequence number.
+This indicates the sequence of this packet, counting from 1. The receiver
+and sender each have their own sequence numbers.
+
+When `git receive-pack` exits, the receiver indicates its exit
+status with a chat message, directed at the sender:
+
+	
+
+### security
+
+Data git-annex sends over XMPP will be visible to the XMPP account's
+buddies, and to the XMPP server (and any attacker who has access to the
+XMPP server). So it's important to consider the security exposure of using
+it.
+
+Even if git-annex sends only a single bit notification, this lets attackers
+know when the user is active and changing files. Although the assistant's other
+syncing activities can somewhat mask this.
+
+As soon as git-annex does anything unlike any other client, an attacker can
+see how many clients are connected for a user, and fingerprint the ones
+running git-annex, and determine how many clients are running git-annex.
+
+An attacker can observe the UUIDs used for pushes and pairing, and determine
+how many different remotes are being used.
+
+An attacker could replay push notification messages, reusing UUIDs it's
+observed. This would make clients pull repeatedly, perhaps as a DOS.
+
+The XMPP server, or an attacker with access to it can reconstruct the git 
+repository from data sent in pushes, in part or in whole.
diff --git a/doc/design/assistant/xmpp/comment_1_f20650f93d7f0ca39b9ba3ce0380193f._comment b/doc/design/assistant/xmpp/comment_1_f20650f93d7f0ca39b9ba3ce0380193f._comment
new file mode 100644
index 0000000000..7ee62eea7f
--- /dev/null
+++ b/doc/design/assistant/xmpp/comment_1_f20650f93d7f0ca39b9ba3ce0380193f._comment
@@ -0,0 +1,10 @@
+[[!comment format=mdwn
+ username="http://meep.pl/"
+ ip="193.23.174.18"
+ subject="xmlns"
+ date="2012-11-11T09:00:01Z"
+ content="""
+A minor point, but is saving a couple of bytes per message worth using a [deprecated feature](http://www.w3.org/TR/REC-xml-names/#iri-use) of the namespaces specification? This is not technically *breaking* the current specification, since \"git-annex\" is of course still a (relative) URI reference; and anyway chances of problems are, I guess, slim. But is it the lesser of two bugs?
+
+The shortest moderately sane absolute URI containing \"git-annex\" would probably be \"data:,git-annex\".
+"""]]
diff --git a/doc/design/assistant/xmpp/comment_2_8c22839a8f5912b4a817415c4a359697._comment b/doc/design/assistant/xmpp/comment_2_8c22839a8f5912b4a817415c4a359697._comment
new file mode 100644
index 0000000000..5b6ea3946c
--- /dev/null
+++ b/doc/design/assistant/xmpp/comment_2_8c22839a8f5912b4a817415c4a359697._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="https://www.google.com/accounts/o8/id?id=AItOawkurjhi0CRJvgm7QNaZDWS9hitBtavqIpc"
+ nickname="Bret"
+ subject="Plans for two factor authentication or oath?"
+ date="2013-04-15T00:58:04Z"
+ content="""
+Are there plans to support google's two factor authentication?  Right now I have to use an application specific password.
+"""]]
diff --git a/doc/design/assistant/xmpp/comment_4_773102522f21844cffc841e6cde9229e._comment b/doc/design/assistant/xmpp/comment_4_773102522f21844cffc841e6cde9229e._comment
new file mode 100644
index 0000000000..f8ddf0a3b8
--- /dev/null
+++ b/doc/design/assistant/xmpp/comment_4_773102522f21844cffc841e6cde9229e._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="hhm"
+ ip="108.17.80.177"
+ subject="file transfer?"
+ date="2013-04-23T10:22:51Z"
+ content="""
+Would it be possible to add optional support for transferring files over XMPP (possibly being disabled out-of-the-box so as not to suck up third-party bandwidth)?
+"""]]
diff --git a/doc/design/assistant/xmpp_security.mdwn b/doc/design/assistant/xmpp_security.mdwn
new file mode 100644
index 0000000000..647bffd797
--- /dev/null
+++ b/doc/design/assistant/xmpp_security.mdwn
@@ -0,0 +1,29 @@
+Currently [[xmpp]] relies on the SSL connection to the server for security.
+The server can see git repository data pushed through it. (Also, the SSL
+connection is not pinned or really checked well at all.)
+
+Add an encryption layer that does not rely on trusting the XMPP server's
+security. There are a few options for how to generate the key for eg,
+AES encryption:
+
+  * Do a simple Diffie-Hellman shared key generation when starting each XMPP
+    push session. Would not protect the users from active MITM by the
+    XMPP server, but would prevent passive data gathering attacks from
+    getting useful data. Since the key is ephemeral, would provide
+    Forward Security.
+  * Do D-H key generation, but at pairing, not push time. Allows for an
+    optional confirmation step, using eg, BubbleBabble to compare the
+    keys out of band. ("I see xebeb-dibyb-gycub-kacyb-modib-pudub-sefab-vifuc-bygoc-daguc-gohec-kuxax .. do you too?")
+  * Prompt both users for a passphrase when XMPP pairing, and 
+    use SPEKE (or similar methods like J-PAKE) to generate a shared key.
+    Avoids active MITM attacks. Makes pairing harder, especially pairing
+    between one's own devices, since the passphrase has to be entered on
+    all devices. Also problematic when pairing more than 2 devices,
+    especially when adding a device to the set later, since there
+    would then be multiple different keys in use.
+  * Rely on the user's gpg key, and do gpg key verification during XMPP
+    pairing. Problematic because who wants to put their gpg key on their
+    phone? Also, require the users be in the WOT and be gpg literate.
+
+Update: This seems unlikely to be worth doing. [[Telehash]] is better.
+--[[Joey]]
diff --git a/doc/design/assistant/xmpp_security/comment_1_c714e86553c02600249795efb224be8a._comment b/doc/design/assistant/xmpp_security/comment_1_c714e86553c02600249795efb224be8a._comment
new file mode 100644
index 0000000000..92cff201df
--- /dev/null
+++ b/doc/design/assistant/xmpp_security/comment_1_c714e86553c02600249795efb224be8a._comment
@@ -0,0 +1,10 @@
+[[!comment format=mdwn
+ username="Sagi"
+ ip="2001:610:1908:8001:25ce:8a0b:53cc:c010"
+ subject="Why not use OTR?"
+ date="2013-07-23T23:53:12Z"
+ content="""
+[OTR](http://www.cypherpunks.ca/otr/) seems a natural fit for end-to-end encryption over xmpp. I can't find Haskell bindings for libotr at this moment, but creating those instead of inventing new protocols might be worth considering. There are also several [native implementations](http://www.cypherpunks.ca/otr/software.php) (Python, Go, Scheme), but I'm not sure if that's worth the effort.
+
+
+"""]]
diff --git a/doc/design/balanced_preferred_content.mdwn b/doc/design/balanced_preferred_content.mdwn
new file mode 100644
index 0000000000..c099642605
--- /dev/null
+++ b/doc/design/balanced_preferred_content.mdwn
@@ -0,0 +1,80 @@
+Say we have 2 backup drives and want to fill them both evenly with files,
+different files in each drive. Currently, preferred content cannot express
+that entirely:
+
+* One way is to use a-m* and n-z*, but that's unlikely to split filenames evenly. 
+* Or, can let both repos take whatever files, perhaps at random, that the
+  other repo is not know to contain, but then repos will race and both get
+  the same file, or similarly if they are not communicating frequently.
+
+So, let's add a new expression: `balanced_amoung(group)`
+
+This would work by taking the list of uuids of all repositories in the
+group, and sorting them, which yields a list from 0..M-1 repositories.
+
+To decide which repository wants key K, convert K to a number N in some
+stable way and then `N mod M` yields the number of the repository that
+wants it, while all the rest don't.
+
+(Since git-annex keys can be pretty long and not all of them are random
+hashes, let's md5sum the key and then use the md5 as a number.)
+
+This expression is stable as long as the members of the group don't change.
+I think that's stable enough to work as a preferred content expression.
+
+Now, you may want to be able to add a third repo and have the data be
+rebalanced, with some moving to it. And that would happen. However, as this
+scheme stands, it's equally likely that adding repo3 will make repo1 and
+repo2 want to swap files between them. So, we'll want to add some
+precautions to avoid a lof of data moving around in this case:
+
+	((balanced_amoung(backup) and not (copies=backup:1)) or present
+
+So once file lands on a backup drive, it stays there, even if more backup
+drives change the balancing.
+
+-----
+
+Some limitations:
+
+* The item size is not taken into account. One repo could end up with a
+  much larger item or items and so fill up faster. And the other repo
+  wouldn't then notice it was full and take up some slack.
+* With the complicated expression above, adding a new repo when one 
+  is full would not necessarily result in new files going to one of the 2
+  repos that still have space. Some items would end up going to the full
+  repo.
+
+These can be dealt with by noticing when a repo is full and moving some
+of it's files (any will do) to other repos in its group. I don't see a way
+to make preferred content express that movement though; it would need to be
+a manual/scripted process.
+
+-----
+
+What if we have 5 backup repos and want each file to land in 3 of them?
+There's a simple change that can support that:
+`balanced_amoung(group:3)`
+
+This works the same as before, but rather than just `N mod M`, take
+`N+I mod M` where I is [0..2] to get the list of 3 repositories that want a
+key.
+
+This does not really avoid the limitations above, but having more repos
+that want each file will reduce the chances that no repo will be able to
+take a given file. In the [[iabackup]] scenario, new clients will just be
+assigned until all the files reach the desired level or replication.
+
+However.. Imagine there are 9 repos, all full, and some files have not
+reached desired level of replication. Seems like adding 1 more repo will make
+only 3 in 10 files be wanted by that new repo. Even if the repo has space
+for all the files, it won't be sufficient, and more repos would need to be
+added.
+
+One way to avoid this problem would be if the preferred content was only
+used for the initial distribution of files to a repo. If the repo has
+gotten all the files it wants, it could make a second pass and
+opportunistically get files it doesn't want but that it has space for
+and that don't have enough copies yet.
+Although this gets back to the original problem of multiple repos racing
+downloads and files getting more than the desired number of copies.
diff --git a/doc/design/caching_database.mdwn b/doc/design/caching_database.mdwn
new file mode 100644
index 0000000000..c27c9557c6
--- /dev/null
+++ b/doc/design/caching_database.mdwn
@@ -0,0 +1,160 @@
+* [[metadata]] for views
+* [direct mode mappings scale badly with thousands of identical files](/bugs/__34__Adding_4923_files__34___is_really_slow)
+* [[bugs/incremental_fsck_should_not_use_sticky_bit]]
+* [[todo/wishlist:_pack_metadata_in_direct_mode]]
+* [[todo/cache_key_info]]
+* [[bugs/indeterminite_preferred_content_state_for_duplicated_file]]
+
+What do all these have in common? They could all be improved by
+using some kind of database to locally store the information in an
+efficient way.
+
+The database should only function as a cache. It should be able to be
+generated and updated by looking at the git repository.
+
+* Metadata can be updated by looking at the git-annex branch,
+  either its current state, or the diff between the old and new versions
+* Direct mode mappings can be updated by looking at the current branch,
+  to see which files map to which key. Or the diff between the old
+  and new versions of the branch.
+* Incremental fsck information is not stored in git, but can be
+  "regenerated" by running fsck again.  
+  (Perhaps doesn't quite fit, but let it slide..)
+
+Store in the database the Ref of the branch that was used to construct it.
+(Update in same transaction as cached data.)
+
+## implementation plan
+
+1. Store incremental fsck info in db, on a branch, with sqlite. **done**
+2. Make sure that builds on all platforms, and works reliably. **done**
+3. Use sqlite db for associated files cache. **done** (only for v6 unlocked
+   files)
+4. Use associated files db when dropping files, to fix
+   [[bugs/indeterminite_preferred_content_state_for_duplicated_file]]
+5. Also, use associated files db to construct views.
+6. Use sqlite db for metadata cache.
+7. Use sqlite db for list of keys present in local annex.
+
+## sqlite or not?
+
+sqllite seems like the most likely thing to work. But it does involve ugh,
+SQL. And even if that's hidden by a layer like persistent, it's still going
+to involve some technical debt (eg, database migrations).
+
+It would be great if there were some haskell thing like acid-state
+that I could use instead. But, acid-state needs to load the whole
+DB into memory. In the comments of
+[[bugs/incremental_fsck_should_not_use_sticky_bit]] I examined several
+other haskell database-like things, and found them all wanting, except for
+possibly TCache. (And TCache is backed by persistent/sqlite anyway.)
+
+## one db or multiple?
+
+Using a single database will use less space. Eg, each Key will only need to
+appear in it once, with proper normalization.
+
+OTOH, it's more complicated, and harder to recover from problems.
+
+Currently leaning toward one database per purpose.
+
+## case study: persistent with sqllite
+
+Here's a non-normalized database schema in persistent's syntax.
+
+
    +CachedKey
+  key Key
+  associatedFiles [FilePath]
+  lastFscked Int Maybe
+  KeyIndex key
+
+CachedMetaData
+  key Key
+  metaDataField MetaDataField
+  metaDataValue MetaDataValue
+
    
+
+Using the above database schema and persistent with sqlite, I made
+a database containing 30k Cache records. This took 5 seconds to create
+and was 7 mb on disk. (Would be rather smaller, if a more packed Key
+show/read instance were used.)
+
+Running 1000 separate queries to get 1000 CachedKeys took 0.688s with warm
+cache. This was more than halved when all 1000 queries were done inside the
+same `runSqlite` call. (Which could be done using a separate thread and some
+MVars.)
+
+(Note that if the database is a cache, there is no need to perform migrations
+when querying it. My benchmarks skip `runMigration`. Instead, if the query
+fails, the database doesn't exist, or uses an incompatible schema, and the
+cache can be rebuilt then. This avoids the problem that persistent's migrations
+can sometimes fail.)
+
+Doubling the db to 60k scaled linearly in disk and cpu and did not affect
+query time.
+
+----
+
+Here's a normalized schema:
+
+
    +CachedKey
+  key Key
+  KeyIndex key
+  deriving Show
+
+AssociatedFiles
+  keyId CachedKeyId Eq
+  associatedFile FilePath
+  KeyIdIndex keyId associatedFile
+  deriving Show
+
+CachedMetaField
+  field MetaField
+  FieldIndex field
+
+CachedMetaData
+  keyId CachedKeyId Eq
+  fieldId CachedMetaFieldId Eq
+  metaValue String
+
+LastFscked
+  keyId CachedKeyId Eq
+  localFscked Int Maybe
+
    
+
+With this, running 1000 joins to get the associated files of 1000
+Keys took 5.6s with warm cache. (When done in the same `runSqlite` call.) Ouch!
+
+Update: This performance was fixed by adding `KeyIdOutdex keyId associatedFile`,
+which adds a uniqueness constraint on the tuple of key and associatedFile.
+With this, 1000 queries takes 0.406s. Note that persistent is probably not
+actually doing a join at the SQL level, so this could be sped up using
+eg, esquelito.
+
+Update2: Using esquelito to do a join got this down to 0.109s.
+See `database` branch for code.
+
+Update3: Converting to a single un-normalized table for AssociatedFiles
+avoids the join, and increased lookup speed to 0.087s. Of course, when
+a key has multiple associated files, this will use more disk space, due
+to not normalizing the key.
+
+Compare the above with 1000 calls to `associatedFiles`, which is approximately
+as fast as just opening and reading 1000 files, so will take well under
+0.05s with a **cold** cache.
+
+So, we're looking at maybe 50% slowdown using sqlite and
+persistent for associated files. OTOH, the normalized schema should
+perform better when adding an associated file to a key that already has many.
+
+For metadata, the story is much nicer. Querying for 30000 keys that all
+have a particular tag in their metadata takes 0.65s. So fast enough to be
+used in views.
+
+Update4: Comparing git-annex fsck using the sticky bit to the final sqlite
+implementation:
+
+sticky bit: 4m30.787s  
+sqlite: 4m40.789s  
diff --git a/doc/design/encryption.mdwn b/doc/design/encryption.mdwn
new file mode 100644
index 0000000000..e2e4bbf155
--- /dev/null
+++ b/doc/design/encryption.mdwn
@@ -0,0 +1,118 @@
+This was the design doc for [[/encryption]] and is preserved for
+the curious. For an example of using git-annex with an encrypted S3 remote,
+see [[tips/using_Amazon_S3]].
+
+[[!toc]]
+
+## encryption key management
+
+[[!template id=note text="""
+The basis of this scheme was originally developed by Lars Wirzenius et al
+[for Obnam](http://liw.fi/obnam/encryption/).
+"""]]
+
+Data is encrypted by GnuPG, using a symmetric cipher. The cipher is
+generated by GnuPG when the special remote is created. By default the
+best entropy pool is used, hence the generation may take a while; One
+can use `initremote` with the `--fast` option
+to speed up things, but at the expense of using random numbers of a
+lower quality. The generated cipher is then checked into your git
+repository, encrypted using one or more OpenPGP public keys. This scheme
+allows new OpenPGP private keys to be given access to content that has
+already been stored in the remote.
+
+Different encrypted remotes need to be able to each use different ciphers.
+Allowing multiple ciphers to be used within a single remote would add a lot
+of complexity, so is not supported.
+Instead, if you want a new cipher, create a new S3 bucket, or whatever.
+There does not seem to be much benefit to using the same cipher for
+two different encrypted remotes.
+
+So, the encrypted cipher is just stored with the rest of a remote's
+configuration in `remotes.log` (see [[internals]]). When `git
+annex intiremote` makes a remote, it generates a random symmetric
+cipher, and encrypt it with the specified gpg key. To allow another gpg
+public key access, update the encrypted cipher to be encrypted to both gpg
+keys.
+
+Note that there's a shared encryption mode where the cipher is not
+encrypted. When this mode is used, any clone of the git repository
+can decrypt files stored in its special remote.
+
+## filename enumeration
+
+If the names of files are encrypted or securely hashed, or whatever is
+chosen, this makes it harder for git-annex (let alone untrusted third parties!)
+to get a list of the files that are stored on a given enrypted remote.
+But, does git-annex really ever need to do such an enumeration?
+
+Apparently not. `git annex unused --from remote` can now check for
+unused data that is stored on a remote, and it does so based only on
+location log data for the remote. This assumes that the location log is
+kept accurately.
+
+What about `git annex fsck --from remote`? Such a command should be able to,
+for each file in the repository, contact the encrypted remote to check
+if it has the file. This can be done without enumeration, although it will
+mean running gpg once per file fscked, to get the encrypted filename.
+
+So, the files stored in the remote should be encrypted. But, it needs to
+be a repeatable encryption, so they cannot just be gpg encrypted, that
+would yeild a new name each time. Instead, HMAC is used. Any hash could
+be used with HMAC. SHA-1 is the default, but [[other_hashes|/encryption]]
+can be chosen for new remotes.
+
+It was suggested that it might not be wise to use the same cipher for both
+gpg and HMAC. Being paranoid, it's best not to tie the security of one
+to the security of the other. So, the encrypted cipher described above is
+actually split in two; the first half is used for HMAC, and the second
+half for gpg.
+
+----
+
+Does the HMAC cipher need to be gpg encrypted? Imagine if it were
+stored in plainext in the git repository. Anyone who can access
+the git repository already knows the actual filenames, and typically also
+the content hashes of annexed content. Having access to the HMAC cipher
+could perhaps be said to only let them verify that data they already
+know.
+
+While this seems a pretty persuasive argument, I'm not 100% convinced, and
+anyway, most times that the HMAC cipher is needed, the gpg cipher is also
+needed. Keeping the HMAC cipher encrypted does slow down two things:
+dropping content from encrypted remotes, and checking if encrypted remotes
+really have content. If it's later determined to be safe to not encrypt the
+HMAC cipher, the current design allows changing that, even for existing
+remotes.
+
+## other use of the symmetric cipher
+
+The symmetric cipher can be used to encrypt other content than the content
+sent to the remote. In particular, it may make sense to encrypt whatever
+access keys are used by the special remote with the cipher, and store that
+in remotes.log. This way anyone whose gpg key has been given access to 
+the cipher can get access to whatever other credentials are needed to
+use the special remote.
+
+For example, the S3 special remote does this if configured with
+embedcreds=yet.
+
+## risks
+
+A risk of this scheme is that, once the symmetric cipher has been
+obtained, it allows full access to all the encrypted content. Indeed
+anyone owning a key that used to be granted access could already have
+decrypted the cipher and stored a copy. While it is in possible to
+remove a key with `keyid-=`, it is designed for a
+[[completely_different_purpose|/encryption]] and does not actually revoke
+access.
+
+If git-annex stores the decrypted symmetric cipher in memory, then there
+is a risk that it could be intercepted from there by an attacker. Gpg
+ameliorates these type of risks by using locked memory. For git-annex, note
+that an attacker with local machine access can tell at least all the
+filenames and metadata of files stored in the encrypted remote anyway,
+and can access whatever content is stored locally.
+
+This design does not address obfuscating the size of files by chunking
+them. However, chunking was later added; see [[design/assistant/chunks]].
diff --git a/doc/design/encryption/comment_1_4715ffafb3c4a9915bc33f2b26aaa9c1._comment b/doc/design/encryption/comment_1_4715ffafb3c4a9915bc33f2b26aaa9c1._comment
new file mode 100644
index 0000000000..f2ecc46d0a
--- /dev/null
+++ b/doc/design/encryption/comment_1_4715ffafb3c4a9915bc33f2b26aaa9c1._comment
@@ -0,0 +1,12 @@
+[[!comment format=mdwn
+ username="https://www.google.com/accounts/o8/id?id=AItOawl9sYlePmv1xK-VvjBdN-5doOa_Xw-jH4U"
+ nickname="Richard"
+ subject="comment 1"
+ date="2011-04-03T20:03:14Z"
+ content="""
+New encryption keys could be used for different directories/files/patterns/times/whatever. One could then encrypt this new key for the public keys of other people/machines and push them out along with the actual data. This would allow some level of access restriction or future revocation. git-annex would need to keep track of which files can be decrypted with which keys. I am undecided if that information needs to be encrypted or not.
+
+Encrypted object files should be checksummed in encrypted form so that it's possible to verify integrity without knowing any keys. Same goes for encrypted keys, etc.
+
+Chunking files in this context seems like needless overkill. This might make sense to store a DVD image on CDs or similar, at some point. But not for encryption, imo. Coming up with sane chunk sizes for all use cases is literally impossible and as you pointed out, correlation by the remote admin is trivial.
+"""]]
diff --git a/doc/design/encryption/comment_2_a610b3d056a059899178859a3a821ea5._comment b/doc/design/encryption/comment_2_a610b3d056a059899178859a3a821ea5._comment
new file mode 100644
index 0000000000..d5461e23c0
--- /dev/null
+++ b/doc/design/encryption/comment_2_a610b3d056a059899178859a3a821ea5._comment
@@ -0,0 +1,10 @@
+[[!comment format=mdwn
+ username="http://joey.kitenet.net/"
+ nickname="joey"
+ subject="comment 2"
+ date="2011-04-05T18:41:49Z"
+ content="""
+I see no use case for verifying encrypted object files w/o access to the encryption key. And possible use cases for not allowing anyone to verify your data.
+
+If there are to be multiple encryption keys usable within a single encrypted remote, than they would need to be given some kind of name (a since symmetric key is used, there is no pubkey to provide a name), and the name encoded in the files stored in the remote. While certainly doable I'm not sold that adding a layer of indirection is worthwhile. It only seems it would be worthwhile if setting up a new encrypted remote was expensive to do. Perhaps that could be the case for some type of remote other than S3 buckets.
+"""]]
diff --git a/doc/design/encryption/comment_3_cca186a9536cd3f6e86994631b14231c._comment b/doc/design/encryption/comment_3_cca186a9536cd3f6e86994631b14231c._comment
new file mode 100644
index 0000000000..d3c483fdf3
--- /dev/null
+++ b/doc/design/encryption/comment_3_cca186a9536cd3f6e86994631b14231c._comment
@@ -0,0 +1,12 @@
+[[!comment format=mdwn
+ username="https://www.google.com/accounts/o8/id?id=AItOawl9sYlePmv1xK-VvjBdN-5doOa_Xw-jH4U"
+ nickname="Richard"
+ subject="comment 3"
+ date="2011-04-05T23:24:17Z"
+ content="""
+Assuming you're storing your encrypted annex with me and I with you, our regular cron jobs to verify all data will catch corruption in each other's annexes.
+
+Checksums of the encrypted objects could be optional, mitigating any potential attack scenarios.
+
+It's not only about the cost of setting up new remotes. It would also be a way to keep data in one annex while making it accessible only in a subset of them. For example, I might need some private letters at work, but I don't want my work machine to be able to access them all.
+"""]]
diff --git a/doc/design/encryption/comment_4_8f3ba3e504b058791fc6e6f9c38154cf._comment b/doc/design/encryption/comment_4_8f3ba3e504b058791fc6e6f9c38154cf._comment
new file mode 100644
index 0000000000..14eb1acac1
--- /dev/null
+++ b/doc/design/encryption/comment_4_8f3ba3e504b058791fc6e6f9c38154cf._comment
@@ -0,0 +1,10 @@
+[[!comment format=mdwn
+ username="http://joey.kitenet.net/"
+ nickname="joey"
+ subject="comment 4"
+ date="2011-04-07T19:59:30Z"
+ content="""
+@Richard the easy way to deal with that scenario is to set up a remote that work can access, and only put in it files work should be able to see. Needing to specify which key a file should be encrypted to when putting it in a remote that supported multiple keys would add another level of complexity which that avoids.
+
+Of course, the right approach is probably to have a separate repository for work. If you don't trust it with seeing file contents, you probably also don't trust it with the contents of your git repository.
+"""]]
diff --git a/doc/design/encryption/comment_5_520e60aa53217b5ba428d4c05d897dee._comment b/doc/design/encryption/comment_5_520e60aa53217b5ba428d4c05d897dee._comment
new file mode 100644
index 0000000000..ff579f49b8
--- /dev/null
+++ b/doc/design/encryption/comment_5_520e60aa53217b5ba428d4c05d897dee._comment
@@ -0,0 +1,16 @@
+[[!comment format=mdwn
+ username="https://www.google.com/accounts/o8/id?id=AItOawkS6aFVrEwOrDuQBTMXxtGHtueA69NS_jo"
+ nickname="Hans"
+ subject="using sshfs + cryptmount is more secure"
+ date="2012-08-14T13:41:47Z"
+ content="""
+\"For git-annex, note that an attacker with local machine access can tell at least all the filenames and metadata of files stored in the encrypted remote anyway, and can access whatever content is stored locally.\"
+
+Better security is given by sshfs + cryptmount, which I used when I recently setup a git-annex repository on a free shell account from a provider I do not trust.
+
+See http://code.cjb.net/free-secure-online-backup.html for what I did to get a really secure solution.
+
+Kind regards,
+
+Hans Ekbrand
+"""]]
diff --git a/doc/design/encryption/comment_6_d677fead0fe0c543f48f07d85f83f592._comment b/doc/design/encryption/comment_6_d677fead0fe0c543f48f07d85f83f592._comment
new file mode 100644
index 0000000000..a2c7013fb0
--- /dev/null
+++ b/doc/design/encryption/comment_6_d677fead0fe0c543f48f07d85f83f592._comment
@@ -0,0 +1,14 @@
+[[!comment format=mdwn
+ username="https://www.google.com/accounts/o8/id?id=AItOawmBUR4O9mofxVbpb8JV9mEbVfIYv670uJo"
+ nickname="Justin"
+ subject="comment 6"
+ date="2012-08-14T14:10:40Z"
+ content="""
+Hans,
+
+You are misunderstanding how git-annex encryption works.  The \"untrusted host\" and the \"local machine\" are not the same machine.  git-annex only transfers pre-encrypted files to the \"untrusted host\".
+
+You should setup a git-annex encrypted remote and watch how it works so you can see for yourself that it is not insecure.
+
+Your solution does not provide better security, it accomplishes the same thing as git-annex in a more complicated way.  In addition, since you are mounting the image from the client your solution will not work with multiple clients.
+"""]]
diff --git a/doc/design/encryption/comment_7_c1c38a09b1276e29adc3ba564dc0fe4e._comment b/doc/design/encryption/comment_7_c1c38a09b1276e29adc3ba564dc0fe4e._comment
new file mode 100644
index 0000000000..259214495a
--- /dev/null
+++ b/doc/design/encryption/comment_7_c1c38a09b1276e29adc3ba564dc0fe4e._comment
@@ -0,0 +1,14 @@
+[[!comment format=mdwn
+ username="https://www.google.com/accounts/o8/id?id=AItOawkS6aFVrEwOrDuQBTMXxtGHtueA69NS_jo"
+ nickname="Hans"
+ subject="comment 7"
+ date="2012-08-15T19:16:10Z"
+ content="""
+Justin,
+
+thanks for clearing that up. It's great that git-annex has implemented mechanisms to work securely on untrusted hosts. My solution is thus only interesting for files that are impractical to manage with git-annex (e.g. data for/from applications that need rw-access to a large number of files). And, possibly, for providers that do not provide rsync.
+
+Your remark that my solution does not work with more than one client, is not entirely accurate. No more than one client can access the repository at any given time, but as long as access is not simultaneous, any number of clients can access the repository. Still, your point is taken, it's a limitation I should mention.
+
+It would be interesting to compare the performance of individually encrypted files to encrypted image-file. My intuition says that encrypted image-file should be faster, but that's just a guess.
+"""]]
diff --git a/doc/design/exporting_trees_to_special_remotes.mdwn b/doc/design/exporting_trees_to_special_remotes.mdwn
new file mode 100644
index 0000000000..6cf7383605
--- /dev/null
+++ b/doc/design/exporting_trees_to_special_remotes.mdwn
@@ -0,0 +1,352 @@
+For publishing content from a git-annex repository, it would be useful to
+be able to export a tree of files to a special remote, using the filenames
+and content from the tree.
+
+(See also [[todo/export]] and [[todo/dumb, unsafe, human-readable_backend]])
+
+[[!toc ]]
+
+## configuring a special remote for tree export
+
+If a special remote already has files stored in it, switching it to be a
+tree export would result in a mix of files named by key and by filename.
+That's not desirable. So, the user should set up a new special remote
+when they want to export a tree. (It would also be possible to drop all content
+from an existing special remote and reuse it, but there does not seem much
+benefit in doing so.)
+
+Add a new `initremote` configuration `exporttree=yes`, that cannot be
+changed by `enableremote`:
+
+	git annex initremote myexport type=... exporttree=yes
+
+It does not make sense to encrypt an export, so exporttree=yes requires
+encryption=none.
+
+Note that the particular tree to export is not specified yet. This is
+because the tree that is exported to a special remote may change.
+
+## exporting a treeish
+
+To export a treeish, the user can run:
+
+	git annex export $treeish --to myexport
+
+That does all necessary uploads etc to make the special remote contain
+the tree of files. The treeish can be a tag, a branch, or a tree.
+
+If a file's content is not present, it won't be exported. Re-running the
+same export later should export files whose content has become present.
+(This likely means a second pass, and needs location tracking to track
+which files are in the export.)
+
+Users may sometimes want to export multiple treeishes to a single special 
+remote. For example, exporting several tags. This interface could be
+complicated to support that, putting the treeishes in subdirectories on the
+special remote etc. But that's not necessary, because the user can use git
+commands to graft trees together into a larger tree, and export that larger
+tree.
+
+If an export is interrupted, running it again should resume where it left
+off.
+
+## updating an export
+
+The user can at any time re-run git-annex export with a new treeish
+to change what's exported. While some use cases for git annex export
+involve publishing datasets that are intended to remain immutable,
+other use cases include eg, making a tree of files available to a computer
+that can't run git-annex, and in such use cases, the tree needs to be able
+to be updated.
+
+To efficiently update an export, git-annex can diff the tree
+that was exported with the new tree. The naive approach is to upload
+new and modified files and remove deleted files.
+
+With rename detection, if the special remote supports moving files,
+more efficient updates can be done. It gets complicated; consider two files
+that swap names.
+
+If the special remote supports copying files, that would also make some
+updates more efficient.
+
+## tracking exports
+
+This lets the user say, "I want to export the master branch", 
+and have git-annex sync and the assistant automatically update the export
+when master changes.
+
+git-annex export could do this by default (if the user doesn't want the
+export to track the branch, they could instead export a tree or a tag). Or
+it could be a --tracking parameter.
+
+How to record the export tracking branch? It could be stored
+as refs/remotes/myexport/master. This says that the master branch
+is being exported to myexport, and the ref points to the last treeish
+that was exported.
+
+But.. master:subdir is a valid treeish, referring to the subdir
+of the current master tree. This is a useful thing to want to export.
+But, that's not a legal ref name. So, perhaps better to record
+the export tracking branch some other way. Perhaps in git config?
+
+## changes to special remote interface
+
+This needs some additional methods added to special remotes, and to
+the [[external_special_remote_protocol]].
+
+Here's the changes to the latter:
+
+* `EXPORTSUPPORTED`  
+  Used to check if a special remote supports exports. The remote
+  responds with either `EXPORTSUPPORTED-SUCCESS` or
+  `EXPORTSUPPORTED-FAILURE`
+* `EXPORT Name`  
+  Comes immediately before each of the following requests, 
+  specifying the name of the exported file. It will be in the form
+  of a relative path, and may contain path separators, whitespace,
+  and other special characters.
+* `TRANSFEREXPORT STORE|RETRIEVE Key File`  
+  Requests the transfer of a File on local disk to or from the previously 
+  provided Name on the special remote.  
+  Note that it's important that, while a file is being stored,
+  CHECKPRESENTEXPORT not indicate it's present until all the data has
+  been transferred.  
+  The remote responds with either `TRANSFER-SUCCESS` or
+  `TRANSFER-FAILURE`, and a remote where exports do not make sense
+  may always fail.
+* `CHECKPRESENTEXPORT Key`  
+  Requests the remote to check if the previously provided Name is present
+  in it.  
+  The remote responds with `CHECKPRESENT-SUCCESS`, `CHECKPRESENT-FAILURE`,
+  or `CHECKPRESENT-UNKNOWN`.
+* `REMOVEEXPORT Key`  
+  Requests the remote to remove content stored by `TRANSFEREXPORT`
+  with the previously provided Name.  
+  The remote responds with either `REMOVE-SUCCESS` or
+  `REMOVE-FAILURE`.
+* `RENAMEEXPORT Key NewName`  
+  Requests the remote rename a file stored on it from the previously
+  provided Name to the NewName.  
+  The remote responds with `RENAMEEXPORT-SUCCESS` or with
+  `RENAMEEXPORT-FAILURE` if an efficient rename cannot be done.
+
+To support old external special remote programs that have not been updated
+to support exports, git-annex will need to handle an `ERROR` response
+when using any of the above.
+
+## location tracking
+
+Since not all the files in an exported treeish may have content
+present when the export is done, location tracking will be needed so that
+getting the files and exporting again transfers their content.
+
+Does a copy of a file exported to a special remote count as a copy
+of a file as far as [[numcopies]] goes? Should git-annex get download
+a file from an export?
+
+The problem is that special remotes with exports are not
+key/value stores. The content of a file can change, and if multiple
+repositories can export a special remote, they can be out of sync about
+what files are exported to it. 
+
+Possible solution: Make exporttree=yes cause the special remote to
+be untrusted, and rely on annex.verify to catch cases where the content
+of a file on a special remote has changed. This would work well enough
+except for when the WORM or URL backend is used. So, prevent the user
+from exporting such keys. Also, force verification on for such special
+remotes, don't let it be turned off.
+
+The same file contents may be in a treeish multiple times under different
+filenames. That complicates using location tracking. One file may have been
+exported and the other not, and location tracking says that the content
+is present in the export. A sqlite database is needed to keep track of
+this.
+
+## recording exported filenames in git-annex branch
+
+In order to download the content of a key from a file exported
+to a special remote, the filename that was exported needs to somehow
+be recorded in the git-annex branch. How to do this? The filename could
+be included in the location tracking log or a related log file, or 
+the exported tree could be grafted into the git-annex branch
+(under eg, `exported/uuid/`). Which way uses less space in the git repository?
+
+Grafting in the exported tree records the necessary data, but the
+file-to-key map needs to be reversed to support downloading from an export.
+It would be too expensive to traverse the tree each time to hunt for a key;
+instead would need a database that gets populated once by traversing the
+tree.
+
+On the other hand, for updating what's exported, having access to the old
+exported tree seems perfect, because it and the new tree can be diffed to
+find what changes need to be made to the special remote. 
+
+If the filenames are stored in the location tracking log, the exported tree
+could be reconstructed, but it would take O(N) queries to git, where N is
+the total number of keys git-annex knows about; updating exports of small
+subsets of large repositories would be expensive. So grafting in the
+exported tree seems the better approach.
+
+## export conflicts
+
+What if different repositories can access the same special remote,
+and different trees get exported to it concurrently?
+
+This would be very hard to untangle, because it's hard to know what
+content was exported to a file last, and thus what content the file
+actually has. The location log's timestamps might give a hint, 
+but clocks vary too much to trust it.
+
+Also, if the exported tree is grafted in to the git-annex branch, 
+there would be a merge conflict. Union merging would *scramble* the exported 
+tree, so even if a smart merge is added, old versions of git-annex would 
+corrupt the exported tree.
+
+To avoid that problem, add a log file `export.log` that contains the uuid
+of the remote that was exported to, and the sha1 of the exported tree.
+To avoid the exported tree being GCed, do graft it in to the git-annex
+branch, but follow that with a commit that removes the tree again,
+and only update `refs/heads/git-annex` after making both commits.
+
+If `export.log` contains multiple active exports of different trees,
+there was an export conflict. Short of downloading the whole export to
+checksum it, or deleting the whole export, what can be done to resolve it?
+
+In this case, git-annex knows both exported trees. Have the user provide
+a tree that resolves the conflict as they desire (it could be the same as
+one of the exported trees, or some merge of them or an entirely new tree).
+The UI to do this can just be another `git annex export $tree --to remote`.
+To resolve, diff each exported tree in turn against the resolving tree
+and delete all files that differ. Then, upload all missing files.
+
+## when to update export.log for efficient resuming of exports
+
+When should `export.log` be updated? Possibilities:
+
+* Before performing any work, to set the goal.
+* After the export is fully successful, to record the current state.
+* After some mid-point.
+
+Lots of things could go wrong during an export. A file might fail to be
+transferred or only part of it be transferred; a file's content might not
+be present to transfer at all. The export could be interrupted part way.
+Updating the export.log at the right point in time is important to handle
+these cases efficiently.
+
+If the export.log is updated first, then it's only a goal and does not tell
+us what's been done already.
+
+If the export.log is updated only after complete success, then the common
+case of some files not having content locally present will prevent it from
+being updated. When we resume, we again don't know what's been done
+already.
+
+If the export.log is updated after deleting any files from the
+remote that are not the same in the new treeish as in the old treeish,
+and as long as TRANSFEREXPORT STORE is atomic, then when resuming we can
+trust CHECKPRESENTEXPORT to only find files that have the correct content
+for the current treeish. (Unless a conflicting export was made from
+elsewhere, but in that case, the conflict resolution will have to fix up
+later.)
+
+## handling renames efficiently
+
+To handle two files that swap names, a temp name is required.
+
+Difficulty with a temp name is picking a name that won't ever be used by
+any exported file.
+
+Interrupted exports also complicate this. While a name could be picked that
+is in neither the old nor the new tree, an export could be interrupted,
+leaving the file at the temp name. There needs to be something to clean
+that up when the export is resumed, even if it's resumed with a different 
+tree.
+
+Could use something like ".git-annex-tmp-content-$key" as the temp name.
+This hides it from casual view, which is good, and it's not depedent on the
+tree, so no state needs to be maintained to clean it up. Also, using the
+key in the name simplifies calculation of complicated renames (eg, renaming
+A to B, B to C, C to A)
+
+Export can first try to rename all files that are deleted/modified
+to their key's temp name (falling back to deleting since not all
+special remotes support rename), and then, in a second pass, rename
+from the temp name to the new name. Followed by deleting the temp name
+of all keys whose files are deleted in the diff. That is more renames and
+deletes than strictly necessary, but it will statelessly clean up 
+an interruped export as long as it's run again with the same new tree.
+
+But, an export of tree B should clean up after 
+an interrupted export of tree A. Some state is needed to handle this.
+Before starting the export of tree A, record it somewhere. Then when
+resuming, diff A..B, and delete the temp names of the keys in the
+diff. (Can't rename here, because we don't know what was the content
+of a file when an export was interrupted.)
+
+So, before an export does anything, need to record the tree that's about
+to be exported to export.log, not as an exported tree, but as a goal.
+Then on resume, the temp files for that can be cleaned up.
+
+## renames and export conflicts
+
+What is there's an export conflict going on at the same time that a file
+in the export gets renamed?
+
+Suppose that there are two git repos A and B, each exporting to the same
+remote. A and B are not currently communicating. A exports T1 which
+contains F. B exports T2, which has a different content for F. 
+
+Then A exports T3, which renames F to G. If that rename is done
+on the remote, then A will think it's successfully exported T3,
+but G will have F's content from T2, not from T1.
+
+When A and B reconnect, the export conflict will be detected. 
+To resolve the export conflict, it says above to:
+
+> To resolve, diff each exported tree in turn against the resolving tree
+> and delete all files that differ. Then, upload all missing files.
+
+Assume that the resolving tree is T3. So B's export of T2 is diffed against
+T3. F differs and is deleted (no change). G differs and is deleted,
+which fixes up the problem that the wrong content was renamed to G.
+G is missing so gets uploaded.
+
+So, this works, as long as "delete all files that differ" means it
+deletes both old and new files. And as long as conflict resolution does not
+itself stash away files in the temp name for later renaming.
+
+## dropping from exports and copying to exports
+
+It might be nice for `git annex drop $file --from myexport` and
+`git annex copy $myfile --to export` to work. However, there are some
+very difficult issues in supporting those, and they don't really
+seem necessary to use exports. Re-running `git annex export`
+to resume an export handles all the cases that copying to an export
+would need to. And, deleting a file from a tree and exporting the new tree
+is the thing to do if a file no longer should be exported.
+
+Here's an example of the kind of problem supporting these needs to deal with:
+
+1. In repo A, file F with content K is exported
+2. In repo B, file F with content K' is exported, since F changed in the
+   exported treeish.
+3. In repo A, file F is removed from the export, which results in
+   K being removed from the location log for the export.
+
+But... did #3 happen before or after #2?
+If #3 occurred before #2, then K' is present in the export
+and the location log is correct.
+If #3 occurred after #2, and A and B's git-annex branches
+were not synced, then K' was accidentially removed
+from the export, and the location log is now wrong.
+
+Is there any reason to allow removeKey from an export?
+Why would someone want to drop a single file from an export?
+Why not remove the file from a tree, and export the new tree?
+
+(Alternatively, removeKey could itself update the exported tree,
+removing the file from it, and update the export log accordingly.
+This would avoid the problem. But that's complication and it would be
+rather slow and bloat the git repo with a lot of intermediate trees
+when dropping multiple keys.)
diff --git a/doc/design/exporting_trees_to_special_remotes/comment_10_75ba45174d3c4b927113a6908061b742._comment b/doc/design/exporting_trees_to_special_remotes/comment_10_75ba45174d3c4b927113a6908061b742._comment
new file mode 100644
index 0000000000..15a28e7ced
--- /dev/null
+++ b/doc/design/exporting_trees_to_special_remotes/comment_10_75ba45174d3c4b927113a6908061b742._comment
@@ -0,0 +1,15 @@
+[[!comment format=mdwn
+ username="yarikoptic"
+ avatar="http://cdn.libravatar.org/avatar/f11e9c84cb18d26a1748c33b48c924b4"
+ subject="export "each revision" -- thinking about quiltdata"
+ date="2017-07-14T20:10:42Z"
+ content="""
+In some cases, if remote supports versioning, might be cool to be able to export all versions (from previously exported point, assuming linear progression).
+Having a chat with [https://quiltdata.com/](https://quiltdata.com/) folks, project which I just got to know about.
+1. They claim/hope to provide infinite storage for public datasets
+2. They do support \"File\" model, so dataset could simply contain files.  If we could (ab)use that -- sounds like a lovely free ride
+3. They do support versioning. If we could export all the versions -- super lovely.  
+
+Might also help to establish interoperability between the tools
+
+"""]]
diff --git a/doc/design/exporting_trees_to_special_remotes/comment_11_0827f5611c8e7e7ffa2633f8c06ae055._comment b/doc/design/exporting_trees_to_special_remotes/comment_11_0827f5611c8e7e7ffa2633f8c06ae055._comment
new file mode 100644
index 0000000000..6cb922e817
--- /dev/null
+++ b/doc/design/exporting_trees_to_special_remotes/comment_11_0827f5611c8e7e7ffa2633f8c06ae055._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="xloem"
+ avatar="http://cdn.libravatar.org/avatar/b8c087f7c5e6a9358748f0727c077f3b"
+ subject="Git History"
+ date="2017-08-10T00:25:27Z"
+ content="""
+It would be great to have an option to include git history in the export, such that a special remote could be used both to rebuild a repository and to view the contents.
+"""]]
diff --git a/doc/design/exporting_trees_to_special_remotes/comment_12_1bb8d57383ca733f3a0069ff30181366._comment b/doc/design/exporting_trees_to_special_remotes/comment_12_1bb8d57383ca733f3a0069ff30181366._comment
new file mode 100644
index 0000000000..a039beb5f8
--- /dev/null
+++ b/doc/design/exporting_trees_to_special_remotes/comment_12_1bb8d57383ca733f3a0069ff30181366._comment
@@ -0,0 +1,10 @@
+[[!comment format=mdwn
+ username="joey"
+ subject="""re: export "each revision""""
+ date="2017-08-15T18:13:38Z"
+ content="""
+That sounds much more like a regular remote with `git annex copy --all`.
+
+This entire design is preducated on exporting a single treeish. If you want
+to make a single treeish containing all versions of every file ...
+"""]]
diff --git a/doc/design/exporting_trees_to_special_remotes/comment_13_f857e15124b70ae1abc74669f63a2d68._comment b/doc/design/exporting_trees_to_special_remotes/comment_13_f857e15124b70ae1abc74669f63a2d68._comment
new file mode 100644
index 0000000000..951827d494
--- /dev/null
+++ b/doc/design/exporting_trees_to_special_remotes/comment_13_f857e15124b70ae1abc74669f63a2d68._comment
@@ -0,0 +1,9 @@
+[[!comment format=mdwn
+ username="joey"
+ subject="""re: Git History"""
+ date="2017-08-15T18:16:31Z"
+ content="""
+That is entirely out of scope. You're looking for a way to store a git
+*repository* someplace like S3. Such things exist already, and are not
+git-annex and I'm not going to replicate them as part of this feature.
+"""]]
diff --git a/doc/design/exporting_trees_to_special_remotes/comment_14_41e8ef74c6e62a4321d2046b2571a246._comment b/doc/design/exporting_trees_to_special_remotes/comment_14_41e8ef74c6e62a4321d2046b2571a246._comment
new file mode 100644
index 0000000000..45b675572e
--- /dev/null
+++ b/doc/design/exporting_trees_to_special_remotes/comment_14_41e8ef74c6e62a4321d2046b2571a246._comment
@@ -0,0 +1,17 @@
+[[!comment format=mdwn
+ username="joey"
+ subject="""re: comments on protocol"""
+ date="2017-08-15T18:18:14Z"
+ content="""
+In `TRANSFEREXPORT STORE|RETRIEVE Key File Name`, it should always
+be possible for the File to not contain spaces in its name. But it could be
+rather painful for git-annex to avoid spaces in some cases (would need to
+link or copy the annexed file content). So well spotted. 
+
+Hmm, it's actually possible for a Key to contain spaces as well,
+at least with the WORM backend.
+[[bugs/external_special_remote_protocol_broken_by_key_with_spaces]]
+
+The protocol `VERSION` is picked by the special remote, it's not
+negotiated.
+"""]]
diff --git a/doc/design/exporting_trees_to_special_remotes/comment_15_3fc518cee1a11b28da769c0915d33e3b._comment b/doc/design/exporting_trees_to_special_remotes/comment_15_3fc518cee1a11b28da769c0915d33e3b._comment
new file mode 100644
index 0000000000..f0de4d49a0
--- /dev/null
+++ b/doc/design/exporting_trees_to_special_remotes/comment_15_3fc518cee1a11b28da769c0915d33e3b._comment
@@ -0,0 +1,13 @@
+[[!comment format=mdwn
+ username="joey"
+ subject="""comment 15"""
+ date="2017-08-28T19:00:10Z"
+ content="""
+Since [[bugs/external_special_remote_protocol_broken_by_key_with_spaces]]
+was fixed, the Key can't contain spaces any longer.
+
+The File could still contain spaces, eg when exporting from a direct mode
+repository where the worktree filename contains spaces.
+
+In `RENAMEEXPORT`, both OldName and NewName could contain spaces.
+"""]]
diff --git a/doc/design/exporting_trees_to_special_remotes/comment_16_29f598eda413c0d5e17536d8f9438d31._comment b/doc/design/exporting_trees_to_special_remotes/comment_16_29f598eda413c0d5e17536d8f9438d31._comment
new file mode 100644
index 0000000000..98a9f93f04
--- /dev/null
+++ b/doc/design/exporting_trees_to_special_remotes/comment_16_29f598eda413c0d5e17536d8f9438d31._comment
@@ -0,0 +1,11 @@
+[[!comment format=mdwn
+ username="joey"
+ subject="""comment 16"""
+ date="2017-08-28T19:32:06Z"
+ content="""
+I've updated the proposed external special remote protocol to avoid the
+whitespace concerns. Not wild about needing a separate EXPORT request,
+which will probably get shoved into a global variable in most
+implementations. But it does avoid needing to use some kind of encoding,
+which would complicate implementations more, I feel.
+"""]]
diff --git a/doc/design/exporting_trees_to_special_remotes/comment_17_32a3240206c5ffff71a47dffa6950c48._comment b/doc/design/exporting_trees_to_special_remotes/comment_17_32a3240206c5ffff71a47dffa6950c48._comment
new file mode 100644
index 0000000000..2afef92a8f
--- /dev/null
+++ b/doc/design/exporting_trees_to_special_remotes/comment_17_32a3240206c5ffff71a47dffa6950c48._comment
@@ -0,0 +1,23 @@
+[[!comment format=mdwn
+ username="yarikoptic"
+ avatar="http://cdn.libravatar.org/avatar/f11e9c84cb18d26a1748c33b48c924b4"
+ subject="protocol message "
+ date="2018-02-06T20:03:27Z"
+ content="""
+joey wrote:
+
+    The protocol VERSION is picked by the special remote, it's not negotiated.
+
+`VERSION ` is provided to git-annex by the special remote to git-annex process.  There is no need to 'negotiate' anything - you could make git-annex understand either of:
+
+- higher `VERSION `, e.g. 
+
+  - `VERSION 2` which would support some new features which that special remote would need.  If parent git-annex is old/doesn't support that version - would fail and demand git annex upgrade
+  - `VERSION 6.20171124` (where `6.20171124` is an example of git-annex version) so if git-annex parent process is older than that it could provide a meaningful message that `git annex >= 6.20171124` is needed
+
+- `VERSION 1 feature1 feature2 ...` where those features could be the ones needed (e.g. `INFO_MSG` for [recent addition](http://git-annex.branchable.com/todo/INFO_message_for_custom_special_remotes/#comment-4dcfb7d4e6db9d5ba8a1bfeb782346b1)). And if parent git-annex doesn't know/support any particular feature, it could fail and inform user that a new annex with that feature support is needed.
+
+In either of those cases the custom special remotes page could outline added features/versions of git-annex supporting them, so may be even those above error messages could point to it.
+
+Overall, it is just a minor change to be done on git-annex side while allowing for clear(er) specification, and I do not see any need for actual \"negotiation\" -- features are either supported or not by the parent process.
+"""]]
diff --git a/doc/design/exporting_trees_to_special_remotes/comment_18_fe77370699b7ce0acd547fd1e045e254._comment b/doc/design/exporting_trees_to_special_remotes/comment_18_fe77370699b7ce0acd547fd1e045e254._comment
new file mode 100644
index 0000000000..bebfee6ed5
--- /dev/null
+++ b/doc/design/exporting_trees_to_special_remotes/comment_18_fe77370699b7ce0acd547fd1e045e254._comment
@@ -0,0 +1,15 @@
+[[!comment format=mdwn
+ username="joey"
+ subject="""comment 18"""
+ date="2018-02-07T16:43:02Z"
+ content="""
+Changing VERSION would prevent any older versions of git-annex from working
+with that external special remote, since they would reject the unknown
+version. (The current parsing of VERSION also happens to preclude adding
+some fields after the number.)
+
+Since it seems completely possible to make the protocol be changed in a way
+that is backwards compatible both ways, while still letting new features to
+be used, I'd rather reserve changing VERSION for whatever future thing
+needs a full breaking bump.
+"""]]
diff --git a/doc/design/exporting_trees_to_special_remotes/comment_19_00d28c758509939974e4583e9b1b9e12._comment b/doc/design/exporting_trees_to_special_remotes/comment_19_00d28c758509939974e4583e9b1b9e12._comment
new file mode 100644
index 0000000000..5e1835eb3d
--- /dev/null
+++ b/doc/design/exporting_trees_to_special_remotes/comment_19_00d28c758509939974e4583e9b1b9e12._comment
@@ -0,0 +1,10 @@
+[[!comment format=mdwn
+ username="yarikoptic"
+ avatar="http://cdn.libravatar.org/avatar/f11e9c84cb18d26a1748c33b48c924b4"
+ subject="comment 19"
+ date="2018-02-07T18:31:45Z"
+ content="""
+> Changing VERSION would prevent any older versions of git-annex from working with that external special remote, since they would reject the unknown version. (The current parsing of VERSION also happens to preclude adding some fields after the number.)
+
+I still do not get it, sorry -- If there is an older git-annex, and a special remote requests some higher VERSION (thus stating that it needs some features older git-annex does not support), IMHO it would be perfectly fine to fail to use that remote since it wouldn't be usable anyways with that older git-annex (i.e. require some special features it does not provide).  If special remote does not need any feature not present in version `1`, it (like all of them ATM) could still keep requesting `VERSION 1` thus staying compatible with whatever old git-annex is out there.
+"""]]
diff --git a/doc/design/exporting_trees_to_special_remotes/comment_19_1eda1af40f6ca82a8bacd19afaa749bc._comment b/doc/design/exporting_trees_to_special_remotes/comment_19_1eda1af40f6ca82a8bacd19afaa749bc._comment
new file mode 100644
index 0000000000..c67370a074
--- /dev/null
+++ b/doc/design/exporting_trees_to_special_remotes/comment_19_1eda1af40f6ca82a8bacd19afaa749bc._comment
@@ -0,0 +1,18 @@
+[[!comment format=mdwn
+ username="joey"
+ subject="""comment 19"""
+ date="2018-02-07T19:04:03Z"
+ content="""
+What if the remote wants to use some feature like NOTE, but can still
+manage to work when an old git-annex does not support it? Hard bumping the
+VERSION cannot support that. If the remote requires to be able to use NOTE
+and sees it cannot, it can still throw an error.
+
+There are a bunch of requests in the protocol that are optional for the
+remote to support; git-annex deals with remotes that don't support them in
+better ways than throwing up its hands because the special remote is too
+old. It's very good that the protocol allowed adding those extensions
+without bumping a version. The protocol is less extensible when it comes
+replies and other messages sent by the special remote, and I want to get
+the same extensibility for those.
+"""]]
diff --git a/doc/design/exporting_trees_to_special_remotes/comment_1_ea84ee9de604e05b8e02483ba8452186._comment b/doc/design/exporting_trees_to_special_remotes/comment_1_ea84ee9de604e05b8e02483ba8452186._comment
new file mode 100644
index 0000000000..c6e7c7aedd
--- /dev/null
+++ b/doc/design/exporting_trees_to_special_remotes/comment_1_ea84ee9de604e05b8e02483ba8452186._comment
@@ -0,0 +1,15 @@
+[[!comment format=mdwn
+ username="yarikoptic"
+ avatar="http://cdn.libravatar.org/avatar/f11e9c84cb18d26a1748c33b48c924b4"
+ subject="note that some remotes could support files versioning "natively""
+ date="2017-07-11T21:59:49Z"
+ content="""
+E.g. when exporting to the S3 bucket with versioning turned on, or OSF (AFAIK).  So upon successful upload special remote could SETURLPRESENT to signal availability of any particular key (associated with the file).
+
+Yet to grasp the cases you outlined better to see if I see any other applicable use-ase
+
+I hope that export would be implemented through extending externals special remote protocol? ;)
+
+[[!meta author=yoh]]
+
+"""]]
diff --git a/doc/design/exporting_trees_to_special_remotes/comment_21_062098e1f54b874467793e4487a45a9b._comment b/doc/design/exporting_trees_to_special_remotes/comment_21_062098e1f54b874467793e4487a45a9b._comment
new file mode 100644
index 0000000000..dfb5cb4569
--- /dev/null
+++ b/doc/design/exporting_trees_to_special_remotes/comment_21_062098e1f54b874467793e4487a45a9b._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="yarikoptic"
+ avatar="http://cdn.libravatar.org/avatar/f11e9c84cb18d26a1748c33b48c924b4"
+ subject="comment 21"
+ date="2018-02-07T20:01:53Z"
+ content="""
+Ok, gotcha.  Shouldn't then EXTENSION entries also be somehow versioned per each one of them? or if needed a new extension would be born by appending a version to its name (e.g. as with all those imap, imap2, imap3, ... ;-))
+"""]]
diff --git a/doc/design/exporting_trees_to_special_remotes/comment_2_d414fb575845770e003a3c8ca4a986be._comment b/doc/design/exporting_trees_to_special_remotes/comment_2_d414fb575845770e003a3c8ca4a986be._comment
new file mode 100644
index 0000000000..f0fc092ad4
--- /dev/null
+++ b/doc/design/exporting_trees_to_special_remotes/comment_2_d414fb575845770e003a3c8ca4a986be._comment
@@ -0,0 +1,10 @@
+[[!comment format=mdwn
+ username="yarikoptic"
+ avatar="http://cdn.libravatar.org/avatar/f11e9c84cb18d26a1748c33b48c924b4"
+ subject="couldn't STATE be used for KEY -> FILENAME(s) mapping?"
+ date="2017-07-11T22:05:49Z"
+ content="""
+just wondered...
+at least in my attempt for zenodo special remote I did store zenodo's file deposition ID within the state to be able to request it back later on
+alternative -- URL(s) I guess.  Could be smth like exported:UUID/filename.
+"""]]
diff --git a/doc/design/exporting_trees_to_special_remotes/comment_3_cb063cdc66df79c40039bce247b7170c._comment b/doc/design/exporting_trees_to_special_remotes/comment_3_cb063cdc66df79c40039bce247b7170c._comment
new file mode 100644
index 0000000000..377d167f73
--- /dev/null
+++ b/doc/design/exporting_trees_to_special_remotes/comment_3_cb063cdc66df79c40039bce247b7170c._comment
@@ -0,0 +1,9 @@
+[[!comment format=mdwn
+ username="yarikoptic"
+ avatar="http://cdn.libravatar.org/avatar/f11e9c84cb18d26a1748c33b48c924b4"
+ subject="does it really need to be a new command ("export") or could be the same old "copy"?"
+ date="2017-07-11T22:14:39Z"
+ content="""
+or it could be just a mode of operation for a special remote depending on \"exporttree=true\" being set, where in one (old) case it would operate based on keys associated with the files pointed on the cmdline (or just keys for --auto or pointed by metadata), whenever when \"exporttree=true\" -- it would operate on filenames pointed on command line (or files found to be associated with the keys as pointed by --auto or by metadata)? 
+Then the same 'copy --to' could be used in both cases, streamlining user experience ;) 
+"""]]
diff --git a/doc/design/exporting_trees_to_special_remotes/comment_4_126ee5332ff88b3993d33d59328d4148._comment b/doc/design/exporting_trees_to_special_remotes/comment_4_126ee5332ff88b3993d33d59328d4148._comment
new file mode 100644
index 0000000000..89bc18d4c8
--- /dev/null
+++ b/doc/design/exporting_trees_to_special_remotes/comment_4_126ee5332ff88b3993d33d59328d4148._comment
@@ -0,0 +1,24 @@
+[[!comment format=mdwn
+ username="joey"
+ subject="""comment 4"""
+ date="2017-07-12T16:45:51Z"
+ content="""
+I've added a section with changes to the external special remote protocol.
+I included the Key in each of the new protocol commands, although it's not
+strictly neeed, to allow the implementation to use SETURLPRESENT, SETSTATE,
+etc.
+
+`git annex copy $file --to myexport` could perhaps work; the difficulty
+though is, what if you've exported branch foo, and then checked out bar,
+and so you told it to export one version of the file, and are running
+git-annex copy on a different version? It seems that git-annex would have
+to cross-check in this and similar commands, to detect such a situation. 
+Unsure how much more work that would be, both CPU time and implementation
+time.
+
+I do think that `git annex get` could download files from exports easily
+enough, but see the "location tracking" section for trust caveats.
+
+I'm not clear about what you're suggesting be done with versioning support
+in external special remotest?
+"""]]
diff --git a/doc/design/exporting_trees_to_special_remotes/comment_5_fcd9890013371dae6ffcd00561b8c625._comment b/doc/design/exporting_trees_to_special_remotes/comment_5_fcd9890013371dae6ffcd00561b8c625._comment
new file mode 100644
index 0000000000..fb73f0622d
--- /dev/null
+++ b/doc/design/exporting_trees_to_special_remotes/comment_5_fcd9890013371dae6ffcd00561b8c625._comment
@@ -0,0 +1,28 @@
+[[!comment format=mdwn
+ username="yarikoptic"
+ avatar="http://cdn.libravatar.org/avatar/f11e9c84cb18d26a1748c33b48c924b4"
+ subject="special remotes with versioning support"
+ date="2017-07-12T17:30:33Z"
+ content="""
+thanks -- I will check those all out!
+
+Meanwhile a quick one regarding \"I'm not clear about what you're suggesting be done with versioning support in external special remotes?\".
+
+I meant that in some cases there might be no need for any custom/special tracking per exported file would be needed -- upon export we could just register a unique URL for that particular version of the file for the corresponding KEY so later on it could be 'annex get'ed even if a new version of the file gets uploaded or removed.  So annex could just store those treeish(es) hexsha on what was exported last without any explicit additional tracking per file.  URL might be some custom one to be handled by the special remote backend.
+
+E.g. here is a list of versions (and corresponding urls) for a sample file on the s3 bucket
+
+[[!format sh \"\"\"
+$> datalad ls -aL s3://datalad-test0-versioned/3versions-allversioned.txt 
+Connecting to bucket: datalad-test0-versioned
+[INFO   ] S3 session: Connecting to the bucket datalad-test0-versioned 
+Bucket info:
+  Versioning: {'MfaDelete': 'Disabled', 'Versioning': 'Enabled'}
+     Website: datalad-test0-versioned.s3-website-us-east-1.amazonaws.com
+         ACL: 
+3versions-allversioned.txt            ...  http://datalad-test0-versioned.s3.amazonaws.com/3versions-allversioned.txt?versionId=Kvuind11HZh._dCPaDAb0OY9dRrQoTMn [OK]
+3versions-allversioned.txt            ...  http://datalad-test0-versioned.s3.amazonaws.com/3versions-allversioned.txt?versionId=b.qCuh7Sg58VIYj8TVHzbRS97EvejzEl [OK]
+3versions-allversioned.txt            ...  http://datalad-test0-versioned.s3.amazonaws.com/3versions-allversioned.txt?versionId=pNsV5jJrnGATkmNrP8.i_xNH6CY4Mo5s [OK]
+3versions-allversioned.txt_sameprefix ...  http://datalad-test0-versioned.s3.amazonaws.com/3versions-allversioned.txt_sameprefix?versionId=Mvsc4FgJWc6gExwSw1d6wsLrnk6wdDVa [OK]
+\"\"\"]]
+"""]]
diff --git a/doc/design/exporting_trees_to_special_remotes/comment_6_3217c2f852e5d9b1e4be2adff995dd24._comment b/doc/design/exporting_trees_to_special_remotes/comment_6_3217c2f852e5d9b1e4be2adff995dd24._comment
new file mode 100644
index 0000000000..4b1265bdd4
--- /dev/null
+++ b/doc/design/exporting_trees_to_special_remotes/comment_6_3217c2f852e5d9b1e4be2adff995dd24._comment
@@ -0,0 +1,19 @@
+[[!comment format=mdwn
+ username="joey"
+ subject="""comment 6"""
+ date="2017-07-12T18:09:00Z"
+ content="""
+That would almost work without any smarts on the git-annex side.
+When it tells the special remote to `REMOVEEXPORT`, the special remote
+could remove the file from the HEAD equivilant but retain the content in its
+versioned snapshots, and keep the url to that registered. But, that
+doesn't actually work, because the url is registered for that special
+remote, not the web special remote. Once git-annex thinks the file has been
+removed from the special remote, it will never try to use the url
+registered for that special remote.
+
+So, to support versioning-capable special remotes, there would need to be
+an additional response to `REMOVEEXPORT` that says "I removed it from HEAD,
+but I still have a copy in this url, which can be accessed using
+the web special remote".
+"""]]
diff --git a/doc/design/exporting_trees_to_special_remotes/comment_7_43a98b4b9d9eb54720a9c92cd8bb3a30._comment b/doc/design/exporting_trees_to_special_remotes/comment_7_43a98b4b9d9eb54720a9c92cd8bb3a30._comment
new file mode 100644
index 0000000000..0ba8327bd8
--- /dev/null
+++ b/doc/design/exporting_trees_to_special_remotes/comment_7_43a98b4b9d9eb54720a9c92cd8bb3a30._comment
@@ -0,0 +1,14 @@
+[[!comment format=mdwn
+ username="yarikoptic"
+ avatar="http://cdn.libravatar.org/avatar/f11e9c84cb18d26a1748c33b48c924b4"
+ subject="side-note about WebDAV&DeltaV"
+ date="2017-07-12T21:54:49Z"
+ content="""
+DAV = “Distributed Authoring and Versioning.”, but versioning was forgotten about in the original RFC. Only some servers/clients implement DeltaV spec (RFC 3253) which came later to fill that gap. 
+But in principle, any DeltaV-compliant WebDAV special remote could then be used for \"export\" while retaining access to all the versions.
+References: 
+- [WebDAV and Autoversioning - Version Control with Subversion](http://archive.oreilly.com/pub/a/opensource/excerpts/9780596510336/webdav-and-autoversioning.html)
+- [RFC 3253](http://www.webdav.org/specs/rfc3253.html)
+
+I have got interested whenever saw that box.com is supported through WebDAV but not sure if DeltaV is anyhow supported and apparently number of versions stored per file is anyways depends on type of the account (and no versions for a free personal one): https://community.box.com/t5/How-to-Guides-for-Managing/How-To-Track-Your-Files-and-File-Versions-Version-History/ta-p/329 
+"""]]
diff --git a/doc/design/exporting_trees_to_special_remotes/comment_8_7e512ef81c529b0392071b8a6dfe853b._comment b/doc/design/exporting_trees_to_special_remotes/comment_8_7e512ef81c529b0392071b8a6dfe853b._comment
new file mode 100644
index 0000000000..eef7d08cab
--- /dev/null
+++ b/doc/design/exporting_trees_to_special_remotes/comment_8_7e512ef81c529b0392071b8a6dfe853b._comment
@@ -0,0 +1,11 @@
+[[!comment format=mdwn
+ username="yarikoptic"
+ avatar="http://cdn.libravatar.org/avatar/f11e9c84cb18d26a1748c33b48c924b4"
+ subject="regarding setting a URL by custom special remote"
+ date="2017-07-12T22:04:38Z"
+ content="""
+I also wonder if `SETURLPRESENT Key Url` could also be extended to be `SETURLPRESENT Key Url Remote`, i.e. that a custom remote could register a URL to Web remote?
+In many cases I expect a \"custom uploader/exporter\" but then public URL being available, so demanding a custom external remote to fetch it would be a bit overkill.
+
+N.B. I already was burnt once on a large scale with our custom remote truthfully replying to CLAIMURL (since it can handle them if needed) to public URLs, thus absorbing them into it instead of relaying responsibility to 'Web' remote. Had to traverse dozens of datasets and duplicate urls from 'datalad' to 'Web' remote.
+"""]]
diff --git a/doc/design/exporting_trees_to_special_remotes/comment_9_6c588170f0b53c74c3c28ff08ed3509d._comment b/doc/design/exporting_trees_to_special_remotes/comment_9_6c588170f0b53c74c3c28ff08ed3509d._comment
new file mode 100644
index 0000000000..54b1f8ada7
--- /dev/null
+++ b/doc/design/exporting_trees_to_special_remotes/comment_9_6c588170f0b53c74c3c28ff08ed3509d._comment
@@ -0,0 +1,9 @@
+[[!comment format=mdwn
+ username="yarikoptic"
+ avatar="http://cdn.libravatar.org/avatar/f11e9c84cb18d26a1748c33b48c924b4"
+ subject="comments on protocol"
+ date="2017-07-12T22:09:54Z"
+ content="""
+- `TRANSFEREXPORT STORE|RETRIEVE Key File Name`  -- note that File could also contain spaces etc (not only the Name), so should be encoded somehow?
+- `old external special remote programs ... need to handle an ERROR response` -- why not just to boost protocol `VERSION` to e.g. `2` so those which implement this would reply with a new version #?
+"""]]
diff --git a/doc/design/external_special_remote_protocol.mdwn b/doc/design/external_special_remote_protocol.mdwn
new file mode 100644
index 0000000000..6f894b3b6f
--- /dev/null
+++ b/doc/design/external_special_remote_protocol.mdwn
@@ -0,0 +1,515 @@
+Communication between git-annex and a program implementing an external
+special remote uses this protocol.
+
+[[!toc]]
+
+## starting the program
+
+The external special remote program has a name like
+`git-annex-remote-$bar`. When 
+`git annex initremote foo type=external externaltype=$bar` is run,
+git-annex finds the appropriate program in PATH.
+
+The program is started by git-annex when it needs to access the special
+remote, and may be left running for a long period of time. This allows
+it to perform expensive setup tasks, etc. Note that git-annex may choose to
+start multiple instances of the program (eg, when multiple git-annex
+commands are run concurrently in a repository).
+
+## protocol overview
+
+Communication is via stdin and stdout. Therefore, the external special
+remote must avoid doing any prompting, or outputting anything like eg,
+progress to stdout. (Such stuff can be sent to stderr instead.)
+
+The protocol is line based. Messages are sent in either direction, from
+git-annex to the special remote, and from the special remote to git-annex.
+
+In order to avoid confusing interactions, one or the other has control
+at any given time, and is responsible for sending requests, while the other
+only sends replies to the requests.
+
+Each protocol line starts with a command, which is followed by the
+command's parameters (a fixed number per command), each separated by a
+single space. The last parameter may contain spaces. Parameters may be
+empty, but the separating spaces are still required in that case.
+
+## example session
+
+The special remote is responsible for sending the first message, indicating
+the version of the protocol it is using.
+
+	VERSION 1
+
+Recent versions of git-annex respond with a message indicating
+protocol extensions that it supports. Older versions of
+git-annex do not send this message. 
+
+	EXTENSIONS INFO
+
+The special remote can respond to that with its own EXTENSIONS message, which
+could have its own protocol extension details, but none are currently used.
+(It's also fine to reply with UNSUPPORTED-REQUEST.)
+
+	EXTENSIONS 
+
+Next, git-annex will generally send a message telling the special
+remote to start up. (Or it might send an INITREMOTE or EXPORTSUPPORTED,
+so don't hardcode this order.)
+
+	PREPARE
+
+The special remote can now ask git-annex for its configuration, as needed,
+and check that it's valid. git-annex responds with the configuration values
+
+	GETCONFIG directory
+	VALUE /media/usbdrive/repo
+	GETCONFIG automount
+	VALUE true
+
+Once the special remote is satisfied with its configuration and is
+ready to go, it tells git-annex that it's done with the PREPARE step:
+
+	PREPARE-SUCCESS
+
+Now git-annex will make a request. Let's suppose it wants to store a key. 
+
+	TRANSFER STORE somekey tmpfile
+
+The special remote can then start reading the tmpfile and storing it.
+While it's doing that, the special remote can send messages back to 
+git-annex to indicate what it's doing, or ask for other information.
+It will typically send progress messages, indicating how many
+bytes have been sent:
+
+	PROGRESS 10240
+	PROGRESS 20480
+
+Once the key has been stored, the special remote tells git-annex the result:
+
+	TRANSFER-SUCCESS STORE somekey
+
+Now git-annex will send its next request.
+
+Once git-annex is done with the special remote, it will close its stdin.
+The special remote program can then exit.
+
+## git-annex request messages
+
+These are messages git-annex sends to the special remote program.
+None of these messages require an immediate reply. The special
+remote can send any messages it likes while handling the requests.
+
+Once the special remote has finished performing the request, it should
+send one of the corresponding replies listed in the next section.
+
+The following requests *must* all be supported by the special remote.
+
+* `INITREMOTE`  
+  Requests the remote to initialize itself. This is where any one-time
+  setup tasks can be done, for example creating an Amazon S3 bucket.  
+  Note: This may be run repeatedly over time, as a remote is initialized in
+  different repositories, or as the configuration of a remote is changed.
+  (Both `git annex initremote` and `git-annex enableremote` run this.)
+  So any one-time setup tasks should be done idempotently.
+* `PREPARE`  
+  Tells the remote that it's time to prepare itself to be used.  
+  Only EXTENSIONS and INITREMOTE or EXPORTSUPPORTED can come before this.
+* `TRANSFER STORE|RETRIEVE Key File`  
+  Requests the transfer of a key. For STORE, the File is the file to upload;
+  for RETRIEVE the File is where to store the download.  
+  Note that the File should not influence the filename used on the remote.  
+  Note that in some cases, the File may contain whitespace.  
+  Note that it's important that, while a Key is being stored, CHECKPRESENT
+  not indicate it's present until all the data has been transferred.  
+* `CHECKPRESENT Key`  
+  Requests the remote to check if a key is present in it.
+* `REMOVE Key`  
+  Requests the remote to remove a key's contents.
+
+The following requests can optionally be supported. If not handled,
+replying with `UNSUPPORTED-REQUEST` is acceptable.
+
+* `EXTENSIONS List`  
+  Sent to indicate protocol extensions which git-annex is capable
+  of using. The list is a space-delimited list of protocol extension
+  keywords. The remote can reply to this with its own EXTENSIONS list.
+* `GETCOST`  
+  Requests the remote to return a use cost. Higher costs are more expensive.
+  (See Config/Cost.hs for some standard costs.)
+* `GETAVAILABILITY`
+  Requests the remote to send back an `AVAILABILITY` reply.
+  If the remote replies with `UNSUPPORTED-REQUEST`, its availability
+  is assumed to be global. So, only remotes that are only reachable
+  locally need to worry about implementing this.
+* `CLAIMURL Url`  
+  Asks the remote if it wishes to claim responsibility for downloading
+  an url. If so, the remote should send back an `CLAIMURL-SUCCESS` reply.
+  If not, it can send `CLAIMURL-FAILURE`.
+* `CHECKURL Url`  
+  Asks the remote to check if the url's content can currently be downloaded
+  (without downloading it). The remote replies with one of `CHECKURL-FAILURE`,
+  `CHECKURL-CONTENTS`, or `CHECKURL-MULTI`.
+* `WHEREIS Key`
+  Asks the remote to provide additional information about ways to access
+  the content of a key stored in it, such as eg, public urls.
+  This will be displayed to the user by eg, `git annex whereis`. The remote
+  replies with `WHEREIS-SUCCESS` or `WHEREIS-FAILURE`.  
+  Note that users expect `git annex whereis` to run fast, without eg,
+  network access.  
+  This is not needed when `SETURIPRESENT` is used, since such uris are
+  automatically displayed by `git annex whereis`.  
+* `GETINFO`  
+  Requests the remote to send some information describing its
+  configuration, for display by `git annex info`.  
+  Reply with a series of `INFOFIELD` each followed by `INFOVALUE`,
+  and concluded with `INFOEND`.
+* `EXPORTSUPPORTED`  
+  Used to check if a special remote supports exports. The remote
+  responds with either `EXPORTSUPPORTED-SUCCESS` or
+  `EXPORTSUPPORTED-FAILURE`. Note that this request may be made before
+  or after `PREPARE`.
+* `EXPORT Name`  
+  Comes immediately before each of the following export-related requests, 
+  specifying the name of the exported file. It will be in the form
+  of a relative path, and may contain path separators, whitespace,
+  and other special characters.  
+  No response is made to this message.
+* `TRANSFEREXPORT STORE|RETRIEVE Key File`  
+  Requests the transfer of a File on local disk to or from the previously 
+  provided Name on the special remote.  
+  Note that it's important that, while a file is being stored,
+  CHECKPRESENTEXPORT not indicate it's present until all the data has
+  been transferred.  
+  The remote responds with either `TRANSFER-SUCCESS` or
+  `TRANSFER-FAILURE`, and a remote where exports do not make sense
+  may always fail.
+* `CHECKPRESENTEXPORT Key`  
+  Requests the remote to check if the previously provided Name is present
+  in it.  
+  The remote responds with `CHECKPRESENT-SUCCESS`, `CHECKPRESENT-FAILURE`,
+  or `CHECKPRESENT-UNKNOWN`.
+* `REMOVEEXPORT Key`  
+  Requests the remote to remove content stored by `TRANSFEREXPORT`
+  with the previously provided Name.  
+  The remote responds with either `REMOVE-SUCCESS` or
+  `REMOVE-FAILURE`.  
+  If the content was already not present in the remote, it should
+  respond with `REMOVE-SUCCESS`.
+* `REMOVEEXPORTDIRECTORY Directory`  
+  Requests the remote remove an exported directory.  
+  If the remote does not use directories, or automatically cleans up
+  empty directories, this does not need to be implemented.  
+  The directory will be in the form of a relative path, and may contain path
+  separators, whitespace, and other special characters.  
+  Typically the directory will be empty, but it could possbly contain
+  files or other directories, and it's ok to remove those.  
+  The remote responds with either `REMOVEEXPORTDIRECTORY-SUCCESS`
+  or `REMOVEEXPORTDIRECTORY-FAILURE`.  
+  Should not fail if the directory was already removed.
+* `RENAMEEXPORT Key NewName`  
+  Requests the remote rename a file stored on it from the previously
+  provided Name to the NewName.  
+  The remote responds with `RENAMEEXPORT-SUCCESS` or
+  `RENAMEEXPORT-FAILURE`.
+
+To support old external special remote programs that have not been updated
+to support exports, git-annex will need to handle an `ERROR` response
+when using any of the above.
+
+More optional requests may be added, without changing the protocol version,
+so if an unknown request is seen, reply with `UNSUPPORTED-REQUEST`.
+
+## special remote replies
+
+These should be sent only in response to the git-annex request messages.
+They do not have to be sent immediately after the request; the special
+remote can send its own messages (listed in the next section below)
+while it's handling a request.
+
+* `PREPARE-SUCCESS`  
+  Sent as a response to PREPARE once the special remote is ready for use.
+* `PREPARE-FAILURE ErrorMsg`
+  Sent as a response to PREPARE if the special remote cannot be used.
+* `TRANSFER-SUCCESS STORE|RETRIEVE Key`  
+  Indicates the transfer completed successfully.
+* `TRANSFER-FAILURE STORE|RETRIEVE Key ErrorMsg`  
+  Indicates the transfer failed.
+* `CHECKPRESENT-SUCCESS Key`  
+  Indicates that a key has been positively verified to be present in the
+  remote.
+* `CHECKPRESENT-FAILURE Key`  
+  Indicates that a key has been positively verified to not be present in the
+  remote.
+* `CHECKPRESENT-UNKNOWN Key ErrorMsg`  
+  Indicates that it is not currently possible to verify if the key is
+  present in the remote. (Perhaps the remote cannot be contacted.)
+* `REMOVE-SUCCESS Key`  
+  Indicates the key has been removed from the remote. May be returned if
+  the remote didn't have the key at the point removal was requested.
+* `REMOVE-FAILURE Key ErrorMsg`  
+  Indicates that the key was unable to be removed from the remote.
+* `EXTENSIONS List`  
+  Sent in response to a EXTENSIONS request, the List could be used to indicate
+  protocol extensions that the special remote uses, but there are currently
+  no such extensions, so the List is empty.
+* `COST Int`  
+  Indicates the cost of the remote.
+* `AVAILABILITY GLOBAL|LOCAL`
+  Indicates if the remote is globally or only locally available.
+  (Ie stored in the cloud vs on a local disk.)
+* `INITREMOTE-SUCCESS`  
+  Indicates the INITREMOTE succeeded and the remote is ready to use.
+* `INITREMOTE-FAILURE ErrorMsg`  
+  Indicates that INITREMOTE failed.
+* `CLAIMURL-SUCCESS`  
+  Indicates that the CLAIMURL url will be handled by this remote.
+* `CLAIMURL-FAILURE`  
+  Indicates that the CLAIMURL url wil not be handled by this remote.
+* `CHECKURL-CONTENTS Size|UNKNOWN Filename`  
+  Indicates that the requested url has been verified to exist.  
+  The Size is the size in bytes, or use "UNKNOWN" if the size could not be
+  determined.  
+  The Filename can be empty (in which case a default is used),
+  or can specify a filename that is suggested to be used for this url.
+* `CHECKURL-MULTI Url1 Size1|UNKNOWN Filename1 Url2 Size2|UNKNOWN Filename2 ...`  
+  Indicates that the requested url has been verified to exist,
+  and contains multiple files, which can each be accessed using
+  their own url.  Each triplet of url, size, and filename should be listed,
+  one after the other.
+  Note that since a list is returned, neither the Url nor the Filename
+  can contain spaces.
+* `CHECKURL-FAILURE`  
+  Indicates that the requested url could not be accessed.
+* `WHEREIS-SUCCESS String`  
+  Indicates a location of a key. Typically an url, the string can
+  be anything that it makes sense to display to the user about content
+  stored in the special remote.
+* `WHEREIS-FAILURE`  
+  Indicates that no location is known for a key.
+* `INFOFIELD` / `INFOVALUE` / `INFOEND`  
+  Reply to a GETINFO request. This can be used to add info about anything,
+  but things like an url to the remote, or details of the remote's
+  configuration are typical. It should not include any sensitive
+  information like passwords, since it will be displayed to the user's
+  screen.
+  
+  There can be zero or more `INFOFIELD` messages, each containing the name of 
+  a field, and each is immediately followed by an `INFOVALUE` message
+  containing its value. The sequence is concluded by `INFOEND`. For example:
+	
+	INFOFIELD repository location
+	INFOVALUE http://example.com/repo/
+	INFOFIELD datacenter
+	INFOVALUE Antarctica
+	INFOEND
+	
+* `EXPORTSUPPORTED-SUCCESS`  
+  Indicates that it makes sense to use this special remote as an export.
+* `EXPORTSUPPORTED-FAILURE`  
+  Indicates that it does not make sense to use this special remote as an
+  export.
+* `RENAMEEXPORT-SUCCESS Key`  
+  Indicates that a `RENAMEEXPORT` was done successfully.
+* `RENAMEEXPORT-FAILURE Key`  
+  Indicates that a `RENAMEEXPORT` failed for whatever reason.
+* `REMOVEEXPORTDIRECTORY-SUCCESS`  
+  Indicates that a `REMOVEEXPORTDIRECTORY` was done successfully.
+* `REMOVEEXPORTDIRECTORY-FAILURE`  
+  Indicates that a `REMOVEEXPORTDIRECTORY` failed for whatever reason.
+* `UNSUPPORTED-REQUEST`  
+  Indicates that the special remote does not know how to handle a request.
+
+## special remote messages
+
+These messages may be sent by the special remote at any time that it's
+handling a request.
+
+* `VERSION Int`  
+  Supported protocol version. Current version is 1. Must be sent first
+  thing at startup, as until it sees this git-annex does not know how to
+  talk with the special remote program!  
+  (git-annex does not send a reply to this message, but may give up if it
+  doesn't support the necessary protocol version.)
+* `PROGRESS Int`  
+  Indicates the current progress of the transfer (in bytes). May be repeated
+  any number of times during the transfer process, but it's wasteful to
+  update the progress until at least another 1% of the file has been sent.
+  This is highly recommended for STORE. (It is optional but good for RETRIEVE.)  
+  (git-annex does not send a reply to this message.)
+* `DIRHASH Key`  
+  Gets a two level hash associated with a Key. Something like "aB/Cd".
+  This is always the same for any given Key, so can be used for eg,
+  creating hash directory structures to store Keys in. This is the same
+  directory hash that git-annex uses inside `.git/annex/objects/`  
+  (git-annex replies with VALUE followed by the value.)
+* `DIRHASH-LOWER Key`  
+  Gets a two level hash associated with a Key, using only lower-case.
+  Something like "abc/def".
+  This is always the same for any given Key, so can be used for eg,
+  creating hash directory structures to store Keys in. This is the same
+  directory hash that is used by eg, the directory special remote.  
+  (git-annex replies with VALUE followed by the value.)  
+  (First supported by git-annex version 6.20160511.)
+* `SETCONFIG Setting Value`  
+  Sets one of the special remote's configuration settings.  
+  Normally this is sent during INITREMOTE, which allows these settings
+  to be stored in the git-annex branch, so will be available if the same
+  special remote is used elsewhere. (If sent after INITREMOTE, the changed
+  configuration will only be available while the remote is running.)  
+  (git-annex does not send a reply to this message.)
+* `GETCONFIG Setting`  
+  Gets one of the special remote's configuration settings, which can have
+  been passed by the user when running `git annex initremote`, or
+  can have been set by a previous SETCONFIG. Can be run at any time.  
+  (git-annex replies with VALUE followed by the value. If the setting is
+  not set, the value will be empty.)
+* `SETCREDS Setting User Password`  
+  When some form of user and password is needed to access a special remote,
+  this can be used to securely store them for later use.
+  (Like SETCONFIG, this is normally sent only during INITREMOTE.)  
+  The Setting indicates which value in a remote's configuration can be
+  used to store the creds.  
+  Note that creds are normally only stored in the remote's configuration
+  when it's surely safe to do so; when gpg encryption is used, in which
+  case the creds will be encrypted using it. If creds are not stored in
+  the configuration, they'll only be stored in a local file.  
+  (embedcreds can be set to yes by the user or by SETCONFIG to force
+   the creds to be stored in the remote's configuration).  
+  (git-annex does not send a reply to this message.)
+* `GETCREDS Setting`  
+  Gets any creds that were previously stored in the remote's configuration
+  or a file.
+  (git-annex replies with "CREDS User Password". If no creds are found,
+  User and Password are both empty.)
+* `GETUUID`  
+  Queries for the UUID of the special remote being used.  
+  (git-annex replies with VALUE followed by the UUID.)
+* `GETGITDIR`  
+  Queries for the path to the git directory of the repository that
+  is using the external special remote.
+  (git-annex replies with VALUE followed by the path.)
+* `SETWANTED PreferredContentExpression`  
+  Can be used to set the preferred content of a repository. Normally
+  this is not configured by a special remote, but it may make sense
+  in some situations to hint at the kind of content that should be stored
+  in the special remote. Note that if a unparsable expression is set,
+  git-annex will ignore it.  
+  (git-annex does not send a reply to this message.)
+* `GETWANTED`  
+  Gets the current preferred content setting of the repository.
+  (git-annex replies with VALUE followed by the preferred content
+  expression.)
+* `SETSTATE Key Value`  
+  Can be used to store some form of state for a Key. The state stored
+  can be anything this remote needs to store, in any format.
+  It is stored in the git-annex branch. Note that this means that if
+  multiple repositories are using the same special remote, and store
+  different state, whichever one stored the state last will win. Also,
+  it's best to avoid storing much state, since this will bloat the
+  git-annex branch. Most remotes will not need to store any state.  
+  (git-annex does not send a reply to this message.)
+* `GETSTATE Key`  
+  Gets any state that has been stored for the key.  
+  (git-annex replies with VALUE followed by the state.)
+* `SETURLPRESENT Key Url`  
+  Records an URL where the Key can be downloaded from.  
+  Note that this does not make git-annex think that the url is present on
+  the web special remote.  
+  Keep in mind that this stores the url in the git-annex branch. This can
+  result in bloat to the branch if the url is large and/or does not delta
+  pack well with other information (such as the names of keys) already
+  stored in the branch.  
+  (git-annex does not send a reply to this message.)
+* `SETURLMISSING Key Url`  
+  Records that the key can no longer be downloaded from the specified
+  URL.  
+  (git-annex does not send a reply to this message.)
+* `SETURIPRESENT Key Uri`  
+  Records an URI where the Key can be downloaded from.  
+  For example, "ipfs:ADDRESS" is used for the ipfs special remote;
+  its CLAIMURL handler checks for such URIS and claims them.  
+  (git-annex does not send a reply to this message.)
+* `SETURIMISSING Key Uri`  
+  Records that the key can no longer be downloaded from the specified
+  URI.  
+  (git-annex does not send a reply to this message.)
+* `GETURLS Key Prefix`  
+  Gets the recorded urls where a Key can be downloaded from.
+  Only urls that start with the Prefix will be returned. The Prefix
+  may be empty to get all urls.
+  (git-annex replies one or more times with VALUE for each url.
+  The final VALUE has an empty value, indicating the end of the url list.)
+* `DEBUG message`
+  Tells git-annex to display the message if --debug is enabled.  
+  (git-annex does not send a reply to this message.)
+
+These messages are protocol extensions; it's only safe to send them to
+git-annex after it sent a EXTENSIONS that included the name of the message.
+
+* `INFO message`
+  Tells git-annex to display the message to the user.  
+  When git-annex is in --json mode, the message will be emitted immediately
+  in its own json object, with an "info" field.  
+  (git-annex does not send a reply to this message.)  
+
+## general messages
+
+These messages can be sent at any time by either git-annex or the special
+remote.
+
+* `ERROR ErrorMsg`  
+  Generic error. Can be sent at any time if things get too messed up
+  to continue. When possible, use a more specific reply from the list above.  
+  The special remote program should exit after sending this, as
+  git-annex will not talk to it any further. If the program receives
+  an ERROR from git-annex, it can exit with its own ERROR.
+
+## long running network connections
+
+Since an external special remote is started only when git-annex needs to
+access the remote, and then left running, it's ok to open a network
+connection in the PREPARE stage, and continue to use that network
+connection as requests are made.
+
+If you're unable to open a network connection, or the connection closes,
+perhaps because the network is down, it's ok to fail to perform any
+requests. Or you can try to reconnect when a new request is made.
+
+Note that the external special remote program may be left running for
+quite a long time, especially when the git-annex assistant is using it.
+The assistant will detect when the system connects to a network, and will
+start a new process the next time it needs to use a remote.
+
+## readonly mode
+
+Some storage services allow downloading the content of a file using a
+regular http connection, with no authentication. An external special remote
+for such a storage service can support a readonly mode of operation.
+
+It works like this:
+
+* When a key's content is stored on the remote, use SETURLPRESENT to
+  tell git-annex the public url from which it can be downloaded.
+* When a key's content is removed from the remote, use SETURLMISSING.
+* Document that this external special remote can be used in readonly mode.
+
+  The user doesn't even need to install your external special remote
+  program to use such a remote! All they need to do is run:
+  `git annex enableremote $remotename readonly=true`
+
+* The readonly=true parameter makes git-annex download content from the
+  urls recorded earlier by SETURLPRESENT.
+
+## TODO
+
+* When storing encrypted files stream the file up/down the pipe, rather
+  than using a temp file. Will probably involve less space and disk IO, 
+  and makes the progress display better, since the encryption can happen
+  concurrently with the transfer. Also, no need to use PROGRESS in this
+  scenario, since git-annex can see how much data it has sent/received from
+  the remote. However, \n and probably \0 need to be escaped somehow in the
+  file data, which adds complication.
+* uuid discovery during INITREMOTE.
+* Hook into webapp. Needs a way to provide some kind of prompt to the user
+  in the webapp, etc.
diff --git a/doc/design/external_special_remote_protocol/comment_11_8d3c35eb0a2a9c57b10566fcaf56d248._comment b/doc/design/external_special_remote_protocol/comment_11_8d3c35eb0a2a9c57b10566fcaf56d248._comment
new file mode 100644
index 0000000000..58c337ccd7
--- /dev/null
+++ b/doc/design/external_special_remote_protocol/comment_11_8d3c35eb0a2a9c57b10566fcaf56d248._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="http://joeyh.name/"
+ ip="209.250.56.43"
+ subject="comment 11"
+ date="2014-01-13T18:45:34Z"
+ content="""
+I've handled the cloud repo check by making external special remotes be assumed by default to be globally available via the cloud. So no need to do anything in most cases. For remotes that are only available locally, the remote can reply with \"AVAILABILITY LOCAL\" when git-annex sends an AVAILABILITY request.
+"""]]
diff --git a/doc/design/external_special_remote_protocol/comment_12_241e57092d9e5631ac0ec4dd962477a6._comment b/doc/design/external_special_remote_protocol/comment_12_241e57092d9e5631ac0ec4dd962477a6._comment
new file mode 100644
index 0000000000..b5b3d1bfc2
--- /dev/null
+++ b/doc/design/external_special_remote_protocol/comment_12_241e57092d9e5631ac0ec4dd962477a6._comment
@@ -0,0 +1,12 @@
+[[!comment format=mdwn
+ username="https://www.google.com/accounts/o8/id?id=AItOawmkBwMWvNKZZCge_YqobCSILPMeK6xbFw8"
+ nickname="TobiasTheViking"
+ subject="Feature request"
+ date="2014-01-11T15:41:48Z"
+ content="""
+The ability to mark a remote as being a \"cloud\" remote. To silence the \"Unable to download files from your other devices. Add a cloud repository\" message in the webapp. 
+
+Maybe as simple as \"SETCONFIG cloud true\", if that is a viable implementation.
+
+
+"""]]
diff --git a/doc/design/external_special_remote_protocol/comment_12_e3029c65d34f78272bc11961ebfd8237._comment b/doc/design/external_special_remote_protocol/comment_12_e3029c65d34f78272bc11961ebfd8237._comment
new file mode 100644
index 0000000000..e8d0dcfe8e
--- /dev/null
+++ b/doc/design/external_special_remote_protocol/comment_12_e3029c65d34f78272bc11961ebfd8237._comment
@@ -0,0 +1,10 @@
+[[!comment format=mdwn
+ username="https://www.google.com/accounts/o8/id?id=AItOawm_YXzEdPHzbSGVwtmTR7g1BqDtTnIBB5s"
+ nickname="Matthias"
+ subject="Chunk it"
+ date="2014-01-20T16:22:09Z"
+ content="""
+> TODO: stream the file up/down the pipe, rather than using a temp file
+
+You might want to use chunked transfer, i.e. a series of \"EXPECT 65536\" followed by that many bytes of binary data and an EOF marker (EXPECT-END or EXPECT 0), instead of escaping three characters (newline, NUL, and the escape prefix) and the additional unnecessary tedious per-character processing that would require.
+"""]]
diff --git a/doc/design/external_special_remote_protocol/comment_13_472748f03ba8dad773da7f35b70cb6e4._comment b/doc/design/external_special_remote_protocol/comment_13_472748f03ba8dad773da7f35b70cb6e4._comment
new file mode 100644
index 0000000000..f222349c13
--- /dev/null
+++ b/doc/design/external_special_remote_protocol/comment_13_472748f03ba8dad773da7f35b70cb6e4._comment
@@ -0,0 +1,15 @@
+[[!comment format=mdwn
+ username="http://schnouki.net/"
+ nickname="Schnouki"
+ subject="comment 13"
+ date="2014-02-10T18:22:55Z"
+ content="""
+First things first: in the documentation, I think `SETCONFIG Setting` should be `SETCONFIG Setting Value`.
+
+Now a few questions:
+
+- why have `SETCREDS` and `GETCREDS` have both a username and a password? I'd like to use them to store a OAuth token, but because of this I also have to store a dummy value, which seems weird to me. Is it possible to just do `SETCREDS oauth_token XXXYYY123456`?
+- about `PROGRESS`: my remote is sending `PROGRESS xxx` every 64kb uploaded or downloaded, but no upload/download progress is displayed by git-annex. Is this normal? Should I do it myself, or will it be done by a future version of git-annex?
+
+Joey, thanks a lot for all your work on git-annex :)
+"""]]
diff --git a/doc/design/external_special_remote_protocol/comment_14_71c3e21a72222250bab933e1c9167fbc._comment b/doc/design/external_special_remote_protocol/comment_14_71c3e21a72222250bab933e1c9167fbc._comment
new file mode 100644
index 0000000000..095f225373
--- /dev/null
+++ b/doc/design/external_special_remote_protocol/comment_14_71c3e21a72222250bab933e1c9167fbc._comment
@@ -0,0 +1,14 @@
+[[!comment format=mdwn
+ username="http://joeyh.name/"
+ ip="209.250.56.172"
+ subject="comment 14"
+ date="2014-02-11T01:36:51Z"
+ content="""
+Schnouki, fixed SETCONFIG docs. Note that this is a wiki. :)
+
+I agree it's a little weird for SETCREDS to have a username and a password. This is just exposing git-annex's existing credential storage which has a tuple of values rather than using, say, a multivalue map. If you only need one it's fine to put in a dummy value for the other one.
+
+Re PROGRESS, it seems I hooked up the progress stuff, so it was visible in the webapp, but forgot to put up a progress display at the command line. Fixed in git.
+
+I look forward to seeing your special remote implementation!
+"""]]
diff --git a/doc/design/external_special_remote_protocol/comment_15_c77386deddc64b2028d9c3a7393d4df7._comment b/doc/design/external_special_remote_protocol/comment_15_c77386deddc64b2028d9c3a7393d4df7._comment
new file mode 100644
index 0000000000..d8ecdc3983
--- /dev/null
+++ b/doc/design/external_special_remote_protocol/comment_15_c77386deddc64b2028d9c3a7393d4df7._comment
@@ -0,0 +1,10 @@
+[[!comment format=mdwn
+ username="http://schnouki.net/"
+ nickname="Schnouki"
+ subject="comment 15"
+ date="2014-02-11T13:44:10Z"
+ content="""
+Joey, thanks for fixing that so quickly. Indeed it works in the webapp; I'll check the CLI version as soon as possible :)
+
+I just released the new remote on . It's for [hubiC](https://hubic.com/), a French personal cloud storage service made by OVH that has free 25GB accounts and only charges €10/month for 10TB (VAT included). It's really experimental (hacked it over the weekend), but it seems to work for me so far.
+"""]]
diff --git a/doc/design/external_special_remote_protocol/comment_16_62b137a138c143a8110886cc0bbb677e._comment b/doc/design/external_special_remote_protocol/comment_16_62b137a138c143a8110886cc0bbb677e._comment
new file mode 100644
index 0000000000..4d196a2df9
--- /dev/null
+++ b/doc/design/external_special_remote_protocol/comment_16_62b137a138c143a8110886cc0bbb677e._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="sjvdwalt@3d195104d8f45061da99fe7f0a97d69dfc49bb5d"
+ nickname="sjvdwalt"
+ subject="Stream encoding"
+ date="2015-08-25T00:36:24Z"
+ content="""
+What encoding is used for the stdin/stdout streams used to communicate with remotes?
+"""]]
diff --git a/doc/design/external_special_remote_protocol/comment_17_8dc7bbf485c9385a4b506e8b8fa934fe._comment b/doc/design/external_special_remote_protocol/comment_17_8dc7bbf485c9385a4b506e8b8fa934fe._comment
new file mode 100644
index 0000000000..55e8c12447
--- /dev/null
+++ b/doc/design/external_special_remote_protocol/comment_17_8dc7bbf485c9385a4b506e8b8fa934fe._comment
@@ -0,0 +1,23 @@
+[[!comment format=mdwn
+ username="https://me.yahoo.com/a/EbvxpTI_xP9Aod7Mg4cwGhgjrCrdM5s-#7c0f4"
+ subject="WHEREIS -- is it better to just report failure to avoid duplicates?"
+ date="2015-08-26T14:22:49Z"
+ content="""
+I wonder how should I utilize this new API (WHEREIS) in my case:  it seems just to lead to duplication of whereis information in my case of a special remote to support extracting of content from archives. If I make it to reply with the same url (which is not \"public\" per se, i.e. can't be used by annex directly) I just get it duplicated:
+
+    $> git annex whereis simple.txt
+    whereis simple.txt (1 copy) 
+      	82025765-5cac-4571-91ed-637620ec6fc7 -- [annexed-archives]
+    
+      annexed-archives: dl+archive:SHA256E-s173--5df2eeab61ea7d6479533d4e6b07c6bcfae46e040cad8cb1fc579f9f18c90790.tar.gz/a/d/%20%22%27%3Ba%26b%26cd%20%60%7C%20
+      annexed-archives: dl+archive:SHA256E-s173--5df2eeab61ea7d6479533d4e6b07c6bcfae46e040cad8cb1fc579f9f18c90790.tar.gz/a/d/%20%22%27%3Ba%26b%26cd%20%60%7C%20
+    ok
+
+if I \"explain\" it a bit, also somewhat duplicate:
+
+    annexed-archives: file a/d/%20%22%27%3Ba%26b%26cd%20%60%7C%20 within archive SHA256E-s173--5df2eeab61ea7d6479533d4e6b07c6bcfae46e040cad8cb1fc579f9f18c90790.tar.gz
+    annexed-archives: dl+archive:SHA256E-s173--5df2eeab61ea7d6479533d4e6b07c6bcfae46e040cad8cb1fc579f9f18c90790.tar.gz/a/d/%20%22%27%3Ba%26b%26cd%20%60%7C%20
+
+But if I just reply with \"WHEREIS-FAILURE\" it becomes more sensible (no duplicates), but I feel that then better documentation for this feature get adjusted to describe
+that it is only to complement information already known to annex, and not really to \"provide any information about ways to access the content of a key stored in it\".  Or have I missed the point? ;)
+"""]]
diff --git a/doc/design/external_special_remote_protocol/comment_18_a0ed457b1b0d71747b6cf0c45897b5e1._comment b/doc/design/external_special_remote_protocol/comment_18_a0ed457b1b0d71747b6cf0c45897b5e1._comment
new file mode 100644
index 0000000000..f56fa64103
--- /dev/null
+++ b/doc/design/external_special_remote_protocol/comment_18_a0ed457b1b0d71747b6cf0c45897b5e1._comment
@@ -0,0 +1,19 @@
+[[!comment format=mdwn
+ username="joey"
+ subject="""re: Stream encoding"""
+ date="2015-09-09T20:44:38Z"
+ content="""
+@sjvdwalt, git-annex does not specify or expect any character encoding to be used
+for this protocol. A robust external special remote shouldn't assume any
+particular character encoding, either.
+
+Lines will be terminated with '\n' (0xA), and words in lines are
+delimited by an ascii space (0x20). The keywords in the protocol are ascii
+too of course. Any values can contain an arbitrary sequence of bytes that may
+or may not be able to be decoded using the current character encoding.
+
+IIRC, character encodings like UTF8 that encode a character to multiple
+bytes avoid ever using 0x0 to 0xFF when doing so. So, every ascii space and
+newline are unambiguously such, and it's safe to split on them even though
+no encoding is specified.
+"""]]
diff --git a/doc/design/external_special_remote_protocol/comment_19_89d532a5013af24f15a0d003cbfbab25._comment b/doc/design/external_special_remote_protocol/comment_19_89d532a5013af24f15a0d003cbfbab25._comment
new file mode 100644
index 0000000000..1a4c180aaf
--- /dev/null
+++ b/doc/design/external_special_remote_protocol/comment_19_89d532a5013af24f15a0d003cbfbab25._comment
@@ -0,0 +1,13 @@
+[[!comment format=mdwn
+ username="joey"
+ subject="""re: WHEREIS -- is it better to just report failure to avoid duplicates?"""
+ date="2015-09-09T21:03:13Z"
+ content="""
+There's no point in implementing WHEREIS if it's going to reply with the
+same values that are passed to SETURIPRESENT.
+
+Some special remotes may not need to use SETURIPRESENT to work at
+all, and yet storing data on the remote makes it available from some public
+url. This is the kind of situation where it makes sense to implement
+WHEREIS.
+"""]]
diff --git a/doc/design/external_special_remote_protocol/comment_1_5baff75d278394a8818c348fb4f13b8a._comment b/doc/design/external_special_remote_protocol/comment_1_5baff75d278394a8818c348fb4f13b8a._comment
new file mode 100644
index 0000000000..aa2e85d900
--- /dev/null
+++ b/doc/design/external_special_remote_protocol/comment_1_5baff75d278394a8818c348fb4f13b8a._comment
@@ -0,0 +1,10 @@
+[[!comment format=mdwn
+ username="https://www.google.com/accounts/o8/id?id=AItOawmicVKRM8vJX4wPuAwlLEoS2cjmFXQkjkE"
+ nickname="Thomas"
+ subject="not useful for "plain directory" special remote?"
+ date="2013-12-16T20:10:16Z"
+ content="""
+It says: \"Note that the File should not influence the filename used on the remote. The filename used should be derived from the Key.\"
+
+Thus this interface might not be useful to implement [[todo/New special remote suggeston - clean directory]]? The clean directory special remote would just do that: save $key content on the remote under the filename $file.
+"""]]
diff --git a/doc/design/external_special_remote_protocol/comment_20_546331742e906a68d760c1bf44be63c4._comment b/doc/design/external_special_remote_protocol/comment_20_546331742e906a68d760c1bf44be63c4._comment
new file mode 100644
index 0000000000..7be874d260
--- /dev/null
+++ b/doc/design/external_special_remote_protocol/comment_20_546331742e906a68d760c1bf44be63c4._comment
@@ -0,0 +1,13 @@
+[[!comment format=mdwn
+ username="https://me.yahoo.com/a/EbvxpTI_xP9Aod7Mg4cwGhgjrCrdM5s-#7c0f4"
+ subject="but which of the 3?"
+ date="2015-09-10T03:16:17Z"
+ content="""
+could you express your expert choice explicitly among those 3 choices how I should react to WHEREIS in my (archives) case
+
+1. report the same url
+2. \"spell it out\"
+3. WHEREIS-FAILURE
+
+or really not implement it at all? (we are still at VERSION 1, so I thought that not implementing it might lead to some undesired side-effects)
+"""]]
diff --git a/doc/design/external_special_remote_protocol/comment_21_a8e8255516dc4741439615fa43b71829._comment b/doc/design/external_special_remote_protocol/comment_21_a8e8255516dc4741439615fa43b71829._comment
new file mode 100644
index 0000000000..f9183fe28a
--- /dev/null
+++ b/doc/design/external_special_remote_protocol/comment_21_a8e8255516dc4741439615fa43b71829._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="joey"
+ subject="""comment 21"""
+ date="2015-09-10T16:33:10Z"
+ content="""
+As the documentation says, it's fine to not implement WHEREIS if you don't
+need it.
+"""]]
diff --git a/doc/design/external_special_remote_protocol/comment_22_0387827d94f141be53458807da9c2fc2._comment b/doc/design/external_special_remote_protocol/comment_22_0387827d94f141be53458807da9c2fc2._comment
new file mode 100644
index 0000000000..50add12291
--- /dev/null
+++ b/doc/design/external_special_remote_protocol/comment_22_0387827d94f141be53458807da9c2fc2._comment
@@ -0,0 +1,9 @@
+[[!comment format=mdwn
+ username="szrc"
+ subject="Local storage of creds"
+ date="2016-04-04T03:59:01Z"
+ content="""
+I have a question about local storage of credentials. I assumed that when creds were stored in the repo (because the remote is encrypted or because embedcreds=yes), they wouldn't be stored locally in .git/annex/creds. But it seems they are stored locally, in plaintext, regardless.
+
+Is there a way to prevent this? Ideally, credentials should not be stored plaintext at all...but maybe there's a technical issue I'm not seeing.
+"""]]
diff --git a/doc/design/external_special_remote_protocol/comment_23_7a5af50f7c482d7c02c53be28ddb6a66._comment b/doc/design/external_special_remote_protocol/comment_23_7a5af50f7c482d7c02c53be28ddb6a66._comment
new file mode 100644
index 0000000000..ba0c1ce402
--- /dev/null
+++ b/doc/design/external_special_remote_protocol/comment_23_7a5af50f7c482d7c02c53be28ddb6a66._comment
@@ -0,0 +1,13 @@
+[[!comment format=mdwn
+ username="joey"
+ subject="""comment 23"""
+ date="2016-04-04T20:17:05Z"
+ content="""
+@szrc, it's pretty expensive to pull encrypted creds out of the git 
+repository and run gpg to decrypt them. Doing so also tends to result
+in a gpg password prompt. 
+
+Rather than do that every time git-annex needs the creds to access the
+remote, it maintains a local cache file, which has its permissions set so
+only the local user (and root, naturally) can read it.
+"""]]
diff --git a/doc/design/external_special_remote_protocol/comment_24_3340d3c34f2ec197f9010d97da8eb7af._comment b/doc/design/external_special_remote_protocol/comment_24_3340d3c34f2ec197f9010d97da8eb7af._comment
new file mode 100644
index 0000000000..b10bf18222
--- /dev/null
+++ b/doc/design/external_special_remote_protocol/comment_24_3340d3c34f2ec197f9010d97da8eb7af._comment
@@ -0,0 +1,11 @@
+[[!comment format=mdwn
+ username="szrc"
+ subject="comment 24"
+ date="2016-04-05T01:18:53Z"
+ content="""
+Thanks for the response, Joey. It seems to me that many/most operations for which credentials are needed will require a gpg prompt anyway, but I can see why it might be too expensive in some cases.
+
+Anyway, if you ever saw fit to add the option to disable or limit local caching, I would definitely use it -- and I'm guessing I'm not the only one who would prefer not to store credentials in plaintext.
+
+Thanks for all of your great work on git-annex.
+"""]]
diff --git a/doc/design/external_special_remote_protocol/comment_25_56f38d934125c28172b6475bdb4de284._comment b/doc/design/external_special_remote_protocol/comment_25_56f38d934125c28172b6475bdb4de284._comment
new file mode 100644
index 0000000000..132c95f5b3
--- /dev/null
+++ b/doc/design/external_special_remote_protocol/comment_25_56f38d934125c28172b6475bdb4de284._comment
@@ -0,0 +1,10 @@
+[[!comment format=mdwn
+ username="grawity@2ea26be48562f66fcb9b66307da72b1e2e37453f"
+ nickname="grawity"
+ subject="comment 25"
+ date="2016-05-03T06:25:02Z"
+ content="""
+Is there a reason that DIRHASH in type=external uses a different format (mixed-case) from that used by type=directory (lower-case)? Or, could there be e.g. `DIRHASH LOWER` to select between the formats?
+
+I'm writing an external remote for SMB filesystem access and I'd like its storage to be usable via type=directory in case of emergencies or other reasons, but  with different dirhash layouts that's not going to work, I assume...
+"""]]
diff --git a/doc/design/external_special_remote_protocol/comment_26_4f8addd76a1e64d05a34910d4719a40c._comment b/doc/design/external_special_remote_protocol/comment_26_4f8addd76a1e64d05a34910d4719a40c._comment
new file mode 100644
index 0000000000..790da04451
--- /dev/null
+++ b/doc/design/external_special_remote_protocol/comment_26_4f8addd76a1e64d05a34910d4719a40c._comment
@@ -0,0 +1,11 @@
+[[!comment format=mdwn
+ username="joey"
+ subject="""comment 26"""
+ date="2016-05-03T17:29:02Z"
+ content="""
+I don't think there's any particularly good reason why DIRHASH uses the
+mixed case format. However, it can't be changed without busting existing
+stuff.
+
+So yeah, I've gone ahead and added a `DIRHASH_LOWER`.
+"""]]
diff --git a/doc/design/external_special_remote_protocol/comment_27_9153a5c49710bf57d11338829f66fa78._comment b/doc/design/external_special_remote_protocol/comment_27_9153a5c49710bf57d11338829f66fa78._comment
new file mode 100644
index 0000000000..5061c0a883
--- /dev/null
+++ b/doc/design/external_special_remote_protocol/comment_27_9153a5c49710bf57d11338829f66fa78._comment
@@ -0,0 +1,10 @@
+[[!comment format=mdwn
+ username="grawity@2ea26be48562f66fcb9b66307da72b1e2e37453f"
+ nickname="grawity"
+ subject="comment 27"
+ date="2016-05-03T18:02:03Z"
+ content="""
+Thanks. It'll probably be safer for my use case of storing data on Windows network shares than the mixed-case version.
+
+_(Speaking of \"too late to change\", `DIRHASH-LOWER` with a dash might be more consistent with the existing responses?)_
+"""]]
diff --git a/doc/design/external_special_remote_protocol/comment_28_74961b12a73a52a700e10a21674e66ee._comment b/doc/design/external_special_remote_protocol/comment_28_74961b12a73a52a700e10a21674e66ee._comment
new file mode 100644
index 0000000000..faaf26a2d4
--- /dev/null
+++ b/doc/design/external_special_remote_protocol/comment_28_74961b12a73a52a700e10a21674e66ee._comment
@@ -0,0 +1,7 @@
+[[!comment format=mdwn
+ username="joey"
+ subject="""comment 28"""
+ date="2016-05-03T18:09:50Z"
+ content="""
+Agreed, DIRHASH-LOWER is more consistent, changed it to that.
+"""]]
diff --git a/doc/design/external_special_remote_protocol/comment_29_c1d97815386453c3e433fca9a44c4667._comment b/doc/design/external_special_remote_protocol/comment_29_c1d97815386453c3e433fca9a44c4667._comment
new file mode 100644
index 0000000000..b714602e39
--- /dev/null
+++ b/doc/design/external_special_remote_protocol/comment_29_c1d97815386453c3e433fca9a44c4667._comment
@@ -0,0 +1,9 @@
+[[!comment format=mdwn
+ username="xloem"
+ subject="Retrieval progress message helpers"
+ date="2016-05-20T19:42:23Z"
+ content="""
+It would be nice if
+- progress could be provided in percentage rather than bytes, when that's all that's available
+- git-annex could inform the special remote how many bytes a file to be downloaded is, if that information is already available
+"""]]
diff --git a/doc/design/external_special_remote_protocol/comment_2_70429b7c4f1e4083a9d5a6e2e238056d._comment b/doc/design/external_special_remote_protocol/comment_2_70429b7c4f1e4083a9d5a6e2e238056d._comment
new file mode 100644
index 0000000000..032136b022
--- /dev/null
+++ b/doc/design/external_special_remote_protocol/comment_2_70429b7c4f1e4083a9d5a6e2e238056d._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="http://joeyh.name/"
+ ip="209.250.56.87"
+ subject="comment 2"
+ date="2013-12-16T20:42:23Z"
+ content="""
+That's not how special remotes work -- they have nothing to do with the symlinks in the work tree, which are managed by git (or git-annex in direct mode). They only store values for keys.
+"""]]
diff --git a/doc/design/external_special_remote_protocol/comment_30_7d045871f9ebbb89e3f9bbfa28e8468f._comment b/doc/design/external_special_remote_protocol/comment_30_7d045871f9ebbb89e3f9bbfa28e8468f._comment
new file mode 100644
index 0000000000..4330e7f971
--- /dev/null
+++ b/doc/design/external_special_remote_protocol/comment_30_7d045871f9ebbb89e3f9bbfa28e8468f._comment
@@ -0,0 +1,18 @@
+[[!comment format=mdwn
+ username="joey"
+ subject="""comment 30"""
+ date="2016-05-23T18:54:35Z"
+ content="""
+You can find out the size of a key by using 
+`git-annex examinekey $key --format='${bytesize}\n'` 
+(There's a --batch option to avoid needing to spin up
+repeated such processes.)
+
+I suppose I could add a PROGRESSPRECENT, but any version of git-annex that
+didn't support it would fail with a protocol error if a special remote
+tried to use that. So, the special remote would need to check git-annex
+version to use it.
+
+So, maybe better to get the size of the key yourself, and convert the
+percentage to bytes for PROGRESS.
+"""]]
diff --git a/doc/design/external_special_remote_protocol/comment_31_907597187020180e7730e27e86f6da5e._comment b/doc/design/external_special_remote_protocol/comment_31_907597187020180e7730e27e86f6da5e._comment
new file mode 100644
index 0000000000..d1d4d753c9
--- /dev/null
+++ b/doc/design/external_special_remote_protocol/comment_31_907597187020180e7730e27e86f6da5e._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="Ilya_Shlyakhter"
+ avatar="http://cdn.libravatar.org/avatar/1647044369aa7747829c38b9dcc84df0"
+ subject="comment 31"
+ date="2018-09-19T11:01:31Z"
+ content="""
+What exacly is the difference between SETURIPRESENT and SETURLPRESENT?
+"""]]
diff --git a/doc/design/external_special_remote_protocol/comment_32_f6b8306cfdcb3792728aa47af0677338._comment b/doc/design/external_special_remote_protocol/comment_32_f6b8306cfdcb3792728aa47af0677338._comment
new file mode 100644
index 0000000000..6ccf84027b
--- /dev/null
+++ b/doc/design/external_special_remote_protocol/comment_32_f6b8306cfdcb3792728aa47af0677338._comment
@@ -0,0 +1,11 @@
+[[!comment format=mdwn
+ username="Ilya_Shlyakhter"
+ avatar="http://cdn.libravatar.org/avatar/1647044369aa7747829c38b9dcc84df0"
+ subject="comment 32"
+ date="2018-09-19T18:19:22Z"
+ content="""
+Some questions about CHECKPRESENT Key: (1) if Key is a URL backend key, should this return true if CHECKURL on the URL would return CHECKURL-CONTENTS?
+(2) Should the external special remote implementation call GETURLS on the key and return true if CHECKURL would return CHECKURL-CONTENTS for any of the URLs?
+(3) Calling GETURLS on a URL key returns an empty list; shouldn't it return a one-element list containing the included URL (at least if a CHECKURL call on that URL 
+would return CHECKURL-CONTENTS)?
+"""]]
diff --git a/doc/design/external_special_remote_protocol/comment_33_bee7ff39f3989dfbec292490e47bcaaa._comment b/doc/design/external_special_remote_protocol/comment_33_bee7ff39f3989dfbec292490e47bcaaa._comment
new file mode 100644
index 0000000000..ac472845f5
--- /dev/null
+++ b/doc/design/external_special_remote_protocol/comment_33_bee7ff39f3989dfbec292490e47bcaaa._comment
@@ -0,0 +1,19 @@
+[[!comment format=mdwn
+ username="joey"
+ subject="""comment 33"""
+ date="2018-09-24T15:35:54Z"
+ content="""
+@Ilya_Shlyakhter,
+
+1. CHECKURL is only used by git-annex add/importfeed when adding a new url.
+   So it does not need to be consistent with CHECKPRESENT, though it would
+   probably make sense for it to be in most cases.
+2. I guess you're asking if it should do that in its CHECKPRESENT
+   implementation. CHECKPRESENT needs to use some method to actively verify
+   that the remote currently contains the content of the key. It doesn't
+   necessarily need to use a recorded url.
+3. GETURLS looks at information stored in the git-annex branch. If the url
+   key has been added to the repository with `git annex add` then its url
+   will be stored there, but if you just generated an url key, it doesn't
+   necessarily have anything stored about it in the git-annex branch.
+"""]]
diff --git a/doc/design/external_special_remote_protocol/comment_34_af6ae46fa9228461573dd7b2f0713ef1._comment b/doc/design/external_special_remote_protocol/comment_34_af6ae46fa9228461573dd7b2f0713ef1._comment
new file mode 100644
index 0000000000..68064e9b02
--- /dev/null
+++ b/doc/design/external_special_remote_protocol/comment_34_af6ae46fa9228461573dd7b2f0713ef1._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="Ilya_Shlyakhter"
+ avatar="http://cdn.libravatar.org/avatar/1647044369aa7747829c38b9dcc84df0"
+ subject="question about special remote protocol"
+ date="2018-09-26T17:16:13Z"
+ content="""
+What exacly is the difference between SETURIPRESENT and SETURLPRESENT?
+"""]]
diff --git a/doc/design/external_special_remote_protocol/comment_3_c763e44d06d9f50f0d357889b180b819._comment b/doc/design/external_special_remote_protocol/comment_3_c763e44d06d9f50f0d357889b180b819._comment
new file mode 100644
index 0000000000..08d405c8e2
--- /dev/null
+++ b/doc/design/external_special_remote_protocol/comment_3_c763e44d06d9f50f0d357889b180b819._comment
@@ -0,0 +1,15 @@
+[[!comment format=mdwn
+ username="https://www.google.com/accounts/o8/id?id=AItOawmkBwMWvNKZZCge_YqobCSILPMeK6xbFw8"
+ nickname="TobiasTheViking"
+ subject="Feature requests"
+ date="2013-12-28T13:57:35Z"
+ content="""
+PREPARE-Failure ErrorMsg (matching INITREMOTE-FAILURE ErrorMsg)
+
+Also, i'd like for the following to overwrite existing credentials/configs 
+MYFOLDER=\"testfolder\" MYLOGIN=\"login\" MYPASSWORD=\"pword\" MYURL=\"http://webdav/\"    git annex enableremote owncloud type=external externaltype=owncloud --debug
+
+This would also be needed for refreshing oauth.
+
+
+"""]]
diff --git a/doc/design/external_special_remote_protocol/comment_4_20ff41d82b5f1872698a5b24adcd0c41._comment b/doc/design/external_special_remote_protocol/comment_4_20ff41d82b5f1872698a5b24adcd0c41._comment
new file mode 100644
index 0000000000..6597294fba
--- /dev/null
+++ b/doc/design/external_special_remote_protocol/comment_4_20ff41d82b5f1872698a5b24adcd0c41._comment
@@ -0,0 +1,12 @@
+[[!comment format=mdwn
+ username="http://joeyh.name/"
+ ip="209.250.56.227"
+ subject="comment 4"
+ date="2013-12-29T17:50:23Z"
+ content="""
+Added PREPARE-FAILURE
+
+git-annex enableremote causes INITREMOTE to be called, so any credentials can be stored etc. (Note that, as with built-in special remotes, credentials are only stored in the git-annex branch when the remote is encrypted. Otherwise, they're stored locally in a .git/annex/creds/ file.)
+
+Also, I'd recommend using environment variables for passing credentials to initremote/enableremote, because that avoids leaking them in `ps`.. but it probably doesn't make sense to use environment variables for other settings, but instead pass them as parameters of initremote/enableremote, which can be looked up using GETCONFIG. Only exception might be if the setting needs to vary between different machines.
+"""]]
diff --git a/doc/design/external_special_remote_protocol/comment_5_3ee158e548002badae5bf44dc0442626._comment b/doc/design/external_special_remote_protocol/comment_5_3ee158e548002badae5bf44dc0442626._comment
new file mode 100644
index 0000000000..fcc70a1860
--- /dev/null
+++ b/doc/design/external_special_remote_protocol/comment_5_3ee158e548002badae5bf44dc0442626._comment
@@ -0,0 +1,13 @@
+[[!comment format=mdwn
+ username="https://www.google.com/accounts/o8/id?id=AItOawmkBwMWvNKZZCge_YqobCSILPMeK6xbFw8"
+ nickname="TobiasTheViking"
+ subject="Feature requests"
+ date="2013-12-31T14:05:07Z"
+ content="""
+Hook should be able to set default configuration for itself.
+
+For instance, clean flickr hook will only upload some files(notably pictures). The user shouldn't have to manage that.
+
+Other hooks have a maximum filesize(though i guess that doesn't matter once splitting works).
+
+"""]]
diff --git a/doc/design/external_special_remote_protocol/comment_6_ee2828ce25b83bbabc9d5dde35d1e57b._comment b/doc/design/external_special_remote_protocol/comment_6_ee2828ce25b83bbabc9d5dde35d1e57b._comment
new file mode 100644
index 0000000000..f57bfabbc4
--- /dev/null
+++ b/doc/design/external_special_remote_protocol/comment_6_ee2828ce25b83bbabc9d5dde35d1e57b._comment
@@ -0,0 +1,12 @@
+[[!comment format=mdwn
+ username="http://joeyh.name/"
+ ip="209.250.56.227"
+ subject="comment 6"
+ date="2013-12-31T17:56:34Z"
+ content="""
+I suppose you're talking about preferred content settings.
+
+I think that it makes sense for hooks to use existing git-annex plumbing etc when it's available. So a hook could just run `git annex wanted` to manage its preferred content.
+
+The only problem is that a hook does not currently have a way to discover the UUID of the repository! So I've added a GETUUID to cover this and other use cases.
+"""]]
diff --git a/doc/design/external_special_remote_protocol/comment_7_edb649d0019a061ef7bf6534a444429d._comment b/doc/design/external_special_remote_protocol/comment_7_edb649d0019a061ef7bf6534a444429d._comment
new file mode 100644
index 0000000000..4e9f91c42c
--- /dev/null
+++ b/doc/design/external_special_remote_protocol/comment_7_edb649d0019a061ef7bf6534a444429d._comment
@@ -0,0 +1,16 @@
+[[!comment format=mdwn
+ username="https://www.google.com/accounts/o8/id?id=AItOawmkBwMWvNKZZCge_YqobCSILPMeK6xbFw8"
+ nickname="TobiasTheViking"
+ subject="comment 7"
+ date="2013-12-31T18:20:32Z"
+ content="""
+I think the hook running anything in shell, to interact with git annex, is a mistake.
+
+I see a lot more potential pitfalls and mistakes(especially crossplatform). 
+
+It should be the existing git annex plumbing (preferred content) as you say. I just really think it should be configurable in the protocol, instead of a having to run a shell command. 
+
+Since you have made this advanced protocol i really see it as a mistake to do anything between the hook and git-annex outside of the protocol, it makes much more sense to have all their interactions happen within the protocol.
+
+IMO anyways.
+"""]]
diff --git a/doc/design/external_special_remote_protocol/comment_8_1f4c205a5ce6f33ccf2f4d80754e5699._comment b/doc/design/external_special_remote_protocol/comment_8_1f4c205a5ce6f33ccf2f4d80754e5699._comment
new file mode 100644
index 0000000000..f9803bee01
--- /dev/null
+++ b/doc/design/external_special_remote_protocol/comment_8_1f4c205a5ce6f33ccf2f4d80754e5699._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="http://joeyh.name/"
+ ip="209.250.56.227"
+ subject="comment 8"
+ date="2013-12-31T19:16:16Z"
+ content="""
+It makes sense to only implement one interface to things, unless there is a reason such as performance to do otherwise.
+"""]]
diff --git a/doc/design/external_special_remote_protocol/comment_9_15c4cfe064be37cc104dcb6aa049a449._comment b/doc/design/external_special_remote_protocol/comment_9_15c4cfe064be37cc104dcb6aa049a449._comment
new file mode 100644
index 0000000000..cc3f7dd235
--- /dev/null
+++ b/doc/design/external_special_remote_protocol/comment_9_15c4cfe064be37cc104dcb6aa049a449._comment
@@ -0,0 +1,14 @@
+[[!comment format=mdwn
+ username="http://joeyh.name/"
+ ip="209.250.56.227"
+ subject="comment 9"
+ date="2014-01-02T00:15:28Z"
+ content="""
+Tobias made some good points:
+
+* git-annex may not be in PATH depending on installation method
+* It would in theory be bad if a special remote ran some git-annex command that used the special remote and ran some git-annex command [...].
+* git-annex would need to tell the special remote what git repo it was being used with.
+
+So, added GETWANTED and SETWANTED. However, if I find myself recapitulating a lot of git-annex's command-line plumbing stuff in this protocol, I will need to revisit this decision and find a better way. Particularly, I narrowly escaped an intractable dependency loop in [[!commit 8e3032df2d5c6ddf07e43de4b3bb89cb578ae048]].
+"""]]
diff --git a/doc/design/gcrypt.mdwn b/doc/design/gcrypt.mdwn
new file mode 100644
index 0000000000..d5b9c064bc
--- /dev/null
+++ b/doc/design/gcrypt.mdwn
@@ -0,0 +1,8 @@
+To integrate with git-remote-gcrypt, a key thing is to have a way to map
+from the gcrypt-id of an encrypted repository to a git-annex repository
+uuid.
+
+To do this, we'll make a v5 UUID, feeding in the gcrypt-id.
+The namespace used is itself a v5 UUID, generated using the URL
+namespace and the URL of this page at the time this scheme was
+developed: "http://git-annex.branchable.com/design/gcrypt/"
diff --git a/doc/design/git-remote-daemon.mdwn b/doc/design/git-remote-daemon.mdwn
new file mode 100644
index 0000000000..41e0c93e18
--- /dev/null
+++ b/doc/design/git-remote-daemon.mdwn
@@ -0,0 +1,173 @@
+# goals
+
+* be configured like a regular git remote, with an unusual url
+  or other configuration
+* receive notifications when a remote has received new commits,
+  and take some action
+* optionally, do receive-pack and send-pack to a remote that
+  is only accessible over an arbitrary network transport
+  (like assistant does with XMPP)
+* optionally, send/receive git-annex objects to remote
+  over an arbitrary network transport
+
+# difficulties
+
+* authentication & configuration
+* multiple nodes may be accessible over a single network transport,
+  with it desirable to sync with any/all of them. For example, with
+  XMPP, there can be multiple friends synced with. This means that
+  one git remote can map to multiple remote nodes. Specific to git-annex,
+  this means that a set of UUIDs known to be associated with the remote
+  needs to be maintained, while currently each remote can only have one
+  annex-uuid in .git/config.
+
+# payoffs
+
+* support [[assistant/telehash]]!
+* Allow running against a normal ssh git remote. This would run
+  git-annex-shell on the remote, watching for changes, and so be able to
+  notify when a commit was pushed to the remote repo. This would let the
+  assistant immediately notice and pull. So the assistant would be fully
+  usable with a single ssh remote and no other configuration!
+  **do this first**
+* clean up existing XMPP support, make it not a special case, and not
+  tightly tied to the assistant
+* git-remote-daemon could be used independantly of git-annex,
+  in any git repository.
+
+# design
+
+Let git-remote-daemon be the name. Or for git-annex,
+`git annex remotedaemon`.
+
+It runs in one of two ways:
+
+1. Forked to background, using a named pipe for the control protocol.
+2. With --foreground, the control protocol goes over stdio.
+
+Either way, behavior is the same:
+
+* Get a list of remotes to act on by looking at .git/config
+* Automatically notices when a remote has changes to branches
+  matching remote.$name.fetch, and pulls them down to the appropriate
+  location.
+* When the control protocol informs it about a new ref that's available,
+  it offers the ref to any interested remotes.
+
+# control protocol
+
+This is an asynchronous protocol. Ie, either side can send any message
+at any time, and the other side does not send a reply.
+
+It is line based and intended to be low volume and not used for large data.
+
+TODO: Expand with commands for sending/receiving git-annex objects, and
+progress during transfer.
+
+TODO: Will probably need to add something for whatever pairing is done by
+the webapp.
+
+## emitted messages
+
+* `CONNECTED uri`
+
+  Sent when a connection has been made with a remote.
+
+* `DISCONNECTED uri`
+
+  Sent when connection with a remote has been lost.
+
+* `SYNCING uri`
+
+  Indicates that a pull or a push with a remote is in progress.
+  Always followed by DONESYNCING.
+
+* `DONESYNCING uri 1|0`
+
+  Indicates that syncing with a remote is done, and either succeeded
+  (1) or failed (0).
+
+* `WARNING uri string`
+
+  A message to display to the user about a remote.
+
+## consumed messages
+
+* `PAUSE`
+
+  The user has requested a pause.  
+  git-remote-daemon should close connections and idle.
+
+* `LOSTNET`
+
+  The network connection has been lost.  
+  git-remote-daemon should close connections and idle.
+
+* `RESUME`
+
+  Undoes PAUSE or LOSTNET.  
+  Start back up network connections.
+
+* `CHANGED ref ...`
+
+  Indicates that a ref is new or has changed. These can be offered to peers,
+  and peers that are interested in them can pull the content.
+
+* `RELOAD`
+
+  Indicates that configs have changed. Daemon should reload .git/config
+  and/or restart.
+
+  Possible config changes include adding a new remote, removing a remote,
+  or setting `remote..annex-sync` to configure whether to sync with a
+  particular remote.
+
+* `STOP`
+
+  Shut down git-remote-daemon
+
+  (When using stdio, it also should shutdown when it reaches EOF on 
+  stdin.)
+
+# encryption & authentication
+
+For simplicity, the network transports have to do their own end-to-end
+encryption. Encryption is not part of this design.
+
+(XMPP does not do end-to-end encryption, but might be supported
+transitionally.)
+
+Ditto for authentication that we're talking to who we intend to talk to.
+Any public key data etc used for authentication is part of the remote's
+configuration (or hidden away in a secure chmodded file, if necessary).
+This design does not concern itself with authenticating the remote node,
+it just takes the auth token and uses it.
+
+For example, in telehash, each node has its own keypair, which is used
+or authentication and encryption, and is all that's needed to route
+messages to that node.
+
+# network level protocol
+
+How do peers communicate with one another over the network?
+
+This seems to need to be network-layer dependant. Telehash will need
+one design, and git-annex-shell on a central ssh server has a very different
+(and much simpler) design.
+
+## ssh
+
+`git-annex-shell notifychanges` is run, and speaks a simple protocol
+over stdio to inform when refs on the remote have changed.
+
+No pushing is done for CHANGED, since git handles ssh natively.
+
+This is implemented and seems to work well.
+
+## telehash 
+
+TODO
+
+## xmpp
+
+Reuse [[assistant/xmpp]]
diff --git a/doc/design/git-remote-daemon/comment_1_bfa8f33a3fdb6e271dfbdd0378b5d364._comment b/doc/design/git-remote-daemon/comment_1_bfa8f33a3fdb6e271dfbdd0378b5d364._comment
new file mode 100644
index 0000000000..d93bab0902
--- /dev/null
+++ b/doc/design/git-remote-daemon/comment_1_bfa8f33a3fdb6e271dfbdd0378b5d364._comment
@@ -0,0 +1,16 @@
+[[!comment format=mdwn
+ username="http://johan.kiviniemi.name/"
+ nickname="Johan"
+ subject="Rolling hash chunking"
+ date="2014-04-04T14:16:25Z"
+ content="""
+I am not sure which page is the best for this comment, but this one seems somewhat relevant.
+
+Given that a future telehash implementation may download files from multiple peers, it might be a good idea to download files in chunks, possibly in parallel. In this case, it might be a good idea to use a rolling hash for chunking (like rsync et al). [There is a package for that on Hackage](http://hackage.haskell.org/package/hash-0.2.0.1/docs/Data-Hash-Rolling.html).
+
+git-annex could store a list of chunk checksums in `.git/annex/objects/…/SHA….chunks` whenever the repository holds a copy of the file. The checksum list would be a small fraction of the file in size, but all the checksum lists for all the files in a repository might take up too much space to store in the `git-annex` branch.
+
+When getting an object, git-annex could first download the `.chunks` file from a remote/peer and then proceed to download missing chunks in a BitTorrent-like fashion.
+
+If git-annex has an idea about what locally present object might be an earlier version of the file, it could compare the checksum lists and only download the parts that have changed (à la rsync).
+"""]]
diff --git a/doc/design/iabackup.mdwn b/doc/design/iabackup.mdwn
new file mode 100644
index 0000000000..0f52ae3b79
--- /dev/null
+++ b/doc/design/iabackup.mdwn
@@ -0,0 +1,242 @@
+This is a fairly detailed design proposal for using git-annex to build
+
+
+[[!toc ]]
+
+## end-user view
+
+What the end user sees is a directory, with a .git subdirectory,
+and 100 thousand little files (actually, they're broken symlinks, on
+Linux/OSX). Over time, some of the symlinks start filling in with
+"random" content from the IA. 
+
+The user can look at that content, or even delete files they don't want to
+host.
+
+The user can control how much total disk space the directory takes up.
+(It will use around 100 mb when empty.)
+
+## sharding to scale
+
+The IA contains some 14 million Items. Inside these Items are 271 million
+files. Around 177 million of those are available for download.
+
+git repositories do not scale well in the 1-10 million file 
+range, and very badly above that. Storing all that in a git repository
+would strain git's scalability badly.
+
+Solution: Create multiple git repositories, and split the files
+amoung them.
+
+* If each git repository holds 100 thousand files, that is 1770
+  repositories, which is not an unmanagable number. 
+  (For comparison, git.debian.org has 18500 repositories.)
+
+* The IA is ~20 Petabytes large. Each shard would thus be around 1
+  terabyte in size, although this will vary considerably.
+
+* Clients are assigned one or more shards, and clone those repositories.
+
+* A client decides which files in its shard to back up, and does
+  so by running "git annex get" on them. This downloads the files
+  over http from the IA.
+
+* A client will typically not back up its entire shard, but maybe
+  only 500 gb or less of it. Also, we want redundancy (LOCKSS)
+  -- say at least 3 copies of each file. So, a given shard will probably
+  have between 3 and 9 clients handling it.
+
+* Add new shards as the IA continues to grow.
+
+Problem: Need to get the checksums for the files, for git-annex
+to use. The census published by the IA only has md5sums in it. While
+git-annex can use md5sums, this allows bad actors to find md5 collisions
+with files from the archive, and upload bogus files that checksum ok
+when restoring.
+
+## creating a shard
+
+This is a simple matter of making a git repository and telling git-annex
+the filenames and urls that belong in it.
+
+A script can do this using the `git annex fromkey` and `git annex
+registerurl` commands. Time to make such a repository with 100k files
+is in the 10 minute range (faster on SSD or ramdisk).
+
+## adding a client
+
+When a client registers to participate:
+
+1. Generate a UUID, which is assigned to this client, and send it to the
+   client, and assign that UUID to a particular shard.
+2. Send the client an appropriate auth token (eg, a locked down ssh private
+   key) to let them access the shard's git repository (or all the shards).
+3. Client clones its assigned shard git repository,
+   runs `git annex init reinit $UUID`.
+
+Note that a client could be assigned to multiple shards, rather than just
+one. Probably good to keep a pool of empty shards that have clients waiting
+for new files to be added.
+
+Note that we may want to enable direct mode in the client's clone, 
+because it lets the user easily delete files to free up space.
+OTOH, direct mode is slow and less safe, so we might prefer to use indirect
+mode, and then the client would need to use `git annex drop` if they
+decided to remove content.
+
+## distributing files
+
+1. Client runs `git annex sync --content`, which downloads as many
+   files from the IA as will fit in their disk's free space
+   (leaving some configurable amount free in reserve by configuring
+   annex.diskreserve)
+2. Note that [[numcopies|copies]] and [[preferred_content]] settings can be
+   used to make clients only want to download an file if it's not yet
+   reached the desired number of copies. Lots of flexibility here in
+   git-annex.
+3. git-annex will push back to the server an updated git-annex branch,
+   which will record when it has successfully stored an file.
+
+## bad actors
+
+Clients can misbehave in probably many ways. The best defense for many
+misbehaviors is to distribute files to enough different clients that we can
+trust some of them.
+
+The main git-annex specific misbehavior is that a client could try to push
+garbage information back to the origin repository on the server.
+
+To guard against this, the server will reject all pushes of branches other
+than the git-annex branch, which is the only one clients need to modify.
+
+Check pushes of the git-annex branch. There are only a few files that
+clients can legitimately modify, and the modifications will always involve
+that client's UUID, not some other client's UUID. Reject anything shady.
+
+These checks can be done in a git `update` hook. Rough estimate is that
+such a hook would be a couple hundred lines of code.
+
+## verification
+
+We want a lightweight verification process, to verify that a client still
+has the data. This can be done using `git annex fsck`, which can be
+configured to eg, check each file only once per month.
+
+git-annex will need a modification here. Currently, a successful fsck
+does not leave any trace in the git-annex branch that it happened. But
+we want the server to track when a client is not fscking (the user probably
+dropped out).
+
+The modification is simple; just have a successful fsck
+update the timestamp in the fscked file's location log.
+It will probably take just a few hours to code.
+
+With that change, the server can check for files that not enough clients
+have verified they have recently, and distribute them to more clients.
+
+(This is now implemented.)
+
+Note that bad actors can lie about this verification; it's not a proof they
+still have the file. But, a bad actor could prove they have a file, and
+refuse to give it back if the IA needed to restore the backup, too.
+
+## fire drill
+
+If we really want to test how well the system is working, we need a fire
+drill.
+
+1. Pick some files that we'll assume the IA has lost in some disaster.
+2. Look up the shard the file belongs to.
+3. Get the git-annex key of the file, and tell git-annex it's been
+   lost from the IA, by running in its shard: `setpresentkey $key $iauuid 0`
+4. The next time a client runs `git annex sync --content`, it will notice
+   that the IA repo doesn't have the file anymore. The client will then
+   send the file back to the origin repo.
+5. To guard against bad actors, that restored file should be checked with
+   `git annex fsck`. If its checksum is good, it can be re-injected back
+   into the IA. (Or, the fire drill was successful.)
+   (Remember to turn off the fire alarm by running
+   `setpresentkey $key $iauuid 1`)
+
+## shard servers
+
+A server at the IA (or otherwise with a fast pipe) is needed to serve
+the shards. One server can probably manage them all.
+Let's consider what this server needs to have on it:
+
+* git and git-annex
+* ssh server
+* The git repository for each shard. A few hundred mb per shard.
+* The git update hook to filter out bad pushes.
+* Some way to learn when a new user has registered to access a shard,
+  so their ssh key is given access.
+
+## other optional nice stuff
+
+The user running a client can delete some or all of their files at any
+time, to free up disk space. The next time `git-annex sync` runs on the client,
+it'll notice and let the server know, and other clients will then take
+over storing it. (Or if the git-annex assistant is run on the client,
+it would inform the server immediately.)
+
+The user is also free to move files around (within the git repository
+directory), modify files, view them, etc. This doesn't affect anyone else.
+
+Offline storage is supported. As long as the user can spin it up from time
+to time to run `git annex fsck`.
+
+More advanced users might have multiple repositories on different disks.
+Each has their own UUID, and they could move files around between them as
+desired; this would be communicated back to the origin repository
+automatically.
+
+Shards could have themes, and users could request to be part of the
+shard that includes Software, or Grateful Dead, etc. This might encourage
+users to devote more resources.
+
+Or, rather than doing a lucky dip and getting one or a couple shards,
+a user could clone em all, and pick just which files to get.
+
+The contents of files sometimes changes.
+This can be reflected by updating the file in the git repository.
+Clients will then download the new version of the file. (They will also
+tend to retain the old version, although this can be dealt with by using
+`git annex unused`).
+
+Items sometimes go dark; this could be reflected by deleting the Item's
+files from the repository. It's up to the clients what they do with the
+content of such Items.
+
+Client's repos could be put into groups to classify them. For example,
+there could be groups per continent, or for trust levels, or whatever.
+These can be used by [[preferred_content]] expressions to fine tune how
+files are spread out amoung the available clients.
+
+## other potential gotchas
+
+If any single file is very large (eg, 10 terabytes), there may not be
+any clients that can handle it. This could be dealt with by splitting up
+the file into smaller chunks. Word is there is a single 2 tb item, and a few
+more around 100 gb, so this is probably not a concern.
+
+A client could add other files to its local repo, and git-annex branch
+pushes would include junk data about those files. It should probably be
+filtered out by the git update hook (rejecting the whole push because of
+this seems excessive).
+
+There may be a thundering herd problem, where many clients end up
+downloading the same file at the same time, and more copies than neecessary
+result. The next `git annex sync --content` in some of the
+redundant clients will notice this and drop that file, and presumably
+download some other file. It would be good to avoid this problem,
+perhaps by having a new client initially download a random set of the
+files in their shard that don't yet have enough copies.
+
+With clients all fscking their part of a shard once a month,
+that will increase the size of the git repository, with new distributed
+fsck updates. I have run some test and this fsck overhead delta compresses
+well. With a 10 thousand file repo and 100 clients all updating the
+location log, the monthly fsck only added 1 mb to the repository size
+(after `git gc --aggressive`). Should scale linearly with number of files
+in repo. Note that `git annex forget` could be used to forget old
+historical data if the repo grew too large from fsck updates.
diff --git a/doc/design/iabackup/comment_1_d33c0910973bc37ce81bf434017e11fd._comment b/doc/design/iabackup/comment_1_d33c0910973bc37ce81bf434017e11fd._comment
new file mode 100644
index 0000000000..ec06675613
--- /dev/null
+++ b/doc/design/iabackup/comment_1_d33c0910973bc37ce81bf434017e11fd._comment
@@ -0,0 +1,11 @@
+[[!comment format=mdwn
+ username="https://www.google.com/accounts/o8/id?id=AItOawnx8kHW66N3BqmkVpgtXDlYMvr8TJ5VvfY"
+ nickname="Yaroslav"
+ subject="great to see such a large scale effort ongoing"
+ date="2015-03-06T04:47:30Z"
+ content="""
+and I would still maintain my view that removing intermediate directory withing .git/annex/objects whose  current roles is simply to provide read-only protection might half the burden on the underlying file system, either annex repo(s) are multitude or a single one [1]. lean view [2] could also be of good use as well[2].  Similar exercises with simulated annex'es with >5M files also \"helped\" to identify problems with ZOL (ZFS on Linux) caching suggesting that even mere handling of such vast arrays of tiny files (as dead symlinks) might give filesystems a good test, so the leaner impact would be -- the better.
+
+[1] e.g. https://github.com/datalad/datalad/issues/32#issuecomment-70523036
+[2] https://github.com/datalad/datalad/issues/25
+"""]]
diff --git a/doc/design/iabackup/comment_2_c0a59549409faa355a461e85a1c3f908._comment b/doc/design/iabackup/comment_2_c0a59549409faa355a461e85a1c3f908._comment
new file mode 100644
index 0000000000..135917b4ee
--- /dev/null
+++ b/doc/design/iabackup/comment_2_c0a59549409faa355a461e85a1c3f908._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="https://www.google.com/accounts/o8/id?id=AItOawkSq2FDpK2n66QRUxtqqdbyDuwgbQmUWus"
+ nickname="Jimmy"
+ subject="comment 2"
+ date="2015-03-09T16:48:18Z"
+ content="""
+I've tried throwing about ~16 million files at git/git-annex in the past where some files were 1-2kb in size (around 30% of them). git/git-annex doesn't work well at that scale.
+"""]]
diff --git a/doc/design/iabackup/comment_3_560d3f65d543c3af9722ed7e9a11e920._comment b/doc/design/iabackup/comment_3_560d3f65d543c3af9722ed7e9a11e920._comment
new file mode 100644
index 0000000000..dca02f8af8
--- /dev/null
+++ b/doc/design/iabackup/comment_3_560d3f65d543c3af9722ed7e9a11e920._comment
@@ -0,0 +1,13 @@
+[[!comment format=mdwn
+ username="https://www.google.com/accounts/o8/id?id=AItOawmsy_GIefGlGGD_XJp_R6EsWIRUC4ev9XU"
+ nickname="David"
+ subject="This is a BIG task"
+ date="2015-03-13T20:48:56Z"
+ content="""
+If I understand it correctly, 20PB at 2400 shards of 8TB each with 3 copies is 24TB/shard at 1TB/client is 2400*24 = ~60K clients assuming no churn. So it would probably need ~100K clients to cover the churn and have a good chance that each shard had 3 copies at all times. That's 1/3 the size of BOINC's active population.
+
+It would take time to scale to that population. And it would take time to get three copies out of the Archive. During that time, the Archive is growing. The back of my envelope says that doing this in 2.5yrs roughly doubles the Archive's outbound bandwidth if you average it across the 2.5 years. But the population would grow slowly to start with, then faster, so that the bandwidth impact would be back-loaded. And at the end of the 2.5 years, you would need a lot more than the 100K users.
+
+A design that used erasure coding or entanglement would reduce the storage and bandwidth demand considerably while providing adequate reliability.
+
+"""]]
diff --git a/doc/design/iabackup/comment_4_465c0966c96a57d189f678d4fa724aa0._comment b/doc/design/iabackup/comment_4_465c0966c96a57d189f678d4fa724aa0._comment
new file mode 100644
index 0000000000..15faa40b59
--- /dev/null
+++ b/doc/design/iabackup/comment_4_465c0966c96a57d189f678d4fa724aa0._comment
@@ -0,0 +1,13 @@
+[[!comment format=mdwn
+ username="https://me.yahoo.com/a/idrn495us85k6mwfdMUolYIsyp4-#cf755"
+ subject="It's not so bad .. only about 10PB"
+ date="2015-04-30T01:19:38Z"
+ content="""
+The good news is that web-archive (.ARC) items are not publicly browsable, and that's about half of the archive's content, so you're only looking at about 10PB to backup.
+
+The bad news is that unless you can work something out with archive.org (which seems unlikely; web-archive items are restricted to protect them legally), or use the old waybackup interface (which I don't think works anymore), or use the wayback machine (which last I heard only supported a few hundred connections per second) you'll only be able to back up half their data.
+
+Still, non-web items seem like a nice place to start.
+
+
+"""]]
diff --git a/doc/design/iabackup/comment_5_7e4d1db9c69c63e79ca13db2ad87c384._comment b/doc/design/iabackup/comment_5_7e4d1db9c69c63e79ca13db2ad87c384._comment
new file mode 100644
index 0000000000..4d24e8f6fc
--- /dev/null
+++ b/doc/design/iabackup/comment_5_7e4d1db9c69c63e79ca13db2ad87c384._comment
@@ -0,0 +1,7 @@
+[[!comment format=mdwn
+ username="db48x"
+ subject="14 of 21PB, actually"
+ date="2015-04-30T02:58:05Z"
+ content="""
+IA helpfully did a quick count for us: https://archive.org/details/ia-bak-census_20150304
+"""]]
diff --git a/doc/design/metadata.mdwn b/doc/design/metadata.mdwn
new file mode 100644
index 0000000000..4e43fe3441
--- /dev/null
+++ b/doc/design/metadata.mdwn
@@ -0,0 +1,182 @@
+[[!toc]]
+
+# metadata
+
+Attach an arbitrary set of metadata to a key. This consists of any number
+of fields. Each field has an unordered set of values. The special field
+"tag" has as its values any tags that are set for the key.
+
+Store in git-annex branch, next to location log files.
+
+Storage needs to support union merging, including removing an old value
+of a field, and adding a new value of a field.
+
+# filtered branches
+
+See [[tips/metadata_driven_views]]
+
+The reason to use specially named filtered branches is because it makes
+self-documenting how the repository is currently filtered.
+
+## unmatched files in filtered branches
+
+TODO Files not matching the view should be able to be included in
+the filtered branch, in a special location, an "other" directory.
+
+For example, it could make a "other" directory containing files
+without a tag when viewing by tag.
+
+It might be nice, if in a two level view, for the other directories
+to nest. For example, `other/2014/file`. However, that leads to a
+performance problem: When adding a level to a view, it has to look at each
+file in the "other" directory and generate a view for it too. With a lot
+of files, that'd be slow.
+
+Instead, why not replicate the parent branch's directory structure inside
+the "other" directory? Then the directory tree only has to be constructed
+once, and can be left alone when refining a view.
+
+## operations while on filtered branch
+
+* If files are removed and git commit called, git-annex should remove the
+  relevant metadata from the files. **done**  
+  (Currently, only metadata used for visible subdirs is added and removed
+  this way.)
+  (Also, this is not usable in direct mode because deleting the
+  file.. actually deletes it...)
+* If a file is moved into a new subdirectory while in a view branch,
+  a tag is added with the subdir name. This allows on the fly tagging.
+  **done**
+* `git annex sync` should avoid pushing out the view branch, but
+  it should check if there are changes to the metadata pulled in, and update
+  the branch to reflect them.
+
+## automatically added metadata
+
+When `annex.genmetadata` is set, git annex add automatically attaches
+some metadata to a file. Currently year and month fields, from its mtime.
+
+There's also a post-commit-annex hook script.
+
+## directory hierarchy metadata
+
+From the original filename used in the master branch, when
+constructing a view, generate fields. For example foo/bar/baz.mp3
+would get /=foo, foo/=bar, foo/bar/=baz, and .=mp3.
+
+Note that dir/=subdir allows a view to use `dir/=*` and only
+match one level of subdirs with the glob. So is better than dir=foo/bar
+as the metadata. (Alternatively, could do special glob matching.)
+
+This allows using whatever directory hierarchy exists to inform the view,
+without locking the view into using it. 
+
+Complication: When refining a view, it only looks at the filenames in
+the view, so it has to map from
+those filenames to derive the same metadata, unless there is persistent
+storage. Luckily, the filenames used in the views currently include the
+subdirs.
+
+# other uses for metadata
+
+Uses are not limited to view branches.
+
+`git annex checkoutmeta year=2014 talk` in a subdir of master could create the
+same tree of files filter would. The user can then commit that if desired.
+Or, they could run additional commands like `git annex fadd` to refine the
+tree of files in the subdir.
+
+Metadata can be used for configuring numcopies. One way would be a
+numcopies=n value attached to a file. But perhaps better would be to make
+the numcopies.log allow configuring numcopies based on which files have
+other metadata.
+
+Other programs could query git-annex for the metadata of files in the work
+tree, and do whatever it wants with it.
+
+# filenames
+
+The hard part of this is actually getting a useful filename to put in the
+view branch, since git-annex only has a key which the user will not
+want to see.
+
+* Could use filename metadata for the key, recorded by git-annex add (which
+  may not correspond to filenames being used in regular git branches like
+  master for the key).
+* Could use the Keys database's associated files. Currently only works for v6
+  unlocked files, and not for locked files.
+* Current approach: Have a reference branch (eg master) and walk it to
+  find filenames and
+  keys. Fine as long as it can be done efficiently. Also allows including
+  the subdirectory a file is in, potentially. cwebber points out that this
+  is essentially a form of tracking branch. Which implies it will need to
+  be updatable when the reference branch changes. Should be doable via
+  diff-tree.
+
+Note that we have to take care to avoid generating conflicting filenames.
+The current approach is to embed the full directory structure inside the
+filename in the view branch.
+
+## union merge properties
+
+While the storage could just list all the current values of a field on a
+line with a timestamp, that's not good enough. Two disconnected
+repositories can make changes to the values of a field (setting and
+unsetting tags for example) and when this is union merged back together,
+the changes need to be able to be replayed in order to determine which
+values we end up with. 
+
+To make that work, we log not only when a field is set to a value, 
+but when a value is unset as well.
+
+For example, here two different remotes added tags, and then later
+a tag was removed:
+
+	1287290776.765152s tag +foo +bar
+	1287290991.152124s tag +baz
+	1291237510.141453s tag -bar
+
+# efficient metadata lookup
+
+Looking up metadata for view generation so far requires traversing all keys
+in the git-annex branch. This is slow. A fast cache is needed.
+
+TODO
+
+# unlocked file issues
+
+View branches can't be used in direct mode repositories.
+
+But, view branches do work with unlocked files in v6 repositories.
+The resulting view branch has all its files locked, although you
+can unlock them again after entering the branch.
+
+# gotchas
+
+* Checking out a view branch can remove the current subdir. May be worth
+  detecting when this happens and help the user.
+  **done**
+
+* Git has a complex set of rules for what is legal in a ref name.
+  View branch names will need to filter out any illegal stuff. **done**
+
+* Metadata should be copied to the new key when adding a modified version
+  of a file. **done**
+
+* Filesystems that are not case sensitive (including case preserving OSX)
+  will cause problems if view branches try to use different cases for 
+  2 directories representing a metadata field.
+  
+  Solution might be to compare fields names case-insensitively, and
+  pick one representation consistently. **done**
+
+* Assistant needs to know about views, so it can update metadata when
+  files are moved around inside them. TODO
+
+* What happens if git annex add or the assistant add a new file while on a
+  view? If the file is not also added to the master branch, it will be lost
+  when exiting the view. TODO
+
+* The filename mangling can result in a filename in a view
+  that is too long for its containing filesystem. Should detect and do
+  something reasonable to avoid. TODO
diff --git a/doc/design/metadata/comment_10_9bc2825b18ce29d2c9b2f085b95aa68c._comment b/doc/design/metadata/comment_10_9bc2825b18ce29d2c9b2f085b95aa68c._comment
new file mode 100644
index 0000000000..107673c7c4
--- /dev/null
+++ b/doc/design/metadata/comment_10_9bc2825b18ce29d2c9b2f085b95aa68c._comment
@@ -0,0 +1,18 @@
+[[!comment format=mdwn
+ username="https://www.google.com/accounts/o8/id?id=AItOawlZF5AC-FSxwkiay5ZgEYZwUzN69Wa6PTE"
+ nickname="Sunke"
+ subject="filename from metadata?"
+ date="2015-04-06T18:00:25Z"
+ content="""
+Hi everbody,
+is it possible to use a metadata field for the filename in a
+metadata driven view?
+
+I am thinking of the following use case:
+
+git annex metadata --set artist=Led\ Zeppelin --set album=Led\ Zeppelin\ IV --set title=04\ Stairway\ to\ heaven  some/weird/filename.mp3
+git annex view --filename-from title artist=* album=*
+
+result:
+Led Zeppelin/Led Zeppelin IV/04 Stairway to heaven.mp3
+"""]]
diff --git a/doc/design/metadata/comment_11_402d7d3e8e7f2df57eb6685134226642._comment b/doc/design/metadata/comment_11_402d7d3e8e7f2df57eb6685134226642._comment
new file mode 100644
index 0000000000..b7b8937c52
--- /dev/null
+++ b/doc/design/metadata/comment_11_402d7d3e8e7f2df57eb6685134226642._comment
@@ -0,0 +1,14 @@
+[[!comment format=mdwn
+ username="joey"
+ subject="""comment 11"""
+ date="2015-04-09T18:31:34Z"
+ content="""
+@Sunke, the reason that views make up their own filenames is to
+avoid the problem of having 2 files in a view that have the same
+name.
+
+In your example, that could happen if you used --set title
+with the same title for 2 separate files.
+
+So, I don't think this can be supported reasonably.
+"""]]
diff --git a/doc/design/metadata/comment_1_22ed80bd8eabaa836e9dfc2432531f04._comment b/doc/design/metadata/comment_1_22ed80bd8eabaa836e9dfc2432531f04._comment
new file mode 100644
index 0000000000..493db43397
--- /dev/null
+++ b/doc/design/metadata/comment_1_22ed80bd8eabaa836e9dfc2432531f04._comment
@@ -0,0 +1,22 @@
+[[!comment format=mdwn
+ username="https://www.google.com/accounts/o8/id?id=AItOawm3vKzS4eOWYpKMoYXqMIjNsIg_nYF-loU"
+ nickname="Konubinix"
+ subject="Already existing metadata implementation "
+ date="2014-02-22T21:45:25Z"
+ content="""
+Hi,
+
+I love the idea behing storing metadata.
+
+I suggest to exchange ideas (and maybe code) with projects already implementing metadata systems.
+
+I have tried several implementations and particularly noticed tmsu (http://tmsu.org/). This tool stores tags into a sqlite database and uses also a SHA-256 fingerprint of the file to be aware of file moves. It provides a fuse view of the tags with the ability to change tags by moving files (like in the git annex metadata view).
+
+Paul Ruane is particularly responsive on the mailing list and he already supports git annexed files (with SHAE-256 fingerprint) (see the end of the thread https://groups.google.com/forum/#!topic/tmsu/A5EGpnCcJ2w).
+
+Even if you cannot reuse the project, they are interresting ideas that might be worth looking at like the implications of tags: a file tagged \"film\" being automatically tagged \"video\".
+
+Tagsistant (http://www.tagsistant.net/) may also be a good source of inspirations. I just don't like the fact that it uses a backstore of tagged files.
+
+Thanks for reading.
+"""]]
diff --git a/doc/design/metadata/comment_2_03ae28acedbe1fa45c366b30b58fcf48._comment b/doc/design/metadata/comment_2_03ae28acedbe1fa45c366b30b58fcf48._comment
new file mode 100644
index 0000000000..c222f75d36
--- /dev/null
+++ b/doc/design/metadata/comment_2_03ae28acedbe1fa45c366b30b58fcf48._comment
@@ -0,0 +1,14 @@
+[[!comment format=mdwn
+ username="https://www.google.com/accounts/o8/id?id=AItOawkSq2FDpK2n66QRUxtqqdbyDuwgbQmUWus"
+ nickname="Jimmy"
+ subject="comment 2"
+ date="2014-02-25T09:51:17Z"
+ content="""
+Some additional ideas for metadata...
+
+Instead of having a simplistic scheme like 'field=value' it might be advantageous to consider a scheme like 'attribute=XXX, value=YYY, unit=ZZZ' that way you could do intesting things with the metadata like adding counters to things, and allow for doing interesting queries like give me all 'things' tagged with a unit of \"audio_file\", this assumes one had trawled through an entire annex and then tagged all files based on type with the unix file tool or something like that.
+
+The above idea is already in use in irods and its a really nice and powerful way to let users add meta-data and to build up more interesting use cases and tools.
+
+btw, I plan on taking a look at seeing if I can map some of the meta that we have in work into this new git-annex feature to see how well/bad it works. Either way this feature looks cool! +1!!!
+"""]]
diff --git a/doc/design/metadata/comment_3_ee850df7d3fa4c56194f13a6e3890a30._comment b/doc/design/metadata/comment_3_ee850df7d3fa4c56194f13a6e3890a30._comment
new file mode 100644
index 0000000000..f77cd86115
--- /dev/null
+++ b/doc/design/metadata/comment_3_ee850df7d3fa4c56194f13a6e3890a30._comment
@@ -0,0 +1,12 @@
+[[!comment format=mdwn
+ username="https://www.google.com/accounts/o8/id?id=AItOawkSq2FDpK2n66QRUxtqqdbyDuwgbQmUWus"
+ nickname="Jimmy"
+ subject="comment 3"
+ date="2014-02-25T09:57:09Z"
+ content="""
+actually in your mp3 example you could have ....
+
+ATTRIBUTE=sample_rate, VALUE=22100, UNIT=Hertz
+
+another example use case is to always be consistent with the AVU order then you could stick in ntriples from RDF to do other cool things by looking up various linked data sources -- see http://www.w3.org/2001/sw/RDFCore/ntriples/ and http://www.freebase.com/, actually this would be quite cool if git-annex examined the mp3's id3 tag, the created an ntriple styled entry can be automatically parsed with the web-based annex gui and automatically pull in additional meta-data from the likes of freebase. I guess the list of ideas can just only get bigger with this potential metadata capability.
+"""]]
diff --git a/doc/design/metadata/comment_4_c32ade1524487e5fdc6f83b2db39f04c._comment b/doc/design/metadata/comment_4_c32ade1524487e5fdc6f83b2db39f04c._comment
new file mode 100644
index 0000000000..01f917ef59
--- /dev/null
+++ b/doc/design/metadata/comment_4_c32ade1524487e5fdc6f83b2db39f04c._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="bremner"
+ ip="198.164.160.48"
+ subject="convenient way to query metadata?"
+ date="2014-03-15T20:58:28Z"
+ content="""
+I'd like to be able to do something like \"git annex metadata -q fieldname\" and have that output the value(s) of fieldname.  I see I could parse the json output but that isn't too convenient in a shell script. Or have I missed something that already exists?
+"""]]
diff --git a/doc/design/metadata/comment_5_0ac3132cd7a84f0e170fbe3a6f235fe7._comment b/doc/design/metadata/comment_5_0ac3132cd7a84f0e170fbe3a6f235fe7._comment
new file mode 100644
index 0000000000..ca98f7f415
--- /dev/null
+++ b/doc/design/metadata/comment_5_0ac3132cd7a84f0e170fbe3a6f235fe7._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="http://joeyh.name/"
+ ip="209.250.56.154"
+ subject="comment 5"
+ date="2014-03-15T21:30:52Z"
+ content="""
+@bremner, you must be up to something interesting.. Added metadata --get for you.
+"""]]
diff --git a/doc/design/metadata/comment_6_fa51ae544b193122334dbae7960ab3d9._comment b/doc/design/metadata/comment_6_fa51ae544b193122334dbae7960ab3d9._comment
new file mode 100644
index 0000000000..7be49a6a92
--- /dev/null
+++ b/doc/design/metadata/comment_6_fa51ae544b193122334dbae7960ab3d9._comment
@@ -0,0 +1,15 @@
+[[!comment format=mdwn
+ username="tdussa"
+ ip="217.84.74.69"
+ subject="Why not automatically add the whole date?"
+ date="2014-04-30T20:41:20Z"
+ content="""
+Hi,
+
+apologies if I am missing something, but from what I understand, git-annex will automatically add the year and the month from a file's mtime to its metadata if instructed to do so.
+
+So... What about the day (or the time, for that matter?)?  What is the reasoning behind the decision not to add those bits automatically?  And, is there a way to get git-annex to add those bits of information automatically as well (besides the obvious way of creating a pre-commit-hook script to that effect)?
+
+THX & Cheers,
+Toby.
+"""]]
diff --git a/doc/design/metadata/comment_7_04cd255a516c8520a7bc1a8fad253533._comment b/doc/design/metadata/comment_7_04cd255a516c8520a7bc1a8fad253533._comment
new file mode 100644
index 0000000000..12e5042fb9
--- /dev/null
+++ b/doc/design/metadata/comment_7_04cd255a516c8520a7bc1a8fad253533._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="https://www.google.com/accounts/o8/id?id=AItOawlM_DRhi_5pJrTA0HbApHR25iAgy-NBXTY"
+ nickname="Tor Arne"
+ subject="comment 7"
+ date="2014-10-01T22:43:40Z"
+ content="""
+I have the same question as Toby, is there a particular reason the whole timestamp is not stored?
+"""]]
diff --git a/doc/design/metadata/comment_8_0a7e55e7626f72f63966fa1e1d2cf100._comment b/doc/design/metadata/comment_8_0a7e55e7626f72f63966fa1e1d2cf100._comment
new file mode 100644
index 0000000000..965b1932e5
--- /dev/null
+++ b/doc/design/metadata/comment_8_0a7e55e7626f72f63966fa1e1d2cf100._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="https://www.google.com/accounts/o8/id?id=AItOawlM_DRhi_5pJrTA0HbApHR25iAgy-NBXTY"
+ nickname="Tor Arne"
+ subject="Can tags/metadata be used for preferred content?"
+ date="2014-10-01T22:45:36Z"
+ content="""
+Would love to be able to \"tag\" something as archived instead of moving it into a special folder. Coupled with a FinderSync extension on OS X Yosemite for right-click menu. This would allow me to also \"view\" the archive and bring things out of there by \"untagging\" it, if I understand the feature correctly?
+"""]]
diff --git a/doc/design/metadata/comment_9_f0bb62c885a925e0da5ae8ce3c5e9003._comment b/doc/design/metadata/comment_9_f0bb62c885a925e0da5ae8ce3c5e9003._comment
new file mode 100644
index 0000000000..fac3bf135a
--- /dev/null
+++ b/doc/design/metadata/comment_9_f0bb62c885a925e0da5ae8ce3c5e9003._comment
@@ -0,0 +1,10 @@
+[[!comment format=mdwn
+ username="https://www.google.com/accounts/o8/id?id=AItOawlM_DRhi_5pJrTA0HbApHR25iAgy-NBXTY"
+ nickname="Tor Arne"
+ subject="comment 9"
+ date="2014-10-01T23:35:39Z"
+ content="""
+Sorry for the noise, I see that tags _can_ be used for preferred content, excellent!
+
+But it seems metadata is tied to a key, not to a specific file/path. If I have 10 different files all with the same content (for some reason, say a simple txt file, Gemspec, or something), and I want to tag one of them as important, it doesn't mean they all are :o
+"""]]
diff --git a/doc/design/new_repo_versions.mdwn b/doc/design/new_repo_versions.mdwn
new file mode 100644
index 0000000000..97bf2c0a73
--- /dev/null
+++ b/doc/design/new_repo_versions.mdwn
@@ -0,0 +1,234 @@
+This page's purpose is to collect and explore plans for a future
+annex.version.
+
+There are two major possible changes that could go in a new repo
+version that would require a hard migration of git-annex repositories:
+
+1. Changing .git/annex/objects/ paths, as appear in the git-annex symlinks.
+
+2. Changing the layout of the git-annex branch in a substantial way.
+
+## object path changes
+
+Any change in this area requires the user make changes to their master
+branch, any other active branches. Old un-converted tags and other
+historical trees in git would also be broken. This is a pretty bad user
+experience. (And it bloats history with a commit that rewrites everything
+too.
+
+For this reason, any changes in this area have been avoided, going all the
+way back to v2 (2011). 
+
+> git-annex had approximately 3 users at the
+> time of that migration, and as one of them, I can say it was a total PITA.
+--[[Joey]] 
+
+So, there would need to be significant payoffs to justify this change.
+
+Note that changing the hash directories might also change where objects are
+stored in special remotes. Because repos can be offline or expensive to
+migrate (or both -- Glacier!) any such changes need to keep looking in the
+old locations for backwards compatability.
+
+Possible reasons to make changes:
+
+* It's annoyingly inconsistent that git-annex uses a different hash
+  directory layout for non-bare repository (on a non-crippled filesystem)
+  than is used for bare repositories and some special remotes.
+
+  Users occasionally stumble over this difference when messing with
+  internals. The code is somewhat complicated by it. In some cases,
+  git-annex checks both locations (eg, a bare repo defaults to xxx/yyy
+  but really old ones might use xX/yY for some keys).
+
+  The mixed case hash directories have caused trouble on case-insensative
+  filesystems, although that has mostly been papered over to avoid
+  problems. One remaining problem users can stuble on occurs
+  when [[moving a repository from OSX to Linux|bugs/OSX_case_insensitive_filesystem]].
+
+* The hash directories, and also the per-key directories
+  can slow down using a repository on a disk (both SSD and spinning).
+  
+  
+
+  Initial benchmarks suggest that going from xX/yY/KEY/OBJ to xX/yY/OBJ
+  directories would improve speed 3x.
+
+  Presumably, removing the yY would also speed it up, unless there are too
+  many objects and the filesystem gets slow w/o the hash directories.
+
+* Removing a directory level would also reduce disk usage, see [[forum/scalability_with_lots_of_files/]] for more info.
+
+## git-annex branch changes
+
+This might involve, eg, rethinking the xxx/yyy/ hash directories used
+in the git-annex branch.
+
+Would this require a hard version transition? It might be possible to avoid
+one, but then git-annex would have to look in both the old and the new
+place. And if a un-transitioned repo was merged into a transitioned one,
+git-annex would have to look in *both* places, and union merge the two sets
+of data on the fly. This doubles the git-cat-file overhead of every
+operation involving the git-annex branch. So a hard transition would
+probably be best.
+
+Also, note that w/o a hard transition, there's the risk that a old
+git-annex version gets ahold of a git-annex branch created by a new
+git-annex version, and sees only half of the story (the un-transitioned
+files). This could be a very confusing failure mode. It doesn't help that
+the git-annex branch does not currently have any kind of
+version number embedded in it, so the old version of git-annex doesn't even
+have a way to check if it can handle the branch.
+
+Possible reasons to make changes:
+
+* There is a discussion of some possible changes to the hash directories here
+   with a
+  goal of reducing the overhead of the git-annex branch in the overall size
+  of the git-annex repository. 
+  
+  Removing the second-level hash directories might improve performance.
+  It doesn't save much space when a repository is having incremental changes
+  made to it. However, if millions of annexed objects are being added
+  in a single commit, removing the second-level hash directories does save
+  space; it halves the number of tree
+  objects[1](https://github.com/datalad/datalad/issues/17#issuecomment-68759754).
+
+  Also,
+  
+  suggests using xxx/yyy.log, where one log contains information for
+  multiple keys. This would probably improve performance too due to
+  caching, although in some cases git-annex would have to process extra
+  information to get to the info about the key it wants, which hurts
+  performance. The disk usage change of this method has not yet been
+  quantified.
+
+* Another reason to do it would be improving git-annex to use vector clocks,
+  instead of its current assumption that client's clocks are close enough to
+  accurate. This would presumably change the contents of the files.
+
+* While not a sufficient reason on its own, the best practices for file
+  formats in the git-annex branch has evolved over time, and there are some
+  files that have unusual formats for historical reasons. Other files have
+  modern formats, but their parsers have to cope with old versions that
+  have other formats. A hard transition would provide an opportunity to
+  clean up a lot of that.
+
+## living on the edge
+
+Rather than a hard transition, git-annex could add a mode
+that could be optionally enabled when initing a repo for the first time.
+
+Users who know they need that mode could then turn it one, and get the
+benefits, while everyone else avoids a transition that doesn't benefit them
+much.
+
+There could even be multiple modes, with different tradeoffs depending on
+how the repo will be used, its size, etc. Of course that adds complexity.
+
+But the main problem with this idea is, how to avoid the foot shooting
+result of merging repo A(v5) into repo B(vNG)? This seems like it would be
+all to easy for a user to do. 
+
+As far as git-annex branch changes go, it might be possible for git-annex
+to paper over the problem by handling both versions in the merged git-annex
+branch, as discussed earlier. But for .git/annex/objects/ changes, there
+does not seem to be a reasonable thing for git-annex to do. When it's
+receiving an object into a mixed v5 and vNG repo, it can't know which
+location that repo expects the object file to be located in. Different
+files in the repo might point to the same object in different locations!
+Total mess. Must avoid this.
+
+Currently, annex.version is a per-local-repo setting. git-annex can't tell 
+if two repos that it's merging have different annex.version's. 
+
+It would be possible to add a git-annex:version file, which would work for
+git-annex branch merging. Ie, `git-annex merge` could detect if different
+git-annex branches have different versions, and refuse to merge them (or
+upgrade the old one before merging it).
+
+Also, that file could be used by git-annex, to automatically set
+annex.version when auto-initing a clone of a repo that was initted with
+a newer than default version.
+
+But git-anex:version won't prevent merging B/master into A's master.
+That merge can be done by git; nothing in git-annex can prevent it.
+
+What we could do is have a .annex-version flag file in the root of the
+repo. Then git merge would at least have a merge conflict. Note that this
+means inflicting the file on all git-annex repos, even ones used by people
+with no intention of living on the edge. And, it would take quite a while
+until all such repos get updated to contain such a file.
+
+Or, we could just document that if you initialize a repo with experimental
+annex.version, you're living on the edge and you can screw up your repo
+by merging with a repo from an old version.
+
+git-annex fsck could also fix up any broken links that do result from the
+inevitable cases where users ignore the docs.
+
+## version numbers vs configuration
+
+A particular annex.version like 5 encompasses a number of somewhat distinct
+things
+
+* git-annex branch layout
+* .git/annex/objects/ layout
+* other git stuff (like eg, the name of the HEAD branch in direct mode)
+
+If the user is specifying at `git annex init` time some nonstandard things
+they want to make the default meet their use case better, that is more
+a matter of configuration than of picking a version. 
+
+For example, we could say that the user is opting out of the second-level
+object hash directories. Or we could say the user is choosing to use vNG,
+which is like  v5 except with different object hash directory structure.
+
+	git annex init --config annex.objects.hashdirectories 1 
+		--config annex.objects.hashlower true
+	git annex init --version 6
+
+The former would be more flexible. The latter is simpler. 
+
+The former also lets the user chose *no* hash directories, or 
+choose 2 levels of hash directories while using the (v5 default) mixed
+case hashing.
+
+## concrete design
+
+Make git-annex:difference.log be used by newer git-annex versions than v5,
+and by nonstandard configurations.
+
+The file contents will be "timestamp uuid [value, ..]", where value is a
+serialized data type that describes divergence from v5 (since v5 and older
+don't have the git-annex:difference.log file).
+
+So, for example, "[Version 6]" could indicate that v6 is being used. Or,
+"[ObjectHashLower True, ObjectHashDirectories 1, BranchHashDirectories 1]"
+indicate a nonstandard configuration on top of v5 (this might turn out to
+be identical to v6; just make the compare equal and no problem).
+
+git-annex merge would check if it's merging in a git-annex:difference.log from
+another repo that doesn't match the git-annex:difference.log of the local repo,
+and abort. git-annex sync (and the assistant) would check the same, but
+before merging master branches either, to avoid a bad merge there.
+
+The git-annex:difference.log of a local repo could be changed by an upgrade
+or some sort of transition. When this happens, the new value is written
+for the uuid of the local repo. git-annex merge would then refuse to merge
+with remote repos until they were also transitioned.
+
+(There's perhaps some overlap here with the existing
+git-annex:transitions.log, however the current transitions involve
+forgetting history/dead remotes and so can be done repeatedly on a
+repository. Also, the current transitions can be performed on remote
+branches before merging them in; that wouldn't work well for version
+changes since those require other changes in the remote repo.)
+
+Not covered:
+
+* git-merge of other branches, such as master (can be fixed by `git annex
+  fix` or `fsck`)
+* Old versions of git-annex will ignore the version file of course,
+  and so merging such repos using them can result in pain.
+
diff --git a/doc/design/new_repo_versions/comment_1_b7b4819211910556838ec37bc2b6b37b._comment b/doc/design/new_repo_versions/comment_1_b7b4819211910556838ec37bc2b6b37b._comment
new file mode 100644
index 0000000000..9077eada90
--- /dev/null
+++ b/doc/design/new_repo_versions/comment_1_b7b4819211910556838ec37bc2b6b37b._comment
@@ -0,0 +1,14 @@
+[[!comment format=mdwn
+ username="https://www.google.com/accounts/o8/id?id=AItOawl9sYlePmv1xK-VvjBdN-5doOa_Xw-jH4U"
+ nickname="Richard"
+ subject="comment 1"
+ date="2015-01-22T01:05:56Z"
+ content="""
+My gut feeling for incompatible changes in v6 is to somehow make v5 and below error out on all merges with a v6 repo.
+
+Conversely, v6 git-annex running in v6 repos would transition all data while merging in v5 repos.
+
+As a migration path for existing repos, you could move `git-annex` and `master` to `git-annex_v5` and `master_v5`. Once you are certain all known repos within that repo group are transitioned, either kill the v5 branches automagically or print info on cli/web UI on what needs to be run to get rid of them.
+
+Having several plain git 1+ GiB repos (i.e. without any annex objects), I would love to transition after v6 and the migration paths have been hung a month or three to dry.
+"""]]
diff --git a/doc/design/new_repo_versions/comment_2_8f35254d2cd5d0c273d5392ddd857c2d._comment b/doc/design/new_repo_versions/comment_2_8f35254d2cd5d0c273d5392ddd857c2d._comment
new file mode 100644
index 0000000000..ef050ac937
--- /dev/null
+++ b/doc/design/new_repo_versions/comment_2_8f35254d2cd5d0c273d5392ddd857c2d._comment
@@ -0,0 +1,24 @@
+[[!comment format=mdwn
+ username="https://www.google.com/accounts/o8/id?id=AItOawnx8kHW66N3BqmkVpgtXDlYMvr8TJ5VvfY"
+ nickname="Yaroslav"
+ subject="comment 2"
+ date="2015-01-26T19:46:12Z"
+ content="""
+\"... merging repo A(v5) into repo B(v6)? This seems like it would be all to easy for a user to do.\"
+
+it would be easy if two separate repos are inited separately of different versions and then merged.  But how frequent could such situation arise?  I  don't think that it is not that common actually in non git-annex assistant initialized  repositories, especially if \"v6\" (or some other custom) layout would be non-default.
+
+This also somewhat reflects upon \"When it's receiving an object into a mixed v5 and v6 repo, it can't know which location that repo expects the object file to be located in.\" -- how plausible/frequent would be to see a mixed v5/6 repo?
+
+\".annex-version flag file\" -- I think it is a good idea.  (may be even better an .annex/version directory/file to reserve for possibly future other branch-specific, thus not for git-annex branch additions).  Indeed it would take time for repos to acquire it, but imho it is ok, if added automagically by new git-annex versions.  Also it is a tiny price to pay for users who do not really care about \"living on the edge\".    But it also might better be coupled with having git-annex:version (i.e. version file under git-annex branch) describing layout of the git-annex branch since those two are somewhat independent, right?
+
+Also a wild idea to mitigate inconvenience for users happen they migrate repositories from old to new formats: to provide 'git annex compat-layout  [version]' command which, for files with content, would  generate symlinks to files in new format to locations in old, with some command to clean them up  later on (e.g. 'git annex compat-layout --cleanup'). Although inefficient per-se it could be a big convenience since would eliminate need to \"migrate/rewrite\" entire history, while still making it possible to get access to previous versions.  With a handling in hooks could be automated to even hide that from users entirely.  or is there a big culprit I don't see?
+
+
+\"version numbers vs configuration\"
+
+having flexibility of configurations, possibly with versions simply providing enumerations to some of them, might be a great feature to have in the long run.  Not sure though if it would not blow up complexity :-/
+
+Overall -- if complete transition to improved/unified v6 is too big of undertaker, allowing for custom non-default configurations while sticking to v5 as the default one and forbidding merges between different versions, could be sufficient to scale up for atypical large uses.  If everyone to migrate to v6 -- more optimal layout should be well thought through to be worthwhile undertaking.
+
+"""]]
diff --git a/doc/design/p2p_protocol.mdwn b/doc/design/p2p_protocol.mdwn
new file mode 100644
index 0000000000..0890c8269c
--- /dev/null
+++ b/doc/design/p2p_protocol.mdwn
@@ -0,0 +1,208 @@
+The git-annex P2P protocol is a custom protocol that git-annex uses to
+communicate between peers.
+
+There's a common line-based serialization of the protocol, but other
+serializations are also possible. The line-based serialization is spoken
+by [[git-annex-shell], and by git-annex over tor.
+
+One peer is known as the client, and is the peer that initiates the
+connection and sends commands. The other peer is known as the server, and
+is the peer that the client connects to. It's possible for two connections
+to be run at the same time between the same two peers, in different
+directions.
+
+## Errors
+
+Either the client or the server may send an error message at any
+time. 
+
+When the client sends an ERROR, the server will close the connection.
+
+If the server sends an ERROR in response to the client's
+request, the connection will remain open, and the client can make
+another request.
+
+	ERROR this repository is read-only; write access denied
+
+## Authentication
+
+The protocol genernally starts with authentication. However, if
+authentication already occurs on another layer, as is the case with
+git-annex-shell, authentication will be skipped.
+
+The client starts by sending an authentication command to the server,
+along with its UUID. The AuthToken is some arbitrary token that has been
+agreed upon beforehand.
+
+	AUTH UUID AuthToken
+
+The server responds with either its own UUID when authentication
+is successful. Or, it can fail the authentication, and close the
+connection.
+
+	AUTH_SUCCESS UUID
+	AUTH_FAILURE
+
+Note that authentication does not guarantee that the client is talking to
+who they expect to be talking to. This, and encryption of the connection,
+are handled at a lower level.
+
+## Protocol version
+
+The default protocol version is 0. The client can choose to
+negotiate a new version with the server. This must come after
+any authentication.
+
+The client sends the highest protocol version it supports:
+
+	VERSION 2
+
+The server responds with the highest protocol version it supports
+that is less than or equal to the version the client sent:
+
+	VERSION 1
+
+Now both client and server should use version 1.
+
+## Binary data
+
+The protocol allows raw binary data to be sent. This is done
+using a DATA message. In the line-based serialization, this comes
+on its own line, followed by a newline and the binary data.
+The Len value tells how many bytes of data to read.
+
+	DATA 3
+	foo1
+
+Note that there is no newline after the binary data; the next protocol
+message will come immediately after it.
+
+If the sender finds itself unable to send as many bytes of data as it
+promised (perhaps because a file got truncated while it was being sent),
+its only option is to close the protocol connection.
+
+## Checking if content is present
+
+To check if a key is currently present on the server, the client sends:
+
+	CHECKPRESENT Key
+
+The server responds with either SUCCESS or FAILURE.
+
+## Locking content
+
+To lock content on the server, preventing it from being removed,
+the client sends:
+
+	LOCKCONTENT Key
+
+The server responds with either SUCCESS or FAILURE.
+The former indicates the content is locked. It will remain
+locked until the connection is broken, or the client
+sends:
+
+	UNLOCKCONTENT Key
+
+The server makes no response to that.
+
+## Removing content
+
+To remove a key's content from the server, the client sends:
+
+	REMOVE Key
+
+The server responds with either SUCCESS or FAILURE.
+
+## Storing content on the server
+
+To store content on the server, the client sends:
+
+	PUT AssociatedFile Key
+
+Here AssociatedFile may be the name of a file in the git
+repository, for information purposes only. Or it can be the
+empty string. It will always have unix directory separators.
+
+(Note that in the line-based serialization. AssociatedFile may not contain any
+spaces, since it's not the last token in the line. Use '%' to indicate
+whitespace.)
+
+The server may respond with ALREADY-HAVE if it already
+had the conent of that key. Otherwise, it responds with:
+
+	PUT-FROM Offset
+
+Offset is the number of bytes into the file that the server wants
+the client to start. This allows resuming transfers.
+
+The client then sends a DATA message with content of the file from
+the offset to the end of file.
+
+In protocol version 1, after the data, the client sends an additional
+message, to indicate if the content of the file has changed while it
+was being sent.
+
+	INVALID
+	VALID
+
+If the server successfully receives the data and stores the content,
+it replies with SUCCESS. Otherwise, FAILURE.
+
+## Getting content from the server
+
+To get content from the server, the client sends:
+
+	GET Offset AssociatedFile Key
+
+The Offset is the number of bytes into the file that the client wants
+the server to skip, which allows resuming transfers.
+See description of AssociatedFile above.
+
+The server then sends a DATA message with the content of the file
+from the offset to end of file.
+
+In protocol version 1, after the data, the server sends an additional
+message, to indicate if the content of the file has changed while it
+was being sent.
+
+	INVALID
+	VALID
+
+The client replies with SUCCESS or FAILURE.
+
+## Connection to services
+
+This is used to connect to services like git-upload-pack and
+git-receive-pack that speak their own protocol.
+
+The client sends a message to request the connection.
+Service is the name of the service, eg "git-upload-pack".
+
+	CONNECT Service
+
+Both client and server may now exchange DATA messages in any order,
+encapsulating the service's protocol.
+
+When the service exits, the server indicates this by telling the client
+its exit code.
+
+	CONNECTDONE ExitCode
+
+## Change notification
+
+The client can request to be notified when a ref in 
+the git repository on the server changes.
+
+	NOTIFYCHANGE
+
+The server will block until at least
+one of the refs changes, and send a list of changed
+refs.
+
+	CHANGED ChangedRefs
+
+For example:
+
+	CHANGED refs/heads/master refs/heads/git-annex
+
+Some servers may not support this command.
diff --git a/doc/design/p2p_protocol/comment_1_367bebae5b84408c7592b5fcca993c1e._comment b/doc/design/p2p_protocol/comment_1_367bebae5b84408c7592b5fcca993c1e._comment
new file mode 100644
index 0000000000..cb34fb951a
--- /dev/null
+++ b/doc/design/p2p_protocol/comment_1_367bebae5b84408c7592b5fcca993c1e._comment
@@ -0,0 +1,30 @@
+[[!comment format=mdwn
+ username="anarcat"
+ avatar="http://cdn.libravatar.org/avatar/4ad594c1e13211c1ad9edb81ce5110b7"
+ subject="late to the party"
+ date="2018-03-12T13:50:33Z"
+ content="""
+eh... i look elsewhere for a week and you design another line
+protocol! ;) so I guess it's too late to do anything to change this,
+but I wanted to share that similar efforts are being done over the
+backup software world, in particular in [restic][],
+which is working with the [rclone][] project to implement an abstract
+get/pull mechanism to store blobs, a lot like what git-annex needs to
+be doing.
+
+they wrote this using a binary protocol for speed (it's basically RPC
+at this point) and I encouraged them to at least use a standard one
+(they use protobufs + HTTP2 AKA gRPC, iirc, and it works over
+stdin/out). you might find the  [full thread][] interesting... it
+would be great if git-annex would support this natively instead of
+rolling its own protocol, because it would mean it could talk with
+other services like rclone or restic servers out of the box, without
+*those* endpoints having to implement yet another custom protocol.
+
+but yeah, i'm way too late it seems. figured you might find it
+interesting anyways... congrats on the performance improvements!
+
+[restic]: https://restic.net
+[rclone]: https://rclone.org/
+[full thread]: https://github.com/restic/restic/issues/1561
+"""]]
diff --git a/doc/design/p2p_protocol/comment_2_dbd118a01c31d17a4223a32a7baa8e39._comment b/doc/design/p2p_protocol/comment_2_dbd118a01c31d17a4223a32a7baa8e39._comment
new file mode 100644
index 0000000000..4eeee45a25
--- /dev/null
+++ b/doc/design/p2p_protocol/comment_2_dbd118a01c31d17a4223a32a7baa8e39._comment
@@ -0,0 +1,15 @@
+[[!comment format=mdwn
+ username="joey"
+ subject="""comment 2"""
+ date="2018-03-12T21:59:26Z"
+ content="""
+This is not a new protocol, it was added 2 years ago for
+git-annex's tor hidden service support.
+
+git-annex already supports rclone via a special remote.
+
+This protocol is only used when there's a git-annex repository
+on the other end, so I don't see any benefit in making it compatible with
+any other programs. And it has stuff in it that I'm sure if not in the
+protocol you talked about, like LOCKCONTENT.
+"""]]
diff --git a/doc/design/preferred_content.mdwn b/doc/design/preferred_content.mdwn
new file mode 100644
index 0000000000..3972b8b583
--- /dev/null
+++ b/doc/design/preferred_content.mdwn
@@ -0,0 +1,21 @@
+The [[preferred_content]] expressions didn't have a design document, but
+it's a small non-turing complete DSL for expressing which objects a
+repository prefers to contain.
+
+One thing that needs to be written down though is the stability analysis
+that must be done of preferred content expressions. 
+
+It's important that when a set of repositories all look at one-another's
+preferred content expressions, and copy/move/drop objects to satisfy them,
+they end up at a steady state. So, a given preferred content expression
+should ideally evaluate to the same answer for each key, from the
+perspective of each repository.
+
+The best way to ensure that is the case is to only use terms in preferred
+content expressions that rely on state that is shared between all
+repositories. So, state in the git-annex branch, or the master branch
+(assuming all repositories have master checked out).
+
+Since git is eventually consistent, there might be disagreements about
+which object belongs where, but once consistency is reached, things will
+settle down.
diff --git a/doc/design/requests_routing.mdwn b/doc/design/requests_routing.mdwn
new file mode 100644
index 0000000000..2391cfae95
--- /dev/null
+++ b/doc/design/requests_routing.mdwn
@@ -0,0 +1,100 @@
+## requesting content
+ 
+In some situations, nodes only want particular files, and not everything.
+(Or don't have the bandwidth to get everything.) A way to handle this,
+that should work in a fully ad-hoc, offline distributed network,
+suggested by Vincenzo Tozzi:
+
+* Nodes generate a request for a specific file they want, committed
+  to git somewhere.
+* This request has a TTL (of eg 3 or 4).
+* When syncing, copy the requests that a node has, and decrease their TTL
+  by 1. Requests with a TTL of 0 have timed out and are not copied.
+  (So, requests are stored in git, but on eg, per-node branches.)
+* Only copy content to nodes that have a request for it (either one
+  originating with them, or one they copied from another node).
+* Each request indicates the requesting node, so once no nodes have an
+  active request for a particular file, it's ok to drop it from the
+  transfer nodes (honoring numcopies etc of course).
+
+## simulation
+
+A simulation of a network using this method is in [[simroutes.hs]].
+
+Question: How efficient is this method? Does the network fill with many
+copies that are not needed, before the request is fulfilled?
+
+## storing requests
+
+Requests could be stored in the location tracking file.
+
+Currently:
+
+	time 0|1 uuid1
+	time 0|1 uuid2
+
+* Use negative numbers for the TTL of a request:
+
+	time -3! uuid1
+	time -2 uuid2
+
+  The `!` indicates that the request originated on
+  that node.
+* To propigate a request, set -1 * (TTL+1) in the line
+  for the uuid of the repository that is propigating it.  
+  This should be done as part of the git-annex branch merge,
+  so if a location tracking file is merged, any open requests
+  get propigated to the current repository automatically.
+* When a requested file reaches a node that requested it,
+  the location is set to 1; this automatically clears the
+  request.
+* When a file has no more originating requests, clear all
+  the copied requests:
+
+	time 1 uuid1
+	time -2 uuid2
+
+  Becomes:
+
+	time 1 uuid1
+	time' 0 uuid2
+
+## generating requests
+
+	git annex request [file...]
+
+Indicates that the file is wanted in the current repository.
+
+(git annex get could also do this on failure, or suggest doing this)
+
+## acting on requests
+
+Add a preferred content expression that looks at request data:
+
+	requestedby=N
+
+Matches files that have been requested by at least N nodes.
+
+	requested
+
+Matches files that the current node has requested.
+
+### Example preferred content expressions
+
+For an immobile node that accumulates files it requests, and also
+temporarily stores files requested by other such nodes:
+
+	present or requestedby=1
+
+For a node that only transfers files between the immobile nodes:
+
+	requestedby=1
+
+For an immobile node that only accumulates files it requests, but never
+stores files requested by other nodes:
+
+	present or requested
+
+TODO: Would be nice to be able to prioritize files that more nodes are
+requesting, or that have some urgent flag set. But currently there is no
+way to do that; content is either preferred or not preferred.
diff --git a/doc/design/requests_routing/comment_1_421b14a4dd9d6c431e00333057df1627._comment b/doc/design/requests_routing/comment_1_421b14a4dd9d6c431e00333057df1627._comment
new file mode 100644
index 0000000000..f2e0a857a2
--- /dev/null
+++ b/doc/design/requests_routing/comment_1_421b14a4dd9d6c431e00333057df1627._comment
@@ -0,0 +1,10 @@
+[[!comment format=mdwn
+ username="http://johan.kiviniemi.name/"
+ nickname="Johan"
+ subject="comment 1"
+ date="2014-05-07T06:27:10Z"
+ content="""
+“Nodes generate a request for a specific file they want”
+
+It would be also convenient to be able to “push” a file to another node you do not have direct access to by creating a request for that file on behalf of the node.
+"""]]
diff --git a/doc/design/requests_routing/comment_2_a409df0566b308bf2b84efd4fb2c8a8f._comment b/doc/design/requests_routing/comment_2_a409df0566b308bf2b84efd4fb2c8a8f._comment
new file mode 100644
index 0000000000..d7bbaed085
--- /dev/null
+++ b/doc/design/requests_routing/comment_2_a409df0566b308bf2b84efd4fb2c8a8f._comment
@@ -0,0 +1,7 @@
+[[!comment format=mdwn
+ username="https://id.koumbit.net/anarcat"
+ subject="design phase only"
+ date="2015-08-18T19:02:30Z"
+ content="""
+just to clarify: this is not implemented yet, as far as i know.
+"""]]
diff --git a/doc/design/requests_routing/simroutes.hs b/doc/design/requests_routing/simroutes.hs
new file mode 100644
index 0000000000..3918160405
--- /dev/null
+++ b/doc/design/requests_routing/simroutes.hs
@@ -0,0 +1,402 @@
+-- Simulation of non-flood syncing of content, across a network of nodes.
+
+module Main where
+
+import System.Random
+import Control.Monad.Random
+import Control.Monad
+import Control.Applicative
+import Data.Ratio
+import Data.Ord
+import Data.List
+import Data.Maybe
+import qualified Data.Set as S
+import qualified Data.Map.Strict as M
+
+{-
+ - Tunable values
+ -}
+
+totalFiles :: Int
+totalFiles = 100
+
+-- How likely is a given file to be wanted by any particular node?
+probabilityFilesWanted :: Probability
+probabilityFilesWanted = 0.10
+
+-- How many different locations can each transfer node move between?
+-- (Min, Max)
+transferDestinationsRange :: (Int, Int)
+transferDestinationsRange = (2, 3)
+
+-- Controls how likely transfer nodes are to move around in a given step
+-- of the simulation.
+-- (They actually move slightly less because they may start to move and
+-- pick the same location they are at.)
+-- (Min, Max)
+transferMoveFrequencyRange :: (Probability, Probability)
+transferMoveFrequencyRange = (0.10, 1.00)
+
+-- counts both immobile and transfer nodes as hops, so double Vince's
+-- theoretical TTL of 3.
+-- (30% loss on mocambos network w/o ttl of 4!)
+maxTTL :: TTL
+maxTTL = TTL (4 * 2)
+
+numImmobileNodes :: Int
+numImmobileNodes = 10
+
+numTransferNodes :: Int
+numTransferNodes = 20
+
+numSteps :: Int
+numSteps = 100
+
+-- IO code
+main :: IO ()
+main = do
+--	initialnetwork <- evalRandIO (seedFiles totalFiles =<< genNetwork)
+	initialnetwork <- evalRandIO (seedFiles totalFiles =<< mocambosNetwork)
+	networks <- evalRandIO (simulate numSteps initialnetwork)
+	let finalnetwork = last networks
+	putStrLn $ summarize initialnetwork finalnetwork
+	putStrLn "location history of file 1:"
+	print $ trace (traceHaveFile (File 1)) networks
+	putStrLn "request history of file 1:"
+	print $ trace (traceWantFile (File 1)) networks
+-- Only pure code below :)
+
+data Network = Network (M.Map NodeName ImmobileNode) [TransferNode]
+	deriving (Show, Eq)
+
+data ImmobileNode = ImmobileNode NodeRepo
+	deriving (Show, Eq)
+
+type NodeName = String
+
+type Route = [NodeName]
+
+data TransferNode = TransferNode
+	{ currentlocation :: NodeName
+	, possiblelocations :: [NodeName]
+	, movefrequency :: Probability
+	, transferrepo :: NodeRepo
+	}
+	deriving (Show, Eq)
+
+data NodeRepo = NodeRepo
+	{ wantFiles :: [Request]
+	, haveFiles :: S.Set File
+	, satisfiedRequests :: S.Set Request
+	}
+	deriving (Show, Eq)
+
+data File = File Int
+	deriving (Show, Eq, Ord)
+
+randomFile :: (RandomGen g) => Rand g File
+randomFile = File <$> getRandomR (0, totalFiles)
+
+data Request = Request File TTL
+	deriving (Show, Ord)
+
+-- compare ignoring TTL
+instance Eq Request where
+	(Request f1 _) == (Request f2 _) = f1 == f2
+
+requestedFile :: Request -> File
+requestedFile (Request f _) = f
+
+requestTTL :: Request -> TTL
+requestTTL (Request _ ttl) = ttl
+
+data TTL = TTL Int
+	deriving (Show, Eq, Ord)
+
+incTTL :: TTL -> TTL
+incTTL (TTL t) = TTL (t + 1)
+
+decTTL :: TTL -> TTL
+decTTL (TTL t) = TTL (t - 1)
+
+staleTTL :: TTL -> Bool
+staleTTL (TTL t) = t < 1
+
+-- Origin of a request starts one higher than max, since the TTL
+-- will decrement the first time the Request is transferred to another node.
+originTTL :: TTL
+originTTL = incTTL maxTTL
+
+randomRequest :: (RandomGen g) => Rand g Request
+randomRequest = Request
+	<$> randomFile
+	<*> pure originTTL
+
+type Probability = Float
+
+randomProbability :: (RandomGen g) => Rand g Probability
+randomProbability = getRandomR (0, 1)
+
+-- Returns the state of the network at each step of the simulation.
+simulate :: (RandomGen g) => Int -> Network -> Rand g [Network]
+simulate n net = go n [net]
+  where
+	go 0 nets = return (reverse nets)
+	go c (prev:nets) = do
+		new <- step prev
+		go (c - 1) (new:prev:nets)
+
+-- Each step of the simulation, check if each TransferNode wants to move,
+-- and if so:
+--   1. It and its current location exchange their Requests.
+--   2. And they exchange any requested files.
+--   3. Move it to a new random location.
+--
+-- Note: This implementation does not exchange requests between two
+-- TransferNodes that both arrive at the same location at the same step,
+-- and then move away in the next step.
+step :: (RandomGen g) => Network -> Rand g Network
+step (Network immobiles transfers) = go immobiles [] transfers
+  where
+	go is c [] = return (Network is c)
+	go is c (t:ts) = do
+		r <- randomProbability
+		if movefrequency t <= r
+			then case M.lookup (currentlocation t) is of
+				Nothing -> go is (c ++ [t]) ts
+				Just currentloc -> do
+					let (currentloc', t') = merge currentloc t
+					t'' <- move t'
+					go (M.insert (currentlocation t) currentloc' is) (c ++ [t'']) ts
+			else go is (c ++ [t]) ts
+
+merge :: ImmobileNode -> TransferNode -> (ImmobileNode, TransferNode)
+merge (ImmobileNode ir) t@(TransferNode { transferrepo = tr }) =
+	( ImmobileNode (go ir tr)
+	, t { transferrepo = go tr ir }
+	)
+  where
+	go r1 r2 = r1
+		{ wantFiles = wantFiles'
+		, haveFiles = haveFiles'
+		, satisfiedRequests = satisfiedRequests' `S.union` checkSatisfied wantFiles' haveFiles'
+		}
+	  where
+		wantFiles' = foldr addRequest (wantFiles r1) (wantFiles r2)
+		haveFiles' = S.foldr (addFile wantFiles' satisfiedRequests') (haveFiles r1) (haveFiles r2)
+		satisfiedRequests' = satisfiedRequests r1 `S.union` satisfiedRequests r2
+
+-- Adds a file to the set, when there's a request for it, and the request
+-- has not already been satisfied.
+addFile :: [Request] -> S.Set Request -> File -> S.Set File -> S.Set File
+addFile rs srs f fs
+	| any (\sr -> f == requestedFile sr) (S.toList srs) = fs
+	| any (\r -> f == requestedFile r) rs = S.insert f fs
+	| otherwise = fs
+
+-- Checks if any requests have been satisfied, and returns them,
+-- to be added to satisfidRequests
+checkSatisfied :: [Request] -> S.Set File -> S.Set Request
+checkSatisfied want have = S.fromList (filter satisfied want)
+  where
+	satisfied r = requestTTL r == originTTL && S.member (requestedFile r) have
+
+-- Decrements TTL, and avoids adding request with a stale TTL, or a
+-- request for an already added file with the same or a lower TTL.
+addRequest :: Request -> [Request] -> [Request]
+addRequest (Request f ttl) rs
+	| staleTTL ttl' = rs
+	| any (\r -> requestTTL r >= ttl) similar = rs
+	| otherwise = r' : other
+  where
+	ttl' = decTTL ttl
+	r' = Request f ttl'
+	(other, similar) = partition (/= r') rs
+
+move :: (RandomGen g) => TransferNode -> Rand g TransferNode
+move t = do
+	newloc <- randomfrom (possiblelocations t)
+	return $ t { currentlocation = newloc }
+
+genNetwork :: (RandomGen g) => Rand g Network
+genNetwork = do
+	let immobiles = M.fromList (zip (map show [1..]) (replicate numImmobileNodes emptyImmobile))
+	transfers <- sequence (replicate numTransferNodes (mkTransfer $ M.keys immobiles))
+	return $ Network immobiles transfers
+
+emptyImmobile :: ImmobileNode
+emptyImmobile = ImmobileNode (NodeRepo [] S.empty S.empty)
+
+mkTransfer :: (RandomGen g) => [NodeName] -> Rand g TransferNode
+mkTransfer immobiles = do
+	-- Transfer nodes are given random routes. May be simplistic.
+	-- Also, some immobile nodes will not be serviced by any transfer nodes.
+	numpossiblelocs <- getRandomR transferDestinationsRange
+	possiblelocs <- sequence (replicate numpossiblelocs (randomfrom immobiles))
+	mkTransferBetween possiblelocs
+
+mkTransferBetween :: (RandomGen g) => [NodeName] -> Rand g TransferNode
+mkTransferBetween possiblelocs = do
+	currentloc <- randomfrom possiblelocs
+	movefreq <- getRandomR transferMoveFrequencyRange
+	-- transfer nodes start out with no files or requests in their repo
+	let repo = (NodeRepo [] S.empty S.empty)
+	return $ TransferNode currentloc possiblelocs movefreq repo
+
+randomfrom :: (RandomGen g) => [a] -> Rand g a
+randomfrom l = do
+	i <- getRandomR (1, length l)
+	return $ l !! (i - 1)
+
+-- Seeds the network with the given number of files. Each file is added to
+-- one of the immobile nodes of the network at random. And, one other node,
+-- at random, is selected which wants to get the file.
+seedFiles :: (RandomGen g) => Int -> Network -> Rand g Network
+seedFiles 0 network = return network
+seedFiles n network@(Network m t) = do
+	(origink, ImmobileNode originr) <- randnode
+	(destinationk, ImmobileNode destinationr) <- randnode
+	let file = File n
+	let origin = ImmobileNode $ originr
+		{ haveFiles = S.insert file (haveFiles originr) }
+	let destination = ImmobileNode $ destinationr
+		{ wantFiles = Request file originTTL : wantFiles destinationr }
+	let m' = M.insert origink origin $
+		M.insert destinationk destination m
+	seedFiles (n - 1) (Network m' t)
+  where
+	randnode = do
+		k <- randomfrom (M.keys m)
+		return (k, fromJust $ M.lookup k m)
+
+summarize :: Network -> Network -> String
+summarize _initial@(Network origis _) _final@(Network is _ts) = format
+	[ ("Total wanted files",
+		show (sum (overis (length . findoriginreqs . wantFiles . repo))))
+	, ("Wanted files that were not transferred to requesting node",
+		show (sum (overis (S.size . findunsatisfied . repo))))
+	, ("Nodes that failed to get files",
+		show (map withinitiallocs $ filter (not . S.null . snd)
+			(M.toList $ M.map (findunsatisfied . repo) is)))
+	, ("Total number of files on immobile nodes at end",
+		show (overis (S.size . haveFiles . repo)))
+	--, ("Immobile nodes at end", show is)
+	]
+  where
+	findoriginreqs = filter (\r -> requestTTL r == originTTL)
+	findunsatisfied r = 
+		let wantedfs = S.fromList $ map requestedFile (findoriginreqs (wantFiles r))
+		in S.difference wantedfs (haveFiles r)
+	repo (ImmobileNode r) = r
+	overis f = map f $ M.elems is
+	format = unlines . map (\(d, s) -> d ++ ": " ++ s)
+
+	withinitiallocs (name, missingfiles) = (name, S.map addinitialloc missingfiles)
+	addinitialloc f = (f, M.lookup f initiallocs)
+
+	initiallocs = M.fromList $ 
+		concatMap (\(k, v) -> map (\f -> (f, k)) (S.toList $ haveFiles $ repo v)) $
+			M.toList origis
+
+trace :: (Network -> S.Set NodeName) -> [Network] -> String
+trace tracer networks = show $ go [] S.empty $ map tracer networks
+  where
+	go c old [] = reverse c
+	go c old (new:l) = go ((S.toList $ new `S.difference` old):c) new l
+
+traceHaveFile :: File -> Network -> S.Set NodeName
+traceHaveFile f (Network m _) = S.fromList $ M.keys $
+	M.filter (\(ImmobileNode r) -> f `S.member` haveFiles r) m 
+
+traceWantFile :: File -> Network -> S.Set NodeName
+traceWantFile f (Network m _) = S.fromList $ M.keys $
+	M.filter (\(ImmobileNode r) -> any wantf (wantFiles r)) m 
+  where
+	wantf (Request rf _ttl) = rf == f
+
+mocambosNetwork :: (RandomGen g) => Rand g Network
+mocambosNetwork = do
+	let major = map (immobilenamed . fst) communities
+	let minor = map immobilenamed (concatMap snd communities)
+	majortransfer <- mapM mkTransferBetween majorroutes
+	minortransfer <- mapM mkTransferBetween (concatMap minorroutes (concat (replicate 5 communities)))
+	return $ Network
+		(M.fromList (major++minor))
+		(majortransfer ++ minortransfer)
+  where
+	immobilenamed name = (name, emptyImmobile)
+
+	-- As a simplification, this only makes 2 hop routes, between minor
+	-- and major communities; no 3-legged routes.
+	minorroutes :: (NodeName, [NodeName]) -> [Route]
+	minorroutes (major, minors) = map (\n -> [major, n]) minors
+
+communities :: [(NodeName, [NodeName])]
+communities =
+	[ ("Tainá/SP",
+		[ "badtas"
+		, "vauedo ribera"
+		, "cofundo"
+		, "jao"
+		, "fazenda"
+		]
+	  )
+	, ("Odomode/RS",
+		[ "moradadapaz"
+		, "pelotas" 
+		]
+	  )
+	, ("MercadoSul/DF",
+		[ "mesquito"
+		, "kalungos"
+		]
+	  )
+	, ("Coco/PE",
+		[ "xambá"
+		, "alafin"
+		, "terreiaos"
+		]
+	  )
+	, ("Linharinho/ES",
+		[ "monte alegne"
+		]
+	  )
+	, ("Boneco/BA",
+		[ "barroso"
+		, "lagoa santa"
+		, "terravista"
+		]
+	  )
+	, ("Zumbidospalmanes/NA",
+		[ "allantana"
+		]
+	  )
+	, ("Casa Pneta/PA",
+		[ "marajó"
+		]
+	  )
+	, ("Purarue/PA",
+		[ "oriaminá"
+		]
+	  )
+	, ("Madiba/NET", [])
+	]
+
+majorroutes :: [Route]
+majorroutes =
+	-- person's routes
+	[ ["Tainá/SP", "Odomode/RS"]
+	, ["Tainá/SP", "MercadoSul/DF"]
+	, ["MercadoSul/DF", "Boneco/BA"]
+	, ["MercadoSul/DF", "Zumbidospalmanes/NA"]
+	, ["Zumbidospalmanes/NA", "Casa Pneta/PA"]
+	, ["Casa Pneta/PA", "Purarue/PA"]
+	, ["Casa Pneta/PA", "Linharinho/ES"]
+	, ["Boneco/BA", "Coco/PE"]
+	-- internet connections
+	, ["Tainá/SP", "MercadoSul/DF", "Coco/PE", "Purarue/PA", "Odomode/RS", "Madiba/NET"]
+	, ["Tainá/SP", "MercadoSul/DF", "Coco/PE", "Purarue/PA", "Odomode/RS", "Madiba/NET"]
+	, ["Tainá/SP", "MercadoSul/DF", "Coco/PE", "Purarue/PA", "Odomode/RS", "Madiba/NET"]
+	, ["Tainá/SP", "MercadoSul/DF", "Coco/PE", "Purarue/PA", "Odomode/RS", "Madiba/NET"]
+	, ["Tainá/SP", "MercadoSul/DF", "Coco/PE", "Purarue/PA", "Odomode/RS", "Madiba/NET"]
+	]
diff --git a/doc/design/roadmap.mdwn b/doc/design/roadmap.mdwn
new file mode 100644
index 0000000000..5e636f33b9
--- /dev/null
+++ b/doc/design/roadmap.mdwn
@@ -0,0 +1,38 @@
+## ahead
+
+* [[design/caching_database]] for metadata views
+* [[assistant/deltas]]
+* [[assistant/gpgkeys]] management for the assistant
+* [[design/requests_routing]]
+* [[design/new_repo_versions]]
+
+## the rearview
+
+* Spring 2016 [[adjusted_branches]]
+* Winter 2015 v6 repo format
+* Mar-Oct 2015 busy busy busy, too busy to update!
+* Feb 2015 user-driven features and polishing, [[design/caching_database]] part 1
+* Jan 2015 Android 5, relative paths, workload [[tuning]]
+* Dec 2014 [[todo/extensible_addurl]], bittorrent special remote
+* Nov 2014 direct mode proxy, undo command, diffdriver
+* Oct 2014 user-driven features and polishing, S3 multipart
+* Sep 2014 vacation
+
+2013-2014 [crowdfunded](https://campaign.joeyh.name/) year
+
+* Aug 2014 user-driven features and polishing
+* Jul 2014 [[!traillink assistant/chunks]]
+* Jun 2014 polish [[assistant/Windows]] port
+* May 2014 Brazil!, [[!traillink assistant/sshpassword]]
+* Apr 2014 [[!traillink git-remote-daemon]]
+* Mar 2014 user-driven features and polishing
+* Feb 2014 get Windows out of beta, [[!traillink design/metadata text="metadata and views"]]
+* Jan 2014 user-driven features and polishing
+* Dec 2013 [[!traillink assistant/windows text="Windows webapp"]], Linux arm, external special remotes
+* Nov 2013 [[!traillink direct_mode]] guard [[!traillink assistant/upgrading]]
+* Oct 2013 [[!traillink assistant/disaster_recovery]]
+* Sep 2013 [[!traillink assistant/encrypted_git_remotes]]
+
+2012-2013 kickstarted year developing [[git-annex assistant|assistant]]
+
+2010-2011 initial git-annex development
diff --git a/doc/devblog.mdwn b/doc/devblog.mdwn
new file mode 100644
index 0000000000..73bc6dd700
--- /dev/null
+++ b/doc/devblog.mdwn
@@ -0,0 +1,9 @@
+Joey blogs about his work here on a semi-daily basis.
+
+[[!sidebar content="""
+[[!calendar type="month" pages="page(devblog/*)"]]
+[[!calendar type="month" month="-1" pages="page(devblog/*)"]]
+[[!calendar type="month" month="-2" pages="page(devblog/*)"]]
+"""]]
+
+[[!inline pages="page(devblog/*)" show=0]]
diff --git a/doc/devblog/day_-1__drop_dead.mdwn b/doc/devblog/day_-1__drop_dead.mdwn
new file mode 100644
index 0000000000..97f7cf1d28
--- /dev/null
+++ b/doc/devblog/day_-1__drop_dead.mdwn
@@ -0,0 +1,5 @@
+Implemented `git annex forget --drop-dead`, which is finally a way to
+remove all references to old repositories that you've marked as dead.
+
+I've still not merged in the `forget` branch, because I developed this
+while slightly ill, and have not tested it very well yet.
diff --git a/doc/devblog/day_-3__.mdwn b/doc/devblog/day_-3__.mdwn
new file mode 100644
index 0000000000..fa1473e06e
--- /dev/null
+++ b/doc/devblog/day_-3__.mdwn
@@ -0,0 +1,29 @@
+John Millikin came through and fixed that haskell-gnutls segfault
+on OSX that I developed a reproducible test case for the other day.
+It's a bit hard to test, since the bug doesn't always happen, but the
+fix is already deployed for Mountain Lion autobuilder.
+
+However, I then found another way to make haskell-gnutls segfault, more
+reliably on OSX, and even sometimes on Linux. Just entering the wrong XMPP
+password in the assistant can trigger this crash. Hopefully John will work
+his magic again.
+
+---
+
+Meanwhile, I fixed the sync-after-forget problem. Now sync always forces
+its push of the git-annex branch (as does the assistant). I considered but
+rejected having sync do the kind of uuid-tagged branch push that the
+assistant sometimes falls back to if it's failing to do a normal sync. It's
+ugly, but worse, it wouldn't work in the workflow where multiple clients
+are syncing to a central bare repository, because they'd not pull down the
+hidden uuid-tagged branches, and without the assistant running on the
+repository, nothing would ever merge their data into the git-annex branch.
+Forcing the push of synced/git-annex was easy, once I satisfied myself
+that it was always ok to do so.
+
+Also factored out a module that knows about all the different log files
+stored on the git-annex branch, which is all the support infrastructure
+that will be needed to make `git annex forget --drop-dead` work. Since this
+is basically a routing module, perhaps I'll get around to making it use
+a nice bidirectional routing library like
+[Zwaluw](http://hackage.haskell.org/package/Zwaluw) one day.
diff --git a/doc/devblog/day_-4__forgetting.mdwn b/doc/devblog/day_-4__forgetting.mdwn
new file mode 100644
index 0000000000..a37b32e33a
--- /dev/null
+++ b/doc/devblog/day_-4__forgetting.mdwn
@@ -0,0 +1,80 @@
+Yesterday I spent making a release, and shopping for a new laptop, since
+this one is dying. (Soon I'll be able to compile git-annex fast-ish! Yay!)
+And thinking about [[todo/wishlist:_dropping_git-annex_history]].
+
+Today, I added the `git annex forget` command. It's currently been lightly
+tested, seems to work, and is living in the `forget` branch until I gain
+confidence with it. It should be perfectly safe to use, even if it's buggy,
+because you can use `git reflog git-annex` to pull out and revert to an old
+version of your git-annex branch. So if you're been wanting this feature,
+please beta test!
+
+----
+
+I actually implemented something more generic than just forgetting git
+history. There's now a whole mechanism for git-annex doing distributed
+transitions of whatever sort is needed.
+
+There were several subtleties involved in distributed transitions:
+
+First is how to tell when a given transition has already been done on a
+branch. At first I was thinking that the transition log should include the
+sha of the first commit on the old branch that got rewritten. However, that
+would mean that after a single transition had been done, every git-annex
+branch merge would need to look up the first commit of the current branch,
+to see if it's done the transition yet. That's slow! Instead, transitions
+are logged with a timestamp, and as long as a branch contains a transition
+with the same timestamp, it's been done.
+
+A really tricky problem is what to do if the local repository has
+transitioned, but a remote has not, and changes keep being made to the
+remote. What it does so far is incorporate the changes from the remote into
+the index, and re-run the transition code over the whole thing to yeild a
+single new commit. This might not be very efficient (once I write the more
+full-featured transition code), but it lets the local repo keep up with
+what's going on in the remote, without directly merging with it (which
+would revert the transition). And once the remote repository has its
+git-annex upgraded to one that knows about transitions, it will finish up
+the transition on its side automatically, and the two branches will once
+again merge.
+
+Related to the previous problem, we don't want to keep trying to merge
+from a remote branch when it's not yet transitioned. So a blacklist is
+used, of untransitioned commits that have already been integrated.
+
+One really subtle thing is that when the user does a transition more
+complicated than `git annex forget`, like the `git annex forget --dead`
+that I need to implement to forget dead remotes, they're not just telling
+git-annex to forget whatever dead remotes it knows right now. They're
+actually telling git-annex to perform the transition one time on every
+existing clone of the repository, at some point in the future. Repositories
+with unfinished transitions could hang around for years, and at some future
+point when git-annex runs in the repository again, it would merge in the
+current state of the world, and re-do the transition. So you might tell it
+to forget dead remotes today, and then the very repository you ran that in
+later becomes dead, and a long-slumbering repo wakes up and forgets about
+the repo that started the whole process! I hope users don't find this
+massively confusing, but that's how the implementation works right now.
+
+----
+
+I think I have at least two more days of work to do to finish up this
+feature.
+
+* I still need to add some extra features like forgetting about dead remotes,
+  and forgetting about keys that are no longer present on any remote.
+
+* After `git annex forget`, `git annex sync`
+  will fail to push the synced/annex branch to remotes, since the branch
+  is no longer a fast-forward of the old one. I will probably fix this by
+  making `git annex sync` do a fallback push of a unique branch in this case,
+  like the assistant already does. Although I may need to adjust that code
+  to handle this case, too..
+
+* For some reason the automatic transitioning code triggers
+  a "(recovery from race)" commit. This is certainly a bug somewhere,
+  because you can't have a race with only 1 participant.
+
+----
+
+Today's work was sponsored by Richard Hartmann.
diff --git a/doc/devblog/day_-4__forgetting/comment_1_f3cc7a25af4c59fda3924c737a789419._comment b/doc/devblog/day_-4__forgetting/comment_1_f3cc7a25af4c59fda3924c737a789419._comment
new file mode 100644
index 0000000000..4c926c1afd
--- /dev/null
+++ b/doc/devblog/day_-4__forgetting/comment_1_f3cc7a25af4c59fda3924c737a789419._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="Sandra.Devil"
+ ip="77.172.73.184"
+ subject="New laptop"
+ date="2013-09-01T09:38:32Z"
+ content="""
+What is the new laptop you are going to use? Specs please :) 
+"""]]
diff --git a/doc/devblog/day_-4__forgetting/comment_2_e47476c80af02a2e9cf76c53fdbb8534._comment b/doc/devblog/day_-4__forgetting/comment_2_e47476c80af02a2e9cf76c53fdbb8534._comment
new file mode 100644
index 0000000000..db2438f97e
--- /dev/null
+++ b/doc/devblog/day_-4__forgetting/comment_2_e47476c80af02a2e9cf76c53fdbb8534._comment
@@ -0,0 +1,9 @@
+[[!comment format=mdwn
+ username="stp"
+ ip="188.193.207.34"
+ subject="Update one forgetting keys no longer present"
+ date="2014-02-17T23:21:49Z"
+ content="""
+I have some repos where due to some hiccups file versions (not in the working tree anymore) were lost and now they come up again and again when fsck is running.
+So I would be happy if I could make my repos forget these not available files via \"git annex forget $key\" and perhaps even have a better solution to show all objects with numcopies=0.
+"""]]
diff --git a/doc/devblog/day_-4__forgetting/comment_3_b57956a8ce372d620f21ea9a497e8013._comment b/doc/devblog/day_-4__forgetting/comment_3_b57956a8ce372d620f21ea9a497e8013._comment
new file mode 100644
index 0000000000..5c0bbc42b4
--- /dev/null
+++ b/doc/devblog/day_-4__forgetting/comment_3_b57956a8ce372d620f21ea9a497e8013._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="http://joeyh.name/"
+ ip="209.250.56.172"
+ subject="comment 3"
+ date="2014-02-20T18:54:48Z"
+ content="""
+@stp, It seems to me if you just delete the symlinks in your git repository that point to the lost files, `git annex fsck` will shut up.
+"""]]
diff --git a/doc/devblog/day_-4__forgetting/comment_4_812b630df01ac35254e3c4e677554e2b._comment b/doc/devblog/day_-4__forgetting/comment_4_812b630df01ac35254e3c4e677554e2b._comment
new file mode 100644
index 0000000000..187545fbc5
--- /dev/null
+++ b/doc/devblog/day_-4__forgetting/comment_4_812b630df01ac35254e3c4e677554e2b._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="stp"
+ ip="84.56.21.11"
+ subject="comment 4"
+ date="2014-02-20T21:07:41Z"
+ content="""
+Yeah true if I remove symlinks from the history (as I understand your suggestion) it would work. I just wanted to suggest that it could be something useful for the git annex forget function as it already cleans out old dead repos and other things.
+"""]]
diff --git a/doc/devblog/day_-4__forgetting/comment_5_a9670eca2aff9ad5f04412a8d8b6df6a._comment b/doc/devblog/day_-4__forgetting/comment_5_a9670eca2aff9ad5f04412a8d8b6df6a._comment
new file mode 100644
index 0000000000..d830225262
--- /dev/null
+++ b/doc/devblog/day_-4__forgetting/comment_5_a9670eca2aff9ad5f04412a8d8b6df6a._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="http://joeyh.name/"
+ ip="209.250.56.172"
+ subject="comment 5"
+ date="2014-02-20T21:14:21Z"
+ content="""
+You don't need to delete them from the history, just from the branch you're running `git annex fsck` in.
+"""]]
diff --git a/doc/devblog/day_-4__forgetting/comment_6_4f87e2ab119f3cd81266159f02952188._comment b/doc/devblog/day_-4__forgetting/comment_6_4f87e2ab119f3cd81266159f02952188._comment
new file mode 100644
index 0000000000..4ecbb1ae65
--- /dev/null
+++ b/doc/devblog/day_-4__forgetting/comment_6_4f87e2ab119f3cd81266159f02952188._comment
@@ -0,0 +1,9 @@
+[[!comment format=mdwn
+ username="stp"
+ ip="84.56.21.11"
+ subject="comment 6"
+ date="2014-02-20T22:19:12Z"
+ content="""
+As discussed on irc.
+Fsck --all does check more then the working tree and therefore for fsck to not complain this would be a worthy feature to be added. (git annex forget $key)
+"""]]
diff --git a/doc/devblog/day_-4__forgetting/comment_7_a865216046aa91a47d0d2b2f0668ea89._comment b/doc/devblog/day_-4__forgetting/comment_7_a865216046aa91a47d0d2b2f0668ea89._comment
new file mode 100644
index 0000000000..dc142edc01
--- /dev/null
+++ b/doc/devblog/day_-4__forgetting/comment_7_a865216046aa91a47d0d2b2f0668ea89._comment
@@ -0,0 +1,12 @@
+[[!comment format=mdwn
+ username="stp"
+ ip="84.56.21.11"
+ subject="New findings"
+ date="2014-02-24T12:28:03Z"
+ content="""
+Another thing I found, which was annoying is that I have objects in my annex not tracked anywhere it seems.
+\"git annex fsck --all\" complains about not having access to the object. \"git log --stat -S '$key'\" doesn't have any record. \"git annex fsck\" has no issues and \"git annex unused\" comes up empty too.
+I'm not sure where these objects still reside or why how to remove this annoying failure.
+
+So not only should \"git annex forget $key\" remove references from within all branches, but should also clean up the aforementioned loose objects, which are neither unused, nor available, nor referenced.
+"""]]
diff --git a/doc/devblog/day_-4__forgetting/comment_8_3f7045a00905b4287d950b08d5a77a82._comment b/doc/devblog/day_-4__forgetting/comment_8_3f7045a00905b4287d950b08d5a77a82._comment
new file mode 100644
index 0000000000..f0ba50990c
--- /dev/null
+++ b/doc/devblog/day_-4__forgetting/comment_8_3f7045a00905b4287d950b08d5a77a82._comment
@@ -0,0 +1,9 @@
+[[!comment format=mdwn
+ username="stp"
+ ip="188.193.207.34"
+ subject="Any update on cleaning up commands?"
+ date="2014-03-21T16:32:43Z"
+ content="""
+Is there any update on cleaning up object/file references to objects/content not at all present and lost. I would love my git annex fsck --all to show current failures and not these old files all the time.
+Thanks
+"""]]
diff --git a/doc/devblog/day_-4__forgetting/comment_9_d9121a5172f02df63364f19eae87d011._comment b/doc/devblog/day_-4__forgetting/comment_9_d9121a5172f02df63364f19eae87d011._comment
new file mode 100644
index 0000000000..a104c2dd26
--- /dev/null
+++ b/doc/devblog/day_-4__forgetting/comment_9_d9121a5172f02df63364f19eae87d011._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="stp"
+ ip="24.134.205.34"
+ subject="Any update"
+ date="2014-10-01T12:47:47Z"
+ content="""
+Any update?
+"""]]
diff --git a/doc/devblog/day_100__git-annex_sync_--content.mdwn b/doc/devblog/day_100__git-annex_sync_--content.mdwn
new file mode 100644
index 0000000000..f35fbd330e
--- /dev/null
+++ b/doc/devblog/day_100__git-annex_sync_--content.mdwn
@@ -0,0 +1,4 @@
+Spent the day building this new feature, which makes `git annex sync --content`
+do the same synchronization of file contents (to satisfy preferred content
+settings) that the assistant does. The result has not been tested a lot
+yet, but seems to work well.
diff --git a/doc/devblog/day_101__old_mistakes.mdwn b/doc/devblog/day_101__old_mistakes.mdwn
new file mode 100644
index 0000000000..4a37416dc4
--- /dev/null
+++ b/doc/devblog/day_101__old_mistakes.mdwn
@@ -0,0 +1,23 @@
+In order to remove some hackishness in `git annex sync --content`, I
+finally fixed a bad design decision I made back at the very beginning
+(before I really knew haskell) when I built the command seek code, which
+had led to a kind of inversion of control. This took most of a night, but
+it made a lot of code in git-annex clearer, and it makes the command
+seeking code much more flexible in what it can do. Some of the oldest, and
+worst code in git-annex was removed in the process.
+
+Also, I've been reworking the numcopies configuration, to allow for a 
+[[todo/preferred_content_numcopies_check]]. That will let the assistant,
+as well as `git annex sync --content` proactively make copies when
+needed in order to satisfy numcopies. 
+
+As part of this, `git config annex.numcopies` is deprecated, and there's a
+new `git annex numcopies N` command that sets the numcopies value that will
+be used by any clone of a repository.
+
+I got the preferred content checking of numcopies working too. However,
+I am unsure if checking for per-file .gitattributes annex.numcopies
+settings will make preferred content expressions be, so I have left
+that out for now.
+
+Today's work was sponsored by Josh Taylor.
diff --git a/doc/devblog/day_101__old_mistakes/comment_1_2c6e991efde3296450189b2821f2ccc3._comment b/doc/devblog/day_101__old_mistakes/comment_1_2c6e991efde3296450189b2821f2ccc3._comment
new file mode 100644
index 0000000000..93fb84aa3d
--- /dev/null
+++ b/doc/devblog/day_101__old_mistakes/comment_1_2c6e991efde3296450189b2821f2ccc3._comment
@@ -0,0 +1,17 @@
+[[!comment format=mdwn
+ username="https://www.google.com/accounts/o8/id?id=AItOawl9sYlePmv1xK-VvjBdN-5doOa_Xw-jH4U"
+ nickname="Richard"
+ subject="comment 1"
+ date="2014-01-21T00:20:34Z"
+ content="""
+Some more details on
+
+* when the old mechanism will stop working
+* how you will ensure interoperability in repo sets which have different versions installed in different places
+* which option weighs more heavily
+
+would be appreciated.
+
+
+Richard
+"""]]
diff --git a/doc/devblog/day_101__old_mistakes/comment_2_524690d69056737dd296e4f7626737d2._comment b/doc/devblog/day_101__old_mistakes/comment_2_524690d69056737dd296e4f7626737d2._comment
new file mode 100644
index 0000000000..9f7ebecf18
--- /dev/null
+++ b/doc/devblog/day_101__old_mistakes/comment_2_524690d69056737dd296e4f7626737d2._comment
@@ -0,0 +1,12 @@
+[[!comment format=mdwn
+ username="http://joeyh.name/"
+ ip="209.250.56.249"
+ subject="comment 2"
+ date="2014-01-21T02:23:00Z"
+ content="""
+I don't plan to remove .git/config annex.numcopies support. It is 5 lines of code. 
+
+A fundamental problem with the .git/config annex.numcopies setting is that it can be inconsistently set between repositories. Therefore, there is no worse potential with interoperability if you use differing versions of git-annex that support and do not support the new global numcopies setting. If this was a concern though, you can use .gitattributes to set annex.numcopies for *, which all recent versions of git-annex support.
+
+The old git/config annex.numcopies setting is overridden as soon as you use `git annex numcopies`. In turn `git annex numcopies` is overridden by any relevant .gitattributes annex.numcopies setting. This is documented on the man page.
+"""]]
diff --git a/doc/devblog/day_102__cleanups.mdwn b/doc/devblog/day_102__cleanups.mdwn
new file mode 100644
index 0000000000..27a1badeaf
--- /dev/null
+++ b/doc/devblog/day_102__cleanups.mdwn
@@ -0,0 +1,10 @@
+Worked on cleaning up and reorganizing all the code that handles numcopies
+settings. Much nicer now. Fixed some bugs.
+
+As expected, making the preferred content numcopies check look at
+.gitattributes slows it down significantly. So, exposed both the slow and
+accurate check and a faster version that ignores .gitattributes.
+
+Also worked on the test suite, removing dependencies between tests.
+This will let tasty-rerun be used later to run only previously failing
+tests.
diff --git a/doc/devblog/day_103__unused.mdwn b/doc/devblog/day_103__unused.mdwn
new file mode 100644
index 0000000000..affb4e532e
--- /dev/null
+++ b/doc/devblog/day_103__unused.mdwn
@@ -0,0 +1,34 @@
+A big missing peice of the assistant is doing something about the content
+of old versions of files, and deleted files. In direct mode, editing or
+deleting a file necessarily loses its content from the local repository,
+but the content can still hang around in other repositories. So, the
+assistant needs to do something about that to avoid eating up disk space
+unnecessarily.
+
+I built on recent work, that lets preferred content expressions be matched
+against keys with no associated file. This means that I can run unused keys
+through all the machinery in the assistant that handles file transfers, and
+they'll end being moved to whatever repository wants them. To control which
+repositories do want to retain unused files, and which not, I added a
+`unused` keyword to preferred content expressions. Client repositories and
+transfer repositories do not want to retain unused files, but backup etc
+repos do.
+
+One nice thing about this `unused` preferred content implementation is that
+it doesn't slow down normal matching of preferred content expressions at
+all. Can you guess why not? See [[!commit 4b55afe9e92c045d72b78747021e15e8dfc16416]]
+
+So, the assistant will run `git annex unused` on a daily basis, and
+cause unused files to flow to repositories that want them. But what if no
+repositories do? To guard against filling up the local disk, there's
+a `annex.expireunused` configuration setting, that can cause old unused
+files to be deleted by the assistant after a number of days.
+
+I made the assistant check if there seem to be a lot of unused files piling
+up. (1000+, or 10% of disk used by them, or more space taken by unused files
+than is free.) If so, it'll pop up an alert to nudge the user to configure
+annex.expireunused.
+
+Still need to build the UI to configure that, and test all of this.
+
+Today's work was sponsored by Samuel Tardieu.
diff --git a/doc/devblog/day_104__unused_II.mdwn b/doc/devblog/day_104__unused_II.mdwn
new file mode 100644
index 0000000000..7dc01c5fdd
--- /dev/null
+++ b/doc/devblog/day_104__unused_II.mdwn
@@ -0,0 +1,7 @@
+Built the UI to manage unused files.
+
+[[!img assistant/unused.png]]
+
+Testing yesterday's work, I found several problems that prevented the
+assistant from moving unused files around, and fixed them. It seems to be
+working pretty well now.
diff --git a/doc/devblog/day_104__unused_II/comment_1_a693a56123497a39c30cbd35b8e35bce._comment b/doc/devblog/day_104__unused_II/comment_1_a693a56123497a39c30cbd35b8e35bce._comment
new file mode 100644
index 0000000000..a4893f63ab
--- /dev/null
+++ b/doc/devblog/day_104__unused_II/comment_1_a693a56123497a39c30cbd35b8e35bce._comment
@@ -0,0 +1,10 @@
+[[!comment format=mdwn
+ username="https://www.google.com/accounts/o8/id?id=AItOawm7AuSfii_tCkLyspL6Mr0ATlO6OxLNYOo"
+ nickname="Georg"
+ subject="state of windows"
+ date="2014-01-28T10:29:41Z"
+ content="""
+Any idea when this stuff will end in the windows build? I am currently fiddling around with git-annex on windows (btw: great work on that so far!) and the unused files config would be highly valueable to me, since I want to mostly use it to push files out to my server from my work machine or my windows tablet, while preventing to clog up too much space. And since I am lazy, I would love to leave the house keeping to git-annex ;)
+
+An additional question on the windows build: which of the backends can actually make use of windows proxy settings? I guess if webdav was in the list of supported backends, it would work fine, since it is only http, but since it is missing, which of the others would work through just a standard configured squid proxy? For my work machine this is the only way to reach the outside world, doing https to my web server.
+"""]]
diff --git a/doc/devblog/day_104__unused_II/comment_2_9833fb9daa50bc838cc46ca2f6c16580._comment b/doc/devblog/day_104__unused_II/comment_2_9833fb9daa50bc838cc46ca2f6c16580._comment
new file mode 100644
index 0000000000..eeaef05bf2
--- /dev/null
+++ b/doc/devblog/day_104__unused_II/comment_2_9833fb9daa50bc838cc46ca2f6c16580._comment
@@ -0,0 +1,10 @@
+[[!comment format=mdwn
+ username="http://joeyh.name/"
+ ip="209.250.56.199"
+ subject="comment 2"
+ date="2014-01-28T20:58:32Z"
+ content="""
+AFAICS, the Windows build does include WebDAV support.
+
+I don't know about Windows proxy stuff. If it requires the program to do something, git-annex almost certianly does not do it.
+"""]]
diff --git a/doc/devblog/day_104__unused_II/comment_3_873a882ec1ddc3be473473cb224a9040._comment b/doc/devblog/day_104__unused_II/comment_3_873a882ec1ddc3be473473cb224a9040._comment
new file mode 100644
index 0000000000..96b870627f
--- /dev/null
+++ b/doc/devblog/day_104__unused_II/comment_3_873a882ec1ddc3be473473cb224a9040._comment
@@ -0,0 +1,12 @@
+[[!comment format=mdwn
+ username="https://www.google.com/accounts/o8/id?id=AItOawm7AuSfii_tCkLyspL6Mr0ATlO6OxLNYOo"
+ nickname="Georg"
+ subject="comment 3"
+ date="2014-01-29T08:22:13Z"
+ content="""
+well, at least in the assistant there is no selection to add a webdav repository - only box.com, rsync.net, S3, remote ssh, and the archives on glazier or internet archive. It is the binary version from 16th this month. I thought it was still excluded due to the compiler crashes you wrote about, because it doesn't show up in the webapp as option. I was scratching my head anyway about that windows build, because there was nothing about setting up encryption in the configuration in the webapp.
+
+Proxy support on Windows is kinda like proxy support on linux when you pull the proxy from gnome settings - just it is pulled from the registry. I don't know how Haskell HTTP client libraries handle it, only know it for Python - there the client libraries usually grab proxy settings automatically from the system configuration.
+
+Another option which could work for me to get through our firewall at work would be ssh - I can use ssh with proxy commands to reach the outside, but didn't yet find the used ssh configs on the windows git-annex installation - it doesn't use the one in ~/.ssh, I added my proxy commands there, but that isn't used.
+"""]]
diff --git a/doc/devblog/day_104__unused_II/comment_4_5ef1bb4d69cf7206f7ca0e542abad6bd._comment b/doc/devblog/day_104__unused_II/comment_4_5ef1bb4d69cf7206f7ca0e542abad6bd._comment
new file mode 100644
index 0000000000..adf3293ffe
--- /dev/null
+++ b/doc/devblog/day_104__unused_II/comment_4_5ef1bb4d69cf7206f7ca0e542abad6bd._comment
@@ -0,0 +1,13 @@
+[[!comment format=mdwn
+ username="https://www.google.com/accounts/o8/id?id=AItOawm7AuSfii_tCkLyspL6Mr0ATlO6OxLNYOo"
+ nickname="Georg"
+ subject="comment 4"
+ date="2014-01-29T08:26:19Z"
+ content="""
+Oh, just noticed in the console window of the assistant:
+
+ssh-keygen: /home/xgwsbae/.ssh/known_hosts: No such file or directory
+ssh-keygen: /home/xgwsbae/.ssh/known_hosts: No such file or directory
+
+Home directory is C:/Users/xgwsbae/ (git bash gives /c/Users/xgwsbae in $HOME) ... looks like something got confuzzled between Windows and Unix there.
+"""]]
diff --git a/doc/devblog/day_104__unused_II/comment_5_1964bfce4887c9c0828fd7f54f5b4f6b._comment b/doc/devblog/day_104__unused_II/comment_5_1964bfce4887c9c0828fd7f54f5b4f6b._comment
new file mode 100644
index 0000000000..12849438c6
--- /dev/null
+++ b/doc/devblog/day_104__unused_II/comment_5_1964bfce4887c9c0828fd7f54f5b4f6b._comment
@@ -0,0 +1,12 @@
+[[!comment format=mdwn
+ username="http://joeyh.name/"
+ ip="209.250.56.199"
+ subject="comment 5"
+ date="2014-01-29T17:58:26Z"
+ content="""
+Some things (not git-annex) default to /home/$USER when appropriate env vars are not set. There is a bug open about that, [[bugs/assistant_using_the_incorrect_path_on_windows?]], although I don't know what it makes sense for git-annex to do to deal with it.
+
+box.com *is* a webdav repository. It is the only webdav repository the webapp currently supports setting up, although you can configure any webdav remote you like at the command line and the assistant will then be able to use it.
+
+Will be curious to hear if the http-conduit library used for webdav supports proxies. If not, that would certianly seem a good place to file a bug. Unofrtunately, git-annex uses 2 or 3 other http stacks in other places too.
+"""]]
diff --git a/doc/devblog/day_104__unused_II/comment_6_0ed4023c9c249024fe0454fc5bab3b14._comment b/doc/devblog/day_104__unused_II/comment_6_0ed4023c9c249024fe0454fc5bab3b14._comment
new file mode 100644
index 0000000000..d900dc5117
--- /dev/null
+++ b/doc/devblog/day_104__unused_II/comment_6_0ed4023c9c249024fe0454fc5bab3b14._comment
@@ -0,0 +1,10 @@
+[[!comment format=mdwn
+ username="https://www.google.com/accounts/o8/id?id=AItOawm7AuSfii_tCkLyspL6Mr0ATlO6OxLNYOo"
+ nickname="Georg"
+ subject="comment 6"
+ date="2014-01-30T10:52:09Z"
+ content="""
+Well, according to the hackage documentation of http conduit it seems it at least supports proxies when youre code gives it the proxy host and port.
+
+But the link about the problem with environment settings helped me to get around my immediate problem by just running annex from the git bash (which is prefereably to me most of the time anyway). So I could set up my annex on my work machine and just tunnel out with ProxyCommand via our ssh gateway. Yay!
+"""]]
diff --git a/doc/devblog/day_105__locking.mdwn b/doc/devblog/day_105__locking.mdwn
new file mode 100644
index 0000000000..b41056bb44
--- /dev/null
+++ b/doc/devblog/day_105__locking.mdwn
@@ -0,0 +1,30 @@
+With yesterday's release, I'm pretty much done with the month's work. Since
+there was no particular goal this month, it's been a grab bag of features
+and bugfixes. Quite a lot of them in this last release.
+
+I'll be away the next couple of days.. But got a start today on the next
+part of the roadmap, which is planned to be all about Windows and Android
+porting. Today, it was all about lock files, mostly on Windows.
+
+Lock files on Windows are horrific. I especially like that programs that
+want to open a file, for any reason, are encouraged in the official
+documentation to retry repeatedly if it fails, because some other
+random program, like a virus checker, might have opened the file first.
+
+Turns out Windows does support a shared file read mode. This was
+just barely enough for me to implement both shared and exclusive
+file locking a-la-flock.
+
+Couldn't avoid a busy wait in a few places that block on a lock.
+Luckily, these are few, and the chances the lock will be taken for a long
+time is small. (I did think about trying to watch the file for close events
+and detect when the lock was released that way, but it seemed much too
+complicated and hard to avoid races.)
+
+Also, Windows only seems to support mandatory locks, while all locking in
+git-annex needs to be advisory locks. Ie, git-annex's locking shouldn't
+prevent a program from opening an annexed file! To work around that,
+I am using dedicated lock files on Windows.
+
+Also switched direct mode's annexed object locking to use dedicated lock
+files. AFAICS, this was pretty well broken in direct mode before.
diff --git a/doc/devblog/day_106__catching_up.mdwn b/doc/devblog/day_106__catching_up.mdwn
new file mode 100644
index 0000000000..86e7acbaec
--- /dev/null
+++ b/doc/devblog/day_106__catching_up.mdwn
@@ -0,0 +1,5 @@
+While I've not been blogging over what amounted to a long weekend, looking
+over the changelog, there were quite a few things done. Mostly various
+improvements and fixes to `git annex sync --content`.
+
+Today, got the test suite to pass on Windows 100% again.
diff --git a/doc/devblog/day_107__TDD.mdwn b/doc/devblog/day_107__TDD.mdwn
new file mode 100644
index 0000000000..0ae41f0d64
--- /dev/null
+++ b/doc/devblog/day_107__TDD.mdwn
@@ -0,0 +1,10 @@
+A more test driven day than usual. Yesterday I noticed a test case was
+failing on Windows in a way not related to what it was intended to test,
+and fixed the test case to not fail.. But knew I'd need to get to the
+bottom of what broke it eventually.
+
+Digging into that today, I eventually (after rather a long time stuck)
+determined the bug involved automatic conflict resolution, but only
+happened on systems without symlink support. This let me reproduce it on
+FAT outside Windows and do some fast TDD iterations in a much less
+unwieldly environment and fix the bug.
diff --git a/doc/devblog/day_108__new_use_for_location_tracking.mdwn b/doc/devblog/day_108__new_use_for_location_tracking.mdwn
new file mode 100644
index 0000000000..62e40672f7
--- /dev/null
+++ b/doc/devblog/day_108__new_use_for_location_tracking.mdwn
@@ -0,0 +1,20 @@
+Added a new feature that started out with me wanting a way to undo a
+`git-annex drop`, but turned into something rather more powerful. The `--in`
+option can now be told to match files that were in a repository at some
+point in the past. For example, `git annex get --in=here@{yesterday}` will
+get any files that have been dropped over the past day.
+
+While git-annex's location tracking info is stored in git and thus versioned,
+very little of it makes use of past versions of the location tracking info
+(only `git annex log`). I'm happy to have finally found a use for it!
+
+OB Windows porting: Fixed a bug in the symlink calculation code.
+Sounds simple; took 2 hours!
+
+Also various bug triage; updated git version on OSX; forwarded bug about
+DAV-0.6 being broken upstream; fixed a bug with initremote in 
+encryption=pubkey mode. Backlog is 65 messages.
+
+---
+
+Today's work was sponsored by Brock Spratlen.
diff --git a/doc/devblog/day_109__elimintating_absNormPath.mdwn b/doc/devblog/day_109__elimintating_absNormPath.mdwn
new file mode 100644
index 0000000000..8381f2956e
--- /dev/null
+++ b/doc/devblog/day_109__elimintating_absNormPath.mdwn
@@ -0,0 +1,18 @@
+git-annex has been using MissingH's `absNormPath` forever, but that's
+not very maintained and doesn't work on Windows. I've been
+wanting to get rid of it for some time, and finally did today, writing a
+`simplifyPath` that does the things git-annex needs and will work with all
+the Windows filename craziness, and takes advantage of the more modern
+System.FilePath to be quite a simple peice of code. A QuickCheck test found
+no important divergences from absNormPath. A good first step to making
+git-annex not depend on MissingH at all. 
+
+That fixed one last Windows bug that was disabled in the test suite:
+`git annex add ..\subdir\file` will now work.
+
+I am re-installing the Android autobuilder for 2 reasons: I noticed I had
+accidentally lost a patch to make a library use the Android SSL cert directory,
+and also a new version of GHC is very near to release and so it makes sense
+to update.
+
+Down to 38 messages in the backlog.
diff --git a/doc/devblog/day_10__lazy_Sunday.mdwn b/doc/devblog/day_10__lazy_Sunday.mdwn
new file mode 100644
index 0000000000..c8843d3747
--- /dev/null
+++ b/doc/devblog/day_10__lazy_Sunday.mdwn
@@ -0,0 +1,23 @@
+Fixed a typo that broke automatic youtube video support in `addurl`.
+
+----
+
+Now there's an easy way to get an overview of how close your repository
+is to meeting the configured numcopies settings (or when it exceeds them).
+
+
    +# time git annex info . 
+[...]
+numcopies stats: 
+	numcopies +0: 6686
+	numcopies +1: 3793
+	numcopies +3: 3156
+	numcopies +2: 2743
+	numcopies -1: 1242
+	numcopies -4: 1098
+	numcopies -3: 1009
+	numcopies +4: 372
+
    
+
+This does make `git annex info` slow when run on a large directory tree,
+so --fast disables that.
diff --git a/doc/devblog/day_110__release_prep.mdwn b/doc/devblog/day_110__release_prep.mdwn
new file mode 100644
index 0000000000..12ba78a4fd
--- /dev/null
+++ b/doc/devblog/day_110__release_prep.mdwn
@@ -0,0 +1,25 @@
+Last night I tracked down and fixed a bug in the DAV library that has been
+affecting WebDAV remotes. I've been deploying the fix for that today,
+including to the android and arm autobuilders. While I finished a clean
+reinstall of the android autobuilder, I ran into problems getting a clean
+reinstall of the arm autobuilder (some type mismatch error building
+yesod-core), so manually fixed its DAV for now.
+
+The WebDAV fix and other recent fixes makes me want to make a release soon,
+probably Monday.
+
+ObWindows: Fixed git-annex to not crash when run on Windows
+in a git repository that has a remote with a unix-style path 
+like "/foo/bar". Seems that not everything aggrees on whether such a path
+is absolute; even sometimes different parts of the same library disagree!
+
+[[!format haskell """
+import System.FilePath.Windows
+
+prop_windows_is_sane :: Bool
+prop_windows_is_sane = isAbsolute upath || ("C:\\STUFF"  upath /= upath)
+  where upath = "/foo/bar"
+"""]]
+
+Perhaps more interestingly, I've been helping dxtrish port git-annex to
+OpenBSD and it seems most of the way there.
diff --git a/doc/devblog/day_111__windows_beta_release.mdwn b/doc/devblog/day_111__windows_beta_release.mdwn
new file mode 100644
index 0000000000..cd880dc0c5
--- /dev/null
+++ b/doc/devblog/day_111__windows_beta_release.mdwn
@@ -0,0 +1,6 @@
+Pushed out the new release. This is the first one where I consider the
+git-annex command line beta quality on Windows.
+
+Did some testing of the webapp on Windows, trying out every part of the UI.
+I now have eleven todo items involving the webapp listed in
+[[todo/windows_support]]. Most of them don't look too bad to fix.
diff --git a/doc/devblog/day_112__metadata_design.mdwn b/doc/devblog/day_112__metadata_design.mdwn
new file mode 100644
index 0000000000..e75d4fc4ed
--- /dev/null
+++ b/doc/devblog/day_112__metadata_design.mdwn
@@ -0,0 +1,18 @@
+There's a new design document for letting git-annex store arbitrary
+metadata. The really neat thing about this is the user can check out only
+files matching the tags or values they care about, and get an automatically
+structuted file tree layout that can be dynamically filtered. It's going to
+be awesome! [[design/metadata]]
+
+In the meantime, spent most of today working on Windows. Very good
+progress, possibly motivated by wanting to get it over with so I can spend
+some time this month on the above. ;)
+
+* webapp can make box.com and S3 remotes. This just involved fixing a hack
+  where the webapp set environment variables to communicate creds to
+  initremote. Can't change environment on Windows (or I don't know how to).
+* webapp can make repos on removable drives.
+* `git annex assistant --stop` works, although this is not likely to really
+  be useful
+* The source tree now has 0 `func = error "Windows TODO"` type stubbed out
+  functions to trip over.
diff --git a/doc/devblog/day_113__metadata_groundwork.mdwn b/doc/devblog/day_113__metadata_groundwork.mdwn
new file mode 100644
index 0000000000..4859c4d33e
--- /dev/null
+++ b/doc/devblog/day_113__metadata_groundwork.mdwn
@@ -0,0 +1,9 @@
+Built the core data types, and log for metadata storage. Making metadata
+union merge well is tricky, but I have a design I'm happy with, that will
+allow distributed changes to metadata.
+
+Finished up the day with a `git annex metadata` command to get/set metadata
+for a file.
+
+This is all the goundwork needed to begin experimenting with generating
+git branches that display different metadata-driven views of annexed files.
diff --git a/doc/devblog/day_114__windows_porting.mdwn b/doc/devblog/day_114__windows_porting.mdwn
new file mode 100644
index 0000000000..2f1cace1b1
--- /dev/null
+++ b/doc/devblog/day_114__windows_porting.mdwn
@@ -0,0 +1,8 @@
+Windows porting all day. Fixed a lot of issues with the webapp,
+so quite productive. Except for the 2 hours wasted finding a way to kill a
+process by PID from Haskell on Windows.
+
+Last night, made `git annex metadata` able to set metadata on a whole
+directory or list of files if desired. And added a `--metadata field=value`
+switch (and corresponding preferred content terminal) which limits
+git-annex to acting on files with the specified metadata.
diff --git a/doc/devblog/day_115__windows_porting.mdwn b/doc/devblog/day_115__windows_porting.mdwn
new file mode 100644
index 0000000000..a9abe4ad2f
--- /dev/null
+++ b/doc/devblog/day_115__windows_porting.mdwn
@@ -0,0 +1,17 @@
+More Windows porting.. Seem to be getting near an end of the easy stuff,
+and also the webapp is getting pretty usable on Windows now, the only
+really important thing lacking is XMPP support.
+
+Made git-annex on Windows set HOME when it's not already set. Several of
+the bundled cygwin tools only look at HOME. This was made a lot harder and
+uglier due to there not being any way to modify the environment of the
+running process.. git-annex has to re-run itself with the fixed
+environment.
+
+Got rsync.net working in the webapp. Although with an extra rsync.net
+password prompt on Windows, which I cannot find a way to avoid.
+
+While testing that, I discovered that openssh 6.5p1 has broken support for
+~/.ssh/config Host lines that contain upper case letters! I have filed a
+bug about this and put a quick fix in git-annex, which sometimes generated
+such lines.
diff --git a/doc/devblog/day_116__views.mdwn b/doc/devblog/day_116__views.mdwn
new file mode 100644
index 0000000000..a7aaf2c939
--- /dev/null
+++ b/doc/devblog/day_116__views.mdwn
@@ -0,0 +1,54 @@
+Working on building [[design/metadata]] filtered branches.
+
+Spent most of the day on types and pure code. Finally at the end 
+I wrote down two actions that I still need to implement to make
+it all work:
+
+[[!format haskell """
+applyView' :: MkFileView -> View -> Annex Git.Branch
+updateView :: View -> Git.Ref -> Git.Ref -> Annex Git.Branch
+"""]]
+
+I know how to implement these, more or less. And in most cases
+they will be pretty fast.
+
+The more interesting part is already done. That was the issue of how to
+generate filenames in the filter branches. That depends on the `View` being
+used to filter and organize the branch, but also on the original filename used
+in the reference branch. Each filter branch has a reference branch (such as
+"master"), and displays a filtered and metadata-driven reorganized tree
+of files from its reference branch.
+
+[[!format haskell """
+fileViews :: View -> (FilePath -> FileView) -> FilePath -> MetaData -> Maybe [FileView]
+"""]]
+
+So, a view that matches files tagged "haskell" or "git-annex"
+and with an author of "J\*" will generate filenames like 
+"haskell/Joachim/interesting_theoretical_talk.ogg" and
+"git-annex/Joey/mytalk.ogg".
+
+It can also work backwards from these 
+filenames to derive the MetaData that is encoded in them.
+
+[[!format haskell """
+fromView :: View -> FileView -> MetaData
+"""]]
+
+So, copying a file to "haskell/Joey/mytalk.ogg" lets it know that
+it's gained a "haskell" tag. I knew I was on the right track when
+`fromView` turned out to be only 6 lines of code!
+
+The trickiest part of all this, which I spent most of yesterday thinking
+about, is what to do if the master branch has files in subdirectories. It
+probably does not makes sense to retain that hierarchical directory
+structure in the filtered branch, because we instead have a
+non-hierarchical metadata structure to express. (And there would probably
+be a lot of deep directory structures containing only one file.) But
+throwing away the subdirectory information entirely means that two files
+with the same basename and same metadata would have colliding names.
+
+I eventually decided to embed the subdirectory information into the filenames
+used on the filter branch. Currently that is done by converting
+`dir/subdir/file.foo` to `file(dir)(subdir).foo`. We'll see how this works
+out in practice..
diff --git a/doc/devblog/day_117__views_implemented.mdwn b/doc/devblog/day_117__views_implemented.mdwn
new file mode 100644
index 0000000000..fa2ae28375
--- /dev/null
+++ b/doc/devblog/day_117__views_implemented.mdwn
@@ -0,0 +1,76 @@
+Today I built `git annex view`, and `git annex vadd` and a few related
+commands. A quick demo:
+
+
    +joey@darkstar:~/lib/talks>ls
+Chaos_Communication_Congress/  FOSDEM/       Linux_Conference_Australia/
+Debian/                        LibrePlanet/  README.md
+joey@darkstar:~/lib/talks>git annex view tag=*
+view  (searching...)
+Switched to branch 'views/_'
+ok
+joey@darkstar:~/lib/talks#_>tree -d
+.
+|-- Debian
+|-- android
+|-- bigpicture
+|-- debhelper
+|-- git
+|-- git-annex
+`-- seen
+
+7 directories
+joey@darkstar:~/lib/talks#_>git annex vadd author=*
+vadd  
+Switched to branch 'views/author=_;_'
+ok
+joey@darkstar:~/lib/talks#author=_;_>tree -d
+.
+|-- Benjamin Mako Hill
+|   `-- bigpicture
+|-- Denis Carikli
+|   `-- android
+|-- Joey Hess
+|   |-- Debian
+|   |-- bigpicture
+|   |-- debhelper
+|   |-- git
+|   `-- git-annex
+|-- Richard Hartmann
+|   |-- git
+|   `-- git-annex
+`-- Stefano Zacchiroli
+    `-- Debian
+
+15 directories
+joey@darkstar:~/lib/talks#author=_;_>git annex vpop
+vpop 1
+Switched to branch 'views/_'
+ok
+joey@darkstar:~/lib/talks#_>git annex vadd tag=git-annex
+vadd  
+Switched to branch 'views/(git-annex)'
+ok
+joey@darkstar:~/lib/talks#(git-annex)>ls
+1025_gitify_your_life_{Debian;2013;DebConf13;high}.ogv@
+git_annex___manage_files_with_git__without_checking_their_contents_into_git_{FOSDEM;2012;lightningtalks}.webm@
+mirror.linux.org.au_linux.conf.au_2013_mp4_gitannex_{Linux_Conference_Australia;2013}.mp4@
+joey@darkstar:~/lib/talks#_>git annex vpop 2
+vpop 2
+Switched to branch 'master'
+ok
+
    
+
+Not 100% happy with the speed -- the generation of the view branch is close
+to optimal, and fast enough (unless the branch has very many matching
+files). And `vadd` can be quite fast if the view has already limited the
+total number of files to a smallish amount. But `view` has to look at every
+file's metadata, and this can take a while in a large repository. Needs indexes.
+
+It also needs integration with `git annex sync`, so the view branches
+update when files are added to the master branch, and moving files around
+inside a view and committing them does not yet update their metadata.
+
+---
+
+Today's work was sponsored by Daniel Atlas.
diff --git a/doc/devblog/day_118__views_refined.mdwn b/doc/devblog/day_118__views_refined.mdwn
new file mode 100644
index 0000000000..bfd102f18f
--- /dev/null
+++ b/doc/devblog/day_118__views_refined.mdwn
@@ -0,0 +1,7 @@
+Still working on views. The most important addition today is that
+`git annex precommit` notices when files have been moved/copied/deleted
+in a view, and updates the metadata to reflect the changes.
+
+Also wrote some walkthrough documentation: [[tips/metadata_driven_views]].  
+And, recorded a screencast demoing views, which I will upload next time
+I have bandwidth.
diff --git a/doc/devblog/day_119__catching_up.mdwn b/doc/devblog/day_119__catching_up.mdwn
new file mode 100644
index 0000000000..c49c3e34dd
--- /dev/null
+++ b/doc/devblog/day_119__catching_up.mdwn
@@ -0,0 +1,15 @@
+Spent the day catching up on the last week or so's traffic. Ended up
+making numerous small big fixes and improvements. Message backlog stands at
+44.
+
+Here's the [[screencast demoing views|videos/git-annex_views_demo]]!
+
+Added to the design today the idea of
+automatically deriving metadata from the location of files in the master
+branch's directory tree. Eg, `git annex view tag=* podcasts/=*` in a
+repository that has a `podcasts/` directory would make a tree like
+"$tag/$podcast". Seems promising. 
+
+So much still to do with views.. I have belatedly added them to
+the roadmap for this month; doing Windows and Android in the same month was
+too much to expect.
diff --git a/doc/devblog/day_119__catching_up/comment_1_8aa413e75cab411b0aec254f0f33ebb9._comment b/doc/devblog/day_119__catching_up/comment_1_8aa413e75cab411b0aec254f0f33ebb9._comment
new file mode 100644
index 0000000000..9c1ba1e43e
--- /dev/null
+++ b/doc/devblog/day_119__catching_up/comment_1_8aa413e75cab411b0aec254f0f33ebb9._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="https://www.google.com/accounts/o8/id?id=AItOawlJEI45rGczFAnuM7gRSj4C6s9AS9yPZDc"
+ nickname="Kevin"
+ subject="Neat!"
+ date="2014-02-20T21:59:05Z"
+ content="""
+When the [[metadata design|day_112__metadata_design]] stuff appeared on the blog I didn't understand what you meant by automatically creating new tree layouts.  I'm really liking these views and can already imagine how useful it would be to tag my photos by person/place/time.  This is awesome!  Keep up the good work.
+"""]]
diff --git a/doc/devblog/day_119__catching_up/comment_2_db31d08690730836ce6277e797fcae1d._comment b/doc/devblog/day_119__catching_up/comment_2_db31d08690730836ce6277e797fcae1d._comment
new file mode 100644
index 0000000000..532b7a4d5f
--- /dev/null
+++ b/doc/devblog/day_119__catching_up/comment_2_db31d08690730836ce6277e797fcae1d._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="https://www.google.com/accounts/o8/id?id=AItOawneiQ3iR9VXOPEP34u7m_L3Qr28H1nEfE0"
+ nickname="Ethan"
+ subject="LFS"
+ date="2014-02-21T00:03:59Z"
+ content="""
+You might be interested in the Logic File System at http://www.padator.org/wiki/wiki-LFS/doku.php or http://www.padator.org/papers/ which has a similar idea with views and metadata.
+"""]]
diff --git a/doc/devblog/day_119__catching_up/comment_3_d44da76b18f53a5f51b46e3e15090a48._comment b/doc/devblog/day_119__catching_up/comment_3_d44da76b18f53a5f51b46e3e15090a48._comment
new file mode 100644
index 0000000000..9d97aff2d4
--- /dev/null
+++ b/doc/devblog/day_119__catching_up/comment_3_d44da76b18f53a5f51b46e3e15090a48._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="https://www.google.com/accounts/o8/id?id=AItOawmZgZuUhZlHpd_AbbcixY0QQiutb2I7GWY"
+ nickname="Jimmy"
+ subject="comment 3"
+ date="2014-02-21T07:05:34Z"
+ content="""
+I agree with Kevin as to the potential usefulness for photos.  Particularly if there's some way of automatically extracting and using tags or other EXIF metadata.
+"""]]
diff --git a/doc/devblog/day_11__webapp_encrypted_drives.mdwn b/doc/devblog/day_11__webapp_encrypted_drives.mdwn
new file mode 100644
index 0000000000..677c02491b
--- /dev/null
+++ b/doc/devblog/day_11__webapp_encrypted_drives.mdwn
@@ -0,0 +1,12 @@
+Now the webapp can set up encrypted repositories on removable drives.
+
+[[assistant/encryptdrive.png]]
+
+This UI needs some work, and the button to create a new key is not wired
+up. Also if you have no gpg agent installed, there will be lots of password
+prompts at the console.
+
+Forked git-remote-gcrypt to fix a bug. Hopefully my patch will be merged;
+for now I recommend installing my worked version.
+
+Today's work was sponsored by Romain Lenglet.
diff --git a/doc/devblog/day_120__more_metadata.mdwn b/doc/devblog/day_120__more_metadata.mdwn
new file mode 100644
index 0000000000..daff68e379
--- /dev/null
+++ b/doc/devblog/day_120__more_metadata.mdwn
@@ -0,0 +1,17 @@
+When generating a view, there's now a way to reuse part of the directory
+hierarchy of the parent branch. For example, `git annex view tag=* podcasts/=*`
+makes a view where the first level is the tags, and the second level is
+whatever `podcasts/*` directories the files were in.
+
+Also, year and month metadata can be automatically recorded when
+adding files to the annex. I made this only be done when annex.genmetadata
+is turned on, to avoid polluting repositories that don't want to use metadata.
+
+It would be nice if there was a way to add a hook script that's run
+when files are added, to collect their metadata. I am not sure yet if
+I am going to add that to git-annex though. It's already possible to do via
+the regular git `post-commit` hook. Just make it look at the commit to see
+what files were added, and then run `git annex metadata` to set their
+metadata appropriately. It would be good to at least have an example of
+such a script to eg, extract EXIF or ID3 metadata. Perhaps someone can
+contribute one?
diff --git a/doc/devblog/day_121__special_remote_maintenance.mdwn b/doc/devblog/day_121__special_remote_maintenance.mdwn
new file mode 100644
index 0000000000..5517048858
--- /dev/null
+++ b/doc/devblog/day_121__special_remote_maintenance.mdwn
@@ -0,0 +1,23 @@
+Turns out that in the last release I broke making box.com, Amazon S3 and
+Glacier remotes from the webapp. Fixed that.
+
+Also, dealt with changes in the haskell DAV library that broke support for
+box.com, and worked around an exception handling bug in the library.
+
+I think I should try to enhance the test suite so it can run live tests
+on special remotes, which would at least have caught the some of these
+recent problems...
+
+----
+
+Since metadata is tied to a particular key, editing an annexed file,
+which causes the key to change, made the metadata seem to get lost.
+
+I've now fixed this; it copies the metadata from the old version to the new
+one. (Taking care to copy the log file identically, so git can reuse its
+blob.) 
+
+That meant that `git annex add` has to check every file it adds to see if
+there's an old version. Happily, that check is fairly fast; I benchmarked my
+laptop running 2500 such checks a second. So it's not going to slow things
+down appreciably.
diff --git a/doc/devblog/day_122_more_windows_porting.mdwn b/doc/devblog/day_122_more_windows_porting.mdwn
new file mode 100644
index 0000000000..7b496f670f
--- /dev/null
+++ b/doc/devblog/day_122_more_windows_porting.mdwn
@@ -0,0 +1,4 @@
+More Windows porting. Made the build completely `-Wall` safe on Windows.
+Fixed some DOS path separator bugs that were preventing WebDav from
+working. Have now tested both box.com and Amazon S3 to be completely
+working in the webapp on Windows.
diff --git a/doc/devblog/day_123__stuck.mdwn b/doc/devblog/day_123__stuck.mdwn
new file mode 100644
index 0000000000..c48507cf0e
--- /dev/null
+++ b/doc/devblog/day_123__stuck.mdwn
@@ -0,0 +1,13 @@
+Not a lot accomplished today. Some release prep, followed up to a few bug
+reports.
+
+Split git-annex's .git/annex/tmp into two directories. .git/annex/tmp will
+now be used only for partially transferred objects, while
+.git/annex/misctmp will be used for everything else. In particular this
+allows symlinking .git/annex/tmp to a ram disk, if you want to do that.
+(It's not possible for .git/annex/misctmp to be on a different filesystem
+from the rest of the repository for various reasons.)
+
+Beat on Windows XMPP for several more painful hours. Got all the haskell
+bindings installed, except for gnuidn. And patched network-client-xmpp to
+build without gnuidn. Have not managed to get it to link.
diff --git a/doc/devblog/day_124__day_off.mdwn b/doc/devblog/day_124__day_off.mdwn
new file mode 100644
index 0000000000..c8c8f0109f
--- /dev/null
+++ b/doc/devblog/day_124__day_off.mdwn
@@ -0,0 +1,13 @@
+Did not plan to work on git-annex today..
+
+Unexpectedly ended up making the webapp support HTTPS. Not by default,
+but if a key and certificate are provided, it'll use them. Great for
+using the webapp remotely! See the new tip: [[tips/remote_webapp_setup]].
+
+Also removed support for --listen with a port, which was buggy and not
+necessary with HTTPS.
+
+Also fixed several webapp/assistant bugs, including one that let it be run in
+a bare git repository.
+
+And, made the quvi version be probed at runtime, rather than compile time.
diff --git a/doc/devblog/day_125__metadata_and_views.mdwn b/doc/devblog/day_125__metadata_and_views.mdwn
new file mode 100644
index 0000000000..471457355a
--- /dev/null
+++ b/doc/devblog/day_125__metadata_and_views.mdwn
@@ -0,0 +1,11 @@
+Worked on metadata and views. Besides bugfixes, two features of note:
+
+Made git-annex run a hook script, pre-commit-annex. And I wrote a
+sample script that extracts metadata from lots of kinds of files,
+including photos and sound files, using extract(1) to do the heavy lifting.
+See [[tips/automatically_adding_metadata]].
+
+Views can be filtered to not include a tag or a field.
+For example, `git annex view tag=* !old year!=2013`
+
+Today's work was sponsored by Stephan Schulz
diff --git a/doc/devblog/day_128__release_prep.mdwn b/doc/devblog/day_128__release_prep.mdwn
new file mode 100644
index 0000000000..d43d2f877b
--- /dev/null
+++ b/doc/devblog/day_128__release_prep.mdwn
@@ -0,0 +1,27 @@
+Preparing for a release (probably tomorrow or Friday).
+
+Part of that was updating the autobuilders. Had to deal with the gnutls
+security hole fix, and upgrading that on the OSX autobuilder turned out to
+be quite complicated due to library version skew. Also, I switched the
+linux autobuilders over to building from Debian unstable, rather than
+stable. That should be ok to do now that the standalone build bundles all
+the libraries it needs... And the arm build has always used unstable, and
+has been reported working on a lot of systems. So I think this will be
+safe, but have backed up the old autobuilder chroots just in case.
+
+Also been catching up on bug reports and traffic and 
+and dealt with quite a lot of things today. Smarter log file
+rotation for the assistant, better webapp behavior when git is not
+installed, and a fix for the webdav 5 second timeout problem.
+
+Perhaps the most interesting change is a new `annex.startupscan` setting,
+which can be disabled to prevent the assistant from doing the expensive
+startup scan. This means it misses noticing any files that changed since it
+last run, but this should be useful for those really big repositories.
+
+(Last night, did more work on the test suite, including even more checking
+of merge conflict resolution.)
+
+----
+
+Today's work was sponsored by Michael Alan Dorman.
diff --git a/doc/devblog/day_12__gpg_key_generation.mdwn b/doc/devblog/day_12__gpg_key_generation.mdwn
new file mode 100644
index 0000000000..c79c49f856
--- /dev/null
+++ b/doc/devblog/day_12__gpg_key_generation.mdwn
@@ -0,0 +1,35 @@
+I decided to keep gpg key generation very simple for now. So it generates a
+special-purpose key that is only intended to be used by git-annex. It
+hardcodes some key parameters, like RSA and 4096 bits (maximum recommended
+by gpg at this time). And there is no password on the key, although you can
+of course edit it and set one. This is because anyone who can access the
+computer to get the key can also look at the files in your git-annex
+repository. Also because I can't rely on gpg-agent being installed
+everywhere. All these simplifying assumptions may be revisited later, but
+are enough for now for someone who doesn't know about gpg (so doesn't 
+have a key already) and just wants an encrypted repo on a
+removable drive.
+
+Put together a simple UI to deal with gpg taking quite a while to
+generate a key ...
+
+[[assistant/genkey.png]]
+
+[[assistant/repoinfo.png]]
+
+Then I had to patch git-remote-gcrypt again, to have a per-remote
+signingkey setting, so that these special-purpose keys get used for signing
+their repo.
+
+Next, need to add support for adding an existing gcrypt repo as a remote
+(assuming it's encrypted to an available key). Then, gcrypt repos on ssh
+servers..
+
+----
+
+Also dealt with build breakage caused by a new version of the Haskell DNS
+library.
+
+----
+
+Today's work was sponsored by Joseph Liu.
diff --git a/doc/devblog/day_12__gpg_key_generation/comment_1_48cdfe3bd71fb348ae05fd90e8cf1dab._comment b/doc/devblog/day_12__gpg_key_generation/comment_1_48cdfe3bd71fb348ae05fd90e8cf1dab._comment
new file mode 100644
index 0000000000..d7a6a86313
--- /dev/null
+++ b/doc/devblog/day_12__gpg_key_generation/comment_1_48cdfe3bd71fb348ae05fd90e8cf1dab._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="http://nullroute.eu.org/~grawity/"
+ nickname="Mantas"
+ subject="comment 1"
+ date="2013-09-18T05:00:12Z"
+ content="""
+Should be \"PGP\" or *at least* \"GnuPG\".
+"""]]
diff --git a/doc/devblog/day_130__post_release.mdwn b/doc/devblog/day_130__post_release.mdwn
new file mode 100644
index 0000000000..b6a2af6c8b
--- /dev/null
+++ b/doc/devblog/day_130__post_release.mdwn
@@ -0,0 +1,17 @@
+Release made yesterday, but only finished up the armel build today. 
+And it turns out the OSX build was missing the webapp, so it's also been
+updated today.
+
+Post release bug triage including:
+
+Added a nice piece of UI to the webapp on user request: A "Sync now" menu
+item in the repository for each repo. (The one for the current repo syncs with
+all its remotes.)
+
+Copying files to a git repository on the same computer turns out to have
+had a resource leak issue, that caused 1 zombie process per file. With
+some tricky monad state caching, fixed that, and also eliminated 8% of the work
+done by git-annex in this case.
+
+Fixed `git annex unused` in direct mode to not think that files that were
+deleted out of the work tree by the user still existed and were unused.
diff --git a/doc/devblog/day_131__more_bug_squashing.mdwn b/doc/devblog/day_131__more_bug_squashing.mdwn
new file mode 100644
index 0000000000..3c256f1fbd
--- /dev/null
+++ b/doc/devblog/day_131__more_bug_squashing.mdwn
@@ -0,0 +1,11 @@
+Squashed three or four more bugs today. Unanswered message backlog is down
+to 27.
+
+The most interesting problem today is that the git-repair code was using
+too much memory when `git-fsck` output a lot of problems (300 thousand!). I
+managed to half the memory use in the worst case (and reduced it much more
+in more likely cases). But, don't really feel I can close that bug yet,
+since really big, really badly broken repositories can still run it out of
+memory. It would be good to find a way to reorganize the code so that the
+broken objects list streams through git-repair and never has to all be
+buffered in memory at once. But this is not easy.
diff --git a/doc/devblog/day_132__database_musings.mdwn b/doc/devblog/day_132__database_musings.mdwn
new file mode 100644
index 0000000000..76ce14c32d
--- /dev/null
+++ b/doc/devblog/day_132__database_musings.mdwn
@@ -0,0 +1,17 @@
+Updated the Debian stable backport to the last release. Also it seems that
+the last release unexpectedly fixed XMPP SIGILL on some OSX machines.
+Apparently when I rebuilt all the libraries recently, it somehow fixed that
+[[old_unsolved_bug|bugs/Share_with_friends_crash_in_osx]].
+
+[RichiH](http://richardhartmann.de/) suggested "wrt ballooning memory on
+repair: can you read in broken
+stuff and simply stop reading once you reach a certain threshold, then
+start repairing, re-run fsck, etc?" .. I had considered that but was
+not sure it would work. I think I've gotten it to work.
+
+Now working on a design for using a [[design/caching_database]]
+for some parts of git-annex. My initial benchmarks using SQLite
+indicate it would slow down associated file lookups by nearly an order of
+magnitude compared with the current ".map files" implementation.
+(But would scale better in edge cases). OTOH, using a SQLite
+database to index metadata for use in views looks very promising.
diff --git a/doc/devblog/day_133__db_and_bugfixes.mdwn b/doc/devblog/day_133__db_and_bugfixes.mdwn
new file mode 100644
index 0000000000..5ba1df20e6
--- /dev/null
+++ b/doc/devblog/day_133__db_and_bugfixes.mdwn
@@ -0,0 +1,20 @@
+Did some more exploration and perf tuning and thinking on caching
+databases, and am pretty sure I know how I want to implement it. Will be
+several stages, starting with using it for generating views, and ending(?)
+with using it for direct mode file mappings.
+
+Not sure I'm ready to dive into that yet, so instead spent the rest of the
+day working on small bugfixes and improvements. Only two significant ones..
+
+Made the webapp use a constant time string comparison (from `securemem`)
+to check if its auth token is valid. This could help avoid a potential
+timing attack to guess the auth token, although that is theoretical.
+Just best practice to do this.
+
+Seems that openssh 6.5p1 had another hidden surprise (in addition to
+its now-fixed bug in handing hostnames in `.ssh/config`) -- it broke
+the method git-annex was using for stopping a cached ssh connection,
+which led to some timeouts for failing DNS lookups. If git-annex seems
+to stall for a few seconds at startup/shutdown, that may be why
+(--debug will say for sure). I seem to have found a workaround that
+avoids this problem.
diff --git a/doc/devblog/day_134-135__avoiding_the_turing_tarpit.mdwn b/doc/devblog/day_134-135__avoiding_the_turing_tarpit.mdwn
new file mode 100644
index 0000000000..d18470ae03
--- /dev/null
+++ b/doc/devblog/day_134-135__avoiding_the_turing_tarpit.mdwn
@@ -0,0 +1,18 @@
+Added some power and convenience to [[preferred_content]] expressions.
+
+Before, "standard" was a special case. Now it's a first-class keyword,
+so you can do things like "standard or present" to use the standard
+preferred content expression, modified to also want any file that happens
+to be present.
+
+Also added a way to write your own reusable preferred content expressions,
+tied to groups. To make a repository use them, set its preferred 
+content to "groupwanted". Of course, "groupwanted" is also a first-class
+keyword, so "not groupwanted" or something can also be done.
+
+While I was at it, I made `vicfg` show the built-in standard preferred
+content expressions, for reference. This little IDE should be pretty
+self-explanatory, I hope.
+
+So, preferred content is almost its own little programming language now.
+Except I was careful to not allow recursion. ;)
diff --git a/doc/devblog/day_136__frustrating_day.mdwn b/doc/devblog/day_136__frustrating_day.mdwn
new file mode 100644
index 0000000000..9088093d4a
--- /dev/null
+++ b/doc/devblog/day_136__frustrating_day.mdwn
@@ -0,0 +1,10 @@
+The website broke and I spent several hours fixing it, changing the
+configuration to not let it break like this again, cleaning up after it,
+etc.
+
+Did manage to make a few minor bugfixes and improvements, but nothing
+stunning.
+
+----
+
+I'll be attending LibrePlanet at MIT this weekend.
diff --git a/doc/devblog/day_137-138__bug_triage_and_too_much_windows.mdwn b/doc/devblog/day_137-138__bug_triage_and_too_much_windows.mdwn
new file mode 100644
index 0000000000..28b197356b
--- /dev/null
+++ b/doc/devblog/day_137-138__bug_triage_and_too_much_windows.mdwn
@@ -0,0 +1,15 @@
+Yesterday, worked on cleaning up the todo list. Fixed Windows slash problem
+with rsync remotes. Today, more Windows work; it turns out to have been
+quite buggy in its handling of non-ASCII characters in filenames. Encoding
+stuff is never easy for me, but I eventually managed to find a way to fix
+that, although I think there are other filename encoding problems lurking
+in git-annex on Windows still to be dealt with.
+
+Implemented an interesting metadata feature yesterday. It turns out that
+metadata can have metadata. Particularly, it can be useful to know when a
+field was last set. That was already beeing tracked, internally (to make
+union merging work), so I was able to quite cheaply expose it as
+"$field-lastchanged" metadata that can be used like any other metadata.
+
+I've been thinking about how to implement [[todo/required_content]]
+expressions, and think I have a reasonably good handle on it.
diff --git a/doc/devblog/day_139-140__traveling.mdwn b/doc/devblog/day_139-140__traveling.mdwn
new file mode 100644
index 0000000000..3025e45b5c
--- /dev/null
+++ b/doc/devblog/day_139-140__traveling.mdwn
@@ -0,0 +1,17 @@
+Yesterday coded up one nice improvement on the plane -- `git annex unannex`
+(and `uninit`) is now tons faster. Before it did a git commit after every
+file processed, now there's just 1 commit at the end. This required using
+some locking to prevent the `pre-commit` hook from running in a confusing
+state.
+
+Today. LibrePlanet and a surprising amount of development. I've
+added [[tips/file_manager_integration]], only for Nautilus so far.
+The main part of this was adding --notify-start and --notify-finish, which
+use dbus desktop notifications to provide feedback. 
+
+(Made possible thanks to Max Rabkin for updating
+[fdo-notify](http://hackage.haskell.org/package/fdo-notify) to use the
+new dbus library, and ion for developing the initial Nautilus integration
+scripts.)
+
+Today's work and LibrePlanet visit was sponsored by Jürgen Lüters.
diff --git a/doc/devblog/day_13__encrypted_sneakernet_working.mdwn b/doc/devblog/day_13__encrypted_sneakernet_working.mdwn
new file mode 100644
index 0000000000..777da255a2
--- /dev/null
+++ b/doc/devblog/day_13__encrypted_sneakernet_working.mdwn
@@ -0,0 +1,13 @@
+Spent basically all of today getting the assistant to be able to handle
+gcrypt special remotes that already exist when it's told to add a USB
+drive. This was quite tricky! And I did have to skip handling gcrypt repos
+that are not git-annex special remotes.
+
+Anyway, it's now almost easy to set up an encrypted sneakernet 
+using a USB drive and some computers running the webapp. The only part
+that the assistant doesn't help with is gpg key management.
+
+Plan is to make a release on Friday, and then try to also add support for
+encrypted git repositories on remote servers. Tomorrow I will try to get
+through some of the communications backlog that has been piling up while I
+was head down working on gcrypt.
diff --git a/doc/devblog/day_141__f-droid_sprint.mdwn b/doc/devblog/day_141__f-droid_sprint.mdwn
new file mode 100644
index 0000000000..2a9e30b553
--- /dev/null
+++ b/doc/devblog/day_141__f-droid_sprint.mdwn
@@ -0,0 +1,3 @@
+Attended at the f-droid sprint at LibrePlanet, and have been getting a
+handle on how their build server works with an eye toward adding git-annex
+to it. Not entirely successful getting vagrant to build an image yet.
diff --git a/doc/devblog/day_141__f-droid_sprint/comment_1_1cc76207020ac478747117c76d7b5f9c._comment b/doc/devblog/day_141__f-droid_sprint/comment_1_1cc76207020ac478747117c76d7b5f9c._comment
new file mode 100644
index 0000000000..9971f33da5
--- /dev/null
+++ b/doc/devblog/day_141__f-droid_sprint/comment_1_1cc76207020ac478747117c76d7b5f9c._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="https://www.google.com/accounts/o8/id?id=AItOawmJuOOkYYguRbWhXzxihIPBavxITJIMyww"
+ nickname="Matt"
+ subject="Excellent News"
+ date="2014-03-26T14:14:35Z"
+ content="""
+It will be great to see git-annex on f-droid!
+"""]]
diff --git a/doc/devblog/day_142__digging_out.mdwn b/doc/devblog/day_142__digging_out.mdwn
new file mode 100644
index 0000000000..fc2ceea365
--- /dev/null
+++ b/doc/devblog/day_142__digging_out.mdwn
@@ -0,0 +1,13 @@
+Catching up on conference backlog. 36 messages backlog remains.
+
+Fixed `git-annex-shell configlist` to automatically initialize a
+git remote when a git-annex branch had been pushed to it. This is necessary
+for gitolite to be easy to use, and I'm sure it used to work.
+
+Updated the Debian backport and made a Debian package of the
+fdo-notify haskell library used for notifications.
+
+Applied a patch from Alberto Berti to fix support for tahoe-lafs
+1.10.
+
+And various other bug fixes and small improvements.
diff --git a/doc/devblog/day_143__foolish_hiatus.mdwn b/doc/devblog/day_143__foolish_hiatus.mdwn
new file mode 100644
index 0000000000..f6763dff30
--- /dev/null
+++ b/doc/devblog/day_143__foolish_hiatus.mdwn
@@ -0,0 +1,20 @@
+Last week's trip was productive, but I came home more tired than I
+realized. Found myself being snappy & stressed, so I have been on break.
+
+I did do a little git-annex dev in the past 5 days. On Saturday I
+implemented [[todo/preferred_content]] (although without the active checks
+I think it probably ought to have.) Yesterday I had a long conversation
+with the Tahoe developers about improving git-annex's tahoe integration.
+
+Today, I have been wrapping up [building propellor](http://joeyh.name/code/propellor/).
+To test its docker support, I used propellor to build and deploy a
+container that is a git-annex autobuilder. I'll be replacing the old
+autobuilder setup with this shortly, and expect to also publish docker
+images for git-annex autobuilders, so anyone who wants to can run their
+own autobuilder really easily.
+
+---
+
+I have April penciled in on the roadmap as the month to do telehash.
+I don't know if telehash-c is ready for me yet, but it has had a lot of
+activity lately, so this schedule may still work out!
diff --git a/doc/devblog/day_144__catching_up.mdwn b/doc/devblog/day_144__catching_up.mdwn
new file mode 100644
index 0000000000..52e25f0d58
--- /dev/null
+++ b/doc/devblog/day_144__catching_up.mdwn
@@ -0,0 +1,12 @@
+Got caught up on all recent bugs and questions, although I still have a
+backlog of 27 older things that I really should find time for.
+
+Fixed a couple of bugs. One was that the assistant set up ssh
+`authorized_keys` that didn't work with the fish shell.
+
+Also got caught up on the current state of telehash-c. Have not quite
+gotten it to work, but it seems pretty close to being able to see it do
+something useful for the first time.
+
+Pushing out a release this evening with a good number of changes left over
+from March.
diff --git a/doc/devblog/day_144__catching_up/comment_1_311a7245dd12f1a7e432168d16041348._comment b/doc/devblog/day_144__catching_up/comment_1_311a7245dd12f1a7e432168d16041348._comment
new file mode 100644
index 0000000000..f13ed82ae3
--- /dev/null
+++ b/doc/devblog/day_144__catching_up/comment_1_311a7245dd12f1a7e432168d16041348._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="https://id.koumbit.net/anarcat"
+ ip="70.82.37.38"
+ subject="comment 1"
+ date="2014-04-02T21:28:15Z"
+ content="""
+awesome! can't wait to see telehash land! :)
+"""]]
diff --git a/doc/devblog/day_145__a_plan.mdwn b/doc/devblog/day_145__a_plan.mdwn
new file mode 100644
index 0000000000..5f18ab8064
--- /dev/null
+++ b/doc/devblog/day_145__a_plan.mdwn
@@ -0,0 +1,16 @@
+I have a plan for this month. While waiting for telehash, I am going to
+build [[design/git-remote-daemon]], which is the infrastructure git-annex
+will need, to use telehash. Since it's generalized to support other protocols,
+I'll be able to start using it before telehash is ready.
+
+In fact, I plan to first make it work with ssh:// remotes, where
+it will talk with git-annex-shell on the remote server. This will let the
+assistant immediately know when the server has received a commit, and that
+will simplify using the assistant with a ssh server -- no more need for
+XMPP in this case! It should also work with git-remote-gcrypt encrypted
+repositories, so also covers the case of an untrusted ssh server where
+everything is end-to-end encrypted.
+
+Building the git-annex-shell part of this should be pretty easy, and
+building enough of the [[design/git-remote-daemon]] design to support it
+also not hard.
diff --git a/doc/devblog/day_145__a_plan/comment_1_c0ceea77443be1172527ed8549f000a4._comment b/doc/devblog/day_145__a_plan/comment_1_c0ceea77443be1172527ed8549f000a4._comment
new file mode 100644
index 0000000000..6bcc18bc2a
--- /dev/null
+++ b/doc/devblog/day_145__a_plan/comment_1_c0ceea77443be1172527ed8549f000a4._comment
@@ -0,0 +1,10 @@
+[[!comment format=mdwn
+ username="https://id.koumbit.net/anarcat"
+ ip="2001:1928:1:9::1"
+ subject="look at mosh"
+ date="2014-04-04T14:07:51Z"
+ content="""
+you may want to look at how \"mosh\" handles authentication: http://mosh.mit.edu/
+
+from what i understand, it negociates an authentication token using SSH and then uses that to encrypt and authenticate UDP traffic. seems like similar issues here...
+"""]]
diff --git a/doc/devblog/day_146__halfway_to_git-remote-daemon.mdwn b/doc/devblog/day_146__halfway_to_git-remote-daemon.mdwn
new file mode 100644
index 0000000000..977123f55a
--- /dev/null
+++ b/doc/devblog/day_146__halfway_to_git-remote-daemon.mdwn
@@ -0,0 +1,17 @@
+Added `git-annex-shell notifychanges` command, which uses inotify (etc)
+to detect when git refs have changed, and informs the caller about the
+changes. This was relatively easy to write; I reused the existing inotify
+code, and factored out code for simple line-based protocols from the
+external special remote protocol. Also implemented the git-remote-daemon
+protocol. 200 lines of code total.
+
+Meanwhile, Johan Kiviniemi improved the dbus notifications, making them
+work on Ubuntu and adding icons. Awesome!
+
+There's going to be some fun to get git-annex-shell upgraded so that the
+assistant can use this new notify feaure. While I have not started working
+on the assistant side of this, you can get a jump by installing today's
+upcoming release of git-annex. I had to push this out early because there
+was a bug that prevented the webapp from running on non-gnome systems. Since
+all changes in this release only affected Linux, today's release will be a
+Linux-only release.
diff --git a/doc/devblog/day_147__git-annex_remotedaemon.mdwn b/doc/devblog/day_147__git-annex_remotedaemon.mdwn
new file mode 100644
index 0000000000..0e500ec406
--- /dev/null
+++ b/doc/devblog/day_147__git-annex_remotedaemon.mdwn
@@ -0,0 +1,5 @@
+Built `git-annex remotedaemon` command today. It's buggy, but it already
+works! If you have a new enough git-annex-shell on a remote server, you can
+run "git annex remotedaemon" in a git-annex repository, and it will notice
+any pushes that get made to that remote from any other clone, and pull down
+the changes.
diff --git a/doc/devblog/day_148__too_many_documents.mdwn b/doc/devblog/day_148__too_many_documents.mdwn
new file mode 100644
index 0000000000..3cb5e5baf8
--- /dev/null
+++ b/doc/devblog/day_148__too_many_documents.mdwn
@@ -0,0 +1,8 @@
+Various bug triage today. Was not good for much after shuffling paper for
+the whole first part of the day, but did get a few little things done.
+
+Re , git-annex does not use OpenSSL itself,
+but when using XMPP, the remote server's key could have been intercepted
+using this new technique. Also, the git-annex autobuilds and this website
+are served over https -- working on generating new https certificates now.
+Be safe out there..
diff --git a/doc/devblog/day_149__remote_control_working.mdwn b/doc/devblog/day_149__remote_control_working.mdwn
new file mode 100644
index 0000000000..56c108b7b7
--- /dev/null
+++ b/doc/devblog/day_149__remote_control_working.mdwn
@@ -0,0 +1,15 @@
+[[design/git-remote-daemon]] is tied into the assistant, and working!
+Since it's not really ready yet, this is in the `remotecontrol` branch.
+
+My test case for this is two client repositories, both running
+the assistant. Both have a bare git repository, accessed over ssh,
+set up as their only remote, and no other way to keep in touch with
+one-another. When I change a file in one repository,
+the other one instantly notices the change and syncs.
+
+This is gonna be *awesome*. Much less need for XMPP. Windows will be fully
+usable even without XMPP. Also, most of the work I did today will be fully
+reused when the telehash backend gets built. The telehash-c developer is
+making noises about it being almost ready for use, too!
+
+Today's work was sponsored by Frédéric Schütz.
diff --git a/doc/devblog/day_149__signal.mdwn b/doc/devblog/day_149__signal.mdwn
new file mode 100644
index 0000000000..2bcb01a728
--- /dev/null
+++ b/doc/devblog/day_149__signal.mdwn
@@ -0,0 +1,16 @@
+[[!meta title="day 150 signal"]]
+
+The git-remote-daemon now robustly handles loss of signal, with
+reconnection backoffs. And it detects if the remote ssh server has too
+old a version of git-annex-shell and the webapp will display a warning
+message.
+
+[[!img /assistant/connection.png]]
+
+Also, made the webapp show a network signal bars icon next to both
+ssh and xmpp remotes that it's currently connected with. And, updated the
+webapp's nudging to set up XMPP to now suggest either an XMPP or a ssh remote.
+
+I think that the `remotecontrol` branch is nearly ready for merging!
+
+Today's work was sponsored by Paul Tagliamonte.
diff --git a/doc/devblog/day_14__gcrypt_refinements_and_OOM_fixes.mdwn b/doc/devblog/day_14__gcrypt_refinements_and_OOM_fixes.mdwn
new file mode 100644
index 0000000000..2ff1d16dda
--- /dev/null
+++ b/doc/devblog/day_14__gcrypt_refinements_and_OOM_fixes.mdwn
@@ -0,0 +1,26 @@
+Spent a few hours improving gcrypt in some minor ways, including adding a
+--check option that the assistant can use to find out if a given repo is
+encrypted with dgit, and also tell if the necessary gpg key is available to
+decrypt it. Also merged in a fix to support subkeys, developed by a
+git-annex user who is the first person I've heard from who is using gcrypt.
+I don't want to maintain gcrypt, so I am glad its author has shown up
+again today.
+
+Got mostly caught up on backlog. The main bug I was able to track down
+today is git-annex using a lot of memory in certian repositories. This
+turns out to have happened when a really large file was committed right
+intoo to the git repository (by mistake or on purpose). Some parts of
+git-annex buffer file contents in memory while trying to work out if
+they're git-annex keys. Fixed by making it first check if a file in git is
+marked as a symlink. Which was really hard to do!
+
+At least 4 people ran into this bug, which makes me suspect that lots of
+people are messing up when using direct mode (probably due to not reading
+the documentation, or having `git commit -a` hardwired into their fingers,
+and forcing git to commit large files into their repos, rather than having
+git-annex manage them. Implementing [[todo/direct_mode_guard]] seems more
+urgent now.
+
+----
+
+Today's work was sponsored by Amitai Schlair.
diff --git a/doc/devblog/day_15-17__Android_rebuild.mdwn b/doc/devblog/day_15-17__Android_rebuild.mdwn
new file mode 100644
index 0000000000..758e2d0977
--- /dev/null
+++ b/doc/devblog/day_15-17__Android_rebuild.mdwn
@@ -0,0 +1,67 @@
+Made a release on Friday. But I had to rebuild the OSX and Linux standalone
+builds today to fix a bug in them.
+
+Spent the past **three days** redoing the whole Android build environment.
+I've been progressively moving from my first hacked up Android build env to
+something more reproducible and sane. Finally I am at the point where I can
+run a shell script (well, actually, 3 shell scripts) and get an Android
+build chroot. It's still not immune to breaking when new versions of
+haskell libs are uploaded, but this is much better, and should be
+maintainable going forward. 
+
+This is a good starting point for getting git-annex into the F-Droid app
+store, or for trying to build with a newer version of the Android SDK and
+NDK, to perhaps get it working on Android 4.3. (Eventually. I am so sick
+of building Android stuff right now..)
+
+Friday was all spent struggling to get ghc-android to build. I had not built
+it successfully since February. I finally did,
+on Saturday, and I have made my own fork of it which builds using a
+known-good snapshot of the current development version of ghc. Building
+this in a Debian stable chroot means that there should be no possibility
+that upstream changes will break the build again.
+
+With ghc built, I moved on to building all the haskell libs git-annex
+needs. Unfortunately my build script for these also has stopped working
+since I made it in April. I failed to pin every package at a defined
+version, and things broke.
+
+So, I redid the build script, and updated all the haskell libs to the
+newest versions while I was at it. I have decided not to pin the library
+versions (at least until I find a foolproof way to do it), so this new
+script will break in the future, but it should break in a way I can fix up
+easily by just refreshing a patch.
+
+The new ghc-android build has a nice feature of at least being able to
+compile Template Haskell code (though still not run it at compile time.
+This made the patching needed in the Haskell libs quite a lot less. Offset
+somewhat by me needing to make general fixes to lots of libs to build with
+ghc head. Including some fun with `==#` changing its type from `Bool` to
+`Int#`. In all, I think I removed around 2.5 thousand lines of patches!
+(Only 6 thousand lines to go...)
+
+Today I improved ghc-android some more so it cross builds several C libraries
+that are needed to build several haskell libraries needed for XMPP.
+I had only ever built those once, and done it by hand, and very hackishly.
+Now they all build automatically too.
+
+And, I put together a script that builds the debian stable chroot and
+installs ghc-android.
+
+And, I hacked on the EvilSplicer (which is sadly still needed) to
+work with the new ghc/yesod/etc.
+
+At this point, I have git-annex successfully building, including the APK!
+
+----
+
+In a bored hour waiting for a compile, I also sped up `git annex add`
+on OSX by I think a factor of 10. Using cryptohash for hash calculation
+now, when external hash programs are not available. It's still a few
+percentage points slower than external hash programs, or I'd use it by
+default.
+
+----
+
+This period of important drudgery was sponsored by an unknown bitcoin
+user, and by Bradley Unterrheiner and Andreas Olsson.
diff --git a/doc/devblog/day_151__birthday_bug.mdwn b/doc/devblog/day_151__birthday_bug.mdwn
new file mode 100644
index 0000000000..251bfb935f
--- /dev/null
+++ b/doc/devblog/day_151__birthday_bug.mdwn
@@ -0,0 +1,18 @@
+Pushed out a new release today, fixing two important bugs, followed by a
+second release which fixed the bugs harder.
+
+Automatic upgrading was broken on OSX. The webapp will tell you upgrading
+failed, and you'll need to manually download the .dmg and install it.
+
+With help from Maximiliano Curia, finally tracked down a bug I have been
+chasing for a while where the assistant would start using a lot of CPU
+while not seeming to be busy doing anything. Turned out to be triggered by
+a scheduled fsck that was configured to run once a month with no particular
+day specified.
+
+That bug turned out to affect users who first scheduled such a fsck job
+after the 11th day of the month. So I expedited putting a release out to
+avoid anyone else running into it starting tomorrow.
+
+(Oddly, the 11th day of this month also happens to be my birthday. I did not
+expect to have to cut 2 releases today..)
diff --git a/doc/devblog/day_152__more_ssh_connection_caching.mdwn b/doc/devblog/day_152__more_ssh_connection_caching.mdwn
new file mode 100644
index 0000000000..ad472b5e55
--- /dev/null
+++ b/doc/devblog/day_152__more_ssh_connection_caching.mdwn
@@ -0,0 +1,37 @@
+Made ssh connection caching be used in several more places. `git annex
+sync` will use it when pushing/pulling to a remote, as will the assistant.
+And `git-annex remotedaemon` also uses connection caching. So, when
+a push lands on a ssh remote, the assistant will immediately notice it, and
+pull down the change over the same TCP connection used for the
+notifications.
+
+This was a bit of a pain to do. Had to set `GIT_SSH=git-annex` and then
+when git invokes git-annex as ssh, it runs ssh with the connection caching
+parameters.
+
+Also, improved the network-manager and wicd code, so it detects when a
+connection has gone down. That propagates through to the remote-daemon,
+which closes all ssh connections. I need to also find out how to detect
+network connections/disconnections on OSX..
+
+Otherwise, the remote-control branch seems ready to be merged. But I want
+to test it for a while first.
+
+----
+
+Followed up on yesterday's bug with writing some test cases for
+Utility.Scheduled, which led to some more bug fixes. Luckily nothing
+I need to rush out a release over. In the end, the code got a lot
+simpler and clearer.
+
+[[!format haskell """
+-- Check if the new Day occurs one month or more past the old Day.
+oneMonthPast :: Day -> Day -> Bool
+new `oneMonthPast` old = fromGregorian y (m+1) d <= new
+  where
+        (y,m,d) = toGregorian old
+"""]]
+
+-------
+
+Today's work was sponsored by Asbjørn Sloth Tønnesen.
diff --git a/doc/devblog/day_153__remotedaemon_has_landed.mdwn b/doc/devblog/day_153__remotedaemon_has_landed.mdwn
new file mode 100644
index 0000000000..5033b32b25
--- /dev/null
+++ b/doc/devblog/day_153__remotedaemon_has_landed.mdwn
@@ -0,0 +1,10 @@
+After fixing a few bugs in the `remotecontrol` branch, It's landed in
+`master`. Try a daily build today, and see if the assistant can keep in
+sync using nothing more than a remote ssh repository!
+
+So, now all the groundwork for telehash is laid too. I only need a
+telehash library to start developing on top of. Development on telehash-c
+is continuing, but I'm more excited that
+[htelehash](https://github.com/alanz/htelehash/tree/v2) 
+has been revived and is being updated to the v2 protocol, seemingly quite
+quickly.
diff --git a/doc/devblog/day_153__remotedaemon_has_landed/comment_1_f19ae6b3d6f33a68e4ffe0c32f788745._comment b/doc/devblog/day_153__remotedaemon_has_landed/comment_1_f19ae6b3d6f33a68e4ffe0c32f788745._comment
new file mode 100644
index 0000000000..5ff090ffde
--- /dev/null
+++ b/doc/devblog/day_153__remotedaemon_has_landed/comment_1_f19ae6b3d6f33a68e4ffe0c32f788745._comment
@@ -0,0 +1,13 @@
+[[!comment format=mdwn
+ username="https://www.google.com/accounts/o8/id?id=AItOawkC0W3ZQERUaTkHoks6k68Tsp1tz510nGo"
+ nickname="Georg"
+ subject=" remotedaemon in pre-built tarballs"
+ date="2014-04-17T07:40:25Z"
+ content="""
+Hi Joey,
+
+can you tell me when the pre-built Linux tarballs will include the remotedaemon?
+Are they updated on a daily basis?
+
+Best regards, Georg
+"""]]
diff --git a/doc/devblog/day_153__remotedaemon_has_landed/comment_2_fbf0c50f772e958af638d2b72dac73f5._comment b/doc/devblog/day_153__remotedaemon_has_landed/comment_2_fbf0c50f772e958af638d2b72dac73f5._comment
new file mode 100644
index 0000000000..5f81fde5cf
--- /dev/null
+++ b/doc/devblog/day_153__remotedaemon_has_landed/comment_2_fbf0c50f772e958af638d2b72dac73f5._comment
@@ -0,0 +1,10 @@
+[[!comment format=mdwn
+ username="http://joeyh.name/"
+ ip="209.250.56.191"
+ subject="comment 2"
+ date="2014-04-17T19:31:30Z"
+ content="""
+The daily builds are updated on a daily basis, so already include it.
+
+The release builds are updated on each release, and so do not.
+"""]]
diff --git a/doc/devblog/day_154__catching_up.mdwn b/doc/devblog/day_154__catching_up.mdwn
new file mode 100644
index 0000000000..7c49a788e2
--- /dev/null
+++ b/doc/devblog/day_154__catching_up.mdwn
@@ -0,0 +1,13 @@
+Worked through message backlog today. Got it down from around 70 to just
+37. Was able to fix some bugs, including making the webapp start up more
+robustly in some misconfigurations.
+
+Added a new `findref` command which may be useful in a git `update` hook to
+deny pushes of refs if the annexed content has not been sent first.
+
+----
+
+BTW, I also added a new `reinit` command a few days ago, which can be
+useful if you're cloning back a deleted repository.
+
+Also a few days ago, I made `uninit` a *lot* faster.
diff --git a/doc/devblog/day_155__missing_bits.mdwn b/doc/devblog/day_155__missing_bits.mdwn
new file mode 100644
index 0000000000..aa8fd9d4ee
--- /dev/null
+++ b/doc/devblog/day_155__missing_bits.mdwn
@@ -0,0 +1,27 @@
+Sometimes you don't notice something is missing for a long time until
+it suddenly demands attention. Like today.
+
+Seems the webapp never had a way to stop using XMPP and delete the XMPP
+password. So I added one.
+
+The new support for instantly noticing changes on a ssh remote forgot to
+start up a connection to a new remote after it was created. Fixed that.
+
+(While doing some testing on Android for unrelated reasons, I noticed that
+my android tablet was pushing photos to a ssh server and my laptop
+immediately noticed and downloaded them from tere, which is an excellent
+demo. I will deploy this on my trip in Brazil next week. Yes, I'm spending
+2 weeks in Brazil with git-annex users; more on this later.)
+
+Finally, it turns out that "installing" git-annex from the standalone
+tarball, or DMG, on a server didn't make it usable by the webapp. Because
+git-annex shell is not in PATH on the server, and indeed git and rsync may
+not be in PATH either if they were installed with the git-annex bundle.
+Fixed this by making the bundle install a ~/.ssh/git-annex-wrapper, which 
+the webapp will detect and use.
+
+Also, quite a lot of other bug chasing activity.
+
+----
+
+Today's work was sponsored by Thomas Koch.
diff --git a/doc/devblog/day_155__missing_bits/comment_1_76424498600ba603946035efffb88023._comment b/doc/devblog/day_155__missing_bits/comment_1_76424498600ba603946035efffb88023._comment
new file mode 100644
index 0000000000..d9ee528e0a
--- /dev/null
+++ b/doc/devblog/day_155__missing_bits/comment_1_76424498600ba603946035efffb88023._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="https://id.koumbit.net/anarcat"
+ ip="72.0.72.144"
+ subject="comment 1"
+ date="2014-04-20T23:58:51Z"
+ content="""
+thanks!
+"""]]
diff --git a/doc/devblog/day_156__release_day.mdwn b/doc/devblog/day_156__release_day.mdwn
new file mode 100644
index 0000000000..7a312b234e
--- /dev/null
+++ b/doc/devblog/day_156__release_day.mdwn
@@ -0,0 +1,14 @@
+I hope this will be a really good release. Didn't get all the way to
+[[design/assistant/telehash]] this month, but the remotedaemon is pretty sweet. Updated [[design/roadmap]]
+pushes telehash back again.
+
+The files in this release are now gpg signed, after recently moving the
+downloads site to a dedicated server, which has a dedicated gpg key.
+You can verify the detached signatures as an additional security check
+over trusting SSL. The automatic upgrade code doesn't check the gpg
+signatures yet.
+
+Sören Brunk has ported the webapp to Bootstrap 3.
+  
+The branch is not ready for merging yet (it would break the Debian stable
+backports), but that was a nice surprise.
diff --git a/doc/devblog/day_157__upgrade_checking.mdwn b/doc/devblog/day_157__upgrade_checking.mdwn
new file mode 100644
index 0000000000..ff70b80939
--- /dev/null
+++ b/doc/devblog/day_157__upgrade_checking.mdwn
@@ -0,0 +1,22 @@
+Now git-annex's self-upgrade code will check the gpg signature of a
+new version before using it.
+
+To do this I had to include the gpg public keys into the
+git-annex distribution, and that raised the question of which public keys
+to include. Currently I have both the dedicated git-annex distribution
+signing key, and my own gpg key as a backup in case I somehow misplace the
+former.
+
+Also spent a while looking at the recent logs on the web server. There
+seem to be around 600 users of the assistant with
+upgrade checking enabled. That breaks down to 68% Linux amd64, 20% Linux
+i386, 11% OSX Mavericks, and 0.5% OSX Lion.
+
+Most are upgrading successfully, but there are a few that seem to
+repeatedly fail for some reason. (Not counting the OSX Lion, which will
+probably never find an upgrade available.) I hope that someone who is
+experiencing an upgrade failure gets in touch with some debug logs.
+
+In the same time period, around 450 unique hosts manually downloaded a
+git-anex distribution. Also compare with Debian popcon, which has 1200
+reporting git-annex users.
diff --git a/doc/devblog/day_158__enroute_to_Brazil.mdwn b/doc/devblog/day_158__enroute_to_Brazil.mdwn
new file mode 100644
index 0000000000..cd9830f2ee
--- /dev/null
+++ b/doc/devblog/day_158__enroute_to_Brazil.mdwn
@@ -0,0 +1,10 @@
+Next month the roadmap has me working on [[design/assistant/sshpassword]].
+That will be a nice UI improvement and I'd be very surprised if it takes
+more than a week, which is great.
+
+Getting a jump on it today, investigating using `SSH_ASKPASS`. It seems this
+will even work on Windows! Preliminary design in [[design/assistant/sshpassword]].
+
+Time to get on a plane to a plane to a plane to Brasilia!
+
+[[!meta date="Fri, 25 Apr 2014 16:32:36 -0400"]]
diff --git a/doc/devblog/day_159__tmp_file_cleanup.mdwn b/doc/devblog/day_159__tmp_file_cleanup.mdwn
new file mode 100644
index 0000000000..d1ef58109f
--- /dev/null
+++ b/doc/devblog/day_159__tmp_file_cleanup.mdwn
@@ -0,0 +1,12 @@
+Today was mostly spent driving across Brazil, but I had energy this evening
+for a little work on git-annex.
+
+Made the assistant delete old temporary files on startup. I've had
+scattered reports of a few users whose `.git/annex/tmp` contained many
+files, apparently put there by the assistant when it locks down a file
+prior to annexing it. That seems it could possibly be a bug -- or it could
+just be unclean shutdowns interrupting the assistant. Anyway, this will
+deal with any source of tmp cruft, and I made sure to preserve
+tmp files for partially downloaded content.
+
+[[!meta date="Sun, 27 Apr 2014 22:12:55 -0300"]]
diff --git a/doc/devblog/day_160__.mdwn b/doc/devblog/day_160__.mdwn
new file mode 100644
index 0000000000..2c20d2f0d8
--- /dev/null
+++ b/doc/devblog/day_160__.mdwn
@@ -0,0 +1,20 @@
+Reviewed Sören's updated bootstrap3 patch, which appeared while I was
+[traveling](http://joeyh.name/blog/entry/the_real_Brazil/). Sören
+kindly fixed it to work with Debian stable's old version of Yesod,
+which was quite a lot of work. The new new bootstrap3 UI looks nice,
+found a few minor issues, but expect to be able to merge it soon.
+
+Started on [[design/assistant/sshpassword]] groundwork. Added a simple
+password cache to the assistant, with automatic expiration, and made
+git-annex be able to be run by ssh as the `SSH_ASKPASS` program.
+
+The main difficulty will be changing the webapp's UI to prompt for the ssh
+password when one is needed. There are several code paths in ssh remote
+setup where a password might be needed. Since the cached password expires,
+it may need to be prompted for at any of those points. Since a new page is
+loading, it can't pop up a prompt on the current page; it needs to redirect
+to a password prompt page and then redirect back to the action that needed
+the password. ...At least, that's one way to do it.
+I'm going to sleep on it and hope I dream up a better way.
+
+[[!meta date="Tue, 29 Apr 2014 18:33:53 -0400"]]
diff --git a/doc/devblog/day_161__routing_design.mdwn b/doc/devblog/day_161__routing_design.mdwn
new file mode 100644
index 0000000000..9c9e65d171
--- /dev/null
+++ b/doc/devblog/day_161__routing_design.mdwn
@@ -0,0 +1,13 @@
+I've moved out of implementation mode (unable to concentrate enough), and
+into high-level design mode.
+
+[[Syncing efficiency|design/assistant/syncing/efficiency]] has been an open TODO for years,
+to find a way to avoid flood filling the network, and find more efficient
+ways to ensure data only gets to the nodes that want it. Relatedly,
+Android devices  often need a way to mark individual files they want to have.
+Had a [very productive discussion with Vince and Fernao](http://joeyh.name/blog/entry/who_needs_whiteboards_when_you_have_strange_seed_pods_from_the_jungle/)
+and I think we're heading toward a design that will address both these
+needs, as well as some more Brazil-specific use cases, about which more
+later.
+
+Today's work was sponsored by Casa do Boneco.
diff --git a/doc/devblog/day_162__routing_sim_and_massive_contribution_landed.mdwn b/doc/devblog/day_162__routing_sim_and_massive_contribution_landed.mdwn
new file mode 100644
index 0000000000..a8f5313ac4
--- /dev/null
+++ b/doc/devblog/day_162__routing_sim_and_massive_contribution_landed.mdwn
@@ -0,0 +1,17 @@
+Sören Brunk's massive bootstrap 3 patch has landed! This is a 43 thousand
+line diff, with 2 thousand lines after the javascript and CSS libraries are
+filtered out. Either way, the biggest patch contributed by anyone to
+git-annex so far, and excellent work.
+
+Meanwhile, I built a [[haskell program|design/assistant/syncing/simroutes.hs]] to simulate
+a network of highly distributed git-annex nodes with ad-hoc connections and
+the selective file syncing algorythm now documented at the bottom of
+[[design/assistant/syncing/efficiency]].
+
+Currently around 33% of requested files never get to their destination
+in this simulation, but this is probably because its network is randomly
+generated, and so contains disconnected islands. So next, some data entry,
+from a map that involves an Amazon not in .com, dotted with names of people
+I have recently met... :)
+
+
diff --git a/doc/devblog/day_163__request_and_routing_design.mdwn b/doc/devblog/day_163__request_and_routing_design.mdwn
new file mode 100644
index 0000000000..c11664a06f
--- /dev/null
+++ b/doc/devblog/day_163__request_and_routing_design.mdwn
@@ -0,0 +1,3 @@
+I have a preliminary design for [[design/requests_routing]]. Won't be
+working on it immediately, but simulations show it can work well in a large
+ad-hoc network.
diff --git a/doc/devblog/day_164__back.mdwn b/doc/devblog/day_164__back.mdwn
new file mode 100644
index 0000000000..48a6ce6f07
--- /dev/null
+++ b/doc/devblog/day_164__back.mdwn
@@ -0,0 +1,17 @@
+My backlog is massive -- 181 items to answer. Will probably take the rest
+of the month to get caught back up. Rather than digging into that yet,
+spent today working on the webapp's ssh password prompting. 
+
+I simplified it so the password is entered on the same form as the rest
+of the server's information. That made the UI easy to build, but
+means that when a user already has a ssh key they want to use, they need
+to select "existing ssh key"; the webapp no longer probes to automatically
+detect that case.
+
+Got the ssh password prompting in the webapp basically working, and it's a
+*really* nice improvement! I even got it to work on Windows (eventually...).
+It's still only in the `sshpassword` branch, since I need to test it more
+and probably fix some bugs. In particular, when enabling a remote that
+already exists, I think it never prompts for the password yet.
+
+Today's work was sponsored by Nicola Chiapolini.
diff --git a/doc/devblog/day_165__sshpassword_merged.mdwn b/doc/devblog/day_165__sshpassword_merged.mdwn
new file mode 100644
index 0000000000..4479df733e
--- /dev/null
+++ b/doc/devblog/day_165__sshpassword_merged.mdwn
@@ -0,0 +1,18 @@
+Spent the day testing the sshpasswd branch. A few interesting things:
+
+* I was able to get rid of 10 lines of Windows specific code for
+  rsync.net, which had been necessary for console ssh password prompting to
+  work. Yay!
+* git-remote-gcrypt turned out to be broken when there is no controlling
+  tty. --no-tty has to be passed to gpg to avoid it falling over in this
+  case, even when a gpg agent is available to be used. I fixed this with
+  a new release of git-remote-gcrypt.
+
+Mostly the new branch just worked! And is merged...
+
+Merged a patch from Robie Basak that adds a new special remote that's sort
+of like bup but supports deletion: [[special_remotes/ddar]]
+
+Backlog: 172
+
+Today's work was sponsored by Andrew Cant.
diff --git a/doc/devblog/day_166__catching_up.mdwn b/doc/devblog/day_166__catching_up.mdwn
new file mode 100644
index 0000000000..51878562e9
--- /dev/null
+++ b/doc/devblog/day_166__catching_up.mdwn
@@ -0,0 +1,40 @@
+Powered through the backlog today, and got it down to 67! Probably most of
+the rest is the hard ones though.
+
+A theme today was: It's stupid hard to get git-annex-shell installed into
+PATH. While that should be the simplest thing in the world, I'm pinned
+between two problems:
+
+1. There's no single portable package format, so all the decades of
+   development nice ways to get things into PATH don't work for everybody.
+2. bash provides not a single dotfile that will work in all circumstances
+   to configure PATH. In particular, "ssh $host git-annex-shell" causes bash
+   to helpfully avoid looking at any dotfiles at all.
+
+Today's flailing to work around that inluded:
+
+* Merged a patch from Fraser Tweedale to allow `git config remote.origin.annex-shell /not/in/path/git-annex-shell`
+* Merged a patch from Justin Lebar to allow symlinking the git-annex-shell
+  etc from the standalone tarball to a directory that is in PATH. (Only
+  on Linux, not OSX yet.)
+* Improved the warning message git-annex prints when a remote server does
+  not have git-annex-shell in PATH, suggesting some things the user could
+  do to try to fix it.
+
+I've found out why OSX machines were retrying upgrades repeatedly. The
+version in the .info file did not match the actual git-annex version for
+OSX. I've fixed the info file version, but will need to come up with a
+system to avoid such mismatches.
+
+Made a few other fixes. A notable one is that dragging and dropping
+repositories in the webapp to reorder the list (and configure costs)
+had been broken since November.
+
+git-annex 5.20140421 finally got into Debian testing today, so I updated
+the backport. I recommend upgrading, especially if you're using the
+assistant with a ssh remote, since you'll get all of last month's nice
+features that make XMPP unnecessary in that configuration.
+
+----
+
+Today's work was sponsored by Geoffrey Irving.
diff --git a/doc/devblog/day_167__growing_the_community_of_git-annex_contributors.mdwn b/doc/devblog/day_167__growing_the_community_of_git-annex_contributors.mdwn
new file mode 100644
index 0000000000..1408e13e2c
--- /dev/null
+++ b/doc/devblog/day_167__growing_the_community_of_git-annex_contributors.mdwn
@@ -0,0 +1,11 @@
+Released git-annex 5.20140517 today. The changelog for this release
+is very unusual, because it's full of contributions from others! There are
+as many patches from others in this release as git-annex got in the first
+entire two years of its existence.
+
+I'd like to keep that going. Also, I could really use help triaging bug
+reports right now. So I have updated the **[[contribute]]** page with more
+info about easy ways to contribute to git-annex. If you read this devblog,
+you're an ideal contributor, and you don't need to know how to write
+haskell either.. So take a [[look at the page|contribute]] and see if
+you can help out.
diff --git a/doc/devblog/day_168__backlog_continued.mdwn b/doc/devblog/day_168__backlog_continued.mdwn
new file mode 100644
index 0000000000..ab19565338
--- /dev/null
+++ b/doc/devblog/day_168__backlog_continued.mdwn
@@ -0,0 +1,16 @@
+Worked on triaging several bugs. Fixed an easy one, which involved the
+assistant choosing the wrong path to a repository that has multiple
+remotes. After today, backlog is down to 43, nearly pre-Brazil levels.
+
+It seems that git-remote-gcrypt [[never quite worked on OSX|bugs/remote_gcrypt_based_repos_and_conflicting_uuids]].
+It looked like it did, but a bug prevented anything being pushed to the remote.
+Tracked down and fixed that bug.
+
+This evening, getting back to working on the armel autobuilder setup
+using propellor. The autobuilder will use a pair of docker containers, one
+armel and a companion amd64, and their quite complex setup will be *almost*
+fully automated (except for the haskell library patching part).
+
+----
+
+Today's work was sponsored by Mica Semrick.
diff --git a/doc/devblog/day_169-171__juggling.mdwn b/doc/devblog/day_169-171__juggling.mdwn
new file mode 100644
index 0000000000..2b34da90ac
--- /dev/null
+++ b/doc/devblog/day_169-171__juggling.mdwn
@@ -0,0 +1,14 @@
+Keeping lots of things going these past few days..
+
+* Rebootstrapping the armel autobuilder with propellor. 
+  Some qemu instability and the need to update haskell
+  library patches meant this took a lot of hand-holding. Finally got a
+  working setup today.
+* Designing and ordering new git-annex stickers on clear viynl backing;
+  have put off sending those to campaign contributors for too long.
+* Added a new feature to the webapp: It now remembers the ssh remotes
+  that it sets up, and makes it easy to enable them elsewhere, the same
+  as other sorts of remotes. Had a very pleasant surprise building this,
+  when I was able to reuse all the UI code for enabling rsync and gcrypt
+  remotes. I think this will be a useful feature as we transition away
+  from XMPP.
diff --git a/doc/devblog/day_172__.mdwn b/doc/devblog/day_172__.mdwn
new file mode 100644
index 0000000000..7f2595cef9
--- /dev/null
+++ b/doc/devblog/day_172__.mdwn
@@ -0,0 +1,7 @@
+Working on moving the android autobuilder to Docker & Propellor, which will
+finish containerizing all the autobuilds that I run. Updated
+ghc-android to use the released ghc 7.8.2, which will make it build more
+reliably. 
+
+Also did bug triage. Bugs are now divided into [[bugs/confirmed]]
+and [[bugs/unconfirmed]] categories.
diff --git a/doc/devblog/day_173-174__android_rebootstrap.mdwn b/doc/devblog/day_173-174__android_rebootstrap.mdwn
new file mode 100644
index 0000000000..6029b1870f
--- /dev/null
+++ b/doc/devblog/day_173-174__android_rebootstrap.mdwn
@@ -0,0 +1,8 @@
+With some help from Sören, have been redoing the android build environment
+for git-annex. This included making propellor put it in a docker container,
+which was easy. But then much struggling with annoying stuff like getting
+the gnutls linking to work, and working around some dependency issues on
+hackage that make cabal's dependency resolver melt down.
+Finally succeeded after much more time than I had wanted to spend on this.
+
+[[!meta date="Mon May 27 16:36:40 JEST 2014"]]
diff --git a/doc/devblog/day_175__encoding_day.mdwn b/doc/devblog/day_175__encoding_day.mdwn
new file mode 100644
index 0000000000..5e23342d5c
--- /dev/null
+++ b/doc/devblog/day_175__encoding_day.mdwn
@@ -0,0 +1,20 @@
+These themed days are inaverdent, but it happened again: Nearly everything
+done today had to do with encoding issues.
+
+The big news is that it turned out everything written to files in the
+git-annex branch had unicode characters truncated to 8 bits. Now fixed so
+you should always get out the same thing you put in, no matter what
+encoding you use (but please use utf8). This affected things like storing
+repository descriptions, but worse, it affected metadata. (Also preferred
+content expressions, I suppose.)
+
+With that fixed, there are still 7 source files left that use Char8 libraries.
+There used to be more; nearly every use of those is a bug. I looked over
+the remaining uses of it, and there might be a problem with Creds using it.
+I should probably make a push to stamp out all remaining uses of Char8.
+
+Other encoding bugs were less reproducible.
+
+And just now, Sören made some progress on
+[[bugs/Bootstrap3_icons_missing_on_Android]] ... and my current theory
+is this is actually caused by an encoding issue too.
diff --git a/doc/devblog/day_176__mostly_a_day_off.mdwn b/doc/devblog/day_176__mostly_a_day_off.mdwn
new file mode 100644
index 0000000000..46bf1c4db1
--- /dev/null
+++ b/doc/devblog/day_176__mostly_a_day_off.mdwn
@@ -0,0 +1,8 @@
+Got a handle on the Android webapp static file problems (no, they were not
+really encoding problems!), and hopefully
+that's all fixed now. Also, only 3 modules use Char8 now. And updated the
+git-annex backport. That's all I did today.
+
+Meanwhile, a complete [[tips/ZSH_completion]] has been contributed by
+Schnouki. And, Ben Gamari sent in a patch moving from the deprecated 
+MonadCatchIO-transformers library to the exceptions library.
diff --git a/doc/devblog/day_177__enabling.mdwn b/doc/devblog/day_177__enabling.mdwn
new file mode 100644
index 0000000000..55a1895aaf
--- /dev/null
+++ b/doc/devblog/day_177__enabling.mdwn
@@ -0,0 +1,21 @@
+After making a release yesterday, I've been fixing some bugs in the
+webapp, all to do with repository configuration stored on the git-annex
+branch. I was led into this by a strange little bug where the webapp stored
+configuration in the wrong repo in one situation. From there, I noticed
+that often when enabling an existing repository, the webapp would stomp on
+its group and preferred content and description, replacing them with
+defaults.
+
+This was a systematic problem, it had to be fixed in several places. And
+some of the fixes were quite tricky. For example, when adding a ssh
+repository, and it turns out there's already a git-annex repository at the
+entered location, it needs to avoid changing its configuration. But also,
+the configuration of that repo won't be known until after the first git
+pull from it. So it doesn't make sense to show the repository edit form
+after enabling such a repository.
+
+Also worked on a couple other bugs, and further cleaned up the [[bugs]]
+page. I think I am finally happy with how the bug list is displayed,
+with confirmed/moreinfo/etc tags.
+
+Today's work was sponsored by François Deppierraz.
diff --git a/doc/devblog/day_177__enabling/comment_1_820d29f84dade09b0e7bb7435c52fcb8._comment b/doc/devblog/day_177__enabling/comment_1_820d29f84dade09b0e7bb7435c52fcb8._comment
new file mode 100644
index 0000000000..05ecdc32d1
--- /dev/null
+++ b/doc/devblog/day_177__enabling/comment_1_820d29f84dade09b0e7bb7435c52fcb8._comment
@@ -0,0 +1,12 @@
+[[!comment format=mdwn
+ username="https://www.google.com/accounts/o8/id?id=AItOawl9sYlePmv1xK-VvjBdN-5doOa_Xw-jH4U"
+ nickname="Richard"
+ subject="comment 1"
+ date="2014-06-02T13:36:00Z"
+ content="""
+Can you make the tags be displayed with background colours or some other mechanism for quick visual recognition? All \"done\" are green, all \"moreinfo\" blue(?), etc pp.
+
+
+Thanks,
+Richard
+"""]]
diff --git a/doc/devblog/day_178-179__screencast_and_what_next.mdwn b/doc/devblog/day_178-179__screencast_and_what_next.mdwn
new file mode 100644
index 0000000000..fe7ba399da
--- /dev/null
+++ b/doc/devblog/day_178-179__screencast_and_what_next.mdwn
@@ -0,0 +1,12 @@
+Yesterday I recorded a new screencast, demoing using the assistant on a
+local network with a small server. [[videos/git-annex_assistant_lan]].
+That's the best screencast yet; having a real framing story was nice;
+recent improvements to git-annex are taken advantage of without being made
+a big deal; and audio and video are improved. (But there are some minor
+encoding glitches which I'd have to re-edit it to fix.)
+
+The [[design/roadmap]] has this month dedicated to improving Android.
+But I think what I'd more like to do is whatever makes the assistant usable
+by the most people. This might mean doing more on Windows, since I hear
+from many who would benefit from that. Or maybe something not related to
+porting?
diff --git a/doc/devblog/day_178-179__screencast_and_what_next/comment_1_eeba788fed45cb22f9cc2a738ceaa074._comment b/doc/devblog/day_178-179__screencast_and_what_next/comment_1_eeba788fed45cb22f9cc2a738ceaa074._comment
new file mode 100644
index 0000000000..d5bb0a1833
--- /dev/null
+++ b/doc/devblog/day_178-179__screencast_and_what_next/comment_1_eeba788fed45cb22f9cc2a738ceaa074._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="https://www.google.com/accounts/o8/id?id=AItOawm8BAEUyzYhORZmMuocRTk4M-3IumDm5VU"
+ nickname="luciusf0"
+ subject="Windows Support +1"
+ date="2014-06-06T21:54:38Z"
+ content="""
+I just also (like others) wanna state my strong interest in a windows port. Actually it's the only thing which is holding me back of using git-annex for real, as I live and work in a mixed-OS environment. The git-annex features would be the answers to an awful lot of our day-to-day problems, and I cannot wait to have it ready for a production environment. So -> Windows Port +1
+"""]]
diff --git a/doc/devblog/day_178-179__screencast_and_what_next/comment_2_d44e67e34615c7b00e29f307556cdd06._comment b/doc/devblog/day_178-179__screencast_and_what_next/comment_2_d44e67e34615c7b00e29f307556cdd06._comment
new file mode 100644
index 0000000000..defb4a80ff
--- /dev/null
+++ b/doc/devblog/day_178-179__screencast_and_what_next/comment_2_d44e67e34615c7b00e29f307556cdd06._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="https://www.google.com/accounts/o8/id?id=AItOawnBJ6Dv1glxzzi4qIzGFNa6F-mfHIvv9Ck"
+ nickname="Jim"
+ subject="Android"
+ date="2014-06-08T02:44:45Z"
+ content="""
+As I don't use Windows and git-annex is already fitting my Linux use cases perfectly, I'd have to vote for Android.  FWIW, I have previously posted some thoughts on what would I see as lacking on Android [here](http://git-annex.branchable.com/forum/Controlling_content_on_mobile_device/).
+"""]]
diff --git a/doc/devblog/day_180__porting.mdwn b/doc/devblog/day_180__porting.mdwn
new file mode 100644
index 0000000000..27b4eaca1c
--- /dev/null
+++ b/doc/devblog/day_180__porting.mdwn
@@ -0,0 +1,13 @@
+Did work on Windows porting today. First, fixed a reversion in the last
+release, that broke the git-annex branch pretty badly on Windows, causing
+\r to be written to files on that branch that should never have DOS line
+endings. Second, fixed a long-standing bug that prevented getting a file
+from a local bare repository on Windows.
+
+Also refreshed all autobuilders to deal with the gnutls and openssl
+security holes-of-the-week. (git-annex uses gnutls only for XMPP,
+and does not use openssl itself, but a few programs bundled with it,
+like curl, do use openssl.)
+
+A nice piece of news: OSX Homebrew now contains git-annex, so it can be
+easily installed with `brew install git-annex`
diff --git a/doc/devblog/day_180__porting/comment_1_133875f4435a298b85ddfb8a2cc11a7a._comment b/doc/devblog/day_180__porting/comment_1_133875f4435a298b85ddfb8a2cc11a7a._comment
new file mode 100644
index 0000000000..2aa67e8233
--- /dev/null
+++ b/doc/devblog/day_180__porting/comment_1_133875f4435a298b85ddfb8a2cc11a7a._comment
@@ -0,0 +1,9 @@
+[[!comment format=mdwn
+ username="Tamschi"
+ ip="178.27.213.202"
+ subject="It's still impossible to use the git-annex repository on Windows."
+ date="2014-06-09T14:44:07Z"
+ content="""
+Since the bug tracker is in the same repository and branch as the main program (why?), any bugs submitted with a colon in the title will prevent access on NTFS file systems.  
+There is at least one such instance, I assume it potentially affects blog posts too.
+"""]]
diff --git a/doc/devblog/day_181__tricky_merge.mdwn b/doc/devblog/day_181__tricky_merge.mdwn
new file mode 100644
index 0000000000..c6bcaf8d42
--- /dev/null
+++ b/doc/devblog/day_181__tricky_merge.mdwn
@@ -0,0 +1,8 @@
+Spent most of today improving behavior when a sync or merge is
+interrupted in direct mode. It was possible for an interrupt at the wrong
+time to leave the merge committed, but the work tree not yet updated. And
+then the next sync would make a commit that reverted the merged changes!
+
+To fix this I had to avoid making any merge commit or indeed updating the
+index until after the work tree is updated. It looked intractable for a
+while; I'm still surprised I eventually succeeded.
diff --git a/doc/devblog/day_182__service.mdwn b/doc/devblog/day_182__service.mdwn
new file mode 100644
index 0000000000..1377252487
--- /dev/null
+++ b/doc/devblog/day_182__service.mdwn
@@ -0,0 +1,6 @@
+Have for the first time gotten git-annex to run as a proper Windows
+service, using nssm.
+([details](http://git-annex.branchable.com/todo/windows_support/#comment-a61be55862ea32e3dc30972f905bb987))
+Not quite ready yet though; doesn't run as the right user.
+
+And a few other windows porting bits.
diff --git a/doc/devblog/day_183__rubbing_sticks_together.mdwn b/doc/devblog/day_183__rubbing_sticks_together.mdwn
new file mode 100644
index 0000000000..ecab379167
--- /dev/null
+++ b/doc/devblog/day_183__rubbing_sticks_together.mdwn
@@ -0,0 +1,23 @@
+Spent all day on some horrible timestamp issues on legacy systems. 
+
+On FAT, timestamps have a 2s granularity, which is ok, but then Linux adds
+a temporary higher resolution cache, which is lost on unmount. This
+confused git-annex since the mtimes seemed to change and it had to
+re-checksum half the files to get unconfused, which was not good.
+I found a way to use the inode sentinal file to detect when on FAT
+and put in a workaround, without degrading git-annex everywhere else.
+
+On Windows, time zones are a utter disaster; it changes the mtime it reports
+for files after the time zone has changed. Also there's a bug in the
+haskell time library which makes it return old time zone data after a time
+zone change. (I just finished developing a fix for that bug..)
+
+Left with nothing but a few sticks, I rubbed them together, and
+actually found a way to deal with this problem too. Scary details in
+[[bugs/Windows_file_timestamp_timezone_madness]]. While I've implemented
+it, it's stuck on a branch until I find a way to make git-annex notice when
+the timezone changes while it's running.
+
+----
+
+Today's work was sponsored by Svenne Krap.
diff --git a/doc/devblog/day_184__windows_month.mdwn b/doc/devblog/day_184__windows_month.mdwn
new file mode 100644
index 0000000000..c99d03e90b
--- /dev/null
+++ b/doc/devblog/day_184__windows_month.mdwn
@@ -0,0 +1,22 @@
+It's officially a Windows porting month. Now that I'm half way through it
+and with the last week of the month going to be a vacation, this makes
+sense.
+
+Today, finished up dealing with the timezone/timestamp issues on Windows.
+This got stranger and stranger the closer I looked at it. After a timestamp
+change, a program that was already running will see one timestamp, while a
+program that is started after the change will see another one! My approach
+works pretty much no matter how Windows goes insane though, and always
+recovers a true timestamp. Yay.
+
+Also fixed a regression test failure on Windows, which turned out to be
+rooted in a bug in the command queue runner, which neglected to pass
+along environment overrides on Windows.
+
+Then I spent 5 hours tracking down a tricky 
+test suite failure on Windows, which turned out to also
+affect FAT and be a recent reversion that has as it's
+root cause a [fun bug in git itself](http://marc.info/?l=git&m=140262402204212&w=2).
+Put in a not very good workaround. Thank goodness for test suites!
+
+Also got the arm autobuilder unstuck. Release tomorrow.
diff --git a/doc/devblog/day_185__service.mdwn b/doc/devblog/day_185__service.mdwn
new file mode 100644
index 0000000000..0613ae2bf9
--- /dev/null
+++ b/doc/devblog/day_185__service.mdwn
@@ -0,0 +1,6 @@
+More work on [[todo/windows_git-annex_service]], but am stuck with a
+permissions problem.
+
+Fixed a bug that prevented two assistants from syncing when there was only
+a uni-directional link between them. Only affected direct mode, and
+was introduced back when I added the direct mode guard.
diff --git a/doc/devblog/day_186__cracked_it.mdwn b/doc/devblog/day_186__cracked_it.mdwn
new file mode 100644
index 0000000000..cff6130cbf
--- /dev/null
+++ b/doc/devblog/day_186__cracked_it.mdwn
@@ -0,0 +1,8 @@
+After despairing of ever solving this yesterday (and for the past 6 months
+really), I've got the webapp running on Windows with no visible DOS box.
+Also have the assistant starting up in the background on login.
+
+It turns out a service was not the way to do. There is a way to write a VB
+Script that runs a "DOS" command in a hidden window, and this is what I
+used. Amazing how hard it was to work this out, probably partly because I
+don't have the Windows vocabulary to know what to look for.
diff --git a/doc/devblog/day_186__cracked_it/comment_1_288b736adf392acd0f45667b2980138d._comment b/doc/devblog/day_186__cracked_it/comment_1_288b736adf392acd0f45667b2980138d._comment
new file mode 100644
index 0000000000..d85798eb88
--- /dev/null
+++ b/doc/devblog/day_186__cracked_it/comment_1_288b736adf392acd0f45667b2980138d._comment
@@ -0,0 +1,11 @@
+[[!comment format=mdwn
+ username="https://launchpad.net/~maestro-alubia"
+ nickname="maestro-alubia"
+ subject="Congratulations!"
+ date="2014-06-18T09:08:45Z"
+ content="""
+I really feel with you and I am glad I currently do not need to develop on/for Windows. I wish you good progress with Windows porting, so you can concentrate on the interesting stuff again :)
+
+Best regards,
+Fabian
+"""]]
diff --git a/doc/devblog/day_186__cracked_it/comment_2_d1d79e93ac420f6b3a6f8a622e8e00bd._comment b/doc/devblog/day_186__cracked_it/comment_2_d1d79e93ac420f6b3a6f8a622e8e00bd._comment
new file mode 100644
index 0000000000..d6ade5c01a
--- /dev/null
+++ b/doc/devblog/day_186__cracked_it/comment_2_d1d79e93ac420f6b3a6f8a622e8e00bd._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="http://id.clacke.se/"
+ nickname="Claes"
+ subject="VBScript + wscript to avoid DOS window"
+ date="2014-06-21T02:09:58Z"
+ content="""
+I use this technique too, to run cygwin xterm without an ugly DOS box popup.
+"""]]
diff --git a/doc/devblog/day_186__cracked_it/comment_3_8ca17a51b10b4e4a63d0672d5ce29024._comment b/doc/devblog/day_186__cracked_it/comment_3_8ca17a51b10b4e4a63d0672d5ce29024._comment
new file mode 100644
index 0000000000..356453e8f2
--- /dev/null
+++ b/doc/devblog/day_186__cracked_it/comment_3_8ca17a51b10b4e4a63d0672d5ce29024._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="http://id.clacke.se/"
+ nickname="Claes"
+ subject="VBScript + wscript to avoid DOS window"
+ date="2014-06-21T02:15:04Z"
+ content="""
+(sorry that I didn't see before that you had run into this annoying problem)
+"""]]
diff --git a/doc/devblog/day_187__release_prep.mdwn b/doc/devblog/day_187__release_prep.mdwn
new file mode 100644
index 0000000000..a5207c373e
--- /dev/null
+++ b/doc/devblog/day_187__release_prep.mdwn
@@ -0,0 +1,10 @@
+Last night, got logging to daemon.log working on Windows. Aside from XMPP
+not working (but it's near to being deprecated anyway), and some possible
+issues with unicode characters in filenames, the Windows port now seems in
+pretty good shape for a beta release.
+
+Today, mostly worked on fixing the release process so the metadata
+accurarely reflects the version from the autobuilder that is included in
+the release. Turns out there was version skew in the last release (now
+manually corrected). This should avoid that happening again, and also
+automates more of my release process.
diff --git a/doc/devblog/day_187__release_prep/comment_1_206692d16177c2a9ca11c0eeff545697._comment b/doc/devblog/day_187__release_prep/comment_1_206692d16177c2a9ca11c0eeff545697._comment
new file mode 100644
index 0000000000..a9e982a211
--- /dev/null
+++ b/doc/devblog/day_187__release_prep/comment_1_206692d16177c2a9ca11c0eeff545697._comment
@@ -0,0 +1,13 @@
+[[!comment format=mdwn
+ username="https://www.google.com/accounts/o8/id?id=AItOawlnoH5btjn_BLib3_IhES5uMhrzuOiwCYo"
+ nickname="András"
+ subject="sync options?"
+ date="2014-06-19T13:16:51Z"
+ content="""
+What options will we have in this beta for windows <=> windows (local network) and windows <=> linux (remote server) synchronization?  I'm interested only in non-cloud sync possibilities.
+
+(I tried to follow the developments but apparently failed...)
+
+Also thank you for working on the windows port!
+
+"""]]
diff --git a/doc/devblog/day_187__release_prep/comment_2_961fb35d9cf7d5e518f8d0bddb8626a6._comment b/doc/devblog/day_187__release_prep/comment_2_961fb35d9cf7d5e518f8d0bddb8626a6._comment
new file mode 100644
index 0000000000..6abcfb7dd3
--- /dev/null
+++ b/doc/devblog/day_187__release_prep/comment_2_961fb35d9cf7d5e518f8d0bddb8626a6._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="http://joeyh.name/"
+ ip="209.250.56.203"
+ subject="comment 2"
+ date="2014-06-19T22:13:23Z"
+ content="""
+windows to linux server (local or remote) to windows will work. 
+"""]]
diff --git a/doc/devblog/day_188__back_sans_laptop.mdwn b/doc/devblog/day_188__back_sans_laptop.mdwn
new file mode 100644
index 0000000000..080d6176ef
--- /dev/null
+++ b/doc/devblog/day_188__back_sans_laptop.mdwn
@@ -0,0 +1,5 @@
+I am back from the beach, but my dev laptop is dead. A replacement is being
+shipped, and I have spent today getting my old netbook into a usable state
+so I can perhaps do some work using it in the meantime.
+
+(Backlog is 95 messages.)
diff --git a/doc/devblog/day_189__finally_working_again.mdwn b/doc/devblog/day_189__finally_working_again.mdwn
new file mode 100644
index 0000000000..54b72b52b5
--- /dev/null
+++ b/doc/devblog/day_189__finally_working_again.mdwn
@@ -0,0 +1,18 @@
+Finally back to work with a new laptop!
+
+Did one fairly major feature today: When using git-annex to pull down
+podcasts, metadata from the feed is copied into git-annex's metadata store,
+if annex.genmetadata is set. Should be great for views etc!
+
+Worked through a lot of the backlog, which is down to 47 messages now.
+
+Only other bug fix of note is a fix on Android. A recent change to git made
+it try to chmod files, which tends to fail on the horrible /sdcard
+filesystem. Patched git to avoid that.
+
+For some reason the autobuilder box rebooted while I was away, and 
+somehow the docker containers didn't come back up -- so they got
+automatically rebuilt. But I have to manually finish up building the
+android and armel ones. Will be babysitting that build this evening.
+
+Today's work was sponsored by Ævar Arnfjörð Bjarmason.
diff --git a/doc/devblog/day_190__fun_fixes.mdwn b/doc/devblog/day_190__fun_fixes.mdwn
new file mode 100644
index 0000000000..d3895df1f4
--- /dev/null
+++ b/doc/devblog/day_190__fun_fixes.mdwn
@@ -0,0 +1,13 @@
+Spent the morning improving behavior when `commit.gpgsign` is set.
+Now git-annex will let gpg sign commits that are made when eg, manually
+running `git annex sync`, but not commits implicitly made to the git-annex
+branch. And any commits made by the assistant are not gpg signed. This was
+slightly tricky, since lots of different places in git-annex ran `git
+commit`, `git merge` and similar.
+
+Then got back to a test I left running over vacation, that added millions
+of files to a git annex repo. This was able to reproduce a problem where
+`git annex add` blew the stack and crashed at the end. There turned out to
+be two different memory issues, one was in git-annex and the other is in
+Haskell's core `getDirectoryContents`. Was able to entirely fix it,
+eventually.
diff --git a/doc/devblog/day_191__semidistracted.mdwn b/doc/devblog/day_191__semidistracted.mdwn
new file mode 100644
index 0000000000..3d4b5b6efc
--- /dev/null
+++ b/doc/devblog/day_191__semidistracted.mdwn
@@ -0,0 +1,7 @@
+Got a bit distracted improving Haskell's directory listing code.
+
+Only real git-annex work today was fixing [[bugs/Assistant_merge_loop]],
+which was caused by changes in the last release (that made direct mode
+merging crash/interrupt-safe). This is a kind of ugly bug, that can result
+in the assistant making lots of empty commits in direct mode repositories.
+So, I plan to make a new release on Monday.
diff --git a/doc/devblog/day_192__release_day.mdwn b/doc/devblog/day_192__release_day.mdwn
new file mode 100644
index 0000000000..ce1a38861c
--- /dev/null
+++ b/doc/devblog/day_192__release_day.mdwn
@@ -0,0 +1,4 @@
+Got the release out. Had to fix various autobuilder issues. The arm autobuilder
+is unfortunatly not working currently.
+
+Updated git-annex to build with a new version of the bloomfilter library.
diff --git a/doc/devblog/day_193-194__ugly_bug.mdwn b/doc/devblog/day_193-194__ugly_bug.mdwn
new file mode 100644
index 0000000000..668b7b2ab9
--- /dev/null
+++ b/doc/devblog/day_193-194__ugly_bug.mdwn
@@ -0,0 +1,37 @@
+**Important** A bug [[caused the assistant to sometimes remove all files|bugs/bad_merge_commit_deleting_all_files]] from the git repository.
+You should check if your repository is ok. If the bug hit you, it should be
+possible to revert the bad commit and recover your files with no data loss.
+See the bug report for details. 
+
+This affected git-annex versions since 5.20140613, and only when using the
+assistant in direct mode. It should be fixed in today's release,
+5.20140709.
+
+I'm available  to help anyone hit by this
+unfortunate bug.
+
+This is another bug in the direct mode merge code. I'm not happy about it.
+It's particularly annoying that I can't fix up after it automatically
+(because there's no way to know if any given commit in the git history that
+deletes all the files is the result of this bug, or a legitimate deletion
+of all files).
+
+The only good thing is that the design of git-annex is pretty robust, and
+in this case, despite stupidly committing the deletion of all the files in
+the repository, git-annex did take care to preserve all their contents and
+so the problem should be able to be resolved without data loss.
+
+Unfortunately, the main autobuilder is down and I've had to spin up
+autobuilders on a different machine (thank goodness that's very automated
+now!), and so I have not been able to build the fixed git-annex for android
+yet. I hope to get that done later this evening.
+
+---
+
+Yesterday, I fixed a few (much less bad) bugs, and did some thinking about
+plans for this month. The [[design/roadmap]] suggests working on some of
+[[!traillink design/assistant/chunks]], [[!traillink design/assistant/deltas]] or [[!traillink design/assistant/gpgkeys]].
+I don't know how to do deltas yet really. Chunks is pretty easily done.
+The gpg keys stuff is pretty open ended and needs some more work to define
+some use cases. But, after today, I am more inclined to want to spend
+time on better testing and other means of avoiding this kind of situation.
diff --git a/doc/devblog/day_195-196__catching_up.mdwn b/doc/devblog/day_195-196__catching_up.mdwn
new file mode 100644
index 0000000000..2c04eaa150
--- /dev/null
+++ b/doc/devblog/day_195-196__catching_up.mdwn
@@ -0,0 +1,13 @@
+Spent past 2 days catching up on backlog and doing bug triage and some
+minor bug fixes and features. Backlog is 27, lowest in quite a while so I
+feel well on top of things.
+
+I was saddened to find [this bug](http://git-annex.branchable.com/bugs/files_lost_during_upgrade/#comment-b265c796b1599d2dde649699cb54fa86)
+where I almost managed to analize the [[ugy_bug|day_193-194__ugly_bug]]'s race
+condition, but not quite (and then went on vacation). BTW, I have not heard
+from anyone else who was hit by that bug so far.
+
+The linux autobuilders are still down; their host server had a disk crash
+in an electrical outage. Might be down for a while. I would not mind
+setting up a redundant autobuilder if anyone else would like to donate a
+linux VM with 4+ gb of ram.
diff --git a/doc/devblog/day_197__autobuilder_rescuscitation.mdwn b/doc/devblog/day_197__autobuilder_rescuscitation.mdwn
new file mode 100644
index 0000000000..12dada287d
--- /dev/null
+++ b/doc/devblog/day_197__autobuilder_rescuscitation.mdwn
@@ -0,0 +1,14 @@
+Yay, the Linux autobuilder is back! Also fixed the Windows build.
+
+Fixed a reversion that prevented the webapp from starting
+properly on Windows, which was introduced by some bad locking when I put in
+the hack that makes it log to the log file on that platform.
+
+Various other minor fixes here and there. There are almost enough
+to do a release again soon.
+
+I've also been trying to bootstrap ghc 7.8 on arm, for Debian. There's a script
+that's supposed to allow building 7.8 using 7.6.3, dealing with a linker
+problem by using the gold linker. Hopefully that will work since otherwise
+Debian could remain stuck with an old ghc or worse lose the arm ports.
+Neither would be great for git-annex..
diff --git a/doc/devblog/day_198__branching_out.mdwn b/doc/devblog/day_198__branching_out.mdwn
new file mode 100644
index 0000000000..cdb3a6d1b6
--- /dev/null
+++ b/doc/devblog/day_198__branching_out.mdwn
@@ -0,0 +1,23 @@
+I have mostly been thinking about gcrypt today.
+[This issue](https://github.com/blake2-ppc/git-remote-gcrypt/issues/9)
+needs to be dealt with. The question is, does it really make sense to
+try to hide the people a git repository is encrypted for? I have 
+[posted some thoughts](http://git-annex.branchable.com/bugs/using_gpg_encryption_with_multiple_keys_fails/?updated#comment-0c4f679d972c63b0b25b6aa5e851af62)
+and am coming to the viewpoint that obscuring the identities of users
+of a repository is not a problem git-annex should try to solve itself,
+although it also shouldn't get in the way of someone who is able and
+wants to do that (by using tor, etc).
+
+Finally, I decided to go ahead and add a gcrypt.publish-participants
+setting to git-remote-gcrypt, and make git-annex set that by default when
+setting up a gcrypt repository.
+
+Some promising news from the ghc build on arm. I got a working ghc, and
+even ghci works. Which would make the template haskell in the webapp etc
+avaialble on arm without the current horrible hacks. Have not managed to
+build the debian ghc package successfully yet though.
+
+Also, fixed a bug that made `git annex sync` not pull/push with a local
+repository that had not yet been initialized for use with git-annex.
+
+Today's work was sponsored by Stanley Yamane.
diff --git a/doc/devblog/day_198__branching_out/comment_1_91ce3dc707ba1ba7c5d9e57e20ffce40._comment b/doc/devblog/day_198__branching_out/comment_1_91ce3dc707ba1ba7c5d9e57e20ffce40._comment
new file mode 100644
index 0000000000..7b9ba45b7d
--- /dev/null
+++ b/doc/devblog/day_198__branching_out/comment_1_91ce3dc707ba1ba7c5d9e57e20ffce40._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="lex"
+ ip="2620:149:7:303:796f:ae98:a0a1:94b"
+ subject="comment 1"
+ date="2014-07-16T21:13:06Z"
+ content="""
+glad my bug inspired a blog post about it.  =]
+"""]]
diff --git a/doc/devblog/day_199__ten_minute_cycle.mdwn b/doc/devblog/day_199__ten_minute_cycle.mdwn
new file mode 100644
index 0000000000..56b48747e2
--- /dev/null
+++ b/doc/devblog/day_199__ten_minute_cycle.mdwn
@@ -0,0 +1,6 @@
+Spent hours today in a 10-minute build/test cycle, tracking down a bug that
+caused the assistant to crash on Windows after exactly 10 minutes uptime.
+Eventually found the cause; this is fallout from last month's work
+that got it logging to the debug.log on Windows.
+
+There was more, but that was the interesting one..
diff --git a/doc/devblog/day_19__moving_on.mdwn b/doc/devblog/day_19__moving_on.mdwn
new file mode 100644
index 0000000000..7f4cd8244a
--- /dev/null
+++ b/doc/devblog/day_19__moving_on.mdwn
@@ -0,0 +1,37 @@
+Finished moving the Android autobuilder over to the new clean build
+environment. Tested the Android app, and it still works. Whew!
+
+There's a small chance that the issue with the Android app not working on
+Android 4.3 has been fixed by this rebuild. I doubt it, but perhaps someone
+can download the daily build and give it another try..
+
+----
+
+I have 7 days left in which I'd like to get remote gcrypt repositories
+working in the assistant. I think that should be fairly easy, but a
+prerequisite for it is making git-annex-shell support being run on a gcrypt
+repository. That's needed so that the assistant's normal locked down ssh
+key setup can also be used for gcrypt repositories.
+
+At the same time, not all gcrypt endpoints will have git-annex-shell
+installed, and it *seems* to make sense to leave in the existing support
+for running raw rsync and git push commands against such a repository. So
+that's going to add some complication.
+
+It will also complicate git-annex-shell to support gcrypt repos. Basically,
+everything it does in git-annex repos will need to be reimplemented in
+gcrypt repositories. Generally in a more simple form; for example it
+doesn't need to (and can't) update location logs in a gcrypt repo.
+
+----
+
+I also need to find a good UI to present the three available choices
+(unencrypted git, encrypted git, encrypted rsync) when setting up a repo
+on a ssh server. I don't want to just remove the encrypted rsync option,
+because it's useful when using xmpp to sync the git repo, and is simpler to
+set up since it uses shared encryption rather than gpg public keys.
+
+My current thought is to offer just 2 choices, encrypted and non-encrypted.
+If they choose encrypted, offer a choice of shared encryption or encrypting
+to a specific key. I think I can word this so it's pretty clear what the
+tradeoffs are.
diff --git a/doc/devblog/day_19__moving_on/comment_1_870106f671f9a055b81e6fc83e0850b5._comment b/doc/devblog/day_19__moving_on/comment_1_870106f671f9a055b81e6fc83e0850b5._comment
new file mode 100644
index 0000000000..b0ed97bcf3
--- /dev/null
+++ b/doc/devblog/day_19__moving_on/comment_1_870106f671f9a055b81e6fc83e0850b5._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="https://www.google.com/accounts/o8/id?id=AItOawmW0kg4uiMIhSHeVuvJFyo2VYMl7Qoej0s"
+ nickname="Chris"
+ subject="comment 1"
+ date="2013-09-23T20:58:45Z"
+ content="""
+The new version of the Android apk doesn't work for me on my Nexus 4.
+"""]]
diff --git a/doc/devblog/day_19__moving_on/comment_2_fad055c8860385ac6c012f897c96408f._comment b/doc/devblog/day_19__moving_on/comment_2_fad055c8860385ac6c012f897c96408f._comment
new file mode 100644
index 0000000000..f2e754b3ab
--- /dev/null
+++ b/doc/devblog/day_19__moving_on/comment_2_fad055c8860385ac6c012f897c96408f._comment
@@ -0,0 +1,10 @@
+[[!comment format=mdwn
+ username="https://www.google.com/accounts/o8/id?id=AItOawmkBwMWvNKZZCge_YqobCSILPMeK6xbFw8"
+ nickname="develop"
+ subject="comment 2"
+ date="2013-09-24T07:11:31Z"
+ content="""
+Yeah, no joy on Cyanogenmod 10.2(Android 4.3).
+
+Would be pretty surprising if it had worked.
+"""]]
diff --git a/doc/devblog/day_19__moving_on/comment_3_69e47d612159587f080ab761566d1830._comment b/doc/devblog/day_19__moving_on/comment_3_69e47d612159587f080ab761566d1830._comment
new file mode 100644
index 0000000000..206fdd8520
--- /dev/null
+++ b/doc/devblog/day_19__moving_on/comment_3_69e47d612159587f080ab761566d1830._comment
@@ -0,0 +1,18 @@
+[[!comment format=mdwn
+ username="https://www.google.com/accounts/o8/id?id=AItOawnR6E5iUghMWdUGlbA9CCs8DKaoigMjJXw"
+ nickname="Efraim"
+ subject="not working on my nexus 4 either"
+ date="2013-09-24T07:37:28Z"
+ content="""
+terminal output reads:
+
+Falling back to hardcoded app location; cannot find expected files in /data/app-lib
+
+git annex webapp
+
+u0_a124@mako:/sdcard/git-annex.home $ git annex webpp
+
+CANNOT LINK EXECUTABLE: git-annex invalid R_ARM_COPY relocation against DT_SYMBOLIC shared library libc.so (built with -Bsymbolic?)
+
+1|u0_a124@mako:/sdcard/git-annex.home $
+"""]]
diff --git a/doc/devblog/day_1__inauspicious_beginning.mdwn b/doc/devblog/day_1__inauspicious_beginning.mdwn
new file mode 100644
index 0000000000..b14f763bbe
--- /dev/null
+++ b/doc/devblog/day_1__inauspicious_beginning.mdwn
@@ -0,0 +1,11 @@
+I try hard to keep this devblog about git-annex development and not me.
+However, it is a shame that what I wanted to be the beginning of my first
+real month of work funded by the new campaign has been marred by my home's
+internet connection being taken out by a lightning strike, and by illness.
+Nearly back on my feet after that, and waiting for my new laptop to
+finally get here.
+
+Today's work: Finished up the `git annex forget` feature and merged it in.
+Fixed the bug that was causing the commit race detection code to
+incorrectly fire on the commit made by the transition code. Few other bits
+and pieces.
diff --git a/doc/devblog/day_1__inauspicious_beginning/comment_1_cc4dea43caf3126c6f814b589b701d70._comment b/doc/devblog/day_1__inauspicious_beginning/comment_1_cc4dea43caf3126c6f814b589b701d70._comment
new file mode 100644
index 0000000000..03e3fec6d6
--- /dev/null
+++ b/doc/devblog/day_1__inauspicious_beginning/comment_1_cc4dea43caf3126c6f814b589b701d70._comment
@@ -0,0 +1,10 @@
+[[!comment format=mdwn
+ username="rjc"
+ ip="86.22.66.200"
+ subject="laptop"
+ date="2013-09-04T21:42:52Z"
+ content="""
+Are you retiring your Dell mini?
+
+What kind of laptop are you getting?
+"""]]
diff --git a/doc/devblog/day_200__one_year_along.mdwn b/doc/devblog/day_200__one_year_along.mdwn
new file mode 100644
index 0000000000..6cfab59b97
--- /dev/null
+++ b/doc/devblog/day_200__one_year_along.mdwn
@@ -0,0 +1,17 @@
+Updated the Debian backport. (Also the git-remote-gcrypt backport.)
+
+Made the assistant install a desktop file to integrate with Konqueror.
+
+Improved `git annex repair`, fixing a bug that could cause it to leave
+broken branch refs and yet think that the repair was successful.
+
+----
+
+A bit surprised to see that now been a full year since I started doing
+development funded by my campaign. Not done yet!
+
+Update on campaign rewards: 
+
+----
+
+Today's work was sponsored by Douglas Butts.
diff --git a/doc/devblog/day_201__chunky.mdwn b/doc/devblog/day_201__chunky.mdwn
new file mode 100644
index 0000000000..b210702a46
--- /dev/null
+++ b/doc/devblog/day_201__chunky.mdwn
@@ -0,0 +1,12 @@
+Working on designs for better [[chunking|design/assistant/chunks]]. Having
+a hard time finding a way to totally obscure file sizes, but otherwise a
+good design seems to be coming together. I particularly like that the new
+design puts the chunk count in the Key (which is then encrypted for special
+remotes, rather than having it be some special extension.
+
+While thinking through chunking, I realized that the current chunking
+method can fail if two repositories have different chunksize settings for
+the same special remote and both upload the same key at the same time.
+Arn't races fun? The new design will eliminate this problem; in the
+meantime updated the docs to recommend never changing a remote's
+chunksize setting.
diff --git a/doc/devblog/day_202__new_chunk_groundwork.mdwn b/doc/devblog/day_202__new_chunk_groundwork.mdwn
new file mode 100644
index 0000000000..c8e0ce7f6e
--- /dev/null
+++ b/doc/devblog/day_202__new_chunk_groundwork.mdwn
@@ -0,0 +1,6 @@
+The design for new style chunks seems done, and I laid the groundwork for it
+today. Added chunk metadata to keys, reorganized the legacy chunking code
+for directory and webdav so it won't get (too badly) in the way, and
+implemented the chunk logs in the git-annex branch.
+
+Today's work was sponsored by LeastAuthority.com.
diff --git a/doc/devblog/day_203__in_the_weeds.mdwn b/doc/devblog/day_203__in_the_weeds.mdwn
new file mode 100644
index 0000000000..a8cf1787ab
--- /dev/null
+++ b/doc/devblog/day_203__in_the_weeds.mdwn
@@ -0,0 +1,39 @@
+A lil bit in the weeds on the chunking rewrite right now. I did succeed in
+writing the core chunk generation code, which can be used for every special
+remote. It was pretty hairy (needs to stream large files in constant
+memory, separating into chunks, and get the progress display right
+across operations on chunks, etc). That took most of the day.
+
+Ended up getting stuck in integrating the encryptable remote code, and had
+to revert changes that could have led to rewriting (or perhaps
+eliminating?) most of the per-remote encryption specific code.
+
+Up till now, this has supported both encrypted and non-encrypted remotes;
+it was simply passed encrypted keys for an encrypted remote:
+
+[[!format haskell """
+remove :: Key -> Annex Bool
+"""]]
+
+But with chunked encrypted keys, it seems it needs to be more complicated:
+
+[[!format haskell """
+remove' :: Maybe (Key -> Key) -> ChunkConfig -> Key -> Annex Bool
+"""]]
+
+So that when the remote is configured to use chunking, it can look up
+the chunk keys, and then encrypt them, in order to remove all the encrypted
+chunk keys.
+
+I don't like that complication, so want to find a cleaner
+abstraction. Will sleep on it.
+
+----
+
+While I was looking at the encryptable remote generator, I realized
+the remote cost was being calculated wrongly for special 
+remotes that are not encrypted. Fixed that bug.
+
+----
+
+Today's work was sponsored by bak.
diff --git a/doc/devblog/day_204__mowing.mdwn b/doc/devblog/day_204__mowing.mdwn
new file mode 100644
index 0000000000..b34f1ba380
--- /dev/null
+++ b/doc/devblog/day_204__mowing.mdwn
@@ -0,0 +1,64 @@
+Remained frustratingly stuck until 3 pm on the same stuff that puzzled
+me yesterday. However, 6 hours later, I have the directory
+special remote 100% working with both new chunk= and legacy chunksize=
+configuration, both with and without encryption.
+
+----
+
+So, the root of why this is was hard, since I thought about it a lot today
+in between beating my head into the wall: git-annex's internal API for remotes
+is really, really simple. It basically comes down to:
+
+[[!format haskell """
+	Remote
+		{ storeKey :: Key -> AssociatedFile -> MeterUpdate -> Annex Bool
+		, retrieveKeyFile :: Key -> AssociatedFile -> FilePath -> MeterUpdate -> Annex Bool
+		, removeKey :: Key -> Annex Bool
+		, hasKey :: Key -> Annex (Either String Bool)
+		}
+"""]]
+
+This simplicity is a Good Thing, because it maps very well to REST-type
+services. And it allows for quite a lot of variety in implementations of
+remotes. Ranging from reguar git remotes, that rsync files around without
+git-annex ever loading them itself, to remotes like webdav that load
+and store files themselves, to remotes like tahoe that intentionally do not
+support git-annex's built-in encryption methods.
+
+However, the simplicity of that API means that lots of complicated stuff,
+like handling chunking, encryption, etc, has to be handled on a per-remote
+basis. Or, more generally, by `Remote -> Remote` transformers that take
+a remote and add some useful feature to it.
+
+One problem is that the API is so simple that a remote transformer that adds
+encryption is not feasible. In fact, every encryptable remote has
+had its own code that loads a file from local disk, encrypts it, and sends
+it to the remote. Because there's no way to make a remote transformer that
+converts a `storeKey` into an encrypted `storeKey`. (Ditto for retrieving
+keys.)
+
+I almost made the API more complicated today. Twice. But both times
+I ended up not, and I think that was the right choice, even though
+it meant I had to write some quite painful code.
+
+----
+
+In the end, I instead wrote a little module that pulls together supporting
+both encryption and chunking. I'm not completely happy because those
+two things should be independent, and so separate. But, 120 lines of
+code that don't keep them separate is not the end of the world.
+
+That module also contains some more powerful, less general APIs, 
+that will work well with the kinds of remotes that will use it.
+
+The really nice result, is that the implementation of the directory
+special remote melts down from 267 lines of code to just 172! (Plus some
+legacy code for the old style chunking, refactored out into a file I can
+delete one day.) It's a lot cleaner too.
+
+With all this done, I expect I can pretty easily add the new style chunking
+to most git-annex remotes, and remove code from them while doing it!
+
+----
+
+Today's work was sponsored by Mark Hepburn.
diff --git a/doc/devblog/day_205__incremental.mdwn b/doc/devblog/day_205__incremental.mdwn
new file mode 100644
index 0000000000..c8535d4390
--- /dev/null
+++ b/doc/devblog/day_205__incremental.mdwn
@@ -0,0 +1,21 @@
+Last night, went over the new chunking interface, tightened up exception
+handling, and improved the API so that things like WebDAV will be able to
+reuse a single connection while all of a key's chunks are being downloaded.
+I am pretty happy with the interface now, and except to convert more
+special remotes to use it soon.
+
+Just finished adding a killer feature: Automatic resuming of interrupted
+downloads from chunked remotes. Sort of a poor man's rsync, that while less
+efficient and awesome, is going to work on *every* remote that gets the new
+chunking interface, from S3 to WebDAV, to all of Tobias's external special
+remotes! Even allows for things like starting a download
+from one remote, interrupting, and resuming from another one, and so on.
+
+I had forgotten about resuming while designing the chunking API. Luckily, I
+got the design right anyway. Implementation was almost trivial, and only
+took about 2 hours! (See [[!commit 9d4a766cd7b8e8b0fc7cd27b08249e4161b5380a]])
+
+I'll later add resuming of interrupted uploads. It's not hard to detect
+such uploads with only one extra query of the remote, but in principle,
+it should be possible to do it with no extra overhead, since git-annex
+already checks if all the chunks are there before starting an upload.
diff --git a/doc/devblog/day_206__zap.mdwn b/doc/devblog/day_206__zap.mdwn
new file mode 100644
index 0000000000..eccee2464b
--- /dev/null
+++ b/doc/devblog/day_206__zap.mdwn
@@ -0,0 +1,83 @@
+Zap! ... My internet gateway was [destroyed by lightning](https://identi.ca/joeyh/note/xogvXTFDR9CZaCPsmKZipA).
+Limping along regardless, and replacement ordered.
+
+Got resuming of uploads to chunked remotes working. Easy!
+
+----
+
+Next I want to convert the external special remotes to have these nice
+new features. But there is a wrinkle: The new chunking interface works
+entirely on ByteStrings containing the content, but the external special
+remote interface passes content around in files.
+
+I could just make it write the ByteString to a temp file, and pass the temp
+file to the external special remote to store. But then, when chunking is
+not being used, it would pointlessly read a file's content, only to write
+it back out to a temp file.
+
+Similarly, when retrieving a key, the external special remote saves it to a
+file. But we want a ByteString. Except, when not doing chunking or
+encryption, letting the external special remote save the content directly
+to a file is optimal.
+
+One approach would be to change the protocol for external special
+remotes, so that the content is sent over the protocol rather than in temp
+files. But I think this would not be ideal for some kinds of external
+special remotes, and it would probably be quite a lot slower and more
+complicated.
+
+Instead, I am playing around with some type class trickery:
+
+[[!format haskell """
+{-# LANGUAGE Rank2Types TypeSynonymInstances FlexibleInstances MultiParamTypeClasses #-}
+
+type Storer p = Key -> p -> MeterUpdate -> IO Bool
+
+-- For Storers that want to be provided with a file to store.
+type FileStorer a = Storer (ContentPipe a FilePath)
+
+-- For Storers that want to be provided with a ByteString to store
+type ByteStringStorer a = Storer (ContentPipe a L.ByteString)
+
+class ContentPipe src dest where
+        contentPipe :: src -> (dest -> IO a) -> IO a
+
+instance ContentPipe L.ByteString L.ByteString where
+        contentPipe b a = a b
+
+-- This feels a lot like I could perhaps use pipes or conduit...
+instance ContentPipe FilePath FilePath where
+        contentPipe f a = a f
+
+instance ContentPipe L.ByteString FilePath where
+        contentPipe b a = withTmpFile "tmpXXXXXX" $ \f h -> do
+                L.hPut h b
+                hClose h
+                a f
+
+instance ContentPipe FilePath L.ByteString where
+        contentPipe f a = a =<< L.readFile f
+"""]]
+
+The external special remote would be a FileStorer, so when a non-chunked,
+non-encrypted file is provided, it just runs on the FilePath with no extra
+work. While when a ByteString is provided, it's swapped out to a temp file
+and the temp file provided. And many other special remotes are ByteStorers,
+so they will just pass the provided ByteStream through, or read in the
+content of a file.
+
+I think that would work. Thoigh it is not optimal for external special
+remotes that are chunked but not encrypted. For that case, it might be worth
+extending the special remote protocol with a way to say "store a chunk of
+this file from byte N to byte M".
+
+---
+
+Also, talked with ion about what would be involved in using rolling checksum
+based chunks. That would allow for rsync or zsync like behavior, where
+when a file changed, git-annex uploads only the chunks that changed, and the
+unchanged chunks are reused.
+
+I am not ready to work on that yet, but I made some changes to the parsing
+of the chunk log, so that additional chunking schemes like this can be added
+to git-annex later without breaking backwards compatability.
diff --git a/doc/devblog/day_207__at_last.mdwn b/doc/devblog/day_207__at_last.mdwn
new file mode 100644
index 0000000000..936ac98f0e
--- /dev/null
+++ b/doc/devblog/day_207__at_last.mdwn
@@ -0,0 +1,34 @@
+It took 9 hours, but I finally got to make [[!commit c0dc134cded6078bb2e5fa2d4420b9cc09a292f7]],
+which both removes 35 lines of code, and adds chunking support to all
+external special remotes!
+
+The groundwork for that commit involved taking the type scheme I sketched
+out yesterday, completely failing to make it work with such high-ranked
+types, and falling back to a simpler set of types that both I and GHC seem
+better at getting our heads around.
+
+Then I also had more fun with types, when it turned out I needed to
+run encryption in the Annex monad. So I had to go convert several parts of
+the utility libraries to use MonadIO and exception lifting. Yurk.
+
+The final and most fun stumbling block caused git-annex to crash when
+retriving a file from an external special remote that had neither
+encryption not chunking. Amusingly it was because I had not put in an
+optimation (namely, just renaming the file that was retrieved in this case,
+rather than unnecessarily reading it in and writing it back out). It's
+not often that a lack of an optimisation causes code to crash!
+
+So, fun day, great result, and it should now be very simple to convert
+the bup, ddar, gcrypt, glacier, rsync, S3, and WebDAV special remotes
+to the new system. Fingers crossed.
+
+But first, I will probably take half a day or so and write a 
+`git annex testremote` that can be run in a repository and does live
+testing of a special remote including uploading and downloading files.
+There are quite a lot of cases to test now, and it seems best to get
+that in place before I start changing a lot of remotes without a way to
+test everything.
+
+----
+
+Today's work was sponsored by Daniel Callahan.
diff --git a/doc/devblog/day_208__testremote.mdwn b/doc/devblog/day_208__testremote.mdwn
new file mode 100644
index 0000000000..87c497fccc
--- /dev/null
+++ b/doc/devblog/day_208__testremote.mdwn
@@ -0,0 +1,10 @@
+Built `git annex testremote` today. 
+
+That took a little bit longer than expected, because it actually found
+several fence post bugs in the chunking code.
+
+It also found a bug in the sample external special remote script.
+
+I am very pleased with this command. Being able to run 640 tests against
+any remote, without any possibility of damaging data already stored in the
+remote, is awesome. Should have written it a looong time ago!
diff --git a/doc/devblog/day_209__mass_conversion.mdwn b/doc/devblog/day_209__mass_conversion.mdwn
new file mode 100644
index 0000000000..23de65b2ac
--- /dev/null
+++ b/doc/devblog/day_209__mass_conversion.mdwn
@@ -0,0 +1,32 @@
+Have started converting lots of special remotes to the new API. Today, S3
+and hook got chunking support. I also converted several remotes to the new
+API without supporting chunking: bup, ddar, and glacier (which should
+support chunking, but there were complications). 
+
+This removed 110 lines of code while adding features! And,
+I seem to be able to convert them faster than `testremote` can test them. :)
+
+Now that S3 supports chunks, they can be used to work around several
+problems with S3 remotes, including file size limits, and a memory leak in
+the underlying S3 library.
+
+The S3 conversion included caching of the S3 connection when
+storing/retrieving chunks. [Update: Actually, it turns out it didn't;
+the hS3 library doesn't support persistent connections. Another reason I
+need to switch to a better S3 library!] 
+
+But the API doesn't yet support caching
+when removing or checking if chunks are present. I should probably expand
+the API, but got into some type checker messes when using generic enough
+data types to support everything. Should probably switch to `ResourceT`.
+
+Also, I tried, but failed to make `testremote` check that storing a key
+is done atomically. The best I could come up with was a test that stored a
+key and had another thread repeatedly check if the object was present on
+the remote, logging the results and timestamps. It then becomes a
+statistical problem -- somewhere toward the end of the log it's ok if the key
+has become present -- but too early might indicate that it wasn't stored
+atomically. Perhaps it's my poor knowledge of statistics, but I could not
+find a way to analize the log that reliably detected non-atomic storage.
+If someone would like to try to work on this, see the `atomic-store-test`
+branch.
diff --git a/doc/devblog/day_20__gcrypt_and_git-annex-shell.mdwn b/doc/devblog/day_20__gcrypt_and_git-annex-shell.mdwn
new file mode 100644
index 0000000000..0e4142b7c0
--- /dev/null
+++ b/doc/devblog/day_20__gcrypt_and_git-annex-shell.mdwn
@@ -0,0 +1,14 @@
+Added support for gcrypt remotes to git-annex-shell. Now gcrypt special
+remotes probe when they are set up to see if the remote system has a
+suitable git-annex-shell, and if so all commands are sent to it. Kept the
+direct rsync mode working as a fallback.
+
+It turns out I made a bad decision when first adding gcrypt support to
+git-annex. To make implementation marginally easier, I decided to not
+put objects inside the usual `annex/objects` directory in a gcrypt remote.
+But that lack of consistency would have made adding support to
+git-annex-shell a lot harder. So, I decided to change this. Which 
+means that anyone already using gcrypt with git-annex will need to
+[[manually_move_files_around|upgrades/gcrypt]].
+
+Today's work was sponsored by Tobias Nix.
diff --git a/doc/devblog/day_210__conversion_and_digression.mdwn b/doc/devblog/day_210__conversion_and_digression.mdwn
new file mode 100644
index 0000000000..2629eb9174
--- /dev/null
+++ b/doc/devblog/day_210__conversion_and_digression.mdwn
@@ -0,0 +1,14 @@
+Just finished converting both rsync and gcrypt to the new API,
+and testing them. Still need to fix 2 test suite failures for gcrypt.
+Otherwise, only WebDAV remains unconverted.
+
+Earlier today, I investigated switching from hS3 to
+. Learned its API, which seemed a
+lot easier to comprehend than the other two times I looked at it. Wrote
+some test programs, which are in the `s3-aws` branch. I was able to stream
+in large files to S3, without ever buffering them in memory (which hS3's
+API precludes). And for chunking, it can reuse an http connection.
+This seems very promising. (Also, it might eventually get Glacier support..)
+
+I have uploaded haskell-aws to Debian, and once it gets into testing and
+backports, I plan to switch git-annex over to it.
diff --git a/doc/devblog/day_211__conversion_complete.mdwn b/doc/devblog/day_211__conversion_complete.mdwn
new file mode 100644
index 0000000000..5e172fae18
--- /dev/null
+++ b/doc/devblog/day_211__conversion_complete.mdwn
@@ -0,0 +1,12 @@
+Converted the webdav special remote to the new API. 
+All done with converting everything now!
+
+I also updated the new API to support doing things like
+reusing the same http connection when removing and checking
+the presence of chunks.
+
+I've been working on improving the haskell DAV library, in a
+number of ways that will let me improve the webdav special remote.
+Including making changes that will let me do connection caching, and
+improving its API to support streaming content without buffering a whole
+file in memory.
diff --git a/doc/devblog/day_212__webdav_rewrite.mdwn b/doc/devblog/day_212__webdav_rewrite.mdwn
new file mode 100644
index 0000000000..714800c9aa
--- /dev/null
+++ b/doc/devblog/day_212__webdav_rewrite.mdwn
@@ -0,0 +1,18 @@
+Today was spent reworking so much of the webdav special remote that it was
+essentially rewritten from scratch.
+
+The main improvement is that it now keeps a http connection open and uses
+it to perform multiple actions. Before, one connection was made per action.
+This is even done for operations on chunks. So, now storing a chunked file
+in webdav makes only 2 http connections total. Before, it would take around
+10 connections *per chunk*. So a big win for performance, although there is
+still room for improvement: It would be possible to reduce that down to
+just 1 connection, and indeed keep a persistent connection reused when
+acting on multiple files.
+
+Finished up by making uploading a large (non-chunked) file to webdav not
+buffer the whole file in memory. 
+
+I still need to make downloading a file from webdav not buffer it, and
+test, and then I'll be done with webdav and can move on to making
+similar changes to S3.
diff --git a/doc/devblog/day_213__newchunks_merged.mdwn b/doc/devblog/day_213__newchunks_merged.mdwn
new file mode 100644
index 0000000000..42ea7a6550
--- /dev/null
+++ b/doc/devblog/day_213__newchunks_merged.mdwn
@@ -0,0 +1,15 @@
+Finished up webdav, and after running `testremote` for a long time, I'm
+satisfied it's good. The newchunks branch has now been merged into master
+completely.
+
+Spent the rest of the day beginning to rework the S3 special remote to use
+the aws library. This was pretty fiddly; I want to keep all the
+configuration exactly the same, so had to do a lot of mapping from hS3
+configuration to aws configuration. Also there is some hairy stuff
+involving escaping from the ResourceT monad with responses and http
+connection managers intact.
+
+Stopped once `initremote` worked. The rest should be pretty easy, although
+Internet Archive support is blocked by
+. This is in the `s3-aws`
+branch until it gets usable.
diff --git a/doc/devblog/day_214-215__wrapping_up_recent_work.mdwn b/doc/devblog/day_214-215__wrapping_up_recent_work.mdwn
new file mode 100644
index 0000000000..7e06e231b2
--- /dev/null
+++ b/doc/devblog/day_214-215__wrapping_up_recent_work.mdwn
@@ -0,0 +1,16 @@
+Yesterday, finished converting S3 to use the aws library. Very happy with
+the result (no memory leaks! connection caching!), but s3-aws is not merged
+into master yet. Waiting on a new release of the aws library so as to not
+break Internet Archive S3 support.
+
+Today, spent a few hours adding more tests to `testremote`. The new tests
+take a remote, and construct a modified version that is intentionally
+unavailable. Then they make sure trying to use it fails in appropriate
+ways. This was a very good thing to test; two bugs were immediately found
+and fixed.
+
+And that wraps up several weeks of hacking on the core of git-annex's
+remotes support, which started with reworking chunking and kind of took
+on a life of its own. I plan a release of this new stuff in a week. The
+next week will be spent catching up on 117 messages of backlog that
+accumulated while I was in deep coding mode.
diff --git a/doc/devblog/day_216__various_minor_bugs.mdwn b/doc/devblog/day_216__various_minor_bugs.mdwn
new file mode 100644
index 0000000000..a9c49a9dd3
--- /dev/null
+++ b/doc/devblog/day_216__various_minor_bugs.mdwn
@@ -0,0 +1,16 @@
+Working on getting caught up with backlog. 73 messages remain.
+
+Several minor bugs were fixed today. All edge cases. The most edge case one
+of all, I could not fix: git-annex cannot add a file that has a newline
+in its filename, because `git cat-file --batch`'s interface does not support such
+filenames.
+
+Added a page [[documenting how verify the signatures of git-annex releases|install/verifying_downloads]].
+
+Over the past couple days, all the autobuilders have been updated to new
+dependencies needed by the recent work. Except for Windows, which needs to
+be updated to the new Haskell Platform first, so hopefully soon.
+
+Turns out that upgrading unix-compat means that inode(like) numbers are
+available even on Windows, which will make git-annex more robust there.
+Win win. ;)
diff --git a/doc/devblog/day_216__various_minor_bugs/comment_1_0d0a0e75b9446f8a1c4cc43f36569473._comment b/doc/devblog/day_216__various_minor_bugs/comment_1_0d0a0e75b9446f8a1c4cc43f36569473._comment
new file mode 100644
index 0000000000..a1cc9c3b7f
--- /dev/null
+++ b/doc/devblog/day_216__various_minor_bugs/comment_1_0d0a0e75b9446f8a1c4cc43f36569473._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="EskildHustvedt"
+ ip="80.202.103.55"
+ subject="comment 1"
+ date="2014-08-14T05:30:46Z"
+ content="""
+What exactly does «git-annex cannot add a file that has a space in its filename» mean? git-annex (/assistant) actually can't handle tracking any file that has a space in its filename?
+"""]]
diff --git a/doc/devblog/day_216__various_minor_bugs/comment_2_6b06b3f46f20a6d2e60684d1d59fca07._comment b/doc/devblog/day_216__various_minor_bugs/comment_2_6b06b3f46f20a6d2e60684d1d59fca07._comment
new file mode 100644
index 0000000000..559689bb79
--- /dev/null
+++ b/doc/devblog/day_216__various_minor_bugs/comment_2_6b06b3f46f20a6d2e60684d1d59fca07._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="http://joeyh.name/"
+ ip="209.250.56.7"
+ subject="comment 2"
+ date="2014-08-15T19:45:28Z"
+ content="""
+That was a typo. Of course, spaces are no problem. Newlines in filenames, on the other hand..
+"""]]
diff --git a/doc/devblog/day_216__various_minor_bugs/comment_3_05177e2ed414d22711dcec57a614e38c._comment b/doc/devblog/day_216__various_minor_bugs/comment_3_05177e2ed414d22711dcec57a614e38c._comment
new file mode 100644
index 0000000000..4b3f0248a5
--- /dev/null
+++ b/doc/devblog/day_216__various_minor_bugs/comment_3_05177e2ed414d22711dcec57a614e38c._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="EskildHustvedt"
+ ip="80.202.103.55"
+ subject="comment 3"
+ date="2014-08-16T15:22:35Z"
+ content="""
+Ah, well then, that sounds a lot more reasonable. Though legal, I have yet to hear of a sane reason for using newlines in filenames.
+"""]]
diff --git a/doc/devblog/day_217__autobuilders.mdwn b/doc/devblog/day_217__autobuilders.mdwn
new file mode 100644
index 0000000000..2a58e0c631
--- /dev/null
+++ b/doc/devblog/day_217__autobuilders.mdwn
@@ -0,0 +1,10 @@
+Over the past couple days, got the arm autobuilder working again. It had
+been down since June with several problems. cabal install tended to crash;
+apparenty this has something to do with threading in user-mode qemu,
+because -j1 avoids that. And strange invalid character problems were fixed
+by downgrading file-embed. Also, with Yury's help I got the Windows
+autobuilder upgraded to the new Haskell Platform and working again.
+
+Today a last few finishing touches, including getting rid of the last
+dependency on the old haskell HTTP library, since http-conduit is being
+used now. Ready for the release!
diff --git a/doc/devblog/day_218__scary_locking.mdwn b/doc/devblog/day_218__scary_locking.mdwn
new file mode 100644
index 0000000000..81afb263b5
--- /dev/null
+++ b/doc/devblog/day_218__scary_locking.mdwn
@@ -0,0 +1,24 @@
+Plan is to be on vacation and/or low activity this week before DebConf.
+However, today I got involved in fixing a bug that caused the assistant to
+keep files open after syncing with repositories on removable media.
+
+Part of that bug involved lock files not being opend close-on-exec, and
+while fixing that I noticed again that the locking code was scattered all
+around and rather repetitive. That led to a lot of refactoring, which is
+always fun when it involves scary locking code. Thanks goodness for
+referential transparency.
+
+Now there's a Utility.LockFile that works on both POSIX and Windows.
+Howver, that module actually exports very different functions for the two.
+While it might be tempting to try to do a portability layer, the
+two locking models are really very different, and there are lots of gotchas
+such a portability layer would face. The only API that's completely the
+same between the two is dropLock.
+
+This refactoring process and the cleaner, more expressive
+code it led to helped me spot a couple of bugs involving locking. See
+[[!commit e386e26ef207db742da6d406183ab851571047ff]]
+and [[!commit 0a4d301051e4933661b7b0a0791afa95bfe9a1d3]]
+Neither bug has ever seemed to cause
+a problem, but it's nice to be able to spot and fix such bugs before they
+do.
diff --git a/doc/devblog/day_219__catching_up_and_looking_back.mdwn b/doc/devblog/day_219__catching_up_and_looking_back.mdwn
new file mode 100644
index 0000000000..8d82d8cb96
--- /dev/null
+++ b/doc/devblog/day_219__catching_up_and_looking_back.mdwn
@@ -0,0 +1,109 @@
+Yesterday and today were the first good solid days working on git-annex in a
+while. There's a big backlog, currently of 133 messages, so I have been
+concentrating on bug reports first. Happily, not many new bugs have been
+reported lately, and I've made good progress on them, fixing 5 bugs today,
+including a file descriptor leak.
+
+## catching up
+
+In this end of summer rush, I've been too busy to blog for the past 20 days,
+but not entirely too busy to work on git-annex. Two releases have been made
+in that time, and a fair amount of improvements worked on.
+
+Including a new feature: When a local git repository is cloned with `git
+clone --shared`, git-annex detects this and defaults to a special mode
+where file contents get hard linked into the clone. It also makes the cloned
+repository be untrusted, to avoid confusing numcopies counting with the
+hard links. This can be useful for temporary working repositories without
+the overhead of lots of copies of files.
+
+## looking back
+
+I want to look back further, over the crowdfunded year of work covered
+by this devblog. There were a lot of things I wanted to accomplish this
+past year, and I managed to get to most of them. As well as a few surprises.
+
+* Windows support improved more than I guessed in my wildest dreams.  
+  git-annex went from working not too well on the command line to
+  being pretty solid there, as well as having a working
+  and almost polished webapp on Windows.  
+  There are still warts -- it's Windows after all!
+
+* Android didn't get many improvements. Most of the time I had budgeted to
+  Android porting ended up being used on Windows porting instead. I did,
+  however, get the Android build environment cleaned up a lot from the initial
+  hacked together one, and generally kept it building and working on Android.
+
+* The [direct mode guard](http://git-annex.branchable.com/devblog/day_48__direct_mode_guard_design/)
+  was not planned, but the need for it became clear, and
+  it's dramatically reduced the amount of command-line foot-shooting
+  that goes on in direct mode.
+
+* Repository repair was planned, and I've very proud of [git-repair](http://git-repair.branchable.com/).
+  Also pleased with the webapp's UI for scheduling repository consistency
+  checks.  
+  Always room for improvement in this kind of thing, but this brings a new
+  capability to both git and git-annex.
+
+* The [[external_special_remote_interface|special_remotes/external]] came
+  together beautifully. External special remotes are now just as well
+  supported as built-in ones, except the webapp cannot be used to configure
+  them.
+
+* Using git-remote-gcrypt for fully encrypted git repositories, including
+  support in the webapp for setting them (and gpg keys if necessary),
+  happened. Still needs testing/more use/improvements. Avoided doing
+  much in the area of gpg key management, which is probably good to avoid when
+  possible, but is probably needed to make this a really suitable option for
+  end users.
+
+* Telehash is still being built, and it's not clear if they've gotten it
+  to work at all yet. The v2 telehash has recently been superseded by a
+  a new v3. So I am not pleased that I didn't get git-annex working with
+  telehash, but it was outside my control. This is a problem that needs to get
+  solved outside git-annex first, either by telehash or something else.
+  The plan is to keep an eye on everything in this space, including for example,
+  Maidsafe.
+
+* In the meantime, the new notifychanges support in git-annex-shell
+  makes XMPP/telehash/whatever unnecessary in a lot of configurations.
+  git-annex's remotedaemon architecture supports that and is designed
+  to support other notification methods later. And the webapp has a lot of
+  improvements in the area of setting up ssh remotes, so fewer users will
+  be stuck with XMPP.
+
+* I didn't quite get to [[design/assistant/deltas]], but the final month
+  of work on chunking provides a lot of new features and hopefully a
+  foundation that will get to deltas eventually. There is a new haskell
+  library that's being developed with the goal of being used for git-annex
+  deltas.
+
+* I hadn't planned to make git-annex be able to upgrade itself, when installed
+  from this website. But there was a need for that, and so it happened.
+  Even got a gpg key trust path for the distribution of git-annex.
+
+* Metadata driven views was an entirely unplanned feature. The current
+  prototype is very exciting, it opens up entire new use cases. 
+  I had to hold myself back to not work on it too much,
+  especially as it shaded into adding a caching database to git-annex.
+  Had too much other stuff planned to do all I wanted.
+  Clearly this is an area I want to spend more time on!
+
+Those are most of the big features and changes, but probably half
+of my work on git-annex this past year was in smaller things, and general
+maintenance. Lots of others have contributed, some with 
+code (like the large effort to switch to bootstrap3),
+and others with documentation, bug reports, etc.
+
+Perhaps it's best to turn to `git diff --stat` to sum up the activity
+and see just how much both the crowdfunding campaign and
+the previous kickstarter have pushed git-annex into high gear:
+ 
+	   campaign: 5410 files changed, 124159 insertions(+), 79395 deletions(-)
+	kickstarter: 4411 files changed, 123262 insertions(+), 13935 deletions(-)
+	year before: 1281 files changed,   7263 insertions(+), 55831 deletions(-)
+
+What's next? The hope is, no more crowdfunded campaigns where I have
+to promise the moon anytime soon. Instead, the goal is to move to a more
+mature and sustainable funding model, and continue to grow the git-annex
+community, and the spaces where it's useful.
diff --git a/doc/devblog/day_219__catching_up_and_looking_back/comment_1_16b13b2510183a9da5f960ae5765e581._comment b/doc/devblog/day_219__catching_up_and_looking_back/comment_1_16b13b2510183a9da5f960ae5765e581._comment
new file mode 100644
index 0000000000..5b839b55cb
--- /dev/null
+++ b/doc/devblog/day_219__catching_up_and_looking_back/comment_1_16b13b2510183a9da5f960ae5765e581._comment
@@ -0,0 +1,10 @@
+[[!comment format=mdwn
+ username="https://www.google.com/accounts/o8/id?id=AItOawkRGMQkg9ck_pr47JXZV_C2DJQXrO8LgpI"
+ nickname="Michael"
+ subject="Hard linking on local clone"
+ date="2014-09-13T06:28:01Z"
+ content="""
+Thanks for this feature. It will save a lot of space when working on one-off projects with big scientific datasets.
+
+Unfortunately, there is probably no easy solution to achieve similar savings across file systems. On our shared cluster individual labs have their data in separate ZFS volumes (to ease individual backup handling), but data is often shared (i.e. copied) across volumes when cloning an annex. We need expensive de-duplication on the backup-server to, at least, prevent this kind of waste to hit the backups -- but the master file server still suffers (de-duplication ratio sometimes approaching a factor of 2.0).
+"""]]
diff --git a/doc/devblog/day_219__catching_up_and_looking_back/comment_2_460c064bebb5061fcba2a6c79f039362._comment b/doc/devblog/day_219__catching_up_and_looking_back/comment_2_460c064bebb5061fcba2a6c79f039362._comment
new file mode 100644
index 0000000000..a00d33dc6f
--- /dev/null
+++ b/doc/devblog/day_219__catching_up_and_looking_back/comment_2_460c064bebb5061fcba2a6c79f039362._comment
@@ -0,0 +1,19 @@
+[[!comment format=mdwn
+ username="https://id.koumbit.net/anarcat"
+ ip="72.0.72.144"
+ subject="comment 2"
+ date="2014-09-14T17:38:34Z"
+ content="""
+thanks so much for your work on git-annex, joeyh. it's hard to imagine that just 4 years ago, we didn't have anything even close to this tool and how far it went since then. 
+
+~~~~
+anarcat@marcos:git-annex$ pepper age
+Loading cache index... done
+git-annex is 3 years old
+git-annex's birthday is in about 1 month (October 19th)
+~~~~
+
+birthday is coming soon! :)
+
+it's also quite impressive how much work can be done in a single year with some (fairly minimal) funding to dedicate a full dev on a project. very inspiring - keep up the good work! -- [[anarcat]]
+"""]]
diff --git a/doc/devblog/day_219__catching_up_and_looking_back/comment_3_9699d5a9de5ea64fbc876352e20261c4._comment b/doc/devblog/day_219__catching_up_and_looking_back/comment_3_9699d5a9de5ea64fbc876352e20261c4._comment
new file mode 100644
index 0000000000..91fc49409c
--- /dev/null
+++ b/doc/devblog/day_219__catching_up_and_looking_back/comment_3_9699d5a9de5ea64fbc876352e20261c4._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="anarcat"
+ ip="70.83.139.100"
+ subject="camlistore"
+ date="2014-09-17T20:18:56Z"
+ content="""
+have you looked at [camlistore](http://camlistore.org/) at all? it's a fairly new project, but it seems like it could be an interesting backend, or at least inspiration for git-annex's design. --[[anarcat]]
+"""]]
diff --git a/doc/devblog/day_219__catching_up_and_looking_back/comment_4_23c4ede3db0ea8165311466881cfa6a2._comment b/doc/devblog/day_219__catching_up_and_looking_back/comment_4_23c4ede3db0ea8165311466881cfa6a2._comment
new file mode 100644
index 0000000000..8e24916db8
--- /dev/null
+++ b/doc/devblog/day_219__catching_up_and_looking_back/comment_4_23c4ede3db0ea8165311466881cfa6a2._comment
@@ -0,0 +1,10 @@
+[[!comment format=mdwn
+ username="https://www.google.com/accounts/o8/id?id=AItOawmH7o6q2l99M-PQolOfbR3_i5B_jtTIcAE"
+ nickname="Giovanni"
+ subject="Camlistore"
+ date="2014-09-17T20:36:43Z"
+ content="""
+anarcat, have you used it? I tried, but it is buggy. They seem to be at [\"the Archivist\"](http://git-annex.branchable.com/) group of people, but if you don't have a hard drive to store the things, everything breaks up. I paid a lot of money to Amazon because I believed I could use Camlistore to organize data stored at S3, but apparently S3 is \"just for backup\", and if it is your only storage, then Camlistore will keep fetching data over and over \"to index it\" and in the end you pay.
+
+Yes, it keeps working, so you need some server online at all times, with Camlistore running.
+"""]]
diff --git a/doc/devblog/day_219__catching_up_and_looking_back/comment_5_7997305d7ec7db072b78dd0c31ecd824._comment b/doc/devblog/day_219__catching_up_and_looking_back/comment_5_7997305d7ec7db072b78dd0c31ecd824._comment
new file mode 100644
index 0000000000..91e4dc5f5d
--- /dev/null
+++ b/doc/devblog/day_219__catching_up_and_looking_back/comment_5_7997305d7ec7db072b78dd0c31ecd824._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="anarcat"
+ ip="70.83.139.100"
+ subject="comment 5"
+ date="2014-09-17T20:53:01Z"
+ content="""
+i haven't tried it at all - only looked at [this one demo video](https://www.youtube.com/watch?v=yxSzQIwXM1k) that reminded me of git-annex a lot...
+"""]]
diff --git a/doc/devblog/day_21__bugfix_day.mdwn b/doc/devblog/day_21__bugfix_day.mdwn
new file mode 100644
index 0000000000..a913fce56a
--- /dev/null
+++ b/doc/devblog/day_21__bugfix_day.mdwn
@@ -0,0 +1,5 @@
+Did various bug fixes and followup today. Amazing how a day can vanish that
+way. Made 4 actual improvements.
+
+I still have 46 messages in unanswered backlog. Although only 8 of
+the are from this month.
diff --git a/doc/devblog/day_220__working_through_backlog.mdwn b/doc/devblog/day_220__working_through_backlog.mdwn
new file mode 100644
index 0000000000..4685e3ff05
--- /dev/null
+++ b/doc/devblog/day_220__working_through_backlog.mdwn
@@ -0,0 +1,14 @@
+Made a release yesterday, which was all bugfixes.
+
+Today, a few more bug fixes. Looked into making the webapp
+create non-bare repositories on removable drives, but before I got too far
+into the code, I noticed [there's a big problem with that idea](http://git-annex.branchable.com/forum/usability:_creating_an_archive_on_a_new_external_drive/).
+
+Rest of day was spent getting caught up on forum posts etc. I'm happy to
+read lots of good answers that have been posted while I've been away.
+Here's an excellent example: 
+
+That led to rewriting the docs for building git-annex from source.
+New page: [[install/fromsource]].
+
+Backlog is now down to 117.
diff --git a/doc/devblog/day_221__another_fine_day_of_bugfixing.mdwn b/doc/devblog/day_221__another_fine_day_of_bugfixing.mdwn
new file mode 100644
index 0000000000..0c26f57359
--- /dev/null
+++ b/doc/devblog/day_221__another_fine_day_of_bugfixing.mdwn
@@ -0,0 +1,10 @@
+Working through the forum posts and bugs. Backlog is down to 95.
+
+Discovered the first known security hole in git-annex!
+Turns out that S3 and Glacier remotes that were configured with embedcreds=yes and encryption=pubkey or encryption=hybrid
+didn't actually encrypt the AWS credentials that get embedded into the git
+repo. This doesn't affect any repos set up by the assistant.
+
+I've fixed the problem and am going to make a release soon.
+If your repo is affected, see 
+[[upgrades/insecure_embedded_creds]] for what to do about it.
diff --git a/doc/devblog/day_222_preparing_for_debian_release.mdwn b/doc/devblog/day_222_preparing_for_debian_release.mdwn
new file mode 100644
index 0000000000..62acc02b22
--- /dev/null
+++ b/doc/devblog/day_222_preparing_for_debian_release.mdwn
@@ -0,0 +1,12 @@
+Made two releases of git-annex, yesterday and today, which turned out to
+contain only Debian changes. So no need for other users to upgrade.
+
+This included fixing building on mips, and arm architectures.
+The mips build was running out of memory, and I was able to work around
+that. Then the arm builds broke today, because of a recent change to the
+version of llvm that has completely trashed ghc. Luckily, I was able
+to work around that too.
+
+Hopefully that will get last week's security fix into Debian testing,
+and otherwise have git-annex in Debian in good shape for the upcoming
+freeze.
diff --git a/doc/devblog/day_223__partial_commit_problem.mdwn b/doc/devblog/day_223__partial_commit_problem.mdwn
new file mode 100644
index 0000000000..b2d1ec4e00
--- /dev/null
+++ b/doc/devblog/day_223__partial_commit_problem.mdwn
@@ -0,0 +1,26 @@
+`git commit $some_unlocked_file` seems like a reasonably common thing for
+someone to do, so it's surprising to find that it's a [[little bit broken|/bugs/modified_permissions_persist_after_unlock__44___commit]],
+leaving the file staged in the index after (correctly) committing the
+annexed symlink.
+
+This is caused by either a bug in git and/or by git-annex abusing the
+git post-commit hook to do something it shouldn't do, although it's not
+unique in using the post-commit hook this way. I'm talking this over with
+Junio, and the fix will depend on the result of that conversation. It might
+involve git-annex detecting this case and canceling the commit, asking the
+user to `git annex add` the file first. Or it might involve a new git hook,
+although I have not had good luck getting hooks added to git before.
+
+----
+
+Meanwhile, today I did some other bug fixing. Fixed the Internet Archive
+support for embedcreds=yes. Made `git annex map` work for remote repos
+in a directory with an implicit ".git" prefix. And fixed a
+strange problem where the repository repair code caused a `git gc` to run
+and then tripped over its pid file.
+
+I seem to have enough fixes to make another release pretty soon.
+Especially since the current release of git-annex doesn't build with yesod
+1.4.
+
+Backlog: 94 messages
diff --git a/doc/devblog/day_224-226__long_rainy_slog.mdwn b/doc/devblog/day_224-226__long_rainy_slog.mdwn
new file mode 100644
index 0000000000..9c26b134b9
--- /dev/null
+++ b/doc/devblog/day_224-226__long_rainy_slog.mdwn
@@ -0,0 +1,14 @@
+3 days spent redoing the Android autobuilder! The new version of
+yesod-routes generates TH splices that break the EvilSplicer. So after
+updating everything to new versions for the Nth time, I instead went back
+to older versions. The autobuilder now uses Debian jessie, instead of
+wheezy. And all haskell packages are pinned to use the same version
+as in jessie, rather than the newest versions. Since jessie is quite near
+to being frozen, this should make the autobuilder much less prone to
+getting broken by new versions of haskell packages that need patches for
+Android.
+
+I happened to stumble over 
+while doing that. This supports setting and unsetting environment variables
+on Windows, which I had not known a way to do from Haskell. Cleaned up
+several ugly corners of the Windows port using it.
diff --git a/doc/devblog/day_227__info.mdwn b/doc/devblog/day_227__info.mdwn
new file mode 100644
index 0000000000..0b5950fe51
--- /dev/null
+++ b/doc/devblog/day_227__info.mdwn
@@ -0,0 +1,33 @@
+Today, I've expanded `git annex info` to also be able to be used on annexed files
+and on remotes. Looking at the info for an individual remote is quite
+useful, especially for answering questions like: Does the remote have
+embedded creds? Are they encrypted? Does it use chunking? Is that old style
+chunking?
+
+
    +remote: rsync.net
+description: rsync.net demo remote
+uuid: 15b42f18-ebf2-11e1-bea1-f71f1515f9f1
+cost: 250.0
+type: rsync
+url: xxx@usw-s002.rsync.net:foo
+encryption: encrypted (to gpg keys: 7321FC22AC211D23 C910D9222512E3C7)
+chunking: 1 MB chunks
+
    
+
+
    +remote: ia3
+description: test [ia3]
+uuid: 12817311-a189-4de3-b806-5f339d304230
+cost: 200.0
+type: S3
+creds: embedded in git repository (not encrypted)
+bucket: joeyh-test-17oct-3
+internet archive item: http://archive.org/details/joeyh-test-17oct-3
+encryption: not encrypted
+chunking: none
+
    
+
+Should be quite useful info for debugging too..
+
+Yesterday, I fixed a bug that prevented retrieving files from Glacier.
diff --git a/doc/devblog/day_229__S3_multipart.mdwn b/doc/devblog/day_229__S3_multipart.mdwn
new file mode 100644
index 0000000000..caccda07ab
--- /dev/null
+++ b/doc/devblog/day_229__S3_multipart.mdwn
@@ -0,0 +1,11 @@
+Some progress on the [[bugs/S3_upload_not_using_multipart]] bug. The aws
+library now includes the multipart API. However, when I dug into it, it
+looks like the API needs some changes to get the ETAG of each uploaded
+part. Once that's fixed, git-annex should be able to support S3 multipart
+uploads, although I think that git-annex's own chunking is better in most
+situations -- it supports resuming uploads and downloads better. The main
+use case for S3 multipart seems to be using git-annex to publish large
+files.
+
+Also, managed to get the backlog down from 100 to just 65 messages,
+including catching up on quite old parts of backlog.
diff --git a/doc/devblog/day_22__gcrypt_on_rsync.net.mdwn b/doc/devblog/day_22__gcrypt_on_rsync.net.mdwn
new file mode 100644
index 0000000000..2c59517952
--- /dev/null
+++ b/doc/devblog/day_22__gcrypt_on_rsync.net.mdwn
@@ -0,0 +1,20 @@
+Being still a little unsure of the UI and complexity
+for configuring gcrypt on ssh servers, I thought I'd start today with the
+special case of gcrypt on rsync.net. Since rsync.net allows running some git
+commands, gcrypt can be used to make encrypted git repositories on it.
+
+Here's the UI I came up with. It's complicated a bit by needing to explain
+the tradeoffs between the rsync and gcrypt special remotes.
+
+[[!img /assistant/rsync.net.encryption.png]]
+
+This works fine, but I did not get a chance to add support for enabling
+existing gcrypt repos on rsync.net. Anyway, most of the changes to make
+this work will also make it easier to add general support for gcrypt on ssh
+servers.
+
+Also spent a while fixing a bug in git-remote-gcrypt. Oddly 
+`gpg --list-keys --fast-list --fingerprint` does not show the fingerprints
+of some keys.
+
+Today's work was sponsored by Cloudier - Thomas Djärv.
diff --git a/doc/devblog/day_230__S3_multipart_round_2.mdwn b/doc/devblog/day_230__S3_multipart_round_2.mdwn
new file mode 100644
index 0000000000..a73579df82
--- /dev/null
+++ b/doc/devblog/day_230__S3_multipart_round_2.mdwn
@@ -0,0 +1,9 @@
+More work on S3 multipart uploads, since the aws library got fixed today
+to return the ETAGs for the parts. I got multipart uploads fully working,
+including progress display.
+
+The code takes care to stream each part in from the file and out the
+socket, so I'd hoped it would have good memory behavior. However, for
+reasons I have not tracked down, something in the aws library is causing
+each part to be buffered in memory. This is a problem, since I want to
+use 1 gb as the default part size.
diff --git a/doc/devblog/day_231__S3_multipart_wrapped_up.mdwn b/doc/devblog/day_231__S3_multipart_wrapped_up.mdwn
new file mode 100644
index 0000000000..df8ecb601c
--- /dev/null
+++ b/doc/devblog/day_231__S3_multipart_wrapped_up.mdwn
@@ -0,0 +1,13 @@
+S3 multipart is finally completely working. I still don't understand the
+memory issue that stumped me yesterday, but rewrote the code to use a
+simpler approach, which avoids the problem. Various other issues, and
+testing it with large files, took all day.
+
+This is now merged into the `s3-aws` branch, so when that branch lands, S3
+support will massively improve, from the current situation of using a buggy
+library that buffers uploaded files in memory, and cannot support very
+large file uploads at all, to being able to support hopefully files of
+arbitrary hugeness (at least up to a few terabytes).
+
+BTW, thanks to Aristid Breitkreuz and Junji Hashimoto for working on the
+multipart support in the `aws` library.
diff --git a/doc/devblog/day_232__OSX_Yosemite.mdwn b/doc/devblog/day_232__OSX_Yosemite.mdwn
new file mode 100644
index 0000000000..468d7030ef
--- /dev/null
+++ b/doc/devblog/day_232__OSX_Yosemite.mdwn
@@ -0,0 +1,6 @@
+The OSX autobuilder has been updated to OSX 10.10 Yosemite. The 
+[resulting build](https://downloads.kitenet.net/git-annex/OSX/current/10.10_Yosemite/git-annex.dmg)
+might also work on 10.9 Mavericks too, and I'd appreciate help testing that.
+
+Went ahead and fixed the [[partial commit problem|bugs/modified_permissions_persist_after_unlock__44___commit]]
+by making the pre-commit hook detect and block problematic partial commits.
diff --git a/doc/devblog/day_233__direct_mode_proxy.mdwn b/doc/devblog/day_233__direct_mode_proxy.mdwn
new file mode 100644
index 0000000000..21a2f2d00c
--- /dev/null
+++ b/doc/devblog/day_233__direct_mode_proxy.mdwn
@@ -0,0 +1,29 @@
+Ever since the direct mode guard was added a year ago, direct mode has been
+a lot safer to use, but very limited in the git commands that could be run
+in a direct mode repository.
+
+The worst limitation was that there was no way to `git revert` unwanted
+changes. But also, there was no way to check out different branches,
+or run commands like `git mv`.
+
+Today I made `git annex proxy`, which allows doing all of those things,
+and more. [documentation here](http://git-annex.branchable.com/direct_mode/#index5h2)
+
+It's so flexible that I'm not sure where the boundries lie yet, but it
+seems it will work for any git command that updates both the work tree and
+the index. Some git commands only update one or the other and not both and
+won't work with the proxy. As an advanced user tool, I think this is a
+great solution. I still want to make a simpler
+[[undo command|todo/direct_mode_undo]] that can nicely integrate into file
+managers.
+
+The implementation of `git annex proxy` is
+[quite simple](http://source.git-annex.branchable.com/?p=source.git;a=blob;f=Command/Proxy.h), because it
+reuses all the complicated work tree update code that was already written
+for `git annex merge`.
+
+----
+
+And here's the lede I buried: I've gotten two years of funding to work
+on git-annex part-time!
+[Details in my personal blog](http://joeyh.name/blog/entry/continuing_to_be_pleasantly_surprised/).
diff --git a/doc/devblog/day_234__undo_undo.mdwn b/doc/devblog/day_234__undo_undo.mdwn
new file mode 100644
index 0000000000..270bee5d77
--- /dev/null
+++ b/doc/devblog/day_234__undo_undo.mdwn
@@ -0,0 +1,6 @@
+Built the `git annex undo` command. This is intended to be a simple
+interface for users who have changed one file, and want to undo the change
+without the complexities of `git revert` or `git annex proxy`. It's simple
+enough that I added undo as an action in the file manager integration.
+
+And yes, you can undo an undo. :)
diff --git a/doc/devblog/day_235__thanksgiving_backlog.mdwn b/doc/devblog/day_235__thanksgiving_backlog.mdwn
new file mode 100644
index 0000000000..4cc28b5a1c
--- /dev/null
+++ b/doc/devblog/day_235__thanksgiving_backlog.mdwn
@@ -0,0 +1,18 @@
+Back from the holiday, catching up on traffic. Backlog stands at 113
+messages.
+
+Here's a nice tip that Giovanni added:
+[[tips/publishing_your_files_to_the_public]] (using a public S3 bucket)
+
+Just before going on break, I added a new feature that I didn't mention
+here. `git annex diffdriver` integrates git-annex with git's external diff
+driver support. So if you have a smart diff program that can diff, say,
+genome sequences, or cat videos, or something in some useful way, it
+can be hooked up to `git diff` and will be able to see the content of
+annexed files.
+
+Also today, I spent a couple hours today updating the license file included
+in the standalone git-annex builds to include the licenses of all the
+haskell libraries git-annex depends on. Which I had for some reason not
+thought to include before, despite them getting built into the git-annex
+binary.
diff --git a/doc/devblog/day_236__release_day.mdwn b/doc/devblog/day_236__release_day.mdwn
new file mode 100644
index 0000000000..48cbce09d1
--- /dev/null
+++ b/doc/devblog/day_236__release_day.mdwn
@@ -0,0 +1,10 @@
+Today's release has a month's accumulated changes, including several nice
+new features: `git annex undo`, `git annex proxy`, `git annex diffdriver`,
+and I was able to land the s3-aws branch in this release too, so lots of
+improvements to the S3 support.
+
+Spent several hours getting the autobuilders updated, with the haskell
+`aws` library installed. Android and armel builds are still out of date.
+
+Also fixed two Windows bugs related to the location of the bundled ssh
+program.
diff --git a/doc/devblog/day_237__extending_addurl.mdwn b/doc/devblog/day_237__extending_addurl.mdwn
new file mode 100644
index 0000000000..e0129398ec
--- /dev/null
+++ b/doc/devblog/day_237__extending_addurl.mdwn
@@ -0,0 +1,14 @@
+Worked on [[todo/extensible_addurl]] today. When `git annex addurl` is run,
+remotes will be asked if they claim the url, and whichever remote does will
+be used to download it, and location tracking will indicate that remote
+contains the object. This is a masive 1000 line patch touching 30 files,
+including follow-on changes in `rmurl` and `whereis` and even `rekey`.
+
+It should now be possible to build an external special remote that handles
+*.torrent and magnet: urls and passes them off to a bittorrent client for
+download, for example.
+
+Another use for this would be to make an external special remote that
+uses youtube-dl or some other program than quvi for downloading web videos.
+The builtin quvi support could probably be moved out of the web special
+remote, to a separate remote. I haven't tried to do that yet.
diff --git a/doc/devblog/day_238__extending_addurl_further.mdwn b/doc/devblog/day_238__extending_addurl_further.mdwn
new file mode 100644
index 0000000000..0dd3764361
--- /dev/null
+++ b/doc/devblog/day_238__extending_addurl_further.mdwn
@@ -0,0 +1,67 @@
+Some more work on the interface that lets remotes claim urls for `git annex
+addurl`. Added support for remotes suggesting a filename to use when
+adding an url. Also, added support for urls that result in multiple files
+when downloaded. The obvious use case for that is an url to a torrent that
+contains multiple files.
+
+Then, got `git annex importfeed` to also check if a remote claims an url.
+
+Finally, I put together a quick demo external remote using this new
+interface. [[special_remotes/external/git-annex-remote-torrent]]
+adds support for torrent files to git-annex, using [aria2c](http://aria2.sourceforge.net/) to download them.
+It supports multi-file torrents, but not magnet links. (I'll probably
+rewrite this more robustly and efficiently in haskell sometime soon.)
+
+Here's a demo:
+
+
    +# git annex initremote torrent type=external encryption=none externaltype=torrent
+initremote torrent ok
+(Recording state in git...)
+# ls
+# git annex addurl  --fast file:///home/joey/my.torrent
+  % Total    % Received % Xferd  Average Speed   Time    Time     Time  Current
+                                 Dload  Upload   Total   Spent    Left  Speed
+100   198  100   198    0     0  3946k      0 --:--:-- --:--:-- --:--:-- 3946k
+addurl _home_joey_my.torrent/bar (using torrent) ok
+addurl _home_joey_my.torrent/baz (using torrent) ok
+addurl _home_joey_my.torrent/foo (using torrent) ok
+(Recording state in git...)
+# ls _home_joey_my.torrent/
+bar@  baz@  foo@
+# git annex get _home_joey_my.torrent/baz
+get _home_joey_my.torrent/baz (from torrent...) 
+  % Total    % Received % Xferd  Average Speed   Time    Time     Time  Current
+                                 Dload  Upload   Total   Spent    Left  Speed
+  0     0    0     0    0     0      0      0 --:--:-- --:--:-- --:--:-100   198  100   198    0     0  3580k      0 --:--:-- --:--:-- --:--:-- 3580k
+
+12/11 18:14:56 [NOTICE] IPv4 DHT: listening on UDP port 6946
+
+12/11 18:14:56 [NOTICE] IPv4 BitTorrent: listening on TCP port 6961
+
+12/11 18:14:56 [NOTICE] IPv6 BitTorrent: listening on TCP port 6961
+
+12/11 18:14:56 [NOTICE] Seeding is over.
+12/11 18:14:57 [NOTICE] Download complete: /home/joey/tmp/tmp.Le89hJSXyh/tor
+
+12/11 18:14:57 [NOTICE] Your share ratio was 0.0, uploaded/downloaded=0B/0B
+                                                                               
+Download Results:
+gid   |stat|avg speed  |path/URI
+======+====+===========+=======================================================
+71f6b6|OK  |       0B/s|/home/joey/tmp/tmp.Le89hJSXyh/tor/baz
+
+Status Legend:
+(OK):download completed.
+ok                      
+(Recording state in git...)
+# git annex find
+_home_joey_my.torrent/baz
+# git annex whereis _home_joey_my.torrent/baz
+whereis _home_joey_my.torrent/baz (2 copies) 
+  	1878241d-ee49-446d-8cce-041c46442d94 -- [torrent]
+   	52412020-2bb3-4aa4-ae16-0da22ba48875 -- joey@darkstar:~/tmp/repo [here]
+
+  torrent: file:///home/joey/my.torrent#2
+ok
+
    
diff --git a/doc/devblog/day_239-240__bittorrent_remote.mdwn b/doc/devblog/day_239-240__bittorrent_remote.mdwn
new file mode 100644
index 0000000000..3b466540f3
--- /dev/null
+++ b/doc/devblog/day_239-240__bittorrent_remote.mdwn
@@ -0,0 +1,16 @@
+Spent a couple days adding a [[bittorrent_special_remote|special_remotes/bittorrent]]
+to git-annex. This is better than the demo external torrent remote I made
+on Friday: It's built into git-annex; it supports magnet links; it even
+parses aria2c's output so the webapp can display progress bars.
+
+Besides needing `aria2` to download torrents, it also currently depends on
+the `btshowmetainfo` command from the original bittorrent client (or
+bittornado). I looked into using
+ instead,
+but that package is out of date and doesn't currently build. I've got a
+patch fixing that, but am waiting to hear back from the library's author.
+
+There is a bit of a behavior change here; while before `git annex addurl` of
+a torrent file would add the torrent file itself to the repository, it now will
+download and add the contents of the torrent. I think/hope this behavior
+change is ok..
diff --git a/doc/devblog/day_23__GNU_day.mdwn b/doc/devblog/day_23__GNU_day.mdwn
new file mode 100644
index 0000000000..4f5b25ca77
--- /dev/null
+++ b/doc/devblog/day_23__GNU_day.mdwn
@@ -0,0 +1,23 @@
+Worked on making the assistant able to merge in existing encrypted
+git repositories from rsync.net.
+
+This had two parts. First, making the webapp UI where you click to enable a
+known special remote work with these encrypted repos. Secondly, handling
+the case where a user knows they have an encrypted repository on rsync.net,
+so enters in its hostname and path, but git-annex doesn't know about that
+special remote. The second case is important, for example, when the
+encrypted repository is a backup and you're restoring from it. It wouldn't
+do for the assistant, in that case, to make a *new* encrypted repo and
+push it over top of your backup!
+
+Handling that was a neat trick. It has to do quite a lot of probing, including
+downloading the whole encrypted git repo so it can decrypt it and merge it,
+to find out about the special remote configuration used for it. This all
+works with just 2 ssh connections, and only 1 ssh password prompt max.
+
+Next, on to generalizing this rsync.net specific code to work with
+arbitrary ssh servers!
+
+----
+
+Today's work was made possible by [RMS's vision 30 years ago](http://article.olduse.net/771@mit-eddie.UUCP).
diff --git a/doc/devblog/day_241-242__end_of_year_cleanup.mdwn b/doc/devblog/day_241-242__end_of_year_cleanup.mdwn
new file mode 100644
index 0000000000..4044ae183f
--- /dev/null
+++ b/doc/devblog/day_241-242__end_of_year_cleanup.mdwn
@@ -0,0 +1,22 @@
+Took a holiday week off from git-annex development, and started a new side
+project building [shell-monad](http://joeyh.name/code/shell-monad/), which
+might eventually be used in some parts of git-annex that generate shell
+scripts.
+
+Message backlog is 165 and I have not dove back into it, but I have started
+spinning back up the development engines in preparation for new year
+takeoff. 
+
+Yesterday, added some minor new features -- `git annex sync` now
+supports git remote groups, and I added a new plumbing command
+`setpresentkey` for those times when you really need to mess with
+git-annex's internal bookkeeping. Also cleaned up a lot of build warning
+messages on OSX and Windows.
+
+Today, first some improvements to make `addurl` more robust.
+Then the rest of the day was spent on Windows. Fixed (again)
+the Windows port's problem with rsync hating DOS style filenames. Got
+the rsync special remote fully working on Windows for the first time.
+
+Best of all, got the Windows autobuilder to run the test
+suite successfully, and fixed a couple test suite failures on Windows.
diff --git a/doc/devblog/day_243__android_5.mdwn b/doc/devblog/day_243__android_5.mdwn
new file mode 100644
index 0000000000..96bf67268a
--- /dev/null
+++ b/doc/devblog/day_243__android_5.mdwn
@@ -0,0 +1,16 @@
+I've finally been clued into why [git-annex isn't working on Android 5](http://git-annex.branchable.com/bugs/__91__Android__93___5.0_needs_PIE_executables___40__git_annex_does_not_work_on_android_5.0__41__/), and
+it seems fixing it is as easy as pie.. That is, passing -pie -FPIE to the
+linker. I've added a 5.0 build to the Android autobuilder. It is currently
+untested, so I hope to get feedback from someone with an Android 5 device;
+a [test build](http://downloads.kitenet.net/git-annex/autobuild/android/5.0/git-annex.apk) is now available.
+
+I've been working through the backlog of messages today, and gotten down
+from 170 to 128. Mostly answered a lot of interesting questions, 
+such as "[Where to start reading the source code?](http://git-annex.branchable.com/install/fromsource/#comment-cb68f2aa0a598d0150db852834ea07da)"
+
+Also did some work to make git-annex check git versions at runtime more
+often, instead of assuming the git version it was built against. 
+It turns out this could be done pretty inexpensively in 2 of 4 cases,
+and one of the 2 fixed was the git check-attr behavior change, 
+which could lead to git-annex add hanging if used with an old version of
+git.
diff --git a/doc/devblog/day_243__android_5/comment_1_59a139108f4cf20dd7443a5fa767f614._comment b/doc/devblog/day_243__android_5/comment_1_59a139108f4cf20dd7443a5fa767f614._comment
new file mode 100644
index 0000000000..a3acbbd42e
--- /dev/null
+++ b/doc/devblog/day_243__android_5/comment_1_59a139108f4cf20dd7443a5fa767f614._comment
@@ -0,0 +1,9 @@
+[[!comment format=mdwn
+ username="https://www.google.com/accounts/o8/id?id=AItOawkGCmVc5qIJaQQgG82Hc5zzBdAVdhe2JEM"
+ nickname="Bruno"
+ subject="comment 1"
+ date="2015-01-06T01:52:03Z"
+ content="""
+I got \"error: only position independent executables (PIE) are supported\" on a Nexus 4 (Android 5.0.1)
+http://i.imgur.com/TXO4fcf.png
+"""]]
diff --git a/doc/devblog/day_244__relative_paths.mdwn b/doc/devblog/day_244__relative_paths.mdwn
new file mode 100644
index 0000000000..f631dbc8fb
--- /dev/null
+++ b/doc/devblog/day_244__relative_paths.mdwn
@@ -0,0 +1,14 @@
+git-annex internally uses all absolute paths all the time.
+For a couple of reasons, I'd like it to use relative paths.
+The best reason is, it would let a repository be moved while git-annex was
+running, without breaking. A lesser reason is that Windows has some
+crazy small limit on the length of a path (260 bytes?!), and using relative
+paths would avoid hitting it so often.
+
+I tried to do this today, in a `relativepaths` branch. I eventually got the
+test suite to pass, but I am very unsure about this change. A lot of random
+assumptions broke, and the test suite won't catch them all. In a few
+places, git-annex commands do change the current directory, and that
+will break with relative paths.
+
+A frustrating day.
diff --git a/doc/devblog/day_245__yak_shaving.mdwn b/doc/devblog/day_245__yak_shaving.mdwn
new file mode 100644
index 0000000000..0e1a7dbaf3
--- /dev/null
+++ b/doc/devblog/day_245__yak_shaving.mdwn
@@ -0,0 +1,9 @@
+Worked more on the `relativepaths` branch last night, and I am actually
+fairly happy with it now, and plan to merge it after I've run it for a bit
+longer myself.
+
+It seems that I did manage to get a git-annex executable that is built PIE
+so it will work on Android 5.0. But all the C programs like busybox
+included in the Android app also have to be built that way. Arranging for
+everything to get built twice and with the right options took up most of
+today.
diff --git a/doc/devblog/day_246__old_todos.mdwn b/doc/devblog/day_246__old_todos.mdwn
new file mode 100644
index 0000000000..6957cff525
--- /dev/null
+++ b/doc/devblog/day_246__old_todos.mdwn
@@ -0,0 +1,9 @@
+Got a release out today.
+
+I'm feeling a little under the weather, so wanted something easy to do in
+the rest of the day that would be nice and constructive. Ended up going
+over the todo list. Old todos come in three groups; hard problems, already
+solved, and easy changes that never got done. I left the first group alone,
+closed many todos in the second group, and implemented a few easy changes.
+Including `git annex sync -m` and adding some more info to `git annex info
+remote`.
diff --git a/doc/devblog/day_247__hooks_and_large_files_on_windows.mdwn b/doc/devblog/day_247__hooks_and_large_files_on_windows.mdwn
new file mode 100644
index 0000000000..59865b36bd
--- /dev/null
+++ b/doc/devblog/day_247__hooks_and_large_files_on_windows.mdwn
@@ -0,0 +1,30 @@
+Today I got The pre-commit-annex hook working on Windows. It turns out that
+msysgit runs hook scripts even when they're not executable, and it parses
+the #! line itself. Now git-annex does too, on Windows.
+
+Also, added a new chapter to the walkthrough,
+[[walkthrough/using_special_remotes]]. They clearly needed to be mentioned,
+especially to show the workflow of running initremote in one repository,
+then syncing another repository and running enableremote to enable the same
+special remote there.
+
+Then more fun Windows porting! Turns out git-annex on Windows didn't handle
+files > 2 gb correctly; the way it was getting file size 
+[uses a too small data type on Windows](https://github.com/jystic/unix-compat/issues/16).
+Luckily git-annex itself treats all file sizes as unbounded Integers,
+so I was easily able to swap in a `getFileSize` that returns correct
+values for large files.
+
+While I haven't blogged since the 13th and have not been too active until
+today, there are still a number of little improvements that have been done
+here and there.
+
+Including a fix for an interesting bug where the assistant would tell
+the remotedaemon that the network connection has been lost, twice in a row,
+and this would make the remotedeamon fail to reconnect to the remote when
+the network came up. I'm not sure what situation triggers this bug (Maybe
+machines with 2 interfaces? Or maybe a double disconnection event for 1
+interface?), but I was able to reproduce it by sending messages to the
+remotedaemon, and so fixed it.
+
+Backlog is down to 118 messages.
diff --git a/doc/devblog/day_248__workload_tuning.mdwn b/doc/devblog/day_248__workload_tuning.mdwn
new file mode 100644
index 0000000000..d0677fc60b
--- /dev/null
+++ b/doc/devblog/day_248__workload_tuning.mdwn
@@ -0,0 +1,54 @@
+Today I put together a lot of things I've been thinking about:
+
+* There's some evidence that git-annex needs tuning to handle some unusual
+  repositories. In particular very big repositories might benefit from
+  different object hashing.
+* It's really hard to handle [[upgrades]] that change the fundamentals of
+  how git-annex repositories work. Such an upgrade would need every
+  git-annex user to upgrade their repository, and would be very painful.
+  It's hard to imagine a change that is worth that amount of pain.
+* There are other changes some would like to see (like lower-case object
+  hash directory names) that are certainly not enough to warrant a flag
+  day repo format upgrade.
+* It would be nice to let people who want to have some flexibility to play
+  around with changes, in their own repos, as long as they don't a)
+  make git-annex a lot more complicated, or b) negatively impact others.
+  (Without having to fork git-annex.)
+
+This is discussed in more depth in [[design/new_repo_versions]].
+
+The solution, which I've built today, is support for
+[[tuning]] settings, when a new repository is first created. The resulting
+repository will be different in some significant way from a default
+git-annex repository, but git-annex will support it just fine. 
+
+The main limitations are:
+
+* You can't change the tuning of an existing repository
+  (unless a tool gets written to transition it).
+* You absolutely don't want to merge repo B, which has been tuned in
+  nonstandard ways, into repo A which has not. Or A into B. (Unless you like
+  watching slow motion car crashes.)
+
+I built all the infrastructure for this today. Basically, the git-annex
+branch gets a record of all tunings that have been applied, and they're
+automatically propagated to new clones of a repository.
+
+And I implemented the first tunable setting:
+
+	git -c annex.tune.objecthashlower=true annex init
+
+This is definitely an experimental feature for now.
+`git-annex merge` and similar commands will detect attempts to merge
+between incompatibly tuned repositories, and error out. But, there are a
+lot of ways to shoot yourself in the foot if you use this feature:
+
+* Nothing stops `git merge` from merging two incompatible repositories.
+* Nothing stops any version of git-annex older from today from merging
+  either.
+
+Now that the groundwork is laid, I can pretty easily, and inexpensively,
+add more tunable settings. The next two I plan to add are already
+documented, `annex.tune.objecthashdirectories` and 
+`annex.tune.branchhashdirectories`. Most new tunables should take about 4
+lines of code to add to git-annex.
diff --git a/doc/devblog/day_249_onward.mdwn b/doc/devblog/day_249_onward.mdwn
new file mode 100644
index 0000000000..dac851c542
--- /dev/null
+++ b/doc/devblog/day_249_onward.mdwn
@@ -0,0 +1,28 @@
+Made a release yesterday, and caught up on most recent messages
+earlier this week. Backlog stands at 128 messages.
+
+Had to deal with an ugly problem with /usr/bin/glacier today. Seems that
+there are multiple programs all using that name, some of them shipping in
+some linux distributions, and the one from boto fails to fail when passed
+parameters it doesn't understand. Yugh! I had to make git-annex probe to
+make sure the right glacier program is installed.
+
+I'm planning to deprecate the glacier special remote at some point.
+Instead, I'd like to make the S3 special remote support the S3-glacier
+lifecycle, so objects can be uploaded to S3, set to transition to
+glacier, and then if necessary pulled back from glacier to S3. That should
+be much simpler and less prone to break.
+
+But not yet; [haskell-aws needs glacier support added](https://github.com/aristidb/aws/issues/81).
+Or I could use the new amazonka library, but I'd rather stick with
+haskell-aws.
+
+Some other minor improvements today included adding `git annex
+groupwanted`, which makes for easier examples than using vicfg, and
+making `git annex import` support options like --include and --exclude.
+
+Also I moved a many file matching options to only be accepted by
+the commands that actually use them. Of the remaining common
+options, most of them make sense for every command to accept (eg, --force
+and --debug). It would make sense to move --backend, --notify-start/finish,
+and perhaps --user-agent. Eventually.
diff --git a/doc/devblog/day_249_onward/comment_1_80af832a132ee37a470d0586ae751f2a._comment b/doc/devblog/day_249_onward/comment_1_80af832a132ee37a470d0586ae751f2a._comment
new file mode 100644
index 0000000000..d5984716d8
--- /dev/null
+++ b/doc/devblog/day_249_onward/comment_1_80af832a132ee37a470d0586ae751f2a._comment
@@ -0,0 +1,10 @@
+[[!comment format=mdwn
+ username="https://www.google.com/accounts/o8/id?id=AItOawlOl_Hf78lWy2iVDHYIey_zc1qVBQjlJgw"
+ nickname="Stefan"
+ subject="comment 1"
+ date="2015-02-07T10:04:31Z"
+ content="""
+Please announce that deprecation well in advance, though. I feel your pain and have started to migrate to an S3->Glacier setup myself, but getting data out of Glacier is fairly expensive if done quickly. 
+
+(And due to git-annex' ease of adding more data, I got a fairly large amount of backups, and would also prefer not to be stuck on an old version of git-annex just for Glacier support.)
+"""]]
diff --git a/doc/devblog/day_249_onward/comment_2_ada958a8df4da36b25a456a7700b310f._comment b/doc/devblog/day_249_onward/comment_2_ada958a8df4da36b25a456a7700b310f._comment
new file mode 100644
index 0000000000..881eef278a
--- /dev/null
+++ b/doc/devblog/day_249_onward/comment_2_ada958a8df4da36b25a456a7700b310f._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="joey"
+ subject="""comment 2"""
+ date="2015-02-09T18:38:27Z"
+ content="""
+Don't worry, by "deprecate glacier", I mean at worst "refuse to make new glacier
+remotes". No plans to remove a remote that someone is using.
+"""]]
diff --git a/doc/devblog/day_24__nearly_done_with_gcrypt.mdwn b/doc/devblog/day_24__nearly_done_with_gcrypt.mdwn
new file mode 100644
index 0000000000..22d3fa70b5
--- /dev/null
+++ b/doc/devblog/day_24__nearly_done_with_gcrypt.mdwn
@@ -0,0 +1,23 @@
+So close to being done with gcrypt support.. But still not quite there.
+
+Today I made the UI changes to support gcrypt when setting up a repository
+on a ssh server, and improved the probing and data types so it can tell
+which options the server supports. Fairly happy with how that is turning
+out.
+
+Have not yet hooked up the new buttons to make gcrypt repos. While I was
+testing that my changes didn't break other stuff, I found a bug in the
+webapp that caused it to sometimes fail to transfer one file to/from a
+remote that was just added, because the transferrer process didn't know
+about the new remote yet, and crashed (and was restarted knowing about it,
+so successfully sent any other files). So got sidetracked on fixing that.
+
+Also did some work to make the gpg bundled with git-annex on OSX be
+compatable with the config files written by MacGPG. At first I was going to
+hack it to not crash on the options it didn't support, but it turned out
+that upgrading to version 1.4.14 actually fixed the problem that was making
+it build without support for DNS.
+
+----
+
+Today's work was sponsored by Thomas Hochstein.
diff --git a/doc/devblog/day_250__backog_bugfixing.mdwn b/doc/devblog/day_250__backog_bugfixing.mdwn
new file mode 100644
index 0000000000..9f5d3239bc
--- /dev/null
+++ b/doc/devblog/day_250__backog_bugfixing.mdwn
@@ -0,0 +1,10 @@
+Plowing through the backlog today, and fixing quite a few bugs! Got the
+backlog down to 87 messages from ~140. And some of the things I got to were
+old and/or hard.
+
+About a third of the day was spent revisiting
+[[bugs/git-annex_branch_shows_commit_with_looong_commitlog]].
+I still don't understand how that behavior can happen, but I have a
+donated repository where it did happen. Made several changes to try to make
+the problem less likely to occur, and not as annoying when it does occur,
+and maybe get me more info if it does happen to someone again.
diff --git a/doc/devblog/day_251-252__dusting_off_the_roadmap.mdwn b/doc/devblog/day_251-252__dusting_off_the_roadmap.mdwn
new file mode 100644
index 0000000000..0a0e50f645
--- /dev/null
+++ b/doc/devblog/day_251-252__dusting_off_the_roadmap.mdwn
@@ -0,0 +1,12 @@
+Many more little improvements made yesterday and part of today. While
+it's only been a week since the last release, it feels almost time
+to make another one, after so many recent bug fixes and small improvements.
+
+I've updated the [[design/roadmap]]. I have been operating without a
+roadmap for half a year, and it would be nice to have some plans.
+Keeping up with bug reports and requests as they come in is a fine mode
+of work, but it can feel a little aimless. It's good to have a planned out
+course, or at least some longer term goals.
+
+After the next release, I've penciled in the second half of this month to
+work on the [[design/caching_database]].
diff --git a/doc/devblog/day_253__sqlite_for_incremental_fsck.mdwn b/doc/devblog/day_253__sqlite_for_incremental_fsck.mdwn
new file mode 100644
index 0000000000..0b5f42c79d
--- /dev/null
+++ b/doc/devblog/day_253__sqlite_for_incremental_fsck.mdwn
@@ -0,0 +1,58 @@
+[[!meta title="day 254  sqlite for incremental fsck"]]
+
+Yesterday I did a little more investigation of key/value stores.
+I'd love a pure haskell key/value store that didn't buffer everything in
+memory, and that allowed concurrent readers, and was ACID, and production
+quality. But so far, I have not found anything that meets all those
+criteria. It seems that sqlite is the best choice for now.
+
+Started working on the `database` branch today. The plan is to use
+sqlite for incremental fsck first, and if that works well, do the rest
+of what's planned in [[design/caching_database]].
+
+At least for now, I'm going to use a dedicated database file for each
+different thing. (This may not be as space-efficient due to lacking
+normalization, but it keeps things simple.) 
+
+So, .git/annex/fsck.db will be used by incremental fsck, and it has
+a super simple Persistent database schema:
+
+[[!format haskell """
+Fscked
+  key SKey
+  UniqueKey key
+"""]]
+
+It was pretty easy to implement this and make incremental fsck use it. The
+hard part is making it both fast and robust.
+
+At first, I was doing everything inside a single `runSqlite` action.
+Including creating the table. But, it turns out that runs as a single
+transaction, and if it was interrupted, this left the database in a
+state where it exists, but has no tables. Hard to recover from.
+
+So, I separated out creating the database, made that be done in a separate
+transation and fully atomically. Now `fsck --incremental` could be crtl-c'd
+and resumed with `fsck --more`, but it would lose the transaction and so
+not remember anything had been checked.
+
+To fix that, I tried making a separate transation per file fscked. That
+worked, and it resumes nicely where it left off, but all those transactions
+made it much slower.
+
+To fix the speed, I made it commit just one transaction per minute. This
+seems like an ok balance. Having fsck re-do one minute's work when restarting
+an interrupted incremental fsck is perfectly reasonable, and now the speed,
+using the sqlite database, is nearly as fast as the old sticky bit hack was.
+(Specifically, 6m7s old vs 6m27s new, fscking 37000 files from cold cache
+in --fast mode.)
+
+There is still a problem with multiple concurrent `fsck --more`
+failing. Probably a concurrent writer problem? And, some porting will be
+required to get sqlite and persistent working on Windows and Android.
+So the branch isn't ready to merge yet, but it seems promising.
+
+In retrospect, while incremental fsck has the simplest database schema, it
+might be one of the harder things listed in [[design/caching_database]], 
+just because it involves so many writes to the database. The other use
+cases are more read heavy.
diff --git a/doc/devblog/day_253__sqlite_for_incremental_fsck/comment_1_683d669ac6af8e314585609f75cfdaf3._comment b/doc/devblog/day_253__sqlite_for_incremental_fsck/comment_1_683d669ac6af8e314585609f75cfdaf3._comment
new file mode 100644
index 0000000000..17d1d11cf2
--- /dev/null
+++ b/doc/devblog/day_253__sqlite_for_incremental_fsck/comment_1_683d669ac6af8e314585609f75cfdaf3._comment
@@ -0,0 +1,7 @@
+[[!comment format=mdwn
+ username="https://id.koumbit.net/anarcat"
+ subject="comment 1"
+ date="2015-02-16T21:26:22Z"
+ content="""
+i am curious: why separate database files while you can have multiple tables in the same database file? --[[anarcat]]
+"""]]
diff --git a/doc/devblog/day_253__sqlite_for_incremental_fsck/comment_2_08dd639180ae79addc007ee47d52048a._comment b/doc/devblog/day_253__sqlite_for_incremental_fsck/comment_2_08dd639180ae79addc007ee47d52048a._comment
new file mode 100644
index 0000000000..481c3df4bb
--- /dev/null
+++ b/doc/devblog/day_253__sqlite_for_incremental_fsck/comment_2_08dd639180ae79addc007ee47d52048a._comment
@@ -0,0 +1,7 @@
+[[!comment format=mdwn
+ username="joey"
+ subject="""comment 2"""
+ date="2015-02-17T20:15:36Z"
+ content="""
+@anarcat, see [[design/caching_database]] for my thinking on that.
+"""]]
diff --git a/doc/devblog/day_253__ssh-options.mdwn b/doc/devblog/day_253__ssh-options.mdwn
new file mode 100644
index 0000000000..805363f3c5
--- /dev/null
+++ b/doc/devblog/day_253__ssh-options.mdwn
@@ -0,0 +1,5 @@
+Spent a couple hours to make the ssh-options git config setting be used
+in more places. Now it's used everywhere that git-annex supports ssh
+caching, including the `git pull` and `git push` done by `sync` and by the
+assistant. Also the `remotedaemon` and the gcrypt, rsync, and ddar
+special remotes.
diff --git a/doc/devblog/day_255__sqlite_concurrent_writers_problem.mdwn b/doc/devblog/day_255__sqlite_concurrent_writers_problem.mdwn
new file mode 100644
index 0000000000..05b431a4dc
--- /dev/null
+++ b/doc/devblog/day_255__sqlite_concurrent_writers_problem.mdwn
@@ -0,0 +1,34 @@
+Worked today on making incremental fsck's use of sqlite be safe with
+multiple concurrent fsck processes.
+
+The first problem was that having `fsck --incremental` running and starting a
+new `fsck --incremental` caused it to crash. And with good reason, since
+starting a new incremental fsck deletes the old database, the old process
+was left writing to a database that had been deleted and recreated out from
+underneath it. Fixed with some locking.
+
+Next problem is harder. Sqlite doesn't support multiple concurrent writers
+at all. One of them will fail to write. It's not even possible to have two
+processes building up separate transactions at the same time. Before using
+sqlite, incremental fsck could work perfectly well with multiple fsck
+processes running concurrently. I'd like to keep that working.
+
+My partial solution, so far, is to make git-annex buffer writes, and every
+so often send them all to sqlite at once, in a transaction. So most of the
+time, nothing is writing to the database. (And if it gets unlucky and
+a write fails due to a collision with another writer, it can just wait and
+retry the write later.) This lets multiple processes write to the database
+successfully.
+
+But, for the purposes of concurrent, incremental fsck, it's not ideal.
+Each process doesn't immediately learn of files that another process has
+checked. So they'll tend to do redundant work. Only way I can see to
+improve this is to use some other mechanism for short-term IPC between the
+fsck processes.
+
+----
+
+Also, I made `git annex fsck --from remote --incremental` use a different
+database per remote. This is a real improvement over the sticky bits;
+multiple incremental fscks can be in progress at once, 
+checking different remotes.
diff --git a/doc/devblog/day_256__sqlite_concurrency_argh.mdwn b/doc/devblog/day_256__sqlite_concurrency_argh.mdwn
new file mode 100644
index 0000000000..df02d47be7
--- /dev/null
+++ b/doc/devblog/day_256__sqlite_concurrency_argh.mdwn
@@ -0,0 +1,28 @@
+Breaking news: gitlab.com repositories now support git-annex!
+
+* [GitLab Annex solves the problem of versioning large binaries with git](https://about.gitlab.com/2015/02/17/gitlab-annex-solves-the-problem-of-versioning-large-binaries-with-git/)
+* [freely licensed source code](https://gitlab.com/gitlab-org/gitlab-shell)
+
+A very nice surprise! More git hosters should do this..
+
+----
+
+Back to sqlite concurrency, I thought I had it dealt with, but more testing
+today has turned up a lot more problems with sqlite and concurrent writers
+(and readers).
+
+First, I noticed that a process can be happily writing changes to the
+database, but if a second process starts reading from the database, this
+will make the writier start failing with BUSY, and keep failing until the
+second process goes idle. It turns out the solution to this is to use WAL 
+mode, which prevents readers from blocking writers.
+
+After several hours (persistent doesn't make it easy to enable WAL mode),
+it seemed pretty robust with concurrent fsck.
+
+But then I saw SELECT fail with BUSY. I don't understand why a reader would
+fail in WAL mode; that's counter to the documentation. My best guess is
+that this happens when a checkpoint is being made.
+
+This seems to be a real bug in sqlite. It may only affect the older
+versions bundled with persistent.
diff --git a/doc/devblog/day_257__release_day.mdwn b/doc/devblog/day_257__release_day.mdwn
new file mode 100644
index 0000000000..bd3057145c
--- /dev/null
+++ b/doc/devblog/day_257__release_day.mdwn
@@ -0,0 +1,17 @@
+Today's release doesn't have the database branch merged of course, but it
+still has a significant amount of changes.
+
+Developed a test case for the sqlite problem, that
+reliably reproduces it, and sent it to the sqlite mailing list. It seems
+that under heavy write load, when a new connection is made to the database,
+SELECT can fail for a little while. Once one SELECT succeeds, that database
+connection becomes solid, and won't fail any more (apparently). This makes
+me think there might be some connection initialization steps that don't end
+up finishing before the SELECT goes through in this situation. I should be
+able to work around this problem by probing new connections for stability,
+and probably will have to, since it'll be years before any bug fixed sqlite
+is available everywhere.
+
+I also noticed that current git-annex incremental parallel fsck doesn't
+really parallelize well; eg the processes do duplicate work. So, the
+database branch is not really a regression in this area.
diff --git a/doc/devblog/day_258__database_branch_merged.mdwn b/doc/devblog/day_258__database_branch_merged.mdwn
new file mode 100644
index 0000000000..97cae78b5d
--- /dev/null
+++ b/doc/devblog/day_258__database_branch_merged.mdwn
@@ -0,0 +1,25 @@
+I'm snowed in, but keeping busy..
+
+Developed a complete workaround for the [sqlite SELECT ErrorBusy bug](http://news.gmane.org/find-root.php?message_id=20150219163255.GA13383%40kitenet.net).
+So after a week, I finally have sqlite working robustly. And, I merged in
+the branch that uses sqlite for incremental fsck.
+
+Benchmarking an incremental fsck --fast run, checking 40 thousand files,
+it used to take 4m30s using sticky bits, and using sqlite slowed it down by
+10s. So one added second per 4 thousand or so files. I think that's ok.
+Incremental fsck is intended to be used in big repos, which are probably not
+checked in --fast most, so the checksumming of files will by far swamp
+that overhead.
+
+Also got sqlite and persistent installed on all the autobuilders. This
+was easier than expected, because persistent bundles its own copy of
+sqlite.
+
+That would have been a good stopping place for the day's work.. But then I
+got to spent 5 more hours getting the EvilSplicer to support Persistent.
+Urgh. :-/
+
+Now I can look forward to using sqlite for something more interesting than
+incremental fsck, like metadata caching for views, or the direct mode mappings.
+But, given all the trouble I had with sqlite, I'm going to put that off for
+a little while, to make sure that I've really gotten sqlite to work robustly.
diff --git a/doc/devblog/day_259__submodules.mdwn b/doc/devblog/day_259__submodules.mdwn
new file mode 100644
index 0000000000..a2178a80fb
--- /dev/null
+++ b/doc/devblog/day_259__submodules.mdwn
@@ -0,0 +1,8 @@
+I had thought that git-annex and git submodules couldn't mix. However,
+looking at it again, it turned out to be possible to use git-annex quite
+sanely in a submodule, with just a little tweaking of how git normally
+configures the repository. Details of this still experimental feature are in
+[[/submodules]].
+
+There is still some work to be done to make git-annex work with submodules
+in repositories on filesystems that don't support symlinks.
diff --git a/doc/devblog/day_25__finishing_up_gcrypt.mdwn b/doc/devblog/day_25__finishing_up_gcrypt.mdwn
new file mode 100644
index 0000000000..9666282d0e
--- /dev/null
+++ b/doc/devblog/day_25__finishing_up_gcrypt.mdwn
@@ -0,0 +1,25 @@
+Long day, but I did finally finish up with gcrypt support. More or less.
+
+Got both creating and enabling existing gcrypt repositories on ssh servers
+working in the webapp. (But I ran out of time to make it detect when the
+user is manually entering a gcrypt repo that already exists. Should be easy
+so maybe tomorrow.)
+
+Fixed several bugs in git-annex's gcrypt support that turned up in testing.
+Made git-annex ensure that a gcrypt repository does not have
+receive.denyNonFastForwards set, because gcrypt relies on always forcing
+the push of the branch it stores its manifest on. Fixed a bug in
+`git-annex-shell recvkey` when it was receiving a file from an annex in
+direct mode.
+
+Also had to add a new `git annex shell gcryptsetup` command, which is
+needed to make setting up a gcrypt repository work when the assistant
+has set up a locked-down ssh key that can only run git-annex-shell. Painted
+myself into a bit of a corner there.
+
+And tested, tested, tested. So many possibilities and edge cases in this
+part of the code..
+
+----
+
+Today's work was sponsored by Hendrik Müller Hofstede.
diff --git a/doc/devblog/day_260__random_month.mdwn b/doc/devblog/day_260__random_month.mdwn
new file mode 100644
index 0000000000..2d71c04dd0
--- /dev/null
+++ b/doc/devblog/day_260__random_month.mdwn
@@ -0,0 +1,27 @@
+This month is going to be a bit more random than usual where git-annex
+development is concerned.
+
+* On Saturday, the [Seven Day Roguelike](http://7drl.org/) competition
+  begins, and I will be spending a week building a game in haskell,
+  to the exclusion of almost all other work.
+* On March 18th, I'll be at the [Boston Haskell User's group](http://www.meetup.com/Boston-Haskell/events/219298257/).
+  (Attending, not presenting.)
+* March 19-20, I'll be at Dartmouth visiting with the DataLad developers
+  and learning more about what it needs from git-annex.
+* March 21-22, I'll be at the FSF's [LibrePlanet](https://libreplanet.org/2015)
+  conference at MIT.
+
+Got started on the randomness today with this 
+[[design proposal for using git-annex to back up the entire Internet Archive|design/iabackup]].
+This is something the Archive Team is [considering taking on](http://archiveteam.org/index.php?title=INTERNETARCHIVE.BAK),
+and I had several hours driving and hiking to think about it and came up
+with a workable design. (Assuming large enough crowd of volunteers.)
+
+Don't know if it will happen, but it was a useful thought problem to see how
+git-annex works, and doesn't work in this unusual use case.
+
+One interesting thing to come out of that is that git-annex fsck does not
+currently make any record of successful fscks. In a very large distributed
+system, it can be useful to have successful fscks of an object's content recorded,
+by updating the timestamp in the location log to say "this repository still
+had the content at this time".
diff --git a/doc/devblog/day_261__random_improvements.mdwn b/doc/devblog/day_261__random_improvements.mdwn
new file mode 100644
index 0000000000..bc0478e5cd
--- /dev/null
+++ b/doc/devblog/day_261__random_improvements.mdwn
@@ -0,0 +1,5 @@
+Fixed a mojibake bug that affected metadata values that included both
+whitespace and unicode characters. This was very fiddly to get right.
+
+Finished up Monday's work to support submodules, getting them working
+on filesystems that don't support symlinks.
diff --git a/doc/devblog/day_262__ipfs.mdwn b/doc/devblog/day_262__ipfs.mdwn
new file mode 100644
index 0000000000..9ae79c8c77
--- /dev/null
+++ b/doc/devblog/day_262__ipfs.mdwn
@@ -0,0 +1,16 @@
+Did a deep dive into [ipfs](http://ipfs.io/) last night. It has great
+promise.
+
+As a first step toward using it with git-annex, I built an experimental
+[[ipfs_special_remote|special_remotes/ipfs]]. It has some nice abilities;
+any ipfs address can be downloaded to a file in the repository:
+
+	git annex addurl ipfs:QmYgXEfjsLbPvVKrrD4Hf6QvXYRPRjH5XFGajDqtxBnD4W --file somefile
+
+And, any file in the git-annex repository can be published to the world
+via ipfs, by simply using `git annex copy --to ipfs`. The ipfs address
+for the file is then visible in `git annex whereis`.
+
+Had to extend the external special remote protocol slightly for that, so
+that ipfs addresses can be recorded as uris in git-annex, and will show up
+in `git annex whereis`.
diff --git a/doc/devblog/day_263__diving_back_in.mdwn b/doc/devblog/day_263__diving_back_in.mdwn
new file mode 100644
index 0000000000..29a73774bc
--- /dev/null
+++ b/doc/devblog/day_263__diving_back_in.mdwn
@@ -0,0 +1,7 @@
+After an intense week away, I didn't mean to work on git-annex today, but I
+got sucked back in..
+
+Worked on some plumbing commands for mass repository creation.
+Made `fromkey` be able to read a stream of files to create from stdin.
+Added a new `registerurl` plumbing command, that reads a stream of keys and
+urls from stdin.
diff --git a/doc/devblog/day_264__catching_up.mdwn b/doc/devblog/day_264__catching_up.mdwn
new file mode 100644
index 0000000000..6c68d0be45
--- /dev/null
+++ b/doc/devblog/day_264__catching_up.mdwn
@@ -0,0 +1,7 @@
+Caught up with most of the recent backlog today. Was not very bad.
+
+Fixed `remotedaemon` to support gcrypt remotes, which was never
+quite working before.
+
+Seem to be on track to making a release tomorrow with a whole month's
+changes.
diff --git a/doc/devblog/day_265__at_Dartmouth.mdwn b/doc/devblog/day_265__at_Dartmouth.mdwn
new file mode 100644
index 0000000000..80d6df54e6
--- /dev/null
+++ b/doc/devblog/day_265__at_Dartmouth.mdwn
@@ -0,0 +1,4 @@
+Spent a couple of days at Dartmouth hanging out in the neuroscience
+department with the [Datalad](http://datalad.org/) developers.
+Added several new plumbing commands and a new post-update-annex hook,
+based on their feedback of how they're using git-annex.
diff --git a/doc/devblog/day_266-267__man_page_split.mdwn b/doc/devblog/day_266-267__man_page_split.mdwn
new file mode 100644
index 0000000000..6239e7dfdf
--- /dev/null
+++ b/doc/devblog/day_266-267__man_page_split.mdwn
@@ -0,0 +1,16 @@
+While traveling for several days, I filled dead time with a rather massive
+reorganization of the git-annex man page, and I finished that up this
+morning.
+
+That man page had gotten rather massive, at around 3 thousand lines. I
+split out 87 man pages, one for each git-annex command. Many of these were
+expanded with additional details, and have become a lot better thanks
+to the added focus and space. See for example, [[git-annex-find]],
+or any of the links on the new [[git-annex]] man page. (Which is still over
+1 thousand lines long..)
+
+Also, `git annex help ` can be used to pull up a command's man
+page now!
+
+I'm taking the rest of the day off to R&R from the big trip north, and
+expect to get back into the backlog of 143 messages starting tomorrow.
diff --git a/doc/devblog/day_266-267__man_page_split/comment_1_d55600917128cb0ce38002fbf6f0b088._comment b/doc/devblog/day_266-267__man_page_split/comment_1_d55600917128cb0ce38002fbf6f0b088._comment
new file mode 100644
index 0000000000..a8d5128e3f
--- /dev/null
+++ b/doc/devblog/day_266-267__man_page_split/comment_1_d55600917128cb0ce38002fbf6f0b088._comment
@@ -0,0 +1,7 @@
+[[!comment format=mdwn
+ username="https://id.koumbit.net/anarcat"
+ subject="nice!"
+ date="2015-03-25T16:28:47Z"
+ content="""
+that's a great idea!! hopefully those smaller manpages can be expanded with more examples and tips as well!
+"""]]
diff --git a/doc/devblog/day_266-267__man_page_split/comment_2_63791201452a6ab30a35cd742434acca._comment b/doc/devblog/day_266-267__man_page_split/comment_2_63791201452a6ab30a35cd742434acca._comment
new file mode 100644
index 0000000000..95bf5dde2e
--- /dev/null
+++ b/doc/devblog/day_266-267__man_page_split/comment_2_63791201452a6ab30a35cd742434acca._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="https://www.google.com/accounts/o8/id?id=AItOawnx8kHW66N3BqmkVpgtXDlYMvr8TJ5VvfY"
+ nickname="Yaroslav"
+ subject="works like charm"
+ date="2015-03-31T02:29:04Z"
+ content="""
+THANKS!
+"""]]
diff --git a/doc/devblog/day_266-267__man_page_split/comment_3_d7764c605ad338c4bf4c4b46b54abd8a._comment b/doc/devblog/day_266-267__man_page_split/comment_3_d7764c605ad338c4bf4c4b46b54abd8a._comment
new file mode 100644
index 0000000000..0e7f793bb9
--- /dev/null
+++ b/doc/devblog/day_266-267__man_page_split/comment_3_d7764c605ad338c4bf4c4b46b54abd8a._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="https://www.google.com/accounts/o8/id?id=AItOawnx8kHW66N3BqmkVpgtXDlYMvr8TJ5VvfY"
+ nickname="Yaroslav"
+ subject="comment 3"
+ date="2015-03-31T02:32:04Z"
+ content="""
+I only wish it was identical to git syntax (git COMMAND --help for manpage full doc, git COMMAND -h for short summary), but iirc I have whined before and it wasn't trivial to achieve.  So current one is next akin ;-) 
+"""]]
diff --git a/doc/devblog/day_268_stressed_out.mdwn b/doc/devblog/day_268_stressed_out.mdwn
new file mode 100644
index 0000000000..e2fd6159be
--- /dev/null
+++ b/doc/devblog/day_268_stressed_out.mdwn
@@ -0,0 +1,18 @@
+While I plowed through a lot of backlog the past several days,
+I still have some 120 messages piled deep.
+
+That work did result in a number of improvements, culminating in a rather
+rushed release of version 5.20150327 today, to fix a regression affecting
+`git annex sync` when using the standalone linux tarballs. Unfortunately, I
+then had to update those tarballs a second time after the release
+as the first fix was incomplete.
+
+And, I'm feeling super stressed out. At this point, I think I should step
+away until the end of the month. Unfortunately, this will mean more backlog
+later. Including lots of noise and hand-holding that I just don't seem to
+have time for if I want to continue making forward progress.
+
+Maybe I'll think of a way to deal with it while I'm away. Currently, all I
+have is that I may have to start ignoring irc and the forum, and
+de-prioritizing bug reports that don't have either a working reproduction
+recipe or multiple independent confirmations that it's a real bug.
diff --git a/doc/devblog/day_268_stressed_out/comment_1_d28352c9fbc988ece19688a687d30ffe._comment b/doc/devblog/day_268_stressed_out/comment_1_d28352c9fbc988ece19688a687d30ffe._comment
new file mode 100644
index 0000000000..bb603e27b7
--- /dev/null
+++ b/doc/devblog/day_268_stressed_out/comment_1_d28352c9fbc988ece19688a687d30ffe._comment
@@ -0,0 +1,10 @@
+[[!comment format=mdwn
+ username="https://www.google.com/accounts/o8/id?id=AItOawmtJGSG74JdKxtaMvHQTTeq8P2yxXmuPD0"
+ nickname="Kenneth"
+ subject="comment 1"
+ date="2015-03-28T03:36:15Z"
+ content="""
+\" and de-prioritizing bug reports that don't have either a working reproduction recipe or multiple independent confirmations that it's a real bug.\"
+
+This seems quite reasonable. Hope you get a little downtime to relax.
+"""]]
diff --git a/doc/devblog/day_268_stressed_out/comment_2_f7b04b2e96f0fc7530f1c21ccfcfb0f9._comment b/doc/devblog/day_268_stressed_out/comment_2_f7b04b2e96f0fc7530f1c21ccfcfb0f9._comment
new file mode 100644
index 0000000000..dffce0772b
--- /dev/null
+++ b/doc/devblog/day_268_stressed_out/comment_2_f7b04b2e96f0fc7530f1c21ccfcfb0f9._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="https://openid.stackexchange.com/user/e65e6d0e-58ba-41de-84cc-1f2ba54cf574"
+ nickname="Mica"
+ subject="You should take a break"
+ date="2015-03-29T20:37:16Z"
+ content="""
+By all means, take a break Joey! 
+"""]]
diff --git a/doc/devblog/day_268_stressed_out/comment_3_0eb4c17d35fba9db6a7446a5f253ace9._comment b/doc/devblog/day_268_stressed_out/comment_3_0eb4c17d35fba9db6a7446a5f253ace9._comment
new file mode 100644
index 0000000000..f098c700ad
--- /dev/null
+++ b/doc/devblog/day_268_stressed_out/comment_3_0eb4c17d35fba9db6a7446a5f253ace9._comment
@@ -0,0 +1,24 @@
+[[!comment format=mdwn
+ username="anarcat"
+ subject="take care and ideas"
+ date="2015-03-30T16:26:05Z"
+ content="""
+we not careful enough with ourselves and with each other... i can only send my warmest support for your self-imposed break, i should probably have done that myself a few times, and never had the lucidity to do it soon enough... so yeah, break time, and enjoy spring! :)
+
+git-annex has grown in leaps and bounds in the last year, and it looks like it's getting more and more the traction it deserves. that means a lot more support requests and yes, a lot more noise in the forums and issue queues. in my experience, having the core devs step away from the support forums isn't necessarily a bad thing: it will leave some room for other people to step up and start cooking up their own answers. i did a very quick analysis of the `doc` subdirectory of the current git repo, and at least a third of the data is injected in the forum:
+
+
    +$ ncdu doc/
+[...]
+   13,0MiB  3 270 [##########] /forum
+   10,4MiB  2 243 [########  ] /bugs
+    4,4MiB    964 [###       ] /design
+    2,2MiB    550 [#         ] /todo
+    1,6MiB    398 [#         ] /devblog
+    1,5MiB    348 [#         ] /tips
+[...]
+ Total disk usage:  38,3MiB  Apparent size:  15,4MiB  Items: 8711
+
    
+
+I wonder if moving this out of the wiki (to, say, a mailing list or Stack-exchange like Q&A site) would be helpful? It seems there is a significant amount of FAQ and divergence from topic on the forum posts... trying another tool may be helpful here (but I know how annoying and crazy this sounds at first...).
+"""]]
diff --git a/doc/devblog/day_268_stressed_out/comment_4_fecf179be92c3a3e25f4a47fcea6b6ec._comment b/doc/devblog/day_268_stressed_out/comment_4_fecf179be92c3a3e25f4a47fcea6b6ec._comment
new file mode 100644
index 0000000000..f042f93943
--- /dev/null
+++ b/doc/devblog/day_268_stressed_out/comment_4_fecf179be92c3a3e25f4a47fcea6b6ec._comment
@@ -0,0 +1,10 @@
+[[!comment format=mdwn
+ username="https://openid.stackexchange.com/user/27ceb3c5-0762-42b8-8f8a-ed21c284748f"
+ nickname="g"
+ subject="SuperUser"
+ date="2015-03-31T01:09:20Z"
+ content="""
+Totally agree that the main developers, and especially you, Joey, shouldn't have to be handling most of the support you are now. There's already a reasonable amount of [git-annex-related action over on the SuperUser Stack Exchage](http://superuser.com/search?q=git-annex), though the actual [`git-annex` tag seems to only have been used once](http://superuser.com/questions/tagged/git-annex)
+
+Echoing #3, it does seem like it would make sense to move more of the basic support work over there from the forum. For this sort of Q&A, I find Stack Exchange a bit better structured, a little harder to miss past questions before re-posting, and like that it has a built-in community of people to help answer.
+"""]]
diff --git a/doc/devblog/day_269__wildcards_and_podcasts.mdwn b/doc/devblog/day_269__wildcards_and_podcasts.mdwn
new file mode 100644
index 0000000000..af17eebfa3
--- /dev/null
+++ b/doc/devblog/day_269__wildcards_and_podcasts.mdwn
@@ -0,0 +1,26 @@
+Turns out that git has a feature I didn't know about; it will expand
+wildcards and other stuff in filenames passed to many git commands. This is
+on top of the shell's expansion.
+
+That led to some broken behavior by `git annex add 'foo.*'`
+and, it could lead to other probably unwanted behavior, like `git annex
+drop 'foo[barred]'` dropping a file named `food` in addition to
+`foo[barred]`
+
+For now, I've disabled this git feature throughout git-annex. If you relied
+on it for something, let me know, I might think about adding it back in
+specific places where it makes sense.
+
+----
+
+Improved `git annex importfeed` to check the itemid of the feed and avoid
+re-downloading a file with the same itemid. Before, it would add duplicate
+files if a feed kept the itemid the same, but changed the url. This was
+easier than expected because annex.genmetadata already caused the itemid
+to be stored in the git-annex metadata. I just had to make it check the
+itemid metadata, and set itemid even when annex.genmetadata isn't set.
+
+----
+
+Also got 4 other bug reports fixed, even though I feel I'm taking it easy
+today. It's good to be relaxed again!
diff --git a/doc/devblog/day_26__gcrypt_really_done_this_time.mdwn b/doc/devblog/day_26__gcrypt_really_done_this_time.mdwn
new file mode 100644
index 0000000000..347e4be5f7
--- /dev/null
+++ b/doc/devblog/day_26__gcrypt_really_done_this_time.mdwn
@@ -0,0 +1,17 @@
+Did I say it would be easy to make the webapp detect when a gcrypt repository 
+already existed and enable it? Well, it wasn't exactly hard, but it took
+over 300 lines of code and 3 hours..
+
+So, gcrypt support is done for now. The glaring omission is gpg key
+management for sharing gcrypt repositories between machines and/or people.
+But despite that, I think it's solid, and easy to use, and covers some
+great use cases.
+
+Pushed out a release.
+
+Now I really need to start thinking about
+[[design/assistant/disaster_recovery]].
+
+----
+
+Today's work was sponsored by Dominik Wagenknecht.
diff --git a/doc/devblog/day_270__distributed_fsck.mdwn b/doc/devblog/day_270__distributed_fsck.mdwn
new file mode 100644
index 0000000000..0e25acb2bf
--- /dev/null
+++ b/doc/devblog/day_270__distributed_fsck.mdwn
@@ -0,0 +1,27 @@
+Added two options to `git annex fsck` that allow for a form of distributed
+fsck.  This is useful in situations where repositiories cannot be trusted to
+continue to exist, and cannot be checked directly, but you'd still like to
+keep track of their status. [[design/iabackup]] is one use case for this.
+
+By running a periodic fsck with the --distributed option,
+the repositories can verify that they still exist and that the
+information about their contents is still accurate. This is done by
+doing an extra update of the location log each time a file is verified by
+fsck to still be in the repository.
+
+The other option looks like --expire="30d somerepo:60d". It checks that
+each specified repository has recorded a distributed fsck within the specified
+time period. If not, the repository is dropped from the location tracking
+log. Of course it can always update that later if it's really still around.
+
+Distributed fsck is not the default because those extra location log updates
+increase the size of the git-annex branch. I did one thing to keep the size
+increase small: An identical line is logged to for each key, including the
+timestamp, so git's delta compression will work as well as is possible. But,
+there's still commit and tree update overhead. 
+
+Probably doesn't make sense to run distributed fscks too often for that and
+other reasons. If the git-annex branch does get too large, there's always
+`git annex forget` ...
+
+**(Update: This was later rethought and works much more efficiently now..)**
diff --git a/doc/devblog/day_271__parallel_get_groundwork.mdwn b/doc/devblog/day_271__parallel_get_groundwork.mdwn
new file mode 100644
index 0000000000..7f7a3afbaa
--- /dev/null
+++ b/doc/devblog/day_271__parallel_get_groundwork.mdwn
@@ -0,0 +1,32 @@
+I've started work on [[todo/parallel_get]].
+Today, laid the groundwork in two areas:
+
+1. Evalulated the ascii-progress haskell library. It can display
+   multiple progress bars in the terminal, portably, and its author
+   Pedro Tacla Yamada has kindly offered to improve it to meet
+   git-annex's needs.
+
+   I ended up filing [10 issues](https://github.com/yamadapc/haskell-ascii-progress/issues)
+   on it today, around 3 of the are blockers for git-annex using it.
+
+2. Worked on making --quiet more quiet. Commands like rsync and wget
+   need to have thier progress output disabled when run in parallel.
+
+   Didn't quite finish this yet.
+
+---
+
+Yesterday I made some improvements to how git-annex behaves when it's
+passed a massive number of directories or files on the command line.
+Eg, when driven by xargs. There turned out to be some bugs in that
+scenario.
+
+One problem one I kind of had to paper over. While git-annex get
+normally is careful to get the files in the same order they were listed on
+the command line, it becomes very expensive to expand directories using
+git-ls-files, and reorder its output to preserve order, when a large number
+offiles are passed on the command line. There was a O(N*M) time blowup.
+
+I worked around it by making it only preserve the order of the first 100
+files. Assumption being that if you're specifying so many files on the
+command line, you probably have less of an attachment to their ordering. :)
diff --git a/doc/devblog/day_272__forest_for_trees.mdwn b/doc/devblog/day_272__forest_for_trees.mdwn
new file mode 100644
index 0000000000..cdbde54a35
--- /dev/null
+++ b/doc/devblog/day_272__forest_for_trees.mdwn
@@ -0,0 +1,13 @@
+Rethought distributed fsck. It's not really a fsck, but an expiration of
+inactive repositories, where fscking is one kind of activity. That insight
+let me reimplement it much more efficiently. Rather than updating all
+the location logs to prove it was active, `git annex fsck` can simply and
+inexpensively update an activity log. It's so cheap it'll do it by default.
+The `git annex expire` command then reads the activity log and expires
+(or unexpires) repositories that have not been active in the desired time
+period. Expiring a repository simply marks it as dead.
+
+Yesterday, finished making --quiet really be quiet. That sounds easy,
+but it took several hours. On the `concurrentprogress` branch, I have
+ascii-progress hooked up and working, but it's not quite ready for prime
+time.
diff --git a/doc/devblog/day_273__unexpected_release.mdwn b/doc/devblog/day_273__unexpected_release.mdwn
new file mode 100644
index 0000000000..5b902c892c
--- /dev/null
+++ b/doc/devblog/day_273__unexpected_release.mdwn
@@ -0,0 +1,15 @@
+I've had to release git-annex twice this week to fix reversions. On Monday,
+just after I made a planned release, I discovered a bug in it, and had to
+update it with a .1 release. Today's release fixes 2 other reversions
+introduced by recent changes, both only affecting the assistant.
+
+Before making today's release, I did a bunch of other minor bugfixes and
+improvements, including adding a new `contentlocationn` plumbing command.
+This release also changes `git annex add` when annex.largefiles is
+configured, so it will `git add` the non-large files. That is particularly
+useful in direct mode.
+
+I feel that the assistant needs some TLC, so I might devote a week to it in
+the latter part of this month. My current funding doesn't cover work
+on the assistant, but I should have some spare time toward the end of the
+month.
diff --git a/doc/devblog/day_274__concurrent_annex_state.mdwn b/doc/devblog/day_274__concurrent_annex_state.mdwn
new file mode 100644
index 0000000000..3aa73c0f3b
--- /dev/null
+++ b/doc/devblog/day_274__concurrent_annex_state.mdwn
@@ -0,0 +1,42 @@
+Back working on `git annex get --jobs=N` today. It was going very well,
+until I realized I had a hard problem on my hands.
+
+The hard problem is that the AnnexState structure at the core of git-annex
+is not able to be shared amoung multiple threads at all. There's too much
+complicated mutable state going on in there for that to be feasible at all.
+
+In the git-annex assistant, which uses many threads, I long ago worked
+around this problem, by having a single shared AnnexState and when a thread
+needs to run an Annex action, it blocks until no other thread is using it.
+This worked ok for the assistant, with a little bit of thought to avoid
+long-duration Annex actions that could stall the rest of it.
+
+That won't work for concurrent `get` etc. I spent a while investigating maybe
+making AnnexState thread safe, but it's just not built for it. Too many
+ways that can go wrong. For example, there's a CatFileHandle in the
+AnnexState. If two threads are running, they can both try to talk to the
+same `git cat-file --batch` command at once, with bad results. Worse, yet,
+some parts of the code do things like modifying the AnnexState's Git repo
+to add environment variables to use when running git commands.
+
+It's not all gloom and doom though. Only very isolated parts of the code
+change the working directory or set environment variables. And the
+assistant has surely smoked out other thread concurrency problems already.
+And, separate `git-annex` programs can be run concurrently with no problems
+at all; it uses file locking to avoid different processes getting in
+each-others' way. So AnnexState is the only remaining obstacle to concurrency.
+
+So, here's how I've worked around it: When `git annex get -J10` is run,
+it will start by allocating 10 job slots. A fresh AnnexState will be
+created, and copied into each slot. Each time a job runs, it uses its
+slot's own AnnexState. This means 10 `git cat-file` processes,
+and maybe some contention over lock files, but generally, a nice, easy,
+and hopefully trouble-free multithreaded mode.
+
+And indeed, I've gotten `git annex get -J10` working robustly!
+And from there it was trivial to enable -J for `move` and `copy` and `mirror`
+too!
+
+The only real blocker to merging the concurrentprogress branch is some bugs
+in the ascii-progress library that make it draw very scrambled progress
+bars the way git-annex uses it.
diff --git a/doc/devblog/day_274__concurrent_annex_state/comment_1_7414fc0dde7a1d1ee456f8eba0b0c2a9._comment b/doc/devblog/day_274__concurrent_annex_state/comment_1_7414fc0dde7a1d1ee456f8eba0b0c2a9._comment
new file mode 100644
index 0000000000..b4e2eeef82
--- /dev/null
+++ b/doc/devblog/day_274__concurrent_annex_state/comment_1_7414fc0dde7a1d1ee456f8eba0b0c2a9._comment
@@ -0,0 +1,11 @@
+[[!comment format=mdwn
+ username="https://id.koumbit.net/anarcat"
+ subject="comment 1"
+ date="2015-04-10T21:33:02Z"
+ content="""
+great news!
+
+one thing i've been wondering after fooling around with the git-annex branch outside of git-annex is why git-annex talks with the commandline git client at all? libgit, for example, seem to access the .git objects directly without a dependency on the git commandline... there doesn't seem to be any haskell shims for libgit, but it seems to me it would reduce the overhead of a bunch of stuff in git-annex...
+
+as an aside, any thoughts of making the [git-annex-specific git library](http://source.git-annex.branchable.com/?p=source.git;a=tree;f=Git;hb=HEAD) portable and standalone? maybe in collaboration with the existing [hs-libgit](https://hackage.haskell.org/package/libgit)?
+"""]]
diff --git a/doc/devblog/day_274__concurrent_annex_state/comment_2_4ca498ee4b4aaac8ee6dbc2c769dbad7._comment b/doc/devblog/day_274__concurrent_annex_state/comment_2_4ca498ee4b4aaac8ee6dbc2c769dbad7._comment
new file mode 100644
index 0000000000..e8629e5323
--- /dev/null
+++ b/doc/devblog/day_274__concurrent_annex_state/comment_2_4ca498ee4b4aaac8ee6dbc2c769dbad7._comment
@@ -0,0 +1,21 @@
+[[!comment format=mdwn
+ username="joey"
+ subject="""comment 2"""
+ date="2015-04-11T14:41:47Z"
+ content="""
+Josh Tripplet has some haskell bindings for libgit2 somewhere.
+My reasons for not using it so far include:
+
+* ABI stability; at least it used to have none. soname is 21 already..
+* Josh told me parts of it are much less optimised than git.
+  (This was several years ago, but I still imagine the git code base
+  has much more work on speed.)
+* It's not even been in a stable release of Debian yet.
+* Adding a C library dependency will make git-annex much harder for
+  users to get started building.
+* The couple of things that I could really use a git library for, like
+  index file access and catting object contents, could be implemented
+  just as well (and likely as fast) in pure haskell
+  code, and would not be particularly hard to do either. There may even
+  be suitable pure haskell libraries for them; haven't checked.
+"""]]
diff --git a/doc/devblog/day_275-276__mostly_Windows.mdwn b/doc/devblog/day_275-276__mostly_Windows.mdwn
new file mode 100644
index 0000000000..bcdc94696d
--- /dev/null
+++ b/doc/devblog/day_275-276__mostly_Windows.mdwn
@@ -0,0 +1,20 @@
+Mostly working on Windows recently. Fixed handling of git
+repos on different drive letters. Fixed crazy start menu loop. Worked around
+stange msysgit version problem.
+
+Also some more work on the `concurrentprogress` branch, making the progress
+display prettier.
+
+Added one nice new feature yesterday: `git annex info $dir` now includes a
+table of repositories that are storing files in the directory, with their
+sizes.
+
+	repositories containing these files: 
+		288.98 MB: ca9c5d52-f03a-11df-ac14-6b772ffe59f9 -- archive-5
+		288.98 MB: f1c0ce8d-d848-4d21-988c-dd78eed172e8 -- archive-8
+		 10.48 MB: 587b9ccf-4548-4d6f-9765-27faecc4105f -- darkstar
+		 15.18 kB: 42d47daa-45fd-11e0-9827-9f142c1630b3 -- origin
+
+Nice thing about this feature is it's done for free, with no extra work other
+than a little bit of addition. All the heavy location lookup work was already
+being done to get the numcopies stats.
diff --git a/doc/devblog/day_277__thanks.mdwn b/doc/devblog/day_277__thanks.mdwn
new file mode 100644
index 0000000000..de82769bb4
--- /dev/null
+++ b/doc/devblog/day_277__thanks.mdwn
@@ -0,0 +1,20 @@
+Recent work has included improving `fsck --from remote` (and fixing a
+reversion caused by the relative path changes in January), and making
+annex.diskreserve be checked in more cases. And added a `git annex
+required` command for setting [[required_content]].
+
+Also, I want to thank several people for their work:
+
+* Roy sent a patch to enable http proxy support.. despite
+  having only learned some haskell by "30 mins with YAHT". I investigated
+  that more, and no patch is actually necessary, but just a newer version
+  of the http-client library.
+* CandyAngel has been posting lots of helpful comments on the website,
+  including [this tip](http://git-annex.branchable.com/forum/__34__git_annex_sync__34___synced_after_8_hours/#comment-890ca1381d800ac833ccbb8c5db175ea)
+  that significantly speeds up a large git repository.
+* Øyvind fixed a lot of typos throughout the git-annex
+  documentation.
+* Yaroslav has created a `git-annex-standalone.deb` package
+  that will work on any system where debian packages can be installed,
+  no matter how out of date it is (within reason), using the same
+  methods as the standalone tarball.
diff --git a/doc/devblog/day_278__release_day.mdwn b/doc/devblog/day_278__release_day.mdwn
new file mode 100644
index 0000000000..22d637c89e
--- /dev/null
+++ b/doc/devblog/day_278__release_day.mdwn
@@ -0,0 +1,25 @@
+I hope that today's git-annex release will be landing in Debian unstable
+toward the end of the month. And I'm looking forward to some changes that
+have been blocked by wanting to keep git-annex buildable on Debian 7.
+
+Yesterday I got rid of the [SHA](http://hackage.haskell.org/package/SHA/)
+dependency, switching git-annex to use a newer version of cryptohash for
+HMAC generation (which its author Vincent Hanquez kindly added to it when I
+requested it, waay back in 2013). I'm considering using the LambdaCase
+extension to clean up a lot of the code next, and there are 500+ lines of
+old yesod compatability code I can eventually remove.
+
+These changes and others will prevent backporting to the soon to be Debian
+oldstable, but the standalone tarball will still work there. And, the
+git-annex-standalone.deb that can be installed on any version of Debian is
+[now available from the NeuroDebian repository](http://neuro.debian.net/install_pkg.html?p=git-annex-standalone),
+and its build support has been merged into the source tree.
+
+In the run up to the release today, I also dealt with getting the
+Windows build tested and working, now that it's been updated to newer
+versions of rsync, ssh, etc from Cygwin. Had to add several more dlls to
+the installer. That testing also turned up a case where `git-annex init`
+could fail, which got a last-minute fix.
+
+PS, scroll down [this 10 year of git timeline](https://www.atlassian.com/git/articles/10-years-of-git/)
+and see what you find!
diff --git a/doc/devblog/day_279__.mdwn b/doc/devblog/day_279__.mdwn
new file mode 100644
index 0000000000..25eb9a64e1
--- /dev/null
+++ b/doc/devblog/day_279__.mdwn
@@ -0,0 +1,10 @@
+Posted a design for [[design/balanced_preferred_content]]. This would let
+preferred content expressions assign each file to N repositories out of a
+group, selected using Math. Adding a repository could optionally be
+configured to automatically rebalance the files (not very bandwidth
+efficiently though). I think some have asked for a feature like this
+before, so read the design and see if it would be useful for you.
+
+Spent a while debugging a problem with a S3 remote, which seems to have
+been a misconfiguration in the end. But several improvements came out of
+it to make it easier to debug S3 in the future etc.
diff --git a/doc/devblog/day_27__locking_fun.mdwn b/doc/devblog/day_27__locking_fun.mdwn
new file mode 100644
index 0000000000..ef0c4131f9
--- /dev/null
+++ b/doc/devblog/day_27__locking_fun.mdwn
@@ -0,0 +1,49 @@
+Started the day by getting the builds updated for yesterday's release. This
+included making it possible to build git-annex with Debian stable's version
+of cryptohash. Also updated the Debian stable backport to the previous
+release.
+
+----
+
+The [[design/roadmap]] has this month devoted to improving git-annex's
+support for recovering from disasters, broken repos, and so on. Today I've
+been working on the first thing on [[the_list|design/assistant/disaster_recovery]],
+stale git index lock files.
+
+It's unfortunate that git uses simple files for locking, and does not use
+fcntl or flock to prevent the stale lock file problem. Perhaps they want
+it to work on broken NFS systems? The problem with that line of thinking is
+is means all non-broken systems end up broken by stale lock files. Not a
+good tradeoff IMHO.
+
+There are actually two lock files that can end up stale when using
+git-annex; both `.git/index.lock` and `.git/annex/index.lock`. Today I
+concentrated on the latter, because I saw a way to prevent it from ever
+being a problem. All updates to that index file are done by git-annex when
+committing to the git-annex branch. git-annex already uses fcntl locking
+when manipulating its journal. So, that can be extended to also cover
+committing to the git-annex branch, and then the git `index.lock` file
+is irrelevant, and can just be removed if it exists when a commit is
+started.
+
+To ensure this makes sense, I used the type system to prove that the journal
+locking was in effect everywhere I need it to be. Very happy I was able to
+do that, although I am very much a novice at using the type system for
+interesting proofs. But doing so made it very easily to build up to a point
+where I could unlink the `.git/annex/index.lock` and be sure it was safe to do
+that.
+
+----
+
+What about stale `.git/index.lock` files? I don't think it's appropriate
+for git-annex to generally recover from those, because it would change
+regular git command line behavior, and risks breaking something. However, I
+do want the assistant to be able to recover if such a file exists when it
+is starting up, since that would prevent it from running. Implemented that
+also today, although I am less happy with the way the assistant detects
+when this lock file is stale, which is somewhat heuristic (but should work
+even on networked filesystems with multiple writing machines).
+
+----
+
+Today's work was sponsored by Torbjørn Thorsen.
diff --git a/doc/devblog/day_27__locking_fun/comment_1_0eb247235fbf8f563934f3548e1d2e10._comment b/doc/devblog/day_27__locking_fun/comment_1_0eb247235fbf8f563934f3548e1d2e10._comment
new file mode 100644
index 0000000000..261fa005c6
--- /dev/null
+++ b/doc/devblog/day_27__locking_fun/comment_1_0eb247235fbf8f563934f3548e1d2e10._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="http://cstork.org/"
+ nickname="Chris Stork"
+ subject="News page not updated"
+ date="2013-10-04T09:38:21Z"
+ content="""
+Already for the last three or so releases the News page wasn't updated. :-( I think many people (including me) used this page to check if their version is uptodate. Posting the newest version there motivates people to try the very latest which seems very desirable for the git-annex development.
+"""]]
diff --git a/doc/devblog/day_27__locking_fun/comment_2_e8b1dfe1b0641e031d05733448b7bc8b._comment b/doc/devblog/day_27__locking_fun/comment_2_e8b1dfe1b0641e031d05733448b7bc8b._comment
new file mode 100644
index 0000000000..c073f735a4
--- /dev/null
+++ b/doc/devblog/day_27__locking_fun/comment_2_e8b1dfe1b0641e031d05733448b7bc8b._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="http://joeyh.name/"
+ ip="4.152.108.243"
+ subject="comment 2"
+ date="2013-10-04T20:13:37Z"
+ content="""
+Sorry about that. It turns out my release script was broken.
+"""]]
diff --git a/doc/devblog/day_27__locking_fun/comment_3_b67f8ef4ed42b49c8c2e6c4e53163b16._comment b/doc/devblog/day_27__locking_fun/comment_3_b67f8ef4ed42b49c8c2e6c4e53163b16._comment
new file mode 100644
index 0000000000..39c840e84f
--- /dev/null
+++ b/doc/devblog/day_27__locking_fun/comment_3_b67f8ef4ed42b49c8c2e6c4e53163b16._comment
@@ -0,0 +1,18 @@
+[[!comment format=mdwn
+ username="https://www.google.com/accounts/o8/id?id=AItOawm5iosFbL2By7UFeViqkc6v-hoAtqILeDA"
+ nickname="Laszlo"
+ subject="comment 3"
+ date="2013-10-07T09:17:30Z"
+ content="""
+Hi,
+
+Can you please enlighten us mere mortals, when this stale file locking problem occcurs?
+The typical situation. I tried to google up, and read locking files on wikipedia, but 
+still have no exact knowledge when the \"stale\" part happens.
+
+How it gets outdated without noticing?
+
+Best, 
+ Laszlo
+
+"""]]
diff --git a/doc/devblog/day_27__locking_fun/comment_4_0759644baf26b75f4e48dbb387d725a5._comment b/doc/devblog/day_27__locking_fun/comment_4_0759644baf26b75f4e48dbb387d725a5._comment
new file mode 100644
index 0000000000..cbeae8b78f
--- /dev/null
+++ b/doc/devblog/day_27__locking_fun/comment_4_0759644baf26b75f4e48dbb387d725a5._comment
@@ -0,0 +1,10 @@
+[[!comment format=mdwn
+ username="http://joeyh.name/"
+ ip="4.154.4.22"
+ subject="comment 4"
+ date="2013-10-12T23:31:16Z"
+ content="""
+Git simply creates a file as a lock file, and does not use any form of locking on it, so if the git process dies for any reason before it gets a chance to remove the lock file, a stale lock file remains, and future git commands will fall over it.
+
+It's really surprisingly bad..
+"""]]
diff --git a/doc/devblog/day_280__slow_week.mdwn b/doc/devblog/day_280__slow_week.mdwn
new file mode 100644
index 0000000000..ed0dce49d9
--- /dev/null
+++ b/doc/devblog/day_280__slow_week.mdwn
@@ -0,0 +1,16 @@
+Reduced activity this week (didn't work on the assistant after all),
+but several things got done:
+
+Monday: Fixed `fsck --fast --from remote` to not fail when the remote
+didn't support fast copy mode. And dealt with an incompatibility in S3 bucket
+names; the old hS3 library supported upper-case bucket names but the new
+one needs them all in lower case.
+
+Wednesday: Caught up on most recent backlog, made some improvements
+to error handling in `import`, and improved integration with KDE's file
+manager to work with newer versions.
+
+Today: Made `import --deduplicate/--clean-duplicates` actively 
+verify that enough copies of a file exist before deleting it. And,
+thinking about some options for batch mode access to git-annex plumbing,
+to speed up things that use it a lot.
diff --git a/doc/devblog/day_281__catching_up__and_arm_autobuilder_needed.mdwn b/doc/devblog/day_281__catching_up__and_arm_autobuilder_needed.mdwn
new file mode 100644
index 0000000000..d55823f755
--- /dev/null
+++ b/doc/devblog/day_281__catching_up__and_arm_autobuilder_needed.mdwn
@@ -0,0 +1,43 @@
+I've not been blogging, but have been busy this week. Backlog is down to
+113 messages.
+
+Tuesday: I got a weird bug report where `git annex get` was deleting
+a file. This turned out to be a bug in `wget ftp://...` where it would
+delete a symlink that was not where it had been told to download the fie
+to. I put a workaround in git-annex; wget is now run in a temp
+directory. But this was a legitimate wget bug, and it's now been reported
+to the wget developers and will hopefully get fixed there.
+
+Wednesday: Added a --batch mode for several plumbing commands
+(contentlocation, examinekey, and lookupkey). This avoids startup overhead,
+and so lets a lot of queries be done much faster. The implementation
+should make it easy to add --batch to more plumbing commands as needed,
+and could probably extend to non-plumbing commands too.
+
+Today: The first 5 hours involved an incompatible mess of ssh and rsync
+versions on Windows. A Gordian knot of brokenness and dependency hell.
+I finally found a solution which involves downgrading the cygwin rsync
+to an older version, and using msysgit's ssh rather than cygwin's.
+
+Finished up today with more post-Debian-release changes. Landed a patch to
+switch from dataenc to sandi that had been waiting since 2013, and got
+sandi installed on all the git-annex autobuilders. Finished up with some
+prep for a release tomorrow.
+
+----
+
+Finally, Debian has a new enough ghc that it can build template haskell
+on arm! So, whenever a new version of git-annex finally gets into Debian
+(I hope soon), the webapp will be available on arm for those arm laptops.
+Yay!
+
+This also means I have the opportunity to make the standalone arm build
+be done much more simply. Currently it involves qemu and a separate
+companion native mode container that it has to ssh to and build stuff,
+that has to have the same versions of all libraries. It's just enormously
+complicated and touchy. With template haskell building support, all that
+complexity can fall away. 
+
+What I'd really like to do is get a fast-ish arm box with 2gb of ram
+hosted somewhere, and use that to do the builds, in native mode.
+Anyone want to help provide such a box for git-annex arm autobuilds?
diff --git a/doc/devblog/day_282__release_day.mdwn b/doc/devblog/day_282__release_day.mdwn
new file mode 100644
index 0000000000..da648c1cdd
--- /dev/null
+++ b/doc/devblog/day_282__release_day.mdwn
@@ -0,0 +1,16 @@
+Got the release out after more struggling with ssh on windows and a last
+minute fix to the quvi support.
+
+The downloads.kitenet.net git annex repository had accumulated 6 gb of past
+builds that were not publically available. I am publishing those 
+[on the Internet Archive now](https://archive.org/details/git-annex-builds),
+so past builds can be downloaded using git-annex in that repository in the
+usual way. This worked great! :)
+
+I have ordered a CubieTruck with 2 gb of ram to use for the new Arm builder.
+Hosting still TBD.
+
+Looks like git-annex is almost ready to
+[be included in stackage](https://github.com/fpco/stackage/pull/519#issuecomment-98590181),
+which will make building it from source much less likely to fail due to
+broken libraries etc.
diff --git a/doc/devblog/day_283__lazy_sunday.mdwn b/doc/devblog/day_283__lazy_sunday.mdwn
new file mode 100644
index 0000000000..2850e2cbb7
--- /dev/null
+++ b/doc/devblog/day_283__lazy_sunday.mdwn
@@ -0,0 +1,7 @@
+Lazy afternoon spent porting git-anenx to build under ghc 7.10. Required
+rather a lot of changes to build, and even more to build cleanly after the
+AMP transition.
+
+Unfortunately, ghc 7.10 has started warning about every line that uses tab
+for indentation. I had to add additional cruft to turn those warnings off
+everywhere, and cannot say I'm happy about this at all.
diff --git a/doc/devblog/day_283__lazy_sunday/comment_1_fdc6da890e2dc6b86b6a5fe2ebceea4a._comment b/doc/devblog/day_283__lazy_sunday/comment_1_fdc6da890e2dc6b86b6a5fe2ebceea4a._comment
new file mode 100644
index 0000000000..e5d4d4c5ea
--- /dev/null
+++ b/doc/devblog/day_283__lazy_sunday/comment_1_fdc6da890e2dc6b86b6a5fe2ebceea4a._comment
@@ -0,0 +1,9 @@
+[[!comment format=mdwn
+ username="spwhitton"
+ subject="comment 1"
+ date="2015-05-11T09:02:10Z"
+ content="""
+I'd be interested to hear your reasons for preferring tabs over spaces, if it's anything more than an aesthetic preference.
+
+The [GHC trac](https://ghc.haskell.org/trac/ghc/ticket/9230) suggests that it was turned on by default because beginners accidentally mix tabs and spaces and get very confused.
+"""]]
diff --git a/doc/devblog/day_284__development.mdwn b/doc/devblog/day_284__development.mdwn
new file mode 100644
index 0000000000..7ecf684301
--- /dev/null
+++ b/doc/devblog/day_284__development.mdwn
@@ -0,0 +1,33 @@
+Implemented `git annex drop --all`. This also added for free drop with
+`--unused` and `--key`, which overlap with `git annexdropunused` and
+`git annex dropkey`.
+
+The `concurrentprogress` branch had gone too long without being merged, and
+had a lot of merge conflicts. I resolved those, and went ahead and merged
+it into master. However, since the ascii-progress library is not ready yet,
+I made it a build flag, and it will build without it by default. So, `git
+annex get -J5` can be used now, but no progress bars will display yet.
+
+When doing concurrent downloads, either with the new -J or by hand by
+running multiple processes, there was a bug in the diskreserve
+checking code. It didn't consider the disk space that was in the process of
+being used by other concurrent downloads, so would let more downloads
+start up than there was space for.
+
+I was able to fix this pretty easily, thanks to the transfer log files.
+Those were originally added just to let the webapp display transfers, but
+proved very helpful here!
+
+Finally, made .git/annex/transfer/failed/ files stop accumulating when the
+assistant is not being used. Looked into also cleaning up stale
+.git/annex/transfer/{upload,download}/ files (from interrupted transfers).
+But, since those are used as lock files, it's difficult to remove them
+in a concurrency safe way.
+
+Update: Unfortunately, I turned out to have stumbled over an apparent bug
+in haskell's implementation of file locking.
+ Had to work around that.
+
+Happily, the workaround also let me implement cleanup of stale transfer
+info files, left behind when a git-annex process was interrupted. So,
+.git/annex/transfer/ will entirely stop accumulating cruft!
diff --git a/doc/devblog/day_285__tuning_git-annex_unused_refs.mdwn b/doc/devblog/day_285__tuning_git-annex_unused_refs.mdwn
new file mode 100644
index 0000000000..b5afaec175
--- /dev/null
+++ b/doc/devblog/day_285__tuning_git-annex_unused_refs.mdwn
@@ -0,0 +1,20 @@
+Today I added a feature to `git annex unused` that lets the user tune which
+refs they are interested in using. Annexed objects that are used by other
+refs then are considered unused.
+
+Did a fairly complicated refspec format for this, with globs and
+include/exclude of refs. Example:
+
+	+refs/heads/*:+HEAD^:+refs/tags/*:-refs/tags/old-tag
+
+----
+
+I think that, since Google dropped openid support, there seems to have been
+less activity on this website. Although possibly also a higher signal to
+noise ratio. :) I have been working on some ikiwiki changes to make it
+easier for users who don't have an openid to contiribute. So git-annex's
+website should soon let you log in and make posts with just an email address.
+
+People sometimes ask for a git-annex mailing list. I wouldn't mind having
+one, and would certianly subscribe, but don't see any reason that I should
+be involved in running it.
diff --git a/doc/devblog/day_285__tuning_git-annex_unused_refs/comment_1_a3f0dd4fc8457af7c72040fbe8fd5fee._comment b/doc/devblog/day_285__tuning_git-annex_unused_refs/comment_1_a3f0dd4fc8457af7c72040fbe8fd5fee._comment
new file mode 100644
index 0000000000..a94383fa23
--- /dev/null
+++ b/doc/devblog/day_285__tuning_git-annex_unused_refs/comment_1_a3f0dd4fc8457af7c72040fbe8fd5fee._comment
@@ -0,0 +1,13 @@
+[[!comment format=mdwn
+ username="anarcat"
+ subject="mailing lists and support sites"
+ date="2015-05-14T20:26:59Z"
+ content="""
+after years subscribing to zillions of them, i have acquired an allergy to mailing lists. it seems that i get this every once in a while: i'm subscribed to a bunch of them and i stop reading and just hit \"d\" on my mail client. :)
+
+i think there are very interesting alternatives, online web forums (yes yes, i know, bear with me) where knowledge is established and shared instead of just having a threaded conversation. i'm thinking of http://ask.debian.net/ or the stackexchange sites. as i [[analysed before|devblog/day_268_stressed_out/#comment-e0814585df0047e6d4e11515aebe1dec]], the forum is the biggest part of this website (and consequently the git-annex source code!), bug tracker coming in second, and the two together forming the majority of disk usage of the git repository.  there are alternative discussion tools like discourse.org that may also be interesting for people wanting to run such a service.
+
+anyways, the ideas of those is to converge on a set of FAQs and knowledge items more than just have random discussion, and force people to reuse existing topics.
+
+if i had more time, i'd probably setup something like this but really, i can only encourage those initiatives at this point. :)
+"""]]
diff --git a/doc/devblog/day_285__tuning_git-annex_unused_refs/comment_2_dae167e473c60dace129c66beb0af384._comment b/doc/devblog/day_285__tuning_git-annex_unused_refs/comment_2_dae167e473c60dace129c66beb0af384._comment
new file mode 100644
index 0000000000..41e0922c82
--- /dev/null
+++ b/doc/devblog/day_285__tuning_git-annex_unused_refs/comment_2_dae167e473c60dace129c66beb0af384._comment
@@ -0,0 +1,7 @@
+[[!comment format=mdwn
+ username="http://joeyh.name/"
+ subject="comment 2"
+ date="2015-05-16T18:35:22Z"
+ content="""
+Unfortunately stack exchange is nearly unusable for me on dialup. In order for me to contribute to any git-annex communication channels, it needs to be something that can be taken offline, like git or email.
+"""]]
diff --git a/doc/devblog/day_285__tuning_git-annex_unused_refs/comment_3_52179fc326e681d8fbf85ee8963f352c._comment b/doc/devblog/day_285__tuning_git-annex_unused_refs/comment_3_52179fc326e681d8fbf85ee8963f352c._comment
new file mode 100644
index 0000000000..f85d63350e
--- /dev/null
+++ b/doc/devblog/day_285__tuning_git-annex_unused_refs/comment_3_52179fc326e681d8fbf85ee8963f352c._comment
@@ -0,0 +1,9 @@
+[[!comment format=mdwn
+ username="https://id.koumbit.net/anarcat"
+ subject="comment 3"
+ date="2015-05-16T21:46:20Z"
+ content="""
+interesting! i didn't think of those requirements, which do seem quite stringent, but i am certainly sensitive to them, considering i will be spending my own significant amount of time with intermittent access soon.
+
+i have opened a todo item to move the discussion somewhere more relevant, see [[todo/enable_a_discussion_forum_or_support_system]].
+"""]]
diff --git a/doc/devblog/day_286-287__rotten_locks.mdwn b/doc/devblog/day_286-287__rotten_locks.mdwn
new file mode 100644
index 0000000000..7db07da578
--- /dev/null
+++ b/doc/devblog/day_286-287__rotten_locks.mdwn
@@ -0,0 +1,44 @@
+There's something rotten in POSIX fctnl locking. It's not composable,
+or thread-safe.
+
+The most obvious problem with it is that if you have 2 threads, and they
+both try to take an exclusive lock of the same file (each opening it
+separately) ... They'll both succeed. Unlike 2 separate processes,
+where only one can take the lock.
+
+Then the really crazy bit: If a process has a lock file open and fcntl
+locked, and then the same process opens the lock file again, for any
+reason, closing the new FD will release the lock that was set
+using the other FD.
+
+So, that's a massive gotcha if you're writing complex multithreaded code.
+Or generally for composition of code. Of course, C programmers deal with
+this kind of thing all the time, but in the clean world of Haskell, this is
+a glaring problem. We don't expect to need to worry about this kind of
+unrelated side effect that breaks composition and thread safety.
+
+After noticing this problem affected git-anenx in at least one place,
+I have to assume there could be more. And I don't want to need to worry
+about this problem forever. So, I have been working today on a clean fix
+that I can cleanly switch all my lock-related code to use.
+
+One reasonable approach would be to avoid fcntl locking, and use flock.
+But, flock works even less well on NFS than fcntl, and git-annex relies on
+some fcntl locking features. On Linux, there's an "open file description
+locks" feature that fixes POSIX fnctl locking to not have this horrible
+wart, but that's not portable.
+
+Instead, my approach is to keep track of which files the process has
+locked. If it tries to do something with a lockfile that it already has
+locked, it avoids opening the same file again, instead implements its own
+in-process locking behavior. I use STM to do that in a thread-safe manner.
+
+I should probably break out git-annex's lock file handling code as a
+library. Eventually.. This was about as much fun as a root canal, and I'm
+having a real one tomorrow. :-/
+
+----
+
+git-annex is now included in [Stackage](http://www.stackage.org/)!
+
+Daniel Kahn Gillmor is doing some work on reproducible builds of git-annex.
diff --git a/doc/devblog/day_288__microrelease_prep.mdwn b/doc/devblog/day_288__microrelease_prep.mdwn
new file mode 100644
index 0000000000..fc1a5f3dfb
--- /dev/null
+++ b/doc/devblog/day_288__microrelease_prep.mdwn
@@ -0,0 +1,22 @@
+After a less active than usual week (dentist), I made a release last Friday.
+Unfortunately, it turns out that the Linux standalone 
+builds in that release don't include the webapp. So, another release is
+planned tomorrow.
+
+Yesterday and part of today I dug into
+the [[bugs/windows_ssh_webapp_password_entry_broken]]
+reversion. Eventually cracked the problem; it seems that
+different versions of ssh for Windows do different things in a `isatty`
+check, and there's a flag that can be passed when starting ssh to make it
+not see a controlling tty. However, this neeeds changes to the
+`process` library, which db48x and I have now coded up. So a fix for this bug
+is waiting on a new release of that library. Oh well.
+
+Rest of today was catching up on recent traffic, and improving the behavior
+of `git annex fsck` when there's a disk IO error while checksumming a file.
+Now it'll detect a hardware fault exception, and take that to mean the file
+is bad, and move it to the bad files directory, instead of just crashing.
+
+I need better tooling to create disk IO errors on demand.
+Yanking disks out works, but is a blunt instrument. Anyone know of
+good tools for that?
diff --git a/doc/devblog/day_288__microrelease_prep/comment_1_99e359976acbdb9727598f8a87027de0._comment b/doc/devblog/day_288__microrelease_prep/comment_1_99e359976acbdb9727598f8a87027de0._comment
new file mode 100644
index 0000000000..3f849e73e8
--- /dev/null
+++ b/doc/devblog/day_288__microrelease_prep/comment_1_99e359976acbdb9727598f8a87027de0._comment
@@ -0,0 +1,7 @@
+[[!comment format=mdwn
+ username="https://id.koumbit.net/anarcat"
+ subject="bup-cron has something"
+ date="2015-05-27T23:32:18Z"
+ content="""
+so the bup people has a little [bup-damage](https://github.com/bup/bup/blob/master/cmd/damage-cmd.py) command to insert noise into files. it doesn't simulate I/O problems per se, but maybe it could help? --[[anarcat]]
+"""]]
diff --git a/doc/devblog/day_288__microrelease_prep/comment_2_27a74d3926083a00c110aafea28420d7._comment b/doc/devblog/day_288__microrelease_prep/comment_2_27a74d3926083a00c110aafea28420d7._comment
new file mode 100644
index 0000000000..461a86e8a9
--- /dev/null
+++ b/doc/devblog/day_288__microrelease_prep/comment_2_27a74d3926083a00c110aafea28420d7._comment
@@ -0,0 +1,7 @@
+[[!comment format=mdwn
+ username="namelessjon"
+ subject="comment 2"
+ date="2015-05-28T10:14:45Z"
+ content="""
+https://stackoverflow.com/questions/1870696/simulate-a-faulty-block-device-with-read-errors ?
+"""]]
diff --git a/doc/devblog/day_288__microrelease_prep/comment_3_17752e96baaf46f3048b14a9ec288e3b._comment b/doc/devblog/day_288__microrelease_prep/comment_3_17752e96baaf46f3048b14a9ec288e3b._comment
new file mode 100644
index 0000000000..b6e65a9293
--- /dev/null
+++ b/doc/devblog/day_288__microrelease_prep/comment_3_17752e96baaf46f3048b14a9ec288e3b._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="http://schnouki.net/"
+ nickname="Schnouki"
+ subject="comment 3"
+ date="2015-05-29T12:20:08Z"
+ content="""
+A while ago I needed a tool to emulate slow IO (low bandwidth and high latency). I ended writing a small FUSE driver (in Go, just for fun) that did this by adding some delay to each function call. Hacky, but simpler than setting up a device mapper or NBD. I guess you could also emulate IO errors with something like this. If you're interested I can clean up my code and push it to a public repo.
+"""]]
diff --git a/doc/devblog/day_288__microrelease_prep/comment_4_2838c1d9da3202616e331bb31fd43f20._comment b/doc/devblog/day_288__microrelease_prep/comment_4_2838c1d9da3202616e331bb31fd43f20._comment
new file mode 100644
index 0000000000..dbb014c7ef
--- /dev/null
+++ b/doc/devblog/day_288__microrelease_prep/comment_4_2838c1d9da3202616e331bb31fd43f20._comment
@@ -0,0 +1,10 @@
+[[!comment format=mdwn
+ username="joey"
+ subject="""comment 4"""
+ date="2015-05-30T16:03:17Z"
+ content="""
+@Schnouki yes please, that's what I was looking for!
+
+Although perhaps google-fu could find someone else who's already done
+it..
+"""]]
diff --git a/doc/devblog/day_289__new_arm_autobuilder.mdwn b/doc/devblog/day_289__new_arm_autobuilder.mdwn
new file mode 100644
index 0000000000..f0df5995a5
--- /dev/null
+++ b/doc/devblog/day_289__new_arm_autobuilder.mdwn
@@ -0,0 +1,21 @@
+On Friday I installed the CubieTruck that is the new autobuilder
+for arm. This autobuilder is hosted at [WetKnee Books](http://www.wetknee.com/),
+so its physical security includes a swamp. 
+
+The hardware is not fast, but it's faster and far more stable than qemu arm
+emulation. By Saturday I got the build environment all installed nicely,
+including building libraries that use template haskell!
+
+But, ghc crashed with an internal error building git-annex.
+I upgraded to ghc 7.10.1 (which took another day), but it also crashed.
+Was almost giving up, but I looked at the ghc parameters, and -j2 stuck
+out in them. Removed the -j2, and the build works w/o crashing! \o/
+(Filed [a bug report on ghc](https://ghc.haskell.org/trac/ghc/ticket/10469).)
+
+----
+
+Anarcat has been working on improving the man pages, including lots of
+linking to related commands.
+
+The [2015 Haskell Communities and Activities Report](http://www.haskell.org/communities/05-2015/html/report.html)
+is out, and includes an entry for git-annex for the first time!
diff --git a/doc/devblog/day_28__lazy_saturday.mdwn b/doc/devblog/day_28__lazy_saturday.mdwn
new file mode 100644
index 0000000000..c2237b28bd
--- /dev/null
+++ b/doc/devblog/day_28__lazy_saturday.mdwn
@@ -0,0 +1,17 @@
+Finished up the automatic recovery from stale lock files. Turns out git
+has quite a few lock files; the assistant handles them all.
+
+Improved URL and WORM keys so the filenames used for them
+will always work on FAT (which has a crazy assortmeny of illegal
+characters). This is a tricky thing to deal with without breaking backwards
+compatability, so it's only dealt with when creating new URL or WORM keys.
+
+-----
+
+I think my next step in this disaster recovery themed month will be adding
+periodic incremental fsck to the assistant. `git annex fsck` can already
+do an incremental fsck, so this should mostly involve adding a user
+interface to the webapp to configure when it should fsck. For example, you
+might choose to run it for up 1 hour every night, with a goal of checking
+all your files once per month. Also will need to make the assistant do
+something useful when fsck finds a bad file (ie, queue a re-download).
diff --git a/doc/devblog/day_290__.mdwn b/doc/devblog/day_290__.mdwn
new file mode 100644
index 0000000000..7b0111485d
--- /dev/null
+++ b/doc/devblog/day_290__.mdwn
@@ -0,0 +1,23 @@
+Worked thru some backlog. Currently stands at 152 messages.
+
+Merged work from Sebastian Reuße to teach the assistant to listen for
+systemd-networkd dbus events when the network connection changes.
+
+Added `git annex get --incomplete`, which can be used to resume whatever it 
+was you were downloading earlier and interrupted, that you've forgotten
+about. ;)
+
+----
+
+The [Isuma Media Players project](http://isuma-media-players.readthedocs.org/en/latest/index.html)
+is using git-annex to "create a two-way, distributed content distribution
+network for communities with poor connexions to the internet".
+My understanding is this involves places [waaay up North](http://www.isuma.tv/did).
+
+Reading over their [design docs](http://isuma-media-players.readthedocs.org/en/latest/design.html)
+is quite interesting, both to see how they've leveraged things
+like git-annex metadata and preferred content expressions and the assistant,
+and areas where git-annex falls short.
+
+Between DataLad, Isuma, Baobáxia, IA.BAK, and more, there are a lot of
+projects being built on top of git-annex now!
diff --git a/doc/devblog/day_291__public_S3.mdwn b/doc/devblog/day_291__public_S3.mdwn
new file mode 100644
index 0000000000..35e1240428
--- /dev/null
+++ b/doc/devblog/day_291__public_S3.mdwn
@@ -0,0 +1,20 @@
+Now git-annex can be used to set up a public S3 remote. If you've cloned a
+repository that knows about such a remote, you can use the S3 remote
+without needing any S3 credentials. Read-only of course.
+
+This tip shows how to do it: [[tips/public_Amazon_S3_remote]]
+
+One rather neat way to use this is to configure the remote with
+`encryption=shared`. Then, the files stored in S3 will be encrypted, and
+anyone with access to the git repository can get and decrypt the files.
+
+This feature will work for at least AWS S3, and for the Internet Archive's
+S3. It may work for other S3 services, that can be configured to publish
+their files over unauthenticated http. There's a `publicurl` configuration
+setting to allow specifying the url when using a service that git-annex
+doesn't know the url for.
+
+Actually, there was a hack for the IA before, that added the public url to
+an item when it was uploaded to the IA. While that hack is now not
+necessary, I've left it in place for now, to avoid breaking anything that
+depended on it.
diff --git a/doc/devblog/day_291__public_S3/comment_1_c0e96b25b764180975cc26b1316f462d._comment b/doc/devblog/day_291__public_S3/comment_1_c0e96b25b764180975cc26b1316f462d._comment
new file mode 100644
index 0000000000..dc51f11108
--- /dev/null
+++ b/doc/devblog/day_291__public_S3/comment_1_c0e96b25b764180975cc26b1316f462d._comment
@@ -0,0 +1,11 @@
+[[!comment format=mdwn
+ username="richih.mailinglist@c4d55c76731cd68299d3983dc5e6f5385128a68a"
+ nickname="richih.mailinglist"
+ subject="comment 1"
+ date="2015-06-08T23:55:20Z"
+ content="""
+Can you backport that to stable? This would be very useful for video.debconf.net
+
+Thanks,
+Richard
+"""]]
diff --git a/doc/devblog/day_291__public_S3/comment_2_d5223684c09725973b2052425fd30773._comment b/doc/devblog/day_291__public_S3/comment_2_d5223684c09725973b2052425fd30773._comment
new file mode 100644
index 0000000000..7fdb9de425
--- /dev/null
+++ b/doc/devblog/day_291__public_S3/comment_2_d5223684c09725973b2052425fd30773._comment
@@ -0,0 +1,10 @@
+[[!comment format=mdwn
+ username="joey"
+ subject="""comment 2"""
+ date="2015-06-09T19:16:11Z"
+ content="""
+Dude, I'm no DD. 
+
+git-annex is backportable to stable in its current state, if/when
+a new version finally gets uploaded to unstable.
+"""]]
diff --git a/doc/devblog/day_292__dead_keys.mdwn b/doc/devblog/day_292__dead_keys.mdwn
new file mode 100644
index 0000000000..3ad66edb5e
--- /dev/null
+++ b/doc/devblog/day_292__dead_keys.mdwn
@@ -0,0 +1,21 @@
+Ever since `git annex fsck --all` was added, people 
+[have](http://bugs.debian.org/753888)
+[[complained|bugs/fsck_reports_unsolvable_problem]] that
+there's no way to stop it complaining about keys whose content is gone for
+good. Well, there is now: `git annex dead --key` can be used when you know
+that a key is no longer available and want fsck to stop complaining about
+it.
+
+Running fsck on a directory will intentionally still complain about files
+in the directory with missing contents, even if the keys have been marked
+dead.
+
+The crucial part was finding a good way to store the information; luckily
+location log files are parsed in a way that lets it be added there without
+breaking backwards compatability. A bonus is that adding a key's content
+back to the annex will automatically bring it back from the dead.
+
+I'm pondering making `git annex drop --force` automatically mark a key as
+dead when the last copy is dropped, but I don't know if it's too DWIM or
+worth the complication. Another approach would be to let fsck mark keys as
+dead, but that would certianly need an extra flag.
diff --git a/doc/devblog/day_293__last_push_before_summer_vacation.mdwn b/doc/devblog/day_293__last_push_before_summer_vacation.mdwn
new file mode 100644
index 0000000000..5cb6bfb4bb
--- /dev/null
+++ b/doc/devblog/day_293__last_push_before_summer_vacation.mdwn
@@ -0,0 +1,26 @@
+Well, not the literal last `push`, but I've caught up on as much backlog as
+I can (142 messages remain) and spent today developing a few final features
+before tomorrow's release.
+
+Some of the newer things displayed by `git annex info` were not included in
+the --json mode output. The json includes everything now.
+
+`git annex sync --all --content` will make it consider all known annexed
+objects, not only those in the current work tree. By default that syncs all
+versions of all files, but of course preferred content can tune what
+repositories want.
+
+To make that work well with preferred content settings like
+"include=*.mp3", it makes two passes. The first pass is over the work tree,
+so preferred content expressions that match files by name will work. The
+second pass is over all known keys, and preferred content expressions
+that don't care about the filename can match those keys. 
+
+Two passes feels a bit like a hack, but it's a lot better than `--all`
+making nothing be synced when the a preferred content expression matches
+against filenames... I actually had to resort to bloom filters to make the
+two passes work.
+
+This new feature led to some
+[slightly tricky follow-on changes](http://source.git-annex.branchable.com/?p=source.git;a=commit;h=8268f7951e9c70760379bd084425b48df99250d6)
+to the [[preferred_content/standard_groups]] preferred content expressions.
diff --git a/doc/devblog/day_294__back_focusing_on_bugs.mdwn b/doc/devblog/day_294__back_focusing_on_bugs.mdwn
new file mode 100644
index 0000000000..0c7b510118
--- /dev/null
+++ b/doc/devblog/day_294__back_focusing_on_bugs.mdwn
@@ -0,0 +1,7 @@
+Back, and have spent all day focusing on new bug reports. All told, I fixed
+4 bugs, followed up on all other bugs reported while I was away, and fixed
+the android autobuilder.
+
+The message backlog started the day at 250 or something, and is down to 178
+now. Looks like others have been following up to forum posts while I was
+away (thanks!) so those should clear quickly.
diff --git a/doc/devblog/day_295__caught_up.mdwn b/doc/devblog/day_295__caught_up.mdwn
new file mode 100644
index 0000000000..cc52882025
--- /dev/null
+++ b/doc/devblog/day_295__caught_up.mdwn
@@ -0,0 +1,8 @@
+Now caught up on nearly all of my backlog of messages, 
+and indeed am getting to some messages that have been waiting for months.
+Backlog is down to 113! Couple of bugfixes resulted, and many questions answered.
+
+Think I'll spend a couple more days dealing with the older part of the
+backlog. Then, when that reaches diminishing returns, I'll move on to some
+big change. I have been thinking about [[design/caching_database]] on and
+off..
diff --git a/doc/devblog/day_296__into_the_backlog.mdwn b/doc/devblog/day_296__into_the_backlog.mdwn
new file mode 100644
index 0000000000..9293db3405
--- /dev/null
+++ b/doc/devblog/day_296__into_the_backlog.mdwn
@@ -0,0 +1,16 @@
+Mostly spent today getting to older messages in the backlog. This did
+result in a few fixes, but with 97 old messages left, I can feel the
+diminishing returns setting in, to try to understand old bug reports that
+are often unclear or lacking necessary info to reproduce them.
+
+By the way, if you feel your bug report or question has gotten lost in my
+backlog, the best thing to do is post an update to it, and help me
+reproduce it, or clarify it.
+
+Moved on to looking through [[todo]], which was a more productive
+way to find useful things to work on. 
+
+Best change made today is that `git annex unused` can now be configured to
+look at the reflog. So, old versions of files are considered still used
+until the reflog expires. If you've wanted a way to only delete (or move
+away) unused files after they get to a certian age, this is a way to do that ...
diff --git a/doc/devblog/day_297__optparse-applicative.mdwn b/doc/devblog/day_297__optparse-applicative.mdwn
new file mode 100644
index 0000000000..0885f0db41
--- /dev/null
+++ b/doc/devblog/day_297__optparse-applicative.mdwn
@@ -0,0 +1,25 @@
+Now working on converting git-annex to use
+[optparse-applicative](https://github.com/pcapriotti/optparse-applicative)
+for its command line parsing. I've wanted to do this for a long time, 
+because the current code for options is generally horrible, runs in
+IO, and is not at all type safe, while optparse-applicative has wonderful
+composable parsers and lets each subcommand have its own data type
+repesenting all its options.
+
+What pushed me over the edge is that optparse-applicative has automatic
+bash completion!
+
+	# source <(git-annex --bash-completion-script `which git-annex`)
+	# git-annex fsck -
+	--all                   --key                   -S
+	--from                  --more                  -U
+
+Since nobody has managed to write a full bash completion for git-annex
+before, let alone keep it up-to-date with changes to the code, automating
+the problem away is a really nice win. :)
+
+The conversion is a rather huge undertaking; the diff is already over 3000
+lines large after 8 hours of work, and I'm maybe 1/3rd done, with
+the groundwork laid (except for global options still todo) and a few
+subcommands  converted. This won't land for this week's release; it'll
+need a lot of testing before it'll be ready for any release.
diff --git a/doc/devblog/day_299__so_many_commands_and_options.mdwn b/doc/devblog/day_299__so_many_commands_and_options.mdwn
new file mode 100644
index 0000000000..04a2c1e2db
--- /dev/null
+++ b/doc/devblog/day_299__so_many_commands_and_options.mdwn
@@ -0,0 +1,9 @@
+Day 3 of the optparse-applicative conversion.  
+116 files changed, 1607 insertions(+), 1135 deletions(-)  
+At this point, everything is done except for around 20 sub-commands.
+Probably takes 15 minutes work for each. Will finish plowing through
+it in the evenings.
+
+Meanwhile, made the release of version 5.20150710. The Android build for
+this version is not available yet, since I broke the autobuilder last week
+and haven't fixed it yet.
diff --git a/doc/devblog/day_29__scheduling.mdwn b/doc/devblog/day_29__scheduling.mdwn
new file mode 100644
index 0000000000..98c928bf8b
--- /dev/null
+++ b/doc/devblog/day_29__scheduling.mdwn
@@ -0,0 +1,10 @@
+Spent most of the day building some generic types for scheduling recurring
+events. Not sure if rolling my own was a good idea, but that's what I did.
+
+In the incrementalfsck branch, I have hooked this up in `git-annex vicfg`,
+which now accepts and parses scheduled events like 
+"fsck self every day at any time for 60 minutes" and 
+"fsck self on day 1 of weeks divisible by 2 at 3:45 for 120 minutes", and
+stores them in the git-annex branch. The exact syntax is of course subject
+to change, but also doesn't matter a whole lot since the webapp will have
+a better interface.
diff --git a/doc/devblog/day_2__new_laptop.mdwn b/doc/devblog/day_2__new_laptop.mdwn
new file mode 100644
index 0000000000..0008007429
--- /dev/null
+++ b/doc/devblog/day_2__new_laptop.mdwn
@@ -0,0 +1,8 @@
+Now I can build git-annex twice as fast! And a typical incremental build is
+down to 10 seconds, from 51 seconds.
+
+Spent a productive evening working with Guilhem to get his encryption
+patches reviewed and merged. Now there is a way to remove revoked gpg keys,
+and there is a new encryption scheme available that uses public key
+encryption by default rather than git-annex's usual approach. That's not
+for everyone, but it is a good option to have available.
diff --git a/doc/devblog/day_2__new_laptop/comment_1_93447dbd4eb09b4db96770644ea663cb._comment b/doc/devblog/day_2__new_laptop/comment_1_93447dbd4eb09b4db96770644ea663cb._comment
new file mode 100644
index 0000000000..15d19b0c95
--- /dev/null
+++ b/doc/devblog/day_2__new_laptop/comment_1_93447dbd4eb09b4db96770644ea663cb._comment
@@ -0,0 +1,10 @@
+[[!comment format=mdwn
+ username="https://www.google.com/accounts/o8/id?id=AItOawm5iosFbL2By7UFeViqkc6v-hoAtqILeDA"
+ nickname="Laszlo"
+ subject="comment 1"
+ date="2013-09-05T15:07:44Z"
+ content="""
+out of curiosity, what laptop model have you choosen finally?
+
+Laszlo
+"""]]
diff --git a/doc/devblog/day_2__new_laptop/comment_2_e1d022b25f2c16dbe72db07ad4b10a2d._comment b/doc/devblog/day_2__new_laptop/comment_2_e1d022b25f2c16dbe72db07ad4b10a2d._comment
new file mode 100644
index 0000000000..0525975216
--- /dev/null
+++ b/doc/devblog/day_2__new_laptop/comment_2_e1d022b25f2c16dbe72db07ad4b10a2d._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="http://joeyh.name/"
+ ip="24.159.78.54"
+ subject="comment 2"
+ date="2013-09-06T18:02:08Z"
+ content="""
+Lenovo Yoga 11s
+"""]]
diff --git a/doc/devblog/day_300__optparse-applicative_landed.mdwn b/doc/devblog/day_300__optparse-applicative_landed.mdwn
new file mode 100644
index 0000000000..e46dfceeea
--- /dev/null
+++ b/doc/devblog/day_300__optparse-applicative_landed.mdwn
@@ -0,0 +1,3 @@
+Worked through the rest of the changes this weekend and morning, and the
+optparse-applicative branch has landed in master,
+including bash completion support.
diff --git a/doc/devblog/day_301__completion_and_er_completion.mdwn b/doc/devblog/day_301__completion_and_er_completion.mdwn
new file mode 100644
index 0000000000..871b4c49d4
--- /dev/null
+++ b/doc/devblog/day_301__completion_and_er_completion.mdwn
@@ -0,0 +1,9 @@
+Worked on bash tab completion some more. Got "git annex" to also tab complete.
+However, for that to work perfectly when using bash-completion to demand-load
+completion scripts, a small improvement is needed in git's own completion
+script, to have it load git-annex's completion script. I sent a 
+[patch for that to the git developers](http://thread.gmane.org/gmane.comp.version-control.git/274034),
+and hopefully it'll get accepted soon.
+
+Then fixed a relatively long-standing bug that prevented uploads to
+chunked remotes from resuming after the last successfully uploaded chunk.
diff --git a/doc/devblog/day_302-305__gitlab.mdwn b/doc/devblog/day_302-305__gitlab.mdwn
new file mode 100644
index 0000000000..05048ebb69
--- /dev/null
+++ b/doc/devblog/day_302-305__gitlab.mdwn
@@ -0,0 +1,30 @@
+I've been working on adding GitLab support to the webapp for the past 3
+days.
+
+That's not the only thing I've been working on; I've continued to work on
+the older parts of the backlog, which is now shrunk to 91 messages, and
+made some minor improvements and bugfixes.
+
+But, GitLab support in the webapp has certianly taken longer than I'd have
+expected. Only had to write 82 lines of GitLab specific code so far, but it
+went slowly. The user will need to cut and paste repository url and
+ssh public key back and forth between the webapp and GitLab for now. And
+the way GitLab repositories use git-annex makes it a bit tricky to set up;
+in one case the webapp has to do a forced push dry run to check if the
+repository on GitLab can be accessed by ssh.
+
+I found a way to adapt the existing code for setting up a ssh server to
+also support GitLab, so beyond the repo url prompt and ssh key setup,
+everything will be reused. I have something that works now, but there are
+lots of cases to test (encrypted repositories, enabling existing
+repositories, etc), so will need to work on it a bit more before merging
+this feature.
+
+Also took some time to split the [centralized git repository tutorial](http://git-annex.branchable.com/tips/centralized_git_repository_tutorial/)
+into three parts, one for each of GitHub, GitLab, and self-administered servers.
+
+----
+
+The git-annex package in Debian unstable hasn't been updated for 8 months.
+This is about to change; Richard Hartmann has stepped up and is preparing
+an upload of a recent version. Yay!
diff --git a/doc/devblog/day_306__release_day.mdwn b/doc/devblog/day_306__release_day.mdwn
new file mode 100644
index 0000000000..dbc0f5e925
--- /dev/null
+++ b/doc/devblog/day_306__release_day.mdwn
@@ -0,0 +1,8 @@
+Made a release today, with recent work, including the optparse-applicative
+transition and initial gitlab.com support in the webapp.
+
+I had time before the release to work out most of the wrinkles in the
+gitlab.com support, but was not able to get gcrypt encrypted repos to work
+with gitlab, for reasons that remain murky. Their git-annex-shell seems to
+be misbehaving somehow. Will need to get some debugging assistance from the
+gitlab.com developers to figure that out.
diff --git a/doc/devblog/day_307__two_release_week.mdwn b/doc/devblog/day_307__two_release_week.mdwn
new file mode 100644
index 0000000000..0b8a1595ec
--- /dev/null
+++ b/doc/devblog/day_307__two_release_week.mdwn
@@ -0,0 +1,11 @@
+Made a release this morning, mostly because the release earlier this week
+turns out to have accidentally removed several options from `git annex copy`.
+
+Spent some time this afternoon improving how git-annex shuts down when
+--time-limit is used. This used to be a quick and dirty shutdown, similar
+to if git-annex were ctrl-c'd, but I reworked things so it does a clean
+shutdown, including running any buffered git commands. 
+This made incremental fsck with --time-limit resume much better, since
+it saves the incremental fsck database on shutdown. Also tuned when the
+database gets checkpointed during an incremental fsck, to resume better after
+it's interrupted.
diff --git a/doc/devblog/day_308__other_peoples_bugs.mdwn b/doc/devblog/day_308__other_peoples_bugs.mdwn
new file mode 100644
index 0000000000..1b0d81828a
--- /dev/null
+++ b/doc/devblog/day_308__other_peoples_bugs.mdwn
@@ -0,0 +1,11 @@
+Work today has started in the git-annex bug tracker, but the real bugs were
+elsewhere. Got a patch into hinotify to fix its handling of filenames
+received from inotify events when used in a non-unicode locale. Tracked
+down why gitlab's git-annex-shell fails to initialize gcrypt repositories,
+and filed a bug on gitlab-shell.
+
+Yesterday, I got the Android autobuilder fixed. I had started upgrading it
+to new versions of yesod etc, 2 months ago, and something in those new
+versions led to character encoding problems that broke the template haskell
+splicing. Had to throw away the work done for that upgrade, but at least
+it's building again, at last.
diff --git a/doc/devblog/day_309__proxy.mdwn b/doc/devblog/day_309__proxy.mdwn
new file mode 100644
index 0000000000..4afc4c23e5
--- /dev/null
+++ b/doc/devblog/day_309__proxy.mdwn
@@ -0,0 +1,5 @@
+Ended up sending most of today working on `git annex proxy`. It had a lot
+of buggy edge cases, which are all cleaned up now.
+
+Spent another couple hours catching up on recent traffic and fixing a
+couple other misc bugs.
diff --git a/doc/devblog/day_30__cronner.mdwn b/doc/devblog/day_30__cronner.mdwn
new file mode 100644
index 0000000000..f368407caa
--- /dev/null
+++ b/doc/devblog/day_30__cronner.mdwn
@@ -0,0 +1,17 @@
+Lots of progress from yesterday's modest start of building data types for
+scheduling. Last night I wrote the hairy calendar code to calculate when
+next to run a scheduled event. (This is actually quite superior to `cron`,
+which checks every second to see if it should run each event!) Today I
+built a "Cronner" thread that handles spawning threads to handle each
+scheduled event. It even notices when changes have been made to the its
+schedule and stops/starts event threads appropriately.
+
+Everything is hooked up, building, and there's a good chance it works
+without too many bugs, but while I've tested all the pure code (mostly
+automatically with quickcheck properties), I have not run the Cronner
+thread at all. And there is some tricky stuff in there, like noticing
+that the machine was asleep past when it expected to wake up, and deciding
+if it should still run a scheduled event, or should wait until next time.
+So tomorrow we'll see..
+
+Today's work was sponsored by Ethan Aubin.
diff --git a/doc/devblog/day_30__cronner/comment_1_53dfd9310e92f5225db52a13e20a46d4._comment b/doc/devblog/day_30__cronner/comment_1_53dfd9310e92f5225db52a13e20a46d4._comment
new file mode 100644
index 0000000000..dc0c7dfe1c
--- /dev/null
+++ b/doc/devblog/day_30__cronner/comment_1_53dfd9310e92f5225db52a13e20a46d4._comment
@@ -0,0 +1,10 @@
+[[!comment format=mdwn
+ username="http://nullroute.eu.org/~grawity/"
+ nickname="Mantas"
+ subject="comment 1"
+ date="2013-10-09T06:04:15Z"
+ content="""
+The SysV Unix *cron* learned this same thing [in 1979][1].
+
+[1]: http://en.wikipedia.org/wiki/Cron#Multi-user_capability
+"""]]
diff --git a/doc/devblog/day_30__cronner/comment_2_f98357c6f7a6da23873ac27c2e1e9638._comment b/doc/devblog/day_30__cronner/comment_2_f98357c6f7a6da23873ac27c2e1e9638._comment
new file mode 100644
index 0000000000..0fc310a780
--- /dev/null
+++ b/doc/devblog/day_30__cronner/comment_2_f98357c6f7a6da23873ac27c2e1e9638._comment
@@ -0,0 +1,10 @@
+[[!comment format=mdwn
+ username="http://joeyh.name/"
+ ip="4.154.4.22"
+ subject="comment 2"
+ date="2013-10-11T17:07:24Z"
+ content="""
+However, if you strace a traditional cron, you will be sad at how it's implemented. So much statting of files, and checking of entries every second..
+
+0 polling or redundant calulcation in my code!
+"""]]
diff --git a/doc/devblog/day_310__half_day.mdwn b/doc/devblog/day_310__half_day.mdwn
new file mode 100644
index 0000000000..01b76d7bff
--- /dev/null
+++ b/doc/devblog/day_310__half_day.mdwn
@@ -0,0 +1,7 @@
+Took a half day and worked on making it simpler to set up ssh remotes.
+The complexity I've gotten rid of is there's no need to take any action to
+get a ssh remote initialized as a git-annex repository. Where before,
+either git-annex init needed to be ran on the remote, or a git-annex branch
+manually pushed to it, now the remote can simply be added and `git annex sync`
+will do the rest. This needed git-annex-shell changes, so will only work
+once servers are upgraded to use a newer version of git-annex.
diff --git a/doc/devblog/day_310__half_day/comment_1_81db682a271ebc92cce893762c3930a8._comment b/doc/devblog/day_310__half_day/comment_1_81db682a271ebc92cce893762c3930a8._comment
new file mode 100644
index 0000000000..57d9bc97ab
--- /dev/null
+++ b/doc/devblog/day_310__half_day/comment_1_81db682a271ebc92cce893762c3930a8._comment
@@ -0,0 +1,9 @@
+[[!comment format=mdwn
+ username="anarcat"
+ subject="great news"
+ date="2015-08-05T18:52:47Z"
+ content="""
+that's great news. this has been a major usability WTF for me in the past!
+
+thanks!
+"""]]
diff --git a/doc/devblog/day_311__SHA-3.mdwn b/doc/devblog/day_311__SHA-3.mdwn
new file mode 100644
index 0000000000..d4936cf8bd
--- /dev/null
+++ b/doc/devblog/day_311__SHA-3.mdwn
@@ -0,0 +1,6 @@
+The SHA-3 specification was released yesterday; git-annex got support for
+using SHA-3 hashes today. I had to add support for building with the new
+cryptonite library, as cryptohash doesn't (correctly) implement SHA-3 yet.
+Of course, nobody is likely to find a use for this for years, since SHA-2
+is still prefectly fine, but it's nice to get support for new hashes
+in early. :)
diff --git a/doc/devblog/day_312__release_prep.mdwn b/doc/devblog/day_312__release_prep.mdwn
new file mode 100644
index 0000000000..4e0db7de26
--- /dev/null
+++ b/doc/devblog/day_312__release_prep.mdwn
@@ -0,0 +1,12 @@
+Catching up on weekend's traffic, and preparing for a release tomorrow.
+
+Found another place where the optparse-applicative conversion broke some
+command-line parsing; using git-annex metadata to dump metadata recursively
+got broken. This is the second known bug caused by that transition, which
+is not too surpising given how large it was.
+
+Tracked down and fixed a [very tricky encoding problem with metadata values](http://git-annex.branchable.com/bugs/view_fails_with___34__invalid_character__34__/).
+
+The arm autobuilder broke so it won't boot; got a serial console hooked up
+to it and looks like a botched upgrade resulting in a udev/systemd/linux
+version mismatch.
diff --git a/doc/devblog/day_313__optimisation.mdwn b/doc/devblog/day_313__optimisation.mdwn
new file mode 100644
index 0000000000..feedf09e0f
--- /dev/null
+++ b/doc/devblog/day_313__optimisation.mdwn
@@ -0,0 +1,7 @@
+Been doing a little bit of optimisation work. Which meant, first improving
+the --debug output to show fractions of a second, and show when commands
+exit.
+
+That let me measure what takes up time when downloading files from ssh remotes.
+Found one place I could spawn a thread to run a cleanup action, and this simple
+change reduced the non-data-transfer overhead to 1/6th of what it had been!
diff --git a/doc/devblog/day_314__pre_trip_catchup.mdwn b/doc/devblog/day_314__pre_trip_catchup.mdwn
new file mode 100644
index 0000000000..15546aa23c
--- /dev/null
+++ b/doc/devblog/day_314__pre_trip_catchup.mdwn
@@ -0,0 +1,13 @@
+Did some work on Friday and Monday to let external special remotes be used
+in a readonly mode. This lets files that are stored in the remote be
+downloaded by git-annex without the user needing to install the 
+external special remote program. For this to work, the external special
+remote just has to tell git-annex the urls to use. This was developed in
+collaboration with Benjamin Gilbert, who is developing
+[gcsannex](https://github.com/bgilbert/gcsannex), a Google Cloud Storage
+special remote.
+
+Today, got caught up with recent traffic, including fixing a couple of
+bugs. The backlog remains in the low 90's, which is a good place to be as I
+prepare for my August vacation week in the SF Bay Area, followed by a week
+for ICFP and the Haskell Symposium in Vancouver.
diff --git a/doc/devblog/day_315__scrambling.mdwn b/doc/devblog/day_315__scrambling.mdwn
new file mode 100644
index 0000000000..5a6f0441b2
--- /dev/null
+++ b/doc/devblog/day_315__scrambling.mdwn
@@ -0,0 +1,5 @@
+Today was a scramble to get caught up after weeks away. Got the message
+backlog down from over 160 to 123. Fixed two reversions, worked around a
+[strange bug](https://github.com/yesodweb/persistent/issues/474),
+and implemented support for the `gpg.program` configuration, and made
+several smaller improvements.
diff --git a/doc/devblog/day_316-318__chasing_dependencies_and_todos.mdwn b/doc/devblog/day_316-318__chasing_dependencies_and_todos.mdwn
new file mode 100644
index 0000000000..9c656d0799
--- /dev/null
+++ b/doc/devblog/day_316-318__chasing_dependencies_and_todos.mdwn
@@ -0,0 +1,24 @@
+Seems that Git for Windows was released a few weeks ago, replacing msysgit.
+There were a couple problems using git-annex with that package of git,
+which I fixed on Thursday. The next release of git-annex won't work with
+msysgit any longer though; only with Git for Windows.
+
+On Friday, I improved the Windows package further, making it work even when
+git is not added to the system PATH. In such an installation, git-annex
+will now work inside the "git bash" window, and I even got the webapp
+starting from the menu working without git in PATH.
+
+----
+
+In other dependency fun, the daily builds for Linux got broken due to a
+glibc bug in Debian unstable/testing, which makes the bundled curl and ssh
+segfault. With some difficulty I tracked that down, and it turns out the
+bug has been [fixed upstream](https://sourceware.org/bugzilla/show_bug.cgi?id=16381)
+for quite a while. The daily builds are now using the fixed glibc 2.21.
+
+----
+
+Today, got back to making useful improvements, rather than chasing
+dependencies. Improved the bash completion for remotes and backends,
+made annex.hardlink be used more, and made special remotes that are
+configured with autoenable=true get automatically enabled by `git annex init`.
diff --git a/doc/devblog/day_319__release_day.mdwn b/doc/devblog/day_319__release_day.mdwn
new file mode 100644
index 0000000000..5679501659
--- /dev/null
+++ b/doc/devblog/day_319__release_day.mdwn
@@ -0,0 +1,6 @@
+Made the release this morning, first one in 3 weeks. A fair lot of good
+stuff in there.
+
+Just in time for the release, git-annex has support for
+[Ceph](http://ceph.com/). Thanks to Mesar Hameed for building 
+[the external special remote](https://github.com/mhameed/git-annex-remote-ceph)!
diff --git a/doc/devblog/day_31__blah.mdwn b/doc/devblog/day_31__blah.mdwn
new file mode 100644
index 0000000000..672adbfd37
--- /dev/null
+++ b/doc/devblog/day_31__blah.mdwn
@@ -0,0 +1,17 @@
+Some neat stuff is coming up, but today was a pretty blah day for me.
+I did get the Cronner tested and working (only had a few little bugs). But
+I got stuck for quite a while making the Cronner stop `git-annex fsck`
+processes it was running when their jobs get removed. I had some code to do
+this that worked when run standalone, but not when run from git-annex.
+
+After considerable head-scratching, I found out this was due to
+`forkProcess` masking aync exceptions, which seems to be probably
+[a bug](http://ghc.haskell.org/trac/ghc/ticket/8433). Luckily was able to
+work around it. Async exceptions continue to strike me as the worst part of
+the worst part of Haskell (the worst part being exceptions in general).
+
+Was more productive after that.. Got the assistant to automatically queue
+re-downloads of any files that fsck throws out due to having bad contents,
+and made the webapp display an alert while fscking is running, which will
+go to the page to configure fsck schedules. Now all I need to do is
+build the UI of that page.
diff --git a/doc/devblog/day_320__caught_up.mdwn b/doc/devblog/day_320__caught_up.mdwn
new file mode 100644
index 0000000000..8c8c85ae1b
--- /dev/null
+++ b/doc/devblog/day_320__caught_up.mdwn
@@ -0,0 +1,13 @@
+I've mostly been chewing through old and new bug reports and support
+requests that past several days. The backlog is waaay low now -- only 82
+messages! Just in time for me to go on another trip, to Louisville on
+Thursday.
+
+Amazon S3 added an "Infrequent Access" storage class last week, and I got a
+patch into the haskell-aws library to support that, as well as partially
+supporting Google Nearline. That patch was accepted today, and git-annex is
+ready to use the new version of the library as soon as it's released.
+
+At the end of today, I found myself rewriting `git annex status` to parse
+and adjust the output of `git status --short`. This new method makes it
+much more capable than before, including displaying Added files.
diff --git a/doc/devblog/day_320__porting_and_such.mdwn b/doc/devblog/day_320__porting_and_such.mdwn
new file mode 100644
index 0000000000..a62510a1e3
--- /dev/null
+++ b/doc/devblog/day_320__porting_and_such.mdwn
@@ -0,0 +1,13 @@
+Lots of porting work ongoing recently:
+
+* I've been working with [Goeke](http://www.goeke.net/) on building git-annex
+  on Solaris/SmartOS. Who knows, this may lead to a binary distribution
+  in some way, but to start with I got the disk free space code ported
+  to Solaris, and have seen git-annex work there.
+* Jirib has also been working on that same disk free code, porting it to
+  OpenBSD. Hope to land an updated patch for that.
+* Yury kindly updated the Windows autobuilder to a new Haskell Platform
+  release, and I was able to land the `winprocfix` branch that fixes
+  ssh password prompting in the webapp on Windows.
+* The arm autobuilder is fixed and back in its colo, and should be making
+  daily builds again.
diff --git a/doc/devblog/day_321__download_verification.mdwn b/doc/devblog/day_321__download_verification.mdwn
new file mode 100644
index 0000000000..e8c944364a
--- /dev/null
+++ b/doc/devblog/day_321__download_verification.mdwn
@@ -0,0 +1,13 @@
+While at the DerbyCon security conference, I got to thinking about
+verifying objects that git-annex downloads from remotes. This can be
+expensive for big files, so git-annex has never done it at download time,
+instead deferring it to fsck time. But, that is a divergence from git,
+which always verifies checksums of objects it receives. So, it violates
+least surprise for git-annex to not verify checksums too. And this could
+weaken security in some use cases.
+
+So, today I changed that. Now whenever git-annex accepts an object into
+.git/annex/objects, it first verifies its checksum and size. I did add a
+setting to disable that and get back the old behavior: `git config
+annex.verify false`, and there's also a per-remote setting if you want to
+verify content from some remotes but not others.
diff --git a/doc/devblog/day_321__download_verification/comment_1_424367fba4db56f4699952572bd3d605._comment b/doc/devblog/day_321__download_verification/comment_1_424367fba4db56f4699952572bd3d605._comment
new file mode 100644
index 0000000000..447bd39433
--- /dev/null
+++ b/doc/devblog/day_321__download_verification/comment_1_424367fba4db56f4699952572bd3d605._comment
@@ -0,0 +1,7 @@
+[[!comment format=mdwn
+ username="spwhitton"
+ subject="comment 1"
+ date="2015-10-02T04:13:07Z"
+ content="""
+It might be nice to set `annex.verify-threshold` to verify all files below a certain size.  This could default to 500MB.
+"""]]
diff --git a/doc/devblog/day_321__download_verification/comment_2_926cc165e99f2e125c909bd706460a05._comment b/doc/devblog/day_321__download_verification/comment_2_926cc165e99f2e125c909bd706460a05._comment
new file mode 100644
index 0000000000..a75caf5351
--- /dev/null
+++ b/doc/devblog/day_321__download_verification/comment_2_926cc165e99f2e125c909bd706460a05._comment
@@ -0,0 +1,7 @@
+[[!comment format=mdwn
+ username="spwhitton"
+ subject="comment 2"
+ date="2015-10-02T04:15:48Z"
+ content="""
+I mean: *only* files below that size.
+"""]]
diff --git a/doc/devblog/day_321__download_verification/comment_3_fb0ed50c27a493c3494bf2efbf9a9c5d._comment b/doc/devblog/day_321__download_verification/comment_3_fb0ed50c27a493c3494bf2efbf9a9c5d._comment
new file mode 100644
index 0000000000..657b61bace
--- /dev/null
+++ b/doc/devblog/day_321__download_verification/comment_3_fb0ed50c27a493c3494bf2efbf9a9c5d._comment
@@ -0,0 +1,7 @@
+[[!comment format=mdwn
+ username="encryptio"
+ subject="comment 3"
+ date="2015-10-03T21:09:31Z"
+ content="""
+Nice! I actually had been doing `git annex get $file && git annex fsck $file` in all my scripts already because I had run into an issue where a bad file got replicated everywhere, and the only good copy I had was `drop`ped before I realized. This would have caught it. Thanks!
+"""]]
diff --git a/doc/devblog/day_321__download_verification/comment_4_fecb2d6349aa61f1c76525178fea4598._comment b/doc/devblog/day_321__download_verification/comment_4_fecb2d6349aa61f1c76525178fea4598._comment
new file mode 100644
index 0000000000..26d60c868f
--- /dev/null
+++ b/doc/devblog/day_321__download_verification/comment_4_fecb2d6349aa61f1c76525178fea4598._comment
@@ -0,0 +1,7 @@
+[[!comment format=mdwn
+ username="sunny256"
+ subject="Yay, man!"
+ date="2015-10-04T23:28:43Z"
+ content="""
+I love this! Especially since I've noticed some read errors on my laptop disk now and then. Not when using git-annex, I've never lost a file with it, but when doing other things. I've been hoping for this, as I've used computers with memory corruption years ago, and in cases like that, all bets are off and you'll only hope that the file copy goes well. I've grown quite paranoid after that computer, and have enabled everything in git (setting `fetch.fsckObjects`, `receive.fsckObjects` and `transfer.fsckObjects` to `true`) to catch potential errors as a result of bad disk, memory corruption or transfer errors over the network. I'd rather wait a bit longer while copying or especially moving files than wait for that single corrupted bit in the only copy of a 4 gig file. Thanks!
+"""]]
diff --git a/doc/devblog/day_322-326__concurrent_drop_safety.mdwn b/doc/devblog/day_322-326__concurrent_drop_safety.mdwn
new file mode 100644
index 0000000000..5dbf420e4c
--- /dev/null
+++ b/doc/devblog/day_322-326__concurrent_drop_safety.mdwn
@@ -0,0 +1,49 @@
+Well, I've spent all week making `git annex drop --from` safe.
+
+On Tuesday I got a sinking feeling in my stomach, as I realized that
+there was hole in git-annex's armor to prevent concurrent drops from
+violating numcopies or even losing the last copy of a file.
+[[The bug involved an unlikely race condition|bugs/concurrent_drop--from_presence_checking_failures]],
+and for all I know it's never happened in real life, but still this is not
+good.
+
+Since this is a potential data loss bug, expect a release pretty soon 
+with the fix. And, there are 2 things to keep in mind about the fix:
+
+1. If a ssh remote is using an old version of git-annex, a drop may fail.
+   Solution will be to just upgrade the git-annex on the remote to the
+   fixed version.
+2. When a file is present in several special remotes, but not in any
+   accessible git repositories, dropping it from one of the special
+   remotes will now fail, where before it was allowed.
+
+   Instead, the file has to be moved from one of the special remotes to
+   the git repository, and can then safely be dropped from the git repository.
+
+   This is a worrysome behavior change, but unavoidable.
+
+Solving this clearly called for more locking, to prevent concurrency
+problems. But, at first I couldn't find a solution that would allow
+dropping content that was only located on special remotes. I didn't want to
+make special remotes need to involve locking; that would be a nightmare to
+implement, and probably some existing special remotes don't have any
+way to do locking anyway.
+
+Happily, after thinking about it all through Wednesday, I found a solution,
+that while imperfect (see above) is probably the best one feasible. If my
+analysis is correct (and it seems so, although I'd like to write a more
+formal proof than the ad-hoc one I have so far), no locking is needed on
+special remotes, as long as the locking is done just right on the git repos
+and remotes. While this is not able to guarantee that numcopies is always
+preserved, it is able to guarantee that the last copy of a file is never
+removed. And, numcopies *will* always be preserved except for when this
+rare race condition occurs.
+
+So, I've been implementing that all of yesterday and today. Getting it
+right involves building up 4 different kinds of evidence, which can be
+used to make sure that the last copy of a file can't possibly end up being
+dropped, no matter what other concurrent drops could be happening.
+I ended up with a very clean and robust implementation of this, and
+a 2,000 line diff.
+
+Whew!
diff --git a/doc/devblog/day_327__soaking.mdwn b/doc/devblog/day_327__soaking.mdwn
new file mode 100644
index 0000000000..d2f033d5f5
--- /dev/null
+++ b/doc/devblog/day_327__soaking.mdwn
@@ -0,0 +1,16 @@
+Feeling kind of ready to cut the next release of git-annex, but am giving
+the recent large changes just a little time to soak in and make sure
+they're ok.
+
+Yesterday, changed the order that `git annex sync --content` and the
+assistant do drops. When dropping from the local repo and also some
+remotes, it now makes more sense to drop from the remotes first, and only
+then the local repo. There are scenaries where that order lets content be
+dropped from all the places that it should be, while the reverse order
+doesn't.
+
+Today, caught up on recent bug reports, including fixing a bad merge
+commit that was made when git merge failed due to filenames not supported by
+a crippled filesystem, and cleaning up a network transport warning that was
+displayed incorrectly. Also developed a patch to the `aws` library to
+support google nearline when creating buckets.
diff --git a/doc/devblog/day_328__git-annex_is_five.mdwn b/doc/devblog/day_328__git-annex_is_five.mdwn
new file mode 100644
index 0000000000..d1dfc9c456
--- /dev/null
+++ b/doc/devblog/day_328__git-annex_is_five.mdwn
@@ -0,0 +1,15 @@
+The first release of git-annex was 5 years ago.
+
+There have been a total of 187 releases, growing to 50k lines of haskell
+code developed by 28 contributors (and another 10 or so external special
+remote contributors). Approximately 2000 people have posted questions,
+answers, bugs, todos, etc to this website, with 18900 posts in total.
+
+I've been funded for 3 of the 5 years to work on git-annex, with support
+from [[1451 individuals and 6 organizations|thanks]].
+
+Released a new version today with rather more significant changes than usual
+(see recent devblog entries).
+
+The [2015 git-annex user survey](http://git-annex-survey.branchable.com/polls/2015/)
+is now live.
diff --git a/doc/devblog/day_328__git-annex_is_five/comment_1_ff7e8d4c37f4a1aa189af0354db8794a._comment b/doc/devblog/day_328__git-annex_is_five/comment_1_ff7e8d4c37f4a1aa189af0354db8794a._comment
new file mode 100644
index 0000000000..ef8bedc5b8
--- /dev/null
+++ b/doc/devblog/day_328__git-annex_is_five/comment_1_ff7e8d4c37f4a1aa189af0354db8794a._comment
@@ -0,0 +1,11 @@
+[[!comment format=mdwn
+ username="anarcat"
+ subject="happy birthday!"
+ date="2015-10-19T20:58:57Z"
+ content="""
+wow, this is great news! :) happy birthday to git-annex, and congratulations on the parents - which is mostly Joey, i guess, but also all those kickstarter contributors, docs writers, bug reporters and patch senders!
+
+long (hopefully *eternal*) life to git-annex and the data it protects! :)
+
+and thanks again, and again joeyh!
+"""]]
diff --git a/doc/devblog/day_328__git-annex_is_five/comment_2_9e18a2c71884229f06b5ba62a6529a33._comment b/doc/devblog/day_328__git-annex_is_five/comment_2_9e18a2c71884229f06b5ba62a6529a33._comment
new file mode 100644
index 0000000000..404eb4ec7e
--- /dev/null
+++ b/doc/devblog/day_328__git-annex_is_five/comment_2_9e18a2c71884229f06b5ba62a6529a33._comment
@@ -0,0 +1,9 @@
+[[!comment format=mdwn
+ username="CandyAngel"
+ subject="comment 2"
+ date="2015-10-21T09:59:06Z"
+ content="""
+Yay! Really glad I got to be part of (part of) the journey so far :)
+
+Thank you Joey. Hip hip, hooray!
+"""]]
diff --git a/doc/devblog/day_32__fsck_config_UI.mdwn b/doc/devblog/day_32__fsck_config_UI.mdwn
new file mode 100644
index 0000000000..66f4cfd54f
--- /dev/null
+++ b/doc/devblog/day_32__fsck_config_UI.mdwn
@@ -0,0 +1,20 @@
+Last night, built this nice user interface for configuring periodic fscks:
+
+[[!img assistant/fsckconfig.png]]
+
+Rather happy that that whole UI needed only 140 lines of code to build.
+Though rather more work behind it, as seen in this blog..
+
+Today I added some support to git-annex for smart fscking of remotes.
+So far only git repos on local drives, but this should get extended to
+git-annex-shell for ssh remotes. The assistant can also run periodic fscks
+of these.
+
+Still need to test that, and find a way to make a removable drive's fsck
+job run when the drive gets plugged in. That's where picking "any time"
+will be useful; it'll let you configure fscking of removable drives when
+they're available, as long as they have not been fscked too recently.
+
+----
+
+Today's work was sponsored by Georg Bauer.
diff --git a/doc/devblog/day_331__concurrent-output_preparations.mdwn b/doc/devblog/day_331__concurrent-output_preparations.mdwn
new file mode 100644
index 0000000000..028147ad53
--- /dev/null
+++ b/doc/devblog/day_331__concurrent-output_preparations.mdwn
@@ -0,0 +1,11 @@
+Today started with getting a release of git-annex out, to deal with a new
+version of the aws library, which broke the build. That also added support
+to the S3 remotes for creating Google Nearline buckets, although only when
+git-annex is built with the newest version of the aws library.
+
+Rest of the day (and most of the past weekend) I've been working on the
+concurrent-output library. Today I finished making it support multi-line
+regions, and color, and even fully optimised its console updates to use
+minimal bandwidth. So, it's got everything git-annex can possibly need
+to display those troublesome concurrent actions. Will be starting to make
+git-annex use it soon!
diff --git a/doc/devblog/day_332__concurrent_output_at_long_last.mdwn b/doc/devblog/day_332__concurrent_output_at_long_last.mdwn
new file mode 100644
index 0000000000..39003557ec
--- /dev/null
+++ b/doc/devblog/day_332__concurrent_output_at_long_last.mdwn
@@ -0,0 +1,6 @@
+Got git-annex using concurrent-output today. It works beautifully.
+Since the library is new, git-annex has to be explicitly configured to use
+it, so it'll be a while until this is available in regular builds.
+
+There are no progress bars yet in concurrent output mode, but that will change
+soon.. Probably tomorrow.
diff --git a/doc/devblog/day_333__cylons.mdwn b/doc/devblog/day_333__cylons.mdwn
new file mode 100644
index 0000000000..36eb738dc0
--- /dev/null
+++ b/doc/devblog/day_333__cylons.mdwn
@@ -0,0 +1,5 @@
+Spent my time today porting concurrent-output to Windows, fixing a
+tricky problem with error handling/thread joining with git-annex -J,
+and improving the concurrent state handling to support the git command
+queue. Got add/addurl working in concurrent mode.
+No concurrent progress bars yet.. maybe tomorrow?
diff --git a/doc/devblog/day_334__too_easy.mdwn b/doc/devblog/day_334__too_easy.mdwn
new file mode 100644
index 0000000000..1f9f7aabc3
--- /dev/null
+++ b/doc/devblog/day_334__too_easy.mdwn
@@ -0,0 +1,6 @@
+Finally concurrent progress bars are working! After all the groundwork,
+it was really easy to add, under a dozen lines of code.
+
+I've found several bugs while testing commands in -Jn mode, and the rest of
+today was spent fixing them. Two of them affected concurrent `git annex
+add`; the worst narrowly avoided being a data loss bug.
diff --git a/doc/devblog/day_334__too_easy/comment_1_2181970ef1a3380407ce9eef311c9ebc._comment b/doc/devblog/day_334__too_easy/comment_1_2181970ef1a3380407ce9eef311c9ebc._comment
new file mode 100644
index 0000000000..4598cc3eab
--- /dev/null
+++ b/doc/devblog/day_334__too_easy/comment_1_2181970ef1a3380407ce9eef311c9ebc._comment
@@ -0,0 +1,7 @@
+[[!comment format=mdwn
+ username="https://id.koumbit.net/anarcat"
+ subject="could / will this help with global status?"
+ date="2015-11-07T16:43:42Z"
+ content="""
+is there some plans to address [[todo/wishlist:_global_progress_status/]] alongside this? thanks, inspiring and innovative stuff here... --- [[anarcat]]
+"""]]
diff --git a/doc/devblog/day_335__catching_up_with_the_bug_reports.mdwn b/doc/devblog/day_335__catching_up_with_the_bug_reports.mdwn
new file mode 100644
index 0000000000..2db6f14375
--- /dev/null
+++ b/doc/devblog/day_335__catching_up_with_the_bug_reports.mdwn
@@ -0,0 +1,4 @@
+Some work today on improving the standalone linux builds and the
+git-annex-standalone.deb. Also, improved fscks's behavior when asked to
+fsck a dead repo, and fixed some places in the assistant where configured
+ssh-options were not used. Backlog is back down to 95.
diff --git a/doc/devblog/day_336__green.mdwn b/doc/devblog/day_336__green.mdwn
new file mode 100644
index 0000000000..2d1bff774b
--- /dev/null
+++ b/doc/devblog/day_336__green.mdwn
@@ -0,0 +1,15 @@
+Test suite is 100% green! Fixed one remaining bug it found, and solved the
+strange sqlite crash, which turned out to be caused by the test suite deleting
+its temporary repository before sqlite was done with the database inside it.
+
+The only remaining blocker for using v6 unlocked files is a bad interaction
+with shared clones. That should be easy to fix, so release of git-annex
+version 6 is now not far away!
+
+While I've only talked about v6/[[todo/smudge]] stuff here lately, I have been
+fixing various other bugs along the way, and have accumulated a dozen bug
+fixes since the last release. Earlier this week I fixed a bug in `git annex
+unused`. Yesterday I noticed that `git annex migrate` didn't copy over
+metadata. Today, fixed a crash of `git annex view` in a non-unicode locale.
+Etc. So it'll be good not to have the release blocked any longer by v6
+stuff.
diff --git a/doc/devblog/day_336__pid_locks.mdwn b/doc/devblog/day_336__pid_locks.mdwn
new file mode 100644
index 0000000000..03ca076e65
--- /dev/null
+++ b/doc/devblog/day_336__pid_locks.mdwn
@@ -0,0 +1,27 @@
+Been working today on getting git-annex to fall back from nice posix fcntl
+locks to pid locks when the former are not supported. There will be an
+`annex.pidlock` to control this. Mostly useful, I think for networked file
+systems like NFS and Lustre. While these *do* support posix locks, I
+guess it can be hard sometimes to get some big server configured
+appropriately, especially when you don't admin it and just want to use
+git-annex there.
+
+Of course, the fun part about pid locks is that it can be pretty hard to
+tell if one is stale or not. Especialy when using a networked filesystem,
+because then the pid in question can be running on a different computer.
+
+Even if you do figure out that a pid lock is stale, how do you then
+take over a stale pid lock, without racing with anther process that
+also wants to take it over? This was the truely tricky question of the
+day.
+
+I have a possibly slightly novel approach to solve that: 
+Put a more modern lock file someplace else (eg, /dev/shm)
+and use that lock file to lock the pid lock file. Then you can tell if 
+a local pid lock file is stale quickly locally, and take it over safely.
+Of course, if the pid is not locked by a local process, this still
+has to fall back to the inevitable retry-and-timeout-and-fail.
+
+I hope the result will work pretty well, although git-annex will not
+support as fine-grained concurrency when using pid locks. Will find out
+tomorrow when I run today's code! ;)
diff --git a/doc/devblog/day_337__who_needs_POSIX.mdwn b/doc/devblog/day_337__who_needs_POSIX.mdwn
new file mode 100644
index 0000000000..3ab185cc91
--- /dev/null
+++ b/doc/devblog/day_337__who_needs_POSIX.mdwn
@@ -0,0 +1,21 @@
+Got the pid locks working pretty easily, as expected.
+
+But then... Detoured into some truely insane behavior of the Lustre
+filesystem. It seems that Lustre is perfectly happy to let link()
+succeed even when there's a file there that it would overwrite. Rather than
+overwriting the file, Lustre picks an even more crazy way to violate
+POSIX.. It lets there be 2 files in a directory with the **same name**, but
+different contents. Has to be seen to be believed:
+
+	hess$ ls pidlock
+	-r--r--r--  1 hess root    70 Nov 13 15:07 pidlock
+	-r--r--r--  1 hess root    70 Nov 13 15:07 pidlock
+	hess$ rm pidlock; ls pidlock
+	-r--r--r--  1 hess root    74 Nov 13 14:35 pidlock	
+
+git-annex's pid locking code now detects this and seems to work
+even on Lustre. Eep.
+
+I'm clutching my "NO WARRANTY" disclaimer pretty hard though, if anyone
+wants to use git-annex on Lustre. When POSIX is being violated this badly,
+it's hard to anticipate what other strangeness might result.
diff --git a/doc/devblog/day_338__week_in_review.mdwn b/doc/devblog/day_338__week_in_review.mdwn
new file mode 100644
index 0000000000..07b37e7abf
--- /dev/null
+++ b/doc/devblog/day_338__week_in_review.mdwn
@@ -0,0 +1,16 @@
+Monday: Some finishing touches on the pid locking support, and released
+5.20151116. After the release I noticed that concurrent downloads didn't
+always include a progress meter, and made the necessary changes to fix
+that.
+
+Wednesday: This was a day of minor bug fixing and responding to questions
+etc. Message backlog got down below 90, not bad.
+
+Thursday: I've been distracted from coding today with an idea of making
+some new stickers. Hexagonal this time, and even better, composable...
+So they can show git-annex getting as big as you want. ;)
+
+[[stickers/hex.svg]]
+
+The design is done, see [[stickers]], and seems to work well, and even
+better is easy to modify. May find time to get these printed at some point.
diff --git a/doc/devblog/day_339_smudging_out_direct_mode.mdwn b/doc/devblog/day_339_smudging_out_direct_mode.mdwn
new file mode 100644
index 0000000000..a4096c2b96
--- /dev/null
+++ b/doc/devblog/day_339_smudging_out_direct_mode.mdwn
@@ -0,0 +1,61 @@
+I'm considering ways to get rid of direct mode, replacing it with something
+better implemented using [[todo/smudge]] filters.
+
+## git-lfs
+
+I started by trying out git-lfs, to see what I can learn from it. My
+feeling is that git-lfs brings an admirable simplicity to using git with
+large files. For example, it uses a push-hook to automatically
+upload file contents before pushing a branch.
+
+But its simplicity comes at the cost of being centralized. You can't make a
+git-lfs repository locally and clone it onto other drive and have the local
+repositories interoperate to pass file contents around. Everything has to
+go back through a centralized server. I'm willing to pay complexity costs
+for decentralization.
+
+Its simplicity also means that the user doesn't have much control over what
+files are present in their checkout of a repository. git-lfs downloads
+all the files in the work tree. It doesn't have facilities for dropping
+files to free up space, or for configuring a repository to only want to get
+a subset of files in the first place. Some of this could be added to it 
+I suppose.
+
+I also noticed that git-lfs uses twice the disk space, at least when
+initially adding files. It keep a copy of the file in .git/lfs/objects/,
+in addition to the copy in the working tree. That copy seems to be
+necessary due to the way git smudge filters work, to avoid data loss. Of
+course, git-annex manages to avoid that duplication when using symlinks,
+and its direct mode also avoids that duplication (at the cost of some
+robustness). I'd like to keep git-annex's single local copy feature 
+if possible.
+
+## replacing direct mode
+
+Anyway, as smudge/clean filters stand now, they can't be used to set up
+git-annex symlinks; their interface doesn't allow it. But, I was able to
+think up a design that uses smudge/clean filters to cover the same use
+cases that direct mode covers now.
+
+Thanks to the clean filter, adding a file with `git add` would check in a
+small file that points to the git-annex object.
+
+In the same repository, you could also use `git annex add` to check
+in a git-annex symlink, which would protect the object from modification,
+in the good old indirect mode way. `git annex lock` and `git annex unlock` 
+could switch a file between those two modes.
+
+So this allows mixing directly writable annexed files and locked down
+annexed files in the same repository. All regular git commands and all
+git-annex commands can be used on both sorts of files. Workflows could
+develop where a file starts out unlocked, but once it's done, is locked to
+prevent accidental edits and archived away or published.
+
+That's much more flexible than the current direct mode, and I think it will
+be able to be implemented in a simpler, more scalable, and robust way too.
+I can lose the direct mode merge code, and remove hundreds of lines of
+other special cases for direct mode.
+
+The downside, perhaps, is that for a repository to be usable on a crippled
+filesystem, all the files in it will need to be unlocked. A file can't
+easily be unlocked in one checkout and locked in another checkout.
diff --git a/doc/devblog/day_339_smudging_out_direct_mode/comment_1_6435db0003d2d8414b1a040b8899c7b8._comment b/doc/devblog/day_339_smudging_out_direct_mode/comment_1_6435db0003d2d8414b1a040b8899c7b8._comment
new file mode 100644
index 0000000000..75d1c2db43
--- /dev/null
+++ b/doc/devblog/day_339_smudging_out_direct_mode/comment_1_6435db0003d2d8414b1a040b8899c7b8._comment
@@ -0,0 +1,11 @@
+[[!comment format=mdwn
+ username="https://id.koumbit.net/anarcat"
+ subject="niiice"
+ date="2015-11-24T01:07:32Z"
+ content="""
+that would be pretty awesome! thanks so much for looking into what others are doing: it takes great maturity and respect for others, something that is so often missing online...
+
+i hope this can solve a bunch of WTF issues i've had with direct mode, which already improved by leaps and bounds, mind you. :)
+
+speaking of which - would it make sense for git-annex to support lfs remotes out of the box? or is considered builtin to git (ie. if you install git-lfs, you can already have a hybrid lfs/annex repo?)
+"""]]
diff --git a/doc/devblog/day_339_smudging_out_direct_mode/comment_2_79d19b2b057c2a0b23875a32dae0c2ba._comment b/doc/devblog/day_339_smudging_out_direct_mode/comment_2_79d19b2b057c2a0b23875a32dae0c2ba._comment
new file mode 100644
index 0000000000..ee6158fbfa
--- /dev/null
+++ b/doc/devblog/day_339_smudging_out_direct_mode/comment_2_79d19b2b057c2a0b23875a32dae0c2ba._comment
@@ -0,0 +1,13 @@
+[[!comment format=mdwn
+ username="joey"
+ subject="""comment 2"""
+ date="2015-11-24T14:22:37Z"
+ content="""
+Sure, it would be great to have a special remote supporting the git-lfs
+storage backend. This would let git-annex repos be uploaded to github along
+with the annexed files, which is a nice diversity to have in addition to
+gitlab's support for git-annex.
+
+The API is documented, so it's certianly doable, as an external special
+remote even.
+"""]]
diff --git a/doc/devblog/day_339_smudging_out_direct_mode/comment_3_43f7c7c6c2fd227466857c3232e13351._comment b/doc/devblog/day_339_smudging_out_direct_mode/comment_3_43f7c7c6c2fd227466857c3232e13351._comment
new file mode 100644
index 0000000000..226fdb2402
--- /dev/null
+++ b/doc/devblog/day_339_smudging_out_direct_mode/comment_3_43f7c7c6c2fd227466857c3232e13351._comment
@@ -0,0 +1,7 @@
+[[!comment format=mdwn
+ username="spwhitton"
+ subject="comment 3"
+ date="2015-11-26T01:34:17Z"
+ content="""
+At present it's possible to check (small) files directly into a git branch alongside annexed files.  To do this one uses `git add` rather than `git annex add`.  But if `git add` would add files via the smudge/clean process, how would one check files directly into git?  Would it no longer be possible?
+"""]]
diff --git a/doc/devblog/day_339_smudging_out_direct_mode/comment_3_43f7c7c6c2fd227466857c3232e13351/comment_1_78e16167ca29eb3ed43706b39bf08603._comment b/doc/devblog/day_339_smudging_out_direct_mode/comment_3_43f7c7c6c2fd227466857c3232e13351/comment_1_78e16167ca29eb3ed43706b39bf08603._comment
new file mode 100644
index 0000000000..f114b48a2b
--- /dev/null
+++ b/doc/devblog/day_339_smudging_out_direct_mode/comment_3_43f7c7c6c2fd227466857c3232e13351/comment_1_78e16167ca29eb3ed43706b39bf08603._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="joey"
+ subject="""comment 1"""
+ date="2015-11-30T19:03:49Z"
+ content="""
+@spwhitton, you'd have to adjust annex.largefiles to not match the small
+files that you want to add to git.
+"""]]
diff --git a/doc/devblog/day_339_smudging_out_direct_mode/comment_4_35d0472cd64335eedeeb17d51cdbaf5b._comment b/doc/devblog/day_339_smudging_out_direct_mode/comment_4_35d0472cd64335eedeeb17d51cdbaf5b._comment
new file mode 100644
index 0000000000..6690bd93ba
--- /dev/null
+++ b/doc/devblog/day_339_smudging_out_direct_mode/comment_4_35d0472cd64335eedeeb17d51cdbaf5b._comment
@@ -0,0 +1,13 @@
+[[!comment format=mdwn
+ username="https://openid.stackexchange.com/user/27ceb3c5-0762-42b8-8f8a-ed21c284748f"
+ nickname="g"
+ subject="The downside"
+ date="2015-12-10T03:45:09Z"
+ content="""
+If I'm understanding correctly, that one downside (requiring all checkouts to have all files be direct if any filesystems require it) seems to be a fairly major limitation, no? Changing the concept of locked/unlocked files from being a local, per-repo concern to a global one seems like quite a major change.
+
+For instance, would mean that any public repo using git annex for distributing a set of data files would either have to have all files be unlocked, or else no one would be able clone onto a FAT32-formatted external hdd?
+
+
+FWIW, the particular use case I'm concerned about personally is having my annexes on my android device.
+"""]]
diff --git a/doc/devblog/day_339_smudging_out_direct_mode/comment_5_499f3553c4efc35e54f121a7d4abc029._comment b/doc/devblog/day_339_smudging_out_direct_mode/comment_5_499f3553c4efc35e54f121a7d4abc029._comment
new file mode 100644
index 0000000000..b4cbe978e2
--- /dev/null
+++ b/doc/devblog/day_339_smudging_out_direct_mode/comment_5_499f3553c4efc35e54f121a7d4abc029._comment
@@ -0,0 +1,30 @@
+[[!comment format=mdwn
+ username="joey"
+ subject="""comment 5"""
+ date="2015-12-10T15:00:52Z"
+ content="""
+I'm concerned about that too. But it may be possible to finesse it,
+when git-annex is running on a crippled filesystem, it may be able to
+unlock all files as it gets content for them, producing a local fork.
+
+The first difficulty would be avoiding or autoresolving conflicts
+between locked and unlocked when merging changes into that fork. I think
+this is very tractable; such a conflict comes down mostly to the symlink
+bit in the tree object.
+
+The real difficulty would be that any pushes from that fork would include
+its change converting all files to unlocked. Although it's fairly mechanical
+to convert such a commit into one that doesn't unlock files, so perhaps
+that could be automated somehow on push or merge.
+
+There's also a small and probably easy to implement git change that
+would avoid all this complexity: If git's smudge filters were optionally
+able to run on the link-text of symlinks, then a file could be unlocked
+locally without changing what's in the repo and all the smudge stuff
+would still work on it.
+
+Crippled filesystems aside, I think there's value in being able to unlock
+files across clones of a repo. For example, a repo could have a workflow
+where the files for the current episiode/experiment/whatever start out
+unlocked and are locked once it's complete.
+"""]]
diff --git a/doc/devblog/day_339_smudging_out_direct_mode/comment_6_ede48107675edfe40d5fdd3377553aa4._comment b/doc/devblog/day_339_smudging_out_direct_mode/comment_6_ede48107675edfe40d5fdd3377553aa4._comment
new file mode 100644
index 0000000000..d52b054e32
--- /dev/null
+++ b/doc/devblog/day_339_smudging_out_direct_mode/comment_6_ede48107675edfe40d5fdd3377553aa4._comment
@@ -0,0 +1,12 @@
+[[!comment format=mdwn
+ username="https://openid.stackexchange.com/user/27ceb3c5-0762-42b8-8f8a-ed21c284748f"
+ nickname="g"
+ subject="comment 6"
+ date="2015-12-11T22:58:22Z"
+ content="""
+Thanks for the quick reply (and all your work on this!)
+
+Interesting, that change to git does sound like it should be relatively small compared to the workarounds needed. But in any case, glad to hear you're thinking about the issue.
+
+Also curious what your thoughts are on the performance issues you had identified previously with using smudge/clean on larger repos. Do the changes in git 2.5 address all your concerns? Or are there still some cases where this will potentially result in significant slow-down?
+"""]]
diff --git a/doc/devblog/day_339_smudging_out_direct_mode/comment_7_ceb1d2a0e5bbc73590b80ff69292102d._comment b/doc/devblog/day_339_smudging_out_direct_mode/comment_7_ceb1d2a0e5bbc73590b80ff69292102d._comment
new file mode 100644
index 0000000000..7de202eac0
--- /dev/null
+++ b/doc/devblog/day_339_smudging_out_direct_mode/comment_7_ceb1d2a0e5bbc73590b80ff69292102d._comment
@@ -0,0 +1,15 @@
+[[!comment format=mdwn
+ username="joey"
+ subject="""comment 7"""
+ date="2015-12-19T18:21:04Z"
+ content="""
+I'm still not entirely happy with the smudge/clean interface's performance.
+At least git doesn't fall over if the clean filter declines to read all the
+content of the large file on stdin anymore, which was the main thing
+preventing an optimised use of it. Still, git has to run the clean filter once
+per file, which is suboptimal. And, the smudge filter can't modify the file in
+the work tree, but instead has to pass the whole file content back to git
+for writing, which is going to result in a lot of unnecessary context
+switches and slowdown. Especially in git-annex's case, where all it really
+needs to do is make a hard link to the content.
+"""]]
diff --git a/doc/devblog/day_33__fsck_on_connect.mdwn b/doc/devblog/day_33__fsck_on_connect.mdwn
new file mode 100644
index 0000000000..36c2260083
--- /dev/null
+++ b/doc/devblog/day_33__fsck_on_connect.mdwn
@@ -0,0 +1,9 @@
+Built everything needed to run a fsck when a remote gets connected. Have
+not tested it; only testing is blocking merging the incrementalfsck branch
+now.
+
+Also updated the OSX and Android builds to use a new gpg release (denial of
+service security fix), and updated the Debian backport, and did a small
+amount of bug fixing. I need to do several more days of bug fixing once 
+I get this incremental fsck feature wrapped up before moving on to recovery
+of corrupt git repositories.
diff --git a/doc/devblog/day_340__post_turkey_catchup.mdwn b/doc/devblog/day_340__post_turkey_catchup.mdwn
new file mode 100644
index 0000000000..c7e22f66d5
--- /dev/null
+++ b/doc/devblog/day_340__post_turkey_catchup.mdwn
@@ -0,0 +1,9 @@
+Spent a couple of days catching up on backlog, and my backlog is down to 80
+messages now. Lowest in recent memory.
+
+Made the annex.largefiles config be honored by `git annex import`, `git
+annex addurl`, and even `git annex importfeed`.
+
+Planning to dive into smudge filters soon. The design seems ready to go,
+although there is some complication in needing to keep track of mappings
+between worktree files and annex keys.
diff --git a/doc/devblog/day_341__starting_smudge.mdwn b/doc/devblog/day_341__starting_smudge.mdwn
new file mode 100644
index 0000000000..d212ca78af
--- /dev/null
+++ b/doc/devblog/day_341__starting_smudge.mdwn
@@ -0,0 +1,24 @@
+I've gotten git-annex working as a smudge/clean filter today in the
+`smudge` branch. It works ok in a local git repository. `git add` lets
+git-annex decide if it wants to annex a file's content, and checking out
+branches and other git commands involving those files works pretty well.
+
+It can sometimes be slow; git's smudge interface necessarily needs to
+copy the content of files around, particularly when checking out files,
+and so it's never going to be as fast as the good old git-annex symlink
+approach. Most of the slow parts are things that can't be done in direct
+mode repos though, like switching branches, so that isn't a regression.
+
+No git-annex commands to manage the annexed content work yet. That
+will need a key to worktree file mapping to be maintained, and implementing
+that mapping and ensuring its always consistent is probably going to be
+the harder part of this.
+
+Also there's the question of how to handle upgrades from direct mode
+repositories. This will be an upgrade from annex.version 5 to 6, and you
+won't want to do it until all computers that have clones of a repository
+have upgraded to git-annex 6.x, since older versions won't be able to work
+with the upgraded repository. So, the repository upgrade will need to be
+run manually initially, and it seems I'll need to keep supporting direct
+mode for v5 repos in a transition period, which will probably be measured
+in years.
diff --git a/doc/devblog/day_342__continuing_smudge.mdwn b/doc/devblog/day_342__continuing_smudge.mdwn
new file mode 100644
index 0000000000..8d66735ad9
--- /dev/null
+++ b/doc/devblog/day_342__continuing_smudge.mdwn
@@ -0,0 +1,30 @@
+Made a lot of progress today. Implemented the database mapping a key to its
+associated files.  As expected this database, when updated by the
+smudge/clean filters, is not always consistent with the current git work tree.
+In particular, commands like `git mv` don't update the database with the
+new filename. So queries of the database will need to do some additional
+work first to get it updated with any staged changes. But the database is
+good enough for a proof of concept, I hope.
+
+Then I got git-annex commands treating smudged files as annexed files.
+So this works:
+
+	joey@darkstar:~/tmp/new>git annex init
+	init  ok
+	(recording state in git...)
+	joey@darkstar:~/tmp/new>cp ~/some.mp3 .
+	joey@darkstar:~/tmp/new>git add some.mp3
+	joey@darkstar:~/tmp/new>git diff --cached
+	diff --git a/some.mp3 b/some.mp3
+	new file mode 100644
+	index 0000000..2df8868
+	--- /dev/null
+	+++ b/some.mp3
+	@@ -0,0 +1 @@
+	+/annex/objects/SHA256E-s191213--e4b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855.mp3
+	joey@darkstar:~/tmp/new>git annex whereis some.mp3
+	whereis some.mp3 (1 copy) 
+	  	7de17427-329a-46ec-afd0-0a088f0d0b1b -- joey@darkstar:~/tmp/new [here]
+	ok
+
+get/drop don't yet update the smudged files, and that's the next step.
diff --git a/doc/devblog/day_343__get_and_drop_for_smudge.mdwn b/doc/devblog/day_343__get_and_drop_for_smudge.mdwn
new file mode 100644
index 0000000000..50046ac08f
--- /dev/null
+++ b/doc/devblog/day_343__get_and_drop_for_smudge.mdwn
@@ -0,0 +1,26 @@
+Well, another day working on smudge filters, or unlocked files as the
+feature will be known when it's ready. Got both `git annex get` and `git
+annex drop` working for these files today.
+
+Get was the easy part; it just has to hard link or copy the object to the
+work tree file(s) that point to it.
+
+Handling dropping was hard. If the user drops a file, but it's unlocked and
+modified, it shouldn't reset it to the pointer file. For this, I reused the
+InodeCache stuff that was built for direct mode. So the sqlite database
+tracks the InodeCaches of unlocked files, and when a key is dropped it can
+check if the file is modified.
+
+But that's not a complete solution, because when git uses a clean filter,
+it will write the file itself, and git-annex won't have an InodeCache for
+it. To handle this case, git-annex will fall back to verifying the content
+of the file when dropping it if its InodeCache isn't known.
+Bit of a shame to need an expensive checksum to drop an unlocked file;
+maybe the git clean filter interface will eventually be improved to let
+git-annex use it more efficiently.
+
+Anyway, smudged aka unlocked files are working now well enough to be a
+proof of concept. I have several missing safety checks that need to be
+added to get the implementation to be really correct, and quite a lot
+of polishing still to do, including making `unlock`, `lock`, `fsck`,
+and `merge` handle them, and finishing repository upgrade code.
diff --git a/doc/devblog/day_344-345__smudging_along.mdwn b/doc/devblog/day_344-345__smudging_along.mdwn
new file mode 100644
index 0000000000..7b7e6f0421
--- /dev/null
+++ b/doc/devblog/day_344-345__smudging_along.mdwn
@@ -0,0 +1,7 @@
+New special remote alert! Chris Kastorff has made
+[a special remote supporting Backblaze's B2 storage servie](https://github.com/encryptio/git-annex-remote-b2).
+
+And I'm still working on v6 unlocked files. After beating on it for 2 more
+days, all git-annex commands should support them. There is still plenty
+of work to do on testing, upgrading, optimisation, merge conflict resolution,
+and reconciling staged changes.
diff --git a/doc/devblog/day_346-347__nearly_ready_to_merge.mdwn b/doc/devblog/day_346-347__nearly_ready_to_merge.mdwn
new file mode 100644
index 0000000000..944e148c50
--- /dev/null
+++ b/doc/devblog/day_346-347__nearly_ready_to_merge.mdwn
@@ -0,0 +1,19 @@
+Two more days working on v6 and the `smudge` branch is almost ready to be
+merged. The test suite is passing again for v5 repos, and is almost
+passing for v6 repos. Also I decided to make `git annex init` create v5
+repos for now, so `git annex init --version=6` or a `git annex upgrade`
+is needed to get a v6 repo. So while I still have plenty of todo items for
+v6 repos, they are working reasonably well and almost ready for early
+adopters.
+
+The only real blocker to merging it is that the database stuff used by v6
+is not optimised yet and probably slow, and even in v5 repos it will query
+the database. I hope to find an optimisation that avoids all database
+overhead unless unlocked files are used in a v6 repo.
+
+I'll probably make one more release before that is merged though. Yesterday
+I fixed a small security hole in `git annex repair`, which could expose the
+contents of an otherwise not world-writable repository to local users.
+
+BTW, the [2015 git-annex user survey](http://git-annex-survey.branchable.com/polls/2015/)
+closes in two weeks, please go fill it out if you haven't yet done so!
diff --git a/doc/devblog/day_348__v6_assistant.mdwn b/doc/devblog/day_348__v6_assistant.mdwn
new file mode 100644
index 0000000000..08659e8e2d
--- /dev/null
+++ b/doc/devblog/day_348__v6_assistant.mdwn
@@ -0,0 +1,8 @@
+Today was mostly spent making the assistant support v6 repositories.
+That was harder than expected, because I have not touched this part of the
+assistant's code much in a long time, and there are lots of tricky races
+and edge cases to deal with.
+
+The smudge branch has a 4500 diff from master now. Not counting
+documentation changes (Another 500 lines.) The todo list for it is
+shrinking slowly now. May not get it done before the new year.
diff --git a/doc/devblog/day_349__v6_database_optimisation.mdwn b/doc/devblog/day_349__v6_database_optimisation.mdwn
new file mode 100644
index 0000000000..bf18614171
--- /dev/null
+++ b/doc/devblog/day_349__v6_database_optimisation.mdwn
@@ -0,0 +1,7 @@
+Got unexpectedly far today on optimising the database that v6 repositories
+use to keep track of unlocked files. The database schema may still need
+optimization, but everything else to do with the database is optimised.
+Writes to the database are queued together. And reads to the database
+avoid creating the database if it doesn't exist yet. Which means v5
+repos, and v6 repos with no unlocked files will avoid any database
+overhead.
diff --git a/doc/devblog/day_34__wrapping_up_fsck.mdwn b/doc/devblog/day_34__wrapping_up_fsck.mdwn
new file mode 100644
index 0000000000..b667086054
--- /dev/null
+++ b/doc/devblog/day_34__wrapping_up_fsck.mdwn
@@ -0,0 +1,7 @@
+Fixed a lot of bugs in the assistant's fsck handling today, and merged
+it into master. There are some enhancments that could be added to it, 
+including fscking ssh remotes via git-annex-shell and adding the ability to
+schedule events to run every 30 days instead of on a specific day of the
+month. But enough on this feature for now.
+
+Today's work was sponsored by Daniel Brockman.
diff --git a/doc/devblog/day_350-351__it_landed_on_xmas_eve.mdwn b/doc/devblog/day_350-351__it_landed_on_xmas_eve.mdwn
new file mode 100644
index 0000000000..12b9c55758
--- /dev/null
+++ b/doc/devblog/day_350-351__it_landed_on_xmas_eve.mdwn
@@ -0,0 +1,9 @@
+If you've got some free holiday time, the v6 repository mode is now
+available in many of the daily builds, and there's documentation at
+[[tips/unlocked_files]]. It would be very useful now if you can give
+it a try. Use a clone or new repository for safety.
+
+Yesterday I checked all parts of the code that special case direct mode,
+and found a few things that needed adjusting for v6 unlocked files. Today,
+I added the annex.thin config. Around 4 other major todo items need to be
+dealt with before this is ready for more than early adopters.
diff --git a/doc/devblog/day_352__had_to_be_done.mdwn b/doc/devblog/day_352__had_to_be_done.mdwn
new file mode 100644
index 0000000000..181fbd7e79
--- /dev/null
+++ b/doc/devblog/day_352__had_to_be_done.mdwn
@@ -0,0 +1,3 @@
+Automatic merge conflict resolver updated to work with unlocked files in v6
+repos. Fairly tricky and painful; thank goodness the test suite tests a lot
+of edge cases in that code. 
diff --git a/doc/devblog/day_353__one_step_forward_and_N_steps_back.mdwn b/doc/devblog/day_353__one_step_forward_and_N_steps_back.mdwn
new file mode 100644
index 0000000000..0e8bf8f794
--- /dev/null
+++ b/doc/devblog/day_353__one_step_forward_and_N_steps_back.mdwn
@@ -0,0 +1,5 @@
+Got the test suite passing 100%, but then added a pass that uses v6
+unlocked files and 30-some more failures appeared. Fixed a couple of the
+bugs today. After sprinting unexpectedly hard all December on v6, I need a
+change of pace, so I started digging into the website message backlog
+and fixed some bugs and posted some comments there.
diff --git a/doc/devblog/day_354-355__beating_on_the_test_suite.mdwn b/doc/devblog/day_354-355__beating_on_the_test_suite.mdwn
new file mode 100644
index 0000000000..2eacb2acb9
--- /dev/null
+++ b/doc/devblog/day_354-355__beating_on_the_test_suite.mdwn
@@ -0,0 +1,8 @@
+Been working hard on the last several test suite failures for v6 unlocked
+files. Now I've solved almost all of them, which is a big improvement to
+my confidence in its (almost) correctness.
+
+Frustratingly, the test suite is still not green after all this work.
+There's some kind of intermittent failure related to the sqlite database.
+Only seems to happen when the test suite is running, and the error
+message is simply "Error" which is making it hard to track down..
diff --git a/doc/devblog/day_356__benchmarking.mdwn b/doc/devblog/day_356__benchmarking.mdwn
new file mode 100644
index 0000000000..05fbce7a5a
--- /dev/null
+++ b/doc/devblog/day_356__benchmarking.mdwn
@@ -0,0 +1,11 @@
+Added `git annex benchmark` which uses the excellent Criterion to benchmark
+parts of git-annex. What I'm interested in benchmarking right now is the
+sqlite database that is used to manage v6 unlocked files, but having a
+built-in benchmark will probably have other uses later.
+
+The benchmark results were pretty good; queries from the database are
+quite fast (60 microseconds warm cache) and scale well as the size increases.
+I did find one scalability issue, which was fixed by adding another index
+to the database. The kind of schema change that it's easy to make now, but
+that would be a painful transition if it had to be done once this was in wide
+use.
diff --git a/doc/devblog/day_357__post_release_catchup.mdwn b/doc/devblog/day_357__post_release_catchup.mdwn
new file mode 100644
index 0000000000..7d997edb8a
--- /dev/null
+++ b/doc/devblog/day_357__post_release_catchup.mdwn
@@ -0,0 +1,8 @@
+After finally releasing git-annex 6 yesterday, I did some catching up
+today, and got the message backlog back down from 120 to 100.
+
+By the way, the first OSX release of git-annex 6 was broken; I had to fix
+an issue on the builder and update the build. If you upgraded at the wrong
+time, you might find that git-annex doesn't run; if so reinstall it.
+I now have an account on a separate OSX machine from the build machine,
+that automatically tests the daily build, to detect such problems.
diff --git a/doc/devblog/day_357__post_release_catchup/comment_1_6963d1c6be69e531726f62f02d50855c._comment b/doc/devblog/day_357__post_release_catchup/comment_1_6963d1c6be69e531726f62f02d50855c._comment
new file mode 100644
index 0000000000..ed91175824
--- /dev/null
+++ b/doc/devblog/day_357__post_release_catchup/comment_1_6963d1c6be69e531726f62f02d50855c._comment
@@ -0,0 +1,7 @@
+[[!comment format=mdwn
+ username="spwhitton"
+ subject="comment 1"
+ date="2016-01-16T05:14:40Z"
+ content="""
+Congratulations!
+"""]]
diff --git a/doc/devblog/day_358__bugfix_release.mdwn b/doc/devblog/day_358__bugfix_release.mdwn
new file mode 100644
index 0000000000..fb2b066532
--- /dev/null
+++ b/doc/devblog/day_358__bugfix_release.mdwn
@@ -0,0 +1,9 @@
+Bugfix release of git-annex today. The release earlier this month had a bug
+that caused `git annex sync --content` to drop files that should be
+preferred content. So I had to rush out a fix after that [bug was reported](http://git-annex.branchable.com/bugs/wanted___61___present_gets_ignored_in___39__git_annex_sync_--content__39__/).
+(Some of the builds for the new release are still updating as I post this.)
+
+In the past week I've been dealing with a blizzard. Snowed in for 6 days
+and counting. That has slightly back-burnered working on git-annex, and
+I've mostly been making enhancements that the DataLad project needs, along
+the lines of more commands supporting --batch and better --json output. 
diff --git a/doc/devblog/day_358__bugfix_release/comment_1_a9aa122010e576c572ca719638ecf2e6._comment b/doc/devblog/day_358__bugfix_release/comment_1_a9aa122010e576c572ca719638ecf2e6._comment
new file mode 100644
index 0000000000..b55f0b073b
--- /dev/null
+++ b/doc/devblog/day_358__bugfix_release/comment_1_a9aa122010e576c572ca719638ecf2e6._comment
@@ -0,0 +1,7 @@
+[[!comment format=mdwn
+ username="CandyAngel"
+ subject="comment 1"
+ date="2016-02-01T12:43:47Z"
+ content="""
+--batch and --with-files on addurl is amazing! I migrated one of my scripts last night, that used to use 'parallel' and disabling autocommit, and it simplified the workflow a *lot*.
+"""]]
diff --git a/doc/devblog/day_359__annex.largefiles_gitattributes.mdwn b/doc/devblog/day_359__annex.largefiles_gitattributes.mdwn
new file mode 100644
index 0000000000..7cd5013f28
--- /dev/null
+++ b/doc/devblog/day_359__annex.largefiles_gitattributes.mdwn
@@ -0,0 +1,18 @@
+For use cases that mix annexed files with files stored in git, the
+annex.largefiles config is more important in v6 repositories than before,
+since it configures the behavior of `git add` and even `git commit -a`. To
+make it possible to set annex.largefiles so it'll stick across clones of
+a repository, I have now made it be supported in `.gitattributes` files
+as well as git config.
+
+Setting it in .gitattributes looks a little bit different, since
+the regular .gitattributes syntax can be used to match on the filename.
+
+	* annex.largefiles=(largerthan=100kb)
+	*.c annex.largefiles=nothing
+
+It seems there's no way to make a git attribute value contain whitespace.
+So, more complicated annex.largefiles expressions need to use parens to
+break up the words.
+
+	* annex.largefiles=(largerthan=100kb)and(not(include=*.c))
diff --git a/doc/devblog/day_359__annex.largefiles_gitattributes/comment_1_ac73ad4147629c25cd51d13cfea206cf._comment b/doc/devblog/day_359__annex.largefiles_gitattributes/comment_1_ac73ad4147629c25cd51d13cfea206cf._comment
new file mode 100644
index 0000000000..d1732a4270
--- /dev/null
+++ b/doc/devblog/day_359__annex.largefiles_gitattributes/comment_1_ac73ad4147629c25cd51d13cfea206cf._comment
@@ -0,0 +1,9 @@
+[[!comment format=mdwn
+ username="anarcat"
+ subject="all those parenthesis..."
+ date="2016-02-02T22:56:51Z"
+ content="""
+... is that lisp showing through your haskell sir? :)
+
+jokes aside, it's cool that you can actually do that, i didn't know! is the \"non-space expression\" new too?
+"""]]
diff --git a/doc/devblog/day_359__annex.largefiles_gitattributes/comment_2_cfbf4fb5a3a1beb582faa051d1627a31._comment b/doc/devblog/day_359__annex.largefiles_gitattributes/comment_2_cfbf4fb5a3a1beb582faa051d1627a31._comment
new file mode 100644
index 0000000000..b3633c209c
--- /dev/null
+++ b/doc/devblog/day_359__annex.largefiles_gitattributes/comment_2_cfbf4fb5a3a1beb582faa051d1627a31._comment
@@ -0,0 +1,13 @@
+[[!comment format=mdwn
+ username="joey"
+ subject="""comment 2"""
+ date="2016-02-03T21:00:55Z"
+ content="""
+Weirdly, that hack wouldn't work if I had used a more lisp-style format for
+the expressions. At least, I don't think lisp can be written with no
+whitespace at all, in general. It was just luck that the expression format
+happened to work out that way.
+
+Anyway, it's a hack, but it works. Would be nice if git got a way to put
+spaces in gitattribute values..
+"""]]
diff --git a/doc/devblog/day_35__anacron_and_bugfixing.mdwn b/doc/devblog/day_35__anacron_and_bugfixing.mdwn
new file mode 100644
index 0000000000..dc8fbb6284
--- /dev/null
+++ b/doc/devblog/day_35__anacron_and_bugfixing.mdwn
@@ -0,0 +1,15 @@
+While I said I was done with fsck scheduling yesterday, I ended up adding
+one more feature to it today: Full anacron style scheduling. So a fsck can
+be scheduled to run once per week, or month, or year, and it'll run the
+fsck the next time it's available after that much time has passed. The nice
+thing about this is I didn't have to change Cronner *at all* to add this,
+just improved the Recurrence data type and the code that calculates when
+to run events.
+
+Rest of the day I've been catching up on some bug reports. The main bug I
+fixed caused git-annex on Android to hang when adding files. This turns out
+to be because it's using a new (unreleased) version of git, and
+`git check-attr -z` output format has changed in an incompatible way.
+
+I am currently 70 messages behind, which includes some ugly looking bug
+reports, so I will probably continue with this over the next couple days.
diff --git a/doc/devblog/day_360__annex.largefiles_mimetype.mdwn b/doc/devblog/day_360__annex.largefiles_mimetype.mdwn
new file mode 100644
index 0000000000..587dcdac35
--- /dev/null
+++ b/doc/devblog/day_360__annex.largefiles_mimetype.mdwn
@@ -0,0 +1,8 @@
+The same parser was used for both preferred content expressions and
+annex.largefiles. Reworked that today, splitting it into two distinct
+parsers. It doesn't make any sense to use terms like "standard" or
+"lackingcopies" in annex.largefiles, and such are now rejected.
+
+That groundwork also let me add a feature that only makes sense for
+annex.largefiles, and not for preferred content expressions: Matching by
+mime type, such as `mimetype=text/*`
diff --git a/doc/devblog/day_360__annex.largefiles_mimetype/comment_2_78a22b96a82f76ab7217d37918a19024._comment b/doc/devblog/day_360__annex.largefiles_mimetype/comment_2_78a22b96a82f76ab7217d37918a19024._comment
new file mode 100644
index 0000000000..3e6f070a55
--- /dev/null
+++ b/doc/devblog/day_360__annex.largefiles_mimetype/comment_2_78a22b96a82f76ab7217d37918a19024._comment
@@ -0,0 +1,9 @@
+[[!comment format=mdwn
+ username="joey"
+ subject="""comment 2"""
+ date="2016-02-19T15:53:46Z"
+ content="""
+Don't know, unless your system somehow has the magic C library installed
+but no magic database file installed. You should file a bug report with
+details about your system and how you installed the magic C library.
+"""]]
diff --git a/doc/devblog/day_360__annex.largefiles_mimetype/comment_3_3145e5843e20a037351e5a14fa3d6e9a._comment b/doc/devblog/day_360__annex.largefiles_mimetype/comment_3_3145e5843e20a037351e5a14fa3d6e9a._comment
new file mode 100644
index 0000000000..659f02f94f
--- /dev/null
+++ b/doc/devblog/day_360__annex.largefiles_mimetype/comment_3_3145e5843e20a037351e5a14fa3d6e9a._comment
@@ -0,0 +1,17 @@
+[[!comment format=mdwn
+ username="davho"
+ subject="Why would it fail?"
+ date="2016-02-19T15:49:00Z"
+ content="""
+Yeah! I've waited for this option for a half a year :-)  Thumbs up for finally including it!
+
+Doesn't work out of the box, though; I do have the MagicMime build flag but it fails to add files now!
+
+I get this error message for every file I try to add:
+
+    /etc/magic, 0: Warning: using regular magic file `/usr/share/misc/magic'
+    
+    git-annex: user error (magicLoadDefault: could not find any valid magic files!)
+    failed
+
+"""]]
diff --git a/doc/devblog/day_360__annex.largefiles_mimetype/comment_3_bf44404de52b00ff4e4f1effd4cef336._comment b/doc/devblog/day_360__annex.largefiles_mimetype/comment_3_bf44404de52b00ff4e4f1effd4cef336._comment
new file mode 100644
index 0000000000..e7f4f40549
--- /dev/null
+++ b/doc/devblog/day_360__annex.largefiles_mimetype/comment_3_bf44404de52b00ff4e4f1effd4cef336._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="https://openid.stackexchange.com/user/27ceb3c5-0762-42b8-8f8a-ed21c284748f"
+ nickname="g"
+ subject="why not mimetypes for preferred content?"
+ date="2016-03-28T08:32:45Z"
+ content="""
+Why does matching by mimetype only make sense for annex.largefiles, but not for preferred content expressions? Sorry if this is a stupid question, but it seems it would in fact be quite useful to easily express that, for example, a certain repo have a preference for my photos and not my music files (though both should be considered large files)
+"""]]
diff --git a/doc/devblog/day_360__annex.largefiles_mimetype/comment_4_70c337d91f8d0458927c111763041118._comment b/doc/devblog/day_360__annex.largefiles_mimetype/comment_4_70c337d91f8d0458927c111763041118._comment
new file mode 100644
index 0000000000..51bff84886
--- /dev/null
+++ b/doc/devblog/day_360__annex.largefiles_mimetype/comment_4_70c337d91f8d0458927c111763041118._comment
@@ -0,0 +1,13 @@
+[[!comment format=mdwn
+ username="joey"
+ subject="""comment 4"""
+ date="2016-03-28T13:22:35Z"
+ content="""
+For the simple reason that git-annex doesn't know the mime type of a file
+before it downloads a copy.
+
+There are some more complicated reasons involving preferred content
+expressions needing to evaluate the same way in different repos, which
+can be on different computers with different versions of the magic
+database, etc.
+"""]]
diff --git a/doc/devblog/day_360__results_of_2015_user_survey.mdwn b/doc/devblog/day_360__results_of_2015_user_survey.mdwn
new file mode 100644
index 0000000000..c515981b65
--- /dev/null
+++ b/doc/devblog/day_360__results_of_2015_user_survey.mdwn
@@ -0,0 +1,56 @@
+The [2015 git-annex user servey](http://git-annex-survey.branchable.com/polls/2015/)
+is over with, and I'm reading through it and comparing with the
+[2013 survey](http://git-annex-survey.branchable.com/polls/2013/).
+
+37% fewer users responded to the 2015 survey than in 2013.
+It's hard to tell if this has anything to do with the total number of
+git-annex users; Debian's popcon suggests the number of users has doubled
+since 2013, although its graph also suggests the number of users has
+flattened off since 2014. The difference may just be that I 
+promoted the 2013 survey better than the 2015 survey, perhaps reaching
+kickstarter backers who I was in touch with back then.
+
+25% use the assistant. Of those, 20% use XMPP, which is good to know as I'd
+like to get rid of it.
+
+Android use has quardrupled, and Windows use has doubled; both are now at
+4%. It's not surprising that Android and Windows users still think more
+porting work is needed for those OSes. iOS is the only unsupported OS that
+more than 1% of users want. Embedded and NAS systems were mentioned much
+less than in 2013; probably the
+[[arm_tarball_build|forum/new_linux_arm_tarball_build]] met many such
+needs.
+
+About the same percentage of users prefer direct mode in 2015 as did in 2013,
+and ditto for indirect mode. But, more users in 2015 only use direct mode
+on platforms that force its use. Correlating with the OS percentages
+suggests that many of these users are using removable media with the
+FAT filesystem, rather than an OS like Windows or Android. Hopefully
+v6 unlocked files will eventually better meet those user's needs.
+
+The percent of users installing git-annex from source has halved since 2013,
+and it seems that builds from this website have taken up most of that slack;
+I would have expected more installs from Debian, Homebrew etc, but that
+seems not to have increased.
+
+The number of repositories per user has gone up quite a lot since 2013,
+when only 7% of users had more than 10 repos. Now, 23% of users do. And,
+2% of users have more than 100 repos! This probably involves both more
+repositories for different purposes, and cloning of repositories
+to more devices.
+
+Similarly, the amount of data stored has gone up. 34% have more than 1
+terabyte stored, up from 18% in 2013. 2% have more than 16 terabytes.
+
+There's some indications of more users sharing repositories or
+otherwise using it in teams of larger groups, although most users still use
+it by themselves.
+
+Users seem happier with git-annex now than in 2013. 16% call it "one of my
+favorite applications of all time". And, significantly fewer find it too
+hard to use than in 2013.
+
+The main blocking problems are documentation, performance with many
+files (a general git problem), and various issues with the assistant.
+Respondants suggest more focus on making it easier for nontechnical users,
+and for use in larger groups/organizations.
diff --git a/doc/devblog/day_360__results_of_2015_user_survey/comment_1_5f30c583cd20c33e2bcd386f674d6f3b._comment b/doc/devblog/day_360__results_of_2015_user_survey/comment_1_5f30c583cd20c33e2bcd386f674d6f3b._comment
new file mode 100644
index 0000000000..4c53ea41bb
--- /dev/null
+++ b/doc/devblog/day_360__results_of_2015_user_survey/comment_1_5f30c583cd20c33e2bcd386f674d6f3b._comment
@@ -0,0 +1,9 @@
+[[!comment format=mdwn
+ username="anarcat"
+ subject="comment 1"
+ date="2016-02-05T23:06:11Z"
+ content="""
+thanks for the review! great to see those surveys every once in a while...
+
+i wonder if [[todo/build_a_user_guide/]] could help with the documentation. the [[todo]] and [[forum]] pages are useful, but they are more \"search\" based than a handbook would be. there is the [[walkthrough]], but it could cover a *lot* more ground and i am not sure the wiki format is appropriate for this...
+"""]]
diff --git a/doc/devblog/day_360__results_of_2015_user_survey/comment_2_68479af6dda2f732436b19160297aacd._comment b/doc/devblog/day_360__results_of_2015_user_survey/comment_2_68479af6dda2f732436b19160297aacd._comment
new file mode 100644
index 0000000000..501e7b455d
--- /dev/null
+++ b/doc/devblog/day_360__results_of_2015_user_survey/comment_2_68479af6dda2f732436b19160297aacd._comment
@@ -0,0 +1,7 @@
+[[!comment format=mdwn
+ username="CandyAngel"
+ subject="comment 2"
+ date="2016-02-08T13:08:30Z"
+ content="""
+I already have a cookbook of sorts (written in POD), but it is notes based on my usage. I'd be happy to expand this if I can get some feedback from newer/Windows users, as my usage is unusual and focused (e.g. no direct mode, limited assistant).
+"""]]
diff --git a/doc/devblog/day_360__results_of_2015_user_survey/comment_3_e38568c6baa30f58c5a7735b2f7985ed._comment b/doc/devblog/day_360__results_of_2015_user_survey/comment_3_e38568c6baa30f58c5a7735b2f7985ed._comment
new file mode 100644
index 0000000000..e43930cc61
--- /dev/null
+++ b/doc/devblog/day_360__results_of_2015_user_survey/comment_3_e38568c6baa30f58c5a7735b2f7985ed._comment
@@ -0,0 +1,7 @@
+[[!comment format=mdwn
+ username="anarcat"
+ subject="publish!"
+ date="2016-02-08T19:12:30Z"
+ content="""
+well, publish those notes somewhere! it's always good to share!
+"""]]
diff --git a/doc/devblog/day_360__results_of_2015_user_survey/comment_4_e0bfb67454df205f2c9c566933e7f121._comment b/doc/devblog/day_360__results_of_2015_user_survey/comment_4_e0bfb67454df205f2c9c566933e7f121._comment
new file mode 100644
index 0000000000..ee565ac2d9
--- /dev/null
+++ b/doc/devblog/day_360__results_of_2015_user_survey/comment_4_e0bfb67454df205f2c9c566933e7f121._comment
@@ -0,0 +1,9 @@
+[[!comment format=mdwn
+ username="CandyAngel"
+ subject="comment 4"
+ date="2016-02-10T12:49:18Z"
+ content="""
+Okay, I'll need to go through it first and remove all the snide comments to myself though :P
+
+I'll throw up a HTML version up in a temporary place, if Joey approves, I'll ask if there is a namespace on CPAN that I can use as a more permanent home (or Joey can do it if he'd prefer control over it, I don't mind).
+"""]]
diff --git a/doc/devblog/day_361__adjusted_branches_design.mdwn b/doc/devblog/day_361__adjusted_branches_design.mdwn
new file mode 100644
index 0000000000..a51bf7885d
--- /dev/null
+++ b/doc/devblog/day_361__adjusted_branches_design.mdwn
@@ -0,0 +1,5 @@
+Working on a design for [[design/adjusted_branches]]. I've been kicking
+this idea around for a while to replace direct mode on crippled filesystems
+with v6 unlocked files. And the same thing would allow for hiding not
+present files. It's somewhat complicated, but the design I have seems like
+it would work.
diff --git a/doc/devblog/day_362__encoding_fun.mdwn b/doc/devblog/day_362__encoding_fun.mdwn
new file mode 100644
index 0000000000..392c5745a0
--- /dev/null
+++ b/doc/devblog/day_362__encoding_fun.mdwn
@@ -0,0 +1,19 @@
+This was one of those days where I somehow end up dealing with tricky
+filename encoding problems all day.
+
+First, worked around inability for concurrent-output to display unicode
+characters when in a non-unicode locale. The normal trick that git-annex
+uses doesn't work in this case. Since it only affected -J, I decided to
+make git-annex detect the problem and make -J behave as if it was not built
+with the concurrent-output feature. So, it just doesn't display concurrent
+output, which is better than crashing with an encoding error.
+
+The other problem affects v6 repos only. Seems that not all Strings will
+round trip through a persistent sqlite database. In particular, unicode
+surrogate characters are replaced with garbage. This is really [a bug in
+persistent](https://github.com/yesodweb/persistent/issues/540).
+But, for git-annex's purposes, it was possible to work around it,
+by detecting such Strings and serializing them differently.
+
+Then I had to enhance `git annex fsck` to fix up repositories that were
+affected by that problem.
diff --git a/doc/devblog/day_363__snow_day.mdwn b/doc/devblog/day_363__snow_day.mdwn
new file mode 100644
index 0000000000..4744816dac
--- /dev/null
+++ b/doc/devblog/day_363__snow_day.mdwn
@@ -0,0 +1,11 @@
+Made a `no-cbits` branch that removes several things that use C code and
+the FFI. I moved one of them out to a new haskell library,
+. Others were replaced with
+other existing libraries. This will simplify git-annex's build process, and
+more library use is good. Planning to merge this branch in a week or two.
+
+v6 unlocked files don't work on Windows. I had assumed that since the build
+was succeeding, the test suite was passing there. But, it turns out the
+test suite was failing and somehow not failing the build. Have now fixed
+several problems with v6 on Windows. Still a couple test suite problems to
+address.
diff --git a/doc/devblog/day_364__more_v6_unlocked.mdwn b/doc/devblog/day_364__more_v6_unlocked.mdwn
new file mode 100644
index 0000000000..38b053136a
--- /dev/null
+++ b/doc/devblog/day_364__more_v6_unlocked.mdwn
@@ -0,0 +1,8 @@
+In a v6 repository on a filesystem not supporting symlinks,
+it makes sense for commands like `git annex add` and `git annex import`
+to add the files unlocked, since locked files are not usable there.
+After implementing that, I also added an `annex.addunlocked` config setting,
+so that the same behavior can be configured in other repositories.
+
+Rest of the day was spent fixing up the test suite's v6 repository tests
+to work on FAT and Windows.
diff --git a/doc/devblog/day_365__some_kind_of_milestone.mdwn b/doc/devblog/day_365__some_kind_of_milestone.mdwn
new file mode 100644
index 0000000000..fdbd4f610d
--- /dev/null
+++ b/doc/devblog/day_365__some_kind_of_milestone.mdwn
@@ -0,0 +1,6 @@
+Should mention that there was a release two days ago. The main reason for
+the timing of that release is because the Linux wstandalone builds include
+glibc, which recently had a nasty security hole and had to be updated.
+
+Today, fixed a memory leak, and worked on getting caught up with backlog,
+which now stands at 112 messages.
diff --git a/doc/devblog/day_366__starting_adjusted_branches.mdwn b/doc/devblog/day_366__starting_adjusted_branches.mdwn
new file mode 100644
index 0000000000..3f50d76bf9
--- /dev/null
+++ b/doc/devblog/day_366__starting_adjusted_branches.mdwn
@@ -0,0 +1,33 @@
+Getting started on [[design/adjusted_branches]], taking a top-down and
+bottom-up approach. Yesterday I worked on improving the design. Today,
+built a `git mktree` interface that supports recursive tree generation and
+filtering, which is the low-level core of what's needed to implement the
+adjusted branches.
+
+To test that, wrote a fun program that generates a git tree with all
+the filenames reversed.
+
+[[!format haskell """
+import Git.Tree
+import Git.CurrentRepo
+import Git.FilePath
+import Git.Types
+import System.FilePath
+
+main = do
+        r <- Git.CurrentRepo.get
+        (Tree t, cleanup) <- getTree (Ref "HEAD") r
+        print =<< recordTree r (Tree (map reverseTree t))
+        cleanup
+
+reverseTree :: TreeContent -> TreeContent
+reverseTree (TreeBlob f m s) = TreeBlob (reverseFile f) m s
+reverseTree (RecordedSubTree f s l) = NewSubTree (reverseFile f) (map reverseTree l)
+
+reverseFile :: TopFilePath -> TopFilePath
+reverseFile = asTopFilePath . joinPath . map reverse . splitPath . getTopFilePath
+"""]]
+
+Also, fixed problems with the Android, Windows, and OSX builds today.
+Made a point release of the OSX dmg, because the last several releases
+of it will SIGILL on some hardware.
diff --git a/doc/devblog/day_367__adjusted_branches_proof_of_concept.mdwn b/doc/devblog/day_367__adjusted_branches_proof_of_concept.mdwn
new file mode 100644
index 0000000000..80576005a9
--- /dev/null
+++ b/doc/devblog/day_367__adjusted_branches_proof_of_concept.mdwn
@@ -0,0 +1,18 @@
+Now I have a proof of concept [[design/adjusted_branches]]
+implementation, that creates a branch where all locked files
+are adjusted to be unlocked. It works!
+
+Building the adjusted branch is pretty fast; around 2 thousand files
+per second. And, I have a trick in my back pocket that could double that
+speed. It's important this be quite fast, because it'll be done often.
+
+Checking out the adjusted branch can be bit slow though, since git runs
+`git annex smudge` once per unlocked file. So that might need to be
+optimised somehow. On the other hand, this should be done only rarely.
+
+I like that it generates reproducible git commits so the same adjustments
+of the same branch will always have the same sha, no matter when and where
+it's done. Implementing that involved parsing git commit objects.
+
+Next step will be merging pulled changes into the adjusted branch, while
+maintaining the desired adjustments.
diff --git a/doc/devblog/day_368__leap.mdwn b/doc/devblog/day_368__leap.mdwn
new file mode 100644
index 0000000000..69766b5e32
--- /dev/null
+++ b/doc/devblog/day_368__leap.mdwn
@@ -0,0 +1,9 @@
+Pushed out a release today, could not resist the leap day in the version
+number, and also there were enough bug fixes accumulated to make it worth
+doing.
+
+I now have `git-annex sync` working inside adjusted branches, so pulls
+get adjusted appropriately before being merged into the adjusted branch.
+Seems to mostly work well, I did just find one bug in it though. Only
+propigating adjusted commits remains to be done to finish my adjusted
+branches prototype.
diff --git a/doc/devblog/day_369-370__paddling_furiously.mdwn b/doc/devblog/day_369-370__paddling_furiously.mdwn
new file mode 100644
index 0000000000..5321079002
--- /dev/null
+++ b/doc/devblog/day_369-370__paddling_furiously.mdwn
@@ -0,0 +1,72 @@
+Tuesday was spent dealing with lock files. Turned out there were some bugs
+in the `annex.pidlock` configuration that prevented it from working, and
+could even lead to data loss.
+
+And then more lock files today, since I needed to lock git's index file the
+same way git does. This involved finding out how to emulate `O_EXCL` under
+Windows. Urgh.
+
+Finally got back to working on [[design/adjusted_branches]] today. And, I've just
+gotten syncing of commits from adjusted branches back to the orginal branch
+working! Time for short demo of what I've been building for the past couple
+weeks:
+
+	joey@darkstar:~/tmp/demo>ls -l
+	total 4
+	lrwxrwxrwx 1 joey joey 190 Mar  3 17:09 bigfile -> .git/annex/objects/zx/X8/SHA256E-s1048576--44ee9fdd91d4bc567355f8b2becd5fe137b9e3aafdfe804341ce2bcc73b8013f/SHA256E-s1048576--44ee9fdd91d4bc567355f8b2becd5fe137b9e3aafdfe804341ce2bcc73b8013f
+	joey@darkstar:~/tmp/demo>git annex adjust
+	Switched to branch 'adjusted/master(unlocked)'
+	ok
+	joey@darkstar:~/tmp/demo#master(unlocked)>ls -l
+	total 4
+	-rw-r--r-- 1 joey joey 1048576 Mar  3 17:09 bigfile
+
+Entering the adjusted branch unlocked all the files.
+
+	joey@darkstar:~/tmp/demo#master(unlocked)>git mv bigfile newname
+	joey@darkstar:~/tmp/demo#master(unlocked)>git commit -m rename
+	[adjusted/master(unlocked) 29e1bc8] rename
+	 1 file changed, 0 insertions(+), 0 deletions(-)
+	  rename bigfile => newname (100%)
+	joey@darkstar:~/tmp/demo#master(unlocked)>git log --pretty=oneline
+	29e1bc835080298bbeeaa4a9faf42858c050cad5 rename
+	a195537dc5beeee73fc026246bd102bae9770389 git-annex adjusted branch
+	5dc1d94d40af4bf4a88b52805e2a3ae855122958 add
+	joey@darkstar:~/tmp/demo#master(unlocked)>git log --pretty=oneline master
+	5dc1d94d40af4bf4a88b52805e2a3ae855122958 add
+
+The commit was made on top of the commit that generated the adjusted branch.
+It's not yet reached the master branch.
+
+	joey@darkstar:~/tmp/demo#master(unlocked)>git annex sync
+	commit  ok
+	joey@darkstar:~/tmp/demo#master(unlocked)>git log --pretty=oneline
+	b60c5d6dfe55107431b80382596f14f4dcd259c9 git-annex adjusted branch
+	9c36848f078a2bb7a304010e962a2b7318c0877c rename
+	5dc1d94d40af4bf4a88b52805e2a3ae855122958 add
+	joey@darkstar:~/tmp/demo#master(unlocked)>git log --pretty=oneline master
+	9c36848f078a2bb7a304010e962a2b7318c0877c rename
+	5dc1d94d40af4bf4a88b52805e2a3ae855122958 add
+
+Now the commit has reached master. Notice how the history of the adjusted
+branch was rebased on top of the updated master branch as well.
+
+	joey@darkstar:~/tmp/demo#master(unlocked)>ls -l
+	total 1024
+	-rw-r--r-- 1 joey joey 1048576 Mar  3 17:09 newname
+	joey@darkstar:~/tmp/demo#master(unlocked)>git checkout master
+	Switched to branch 'master'
+	joey@darkstar:~/tmp/demo>ls -l
+	total 4
+	lrwxrwxrwx 1 joey joey 190 Mar  3 17:12 newname -> .git/annex/objects/zx/X8/SHA256E-s1048576--44ee9fdd91d4bc567355f8b2becd5fe137b9e3aafdfe804341ce2bcc73b8013f/SHA256E-s1048576--44ee9fdd91d4bc567355f8b2becd5fe137b9e3aafdfe804341ce2bcc73b8013f
+
+Just as we'd want, the file is locked in master, and unlocked in 
+the adjusted branch.
+
+(Not shown: git annex sync will also merge in and adjust changes from remotes.)
+
+So, that all looks great! But, it's cheating a bit, because it locks
+all files when updating the master branch. I need to make it remember,
+somehow, when files were originally unlocked, and keep them unlocked. Also
+want to implement other adjustments, like hiding files whose content is not
+present.
diff --git a/doc/devblog/day_36__bugfixing.mdwn b/doc/devblog/day_36__bugfixing.mdwn
new file mode 100644
index 0000000000..0e31d54bf4
--- /dev/null
+++ b/doc/devblog/day_36__bugfixing.mdwn
@@ -0,0 +1 @@
+Productive day, but I'm wiped out. Backlog down to 51.
diff --git a/doc/devblog/day_371__catching_up.mdwn b/doc/devblog/day_371__catching_up.mdwn
new file mode 100644
index 0000000000..02cdf8e7b5
--- /dev/null
+++ b/doc/devblog/day_371__catching_up.mdwn
@@ -0,0 +1,21 @@
+Over the weekend, I converted the linux "ancient" autobuilder to use stack.
+This makes it easier to get all the recent versions of all the haskell
+dependencies installed there.
+
+Also, merged my no-ffi branch, removing some library code from git-annex
+and adding new dependencies. It's good to remove code.
+
+Today, fixed the OSX dmg file -- its bundled gpg was broken. I pushed out a
+new version of the OSX dmg file with the fix.
+
+With the recent incident in mind of malware inserted into the Transmission
+dmg, I've added a virus scan step to the release process
+for all the git-annex images. This way, we'll notice if an autobuilder
+gets a virus.
+
+Also caught up on some backlog, although the remaining backlog is a little
+larger than I'd like at 135 messages.
+
+Hope to work some more on adjusted branches this week. A few mornings ago,
+I had what may be a key insight about how to reverse adjustments when
+propigating changes back from the adjusted branch.
diff --git a/doc/devblog/day_372__adjusted_branches_improved.mdwn b/doc/devblog/day_372__adjusted_branches_improved.mdwn
new file mode 100644
index 0000000000..625169eafd
--- /dev/null
+++ b/doc/devblog/day_372__adjusted_branches_improved.mdwn
@@ -0,0 +1,18 @@
+After a real brain-bender of a day, I have commit propagation from the
+adjusted branch back to the original branch working, without needing to
+reverse adjust the whole tree. This is faster, but the really nice thing
+is that it makes individual adjustments simpler to write.
+
+In fact, it's so simple that I took 10 minutes just now to implement a second
+adjustment!
+
+[[!format haskell """
+adjustTreeItem HideMissingAdjustment h ti@(TreeItem _ _ s) = do
+         mk <- catKey s
+         case mk of
+                 Just k -> ifM (inAnnex k)
+                         ( return (Just ti)
+      	                 , return Nothing
+                         )
+                 Nothing -> return (Just ti)
+"""]]
diff --git a/doc/devblog/day_373__away.mdwn b/doc/devblog/day_373__away.mdwn
new file mode 100644
index 0000000000..e8761b4e2c
--- /dev/null
+++ b/doc/devblog/day_373__away.mdwn
@@ -0,0 +1,6 @@
+Caught up with a few last things today, before I leave for 
+[a week in Boston](http://joeyh.name/trips/2016/boston/).
+
+Converted several places that ran git hash-object repeatedly to feed data
+to a running process. This sped up git-annex add in direct mode and with
+v6 unlocked files, by up to 2x.
diff --git a/doc/devblog/day_374__security_fix.mdwn b/doc/devblog/day_374__security_fix.mdwn
new file mode 100644
index 0000000000..c1d20bbdf1
--- /dev/null
+++ b/doc/devblog/day_374__security_fix.mdwn
@@ -0,0 +1,4 @@
+Pushed out a git-annex release this morning mostly because of the recent
+[[bugs/git_security_fix]]. Several git-annex builds bundle a copy of git and
+needed to be updated. Note that the OSX autobuilder is temporarily down and
+so it's not been updated yet -- hopefully soon.
diff --git a/doc/devblog/day_375__back.mdwn b/doc/devblog/day_375__back.mdwn
new file mode 100644
index 0000000000..c53cebaba3
--- /dev/null
+++ b/doc/devblog/day_375__back.mdwn
@@ -0,0 +1,10 @@
+Back from Libreplanet and a week of spring break. Backlog is not too bad
+for two weeks mostly away; 143 messages.
+
+Finally got the OSX app updated for the git security fix yesterday. Had to
+drop builds for old OSX releases.
+
+Getting back into working on adjusted branches now. Polishing up the UI and
+docs today. Nearly ready to merge the feature; the only blocker is there
+seems to be something a little bit wrong with how pulled changes are merged
+into the adjusted branch that I noticed in testing.
diff --git a/doc/devblog/day_376__in_the_weeds.mdwn b/doc/devblog/day_376__in_the_weeds.mdwn
new file mode 100644
index 0000000000..4454d7672c
--- /dev/null
+++ b/doc/devblog/day_376__in_the_weeds.mdwn
@@ -0,0 +1,18 @@
+Spent all day fixing sync in adjusted branches. I was lost in the weeds for
+a long time. Eventually, drawing this diagram helped me find my way
+to a solution:
+
+	origin/master    adjusted/master     master
+	A                                    A
+	|--------------->A'                  |
+	|                |                   |
+	|                C'- - - - - - - - > C
+	B                                    |
+	|                                    |
+	|--------------->M'<-----------------|
+
+After implementing that, syncing in adjusted branches seems to work much
+better now. And I've finally merged support for them into master.
+
+There's still several bugs and race conditions and upgrade things to sort
+out around adjusted branches. Proably another week's work all told.
diff --git a/doc/devblog/day_377__will_adjusted_branches_ever_end.mdwn b/doc/devblog/day_377__will_adjusted_branches_ever_end.mdwn
new file mode 100644
index 0000000000..9e6499b4e1
--- /dev/null
+++ b/doc/devblog/day_377__will_adjusted_branches_ever_end.mdwn
@@ -0,0 +1,21 @@
+Feels like I've been working on adjusted branches too long.
+
+Did make some excellent progress today. Upgrading a direct mode repo to v6
+will now enter an adjusted branch where all files are unlocked. Using an
+adjusted branch like this avoids unlocking all files in the master branch
+of the repo, which means that different clones of a repo can be
+upgraded to v6 mode at different times. This should let me advance the
+timetable for enabling v6 by default, and getting rid of direct mode.
+
+Also, cloning a repository that has an adjusted branch checked out will
+now work; the clone starts out in the same adjusted branch.
+
+But, I realized today that the way merges from origin/master into
+adjusted/master are done will often lead to merge conflicts. I have came up
+with a better way to handle these merges that won't unncessarily conflict,
+but didn't feel ready to implement that today.
+
+----
+
+Instead, I spent the latter half of the day getting caught up on some
+of the backlog. Got it down from some 200 messages to 150.
diff --git a/doc/devblog/day_378__finishing_adjusted_branches_merge.mdwn b/doc/devblog/day_378__finishing_adjusted_branches_merge.mdwn
new file mode 100644
index 0000000000..aa0cd1f8d8
--- /dev/null
+++ b/doc/devblog/day_378__finishing_adjusted_branches_merge.mdwn
@@ -0,0 +1,23 @@
+Well, I had to rethink how merges into adjusted branches should be handled.
+The old method often led to unnecessary merge conflicts. My new approach
+should always avoid unncessary merge conflicts, but it's quite a trick.
+
+To merge origin/master into adjusted/master, it first merges origin/master
+into master. But, since adjusted/master is checked out, it has to do the
+merge in a temporary work tree. Luckily this can be done fairly
+inexpensively. To handle merge conflicts at this stage, git-annex's
+automatic merge conflict resolver is used. This approach wouldn't be
+feasible without a way to automatically resolve merge conflicts, because
+the user can't help with conflict resolution when the merge is not
+happening in their working tree.
+
+Once that out-of-tree merge is done, the result is adjusted, and merged
+into the adjusted branch. Since we know the adjusted branch is a child of
+the old master branch, this merge can be forced to always be a
+fast-forward. This second merge will only ever have conflicts if the work
+tree has something uncommitted in it that causes a merge conflict.
+
+Wow! That's super tricky, but it seems to work well. While I ended up
+throwing away everything I did [[last Thursday|day_376__in_the_weeds]]
+due to this new approach, the code is in some ways simpler than that
+old, busted approach.
diff --git a/doc/devblog/day_379__bugs_race_conditions_and_taxes.mdwn b/doc/devblog/day_379__bugs_race_conditions_and_taxes.mdwn
new file mode 100644
index 0000000000..3d323d2d49
--- /dev/null
+++ b/doc/devblog/day_379__bugs_race_conditions_and_taxes.mdwn
@@ -0,0 +1,12 @@
+Think I'm really finished with adjusted branches now. Fixed a bug in
+annex symlink calculation when merging into an adjusted branch. And, fixed
+a race condition involving a push of master from another repository.
+
+While `git annex adjust --unlock` is reason enough to have adjusted
+branches, I do want to at some point look into implementing `git annex
+adjust --hide-missing`, and perhaps rewrite the view branches to use
+adjusted branches, which would allow for updating view branches when
+pulling from a remote.
+
+Also, turns out Windows supports hard links, so I got annex.thin working
+on Windows, as well as a few other things that work better with hard links.
diff --git a/doc/devblog/day_37__long_day.mdwn b/doc/devblog/day_37__long_day.mdwn
new file mode 100644
index 0000000000..7882a746d8
--- /dev/null
+++ b/doc/devblog/day_37__long_day.mdwn
@@ -0,0 +1,6 @@
+A long day of bugfixing. Split into two major parts. First I got back to a
+bug I filed in August to do with the assistant misbehaving when run in a
+subdirectory of a git repository, and did a nice type-driven fix of the
+underlying problem (that also found and fixed some other related bugs that
+would not normally occur). Then, spent 4 hours in Windows purgatory working
+around crazy path separator issues.
diff --git a/doc/devblog/day_380__post_release.mdwn b/doc/devblog/day_380__post_release.mdwn
new file mode 100644
index 0000000000..d36aa27d65
--- /dev/null
+++ b/doc/devblog/day_380__post_release.mdwn
@@ -0,0 +1,8 @@
+Yesterday I released version 6.20160412, which is the first to support
+adjusted branches.
+
+Today, some planning for ways to better support `annex.thin`, but that
+seems to be stuck on needing a way to update git's index file. Which is the
+main thing needed to fix various problems with v6 unlocked files.
+
+Dove back into the backlog, got it down to 144 messages. Several bug fixes.
diff --git a/doc/devblog/day_381__executable_unlocked_files.mdwn b/doc/devblog/day_381__executable_unlocked_files.mdwn
new file mode 100644
index 0000000000..cac006205b
--- /dev/null
+++ b/doc/devblog/day_381__executable_unlocked_files.mdwn
@@ -0,0 +1,2 @@
+Seems I forgot about executable files entirely when implementing v6
+unlocked files. Fixed that oversight today.
diff --git a/doc/devblog/day_382-384__pretty_well_caught_up.mdwn b/doc/devblog/day_382-384__pretty_well_caught_up.mdwn
new file mode 100644
index 0000000000..86089f568a
--- /dev/null
+++ b/doc/devblog/day_382-384__pretty_well_caught_up.mdwn
@@ -0,0 +1,9 @@
+The past three days have felt kind of low activity days, but somehow a lot
+of stuff still got done, both bug fixes and small features, and I am
+feeling pretty well caught up with backlog for the first time in over a
+month. Although as always there is some left, 110 messages.
+
+On Monday I fixed a bug that could cause a hang when dropping content, if
+git-annex had to verify the content was present on a ssh remote. That bug
+was bad enough to make an immediate release for, even though it was only a
+week since the last release.
diff --git a/doc/devblog/day_385__new_features.mdwn b/doc/devblog/day_385__new_features.mdwn
new file mode 100644
index 0000000000..a2ff3ed195
--- /dev/null
+++ b/doc/devblog/day_385__new_features.mdwn
@@ -0,0 +1,16 @@
+Something that has come up repeatedly is that `git annex reinject` is
+too hard to use since you have to tell it which annexed file you're providing
+the content for. Now `git-annex reinject --known` can be passed a list of
+files and it will reinject any that hash to known annexed contents
+and ignore the rest. That works best when only one backend is used in a 
+repository; otherwise it would need to be run repeatedly with different
+`--backend` values.
+
+Turns out that the `GIT_COMMON_DIR` feature used by adjusted branches
+is only a couple years old, so don't let adjusted branches be used with
+a too old git.
+
+And, `git merge` is getting a new sanity check that prevents merging
+in a branch with a disconnected history. `git annex sync` will inherit that
+sanity check, but the assistant needs to let such merges happen when eg,
+pairing repositories, so more git version checking there.
diff --git a/doc/devblog/day_385__new_features/comment_1_f76ff85897f24692e96f88236b7f4b7d._comment b/doc/devblog/day_385__new_features/comment_1_f76ff85897f24692e96f88236b7f4b7d._comment
new file mode 100644
index 0000000000..91d4b2ddab
--- /dev/null
+++ b/doc/devblog/day_385__new_features/comment_1_f76ff85897f24692e96f88236b7f4b7d._comment
@@ -0,0 +1,11 @@
+[[!comment format=mdwn
+ username="CandyAngel"
+ subject="comment 1"
+ date="2016-04-24T10:56:01Z"
+ content="""
+    git-annex reinject --known can be passed a list of files and it will reinject any that hash to known annexed contents and ignore the rest.
+
+I **so** can't wait for this to get released, it will save so much time for me!
+
+Thank you Joey ^_^
+"""]]
diff --git a/doc/devblog/day_386__day_off.mdwn b/doc/devblog/day_386__day_off.mdwn
new file mode 100644
index 0000000000..ae73307bb4
--- /dev/null
+++ b/doc/devblog/day_386__day_off.mdwn
@@ -0,0 +1,12 @@
+I'm on a long weekend. This did not prevent git-annex from getting an
+impressive lot of features though, as Daniel Dent contributed
+ which uses
+[rclone](http://rclone.org/) to add support for a ton of additional cloud
+storage things, including: 
+
+Google Drive, Openstack Swift, Rackspace cloud files, Memset Memstore, Dropbox,
+Google Cloud Storage, Amazon Cloud Drive, Microsoft One Drive, Hubic, Backblaze
+B2, Yandex Disk
+
+Wow! I hope that rclone will end up packaged in more distributions (eg Debian)
+so this will be easier to set up.
diff --git a/doc/devblog/day_387__release_day.mdwn b/doc/devblog/day_387__release_day.mdwn
new file mode 100644
index 0000000000..c7a294aed2
--- /dev/null
+++ b/doc/devblog/day_387__release_day.mdwn
@@ -0,0 +1,10 @@
+git-annex 6.20160419 has a rare security fix.
+A [bug](http://git-annex.branchable.com/bugs/External_special_remote_broken__63__/) made encrypted special
+remotes that are configured to use chunks accidentally expose the checksums
+of content that is uploaded to the remote. Such information is supposed to
+be hidden from the remote's view by the encryption. The same bug also made
+resuming interrupted uploads to such remotes start over from the beginning.
+
+After releasing that, I've been occupied today with fixing the Android
+autobuilder, which somehow got its build environment broken (unsure how),
+and fixing some other dependency issues.
diff --git a/doc/devblog/day_387__release_day/comment_1_3e48e930616917f6db1fc351d0f7e6df._comment b/doc/devblog/day_387__release_day/comment_1_3e48e930616917f6db1fc351d0f7e6df._comment
new file mode 100644
index 0000000000..9c20baad79
--- /dev/null
+++ b/doc/devblog/day_387__release_day/comment_1_3e48e930616917f6db1fc351d0f7e6df._comment
@@ -0,0 +1,12 @@
+[[!comment format=mdwn
+ username="http://svario.it/gioele"
+ nickname="gioele"
+ subject="comment 1"
+ date="2016-04-29T10:26:22Z"
+ content="""
+> A bug made encrypted special remotes that are configured to use chunks accidentally expose the checksums of content that is uploaded to the remote. Such information is supposed to be hidden from the remote's view by the encryption.
+
+What should the users do to repair this situation? How can the exposed checksums be removed from the remote? Is a `git annex sync` enough?
+
+Thank you for the timely release!
+"""]]
diff --git a/doc/devblog/day_387__release_day/comment_2_86262439c9056c0e06623cdf79eaf9a6._comment b/doc/devblog/day_387__release_day/comment_2_86262439c9056c0e06623cdf79eaf9a6._comment
new file mode 100644
index 0000000000..19492995ab
--- /dev/null
+++ b/doc/devblog/day_387__release_day/comment_2_86262439c9056c0e06623cdf79eaf9a6._comment
@@ -0,0 +1,15 @@
+[[!comment format=mdwn
+ username="joey"
+ subject="""comment 2"""
+ date="2016-05-03T17:56:44Z"
+ content="""
+The exposed information is not stored in the remote. If it's stored
+anywhere, it would be in a server log, which might log an attempt
+to access an un-encrypted key filename (which typically includes the
+checksum and maybe the file's extension).
+
+So on the one hand, you don't need to do anything other than upgrade
+git-annex to recover from the problem. On the other hand, if the potential
+that a un-encrypted filename of a git-annex key having leaked into a server
+log somewhere is a problem, I don't have a solution to the problem. :-/
+"""]]
diff --git a/doc/devblog/day_388-389__various_and_windows.mdwn b/doc/devblog/day_388-389__various_and_windows.mdwn
new file mode 100644
index 0000000000..b929882b8f
--- /dev/null
+++ b/doc/devblog/day_388-389__various_and_windows.mdwn
@@ -0,0 +1,13 @@
+Lots of little fixes and improvements here and there over the past couple
+days.
+
+The main thing was fixing several bugs with adjusted branches and Windows.
+They seem to work now, and commits made on the adjusted branch are
+propigated back to master correctly.
+
+It would be good to finish up the last todos for v6 mode this month.
+The sticking point is I need a way to update the file stat in the git index
+when git-annex gets/drops/etc an unlocked file. I have not decided yet if
+it makes the most sense to add a dependency on libgit2 for that, or extend
+`git update-index`, or even write a pure haskell library to manipulate
+index files. Each has its pluses and its minuses.
diff --git a/doc/devblog/day_38__starting_git_repo_repair.mdwn b/doc/devblog/day_38__starting_git_repo_repair.mdwn
new file mode 100644
index 0000000000..3808abe389
--- /dev/null
+++ b/doc/devblog/day_38__starting_git_repo_repair.mdwn
@@ -0,0 +1,11 @@
+Goal for the rest of the month is to build automatic recovery git
+repository corruption. Spent today investigating how to do it and came up
+with a fairly [[detailed_design|design/assistant/disaster_recovery]]. It
+will have two parts, first to handle repository problems that can be fixed
+by fetching objects from remotes, and secondly to recover from problems
+where data never got sent to a remote, and has been lost.
+
+In either case, the assistant should be able to detect the problem and
+automatically recover well enough to keep running. Since this also affects
+non-git-annex repositories, it will also be available in a standalone
+`git-recover-repository` command.
diff --git a/doc/devblog/day_38__starting_git_repo_repair/comment_1_321468d9686db5dde072500bdaeb7d29._comment b/doc/devblog/day_38__starting_git_repo_repair/comment_1_321468d9686db5dde072500bdaeb7d29._comment
new file mode 100644
index 0000000000..3a1ce7851c
--- /dev/null
+++ b/doc/devblog/day_38__starting_git_repo_repair/comment_1_321468d9686db5dde072500bdaeb7d29._comment
@@ -0,0 +1,10 @@
+[[!comment format=mdwn
+ username="http://cstork.org/"
+ nickname="Chris Stork"
+ subject="Idea: checksum left-over files in .git/annex/tmp for potential recovery"
+ date="2013-10-19T12:30:58Z"
+ content="""
+This just came to mind when I thought about the second case.  I noticed quite often that files were left in .git/annex/tmp (seen on OS X) and sometimes these are the only instances of files on this particular computer.
+
+(It's on my todo list to file several bug reports about this and other issues on OS X...)
+"""]]
diff --git a/doc/devblog/day_390__sharedpubkey.mdwn b/doc/devblog/day_390__sharedpubkey.mdwn
new file mode 100644
index 0000000000..805b5b6c9a
--- /dev/null
+++ b/doc/devblog/day_390__sharedpubkey.mdwn
@@ -0,0 +1,5 @@
+It's not every day I add a new special remote encryption mode to
+git-annex! The new encryption=sharedpubkey mode
+lets anyone with a clone of the git repository (and access to the remote)
+store files in the remote, but then only the private key owner can access
+those files. Which opens up some interesting new use cases...
diff --git a/doc/devblog/day_391__git_smudge_clean_interface_proposal.mdwn b/doc/devblog/day_391__git_smudge_clean_interface_proposal.mdwn
new file mode 100644
index 0000000000..0a9d676aa7
--- /dev/null
+++ b/doc/devblog/day_391__git_smudge_clean_interface_proposal.mdwn
@@ -0,0 +1,3 @@
+Posted a [proposal for extending git smudge/clean filters with raw file access](http://thread.gmane.org/gmane.comp.version-control.git/294425).
+If git gets an interface like that, it will make it easy to deal with most of
+the remaining [[v6_todo_list|todo/smudge]].
diff --git a/doc/devblog/day_392__v6_fixes.mdwn b/doc/devblog/day_392__v6_fixes.mdwn
new file mode 100644
index 0000000000..10d2de263e
--- /dev/null
+++ b/doc/devblog/day_392__v6_fixes.mdwn
@@ -0,0 +1,12 @@
+Fixed several problems with v6 mode today. The assistant was doing some
+pretty wrong things when changes were synced into v6 repos, and that
+behavior is fixed. Also dealt with a race that caused updates made to the
+keys database by one process to not be seen by another process.
+And, made `git annex add` of a unlocked pointer file not annex the pointer
+file's content, but just add it to git as-is.
+
+Also, Thowz pointed out that adjusted branches could be used to locally adjust
+where annex symlinks point to, when a repository's git directory is not in
+the usual location. I've added that, as `git annex adjust --fix`. It
+was quite easy to implement this, which makes me very happy with the
+adjusted branches code!
diff --git a/doc/devblog/day_392__v6_fixes/comment_1_6da0787fa9749ee2d1f91842f10a5d3c._comment b/doc/devblog/day_392__v6_fixes/comment_1_6da0787fa9749ee2d1f91842f10a5d3c._comment
new file mode 100644
index 0000000000..a123937b53
--- /dev/null
+++ b/doc/devblog/day_392__v6_fixes/comment_1_6da0787fa9749ee2d1f91842f10a5d3c._comment
@@ -0,0 +1,7 @@
+[[!comment format=mdwn
+ username="xloem"
+ subject="dangling objects"
+ date="2016-05-18T01:49:11Z"
+ content="""
+My repos have accumulated thousands of dangling objects recently (every one).  I'm wondering, would this issue be resolved with the syncing fix?  If so, should the objects just be deleted?
+"""]]
diff --git a/doc/devblog/day_392__v6_fixes/comment_2_15434848be1951e93a1e5b9fcbccedb0._comment b/doc/devblog/day_392__v6_fixes/comment_2_15434848be1951e93a1e5b9fcbccedb0._comment
new file mode 100644
index 0000000000..129f88e98f
--- /dev/null
+++ b/doc/devblog/day_392__v6_fixes/comment_2_15434848be1951e93a1e5b9fcbccedb0._comment
@@ -0,0 +1,12 @@
+[[!comment format=mdwn
+ username="joey"
+ subject="""comment 2"""
+ date="2016-05-21T13:55:01Z"
+ content="""
+Dangling objects are not related to the v6 syncing problems mentioned in
+this post.
+
+You can generally safely delete dangling objects from a git repository;
+git gc does it automatically after 30 days or so. git-annex does not
+normally create dangling objects but several git operations can.
+"""]]
diff --git a/doc/devblog/day_392__v6_fixes/comment_3_74e696f147f80162a941a5a5435327ce._comment b/doc/devblog/day_392__v6_fixes/comment_3_74e696f147f80162a941a5a5435327ce._comment
new file mode 100644
index 0000000000..e331dd8792
--- /dev/null
+++ b/doc/devblog/day_392__v6_fixes/comment_3_74e696f147f80162a941a5a5435327ce._comment
@@ -0,0 +1,7 @@
+[[!comment format=mdwn
+ username="xloem"
+ subject="comment 3"
+ date="2016-05-22T15:58:22Z"
+ content="""
+Thanks.
+"""]]
diff --git a/doc/devblog/day_393__fun_and_more_fun.mdwn b/doc/devblog/day_393__fun_and_more_fun.mdwn
new file mode 100644
index 0000000000..9d1fc0c326
--- /dev/null
+++ b/doc/devblog/day_393__fun_and_more_fun.mdwn
@@ -0,0 +1,15 @@
+Over the weekend, I noticed that a relative path to `GIT_INDEX_FILE` is
+interpreted in several different, inconsistent ways by git. git-annex
+mostly used absolute paths, but did use a relative path in `git annex
+view`. Now it will only use absolute paths to avoid git's wacky behavior.
+
+Integrated some patches to support building with ghc 8.0.1, which was
+recently released.
+
+The gnupg-options git configs were not always passed to gpg. Fixing this
+involved quite a lot of plumbing to get the options to the right functions,
+and consumed half of today.
+
+Also did some design work on [[design/external_special_remote_protocol]]
+to avoid backwards compatability problems when adding new protocol
+features.
diff --git a/doc/devblog/day_394__implicit_vs_explicit.mdwn b/doc/devblog/day_394__implicit_vs_explicit.mdwn
new file mode 100644
index 0000000000..300ac4c570
--- /dev/null
+++ b/doc/devblog/day_394__implicit_vs_explicit.mdwn
@@ -0,0 +1,23 @@
+git-annex has always balanced implicit and explicit behavior.
+Enabling a git repository to be used with git-annex needs an explicit init,
+to avoid foot-shooting; but a clone of a repository that is already
+using git-annex will be implicitly initialized. Git remotes implicitly
+are checked to see if they use git-annex, so the user can immediately
+follow `git remote add` with `git annex get` to get files from it.
+
+There's a fine line here, and implicit git remote enabling sometimes
+crosses it; sometimes the remote doesn't have git-annex-shell, and so
+there's an ugly error message and annex-ignore has to be set to avoid
+trying to enable that git remote again. Sometimes the probe of a remote
+can occur when the user doesn't really expect it to (and it can involve a
+ssh password prompt).
+
+Part of the problem is, there's not an explicit way to enable a git remote
+to be used by git-annex. So, today, I made `git annex enableremote` do
+that, when the remote name passed to it is a git remote rather than a
+special remote. This way, you can avoid the implicit behavior if you want
+to.
+
+I also made `git annex enableremote` un-set annex-ignore, so if a remote
+got that set due to a transient configuration problem, it can be explicitly
+enabled.
diff --git a/doc/devblog/day_395__leaky_abstractions.mdwn b/doc/devblog/day_395__leaky_abstractions.mdwn
new file mode 100644
index 0000000000..41bce37cc1
--- /dev/null
+++ b/doc/devblog/day_395__leaky_abstractions.mdwn
@@ -0,0 +1,7 @@
+Release today includes a last-minute fix to parsing lines from the
+git-annex branch that might have one or more carriage returns at the end.
+This comes from Windows of course, where since some things transparently
+add/remove `\r` before the end of lines, while other things don't,
+it could result in quite a mess. Luckily it was not hard or expensive to
+handle. If you are lucky enough not to use Windows, the release also
+has several more interesting improvements.
diff --git a/doc/devblog/day_396__misc_fixes.mdwn b/doc/devblog/day_396__misc_fixes.mdwn
new file mode 100644
index 0000000000..8f33d66ea3
--- /dev/null
+++ b/doc/devblog/day_396__misc_fixes.mdwn
@@ -0,0 +1,11 @@
+A productive day of small fixes. Including a change to deal with an
+incompatibility in git 2.9's commit.gpgsign, and couple of fixes
+involving gcrypt repositories.
+
+Also several improvements to cloning from repositories where an adjusted
+branch is checked out. The clone automatically ends up with the adjusted
+branch checked out too.
+
+The test suite has 3 failures when run on a FAT repository, all involving
+adjusted branches. Managed to fix one of them today, hope to get to the
+others soon.
diff --git a/doc/devblog/day_397__befuddled.mdwn b/doc/devblog/day_397__befuddled.mdwn
new file mode 100644
index 0000000000..a7d35fc75f
--- /dev/null
+++ b/doc/devblog/day_397__befuddled.mdwn
@@ -0,0 +1,28 @@
+Been having a difficult time fixing the two remaining test suite failures
+when run on a FAT filesystem.
+
+On Friday, I got quite lost trying to understand the first failure. At
+first I thought it had something to do with queued git staging commands not
+being run in the right git environment when git-annex is using a different
+index file or work tree. I did find and fix a potential bug in that area.
+It might be that some reports long ago of git-annex branch files getting
+written to the master branch was caused by that. But, fixing it did not
+help with the test suite failure at hand.
+
+Today, I quickly found the actual cause of the first failure.
+Of course, it had nothing to do with queued git commands at
+all, and was a simple fix in the end.
+
+But, I've been staring at the second failure for hours and am not much
+wiser. All I know is, an invalid tree object gets generated by the adjusted
+branch code that contains some files more than once. (git gets very
+confused when a repository contains such tree objects; if you wanted to break a
+git repository, getting such trees into it might be a good way. *cough*) This
+invalid tree object seems to be caused by the basis ref for the adjusted branch
+diverging somehow from the adjusted branch itself. I have not been able to
+determine why or how the basis ref can diverge like that.
+
+Also, this failure is somewhat indeterminite, doesn't always occur and
+reordering the tests in the test suite can hide it. Weird.
+
+Well, hopefully looking at it again later with fresh eyes will help.
diff --git a/doc/devblog/day_398__fresh_eyes.mdwn b/doc/devblog/day_398__fresh_eyes.mdwn
new file mode 100644
index 0000000000..ce5c2adb7b
--- /dev/null
+++ b/doc/devblog/day_398__fresh_eyes.mdwn
@@ -0,0 +1,11 @@
+Today I was indeed able to get to the bottom of and fix the bug that
+had stumped me the other day.
+
+Rest of the day was taken up by catching up to some bug requests and
+suggestions for v6 mode. Like making unlock and lock work for files
+that are not locally present. And, improving the behavior of the clean
+filter so it remembers what backend was used for a file before and
+continues using that same backend.
+
+About ready to make a release, but IIRC there's one remaining test suite
+failure on FAT.
diff --git a/doc/devblog/day_399__weird_git_merge_bug.mdwn b/doc/devblog/day_399__weird_git_merge_bug.mdwn
new file mode 100644
index 0000000000..76e56384af
--- /dev/null
+++ b/doc/devblog/day_399__weird_git_merge_bug.mdwn
@@ -0,0 +1,16 @@
+There was one more test suite failure when run on FAT, which I've
+investigated today. It turns out that
+[a bug report](https://git-annex.branchable.com/bugs/Assistant_keeps_deleting_all_the_files_in_my_repo/)
+was filed about the same problem, and at root
+it seems to be a [bug in git merge](http://thread.gmane.org/gmane.comp.version-control.git/297237).
+Luckily, it was not hard to work around the strange merge behavior.
+
+It's been very worthwhile running the test suite on FAT; it's pointed me at
+several problems with adjusted branches over the past weeks. It would be
+good to add another test suite pass to test adjusted branches explicitly,
+but when I tried adding that, there were a lot of failures where the test
+suite is confused by adjusted branch behavior and would need to be taught
+about it.
+
+I've released git-annex 6.20160613. If you're using v6 repositories and
+especially adjusted branches, you should upgrade since it has many fixes.
diff --git a/doc/devblog/day_39__git-recover-repository.mdwn b/doc/devblog/day_39__git-recover-repository.mdwn
new file mode 100644
index 0000000000..f2c552eef8
--- /dev/null
+++ b/doc/devblog/day_39__git-recover-repository.mdwn
@@ -0,0 +1,54 @@
+Built a `git-recover-repository` command today. So far it only does the
+detection and deletion of corrupt objects, and retrieves them from remotes
+when possible. No handling yet of missing objects that cannot be recovered
+from remotes.
+
+Here's a couple of sample runs where I do bad things to the git
+repository and it fixes them:
+
+
    +joey@darkstar:~/tmp/git-annex>chmod 644 .git/objects/pack/*
+joey@darkstar:~/tmp/git-annex>echo > .git/objects/pack/pack-a1a770c1569ac6e2746f85573adc59477b96ebc5.pack 
+joey@darkstar:~/tmp/git-annex>~/src/git-annex/git-recover-repository 
+Running git fsck ...
+git fsck found a problem but no specific broken objects. Perhaps a corrupt pack file? Unpacking all pack files.
+fatal: early EOF
+Unpacking objects: 100% (148/148), done.
+Unpacking objects: 100% (354/354), done.
+Re-running git fsck to see if it finds more problems.
+Re-running git fsck to see if it finds more problems.
+Initialized empty Git repository in /home/joey/tmp/tmprepo.0/.git/
+Trying to recover missing objects from remote origin
+Successfully recovered repository!
+You should run "git fsck" to make sure, but it looks like
+everything was recovered ok.
+
    
+
+----
+
+
    +joey@darkstar:~/tmp/git-annex>chmod 644 .git/objects/00/0800742987b9f9c34caea512b413e627dd718e
+joey@darkstar:~/tmp/git-annex>echo > .git/objects/00/0800742987b9f9c34caea512b413e627dd718e
+joey@darkstar:~/tmp/git-annex>~/src/git-annex/git-recover-repository 
+Running git fsck ...
+error: unable to unpack 000800742987b9f9c34caea512b413e627dd718e header
+error: inflateEnd: stream consistency error (no message)
+error: unable to unpack 000800742987b9f9c34caea512b413e627dd718e header
+error: inflateEnd: stream consistency error (no message)
+git fsck found 1 broken objects. Unpacking all pack files.
+removing 1 corrupt loose objects
+Re-running git fsck to see if it finds more problems.
+Re-running git fsck to see if it finds more problems.
+Initialized empty Git repository in /home/joey/tmp/tmprepo.0/.git/
+Trying to recover missing objects from remote origin
+Successfully recovered repository!
+You should run "git fsck" to make sure, but it looks like
+everything was recovered ok.
+
    
+
+Works great! I need to move this and `git-union-merge` out of the git-annex
+source tree sometime.
+
+----
+
+Today's work was sponsored by Francois Marier.
diff --git a/doc/devblog/day_3__gcrypt_uuids.mdwn b/doc/devblog/day_3__gcrypt_uuids.mdwn
new file mode 100644
index 0000000000..3182aca633
--- /dev/null
+++ b/doc/devblog/day_3__gcrypt_uuids.mdwn
@@ -0,0 +1,63 @@
+Started work on [gcrypt](https://github.com/blake2-ppc/git-remote-gcrypt)
+support.
+
+The first question is, should git-annex leave it up to gcrypt to transport
+the data to the encrypted repository on a push/pull? gcrypt hooks into git
+nicely to make that just work. However, if I go this route, it limits
+the places the encrypted git repositores can be stored to regular git
+remotes (and rsync). The alternative is to somehow use gcrypt to
+generate/consume the data, but use the git-annex special remotes to store
+individual files. Which would allow for a git repo stored on S3, etc.
+For now, I am going with the simple option, but I have not ruled out
+trying to make the latter work. It seems it would need changes to gcrypt
+though.
+
+Next question: Given a remote that uses gcrypt, how do I determine the
+annex.uuid of that repository. I found a nice solutuon to this. gcrypt has
+its own gcrypt-id, and I convert it to a UUID in a 
+[[reproducible, and even standards-compliant way|design/gcrypt]]. So
+the same encrypted remote will automatically get the same annex.uuid
+wherever it's used. Nice. Does mean that git-annex cannot find a uuid
+until `git pull` or `git push` has been used, to let gcrypt get the
+gcrypt-id. Implemented that.
+
+The next step is actually making git-annex store data on gcrypt remotes.
+And it needs to store it encrypted of course. It seems best to avoid
+needing a `git annex initremote` for these gcrypt remotes, and just have
+git-annex automatically encrypt data stored on them. But I don't
+know. Without initializing them like a special remote is, I'm limited to
+using the gpg keys that gcrypt is configured to encrypt to, and cannot use
+the regular git-annex hybrid encryption scheme. Also, I need to generate
+and store a nonce anyway to HMAC ecrypt keys. (Or modify gcrypt
+to put enough entropy in gcrypt-id that I can use it?) 
+
+Another concern I have is that gcrypt's own encryption scheme is simply
+to use a list of public keys to encrypt to. It would be nicer if the
+full set of git-annex encryption schemes could be used. Then the webapp
+could use shared encryption to avoid needing to make the user set up a gpg
+key, or hybrid encryption could be used to add keys later, etc.
+
+But I see why gcrypt works the way it does. Otherwise, you can't make an
+encrypted repo with a friend set as one of the particpants and have them be
+able to git clone it. Both hybrid and shared encryption store a secret
+inside the repo, which is not accessible if it's encrypted using that
+secret. There are use cases where not being able to blindly clone a gcrypt
+repo would be ok. For example, you use the assistant to pair with a friend
+and then set up an encrypted repo in the cloud for both of you to use.
+
+Anyway, for now, I will need to deal with
+setting up gpg keys etc in the assistant. I don't want to tackle
+full [[design/assistant/gpgkeys]] yet. Instead, I think I will start by
+adding some simple stuff to the assistant:
+
+* When adding a USB drive, offer to encrypt the repository on the drive
+  so that only you can see it.
+* When adding a ssh remote make a similar offer.
+* Add a UI to add an arbitrary git remote with encryption.
+  Let the user paste in the url to an empty remote they have,
+  which could be to eg github. (In most cases this won't be used for
+  annexed content..)
+* When the user has no gpg key, prompt to set one up. (Securely!)
+* Maybe have an interface to add another gpg key that can access the gcrypt
+  repo. Note that this will need to re-encrypt and re-push the whole
+  git history.
diff --git a/doc/devblog/day_400-401__git_development.mdwn b/doc/devblog/day_400-401__git_development.mdwn
new file mode 100644
index 0000000000..a3b7f5d564
--- /dev/null
+++ b/doc/devblog/day_400-401__git_development.mdwn
@@ -0,0 +1,11 @@
+Working on git, not git-annex the past two days, I have implemented the
+smudge-to-file/clean-from-file extension to the smudge/clean filter
+interface. Patches have been [sent to the git developers](http://thread.gmane.org/gmane.comp.version-control.git/297475), 
+and hopefully they'll like it and include it. This will make git-annex
+v6 work a lot faster and better.
+
+Amazing how much harder it is to code on git than on git-annex! While I'm
+certianly not as familiar with the git code base, this is mostly because C
+requires so much more care about innumerable details and so much verbosity
+to do anything. I probably could have implemented this interface in
+git-annex in 2 hours, not 2 days.
diff --git a/doc/devblog/day_402__enhanced_smudge_clean_interface.mdwn b/doc/devblog/day_402__enhanced_smudge_clean_interface.mdwn
new file mode 100644
index 0000000000..a7739755ea
--- /dev/null
+++ b/doc/devblog/day_402__enhanced_smudge_clean_interface.mdwn
@@ -0,0 +1,14 @@
+Continued working on the enhancaed smudge/clean interface in git,
+incorporating feedback from the git developers.
+
+In a spare half an hour, I made an `improved-smudge-filters` branch
+that teaches git-annex smudge to use the new interface. 
+
+Doing a quick benchmark, `git checkout` of a deleted 1 gb file took:
+
+* 19 seconds before
+* 11 seconds with the new interface
+* 0.1 seconds with the new interface and annex.thin set  
+  (while also saving 1 gb of disk space!)
+
+So, this new interface is very much worthwhile.
diff --git a/doc/devblog/day_403__update_and_away.mdwn b/doc/devblog/day_403__update_and_away.mdwn
new file mode 100644
index 0000000000..02c1834354
--- /dev/null
+++ b/doc/devblog/day_403__update_and_away.mdwn
@@ -0,0 +1,4 @@
+Continued working on the enhanced smudge/clean interface in git today.
+Sent in a third version of the patch set, which is now quite complete.
+
+I'll be away for the next week and a half, on vacation.
diff --git a/doc/devblog/day_404__low_hanging_fruit.mdwn b/doc/devblog/day_404__low_hanging_fruit.mdwn
new file mode 100644
index 0000000000..73f8ee7fb9
--- /dev/null
+++ b/doc/devblog/day_404__low_hanging_fruit.mdwn
@@ -0,0 +1,15 @@
+Back from vacation, with a message backlog of 181. I'm concentrating first on
+low-hanging fruit of easily implemented todos, and well reproducible bugs,
+to get started again.
+
+Implemented --batch mode for `git annex get` and `git annex drop`, and also
+enabled --json for those.
+
+Investigated git-annex startup time; see
+.
+Turns out that cabal has a bug that causes many thousands of unnecessary
+syscalls when linking in the shared libraries. Working around it halved
+git-annex's startup time.
+
+Fixed a bug that caused `git annex testremote` to crash when testing a
+freshly made external special remote.
diff --git a/doc/devblog/day_405__more_git_development.mdwn b/doc/devblog/day_405__more_git_development.mdwn
new file mode 100644
index 0000000000..2a2485db89
--- /dev/null
+++ b/doc/devblog/day_405__more_git_development.mdwn
@@ -0,0 +1,3 @@
+Revisited my enhanced smudge/clean patch set for git, updating it for code
+review and to deal with changes in git since I've been away. This took
+several hours unfortunately.
diff --git a/doc/devblog/day_406__low_handing_fruit_continued.mdwn b/doc/devblog/day_406__low_handing_fruit_continued.mdwn
new file mode 100644
index 0000000000..e417f1856a
--- /dev/null
+++ b/doc/devblog/day_406__low_handing_fruit_continued.mdwn
@@ -0,0 +1,3 @@
+Worked on recent bug reports. Two bugs fixed today were both reversions
+introduced when the v6 repository support was added. Backlog is down to
+153.
diff --git a/doc/devblog/day_407__lazy_sunday.mdwn b/doc/devblog/day_407__lazy_sunday.mdwn
new file mode 100644
index 0000000000..7fe5e19176
--- /dev/null
+++ b/doc/devblog/day_407__lazy_sunday.mdwn
@@ -0,0 +1,9 @@
+Noticed that in one of my git-annex repositories, git-annex was spending
+a full second at startup checking all the git-annex branches from remotes
+to see if they contained changes that needed to be merged in. So, I added a
+cache of recently merged branches to avoid that. I remember considering
+this optimisation years ago; don't know why I didn't do it then. Not every
+day that I can speed up git-annex so much!
+
+Also, made `git annex log --all` show location log changes for all keys.
+This was tricky to get right and fast.
diff --git a/doc/devblog/day_408__release_day.mdwn b/doc/devblog/day_408__release_day.mdwn
new file mode 100644
index 0000000000..647b64eac8
--- /dev/null
+++ b/doc/devblog/day_408__release_day.mdwn
@@ -0,0 +1,6 @@
+First release in over a month. Before making this release, a few last
+minute fixes, including a partial workaround for the problem that
+Sqlite databases don't work on Lustre filesystems.
+
+Backlog is now down to 140 messages, and only 3 of those are from this
+month. Still higher than I like.
diff --git a/doc/devblog/day_408__release_day/comment_1_RichiH._comment b/doc/devblog/day_408__release_day/comment_1_RichiH._comment
new file mode 100644
index 0000000000..36f8272022
--- /dev/null
+++ b/doc/devblog/day_408__release_day/comment_1_RichiH._comment
@@ -0,0 +1,3 @@
+You typo'ed the tag and the release date in debian/changelog: 6.20160619 instead of 6.20160719
+
+I will release 6.20160719-1 in Debian on the assumption that you will fix that tag :)
diff --git a/doc/devblog/day_409__--branch.mdwn b/doc/devblog/day_409__--branch.mdwn
new file mode 100644
index 0000000000..6ede436c76
--- /dev/null
+++ b/doc/devblog/day_409__--branch.mdwn
@@ -0,0 +1,17 @@
+A common complaint is that `git annex fsck` in a bare repository complains
+about missing content of deleted files. That's because in a bare
+repository, git-annex operates on all versions of all files. Today I added
+a --branch option, so if you only want to check say, the master branch, you
+can: `git annex fsck --branch master`
+
+The new option has other uses too. Want to get all the files in the v1.0
+tag? `git annex get --branch v1.0`
+
+It might be worth revisiting the implicit --all behavior for bare
+repositories. It could instead default to --branch HEAD or something like
+that. But I'd only want to change that if there was a strong consensus in
+favor.
+
+Over 3/4th of the time spent implementing --branch was spent in
+adjusting the output of commands, to show "branch:file" is being
+operated on. How annoying.
diff --git a/doc/devblog/day_40__another_fine_mess.mdwn b/doc/devblog/day_40__another_fine_mess.mdwn
new file mode 100644
index 0000000000..dfe5bc5642
--- /dev/null
+++ b/doc/devblog/day_40__another_fine_mess.mdwn
@@ -0,0 +1,15 @@
+Solid day of working on repository recovery. Got `git recover-repository
+--force` working, which involves fixing up branches that refer to missing
+objects. Mostly straightforward traversal of git commits, trees, blobs, to
+find when a branch has a problem, and identify an old version of it that
+predates the missing object. (Can also find them in the reflog.) 
+
+The main complication turned out to be that `git branch -D` and `git
+show-ref` don't behave very well when the commit objects pointed to by refs
+are themselves missing. And git has no low-level plumbing that avoids
+falling over these problems, so I had to write it myself.
+
+Testing has turned up one unexpected problem: Git's index can itself refer
+to missing objects, and that will break future commits, etc. So I need to
+find a way to validate the index, and when it's got problems,
+either throw it out, or possibly recover some of the staged data from it.
diff --git a/doc/devblog/day_410__better_JSON_for_metadata.mdwn b/doc/devblog/day_410__better_JSON_for_metadata.mdwn
new file mode 100644
index 0000000000..44648d9992
--- /dev/null
+++ b/doc/devblog/day_410__better_JSON_for_metadata.mdwn
@@ -0,0 +1,31 @@
+I've had to change the output of `git annex metadata --json`.
+The old output looked like this:
+
+	{"command":"metadata","file":"foo","key":"...","author":["bar"],...,"note":"...","success":true}
+
+That was not good, because it didn't separate the metadata fields
+from the rest of the JSON object. What if a metadata field is named
+"note" or "success"? It would collide with the other "note" and "success"
+in the JSON.
+
+So, changed this to a new format, which moves the metadata fields into
+a "fields" object:
+
+	{"command":"metadata","file":"foo","key":"...","fields":{"author":["bar"],...},"note":"...","success":true}
+
+I don't like breaking backwards compatability of JSON output, but in this
+case I could see no real alternative. I don't know if anyone
+is using `metadata --batch` anyway. If you are and this will cause a
+problem, get in touch.
+
+----
+
+While making that change, I also improved the JSON output layer, so it can
+use Aeson. Update: And switched everything over to using Aeson, so
+git-annex no longer depends on two different JSON libraries.
+
+This let me use Aeson to generate the "fields" object for `metadata
+--json`. And it was also easy enough to use Aeson to parse the output of
+that command (and some simplified forms of it).
+
+So, I've laid the groundwork for `git annex metadata --batch` today.
diff --git a/doc/devblog/day_411__metadata_--batch.mdwn b/doc/devblog/day_411__metadata_--batch.mdwn
new file mode 100644
index 0000000000..acd1429c8a
--- /dev/null
+++ b/doc/devblog/day_411__metadata_--batch.mdwn
@@ -0,0 +1,4 @@
+With yesterday's JSON groundwork in place, I quickly implemented `git annex
+metadata --batch` today in only 45 LoC. The interface is nicely elegant;
+the same JSON format that git-annex metadata outputs can be fed into it
+to get, set, delete, and modify metadata.
diff --git a/doc/devblog/day_412__if_at_first_you_dont_succeed.mdwn b/doc/devblog/day_412__if_at_first_you_dont_succeed.mdwn
new file mode 100644
index 0000000000..958564cab3
--- /dev/null
+++ b/doc/devblog/day_412__if_at_first_you_dont_succeed.mdwn
@@ -0,0 +1,49 @@
+[[!meta title="if at first you don't succeed.."]]
+
+A user suggested [adding --failed](http://git-annex.branchable.com/todo/__34__copy_--failed__34__/)
+to retry failed transfers. That was a great idea and I landed a patch for it
+3 hours later. Love it when a user suggests something so clearly right and
+I am able to quickly make it happen!
+
+----
+
+Unfortunately, my funding from the [DataLad](https://datalad.org/) project
+to work on git-annex is running out. It's been a very good two years funded
+that way, with an enormous amount of improvements and support and bug
+fixes, but all good things must end. I'll continue to get some funding
+from them for the next year, but only for half as much time as the past two
+years.
+
+I need to decide it it makes sense to keep working on git-annex to the
+extent I have been. There are definitely a few (hundred) things I still
+want to do on git-annex, starting with getting the git patches landed to
+make v6 mode really shine. Past that, it's mostly up to the users. If they
+keep suggesting great ideas and finding git-annex useful, I'll want to
+work on it more.
+
+What to do about funding? Maybe some git-annex users can contribute a
+small amount each month to fund development. I've set up a Patreon
+page for this, ****
+
+----
+
+Anyhoo... Back to today's (unfunded) work.
+
+`--failed` can be used with `get`, `move`, `copy`, and
+`mirror`. Of course those commands can all be simply re-ran if some
+of the transfers fail and will pick up where they left off. But using
+`--failed` is faster because it does not need to scan all files to find
+out which still need to be transferred. And accumulated failures from
+multiple commands can be retried with a single use of `--failed`. 
+
+It's even possible to do things like `git annex get --from foo; git annex
+get --failed --from bar`, which first downloads everything it can from the
+foo remote and falls back to using the bar remote for the rest. Although
+setting remote costs is probably a better approach most of the time.
+
+Turns out that I had earlier disabled writing failure log files, except by
+the assistant, because only the assistant was using them. So, that had to
+be undone. There's some potential for failure log files to accumulate
+annoyingly, so perhaps some expiry mechanism will be needed. This is why
+`--failed` is documented as retrying "recent" transfers. Anyway, the
+failure log files are cleaned up after successful transfers.
diff --git a/doc/devblog/day_412__if_at_first_you_dont_succeed/comment_1_8980f56815a2c016ca950aec09e6e839._comment b/doc/devblog/day_412__if_at_first_you_dont_succeed/comment_1_8980f56815a2c016ca950aec09e6e839._comment
new file mode 100644
index 0000000000..b299eb2a71
--- /dev/null
+++ b/doc/devblog/day_412__if_at_first_you_dont_succeed/comment_1_8980f56815a2c016ca950aec09e6e839._comment
@@ -0,0 +1,18 @@
+[[!comment format=mdwn
+ username="rasmus"
+ avatar="http://cdn.libravatar.org/avatar/cb09dc4c52e771aa888e14b931d3acc3"
+ subject="support"
+ date="2016-10-27T10:45:06Z"
+ content="""
+Hi Joey,
+
+git annex has become great.  In the past I spent a lot of time fiddeling with it and even lost files.  Now, it all runs in the background flawlessly.
+Even the \"spin up\" of the disk is
+
+In any case, I would be very happy to help fund the development in what way I can.
+However, it would be great if you could point out a way to do single transfers to you, either by credit card or paypal (or a European bank transfer, which I guess is less interesting).
+I'm also happy to support a rebooted campaign.joeyh.name.
+
+Thanks,
+Rasmus
+"""]]
diff --git a/doc/devblog/day_412__if_at_first_you_dont_succeed/comment_2_641ca9bce852cea42cac25fd77518885._comment b/doc/devblog/day_412__if_at_first_you_dont_succeed/comment_2_641ca9bce852cea42cac25fd77518885._comment
new file mode 100644
index 0000000000..9754a97ced
--- /dev/null
+++ b/doc/devblog/day_412__if_at_first_you_dont_succeed/comment_2_641ca9bce852cea42cac25fd77518885._comment
@@ -0,0 +1,7 @@
+[[!comment format=mdwn
+ username="joey"
+ subject="""comment 2"""
+ date="2016-10-27T19:23:41Z"
+ content="""
+Nice to hear.  has some pointers.
+"""]]
diff --git a/doc/devblog/day_413__back.mdwn b/doc/devblog/day_413__back.mdwn
new file mode 100644
index 0000000000..ebd2335ee4
--- /dev/null
+++ b/doc/devblog/day_413__back.mdwn
@@ -0,0 +1,19 @@
+Back after taking most of August off and working on other projects.
+
+Got the unanswered messages backlog down from 222 to 170. Still scary high.
+
+Numerous little improvements today. Notable ones:
+
+* Windows: Handle shebang in external special remote program. This is
+  needed for [git-annex-remote-rclone to work on Windows](https://github.com/DanielDent/git-annex-remote-rclone/pull/10).
+  Nice to see that external special remote is getting ported and
+  apparently lots of use.
+* Make --json and --quiet suppress automatic init messages, and any
+  other messages that might be output before a command starts. This was a
+  reversion introduced in the optparse-applicative changes over a year ago.
+
+Also I'm developing a plan to improve parallel downloading when multiple
+remotes have the same cost. See [[todo/get_round_robin]].
+
+Today's work was sponsored by Jake Vosloo on
+[Patreon](https://www.patreon.com/joeyh).
diff --git a/doc/devblog/day_414__improved_parallel_get.mdwn b/doc/devblog/day_414__improved_parallel_get.mdwn
new file mode 100644
index 0000000000..28f6c8ad8d
--- /dev/null
+++ b/doc/devblog/day_414__improved_parallel_get.mdwn
@@ -0,0 +1,6 @@
+Turned out to not be very hard at all to make `git annex get -JN`
+assign different threads to different remotes that have the same cost.
+Something like that was requested back in 2011, but it didn't really make
+sense until parallel get was implemented last year.
+
+(Also spent too much time fixing up broken builds.)
diff --git a/doc/devblog/day_415__catching_up.mdwn b/doc/devblog/day_415__catching_up.mdwn
new file mode 100644
index 0000000000..d2828a0c8b
--- /dev/null
+++ b/doc/devblog/day_415__catching_up.mdwn
@@ -0,0 +1,19 @@
+Catching up on backlog today. I hope to be back to a regular work schedule
+now. Unanswered messages down to 156. A lot of time today spent answering
+questions.
+
+There were several problems involving git branches with slashes in their
+name, such as "foo/bar" (but not "origin/master" or "refs/heads/foo").
+Some branch names based on such a branch would take only the "bar" part.
+In `git annex sync`, this led to perhaps merging "foo/bar" into "other/bar"
+or "bar". And the adjusted branch code was entirely broken for such
+branches. I've fixed it now.
+
+Also made `git annex addurl` behave better when the file it wants to 
+add is gitignored.
+
+Thinking about implementing `git annex copy --from A --to B`.
+It does not seem *too* hard to do that, at least with a temp file
+used inbetween. See [[todo/transitive_transfers]].
+
+Today's work was sponsored by Thomas Hochstein on Patreon.
diff --git a/doc/devblog/day_416__measure_twice.mdwn b/doc/devblog/day_416__measure_twice.mdwn
new file mode 100644
index 0000000000..e3d9f00c47
--- /dev/null
+++ b/doc/devblog/day_416__measure_twice.mdwn
@@ -0,0 +1,8 @@
+Only had a couple hours today, which were spent doing some profiling of
+git-annex in situations where it has to look through a large working tree in
+order to find files to act on. The top five hot spots this found are
+responsible for between 50% and 80% of git-annex's total CPU use in these
+situations.
+
+The first optimisation sped up `git annex find` by around 18%.
+More tomorrow..
diff --git a/doc/devblog/day_417__cut_once.mdwn b/doc/devblog/day_417__cut_once.mdwn
new file mode 100644
index 0000000000..419e279f78
--- /dev/null
+++ b/doc/devblog/day_417__cut_once.mdwn
@@ -0,0 +1,6 @@
+Did most of the optimisations that recent profiling suggested. 
+This sped up a `git annex find` from 3.53 seconds to 1.73 seconds.
+And, `git annex find --not --in remote` from 12.41 seconds to 5.24 seconds.
+One of the optimisations sped up git-annex branch querying by up to 50%,
+which should also speed up use of some preferred content expressions.
+All in all, a very nice little optimisation pass.
diff --git a/doc/devblog/day_418__concurrent_externals.mdwn b/doc/devblog/day_418__concurrent_externals.mdwn
new file mode 100644
index 0000000000..6dd6916dd2
--- /dev/null
+++ b/doc/devblog/day_418__concurrent_externals.mdwn
@@ -0,0 +1,12 @@
+Realized recently that despite all the nice concurrency support in
+git-annex, external special remotes were limited to handling one request at
+a time.
+
+While the external special remote prococol could almost support concurrent
+requests, that would complicate implementing them, and probably need a
+version flag to enable to avoid breaking existing ones.
+
+Instead, made git-annex start up multiple external special remote processes
+as needed to handle concurrency.
+
+Today's work was sponsored by Josh Taylor on Patreon.
diff --git a/doc/devblog/day_418__concurrent_externals/comment_1_75e6569b143cf3f595ba9b356415a747._comment b/doc/devblog/day_418__concurrent_externals/comment_1_75e6569b143cf3f595ba9b356415a747._comment
new file mode 100644
index 0000000000..dfad031d78
--- /dev/null
+++ b/doc/devblog/day_418__concurrent_externals/comment_1_75e6569b143cf3f595ba9b356415a747._comment
@@ -0,0 +1,11 @@
+[[!comment format=mdwn
+ username="Adam"
+ subject="Question"
+ date="2016-10-04T05:27:31Z"
+ content="""
+Hi Joey,
+
+Does this mean that, for example, when uploading many small files to a special remote, that git-annex could upload more than one at a time?  Sometimes it seems like the time spent waiting for a small file to upload is mostly spent waiting for the server to begin accepting the file.   With lots of small files, this adds up (similar to how round-trip time slows down Obnam significantly when using a network repo).
+
+Thanks.
+"""]]
diff --git a/doc/devblog/day_418__concurrent_externals/comment_2_067909ed54f3f9295f90233f25ccfd70._comment b/doc/devblog/day_418__concurrent_externals/comment_2_067909ed54f3f9295f90233f25ccfd70._comment
new file mode 100644
index 0000000000..c075785763
--- /dev/null
+++ b/doc/devblog/day_418__concurrent_externals/comment_2_067909ed54f3f9295f90233f25ccfd70._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="joey"
+ subject="""comment 2"""
+ date="2016-10-04T15:16:01Z"
+ content="""
+Well yes, git-annex -J4 or something can parallelize any operation and
+avoid such latencies.
+"""]]
diff --git a/doc/devblog/day_418__concurrent_externals/comment_3_39c729fc2ea4b597d13e54d60bb36d0d._comment b/doc/devblog/day_418__concurrent_externals/comment_3_39c729fc2ea4b597d13e54d60bb36d0d._comment
new file mode 100644
index 0000000000..7f4202ab9c
--- /dev/null
+++ b/doc/devblog/day_418__concurrent_externals/comment_3_39c729fc2ea4b597d13e54d60bb36d0d._comment
@@ -0,0 +1,7 @@
+[[!comment format=mdwn
+ username="Adam"
+ subject="comment 3"
+ date="2016-10-04T21:24:50Z"
+ content="""
+Oh, I see, thanks.  Have you thought about adding an option for that to the assistant?  For example, I tested syncing a bunch of text files through a special remote, and even though the files were small, the per-command latency added up, so the effective transfer rate wasn't very high.  If it were easy to set the concurrency option in the assistant, I guess it would help with that.
+"""]]
diff --git a/doc/devblog/day_418__concurrent_externals/comment_4_6b254fbe92d40abc30d01afe5d039afc._comment b/doc/devblog/day_418__concurrent_externals/comment_4_6b254fbe92d40abc30d01afe5d039afc._comment
new file mode 100644
index 0000000000..8f3387ee1f
--- /dev/null
+++ b/doc/devblog/day_418__concurrent_externals/comment_4_6b254fbe92d40abc30d01afe5d039afc._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="evilmoo@0c9697a666c2853b3b3229b6646d3a4e849f7ec8"
+ nickname="evilmoo"
+ subject="bittorrent"
+ date="2016-10-05T03:53:05Z"
+ content="""
+Will this speed up the problems with added Bittorrent URLs only downloading one file at a time?
+"""]]
diff --git a/doc/devblog/day_418__concurrent_externals/comment_5_5e851b62bdca1cb8d8062d4890c77956._comment b/doc/devblog/day_418__concurrent_externals/comment_5_5e851b62bdca1cb8d8062d4890c77956._comment
new file mode 100644
index 0000000000..a3f39a5cd5
--- /dev/null
+++ b/doc/devblog/day_418__concurrent_externals/comment_5_5e851b62bdca1cb8d8062d4890c77956._comment
@@ -0,0 +1,7 @@
+[[!comment format=mdwn
+ username="joey"
+ subject="""comment 5"""
+ date="2016-10-05T15:31:37Z"
+ content="""
+@evilmoo is that problem detailed somewhere?
+"""]]
diff --git a/doc/devblog/day_418__concurrent_externals/comment_6_ea5d42bcb03af937a604119039f9073a._comment b/doc/devblog/day_418__concurrent_externals/comment_6_ea5d42bcb03af937a604119039f9073a._comment
new file mode 100644
index 0000000000..cb7a23ccec
--- /dev/null
+++ b/doc/devblog/day_418__concurrent_externals/comment_6_ea5d42bcb03af937a604119039f9073a._comment
@@ -0,0 +1,7 @@
+[[!comment format=mdwn
+ username="joey"
+ subject="""comment 6"""
+ date="2016-10-05T15:32:15Z"
+ content="""
+@Adam adding a concurrency setting to the assistant would indeed be nice.
+"""]]
diff --git a/doc/devblog/day_419__catching_up.mdwn b/doc/devblog/day_419__catching_up.mdwn
new file mode 100644
index 0000000000..90e0da1ff9
--- /dev/null
+++ b/doc/devblog/day_419__catching_up.mdwn
@@ -0,0 +1,23 @@
+Several bug fixes today and got caught up on most recent messages. Backlog
+is 157.
+
+The most significant one prevents git-annex from reading in the whole
+content of a large git object when it wants to check if it's an annex
+symlink. In several situations where large files were committed to git, or
+staged, git-annex could do a lot of work, and use a lot of memory and maybe
+crash. Fixed by checking the size of an object before asking 
+`git cat-file` for its content.
+
+Also a couple of improvements around versions and upgrading. IIRC git-annex
+used to only support one repository version at a time, but this was changed
+to support V6 as an optional upgrade from V5, and so the supported versions
+became a list. Since V3 repositories are identical to V5 other than the
+version, I added it to the supported version list, and any V3 repos out
+there can be used without upgading. Particularly useful if they're on
+read-only media.
+
+And, there was a bug in the automatic upgrading of a remote that caused it
+to be upgraded all the way to V6. Now it will only be upgraded to V5.
+
+Today's work was sponsored by Jake Vosloo on
+[Patreon](https://patreon.com/joeyh/).
diff --git a/doc/devblog/day_41__onward.mdwn b/doc/devblog/day_41__onward.mdwn
new file mode 100644
index 0000000000..fd393b7340
--- /dev/null
+++ b/doc/devblog/day_41__onward.mdwn
@@ -0,0 +1,17 @@
+I think that git-recover-repository is ready now. Made it deal with the
+index file referencing corrupt objects. The best approach I could think of
+for that is to just remove those objects from the index, so the user can
+re-add files from their work tree after recovery.
+
+Now to integrate this git repository repair capability into the git-annex 
+assistant. I decided to run `git fsck` as part of a scheduled
+repository consistency check. It may also make sense for the assistant to
+notice when things are going wrong, and suggest an immediate check. I've
+started on the webapp UI to run a repository repair when fsck detects
+problems.
+
+[[!img /assistant/brokenrepositoryalert.png]]
+
+[[!img /assistant/repairrepository.png]]
+
+[[!meta title="the user interface I hope noone ever sees"]]
diff --git a/doc/devblog/day_41__onward/comment_1_a716c7b5a9ea3c949ff047cfb4e9a0a4._comment b/doc/devblog/day_41__onward/comment_1_a716c7b5a9ea3c949ff047cfb4e9a0a4._comment
new file mode 100644
index 0000000000..0457a4bf04
--- /dev/null
+++ b/doc/devblog/day_41__onward/comment_1_a716c7b5a9ea3c949ff047cfb4e9a0a4._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="aarone"
+ ip="2607:f470:8:d008:290:f5ff:fec3:a9e6"
+ subject="Email alert"
+ date="2013-10-22T21:23:24Z"
+ content="""
+I have at least one git-annex repository where I seldom open the web UI, but rather have the assistant humming along in the background.  I think for my use case it would be best to receive an email notification that the assistant had detected a problem, which would prompt me to open the web UI and perform the recovery.
+"""]]
diff --git a/doc/devblog/day_41__onward/comment_2_33149e424cd5f03fac376288bcc4dfdc._comment b/doc/devblog/day_41__onward/comment_2_33149e424cd5f03fac376288bcc4dfdc._comment
new file mode 100644
index 0000000000..fd26f92903
--- /dev/null
+++ b/doc/devblog/day_41__onward/comment_2_33149e424cd5f03fac376288bcc4dfdc._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="http://joeyh.name/"
+ ip="64.134.31.139"
+ subject="excellent idea"
+ date="2013-10-22T22:13:58Z"
+ content="""
+Added to my todo. I think it would require the system have an MTA though, I really don't want git-annex to grow to the point it can send email. ;)
+"""]]
diff --git a/doc/devblog/day_41__onward/comment_3_3b07503bd79089ad3ce3ddd7535ed116._comment b/doc/devblog/day_41__onward/comment_3_3b07503bd79089ad3ce3ddd7535ed116._comment
new file mode 100644
index 0000000000..1116b4d1b0
--- /dev/null
+++ b/doc/devblog/day_41__onward/comment_3_3b07503bd79089ad3ce3ddd7535ed116._comment
@@ -0,0 +1,14 @@
+[[!comment format=mdwn
+ username="http://jasonwoof.com/"
+ nickname="JasonWoof"
+ subject="Please check if it needs help from the user first"
+ date="2013-10-23T23:30:10Z"
+ content="""
+Awesome that you're working on recovery, and recovery automation!
+
+Please only bother the user if there is a serious problem _and_ it can't be fixed without their help. Otherwise I fear people will learn to ignore your dialog boxes, like they do most dialog boxes. If you want to notify your user that some hard drive corruption happened (and it's been fixed already) then put a little yellow/orange line across somewhere with a warning message.
+
+When I read to the part about how it can (probably) be fixed automatically, I got a flash of annoyance and thought \"well, then fix it automatically, why are you bothering me?\"
+
+Please (if you aren't already) check if it can be automatically fixed without help from the user before telling the user about it. Then you can say \"The data to fix this could not be reached, please plug in another repo or something.\"
+"""]]
diff --git a/doc/devblog/day_420__delayed_debugging.mdwn b/doc/devblog/day_420__delayed_debugging.mdwn
new file mode 100644
index 0000000000..8ab3ecd972
--- /dev/null
+++ b/doc/devblog/day_420__delayed_debugging.mdwn
@@ -0,0 +1,17 @@
+Over a month ago, I had some reports that syncing into adjusted branches
+was losing some files that had been committed. I couldn't reproduce it, but
+IIRC both felix and tbm reported problems in this area. And, felix kindly
+sent me enough of his git repo to hopefully reproduce it the problem.
+
+Finally got back to that today. Luckily, I *was* able to reproduce the bug
+using felix's repo. The bug only occurs when there's a change deep in a
+tree of an adjusted branch, and not always then. After staring at it for a
+couple of hours, I finally found the problem; a modification flag was not
+getting propagated in this case, and some changes made deep in the tree
+were not getting included into parent trees.
+
+So, I think I've fixed it, but need to look at it some more to be sure, and
+develop a test case. And fixing that exposed another bug in the same code.
+Gotta run unfortunately, so will finish this tomorrow..
+
+Today's work was sponsored by Riku Voipio.
diff --git a/doc/devblog/day_421__lost_in_the_trees.mdwn b/doc/devblog/day_421__lost_in_the_trees.mdwn
new file mode 100644
index 0000000000..1cb09b63da
--- /dev/null
+++ b/doc/devblog/day_421__lost_in_the_trees.mdwn
@@ -0,0 +1,6 @@
+Finished up where I left off yesterday, writing test cases and fixing
+bugs with syncing in adjusted branches. While adjusted branches need v6
+mode, and v6 mode is still considered experimental, this is still a rather
+nasty bug, since it can make files go missing (though still available
+in git history of course). So, planning to release a new version with these
+fixes as soon as the autobuilders build it.
diff --git a/doc/devblog/day_422__bugfixes_for_v6_mode.mdwn b/doc/devblog/day_422__bugfixes_for_v6_mode.mdwn
new file mode 100644
index 0000000000..3cd7bc35b9
--- /dev/null
+++ b/doc/devblog/day_422__bugfixes_for_v6_mode.mdwn
@@ -0,0 +1,16 @@
+Several bug fixes involving v6 unlocked files today. Several related bugs
+were caused by relying on the inode cache information, without a fallback
+to handle the case where the inode cache had not gotten updated. While the
+inode cache is generally kept up-to-date well by the smudge/clean
+filtering, it is just a cache and can be out of date. Did some auditing for
+such problems and hopefully I've managed to find them all.
+
+Also, there was a tricky upgrade case where a v5 repository contained a
+v6 unlocked file, and the annexed content got copied into it. This
+triggered the above-described bugs, and in this case the worktree needs
+to be updated on upgrade, to replace the pointer file with the content.
+
+As I caught up with recent activity, it was nice to see some contributions
+from others. James MacMahon sent in a patch to improve the filenames
+generated by `importfeed`. And, xloem is writing workflow documentation for
+git-annex in [[todo/Workflow_guide]].
diff --git a/doc/devblog/day_423__ssh_fun.mdwn b/doc/devblog/day_423__ssh_fun.mdwn
new file mode 100644
index 0000000000..1fe12bc368
--- /dev/null
+++ b/doc/devblog/day_423__ssh_fun.mdwn
@@ -0,0 +1,19 @@
+Made a significant change today: Enabled automatic retrying of transfers
+that fail. It's only done if the previous try managed to advance the
+progress by some amount. The assistant has already had that retrying for
+many years, but now it will also be done when using git-annex at the
+command line.
+
+One good reason for a transfer to fail and need a retry is when the network
+connection stalls. You'd think that TCP keepalives would detect this kind
+of thing and kill the connection but I've had enough complaints, that I
+suppose that doesn't always work or gets disabled. Ssh has a
+ServerAliveInterval that detects such stalls nicely for the kind of batch
+transfers git-annex uses ssh for, but it's not enabled by default. So I
+found a way to make git-annex enable it, while still letting ~/.ssh/config
+settings override that.
+
+Also got back to analizing an old bug report about proliferating
+".nfs*.lock" files when using git-annex on nfs; this was caused by the
+wacky NFS behavior of renaming deleted files, and I found a change to the
+ssh connection caching cleanup code that should avoid the problem.
diff --git a/doc/devblog/day_424__the_dog.mdwn b/doc/devblog/day_424__the_dog.mdwn
new file mode 100644
index 0000000000..0c117a8563
--- /dev/null
+++ b/doc/devblog/day_424__the_dog.mdwn
@@ -0,0 +1,11 @@
+[[!meta title="the dog that didn't bark"]]
+
+Worked on several bug reports today, fixing some easy ones, and following
+up on others. And then there are the hard bugs.. Very pleased that I was
+able to eventually reproduce a bug based entirely on the information
+that git-annex's output did not include a filename. Didn't quite get that
+bug fixed though.
+
+At the end of the day, got a bug report that `git annex add` of filenames
+containing spaces has broken. This is a recent reversion and I'm pushing
+out a release with a fix ASAP.
diff --git a/doc/devblog/day_425__tor.mdwn b/doc/devblog/day_425__tor.mdwn
new file mode 100644
index 0000000000..08fe21cdd5
--- /dev/null
+++ b/doc/devblog/day_425__tor.mdwn
@@ -0,0 +1,23 @@
+Have waited too long for some next-generation encrypted P2P network, like
+telehash to emerge. Time to stop waiting; tor hidden services are not as
+cutting edge, but should work. Updated the [[design|design/assistant/telehash]]
+and started implementation in the `tor` branch.
+
+Unfortunately, Tor's default configuration does not enable the ControlPort.
+And, changing that in the configuration could be problimatic. This
+makes it harder than it ought to be to register a tor hidden service.
+So, I implemented a `git annex enable-tor` command, which can be run as root
+to set it up. The webapp will probably use `su-to-root` or `gksu` to run it. 
+There's some Linux-specific parts in there, and it uses a socket for
+communication between tor and the hidden service, which may cause problems
+for Windows porting later.
+
+Next step will be to get `git annex remotedaemon` to run as a tor hidden
+service.
+
+Also made a `no-xmpp` branch which removes xmpp support from the assistant.
+That will remove 3000 lines of code when it's merged. Will probably wait
+until after tor hidden services are working.
+
+Today's work was sponsored by Jake Vosloo on
+[Patreon](https://www.patreon.com/joeyh/).
diff --git a/doc/devblog/day_425__tor/comment_1_1dd41fa32eb3867d764f3238005b5b81._comment b/doc/devblog/day_425__tor/comment_1_1dd41fa32eb3867d764f3238005b5b81._comment
new file mode 100644
index 0000000000..fe609cab01
--- /dev/null
+++ b/doc/devblog/day_425__tor/comment_1_1dd41fa32eb3867d764f3238005b5b81._comment
@@ -0,0 +1,11 @@
+[[!comment format=mdwn
+ username="grawity@2ea26be48562f66fcb9b66307da72b1e2e37453f"
+ nickname="grawity"
+ avatar="http://cdn.libravatar.org/avatar/7003e967f47003bae82966aa373de8ef"
+ subject="comment 1"
+ date="2016-11-15T18:01:18Z"
+ content="""
+…or `pkexec`, which is present on many systems and generally integrates better with whatever DE/non-DE the user may be running.
+
+(OTOH, pkexec does not set up X11 access – then again, root helpers shouldn't need it.)
+"""]]
diff --git a/doc/devblog/day_426__grab_bag.mdwn b/doc/devblog/day_426__grab_bag.mdwn
new file mode 100644
index 0000000000..36e32077ee
--- /dev/null
+++ b/doc/devblog/day_426__grab_bag.mdwn
@@ -0,0 +1,63 @@
+Fixed one howler of a bug today. Turns out that
+`git annex fsck --all --from remote` didn't actually check the content of
+the remote, but checked the local repository. Only `--all` was buggy;
+`git annex fsck --from remote` was ok. Don't think this is crash priority
+enough to make a release for, since only `--all` is affected.
+
+Somewhat uncomfortably made `git annex sync` pass
+`--allow-unrelated-histories` to git merge. While I do think that git's
+recent refusal to merge unrelated histories is good in general, the
+problem is that initializing a direct mode repository involves making an
+empty commit. So merging from a remote into such a direct mode repository
+means merging unrelated histories, while an indirect mode repository doesn't.
+Seems best to avoid such inconsistencies, and the only way I could see to
+do it is to always use `--allow-unrelated-histories`. May revisit this once
+direct mode is finally removed.
+
+Using the git-annex arm standalone bundle on some WD NAS boxes used to
+work, and then it seems they changed their kernel to use a nonstandard page
+size, and broke it. This actually seems to be a 
+[bug in the gold linker](http://bugs.debian.org/844467), which defaults to an
+unncessarily small page size on arm. The git-annex arm bundle is being
+adjusted to try to deal with this.
+
+ghc 8 made `error` include some backtrace information. While it's really
+nice to have backtraces for unexpected exceptions in Haskell, it turns
+out that git-annex used `error` a lot with the intent of showing an error
+message to the user, and a backtrace clutters up such messages. So,
+bit the bullet and checked through every `error` in git-annex and made such
+ones not include a backtrace.
+
+Also, I've been considering what protocol to use between git-annex nodes
+when communicating over tor. One way would be to make it very similar to
+`git-annex-shell`, using rsync etc, and possibly reusing code from
+git-annex-shell. However, it can take a while to make a connection across
+the tor network, and that method seems to need a new connection for each
+file transfered etc. Also thought about using a http based protocol. The
+servant library is great for that, you get both http client and server
+implementations almost for free. Resuming interrupted transfers might
+complicate it, and the hidden service side would need to listen on a unix
+socket, instead of the regular http port. It might be worth it to use http
+for tor, if it could be reused for git-annex http servers not on the tor
+network. But, then I'd have to make the http server support git pull and
+push over http in a way that's compatable with how git uses http, including
+authentication. Which is a whole nother ball of complexity. So, I'm leaning
+instead to using a simple custom protocol something like:
+
+		> AUTH $localuuid $token
+		< AUTH-SUCCESS $remoteuuid
+		> SENDPACK $length
+		> $gitdata
+		< RECVPACK $length
+		< $gitdata
+		> GET $pos $key
+		< DATA $length
+		< $bytes
+		> SUCCESS
+		> PUT $key
+		< PUT-FROM $pos
+		> DATA $length
+		> $bytes
+		< SUCCESS
+
+Today's work was sponsored by Riku Voipio.
diff --git a/doc/devblog/day_426__grab_bag/comment_1_4d01c756850032d351fa99188a3301a7._comment b/doc/devblog/day_426__grab_bag/comment_1_4d01c756850032d351fa99188a3301a7._comment
new file mode 100644
index 0000000000..7b5a2949a9
--- /dev/null
+++ b/doc/devblog/day_426__grab_bag/comment_1_4d01c756850032d351fa99188a3301a7._comment
@@ -0,0 +1,11 @@
+[[!comment format=mdwn
+ username="https://anarc.at/openid/"
+ nickname="anarcat"
+ avatar="http://cdn.libravatar.org/avatar/b36dcf65657dd36128161355d8920a99503def9461c1bb212410980fe6f07125"
+ subject="how about reusing the special remote protocol?"
+ date="2016-11-16T21:58:08Z"
+ content="""
+git-annex already has a custom protocol detailed in [[design/external_special_remote_protocol]]. it could be quite useful to have that protocol extended to support direct object transfer instead of having to mess around with temporary files like may remotes do, for example...
+
+maybe that makes no sense at all, i don't know. :) --[[anarcat]]
+"""]]
diff --git a/doc/devblog/day_427__free_p2p.mdwn b/doc/devblog/day_427__free_p2p.mdwn
new file mode 100644
index 0000000000..7c727587b1
--- /dev/null
+++ b/doc/devblog/day_427__free_p2p.mdwn
@@ -0,0 +1,51 @@
+For a Haskell programmer, and day where a big thing is implemented
+without the least scrap of code that touches the IO monad is a good day.
+And this was a good day for me!
+
+Implemented the p2p protocol for tor hidden services. Its needs are somewhat
+similar to the external special remote protocol, but the two protocols are
+not fully overlapping with one-another. Rather than try to unify them, and
+so complicate both cases, I prefer to reuse as much code as possible between
+separate protocol implementations. The generating and parsing of messages
+is largely shared between them. I let the new p2p protocol otherwise
+develop in its own direction.
+
+But, I *do* want to make this p2p protocol reusable for other types of p2p
+networks than tor hidden services. This was an opportunity to use the Free
+monad, which I'd never used before. It worked out great, letting me write
+monadic code to handle requests and responses in the protocol, that reads
+the content of files and resumes transfers and so on, all independent
+of any concrete implementation.
+
+The whole implementation of the protocol only needed 74 lines of monadic code.
+It helped that I was able to factor out functions like this one, that is used
+both for handling a download, and by the remote when an upload is sent to it:
+
+	receiveContent :: Key -> Offset -> Len -> Proto Bool
+	receiveContent key offset len = do
+	        content <- receiveBytes len
+	        ok <- writeKeyFile key offset content
+	        sendMessage $ if ok then SUCCESS else FAILURE
+	        return ok
+
+To get transcripts of the protocol in action, the Free monad can be evaluated
+purely, providing the other side of the conversation:
+
+	ghci> putStrLn $ protoDump $ runPure (put (fromJust $ file2key "WORM--foo")) [PUT_FROM (Offset 10), SUCCESS]
+	> PUT WORM--foo
+	< PUT-FROM 10
+	> DATA 90
+	> bytes
+	< SUCCESS
+	result: True
+
+	ghci> putStrLn $ protoDump $ runPure (serve (toUUID "myuuid")) [GET (Offset 0) (fromJust $ file2key "WORM--foo")]
+	< GET 0 WORM--foo
+	> PROTO-ERROR must AUTH first
+	result: ()
+
+Am very happy with all this pure code and that I'm finally using Free monads.
+Next I need to get down the the dirty business of wiring this up to
+actual IO actions, and an actual network connection.
+
+Today's work was sponsored by Jake Vosloo on Patreon.
diff --git a/doc/devblog/day_428-429__git_push_to_hiddden_service.mdwn b/doc/devblog/day_428-429__git_push_to_hiddden_service.mdwn
new file mode 100644
index 0000000000..ec1bf12fd4
--- /dev/null
+++ b/doc/devblog/day_428-429__git_push_to_hiddden_service.mdwn
@@ -0,0 +1,31 @@
+The `tor` branch is coming along nicely.
+
+This weekend, I continued working on the P2P protocol, implementing
+it for network sockets, and extending it to support connecting up
+git-send-pack/git-receive-pack. 
+
+There was a bit of a detour when I split the Free monad into two separate
+ones, one for Net operations and the other for Local filesystem operations.
+
+This weekend's work was sponsored by Thomas Hochstein on Patreon.
+
+----
+
+Today, implemented a `git-remote-tor-annex` command that git will
+use for tor-annex:: urls,  and made `git annex remotedaemon`
+serve the tor hidden service.
+
+Now I have git push/pull working to the hidden service, for example:
+
+	git pull tor-annex::eeaytkuhaupbarfi.onion:47651
+
+That works very well, but does not yet check that the user is authorized
+to use the repo, beyond knowing the onion address. And currently
+it only works in git-annex repos; with some tweaks it should
+also work in plain git repos.
+
+Next, I need to teach git-annex how to access tor-annex remotes.
+And after that, an interface in the webapp for setting them up and
+connecting them together.
+
+Today's work was sponsored by Josh Taylor on Patreon.
diff --git a/doc/devblog/day_42__repair_milestone.mdwn b/doc/devblog/day_42__repair_milestone.mdwn
new file mode 100644
index 0000000000..61a1bea279
--- /dev/null
+++ b/doc/devblog/day_42__repair_milestone.mdwn
@@ -0,0 +1,35 @@
+The webapp now fully handles repairing damage to the repository.
+
+Along with all the git repository repair stuff already built, I added
+additional repairs of the git-annex branch and git-annex's index file.
+That was pretty easy actually, since git-annex already handles merging
+git-annex branches that can sometimes be quite out of date. So when git repo
+repair has to throw away recent changes to the git-annex branch, it just
+effectively becomes out of date. Added a `git annex fsck --fast` run to
+ensure that the git-annex branch reflects the current state of the
+repository.
+
+When the webapp runs a repair, it first stops the assistant from committing
+new files. Once the repair is done, that's started back up, and it runs a
+startup scan, which is just what is needed in this sitation; it will add
+any new files, as well as any old files that the git repository damange
+caused to be removed from the index.
+
+Also made `git annex repair` run the git repository repair code, 
+for those with a more command-line bent. It can be used in non-git-annex
+repos too!
+
+----
+
+So, I'm nearly ready to wrap up working on disaster recovery. Lots has been
+accomplished this month. And I have put off making a release for entirely
+too long!
+
+The big missing piece is repair of git remotes located on removable drive.
+I may make a release before adding that, but removable drives are probably
+where git repository corruption is most likely to occur, so I certainly
+need to add that.
+
+----
+
+Today's work was sponsored by Scott Robinson.
diff --git a/doc/devblog/day_430__tor_socket_problem.mdwn b/doc/devblog/day_430__tor_socket_problem.mdwn
new file mode 100644
index 0000000000..7e7c8d1bd0
--- /dev/null
+++ b/doc/devblog/day_430__tor_socket_problem.mdwn
@@ -0,0 +1,13 @@
+Debian's tor daemon is very locked down in the directories it can read
+from, and so I've had a hard time finding a place to put the unix socket
+file for git-annex's tor hidden service. Painful details in 
+. At least for now, I'm putting it under
+/etc/tor/, which is probably a FHS violation, but seems to be the only
+option that doesn't involve a lot of added complexity.
+
+--- 
+
+The Windows autobuilder is moving, since
+[NEST](http://nest-initiative.org/) is shutting down the server it has been
+using. Yury Zaytsev has set up a new Windows autobuilder, hosted at
+Dartmouth College this time.
diff --git a/doc/devblog/day_431__p2p_linking.mdwn b/doc/devblog/day_431__p2p_linking.mdwn
new file mode 100644
index 0000000000..1e53ffefc3
--- /dev/null
+++ b/doc/devblog/day_431__p2p_linking.mdwn
@@ -0,0 +1,27 @@
+Today I finished the second-to-last big missing peice for tor hidden service
+remotes. Networks of these remotes are P2P networks, and there needs to be
+a way for peers to find one-another, and to authenticate with one-another.
+The `git annex p2p` command sets up links between peers in such a network.
+
+So far it has only a basic interface that sets up a one way link between
+two peers. In the first repository, run `git annex p2p --gen-address`.
+That outputs a long address. In the second repository, run
+`git annex p2p --link peer1`, and paste the address into it. That sets up a
+git remote named "peer1" that connects back to the first repository over tor.
+
+That is a one-directional link, while a bi-directional link would be
+much more convenient to have between peers. Worse, the address can be reused by
+anyone who sees it, to link into the repository. And, the address is far
+too long to communicate in any way except for pasting it.
+
+So I want to improve that later. What I'd really like to have is an
+interface that displays a one-time-use phrase of five to ten words, that
+can be read over the phone or across the room. Exchange phrases with a
+friend, and get your repositories securely linked together with tor.
+
+But, `git annex p2p` is good enough for now. I can move on to the final
+keystone of the tor support, which is file transfer over tor.
+That should, fingers crossed, be relatively easy, and the `tor` branch is
+close to mergeable now.
+
+Today's work was sponsored by Riku Voipio.
diff --git a/doc/devblog/day_431__p2p_linking/comment_1_1d5f809564c25e765f82594af8e174ab._comment b/doc/devblog/day_431__p2p_linking/comment_1_1d5f809564c25e765f82594af8e174ab._comment
new file mode 100644
index 0000000000..9eceb71edb
--- /dev/null
+++ b/doc/devblog/day_431__p2p_linking/comment_1_1d5f809564c25e765f82594af8e174ab._comment
@@ -0,0 +1,49 @@
+[[!comment format=mdwn
+ username="https://anarc.at/openid/"
+ nickname="anarcat"
+ avatar="http://cdn.libravatar.org/avatar/b36dcf65657dd36128161355d8920a99503def9461c1bb212410980fe6f07125"
+ subject="magic wormhole"
+ date="2016-11-30T22:16:19Z"
+ content="""
+> What I'd really like to have is an interface that displays a
+> one-time-use phrase of five to ten words, that can be read over the
+> phone or across the room. Exchange phrases with a friend, and get
+> your repositories securely linked together with tor.
+
+I already mentionned the project in [[design/assistant/telehash/]],
+but [magic-wormhole](https://github.com/warner/magic-wormhole) does
+exactly that:
+
+    % wormhole send README.md
+    Sending 7924 byte file named 'README.md'
+    On the other computer, please run: wormhole receive
+    Wormhole code is: 7-crossover-clockwork
+    
+    Sending (<-10.0.1.43:58988)..
+    100%|=========================| 7.92K/7.92K [00:00<00:00, 6.02MB/s]
+    File sent.. waiting for confirmation
+    Confirmation received. Transfer complete.
+
+Receiver:
+
+    % wormhole receive
+    Enter receive wormhole code: 7-crossover-clockwork
+    Receiving file (7924 bytes) into: README.md
+    ok? (y/n): y
+    Receiving (->tcp:10.0.1.43:58986)..
+    100%|===========================| 7.92K/7.92K [00:00<00:00, 120KB/s]
+    Received file written to README.md
+
+While that example shows a file transfer, arbitrary data can be
+transfered this way. There's a documented protocol, and it's not
+completely peer-to-peer: there are relay servers to deal with NAT'd
+machines. But the [PAKE
+protocol](https://en.wikipedia.org/wiki/Password-authenticated_key_agreement)
+(basically SPAKE2) could be a good inspiration here.
+
+Otherwise, I must say that, as a user, I don't mind copy-pasting a
+hidden service string (if that's what it's about): i can do that over
+a secure medium (email + OpenPGP or IM + OTR) easily... But I
+understand it can be difficult to do for new users.
+
+"""]]
diff --git a/doc/devblog/day_432-433__almost_there.mdwn b/doc/devblog/day_432-433__almost_there.mdwn
new file mode 100644
index 0000000000..b41ce3f701
--- /dev/null
+++ b/doc/devblog/day_432-433__almost_there.mdwn
@@ -0,0 +1,13 @@
+Friday and today were spent implementing both sides of the P2P protocol for
+git-annex content transfers.
+
+There were some tricky cases to deal with. For example, when a file is being
+sent from a direct mode repository, or v6 annex.thin repository, the
+content of the file can change as it's being transferred. Including being
+appended to or truncated. Had to find a way to deal with that, to avoid
+breaking the protocol by not sending the indicated number of bytes of data.
+
+It all seems to be done now, but it's not been tested at all, and there are
+probably some bugs to find. (And progress info is not wired up yet.)
+
+Today's work was sponsored by Trenton Cronholm on Patreon.
diff --git a/doc/devblog/day_434__it_works.mdwn b/doc/devblog/day_434__it_works.mdwn
new file mode 100644
index 0000000000..75d096b31c
--- /dev/null
+++ b/doc/devblog/day_434__it_works.mdwn
@@ -0,0 +1,27 @@
+Git annex transfers over Tor worked correctly the first time I tried them
+today. I had been expecting protocol implementation bugs, so this was a
+nice surprise!
+
+Of course there were some bugs to fix. I had forgotten to add UUID
+discovery to `git annex p2p --link`. And, resuming interrupted transfers
+was buggy.
+
+Spent some time adding progress updates to the Tor remote. I was curious to
+see what speed transfers would run. Speed will of course vary depending on
+the Tor relays being used, but this example with a 100 mb file is not bad:
+
+	copy big4 (to peer1...) 
+	62%          1.5MB/s 24s
+
+There are still a couple of [[known bugs|todo/tor]], 
+but I've merged the `tor` branch into `master` already.
+
+----
+
+Alpernebbi has built a GUI for editing git-annex metadata.
+Something I always wanted!  
+[[Read about it here|tips/a_gui_for_metadata_operations]]
+
+----
+
+Today's work was sponsored by Ethan Aubin.
diff --git a/doc/devblog/day_435-436_post_tor_merge.mdwn b/doc/devblog/day_435-436_post_tor_merge.mdwn
new file mode 100644
index 0000000000..2f05e02521
--- /dev/null
+++ b/doc/devblog/day_435-436_post_tor_merge.mdwn
@@ -0,0 +1,20 @@
+More improvements to tor support. Yesterday, debugged a reversion that
+broke push/pull over tor, and made actual useful error messages be
+displayed when there were problems. Also fixed a memory leak, although I
+fixed it by reorganizing code and could not figure out quite why it happened,
+other than that the ghc runtime was not managing to be as lazy as I would
+expect. 
+
+Today, added git ref change notification to the
+P2P protocol, and made the remotedaemon automatically fetch changes from
+tor remotes. So, it should work to use the assistant to keep
+repositories in sync over tor. I have not tried it yet, and linking over tor
+still needs to be done at the command line, so it's not really ready for
+webapp users yet.
+
+Also fixed a denial of service attack in git-annex-shell and git-annex when
+talking to a remote git-annex-shell. It was possible to feed either a large
+amount of data when they tried to read a line of data, and summon the OOM
+killer. Next release will be expedited some because of that.
+
+Today's work was sponsored by Thomas Hochstein on Patreon.
diff --git a/doc/devblog/day_437__catching_up.mdwn b/doc/devblog/day_437__catching_up.mdwn
new file mode 100644
index 0000000000..1deb54459c
--- /dev/null
+++ b/doc/devblog/day_437__catching_up.mdwn
@@ -0,0 +1,20 @@
+Quite a backlog developed in the couple of weeks I was concentrating on tor
+support. I've taken a first pass through it and fixed the most pressing
+issues now. 
+
+Most important was an ugly memory corruption problem in the GHC runtime
+system that may have led to data corruption when using git-annex with Linux
+kernels older than 4.5. All the Linux standalone builds of git-annex have
+been updated to fix that issue.
+
+Today dealt with several more things, including fixing a buggy timestamp
+issue with `metadata --batch`, reverting the ssh ServerAliveInterval
+setting (broke on too many systems with old ssh or complicated ssh
+configurations), making batch input not be rejected when it can't be decoded
+as UTF-8, and more.
+
+Also, spent some time learning a little bit about Magic Wormhole and SPAKE,
+as a way to exchange tor remote addresses. Using Magic Wormhole for that
+seems like a reasonable plan. I did file a couple bugs on it which will
+need to get fixed, and then using it is mostly a question of whether it's
+easy enough to install that git-annex can rely on it.
diff --git a/doc/devblog/day_438__bi-directional_p2p_links.mdwn b/doc/devblog/day_438__bi-directional_p2p_links.mdwn
new file mode 100644
index 0000000000..abcbed1229
--- /dev/null
+++ b/doc/devblog/day_438__bi-directional_p2p_links.mdwn
@@ -0,0 +1,6 @@
+Improved `git annex p2p --link` to create a bi-directional link
+automatically. Bi-directional links are desirable more often than not, so
+it's the default behavior.
+
+Also continued thinking about using magic wormhole for communicating
+p2p addresses for pairing. And filed some more bugs on magic wormhole.
diff --git a/doc/devblog/day_439__wormhole_pairing.mdwn b/doc/devblog/day_439__wormhole_pairing.mdwn
new file mode 100644
index 0000000000..cc988a2db3
--- /dev/null
+++ b/doc/devblog/day_439__wormhole_pairing.mdwn
@@ -0,0 +1,51 @@
+`git annex p2p --pair` implemented, using Magic Wormhole codes
+that have to be exchanged between the repositories being paired.
+
+It looks like this, with the same thing being done at the same time
+in the other repository.
+
+	joey@elephant:~/tmp/bench3/a>git annex p2p --pair
+	p2p pair peer1 (using Magic Wormhole) 
+	
+	This repository's pairing code is: 1-select-bluebird
+	
+	Enter the other repository's pairing code: (here I entered 8-fascinate-sawdust) 
+	Exchanging pairing data...
+	Successfully exchanged pairing data. Connecting to peer1...
+	ok
+
+And just that simply, the two repositories find one another, 
+Tor onion addresses and authentication data is exchanged, and a git remote
+is set up connecting via Tor.
+
+	joey@elephant:~/tmp/bench3/a>git annex sync peer1
+	commit  
+	ok
+	pull peer1 
+	warning: no common commits
+	remote: Counting objects: 5, done.
+	remote: Compressing objects: 100% (3/3), done.
+	remote: Total 5 (delta 0), reused 0 (delta 0)
+	Unpacking objects: 100% (5/5), done.
+	From tor-annex::5vkpoyz723otbmzo.onion:61900
+	 * [new branch]      git-annex  -> peer1/git-annex
+
+Very pleased with this, and also the whole thing worked on the very first
+try!
+
+It might be slightly annoying to have to exchange two codes during pairing.
+It would be possible to make this work with only one code. I decided to go
+with two codes, even though it's only marginally more secure than one,
+mostly for UI reasons. The pairing interface and 
+[[instructions for using it|tips/peer_to_peer_network_with_tor]] is simplfied
+by being symmetric.
+
+(I also decided to revert the work I did on Friday to make `p2p --link`
+set up a bidirectional link. Better to keep `--link` the simplest possible
+primitive, and pairing makes bidirectional links more easily.)
+
+Next: Some more testing of this and the Tor hidden services, a webapp UI
+for P2P peering, and then finally removing XMPP support. I hope to finish
+that by New Years.
+
+Today's work was sponsored by Jake Vosloo on Patreon.
diff --git a/doc/devblog/day_43__bugfix_day.mdwn b/doc/devblog/day_43__bugfix_day.mdwn
new file mode 100644
index 0000000000..ad150f5120
--- /dev/null
+++ b/doc/devblog/day_43__bugfix_day.mdwn
@@ -0,0 +1,26 @@
+Got well caught up on bug fixes and traffic. Backlog is down to 40.
+
+Made the assistant wait for a few seconds before doing the startup
+scan when it's autostarted, since the desktop is often busy starting
+up at that same time.
+
+Fixed an ugly bug with chunked webdav and directory special remotes
+that caused it to not write a "chunkcount" file when storing data,
+so it didn't think the data was present later. I was able to make it
+recover nicely from that mistake, by probing for what chunks are actually
+present.
+
+Several people turn out to have had problems with `git annex sync` not
+working because receive.denyNonFastForwards is enabled. I made the webapp
+not enable it when setting up a ssh repository, and I made `git annex sync`
+print out a hint about this when it's failed to push. (I don't think this
+problem affects the assistant's own syncing.)
+
+Made the assistant try to repair a damaged git repository without
+prompting. It will only prompt when it fails to fetch all the lost
+objects from remotes.
+
+Glad to see that others have managed to 
+[get git-annex to build on Max OS X 10.9](http://git-annex.branchable.com/bugs/git_annex_doesn__39__t_work_in_Max_OS_X_10.9/#comment-8e8ee5e50506a6fde029d236f4809df8). 
+Now I just need someone to offer up a ssh account on that OS, and I could
+set up an autobuilder for it.
diff --git a/doc/devblog/day_440__holidaze.mdwn b/doc/devblog/day_440__holidaze.mdwn
new file mode 100644
index 0000000000..e002d0d157
--- /dev/null
+++ b/doc/devblog/day_440__holidaze.mdwn
@@ -0,0 +1,11 @@
+Have been working on some improvements to `git annex enable-tor`. Made it
+su to root, using any su-like program that's available. And made it test
+the hidden service it sets up, and wait until it's propigated the the Tor
+directory authorities. The webapp will need these features, so I thought I
+might as well add them at the command-line level.
+
+Also some messing about with locale and encoding issues. About most of
+which the less said the better. One significant thing is that I've made the
+filesystem encoding be used for all IO by git-annex, rather than needing to
+explicitly enable it for each file and process. So, there should be much
+less bother with encoding problems going forward.
diff --git a/doc/devblog/day_441__webapp_wormhole_pairing.mdwn b/doc/devblog/day_441__webapp_wormhole_pairing.mdwn
new file mode 100644
index 0000000000..b27723b9dc
--- /dev/null
+++ b/doc/devblog/day_441__webapp_wormhole_pairing.mdwn
@@ -0,0 +1,18 @@
+Added the Magic Wormhole UI to the webapp for pairing Tor remotes.
+This replaces the XMPP pairing UI when using "Share with a friend" 
+and "Share with your other devices" in the webapp.
+
+[[!img assistant/share_with_a_friend_walkthrough/enabletor.png]]
+
+[[!img assistant/share_with_a_friend_walkthrough/wormholepairing.png]]
+
+I have not been able to fully test it yet, and it's part of the `no-xmpp`
+branch until I can.
+
+It's been a while since I worked on the webapp. It was not as hard as I
+remembered to deal with Yesod. The inversion of control involved in coding
+for the web is as annoying as I remembered.
+
+----
+
+Today's work was sponsored by Riku Voipio.
diff --git a/doc/devblog/day_442__xmpp_removal.mdwn b/doc/devblog/day_442__xmpp_removal.mdwn
new file mode 100644
index 0000000000..d66764314f
--- /dev/null
+++ b/doc/devblog/day_442__xmpp_removal.mdwn
@@ -0,0 +1,15 @@
+The webapp's wormhole pairing almost worked perfectly on the first test.
+Turned out the remotedaemon was not noticing that the tor hidden service
+got enabled. After fixing that, it worked perfectly!
+
+So, I've merged that feature, and removed XMPP support from the assistant
+at the same time. If all goes well, the autobuilds will be updated soon,
+and it'll be released in time for new year's.
+
+Anyone who's been using XMPP to keep repositories in sync will need to
+either switch to Tor, or could add a remote on a ssh server to sync by
+instead. See
+
+for the pointy-clicky way to do it, and
+ for
+the command-line way.
diff --git a/doc/devblog/day_442__xmpp_removal/comment_1_22ff11b84f855ab5e6de6dcf2553a050._comment b/doc/devblog/day_442__xmpp_removal/comment_1_22ff11b84f855ab5e6de6dcf2553a050._comment
new file mode 100644
index 0000000000..ccb079ed59
--- /dev/null
+++ b/doc/devblog/day_442__xmpp_removal/comment_1_22ff11b84f855ab5e6de6dcf2553a050._comment
@@ -0,0 +1,35 @@
+[[!comment format=mdwn
+ username="Apichat"
+ avatar="http://cdn.libravatar.org/avatar/af7b465d7cd067b9c5acd365bdef92ac"
+ subject="Maintain the XMPP function ?"
+ date="2017-01-05T14:00:34Z"
+ content="""
+Is it realy a good idea to remove now the XMPP function ? While XMPP is a good tool to manage social relations and is getting improve ? [As you say it is an easy way to share with friends](http://joeyh.name/blog/entry/p2p_dreams/)
+
+The lake of use of the XMPP function in Git-annex is may be due to the fact that Git-annex assistant and the XMPP feature are difficult to use on Windows - so 90% of social relations can't exist.
+
+Can't you a least maintain the XMPP function ?
+ 
+XMPP is getting improve a lot those years :
+
+  * Host services, finally we can easily use a host service up to date and with our own domain name :
+    * [[https://account.conversations.im/domain]]
+    * [[https://support-en.mailbox.org/knowledge-base/article/introduction-to-jabber]]
+    * [[https://gultsch.de/compliance_ranked.html]]
+
+  * Protocol Specifications 
+    * [The State of Mobile XMPP in 2016](https://gultsch.de/xmpp_2016.html)
+
+  * Client software :
+    * [[https://conversations.im]]
+    * [[https://jitsi.org]]
+    * [[https://conversejs.org]]
+    * [Tor Messenger](https://trac.torproject.org/projects/tor/wiki/doc/TorMessenger)
+
+  * Server software :
+    * [[https://prosody.im]]
+    * [[https://www.ejabberd.im]]
+    * [Mongoose IM platform](https://www.erlang-solutions.com/products/mongooseim.html)
+
+
+"""]]
diff --git a/doc/devblog/day_442__xmpp_removal/comment_2_eaf64ce3daae790bb27d21887886d73b._comment b/doc/devblog/day_442__xmpp_removal/comment_2_eaf64ce3daae790bb27d21887886d73b._comment
new file mode 100644
index 0000000000..5a5f327bb8
--- /dev/null
+++ b/doc/devblog/day_442__xmpp_removal/comment_2_eaf64ce3daae790bb27d21887886d73b._comment
@@ -0,0 +1,18 @@
+[[!comment format=mdwn
+ username="Apichat"
+ avatar="http://cdn.libravatar.org/avatar/af7b465d7cd067b9c5acd365bdef92ac"
+ subject="OMEMO to encrypt content end to end"
+ date="2017-01-28T11:05:29Z"
+ content="""
+One of your argument for removing XMPP is 
+
+> _it was not entirely secure, since the XMPP server saw the contents of git pushes without encryption_
+
+But since one years there is a new way to encrypt XMPP content end to end : [OMEMO Multi-End Message and Object Encryption](https://conversations.im/omemo/)
+
+> _OMEMO is an XMPP Extension Protocol (XEP) for secure multi-client end-to-end encryption. It is an open standard based on a Double Ratchet and PEP which can be freely used and implemented by anyone. The protocol has been audited by a third party._
+
+Here are other informations : [[https://gultsch.de/talks/omemo.html]]
+
+You can see here the xmpp clients which are using it : [[https://omemo.top]]
+"""]]
diff --git a/doc/devblog/day_443__yes_it_has_been_a_while.mdwn b/doc/devblog/day_443__yes_it_has_been_a_while.mdwn
new file mode 100644
index 0000000000..0af4201257
--- /dev/null
+++ b/doc/devblog/day_443__yes_it_has_been_a_while.mdwn
@@ -0,0 +1,30 @@
+First day working on git-annex in over a month. I've been away preparing
+for and giving two talks at Linux Conf Australia and then recovering from
+conference flu, but am now raring to dive back into git-annex development!
+
+The backlog stood at over 300 messages this morning, and is down to 274 now. So
+still lots of catching up to do. But nothing seems to have blown up badly in my
+absence. The [[tips/antipatterns]] page was a nice development while I was
+away, listing some ways people sometimes find to shoot their feet.
+Read and responded to lots of questions, including one user who mentioned
+a scientific use case: "We are exploring use of git-annex to manage the
+large boundary conditions used within our weather model."
+
+The main bit of coding today was adding a new `git annex config` command.
+This is fairly similar to `git config`, but it stores the settings in the
+git-annex branch, so they're visible in all clones of the repo (aka
+"global"). Not every
+setting will be configurable this way (that would be too expensive, and too
+foot-shooty), but starting with annex.autocommit I plan to enable
+selected settings that make sense to be able to set globally. 
+If you've wanted to be able to configure some part of git-annex in all
+clones of a repository, suggestions are welcome in the
+[todo item about this](https://git-annex.branchable.com/todo/wishlist__58___per-repository_autocommit__61__false/)
+
+`git annex vicfg` can also be used to edit the global settings, and I also
+made it able to edit the global `git annex numcopies` setting which was
+omitted before. There's no real reason to have a separate `git annex
+numcopies` command now, since `git annex config` could configure
+global annex.numcopies.. but it's probably not worth changing that.
+
+Today's work was sponsored by Trenton Cronholm on Patreon.
diff --git a/doc/devblog/day_444__memory_leak_with_a_cold.mdwn b/doc/devblog/day_444__memory_leak_with_a_cold.mdwn
new file mode 100644
index 0000000000..bed714e95d
--- /dev/null
+++ b/doc/devblog/day_444__memory_leak_with_a_cold.mdwn
@@ -0,0 +1,21 @@
+Spent rather too long today tracking down a memory leak in `git annex unused`. 
+Actually, it was three memory leaks; one of them was a reversion introduced
+while otherwise improving a function to not be partial. Another only
+happened in very rare circumstances. The third, which took several more
+hours staring at the code, turned out to simply be an unnecessary use of an
+accumulating list. Feel like I should have seen that one sooner, but then I
+am under the weather and was running profiles in a daze for several hours..
+In the end, `git-annex unused` went from needing 1 gb of memory to 150 mb
+in my big repo.
+
+One advantage to all the profiling though, was I noticed that the `split`
+function was allocating a lot of memory, and seemed generally ineficient. This
+has to do with it splitting on a string; splitting on a single character
+can run twice as fast and churn the GC quite a bit less, so I wrote up a
+specialized version of that, and it's used extensively in git-annex now, so
+it may run up to 50% faster in some cases. Seems like haskell libraries
+with a `split` function should perhaps use the more optimal version
+when splitting on a single character, and I'm going to file bugs to that
+effect.
+
+Today's work was sponsored by Jake Vosloo on Patreon.
diff --git a/doc/devblog/day_445__configs.mdwn b/doc/devblog/day_445__configs.mdwn
new file mode 100644
index 0000000000..148954d507
--- /dev/null
+++ b/doc/devblog/day_445__configs.mdwn
@@ -0,0 +1,11 @@
+Finished the repository-clone-global configuration settings I started
+adding on Monday. Came up with a nice type-driven way to make sure
+that configuration is loaded when needed, and only loaded once.
+Then it was easy to make annex.autocommit be configurable by git-annex
+config. Also added a new annex.synccontent configuration, which can also be
+set by git-annex config.
+
+Also resolved a tricky situation with providing an appid to magic wormhole.
+It will happen on a flag day in 2021. I've marked my calendar..
+
+Today's work was sponsored by Thomas Hochstein on Patreon.
diff --git a/doc/devblog/day_446__quiet_progress.mdwn b/doc/devblog/day_446__quiet_progress.mdwn
new file mode 100644
index 0000000000..87c7c64605
--- /dev/null
+++ b/doc/devblog/day_446__quiet_progress.mdwn
@@ -0,0 +1,14 @@
+Last week I only had energy to work most of each day on git-annex,
+or to blog about it. I chose quiet work. The changelog did grow
+a good amount.
+
+Today, fixed some autobuilder problems, and I am gearing up to add another
+autobuild, targeting arm boxes with older linux kernels, since I got a
+chance to upgrade the arm autobuilder's disk this weekend.
+
+Also, some work on the S3 special remote, and worked around a bug in
+sqlite's handling of umask.
+
+Backlog is down to 243 messages.
+
+Today's work was sponsored by Trenton Cronholm on Patreon.
diff --git a/doc/devblog/day_447__bug_class.mdwn b/doc/devblog/day_447__bug_class.mdwn
new file mode 100644
index 0000000000..c7b3d386c5
--- /dev/null
+++ b/doc/devblog/day_447__bug_class.mdwn
@@ -0,0 +1,48 @@
+When you see a command like "ssh somehost rm -f file", you probably don't
+think that consumes stdin. After all, the `rm -f` doesn't. But, `ssh`
+can pass stdin over the network even if it's not being consumed, and it
+turns out git-annex was bitten by this.
+
+That bug made `git-annex-checkpresentkey --batch` with remote accessed
+over ssh not see all the batch-mode input that was passed into it, because
+ssh sometimes consumed some of it.
+
+Shell scripts using git-annex could also be impacted by the bug, for
+example:
+
+	#!/bin/sh
+	find . -type l -atime 100 | \
+		while read file; do
+			echo "gonna drop $file that has not been used in a while"
+			git annex drop "$file"
+		done
+
+Depending on what remotes `git annex drop` talks to, it might consume
+parts of the output of find.
+
+I've fixed this in git-annex now (using `ssh -n` when running commands
+that are not fed some stdin of their own), but this seems like a class of
+bug that could impact lots of programs that run ssh.
+
+----
+
+I've been thinking about [[simpler_setup_for_remote_worktree_update_on_push]].
+
+One nice way to make a remote update its worktree on push is available
+in recent-ish gits, receive.denyCurrentBranch=updateInstead. That could
+already be used with `git annex sync`, but it hid any error messages
+when pushing the master branch to the remote (since that push fails with
+a large error message in default configurations). Found a way to make
+the error message be displayed when the remote's receive.denyCurrentBranch
+does not have the default configuration.
+
+The remaining problem is that direct mode and adjusted branch remotes
+won't get their works trees updated even when configured that way. I am
+thinking about adding a post-update hook to support those.
+
+----
+
+Also continuing to bring up the ancient kernel arm autobuilder. It's running
+its first build now.
+
+Today's work was sponsored by Riku Voipio.
diff --git a/doc/devblog/day_448__git_push_to_update_remote.mdwn b/doc/devblog/day_448__git_push_to_update_remote.mdwn
new file mode 100644
index 0000000000..edcccc2b7a
--- /dev/null
+++ b/doc/devblog/day_448__git_push_to_update_remote.mdwn
@@ -0,0 +1,24 @@
+Today was all about writing
+[[tips/making_a_remote_repo_update_when_changes_are_pushed_to_it]].
+
+That's a fairly simple page, because I added workarounds for all the
+complexity of making it work in direct mode repos, adjusted branches, and
+repos on filesystems not supporting executable git hooks. Basically,
+the user should be able to set the standard
+receive.denyCurrentBranch=updateInstead configuration on a remote, and 
+then `git push` or `git annex sync` should update that remote's working tree.
+
+There are a couple of unhandled cases; `git push` to a remote on a
+filesystem like FAT won't update it, and `git annex sync` will only update
+it if it's local, not accessed over ssh. Also, the emulation of git's
+updateInstead behavior is not perfect for direct mode repos and adjusted
+branches.
+
+Still, it's good enough that most users should find it meets
+their needs, I hope. How to set this kind of thing up is a fairly common
+FAQ, and this makes it much simpler.
+
+(Oh yeah, the first ancient kernel arm build is *still*
+running. May finish before tomorrow.)
+
+Today's work was sponsored by Jake Vosloo on Patreon.
diff --git a/doc/devblog/day_449__SHA1_break_day.mdwn b/doc/devblog/day_449__SHA1_break_day.mdwn
new file mode 100644
index 0000000000..df70856029
--- /dev/null
+++ b/doc/devblog/day_449__SHA1_break_day.mdwn
@@ -0,0 +1,21 @@
+[The first SHA1 collision](https://shattered.io/) was announced today,
+produced by an identical-prefix collision attack.
+
+After looking into it all day, it does not appear to impact git's security
+immediately, except for targeted attacks against specific projects by
+very wealthy attackers. But we're well past the time when it seemed ok that git
+uses SHA1. If this gets improved into a chosen-prefix collision
+attack, git will start to be rather insecure.
+
+Projects that store binary files in git, that might be worth $100k for an
+attacker to backdoor **should** be concerned by the SHA1 collisions.
+A good example of such a project is
+.
+
+Using git-annex (with a suitable backend like SHA256) and signed commits
+together is a good way to secure such repositories. 
+
+Update 12:25 am: However, there are some ways to embed SHA1-colliding data
+in the names of git-annex keys. That makes git-annex with signed
+commits be no more secure than git with signed commits. I am working
+to fix git-annex to not use keys that have such problems.
diff --git a/doc/devblog/day_44__automatic_removable_drive_repair.mdwn b/doc/devblog/day_44__automatic_removable_drive_repair.mdwn
new file mode 100644
index 0000000000..444cf5f2e5
--- /dev/null
+++ b/doc/devblog/day_44__automatic_removable_drive_repair.mdwn
@@ -0,0 +1,16 @@
+Finally got the assistant to repair git repositories on removable drives,
+or other local repos. Mostly this happens entirely automatically, whatever
+data in the git repo on the drive has been corrupted can just be copied
+to it from `~/annex/.git`.
+
+And, the assistant will launch a git fsck of such a repo whenever it fails
+to sync with it, so the user does not even need to schedule periodic fscks.
+Although it's still a good idea, since some git repository problems don't
+prevent syncing from happening.
+
+Watching git annex heal problems like this is quite cool!
+
+One thing I had to defer till later is repairing corrupted gcrypt
+repositories. I don't see a way to do it without deleting all the objects
+in the gcrypt repository, and re-pushing everything. And even doing that
+is tricky, since the `gcrypt-id` needs to stay the same.
diff --git a/doc/devblog/day_450__hardening_against_SHA_attacks.mdwn b/doc/devblog/day_450__hardening_against_SHA_attacks.mdwn
new file mode 100644
index 0000000000..5e419dea7a
--- /dev/null
+++ b/doc/devblog/day_450__hardening_against_SHA_attacks.mdwn
@@ -0,0 +1,13 @@
+Yesterday I said that a git-annex repository using signed commits and SHA2
+backend would be secure from SHA1 collision attacks. Then I noticed that
+there were two ways to embed the necessary collision generation data inside
+git-annex key names. I've fixed both of them today, and cannot find any
+other ways to embed collision generation data in between a signed commit
+and the annexed files.
+
+I also have a design for a way to configure git-annex to expect to see only
+keys using secure hash backends, which will make it easier to work with
+repositories that want to use signed commits and SHA2. Planning to implement
+that tomorrow.
+
+[[todo/sha1_collision_embedding_in_git-annex_keys]] has the details.
diff --git a/doc/devblog/day_451__annex.securehashesonly.mdwn b/doc/devblog/day_451__annex.securehashesonly.mdwn
new file mode 100644
index 0000000000..d0407d0e8e
--- /dev/null
+++ b/doc/devblog/day_451__annex.securehashesonly.mdwn
@@ -0,0 +1,16 @@
+The new annex.securehashesonly config setting prevents annexed content
+that does not use a cryptographically secure hash from being downloaded or
+otherwise added to a repository.
+
+Using that and signed commits prevents SHA1 collisions from causing
+problems with annexed files. See [[tips/using_signed_git_commits]] for
+details about how to use it, and why I believe it makes git-annex
+safe despite git's vulnerability to SHA1 collisions in general.
+
+If you are using git-annex to publish binary files in a repository,
+you should follow the instructions in [[tips/using_signed_git_commits]].
+
+If you're using git to publish binary files, you can improve the security
+of your repository by switchingto git-annex and signed commits.
+
+Today's work was sponsored by Riku Voipio.
diff --git a/doc/devblog/day_452__GIT_SSH.mdwn b/doc/devblog/day_452__GIT_SSH.mdwn
new file mode 100644
index 0000000000..a1ce5cfade
--- /dev/null
+++ b/doc/devblog/day_452__GIT_SSH.mdwn
@@ -0,0 +1,17 @@
+Found a bug in git-annex-shell where verbose messages would sometimes make
+it output things git-annex didn't expect.
+
+While fixing that, I wanted to add a test case, but the test suite actually
+does not test git-annex-shell at all. It would need to ssh, which test
+suites should not do. So, I took a detour..
+
+Support for `GIT_SSH` and `GIT_SSH_COMMAND` has been requested before for
+various reasons. So I implemented that, which took 4 hours. (With one
+little possible compatibility caveat, since git-annex needs to pass the -n
+parameter to ssh sometimes, and git's interface doesn't allow for such a
+parameter.)
+
+Now the test suite can use those environment variables to make mock ssh
+remotes be accessed using local `sh` instead of `ssh`.
+
+Today's work was sponsored by Trenton Cronholm on Patreon.
diff --git a/doc/devblog/day_453_release_prep.mdwn b/doc/devblog/day_453_release_prep.mdwn
new file mode 100644
index 0000000000..8a0a90dadf
--- /dev/null
+++ b/doc/devblog/day_453_release_prep.mdwn
@@ -0,0 +1,13 @@
+Preparing for a release tomorrow. Yury fixed the Windows autobuilder over
+the weekend. The OSX autobuilder was broken by my changes Friday, which
+turned out to have a simple bug that took quite a long time to chase down.
+
+Also added `git annex sync --content-of=path` to sync the contents of files
+in a path, rather than in the whole work tree as `--content` does. I would
+have rather made this be `--content=path` but optparse-applicative 
+[does not support](https://github.com/pcapriotti/optparse-applicative/issues/243)
+options that can be either boolean or have a string value. Really,
+I'd rather `git annex sync path` do it, but that would be ambiguous
+with the remote name parameter.
+
+Today's work was sponsored by Jake Vosloo on Patreon.
diff --git a/doc/devblog/day_454__multicast.mdwn b/doc/devblog/day_454__multicast.mdwn
new file mode 100644
index 0000000000..1972c5ead9
--- /dev/null
+++ b/doc/devblog/day_454__multicast.mdwn
@@ -0,0 +1,15 @@
+Earlier this week I had the opportunity to sit in on a workshop at MIT where
+students were taught how to use git-annex as part of a stack of tools for
+reproducible scientific data research. That was great!
+
+One thing we noticed there is, it can be hard to distribute files to such a
+class; downloading them individually wastes network bandwidth. Today, I
+added [[git annex multicast|git-annex-multicast]] which uses `uftp`
+to multicast files to other clones of a repository on a LAN.
+An "easy" 500 lines of code and 7 hour job.
+
+There is encryption and authentication, but the key management for this
+turned out to be simple, since the public key fingerprints can be stored on
+the git-annex branch, and easily synced around that way. So, I expect
+this should be not hard to use in a classroom setting such as the one
+I was in earlier this week.
diff --git a/doc/devblog/day_454__multicast/comment_1_137ea34cd11004513a03f24955719756._comment b/doc/devblog/day_454__multicast/comment_1_137ea34cd11004513a03f24955719756._comment
new file mode 100644
index 0000000000..5f842eec46
--- /dev/null
+++ b/doc/devblog/day_454__multicast/comment_1_137ea34cd11004513a03f24955719756._comment
@@ -0,0 +1,38 @@
+[[!comment format=mdwn
+ username="anarcat"
+ avatar="http://cdn.libravatar.org/avatar/4ad594c1e13211c1ad9edb81ce5110b7"
+ subject="point to point?"
+ date="2017-03-31T13:08:59Z"
+ content="""
+I wonder: could this be used to send blobs to arbitrary hosts as well? I've been looking at the problem of sharing blobs without sharing the git repository (in [[tips/semi-synchronized_remotes]]), and while we have the tor/wormhole and SSH remotes that can be (ab)used for this purpose, uftp seems like a much better fit, feature-wise.
+
+When you run: `git annex multicast --gen-address; git annex sync`, does the `sync` command exchange git refs over multicast?
+
+When you run: `git annex multicast --send`, will that work for all git-annex blobs, or just the ones that are in the local checkout?
+
+Looking at the [server usage](http://uftp-multicast.sourceforge.net/server_usage.txt), i see that multicast is the default, but that you can also unicast to specific hosts:
+
+       Or in unicast mode:
+
+	    uftp -M host_or_ip file
+
+       Where host_or_ip is the hostname or unicast IP address of the  host  to
+       send to.
+
+       To send only to certain hosts:
+
+	    uftp -H client_id_1,client_id_2,client_id_3 file_to_send
+
+       or:
+
+	    uftp -H @file_containing_list_of_clients file_to_send
+
+Could this be used to send to arbitrary, non-local hosts on the internet?
+
+And how about tor support? Could this simplify the encrypted setup?
+
+This amazing feature just brings up more questions for me, but i'm really glad to see this come. It certainly addresses parts of [[todo/Bittorrent-like_features]], and is probably the first remote that supports bandwidth optimizations for multiple downloads, or rather, in this case, uploads. :)
+
+Thanks so much for your hard work!
+The feature set of git-annex never ceases to amaze me - new features just keep on coming, this is great!
+"""]]
diff --git a/doc/devblog/day_454__multicast/comment_2_09908606fc362cbee516999cd696f718._comment b/doc/devblog/day_454__multicast/comment_2_09908606fc362cbee516999cd696f718._comment
new file mode 100644
index 0000000000..b7b5279498
--- /dev/null
+++ b/doc/devblog/day_454__multicast/comment_2_09908606fc362cbee516999cd696f718._comment
@@ -0,0 +1,19 @@
+[[!comment format=mdwn
+ username="joey"
+ subject="""comment 2"""
+ date="2017-04-05T15:20:20Z"
+ content="""
+Multicast is only being used to send git-annex objects around, not git
+objects. There's assumed to be some way to sync git repositories, which is
+how the encryption keys for uftp are distributed.
+
+`git annex multicast --send` operates on files in the working tree.
+It would be possible to make it support `--all`.
+
+I'm not sure if uftp can send outside the local LAN.
+
+It would certianly be possible to have a special remote backed by uftp
+that thus only sends to a single host. Since multicast does not send
+to any particular remote, it did not make sense to implement it
+as a special remote.
+"""]]
diff --git a/doc/devblog/day_455__semi-synchronized.mdwn b/doc/devblog/day_455__semi-synchronized.mdwn
new file mode 100644
index 0000000000..2a53f06bb5
--- /dev/null
+++ b/doc/devblog/day_455__semi-synchronized.mdwn
@@ -0,0 +1,9 @@
+Catching up on some recent backlog after a trip and post-trip flu.
+
+Anarcat wrote up an anlysis of [[tips/semi-synchronized_remotes]], and
+based on that I implemented `remote..annex-push` and
+`remote..annex-pull`
+
+Also fixed the Windows build.
+
+Today's work was sponsored by Thomas Hochstein on Patreon.
diff --git a/doc/devblog/day_456__digging_in.mdwn b/doc/devblog/day_456__digging_in.mdwn
new file mode 100644
index 0000000000..024e0b90b8
--- /dev/null
+++ b/doc/devblog/day_456__digging_in.mdwn
@@ -0,0 +1,16 @@
+Digging in to some of the meatier backlog today. Backlog down to 225.
+
+A lot of fixes around using `git annex enableremote` to add new gpg keys to
+a gcrypt special remote. 
+
+Had to make git-annex's use of `GIT_SSH`/`GIT_SSH_COMMAND`
+contingent on `GIT_ANNEX_USE_GIT_SSH=1` being set. Unfortunate, but
+difference from git made at least one existing use of that environment
+variable break, and so it will need to be whitelisted in places where
+git-annex should use it.
+
+Added support for `git annex add --update`
+
+----
+
+Today's work was sponsored by Trenton Cronholm on Patreon.
diff --git a/doc/devblog/day_457__improved_ssh_password_prompting.mdwn b/doc/devblog/day_457__improved_ssh_password_prompting.mdwn
new file mode 100644
index 0000000000..df38304e4e
--- /dev/null
+++ b/doc/devblog/day_457__improved_ssh_password_prompting.mdwn
@@ -0,0 +1,17 @@
+After a month away building [debug-me](https://debug-me.branchable.com/)
+I'm back working on git-annex. I hope debug-me will be useful for debugging
+git-annex in some situations BTW.
+
+Pushed a release yesterday that was mostly changes from back in March.
+It also updated the git bundled with git-annex to fix the recent git-shell
+security hole.
+
+After work on Monday and today, I am caught up with all the recent month's
+backlog, but still have 230 old backlogged messages to get to.
+
+The first consequental thing I got back to was improving ssh password
+prompting when git-annex is running concurrently with -J. It used to start
+up several ssh's at the same time, so connection caching didn't kick in,
+and there could be a bunch of ssh password prompts at the same time. Now
+there will never be more than one ssh password prompt at once, and only one
+prompt per host. (As long as connection caching is enabled.)
diff --git a/doc/devblog/day_458__adeiu_MissingH.mdwn b/doc/devblog/day_458__adeiu_MissingH.mdwn
new file mode 100644
index 0000000000..5d6a7870e8
--- /dev/null
+++ b/doc/devblog/day_458__adeiu_MissingH.mdwn
@@ -0,0 +1,18 @@
+Wasn't planning to, but spent the day making git-annex not depend on the
+MissingH library. This has been a long-term goal, as MissingH pulls in
+several other libraries and is not modern or principled.
+
+The first part was to using cryptonite for MD5 calculation. While
+converting to the form git-annex uses to make hash directories involved
+some math, this did make git-annex garbage-collect less, and
+probably made it faster.
+
+Then I had to write my own progress meter display, since git-annex was
+using MissingH's display. That was fairly simple (73 LoC), and let me
+make it more efficient and tuned for the git-annex use case. As a bonus, it
+got progress displays when transferring files of unknown sizes, which
+wasn't done before.
+
+MissingH was handy training wheels when I was coming over from perl,
+but it's been training wheels on some old cars in the middle of a 
+500 car train for a while, so glad that's over.
diff --git a/doc/devblog/day_459__git_bug.mdwn b/doc/devblog/day_459__git_bug.mdwn
new file mode 100644
index 0000000000..3ec5f167fa
--- /dev/null
+++ b/doc/devblog/day_459__git_bug.mdwn
@@ -0,0 +1,29 @@
+Seems that the recent release of git 2.13.0 contained a reversion that
+broke `git-annex sync` in an adjusted branch. After bisecting git,
+producing a minimal test case, and reporting that to the git developers, I
+was able to work around it in git-annex. The workaround is normally not
+expensive, but could be when a repository has thousands of unpacked refs.
+So I hope this will get fixed in git and I can remove the workaround.
+
+I think I will hurry up the next git-annex release somewhat to get the
+workaround out. It's not a super bad bug, but it does make the test suite
+fail and I've already had 3 people report the problem.
+
+Seems it would be good to have an integration test that runs git-annex's
+test suite against new commits to git. Ævar Arnfjörð Bjarmason has stepped
+up to add that to git's test suite.
+
+----
+
+Also, I dealt with some fallout from removing MissingH; a exponential
+speed blowup in a directory traversal function.
+
+Getting back to the ssh password prompting with -J I was working on last
+week, dealt with the ssh prompt interfering with the regional display
+manager. The fix is not perfect, but good enough; before ssh prompts (and
+only if it prompts), git-annex temporarily clears the regional display.
+Then the display gets redrawn under the ssh output. That needed some
+changes to concurrent-output (which I did over the weekend), so will only
+be done when it's built with a new enough version. A better approach would
+be to save and restore the cursor position, but the ansi-terminal library
+does not yet support that.
diff --git a/doc/devblog/day_45__command_line.mdwn b/doc/devblog/day_45__command_line.mdwn
new file mode 100644
index 0000000000..fa6a7ffc74
--- /dev/null
+++ b/doc/devblog/day_45__command_line.mdwn
@@ -0,0 +1,9 @@
+All command line stuff today..
+
+Added --want-get and --want-drop, which can be used to test preferred content settings
+of a repository. For example `git annex find --in . --want-drop` will list the same
+files that `git annex drop --auto` would try to drop. (Also renamed `git annex content`
+to `git annex wanted`.)
+
+Finally laid to rest problems with `git annex unannex` when multiple files point to the
+same key. It's a lot slower, but I'll stop getting bug reports about that.
diff --git a/doc/devblog/day_460__move_--to_here.mdwn b/doc/devblog/day_460__move_--to_here.mdwn
new file mode 100644
index 0000000000..64cb002853
--- /dev/null
+++ b/doc/devblog/day_460__move_--to_here.mdwn
@@ -0,0 +1,3 @@
+From the why-did-I-never-think-of-that-before department:
+`git annex move --to=here` implemented today. Useful if you want to
+move a file from whatever remotes might contain it to the local repository.
diff --git a/doc/devblog/day_461__shell_completions.mdwn b/doc/devblog/day_461__shell_completions.mdwn
new file mode 100644
index 0000000000..e2bd652fcb
--- /dev/null
+++ b/doc/devblog/day_461__shell_completions.mdwn
@@ -0,0 +1,8 @@
+A new version of optparse-applicative supports zsh and fish shell
+completions. Got that integrated into git-annex, although it will be a
+while until most builds are updated to that version of the library.
+Also, re-submitted my old patch from 2015 to make "git annex"
+always tab complete in bash.
+
+Enough other small fixes and improvements have accumulated that a release
+is due soon..
diff --git a/doc/devblog/day_462__the_feature_youve_all_been_waiting_for.mdwn b/doc/devblog/day_462__the_feature_youve_all_been_waiting_for.mdwn
new file mode 100644
index 0000000000..dfa4080941
--- /dev/null
+++ b/doc/devblog/day_462__the_feature_youve_all_been_waiting_for.mdwn
@@ -0,0 +1,12 @@
+Have been working on a design for [[design/exporting_trees_to_special_remotes]].
+As well as being handy for publishing scientific data sets out of git-annex
+repositories, that covers long-requested features like
+[[todo/dumb, unsafe, human-readable_backend]].
+
+I had not been optimistic about such requests, which seemed half-baked, but
+Yoh came up with idea of exporting a git treeish, and remembering the last
+exported treeish so a subsequent export can be done incrementally, and can
+fully sync the exported tree.
+
+Please take a look at the design if you've wanted to use git-annex for some
+sort of tree export before, and see if it meets your needs.
diff --git a/doc/devblog/day_462__the_feature_youve_all_been_waiting_for/comment_1_6b13defcdf76691049b9bcd7ec675da6._comment b/doc/devblog/day_462__the_feature_youve_all_been_waiting_for/comment_1_6b13defcdf76691049b9bcd7ec675da6._comment
new file mode 100644
index 0000000000..a4638c14bf
--- /dev/null
+++ b/doc/devblog/day_462__the_feature_youve_all_been_waiting_for/comment_1_6b13defcdf76691049b9bcd7ec675da6._comment
@@ -0,0 +1,9 @@
+[[!comment format=mdwn
+ username="https://openid.stackexchange.com/user/3ee5cf54-f022-4a71-8666-3c2b5ee231dd"
+ nickname="Anthony DeRobertis"
+ avatar="http://cdn.libravatar.org/avatar/1007bfece547e9f2d86fafa142cd240a62456c37f104a9d96ba9db5bf18e1934"
+ subject="Link broken?"
+ date="2017-07-14T07:37:40Z"
+ content="""
+The link isn't working for me, but found it at [[http://git-annex.branchable.com/design/exporting_trees_to_special_remotes/]]
+"""]]
diff --git a/doc/devblog/day_463-465__back_buggy_external_special_remotes.mdwn b/doc/devblog/day_463-465__back_buggy_external_special_remotes.mdwn
new file mode 100644
index 0000000000..f374a51cb6
--- /dev/null
+++ b/doc/devblog/day_463-465__back_buggy_external_special_remotes.mdwn
@@ -0,0 +1,30 @@
+I'm back working on git-annex after a month away! It's good to be back, and
+I've made some decent improvements this week. 
+
+New features include a `GIT_ANNEX_VECTOR_CLOCK` environment variable that
+may be useful for those using git-annex in a HIPPA compliance setting,
+where timestamps are verboten (but verifying full compliance is up to
+you!), and remote.name.annex-ignore-command and
+remote.name.annex-sync-command configurations that let shell commands be
+run to vary what remotes are used depending on eg, what network it's on.
+
+Also, I took a look at the external special remote protocol, and noticed
+two problems with it. First, keys with spaces in their names can't be used
+with it. This only affects the WORM backend, and it seems no one has ever
+run into the problem. Rather than complicate the implementation of external
+special remotes, I decided to deprecate having spaces in key names. Which
+is just asking for trouble anyway. So now there's a nice error message, and
+a migration path.
+
+The other problem was that the external special remote documentation
+incorrectly said that a filename parameter never contained spaces. But in
+fact, there are situations where it can. This was not a problem with the
+protocol, but only with its documentation, and potentially with the
+implementation of some special remotes. So, I've spent some time today
+auditing every git-annex special remote that I know about. A few scripts
+that are bundled with git-annex were buggy and got fixed, and I
+filed bugs on 9 other external special remotes. A few did already get it
+right!
+
+Today's work was sponsored by Trenton Cronholm on 
+[Patreon](https://patreon.com/joeyh/)
diff --git a/doc/devblog/day_466__export_prototype.mdwn b/doc/devblog/day_466__export_prototype.mdwn
new file mode 100644
index 0000000000..cdc1926f83
--- /dev/null
+++ b/doc/devblog/day_466__export_prototype.mdwn
@@ -0,0 +1,6 @@
+Put together a prototype of `git annex export` in the "export" branch.
+Exporting to a directory special remote is basically working, but this is
+only the beginning.
+
+Today's work was sponsored by Jake Vosloo on
+[Patreon](https://patreon.com/joeyh/)
diff --git a/doc/devblog/day_467__export_progress.mdwn b/doc/devblog/day_467__export_progress.mdwn
new file mode 100644
index 0000000000..d5d32d5446
--- /dev/null
+++ b/doc/devblog/day_467__export_progress.mdwn
@@ -0,0 +1,11 @@
+Good progress on `git annex export` today. Changing the exported tree now
+works and is done efficiently. Resuming an export is working. Even
+detecting and resolving export conflicts should work (have not tested it).
+The necessary information about the export is recorded in the git-annex
+branch, including grafting in the exported tree there.
+
+There are some known problems when the tree that is exported contains
+multiple files with the same content. And git-annex is not yet able
+to download exported files from a special remote. Handling both of those
+needs way to get from keys to exported filenames. So, I plan to
+populate a sqlite database with that information next.
diff --git a/doc/devblog/day_467__firming_up_export.mdwn b/doc/devblog/day_467__firming_up_export.mdwn
new file mode 100644
index 0000000000..6b5e193ca0
--- /dev/null
+++ b/doc/devblog/day_467__firming_up_export.mdwn
@@ -0,0 +1,27 @@
+More work on `git annex export`. Made `initremote exporttree=yes` be
+required to enable exporting to a special remote. Added a sqlite database
+to keep track of what files have been exported. That let me fix the known
+problems with exporting multiple files that have the same content. 
+
+The same database lets `git annex get` (etc) download content from exports.
+Since an export is not a key/value store, git-annex has to do more
+verification of content downloaded from an export. Some types of keys,
+that are not based on checksums (eg WORM and URL), 
+cannot be downloaded from an export. And, git-annex will never trust
+an export to retain the content of a key, since some other tree could
+be exported over it at any time.
+
+With `git annex get` working from exports, it might be nice to also support
+`git annex copy --to export` for exporting specific files to them. However,
+that needs information that is not currently stored in the sqlite database
+until the export has already completed. One way it could work is for `git
+annex export --fast treeish --to export` to put all the filenames in the
+database but not export anything, and then `git annex copy --to export` (or
+even `git annex sync --content` to send the contents). I don't know if this
+complication is worth it.
+
+Otherwise, the export feature is fairly close to being complete now.
+Still need to make renames be handled efficiently, and add support for
+exporting to more special remotes.
+
+Today's work was supported by the NSF-funded DataLad project.
diff --git a/doc/devblog/day_467__firming_up_export/comment_1_382fac60766340481acbcb8c05b70f42._comment b/doc/devblog/day_467__firming_up_export/comment_1_382fac60766340481acbcb8c05b70f42._comment
new file mode 100644
index 0000000000..09277b2f2b
--- /dev/null
+++ b/doc/devblog/day_467__firming_up_export/comment_1_382fac60766340481acbcb8c05b70f42._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="EskildHustvedt"
+ avatar="http://cdn.libravatar.org/avatar/0be1310904ded29624b9edb4824d451b"
+ subject="Partial exports"
+ date="2017-09-05T09:16:26Z"
+ content="""
+For what it's worth, partial exports (being able to only copy certain files to an export) would be very useful for me. My main usecase is exporting to my android phone (which has an sshd in termux that I use) from my desktop. I've got some large repos where having it all on my phone isn't possible, but it would be very useful to use git-annex to upload partials (right now I'm just using plain-old-rsync for that).
+"""]]
diff --git a/doc/devblog/day_468__export_renames.mdwn b/doc/devblog/day_468__export_renames.mdwn
new file mode 100644
index 0000000000..e40ebac907
--- /dev/null
+++ b/doc/devblog/day_468__export_renames.mdwn
@@ -0,0 +1,23 @@
+I knew that making `git annex export` handle renames efficiently would take
+a whole day somehow.
+
+Indeed, thinking it over, it is a seriously super hairy thing. Renames can swap
+contents between two or more files, and so temp files are needed. It has to
+handle cleaning up temp files after interrupted exports, which may be
+resumed with the same or a different tree. It also has to recover from
+export conflicts, which could cause the wrong content to be renamed to a file.
+
+I think I've thought through everything and found a way to deal with it all.
+Here's how it looks in operation swapping two files:
+
+	git annex export master --to dir
+	rename bar -> .git-annex-tmp-content-SHA256E-s30--472b01bf6234c98ce03d1386483ae578f6e58033974a1363da2606f9fa0e222a ok
+	rename foo -> .git-annex-tmp-content-SHA256E-s4--b5bb9d8014a0f9b1d61e21e796d78dccdf1352f23cd32812f4850b878ae4944c ok
+	rename .git-annex-tmp-content-SHA256E-s4--b5bb9d8014a0f9b1d61e21e796d78dccdf1352f23cd32812f4850b878ae4944c -> bar ok
+	rename .git-annex-tmp-content-SHA256E-s30--472b01bf6234c98ce03d1386483ae578f6e58033974a1363da2606f9fa0e222a -> foo ok
+	(recording state in git...)
+
+The export todo list is only getting longer.. But the branch may
+be close to being merged.
+
+Today's work was supported by the NSF-funded DataLad project.
diff --git a/doc/devblog/day_468__export_renames/comment_1_4c37e6f9ac1e1571495b0d355c8af9ff._comment b/doc/devblog/day_468__export_renames/comment_1_4c37e6f9ac1e1571495b0d355c8af9ff._comment
new file mode 100644
index 0000000000..b9bc07ee43
--- /dev/null
+++ b/doc/devblog/day_468__export_renames/comment_1_4c37e6f9ac1e1571495b0d355c8af9ff._comment
@@ -0,0 +1,11 @@
+[[!comment format=mdwn
+ username="anthony@ad39673d230d75cbfd19d2757d754030049c7673"
+ nickname="anthony"
+ avatar="http://cdn.libravatar.org/avatar/05b48b72766177b3b0a6ff4afdb70790"
+ subject="comment 1"
+ date="2017-09-06T22:01:57Z"
+ content="""
+Looking forward to the export support! 
+
+Just curious (and I apologize in advance for any nightmares induced) did you consider the case of four files, A–D, where (content-wise) A=B and C=D and the change is to swap A/C and B/D? That'd potentially be an issue, since four files want to share two temporary names. Unless of course it's all only done with pairwise swaps.
+"""]]
diff --git a/doc/devblog/day_468__export_renames/comment_2_7fbbc5beb80acf32eff617ec704d5466._comment b/doc/devblog/day_468__export_renames/comment_2_7fbbc5beb80acf32eff617ec704d5466._comment
new file mode 100644
index 0000000000..1dfdfe6dee
--- /dev/null
+++ b/doc/devblog/day_468__export_renames/comment_2_7fbbc5beb80acf32eff617ec704d5466._comment
@@ -0,0 +1,26 @@
+[[!comment format=mdwn
+ username="joey"
+ subject="""comment 2"""
+ date="2017-09-07T19:54:41Z"
+ content="""
+Did not consider such a case. However, that's closely related to exporting
+files with same content being inefficient. There's a move
+operation but no copy operation. I might add a copy operation eventually,
+unsure.
+
+If a copy operation is added, then that rename case can be handled more
+efficiently, by moving to the single temp file and copying. Although it
+might still involve the special remote doing more work than strictly
+necessary depending on how it implements copy.
+
+Anyway, if the user is exporting copys of files, they're probably going to
+care more about that being somewhat more efficient than about renames of
+pairs of those copies being optimally efficient..
+
+Handling it fully optimally, with only one temp file per key,
+would require analizing the change and finding pairs of renames
+that swap filenames and handling each pair in turn. I suppose that
+is doable, just needs a better data structure than I have now.
+I've added a note to my todo list and the design document, but
+no promises.
+"""]]
diff --git a/doc/devblog/day_469__export_merged.mdwn b/doc/devblog/day_469__export_merged.mdwn
new file mode 100644
index 0000000000..5cd54d878d
--- /dev/null
+++ b/doc/devblog/day_469__export_merged.mdwn
@@ -0,0 +1,8 @@
+I've merged the `export` branch, after fixing most of the remaining known
+warts, and testing clean-up from interrupted exports and export conflicts.
+
+The main thing remaining to be done is adding the new commands to the
+external special remote interface, and adding export support to S3, webdav,
+and rsync special remotes.
+
+Today's work was supported by the NSF-funded DataLad project.
diff --git a/doc/devblog/day_46__wrapping_up_the_month.mdwn b/doc/devblog/day_46__wrapping_up_the_month.mdwn
new file mode 100644
index 0000000000..c6045178c5
--- /dev/null
+++ b/doc/devblog/day_46__wrapping_up_the_month.mdwn
@@ -0,0 +1,18 @@
+Spent today reviewing my [[plans_for_the_month|assistant/disaster_recovery]]
+and filling in a couple of missing peices.
+
+Noticed that I had forgotten to make repository repair clean up any stale
+git locks, despite writing that code at the beginning of the month, and
+added that in.
+
+Made the webapp notice when a repository that is being used does not have
+any consistency checks configured, and encourage the user to set up checks.
+This happens when the assistant is started (for the local repository),
+and when removable drives containing repositories are plugged in. If the
+reminders are annoying, they can be disabled with a couple clicks.
+
+And I think that just about wraps up the month. (If I get a chance, I would
+still like to add recovery of git-remote-gcrypt encrypted git repositories.)
+
+My [[design/roadmap]] has next month dedicated to user-driven features
+and polishing and bugfixing.
diff --git a/doc/devblog/day_470__export_to_external_and_S3.mdwn b/doc/devblog/day_470__export_to_external_and_S3.mdwn
new file mode 100644
index 0000000000..df27b06ed7
--- /dev/null
+++ b/doc/devblog/day_470__export_to_external_and_S3.mdwn
@@ -0,0 +1,12 @@
+Got `git annex export` working to external special remotes. Each external
+special remote will need some modifications to allow exporting. Exporting
+to some things doesn't make sense, but often there's a way to browse a tree
+of files stored on the special remote and so export is worth supporting.
+Now would be a good time to contact the author of your favorite special
+remote about supporting export..
+
+Also had time to get `git annex export` working to S3. The tip 
+[[tips/publishing_your_files_to_the_public]] had a clumsy method for
+publishing files via S3 before, and is now quite simple!
+
+Today's work was supported by the NSF-funded DataLad project.
diff --git a/doc/devblog/day_471__export_to_more_remotes.mdwn b/doc/devblog/day_471__export_to_more_remotes.mdwn
new file mode 100644
index 0000000000..bf8c1e66a8
--- /dev/null
+++ b/doc/devblog/day_471__export_to_more_remotes.mdwn
@@ -0,0 +1,13 @@
+Got `git annex export` working to webdav and rsync special remotes. Tested
+exporting to the Internet Archive via S3, and to box.com via webdav. Both
+had little weirdnesses in their handling of the protocols, which were
+worked around, and it's quite nice to be able to export trees to those
+services, as well as Amazon S3.
+
+Also added connection caching for exports, so S3 and webdav exports only
+make one http connection, instead of one per file.
+
+Had to change the format of `git-annex:export.log`; the old format didn't
+take into account that a repository can export to several different remotes.
+
+Today's work was supported by the NSF-funded DataLad project.
diff --git a/doc/devblog/day_472__removing_empty_directories.mdwn b/doc/devblog/day_472__removing_empty_directories.mdwn
new file mode 100644
index 0000000000..25b36fd867
--- /dev/null
+++ b/doc/devblog/day_472__removing_empty_directories.mdwn
@@ -0,0 +1,20 @@
+After doing some fine-tuning of webdav export on Wednesday, I noticed a
+problem: There seems to be no way in the webdav spec to delete a collection
+(directory) only when it's empty like `rmdir(2)` does. 
+It would be possible to check the contents of the collection before
+deleting it, but that's complex (involving XML parsing) and race-prone.
+
+So, I decided to add a remote method to delete a directory, and make
+git-annex keep track of when a directory in an export is empty, and delete
+it. While it does complicate the design some to need to do this, that seems
+better than complicating the implementation of remotes like webdav. And
+some remotes may not have a `rmdir(2)` equivalent or a way to check if a
+directory is empty.
+
+Spent most of today implementing that, including some rather hairy
+eSQL to maintain a table of exported directories.
+
+Still not quite done with export..
+
+Today's work was sponsored by Trenton Cronholm on
+[Patreon](https://patreon.com/joeyh)
diff --git a/doc/devblog/day_473__distributed_use_of_exports.mdwn b/doc/devblog/day_473__distributed_use_of_exports.mdwn
new file mode 100644
index 0000000000..3113292460
--- /dev/null
+++ b/doc/devblog/day_473__distributed_use_of_exports.mdwn
@@ -0,0 +1,17 @@
+The tricky part of the `git annex export` feature has definitely been
+making it work in a distributed situation. The last details of that seem to
+have been worked out now.
+
+I had to remove support for dropping individual files from export remotes.
+The [[design|design/exporting_trees_to_special_remotes]] has a scenario
+where that makes distributed use of exports inconsistent.
+
+But, what is working now is `git annex export` being run in one repository,
+and then another repository, after syncing, can get files from the export.
+
+Most of export is done now. The only thing I'm thinking about adding is
+a way to make an export track a branch. so `git annex sync` can update
+the export.
+
+Today's work was sponsored by Jake Vosloo on
+[Patreon](https://patreon.com/joeyh/).
diff --git a/doc/devblog/day_474__tracking_exports.mdwn b/doc/devblog/day_474__tracking_exports.mdwn
new file mode 100644
index 0000000000..7b06027c45
--- /dev/null
+++ b/doc/devblog/day_474__tracking_exports.mdwn
@@ -0,0 +1,26 @@
+Built a way to make an export track changes to a branch.
+
+	git annex export --tracking master --to myexport
+
+That ties in nicely with `git annex sync`:
+
+	joey@darkstar:~/tmp/bench/a> echo hello > foo
+	joey@darkstar:~/tmp/bench/a> git annex add
+	add foo ok
+	joey@darkstar:~/tmp/bench/a> git annex sync --content
+	commit  
+	[master 8edbc6f] git-annex in joey@darkstar:~/tmp/bench/a
+	 1 file changed, 1 insertion(+)
+	 create mode 120000 foo
+	ok
+	export myexport foo 
+	ok                          
+	joey@darkstar:~/tmp/bench/a> git mv foo bar
+	joey@darkstar:~/tmp/bench/a> git annex sync --content
+	commit  
+	[master 3ab6e73] git-annex in joey@darkstar:~/tmp/bench/a
+	 1 file changed, 0 insertions(+), 0 deletions(-)
+	 rename foo => bar (100%)
+	ok
+	rename myexport foo -> .git-annex-tmp-content-SHA256E-s6--5891b5b522d5df086d0ff0b110fbd9d21bb4fc7163af34d08286a2e846f6be03 ok
+	rename myexport .git-annex-tmp-content-SHA256E-s6--5891b5b522d5df086d0ff0b110fbd9d21bb4fc7163af34d08286a2e846f6be03 -> bar ok
diff --git a/doc/devblog/day_475__assistant_exports.mdwn b/doc/devblog/day_475__assistant_exports.mdwn
new file mode 100644
index 0000000000..789fa0728b
--- /dev/null
+++ b/doc/devblog/day_475__assistant_exports.mdwn
@@ -0,0 +1,4 @@
+Got the git-annex assistant updating exports. The assistant is pretty
+complicated, so that took most of the day.
+
+Exports are done!
diff --git a/doc/devblog/day_476__third_time_lucky.mdwn b/doc/devblog/day_476__third_time_lucky.mdwn
new file mode 100644
index 0000000000..1a07b1b036
--- /dev/null
+++ b/doc/devblog/day_476__third_time_lucky.mdwn
@@ -0,0 +1,29 @@
+There's been a lot of little bug fixes and improvements going on
+in the ... oops ... almost a month since I last updated the devblog.
+Including a release of git-annex on the 3rd, and another release
+that's almost ready to go now. Just have not had the energy to blog about
+it all.
+
+Anyway, today I spent way too long fixing a minor wart. When multiple
+annexed files have the same content, transferring them with concurrency
+enabled could make it complain that "transfer already in progress".
+Which is better than transferring the same content twice, but it did make
+there seem to be a failure.
+
+I implemented two and a half different fixes for that. The first half a fix
+was too intrusive and I couldn't get it to work. Then came a fix that
+avoided the problem pretty cleanly, except it actually led to worse
+behavior, because it would sometimes transfer the same content twice, and
+needed non-obvious tweaks here and there to prevent that. Finally, around
+an hour ago, having actually given up unhappily for the day, I realized a
+much better way to fix it, that was minimally intrusive and works
+perfectly. 
+
+So it goes.. I'd say "concurrency is hard", but it's more that big complex
+code bases can make things that seem simple not really that simple.
+Yesterday I had a much easier time fixing a related problem with `git annex
+add -J`, which was really a lot hairier (involving a race condition and
+a lack of atomicity), but didn't cut across the code base in the same broad
+way.
+
+Today's work was supported by the NSF-funded DataLad project.
diff --git a/doc/devblog/day_477__windows_build_fixed.mdwn b/doc/devblog/day_477__windows_build_fixed.mdwn
new file mode 100644
index 0000000000..c957815a1e
--- /dev/null
+++ b/doc/devblog/day_477__windows_build_fixed.mdwn
@@ -0,0 +1,10 @@
+Got the Windows build fixed, with help from Yury. The toolchain had been
+broken for months. We switched to using stack, which should make the
+Windows build more reproducible and easy to manage.
+
+Unfortunately, there was a link problem, and I had to disable some FFI code
+that was needed to terminate processes on Windows. Until that gets fixed,
+restarting and stopping the assistant won't work right on Windows.
+
+Aaand: The EvilLinker is not needed any longer, so I was very happy to be
+able to delete that hack. \o/
diff --git a/doc/devblog/day_478__windows_the_blackhole.mdwn b/doc/devblog/day_478__windows_the_blackhole.mdwn
new file mode 100644
index 0000000000..f1c65dabc7
--- /dev/null
+++ b/doc/devblog/day_478__windows_the_blackhole.mdwn
@@ -0,0 +1,14 @@
+Good grief, I've spent another whole day on Windows porting issues.
+
+Tested the new windows builds. Got win32 patched with terminateProcessId,
+and got the windows build using that. Unfortunately this made stack not
+incrementally rebuild very well, so Windows builds 
+got very slow. ([Bug report](https://github.com/commercialhaskell/stack/issues/2960))
+
+Found and fixed an ugly bug with annex link generation on Windows,
+which probably dates back to this spring. Code was comparing `"C:\"` with
+`"C:/"` and thinking the drives were different, argh. Still waiting on
+sloooow builds to test that.
+
+There will probably be an almost Windows-dedicated release tomorrow. Only
+"almost" because Sean T Parsons sent in some patches.
diff --git a/doc/devblog/day_47__fell_off_the_blogging_wagon.mdwn b/doc/devblog/day_47__fell_off_the_blogging_wagon.mdwn
new file mode 100644
index 0000000000..fd312a0d8d
--- /dev/null
+++ b/doc/devblog/day_47__fell_off_the_blogging_wagon.mdwn
@@ -0,0 +1,3 @@
+Low activity the past couple of days. Released a new version of git-annex
+yesterday. Today fixed three bugs (including a local pairing one that was
+pretty compicated) and worked on getting caught up with traffic.
diff --git a/doc/devblog/day_482__website_login_problem.mdwn b/doc/devblog/day_482__website_login_problem.mdwn
new file mode 100644
index 0000000000..65584ff8e3
--- /dev/null
+++ b/doc/devblog/day_482__website_login_problem.mdwn
@@ -0,0 +1,31 @@
+I noticed a large drop in bug reports and comments on the git-annex website
+over the holiday period of December. At first I thought this was just due
+to the holidays, even though holidays are often busy times for free
+software projects since lots of people have more time. But, traffic is
+still down this week, and several people emailed me about problems logging
+into the website.
+
+So, lacking much detail at all about what people were doing that didn't
+work, I've the past day and a some trying to guess at and reproduce
+the problem. And I think I have, ,
+and once reproduced it was of course easily fixed.
+
+If you tried to post something and got a login prompt instead of seeing it
+on the website, now would be a good time to post it again.
+
+If you still have login problems with the website (other than openid which
+has lot of broken providers and badly specified protocol and stuff), please
+get in touch and try to provide enough detail to reproduce the problem, cuz
+my guessing muscles are feeling sprained after this experience.
+
+In the meantime, there has still been git-annex development happening.
+I added a new `git annex inprogress` command over the holidays that 
+allows doing things like streaming videos while `git annex get` is still
+downloading them. Several fixes to problems with the switch to youtube-dl 
+are fixed, core.sharedRepository is handled better, and the cabal file's
+custom-setup stanza was added back after quite a lot of refactoring of
+library code.
+
+---
+
+Today's work was sponsored by an anonymous bitcoin donor.
diff --git a/doc/devblog/day_483__faster_start_with_removable_drives.mdwn b/doc/devblog/day_483__faster_start_with_removable_drives.mdwn
new file mode 100644
index 0000000000..dfbba15eff
--- /dev/null
+++ b/doc/devblog/day_483__faster_start_with_removable_drives.mdwn
@@ -0,0 +1,27 @@
+git-annex does a little bit of work at startup to learn about the git
+repository it's running in. That's been optimised some before, but
+not entirely eliminated; it's just too useful to have that information
+always available inside git-annex. But it turned out that it was doing more work
+than needed for many commands, by checking the git config of local remotes.
+Thas caused unncessary spin up of removable drives, or automount timeouts,
+or generally more work than needed when running commands like `git annex
+find` and even tab completing git-annex. That's fixed now, so it
+avoids checking the git config of remotes except when running commands that
+access some remote.
+
+There's also a new config setting, `remote..annex-checkuuid` that can
+be set to false to defer checking the uuid of local repositories until
+git-annex actually uses them. That can avoid even more spinup/automounts,
+but that config prevents git-annex from transparently handling the case
+where different removable drives get mounted to the same place at different
+times.
+
+Speaking of speed, I benchmarked linux kernel mitigation for the meltdown
+attack making `git status` 5% slower from a warm cache. It did not
+slow down `git annex find` or `git annex find --in remote` enough to be
+measured by my benchmark. I expect that git-annex commands that transfer
+data are bottlenecked on IO and won't be slowed down appreciably by the
+meltdown mitigation either.
+
+Today's work was sponsored by Jake Vosloo on
+[Patreon](https://patreon.com/joeyh/).
diff --git a/doc/devblog/day_484__special_remote_protocol_extensions.mdwn b/doc/devblog/day_484__special_remote_protocol_extensions.mdwn
new file mode 100644
index 0000000000..68140c6fee
--- /dev/null
+++ b/doc/devblog/day_484__special_remote_protocol_extensions.mdwn
@@ -0,0 +1,12 @@
+The 
+[[external special remote protocol|design/external_special_remote_protocol]]
+had extensibility built into it for messages git-annex sends, but not
+for messages that the remote sends back to git-annex. To fix this
+asymmetry, I've added a new EXTENSIONS to the protocol, which can be used
+to find out about what new protocol extensions are supported.
+
+There was the possibility that adding that might break some external
+special remote that hardcoded the intial protocol messages. So, I checked
+all of them that I know of, and all were ok, except for older versions of
+datalad, which we were able to deal with. If you have your own external
+special remote implementation, now would be a good time to check it.
diff --git a/doc/devblog/day_485__slow_and_steady.mdwn b/doc/devblog/day_485__slow_and_steady.mdwn
new file mode 100644
index 0000000000..22a011d99f
--- /dev/null
+++ b/doc/devblog/day_485__slow_and_steady.mdwn
@@ -0,0 +1,9 @@
+I've been recovering from some stuff over the past month, so
+progress lately has been slow, but still steadily progressing. Yesterday's
+release of git-annex had a month and a half of improvements, including JSON
+enhancements, adding extension support to the external special remote
+protocol, and making fsck warn about required content that's missing.
+
+Today I've been working on `git annex export` to rsync special remotes.
+
+Today's work was sponsored by Jake Vosloo [on Patreon](https://www.patreon.com/joeyh).
diff --git a/doc/devblog/day_486__time_to_ditch_rsync.mdwn b/doc/devblog/day_486__time_to_ditch_rsync.mdwn
new file mode 100644
index 0000000000..bd7cf0d4bb
--- /dev/null
+++ b/doc/devblog/day_486__time_to_ditch_rsync.mdwn
@@ -0,0 +1,14 @@
+I'm excited by this new design
+[[todo/accellerate_ssh_remotes_with_git-annex-shell_mass_protocol]].
+
+git-annex's use of rsync got transfers over ssh working quickly early on,
+but other than resuming interrupted transfers, using rsync doesn't really
+gain git-annex much, since annexed objects don't change over time. And
+rsync has always involved a certian amount of overhead that a custom
+protocol would avoid.
+
+It's especially handy that such a protocol was already developed for
+`git-annex p2p` when using tor. I've not heard of a lot of people using that
+feature (but maybe people who do have reason not to talk about it), but
+it's a good solid thing, implemented very generically with a free monad,
+and reusing it for git-annex-shell would be great.
diff --git a/doc/devblog/day_487__git-annex-shell_p2pstdio.mdwn b/doc/devblog/day_487__git-annex-shell_p2pstdio.mdwn
new file mode 100644
index 0000000000..2475df87e5
--- /dev/null
+++ b/doc/devblog/day_487__git-annex-shell_p2pstdio.mdwn
@@ -0,0 +1,38 @@
+It was rather easy to implement `git-annex-shell p2pstdio`, the P2P
+protocol implementation done for tor is quite generic and easy to adapt for
+this. 
+
+The only complication was that git-annex-shell has a readonly mode,
+so the protocol server needed modifications to support that.
+Well, there's also some innefficiency around unncessary verification
+of transferred content in some cases, which will probably need extensions
+to the P2P protocol later.
+
+Also wrote up some documentation of what the P2P protocol looks like,
+for anyone who might want to communiate with git-annex-shell using it,
+for some reason, and doesn't understand Haskell and free monads.
+[[design/P2P_protocol]]
+
+While comparing the code of the P2P server and git-annex-shell commands, I
+noticed that the P2P server didn't check inAnnex when locking content,
+while `git-annex-shell lockcontent` did check inAnnex. This turned out to
+be a ugly data loss bug involving direct mode repositories, where a
+modified direct mode file was allowed when locking a key in the repository.
+Turned out that the bug happened when locking a file over tor to drop it
+locally, but also when locking a file locally in a direct mode repository
+to allow dropping it from any remote. 
+
+Very glad I noticed that, and I've changed the API to prevent that class of
+bug. I feel this is not a severe data loss bug, because when a direct mode
+repository is involved, dropping from somewhere and then modifying the file
+in the direct mode repository can have the same effect of losing the old copy.
+The bug just made data loss happen when running the same operations in the
+other order.
+
+Next will be making git-annex use this new git-annex-shell feature
+when available.
+
+----
+
+Today's work was sponsored by Trenton Cronholm on
+[Patreon](https://patreon.com/joeyh)
diff --git a/doc/devblog/day_488__groundwork_for_using_p2pstdio.mdwn b/doc/devblog/day_488__groundwork_for_using_p2pstdio.mdwn
new file mode 100644
index 0000000000..e09da769ae
--- /dev/null
+++ b/doc/devblog/day_488__groundwork_for_using_p2pstdio.mdwn
@@ -0,0 +1,30 @@
+Spent most of the day laying groundwork for using git-annex-shell p2pstdio.
+Implemented pools of ssh connections to it, and added uuid verification.
+Then generalized code from the p2p remote so it can be reused in the git
+remote. The types got super hairy in there, but the code reuse level is
+excellent.
+
+Finally it was time to convert the first ssh remote method
+to use the P2P protocol. I chose key removal, since benchmarking it doesn't
+involve the size of annexed objects.
+
+Here's the P2P protocol in action over ssh:
+
+	[2018-03-08 17:02:47.688627136] chat: ssh ["localhost","-S",".git/annex/ssh/localhost","-o","ControlMaster=auto","-o","ControlPersist=yes","-T","git-annex-shell 'p2pstdio' '/~/tmp/bench/a' '--debug' 'da72c285-2615-4a67-828f-eaae4f42fc3d' --uuid db017fac-eb8f-42d9-9d09-2780b193cef1"]
+	[2018-03-08 17:02:47.901897195] P2P < AUTH-SUCCESS db017fac-eb8f-42d9-9d09-2780b193cef1
+	[2018-03-08 17:02:47.902025504] P2P > REMOVE SHA256E-s4--97b912eb4a61df5f806ca6239dde3e1a4f51ad20aced1642cbb83dc510a5fa6b
+	[2018-03-08 17:02:47.910074003] P2P < SUCCESS
+	[2018-03-08 17:02:47.914181701] P2P > REMOVE SHA256E-s4--6af2f5b785a8930f0bd3edc833e18fa191167ab0535ef359b19a1982a6984e96
+	[2018-03-08 17:02:47.918699806] P2P < SUCCESS
+
+For a benchmark, I set up a repository with 1000 annexed files,
+and cloned it from localhost, then ran `git annex drop --from origin`.
+
+before: 41 seconds  
+after: 10 seconds
+
+400% speedup for dropping is pretty great.. And when there's more latency
+than loopback has, the improvement should be more pronounced. 
+Will test it this evening over my satellite internet. :)
+
+Today's work was sponsored by Trenton Cronholm on [Patreon](https://patreon.com/joeyh/).
diff --git a/doc/devblog/day_489__zooming.mdwn b/doc/devblog/day_489__zooming.mdwn
new file mode 100644
index 0000000000..454d9a5846
--- /dev/null
+++ b/doc/devblog/day_489__zooming.mdwn
@@ -0,0 +1,31 @@
+Andrew Wringler has released
+[git-annex-turtle](https://github.com/andrewringler/git-annex-turtle)
+which provides Apple Finder integration for git-annex on macOS,
+including custom badge icons, contextual menus and a Menubar icon.
+This looks really nice!
+
+---
+
+I've completed the P2P protocol with git-annex-shell. It turned out just as
+fast and good as I'd hoped. 
+[[todo/accellerate_ssh_remotes_with_git-annex-shell_mass_protocol]] has the
+benchmark details. 
+
+Even transferring of large files speeds up somewhat;
+git-annex is actually faster than rsync at shoving bytes down a pipe.
+(Though rsync still wins in lots of other benchmarks I'm sure.)
+
+Surprisingly, in one benchmark, I found accessing a repository on localhost
+via ssh is now slightly *faster* than accessing that same repository by
+path. I think that this is because when git-annex is talking to
+git-annex-ssh, the programs run on different CPU cores, so there's some
+extra concurrency.
+
+There are still some implementation todos, some of which will make it
+faster yet, and others involving potential edge cases. This is a big change
+and will need some time to be considered stable.
+
+----
+
+Today's work was sponsored by Jake Vosloo on
+[Patreon](https://patreon.com/joeyh)
diff --git a/doc/devblog/day_489__zooming/comment_1_d840f371f1d22b0ee8f3c8df25ebe3d3._comment b/doc/devblog/day_489__zooming/comment_1_d840f371f1d22b0ee8f3c8df25ebe3d3._comment
new file mode 100644
index 0000000000..38410b5ea7
--- /dev/null
+++ b/doc/devblog/day_489__zooming/comment_1_d840f371f1d22b0ee8f3c8df25ebe3d3._comment
@@ -0,0 +1,12 @@
+[[!comment format=mdwn
+ username="anarcat"
+ avatar="http://cdn.libravatar.org/avatar/4ad594c1e13211c1ad9edb81ce5110b7"
+ subject="progress bars?"
+ date="2018-03-12T13:44:50Z"
+ content="""
+how do progress bars work in this new design? we had those for free with rsync before (i assume?), do we still have them now?
+
+how about [[todo/wishlist__58___global_progress_status]]? is that more feasible now?
+
+thanks!
+"""]]
diff --git a/doc/devblog/day_489__zooming/comment_2_ec579c84aeb244798a818a20ff0f50b5._comment b/doc/devblog/day_489__zooming/comment_2_ec579c84aeb244798a818a20ff0f50b5._comment
new file mode 100644
index 0000000000..db39387248
--- /dev/null
+++ b/doc/devblog/day_489__zooming/comment_2_ec579c84aeb244798a818a20ff0f50b5._comment
@@ -0,0 +1,11 @@
+[[!comment format=mdwn
+ username="joey"
+ subject="""comment 2"""
+ date="2018-03-12T21:36:45Z"
+ content="""
+It will use git-annex's native progress display, same as it already did
+when -J is used.
+
+Rsync's progress display was not blocking
+[[todo/wishlist__58___global_progress_status]].
+"""]]
diff --git a/doc/devblog/day_48__direct_mode_guard_design.mdwn b/doc/devblog/day_48__direct_mode_guard_design.mdwn
new file mode 100644
index 0000000000..c047e7f750
--- /dev/null
+++ b/doc/devblog/day_48__direct_mode_guard_design.mdwn
@@ -0,0 +1,29 @@
+I've been investigating ways to implement a [[/todo/direct_mode_guard]].
+Preventing a stray `git commit -a` or `git add` doing bad things in a
+direct mode repository seems increasingly important.
+
+First, considered moving `.git`, so git won't know it's a git repository.
+This doesn't seem *too* hard to do, but there will certainly be unexpected
+places that assume `.git` is the directory name.
+
+I dislike it more and more as I think about it though, because it moves
+direct mode git-annex toward being entirely separate from git, and I don't
+want to write my own version control system. Nor do I want to complicate
+the git ecosystem with tools needing to know about git-annex to work in
+such a repository.
+
+So, I'm happy that one of the other ideas I tried today seems quite
+promising. Just set core.bare=true in a direct mode repository. This nicely
+blocks all git commands that operate on the working tree from doing
+anything, which is just what's needed in direct mode, since they don't know
+how to handle the direct mode files. But it lets all git commands and other
+tools that don't touch the working tree continue to be used. You can even
+run `git log file` in such a repository (surprisingly!)
+
+It also gives an easy out for anyone who really wants to use git commands
+that operate on the work tree of their direct mode repository, by just
+passing `-c core.bare=false`. And it's really easy to implement in
+git-annex too -- it can just notice if a repo has core.bare and
+annex.direct both set, and pass that parameter to every git command it
+runs. I should be able to get by with only modifying 2 functions to
+implement this.
diff --git a/doc/devblog/day_48__direct_mode_guard_design/comment_1_ec0147ccc55bad3a38652383f4098a65._comment b/doc/devblog/day_48__direct_mode_guard_design/comment_1_ec0147ccc55bad3a38652383f4098a65._comment
new file mode 100644
index 0000000000..3f297d505c
--- /dev/null
+++ b/doc/devblog/day_48__direct_mode_guard_design/comment_1_ec0147ccc55bad3a38652383f4098a65._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="https://id.koumbit.net/anarcat"
+ ip="70.82.37.38"
+ subject="awesome!"
+ date="2013-11-05T15:50:15Z"
+ content="""
+`core.bare=true` seems like an awesome idea!!! it probably means this can be factored into current installs without any change too right?
+"""]]
diff --git a/doc/devblog/day_490__kind_of_annoying.mdwn b/doc/devblog/day_490__kind_of_annoying.mdwn
new file mode 100644
index 0000000000..bf3344b332
--- /dev/null
+++ b/doc/devblog/day_490__kind_of_annoying.mdwn
@@ -0,0 +1,28 @@
+Working on getting the git-annex-shell P2P protocol into a releasable
+state. This was kind of annoying.
+
+I started out wanting to make annex.verify=false disable verification when
+using the P2P protocol. But, that needed protocol changes, and
+unfortunately the protocol was not extensible. I thought it was supposed to
+reject unknown commands and keep the connection open, which would make
+extensions easy, but unfortunately it actually closed the connection after
+an unknown command.
+
+So, I added a version negotiation to the P2P protocol, but it's not done
+for tor remotes yet, and will be turned on for them in some
+[[future flag day|todo/p2p_protocol_flag_days]], once all of them get
+upgraded.
+
+After all that, I got completely stuck on the annex.verify change.
+Multiple problems are preventing me from seeing a way to do it at all.
+[[todo/support_disabling_verification_of_transfer_over_p2p_protocol]]
+This must be why I didn't support it in the first place when building the
+P2P protocol two years ago.
+
+Also fixed performance when a ssh remote is unavailable, where it was
+trying to connect twice to the remote for each action. And confirmed that
+the assistant will behave ok when moving between networks while it has
+P2P connections open. So, other than annex.verify not being supported,
+I feel fairly ready to release this new feature.
+
+Today's work was supported by an anonymous bitcoin donor.
diff --git a/doc/devblog/day_491__annex.verify_redux.mdwn b/doc/devblog/day_491__annex.verify_redux.mdwn
new file mode 100644
index 0000000000..1d061f679b
--- /dev/null
+++ b/doc/devblog/day_491__annex.verify_redux.mdwn
@@ -0,0 +1,19 @@
+With fresh eyes I stopped being confused by P2P protocol free monad stuff,
+and got annex.verify=false supported when it's safe to skip verification.
+
+And, I found some cases where resuming a download with annex.verify=false
+could let corrupt data into the repository. This is not a new problem; as
+well as with the P2P protocol, it could happen when downloading from the
+web, and possibly with some external special remotes that support resuming.
+So, it seemed best to override annex.verify configuration when
+resuming a download.
+
+Also fixed up some progress bar stuff related to the P2P protocol. 
+Including dealing with the case where the size of a key being downloaded is
+not known until the peer starts sending its data. The progress bar will
+now be updated with the size from the P2P protocol, so it can display
+a percentage even in this case.
+
+I hope that's the end of the P2P protocol stuff for now.
+
+Today's work was supported by the NSF-funded DataLad project.
diff --git a/doc/devblog/day_492__concurrency_is_hard.mdwn b/doc/devblog/day_492__concurrency_is_hard.mdwn
new file mode 100644
index 0000000000..6a18072d95
--- /dev/null
+++ b/doc/devblog/day_492__concurrency_is_hard.mdwn
@@ -0,0 +1,12 @@
+In the past 24 hours, I've fixed two extremely hairy problems with 
+`git annex get -J`. One was a locking problem.
+And the other involved thundering herds and ssh connection multiplexing
+and inherited file descriptors and races, and ...
+That took 4 hours of investigation to understand well enough to fix it.
+
+Neither of those involved the ssh P2P changes, other than perhaps they
+exposed one of the issues more than it was exposed before,
+but on the plus side I've been testing that new code
+quite a lot as I worked on them..
+
+Today's work was supported by the NSF-funded DataLad project.
diff --git a/doc/devblog/day_493__two_new_special_remotes.mdwn b/doc/devblog/day_493__two_new_special_remotes.mdwn
new file mode 100644
index 0000000000..478450ba81
--- /dev/null
+++ b/doc/devblog/day_493__two_new_special_remotes.mdwn
@@ -0,0 +1,18 @@
+I've been traveling and at conferences.
+
+In the meantime, Lykos has released
+[git-annex-remote-googledrive](https://github.com/Lykos153/git-annex-remote-googledrive),
+a replacement for an older, unmaintained Google Drive special remote.
+
+Today I added a special remote that stores files on an Android device
+using `adb`. It supports `git annex export`, so the files stored on 
+the Android device can have the same filenames as in the git-annex
+repository. I have plans for making `git annex import` support
+special remotes, and other features to make bi-directional
+sync with Android work well.
+
+Of course, there is some overlap between that and the Android port,
+but they probably serve different use cases.
+
+Today's work was sponsored by Trenton Cronholm on
+[Patreon](https://patreon.com/joeyh/)
diff --git a/doc/devblog/day_494__url_download_changes.mdwn b/doc/devblog/day_494__url_download_changes.mdwn
new file mode 100644
index 0000000000..3d542dd57b
--- /dev/null
+++ b/doc/devblog/day_494__url_download_changes.mdwn
@@ -0,0 +1,25 @@
+To make git-annex faster when it's dealing with a lot of urls,
+I decided to make it use the http-conduit library for all url access by
+default. That way, http pipelining will speed up repeated requests to the
+same web servers. This is kind of a follow-up to the recent elimination of
+rsync.
+
+Some users rely on some annex.web-options or a .netrc file to configure
+how git-annex downloads urls. To keep that supported, when
+annex.web-options is set, git-annex will use curl.
+To use a .netrc file, curl needs an option, so you would configure:
+
+	git config annex.web-options --netrc
+
+I get the feeling that nobody has implemented resuming interrupted
+downloads of files using http-conduit before, because it was unexpectedly
+kind of hard and http-types lacks support for some of the necessary
+range-related HTTP stuff.
+
+Today's work was supported by the NSF-funded DataLad project.
+
+----
+
+Stewart V. Wright [announced recastex](http://git-annex.branchable.com/tips/Announcing_recastex_-___40__re__41__podcast__from_your_annex/),
+a program that publishes podcasts and other files from by git-annex to your
+phone.
diff --git a/doc/devblog/day_495__move_numcopies_safety.mdwn b/doc/devblog/day_495__move_numcopies_safety.mdwn
new file mode 100644
index 0000000000..0161244911
--- /dev/null
+++ b/doc/devblog/day_495__move_numcopies_safety.mdwn
@@ -0,0 +1,20 @@
+New version released today with adb special remote, http connection
+caching, improved progress displays, annex.retry, and other changes.
+
+I've been rethinking `git annex move` in the context of numcopies
+checking. Thanks to a user posting
+[[forum/git-annex_move_does_not_appear_to_respect_numcopies]].
+Of course, move is known not to do that, but it's useful to get a
+perspective that this is susprising behavior and not wanted by that user,
+and poorly documented besides.
+
+So, I added `git annex move --safe` which does honor numcopies, so 
+it only does a copy when there are not enough copies to move.
+
+I'm leaning toward making that the default behavior, and
+needing `git annex move --unsafe` to get the current behavior of moving
+without a net. Of course, lots of us probably use move and like the current
+behavior, and such a change can break workflows and scripts.
+There might be a transition period where move warns when run without --safe
+or --unsafe. Feedback welcomed on the bug report 
+[[bugs/move_violates_numcopies]].
diff --git a/doc/devblog/day_496__move_numcopies_safety_revisited.mdwn b/doc/devblog/day_496__move_numcopies_safety_revisited.mdwn
new file mode 100644
index 0000000000..9d96e0cdd5
--- /dev/null
+++ b/doc/devblog/day_496__move_numcopies_safety_revisited.mdwn
@@ -0,0 +1,12 @@
+After talking it over in [[bugs/move_violates_numcopies]], we found a nicer
+compromise for `git annex move`. Rather than strictly enforcing numcopies,
+it avoids making any bad situations worse. For example, when there's
+only one copy of a file, it can be moved even if numcopies is higher.
+But, when numcopies is 2 and the source and destination repos have a copy,
+move will not drop from the source repo, since that would make it worse.
+
+Implemented that today. While doing so I got bit by the inverted Ord
+instance for TrustLevel, so spent a while cleaning that up.
+
+Today's work was sponsored by Jake Vosloo on
+[Patreon](https://patreon.com/joeyh/).
diff --git a/doc/devblog/day_497__rethinking_the_android_port.mdwn b/doc/devblog/day_497__rethinking_the_android_port.mdwn
new file mode 100644
index 0000000000..5d1b74124c
--- /dev/null
+++ b/doc/devblog/day_497__rethinking_the_android_port.mdwn
@@ -0,0 +1,24 @@
+I've long been unsatisfied with the amount of effort needed to maintain the
+Android port in its current state, the hacky cross-compiler toolchain 
+needs days of wasted work to update, and is constantly out of date and
+breaking in one way or other. This sucks up any time I might spare to
+actually improve the Android port.
+
+So, it was quite a surprise yesterday when I downloaded the git-annex
+standalone Linux tarball into the Termux Android shell and unpacked it, and
+it more or less worked!
+
+The result, after a [[few minor fixes|todo/termux_package]], works 
+just as well as the git-annex Android app, and probably better. Even the 
+webapp works well, and with the Termux:Boot app, it can even
+autostart the assistant on boot as a daemon. If you want to give it a
+try, see [[tips/install_on_Android_in_Termux]].
+
+So, I am leaning toward deprecating the android port for this, removing 14
+thousand lines of patches and android-specific code. Not going to do it
+just yet, but I feel a weight lifting...
+
+----
+
+Today's work was sponsored by Jake Vosloo on
+[Patreon](https://patreon.com/joeyh).
diff --git a/doc/devblog/day_497__rethinking_the_android_port/comment_1_cb30ab18753e5f221ed4d92b3ff8a32e._comment b/doc/devblog/day_497__rethinking_the_android_port/comment_1_cb30ab18753e5f221ed4d92b3ff8a32e._comment
new file mode 100644
index 0000000000..f288912e65
--- /dev/null
+++ b/doc/devblog/day_497__rethinking_the_android_port/comment_1_cb30ab18753e5f221ed4d92b3ff8a32e._comment
@@ -0,0 +1,13 @@
+[[!comment format=mdwn
+ username="https://openid.stackexchange.com/user/3ee5cf54-f022-4a71-8666-3c2b5ee231dd"
+ nickname="Anthony DeRobertis"
+ avatar="http://cdn.libravatar.org/avatar/1007bfece547e9f2d86fafa142cd240a62456c37f104a9d96ba9db5bf18e1934"
+ subject="Looking forward to trying this"
+ date="2018-04-26T13:10:30Z"
+ content="""
+I use the current Android port on a few tablets, and it's... well, clearly labeled beta. On one of the tablets, it randomly segfaults [no idea if that's a git-annex bug or a hardware problem, not like I have a memtest86 available]. On both it sometimes just decides to add commits to remove files recently added on other machines (easy enough to fix with git revert). Sometimes it puts in place small placeholder files (git annex fsck fixes). Still far better than manually copying & sync'ing, though!
+
+At least for me, v6 mode just doesn't work on Android — even on a new empty repository, last I checked, switching to v6 mode gave errors.
+
+So I look forward to trying it in Termux! 
+"""]]
diff --git a/doc/devblog/day_498__unexpected_release_prep.mdwn b/doc/devblog/day_498__unexpected_release_prep.mdwn
new file mode 100644
index 0000000000..8ceae0a808
--- /dev/null
+++ b/doc/devblog/day_498__unexpected_release_prep.mdwn
@@ -0,0 +1,21 @@
+I'm unexpectedly preparing for a release soon, because the last
+release turned out to have a crasher bug when using a bare repository or
+--all, and a bug that prevented the webapp starting on OSX.
+
+As well as fixing those, the new release will have
+several smaller improvements and fixes all done today. It's been a rather
+productive day.
+
+And, using git-annex in Termux is now working even on newer versions of
+Android, that use seccomp filtering to filter out system calls that the ghc
+runtime uses. The `proot` program on Termux worked around that nasty
+problem.
+
+The old Android app is now deprecated, and I'll probably remove it entirely
+within a few months unless I find a reason not to.
+So, I also closed almost all the old Android-specific bug reports today. I
+don't normally do mass bug closures without followup, but it was warranted
+here; almost all of those bugs are specific to the old Android app.
+
+Today's work was sponsored by Trenton Cronholm on
+[Patreon](https://patreon.com/joeyh)
diff --git a/doc/devblog/day_499__security_hole.mdwn b/doc/devblog/day_499__security_hole.mdwn
new file mode 100644
index 0000000000..635dc3f073
--- /dev/null
+++ b/doc/devblog/day_499__security_hole.mdwn
@@ -0,0 +1,27 @@
+I'm writing this on a private branch, it won't be posted until a week from
+now when the security hole is disclosed.
+
+Security is not compositional. You can have one good feature, and add
+another good feature, and the result is not two good features, but a new
+security hole. In this case
+[[bugs/security_hole_private_data_exposure_via_addurl]] (CVE-2018-10857). 
+And it can be hard to spot this kind of security hole, but then once it's
+known it seems blindly obvious.
+
+It came to me last night and by this morning I had decided the potential
+impact was large enough to do a coordinated disclosure. Spent the first
+half of the day thinking through ways to fix it that don't involve writing
+my own http library. Then started getting in touch with all the
+distributions' security teams. And then coded up a fairly complete fix for
+the worst part of the security hole, although a secondary part is going to
+need considerably more work.
+
+It looks like the external special remotes are going to need at least some
+security review too, and I'm still thinking that part of the problem over.
+
+Exhausted.
+
+Today's work was sponsored by Trenton Cronholm 
+[on Patreon](https://patreon.com/joeyh).
+
+[[!meta date="Jun 15 2018 7:00 pm"]]
diff --git a/doc/devblog/day_49__direct_mode_guard_implementation.mdwn b/doc/devblog/day_49__direct_mode_guard_implementation.mdwn
new file mode 100644
index 0000000000..ebc0dd4946
--- /dev/null
+++ b/doc/devblog/day_49__direct_mode_guard_implementation.mdwn
@@ -0,0 +1,14 @@
+Long, long day coding up the direct mode guard today. About 90% of the fun
+is dealing with `receive.denyCurrentBranch` not preventing pushes that
+change the current branch, now that core.bare is set in direct mode.
+My current solution to this involves using a special branch when using
+direct mode, which nothing will ever push to (hopefully). A much nicer
+solution would be to use a `update` hook to deny pushes of the current
+branch -- but there are filesystems where repos cannot have git hooks.
+
+The test suite is falling over, but the `directguard` branch otherwise
+seems usable.
+
+----
+
+Today's work was sponsored by Carlo Matteo Capocasa.
diff --git a/doc/devblog/day_49__direct_mode_guard_implementation/comment_1_3ebe5c3f708070f164ecaf36b79f7bfc._comment b/doc/devblog/day_49__direct_mode_guard_implementation/comment_1_3ebe5c3f708070f164ecaf36b79f7bfc._comment
new file mode 100644
index 0000000000..2a77d6d815
--- /dev/null
+++ b/doc/devblog/day_49__direct_mode_guard_implementation/comment_1_3ebe5c3f708070f164ecaf36b79f7bfc._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="https://www.google.com/accounts/o8/id?id=AItOawnZEanlyzay_QlEAL0CWpyZcRTyN7vay8U"
+ nickname="Carlo"
+ subject="comment 1"
+ date="2013-11-06T11:50:00Z"
+ content="""
+Thanks, Joey! I am very proud!
+"""]]
diff --git a/doc/devblog/day_4__unexpected_windows_day.mdwn b/doc/devblog/day_4__unexpected_windows_day.mdwn
new file mode 100644
index 0000000000..6a448c3fc1
--- /dev/null
+++ b/doc/devblog/day_4__unexpected_windows_day.mdwn
@@ -0,0 +1,10 @@
+Woke up with a pretty solid plan for gcrypt. It will be structured as a
+separate special remote, so `initremote` will be needed, with a gitrepo=
+parameter (unless the remote already exists). git-annex will then set up
+the git remote, including pushing to it (needed to get a gcrypt-id).
+
+Didn't feel up to implementing that today. Instead I expectedly spent
+the day doing mostly Windows work, including setting up a VM on my new
+laptop for development. Including a ssh server in Windows, so I can
+script local builds and tests on Windows without ever having to
+touch the desktop. Much better!
diff --git a/doc/devblog/day_500__security_hole_part_2.mdwn b/doc/devblog/day_500__security_hole_part_2.mdwn
new file mode 100644
index 0000000000..3d485ee73c
--- /dev/null
+++ b/doc/devblog/day_500__security_hole_part_2.mdwn
@@ -0,0 +1,26 @@
+Most of the day was spent staring at the http-client source code and trying
+to find a way to add the IP address checks to it that I need to fully close
+the security hole.
+
+In the end, I did find a way, with the duplication of a couple dozen lines
+of code from http-client. It will let the security fix be used with
+libraries like aws and DAV that build on top of http-client, too.
+
+While the code is in git-annex for now, it's fully disconnected and
+would also be useful if a web browser were implemented in Haskell,
+to implement same-origin restrictions while avoiding DNS rebinding attacks.
+
+Looks like http proxies and curl will need to be disabled by default,
+since this fix can't support either of them securely. I wonder how web
+browsers deal with http proxies, DNS rebinding attacks and same-origin?
+I can't think of a secure way.
+
+Next I need a function that checks if an IP address is a link-local address
+or a private network address. For both ipv4 and ipv6. Could not find
+anything handy on hackage, so I'm gonna have to stare at some RFCs. Perhaps
+this evening, for now, it's time to swim in the river.
+
+Today's work was sponsored by  Jake Vosloo
+[on Patreon](https://patreon.com/joeyh)
+
+[[!meta date="June 16 2018 4:00 pm"]]
diff --git a/doc/devblog/day_501__security_hole_part_3.mdwn b/doc/devblog/day_501__security_hole_part_3.mdwn
new file mode 100644
index 0000000000..6187f7d534
--- /dev/null
+++ b/doc/devblog/day_501__security_hole_part_3.mdwn
@@ -0,0 +1,13 @@
+Got the IP address restrictions for http implemented. (Except for http
+proxies.)
+
+Unforunately as part of this, had to make youtube-dl and curl not be used
+by default. The annex.security.allowed-http-addresses config has to be
+opened up by the user in order to use those external commands, since they
+can follow arbitrary redirects.
+
+Also thought some more about how external special remotes might be
+affected, and sent their authors' a heads-up.
+
+[[!meta date="June 17 2018 4:00 pm"]]
+
diff --git a/doc/devblog/day_502__security_hole_part_4.mdwn b/doc/devblog/day_502__security_hole_part_4.mdwn
new file mode 100644
index 0000000000..8988a7f15d
--- /dev/null
+++ b/doc/devblog/day_502__security_hole_part_4.mdwn
@@ -0,0 +1,24 @@
+Spent several hours dealing with the problem of http proxies, which
+bypassed the IP address checks added to prevent the security hole.
+Eventually got it filtering out http proxies located on private IP
+addresses.
+
+Other than the question of what to do about external special remotes
+that may be vulerable to related problems, it looks like the security
+hole is all closed off in git-annex now.
+
+Added a new page [[security]] with details of this and past security holes
+in git-annex.
+
+Several people I reached out to for help with special remotes have gotten
+back to me, and we're discussing how the security hole may affect them and
+what to do. Thanks especially to Robie Basak and Daniel Dent for their
+work on security analysis.
+
+Also prepared a minimal backport of the security fixes for the git-annex in
+Debian stable, which will probably be more palatable to their security team
+than the full 2000+ lines of patches I've developed so far. 
+The minimal fix is secure, but suboptimal; it prevents even safe urls from
+being downloaded from the web special remote by default.
+
+[[!meta date="June 18 2018 4:00 pm"]]
diff --git a/doc/devblog/day_503__security_hole_part_5.mdwn b/doc/devblog/day_503__security_hole_part_5.mdwn
new file mode 100644
index 0000000000..86c760c6c2
--- /dev/null
+++ b/doc/devblog/day_503__security_hole_part_5.mdwn
@@ -0,0 +1,19 @@
+Started testing that the security fix will build everywhere on
+release day. This is being particularly painful for the android build,
+which has very old libraries and needed http-client updated, with many
+follow-on changes, and is not successfully building yet after 5 hours. 
+I really need to finish deprecating the android build.
+
+Pretty exhausted from all this, and thinking what to do about
+external special remotes, I elaborated on an idea that Daniel Dent had
+raised in discussions about vulnerability, and realized that git-annex
+has a second, worse vulnerability. This new one could be used to trick a
+git-annex user into decrypting gpg encrypted data that they had 
+never stored in git-annex. The attacker needs to have control of both an
+encrypted special remote and a git remote, so it's not an easy exploit to
+pull off, but it's still super bad.
+
+This week is going to be a lot longer than I thought, and it's already
+feeling kind of endless..
+
+[[!meta date="June 19 2018 8:00 pm"]]
diff --git a/doc/devblog/day_504__security_hole_part_6.mdwn b/doc/devblog/day_504__security_hole_part_6.mdwn
new file mode 100644
index 0000000000..17680941c2
--- /dev/null
+++ b/doc/devblog/day_504__security_hole_part_6.mdwn
@@ -0,0 +1,12 @@
+Was getting dangerously close to burnt out, or exhaustion leading to
+mistakes, so yesterday I took the day off, aside from spending the morning
+babysitting the android build every half hour. (It did finally succeed.)
+
+Today, got back into it, and implemented a fix for CVE-2018-10859 and also
+the one case of CVE-2018-10857 that had not been dealt with before.
+This fix was really a lot easier than the previous fixes for
+CVE-2018-10857.
+Unfortunately this did mean not letting URL and WORM keys be downloaded
+from many special remotes by default, which is going to be painful for some.
+
+[[!meta date="June 20 2018 5:00 pm"]]
diff --git a/doc/devblog/day_505__security_fix_release.mdwn b/doc/devblog/day_505__security_fix_release.mdwn
new file mode 100644
index 0000000000..2f2d486675
--- /dev/null
+++ b/doc/devblog/day_505__security_fix_release.mdwn
@@ -0,0 +1,15 @@
+Just released git-annex 6.20180626 with important security fixes!
+
+Please go upgrade now, read the [[release_notes|news/security_fix_release]]
+for details about some necessary behavior changes,
+and if you're curious about the details of the security holes,
+see [[the advisory|security/CVE-2018-10857_and_CVE-2018-10859]].
+
+I've been dealing with these security holes for the past week and a half,
+and decided to use a security embargo while fixes were being developed
+due to the complexity of addressing security holes that impact both
+git-annex and external special remote programs. For the full story
+see past 5 posts in this devblog, which are being published all together
+now that the embargo is lifted.
+
+[[!meta date="Jun 26 2018 12:00 pm"]]
diff --git a/doc/devblog/day_506__summer_features.mdwn b/doc/devblog/day_506__summer_features.mdwn
new file mode 100644
index 0000000000..52eea6a454
--- /dev/null
+++ b/doc/devblog/day_506__summer_features.mdwn
@@ -0,0 +1,38 @@
+After the big security fix push, I've had a bit of a vacation. Several new
+features have also landed in git-annex though.
+
+git-worktree support is a feature I'm fairly excited by.
+It turned out to be possible to make git-annex just work in working trees
+set up by `git worktree`, and they share the same object files. So,
+if you need several checkouts of a repository for whatever reason,
+this makes it really efficient to do. It's much better than the old
+method of using `git clone --shared`.
+
+A new `--accessedwithin` option matches files whose content was accessed
+within a given amount of time. (Using the atime.) Of course it can
+be combined with other options, for example
+`git annex move --to archive --not --accessedwithin=30d`  
+There are a few open requests for other new file matching options that I
+hope to get to soon.
+
+A small configuration addition of remote.name.annex-speculate-present
+to make git-annex try to get content from a remote even if its records
+don't indicate the remote contains the content allows setting up an interesting
+kind of [local cache of annexed files](https://git-annex.branchable.com/tips/local_caching_of_annexed_files/)
+which can even be shared between unrelated git-annex repositories, with
+inter-repository deduplication. 
+
+I suspect that remote.name.annex-speculate-present may also have other
+uses. It warps git-annex's behavior in a small but fundamental way which
+could let it fit into new places. Will be interesting to see.
+
+There's also a annex.commitmessage config, which I am much less excited by,
+but enough people have asked for it over the years.
+
+Also fixed a howler of a bug today: In -J mode, remotes were sorted not
+by cost, but by UUID! How did that not get noticed for 2 years?
+
+Much of this work was sponsored by NSF-funded DataLad project at Dartmouth
+Colledge, as has been the case for the past 4 years. All told they've
+funded over 1000 hours of work on git-annex. This is the last month of that
+funding.
diff --git a/doc/devblog/day_507__v6_revisited.mdwn b/doc/devblog/day_507__v6_revisited.mdwn
new file mode 100644
index 0000000000..cb8bcaa473
--- /dev/null
+++ b/doc/devblog/day_507__v6_revisited.mdwn
@@ -0,0 +1,23 @@
+Plan is to take some time this August and revisit v6, hoping to move it
+toward being production ready.
+
+Today I studied the "Long Running Filter Process" documentation in
+gitattributes(5), as well as the supplimental documentation in git about
+the protocol they use. This interface was added to git after v6 mode was
+implemented, and hopefully some of v6's issues can be fixed by using it in
+some way. But I don't know how yet, it's not as simple as using this
+interface as-is (it was designed for something different), but
+finding a creative trick using it.
+
+So far I have [this idea](http://git-annex.branchable.com/todo/Long_Running_Filter_Process/#comment-7c571c4ed26ce370ccd48db0a4aff4fc)
+to explore. It's promising, might fix the worst of the problems.
+
+Also, reading over all the notes in [[todo/smudge]], I finally
+checked and yes, git doesn't require filters to consume all stdin anymore,
+and when they don't consume stdin, git doesn't leak memory anymore either.
+Which let me massively speed up `git add` in v6 repos. While before `git
+add` of a gigabyte file made git grow to a gigabyte in memory and copied a
+gigabyte through a pipe, it's now just as fast as `git annex add` in v5
+mode is.
+
+This work is supported by the NSF-funded DataLad project.
diff --git a/doc/devblog/day_508__git-protocol.mdwn b/doc/devblog/day_508__git-protocol.mdwn
new file mode 100644
index 0000000000..b3de2ffa8a
--- /dev/null
+++ b/doc/devblog/day_508__git-protocol.mdwn
@@ -0,0 +1,20 @@
+Spent today implementing the git pkt-line protocol. Git uses it for a bunch
+of internal stuff, but also to talk to long-running filter processes.
+
+This was my first time using attoparsec, which I quite enjoyed aside from
+some difficulty in parsing a 4 byte hex number. Even though parsing to a
+Word16 should naturally only consume 4 bytes, attoparsec will actually
+consume subsequent bytes that look like hex. And it may parse fewer than 4
+bytes too. So my parser had to take 4 bytes and feed them back into a call
+to attoparsec. Which seemed weird, but works. I also used
+bytestring-builder, and between the two libraries, this should be quite a
+fast implementation of the protocol.
+
+With that 300 lines of code written, it should be easy to implement support
+for the rest of the long-running filter process protocol. Which will surely
+speed up v6 a bit, since at least git won't be running git-annex over and
+over again for each file in the worktree. I hope it will also avoid a memory
+leak in git. That'll be the rest of the low-hanging fruit, before v6
+improvements get really interesting.
+
+This work is supported by the NSF-funded DataLad project.
diff --git a/doc/devblog/day_509__filterdriver.mdwn b/doc/devblog/day_509__filterdriver.mdwn
new file mode 100644
index 0000000000..8bc6fc9c4a
--- /dev/null
+++ b/doc/devblog/day_509__filterdriver.mdwn
@@ -0,0 +1,16 @@
+Working on a "filterdriver" branch, I've implemented support for the
+long-running smudge/clean process interface.
+
+It works, but not really any better than the old smudge/clean interface.
+Unfortunately git leaks memory just as badly in the new interface as it did
+in the old interface when sending large data to the smudge filter. Also,
+the new interface requires that the clean filter read all the content of the
+file from git, even when it's just going to look at the file on disk, so
+that's worse performance.
+
+So, I don't think I'll be merging that branch yet, but git's interface does
+support adding capabilities, and perhaps a capability could be added that
+avoids it schlepping the file content over the pipe. Same as my old git
+patches tried to do with the old smudge/clean interface.
+
+This work is supported by the NSF-funded DataLad project.
diff --git a/doc/devblog/day_50__grab_bag.mdwn b/doc/devblog/day_50__grab_bag.mdwn
new file mode 100644
index 0000000000..e38ec065c7
--- /dev/null
+++ b/doc/devblog/day_50__grab_bag.mdwn
@@ -0,0 +1,34 @@
+Started by tracking down a strange bug that was apparently
+ubuntu-specific and caused git-annex branch changes to get committed to
+master. Root cause turned out to failing to recover from an
+exception. I'm kicking myself about that, because I remember looking at the
+code where the bug was at least twice before and thinking "hmm, should add
+exception handling here? nah..". Exceptions are horrible.
+
+Made a release with a fix for that and a few minor other accumulated
+changes since last Friday's release. The pain point of this release is to
+fix building without the webapp (so it will propigate to Debian testing,
+etc). This release does not include the direct mode guard, so I'll have a
+few weeks until the next release to get that tested.
+
+Fixed the test suite in `directguard`. This branch is now nearly ready to
+merge to master, but one command that is badly needed in guarded direct
+mode is "git status". So I am planning to rename "git annex status" to
+"git annex info", and make "git annex status" display something similar
+to "git status".
+
+Also took half an hour and added optional [[EKG]] support to git-annex.
+This is a Haskell library that can add a terrific monitoring console web
+UI to any program in 2 lines of code. Here we can see the git-annex
+webapp using resources at startup, followed in a few seconds by the
+assistant's startup scan of the repository.
+
+[[!img ekg/ekg.png]]
+
+BTW, Kevin tells me that the machine used to build git-annex for OSX is
+going to be upgraded to 10.9 soon. So, hopefully I'll be making autobuilds
+of that. I may have to stop the 10.8.2 autobuilds though.
+
+----
+
+Today's work was sponsored by [Protonet](http://protonet.info/).
diff --git a/doc/devblog/day_50__grab_bag/comment_1_01846f6494fe843889391fd09fd127a0._comment b/doc/devblog/day_50__grab_bag/comment_1_01846f6494fe843889391fd09fd127a0._comment
new file mode 100644
index 0000000000..1c17171808
--- /dev/null
+++ b/doc/devblog/day_50__grab_bag/comment_1_01846f6494fe843889391fd09fd127a0._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="https://me.yahoo.com/a/2grhJvAC049fJnvALDXek.6MRZMTlg--#eec89"
+ nickname="John"
+ subject="OS X builds"
+ date="2013-11-07T05:12:13Z"
+ content="""
+Joey, were you not interested in my offer of an OS X build server that I've posted elsewhere on this list, and also in e-mail to you?
+"""]]
diff --git a/doc/devblog/day_50__grab_bag/comment_2_12736014aa2c1af81e4b83072505e7d5._comment b/doc/devblog/day_50__grab_bag/comment_2_12736014aa2c1af81e4b83072505e7d5._comment
new file mode 100644
index 0000000000..70d7b7cd16
--- /dev/null
+++ b/doc/devblog/day_50__grab_bag/comment_2_12736014aa2c1af81e4b83072505e7d5._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="http://joeyh.name/"
+ ip="209.250.56.47"
+ subject="comment 2"
+ date="2013-11-07T16:01:00Z"
+ content="""
+John, must have missed that; can't see to find it anywhere..
+"""]]
diff --git a/doc/devblog/day_510__v6_get_drop_index.mdwn b/doc/devblog/day_510__v6_get_drop_index.mdwn
new file mode 100644
index 0000000000..9cd7e805b7
--- /dev/null
+++ b/doc/devblog/day_510__v6_get_drop_index.mdwn
@@ -0,0 +1,14 @@
+I've now fixed the worst problem with v6 mode, which was that get/drop of
+unlocked files would cause git to think that the files were modified.
+
+Since the clean filter now runs quite fast, I was able to fix that by,
+after git-annex updates the worktree, restaging the not-really-modified 
+file in the index.
+
+This approach is not optimal; index file updates have overhead; and only
+one process can update the index file at one time. [[todo/smudge]] has a
+bunch of new todo items for cases where this change causes problems. Still,
+it seems a lot better than the old behavior, which made v6 mode nearly
+unusable IMHO.
+
+This work is supported by the NSF-funded DataLad project.
diff --git a/doc/devblog/day_510__v6_get_drop_index/comment_1_0e2eff8c8c3aaef3499b8f94b2733148._comment b/doc/devblog/day_510__v6_get_drop_index/comment_1_0e2eff8c8c3aaef3499b8f94b2733148._comment
new file mode 100644
index 0000000000..42a871a147
--- /dev/null
+++ b/doc/devblog/day_510__v6_get_drop_index/comment_1_0e2eff8c8c3aaef3499b8f94b2733148._comment
@@ -0,0 +1,16 @@
+[[!comment format=mdwn
+ username="spwhitton"
+ avatar="http://cdn.libravatar.org/avatar/9c3f08f80e67733fd506c353239569eb"
+ subject="comment 1"
+ date="2018-08-17T14:44:13Z"
+ content="""
+> I've now fixed the worst problem with v6 mode, which was that get/drop of
+> unlocked files would cause git to think that the files were modified.
+
+\o/
+
+I have been using v6 lately and did not know that this was a known
+issue; I assumed I was doing something wrong.
+
+Thank you!
+"""]]
diff --git a/doc/devblog/day_511__v6_improved_index_update.mdwn b/doc/devblog/day_511__v6_improved_index_update.mdwn
new file mode 100644
index 0000000000..989aa1feab
--- /dev/null
+++ b/doc/devblog/day_511__v6_improved_index_update.mdwn
@@ -0,0 +1,13 @@
+Found a better way to update the index after get/drop in v6 repositories.
+I was able to close all the todos around that.
+
+Only problem is there is a race where a modification that happens to a file
+soon after get/drop gets unexpectedly staged by the index update. I made
+this race's window as small as I reasonably can. Fully fixing it would
+involve improvements to the git update-index interface, or another way to
+update the index.
+
+Only two todos remain in [[todo/smudge]] that
+I want to fix in the remainder of this v6 sprint.
+
+This work is supported by the NSF-funded DataLad project.
diff --git a/doc/devblog/day_512__fixed_race.mdwn b/doc/devblog/day_512__fixed_race.mdwn
new file mode 100644
index 0000000000..94f5d920ad
--- /dev/null
+++ b/doc/devblog/day_512__fixed_race.mdwn
@@ -0,0 +1,5 @@
+Sleeping on that race from yesterday, I realized there is a way to fix it,
+and have implemented the fix. It doubled the overhead of updating the
+index, but that's worth it to not have a race condition to worry about.
+
+This work is supported by the NSF-funded DataLad project.
diff --git a/doc/devblog/day_513__v6_reconciling_staged_changes.mdwn b/doc/devblog/day_513__v6_reconciling_staged_changes.mdwn
new file mode 100644
index 0000000000..aeb22aad5d
--- /dev/null
+++ b/doc/devblog/day_513__v6_reconciling_staged_changes.mdwn
@@ -0,0 +1,21 @@
+More v6 work. Got most of the way to a solution to the problem of updating
+the associated files database for staged changes to unlocked files, eg a
+`git mv`.
+
+While writing the test case, I was surprised to find that the problem
+is timing dependent. If a `git mv` is run less than a second after `git
+add`, git runs the smudge filter for whatever reason, which avoids the
+problem. With a longer delay, it doesn't run the smudge filter. Seems this
+could be the cause of intermittent glitches with v6 mode, and I've seen a
+few such glitches before.
+
+Anyway, I developed an inexpensive way to find the relevant staged changes,
+using `git diff` with a full page of options to tweak its behavior just
+right. Still need to make that only run when the index has changed, not
+every time git-annex runs.
+
+There's still a race between a command like `git mv` and `git annex
+drop/get`, that can result in the unlocked file's content not being
+updated. Don't have a solution to that yet.
+
+This work is supported by the NSF-funded DataLad project.
diff --git a/doc/devblog/day_514__fixed_5_races_in_v6.mdwn b/doc/devblog/day_514__fixed_5_races_in_v6.mdwn
new file mode 100644
index 0000000000..b400555290
--- /dev/null
+++ b/doc/devblog/day_514__fixed_5_races_in_v6.mdwn
@@ -0,0 +1,9 @@
+That's a lot of races! Well, 4 of them were all related, but the fixes to
+them had to be made in two different places.
+
+Hopefully that's all the v6 races fixed. I've been finding these races by
+inspection, who knows if I missed some. Anyway, I'm now down to one todo
+item left on the v6 sprint. Gonna take a break for a couple days before
+tackling it.
+
+This work is supported by the NSF-funded DataLad project.
diff --git a/doc/devblog/day_514__v6_bug_review.mdwn b/doc/devblog/day_514__v6_bug_review.mdwn
new file mode 100644
index 0000000000..f50571f26b
--- /dev/null
+++ b/doc/devblog/day_514__v6_bug_review.mdwn
@@ -0,0 +1,25 @@
+Looked over bugs filed about v6 mode and did some triage and analysis.
+[[todo/smudge]] has the details.
+
+This led to changing what's done by `git add` and `git commit file` when
+annex.largefiles is not configured. Rather than behaving like `git annex
+add` and always storing the file in the annex, it will store it in the
+annex if the old version was annexed, and in git if the old version was
+stored in git. This avoids accidental conversions.
+
+It might make sense to have `git annex add` also do this, even in v5
+repositories, but I want to concentrate on v6 for now, and also don't
+think that `git add` and `git annex add` necessarily need to behave
+identically in v6 mode. While using `git commit -a` doesn't imply
+anything about whether you want the file in git or the annex,
+using `git-annex add` seems to imply they you want it in
+the annex, unless you've gone out of your way to configure otherwise.
+
+----
+
+Also did some design work on supporting versioned S3 buckets with
+`git-annex export`.
+
+----
+
+This work is supported by the NSF-funded DataLad project.
diff --git a/doc/devblog/day_515__S3_exporttree_with_versioning.mdwn b/doc/devblog/day_515__S3_exporttree_with_versioning.mdwn
new file mode 100644
index 0000000000..27ff00d5aa
--- /dev/null
+++ b/doc/devblog/day_515__S3_exporttree_with_versioning.mdwn
@@ -0,0 +1,10 @@
+Most of the way done with implementing support for export to S3 buckets
+with versioning enabled. This will make the files from the most recent
+`git annex export` be visible to users browsing the bucket, while letting
+git-annex download any of the content from previous exports too.
+
+Still need to test it. And, deletion of old content from such a bucket is
+not supported, and my initial thoughts are that it might not be possible
+in a multi-writer situation. I need to think about it more.
+
+This work is supported by the NSF-funded DataLad project.
diff --git a/doc/devblog/day_516__S3_exporttree_with_versioning_continued.mdwn b/doc/devblog/day_516__S3_exporttree_with_versioning_continued.mdwn
new file mode 100644
index 0000000000..cd57d5be94
--- /dev/null
+++ b/doc/devblog/day_516__S3_exporttree_with_versioning_continued.mdwn
@@ -0,0 +1,19 @@
+Finished this feature, and I'm liking it quite a lot!
+Though I had to put off [[todo/support_public_versioned_S3_access]] until
+some other time as I'm all out of time and energy.
+
+The storage of S3 version IDs got rethought -- I was not comfortable with
+using per-remote state in the git-annex branch which would have caused
+problems if dropping from these remotes later gets supported.
+
+So, I added per-remote metadata in about 1 hour! It's like git-annex's regular
+metadata, but scoped so only the remote that owns it can see it. This is
+perfect for storing things like S3 version IDs. It probably ought to be added
+to the external special remote interface since it could be used for lots of
+stuff.
+
+Here's how that looks when S3 version IDs are stored in it:
+
+	1535737778.867692782s 31ea6c94-fba3-4952-99b5-285ae192d92a:V +woYHK59DD2VUkJfg527mEBBqtCaPlSXn#myfile
+
+This work was supported by the NSF-funded DataLad project.
diff --git a/doc/devblog/day_517__return_to_crowdfunding.mdwn b/doc/devblog/day_517__return_to_crowdfunding.mdwn
new file mode 100644
index 0000000000..0d4f03db97
--- /dev/null
+++ b/doc/devblog/day_517__return_to_crowdfunding.mdwn
@@ -0,0 +1,9 @@
+Back to being only [crowdfunded](https://patreon.com/joeyh) now.
+
+Several little things today, including a git-annex.cabal patch from
+fftehnik that fixed building without the assistant, and supporting
+`AWS_SESSION_TOKEN`. The main work was on making `git annex drop --dead`
+prune obsolete per-remote metadata, and on fixing a bug in v6 mode that
+left git-annex object files writable.
+
+Today's work was sponsored by Paul Walmsley in honor of Mark Phillips.
diff --git a/doc/devblog/day_518__S3_versioning_finishing_touches.mdwn b/doc/devblog/day_518__S3_versioning_finishing_touches.mdwn
new file mode 100644
index 0000000000..57c79049cb
--- /dev/null
+++ b/doc/devblog/day_518__S3_versioning_finishing_touches.mdwn
@@ -0,0 +1,39 @@
+Got git-annex downloading versioned files from S3, without needing S3
+credentials. This makes a S3 special remote be equally capable as a
+git-annex repository exported over http, other than of course not including
+the git objects.
+
+An example of this new feature:
+
+	AWS_SECRET_ACCESS_KEY=... AWS_ACCESS_KEY_ID=...
+	git annex initremote s3 type=S3 public=yes exporttree=yes versioning=yes
+	git annex export --tracking master --to s3
+	git tag 1.0
+	# modify some files here
+	git annex sync --content s3
+
+And then in a clone without the credentials:
+
+	git annex enableremote s3
+	git checkout 1.0
+	git annex get somefile
+
+This is nice; I only wish it were supported by other special remotes.
+It seems that any special remote could be made to support it, but ones not
+supporting some kind of versioning would need to store each file twice,
+and many would also need each file to be uploaded to them twice. But
+perhaps there are others that do have a form of versioning.
+WebDAV for one has a versioning extension in RFC 3253.
+
+Also did a final review of a patch Antoine Beaupré is working on to
+backport the recent git-annex security fixes to debian oldstable,
+git-annex 5.20141125. He described the backport [in his blog](https://anarc.at/blog/2018-08-31-report/):
+
+> This time again, Haskell was nice to work with: by changing type
+> configurations and APIs, the compiler makes sure that everything works out and
+> there are no inconsistencies. This logic is somewhat backwards to what we are
+> used to: normally, in security updates, we avoid breaking APIs at all costs.
+> But in Haskell, it's a fundamental way to make sure the system is still
+> coherent.
+
+Today's work was sponsored by Trenton Cronholm [on Patreon](https://patreon.com/joeyh).
diff --git a/doc/devblog/day_518__S3_versioning_finishing_touches/comment_1_aed297f65b6e555f338fc622a8678c0f._comment b/doc/devblog/day_518__S3_versioning_finishing_touches/comment_1_aed297f65b6e555f338fc622a8678c0f._comment
new file mode 100644
index 0000000000..b4b7e84750
--- /dev/null
+++ b/doc/devblog/day_518__S3_versioning_finishing_touches/comment_1_aed297f65b6e555f338fc622a8678c0f._comment
@@ -0,0 +1,121 @@
+[[!comment format=mdwn
+ username="yarikoptic"
+ avatar="http://cdn.libravatar.org/avatar/f11e9c84cb18d26a1748c33b48c924b4"
+ subject="what to be done for previously exported instances?"
+ date="2018-09-10T17:03:16Z"
+ content="""
+OpenNeuro folks have been exporting their datasets for a while by now using older git-annex, which didn't have all the recent \"public S3\" support implemented.  What changes should be done on their end so git-annex could get those files (ATM --debug output and error message give no information on why access is failing)?  Here is an example repository/dataset:
+[[!format sh \"\"\"
+$> git clone https://github.com/OpenNeuroDatasets/ds001506 ; cd ds001506; 
+Cloning into 'ds001506'...                                                
+remote: Counting objects: 10911, done.
+remote: Compressing objects: 100% (6624/6624), done.
+remote: Total 10911 (delta 1820), reused 10911 (delta 1820), pack-reused 0
+Receiving objects: 100% (10911/10911), 1.16 MiB | 0 bytes/s, done.
+Resolving deltas: 100% (1820/1820), done.
+
+$> git annex info s3-PUBLIC                                              
+(merging origin/git-annex into git-annex...)
+(recording state in git...)
+download failed: Not Found
+
+  Remote origin not usable by git-annex; setting annex-ignore
+uuid: ca9b233b-7567-48b3-89c7-efe7f6a97d4a
+description: s3-PUBLIC
+remote annex keys: 1288
+remote annex size: 103.29 gigabytes (+ 632 unknown size)
+
+$> git annex info          
+repository mode: indirect
+trusted repositories: 0
+semitrusted repositories: 6
+	00000000-0000-0000-0000-000000000001 -- web
+ 	00000000-0000-0000-0000-000000000002 -- bittorrent
+ 	810b2f6b-fb98-4401-9130-6c84dd7ddc50 -- root@b3ba225d5547:/datalad/ds001506
+ 	8fdca7f0-84c3-4b1b-af9a-453effd5380c -- s3-PRIVATE
+ 	b2a25c10-6db6-4e7c-8490-b16837fff749 -- yoh@hopa:/tmp/ds001506 [here]
+ 	ca9b233b-7567-48b3-89c7-efe7f6a97d4a -- s3-PUBLIC
+untrusted repositories: 0
+transfers in progress: none
+available local disk space: 21.98 gigabytes (+1 megabyte reserved)
+local annex keys: 0
+local annex size: 0 bytes
+annexed files in working tree: 665
+size of annexed files in working tree: 104.83 gigabytes
+bloom filter size: 32 mebibytes (0% full)
+backend usage: 
+	MD5E: 665
+
+$> git annex enableremote s3-PUBLIC                                          
+
+enableremote s3-PUBLIC ok          
+(recording state in git...)
+
+$> git annex get --from s3-PUBLIC sub-01/ses-anatomy/anat/sub-01_ses-anatomy_T1w.nii.gz
+get sub-01/ses-anatomy/anat/sub-01_ses-anatomy_T1w.nii.gz (from s3-PUBLIC...) failed
+git-annex: get: 1 failed
+
+$> git annex get --debug --from s3-PUBLIC sub-01/ses-anatomy/anat/sub-01_ses-anatomy_T1w.nii.gz 
+[2018-09-10 12:53:35.937741872] read: git [\"--git-dir=.git\",\"--work-tree=.\",\"--literal-pathspecs\",\"show-ref\",\"git-annex\"]
+[2018-09-10 12:53:35.946999706] process done ExitSuccess
+[2018-09-10 12:53:35.947122637] read: git [\"--git-dir=.git\",\"--work-tree=.\",\"--literal-pathspecs\",\"show-ref\",\"--hash\",\"refs/heads/git-annex\"]
+[2018-09-10 12:53:35.951493681] process done ExitSuccess
+[2018-09-10 12:53:35.951694627] read: git [\"--git-dir=.git\",\"--work-tree=.\",\"--literal-pathspecs\",\"log\",\"refs/heads/git-annex..566bcf31d8d24f0e03df73d7c2196092e6fce39e\",\"--pretty=%H\",\"-n1\"]
+[2018-09-10 12:53:35.955844481] process done ExitSuccess
+[2018-09-10 12:53:35.956107478] chat: git [\"--git-dir=.git\",\"--work-tree=.\",\"--literal-pathspecs\",\"cat-file\",\"--batch\"]
+[2018-09-10 12:53:35.956674577] chat: git [\"--git-dir=.git\",\"--work-tree=.\",\"--literal-pathspecs\",\"cat-file\",\"--batch-check=%(objectname) %(objecttype) %(objectsize)\"]
+[2018-09-10 12:53:35.963580866] read: git [\"--git-dir=.git\",\"--work-tree=.\",\"--literal-pathspecs\",\"ls-files\",\"--cached\",\"-z\",\"--\",\"sub-01/ses-anatomy/anat/sub-01_ses-anatomy_T1w.nii.gz\"]
+get sub-01/ses-anatomy/anat/sub-01_ses-anatomy_T1w.nii.gz (from s3-PUBLIC...) failed
+[2018-09-10 12:53:35.991800587] process done ExitSuccess
+[2018-09-10 12:53:35.992039867] process done ExitSuccess
+git-annex: get: 1 failed
+
+$> git annex version
+git-annex version: 6.20180807+git230-gaa291acfe-1~ndall+1
+build flags: Assistant Webapp Pairing S3(multipartupload)(storageclasses) WebDAV Inotify DBus DesktopNotify ConcurrentOutput TorrentParser MagicMime Feeds Testsuite
+dependency versions: aws-0.19 bloomfilter-2.0.1.0 cryptonite-0.25 DAV-1.3.2 feed-1.0.0.0 ghc-8.2.2 http-client-0.5.12 persistent-sqlite-2.8.1.2 torrent-10000.1.1 uuid-1.3.13 yesod-1.6.0
+key/value backends: SHA256E SHA256 SHA512E SHA512 SHA224E SHA224 SHA384E SHA384 SHA3_256E SHA3_256 SHA3_512E SHA3_512 SHA3_224E SHA3_224 SHA3_384E SHA3_384 SKEIN256E SKEIN256 SKEIN512E SKEIN512 BLAKE2B256E BLAKE2B256 BLAKE2B512E BLAKE2B512 BLAKE2B160E BLAKE2B160 BLAKE2B224E BLAKE2B224 BLAKE2B384E BLAKE2B384 BLAKE2S256E BLAKE2S256 BLAKE2S160E BLAKE2S160 BLAKE2S224E BLAKE2S224 BLAKE2SP256E BLAKE2SP256 BLAKE2SP224E BLAKE2SP224 SHA1E SHA1 MD5E MD5 WORM URL
+remote types: git gcrypt p2p S3 bup directory rsync web bittorrent webdav adb tahoe glacier ddar hook external
+operating system: linux x86_64
+supported repository versions: 3 5 6
+upgrade supported from repository versions: 0 1 2 3 4 5
+local repository version: 5
+
+$> git annex whereis sub-01/ses-anatomy/anat/sub-01_ses-anatomy_T1w.nii.gz
+whereis sub-01/ses-anatomy/anat/sub-01_ses-anatomy_T1w.nii.gz (2 copies) 
+  	810b2f6b-fb98-4401-9130-6c84dd7ddc50 -- root@b3ba225d5547:/datalad/ds001506
+   	8fdca7f0-84c3-4b1b-af9a-453effd5380c -- s3-PRIVATE
+
+  The following untrusted locations may also have copies:
+  	ca9b233b-7567-48b3-89c7-efe7f6a97d4a -- [s3-PUBLIC]
+ok
+
+$> datalad ls s3://openneuro.org/ds001506/sub-01/ses-anatomy/anat/sub-01_ses-anatomy_T1w.nii.gz
+Connecting to bucket: openneuro.org                                                            
+[INFO   ] S3 session: Connecting to the bucket openneuro.org anonymously 
+Bucket info:
+  Versioning: S3ResponseError: 403 Forbidden
+     Website: S3ResponseError: 403 Forbidden
+         ACL: 
+ds001506/sub-01/ses-anatomy/anat/sub-01_ses-anatomy_T1w.nii.gz 2018-09-10T14:20:22.000Z 11697980 
+
+$> datalad ls --list-content md5 -L s3://openneuro.org/ds001506/sub-01/ses-anatomy/anat/sub-01_ses-anatomy_T1w.nii.gz 
+Connecting to bucket: openneuro.org
+[INFO   ] S3 session: Connecting to the bucket openneuro.org anonymously 
+Bucket info:
+  Versioning: S3ResponseError: 403 Forbidden
+     Website: S3ResponseError: 403 Forbidden
+         ACL: 
+ds001506/sub-01/ses-anatomy/anat/sub-01_ses-anatomy_T1w.nii.gz 2018-09-10T14:20:22.000Z 11697980 ver:snn1VMlJ8hgxd42_Dy1dXuiyBCwd00eZ  acl:AccessDenied  http://openneuro.org.s3.amazonaws.com/ds001506/sub-01/ses-anatomy/anat/sub-01_ses-anatomy_T1w.nii.gz?versionId=snn1VMlJ8hgxd42_Dy1dXuiyBCwd00eZ [OK] f8e1e6ec4efe752b86a7e5885d64e131
+
+$> ls -ld sub-01/ses-anatomy/anat/sub-01_ses-anatomy_T1w.nii.gz                            
+lrwxrwxrwx 1 yoh yoh 145 Sep 10 12:53 sub-01/ses-anatomy/anat/sub-01_ses-anatomy_T1w.nii.gz -> ../../../.git/annex/objects/fq/V7/MD5E-s11697980--f8e1e6ec4efe752b86a7e5885d64e131.nii.gz/MD5E-s11697980--f8e1e6ec4efe752b86a7e5885d64e131.nii.gz
+
+\"\"\"]]
+
+so file is available from S3, matches the checksum, but `git annex get` just fails to get it without giving any clarification on why.
+If I add `versioning=yes` to enableremote then it starts asking for the AWS credentials.
+
+Thank you in advance for looking into it.
+
+"""]]
diff --git a/doc/devblog/day_518__S3_versioning_finishing_touches/comment_2_a981371786ba702ac616cd55b296bc1c._comment b/doc/devblog/day_518__S3_versioning_finishing_touches/comment_2_a981371786ba702ac616cd55b296bc1c._comment
new file mode 100644
index 0000000000..3c7a5736e3
--- /dev/null
+++ b/doc/devblog/day_518__S3_versioning_finishing_touches/comment_2_a981371786ba702ac616cd55b296bc1c._comment
@@ -0,0 +1,39 @@
+[[!comment format=mdwn
+ username="joey"
+ subject="""comment 2"""
+ date="2018-09-11T16:17:09Z"
+ content="""
+What they'll need to do is:
+
+1. Enableremote with versioning=yes
+2. Manually update the git-annex branch location tracking
+   info for older versions of exported files so git-annex knows that the files
+   are still stored in S3. Ie, `git-annex setpresentkey`
+3. Add remote metadata log files to git-annex branch so that the S3 remote knows
+   the S3 version ID for all the already updated files.
+
+The third item is surely going to involve some work since they'll have to
+gather the S3 version IDs, match them up with git-annex keys for the file
+that was uploaded at that point, and manually generate files to
+commit to the git-annex branch. The rmet log file for S3 looks like:
+
+	timestamp uuid:V +versionid#filename
+
+For example, for a file that was uploaded to the bucket as "myfile":
+
+	1535737778.867692782s 31ea6c94-fba3-4952-99b5-285ae192d92a:V +woYHK59DD2VUkJfg527mEBBqtCaPlSXn#myfile
+
+Note that if the filename contains spaces it gets more complicated and it
+would probably be worth adding some plumbing command to git-annex to set
+per-remote metadata.
+
+----
+
+As to the problems you're having with that S3 remote, it looks like
+it does not have public=yes enabled in its configuration at all, which is
+why git-annex is failing to download from it. It kind of looks like you
+have some other S3 creds in the environment, otherwise I'd expect git-annex
+to complain that the remote is not public and no creds are set. Or maybe a
+bug in the handling of that case. devblog doesn't seem like the place to
+dig into that..
+"""]]
diff --git a/doc/devblog/day_518__S3_versioning_finishing_touches/comment_3_08de1605f1aa8b042ba3c4b710939989._comment b/doc/devblog/day_518__S3_versioning_finishing_touches/comment_3_08de1605f1aa8b042ba3c4b710939989._comment
new file mode 100644
index 0000000000..bca50a3620
--- /dev/null
+++ b/doc/devblog/day_518__S3_versioning_finishing_touches/comment_3_08de1605f1aa8b042ba3c4b710939989._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="yarikoptic"
+ avatar="http://cdn.libravatar.org/avatar/f11e9c84cb18d26a1748c33b48c924b4"
+ subject="comment 3"
+ date="2018-09-11T16:53:38Z"
+ content="""
+Thank you Joey!
+"""]]
diff --git a/doc/devblog/day_519__release_prep.mdwn b/doc/devblog/day_519__release_prep.mdwn
new file mode 100644
index 0000000000..bff6916bc7
--- /dev/null
+++ b/doc/devblog/day_519__release_prep.mdwn
@@ -0,0 +1,10 @@
+I'm in release prep mode now, fixing build problems and a few bugs, but
+the v6 sprint is well over, though v6 still has its issues.
+Might as well release all the last month's work.
+
+Yesterday was taken up with dealing with some very ugly git interface
+stuff that changed between versions. An July workaround for a bug in
+git turns out to have caused reversions with older versions, and was not
+a complete fix either. Tuned it to hopefully work better.
+
+This was sponsored by Jake Vosloo [on Patreon](https://patreon.com/joeyh).
diff --git a/doc/devblog/day_51__direct_mode_guard_finished.mdwn b/doc/devblog/day_51__direct_mode_guard_finished.mdwn
new file mode 100644
index 0000000000..9c637aa949
--- /dev/null
+++ b/doc/devblog/day_51__direct_mode_guard_finished.mdwn
@@ -0,0 +1,6 @@
+Finished the direct mode guard, including the new `git annex status`
+command.
+
+Spent the rest of the day working on various bug fixes. One of them turned
+into rather a lot of work to make the webapp's UI better for git
+remotes that do not have an annex.uuid.
diff --git a/doc/devblog/day_520__storm_before_the_calm.mdwn b/doc/devblog/day_520__storm_before_the_calm.mdwn
new file mode 100644
index 0000000000..62acfd0d8f
--- /dev/null
+++ b/doc/devblog/day_520__storm_before_the_calm.mdwn
@@ -0,0 +1,10 @@
+Well it took the whole day to finish the release. Including fixing a
+deadlock when the new v6 code runs with an older git, and some build
+errors. Ant there's still an intermittent test suite failure involving v6 on
+one autobuilder, which will need to be dealt with later.
+
+This is a big release, lots of bug fixes, lots of v6 improvements, and
+significant S3 improvements.
+
+Today's work was sponsored by Paul Walmsley 
+[on Patreon](https://patreon.com/joeyh).
diff --git a/doc/devblog/day_521__newlines_in_filenames.mdwn b/doc/devblog/day_521__newlines_in_filenames.mdwn
new file mode 100644
index 0000000000..1eccb59bfe
--- /dev/null
+++ b/doc/devblog/day_521__newlines_in_filenames.mdwn
@@ -0,0 +1,21 @@
+Unix would be better if filenames could not contain newlines. But they can,
+and so today was spent dealing with some technical debt.
+
+The main problem with using git-annex with filenames with newlines is that
+`git cat-file --batch` uses a line-based protocol. It would be nice if that
+were extended to support `-z` like most of the rest of git does, but I
+realized I could work around this by not using batch mode for the rare
+filename with a newline. Handling such files will be slower than other
+files, but at least it will work.
+
+Then I realized that git-annex has its own problems with its `--batch`
+option and files with newlines. So I added support for `-z` to every batchable
+command in git-annex, including a couple of commands that did batch input
+without a `--batch` option.
+
+Now git-annex should fully support filenames containing newlines, as well 
+as anything else. The best thing to do if you have such a file is to
+commit it and then `git mv` it to a better name.
+
+Today's work was sponsored by Trenton Cronholm 
+[on Patreon](https://patreon.com/joeyh).
diff --git a/doc/devblog/day_522__multi.mdwn b/doc/devblog/day_522__multi.mdwn
new file mode 100644
index 0000000000..99b3dc01bb
--- /dev/null
+++ b/doc/devblog/day_522__multi.mdwn
@@ -0,0 +1,7 @@
+Started work on .
+It's going slow, I had to start with a large refactoring. So far, option
+parsing is working, and a few commands are almost working, but concurrency
+is not working right, and concurrency is the main reason to want to support
+this (along with remote groups).
+
+Today's work was supported by Jake Vosloo [on Patreon](https://patreon.com/joeyh).
diff --git a/doc/devblog/day_52__slowly_but_surely.mdwn b/doc/devblog/day_52__slowly_but_surely.mdwn
new file mode 100644
index 0000000000..01019355bd
--- /dev/null
+++ b/doc/devblog/day_52__slowly_but_surely.mdwn
@@ -0,0 +1,5 @@
+Been chipping away at my backlog of messages, and it's down to 23 items.
+
+Finally managed to get ghc to build with a newer version of the NDK.
+This *might* mean a solution to git-annex on Android 4.2. I 
+[need help with testing](http://git-annex.branchable.com/bugs/git-annex_broken_on_Android_4.3/#comment-90b82735cd6090a7765f423b743fffd3).
diff --git a/doc/devblog/day_54__android_bisection_minions.mdwn b/doc/devblog/day_54__android_bisection_minions.mdwn
new file mode 100644
index 0000000000..ed86c68d07
--- /dev/null
+++ b/doc/devblog/day_54__android_bisection_minions.mdwn
@@ -0,0 +1,9 @@
+Finally found the [root cause](http://git-annex.branchable.com/bugs/git-annex_broken_on_Android_4.3/#comment-452bee7d0a816300ccb4a34f9758134e)
+of the Android 4.3/4.4 trouble, and a fix is now in place!
+
+As a bonus, it looks like I've fixed a problem accessing the
+environment on Android that had been worked around in an ugly way before.
+
+Big thanks to my remote hands Michael Alan, Sören, and subito. All
+told they ran 19 separate tests to help me narrow down this tricky
+problem, often repeating long command lines on software keyboards.
diff --git a/doc/devblog/day_54__android_bisection_minions/comment_1_bea8fbe2b87d4a4865b92fa796298fa0._comment b/doc/devblog/day_54__android_bisection_minions/comment_1_bea8fbe2b87d4a4865b92fa796298fa0._comment
new file mode 100644
index 0000000000..0f8e588273
--- /dev/null
+++ b/doc/devblog/day_54__android_bisection_minions/comment_1_bea8fbe2b87d4a4865b92fa796298fa0._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="https://www.google.com/accounts/o8/id?id=AItOawmW0kg4uiMIhSHeVuvJFyo2VYMl7Qoej0s"
+ nickname="Chris"
+ subject="Woot!"
+ date="2013-11-12T15:52:48Z"
+ content="""
+Thanks for getting this fixed.  I'm about to go on a long plane ride, and this will make it a lot easier for me to put movies on my tablet.
+"""]]
diff --git a/doc/devblog/day_55__fireside_porting.mdwn b/doc/devblog/day_55__fireside_porting.mdwn
new file mode 100644
index 0000000000..c7469d953b
--- /dev/null
+++ b/doc/devblog/day_55__fireside_porting.mdwn
@@ -0,0 +1,22 @@
+Annoyingly, the Android 4.3 fix breaks git-annex on Android 4.0 (probably
+through 4.2), so I now have two separate builds of the Android app.
+
+---
+
+Worked on Windows porting today. I've managed to get the assistant
+and watcher (but not yet webapp) to build on Windows.
+The `git annex transferrer` interface needs POSIX stuff, and seems to be
+the main thing that will need porting for Windows for the assistant to
+work, besides of course file change detection. For that, I've hooked up
+[Win32-notify](http://hackage.haskell.org/package/Win32-notify).
+
+So the watcher might work on Windows. 
+At least in theory. Problem is, while all the code builds ok, 
+it fails to link:
+
+	ghc.exe: could not execute: C:\Program Files (x86)\Haskell Platform\2012.4.0.0\lib/../mingw/bin/gcc.exe
+
+I wonder if this is case of too many parameters being passed?
+
+This happens both on the autobuilder and on my laptop, so I'm stuck here.
+Oh well, I was not planning to work on this anyway until February...
diff --git a/doc/devblog/day_55__fireside_porting/comment_1_d690a52db82f9594d99ae65fe51e1f1a._comment b/doc/devblog/day_55__fireside_porting/comment_1_d690a52db82f9594d99ae65fe51e1f1a._comment
new file mode 100644
index 0000000000..74acc83221
--- /dev/null
+++ b/doc/devblog/day_55__fireside_porting/comment_1_d690a52db82f9594d99ae65fe51e1f1a._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="http://nullroute.eu.org/~grawity/"
+ nickname="Mantas"
+ subject="comment 1"
+ date="2013-11-13T06:22:49Z"
+ content="""
+[ProcExp](http://technet.microsoft.com/en-us/sysinternals/bb896645.aspx) may be useful -- it can show process-related syscalls and their return codes _(among other things like file or registry access)_ so it should tell exactly why the exec failed.
+"""]]
diff --git a/doc/devblog/day_56__git-annex_user_survey.mdwn b/doc/devblog/day_56__git-annex_user_survey.mdwn
new file mode 100644
index 0000000000..8c1c67994e
--- /dev/null
+++ b/doc/devblog/day_56__git-annex_user_survey.mdwn
@@ -0,0 +1,20 @@
+One of my goals for this month is to get a better sense of how git-annex is
+being used, how it's working out for people, and what areas need to be
+concentrated on. To start on that, I am doing the 
+[2013 git-annex user survey](http://git-annex-survey.branchable.com/polls/2013/), similar to the git user
+surveys. I will be adding some less general polls later (suggestions for
+topics appreciated!), but you can go vote in any or all of 10 polls now.
+
+----
+
+Found a workaround for yesterday's Windows build problem. Seems that only
+cabal runs gcc in a way that fails, so `ghc --make` builds is successfully.
+However, the watcher doesn't quite work on Windows. It does get events when
+files are created, but it seems to then hang before it can add the file to
+git, or indeed finish printing out a debug log message about the event.
+This looks like it could be a problem with the threaded ghc runtime on
+Windows, or something like that.
+
+Main work today was improving the git repository repair to handle corrupt
+index files. The assistant can now start up, detect that the index file is
+corrupt, and regenerate it all automatically.
diff --git a/doc/devblog/day_57__mavericks.mdwn b/doc/devblog/day_57__mavericks.mdwn
new file mode 100644
index 0000000000..1031895bed
--- /dev/null
+++ b/doc/devblog/day_57__mavericks.mdwn
@@ -0,0 +1,14 @@
+The user survey is producing some interesting and useful results!  
+Added two more polls: [using with](http://git-annex-survey.branchable.com/polls/2013/using_with/) and [blocking problems](http://git-annex-survey.branchable.com/polls/2013/blocking_problems/)  
+(There were some load issues so if you were unable to vote yesterday, try
+again..)
+
+Worked on getting the autobuilder for OS X Mavericks set up. Eventually
+succeeded, after patching a few packages to work around a cpp that thinks
+it should parse haskell files as if they're C code.
+Also, Jimmy has resuscitated the OS X Lion autobuilder.
+
+A not too bad bug in automatic merge conflict resolution has been reported,
+so I will need to dig into that tomorrow. Didn't feel up to it today, so
+instead have been spending the remaining time finishing up a branch that
+switches the test suite to use the tasty test framework.
diff --git a/doc/devblog/day_58__urgle.mdwn b/doc/devblog/day_58__urgle.mdwn
new file mode 100644
index 0000000000..f6d930d806
--- /dev/null
+++ b/doc/devblog/day_58__urgle.mdwn
@@ -0,0 +1,16 @@
+Fixed two difficult bugs with direct mode. One happened (sometimes) when a
+file was deleted and replaced with a directory by the same name and then
+those changes were merged into a direct mode repository.
+
+The other problem was that direct mode did not prevent writes to
+.git/annex/objects the way that indirect mode does, so when a file in the
+repository was not currently present, writing to the dangling symlink would
+follow it and write into the object directory.
+
+Hmm, I was going to say that it's a pity that direct mode still has so many
+bugs being found and fixed, but the last real bug fix to direct mode was
+made last May! Instead, I probably have to thank Tim for being a very
+thorough tester.
+
+Finished switching the test suite to use the tasty framework, and prepared
+tasty packages for Debian.
diff --git a/doc/devblog/day_58__urgle/comment_1_bd279f58f614b103a53215dfb0211007._comment b/doc/devblog/day_58__urgle/comment_1_bd279f58f614b103a53215dfb0211007._comment
new file mode 100644
index 0000000000..0fec5fa8c9
--- /dev/null
+++ b/doc/devblog/day_58__urgle/comment_1_bd279f58f614b103a53215dfb0211007._comment
@@ -0,0 +1,12 @@
+[[!comment format=mdwn
+ username="https://www.google.com/accounts/o8/id?id=AItOawkx5V3MTbzCXS3J7Mn9FEq8M9bPPYMkAHY"
+ nickname="Pedro"
+ subject="Is direct mode still using symlinks then?"
+ date="2013-11-16T16:28:39Z"
+ content="""
+Is direct mode still using symlinks? I see the bug I opened is still unclosed:
+
+http://git-annex.branchable.com/bugs/Direct_mode_repositories_still_use_symlinks_sometimes/
+
+Any opinion on this? This is still my main problem with replacing dropbox with git-annex.
+"""]]
diff --git a/doc/devblog/day_59__release_day.mdwn b/doc/devblog/day_59__release_day.mdwn
new file mode 100644
index 0000000000..469ccec445
--- /dev/null
+++ b/doc/devblog/day_59__release_day.mdwn
@@ -0,0 +1,11 @@
+Release today, right on bi-weekly schedule. Rather startled
+at the size of the changelog for this one; along with the direct mode
+guard, it adds support for OS X Mavericks, Android 4.3/4.4, and fixes
+numerous bugs.
+
+Posted another question in the survey,
+.
+
+Spun off git-repair as an independant package from git-annex. Of course,
+most of the source code is shared with git-annex. I need to do something
+with libraries eventually..
diff --git a/doc/devblog/day_5__gcrypt_special_remote_part_1.mdwn b/doc/devblog/day_5__gcrypt_special_remote_part_1.mdwn
new file mode 100644
index 0000000000..626b5edaf9
--- /dev/null
+++ b/doc/devblog/day_5__gcrypt_special_remote_part_1.mdwn
@@ -0,0 +1,7 @@
+About half way done with a gcrypt special remote. I can initremote it (the
+hard part to get working), and can send files to it. Can't yet get files
+back, or remove files, and only local repositories work so far, but this is
+enough to know it's going to be pretty nice!
+
+Did find one issue in gcrypt that I may need to develop a patch for:
+
diff --git a/doc/devblog/day_60__damage_driven_development.mdwn b/doc/devblog/day_60__damage_driven_development.mdwn
new file mode 100644
index 0000000000..64460e86a9
--- /dev/null
+++ b/doc/devblog/day_60__damage_driven_development.mdwn
@@ -0,0 +1,36 @@
+Wrote some evil code you don't want to run today. Git.Destroyer randomly
+generates Damage, and applies it to a git repository, in a way that is
+reproducible -- applying the same Damage to clones of the same git repo 
+will always yeild the same result.
+
+This let me build a test harness for git-repair, which repeatedly clones,
+damages, and repairs a repository. And when it fails, I can just ask it to
+retry after fixing the bug and it'll re-run every attempt it's logged.
+
+This is already yeilding improvements to the git-repair code.
+The first randomly constructed Damage that it failed to recover
+turned out to be a truncated index file that hid some other
+corrupted object files from being repaired.
+
+	[Damage Empty (FileSelector 1),
+	 Damage Empty (FileSelector 2),
+	 Damage Empty (FileSelector 3),
+	 Damage Reverse (FileSelector 3),
+	 Damage (ScrambleFileMode 3) (FileSelector 5),
+	 Damage Delete (FileSelector 9),
+	 Damage (PrependGarbage "¥SOH¥STX¥ENQ¥f¥a¥ACK¥b¥DLE¥n") (FileSelector 9),
+	 Damage Empty (FileSelector 12),
+	 Damage (CorruptByte 11 25) (FileSelector 6),
+	 Damage Empty (FileSelector 5),
+	 Damage (ScrambleFileMode 4294967281) (FileSelector 14)
+	]
+
+I need to improve the ranges of files that it damages -- currently QuickCheck
+seems to only be selecting one of the first 20 or so files. Also, it's quite
+common that it will damage `.git/config` so badly that git thinks it's not
+a git repository anymore. I am not sure if that is something `git-repair`
+should try to deal with.
+
+---
+
+Today's work was sponsored by the WikiMedia Foundation.
diff --git a/doc/devblog/day_61__damage_driven_development__II.mdwn b/doc/devblog/day_61__damage_driven_development__II.mdwn
new file mode 100644
index 0000000000..605a3878d8
--- /dev/null
+++ b/doc/devblog/day_61__damage_driven_development__II.mdwn
@@ -0,0 +1,15 @@
+Pushed out a minor release of git-annex today, mostly to fix build problems
+on Debian. No strong reason to upgrade to it otherwise.
+
+Continued where I left off with the Git.Destroyer. Fixed quite a lot of
+edge cases where git repair failed due to things like a corrupted .git/HEAD
+file (this makes git think it's not in a git repository), corrupt
+git objects that have an unknown object type and so crash git hard, and
+an interesting failure mode where git fsck wants to allocate 116 GB of
+memory due to a corrupted object size header. Reported that last to the git
+list, as well as working around it.
+
+At the end of the day, I ran a test creating 10000 corrupt git
+repositories, and **all** of them were recovered! Any improvements will
+probably involve finding new ways to corrupt git repositories that my code
+can't think of. ;)
diff --git a/doc/devblog/day_62__upgrade_alerts.mdwn b/doc/devblog/day_62__upgrade_alerts.mdwn
new file mode 100644
index 0000000000..0b30785631
--- /dev/null
+++ b/doc/devblog/day_62__upgrade_alerts.mdwn
@@ -0,0 +1,22 @@
+Still working on the git repair code. Improved the test suite, which found
+some more bugs, and so I've been running tests all day and occasionally
+going and fixing a bug in the repair code. The hardest part of repairing a
+git repo has turned out to be reliably determining which objects in it are
+broken. Bugs in git don't help (but the git devs are going to fix the one I
+reported).
+
+But the interesting new thing today is that I added some upgrade alert code
+to the webapp. Ideally everyone would get git-annex and other software as
+part of an OS distribution, which would include its own upgrade system -- 
+But the  [survey](http://git-annex-survey.branchable.com/polls/2013/how_installed/)
+tells me that a quarter of installs are from the prebuilt binaries I
+distribute.
+
+So, those builds are going to be built with knowledge of an upgrade url,
+and will periodically download a small info file (over https) to see if a
+newer version is available, and show an alert.
+
+I think all that's working, though I have not yet put the info files in
+place and tested it. The actual upgrade process will be a manual
+download and reinstall, to start with, and then perhaps I'll automate it
+further, depending on how hard that is on the different platforms.
diff --git a/doc/devblog/day_62__upgrade_alerts/comment_1_cdb44aaa1d2a784a72613cbf16038f89._comment b/doc/devblog/day_62__upgrade_alerts/comment_1_cdb44aaa1d2a784a72613cbf16038f89._comment
new file mode 100644
index 0000000000..cb0bfc08b8
--- /dev/null
+++ b/doc/devblog/day_62__upgrade_alerts/comment_1_cdb44aaa1d2a784a72613cbf16038f89._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="https://www.google.com/accounts/o8/id?id=AItOawniayrgSdVLUc3c6bf93VbO-_HT4hzxmyo"
+ nickname="Tobias"
+ subject="gitannex-install"
+ date="2013-11-22T07:13:02Z"
+ content="""
+To install and upgrade the prebuilt binary on Linux I use the fantastic [gitannex-install](https://github.com/zerodogg/scriptbucket/blob/master/gitannex-install) script, together with the upgrade alert this is just great.
+"""]]
diff --git a/doc/devblog/day_62__upgrade_alerts/comment_2_b08bb946e4760d7f03b45c852c745b2e._comment b/doc/devblog/day_62__upgrade_alerts/comment_2_b08bb946e4760d7f03b45c852c745b2e._comment
new file mode 100644
index 0000000000..3a5ec5b7db
--- /dev/null
+++ b/doc/devblog/day_62__upgrade_alerts/comment_2_b08bb946e4760d7f03b45c852c745b2e._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="http://id.clacke.se/"
+ nickname="clacke"
+ subject="git-annex release annex"
+ date="2013-11-25T06:15:23Z"
+ content="""
+Immediate thought: instead of an info file, why not sync a public git repo with releases annexed with URLs?
+"""]]
diff --git a/doc/devblog/day_63__leverage.mdwn b/doc/devblog/day_63__leverage.mdwn
new file mode 100644
index 0000000000..6cf8acfa25
--- /dev/null
+++ b/doc/devblog/day_63__leverage.mdwn
@@ -0,0 +1,24 @@
+The difference picking the right type can make! Last night, I realized that
+the where I had a `distributionSha256sum :: String`, I should instead use
+`distributionKey :: Key`. This means that when git-annex is eventually
+downloading an upgrade, it can treat it as just another Key being
+downloaded from the web. So the webapp will show that transfer along with
+all the rest, and I can leverage tons of code for a new purpose. For
+example, it can simply fsck the key once it's downloaded to verify its
+checksum.
+
+Also, built a DistriutionUpdate program, which I'll run to generate the
+info files for a new version. And since I keep git-annex releases in a
+git-annex repo, this too leverages a lot of git-annex modules, and ended up
+being just 60 easy lines of code. The upgrade notification code is tested
+and working now.
+
+And, I made the assistant detect when the git-annex program binary is
+replaced or modified. Used my existing DirWatcher code for that. The plan
+is to restart the assistant on upgrade, although I need to add some sanity
+checks (eg, reuse the lsof code) first. And yes, this will work even for
+`apt-get upgrade`!
+
+----
+
+Today's work was sponsored by Paul Tötterman
diff --git a/doc/devblog/day_64__overkill.mdwn b/doc/devblog/day_64__overkill.mdwn
new file mode 100644
index 0000000000..fd5d66f4ec
--- /dev/null
+++ b/doc/devblog/day_64__overkill.mdwn
@@ -0,0 +1,31 @@
+Completely finished up with making the assistant detect when git-annex's
+binary has changed and handling the restart.
+
+It's a bit tricky because during an upgrade there can be two assistant
+daemons running at the same time, in the same repository. Although I
+disable the watcher of the old one first. Luckily, git-annex has long
+supported running multiple concurrent git-annex processes in the same
+repository.
+
+The surprisingly annoying part turned out to be how to make the webapp
+redirect the browser to the new url when it's upgraded. Particularly needed
+when automatic upgrades are enabled, since the user will not then be taking
+any action in the webapp that could result in a redirect. My solution to this
+feels like overkill; the webapp does ajax long polling until it gets an
+url, and then redirects to it. Had to write javascript code and ugh.
+
+But, that turned out to also be useful when manually restarting the webapp
+(removed some horrible old code that ran a shell script to do it before),
+and also when shutting the webapp down.
+
+[[!img assistant/downloadupgrade.png alt="assistant downloading an upgrade to itself"]]
+
+Getting back to upgrades, I have the assistant downloading the upgrade, and
+running a hook action once the key is transferred. Now all I need is some
+platform-specific code to install it. Will probably be hairy, especially on
+OSX where I need to somehow unmount the old git-annex dmg and mount the new
+one, from within a program running on the old dmg.
+
+----
+
+Today's work was sponsored by Evan Deaubl.
diff --git a/doc/devblog/day_64__overkill/comment_1_e1db7678aae37af281d31ae211677786._comment b/doc/devblog/day_64__overkill/comment_1_e1db7678aae37af281d31ae211677786._comment
new file mode 100644
index 0000000000..b04af39ffb
--- /dev/null
+++ b/doc/devblog/day_64__overkill/comment_1_e1db7678aae37af281d31ae211677786._comment
@@ -0,0 +1,10 @@
+[[!comment format=mdwn
+ username="abg"
+ ip="184.75.210.234"
+ subject="Upgrade installation in OSX"
+ date="2013-11-24T00:28:03Z"
+ content="""
+Hey,
+
+You don't have to implement any whacky DMG replacement logic for OSX upgrades. No sane user will be using the application from the DMG. They will drag the .app directory to /Applications or some non-standard location and unmount the DMG.  So your upgrade logic for OSX should end up looking very similar to the upgrade logic for Linux.
+"""]]
diff --git a/doc/devblog/day_64__overkill/comment_3_f7a96f0b6d942d0b59d9d0ec1b21c4bf._comment b/doc/devblog/day_64__overkill/comment_3_f7a96f0b6d942d0b59d9d0ec1b21c4bf._comment
new file mode 100644
index 0000000000..c9e76b577e
--- /dev/null
+++ b/doc/devblog/day_64__overkill/comment_3_f7a96f0b6d942d0b59d9d0ec1b21c4bf._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="https://www.google.com/accounts/o8/id?id=AItOawmuh_0-Mjz7QkVz7gT_7PRpCgmMcBFkj14"
+ nickname="Wichert"
+ subject="Running from DMG is not a sane use-case"
+ date="2013-11-25T08:23:38Z"
+ content="""
+I don't think you should bother trying to support upgrades when people are running from a dmg. A DMG is just a disk image, and a format commonly used to distribute software. Running software from a DMG is not something anyone does beyond a quick first test of an application. A more reasonable thing to do, which many applications do now, is to offer to move the application from the DMG to /Applications if you detect someone is running it from a DMG.
+"""]]
diff --git a/doc/devblog/day_65__wrapping_up_upgrades.mdwn b/doc/devblog/day_65__wrapping_up_upgrades.mdwn
new file mode 100644
index 0000000000..9faa31a63c
--- /dev/null
+++ b/doc/devblog/day_65__wrapping_up_upgrades.mdwn
@@ -0,0 +1,5 @@
+[[!img assistant/upgradecomplete.png]]
+
+Upgrades are fully working on Linux. OSX code is written but intested and I
+thought of one bug it certainly has on my evening walk. Probably another
+hour's work left later this evening to finish it off.
diff --git a/doc/devblog/day_66__upgrade_testing.mdwn b/doc/devblog/day_66__upgrade_testing.mdwn
new file mode 100644
index 0000000000..741ec494ee
--- /dev/null
+++ b/doc/devblog/day_66__upgrade_testing.mdwn
@@ -0,0 +1,17 @@
+Upgrades should be working on OSX Mavericks, Linux, and sort of on Android.
+This needs more testing, so I have temporarily made the daily builds think
+they are an older version than the last git-annex release. So when you
+install a daily build, and start the webapp, it should try to upgrade
+(really downgrade) to the last release. Tests appreciated.
+
+Looking over the whole upgrade code base, it took 700 lines of code
+to build the whole thing, of which 75 are platform specific (and mostly
+come down to just 3 or 4 shell commands). Not bad..
+
+----
+
+Last night, added support for quvi 0.9, which has a completely changed
+command line interface from the 0.4 version.
+
+Plan to spend tomorrow catching up on bug reports etc and then low activity
+for rest of the week.
diff --git a/doc/devblog/day_67_thanksgiving_rush.mdwn b/doc/devblog/day_67_thanksgiving_rush.mdwn
new file mode 100644
index 0000000000..7ac6ba523f
--- /dev/null
+++ b/doc/devblog/day_67_thanksgiving_rush.mdwn
@@ -0,0 +1,19 @@
+My last day before thanksgiving, getting caught up with some recent bug
+reports and, quite a rush to get a lot of fixes in. Adding to the fun,
+wintery weather means very limited power today.
+
+It was a very productive day, especially for Android, which hopefully has
+XMPP working again (at least it builds..), halved the size of the package,
+etc.
+
+Fixed a stupid bug in the automatic v5 upgrade code; annex.version was not
+being set to 5, and so every git annex command was
+actually re-running the upgrade.
+
+Fixed another bug I introduced last Friday, which the test suite luckily
+caught, that broke using some local remotes in direct mode.
+
+Tracked down a behavior that makes `git annex sync` quite slow on
+filesystems that don't support symlinks. I need to switch direct mode to 
+not using `git commit` at all, and use plumbing to make commits there.
+Will probably work on this over the holiday.
diff --git a/doc/devblog/day_68__bits_and_pieces.mdwn b/doc/devblog/day_68__bits_and_pieces.mdwn
new file mode 100644
index 0000000000..1caa6e38bb
--- /dev/null
+++ b/doc/devblog/day_68__bits_and_pieces.mdwn
@@ -0,0 +1,14 @@
+Made a release yesterday to fix a bug that made git-annex init in a bare
+repository set core.bare=false. This bug only affected git-annex 5, it
+was introduced when building the direct mode guard. Currently recovering
+from it is a [manual (pretty easy) process](http://git-annex.branchable.com/bugs/assistant_creating_.git_directory_inside_bare_repo/#comment-73a8ce8aa100baa7c03861b769fdca29).
+Perhas I should automate that, but I mostly wanted to get a fix out
+before too many people encountered the bug.
+
+Today, I made the assistant run batch jobs with ionice and nocache, when
+those commands are available. Also, when the assistant transfers files,
+that also runs as a batch job.
+
+Changed how git-annex does commits, avoiding using `git commit` in direct
+mode, since in some situations `git commit` (not with `-a`!) wants to
+read the contents of files in the work tree, which can be very slow.
diff --git a/doc/devblog/day_69__catching_up.mdwn b/doc/devblog/day_69__catching_up.mdwn
new file mode 100644
index 0000000000..45b1e38e6c
--- /dev/null
+++ b/doc/devblog/day_69__catching_up.mdwn
@@ -0,0 +1,14 @@
+Still working through thanksgiving backlog. Around 55 messages to go.
+
+Wrote hairy code to automatically fix up bad bare repositories created by
+recent versions of git-annex. Managed to do it with only 1 stat call
+overhead (per local repository). Will probably keep that code in git-annex
+for a year or so, despite the bug only being present for a few weeks,
+because the repositories that need to be fixed might be on removable drives
+that are rarely used.
+
+Various other small bug fixes, including dealing with box.com having
+changed their WebDAV endpoint url.
+
+Spent a while evaluating various key/value storage possibilities. 
+[[bugs/incremental_fsck_should_not_use_sticky_bit]] has the details.
diff --git a/doc/devblog/day_6__gcrypt_fully_working.mdwn b/doc/devblog/day_6__gcrypt_fully_working.mdwn
new file mode 100644
index 0000000000..58abdbb602
--- /dev/null
+++ b/doc/devblog/day_6__gcrypt_fully_working.mdwn
@@ -0,0 +1,8 @@
+gcrpyt is fully working now. *Most* of the examples in
+[[tips/fully_encrypted_git_repositories_with_gcrypt]] should work.
+
+A few known problems:
+
+* `git annex sync` refuses to sync with gcrypt remotes. some url parsing issue.
+* Swapping two drives with gcrypt repositories on the same mount point doesn't work yet.
+* http urls are not supported
diff --git a/doc/devblog/day_6__gcrypt_fully_working/comment_1_136bb7537a9ba93d400ce6f6ea1932ac._comment b/doc/devblog/day_6__gcrypt_fully_working/comment_1_136bb7537a9ba93d400ce6f6ea1932ac._comment
new file mode 100644
index 0000000000..adb298fd33
--- /dev/null
+++ b/doc/devblog/day_6__gcrypt_fully_working/comment_1_136bb7537a9ba93d400ce6f6ea1932ac._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="spwhitton"
+ ip="82.36.235.9"
+ subject="Converting an rsync special remote to a gcrypt special remote"
+ date="2013-09-11T17:36:09Z"
+ content="""
+I'm guessing there is no way to convert an rsync special remote to a gcrypt special remote?  It would be cool not to have to upload 100GB across the Atlantic again!
+"""]]
diff --git a/doc/devblog/day_6__gcrypt_fully_working/comment_2_1f8faa65bbd56a12588b43a5bc822d96._comment b/doc/devblog/day_6__gcrypt_fully_working/comment_2_1f8faa65bbd56a12588b43a5bc822d96._comment
new file mode 100644
index 0000000000..a6859cf2f7
--- /dev/null
+++ b/doc/devblog/day_6__gcrypt_fully_working/comment_2_1f8faa65bbd56a12588b43a5bc822d96._comment
@@ -0,0 +1,10 @@
+[[!comment format=mdwn
+ username="http://joeyh.name/"
+ ip="4.154.4.51"
+ subject="comment 2"
+ date="2013-09-12T16:53:02Z"
+ content="""
+Well, rsync and gcrypt use the same locations for annexed content, so it's theoretically possible.
+
+However, it seems a lot easier to just add a gcrypt remote and let git-annex use the existing rsync remote for the content that is already stored in it.
+"""]]
diff --git a/doc/devblog/day_70__preliminary_user_survey_analysis.mdwn b/doc/devblog/day_70__preliminary_user_survey_analysis.mdwn
new file mode 100644
index 0000000000..0202d6c4af
--- /dev/null
+++ b/doc/devblog/day_70__preliminary_user_survey_analysis.mdwn
@@ -0,0 +1,104 @@
+The [2013 git-annex user survey](http://git-annex-survey.branchable.com/polls/2013/)
+has been running for several weeks and around 375 people have answered
+at least the first question. While I am going to leave it up through the
+end of the year, I went over the data today to see what interesting
+preliminary conclusions I can draw.
+
+* 11% build git-annex from source. More than I would have guessed.
+
+* 20% use the prebuilt versions from the git-annex website.
+  
+  This is a number to keep in mind later, when more people have upgraded to
+  the last release, which checks for upgrades. I can run some stats on
+  the number of upgrade checks I receive, and multiplying that by 5 would
+  give a good approximation of the total number of computers running
+  git-annex.
+
+* I'm surprised to see so many more Linux (79%) than OSX (15%) users.
+  Also surprising is there are more Windows (2%) than Android (1%) users.
+  (Android numbers may be artificially low since many users will use it in
+  addition to one of the other OSes.)
+
+* Android and Windows unsurprisingly lead in ports requested, but the 
+  Synology NAS is a surprise runner up, with 5% (more than IOS).
+
+  In theory it would not be too hard to make a standalone arm tarball,
+  which could be used on such a device, although IIRC the Synology had
+  problems with a too old linker and libc. It would help if I could make
+  the standalone tarball not depend on the system linker at all.
+  
+  A susprising number (3%) want some kind of port the the Raspberry Pi, which
+  is weird because I'd think they'd just be using Raspbian on it.. but a
+  standalone arm tarball would also cover that use case.
+
+* A minimum of 1664 (probably closer to 2000) git annex repositories are being
+  used by the 248 people who answered that question. Around 7 repositories
+  per person average, which is either one repository checked out on 7
+  different machines or two repositories on 3 machines, etc.
+
+* At least 143 terabytes of data are being stored in git-annex. This does
+  not count redundant data. (It also excludes several hundred terabytes from
+  one instituion that I don't think are fully online yet.)
+  Average user has more than half a terabyte of data.
+
+* 8% of users store scientific data in git-annex! :) A couple of users are
+  using it for game development assets, and 5% of users are using it for
+  some form of business data.
+
+* Only 10% of users are sharing a git-annex repository with at least one
+  other person. 27% use it by themselves, but want to get others using
+  their repositories. This probably points to it needing to be easier for
+  nontechnical users.
+
+* 61% of git-annex users have good or very good knowledge of git.
+  This question intentionally used the same wording as the 
+  [general git user survey](https://git.wiki.kernel.org/index.php/GitSurvey2012),
+  so the results can be compared. The curves have somewhat different
+  shapes, with git-annex's users being biased more toward the higher
+  knowledge levels than git's users.
+
+* The question about how happy users are also used the same wording.
+  While 74% of users are happy with git-annex, 94% are similarly happy with
+  git, and a while the median git-annex user is happy, the median git user
+  is very happy.
+  
+  The 10% who wrote in "very enthusiastic, but still often
+  bitten by quirks (so not very happy yet, but with lots of confidence in
+  the potential" might have thrown off this comparison some, but they
+  certianly made their point!
+
+* 3% of respondants say that a bug is preventing them from using git-annex,
+  but that they have not reported the bug yet. Frustrating! 1% say that a
+  bug that's been reported already is blocking them.
+
+* 18% wrote in that they need the webapp to support using github (etc)
+  as a central server. I've been moving in that direction with the
+  encryption and some other changes, so it's probably time to make a UI for
+  that.
+
+* 12% want more control over which files are stored locally when using the
+  assistant.
+
+* A really surprising thing happened when someone wrote in that I should
+  work on "not needing twice disk space of repo in direct mode", and 5% of
+  people then picked this choice. This is some kind of documentation
+  problem, because of course git-annex never needs 2x disk space, whether
+  using direct mode or not. That's one of its advantages over git!
+
+* Somewhere between 59 and 161 of the survey respondants use Debian.
+  I can compare this with [Debian popularity contest data](http://qa.debian.org/popcon-graph.php?packages=git-annex)
+  which has 400 active installations and 1000 total installations,
+  and make guesses about what fraction of all git-annex users have answered
+  the survey. By making different assumptions I got guesses that varied by
+  2 orders of magnitude, so not worth bothering with. Explicitly asking how
+  many people use each Linux distribution would be a good idea in next
+  year's survey.
+
+----
+
+Main work today was fixing Android DNS lookups, which was trying to use
+/etc/resolv.conf to look up SRV records for XMPP, and had to be changed to
+use a getprop command instead. Since I can't remember dealing with this
+before (not impossible I made some quick fix to the dns library before and
+lost it though), I'm wondering if XMPP was ever usable on Android before.
+Cannot remember. May work now, anyway...
diff --git a/doc/devblog/day_71__that_was_unexpected.mdwn b/doc/devblog/day_71__that_was_unexpected.mdwn
new file mode 100644
index 0000000000..92de97b7dc
--- /dev/null
+++ b/doc/devblog/day_71__that_was_unexpected.mdwn
@@ -0,0 +1,30 @@
+Had planned to spend all day not working on git-annex and instead getting
+caught up on conference videos. However, got a little bit multitasky while
+watching those, and started investigating why, last time I worked on
+Windows port, git-annex was failing to link. 
+
+A good thing to do while watching conference videos since it involved lots of 
+test builds with different flags. Eventially solved it. 
+Building w/o WebDAV avoids crashing the compiler anyhow.
+
+Thought I'd try the resulting binary and see if perhaps I had forgotten to
+use the threaded RTS when I was running ghc by hand to link it last time,
+and perhaps that was why threads
+[[seemed to have hung|day_56__git-annex_user_survey]] back then.
+
+It was. This became clear when I saw a "deadlocked indefinitely in MVar"
+error message, which tells me that it's at least using the threaded RTS.
+So, I fixed that, and a few other minor things, and ran this command
+in a DOS prompt box:
+
+	git annex watch --force --foreground --debug
+
+And I've been making changes to files in that repository, and amazingly,
+the watcher is noticing them, and committing them!
+
+So, I was almost entirely there to a windows port of the watcher a month
+ago, and didn't know. It has some rough edges, including not doing anything
+to check if a newly created file is open for write when adding it, and
+getting the full assistant ported will be more work, and the full webapp
+may be a whole other set of problems, but this is a quite nice milestone
+for the Windows port.
diff --git a/doc/devblog/day_72__windows_webapp_not.mdwn b/doc/devblog/day_72__windows_webapp_not.mdwn
new file mode 100644
index 0000000000..985afe7c42
--- /dev/null
+++ b/doc/devblog/day_72__windows_webapp_not.mdwn
@@ -0,0 +1,22 @@
+Got the entire webapp to build on Windows.
+
+Compiling was easy. One line of code had to be #ifdefed out, and the whole
+rest of the webapp UI just built!
+
+Linking was epic. It seems that I really am runninginto a 32kb command line length
+limit, which causes the link command to fail on Windows. git-annex with all
+its bells and whistles enabled is just too big. Filed a 
+[ghc bug report](https://ghc.haskell.org/trac/ghc/ticket/8596), and got back a
+helpful response about using  to
+work around.
+
+6 hours of slogging through compiling dependencies and fighting with
+toolchain later, I have managed to link git-annex with the webapp!
+
+The process is not automated yet. While I was able to automate
+passing gcc a @file with its parameters, gcc then calls collect2, which
+calls ld, and both are passed too many parameters. I have not found a way
+to get gcc to generate a response file. So I did it manually. Urgh.
+
+Also, it crashes on startup with `getAddrInfo` failure. But some more
+porting is to be expected, now that the windows webapp links.. ;)
diff --git a/doc/devblog/day_73__EvilLinker.mdwn b/doc/devblog/day_73__EvilLinker.mdwn
new file mode 100644
index 0000000000..68aa5d0653
--- /dev/null
+++ b/doc/devblog/day_73__EvilLinker.mdwn
@@ -0,0 +1,28 @@
+Android has the EvilSplicer, now Windows gets the EvilLinker. Fully
+automated, and truly horrible solution to the too long command line problem.
+
+Now when I run `git annex webapp` on windows, it almost manages to open
+the web browser.
+
+At the same time, I worked with Yuri to upgrade the Windows autobuilder to a
+newer Haskell platform, which can install Yesod. I have not quite achieved
+a successful webapp build on the autobuilder, but it seems close.
+
+----
+
+Here's a nice Haskell exercise for someone. I wrote this quick and dirty
+function in the EvilSplicer, but it's crying out for a generalized solution.
+
+[[!format haskell """
+{- Input contains something like 
+ - c:/program files/haskell platform/foo -LC:/Program Files/Haskell Platform/ -L...
+ - and the *right* spaces must be escaped with \
+ -
+ - Argh.
+ -}
+escapeDosPaths :: String -> String
+escapeDosPaths = replace "Program Files" "Program\\ Files"
+        . replace "program files" "program\\ files"
+        . replace "Haskell Platform" "Haskell\\ Platform"
+        . replace "haskell platform" "haskell\\ platform"
+"""]]
diff --git a/doc/devblog/day_74__so_close.mdwn b/doc/devblog/day_74__so_close.mdwn
new file mode 100644
index 0000000000..1f63eaf4d8
--- /dev/null
+++ b/doc/devblog/day_74__so_close.mdwn
@@ -0,0 +1,20 @@
+Windows webapp now starts, opens a web browser, and ... crashes. 
+
+
+
+This  is [a bug in warp](https://github.com/yesodweb/wai/issues/202)
+or a deep level of the stack. I know that yesod apps have run on Windows
+before, so apparently something has changed and introduced this problem.
+
+Also have a problem with the autobuilder; the EvilSplicer or something
+it runs is locking up on that system for reasons not yet determined.
+
+Looks like I will need to wait a bit longer for the windows webapp, but I
+could keep working on porting the assistant in the meantime. 
+
+The most important thing that I need to port is how to check if a file
+is being written to at the same time the assistant adds it to the
+repository. No real `lsof` equivilant on Windows. I might be able to do
+something with exclusive locking to detect if there's a writer (but this
+would also block using the file while it was being added). Or I may be able
+to avoid the need for this check, at least in direct mode.
diff --git a/doc/devblog/day_74__so_close/comment_1_b1aa185734c3d74830b81def4fe63bff._comment b/doc/devblog/day_74__so_close/comment_1_b1aa185734c3d74830b81def4fe63bff._comment
new file mode 100644
index 0000000000..2abbc8dd33
--- /dev/null
+++ b/doc/devblog/day_74__so_close/comment_1_b1aa185734c3d74830b81def4fe63bff._comment
@@ -0,0 +1,10 @@
+[[!comment format=mdwn
+ username="http://nullroute.eu.org/~grawity/"
+ nickname="Mantas"
+ subject="comment 1"
+ date="2013-12-08T12:52:26Z"
+ content="""
+Take a look at how Process Hacker implements the \"Search for open handles\" function; it is rather close to `lsof`. (Unlocker probably does it the same way, too.)
+
+But locking (e.g. I think the \"share mode\" could prevent other programs from writing but not reading) may be more reliable, if a program starts writing to file *after* the lsof check happens...
+"""]]
diff --git a/doc/devblog/day_75__hallelujah.mdwn b/doc/devblog/day_75__hallelujah.mdwn
new file mode 100644
index 0000000000..42fb4fdf9c
--- /dev/null
+++ b/doc/devblog/day_75__hallelujah.mdwn
@@ -0,0 +1,8 @@
+I have seen the glory of the webapp running on Windows.
+
+One of the warp developers pointed me in the right direction and I
+developed a fix for the `recv` bug.
+
+My Windows and MSIE are old and fall over on some of the
+javascript, so it's not glorious enough for a screenshot. But large chunks
+of it do seem to work.
diff --git a/doc/devblog/day_75__hallelujah/comment_1_df04c456e99d47743494a18c1badba8c._comment b/doc/devblog/day_75__hallelujah/comment_1_df04c456e99d47743494a18c1badba8c._comment
new file mode 100644
index 0000000000..bd83ec1ff4
--- /dev/null
+++ b/doc/devblog/day_75__hallelujah/comment_1_df04c456e99d47743494a18c1badba8c._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="https://launchpad.net/~maestro-alubia"
+ nickname="maestro-alubia"
+ subject="comment 1"
+ date="2013-12-09T19:47:32Z"
+ content="""
+Yeah! Looking forward to finally introduce git-annex in my heterogeneous network. Thanks and keep up the good work!
+"""]]
diff --git a/doc/devblog/day_76__results.mdwn b/doc/devblog/day_76__results.mdwn
new file mode 100644
index 0000000000..4fc71eaa63
--- /dev/null
+++ b/doc/devblog/day_76__results.mdwn
@@ -0,0 +1,15 @@
+Fixed up a few problems with the Windows webapp, and it's now completely
+usable, using any browser other than MSIE. While there are missing
+features in the windows port, all the UI for the features it does have
+seems to just work in the webapp.
+
+Fixed a ugly problem with Firefox, which turned out to have been introduced
+a while ago by a workaround for an ugly problem in Chrome. Web browsers are
+so wonderful, until they're crap.
+
+Think I've fixed the bug in the EvilLinker that was causing it to hang on
+the autobuilder, but still don't have a Windows autobuild with the webapp
+just yet.
+
+Also improved `git annex import` some more, and worked on a bug in git
+repository repair, which I will need to spend some more time on tomorrow.
diff --git a/doc/devblog/day_77__it_builds.mdwn b/doc/devblog/day_77__it_builds.mdwn
new file mode 100644
index 0000000000..421bb5efec
--- /dev/null
+++ b/doc/devblog/day_77__it_builds.mdwn
@@ -0,0 +1,8 @@
+Got the Windows autobuilder building the webapp. Have not tried that build
+yet myself, but I have high hopes it will work.
+
+Made other Windows improvements, including making the installer
+write a start menu entry file, and adding free disk space checking.
+
+Spent rest of the day improving git repair code on a real-world corrupted
+repository.
diff --git a/doc/devblog/day_78__desidetracked.mdwn b/doc/devblog/day_78__desidetracked.mdwn
new file mode 100644
index 0000000000..5b8ca8a583
--- /dev/null
+++ b/doc/devblog/day_78__desidetracked.mdwn
@@ -0,0 +1,34 @@
+I've switched over to mostly working on Windows porting in the evenings
+when bored, with days spent on other git-annex stuff. So, getting back to
+the planned [[design/roadmap]] for this month..
+
+Set up a [tip4commit for git-annex](http://tip4commit.com/projects/152).
+Anyone who gets a commit merged in will receive a currently small amount of
+bitcoin. This would almost be a good way to encourage more committers
+other than me, by putting say, half the money I have earmarked for that into
+the tip jar. The problem is, I make too many commits myself, so most of the
+money would be quickly tipped back out to me! I have gotten in touch with the
+tip4commit people, and hope they will give me a way to blacklist
+myself from being tipped.
+
+Designed a [[design/external_special_remote_protocol]] that seems pretty
+good for first-class special remotes implemented outside git-annex.
+It's moderately complicated on the git-annex side to make it simple and
+flexible on the special remote side, but I estimate only a few days to build
+it once I have the design finalized.
+
+# windows
+
+Tested the autobuilt windows webapp. It works! Sorted out some issues with
+the bundled libraries.
+
+Reworked how `git annex transferkeys` communicates, to make it easier to
+port it to Windows. Narrowly managed to avoid needing to write Haskell
+bindings to Windows's equivilant of `pipe(2)`. I think the Windows
+assistant can transfer keys now. and the webapp UI may even be able to be
+used to stop transfers. Needs testing.
+
+Investigated what I'll need to get XMPP working on Windows. Most of the
+libs are available in cygwin, but gsasl would need to be built from source.
+Also some kind of space-in-path problem is preventing cabal installing some
+of the necessary dependencies.
diff --git a/doc/devblog/day_79__catch_up.mdwn b/doc/devblog/day_79__catch_up.mdwn
new file mode 100644
index 0000000000..0206a0acf6
--- /dev/null
+++ b/doc/devblog/day_79__catch_up.mdwn
@@ -0,0 +1,3 @@
+Spent most of today catching up with a weeks's worth of traffic.
+
+Fixed 2 bugs. Message backlog is 23 messages.
diff --git a/doc/devblog/day_7__release_day.mdwn b/doc/devblog/day_7__release_day.mdwn
new file mode 100644
index 0000000000..f377bb4be2
--- /dev/null
+++ b/doc/devblog/day_7__release_day.mdwn
@@ -0,0 +1,10 @@
+Got git annex sync working with gcrypt. So went ahead and made a release
+today. Lots of nice new features!
+
+Unfortunately the linux 64 bit daily build is failing, because my build
+host only has 2 gb of memory and it is no longer enough. I am looking for a
+new build host, ideally one that doesn't cost me $40/month for 3 gb of ram
+and 15 gb of disk. (Extra special ideally one that I can run multiple builds
+per day on, rather than the current situation of only building overnight to
+avoid loading the machine during the day.) Until this is sorted out, no
+new 64 bit linux builds..
diff --git a/doc/devblog/day_7__release_day/comment_1_12bb94d903868ecddb3e348c9c4afeaf._comment b/doc/devblog/day_7__release_day/comment_1_12bb94d903868ecddb3e348c9c4afeaf._comment
new file mode 100644
index 0000000000..266e843fd5
--- /dev/null
+++ b/doc/devblog/day_7__release_day/comment_1_12bb94d903868ecddb3e348c9c4afeaf._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="https://me.yahoo.com/a/2grhJvAC049fJnvALDXek.6MRZMTlg--#eec89"
+ nickname="John"
+ subject="comment 1"
+ date="2013-09-09T21:51:47Z"
+ content="""
+Joey, I'd be happy to host 64-bit builds on both Mac and Linux using my personal Jenkins server (both machines are local and have 16+ GB of RAM).  I might even be able to get you SSH jail access to the build tree for the 64-bit Linux build.  Just let me know at johnw@fpcomplete.com.
+"""]]
diff --git a/doc/devblog/day_7__release_day/comment_2_d3e38d6f6bba179dab40d4d75ff061de._comment b/doc/devblog/day_7__release_day/comment_2_d3e38d6f6bba179dab40d4d75ff061de._comment
new file mode 100644
index 0000000000..96fb5dce30
--- /dev/null
+++ b/doc/devblog/day_7__release_day/comment_2_d3e38d6f6bba179dab40d4d75ff061de._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="https://me.yahoo.com/a/2grhJvAC049fJnvALDXek.6MRZMTlg--#eec89"
+ nickname="John"
+ subject="comment 2"
+ date="2013-09-10T23:11:27Z"
+ content="""
+Everything is all setup now to build git-annex on both systems, at most once per hour (if there have been new Git commits), and only between 2am and 4pm.  Just contact me on IRC or e-mail and I'll setup you up with credentials so that you can manipulate the jobs and access the build artifacts.
+"""]]
diff --git a/doc/devblog/day_80__plumbing.mdwn b/doc/devblog/day_80__plumbing.mdwn
new file mode 100644
index 0000000000..3ef76a3730
--- /dev/null
+++ b/doc/devblog/day_80__plumbing.mdwn
@@ -0,0 +1,9 @@
+Made some improvements to git-annex's plumbing level commands today. Added
+new lookupkey and examinekey commands. Also expanded the things that
+`git annex find` can report about files. Among other things, the elusive
+hash directory locations can now be looked up, which IIRC a few people have
+asked for a way to do.
+
+Also did some work on the linux standalone tarball and OSX app. Both now
+include man pages, and it's also now possible to just unpack it and symlink
+git-annex into ~/bin or similar to add it to PATH.
diff --git a/doc/devblog/day_81__more_standalone.mdwn b/doc/devblog/day_81__more_standalone.mdwn
new file mode 100644
index 0000000000..646f280c12
--- /dev/null
+++ b/doc/devblog/day_81__more_standalone.mdwn
@@ -0,0 +1,15 @@
+Made the Linux standalone builds more self-contained, now they include
+their own linker and glibc, and ugly hacks to make them be used when
+running the included programs. This should make them more portable
+to older systems.
+
+Set up an arm autobuilder. 
+This autobuilder runs in an Debian armel chroot, using
+qemu-user-static (with a patch to make it support some syscalls ghc uses).
+No webapp yet; waiting on feedback of how well it works. I *hope* this
+build will be usable on eg, Synology NAS and Raspberry PI.
+
+Also worked on improving the assistant's batching of commits during the
+startup scan. And some other followups and bug triage.
+
+Today's work was sponsored by Hamish Coleman.
diff --git a/doc/devblog/day_81__more_standalone/comment_1_25ceb116406b55a8ff28f7b392806bc9._comment b/doc/devblog/day_81__more_standalone/comment_1_25ceb116406b55a8ff28f7b392806bc9._comment
new file mode 100644
index 0000000000..4a18b79a66
--- /dev/null
+++ b/doc/devblog/day_81__more_standalone/comment_1_25ceb116406b55a8ff28f7b392806bc9._comment
@@ -0,0 +1,10 @@
+[[!comment format=mdwn
+ username="https://www.google.com/accounts/o8/id?id=AItOawnRai_qFYPVvEgC6i1nlM1bh-C__jbhqS0"
+ nickname="Matthew"
+ subject="Binaries"
+ date="2013-12-17T12:49:26Z"
+ content="""
+Are there binaries for the RPi version anywhere, it would be great to try it.
+
+Thanks
+"""]]
diff --git a/doc/devblog/day_82__rpi_and_synology.mdwn b/doc/devblog/day_82__rpi_and_synology.mdwn
new file mode 100644
index 0000000000..3749f474ad
--- /dev/null
+++ b/doc/devblog/day_82__rpi_and_synology.mdwn
@@ -0,0 +1,21 @@
+Fixed a few problems in the [[armel build|install/Linux_standalone]], and
+it's been confirmed to work on Raspberry Pi and Synology NAS. Since none of
+the fixes were specific to those platforms, it will probably work anywhere
+the kernel is new enough. That covers 9+% of the 
+[missing ports in the user survey](http://git-annex-survey.branchable.com/polls/2013/missing_ports/)!
+
+Thought through the possible issues with the assistant on Windows not being
+able to use lsof. I've convinced myself it's probably safe. (In fact, it
+might be safe to stop checking with lsof when using the assistant in direct
+mode entirely.) Also did some testing of some specific interesting
+circumstances (including 2 concurrent writers to a single file).
+
+I've been working on adding the webapp to the armel build. This can mostly
+reuse the patches and EvilSplicer developed for Android, but it's taking
+some babysitting of the build to get yesod etc installer for various reasons.
+Will be surprised if I don't get there tomorrow.
+
+One other thing.. I notice that  is up and running.
+This was set up by Subito, who offered me the domain, but I suggested he
+keep it and set up a pretty start page that points new users at the
+relevant parts of the wiki. I think he's done a good job with that!
diff --git a/doc/devblog/day_82__rpi_and_synology/comment_1_d154ddcf22027fd06acf9da73e12c006._comment b/doc/devblog/day_82__rpi_and_synology/comment_1_d154ddcf22027fd06acf9da73e12c006._comment
new file mode 100644
index 0000000000..33bc12e634
--- /dev/null
+++ b/doc/devblog/day_82__rpi_and_synology/comment_1_d154ddcf22027fd06acf9da73e12c006._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="Attila"
+ ip="213.163.19.162"
+ subject="git-annex.org"
+ date="2013-12-27T14:14:07Z"
+ content="""
+Minor glitch on git-annex.org: the first link to http://git-annex.branchable.com is missing http:// prefix and so points to http://www.git-annex.org/git-annex.branchable.com
+"""]]
diff --git a/doc/devblog/day_83__armel_webapp.mdwn b/doc/devblog/day_83__armel_webapp.mdwn
new file mode 100644
index 0000000000..95c70010b8
--- /dev/null
+++ b/doc/devblog/day_83__armel_webapp.mdwn
@@ -0,0 +1,19 @@
+Got the arm webapp to build! (I have not tried to run it.) The build
+process for this is quite elaborate; 2 chroots, one amd64 and one armel,
+with the same versions of everything installed in each, and git-annex is
+built in the first to get the info the EvilSplicer needs to build it in the
+second.
+
+Fixed a nasty bug in the assistant on OSX, where at startup it would follow
+symlinks in the repository that pointed to directories outside the
+repository, and add the files found there. Didn't cause data loss itself
+(in direct mode the assistant doesn't touch the files), but certainly
+confusingly breaks things and makes it easy to shoot your foot off. I will
+be moving up the next scheduled release because of this bug, probably to
+Saturday.
+
+Looped the git developers in on a problem with git failing on some kernels
+due to `RLIMIT_NOFILE` not working. Looks like git will get more robust and
+this should make the armel build work on even more embedded devices.
+
+Today's work was sponsored by Johan Herland.
diff --git a/doc/devblog/day_84__ho_uh_oh.mdwn b/doc/devblog/day_84__ho_uh_oh.mdwn
new file mode 100644
index 0000000000..516ceb5321
--- /dev/null
+++ b/doc/devblog/day_84__ho_uh_oh.mdwn
@@ -0,0 +1,11 @@
+Resurfaced today to fix some problems with the Linux standalone builds
+in the Solstice release. The worst of these prevented the amd64 build from
+running on some systems, and that build has been updated. The other
+problems all involved the binary shimming, and were less serious.
+
+As part of that work, replaced the hacky shell script that handled the
+linux library copying and binary shimming with a haskell program.
+
+Also worked on some Windows bugs, and fixed a typo in the test suite.
+Got my own little present: haskell-tasty finally got out of Incoming, so
+the next Debian package build will once again include the test suite.
diff --git a/doc/devblog/day_85__external_special_remote_protocol_types.mdwn b/doc/devblog/day_85__external_special_remote_protocol_types.mdwn
new file mode 100644
index 0000000000..9b3e4d9b1f
--- /dev/null
+++ b/doc/devblog/day_85__external_special_remote_protocol_types.mdwn
@@ -0,0 +1,25 @@
+Only did a few hours today, getting started on implementing the
+[[design/external_special_remote_protocol]].
+
+Mostly this involved writing down types for the various messages,
+and code to parse them. I'm very happy with how the parsing turned out;
+nearly all the work is handled by the data types and type classes,
+and so only one line of very simple code is needed to parse each message:
+
+[[!format haskell """
+instance Receivable Response where
+       parseCommand "PREPARE-SUCCESS" = parse0 PREPARE_SUCCESS
+       parseCommand "TRANSFER-SUCCESS" = parse2 TRANSFER_SUCCESS
+       parseCommand "TRANSFER-FAILURE" = parse3 TRANSFER_FAILURE
+"""]]
+
+An especially nice part of this implementation is that it knows exactly how
+many parameters each message should have (and their types of course), and so
+can both reject invalid messages, and avoid ambiguity in tokenizing the
+parameters. For example, the 3rd parameter of TRANSFER-FAILURE is an error
+message, and as it's the last parameter, it can contain multiple words.
+
+	*Remote.External> parseMessage "TRANSFER-FAILURE STORE SHA1--foo doesn't work on Christmas" :: Maybe Response
+	Just (TRANSFER_FAILURE Upload (Key {keyName = "foo", keyBackendName = "SHA1", keySize = Nothing, keyMtime = Nothing}) "doesn't work on Christmas")
+
+That's the easy groundwork for external special remotes, done.
diff --git a/doc/devblog/day_86__external_special_remote_implementation.mdwn b/doc/devblog/day_86__external_special_remote_implementation.mdwn
new file mode 100644
index 0000000000..c40bfbaea3
--- /dev/null
+++ b/doc/devblog/day_86__external_special_remote_implementation.mdwn
@@ -0,0 +1,11 @@
+Built most of the external special remote today. While I've written 600
+lines of code for this, and think it's probably working, and complete
+(except for a couple of features), all I know is that it compiles.
+
+I've also written an [[example external special remote program in shell script|special_remotes/external/example.sh]],
+so the next step is to put the two together and see how it works.
+I also hope that some people who have built hook special remotes
+in the past will update them to the new external special remote interface,
+which is quite a lot better.
+
+Today's work was sponsored by Justine Lam.
diff --git a/doc/devblog/day_86__external_special_remote_implementation/comment_1_5116bcf4b60030cb46683df94a75d7ee._comment b/doc/devblog/day_86__external_special_remote_implementation/comment_1_5116bcf4b60030cb46683df94a75d7ee._comment
new file mode 100644
index 0000000000..3580623086
--- /dev/null
+++ b/doc/devblog/day_86__external_special_remote_implementation/comment_1_5116bcf4b60030cb46683df94a75d7ee._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="https://www.google.com/accounts/o8/id?id=AItOawleVyKk2kQsB_HgEdS7w1s0BmgRGy1aay0"
+ nickname="Milan"
+ subject="comment 1"
+ date="2013-12-27T08:44:44Z"
+ content="""
+Will it support splitting of large files for remotes that limits size of stored file?
+"""]]
diff --git a/doc/devblog/day_86__external_special_remote_implementation/comment_2_7b6e734f785fbd9db7883b63150023dc._comment b/doc/devblog/day_86__external_special_remote_implementation/comment_2_7b6e734f785fbd9db7883b63150023dc._comment
new file mode 100644
index 0000000000..7a0c857834
--- /dev/null
+++ b/doc/devblog/day_86__external_special_remote_implementation/comment_2_7b6e734f785fbd9db7883b63150023dc._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="http://joeyh.name/"
+ ip="209.250.56.227"
+ subject="comment 2"
+ date="2013-12-27T19:12:17Z"
+ content="""
+That's on the TODO list and the protocol should be easily flexible enough to let it be added. The existing code to handle chunks is a bit of a mess though.
+"""]]
diff --git a/doc/devblog/day_87__external_special_remotes_done.mdwn b/doc/devblog/day_87__external_special_remotes_done.mdwn
new file mode 100644
index 0000000000..b8bba606bb
--- /dev/null
+++ b/doc/devblog/day_87__external_special_remotes_done.mdwn
@@ -0,0 +1,16 @@
+The [[external_special_remote|special_remotes/external]] interface is now
+done, and tested working great! Now we just need all the old hook special
+remotes to be converted to use it..
+
+I punted on per-special-remote, per-key state storage in the git-annex
+branch for now. If I find an example of a remote that needs it (Tahoe-LAFS
+may, but still TBD), I'll add it. Added suppport for using the same
+credential storage that git-annex uses for S3 and WebDAV credentials.
+
+The main improvement I'd like to make is to add an interface for transferring
+files where the file is streamed to/from the external special remote,
+rather than using temp files as it does now. This would be more efficient
+(sometimes) and make the progress bars better. But it needs to either use a
+named pipe, which is complicated and non-portable, or serialize the file's
+contents over a currently line-based protocol, which would be a pain.
+Anyway, this can be added later, the protocol is extensible.
diff --git a/doc/devblog/day_88__lazy_sunday.mdwn b/doc/devblog/day_88__lazy_sunday.mdwn
new file mode 100644
index 0000000000..b06c76bd2f
--- /dev/null
+++ b/doc/devblog/day_88__lazy_sunday.mdwn
@@ -0,0 +1,15 @@
+Fixed a bug that could leave a direct mode repository stuck at
+annex.version 3. As part of that, v3 indirect mode repositories will be
+automatically updated to v5. There's no actual change in that upgrade, it
+just simplifies things to have only one supported annex.version.
+
+Added youtube playlist support to git-annex. Seems I had almost all the
+pieces needed, and didn't know it. Only about a dozen lines of code!
+
+Added PREPARE-FAILURE support to the external special remote interface.
+
+After I found the cable my kitten stole (her apport level is high), fixed
+file transfers to/from Android. This broke because git-annex assistant
+tries to use ionice, if it's in PATH, and Android's ionice is not suitable.
+It could probably include ionice in the busybox build and use that one, but
+I wanted a quick fix for this before the upcoming release.
diff --git a/doc/devblog/day_89__reflections.mdwn b/doc/devblog/day_89__reflections.mdwn
new file mode 100644
index 0000000000..94bbf87e2b
--- /dev/null
+++ b/doc/devblog/day_89__reflections.mdwn
@@ -0,0 +1,18 @@
+Worked on bug report and forum backlog (24 messages left), and made a few
+bug fixes. The main one was a fix for a Windows-specific direct mode merge
+bug.
+
+This month didn't go entirely to plan. I had not expected to work on the
+Windows assistant and webapp and get it so close to fully working. Nor had
+I expected to spend time and make significant progress on porting git-annex
+to *Linux* -- particularly to embedded NAS devices! I had hoped to
+encourage some others to develop git-annex, but only had one bite from a
+student and it didn't work out. Meanwhile, automatically rewarding
+committers with bitcoin is an interesting alternative approach to possibly
+motivating contributors, and I would like to set that up, but the software
+is new and I haven't had time yet. The only thing that went exactly as
+planned was the external special remote implementation.
+
+A special surprise this month is that I have started hearing privately from
+several institutions that are starting using git-annex in interesting ways.
+Hope I can share details of some of that 2014!
diff --git a/doc/devblog/day_8__ill.mdwn b/doc/devblog/day_8__ill.mdwn
new file mode 100644
index 0000000000..c086a9dc1b
--- /dev/null
+++ b/doc/devblog/day_8__ill.mdwn
@@ -0,0 +1,20 @@
+I've been out sick. However, some things kept happening. Mesar contributed
+a build host, and the linux and android builds are now happening, hourly,
+there. (Thanks as well to the two other people who also offered hostng.)
+And I made a minor release to fix a bug in the test suite that I was pleased
+three different people reported.
+
+Today, my main work was getting git-annex to notice when a gcrypt remote
+located on some removable drive mount point is not the same gcrypt remote
+that was mounted there before. I was able to finesse this so it
+re-configures things to use the new gcrypt remote, as long as it's a
+special remote it knows about. (Otherwise it has to ignore the remote.)
+So, encrypted repos on removable drives will work just as well as
+non-encrypted repos!
+
+Also spent a while with rsync.net tech support trying to work out why
+someone's git-annex apparently opened a lot of concurrent ssh connections
+to rsync.net. Have not been able to reproduce the problem though.
+
+Also, a lot of catch-up to traffic. Still 63 messages backlogged however,
+and still not entirely well..
diff --git a/doc/devblog/day_90__slow_start.mdwn b/doc/devblog/day_90__slow_start.mdwn
new file mode 100644
index 0000000000..09d1f7270c
--- /dev/null
+++ b/doc/devblog/day_90__slow_start.mdwn
@@ -0,0 +1,23 @@
+Various work on Debian, OSX, and Windows stuff. Mostly uninteresting, but
+took most of the day.
+
+Made `git annex mirror --all` work. I can see why I left it out; when the
+mirroring wants to drop an object, in --all mode it doesn't have an
+associated file in the tree, so it cannot look at the annex.numcopies in
+gitattributes. Same reason why `git annex drop --all` is not implemented.
+But decided to go ahead and only use other numcopies configuration for
+mirroring.
+
+Added GETWANTED and SETWANTED to the external special remote protocol, and
+that is as far as I want to go on adding git-annex plumbing stuff to the
+protocol. I expect Tobias will release a boatload of special remotes
+updated to the new protocol soon, which seems to prove it has everything
+that could reasonably be needed.
+
+This is a nice public git-annex repository containing a growing collection
+of tech conference videos. 
+
+Did some design work on [[todo/untracked_remotes]], which I think will turn
+out to be read-only remotes. Being able to clone a repository and use
+git-annex in the clone without anything leaking back upstream is often
+desirable when using public repository, or a repository with many users.
diff --git a/doc/devblog/day_91__wintry_mix.mdwn b/doc/devblog/day_91__wintry_mix.mdwn
new file mode 100644
index 0000000000..bfa9ddd35e
--- /dev/null
+++ b/doc/devblog/day_91__wintry_mix.mdwn
@@ -0,0 +1,11 @@
+Implemented read-only remotes. This may not cover every use case around
+wanting to clone a repository and use git-annex without leaking the
+existence of your clone back to it, but I think it hits most of them in a
+quite easy way, and allows for some potentially interesting stuff like
+partitioned networks of git-annex repositories.
+
+Zooko and I have been talking things over (for rather too long),
+and I think have now agreed on a how a more advanced git-annex
+Tahoe-LAFS special remote should work. This includes storing the
+tahoe file-caps in the git-annex branch. So, I really need to add that
+per-special-remote data storage feature I've been thinking about.
diff --git a/doc/devblog/day_92-93__reconnection.mdwn b/doc/devblog/day_92-93__reconnection.mdwn
new file mode 100644
index 0000000000..aa83bde38c
--- /dev/null
+++ b/doc/devblog/day_92-93__reconnection.mdwn
@@ -0,0 +1,34 @@
+Yesterday, added per-remote, per-key state storage. This is exported via
+the external special remote protocol, and I expect to use it at least for
+Tahoe Lafs.
+
+Also, made the assistant write ssh config files with better permissions,
+so ssh won't refuse to use them. (The only case I know of where that
+happened was on Windows.)
+
+Today, made `addurl` and `importfeed` honor annex.diskreserve. Found out about
+this the hard way, when an importfeed cron job filled up my server with
+youtube videos. I should probably also make `import` honor
+annex.diskreserve.
+
+---
+
+I've been working, so far inconclusively, on making the assistant
+deal with remotes that might open a long duration network connection.
+Problem being that if the connection is lost, and the remote is not smart
+enough to reconnect, all further use of it could fail.
+
+In a `restarttransferrer` branch, I have made the assistant start separate
+`transferkeys` processes for each remote. So if a remote starts to
+fail, the assistant can stop its `transferkeys` process, and restart it,
+solving the problem.
+
+But, if a resource needed for a remote is not available, this degrades to
+every transfer attempt to that remote restarting it. So I don't know if this is the
+right approach.
+
+Other approaches being considered include asking that implementors of
+external special remotes deal with reconnection themselves (Tobias, do you
+deal with this in your remotes?), or making the
+assistant only restart failing remotes after it detects there's been 
+a network connection change.
diff --git a/doc/devblog/day_94__leaks.mdwn b/doc/devblog/day_94__leaks.mdwn
new file mode 100644
index 0000000000..3aa4e1552c
--- /dev/null
+++ b/doc/devblog/day_94__leaks.mdwn
@@ -0,0 +1,12 @@
+Spent ages tracking down a memory leak in the assistant that showed up when
+a lot of files were added. Turned out to be a standard haskell laziness
+induced problem, fixed by adding strictness annotations. Actually there
+were several of them, that leaked at different rates. Eventually, I seem to
+have gotten them all fixed:
+
+Before: [[bugs/import_memleak_from_the_assistant/leakbefore.png]]
+After: [[bugs/import_memleak_from_the_assistant/leakafter.png]]
+
+Also fixed a bug in `git annex add` when the disk was completely full.
+In that situation, it could sometimes move the file from the work tree to
+.git/annex/objects and fail to put the symlink in place.
diff --git a/doc/devblog/day_95__reconnection_revisited.mdwn b/doc/devblog/day_95__reconnection_revisited.mdwn
new file mode 100644
index 0000000000..b2d5b28a58
--- /dev/null
+++ b/doc/devblog/day_95__reconnection_revisited.mdwn
@@ -0,0 +1,13 @@
+Taught the assistant to stop reusing an existing `git annex transferkeys`
+process after it detects a network connection change. I don't think this is
+a complete solution to what to do about long-duration network connections
+in remotes. For one thing a remote could take a long time to time out
+when the network is disconnected, and block other transfers (eg to
+local drives) in the meantime. But at least if a remote loses its network
+connection and does not try to reconnect on its own, and so is continually
+failing, this will get it back into a working state eventually.
+
+Also, fixed a problem with the OSX Mavericks build, it seems that the
+versions of wget and coreutils stuff that I was including in it were built
+by homebrew with full optimisations turned on, so didn't work on some CPUs.
+Replaced those with portable builds.
diff --git a/doc/devblog/day_95__reconnection_revisited/comment_1_c1106e573fcf9f3d4524c0e4f4254790._comment b/doc/devblog/day_95__reconnection_revisited/comment_1_c1106e573fcf9f3d4524c0e4f4254790._comment
new file mode 100644
index 0000000000..67618f64d3
--- /dev/null
+++ b/doc/devblog/day_95__reconnection_revisited/comment_1_c1106e573fcf9f3d4524c0e4f4254790._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="https://www.google.com/accounts/o8/id?id=AItOawnX1msQxnLoSeu7q-i-c9BWghonsN7Qmns"
+ nickname="Jan Ulrich"
+ subject="comment 1"
+ date="2014-01-07T16:02:01Z"
+ content="""
+Thanks for the Mavericks fix! 
+"""]]
diff --git a/doc/devblog/day_95__reconnection_revisited/comment_2_40478739f95e0b56ce0103db6e405ef4._comment b/doc/devblog/day_95__reconnection_revisited/comment_2_40478739f95e0b56ce0103db6e405ef4._comment
new file mode 100644
index 0000000000..331965e3e7
--- /dev/null
+++ b/doc/devblog/day_95__reconnection_revisited/comment_2_40478739f95e0b56ce0103db6e405ef4._comment
@@ -0,0 +1,10 @@
+[[!comment format=mdwn
+ username="https://www.google.com/accounts/o8/id?id=AItOawnxp2XU8gIribhhGhGuYtU6eMMwHv5gUGI"
+ nickname="Amitai"
+ subject="obligatory "bah homebrew, yay pkgsrc" comment"
+ date="2014-01-12T03:01:17Z"
+ content="""
+Bah homebrew, yay pkgsrc.
+
+:-)
+"""]]
diff --git a/doc/devblog/day_96__catching_up.mdwn b/doc/devblog/day_96__catching_up.mdwn
new file mode 100644
index 0000000000..c41e962b13
--- /dev/null
+++ b/doc/devblog/day_96__catching_up.mdwn
@@ -0,0 +1,7 @@
+Been on reduced activity the past several days. I did spend a full day
+somewhere in there building the Tahoe LAFS special remote. Also, Tobias has
+finished updating his full suite of external special remotes to use the new
+interface!
+
+Worked on closing up the fundraising campaign today (long overdue).
+This included adding a new wall-o-names to [[/thanks]].
diff --git a/doc/devblog/day_97__exciting_telehash_possiblities.mdwn b/doc/devblog/day_97__exciting_telehash_possiblities.mdwn
new file mode 100644
index 0000000000..2477949768
--- /dev/null
+++ b/doc/devblog/day_97__exciting_telehash_possiblities.mdwn
@@ -0,0 +1,20 @@
+If you've been keeping an eye on the [[design/roadmap]], you'll have seen that
+[[design/assistant/xmpp_security]] keeps being pushed back. This was because
+it's a hard and annoying problem requiring custom crypto and with an
+ugly key validation problem built into it too. I've now removed it from the
+roadmap entirely, replacing it with a [[design/assistant/telehash]] design.
+
+I'm excited by the possibilities of using telehash with git-annex. It seems
+it would be quite easy to make it significantly more peer-to-peer and
+flexible. The only issue is that telehash is still under heavy
+development and the C implementation is not even usable yet..
+(I'll probably end up writing Haskell bindings to that.)
+So I've pushed it down the roadmap to at least March.
+
+Spent the rest of the day making some minor improvements to external special
+remote protocol and doing some other minor bug fixes and backlog catch up.
+My backlog has exploded to nearly 50 messages remaining.
+
+----
+
+Today's work was sponsored by Chad Horohoe.
diff --git a/doc/devblog/day_97__exciting_telehash_possiblities/comment_1_7c775d93cbeed0d553e224751d30fbaa._comment b/doc/devblog/day_97__exciting_telehash_possiblities/comment_1_7c775d93cbeed0d553e224751d30fbaa._comment
new file mode 100644
index 0000000000..78750716ed
--- /dev/null
+++ b/doc/devblog/day_97__exciting_telehash_possiblities/comment_1_7c775d93cbeed0d553e224751d30fbaa._comment
@@ -0,0 +1,10 @@
+[[!comment format=mdwn
+ username="https://www.google.com/accounts/o8/id?id=AItOawkGCmVc5qIJaQQgG82Hc5zzBdAVdhe2JEM"
+ nickname="Bruno"
+ subject="is telehash udp only?"
+ date="2014-01-14T14:21:36Z"
+ content="""
+Is telehash UDP only?
+
+I like that XMPP works with HTTPS so it also works at work with proxies that only allows http traffic.
+"""]]
diff --git a/doc/devblog/day_98__old_bug.mdwn b/doc/devblog/day_98__old_bug.mdwn
new file mode 100644
index 0000000000..565efbdbcc
--- /dev/null
+++ b/doc/devblog/day_98__old_bug.mdwn
@@ -0,0 +1,17 @@
+Fixed a bug that one or two people had mentioned years ago, but I was never
+able to reproduce myself or get anyone to reproduce in a useful way. It
+caused log files that were supposed to be committed to the git-annex branch
+to end up in master. Turned out to involve weird stuff when the environment
+contains two different settings for a single variable. So was easily fixed
+at last. (I'm pretty sure the code would have never had this bug if
+Data.AssocList was not buried inside an xml library, which rather
+discourages using it when dealing with the environment.)
+
+Also worked on, and hopefully fixed, another OSX cpu optimisations problem.
+This one involving shared libraries that git-annex uses for XMPP.
+
+Also made the assistant detect corrupt .git/annex/index files on startup
+and remove them. It was already able to recover from corrupt .git/index
+files.
+
+Today's work was sponsored by David Wagner.
diff --git a/doc/devblog/day_99__catching_up_again.mdwn b/doc/devblog/day_99__catching_up_again.mdwn
new file mode 100644
index 0000000000..118a21f8d2
--- /dev/null
+++ b/doc/devblog/day_99__catching_up_again.mdwn
@@ -0,0 +1,19 @@
+Activity has been a bit low again this week. It seems to make sense to do
+weekly releases currently (rather than bi-monthly), and Thursday's
+release had only one new feature (Tahoe LAFS) and a bunch of bug fixes.
+
+Looks like git-annex will get back into Debian testing soon, after various
+fixes to make it build on all architectures again, and then the
+backport can be updated again too.
+
+I have been struggling with a problem with the OSX builds, which fail with
+a SIGKILL on some machines. It seems that homebrew likes to agressively
+optimise things it builds, and while I have had some success with its
+`--build-bottle` option, something in the gnutls stack used for XMPP is
+still over-optimised. Waiting to hear back from Kevin on cleaning up some
+optimised system libraries on the OSX host I use. (Is there some way to make
+a clean chrooot on OSX that can be accessed by a non-root user?)
+
+Today I did some minor work involving the --json switch, and also 
+a small change (well, under 300 line diff) allowing
+--all to be mixed with options like --copies and --in.
diff --git a/doc/devblog/day_99__catching_up_again/comment_1_b871bf0606dc29be9b8c2e5dc150f708._comment b/doc/devblog/day_99__catching_up_again/comment_1_b871bf0606dc29be9b8c2e5dc150f708._comment
new file mode 100644
index 0000000000..933de78737
--- /dev/null
+++ b/doc/devblog/day_99__catching_up_again/comment_1_b871bf0606dc29be9b8c2e5dc150f708._comment
@@ -0,0 +1,10 @@
+[[!comment format=mdwn
+ username="https://www.google.com/accounts/o8/id?id=AItOawkq2cjugiSvKWiWmcah3CPBqviQV_cin9I"
+ nickname="Yury"
+ subject="The world of Mac"
+ date="2014-01-19T14:13:55Z"
+ content="""
+I think that there is nothing fundamental about Mac OS X, which would prevent one from making a clean chroot that would work for non-root users. For instance, I believe that jailkit has been reported to work just fine. The problem is that you'll have to rebuild most of the stuff you need from scratch, which is very tedious. Might be easier to find instances where Homebrew has something like -march=native added to the $CFLAGS (not that I particularly fancy Homebrew of all 'missing' package managers on OS X).
+
+Anyways, it sounds like I'm going to get a Mac Mini donated sometime soon and I'm curious as to what would be the best way to make use of it for the CI system...
+"""]]
diff --git a/doc/devblog/day_99__catching_up_again/comment_2_c8363d47223e7bb899420e800bde3e27._comment b/doc/devblog/day_99__catching_up_again/comment_2_c8363d47223e7bb899420e800bde3e27._comment
new file mode 100644
index 0000000000..fb2377845d
--- /dev/null
+++ b/doc/devblog/day_99__catching_up_again/comment_2_c8363d47223e7bb899420e800bde3e27._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="https://www.google.com/accounts/o8/id?id=AItOawnE6kFAbud1LWrQuyX76yMYnUjHt9tR-A8"
+ nickname="Leonardo"
+ subject="Chroots"
+ date="2014-01-21T13:52:29Z"
+ content="""
+How about simply having sshd inside the chroot?
+"""]]
diff --git a/doc/devblog/day_9__Friday_the_13th.mdwn b/doc/devblog/day_9__Friday_the_13th.mdwn
new file mode 100644
index 0000000000..77d9039eb8
--- /dev/null
+++ b/doc/devblog/day_9__Friday_the_13th.mdwn
@@ -0,0 +1,21 @@
+Worked to get git-remote-gcrypt included in every git-annex autobuild
+bundle.
+(Except Windows; running a shell script there may need some work later..)
+
+Next I want to work on making the assistant easily able to create encrypted
+git repositories on removable drives. Which will involve a UI to select
+which gpg key to use, or creating (and backing up!) a gpg key.
+
+But, I got distracted chasing down some bugs on Windows. These were
+quite ugly; more direct mode mapping breakage which resulted in 
+files not being accessible. Also fsck on Windows failed to detect and fix
+the problem. All fixed now. (If you use git-annex on Windows, you should
+certainly upgrade and run `git annex fsck`.)
+
+As with most bugs in the Windows port, the underlying cause turned out to
+be stupid: `isSymlink` always returned False on Windows. Which makes sense
+from the perspective of Windows not quite having anything entirely like
+symlinks. But failed when that was being used to detect when files in the
+git tree being merged into the repository had the symlink bit set..
+
+Did bug triage. Backlog down to 32 (mostly messages from August).
diff --git a/doc/devblog/day_9__Friday_the_13th/comment_1_07195b4ec399ba1be6c8bdb3ae0fa50b._comment b/doc/devblog/day_9__Friday_the_13th/comment_1_07195b4ec399ba1be6c8bdb3ae0fa50b._comment
new file mode 100644
index 0000000000..aff4dc264a
--- /dev/null
+++ b/doc/devblog/day_9__Friday_the_13th/comment_1_07195b4ec399ba1be6c8bdb3ae0fa50b._comment
@@ -0,0 +1,12 @@
+[[!comment format=mdwn
+ username="http://nullroute.eu.org/~grawity/"
+ nickname="Mantas"
+ subject="comment 1"
+ date="2013-09-13T22:05:37Z"
+ content="""
+Windows *does* have something very much like symlinks – they're called *symlinks* (\"symbolic links\") and they are meant to [\"function just like UNIX links\"][1] as the official docs say.
+
+But on the other hand, yes – `isSymlink` makes less sense on Windows, because symlinks are just a type of reparse points, and there may be several other types (e.g. directory junctions should also be treated like symlinks, but mount points shouldn't...)
+
+[1]: http://msdn.microsoft.com/en-us/library/windows/desktop/aa365680(v=vs.85).aspx
+"""]]
diff --git a/doc/devblog/day__126-127__merge_fixes.mdwn b/doc/devblog/day__126-127__merge_fixes.mdwn
new file mode 100644
index 0000000000..f4a57a09d7
--- /dev/null
+++ b/doc/devblog/day__126-127__merge_fixes.mdwn
@@ -0,0 +1,61 @@
+Yesterday I learned of a nasty bug in handling of merges in direct mode. It
+turns out that if the remote repository has added a file, and there is a
+conflicting file in the local work tree, which has not been added to git, the
+local file was overwritten when git-annex did a merge. That's really bad, I'm
+very unhappy this bug lurked undetected for so long.
+
+Understanding the bug was easy. Fixing it turned out to be hard, because
+the automatic merge conflict resolution code was quite a mess. In
+particular, it wrote files to the work tree, which made it difficult for a
+later stage to detect and handle the abovementioned case. Also, the
+automatic merge resolution code had weird asymmetric structure that I never
+fully understood, and generally needed to be stared at for an hour to begin
+to understand it.
+
+In the process of cleaning that up, I wrote several more tests,
+to ensure that every case was handled correctly. Coverage was about 50%
+of the cases, and should now be 100%.
+
+To add to the fun, a while ago I had dealt with a bug on FAT/Windows where
+it sometimes lost the symlink bit during automatic merge resolution. Except
+it turned out my test case for it had a heisenbug, and I had not actually
+fixed it (I think). In any case, my old fix for it was a large part
+of the ugliness I was cleaning up, and had to be rewritten.
+Fully tracking down and dealing with that took a large part of today.
+
+Finally this evening, I added support for automatically handling merge
+conflicts where one side is an annexed file, and the other side has the
+same filename committed to git in the normal way. This is not an important
+case, but it's worth it for completeness. There was an unexpected benefit
+to doing it; it turned out that the weird asymmetric part of the code went
+away.
+
+The final core of the automatic merge conflict resolver has morphed from
+a mess I'd not want to paste here to a quite consise and easy to follow
+bit of code.
+
+[[!format haskell """
+        case (kus, kthem) of
+                -- Both sides of conflict are annexed files
+                (Just keyUs, Just keyThem) -> resolveby $
+                        if keyUs == keyThem
+                                then makelink keyUs
+                                else do
+                                        makelink keyUs
+                                        makelink keyThem
+                -- Our side is annexed file, other side is not.
+                (Just keyUs, Nothing) -> resolveby $ do
+                        graftin them file
+                        makelink keyUs
+                -- Our side is not annexed file, other side is.
+                (Nothing, Just keyThem) -> resolveby $ do
+                        graftin us file
+                        makelink keyThem
+                -- Neither side is annexed file; cannot resolve.
+                (Nothing, Nothing) -> return Nothing
+"""]]
+
+Since the bug that started all this is so bad, I want to make a release
+pretty soon.. But I will probably let it soak and whale on the test suite
+a bit more first. (This bug is also probably worth backporting to old
+versions of git-annex in eg Debian stable.)
diff --git a/doc/devblog/day__228_new_AWS.mdwn b/doc/devblog/day__228_new_AWS.mdwn
new file mode 100644
index 0000000000..3003a79852
--- /dev/null
+++ b/doc/devblog/day__228_new_AWS.mdwn
@@ -0,0 +1,13 @@
+New AWS region in Germany announced today. git-annex doesn't support it
+yet, unless you're using the `s3-aws` branch.
+
+I cleaned up that branch, got it building again, and re-tested it with
+`testremote`, and then fixed a problem the test suite found that was
+caused by some changes in the haskell aws library.
+
+Unfortunately, s3-aws is [not ready to be merged](http://git-annex.branchable.com/bugs/new_AWS_region___40__eu-central-1__41__)
+because of some cabal dependency problems involving `dbus` and `random`. I did
+go ahead and update Debian's haskell-aws package to cherry-pick
+from a newer version the change needed for Inernet Archive
+support, which allows building the s3-aws branch on Debian.
+Getting closer..
diff --git a/doc/devblog/day__329-330__a_rising_tide.mdwn b/doc/devblog/day__329-330__a_rising_tide.mdwn
new file mode 100644
index 0000000000..b23d2677f4
--- /dev/null
+++ b/doc/devblog/day__329-330__a_rising_tide.mdwn
@@ -0,0 +1,15 @@
+Things have been relatively quiet on git-annex this week. I've been
+distracted with other projects. But, a library that I developed for
+[propellor](http://propellor.branchable.com/) to help with 
+[concurrent console output](http://joeyh.name/blog/entry/concurrent_output_library/)
+has been rapidly developing into a kind of
+[tiling region manager for the console](http://joeyh.name/blog/entry/a_tiling_region_manager_for_the_console),
+which may be just the thing git-annex needs on the concurrent
+download progress display front.
+
+After seeing it could go that way, and working on it around the clock to
+add features git-annex will need, here's a teaser of its abilities.
+
+
+
+Probably coming soonish to a `git-annex -J` near you!
diff --git a/doc/devblog/moving_blogs.mdwn b/doc/devblog/moving_blogs.mdwn
new file mode 100644
index 0000000000..f6dac56c9e
--- /dev/null
+++ b/doc/devblog/moving_blogs.mdwn
@@ -0,0 +1,5 @@
+I've started a new page for my devblog, since I'm not focusing extensively
+on the assistant and so keeping the blog [[here|design/assistant/blog]]
+increasingly felt wrong. Also, my new year of
+[crowdfunded development](https://campaign.joeyh.name)
+formally starts in September, so a new blog seemed good.
diff --git a/doc/devblog/moving_blogs/comment_1_6caa7e67461a6ea5de8155ae9cf75fab._comment b/doc/devblog/moving_blogs/comment_1_6caa7e67461a6ea5de8155ae9cf75fab._comment
new file mode 100644
index 0000000000..46df4a7f68
--- /dev/null
+++ b/doc/devblog/moving_blogs/comment_1_6caa7e67461a6ea5de8155ae9cf75fab._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="http://pnijjar.livejournal.com/"
+ ip="99.236.22.229"
+ subject="comment 1"
+ date="2013-08-31T00:05:16Z"
+ content="""
+Do we need to update our RSS feeds? I appear to be getting your devblog posts in my old feed, but I do not know whether that will continue working.
+"""]]
diff --git a/doc/devblog/moving_blogs/comment_2_e3e2048fc2397b87a2f29c9fe49394cb._comment b/doc/devblog/moving_blogs/comment_2_e3e2048fc2397b87a2f29c9fe49394cb._comment
new file mode 100644
index 0000000000..19b5ae1c6f
--- /dev/null
+++ b/doc/devblog/moving_blogs/comment_2_e3e2048fc2397b87a2f29c9fe49394cb._comment
@@ -0,0 +1,10 @@
+[[!comment format=mdwn
+ username="https://www.google.com/accounts/o8/id?id=AItOawmkBwMWvNKZZCge_YqobCSILPMeK6xbFw8"
+ nickname="develop"
+ subject="comment 2"
+ date="2013-08-31T10:03:04Z"
+ content="""
+The old RSS feed will continue working.
+
+So sit back, relax, and enjoy the show.
+"""]]
diff --git a/doc/devblog/whither_XMPP.mdwn b/doc/devblog/whither_XMPP.mdwn
new file mode 100644
index 0000000000..9e99ba1acd
--- /dev/null
+++ b/doc/devblog/whither_XMPP.mdwn
@@ -0,0 +1,30 @@
+Pushed a release today. Rest of day spent beating head against Windows XMPP
+brick wall.
+
+Actually made a lot of progress -- Finally found the right approach, and
+got a clean build of the XMPP haskell libraries. But.. ghc fails to
+load the libraries when running Template Haskell. 
+"Misaligned section: 18206e5b".
+Filed [a bug report](https://ghc.haskell.org/trac/ghc/ticket/8830), 
+and I'm sure this alignment problem can be fixed,
+but I'm not hopeful about fixing it myself.
+
+One workaround would be to use the EvilSplicer, building once without the
+XMPP library linked in, to get the TH splices expanded, and then a second
+time with the XMPP library and no TH. Made a `winsplicehack` branch with
+tons of ifdefs that allows doing this. However, several dozen haskell libraries
+would need to be patched to get it to work. I have the patches from
+Android, but would rather avoid doing all that again on Windows.
+
+Another workaround would be to move XMPP into a separate process from the
+webapp. This is not very appealing either, the IPC between them would be
+fairly complicated since the webapp does stuff like show lists of XMPP
+buddies, etc. But, one thing this idea has to recommend it is I am already
+considering using a separate helper daemon like this for
+[[design/assistant/Telehash]].
+
+So there could be synergies between XMPP and Telehash support, possibly
+leading to some kind of plugin interface in git-annex for this sort of
+thing. But then, once Telehash or something like it is available and
+working well, I plan to deprecate XMPP entirely. It's been a flakey pain
+from the start, so that can't come too soon.
diff --git a/doc/devblog/youtube-dl.mdwn b/doc/devblog/youtube-dl.mdwn
new file mode 100644
index 0000000000..0a7fbb51b7
--- /dev/null
+++ b/doc/devblog/youtube-dl.mdwn
@@ -0,0 +1,30 @@
+Working on [[todo/switch_from_quvi_to_youtube-dl]], because
+quvi is not being maintained and youtube-dl can download a lot more stuff.
+
+Unfortunately, youtube-dl's interface is not a good fit for git-annex,
+compared with quvi's interface which was a near-perfect fit. Two things
+git-annex relied on quvi for are a way to check if a url has embedded media
+without downloading the url, and a way to get the url from which the
+embedded media can be downloaded. Youtube-dl supports neither. Also it has
+some other warts that make it unncessarily hard to interface with, like not
+always [storing the download in the location specified by --output](https://github.com/rg3/youtube-dl/issues/14864),
+and [sometimes crashing when downloading non-media urls (eg over my satellite internet)](http://bugs.debian.org/874321).
+
+I've found ways to avoid all these problems. For example, to make
+`git annex addurl` avoid unncessarily overhead of running youtube-dl
+in the common case of downloading some non-web-page file, I'll have it
+download the url content, and check if it looks like a html page.
+Only then will it use youtube-dl. So addurl of html pages without
+embedded media will get slower, but addurl of everything else
+will be as fast as before.
+
+But there's an unavoidable change to `addurl --relaxed`. It will not check
+for embedded media and more, because that would make it a lot slower, since
+it would have to hit the network. `addurl --fast` will have to be used for
+such urls instead. I hope this behavior change won't affect workflows
+badly.
+
+Today was all coding groundwork, and I just got to the point that I'm
+ready to have it run youtube-dl. Hope to finish it tomorrow.
+
+Today's work was sponsored by Jake Vosloo [on Patreon](https://www.patreon.com/joeyh).
diff --git a/doc/devblog/youtube-dl/comment_1_ff062ea66ea50d2f31a92c25017bb135._comment b/doc/devblog/youtube-dl/comment_1_ff062ea66ea50d2f31a92c25017bb135._comment
new file mode 100644
index 0000000000..172eee1b21
--- /dev/null
+++ b/doc/devblog/youtube-dl/comment_1_ff062ea66ea50d2f31a92c25017bb135._comment
@@ -0,0 +1,10 @@
+[[!comment format=mdwn
+ username="svw"
+ avatar="http://cdn.libravatar.org/avatar/539b90cb79086a2b1d7d3fbe48fd4184"
+ subject="Passing flags to youtube-dl"
+ date="2018-04-20T05:57:39Z"
+ content="""
+Joey - is there some way to pass flags to youtube-dl?
+
+For example the `--extract-audio` flag allows the extraction of just the audio stream.
+"""]]
diff --git a/doc/devblog/youtube-dl_day_2.mdwn b/doc/devblog/youtube-dl_day_2.mdwn
new file mode 100644
index 0000000000..a113339f72
--- /dev/null
+++ b/doc/devblog/youtube-dl_day_2.mdwn
@@ -0,0 +1,13 @@
+It's mostly working now. Still need to fix --fast and --relaxed, and avoid
+youtube-dl running out of the annex.diskreserve.
+
+The first hour or two was spent adding support for per-key temp
+directories. youtube-dl is run inside such a directory, to let it write
+whatever files it needs. Like the per-key temp files, these temp directories
+are not cleaned up when a download fails or is interrupted, so resuming can
+pick up where it left off. Taught `git annex dropunused` and everything
+else that cleans up per-key temp files to also clean up the temp
+directories.
+
+Today's work was sponsored by Trenton Cronholm on
+[Patreon](https://patreon.com/joeyh/)
diff --git a/doc/devblog/youtube-dl_day_3.mdwn b/doc/devblog/youtube-dl_day_3.mdwn
new file mode 100644
index 0000000000..bc868d25cd
--- /dev/null
+++ b/doc/devblog/youtube-dl_day_3.mdwn
@@ -0,0 +1,7 @@
+Finished up youtube-dl integration today, including all the edge cases in
+`addurl` and honoring annex.diskreserve.
+
+I changed my mind about `git annex addurl --relaxed`; it seems better for
+it to be slower than before, but not have surprising behavior, than to be
+fast but potentially surprising. If it's too slow, add `--raw` to avoid
+using youtube-dl.
diff --git a/doc/devblog/youtube-dl_day_3/comment_1_bcf6ccc65f191c446ae4ea70c4e86c87._comment b/doc/devblog/youtube-dl_day_3/comment_1_bcf6ccc65f191c446ae4ea70c4e86c87._comment
new file mode 100644
index 0000000000..f558092575
--- /dev/null
+++ b/doc/devblog/youtube-dl_day_3/comment_1_bcf6ccc65f191c446ae4ea70c4e86c87._comment
@@ -0,0 +1,27 @@
+[[!comment format=mdwn
+ username="Joseph Rawson"
+ subject="""comment 1"""
+ date="2017-12-31T16:23:04Z"
+ content="""
+Merry Christmas!  Thanks for adding this.  I was falling back to yt-dl when
+quvi failed.  I built from git last night and tested this.  I have a few
+comments and questions.
+
+- I am using debian/stretch.  Is yt-dl being used from system python package,
+or my virtualenv? (I have been preparing for py3 being default version in a
+couple of years.)
+
+- I have often used a terminal window and prefixed my input with "git-annex
+addurl" then drag links to the window for pasting.  Often, I have to press the
+up-arrow and run the command again.  The addurl behavior with urls that are
+already locally present, quvi responds "ok."  However, when repeating a command
+using yt-dl, when the url is already local, yt-dl refuses to overwrite, yet
+returns "failed."  I didn't know if you were aware of this.  This isn't a
+show-stopper, but just something I noticed.  I generally do "addurl" manually.
+
+- Playlist support.  Currently youtube-dl will allow you to download a whole
+playlist.  When I add a url "yt.com/watch?v=vidID&list=playlistID" only the
+video identified is downloaded.  This is a good default behavior.  When I add "
+yt.com/watch?list=listID" yt-dl successfully parses the whole list, but will
+only download one video from the list.
+"""]]
diff --git a/doc/devblog/youtube-dl_day_3/comment_2_423d1896981b9d5de85c89fdcde2d1ab._comment b/doc/devblog/youtube-dl_day_3/comment_2_423d1896981b9d5de85c89fdcde2d1ab._comment
new file mode 100644
index 0000000000..8ab6065d1f
--- /dev/null
+++ b/doc/devblog/youtube-dl_day_3/comment_2_423d1896981b9d5de85c89fdcde2d1ab._comment
@@ -0,0 +1,18 @@
+[[!comment format=mdwn
+ username="joey"
+ subject="""comment 2"""
+ date="2017-12-31T16:25:07Z"
+ content="""
+It uses whatever youtube-dl you have in PATH.
+
+I was not aware of the changed behavior when adding an url that's already
+locally present. I've opened [[bugs/addurl_youtube-dl_behavior_change]]
+to remember to look into it.
+
+The reason it only downloads one video from a playlist or channel is that
+git-annex needs to associate an url with the annexed file, so it can later
+download that single file when git-annex get is used, and there's no clear
+way to associate an url with a particular file that youtube-dl downloaded
+from a playlist or channel or whatever. [[tips/downloading_podcasts]] has a
+way to use `git annex importfeed` with a youtube playlist though.
+"""]]
diff --git a/doc/devblog/youtube-dl_day_3/comment_3_c3735d169bfe81eaba5557197b1c0eb0._comment b/doc/devblog/youtube-dl_day_3/comment_3_c3735d169bfe81eaba5557197b1c0eb0._comment
new file mode 100644
index 0000000000..904580913f
--- /dev/null
+++ b/doc/devblog/youtube-dl_day_3/comment_3_c3735d169bfe81eaba5557197b1c0eb0._comment
@@ -0,0 +1,9 @@
+[[!comment format=mdwn
+ username="joseph.rawson.works@85a210ab8c0e37a0b2d6bb235738b20e23e8878f"
+ nickname="joseph.rawson.works"
+ avatar="http://cdn.libravatar.org/avatar/6b473d5484b68803e8c47eeff9197397"
+ subject="Download playlist"
+ date="2018-01-06T20:34:31Z"
+ content="""
+Thanks!  I'll try testing the ```importfeed``` now!
+"""]]
diff --git a/doc/devblog/youtube-dl_day_3/comment_4_d19a698b28a5d6b583c527b9a909370d._comment b/doc/devblog/youtube-dl_day_3/comment_4_d19a698b28a5d6b583c527b9a909370d._comment
new file mode 100644
index 0000000000..a2a9be207c
--- /dev/null
+++ b/doc/devblog/youtube-dl_day_3/comment_4_d19a698b28a5d6b583c527b9a909370d._comment
@@ -0,0 +1,12 @@
+[[!comment format=mdwn
+ username="joseph.rawson.works@85a210ab8c0e37a0b2d6bb235738b20e23e8878f"
+ nickname="joseph.rawson.works"
+ avatar="http://cdn.libravatar.org/avatar/6b473d5484b68803e8c47eeff9197397"
+ subject="testing importfeed"
+ date="2018-01-06T21:57:06Z"
+ content="""
+I tested importfeed.  It seemed to be stuck on ```importfeed checking known urls```.  The first playlist I tried had 70+ videos, so I tested a shorter playlist and got the same response.  I noticed that on the [[tips/downloading_podcasts]] page, only channels are mentions, but not playlists.  However, you mentioned playlists in your response.  I was able to work around the issue:
+
+```for _id in `youtube-dl 'https://www.youtube.com/watch?v=ps40Uminvno&list=PLWnVxuqvY7Jj2Z8aeR-qKXACCXUN4Ouv_' --get-id` ; do echo \"https://www.youtube.com/watch?v=${_id}\" ; done | git-annex addurl --batch --jobs 2```
+
+"""]]
diff --git a/doc/direct_mode.mdwn b/doc/direct_mode.mdwn
new file mode 100644
index 0000000000..d3e1067f9a
--- /dev/null
+++ b/doc/direct_mode.mdwn
@@ -0,0 +1,129 @@
+Normally, git-annex repositories consist of symlinks that are checked into
+git, and in turn point at the content of large files that is stored in
+`.git/annex/objects/`. Direct mode gets rid of the symlinks.
+
+The advantage of direct mode is that you can access files directly,
+including modifying them. The disadvantage is that many regular git
+commands cannot be used in a direct mode repository, since they don't
+understand how to update its working tree.
+
+[[!toc]]
+
+## deprecated
+
+Direct mode is deprecated! Intead, git-annex v6 repositories can simply
+have files that are unlocked and thus can be directly accessed and
+modified. See [[upgrades]] for details about the transition to v6
+repositories.
+
+## enabling (and disabling) direct mode
+
+Normally, git-annex repositories start off in indirect mode. With some
+exceptions:
+
+* Repositories created by the [[assistant]] use direct mode by default.
+* Repositories on FAT and other less than stellar filesystems
+  that don't support things like symlinks will be automatically put
+  into direct mode.
+* Windows always uses direct mode.
+
+Any repository can be converted to use direct mode at any time, and if you
+decide not to use it, you can convert back to indirect mode just as easily.
+Also, you can have one clone of a repository using direct mode, and another
+using indirect mode.
+
+To start using direct mode:
+
+	git annex direct
+
+To stop using direct mode:
+
+	git annex indirect
+
+## safety of using direct mode
+
+With direct mode, you're operating without large swathes of git-annex's
+carefully constructed safety net, which ensures that past versions of
+files are preserved and can be accessed. 
+With direct mode, any file can be edited directly, or deleted at any time,
+and there's no guarantee that the old version is backed up somewhere else.
+
+So if you care about preserving the history of files, you're strongly
+encouraged to tell git-annex that your direct mode repository cannot be
+trusted to retain the content of a file. To do so:
+
+	git annex untrust .
+
+On the other hand, if you only care about the current versions of files,
+and are using git-annex with direct mode to keep files synchronised between
+computers, and manage your files, this should not be a concern for you.
+
+## use a direct mode repository
+
+You can use most git-annex commands as usual in a direct mode repository.
+
+Direct mode also works well with the git-annex assistant.
+
+The most important command to use in a direct mode repository is `git annex
+sync`. This will commit any files you have run `git annex add` on, as well
+as files that were added earlier and have been modified. It will push 
+the changes to other repositories for `git annex sync` there to pick up,
+and will pull and merge any changes made on other repositories into the
+local repository.
+
+## what doesn't work in direct mode
+
+A very few git-annex commands don't work in direct mode, and will refuse
+to do anything. For example, `git annex unlock` doesn't make sense in
+direct mode.
+
+As for git commands, direct mode prevents using any git command that would
+modify or access the work tree. So you cannot `git commit` or `git pull`
+(use `git annex sync` for both instead), or run `git status` (use `git
+annex status` instead). These git commands will complain "fatal: This
+operation must be run in a work tree".
+
+The reason for this is that git doesn't understand how git-annex uses the
+work tree in direct mode. Where git expects the symlinks that get checked
+into git to be checked out in the work tree, direct mode instead replaces
+them with the actual content of files, as managed by git-annex.
+
+There are still lots of git commands you can use in direct mode. For
+example, you can run `git log` on files, run `git push`, `git fetch`,
+`git config`, `git remote add` etc.
+
+## proxing git commands in direct mode
+
+For those times when you really need to run a command like `git revert
+HEAD` in a direct mode repository, git-annex has the ability to proxy
+the command to work in direct mode.
+
+For example:
+
+	git annex proxy -- git revert HEAD
+
+	git annex proxy -- git checkout HEAD^^
+
+	git annex proxy -- git mv mydir newname
+
+This works by setting up a temporary work tree, letting the git
+command run on that work tree, and then updating the real work
+tree to reflect any changes staged or committed by the git command,
+with appropriate handling of the direct mode files.
+
+## undoing changes in direct mode
+
+There is also the `undo` command to do the equivalent of the above revert
+in a simpler way. Say you made a change in direct mode, the assistant
+dutifully committed it and you realise your mistake, you can try:
+
+    git annex undo file
+
+## forcing git to use the work tree in direct mode
+
+This is for experts only. You can lose data doing this, or check enormous
+files directly into your git repository, and it's your fault if you do!
+
+Ok, with the warnings out of the way, all you need to do to make any
+git command access the work tree in direct mode is pass it 
+`-c core.bare=false`
diff --git a/doc/direct_mode/comment_11_1c79c93f4b17cfc354ab920e3775cc60._comment b/doc/direct_mode/comment_11_1c79c93f4b17cfc354ab920e3775cc60._comment
new file mode 100644
index 0000000000..ad1b66bc07
--- /dev/null
+++ b/doc/direct_mode/comment_11_1c79c93f4b17cfc354ab920e3775cc60._comment
@@ -0,0 +1,26 @@
+[[!comment format=mdwn
+ username="http://www.gl-como.it/author/valhalla/"
+ nickname="valhalla"
+ subject="Direct mode clone of an indirect repo"
+ date="2013-08-18T08:47:35Z"
+ content="""
+I too have issues with mixing direct and indirect mode repositories.
+
+ I have a regular, existing repository with ebooks, shared between various clones on proper :) filesystems; now I would need a copy of some of them on an ereader which only offers a FAT filesystem, so it has to be direct mode.
+
+    mount $READER
+    cd $reader
+    git clone $REPO
+
+I get a directory full of small files, the way git manages links on FAT.
+
+    git annex init \"ebook reader\"
+
+This detects the fact that it is working on a crippled filesystem, enables direct mode and disables ssh connection caching; up to now everything seems to be fine, but then
+
+    git annex get $SOME_BOOK
+
+seems to work, downloads the file somewhere, but when I try to open $SOME_BOOK it is still the fake link, and the file has been downloaded in its destination, as if the repo wasn't in direct mode.
+
+I use version 4.20130723 on debian jessie
+"""]]
diff --git a/doc/direct_mode/comment_12_1b5218fdb6ee362d6df68ff1229590d4._comment b/doc/direct_mode/comment_12_1b5218fdb6ee362d6df68ff1229590d4._comment
new file mode 100644
index 0000000000..1743d5df99
--- /dev/null
+++ b/doc/direct_mode/comment_12_1b5218fdb6ee362d6df68ff1229590d4._comment
@@ -0,0 +1,10 @@
+[[!comment format=mdwn
+ username="http://joeyh.name/"
+ ip="4.154.0.63"
+ subject="comment 12"
+ date="2013-08-23T17:48:54Z"
+ content="""
+There should be no obstacles to using direct mode on one clone of a git repository, and indirect mode on another clone. The data stored in git for either mode is identical, and I do this myself for some repositories.
+
+@valhalla, you probably need to run `git annex fsck`, and if that does not solve your problem, you need to file a bug report.
+"""]]
diff --git a/doc/direct_mode/comment_12_7d507b6f87085a19d8dd5014f580922b._comment b/doc/direct_mode/comment_12_7d507b6f87085a19d8dd5014f580922b._comment
new file mode 100644
index 0000000000..bc8f006800
--- /dev/null
+++ b/doc/direct_mode/comment_12_7d507b6f87085a19d8dd5014f580922b._comment
@@ -0,0 +1,24 @@
+[[!comment format=mdwn
+ username="http://mildred.fr/"
+ ip="82.247.184.53"
+ subject="Fixing symlinks to the annex store in direct mode"
+ date="2014-07-16T06:52:33Z"
+ content="""
+I have an issue with direct mode: I have tons of symlinks that points to the git-annex store, while in direct mode. After investigation, I found that these files don't seem to be part of the repository. I can check with:
+
+    $ git -c core.bare=false status --porcelain tr.html
+    ?? Documentation/Mozilla/developer.mozilla.org/tr.html
+
+    $ ls -l tr.html
+    lrwxrwxrwx 1 mildred mildred 205 10 juin  16:22 tr.html -> ../../../.git/annex/objects/gF/z1/SHA256E-s31895--c873982bd742ba8db6e026afee26b7ab2f75f54f587304d8c2d877db3900c0f6.html/SHA256E-s31895--c873982bd742ba8db6e026afee26b7ab2f75f54f587304d8c2d877db3900c0f6.html
+
+The link is valid, and is probably pointing to a unused file in the annex store. How to add these files back?
+
+If I was in indirect mode, I could simply use `git add tr.html` (and **not** `git annex add`). This would stage to the git staging area the symlink, and all would be well.
+
+I found that in direct mode, the same was true. The command is: 
+
+    git -c core.bare=false add tr.html
+
+The file is added to the repository, and the link is converted to the target file. The question now is why? There is no hook on the add command. Could it be the annex assistant? If that didn't worked, my question would have been: how to checkout a file in direct mode?
+"""]]
diff --git a/doc/direct_mode/comment_13_5169c5541970d3b3bc1e080e07539b22._comment b/doc/direct_mode/comment_13_5169c5541970d3b3bc1e080e07539b22._comment
new file mode 100644
index 0000000000..7d15fca6f8
--- /dev/null
+++ b/doc/direct_mode/comment_13_5169c5541970d3b3bc1e080e07539b22._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="http://mildred.fr/"
+ ip="82.247.184.53"
+ subject="Re: Fixing symlinks to the annex store in direct mode"
+ date="2014-07-16T07:04:08Z"
+ content="""
+What I said was just wrong. Instead of trying on tr.html, I tried with a copy of that file. But making the copy of that file had the wanted effect, transforming the symlink to the actual file. Probably the assistant monitoring file creation, and transforming indirect file to direct file. Or perhaps the `cp` command follows symlinks by default (I thought it didn't).
+"""]]
diff --git a/doc/direct_mode/comment_13_55108ac736ea450df89332ba5de4a208._comment b/doc/direct_mode/comment_13_55108ac736ea450df89332ba5de4a208._comment
new file mode 100644
index 0000000000..b8d80cd7ea
--- /dev/null
+++ b/doc/direct_mode/comment_13_55108ac736ea450df89332ba5de4a208._comment
@@ -0,0 +1,12 @@
+[[!comment format=mdwn
+ username="http://joeyh.name/"
+ ip="4.154.0.63"
+ subject="comment 13"
+ date="2013-08-23T17:50:15Z"
+ content="""
+@obergix asked:
+
+> But then, how can a direct repo sync with changes made in other remotes, if there no pull/fetch available.
+
+The answer is simple: By running `git annex sync`, which handles all that.
+"""]]
diff --git a/doc/direct_mode/comment_14_03a02e689d92faa596de98e02b2ffe28._comment b/doc/direct_mode/comment_14_03a02e689d92faa596de98e02b2ffe28._comment
new file mode 100644
index 0000000000..ce47a5eb8c
--- /dev/null
+++ b/doc/direct_mode/comment_14_03a02e689d92faa596de98e02b2ffe28._comment
@@ -0,0 +1,10 @@
+[[!comment format=mdwn
+ username="http://joeyh.name/"
+ ip="209.250.56.2"
+ subject="comment 14"
+ date="2014-07-16T17:51:28Z"
+ content="""
+I kinda wish people would post questions to the forum, and not clutter up this page..
+
+Anyway, there have been past bugs in the direct mode code that caused some files to not be checked out in direct mode, but stay as symlinks pointing at the content. That can be fixed by running `git annex fsck`. But, I am not aware of any problem that can leave a git-annex symlink that is not checked into git at all. Perhaps you copied the symlink from another location?
+"""]]
diff --git a/doc/direct_mode/comment_14_ff4ffc2aabc5fd174d7386ef13860f78._comment b/doc/direct_mode/comment_14_ff4ffc2aabc5fd174d7386ef13860f78._comment
new file mode 100644
index 0000000000..3a538e00b3
--- /dev/null
+++ b/doc/direct_mode/comment_14_ff4ffc2aabc5fd174d7386ef13860f78._comment
@@ -0,0 +1,10 @@
+[[!comment format=mdwn
+ username="http://olivier.berger.myopenid.com/"
+ nickname="obergix"
+ subject="Git annex copy needed before git annex sync"
+ date="2013-08-23T19:59:35Z"
+ content="""
+Thanks for these details @joeyh. But AFAIU, one needs to proceed to the git annex copy before doing the git annex sync, otherwise, symlinks (or files containing the symlink path on SMB) will be created, instead of the plain \"direct\" files that are expected.
+
+I'm still not sure whether the git annex sync needs to be issued on either of the indirect or direct remotes first, or both, then in which sequence. I think a \"walkthrough\" script would help.
+"""]]
diff --git a/doc/direct_mode/comment_15_1cd32456630b25d5aaa6d2763e6eb384._comment b/doc/direct_mode/comment_15_1cd32456630b25d5aaa6d2763e6eb384._comment
new file mode 100644
index 0000000000..d265118a04
--- /dev/null
+++ b/doc/direct_mode/comment_15_1cd32456630b25d5aaa6d2763e6eb384._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="http://joeyh.name/"
+ ip="4.154.0.63"
+ subject="comment 15"
+ date="2013-08-24T15:56:47Z"
+ content="""
+No, you can sync before you copy, get, or whatever. git-annex will replace the symlinks with the actual files when they arrive at the repository.
+"""]]
diff --git a/doc/direct_mode/comment_15_599b2285d24ae1244a1945d572b2c397._comment b/doc/direct_mode/comment_15_599b2285d24ae1244a1945d572b2c397._comment
new file mode 100644
index 0000000000..d8c5e825fc
--- /dev/null
+++ b/doc/direct_mode/comment_15_599b2285d24ae1244a1945d572b2c397._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="https://www.google.com/accounts/o8/id?id=AItOawmjjrCHEIa4vpDIJoBuJsrF3y8wZQElVHw"
+ nickname="Siyuan"
+ subject="Non-direct mode for Windows"
+ date="2014-10-08T15:55:04Z"
+ content="""
+Why Windows is restricted to direct mode? NTFS has symbolic links too. Is that fundamentally different from POSIX symlinks that it cannot be done?
+"""]]
diff --git a/doc/direct_mode/comment_16_7f6805e090d0acd8a077b65214da5837._comment b/doc/direct_mode/comment_16_7f6805e090d0acd8a077b65214da5837._comment
new file mode 100644
index 0000000000..d40ee9e985
--- /dev/null
+++ b/doc/direct_mode/comment_16_7f6805e090d0acd8a077b65214da5837._comment
@@ -0,0 +1,7 @@
+[[!comment format=mdwn
+ username="https://id.koumbit.net/anarcat"
+ subject="comment 16"
+ date="2015-02-17T05:22:00Z"
+ content="""
+i believe this is [answered here](https://git-annex.branchable.com/todo/windows_support/#comment-e72601243c643d7821e68d3a04489fcb). TLDR; basically NTFS + symlink works in Linux, but not in Windows/Cygwin, which git-annex seems to be using. YMMV.
+"""]]
diff --git a/doc/direct_mode/comment_17_e7c066fba8e28c61e8517c7a18a02457._comment b/doc/direct_mode/comment_17_e7c066fba8e28c61e8517c7a18a02457._comment
new file mode 100644
index 0000000000..6847c5f9ee
--- /dev/null
+++ b/doc/direct_mode/comment_17_e7c066fba8e28c61e8517c7a18a02457._comment
@@ -0,0 +1,7 @@
+[[!comment format=mdwn
+ username="mitzip"
+ subject="--depth undo option"
+ date="2015-05-27T02:54:17Z"
+ content="""
+Well I just spent 4 hours building a OSX GUI around git annex undo --depth and it responds with \"git-annex: unrecognized option '--depth'\"... lol Please tell me that option exists somewhere other than the documentation... lol
+"""]]
diff --git a/doc/direct_mode/comment_18_e46f70efa6d8d65d5ef81cdcbd844869._comment b/doc/direct_mode/comment_18_e46f70efa6d8d65d5ef81cdcbd844869._comment
new file mode 100644
index 0000000000..ed9d016519
--- /dev/null
+++ b/doc/direct_mode/comment_18_e46f70efa6d8d65d5ef81cdcbd844869._comment
@@ -0,0 +1,13 @@
+[[!comment format=mdwn
+ username="joey"
+ subject="""comment 1"""
+ date="2015-05-27T18:31:26Z"
+ content="""
+This page mentioned a --depth option, but no, it was never implemented.
+I have updated this page.
+
+If you need something simpler than the basic undo, you can use `git annex
+proxy` to run eg, a `git revert`.
+
+(Deleted a forum thread that duplicated the previous comment.)
+"""]]
diff --git a/doc/direct_mode/comment_19_f4f23d9be76fa3c201b6c67efab0d69c._comment b/doc/direct_mode/comment_19_f4f23d9be76fa3c201b6c67efab0d69c._comment
new file mode 100644
index 0000000000..1d47744bbb
--- /dev/null
+++ b/doc/direct_mode/comment_19_f4f23d9be76fa3c201b6c67efab0d69c._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="Ilya_Shlyakhter"
+ avatar="http://cdn.libravatar.org/avatar/1647044369aa7747829c38b9dcc84df0"
+ subject="comment 19"
+ date="2018-09-19T19:12:58Z"
+ content="""
+This page says direct mode is \"deprecated\"; but, v6 is not yet official ( https://git-annex.branchable.com/forum/default_repo_version_is_still_5__63__/ )?  So, for now, direct mode is the recommended thing to use?
+"""]]
diff --git a/doc/direct_mode/comment_20_4d0077d7881df05dcbeb460bf7734b6b._comment b/doc/direct_mode/comment_20_4d0077d7881df05dcbeb460bf7734b6b._comment
new file mode 100644
index 0000000000..803b263cd0
--- /dev/null
+++ b/doc/direct_mode/comment_20_4d0077d7881df05dcbeb460bf7734b6b._comment
@@ -0,0 +1,11 @@
+[[!comment format=mdwn
+ username="joey"
+ subject="""comment 20"""
+ date="2018-09-24T15:40:48Z"
+ content="""
+Indirect mode is the recommended thing to use, except for situations where
+it doesn't work, then direct mode is used as a fallback.
+
+v6 is getting fairly close to working better than direct mode at this
+point.
+"""]]
diff --git a/doc/direct_mode/comment_3_8020d74bddf0e38b0a297e5dae7c217b._comment b/doc/direct_mode/comment_3_8020d74bddf0e38b0a297e5dae7c217b._comment
new file mode 100644
index 0000000000..f89cafe533
--- /dev/null
+++ b/doc/direct_mode/comment_3_8020d74bddf0e38b0a297e5dae7c217b._comment
@@ -0,0 +1,12 @@
+[[!comment format=mdwn
+ username="https://www.google.com/accounts/o8/id?id=AItOawl2Jj8q2upJL4ZQAc2lp7ugTxJiGtcICv8"
+ nickname="Michael"
+ subject="comment 3"
+ date="2013-02-19T03:03:14Z"
+ content="""
+So, if I edit a \"content file\" (change a music file's metadata, say), what's the workflow to record that fact and then synchronise it to other repositories?
+
+I can't do a `git add`, so I don't understand what has to happen as a first step.  (Thanks for your quick reply above, BTW.)
+
+
+"""]]
diff --git a/doc/direct_mode/comment_4_97c26bd82f623a3b2d56bab4afff0126._comment b/doc/direct_mode/comment_4_97c26bd82f623a3b2d56bab4afff0126._comment
new file mode 100644
index 0000000000..cf5ae2fb64
--- /dev/null
+++ b/doc/direct_mode/comment_4_97c26bd82f623a3b2d56bab4afff0126._comment
@@ -0,0 +1,12 @@
+[[!comment format=mdwn
+ username="http://joeyh.name/"
+ ip="4.152.108.183"
+ subject="comment 4"
+ date="2013-02-19T03:05:35Z"
+ content="""
+
    +git annex add $file
+git annex sync
+git annex copy $file --to otherrepo
+
    
+"""]]
diff --git a/doc/direct_mode/comment_5_42363bf0367f935b3eee8ad3d2eaf5cf._comment b/doc/direct_mode/comment_5_42363bf0367f935b3eee8ad3d2eaf5cf._comment
new file mode 100644
index 0000000000..d4e2962a64
--- /dev/null
+++ b/doc/direct_mode/comment_5_42363bf0367f935b3eee8ad3d2eaf5cf._comment
@@ -0,0 +1,10 @@
+[[!comment format=mdwn
+ username="mildred"
+ ip="2a01:e35:2f7b:8350:225:22ff:fe40:fdfc"
+ subject="What happens to object database in direct mode?"
+ date="2013-07-08T13:27:21Z"
+ content="""
+What happens to the object database (`.git/annex/objects`) when going to direct mode? Are the objects deleted, moved to another location, kept?
+
+If the objects are kept, does it means that the file on the repository in direct mode is duplicated in the object database? If so, would it be relevant to use `cp --reflink=auto` to populate the working directory to enable copy on write on filesystems that supports it?
+"""]]
diff --git a/doc/direct_mode/comment_6_5f03b1686c1fb3f7606a5bc724ac3812._comment b/doc/direct_mode/comment_6_5f03b1686c1fb3f7606a5bc724ac3812._comment
new file mode 100644
index 0000000000..1cdb9a79c7
--- /dev/null
+++ b/doc/direct_mode/comment_6_5f03b1686c1fb3f7606a5bc724ac3812._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="http://joeyh.name/"
+ ip="4.153.254.222"
+ subject="comment 6"
+ date="2013-07-08T16:11:32Z"
+ content="""
+`.git/annex/objects` does not typically contain any file contents in direct mode. The file contents are stored directly in the working tree.
+"""]]
diff --git a/doc/direct_mode/comment_7_5355ac418bfb26e990762b80f4c36b77._comment b/doc/direct_mode/comment_7_5355ac418bfb26e990762b80f4c36b77._comment
new file mode 100644
index 0000000000..d416a01f62
--- /dev/null
+++ b/doc/direct_mode/comment_7_5355ac418bfb26e990762b80f4c36b77._comment
@@ -0,0 +1,12 @@
+[[!comment format=mdwn
+ username="http://caust1c.myopenid.com/"
+ nickname="asbraithwaite"
+ subject="comment 7"
+ date="2013-08-12T18:06:21Z"
+ content="""
+Would it be safe to add largefiles to gitignore in direct mode?
+
+Can git-annex still track large files ignored by git?
+
+Thanks. :-)
+"""]]
diff --git a/doc/direct_mode/comment_8_6cd15e2c5fd0bef48f60c6993322c2fc._comment b/doc/direct_mode/comment_8_6cd15e2c5fd0bef48f60c6993322c2fc._comment
new file mode 100644
index 0000000000..72ba16baf4
--- /dev/null
+++ b/doc/direct_mode/comment_8_6cd15e2c5fd0bef48f60c6993322c2fc._comment
@@ -0,0 +1,9 @@
+[[!comment format=mdwn
+ username="arand"
+ ip="130.243.226.21"
+ subject="comment 8"
+ date="2013-08-12T18:12:32Z"
+ content="""
+asbraithwaite:
+No, as far as I know it can not.
+"""]]
diff --git a/doc/distributed_version_control.mdwn b/doc/distributed_version_control.mdwn
new file mode 100644
index 0000000000..ccf82b91c7
--- /dev/null
+++ b/doc/distributed_version_control.mdwn
@@ -0,0 +1,21 @@
+In git, there can be multiple clones of a repository, each clone can 
+be independently modified, and clones can push or pull changes to
+one-another to get back in sync.
+
+git-annex preserves that fundamental distributed nature of git, while
+dropping the requirement that, once in sync, each clone contains all the data
+that was committed to each other clone. Instead of storing the content
+of a file in the repository, git-annex stores a pointer to the content.
+
+Each git-annex repository is responsible for storing some of the content,
+and can copy it to or from other repositories. [[Location_tracking]]
+information is committed to git, to let repositories inform other
+repositories what file contents they have available.
+
+---
+
+The [[walkthrough]] shows how to create a distributed set of git-annex
+repositories with no central repository.
+
+Prefer a central repository like GitHub? See the
+[[tips/centralized_git_repository_tutorial]].
diff --git a/doc/download.mdwn b/doc/download.mdwn
new file mode 100644
index 0000000000..cda592589e
--- /dev/null
+++ b/doc/download.mdwn
@@ -0,0 +1,31 @@
+The main git repository for git-annex is `git://git-annex.branchable.com/`
+
+(You can push changes to this wiki from that anonymous git checkout.)
+
+Other mirrors of the git repository:
+
+* `git://git.kitenet.net/git-annex` [[gitweb](http://git.kitenet.net/?p=git-annex.git;a=summary)]
+
+Releases of git-annex are uploaded
+[to hackage](http://hackage.haskell.org/package/git-annex). Note that the
+tarball there is not the complete git-annex source tree, but only a subset
+to make `cabal install` work. Use git to checkout the full source tree.
+
+Some operating systems include git-annex in easily prepackaged form and
+others need some manual work. See [[install]] for details.
+
+## git branches
+
+The git repository has some branches:
+
+* `ghc7.0` is a by now very out of date branch that can be built with
+  ghc 7.0.
+* `debian-*-backport` contains the latest backport of git-annex.
+* `tweak-fetch` adds support for the git tweak-fetch hook, which has
+  been proposed and implemented but not yet accepted into git.
+* `setup` contains configuration for this website
+
+----
+
+Developing git-annex? Patches are very welcome.
+You should read [[coding_style]].
diff --git a/doc/download/comment_1_ec2578241a966cfcdd43f2a26a5c8709._comment b/doc/download/comment_1_ec2578241a966cfcdd43f2a26a5c8709._comment
new file mode 100644
index 0000000000..e242230a05
--- /dev/null
+++ b/doc/download/comment_1_ec2578241a966cfcdd43f2a26a5c8709._comment
@@ -0,0 +1,13 @@
+[[!comment format=mdwn
+ username="https://www.google.com/accounts/o8/id?id=AItOawm3uJkdiJJejvqix9dULvw_Ma7DCtB-6zA"
+ nickname="Ian"
+ subject="git clone git://git-annex.branchable.com/ gives an error"
+ date="2012-08-13T20:57:34Z"
+ content="""
+Thought you would want to know
+
+error: unable to create file doc/forum/__91__Installation__93___base-3.0.3.2_requires_syb___61____61__0.1.0.2_however_syb-0.1.0.2_was_excluded_because_json-0.5_requires_syb___62____61__0.3.3.mdwn (File name too long)
+fatal: cannot create directory at 'doc/forum/__91__Installation__93___base-3.0.3.2_requires_syb___61____61__0.1.0.2_however_syb-0.1.0.2_was_excluded_because_json-0.5_requires_syb___62____61__0.3.3': File name too long
+
+
+"""]]
diff --git a/doc/download/comment_2_ee0d158ac59903737dbc4ef632f11fe3._comment b/doc/download/comment_2_ee0d158ac59903737dbc4ef632f11fe3._comment
new file mode 100644
index 0000000000..d3024217a9
--- /dev/null
+++ b/doc/download/comment_2_ee0d158ac59903737dbc4ef632f11fe3._comment
@@ -0,0 +1,7 @@
+[[!comment format=mdwn
+ username="http://joeyh.name/"
+ subject="comment 2"
+ date="2012-08-16T23:28:30Z"
+ content="""
+Ok, I've renamed that long-ish filename.
+"""]]
diff --git a/doc/download/comment_3_b59fc32a18a9fda8da120da82fade0a6._comment b/doc/download/comment_3_b59fc32a18a9fda8da120da82fade0a6._comment
new file mode 100644
index 0000000000..27fa71ca08
--- /dev/null
+++ b/doc/download/comment_3_b59fc32a18a9fda8da120da82fade0a6._comment
@@ -0,0 +1,9 @@
+[[!comment format=mdwn
+ username="jckuester@11e9fceb442172b6db63320bb545797a47a4c103"
+ nickname="jckuester"
+ subject="error during "git clone git://git.kitenet.net/git-annex""
+ date="2015-11-22T15:38:57Z"
+ content="""
+error: unable to create file doc/bugs/On_Lubuntu_14.04_assistant_fails_to_create_new_setup_or_actually_work___40__fixed_by_regular_lxsession_package_update_from_2014-06-30__41__.mdwn (File name too long)
+fatal: unable to checkout working tree
+"""]]
diff --git a/doc/download/comment_4_bf102d87552f97ac293dd20f6822b84f._comment b/doc/download/comment_4_bf102d87552f97ac293dd20f6822b84f._comment
new file mode 100644
index 0000000000..2993a6e101
--- /dev/null
+++ b/doc/download/comment_4_bf102d87552f97ac293dd20f6822b84f._comment
@@ -0,0 +1,11 @@
+[[!comment format=mdwn
+ username="joey"
+ subject="""comment 4"""
+ date="2015-11-30T19:13:34Z"
+ content="""
+I guess you guys with the badly limited filename lenghs are using encfs or
+something like that. encfs is [really shoddy](https://defuse.ca/audits/encfs.htm),
+and I feel kind of like someone is complaining that the git repo can't be
+checked out on MS-DOS. 
+8.3 filenames are so 1985, and 128 filenames are so.. 1995?
+"""]]
diff --git a/doc/ekg.mdwn b/doc/ekg.mdwn
new file mode 100644
index 0000000000..26d0128a63
--- /dev/null
+++ b/doc/ekg.mdwn
@@ -0,0 +1,31 @@
+You can `cabal configure -fEKG` to build a git-annex that includes
+the EKG remote monitoring interface.
+
+To access the EKG control panel, go to
+ while a git-annex command is running.
+
+This EKG build is mostly useful for debugging resource usage problems.
+
+[[!img ekg.png caption="git-annex webapp startup, and assistant startup scan"]]
+
+Note that since only one process can open port 4242 at a time, running
+more than one git-annex process with EKG support at the same time can
+result in some "resource busy (Address already in use)" messages -- but
+git-annex will continue to work.
+
+----
+
+## full profiling
+
+For the really tricky memory leaks, here's how to make a profiling build of
+git-annex.
+
+1. `cabal configure --enable-profiling`  
+  This will probably fail due to some missing profiling libraries.
+  You have to get the profiling versions of all needed haskell libraries
+  installed somehow.
+2. `cabal build`
+3. Run git-annex with the special flags `+RTS -hc -p`
+4. Reproduce the memory leak problem.
+5. If the assistant was run, stop it.
+6. `hp2ps -e8in -c git-annex.hp` to generate a .ps graph of memory usage.
diff --git a/doc/ekg/ekg.png b/doc/ekg/ekg.png
new file mode 100644
index 0000000000..a8d16a2d81
Binary files /dev/null and b/doc/ekg/ekg.png differ
diff --git a/doc/encryption.mdwn b/doc/encryption.mdwn
new file mode 100644
index 0000000000..ddde431b1c
--- /dev/null
+++ b/doc/encryption.mdwn
@@ -0,0 +1,131 @@
+[[!toc]]
+
+git-annex mostly does not use encryption. Anyone with access to a git
+repository can see all the filenames in it, its history, and can access
+any annexed file contents.
+
+Encryption is needed when using [[special_remotes]] like Amazon S3, where
+file content is sent to an untrusted party who does not have access to the
+git repository.
+
+Such an encrypted remote uses strong ([[symmetric|design/encryption]] or
+asymmetric) encryption on the contents of files, as well as HMAC hashing
+of the filenames. The size of the encrypted files, and access patterns
+of the data, should be the only clues to what is stored in such a
+remote.
+
+You should decide whether to use encryption with a special remote before
+any data is stored in it. So, `git annex initremote` requires you
+to specify "encryption=none" when first setting up a remote in order
+to disable encryption. To use encryption, you run
+`git-annex initremote` in one of these ways:
+
+* `git annex initremote newremote type=... encryption=hybrid keyid=KEYID`
+* `git annex initremote newremote type=... encryption=shared`
+* `git annex initremote newremote type=... encryption=pubkey keyid=KEYID`
+* `git annex initremote newremote type=... encryption=sharedpubkey keyid=KEYID`
+
+To see what encryption is used for a special remote, run
+`git annex info $remote` and look for a line like:
+
+	encryption: hybrid (to gpg keys: AEC828149D85C538 C910D9122512E3C8)
+
+## hybrid encryption keys (encryption=hybrid)
+
+The [[hybrid_key_design|design/encryption]] allows additional
+encryption keys to be added on to a special remote later. Due to this
+flexibility, it is the default and recommended encryption scheme.
+
+	git annex initremote newremote type=... [encryption=hybrid] keyid=KEYID
+
+The KEYID is passed to `gpg` to find gpg keys.
+Typically, you will say "keyid=2512E3C7" to use a specific gpg key.
+Or, you might say "keyid=id@joeyh.name" to search for matching keys.
+
+To add a new key and allow it to access all the content that is stored
+in the encrypted special remote, just run `git annex
+enableremote` specifying the keyid to add:
+
+	git annex enableremote myremote keyid+=788A3F4C
+
+You can repeat this process to add any number of gpg keys, including
+your own gpg keys and any public keys of others who you want to give
+access. Anyone with a corresponding secret key will be able to decrypt
+all content that is stored in the remote.
+
+While a key can later be removed from the list, note that it will **not**
+prevent the owner of the key from accessing data on the remote (which is by
+design impossible to prevent, short of deleting the remote). In fact the
+only sound use of `keyid-=` is probably to replace a revoked key:
+
+	git annex enableremote myremote keyid-=2512E3C7 keyid+=788A3F4C
+
+See also [[encryption_design|design/encryption]] for other security
+risks associated with encryption.
+
+## shared encryption key (encryption=shared)
+
+Alternatively, you can configure git-annex to use a shared cipher to
+encrypt data stored in a remote. This shared cipher is stored,
+**unencrypted** in the git repository. So it's shared among every
+clone of the git repository.
+
+	git annex initremote newremote type=... encryption=shared
+
+The advantage is you don't need to set up gpg keys. The disadvantage is
+that this is **insecure** unless you trust every clone of the git
+repository with access to the encrypted data stored in the special remote.
+
+## regular public key encryption (encryption=pubkey)
+
+This alternative simply encrypts the files in the special remotes to one or
+more public keys. The corresponding private key is needed to store
+anything in the remote, or access anything stored in it.
+It might be considered more secure due to its simplicity and since
+it's exactly the way everyone else uses gpg.
+
+	git annex initremote newremote type=.... encryption=pubkey keyid=KEYID
+
+A disadvantage is that it is not easy to later add additional public keys
+to the special remote. While the `enableremote` parameters `keyid+=` and
+`keyid-=` can be used, they have **no effect** on encrypted files that
+are already stored in the remote.
+
+One use for these parameters is to replace a revoked key:
+
+	git annex enableremote myremote keyid-=2512E3C7 keyid+=788A3F4C
+
+But even in this case, since the files are not re-encrypted, the revoked
+key has to be kept around to be able to decrypt those files.
+(Of course, if the reason for revocation is
+that the key has been compromised, it is **insecure** to leave files
+encrypted using that old key, and the user should re-encrypt everything.)
+
+(A cipher still needs to be generated (and is encrypted to the given key IDs).
+It is only used for HMAC encryption of filenames.)
+
+## regular public key encryption with shared filename encryption (encryption=sharedpubkey)
+
+This is a variation on encryption=pubkey which lets anyone who
+has access to the gpg public keys store files in the special remote.
+But, only owners of the corresponding gpg private keys can retrieve the files
+from the special remote.
+
+	git annex initremote newremote type=... encryption=sharedpubkey keyid=KEYID
+
+This might be useful if you want to let others drop off files for you in a
+special remote, so that only you can access them.
+
+The filenames used on the special remote are encrypted using HMAC,
+which prevents the special remote from seeing the filenames. But, anyone
+who can clone the git repository can access the HMAC cipher; it's stored
+**unencrypted** in the git repository.
+
+## MAC algorithm
+
+The default MAC algorithm to be applied on the filenames is HMACSHA1. A
+stronger one, for instance HMACSHA512, can be chosen upon creation
+of the special remote with the option `mac=HMACSHA512`. The available
+MAC algorithms are HMACSHA1, HMACSHA224, HMACSHA256, HMACSHA384, and
+HMACSHA512. Note that it is not possible to change algorithm for a
+non-empty remote.
diff --git a/doc/encryption/comment_10_6416ee43ffad1c306ef71247ae71a6c5._comment b/doc/encryption/comment_10_6416ee43ffad1c306ef71247ae71a6c5._comment
new file mode 100644
index 0000000000..ae2fb88a49
--- /dev/null
+++ b/doc/encryption/comment_10_6416ee43ffad1c306ef71247ae71a6c5._comment
@@ -0,0 +1,17 @@
+[[!comment format=mdwn
+ username="joey"
+ subject="""comment 10"""
+ date="2017-05-24T17:47:46Z"
+ content="""
+@Yurt, git-annex will let you specify the gpg key id using anything that gpg
+accepts, including a keyid with a appended '!'. However, when I tried that,
+gpg seemed to still pick the master key instead of the subkey. That
+happens because git-annex runs the input through `gpg --list-public-keys`
+(in order to convert eg, email addresses to key ids)
+which always lists the master key even when given a subkey.
+
+I made a small change to git-annex to special case this '!' suffix
+behavior. Seems to work in my very limited testing.
+
+Please file bug reports about this kind of thing!
+"""]]
diff --git a/doc/encryption/comment_11_30b926fbabe9a0089de1f55f6f9a5d2d._comment b/doc/encryption/comment_11_30b926fbabe9a0089de1f55f6f9a5d2d._comment
new file mode 100644
index 0000000000..5cddcfee49
--- /dev/null
+++ b/doc/encryption/comment_11_30b926fbabe9a0089de1f55f6f9a5d2d._comment
@@ -0,0 +1,15 @@
+[[!comment format=mdwn
+ username="anarcat"
+ avatar="http://cdn.libravatar.org/avatar/4ad594c1e13211c1ad9edb81ce5110b7"
+ subject="headless configs"
+ date="2018-05-17T21:02:32Z"
+ content="""
+is there some combination of this and the gcrypt special remote that would give me the following properties:
+
+ 1. password-less operation (ie. allow uploading content without the private key)
+ 2. easy revocation and key rotation (ie. not encrypt directly with GnuPG but instead encrypt a keyfile with the public keys)
+
+It seems to me this would be technically possible, no? A mix of \"hybrid\" and \"sharedpubkey\", basically...?
+
+Hybrid works great, except I can't use it in my scenario because I am trying to automate backups and it will prompt me for the private key password. I guess the solution here is to have a special unencrypted private key for the batch job? Thanks! -- [[anarcat]
+"""]]
diff --git a/doc/encryption/comment_1_4257e3c4ae559f1c0595a903f738fd7e._comment b/doc/encryption/comment_1_4257e3c4ae559f1c0595a903f738fd7e._comment
new file mode 100644
index 0000000000..e1decc397c
--- /dev/null
+++ b/doc/encryption/comment_1_4257e3c4ae559f1c0595a903f738fd7e._comment
@@ -0,0 +1,28 @@
+[[!comment format=mdwn
+ username="https://www.google.com/accounts/o8/id?id=AItOawlZ-6dtxJY4cP7shhvV8E6YyuV0Rak8it4"
+ nickname="Giovanni"
+ subject="comment 1"
+ date="2015-03-10T22:16:09Z"
+ content="""
+I have a gcrypt special remote encrypted in hybrid mode, when I try to add a keyid using:
+
+     git annex enableremote myremote keyid+=XXXXXXXX
+
+I get this error:
+
+     enableremote myremote (encryption update) (hybrid cipher with gpg keys XXXXXXXX XXXXXXX) fatal: remote myremote already exists. 
+     git-annex: git [Params \"remote add\",Param \"myremote\",Param \"gcrypt::XXXXXXXXXXX:gcrypt-tests\"] failed
+
+this is my git-annex version info: 
+
+     git-annex version: 5.20141125 
+     build flags: Assistant Webapp Webapp-secure Pairing Testsuite S3 WebDAV Inotify DBus DesktopNotify XMPP DNS Feeds Quvi TDFA CryptoHash 
+     key/value backends: SHA256E SHA1E SHA512E SHA224E SHA384E SKEIN256E SKEIN512E SHA256 SHA1 SHA512 SHA224 SHA384 SKEIN256 SKEIN512 WORM URL 
+     remote types: git gcrypt S3 bup directory rsync web webdav tahoe glacier ddar hook external 
+     local repository version: 5 
+     supported repository version: 5 
+     upgrade supported from repository versions: 0 1 2 4
+
+am I doing something wrong? thank you Giovanni
+
+"""]]
diff --git a/doc/encryption/comment_2_5c2da865082de475254ebfd53feb2d0a._comment b/doc/encryption/comment_2_5c2da865082de475254ebfd53feb2d0a._comment
new file mode 100644
index 0000000000..c8a571813b
--- /dev/null
+++ b/doc/encryption/comment_2_5c2da865082de475254ebfd53feb2d0a._comment
@@ -0,0 +1,13 @@
+[[!comment format=mdwn
+ username="joey"
+ subject="""comment 2"""
+ date="2015-07-06T19:57:12Z"
+ content="""
+@Giovanni, git is complaining that there is already a remote named
+"myremote" enabled in the current repository. Perhaps you have reused this
+name for a different remote.
+
+(This seems to have nothing to do with the page the comment was posted to, which 
+is a bit annoying. Please post questions in the forum and not attached to
+random other pages.)
+"""]]
diff --git a/doc/encryption/comment_3_46e64e4856975706e06e2a012a5d8f67._comment b/doc/encryption/comment_3_46e64e4856975706e06e2a012a5d8f67._comment
new file mode 100644
index 0000000000..eba9a883cd
--- /dev/null
+++ b/doc/encryption/comment_3_46e64e4856975706e06e2a012a5d8f67._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="whlabratz@d10941ab2681deb87122fd8f6da51f3dcdb8dbe5"
+ nickname="whlabratz"
+ subject="Listing GPG keys enabled for a remote with hybrid encryption"
+ date="2015-09-15T22:37:35Z"
+ content="""
+Is there a straight-forward way to list which GPG keys are enabled for a particular remote?
+"""]]
diff --git a/doc/encryption/comment_4_fe8b181adef9a39a039ff96a0d587188._comment b/doc/encryption/comment_4_fe8b181adef9a39a039ff96a0d587188._comment
new file mode 100644
index 0000000000..4b8014f00d
--- /dev/null
+++ b/doc/encryption/comment_4_fe8b181adef9a39a039ff96a0d587188._comment
@@ -0,0 +1,11 @@
+[[!comment format=mdwn
+ username="joey"
+ subject="""listing gpg keys"""
+ date="2015-09-17T17:01:38Z"
+ content="""
+Run "git annex info specialremote" and it will describe the encryption
+settings of the remote, including gpg keys where applicable.
+
+Needs a fairly recent git-annex. `git show git-annex:remote.log` can
+also be used.
+"""]]
diff --git a/doc/encryption/comment_5_5c9897663aaa83ca39a7e8cb292a3fd1._comment b/doc/encryption/comment_5_5c9897663aaa83ca39a7e8cb292a3fd1._comment
new file mode 100644
index 0000000000..ee4c2f3287
--- /dev/null
+++ b/doc/encryption/comment_5_5c9897663aaa83ca39a7e8cb292a3fd1._comment
@@ -0,0 +1,10 @@
+[[!comment format=mdwn
+ username="gavinwahl@d3a94ffbd6df3b8833a53ff11ec435f58f7c44a9"
+ nickname="gavinwahl"
+ subject="Shared key - how many keys?"
+ date="2016-04-03T03:43:58Z"
+ content="""
+In shared mode, is a single key used to encrypt every file in the repository? Or is a new key created for each file?
+
+Shared mode has the properties I need - getting access to the git repo should give you access to all the content. BUT, if one loses access to updates to the git repo, they should not have access to files added after they lost access.
+"""]]
diff --git a/doc/encryption/comment_6_1756ce62906586f876a3491e5d9befde._comment b/doc/encryption/comment_6_1756ce62906586f876a3491e5d9befde._comment
new file mode 100644
index 0000000000..81802c6254
--- /dev/null
+++ b/doc/encryption/comment_6_1756ce62906586f876a3491e5d9befde._comment
@@ -0,0 +1,13 @@
+[[!comment format=mdwn
+ username="joey"
+ subject="""comment 6"""
+ date="2016-04-04T18:48:34Z"
+ content="""
+@gavinwahl, it's a single shared key that any clone of the repository
+provides access to.
+
+If you use the [[tahoe]] special remote, storing files in tahoe-lafs does result
+in a new capability (a kind of key) being stored in the git repo.
+So someone with an old clone can't access the files from tahoe-lafs.
+Tahoe is unique in providing that ability.
+"""]]
diff --git a/doc/encryption/comment_8_3849eb24c3682644e263bae107747dee._comment b/doc/encryption/comment_8_3849eb24c3682644e263bae107747dee._comment
new file mode 100644
index 0000000000..393d18ecc5
--- /dev/null
+++ b/doc/encryption/comment_8_3849eb24c3682644e263bae107747dee._comment
@@ -0,0 +1,20 @@
+[[!comment format=mdwn
+ username="joern.mankiewicz@06fb5bc9b732f143dee3606866362f562531310d"
+ nickname="joern.mankiewicz"
+ avatar="http://cdn.libravatar.org/avatar/446365f4d50dddc42965fa0432e70cdb"
+ subject="Workflow for adding a keyid in hybrid-enc lateron and re-encrypting? "
+ date="2017-03-21T22:08:30Z"
+ content="""
+Hi folks!
+
+We are considering introducing git-annex with gcrypt in hybrid mode as secure storage for common data in our company and I'd rather not delete and reinit the repo everytime when somebody new is granted access.
+A little testing with current git-annex showed, that GCRYPT_FULL_REPACK with a forced git-push of all branches makes the git-repo accessible (I get the files) to the newcomer but not the annexed data (gpg error \"No secret key\" in git annex get, git annex info secretRepo just lists my first key).
+
+Has anybody sucessfully tested adding keyids in hybrid-encryption later on? Which further steps where needed to make it work? 
+
+Thanks for any input! :)
+
+Cheers 
+
+Jörn
+"""]]
diff --git a/doc/encryption/comment_8_66e81e89fd483ca95620522b0f63c4fd._comment b/doc/encryption/comment_8_66e81e89fd483ca95620522b0f63c4fd._comment
new file mode 100644
index 0000000000..bca3286d07
--- /dev/null
+++ b/doc/encryption/comment_8_66e81e89fd483ca95620522b0f63c4fd._comment
@@ -0,0 +1,11 @@
+[[!comment format=mdwn
+ username="joey"
+ subject="""comment 8"""
+ date="2017-04-07T16:52:59Z"
+ content="""
+@joern.mankiewicz, I see you found a bug and filed it, so will answer in
+the but report.
+
+Barring bugs, adding another gpg key to a hybrid encryption special remote
+is as simple as `git annex enableremote $theremote keyid+=$newkey`
+"""]]
diff --git a/doc/encryption/comment_9_5ca10891d642392aaff342c1478b0550._comment b/doc/encryption/comment_9_5ca10891d642392aaff342c1478b0550._comment
new file mode 100644
index 0000000000..723e3df2d4
--- /dev/null
+++ b/doc/encryption/comment_9_5ca10891d642392aaff342c1478b0550._comment
@@ -0,0 +1,10 @@
+[[!comment format=mdwn
+ username="Yurt"
+ avatar="http://cdn.libravatar.org/avatar/28d2177f7678c6cc0b677c34927eba06"
+ subject="Encrypt to different subkeys?"
+ date="2017-05-16T20:25:01Z"
+ content="""
+Is it possible to encrypt with subkeys. I have a few subkeys distributed to different computers and I'd like to be able to sync to a special remote with all of them. Right now, if the master key is stripped, I get an error from gpg.
+
+I do this exact thing with password-store. Appending \"!\" to the subkey id should force gpg to use that specific key: [https://lists.zx2c4.com/pipermail/password-store/2014-September/001131.html](https://lists.zx2c4.com/pipermail/password-store/2014-September/001131.html).
+"""]]
diff --git a/doc/favicon.ico b/doc/favicon.ico
new file mode 100644
index 0000000000..e754f5a48b
Binary files /dev/null and b/doc/favicon.ico differ
diff --git a/doc/footer/column_a.mdwn b/doc/footer/column_a.mdwn
new file mode 100644
index 0000000000..92c77d8406
--- /dev/null
+++ b/doc/footer/column_a.mdwn
@@ -0,0 +1,7 @@
+### Recent [[news]]
+
+[[!inline pages="news/* and !*/Discussion" archive=yes show=2 feeds=no]]
+
+### [[devblog|devblog]]
+
+[[!inline pages="devblog/* and !*/Discussion" archive=yes show=5 feeds=no]]
diff --git a/doc/footer/column_b.mdwn b/doc/footer/column_b.mdwn
new file mode 100644
index 0000000000..4d2d3dd983
--- /dev/null
+++ b/doc/footer/column_b.mdwn
@@ -0,0 +1,7 @@
+### Recent [[videos]]
+
+[[!inline pages="videos/* and !*/Discussion" archive=yes show=2 feeds=no]]
+
+### Recent [[forum posts|forum]]
+
+[[!inline pages="forum/* and !*/Discussion" archive=yes show=5 feeds=no]]
diff --git a/doc/forum.mdwn b/doc/forum.mdwn
new file mode 100644
index 0000000000..c6759aed07
--- /dev/null
+++ b/doc/forum.mdwn
@@ -0,0 +1,9 @@
+This is a place to discuss using git-annex.
+If you need help, advice, or anything, post about it here.
+
+But, please don't post bug reports here. Put them in [[bugs]].  
+And please don't make wishlist requests here. Put them in [[todo]].  
+(If you post bugs/todo here, it'll just get moved.)
+
+[[!inline pages="forum/* and !*/Discussion" archive=yes feedlimit=10 
+rootpage=forum postformtext="Add a new thread titled:"]]
diff --git a/doc/forum/2_clients_using_an_encrypted_server_for_syncing_-_possible__63__.mdwn b/doc/forum/2_clients_using_an_encrypted_server_for_syncing_-_possible__63__.mdwn
new file mode 100644
index 0000000000..84f92ae7ea
--- /dev/null
+++ b/doc/forum/2_clients_using_an_encrypted_server_for_syncing_-_possible__63__.mdwn
@@ -0,0 +1,3 @@
+I have 2 clients (laptops) that  would (usually) not be online at the same time. Is it possible to use git-annex for syncing these 2 clients through an encrypted server?
+
+I have tried to set this up with the assistant by using an USB-disk for copying the repository from one client to the other. The server is configured through the assistant as repository group 'transfer'. Now both clients see the encrypted ssh-server, each client copy stuff to the encrypted server, but no files are copied between the 2 clients. (In my case, running git-annex on the server is not an option. I have read through this forum, but it doesn't seem anyone has been able to get it to work.)
diff --git a/doc/forum/2_clients_using_an_encrypted_server_for_syncing_-_possible__63__/comment_1_924521ad5972046bac44d2e04ec296c7._comment b/doc/forum/2_clients_using_an_encrypted_server_for_syncing_-_possible__63__/comment_1_924521ad5972046bac44d2e04ec296c7._comment
new file mode 100644
index 0000000000..90b47a3a9a
--- /dev/null
+++ b/doc/forum/2_clients_using_an_encrypted_server_for_syncing_-_possible__63__/comment_1_924521ad5972046bac44d2e04ec296c7._comment
@@ -0,0 +1,10 @@
+[[!comment format=mdwn
+ username="https://www.google.com/accounts/o8/id?id=AItOawnvVfFLW4CTKs7UjdiLIsOn_cxj1Jnh64I"
+ nickname="Charl"
+ subject="Change the encrypted server repository group to "full backup""
+ date="2014-03-23T21:00:01Z"
+ content="""
+Have you tried changing the repository group of the encrypted server to \"full backup\" instead of \"transfer\"?
+
+I've just started experimenting with git-annex, and this setup (two possibly remote laptops, one encrypted server) seems to be working after very limited testing. I'm currently importing a larger collection of about 40000 files, will see if it still does (it seems whilst one client is transferring to the remote server, the other client laptop is not beeing synced; it did sync after my first batch of copying however.)
+"""]]
diff --git a/doc/forum/2_clients_using_an_encrypted_server_for_syncing_-_possible__63__/comment_2_e2a7f34a3ccc1b6467e6da611c067d66._comment b/doc/forum/2_clients_using_an_encrypted_server_for_syncing_-_possible__63__/comment_2_e2a7f34a3ccc1b6467e6da611c067d66._comment
new file mode 100644
index 0000000000..b50ba4b469
--- /dev/null
+++ b/doc/forum/2_clients_using_an_encrypted_server_for_syncing_-_possible__63__/comment_2_e2a7f34a3ccc1b6467e6da611c067d66._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="https://www.google.com/accounts/o8/id?id=AItOawkvN0eFnwvgw2JyTSSHw0QouytcxtPLln8"
+ nickname="Stein Roald"
+ subject="No success"
+ date="2014-03-26T19:43:24Z"
+ content="""
+Thank you, Charl, for your suggestion. I've tried it, without success. (It doesn't seem like that change changed anything on the encrypted ssh-server.)
+"""]]
diff --git a/doc/forum/2_clients_using_an_encrypted_server_for_syncing_-_possible__63__/comment_3_f9a369a6ac69f091e6128990274d3228._comment b/doc/forum/2_clients_using_an_encrypted_server_for_syncing_-_possible__63__/comment_3_f9a369a6ac69f091e6128990274d3228._comment
new file mode 100644
index 0000000000..1e42061447
--- /dev/null
+++ b/doc/forum/2_clients_using_an_encrypted_server_for_syncing_-_possible__63__/comment_3_f9a369a6ac69f091e6128990274d3228._comment
@@ -0,0 +1,10 @@
+[[!comment format=mdwn
+ username="http://joeyh.name/"
+ ip="209.250.56.41"
+ subject="comment 3"
+ date="2014-03-26T21:10:39Z"
+ content="""
+What kind of encrypted remote are you using? An encrypted rsync special remote does not include the git repository, but only the content of the files, so cannot be used in this way.
+
+If you set up a [[gcrypt special remote|special_remotes/gcrypt]], it will be encrypted and includes the full git repository, as well as the content of the files, so can be used like this.
+"""]]
diff --git a/doc/forum/2_clients_using_an_encrypted_server_for_syncing_-_possible__63__/comment_4_91b422f8d55b68077245c606c4f7f87c._comment b/doc/forum/2_clients_using_an_encrypted_server_for_syncing_-_possible__63__/comment_4_91b422f8d55b68077245c606c4f7f87c._comment
new file mode 100644
index 0000000000..4d8915b8df
--- /dev/null
+++ b/doc/forum/2_clients_using_an_encrypted_server_for_syncing_-_possible__63__/comment_4_91b422f8d55b68077245c606c4f7f87c._comment
@@ -0,0 +1,28 @@
+[[!comment format=mdwn
+ username="https://www.google.com/accounts/o8/id?id=AItOawkvN0eFnwvgw2JyTSSHw0QouytcxtPLln8"
+ nickname="Stein Roald"
+ subject="Need some clarification"
+ date="2014-03-28T22:49:14Z"
+ content="""
+OK, thank you Joey for your help, and I think you have made some really excellent software. I've spent a couple of days to configure gpg and understand how to use it (glad I did, it has been on my todo-list for a long time...)
+
+First to your question: When I started, I just used the git-annex assistant to \"set up a repository on a remote server using ssh\" as a \"transfer repository\". Now I'll use git-remote-gcrypt instead.
+
+Now I have 2 questions:
+
+---
+
+1) On this webpage: [[https://github.com/joeyh/git-remote-gcrypt]] these are the instructions for setting up a remote for two participants:
+
+    git remote add cryptremote gcrypt::rsync://example.com:repo
+    git config remote.cryptremote.gcrypt-participants \"KEY1 KEY2\"
+    git push cryptremote master
+
+**Question 1:** As I own both computers myself, will syncing between the 2 computers work if I only use 1 KEY (the same KEY on both computers)?
+
+---
+
+2) I am still struggling to understand git-annex assistant and the use of special remotes. Important reasons for using git-annex are backups and moving data to the cloud to free up disk space locally. If I've understood it correctly, git-annex assistant always encrypts data sent to special remotes. And it seems to me that this page [[http://git-annex.branchable.com/special_remotes/]] suggest that git-annex assistant makes it easy to transfer files between different computers that do not communicate directly. I must clearly have misunderstood something, as Joey's comment above seem to state that I can't use the assistant for setting up syncing between computers.
+
+**Question 2:** If I loose my computer, or it becomes corrupted, how can I get my files back if I don't set up special remotes as a gcrypt special remote? (And if I can't, what is the purpose of the special remotes made by the assistant?)
+"""]]
diff --git a/doc/forum/2_clients_using_an_encrypted_server_for_syncing_-_possible__63__/comment_5_f6128fe75ff3453747f69f12e0fd0a5b._comment b/doc/forum/2_clients_using_an_encrypted_server_for_syncing_-_possible__63__/comment_5_f6128fe75ff3453747f69f12e0fd0a5b._comment
new file mode 100644
index 0000000000..8dcddbeef5
--- /dev/null
+++ b/doc/forum/2_clients_using_an_encrypted_server_for_syncing_-_possible__63__/comment_5_f6128fe75ff3453747f69f12e0fd0a5b._comment
@@ -0,0 +1,12 @@
+[[!comment format=mdwn
+ username="https://www.google.com/accounts/o8/id?id=AItOawmNu4V5fvpLlBhaCUfXXOB0MI5NXwh8SkU"
+ nickname="Adam"
+ subject="Confused"
+ date="2014-03-31T03:18:41Z"
+ content="""
+>What kind of encrypted remote are you using? An encrypted rsync special remote does not include the git repository, but only the content of the files, so cannot be used in this way.
+
+Forgive me, Joey, I've been following your work on the assistant for almost as long as you've been working on it, and I use git a little bit, but I have yet to wrap my head around the complexities of git and how git-annex and the assistant...morph them.  I've read all the design docs you've written, but I still don't understand this.  From reading the OP it sounds like this is exactly what the encrypted rsync special remote is for: using a transfer repo to sync two devices that don't connect directly to each other.  Why would he need to use a gcrypt repo instead?
+
+Thanks for your patience.  I know you get asked these basic questions over and over again.
+"""]]
diff --git a/doc/forum/2_clients_using_an_encrypted_server_for_syncing_-_possible__63__/comment_6_9b90b4031a5ed26c375903b33ed65a10._comment b/doc/forum/2_clients_using_an_encrypted_server_for_syncing_-_possible__63__/comment_6_9b90b4031a5ed26c375903b33ed65a10._comment
new file mode 100644
index 0000000000..5fc4286b27
--- /dev/null
+++ b/doc/forum/2_clients_using_an_encrypted_server_for_syncing_-_possible__63__/comment_6_9b90b4031a5ed26c375903b33ed65a10._comment
@@ -0,0 +1,17 @@
+[[!comment format=mdwn
+ username="http://joeyh.name/"
+ ip="209.250.56.244"
+ subject="comment 6"
+ date="2014-04-02T20:11:35Z"
+ content="""
+@Adam, an encrypted special remote does not contain the git repository (the gcrypt special remote is an exception to this rule). So you need to use another method to sync the git repository between machines.
+
+@Stein Roald once you have a clone of the git repository, you have the information that git-annex needs in able to get files from encrypted special remotes.
+
+Question 1: While gcrypt supports multi-key setups, when you use the git-annex assistant to set up a gcrypt remote it only sets it up to use one key. It's left to you to arrange for this key to be on every computer that needs it. Or use command-line stuff to add additional keys later.
+
+Question 2: You can certianly use the assistant to set up syncing between computers, but the only setup that currently provides 100% end-to-end enctyption of the git repository is using gcrypt. (Special remotes are 100% end-to-end encrypted, but as I've stated several times, do not contain the git repository data.)
+Since you seemed to want 100% end to end encryption I suggested using gcrypt. There are simpler setups, like using XMPP, that encrypt everything but not end-to-end, so the XMPP server could snoop on it.
+
+If you loose your computer, you can get your files back from any other device where you've set up a clone of that repository. A backup drive, another computer , etc. 
+"""]]
diff --git a/doc/forum/2_clients_using_an_encrypted_server_for_syncing_-_possible__63__/comment_7_acd64ce1b08a97ddf730622272e9f611._comment b/doc/forum/2_clients_using_an_encrypted_server_for_syncing_-_possible__63__/comment_7_acd64ce1b08a97ddf730622272e9f611._comment
new file mode 100644
index 0000000000..f850253b18
--- /dev/null
+++ b/doc/forum/2_clients_using_an_encrypted_server_for_syncing_-_possible__63__/comment_7_acd64ce1b08a97ddf730622272e9f611._comment
@@ -0,0 +1,14 @@
+[[!comment format=mdwn
+ username="https://www.google.com/accounts/o8/id?id=AItOawkvN0eFnwvgw2JyTSSHw0QouytcxtPLln8"
+ nickname="Stein Roald"
+ subject="comment 7"
+ date="2014-04-03T21:55:14Z"
+ content="""
+Thanks again, Joey, for your time and for your kind answers.
+
+I have tried to set up git-annex with 2 laptops, each with their own gpg-key in order to use a remote server with gcrypt (and those laptops have shared the keys with each other). I regularly use git, but so far I haven't been able to get this setup to work. But before I bother the world with these problems, I would be interested in learning how to do the following:
+
+*Question 3*: How can I use the git-annex assistant to set up a gcrypt remote (with one key)? It doesn't show up as an option as far as I can see (git-annex version: 4.20130815).
+
+*Promise*: When I get git-annex to work with gcrypt, I'll make a summary on how I did it (and maybe which mistakes I made so others can avoid them).
+"""]]
diff --git a/doc/forum/2_clients_using_an_encrypted_server_for_syncing_-_possible__63__/comment_8_9baacb14fc5eb449cb13e0b4a4995fb0._comment b/doc/forum/2_clients_using_an_encrypted_server_for_syncing_-_possible__63__/comment_8_9baacb14fc5eb449cb13e0b4a4995fb0._comment
new file mode 100644
index 0000000000..0ecc2fb3ba
--- /dev/null
+++ b/doc/forum/2_clients_using_an_encrypted_server_for_syncing_-_possible__63__/comment_8_9baacb14fc5eb449cb13e0b4a4995fb0._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="http://joeyh.name/"
+ ip="216.145.95.162"
+ subject="comment 8"
+ date="2014-05-19T16:48:09Z"
+ content="""
+4.20130815 is too old. Get a current version.
+"""]]
diff --git a/doc/forum/4hr+_sync_on_new_remote___40__USB_drive__41__.mdwn b/doc/forum/4hr+_sync_on_new_remote___40__USB_drive__41__.mdwn
new file mode 100644
index 0000000000..167c090f18
--- /dev/null
+++ b/doc/forum/4hr+_sync_on_new_remote___40__USB_drive__41__.mdwn
@@ -0,0 +1,92 @@
+I need some help understanding what would cause ``git-annex sync`` to still be running 4hrs+ on a new remote (FAT32 USB drive - BTEST) ? From all my searching, it appears to be part of git's optimization routines. But to be this long and slow seems odd. Also I am not sure what there would be optimize since it's a brand new remote ? The src (WTEST) doesn't seem to show any need to run ``git gc``.
+
+I followed the walkthrough's setup of a remote and followed that up with ``git-annex sync``. I did not specify the source for the sync as there is only one other (WTEST). The content is mostly ISO files and Win32 executables in 7 separate commits. In between ``git-annex import`` and ``git commit``, the working directory was removed with ``git rm '*'`` but no ``git-annex drop``. There were lots of duplicate files and it truly was satisfying to not see the disk usage increase. My backend is also MD5E for purposes of quicker imports and ease of hash lookup from other sources.
+
+I would like to have 20-30 USB remotes. The length of time to get one remote up and running at this point is horrendous. Other people seem to have better experiences with it. Only odd thing that I see with my configuration is that the remote FAT32 drive is in ``indirect`` mode. My understanding is that ``git-annex`` would automatically switch to ``direct`` if it detected a filesystem that did not support symbolic links. 
+
+What is wrong with my setup ? How can I fix it ?
+
+Here's the basic config of WTEST.
+
+[[!format sh """
+
+WTEST$ git-annex info
+
+repository mode: indirect
+trusted repositories: 0
+semitrusted repositories: 4
+        00000000-0000-0000-0000-000000000001 -- web
+        00000000-0000-0000-0000-000000000002 -- bittorrent
+        98dfasdf-ab83-4a0e-8b73-4dfasffdsaff1ae -- WTEST [here]
+        9dfdsfdf8-c5d5-4761-ab67-ffsadfsadfsa83 -- BTEST
+untrusted repositories: 0
+transfers in progress: none
+available local disk space: 488.16 gigabytes (+1 megabyte reserved)
+local annex keys: 57327
+local annex size: 58.57 gigabytes
+annexed files in working tree: 4322
+size of annexed files in working tree: 6.82 gigabytes
+bloom filter size: 16 mebibytes (11.5% full)
+backend usage:
+        MD5E: 61649
+"""]]
+
+[[!format sh """
+WTEST$git object-count -v
+
+count: 521829
+size: 66794240
+in-pack: 0
+packs: 0
+size-pack: 0
+prune-packable: 0
+garbage: 0
+size-garbage: 0
+"""]]
+
+
+[[!format sh """
+WTEST$git config -l
+
+core.repositoryformatversion=0
+core.filemode=false
+core.bare=false
+core.logallrefupdates=true
+core.ignorecase=true
+core.precomposeunicode=true
+annex.uuid=98dfasdf-ab83-4a0e-8b73-4dfasffdsaff1ae
+annex.sshcaching=false
+annex.version=5
+annex.backends=MD5E
+annex.queuesize=102400
+annex.genmetadata=true
+remote.BTEST.url=/Volumes/BTEST
+remote.BTEST.fetch=+refs/heads/*:refs/remotes/BTEST/*
+remote.BTEST.annex-uuid=9dfdsfdf8-c5d5-4761-ab67-ffsadfsadfsa83
+"""]]
+
+**The long running sync.**
+
+[[!format sh """ 
+BTEST$ git-annex sync
+
+commit  ok
+
+pull origin
+
+Auto packing the repository for optimum performance. You may also
+
+run "git gc" manually. See "git help gc" for more information.
+
+Counting objects: 521834, done.
+
+Delta compression using up to 4 threads.
+
+Compressing objects:  12% (59687/462488)
+"""]]
+
+**Update 7hrs later:**
+
+[[!format sh """
+Writing objects:  70% (366184/521834)
+"""]]
diff --git a/doc/forum/4hr+_sync_on_new_remote___40__USB_drive__41__/comment_1_01a6020d25826a3e0afb087899f6919c._comment b/doc/forum/4hr+_sync_on_new_remote___40__USB_drive__41__/comment_1_01a6020d25826a3e0afb087899f6919c._comment
new file mode 100644
index 0000000000..d7209a89b8
--- /dev/null
+++ b/doc/forum/4hr+_sync_on_new_remote___40__USB_drive__41__/comment_1_01a6020d25826a3e0afb087899f6919c._comment
@@ -0,0 +1,14 @@
+[[!comment format=mdwn
+ username="joey"
+ subject="""comment 1"""
+ date="2015-03-16T17:38:02Z"
+ content="""
+git gc is using the time here.
+
+It's easy to configure git to not do auto-gc. See the git-config man
+page's documentation of gc.auto.
+
+(It's impossible to use git-annex in indirect mode on a fat filesystem,
+unless FAT has suddenly grown symlink support overnight and I didn't
+notice..)
+"""]]
diff --git a/doc/forum/ARM_build_on_Zyxel_NAS.mdwn b/doc/forum/ARM_build_on_Zyxel_NAS.mdwn
new file mode 100644
index 0000000000..4fe46bedbc
--- /dev/null
+++ b/doc/forum/ARM_build_on_Zyxel_NAS.mdwn
@@ -0,0 +1,15 @@
+I am trying to run the linux standalone ARM build on my Zyxel NAS, but I get the following error
+
+`FATAL: kernel too old`
+
+The system runs the following:
+
+`
+uname -a
+`
+
+`
+Linux nas 2.6.31.8 #2 Thu Dec 19 14:31:05 CST 2013 armv5tel GNU/Linux
+`
+
+Help would be much appreciated.  
diff --git a/doc/forum/ARM_build_on_Zyxel_NAS/comment_1_38f38755c0afd76a2b968836fec395e8._comment b/doc/forum/ARM_build_on_Zyxel_NAS/comment_1_38f38755c0afd76a2b968836fec395e8._comment
new file mode 100644
index 0000000000..02d2716ac7
--- /dev/null
+++ b/doc/forum/ARM_build_on_Zyxel_NAS/comment_1_38f38755c0afd76a2b968836fec395e8._comment
@@ -0,0 +1,11 @@
+[[!comment format=mdwn
+ username="Joey"
+ subject="""comment 1"""
+ date="2014-10-20T15:11:49Z"
+ content="""
+The git-annex arm build is built using the libc from Debian stable,
+which needs a newer version of the Linux kerenl than is on your device.
+
+It would be possible to build git-annex against an older libc, but
+not easily, which is why I don't.
+"""]]
diff --git a/doc/forum/ARM_build_on_Zyxel_NAS/comment_2_44c8f1af0cbe9ad51794e6d8d16be627._comment b/doc/forum/ARM_build_on_Zyxel_NAS/comment_2_44c8f1af0cbe9ad51794e6d8d16be627._comment
new file mode 100644
index 0000000000..9cec2bf69b
--- /dev/null
+++ b/doc/forum/ARM_build_on_Zyxel_NAS/comment_2_44c8f1af0cbe9ad51794e6d8d16be627._comment
@@ -0,0 +1,9 @@
+[[!comment format=mdwn
+ username="musella"
+ ip="84.73.42.152"
+ subject="comment 2"
+ date="2014-10-21T23:35:35Z"
+ content="""
+what is the minimal kernel version that I would need?
+
+"""]]
diff --git a/doc/forum/ARM_build_on_Zyxel_NAS/comment_3_b4f6e5ac672e8ece36cceb74ff3315dd._comment b/doc/forum/ARM_build_on_Zyxel_NAS/comment_3_b4f6e5ac672e8ece36cceb74ff3315dd._comment
new file mode 100644
index 0000000000..2ff40efe65
--- /dev/null
+++ b/doc/forum/ARM_build_on_Zyxel_NAS/comment_3_b4f6e5ac672e8ece36cceb74ff3315dd._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="http://joeyh.name/"
+ ip="209.250.56.96"
+ subject="comment 3"
+ date="2014-10-22T16:22:24Z"
+ content="""
+I know kernel 3.2 would work. I don't know what the minimum kernel supported by glibc 2.13 is.
+"""]]
diff --git a/doc/forum/A_git-annex_helper.mdwn b/doc/forum/A_git-annex_helper.mdwn
new file mode 100644
index 0000000000..23a568eeb2
--- /dev/null
+++ b/doc/forum/A_git-annex_helper.mdwn
@@ -0,0 +1,6 @@
+Hi,
+I have developed a [sort-of-a-GUI](http://github.com/atrent/AdMinchiam/tree/master/ga-gui) for git-annex and I'd like you to test it if you have time, see README and [screenshots](http://github.com/atrent/AdMinchiam/blob/master/ga-gui/Screenshots/GitAnnexGUI_020.png), the idea is very simple: visually tag items then generate a script based on editable templates.
+
+The only problem at the moment is the very slow git-annex data acquisition; not my fault, on very large annexes, in terms of number of files, the 'git-annex list' and 'git-annex metadata' commands take AGES...
+
+Thank you
diff --git a/doc/forum/A_little_help_with_headless_server_configuration.mdwn b/doc/forum/A_little_help_with_headless_server_configuration.mdwn
new file mode 100644
index 0000000000..3dc52be306
--- /dev/null
+++ b/doc/forum/A_little_help_with_headless_server_configuration.mdwn
@@ -0,0 +1,2 @@
+Attempting to setup a multiple server file share. I am really not sure how to configure and setup the headless system for sharing? Seems easy enough with the webapp and have my windows system appear to be working not sure how to get my Linux headless systems to sync up as well any help would be greatly appreciated.
+
diff --git a/doc/forum/A_little_help_with_headless_server_configuration/comment_1_d32e28e8dcacf89eb357b78aefded274._comment b/doc/forum/A_little_help_with_headless_server_configuration/comment_1_d32e28e8dcacf89eb357b78aefded274._comment
new file mode 100644
index 0000000000..9413b10cb8
--- /dev/null
+++ b/doc/forum/A_little_help_with_headless_server_configuration/comment_1_d32e28e8dcacf89eb357b78aefded274._comment
@@ -0,0 +1,22 @@
+[[!comment format=mdwn
+ username="joey"
+ subject="""comment 1"""
+ date="2015-11-18T18:03:30Z"
+ content="""
+It's not really clear from that what you want this server 
+(or servers?) to do.
+
+I think if you explain some more we can help better.
+
+I guess you have a repository on Windows using the webapp and you want
+that repository to be synced to one or more servers. The webapp has a
+dialog to Add a Repository on a Remote server. You'll need to install
+git-annex on the server, and tell the webapp the password to use to ssh to
+it, and otherwise that's pretty simple.
+
+That will get you a git repository on the server that will store all the files
+from your Windows repository. Since it's a bare git repository by default,
+all you can do with it is clone it to somewhere else, which can then get
+files from it. That could be another server or another webapp client or
+whatever, depending on what you're trying to accomplish.
+"""]]
diff --git a/doc/forum/A_question_an_the_nomad_use_cases__58___files_to_fetch__44___files_to_delete__44___files_to_keep__63__.mdwn b/doc/forum/A_question_an_the_nomad_use_cases__58___files_to_fetch__44___files_to_delete__44___files_to_keep__63__.mdwn
new file mode 100644
index 0000000000..c6c67a2917
--- /dev/null
+++ b/doc/forum/A_question_an_the_nomad_use_cases__58___files_to_fetch__44___files_to_delete__44___files_to_keep__63__.mdwn
@@ -0,0 +1,12 @@
+Hello,
+
+I went through several of the questions on the forum but could not find the answer to this. I have the following scenario, which seems very similar to the nomad case.
+
+I have a server at home with many big media files. I want to keep a subset of these files on my laptop, to watch when I travel. When I'm done watching the files, I want to either delete them (i.e, they should no longer be on any machine) or archive them (i.e., they should remain on the server, but no longer be on the laptop). I think I know how to do this from the command line, but I still have a few questions.
+
+- How do I drop a file from _every_ repository? If I understood correctly, `git annex drop foo` will only drop foo locally.
+- Is it possible to follow this use case using the assistant, or should I use the command line instead (which would be very fine with me)? In particular, how do I choose the file I want to put on the laptop? (It seems `git annex get` is the command line way, is there an assistant equivalent?)
+- Can I use both the assistant and the command line at the same time? Is there a description of the command-line version of what the assistant is doing behind the scenes?
+- Can the server repository be both a "transfer" repository (to hold the files I haven't watch) and an "archive" repository (to keep the files I've watched and decided to keep), or do I need two repositories (on the server) for that?
+
+Thanks a lot for any suggestion.
diff --git a/doc/forum/A_question_an_the_nomad_use_cases__58___files_to_fetch__44___files_to_delete__44___files_to_keep__63__/comment_1_fe291cd6cd8e2d5b8e23f8e3689d824b._comment b/doc/forum/A_question_an_the_nomad_use_cases__58___files_to_fetch__44___files_to_delete__44___files_to_keep__63__/comment_1_fe291cd6cd8e2d5b8e23f8e3689d824b._comment
new file mode 100644
index 0000000000..e29f576577
--- /dev/null
+++ b/doc/forum/A_question_an_the_nomad_use_cases__58___files_to_fetch__44___files_to_delete__44___files_to_keep__63__/comment_1_fe291cd6cd8e2d5b8e23f8e3689d824b._comment
@@ -0,0 +1,15 @@
+[[!comment format=mdwn
+ username="http://joeyh.name/"
+ ip="64.134.31.139"
+ subject="comment 1"
+ date="2013-10-21T22:41:44Z"
+ content="""
+You can use `git annex drop foo --from remote` to drop the file from the remote. Of course, the remote has to be accessible. You can't drop files from an offline drive that's stored in a safe.. 
+
+What you might be looking for is `git annex drop foo; rm foo`, followed by a git commit, and then later you can run `git annex unused` on a remote and it will let you then easily drop the files you've removed from the repository. This approach of just deleting files when you're done with them also works when using the assistant.
+
+The assistant isn't really about only keeping a subset of files on your laptop. It would like to get them all, except for ones in \"archive\" directories.
+You can configure the assistant to use manual mode, and then it doesn't download any files on its own (so you have to manually `git annex get` them), but it will still handle all the other stuff the assistant does, like automatically committing and syncing changes.
+
+An archive repository wants one copy of every file that is not already stored in some other archive repository. So an archive repository could certainly be used instead of a transfer repository. (But not a small archive repository; those only want files that are moved to \"archive\" directories.) A better choice might be to make that server be a backup repository. That makes it want *every* file, no matter what, and it follows that it would archive everything, and have everything the client wants available for transferring to it.
+"""]]
diff --git a/doc/forum/A_question_an_the_nomad_use_cases__58___files_to_fetch__44___files_to_delete__44___files_to_keep__63__/comment_2_f0dbc3c723999bf0f22502e3a89d1d4a._comment b/doc/forum/A_question_an_the_nomad_use_cases__58___files_to_fetch__44___files_to_delete__44___files_to_keep__63__/comment_2_f0dbc3c723999bf0f22502e3a89d1d4a._comment
new file mode 100644
index 0000000000..3ae6b94c5c
--- /dev/null
+++ b/doc/forum/A_question_an_the_nomad_use_cases__58___files_to_fetch__44___files_to_delete__44___files_to_keep__63__/comment_2_f0dbc3c723999bf0f22502e3a89d1d4a._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="http://alan.petitepomme.net/"
+ nickname="Alan Schmitt"
+ subject="comment 2"
+ date="2013-10-23T10:06:53Z"
+ content="""
+Thank you for the suggestions and explanations. I'm using a manual repo and I find that it works quite well.
+"""]]
diff --git a/doc/forum/A_really_stupid_question.mdwn b/doc/forum/A_really_stupid_question.mdwn
new file mode 100644
index 0000000000..38c7bcb56a
--- /dev/null
+++ b/doc/forum/A_really_stupid_question.mdwn
@@ -0,0 +1,3 @@
+Sorry, but all this wiki and the manpage seem to gloss over the most obvious question:
+
+What happens when you commit conflicting edits in different repositories?
diff --git a/doc/forum/A_really_stupid_question/comment_1_40e02556de0b00b94f245a0196b5a89f._comment b/doc/forum/A_really_stupid_question/comment_1_40e02556de0b00b94f245a0196b5a89f._comment
new file mode 100644
index 0000000000..2a400db3b0
--- /dev/null
+++ b/doc/forum/A_really_stupid_question/comment_1_40e02556de0b00b94f245a0196b5a89f._comment
@@ -0,0 +1,31 @@
+[[!comment format=mdwn
+ username="http://joey.kitenet.net/"
+ nickname="joey"
+ subject="Good question!"
+ date="2011-12-20T23:07:25Z"
+ content="""
+You get a regular git merge conflict, which can be resolved in any of the regular ways, except that conflicting files are just symlinks.
+
+Example:
+
+
    +$ git pull
+...
+Auto-merging myfile
+CONFLICT (add/add): Merge conflict in myfile
+Automatic merge failed; fix conflicts and then commit the result.
+$ git status
+# On branch master
+# Your branch and 'origin/master' have diverged,
+# and have 1 and 1 different commit(s) each, respectively.
+#
+# Unmerged paths:
+#   (use \"git add/rm ...\" as appropriate to mark resolution)
+#
+#	both added:         myfile
+#
+no changes added to commit (use \"git add\" and/or \"git commit -a\")
+$ git add myfile
+$ git commit -m \"took local version of the conflicting file\"
+
    
+"""]]
diff --git a/doc/forum/A_tiny_filesystem__63__.mdwn b/doc/forum/A_tiny_filesystem__63__.mdwn
new file mode 100644
index 0000000000..44a397e2f6
--- /dev/null
+++ b/doc/forum/A_tiny_filesystem__63__.mdwn
@@ -0,0 +1,7 @@
+First of all, thanks for the amazing work! I've already tried Owncloud, SparkleShare, Unison, rsync and lsycnd. Finally I may have found the real deal.
+
+My main problem with git-annex right now is that it is not fast/reliable enough. The main issues being: broken links now and then (I've probably done something wrong) and the slow: "startup scans", "consistency checks", "attempting to fix here" and "syncing with server". (50 Giga, Work-Server-Home, setup via webapp assistant)
+
+Please, tell me if I'm wrong, but I have the impression that git-annex would get much more robust if it was 100% sure that nobody could mess with its file tree. One possibility would be to add the option (when creating a repo) to make a protected filesystem controlled by git-annex only via FUSE (not something like ShareBox). This could be a tiny filesystem (such as loggedfs) that does nothing but make sure that only git-annex can mount/change files. This would definitely add to speed (reducing checks) and stability.
+
+I'm I being too naive? Most likely there is something I'm overlooking, like the amount of work this would involve.
diff --git a/doc/forum/A_tiny_filesystem__63__/comment_1_993e3f5dbe4bcbb5b7bd9e08ab9554f3._comment b/doc/forum/A_tiny_filesystem__63__/comment_1_993e3f5dbe4bcbb5b7bd9e08ab9554f3._comment
new file mode 100644
index 0000000000..aa246b5ee7
--- /dev/null
+++ b/doc/forum/A_tiny_filesystem__63__/comment_1_993e3f5dbe4bcbb5b7bd9e08ab9554f3._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="http://joeyh.name/"
+ ip="209.250.56.244"
+ subject="comment 1"
+ date="2014-04-05T22:34:32Z"
+ content="""
+git-annex can indeed be sped up by preventing modification of files in the tree. you can do this by running \"git annex indirect\"
+"""]]
diff --git a/doc/forum/A_tiny_filesystem__63__/comment_2_af57591d42868c8aa1cc1eda43ca8b98._comment b/doc/forum/A_tiny_filesystem__63__/comment_2_af57591d42868c8aa1cc1eda43ca8b98._comment
new file mode 100644
index 0000000000..1d4bc4be7a
--- /dev/null
+++ b/doc/forum/A_tiny_filesystem__63__/comment_2_af57591d42868c8aa1cc1eda43ca8b98._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="https://id.koumbit.net/anarcat"
+ ip="2001:1928:1:9::1"
+ subject="comment 2"
+ date="2014-04-07T04:25:43Z"
+ content="""
+what's wrong with [sharebox](https://github.com/chmduquesne/sharebox-fs), actually?
+"""]]
diff --git a/doc/forum/A_tiny_filesystem__63__/comment_3_3869c0472b50d7cf5e29ac0720f4f20f._comment b/doc/forum/A_tiny_filesystem__63__/comment_3_3869c0472b50d7cf5e29ac0720f4f20f._comment
new file mode 100644
index 0000000000..9c1613ca7c
--- /dev/null
+++ b/doc/forum/A_tiny_filesystem__63__/comment_3_3869c0472b50d7cf5e29ac0720f4f20f._comment
@@ -0,0 +1,10 @@
+[[!comment format=mdwn
+ username="augusto"
+ ip="177.98.104.136"
+ subject="comment 3"
+ date="2014-04-08T22:46:18Z"
+ content="""
+When I saw Sharebox's page on Github I had the impression it was vaporware. It has a section named \"Planned Interface\" and there are no updates for quite a while.
+
+Is it working? How to install/use it?
+"""]]
diff --git a/doc/forum/Accessing_files_directly_on__a_USB_device.mdwn b/doc/forum/Accessing_files_directly_on__a_USB_device.mdwn
new file mode 100644
index 0000000000..7b16f65231
--- /dev/null
+++ b/doc/forum/Accessing_files_directly_on__a_USB_device.mdwn
@@ -0,0 +1,11 @@
+Using the assistant, I have created a repository on my laptop plus a synced repository on a USB disk. Looking into the first repository, I see my files accompanied by a .git directory. However, looking on the USB disk (e.g. /media/usb/annex), all I see is what looks like the content of a .git directory.
+
+This means that it is difficult to retrieve any file directly from this disk -- it has to be synced to another local repository first.
+
+Is there any way to change this ? E.g. to have a copy of the working tree, plus a .git directory, on the disk ?
+
+My use case: I have added plenty of media files to my repository. In addition to using the USB disk as a backup/medium for transfering these files to another computer, I'd like to be able to plug the disk to e.g. a media player and read the files directly from the tree, but it does not work at the moment.
+
+Is there anything I am missing ?
+
+Edited: yes, there is something I was missing: the forum entry at [[forum/USB_backup_with_files_visible/]]
diff --git a/doc/forum/Accessing_files_in_SSH_remotes_while_SSHed_into_remote.mdwn b/doc/forum/Accessing_files_in_SSH_remotes_while_SSHed_into_remote.mdwn
new file mode 100644
index 0000000000..5d0addf6c4
--- /dev/null
+++ b/doc/forum/Accessing_files_in_SSH_remotes_while_SSHed_into_remote.mdwn
@@ -0,0 +1 @@
+Hello, I would like to use git-annex to sync files between my desktop, laptop, and a remote server.  The remote server is part of a Beowulf cluster, these files are used to run simulations on the cluster, but I want the same files on my laptop and desktop to mirror those on the cluster.  I already have git annex assistant syncing files between the desktop and laptop.  I do not and will not ever have root access on the cluster, so I can't install git-annex there.  I can access the cluster server via ssh or by mounting it as NFS (OS X 10.11.6).  The NFS solution doesn't work due to the issues with file locking previously discussed on these forums.  I can set up an ssh repo but that's a "special remote" and I cannot access the files when ssh'ed in there, which I need to do to run the simulations.  Is there a strategy to do what I want with git-annex?  Thanks!
diff --git a/doc/forum/Accessing_files_in_SSH_remotes_while_SSHed_into_remote/comment_1_b2f7b4aa5e7c859de9741db440292365._comment b/doc/forum/Accessing_files_in_SSH_remotes_while_SSHed_into_remote/comment_1_b2f7b4aa5e7c859de9741db440292365._comment
new file mode 100644
index 0000000000..fa475cba09
--- /dev/null
+++ b/doc/forum/Accessing_files_in_SSH_remotes_while_SSHed_into_remote/comment_1_b2f7b4aa5e7c859de9741db440292365._comment
@@ -0,0 +1,10 @@
+[[!comment format=mdwn
+ username="xloem"
+ avatar="http://cdn.libravatar.org/avatar/b8c087f7c5e6a9358748f0727c077f3b"
+ subject="comment 1"
+ date="2017-01-16T12:29:59Z"
+ content="""
+It sounds like you want to have a git-annex working directory on the server.
+
+You can put git-annex on the server without root access.  Just place it in a user folder such as ~/local .
+"""]]
diff --git a/doc/forum/Accessing_files_in_SSH_remotes_while_SSHed_into_remote/comment_2_6450ee0b5729beec155fbe0ce304e223._comment b/doc/forum/Accessing_files_in_SSH_remotes_while_SSHed_into_remote/comment_2_6450ee0b5729beec155fbe0ce304e223._comment
new file mode 100644
index 0000000000..72348528aa
--- /dev/null
+++ b/doc/forum/Accessing_files_in_SSH_remotes_while_SSHed_into_remote/comment_2_6450ee0b5729beec155fbe0ce304e223._comment
@@ -0,0 +1,9 @@
+[[!comment format=mdwn
+ username="jaylangseth@fa84aeedaf6dd96a51e93b012339ac09fb4dc135"
+ nickname="jaylangseth"
+ avatar="http://cdn.libravatar.org/avatar/a40561e76777f90b29891f8fb86b5d05"
+ subject="comment 2"
+ date="2017-01-17T16:37:39Z"
+ content="""
+Ok thanks!  I'll give that a try.
+"""]]
diff --git a/doc/forum/Accessing_files_in_SSH_remotes_while_SSHed_into_remote/comment_3_9d8d03bda09596d03d9cfb7138a152f0._comment b/doc/forum/Accessing_files_in_SSH_remotes_while_SSHed_into_remote/comment_3_9d8d03bda09596d03d9cfb7138a152f0._comment
new file mode 100644
index 0000000000..b689938a9e
--- /dev/null
+++ b/doc/forum/Accessing_files_in_SSH_remotes_while_SSHed_into_remote/comment_3_9d8d03bda09596d03d9cfb7138a152f0._comment
@@ -0,0 +1,9 @@
+[[!comment format=mdwn
+ username="jaylangseth@fa84aeedaf6dd96a51e93b012339ac09fb4dc135"
+ nickname="jaylangseth"
+ avatar="http://cdn.libravatar.org/avatar/a40561e76777f90b29891f8fb86b5d05"
+ subject="comment 3"
+ date="2017-01-18T21:11:15Z"
+ content="""
+Ok!  Got it working, and I can ssh between the two machines and access git-annex-shell on both sides, got all the $PATH stuff right.  I set up one repo on the desktop and one on the server, made them remotes of each other, and I have managed to transfer files back and forth using \"git annex sync.\"  But how do I get the assistant to do this automatically without command line intervention?  Just dropping the file in one folder, it doesn't show up in another, unlike the other git-annex repos I have between my two local computers.
+"""]]
diff --git a/doc/forum/Accessing_files_in_SSH_remotes_while_SSHed_into_remote/comment_4_650ebbe4e7b57361a42882f3092122f0._comment b/doc/forum/Accessing_files_in_SSH_remotes_while_SSHed_into_remote/comment_4_650ebbe4e7b57361a42882f3092122f0._comment
new file mode 100644
index 0000000000..8d08278600
--- /dev/null
+++ b/doc/forum/Accessing_files_in_SSH_remotes_while_SSHed_into_remote/comment_4_650ebbe4e7b57361a42882f3092122f0._comment
@@ -0,0 +1,15 @@
+[[!comment format=mdwn
+ username="joey"
+ subject="""comment 4"""
+ date="2017-02-13T20:14:31Z"
+ content="""
+In your original message, you mentioned NFS won't work due to locking.
+However, as far as I know, those NFS locking problems can be worked
+around by `git config annex.pidlock true`. That said, I encourage
+*not* using NFS if there is any alternative and it sounds like you've
+found an alternative, so great.
+
+Re your question about the assistant, have you tried running the assistant
+in the repo on the server? Once it's running, it should update the
+repository whenever the client pushes over a change.
+"""]]
diff --git a/doc/forum/Accessing_files_in_bare_repository.mdwn b/doc/forum/Accessing_files_in_bare_repository.mdwn
new file mode 100644
index 0000000000..2ea6ba9d32
--- /dev/null
+++ b/doc/forum/Accessing_files_in_bare_repository.mdwn
@@ -0,0 +1,5 @@
+I have set up a remote server repository using the git-annex web assistant, accessed via ssh. The repository is a bare one according to the config file in the annex directory on the server.
+
+I am wondering how I could access any of the files in the repository while logged in to the server - they don't have their usual file names to look for clearly. Is there a way to get a list of the files for starters? I tried using git annex find, but it never returns any files. git annex get followed by a filename of a document in the repository also doesn't work.
+
+Thanks to help me understand better how to approach this.
diff --git a/doc/forum/Accessing_files_in_bare_repository/comment_10_7eb66e3806f9524e043fae2da9d57d64._comment b/doc/forum/Accessing_files_in_bare_repository/comment_10_7eb66e3806f9524e043fae2da9d57d64._comment
new file mode 100644
index 0000000000..2b9bb86317
--- /dev/null
+++ b/doc/forum/Accessing_files_in_bare_repository/comment_10_7eb66e3806f9524e043fae2da9d57d64._comment
@@ -0,0 +1,10 @@
+[[!comment format=mdwn
+ username="http://edheil.wordpress.com/"
+ ip="173.162.44.162"
+ subject="comment 10"
+ date="2013-05-13T04:08:14Z"
+ content="""
+I'm not sure what's up.  I hope Joey can weigh in.
+
+\"git annex sync\" ought to go get the symlinks from your remote, and merge them into your master branch, which is currently checked out, so you should see them.  :(
+"""]]
diff --git a/doc/forum/Accessing_files_in_bare_repository/comment_11_f0165d66865ad14f7eb5d50e900c1df4._comment b/doc/forum/Accessing_files_in_bare_repository/comment_11_f0165d66865ad14f7eb5d50e900c1df4._comment
new file mode 100644
index 0000000000..abacbc30e4
--- /dev/null
+++ b/doc/forum/Accessing_files_in_bare_repository/comment_11_f0165d66865ad14f7eb5d50e900c1df4._comment
@@ -0,0 +1,10 @@
+[[!comment format=mdwn
+ username="http://joeyh.name/"
+ nickname="joey"
+ subject="comment 11"
+ date="2013-05-13T17:33:26Z"
+ content="""
+When the assistant (or git annex sync) pushes to a repository, it creates a `synced/master` branch. So your bare repository has no regular `master` branch. So when you clone it, you get a repository with no branch checked out, which is what git-annex sync complains about.
+
+There are several solutions. The easiest is to run `git merge origin/synced/master`; then your checkout will have a master branch and you can use `git annex sync` from then on.
+"""]]
diff --git a/doc/forum/Accessing_files_in_bare_repository/comment_12_0e7ea5161b6da6e9bb9425bdb953de33._comment b/doc/forum/Accessing_files_in_bare_repository/comment_12_0e7ea5161b6da6e9bb9425bdb953de33._comment
new file mode 100644
index 0000000000..8d083d32db
--- /dev/null
+++ b/doc/forum/Accessing_files_in_bare_repository/comment_12_0e7ea5161b6da6e9bb9425bdb953de33._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="https://me.yahoo.com/a/bBy7WkgQicYHIiiyj.Vm0TcMbxi2quzbPFef#6f9f7"
+ nickname="Frederik Vanrenterghem"
+ subject="comment 12"
+ date="2013-05-14T04:07:37Z"
+ content="""
+Thanks Joey, this worked. The symlinks are in the directory and I can get the actual file contents (except for one attempt that failed - I'll need to check why that didn't seem to have propagated to the bare repository.)
+"""]]
diff --git a/doc/forum/Accessing_files_in_bare_repository/comment_13_f804b9bf71f7d04bd23ce32d813dc340._comment b/doc/forum/Accessing_files_in_bare_repository/comment_13_f804b9bf71f7d04bd23ce32d813dc340._comment
new file mode 100644
index 0000000000..470541f1d1
--- /dev/null
+++ b/doc/forum/Accessing_files_in_bare_repository/comment_13_f804b9bf71f7d04bd23ce32d813dc340._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="http://joeyh.name/"
+ nickname="joey"
+ subject="comment 13"
+ date="2013-06-12T18:56:39Z"
+ content="""
+Regarding my comment above, I've changed things so now `git annex sync` to a bare repository *does* push to the regular branch. So `git clone` will work without this bother.
+"""]]
diff --git a/doc/forum/Accessing_files_in_bare_repository/comment_1_6de649d38febd2240eb5b703da77c2d6._comment b/doc/forum/Accessing_files_in_bare_repository/comment_1_6de649d38febd2240eb5b703da77c2d6._comment
new file mode 100644
index 0000000000..ec14394b88
--- /dev/null
+++ b/doc/forum/Accessing_files_in_bare_repository/comment_1_6de649d38febd2240eb5b703da77c2d6._comment
@@ -0,0 +1,15 @@
+[[!comment format=mdwn
+ username="http://edheil.wordpress.com/"
+ ip="173.162.44.162"
+ subject="comment 1"
+ date="2013-05-08T13:40:16Z"
+ content="""
+Do you have git-annex available on the server from the command line?
+
+If so, the easiest way would probably be to clone the repo in git, and do a \"git annex init\" in it.  This would give you a copy of the repo with none of the files in it.  It would be tiny, just full of symlinks, no content.  Then pull down copies of *only* the individual files you need using \"git annex get.\"
+
+Maybe set that repo to \"untrusted\" to make sure other repos don't start depending on it to store copies of files to fulfill numcopies.
+
+When you're done with a copy of a file, just \"git annex drop\" it again so it disappears from this skeletal repo.
+
+"""]]
diff --git a/doc/forum/Accessing_files_in_bare_repository/comment_2_7e8dd09915ddc3267377e900891cb02c._comment b/doc/forum/Accessing_files_in_bare_repository/comment_2_7e8dd09915ddc3267377e900891cb02c._comment
new file mode 100644
index 0000000000..9c4df2f054
--- /dev/null
+++ b/doc/forum/Accessing_files_in_bare_repository/comment_2_7e8dd09915ddc3267377e900891cb02c._comment
@@ -0,0 +1,24 @@
+[[!comment format=mdwn
+ username="https://me.yahoo.com/a/bBy7WkgQicYHIiiyj.Vm0TcMbxi2quzbPFef#6f9f7"
+ nickname="Frederik Vanrenterghem"
+ subject="comment 2"
+ date="2013-05-10T03:21:57Z"
+ content="""
+I have git-annex on the VPS. The bare repository is in the directory \"annex\"
+
+From the user's root directory, I execute:
+
+git clone annex/ Documents
+Cloning into 'Documents'...
+done.
+warning: remote HEAD refers to nonexistent ref, unable to checkout.
+
+cd Documents/
+~/Documents$ git annex init 'Documents on VPS'
+init Documents on VPS ok
+(Recording state in git...)
+
+Still, when I try to get a file now, or do a git annex find, nothing shows up.
+
+
+"""]]
diff --git a/doc/forum/Accessing_files_in_bare_repository/comment_3_80eae4a73f38d1a7e35f97c33b6401f8._comment b/doc/forum/Accessing_files_in_bare_repository/comment_3_80eae4a73f38d1a7e35f97c33b6401f8._comment
new file mode 100644
index 0000000000..31aa5b1a39
--- /dev/null
+++ b/doc/forum/Accessing_files_in_bare_repository/comment_3_80eae4a73f38d1a7e35f97c33b6401f8._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="http://edheil.wordpress.com/"
+ ip="99.54.57.201"
+ subject="comment 3"
+ date="2013-05-10T04:11:22Z"
+ content="""
+OK, I'm not sure what is up, but perhaps try \"git annex sync\" and see if that gives you the treeful of symlinks that you want.
+"""]]
diff --git a/doc/forum/Accessing_files_in_bare_repository/comment_4_5ec13e98d3ecb69426e974d34f712f9b._comment b/doc/forum/Accessing_files_in_bare_repository/comment_4_5ec13e98d3ecb69426e974d34f712f9b._comment
new file mode 100644
index 0000000000..720d787666
--- /dev/null
+++ b/doc/forum/Accessing_files_in_bare_repository/comment_4_5ec13e98d3ecb69426e974d34f712f9b._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="http://edheil.wordpress.com/"
+ ip="99.54.57.201"
+ subject="comment 4"
+ date="2013-05-10T04:11:53Z"
+ content="""
+that is, do \"git annex sync\" from within your new clone.
+"""]]
diff --git a/doc/forum/Accessing_files_in_bare_repository/comment_5_dccbf5793998c6381e23eb8ff6497ebf._comment b/doc/forum/Accessing_files_in_bare_repository/comment_5_dccbf5793998c6381e23eb8ff6497ebf._comment
new file mode 100644
index 0000000000..d89ab793ec
--- /dev/null
+++ b/doc/forum/Accessing_files_in_bare_repository/comment_5_dccbf5793998c6381e23eb8ff6497ebf._comment
@@ -0,0 +1,17 @@
+[[!comment format=mdwn
+ username="https://me.yahoo.com/a/bBy7WkgQicYHIiiyj.Vm0TcMbxi2quzbPFef#6f9f7"
+ nickname="Frederik Vanrenterghem"
+ subject="comment 5"
+ date="2013-05-10T04:23:28Z"
+ content="""
+Thanks for your help. Unfortunately that doesn't do the trick either:
+
+git annex sync
+(merging origin/git-annex into git-annex...)
+(Recording state in git...)
+commit
+ok
+git-annex: no branch is checked out
+
+After this, git annex find still yields no results.
+"""]]
diff --git a/doc/forum/Accessing_files_in_bare_repository/comment_6_42d923916232c81f3b8bdbefa34a89d3._comment b/doc/forum/Accessing_files_in_bare_repository/comment_6_42d923916232c81f3b8bdbefa34a89d3._comment
new file mode 100644
index 0000000000..f33b521466
--- /dev/null
+++ b/doc/forum/Accessing_files_in_bare_repository/comment_6_42d923916232c81f3b8bdbefa34a89d3._comment
@@ -0,0 +1,18 @@
+[[!comment format=mdwn
+ username="http://edheil.wordpress.com/"
+ ip="173.162.44.162"
+ subject="comment 6"
+ date="2013-05-10T12:52:48Z"
+ content="""
+OK, I'm going to guess the problem is there is no \"master\" branch in your cloned repository, because there is no \"master\" branch in a bare remote. (Not sure about this, just guessing.)  Try \"git branch\" in your clone.  If there is no \"master\" branch, that's the problem.
+
+If that's true, the right thing to do would probably have been not to clone from it, but to create a fresh git repo and then set your bare repo as a remote.
+
+You could try doing that -- create a fresh git repo, git annex init it, and then try something like \"git remote add barerepo /path/to/bare/repo\" and then \"git annex sync\" which should pull changes from the bare repo's 'synced/master' branch into your 'synced/master' branch and from there into your master branch, giving you all your symlinks.
+
+If that works you could declare your previous attempt \"dead\" and you'll be good.
+
+It *may* also be possible to just fix the clone you've got by creating a master branch, maybe by branching off of synced/master.  Do a \"git branch\" and see whether synced/master exists, and just try \"git branch master synced/master\" and then \"git checkout master\" and that might do the job!
+
+But I'm just throwing ideas around here, so I hope somebody who actually knows what's going on (joey? are you there?) will pipe up.
+"""]]
diff --git a/doc/forum/Accessing_files_in_bare_repository/comment_7_43a0a7d222faee582aeb3150a59cef87._comment b/doc/forum/Accessing_files_in_bare_repository/comment_7_43a0a7d222faee582aeb3150a59cef87._comment
new file mode 100644
index 0000000000..7684b0c461
--- /dev/null
+++ b/doc/forum/Accessing_files_in_bare_repository/comment_7_43a0a7d222faee582aeb3150a59cef87._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="https://me.yahoo.com/a/bBy7WkgQicYHIiiyj.Vm0TcMbxi2quzbPFef#6f9f7"
+ nickname="Frederik Vanrenterghem"
+ subject="comment 7"
+ date="2013-05-12T14:04:29Z"
+ content="""
+That all sounded very plausible, but for some reason it failed to work for me. I went back to basics and tried getting there by replicating steps 1 to 3 of the walkthrough, but also to no aval. I'll still try one more thing - setting up a local repository on the VPS and syncing it to my main one over XMPP. That seems rather convoluted given the bare repository is on the same server, but who knows...
+"""]]
diff --git a/doc/forum/Accessing_files_in_bare_repository/comment_8_ec1024235c1c74c113483a833df84654._comment b/doc/forum/Accessing_files_in_bare_repository/comment_8_ec1024235c1c74c113483a833df84654._comment
new file mode 100644
index 0000000000..e609a4c902
--- /dev/null
+++ b/doc/forum/Accessing_files_in_bare_repository/comment_8_ec1024235c1c74c113483a833df84654._comment
@@ -0,0 +1,12 @@
+[[!comment format=mdwn
+ username="http://edheil.wordpress.com/"
+ ip="173.162.44.162"
+ subject="comment 8"
+ date="2013-05-12T19:06:14Z"
+ content="""
+So what *does* show up?  Is your repo an empty directory?  Or are the symlinks that should point to content there?
+
+as far as git goes, are you on the master branch?  *is* there a master branch?  If you do \"git status\" what do you see?
+
+There's no reason this shouldn't work for you....
+"""]]
diff --git a/doc/forum/Accessing_files_in_bare_repository/comment_9_c156b8c1ae0f2905566bbdb13b84e577._comment b/doc/forum/Accessing_files_in_bare_repository/comment_9_c156b8c1ae0f2905566bbdb13b84e577._comment
new file mode 100644
index 0000000000..43b93b271c
--- /dev/null
+++ b/doc/forum/Accessing_files_in_bare_repository/comment_9_c156b8c1ae0f2905566bbdb13b84e577._comment
@@ -0,0 +1,37 @@
+[[!comment format=mdwn
+ username="https://me.yahoo.com/a/bBy7WkgQicYHIiiyj.Vm0TcMbxi2quzbPFef#6f9f7"
+ nickname="Frederik Vanrenterghem"
+ subject="comment 9"
+ date="2013-05-12T23:51:16Z"
+ content="""
+I get the following:
+
+frederik@niihau:~/Documents$ git status
+# On branch master
+#
+# Initial commit
+#
+nothing to commit (create/copy files and use \"git add\" to track)
+
+And this is the config:
+
+frederik@niihau:~/Documents$ cat .git/config 
+[core]
+        repositoryformatversion = 0
+        filemode = true
+        bare = false
+        logallrefupdates = true
+[remote \"origin\"]
+        fetch = +refs/heads/*:refs/remotes/origin/*
+        url = /home/frederik/annex
+        annex-uuid = 1648a298-27aa-4cb1-xx-xxx
+[annex]
+        uuid = 4406bd35-9ff7-4008-yy-yyy
+        version = 3
+[remote \"niihau\"]
+        url = /home/frederik/annex
+        fetch = +refs/heads/*:refs/remotes/niihau/*
+        annex-uuid = 1648a298-27aa-4cb1-xx-xxx
+
+Not having worked with git before, all this is still very new for me. I have 33M of data in the .git directory, mostly in objects, so it does seem to have synced something already.
+"""]]
diff --git a/doc/forum/Actions_very_slow_on_a_direct___40__and_crippled__41___annex_repository.mdwn b/doc/forum/Actions_very_slow_on_a_direct___40__and_crippled__41___annex_repository.mdwn
new file mode 100644
index 0000000000..f09b2ea3de
--- /dev/null
+++ b/doc/forum/Actions_very_slow_on_a_direct___40__and_crippled__41___annex_repository.mdwn
@@ -0,0 +1,28 @@
+Hi!
+
+One of my annex repository has a very strange behavior. Every `git annex` command is very slow.
+
+I'm using MacOSX Mavericks, and this repository is a network drive, mounted with Samba. It's using direct mode, and the filesystem is crippled. I use *annex* for storing huge files, for example movies. I moved some files in this directory, and used `git annex add`. It was long (as checksum was performed) and I thought that everything was OK. I tried `git log -p`, it was OK too:
+
+    new file mode 120000
+    index 0000000..e58c65a
+    --- /dev/null
+    +++ b/Movies/movie.mp4
+    @@ -0,0 +1 @@
+    +../.git/annex/objects/FK/60/SHA256E-s346858581--053dca6a842376ab8022722df306ad5
+    \ No newline at end of file
+
+However it was not. I tried to launch `git annex sync another_repo` (with *another_repo* indirect and on a local disk) and it took ages. Even `git annex list` takes ages, **on every repository linked to this one**. With `ps -A`, I found out that the issue was created by `git --git-dir=/Volumes/SAMBA_REMOTE/.git --work-tree=/Volumes/SAMBA_REMOTE -c core.bare=false checkout -q -B annex/direct/master`.
+
+**Have you ever noticed this behavior? Have I done something wrong?**
+
+Here is the output of `git annex version`:
+
+    git-annex version: 5.20131117-gbd514dc
+    build flags: Assistant Webapp Pairing Testsuite S3 WebDAV FsEvents XMPP DNS Feeds Quvi TDFA CryptoHash
+    key/value backends: SHA256E SHA1E SHA512E SHA224E SHA384E SKEIN256E SKEIN512E SHA256 SHA1 SHA512 SHA224 SHA384 SKEIN256 SKEIN512 WORM URL
+    remote types: git gcrypt S3 bup directory rsync web webdav glacier hook
+    local repository version: 4
+    default repository version: 3
+    supported repository versions: 3 5
+    upgrade supported from repository versions: 0 1 2 4
diff --git a/doc/forum/Actions_very_slow_on_a_direct___40__and_crippled__41___annex_repository/bugs/direct_mode_sync_should_avoid_git_commit.mdwn b/doc/forum/Actions_very_slow_on_a_direct___40__and_crippled__41___annex_repository/bugs/direct_mode_sync_should_avoid_git_commit.mdwn
new file mode 100644
index 0000000000..ec7ce2ccef
--- /dev/null
+++ b/doc/forum/Actions_very_slow_on_a_direct___40__and_crippled__41___annex_repository/bugs/direct_mode_sync_should_avoid_git_commit.mdwn
@@ -0,0 +1 @@
+See [[/bugs/direct_mode_sync_should_avoid_git_commit/]] instead.
diff --git a/doc/forum/Actions_very_slow_on_a_direct___40__and_crippled__41___annex_repository/comment_10_06dae5709750ea1da4f7fdbee4e84efc._comment b/doc/forum/Actions_very_slow_on_a_direct___40__and_crippled__41___annex_repository/comment_10_06dae5709750ea1da4f7fdbee4e84efc._comment
new file mode 100644
index 0000000000..c8feb05b93
--- /dev/null
+++ b/doc/forum/Actions_very_slow_on_a_direct___40__and_crippled__41___annex_repository/comment_10_06dae5709750ea1da4f7fdbee4e84efc._comment
@@ -0,0 +1,16 @@
+[[!comment format=mdwn
+ username="http://joeyh.name/"
+ ip="209.250.56.64"
+ subject="comment 10"
+ date="2013-11-26T21:48:11Z"
+ content="""
+Hmm. Seems to me that `git commit` is trying to download the whole big file from the SMB share. Perhaps just to compare it with the symlink it expects to be there?
+
+When I try this, in a large direct mode repository with some video from *my* family, `git commit` does not open the file (verified with strace), let alone read it, and so finishes in well under 1 second. 
+
+Aha, I was able to reproduce `git commit` doing that with a repo on a FAT filesystem. git opens the file, and mmaps it, and I guess it proceeds to try to diff it against the symlink standin file it expected to be there.
+
+This is particularly unfortunate, since `git commit` is only, I think, doing this so that it can print out a \"Changes not staged for commit\" message. Which git-annex sync throws away. There seems to be no git commit option that disables this behavior.
+
+I think that this calls for making `git annex sync` not use `git commit` any longer, and instead manually build the commit using `git-write-tree` and `git-commit-tree`. Since I don't know if I will get to this before the thanksgiving holiday, I am creating a bug: [[bugs/direct_mode_sync_should_avoid_git_commit]]
+"""]]
diff --git a/doc/forum/Actions_very_slow_on_a_direct___40__and_crippled__41___annex_repository/comment_11_2069c5c41882ae0a1973fb7da583b60e._comment b/doc/forum/Actions_very_slow_on_a_direct___40__and_crippled__41___annex_repository/comment_11_2069c5c41882ae0a1973fb7da583b60e._comment
new file mode 100644
index 0000000000..76ecd13c2a
--- /dev/null
+++ b/doc/forum/Actions_very_slow_on_a_direct___40__and_crippled__41___annex_repository/comment_11_2069c5c41882ae0a1973fb7da583b60e._comment
@@ -0,0 +1,10 @@
+[[!comment format=mdwn
+ username="TroisSinges"
+ ip="82.227.207.5"
+ subject="comment 11"
+ date="2013-11-26T21:57:52Z"
+ content="""
+Awesome! Thanks for your very efficient investigations. I'll stay tuned to the bug report.
+
+Happy Thanksgiving!
+"""]]
diff --git a/doc/forum/Actions_very_slow_on_a_direct___40__and_crippled__41___annex_repository/comment_12_b35e3a87c30974eedd71ebe52ecbed96._comment b/doc/forum/Actions_very_slow_on_a_direct___40__and_crippled__41___annex_repository/comment_12_b35e3a87c30974eedd71ebe52ecbed96._comment
new file mode 100644
index 0000000000..282f1fb4ce
--- /dev/null
+++ b/doc/forum/Actions_very_slow_on_a_direct___40__and_crippled__41___annex_repository/comment_12_b35e3a87c30974eedd71ebe52ecbed96._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="TroisSinges"
+ ip="82.227.207.5"
+ subject="comment 12"
+ date="2013-11-27T12:08:29Z"
+ content="""
+I corrected the bug address: See [[/bugs/direct_mode_sync_should_avoid_git_commit/]]
+"""]]
diff --git a/doc/forum/Actions_very_slow_on_a_direct___40__and_crippled__41___annex_repository/comment_13_84e026f9bda87bfd12a3769dcef77f8b._comment b/doc/forum/Actions_very_slow_on_a_direct___40__and_crippled__41___annex_repository/comment_13_84e026f9bda87bfd12a3769dcef77f8b._comment
new file mode 100644
index 0000000000..dfc0db3d2d
--- /dev/null
+++ b/doc/forum/Actions_very_slow_on_a_direct___40__and_crippled__41___annex_repository/comment_13_84e026f9bda87bfd12a3769dcef77f8b._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="http://joeyh.name/"
+ ip="209.250.56.64"
+ subject="comment 13"
+ date="2013-11-27T12:13:04Z"
+ content="""
+thanks!
+"""]]
diff --git a/doc/forum/Actions_very_slow_on_a_direct___40__and_crippled__41___annex_repository/comment_14_b2ac2ea300a5026832b40a1a6b27a7cd._comment b/doc/forum/Actions_very_slow_on_a_direct___40__and_crippled__41___annex_repository/comment_14_b2ac2ea300a5026832b40a1a6b27a7cd._comment
new file mode 100644
index 0000000000..4dfc762ac3
--- /dev/null
+++ b/doc/forum/Actions_very_slow_on_a_direct___40__and_crippled__41___annex_repository/comment_14_b2ac2ea300a5026832b40a1a6b27a7cd._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="http://joeyh.name/"
+ ip="209.250.56.64"
+ subject="comment 14"
+ date="2013-12-02T20:05:00Z"
+ content="""
+Should be fixed; the git commit is now avoided.
+"""]]
diff --git a/doc/forum/Actions_very_slow_on_a_direct___40__and_crippled__41___annex_repository/comment_1_ed3534196164c6736a8dbf21c65c119d._comment b/doc/forum/Actions_very_slow_on_a_direct___40__and_crippled__41___annex_repository/comment_1_ed3534196164c6736a8dbf21c65c119d._comment
new file mode 100644
index 0000000000..ed104f488e
--- /dev/null
+++ b/doc/forum/Actions_very_slow_on_a_direct___40__and_crippled__41___annex_repository/comment_1_ed3534196164c6736a8dbf21c65c119d._comment
@@ -0,0 +1,10 @@
+[[!comment format=mdwn
+ username="http://joeyh.name/"
+ ip="209.250.56.64"
+ subject="comment 1"
+ date="2013-11-26T17:26:12Z"
+ content="""
+If you run git-annex with --debug, it will print out every command it is running. This is useful because A) I can see the commands in context and B) it includes timestamps, which is a little bit more informative than \"it took ages\". (Which can mean anything.)
+
+Anyway, I figured out the problem. Upgrade from v4 to v5 does what should be a one-time git checkout, but it seems that the auto upgrade code neglected to update annex.version, so it started doing it on every command run in a v4 repo. Fixed in git. You can work around the bug by running \"git config annex.version 5\".
+"""]]
diff --git a/doc/forum/Actions_very_slow_on_a_direct___40__and_crippled__41___annex_repository/comment_2_1e29bcf568f02765c48f0eac6c640673._comment b/doc/forum/Actions_very_slow_on_a_direct___40__and_crippled__41___annex_repository/comment_2_1e29bcf568f02765c48f0eac6c640673._comment
new file mode 100644
index 0000000000..ab0cd9c9d4
--- /dev/null
+++ b/doc/forum/Actions_very_slow_on_a_direct___40__and_crippled__41___annex_repository/comment_2_1e29bcf568f02765c48f0eac6c640673._comment
@@ -0,0 +1,45 @@
+[[!comment format=mdwn
+ username="TroisSinges"
+ ip="82.227.207.5"
+ subject="comment 2"
+ date="2013-11-26T20:42:03Z"
+ content="""
+My problem isn't solved. I cloned the repository from my netbook to the SMD drive, and added a file in this new direct annex repository :
+
+    [3singes:/Volumes/Video/Videos/Films] $ git annex add MyFamily.mkv --debug
+    [2013-11-26 19:52:59 CET] read: git [\"--git-dir=/Volumes/Video/Videos/.git\",\"--work-tree=/Volumes/Video/Videos\",\"-c\",\"core.bare=false\",\"ls-files\",\"--others\",\"--exclude-standard\",\"-z\",\"--\",\"MyFamily.mkv\"]
+    [2013-11-26 19:52:59 CET] read: git [\"--git-dir=/Volumes/Video/Videos/.git\",\"--work-tree=/Volumes/Video/Videos\",\"-c\",\"core.bare=false\",\"ls-files\",\"--modified\",\"-z\",\"--\",\"MyFamily.mkv\"]
+    [2013-11-26 19:52:59 CET] chat: git [\"--git-dir=/Volumes/Video/Videos/.git\",\"--work-tree=/Volumes/Video/Videos\",\"-c\",\"core.bare=false\",\"cat-file\",\"--batch\"]
+    add MyFamily.mkv [2013-11-26 19:52:59 CET] chat: git [\"--git-dir=/Volumes/Video/Videos/.git\",\"--work-tree=/Volumes/Video/Videos\",\"-c\",\"core.bare=false\",\"check-attr\",\"-z\",\"--stdin\",\"annex.backend\",\"annex.numcopies\",\"--\"]
+    (checksum...) [2013-11-26 19:52:59 CET] read: gsha256sum [\"MyFamily.mkv\"]
+    [2013-11-26 20:55:35 CET] chat: git [\"--git-dir=/Volumes/Video/Videos/.git\",\"--work-tree=/Volumes/Video/Videos\",\"-c\",\"core.bare=false\",\"cat-file\",\"--batch\"]
+    [2013-11-26 20:55:35 CET] chat: git [\"--git-dir=/Volumes/Video/Videos/.git\",\"--work-tree=/Volumes/Video/Videos\",\"-c\",\"core.bare=false\",\"hash-object\",\"-t\",\"blob\",\"-w\",\"--stdin\",\"--no-filters\"]
+    ok
+    (Recording state in git...)
+    [2013-11-26 20:55:35 CET] feed: git [\"--git-dir=/Volumes/Video/Videos/.git\",\"--work-tree=/Volumes/Video/Videos\",\"-c\",\"core.bare=false\",\"update-index\",\"-z\",\"--index-info\"]
+    [2013-11-26 20:55:35 CET] chat: git [\"--git-dir=/Volumes/Video/Videos/.git\",\"--work-tree=/Volumes/Video/Videos\",\"-c\",\"core.bare=false\",\"hash-object\",\"-w\",\"--stdin-paths\",\"--no-filters\"]
+    [2013-11-26 20:55:35 CET] feed: git [\"--git-dir=/Volumes/Video/Videos/.git\",\"--work-tree=/Volumes/Video/Videos\",\"-c\",\"core.bare=false\",\"update-index\",\"-z\",\"--index-info\"]
+    [2013-11-26 20:55:35 CET] read: git [\"--git-dir=/Volumes/Video/Videos/.git\",\"--work-tree=/Volumes/Video/Videos\",\"-c\",\"core.bare=false\",\"show-ref\",\"--hash\",\"refs/heads/git-annex\"]
+    [2013-11-26 20:55:36 CET] read: git [\"--git-dir=/Volumes/Video/Videos/.git\",\"--work-tree=/Volumes/Video/Videos\",\"-c\",\"core.bare=false\",\"write-tree\"]
+    [2013-11-26 20:55:36 CET] chat: git [\"--git-dir=/Volumes/Video/Videos/.git\",\"--work-tree=/Volumes/Video/Videos\",\"-c\",\"core.bare=false\",\"commit-tree\",\"9bdb7b18b44020441ec8dc179e676120e769ff32\",\"-p\",\"refs/heads/git-annex\"]
+    [2013-11-26 20:55:37 CET] call: git [\"--git-dir=/Volumes/Video/Videos/.git\",\"--work-tree=/Volumes/Video/Videos\",\"-c\",\"core.bare=false\",\"update-ref\",\"refs/heads/git-annex\",\"293332232ca4e7428090171b52dcc5b3b80d1df6\"]
+
+Let's try to sync to my macbookpro annex repository:
+
+    [3singes:/Volumes/Video/Videos/Films] $ git annex sync macbookpro --debug
+    [2013-11-26 20:57:48 CET] read: git [\"--git-dir=/Volumes/Video/Videos/.git\",\"--work-tree=/Volumes/Video/Videos\",\"-c\",\"core.bare=false\",\"show-ref\",\"git-annex\"]
+    [2013-11-26 20:57:48 CET] read: git [\"--git-dir=/Volumes/Video/Videos/.git\",\"--work-tree=/Volumes/Video/Videos\",\"-c\",\"core.bare=false\",\"show-ref\",\"--hash\",\"refs/heads/git-annex\"]
+    [2013-11-26 20:57:49 CET] read: git [\"--git-dir=/Volumes/Video/Videos/.git\",\"--work-tree=/Volumes/Video/Videos\",\"-c\",\"core.bare=false\",\"log\",\"refs/heads/git-annex..293332232ca4e7428090171b52dcc5b3b80d1df6\",\"--oneline\",\"-n1\"]
+    [2013-11-26 20:57:49 CET] read: git [\"--git-dir=/Volumes/Video/Videos/.git\",\"--work-tree=/Volumes/Video/Videos\",\"-c\",\"core.bare=false\",\"log\",\"refs/heads/git-annex..b2146cf0abc7d6a7e5c79d3b133181b56cd77461\",\"--oneline\",\"-n1\"]
+    [2013-11-26 20:57:49 CET] read: git [\"--git-dir=/Volumes/Video/Videos/.git\",\"--work-tree=/Volumes/Video/Videos\",\"-c\",\"core.bare=false\",\"log\",\"refs/heads/git-annex..c2cf7488b4745d4168491d8e62f5bdc354869133\",\"--oneline\",\"-n1\"]
+    [2013-11-26 20:57:49 CET] chat: git [\"--git-dir=/Volumes/Video/Videos/.git\",\"--work-tree=/Volumes/Video/Videos\",\"-c\",\"core.bare=false\",\"cat-file\",\"--batch\"]
+    [2013-11-26 20:57:49 CET] read: git [\"config\",\"--null\",\"--list\"]
+    [2013-11-26 20:57:49 CET] read: git [\"--git-dir=/Volumes/Video/Videos/.git\",\"--work-tree=/Volumes/Video/Videos\",\"-c\",\"core.bare=false\",\"ls-files\",\"--stage\",\"-z\",\"--others\",\"--exclude-standard\",\"--\",\"/Volumes/Video/Videos\"]
+    [2013-11-26 20:57:50 CET] chat: git [\"--git-dir=/Volumes/Video/Videos/.git\",\"--work-tree=/Volumes/Video/Videos\",\"-c\",\"core.bare=false\",\"cat-file\",\"--batch\"]
+    (Recording state in git...)
+    [2013-11-26 20:57:55 CET] feed: xargs [\"-0\",\"git\",\"--git-dir=/Volumes/Video/Videos/.git\",\"--work-tree=/Volumes/Video/Videos\",\"-c\",\"core.bare=false\",\"add\",\"-f\"]
+    commit
+    [2013-11-26 20:57:55 CET] read: git [\"--git-dir=/Volumes/Video/Videos/.git\",\"--work-tree=/Volumes/Video/Videos\",\"-c\",\"core.bare=false\",\"commit\",\"-m\",\"git-annex automatic sync\"]
+
+Now it's 21:41 CET, and it's been stuck for circa 1 hour.
+"""]]
diff --git a/doc/forum/Actions_very_slow_on_a_direct___40__and_crippled__41___annex_repository/comment_3_9ea6803a94b1de15079a3fa20d59c9af._comment b/doc/forum/Actions_very_slow_on_a_direct___40__and_crippled__41___annex_repository/comment_3_9ea6803a94b1de15079a3fa20d59c9af._comment
new file mode 100644
index 0000000000..f2c4fd77e1
--- /dev/null
+++ b/doc/forum/Actions_very_slow_on_a_direct___40__and_crippled__41___annex_repository/comment_3_9ea6803a94b1de15079a3fa20d59c9af._comment
@@ -0,0 +1,10 @@
+[[!comment format=mdwn
+ username="http://joeyh.name/"
+ ip="209.250.56.64"
+ subject="comment 3"
+ date="2013-11-26T20:51:59Z"
+ content="""
+The problem you reported *is* solved. You were not talking about syncing before.
+
+If your setup is such that `git commit` takes a long time to run, then `git annex sync` is also necessarily going to take a long time to run.
+"""]]
diff --git a/doc/forum/Actions_very_slow_on_a_direct___40__and_crippled__41___annex_repository/comment_4_3fae5a7fa5d99d0eb4473adb43e7f6f5._comment b/doc/forum/Actions_very_slow_on_a_direct___40__and_crippled__41___annex_repository/comment_4_3fae5a7fa5d99d0eb4473adb43e7f6f5._comment
new file mode 100644
index 0000000000..e9b7655cf7
--- /dev/null
+++ b/doc/forum/Actions_very_slow_on_a_direct___40__and_crippled__41___annex_repository/comment_4_3fae5a7fa5d99d0eb4473adb43e7f6f5._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="TroisSinges"
+ ip="82.227.207.5"
+ subject="comment 4"
+ date="2013-11-26T20:55:39Z"
+ content="""
+Thanks for your answer. On the second part of my problem, *why* should `git commit` be slow? It's only committing a file with a line (../.git/annex/...), isn't it?
+"""]]
diff --git a/doc/forum/Actions_very_slow_on_a_direct___40__and_crippled__41___annex_repository/comment_5_57a5b73cff480266355e75c7bdc762c2._comment b/doc/forum/Actions_very_slow_on_a_direct___40__and_crippled__41___annex_repository/comment_5_57a5b73cff480266355e75c7bdc762c2._comment
new file mode 100644
index 0000000000..51e31d9b4e
--- /dev/null
+++ b/doc/forum/Actions_very_slow_on_a_direct___40__and_crippled__41___annex_repository/comment_5_57a5b73cff480266355e75c7bdc762c2._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="TroisSinges"
+ ip="82.227.207.5"
+ subject="comment 5"
+ date="2013-11-26T21:02:17Z"
+ content="""
+Err, I read my post again, and I definitely *was* talking about syncing ;-)
+"""]]
diff --git a/doc/forum/Actions_very_slow_on_a_direct___40__and_crippled__41___annex_repository/comment_6_bbcf5e863c8f152e1079536e9011a404._comment b/doc/forum/Actions_very_slow_on_a_direct___40__and_crippled__41___annex_repository/comment_6_bbcf5e863c8f152e1079536e9011a404._comment
new file mode 100644
index 0000000000..4640a91bfc
--- /dev/null
+++ b/doc/forum/Actions_very_slow_on_a_direct___40__and_crippled__41___annex_repository/comment_6_bbcf5e863c8f152e1079536e9011a404._comment
@@ -0,0 +1,16 @@
+[[!comment format=mdwn
+ username="http://joeyh.name/"
+ ip="209.250.56.64"
+ subject="comment 6"
+ date="2013-11-26T21:04:44Z"
+ content="""
+I think the most likely reason for git commit to be slow on your setup is that it probably rewrites .git/index. If you have a lot of files in your repository, the index file will be large and rewriting the index file will involve re-transferring it all over the network to the SMB share. 
+
+It's also possible that git commit scans the whole work tree, although I don't think it should -- it's not been told to with -a.
+
+You may be able to find what's taking a long time by
+
+    strace git --git-dir=/Volumes/Video/Videos/.git --work-tree=/Volumes/Video/Videos -c core.bare=false commit -m \"git-annex automatic sync\"
+
+(or ltrace)
+"""]]
diff --git a/doc/forum/Actions_very_slow_on_a_direct___40__and_crippled__41___annex_repository/comment_7_fdcd144c22601bdf98ff844254b0120d._comment b/doc/forum/Actions_very_slow_on_a_direct___40__and_crippled__41___annex_repository/comment_7_fdcd144c22601bdf98ff844254b0120d._comment
new file mode 100644
index 0000000000..033a1af45a
--- /dev/null
+++ b/doc/forum/Actions_very_slow_on_a_direct___40__and_crippled__41___annex_repository/comment_7_fdcd144c22601bdf98ff844254b0120d._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="http://joeyh.name/"
+ ip="209.250.56.64"
+ subject="comment 7"
+ date="2013-11-26T21:07:12Z"
+ content="""
+If you kept your git repository on your local disk, and used a [[directory special remote|special_remotes/directory]] on the SMB share, you could then `git annex copy --to smbshare` and `git annex get --from smbshare` as desired, which would probably be much more efficient.
+"""]]
diff --git a/doc/forum/Actions_very_slow_on_a_direct___40__and_crippled__41___annex_repository/comment_8_b77243e765b2af7ba71e963fcb5cbbb1._comment b/doc/forum/Actions_very_slow_on_a_direct___40__and_crippled__41___annex_repository/comment_8_b77243e765b2af7ba71e963fcb5cbbb1._comment
new file mode 100644
index 0000000000..273b9d8e91
--- /dev/null
+++ b/doc/forum/Actions_very_slow_on_a_direct___40__and_crippled__41___annex_repository/comment_8_b77243e765b2af7ba71e963fcb5cbbb1._comment
@@ -0,0 +1,71 @@
+[[!comment format=mdwn
+ username="TroisSinges"
+ ip="82.227.207.5"
+ subject="comment 8"
+ date="2013-11-26T21:17:38Z"
+ content="""
+Thanks again for your answers.
+
+`.git/index` is only 30Ko big. It shouldn't be an issue, even through the network.
+
+However, I ran `lsof`, and strangely it lock the big file I added before (`/Volumes/Video/Videos/Films/MyFamily.mkv`)
+
+    COMMAND   PID    USER   FD   TYPE DEVICE    SIZE/OFF                 NODE NAME
+    git     68047 3singes  cwd    DIR   46,7       16384  5152641528667254045 /Volumes/Video/Videos
+    git     68047 3singes  txt    REG    1,4     3035984             77501447 /Applications/git-annex.app/Contents/MacOS/bundle/git
+    git     68047 3singes  txt    REG   46,7        8632  9219631544499731818 /Volumes/Video/Videos/.git/objects/pack/pack-99901c82bacfa26c04f505ceb27c4af9ccda414f.idx
+    git     68047 3singes  txt    REG    1,4      403392             77501680 /Applications/git-annex.app/Contents/MacOS/bundle/usr/lib/libpcre.0.dylib
+    git     68047 3singes  txt    REG   46,7        4012  8204150806531008435 /Volumes/Video/Videos/.git/objects/pack/pack-bb60a2315b13bda720682a4213507e5ee9ba029f.idx
+    git     68047 3singes  txt    REG   46,7       15969 12003107822997399035 /Volumes/Video/Videos/.git/objects/pack/pack-bb60a2315b13bda720682a4213507e5ee9ba029f.pack
+    git     68047 3singes  txt    REG    1,4      169392             77501683 /Applications/git-annex.app/Contents/MacOS/bundle/usr/lib/libz.1.dylib
+    git     68047 3singes  txt    REG    1,4     2088992             77501673 /Applications/git-annex.app/Contents/MacOS/bundle/usr/lib/libiconv.2.dylib
+    git     68047 3singes  txt    REG    1,4      600832             72809225 /usr/lib/dyld
+    git     68047 3singes  txt    REG    1,4   342937194             75930310 /private/var/db/dyld/dyld_shared_cache_x86_64
+    git     68047 3singes    0u   CHR  16,10    0t718314                 2413 /dev/ttys010
+    git     68047 3singes    1w   CHR    3,2         0t0                  308 /dev/null
+    git     68047 3singes    2w   CHR    3,2         0t0                  308 /dev/null
+    git     68047 3singes    3u   REG   46,7           0 17793593705208135594 /Volumes/Video/Videos/.git/index.lock
+    git     68047 3singes    4r   REG   46,7 15581987050 12221620371912261305 /Volumes/Video/Videos/Films/MyFamily.mkv
+    git     68047 3singes    8w   CHR    3,2         0t0                  308 /dev/null
+
+I'm using MacOSX, so I can only use `dtrace`, which I don't know so much.
+
+**The process is over, after 1H30:**
+
+    ok
+    [2013-11-26 22:13:48 CET] read: git [\"--git-dir=/Volumes/Video/Videos/.git\",\"--work-tree=/Volumes/Video/Videos\",\"-c\",\"core.bare=false\",\"symbolic-ref\",\"HEAD\"]
+    [2013-11-26 22:13:48 CET] read: git [\"--git-dir=/Volumes/Video/Videos/.git\",\"--work-tree=/Volumes/Video/Videos\",\"-c\",\"core.bare=false\",\"show-ref\",\"refs/heads/annex/direct/master\"]
+    [2013-11-26 22:13:48 CET] call: git [\"--git-dir=/Volumes/Video/Videos/.git\",\"--work-tree=/Volumes/Video/Videos\",\"-c\",\"core.bare=false\",\"show-ref\",\"--verify\",\"-q\",\"refs/heads/synced/master\"]
+    [2013-11-26 22:13:49 CET] read: git [\"--git-dir=/Volumes/Video/Videos/.git\",\"--work-tree=/Volumes/Video/Videos\",\"-c\",\"core.bare=false\",\"log\",\"refs/heads/annex/direct/master..refs/heads/synced/master\",\"--oneline\",\"-n1\"]
+    pull macbookpro
+    [2013-11-26 22:13:49 CET] call: git [\"--git-dir=/Volumes/Video/Videos/.git\",\"--work-tree=/Volumes/Video/Videos\",\"-c\",\"core.bare=false\",\"fetch\",\"macbookpro\"]
+    From /Users/3singes/Movies/NE PAS SAUVEGARDER/Videos
+       b2146cf..c2cf748  git-annex  -> macbookpro/git-annex
+    [2013-11-26 22:13:50 CET] call: git [\"--git-dir=/Volumes/Video/Videos/.git\",\"--work-tree=/Volumes/Video/Videos\",\"-c\",\"core.bare=false\",\"show-ref\",\"--verify\",\"-q\",\"refs/remotes/macbookpro/annex/direct/master\"]
+    [2013-11-26 22:13:50 CET] call: git [\"--git-dir=/Volumes/Video/Videos/.git\",\"--work-tree=/Volumes/Video/Videos\",\"-c\",\"core.bare=false\",\"show-ref\",\"--verify\",\"-q\",\"refs/remotes/macbookpro/synced/master\"]
+    [2013-11-26 22:13:50 CET] read: git [\"--git-dir=/Volumes/Video/Videos/.git\",\"--work-tree=/Volumes/Video/Videos\",\"-c\",\"core.bare=false\",\"log\",\"refs/heads/synced/master..refs/remotes/macbookpro/synced/master\",\"--oneline\",\"-n1\"]
+    ok
+    [2013-11-26 22:13:50 CET] read: git [\"--git-dir=/Volumes/Video/Videos/.git\",\"--work-tree=/Volumes/Video/Videos\",\"-c\",\"core.bare=false\",\"show-ref\",\"git-annex\"]
+    [2013-11-26 22:13:50 CET] read: git [\"--git-dir=/Volumes/Video/Videos/.git\",\"--work-tree=/Volumes/Video/Videos\",\"-c\",\"core.bare=false\",\"show-ref\",\"--hash\",\"refs/heads/git-annex\"]
+    [2013-11-26 22:13:50 CET] read: git [\"--git-dir=/Volumes/Video/Videos/.git\",\"--work-tree=/Volumes/Video/Videos\",\"-c\",\"core.bare=false\",\"log\",\"refs/heads/git-annex..293332232ca4e7428090171b52dcc5b3b80d1df6\",\"--oneline\",\"-n1\"]
+    [2013-11-26 22:13:51 CET] read: git [\"--git-dir=/Volumes/Video/Videos/.git\",\"--work-tree=/Volumes/Video/Videos\",\"-c\",\"core.bare=false\",\"log\",\"refs/heads/git-annex..b2146cf0abc7d6a7e5c79d3b133181b56cd77461\",\"--oneline\",\"-n1\"]
+    [2013-11-26 22:13:51 CET] read: git [\"--git-dir=/Volumes/Video/Videos/.git\",\"--work-tree=/Volumes/Video/Videos\",\"-c\",\"core.bare=false\",\"log\",\"refs/heads/git-annex..c2cf7488b4745d4168491d8e62f5bdc354869133\",\"--oneline\",\"-n1\"]
+    [2013-11-26 22:13:51 CET] call: git [\"--git-dir=/Volumes/Video/Videos/.git\",\"--work-tree=/Volumes/Video/Videos\",\"-c\",\"core.bare=false\",\"branch\",\"-f\",\"synced/master\"]
+    [2013-11-26 22:13:51 CET] call: git [\"--git-dir=/Volumes/Video/Videos/.git\",\"--work-tree=/Volumes/Video/Videos\",\"-c\",\"core.bare=false\",\"branch\",\"-f\",\"master\"]
+    [2013-11-26 22:13:51 CET] call: git [\"--git-dir=/Volumes/Video/Videos/.git\",\"--work-tree=/Volumes/Video/Videos\",\"-c\",\"core.bare=false\",\"show-ref\",\"--verify\",\"-q\",\"refs/remotes/macbookpro/synced/master\"]
+    [2013-11-26 22:13:51 CET] read: git [\"--git-dir=/Volumes/Video/Videos/.git\",\"--work-tree=/Volumes/Video/Videos\",\"-c\",\"core.bare=false\",\"log\",\"refs/remotes/macbookpro/synced/master..refs/heads/synced/master\",\"--oneline\",\"-n1\"]
+    [2013-11-26 22:13:51 CET] call: git [\"--git-dir=/Volumes/Video/Videos/.git\",\"--work-tree=/Volumes/Video/Videos\",\"-c\",\"core.bare=false\",\"show-ref\",\"--verify\",\"-q\",\"refs/remotes/macbookpro/git-annex\"]
+    [2013-11-26 22:13:51 CET] read: git [\"--git-dir=/Volumes/Video/Videos/.git\",\"--work-tree=/Volumes/Video/Videos\",\"-c\",\"core.bare=false\",\"log\",\"refs/remotes/macbookpro/git-annex..git-annex\",\"--oneline\",\"-n1\"]
+    push macbookpro
+    [2013-11-26 22:13:52 CET] call: git [\"--git-dir=/Volumes/Video/Videos/.git\",\"--work-tree=/Volumes/Video/Videos\",\"-c\",\"core.bare=false\",\"push\",\"macbookpro\",\"+git-annex:synced/git-annex\",\"annex/direct/master:synced/master\"]
+    Counting objects: 9, done.
+    Delta compression using up to 8 threads.
+    Compressing objects: 100% (4/4), done.
+    Writing objects: 100% (5/5), 440 bytes | 0 bytes/s, done.
+    Total 5 (delta 3), reused 0 (delta 0)
+    To /Users/3singes/Movies/NE PAS SAUVEGARDER/Videos
+       c2cf748..2933322  git-annex -> synced/git-annex
+    [2013-11-26 22:13:57 CET] read: git [\"--git-dir=/Volumes/Video/Videos/.git\",\"--work-tree=/Volumes/Video/Videos\",\"-c\",\"core.bare=false\",\"push\",\"macbookpro\",\"master\"]
+    ok
+
+"""]]
diff --git a/doc/forum/Actions_very_slow_on_a_direct___40__and_crippled__41___annex_repository/comment_9_cb0815e96ee211d4778f2e7a4274e855._comment b/doc/forum/Actions_very_slow_on_a_direct___40__and_crippled__41___annex_repository/comment_9_cb0815e96ee211d4778f2e7a4274e855._comment
new file mode 100644
index 0000000000..041f3c5477
--- /dev/null
+++ b/doc/forum/Actions_very_slow_on_a_direct___40__and_crippled__41___annex_repository/comment_9_cb0815e96ee211d4778f2e7a4274e855._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="TroisSinges"
+ ip="82.227.207.5"
+ subject="comment 9"
+ date="2013-11-26T21:19:42Z"
+ content="""
+I thought to a directory special remote, but it doesn't meet my needs: the SMB drive is used by a MediaCenter, so I need the working tree.
+"""]]
diff --git a/doc/forum/Add_a___34__local__34___remote.mdwn b/doc/forum/Add_a___34__local__34___remote.mdwn
new file mode 100644
index 0000000000..c35a637a85
--- /dev/null
+++ b/doc/forum/Add_a___34__local__34___remote.mdwn
@@ -0,0 +1,13 @@
+I have been playing around with annex assistent today (way late as I am one of the Kickstarter backers) and found the following puzzling.
+
+I am running a fairly new version, my package manager calls it 4.20130323 (git-annex does not seem to have a --version switch). 
+
+I have been playing around with the annex assistent webapp today and have the following questions/observations: 
+
+1) The "remote" server creates a bare git repository as far as I can tell, the "local computer" probably does the right thing (but I cannot run it as the "other box" has no X or browser). Is there any way to remotely create a "local" copy (i.e. non-bare git repository, which can later be managed by the cli?)
+2) It seems like git-annex-assistent (webapp) binds to localhost, is it possible to let it bind to either a specific ip-address (or all)? (Yes, I understand the security implications, but the use-case is the box from question 2 - i.e. on local network but no X and no browser). 
+3) what is being launched when I hit "files", on the video it starts a file-manager on my box nothing happens (and no errors as far as I can tell). 
+
+Thank you in advance
+
+Svenne
diff --git a/doc/forum/Add_a___34__local__34___remote/comment_1_c68ad724b465c4be5243be687168c0b3._comment b/doc/forum/Add_a___34__local__34___remote/comment_1_c68ad724b465c4be5243be687168c0b3._comment
new file mode 100644
index 0000000000..251e89e034
--- /dev/null
+++ b/doc/forum/Add_a___34__local__34___remote/comment_1_c68ad724b465c4be5243be687168c0b3._comment
@@ -0,0 +1,12 @@
+[[!comment format=mdwn
+ username="http://joeyh.name/"
+ nickname="joey"
+ subject="comment 1"
+ date="2013-04-08T19:11:33Z"
+ content="""
+1. If you want to use the cli for managing the repository, why not `git clone` it using the cli as well? Set up a remote in your existing repository pointing at the repository you cloned, and the assistant will push changes to it. This is all that the webapp would do when creating the repository. The webapp does not set up remote non-bare repositories because unless the user is comfortable with the cli, nothing is going to keep them up-to-date.
+
+2. Setting up the assistant on a headless box is something I need to support better, indeed. I've just now made `git annex webapp --listen=IP` listen on the specified IP address, and output the URL you can connect to from the remote side. Note that this does not yet use HTTPS, so use with caution.
+
+3. It uses the `xdg-open` program to open a file manager. If `xdg-open` was not in your PATH, it would instead point the web browser at a file:/// URI as a fallback. If it seems to do nothing, this probably means `xdg-open` is in PATH but not working for some reason. Try running `xdg-open /path/to/your/repository`
+"""]]
diff --git a/doc/forum/Add_annex_files_outside_git_root_directory.mdwn b/doc/forum/Add_annex_files_outside_git_root_directory.mdwn
new file mode 100644
index 0000000000..8d90cbf99a
--- /dev/null
+++ b/doc/forum/Add_annex_files_outside_git_root_directory.mdwn
@@ -0,0 +1,24 @@
+Hi,
+
+I love the idea of git-annex, and I'm trying to do my first steps with it. I'm stuck on the following issue:
+
+When creating a git repository and annexing a file in the root directory, everything works as expected:
+
+    git init
+    git annex init "mytest"
+    echo 1 > annexfile
+    git annex add annexfile
+    # add annexfile ok
+    # (Recording state in git...)
+
+However, when I try to add a file in a subdirectory of the repository, the adding command fails without any error message:
+
+    mkdir mydir
+    echo 1 > mydir/myfile
+    git annex add mydir/myfile
+    # no message, no status change, nothing
+    # note that I can add the file in the root dir and then move it to `mydir`
+
+Am I doing something wrong here?
+
+Thank you!
diff --git a/doc/forum/Add_annex_files_outside_git_root_directory/comment_1_3d7c8917633a6632f22385b8f7a98ccf._comment b/doc/forum/Add_annex_files_outside_git_root_directory/comment_1_3d7c8917633a6632f22385b8f7a98ccf._comment
new file mode 100644
index 0000000000..c0a433f947
--- /dev/null
+++ b/doc/forum/Add_annex_files_outside_git_root_directory/comment_1_3d7c8917633a6632f22385b8f7a98ccf._comment
@@ -0,0 +1,18 @@
+[[!comment format=mdwn
+ username="joey"
+ subject="""comment 1"""
+ date="2015-02-04T17:19:46Z"
+ content="""
+git-annex should have no difficulty doing that, and in fact its test
+suite includes some adds of files in subdirs.
+
+The most likely reason for this behavior would be if you have configured
+a `.gitignore` file (or other method of configuring gitignore) to 
+exclude the file you want to add. If so, "git annex add --force
+mydir/myfile" would override the gitignore.
+
+Internally, git-annex add runs "git ls-files --others --exclude-standard"
+and passes it the files/dirs you specified to add, and then adds
+the files that command lists. So you can run that command and see
+what it lists too.
+"""]]
diff --git a/doc/forum/Add_annex_files_outside_git_root_directory/comment_2_312a57883d809857a69216fa47e04f17._comment b/doc/forum/Add_annex_files_outside_git_root_directory/comment_2_312a57883d809857a69216fa47e04f17._comment
new file mode 100644
index 0000000000..0e5be5eec8
--- /dev/null
+++ b/doc/forum/Add_annex_files_outside_git_root_directory/comment_2_312a57883d809857a69216fa47e04f17._comment
@@ -0,0 +1,15 @@
+[[!comment format=mdwn
+ username="https://www.google.com/accounts/o8/id?id=AItOawlJl0OCe6AJEnIFIcg-t5Rhk-lI_Y-tWUs"
+ nickname="Michael"
+ subject="comment 2"
+ date="2015-02-12T16:22:18Z"
+ content="""
+Ok, seems that I assumed incorrectly that `.gitignore` files were not valid for annex.
+
+However, if I `git add file_that_is_ignored`, I get an error message saying:
+
+    The following paths are ignored by one of your .gitignore files:
+    Use -f if you really want to add them.
+
+Would it be possible for git-annex to be consistent with git behaviour here?
+"""]]
diff --git a/doc/forum/Add_file_to_git_index_while_in_direct_mode.mdwn b/doc/forum/Add_file_to_git_index_while_in_direct_mode.mdwn
new file mode 100644
index 0000000000..e1678e95c7
--- /dev/null
+++ b/doc/forum/Add_file_to_git_index_while_in_direct_mode.mdwn
@@ -0,0 +1,20 @@
+Hi, 
+
+Is there a preferred way to add files to git's index (instead of the annex) while in direct mode? 
+
+I am trying to use git-annex for scientific data. On linux and mac in indirect mode, the git workflow is basically unchanged, and I really like the way git annex works. Unfortunately when I use git-annex on Windows, I have not found a good way to add things to git's tree. 
+
+I've tried two things:
+
+```
+$ git annex proxy -- git add smallfile
+```
+
+This is slow because it has to copy things over to create a work tree
+
+```
+$ git config annex.largefiles="*.pdf or *.h5"
+$ git annex add largefile.pdf
+```
+
+This doesn't seem to work as well as I'd like either. I'd like the equivalent of a ```git add``` command in direct mode.
diff --git a/doc/forum/Add_file_to_git_index_while_in_direct_mode/comment_1_3eb43a983eba09f500b776f75b299341._comment b/doc/forum/Add_file_to_git_index_while_in_direct_mode/comment_1_3eb43a983eba09f500b776f75b299341._comment
new file mode 100644
index 0000000000..df4fd7655f
--- /dev/null
+++ b/doc/forum/Add_file_to_git_index_while_in_direct_mode/comment_1_3eb43a983eba09f500b776f75b299341._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="joey"
+ subject="""comment 1"""
+ date="2015-09-10T22:02:19Z"
+ content="""
+In what way does setting annex.largefiles and using git annex add not work
+as well as you'd like?
+"""]]
diff --git a/doc/forum/Add_file_to_git_index_while_in_direct_mode/comment_2_6945d9e930e7385f0abe6538622a9aa3._comment b/doc/forum/Add_file_to_git_index_while_in_direct_mode/comment_2_6945d9e930e7385f0abe6538622a9aa3._comment
new file mode 100644
index 0000000000..7dc65c2dc6
--- /dev/null
+++ b/doc/forum/Add_file_to_git_index_while_in_direct_mode/comment_2_6945d9e930e7385f0abe6538622a9aa3._comment
@@ -0,0 +1,31 @@
+[[!comment format=mdwn
+ username="aloukian@c071735c08bc04266b792aa478ad1bae2d6a8b50"
+ nickname="aloukian"
+ subject="comment 2"
+ date="2015-09-11T00:42:03Z"
+ content="""
+Sorry, I should have clarified. I was not sure how to add a commit message.
+
+I played with a test repo and I think I figured it out. What I tried was this:
+   
+```
+$ git init test-repo
+$ touch test.txt
+$ git add test.txt
+$ git commit -m \"initial commit\"
+$ git annex init
+$ git config annex.largefiles=\"*.pdf\"
+$ echo \"updated\" > test.txt
+$ git annex add test.txt
+$ git annex sync -m \"updated test.txt\"
+```
+
+So if I understand correctly, in direct mode 
+
+- ```git add``` ->  ```git annex add``` 
+- ```git commit -m MSG``` -> ```git annex sync -m MSG```
+
+```git log``` now shows the right message, but I am still not sure how to check whether a file is in the annex or directly in the git tree. I suppose I could just make aliases for git add and git commit, but I wouldn't be able to add arbitrary files to git annex without playing with annex.largefiles. For example, if I had many small ```.txt``` files and one large file and I wanted to just add that large file to the git annex, I'd have to change the largefiles setting.
+
+Is there a better way of doing this or is loss of functionality intrinsic until Windows symlinks work?
+"""]]
diff --git a/doc/forum/Add_file_to_git_index_while_in_direct_mode/comment_3_8f77b5dcf582e88099b2e6e77563fb11._comment b/doc/forum/Add_file_to_git_index_while_in_direct_mode/comment_3_8f77b5dcf582e88099b2e6e77563fb11._comment
new file mode 100644
index 0000000000..e597917e3e
--- /dev/null
+++ b/doc/forum/Add_file_to_git_index_while_in_direct_mode/comment_3_8f77b5dcf582e88099b2e6e77563fb11._comment
@@ -0,0 +1,10 @@
+[[!comment format=mdwn
+ username="joey"
+ subject="""comment 3"""
+ date="2015-09-11T17:39:05Z"
+ content="""
+You can temporarily set annex.largefiles to match no files, while running
+git-annex add, to force it to add the file to git instead of the annex.
+
+git -c annex.largefiles='exclude=*' annex add somefile
+"""]]
diff --git a/doc/forum/Add_files_to_direct_mode_repos.mdwn b/doc/forum/Add_files_to_direct_mode_repos.mdwn
new file mode 100644
index 0000000000..a5cb2d691c
--- /dev/null
+++ b/doc/forum/Add_files_to_direct_mode_repos.mdwn
@@ -0,0 +1 @@
+I started a git annex repo in a crippled filesystem (FAT32). I git-annex-add'ed some files but then I learned one cannot then git-commit them if on a direct mode repo (“fatal: This operation must be run in a work tree”). How do I add files in a crippled filesystem and then have them propagate to other repost if numcopies > 1?
diff --git a/doc/forum/Add_files_to_direct_mode_repos/comment_1_41d632853d3160899b04da7e4b95475e._comment b/doc/forum/Add_files_to_direct_mode_repos/comment_1_41d632853d3160899b04da7e4b95475e._comment
new file mode 100644
index 0000000000..5ca3a4edfd
--- /dev/null
+++ b/doc/forum/Add_files_to_direct_mode_repos/comment_1_41d632853d3160899b04da7e4b95475e._comment
@@ -0,0 +1,11 @@
+[[!comment format=mdwn
+ username="Gus"
+ avatar="http://cdn.libravatar.org/avatar/665626c67ab3ee7e842183f6f659e120"
+ subject="comment 1"
+ date="2017-09-24T13:14:53Z"
+ content="""
+You should reat the [dirct mode page](https://git-annex.branchable.com/direct_mode/#index5h2).
+
+In short, use `git-annex sync` as a replacement for `git commit`.
+
+"""]]
diff --git a/doc/forum/Adding_a_URL_from_wayback_machine.mdwn b/doc/forum/Adding_a_URL_from_wayback_machine.mdwn
new file mode 100644
index 0000000000..d019e255b6
--- /dev/null
+++ b/doc/forum/Adding_a_URL_from_wayback_machine.mdwn
@@ -0,0 +1,64 @@
+Is there a nice way to add a URL from the Wayback machine?
+I don't want to add HTML documents but rather PDFs etc.
+The problem is that web.archive.org doesn't send a Content-Length header:
+
+    $ curl -I http://web.archive.org/web/20171117120847/https://downloads.kitenet.net/videos/git-annex/git-annex_views_demo.ogg
+    HTTP/1.1 200 OK
+    Server: Tengine/2.1.0
+    Date: Fri, 17 Nov 2017 12:09:35 GMT
+    Content-Type: audio/ogg
+    Connection: keep-alive
+    X-Archive-Orig-content-length: 29784324
+    X-Archive-Orig-accept-ranges: bytes
+    X-Archive-Orig-server: Apache/2.4.29 (Debian)
+    X-Archive-Orig-last-modified: Wed, 11 Feb 2015 01:31:47 GMT
+    X-Archive-Orig-connection: close
+    X-Archive-Orig-etag: "1c67904-50ec5f783257c"
+    X-Archive-Orig-date: Fri, 17 Nov 2017 12:08:49 GMT
+    Cache-Control: max-age=1800
+    X-Archive-Guessed-Content-Type: audio/ogg
+    Memento-Datetime: Fri, 17 Nov 2017 12:08:47 GMT
+    Link: ; rel="original", ; rel="timemap"; type="application/link-format", ; rel="timegate", ; rel="first memento"; datetime="Sat, 26 Apr 2014 04:57:33 GMT", ; rel="prev memento"; datetime="Tue, 22 Aug 2017 17:46:08 GMT", ; rel="memento"; datetime="Fri, 17 Nov 2017 12:08:47 GMT", ; rel="last memento"; datetime="Fri, 17 Nov 2017 12:08:47 GMT"
+    Content-Security-Policy: default-src 'self' 'unsafe-eval' 'unsafe-inline' data: archive.org web.archive.org analytics.archive.org
+    X-App-Server: wwwb-app39
+    X-ts: ----
+    X-Archive-Playback: 0
+    X-location: All
+    X-Page-Cache: MISS
+
+Therefore, I'd have to pass the `--relaxed` flag when calling `addurl`.
+Is there maybe a way to tell git-annex to download the file before checking its size?
+Or is there a way to use the Wayback machine via the S3 special remote?
+
+Eventually, I'd like to have a script that automatically saves all web content in my repo
+into the Wayback machine and adds the new URLs for redundancy.
+
+This is what I've got so far:
+
+    #! /usr/bin/env python3
+    
+    from subprocess import check_output, Popen, PIPE
+    import json
+    from urllib import request
+    from sys import stdout
+    
+    output = check_output(['git-annex', 'find', '--in', 'web', '--json'])
+    files = [json.loads(line)['file'] for line in output.split(b'\n')[:-1]]
+    p = Popen(['git-annex', 'whereis', '--batch', '--json'], stdin=PIPE, stdout=PIPE)
+    p.stdin.writelines(str.encode(f + '\n') for f in files)
+    out, err = p.communicate()
+    urls = []
+    for line in out.split(b'\n')[:-1]:
+        for loc in json.loads(line)['untrusted']:
+            if loc['uuid'] == '00000000-0000-0000-0000-000000000001':
+                url = loc['urls'][0]
+                import urllib.request
+                print('GET https://web.archive.org/save/' + url)
+                r = request.urlopen('https://web.archive.org/save/' + url)
+                assert(r.getcode() == 200)
+                urls.append(r.geturl())
+                break
+    assert(len(files) == len(urls))
+    p = Popen(['git-annex', 'addurl', '--batch', '--with-files', '--relaxed'], stdin=PIPE, stdout=stdout)
+    p.stdin.writelines(str.encode(f + ' ' + u) for f, u in zip(urls, files))
+    p.communicate()
diff --git a/doc/forum/Adding_a_URL_from_wayback_machine/comment_1_bc78c24becbf758da7873cdf01e2adc5._comment b/doc/forum/Adding_a_URL_from_wayback_machine/comment_1_bc78c24becbf758da7873cdf01e2adc5._comment
new file mode 100644
index 0000000000..4660977462
--- /dev/null
+++ b/doc/forum/Adding_a_URL_from_wayback_machine/comment_1_bc78c24becbf758da7873cdf01e2adc5._comment
@@ -0,0 +1,9 @@
+[[!comment format=mdwn
+ username="joey"
+ subject="""comment 1"""
+ date="2017-12-05T17:37:21Z"
+ content="""
+Lack of a Content-Length header does not prevent `git annex addurl` from
+working as far as I can see. Perhaps you should show what the problem
+you're experiencing looks like.
+"""]]
diff --git a/doc/forum/Adding_a_URL_from_wayback_machine/comment_2_6df0501b103a7cfcd1eaa4120f7fa818._comment b/doc/forum/Adding_a_URL_from_wayback_machine/comment_2_6df0501b103a7cfcd1eaa4120f7fa818._comment
new file mode 100644
index 0000000000..c438596bf2
--- /dev/null
+++ b/doc/forum/Adding_a_URL_from_wayback_machine/comment_2_6df0501b103a7cfcd1eaa4120f7fa818._comment
@@ -0,0 +1,26 @@
+[[!comment format=mdwn
+ username="robert.schuetz@7942237bf71a2ae4f5d3cb047d167612b8c9e311"
+ nickname="robert.schuetz"
+ avatar="http://cdn.libravatar.org/avatar/89879460a9e84b9c736d982d9489d3d9"
+ subject="comment 2"
+ date="2017-12-09T11:50:09Z"
+ content="""
+I'm adding a file and later want to add a URL, that doesn't give a Content-Length header, to the file.
+
+It is essantially the following:
+
+    mkdir test
+    cd test
+    git init
+    git annex init
+    wget http://web.archive.org/web/20171117120847/https://downloads.kitenet.net/videos/git-annex/git-annex_views_demo.ogg
+    git annex add
+    git annex addurl \"http://web.archive.org/web/20171117120847/https://downloads.kitenet.net/videos/git-annex/git-annex_views_demo.ogg\" --file git-annex_views_demo.ogg
+
+and the last command gives me the result
+
+    addurl git-annex_views_demo.ogg 
+      while adding a new url to an already annexed file, url does not have expected file size (use --relaxed to bypass this check) http://web.archive.org/web/20171117120847/https://downloads.kitenet.net/videos/git-annex/git-annex_views_demo.ogg
+    failed
+    git-annex: addurl: 1 failed
+"""]]
diff --git a/doc/forum/Adding_a_URL_from_wayback_machine/comment_3_aefbb393c11ab233d145311409d405c7._comment b/doc/forum/Adding_a_URL_from_wayback_machine/comment_3_aefbb393c11ab233d145311409d405c7._comment
new file mode 100644
index 0000000000..1711ccb911
--- /dev/null
+++ b/doc/forum/Adding_a_URL_from_wayback_machine/comment_3_aefbb393c11ab233d145311409d405c7._comment
@@ -0,0 +1,31 @@
+[[!comment format=mdwn
+ username="joey"
+ subject="""comment 3"""
+ date="2017-12-11T17:56:53Z"
+ content="""
+Ah, gotcha.
+
+When adding an url to an annexed file, git-annex doesn't download the
+content again, because that could be a lot of work, instead it checks
+if the size is the same, which is the same check that git-annex always uses
+to see if the web still seems to have the content of a file.
+
+It doesn't feel safe to relax the size check when the web server doesn't
+send a Content-Length, because then there's no indication at all 
+that this url really has the same content as the file. It *might* make
+sense for git-annex to download all the content again in that case, and
+check if it's the expected content and only then add it. However, that
+could be a whole lot of unwanted work to do.
+
+If the user is sure that the url has the same content, it does make sense
+for them to add --relaxed. So, perhaps what's needed is a --strict
+for the users who are not sure and want to force a full download to check.
+
+Here's how to check it yourself, without any changes to git-annex.
+
+	wget -O tmp.git-annex_views_demo.ogg http://web.archive.org/web/20171117120847/https://downloads.kitenet.net/videos/git-annex/git-annex_views_demo.ogg
+	if [ "$(git annex calckey tmp.git-annex_views_demo.ogg)" == "$(git annex find --format '${key}' git-annex_views_demo.ogg)" ]; then
+		git annex addurl \"http://web.archive.org/web/20171117120847/https://downloads.kitenet.net/videos/git-annex/git-annex_views_demo.ogg\" --file git-annex_views_demo.ogg --relaxed
+	fi
+	rm tmp.git-annex_views_demo.ogg
+"""]]
diff --git a/doc/forum/Adding_a_mounted_network.mdwn b/doc/forum/Adding_a_mounted_network.mdwn
new file mode 100644
index 0000000000..5f62e79a2b
--- /dev/null
+++ b/doc/forum/Adding_a_mounted_network.mdwn
@@ -0,0 +1 @@
+I would like to add (bare) repositories on network directories that are locally mounted. "Add more repositories" gives me many choices, which apparently does not include this specifically. It does have "Removable drive" which sees the CIFS directory (perhaps because it's under /media?) but not the AFS directory, and there doesn't seem to be a way to enter a path where it's not inclined to look ("rescan for removable drives" still misses /afs). I am comfortable using shell commands, but I don't know what commands to use.
diff --git a/doc/forum/Adding_a_mounted_network/comment_1_9b4bab6177856569adfec26ada6b01cd._comment b/doc/forum/Adding_a_mounted_network/comment_1_9b4bab6177856569adfec26ada6b01cd._comment
new file mode 100644
index 0000000000..cda6d5fafc
--- /dev/null
+++ b/doc/forum/Adding_a_mounted_network/comment_1_9b4bab6177856569adfec26ada6b01cd._comment
@@ -0,0 +1,13 @@
+[[!comment format=mdwn
+ username="https://www.google.com/accounts/o8/id?id=AItOawmBUR4O9mofxVbpb8JV9mEbVfIYv670uJo"
+ nickname="Justin"
+ subject="comment 1"
+ date="2015-04-09T13:17:41Z"
+ content="""
+It's just a git remote..
+
+    git remote add somerepo /path/to/repo
+    git annex sync
+
+should be all you need.
+"""]]
diff --git a/doc/forum/Adding_a_mounted_network/comment_2_ae3d4ac918ef002ead859ed0c962ae32._comment b/doc/forum/Adding_a_mounted_network/comment_2_ae3d4ac918ef002ead859ed0c962ae32._comment
new file mode 100644
index 0000000000..eaf0665581
--- /dev/null
+++ b/doc/forum/Adding_a_mounted_network/comment_2_ae3d4ac918ef002ead859ed0c962ae32._comment
@@ -0,0 +1,14 @@
+[[!comment format=mdwn
+ username="joey"
+ subject="""comment 2"""
+ date="2015-04-09T18:33:47Z"
+ content="""
+If you want to do with using the webapp, go to the "Repository"
+menu in the upper-left, and select "Add another local repository."
+
+You can then enter the path to your repository, whevever it is, 
+and click on "combine the repositories"
+
+But, like Justin said, this just sets up a git remote, so doing it at
+the command line will work just as well.
+"""]]
diff --git a/doc/forum/Adding_a_mounted_network/comment_3_559cfec9210f8c86de6ee13de0ec2175._comment b/doc/forum/Adding_a_mounted_network/comment_3_559cfec9210f8c86de6ee13de0ec2175._comment
new file mode 100644
index 0000000000..b45f741760
--- /dev/null
+++ b/doc/forum/Adding_a_mounted_network/comment_3_559cfec9210f8c86de6ee13de0ec2175._comment
@@ -0,0 +1,7 @@
+[[!comment format=mdwn
+ username="http://lhealy.livejournal.com/"
+ subject="comment 3"
+ date="2015-04-10T15:35:16Z"
+ content="""
+Thanks for both these answers. For the first, one does the repository have to be made first? I.e., do a `git init --bare` first? I discovered the second approach before reading the comment and it worked, but it did not make a bare repository as happens with the \"removable drive\" option in the assistant.
+"""]]
diff --git a/doc/forum/Adding_existing_S3_bucket_to_sync_with.mdwn b/doc/forum/Adding_existing_S3_bucket_to_sync_with.mdwn
new file mode 100644
index 0000000000..af138c1015
--- /dev/null
+++ b/doc/forum/Adding_existing_S3_bucket_to_sync_with.mdwn
@@ -0,0 +1,16 @@
+Hello,
+
+I am just starting to learn git-annex so forgive me if this is a naive question.
+
+I have the following repositories:
+
+1. Home (laptop)
+2. Work (mac mini)
+3. S3 bucket
+4. USB drive, full backup, attached to Home laptop.
+
+I want to sync files between Home and Work via the S3 bucket.  It is not clear to me how to accomplish this through the git-annex assistant.  Is this possible? Are there instructions online?
+
+Thanks in advance!
+
+Scott
diff --git a/doc/forum/Adding_existing_S3_bucket_to_sync_with/comment_1_30b9a70d367dd5b8781e9a86e42d4c3e._comment b/doc/forum/Adding_existing_S3_bucket_to_sync_with/comment_1_30b9a70d367dd5b8781e9a86e42d4c3e._comment
new file mode 100644
index 0000000000..d68af19f95
--- /dev/null
+++ b/doc/forum/Adding_existing_S3_bucket_to_sync_with/comment_1_30b9a70d367dd5b8781e9a86e42d4c3e._comment
@@ -0,0 +1,10 @@
+[[!comment format=mdwn
+ username="http://marco.paga.myopenid.com/"
+ ip="2001:4dd0:ff00:917b:d575:780:2c27:d982"
+ subject="Yes, it is possible"
+ date="2013-08-19T17:59:56Z"
+ content="""
+You can sync both repos. The simplest way is to have both clients signed on to jabber. This way the clients exchange information when the two are online.
+
+Additionally you need to have a cloud special remote available on both ends. The cloud repository is needed to push and retrieve data for the clients. You need to configure the cloud as a transfer reposity. The clients as - what else - client. The USB Drive can be used as a backup repository on one or both ends and perhaps even for syncing very large files around.
+"""]]
diff --git a/doc/forum/Adding_existing_S3_bucket_to_sync_with/comment_2_a8525c1a7e5f89c30c9503fe8bfed02e._comment b/doc/forum/Adding_existing_S3_bucket_to_sync_with/comment_2_a8525c1a7e5f89c30c9503fe8bfed02e._comment
new file mode 100644
index 0000000000..9e1d27af4e
--- /dev/null
+++ b/doc/forum/Adding_existing_S3_bucket_to_sync_with/comment_2_a8525c1a7e5f89c30c9503fe8bfed02e._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="http://marco.paga.myopenid.com/"
+ ip="2001:4dd0:ff00:917b:d575:780:2c27:d982"
+ subject="I forgot to mention"
+ date="2013-08-19T18:01:47Z"
+ content="""
+You might want to have a look at this: https://git-annex.branchable.com/assistant/remote_sharing_walkthrough/
+"""]]
diff --git a/doc/forum/Adding_existing_S3_bucket_to_sync_with/comment_3_c3878989f74e740c0ed42f440750f3a4._comment b/doc/forum/Adding_existing_S3_bucket_to_sync_with/comment_3_c3878989f74e740c0ed42f440750f3a4._comment
new file mode 100644
index 0000000000..9925f8274d
--- /dev/null
+++ b/doc/forum/Adding_existing_S3_bucket_to_sync_with/comment_3_c3878989f74e740c0ed42f440750f3a4._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="https://www.google.com/accounts/o8/id?id=AItOawkjQhXk8KAh9yD0p1R6QzT-Sw7FtHE3d54"
+ nickname="Scott"
+ subject="comment 3"
+ date="2013-08-19T18:03:41Z"
+ content="""
+Great thanks!  I'll try it out tonight and report back.
+"""]]
diff --git a/doc/forum/Adding_existing_S3_bucket_to_sync_with/comment_4_c06cc86496f9d4c0c42a8c89aa5a7b35._comment b/doc/forum/Adding_existing_S3_bucket_to_sync_with/comment_4_c06cc86496f9d4c0c42a8c89aa5a7b35._comment
new file mode 100644
index 0000000000..5e1a397135
--- /dev/null
+++ b/doc/forum/Adding_existing_S3_bucket_to_sync_with/comment_4_c06cc86496f9d4c0c42a8c89aa5a7b35._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="https://www.google.com/accounts/o8/id?id=AItOawkjQhXk8KAh9yD0p1R6QzT-Sw7FtHE3d54"
+ nickname="Scott"
+ subject="comment 4"
+ date="2013-08-20T16:18:24Z"
+ content="""
+When I try to setup my Jabber client, using my gmail address, the assistant crashes.  I thought this might be due to having two-factor authentication enabled so I generated an application specific password and tried that.  The assistant still crashes.  The log is huge, I am not sure what would be relevant to show here.
+"""]]
diff --git a/doc/forum/Adding_existing_S3_bucket_to_sync_with/comment_5_0a1c2dd0929511ff824be8de2c8d85eb._comment b/doc/forum/Adding_existing_S3_bucket_to_sync_with/comment_5_0a1c2dd0929511ff824be8de2c8d85eb._comment
new file mode 100644
index 0000000000..2c4927b195
--- /dev/null
+++ b/doc/forum/Adding_existing_S3_bucket_to_sync_with/comment_5_0a1c2dd0929511ff824be8de2c8d85eb._comment
@@ -0,0 +1,9 @@
+[[!comment format=mdwn
+ username="http://marco.paga.myopenid.com/"
+ ip="2001:4dd0:ff00:917b:f87b:5579:3820:5db"
+ subject="Please post the log"
+ date="2013-08-21T15:49:00Z"
+ content="""
+I don't know what is going wrong. It is not a bad idea to post the log information.
+
+"""]]
diff --git a/doc/forum/Adding_existing_S3_bucket_to_sync_with/comment_6_1444c2f89885f028f20a4d3ce225a403._comment b/doc/forum/Adding_existing_S3_bucket_to_sync_with/comment_6_1444c2f89885f028f20a4d3ce225a403._comment
new file mode 100644
index 0000000000..21c3d80e35
--- /dev/null
+++ b/doc/forum/Adding_existing_S3_bucket_to_sync_with/comment_6_1444c2f89885f028f20a4d3ce225a403._comment
@@ -0,0 +1,23 @@
+[[!comment format=mdwn
+ username="https://www.google.com/accounts/o8/id?id=AItOawkjQhXk8KAh9yD0p1R6QzT-Sw7FtHE3d54"
+ nickname="Scott"
+ subject="comment 6"
+ date="2013-08-21T20:05:31Z"
+ content="""
+I tried a different approach and attempted to setup s3 at the command line.  This is what I did:
+
+zombie:annex scott$ export AWS_ACCESS_KEY_ID=\"my key\"
+zombie:annex scott$ export AWS_SECRET_ACCESS_KEY=\"my secret key\"
+zombie:annex scott$ git annex initremote S3 type=S3 encryption=shared
+[2013-08-21 13:03:42 PDT] read: git [\"--git-dir=/Network/Servers/filer004/vol/office_homes/scott/annex/.git\",\"--work-tree=/Network/Servers/filer004/vol/office_homes/scott/annex\",\"show-ref\",\"git-annex\"]
+[2013-08-21 13:03:42 PDT] read: git [\"--git-dir=/Network/Servers/filer004/vol/office_homes/scott/annex/.git\",\"--work-tree=/Network/Servers/filer004/vol/office_homes/scott/annex\",\"show-ref\",\"--hash\",\"refs/heads/git-annex\"]
+[2013-08-21 13:03:42 PDT] read: git [\"--git-dir=/Network/Servers/filer004/vol/office_homes/scott/annex/.git\",\"--work-tree=/Network/Servers/filer004/vol/office_homes/scott/annex\",\"log\",\"refs/heads/git-annex..da801570f9ed8d28e5a0cea6cc51f1a2003317d6\",\"--oneline\",\"-n1\"]
+[2013-08-21 13:03:42 PDT] read: git [\"--git-dir=/Network/Servers/filer004/vol/office_homes/scott/annex/.git\",\"--work-tree=/Network/Servers/filer004/vol/office_homes/scott/annex\",\"log\",\"refs/heads/git-annex..8015f3fed32792b558d16008d20816fab0fc50c2\",\"--oneline\",\"-n1\"]
+[2013-08-21 13:03:42 PDT] chat: git [\"--git-dir=/Network/Servers/filer004/vol/office_homes/scott/annex/.git\",\"--work-tree=/Network/Servers/filer004/vol/office_homes/scott/annex\",\"cat-file\",\"--batch\"]
+[2013-08-21 13:03:42 PDT] read: git [\"config\",\"--null\",\"--list\"]
+initremote S3 (encryption setup) [2013-08-21 13:03:42 PDT] read: gpg [\"--quiet\",\"--trust-model\",\"always\",\"--gen-random\",\"--armor\",\"2\",\"512\"]
+(shared cipher) (checking bucket...)
+git-annex: connect: does not exist (Connection refused)
+failed
+git-annex: initremote: 1 failed
+"""]]
diff --git a/doc/forum/Adding_existing_S3_bucket_to_sync_with/comment_7_1c30944010d541096baff18198a5560d._comment b/doc/forum/Adding_existing_S3_bucket_to_sync_with/comment_7_1c30944010d541096baff18198a5560d._comment
new file mode 100644
index 0000000000..7e05bf59af
--- /dev/null
+++ b/doc/forum/Adding_existing_S3_bucket_to_sync_with/comment_7_1c30944010d541096baff18198a5560d._comment
@@ -0,0 +1,23 @@
+[[!comment format=mdwn
+ username="https://www.google.com/accounts/o8/id?id=AItOawkjQhXk8KAh9yD0p1R6QzT-Sw7FtHE3d54"
+ nickname="Scott"
+ subject="comment 7"
+ date="2013-08-21T20:07:58Z"
+ content="""
+Sorry that didnt format properly
+
+    zombie:annex scott$ export AWS_ACCESS_KEY_ID=\"key\"
+    zombie:annex scott$ export AWS_SECRET_ACCESS_KEY=\"secret\"
+    zombie:annex scott$ git annex initremote S3 type=S3 encryption=shared
+    [2013-08-21 13:03:42 PDT] read: git [\"--git-dir=/Network/Servers/filer004/vol/office_homes/scott/annex/.git\",\"--work-tree=/Network/Servers/filer004/vol/office_homes/scott/annex\",\"show-ref\",\"git-annex\"]
+    [2013-08-21 13:03:42 PDT] read: git [\"--git-dir=/Network/Servers/filer004/vol/office_homes/scott/annex/.git\",\"--work-tree=/Network/Servers/filer004/vol/office_homes/scott/annex\",\"show-ref\",\"--hash\",\"refs/heads/git-annex\"]
+    [2013-08-21 13:03:42 PDT] read: git [\"--git-dir=/Network/Servers/filer004/vol/office_homes/scott/annex/.git\",\"--work-tree=/Network/Servers/filer004/vol/office_homes/scott/annex\",\"log\",\"refs/heads/git-annex..da801570f9ed8d28e5a0cea6cc51f1a2003317d6\",\"--oneline\",\"-n1\"]  
+    [2013-08-21 13:03:42 PDT] read: git [\"--git-dir=/Network/Servers/filer004/vol/office_homes/scott/annex/.git\",\"--work-tree=/Network/Servers/filer004/vol/office_homes/scott/annex\",\"log\",\"refs/heads/git-annex..8015f3fed32792b558d16008d20816fab0fc50c2\",\"--oneline\",\"-n1\"  ]
+    [2013-08-21 13:03:42 PDT] chat: git [\"--git-dir=/Network/Servers/filer004/vol/office_homes/scott/annex/.git\",\"--work-tree=/Network/Servers/filer004/vol/office_homes/scott/annex\",\"cat-file\",\"--batch\"]
+    [2013-08-21 13:03:42 PDT] read: git [\"config\",\"--null\",\"--list\"]
+    initremote S3 (encryption setup) [2013-08-21 13:03:42 PDT] read: gpg [\"--quiet\",\"--trust-model\",\"always\",\"--gen-random\",\"--armor\",\"2\",\"512\"]
+    (shared cipher) (checking bucket...)
+    git-annex: connect: does not exist (Connection refused)
+    failed
+    git-annex: initremote: 1 failed
+"""]]
diff --git a/doc/forum/Adding_files_with_wildcard_on_Mac_Yosemite.mdwn b/doc/forum/Adding_files_with_wildcard_on_Mac_Yosemite.mdwn
new file mode 100644
index 0000000000..589877860a
--- /dev/null
+++ b/doc/forum/Adding_files_with_wildcard_on_Mac_Yosemite.mdwn
@@ -0,0 +1,10 @@
+I've been trying to get all image files added to git annex with a wildcard, with no luck. If I use 'git annex add image.jpg', the file gets added and I can commit it. Also using 'git annex add .' works for adding all files. 
+
+Comands I've tried-
+
+git annex add *.jpg     * This outputs to the screen that it's adding each .jpg it finds, but when I go to commit there are 0 files staged.
+
+git annex add . --include='*.jpg'   * This doesn't return any results.
+
+
+What command should I be using on a Mac to just add all files with a specific extension?
diff --git a/doc/forum/Adding_files_with_wildcard_on_Mac_Yosemite/comment_1_e0784676aab3388254cdc9e7dd2d97e7._comment b/doc/forum/Adding_files_with_wildcard_on_Mac_Yosemite/comment_1_e0784676aab3388254cdc9e7dd2d97e7._comment
new file mode 100644
index 0000000000..d7efcec954
--- /dev/null
+++ b/doc/forum/Adding_files_with_wildcard_on_Mac_Yosemite/comment_1_e0784676aab3388254cdc9e7dd2d97e7._comment
@@ -0,0 +1,11 @@
+[[!comment format=mdwn
+ username="joey"
+ subject="""comment 1"""
+ date="2015-03-27T18:21:11Z"
+ content="""
+There is no difference between "git annex add *" and "git annex
+add list of files". The wildcard is expanded by your shell, not by
+git-annex.
+
+Post a transcript of the problem.
+"""]]
diff --git a/doc/forum/Adding_files_with_wildcard_on_Mac_Yosemite/comment_2_d9a7d2669d1524cab71e7e3139631f2a._comment b/doc/forum/Adding_files_with_wildcard_on_Mac_Yosemite/comment_2_d9a7d2669d1524cab71e7e3139631f2a._comment
new file mode 100644
index 0000000000..c6a410a657
--- /dev/null
+++ b/doc/forum/Adding_files_with_wildcard_on_Mac_Yosemite/comment_2_d9a7d2669d1524cab71e7e3139631f2a._comment
@@ -0,0 +1,55 @@
+[[!comment format=mdwn
+ username="https://www.google.com/accounts/o8/id?id=AItOawmFe2gzaJequ06hlza8N1-lFqQfwtFnw68"
+ nickname="Michael"
+ subject="Works in a subfolder, but not from root of site"
+ date="2015-03-27T19:23:06Z"
+ content="""
+So if I drill all the way down into a folder containing images, the git annex add and git commit are successful. When trying to do the same from the root of the site, I get the add messages, but nothing appears to be staged when committing.
+
+
+Here is the output when trying to add all images recursively from the root of the site
+
+    macbook-pro:domain.com mhoskison$ git annex add *.jpg
+    add webroot/HTML/images/a-expanded.jpg ok
+    add webroot/HTML/images/about-logo1.jpg ok
+    
+    add webroot/wp-content/uploads/2015/01/shutterstock_98870129-300x227.jpg ok
+    add webroot/wp-content/uploads/2015/01/shutterstock_98870129-940x198.jpg ok
+    add webroot/wp-content/uploads/2015/01/shutterstock_98870129.jpg ok
+    add webroot/wp-content/uploads/2015/01/smart-beauty-guide-logo.jpg ok
+    git-annex: *.jpg not found
+    macbook-pro:domain.com mhoskison$ git commit -m \"annex jpg wildcard\"
+    (recording state in git...)
+    On branch master
+    Your branch is up-to-date with 'origin/master'.
+    nothing added to commit but untracked files present
+    macbook-pro:domain.com mhoskison$ 
+
+
+Here is the successful output when I run the same commands from a subfolder
+
+    macbook-pro:04 mhoskison$ git annex add *.jpg
+    add april-150x150.jpg ok
+    add april.jpg ok
+    add images2-150x150.jpg ok
+    add images2.jpg ok
+    add p44242.tf_3B4F1611-AE82-BF8C-9E3190356674F495.Clear-and-Brilliant-logo-219x305_1_-150x150.jpg ok
+    add p44242.tf_3B4F1611-AE82-BF8C-9E3190356674F495.Clear-and-Brilliant-logo-219x305_1_-215x300.jpg ok
+    add p44242.tf_3B4F1611-AE82-BF8C-9E3190356674F495.Clear-and-Brilliant-logo-219x305_1_-219x198.jpg ok
+    add p44242.tf_3B4F1611-AE82-BF8C-9E3190356674F495.Clear-and-Brilliant-logo-219x305_1_.jpg ok
+    (recording state in git...)
+    macbook-pro:04 mhoskison$ git commit -m \"annex test from subfolder\"
+    [master 9859afd] annex test from subfolder
+     8 files changed, 8 insertions(+)
+     create mode 120000 webroot/wp-content/uploads/2014/04/april-150x150.jpg
+     create mode 120000 webroot/wp-content/uploads/2014/04/april.jpg
+     create mode 120000 webroot/wp-content/uploads/2014/04/images2-150x150.jpg
+     create mode 120000 webroot/wp-content/uploads/2014/04/images2.jpg
+     create mode 120000 webroot/wp-content/uploads/2014/04/p44242.tf_3B4F1611-AE82-BF8C-9E3190356674F495.Clear-and-Brilliant-logo-219x305_1_-
+    F8C-9E3190356674F495.Clear-and-Brilliant-logo-219x305_1_-150x150.jpg
+     create mode 120000 webroot/wp-content/uploads/2014/04/p44242.tf_3B4F1611-AE82-BF8C-9E3190356674F495.Clear-and-Brilliant-logo-219x305_1_-215x300.jpg
+     create mode 120000 webroot/wp-content/uploads/2014/04/p44242.tf_3B4F1611-AE82-BF8C-9E3190356674F495.Clear-and-Brilliant-logo-219x305_1_-219x198.jpg
+     create mode 120000 webroot/wp-content/uploads/2014/04/p44242.tf_3B4F1611-AE82-BF8C-9E3190356674F495.Clear-and-Brilliant-logo-219x305_1_.jpg
+
+
+"""]]
diff --git a/doc/forum/Adding_files_with_wildcard_on_Mac_Yosemite/comment_3_a0fdcd19954a38141b1dc2a51638ba30._comment b/doc/forum/Adding_files_with_wildcard_on_Mac_Yosemite/comment_3_a0fdcd19954a38141b1dc2a51638ba30._comment
new file mode 100644
index 0000000000..6af065cb54
--- /dev/null
+++ b/doc/forum/Adding_files_with_wildcard_on_Mac_Yosemite/comment_3_a0fdcd19954a38141b1dc2a51638ba30._comment
@@ -0,0 +1,24 @@
+[[!comment format=mdwn
+ username="joey"
+ subject="""comment 3"""
+ date="2015-03-27T19:25:26Z"
+ content="""
+The shell does not expand *.jpg to match files inside subdirectories.
+
+I suggest you read up on how shell wildcards work, and how to use them
+properly.
+
+There is a bit of a bug here, which is that the shell, when `*.jpg` does not
+expand to anything, passes the `"*.jpg"` parameter to git-annex un-expanded,
+which then passes it off to git ls-files, which then unexpectedly expands
+it to recursively match jpegs in subirectories. Only then does git-annex
+try to add a `*.jpg` file, and when it doesn't exist, it exits with an
+error, leaving the files it did add unstaged in git.
+
+To get out of that situation, just re-run git-annex add and actually list
+the files or directories to add, rather than a wildcard that won't expand
+to them..
+
+I'll see if I can fix the behavior here, it should just fail when given an
+un-expanded wildcard.
+"""]]
diff --git a/doc/forum/Adding_files_with_wildcard_on_Mac_Yosemite/comment_3_bc36d8f7b1d4d475018a236f71d06bb5._comment b/doc/forum/Adding_files_with_wildcard_on_Mac_Yosemite/comment_3_bc36d8f7b1d4d475018a236f71d06bb5._comment
new file mode 100644
index 0000000000..d04ab1c774
--- /dev/null
+++ b/doc/forum/Adding_files_with_wildcard_on_Mac_Yosemite/comment_3_bc36d8f7b1d4d475018a236f71d06bb5._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="https://www.google.com/accounts/o8/id?id=AItOawmFe2gzaJequ06hlza8N1-lFqQfwtFnw68"
+ nickname="Michael"
+ subject="Symlinks added"
+ date="2015-03-27T19:33:07Z"
+ content="""
+Git-annex does replace all jpg files with symlinks when I run the add command from the root directory. It's just when I try to commit and sync that no changes are detected.
+"""]]
diff --git a/doc/forum/Adding_files_with_wildcard_on_Mac_Yosemite/comment_5_a02c3cd804c2f6fd80d8839cc5c81af4._comment b/doc/forum/Adding_files_with_wildcard_on_Mac_Yosemite/comment_5_a02c3cd804c2f6fd80d8839cc5c81af4._comment
new file mode 100644
index 0000000000..14560b0231
--- /dev/null
+++ b/doc/forum/Adding_files_with_wildcard_on_Mac_Yosemite/comment_5_a02c3cd804c2f6fd80d8839cc5c81af4._comment
@@ -0,0 +1,12 @@
+[[!comment format=mdwn
+ username="joey"
+ subject="""comment 5"""
+ date="2015-03-27T20:42:21Z"
+ content="""
+I've fixed this; now you'll see:
+
+
    +git annex add *.jpg
+git-annex: *.jpg not found
+
    
+"""]]
diff --git a/doc/forum/Adding_files_with_wildcard_on_Mac_Yosemite/comment_6_703ecd8e1dfc5b5b58655e27c9db838a._comment b/doc/forum/Adding_files_with_wildcard_on_Mac_Yosemite/comment_6_703ecd8e1dfc5b5b58655e27c9db838a._comment
new file mode 100644
index 0000000000..30a6bd26fc
--- /dev/null
+++ b/doc/forum/Adding_files_with_wildcard_on_Mac_Yosemite/comment_6_703ecd8e1dfc5b5b58655e27c9db838a._comment
@@ -0,0 +1,9 @@
+[[!comment format=mdwn
+ username="clacke"
+ subject="regression"
+ date="2015-04-22T11:39:19Z"
+ content="""
+The fix is a regression compared to previous behavior.
+
+Being able to do `git annex add '*'.{JPG,jp{,e}g,AVI,avi,MOV,mov,3GP,3gp,mp4,tif,pdf,PDF,doc,pps,bmp,png,mp{,e}g,MP{,E}G,wav,WAV,nef,NEF,thm,THM,key.gz,ogg,OGG,ppt,GIF,gif,m4a}` in the root of my binary files archive to catch any newly copied files was a real feature. This worked until recently. I was just about to add a bug report stating the same problem as the OP for the OSX homebrew version, but I verified with the latest Linux standalone and there wildcards were completely turned off.
+"""]]
diff --git a/doc/forum/Adding_files_with_wildcard_on_Mac_Yosemite/comment_7_dbe40fef2ba65cc0f1c20f41f2daab4d._comment b/doc/forum/Adding_files_with_wildcard_on_Mac_Yosemite/comment_7_dbe40fef2ba65cc0f1c20f41f2daab4d._comment
new file mode 100644
index 0000000000..47b00a5f06
--- /dev/null
+++ b/doc/forum/Adding_files_with_wildcard_on_Mac_Yosemite/comment_7_dbe40fef2ba65cc0f1c20f41f2daab4d._comment
@@ -0,0 +1,11 @@
+[[!comment format=mdwn
+ username="clacke"
+ subject="workaround"
+ date="2015-04-22T13:31:10Z"
+ content="""
+This is a workaround to the missing functionality, generating an `--include` expression:
+
+[[!format sh \"\"\"
+git annex add $(printf -- '--include=*.%s --or ' jp{,e}g avi mov 3gp mp4 tif pdf doc pps bmp png mp{,e}g wav nef thm key.gz ogg ppt gif) --include='*.m4a'
+\"\"\"]]
+"""]]
diff --git a/doc/forum/Adding_files_with_wildcard_on_Mac_Yosemite/comment_8_d8620ce7b3dbb81c0d3d0b09ded1deb0._comment b/doc/forum/Adding_files_with_wildcard_on_Mac_Yosemite/comment_8_d8620ce7b3dbb81c0d3d0b09ded1deb0._comment
new file mode 100644
index 0000000000..ac95deb7fe
--- /dev/null
+++ b/doc/forum/Adding_files_with_wildcard_on_Mac_Yosemite/comment_8_d8620ce7b3dbb81c0d3d0b09ded1deb0._comment
@@ -0,0 +1,18 @@
+[[!comment format=mdwn
+ username="joey"
+ subject="""comment 8"""
+ date="2015-04-24T17:26:01Z"
+ content="""
+The old behavior was accidental, and was never documented. When you use
+such undocumented behaviors, you're taking the risk of bugfixes breaking
+things. It's not fair to call that a "regression". If I had to worry about
+every bugfix breaking users who relied on the old buggy behavior in some
+way, I could just stop working now.
+
+It might be reasonable to add the "expand wildcards rather than letting the
+shell do it" feature, as an option. Of course, it would need to be tested
+for every git-annex command and problems like the one that caused this bug
+to be noticed in the first place dealt with, for every git-annex command.
+Using --include and --exclude, which already work seems pretty reasonable
+instead.
+"""]]
diff --git a/doc/forum/Adding_files_with_wildcard_on_Mac_Yosemite/comment_9_6a43f52449c4a38a986772ec9d65f9d5._comment b/doc/forum/Adding_files_with_wildcard_on_Mac_Yosemite/comment_9_6a43f52449c4a38a986772ec9d65f9d5._comment
new file mode 100644
index 0000000000..210ece464f
--- /dev/null
+++ b/doc/forum/Adding_files_with_wildcard_on_Mac_Yosemite/comment_9_6a43f52449c4a38a986772ec9d65f9d5._comment
@@ -0,0 +1,7 @@
+[[!comment format=mdwn
+ username="clacke"
+ subject="not regression"
+ date="2015-04-27T12:13:12Z"
+ content="""
+Fair enough. And I have got over my initial frustration with the change in behavior. The workaround works.
+"""]]
diff --git a/doc/forum/Adding_normal___40__non-annexed__41___files_in_Windows.mdwn b/doc/forum/Adding_normal___40__non-annexed__41___files_in_Windows.mdwn
new file mode 100644
index 0000000000..b9d118cd81
--- /dev/null
+++ b/doc/forum/Adding_normal___40__non-annexed__41___files_in_Windows.mdwn
@@ -0,0 +1,31 @@
+Hi,
+I'm working on a game project (Unity3D) which I want to use git-annex for all those binary assets,
+
+but I still want to be able to commit normal code files in git,
+
+is there a way to do that?
+
+below is what I tried (but failed) to do.
+
+Thanks.
+
+Chris
+
+
+
+    C:\Users\Chris\Projects\testing>git annex init "testing"
+    init testing
+      Detected a filesystem without fifo support.
+    
+      Disabling ssh connection caching.
+    
+      Detected a crippled filesystem.
+    
+      Enabling direct mode.
+    ok
+    (recording state in git...)
+    
+    C:\Users\Chris\Projects\testing>echo test > testing.txt
+    
+    C:\Users\Chris\Projects\testing>git add testing.txt
+    fatal: This operation must be run in a work tree
diff --git a/doc/forum/Adding_normal___40__non-annexed__41___files_in_Windows/comment_1_e40e129fbdae305ad48f810c578fa69f._comment b/doc/forum/Adding_normal___40__non-annexed__41___files_in_Windows/comment_1_e40e129fbdae305ad48f810c578fa69f._comment
new file mode 100644
index 0000000000..e2c1df6bef
--- /dev/null
+++ b/doc/forum/Adding_normal___40__non-annexed__41___files_in_Windows/comment_1_e40e129fbdae305ad48f810c578fa69f._comment
@@ -0,0 +1,9 @@
+[[!comment format=mdwn
+ username="https://me.yahoo.com/a/5j.FKrMpxZS.luSB.5ahyosMU6RcaYq2#74c60"
+ nickname="Jarno"
+ subject="Proxy command"
+ date="2015-06-26T17:32:42Z"
+ content="""
+Have you tried the `git annex proxy --` command? Another way might be to just bypass the annex checks with `git -c core.bare=false`.
+Check  for details.
+"""]]
diff --git a/doc/forum/Adding_normal___40__non-annexed__41___files_in_Windows/comment_2_3f2556b0a0cd8b7523aed1bea1be8484._comment b/doc/forum/Adding_normal___40__non-annexed__41___files_in_Windows/comment_2_3f2556b0a0cd8b7523aed1bea1be8484._comment
new file mode 100644
index 0000000000..96bc4f42b7
--- /dev/null
+++ b/doc/forum/Adding_normal___40__non-annexed__41___files_in_Windows/comment_2_3f2556b0a0cd8b7523aed1bea1be8484._comment
@@ -0,0 +1,29 @@
+[[!comment format=mdwn
+ username="chris.forum.email@596f849a9d27c1f0bcdd653d1d2dc992f21ce603"
+ nickname="chris.forum.email"
+ subject="comment 2"
+ date="2015-06-27T11:23:05Z"
+ content="""
+if I use `git annex proxy` I can't see the changes:
+
+    C:\Users\Chris\Projects\testing>git annex proxy -- git status
+    On branch annex/direct/master
+    nothing to commit, working directory clean
+
+and I noticed that I'm on a branch called `annex/direct/master`, not the `master` I'd expect...
+
+if I use `git -c core.bare=false`, I get the normal output:
+
+    C:\Users\Chris\Projects\testing>git -c core.bare=false status
+    On branch annex/direct/master
+    Untracked files:
+      (use \"git add ...\" to include in what will be committed)
+    
+            test.txt
+    
+    nothing added to commit but untracked files present (use \"git add\" to track)
+
+but I'm still on a \"wrong\" branch? what is this `annex/direct/master` branch and am I supposed to commit and push this branch?
+
+Thanks, Chris
+"""]]
diff --git a/doc/forum/Adding_normal___40__non-annexed__41___files_in_Windows/comment_3_d50b99393009ba4adf2040f1d5d5078b._comment b/doc/forum/Adding_normal___40__non-annexed__41___files_in_Windows/comment_3_d50b99393009ba4adf2040f1d5d5078b._comment
new file mode 100644
index 0000000000..eaef967b41
--- /dev/null
+++ b/doc/forum/Adding_normal___40__non-annexed__41___files_in_Windows/comment_3_d50b99393009ba4adf2040f1d5d5078b._comment
@@ -0,0 +1,23 @@
+[[!comment format=mdwn
+ username="joey"
+ subject="""comment 3"""
+ date="2015-07-06T18:41:35Z"
+ content="""
+One nice way to do this is to use the annex.largefiles configuration
+setting. With a recent version of git-anne (5.20150409 or newer),
+adding a file that doesn't match annex.largefiles will cause it to be
+checked into git, rather than annexed.
+
+So, for example, `git -c annex.largefiles='exclude=*' annex add foo`
+will temporarily configure it to not annex any files, and add file foo to
+git. Or you can set annex.largefiles more permanantly to match specific
+file extensions.
+
+As to the annex/direct/master branch, this branch will get merged into the
+master branch on a push, so don't worry about it.
+
+(The `-c core.bare=false` approach will also work, just run regular git
+commands to add the file and commit. But, it's best to avoid this approach,
+because it's very easy to shoot yourself in the foot, by eg using
+`git commit -a` with it, which will check *all* file contents into git.)
+"""]]
diff --git a/doc/forum/Adding_public_key_to_hybrid_encrypted_repo_causes___34__remote_already_exists__34___error.mdwn b/doc/forum/Adding_public_key_to_hybrid_encrypted_repo_causes___34__remote_already_exists__34___error.mdwn
new file mode 100644
index 0000000000..8032584f1e
--- /dev/null
+++ b/doc/forum/Adding_public_key_to_hybrid_encrypted_repo_causes___34__remote_already_exists__34___error.mdwn
@@ -0,0 +1,5 @@
+Hello, I'm running git-annex version: 5.20151208-1build1.  I am trying to add a public key to my git annex repo but it tells me the repo already exists.  I worked around this by marking the repo as dead and re-enabling but I feel like this isn't the correct course of action.  Is there another way to avoid this error?
+
+    $ git annex enableremote annex.repo keyid+=920D7A56
+    enableremote annex.repo (encryption update) (hybrid cipher with gpg keys 97A273201BA73A76 A50E9160920D7A56) fatal: remote annex.repo already exists.
+    git-annex: git [Param "remote",Param "add",Param "annex.repo",Param "gcrypt::ssh://ben@annex/dataset/annex/share"] failed
diff --git a/doc/forum/Adding_public_key_to_hybrid_encrypted_repo_causes___34__remote_already_exists__34___error/comment_1_fe2f0e7a4688865faf21c2c0467852b7._comment b/doc/forum/Adding_public_key_to_hybrid_encrypted_repo_causes___34__remote_already_exists__34___error/comment_1_fe2f0e7a4688865faf21c2c0467852b7._comment
new file mode 100644
index 0000000000..be3a213144
--- /dev/null
+++ b/doc/forum/Adding_public_key_to_hybrid_encrypted_repo_causes___34__remote_already_exists__34___error/comment_1_fe2f0e7a4688865faf21c2c0467852b7._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="joey"
+ subject="""comment 1"""
+ date="2018-03-02T19:04:08Z"
+ content="""
+Your version of git-annex is impressively out of date.
+When I try this with the current version, it works ok.
+"""]]
diff --git a/doc/forum/Adding_selected_big_binaries_recursively.mdwn b/doc/forum/Adding_selected_big_binaries_recursively.mdwn
new file mode 100644
index 0000000000..b870a39fac
--- /dev/null
+++ b/doc/forum/Adding_selected_big_binaries_recursively.mdwn
@@ -0,0 +1,12 @@
+I have a very large stack of files, with many large binaries.
+Unless I am told its a bad idea, I want to initialize a git annex archive with the big files 'git annex added' ( symlinked to hidden annex) and the rest of the files 'git added'
+
+I tried-
+git annex add "*.adi"
+find . -name "*.adi" | git annex add
+find . -name "*.adi" | xargs git annex add
+git annex add robots/\*.adi
+
+
+Whats a good ( correct) way to do only the *.adi files recursively?
+( its Friday PST here- will return Monday)
diff --git a/doc/forum/Adding_selected_big_binaries_recursively/comment_1_6ff6031a5a44caf17f36c3569cca5512._comment b/doc/forum/Adding_selected_big_binaries_recursively/comment_1_6ff6031a5a44caf17f36c3569cca5512._comment
new file mode 100644
index 0000000000..abfbbad5c2
--- /dev/null
+++ b/doc/forum/Adding_selected_big_binaries_recursively/comment_1_6ff6031a5a44caf17f36c3569cca5512._comment
@@ -0,0 +1,23 @@
+[[!comment format=mdwn
+ username="joey"
+ subject="""comment 1"""
+ date="2016-07-30T16:36:51Z"
+ content="""
+It's fine to annex the big files and store the small files in git in
+the usual way.
+
+The `find | xargs` approach should work.
+
+You can also use the [[git-annex-matching-options]], eg:
+
+	git annex add --include='*.adi'
+
+Or:
+
+	git annex add --largerthan=1mb
+
+
+You can also configure git-annex to know which files you consider
+large, so that `git annex add` will annex the large ones and add 
+the rest to git not the annex. See [[tips/largefiles]]
+"""]]
diff --git a/doc/forum/Adding_selected_big_binaries_recursively/comment_2_872bef6902ddb362b62d5e9ec1afe864._comment b/doc/forum/Adding_selected_big_binaries_recursively/comment_2_872bef6902ddb362b62d5e9ec1afe864._comment
new file mode 100644
index 0000000000..3950b19577
--- /dev/null
+++ b/doc/forum/Adding_selected_big_binaries_recursively/comment_2_872bef6902ddb362b62d5e9ec1afe864._comment
@@ -0,0 +1,26 @@
+[[!comment format=mdwn
+ username="JohnFisher"
+ subject="didn't work for me, perhaps I did the attributes wrong?"
+ date="2016-08-04T20:43:15Z"
+ content="""
+I created .gitattributes in the root of my repo:
+jfisher@buildos1:~/annex$ cat .gitattributes
+* annex.largefiles=(largerthan=100mb)
+jfisher@buildos1:~/annex$ git config -l
+user.email=jfisher@intouchhealth.com
+user.name=John Fisher
+core.repositoryformatversion=0
+core.filemode=true
+core.bare=false
+core.logallrefupdates=true
+annex.uuid=75d91aa9-f5fc-45b8-b62c-e6050ed7c3d5
+annex.version=5
+
+where I intend for all users of the repo to automatically add all files larger than 100Mb as annexed and add all other files as conventional git.
+
+I ran 
+git annex add
+and small text files were staged as links, as well as a large file
+
+Comment? Ideas?
+"""]]
diff --git a/doc/forum/Adding_selected_big_binaries_recursively/comment_3_4be5ca7a22db19a3152b54bee943f58c._comment b/doc/forum/Adding_selected_big_binaries_recursively/comment_3_4be5ca7a22db19a3152b54bee943f58c._comment
new file mode 100644
index 0000000000..c0f9a03e78
--- /dev/null
+++ b/doc/forum/Adding_selected_big_binaries_recursively/comment_3_4be5ca7a22db19a3152b54bee943f58c._comment
@@ -0,0 +1,19 @@
+[[!comment format=mdwn
+ username="JohnFisher"
+ subject="re-format for readability"
+ date="2016-08-04T20:48:55Z"
+ content="""
+    jfisher@buildos1:~/annex$ cat .gitattributes
+    * annex.largefiles=(largerthan=100mb)
+
+    jfisher@buildos1:~/annex$ git config -l
+    user.email=jfisher@intouchhealth.com
+    user.name=John Fisher
+    core.repositoryformatversion=0
+    core.filemode=true
+    core.bare=false
+    core.logallrefupdates=true
+    annex.uuid=75d91aa9-f5fc-45b8-b62c-e6050ed7c3d5
+annex.version=5
+
+"""]]
diff --git a/doc/forum/Adding_selected_big_binaries_recursively/comment_4_83b62738f79c434f5a2cacc9316e5d2b._comment b/doc/forum/Adding_selected_big_binaries_recursively/comment_4_83b62738f79c434f5a2cacc9316e5d2b._comment
new file mode 100644
index 0000000000..321d1c7c46
--- /dev/null
+++ b/doc/forum/Adding_selected_big_binaries_recursively/comment_4_83b62738f79c434f5a2cacc9316e5d2b._comment
@@ -0,0 +1,9 @@
+[[!comment format=mdwn
+ username="joey"
+ subject="""comment 4"""
+ date="2016-10-05T21:01:38Z"
+ content="""
+That .gitattributes work when I try it here. Perhaps your version of
+git-annex is too old to support it. Configuring annex.largefiles via
+.gitattributes was first supported in version 6.20160211.
+"""]]
diff --git a/doc/forum/Adding_the_same_content_under_different_file_names.mdwn b/doc/forum/Adding_the_same_content_under_different_file_names.mdwn
new file mode 100644
index 0000000000..57c31199ed
--- /dev/null
+++ b/doc/forum/Adding_the_same_content_under_different_file_names.mdwn
@@ -0,0 +1,17 @@
+Working on the Baobáxia project, we have this interesting problem: If some content is added twice with different file names, it will appear in the git-annex repository as two symlinks pointing to the same file in the annex.
+
+So far so good.
+
+But this means that if I want to delete a file, I have two cases:
+
+* If this content was added only once, I need to drop the binary content with git annex drop and afterwards remove the symlink with git rm.
+* If this content exists under more than one file name, I need to remove the symlink with git rm only; I should *not* drop the content.
+
+So the question is: How do I know that, apart from searching through the whole repository? Is there a more efficient way? After going through the manual I can't find it, but I may have overlooked something.
+
+Best regards and TIA for any response
+Carsten Agger
+
+
+Note: This question references bug #191 in the Baobáxia project, https://github.com/RedeMocambos/baobaxia/issues/191
+
diff --git a/doc/forum/Adding_the_same_content_under_different_file_names/comment_1_7c659668f62577612eaf8f463383a185._comment b/doc/forum/Adding_the_same_content_under_different_file_names/comment_1_7c659668f62577612eaf8f463383a185._comment
new file mode 100644
index 0000000000..e71a6901ed
--- /dev/null
+++ b/doc/forum/Adding_the_same_content_under_different_file_names/comment_1_7c659668f62577612eaf8f463383a185._comment
@@ -0,0 +1,16 @@
+[[!comment format=mdwn
+ username="ghen1"
+ avatar="http://cdn.libravatar.org/avatar/efd0e92b6198291138f0cd7aedbf86b6"
+ subject="comment 1"
+ date="2017-01-07T15:41:36Z"
+ content="""
+You don't need to keep track of links. Just delete the links, then run:
+
+    git-annex unused
+
+This will find any content that isn't linked to. You can then run *git-annex dropunused* to free up space.
+
+See these man pages:
    
+
    
+
+"""]]
diff --git a/doc/forum/Adding_the_same_content_under_different_file_names/comment_2_217052fe47822f73ffc765afd6386a7e._comment b/doc/forum/Adding_the_same_content_under_different_file_names/comment_2_217052fe47822f73ffc765afd6386a7e._comment
new file mode 100644
index 0000000000..c2fe6e94cf
--- /dev/null
+++ b/doc/forum/Adding_the_same_content_under_different_file_names/comment_2_217052fe47822f73ffc765afd6386a7e._comment
@@ -0,0 +1,21 @@
+[[!comment format=mdwn
+ username="joey"
+ subject="""comment 2"""
+ date="2017-01-31T16:35:15Z"
+ content="""
+This deduplication is generally considered a feature.
+(If you don't want it, you can eg use the WORM backend.)
+
+The `git annex unused` command can be used in this case. It looks through
+all files (in all branches) in the repo and finds content that no files
+are using, which can then be dropped.
+
+So the question is, if you've been using `git annex drop` before deleting a
+file, how do you know when you need to not do that, and 
+instead use `git annex unused` after deleting a file? Well, my suggestion
+is to just never get in the habit of `git annex drop` before deleting a
+file. Reserve using `git annex drop` for times when you are getting low on
+disk space and need to free some up by dropping content from local
+storage; don't try to manually manage which content is stored in the local
+repository as long as you have enough free disk space.
+"""]]
diff --git a/doc/forum/Advice_needed__58___git-annex_slows_down_my_macbook.mdwn b/doc/forum/Advice_needed__58___git-annex_slows_down_my_macbook.mdwn
new file mode 100644
index 0000000000..459e738382
--- /dev/null
+++ b/doc/forum/Advice_needed__58___git-annex_slows_down_my_macbook.mdwn
@@ -0,0 +1,21 @@
+I need some advice from more experienced users. 
+
+I've created four repositories on my macbook and paired them with repos on my ubuntu desktop. They are all created with the webapp so they might have some configurations that are not optimal for me.
+
+In fact, when I start my macbook it is unusual for more than an hour because git-annex is reading and writing to and from the disk in the background. 
+
+I don't change much in three of the four repos, only one repo contains my daily work. 
+
+What configuration would enhance my situation?
+
+The four repos are called:
+
+- movies
+- pictures
+- music
+- documents (many changes each day)
+
+I have also problems to switch to the documents repo in the webapp.
+
+TIA
+juh
diff --git a/doc/forum/Advice_needed__58___git-annex_slows_down_my_macbook/comment_1_af8ab0a47cd379fcb1445e50782ad086._comment b/doc/forum/Advice_needed__58___git-annex_slows_down_my_macbook/comment_1_af8ab0a47cd379fcb1445e50782ad086._comment
new file mode 100644
index 0000000000..a7cb701b6f
--- /dev/null
+++ b/doc/forum/Advice_needed__58___git-annex_slows_down_my_macbook/comment_1_af8ab0a47cd379fcb1445e50782ad086._comment
@@ -0,0 +1,12 @@
+[[!comment format=mdwn
+ username="https://www.google.com/accounts/o8/id?id=AItOawmTIL7ubr5opWM69Q5VtCxuxC2H0SSnzic"
+ nickname="Petr"
+ subject="Where are your repositories located?"
+ date="2014-01-14T08:58:31Z"
+ content="""
+First - Where are the repos actually located ~/Desktop or ~/annex/? Other?
+ 
+I can confirm, that like two weeks back I have had similar issue using git-annex-assistant with other synced repos on local LAN and at Box.com. The git-annex processes actually overloaded the CPU and I was forced to kill annex processes to get over.
+
+I also found myself that NFS mounted shares, linked by symlink to home directory, can slowdown mac finder app as well if NFS has slight delays in response.
+"""]]
diff --git a/doc/forum/Advice_needed__58___git-annex_slows_down_my_macbook/comment_2_a7202bcbdda36a3801833d2432db1965._comment b/doc/forum/Advice_needed__58___git-annex_slows_down_my_macbook/comment_2_a7202bcbdda36a3801833d2432db1965._comment
new file mode 100644
index 0000000000..478d2804f6
--- /dev/null
+++ b/doc/forum/Advice_needed__58___git-annex_slows_down_my_macbook/comment_2_a7202bcbdda36a3801833d2432db1965._comment
@@ -0,0 +1,25 @@
+[[!comment format=mdwn
+ username="https://www.google.com/accounts/o8/id?id=AItOawnX1msQxnLoSeu7q-i-c9BWghonsN7Qmns"
+ nickname="Jan Ulrich"
+ subject="comment 2"
+ date="2014-01-14T10:23:21Z"
+ content="""
+All repositories are located in my home folders:
+~/Pictures
+~/Movies
+~/Music
+~/Documents
+
+On my Ubuntu desktop machine and my macbook. They are connected in my local network via WLAN. 
+
+I use different git annex versions on these two machines. Maybe this causes problems.
+
+On Ubuntu I have 
+Version: 4.20130815
+Build flags: Assistant Webapp Pairing Testsuite S3 WebDAV Inotify DBus XMPP 
+
+On my macbook:
+Version: 5-20140106-gcb3351b
+
+juh
+"""]]
diff --git a/doc/forum/All_repos_on_same_filesystem.mdwn b/doc/forum/All_repos_on_same_filesystem.mdwn
new file mode 100644
index 0000000000..ee61ab7254
--- /dev/null
+++ b/doc/forum/All_repos_on_same_filesystem.mdwn
@@ -0,0 +1,14 @@
+Hi,
+ I am looking to find out the best way to use git annex, when all repos will live on the same filesystem, using a central repo.
+
+What I have so far is, after creating the main repo (mainrepo).
+
+Create clones via: git clone -shared mainrepo clonerepo
+
+Then  use "git annex add" and "git add". When it comes to making the data accessible to the mainrepo, it is a little unclear to me. 
+There is a lot of disjoint information regarding pull/pushing content and which directions use hardlinks vs copies etc. So I 
+was hoping that someone could fill me in on best practices.
+
+Thanks in advance!
+
+Pete
diff --git a/doc/forum/All_repos_on_same_filesystem/comment_1_a6666a89f62dc7e2d40e028782e5a25a._comment b/doc/forum/All_repos_on_same_filesystem/comment_1_a6666a89f62dc7e2d40e028782e5a25a._comment
new file mode 100644
index 0000000000..9074c4e16c
--- /dev/null
+++ b/doc/forum/All_repos_on_same_filesystem/comment_1_a6666a89f62dc7e2d40e028782e5a25a._comment
@@ -0,0 +1,14 @@
+[[!comment format=mdwn
+ username="joey"
+ subject="""comment 1"""
+ date="2015-12-19T17:55:40Z"
+ content="""
+When you clone a repository with --shared, all git-annex commands
+that transfer data into or out of the repository will use hardlinks
+whenever possible.
+
+(But, before version 5.20150916, it didn't manage to use them all the
+time.)
+
+To get started, just use `git annex sync --content`
+"""]]
diff --git a/doc/forum/Alternative_to_XMPP_on_Windows__63__.mdwn b/doc/forum/Alternative_to_XMPP_on_Windows__63__.mdwn
new file mode 100644
index 0000000000..7edd703e01
--- /dev/null
+++ b/doc/forum/Alternative_to_XMPP_on_Windows__63__.mdwn
@@ -0,0 +1 @@
+Is there an alternative to XMPP to sync Windows machines?
diff --git a/doc/forum/Alternative_to_XMPP_on_Windows__63__/comment_1_4084abf5b26fcc92e577eea34c331a15._comment b/doc/forum/Alternative_to_XMPP_on_Windows__63__/comment_1_4084abf5b26fcc92e577eea34c331a15._comment
new file mode 100644
index 0000000000..39ab7a8a67
--- /dev/null
+++ b/doc/forum/Alternative_to_XMPP_on_Windows__63__/comment_1_4084abf5b26fcc92e577eea34c331a15._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="joey"
+ subject="""comment 1"""
+ date="2015-02-04T17:16:43Z"
+ content="""
+The best alternative currently is to use a server with git-annex installed,
+which the windows instance can then communicate with.
+"""]]
diff --git a/doc/forum/Alternative_to_git_annex_copy_which_is_as_fast_as_git_annex_get.mdwn b/doc/forum/Alternative_to_git_annex_copy_which_is_as_fast_as_git_annex_get.mdwn
new file mode 100644
index 0000000000..61d0ea699e
--- /dev/null
+++ b/doc/forum/Alternative_to_git_annex_copy_which_is_as_fast_as_git_annex_get.mdwn
@@ -0,0 +1,15 @@
+I have a big git-annex repository. If I add a few files on my laptop I want to sync it with my server which consists of one bare git-annex repository (server_bare) and one regular git-annex repository (server).
+
+I run
+
+git annex sync
+git annex copy * --to server
+
+The second command is really slow since it runs git annex copy on every single file that I have locally (it sometimes takes an hour to complete). If I would instead run
+
+git annex sync
+git annex get .
+
+This is usually quite fast, but it is not so easy to do since I don't keep an SSH port open on my laptop.
+
+My question: Is there any command that I can run on my laptop which basically invokes "git annex get ." from the server?
diff --git a/doc/forum/Alternative_to_git_annex_copy_which_is_as_fast_as_git_annex_get/comment_1_7f62ed07749308a46f0028a9586a10a4._comment b/doc/forum/Alternative_to_git_annex_copy_which_is_as_fast_as_git_annex_get/comment_1_7f62ed07749308a46f0028a9586a10a4._comment
new file mode 100644
index 0000000000..b9aabb3098
--- /dev/null
+++ b/doc/forum/Alternative_to_git_annex_copy_which_is_as_fast_as_git_annex_get/comment_1_7f62ed07749308a46f0028a9586a10a4._comment
@@ -0,0 +1,13 @@
+[[!comment format=mdwn
+ username="joey"
+ subject="""comment 1"""
+ date="2017-02-02T20:33:31Z"
+ content="""
+Try `git annex copy --fast --to server`
+
+That avoids checking if the server still has every file, and only sends
+files it knows the server does not have. If you've just synced with the
+server, it knows what's on the server, so this can't go wrong.
+
+That'll be almost as fast as `git annex get`
+"""]]
diff --git a/doc/forum/Alternative_to_git_annex_copy_which_is_as_fast_as_git_annex_get/comment_2_a0c58c65091f36b426e016c700ec2f1c._comment b/doc/forum/Alternative_to_git_annex_copy_which_is_as_fast_as_git_annex_get/comment_2_a0c58c65091f36b426e016c700ec2f1c._comment
new file mode 100644
index 0000000000..60239f93a9
--- /dev/null
+++ b/doc/forum/Alternative_to_git_annex_copy_which_is_as_fast_as_git_annex_get/comment_2_a0c58c65091f36b426e016c700ec2f1c._comment
@@ -0,0 +1,9 @@
+[[!comment format=mdwn
+ username="kristianrumberg@85de814abe9398ab5573b77bb8a7a80ceaf3b9e1"
+ nickname="kristianrumberg"
+ avatar="http://cdn.libravatar.org/avatar/bb8d029db59da98a0c5e0053b1edc459"
+ subject="comment 2"
+ date="2017-02-05T17:02:10Z"
+ content="""
+Thanks :) It indeed speeds it up a lot
+"""]]
diff --git a/doc/forum/Alternative_to_git_annex_copy_which_is_as_fast_as_git_annex_get/comment_3_102047d2676af0bc62791fa0f18ac49f._comment b/doc/forum/Alternative_to_git_annex_copy_which_is_as_fast_as_git_annex_get/comment_3_102047d2676af0bc62791fa0f18ac49f._comment
new file mode 100644
index 0000000000..bdd7907796
--- /dev/null
+++ b/doc/forum/Alternative_to_git_annex_copy_which_is_as_fast_as_git_annex_get/comment_3_102047d2676af0bc62791fa0f18ac49f._comment
@@ -0,0 +1,25 @@
+[[!comment format=mdwn
+ username="kanak@3c4f6e7d832d88751c617b25bdbac896417eb93b"
+ nickname="kanak"
+ avatar="http://cdn.libravatar.org/avatar/708121dfec06e554300b2a3a73a26818"
+ subject="get appears to be doing less work?"
+ date="2017-08-22T11:55:41Z"
+ content="""
+I'm running v6.20170519 on Fedora. Syncing between my desktop and my external drive (USB3).
+
+When I do the following:
+
+1. desktop$ git annex sync usbdrive
+2. desktop$ git annex copy --fast --to usbdrive
+
+The output shows every single file in the respository followed quickly by an OK. Output-wise it's no different than if i omitted the --fast flag. 
+
+However, when I do the following, 
+
+1. usbdrive$ git annex sync desktop
+2. usbdrive$ git annex get .
+
+The only output is the files being copied over. It also appears to be faster for me than a copy.
+
+I feel like i'm missing something basic or some step? Is copy --fast doing the same amount of work as get but just displaying more output?
+"""]]
diff --git a/doc/forum/Alternative_to_git_annex_copy_which_is_as_fast_as_git_annex_get/comment_4_717078ebeb3b69dd9c52c8abe457824a._comment b/doc/forum/Alternative_to_git_annex_copy_which_is_as_fast_as_git_annex_get/comment_4_717078ebeb3b69dd9c52c8abe457824a._comment
new file mode 100644
index 0000000000..8145210a70
--- /dev/null
+++ b/doc/forum/Alternative_to_git_annex_copy_which_is_as_fast_as_git_annex_get/comment_4_717078ebeb3b69dd9c52c8abe457824a._comment
@@ -0,0 +1,16 @@
+[[!comment format=mdwn
+ username="kanak@3c4f6e7d832d88751c617b25bdbac896417eb93b"
+ nickname="kanak"
+ avatar="http://cdn.libravatar.org/avatar/708121dfec06e554300b2a3a73a26818"
+ subject="get appears to be doing less work? (Part 2)"
+ date="2017-08-22T12:20:56Z"
+ content="""
+I tried git annex copy --to desktop from my laptop:
+
+without --fast, I see a \"(checking desktop)\" line
+with --fast, I only see the lines corresponding to transfer between files.
+
+Given that I don't see a \"checking\" line when I do this with my usb, I assume that it's not actually checking. However, I'm puzzled by the difference in output between the USB and the SSH scenarios.
+
+I still feel like I have a basic misunderstanding here, and I apologize in advance.
+"""]]
diff --git a/doc/forum/Alternative_to_git_annex_copy_which_is_as_fast_as_git_annex_get/comment_5_0630abbcb27e1f9dc5e9269b09a99b09._comment b/doc/forum/Alternative_to_git_annex_copy_which_is_as_fast_as_git_annex_get/comment_5_0630abbcb27e1f9dc5e9269b09a99b09._comment
new file mode 100644
index 0000000000..ad4043f3b5
--- /dev/null
+++ b/doc/forum/Alternative_to_git_annex_copy_which_is_as_fast_as_git_annex_get/comment_5_0630abbcb27e1f9dc5e9269b09a99b09._comment
@@ -0,0 +1,13 @@
+[[!comment format=mdwn
+ username="joey"
+ subject="""comment 5"""
+ date="2017-09-29T20:23:21Z"
+ content="""
+The behavior is different when the remote repository is on the USB drive,
+because git-annex knows it can quickly check if the remote really contains
+a file, and so it always checks it, even with --fast.
+
+I'm not sure if you're actually seeing a significant speed difference, or
+are psycholicically feeling it's slower since it outputs more. Perhaps you
+should time it.
+"""]]
diff --git a/doc/forum/Always_sync_all_content_to_remote.mdwn b/doc/forum/Always_sync_all_content_to_remote.mdwn
new file mode 100644
index 0000000000..fa6cc08719
--- /dev/null
+++ b/doc/forum/Always_sync_all_content_to_remote.mdwn
@@ -0,0 +1,5 @@
+I am trying to set up two repositories, one on my laptop and one on my server, and then have "git annex sync" always push everything on the laptop to the server.
+
+So far, I have created repositories on the laptop and on the server and then added the server as an ssh remote on the laptop. At the root of the repository on the server, I ran "git annex group . backup" and "git annex wanted . standard". At the root of the repository on the laptop, I ran "git annex group . manual" and "git annex wanted . standard". When I run "git annex sync" or "git annex sync --content" on the laptop now, it seems to only copy the symlink information to the server but not the actual data. When I go to the server and run git annex sync, it updates the working tree symlinks but does not have the content data they point to. If I use "git annex copy file --to server" then the symlink created when I ran "git annex sync" on the server points to the correct file. Do I not have the correct content preferences set up? Or do I need to alias "git annex sync --content" to "git annex sync --content && git annex copy . --to server" on the laptop?
+
+I want to be able to drop large amounts of data from the laptop and get it back if needed but always have it available on the server any way without having to push it explicitly. I would think that this would be a fairly common use case. Maybe I have missed something in the documentation? I have not set up the laptop as a remote on the server (I'd prefer not to -- I only plan to push/pull from the laptop, not push/pull from the server).
diff --git a/doc/forum/Always_sync_all_content_to_remote/comment_1_241c19cad6249fbd0832c76028c8b373._comment b/doc/forum/Always_sync_all_content_to_remote/comment_1_241c19cad6249fbd0832c76028c8b373._comment
new file mode 100644
index 0000000000..3ceecdf8de
--- /dev/null
+++ b/doc/forum/Always_sync_all_content_to_remote/comment_1_241c19cad6249fbd0832c76028c8b373._comment
@@ -0,0 +1,11 @@
+[[!comment format=mdwn
+ username="joey"
+ subject="""comment 1"""
+ date="2015-05-30T15:33:13Z"
+ content="""
+The configuration you describe sounds right and should work.
+
+Are you sure that the laptop had seen the changed configuration from the
+server when `git annex sync --content` didn't send files to the server?
+Ie, did you try running it a second time?
+"""]]
diff --git a/doc/forum/Always_sync_all_content_to_remote/comment_4_3eda671cea0f8869f1aac0194cae7146._comment b/doc/forum/Always_sync_all_content_to_remote/comment_4_3eda671cea0f8869f1aac0194cae7146._comment
new file mode 100644
index 0000000000..2a2c02d52d
--- /dev/null
+++ b/doc/forum/Always_sync_all_content_to_remote/comment_4_3eda671cea0f8869f1aac0194cae7146._comment
@@ -0,0 +1,34 @@
+[[!comment format=mdwn
+ username="wsha.code+ga@b38779424f41c5701bbe5937340be43ff1474b2d"
+ nickname="wsha.code+ga"
+ subject="comment 4"
+ date="2015-06-02T21:37:36Z"
+ content="""
+Okay, sorry, in the previous posts, I misunderstood parts of how git annex works. Let me break down what I want to do again and how I have tried to set it up.
+
+Use case
+--------
+The basic use case is to have a client (could be more than one) generating content that is pushed to a server. The client would like to be able to drop any data at any time, while the server is expected to keep all data at all times. Additionally, the data stored in the server should be visible in the working tree to other users on the server (and the working tree should stay up to date). The client has push access to the server, but the server does not have pull access to the client.
+
+Set up
+------
+I have taken the following steps to try to set this up so far:
+
+* Create git annex repos on the client and server.
+* Add the server as a remote for the client.
+* Set the client's group as manual and the server's group as backup.
+* Set up the server so that its working tree is kept up to date
+  - This step was one of the ones I was expecting to work when I first asked the question
+  - To do this, I added `git merge synced/master master` to the `annex-content` hook described [here](https://git-annex.branchable.com/git-annex-shell/) and to the `post-receive` hook (I did this because the `annex-content` hook was not being called for updates pushed by the assistant -- should it be?).
+* Set up the server so its working tree remains visible to other users
+  - Set `git config core.sharedrepository group` on the server repo as described [here](http://git-annex.branchable.com/tips/shared_git_annex_directory_between_multiple_users/) so that the server's files are accessible to other users in the same group.
+  - I'm not sure if the previous step was doing anything because updated files were still being changed to the server's group. To deal with this, I added `chgrp -R shared \"$(git rev-parse --show-toplevel)\"` to the `annex-content`, `post-receive`, and `post-merge` hooks.
+
+Questions
+---------
+The above set up seems to be working now. My remaining questions are:
+
+1. Is it okay to auto merge synced/master like this? Is there a better way to do the same thing? The repo contents should only see light editing, but could I run into problems if the client is running the assistant and I try to edit files on the server directly (e.g. if some file is modified on the client and the assistant triggers it auto-merging into the server while I am modifying some other file in the server's working tree)? Should I use a separate client repo on the server if I want to modify files on the server?
+2. Is there a better way to make the group permissions stick?
+3. Is it okay to symlink just `git-annex` and `git-annex-shell` onto my path and not `git`, `git-upload-pack`, `git-receive-pack`, etc? Maybe there is no downside to using the versions that are packaged with git annex for all of my general git usage, but I'd rather just use the system git for things other than git annex.
+"""]]
diff --git a/doc/forum/Ambigous_argument___40__unknown_revision_or_path__41__.mdwn b/doc/forum/Ambigous_argument___40__unknown_revision_or_path__41__.mdwn
new file mode 100644
index 0000000000..a4cc6a0a50
--- /dev/null
+++ b/doc/forum/Ambigous_argument___40__unknown_revision_or_path__41__.mdwn
@@ -0,0 +1,148 @@
+Okey, so I set up a repo on a client computer with files in a directory. All went fine without errors. Then I created the remote server in backup mode. Everything went well with the sync. Back at the office I now want the same repo content on this computer, so I fire up the assistant, add a local repo directory (empty of course) and then add a remote server. Then this happens
+
+    [2014-11-05 09:17:24 CET] main: Syncing with server__volume1_work_user_work
+    From ssh://git-annex-server-username_7000_.2Fvolume1.2Fwork_user.2Fwork/volume1/work_user/work
+     * [new branch]      git-annex  -> server__volume1_work_user_work/git-annex
+     * [new branch]      master     -> server__volume1_work_user_work/master
+     * [new branch]      synced/git-annex -> server__volume1_work_user_work/synced/git-annex
+     * [new branch]      synced/master -> server__volume1_work_user_work/synced/master
+    (merging server__volume1_work_user_work/git-annex server__volume1_work_user_work/synced/git-annex into git-annex...)
+    (Recording state in git...)
+    fatal: ambiguous argument 'refs/heads/synced/master..refs/remotes/server__volume1_work_user_work/synced/master': unknown revision or path not in the working tree.
+    Use '--' to separate paths from revisions, like this:
+    'git  [...] -- [...]'
+
+    Automatic merge went well; stopped before committing as requested
+    To ssh://username@git-annex-server-username_7000_.2Fvolume1.2Fwork_user.2Fwork/volume1/work_user/work/
+       3943547..00aa6bd  git-annex -> synced/git-annex
+     ! [rejected]        annex/direct/master -> synced/master (non-fast-forward)
+    error: failed to push some refs to 'ssh://username@git-annex-server-username_7000_.2Fvolume1.2Fwork_user.2Fwork/volume1/work_user/work/'
+    hint: Updates were rejected because a pushed branch tip is behind its remote
+    hint: counterpart. Check out this branch and integrate the remote changes
+    hint: (e.g. 'git pull ...') before pushing again.
+    hint: See the 'Note about fast-forwards' in 'git push --help' for details.
+    [2014-11-05 09:20:09 CET] Pusher: Syncing with server__volume1_work_user_work
+    [2014-11-05 09:20:09 CET] Committer: Committing changes to git
+    (Recording state in git...)
+
+    SHA256E-s1570--7437c0b77825d2636723cde82f0094854357aec43619b7d62707b5c26c4b0379.c
+
+                  0   0%    0.00kB/s    0:00:00
+              1,570 100%    1.50MB/s    0:00:00 (xfr#1, to-chk=0/1)
+    [2014-11-05 09:20:10 CET] Transferrer: Downloaded timing_ds20.c
+    To ssh://username@git-annex-server-username_7000_.2Fvolume1.2Fwork_user.2Fwork/volume1/work_user/work/
+       0e52256..2515592  annex/direct/master -> synced/master
+    error: Ref refs/heads/synced/master is at 2515592b1a0b2a355b5334c315d8857619b4a9c3 but expected 0e522560b9b6f0e52f789187e2fbd9076174bc4b
+    remote: error: failed to lock refs/heads/synced/master
+    To ssh://username@git-annex-server-username_7000_.2Fvolume1.2Fwork_user.2Fwork/volume1/work_user/work/
+     ! [remote rejected] annex/direct/master -> synced/master (failed to lock)
+    error: failed to push some refs to 'ssh://username@git-annex-server-username_7000_.2Fvolume1.2Fwork_user.2Fwork/volume1/work_user/work/'
+    To ssh://username@git-annex-server-username_7000_.2Fvolume1.2Fwork_user.2Fwork/volume1/work_user/work/
+     * [new branch]      git-annex -> refs/synced/a455ea74-d6fe-4f82-b796-72c20fc113fd/git-annex
+     * [new branch]      annex/direct/master -> refs/synced/a455ea74-d6fe-4f82-b796-72c20fc113fd/annex/direct/master
+
+    SHA256E-s7--19d210b22987ea279fe707c89b3afdbe530537118d3bac62c722b6ec54901e5a
+
+                  0   0%    0.00kB/s    0:00:00
+                  7 100%    6.84kB/s    0:00:00 (xfr#1, to-chk=0/1)
+    [2014-11-05 09:20:13 CET] Transferrer: Downloaded somefile
+    [2014-11-05 09:20:13 CET] Pusher: Syncing with server__volume1_work_user_work
+    (Recording state in git...)
+
+    SHA256E-s7762408--27c11674fe8c0b23f469aa90342dd45bf9ec9c83108251985b3231178f0f4ed5.pdf
+
+                  0   0%    0.00kB/s    0:00:00
+          5,406,720  69%    5.12MB/s    0:00:00
+          7,762,408 100%    5.01MB/s    0:00:01 (xfr#1, to-chk=0/1)
+    To ssh://username@git-annex-server-username_7000_.2Fvolume1.2Fwork_user.2Fwork/volume1/work_user/work/
+       00aa6bd..841a157  git-annex -> synced/git-annex
+    [2014-11-05 09:20:16 CET] Transferrer: Downloaded anotherfile.pdf
+
+    SHA256E-s322841--452ddf060131e1cfe44a02ee23840494d4566c4b1ad58f0131d52f17fbf65ea2.pdf
+
+And then it continues transfering the files. Then this happens
+
+    error: Ref refs/heads/synced/git-annex is at 4a802d679a5a7e2eae3c16b2ed0a8d724b665b0a but expected 45004c24db2eb24dddc04d1542d3689aa27b9507
+    remote: error: failed to lock refs/heads/synced/git-annex
+    To ssh://username@git-annex-server-username_7000_.2Fvolume1.2Fwork_user.2Fwork/volume1/work_user/work/
+     ! [remote rejected] git-annex -> synced/git-annex (failed to lock)
+    error: failed to push some refs to 'ssh://username@git-annex-server-username_7000_.2Fvolume1.2Fwork_user.2Fwork/volume1/work_user/work/'
+    [2014-11-05 10:13:48 CET] read: git ["--git-dir=/home/officeusername/work/.git","--work-tree=/home/officeusername/work","-c","core.bare=false","push","server__volume1_work_user_work","master"]
+
+    SHA256E-s115543--4aca4053b23d914ea034f06439edcb7d3d9bcc24b5d9865f5e062394ca1263e0
+   
+                  0   0%    0.00kB/s    0:00:00
+            115,543 100%    2.98MB/s    0:00:00 (xfr#1, to-chk=0/1)
+    [2014-11-05 10:13:48 CET] feed: ssh ["-S",".git/annex/ssh/abf79f467a5dcf714ddaf5162ffc09ba","-o","ControlMaster=auto","-o","ControlPersist=yes","-T","username@git-annex-server-username_7000_.2Fvolume1.2Fwork_user.2Fwork","git-annex-shell 'transferinfo' '/volume1/work_user/work/' 'SHA256E-s115543--4aca4053b23d914ea034f06439edcb7d3d9bcc24b5d9865f5e062394ca1263e0' --uuid 05bdd8d5-da30-4024-a921-0f8b1a2f33df '--' 'remoteuuid=a455ea74-d6fe-4f82-b796-72c20fc113fd' 'associatedfile=pathtosomefile' '--'"]
+    [2014-11-05 10:13:48 CET] NetWatcherFallback: trying manual pull to resolve failed pushes
+    [2014-11-05 10:13:48 CET] call: git ["--git-dir=/home/officeusername/work/.git","--work-tree=/home/officeusername/work","-c","core.bare=false","fetch","server__volume1_work_user_work"]
+    [2014-11-05 10:13:48 CET] Watcher: add symlink pathtosomefile
+    [2014-11-05 10:13:48 CET] Transferrer: Downloaded file
+    [2014-11-05 10:13:48 CET] TransferWatcher: transfer finishing: Transfer {transferDirection = Download, transferUUID = UUID "05bdd8d5-da30-4024-a921-0f8b1a2f33df", transferKey = Key {keyName = "4aca4053b23d914ea034f06439edcb7d3d9bcc24b5d9865f5e062394ca1263e0", keyBackendName = "SHA256E", keySize = Just 115543, keyMtime = Nothing, keyChunkSize = Nothing, keyChunkNum = Nothing}}
+    [2014-11-05 10:13:48 CET] Transferrer: Transferring: Download UUID "05bdd8d5-da30-4024-a921-0f8b1a2f33df" pathtosomefile2 Nothing
+    [2014-11-05 10:13:48 CET] TransferWatcher: transfer starting: Download UUID "05bdd8d5-da30-4024-a921-0f8b1a2f33df" SHA256E-s145572--50e4d0d2fa52fb6843258509637307b8efc87d31abab834e7bc3174296689e25 Nothing
+    [2014-11-05 10:13:48 CET] TransferWatcher: transfer starting: Download UUID "05bdd8d5-da30-4024-a921-0f8b1a2f33df" pathtosomefile2 Nothing
+    [2014-11-05 10:13:48 CET] read: rsync ["--progress","--inplace","--perms","-e","'ssh' '-S' '.git/annex/ssh/abf79f467a5dcf714ddaf5162ffc09ba' '-o' 'ControlMaster=auto' '-o' 'ControlPersist=yes' '-T' 'username@git-annex-server-username_7000_.2Fvolume1.2Fwork_user.2Fwork' 'git-annex-shell ''sendkey'' ''/volume1/work_user/work/'' ''SHA256E-s145572--50e4d0d2fa52fb6843258509637307b8efc87d31abab834e7bc3174296689e25'' --uuid 05bdd8d5-da30-4024-a921-0f8b1a2f33df ''--'' ''remoteuuid=a455ea74-d6fe-4f82-b796-72c20fc113fd'' ''direct=1'' ''associatedfile=pathtosomefile2'' ''--'''","--","dummy:","/home/officeusername/work/.git/annex/tmp/SHA256E-s145572--50e4d0d2fa52fb6843258509637307b8efc87d31abab834e7bc3174296689e25"]
+    [2014-11-05 10:13:49 CET] Committer: committing 2 changes
+    [2014-11-05 10:13:49 CET] Committer: Committing changes to git
+    (Recording state in git...)
+    [2014-11-05 10:13:49 CET] feed: git ["--git-dir=/home/officeusername/work/.git","--work-tree=/home/officeusername/work","-c","core.bare=false","update-index","-z","--index-info"]
+    [2014-11-05 10:13:49 CET] read: git ["--git-dir=/home/officeusername/work/.git","--work-tree=/home/officeusername/work","-c","core.bare=false","symbolic-ref","HEAD"]
+    [2014-11-05 10:13:49 CET] read: git ["--git-dir=/home/officeusername/work/.git","--work-tree=/home/officeusername/work","-c","core.bare=false","show-ref","--hash","refs/heads/annex/direct/master"]
+    [2014-11-05 10:13:49 CET] read: git ["--git-dir=/home/officeusername/work/.git","--work-tree=/home/officeusername/work","-c","core.bare=false","write-tree"]
+    [2014-11-05 10:13:49 CET] read: git ["--git-dir=/home/officeusername/work/.git","--work-tree=/home/officeusername/work","-c","core.bare=false","rev-parse","2515592b1a0b2a355b5334c315d8857619b4a9c3:"]
+    [2014-11-05 10:13:49 CET] read: git ["--git-dir=/home/officeusername/work/.git","--work-tree=/home/officeusername/work","-c","core.bare=false","symbolic-ref","HEAD"]
+    [2014-11-05 10:13:49 CET] read: git ["--git-dir=/home/officeusername/work/.git","--work-tree=/home/officeusername/work","-c","core.bare=false","show-ref","refs/heads/annex/direct/master"]
+    [2014-11-05 10:13:49 CET] call: git ["--git-dir=/home/officeusername/work/.git","--work-tree=/home/officeusername/work","-c","core.bare=false","branch","-f","synced/master"]
+    [2014-11-05 10:13:49 CET] call: git ["--git-dir=/home/officeusername/work/.git","--work-tree=/home/officeusername/work","-c","core.bare=false","branch","-f","master"]
+    drop server__volume1_work_user_work pathtosomefile2
+    [2014-11-05 10:13:49 CET] call: ssh ["-S",".git/annex/ssh/abf79f467a5dcf714ddaf5162ffc09ba","-o","ControlMaster=auto","-o","ControlPersist=yes","-T","username@git-annex-server-username_7000_.2Fvolume1.2Fwork_user.2Fwork","git-annex-shell 'dropkey' '/volume1/work_user/work/' '--quiet' '--force' 'SHA256E-s674028--398092ddec93e33310ca8be87b4d48b2c84a873b579b80002cad3738c1461266' --uuid 05bdd8d5-da30-4024-a921-0f8b1a2f33df"]
+
+    SHA256E-s145572--50e4d0d2fa52fb6843258509637307b8efc87d31abab834e7bc3174296689e25
+
+Again it transfer some files...and then
+
+    error: Ref refs/heads/synced/git-annex is at ac6d09794a274e861db1ef00295e361e15d85ca3 but expected 4a802d679a5a7e2eae3c16b2ed0a8d724b665b0a
+    remote: error: failed to lock refs/heads/synced/git-annex
+    To ssh://username@git-annex-server-username_7000_.2Fvolume1.2Fwork_user.2Fwork/volume1/work_user/work/
+     ! [remote rejected] git-annex -> synced/git-annex (failed to lock)
+    error: failed to push some refs to 'ssh://username@git-annex-server-username_7000_.2Fvolume1.2Fwork_user.2Fwork/volume1/work_user/work/'
+    [2014-11-05 10:13:52 CET] read: git ["--git-dir=/home/officeusername/work/.git","--work-tree=/home/officeusername/work","-c","core.bare=false","push","server.me__volume1_work_user_work","master"]
+
+          8,454,144   2%    2.67MB/s    0:02:24  [2014-11-05 10:13:53 CET] NetWatcherFallback: fallback pushing to [Remote { name ="server__volume1_work_user_work" }]
+    [2014-11-05 10:13:53 CET] call: git ["--git-dir=/home/officeusername/work/.git","--work-tree=/home/officeusername/work","-c","core.bare=false","push","server.me__volume1_work_user_work","+git-annex:refs/synced/a455ea74-d6fe-4f82-b796-72c20fc113fd/git-annex","refs/heads/annex/direct/master:refs/synced/a455ea74-d6fe-4f82-b796-72c20fc113fd/annex/direct/master"]
+
+         12,058,624   2%    2.86MB/s    0:02:13  To ssh://username@git-annex-server-username_7000_.2Fvolume1.2Fwork_user.2Fwork/volume1/work_user/work/
+       00aa6bd..ac6d097  git-annex -> refs/synced/a455ea74-d6fe-4f82-b796-72c20fc113fd/git-annex
+
+         16,875,520   4%    3.34MB/s    0:01:53  [2014-11-05 10:13:55 CET] Pusher: Syncing with server__volume1_work_user_work
+
+Again
+
+    error: Ref refs/heads/synced/git-annex is at 383d9581c2722ab87647cf26f1ab8f25758866a4 but expected d158cddb1055adbe44973208d5562961acb0c42b
+    remote: error: failed to lock refs/heads/synced/git-annex
+    To ssh://username@git-annex-server-username_7000_.2Fvolume1.2Fwork_user.2Fwork/volume1/work_user/work/
+     ! [remote rejected] git-annex -> synced/git-annex (failed to lock)
+    error: failed to push some refs to 'ssh://username@git-annex-server-username_7000_.2Fvolume1.2Fwork_user.2Fwork/volume1/work_user/work/'
+    [2014-11-05 11:13:58 CET] read: git ["--git-dir=/home/officeusername/work/.git","--work-tree=/home/officeusername/work","-c","core.bare=false","push","server__volume1_work_user_work","master"]
+    ok
+    [2014-11-05 11:13:58 CET] dropped somefile (from UUID "05bdd8d5-da30-4024-a921-0f8b1a2f33df") (copies now 2) : file renamed
+    [2014-11-05 11:13:58 CET] TransferWatcher: transfer starting: Download UUID "05bdd8d5-da30-4024-a921-0f8b1a2f33df" someotherfile Nothing
+
+Then some other stuff
+
+    error: Ref refs/remotes/server__volume1_work_user_work/synced/git-annex is at 2e867456306f9ac0dd61e29c221ea3968dc4cb70 but expected 086e08a168498b84a74065c1f71e114a9868867d
+    From ssh://git-annex-server-username_7000_.2Fvolume1.2Fwork_user.2Fwork/volume1/work_user/work
+    ! 086e08a..2e86745  synced/git-annex -> server__volume1_work_user_work/synced/git-annex  (unable to update local ref)
+
+And so on. When submitting this, the system is still transfering files (arrow pointing to the left, so that has to be from the server to the office computer I suppose). Checking the office computer file index by running
+
+    git ls-files | wl -l
+
+yields the expected number of files. So, in addition to the errors above this seems a bit strange to me that it is still transferring files.
+
+Thinking about the error messages above...is it so that the download and git repo updates work in parallel, so if git tries to update on a partly transferred files the checksum is wrong and then I get the errors above? If so, is the file transferred again later and is there a way to suppress the "failed to sync" in the webapp? 
+
+Another thing, then I add the remote server it automatically goes into transfer mode, even though I configured it to be a backup from home before coming into the office. Maybe it is not autodetected. Would it cause problem to later change it from transfer to backup mode?
+
+Hope someone can point me in the right direction. And thanks in advance!
diff --git a/doc/forum/Ambiguous_repo_names__63__.mdwn b/doc/forum/Ambiguous_repo_names__63__.mdwn
new file mode 100644
index 0000000000..fb96240fe7
--- /dev/null
+++ b/doc/forum/Ambiguous_repo_names__63__.mdwn
@@ -0,0 +1,8 @@
+The output of `git annex info` includes this:
+
+ 	34xx3351-xx14-4039-6xxx-xxx9x5xxxxxx -- wx550 (wx312)
+ 	9x9x07xx-x6x9-464x-7x5x-x3xx42xx7x70 -- wx550
+
+What does this mean? Do I have two different repos both named wx550?
+What is wx312? 
+
diff --git a/doc/forum/Ambiguous_repo_names__63__/comment_1_9695c42113ad6c873b1eae1c1f04e70a._comment b/doc/forum/Ambiguous_repo_names__63__/comment_1_9695c42113ad6c873b1eae1c1f04e70a._comment
new file mode 100644
index 0000000000..13f7dbe78d
--- /dev/null
+++ b/doc/forum/Ambiguous_repo_names__63__/comment_1_9695c42113ad6c873b1eae1c1f04e70a._comment
@@ -0,0 +1,13 @@
+[[!comment format=mdwn
+ username="joey"
+ subject="""comment 1"""
+ date="2015-06-03T17:35:38Z"
+ content="""
+You have two repositories with a description of "wx550".
+You can set the description to anything you want using `git annex
+describe`.
+
+One of the repositories is configured as a remote of the local repositry.
+This means that the local repository can access it. The name of that
+remote is "wx312".
+"""]]
diff --git a/doc/forum/Android_-_ext3__47__4__47__....mdwn b/doc/forum/Android_-_ext3__47__4__47__....mdwn
new file mode 100644
index 0000000000..5227228a97
--- /dev/null
+++ b/doc/forum/Android_-_ext3__47__4__47__....mdwn
@@ -0,0 +1 @@
+Is there a good way to use a ext4 formated external sdcard on a rooted android with symlinks. If android mounts it (with the sdcard command i think) symlinks don't work. If i mount it with mount the permissions are a problem because every app has a different user.
diff --git a/doc/forum/Android__58___Cabal_hell.mdwn b/doc/forum/Android__58___Cabal_hell.mdwn
new file mode 100644
index 0000000000..e31b1b0dce
--- /dev/null
+++ b/doc/forum/Android__58___Cabal_hell.mdwn
@@ -0,0 +1,32 @@
+I tried to build the Android app according to the instructions on [the install page](https://git-annex.branchable.com/install/Android/).
+
+But during step 2 (In the chroot, run install-haskell-packages) cabal fails to handle the dependencies correctly. So I tried to change the standalone/android/cabal.config file to update all the dependencies, but this had me running into something I think is unresolvable in another package's
+dependencies. Eventually I had the following output:
+
+    # ./standalone/android/install-haskell-packages 
+    Downloading the latest package list from hackage.haskell.org
+    Skipping download: Local and remote files match.
+    /home/repo/git-annex /home/repo/git-annex/standalone/android
+    Resolving dependencies...
+    cabal: Could not resolve dependencies:
+    trying: git-annex-6.20160318 (user goal)
+    trying: persistent-template-2.1.6/installed-08b... (dependency of
+    git-annex-6.20160318)
+    next goal: monad-control (dependency of git-annex-6.20160318)
+    rejecting: monad-control-1.0.0.5/installed-cac..., 1.0.0.5, 1.0.0.4, 1.0.0.3,
+    1.0.0.2, 1.0.0.1, 1.0.0.0, 0.3.3.1, 0.3.3.0, 0.3.2.3 (global constraint
+    requires ==0.3.2.2)
+    rejecting: monad-control-0.3.2.2 (conflict: persistent-template =>
+    monad-control==1.0.0.5/installed-cac...)
+    rejecting: monad-control-0.3.2.1, 0.3.2, 0.3.1.4, 0.3.1.3, 0.3.1.2, 0.3.1.1,
+    0.3.1, 0.3.0.1, 0.3, 0.2.0.3, 0.2.0.2, 0.2.0.1, 0.2, 0.1 (global constraint
+    requires ==0.3.2.2)
+
+This tells me that the package persistent-template depends on the package monad-control at a version of 1.0.0.5. So I look at the [persistent-template hackage page](https://hackage.haskell.org/package/persistent-template-2.1.6) and see that it lists its dependency on monad-control as:
+
+    monad-control (>=0.2 && <1.1)
+
+And I don't think that's possible to resolve.
+
+All I wanted to do was change the icons on the Android package, so if you'll just accept the changed
+icons folder, I can submit the patch without testing the Android build, but currently I'm unable to build the Android port of git-annex.
diff --git a/doc/forum/Android__58___Cabal_hell/comment_1_34e869c1092f2885f54f627fb77a6057._comment b/doc/forum/Android__58___Cabal_hell/comment_1_34e869c1092f2885f54f627fb77a6057._comment
new file mode 100644
index 0000000000..e96bdf2222
--- /dev/null
+++ b/doc/forum/Android__58___Cabal_hell/comment_1_34e869c1092f2885f54f627fb77a6057._comment
@@ -0,0 +1,13 @@
+[[!comment format=mdwn
+ username="joey"
+ subject="""comment 1"""
+ date="2016-03-28T13:20:06Z"
+ content="""
+The android packages are all pinned at old versions, so the build is
+supposed to be reproducible using those versions. Diverging certainly
+invites cabal hell.
+
+Any chance you could show the original failure?
+
+I could probably integrate new icons if you want to just submit them.
+"""]]
diff --git a/doc/forum/Android__58___Cabal_hell/comment_2_ed5b2b37d37278458fdf2dc37539e741._comment b/doc/forum/Android__58___Cabal_hell/comment_2_ed5b2b37d37278458fdf2dc37539e741._comment
new file mode 100644
index 0000000000..0dba07c3c5
--- /dev/null
+++ b/doc/forum/Android__58___Cabal_hell/comment_2_ed5b2b37d37278458fdf2dc37539e741._comment
@@ -0,0 +1,10 @@
+[[!comment format=mdwn
+ username="freewheelinfranks@f936f2e420801f565d39e40036c0a22c71a79425"
+ nickname="freewheelinfranks"
+ subject="Makes sense."
+ date="2016-04-02T23:41:23Z"
+ content="""
+Okay, thanks for the information.
+
+How should I send you the icons? I have sent you an email with a zip file of the icons. If that is not appropriate, please let me know.
+"""]]
diff --git a/doc/forum/Android__58___Cabal_hell/comment_3_fe71aa84ed5e1bb1e19a7d0f4e42eccd._comment b/doc/forum/Android__58___Cabal_hell/comment_3_fe71aa84ed5e1bb1e19a7d0f4e42eccd._comment
new file mode 100644
index 0000000000..c9d865c648
--- /dev/null
+++ b/doc/forum/Android__58___Cabal_hell/comment_3_fe71aa84ed5e1bb1e19a7d0f4e42eccd._comment
@@ -0,0 +1,9 @@
+[[!comment format=mdwn
+ username="joey"
+ subject="""comment 3"""
+ date="2016-04-14T19:01:42Z"
+ content="""
+An email would be fine, but I have not received that email.
+
+Post a link to somewhere?
+"""]]
diff --git a/doc/forum/Android__58___Cabal_hell/comment_4_da004cee7c0cf5617291dbd520af2e96._comment b/doc/forum/Android__58___Cabal_hell/comment_4_da004cee7c0cf5617291dbd520af2e96._comment
new file mode 100644
index 0000000000..043ed6d552
--- /dev/null
+++ b/doc/forum/Android__58___Cabal_hell/comment_4_da004cee7c0cf5617291dbd520af2e96._comment
@@ -0,0 +1,10 @@
+[[!comment format=mdwn
+ username="freewheelinfranks@f936f2e420801f565d39e40036c0a22c71a79425"
+ nickname="freewheelinfranks"
+ subject="Here's a link to the image files"
+ date="2016-05-03T06:14:40Z"
+ content="""
+The file \"icons.zip\" can replace the icons folder in the git-annex repo. The folder to be replaced is found at standalone/android/icons.
+
+[https://stanford.edu/~farhank/icons.zip](https://stanford.edu/~farhank/icons.zip)
+"""]]
diff --git a/doc/forum/Android__58___Cabal_hell/comment_5_12e2cb09ac2ee07b392cd9be064a3cbe._comment b/doc/forum/Android__58___Cabal_hell/comment_5_12e2cb09ac2ee07b392cd9be064a3cbe._comment
new file mode 100644
index 0000000000..2fc689cb3c
--- /dev/null
+++ b/doc/forum/Android__58___Cabal_hell/comment_5_12e2cb09ac2ee07b392cd9be064a3cbe._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="joey"
+ subject="""comment 5"""
+ date="2016-05-03T17:26:34Z"
+ content="""
+Indeed, I can see these are better scaled and rounded corners.
+Thanks for the contribution, and for testing it! Merged.
+"""]]
diff --git a/doc/forum/Android__58___Encrypted_Remotes__63__.mdwn b/doc/forum/Android__58___Encrypted_Remotes__63__.mdwn
new file mode 100644
index 0000000000..b8342392fc
--- /dev/null
+++ b/doc/forum/Android__58___Encrypted_Remotes__63__.mdwn
@@ -0,0 +1,3 @@
+Hi,
+
+Does the Android app support encryption at all? The assistant allows me to create encrypted repositories but it just keeps telling me to install git-remote-gcrypt, which did not work.
diff --git a/doc/forum/Android__58___Encrypted_Remotes__63__/comment_1_6b16cd372a9bd4f99d4c8b09a82ce3ed._comment b/doc/forum/Android__58___Encrypted_Remotes__63__/comment_1_6b16cd372a9bd4f99d4c8b09a82ce3ed._comment
new file mode 100644
index 0000000000..3edcc5f900
--- /dev/null
+++ b/doc/forum/Android__58___Encrypted_Remotes__63__/comment_1_6b16cd372a9bd4f99d4c8b09a82ce3ed._comment
@@ -0,0 +1,10 @@
+[[!comment format=mdwn
+ username="http://joeyh.name/"
+ ip="209.250.56.55"
+ subject="comment 1"
+ date="2014-07-03T19:14:37Z"
+ content="""
+Encrypted [[special_remotes]] are supported just fine on Android.
+
+As for storing the [[encrypted git repository itself on a remote|special_remotes/gcrypt]], which is what git-remote-gcrypt is used for, it would need porting that to Android. I don't think that should be very hard (it's just a rather complicated shell script and all the tools it uses are already included in the git-annex bundle). I may eventully end up reimplementing git-remote-gcrypt as part of git-annex in Haskell, since porting it to eg, Windows is likely to be a lot harder..
+"""]]
diff --git a/doc/forum/Android__58___git_annex_sync_--content_not_run_in_all_repos__63__.mdwn b/doc/forum/Android__58___git_annex_sync_--content_not_run_in_all_repos__63__.mdwn
new file mode 100644
index 0000000000..4d4b679417
--- /dev/null
+++ b/doc/forum/Android__58___git_annex_sync_--content_not_run_in_all_repos__63__.mdwn
@@ -0,0 +1,24 @@
+I have two repositories in an Android device, and each is synced with a corresponding repo in a computer:
+
+A-android <-> A-computer
+
+B-android <-> B-computer
+
+Both are configured as "client: a repository on your computer".
+
+
+One of the repositories (say, A) has the expected behavior (changes in the Android device show in the computer and viceversa).
+
+The other repo pair doesn't. Changes in the computer do appear in the Android device, but not the other way around. I can get them to appear if, in the computer, I do
+
+~/B-computer$ git annex sync --content
+
+but not otherwise. For example, from the Android web app clicking "sync" in B-android or B-computer does not make the changes in the Android appear in the computer. 
+
+I am very new to git annex, but I do not understand this; I've searched around to no avail; I think that I should expect sync to run for all repositories  
+[http://git-annex.branchable.com/design/assistant/#comment-6ddc18af9685a7ddbbe6e639e7b15feb](http://git-annex.branchable.com/design/assistant/#comment-6ddc18af9685a7ddbbe6e639e7b15feb), but 
+then maybe not to do what I expect: [http://git-annex.branchable.com/sync/#comment-1a0a053001ef684a0f3c5325e8062483](http://git-annex.branchable.com/sync/#comment-1a0a053001ef684a0f3c5325e8062483).
+
+
+The problem with the behavior I see is that in addition to making me unsure if I am seeing the latest stuff when I do, say, "ls" or "cat", it leads to trouble (if I edit the same file in both the Android and the computer, eventually I will get a conflict, of course). If this is by design, is there a way to get all of the repositories to show the first behavior (changes propagated android <-> computer)?
+
diff --git a/doc/forum/Android__58___git_annex_sync_--content_not_run_in_all_repos__63__/comment_1_aff6a984ab076433fbce32bacae6660e._comment b/doc/forum/Android__58___git_annex_sync_--content_not_run_in_all_repos__63__/comment_1_aff6a984ab076433fbce32bacae6660e._comment
new file mode 100644
index 0000000000..8d62f0bf59
--- /dev/null
+++ b/doc/forum/Android__58___git_annex_sync_--content_not_run_in_all_repos__63__/comment_1_aff6a984ab076433fbce32bacae6660e._comment
@@ -0,0 +1,13 @@
+[[!comment format=mdwn
+ username="joey"
+ subject="""comment 1"""
+ date="2015-02-25T19:22:17Z"
+ content="""
+It sounds like the android device is sending the files to repository
+B-computer, but it is not updating its repository until you run
+`git annex sync` manually.
+
+Are you sure that the git-annex assistant is actively managing the `B-computer`
+repository? I would `tail -f .git/annex/daemon.log` in that repository 
+when making a change to B-android, and see what it does.
+"""]]
diff --git a/doc/forum/Android__58___git_annex_sync_--content_not_run_in_all_repos__63__/comment_2_5b1565955ed88554e5ef4a00c0f4a754._comment b/doc/forum/Android__58___git_annex_sync_--content_not_run_in_all_repos__63__/comment_2_5b1565955ed88554e5ef4a00c0f4a754._comment
new file mode 100644
index 0000000000..056c78547b
--- /dev/null
+++ b/doc/forum/Android__58___git_annex_sync_--content_not_run_in_all_repos__63__/comment_2_5b1565955ed88554e5ef4a00c0f4a754._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="https://www.google.com/accounts/o8/id?id=AItOawnbPKLjBONawBd74MKJZo05juCqdsP1jAU"
+ nickname="Ramon"
+ subject="comment 2"
+ date="2015-02-26T08:08:37Z"
+ content="""
+Thanks. I'll check as suggested if it happens again. Since switching between repos in android was complicated, I've started all over putting everything in a single repo (the default one).
+"""]]
diff --git a/doc/forum/Android__58___is_constant_high_cpu_usage_to_be_expected__63__.mdwn b/doc/forum/Android__58___is_constant_high_cpu_usage_to_be_expected__63__.mdwn
new file mode 100644
index 0000000000..0392dbb38f
--- /dev/null
+++ b/doc/forum/Android__58___is_constant_high_cpu_usage_to_be_expected__63__.mdwn
@@ -0,0 +1,3 @@
+While running the Git Annex App on Android, the app causes a constant cpu usage of about 50% when idling. I've seen this behavior on two devices (phone and tablet) with a CM 10.1 nightly build. The app causes this high cpu usage even when it is in the background, not performing any synchronization and managing only one repository containing just one file. Unfortunately I couldn't figure out what causes the cpu usage. The daemon.log file remains unchanged and I couldn't find any other log files.
+
+Is this expected behavior or unusual high cpu usage?
diff --git a/doc/forum/Android__58___is_constant_high_cpu_usage_to_be_expected__63__/comment_1_7880fc38792a1fcbf3e5c47e8bcaabce._comment b/doc/forum/Android__58___is_constant_high_cpu_usage_to_be_expected__63__/comment_1_7880fc38792a1fcbf3e5c47e8bcaabce._comment
new file mode 100644
index 0000000000..28f3dfb925
--- /dev/null
+++ b/doc/forum/Android__58___is_constant_high_cpu_usage_to_be_expected__63__/comment_1_7880fc38792a1fcbf3e5c47e8bcaabce._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="https://www.google.com/accounts/o8/id?id=AItOawmKKg3Vmzk7KwRGRKjHVdtyoj1JfxLX6NM"
+ nickname="Tom"
+ subject="comment 1"
+ date="2013-10-01T17:38:03Z"
+ content="""
+I've had this issue as well. Saw a comment on Joey's blog that implies he knows about it and that a fix will be released soon.
+"""]]
diff --git a/doc/forum/Android__58___is_constant_high_cpu_usage_to_be_expected__63__/comment_2_840fbce18b4fdec21ee557fdf52c366e._comment b/doc/forum/Android__58___is_constant_high_cpu_usage_to_be_expected__63__/comment_2_840fbce18b4fdec21ee557fdf52c366e._comment
new file mode 100644
index 0000000000..eaf42c9e2a
--- /dev/null
+++ b/doc/forum/Android__58___is_constant_high_cpu_usage_to_be_expected__63__/comment_2_840fbce18b4fdec21ee557fdf52c366e._comment
@@ -0,0 +1,10 @@
+[[!comment format=mdwn
+ username="http://joeyh.name/"
+ ip="64.134.31.139"
+ subject="comment 2"
+ date="2013-10-16T16:57:02Z"
+ content="""
+This was fixed.
+
+(Please file bug reports for bugs in the future, so they don't clutter up this forum once fixed.)
+"""]]
diff --git a/doc/forum/Android__58___unusably_slow_with_moderately_sized_repositories.mdwn b/doc/forum/Android__58___unusably_slow_with_moderately_sized_repositories.mdwn
new file mode 100644
index 0000000000..ca2b3906fa
--- /dev/null
+++ b/doc/forum/Android__58___unusably_slow_with_moderately_sized_repositories.mdwn
@@ -0,0 +1,67 @@
+I am trying to sync a repository between a computer and two Android devices. The data in the directory of files (i.e., not the .git directory) takes about 7.4 GB and there are about 10000 files and directories in total (as reported by find . | wc -l) of which 7000 are files (as reported by find - -type f | wc -l).
+
+I've used two different routes: 1. start the assistant in the Android in a directory with all the files: 2. start the assistant in the Android in an empty dir and sync with the git-annex repository in the computer. None are working. I am using, in the androids, v. 5.20150224-g9dca034 and in the computer the latest Debian version (5.20141125).
+
+Details follow:
+
+
+Route 1. 
+=========
+
+In one Android, I had already transmitted all the files (nothing git-related; just the directory). So I placed that directory under /sdcard/annex, and started the assistant. I did not add any other repository (i.e., the computer was not added). After more than 12 hours, it is not done. The webapp in the Android shows "Adding blablabla".
+
+If I compare the sizes of directories between the computer and the android with du I get:
+
+Computer:~  du -H -d 1
+215624  ./.git
+7710564 ./directory
+7926192 .
+
+
+Android (using adb shell):~ du -H -d 1
+7681744 ./directory
+35388   ./.git
+7717136 .
+
+
+Over the last six hours it seems the size in the Android is getting closer and closer to that in the computer, but it seems is never quite getting there. And then, I think after this is completed (if ever), I'll need to sync with the computer.
+
+
+Route 2. 
+========
+
+In the other Android I started the assistant, and added the computer as the other repository. Let them sync. Initially I was getting the TransferScanner crashed problem in
+[Android version does not sync](http://git-annex.branchable.com/forum/Android_version_does_not_sync/). So I issued the git-annex sync as suggested. Things started syncing. But again, after more than 12 hourse, it is not done and I have no idea how long I'd have to wait. In this case, git-annex seems to be placing stuff in ./git, as the things downloaded are actually links. A du shows this:
+
+392304  ./.git
+82464   ./directory
+474776  .
+
+-------
+
+I think I am experiencing two different problems:
+
+Problem 1. 
+==========
+
+git is just taking forever (route 1.) in the Android. There are not network issues possible there, since that Android is not downloading/syncing anything with anyone. Interestingly, in case this is of any use, from adb shell a top shows: 
+
+User 27%, System 16%, IOW 1%, IRQ 0%
+User 318 + Nice 28 + Sys 199 + Idle 674 + IOW 20 + IRQ 0 + SIRQ 0 = 1239
+
+
+and the git-annex process taking between 3% to 15% CPU.
+
+
+Problem 2
+==========
+
+git + network issues in route 2.  But I doubt this is mainly a network problem. As a comparison, using syncthing I was able to initially sync (i.e., download to the android device) the complete set of files in about four hours and then checking for changes in the Android takes syncthing about 1 to 3 minutes. I've also repeated this using a private seafile server (seafile server running in the computer); downloading all files takes about the same (about four hours), and then checking for changes against the seafile server takes between 45 minutes and an hour. For comparison, a top also shows that git-annex is taking between 2 and 15% CPU.
+
+
+So I am wondering if I am making some silly mistake here, or maybe this is not the intended use case (because of number or size of files for an Android device). Of course, computer-to-computer things work just fine. It is the Android part which is not working. 
+
+
+
+
+
diff --git a/doc/forum/Android__58___unusably_slow_with_moderately_sized_repositories/comment_3_c5c655ee3ec2501c1c3c60fd83574914._comment b/doc/forum/Android__58___unusably_slow_with_moderately_sized_repositories/comment_3_c5c655ee3ec2501c1c3c60fd83574914._comment
new file mode 100644
index 0000000000..42cab60b5e
--- /dev/null
+++ b/doc/forum/Android__58___unusably_slow_with_moderately_sized_repositories/comment_3_c5c655ee3ec2501c1c3c60fd83574914._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="https://www.google.com/accounts/o8/id?id=AItOawnbPKLjBONawBd74MKJZo05juCqdsP1jAU"
+ nickname="Ramon"
+ subject="comment 3"
+ date="2015-02-26T23:44:40Z"
+ content="""
+Please forgive the noise. I think I've been doing something very, very stupid. Sorry. I'd delete the whole thread if I could
+"""]]
diff --git a/doc/forum/Android_version_does_not_sync.mdwn b/doc/forum/Android_version_does_not_sync.mdwn
new file mode 100644
index 0000000000..4c2d288be9
--- /dev/null
+++ b/doc/forum/Android_version_does_not_sync.mdwn
@@ -0,0 +1,23 @@
+Hey!
+
+I have got a Kindle Fire HD (2nd generation) and want to use Git Annex to sync my data round. However, it does not sync my data at all (my other setups, e.g. PC or laptop do work quite well). The reason for that seems to be quite simple, but I do not know how to fix that:
+
+I always get the warning message:
+
+> TransferScanner crashed: /storage/emulated/legacy/annex/.git/index: copyFile: does not exist (No such file or directory).
+
+I found the same error in the log:
+
+> git-annex: /storage/emulated/legacy/annex/.git/index: copyFile: does not exist (No such file or directory)
+> ...
+> fatal: Run with no arguments or with -c cmd
+> git-annex-shell: git-shell failed
+
+I am using the latest versions for android and arm (server, raspberry pi) from this site from today (13.07.2014)
+
+http://git-annex.branchable.com/install/
+
+I hope somebody can help me. I tried restaring the TransferScanner, restarting git-annex, restarting the OS, resetting the OS (to shipping defaults), nothing helped.
+
+Cheers,
+Stephan
diff --git a/doc/forum/Android_version_does_not_sync/comment_1_ed9e33eef2c6d651847dca9d3f7a63f6._comment b/doc/forum/Android_version_does_not_sync/comment_1_ed9e33eef2c6d651847dca9d3f7a63f6._comment
new file mode 100644
index 0000000000..589b230c39
--- /dev/null
+++ b/doc/forum/Android_version_does_not_sync/comment_1_ed9e33eef2c6d651847dca9d3f7a63f6._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="sts"
+ ip="134.147.119.104"
+ subject="comment 1"
+ date="2014-07-14T09:12:13Z"
+ content="""
+Ok, I checked and the index file was missing and not sure why, but I had to manually sync one time. So I had to go in the annex folder with the command line and make a \"git annex sync\". It works pretty well now, it seems not to automatically sync my data (have to force it with \"Sync Now\"), but at least I can sync data to my tablet and thats awesome :).
+"""]]
diff --git a/doc/forum/Android_version_does_not_sync/comment_2_4eafd3e989611f835c489b379bd6ec8a._comment b/doc/forum/Android_version_does_not_sync/comment_2_4eafd3e989611f835c489b379bd6ec8a._comment
new file mode 100644
index 0000000000..bf2b412633
--- /dev/null
+++ b/doc/forum/Android_version_does_not_sync/comment_2_4eafd3e989611f835c489b379bd6ec8a._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="http://joeyh.name/"
+ ip="209.250.56.2"
+ subject="comment 2"
+ date="2014-07-14T18:21:15Z"
+ content="""
+How did you create this repository on the android device? Normally a git repository has an index file, unless it has just been manually created with `git init` and has not yet had anything added to it. When the webapp is used to create a repository, it should always leave it with an index file already present.
+"""]]
diff --git a/doc/forum/Android_version_does_not_sync/comment_3_2a4efec37015ea44509e7ed16b36a72d._comment b/doc/forum/Android_version_does_not_sync/comment_3_2a4efec37015ea44509e7ed16b36a72d._comment
new file mode 100644
index 0000000000..71c5b91d19
--- /dev/null
+++ b/doc/forum/Android_version_does_not_sync/comment_3_2a4efec37015ea44509e7ed16b36a72d._comment
@@ -0,0 +1,13 @@
+[[!comment format=mdwn
+ username="https://www.google.com/accounts/o8/id?id=AItOawnjX_O_VChwfDjcKlcRHbc2Aah8aYQlqts"
+ nickname="Ludovic"
+ subject="comment 3"
+ date="2014-10-05T11:01:39Z"
+ content="""
+I have the same problem.
+I checked again, removed the previous annex/ directory and created a new annex repository from the webapp. No annex/.git/index file is created.
+
+I then add a remore ssh server. The synchronisation fails with the error in the log: \"/storage/emulated/legacy/annex/.git/index: copyFile: does not exist (No such file or directory).\"
+
+Running \"git annex sync\" solved the problem.
+"""]]
diff --git a/doc/forum/Android_version_does_not_sync/comment_4_1ad06842dd96be14e66cde67f32d50f8._comment b/doc/forum/Android_version_does_not_sync/comment_4_1ad06842dd96be14e66cde67f32d50f8._comment
new file mode 100644
index 0000000000..3ae4f77847
--- /dev/null
+++ b/doc/forum/Android_version_does_not_sync/comment_4_1ad06842dd96be14e66cde67f32d50f8._comment
@@ -0,0 +1,26 @@
+[[!comment format=mdwn
+ username="https://www.google.com/accounts/o8/id?id=AItOawkyMs6WP6rXEnFqQg9oooOjYZzqO1j9_kE"
+ nickname="John"
+ subject="Same problem here"
+ date="2015-03-29T02:30:36Z"
+ content="""
+**Here's the log right from the first start up after creating a repo, and also the warning I get when I open the webapp:**
+
+[2015-03-29 10:18:01 JST] main: starting assistant version 5.20150327-ge414f2d [2015-03-29 10:18:01 JST] 
+Cronner: You should enable consistency checking to protect your data. (scanning...) 
+
+[2015-03-29 10:18:02 JST] Watcher: Performing startup scan (started...) 
+
+Detected a filesystem without fifo support. 
+
+Disabling ssh connection caching. 
+
+Detected a crippled filesystem. (recording state in git...) error: cannot run git-upload-pack '/storage/extSdCard/annex-sync': No such file or directory fatal: unable to fork
+
+
+**Each time I start the webapp, I get this warning:**
+
+Transfer scanner crashed: /storage/sdcard0/annex/.git/index: copyFile does not exist (No such file or directory)
+
+
+"""]]
diff --git a/doc/forum/Annex_contents_just_disappeared__63__.mdwn b/doc/forum/Annex_contents_just_disappeared__63__.mdwn
new file mode 100644
index 0000000000..d18e2bf91c
--- /dev/null
+++ b/doc/forum/Annex_contents_just_disappeared__63__.mdwn
@@ -0,0 +1,12 @@
+Joey,
+
+I have git-annex now to manage many of the repositories on my system.  I have them both on my local machine, and on a very large file server, and a backup system on the Internet.
+
+Today I went to look at a file in one of my annexes and it wasn't there.  This really surprised me.  But what surprised me most is that around 90% of the files in *all* of my annexes on both my local system and my file server are completely missing.  Only the Internet backup system has them.
+
+How could something like this happen, when I haven't been interacting with these annexes at all during this time?  Can you think of any scenario that might lead to this?  This is pretty much the absolute worst case scenario for an archival data system.
+
+I am running on Mac OS X 10.8, using GHC 7.6.3 to build git-annex, and I keep my git-annex binary updated often.
+
+Thanks,
+  John
diff --git a/doc/forum/Annex_contents_just_disappeared__63__/comment_1_4ab5ca00f912c0c95fabc10f2d9600d3._comment b/doc/forum/Annex_contents_just_disappeared__63__/comment_1_4ab5ca00f912c0c95fabc10f2d9600d3._comment
new file mode 100644
index 0000000000..fe4edbc27c
--- /dev/null
+++ b/doc/forum/Annex_contents_just_disappeared__63__/comment_1_4ab5ca00f912c0c95fabc10f2d9600d3._comment
@@ -0,0 +1,12 @@
+[[!comment format=mdwn
+ username="https://me.yahoo.com/a/2grhJvAC049fJnvALDXek.6MRZMTlg--#eec89"
+ nickname="John"
+ subject="comment 1"
+ date="2013-08-23T03:06:00Z"
+ content="""
+Wait, I think this comes from a backend switch.  I changed my .gitattributes file at one point to read:
+
+* annex.backend=SHA512E annex.numcopies=2
+
+I thought this would just affect new files, not existing annexed content.  Could this do it?
+"""]]
diff --git a/doc/forum/Annex_contents_just_disappeared__63__/comment_2_657f737c5d64d440aa133ddb41408fbc._comment b/doc/forum/Annex_contents_just_disappeared__63__/comment_2_657f737c5d64d440aa133ddb41408fbc._comment
new file mode 100644
index 0000000000..e30ff02ca1
--- /dev/null
+++ b/doc/forum/Annex_contents_just_disappeared__63__/comment_2_657f737c5d64d440aa133ddb41408fbc._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="https://me.yahoo.com/a/2grhJvAC049fJnvALDXek.6MRZMTlg--#eec89"
+ nickname="John"
+ subject="comment 2"
+ date="2013-08-23T03:14:15Z"
+ content="""
+Yes, that was the problem, sorry to bother you.  None of my data is gone, it's just sitting there under unknown names.  I can roll back to when it knew the names, and migrate them forward.
+"""]]
diff --git a/doc/forum/Annex_contents_just_disappeared__63__/comment_3_9b4c35feb14b37d43d053d7430da9abf._comment b/doc/forum/Annex_contents_just_disappeared__63__/comment_3_9b4c35feb14b37d43d053d7430da9abf._comment
new file mode 100644
index 0000000000..6c40a1cac9
--- /dev/null
+++ b/doc/forum/Annex_contents_just_disappeared__63__/comment_3_9b4c35feb14b37d43d053d7430da9abf._comment
@@ -0,0 +1,12 @@
+[[!comment format=mdwn
+ username="http://joeyh.name/"
+ ip="4.154.0.63"
+ subject="comment 3"
+ date="2013-08-23T04:28:45Z"
+ content="""
+If you change the backend, and then in one repository you run `git annex migrate`, other repositories that have the old keys will not know about the new names. For this reason, then multiple repositories have the files, it's best to run it redundantly in each repository.
+
+TBH, migration is a bit of a PITA because of this. Best to aovid it in most cases.
+
+Git-annex will never perform a migration begind your back. You must have run `git annex migrate` at some point. You can check the git history for details.
+"""]]
diff --git a/doc/forum/Annex_contents_just_disappeared__63__/comment_4_c3625409652bff5f2165260803269a8a._comment b/doc/forum/Annex_contents_just_disappeared__63__/comment_4_c3625409652bff5f2165260803269a8a._comment
new file mode 100644
index 0000000000..236f8cfdef
--- /dev/null
+++ b/doc/forum/Annex_contents_just_disappeared__63__/comment_4_c3625409652bff5f2165260803269a8a._comment
@@ -0,0 +1,10 @@
+[[!comment format=mdwn
+ username="https://me.yahoo.com/a/2grhJvAC049fJnvALDXek.6MRZMTlg--#eec89"
+ nickname="John"
+ subject="comment 4"
+ date="2013-08-30T06:06:16Z"
+ content="""
+Just to confirm, this wasn't a git-annex problem at all, but just a misstep during migration as you suggested.
+
+I think what I'm going to do now is to just wipe the slate clean and start over again, by using `unannex -fast` on all the files, wiping `.git`, and then adding everything back in using my new default backend of SHA512E.  The bigger pain is doing the same thing on all the servers where I have this data (to avoid having to upload it again), but in such a way that I'm not replicating file history.  I think I should be able to just clone, `mv $OLDREPO/.git/annex/objects objects`, `git annex add objects`, `git rm -r --cached objects`, and then everything should be good without even needing a new commit on the remote machine, just a git-annex sync.
+"""]]
diff --git a/doc/forum/Annex_dropping_files.mdwn b/doc/forum/Annex_dropping_files.mdwn
new file mode 100644
index 0000000000..7ec19caa15
--- /dev/null
+++ b/doc/forum/Annex_dropping_files.mdwn
@@ -0,0 +1,12 @@
+It seems like one of my repos dropped files by itself, my setup involved 3 distributed repos (non bare repos, not using assistant), A/B/C then one client died I've created another one in its place called D cloned from B and marked C as dead. I have now A/B/D. I have updated files in A synced it to D (central) I then tried dropping unused, it listed none (I've dropped the files from A which has full copy) at this point I thought B did not sync to it maybe thats why it did not drop them. I went home sync my laptop B (B dropped its share of unused files) but cental D still show no unused. 
+
+A-D has full copies of the repo
+B has partial.
+
+At this point I checked out a old commit from a week ago, checked where the links for the files are pointing to, they point to no where files are not there. Repo sized reported with du -hcs show both repos are the same size (132GB) looks like D managed to drop unused files by itself. I am wondering what caused this?
+
+My annex version is,
+
+git-annex version: 4.20130902-g307537a
+
+on all clients.
diff --git a/doc/forum/Annex_dropping_files/comment_1_62fbea95248fda2ff075b5a8952a728f._comment b/doc/forum/Annex_dropping_files/comment_1_62fbea95248fda2ff075b5a8952a728f._comment
new file mode 100644
index 0000000000..3844d3f0d3
--- /dev/null
+++ b/doc/forum/Annex_dropping_files/comment_1_62fbea95248fda2ff075b5a8952a728f._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="http://joeyh.name/"
+ ip="4.154.4.51"
+ subject="comment 1"
+ date="2013-09-12T21:01:03Z"
+ content="""
+If you have just created repository D and have just copied some files to it, then it will only contain those files, and not old deleted files or old versions of files. You can run `git annex copy --all --to D` to copy all those files to D, although if you're only wanting to dropunused them, there is not much point in that..
+"""]]
diff --git a/doc/forum/Annex_keeps_dropping_content.mdwn b/doc/forum/Annex_keeps_dropping_content.mdwn
new file mode 100644
index 0000000000..d3ff813f85
--- /dev/null
+++ b/doc/forum/Annex_keeps_dropping_content.mdwn
@@ -0,0 +1,9 @@
+I'm experiencing a strange behaviour...
+
+I have a few annexes (local, ssh, gitolite) in my group, some of them  are "manual standard" and some are "backup".
+
+If I 'get' files on my laptop annex and then I 'sync --content' local files are dropped! Local is "manual standard".
+
+Am I doing something wrong?
+
+Thanks
diff --git a/doc/forum/Annex_slow_on_Windows__47__direct_mode.mdwn b/doc/forum/Annex_slow_on_Windows__47__direct_mode.mdwn
new file mode 100644
index 0000000000..57247ac5dd
--- /dev/null
+++ b/doc/forum/Annex_slow_on_Windows__47__direct_mode.mdwn
@@ -0,0 +1,11 @@
+I've been using annex for some weeks now and while I more and more love how it behaves on my Linux machines, I just can't get it working on Windows...
+
+The setup consists of two Ubuntu machines (one being an always-on server) and a Windows laptop I keep for the occasional moment of gaming. My wife's Windows machine is a candidate to join the annex setup, as well as some other computers I still use every now and then.
+
+The first thing I started annexing was my pictures folder. It consists of about 40k files and occupies about 350GB. The Ubuntu server is running a hidden service as a substitute for a DynDNS with ssh basically the only thing going out/in. First, I couldn't get annex to properly sync its data via TOR, but when I instead set up a directory special remote on the server (with the annex repository there being "bare", i.e. not containing any actual data) I got them to sync as they should.
+
+Not the Windows machine though. It's just too slow. It seems that on every sync, every add, every anything it scans... well, everything. I added some files and after an hour I checked the resource monitor which files were being accessed and it seemed to be every last one of the files present in the pictures folder. I'm not sure what's going on there, but it's really getting to the point of being a dealbreaker... I think it's something with annex only supporting direct mode repositories on Windows and all files just lying around (as that's really the only large difference between those setups), although I'm of course not entirely sure.
+
+I'd really like to run annex in indirect mode on Windows. I can't really find any information on that (except those few "official" pages that just state that annex is running in direct mode on Windows). I know that creating symlinks needs elevated priviledges on Windows, but accepting a prompt seems to be much more realistic than waiting hours for a sync.
+
+So... Is there any way to get indirect running on Windows? And if there isn't, is there any other way to speed up direct mode / Windows performance?
diff --git a/doc/forum/Annex_slow_on_Windows__47__direct_mode/comment_1_d80839f4582fc2a6269db31e30e1dbab._comment b/doc/forum/Annex_slow_on_Windows__47__direct_mode/comment_1_d80839f4582fc2a6269db31e30e1dbab._comment
new file mode 100644
index 0000000000..c68c727385
--- /dev/null
+++ b/doc/forum/Annex_slow_on_Windows__47__direct_mode/comment_1_d80839f4582fc2a6269db31e30e1dbab._comment
@@ -0,0 +1,9 @@
+[[!comment format=mdwn
+ username="https://www.google.com/accounts/o8/id?id=AItOawnOSgFb3l7nL3Fs7Y9gPGJJjFiV7aJ1tek"
+ nickname="Phil"
+ subject="Same boat"
+ date="2014-07-03T19:38:15Z"
+ content="""
+I'd love to know this too. I'm setting up on 2 Windows 8.1 machines (with a 3rd Windows 7 hopefully joining the party as well as a couple of Linux boxes) but doing anything is really slow. I just ran git annex status on a 50GB, 7000 file Music repo and it took 6 minutes. I've not even setup the remotes yet but as soon as I'd added and sync'd, things slowed down. I'm running on a nippy SSD and CPU usage is around 12%.
+
+"""]]
diff --git a/doc/forum/Annex_slow_on_Windows__47__direct_mode/comment_2_593e1e01b70a2b6a15ad6bca09a80c7b._comment b/doc/forum/Annex_slow_on_Windows__47__direct_mode/comment_2_593e1e01b70a2b6a15ad6bca09a80c7b._comment
new file mode 100644
index 0000000000..0e357f4d6d
--- /dev/null
+++ b/doc/forum/Annex_slow_on_Windows__47__direct_mode/comment_2_593e1e01b70a2b6a15ad6bca09a80c7b._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="https://www.google.com/accounts/o8/id?id=AItOawnOSgFb3l7nL3Fs7Y9gPGJJjFiV7aJ1tek"
+ nickname="Phil"
+ subject="comment 2"
+ date="2014-07-03T19:39:12Z"
+ content="""
+Fortunately, git annex sync is pretty quick
+"""]]
diff --git a/doc/forum/Annex_slow_on_Windows__47__direct_mode/comment_3_c4e4c596f31aa97645fe1e1527dc2c31._comment b/doc/forum/Annex_slow_on_Windows__47__direct_mode/comment_3_c4e4c596f31aa97645fe1e1527dc2c31._comment
new file mode 100644
index 0000000000..a97ccda2e3
--- /dev/null
+++ b/doc/forum/Annex_slow_on_Windows__47__direct_mode/comment_3_c4e4c596f31aa97645fe1e1527dc2c31._comment
@@ -0,0 +1,12 @@
+[[!comment format=mdwn
+ username="http://joeyh.name/"
+ ip="209.250.56.55"
+ subject="comment 3"
+ date="2014-07-04T20:57:54Z"
+ content="""
+Yall need to tell me your git-annex versions. [[devblog/day_183__rubbing_sticks_together]] fixed a horrible slowdown that indeed causes git-annex on windows to unnecessarily look at lots of files.
+
+Also, is this using the git-annex assistant, or using git-annex at the command line that's being slow?
+
+As to using symlinks on windows, cygwin's build of git is able to handle them somehow. I have not tried to get it working with git-annex, but it seems at least possible that indirect mode could be accomplished that way.
+"""]]
diff --git a/doc/forum/Annex_slow_on_Windows__47__direct_mode/comment_4_92db0b99ada9af15a5383da41397ebd7._comment b/doc/forum/Annex_slow_on_Windows__47__direct_mode/comment_4_92db0b99ada9af15a5383da41397ebd7._comment
new file mode 100644
index 0000000000..39e472606b
--- /dev/null
+++ b/doc/forum/Annex_slow_on_Windows__47__direct_mode/comment_4_92db0b99ada9af15a5383da41397ebd7._comment
@@ -0,0 +1,19 @@
+[[!comment format=mdwn
+ username="divB"
+ ip="171.67.172.69"
+ subject="Also in the same boat"
+ date="2014-07-07T01:12:18Z"
+ content="""
+Even worse here. git-annex looked (and still looks) so promising for exactly I want (share >100GB photos/data with my girlfriend which are on my private DSL connected server but each of us should only download whatever he currently needs for offline usage and everything should stay consistent, e.g. is one edits, adds or deletes files). However, after long testing in Windows I gave up. I had so many issues and speed was one of them. So I looked again at other solutions (ownCloud pydio) but they are not good either. unison is one thing that just works perfectly but it is not suited to sync huge stuff where just parts are available on one machine.
+
+I still put much hope in git-annex that it's useable at some point also in Windows. If there would be another of these kickstarters for that, I'd be in.
+
+One BIG dealbreaker for me is SSH. It popped up a couple of times already there is inconsistency between ssh.exe, plink.exe and %GIT_SSH%.
+But the biggest issue is that ssh.exe in Windows does not do connection sharing: For each single file a new SSH session is created! That way it works only for huge files (movies) where the overhead is small. In my tests I had smaller files, that's just unuseable.
+
+I know it's a conundrum. unison just leaves one ssh connection open, that would be nice. I know however that this requires bigger changes. But what about for example starting something in background (during a git-annex command) that leaves the SSH session open and pumps data via some IPC?
+
+Last but not least: links. I agree windows links are unuseable. But would it really be real links? What about just basic *.lnk files?
+
+
+"""]]
diff --git a/doc/forum/Annex_slow_on_Windows__47__direct_mode/comment_5_3d628c9db9ebdfd5bff92af105c47719._comment b/doc/forum/Annex_slow_on_Windows__47__direct_mode/comment_5_3d628c9db9ebdfd5bff92af105c47719._comment
new file mode 100644
index 0000000000..9b7b166b3a
--- /dev/null
+++ b/doc/forum/Annex_slow_on_Windows__47__direct_mode/comment_5_3d628c9db9ebdfd5bff92af105c47719._comment
@@ -0,0 +1,13 @@
+[[!comment format=mdwn
+ username="divB"
+ ip="171.67.172.69"
+ subject="comment 5"
+ date="2014-07-07T01:16:21Z"
+ content="""
+One more thing: integration in TortoiseGit would be sooooooooo great! You would then even be able to let novice computer users use git-annex.
+
+I added a feature request long time ago but it's on hold: http://code.google.com/p/tortoisegit/issues/detail?id=2166
+
+I do hope that git-annex becomes more popular to increase the chance that it will be built into that...
+
+"""]]
diff --git a/doc/forum/Annex_slow_on_Windows__47__direct_mode/comment_6_db7965fa928c093233769ed52b2fcd43._comment b/doc/forum/Annex_slow_on_Windows__47__direct_mode/comment_6_db7965fa928c093233769ed52b2fcd43._comment
new file mode 100644
index 0000000000..90b6094fa8
--- /dev/null
+++ b/doc/forum/Annex_slow_on_Windows__47__direct_mode/comment_6_db7965fa928c093233769ed52b2fcd43._comment
@@ -0,0 +1,10 @@
+[[!comment format=mdwn
+ username="Idhup"
+ ip="178.175.139.138"
+ subject="Connection sharing workaround"
+ date="2014-07-07T10:20:23Z"
+ content="""
+I had the same issues but was able to work around most of them. I simply created keypairs on my Windows machines that allowed me to log in to my Ubuntu server. Means that while the overhead does make it smaller (I guess), the whole sync process can simply be run in the background without me having to check for a password prompt for 30 minutes, just to miss the short period during which I can enter it. Also, the actual object storage was moved to a directory special remote on the server, so Windows clients can simply automount a samba share and push the content over that line, which is less of a hazzle.
+
+Also, the speed issue got *much* better when I added all files. I still had a lot of files just lying around on my Windows machine. Once I added them all, the sync process sped up by several magnitudes. So, if https://git-annex.branchable.com/forum/fatal:_Out_of_memory__63___mmap_failed:_No_such_file_or_directory/ can be solved, I guess I'm happy on Windows.
+"""]]
diff --git a/doc/forum/Annex_slow_on_Windows__47__direct_mode/comment_7_674f52c5e5484207db403b18efc986c6._comment b/doc/forum/Annex_slow_on_Windows__47__direct_mode/comment_7_674f52c5e5484207db403b18efc986c6._comment
new file mode 100644
index 0000000000..2b87a11e01
--- /dev/null
+++ b/doc/forum/Annex_slow_on_Windows__47__direct_mode/comment_7_674f52c5e5484207db403b18efc986c6._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="Idhup"
+ ip="178.175.139.138"
+ subject="typo"
+ date="2014-07-07T10:23:23Z"
+ content="""
+Slower. It's not smaller, it's slower. Gee.
+"""]]
diff --git a/doc/forum/Any_plan_for___40__official__41___OpenBSD_support__63__.mdwn b/doc/forum/Any_plan_for___40__official__41___OpenBSD_support__63__.mdwn
new file mode 100644
index 0000000000..389201f60c
--- /dev/null
+++ b/doc/forum/Any_plan_for___40__official__41___OpenBSD_support__63__.mdwn
@@ -0,0 +1,10 @@
+Hello
+
+Is OpenBSD support on the roadmap for the near future?
+Both cabal and stack are problematic on OpenBSD (for now). 
+
+I revisit this issue once in a while and while it is literally the last thing keeping me on Linux, my whole daily workflow revolves around git-annex, so I can't be without it. I also can't have it break on me every time I will need to update.
+
+Just wanted to know if anybody else had success or if a more formal way of installation on OpenBSD is in the cards.
+
+Thanks a lot
diff --git a/doc/forum/Any_plan_for___40__official__41___OpenBSD_support__63__/comment_1_0d87a7d0f6ae556deb999403a2fb5461._comment b/doc/forum/Any_plan_for___40__official__41___OpenBSD_support__63__/comment_1_0d87a7d0f6ae556deb999403a2fb5461._comment
new file mode 100644
index 0000000000..b84bfc11f5
--- /dev/null
+++ b/doc/forum/Any_plan_for___40__official__41___OpenBSD_support__63__/comment_1_0d87a7d0f6ae556deb999403a2fb5461._comment
@@ -0,0 +1,13 @@
+[[!comment format=mdwn
+ username="joey"
+ subject="""comment 1"""
+ date="2018-08-17T20:16:02Z"
+ content="""
+People have told me once or twice that they were able to build git-annex on
+openbsd using stack. If it doesn't build, file a bug (unless it's clearly a
+toolchain/OS issue).
+
+If someone can confirm it builds, I could try to set up an openbsd
+autobuilder in a VM. But I have no openbsd experience so would probably
+need some hand-holding.
+"""]]
diff --git a/doc/forum/Any_plan_for___40__official__41___OpenBSD_support__63__/comment_2_c7fc9be88dad09f71774664b233bc8a3._comment b/doc/forum/Any_plan_for___40__official__41___OpenBSD_support__63__/comment_2_c7fc9be88dad09f71774664b233bc8a3._comment
new file mode 100644
index 0000000000..d04ed82d62
--- /dev/null
+++ b/doc/forum/Any_plan_for___40__official__41___OpenBSD_support__63__/comment_2_c7fc9be88dad09f71774664b233bc8a3._comment
@@ -0,0 +1,11 @@
+[[!comment format=mdwn
+ username="hobbes@b2cacef69071743c3a831e60511062f7e014e52f"
+ nickname="hobbes"
+ avatar="http://cdn.libravatar.org/avatar/44b70169c4d862b3619812c360aa8f1e"
+ subject="Building with OpenBSD"
+ date="2018-08-23T19:30:09Z"
+ content="""
+I have successfully built git-annex using plain cabal on OpenBSD, and ran into significant problems using stack.
+
+The main thing was that I needed to add a lot of swap space, and the build took forever. Running out of memory pretty much just killed the machine. I would love to have an official build available. Would make my life easier too, and put the effort in one place.
+"""]]
diff --git a/doc/forum/Any_plan_for___40__official__41___OpenBSD_support__63__/comment_3_89330ea5d2754d59766a096e0c35d791._comment b/doc/forum/Any_plan_for___40__official__41___OpenBSD_support__63__/comment_3_89330ea5d2754d59766a096e0c35d791._comment
new file mode 100644
index 0000000000..90969aaf85
--- /dev/null
+++ b/doc/forum/Any_plan_for___40__official__41___OpenBSD_support__63__/comment_3_89330ea5d2754d59766a096e0c35d791._comment
@@ -0,0 +1,19 @@
+[[!comment format=mdwn
+ username="bnyn"
+ avatar="http://cdn.libravatar.org/avatar/086ca5cb07dc2022f472433c1c5edac1"
+ subject="Update"
+ date="2018-09-11T19:29:39Z"
+ content="""
+Hey joey (and anyone else struggling to compile on OpenBSD)
+
+Just thought I'd give you this update. Git annex now successfully builds for me on OpenBSD 6.3
+I think a recent change regarding the building without assistant fixed it.
+I was playing around with constraints to try to force my way around problematic hackage package versions and decided to start from scrap a couple of days later.
+6.20180808 then simply worked for me. 
+
+Please be aware that I built using cabal with the following flags, otherwise as per the instructions on the \"fromsource\" page here.
+\"-s3 -webdav -assistant -webapp -pairing -torrentparser -magicmime -dbus\"
+
+I don't know if you have intentionally worked on support for the BSDs or another fix applied here as well, but it works now. 
+Thanks a lot, joey! I really appreciate this.
+"""]]
diff --git a/doc/forum/Any_way_to_add_a_folder_called___34__.git__34___to_git-annex__63__.mdwn b/doc/forum/Any_way_to_add_a_folder_called___34__.git__34___to_git-annex__63__.mdwn
new file mode 100644
index 0000000000..425301a0da
--- /dev/null
+++ b/doc/forum/Any_way_to_add_a_folder_called___34__.git__34___to_git-annex__63__.mdwn
@@ -0,0 +1,9 @@
+I'm trying to use git-annex as a way to sync fairly large folders between machines. These folders contain documents, projects, ...
+
+There are several issues with files that just won't get added to my git-annex repository:
+
+- files mentioned in .gitignore are not added -- this is good.
+- files starting with a dot are not added unless I explicitly mention them to git add -- this is strange, but I can work around it.
+- folders called .git/ are not added no matter what I try. git annex add --force doesn't help, either.
+
+Any idea how I could solve the third issue? It might not even be specific to git-annex... maybe git just works that way... but I'd be grateful for hints.
diff --git a/doc/forum/Any_way_to_add_a_folder_called___34__.git__34___to_git-annex__63__/comment_1_51f4392e718d857e2f155d6217727a53._comment b/doc/forum/Any_way_to_add_a_folder_called___34__.git__34___to_git-annex__63__/comment_1_51f4392e718d857e2f155d6217727a53._comment
new file mode 100644
index 0000000000..3aecd39881
--- /dev/null
+++ b/doc/forum/Any_way_to_add_a_folder_called___34__.git__34___to_git-annex__63__/comment_1_51f4392e718d857e2f155d6217727a53._comment
@@ -0,0 +1,10 @@
+[[!comment format=mdwn
+ username="https://www.google.com/accounts/o8/id?id=AItOawlq4ClC5EMN1Vq1DpWXAqP5TiDnCK1mSfk"
+ nickname="Jonas"
+ subject="comment 1"
+ date="2014-01-09T22:58:30Z"
+ content="""
+I just found [this thread](http://git-annex.branchable.com/forum/Tracking_a_directory_with_some_hg_and_git_repositories/) where a user was discouraged from syncing git repositories with annex.
+
+I've used unison in the past for this. Sure, it would occasionally create a few unneeded object files; but these got removed during the next \"git gc\", and never corrupted a repository... Do people know of any better way to handle this problem?
+"""]]
diff --git a/doc/forum/Any_way_to_add_a_folder_called___34__.git__34___to_git-annex__63__/comment_2_9698c4a8f0d8785ee89a6228e0e85ca9._comment b/doc/forum/Any_way_to_add_a_folder_called___34__.git__34___to_git-annex__63__/comment_2_9698c4a8f0d8785ee89a6228e0e85ca9._comment
new file mode 100644
index 0000000000..0c933fe312
--- /dev/null
+++ b/doc/forum/Any_way_to_add_a_folder_called___34__.git__34___to_git-annex__63__/comment_2_9698c4a8f0d8785ee89a6228e0e85ca9._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="https://launchpad.net/~timo-linux"
+ nickname="Tim O'Callaghan"
+ subject="How about automatically adding it as a submodule?"
+ date="2014-01-11T10:30:57Z"
+ content="""
+Adding as a submodule and automatically adding the annexe destination as a remote? that should work in most cases..
+"""]]
diff --git a/doc/forum/Archive_USB_drive_not_working_as_documented.mdwn b/doc/forum/Archive_USB_drive_not_working_as_documented.mdwn
new file mode 100644
index 0000000000..73c2db6de2
--- /dev/null
+++ b/doc/forum/Archive_USB_drive_not_working_as_documented.mdwn
@@ -0,0 +1,171 @@
+I'm hoping to use git-annex in a project to manage large image files and maintain archived copies of them on removable USB drives. But I can't get this to work the way it's described in the walkthrough and in the "git-annex/tips/offline archive drives" example. Here's what I did:
+
+    marshal@home[~]> git-annex version
+    git-annex version: 5.20140412ubuntu1
+    build flags: Assistant Webapp Webapp-secure Pairing Testsuite S3 WebDAV Inotify DBus DesktopNotify XMPP DNS Feeds Quvi TDFA CryptoHash
+    key/value backends: SHA256E SHA1E SHA512E SHA224E SHA384E SKEIN256E SKEIN512E SHA256 SHA1 SHA512 SHA224 SHA384 SKEIN256 SKEIN512 WORM URL
+    remote types: git gcrypt S3 bup directory rsync web webdav tahoe glacier hook external
+
+Create the local repo:
+
+    marshal@home[~]> mkdir ~/annex
+    marshal@home[~]> cd ~/annex
+    marshal@home[~/annex]> git init
+    Initialized empty Git repository in /home/marshal/annex/.git/
+    marshal@home[~/annex]> git annex init "my laptop"
+    init my laptop ok
+    (Recording state in git...)
+
+Create the remote repo (assume the warning can be ignored?)
+
+    marshal@home[~/annex]> cd /media/marshal/Sony\ USB/
+    marshal@home[/media/marshal/Sony USB]> git clone ~/annex
+    Cloning into 'annex'...
+    done.
+    warning: remote HEAD refers to nonexistent ref, unable to checkout.
+    marshal@home[/media/marshal/Sony USB]> cd annex
+    marshal@home[/media/marshal/Sony USB/annex]> git annex init "usbdrive"
+    init usbdrive ok
+    (Recording state in git...)
+
+Make them remotes of each other
+
+    marshal@home[/media/marshal/Sony USB/annex]> git remote add laptop ~/annex
+    marshal@home[/media/marshal/Sony USB/annex]> cd ~/annex
+    marshal@home[~/annex]> git remote add usbdrive /media/marshal/Sony\ USB/annex/
+
+Add some files into the local repo
+
+    marshal@home[~/annex]> cp ~/Desktop/file* .
+    marshal@home[~/annex]> git annex add .
+    add file1.m3u ok
+    add file2.sh ok
+    add file3.pdf ok
+    (Recording state in git...)
+    marshal@home[~/annex]> git commit -a -m "added files"
+    [master (root-commit) 4ea3be7] added files
+     3 files changed, 3 insertions(+)
+     create mode 120000 file1.m3u
+     create mode 120000 file2.sh
+     create mode 120000 file3.pdf
+
+Change the preferred content settings
+
+    marshal@home[~/annex]> git annex group usbdrive archive
+    group usbdrive ok
+    (Recording state in git...)
+    marshal@home[~/annex]> git annex wanted usbdrive standard
+    wanted usbdrive ok
+    (Recording state in git...)
+
+Sync to the usbdrive repo
+
+    marshal@home[~/annex]> git annex sync usbdrive
+    commit  ok
+    pull usbdrive 
+    remote: Counting objects: 7, done.
+    remote: Compressing objects: 100% (3/3), done.
+    remote: Total 5 (delta 1), reused 0 (delta 0)
+    Unpacking objects: 100% (5/5), done.
+    From /media/marshal/Sony USB/annex
+     * [new branch]      git-annex  -> usbdrive/git-annex
+    ok
+    (merging usbdrive/git-annex into git-annex...)
+    (Recording state in git...)
+    push usbdrive 
+    Counting objects: 29, done.
+    Delta compression using up to 8 threads.
+    Compressing objects: 100% (22/22), done.
+    Writing objects: 100% (25/25), 2.44 KiB | 0 bytes/s, done.
+    Total 25 (delta 3), reused 0 (delta 0)
+    To /media/marshal/Sony USB/annex/
+     * [new branch]      git-annex -> synced/git-annex
+     * [new branch]      master -> synced/master
+    ok
+
+Copy eveything to the usbdrive repo
+
+    marshal@home[~/annex]> git-annex copy --auto --to usbdrive
+    copy file1.m3u (to usbdrive...) 
+    SHA256E-s64061--c388e2c927cb4c88c11b1b5a8e166c4d8080dc528d53954dad1ee6d4933ce3c7.m3u
+             64,061 100%   29.84MB/s    0:00:00 (xfr#1, to-chk=0/1)
+    ok
+    copy file2.sh (to usbdrive...) 
+    SHA256E-s318--b8729560cc759d2256903feaa4ba65994ccbac94f30515d52a39083ad52e1bad.sh
+                318 100%    0.00kB/s    0:00:00 (xfr#1, to-chk=0/1)
+    ok
+    copy file3.pdf (to usbdrive...) 
+    SHA256E-s32239--9b051067d5ef9f0cb4a01750901bf6bee3f9348e5b10138c8ef416ac8d51e5df.pdf
+             32,239 100%    0.00kB/s    0:00:00 (xfr#1, to-chk=0/1)
+    ok
+    (Recording state in git...)
+
+Check to see where git-annex "thinks" the files are
+
+    marshal@home[~/annex]> git-annex whereis file1.m3u 
+    whereis file1.m3u (2 copies) 
+          	80fa5a68-3843-4e25-b4c9-53a8c51ba9ff -- here (my laptop)
+           	d642b276-5f9b-4a5c-a42e-ea29cd9540b5 -- usbdrive
+    ok
+
+Check the filesystem results
+
+    marshal@home[~/annex]> ls -la ~/annex
+    total 48
+    drwxrwxr-x  3 marshal marshal  4096 Jul 19 14:59 .
+    drwxr-xr-x 99 marshal marshal 24576 Jul 19 14:55 ..
+    lrwxrwxrwx  1 marshal marshal   194 Jul 19 14:59 file1.m3u -> .git/annex/objects/jq/MW/SHA256E-s64061--c388e2c927cb4c88c11b1b5a8e166c4d8080dc528d53954dad1ee6d4933ce3c7.m3u/SHA256E-s64061--c388e2c927cb4c88c11b1b5a8e166c4d8080dc528d53954dad1ee6d4933ce3c7.m3u
+    lrwxrwxrwx  1 marshal marshal   188 Jul 19 14:59 file2.sh -> .git/annex/objects/1J/2K/SHA256E-s318--b8729560cc759d2256903feaa4ba65994ccbac94f30515d52a39083ad52e1bad.sh/SHA256E-    s318--b8729560cc759d2256903feaa4ba65994ccbac94f30515d52a39083ad52e1bad.sh
+    lrwxrwxrwx  1 marshal marshal   194 Jul 19 14:59 file3.pdf -> .git/annex/objects/77/Wj/SHA256E-s32239--9b051067d5ef9f0cb4a01750901bf6bee3f9348e5b10138c8ef416ac8d51e5df.pdf/SHA256E-s32239--9b051067d5ef9f0cb4a01750901bf6bee3f9348e5b10138c8ef416ac8d51e5df.pdf
+    drwxrwxr-x  9 marshal marshal  4096 Jul 19 15:04 .git
+    marshal@home[~/annex]> ls -la /media/marshal/Sony\ USB/annex/
+    total 4
+    drwx------ 1 marshal marshal  144 Jul 19 14:56 .
+    drwx------ 1 marshal marshal 4096 Jul 19 14:56 ..
+    drwx------ 1 marshal marshal  448 Jul 19 14:57 .git
+
+Why is there nothing in the usbdrive repo? And worse, why is the whereis command saying there are 2 copies?
+
+But if I sync from the usbdrive repo, the files are copied
+
+    marshal@home[~/annex]> cd /media/marshal/Sony\ USB/annex/
+    marshal@home[/media/marshal/Sony USB/annex]> git-annex sync laptop
+    (merging synced/git-annex origin/git-annex into git-annex...)
+    (Recording state in git...)
+    commit  ok
+    pull laptop 
+    remote: Counting objects: 23, done.
+    remote: Compressing objects: 100% (8/8), done.
+    remote: Total 11 (delta 3), reused 0 (delta 0)
+    Unpacking objects: 100% (11/11), done.
+    From /home/marshal/annex
+     * [new branch]      git-annex  -> laptop/git-annex
+     * [new branch]      master     -> laptop/master
+     * [new branch]      synced/master -> laptop/synced/master
+    
+    
+    Already up-to-date.
+    ok
+    (merging laptop/git-annex into git-annex...)
+    (Recording state in git...)
+    push laptop 
+    Counting objects: 56, done.
+    Delta compression using up to 8 threads.
+    Compressing objects: 100% (24/24), done.
+    Writing objects: 100% (33/33), 2.34 KiB | 0 bytes/s, done.
+    Total 33 (delta 15), reused 0 (delta 0)
+    To /home/marshal/annex
+     * [new branch]      git-annex -> synced/git-annex
+    ok
+    marshal@home[/media/marshal/Sony USB/annex]> ls -la /media/marshal/Sony\ USB/annex/
+    total 6
+    drwx------ 1 marshal marshal  456 Jul 19 15:07 .
+    drwx------ 1 marshal marshal 4096 Jul 19 14:56 ..
+    lrwxrwxrwx 1 marshal marshal  396 Jul 19 15:07 file1.m3u -> .git/annex/objects/jq/MW/SHA256E-s64061--c388e2c927cb4c88c11b1b5a8e166c4d8080dc528d53954dad1ee6d4933ce3c7.m3u/SHA256E-s64061--c388e2c927cb4c88c11b1b5a8e166c4d8080dc528d53954dad1ee6d4933ce3c7.m3u
+    lrwxrwxrwx 1 marshal marshal  384 Jul 19 15:07 file2.sh -> .git/annex/objects/1J/2K/SHA256E-s318--b8729560cc759d2256903feaa4ba65994ccbac94f30515d52a39083ad52e1bad.sh/SHA256E-s318--b8729560cc759d2256903feaa4ba65994ccbac94f30515d52a39083ad52e1bad.sh
+    lrwxrwxrwx 1 marshal marshal  396 Jul 19 15:07 file3.pdf -> .git/annex/objects/77/Wj/SHA256E-s32239--9b051067d5ef9f0cb4a01750901bf6bee3f9348e5b10138c8ef416ac8d51e5df.pdf/SHA256E-s32239--9b051067d5ef9f0cb4a01750901bf6bee3f9348e5b10138c8ef416ac8d51e5df.pdf
+    drwx------ 1 marshal marshal  448 Jul 19 15:07 .git
+
+Why did I have to sync from the usbdrive? I expected the repo setup and the --auto option to copy the files. What am I missing?
+
+Any help much appreciated.
diff --git a/doc/forum/Archive_USB_drive_not_working_as_documented/comment_1_59de1e101e5e427abb1df3a71c6f1b04._comment b/doc/forum/Archive_USB_drive_not_working_as_documented/comment_1_59de1e101e5e427abb1df3a71c6f1b04._comment
new file mode 100644
index 0000000000..e7f5eea08a
--- /dev/null
+++ b/doc/forum/Archive_USB_drive_not_working_as_documented/comment_1_59de1e101e5e427abb1df3a71c6f1b04._comment
@@ -0,0 +1,23 @@
+[[!comment format=mdwn
+ username="https://www.google.com/accounts/o8/id?id=AItOawmcS2gG2R_AIiBBOsWuxGf1yEn_l797jjU"
+ nickname="Mesar"
+ subject="comment 1"
+ date="2014-07-20T00:50:15Z"
+ content="""
+
+When you did:
+
+    marshal@home[~/annex]> git-annex copy --auto --to usbdrive
+
+this copies the file content (see .git/annex/objects/*), and updates the metadata in both repos.
+The reason why you are not seeing the files on the usb is because the metadata has not been merged into your master branch.
+This is verified by the output of:
+
+    marshal@home[/media/marshal/Sony USB/annex]> git-annex sync laptop
+
+To verify that you indeed do have the files and their content, don't do the above sync, but instead do:
+
+    marshal@home[/media/marshal/Sony USB/annex]> git annex merge
+
+the local metadata in the sync/git-annex branch will be merged in, and the local directory will be populated with the new symlinks to the content which you already have in the /media/marshal/Sony USB/annex/.git/annex/objects/*
+"""]]
diff --git a/doc/forum/Archive_USB_drive_not_working_as_documented/comment_2_3541fdd31d398a494a8fa452ac2c277f._comment b/doc/forum/Archive_USB_drive_not_working_as_documented/comment_2_3541fdd31d398a494a8fa452ac2c277f._comment
new file mode 100644
index 0000000000..1ec09b0ef0
--- /dev/null
+++ b/doc/forum/Archive_USB_drive_not_working_as_documented/comment_2_3541fdd31d398a494a8fa452ac2c277f._comment
@@ -0,0 +1,14 @@
+[[!comment format=mdwn
+ username="https://www.google.com/accounts/o8/id?id=AItOawnwSZggfi3YE0EAuxAB9jT6pMcB73V8ae4"
+ nickname="Marshal"
+ subject="comment 2"
+ date="2014-07-20T01:33:25Z"
+ content="""
+Thank you Mesar! 
+
+I think I'm beginning to understand. Indeed, the files were there on the usbdrive repo (in .git/annex/objects). No need to sync or merge. I was able to drop a file from the laptop repo and get it back from the usbdrive.
+
+What is a bit unnerving, is that the symlinks are not present on the usbdrive, so it looks empty except for the .git directory.
+
+Thanks so much for clearing this up. More to explore... 
+"""]]
diff --git a/doc/forum/Assistant_-_pausing___47___using_lock_file_to_get_atomic__40__ish__41___states__63__.mdwn b/doc/forum/Assistant_-_pausing___47___using_lock_file_to_get_atomic__40__ish__41___states__63__.mdwn
new file mode 100644
index 0000000000..b75a6eb273
--- /dev/null
+++ b/doc/forum/Assistant_-_pausing___47___using_lock_file_to_get_atomic__40__ish__41___states__63__.mdwn
@@ -0,0 +1,10 @@
+Is there functionality to pause the assistant so that a group of changes all get committed at once?
+
+The case I'm thinking of is using git-annex to manage distribution of my bup backups.  I want to make sure that all the backup files are written - and then the par2 files generated from `fsck`ing - before they are committed and shipped.
+
+Naively, I'm looking for something as simple as a lockfile to touch at the start of the backup and then remove afterwards.
+
+Of course I could just run `git annex sync --content` after the backup is run, but, I'm was thinking that the assistant might be more set and forget...
+
+
+Does anyone do pausing or something similar?
diff --git a/doc/forum/Assistant_-_pausing___47___using_lock_file_to_get_atomic__40__ish__41___states__63__/comment_1_44c97c54066ca0487d8903cc01a356a3._comment b/doc/forum/Assistant_-_pausing___47___using_lock_file_to_get_atomic__40__ish__41___states__63__/comment_1_44c97c54066ca0487d8903cc01a356a3._comment
new file mode 100644
index 0000000000..42f21761fd
--- /dev/null
+++ b/doc/forum/Assistant_-_pausing___47___using_lock_file_to_get_atomic__40__ish__41___states__63__/comment_1_44c97c54066ca0487d8903cc01a356a3._comment
@@ -0,0 +1,10 @@
+[[!comment format=mdwn
+ username="olaf"
+ avatar="http://cdn.libravatar.org/avatar/4ae498d3d6ee558d6b65caa658f72572"
+ subject="manual mode?"
+ date="2017-08-17T07:35:41Z"
+ content="""
+It looks like setting the repo to 'manual mode' is sufficient.  I just need to then remember to `git annex add` to ensure everything is committed.
+
+Not ideal, but...
+"""]]
diff --git a/doc/forum/Assistant_-_pausing___47___using_lock_file_to_get_atomic__40__ish__41___states__63__/comment_2_98eb671596e84466e24004f55724660a._comment b/doc/forum/Assistant_-_pausing___47___using_lock_file_to_get_atomic__40__ish__41___states__63__/comment_2_98eb671596e84466e24004f55724660a._comment
new file mode 100644
index 0000000000..7478367b92
--- /dev/null
+++ b/doc/forum/Assistant_-_pausing___47___using_lock_file_to_get_atomic__40__ish__41___states__63__/comment_2_98eb671596e84466e24004f55724660a._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="CandyAngel"
+ avatar="http://cdn.libravatar.org/avatar/15c0aade8bec5bf004f939dd73cf9ed8"
+ subject="comment 2"
+ date="2017-08-17T10:22:54Z"
+ content="""
+You might also want to check out the annex.alwayscommit and annex.autocommit options.
+"""]]
diff --git a/doc/forum/Assistant_-_pausing___47___using_lock_file_to_get_atomic__40__ish__41___states__63__/comment_3_e3c506ae998a8ae681727d3c432f95d2._comment b/doc/forum/Assistant_-_pausing___47___using_lock_file_to_get_atomic__40__ish__41___states__63__/comment_3_e3c506ae998a8ae681727d3c432f95d2._comment
new file mode 100644
index 0000000000..c11b693578
--- /dev/null
+++ b/doc/forum/Assistant_-_pausing___47___using_lock_file_to_get_atomic__40__ish__41___states__63__/comment_3_e3c506ae998a8ae681727d3c432f95d2._comment
@@ -0,0 +1,13 @@
+[[!comment format=mdwn
+ username="olaf"
+ avatar="http://cdn.libravatar.org/avatar/4ae498d3d6ee558d6b65caa658f72572"
+ subject="comment 3"
+ date="2017-08-18T04:37:57Z"
+ content="""
+Awesome.  Thanks.  I'd missed these entirely.
+
+Looks like my lockfile should be as simple as a combination of those two.
+
+
+I'll report back a working example when I get time to test.
+"""]]
diff --git a/doc/forum/Assistant_Droping_Files.mdwn b/doc/forum/Assistant_Droping_Files.mdwn
new file mode 100644
index 0000000000..e26edc92dc
--- /dev/null
+++ b/doc/forum/Assistant_Droping_Files.mdwn
@@ -0,0 +1,8 @@
+I have a setup as follows,
+
+3 drives on a remote ssh server regular git annex repos.
+One untrusted clone of this repository on my laptop. 
+
+On my laptop in webapp I have set num of copies to 2 (each file does have two copies on the server already). all repos plus laptop is set to manual mode. What happens is whenever I get a file to the laptop it would drop it from the disk on the server. Even though laptop is marked as untrusted this does not happen when using annex at the command line when I do a git annex get . --auto on the server it does not take the files on the untrusted repos into account which is what I want so I can drop files without moving them back to the server.
+
+Is this the intended behavior or am I doing something wrong?
diff --git a/doc/forum/Assistant__58___configure_auto-sync.mdwn b/doc/forum/Assistant__58___configure_auto-sync.mdwn
new file mode 100644
index 0000000000..48c4dc4552
--- /dev/null
+++ b/doc/forum/Assistant__58___configure_auto-sync.mdwn
@@ -0,0 +1,11 @@
+I have large central repositories of data. Therefore, on each client I want to save part data(to save space of disk). In command line I do 
+
+	[...]
+	git-annex webapp
+	git-annex drop [DeleteContentDirectory]
+	[...]
+
+After this command Assistant performs automatic synchronization getting content of files from this directory(DeleteContentDirectory), but I don't want. I want it's was only symlink of file in this directory.
+
+How can I configure Assistant which files have to get content on the client?  It's possible?
+ 
diff --git a/doc/forum/Assistant__58___configure_auto-sync/comment_1_c8cabd38114582bbdbad49f2d81959d7._comment b/doc/forum/Assistant__58___configure_auto-sync/comment_1_c8cabd38114582bbdbad49f2d81959d7._comment
new file mode 100644
index 0000000000..34fffb8463
--- /dev/null
+++ b/doc/forum/Assistant__58___configure_auto-sync/comment_1_c8cabd38114582bbdbad49f2d81959d7._comment
@@ -0,0 +1,13 @@
+[[!comment format=mdwn
+ username="https://www.google.com/accounts/o8/id?id=AItOawmWg4VvDTer9f49Y3z-R0AH16P4d1ygotA"
+ nickname="Tobias"
+ subject="comment 1"
+ date="2013-08-26T07:44:22Z"
+ content="""
+You should set your repository to \"manual\" mode instead of \"client\" mode.
+
+But then no data will be synced at all by the assistant, only metadata. You would have to do \"get/drop\" manually for commandline for all files.
+
+Alternatively you could use the special \"archive\" folders that are supported by the assistant.
+
+"""]]
diff --git a/doc/forum/Assistant__58___run_with___34__sync_--fast__34___to_automatically_use_only_available_remotes.mdwn b/doc/forum/Assistant__58___run_with___34__sync_--fast__34___to_automatically_use_only_available_remotes.mdwn
new file mode 100644
index 0000000000..3a6e4a7108
--- /dev/null
+++ b/doc/forum/Assistant__58___run_with___34__sync_--fast__34___to_automatically_use_only_available_remotes.mdwn
@@ -0,0 +1,18 @@
+I have a set of machines, call them A, B, C, I am trying to keep synced. However, all three of them do not always see all the other two (e.g., I might need a vpn tunnel up to see C from A, etc, or C is down, etc). 
+Everything works as expected (i.e., content gets from one machine to the others) if in the webapp I disable the non-reachable nodes (or, equivalently, I set annex-sync to false in .git/config). But I wonder if there is a way to have git annex not use the non-reachable remotes. Using git-annex sync --fast does not seem to do it (since all remotes I guess have the same annex-cost value).
+
+
+I understand a possible solution involves using 
+
+```
+remote..annex-cost-command
+```
+
+so that I set the cost of the non-available remote to a large value. But then, I'd need to tell the assistant to always use, by default
+
+
+```
+git-annex sync --fast --content
+```
+
+and I do not know how to make it use "--fast". 
diff --git a/doc/forum/Assistant_fails_to_authenticate_to_rsync_remote.mdwn b/doc/forum/Assistant_fails_to_authenticate_to_rsync_remote.mdwn
new file mode 100644
index 0000000000..29a851595c
--- /dev/null
+++ b/doc/forum/Assistant_fails_to_authenticate_to_rsync_remote.mdwn
@@ -0,0 +1,17 @@
+I have an rsync special remote for rsync.net. Recently I noticed that the assistant was not uploading files to this remote. When I look at the log in the webapp I see that it fails to authenticate:
+
+    [2016-04-22 13:28:49.145178] NetWatcherFallback: Syncing with rsync.net
+    Permission denied, please try again.
+    Permission denied, please try again.
+    Received disconnect from 1.1.1.1 port 22:2: Too many authentication failures for 12345
+    Connection to host.rsync.net closed by remote host.
+    rsync: connection unexpectedly closed (0 bytes received so far) [sender]
+    rsync error: unexplained error (code 255) at io.c(226) [sender=3.1.2]
+
+(For privacy I replaced the IP of the rsync.net host with 1.1.1.1 and the numerical user with 12345 and the hostname with host.rsync.net. In the actual log output those 3 things have real values that are correct!)
+
+If I change into this annex in the terminal I can `git annex copy . --to rsync.net` and everything uploads properly, so it is just the assistant which is failing to authenticate. I'm not sure how to go about troubleshooting this. I do use an ssh key to authenticate to the remote. Maybe the assistant doesn't know about this key?
+
+I did create this repo and add the remote manually, only later telling the assistant about the repo through the webapp.
+
+What should I look into to debug this?
diff --git a/doc/forum/Assistant_fails_to_authenticate_to_rsync_remote/comment_1_fdbadd7ae4c0236301944136ac62346a._comment b/doc/forum/Assistant_fails_to_authenticate_to_rsync_remote/comment_1_fdbadd7ae4c0236301944136ac62346a._comment
new file mode 100644
index 0000000000..51b6496716
--- /dev/null
+++ b/doc/forum/Assistant_fails_to_authenticate_to_rsync_remote/comment_1_fdbadd7ae4c0236301944136ac62346a._comment
@@ -0,0 +1,12 @@
+[[!comment format=mdwn
+ username="joey"
+ subject="""comment 1"""
+ date="2016-05-03T19:18:29Z"
+ content="""
+The assistant just uses whatever ssh configuration you have.
+
+Desktop environments have password safes and ssh agents. If the assistant
+is being started up outside the desktop environment, it won't have access
+to those. Try stopping the assistant and re-starting it inside a terminal
+in your desktop environment.
+"""]]
diff --git a/doc/forum/Assistant_fails_to_authenticate_to_rsync_remote/comment_2_a395406abd3eb4cf07dc6467506a7e77._comment b/doc/forum/Assistant_fails_to_authenticate_to_rsync_remote/comment_2_a395406abd3eb4cf07dc6467506a7e77._comment
new file mode 100644
index 0000000000..e816a5392b
--- /dev/null
+++ b/doc/forum/Assistant_fails_to_authenticate_to_rsync_remote/comment_2_a395406abd3eb4cf07dc6467506a7e77._comment
@@ -0,0 +1,7 @@
+[[!comment format=mdwn
+ username="annexuser"
+ subject="comment 2"
+ date="2016-05-04T19:16:31Z"
+ content="""
+That makes sense! I'm starting the assistant via systemd, so it probably doesn't have access to my environment to get the ssh keys. Thanks!
+"""]]
diff --git a/doc/forum/Assistant_loosing_advantages__63__.mdwn b/doc/forum/Assistant_loosing_advantages__63__.mdwn
new file mode 100644
index 0000000000..113a436bf7
--- /dev/null
+++ b/doc/forum/Assistant_loosing_advantages__63__.mdwn
@@ -0,0 +1,10 @@
+Hey,
+
+I write an article for the univerity about git-annex assistant and I'm a little bit confused about the function of the assistant. 
+Because an advantage of git-annex is that you can hold different files at different places in one repository, but this isn't anymore, or? The assistant syncs every file to every place of the repository. And this makes it just to another Dropbox (not direct the same) pentant or I'm not correct?
+
+The transfer group is just for transfer files to other repositories, which are not connected with the network or?
+
+I'm using ppa, ubuntu 12.04 lts, vers, 2013-12-13
+
+JP Lührig
diff --git a/doc/forum/Assistant_loosing_advantages__63__/comment_1_cdbc827d9e00aeeaefafe45de64b8d2c._comment b/doc/forum/Assistant_loosing_advantages__63__/comment_1_cdbc827d9e00aeeaefafe45de64b8d2c._comment
new file mode 100644
index 0000000000..171019b16c
--- /dev/null
+++ b/doc/forum/Assistant_loosing_advantages__63__/comment_1_cdbc827d9e00aeeaefafe45de64b8d2c._comment
@@ -0,0 +1,10 @@
+[[!comment format=mdwn
+ username="http://joeyh.name/"
+ ip="209.250.56.87"
+ subject="good question.."
+ date="2013-12-19T17:04:53Z"
+ content="""
+The assistant necessarily simplifies some things git-annex does, so that it can automate them, and this includes keeping files synced to everywhere by default. There are ways to configure the assistant to only keep a subset of files on a computer, including [[archive_directories|assistant/archival_walkthrough]]. Underneath, there is the quite powerful [[preferred_content]] DSL, which can be used to configure lots more interesting behavior about what goes where. So far only some common scenarios have been made easily available in the webappp, but there's potential for lots more configurability here, if a good UI for it is built.
+
+I think the assistant is probably better than dropbox at handling external removable media, used for backups or archiving. (But I have not evaluated, or actually even used dropbox myself. My goal is *not* to copy dropbox.) And it's better at encryption, and it allows using a variety of cloud services rather than only one provider.
+"""]]
diff --git a/doc/forum/Assistant_not_syncing_to_Rsync.mdwn b/doc/forum/Assistant_not_syncing_to_Rsync.mdwn
new file mode 100644
index 0000000000..18adc733d5
--- /dev/null
+++ b/doc/forum/Assistant_not_syncing_to_Rsync.mdwn
@@ -0,0 +1,15 @@
+I have 3 remotes for an annex - one on my laptop, one on a USB drive that's plugged in 50% of the time, and an rsync special remote on a friend's machine that I can access using SSH.
+
+I have tried various things (annex copy, numcopies=3) to make it move data to the rsync remote, but it doesn't seem to want to work. I have tried git annex sync. I added the remote repo using the webapp, and it looked as if it was copying my files for a bit then stopped. I have the remote repo setup as a backup in the webapp.
+
+The sizes of the directorys:
+    ~ $ du -sh Documents
+    126M	Documents
+    ~ $ du -sh /Volumes/Backup/Documents
+    227M	/Volumes/Backup/Documents
+
+and the remote
+    [c0g@womb Tom]$ du -sh MacDocuments/
+    21M	MacDocuments/
+
+I'm also curious as to why the usb drive repo is so much bigger than the thing I'm copying, but I'll worry about that later.
diff --git a/doc/forum/Assistant_not_syncing_to_Rsync/comment_1_2178a7fc0d66643e84597b0938ef65f2._comment b/doc/forum/Assistant_not_syncing_to_Rsync/comment_1_2178a7fc0d66643e84597b0938ef65f2._comment
new file mode 100644
index 0000000000..e701b7cd8f
--- /dev/null
+++ b/doc/forum/Assistant_not_syncing_to_Rsync/comment_1_2178a7fc0d66643e84597b0938ef65f2._comment
@@ -0,0 +1,10 @@
+[[!comment format=mdwn
+ username="http://joeyh.name/"
+ ip="4.154.7.238"
+ subject="comment 1"
+ date="2013-01-10T18:36:04Z"
+ content="""
+What does it say when you run:
+
+  git annex copy --to $rsyncremote
+"""]]
diff --git a/doc/forum/Assistant_not_syncing_to_Rsync/comment_2_650651398443e128c2adc6a2a2d320d0._comment b/doc/forum/Assistant_not_syncing_to_Rsync/comment_2_650651398443e128c2adc6a2a2d320d0._comment
new file mode 100644
index 0000000000..28d043c6d5
--- /dev/null
+++ b/doc/forum/Assistant_not_syncing_to_Rsync/comment_2_650651398443e128c2adc6a2a2d320d0._comment
@@ -0,0 +1,12 @@
+[[!comment format=mdwn
+ username="https://www.google.com/accounts/o8/id?id=AItOawkjWgG_L90szs6o4E8cBusol57zkcDhg7c"
+ nickname="Tom"
+ subject="comment 2"
+ date="2013-01-10T18:40:13Z"
+ content="""
+....
+copy FrenchBasicCourserevised-Volume1-StudentText.pdf (checking Womb_MacDocuments...) ok
+....
+
+for each file in the directory.
+"""]]
diff --git a/doc/forum/Assistant_not_syncing_to_Rsync/comment_3_e6d0c9620b148acc72342862a8b4cfef._comment b/doc/forum/Assistant_not_syncing_to_Rsync/comment_3_e6d0c9620b148acc72342862a8b4cfef._comment
new file mode 100644
index 0000000000..99b78a73c4
--- /dev/null
+++ b/doc/forum/Assistant_not_syncing_to_Rsync/comment_3_e6d0c9620b148acc72342862a8b4cfef._comment
@@ -0,0 +1,10 @@
+[[!comment format=mdwn
+ username="http://joeyh.name/"
+ ip="4.154.7.238"
+ subject="comment 3"
+ date="2013-01-10T18:44:00Z"
+ content="""
+Sounds like copy works then?
+
+What version of git-annex do you have installed? A bug was fixed on December 11th, before that the assistant never stored things in repositories configured as backups.
+"""]]
diff --git a/doc/forum/Assistant_not_syncing_to_Rsync/comment_4_b91f9febdb8b69d8b487ba4ea08c119a._comment b/doc/forum/Assistant_not_syncing_to_Rsync/comment_4_b91f9febdb8b69d8b487ba4ea08c119a._comment
new file mode 100644
index 0000000000..8303babe3d
--- /dev/null
+++ b/doc/forum/Assistant_not_syncing_to_Rsync/comment_4_b91f9febdb8b69d8b487ba4ea08c119a._comment
@@ -0,0 +1,11 @@
+[[!comment format=mdwn
+ username="https://www.google.com/accounts/o8/id?id=AItOawkjWgG_L90szs6o4E8cBusol57zkcDhg7c"
+ nickname="Tom"
+ subject="comment 4"
+ date="2013-01-10T18:48:16Z"
+ content="""
+Sorry for lack of extra information: The sizes of the remote folder is still at 21MB, well below even the size of the PDF I posted about (that's about 75MB). Copy claims to work, but I see no network use and the remote repo doesn't grow. I install using cabal on my max yesterday, git-annex version: 3.20130107
+
+
+
+"""]]
diff --git a/doc/forum/Assistant_not_syncing_to_Rsync/comment_5_c5ad7c1546a17d8459c995c9c8c26414._comment b/doc/forum/Assistant_not_syncing_to_Rsync/comment_5_c5ad7c1546a17d8459c995c9c8c26414._comment
new file mode 100644
index 0000000000..653b7cc51f
--- /dev/null
+++ b/doc/forum/Assistant_not_syncing_to_Rsync/comment_5_c5ad7c1546a17d8459c995c9c8c26414._comment
@@ -0,0 +1,31 @@
+[[!comment format=mdwn
+ username="http://joeyh.name/"
+ ip="4.154.7.238"
+ subject="comment 5"
+ date="2013-01-10T18:58:49Z"
+ content="""
+Ok, so copy is checking if the file is present, and it seems to think it is, so doesn't try again to send it.
+
+Here's what you need to do: Run `git annex copy FrenchBasicCourserevised-Volume1-StudentText.pdf  --to Womb_MacDocuments --debug`
+
+It'll output something like this:
+
+    copy new (checking rsync...) [2013-01-10 14:51:43 JEST] read: rsync [\"localhost:/tmp/r/f87/4d5/'SHA256E-s0--e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855/SHA256E-s0--e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855'\"]
+
+Now try running that command at the shell, and check its exit status. For the above, I ran:
+
+    rsync \"localhost:/tmp/r/f87/4d5/'SHA256E-s0--e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855/SHA256E-s0--e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855'\"; echo $?
+
+It output:
+
+    -rw-r--r--           0 2013/01/10 14:51:40 SHA256E-s0--e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855
+    0
+
+Compare with the same command but with the filename mangled. 
+
+    rsync: link_stat \"/tmp/r/f87/4d5/'SHA256E-s0--e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855/mangled'\" failed: No such file or directory (2)
+    rsync error: some files/attrs were not transferred (see previous errors) (code 23) at main.c(1536) [Receiver=3.0.9]
+    27
+
+It seems that rsync might be exiting 0 for content that's not there, in your case. This test will confirm or disprove that hypothesis and might provide some useful debug info.
+"""]]
diff --git a/doc/forum/Assistant_not_syncing_to_Rsync/comment_6_4c12587f972eced91c5128d4885800b5._comment b/doc/forum/Assistant_not_syncing_to_Rsync/comment_6_4c12587f972eced91c5128d4885800b5._comment
new file mode 100644
index 0000000000..3746fa2e37
--- /dev/null
+++ b/doc/forum/Assistant_not_syncing_to_Rsync/comment_6_4c12587f972eced91c5128d4885800b5._comment
@@ -0,0 +1,30 @@
+[[!comment format=mdwn
+ username="https://www.google.com/accounts/o8/id?id=AItOawkjWgG_L90szs6o4E8cBusol57zkcDhg7c"
+ nickname="Tom"
+ subject="comment 6"
+ date="2013-01-11T07:44:11Z"
+ content="""
+I did what you said for a file, and checked the remote size as well. Local size of this file was 11.4 meg, remote was 1.3 megabytes.
+
+    copy Paradox Interactive/Crusader Kings II/save games/Yngling/1066_09_15_2.ck2 (checking Womb_MacDocuments...) [2013-01-11 07:30:26 GMT] read: rsync [\"c0g@git-    annex-c0g:/raid/Tom/MacDocuments/87f/7f7/'GPGHMACSHA1--0148d706315885fac29c3a1dc0ec37b2835739a7/GPGHMACSHA1--0148d706315885fac29c3a1dc0ec37b2835739a7'\"]
+     ok
+
+    rsync c0g@git-annex-c0g:/raid/Tom/MacDocuments/87f/7f7/'GPGHMACSHA1--0148d706315885fac29c3a1dc0ec37b2835739a7/GPGHMACSHA1--0148d706315885fac29c3a1dc0ec37b2835739a7'
+    -rw-r--r--     1290004 2013/01/10 17:37:14 GPGHMACSHA1--0148d706315885fac29c3a1dc0ec37b2835739a7
+    0
+
+    ~/Documents ± la Paradox\ Interactive/Crusader\ Kings\ II/save\ games/Yngling
+    total 44728
+    drwx------  4 c0g  staff       136 10 Jan 17:52 . 
+    drwx------  3 c0g  staff       102 10 Nov 20:53 ..
+    -rw-r--r--  1 c0g  staff  11413707 10 Jan 17:52 1066_09_15.ck2
+    -rw-r--r--  1 c0g  staff  11484448 10 Jan 17:52 1066_09_15_2.ck2
+
+    [c0g@womb GPGHMACSHA1--0148d706315885fac29c3a1dc0ec37b2835739a7]$ ls -la
+    total 1268
+    drwxr-xr-x 2 c0g users    4096 Jan 10 12:37 .
+    drwxr-xr-x 3 c0g users    4096 Jan 10 12:37 ..
+    -rw-r--r-- 1 c0g users 1290004 Jan 10 12:37 GPGHMACSHA1--0148d706315885fac29c3a1dc0ec37b2835739a7
+
+
+"""]]
diff --git a/doc/forum/Assistant_not_syncing_to_Rsync/comment_7_6ecaaee9316bcf0c65688676d60fc055._comment b/doc/forum/Assistant_not_syncing_to_Rsync/comment_7_6ecaaee9316bcf0c65688676d60fc055._comment
new file mode 100644
index 0000000000..38f755c86e
--- /dev/null
+++ b/doc/forum/Assistant_not_syncing_to_Rsync/comment_7_6ecaaee9316bcf0c65688676d60fc055._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="https://www.google.com/accounts/o8/id?id=AItOawkjWgG_L90szs6o4E8cBusol57zkcDhg7c"
+ nickname="Tom"
+ subject="comment 7"
+ date="2013-01-11T07:52:17Z"
+ content="""
+I just compressed the documents folder, and it compresses down to 20 MB... I'm going to guess it stores as a compressed encrypted archive? How embarrasing for me!
+"""]]
diff --git a/doc/forum/Assistant_not_syncing_to_Rsync/comment_8_daa9a9a6188afa0394833e1b682f7cd4._comment b/doc/forum/Assistant_not_syncing_to_Rsync/comment_8_daa9a9a6188afa0394833e1b682f7cd4._comment
new file mode 100644
index 0000000000..f0a1c88fc7
--- /dev/null
+++ b/doc/forum/Assistant_not_syncing_to_Rsync/comment_8_daa9a9a6188afa0394833e1b682f7cd4._comment
@@ -0,0 +1,10 @@
+[[!comment format=mdwn
+ username="http://joeyh.name/"
+ ip="4.154.7.238"
+ subject="comment 8"
+ date="2013-01-11T17:20:19Z"
+ content="""
+Aha! Yes, encryption includes compression. 
+
+So, only remaining question is why your USB drive seems to be using 2x the uncompressed file size. One possibility would be it could have old versions of files, or deleted files still stored in it. You could try running \"git annex unused\" in its repository to find them. Other than that, I'm not sure what it could be. You can look around in its .git/annex directory, and compare to your laptop's repository, and perhaps see what's using the space.
+"""]]
diff --git a/doc/forum/Attempting_to_repair_fails_with_everincreasing_deltas.mdwn b/doc/forum/Attempting_to_repair_fails_with_everincreasing_deltas.mdwn
new file mode 100644
index 0000000000..e71bf040bf
--- /dev/null
+++ b/doc/forum/Attempting_to_repair_fails_with_everincreasing_deltas.mdwn
@@ -0,0 +1,21 @@
+Hello,
+
+I am using the latest git-annex with the webui having two local folders (one over nfs) connected as a full backup group. 
+
+On every reboot I get a jumping ball icon with the text:
+
+"Attempting to repair [tr2]"
+
+And the later the text:
+
+"failed to sync to tr2"
+
+The debug log is filled with entries like this, where the number of deltas is increasing:
+
+[2014-08-26 20:34:50 CEST] PushRetrier: Syncing with tr2 
+fatal: pack has 15 unresolved deltas
+error: unpack failed: index-pack abnormal exit
+To /nfs/backup
+ ! [remote rejected] git-annex -> synced/git-annex (n/a (unpacker error))
+ ! [remote rejected] annex/direct/master -> synced/master (n/a (unpacker error))
+error: failed to push some refs to '/nfs/backup''
diff --git a/doc/forum/Attempting_to_repair_fails_with_everincreasing_deltas/comment_1_a8effe196e4a040630d183803768c5a1._comment b/doc/forum/Attempting_to_repair_fails_with_everincreasing_deltas/comment_1_a8effe196e4a040630d183803768c5a1._comment
new file mode 100644
index 0000000000..41f28a0cf2
--- /dev/null
+++ b/doc/forum/Attempting_to_repair_fails_with_everincreasing_deltas/comment_1_a8effe196e4a040630d183803768c5a1._comment
@@ -0,0 +1,10 @@
+[[!comment format=mdwn
+ username="anders7788"
+ ip="212.247.195.173"
+ subject="comment 1"
+ date="2014-08-27T06:59:08Z"
+ content="""
+My version is:
+
+assistant version 5.20140517.4
+"""]]
diff --git a/doc/forum/Attempting_to_repair_fails_with_everincreasing_deltas/comment_2_9f032e43b132bcad656e1337ab2551ad._comment b/doc/forum/Attempting_to_repair_fails_with_everincreasing_deltas/comment_2_9f032e43b132bcad656e1337ab2551ad._comment
new file mode 100644
index 0000000000..c139bd963f
--- /dev/null
+++ b/doc/forum/Attempting_to_repair_fails_with_everincreasing_deltas/comment_2_9f032e43b132bcad656e1337ab2551ad._comment
@@ -0,0 +1,10 @@
+[[!comment format=mdwn
+ username="anders7788"
+ ip="89.160.15.173"
+ subject="[Solved]"
+ date="2014-09-04T15:47:24Z"
+ content="""
+[Solved]
+
+The following of the steps at: http://git-annex.branchable.com/tips/what_to_do_when_a_repository_is_corrupted/ provided some clues and I discovered that the disk was broken. But a big thanks to git-annex which made it possible to discover this issue early!
+"""]]
diff --git a/doc/forum/Attempting_to_repair_fails_with_everincreasing_deltas/comment_3_5a09f65c77dce3c62236c13aa90a1191._comment b/doc/forum/Attempting_to_repair_fails_with_everincreasing_deltas/comment_3_5a09f65c77dce3c62236c13aa90a1191._comment
new file mode 100644
index 0000000000..1b4c2499ac
--- /dev/null
+++ b/doc/forum/Attempting_to_repair_fails_with_everincreasing_deltas/comment_3_5a09f65c77dce3c62236c13aa90a1191._comment
@@ -0,0 +1,10 @@
+[[!comment format=mdwn
+ username="http://joeyh.name/"
+ ip="209.250.56.54"
+ subject="comment 3"
+ date="2014-10-12T17:42:28Z"
+ content="""
+Sorry I didn't get to this earlier. I'm glad you managed to solve the problem, but there is something I am curious about: You say you had 2 folders set as full backups. Were these git repositories in their own right, or were they something else, eg directory special remotes?
+
+I ask because, if there were git repositories then the repair should have been able to pull the missing git objects from them, and fix your repository. Unless the broken disk somehow kept corrupting it, I suppose..
+"""]]
diff --git a/doc/forum/Auto_archiving.mdwn b/doc/forum/Auto_archiving.mdwn
new file mode 100644
index 0000000000..f32a648c23
--- /dev/null
+++ b/doc/forum/Auto_archiving.mdwn
@@ -0,0 +1,17 @@
+I've been toying with the idea of auto archiving files (that is - removing them from .) based on a set of rules.
+
+Git already provides attribute management for files and I put together a simple script that tries to achieve the following:
+
+* Look for files with X copies (including a local one)
+* Verify that the file is not recent (defined as not having been dropped or getted within X days )
+* Verify that the file has an annex.archive attribute
+* Archive if the above is met
+
+Here is the script:
+http://pastebin.com/53iLqyPd
+
+You just add the annex.archive attribute to files via .gitattributes to use
+
+The script runs in preview mode... exec with autoArchive.sh commit to drop the files.
+
+Open to thoughts / suggestions
diff --git a/doc/forum/Auto_ignore_non-reachable_repo.mdwn b/doc/forum/Auto_ignore_non-reachable_repo.mdwn
new file mode 100644
index 0000000000..c56872536e
--- /dev/null
+++ b/doc/forum/Auto_ignore_non-reachable_repo.mdwn
@@ -0,0 +1,15 @@
+Hello,
+
+having multiple remotes, which are often unreachable, I get command output like:
+
+    git annex drop .
+    drop 2017 Haus Idstein/26108065_10212412239435248_1115645256_o.jpg (locking horus...) (lockcontent failed) (checking horus...) ssh: Could not resolve hostname horus.local: Name or service not known
+    (checking S3...) ok
+    drop 2017 Haus Idstein/26177516_10212412238475224_326557077_o.jpg (locking horus...) (lockcontent failed) (checking horus...) ssh: Could not resolve hostname horus.local: Name or service not known
+    (checking S3...) ok
+    drop 2017 Haus Idstein/26178378_10212412238835233_820819105_o.jpg (locking horus...) (lockcontent failed) (checking horus...) ssh: Could not resolve hostname horus.local: Name or service not known
+    (checking S3...) ok
+
+which takes a long time, because it tries to lock horus first for each, waits for timeout and then tries the next remote. Of course I could use ```git annex drop --from=S3``` but I do not always remember that. Is there a way to auto ignore a remote which is not reachable?
+
+Thanks!
diff --git a/doc/forum/Auto_ignore_non-reachable_repo/comment_1_40d7aab5e43c75ede3414ed48f0c7066._comment b/doc/forum/Auto_ignore_non-reachable_repo/comment_1_40d7aab5e43c75ede3414ed48f0c7066._comment
new file mode 100644
index 0000000000..fe3b73a244
--- /dev/null
+++ b/doc/forum/Auto_ignore_non-reachable_repo/comment_1_40d7aab5e43c75ede3414ed48f0c7066._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="Horus"
+ avatar="http://cdn.libravatar.org/avatar/8f0ee08b98ea5bba76c3fe112c08851c"
+ subject="comment 1"
+ date="2018-02-28T09:31:26Z"
+ content="""
+Should be a edit: ```git annex drop --from=S3``` does not force checking the repo S3, but drops from S3.
+"""]]
diff --git a/doc/forum/Auto_ignore_non-reachable_repo/comment_2_74c297d8d5e9f7d131c461ed5e803071._comment b/doc/forum/Auto_ignore_non-reachable_repo/comment_2_74c297d8d5e9f7d131c461ed5e803071._comment
new file mode 100644
index 0000000000..4a80489010
--- /dev/null
+++ b/doc/forum/Auto_ignore_non-reachable_repo/comment_2_74c297d8d5e9f7d131c461ed5e803071._comment
@@ -0,0 +1,16 @@
+[[!comment format=mdwn
+ username="joey"
+ subject="""comment 2"""
+ date="2018-02-28T15:55:39Z"
+ content="""
+You can temporarily enable annex-ignore for a remote.
+
+For example:
+
+git -c remote.horus.annex-ignore=true git annex drop ...
+
+Adjusting the annex-cost of the remote may also be useful, so git-annex
+will avoid using that remote when it can use a lower-cost remote instead.
+The annex-cost-command could be used to detect the network where the remote
+access times out and make its cost be very high then.
+"""]]
diff --git a/doc/forum/Auto_sync_with_music_player.mdwn b/doc/forum/Auto_sync_with_music_player.mdwn
new file mode 100644
index 0000000000..e4a5543b45
--- /dev/null
+++ b/doc/forum/Auto_sync_with_music_player.mdwn
@@ -0,0 +1 @@
+I have a music directory under Nexus 5 which is a git-annex repository. If I sync the same with remote repository, newly added songs or modified songs(modifed id3 tags) will not sync with music player. But if I use Android File Transfer to transfer songs, it will sync music player. The songs which are transferred using git-annex will reflect in music player only after restart. Do we have to execute any command which will sync music player.
diff --git a/doc/forum/Auto_sync_with_music_player/comment_1_81ad1c15cfd753531c01dae8945d1caf._comment b/doc/forum/Auto_sync_with_music_player/comment_1_81ad1c15cfd753531c01dae8945d1caf._comment
new file mode 100644
index 0000000000..3ed75f9e9d
--- /dev/null
+++ b/doc/forum/Auto_sync_with_music_player/comment_1_81ad1c15cfd753531c01dae8945d1caf._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="http://joeyh.name/"
+ ip="209.250.56.146"
+ subject="comment 1"
+ date="2014-03-05T20:40:50Z"
+ content="""
+This is probably done via intents or a similar interface. git-annex is not yet able to use such android api's, but if there is a command line way to send the message that a file has changed, which there probably is, I could look into making git-annex use it.
+"""]]
diff --git a/doc/forum/Auto_sync_with_music_player/comment_2_a15e3f298c3d3faa5b3295355f9bb794._comment b/doc/forum/Auto_sync_with_music_player/comment_2_a15e3f298c3d3faa5b3295355f9bb794._comment
new file mode 100644
index 0000000000..fc0d069683
--- /dev/null
+++ b/doc/forum/Auto_sync_with_music_player/comment_2_a15e3f298c3d3faa5b3295355f9bb794._comment
@@ -0,0 +1,11 @@
+[[!comment format=mdwn
+ username="EvanDeaubl"
+ ip="67.128.198.190"
+ subject="Workaround using external app"
+ date="2014-03-07T14:57:39Z"
+ content="""
+A workaround for this is using an external app that pings the media scanner to rescan. It's an extra step to remember, but I haven't found it too much of a burden.
+
+I use this one: , but there are at least a dozen to choose from.
+
+"""]]
diff --git a/doc/forum/Auto_sync_with_music_player/comment_3_99f65a0efaf5d5f9b8ff530acc122860._comment b/doc/forum/Auto_sync_with_music_player/comment_3_99f65a0efaf5d5f9b8ff530acc122860._comment
new file mode 100644
index 0000000000..b0275abdc2
--- /dev/null
+++ b/doc/forum/Auto_sync_with_music_player/comment_3_99f65a0efaf5d5f9b8ff530acc122860._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="http://joeyh.name/"
+ ip="209.250.56.154"
+ subject="comment 3"
+ date="2014-03-18T19:59:07Z"
+ content="""
+Turns out there is an open todo about this with details: [[todo/Use_MediaScannerConnection_on_Android]]
+"""]]
diff --git a/doc/forum/Auto_update_not_working.mdwn b/doc/forum/Auto_update_not_working.mdwn
new file mode 100644
index 0000000000..1c9fa60bf3
--- /dev/null
+++ b/doc/forum/Auto_update_not_working.mdwn
@@ -0,0 +1,6 @@
+Hello,
+
+I've installed to ~/software/ using the prebuilt tarballs. I'm using the assistant with auto updates set to ask. Everytime I start the assistant it claims a new that a new version of git-annex has been installed. I click Finish Upgrade, it does the upgrade and it says to have finished upgrading to version 5.20141105-g8b19598. Next boot / restart everything starts again and it upgrades always to the same version.
+
+Thanks!
+Florian
diff --git a/doc/forum/Automatic___96__git_annex_get__96___after_invalidation_of_local_files_due_to_external_modification__63__.mdwn b/doc/forum/Automatic___96__git_annex_get__96___after_invalidation_of_local_files_due_to_external_modification__63__.mdwn
new file mode 100644
index 0000000000..50518e33ad
--- /dev/null
+++ b/doc/forum/Automatic___96__git_annex_get__96___after_invalidation_of_local_files_due_to_external_modification__63__.mdwn
@@ -0,0 +1,46 @@
+## Use case
+
+A laptop with a relatively small hard drive has copies of a subset of
+all annexed files.  When annexed files are changed externally and `git
+annex sync` is run on the laptop, the stale local copies are
+invalidated and their symlinks break.  How can I automatically fetch
+the updated versions of these previously locally-cached files?
+
+Because I only want a subset of files, I can't do
+
+    git annex add --not --in here --and --in superset.
+
+Because files may be renamed, the
+[[tips/automatically_getting_files_on_checkout/]] solution, by making
+`dir` specify the subset, will require manually and redundantly
+tracking renames.
+
+## Simple ( (?) ) feature addition to git-annex to support this
+
+When locally-cached files are invalidated by `git-annex sync`,
+git-annex could notify the user, and give them the option to
+`git-annex get` the invalidated files.  Bonus points if the mechanism
+allows this to be done at any point in the future, not just when
+running `git-annex sync`.  The idea is that git-annex could track
+which files, previously cached locally, have been invalidated
+*unintentionally* by syncs, and treat them differently from files,
+previously cached locally, that have been *intentionally* dropped
+using `git-annex drop` or `git-annex move`.
+
+## More generally
+
+The ability to specify a collection of files to always cache locally
+(something like a numcopies.here=1), which is robust to renames, would
+work.  The "robust to renames" part seems tricky in git: whereas svn
+attaches properties to files, and so properties are propagated by `svn
+mv`, I believe git attributes are only specified by patterns in
+.gitattributes files.
+
+## Related questions / possible approaches
+
+Other forum posts mention [[`git
+subtree`|forum/git-subtree_support__63__/]] and [[sparse git
+checkouts|forum/sparse_git_checkouts_with_annex/]], but I'm not
+familiar with these features and from reading those questions it's
+unclear if those approaches will work for me.  Does anyone more
+familiar see how to adapt one of those features to my use case?
diff --git a/doc/forum/Automatic___96__git_annex_get__96___after_invalidation_of_local_files_due_to_external_modification__63__/comment_1_dab1099ee56541c194de319c593f1268._comment b/doc/forum/Automatic___96__git_annex_get__96___after_invalidation_of_local_files_due_to_external_modification__63__/comment_1_dab1099ee56541c194de319c593f1268._comment
new file mode 100644
index 0000000000..fa561ef04c
--- /dev/null
+++ b/doc/forum/Automatic___96__git_annex_get__96___after_invalidation_of_local_files_due_to_external_modification__63__/comment_1_dab1099ee56541c194de319c593f1268._comment
@@ -0,0 +1,9 @@
+[[!comment format=mdwn
+ username="http://joeyh.name/"
+ subject="comment 1"
+ date="2012-06-04T19:56:05Z"
+ content="""
+Personally, I deal with this problem by having a directory, or directories where I put files that I want to have on my partial checkout laptop, and run `git annex get` in that directory. 
+
+It's not a perfect solution, but I don't know that a perfect solution exists.
+"""]]
diff --git a/doc/forum/Automatic___96__git_annex_get__96___after_invalidation_of_local_files_due_to_external_modification__63__/comment_2_b5faccf132fb47e3cda778a6600fd9ef._comment b/doc/forum/Automatic___96__git_annex_get__96___after_invalidation_of_local_files_due_to_external_modification__63__/comment_2_b5faccf132fb47e3cda778a6600fd9ef._comment
new file mode 100644
index 0000000000..2cd1001ef3
--- /dev/null
+++ b/doc/forum/Automatic___96__git_annex_get__96___after_invalidation_of_local_files_due_to_external_modification__63__/comment_2_b5faccf132fb47e3cda778a6600fd9ef._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="https://www.google.com/accounts/o8/id?id=AItOawlu7K3h7Ry1uDAU_ERYGuqt0LoGNJqGuRo"
+ nickname="Nathan"
+ subject="comment 2"
+ date="2012-06-04T21:01:52Z"
+ content="""
+Joey, that sounds reasonable; I'll try it. Thanks!
+"""]]
diff --git a/doc/forum/Automatic_commit_messages_for_git_annex_sync.mdwn b/doc/forum/Automatic_commit_messages_for_git_annex_sync.mdwn
new file mode 100644
index 0000000000..60b8e034d0
--- /dev/null
+++ b/doc/forum/Automatic_commit_messages_for_git_annex_sync.mdwn
@@ -0,0 +1 @@
+Not really important (who reads git annex commit messages anyways ;-)), but nice to have and maybe a nice task for someone who wants to play around with Haskell and git annex: It would be shiny if the auto-commit done by git annex sync would automatically create a sensible commmit message and description. E.g. if just one file is added, it could say „Added blubb“. If a few files are added, it could say „Added blubb, bla and n other files“, based on the file name length, and list the files in the long description. Lots of room for playing around :-)
diff --git a/doc/forum/Automatic_commit_messages_for_git_annex_sync/comment_1_ea2ec57bc695da4df8a30a35d433959d._comment b/doc/forum/Automatic_commit_messages_for_git_annex_sync/comment_1_ea2ec57bc695da4df8a30a35d433959d._comment
new file mode 100644
index 0000000000..c07dd49999
--- /dev/null
+++ b/doc/forum/Automatic_commit_messages_for_git_annex_sync/comment_1_ea2ec57bc695da4df8a30a35d433959d._comment
@@ -0,0 +1,15 @@
+[[!comment format=mdwn
+ username="http://joey.kitenet.net/"
+ nickname="joey"
+ subject="comment 1"
+ date="2012-02-24T20:52:51Z"
+ content="""
+Took me a minute to see this is not about [[bugs/wishlist:_more_descriptive_commit_messages_in_git-annex_branch]], but about the \"git-annex automatic sync\" message that is used when committing any changes currently on the master branch before doing the rest of the sync.
+
+So.. It would be pretty easy to `ls-files` the relevant files before the commit and make a message. Although this would roughly double the commit time in a large tree, since that would walk the whole tree again (git commit -a already does it once). Smarter approaches could be faster.. perhaps it could find unstaged files, stage them, generate the message, and then `git commit` the staged changes.
+
+But, would this really be useful? It's already easy to get `git log` to show a summary of the changes made in such a commit. So it's often seen as bad form to unnecessarily mention which files a commit changes in the commit message.
+
+Perhaps more useful would be to expand the current message with details like where the sync is being committed, or what
+remotes it's going to sync from, or something like that.
+"""]]
diff --git a/doc/forum/Automatic_commit_messages_for_git_annex_sync/comment_2_af71f53dbbca35d5a5c66ff131887ada._comment b/doc/forum/Automatic_commit_messages_for_git_annex_sync/comment_2_af71f53dbbca35d5a5c66ff131887ada._comment
new file mode 100644
index 0000000000..a71f72b9e4
--- /dev/null
+++ b/doc/forum/Automatic_commit_messages_for_git_annex_sync/comment_2_af71f53dbbca35d5a5c66ff131887ada._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="http://www.joachim-breitner.de/"
+ nickname="nomeata"
+ subject="comment 2"
+ date="2012-02-24T23:09:03Z"
+ content="""
+Yes, it is really a minor point. And indeed, \"git log --summary\" is pretty good already. But I’d still think that at least the title could deserves some love. Including the Hostname or the name of the repository there is a good idea as well.
+"""]]
diff --git a/doc/forum/Automatically_dropping_files.mdwn b/doc/forum/Automatically_dropping_files.mdwn
new file mode 100644
index 0000000000..dd47d87d68
--- /dev/null
+++ b/doc/forum/Automatically_dropping_files.mdwn
@@ -0,0 +1,7 @@
+I can make `git-annex` automatically fetch files with the [[/preferred content]] setting and the `--auto` flag, and it works almost exactly like I expect it to work.
+
+What I am missing is a way to make `git annex drop --auto` drop all files that are not wanted.
+
+I would like to work with metadata and tags in such a way that I can have clones (with views) that have only exactly those files available which carry a tag (done), and all other files automatically removed from the annex (unless that would be unsafe).
+
+Does anyone know how to achieve this?
diff --git a/doc/forum/Automatically_dropping_files/comment_1_f5fc608f9cb0edf3272b586b62050637._comment b/doc/forum/Automatically_dropping_files/comment_1_f5fc608f9cb0edf3272b586b62050637._comment
new file mode 100644
index 0000000000..28c9c0a565
--- /dev/null
+++ b/doc/forum/Automatically_dropping_files/comment_1_f5fc608f9cb0edf3272b586b62050637._comment
@@ -0,0 +1,12 @@
+[[!comment format=mdwn
+ username="http://joeyh.name/"
+ ip="216.145.95.162"
+ subject="comment 1"
+ date="2014-05-19T15:41:05Z"
+ content="""
+`git annex drop --auto` *does* automatically drop files that are not wanted, according to your preferred content settings.
+
+If your preferred content for a repo is `metadata=tag=done`, then only files tagged \"done\" will be kept in the repository. 
+
+Of course, files are only dropped if enough other copies can be verified to exist in other repositories..
+"""]]
diff --git a/doc/forum/Automatically_syncronise_centralised_repository.mdwn b/doc/forum/Automatically_syncronise_centralised_repository.mdwn
new file mode 100644
index 0000000000..fe93a814d0
--- /dev/null
+++ b/doc/forum/Automatically_syncronise_centralised_repository.mdwn
@@ -0,0 +1,14 @@
+I would like to use git annex between two locations (work and home) where essentially the two computers are never on at the same time.
+
+I have set up a special remote (S3) to cater for file transfer between the sites, but still need some way of syncronising the git repositories between them.
+I have access to a git server, but which doesn't have git-annex on it. So, I think that is all the components I need to get this working.
+
+However, I don't want to have to manually sync my computers with the central server, so I would like the assistant to do it for me, in what is essentially the complement of the special remote.
+
+What is the best way to accomplish this? I guess that this is a general git question, not specific to git annex.
+I see some solutions [[http://stackoverflow.com/questions/3583061/automatically-mirror-a-git-repository]], but my git isn't really up to evaluating the options properly.
+
+So, what do other people do in this situation? 
+
+
+--Walter
diff --git a/doc/forum/Automatically_syncronise_centralised_repository/comment_1_6a2047daa9faf4309d2ed27d5cc48b76._comment b/doc/forum/Automatically_syncronise_centralised_repository/comment_1_6a2047daa9faf4309d2ed27d5cc48b76._comment
new file mode 100644
index 0000000000..cbf8f0d047
--- /dev/null
+++ b/doc/forum/Automatically_syncronise_centralised_repository/comment_1_6a2047daa9faf4309d2ed27d5cc48b76._comment
@@ -0,0 +1,10 @@
+[[!comment format=mdwn
+ username="http://joeyh.name/"
+ nickname="joey"
+ subject="comment 1"
+ date="2013-04-22T18:58:08Z"
+ content="""
+The assistant can handle this syncing via a central bare remote. The only problem is that, since your server does not have git-annex installed, that remote will have annex-ignore set on it. That made the assistant not use it for syncing at all.
+
+I've just committed a change to git that makes the assistant still use such a git remote for syncing the git repository, even though it cannot store file contents there.
+"""]]
diff --git a/doc/forum/Automatically_syncronise_centralised_repository/comment_2_3be7b45bc2284019f17a81375637a576._comment b/doc/forum/Automatically_syncronise_centralised_repository/comment_2_3be7b45bc2284019f17a81375637a576._comment
new file mode 100644
index 0000000000..956e6b00a3
--- /dev/null
+++ b/doc/forum/Automatically_syncronise_centralised_repository/comment_2_3be7b45bc2284019f17a81375637a576._comment
@@ -0,0 +1,10 @@
+[[!comment format=mdwn
+ username="https://www.google.com/accounts/o8/id?id=AItOawnSenxKyE_2Z6Wb-EBMO8FciyRywjx1ZiQ"
+ nickname="Walter"
+ subject="comment 2"
+ date="2013-04-22T20:37:59Z"
+ content="""
+That looks like exactly what I'm after!
+
+I'm very impressed by the quick response. Keep up the great work!
+"""]]
diff --git a/doc/forum/Autostart_the_assistant.mdwn b/doc/forum/Autostart_the_assistant.mdwn
new file mode 100644
index 0000000000..06439750e9
--- /dev/null
+++ b/doc/forum/Autostart_the_assistant.mdwn
@@ -0,0 +1,9 @@
+Hello,
+
+I'm using the Arch git-annex package: https://www.archlinux.org/packages/community/x86_64/git-annex/
+
+Checking the autostart checkbox in the assistant's web interface has no effect. There is no git-annex desktop file in ~/.config/autostart/ or alike. Neither is one contained in the package, however in the tarball I haven't found either.
+
+How does autostarting the assistant works?
+
+Best Thanks!
diff --git a/doc/forum/Autostart_the_assistant/comment_1_a5421711102fb1649d98d916e5ced86b._comment b/doc/forum/Autostart_the_assistant/comment_1_a5421711102fb1649d98d916e5ced86b._comment
new file mode 100644
index 0000000000..88ad0fc3ad
--- /dev/null
+++ b/doc/forum/Autostart_the_assistant/comment_1_a5421711102fb1649d98d916e5ced86b._comment
@@ -0,0 +1,7 @@
+[[!comment format=mdwn
+ username="Horus"
+ subject="comment 1"
+ date="2016-03-14T08:18:49Z"
+ content="""
+Bump that, since a vast number of ancient posts somehow were resurrected...
+"""]]
diff --git a/doc/forum/Autostart_the_assistant/comment_2_df70bd0f5caab43503a3c9d805323289._comment b/doc/forum/Autostart_the_assistant/comment_2_df70bd0f5caab43503a3c9d805323289._comment
new file mode 100644
index 0000000000..3bad9e467b
--- /dev/null
+++ b/doc/forum/Autostart_the_assistant/comment_2_df70bd0f5caab43503a3c9d805323289._comment
@@ -0,0 +1,35 @@
+[[!comment format=mdwn
+ username="joey"
+ subject="""comment 2"""
+ date="2016-03-14T17:41:38Z"
+ content="""
+The way autostart works is, the repository gets listed in
+`~/.config/git-annex/autostart` -- setting up that file is all that
+the webapp preferences does.
+
+`git annex assistant --autostart` looks at that files and gets the
+assistant running in each listed repository.
+
+Now, something has to take care of running that command when the system
+starts or you log in or whatever. At this point it gets OS dependant.
+
+For example, on Debian, this is handled by the git-annex package containing
+`/etc/xdg/autostart/git-annex.desktop`
+
+Sounds like they may have left this file out of the Arch package. 
+
+git-annex's `make install` handles building that desktop file. It only
+generates it if run as root, or with DESTDIR set. One or the other should
+normally be the case when building a package.
+
+Here's the content of `/etc/xdg/autostart/git-annex.desktop`
+
+	[Desktop Entry]
+	Type=Application
+	Version=1.0
+	Name=Git Annex Assistant
+	Comment=Autostart
+	Terminal=false
+	Exec=/usr/bin/git-annex assistant --autostart
+	Categories=
+"""]]
diff --git a/doc/forum/Autostart_the_assistant/comment_3_42de896f64adb49c60ec3f7e0aeebda6._comment b/doc/forum/Autostart_the_assistant/comment_3_42de896f64adb49c60ec3f7e0aeebda6._comment
new file mode 100644
index 0000000000..3f23c03e54
--- /dev/null
+++ b/doc/forum/Autostart_the_assistant/comment_3_42de896f64adb49c60ec3f7e0aeebda6._comment
@@ -0,0 +1,19 @@
+[[!comment format=mdwn
+ username="Horus"
+ subject="comment 3"
+ date="2016-03-15T08:53:14Z"
+ content="""
+I use the Arch package, it is built / installed like that 
+
+    runhaskell Setup configure -O --prefix=/usr --docdir=\"/usr/share/doc/${pkgname}\" \
+        -fcryptonite -fnetwork-uri -f-ekg -fconcurrentoutput -ftorrentparser \
+        -ftestsuite -f-androidsplice -f-android -fproduction -fpairing -fwebapp \
+        -fassistant -fwebdav -fs3 -f-benchmark -fdbus -fxmpp -fmagicmime
+    runhaskell Setup build
+
+    runhaskell Setup copy --destdir=\"${pkgdir}\"
+
+Seem to be no classical make install involved. I try to peek your build system, but I know way to little about Haskell.
+
+Is there a way to get all the generated desktop files? Maybe add them to the prebuilt tarball?
+"""]]
diff --git a/doc/forum/Autostart_the_assistant/comment_4_5c6e085b1077b0792a1805f032ac6a4b._comment b/doc/forum/Autostart_the_assistant/comment_4_5c6e085b1077b0792a1805f032ac6a4b._comment
new file mode 100644
index 0000000000..23416f76b8
--- /dev/null
+++ b/doc/forum/Autostart_the_assistant/comment_4_5c6e085b1077b0792a1805f032ac6a4b._comment
@@ -0,0 +1,11 @@
+[[!comment format=mdwn
+ username="joey"
+ subject="""comment 4"""
+ date="2016-04-20T16:56:01Z"
+ content="""
+The only supported way is to either run "make install", or use a
+distribution package that is built using "make install". 
+
+If your distribution is building the package in some other way, please file
+a bug with it to get its package fixed.
+"""]]
diff --git a/doc/forum/Backend_migration_and_special_remotes.mdwn b/doc/forum/Backend_migration_and_special_remotes.mdwn
new file mode 100644
index 0000000000..b1cf885116
--- /dev/null
+++ b/doc/forum/Backend_migration_and_special_remotes.mdwn
@@ -0,0 +1,22 @@
+I have a large repository that is backed up to hubiC with the [special remote that I wrote](https://github.com/Schnouki/git-annex-remote-hubic). When I created this repository I used the WORM backend. But as I have some duplicated files I'd like to migrate to SHA256E to avoid storing them twice. However, when migrating a file, its location on the remote is not kept:
+
+    % git annex whereis "2015-07-25 12.28.16.jpg"
+    whereis 2015-07-25 12.28.16.jpg (2 copies) 
+            23fa590d-682d-43be-9851-8ba47186d763 -- [hubic]
+            f6642b89-2a0f-4788-8e13-be540fdfec58 -- baldr [here]
+    ok
+    
+    % git annex migrate "2015-07-25 12.28.16.jpg" --backend SHA256E
+    migrate 2015-07-25 12.28.16.jpg ok 
+    (recording state in git...)
+    
+    % git annex whereis "2015-07-25 12.28.16.jpg"
+    whereis 2015-07-25 12.28.16.jpg (1 copy) 
+            f6642b89-2a0f-4788-8e13-be540fdfec58 -- baldr [here]
+    ok
+
+Is there any way to store in the git-annex branch that migrated files are still available on remotes without having to re-upload all of them? hubiC is quite slow and I don't really want to re-upload 90 GB from a DSL connection…
+
+Or would it be possible to add something in the special remote protocol to "move" a key without re-uploading it?
+
+Thanks!
diff --git a/doc/forum/Backend_migration_and_special_remotes/comment_1_cf6d45c43b14719fc5ea41d0cbfb612d._comment b/doc/forum/Backend_migration_and_special_remotes/comment_1_cf6d45c43b14719fc5ea41d0cbfb612d._comment
new file mode 100644
index 0000000000..877594007e
--- /dev/null
+++ b/doc/forum/Backend_migration_and_special_remotes/comment_1_cf6d45c43b14719fc5ea41d0cbfb612d._comment
@@ -0,0 +1,20 @@
+[[!comment format=mdwn
+ username="joey"
+ subject="""comment 1"""
+ date="2015-09-18T16:47:02Z"
+ content="""
+A rename operation could be added to the interface for remotes. I suspect
+most remotes wouldn't be able to implement it (could hubic?), but it could
+be used when available.
+
+I'm not sure what to do in the migrate command to make it use such an
+interface though. It could try to move the files to their new names on each
+remote that supports the interface. But what about remotes that cannot be
+accessed?
+
+Could make a `git annex migrate --on remote`, to 
+migrate remotes peicemeal. Which would have to look back in the git history
+to find the old key that a file had been migrated from.
+Although then the user has to be careful to specify the same set of
+files that were migrated before.
+"""]]
diff --git a/doc/forum/Backing_up_photos_to_the_cloud.mdwn b/doc/forum/Backing_up_photos_to_the_cloud.mdwn
new file mode 100644
index 0000000000..0af14cbb80
--- /dev/null
+++ b/doc/forum/Backing_up_photos_to_the_cloud.mdwn
@@ -0,0 +1,13 @@
+I'm using git annex to manage my photo collection. The main reason is because my laptop doesn't have enough space to store all my photos, so I'm using git annex to create a sort of split repository between my laptop (which has some photos) and an external drive (which has everything). So far this has worked well, I have around 15,000 photos which is around 40GB.
+
+Now I also want to see if I can use git annex to improve my backup workflow. Previously I've just exported albums from my photo manager (iPhoto on OS X), zipped them up, and uploaded them to S3. I have lifecycle rules setup so that they are automatically replicated to a different region and archived to Glacier (it's a lot easier than dealing with Glacier directly). I am using this as a last resort backup in case everything else is lost, so it doesn't matter if it takes a while to access. This works well, except on it's own I don't really know what photos are stored where, which is where I'm hoping git annex can help.
+
+I've tried using the S3 remote, but there are a few things which I don't like:
+
+1) If the git repository is lost I can't recover the original paths, so I won't know which photo belongs in which album. As this is a last resort backup, if I ever need to get anything from here it's likely that the git repository is also lost. [JGit supports storing Git repositories in S3](http://www.fancybeans.com/blog/2012/08/24/how-to-use-s3-as-a-private-git-repository/), but that seems like the wrong way to solve this, I'd prefer just to have the original folder structure maintained.
+
+2) As there are 15,000 photos, that means 15,000 requests to S3 to upload them and another 15,000 each time I check them. On my connection I can upload to AWS at around 5MB/s, but due to latency that only means one or two photos per second. I'd prefer to just upload archives.
+
+As I understand encryption + chunking with a sufficiently large size (say 100Mb) would help solve the second problem, but as this is a last resort backup I don't want to have to worry about encryption keys or passphrases.
+
+It looks like a wrapper around the [archivedrive feature](https://git-annex.branchable.com/tips/offline_archive_drives/) (which archives, zips and uploads it to S3) would do what I want, but I'm wondering if there is a better way?
diff --git a/doc/forum/Backup_of_git-annex_repos.mdwn b/doc/forum/Backup_of_git-annex_repos.mdwn
new file mode 100644
index 0000000000..1e5663b2fd
--- /dev/null
+++ b/doc/forum/Backup_of_git-annex_repos.mdwn
@@ -0,0 +1,15 @@
+Hello,
+
+I use CrashPlan, a proprietary offsite backup system to backup a couple of git-annex repositories.
+
+CrashPlan backups the symlinks as symlinks as well as the .git directory. So all data is backuped. However, restoring it tedious.
+
+I identify a file that I want to restore, download it and get only the symlink. Now I can descent into the .git directory and get the correct blob the symlink points to.
+
+Are there any simple strategies getting a specific file from a git-annex repo if you only have file-based view on the repo?
+
+I can, of course, use direct mode, that would resolve this issue, but would prefer to use the safer indirect mode.
+
+Thanks for any ideas!
+
+Best!
diff --git a/doc/forum/Backup_of_git-annex_repos/comment_1_ed90e524606e394f7669fcba10846930._comment b/doc/forum/Backup_of_git-annex_repos/comment_1_ed90e524606e394f7669fcba10846930._comment
new file mode 100644
index 0000000000..3c38efe7ab
--- /dev/null
+++ b/doc/forum/Backup_of_git-annex_repos/comment_1_ed90e524606e394f7669fcba10846930._comment
@@ -0,0 +1,11 @@
+[[!comment format=mdwn
+ username="joey"
+ subject="""comment 1"""
+ date="2017-02-13T19:36:11Z"
+ content="""
+Suggest you restore everything except for `.git/annex/objects`.
+That should not be a lot of data to restore, but it will get
+you a fully working git repository back. Then you won't need the two-step
+process to restore a file, and can instead just restore the `.git/annex/objects`
+file that a symlink points to.
+"""]]
diff --git a/doc/forum/Bare_repo_on_USB_drive_not_providing_files.mdwn b/doc/forum/Bare_repo_on_USB_drive_not_providing_files.mdwn
new file mode 100644
index 0000000000..c4ac2c5b72
--- /dev/null
+++ b/doc/forum/Bare_repo_on_USB_drive_not_providing_files.mdwn
@@ -0,0 +1,17 @@
+Dear all,
+     In the first place I will explain what I want to achieve and then the way I figured out to do it. I have a lot of scientific simulations which I need to be traceable (cross check experiment version with code version, etc). I run them mostly on my ubuntu box (14.10, ext4)  and store them on an indirect repo. From time to time, I find useful to work on them on my laptop (OSX 10.10, low disk space, hfs+). 
+My idea was to full sync the main repo with a usb drive, and then at home get only the files I need to work from the USB drive.
+I faced a huge interoperability problem regarding the filesystem format. I tried using ext4, hfs+, fat32 and exfat. Regarding performance and stability, exfat is the only usable option between linux and osx (using hfs+ got the repo corrupted many times while writing from linux). However that forces me to use either a direct mode repo on the USB drive or a bare one. 
+
+Scenario 1: Direct mode on USB drive 
+While this worked in another case, in this case when setting up the repo with the assistant it deleted all symlinks from the main repo. I wasn't able to correctly do this and finally gave up. 
+Can anyone help me setting up this scenario? It will be like the following: 
+
+* ubuntu box (ext4): main repo, indirect mode, all files
+* usb drive (exfat): sync repo, direct mode, all files (having the files available is a plus)
+* osx laptop (hpfs+): partial repo, indirect mode, some files
+
+
+Scenario 2: Bare repo on USB drive 
+In this case the bare repo gets synced, with all data (2.6Gb). However when I clone it on the laptop, I get the  message that files are not available, it says that they are only on main repo. This puzzles me because files seem to be on the bare repo.
+What's wrong in this case? How can I debug this?
diff --git a/doc/forum/Bare_repo_on_USB_drive_not_providing_files/comment_1_0738174fb6984b777ec0a221502106ac._comment b/doc/forum/Bare_repo_on_USB_drive_not_providing_files/comment_1_0738174fb6984b777ec0a221502106ac._comment
new file mode 100644
index 0000000000..739ccc84fe
--- /dev/null
+++ b/doc/forum/Bare_repo_on_USB_drive_not_providing_files/comment_1_0738174fb6984b777ec0a221502106ac._comment
@@ -0,0 +1,30 @@
+[[!comment format=mdwn
+ username="joey"
+ subject="""comment 1"""
+ date="2014-12-01T22:03:52Z"
+ content="""
+You should be able to use a direct mode repo on a USB drive.
+Of course, if you delete files from there, the deletion will propigate
+to other clones of that repo. It could be that there's some bug
+with direct mode repos that causes file deletions to be incorrectly
+committed; if so you could a) revert the bad commits and b) file a bug
+report with details to reproduce the problem.
+
+As to a bare repo on a USB drive, it will work too. It sounds to me like
+you forgot to run `git annex sync` after copying the files to that repo.
+And/or forgot to run `git annex sync` before asking git-annex to get
+the files from the drive.
+
+That is to say, the fulll process would be:
+
+Plug drive into first computer, and run in that computer's repo:
+
+	git annex copy --to usbdrive
+	git annex sync
+
+And then move drive to other computer, and run in that computer's
+repo:
+	
+	git annex sync
+	git annex copy --from usbdrive
+"""]]
diff --git a/doc/forum/Bare_repo_on_USB_drive_not_providing_files/comment_2_1e07d1a9ddb88c0e1d84d8d88b7b4cc4._comment b/doc/forum/Bare_repo_on_USB_drive_not_providing_files/comment_2_1e07d1a9ddb88c0e1d84d8d88b7b4cc4._comment
new file mode 100644
index 0000000000..778471f5ab
--- /dev/null
+++ b/doc/forum/Bare_repo_on_USB_drive_not_providing_files/comment_2_1e07d1a9ddb88c0e1d84d8d88b7b4cc4._comment
@@ -0,0 +1,10 @@
+[[!comment format=mdwn
+ username="joey"
+ subject="""comment 2"""
+ date="2014-12-02T17:33:27Z"
+ content="""
+Well, I'm glad you solved it.
+
+I'd recommend a bare repo on a USB drive because it's the simplest thing,
+but a non-bare repo will also work ok.
+"""]]
diff --git a/doc/forum/Bare_repo_on_USB_drive_not_providing_files/comment_2_65d007284287e709a8f6f07dd8630f15._comment b/doc/forum/Bare_repo_on_USB_drive_not_providing_files/comment_2_65d007284287e709a8f6f07dd8630f15._comment
new file mode 100644
index 0000000000..018d69afdb
--- /dev/null
+++ b/doc/forum/Bare_repo_on_USB_drive_not_providing_files/comment_2_65d007284287e709a8f6f07dd8630f15._comment
@@ -0,0 +1,12 @@
+[[!comment format=mdwn
+ username="https://www.google.com/accounts/o8/id?id=AItOawmTNrhkVQ26GBLaLD5-zNuEiR8syTj4mI8"
+ nickname="Juan"
+ subject="I finally solved it"
+ date="2014-12-02T01:32:34Z"
+ content="""
+I fnally made it work. I tried git-annex sync wihtout luck. What tricked me was the size of the bare repo, it made me thought that it had the files. However after I've made a git annex get on the bare repo it got all the files and after that everthing worked as expected.
+So, if I don't get you wrong, it is safer to use a bare repo on the USB drive?
+Thanks for the help.
+Best regards,
+                     Juan
+"""]]
diff --git a/doc/forum/Bare_repo_on_USB_drive_not_providing_files/comment_4_3bbce5d12ece481b669be10ef9a70f40._comment b/doc/forum/Bare_repo_on_USB_drive_not_providing_files/comment_4_3bbce5d12ece481b669be10ef9a70f40._comment
new file mode 100644
index 0000000000..e70066b62b
--- /dev/null
+++ b/doc/forum/Bare_repo_on_USB_drive_not_providing_files/comment_4_3bbce5d12ece481b669be10ef9a70f40._comment
@@ -0,0 +1,28 @@
+[[!comment format=mdwn
+ username="https://www.google.com/accounts/o8/id?id=AItOawmTNrhkVQ26GBLaLD5-zNuEiR8syTj4mI8"
+ nickname="Juan"
+ subject="I think I got it working, however, same setup works different on fat and exfat"
+ date="2014-12-09T15:04:37Z"
+ content="""
+After struggling a lot, I tried to do the same but with an fat32 drive. Seems that some of the problems I'm experiencing are due to some (linux) filesystem functions working different with fat and exfat.
+Just to get this right, using repos on USB drive formatted as fat32 works perfectly under linux and mac. When I go to the exfat drive, linux fails, and osx keeps working.
+Are you able to test it?
+Steps to reproduce it: 
+1)make a test repo on linux: cd /tmp; mkdir test_repo; cd test repo; git init; git annex init \"test repo on linux box\" 
+2) go to exfat drive and clone it: git clone /tmp/rest_repo
+
+> git clone /home/juan/juanc/tmp/prueba_git 
+Cloning into 'prueba_git'... 
+error: chmod on /media/juan/USB_LIVE/tmp/test1/prueba_git/.git/config.lock failed: Function not implemented 
+error: chmod on /media/juan/USB_LIVE/tmp/test1/prueba_git/.git/config.lock failed: Function not implemented 
+error: chmod on /media/juan/USB_LIVE/tmp/test1/prueba_git/.git/config.lock failed: Function not implemented 
+error: chmod on /media/juan/USB_LIVE/tmp/test1/prueba_git/.git/config.lock failed: Function not implemented 
+error: chmod on /media/juan/USB_LIVE/tmp/test1/prueba_git/.git/config.lock failed: Function not implemented  
+error: chmod on /media/juan/USB_LIVE/tmp/test1/prueba_git/.git/config.lock failed: Function not implemented 
+fatal: 'origin' does not appear to be a git repository 
+fatal: Could not read from remote repository. 
+ 
+Please make sure you have the correct access rights 
+and the repository exists. 
+
+"""]]
diff --git a/doc/forum/Bare_repo_on_USB_drive_not_providing_files/comment_5_5e6202c319e75a9758bb8c2f32152e0f._comment b/doc/forum/Bare_repo_on_USB_drive_not_providing_files/comment_5_5e6202c319e75a9758bb8c2f32152e0f._comment
new file mode 100644
index 0000000000..d833e15e6f
--- /dev/null
+++ b/doc/forum/Bare_repo_on_USB_drive_not_providing_files/comment_5_5e6202c319e75a9758bb8c2f32152e0f._comment
@@ -0,0 +1,16 @@
+[[!comment format=mdwn
+ username="joey"
+ subject="""comment 5"""
+ date="2015-02-09T19:09:53Z"
+ content="""
+I had never heard of [exFAT](http://en.wikipedia.org/wiki/ExFAT) before.
+
+I guess you were mounting it in linux using the FUSE implementation of
+exFAT. It's not very surprising when FUSE filesystems leave off support
+for some syscalls. In this case, chmod.
+
+Since even git clone fails, git-annex obviously cannot work.
+
+A bare repository might be the only option if you're stuck using exFAT for
+some reason.
+"""]]
diff --git a/doc/forum/Basic_set_up_with_one_remote.mdwn b/doc/forum/Basic_set_up_with_one_remote.mdwn
new file mode 100644
index 0000000000..669c281d3d
--- /dev/null
+++ b/doc/forum/Basic_set_up_with_one_remote.mdwn
@@ -0,0 +1,14 @@
+Hi, I'm new to git annex and so far it's looking really useful. I'm convinced what I'm trying to achieve is possible, but I can't seem to hit upon the right combination. Here's what I'd like to set up.
+
+One server with git annex / rsync / ssh access, but configured such that it contains a history of all the files as a full backup.
+Two clients WITHOUT ssh access (these are portable machines and I'd rather not open any ports I don't absolutely have to).
+
+I've used the assistant to set up syncing between all three, using a jabber account to send signals and the server to pass on the files between the machines. This all works beautifully. But what I can't seem to get right is the server / backup aspect. If I set up the server using the assistant as a remote, and set it to "full backup" from the drop down list, it doesn't seem to have any files in the repository except in the .git/annex/objects folder. Which are all cryptic file names and not really accessible. If I run commands such as 
+
+> git annex find
+
+(which should list all files as far as I can tell) in the server repository, it doesn't list any files at all. As the clients are all set up as direct repositories I can't access any history of the files there, so this doesn't seem to be quite what I'm after. The sync is working, but I really want the backup / history aspect as well.
+
+Does anyone know if what I want is possible? If so hopefully I'm close and someone can point out where I've gone wrong. I'm not afraid of the command line, but the assistant is a very convenient way to get started. 
+
+Thanks for your help.
diff --git a/doc/forum/Basic_set_up_with_one_remote/comment_1_3ad06a0dbebf62e6440f549e77af59b6._comment b/doc/forum/Basic_set_up_with_one_remote/comment_1_3ad06a0dbebf62e6440f549e77af59b6._comment
new file mode 100644
index 0000000000..c262611075
--- /dev/null
+++ b/doc/forum/Basic_set_up_with_one_remote/comment_1_3ad06a0dbebf62e6440f549e77af59b6._comment
@@ -0,0 +1,15 @@
+[[!comment format=mdwn
+ username="https://www.google.com/accounts/o8/id?id=AItOawk3HGoDpnOPob5jOjvIootmkve1-nCpRiI"
+ nickname="Kalle"
+ subject="ssh remote defaults to bare repo"
+ date="2014-05-06T06:10:06Z"
+ content="""
+Hi,
+
+When setting up a ssh remote the default is to create a repo without a human readable file structure. To achieve what you are looking for you have to initialize a git annex repo on the server and then add the remote to the clients. afaik this has to be done using the command line interface. 
+
+If i remember correctly you add remotes with `git remote add ssh://example.com/path/to/annex`
+
+Cant find the docs atm (I'm commuting and at my final dest)
+
+"""]]
diff --git a/doc/forum/Basic_set_up_with_one_remote/comment_2_59e18e759c907b8adabf8c34eef08065._comment b/doc/forum/Basic_set_up_with_one_remote/comment_2_59e18e759c907b8adabf8c34eef08065._comment
new file mode 100644
index 0000000000..40ed2c8b5a
--- /dev/null
+++ b/doc/forum/Basic_set_up_with_one_remote/comment_2_59e18e759c907b8adabf8c34eef08065._comment
@@ -0,0 +1,10 @@
+[[!comment format=mdwn
+ username="https://www.google.com/accounts/o8/id?id=AItOawl2I59wfwq505F9W0a6f7sFDcC9neWcc24"
+ nickname="Andrew"
+ subject="comment 2"
+ date="2014-05-06T12:45:05Z"
+ content="""
+OK that sounds reasonable. I'll give it a go.
+
+I don't mind so much that the remote is not human readable as I wont be browsing directly on the remote, but what I did find disconcerting is that I didn't seem to be able to access any files. For example if I lose both the client machines, how do I get access to the files on the remote?
+"""]]
diff --git a/doc/forum/Basic_set_up_with_one_remote/comment_3_5221a713ee3f65fa2740c9fa6cb1db0f._comment b/doc/forum/Basic_set_up_with_one_remote/comment_3_5221a713ee3f65fa2740c9fa6cb1db0f._comment
new file mode 100644
index 0000000000..43dbb5d635
--- /dev/null
+++ b/doc/forum/Basic_set_up_with_one_remote/comment_3_5221a713ee3f65fa2740c9fa6cb1db0f._comment
@@ -0,0 +1,27 @@
+[[!comment format=mdwn
+ username="https://www.google.com/accounts/o8/id?id=AItOawl2I59wfwq505F9W0a6f7sFDcC9neWcc24"
+ nickname="Andrew"
+ subject="Still missing something"
+ date="2014-05-06T17:35:20Z"
+ content="""
+On the server I ran
+
+git init
+git annex init
+
+On client I ran
+
+git init
+git annex init
+git annex direct
+
+then dropped some files in and ran
+
+git sync
+
+This didn't upload any files. I now have a broken link on the server without the actual file.
+
+When I tried to set up the server in the same way, then used the web assistant to create the repository on the client and then combine it with the manually created repository on the server, it actually worked for the first file. I could see a working link to the file it had just uploaded. But any subsequent files didn't reflect in the file structure. Additional files would show up in the .git/annex/objects folder, but again I have no way to explore these. git annex find showed nothing, git annex status showed only the first file and git log didn't show much. I tried re-adding this remote to a new annex repository, but it didn't give me access to the files or the history of the files. I'm confused!
+
+Is it me or is this not working as expected?
+"""]]
diff --git a/doc/forum/Basic_set_up_with_one_remote/comment_4_62e554a546e4b50d211f5f65446fd289._comment b/doc/forum/Basic_set_up_with_one_remote/comment_4_62e554a546e4b50d211f5f65446fd289._comment
new file mode 100644
index 0000000000..8cbe86666a
--- /dev/null
+++ b/doc/forum/Basic_set_up_with_one_remote/comment_4_62e554a546e4b50d211f5f65446fd289._comment
@@ -0,0 +1,13 @@
+[[!comment format=mdwn
+ username="https://www.google.com/accounts/o8/id?id=AItOawk3HGoDpnOPob5jOjvIootmkve1-nCpRiI"
+ nickname="Kalle"
+ subject="Some guesses"
+ date="2014-05-06T19:43:02Z"
+ content="""
+If you don't need human readable file structure on the server you don't have to initialize the annex repo. Just create a \"remote server\" repo with the assistant and set it to \"full backup\" if you want all the files there all the time. Without the \"full backup\" setting I believe only files it thinks someone else needs will be uploaded.* This might solve your problem with symlinks as `git annex sync` syncs metadata, only moving actual data if required.
+
+I'm not sure if the `git-annex find` command will work on a bare repo I can't test as all my repos are configured so that I can access the files in a normal way. Running `git-annex whereis $file` on the client should allow you to check if a file has been uploaded.
+
+
+*Massive caveat as I'm really not sure
+"""]]
diff --git a/doc/forum/Beginners_questions__58___limit_content_on_special_remotes___38___use_non-git_clients_for_r__47__w_access_on_shared_git-annex_server.mdwn b/doc/forum/Beginners_questions__58___limit_content_on_special_remotes___38___use_non-git_clients_for_r__47__w_access_on_shared_git-annex_server.mdwn
new file mode 100644
index 0000000000..7291bd4b0d
--- /dev/null
+++ b/doc/forum/Beginners_questions__58___limit_content_on_special_remotes___38___use_non-git_clients_for_r__47__w_access_on_shared_git-annex_server.mdwn
@@ -0,0 +1,13 @@
+Hi!
+
+I just discovered *git-annex* as a great way to move away from Dropbox, but there are some open questions I couldn't find answers for and hope someone will be able to answer them here:
+
+I didn't completely understand how remotes are treated. As far as I understand there are two different parts to sync, the metadata (which is stored in a regular git repository) and the regular file data (or only chunks of it?). When syncing, I can choose whether I want to only sync the git (-metadata) part, or if I want to include the file contents, right?
+
+1. Is it possible to sync only a *part* of the file contents to certain remotes? For example I would want all my ebooks and photos to be synced to my remote server, but wouldn't want my private accounting and tax stuff to sync over the internet. 
+2. I have several (regular) git repositories used for e.g. development. Can I store a regular (bare) git repository within a git-annex repository? Does this make sense, or should I (for whatever reason) avoid it?
+3. A great feature of the Dropbox app is the automatic upload of photos taken with my iPhone. I would like to adopt this kind of backup. So considering a remotely accessible server. Is there some way to make use of *special remotes* like WebDAV to allow uploads over a *non-git interface*?
+4. Is there a way to access files (read-only) over a non-git interface from some git-annex remote? So could I access e.g. some pdf document from a git-annex repository, when there is a WebDAV special remote? Or do I just misunderstand the concept of *special remotes*? Is there another (useful) way? Maybe some kind of (remotely accessible) web interface? 
+5. Why is git-annex [not a backup system](http://git-annex.branchable.com/not/)?
+
+Thank you in advance!
diff --git a/doc/forum/Beginners_questions__58___limit_content_on_special_remotes___38___use_non-git_clients_for_r__47__w_access_on_shared_git-annex_server/comment_1_871b5a42f1134a059df520993bb55268._comment b/doc/forum/Beginners_questions__58___limit_content_on_special_remotes___38___use_non-git_clients_for_r__47__w_access_on_shared_git-annex_server/comment_1_871b5a42f1134a059df520993bb55268._comment
new file mode 100644
index 0000000000..35346757a5
--- /dev/null
+++ b/doc/forum/Beginners_questions__58___limit_content_on_special_remotes___38___use_non-git_clients_for_r__47__w_access_on_shared_git-annex_server/comment_1_871b5a42f1134a059df520993bb55268._comment
@@ -0,0 +1,11 @@
+[[!comment format=mdwn
+ username="spwhitton"
+ subject="Replies to questions"
+ date="2015-01-28T05:34:16Z"
+ content="""
+1. Yes, see [[/preferred content]].
+2. Every time you change a file, git-annex stores that version of it indefinitely, until manually dropped or automatically expired by the git annex assistant daemon.  So every time you checked out a branch or made a commit or whatever, you'd be cluttering your annex with many different versions of the same files.  Since git is a pretty good way of keeping track of versions of files, and indeed syncing with other machines using `git push` and `git pull`, why would you want to put your git repos into another synchronisation system?  (You can certainly tar up your git repos and put them in your annex as a backup.)
+3. Git-annex can run on Android phones but there is no iOS app.  You could have some other app uploading to the server, and then the git-annex assistant could watch the directory the files were being uploaded to and automatically add the files to the annex and sync it to your workstation.
+4. If it's just a normal git remote then you can access it in the usual ways.  Special remotes generally either lack knowledge of the names of your files, or this information is stored opaquely in git.  That means that access via e.g. WebDAV would only let you see your files organised by SHA checksum which would be useless.  So you'd want the git-annex remote you were accessing with a non-git interface to be a normal git remote, not a special remote.
+5. I think that Joey Hess wrote that git-annex is not a backup system to mean that what it does is quite different to something like duplicity, and also to mean that if you just install git-annex and create a repository, nothing is actually being backed up at all: you've got to manually add a remote, and instruct git-annex to send stuff there.  You can certainly use git-annex to backup so long as you understand to what extent it is doing so.
+"""]]
diff --git a/doc/forum/Beginners_questions__58___limit_content_on_special_remotes___38___use_non-git_clients_for_r__47__w_access_on_shared_git-annex_server/comment_2_68d675096d9ea32780b8ec8526544b12._comment b/doc/forum/Beginners_questions__58___limit_content_on_special_remotes___38___use_non-git_clients_for_r__47__w_access_on_shared_git-annex_server/comment_2_68d675096d9ea32780b8ec8526544b12._comment
new file mode 100644
index 0000000000..2965054332
--- /dev/null
+++ b/doc/forum/Beginners_questions__58___limit_content_on_special_remotes___38___use_non-git_clients_for_r__47__w_access_on_shared_git-annex_server/comment_2_68d675096d9ea32780b8ec8526544b12._comment
@@ -0,0 +1,7 @@
+[[!comment format=mdwn
+ username="janis_e"
+ subject="comment 2"
+ date="2015-01-29T08:39:02Z"
+ content="""
+Thank you for these answers, especially the 4) makes sense to me and prevented me form blowing up my repository!
+"""]]
diff --git a/doc/forum/Behaviour_of_fsck.mdwn b/doc/forum/Behaviour_of_fsck.mdwn
new file mode 100644
index 0000000000..cd27d49f76
--- /dev/null
+++ b/doc/forum/Behaviour_of_fsck.mdwn
@@ -0,0 +1,13 @@
+The current behaviour of 'fsck' is a bit verbose. I have an annex'd directory of tarballs for my own build system for "science" applications, there's about ~600 or so blobs in my repo, I do occassionally like to run fsck across all my data to see what files don't meet the min num copies requirement that I have set.
+
+Would it be better for the default behaviour of fsck when it has not been given a path to only output errors and not bother to show that a file is ok for every single file in a repo. i.e.
+
+    git annex fsck
+
+should show only 'errors' and maybe a simple indicator showing the status (show a spinner or dots?) and when 
+
+    git annex fsck PATH/FILE 
+
+it should have the current behaviour? 
+
+Right now the current fsck behaviour might get annoying for anyone who would want to run fsck with repos with lots of big files.
diff --git a/doc/forum/Behaviour_of_fsck/comment_1_0e40f158b3f4ccdcaab1408d858b68b8._comment b/doc/forum/Behaviour_of_fsck/comment_1_0e40f158b3f4ccdcaab1408d858b68b8._comment
new file mode 100644
index 0000000000..dc48e2f943
--- /dev/null
+++ b/doc/forum/Behaviour_of_fsck/comment_1_0e40f158b3f4ccdcaab1408d858b68b8._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="http://joey.kitenet.net/"
+ nickname="joey"
+ subject="comment 1"
+ date="2011-03-24T17:45:08Z"
+ content="""
+I tend to agree that the default output of fsck is not quite right. I often use git annex fsck -q. A progress spinner display is a good idea.
+"""]]
diff --git a/doc/forum/Behaviour_of_fsck/comment_2_ead36a23c3e6efa1c41e4555f93e014e._comment b/doc/forum/Behaviour_of_fsck/comment_2_ead36a23c3e6efa1c41e4555f93e014e._comment
new file mode 100644
index 0000000000..357b48a234
--- /dev/null
+++ b/doc/forum/Behaviour_of_fsck/comment_2_ead36a23c3e6efa1c41e4555f93e014e._comment
@@ -0,0 +1,19 @@
+[[!comment format=mdwn
+ username="https://www.google.com/accounts/o8/id?id=AItOawkSq2FDpK2n66QRUxtqqdbyDuwgbQmUWus"
+ nickname="Jimmy"
+ subject="comment 2"
+ date="2011-03-26T10:57:41Z"
+ content="""
+After some thought, perhaps the default fsck output should be at least machine readable and copy and pasteable i.e.
+
+
    +$ git annex fsck
+Files with errors
+
+    file1
+    file2
+
+
    
+
+so I can then copy the list of borked files and then just paste it into a for loop in my shell to recover the files. it's just an idea.
+"""]]
diff --git a/doc/forum/Behaviour_of_fsck/comment_3_97848f9a3db89c0427cfb671ba13300e._comment b/doc/forum/Behaviour_of_fsck/comment_3_97848f9a3db89c0427cfb671ba13300e._comment
new file mode 100644
index 0000000000..be34473c0a
--- /dev/null
+++ b/doc/forum/Behaviour_of_fsck/comment_3_97848f9a3db89c0427cfb671ba13300e._comment
@@ -0,0 +1,19 @@
+[[!comment format=mdwn
+ username="https://www.google.com/accounts/o8/id?id=AItOawl9sYlePmv1xK-VvjBdN-5doOa_Xw-jH4U"
+ nickname="Richard"
+ subject="comment 3"
+ date="2011-03-28T01:16:21Z"
+ content="""
+Another nice thing would be a summary of _what_ is wrong. I.e.
+
+    % git fsck
+    [...]
+    git-annex: 100 total failed
+      50 checksum failed
+      50 not enough copies exit
+
+And the same/similar for all other failure modes.
+
+
+-- RichiH
+"""]]
diff --git a/doc/forum/Behaviour_of_fsck/comment_4_e4911dc6793f98fb81151daacbe49968._comment b/doc/forum/Behaviour_of_fsck/comment_4_e4911dc6793f98fb81151daacbe49968._comment
new file mode 100644
index 0000000000..e8c9837462
--- /dev/null
+++ b/doc/forum/Behaviour_of_fsck/comment_4_e4911dc6793f98fb81151daacbe49968._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="https://www.google.com/accounts/o8/id?id=AItOawl9sYlePmv1xK-VvjBdN-5doOa_Xw-jH4U"
+ nickname="Richard"
+ subject="comment 2"
+ date="2011-03-25T11:23:04Z"
+ content="""
+FWIW, I wanted to suggest exactly the same thing.
+"""]]
diff --git a/doc/forum/Benefit_and_drawback_of_new_unlocked_file_mode_V6_vs_locked_mode.mdwn b/doc/forum/Benefit_and_drawback_of_new_unlocked_file_mode_V6_vs_locked_mode.mdwn
new file mode 100644
index 0000000000..e7be125197
--- /dev/null
+++ b/doc/forum/Benefit_and_drawback_of_new_unlocked_file_mode_V6_vs_locked_mode.mdwn
@@ -0,0 +1,4 @@
+Someone wrote: "Unlocked files are more difficult to synchronize (to check for
+modifications and to synchronize throught multiple remotes)."  Does it mean new
+unlocked file mode V6 takes more time to find modifications?  If not what is the
+drawbacks of this new unlocked file mode?
diff --git a/doc/forum/Benefit_and_drawback_of_new_unlocked_file_mode_V6_vs_locked_mode/comment_1_fef0122752291bdc583fdcfdd400d250._comment b/doc/forum/Benefit_and_drawback_of_new_unlocked_file_mode_V6_vs_locked_mode/comment_1_fef0122752291bdc583fdcfdd400d250._comment
new file mode 100644
index 0000000000..657839f467
--- /dev/null
+++ b/doc/forum/Benefit_and_drawback_of_new_unlocked_file_mode_V6_vs_locked_mode/comment_1_fef0122752291bdc583fdcfdd400d250._comment
@@ -0,0 +1,10 @@
+[[!comment format=mdwn
+ username="andrew"
+ avatar="http://cdn.libravatar.org/avatar/acc0ece1eedf07dd9631e7d7d343c435"
+ subject="comment 1"
+ date="2018-05-03T19:56:03Z"
+ content="""
+I guess you were reading this blogpost  from 2016?
+
+I have not run into any performance issues, but apparently others have. Namely, direct mode (V5) is faster than using unlocked files with a V6 repo, see [How to improve performance with v6 repos](https://git-annex.branchable.com/forum/How_to_improve_performance_with_v6_repos__63__/) related to the difference between direct-mode and V6 unlocked. Joey has a long [TODO related to git smudge filters](https://git-annex.branchable.com/todo/smudge/) that I believe would resolve this performance issue. The other drawback of V6 is that Joey considers it beta: “I'd encourage you to not upgrade any production repositories to v6 yet. It's still in beta state and has known problems. Currently all known problems are only innefficiencies, but there could also still be bugs.” [v5_to_v6_upgrade_strategy](https://git-annex.branchable.com/forum/v5_to_v6_upgrade_strategy/).
+"""]]
diff --git a/doc/forum/Best_approach_for_central_sharing_and_multiple_users.mdwn b/doc/forum/Best_approach_for_central_sharing_and_multiple_users.mdwn
new file mode 100644
index 0000000000..6fc104cb8f
--- /dev/null
+++ b/doc/forum/Best_approach_for_central_sharing_and_multiple_users.mdwn
@@ -0,0 +1,8 @@
+Just a quick question: Can anyone recomment me a good to handle a centralized repository which is used by different users?
+
+Requirements: must work in direct mode on both sides, server is Linux, clients are Windows and operated by 2 different users.
+
+- Should the repository create bare or not?
+- Must the repository created on the server or can it also be "pushed" from one client to the server (via SSH on the server)?
+- Is there a better way for data transfer than SSH (WebDAV as in SVN)? I am still struggling with an elegant way to handle public keys in windows
+- What is the best way to not only keep meta data in sync but also the data each of the users has checked out?
diff --git a/doc/forum/Best_approach_for_central_sharing_and_multiple_users/comment_1_48ffb50b92588daec6887bf08f1b97f5._comment b/doc/forum/Best_approach_for_central_sharing_and_multiple_users/comment_1_48ffb50b92588daec6887bf08f1b97f5._comment
new file mode 100644
index 0000000000..17860d0372
--- /dev/null
+++ b/doc/forum/Best_approach_for_central_sharing_and_multiple_users/comment_1_48ffb50b92588daec6887bf08f1b97f5._comment
@@ -0,0 +1,10 @@
+[[!comment format=mdwn
+ username="http://joeyh.name/"
+ ip="209.250.56.2"
+ subject="comment 1"
+ date="2014-07-11T19:31:52Z"
+ content="""
+I suggest following this screencast: [[videos/git-annex_assistant_lan]]
+
+While it talks about a local network, this will also work with a server on the internet.
+"""]]
diff --git a/doc/forum/Best_approach_for_central_sharing_and_multiple_users/comment_2_6b9a20f9707da9d2cfc3697a538d6935._comment b/doc/forum/Best_approach_for_central_sharing_and_multiple_users/comment_2_6b9a20f9707da9d2cfc3697a538d6935._comment
new file mode 100644
index 0000000000..c77e9a8fd1
--- /dev/null
+++ b/doc/forum/Best_approach_for_central_sharing_and_multiple_users/comment_2_6b9a20f9707da9d2cfc3697a538d6935._comment
@@ -0,0 +1,12 @@
+[[!comment format=mdwn
+ username="divB"
+ ip="171.67.173.91"
+ subject="comment 2"
+ date="2014-07-11T21:56:29Z"
+ content="""
+Thanks, great work!
+
+I wanted to avoid the assistant so far because I feel I do not get so much control (particularly which files are synced to the clients) but I think I'll start with the assistant ...
+
+
+"""]]
diff --git a/doc/forum/Best_way_to_manage_files_on_removable_media__63__.mdwn b/doc/forum/Best_way_to_manage_files_on_removable_media__63__.mdwn
new file mode 100644
index 0000000000..d6d1206ae0
--- /dev/null
+++ b/doc/forum/Best_way_to_manage_files_on_removable_media__63__.mdwn
@@ -0,0 +1,18 @@
+I have a bunch of removable storage devices and was planning on storing my data across
+all of them.  I've run into an annoyance, and would like to see if anybody has any
+ideas.
+
+My goal was to have the full file tree on all the devices, but only a subset of the
+annexed data.  Where I have run into trouble is removing data from the system.  It
+seems that the "git annex unused" command checks remote branches as well as local ones
+when determining whether an object is referred to.
+
+This means that if I remove a file that is stored locally, "git annex unused" doesn't
+report the corresponding object as unused until I either connect and update all
+removable storage *or* remove the remote corresponding to the removable storage.  I
+posted a bug about this inconsistency named
+[[bugs/git annex unused considers remote branches which makes it inconsistent]].
+
+If I used the removable storage as a special remote, then I wouldn't have this issue,
+but I also wouldn't be able to conveniently use the files on it and manage the repo
+from it either.
diff --git a/doc/forum/Best_way_to_re-add_a_file_in_a_direct_and_crippled_annex_repository.mdwn b/doc/forum/Best_way_to_re-add_a_file_in_a_direct_and_crippled_annex_repository.mdwn
new file mode 100644
index 0000000000..c126f83735
--- /dev/null
+++ b/doc/forum/Best_way_to_re-add_a_file_in_a_direct_and_crippled_annex_repository.mdwn
@@ -0,0 +1,17 @@
+Hi!
+
+What is the best way to add again the content of file in an direct and crippled annex repository? I'm using such a repository on a SMB drive, with no symbolic links, that's why the repository is in direct mode.
+
+For example:
+
+`/disk/annex-rep/a.bin` contains the text `../.git/annex/objects/W3/M8/SHA256E-s4701522386--e486cce01b1870cee394a376a3acc64a608e84e38b85e6c21c29721cf14328e6.bin/SHA256E-s4701522386--e486cce01b1870cee394a376a3acc64a608e84e38b85e6c21c29721cf14328e6.bin`
+
+and I've got the real content in another directory, outside annex: /disk/a.bin (contains the binary data)
+
+I imagined this process:
+* replace `/disk/annex-rep/a.bin` with `/disk/a.bin``
+* update `git-annex:122/4b8/SHA256E-s4701522386--e486cce01b1870cee394a376a3acc64a608e84e38b85e6c21c29721cf14328e6.bin.log` by adding this repository uuid
+
+I tried `git annex reinject` but it didn't work.
+
+Is there another way?
diff --git a/doc/forum/Best_way_to_re-add_a_file_in_a_direct_and_crippled_annex_repository/comment_1_552e74f9573a34ec178f396b83252c3e._comment b/doc/forum/Best_way_to_re-add_a_file_in_a_direct_and_crippled_annex_repository/comment_1_552e74f9573a34ec178f396b83252c3e._comment
new file mode 100644
index 0000000000..439c985dcb
--- /dev/null
+++ b/doc/forum/Best_way_to_re-add_a_file_in_a_direct_and_crippled_annex_repository/comment_1_552e74f9573a34ec178f396b83252c3e._comment
@@ -0,0 +1,12 @@
+[[!comment format=mdwn
+ username="http://joeyh.name/"
+ ip="209.250.56.64"
+ subject="comment 1"
+ date="2013-11-26T19:54:21Z"
+ content="""
+There's a bug report asking that reinject work in direct mode. I just have not gotten around to it.
+
+Another way to do it would be to make a temp directory in your annex repository, move or copy the file to there, and `git annex add`. As long as you're not using the WORM backend, the checksum will be used, and so it'll see it's added the content of the file, and make the a.bin file have that content too.
+
+Off to see if I can easily add direct mode reinject now..
+"""]]
diff --git a/doc/forum/Best_way_to_re-add_a_file_in_a_direct_and_crippled_annex_repository/comment_2_33c57922714f204fc63c260b838f3712._comment b/doc/forum/Best_way_to_re-add_a_file_in_a_direct_and_crippled_annex_repository/comment_2_33c57922714f204fc63c260b838f3712._comment
new file mode 100644
index 0000000000..afc449ac2e
--- /dev/null
+++ b/doc/forum/Best_way_to_re-add_a_file_in_a_direct_and_crippled_annex_repository/comment_2_33c57922714f204fc63c260b838f3712._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="http://joeyh.name/"
+ ip="209.250.56.64"
+ subject="comment 2"
+ date="2013-11-26T20:18:56Z"
+ content="""
+Turned out to be trivial, so it's supported now.
+"""]]
diff --git a/doc/forum/Best_way_to_remove_old_version_of_specific_file.mdwn b/doc/forum/Best_way_to_remove_old_version_of_specific_file.mdwn
new file mode 100644
index 0000000000..4727c75c01
--- /dev/null
+++ b/doc/forum/Best_way_to_remove_old_version_of_specific_file.mdwn
@@ -0,0 +1,24 @@
+Hi,
+
+I store lots of books under annex and every once in a while one of them gets updated and I don't really want to keep the old version. I know about the `unused` subcommand but it makes a bit difficult to identify what I would be dropping. I mean, at the point in time when I'm replacing a file I already know I don't want to keep in the annex, maybe there is a method that directly referrs to that file instead of globally garbage collecting obscurely named files. Now, this is what comes to my mind:
+
+Alternative a
+
+1. git-annex drop f
+2. git rm f
+3. mv new-f f
+4. git-annex add f
+
+Alternative b
+
+1. git-annex unlock f
+2. mv new-f f
+3. git-annex add f
+4. git commit -am blah
+5. git checkout HEAD^
+6. git-annex drop f
+7. git checkout master
+
+As you can see neither option is that pretty. Is there any way to say 'drop f in revision r' or something similar?
+
+Thank you!
diff --git a/doc/forum/Best_way_to_remove_old_version_of_specific_file/comment_1_1663dd9118cb1fee92ae3f8d812df79c._comment b/doc/forum/Best_way_to_remove_old_version_of_specific_file/comment_1_1663dd9118cb1fee92ae3f8d812df79c._comment
new file mode 100644
index 0000000000..8e075df1be
--- /dev/null
+++ b/doc/forum/Best_way_to_remove_old_version_of_specific_file/comment_1_1663dd9118cb1fee92ae3f8d812df79c._comment
@@ -0,0 +1,17 @@
+[[!comment format=mdwn
+ username="joey"
+ subject="""comment 1"""
+ date="2017-06-26T16:52:25Z"
+ content="""
+Your alternative B does not actually remove the old version of the file,
+because git-annex unlock keeps a copy in .git/annex/objects
+(except when using v6 mode with annex.thin).
+
+Alternative A is fine, if you just want to remove the old version
+of the file from the local repository. Other clones of the repository
+may still have the content of that file though.
+
+I use alternative A fairly frequently in my own repositories, and then
+sometimes followup with `git annex unused` in clones and dropping all
+unused files.
+"""]]
diff --git a/doc/forum/Best_way_to_remove_old_version_of_specific_file/comment_2_6580f6790404694b55d07a0e57d83336._comment b/doc/forum/Best_way_to_remove_old_version_of_specific_file/comment_2_6580f6790404694b55d07a0e57d83336._comment
new file mode 100644
index 0000000000..77b39470c4
--- /dev/null
+++ b/doc/forum/Best_way_to_remove_old_version_of_specific_file/comment_2_6580f6790404694b55d07a0e57d83336._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="memeplex"
+ avatar="http://cdn.libravatar.org/avatar/84a611000e819ef825421de06c9bca90"
+ subject="comment 2"
+ date="2017-07-08T15:37:13Z"
+ content="""
+Hi Joey, thank you for answering. I don't understand why alternative b won't work. Doesn't the add after the unlock revert to locked state?
+"""]]
diff --git a/doc/forum/Best_way_to_remove_old_version_of_specific_file/comment_3_c1f34ec908d7c4ad05c4360e29e68fb4._comment b/doc/forum/Best_way_to_remove_old_version_of_specific_file/comment_3_c1f34ec908d7c4ad05c4360e29e68fb4._comment
new file mode 100644
index 0000000000..6fd7012d4f
--- /dev/null
+++ b/doc/forum/Best_way_to_remove_old_version_of_specific_file/comment_3_c1f34ec908d7c4ad05c4360e29e68fb4._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="memeplex"
+ avatar="http://cdn.libravatar.org/avatar/84a611000e819ef825421de06c9bca90"
+ subject="comment 3"
+ date="2017-07-08T15:47:47Z"
+ content="""
+I mean, as I understand it I unlock the file, modify it, re-add it, commit, go to the previous revision and drop the annexed old version there. I see this as an instance of the remove-old-annexed-version general case, which AFAIK is valid. What am I missing here? Should I lock instead of adding the new file?
+"""]]
diff --git a/doc/forum/Big_repository_vs._multiple_small.mdwn b/doc/forum/Big_repository_vs._multiple_small.mdwn
new file mode 100644
index 0000000000..c77dd02a55
--- /dev/null
+++ b/doc/forum/Big_repository_vs._multiple_small.mdwn
@@ -0,0 +1,8 @@
+I am new to git (but extensively used SVN).
+
+In SVN I could have a big fat repository but only check out sub-trees is it.
+Is that also common in git(-annex) / recommended?
+
+E.g., should I create a big-fat repos with all data I have (personal data, music, videos, ...) and check out only the appropriate subtress or create a repository for each purpose? E.g., one for Fotos, Music, OnTheGoData, ebooks, ...
+
+What happens if I have a git-annex repository checked out at my laptop (say, d:\Files) and within it, check out another one (e.g. d:\Files\Library)?
diff --git a/doc/forum/Big_repository_vs._multiple_small/comment_1_8e21ee3c674ef6e595bdab53dd5c2356._comment b/doc/forum/Big_repository_vs._multiple_small/comment_1_8e21ee3c674ef6e595bdab53dd5c2356._comment
new file mode 100644
index 0000000000..707a1cca6e
--- /dev/null
+++ b/doc/forum/Big_repository_vs._multiple_small/comment_1_8e21ee3c674ef6e595bdab53dd5c2356._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="https://me.yahoo.com/a/FHnTlSBo1eCGJRwueeKeB6.RCaPbGMPr5jxx8A--#ce0d8"
+ nickname="Hamza"
+ subject="comment 1"
+ date="2014-04-14T15:32:34Z"
+ content="""
+In my setup I have one repository for each category (photos documents videos) it is faster then single giant repository plus it makes sharing with other people easier since you can let people clone one category at a time.
+"""]]
diff --git a/doc/forum/Big_repository_vs._multiple_small/comment_2_656c62351502492d20e8490242e51169._comment b/doc/forum/Big_repository_vs._multiple_small/comment_2_656c62351502492d20e8490242e51169._comment
new file mode 100644
index 0000000000..22fa765428
--- /dev/null
+++ b/doc/forum/Big_repository_vs._multiple_small/comment_2_656c62351502492d20e8490242e51169._comment
@@ -0,0 +1,14 @@
+[[!comment format=mdwn
+ username="divB"
+ ip="128.12.90.218"
+ subject="comment 2"
+ date="2014-04-15T18:23:13Z"
+ content="""
+Thank you, that's a good point!
+
+Does this also mean that a repository in git(-annex) is \"all or nothing\"?
+
+For example, I cannot share/clone parts of it? Or define access rights within a repository?
+
+With SVN for example, I have a big repository \"university\" and it contains all stuff of projects/research. Each individual directory is only shared with the persons whom I worked together in this particular project. In short: In git, this should not be that way, right?
+"""]]
diff --git a/doc/forum/Big_repository_vs._multiple_small/comment_3_e9c44ea364513f090844f46af2ea46a1._comment b/doc/forum/Big_repository_vs._multiple_small/comment_3_e9c44ea364513f090844f46af2ea46a1._comment
new file mode 100644
index 0000000000..063260ba10
--- /dev/null
+++ b/doc/forum/Big_repository_vs._multiple_small/comment_3_e9c44ea364513f090844f46af2ea46a1._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="http://id.clacke.se/"
+ nickname="Claes"
+ subject="comment 3"
+ date="2014-04-17T14:04:31Z"
+ content="""
+Yeah, git does not have the concept of checking out subdirectories that subversion does. You could, however, have different branches with different content that live in the same repo and therefore share the same git-annex backend, so there could be overlap between what files are in what branch without them using up much extra disk space.
+"""]]
diff --git a/doc/forum/Big_repository_vs._multiple_small/comment_4_82e13580426dc648688e4c26e7ed91ec._comment b/doc/forum/Big_repository_vs._multiple_small/comment_4_82e13580426dc648688e4c26e7ed91ec._comment
new file mode 100644
index 0000000000..e79e19814a
--- /dev/null
+++ b/doc/forum/Big_repository_vs._multiple_small/comment_4_82e13580426dc648688e4c26e7ed91ec._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="https://me.yahoo.com/a/FHnTlSBo1eCGJRwueeKeB6.RCaPbGMPr5jxx8A--#ce0d8"
+ nickname="Hamza"
+ subject="comment 4"
+ date="2014-04-18T09:16:44Z"
+ content="""
+with git you can not checkout parts of a repo it is all or nothing but with annex you can download your files dir by dir. When you clone you get symlinks for all files but only the files you want are downloaded.
+"""]]
diff --git a/doc/forum/Big_repository_vs._multiple_small/comment_5_632aceb71dc6a4a9a4bb03de25a9b21a._comment b/doc/forum/Big_repository_vs._multiple_small/comment_5_632aceb71dc6a4a9a4bb03de25a9b21a._comment
new file mode 100644
index 0000000000..182263f397
--- /dev/null
+++ b/doc/forum/Big_repository_vs._multiple_small/comment_5_632aceb71dc6a4a9a4bb03de25a9b21a._comment
@@ -0,0 +1,23 @@
+[[!comment format=mdwn
+ username="divB"
+ ip="74.61.144.53"
+ subject="comment 5"
+ date="2014-04-19T05:38:43Z"
+ content="""
+Thanks!
+
+But what to my question regarding checking out a repository within a repository?
+
+I will most likely have a \"home\" repository which I would check out at first level.
+
+Then, within one directory I would like to (only locally!) check out another, such as \"Library\".
+
+Something like mount ... to mount one FS within another mount point.
+
+For SVN I think there exist externals for that ...
+
+Does this work?
+
+
+
+"""]]
diff --git a/doc/forum/Bitbucket_gcrypt_repository_doesn__39__t_appear_in_the_Assistant.mdwn b/doc/forum/Bitbucket_gcrypt_repository_doesn__39__t_appear_in_the_Assistant.mdwn
new file mode 100644
index 0000000000..0079ecfee8
--- /dev/null
+++ b/doc/forum/Bitbucket_gcrypt_repository_doesn__39__t_appear_in_the_Assistant.mdwn
@@ -0,0 +1,37 @@
+Hello everyone,
+
+my problem is: I have added a Bitbucket gcrypt repository (to be used for metadata only); it works via git-annex sync but the Assistant ignores it completely. I can check from the Bitbucket webpage that the remote repository never gets updated by the Assistant, it only does when i run 'git-annex sync' manually.
+
+## Adding the repository:
+
+    git remote add bitbucket gcrypt::https://bitbucket.org/omissis/basic.git
+
+## Syncing to it is ok:
+
+    $ git-annex sync
+    commit  ok
+    pull bitbucket 
+    gcrypt: Development version -- Repository format MAY CHANGE
+    gcrypt: Decrypting manifest
+    [...]
+    ok
+    push bitbucket 
+    gcrypt: Development version -- Repository format MAY CHANGE
+    gcrypt: Decrypting manifest
+    [...]
+    ok
+
+## But it doesn't appear at all in the Assistant webapp!
+
+Even if I do "restart daemon".
+
+BTW, adding a local gcrypt repo works fine and it gets synced normally by the Assistant  (after creating a bare Git repo):
+
+    git remote add enc gcrypt::/tmp/enc
+
+My Git-annex version:
+
+    $ git-annex version
+    git-annex version: 5.20140412ubuntu1
+
+Anybody has any idea? My use case is that I have a computer behind a very restrictive firewall (only http and https are allowed) and I don't know how to sync it otherwise. I would like to avoid XMPP. Thanks!
diff --git a/doc/forum/Bitbucket_gcrypt_repository_doesn__39__t_appear_in_the_Assistant/comment_1_239ae08d8eb8b0d7fe52991b0bddc653._comment b/doc/forum/Bitbucket_gcrypt_repository_doesn__39__t_appear_in_the_Assistant/comment_1_239ae08d8eb8b0d7fe52991b0bddc653._comment
new file mode 100644
index 0000000000..7bce01150c
--- /dev/null
+++ b/doc/forum/Bitbucket_gcrypt_repository_doesn__39__t_appear_in_the_Assistant/comment_1_239ae08d8eb8b0d7fe52991b0bddc653._comment
@@ -0,0 +1,11 @@
+[[!comment format=mdwn
+ username="joey"
+ subject="""comment 1"""
+ date="2015-07-30T18:07:13Z"
+ content="""
+I think this is due to the format of the url. The webapp only supports
+gcrypt repos that are using ssh uris, or are on local drives.
+
+I think, though that this might only prevent the assistant from displaying the
+repo in the list of repos. I think it might still sync with it?
+"""]]
diff --git a/doc/forum/Bitbucket_gcrypt_repository_doesn__39__t_appear_in_the_Assistant/comment_2_8b91328473b384f347710ec634be710a._comment b/doc/forum/Bitbucket_gcrypt_repository_doesn__39__t_appear_in_the_Assistant/comment_2_8b91328473b384f347710ec634be710a._comment
new file mode 100644
index 0000000000..fc79c17ba6
--- /dev/null
+++ b/doc/forum/Bitbucket_gcrypt_repository_doesn__39__t_appear_in_the_Assistant/comment_2_8b91328473b384f347710ec634be710a._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="Agenta"
+ subject="comment 2"
+ date="2015-07-31T08:34:26Z"
+ content="""
+Unfortunately the Assistant doesn't sync to the https gcrypt repository at all! E.g. if I add a file, the file gets added to the other repositories (the Assistant syncs it in the background) but it doesn't get added to Bitbucket (https gcrypt).
+I know that because Bitbucket tells me when the latest change has been made; I can see that no change is reported unless I perform 'git-annex sync' manually.
+"""]]
diff --git a/doc/forum/Bitbucket_gcrypt_repository_doesn__39__t_appear_in_the_Assistant/comment_3_12a6bc61a7492650fd2baf9a1b796997._comment b/doc/forum/Bitbucket_gcrypt_repository_doesn__39__t_appear_in_the_Assistant/comment_3_12a6bc61a7492650fd2baf9a1b796997._comment
new file mode 100644
index 0000000000..4a7ad554a8
--- /dev/null
+++ b/doc/forum/Bitbucket_gcrypt_repository_doesn__39__t_appear_in_the_Assistant/comment_3_12a6bc61a7492650fd2baf9a1b796997._comment
@@ -0,0 +1,17 @@
+[[!comment format=mdwn
+ username="joey"
+ subject="""comment 3"""
+ date="2015-08-04T20:23:58Z"
+ content="""
+Well, the file is not uploaded to the bitbucket remote, because bitbucket
+doesn't support git-annex. Without git-annex-shell on the remote, git-annex
+cannot send files to it.
+
+Even if it did, it would need to be a ssh
+transport, not a http transport, since git-annex doesn't upload files over
+http (except to S3 or webdav remotes).
+
+I think that the assistant also assumes that http transport git remotes are
+read-only, and doesn't try to push to them. Which might be improvable,
+perhaps.
+"""]]
diff --git a/doc/forum/Blacklisting_files___40__so_that_they_get_removed_any_time_a_copy_is_found__41__.mdwn b/doc/forum/Blacklisting_files___40__so_that_they_get_removed_any_time_a_copy_is_found__41__.mdwn
new file mode 100644
index 0000000000..ca96a074f6
--- /dev/null
+++ b/doc/forum/Blacklisting_files___40__so_that_they_get_removed_any_time_a_copy_is_found__41__.mdwn
@@ -0,0 +1,55 @@
+Hello,
+
+I'm not sure this is mature enough as a wishlist item so I put it in the forum.  Please move if it is appropriate.
+
+# Context, general need
+
+I'm considering using git-annex to store photos, yet what I'm needing might be useful in more general cases.
+
+The need in one sentence: **deal with files that change status from "precious, please keep n copies" to "junk, please delete it from everywhere you find it, now and forever"**.
+
+# More concrete need
+
+I take many photographs and am interested in git-annex comfort to ensure a number of copies of files exist at any time.
+
+Sometimes I can sort photos and reject bad ones shortly after taking a roll, typically before I could `git annex add` them.  Sometimes I do it much later, after they have been included in `git-annex` and stored in several places.
+
+## Rationale 1: releasing storage space
+
+I'm worried about having a lot of space still used by photographs that I actually don't want to keep.
+
+It's not marginal.  For example, at 30Mb per RAW photo, 300+ photos in an ordinary afternoon take 10Gb, from which up to one half could turn out rejected.  Whole photo/video collection is already probably much over 1Tb and growing.
+
+So we're talking about 5Gb per shooting day to be freed and already probably 100+Gb to free from remotes, so definitely something to consider.
+
+## Rationale 2: rejecting files once and for all, not having to repeat it
+
+Once I rejected a photograph, I'd like to never have to `rm`, `forget`, `drop` them or whatever again.  Ideally it would just be dropped from any back-end (that supports removing information) at any time in the future, perhaps with just a generic command/script like `git annex purge_the_junk_files`.
+
+# Questions
+
+* Q1 Is there a simple way to somehow "blacklist" a particular file so that from that moment on, any `sync`-like operation of `git-annex` involving a remote will remove the data of this file in that remote.
+* Q2 UI considerations. In my dreams, files could just be moved to a "junk" sub-folder (using photo sorting tools like e.g. geeqie), then a batch command would assign them the "blacklisted" or "junk" status.
+* Q3 I don't mind at this point if there are traces of where (in the filesystem tree) the now-rejected files were.  Perhaps it's easier to perform a different operation, that is completely forgetting the history of blacklisted files in the spirit of `git filter-branch`.
+
+Perhaps most of this can be done with simple configuration and a helper script.
+
+# Additional information
+
+* I'm wondering if some simple scheme like an actual `git filter-branch` in the local `git-annex` repo then some cleanup command (`git annex fsck`) then push to remote would have the intended effect.  Since this involves rewriting history I don't know how `git-annex` would like it.  But that's just a thought-experiment at this point.
+* The number of files to blacklist can quickly go to a few hundreds later thousands.  This might rule out some very naive implementations.
+* I can hack a bash script to perform whatever appropriate command so that given a solution to Q1 I have a solution to Q2.
+
+# Search before you post
+
+I've found other topics but they don't seem to even remotely deal with the topic here.  E.g.:
+
+* [[https://git-annex.branchable.com/forum/Backing_up_photos_to_the_cloud/]]
+* [[https://git-annex.branchable.com/forum/best_practices_for_importing_photos__63__/]]
+* [[https://git-annex.branchable.com/tips/automatically_adding_metadata/]] 
+
+[[https://git-annex.branchable.com/git-annex-forget/]] seems to be orthogonal (forget history of everything, but not delete data)
+
+This might be more-or-less on topic but confusing to me at this point : [[https://git-annex.branchable.com/forum/How_to_get_git-annex_to_forget_a_commit__63__/]]
+
+Thank you for your attention.
diff --git a/doc/forum/Blacklisting_files___40__so_that_they_get_removed_any_time_a_copy_is_found__41__/comment_1_a224bf8ed9502053e9b95317db7b4bfa._comment b/doc/forum/Blacklisting_files___40__so_that_they_get_removed_any_time_a_copy_is_found__41__/comment_1_a224bf8ed9502053e9b95317db7b4bfa._comment
new file mode 100644
index 0000000000..46e78c2a8e
--- /dev/null
+++ b/doc/forum/Blacklisting_files___40__so_that_they_get_removed_any_time_a_copy_is_found__41__/comment_1_a224bf8ed9502053e9b95317db7b4bfa._comment
@@ -0,0 +1,24 @@
+[[!comment format=mdwn
+ username="https://launchpad.net/~stephane-gourichon-lpad"
+ nickname="stephane-gourichon-lpad"
+ subject="Sketch of implementation, request for comment"
+ date="2016-10-01T17:25:46Z"
+ content="""
+From forum entry [git annex drop not freeing space on filesystem](http://git-annex.branchable.com/forum/git_annex_drop_not_freeing_space_on_filesystem/) I understand that:
+
+* removing a file from the tree hierarchy (with `git rm`, maybe other ways) opens the possibility of freeing space locally
+* syncing with remotes propagates this possibility
+* `git annex dropunused --force` can free space
+
+So, if we have a way to remove files by content (through git, git annex, etc), e.g. starting from http://stackoverflow.com/questions/460331/git-finding-a-filename-from-a-sha1 , we can sketch a naive implementation:
+
+* (option 1, provides Q1 and Q2 above) run `git ls-tree` and filter output for blacklisted hashes (anyone knows a way to efficiently filter lines matching a pattern from a potentially big collection?), run git rm on the paths obtained
+* (option 2, provides Q3 above) run `git gilter-branch` with a filter spec that removes files by hash (is that possible?)
+* git sync with remotes
+* on each remote `git annex dropunused --force`
+
+Does that make sense?
+
+Thank you.
+
+"""]]
diff --git a/doc/forum/Blacklisting_files___40__so_that_they_get_removed_any_time_a_copy_is_found__41__/comment_2_0f2e7cf29ad9d41e0041924e69425050._comment b/doc/forum/Blacklisting_files___40__so_that_they_get_removed_any_time_a_copy_is_found__41__/comment_2_0f2e7cf29ad9d41e0041924e69425050._comment
new file mode 100644
index 0000000000..672ea738d3
--- /dev/null
+++ b/doc/forum/Blacklisting_files___40__so_that_they_get_removed_any_time_a_copy_is_found__41__/comment_2_0f2e7cf29ad9d41e0041924e69425050._comment
@@ -0,0 +1,27 @@
+[[!comment format=mdwn
+ username="joey"
+ subject="""comment 2"""
+ date="2016-10-04T15:17:49Z"
+ content="""
+The answer to this question depends on how quickly you need to get rid of
+the files, and whether you want git-annex to generally keep the content of
+deleted and old versions of modified files or not.
+
+If you only want git-annex to keep the content of files that are present in
+the current branches and tags, then simply delete a file and commit will
+work. Later using `git annex unused` to finish getting rid of the content
+of deleted files that are not used by any branches/tags.
+
+If you want to keep the full history in general, but drop the content of
+specific files, then you need to use git-annex to drop the content before
+you delete the file. You can use `git annex whereis $file` to see everywhere that
+the content has gotten to, and then `git annex drop $file --from` each location
+(and from the local repository).
+
+If you need to immediately get rid of the content of some file, you can use
+the same procedure to check where it is and drop it from those locations.
+
+You don't need to filter old commits out of branches to use `git annex
+unused`; it only looks at the most recent commit in each branch, so once a
+file has been deleted from all branches it will be identified as unused.
+"""]]
diff --git a/doc/forum/Blacklisting_files___40__so_that_they_get_removed_any_time_a_copy_is_found__41__/comment_3_a2cac051ef2d36e1ffb550c4a788c111._comment b/doc/forum/Blacklisting_files___40__so_that_they_get_removed_any_time_a_copy_is_found__41__/comment_3_a2cac051ef2d36e1ffb550c4a788c111._comment
new file mode 100644
index 0000000000..c540d55499
--- /dev/null
+++ b/doc/forum/Blacklisting_files___40__so_that_they_get_removed_any_time_a_copy_is_found__41__/comment_3_a2cac051ef2d36e1ffb550c4a788c111._comment
@@ -0,0 +1,101 @@
+[[!comment format=mdwn
+ username="https://launchpad.net/~stephane-gourichon-lpad"
+ nickname="stephane-gourichon-lpad"
+ avatar="http://cdn.libravatar.org/avatar/02d4a0af59175f9123720b4481d55a769ba954e20f6dd9b2792217d9fa0c6089"
+ subject="Work-in-progress, yet already usable, solution"
+ date="2017-03-09T12:19:49Z"
+ content="""
+TL;DR:  Thanks for clarifying the possibilities.  I chose to maintain an explicit blacklist of files that are definitely uninteresting, because it is safer and more convenient at several level (protect against mistakes, makes reviewing easier, fewer assumptions).  I share the proof-of-concept here in case anyone is interested in the future.  I might share some scripts on a public repo, especially if anyone shows interest.
+
+## Discussion
+
+> You don't need to filter old commits out of branches to use git annex unused; it only looks at the most recent commit in each branch, so once a file has been deleted from all branches it will be identified as unused.
+
+Okay, this is important to know (perhaps something to add to the documentation there).
+
+> (...) depends on how quickly you need to get rid of the files, and whether you want git-annex to generally keep the content of deleted and old versions of modified files or not.
+>
+> If you only want git-annex to keep the content of files that are present in the current branches and tags (...)
+
+In a perfect world, that would be enough.
+
+> If you want to keep the full history in general, but drop the content of specific files, (...)
+
+I realize that I need to keep the full history in general, because (it already happened) although I almost always make small independent commits and review changes before commit, large changes are relatively common.   For example renaming a directory that has many sub-directories full of files.  If some files are removed by mistake since the last commit, the information is drowned in a big change.  As git does not always track renames, it sometimes shows many additions/removal.  Also, changes are sometimes committed by a `git annex sync` (which I avoid for this reason).
+
+So, a file reported by `git unused` is not a proof that it should be deleted.
+
+So all in all, yes I \"want to keep the full history in general, but drop the content of specific files\".
+
+> then you need to use git-annex to drop the content before you delete the file.
+
+Okay, this works locally.  Yet I have started to work another way, see below.
+
+> You can use git annex whereis $file to see everywhere that the content has gotten to, and then git annex drop $file --from each location (and from the local repository).
+>
+> If you need to immediately get rid of the content of some file, you can use the same procedure to check where it is and drop it from those locations.
+
+This somehow assumes connectivity to other repositories at cleanup time, which is not practical.
+
+I might make a number of commits including some that locally prune hundreds of files, several times, before having access to any other repository.
+
+This does not leave a clear state of \"globally blacklisted\" file.
+
+
+
+**It seems to me that the situation needs maintain some state allowing to defer the removal to the time when repositories are available.**
+
+
+## Work-in-progress solution
+
+This evolved to working another way:
+
+* maintain (grow) a list of blacklisted keys that's stored somewhere in the repository and synced like any other (regular git) file.
+* have some script that can be run at any time from any repo and locally delete any file that's blacklisted.
+* variant: before locally dropping the file, check if, as far as this repo knows, it's really unused anywhere.
+
+### Proof-of-concept implementation
+
+#### Script `git_annex_drop_all_files_in_blacklist.sh`
+
+Good for a small blacklist when `git annex unused` is slow.
+
+    #!/bin/bash
+
+    set -eu # Safety, abort on error.
+
+    while read KEY REASON
+    do
+            echo Will drop blacklisted \"$KEY\"
+            git annex dropkey --force \"$KEY\"
+    done 
+
+
+"""]]
diff --git a/doc/forum/Blacklisting_files___40__so_that_they_get_removed_any_time_a_copy_is_found__41__/comment_4_d95647d91a33d1164a87344939cb769a._comment b/doc/forum/Blacklisting_files___40__so_that_they_get_removed_any_time_a_copy_is_found__41__/comment_4_d95647d91a33d1164a87344939cb769a._comment
new file mode 100644
index 0000000000..64282fb320
--- /dev/null
+++ b/doc/forum/Blacklisting_files___40__so_that_they_get_removed_any_time_a_copy_is_found__41__/comment_4_d95647d91a33d1164a87344939cb769a._comment
@@ -0,0 +1,10 @@
+[[!comment format=mdwn
+ username="CandyAngel"
+ avatar="http://cdn.libravatar.org/avatar/15c0aade8bec5bf004f939dd73cf9ed8"
+ subject="comment 4"
+ date="2017-03-10T08:11:13Z"
+ content="""
+I use git-annex-metadata to tag files I don't want, then drop the content with that tag[1]. Then it doesn't matter if the content reappears, it'll get pruned at some point. Also easy to find any symlinks pointing at the unwanted content using git-annex-find :)
+
+[1] I actually move it to another remote on a 3TB drive which only has such unwanted content, and then delete it on there when I need more space on it. Doesn't matter if that drive fails, but allows a bit of a window for an \"actually, I do need that\" recovery.
+"""]]
diff --git a/doc/forum/Box.com_hasn__39__t_been_working_for_a_few_days.mdwn b/doc/forum/Box.com_hasn__39__t_been_working_for_a_few_days.mdwn
new file mode 100644
index 0000000000..77b0928916
--- /dev/null
+++ b/doc/forum/Box.com_hasn__39__t_been_working_for_a_few_days.mdwn
@@ -0,0 +1,72 @@
+I've been experiencing problems with Box.com for a few days now and I don't know what's causing them. Is anyone else experiencing anything similar?
+
+I paste the log.
+
+    [2013-09-02 12:27:26 CEST] TransferWatcher: transfer finishing: Transfer {transferDirection = Upload, transferUUID = UUID "72111b4c-28fe-42fd-a77b-e4cb9240a1c9", transferKey = Key {keyName = "c9e1d5421e78924c21e3d68e84f80a8d1f64f9488020107ca0eeee0c4f10e763.py", keyBackendName = "SHA256E", keySize = Just 1891, keyMtime = Nothing}}
+    [2013-09-02 12:27:26 CEST] TransferScanner: queued Upload UUID "72111b4c-28fe-42fd-a77b-e4cb9240a1c9" Documentos/diveintopython-5.4/py/kgp/kant.xml Nothing : expensive scan found missing object
+    [2013-09-02 12:27:26 CEST] Transferrer: Transferring: Upload UUID "72111b4c-28fe-42fd-a77b-e4cb9240a1c9" Documentos/diveintopython-5.4/py/argecho.py Nothing
+    [2013-09-02 12:27:26 CEST] TransferWatcher: transfer starting: Upload UUID "72111b4c-28fe-42fd-a77b-e4cb9240a1c9" Documentos/diveintopython-5.4/py/argecho.py Nothing
+    [2013-09-02 12:27:26 CEST] TransferWatcher: transfer starting: Upload UUID "72111b4c-28fe-42fd-a77b-e4cb9240a1c9" Documentos/diveintopython-5.4/py/argecho.py Nothing
+    
+    
+    100%          0.0 B/s 0s[2013-09-02 12:27:26 CEST] TransferWatcher: transfer starting: Upload UUID "72111b4c-28fe-42fd-a77b-e4cb9240a1c9" Documentos/diveintopython-5.4/py/argecho.py Just 437
+    ResponseTimeout
+    ResponseTimeout
+    
+                            
+    [2013-09-02 12:27:44 CEST] TransferWatcher: transfer finishing: Transfer {transferDirection = Upload, transferUUID = UUID "72111b4c-28fe-42fd-a77b-e4cb9240a1c9", transferKey = Key {keyName = "dd3cc45d91430c6f7d68eb807f4ac1561cd0297b11a2de77b5fe66017d125798.py", keyBackendName = "SHA256E", keySize = Just 437, keyMtime = Nothing}}
+    [2013-09-02 12:27:44 CEST] TransferScanner: queued Upload UUID "72111b4c-28fe-42fd-a77b-e4cb9240a1c9" Documentos/diveintopython-5.4/py/kgp/kgp.dtd Nothing : expensive scan found missing object
+    [2013-09-02 12:27:44 CEST] Transferrer: Transferring: Upload UUID "72111b4c-28fe-42fd-a77b-e4cb9240a1c9" Documentos/diveintopython-5.4/py/autosize.py Nothing
+    [2013-09-02 12:27:44 CEST] TransferWatcher: transfer starting: Upload UUID "72111b4c-28fe-42fd-a77b-e4cb9240a1c9" Documentos/diveintopython-5.4/py/autosize.py Nothing
+    [2013-09-02 12:27:44 CEST] TransferWatcher: transfer starting: Upload UUID "72111b4c-28fe-42fd-a77b-e4cb9240a1c9" Documentos/diveintopython-5.4/py/autosize.py Nothing
+    
+    
+    100%          0.0 B/s 0s[2013-09-02 12:27:44 CEST] TransferWatcher: transfer starting: Upload UUID "72111b4c-28fe-42fd-a77b-e4cb9240a1c9" Documentos/diveintopython-5.4/py/autosize.py Just 2861
+    ResponseTimeout
+    ResponseTimeout
+    
+                            
+    [2013-09-02 12:28:02 CEST] TransferWatcher: transfer finishing: Transfer {transferDirection = Upload, transferUUID = UUID "72111b4c-28fe-42fd-a77b-e4cb9240a1c9", transferKey = Key {keyName = "d6b7940ac68768a8e37e72f248e2d94f19fb0d47062084d9baf0ec08cebbf692.py", keyBackendName = "SHA256E", keySize = Just 2861, keyMtime = Nothing}}
+    [2013-09-02 12:28:02 CEST] TransferScanner: queued Upload UUID "72111b4c-28fe-42fd-a77b-e4cb9240a1c9" Documentos/diveintopython-5.4/py/kgp/kgp.py Nothing : expensive scan found missing object
+    [2013-09-02 12:28:02 CEST] Transferrer: Transferring: Upload UUID "72111b4c-28fe-42fd-a77b-e4cb9240a1c9" Documentos/diveintopython-5.4/py/builddialectexamples.py Nothing
+    [2013-09-02 12:28:03 CEST] TransferWatcher: transfer starting: Upload UUID "72111b4c-28fe-42fd-a77b-e4cb9240a1c9" Documentos/diveintopython-5.4/py/builddialectexamples.py Nothing
+    [2013-09-02 12:28:03 CEST] TransferWatcher: transfer starting: Upload UUID "72111b4c-28fe-42fd-a77b-e4cb9240a1c9" Documentos/diveintopython-5.4/py/builddialectexamples.py Nothing
+    
+    
+    100%          0.0 B/s 0s[2013-09-02 12:28:03 CEST] TransferWatcher: transfer starting: Upload UUID "72111b4c-28fe-42fd-a77b-e4cb9240a1c9" Documentos/diveintopython-5.4/py/builddialectexamples.py Just 1090
+    ResponseTimeout
+    ResponseTimeout
+    
+                            
+    [2013-09-02 12:28:21 CEST] TransferWatcher: transfer finishing: Transfer {transferDirection = Upload, transferUUID = UUID "72111b4c-28fe-42fd-a77b-e4cb9240a1c9", transferKey = Key {keyName = "f1492b80d05b96cc7cf2904d461c99d430fa86a4eb1d99f1b155c9147ff4420f.py", keyBackendName = "SHA256E", keySize = Just 1090, keyMtime = Nothing}}
+    [2013-09-02 12:28:21 CEST] TransferScanner: queued Upload UUID "72111b4c-28fe-42fd-a77b-e4cb9240a1c9" Documentos/diveintopython-5.4/py/kgp/russiansample.xml Nothing : expensive scan found missing object
+    [2013-09-02 12:28:21 CEST] Transferrer: Transferring: Upload UUID "72111b4c-28fe-42fd-a77b-e4cb9240a1c9" Documentos/diveintopython-5.4/py/colorize.py Nothing
+    [2013-09-02 12:28:21 CEST] TransferWatcher: transfer starting: Upload UUID "72111b4c-28fe-42fd-a77b-e4cb9240a1c9" Documentos/diveintopython-5.4/py/colorize.py Nothing
+    [2013-09-02 12:28:21 CEST] TransferWatcher: transfer starting: Upload UUID "72111b4c-28fe-42fd-a77b-e4cb9240a1c9" Documentos/diveintopython-5.4/py/colorize.py Nothing
+    
+    
+    100%          0.0 B/s 0s[2013-09-02 12:28:21 CEST] TransferWatcher: transfer starting: Upload UUID "72111b4c-28fe-42fd-a77b-e4cb9240a1c9" Documentos/diveintopython-5.4/py/colorize.py Just 4864
+    ResponseTimeout
+    ResponseTimeout
+    
+    
+    [2013-09-02 12:28:40 CEST] TransferWatcher: transfer finishing: Transfer {transferDirection = Upload, transferUUID = UUID "72111b4c-28fe-42fd-a77b-e4cb9240a1c9", transferKey = Key {keyName = "b577eaf8b6ddbf9fef866c455cae248aec3b22e3f2e91aa2b75ece90f1801689.py", keyBackendName = "SHA256E", keySize = Just 4864, keyMtime = Nothing}}
+    [2013-09-02 12:28:40 CEST] TransferScanner: queued Upload UUID "72111b4c-28fe-42fd-a77b-e4cb9240a1c9" Documentos/diveintopython-5.4/py/kgp/stderr.py Nothing : expensive scan found missing object
+    [2013-09-02 12:28:40 CEST] Transferrer: Transferring: Upload UUID "72111b4c-28fe-42fd-a77b-e4cb9240a1c9" Documentos/diveintopython-5.4/py/dialect.py Nothing
+    [2013-09-02 12:28:40 CEST] TransferWatcher: transfer starting: Upload UUID "72111b4c-28fe-42fd-a77b-e4cb9240a1c9" Documentos/diveintopython-5.4/py/dialect.py Nothing
+    [2013-09-02 12:28:40 CEST] TransferWatcher: transfer starting: Upload UUID "72111b4c-28fe-42fd-a77b-e4cb9240a1c9" Documentos/diveintopython-5.4/py/dialect.py Nothing
+    
+    
+    100%          0.0 B/s 0s[2013-09-02 12:28:40 CEST] TransferWatcher: transfer starting: Upload UUID "72111b4c-28fe-42fd-a77b-e4cb9240a1c9" Documentos/diveintopython-5.4/py/dialect.py Just 4449
+    ResponseTimeout
+    ResponseTimeout
+    
+                            
+    [2013-09-02 12:28:58 CEST] TransferWatcher: transfer finishing: Transfer {transferDirection = Upload, transferUUID = UUID "72111b4c-28fe-42fd-a77b-e4cb9240a1c9", transferKey = Key {keyName = "c5e5d9b1bee2710c7ed05270a363d3e93270b0fb6779c4c8d59ace06c11db684.py", keyBackendName = "SHA256E", keySize = Just 4449, keyMtime = Nothing}}
+    [2013-09-02 12:28:58 CEST] TransferScanner: queued Upload UUID "72111b4c-28fe-42fd-a77b-e4cb9240a1c9" Documentos/diveintopython-5.4/py/kgp/stdout.py Nothing : expensive scan found missing object
+    [2013-09-02 12:28:58 CEST] Transferrer: Transferring: Upload UUID "72111b4c-28fe-42fd-a77b-e4cb9240a1c9" Documentos/diveintopython-5.4/py/fibonacci.py Nothing
+    [2013-09-02 12:28:58 CEST] TransferWatcher: transfer starting: Upload UUID "72111b4c-28fe-42fd-a77b-e4cb9240a1c9" Documentos/diveintopython-5.4/py/fibonacci.py Nothing
+    [2013-09-02 12:28:58 CEST] TransferWatcher: transfer starting: Upload UUID "72111b4c-28fe-42fd-a77b-e4cb9240a1c9" Documentos/diveintopython-5.4/py/fibonacci.py Nothing
+
+
+    100%          0.0 B/s 0s[2013-09-02 12:28:58 CEST] TransferWatcher: transfer starting: Upload UUID "72111b4c-28fe-42fd-a77b-e4cb9240a1c9" Documentos/diveintopython-5.4/py/fibonacci.py Just 532
diff --git a/doc/forum/Box.com_hasn__39__t_been_working_for_a_few_days/comment_1_6ca872c241399b9129cf9a18f42ebd43._comment b/doc/forum/Box.com_hasn__39__t_been_working_for_a_few_days/comment_1_6ca872c241399b9129cf9a18f42ebd43._comment
new file mode 100644
index 0000000000..2c7d98b49d
--- /dev/null
+++ b/doc/forum/Box.com_hasn__39__t_been_working_for_a_few_days/comment_1_6ca872c241399b9129cf9a18f42ebd43._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="http://sunny256.sunbase.org/"
+ nickname="sunny256"
+ subject="It works here"
+ date="2013-09-02T11:47:00Z"
+ content="""
+I set up a box.com remote a couple of months ago or so just for testing. Haven't used it that much, but I tested it now to see if it still works. No errors or problems. I have pasted the output from a session where I copied a file to box.com, dropped it locally, then got it back from box.com here: . The computer I ran the test on is using the newest git-annex binary (v4.20130827), Ubuntu 10.04.4 LTS. Pretty old distro, but it still works.
+"""]]
diff --git a/doc/forum/Broken_symlinks_remain_after_drop.mdwn b/doc/forum/Broken_symlinks_remain_after_drop.mdwn
new file mode 100644
index 0000000000..bc24049010
--- /dev/null
+++ b/doc/forum/Broken_symlinks_remain_after_drop.mdwn
@@ -0,0 +1,7 @@
+This is a newb question. I don't know whether this is a bug or the way git-annex is intended to function.
+
+I have two annex repos connected to each other. My idea was to have the first repository add files, which would then be moved to the second repository for storage. After moving, repo1 would be empty again, empty and clean of any symlinks.
+
+But after I 'git-annex move * --to repo2' broken symlinks remain in repo1. I don't want any broken/unused symlinks to remain in repo1 for object data it doesn't currently have (even if those files remain in the repository itself).
+
+Is there a way I can clean/remove broken symlinks to object data when those objects aren't present, so the directory only contains symlinks when the repo currently has the object data for those files?
diff --git a/doc/forum/Broken_symlinks_remain_after_drop/comment_1_d4a59b9e58d43d7a3d437e521dd5c4e1._comment b/doc/forum/Broken_symlinks_remain_after_drop/comment_1_d4a59b9e58d43d7a3d437e521dd5c4e1._comment
new file mode 100644
index 0000000000..33a9cb0581
--- /dev/null
+++ b/doc/forum/Broken_symlinks_remain_after_drop/comment_1_d4a59b9e58d43d7a3d437e521dd5c4e1._comment
@@ -0,0 +1,12 @@
+[[!comment format=mdwn
+ username="http://joeyh.name/"
+ ip="209.250.56.54"
+ subject="comment 1"
+ date="2014-10-06T15:30:54Z"
+ content="""
+git-annex is behaving as expected here. The broken symlink allows you to run `git annex get` on it to get the file content back into the repository, or `git mv` to rename the file even though its content is not present, etc.
+
+You can probably accomplish what you want by using git branches. You want a branch for repo2 that has all the files, and a branch for repo1 that has only the files in repo1. git-annex doesn't maintain such branches for you, but you can probably come up with a way to create such branches (`git annex find` will be useful when scripting up a solution).
+
+Or you can adopt the approach the git-annex assistant uses for archived files -- a archive/ directory, where files are moved when they're no longer wanted in the local system, so that their symlinks don't clutter up the view, while still being easily accessible when the time comes to pull something out of the archive.
+"""]]
diff --git a/doc/forum/Broken_symlinks_remain_after_drop/comment_2_399ba969a17a41a022c69a1f7c480857._comment b/doc/forum/Broken_symlinks_remain_after_drop/comment_2_399ba969a17a41a022c69a1f7c480857._comment
new file mode 100644
index 0000000000..f9225b0661
--- /dev/null
+++ b/doc/forum/Broken_symlinks_remain_after_drop/comment_2_399ba969a17a41a022c69a1f7c480857._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="ghen1"
+ ip="66.41.70.34"
+ subject="comment 2"
+ date="2014-10-09T02:20:14Z"
+ content="""
+Thank you for your reply and suggestions. How could I maintain an archive folder on a local repo without those changes (moved files) being reflected in the other repos? Would this require branching as well? How does the assistant do it?
+"""]]
diff --git a/doc/forum/Building_Android_app_fails.mdwn b/doc/forum/Building_Android_app_fails.mdwn
new file mode 100644
index 0000000000..bf0e69ecb1
--- /dev/null
+++ b/doc/forum/Building_Android_app_fails.mdwn
@@ -0,0 +1,69 @@
+When trying to build the Android app, I'm able to successfully set up the chroot, and enter it. Once I am inside, as the user "builder" I get a failure on the second step of the installation process "In the chroot, run standalone/android/install-haskell-packages".
+
+When I run this command, I get the following error:
+
+    cbits/cryptonite_rdrand.c: In function ‘cryptonite_get_rand_bytes’:
+    
+    cbits/cryptonite_rdrand.c:63:2:
+         error: inconsistent operand constraints in an ‘asm’
+          asm(".byte 0x48,0x0f,0xc7,0xf0; setc %1" \
+          ^
+    
+    cbits/cryptonite_rdrand.c:78:3:
+         note: in expansion of macro ‘inline_rdrand_rax’
+           inline_rdrand_rax(tmp, ok);
+           ^
+    
+    cbits/cryptonite_rdrand.c:63:2:
+         error: inconsistent operand constraints in an ‘asm’
+          asm(".byte 0x48,0x0f,0xc7,0xf0; setc %1" \
+          ^
+    
+    cbits/cryptonite_rdrand.c:87:3:
+         note: in expansion of macro ‘inline_rdrand_rax’
+           inline_rdrand_rax(tmp, ok);
+           ^
+    
+    cbits/cryptonite_rdrand.c:63:2:
+         error: inconsistent operand constraints in an ‘asm’
+          asm(".byte 0x48,0x0f,0xc7,0xf0; setc %1" \
+          ^
+    
+    cbits/cryptonite_rdrand.c:87:3:
+         note: in expansion of macro ‘inline_rdrand_rax’
+           inline_rdrand_rax(tmp, ok);
+           ^
+    
+    cbits/cryptonite_rdrand.c:63:2:
+         error: inconsistent operand constraints in an ‘asm’
+          asm(".byte 0x48,0x0f,0xc7,0xf0; setc %1" \
+          ^
+    
+    cbits/cryptonite_rdrand.c:94:3:
+         note: in expansion of macro ‘inline_rdrand_rax’
+           inline_rdrand_rax(tmp, ok);
+           ^
+    cabal: Error: some packages failed to install:
+    cryptonite-0.16 failed during the building phase. The exception was:
+    ExitFailure 1
+
+So, there is a problem with building the haskell package "cryptonite-0.16".
+
+I also get this output a little further up:
+
+    Failed to install cryptonite-0.16
+    Build log ( /home/builder/.cabal/logs/cryptonite-0.16.log ):
+    Warning: cryptonite.cabal: Unknown fields: extra-doc-files (line 37)
+    Fields allowed in this section:
+    name, version, cabal-version, build-type, license, license-file,
+    copyright, maintainer, build-depends, stability, homepage,
+    package-url, bug-reports, synopsis, description, category, author,
+    tested-with, data-files, data-dir, extra-source-files,
+    extra-tmp-files
+
+So perhaps that is helpful as well.
+
+
+If anyone has any guidance on how to move beyond this or what needs to be changed, I'd appreciate it!
+
+Thanks!
diff --git a/doc/forum/Building_Android_app_fails/comment_1_094b90cbc1e2e069900c0037fec145ad._comment b/doc/forum/Building_Android_app_fails/comment_1_094b90cbc1e2e069900c0037fec145ad._comment
new file mode 100644
index 0000000000..6d735d4cf9
--- /dev/null
+++ b/doc/forum/Building_Android_app_fails/comment_1_094b90cbc1e2e069900c0037fec145ad._comment
@@ -0,0 +1,11 @@
+[[!comment format=mdwn
+ username="joey"
+ subject="""comment 1"""
+ date="2016-06-09T19:27:03Z"
+ content="""
+FWIW, my android autobuilder currently has cryptonite-0.15 installed.
+
+Seems that version is not frozen in the cabal.config for android,
+so tried a newer one in your case. I'm not sure if that will fix your
+problem, but it may be worth a try.
+"""]]
diff --git a/doc/forum/Building_a_Debian_package_of_git-annex.mdwn b/doc/forum/Building_a_Debian_package_of_git-annex.mdwn
new file mode 100644
index 0000000000..ee222df29d
--- /dev/null
+++ b/doc/forum/Building_a_Debian_package_of_git-annex.mdwn
@@ -0,0 +1,27 @@
+I'd like to have a Debian/Ubuntu package of the latest git-annex (f.e. to have it installed systemwide). In the source I can find the debian/ folder with all needed information to build the package. But it looks like I don't have all needed dependencies on my Ubuntu 12.10:
+
+    user@laptop(pc) ~/src/git/git-annex (git)-[master] % debuild -us -uc
+    dpkg-buildpackage -rfakeroot -D -us -uc
+    dpkg-buildpackage: source package git-annex
+    dpkg-buildpackage: source version 3.20130115
+    dpkg-buildpackage: source changed by Joey Hess 
+    dpkg-source --before-build git-annex
+    dpkg-buildpackage: host architecture amd64
+    dpkg-checkbuilddeps: Unmet build dependencies: libghc-dav-dev (>= 0.3) libghc-dbus-dev (>= 0.10.3) libghc-yesod-dev libghc-yesod-static-dev libghc-yesod-default-dev libghc-hamlet-dev libghc-clientsession-dev libghc-warp-dev libghc-wai-dev libghc-wai-logger-dev libghc-case-insensitive-dev libghc-http-types-dev libghc-blaze-builder-dev libghc-crypto-api-dev libghc-network-multicast-dev libghc-network-info-dev libghc-safesemaphore-dev libghc-network-protocol-xmpp-dev (>= 0.4.3-1+b1) libghc-gnutls-dev (>= 0.1.4) libghc-xml-types-dev
+    dpkg-buildpackage: warning: build dependencies/conflicts unsatisfied; aborting
+    dpkg-buildpackage: warning: (Use -d flag to override.)
+    debuild: fatal error at line 1357:
+    dpkg-buildpackage -rfakeroot -D -us -uc failed
+    user@laptop(pc) ~/src/git/git-annex (git)-[master] % apt-get install libghc-dav-dev libghc-dbus-dev libghc-yesod-dev libghc-yesod-static-dev libghc-yesod- default-dev libghc-hamlet-dev libghc-clientsession-dev libghc-warp-dev libghc-wai-dev libghc-wai-logger-dev libghc-case-insensitive-dev libghc-http-types-dev libghc-blaze-builder-dev libghc-crypto-api-dev libghc-network-multicast-dev libghc-network-info-dev libghc-safesemaphore-dev libghc-network-protocol-xmpp-dev libghc-gnutls-dev libghc-xml-types-dev
+    Reading package lists... Done
+    Building dependency tree       
+    Reading state information... Done
+    E: Unable to locate package libghc-dav-dev
+    E: Unable to locate package libghc-network-multicast-dev
+    E: Unable to locate package libghc-network-info-dev
+    E: Unable to locate package libghc-safesemaphore-dev
+
+How can I build the package? Or would it be possible for you to build the package and add it to the "Install" page (like the prebuilt linux tarball)? That would be great!
+
+Cheers and thanks for the hard work,
+Tobias
diff --git a/doc/forum/Building_a_Debian_package_of_git-annex/comment_1_0848513c46f3efa21bc34784554ae88a._comment b/doc/forum/Building_a_Debian_package_of_git-annex/comment_1_0848513c46f3efa21bc34784554ae88a._comment
new file mode 100644
index 0000000000..39790c5ffd
--- /dev/null
+++ b/doc/forum/Building_a_Debian_package_of_git-annex/comment_1_0848513c46f3efa21bc34784554ae88a._comment
@@ -0,0 +1,10 @@
+[[!comment format=mdwn
+ username="http://joeyh.name/"
+ ip="4.154.7.238"
+ subject="comment 1"
+ date="2013-01-16T23:05:13Z"
+ content="""
+There is a package available already: 
+
+This will install even on Debian stable, I think.
+"""]]
diff --git a/doc/forum/Building_git-annex-3.20121112-19309.mdwn b/doc/forum/Building_git-annex-3.20121112-19309.mdwn
new file mode 100644
index 0000000000..68d5e32050
--- /dev/null
+++ b/doc/forum/Building_git-annex-3.20121112-19309.mdwn
@@ -0,0 +1,78 @@
+Hi,
+
+I have Problems building git-annex-3.20121112-19309, I rceive the following error:
+
+...
+  Loading object (static) dist/build/git-annex/git-annex-tmp/Utility/libmounts.o ... done
+  final link ... done
+
+  Assistant/Alert.hs:66:30:
+      Warning: default newline style has changed, using an explicit $newline is recommended
+
+  Assistant/Alert.hs:69:31:
+      Warning: default newline style has changed, using an explicit $newline is recommended
+  [157 of 279] Compiling Assistant.Types.DaemonStatus ( Assistant/Types/DaemonStatus.hs, dist/build/git-annex/git-annex-tmp/Assistant/Types/DaemonStatus.o )
+  [158 of 279] Compiling Assistant.Monad  ( Assistant/Monad.hs, dist/build/git-annex/git-annex-tmp/Assistant/Monad.o )
+
+  Assistant/Monad.hs:86:16:
+      Couldn't match expected type `Assistant a'
+                  with actual type `Reader AssistantData a'
+      Expected type: (AssistantData -> a) -> Assistant a
+        Actual type: (AssistantData -> a) -> Reader AssistantData a
+      In the expression: reader
+      In an equation for `getAssistant': getAssistant = reader
+
+  Assistant/Monad.hs:93:15:
+      Couldn't match expected type `Assistant t0'
+                  with actual type `Reader r0 a0'
+      In the return type of a call of `reader'
+      In a stmt of a 'do' block: st <- reader threadState
+      In the expression:
+        do { st <- reader threadState;
+             liftIO $ runThreadState st a }
+
+  Assistant/Monad.hs:99:14:
+      Couldn't match expected type `Assistant t0'
+                  with actual type `Reader r0 a0'
+      In the return type of a call of `reader'
+      In a stmt of a 'do' block: d <- reader id
+      In the expression:
+        do { d <- reader id;
+             liftIO $ io $ runAssistant d a }
+
+  Assistant/Monad.hs:105:14:
+      Couldn't match expected type `Assistant t0'
+                  with actual type `Reader r0 a0'
+      In the return type of a call of `reader'
+      In a stmt of a 'do' block: d <- reader id
+      In the expression:
+        do { d <- reader id;
+             return $ runAssistant d a }
+
+  Assistant/Monad.hs:110:14:
+      Couldn't match expected type `Assistant t0'
+                  with actual type `Reader r0 a0'
+      In the return type of a call of `reader'
+      In a stmt of a 'do' block: d <- reader id
+      In the expression:
+        do { d <- reader id;
+             return $ \ v -> runAssistant d $ a v }
+
+  Assistant/Monad.hs:115:14:
+      Couldn't match expected type `Assistant t0'
+                  with actual type `Reader r0 a0'
+      In the return type of a call of `reader'
+      In a stmt of a 'do' block: d <- reader id
+      In the expression:
+        do { d <- reader id;
+             return $ \ v1 v2 -> runAssistant d (a v1 v2) }
+
+  Assistant/Monad.hs:120:12:
+      Couldn't match expected type `Assistant a0'
+                  with actual type `Reader r0 a1'
+      In the return type of a call of `reader'
+      In the first argument of `(>>=)', namely `reader v'
+      In the expression: reader v >>= liftIO . io
+  cabal: Error: some packages failed to install:
+  git-annex-3.20121112 failed during the building phase. The exception was:
+  ExitFailure 1
diff --git a/doc/forum/Building_git-annex-3.20121112-19309/comment_1_b115e28c77fe748ee6643c41f766beb4._comment b/doc/forum/Building_git-annex-3.20121112-19309/comment_1_b115e28c77fe748ee6643c41f766beb4._comment
new file mode 100644
index 0000000000..3b75dfeb6d
--- /dev/null
+++ b/doc/forum/Building_git-annex-3.20121112-19309/comment_1_b115e28c77fe748ee6643c41f766beb4._comment
@@ -0,0 +1,12 @@
+[[!comment format=mdwn
+ username="http://joeyh.name/"
+ ip="4.252.11.120"
+ subject="comment 1"
+ date="2012-11-18T17:49:57Z"
+ content="""
+What platform are you building on? 
+
+What version of ghc is installed?
+
+What version of the haskell mtl library is installed? (`cabal list --installed --simple-output mtl`)
+"""]]
diff --git a/doc/forum/Building_git-annex-3.20121112-19309/comment_2_8c6ae1fd74f14da12ccfa77dbd27fc65._comment b/doc/forum/Building_git-annex-3.20121112-19309/comment_2_8c6ae1fd74f14da12ccfa77dbd27fc65._comment
new file mode 100644
index 0000000000..195f639919
--- /dev/null
+++ b/doc/forum/Building_git-annex-3.20121112-19309/comment_2_8c6ae1fd74f14da12ccfa77dbd27fc65._comment
@@ -0,0 +1,16 @@
+[[!comment format=mdwn
+ username="https://www.google.com/accounts/o8/id?id=AItOawmU_2tE75oyG0h2ZPN4lcroIKEMC8G-otE"
+ nickname="Michael"
+ subject="comment 2"
+ date="2012-11-19T06:57:54Z"
+ content="""
+This is an Ubuntu 12.04 system (3.2.0-29-generic-pae #46-Ubuntu SMP Fri Jul 27 17:25:43 UTC 2012 i686 i686 i386 GNU/Linux)
+
+ghc is 7.4.1
+
+mtl info is:
+
+MonadCatchIO-mtl 0.3.0.5
+mtl 2.0.1.0
+
+"""]]
diff --git a/doc/forum/Building_git-annex-3.20121112-19309/comment_3_2f30b301c14f3a7fa0f52715d6140353._comment b/doc/forum/Building_git-annex-3.20121112-19309/comment_3_2f30b301c14f3a7fa0f52715d6140353._comment
new file mode 100644
index 0000000000..d4f48434cb
--- /dev/null
+++ b/doc/forum/Building_git-annex-3.20121112-19309/comment_3_2f30b301c14f3a7fa0f52715d6140353._comment
@@ -0,0 +1,13 @@
+[[!comment format=mdwn
+ username="http://joeyh.name/"
+ ip="4.154.6.49"
+ subject="comment 3"
+ date="2012-11-25T18:56:01Z"
+ content="""
+On my Debian system, where it builds ok, I have the same ghc version, but version 2.1.1 of mtl. I suspect your old version of that is the problem. You can try using cabal to install a newer mtl:
+
+
    +  cabal update
+  cabal install mtl
+
    
+"""]]
diff --git a/doc/forum/Building_git-annex-3.20121112-19309/comment_4_1e3c3903a71a2ff7109372aa4dd5742a._comment b/doc/forum/Building_git-annex-3.20121112-19309/comment_4_1e3c3903a71a2ff7109372aa4dd5742a._comment
new file mode 100644
index 0000000000..1b59c3d2d0
--- /dev/null
+++ b/doc/forum/Building_git-annex-3.20121112-19309/comment_4_1e3c3903a71a2ff7109372aa4dd5742a._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="http://joeyh.name/"
+ ip="4.154.6.49"
+ subject="comment 4"
+ date="2012-11-25T20:26:33Z"
+ content="""
+I was able to reproduce this build failure with the same old version of mtl you have. I've adjusted the cabal build dependencies to require a newer version.
+"""]]
diff --git a/doc/forum/CD-R_as_special_remote.mdwn b/doc/forum/CD-R_as_special_remote.mdwn
new file mode 100644
index 0000000000..b106df9cd5
--- /dev/null
+++ b/doc/forum/CD-R_as_special_remote.mdwn
@@ -0,0 +1,10 @@
+I would like to use a CD-R as a special remote for one of my repositories. I'm not sure what the best way to do this is. I'm thinking aomething like:
+
+1. Make a new directory
+2. Add the directory as a special remote using type=directory
+3. Copy the files to the special remote
+4. Create an iso of the directory and burn it
+
+Does that sound right?
+
+I'm not sure how I can do stuff like tell git-annex that the maximum size of the remote is 700MB and that (after the initial push) the remote should be considered read only. I intend to have multiple of these CD-R/directory remotes for the one annex. Has anybody done anything like this before?
diff --git a/doc/forum/CD-R_as_special_remote/comment_1_e5d6d72953575d2bbfb6c620bb4c9454._comment b/doc/forum/CD-R_as_special_remote/comment_1_e5d6d72953575d2bbfb6c620bb4c9454._comment
new file mode 100644
index 0000000000..dd84eb60d2
--- /dev/null
+++ b/doc/forum/CD-R_as_special_remote/comment_1_e5d6d72953575d2bbfb6c620bb4c9454._comment
@@ -0,0 +1,10 @@
+[[!comment format=mdwn
+ username="joey"
+ subject="""comment 1"""
+ date="2016-04-13T18:14:33Z"
+ content="""
+This was previously discussed in .
+
+I don't think that CDs/DVDs are a particularly good fit for git-annex, but it
+seems a couple of users have gotten something working.
+"""]]
diff --git a/doc/forum/Cabal__58___Could_not_resolve_dependencies___40__yesod__41__.mdwn b/doc/forum/Cabal__58___Could_not_resolve_dependencies___40__yesod__41__.mdwn
new file mode 100644
index 0000000000..f4878da200
--- /dev/null
+++ b/doc/forum/Cabal__58___Could_not_resolve_dependencies___40__yesod__41__.mdwn
@@ -0,0 +1,19 @@
+Hi,
+
+I try to install git-annex master with cabal, so I cloned the git repo and run "cabal install --only-dependencies". This gives me the following error:
+
+    Resolving dependencies...
+    cabal: Could not resolve dependencies:
+    trying: git-annex-3.20120826 (user goal)
+    trying: git-annex-3.20120826:+oldyesod
+    trying: git-annex-3.20120826:+currentyesod
+    next goal: yesod-default (dependency of git-annex-3.20120826:+oldyesod)
+    rejecting: yesod-default-1.1.0 (conflict: git-annex-3.20120826:oldyesod =>
+    yesod-default(<=1.0.1.1))
+    rejecting: yesod-default-1.0.1.1, 1.0.1, 1.0.0, 0.6.1, 0.5.0, 0.4.1, 0.4.0,
+    0.3.1 (conflict: git-annex-3.20120826:currentyesod => yesod-default(>=1.1.0))
+
+Any idea how to fix this? (PS: I'm running Kubuntu 12.04)
+
+Cheers,
+Tobias
diff --git a/doc/forum/Cabal__58___Could_not_resolve_dependencies___40__yesod__41__/comment_1_2eb4f410b54a25fcc895893a3c631c43._comment b/doc/forum/Cabal__58___Could_not_resolve_dependencies___40__yesod__41__/comment_1_2eb4f410b54a25fcc895893a3c631c43._comment
new file mode 100644
index 0000000000..b18e74f7a1
--- /dev/null
+++ b/doc/forum/Cabal__58___Could_not_resolve_dependencies___40__yesod__41__/comment_1_2eb4f410b54a25fcc895893a3c631c43._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="http://joeyh.name/"
+ ip="4.152.246.16"
+ subject="comment 1"
+ date="2012-08-29T16:01:00Z"
+ content="""
+Hmm, cabal is not as smart as I'd hoped it could be. I've removed the support for old version of yesod.
+"""]]
diff --git a/doc/forum/Cabal__58___Could_not_resolve_dependencies___40__yesod__41__/comment_2_44cd6f6dd674df105d6f0b3f320f3236._comment b/doc/forum/Cabal__58___Could_not_resolve_dependencies___40__yesod__41__/comment_2_44cd6f6dd674df105d6f0b3f320f3236._comment
new file mode 100644
index 0000000000..4d8b0a8aa9
--- /dev/null
+++ b/doc/forum/Cabal__58___Could_not_resolve_dependencies___40__yesod__41__/comment_2_44cd6f6dd674df105d6f0b3f320f3236._comment
@@ -0,0 +1,19 @@
+[[!comment format=mdwn
+ username="https://www.google.com/accounts/o8/id?id=AItOawniayrgSdVLUc3c6bf93VbO-_HT4hzxmyo"
+ nickname="Tobias"
+ subject="comment 2"
+ date="2012-08-30T10:01:24Z"
+ content="""
+thanks, that resolved the dependency trouble.
+But now:
+
+    Building git-annex-3.20120826...
+    Preprocessing executable 'git-annex' for git-annex-3.20120826...
+
+    Utility/Yesod.hs:15:8:
+        Could not find module `Data.Default'
+        It is a member of the hidden package `data-default-0.5.0'.
+        Perhaps you need to add `data-default' to the build-depends in your .cabal file.
+        Use -v to see a list of the files searched for.
+
+"""]]
diff --git a/doc/forum/Cabal__58___Could_not_resolve_dependencies___40__yesod__41__/comment_3_992af6855901df79a2018a07941cb8b6._comment b/doc/forum/Cabal__58___Could_not_resolve_dependencies___40__yesod__41__/comment_3_992af6855901df79a2018a07941cb8b6._comment
new file mode 100644
index 0000000000..09d0dbd9af
--- /dev/null
+++ b/doc/forum/Cabal__58___Could_not_resolve_dependencies___40__yesod__41__/comment_3_992af6855901df79a2018a07941cb8b6._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="http://joeyh.name/"
+ ip="4.152.246.16"
+ subject="comment 3"
+ date="2012-08-30T16:59:07Z"
+ content="""
+The Data.Default problem is fixed now.
+"""]]
diff --git a/doc/forum/Calculating_Annex_Cost_by_Ping_Times.mdwn b/doc/forum/Calculating_Annex_Cost_by_Ping_Times.mdwn
new file mode 100644
index 0000000000..996e864c30
--- /dev/null
+++ b/doc/forum/Calculating_Annex_Cost_by_Ping_Times.mdwn
@@ -0,0 +1 @@
+I threw together a pair of shell scripts for calculating the cost of a remote using ping times.  I don't know how useful this is in practice, but the theory seemed sound to me.  If I'm in a hotel room with my two laptops, I'd rather annex try to get a file from the other laptop than from my NAS all the way back home.  I'd love to figure out how to also detect if I'm on my VerizonWireless connection at the time and multiply the cost of all connections over the Internet accordingly, but that's down the road.  Latest versions of the pair of scripts will be at . I'm interested in feedback, so please fork the git repo on gist and send me changes/updates. Also of note is that these were written for MacOSX. If you're interested in using them on a different linux, pay attention to the format of the summary line of your ping command.
diff --git a/doc/forum/Calculating_Annex_Cost_by_Ping_Times/comment_1_9b4a6bc8d52ecbbdd537e8cf76757a80._comment b/doc/forum/Calculating_Annex_Cost_by_Ping_Times/comment_1_9b4a6bc8d52ecbbdd537e8cf76757a80._comment
new file mode 100644
index 0000000000..2a2554f8d3
--- /dev/null
+++ b/doc/forum/Calculating_Annex_Cost_by_Ping_Times/comment_1_9b4a6bc8d52ecbbdd537e8cf76757a80._comment
@@ -0,0 +1,15 @@
+[[!comment format=mdwn
+ username="https://www.google.com/accounts/o8/id?id=AItOawl9sYlePmv1xK-VvjBdN-5doOa_Xw-jH4U"
+ nickname="Richard"
+ subject="comment 1"
+ date="2012-12-30T23:01:39Z"
+ content="""
+You may want to use hop count, not ping times, for a rough guesstimate of distance.
+
+Generally speaking, this is a highly non-trivial problem that's subject to university research projects so unless you find a well-maintained third party tool to solve this problem, it's unlikely that you will find a truly general solution.
+
+Anyway, look into mtr for your distance measurements.
+
+
+Richard
+"""]]
diff --git a/doc/forum/Calculating_Annex_Cost_by_Ping_Times/comment_2_7e04f85c6ba74c18c8dde148aef9bf80._comment b/doc/forum/Calculating_Annex_Cost_by_Ping_Times/comment_2_7e04f85c6ba74c18c8dde148aef9bf80._comment
new file mode 100644
index 0000000000..7dc4072eb1
--- /dev/null
+++ b/doc/forum/Calculating_Annex_Cost_by_Ping_Times/comment_2_7e04f85c6ba74c18c8dde148aef9bf80._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="https://www.google.com/accounts/o8/id?id=AItOawmRFKwny4rArBaz-36xTcsJYqKIgdDaw5Q"
+ nickname="Andrew"
+ subject="comment 2"
+ date="2013-01-06T11:08:42Z"
+ content="""
+Yeah, I was debating between ping times (max or avg) and hop count and ultimately went with ping time because of this situation: From my office to my house is ~12 hops but because we peer with the cable modem provider the ping time is still ~5ms. But from my cell modem, it's ~8 hops but ping times of ~200ms.
+"""]]
diff --git a/doc/forum/Can_I_purge_.git__47__annex__47__transfer_directory___63__.mdwn b/doc/forum/Can_I_purge_.git__47__annex__47__transfer_directory___63__.mdwn
new file mode 100644
index 0000000000..9650f26f57
--- /dev/null
+++ b/doc/forum/Can_I_purge_.git__47__annex__47__transfer_directory___63__.mdwn
@@ -0,0 +1,11 @@
+Hello.
+
+There is a git-annex repository managing about 172.000 annexed files (plus a number of small regular files in git history), for a total of about 1.7TB. At filesystem level, `find` reports about 924.000 entries (directories, files, symlinks).
+
+Inspecting it I see that `.git/annex/transfer` containes over 86000 entries (mostly files).
+
+I tried to find more information about it and only https://git-annex.branchable.com/design/assistant/syncing/ seemed to provide some information but not enough.
+
+Are they needed permanently? Can I just delete `.git/annex/transfer` without damage?
+
+Thanks.
diff --git a/doc/forum/Can_I_purge_.git__47__annex__47__transfer_directory___63__/comment_1_3510190e7c5d08f906b24e5743245f87._comment b/doc/forum/Can_I_purge_.git__47__annex__47__transfer_directory___63__/comment_1_3510190e7c5d08f906b24e5743245f87._comment
new file mode 100644
index 0000000000..2a0bfa1189
--- /dev/null
+++ b/doc/forum/Can_I_purge_.git__47__annex__47__transfer_directory___63__/comment_1_3510190e7c5d08f906b24e5743245f87._comment
@@ -0,0 +1,13 @@
+[[!comment format=mdwn
+ username="supernaught"
+ avatar="http://cdn.libravatar.org/avatar/55f92a50f2617099e2dc7509130ce158"
+ subject="comment 1"
+ date="2017-07-18T17:03:54Z"
+ content="""
+I'm not certain, but I think 
+    
+    $> git annex unused --fast
+    $> git annex drop --unused
+
+will work.
+"""]]
diff --git a/doc/forum/Can_I_purge_.git__47__annex__47__transfer_directory___63__/comment_2_bfd829aed28817a5aada3f237496c865._comment b/doc/forum/Can_I_purge_.git__47__annex__47__transfer_directory___63__/comment_2_bfd829aed28817a5aada3f237496c865._comment
new file mode 100644
index 0000000000..cd4a3bff74
--- /dev/null
+++ b/doc/forum/Can_I_purge_.git__47__annex__47__transfer_directory___63__/comment_2_bfd829aed28817a5aada3f237496c865._comment
@@ -0,0 +1,25 @@
+[[!comment format=mdwn
+ username="https://launchpad.net/~stephane-gourichon-lpad"
+ nickname="stephane-gourichon-lpad"
+ avatar="http://cdn.libravatar.org/avatar/02d4a0af59175f9123720b4481d55a769ba954e20f6dd9b2792217d9fa0c6089"
+ subject="How about find .git/annex/transfer .git/annex/bad -type f -print0 | xargs -0 rm -fv"
+ date="2017-07-19T19:42:25Z"
+ content="""
+> I'm not certain, but I think
+> 
+> $> git annex unused --fast
+> $> git annex drop --unused
+> 
+> will work.
+
+This (quickly) finds only two entries: one regarding a file in `.git/annex/transfer` and one other in `.git/annex/bad/`.
+
+Anyway, `git annex drop --unused` is too general and would potentially involve a number of other files.
+
+I'm just considering something that should probably be safe:
+
+    find .git/annex/transfer .git/annex/bad -type f -print0 | xargs -0 rm -fv
+
+unless people tell me it's dangerous somehow.  Having to do a `git annex fsck` in the future is okay.  Having important features broken until I do a `git annex fsck` (or worse) is less cool. :-)
+
+"""]]
diff --git a/doc/forum/Can_I_purge_.git__47__annex__47__transfer_directory___63__/comment_3_cf2ae4db1fdb421b59d85edd9e58b868._comment b/doc/forum/Can_I_purge_.git__47__annex__47__transfer_directory___63__/comment_3_cf2ae4db1fdb421b59d85edd9e58b868._comment
new file mode 100644
index 0000000000..3c2f8016aa
--- /dev/null
+++ b/doc/forum/Can_I_purge_.git__47__annex__47__transfer_directory___63__/comment_3_cf2ae4db1fdb421b59d85edd9e58b868._comment
@@ -0,0 +1,19 @@
+[[!comment format=mdwn
+ username="joey"
+ subject="""comment 3"""
+ date="2017-10-02T16:55:59Z"
+ content="""
+It's fine to delete .git/annex/transfer/ at any time.
+
+Those files are used to make `git annex info` (and the webapp) be able to
+show what transfers are in progress. Normally they're cleaned up when a
+transfer finishes; if you interrupt git-annex they will be left behind.
+
+Running `git annex info` is also supposed to clean up the stale transfer
+info files, but it was broken -- now fixed.
+
+The only time you get files in .git/annex/bad/ is when fsck finds a file
+got corrupted somehow and moves its content there. It's fine to delete
+those unless you were planning to recover or examine the corrupted file in
+some way.
+"""]]
diff --git a/doc/forum/Can_I_store_normal_files_in_the_git-annex_git_repository__63__.mdwn b/doc/forum/Can_I_store_normal_files_in_the_git-annex_git_repository__63__.mdwn
new file mode 100644
index 0000000000..6d1083dd57
--- /dev/null
+++ b/doc/forum/Can_I_store_normal_files_in_the_git-annex_git_repository__63__.mdwn
@@ -0,0 +1,6 @@
+Is it possible to story ordinary files in the git repository, or is this going to confuse git-annex? In other words, can I safely run
+
+    git add .gitattributes
+    git commit -m 'remember attributes' .gitattributes
+
+..., or do I have to use `git-annex add` all time?
diff --git a/doc/forum/Can_I_store_normal_files_in_the_git-annex_git_repository__63__/comment_1_c8f9923d8dc76b8bed25dce5ae09b520._comment b/doc/forum/Can_I_store_normal_files_in_the_git-annex_git_repository__63__/comment_1_c8f9923d8dc76b8bed25dce5ae09b520._comment
new file mode 100644
index 0000000000..8873edcde8
--- /dev/null
+++ b/doc/forum/Can_I_store_normal_files_in_the_git-annex_git_repository__63__/comment_1_c8f9923d8dc76b8bed25dce5ae09b520._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="http://peter-simons.myopenid.com/"
+ ip="77.12.196.1"
+ subject="Solved"
+ date="2011-07-13T16:21:25Z"
+ content="""
+I got my answer on #vcs-home: Yes, git-annex and git get along fine.
+"""]]
diff --git a/doc/forum/Can_Not_Sync_to_Git_Repo.mdwn b/doc/forum/Can_Not_Sync_to_Git_Repo.mdwn
new file mode 100644
index 0000000000..3b57829961
--- /dev/null
+++ b/doc/forum/Can_Not_Sync_to_Git_Repo.mdwn
@@ -0,0 +1 @@
+On one my repos (git repo on github data on S3) I've started getting, `fatal: Cannot force update the current branch` `git-annex: failed to update refs/heads/master` other clones of this repository can sync fine but this one started failing after adding a couple of files.
diff --git a/doc/forum/Can_Not_Sync_to_Git_Repo/comment_10_e0f82074eb1a4b8258729d9a23a7f421._comment b/doc/forum/Can_Not_Sync_to_Git_Repo/comment_10_e0f82074eb1a4b8258729d9a23a7f421._comment
new file mode 100644
index 0000000000..bf3cc836af
--- /dev/null
+++ b/doc/forum/Can_Not_Sync_to_Git_Repo/comment_10_e0f82074eb1a4b8258729d9a23a7f421._comment
@@ -0,0 +1,33 @@
+[[!comment format=mdwn
+ username="http://cstork.org/"
+ nickname="Chris Stork"
+ subject="no annex/direct/master branch"
+ date="2013-12-21T04:19:33Z"
+ content="""
+Same problem here:
+
+    % git branch -a
+      git-annex
+    * master
+      synced/git-annex
+      synced/master
+      remotes/HD02/git-annex
+      remotes/HD02/master
+      remotes/HD02/synced/git-annex
+      remotes/HD02/synced/master
+
+This is a direct mode repo, too.
+
+It seems that git-annex didn't properly update the repo.  .git/config shows:
+
+    [core]
+        repositoryformatversion = 0
+        filemode = true
+        bare = false
+        ...
+    [annex]
+        ...
+        version = 3
+        direct = true
+
+"""]]
diff --git a/doc/forum/Can_Not_Sync_to_Git_Repo/comment_11_842aaf685aa843c21bf3eef0b61f8630._comment b/doc/forum/Can_Not_Sync_to_Git_Repo/comment_11_842aaf685aa843c21bf3eef0b61f8630._comment
new file mode 100644
index 0000000000..a62cd7877f
--- /dev/null
+++ b/doc/forum/Can_Not_Sync_to_Git_Repo/comment_11_842aaf685aa843c21bf3eef0b61f8630._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="http://joeyh.name/"
+ ip="209.250.56.227"
+ subject="comment 11"
+ date="2013-12-29T17:09:26Z"
+ content="""
+I've found a bug where a repo would for some unknown reason be in direct mode but v3, and so would not be auto-upgraded to v5. This explains at least what Chris Stork shows. But I think not what previous posters are experiencing. I've fixed that bug in git.
+"""]]
diff --git a/doc/forum/Can_Not_Sync_to_Git_Repo/comment_12_b4f8be428a08db01dbd004e1f06dcffd._comment b/doc/forum/Can_Not_Sync_to_Git_Repo/comment_12_b4f8be428a08db01dbd004e1f06dcffd._comment
new file mode 100644
index 0000000000..883ae6d96f
--- /dev/null
+++ b/doc/forum/Can_Not_Sync_to_Git_Repo/comment_12_b4f8be428a08db01dbd004e1f06dcffd._comment
@@ -0,0 +1,10 @@
+[[!comment format=mdwn
+ username="http://cstork.org/"
+ nickname="Chris Stork"
+ subject="Bug fix confirmed"
+ date="2014-01-30T15:29:56Z"
+ content="""
+Thank you, Joey!
+FYI, my repo was automatically converted to version 5 and put on the annex/direct/master branch.
+As Joey mentioned, this might only apply to my bug.
+"""]]
diff --git a/doc/forum/Can_Not_Sync_to_Git_Repo/comment_13_45132d348807fbf8ed32198e110d2caa._comment b/doc/forum/Can_Not_Sync_to_Git_Repo/comment_13_45132d348807fbf8ed32198e110d2caa._comment
new file mode 100644
index 0000000000..f30f2907f6
--- /dev/null
+++ b/doc/forum/Can_Not_Sync_to_Git_Repo/comment_13_45132d348807fbf8ed32198e110d2caa._comment
@@ -0,0 +1,26 @@
+[[!comment format=mdwn
+ username="https://www.google.com/accounts/o8/id?id=AItOawlo7XnC4EU3u_9t80JUYXV7XPnUqBQ7mD4"
+ nickname="Ryan"
+ subject="comment 13"
+ date="2014-12-19T00:09:35Z"
+ content="""
+I'm having the same problem with my direct repo:
+
+git branch -a
+  git-annex
+* master
+  synced/git-annex
+  synced/master
+  remotes/cluster/git-annex
+  remotes/cluster/master
+  remotes/cluster/synced/master
+
+git annex version
+git-annex version: 5.20140818-g10bf03a
+build flags: Assistant Webapp Webapp-secure Pairing Testsuite S3 WebDAV Inotify DBus DesktopNotify XMPP DNS Feeds Quvi TDFA CryptoHash
+key/value backends: SHA256E SHA1E SHA512E SHA224E SHA384E SKEIN256E SKEIN512E SHA256 SHA1 SHA512 SHA224 SHA384 SKEIN256 SKEIN512 WORM URL
+remote types: git gcrypt S3 bup directory rsync web webdav tahoe glacier ddar hook external
+local repository version: 5
+supported repository version: 5
+upgrade supported from repository versions: 0 1 2 4
+"""]]
diff --git a/doc/forum/Can_Not_Sync_to_Git_Repo/comment_14_d2bb44a7607f3b2cddfd98cc2fa456d6._comment b/doc/forum/Can_Not_Sync_to_Git_Repo/comment_14_d2bb44a7607f3b2cddfd98cc2fa456d6._comment
new file mode 100644
index 0000000000..de76cc210d
--- /dev/null
+++ b/doc/forum/Can_Not_Sync_to_Git_Repo/comment_14_d2bb44a7607f3b2cddfd98cc2fa456d6._comment
@@ -0,0 +1,12 @@
+[[!comment format=mdwn
+ username="http://www.ryantm.com/"
+ subject="Possible workaround"
+ date="2015-09-08T14:46:03Z"
+ content="""
+The `git annex checkout annex/direct/master` workaround mentioned above did not work for me, but this did:
+
+git annex indirect
+git annex direct
+
+after that, my repo was on the annex/direct/master branch
+"""]]
diff --git a/doc/forum/Can_Not_Sync_to_Git_Repo/comment_1_80344c54804ddee81d89c0b40731fb9c._comment b/doc/forum/Can_Not_Sync_to_Git_Repo/comment_1_80344c54804ddee81d89c0b40731fb9c._comment
new file mode 100644
index 0000000000..93c4ce7791
--- /dev/null
+++ b/doc/forum/Can_Not_Sync_to_Git_Repo/comment_1_80344c54804ddee81d89c0b40731fb9c._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="Remy"
+ ip="83.87.167.98"
+ subject="comment 1"
+ date="2013-12-02T19:32:18Z"
+ content="""
+i have the same problem on one of my repo's as well.
+"""]]
diff --git a/doc/forum/Can_Not_Sync_to_Git_Repo/comment_2_1797c2fef5c20e885b56b8a2c6330ff0._comment b/doc/forum/Can_Not_Sync_to_Git_Repo/comment_2_1797c2fef5c20e885b56b8a2c6330ff0._comment
new file mode 100644
index 0000000000..8d08c32760
--- /dev/null
+++ b/doc/forum/Can_Not_Sync_to_Git_Repo/comment_2_1797c2fef5c20e885b56b8a2c6330ff0._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="http://joeyh.name/"
+ ip="209.250.56.64"
+ subject="comment 2"
+ date="2013-12-02T19:52:04Z"
+ content="""
+Post the version of git-annex you're using, and `git annex sync --debug` output.
+"""]]
diff --git a/doc/forum/Can_Not_Sync_to_Git_Repo/comment_3_fb23d640f9634cab2da91848f1848627._comment b/doc/forum/Can_Not_Sync_to_Git_Repo/comment_3_fb23d640f9634cab2da91848f1848627._comment
new file mode 100644
index 0000000000..10206a898f
--- /dev/null
+++ b/doc/forum/Can_Not_Sync_to_Git_Repo/comment_3_fb23d640f9634cab2da91848f1848627._comment
@@ -0,0 +1,58 @@
+[[!comment format=mdwn
+ username="Remy"
+ ip="83.87.167.98"
+ subject="comment 3"
+ date="2013-12-03T08:35:21Z"
+ content="""
+> git-annex version
+
+	git-annex version: 5.20131117-gbd514dc
+	build flags: Assistant Webapp Pairing Testsuite S3 WebDAV FsEvents XMPP DNS Feeds Quvi TDFA CryptoHash
+	key/value backends: SHA256E SHA1E SHA512E SHA224E SHA384E SKEIN256E SKEIN512E SHA256 SHA1 SHA512 SHA224 SHA384 SKEIN256 SKEIN512 WORM URL
+	remote types: git gcrypt S3 bup directory rsync web webdav glacier hook
+	local repository version: 5
+	default repository version: 3
+	supported repository versions: 3 5
+	upgrade supported from repository versions: 0 1 2 4
+
+
+> git-annex sync --debug bonus_remote
+
+	[2013-12-03 09:23:07 CET] read: git [\"--git-dir=/Users/remy/Sync/.git\",\"--work-tree=/Users/remy/Sync\",\"show-ref\",\"git-annex\"]
+	[2013-12-03 09:23:07 CET] read: git [\"--git-dir=/Users/remy/Sync/.git\",\"--work-tree=/Users/remy/Sync\",\"show-ref\",\"--hash\",\"refs/heads/git-annex\"]
+	[2013-12-03 09:23:07 CET] read: git [\"--git-dir=/Users/remy/Sync/.git\",\"--work-tree=/Users/remy/Sync\",\"log\",\"refs/heads/git-annex..37a902328b4806d1549bf174241339432105d756\",\"--oneline\",\"-n1\"]
+	[2013-12-03 09:23:07 CET] read: git [\"--git-dir=/Users/remy/Sync/.git\",\"--work-tree=/Users/remy/Sync\",\"log\",\"refs/heads/git-annex..5e25b4c5c8c28d48b212c079767827e27819c379\",\"--oneline\",\"-n1\"]
+	[2013-12-03 09:23:07 CET] read: git [\"--git-dir=/Users/remy/Sync/.git\",\"--work-tree=/Users/remy/Sync\",\"log\",\"refs/heads/git-annex..2245416c2355c9e00f560bcde15b8856d8f924be\",\"--oneline\",\"-n1\"]
+	[2013-12-03 09:23:07 CET] read: git [\"--git-dir=/Users/remy/Sync/.git\",\"--work-tree=/Users/remy/Sync\",\"log\",\"refs/heads/git-annex..18e56479af7031d179cd4f4af4138b15211bea26\",\"--oneline\",\"-n1\"]
+	[2013-12-03 09:23:07 CET] read: git [\"--git-dir=/Users/remy/Sync/.git\",\"--work-tree=/Users/remy/Sync\",\"log\",\"refs/heads/git-annex..b74c5b5cfeecfeace64a4b31aabd7163c6c28f42\",\"--oneline\",\"-n1\"]
+	[2013-12-03 09:23:07 CET] read: git [\"--git-dir=/Users/remy/Sync/.git\",\"--work-tree=/Users/remy/Sync\",\"log\",\"refs/heads/git-annex..88fd933c3dbb56d2527c19d6b462c61d9213535c\",\"--oneline\",\"-n1\"]
+	[2013-12-03 09:23:07 CET] chat: git [\"--git-dir=/Users/remy/Sync/.git\",\"--work-tree=/Users/remy/Sync\",\"cat-file\",\"--batch\"]
+	[2013-12-03 09:23:07 CET] read: git [\"--git-dir=/Users/remy/Sync/.git\",\"--work-tree=/Users/remy/Sync\",\"ls-files\",\"--stage\",\"-z\",\"--others\",\"--exclude-standard\",\"--\",\"/Users/remy/Sync\"]
+	[2013-12-03 09:23:07 CET] chat: git [\"--git-dir=/Users/remy/Sync/.git\",\"--work-tree=/Users/remy/Sync\",\"cat-file\",\"--batch\"]
+	commit  
+	[2013-12-03 09:23:13 CET] read: git [\"--git-dir=/Users/remy/Sync/.git\",\"--work-tree=/Users/remy/Sync\",\"commit\",\"-m\",\"git-annex automatic sync\"]
+	ok
+	[2013-12-03 09:23:13 CET] read: git [\"--git-dir=/Users/remy/Sync/.git\",\"--work-tree=/Users/remy/Sync\",\"symbolic-ref\",\"HEAD\"]
+	[2013-12-03 09:23:13 CET] read: git [\"--git-dir=/Users/remy/Sync/.git\",\"--work-tree=/Users/remy/Sync\",\"show-ref\",\"refs/heads/master\"]
+	[2013-12-03 09:23:13 CET] call: git [\"--git-dir=/Users/remy/Sync/.git\",\"--work-tree=/Users/remy/Sync\",\"show-ref\",\"--verify\",\"-q\",\"refs/heads/synced/master\"]
+	[2013-12-03 09:23:13 CET] read: git [\"--git-dir=/Users/remy/Sync/.git\",\"--work-tree=/Users/remy/Sync\",\"log\",\"refs/heads/master..refs/heads/synced/master\",\"--oneline\",\"-n1\"]
+	pull bonus_remote 
+	[2013-12-03 09:23:13 CET] call: git [\"--git-dir=/Users/remy/Sync/.git\",\"--work-tree=/Users/remy/Sync\",\"fetch\",\"bonus_remote\"]
+	[2013-12-03 09:23:14 CET] call: git [\"--git-dir=/Users/remy/Sync/.git\",\"--work-tree=/Users/remy/Sync\",\"show-ref\",\"--verify\",\"-q\",\"refs/remotes/bonus_remote/master\"]
+	[2013-12-03 09:23:14 CET] read: git [\"--git-dir=/Users/remy/Sync/.git\",\"--work-tree=/Users/remy/Sync\",\"log\",\"refs/heads/master..refs/remotes/bonus_remote/master\",\"--oneline\",\"-n1\"]
+	[2013-12-03 09:23:14 CET] call: git [\"--git-dir=/Users/remy/Sync/.git\",\"--work-tree=/Users/remy/Sync\",\"show-ref\",\"--verify\",\"-q\",\"refs/remotes/bonus_remote/synced/master\"]
+	[2013-12-03 09:23:14 CET] read: git [\"--git-dir=/Users/remy/Sync/.git\",\"--work-tree=/Users/remy/Sync\",\"log\",\"refs/heads/synced/master..refs/remotes/bonus_remote/synced/master\",\"--oneline\",\"-n1\"]
+	ok
+	[2013-12-03 09:23:14 CET] read: git [\"--git-dir=/Users/remy/Sync/.git\",\"--work-tree=/Users/remy/Sync\",\"show-ref\",\"git-annex\"]
+	[2013-12-03 09:23:14 CET] read: git [\"--git-dir=/Users/remy/Sync/.git\",\"--work-tree=/Users/remy/Sync\",\"show-ref\",\"--hash\",\"refs/heads/git-annex\"]
+	[2013-12-03 09:23:14 CET] read: git [\"--git-dir=/Users/remy/Sync/.git\",\"--work-tree=/Users/remy/Sync\",\"log\",\"refs/heads/git-annex..37a902328b4806d1549bf174241339432105d756\",\"--oneline\",\"-n1\"]
+	[2013-12-03 09:23:14 CET] read: git [\"--git-dir=/Users/remy/Sync/.git\",\"--work-tree=/Users/remy/Sync\",\"log\",\"refs/heads/git-annex..5e25b4c5c8c28d48b212c079767827e27819c379\",\"--oneline\",\"-n1\"]
+	[2013-12-03 09:23:14 CET] read: git [\"--git-dir=/Users/remy/Sync/.git\",\"--work-tree=/Users/remy/Sync\",\"log\",\"refs/heads/git-annex..2245416c2355c9e00f560bcde15b8856d8f924be\",\"--oneline\",\"-n1\"]
+	[2013-12-03 09:23:14 CET] read: git [\"--git-dir=/Users/remy/Sync/.git\",\"--work-tree=/Users/remy/Sync\",\"log\",\"refs/heads/git-annex..18e56479af7031d179cd4f4af4138b15211bea26\",\"--oneline\",\"-n1\"]
+	[2013-12-03 09:23:14 CET] read: git [\"--git-dir=/Users/remy/Sync/.git\",\"--work-tree=/Users/remy/Sync\",\"log\",\"refs/heads/git-annex..b74c5b5cfeecfeace64a4b31aabd7163c6c28f42\",\"--oneline\",\"-n1\"]
+	[2013-12-03 09:23:14 CET] read: git [\"--git-dir=/Users/remy/Sync/.git\",\"--work-tree=/Users/remy/Sync\",\"log\",\"refs/heads/git-annex..88fd933c3dbb56d2527c19d6b462c61d9213535c\",\"--oneline\",\"-n1\"]
+	[2013-12-03 09:23:14 CET] call: git [\"--git-dir=/Users/remy/Sync/.git\",\"--work-tree=/Users/remy/Sync\",\"branch\",\"-f\",\"synced/master\"]
+	[2013-12-03 09:23:14 CET] call: git [\"--git-dir=/Users/remy/Sync/.git\",\"--work-tree=/Users/remy/Sync\",\"branch\",\"-f\",\"master\"]
+	fatal: Cannot force update the current branch.
+	git-annex: failed to update refs/heads/master
+"""]]
diff --git a/doc/forum/Can_Not_Sync_to_Git_Repo/comment_4_a947736911e68856f2c3494963063df8._comment b/doc/forum/Can_Not_Sync_to_Git_Repo/comment_4_a947736911e68856f2c3494963063df8._comment
new file mode 100644
index 0000000000..c2a48b5573
--- /dev/null
+++ b/doc/forum/Can_Not_Sync_to_Git_Repo/comment_4_a947736911e68856f2c3494963063df8._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="https://www.google.com/accounts/o8/id?id=AItOawmu416zAYgYzbXVZAe30MiXoOWO4z6nGX8"
+ nickname="Johannes"
+ subject="comment 4"
+ date="2013-12-05T16:31:04Z"
+ content="""
+I have the same problem. \"git annex repair\" does not detect any problems.
+"""]]
diff --git a/doc/forum/Can_Not_Sync_to_Git_Repo/comment_5_ce0e77143cfd2d578b1e5a71e35060da._comment b/doc/forum/Can_Not_Sync_to_Git_Repo/comment_5_ce0e77143cfd2d578b1e5a71e35060da._comment
new file mode 100644
index 0000000000..9b80220fb6
--- /dev/null
+++ b/doc/forum/Can_Not_Sync_to_Git_Repo/comment_5_ce0e77143cfd2d578b1e5a71e35060da._comment
@@ -0,0 +1,18 @@
+[[!comment format=mdwn
+ username="https://www.google.com/accounts/o8/id?id=AItOawn26WQjIP5fnMgQF_L_k3Q3UrR5v8mjRTY"
+ nickname="Ellis"
+ subject="comment 5"
+ date="2013-12-07T18:14:21Z"
+ content="""
+I'm experiencing the same problem in one of my repositories on a couple computers now.
+
+> git-annex version: 5.20131118-gc7e5cde
+> build flags: Assistant Webapp Pairing S3 WebDAV Inotify DBus XMPP Feeds Quvi TDFA
+> key/value backends: SHA256E SHA1E SHA512E SHA224E SHA384E SHA256 SHA1 SHA512 SHA224 SHA384 WORM URL
+> remote types: git gcrypt S3 bup directory rsync web webdav glacier hook
+> local repository version: 3
+> default repository version: 3
+> supported repository versions: 3 5
+> upgrade supported from repository versions: 0 1 2 4
+
+"""]]
diff --git a/doc/forum/Can_Not_Sync_to_Git_Repo/comment_6_10e114da6a2bb54b860b44767ba1ca94._comment b/doc/forum/Can_Not_Sync_to_Git_Repo/comment_6_10e114da6a2bb54b860b44767ba1ca94._comment
new file mode 100644
index 0000000000..b605de7262
--- /dev/null
+++ b/doc/forum/Can_Not_Sync_to_Git_Repo/comment_6_10e114da6a2bb54b860b44767ba1ca94._comment
@@ -0,0 +1,17 @@
+[[!comment format=mdwn
+ username="https://www.google.com/accounts/o8/id?id=AItOawniayrgSdVLUc3c6bf93VbO-_HT4hzxmyo"
+ nickname="Tobias"
+ subject="same here"
+ date="2013-12-11T19:57:09Z"
+ content="""
+I'm facing the same situation as the other commenters.
+
+    git-annex version: 5.20131130-gc25be33
+    [...]
+    (merging gitlab/synced/git-annex into git-annex...)
+    (Recording state in git...)
+    fatal: Cannot force update the current branch.
+    git-annex: failed to update refs/heads/master
+
+Seems to be a problem with the command `git branch -f master`
+"""]]
diff --git a/doc/forum/Can_Not_Sync_to_Git_Repo/comment_7_415bab6a7ab564e671f42cfad83e0e58._comment b/doc/forum/Can_Not_Sync_to_Git_Repo/comment_7_415bab6a7ab564e671f42cfad83e0e58._comment
new file mode 100644
index 0000000000..8b1824ede5
--- /dev/null
+++ b/doc/forum/Can_Not_Sync_to_Git_Repo/comment_7_415bab6a7ab564e671f42cfad83e0e58._comment
@@ -0,0 +1,10 @@
+[[!comment format=mdwn
+ username="http://joeyh.name/"
+ ip="209.250.56.87"
+ subject="comment 7"
+ date="2013-12-12T17:07:37Z"
+ content="""
+What branch does `git branch` say is checked out?
+
+If you're using direct mode, you should not have the master branch checked out. Instead you should have an `annex/direct/master` branch that direct mode sets up and automatically switches you to. (Commits are still made to the master branch when syncing.)
+"""]]
diff --git a/doc/forum/Can_Not_Sync_to_Git_Repo/comment_8_36abd829ea71a44c7cded1123a7c913d._comment b/doc/forum/Can_Not_Sync_to_Git_Repo/comment_8_36abd829ea71a44c7cded1123a7c913d._comment
new file mode 100644
index 0000000000..66f75327ee
--- /dev/null
+++ b/doc/forum/Can_Not_Sync_to_Git_Repo/comment_8_36abd829ea71a44c7cded1123a7c913d._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="https://www.google.com/accounts/o8/id?id=AItOawniayrgSdVLUc3c6bf93VbO-_HT4hzxmyo"
+ nickname="Tobias"
+ subject="comment 8"
+ date="2013-12-14T13:15:13Z"
+ content="""
+The master branch was checkout out (I don't know why). Checking out the mentioned branch fixed the syncing issue: `git checkout annex/direct/master`
+"""]]
diff --git a/doc/forum/Can_Not_Sync_to_Git_Repo/comment_9_2fb745aaffe544f97bbdc670261fd4fd._comment b/doc/forum/Can_Not_Sync_to_Git_Repo/comment_9_2fb745aaffe544f97bbdc670261fd4fd._comment
new file mode 100644
index 0000000000..3bf1df783a
--- /dev/null
+++ b/doc/forum/Can_Not_Sync_to_Git_Repo/comment_9_2fb745aaffe544f97bbdc670261fd4fd._comment
@@ -0,0 +1,14 @@
+[[!comment format=mdwn
+ username="etset"
+ ip="82.155.112.106"
+ subject="comment 9"
+ date="2013-12-19T12:45:06Z"
+ content="""
+I seem to have the same problem. I have two repositories added as ssh remotes to one another with the assistant. `git-annex sync` works on one of these, but not on the other, failing with the same error as above. Also, running `git-annex sync --debug` results in the same output as in Remy's above post. `git-annex repair` doesn't find any problem. Running `git branch` on both shows that neither repository has an `annex/direct/master` branch, and both have `master` checked out. 
+
+One of the repositories has the current version from debian wheezy-backports, 4.20131106~bpo70+1, installed, and the other one had 5.20131130 from unstable as of when I discovered the problem (I upgraded it today, but the problem persists). I'm not sure, but I think those were the versions I used to create the repositores, too.
+
+Trying to reproduce the problem, I created a new repository via assistant on both computers, the one running 4.20131106~bpo70+1 and the one recently upgraded to 5.20131213. On the 5.* one, a `annex/direct/master` branch was created and checked out, but not on the 4.* one. However, adding each other as ssh remotes with the assistant makes the `annex/direct/master` on 5.* disappear and `master` to be checked out on both.
+
+Is there anything I can do to fix the repositories? Is there any other info I could provide to help with this?
+"""]]
diff --git a/doc/forum/Can__39__t_get_git-annex_merge_to_work_from_git_hook.mdwn b/doc/forum/Can__39__t_get_git-annex_merge_to_work_from_git_hook.mdwn
new file mode 100644
index 0000000000..9c0f5c7d72
--- /dev/null
+++ b/doc/forum/Can__39__t_get_git-annex_merge_to_work_from_git_hook.mdwn
@@ -0,0 +1,41 @@
+I'm trying to automate syncing of two repos A and B. My goal is to run `git annex sync` from A and have the working copy of B updated automatically. According to the manual page, `git annex merge` should to the trick. It works just fine when I run it manually in B, but not when I run it from the post-receive hook, as suggested in the manual page.
+
+Here is a test script that illustrates the issue: 
+
+The output I get:
+
+    [...]
+    file1 exists after manual git annex merge
+    [...]
+    file2 does not exist after git annex merge in post-receive
+
+From the output I can see that `git annex merge` is run on the remote end, and seems to do it's thing (`file2` is added):
+
+    remote: merge git-annex (merging synced/git-annex into git-annex...)
+    remote: ok
+    remote: merge synced/master Updating 6e5bfba..0dcbcfd
+    remote: Fast-forward
+    remote:  file2 |    1 +
+    remote:  1 file changed, 1 insertion(+)
+    remote:  create mode 120000 file2
+    remote: 
+    remote: ok
+
+However, the working copy in B does not have the file `file2`. Even worse, `git status` in B shows the file as deleted:
+
+    # On branch master
+    # Your branch is ahead of 'origin/master' by 2 commits.
+    #
+    # Changes not staged for commit:
+    #   (use "git add/rm ..." to update what will be committed)
+    #   (use "git checkout -- ..." to discard changes in working directory)
+    #
+    #	deleted:    file2
+    #
+    no changes added to commit (use "git add" and/or "git commit -a")
+
+So when running `git annex sync` from B now, the file will be deleted from A as well, which is not what I expected.
+
+This is on Ubuntu 12.04, using the precompiled git-annex tarball (amd64).
+
+What am I doing wrong?
diff --git a/doc/forum/Can__39__t_get_git-annex_merge_to_work_from_git_hook/comment_1_8b71cb6772b219c27c17392d5099907a._comment b/doc/forum/Can__39__t_get_git-annex_merge_to_work_from_git_hook/comment_1_8b71cb6772b219c27c17392d5099907a._comment
new file mode 100644
index 0000000000..cfaff89b12
--- /dev/null
+++ b/doc/forum/Can__39__t_get_git-annex_merge_to_work_from_git_hook/comment_1_8b71cb6772b219c27c17392d5099907a._comment
@@ -0,0 +1,12 @@
+[[!comment format=mdwn
+ username="http://kallesoderman.myopenid.com/"
+ ip="85.224.35.48"
+ subject="Same problem here"
+ date="2013-08-14T20:20:42Z"
+ content="""
+I'm trying to achieve [[USB_backup_with_files_visible/]] but can't make the post-recieve or post-update hooks have any effect. The filesystem is vfat so there are some issues with mounting defaults and executables (do the hooks have to be executable?) but regardless of mount options no dice.
+
+ ``git-annex merge`` at the cli works perfectly. I havent been bitten by  files being deleted and propagated though.
+
+Manually merging ruins my \"late for work need to bring data but to tired to think properly\" use case.
+"""]]
diff --git a/doc/forum/Can__39__t_get_git-annex_merge_to_work_from_git_hook/comment_2_af2a2634d8d128868022d033d6adb549._comment b/doc/forum/Can__39__t_get_git-annex_merge_to_work_from_git_hook/comment_2_af2a2634d8d128868022d033d6adb549._comment
new file mode 100644
index 0000000000..0623ed1dbb
--- /dev/null
+++ b/doc/forum/Can__39__t_get_git-annex_merge_to_work_from_git_hook/comment_2_af2a2634d8d128868022d033d6adb549._comment
@@ -0,0 +1,13 @@
+[[!comment format=mdwn
+ username="http://joeyh.name/"
+ ip="4.154.0.63"
+ subject="comment 2"
+ date="2013-08-23T18:35:01Z"
+ content="""
+Having just set up  which uses a git hook to run `git annex merge`, I can say that the main problems you are likely to run into are:
+
+1. The hook might be run with the cwd not set to the top of the git repository. cd to the git repository in the hook to fix.
+2. The hook might be run with `GIT_DIR` set to a strange value (in my case, it was set to \".\"), which is not the actual .git directory location. Unsetting it fixes that.
+
+I don't know about how to get git hooks to work on FAT filesystems though. Hooks have to be executable, and most systems probably don't mount such filesystems with executability allowed.
+"""]]
diff --git a/doc/forum/Can__39__t_get_git-annex_merge_to_work_from_git_hook/comment_3_31ec762a0684d2ce87d229ed2924db93._comment b/doc/forum/Can__39__t_get_git-annex_merge_to_work_from_git_hook/comment_3_31ec762a0684d2ce87d229ed2924db93._comment
new file mode 100644
index 0000000000..2a56b57e92
--- /dev/null
+++ b/doc/forum/Can__39__t_get_git-annex_merge_to_work_from_git_hook/comment_3_31ec762a0684d2ce87d229ed2924db93._comment
@@ -0,0 +1,16 @@
+[[!comment format=mdwn
+ username="https://www.google.com/accounts/o8/id?id=AItOawlq495p0WtZDUpxzYN9YnToZGODfCGnqOw"
+ nickname="Stanis"
+ subject="comment 3"
+ date="2013-08-24T13:30:52Z"
+ content="""
+Thanks joey, that's exactly what was happening.
+
+Just to repeat it here, if anyone else runs into the same problem, your post-receive hook has to look like this:
+
+    #!/bin/sh
+    unset GIT_DIR
+    cd ..
+    git annex merge
+
+"""]]
diff --git a/doc/forum/Can__39__t_get_git-annex_merge_to_work_from_git_hook/comment_4_cfa52c727baaf683e91c3fbb1c78072c._comment b/doc/forum/Can__39__t_get_git-annex_merge_to_work_from_git_hook/comment_4_cfa52c727baaf683e91c3fbb1c78072c._comment
new file mode 100644
index 0000000000..1890eabae0
--- /dev/null
+++ b/doc/forum/Can__39__t_get_git-annex_merge_to_work_from_git_hook/comment_4_cfa52c727baaf683e91c3fbb1c78072c._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="joey"
+ subject="""comment 4"""
+ date="2017-02-17T19:34:55Z"
+ content="""
+An easier way to set this up is available now, see 
+
+"""]]
diff --git a/doc/forum/Can__39__t_get_jabber_working.mdwn b/doc/forum/Can__39__t_get_jabber_working.mdwn
new file mode 100644
index 0000000000..6f129ee311
--- /dev/null
+++ b/doc/forum/Can__39__t_get_jabber_working.mdwn
@@ -0,0 +1,7 @@
+I've been running circles around the "jabber account setup" in assistant. I tried using jabber.org (isode) and jabber.de (openfire), but whatever I do I get thrown back to an Error page
+
+"Firefox can't establish a connection to the server at 127.0.0.1:61967." 
+
+Assistant definetly tries to connect to the server (LittleSnitch is giving me a hint) and the connections are unblocked. Even after turning LittleSnitch off completely it won't work. On the other hand, I can connect to the servers using Jitsi without any problems. I'd be really thankful for any help.
+
+David 
diff --git a/doc/forum/Can__39__t_get_jabber_working/comment_1_def20bf0b3c1a188e4dad5ec67b455d8._comment b/doc/forum/Can__39__t_get_jabber_working/comment_1_def20bf0b3c1a188e4dad5ec67b455d8._comment
new file mode 100644
index 0000000000..e2491de644
--- /dev/null
+++ b/doc/forum/Can__39__t_get_jabber_working/comment_1_def20bf0b3c1a188e4dad5ec67b455d8._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="https://www.google.com/accounts/o8/id?id=AItOawn_3tllXjSmtgm__aGr9Z4gVNFgJgGyJ30"
+ nickname="David Alan"
+ subject="Edit:"
+ date="2014-01-14T16:31:05Z"
+ content="""
+I tried this on a OS X 10.9.1 and OS X 10.8.5 as well as the regular dmg download and the autobuild.
+"""]]
diff --git a/doc/forum/Can__39__t_get_jabber_working/comment_2_7cb49c0ebfec6e0fe6784e189ed65d40._comment b/doc/forum/Can__39__t_get_jabber_working/comment_2_7cb49c0ebfec6e0fe6784e189ed65d40._comment
new file mode 100644
index 0000000000..d4b296cd52
--- /dev/null
+++ b/doc/forum/Can__39__t_get_jabber_working/comment_2_7cb49c0ebfec6e0fe6784e189ed65d40._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="http://joeyh.name/"
+ ip="209.250.56.43"
+ subject="comment 2"
+ date="2014-01-14T18:56:44Z"
+ content="""
+You apprear to be experiencing this bug: [[bugs/Share_with_friends_crash_in_osx]]
+"""]]
diff --git a/doc/forum/Can__39__t_get_jabber_working/comment_3_852dc402a286e38b77b99f174c33f8d1._comment b/doc/forum/Can__39__t_get_jabber_working/comment_3_852dc402a286e38b77b99f174c33f8d1._comment
new file mode 100644
index 0000000000..251047f264
--- /dev/null
+++ b/doc/forum/Can__39__t_get_jabber_working/comment_3_852dc402a286e38b77b99f174c33f8d1._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="https://www.google.com/accounts/o8/id?id=AItOawn_3tllXjSmtgm__aGr9Z4gVNFgJgGyJ30"
+ nickname="David Alan"
+ subject="Yes"
+ date="2014-01-15T01:41:12Z"
+ content="""
+Will put future comments there. Thanks you.
+"""]]
diff --git a/doc/forum/Can__39__t_get_jabber_working/comment_4_259741e146906ff70540390bdfe07002._comment b/doc/forum/Can__39__t_get_jabber_working/comment_4_259741e146906ff70540390bdfe07002._comment
new file mode 100644
index 0000000000..005759caf9
--- /dev/null
+++ b/doc/forum/Can__39__t_get_jabber_working/comment_4_259741e146906ff70540390bdfe07002._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="https://www.google.com/accounts/o8/id?id=AItOawlm_3m5gLhML9bHbZ8FpJ-HBZhWaRfFeO8"
+ nickname="Corey"
+ subject="Same bug? OSX 10.9.1"
+ date="2014-01-23T23:38:32Z"
+ content="""
+… using Chrome. Connection to 127.0.0.1:XXXXX interrupted when setting up Jabber.
+"""]]
diff --git a/doc/forum/Can__39__t_get_pairing_to_work.mdwn b/doc/forum/Can__39__t_get_pairing_to_work.mdwn
new file mode 100644
index 0000000000..fc45bffcb4
--- /dev/null
+++ b/doc/forum/Can__39__t_get_pairing_to_work.mdwn
@@ -0,0 +1,5 @@
+I'm trying to pair my ~/music repositories on my two laptops (Ubuntu 10.04 and 12.04) using the Linux standalone tarball on my home WiFi network. After entering the same passphrase on both machines, nothing happens, both remain in "Pairing in progress" state.
+
+The router I'm using is, I think, fairly standard, it's a ZyXEL P-2812HNU-F1 with factory settings.
+
+Are others having the problem too? Any advice where I should start looking for what goes wrong?
diff --git a/doc/forum/Can__39__t_get_pairing_to_work/comment_1_b981977b4fb942fd109c37fcf40f35d7._comment b/doc/forum/Can__39__t_get_pairing_to_work/comment_1_b981977b4fb942fd109c37fcf40f35d7._comment
new file mode 100644
index 0000000000..5e6690354a
--- /dev/null
+++ b/doc/forum/Can__39__t_get_pairing_to_work/comment_1_b981977b4fb942fd109c37fcf40f35d7._comment
@@ -0,0 +1,22 @@
+[[!comment format=mdwn
+ username="http://joeyh.name/"
+ ip="4.152.246.8"
+ subject="comment 1"
+ date="2012-10-11T23:48:37Z"
+ content="""
+It's not clear from your description whether you
+
+a) Started the pairing process independently on both machines... which probably doesn't work.
+
+b) Started pairing on one machine, and the other one noticed and popped up a pair request alert where
+you re-entered the password.
+
+If b) didn't happen, then the the pairing broadcast is not being seen by the second machine.
+You can try using tcpdump or wireshark to see the traffic. The traffic will look like this:
+
+
    +19:45:11.125893 IP 10.1.1.2.43376 > 224.0.0.1.55556: UDP, length 692
+
    
+
+If all is going well, you should be able to see that on both the machine that's initiating the pairing and the other machine. And every other machine on the network for that matter.
+"""]]
diff --git a/doc/forum/Can__39__t_get_pairing_to_work/comment_2_341e2ff6c88ace1b1422e16781edf580._comment b/doc/forum/Can__39__t_get_pairing_to_work/comment_2_341e2ff6c88ace1b1422e16781edf580._comment
new file mode 100644
index 0000000000..541b0e5812
--- /dev/null
+++ b/doc/forum/Can__39__t_get_pairing_to_work/comment_2_341e2ff6c88ace1b1422e16781edf580._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="https://www.google.com/accounts/o8/id?id=AItOawlZvtBCVyJw4O71OPsdwGpVh6iJ1W-xaPc"
+ nickname="Kilian"
+ subject="comment 2"
+ date="2012-10-17T14:39:58Z"
+ content="""
+Thanks - it seems to be a problem with my router's firmware.
+"""]]
diff --git a/doc/forum/Can__39__t_get_pairing_to_work/comment_3_0c8cce48f179f2564ff0844bb7ef6bd1._comment b/doc/forum/Can__39__t_get_pairing_to_work/comment_3_0c8cce48f179f2564ff0844bb7ef6bd1._comment
new file mode 100644
index 0000000000..f1ade11f97
--- /dev/null
+++ b/doc/forum/Can__39__t_get_pairing_to_work/comment_3_0c8cce48f179f2564ff0844bb7ef6bd1._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="http://joeyh.name/"
+ ip="2001:4978:f:21a::2"
+ subject="comment 3"
+ date="2012-10-17T17:35:11Z"
+ content="""
+Well, that's interesting. I wonder if other broadcast network services work over your router? The most common one is probably the avahi/zeroconf services.
+"""]]
diff --git a/doc/forum/Can__39__t_get_pairing_to_work/comment_4_169d77b30cea05125068ee1eeb2ef328._comment b/doc/forum/Can__39__t_get_pairing_to_work/comment_4_169d77b30cea05125068ee1eeb2ef328._comment
new file mode 100644
index 0000000000..7725a3a62a
--- /dev/null
+++ b/doc/forum/Can__39__t_get_pairing_to_work/comment_4_169d77b30cea05125068ee1eeb2ef328._comment
@@ -0,0 +1,25 @@
+[[!comment format=mdwn
+ username="https://www.google.com/accounts/o8/id?id=AItOawlZvtBCVyJw4O71OPsdwGpVh6iJ1W-xaPc"
+ nickname="Kilian"
+ subject="comment 4"
+ date="2012-10-18T22:44:40Z"
+ content="""
+Hmm... using avahi-discover the two machines can indeed see some services offered by each other. A git-annex-assistant specific problem after all?
+
+So when I initiate a pairing on 192.168.1.63, and run tcpdump on the same machine, it looks like this:
+
+ke@thot:~/opt/git-annex.linux$ sudo tcpdump -i eth1 host 224.0.0.1 -n
+tcpdump: verbose output suppressed, use -v or -vv for full protocol decode
+listening on eth1, link-type EN10MB (Ethernet), capture size 65535 bytes
+00:33:47.723989 IP 192.168.1.63.56243 > 224.0.0.1.55556: UDP, length 691
+00:33:49.729242 IP 192.168.1.63.55072 > 224.0.0.1.55556: UDP, length 691
+00:33:51.733358 IP 192.168.1.63.55115 > 224.0.0.1.55556: UDP, length 691
+00:33:53.736730 IP 192.168.1.63.60249 > 224.0.0.1.55556: UDP, length 691
+00:33:55.741641 IP 192.168.1.63.59753 > 224.0.0.1.55556: UDP, length 691
+
+The same command (modulo different interface name) on the other machine (192.168.1.59) turns up nothing:
+
+ke@apis:~/opt/git-annex.linux$ sudo tcpdump -i wlan0 host 224.0.0.1 -n
+tcpdump: verbose output suppressed, use -v or -vv for full protocol decode
+listening on wlan0, link-type EN10MB (Ethernet), capture size 96 bytes
+"""]]
diff --git a/doc/forum/Can__39__t_get_pairing_to_work/comment_5_70e6c4f4f01277be1767b38ca8374793._comment b/doc/forum/Can__39__t_get_pairing_to_work/comment_5_70e6c4f4f01277be1767b38ca8374793._comment
new file mode 100644
index 0000000000..2b249ab575
--- /dev/null
+++ b/doc/forum/Can__39__t_get_pairing_to_work/comment_5_70e6c4f4f01277be1767b38ca8374793._comment
@@ -0,0 +1,11 @@
+[[!comment format=mdwn
+ username="http://joeyh.name/"
+ ip="4.154.0.23"
+ subject="comment 5"
+ date="2012-10-22T16:18:56Z"
+ content="""
+So, the only difference I can think of is that avahi uses a different multicast address than the one I picked for git-annex. It may be that your router is only letting that one address through even though it's supposed to let a whole range through.
+
+I've prepared a test build of git-annex that uses the same 224.0.0.251 address avahi does. http://downloads.kitenet.net/tmp/git-annex-standalone-i386.tar.gz (i386 build)
+If you can try that and see how it behaves, I'll see if I need to change the address.
+"""]]
diff --git a/doc/forum/Can__39__t_get_pairing_to_work/comment_6_2cd014a76fac6e08269dfd8146957418._comment b/doc/forum/Can__39__t_get_pairing_to_work/comment_6_2cd014a76fac6e08269dfd8146957418._comment
new file mode 100644
index 0000000000..49a5a9ac90
--- /dev/null
+++ b/doc/forum/Can__39__t_get_pairing_to_work/comment_6_2cd014a76fac6e08269dfd8146957418._comment
@@ -0,0 +1,10 @@
+[[!comment format=mdwn
+ username="https://www.google.com/accounts/o8/id?id=AItOawlZvtBCVyJw4O71OPsdwGpVh6iJ1W-xaPc"
+ nickname="Kilian"
+ subject="comment 6"
+ date="2012-10-22T19:44:45Z"
+ content="""
+Yep, with the test build I see the pair request and can respond to it!
+
+Pairing doesn't seem to succeed, but that looks like a different issue. I'll try around some more and report back, probably in a new thread.
+"""]]
diff --git a/doc/forum/Can__39__t_get_pairing_to_work/comment_7_b9b715084d5a5562998b1724699d49e5._comment b/doc/forum/Can__39__t_get_pairing_to_work/comment_7_b9b715084d5a5562998b1724699d49e5._comment
new file mode 100644
index 0000000000..69d6e12e09
--- /dev/null
+++ b/doc/forum/Can__39__t_get_pairing_to_work/comment_7_b9b715084d5a5562998b1724699d49e5._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="http://joeyh.name/"
+ ip="4.154.0.23"
+ subject="comment 7"
+ date="2012-10-22T19:49:44Z"
+ content="""
+All right, I'll change the address.
+"""]]
diff --git a/doc/forum/Can__39__t_init_git_annex.mdwn b/doc/forum/Can__39__t_init_git_annex.mdwn
new file mode 100644
index 0000000000..85be0e1071
--- /dev/null
+++ b/doc/forum/Can__39__t_init_git_annex.mdwn
@@ -0,0 +1,15 @@
+It seems I can't initialize git annex:
+
+    $ git annex init "files2"
+    init files2 
+      pre-commit hook (/Volumes/project/annex/.git/hooks/pre-commit) already exists, not configuring
+    
+    git-annex: waitToSetLock: failed (Operation not supported)
+    failed
+    git-annex: init: 1 failed
+    $
+
+
+  `project` is a remote file server connected via `smb://`.
+
+   Any ideas why and how to fix?
diff --git a/doc/forum/Can__39__t_init_git_annex/comment_10_c4d2ab1ecf69718a2211c3ea7b27092b._comment b/doc/forum/Can__39__t_init_git_annex/comment_10_c4d2ab1ecf69718a2211c3ea7b27092b._comment
new file mode 100644
index 0000000000..273d0c1351
--- /dev/null
+++ b/doc/forum/Can__39__t_init_git_annex/comment_10_c4d2ab1ecf69718a2211c3ea7b27092b._comment
@@ -0,0 +1,10 @@
+[[!comment format=mdwn
+ username="http://joeyh.name/"
+ ip="4.152.246.8"
+ subject="comment 10"
+ date="2012-10-16T05:52:08Z"
+ content="""
+If you need a git repository with your regular file names on the smb share, none of the special remotes will meet your needs. You'd need to find a way to make it support POSIX locking to use git-annex on it in a full git repository.
+
+But I think that in most cases a directory special remote on such a share, with the git repository kept locally and git-annex used to pull files down to it as needed, would work ok.
+"""]]
diff --git a/doc/forum/Can__39__t_init_git_annex/comment_12_fca9ed3707e097bee2cd642424681005._comment b/doc/forum/Can__39__t_init_git_annex/comment_12_fca9ed3707e097bee2cd642424681005._comment
new file mode 100644
index 0000000000..adc9fba3bb
--- /dev/null
+++ b/doc/forum/Can__39__t_init_git_annex/comment_12_fca9ed3707e097bee2cd642424681005._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="https://www.google.com/accounts/o8/id?id=AItOawnZ0g2UAijV7RGrKtWPljCCAYHBJ3pwPvM"
+ nickname="Meng"
+ subject="comment 12"
+ date="2012-10-16T06:01:44Z"
+ content="""
+OK;   The purpose of putting files on the remote `smb://` partition is to publish these big data files, so other (potentially non-technical) people can download and use them.  They'll not be willing to learn how to use git, let alone git-annex.  And i have multiple development machines that can generate revisions/updates to these big files.  I previously just rsync from/to various development machines to/from this smb partition, but i find sometimes I don't always srync in correct direction.  The hope is i can have annex on these development machines, and set the smb:// partition as the remote for these git annex.    But it sounds like none of the special remote will have the original form, and a normal remote is not possible on smb:// partition.  
+"""]]
diff --git a/doc/forum/Can__39__t_init_git_annex/comment_1_a294b5e7e52aa9f66a708866be16f137._comment b/doc/forum/Can__39__t_init_git_annex/comment_1_a294b5e7e52aa9f66a708866be16f137._comment
new file mode 100644
index 0000000000..2dd0575eca
--- /dev/null
+++ b/doc/forum/Can__39__t_init_git_annex/comment_1_a294b5e7e52aa9f66a708866be16f137._comment
@@ -0,0 +1,10 @@
+[[!comment format=mdwn
+ username="http://joeyh.name/"
+ ip="4.152.246.8"
+ subject="comment 1"
+ date="2012-10-16T02:52:26Z"
+ content="""
+The smb:// is a red flag to me. git-annex relies on POSIX file locking, and I'll bet smb does not provide that, or, perhaps, your samba server needs a configuration change to support it.
+
+You might consider putting your git annex repository on the local filesystem, and setting up a [[special_remote|special_remotes]] on smb. The [[special_remotes/directory]] special remote is a likely choice.
+"""]]
diff --git a/doc/forum/Can__39__t_init_git_annex/comment_2_fcf678d5188821d63b4c9ea5b59474a8._comment b/doc/forum/Can__39__t_init_git_annex/comment_2_fcf678d5188821d63b4c9ea5b59474a8._comment
new file mode 100644
index 0000000000..c3596a5185
--- /dev/null
+++ b/doc/forum/Can__39__t_init_git_annex/comment_2_fcf678d5188821d63b4c9ea5b59474a8._comment
@@ -0,0 +1,13 @@
+[[!comment format=mdwn
+ username="https://www.google.com/accounts/o8/id?id=AItOawnZ0g2UAijV7RGrKtWPljCCAYHBJ3pwPvM"
+ nickname="Meng"
+ subject="comment 2"
+ date="2012-10-16T03:19:43Z"
+ content="""
+Thanks for getting back so quick; So, after doing
+
+    git annex initremote smb type=directory directory=/Volumes/subproject/ encryption=none
+    git annex describe smb \"smb://XXXhost.com/subproject\"
+
+What else do i need to do?  Do i need to `git init` and/or `git annex init` in `/Volumes/subproject/`?
+"""]]
diff --git a/doc/forum/Can__39__t_init_git_annex/comment_3_c83f7dea7d5304e226e52eb3c43ef14a._comment b/doc/forum/Can__39__t_init_git_annex/comment_3_c83f7dea7d5304e226e52eb3c43ef14a._comment
new file mode 100644
index 0000000000..c7680bc936
--- /dev/null
+++ b/doc/forum/Can__39__t_init_git_annex/comment_3_c83f7dea7d5304e226e52eb3c43ef14a._comment
@@ -0,0 +1,9 @@
+[[!comment format=mdwn
+ username="http://joeyh.name/"
+ ip="4.152.246.8"
+ subject="comment 3"
+ date="2012-10-16T04:12:55Z"
+ content="""
+Nope, you're set with just those two commands. You can now do things like \"git annex copy --to smb\"
+to put files there, and \"git annex get\" will get files from there as necessary.
+"""]]
diff --git a/doc/forum/Can__39__t_init_git_annex/comment_4_06a01dd51ffbfd006c0afb8eab40b530._comment b/doc/forum/Can__39__t_init_git_annex/comment_4_06a01dd51ffbfd006c0afb8eab40b530._comment
new file mode 100644
index 0000000000..421f98e626
--- /dev/null
+++ b/doc/forum/Can__39__t_init_git_annex/comment_4_06a01dd51ffbfd006c0afb8eab40b530._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="https://www.google.com/accounts/o8/id?id=AItOawnZ0g2UAijV7RGrKtWPljCCAYHBJ3pwPvM"
+ nickname="Meng"
+ subject="comment 4"
+ date="2012-10-16T04:17:11Z"
+ content="""
+Could i use (some variant of) `git annex sync` to do two-way sync between local and `smb`?
+"""]]
diff --git a/doc/forum/Can__39__t_init_git_annex/comment_5_53c33484bded57abc60f0449331c7b05._comment b/doc/forum/Can__39__t_init_git_annex/comment_5_53c33484bded57abc60f0449331c7b05._comment
new file mode 100644
index 0000000000..5bbbb73608
--- /dev/null
+++ b/doc/forum/Can__39__t_init_git_annex/comment_5_53c33484bded57abc60f0449331c7b05._comment
@@ -0,0 +1,11 @@
+[[!comment format=mdwn
+ username="http://joeyh.name/"
+ ip="4.152.246.8"
+ subject="comment 5"
+ date="2012-10-16T04:23:59Z"
+ content="""
+`git annex sync` only syncs the git repository, not file contents, and the special remote has no git repo, only file contents. `git annex copy` will avoid transferring things that are already there, so it's the way to go.
+
+If you want automatic syncing of file contents and lots of other magic including automatic commit of new files, you could try the [[assistant]].
+
+"""]]
diff --git a/doc/forum/Can__39__t_init_git_annex/comment_6_9e0ff44f6e62581bfc83f9f1da3e0100._comment b/doc/forum/Can__39__t_init_git_annex/comment_6_9e0ff44f6e62581bfc83f9f1da3e0100._comment
new file mode 100644
index 0000000000..36c0bb5d59
--- /dev/null
+++ b/doc/forum/Can__39__t_init_git_annex/comment_6_9e0ff44f6e62581bfc83f9f1da3e0100._comment
@@ -0,0 +1,14 @@
+[[!comment format=mdwn
+ username="https://www.google.com/accounts/o8/id?id=AItOawnZ0g2UAijV7RGrKtWPljCCAYHBJ3pwPvM"
+ nickname="Meng"
+ subject="comment 6"
+ date="2012-10-16T04:38:40Z"
+ content="""
+OK;  The reason i ask about `git annex sync` is because I had previously copied the large files onto `smb://` before setting up the `annex` and don't want to unnecessarily transfer large data over Internet again.  But now if i do `git annex copy --to smb`, and check what's added/changed in `smb` end, i found 
+ 
+    $ cd /Volumes/subproject
+    $ ls
+    091		383		bigdir1    bigfile1  ...
+
+The files with numbers in names such as`091`, `383` are being added, the `bigfile1` and `bigdir1` are the ones previously copied there.  What are those `091`, `383` etc.
+"""]]
diff --git a/doc/forum/Can__39__t_init_git_annex/comment_7_7f96b5ef05e2faf4a3dbe8bfc39b810e._comment b/doc/forum/Can__39__t_init_git_annex/comment_7_7f96b5ef05e2faf4a3dbe8bfc39b810e._comment
new file mode 100644
index 0000000000..9c29567f68
--- /dev/null
+++ b/doc/forum/Can__39__t_init_git_annex/comment_7_7f96b5ef05e2faf4a3dbe8bfc39b810e._comment
@@ -0,0 +1,10 @@
+[[!comment format=mdwn
+ username="http://joeyh.name/"
+ ip="4.152.246.8"
+ subject="comment 7"
+ date="2012-10-16T04:52:12Z"
+ content="""
+That's where it stores files in the directory special remote. You can move the similar contents of /Volumes/subproject/.git/annex/objects/ to the same place as those and then run `git annex fsck --from smb --fast` and it'll learn about those files you already transferred.
+
+You probably also want to delete /Volumes/subproject/.git afterwards, and any git working tree that was checked out there before; the directory special remote does not use the git repository that you had there before.
+"""]]
diff --git a/doc/forum/Can__39__t_init_git_annex/comment_8_65ab8463716f4ddd7721a5bcfcd18fa0._comment b/doc/forum/Can__39__t_init_git_annex/comment_8_65ab8463716f4ddd7721a5bcfcd18fa0._comment
new file mode 100644
index 0000000000..8f37408f77
--- /dev/null
+++ b/doc/forum/Can__39__t_init_git_annex/comment_8_65ab8463716f4ddd7721a5bcfcd18fa0._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="https://www.google.com/accounts/o8/id?id=AItOawnZ0g2UAijV7RGrKtWPljCCAYHBJ3pwPvM"
+ nickname="Meng"
+ subject="comment 8"
+ date="2012-10-16T05:22:54Z"
+ content="""
+Oh,  I see.   But what I need on the remote partition `/Volumes/subproject` connected via `smb://` is the actual `bigdir1` and `bigfile`, not the encoded objects.  Maybe I should use some other types of special remote?
+"""]]
diff --git a/doc/forum/Can__39__t_init_git_annex/comment_9_31a45f6a72266190b3ed7a7b02e03d5b._comment b/doc/forum/Can__39__t_init_git_annex/comment_9_31a45f6a72266190b3ed7a7b02e03d5b._comment
new file mode 100644
index 0000000000..94db3f2fef
--- /dev/null
+++ b/doc/forum/Can__39__t_init_git_annex/comment_9_31a45f6a72266190b3ed7a7b02e03d5b._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="https://www.google.com/accounts/o8/id?id=AItOawnZ0g2UAijV7RGrKtWPljCCAYHBJ3pwPvM"
+ nickname="Meng"
+ subject="comment 9"
+ date="2012-10-16T05:33:06Z"
+ content="""
+Maybe i didn't explain clearly at the beginning:  `/Volumes/subproject` is a remote partition connected via smb://.  It doesn't have a `.git`.  The content I mentioned previously \"copied\" there was copied via `cp -R`.  And my local annex is `~/project/subproject` and it does have `.git` and `.git/annex` in it.
+"""]]
diff --git a/doc/forum/Can__39__t_install__58___Mac_OS_10.8.2.mdwn b/doc/forum/Can__39__t_install__58___Mac_OS_10.8.2.mdwn
new file mode 100644
index 0000000000..d63acdcb8b
--- /dev/null
+++ b/doc/forum/Can__39__t_install__58___Mac_OS_10.8.2.mdwn
@@ -0,0 +1,36 @@
+###Tried to install git-annex.app
+
+- App hangs up
+- Cpu "git" load is 100
+- Had to force quite git-annex
+
+###Install thru command line: using Brew
+
+- Installed haskell
+- updated cabal
+- But when I do:
+
+`
+cabal install git-annex --bindir=$HOME/bin
+`
+
+**I get this**
+
+    Resolving dependencies...
+    Configuring gnuidn-0.2...
+    cabal: The program c2hs is required but it could not be found.
+    Configuring libxml-sax-0.7.3...
+    cabal: The pkg-config package libxml-2.0 is required but it could not be found.
+    cabal: Error: some packages failed to install:
+    git-annex-3.20121127.1 depends on libxml-sax-0.7.3 which failed to install.
+    gnuidn-0.2 failed during the configure step. The exception was:
+    ExitFailure 1
+    libxml-sax-0.7.3 failed during the configure step. The exception was:
+    ExitFailure 1
+    network-protocol-xmpp-0.4.4 depends on libxml-sax-0.7.3 which failed to install.
+
+Any help would be greatly appreciated. 
+
+Thanks,
+
+Carlito
diff --git a/doc/forum/Can__39__t_install__58___Mac_OS_10.8.2/comment_1_c44023d81e9e4f7c9341af0e4271a1e4._comment b/doc/forum/Can__39__t_install__58___Mac_OS_10.8.2/comment_1_c44023d81e9e4f7c9341af0e4271a1e4._comment
new file mode 100644
index 0000000000..43f86c8b04
--- /dev/null
+++ b/doc/forum/Can__39__t_install__58___Mac_OS_10.8.2/comment_1_c44023d81e9e4f7c9341af0e4271a1e4._comment
@@ -0,0 +1,10 @@
+[[!comment format=mdwn
+ username="http://joeyh.name/"
+ ip="4.153.253.113"
+ subject="comment 1"
+ date="2012-12-06T18:09:13Z"
+ content="""
+The lack of c2hs can be dealt with by `cabal install c2hs` (which the install instructions already say to do).
+
+Nobody currently understands why git commands are spinning and using all CPU when run by the app. This is being tracked at [[bugs/OSX_app_issues]].
+"""]]
diff --git a/doc/forum/Can__39__t_install__58___Mac_OS_10.8.2/comment_2_dfbcd39eedff28dc9ed866a8f1411ef3._comment b/doc/forum/Can__39__t_install__58___Mac_OS_10.8.2/comment_2_dfbcd39eedff28dc9ed866a8f1411ef3._comment
new file mode 100644
index 0000000000..904fbb15c6
--- /dev/null
+++ b/doc/forum/Can__39__t_install__58___Mac_OS_10.8.2/comment_2_dfbcd39eedff28dc9ed866a8f1411ef3._comment
@@ -0,0 +1,30 @@
+[[!comment format=mdwn
+ username="https://www.google.com/accounts/o8/id?id=AItOawlxKQEwwP6J58W0Dmfq0-v0j3LXtT5gNAA"
+ nickname="Carlito"
+ subject="Thanks"
+ date="2012-12-06T19:23:48Z"
+ content="""
+Thanks joey!
+
+Sorry missed `cabal install c2hs` in the instruction. But got passed that and still can't install.
+
+    Resolving dependencies...
+    Configuring libxml-sax-0.7.3...
+    cabal: The pkg-config package libxml-2.0 is required but it could not be found.
+    cabal: Error: some packages failed to install:
+    git-annex-3.20121127.1 depends on libxml-sax-0.7.3 which failed to install.
+    libxml-sax-0.7.3 failed during the configure step. The exception was:
+    ExitFailure 1
+    network-protocol-xmpp-0.4.4 depends on libxml-sax-0.7.3 which failed to install.
+
+I tried
+
+    brew install libxml2
+    cabal install libxml
+
+Also I added $HOME/.cabal/bin to my path
+
+Thanks,
+
+Carlito
+"""]]
diff --git a/doc/forum/Can__39__t_install__58___Mac_OS_10.8.2/comment_3_b37b2a9906ffb956cca91adb4bb4e521._comment b/doc/forum/Can__39__t_install__58___Mac_OS_10.8.2/comment_3_b37b2a9906ffb956cca91adb4bb4e521._comment
new file mode 100644
index 0000000000..dbfff2a84e
--- /dev/null
+++ b/doc/forum/Can__39__t_install__58___Mac_OS_10.8.2/comment_3_b37b2a9906ffb956cca91adb4bb4e521._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="http://joeyh.name/"
+ ip="4.153.14.162"
+ subject="comment 3"
+ date="2012-12-06T21:20:19Z"
+ content="""
+I can't help with installing libraries on OSX, but you can disable XMPP support in git-annex like this: \"cabal install git-annex -f-XMPP\"
+"""]]
diff --git a/doc/forum/Can__39__t_install__58___Mac_OS_10.8.2/comment_4_afddf16f8faedc78d458835480f10dc3._comment b/doc/forum/Can__39__t_install__58___Mac_OS_10.8.2/comment_4_afddf16f8faedc78d458835480f10dc3._comment
new file mode 100644
index 0000000000..c221455fcb
--- /dev/null
+++ b/doc/forum/Can__39__t_install__58___Mac_OS_10.8.2/comment_4_afddf16f8faedc78d458835480f10dc3._comment
@@ -0,0 +1,15 @@
+[[!comment format=mdwn
+ username="https://www.google.com/accounts/o8/id?id=AItOawlxKQEwwP6J58W0Dmfq0-v0j3LXtT5gNAA"
+ nickname="Carlito"
+ subject="Got it to work!"
+ date="2012-12-06T22:17:55Z"
+ content="""
+Had to link it: 
+
+    brew install libxml2 
+    brew link libxml2
+
+Thanks for your help.
+
+Will test out annex and leave any bug reports.
+"""]]
diff --git a/doc/forum/Can__39__t_upload_data_to_glacier_remote.mdwn b/doc/forum/Can__39__t_upload_data_to_glacier_remote.mdwn
new file mode 100644
index 0000000000..fbf95df7bc
--- /dev/null
+++ b/doc/forum/Can__39__t_upload_data_to_glacier_remote.mdwn
@@ -0,0 +1,55 @@
+I'm trying to follow the directions on [this tips page](https://git-annex.branchable.com/tips/using_Amazon_Glacier/) to easily back up some large home videos to Glacier. I followed the steps and everything worked fine until the `git annex copy`, at which point it claimed it was successful but had uploaded 0 bytes, as well as dumping the usage message for `glacier-cli` at the terminal (without any error):
+
+    Emily $ git annex copy --to glacier README
+    copy README (gpg) (checking glacier...) (to glacier...) 
+    100%          0.0 B/s 0s
+    glacier  [args]
+
+        Commands
+            vaults    - Operations with vaults
+            jobs      - Operations with jobs
+            upload    - Upload files to a vault. If the vault doesn't exits, it is
+                        created
+
+        Common args:
+            --access_key - Your AWS Access Key ID.  If not supplied, boto will
+                           use the value of the environment variable
+                           AWS_ACCESS_KEY_ID
+            --secret_key - Your AWS Secret Access Key.  If not supplied, boto
+                           will use the value of the environment variable
+                           AWS_SECRET_ACCESS_KEY
+            --region     - AWS region to use. Possible values: us-east-1, us-west-1,
+                           us-west-2, ap-northeast-1, eu-west-1.
+                           Default: us-east-1
+
+        Vaults operations:
+
+            List vaults:
+                glacier vaults 
+
+        Jobs operations:
+
+            List jobs:
+                glacier jobs 
+
+        Uploading files:
+
+            glacier upload  
+
+            Examples : 
+                glacier upload pics *.jpg
+                glacier upload pics a.jpg b.jpg
+
+    ok                      
+    (Recording state in git...)
+
+Doing a `glacier vaults` also does not show any new vaults, and getting the usage message is obviously not normal.
+
+I tried doing a manual upload to a vault I already had sitting around from some years ago called `TVault`, and that looked to work fine:
+
+    Emily $ glacier upload TVault README 
+    Uploading README to TVault... done. Vault returned ArchiveID [omitted]
+
+(The update date hasn't updated on the management console yet, but I understand that may take up to a day.)
+
+Does anyone know what's going on, or is there at least a way to get a useful error message to output?
diff --git a/doc/forum/Can__39__t_upload_data_to_glacier_remote/comment_1_cb18d5d63663f73a343c8972faadc83a._comment b/doc/forum/Can__39__t_upload_data_to_glacier_remote/comment_1_cb18d5d63663f73a343c8972faadc83a._comment
new file mode 100644
index 0000000000..5c18f19271
--- /dev/null
+++ b/doc/forum/Can__39__t_upload_data_to_glacier_remote/comment_1_cb18d5d63663f73a343c8972faadc83a._comment
@@ -0,0 +1,9 @@
+[[!comment format=mdwn
+ username="basak"
+ subject="comment 1"
+ date="2015-12-13T21:35:59Z"
+ content="""
+Sounds like you're running the boto glacier command rather than the glacier-cli tool. Unfortunately they have the same name (my intention was to deprecate the boto tool but boto upstream deprecated boto entirely instead). The current plan is to rename glacier-cli to glcr, but I haven't got round to doing that yet.
+
+Try making sure that \"glacier\" runs the glacier-cli tool rather than boto's glacier.
+"""]]
diff --git a/doc/forum/Can__39__t_upload_data_to_glacier_remote/comment_2_9b6a397d1c85da87481d36b8a4bd1ba6._comment b/doc/forum/Can__39__t_upload_data_to_glacier_remote/comment_2_9b6a397d1c85da87481d36b8a4bd1ba6._comment
new file mode 100644
index 0000000000..321d4cc37a
--- /dev/null
+++ b/doc/forum/Can__39__t_upload_data_to_glacier_remote/comment_2_9b6a397d1c85da87481d36b8a4bd1ba6._comment
@@ -0,0 +1,252 @@
+[[!comment format=mdwn
+ username="scorchgeek"
+ subject="Indeed"
+ date="2015-12-13T22:30:47Z"
+ content="""
+Nice catch! I couldn't find any install option for `glacier-cli`, so I just added a symlink to the source distribution and figured that would take care of it.
+
+    Emily $ ls /usr/bin -l | grep glacier
+    lrwxrwxrwx 1 root   root          38 Dec 11 16:19 glacier -> /home/soren/bin/glacier-cli/glacier.py
+
+But:
+
+    Emily $ which glacier
+    /usr/local/bin/glacier
+
+Oops. I changed the location of the symlink to somewhere earlier in the path, uninited and reinited the repository, and it uploaded the README.
+
+Unfortunately, now I'm getting a different error, some kind of Unicode issue, when I try to upload one of the files I actually want to back up:
+
+    Emily $ git annex add family-videos/Tape01.mpg
+    add family-videos/Tape01.mpg ok
+    (Recording state in git...)
+
+    Emily $ git commit
+    [master 5af0257] add tape01 as test
+    1 file changed, 1 insertion(+)
+    create mode 120000 family-videos/Tape01.mpg
+
+    Emily $ git annex copy --to glacier
+    copy family-videos/Tape01.mpg (gpg) (checking glacier...) (to glacier...) 
+    0%        8.1MB/s 11m32sTraceback (most recent call last):
+      File \"/home/soren/bin/glacier\", line 730, in 
+        App().main()
+      File \"/home/soren/bin/glacier\", line 716, in main
+        self.args.func()
+      File \"/home/soren/bin/glacier\", line 498, in archive_upload
+        file_obj=self.args.file, description=name)
+      File \"/usr/local/lib/python2.7/dist-packages/boto/glacier/vault.py\", line 177, in create_archive_from_file
+        writer.write(data)
+      File \"/usr/local/lib/python2.7/dist-packages/boto/glacier/writer.py\", line 219, in write
+        self.partitioner.write(data)
+      [...many more things in boto]
+      File \"/usr/lib/python2.7/httplib.py\", line 979, in request
+        self._send_request(method, url, body, headers)
+      File \"/usr/lib/python2.7/httplib.py\", line 1013, in _send_request
+        self.endheaders(body)
+      File \"/usr/lib/python2.7/httplib.py\", line 975, in endheaders
+        self._send_output(message_body)
+      File \"/usr/lib/python2.7/httplib.py\", line 833, in _send_output
+        msg += message_body
+    UnicodeDecodeError: 'ascii' codec can't decode byte 0x8c in position 0: ordinal not in range(128)
+    gpg: [stdout]: write error: Broken pipe
+    gpg: DBG: deflate: iobuf_write failed
+    gpg: build_packet failed: file write error
+    gpg: [stdout]: write error: Broken pipe
+    gpg: iobuf_flush failed on close: file write error
+    gpg: symmetric encryption of `[stdin]' failed: file write error
+    git-annex: fd:14: hPutBuf: resource vanished (Broken pipe)
+    failed
+    git-annex: copy: 1 failed
+
+Here's my complete session setting up the repository, in case it contains other useful information:
+
+    Emily 1020 [/media/backup/Videos]$ git init
+    Initialized empty Git repository in /media/backup/Videos/.git/
+    Emily 1021 [/media/backup/Videos](.)$ git remote add origin https://[myusername]@bitbucket.org/[myusername]/[myreponame].git
+    Emily 1022 [/media/backup/Videos](.)$ git push
+    fatal: The current branch master has no upstream branch.
+    To push the current branch and set the remote as upstream, use
+
+        git push --set-upstream origin master
+
+    Emily 1023 [/media/backup/Videos](.)$ git push -u origin master
+    error: src refspec master does not match any.
+    error: failed to push some refs to [URL]
+    Emily 1024 [/media/backup/Videos](.)$ git add README
+    Emily 1025 [/media/backup/Videos](@)$ ls
+    family-videos  README
+    Emily 1025 [/media/backup/Videos](@)$ git push -u origin master^C
+    Emily 1025 [/media/backup/Videos](@)$ git commit -am \"test commit\"
+    [master (root-commit) 03fb2a7] test commit
+     1 file changed, 1 insertion(+)
+     create mode 100644 README
+    Emily 1026 [/media/backup/Videos](.)$ git push -u origin master
+    Counting objects: 3, done.
+    Writing objects: 100% (3/3), 223 bytes | 0 bytes/s, done.
+    Total 3 (delta 0), reused 0 (delta 0)
+    remote:
+    remote: We're changing our IP addresses on 15 December 2015 at 00:00 UTC.
+    remote: Please make sure your firewalls are up to date:
+    remote: https://blog.bitbucket.org/?p=2677
+    To https://[myusername]@bitbucket.org/[myusername]/[myreponame].git
+     * [new branch]      master -> master
+    Branch master set up to track remote branch master from origin.
+    Emily 1027 [/media/backup/Videos](.)$ git annex init
+    init  ok
+    (Recording state in git...)
+    Emily 1028 [/media/backup/Videos](.)$ git annex --fast initremote glacier type=glacier encryption=hybrid keyid=[my GPG key ID]
+
+      Remote origin not usable by git-annex; setting annex-ignore
+    initremote glacier (encryption setup) (hybrid cipher with gpg key [hybrid key ID]) ok
+    (Recording state in git...)
+    Emily 1029 [/media/backup/Videos](.)$ git annex enableremote glacier
+    enableremote glacier (gpg) (encryption update) (hybrid cipher with gpg key [hybrid key ID]) ok
+    (Recording state in git...)
+    Emily 1030 [/media/backup/Videos](.)$ git annex copy README --to glacier
+    Emily 1031 [/media/backup/Videos](.)$ git annex sync
+    commit  ok
+    pull origin 
+    ok
+    push origin 
+    Counting objects: 13, done.
+    Delta compression using up to 4 threads.
+    Compressing objects: 100% (10/10), done.
+    Writing objects: 100% (12/12), 2.15 KiB | 0 bytes/s, done.
+    Total 12 (delta 3), reused 0 (delta 0)
+    remote: 
+    remote: We're changing our IP addresses on 15 December 2015 at 00:00 UTC.
+    remote: Please make sure your firewalls are up to date:
+    remote: https://blog.bitbucket.org/?p=2677
+    To [repository URL]
+     * [new branch]      git-annex -> synced/git-annex
+     * [new branch]      master -> synced/master
+    ok
+    Emily 1032 [/media/backup/Videos](.)$ git annex status
+    ? family-videos/Tape01.mpg
+    ? family-videos/Tape02.mpg
+    ? family-videos/Tape03.mpg
+    ? family-videos/Tape04.mpg
+    ? family-videos/Tape05.mpg
+    ? family-videos/Tape06.mpg
+    ? family-videos/Tape07.mpg
+    ? family-videos/Tape08.mpg
+    ? family-videos/Tape09.mpg
+    ? family-videos/Tape10.mpg
+    ? family-videos/Tape11.mpg
+    ? family-videos/Tape12.mpg
+    ? family-videos/Tape13.mpg
+    ? family-videos/Tape14.mpg
+    ? family-videos/Tape15.mpg
+    ? family-videos/Tape16.mpg
+    Emily 1033 [/media/backup/Videos](.)$ gitk
+    Emily 1034 [/media/backup/Videos](.)$ git annex copy README --to glacier^C
+    Emily 1034 [/media/backup/Videos](.)$ git annex add family-videos/Tap
+    ...01.mpg  ...04.mpg  ...07.mpg  ...10.mpg  ...13.mpg  ...16.mpg  
+    ...02.mpg  ...05.mpg  ...08.mpg  ...11.mpg  ...14.mpg  
+    ...03.mpg  ...06.mpg  ...09.mpg  ...12.mpg  ...15.mpg  
+    Emily 1034 [/media/backup/Videos](.)$ git annex add family-videos/Tape01.mpg
+    add family-videos/Tape01.mpg ok
+    (Recording state in git...)
+    Emily 1035 [/media/backup/Videos](@)$ git commit
+    [master 5af0257] add tape01 as test
+     1 file changed, 1 insertion(+)
+     create mode 120000 family-videos/Tape01.mpg
+    Emily 1036 [/media/backup/Videos](+)$ git annex copy --to glacier
+    copy family-videos/Tape01.mpg (gpg) (checking glacier...) (to glacier...) 
+    0%        8.1MB/s 11m32sTraceback (most recent call last):
+      File \"/home/soren/bin/glacier\", line 730, in 
+        App().main()
+      File \"/home/soren/bin/glacier\", line 716, in main
+        self.args.func()
+      File \"/home/soren/bin/glacier\", line 498, in archive_upload
+        file_obj=self.args.file, description=name)
+      File \"/usr/local/lib/python2.7/dist-packages/boto/glacier/vault.py\", line 177, in create_archive_from_file
+        writer.write(data)
+      File \"/usr/local/lib/python2.7/dist-packages/boto/glacier/writer.py\", line 219, in write
+        self.partitioner.write(data)
+      File \"/usr/local/lib/python2.7/dist-packages/boto/glacier/writer.py\", line 61, in write
+        self._send_part()
+      File \"/usr/local/lib/python2.7/dist-packages/boto/glacier/writer.py\", line 75, in _send_part
+        self.send_fn(part)
+      File \"/usr/local/lib/python2.7/dist-packages/boto/glacier/writer.py\", line 222, in _upload_part
+        self.uploader.upload_part(self.next_part_index, part_data)
+      File \"/usr/local/lib/python2.7/dist-packages/boto/glacier/writer.py\", line 129, in upload_part
+        content_range, part_data)
+      File \"/usr/local/lib/python2.7/dist-packages/boto/glacier/layer1.py\", line 1279, in upload_part
+        response_headers=response_headers)
+      File \"/usr/local/lib/python2.7/dist-packages/boto/glacier/layer1.py\", line 114, in make_request
+        data=data)
+      File \"/usr/local/lib/python2.7/dist-packages/boto/connection.py\", line 1071, in make_request
+        retry_handler=retry_handler)
+      File \"/usr/local/lib/python2.7/dist-packages/boto/connection.py\", line 943, in _mexe
+        request.body, request.headers)
+      File \"/usr/lib/python2.7/httplib.py\", line 979, in request
+        self._send_request(method, url, body, headers)
+      File \"/usr/lib/python2.7/httplib.py\", line 1013, in _send_request
+        self.endheaders(body)
+      File \"/usr/lib/python2.7/httplib.py\", line 975, in endheaders
+        self._send_output(message_body)
+      File \"/usr/lib/python2.7/httplib.py\", line 833, in _send_output
+        msg += message_body
+    UnicodeDecodeError: 'ascii' codec can't decode byte 0x8c in position 0: ordinal not in range(128)
+    gpg: [stdout]: write error: Broken pipe
+    gpg: DBG: deflate: iobuf_write failed
+    gpg: build_packet failed: file write error
+    gpg: [stdout]: write error: Broken pipe
+    gpg: iobuf_flush failed on close: file write error
+    gpg: symmetric encryption of `[stdin]' failed: file write error
+    git-annex: fd:15: hPutBuf: resource vanished (Broken pipe)
+    failed
+    git-annex: copy: 1 failed
+    Emily 1037 [/media/backup/Videos](+)$ git annex copy --to glacier
+    copy family-videos/Tape01.mpg (gpg) (checking glacier...) (to glacier...) 
+    0%        8.1MB/s 11m32sTraceback (most recent call last):
+      File \"/home/soren/bin/glacier\", line 730, in 
+        App().main()
+      File \"/home/soren/bin/glacier\", line 716, in main
+        self.args.func()
+      File \"/home/soren/bin/glacier\", line 498, in archive_upload
+        file_obj=self.args.file, description=name)
+      File \"/usr/local/lib/python2.7/dist-packages/boto/glacier/vault.py\", line 177, in create_archive_from_file
+        writer.write(data)
+      File \"/usr/local/lib/python2.7/dist-packages/boto/glacier/writer.py\", line 219, in write
+        self.partitioner.write(data)
+      File \"/usr/local/lib/python2.7/dist-packages/boto/glacier/writer.py\", line 61, in write
+        self._send_part()
+      File \"/usr/local/lib/python2.7/dist-packages/boto/glacier/writer.py\", line 75, in _send_part
+        self.send_fn(part)
+      File \"/usr/local/lib/python2.7/dist-packages/boto/glacier/writer.py\", line 222, in _upload_part
+        self.uploader.upload_part(self.next_part_index, part_data)
+      File \"/usr/local/lib/python2.7/dist-packages/boto/glacier/writer.py\", line 129, in upload_part
+        content_range, part_data)
+      File \"/usr/local/lib/python2.7/dist-packages/boto/glacier/layer1.py\", line 1279, in upload_part
+        response_headers=response_headers)
+      File \"/usr/local/lib/python2.7/dist-packages/boto/glacier/layer1.py\", line 114, in make_request
+        data=data)
+      File \"/usr/local/lib/python2.7/dist-packages/boto/connection.py\", line 1071, in make_request
+        retry_handler=retry_handler)
+      File \"/usr/local/lib/python2.7/dist-packages/boto/connection.py\", line 943, in _mexe
+        request.body, request.headers)
+      File \"/usr/lib/python2.7/httplib.py\", line 979, in request
+        self._send_request(method, url, body, headers)
+      File \"/usr/lib/python2.7/httplib.py\", line 1013, in _send_request
+        self.endheaders(body)
+      File \"/usr/lib/python2.7/httplib.py\", line 975, in endheaders
+        self._send_output(message_body)
+      File \"/usr/lib/python2.7/httplib.py\", line 833, in _send_output
+        msg += message_body
+    UnicodeDecodeError: 'ascii' codec can't decode byte 0x8c in position 0: ordinal not in range(128)
+    gpg: [stdout]: write error: Broken pipe
+    gpg: DBG: deflate: iobuf_write failed
+    gpg: build_packet failed: file write error
+    gpg: [stdout]: write error: Broken pipe
+    gpg: iobuf_flush failed on close: file write error
+    gpg: symmetric encryption of `[stdin]' failed: file write error
+    git-annex: fd:14: hPutBuf: resource vanished (Broken pipe)
+    failed                  
+    git-annex: copy: 1 failed
+    Emily 1037 [/media/backup/Videos](+)$ 
+
+Thanks a lot for your help!
+"""]]
diff --git a/doc/forum/Can__39__t_upload_data_to_glacier_remote/comment_3_6625da96f8cdc8b367b4c2bc275f2aee._comment b/doc/forum/Can__39__t_upload_data_to_glacier_remote/comment_3_6625da96f8cdc8b367b4c2bc275f2aee._comment
new file mode 100644
index 0000000000..3804eacf5b
--- /dev/null
+++ b/doc/forum/Can__39__t_upload_data_to_glacier_remote/comment_3_6625da96f8cdc8b367b4c2bc275f2aee._comment
@@ -0,0 +1,14 @@
+[[!comment format=mdwn
+ username="joey"
+ subject="""comment 3"""
+ date="2015-12-19T18:28:42Z"
+ content="""
+Note that, since version 5.20150219, git-annex probes to see if the
+"glacier" program in PATH is the one from boto, and fails with a nicer
+error message.
+
+The UnicodeDecodeError is mentioned in the thread for
+[[special_remotes/glacier]] too. There is a workaround posted in that
+thread. It would probably be good to nudge the boto maintainers to apply
+the fix, which has been available for at least 3 months now.
+"""]]
diff --git a/doc/forum/Can_a_repo_be_marked_as___34__must_have_file__63____34__.mdwn b/doc/forum/Can_a_repo_be_marked_as___34__must_have_file__63____34__.mdwn
new file mode 100644
index 0000000000..7eb024adfe
--- /dev/null
+++ b/doc/forum/Can_a_repo_be_marked_as___34__must_have_file__63____34__.mdwn
@@ -0,0 +1,5 @@
+Is there a way to mark a repo as needing to have a copy of a file?
+
+I know there's the `remote..annex-cost` setting but that's done just on a single copy of the repo.
+
+Obviously the problem here would be determining the path to get a file to a repo, so maybe this isn't possible.
diff --git a/doc/forum/Can_a_repo_be_marked_as___34__must_have_file__63____34__/comment_1_14d4d3f84eacc36dcc55af783d189619._comment b/doc/forum/Can_a_repo_be_marked_as___34__must_have_file__63____34__/comment_1_14d4d3f84eacc36dcc55af783d189619._comment
new file mode 100644
index 0000000000..adf1d1c2d8
--- /dev/null
+++ b/doc/forum/Can_a_repo_be_marked_as___34__must_have_file__63____34__/comment_1_14d4d3f84eacc36dcc55af783d189619._comment
@@ -0,0 +1,10 @@
+[[!comment format=mdwn
+ username="kevin@f9ce8e46ad8c29b25093b82cf68fae38f54b89a1"
+ nickname="kevin"
+ subject="Groups..."
+ date="2016-09-18T20:35:00Z"
+ content="""
+The answer seems to be [groups](https://git-annex.branchable.com/preferred_content/standard_groups/).
+
+If I set the two repos I want to have content to be in the backup group, they'll always try to get every file the see.
+"""]]
diff --git a/doc/forum/Can_a_repo_be_marked_as___34__must_have_file__63____34__/comment_2_409d6a97f5c3d1f1fc5e932a2965e227._comment b/doc/forum/Can_a_repo_be_marked_as___34__must_have_file__63____34__/comment_2_409d6a97f5c3d1f1fc5e932a2965e227._comment
new file mode 100644
index 0000000000..65ff9ed903
--- /dev/null
+++ b/doc/forum/Can_a_repo_be_marked_as___34__must_have_file__63____34__/comment_2_409d6a97f5c3d1f1fc5e932a2965e227._comment
@@ -0,0 +1,16 @@
+[[!comment format=mdwn
+ username="joey"
+ subject="""comment 2"""
+ date="2016-09-21T16:46:16Z"
+ content="""
+Cost settings have nothing to do with this.
+
+[[Preferred_content]] can be configured to include specific files
+that you want to have in the repository, by matching on the filename.
+For example:
+
+	git annex wanted . "include=somefile or include=otherfile or include=*.mp3"
+
+Putting the repository in the backup group is similar to setting
+"include=*"
+"""]]
diff --git a/doc/forum/Can_git-annex-import_--clean-duplicates_honour_multiple_backends__63__.mdwn b/doc/forum/Can_git-annex-import_--clean-duplicates_honour_multiple_backends__63__.mdwn
new file mode 100644
index 0000000000..a75dd0f1b8
--- /dev/null
+++ b/doc/forum/Can_git-annex-import_--clean-duplicates_honour_multiple_backends__63__.mdwn
@@ -0,0 +1,25 @@
+Is there a better way to de-duplicate in a way that considers multiple backends?
+
+Multiple backends can be added to the .git/config annex.backends entry, but what is the purpose of the secondary backends? The first is used when adding new files, but the second (third, fourth, ...) do not seem to serve any purpose. (Or am I missing something?)
+
+Here's my use case, problem, and a possible solution. I frequently use git-annex to de-duplicate. The default SHA256E backend has caused issues since filename case is significant, so I have partially switched to SHA256. I also occasionally use other backends. Now when I'm given an arbitrary file, as far as I can tell, I have to try de-duplicate once for every possible backend which amounts to something like
+
+    for i SHA256E SHA256 SKEIN256 ... ; do
+        [ -f /tmp/afile.pdf ] && git annex import --clean-duplicates --backend=$i /tmp/afile.pdf
+    done
+
+even though my .git/config has annex.backends = "SHA256E SHA256 SKEIN256 ...". I was surprised that `--clean-duplicates` does not honour all listed annex.backends. In this case hashing multiple times as needed seems quite reasonable IMO, so adding multiple backend support for `--clean-duplicates` would solve the problem. If you're not keen to modify this existing behaviour, it might be instead sensible to have to opt-in by explicitly specifying all backends to consider, like
+
+    git annex import --clean-duplicates --backends="SHA256E SHA256 SKEIN256" /tmp/afile.pdf
+
+or
+
+    git annex import --clean-duplicates --backends="$( git config --get annex.backends )" /tmp/afile.pdf
+
+Moving this loop into git-annex would also allow hashing to be parallelized; it currently cannot because the file could disappear.
+
+- - -
+
+PS. Thanks for git-annex Joey. I have around 100 annexes and rely on them on a daily basis.
+
+-supernaught
diff --git a/doc/forum/Can_git-annex-import_--clean-duplicates_honour_multiple_backends__63__/comment_1_5730767d5247e8997b4fa5b20c4cb281._comment b/doc/forum/Can_git-annex-import_--clean-duplicates_honour_multiple_backends__63__/comment_1_5730767d5247e8997b4fa5b20c4cb281._comment
new file mode 100644
index 0000000000..1b6ebca993
--- /dev/null
+++ b/doc/forum/Can_git-annex-import_--clean-duplicates_honour_multiple_backends__63__/comment_1_5730767d5247e8997b4fa5b20c4cb281._comment
@@ -0,0 +1,35 @@
+[[!comment format=mdwn
+ username="joey"
+ subject="""comment 1"""
+ date="2017-05-09T18:06:34Z"
+ content="""
+When git-annex is adding a file, a backend can chose to not
+generate any key, and then it will try the next backend in the list.
+
+The only backend that does that is the URL backend.
+So if someone lists URL first for some reason, it'll fall back to a backend
+that is usable. It could just as well crash in that edge case; the
+annex.backends UI happened before the needs of backends were perfectly
+understood. (As did the "backend" name...)
+
+Anyway, I see the use case, but.. `git annex import` actually honors
+annex.backend settings in .gitattributes before annex.backends in
+git-config. So, relying on it using the latter to make it check multiple
+backends won't always work. I don't think it would be good to complicate
+the .gitattributes annex.backends and --backend to support a list of
+backends.
+
+It seems it would be just as fast for you to run git-annex import once per
+backend, rather than compliciating it to try multiple backends. 
+
+I think that if annex.backends were not a list for historical reasons, 
+I'd be suggesting a small shell script is your best option. 
+
+And so rather than add a new feature just because annex.backends is
+historically a list, I'd rather perhaps deprecate annex.backends as
+unncessarily complicated, and make annex.backend be a single-backend
+setting. (Just did that.)
+
+Sorry this didn't quite go the way you wanted! If there is a disadvantage
+to the simple shell script option, please do let me know..
+"""]]
diff --git a/doc/forum/Can_git-annex-import_--clean-duplicates_honour_multiple_backends__63__/comment_2_c2846031bd79fa60cd903fb4d5bcebaf._comment b/doc/forum/Can_git-annex-import_--clean-duplicates_honour_multiple_backends__63__/comment_2_c2846031bd79fa60cd903fb4d5bcebaf._comment
new file mode 100644
index 0000000000..7a381bf518
--- /dev/null
+++ b/doc/forum/Can_git-annex-import_--clean-duplicates_honour_multiple_backends__63__/comment_2_c2846031bd79fa60cd903fb4d5bcebaf._comment
@@ -0,0 +1,10 @@
+[[!comment format=mdwn
+ username="supernaught"
+ avatar="http://cdn.libravatar.org/avatar/55f92a50f2617099e2dc7509130ce158"
+ subject="comment 2"
+ date="2017-05-09T22:07:59Z"
+ content="""
+Your solution is reasonable. Simpler is better.
+
+Thanks.
+"""]]
diff --git a/doc/forum/Can_git_annex___40__mostly__41___be_an_Ubuntu_One___40__or_Dropbox...__41___substitute__63__.mdwn b/doc/forum/Can_git_annex___40__mostly__41___be_an_Ubuntu_One___40__or_Dropbox...__41___substitute__63__.mdwn
new file mode 100644
index 0000000000..0b49f422fa
--- /dev/null
+++ b/doc/forum/Can_git_annex___40__mostly__41___be_an_Ubuntu_One___40__or_Dropbox...__41___substitute__63__.mdwn
@@ -0,0 +1,76 @@
+Hi, I tried with several git annex versions, on 3 different clients, and using 2 different remotes (an home server and box.com) and trying out different remotes setups but I've been unable to get git annex working as I expected.
+
+
+I used the assistant for setup, I can sync files between clients just fine if they're connected at the same time, but if I sync a file from client A to the remotes when client B is off, and then later I turn off client A and power on client B, I've never been able to successfully sync files.
+
+on client B, inside .git/annex/daemon.log there's nothing interesting: just messages like:
+
+```(scanning...) [2014-06-03 14:53:26 CEST] Watcher: Performing startup scan
+
+Already up-to-date.```
+
+`git annex sync` just outputs
+
+`"commit  ok"`
+
+and `git annex list`:
+
+```
+here
+
+|berdario
+
+||soloud
+
+|||web
+
+||||box.com
+
+|||||
+
+XX__X IMG_20130202_100444.jpg
+
+XXX_X cookies.png
+
+XXX_X dancingllama.png
+
+XXX_X dario_bertini_cv.pdf
+
+XX__X steam_latest.deb
+```
+
+while this is what I get on client A:
+
+```
+here
+
+|berdario
+
+||web 
+
+|||box.com 
+
+||||
+
+XX_X IMG_20130202_100444.jpg
+
+X__X Russian Lesson 5 - Wikibooks, open books for an open world.html
+
+XX_X cookies.png
+
+XX_X dancingllama.png
+
+XX_X dario_bertini_cv.pdf
+
+XX_X steam_latest.deb
+```
+
+(they're just a bunch of random files I'm using to test it, both clients are called berdario, soloud is the home server (currently down) and the other working remote is box.com)
+
+As you can see the russian wikibooks html file is successfully synced with the remotes, but client B is unable to see it...
+
+I tried to set the remotes as incremental backup, full backup, transfer, client (!?) but none of these settings work.
+
+Is git annex not what I'm looking for? Is it supposed to work only on contemporarily connected clients?
+
+Thanks
diff --git a/doc/forum/Can_git_annex___40__mostly__41___be_an_Ubuntu_One___40__or_Dropbox...__41___substitute__63__/comment_1_a3fbae205c0312436f8861f432643811._comment b/doc/forum/Can_git_annex___40__mostly__41___be_an_Ubuntu_One___40__or_Dropbox...__41___substitute__63__/comment_1_a3fbae205c0312436f8861f432643811._comment
new file mode 100644
index 0000000000..a5baa31f3b
--- /dev/null
+++ b/doc/forum/Can_git_annex___40__mostly__41___be_an_Ubuntu_One___40__or_Dropbox...__41___substitute__63__/comment_1_a3fbae205c0312436f8861f432643811._comment
@@ -0,0 +1,10 @@
+[[!comment format=mdwn
+ username="http://joeyh.name/"
+ ip="108.236.230.124"
+ subject="comment 1"
+ date="2014-06-04T18:21:13Z"
+ content="""
+your box.com repository only stores encrypted file contents, it does not include the git repository. So you need to either set up XMPP, which will allow the clients to sync while they're turned on, or you need to put a git repository on a remote server, which the clients can use to sync anytime.
+
+Recent versions of git-annex allow encrypting the git repository. Use the \"remote server\" option in the webapp UI to set it up.
+"""]]
diff --git a/doc/forum/Can_git_annex___40__mostly__41___be_an_Ubuntu_One___40__or_Dropbox...__41___substitute__63__/comment_2_de49cf261c644a6e7f6ac881a48d4e6c._comment b/doc/forum/Can_git_annex___40__mostly__41___be_an_Ubuntu_One___40__or_Dropbox...__41___substitute__63__/comment_2_de49cf261c644a6e7f6ac881a48d4e6c._comment
new file mode 100644
index 0000000000..552a6db2b6
--- /dev/null
+++ b/doc/forum/Can_git_annex___40__mostly__41___be_an_Ubuntu_One___40__or_Dropbox...__41___substitute__63__/comment_2_de49cf261c644a6e7f6ac881a48d4e6c._comment
@@ -0,0 +1,10 @@
+[[!comment format=mdwn
+ username="https://www.google.com/accounts/o8/id?id=AItOawk0Sr9y7EcI5OhPk4GteZiHVxaoC0DmJ-E"
+ nickname="Dario"
+ subject="comment 2"
+ date="2014-06-05T09:10:40Z"
+ content="""
+I have set up XMPP already. I thought it was obvious, since otherwise you wouldn't even be able to share a remote with other clients (it would overwrite the contents)
+
+And the same problem happens with a remote server
+"""]]
diff --git a/doc/forum/Can_not_delete_Repository.mdwn b/doc/forum/Can_not_delete_Repository.mdwn
new file mode 100644
index 0000000000..549b47c3d3
--- /dev/null
+++ b/doc/forum/Can_not_delete_Repository.mdwn
@@ -0,0 +1,3 @@
+I have one repository that I deleted a while back. When I mark it as dead in command line it disappears from git annex info however when I run webapp it pops back webapp shows it as syncing disabled. When I try to delete it from the webapp it does not delete. I tried shutting down the daemon mark it as dead again then run git annex forget --drop-dead --force but running it makes the repo active again instead of deleting it.
+
+Repo in question was a S3 repo. I tried deleting it using both its name and uuid.
diff --git a/doc/forum/Can_not_delete_Repository/comment_1_b1a9420974e2e50c9c86a379ad62502c._comment b/doc/forum/Can_not_delete_Repository/comment_1_b1a9420974e2e50c9c86a379ad62502c._comment
new file mode 100644
index 0000000000..050127c24b
--- /dev/null
+++ b/doc/forum/Can_not_delete_Repository/comment_1_b1a9420974e2e50c9c86a379ad62502c._comment
@@ -0,0 +1,12 @@
+[[!comment format=mdwn
+ username="http://joeyh.name/"
+ ip="209.250.56.172"
+ subject="comment 1"
+ date="2014-02-21T17:36:03Z"
+ content="""
+Marking a repository as dead does not mean it is deleted. The way this is supposed to work, using the webapp is you tell it to start deleting the repository, which causes it to mark it as dead, and also, crucially, puts it into the \"unwanted\" group. This does not remove it from the list yet, but it causes all files located in the repository to be moved off it (assuming the repository can still be accessed).
+
+Then once the webapp detects that the repository is empty it will prompt you to continue the deletion process and truely remove it. It can take quite a while for the webapp to get around to this last step, since it's done during the expensive transfer scan, which it tries to avoid running unnecessarily.
+
+In any case, you can always edit .git/config and delete the remote, which should make the webapp no longer show it, as long as it's marked as dead.
+"""]]
diff --git a/doc/forum/Can_not_drop_unused_file.mdwn b/doc/forum/Can_not_drop_unused_file.mdwn
new file mode 100644
index 0000000000..8ec48eb14f
--- /dev/null
+++ b/doc/forum/Can_not_drop_unused_file.mdwn
@@ -0,0 +1,14 @@
+I have encrypted directory remote on a usb drive over time it accumulated some unused files. I would like to drop them running,
+
+    git annex --unused --from external
+
+returns a list of unused files when I try to drop them with,
+
+
+    git annex dropunused --force --from external 1-XX
+
+I get,
+
+    dropunused XX (from external...) failed
+
+I can not seem to get rid of these files.
diff --git a/doc/forum/Can_not_drop_unused_file/comment_1_cea83dfdf4cdb4f6efb3f2b33a39a51f._comment b/doc/forum/Can_not_drop_unused_file/comment_1_cea83dfdf4cdb4f6efb3f2b33a39a51f._comment
new file mode 100644
index 0000000000..4a6eacc510
--- /dev/null
+++ b/doc/forum/Can_not_drop_unused_file/comment_1_cea83dfdf4cdb4f6efb3f2b33a39a51f._comment
@@ -0,0 +1,16 @@
+[[!comment format=mdwn
+ username="http://joeyh.name/"
+ ip="209.250.56.68"
+ subject="comment 1"
+ date="2014-01-18T22:39:37Z"
+ content="""
+One reason it might fail (especially if it's not printing any useful message beyond \"(failed)\") is if it thought the file was present in the repositort, but something has happened to it.
+
+So, I suggest you try:
+
+git annex fsck --from external
+
+Followed by dropping again.
+
+If that doesn't help, I'd recommend stracing the dropping of one of the unused files, and see what's going wrong toward the end.
+"""]]
diff --git a/doc/forum/Can_not_drop_unused_file/comment_2_ed1543cff5e6e81ca18c43b716ca8199._comment b/doc/forum/Can_not_drop_unused_file/comment_2_ed1543cff5e6e81ca18c43b716ca8199._comment
new file mode 100644
index 0000000000..321e107615
--- /dev/null
+++ b/doc/forum/Can_not_drop_unused_file/comment_2_ed1543cff5e6e81ca18c43b716ca8199._comment
@@ -0,0 +1,42 @@
+[[!comment format=mdwn
+ username="https://me.yahoo.com/a/FHnTlSBo1eCGJRwueeKeB6.RCaPbGMPr5jxx8A--#ce0d8"
+ nickname="Hamza"
+ subject="comment 2"
+ date="2014-01-19T23:56:17Z"
+ content="""
+I've tried,
+
+    git annex fsck --from external
+
+and
+
+    git annex fsck --all --from external
+
+did not solve the problem. Running,
+
+    strace git annex dropunused 1 --from external --force
+
+tail of strace produces,
+
+    clone(child_stack=0, flags=CLONE_CHILD_CLEARTID|CLONE_CHILD_SETTID|SIGCHLD, child_tidp
+    tr=0x7f3a8d6d8a10) = 5746                                                            
+    rt_sigaction(SIGINT, {0x4e1fb0, [INT], SA_RESTORER|SA_RESTART, 0x7f3a8c8b60b0}, {SIG_D
+    FL, [], 0}, 8) = 0                                                                   
+    rt_sigaction(SIGHUP, {0x4e1fb0, [HUP], SA_RESTORER|SA_RESTART, 0x7f3a8c8b60b0}, {SIG_D
+    FL, [], 0}, 8) = 0                                                                   
+    rt_sigaction(SIGTERM, {0x4e1fb0, [TERM], SA_RESTORER|SA_RESTART, 0x7f3a8c8b60b0}, {SIG
+    _DFL, [], 0}, 8) = 0                                                                 
+    rt_sigaction(SIGQUIT, {0x4e1fb0, [QUIT], SA_RESTORER|SA_RESTART, 0x7f3a8c8b60b0}, {SIG
+    _DFL, [], 0}, 8) = 0                                                                 
+    rt_sigaction(SIGPIPE, {0x4e1fb0, [PIPE], SA_RESTORER|SA_RESTART, 0x7f3a8c8b60b0}, {SIG
+    _DFL, [], 0}, 8) = 0                                                                 
+    close(5)                                = 0
+    read(4, \"\", 1)                          = 0
+    close(4)                                = 0
+    wait4(5746, dropunused 1 (from external...) (gpg) failed
+    git-annex: dropunused: 1 failed
+    [{WIFEXITED(s) && WEXITSTATUS(s) == 1}], 0, NULL) = 5746
+    --- SIGCHLD (Child exited) @ 0 (0) ---
+    exit_group(1)                           = ?
+
+"""]]
diff --git a/doc/forum/Can_not_drop_unused_file/comment_3_0c9c9c0ed557af4845a67434c21bb4bc._comment b/doc/forum/Can_not_drop_unused_file/comment_3_0c9c9c0ed557af4845a67434c21bb4bc._comment
new file mode 100644
index 0000000000..8a1ecfdd24
--- /dev/null
+++ b/doc/forum/Can_not_drop_unused_file/comment_3_0c9c9c0ed557af4845a67434c21bb4bc._comment
@@ -0,0 +1,10 @@
+[[!comment format=mdwn
+ username="http://joeyh.name/"
+ ip="209.250.56.68"
+ subject="comment 3"
+ date="2014-01-20T16:33:27Z"
+ content="""
+I see you're using encryption. That could have something to do with the problem. Which type of encryption was used for this special remote? encryption=shared or one of the other options?
+
+Look through the whole strace output for attempts to access the directory special remote and show those. Or put up the full strace somewhere.
+"""]]
diff --git a/doc/forum/Can_the_assistant_sync_files_if_committed_manually___40__autocommit__61__false__41____63__.mdwn b/doc/forum/Can_the_assistant_sync_files_if_committed_manually___40__autocommit__61__false__41____63__.mdwn
new file mode 100644
index 0000000000..2f1491afd6
--- /dev/null
+++ b/doc/forum/Can_the_assistant_sync_files_if_committed_manually___40__autocommit__61__false__41____63__.mdwn
@@ -0,0 +1,6 @@
+I have some repositories where I want to manually control what files are added (autocommit=false), but would like to have the assistant/webapp automatically sync commits and transfer files.
+However it seems that the assistant is not notified if I manually do a commit.
+
+The only solution right now seems to frequently restart the assistant since the startup scan will transfer files if required or run git annex get/copy manually.
+
+Is there another possibility?
diff --git a/doc/forum/Can_the_assistant_sync_files_if_committed_manually___40__autocommit__61__false__41____63__/comment_1_bc838634442883e541de1ceab520d71e._comment b/doc/forum/Can_the_assistant_sync_files_if_committed_manually___40__autocommit__61__false__41____63__/comment_1_bc838634442883e541de1ceab520d71e._comment
new file mode 100644
index 0000000000..fb0fbf40b1
--- /dev/null
+++ b/doc/forum/Can_the_assistant_sync_files_if_committed_manually___40__autocommit__61__false__41____63__/comment_1_bc838634442883e541de1ceab520d71e._comment
@@ -0,0 +1,13 @@
+[[!comment format=mdwn
+ username="joey"
+ subject="""comment 1"""
+ date="2015-07-15T16:01:30Z"
+ content="""
+Right, the assistant doesn't have a way to get notified of your manual
+commits, so it doesn't promptly push them.
+
+While the assistant could be modified to watch git branch files for changes,
+I think that for most people wanting to make manual commits, running `git
+annex sync --content` is a better approach. It does all the same syncing
+that the assistant would.
+"""]]
diff --git a/doc/forum/Can_we_have_remotes_that_aren__39__t_tracked__63___.mdwn b/doc/forum/Can_we_have_remotes_that_aren__39__t_tracked__63___.mdwn
new file mode 100644
index 0000000000..684893e84e
--- /dev/null
+++ b/doc/forum/Can_we_have_remotes_that_aren__39__t_tracked__63___.mdwn
@@ -0,0 +1,13 @@
+I'm wondering if it is possible to have remotes that don't have the *content* of git-annex tracked.
+
+# My use case:
+
+I have a number of projects that I am working on at any one time.  They all are tracking independently by `git` and more recently I am using `git annex` to manage the large files.
+
+However because I have so many projects I work on one (called `AAA`), move to another, delete `AAA` to save disk space, ...time passes... return to `AAA`.
+
+Now, prior to `git-annex` I could just clone `AAA` from my central repository folder do work, commit, push, repeat and then delete and there is no indication that I had one, or many copies of `AAA` floating around.  Now with `git-annex` there is some trail of me cloning, running `git annex get`, etc.
+
+Is there some way to set a remote as `untracked`?  By that I mean it is classed as `untrusted` - so I can move files around, add them, copy to trusted remotes and delete the whole repository without worrying about losing data - but it also doesn't push any of the git-annex tracking info of where a copy of a file actually is.  I don't want to know if any or all of my other `untracked` repositories have a copy of a file or not. 
+
+I don't want my `git annex whereis` polluted with many references to repositories that just don't exist any more.  I guess I could set them to dead but that still keeps all of the tracking info around in all the repos, which seems unnecessary...
diff --git a/doc/forum/Can_we_have_remotes_that_aren__39__t_tracked__63___/comment_1_35e5a963b9e58ed7773dfcb884f8ecbd._comment b/doc/forum/Can_we_have_remotes_that_aren__39__t_tracked__63___/comment_1_35e5a963b9e58ed7773dfcb884f8ecbd._comment
new file mode 100644
index 0000000000..955c256ca9
--- /dev/null
+++ b/doc/forum/Can_we_have_remotes_that_aren__39__t_tracked__63___/comment_1_35e5a963b9e58ed7773dfcb884f8ecbd._comment
@@ -0,0 +1,10 @@
+[[!comment format=mdwn
+ username="http://joeyh.name/"
+ ip="2001:4978:f:21a::2"
+ subject="comment 1"
+ date="2013-08-07T16:27:21Z"
+ content="""
+This seems like a reasonable request, so I've opened [[todo/untracked_remotes]]
+
+I will note that if you know the annex.uuid of the previous AAA repository, you can make the new one use that same uuid, just `git config annex.uuid $UUID`. (`git annex fsck --fast` would be a good idea after doing that.)
+"""]]
diff --git a/doc/forum/Can_we_have_remotes_that_aren__39__t_tracked__63___/comment_2_7cde9785886c8450e1475f0b54481ae3._comment b/doc/forum/Can_we_have_remotes_that_aren__39__t_tracked__63___/comment_2_7cde9785886c8450e1475f0b54481ae3._comment
new file mode 100644
index 0000000000..92713db013
--- /dev/null
+++ b/doc/forum/Can_we_have_remotes_that_aren__39__t_tracked__63___/comment_2_7cde9785886c8450e1475f0b54481ae3._comment
@@ -0,0 +1,10 @@
+[[!comment format=mdwn
+ username="http://joeyh.name/"
+ ip="209.250.56.227"
+ subject="comment 2"
+ date="2014-01-02T17:26:35Z"
+ content="""
+I have now implemented a way to mark a remote as readonly. This prevents git-annex from pushing anything to the remote, or modifying it in other ways.
+
+It seems that in your use case, you might want to avoid git-annex sync pushing the git-annex branch, but still push a branch like master. So far, it's up to you to decide which branches to manually push; readonly disables all pushing.
+"""]]
diff --git a/doc/forum/Cannot_find_git-annex_in_server.mdwn b/doc/forum/Cannot_find_git-annex_in_server.mdwn
new file mode 100644
index 0000000000..6208c288e5
--- /dev/null
+++ b/doc/forum/Cannot_find_git-annex_in_server.mdwn
@@ -0,0 +1,10 @@
+My server is running the precompiled tarball https://downloads.kitenet.net/git-annex/linux/current/
+
+git-annex version: 4.20130531-g5df09b5
+build flags: Assistant Webapp Pairing Testsuite S3 WebDAV Inotify DBus XMPP
+
+The tarball is untared in "/opt/git-annex.linux" and this location is added to the users path in ".profile", who can launch the webapp as usual and so on.
+
+But when a git-annex client from another computer tries to stablish a remote server repository with the server via ssh, it will complain "/usr/bin/git-annex", "runshell" and so on are missing. And if the binaries in "/opt/git-annex.linux" are symlinked in "/usr/bin" they will start to miss the other bin and libs in the "/opt/git-annex.linux" source tree.
+
+As you can understand, I can't put the whole "/opt/git-annex.linux" folder tree in "/usr/bin". Is there any solution to make the precompiled tarball work properly as a git-annex server?
diff --git a/doc/forum/Cannot_find_git-annex_in_server/comment_1_bf7e98e6130698ad0dc92e3a6a63ade3._comment b/doc/forum/Cannot_find_git-annex_in_server/comment_1_bf7e98e6130698ad0dc92e3a6a63ade3._comment
new file mode 100644
index 0000000000..b5685c901c
--- /dev/null
+++ b/doc/forum/Cannot_find_git-annex_in_server/comment_1_bf7e98e6130698ad0dc92e3a6a63ade3._comment
@@ -0,0 +1,15 @@
+[[!comment format=mdwn
+ username="http://joeyh.name/"
+ nickname="joey"
+ subject="comment 1"
+ date="2013-06-20T15:23:32Z"
+ content="""
+The standalone tarball does not need to be installed in /usr/bin or any other particular location. I *never* hardcode /usr/bin in anything.
+
+It seems likely to me that you have not correctly added /opt/git-annex.linux to PATH. You mention `.profile` -- but this is only used
+by bash when starting a login shell. So any non-login shells won't have it in path. You need to put something in .bashrc for that.
+
+Also, to use git-annex on a server, there is no reason at all to install the latest and greatest version. Any version 3 or greater build of git-annex will work fine on a server with newer git-annex versions on clients. So `apt-get install git-annex` is a much easier and nicer way to install it on a server. Most linux distributions have a package of git-annex and an easy way to install it.
+
+(It would also help if you pasted actual error messages, rather than a summary of an error message.)
+"""]]
diff --git a/doc/forum/Cannot_find_git-annex_in_server/comment_2_168dda4aed09f90a510bc453e8a7cda7._comment b/doc/forum/Cannot_find_git-annex_in_server/comment_2_168dda4aed09f90a510bc453e8a7cda7._comment
new file mode 100644
index 0000000000..48c0a65aff
--- /dev/null
+++ b/doc/forum/Cannot_find_git-annex_in_server/comment_2_168dda4aed09f90a510bc453e8a7cda7._comment
@@ -0,0 +1,10 @@
+[[!comment format=mdwn
+ username="https://www.google.com/accounts/o8/id?id=AItOawmCEE7m7pm6lqvYmgRSESRP9xqmmMm9ox8"
+ nickname="Daniel"
+ subject="comment 2"
+ date="2013-06-20T20:14:25Z"
+ content="""
+I'd rather use the package from the repositories. But this server is running an openSUSE install and the package manager could not resolve the Haskell dependencies of the unofficial git-annex rpm package.
+
+Indeed, performing the PATH extension in \".bashrc\", not in \".profile\", solved the issue. I feel like a newbie. Thanks for the express answer!
+"""]]
diff --git a/doc/forum/Cannot_launch_webapp_on_ubuntu_12.04_using_ppa.mdwn b/doc/forum/Cannot_launch_webapp_on_ubuntu_12.04_using_ppa.mdwn
new file mode 100644
index 0000000000..351c6f92d6
--- /dev/null
+++ b/doc/forum/Cannot_launch_webapp_on_ubuntu_12.04_using_ppa.mdwn
@@ -0,0 +1,6 @@
+Hi, I have installed latest version from https://launchpad.net/~rubiojr/+archive/git-annex, that is git-annex version: 3.20121017-ubuntu1ppa1~precise
+When running git annex webapp I get 
+
+git-annex: unknown command webapp
+
+I only installed git-annex. Are there more packages to be installed to make it work?
diff --git a/doc/forum/Cannot_launch_webapp_on_ubuntu_12.04_using_ppa/comment_1_9345551f5772c3a6f1490b00e1edbf69._comment b/doc/forum/Cannot_launch_webapp_on_ubuntu_12.04_using_ppa/comment_1_9345551f5772c3a6f1490b00e1edbf69._comment
new file mode 100644
index 0000000000..aae67b95e3
--- /dev/null
+++ b/doc/forum/Cannot_launch_webapp_on_ubuntu_12.04_using_ppa/comment_1_9345551f5772c3a6f1490b00e1edbf69._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="https://www.google.com/accounts/o8/id?id=AItOawniayrgSdVLUc3c6bf93VbO-_HT4hzxmyo"
+ nickname="Tobias"
+ subject="comment 1"
+ date="2013-04-23T08:39:50Z"
+ content="""
+This version does not have the webapp. Use a newer one (maybe you need the standalone build)
+"""]]
diff --git a/doc/forum/Cannot_launch_webapp_on_ubuntu_12.04_using_ppa/comment_2_0b688a442b6a911a0353e73097a24cb6._comment b/doc/forum/Cannot_launch_webapp_on_ubuntu_12.04_using_ppa/comment_2_0b688a442b6a911a0353e73097a24cb6._comment
new file mode 100644
index 0000000000..b83eb40ec8
--- /dev/null
+++ b/doc/forum/Cannot_launch_webapp_on_ubuntu_12.04_using_ppa/comment_2_0b688a442b6a911a0353e73097a24cb6._comment
@@ -0,0 +1,12 @@
+[[!comment format=mdwn
+ username="tomas"
+ ip="64.103.25.104"
+ subject="comment 2"
+ date="2013-04-23T11:50:21Z"
+ content="""
+According to documentation:
+http://git-annex.branchable.com/assistant/
+The git-annex assistant comes as part of git-annex, starting with version 3.20120924.
+
+So it should be there in 3.20121017, no?
+"""]]
diff --git a/doc/forum/Cannot_launch_webapp_on_ubuntu_12.04_using_ppa/comment_3_7e246caa00005560bb489c927c663046._comment b/doc/forum/Cannot_launch_webapp_on_ubuntu_12.04_using_ppa/comment_3_7e246caa00005560bb489c927c663046._comment
new file mode 100644
index 0000000000..e2463ba23c
--- /dev/null
+++ b/doc/forum/Cannot_launch_webapp_on_ubuntu_12.04_using_ppa/comment_3_7e246caa00005560bb489c927c663046._comment
@@ -0,0 +1,12 @@
+[[!comment format=mdwn
+ username="http://joeyh.name/"
+ nickname="joey"
+ subject="comment 3"
+ date="2013-04-23T16:05:31Z"
+ content="""
+git-annex can be built without the webapp, and if some of the necessary dependencies for the webapp are not available, it will build without it by default. More recent versions than that one let you run `git annex version` and see the build flags, which can all be disabled due to missing dependencies. For example:
+
+    build flags: Assistant Webapp Pairing Testsuite S3 WebDAV Inotify DBus XMPP
+
+Sergio does seem to be trying to backport all the dependencies of the webapp, but it looks to me like some of them, particularly haskell-xml-hamlet, have not successfully built yet.
+"""]]
diff --git a/doc/forum/Cannot_launch_webapp_on_ubuntu_12.04_using_ppa/comment_4_1d8025aabe8bc72711a77f691f67da5f._comment b/doc/forum/Cannot_launch_webapp_on_ubuntu_12.04_using_ppa/comment_4_1d8025aabe8bc72711a77f691f67da5f._comment
new file mode 100644
index 0000000000..4c050f0520
--- /dev/null
+++ b/doc/forum/Cannot_launch_webapp_on_ubuntu_12.04_using_ppa/comment_4_1d8025aabe8bc72711a77f691f67da5f._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="fmarier"
+ ip="121.98.93.240"
+ subject="My PPA (Ubuntu Precise) has git-annex 4.20130417"
+ date="2013-04-27T23:31:54Z"
+ content="""
+It has the 90+ Haskell packages that are required to build git annex with every backend: https://launchpad.net/~fmarier/+archive/ppa
+"""]]
diff --git a/doc/forum/Cannot_launch_webapp_on_ubuntu_12.04_using_ppa/comment_5_7c2f95da65190016192424e7c622122f._comment b/doc/forum/Cannot_launch_webapp_on_ubuntu_12.04_using_ppa/comment_5_7c2f95da65190016192424e7c622122f._comment
new file mode 100644
index 0000000000..6821da22d8
--- /dev/null
+++ b/doc/forum/Cannot_launch_webapp_on_ubuntu_12.04_using_ppa/comment_5_7c2f95da65190016192424e7c622122f._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="https://www.google.com/accounts/o8/id?id=AItOawk3HGoDpnOPob5jOjvIootmkve1-nCpRiI"
+ nickname="Kalle"
+ subject="Debian Sid PowerPC is also without webapp"
+ date="2013-05-07T09:44:31Z"
+ content="""
+Took me a couple of minutes to figure out why git-annex on my old Apple G4 wouldn't recognise the webapp command. Comparing the version command output on AMD64 and PowerPC made it obvious.
+"""]]
diff --git a/doc/forum/Cannot_launch_webapp_on_ubuntu_12.04_using_ppa/comment_6_9b8465cefe609e7a696e7573b8892e38._comment b/doc/forum/Cannot_launch_webapp_on_ubuntu_12.04_using_ppa/comment_6_9b8465cefe609e7a696e7573b8892e38._comment
new file mode 100644
index 0000000000..b2a6e55bb9
--- /dev/null
+++ b/doc/forum/Cannot_launch_webapp_on_ubuntu_12.04_using_ppa/comment_6_9b8465cefe609e7a696e7573b8892e38._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="tomas"
+ ip="188.167.111.235"
+ subject="thanks for the ppa"
+ date="2013-05-08T10:04:25Z"
+ content="""
+Thanks François for your work
+"""]]
diff --git a/doc/forum/Cannot_launch_webapp_on_ubuntu_12.04_using_ppa/comment_7_af6472762a598a454ba52ac0caa059aa._comment b/doc/forum/Cannot_launch_webapp_on_ubuntu_12.04_using_ppa/comment_7_af6472762a598a454ba52ac0caa059aa._comment
new file mode 100644
index 0000000000..d205e653de
--- /dev/null
+++ b/doc/forum/Cannot_launch_webapp_on_ubuntu_12.04_using_ppa/comment_7_af6472762a598a454ba52ac0caa059aa._comment
@@ -0,0 +1,10 @@
+[[!comment format=mdwn
+ username="https://openid.fmarier.org/"
+ nickname="fmarier"
+ subject="New location for my PPA"
+ date="2013-06-15T07:41:03Z"
+ content="""
+My Ubuntu Precise PPA has moved here: https://launchpad.net/~fmarier/+archive/git-annex
+
+It contains version 20130601 of git-annex.
+"""]]
diff --git a/doc/forum/Cant_see_git-annex-shell_via_SSH_in_OSX.mdwn b/doc/forum/Cant_see_git-annex-shell_via_SSH_in_OSX.mdwn
new file mode 100644
index 0000000000..76b1d5a8f9
--- /dev/null
+++ b/doc/forum/Cant_see_git-annex-shell_via_SSH_in_OSX.mdwn
@@ -0,0 +1,119 @@
+### Sync Problems using SSH remote in OSX
+
+- Im trying to work out SSH remotes by trying to sync up repos on my home network, following the walkthrough. 
+- I have two machines (mini and mbp ) running OSX Mavericks, with RLogin enabled for all users to enable ssh.
+- I can SSH into the remote machine and see *git-annex-shell*, which seems to have ok permissions
+
+```
+
+    johns-mbp:annex johnmccallum$ ssh john@johns-mini-5.home 
+    
+    Last login: Sun Apr 12 07:31:07 2015 from johns-mbp.home
+
+    johns-mini-5:~ john$ which git-annex-shell
+
+    /usr/local/bin/git-annex-shell
+
+    johns-mini-5:~ john$ ls -l /usr/local/bin/git-annex-shell
+
+    -rwxr-xr-x@ 1 john  admin  668 12 Apr 07:03 /usr/local/bin/git-annex-shell
+
+```
+
+- Previously on mini I created and populated a repo
+
+``` 
+
+    494  mkdir annex
+
+    495  cd annex
+
+    496  git init
+
+    497  git annex init
+
+    498  cp ~/Pictures/*.png .
+
+    499  git annex add .
+
+    500  git commit -a -m 'added png'
+
+```
+
+- I can git clone this repo to MBP by SSH
+
+
+```
+	johns-mbp:~ johnmccallum$ git clone ssh://john@johns-mini-5.home/Users/john/annex ~/annex
+
+	Cloning into '/Users/johnmccallum/annex'...
+
+	remote: Counting objects: 24, done.
+
+	remote: Compressing objects: 100% (19/19), done.
+
+	remote: Total 24 (delta 3), reused 0 (delta 0)
+
+	Receiving objects: 100% (24/24), done.
+
+	Resolving deltas: 100% (3/3), done.
+
+	Checking connectivity... done
+
+	johns-mbp:~ johnmccallum$ cd annex
+
+	johns-mbp:annex johnmccallum$ git annex init 'MBP'
+
+	init MBP (merging origin/git-annex into git-annex...)
+
+	(recording state in git...)
+
+	ok
+
+	(recording state in git...)
+
+	johns-mbp:annex johnmccallum$ ls -l
+
+	total 16
+
+	lrwxr-xr-x  1 johnmccallum  staff  196 12 Apr 08:20 CoGe-Snapshot at 2013-03-22 - 11-27-20.png -> .git/annex/objects/gf/Xp/SHA256E-s367697--	fce3f47f218805cd9855ec3fd4203b52e83587148b34c8e706df512783eb7557.png/SHA256E-s367697--fce3f47f218805cd9855ec3fd4203b52e83587148b34c8e706df512783eb7557.png
+
+	lrwxr-xr-x  1 johnmccallum  staff  196 12 Apr 08:20 delicious.png -> .git/annex/objects/ZJ/vX/SHA256E-s112714--057d0faa464f8d588c053dae460838d68ea7803d7eaf7330798679e63f92cecb.png/SHA256E-s112714--057d0faa464f8d588c053dae460838d68ea7803d7eaf7330798679e63f92cecb.png
+
+
+```
+
+ **HOWEVER**   _git annex get_  fails as follows:
+
+```
+
+	johns-mbp:annex johnmccallum$ git annex get delicious.png 
+
+	get delicious.png bash: git-annex-shell: command not found
+
+ 	 Remote origin does not have git-annex installed; setting annex-ignore
+
+ 	 This could be a problem with the git-annex installation on the remote. Please make sure that git-annex-shell is available in PATH when you ssh into the remote. Once you have fixed the git-annex installation, run: git config remote.origin.annex-ignore false
+	(not available) 
+ 	 Try making some of these repositories available:
+  		129620b2-91b1-4541-b7b1-9e5a9d31d5d3 -- john@johns-mini-5.home:~/annex
+	failed
+	git-annex: get: 1 failed
+
+```
+
+This is not the case on the remote host when I SSH in as the same user
+
+```
+
+    johns-mini-5:~ john$ which git-annex-shell
+
+
+    /usr/local/bin/git-annex-shell
+
+```
+
+
+ The only thread on this seems to be https://git-annex.branchable.com/forum/not_finding_git-annex-shell_on_remote/ and Im at a loss to understand it.  
+
+Any suggestions would be welcome
diff --git a/doc/forum/Cant_see_git-annex-shell_via_SSH_in_OSX/comment_1_1fafdc4ed4a0f601918361dca688aa6c._comment b/doc/forum/Cant_see_git-annex-shell_via_SSH_in_OSX/comment_1_1fafdc4ed4a0f601918361dca688aa6c._comment
new file mode 100644
index 0000000000..edbf2edfc4
--- /dev/null
+++ b/doc/forum/Cant_see_git-annex-shell_via_SSH_in_OSX/comment_1_1fafdc4ed4a0f601918361dca688aa6c._comment
@@ -0,0 +1,14 @@
+[[!comment format=mdwn
+ username="joey"
+ subject="""comment 1"""
+ date="2015-04-18T20:05:42Z"
+ content="""
+This is a common gotcha; bash sources something like .bashrc
+or whatever that puts /usr/local/bin/ in your path when it's a login shell.
+However, when git-annex is sshing in, there is no login shell,
+and bash does not source any dotfiles, and so git-annex-shell in not in
+PATH.
+
+The solution is probably to move or symlink it to some other directory
+that is in PATH always.
+"""]]
diff --git a/doc/forum/Case-insensitive_search_with_find_list_whereis__63__.mdwn b/doc/forum/Case-insensitive_search_with_find_list_whereis__63__.mdwn
new file mode 100644
index 0000000000..bb14a81794
--- /dev/null
+++ b/doc/forum/Case-insensitive_search_with_find_list_whereis__63__.mdwn
@@ -0,0 +1 @@
+Are case-insensitive searchs possible with find, list, and whereis?
diff --git a/doc/forum/Case-insensitive_search_with_find_list_whereis__63__/comment_1_b893fd9a250d25ebc62913131cb0a479._comment b/doc/forum/Case-insensitive_search_with_find_list_whereis__63__/comment_1_b893fd9a250d25ebc62913131cb0a479._comment
new file mode 100644
index 0000000000..5722906690
--- /dev/null
+++ b/doc/forum/Case-insensitive_search_with_find_list_whereis__63__/comment_1_b893fd9a250d25ebc62913131cb0a479._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="joey"
+ subject="""comment 1"""
+ date="2015-03-26T16:09:02Z"
+ content="""
+`git annex find | grep -i whatever`
+
+"""]]
diff --git a/doc/forum/Case-insensitive_search_with_find_list_whereis__63__/comment_2_e19abfe33b0f071ffc059e9976fc6d91._comment b/doc/forum/Case-insensitive_search_with_find_list_whereis__63__/comment_2_e19abfe33b0f071ffc059e9976fc6d91._comment
new file mode 100644
index 0000000000..26d9efaf35
--- /dev/null
+++ b/doc/forum/Case-insensitive_search_with_find_list_whereis__63__/comment_2_e19abfe33b0f071ffc059e9976fc6d91._comment
@@ -0,0 +1,7 @@
+[[!comment format=mdwn
+ username="ghen1"
+ subject="comment 2"
+ date="2015-03-29T23:14:00Z"
+ content="""
+I feel stupid for not thinking of that. Thank you.
+"""]]
diff --git a/doc/forum/Central_git_annex_server_that_always_keeps_one_copy.mdwn b/doc/forum/Central_git_annex_server_that_always_keeps_one_copy.mdwn
new file mode 100644
index 0000000000..166a22deef
--- /dev/null
+++ b/doc/forum/Central_git_annex_server_that_always_keeps_one_copy.mdwn
@@ -0,0 +1 @@
+Is there a way to configure a central git repository that keeps track of large files with git annex so that multiple users can clone the repository but no repository clone can drop files from the server. Essentially, I'm looking for a way to have one repository that is always populated with at least one copy of each file. Other users shouldn't be able to tell that repository to drop any files (but would be able to add files it). The term "user" in that last sentence really refers to other clones...
diff --git a/doc/forum/Central_git_annex_server_that_always_keeps_one_copy/comment_1_e786c8df6e48d88cf15b555af1b8639a._comment b/doc/forum/Central_git_annex_server_that_always_keeps_one_copy/comment_1_e786c8df6e48d88cf15b555af1b8639a._comment
new file mode 100644
index 0000000000..19339a3aa2
--- /dev/null
+++ b/doc/forum/Central_git_annex_server_that_always_keeps_one_copy/comment_1_e786c8df6e48d88cf15b555af1b8639a._comment
@@ -0,0 +1,10 @@
+[[!comment format=mdwn
+ username="https://www.google.com/accounts/o8/id?id=AItOawn-TDneVW-8kwb1fyTRAJfH3l1xs2VSEmk"
+ nickname="James"
+ subject="comment 1"
+ date="2014-07-30T20:37:27Z"
+ content="""
+It might not suit all your needs but you could try using gitolite and set permissions on the git-annex branch of your repository
+http://gitolite.com/gitolite/conf.html#write-types
+
+"""]]
diff --git a/doc/forum/Central_git_annex_server_that_always_keeps_one_copy/comment_2_b0a091e11c18c8640888b0aedfbff5c5._comment b/doc/forum/Central_git_annex_server_that_always_keeps_one_copy/comment_2_b0a091e11c18c8640888b0aedfbff5c5._comment
new file mode 100644
index 0000000000..35778ad9ab
--- /dev/null
+++ b/doc/forum/Central_git_annex_server_that_always_keeps_one_copy/comment_2_b0a091e11c18c8640888b0aedfbff5c5._comment
@@ -0,0 +1,19 @@
+[[!comment format=mdwn
+ username="joey"
+ subject="""comment 2"""
+ date="2014-10-28T20:28:26Z"
+ content="""
+If you configure the central repository to be in the "full backup" group,
+then the assistant and `git annex sync --content` etc will send the
+contents of all files to it, and never remove the contents of files from
+it.
+
+This doesn't prevent someone manually running `git annex drop --from
+$centralrepo --force`, or similar. But then, nothing prevents users from
+using git to push annexed files to the central repo, and never sending the
+contents of the files along to it.
+
+It would be possible to make `git-annex-shell` have a config setting that
+would prevent ever dropping keys. But I'd like to talk about this use case
+more, to make sure that really makes sense.
+"""]]
diff --git a/doc/forum/Central_server_with_automatically_merged_working_tree.mdwn b/doc/forum/Central_server_with_automatically_merged_working_tree.mdwn
new file mode 100644
index 0000000000..8b2bb87d6c
--- /dev/null
+++ b/doc/forum/Central_server_with_automatically_merged_working_tree.mdwn
@@ -0,0 +1,30 @@
+I would appreciate some feedback on whether the following scheme is feasible or will prove unworkable. I have a few specific questions at the bottom.
+
+# Components
+* An archive of documents to be shared amongst computers
+* A server that stores the archive and retains all of its contents
+* A set of client machines that store only the files that they need and drop the rest
+
+# Desired behavior
+* Clients will run the assistant to handle sync'ing
+* On the server, the archive should be accessible in the working tree so that other users can read the files (they do not need write access).
+* It should be possible to modify the archive on the server while logged into the server
+* Clients can push and pull data. The server does not push or pull data.
+
+# Basic setup
+## Set up the server and one client
+* Create git annex repos on the client and server
+* Add the server as a remote in the client repo
+
+## Keep the server's working directory up to date
+* Add `git merge synced/master master` to the `annex-content` hook described [here](https://git-annex.branchable.com/git-annex-shell/).
+* Add `git merge synced/master master` to the `post-receive` hook.
+
+## Keep server working tree visible to a different group
+* Set `git config core.sharedrepository group` for the repository
+* Add `chgrp -R shared "$(git rev-parse --show-toplevel)"` to the `annex-content`, `post-receive`, and `post-merge` hooks, where `shared` is the name of the group that you want to be able to access the server files.
+
+# Questions
+* I had to add the `post-receive` hook because updates from the assistant were not trigger the `annex-content` hook. Should they trigger it?
+* Are there downsides to merging `synced/master` like this?
+* If I want to edit files on the server, is it safe to edit them in the repo with this set up? Or should I create a second client repo on the server, check out the necessary files there, and then push them to the server like I would from any other client?
diff --git a/doc/forum/Central_server_with_automatically_merged_working_tree/comment_1_bc850e25e48598a09353850021c6368c._comment b/doc/forum/Central_server_with_automatically_merged_working_tree/comment_1_bc850e25e48598a09353850021c6368c._comment
new file mode 100644
index 0000000000..2680a14115
--- /dev/null
+++ b/doc/forum/Central_server_with_automatically_merged_working_tree/comment_1_bc850e25e48598a09353850021c6368c._comment
@@ -0,0 +1,17 @@
+[[!comment format=mdwn
+ username="joey"
+ subject="""comment 1"""
+ date="2015-09-21T16:42:29Z"
+ content="""
+A way to do this pretty cleanly is described here:
+
+
+The annex-content hook is not meant to be run when a regular git push
+is made to the repository. That would be unnecessarily redundant with
+existing git hooks.
+
+I'd expect it to be pretty safe to edit files in such a repository,
+but you have to keep in mind that if changes are pushed into it at the same
+time as an edit is occuring, you can get into various situations with the
+hooks etc.
+"""]]
diff --git a/doc/forum/Centralized_repository_with_webapp.mdwn b/doc/forum/Centralized_repository_with_webapp.mdwn
new file mode 100644
index 0000000000..6a9fb285de
--- /dev/null
+++ b/doc/forum/Centralized_repository_with_webapp.mdwn
@@ -0,0 +1,13 @@
+Hi,
+
+I'm kind of new to git-annex, I've been following it for a while and tried small task, but never used it in a real situation.
+I'm now trying to sync various computers through a central server and I'm having some problems, so I think I might be doing something wrong.
+
+I have a remote server that I want to use as central server. I use the webapp to configure client 1 to use that server as remote server and using git (so I assume it stores the files and the tree). I then create the client 2 in another computer and doing the exact same steps.
+Initially everything seams to work, but after a few modifications in the clients weird things start to happen, files only in one client, some files don't get updated, etc.
+
+I guess I'm doing something wrong (maybe I need to clone the repo in client 2 instead of creating a new one?) but I can't figure out how to solve it.
+
+Any tip that could help me?
+
+Thanks!
diff --git a/doc/forum/Centralized_repository_with_webapp/comment_1_dcb9b07fd154f4d4fdef4809cc37ce77._comment b/doc/forum/Centralized_repository_with_webapp/comment_1_dcb9b07fd154f4d4fdef4809cc37ce77._comment
new file mode 100644
index 0000000000..78866072e3
--- /dev/null
+++ b/doc/forum/Centralized_repository_with_webapp/comment_1_dcb9b07fd154f4d4fdef4809cc37ce77._comment
@@ -0,0 +1,18 @@
+[[!comment format=mdwn
+ username="http://joeyh.name/"
+ nickname="joey"
+ subject="comment 1"
+ date="2013-03-11T15:13:32Z"
+ content="""
+Your setup is fine as far as it goes. The problem is that a client has no way to know when another client has pushed a change to the centralized repository. It might take quite a while before it decides on its own to go pull changes from there (most often because *it* has now changed a file, and realizes it needs to sync), and so it won't see files the other client has added right away.
+
+The solution to this is to go to Configuration -> Jabber account, and set both clients up using the same Jabber account. (Or, if these two machines belong to different people, you can pick \"pair with a friend\" to link one with the other over Jabber.)
+
+Now when one client pushes to the centralized repository, it will immediately send the other a message letting it know something has changed.
+
+The webapp actually pops up an alert when you add that centralized ssh repository, to nudge you to do this:
+
+[[/assistant/xmppnudge.png]]
+
+I'm trying to find ways to make the need to do this more clear...
+"""]]
diff --git a/doc/forum/Centralized_repository_with_webapp/comment_2_08c84f2703f89dc12982eba9dd2a06d1._comment b/doc/forum/Centralized_repository_with_webapp/comment_2_08c84f2703f89dc12982eba9dd2a06d1._comment
new file mode 100644
index 0000000000..a6686f6a3d
--- /dev/null
+++ b/doc/forum/Centralized_repository_with_webapp/comment_2_08c84f2703f89dc12982eba9dd2a06d1._comment
@@ -0,0 +1,11 @@
+[[!comment format=mdwn
+ username="https://www.google.com/accounts/o8/id?id=AItOawk9nck8WX8-ADF3Fdh5vFo4Qrw1I_bJcR8"
+ nickname="Jon Ander"
+ subject="comment 2"
+ date="2013-03-13T23:52:32Z"
+ content="""
+Hi Joey,
+
+My problem isn't that the repositories don't sync immediately, they are never up at the same time and I understand that the assistant is not constantly checking for changes, so that's not an issue.
+The problem is that with this setup I'm experiencing data loss (files being completely removed from both repos), it has happened a few times, but I cant seam to reproduce it reliably. I'll keep investigating and get back to you if I have a proper bug.
+"""]]
diff --git a/doc/forum/Change_or_add_S3_credentials.mdwn b/doc/forum/Change_or_add_S3_credentials.mdwn
new file mode 100644
index 0000000000..736f9e4e1b
--- /dev/null
+++ b/doc/forum/Change_or_add_S3_credentials.mdwn
@@ -0,0 +1,6 @@
+How do I change or add S3 credentials, when a S3 special remote is already initialised/enabled?
+
+I have a repository with a `public=yes` S3 remote, such that people can read the data without credentials.
+But then when they need to upload files, how do they add their credentials?
+
+Setting the `AWS_*` environment variables when running `git annex copy --to=s3` works, but then the credentials are not stored.
diff --git a/doc/forum/Change_or_add_S3_credentials/comment_1_864932114dcc7aaf1d88edb0673f1d86._comment b/doc/forum/Change_or_add_S3_credentials/comment_1_864932114dcc7aaf1d88edb0673f1d86._comment
new file mode 100644
index 0000000000..bd9f92c8ad
--- /dev/null
+++ b/doc/forum/Change_or_add_S3_credentials/comment_1_864932114dcc7aaf1d88edb0673f1d86._comment
@@ -0,0 +1,10 @@
+[[!comment format=mdwn
+ username="Mara"
+ avatar="http://cdn.libravatar.org/avatar/9b8abe5a5b0a41b88fc9970a88c2e317"
+ subject="Also with embedcreds=yes"
+ date="2018-08-13T12:28:09Z"
+ content="""
+Also with `public=no` but `embedcreds=yes`.
+
+It can be useful to embed read-only credentials, but allow users to easily add/store their own credentials (locally) with write-access.
+"""]]
diff --git a/doc/forum/Change_remote_server_address.mdwn b/doc/forum/Change_remote_server_address.mdwn
new file mode 100644
index 0000000000..722b4ada4e
--- /dev/null
+++ b/doc/forum/Change_remote_server_address.mdwn
@@ -0,0 +1,6 @@
+Hi again,
+
+I have a SSH remote server that I registered to my git-annex repository via git-annex assistant. When I go to edit the settings for the repository from within git-annex assistant I noticed I can't edit the server address. If the server IP changes, how should I go about letting git-annex know of this? Can I just (1) shutdown git-annex assistant, (2) edit the 'url' line of the remote entry inside the repository's ``.git/config`` file, and then (3) start up git-annex assistant again? Is this a safe method for doing this?
+
+Regards,
+Blake
diff --git a/doc/forum/Change_remote_server_address/comment_1_401c3d2530ac7ba41dd3857ab4737ed5._comment b/doc/forum/Change_remote_server_address/comment_1_401c3d2530ac7ba41dd3857ab4737ed5._comment
new file mode 100644
index 0000000000..d4e4068296
--- /dev/null
+++ b/doc/forum/Change_remote_server_address/comment_1_401c3d2530ac7ba41dd3857ab4737ed5._comment
@@ -0,0 +1,10 @@
+[[!comment format=mdwn
+ username="http://joeyh.name/"
+ ip="209.250.56.47"
+ subject="comment 1"
+ date="2013-11-01T16:35:47Z"
+ content="""
+Yes, you can certainly edit .git/config in any way you like and the assistant will use the new values when started back up.
+
+It shouldn't be using a hardcoded IP address normally, unless you manually entered an IP address when setting up that ssh remote. Using DNS is better..
+"""]]
diff --git a/doc/forum/Change_remote_server_address/comment_2_93a4c44d552efe6d51584e0aab3605e7._comment b/doc/forum/Change_remote_server_address/comment_2_93a4c44d552efe6d51584e0aab3605e7._comment
new file mode 100644
index 0000000000..84ec29e051
--- /dev/null
+++ b/doc/forum/Change_remote_server_address/comment_2_93a4c44d552efe6d51584e0aab3605e7._comment
@@ -0,0 +1,10 @@
+[[!comment format=mdwn
+ username="hasard@aa44fbf6c0a991a7b76fb8a118cc0b274f528de1"
+ nickname="hasard"
+ subject="like dropbox, with your own cloud "
+ date="2015-08-26T17:49:16Z"
+ content="""
+just to be complete: during setup some useful assistant may have created rsa keys so that the assistant can access remote hosts without password and following the update of .git/config you probably will need to update .ssh/config as well.
+ 
+Like many things with git annex and the assistant, its just \"like dropbox with your own cloud\", but you need first to become expert in editing .git/config and .ssh/config. 
+"""]]
diff --git a/doc/forum/Changing_files_during_git_annex_runs.mdwn b/doc/forum/Changing_files_during_git_annex_runs.mdwn
new file mode 100644
index 0000000000..7ff362d1cc
--- /dev/null
+++ b/doc/forum/Changing_files_during_git_annex_runs.mdwn
@@ -0,0 +1,12 @@
+Hello,
+
+I have my music git annexed, direct mode. It's about 30k files of 429GB size. Some actions take considerable time (sync, add and of course transfer to/from other repos). During this time I don't hear music because of my player changes files. :-(
+
+When is it a problem when a files changes during git annex operations?
+
+git annex get gives a wrong checksum I guess and you need to re-transfer later.
+
+What about git annex add?
+
+Thx!
+Florian
diff --git a/doc/forum/Changing_files_during_git_annex_runs/comment_1_8067077c49dafbe2afa7d182b3314df4._comment b/doc/forum/Changing_files_during_git_annex_runs/comment_1_8067077c49dafbe2afa7d182b3314df4._comment
new file mode 100644
index 0000000000..312100b116
--- /dev/null
+++ b/doc/forum/Changing_files_during_git_annex_runs/comment_1_8067077c49dafbe2afa7d182b3314df4._comment
@@ -0,0 +1,14 @@
+[[!comment format=mdwn
+ username="http://joeyh.name/"
+ ip="209.250.56.111"
+ subject="comment 1"
+ date="2014-10-13T21:33:35Z"
+ content="""
+git-annex will detect things like files being changed at the same time that `git annex add` is run. It doesn't currently do a full lsof check to make sure nothing has a file open when `git annex add` is run (the assistant does do such extra checks).
+
+I don't see why syncing or transferring files to other repos should prevent your music player or whatever from using the files that you have locally present.
+
+If you're using direct mode and your music player is modifying the contents of files in the repository, then that will prevent other repositories downloading older versions of those files, since the old version is no longer present in the direct mode repository. There is the possibility for some innefficient attempts to transfer a file, that would fail because it got modified in between. git-annex detects this, but it can still waste bandwidth.
+
+(If my music player was modifying files all the time,  I'd give it a good talking to; that's not the job of a music player. But, I keep my music in an indirect mode repo, so any music player would find it hard to modify annexed files anyway.)
+"""]]
diff --git a/doc/forum/Changing_the_target_of_an_external_special_remote.mdwn b/doc/forum/Changing_the_target_of_an_external_special_remote.mdwn
new file mode 100644
index 0000000000..84b440ccf6
--- /dev/null
+++ b/doc/forum/Changing_the_target_of_an_external_special_remote.mdwn
@@ -0,0 +1 @@
+I've initialized an external special remote (specifically, an rclone remote). It is working great. However, I would like to edit my rclone config and change the name of the rclone remote. This is the value I passed as `target=` when running `git annex initremote`. Can I change the target of the special remote in git annex? I grepped through `~/.git`, but didn't see it anywhere.
diff --git a/doc/forum/Changing_the_target_of_an_external_special_remote/comment_1_db1112fb1b4626f2d2c1b7b78ab3f5e5._comment b/doc/forum/Changing_the_target_of_an_external_special_remote/comment_1_db1112fb1b4626f2d2c1b7b78ab3f5e5._comment
new file mode 100644
index 0000000000..d6dc0ff10d
--- /dev/null
+++ b/doc/forum/Changing_the_target_of_an_external_special_remote/comment_1_db1112fb1b4626f2d2c1b7b78ab3f5e5._comment
@@ -0,0 +1,11 @@
+[[!comment format=mdwn
+ username="joey"
+ subject="""comment 1"""
+ date="2016-09-05T16:49:37Z"
+ content="""
+This is generally the kind of thing that `git annex enableremote` is able
+to do, by specifying a new value for target=
+
+I am not very familiar with the rclone remote, so don't know if it will
+work to change its target.
+"""]]
diff --git a/doc/forum/Check_if_remote_is_using_GPG__63__.mdwn b/doc/forum/Check_if_remote_is_using_GPG__63__.mdwn
new file mode 100644
index 0000000000..d719b2feb7
--- /dev/null
+++ b/doc/forum/Check_if_remote_is_using_GPG__63__.mdwn
@@ -0,0 +1 @@
+Is there a way to check if a special remote is setup to use GPG? And, if so, see the ID of keys that it is encrypting to?
diff --git a/doc/forum/Check_if_remote_is_using_GPG__63__/comment_1_db8ce8ef50fc33a28860ee475988450f._comment b/doc/forum/Check_if_remote_is_using_GPG__63__/comment_1_db8ce8ef50fc33a28860ee475988450f._comment
new file mode 100644
index 0000000000..ff073cd9a0
--- /dev/null
+++ b/doc/forum/Check_if_remote_is_using_GPG__63__/comment_1_db8ce8ef50fc33a28860ee475988450f._comment
@@ -0,0 +1,14 @@
+[[!comment format=mdwn
+ username="http://joeyh.name/"
+ nickname="joey"
+ subject="comment 1"
+ date="2013-03-17T18:59:40Z"
+ content="""
+You can tell this by looking at remote.log:
+
+git show git-annex:remote.log
+
+Remotes that are encrypted will have a big \"cipher=\" block in there, and ones that are encrypted to gpg public keys will have a list of the keys following \"cipherkeys=\"
+
+
+"""]]
diff --git a/doc/forum/Check_if_remote_is_using_GPG__63__/comment_2_11c7033904c9c7a1df766e915632c386._comment b/doc/forum/Check_if_remote_is_using_GPG__63__/comment_2_11c7033904c9c7a1df766e915632c386._comment
new file mode 100644
index 0000000000..a88021352e
--- /dev/null
+++ b/doc/forum/Check_if_remote_is_using_GPG__63__/comment_2_11c7033904c9c7a1df766e915632c386._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="annexuser"
+ ip="77.247.182.246"
+ subject="comment 2"
+ date="2013-03-17T22:25:58Z"
+ content="""
+If a special remote was setup using shared encryption, can I change it to using keys by simply doing `initremote myremote encryption=keyid`? Or should I remove the remote and start over?
+"""]]
diff --git a/doc/forum/Check_if_remote_is_using_GPG__63__/comment_3_a7ab70ad87a334c36761ddb3d830d99b._comment b/doc/forum/Check_if_remote_is_using_GPG__63__/comment_3_a7ab70ad87a334c36761ddb3d830d99b._comment
new file mode 100644
index 0000000000..f467495b8e
--- /dev/null
+++ b/doc/forum/Check_if_remote_is_using_GPG__63__/comment_3_a7ab70ad87a334c36761ddb3d830d99b._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="http://joeyh.name/"
+ nickname="joey"
+ subject="comment 3"
+ date="2013-03-18T15:09:53Z"
+ content="""
+git-annex won't let you change a remote's encryption scheme, as this would make any data stored in the remote using the old scheme be no longer accessible. You need to make a new remote. It should be fine to point the new remote at the same location as the old one, though.
+"""]]
diff --git a/doc/forum/Check_when_your_last_fsck_was__63__.mdwn b/doc/forum/Check_when_your_last_fsck_was__63__.mdwn
new file mode 100644
index 0000000000..4a655cc1de
--- /dev/null
+++ b/doc/forum/Check_when_your_last_fsck_was__63__.mdwn
@@ -0,0 +1,4 @@
+Hey Joey,
+    Is there a way to see when the last fsck was?  I know about --incremental and the related options, but sometimes I'd like to know when the last time I fsck'd a file (or even the whole repo) was.  There doesn't seem to be a command line option for it, but I know that info is saved somewhere...maybe this could be part of 'git annex status'?
+
+$(words of gratitude and encouragement)
diff --git a/doc/forum/Check_when_your_last_fsck_was__63__/comment_1_ee98a1fcd796fe4fd7af6f77d0c1837d._comment b/doc/forum/Check_when_your_last_fsck_was__63__/comment_1_ee98a1fcd796fe4fd7af6f77d0c1837d._comment
new file mode 100644
index 0000000000..ae8ffe42fa
--- /dev/null
+++ b/doc/forum/Check_when_your_last_fsck_was__63__/comment_1_ee98a1fcd796fe4fd7af6f77d0c1837d._comment
@@ -0,0 +1,10 @@
+[[!comment format=mdwn
+ username="http://joeyh.name/"
+ nickname="joey"
+ subject="comment 1"
+ date="2013-05-19T18:50:06Z"
+ content="""
+You're quite right. that info is stored in `.git/annex/fsckstate`. The timestamp of the file is when the last incremental fsck was started. The contents of the file is the same date, represented as seconds from epoch.
+
+That is only written to when doing an incremental fsck though. A non-incremental fsck does not touch the file.
+"""]]
diff --git a/doc/forum/Check_when_your_last_fsck_was__63__/comment_2_7238740d7904a6a9715c1cd7d8a529d4._comment b/doc/forum/Check_when_your_last_fsck_was__63__/comment_2_7238740d7904a6a9715c1cd7d8a529d4._comment
new file mode 100644
index 0000000000..52e9841747
--- /dev/null
+++ b/doc/forum/Check_when_your_last_fsck_was__63__/comment_2_7238740d7904a6a9715c1cd7d8a529d4._comment
@@ -0,0 +1,21 @@
+[[!comment format=mdwn
+ username="anthony@ad39673d230d75cbfd19d2757d754030049c7673"
+ nickname="anthony"
+ avatar="http://cdn.libravatar.org/avatar/05b48b72766177b3b0a6ff4afdb70790"
+ subject="Seems this has been somewhat implemented?"
+ date="2018-07-08T07:15:38Z"
+ content="""
+From one of my repositories (UUIDs redacted, I doubt that's required but better safe than sorry):
+
+    $ git show git-annex:activity.log
+    XXXXXXXX-XXXX-XXXX-XXXX-XXXXXXXXXX4e Fsck timestamp=1509040059.713438476s
+    XXXXXXXX-XXXX-XXXX-XXXX-XXXXXXXXXX41 Fsck timestamp=1509495891.418704179s
+    XXXXXXXX-XXXX-XXXX-XXXX-XXXXXXXXXX28 Fsck timestamp=1496379403.744140736s
+    XXXXXXXX-XXXX-XXXX-XXXX-XXXXXXXXXX20 Fsck timestamp=1527838142.197908353s
+    XXXXXXXX-XXXX-XXXX-XXXX-XXXXXXXXXXdb Fsck timestamp=1530951201.006459518s
+    XXXXXXXX-XXXX-XXXX-XXXX-XXXXXXXXXX37 Fsck timestamp=1509828389.73059196s
+
+Those appear to be Unix timestamps, easily converted to human-readable with, e.g., `date -d @TIMESTAMP` (without the `s` at the end; at least with GNU date). E.g., the last one becomes Sat Nov  4 16:46:29 EDT 2017.
+
+Sometimes there are multiple lines per-repository; not sure what determines how long they're kept. Before running `git-annex sync`, this one had dozens of lines of history; sync cleaned it up. Of course, those lines of history remain in the git history.
+"""]]
diff --git a/doc/forum/Check_when_your_last_fsck_was__63__/comment_3_8fa3680b084fad45d7a184d1191fedce._comment b/doc/forum/Check_when_your_last_fsck_was__63__/comment_3_8fa3680b084fad45d7a184d1191fedce._comment
new file mode 100644
index 0000000000..42faf88466
--- /dev/null
+++ b/doc/forum/Check_when_your_last_fsck_was__63__/comment_3_8fa3680b084fad45d7a184d1191fedce._comment
@@ -0,0 +1,9 @@
+[[!comment format=mdwn
+ username="joey"
+ subject="""comment 3"""
+ date="2018-08-03T18:28:07Z"
+ content="""
+That was implemented for a different use-case (`git-annex expire`) but
+you can certianly use it to keep track of the last time some fsck, perhaps
+a partial one, was run on a repository.
+"""]]
diff --git a/doc/forum/Checking_for_missing_files_in_S3___40__reduced_redundancy__41__.mdwn b/doc/forum/Checking_for_missing_files_in_S3___40__reduced_redundancy__41__.mdwn
new file mode 100644
index 0000000000..97299a9662
--- /dev/null
+++ b/doc/forum/Checking_for_missing_files_in_S3___40__reduced_redundancy__41__.mdwn
@@ -0,0 +1,5 @@
+I would like to use git-annex with [S3 reduced redundancy storace (RRS)](https://aws.amazon.com/s3/faqs/#rrs_anchor), a cheaper S3 service with an higher chance of seeing your files disappear overnight. This means that git-annex may think that a certain files has been copied to a S3 remote while, in fact, it is no longer there.
+
+I would like to check that the files S3 are really there and, if some files are not, record that we lost that copy. In the case of S3, either the data is there and it is complete or it is not there at all, so there is no need to check the data itself, just the presence of the files.
+
+Is there a way to check that all the files that are supposed to be in a S3 remote are still there?
diff --git a/doc/forum/Checking_for_missing_files_in_S3___40__reduced_redundancy__41__/comment_2_793b3d9f78562f3aecf27dd926bbcf82._comment b/doc/forum/Checking_for_missing_files_in_S3___40__reduced_redundancy__41__/comment_2_793b3d9f78562f3aecf27dd926bbcf82._comment
new file mode 100644
index 0000000000..404ba8b878
--- /dev/null
+++ b/doc/forum/Checking_for_missing_files_in_S3___40__reduced_redundancy__41__/comment_2_793b3d9f78562f3aecf27dd926bbcf82._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="http://joeyh.name/"
+ ip="209.250.56.114"
+ subject="comment 2"
+ date="2014-04-24T18:12:59Z"
+ content="""
+git annex fsck --fast --from $yours3remote
+"""]]
diff --git a/doc/forum/Checkout_only_some_files_with_the_assistant.mdwn b/doc/forum/Checkout_only_some_files_with_the_assistant.mdwn
new file mode 100644
index 0000000000..7a2270d920
--- /dev/null
+++ b/doc/forum/Checkout_only_some_files_with_the_assistant.mdwn
@@ -0,0 +1,14 @@
+Hi,
+
+I really like git-annex for sharing contents with all my devices. But i have a problem with the assistant on my phone, i don't know how to git annex get some file but not all.
+For example i have this on a remote:
+
+repository:
+ - big file1
+ - big file2
+ - big file3
+
+I have this repository on my phone. I would like to be able to checkout out only "big file2" directly within the assistant.
+How can i achieve that?
+
+Thank you. 
diff --git a/doc/forum/Checkout_only_some_files_with_the_assistant/comment_1_23d8ab1a05e3e9d3611bd12a4ba70b0c._comment b/doc/forum/Checkout_only_some_files_with_the_assistant/comment_1_23d8ab1a05e3e9d3611bd12a4ba70b0c._comment
new file mode 100644
index 0000000000..85325c60ce
--- /dev/null
+++ b/doc/forum/Checkout_only_some_files_with_the_assistant/comment_1_23d8ab1a05e3e9d3611bd12a4ba70b0c._comment
@@ -0,0 +1,10 @@
+[[!comment format=mdwn
+ username="http://joeyh.name/"
+ ip="209.250.56.114"
+ subject="comment 1"
+ date="2014-04-24T18:15:15Z"
+ content="""
+[[preferred_content]] expressions can be used to do that. Works best if all the files you want have some extension or are in a specific directory.
+
+There are some [[preferred_content/standard_groups]] that might do what you want, particularly the \"manual\" one.
+"""]]
diff --git a/doc/forum/Checkout_only_some_files_with_the_assistant/comment_2_bf095ff5b5af95b062ae1f7da566a279._comment b/doc/forum/Checkout_only_some_files_with_the_assistant/comment_2_bf095ff5b5af95b062ae1f7da566a279._comment
new file mode 100644
index 0000000000..cf35f3541d
--- /dev/null
+++ b/doc/forum/Checkout_only_some_files_with_the_assistant/comment_2_bf095ff5b5af95b062ae1f7da566a279._comment
@@ -0,0 +1,12 @@
+[[!comment format=mdwn
+ username="elfangor"
+ ip="176.57.244.101"
+ subject="comment 2"
+ date="2014-04-28T13:12:07Z"
+ content="""
+Thanks for your reply.
+
+The Standard group \"manual\" seems to be what i need. The last thing i'd like to now is, is there a way to do git annex get on the assistant(on the webui) on my phone? (I know i can do this on the shell opened, but it's not that easy to write from the phone.) I guess it's not implemented, and i understand that it should be a lot of work to add a file browser in the ui where you can git annex get, and git annex add from it.
+
+Thanks again for your answer, really helpfull: I have a better idea on how i will structur my git annex repositories.
+"""]]
diff --git a/doc/forum/Checkout_only_some_files_with_the_assistant/comment_3_7c026e36e1cdd52053c34638c87d793c._comment b/doc/forum/Checkout_only_some_files_with_the_assistant/comment_3_7c026e36e1cdd52053c34638c87d793c._comment
new file mode 100644
index 0000000000..ae18c8caff
--- /dev/null
+++ b/doc/forum/Checkout_only_some_files_with_the_assistant/comment_3_7c026e36e1cdd52053c34638c87d793c._comment
@@ -0,0 +1,10 @@
+[[!comment format=mdwn
+ username="http://joeyh.name/"
+ ip="216.145.95.162"
+ subject="comment 3"
+ date="2014-05-19T15:45:57Z"
+ content="""
+There is not currently any UI to do git-annex get/drop. 
+
+There is some recent work on [[tips/file_manager_integration]] which lets you use a file manager and pick files you want and files to drop. Nobody has gotten it working with android file manager yet AFAIK.
+"""]]
diff --git a/doc/forum/Cleaning_up_after_aborted_sync_in_direct_mode.mdwn b/doc/forum/Cleaning_up_after_aborted_sync_in_direct_mode.mdwn
new file mode 100644
index 0000000000..f0912fd74a
--- /dev/null
+++ b/doc/forum/Cleaning_up_after_aborted_sync_in_direct_mode.mdwn
@@ -0,0 +1,19 @@
+I recently began experimenting with direct mode on a repository where only a very limited number of commands (`git annex sync`, `git annex add .`, etc.) are being used (through desktop buttons).
+
+Things were working well, and then on another repository in indirect mode I moved a large folder and synced that data with a bare repo, and synced the repo in direct mode with the same bare repo (to get everything in sync).
+
+At that point, git annex seemed to hang, taking forever to complete, and from looking a processes and files it seemed like it was going nowhere, so I killed it. I wish I had done a bit more diagnostics before that, but unfortunately I didn't.
+
+At that point, I noticed that the old directory was still there and that there were still many files in it, but the new folder had also been created and there were files there, too. So I thought the transfer was done, and for whatever reason `git annex sync` had just not cleaned up properly.
+
+In fact, the sync was only half done and the remaining files in the old directory were the only (local) copies (on the direct mode repo). I removed them and then synced again, which actually told git to delete those symlinks.
+
+When I updated another (indirect mode) repository I noticed this, so I reverted the commit in question and got the symlinks back, no data lost. Then I went back to the direct mode repo, switched to indirect mode because I was worried about direct mode, `git annex sync`ed, then `git annex get` to get the files again from a usb and everything was back to normal.
+
+Except that when I tried to go back to direct mode in that original repo, I got an error saying that git could not stat a file which is in the old (deleted) folder. Searching around, I noticed that in the `.git/annex/objects` directory, there are many remaining `.map` files with lines referring to the old (deleted) directory.
+
+Is there any way to "reset" this somehow? I would like to switch back to direct mode, and I'd prefer not to recreate the repo.
+
+Oh and I'm using version: 4.20130501-g4a5bfb3.
+
+Thanks!
diff --git a/doc/forum/Cleaning_up_after_aborted_sync_in_direct_mode/comment_1_3440b2e1662d3b113c18283afcbf4520._comment b/doc/forum/Cleaning_up_after_aborted_sync_in_direct_mode/comment_1_3440b2e1662d3b113c18283afcbf4520._comment
new file mode 100644
index 0000000000..77c0ebbba6
--- /dev/null
+++ b/doc/forum/Cleaning_up_after_aborted_sync_in_direct_mode/comment_1_3440b2e1662d3b113c18283afcbf4520._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="https://www.google.com/accounts/o8/id?id=AItOawnkBYpLu_NOj7Uq0-acvLgWhxF8AUEIJbo"
+ nickname="Chris"
+ subject="comment 1"
+ date="2013-05-10T03:50:26Z"
+ content="""
+Maybe a simpler question: if I delete all the `.map` and `.cache` files that are leftover in the `.git/annex/objects` directory, will that effectively \"reset\" the status so that I can then switch to direct mode?
+"""]]
diff --git a/doc/forum/Cleaning_up_after_aborted_sync_in_direct_mode/comment_2_9a61ba8ac4a375f1d69cd09b5a6f8091._comment b/doc/forum/Cleaning_up_after_aborted_sync_in_direct_mode/comment_2_9a61ba8ac4a375f1d69cd09b5a6f8091._comment
new file mode 100644
index 0000000000..e471f113ea
--- /dev/null
+++ b/doc/forum/Cleaning_up_after_aborted_sync_in_direct_mode/comment_2_9a61ba8ac4a375f1d69cd09b5a6f8091._comment
@@ -0,0 +1,14 @@
+[[!comment format=mdwn
+ username="https://www.google.com/accounts/o8/id?id=AItOawnkBYpLu_NOj7Uq0-acvLgWhxF8AUEIJbo"
+ nickname="Chris"
+ subject="comment 2"
+ date="2013-05-10T04:01:38Z"
+ content="""
+After reading [this](http://git-annex.branchable.com/internals/):
+
+> Also in direct mode, some additional data is stored in these directories. .cache files contain cached file stats used in detecting when a file has changed, and .map files contain a list of file(s) in the work directory that contain the key.
+
+I decided that indeed, deleting the `.map` and `.cache` files should do the trick, so I deleted them and direct mode works again.
+
+Still not sure what went wrong with the original sync, but at least I got things back on track.
+"""]]
diff --git a/doc/forum/Cleaning_up_after_aborted_sync_in_direct_mode/comment_3_6b9d8c48547f3d0a911310622ba91df7._comment b/doc/forum/Cleaning_up_after_aborted_sync_in_direct_mode/comment_3_6b9d8c48547f3d0a911310622ba91df7._comment
new file mode 100644
index 0000000000..b7d495dea4
--- /dev/null
+++ b/doc/forum/Cleaning_up_after_aborted_sync_in_direct_mode/comment_3_6b9d8c48547f3d0a911310622ba91df7._comment
@@ -0,0 +1,13 @@
+[[!comment format=mdwn
+ username="http://joeyh.name/"
+ nickname="joey"
+ subject="comment 3"
+ date="2013-05-13T18:00:42Z"
+ content="""
+This sounds like one or more bugs that I should address, but I'm having trouble understanding exactly what you did, from your description, at the level of detail I need to replicate a problem.
+
+Do you still have a copy of the corrupted repository? Or any error messages to show me?
+
+It sounds like at least one problem might have been this bug: [[bugs/Unable_to_switch_back_to_direct_mode]]
+At least that is fixed in current git.
+"""]]
diff --git a/doc/forum/Combine_tags_in_view_branches.mdwn b/doc/forum/Combine_tags_in_view_branches.mdwn
new file mode 100644
index 0000000000..8dfb8208fc
--- /dev/null
+++ b/doc/forum/Combine_tags_in_view_branches.mdwn
@@ -0,0 +1,24 @@
+Is there a way to 'combine' tags in view branches? For example:
+
+```
+git annex view photos videos
+```
+
+produces a directory tree like so:
+
++ photos
+  - a.jpg
+  - b.jpg
+  - ...
++ videos
+  - a.mp4
+  - b.mp4
+  - ...
+
+Is there a way to achieve the following output?:
+
+- a.jpg
+- b.jpg
+- a.mp4
+- b.mp4
+- ...
diff --git a/doc/forum/Coming_from_git_world.mdwn b/doc/forum/Coming_from_git_world.mdwn
new file mode 100644
index 0000000000..ee8b844e81
--- /dev/null
+++ b/doc/forum/Coming_from_git_world.mdwn
@@ -0,0 +1,9 @@
+So i am coming from git world, All documentation and posts seemed to be for media etc. I have a simple question.
+
+Can i use git annex to manage large files , Which i think i can. But will it work with branching/tagging etc.
+
+e.g. We currently use git have some large files in git repo.... I am planning to move the large files to use annex. But would like to maintain branches, i.e. each branch might have different version of files and maybe tagged, So i can get back some old version from a branch.
+
+Is this possible ? I did not get any explicit  answer or examples on how this works or even if its supported.
+
+Also can i use same folder for git and git-annex ? 
diff --git a/doc/forum/Coming_from_git_world/comment_10_098bef38c2688607e869425a557cc482._comment b/doc/forum/Coming_from_git_world/comment_10_098bef38c2688607e869425a557cc482._comment
new file mode 100644
index 0000000000..85ba7fe7f0
--- /dev/null
+++ b/doc/forum/Coming_from_git_world/comment_10_098bef38c2688607e869425a557cc482._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="http://joeyh.name/"
+ ip="4.154.7.235"
+ subject="comment 10"
+ date="2013-07-10T17:05:49Z"
+ content="""
+It has its own ssh server written in java too, which hardcodes the set of commands it supports. AFAICS it would require some java coding to add a command to this, it does not seem to be designed for easy extensability.
+"""]]
diff --git a/doc/forum/Coming_from_git_world/comment_11_98d75a1415e0c3689ab4231855e61233._comment b/doc/forum/Coming_from_git_world/comment_11_98d75a1415e0c3689ab4231855e61233._comment
new file mode 100644
index 0000000000..3dba0760b5
--- /dev/null
+++ b/doc/forum/Coming_from_git_world/comment_11_98d75a1415e0c3689ab4231855e61233._comment
@@ -0,0 +1,12 @@
+[[!comment format=mdwn
+ username="https://www.google.com/accounts/o8/id?id=AItOawn5RcmefXjrl1vmbHIiOWQyXGXVKxlm3rg"
+ nickname="Kavin"
+ subject="comment 11"
+ date="2013-07-10T20:14:52Z"
+ content="""
+:( Ahh... So i guess this might not be possible in near future.
+
+Ill go to plan B, Setup another server which users will connect to for large files which will have git-annex installed. (So a central Repository for git annex). They will need to add another remote in git config.
+
+Does git annex have a way to manage ssh keys ?  (Please can you point me to the location where i can read how git-annex-shell ? )
+"""]]
diff --git a/doc/forum/Coming_from_git_world/comment_12_5e7079e9bf3e4d97191333c66ac00e52._comment b/doc/forum/Coming_from_git_world/comment_12_5e7079e9bf3e4d97191333c66ac00e52._comment
new file mode 100644
index 0000000000..2d76687371
--- /dev/null
+++ b/doc/forum/Coming_from_git_world/comment_12_5e7079e9bf3e4d97191333c66ac00e52._comment
@@ -0,0 +1,10 @@
+[[!comment format=mdwn
+ username="http://joeyh.name/"
+ ip="4.154.7.235"
+ subject="comment 12"
+ date="2013-07-10T21:10:03Z"
+ content="""
+Yes, a separate repository is one way to go, although I do not think it would be very hard to get gerrit's internal ssh to run git-annex-shell.
+
+[[git-annex-shell]] is a low-level restricted shell like git-shell, not a login manager, so it has nothing at all to do with ssh keys. Things like gitosis and gitolite can be built on top of git-shell and git-annex-shell and handle ssh key management.
+"""]]
diff --git a/doc/forum/Coming_from_git_world/comment_1_357443dc601ae38784c01cf18552f4d5._comment b/doc/forum/Coming_from_git_world/comment_1_357443dc601ae38784c01cf18552f4d5._comment
new file mode 100644
index 0000000000..adc65d50f2
--- /dev/null
+++ b/doc/forum/Coming_from_git_world/comment_1_357443dc601ae38784c01cf18552f4d5._comment
@@ -0,0 +1,14 @@
+[[!comment format=mdwn
+ username="http://joeyh.name/"
+ ip="4.154.7.235"
+ subject="comment 1"
+ date="2013-07-08T20:09:54Z"
+ content="""
+Yes, git-annex would not do very well at adding large file support to git if it did not handle tagging, branches, etc! So of course it does. It's in a sense too obvious a thing to get much mention. And so people sometimes get confused about it.
+
+The only thing to need to be aware of coming from git is that not every repository will have every version of every file locally available. When you check out a branch, you may need to run `git annex get` to retrieve those versions from origin or elsewhere.
+
+And, `git annex unused` can be used to find versions of files that no existing tag or branch refers to, and `git annex dropunused` can then delete those versions. If you want to ensure every revision in your git repo is accessible, you should avoid using those two commands; otherwise git-annex will never delete old versions of files.
+
+The unreleased git master adds a new feature, a --all switch that makes git annex commands operate on all versions of files. While normally `git annex get` will only do what it needs to to get all files in the currently checked out branch, `git annex get --all` will pull down every version of every file in the whole history. Similarly, `git annex copy --all --to origin` will ensure that every locally available version of every file is sent to origin.
+"""]]
diff --git a/doc/forum/Coming_from_git_world/comment_2_ed1847dd3f47a9d013b8dd0455fb80ff._comment b/doc/forum/Coming_from_git_world/comment_2_ed1847dd3f47a9d013b8dd0455fb80ff._comment
new file mode 100644
index 0000000000..33d7d44d35
--- /dev/null
+++ b/doc/forum/Coming_from_git_world/comment_2_ed1847dd3f47a9d013b8dd0455fb80ff._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="http://joeyh.name/"
+ ip="4.154.7.235"
+ subject="comment 2"
+ date="2013-07-08T20:18:27Z"
+ content="""
+And yes, you can use the same git repository for files added to git `git add file` and added to git annex `git annex add file`
+"""]]
diff --git a/doc/forum/Coming_from_git_world/comment_3_09c6bb83a73d34dff2b8bc185a14a1db._comment b/doc/forum/Coming_from_git_world/comment_3_09c6bb83a73d34dff2b8bc185a14a1db._comment
new file mode 100644
index 0000000000..07023ffe3c
--- /dev/null
+++ b/doc/forum/Coming_from_git_world/comment_3_09c6bb83a73d34dff2b8bc185a14a1db._comment
@@ -0,0 +1,18 @@
+[[!comment format=mdwn
+ username="https://www.google.com/accounts/o8/id?id=AItOawn5RcmefXjrl1vmbHIiOWQyXGXVKxlm3rg"
+ nickname="Kavin"
+ subject="comment 3"
+ date="2013-07-08T23:27:45Z"
+ content="""
+Awesome!! thanks, As i am kicking the tires, Based on my current scenario, How can i deploy this in my company , So each developer to do the bare minimum, Like just install git annex and it should work.
+
+i.e. is it possible to avoid the step git remote add backup ? Then they can just do
+
+$ yum install git-annex
+$ cd repo
+$ git checkout staging
+$ git pull
+$ git annex get large_file
+
+And it then just works.
+"""]]
diff --git a/doc/forum/Coming_from_git_world/comment_4_6c731bb9a8d21dd9ab8c09612b23f908._comment b/doc/forum/Coming_from_git_world/comment_4_6c731bb9a8d21dd9ab8c09612b23f908._comment
new file mode 100644
index 0000000000..f09281bc45
--- /dev/null
+++ b/doc/forum/Coming_from_git_world/comment_4_6c731bb9a8d21dd9ab8c09612b23f908._comment
@@ -0,0 +1,16 @@
+[[!comment format=mdwn
+ username="http://joeyh.name/"
+ ip="4.154.7.235"
+ subject="comment 4"
+ date="2013-07-09T18:14:07Z"
+ content="""
+I don't know where you got the \"git remote add backup\" step from. Obviously this is not necessary unless you want to add a remote named \"backup\".
+
+To use git-annex in a centralized git environment, which it sounds like you have here, you just need to install git-annex on the central git server, and arrange for all the developers to have ssh access to it. Then any git repository that is cloned from that server using ssh as the transport can support git-annex without the user needing to do anything special to set it up.
+
+This assumes that users have shell accounts on the server. git-annex includes the git-annex-shell program, which is similar to the git-shell in git. User accounts can be locked down to use this restricted shell if giving them full shell access to the server is not desired.
+
+If the server is using a git repository manager like gitolite or gitosis, those can also be adapted to use git-annex-shell. I got gitolite patched to support it earlier, see [[tips/using_gitolite_with_git-annex]].
+
+PS: I'm available for consulting on deploying git-annex in production environments. ;)
+"""]]
diff --git a/doc/forum/Coming_from_git_world/comment_5_e719d99af5afd90da3d3db692eff28dc._comment b/doc/forum/Coming_from_git_world/comment_5_e719d99af5afd90da3d3db692eff28dc._comment
new file mode 100644
index 0000000000..ffeaaa6e06
--- /dev/null
+++ b/doc/forum/Coming_from_git_world/comment_5_e719d99af5afd90da3d3db692eff28dc._comment
@@ -0,0 +1,11 @@
+[[!comment format=mdwn
+ username="https://www.google.com/accounts/o8/id?id=AItOawn5RcmefXjrl1vmbHIiOWQyXGXVKxlm3rg"
+ nickname="Kavin"
+ subject="comment 5"
+ date="2013-07-09T20:32:51Z"
+ content="""
+Thank you very much!!!!! Yes, Planning to use Centralized repo. Basically what i understood was there will be a git server and separate server for hosting the large files. Its great to know i can use the same server and do not need to explicitly point to the server.
+
+As we are using Gerrit, What is the general best practice for it. (After i am done with it, I will write a blog post, As i am sure many people with centralized repo's might need this)  
+Thanks! 
+"""]]
diff --git a/doc/forum/Coming_from_git_world/comment_6_85a42106944dba9995fb3f4bfee3443a._comment b/doc/forum/Coming_from_git_world/comment_6_85a42106944dba9995fb3f4bfee3443a._comment
new file mode 100644
index 0000000000..840553025e
--- /dev/null
+++ b/doc/forum/Coming_from_git_world/comment_6_85a42106944dba9995fb3f4bfee3443a._comment
@@ -0,0 +1,10 @@
+[[!comment format=mdwn
+ username="http://joeyh.name/"
+ ip="4.154.7.235"
+ subject="comment 6"
+ date="2013-07-09T20:39:25Z"
+ content="""
+Yes, the default for a repository accessed using ssh is to also store the large files in that repository. Of course you can set up a second remote to hold the large files if that works better.
+
+I don't have any personal experience with Gerrit. All I can say is that files managed by git-annex will appear as symlinks.
+"""]]
diff --git a/doc/forum/Coming_from_git_world/comment_7_90623294b910ceca3dc8ebd41b50fc9b._comment b/doc/forum/Coming_from_git_world/comment_7_90623294b910ceca3dc8ebd41b50fc9b._comment
new file mode 100644
index 0000000000..50b34fb86d
--- /dev/null
+++ b/doc/forum/Coming_from_git_world/comment_7_90623294b910ceca3dc8ebd41b50fc9b._comment
@@ -0,0 +1,38 @@
+[[!comment format=mdwn
+ username="https://www.google.com/accounts/o8/id?id=AItOawn5RcmefXjrl1vmbHIiOWQyXGXVKxlm3rg"
+ nickname="Kavin"
+ subject="comment 7"
+ date="2013-07-09T23:48:17Z"
+ content="""
+Thanks, So i made some progress, In the gerrit repo server,  (which has git-annex installed)
+
+I ran: 
+    git annex init origin,
+
+Did a clone of the repo, it had the branch git-annex, So i assumed all is working.
+
+I did a 
+    git annex add 
+and 
+    git push
+
+Then when i try to 
+    git annex copy --to origin
+
+I get error , But git-annex-shell is installed on the machine:
+
+    $ echo $PATH
+    /usr/lib64/qt-3.3/bin:/usr/local/bin:/bin:/usr/bin:/usr/local/sbin:/usr/sbin:/sbin:/home/gerrit2/bin
+    $ which git-annex-shell
+    /usr/bin/git-annex-shell
+
+
+
+    Gerrit Code Review: git-annex-shell: not found
+    rsync: connection unexpectedly closed (0 bytes received so far) [sender]
+    rsync error: error in rsync protocol data stream (code 12) at io.c(600) [sender=3.0.6]
+    
+    rsync failed -- run git annex again to resume file transfer failed
+    
+Any idea what i  need to do to make it work ?
+"""]]
diff --git a/doc/forum/Coming_from_git_world/comment_8_28dbee30eb54877418f72eb8935302d8._comment b/doc/forum/Coming_from_git_world/comment_8_28dbee30eb54877418f72eb8935302d8._comment
new file mode 100644
index 0000000000..668cfbd95a
--- /dev/null
+++ b/doc/forum/Coming_from_git_world/comment_8_28dbee30eb54877418f72eb8935302d8._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="http://joeyh.name/"
+ ip="4.154.7.235"
+ subject="comment 8"
+ date="2013-07-10T16:42:18Z"
+ content="""
+If gerrit is handling incoming ssh connections, it's probable running its own restricted shell. That shell would need to be configured to spawn git-annex-shell when asked to do so in order for git-annex to transfer files to that repository.
+"""]]
diff --git a/doc/forum/Coming_from_git_world/comment_9_6edb36ea9535030fa3766937398e5bc7._comment b/doc/forum/Coming_from_git_world/comment_9_6edb36ea9535030fa3766937398e5bc7._comment
new file mode 100644
index 0000000000..2f1034d173
--- /dev/null
+++ b/doc/forum/Coming_from_git_world/comment_9_6edb36ea9535030fa3766937398e5bc7._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="https://www.google.com/accounts/o8/id?id=AItOawln3ckqKx0x_xDZMYwa9Q1bn4I06oWjkog"
+ nickname="Michael"
+ subject="comment 9"
+ date="2013-07-10T16:57:29Z"
+ content="""
+AFAIK Gerrit relies on jgit implementation of git in java; allowing it to execute helper programs like git-annex-shell may be tricky.
+"""]]
diff --git a/doc/forum/Comparison_with_other_big_files_solutions.mdwn b/doc/forum/Comparison_with_other_big_files_solutions.mdwn
new file mode 100644
index 0000000000..cfbd12d29b
--- /dev/null
+++ b/doc/forum/Comparison_with_other_big_files_solutions.mdwn
@@ -0,0 +1,11 @@
+# Comparison
+
+Based on my understanding of the http://git-annex.branchable.com/not/ page and of some research,
+I have made a draft of an [Online excel document][1] that compares different git file solutions.
+
+Feel free to edit the document and help the community to understand the different solutions out there.
+
+[1]: https://onedrive.live.com/redir?resid=C66C90783BD2C185!1200&authkey=!ACF-Ol_mG3DwQ2k&ithint=file%2c.xlsx
+
+Thank you very much,
+Alex
diff --git a/doc/forum/Compile_Problem_with_latest_git_pull_version.mdwn b/doc/forum/Compile_Problem_with_latest_git_pull_version.mdwn
new file mode 100644
index 0000000000..761615a3e7
--- /dev/null
+++ b/doc/forum/Compile_Problem_with_latest_git_pull_version.mdwn
@@ -0,0 +1,23 @@
+Hi,
+I try to compile the latest version of git annex on my server; 
+As you can see some package (WARP) failed to install : 
+
+    [root@git-annex]$ cat /home/reyman64/.cabal/logs/warp-3.1.11.log
+    Configuring warp-3.1.11...
+    Building warp-3.1.11...
+    Preprocessing library warp-3.1.11...
+    [ 1 of 35] Compiling Network.Wai.Handler.Warp.Windows ( Network/Wai/Handler/Warp/Windows.hs, dist/build/Network/Wai/Handler/Warp/Windows.o )
+    [ 2 of 35] Compiling Paths_warp       ( dist/build/autogen/Paths_warp.hs, dist/build/Paths_warp.o )
+    [ 3 of 35] Compiling Network.Wai.Handler.Warp.ReadInt ( Network/Wai/Handler/Warp/ReadInt.hs, dist/build/Network/Wai/Handler/Warp/ReadInt.o )
+    [ 4 of 35] Compiling Network.Wai.Handler.Warp.HTTP2.EncodeFrame ( Network/Wai/Handler/Warp/HTTP2/EncodeFrame.hs, dist/build/Network/Wai/Handler/Warp/HTTP2/EncodeFrame.o )
+    [ 5 of 35] Compiling Network.Wai.Handler.Warp.MultiMap ( Network/Wai/Handler/Warp/MultiMap.hs, dist/build/Network/Wai/Handler/Warp/MultiMap.o )
+    [ 6 of 35] Compiling Network.Wai.Handler.Warp.IORef ( Network/Wai/Handler/Warp/IORef.hs, dist/build/Network/Wai/Handler/Warp/IORef.o )
+    [ 7 of 35] Compiling Network.Wai.Handler.Warp.HTTP2.Manager ( Network/Wai/Handler/Warp/HTTP2/Manager.hs, dist/build/Network/Wai/Handler/Warp/HTTP2/Manager.o )
+    [ 8 of 35] Compiling Network.Wai.Handler.Warp.Counter ( Network/Wai/Handler/Warp/Counter.hs, dist/build/Network/Wai/Handler/Warp/Counter.o )
+    [ 9 of 35] Compiling Network.Wai.Handler.Warp.FileInfoCache ( Network/Wai/Handler/Warp/FileInfoCache.hs, dist/build/Network/Wai/Handler/Warp/FileInfoCache.o )
+    [10 of 35] Compiling Network.Wai.Handler.Warp.Timeout ( Network/Wai/Handler/Warp/Timeout.hs, dist/build/Network/Wai/Handler/Warp/Timeout.o )
+
+    Network/Wai/Handler/Warp/Timeout.hs:86:15:
+        Not in scope: ‘reaperKill’
+        Perhaps you meant ‘reaperNull’ (imported from Control.Reaper)
+
diff --git a/doc/forum/Compile_Problem_with_latest_git_pull_version/comment_1_0b87183756e860d93b8c0197bac3c0b1._comment b/doc/forum/Compile_Problem_with_latest_git_pull_version/comment_1_0b87183756e860d93b8c0197bac3c0b1._comment
new file mode 100644
index 0000000000..e95b87357b
--- /dev/null
+++ b/doc/forum/Compile_Problem_with_latest_git_pull_version/comment_1_0b87183756e860d93b8c0197bac3c0b1._comment
@@ -0,0 +1,16 @@
+[[!comment format=mdwn
+ username="reyman64@740e43250854ada1b8484b3bc44b53be03367d70"
+ nickname="reyman64"
+ subject="comment 1"
+ date="2016-01-21T18:21:42Z"
+ content="""
+I think the problem arise here, with the auto-update version : 
+
+--dependency=auto-update=auto-update-0.1.2.2-082ccf928dc1747301e11c3fe47c54fa
+
+You need 0.1.3 to solve problem
+
+https://github.com/yesodweb/wai/issues/495
+
+Waiting for new commit :)
+"""]]
diff --git a/doc/forum/Compile_Problem_with_latest_git_pull_version/comment_2_1dd40ce31bcf994656fdfee7f8af43d2._comment b/doc/forum/Compile_Problem_with_latest_git_pull_version/comment_2_1dd40ce31bcf994656fdfee7f8af43d2._comment
new file mode 100644
index 0000000000..e006d86e83
--- /dev/null
+++ b/doc/forum/Compile_Problem_with_latest_git_pull_version/comment_2_1dd40ce31bcf994656fdfee7f8af43d2._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="joey"
+ subject="""comment 2"""
+ date="2016-01-22T15:53:52Z"
+ content="""
+The best way to avoid these kind of transient problems building
+dependencies is to use stack to build from source.
+"""]]
diff --git a/doc/forum/Compile_Problem_with_latest_git_pull_version/comment_3_f8e52610a592c68f9bc733bd31fa9659._comment b/doc/forum/Compile_Problem_with_latest_git_pull_version/comment_3_f8e52610a592c68f9bc733bd31fa9659._comment
new file mode 100644
index 0000000000..c029d967ee
--- /dev/null
+++ b/doc/forum/Compile_Problem_with_latest_git_pull_version/comment_3_f8e52610a592c68f9bc733bd31fa9659._comment
@@ -0,0 +1,10 @@
+[[!comment format=mdwn
+ username="reyman64@740e43250854ada1b8484b3bc44b53be03367d70"
+ nickname="reyman64"
+ subject="comment 3"
+ date="2016-01-22T22:12:24Z"
+ content="""
+You're right, works much much better with stack.
+
+If other user are interested, i create a tutorial here to install cabal / stack on : [webfaction hosting](https://community.webfaction.com/questions/19167/install-haskell-ghc-and-cabal-from-source-on-webfaction) After stack install, compiling git-annex is very easy (compared to cabal instal ....)
+"""]]
diff --git a/doc/forum/Compression_in_special_remotes___40__specifically_S3__41____63__.mdwn b/doc/forum/Compression_in_special_remotes___40__specifically_S3__41____63__.mdwn
new file mode 100644
index 0000000000..49f3d75ef3
--- /dev/null
+++ b/doc/forum/Compression_in_special_remotes___40__specifically_S3__41____63__.mdwn
@@ -0,0 +1 @@
+I use git-annex mostly to archive my photos.  They're generally large raw files and I am encrypting them going to the S3 special remote.  Are there any plans to compress objects before encrypting them?  It would save on S3 costs, only add some overhead on get and copy, and I don't think there would be any problems looking up data later.
diff --git a/doc/forum/Compression_in_special_remotes___40__specifically_S3__41____63__/comment_1_9c6c4ca0c9dc6976ba7cf27e84683bf0._comment b/doc/forum/Compression_in_special_remotes___40__specifically_S3__41____63__/comment_1_9c6c4ca0c9dc6976ba7cf27e84683bf0._comment
new file mode 100644
index 0000000000..7c5f1ecc1d
--- /dev/null
+++ b/doc/forum/Compression_in_special_remotes___40__specifically_S3__41____63__/comment_1_9c6c4ca0c9dc6976ba7cf27e84683bf0._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="http://joeyh.name/"
+ ip="4.153.8.117"
+ subject="comment 1"
+ date="2012-12-14T18:53:04Z"
+ content="""
+gpg compresses encrypted content by default, so all encrypted special remotes get compression for free
+"""]]
diff --git a/doc/forum/Confusion_about_Webapp___40__on_OSX__41__.mdwn b/doc/forum/Confusion_about_Webapp___40__on_OSX__41__.mdwn
new file mode 100644
index 0000000000..a34d5cbc61
--- /dev/null
+++ b/doc/forum/Confusion_about_Webapp___40__on_OSX__41__.mdwn
@@ -0,0 +1,9 @@
+I am confused on how to invoke the web app. I am learning how to use this tool, however I don't feel quite confident yet. I wanted to test out the web app, to play around with configurations and auto syncing, however I stumbled over this:
+
+    ➜  ~  git annex help webapp // works fine
+    ➜  ~  git annex webapp
+    Invalid argument `webapp' // uhh what.
+
+git-annex-webapp didn't work either.
+
+I installed with brew. El Capitan, latest update as of right now.
diff --git a/doc/forum/Confusion_about_Webapp___40__on_OSX__41__/comment_1_868b326cc5eb0767390ffdf45a594cff._comment b/doc/forum/Confusion_about_Webapp___40__on_OSX__41__/comment_1_868b326cc5eb0767390ffdf45a594cff._comment
new file mode 100644
index 0000000000..ed288634ec
--- /dev/null
+++ b/doc/forum/Confusion_about_Webapp___40__on_OSX__41__/comment_1_868b326cc5eb0767390ffdf45a594cff._comment
@@ -0,0 +1,10 @@
+[[!comment format=mdwn
+ username="joey"
+ subject="""comment 1"""
+ date="2015-11-30T20:13:39Z"
+ content="""
+git-annex can be built with or without support for the webapp. I suppose
+that your brew install is of the latter variety. For a build with the
+webappp included:  (or ask
+the brew developers to include the webapp in their build)
+"""]]
diff --git a/doc/forum/Confusion_about_Webapp___40__on_OSX__41__/comment_2_9b38b92d584395e5468bed03395c927f._comment b/doc/forum/Confusion_about_Webapp___40__on_OSX__41__/comment_2_9b38b92d584395e5468bed03395c927f._comment
new file mode 100644
index 0000000000..5b00be7ff1
--- /dev/null
+++ b/doc/forum/Confusion_about_Webapp___40__on_OSX__41__/comment_2_9b38b92d584395e5468bed03395c927f._comment
@@ -0,0 +1,12 @@
+[[!comment format=mdwn
+ username="oliverconzen@8cff8c5ab3868de177f748566f90c79720f9cf4b"
+ nickname="oliverconzen"
+ subject="comment 2"
+ date="2016-01-08T19:21:28Z"
+ content="""
+After revisiting this issue the guys over at Homebrew have no idea whats going on apparently. The build process seems to be fairly complicated although obviously automated by cabal. The feature just seemed to went missing and nobody has an idea where to search. I am currently compiling from source and fairly confident that I will get it working.
+
+Since not everyone has the time to deal with this in their freetime, and brew is pretty much done to to this kind of thing automagically, maybe it is worth checking with them what went wrong.
+
+ reveals two issues about the webapp and S3 features disappearing in between updates with no changes to the formula.
+"""]]
diff --git a/doc/forum/Confusion_about_Webapp___40__on_OSX__41__/comment_3_7d790a158b7c22206b0b523a4227bacb._comment b/doc/forum/Confusion_about_Webapp___40__on_OSX__41__/comment_3_7d790a158b7c22206b0b523a4227bacb._comment
new file mode 100644
index 0000000000..72643f0232
--- /dev/null
+++ b/doc/forum/Confusion_about_Webapp___40__on_OSX__41__/comment_3_7d790a158b7c22206b0b523a4227bacb._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="joey"
+ subject="""comment 3"""
+ date="2016-01-11T16:32:47Z"
+ content="""
+Seems to have been resolved in brew by forcing the Webapp flag, which I
+think is the right solution.
+"""]]
diff --git a/doc/forum/Consistency_Check_for_S3.mdwn b/doc/forum/Consistency_Check_for_S3.mdwn
new file mode 100644
index 0000000000..0c06f86090
--- /dev/null
+++ b/doc/forum/Consistency_Check_for_S3.mdwn
@@ -0,0 +1 @@
+Wepapp only allows me to setup consistency checks for the local repo. Non of my computers has full copy of repos. How can I setup annex so that every night for one hour it checks consistency of s3 data?
diff --git a/doc/forum/Consistency_Check_for_S3/comment_1_40385806ef1cc082232cd2723a24be1a._comment b/doc/forum/Consistency_Check_for_S3/comment_1_40385806ef1cc082232cd2723a24be1a._comment
new file mode 100644
index 0000000000..a641ded34d
--- /dev/null
+++ b/doc/forum/Consistency_Check_for_S3/comment_1_40385806ef1cc082232cd2723a24be1a._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="http://joeyh.name/"
+ ip="209.250.56.227"
+ subject="comment 1"
+ date="2013-12-31T19:30:00Z"
+ content="""
+The webapp does not let you set up a consistency check for data in a S3 remote because the only way to check it would be to download the data, which is expensive in bandwidth and potentially, Amazon S3 fees.
+"""]]
diff --git a/doc/forum/Consistency_Check_for_S3/comment_2_ebfe40d9f777c9c0a83c44afd0f5802d._comment b/doc/forum/Consistency_Check_for_S3/comment_2_ebfe40d9f777c9c0a83c44afd0f5802d._comment
new file mode 100644
index 0000000000..d78a1dc52d
--- /dev/null
+++ b/doc/forum/Consistency_Check_for_S3/comment_2_ebfe40d9f777c9c0a83c44afd0f5802d._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="https://me.yahoo.com/a/FHnTlSBo1eCGJRwueeKeB6.RCaPbGMPr5jxx8A--#ce0d8"
+ nickname="Hamza"
+ subject="comment 2"
+ date="2014-01-01T01:50:57Z"
+ content="""
+Is it possible to do it from command line? I was thinking with a time limit I would spread the fsck over a mount run it once a while to make sure all data is actually there when I need it. 
+"""]]
diff --git a/doc/forum/Consistency_Check_for_S3/comment_3_ec9de6882a0eef4d2786e55b583ad020._comment b/doc/forum/Consistency_Check_for_S3/comment_3_ec9de6882a0eef4d2786e55b583ad020._comment
new file mode 100644
index 0000000000..e0e68fe6df
--- /dev/null
+++ b/doc/forum/Consistency_Check_for_S3/comment_3_ec9de6882a0eef4d2786e55b583ad020._comment
@@ -0,0 +1,10 @@
+[[!comment format=mdwn
+ username="http://joeyh.name/"
+ ip="209.250.56.227"
+ subject="comment 3"
+ date="2014-01-01T21:05:20Z"
+ content="""
+Adapting an example from the man page:
+
+`git annex fsck --incremental-schedule 30d --time-limit 1h --from S3`
+"""]]
diff --git a/doc/forum/Content_branch.mdwn b/doc/forum/Content_branch.mdwn
new file mode 100644
index 0000000000..6eb268eade
--- /dev/null
+++ b/doc/forum/Content_branch.mdwn
@@ -0,0 +1,5 @@
+I was not able to find a convenient way to _only_ see locally available files.
+
+Is there a command to create a branch that only contain files taht are present locally, or is there another way to get a "locally available view" ?
+
+Thanks. 
diff --git a/doc/forum/Content_branch/comment_1_564724cb1508524700b71950b81a5058._comment b/doc/forum/Content_branch/comment_1_564724cb1508524700b71950b81a5058._comment
new file mode 100644
index 0000000000..e843b343fc
--- /dev/null
+++ b/doc/forum/Content_branch/comment_1_564724cb1508524700b71950b81a5058._comment
@@ -0,0 +1,10 @@
+[[!comment format=mdwn
+ username="jd.schroeder@0c8632a8f2bffdd4b0de05a0a3660f32acdfeeca"
+ nickname="jd.schroeder"
+ subject="comment 1"
+ date="2016-07-29T19:12:24Z"
+ content="""
+I'm not sure about creating a branch, but I use [[https://git-annex.branchable.com/git-annex-find/]] to list locally available files.
+
+Good luck!
+"""]]
diff --git a/doc/forum/Controlling_content_on_mobile_device.mdwn b/doc/forum/Controlling_content_on_mobile_device.mdwn
new file mode 100644
index 0000000000..b1f145ccb2
--- /dev/null
+++ b/doc/forum/Controlling_content_on_mobile_device.mdwn
@@ -0,0 +1,46 @@
+I have some questions about how to best manage content on mobile
+devices.  This is probably more of a wishlist discussion, although I
+haven't had enough experience with the assistant and preferred content
+features enough to know how much of this might already be possible.
+
+(While I did donate to both campaigns, I also didn't donate at the
+"personal git-annex assistant" level, so I also know not to expect
+personal undivided attention :) . Any tips would be appreciated.)
+
+In short, I don't think git-annex really fits with how I'd like to use
+it on my Android phone.  My current setup is as follows:
+
+- I have a large repository with various media files.  This is stored
+  on several computers, at least one of which generally has a copy of
+  everything.  I use a centralized bare repo to automatically
+  synchronize between computers with "`git annex sync origin`" using
+  cron, and manually control what is on each one using "`copy --to
+  host`", "`get`", "`drop`", etc.  It is *awesome*.
+
+What I would like for Android is:
+
+- The phone keeps in sync with the origin repository.
+
+- On the phone, I'd like to browse through and choose content to get
+  or drop.  I could use the command-line "`get`" and "`drop`" for
+  this, of course, but being able to do it through the webapp would be
+  better, as typing on a phone is no fun.
+
+- From _off_ the phone, I'd like to be able to choose which content
+  should be _on_ the phone.  For example, using the command line or
+  webapp on my desktop, I'd like to be able to choose media files that
+  should be downloaded to (or removed from) the phone.  However, I
+  don't want to have to move files between directories in the repository
+  to achieve this.
+
+- The only useful content that gets created on the phone is photos and
+  videos.  Those should get uploaded to the main server, but only the
+  most recent files should remain on the phone by default (say,
+  anything older than 2 weeks gets dropped).  Both on the phone, and
+  off the phone, I'd be able to override that using the previously
+  described features, so that certain photos remain on the phone
+  forever, or get dropped earlier.
+
+Thoughts?  I feel like git-annex is close to being able to do this.
+
+-jim
diff --git a/doc/forum/Controlling_content_on_mobile_device/comment_1_708649b7f30d8619d7b34dcb0ef46515._comment b/doc/forum/Controlling_content_on_mobile_device/comment_1_708649b7f30d8619d7b34dcb0ef46515._comment
new file mode 100644
index 0000000000..c9cbc9e79c
--- /dev/null
+++ b/doc/forum/Controlling_content_on_mobile_device/comment_1_708649b7f30d8619d7b34dcb0ef46515._comment
@@ -0,0 +1,12 @@
+[[!comment format=mdwn
+ username="https://www.google.com/accounts/o8/id?id=AItOawl_EUsflI4UOGtx0-2yrRJuGAzNyN_5UYY"
+ nickname="Chris"
+ subject="comment 1"
+ date="2014-01-03T16:33:51Z"
+ content="""
+Jim,
+
+I have nothing in terms of a solution, but wanted to say thanks for writing this up.  This describes precisely the use case I am also seeking on my Android phone.
+
+-- Chris
+"""]]
diff --git a/doc/forum/Controlling_content_on_mobile_device/comment_2_dba1a1b0917332a1dee387b1183bd2cb._comment b/doc/forum/Controlling_content_on_mobile_device/comment_2_dba1a1b0917332a1dee387b1183bd2cb._comment
new file mode 100644
index 0000000000..60aea0bdda
--- /dev/null
+++ b/doc/forum/Controlling_content_on_mobile_device/comment_2_dba1a1b0917332a1dee387b1183bd2cb._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="https://www.google.com/accounts/o8/id?id=AItOawnBJ6Dv1glxzzi4qIzGFNa6F-mfHIvv9Ck"
+ nickname="Jim"
+ subject="Metadata?"
+ date="2014-02-21T18:44:10Z"
+ content="""
+I suspect the new [metadata](http://git-annex.branchable.com/design/metadata/) support could be used to help implement this.
+"""]]
diff --git a/doc/forum/Convert_regular_git-annex_repo_to_a_rsync_repo.mdwn b/doc/forum/Convert_regular_git-annex_repo_to_a_rsync_repo.mdwn
new file mode 100644
index 0000000000..c941725356
--- /dev/null
+++ b/doc/forum/Convert_regular_git-annex_repo_to_a_rsync_repo.mdwn
@@ -0,0 +1 @@
+Is it possible to convert a regular git annex repo (git clone then git annex init in the folder), to an rsync remote. I have an annex with alot of remotes which makes the sync operation take a really long time. I would like to convert some of those remotes to rsync. This particular repo has a TB of data so I would like to avoid dropping content from the remote than re download everything.
diff --git a/doc/forum/Convert_regular_git-annex_repo_to_a_rsync_repo/comment_1_e6065f9c44c85030c7628e2cfa0fd0fa._comment b/doc/forum/Convert_regular_git-annex_repo_to_a_rsync_repo/comment_1_e6065f9c44c85030c7628e2cfa0fd0fa._comment
new file mode 100644
index 0000000000..46ba67c302
--- /dev/null
+++ b/doc/forum/Convert_regular_git-annex_repo_to_a_rsync_repo/comment_1_e6065f9c44c85030c7628e2cfa0fd0fa._comment
@@ -0,0 +1,12 @@
+[[!comment format=mdwn
+ username="http://joeyh.name/"
+ ip="209.250.56.172"
+ subject="comment 1"
+ date="2014-02-23T18:57:42Z"
+ content="""
+This is doable. It works best if the remote repo is a bare git repository, because then the filenames line up 100% with the filenames used in a rsync special remote. If the git repo is not bare, the rsync special remote will first try the paths it expects, and only then fall back to the right paths, so a little extra work done. (If this became a big problem, it would not be infesable to move the files around with a script.)
+
+Anyway, if it's a bare repo, then repo.git/annex/objects is where you want to point the rsync special remote at. With a non-bare repo, repo/.git/annex/objects/ is the location. I'd recommend moving the objects directory out to a new location, and pointing the rsyncurl at that. This way, there's no possibility of git-annex thinking one files accessed 2 ways is 2 copies.
+
+Of course, you can't use encryption for the rsync special remote.
+"""]]
diff --git a/doc/forum/Convert_regular_git-annex_repo_to_a_rsync_repo/comment_2_76bfb11396dc20a5105376b22e7e773b._comment b/doc/forum/Convert_regular_git-annex_repo_to_a_rsync_repo/comment_2_76bfb11396dc20a5105376b22e7e773b._comment
new file mode 100644
index 0000000000..8ed4f65081
--- /dev/null
+++ b/doc/forum/Convert_regular_git-annex_repo_to_a_rsync_repo/comment_2_76bfb11396dc20a5105376b22e7e773b._comment
@@ -0,0 +1,10 @@
+[[!comment format=mdwn
+ username="http://joeyh.name/"
+ ip="209.250.56.172"
+ subject="comment 2"
+ date="2014-02-23T19:07:59Z"
+ content="""
+However, if the only problem is that pushing and pulling with a git repository makes `git annex sync` take too long, another option is setting `git config remote.$foo.annex-sync false`. You can still then use git-annex commands to get and push data to the remote, and can even `git annex sync $foo` from time to time, but it won't slow down the normal `git annex sync`.
+
+However, this also prevents the assistant from uploading new files to the remote automatically.
+"""]]
diff --git a/doc/forum/Convert_regular_git-annex_repo_to_a_rsync_repo/comment_3_b34d6ae0718ab0ff6bc1d7b8f2470b9b._comment b/doc/forum/Convert_regular_git-annex_repo_to_a_rsync_repo/comment_3_b34d6ae0718ab0ff6bc1d7b8f2470b9b._comment
new file mode 100644
index 0000000000..a8040cd3fe
--- /dev/null
+++ b/doc/forum/Convert_regular_git-annex_repo_to_a_rsync_repo/comment_3_b34d6ae0718ab0ff6bc1d7b8f2470b9b._comment
@@ -0,0 +1,16 @@
+[[!comment format=mdwn
+ username="https://me.yahoo.com/a/FHnTlSBo1eCGJRwueeKeB6.RCaPbGMPr5jxx8A--#ce0d8"
+ nickname="Hamza"
+ subject="comment 3"
+ date="2014-02-23T19:39:28Z"
+ content="""
+Thanks for the reply, just to make sure I got you right,
+
+It is indeed a non bare git repo. So I will move the folder repo/.git/annex/objects/ to repo/
+
+then run,
+
+git annex initremote myrsync type=rsync rsyncurl=ssh.example.com:~/repo
+
+and enable the remote on other annexes (disks are connected to an ssh server there is no encryption setup right now so I do not mind not having it.). And everything should be setup correctly.
+"""]]
diff --git a/doc/forum/Convert_regular_git-annex_repo_to_a_rsync_repo/comment_4_8f5e323b29745591f9f2f0f867353f69._comment b/doc/forum/Convert_regular_git-annex_repo_to_a_rsync_repo/comment_4_8f5e323b29745591f9f2f0f867353f69._comment
new file mode 100644
index 0000000000..c8305aaa92
--- /dev/null
+++ b/doc/forum/Convert_regular_git-annex_repo_to_a_rsync_repo/comment_4_8f5e323b29745591f9f2f0f867353f69._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="https://me.yahoo.com/a/FHnTlSBo1eCGJRwueeKeB6.RCaPbGMPr5jxx8A--#ce0d8"
+ nickname="Hamza"
+ subject="comment 4"
+ date="2014-02-23T19:45:09Z"
+ content="""
+and as a follow up do I have rename the repos or can I reuse the same names for the repos?
+"""]]
diff --git a/doc/forum/Convert_regular_git-annex_repo_to_a_rsync_repo/comment_5_9824c953694770afa0611ff7276737bf._comment b/doc/forum/Convert_regular_git-annex_repo_to_a_rsync_repo/comment_5_9824c953694770afa0611ff7276737bf._comment
new file mode 100644
index 0000000000..c6052232b8
--- /dev/null
+++ b/doc/forum/Convert_regular_git-annex_repo_to_a_rsync_repo/comment_5_9824c953694770afa0611ff7276737bf._comment
@@ -0,0 +1,12 @@
+[[!comment format=mdwn
+ username="http://joeyh.name/"
+ ip="209.250.56.172"
+ subject="comment 5"
+ date="2014-02-24T19:07:33Z"
+ content="""
+That looks all-right, although initremote will ask you to tell it what encryption to use, and you'll need to specify `encryption=none`
+
+One thing I forgot to mention is that the UUID of the new rsync repository won't be the same, so git-annex won't know about the files in there. This can be fixed by `git annex fsck --fast --from myrsync`. Which doesn't re-download all the files, but you still may want to run it on a repository close to or on the server for speed.
+
+You can re-use the name you're currently using for the git remote for the new rsync special remote if you like.
+"""]]
diff --git a/doc/forum/Convert_regular_git-annex_repo_to_a_rsync_repo/comment_6_5899741cb7f83e1b22c5ee3509c5ff21._comment b/doc/forum/Convert_regular_git-annex_repo_to_a_rsync_repo/comment_6_5899741cb7f83e1b22c5ee3509c5ff21._comment
new file mode 100644
index 0000000000..8b7c3b52e1
--- /dev/null
+++ b/doc/forum/Convert_regular_git-annex_repo_to_a_rsync_repo/comment_6_5899741cb7f83e1b22c5ee3509c5ff21._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="https://me.yahoo.com/a/FHnTlSBo1eCGJRwueeKeB6.RCaPbGMPr5jxx8A--#ce0d8"
+ nickname="Hamza"
+ subject="comment 6"
+ date="2014-02-25T09:34:16Z"
+ content="""
+assuming the remote I am converting is called some-repo should mark it as dead before converting and reinitting as rsync some-repo again?
+"""]]
diff --git a/doc/forum/Copy_contents_of_directory_to_rsync_remotes_possible__63__.mdwn b/doc/forum/Copy_contents_of_directory_to_rsync_remotes_possible__63__.mdwn
new file mode 100644
index 0000000000..abc2f8f263
--- /dev/null
+++ b/doc/forum/Copy_contents_of_directory_to_rsync_remotes_possible__63__.mdwn
@@ -0,0 +1,16 @@
+I'm having trouble sharing an rsync remote between two users (as a "transfer" repository).  `git-annex` thinks `rsync` fails because it can't set the permissions on the root directory on the destination.  I've added `--perms --chmod=Dug+rwxs,Fug+rw,o-rwx` to `annex-rsync-options` so all users in the specified group on the destination can create and delete files.  However, I see in the logs errors like:
+
+    sending incremental file list
+    ./
+    rsync: failed to set permissions on "/home/shared-xfer/.": Operation not permitted (1)
+    2fe/985/GPGHMACSHA1--b36ce0a18718e7ea7cf9827c2748608df7152dfc/GPGHMACSHA1--b36ce0a18718e7ea7cf9827c2748608df7152dfc
+    
+             32,768   0%    0.00kB/s    0:00:00  
+          3,359,405 100%    4.78MB/s    0:00:00 (xfr#1, to-chk=0/5)
+    rsync error: some files/attrs were not transferred (see previous errors) (code 23) at main.c(1165) [sender=3.1.1]
+    [2014-10-18 15:16:24 EDT] chat: gpg ["--batch","--no-tty","--use-agent","--quiet","--trust-model","always","--batch","--passphrase-fd","22","--symmetric","--force-mdc","--no-textmode"]
+    [2014-10-18 15:16:24 EDT] read: rsync ["-e","'ssh' '-S' '.git/annex/ssh/blah@blah' '-o' 'ControlMaster=auto' '-o' 'ControlPersist=yes' '-l' 'blah' '-T'","--perms","--chmod=Dug+rwxs,Fug+rw,o-rwx","--progress","--recursive","--partial","--partial-dir=.rsync-partial","/blah/.git/annex/tmp/rsynctmp/24863/","blah@blah:/home/shared-xfer/"]
+
+      rsync failed -- run git annex again to resume file transfer
+
+The transfer actually completes fine, except for setting permissions of the root directory on the destination (which I don't really care about, because I have manually set them to be correct).  Is it possible for `git-annex` to copy the contents of the directory to rsync remotes?  For the example above, instead of `/blah/.git/annex/tmp/rsynctmp/24863/`, `/blah/.git/annex/tmp/rsynctmp/24863/2fe` would work.  I can't come up with an alternate solution for this server, except using the same remote user (which I'd prefer not to do).
diff --git a/doc/forum/Copy_contents_of_directory_to_rsync_remotes_possible__63__/comment_1_6a598b2a0fc6c241aa051f663dca9842._comment b/doc/forum/Copy_contents_of_directory_to_rsync_remotes_possible__63__/comment_1_6a598b2a0fc6c241aa051f663dca9842._comment
new file mode 100644
index 0000000000..6e4186622b
--- /dev/null
+++ b/doc/forum/Copy_contents_of_directory_to_rsync_remotes_possible__63__/comment_1_6a598b2a0fc6c241aa051f663dca9842._comment
@@ -0,0 +1,11 @@
+[[!comment format=mdwn
+ username="joey"
+ subject="""comment 1"""
+ date="2014-10-28T19:26:01Z"
+ content="""
+git-annex has to tell rsync to transfer the "24863" directory,
+otherwise rsync would not create that directory, and would then
+fail to create "24863/2fe" since its parent directory doesn't exist.
+
+Might be able to use use ACLS, I suppose, but they bring a lot of complexity.
+"""]]
diff --git a/doc/forum/Copy_contents_of_directory_to_rsync_remotes_possible__63__/comment_3_09b7f1dd1fa4589b3fe0be0086849bf3._comment b/doc/forum/Copy_contents_of_directory_to_rsync_remotes_possible__63__/comment_3_09b7f1dd1fa4589b3fe0be0086849bf3._comment
new file mode 100644
index 0000000000..1d6e912bc9
--- /dev/null
+++ b/doc/forum/Copy_contents_of_directory_to_rsync_remotes_possible__63__/comment_3_09b7f1dd1fa4589b3fe0be0086849bf3._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="https://www.google.com/accounts/o8/id?id=AItOawlVB7wf1TFFrNPvM62fg78X-simVV-R1Cg"
+ nickname="Joe"
+ subject="comment 3"
+ date="2014-11-03T02:21:32Z"
+ content="""
+I was thinking that a call to `rsync /blah/.git/annex/tmp/rsynctmp/24863/2fe blah@blah:/home/shared-xfer/24863/` would work just as well, creating the necessary directories.  But this requires knowing all the subdirectories of `24863`.  Not sure if `git-annex` knows this.
+"""]]
diff --git a/doc/forum/Copying_and_dropping_right_after.mdwn b/doc/forum/Copying_and_dropping_right_after.mdwn
new file mode 100644
index 0000000000..65680c0629
--- /dev/null
+++ b/doc/forum/Copying_and_dropping_right_after.mdwn
@@ -0,0 +1,18 @@
+Hello,
+
+I have three repos horus (client, this one), astarte (transfer, remote) and external_2tb (client, connected usb). On a git annex sync --content it first copies the content to external and astarte and then dropping the content from astarte right after. 
+
+    copy FLAC/Corvus Corax - Cantus Buranus/Corvus Corax - 08 - Sol solo.flac copy FLAC/Corvus Corax - Cantus Buranus/Corvus Corax - 08 - Sol solo.flac (to external_2TB...) 
+    Corvus Corax - 08 - Sol solo.flac
+         33,541,833 100%  257.72MB/s    0:00:00 (xfr#1, to-chk=0/1)
+    ok
+    copy FLAC/Corvus Corax - Cantus Buranus/Corvus Corax - 08 - Sol solo.flac copy FLAC/Corvus Corax - Cantus Buranus/Corvus Corax - 08 - Sol solo.flac (checking astarte...) (to astarte...) 
+    Corvus Corax - 08 - Sol solo.flac
+        33,541,833 100%  437.82kB/s    0:01:14 (xfr#1, to-chk=0/1)
+    ok
+    drop astarte FLAC/Corvus Corax - Cantus Buranus/Corvus Corax - 08 - Sol solo.flac ok
+
+Why copying it anyway?
+
+Regards,
+Florian
diff --git a/doc/forum/Copying_and_dropping_right_after/comment_1_87d81911964c2628ccb6156b0b631aa0._comment b/doc/forum/Copying_and_dropping_right_after/comment_1_87d81911964c2628ccb6156b0b631aa0._comment
new file mode 100644
index 0000000000..dbe357d66e
--- /dev/null
+++ b/doc/forum/Copying_and_dropping_right_after/comment_1_87d81911964c2628ccb6156b0b631aa0._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="joey"
+ subject="""comment 1"""
+ date="2015-01-05T20:31:10Z"
+ content="""
+This must have something to do with the preferred content settings of
+astarte. Can you show the output of this command: `git annex wanted astarte`
+"""]]
diff --git a/doc/forum/Copying_and_dropping_right_after/comment_2_87a14d14014aefc33f811bb15ea27e13._comment b/doc/forum/Copying_and_dropping_right_after/comment_2_87a14d14014aefc33f811bb15ea27e13._comment
new file mode 100644
index 0000000000..b58b61a757
--- /dev/null
+++ b/doc/forum/Copying_and_dropping_right_after/comment_2_87a14d14014aefc33f811bb15ea27e13._comment
@@ -0,0 +1,15 @@
+[[!comment format=mdwn
+ username="https://www.google.com/accounts/o8/id?id=AItOawk9SYh6N-JUMkYkW4aOk55zC3Vr9KonDV4"
+ nickname="Florian"
+ subject="comment 2"
+ date="2015-01-14T22:01:24Z"
+ content="""
+Sorry for the late reply. I had redone my git-annex repos, but I just had exactly this behaviour again
+
+    % git annex wanted astarte
+    standard 
+    % git annex group astarte 
+    transfer
+
+Best Regards...
+"""]]
diff --git a/doc/forum/Copying_to_S3_does_not_work_-_chunking_does_not_work.mdwn b/doc/forum/Copying_to_S3_does_not_work_-_chunking_does_not_work.mdwn
new file mode 100644
index 0000000000..2865441e78
--- /dev/null
+++ b/doc/forum/Copying_to_S3_does_not_work_-_chunking_does_not_work.mdwn
@@ -0,0 +1,47 @@
+Posting this here because I am not sure if it is a bug or if I am missing something.
+
+I have a ~10 GB file in git-annex. I can't get it to go to S3 whatever I do.
+
+I added a S3 special remote:
+
+    git annex initremote s3-mybucket type=S3 chunk=1MiB keyid=ABCD1234 bucket=mybucket
+
+Then, I tried copying files to the remote.  Small files worked, but big files don't:
+
+    $ git annex copy bigfile.tgz --to s3-mybucket
+    copy bigfile.tgz (gpg)
+    You need a passphrase to unlock the secret key for
+    user: "user"
+    2048-bit RSA key, ID ABCD1234, created 2014-10-13 (main key ID ABCD1234)
+
+    (checking s3-mybucket...) (to s3-mybucket...)
+
+      Your proposed upload exceeds the maximum allowed size
+    failed
+    git-annex: copy: 1 failed
+
+I tried some stuff like this too:
+
+    git annex enableremote s3-mybucket chunk=100MiB
+    git annex enableremote s3-mybucket chunksize=100MiB
+
+It didn't work. Same result.
+
+    $ git annex version
+    git-annex version: 5.20140717
+    build flags: Assistant Webapp Webapp-secure Pairing Testsuite S3 WebDAV FsEvents XMPP DNS Feeds Quvi TDFA CryptoHash
+    key/value backends: SHA256E SHA1E SHA512E SHA224E SHA384E SKEIN256E SKEIN512E SHA256 SHA1 SHA512 SHA224 SHA384 SKEIN256 SKEIN512 WORM URL
+    remote types: git gcrypt S3 bup directory rsync web webdav tahoe glacier ddar hook external
+    local repository version: 5
+    supported repository version: 5
+    upgrade supported from repository versions: 0 1 2 4
+
+The chunk size did seem to be set properly:
+
+    $ git checkout git-annex
+    $ cat remote.log
+    xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxx bucket=mybucket chunk=100MiB chunksize=100MiB cipher=....
+
+I'm on OSX 10.9.4 and I installed git-annex via homebrew.
+
+Any ideas?
diff --git a/doc/forum/Copying_to_S3_does_not_work_-_chunking_does_not_work/comment_1_ec390a7d521c697eb6b17e8db1dc9d1d._comment b/doc/forum/Copying_to_S3_does_not_work_-_chunking_does_not_work/comment_1_ec390a7d521c697eb6b17e8db1dc9d1d._comment
new file mode 100644
index 0000000000..7b8a0e1f5d
--- /dev/null
+++ b/doc/forum/Copying_to_S3_does_not_work_-_chunking_does_not_work/comment_1_ec390a7d521c697eb6b17e8db1dc9d1d._comment
@@ -0,0 +1,10 @@
+[[!comment format=mdwn
+ username="http://joeyh.name/"
+ ip="209.250.56.111"
+ subject="comment 1"
+ date="2014-10-15T15:25:14Z"
+ content="""
+You're using an old version of git-annex from before the recent complete rewrite of the chunking code. That old version did not support chunking for S3. QED. Upgrade.
+
+Note that your configuration has both the chunksize= and chunk= set. This is not a good idea, since they enable different types of chunking that are not compatible. If I were you, I'd delete that special remote and make a new one after upgrading, and be careful to only set `chunk=` in that new one.
+"""]]
diff --git a/doc/forum/Copying_to_S3_does_not_work_-_chunking_does_not_work/comment_2_14a584567ef42d5b7955ee970200e74d._comment b/doc/forum/Copying_to_S3_does_not_work_-_chunking_does_not_work/comment_2_14a584567ef42d5b7955ee970200e74d._comment
new file mode 100644
index 0000000000..744b9f182c
--- /dev/null
+++ b/doc/forum/Copying_to_S3_does_not_work_-_chunking_does_not_work/comment_2_14a584567ef42d5b7955ee970200e74d._comment
@@ -0,0 +1,10 @@
+[[!comment format=mdwn
+ username="http://digiuser.livejournal.com/"
+ ip="67.161.4.185"
+ subject="Homebrew"
+ date="2014-10-15T15:37:34Z"
+ content="""
+Oh, ok, thank you. I am using the latest version from homebrew. Is there a way to get homebrew to install the latest version?
+
+Thanks.
+"""]]
diff --git a/doc/forum/Copying_to_S3_does_not_work_-_chunking_does_not_work/comment_3_6cbd7329f1f11edf8dd90df27d45158f._comment b/doc/forum/Copying_to_S3_does_not_work_-_chunking_does_not_work/comment_3_6cbd7329f1f11edf8dd90df27d45158f._comment
new file mode 100644
index 0000000000..ecba0fc838
--- /dev/null
+++ b/doc/forum/Copying_to_S3_does_not_work_-_chunking_does_not_work/comment_3_6cbd7329f1f11edf8dd90df27d45158f._comment
@@ -0,0 +1,10 @@
+[[!comment format=mdwn
+ username="http://joeyh.name/"
+ ip="209.250.56.111"
+ subject="comment 3"
+ date="2014-10-15T17:25:27Z"
+ content="""
+I think that if you have installed all the build deps for git-annex using homebrew, you should be able to just install git-annex from source.
+
+Or, ask the homebrewers to update the package there..
+"""]]
diff --git a/doc/forum/Correct_way_to_use_somewhat_unreliable_private_server_and_box.com.mdwn b/doc/forum/Correct_way_to_use_somewhat_unreliable_private_server_and_box.com.mdwn
new file mode 100644
index 0000000000..0aa122203f
--- /dev/null
+++ b/doc/forum/Correct_way_to_use_somewhat_unreliable_private_server_and_box.com.mdwn
@@ -0,0 +1,3 @@
+I have a private server (a computer at work) which I use as the "central repository" with git-annex. Unfortunately it sometimes goes down and occasionally I can't easilly put it on again (think holidays). So I'd like to use box.com as a backup.
+
+What's the most efficient way to do it? Simply add both my private serer and box.com as "transfer: distributes files to clients" repositiories? I'm afraid that if I do that then box.com will slow things down for me (I haven't tried it yet because I'm literally starting with git-annex today). Ideally I'd like all transfers to/from box.com to be somehow in the background so that they don't interfere with the transfers to/from my private server.
diff --git a/doc/forum/Correct_way_to_use_somewhat_unreliable_private_server_and_box.com/comment_1_9c357a4a99930936124a58ffe105f87f._comment b/doc/forum/Correct_way_to_use_somewhat_unreliable_private_server_and_box.com/comment_1_9c357a4a99930936124a58ffe105f87f._comment
new file mode 100644
index 0000000000..e159e2e97e
--- /dev/null
+++ b/doc/forum/Correct_way_to_use_somewhat_unreliable_private_server_and_box.com/comment_1_9c357a4a99930936124a58ffe105f87f._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="https://www.google.com/accounts/o8/id?id=AItOawnmF_9CAtfqdZkC4e-_dCX-rK5bqh4RWkw"
+ nickname="Carl"
+ subject="Cost"
+ date="2015-09-26T08:59:39Z"
+ content="""
+If I understand you correctly, the concept of \"cost\" is exactly what you need. You give the box remote a higher cost, and thus you will prefer to download from the remote with a lower cost, when available.
+"""]]
diff --git a/doc/forum/Correct_way_to_use_somewhat_unreliable_private_server_and_box.com/comment_2_b4aec96cbcd8752f273ecd810400acef._comment b/doc/forum/Correct_way_to_use_somewhat_unreliable_private_server_and_box.com/comment_2_b4aec96cbcd8752f273ecd810400acef._comment
new file mode 100644
index 0000000000..cac39d7bfb
--- /dev/null
+++ b/doc/forum/Correct_way_to_use_somewhat_unreliable_private_server_and_box.com/comment_2_b4aec96cbcd8752f273ecd810400acef._comment
@@ -0,0 +1,41 @@
+[[!comment format=mdwn
+ username="joey"
+ subject="""comment 2"""
+ date="2015-09-29T15:42:46Z"
+ content="""
+I'm going to answer this both for the webapp and for command line usage.
+
+webapp:
+
+Drag and drop the box.com remote to be below the remote for your private
+server. This will adjust the cost settings, and let git-annex know it
+should try to get files from your private server first.
+
+Click on "syncing enabled" at any time to toggle it to disabled.
+You could choose to leave box.com disabled most of the time, to avoid files
+being uploaded to it etc. When your private server is not available, you
+can toggle box.com back to enabled to use it temporarily.
+
+command line:
+
+Let's assume the box.com remote is named "box".
+
+You can use `remote.box.annex-cost` in git config, to give that remote a
+higher use-cost than your private server's remote. git-annex will then know
+you prefer using your private server when possible, and so will prefer to
+use it when eg, downloading files. (Same as dragging and dropping in the
+webapp.)
+
+You could configure a cron job (or run the git-annex assistant) on the
+server to send the files on to box.com. Just `git annex enableremote box`
+on the server so it knows how to use that remote. This way, you don't
+need to upload the same file twice to both box.com and your server over
+your home network link, but files will end up in both locations.
+
+Uploading files with `git annex copy --to` can send them to your private
+server. Or, if you want to use `git annex sync --content` or the assistant,
+you could set `remote.box.annex-sync` to false for the box.com
+remote, to prevent sending content to it. When your private server is
+unavailable, you could override that setting temporarily and sync with
+box.com by running `git annex sync --content box`
+"""]]
diff --git a/doc/forum/Corrupt_Repository_Invalid_Object.mdwn b/doc/forum/Corrupt_Repository_Invalid_Object.mdwn
new file mode 100644
index 0000000000..af6d8e3538
--- /dev/null
+++ b/doc/forum/Corrupt_Repository_Invalid_Object.mdwn
@@ -0,0 +1,10 @@
+One of my repositories got corrupted. I am not exactly sure how it happened (was running a series of commands) but I think I accidentally ran regular mv instead of git mv. To fix it I deleted the moved file then checkout the original link however this did not fixed the problem. I ended up with a corrupted repo. Now running any command ends with the following error,
+
+    ga sync
+    (merging origin/git-annex origin/synced/git-annex into git-annex...)
+    (Recording state in git...)
+    error: invalid object 040000 6ad564920e3d78d31c9456f5be3869a0319f9f08 for'3fd/d44'                                                                                
+    fatal: git-write-tree: error building trees
+    git-annex: failed to read sha from git write-tree
+
+Was wondering how to fix this? I did run git fsck and git annex fsck but non fixed the problem.
diff --git a/doc/forum/Corrupt_Repository_Invalid_Object/comment_1_b7fd4b6212b50400342931e70684b96c._comment b/doc/forum/Corrupt_Repository_Invalid_Object/comment_1_b7fd4b6212b50400342931e70684b96c._comment
new file mode 100644
index 0000000000..a62e6d345c
--- /dev/null
+++ b/doc/forum/Corrupt_Repository_Invalid_Object/comment_1_b7fd4b6212b50400342931e70684b96c._comment
@@ -0,0 +1,14 @@
+[[!comment format=mdwn
+ username="http://joeyh.name/"
+ ip="209.250.56.191"
+ subject="comment 1"
+ date="2014-04-17T19:47:54Z"
+ content="""
+git is complaining about one of the files in `.git/objects` being missing or corrupt.
+
+It's not likely that some accidental command you ran caused this. More likely a disk error or an unclean shutdown could have left the repository in this state.
+
+You can try running `git annex repair` which should be able to repair git repositories with this kind of damage.
+
+Or you can just `git clone` the repository from any other place you have it and move over .git/annex/objects and .git/config to the new repository, discarding the damaged one.
+"""]]
diff --git a/doc/forum/Corrupted_repository__44___can_not_be_repaired.mdwn b/doc/forum/Corrupted_repository__44___can_not_be_repaired.mdwn
new file mode 100644
index 0000000000..e2b4357bf9
--- /dev/null
+++ b/doc/forum/Corrupted_repository__44___can_not_be_repaired.mdwn
@@ -0,0 +1,40 @@
+Hey, I hope somebody can help me:
+
+I recently moved a bunch of files to another location shortly after I renamed them. The assistant was running (repo: indirect mode) and I guess the assistant was confused about whether to commit the rename changes or to fix the symlinks because of the location move (and to commit these changes). I don't know much about the internal design, but after that my repository was kinda broken. Every git command failed and also "git fsck" gave me some error messages. However, I mounted my external drive (backup repo) and tried a "git annex repair". It took quite a while, but it could fix the repo....somehow. This is the current status:
+
+    ~/private [master●] » git annex info
+    repository mode: indirect
+    trusted repositories: error: refs/remotes/extern/git-annex does not point to a valid object!
+    error: refs/remotes/extern/git-annex does not point to a valid object!
+    error: unable to resolve reference refs/heads/git-annex: Datei oder Verzeichnis nicht gefunden // (english: No such file or directory)
+    fatal: Cannot lock the ref 'refs/heads/git-annex'.
+    git-annex: git [Param "update-ref",Param "refs/heads/git-annex",Param "ea4d4b5e09bfd10c714c24ca76ab5af0625bb6a7"] failed
+
+    ~/private [master●] » git fsck
+    Checking object directories: 100% (256/256), done.
+    Checking objects: 100% (654755/654755), done.
+    error: refs/remotes/extern/git-annex does not point to a valid object!
+    error: bad ref for refs/heads/git-annex
+    Checking connectivity: 654760, done.
+    dangling blob cb2a012e65b15baf0bc51cb8ac6ab05c4ac4e543
+    dangling blob 6a48216d54be6fbf0539c86501ce9e4567ebf678
+    dangling blob 3f8f21304c6dcfc6ace92093425d7f057138b5e8
+    dangling blob 4943840264ad7c06071f510261151c28b4fb5168
+    dangling blob d67e845b06b17c3ba5b44a454d3128dea6ebe8cc
+    dangling blob 38b7e4ce06a84c03b9a0f7f3a84872b2ed202421
+    dangling blob ca0a65e010c69e02953d63f33c5f2030f793c8ff
+    dangling blob aeba65035f566ec9f18ad2ba71b155b0efd2193a
+    dangling blob d3cea542db4c2904f5c8c6dd98ca3868074e6d36
+    dangling commit fe1026edee5dcc6306f04ebe1a7762b86678db8a
+    [... some more dangling objects ...]
+
+However, git annex repair does not want to repair my repo a second time:
+
+    ~/private [master●] » git annex repair
+    Running git fsck ...
+    No problems found.
+    ok
+My directory seems to be fine, also the data (symlinks) I renamed & moved are also fine. Just the "git-annex" branch seems to be broken now and I don't know how to fix that.
+
+Cheers,
+Stephan
diff --git a/doc/forum/Corrupted_repository__44___can_not_be_repaired/comment_1_297ea9c13096f6360358b3a3e7197c03._comment b/doc/forum/Corrupted_repository__44___can_not_be_repaired/comment_1_297ea9c13096f6360358b3a3e7197c03._comment
new file mode 100644
index 0000000000..63878f19ef
--- /dev/null
+++ b/doc/forum/Corrupted_repository__44___can_not_be_repaired/comment_1_297ea9c13096f6360358b3a3e7197c03._comment
@@ -0,0 +1,37 @@
+[[!comment format=mdwn
+ username="sts"
+ ip="134.147.73.126"
+ subject="comment 1"
+ date="2014-07-18T09:46:01Z"
+ content="""
+I forgot, this might be interesting
+
+    git show ea4d4b5e09bfd10c714c24ca76ab5af0625bb6a7
+    
+    commit ea4d4b5e09bfd10c714c24ca76ab5af0625bb6a7
+    Author: Stephan Schindel 
+    Date:   Fri Jul 18 11:28:07 2014 +0200
+
+        branch created
+
+and
+
+    ~/private [master●] » git annex fsck
+    error: refs/remotes/extern/git-annex does not point to a valid object!
+    error: refs/remotes/extern/git-annex does not point to a valid object!
+    error: unable to resolve reference refs/heads/git-annex: Datei oder Verzeichnis nicht gefunden
+    fatal: Cannot lock the ref 'refs/heads/git-annex'.
+    git-annex: git [Param \"update-ref\",Param \"refs/heads/git-annex\",Param \"199fbe3e7ca1dbce99ffa04e440209eb93446374\"] failed
+
+
+and
+
+    git show 199fbe3e7ca1dbce99ffa04e440209eb93446374
+
+    commit 199fbe3e7ca1dbce99ffa04e440209eb93446374
+    Author: Stephan Schindel 
+    Date:   Fri Jul 18 11:46:29 2014 +0200
+
+        branch created
+
+"""]]
diff --git a/doc/forum/Corrupted_repository__44___can_not_be_repaired/comment_2_c3aabffbe9568cf7acb4018e445f22a5._comment b/doc/forum/Corrupted_repository__44___can_not_be_repaired/comment_2_c3aabffbe9568cf7acb4018e445f22a5._comment
new file mode 100644
index 0000000000..04e11a78cf
--- /dev/null
+++ b/doc/forum/Corrupted_repository__44___can_not_be_repaired/comment_2_c3aabffbe9568cf7acb4018e445f22a5._comment
@@ -0,0 +1,13 @@
+[[!comment format=mdwn
+ username="sts"
+ ip="134.147.73.126"
+ subject="comment 2"
+ date="2014-07-18T11:17:49Z"
+ content="""
+Ok, I could fix this issue by doing a fresh clone of one of my remotes:
+
+    git clone path/to/remote
+    [move objects folder from broken repo .git/annex/objects to new clone]
+    git annex info (look for UUID or DESC)
+    git annex reinit \"DESC\"
+"""]]
diff --git a/doc/forum/Corrupted_repository__44___can_not_be_repaired/comment_3_baee46af6b2dbaf92be406ab22007a85._comment b/doc/forum/Corrupted_repository__44___can_not_be_repaired/comment_3_baee46af6b2dbaf92be406ab22007a85._comment
new file mode 100644
index 0000000000..662e8a2b34
--- /dev/null
+++ b/doc/forum/Corrupted_repository__44___can_not_be_repaired/comment_3_baee46af6b2dbaf92be406ab22007a85._comment
@@ -0,0 +1,12 @@
+[[!comment format=mdwn
+ username="http://joeyh.name/"
+ ip="108.236.230.124"
+ subject="comment 3"
+ date="2014-07-21T22:57:32Z"
+ content="""
+There was a bug in the repair code that could leave the repair incomplete like this; I've fixed it.
+
+> I recently moved a bunch of files to another location shortly after I renamed them. The assistant was running (repo: indirect mode) and I guess the assistant was confused
+
+No, I don't think that this was caused by the assistant somehow getting confused. Your git repository was missing objects that had been previously committed to git. The only way this can happen is if your computer lost data. Most commonly due to crashing or being powered off or the drive the repository is in being removed while git is in the middle of committing changes.
+"""]]
diff --git a/doc/forum/Create_lightweight_checkouts_on_the_same_filesystem.mdwn b/doc/forum/Create_lightweight_checkouts_on_the_same_filesystem.mdwn
new file mode 100644
index 0000000000..162185fece
--- /dev/null
+++ b/doc/forum/Create_lightweight_checkouts_on_the_same_filesystem.mdwn
@@ -0,0 +1,7 @@
+I'm sorry if this has been answered before, I did my best searching and couldn't find a fitting solution.
+
+I envision the following setup. There is one central git-annex repository where all modifications are to be performed. At the same time, I want to create lightweight clones of that repository on the same machine, the same filesystem, that would contain all git metadata (so that I can navigate the history inside the child repository), but would reuse binary objects from the parent repository. The child repositories can be read-only, I don't plan to use them for anything else but checking out the specific version of the parent repository.
+
+I found out about --shared flag and it seemed like it was exactly what I need. However, after cloning the parent repository with --shared, the symlinks in the child repository still pointed to nowhere. After I did `git annex sync --content`, the binary files were copied into the child repository's .git/ directory.
+
+Is it possible to achieve what I want? Thanks in advance!
diff --git a/doc/forum/Create_lightweight_checkouts_on_the_same_filesystem/comment_1_dc219eec54b62803831c854a620aceae._comment b/doc/forum/Create_lightweight_checkouts_on_the_same_filesystem/comment_1_dc219eec54b62803831c854a620aceae._comment
new file mode 100644
index 0000000000..2c49a613d2
--- /dev/null
+++ b/doc/forum/Create_lightweight_checkouts_on_the_same_filesystem/comment_1_dc219eec54b62803831c854a620aceae._comment
@@ -0,0 +1,20 @@
+[[!comment format=mdwn
+ username="https://launchpad.net/~stephane-gourichon-lpad"
+ nickname="stephane-gourichon-lpad"
+ avatar="http://cdn.libravatar.org/avatar/02d4a0af59175f9123720b4481d55a769ba954e20f6dd9b2792217d9fa0c6089"
+ subject="comment 1"
+ date="2017-07-20T17:47:42Z"
+ content="""
+Symbolic links point to `......./.git/annex/objects/.....`
+
+So, you can have them work by making your `.git/annex/objects` a link to the main repo's `.git/annex/objects`.
+
+    cd $mylightweightclone/.git/annex
+    mv objects objects.empty # move away but keep, just in case
+    ln -s $centralrepository/.git/annex/objects
+
+If the lightweight clone only performs read operations, I would expect things to work fine.
+
+I don't know if it can be dangerous to the health of your central repository besides that, so be careful.
+
+"""]]
diff --git a/doc/forum/Create_lightweight_checkouts_on_the_same_filesystem/comment_2_1263830d015a458024a2346540f10f31._comment b/doc/forum/Create_lightweight_checkouts_on_the_same_filesystem/comment_2_1263830d015a458024a2346540f10f31._comment
new file mode 100644
index 0000000000..fbb87f448a
--- /dev/null
+++ b/doc/forum/Create_lightweight_checkouts_on_the_same_filesystem/comment_2_1263830d015a458024a2346540f10f31._comment
@@ -0,0 +1,11 @@
+[[!comment format=mdwn
+ username="joey"
+ subject="""comment 2"""
+ date="2017-10-02T16:40:44Z"
+ content="""
+The way to do this is to use `git clone --shared`.
+Since git-annex version, 5.20140915, it detects when a new
+repository has been cloned that way and enables annex.hardlink,
+so files are inexpensively linked into place in the clone when getting them
+from the origin on the same filesystem.
+"""]]
diff --git a/doc/forum/Creating_a_more_secure_shared_encryption.mdwn b/doc/forum/Creating_a_more_secure_shared_encryption.mdwn
new file mode 100644
index 0000000000..5b2019f37c
--- /dev/null
+++ b/doc/forum/Creating_a_more_secure_shared_encryption.mdwn
@@ -0,0 +1,7 @@
+I was wondering if there was a way to make shared encryption more secure.  Here is my suggestion:  
+The shared repository is encrypted using a key for the whole repository, just the way normal encryption would work.  
+The server additionally keeps a copy of every user's public key.  
+When a user is authorized, their repository is initialized and they receive the common key, encrypted by their public key.  
+The only issue would be storage of the common key, which would have to be restricted to repository on a trusted machine.  
+
+Not sure if this would be easy for you to implement, but I figured I'd submit a post detailing it, to see if maybe it was doable.
diff --git a/doc/forum/Crippled_FS__58___remove_read_perms_from_dropped_files__63___.mdwn b/doc/forum/Crippled_FS__58___remove_read_perms_from_dropped_files__63___.mdwn
new file mode 100644
index 0000000000..18dc2fe66f
--- /dev/null
+++ b/doc/forum/Crippled_FS__58___remove_read_perms_from_dropped_files__63___.mdwn
@@ -0,0 +1,3 @@
+While experimenting with direct mode on Windows, I've got confused (and feeling momentary dread about apparent loss of data) many times when opening a file, and finding out that it in fact only contained the checksum; that is, was in the "dropped" state. As an **idea to prevent this confusion**, would it perhaps be possible to make the dropped files non-readable (permission denied) to the user somehow?
+
+(Background: I'm trying to build a version controlled development environment for a bunch of artists collaborating on large binaries, and making the process as fool-proof as possible is a priority.)
diff --git a/doc/forum/Crippled_FS__58___remove_read_perms_from_dropped_files__63___/comment_1_53941b83ff173025934e002a26031d9b._comment b/doc/forum/Crippled_FS__58___remove_read_perms_from_dropped_files__63___/comment_1_53941b83ff173025934e002a26031d9b._comment
new file mode 100644
index 0000000000..243f839aaa
--- /dev/null
+++ b/doc/forum/Crippled_FS__58___remove_read_perms_from_dropped_files__63___/comment_1_53941b83ff173025934e002a26031d9b._comment
@@ -0,0 +1,15 @@
+[[!comment format=mdwn
+ username="joey"
+ subject="""comment 1"""
+ date="2015-02-09T19:43:07Z"
+ content="""
+I think that would be a great idea. If someone can tell me how to do that
+on Windows.. 
+
+Note that even if you opened such a file, and made a change, and then
+saved it, this would at the worst result in a new commit being made.
+Other repositories that have a copy of the file would not lose their
+copy, at least not immediately (`git annex dropunused` might eventually
+remove it). So, to recover from such a mistake, it suffices to run `git
+annex undo $file`.
+"""]]
diff --git a/doc/forum/DBus_on_Ubuntu_12.04__63__.mdwn b/doc/forum/DBus_on_Ubuntu_12.04__63__.mdwn
new file mode 100644
index 0000000000..b4271d172c
--- /dev/null
+++ b/doc/forum/DBus_on_Ubuntu_12.04__63__.mdwn
@@ -0,0 +1,3 @@
+I tried to compile the assitant branch on Ubuntu 12.04. But i depends on the DBus libraryw hich does not compile with some glibberish errors. Is there a way to solve this?
+
+
diff --git a/doc/forum/DBus_on_Ubuntu_12.04__63__/comment_1_dc14a40b64b7eda94d1a3fd766cd39cc._comment b/doc/forum/DBus_on_Ubuntu_12.04__63__/comment_1_dc14a40b64b7eda94d1a3fd766cd39cc._comment
new file mode 100644
index 0000000000..0ef2469d3c
--- /dev/null
+++ b/doc/forum/DBus_on_Ubuntu_12.04__63__/comment_1_dc14a40b64b7eda94d1a3fd766cd39cc._comment
@@ -0,0 +1,28 @@
+[[!comment format=mdwn
+ username="http://joeyh.name/"
+ ip="4.152.246.119"
+ subject="comment 1"
+ date="2012-08-25T13:06:31Z"
+ content="""
+Hmm, let's see... 
+ 
+If the gibberish error is ouyay orgotfay otay otay elltay emay utwhay ethay
+roreay asway, then we can figure it out, surely..
+
+If the gibberish error looks something like Ḩ̶̞̗̓ͯ̅͒ͪͫe̢ͦ̊ͭͭͤͣ̂͏̢̳̦͔̬ͅ ̣̘̹̄̕͢Ç̛͈͔̹̮̗͈͓̞ͨ͂͑ͅo̿ͥͮ̿͢͏̧̹̗̪͇̫m̷̢̞̙͑̊̔ͧ̍ͩ̇̚ę̜͑̀͝s̖̱̝̩̞̻͐͂̐́̂̇̆͂ 
+
+.. your use of cabal  
+has accidentually summoned Cthulu! Back slowly away from the monitor!
+
+Otherwise, you might try installing the `libdbus-1-dev` package with apt, 
+which might make cabal install the haskell dbus bindings successfully. Or
+you could just install the `libghc-dbus-dev` package, which contains the 
+necessary haskell library pre-built. But I don't know if it's in Ubuntu 
+12.04; it only seems to be available in quantal 
+
+
+Or you could even build it with the Makefile, rather than using cabal.
+The Makefile has a `-DWITH_DBUS` setting in it that can be removed to build
+the fallback mode that doesn't use dbus.
+
+"""]]
diff --git a/doc/forum/DBus_on_Ubuntu_12.04__63__/comment_2_608a30e274e6a691a39f69503720e320._comment b/doc/forum/DBus_on_Ubuntu_12.04__63__/comment_2_608a30e274e6a691a39f69503720e320._comment
new file mode 100644
index 0000000000..02bda9eaa6
--- /dev/null
+++ b/doc/forum/DBus_on_Ubuntu_12.04__63__/comment_2_608a30e274e6a691a39f69503720e320._comment
@@ -0,0 +1,10 @@
+[[!comment format=mdwn
+ username="http://joeyh.name/"
+ ip="4.152.246.119"
+ subject="comment 2"
+ date="2012-08-25T13:11:37Z"
+ content="""
+I fnordgot to mention, cabal can be configured to not build with dbus too. The relevant incantation is:
+
+cabal install git-annex --flags=\"-Dbus\"
+"""]]
diff --git a/doc/forum/DBus_on_Ubuntu_12.04__63__/comment_3_791b9978b410c1aff7fd8ef05c38f5f9._comment b/doc/forum/DBus_on_Ubuntu_12.04__63__/comment_3_791b9978b410c1aff7fd8ef05c38f5f9._comment
new file mode 100644
index 0000000000..4097070fd2
--- /dev/null
+++ b/doc/forum/DBus_on_Ubuntu_12.04__63__/comment_3_791b9978b410c1aff7fd8ef05c38f5f9._comment
@@ -0,0 +1,40 @@
+[[!comment format=mdwn
+ username="https://www.google.com/accounts/o8/id?id=AItOawmU_2tE75oyG0h2ZPN4lcroIKEMC8G-otE"
+ nickname="Michael"
+ subject="Shame on me..."
+ date="2012-08-25T15:43:19Z"
+ content="""
+The build error is:
+
+$ cabal install DBus
+Resolving dependencies...
+Configuring DBus-0.4...
+checking for pkg-config... /usr/bin/pkg-config
+checking pkg-config is at least version 0.9.0... yes
+checking for DBUS... yes
+configure: creating ./config.status
+config.status: creating DBus.buildinfo
+Building DBus-0.4...
+Preprocessing library DBus-0.4...
+
+DBus/Message.hsc:1:14:
+    Warning: -XPatternSignatures is deprecated: use -XScopedTypeVariables or pragma {-# LANGUAGE ScopedTypeVariables #-} instead
+
+DBus/Message.hsc:2:12:
+    Warning: -fglasgow-exts is deprecated: Use individual extensions instead
+[1 of 5] Compiling DBus.Shared      ( dist/build/DBus/Shared.hs, dist/build/DBus/Shared.o )
+[2 of 5] Compiling DBus             ( dist/build/DBus.hs, dist/build/DBus.o )
+
+DBus.hsc:26:49:
+    Warning: In the use of `mkTyCon'
+             (imported from Data.Typeable):
+             Deprecated: \"either derive Typeable, or use mkTyCon3 instead\"
+[3 of 5] Compiling DBus.Internal    ( dist/build/DBus/Internal.hs, dist/build/DBus/Internal.o )
+
+DBus/Internal.hsc:12:27:
+    Module `Control.Exception' does not export `throwDyn'
+cabal: Error: some packages failed to install:
+DBus-0.4 failed during the building phase. The exception was:
+ExitFailure 1
+
+"""]]
diff --git a/doc/forum/DBus_on_Ubuntu_12.04__63__/comment_4_8665c95299916138c4af375626d9ec7d._comment b/doc/forum/DBus_on_Ubuntu_12.04__63__/comment_4_8665c95299916138c4af375626d9ec7d._comment
new file mode 100644
index 0000000000..abd154a6bc
--- /dev/null
+++ b/doc/forum/DBus_on_Ubuntu_12.04__63__/comment_4_8665c95299916138c4af375626d9ec7d._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="http://joeyh.name/"
+ ip="4.152.246.119"
+ subject="in the language of the ancients, small details matter"
+ date="2012-08-25T15:50:47Z"
+ content="""
+It seems that to cabal, \"DBus\" and \"dbus\" are different packages, and it seems you should install the latter, which has a current version of 0.10 which works and not some old 0.4 version.
+"""]]
diff --git a/doc/forum/DO_NOT_use_ntfs-3g_on_mac_osx_for_annex.mdwn b/doc/forum/DO_NOT_use_ntfs-3g_on_mac_osx_for_annex.mdwn
new file mode 100644
index 0000000000..16750a1fb2
--- /dev/null
+++ b/doc/forum/DO_NOT_use_ntfs-3g_on_mac_osx_for_annex.mdwn
@@ -0,0 +1,41 @@
+After testing around git-annex on my mac using ntfs-3g to access the removable harddrive, I conclude that this is very dangerous. 
+
+Here are some example error snippets:
+
+    add man/man1/git-tar-tree.1 
+    git-annex: /Volumes/SeagateExpansion/annex/testdirect/.git/annex/objects/0K/JJ/SHA256E-s3805--948fd508a364e036ec28f5287d11825062aea2c0ed86a3a89ed871edc6ddc7a8.1/: openTempFile: interrupted (Interrupted system call)
+    failed
+
+    add man/man1/git-bundle.1 
+    git-annex: /Volumes/SeagateExpansion/annex/testdirect/.git/annex/tmp/e3f_804_SHA256E-s8739--569a4b4fe1072e17ff0aa9a2f67d0130b9ec3ed23f96c5a4ea38ada82f50b4aa.1.log: rename: does not exist (No such file or directory)
+    failed
+
+    add git-core/git-completion.bash failed
+    add man/man7/gitglossary.7 failed
+    (Recording state in git...)
+
+    git-annex: /Volumes/SeagateExpansion/annex/testdirect/.git/annex/journal/.fuse_hidden00005dd50000010a: removeLink: does not exist (No such file or directory)
+    failed
+    git-annex: add: 3 failed
+
+It's interesting that after each git-annex add ., it would got fewer failures. But in the end it stuck at 2 files. After 60 times doing: git-annex add .
+
+It went from this:
+
+    git-annex: add: 248 failed
+
+to this:
+
+    git-annex: add: 2 failed
+
+Now the repository has 246 objects, let's try to convert it into an indirect one:
+
+    git-annex indirect
+    ...
+    git-annex info
+    ...
+    annexed files in working tree: 9
+
+Only 9 files made into the indirect repository! Remaining files stay as original state. Now it would be a nightmare to try to recover form this mess.
+
+I discourage anyone from using ntfs-3g on mac for annexing. Especially direct & indirect mode. Bare repo seems to be fine however.
diff --git a/doc/forum/DS__95__Store_files_are_not_added.mdwn b/doc/forum/DS__95__Store_files_are_not_added.mdwn
new file mode 100644
index 0000000000..f0ccf2023b
--- /dev/null
+++ b/doc/forum/DS__95__Store_files_are_not_added.mdwn
@@ -0,0 +1,3 @@
+The "git annex add" command adds new file to the annex. However ".DS_Store" files are ignored by git-annex. Is there a list of files that are being ignored? 
+
+Maybe sometimes it's useful to add .DS_Store extended attribute data to the annex to ensure a complete sync of Mac files...
diff --git a/doc/forum/DS__95__Store_files_are_not_added/comment_1_30687306da9bd35ec02a806193c5e240._comment b/doc/forum/DS__95__Store_files_are_not_added/comment_1_30687306da9bd35ec02a806193c5e240._comment
new file mode 100644
index 0000000000..3402b84c61
--- /dev/null
+++ b/doc/forum/DS__95__Store_files_are_not_added/comment_1_30687306da9bd35ec02a806193c5e240._comment
@@ -0,0 +1,7 @@
+[[!comment format=mdwn
+ username="http://joeyh.name/"
+ subject="comment 1"
+ date="2012-08-02T23:29:38Z"
+ content="""
+git-annex ignores whatever git does. See `git-ls-files` and the gitignore configuration.
+"""]]
diff --git a/doc/forum/DVD-R-based_archive_--_lack_of_backward_compatibility_seems_like_a_showstopper.mdwn b/doc/forum/DVD-R-based_archive_--_lack_of_backward_compatibility_seems_like_a_showstopper.mdwn
new file mode 100644
index 0000000000..bc89b1352d
--- /dev/null
+++ b/doc/forum/DVD-R-based_archive_--_lack_of_backward_compatibility_seems_like_a_showstopper.mdwn
@@ -0,0 +1,15 @@
+I've been planning to use git-annex to manage a collection of DVD-Rs (of photos and the like), something along [these lines](https://git-annex.branchable.com/forum/How_do_I_backup_my_data_to_blue_ray_disks__63__/), and especially this paragraph therein:
+
+> If the [media is] not rewritable [which in my case it won't be; see below], I might try making the git-annex repo in a temporary directory on the hard disk, and then generating the ISO from that once I've filled it up. Should work fine, I might set "remote..annex-readonly" to true in git repos that had such a disk as a remote to let git-annex know to not try to write to it.
+
+But then there's this bug report: [read-only filesystem on remote prevents auto-upgrade from v3 to v5, and prevents using a remote](https://git-annex.branchable.com/bugs/annex_get_fails_from_read-only_filesystem/), and [Joey's comment]((http://git-annex.branchable.com/bugs/annex_get_fails_from_read-only_filesystem/#comment-67f204c3a4312bbda8ace305dbe0afbf):
+
+> From a code complication POV, it's useful for git-annex to only support one version of repository at a time.
+
+My concept depends on being able to restore files from my archive, N years and M git-annex versions later, using `git annex get` (after `git annex whereis` or the like to find out which volume to mount).  If I can't count on the *get* to work -- and indeed, can pretty much count on it *not* to, after the next format-version bump -- this whole scheme sounds like a nonstarter.
+
+> As far as backwards compatablity goes, I don't anticipate ever removing the upgrade code from git-annex.
+
+This is an archive; therefore it will use write-once media -- if it were rewritable, it would be clobberable should something (including `git-annex upgrade`!) go wrong.
+
+So, is there a way around this?  Or am I (and others) out of luck on using git-annex to manage long-term archival storage?
diff --git a/doc/forum/DVD-R-based_archive_--_lack_of_backward_compatibility_seems_like_a_showstopper/comment_1_d2640d4cd5bd28df5ecf389d17a2e52b._comment b/doc/forum/DVD-R-based_archive_--_lack_of_backward_compatibility_seems_like_a_showstopper/comment_1_d2640d4cd5bd28df5ecf389d17a2e52b._comment
new file mode 100644
index 0000000000..de950c8825
--- /dev/null
+++ b/doc/forum/DVD-R-based_archive_--_lack_of_backward_compatibility_seems_like_a_showstopper/comment_1_d2640d4cd5bd28df5ecf389d17a2e52b._comment
@@ -0,0 +1,13 @@
+[[!comment format=mdwn
+ username="joey"
+ subject="""comment 1"""
+ date="2016-10-05T20:54:02Z"
+ content="""
+I understand your pain (and have just made git-annex support reading V3
+repos without upgrading them). But, it's not a good idea to make git-annex
+support reading all previous repository versions without an upgrade step.
+
+Suggest you plan on copying your DVD-R to a writable medium to access it,
+if git-annex at some point stops supporting the old repo version without
+upgrade.
+"""]]
diff --git a/doc/forum/Dealing_with_crippled_Android_file_system.mdwn b/doc/forum/Dealing_with_crippled_Android_file_system.mdwn
new file mode 100644
index 0000000000..bd957f459f
--- /dev/null
+++ b/doc/forum/Dealing_with_crippled_Android_file_system.mdwn
@@ -0,0 +1,15 @@
+When working on Android in [[termux|tips/Install on Android in Termux]], user has limited choices in where to check out an annex:
+
+* in the home directory (`/data/data/com.termux/files/home`): well-behaved ext4 file system that allow symlinks, but is only accessible from inside termux.
+* in shared storage (`/storage/emulated/0`, symlinked as `~/storage/...`): accessible to media apps, but does not support symlinks and is therefore a crippled file system with all its drawbacks. The file system seems to be backed by the same ext4 partition as above, but mounted through some fuse mechanism which I can only assume is responsible for enforcing Android's permissions model.
+
+It would be nice to find a way to have the best of both; here's what I tried so far:
+
+* running git-annex as root for experimental purposes: Does not help, the fuse file system does not support symlinks (`ln -s foo bar` → `ln: cannot create symbolic link from 'foo' to 'bar': Function not implemented`).
+  * If that had worked, I would have had a look at whether it's just a matter of permissions that termux could obtain, but it's obviously a matter of principle.
+* running git-annex in home, but ro-bind-mounting a view of it into emulated storage: Only made the files visible to root, not even to termux let alone other processes (I tried adding `--make-shared` without knowing what it'd do exactly)
+* I saw there'd be a way to [allow symlinks on Android](https://android.stackexchange.com/questions/84022/how-to-make-symlinks-work-inside-storage-emulated-in-4-4-4#111669), but honestly that appears too scary, and given the root ln issue, I think it does not apply any more.
+
+The use case I have in mind is syncing photos to an Android tablet in a v5 repository.
+
+Are there other ideas around on how git-annex can be used with classical symlinks on Android?
diff --git a/doc/forum/Dealing_with_crippled_Android_file_system/comment_1_1b17852211108d9ab4d3959d7f57a9df._comment b/doc/forum/Dealing_with_crippled_Android_file_system/comment_1_1b17852211108d9ab4d3959d7f57a9df._comment
new file mode 100644
index 0000000000..daa1b01b65
--- /dev/null
+++ b/doc/forum/Dealing_with_crippled_Android_file_system/comment_1_1b17852211108d9ab4d3959d7f57a9df._comment
@@ -0,0 +1,20 @@
+[[!comment format=mdwn
+ username="andrew"
+ avatar="http://cdn.libravatar.org/avatar/acc0ece1eedf07dd9631e7d7d343c435"
+ subject="use case"
+ date="2018-05-23T00:12:11Z"
+ content="""
+Hmmm. Could you talk more about your use case “The use case I have in mind is syncing photos to an Android tablet in a v5 repository.”?
+
+Lets say I have a nice collection of family photos on my desktop computer annexed in a folder called `family`. Lets say we have `numcopies` set at 1. I keep only the best photos, so it always takes up about 1gb. In the family folder I have nice organized sub-folders and I add new images occasionally. I have an Android tablet that sits on my coffee table and I want visitors to be able to open the tablet, run some gallery app and view all my family photos. Sometimes i'll grab the tablet and take it with me to family gatherings, so I want all the photos on in, ready for offline viewing.
+
+With this use case, perhaps I run assistant inside termux. I clone family into `/data/data/com.termux/files/home` set it as standard group transfer and call it `t`, I also clone it into `/storage/emulated/0`, set it as standard group client, and call this crippled client repo `c`. Our `c` folder should now be accessible to any Android photo gallery app. As we make changes to `family`, on our desktop, assistant on the Android tablet brings them in, when we have wifi, it will probably copy them directly to `c`?
+
+If I take a photo on my tablet, put it in the `c` directory (which is accessible from my photo app), then this photo should be copied directly to family (if I have wifi). If I don't have have wifi, this photo should be immediately copied to `t` in which case I have two copies of the file on my tablet (I think)? If I delete the photo from the crippled filesystem `c` I still have the one that was already copied to `t`. If I re-connect to wifi, perhaps the photo will get sent to `family` from `t`. It might stay in `t` since it is missing from `c`, not sure how this would work (but anyhow, see archive folders in the next paragraph).
+
+Maybe I want to save some space on my tablet (deleting from photo app might work?, but archiving is more likely to work). So I open up my photo app, browse to my `c` folder and start putting a bunch of files in a sub-folder named `archive`. They are then immediately deleted from my tablet (unless they were never synced to `family` in which case I have two copies of them, so I would still have a copy sitting in transfer `t`). If I archive files when I have no wifi, presumably they will just get moved to `t` by assistant since git-annex cannot verify the copy at `family`?
+
+Would something like that work for your use case? I don't actually use Android termux, so I am sure others might have some insight into use of the various directories.
+
+—Andrew
+"""]]
diff --git a/doc/forum/Dealing_with_crippled_Android_file_system/comment_2_af66edabeeafcefb45b1d5c5d13dda92._comment b/doc/forum/Dealing_with_crippled_Android_file_system/comment_2_af66edabeeafcefb45b1d5c5d13dda92._comment
new file mode 100644
index 0000000000..eb5b340e55
--- /dev/null
+++ b/doc/forum/Dealing_with_crippled_Android_file_system/comment_2_af66edabeeafcefb45b1d5c5d13dda92._comment
@@ -0,0 +1,21 @@
+[[!comment format=mdwn
+ username="https://christian.amsuess.com/chrysn"
+ nickname="chrysn"
+ avatar="http://christian.amsuess.com/avatar/c6c0d57d63ac88f3541522c4b21198c3c7169a665a2f2d733b4f78670322ffdc"
+ subject="Re: use case"
+ date="2018-05-23T07:56:34Z"
+ content="""
+The particular repository I'm looking at for a first stage is a wedding photo repository. It contains both the working directories of the photographers involved (some JPEG, some RAW-based with git-versioned Makefiles and sidecars to get the JPEGs which are then checked in with numcopies=0), where some photographers have their files in multiples copies to reflect their pre-sorting. The final photos are also copied to a release directory in various combinations (\"complete couple's selection\", \"pictures released to guests\", \"other pictures of guests\") which were then distributed by a process outside of git-annex' control (zip files were created and distributed over a web server).
+
+This copying workflow usually works greatly with git-annex v5 (or v6 when files are always kept locked): even if files were checked in by different contributors with different default hash settings, copying a file means copying a symlink.
+
+The whole repository amounts to 125gb, with 3gb in the selection.
+
+On the tablet, when I clone the repository on the crippled side of the file system, the initial checkout takes quite some time, and the 3gb get blown to 8gb due to the file being copied not only to the \"complete couple's selection\", but also to the other selections and the occasional picture in the photographers' directories.
+
+Note that on this repository, the tablet does not even have write access to the repository. It is set to untrusted and \"wants\" the selection folder.
+
+---
+
+The next repository I'd like to do this with is our family photo repository, containing an average of four persons' digitized analog and digital photos starting in 2002 and contains about 700gb of images; that repository has not that much duplication, but the interesting files (often a \"Selection\" folder per event that contains copies with appropriately named files for a sequence) are still checked in in duplicate.
+"""]]
diff --git a/doc/forum/Dealing_with_crippled_Android_file_system/comment_3_8a555490bf1735c2f05a95cdb281ec80._comment b/doc/forum/Dealing_with_crippled_Android_file_system/comment_3_8a555490bf1735c2f05a95cdb281ec80._comment
new file mode 100644
index 0000000000..9a32c41ebe
--- /dev/null
+++ b/doc/forum/Dealing_with_crippled_Android_file_system/comment_3_8a555490bf1735c2f05a95cdb281ec80._comment
@@ -0,0 +1,11 @@
+[[!comment format=mdwn
+ username="andrew"
+ avatar="http://cdn.libravatar.org/avatar/acc0ece1eedf07dd9631e7d7d343c435"
+ subject="comment 3"
+ date="2018-05-23T16:01:18Z"
+ content="""
+Hmmm, yes, my solution above wouldn't handle de-duplication of the same file in multiple folders. I have a one thought that might work (I can't test right now):
+
+Use [proot](https://github.com/termux/proot) inside termux to [bind a path](https://github.com/proot-me/PRoot/blob/master/doc/proot/manual.txt) `/data/data/com.termux/files/home/annex-photos` to a mount point `~/storage/pictures/annex`. Then perhaps symlinks within annex-photos will be presented as files at `~/storage/pictures/annex` without increasing actual storage.
+
+"""]]
diff --git a/doc/forum/Dealing_with_crippled_Android_file_system/comment_4_051a08b7acd718ebae66a6aab69edb6e._comment b/doc/forum/Dealing_with_crippled_Android_file_system/comment_4_051a08b7acd718ebae66a6aab69edb6e._comment
new file mode 100644
index 0000000000..9a067b79be
--- /dev/null
+++ b/doc/forum/Dealing_with_crippled_Android_file_system/comment_4_051a08b7acd718ebae66a6aab69edb6e._comment
@@ -0,0 +1,11 @@
+[[!comment format=mdwn
+ username="https://christian.amsuess.com/chrysn"
+ nickname="chrysn"
+ avatar="http://christian.amsuess.com/avatar/c6c0d57d63ac88f3541522c4b21198c3c7169a665a2f2d733b4f78670322ffdc"
+ subject="Re: proot"
+ date="2018-05-23T17:05:54Z"
+ content="""
+proot would only change the view of programs launched under it. I could probably do that (if git annex works under proot on the tablet in question at all; [[I had to disable it|tips/install_on_Android_in_Termux/#comment-48d995d39f783b4c59b1a2a66b387b55]] to get it running on Android 7.1 (compatible; LineageOS 14.1)).
+
+The gallery and similar apps would still only see the fuse file system.
+"""]]
diff --git a/doc/forum/Dealing_with_crippled_Android_file_system/comment_5_a210b77d8567643667bbe92b40b5bef0._comment b/doc/forum/Dealing_with_crippled_Android_file_system/comment_5_a210b77d8567643667bbe92b40b5bef0._comment
new file mode 100644
index 0000000000..33c86afb5b
--- /dev/null
+++ b/doc/forum/Dealing_with_crippled_Android_file_system/comment_5_a210b77d8567643667bbe92b40b5bef0._comment
@@ -0,0 +1,19 @@
+[[!comment format=mdwn
+ username="andrew"
+ avatar="http://cdn.libravatar.org/avatar/acc0ece1eedf07dd9631e7d7d343c435"
+ subject="comment 5"
+ date="2018-05-30T14:43:37Z"
+ content="""
+Aaaah. OK, that makes sense.
+
+So, when you call [termux-setup-storage](https://wiki.termux.com/wiki/Internal_and_external_storage) in Termux it creates a bunch of symlinks to the standard Android directories, calling [setupStorageSymlinks](https://github.com/termux/termux-app/blob/master/app/src/main/java/com/termux/app/TermuxInstaller.java). But yeah, the new folders sitting in $HOME are just symlinks to Android's FAT32 view of these folders. I was thinking maybe they were something else…
+
+So after reading [Diving into SDCardFS: How Google’s FUSE Replacement Will Reduce I/O Overhead](https://www.xda-developers.com/diving-into-sdcardfs-how-googles-fuse-replacement-will-reduce-io-overhead/) it appears that FAT32 is very much the baked-in standard. The article also has some nice information on the naming conventions in Android of various storage types. Some Google-ing does imply that people have changed the underlying filesystem of the root OS, with rooting, but the solutions seems to now, at least, be [device specific](https://stackoverflow.com/a/29727734/8671834).
+
+I can't find any elegant, filesystem level solution for de-duplication of files that would be visible as media to all Android apps in `external storage`.
+
+Maybe it is possible that with Android's new [Adoptable Storage](https://source.android.com/devices/storage/adoptable) one could create symlinks, but I haven't tried this and I am not optimistic. Adoptable storage is not widely supported, but essentially it fully encrypts your device and re-formats your “external storage” as ext4 or f2fs. But I would imagine that any public facing views of external storage would probably still be FAT32 so their wouldn't be any way to create symlinks.
+
+My other thought would be to create an app specific solution to your issue. For example you are trying to make photos visible to Gallery apps and you have a lot of duplicate photos present in multiple folders. One solution would be to use albums, which I am pretty sure already support having the same photo in multiple albums (at least in Google Photos), while only taking up storage for a single photo. So, you could copy a single copy of each of your photos to `~/storage/pictures`, then using Android APIs [add the photos into multiple albums](https://stackoverflow.com/a/11983767/8671834). You could probably automate this process with scripts or perhaps a Termux addon? Or, perhaps there is some other album supporting Android Gallery app that uses a simpler album format, like a JSON file or something.
+
+"""]]
diff --git a/doc/forum/Dealing_with_crippled_Android_file_system/comment_6_91f81669519dbf1dfbad2259f48ab99d._comment b/doc/forum/Dealing_with_crippled_Android_file_system/comment_6_91f81669519dbf1dfbad2259f48ab99d._comment
new file mode 100644
index 0000000000..95a08e122a
--- /dev/null
+++ b/doc/forum/Dealing_with_crippled_Android_file_system/comment_6_91f81669519dbf1dfbad2259f48ab99d._comment
@@ -0,0 +1,23 @@
+[[!comment format=mdwn
+ username="https://christian.amsuess.com/chrysn"
+ nickname="chrysn"
+ avatar="http://christian.amsuess.com/avatar/c6c0d57d63ac88f3541522c4b21198c3c7169a665a2f2d733b4f78670322ffdc"
+ subject="Re: comment 5"
+ date="2018-06-12T19:01:09Z"
+ content="""
+ad SDCardFS / FAT32: It seems that both the fuse wrapper around the SD card (be it actually formatted FAT32 because it's the external one, or ext4 as is the encrypted internal one on my device) and SDCardFS (which, it appears to me, is just Google moving the fuse wrapper into the kernel) won't expose the functionality we want.
+
+ad adoptable storage: I think that's what's happening on my device (but am not sure), so if it is, the there's still the fuse wrapper / SDCardFS inbetween.
+
+ad specific app: That'd still mean that git-annex needs to operate on crippled file system, and I don't want to rely on special solutions for each app.
+
+---
+
+What I've seen when just setting up a new device is that the Storage Access Framework (looks like Android's idea of gvfs / kvfs with a spike of flatpak-portals to me: a userland file system that operates on `saf:/` and `content://` URIs rather than files, probably does file operations by means of IPC and receiving file handlers) might help:
+
+In the LineageOS built-in file manager, I don't see two \"virtual SD cards\" (the internal and the external one), but also a \"Termux\" \"device\" with proper logo and \"0B free\", which allows me to peek into the termux home directory (but not its usr directory). From that file manager I even see the photos inside a regular (symlinked) checkout, and can open them with the Simple Gallery app. (Provided they end with '.jpg' and not '.JPG', but that's probably a detail of the file manager).
+
+I did not yet find another app that sees this \"Termux\" device; Ghost Commander only sees \"Documents\" and the internal and external SD cards; the \"Dir\" file manager exposes a SAF selection only when creating directories on external SD (and then shows the same selection as Ghost Commander), the Simple tools (Simple File Manager, Simple gallery) show \"Internal\", \"SD Card\" and \"Root\" in their own storage selection (might be pre-SAF; similar situation also for the Amaze file manager).
+
+If we could reproduce what the builtin file manager does to access the files, that might be a solution here. Termux obviously already helps in this by exporting the home directory ([since 0.34](https://termux.com/changelog.html)), but maybe they also can do more (like exporting XDG user directories for quick access?).
+"""]]
diff --git a/doc/forum/Dealing_with_crippled_Android_file_system/comment_7_087fdae633144482473c7391ea338c5c._comment b/doc/forum/Dealing_with_crippled_Android_file_system/comment_7_087fdae633144482473c7391ea338c5c._comment
new file mode 100644
index 0000000000..12c667154a
--- /dev/null
+++ b/doc/forum/Dealing_with_crippled_Android_file_system/comment_7_087fdae633144482473c7391ea338c5c._comment
@@ -0,0 +1,15 @@
+[[!comment format=mdwn
+ username="https://christian.amsuess.com/chrysn"
+ nickname="chrysn"
+ avatar="http://christian.amsuess.com/avatar/c6c0d57d63ac88f3541522c4b21198c3c7169a665a2f2d733b4f78670322ffdc"
+ subject="Update on using SAF"
+ date="2018-06-13T14:57:10Z"
+ content="""
+I've looked around for how this whole SAF thing works and where it is supported; it seems to me that it is indeed the way to go:
+
+* Termux added this with the very intention of making the home directory usable ([initial issue](https://github.com/termux/termux-app/issues/79)).
+  * That .JPG files are not image/jpeg is probably the fault of a library termux uses ([reported](https://github.com/termux/termux-app/issues/721))
+* Some applications can find files from a termux-home out-of-the-box, like the [billthefarmer editor](https://f-droid.org/en/packages/org.billthefarmer.editor/) -- just that if it was just about editing text files, we could just have used ~/storage/shared and plain git.
+* Many applications accept files from there when opened from the file manager (VLC, Simple Gallery, Ghost Commander's picture viewer); I have not found one yet that does not.
+* A good way forward would be to encourage the authors of apps we care about to allow opening SAF locations directly; it appears to me that they sometimes add SAF support for a particular function (eg. to support USB-OTG or writing to external SD cards) and then limit it to that path because the usual other options appear to confusing. [Requested support for it in Simple Gallery](https://github.com/SimpleMobileTools/Simple-Gallery/issues/806), let's see where that takes this.
+"""]]
diff --git a/doc/forum/Dealing_with_crippled_Android_file_system/comment_8_a2bafe08ff3e7a236ba560dafc9c4d85._comment b/doc/forum/Dealing_with_crippled_Android_file_system/comment_8_a2bafe08ff3e7a236ba560dafc9c4d85._comment
new file mode 100644
index 0000000000..87faebd6c2
--- /dev/null
+++ b/doc/forum/Dealing_with_crippled_Android_file_system/comment_8_a2bafe08ff3e7a236ba560dafc9c4d85._comment
@@ -0,0 +1,18 @@
+[[!comment format=mdwn
+ username="andrew"
+ avatar="http://cdn.libravatar.org/avatar/acc0ece1eedf07dd9631e7d7d343c435"
+ subject="comment 8"
+ date="2018-06-15T15:59:04Z"
+ content="""
+Oh, Awesome, thanks! Great find. I'll write out some of the things I have tried:
+
+Create a text file called `hi.txt` from Termux in `/data/data/com.termux/files/home` AKA `$HOME`. Launch `Turbo Editor` enable `Use the Storage Access Framework` in Preferences menu. Click `Open a file` | `Open from` `Termux`, click `hi.txt`, edit file, save. Now I can see changes in Termux. Nice.
+
+From Termux create a new soft link, `cd`, `ln -s hi.txt hilink.txt`, then in Turbo Editor I can see both files, I can edit and save either, and see changes correctly in Termux.
+
+I wasn't able to find the Termux `$HOME` folder in any File Manager I tried, nor any Gallery App.
+
+
+
+
+"""]]
diff --git a/doc/forum/Dealing_with_long_filenames.mdwn b/doc/forum/Dealing_with_long_filenames.mdwn
new file mode 100644
index 0000000000..5411c32153
--- /dev/null
+++ b/doc/forum/Dealing_with_long_filenames.mdwn
@@ -0,0 +1,8 @@
+- External HD A: annex (contains files with long paths)
+- External HD B: annex (synced with A, doesn't have the contents for files with long paths)
+- laptop HD: annex     (*syncing with B fails*: paths too long)
+
+How should I resolve this? 
+
+- Can I rename the broken symlinks on B to shorten the paths, and then sync to the laptop? Would this be propagated to A when it's connected again?
+- Can I `git rm too/long/paths` on B and then sync?
diff --git a/doc/forum/Dealing_with_long_filenames/comment_1_373798ea9763d3a0be5ce78f7e3f79f0._comment b/doc/forum/Dealing_with_long_filenames/comment_1_373798ea9763d3a0be5ce78f7e3f79f0._comment
new file mode 100644
index 0000000000..32056270e9
--- /dev/null
+++ b/doc/forum/Dealing_with_long_filenames/comment_1_373798ea9763d3a0be5ce78f7e3f79f0._comment
@@ -0,0 +1,16 @@
+[[!comment format=mdwn
+ username="joey"
+ subject="""comment 1"""
+ date="2017-03-02T18:01:51Z"
+ content="""
+It depends, is it failing because the filenames checked into git are too
+long? Or because the symlinks are links to too long paths?
+
+In the former case, you can `git rename` files as you like.
+As with any change you make to files in a git repository, that will
+propigate to other git repositories in the usual ways.
+
+In the latter case, you might be using a hash like SHA512 that causes
+problimatically long paths, and then `git annex migrate --backend=SHA256` might
+help.
+"""]]
diff --git a/doc/forum/Debugging_Git_Annex.mdwn b/doc/forum/Debugging_Git_Annex.mdwn
new file mode 100644
index 0000000000..fc4d829a2a
--- /dev/null
+++ b/doc/forum/Debugging_Git_Annex.mdwn
@@ -0,0 +1,4 @@
+Hi, 
+May I know, how can I debug git-annex code.
+I am new to Haskell Platform, I would like to know which IDE can be used to debug haskell code.
+Thank You.
diff --git a/doc/forum/Debugging_Git_Annex/comment_1_ce63b2ee641a2338f1ad5ded9e6f09a8._comment b/doc/forum/Debugging_Git_Annex/comment_1_ce63b2ee641a2338f1ad5ded9e6f09a8._comment
new file mode 100644
index 0000000000..84781a00a4
--- /dev/null
+++ b/doc/forum/Debugging_Git_Annex/comment_1_ce63b2ee641a2338f1ad5ded9e6f09a8._comment
@@ -0,0 +1,7 @@
+[[!comment format=mdwn
+ username="http://joeyh.name/"
+ subject="This is not an easy question to answer..."
+ date="2012-06-04T19:49:46Z"
+ content="""
+Do you have a bug in git-annex that you need fixed, or are you just curious?
+"""]]
diff --git a/doc/forum/Debugging_Git_Annex/comment_2_1d70ff052d00f33c34fd45730ea13040._comment b/doc/forum/Debugging_Git_Annex/comment_2_1d70ff052d00f33c34fd45730ea13040._comment
new file mode 100644
index 0000000000..017b34b0de
--- /dev/null
+++ b/doc/forum/Debugging_Git_Annex/comment_2_1d70ff052d00f33c34fd45730ea13040._comment
@@ -0,0 +1,12 @@
+[[!comment format=mdwn
+ username="https://www.google.com/accounts/o8/id?id=AItOawlYu7QmD7wrbHWkoxuriaA9XcijM-g5vrQ"
+ nickname="Royal"
+ subject="comment 2"
+ date="2012-06-05T17:19:16Z"
+ content="""
+Hi,
+
+I want to replace rsync with aspera-rsync. Whenever there is file transfer between 2 repositories which are in two different hosts, git-annex will use rsync protocol. I am trying to replace that rsync call with aspera-rsync so that transfer can be more faster. Since I am new to Haskell I am finding difficulties to understand the flow of execution. Is there any way I can debug so that I can get the flow?
+
+Thanks
+"""]]
diff --git a/doc/forum/Debugging_of_transfer_considerations.mdwn b/doc/forum/Debugging_of_transfer_considerations.mdwn
new file mode 100644
index 0000000000..eb75e39772
--- /dev/null
+++ b/doc/forum/Debugging_of_transfer_considerations.mdwn
@@ -0,0 +1,6 @@
+I have a client repository with an archive/ directory. If I let the assistant sync the repos or initiate a manual `git annex sync --content` the contents of the archive directory is filled up.
+
+I can't see why this happens. The repo is a client repo with the standard preferred contents and no required contents onfig.
+One of the files that is wanted has 2 copies in backup repositories (one is trusted and one is reachable and semitrusted). `git annex numcopies` is 2 for one of these files. The repos were checked by `git annex fsck` and synced.
+
+I'm not sure what to look for or how to debug this kind of situation. Any ideas? 
diff --git a/doc/forum/Debugging_of_transfer_considerations/comment_1_e68ab187c15b3b6f9384a990da07f358._comment b/doc/forum/Debugging_of_transfer_considerations/comment_1_e68ab187c15b3b6f9384a990da07f358._comment
new file mode 100644
index 0000000000..ba943b8945
--- /dev/null
+++ b/doc/forum/Debugging_of_transfer_considerations/comment_1_e68ab187c15b3b6f9384a990da07f358._comment
@@ -0,0 +1,39 @@
+[[!comment format=mdwn
+ username="joey"
+ subject="""comment 1"""
+ date="2017-03-06T16:08:55Z"
+ content="""
+Currently the best way to debug this kind of thing is to use git annex find
+with options to find files that match the preferred content expression.
+Once you have gotten git annex find to list the same files that are being
+transferred, you can then modify/cut down the options to narrow down
+what's going on.
+
+The preferred content expression for a client repository is:
+
+	(include=* and ((exclude=*/archive/* and exclude=archive/*) or (not (copies=archive:1 or copies=smallarchive:1)))) or approxlackingcopies=1
+
+Translating this to command-line options:
+
+	git annex find '-(' '--include=*' --and '-(' '-(' '--exclude=*/archive/*' --and '--exclude=archive/*' '-)' --or '-(' --not '-(' --copies=archive:1 --or --copies=smallarchive:1 '-)' '-)' '-)' '-)' --or --approxlackingcopies=1
+
+You'll want to run that when the files are not located in the archive/
+directory, and run it from the top of the repository.
+
+Assuming that lists the files that are getting transferred, then you can
+split it into two commands, each of which checks one of the two parts
+of the expression that are ORed together:
+
+	git annex find '-(' '--include=*' --and '-(' '-(' '--exclude=*/archive/*' --and '--exclude=archive/*' '-)' --or '-(' --not '-(' --copies=archive:1 --or --copies=smallarchive:1 '-)' '-)' '-)' '-)'
+	git annex find --approxlackingcopies=1
+
+Assuming the first of those lists the files and not the second, you can
+then split it further. The include=* part must be matching then, so
+checking two parts ORed within the second part:
+	
+	git annex find '-(' '--exclude=*/archive/*' --and '--exclude=archive/*' '-)'
+	git annex find '-(' --not '-(' --copies=archive:1 --or --copies=smallarchive:1 '-)' '-)'
+
+Probably one of those will list the files and the other won't.
+Which will point fairly strongly at what's happening.
+"""]]
diff --git a/doc/forum/Debugging_of_transfer_considerations/comment_2_f7cd207b08fe365d0a6248316c91ed78._comment b/doc/forum/Debugging_of_transfer_considerations/comment_2_f7cd207b08fe365d0a6248316c91ed78._comment
new file mode 100644
index 0000000000..d95274ef52
--- /dev/null
+++ b/doc/forum/Debugging_of_transfer_considerations/comment_2_f7cd207b08fe365d0a6248316c91ed78._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="Horus"
+ avatar="http://cdn.libravatar.org/avatar/8f0ee08b98ea5bba76c3fe112c08851c"
+ subject="comment 2"
+ date="2017-03-14T08:58:54Z"
+ content="""
+Maybe it's the problem that the client group requires a file in archive/ to reach a archive repo, not a backup repo. To combine the behavior of both, I have set the group to archive and wanted to anything.
+"""]]
diff --git a/doc/forum/Debugging_of_transfer_considerations/comment_3_508b5b4155f0f9c0c2a7d2e990a1bf92._comment b/doc/forum/Debugging_of_transfer_considerations/comment_3_508b5b4155f0f9c0c2a7d2e990a1bf92._comment
new file mode 100644
index 0000000000..99a8be0a18
--- /dev/null
+++ b/doc/forum/Debugging_of_transfer_considerations/comment_3_508b5b4155f0f9c0c2a7d2e990a1bf92._comment
@@ -0,0 +1,14 @@
+[[!comment format=mdwn
+ username="joey"
+ subject="""comment 3"""
+ date="2017-04-07T20:26:54Z"
+ content="""
+Yes, files in archive/ directories are only dropped from clients once they
+reach an archive. Backup repositories are not considered to be archives.
+
+Of course, you can tweak the preferred content expressions to change this
+behavior.
+
+I don't know if the arhive vs backup distinction makes sense really but
+I have heard of some users doing things that depend on it.
+"""]]
diff --git a/doc/forum/Deduplication_in_direct_mode.mdwn b/doc/forum/Deduplication_in_direct_mode.mdwn
new file mode 100644
index 0000000000..076d0ab5a2
--- /dev/null
+++ b/doc/forum/Deduplication_in_direct_mode.mdwn
@@ -0,0 +1,18 @@
+Hi,
+
+I'm using git-annex across a number of (indirect) repositories, making heavy use of deduplication for organizing files according to various different aspects.
+
+Now I want to keep part of the files also on a VFAT device, which doesn't let me use indirect mode.  In direct mode, however, git-annex "get" or "copy" places a separate copy of each file in the repository, whereas in indirect mode, it would just keep a single copy and maintain a number of (inexpensive) symbolic links.  Since space on the VFAT drive is limited, I would like to just keep one, specific copy, not caring about the others.  If I "drop" an unneeded copy of the file, it also gets replaced by the ASCII "link" in all other places that contained the same file.  Therefore, I can either have multiple copies of the same data or none at all.
+
+Imagine you have a bunch of photos sorted into a directories in meant to make it easy to find them (same file name means same file content):
+
+./photo1.jpg
+./photo2.jpg
+./by-date/2014-10-27/photo1.jpg
+./by-date/2014-10-28/photo2.jpg
+./by-event/holiday-by-the-sea/photo1.jpg
+./by-event/her-birthday/photo2.jpg
+
+I want to keep a copy of ./photo?.jpg in the VFAT repository, but not the other (identical) files.  How do I do that?  Or is there really no way of doing this?
+
+Thanks.
diff --git a/doc/forum/Deduplication_in_direct_mode/comment_1_cb8eaa8c19f5bb9e145a5e4c8995a8ca._comment b/doc/forum/Deduplication_in_direct_mode/comment_1_cb8eaa8c19f5bb9e145a5e4c8995a8ca._comment
new file mode 100644
index 0000000000..63367fcc14
--- /dev/null
+++ b/doc/forum/Deduplication_in_direct_mode/comment_1_cb8eaa8c19f5bb9e145a5e4c8995a8ca._comment
@@ -0,0 +1,22 @@
+[[!comment format=mdwn
+ username="joey"
+ subject="""comment 1"""
+ date="2014-10-31T20:15:58Z"
+ content="""
+There is really no way to do this.
+
+We could consider hard-linking the files, but then modifying one would
+modify the other, which is likely to be confusing. And, FAT doesn't support
+hard links anyway.
+
+I don't want to complicate git-annex's notion of whether an object is
+present or not with the possibility that it might be present for some
+files but not for others. For example, `git annex get` would then need
+to make a copy of content that was already locally present, while
+currently it knows that if the file is locally present, it has nothing to
+do.
+
+I think that the solution is to use either a better filesystem
+which can support the suprerior indirect mode, or to switch your
+repository to use the WORM backend which does not do deduplication.
+"""]]
diff --git a/doc/forum/Default_annex.largefiles.mdwn b/doc/forum/Default_annex.largefiles.mdwn
new file mode 100644
index 0000000000..9da3220e68
--- /dev/null
+++ b/doc/forum/Default_annex.largefiles.mdwn
@@ -0,0 +1 @@
+I'm new to git annex, so if this has been discussed before, please forgive me.  The documentation of annex.largefiles seems to say that all files are added to the annex by default.  However, when I tried it, several of my smaller files were not added to the annex.  I admit that I haven't tried changing this value yet.  Has the default changed?  I'm using the package from Arch AUR git-annex-bin. Possibly this version has a different default?
diff --git a/doc/forum/Default_annex.largefiles/comment_1_74a3ad2388e41f1ff17f64a00485a35a._comment b/doc/forum/Default_annex.largefiles/comment_1_74a3ad2388e41f1ff17f64a00485a35a._comment
new file mode 100644
index 0000000000..d57e4f71d3
--- /dev/null
+++ b/doc/forum/Default_annex.largefiles/comment_1_74a3ad2388e41f1ff17f64a00485a35a._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="joey"
+ subject="""comment 1"""
+ date="2014-10-20T15:26:35Z"
+ content="""
+The default is to treat all files as "large", so any file it sees
+should be added, except for those that are `.gitignored`.
+"""]]
diff --git a/doc/forum/Default_text__47__html_handler.mdwn b/doc/forum/Default_text__47__html_handler.mdwn
new file mode 100644
index 0000000000..a6bf4aaab9
--- /dev/null
+++ b/doc/forum/Default_text__47__html_handler.mdwn
@@ -0,0 +1,2 @@
+I've had to change my default `text/html .html` handler from a text editor to a browser to support the opening sequence of `git annex webapp`. any other way around this? perhaps to have it open the page in the default browser rather than the default text/html handler?
+
diff --git a/doc/forum/Default_text__47__html_handler/comment_1_4730061916c7e12b7a41906152f847ee._comment b/doc/forum/Default_text__47__html_handler/comment_1_4730061916c7e12b7a41906152f847ee._comment
new file mode 100644
index 0000000000..e2424f1ccc
--- /dev/null
+++ b/doc/forum/Default_text__47__html_handler/comment_1_4730061916c7e12b7a41906152f847ee._comment
@@ -0,0 +1,12 @@
+[[!comment format=mdwn
+ username="http://joeyh.name/"
+ ip="4.252.11.120"
+ subject="comment 1"
+ date="2012-11-13T17:25:07Z"
+ content="""
+Not without exposing the secret token to other users in the system by passing it as a parameter to the browser command. Which is why it uses the method it does.
+
+But, you can configure it to use a specific browser in the standard git way:
+
+git config web.browser chromium
+"""]]
diff --git a/doc/forum/Delete_unused_files__47__metadata.mdwn b/doc/forum/Delete_unused_files__47__metadata.mdwn
new file mode 100644
index 0000000000..5c6b413996
--- /dev/null
+++ b/doc/forum/Delete_unused_files__47__metadata.mdwn
@@ -0,0 +1,7 @@
+After moving some files (about 1G, some big and some small files) in and out the annex, I noticed that the size of the repository has grown quite a bit. My empty repository now is over 100 MB (even after  "git annex dropunused ..." and "git gc").
+
+Most of this size is not Git metadata but many small files in the git-annex branch that seem to hold information about files I deleted (even in the other known repositories).
+
+So is there a way to get rid of these useless but space consuming information?
+
+(Maybe there is no (elegant) way to remove the symlink versions from the Git history (is there?), but it would already be nice if those small git-annex metadata files could be removed)
diff --git a/doc/forum/Delete_unused_files__47__metadata/comment_1_3efc19895c8dec89b71ae3778b583fea._comment b/doc/forum/Delete_unused_files__47__metadata/comment_1_3efc19895c8dec89b71ae3778b583fea._comment
new file mode 100644
index 0000000000..c0d779abd1
--- /dev/null
+++ b/doc/forum/Delete_unused_files__47__metadata/comment_1_3efc19895c8dec89b71ae3778b583fea._comment
@@ -0,0 +1,11 @@
+[[!comment format=mdwn
+ username="http://joeyh.name/"
+ subject="comment 1"
+ date="2012-08-07T20:06:26Z"
+ content="""
+You must have quite a number of files and/or location tracking churn to get the branch that large. 
+
+Removing location tracking files for vanished files from the git-annex branch would not save much space.. Only a very little bit of tree object size going forward. The way the git-annex branch is merged does not really allow deleting files from it. And of course git uses space for even deleted files.
+
+It'd be possible to delete the git-annex branch entirely, first backing up git-annex:uuid.log  git-annex:remote.log and git-annex:trust.log and then making a new, clean branch that those files are added back to. Then run git annex fsck on every clone to repopulate the location tracking info for files that still exit. (Note that this would also lose urls stored by `git-annex addurl`.)
+"""]]
diff --git a/doc/forum/Delete_unused_files__47__metadata/comment_2_23597d9468347b3d94257f3c02afe1b8._comment b/doc/forum/Delete_unused_files__47__metadata/comment_2_23597d9468347b3d94257f3c02afe1b8._comment
new file mode 100644
index 0000000000..340debc11e
--- /dev/null
+++ b/doc/forum/Delete_unused_files__47__metadata/comment_2_23597d9468347b3d94257f3c02afe1b8._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="https://www.google.com/accounts/o8/id?id=AItOawkSq2FDpK2n66QRUxtqqdbyDuwgbQmUWus"
+ nickname="Jimmy"
+ subject="comment 2"
+ date="2012-08-08T14:54:42Z"
+ content="""
+I'm sort of getting into the same issue as I run 'git-annex get .' and 'git annex sync' with one of the annexes that I have on a nightly basis to collect data for back up reasons. But it's good to know there is a work around for it.
+"""]]
diff --git a/doc/forum/Deleting_remotes_doesn__39__t_work__8230__.mdwn b/doc/forum/Deleting_remotes_doesn__39__t_work__8230__.mdwn
new file mode 100644
index 0000000000..012afe8ec0
--- /dev/null
+++ b/doc/forum/Deleting_remotes_doesn__39__t_work__8230__.mdwn
@@ -0,0 +1 @@
+I'm trying to get rid of two remotes through the assistant, but when I'm trying to delete it, it doesn't start deletion process, as far as I can tell, since the symbol where it say syncing enabled/disabled is not changed to "delete" or whatever it is.
diff --git a/doc/forum/Deleting_remotes_doesn__39__t_work__8230__/comment_1_01d244f44d439ad08681af88284103c8._comment b/doc/forum/Deleting_remotes_doesn__39__t_work__8230__/comment_1_01d244f44d439ad08681af88284103c8._comment
new file mode 100644
index 0000000000..3e1f171bd0
--- /dev/null
+++ b/doc/forum/Deleting_remotes_doesn__39__t_work__8230__/comment_1_01d244f44d439ad08681af88284103c8._comment
@@ -0,0 +1,7 @@
+[[!comment format=mdwn
+ username="openmedi"
+ subject="comment 1"
+ date="2016-05-12T06:52:50Z"
+ content="""
+This didn't work, because syncing was disabled before trying to delete the remote. Deleting involves putting the remote in the unwanted group and through active syncing remove all the content from it, before deleting it fully. The assistant should be able to handle starting deletion processes, even when synching is disabled. At least there should be a dialogue telling the user that synching is disabled and therefore the deletion process will not start.
+"""]]
diff --git a/doc/forum/Deleting_remotes_doesn__39__t_work__8230__/comment_2_a33c699726e5a913da72671ceb9be989._comment b/doc/forum/Deleting_remotes_doesn__39__t_work__8230__/comment_2_a33c699726e5a913da72671ceb9be989._comment
new file mode 100644
index 0000000000..972db0e058
--- /dev/null
+++ b/doc/forum/Deleting_remotes_doesn__39__t_work__8230__/comment_2_a33c699726e5a913da72671ceb9be989._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="joey"
+ subject="""comment 2"""
+ date="2016-05-12T19:14:05Z"
+ content="""
+Thanks, good analysis! I've opened a bug about this:
+[[bugs/webapp_should_notice_when_remote_deletion_cannot_complete_due_to_not_syncing]]
+"""]]
diff --git a/doc/forum/Desktop_as_transfer_repository.mdwn b/doc/forum/Desktop_as_transfer_repository.mdwn
new file mode 100644
index 0000000000..bd6813e366
--- /dev/null
+++ b/doc/forum/Desktop_as_transfer_repository.mdwn
@@ -0,0 +1,33 @@
+Hi,
+
+I try to achieve this with my podcast repository:
+
+I have three repositories all in indirect mode:
+
+1. desktop
+2. backup on usb disk at desktop
+3. raspberrypi
+
+I want to use the desktop as transfer repository. 
+
+If I add a new url on the raspberrypi I want to sync the content via the desktop repository to the backup repository which is no directly accessible from the raspberrypi.
+
+If I add a new url on desktop the content should be moved to raspberrypi and backup. 
+
+So I don't want to have the content of the files on the desktop. All content should be in backup. 
+
+If content is not in raspberrypi I want to get it from backup via desktop.
+
+What is the best workflow to achieve this?
+
+Currently my problem is that if I do a "git annex sync --content" the content is also copied to desktop. Maybe because the desktop repo is origin? 
+
+- backup is in group backup
+- desktop is in group transfer
+- raspberrypi is in no special group
+
+To do further experiments I need a way to see in what group a repo is, but "git annex show" has no information about this.
+
+TIA
+
+juh
diff --git a/doc/forum/Desktop_as_transfer_repository/comment_1_b211b8f0ab58b8e59ce31d6ed6cb3c8c._comment b/doc/forum/Desktop_as_transfer_repository/comment_1_b211b8f0ab58b8e59ce31d6ed6cb3c8c._comment
new file mode 100644
index 0000000000..afce0e5674
--- /dev/null
+++ b/doc/forum/Desktop_as_transfer_repository/comment_1_b211b8f0ab58b8e59ce31d6ed6cb3c8c._comment
@@ -0,0 +1,46 @@
+[[!comment format=mdwn
+ username="juh"
+ subject="My current workflow"
+ date="2015-11-17T14:47:59Z"
+ content="""
+Ok,
+
+here is my current workflow to clarify my problem:
+
+On raspberrypi:
+
+    Adding new podcasts
+    git annex sync
+
+On desktop:
+
+    git annex sync
+    git annex get --want-get
+
+New podcast content from raspberrypi is downloaded.
+
+In backup:
+
+    git annex sync
+    git annex get --want-get
+
+New podcast content from desktop is downloaded.
+
+On desktop:
+
+    git annex drop
+
+Everything on desktop is dropped.
+
+**Now the problem**
+
+Next time I do 
+
+    git annex get --want-get
+
+on **desktop** all files are downloaded again from either backup or raspberrypi.
+
+What can I do to have only content in desktop that is not in the raspberrypi repository or in the backup repository? 
+
+I thought that this is the behaviour of a transfer repository.
+"""]]
diff --git a/doc/forum/Desktop_as_transfer_repository/comment_2_9a943bcfe7679e469acc6fa1f78324b7._comment b/doc/forum/Desktop_as_transfer_repository/comment_2_9a943bcfe7679e469acc6fa1f78324b7._comment
new file mode 100644
index 0000000000..93e0778e3e
--- /dev/null
+++ b/doc/forum/Desktop_as_transfer_repository/comment_2_9a943bcfe7679e469acc6fa1f78324b7._comment
@@ -0,0 +1,35 @@
+[[!comment format=mdwn
+ username="joey"
+ subject="""comment 2"""
+ date="2015-11-18T17:15:34Z"
+ content="""
+I suspect that if you run `git annex drop --auto` on desktop, it won't drop
+any files. This is because it's configured as a transfer repository, and
+transfer repositories wait for files to reach all client repositories
+before wanting to drop them.
+
+Since you have no client repositories, your transfer repository will prefer
+to hang onto files. In fact, transfer repositories assume there will
+eventually be at least 2 client repositories (otherwise what good would a
+transfer repository between clients be?) and so hang onto files that are
+not in 2 clients, even when less than 2 clients exist yet. That is generally
+useful behavior when setting things up in eg the webapp.
+
+So, the standard preferred content expression for a transfer repository
+won't work in this configuration. Let's instead express the things you want
+as a custom setting.
+
+> If I add a new url on desktop the content should be moved to raspberrypi and backup.
+> 
+> So I don't want to have the content of the files on the desktop. All
+> content should be in backup.
+> 
+> If content is not in raspberrypi I want to get it from backup via desktop.
+
+	git annex wanted desktop "not (inallgroup=backup and inallgroup=client)"
+
+That assumes you put the raspberrypi in the client group, which is what I think it
+basically is. Then desktop will want to get and hold any files that
+have not yet reached all backup and client repositories, at which point
+desktop will drop files.
+"""]]
diff --git a/doc/forum/Desktop_as_transfer_repository/comment_3_17165386df0c8c0085de29659f46f084._comment b/doc/forum/Desktop_as_transfer_repository/comment_3_17165386df0c8c0085de29659f46f084._comment
new file mode 100644
index 0000000000..d29067480c
--- /dev/null
+++ b/doc/forum/Desktop_as_transfer_repository/comment_3_17165386df0c8c0085de29659f46f084._comment
@@ -0,0 +1,7 @@
+[[!comment format=mdwn
+ username="juh"
+ subject="Thanks"
+ date="2015-11-19T08:01:09Z"
+ content="""
+This line seems to do the trick. The flexibility of git annex is amazing – and frightening. I fear I would have never found the line by myself.
+"""]]
diff --git a/doc/forum/Desktop_as_transfer_repository/comment_4_986e7d7bb536685f48731ffbbab4dc7c._comment b/doc/forum/Desktop_as_transfer_repository/comment_4_986e7d7bb536685f48731ffbbab4dc7c._comment
new file mode 100644
index 0000000000..89b773a5f1
--- /dev/null
+++ b/doc/forum/Desktop_as_transfer_repository/comment_4_986e7d7bb536685f48731ffbbab4dc7c._comment
@@ -0,0 +1,10 @@
+[[!comment format=mdwn
+ username="joey"
+ subject="""comment 4"""
+ date="2015-12-02T17:29:25Z"
+ content="""
+It should, in theory, be possible to read [[preferred_content]] and learn
+how to write such an expression. Of course, it is a limited form of
+programming, and so not for everyone. Which is why git-annex ships with
+standard expressions to cover as many common cases as we can think of.
+"""]]
diff --git a/doc/forum/Detached_git_work_tree__63__.mdwn b/doc/forum/Detached_git_work_tree__63__.mdwn
new file mode 100644
index 0000000000..3c1f6ae249
--- /dev/null
+++ b/doc/forum/Detached_git_work_tree__63__.mdwn
@@ -0,0 +1,11 @@
+Does git-annex (safely) handle detached work trees?
+
+That is, in git I can set `GIT_WORK_TREE=/dir/A` and `GIT_DIR=/dir/B` in the environment and have all my .git stuff in /dir/B and all my files in /dir/A.
+
+I can see this coming in useful for a few situations, but in particular for difficult file systems - like SMB or old implementations of NFS.
+
+In my particular case I have a Drobo (something like a proprietary NAS).  The Drobo is linux based, but by default mounts as a samba share or if you install `unfsd` it can be mounted via NFS.  Unfortunately, the nfs is v3 and doesn't allow locks, so git-annex barfs.  :-(
+
+What I'd like to be able to do is have a direct mode annex on the drobo, with the git directory sitting on one of my linux machines.  That machine would be the only one that would directly access the drobo data as an annex but other systems that look at the drobo would see what looks like a normal directory structure; for example my media centre - `mythtv` naturally! - would see "normal" names for my music collection, not SHA256 hashes...
+
+I guess there would be an issue if there were different `GIT_DIR`s pointing to the one `GIT_WORK_TREE`, but that is a caveat emptor IMHO. 
diff --git a/doc/forum/Detached_git_work_tree__63__/comment_10_656c737772bf92be2c7a2f33bd2bb0f0._comment b/doc/forum/Detached_git_work_tree__63__/comment_10_656c737772bf92be2c7a2f33bd2bb0f0._comment
new file mode 100644
index 0000000000..0bf9c12f45
--- /dev/null
+++ b/doc/forum/Detached_git_work_tree__63__/comment_10_656c737772bf92be2c7a2f33bd2bb0f0._comment
@@ -0,0 +1,10 @@
+[[!comment format=mdwn
+ username="https://www.google.com/accounts/o8/id?id=AItOawlRThEwuPnr8_bcuuCTQ0rQd3w6AfeMiLY"
+ nickname="Alex"
+ subject="comment 10"
+ date="2013-02-24T12:04:21Z"
+ content="""
+Been using it a few hours now and it works like a charm - first thing I did was get a sitemap of the LCA and FOSDEM talks and use addurl --fast on them, since I've been wanting to do that since I saw the lightning talk last year at FOSDEM.
+
+Thanks again for making this awesome utility!
+"""]]
diff --git a/doc/forum/Detached_git_work_tree__63__/comment_1_28ac35a325fba250721d9f1b7c994960._comment b/doc/forum/Detached_git_work_tree__63__/comment_1_28ac35a325fba250721d9f1b7c994960._comment
new file mode 100644
index 0000000000..83721eac36
--- /dev/null
+++ b/doc/forum/Detached_git_work_tree__63__/comment_1_28ac35a325fba250721d9f1b7c994960._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="http://joeyh.name/"
+ ip="4.154.7.238"
+ subject="comment 1"
+ date="2013-01-14T16:32:51Z"
+ content="""
+Recent versions of git-annex support these variables. I haven't tested it extensively, but AFAIK it works.
+"""]]
diff --git a/doc/forum/Detached_git_work_tree__63__/comment_2_7128c26bbc8efea04a5a317edf0ca9f2._comment b/doc/forum/Detached_git_work_tree__63__/comment_2_7128c26bbc8efea04a5a317edf0ca9f2._comment
new file mode 100644
index 0000000000..e9154c6933
--- /dev/null
+++ b/doc/forum/Detached_git_work_tree__63__/comment_2_7128c26bbc8efea04a5a317edf0ca9f2._comment
@@ -0,0 +1,13 @@
+[[!comment format=mdwn
+ username="https://www.google.com/accounts/o8/id?id=AItOawlRThEwuPnr8_bcuuCTQ0rQd3w6AfeMiLY"
+ nickname="Alex"
+ subject="Having issues here"
+ date="2013-02-23T14:44:15Z"
+ content="""
+I'm trying to use git-annex with vcsh, and it doesn't seem to be respecting the GIT_WORK_TREE and GIT_DIR variables vcsh sets. If I try it manually, the same happens.
+
+Any time I try to invoke a git-annex subcommand (init, add) via vcsh (vcsh $repo annex init/add/etc) it spits out an error that \"git-annex: Not in a git repository.\"
+
+I'd really like this to work, so that I can manage the bigger files in my home with topic-based annexes alongside plain-git repos for dotfiles.
+
+"""]]
diff --git a/doc/forum/Detached_git_work_tree__63__/comment_3_a3c22f905748ff2c803e8621c74a87a0._comment b/doc/forum/Detached_git_work_tree__63__/comment_3_a3c22f905748ff2c803e8621c74a87a0._comment
new file mode 100644
index 0000000000..0e59edd76e
--- /dev/null
+++ b/doc/forum/Detached_git_work_tree__63__/comment_3_a3c22f905748ff2c803e8621c74a87a0._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="http://joeyh.name/"
+ ip="4.153.253.75"
+ subject="comment 3"
+ date="2013-02-23T15:01:57Z"
+ content="""
+@Alex, you forgot to say what version of git-annex you are having trouble with.
+"""]]
diff --git a/doc/forum/Detached_git_work_tree__63__/comment_4_8063921241760458349e7cb0cadf3d4e._comment b/doc/forum/Detached_git_work_tree__63__/comment_4_8063921241760458349e7cb0cadf3d4e._comment
new file mode 100644
index 0000000000..b0d8fcd419
--- /dev/null
+++ b/doc/forum/Detached_git_work_tree__63__/comment_4_8063921241760458349e7cb0cadf3d4e._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="https://www.google.com/accounts/o8/id?id=AItOawlRThEwuPnr8_bcuuCTQ0rQd3w6AfeMiLY"
+ nickname="Alex"
+ subject="comment 4"
+ date="2013-02-23T15:07:52Z"
+ content="""
+3.20130216.1 from Hackage - I packaged it for Exherbo specifically to try this, and installed it about two hours ago.
+"""]]
diff --git a/doc/forum/Detached_git_work_tree__63__/comment_5_4510a787255cb03e7d0c3e7b830b7d52._comment b/doc/forum/Detached_git_work_tree__63__/comment_5_4510a787255cb03e7d0c3e7b830b7d52._comment
new file mode 100644
index 0000000000..693e8ef4dd
--- /dev/null
+++ b/doc/forum/Detached_git_work_tree__63__/comment_5_4510a787255cb03e7d0c3e7b830b7d52._comment
@@ -0,0 +1,16 @@
+[[!comment format=mdwn
+ username="http://joeyh.name/"
+ ip="4.153.253.75"
+ subject="comment 5"
+ date="2013-02-23T16:44:48Z"
+ content="""
+So, I found and fixed that bug. It really seems to be working now. Although I've thought that at least twice before!
+
+There is an important caveat though, with using `GIT_DIR` and/or `GIT_WORK_TREE`.
+
+git-annex needs to check in symlinks that point at the git repository. If using `GIT_DIR`, those symlinks do not look like \"-> .git/annex/objects/\"; they instead point off to some git repository elsewhere, as a relative path. For example they could look like \"-> ../gitrepo/annex/objects/\". Similar when using `GIT_WORK_TREE`.
+
+But this is a problem if you want to use the same git repo elsewhere, with a git work tree and repo that are not set up the same relative to one-another, because the links will all be wrong. So, if you decide to use `GIT_DIR` or `GIT_WORK_TREE` with git-annex, any clone of the repository will need to place the git directory in the same place relative to the working tree. 
+
+If you later want to change the relative paths between git directory and work tree, it would need to be changed in all clones of the repository, and then you could use `git annex fix` to update the symlinks.
+"""]]
diff --git a/doc/forum/Detached_git_work_tree__63__/comment_6_ffd9c67ecc5b46ae98996018573f5591._comment b/doc/forum/Detached_git_work_tree__63__/comment_6_ffd9c67ecc5b46ae98996018573f5591._comment
new file mode 100644
index 0000000000..faecbf2f55
--- /dev/null
+++ b/doc/forum/Detached_git_work_tree__63__/comment_6_ffd9c67ecc5b46ae98996018573f5591._comment
@@ -0,0 +1,10 @@
+[[!comment format=mdwn
+ username="https://www.google.com/accounts/o8/id?id=AItOawlRThEwuPnr8_bcuuCTQ0rQd3w6AfeMiLY"
+ nickname="Alex"
+ subject="comment 6"
+ date="2013-02-23T21:23:18Z"
+ content="""
+Cool, thanks! The caveat shouldn't be much of a problem for my use case - with vcsh, the workdir is always $HOME, and I'm using the recommended mr-based workflow so the gitdir will always be ~/.config/vcsh/repo.d/$name.git in each clone.
+
+EDIT: fix erroneous path
+"""]]
diff --git a/doc/forum/Detached_git_work_tree__63__/comment_7_36ca007643c983604fc4aed6ec8cb3d2._comment b/doc/forum/Detached_git_work_tree__63__/comment_7_36ca007643c983604fc4aed6ec8cb3d2._comment
new file mode 100644
index 0000000000..9e5d241a25
--- /dev/null
+++ b/doc/forum/Detached_git_work_tree__63__/comment_7_36ca007643c983604fc4aed6ec8cb3d2._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="https://www.google.com/accounts/o8/id?id=AItOawlRThEwuPnr8_bcuuCTQ0rQd3w6AfeMiLY"
+ nickname="Alex"
+ subject="Further possible issues"
+ date="2013-02-23T23:17:43Z"
+ content="""
+Mm, looking at the actual code changes it may still not work with vcsh - since multiple GIT_DIRS share one GIT_WORKTREE, some symlinks will point to ~/.config/vcsh/repo.d/foo.git and some will point to ~/.config/vcsh/repo.d/bar.git. When git annex is invoked with GIT_DIR='~/.config/vcsh/repo.d/bar.git', with the changes you made files annexed by ~/.config/vcsh/repo.d/foo.git will still match isLinkToAnnex.
+"""]]
diff --git a/doc/forum/Detached_git_work_tree__63__/comment_8_b7a2da4fbace7156e11c48a496a19dc9._comment b/doc/forum/Detached_git_work_tree__63__/comment_8_b7a2da4fbace7156e11c48a496a19dc9._comment
new file mode 100644
index 0000000000..7d9d13a7a6
--- /dev/null
+++ b/doc/forum/Detached_git_work_tree__63__/comment_8_b7a2da4fbace7156e11c48a496a19dc9._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="http://joeyh.name/"
+ ip="4.152.108.210"
+ subject="comment 8"
+ date="2013-02-23T23:23:01Z"
+ content="""
+That should not be a problem, because git-annex only acts on files that `git ls-files` lists.
+"""]]
diff --git a/doc/forum/Detached_git_work_tree__63__/comment_9_f9fa237a693d28178f0451799209f7e2._comment b/doc/forum/Detached_git_work_tree__63__/comment_9_f9fa237a693d28178f0451799209f7e2._comment
new file mode 100644
index 0000000000..16702141c0
--- /dev/null
+++ b/doc/forum/Detached_git_work_tree__63__/comment_9_f9fa237a693d28178f0451799209f7e2._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="https://www.google.com/accounts/o8/id?id=AItOawlRThEwuPnr8_bcuuCTQ0rQd3w6AfeMiLY"
+ nickname="Alex"
+ subject="comment 9"
+ date="2013-02-24T00:52:52Z"
+ content="""
+Alright then, sounds like it should be perfect! Thanks for the quick response!
+"""]]
diff --git a/doc/forum/Difference_between_copy__44___move_and_get__63__.mdwn b/doc/forum/Difference_between_copy__44___move_and_get__63__.mdwn
new file mode 100644
index 0000000000..a94281a1fc
--- /dev/null
+++ b/doc/forum/Difference_between_copy__44___move_and_get__63__.mdwn
@@ -0,0 +1,24 @@
+I'm starting to experiment with git annex.  I'd like to use it for a centralized git repo that will be checked out often, but the clones will rarely need some large binary files (used for testing).  Therefore, I've set up a centralized/bare git repo and a clone of that repo using the instructions at [centralized_git_repository_tutorial](http://git-annex.branchable.com/tips/centralized_git_repository_tutorial/) and [bare_repositories](http://git-annex.branchable.com/bare_repositories/).  I've added some files to the annex in the clone.
+
+I'm struggling to understand the difference between copy, move, and get.  Here's a sequence of commands:
+
+    >> git annex add shared/1bel.maegz
+    >> git commit -m "added first file"
+    >> git push
+    >> git annex move shared/1bel.maegz --to origin
+    ## Now it no longer exists in my local repo
+    >> git annex get shared/1bel.maegz
+    fails.
+    >> git annex get shared/1bel.maegz --from origin
+    fails.
+    >> git annex copy shared/1bel.maegz --from origin
+    fails.
+    >> git annex move shared/1bel.maegz --from origin
+    succeeds! Now I have the file in my clone.
+
+Each failure message is:
+
+    fatal: Could not switch to '../.git/annex/objects/W8/gZ/SHA256-s99196--62874e9b58e652c9c01e796c2bf38b2234a80e0cef95c185bb7f0857d9765df2': No such file or directory
+    git-annex: : hGetLine: end of file
+
+How are copy, move, and get different? Which one *should* I be using to move my large data into the central (bare) repo?  Will it then be available to other clones?
diff --git a/doc/forum/Difference_between_copy__44___move_and_get__63__/comment_1_26ee8192af3a62178c1ccf17c6da5ca5._comment b/doc/forum/Difference_between_copy__44___move_and_get__63__/comment_1_26ee8192af3a62178c1ccf17c6da5ca5._comment
new file mode 100644
index 0000000000..fb1aed5da1
--- /dev/null
+++ b/doc/forum/Difference_between_copy__44___move_and_get__63__/comment_1_26ee8192af3a62178c1ccf17c6da5ca5._comment
@@ -0,0 +1,10 @@
+[[!comment format=mdwn
+ username="http://joeyh.name/"
+ nickname="joey"
+ subject="comment 1"
+ date="2013-03-09T02:00:11Z"
+ content="""
+There is no particular reason to use any of copy, move, or get. Use which ever command makes sense at the time.
+
+The problem you're encoutering is that you have an old, and broken version of git-annex installed. Upgrade and the error message will go away and both get and copy, in your example, will work.
+"""]]
diff --git a/doc/forum/Different_annexes_pointing_to_same_special_remote__63__.mdwn b/doc/forum/Different_annexes_pointing_to_same_special_remote__63__.mdwn
new file mode 100644
index 0000000000..8413a6e08a
--- /dev/null
+++ b/doc/forum/Different_annexes_pointing_to_same_special_remote__63__.mdwn
@@ -0,0 +1,6 @@
+Is there likely to be any problem in pointing different annexes to the same special remote (i.e. rsync/box.com/etc.) ?
+
+As the objects are stored based on their SHA256 key the expectation is that the chance of collision is is small.
+
+The only problem I can foresee is where the same content is stored in more than one annex and it is deleted in the remote in one annex, but not the other - there won't be any protection against that, but for non-overlapping content this risk should be negligible.
+
diff --git a/doc/forum/Different_annexes_pointing_to_same_special_remote__63__/comment_1_359f46805e6508d03aadd90429937546._comment b/doc/forum/Different_annexes_pointing_to_same_special_remote__63__/comment_1_359f46805e6508d03aadd90429937546._comment
new file mode 100644
index 0000000000..282ed82c8b
--- /dev/null
+++ b/doc/forum/Different_annexes_pointing_to_same_special_remote__63__/comment_1_359f46805e6508d03aadd90429937546._comment
@@ -0,0 +1,10 @@
+[[!comment format=mdwn
+ username="http://joeyh.name/"
+ ip="4.152.108.210"
+ subject="comment 1"
+ date="2013-02-24T21:51:19Z"
+ content="""
+You're basically right. Additionally, if you use encryption, the keys used on the remote are themselves encrypted and so even the same content will result in different keys.
+
+With this said, it's easy to use a subdirectory of a rsync or box.com remote, and I'd rather recommend doing that, rather than jumbling multiple repositories data into a single directory.
+"""]]
diff --git a/doc/forum/Different_branches_for_different_devices__63__.mdwn b/doc/forum/Different_branches_for_different_devices__63__.mdwn
new file mode 100644
index 0000000000..4b9154aa5a
--- /dev/null
+++ b/doc/forum/Different_branches_for_different_devices__63__.mdwn
@@ -0,0 +1,17 @@
+Hi,
+
+I'm currently trying to figure out how to best use git-annex for my needs. And I have a specific use case on which I can't really wrap my head around:
+
+Having a large photo library that collects all photos from all my devices.
+
+What's special about that? Some devices (e.g. mobile phone, laptop, laptop at work) should have/keep their own "view" on the data. Meaning that, e.g., on my laptop at work I don't want to have private pictures, but I might want to have work pictures on my private laptop. Similar for my mobile phone: I want to have all photos for my mobile phone on my laptop, but I don't want all pictures I have ever made on my (tiny) phone.
+
+So, first of all, git-annex supports get, drop, preferred content, etc. which makes it perfect for this use case. Still there is one problem: It's not only that I don't want the content on some devices, I also don't want the respective directory names, simlinks, etc. Say, a directory name like "crazy party pictures" should not be visible on my work laptop, at all...
+
+For me, using directories like "phone", "work", etc. would be a start and a reasonable way to sort which content should be available on these devices. However, as far as I know, it's not possible with GIT to just check out a single (or a few) directories. So I thought different branches could be what I want. But I don't understand how I can copy/move content from one branch to another. Or even, how can I see all the content at once on some machines (e.g. private laptop) without switching branches all the time. I could have several copies of the same repo all set to a different branch, but does this make sense..?
+
+I think what I want is similar to git-annex view, but only for its filter capabilities. Still, subdirectories should work the same as before and filenames should not change. Also the repository should still be fully functional and behaving as usual but just hiding some of the content.
+
+Do you people have similar use cases? What are your best practices?
+
+Best, Mario
diff --git a/doc/forum/Different_branches_for_different_devices__63__/comment_1_a7d2c26298e8da715e1deb13baaf002e._comment b/doc/forum/Different_branches_for_different_devices__63__/comment_1_a7d2c26298e8da715e1deb13baaf002e._comment
new file mode 100644
index 0000000000..edeee3b435
--- /dev/null
+++ b/doc/forum/Different_branches_for_different_devices__63__/comment_1_a7d2c26298e8da715e1deb13baaf002e._comment
@@ -0,0 +1,9 @@
+[[!comment format=mdwn
+ username="joey"
+ subject="""comment 1"""
+ date="2017-05-24T17:41:08Z"
+ content="""
+Don't underestimate the simple power of branches for this kind of thing.
+If you have a limited `phone` branch, you can `git merge` it into a larger
+branch at home.
+"""]]
diff --git a/doc/forum/Different_branches_for_different_devices__63__/comment_2_214c71d2ab43e4d1bf564ea8337909b7._comment b/doc/forum/Different_branches_for_different_devices__63__/comment_2_214c71d2ab43e4d1bf564ea8337909b7._comment
new file mode 100644
index 0000000000..f06be601af
--- /dev/null
+++ b/doc/forum/Different_branches_for_different_devices__63__/comment_2_214c71d2ab43e4d1bf564ea8337909b7._comment
@@ -0,0 +1,10 @@
+[[!comment format=mdwn
+ username="mario"
+ avatar="http://cdn.libravatar.org/avatar/4c63b0935789d29210d0bd8cad8d7ac7"
+ subject="comment 2"
+ date="2017-05-25T12:01:46Z"
+ content="""
+Hi, thanks for your help.
+
+But I don't get it entirely. Let's say I move photos from the phone folder into another folder (on the laptop, after merging, in the master branch). How do I propagate these changes back into the phone branch? Since, I guess, merging would only work one-way in this setting. Or let's say I want to push a new picture from my laptop into the phone branch, how would I do this?
+"""]]
diff --git a/doc/forum/Different_branches_for_different_devices__63__/comment_3_534f1e2cfcc0b6d184ef4968ec33dc89._comment b/doc/forum/Different_branches_for_different_devices__63__/comment_3_534f1e2cfcc0b6d184ef4968ec33dc89._comment
new file mode 100644
index 0000000000..54642fdb2f
--- /dev/null
+++ b/doc/forum/Different_branches_for_different_devices__63__/comment_3_534f1e2cfcc0b6d184ef4968ec33dc89._comment
@@ -0,0 +1,10 @@
+[[!comment format=mdwn
+ username="joey"
+ subject="""comment 3"""
+ date="2017-05-25T17:44:59Z"
+ content="""
+You'd need to checkout the phone branch on the laptop and make the changes
+to it. So there can be some extra work involved since you have to maintain
+two branches. In some cases `git cherry-pick` could be used to pick a
+change from one branch to the other.
+"""]]
diff --git a/doc/forum/Direct_Mode_-_Restore_file_from_Full_Backup_Repository__63__.mdwn b/doc/forum/Direct_Mode_-_Restore_file_from_Full_Backup_Repository__63__.mdwn
new file mode 100644
index 0000000000..47c327f1be
--- /dev/null
+++ b/doc/forum/Direct_Mode_-_Restore_file_from_Full_Backup_Repository__63__.mdwn
@@ -0,0 +1,14 @@
+Hi, I'm using the webapp and created a repository on my local computer. Then I created another remote repository (encrypted remote with gcrypt), this remote repository is selected as type "full backup". 
+
+I've added some files to the local repository, then changed some of them and watched the sync happen. Then I deleted some files, and these also get synced to the remote.
+
+Now, how can I recover those files from the foreign repo, using the webapp or the command line? I could not find any solution.
+
+I tried:
+git log --diff-filter=D --summary
+and then
+git checkout 488408bfcd58eced685d9e3ca5daf55250850f5d -- .
+to recover the file listed in this remote but got the following response:
+fatal: This operation must be run in a work tree
+
+What do I miss and how does the "Restore" part work when using "full backup" remote repository?
diff --git a/doc/forum/Direct_Mode_-_Restore_file_from_Full_Backup_Repository__63__/comment_1_67ac7e8b53a4374baf640d32dac79030._comment b/doc/forum/Direct_Mode_-_Restore_file_from_Full_Backup_Repository__63__/comment_1_67ac7e8b53a4374baf640d32dac79030._comment
new file mode 100644
index 0000000000..21adc4ef99
--- /dev/null
+++ b/doc/forum/Direct_Mode_-_Restore_file_from_Full_Backup_Repository__63__/comment_1_67ac7e8b53a4374baf640d32dac79030._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="http://joeyh.name/"
+ ip="209.250.56.144"
+ subject="comment 1"
+ date="2014-09-25T15:52:04Z"
+ content="""
+Yes, you need to use git to either revert the repository to a previous version that had the file, or perhaps just revert the commit where the file was deleted. Either way, this requires letting git modify files in the repository, which is prevented by direct mode. So, if you can `git annex indirect` to switch to indirect mode, your git commands will work then.
+"""]]
diff --git a/doc/forum/Direct_Mode_-_Restore_file_from_Full_Backup_Repository__63__/comment_2_eb6df2bfcb3892ae22050a8c5f67ee90._comment b/doc/forum/Direct_Mode_-_Restore_file_from_Full_Backup_Repository__63__/comment_2_eb6df2bfcb3892ae22050a8c5f67ee90._comment
new file mode 100644
index 0000000000..5963d4b747
--- /dev/null
+++ b/doc/forum/Direct_Mode_-_Restore_file_from_Full_Backup_Repository__63__/comment_2_eb6df2bfcb3892ae22050a8c5f67ee90._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="https://www.google.com/accounts/o8/id?id=AItOawmK0703vNSIQsP1mGf-4MAPnsBZiSc6yVo"
+ nickname="Emre"
+ subject="comment 2"
+ date="2014-09-25T19:25:06Z"
+ content="""
+Thanks Joeyh, of course i guess checking stuff in git would then lose my timestamps as in the previous post. I recommend, as a feature request, to make file recovery a bit easier if possible. I'm not a git expert and definitely wont use command line stuff for doing this on the long run, unless some more intuitive commands like \"git annex list-old-versions\" or \"git annex show-deleted\" and then \"git annex restore \"filename\" version_no  etc.
+"""]]
diff --git a/doc/forum/Direct_Mode_-_Restore_file_from_Full_Backup_Repository__63__/comment_3_15f36487383a631f16e041e2885c44ec._comment b/doc/forum/Direct_Mode_-_Restore_file_from_Full_Backup_Repository__63__/comment_3_15f36487383a631f16e041e2885c44ec._comment
new file mode 100644
index 0000000000..99dd7e81be
--- /dev/null
+++ b/doc/forum/Direct_Mode_-_Restore_file_from_Full_Backup_Repository__63__/comment_3_15f36487383a631f16e041e2885c44ec._comment
@@ -0,0 +1,10 @@
+[[!comment format=mdwn
+ username="http://joeyh.name/"
+ ip="209.250.56.54"
+ subject="comment 3"
+ date="2014-10-02T15:40:40Z"
+ content="""
+Using git does not affect the timestamps or other metadata of files stored by git-annex, which git knows nothing about. It will perhaps change the timestamps of the symlinks that git changes. It you really wanted to avoid that, you could `git clone` the repository and do all the git commands in the clone of the repository, without touching the original repo.
+
+Whether commands like `git checkout` and `git revert` are intuitive depends on how intuitive you find git, I suppose. It sure seems more intuitive to me to reuse git commands that work just fine, rather than adding a whole new set of commands.
+"""]]
diff --git a/doc/forum/Direct_Mode_-_Restore_file_from_Full_Backup_Repository__63__/comment_4_9293831aff5b6cef490f65d03638d34d._comment b/doc/forum/Direct_Mode_-_Restore_file_from_Full_Backup_Repository__63__/comment_4_9293831aff5b6cef490f65d03638d34d._comment
new file mode 100644
index 0000000000..176c5d915b
--- /dev/null
+++ b/doc/forum/Direct_Mode_-_Restore_file_from_Full_Backup_Repository__63__/comment_4_9293831aff5b6cef490f65d03638d34d._comment
@@ -0,0 +1,12 @@
+[[!comment format=mdwn
+ username="https://www.google.com/accounts/o8/id?id=AItOawmK0703vNSIQsP1mGf-4MAPnsBZiSc6yVo"
+ nickname="Emre"
+ subject="comment 4"
+ date="2014-10-02T21:57:02Z"
+ content="""
+If git-annex target user community are the ones with some level of git know-how, then you may be right. But I'm not one of them. The only time I use git is to clone a repo from the internet when I want to compile latest source code. Nothing more. Reverting etc are alien to me. 
+
+Besides, what happens if there are multiple files in a commit but you want to revert only one file? This may be a stupid question if git is able to filter files when reverting, but accept my apologies if so, since I do not know git.
+
+I simply want to use git-annex to sync files, keeping some versions of the files in case needed.  
+"""]]
diff --git a/doc/forum/Direct_special_remotes.mdwn b/doc/forum/Direct_special_remotes.mdwn
new file mode 100644
index 0000000000..bf80d57f85
--- /dev/null
+++ b/doc/forum/Direct_special_remotes.mdwn
@@ -0,0 +1,26 @@
+I have a NAS at home which I access both as AFP/SMB shares and thru ssh/rsync.  Now, I'd like to keep on using the shares as before, i.e., browsable via AFP/SMB with conventional file names.  Ideally, I'd also like to git-annex some NAS shares, preferably, in direct mode. However, it seems out of the question to install git-annex on the NAS (hopefully, I'm wrong about this in the long run).
+
+Two **non-special remote setups** would be:
+
+1. Mount the share and turn it into a direct mode git annex repo.  Does anybody have experience with this?  I'd suspect this to be very inefficient due to the use of all the files in .git over AFP/SMB.  Configuration as a WORM backend seems to be advised? (Edit: Well, I just tried this and 'git annex init' failed as described in this [forum post](http://git-annex.branchable.com/forum/Can__39__t_init_git_annex/).  So AFP/SMB seem to be non-starters. :-(  Furthermore, AFP/SMB are immediately detected as crippled and set to direct mode automatically.)
+2. Same as 1. but with a local GIT_DIR.  This should work by having .git on the NAS link to the local GIT_DIR.
+
+Alternatively, I could treat my NAS as a [**web special remote**](http://git-annex.branchable.com/tips/using_the_web_as_a_special_remote/).  Some URL schemes come to mind:
+
+* **file:**  This would benefit from some wish list items ([recursive directory remote setup/addurl](http://git-annex.branchable.com/todo/wishlist:_recursive_directory_remote_setup__47__addurl/)).
+* **rsync:** AFAIK not implemented (yet?) as an option for web special remotes.
+* **sftp:** (Might also include ssh access.)
+
+The problem is that the "web semantics" don't really work in my use case:
+
+* Files might change/move on the NAS.
+* I'd like changes (e.g., renamed files) in my local repo to propagate to the NAS.  Currently, git-annex would use git push for this purpose IIUC, however that's not available on the NAS...
+* the web semantics seem to imply that there is exactly one "web" repository (and the URL is fixed)
+
+All of these indicate a mismatch between my use case and web special remotes.
+
+Hence my question: **Would something like a "direct special remote" make sense?**
+
+As a starting point I'd look at a setup similar to 2. above, i.e., a remote "working copy" with local GIT_DIR.  Except that instead of a whole local .git directory a branch in an existing .git dir might be more appropriate...
+
+--Chris
diff --git a/doc/forum/Direct_special_remotes/comment_1_50357130a1c57ad2fab70f71925faf02._comment b/doc/forum/Direct_special_remotes/comment_1_50357130a1c57ad2fab70f71925faf02._comment
new file mode 100644
index 0000000000..09e9771280
--- /dev/null
+++ b/doc/forum/Direct_special_remotes/comment_1_50357130a1c57ad2fab70f71925faf02._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="https://www.google.com/accounts/o8/id?id=AItOawmVICFY2CDP08xdsPr3cgmScomy9HA-1sk"
+ nickname="Andrew"
+ subject="comment 1"
+ date="2013-04-04T00:33:15Z"
+ content="""
+\"Dumb NAS in direct mode, without git-annex installed\" would greatly expand the possible use cases IMO - it would transform git-annex from a distributed filesystem to a general sync tool. Or maybe I'm dreaming.
+"""]]
diff --git a/doc/forum/Direct_special_remotes/comment_2_e94a722ca056a068bcc16eb822008602._comment b/doc/forum/Direct_special_remotes/comment_2_e94a722ca056a068bcc16eb822008602._comment
new file mode 100644
index 0000000000..4b4d8632c8
--- /dev/null
+++ b/doc/forum/Direct_special_remotes/comment_2_e94a722ca056a068bcc16eb822008602._comment
@@ -0,0 +1,18 @@
+[[!comment format=mdwn
+ username="https://pradermecker.myopenid.com/"
+ ip="81.244.179.73"
+ subject="Local rsync ssh remote (I am confused)"
+ date="2013-04-13T17:54:06Z"
+ content="""
+If I understand this right, it is not possible to use git-annex as a simple wrapper around rsync.
+
+My use case is this: I have a music folder on a plug computer running debian squeeze. The plug computer acts as a streaming music server. My wish is to create a local ssh remote to have my laptop git-annex music repo in sync with the plug computer. So ideally I don't want to install git-annex on the plug computer (but git is there so a git bare repo can be used).
+
+According to the UI, if I use \"Pairing with a local computer\", both computers needs to have git-annex installed. If I try \"Remote server\" either I need both machines git-annex aware or I need the content of the plug computer to be encrypted (which is obviously not what I want in this case).
+
+Is the use case supported by the UI ? Should I try with the CLI ?
+
+Thanks for your help
+
+
+"""]]
diff --git a/doc/forum/Direct_special_remotes/comment_4_187036bbfee0508e2914afb51ead3c71._comment b/doc/forum/Direct_special_remotes/comment_4_187036bbfee0508e2914afb51ead3c71._comment
new file mode 100644
index 0000000000..1f62871115
--- /dev/null
+++ b/doc/forum/Direct_special_remotes/comment_4_187036bbfee0508e2914afb51ead3c71._comment
@@ -0,0 +1,16 @@
+[[!comment format=mdwn
+ username="https://pradermecker.myopenid.com/"
+ ip="81.244.190.181"
+ subject="comment 4"
+ date="2013-04-14T09:01:27Z"
+ content="""
+Thanks !
+
+Before trying the CLI, I will install have a go with installing git-annex on the plug computer.
+
+Do you know by any chance if version 3.20120629~bpo60+2 (the available version on squeeze-backport) will be compatible with the current latest release (4.20130405) ?
+
+I guess I am asking if version 3 is compatible with version 4.
+
+
+"""]]
diff --git a/doc/forum/Direct_special_remotes/comment_4_6bfbf60f2061d49b7d34c844e7e1dea2._comment b/doc/forum/Direct_special_remotes/comment_4_6bfbf60f2061d49b7d34c844e7e1dea2._comment
new file mode 100644
index 0000000000..86621a00e3
--- /dev/null
+++ b/doc/forum/Direct_special_remotes/comment_4_6bfbf60f2061d49b7d34c844e7e1dea2._comment
@@ -0,0 +1,66 @@
+[[!comment format=mdwn
+ username="andy"
+ ip="99.48.75.171"
+ subject="comment 4"
+ date="2013-04-14T01:07:19Z"
+ content="""
+@pradermecker: I think that the CLI would be able to do what you're describing (to some extent), but the issue is that the filenames it creates will be hashes, and not in the folder pattern that is on your laptop. Of course, if the plug computer uses file metadata to determine file info, and will search a directory tree, then it probably won't care that the files are named strange things.
+
+As a quick example:
+
+I have a repo with three files in it:
+
+    $ ls -R
+    .:
+    File 1  File 2  File 3
+
+I copy those files to a directory remote:
+
+    $ git annex copy --to=test-dir *
+    copy File 1 (to test-dir...) 
+    ok                      
+    copy File 2 (to test-dir...) 
+    ok                      
+    copy File 3 (to test-dir...) 
+    ok                      
+    (Recording state in git...)
+
+Then I look at what's in my directory remote:
+
+    $ cd ../remote
+    $ ls -R
+    .:
+    0d4  d35  f15
+
+    ./0d4:
+    d37
+
+    ./0d4/d37:
+    SHA256-s7--a940d8aadc02f798331b2d46f1a8ad2c9821783060f4a0810da42bf785855c1c
+
+    ./0d4/d37/SHA256-s7--a940d8aadc02f798331b2d46f1a8ad2c9821783060f4a0810da42bf785855c1c:
+    SHA256-s7--a940d8aadc02f798331b2d46f1a8ad2c9821783060f4a0810da42bf785855c1c
+
+    ./d35:
+    55c
+
+    ./d35/55c:
+    SHA256-s7--ab1ad6a49c022416008887464b8dc03b523b9e81530cf47d1f6f7712c1b30955
+
+    ./d35/55c/SHA256-s7--ab1ad6a49c022416008887464b8dc03b523b9e81530cf47d1f6f7712c1b30955:
+    SHA256-s7--ab1ad6a49c022416008887464b8dc03b523b9e81530cf47d1f6f7712c1b30955
+
+    ./f15:
+    3f2
+
+    ./f15/3f2:
+    SHA256-s7--d1993f1115215aa6389e33fa9979fb39bb29e5bed14661baf1cf6af3182f0164
+
+    ./f15/3f2/SHA256-s7--d1993f1115215aa6389e33fa9979fb39bb29e5bed14661baf1cf6af3182f0164:
+    SHA256-s7--d1993f1115215aa6389e33fa9979fb39bb29e5bed14661baf1cf6af3182f0164
+
+I haven't checked, but I think that the assistant will be able to use a special remote that you create from the CLI once you create it.
+
+To create a remote without encryption using the CLI, try a command of the form `git annex initremote remote-name blah blah blah encryption=none`
+Also see the information at [[special_remotes/directory]], [[special_remotes/rsync]], and [[Repo accessible from \"dumb\" client without git-annex]].
+"""]]
diff --git a/doc/forum/Direct_special_remotes/comment_5_69c34c655e4b153dfc0d1b8580091124._comment b/doc/forum/Direct_special_remotes/comment_5_69c34c655e4b153dfc0d1b8580091124._comment
new file mode 100644
index 0000000000..bad1f635cb
--- /dev/null
+++ b/doc/forum/Direct_special_remotes/comment_5_69c34c655e4b153dfc0d1b8580091124._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="andy"
+ ip="99.48.75.171"
+ subject="comment 4"
+ date="2013-04-16T07:57:05Z"
+ content="""
+Not I. Sorry. You might want to make a forum post about this, if you haven't already: [[forum]]
+"""]]
diff --git a/doc/forum/Direct_special_remotes/comment_6_b054cfc3d3f81873f3faae7eb4f5337c._comment b/doc/forum/Direct_special_remotes/comment_6_b054cfc3d3f81873f3faae7eb4f5337c._comment
new file mode 100644
index 0000000000..5ade3f1d01
--- /dev/null
+++ b/doc/forum/Direct_special_remotes/comment_6_b054cfc3d3f81873f3faae7eb4f5337c._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="http://joeyh.name/"
+ nickname="joey"
+ subject="comment 6"
+ date="2013-04-16T17:03:36Z"
+ content="""
+Version 3.2012* is compatible with the current version. You can install that version anywhere you don't need the full assistant.
+"""]]
diff --git a/doc/forum/Direct_special_remotes/comment_7_5f9d8a0ef2f13f242594848825d29ee7._comment b/doc/forum/Direct_special_remotes/comment_7_5f9d8a0ef2f13f242594848825d29ee7._comment
new file mode 100644
index 0000000000..fa1f07daac
--- /dev/null
+++ b/doc/forum/Direct_special_remotes/comment_7_5f9d8a0ef2f13f242594848825d29ee7._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="https://www.google.com/accounts/o8/id?id=AItOawkI9AR8BqG4RPw_Ov2lnDCJWMuM6WMRobQ"
+ nickname="Dav"
+ subject="Curious to know if there's progress on this"
+ date="2013-12-09T00:49:45Z"
+ content="""
+The above scenario is also quite interesting to me. Similarly running and old OS (Debian 4.x) on a NAS, and it would be great to be able to access files directly (via direct mode) and manage via git annex, along with a true backup elsewhere. Mostly commenting so I'll get emails for further discussion here.
+"""]]
diff --git a/doc/forum/Disadvantages_of_WORM__58___long_filename__63__.mdwn b/doc/forum/Disadvantages_of_WORM__58___long_filename__63__.mdwn
new file mode 100644
index 0000000000..bff1a3ffe7
--- /dev/null
+++ b/doc/forum/Disadvantages_of_WORM__58___long_filename__63__.mdwn
@@ -0,0 +1,13 @@
+Hello,
+
+first, I felt very comfortable with git-annex, but now I run into problems more often, and I think they are related to the WORM backend. I use this backend for my photos, since they are big (25MB) and never modified (camera raws, I don't touch them). So I edited my `.gitattributes` to use `WORM`.
+
+The problems usually are as follows: I perform an operation, and git (I think it is git which has the problems), and the process exits with exit code 1 and tells me "cannot open binary file: File name too long". The same happens from time to time when I have filenames containing special characters such as ', which can happen when I tag my music collection. I suspect the WORM backend to cause the problems, since the file names in `.git/annex` are indeed very long and ugly beccause of all the escaping done here.
+
+I am sorry that I cannot give a recipe how to reproduce this behaviour, and sometimes it helps to enter a subdirectory and perform the operation from there, for example:
+
+`$ git annex copy --to dest Pictures` fails, but `$ cd Pictures/dir_with_problematic_file; git annex copy --to dest .` works.
+
+Is there some advice or did anyone here observe similar behaviour?
+
+Best regards, Philipp
diff --git a/doc/forum/Disadvantages_of_WORM__58___long_filename__63__/comment_1_cc5502b55fb71c8fdb235ea9d17216ff._comment b/doc/forum/Disadvantages_of_WORM__58___long_filename__63__/comment_1_cc5502b55fb71c8fdb235ea9d17216ff._comment
new file mode 100644
index 0000000000..9cd635eba2
--- /dev/null
+++ b/doc/forum/Disadvantages_of_WORM__58___long_filename__63__/comment_1_cc5502b55fb71c8fdb235ea9d17216ff._comment
@@ -0,0 +1,12 @@
+[[!comment format=mdwn
+ username="joey"
+ subject="""comment 1"""
+ date="2015-01-05T16:47:16Z"
+ content="""
+Please show `ls -l` output listing one or more of the files with the
+problem. Also I'd need to know the filesystem your repository is using
+in order to reproduce the problem.
+
+Note that you can use `git annex migrate` to switch away from the WORM
+backend..
+"""]]
diff --git a/doc/forum/Disadvantages_of_WORM__58___long_filename__63__/comment_2_d04b7adcc17a305fb1d7f1c1a3641be1._comment b/doc/forum/Disadvantages_of_WORM__58___long_filename__63__/comment_2_d04b7adcc17a305fb1d7f1c1a3641be1._comment
new file mode 100644
index 0000000000..9cf4f8c202
--- /dev/null
+++ b/doc/forum/Disadvantages_of_WORM__58___long_filename__63__/comment_2_d04b7adcc17a305fb1d7f1c1a3641be1._comment
@@ -0,0 +1,24 @@
+[[!comment format=mdwn
+ username="joey"
+ subject="""comment 2"""
+ date="2015-01-05T17:01:12Z"
+ content="""
+I refreshed my memory and did some testing, and the WORM backend
+deals with extremely large filenames by detecting the filename length
+limitations of the filesystem where the repository is located, and if a
+WORM key is longer than the maximum filename length, it will truncate it
+down to fit.
+
+This avoids any problems in my testing, as long as the WORM keys are being
+generated and used all on filesystems with similar filename length limits.
+If you are using a version of git-annex older than 4.20131024, it won't
+do that, and you should upgrade.
+
+If you are using a mixture of filesystems, eg EXT4 and VFAT, this can still
+result in WORM key names generated on EXT4 being too long to fit on the
+VFAT filesystem. In this case, I would recommend not using WORM.
+
+Incidentially, that version also made many problematic characters
+not be included in WORM key names, so they're more portable to eg, FAT
+filesystems.
+"""]]
diff --git a/doc/forum/Disk_usage_during___96__import__96__.mdwn b/doc/forum/Disk_usage_during___96__import__96__.mdwn
new file mode 100644
index 0000000000..9ece26e330
--- /dev/null
+++ b/doc/forum/Disk_usage_during___96__import__96__.mdwn
@@ -0,0 +1,5 @@
+Does `git-annex import` move data to the working directory before moving it to `.git/annex/objects`?
+
+The reason I'm asking is that I use the split SSD/HDD setup with some annexes, i.e., everything resides on an SSD except for the git-annex objects directory.
+
+If git-annex wrote imported data to the working directoy (on the SSD) first and moved it to the objects directory (on the HDD) later, it would cause lots of unnecessary writes to the SSD.
diff --git a/doc/forum/Disk_usage_during___96__import__96__/comment_1_3ed4d9a36cacb9be96860cfaae8cee90._comment b/doc/forum/Disk_usage_during___96__import__96__/comment_1_3ed4d9a36cacb9be96860cfaae8cee90._comment
new file mode 100644
index 0000000000..33638e076d
--- /dev/null
+++ b/doc/forum/Disk_usage_during___96__import__96__/comment_1_3ed4d9a36cacb9be96860cfaae8cee90._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="joey"
+ subject="""comment 1"""
+ date="2015-01-05T20:08:52Z"
+ content="""
+Yes, it does; `git annex import` is implemented quite similarly to
+`copy /file .; git annex add $file'
+"""]]
diff --git a/doc/forum/Distributing_data_to_a_set_of_drives.mdwn b/doc/forum/Distributing_data_to_a_set_of_drives.mdwn
new file mode 100644
index 0000000000..406379715d
--- /dev/null
+++ b/doc/forum/Distributing_data_to_a_set_of_drives.mdwn
@@ -0,0 +1 @@
+Here is my situation, Say I have a repo that is 3 TBs (lets call this repo A has full copy of everything.) and I have 3 other drives all 1 TB each lets call them B C D, then I have partial checkouts on my laptops D E. What I would like to do is, have two copies of all files in A and BCD I would like to threat BCD as a group acting like a single repo so A distributes files evenly to drives in group BCD. I also want copies in D and E to not count towards num of files. As for the latter even though I set D and E as untrusted annex still counts copies on those repos If a get a file in D assistant drops a copy from one of the trusted repos A or BCD I have to move it back instead of just dropping it. Also How can I or is it possible to set BCD to act as a group so A distributes files among drives currently I can do this using find/get but it turns it into a math problem every time a add a file to A I have to manually check which disk has most space navigate to it check files with less copies then 2 and get them.
diff --git a/doc/forum/Distributing_data_to_a_set_of_drives/comment_1_f1fa72879f4e1db13bf59dea33c91624._comment b/doc/forum/Distributing_data_to_a_set_of_drives/comment_1_f1fa72879f4e1db13bf59dea33c91624._comment
new file mode 100644
index 0000000000..f0c0a1c2be
--- /dev/null
+++ b/doc/forum/Distributing_data_to_a_set_of_drives/comment_1_f1fa72879f4e1db13bf59dea33c91624._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="https://www.google.com/accounts/o8/id?id=AItOawmNu4V5fvpLlBhaCUfXXOB0MI5NXwh8SkU"
+ nickname="Adam"
+ subject="comment 1"
+ date="2014-05-08T16:56:34Z"
+ content="""
+I recommend rewriting your comment.  Use paragraphs and be more clear and specific.
+"""]]
diff --git a/doc/forum/Distributing_data_to_a_set_of_drives/comment_2_e13b4e5c1e6f1f503f93d521b504d5c1._comment b/doc/forum/Distributing_data_to_a_set_of_drives/comment_2_e13b4e5c1e6f1f503f93d521b504d5c1._comment
new file mode 100644
index 0000000000..5843ab1779
--- /dev/null
+++ b/doc/forum/Distributing_data_to_a_set_of_drives/comment_2_e13b4e5c1e6f1f503f93d521b504d5c1._comment
@@ -0,0 +1,16 @@
+[[!comment format=mdwn
+ username="http://joeyh.name/"
+ ip="216.145.95.162"
+ subject="comment 2"
+ date="2014-05-17T17:00:19Z"
+ content="""
+git-annex doesn't allow bonding repositories like that, but it does have repository groups which can accomplish the same thing.
+
+For example, if you put B C and D into the archive group, and set the [[preferred_content]] expression to \"standard\", then each repository will only want files that have not yet reached some other archive repository.
+
+Or, you can make up your own group name, like \"BCD\" and set the preferred content of each repository to something like \"not (copies=BCD:2)\"  -- and now it will try to have 2 copies of each file on one of the drives.
+
+Once it's configured, using either the git-annex assistant, or `git annex sync --content` will copy files around according to the configuration.
+
+This will tend to fill up the first drive, or the first 2 drives, and only use subsequent drives if files don't fit on the first drive. You can use [[cost]] settings to control which drives files are put on.
+"""]]
diff --git a/doc/forum/Do_I_have_naming_ssh_remote_issue__63__.mdwn b/doc/forum/Do_I_have_naming_ssh_remote_issue__63__.mdwn
new file mode 100644
index 0000000000..c6f7fd734d
--- /dev/null
+++ b/doc/forum/Do_I_have_naming_ssh_remote_issue__63__.mdwn
@@ -0,0 +1,45 @@
+I'm running [git-annex](http://ix.io/gJJ) from  btw on Archlinux.
+
+I fetched a copy of my [git-annex wedding test repo](https://github.com/kaihendry/krwedding) to a machine on ssh called 'bible'.
+
+Now I'm trying to fetch via ssh from my local machine "X1C3".
+
+I can't work out how to fetch from it from b1b15a9b-1aa1-4f94-8b9a-2186d71c0d1a .... what am I missing?
+
+    X1C3:~/annex/krwedding$ git-annex whereis krfeature.mp4
+     whereis krfeature.mp4 (3 copies)
+        00000000-0000-0000-0000-000000000001 -- web
+        10418340-834d-41c2-b38f-7ee84bf6a23a -- s3
+        b1b15a9b-1aa1-4f94-8b9a-2186d71c0d1a -- Jamie's bible
+      web: http://r2d2.webconverger.org/2013-12-22/krfeature.mp4
+      web: http://static.prazefarm.co.uk/krfeature.mp4
+      web: https://objects.dreamhost.com/wedding-video/krfeature.mp4
+    ok
+    X1C3:~/annex/krwedding$ git-annex get . --from "Jamie's bible"
+    git-annex: there is no available git remote named "Jamie's bible"
+    X1C3:~/annex/krwedding$ git-annex enableremote Jamie's bible
+    > ^C
+    X1C3:~/annex/krwedding$ git-annex enableremote "Jamie's bible"
+    git-annex: Unknown special remote name.
+    Known special remotes: s3
+    X1C3:~/annex/krwedding$ git-annex get . --from b1b15a9b-1aa1-4f94-8b9a-2186d71c0d1a
+    git-annex: there is no available git remote named "b1b15a9b-1aa1-4f94-8b9a-2186d71c0d1a"
+
+Why doesn't the UUID work? :/
+
+I even [tried renaming the remote to the UUID... didn't work](http://ix.io/gJI)
+
+**Solution**: Neither UUID or the description is used by get. I also should not have resorted to [[special_remotes]] setup for setting up a git remote.
+
+# Issue 1
+
+Keep getting `git-annex-shell: user error (git ["config","--null","--list"] exited 126)` even though when I run `git config` my return error is 0: 
+
+**Solution**: This was because my ssh git URL was incorrect. A better error message has been implemented: 
+
+
+# Issue 2
+
+I can't work out the [git-annex remote type for ssh, in order to rename the remote](http://ix.io/gJH). I think the issue here is that my ssh remote name "Jamie's bible" doesn't match with the `git remote` name bible.
+
+**Solution**: A _rw_ git URL configured with `git remote` are not [[special_remotes]]. I confused the two. If you need to define public git URL ([[time capsule use case|future_proofing]]), it is possible with an undocumented `git annex initremote foo type=git location=url`. So to summarise, just manually setup the git remote `git remote add ssh://someplace/path/to/repo` (don't worry about the name) and git-annex will find it!
diff --git a/doc/forum/Do_I_have_naming_ssh_remote_issue__63__/comment_1_f3e5ef9d7440cba467ebda1f2e8b12ea._comment b/doc/forum/Do_I_have_naming_ssh_remote_issue__63__/comment_1_f3e5ef9d7440cba467ebda1f2e8b12ea._comment
new file mode 100644
index 0000000000..65d2336e41
--- /dev/null
+++ b/doc/forum/Do_I_have_naming_ssh_remote_issue__63__/comment_1_f3e5ef9d7440cba467ebda1f2e8b12ea._comment
@@ -0,0 +1,25 @@
+[[!comment format=mdwn
+ username="joey"
+ subject="""comment 1"""
+ date="2015-03-06T00:43:32Z"
+ content="""
+You cannot use a UUID like that. How is git-anndx supposed to know where
+that UUID is, or how to access it?
+
+You do not use git-annex initremote with normal git remotes, only special
+remotes.
+
+Set up a git remote, and use `git-annex get --from` with that.
+
+	git remote add ssh://someplace/path/to/repo
+
+git-annex will then figure out that that git remote is the one with that
+UUID, and be able to use it.
+
+Names of git remotes are completely separate from **descriptions** of
+git-annex repositories, which is what git-annex info displays.
+
+AFAIK, all the documentation is clear about the several things that you
+seem to have gotten confused about. Pointers to any places in
+the documentation that are confusing about these matters are appreciated..
+"""]]
diff --git a/doc/forum/Does_Jabber_syncing_work_when_the_buddy_is_offline__63__.mdwn b/doc/forum/Does_Jabber_syncing_work_when_the_buddy_is_offline__63__.mdwn
new file mode 100644
index 0000000000..d5a3d3a928
--- /dev/null
+++ b/doc/forum/Does_Jabber_syncing_work_when_the_buddy_is_offline__63__.mdwn
@@ -0,0 +1 @@
+Do both friends need to be online in order for Jabber syncing to work? Or will the pushed changes be stored on the Jabber server, so that when the remote machine does come online it can pull them even if the original machine is now offline?
diff --git a/doc/forum/Does_Jabber_syncing_work_when_the_buddy_is_offline__63__/comment_1_f290dd8547176793934f8077374e1c0a._comment b/doc/forum/Does_Jabber_syncing_work_when_the_buddy_is_offline__63__/comment_1_f290dd8547176793934f8077374e1c0a._comment
new file mode 100644
index 0000000000..8df82612a9
--- /dev/null
+++ b/doc/forum/Does_Jabber_syncing_work_when_the_buddy_is_offline__63__/comment_1_f290dd8547176793934f8077374e1c0a._comment
@@ -0,0 +1,17 @@
+[[!comment format=mdwn
+ username="http://joeyh.name/"
+ nickname="joey"
+ subject="comment 1"
+ date="2013-02-27T17:54:47Z"
+ content="""
+Both friends need to be online at the same time to sync with Jabber.
+
+There are some extensions that store and forward jabber messages, but:
+
+* They are probably not widely available.
+* AFAIK they only forward chat messages. But we try not to use visible messages, because that would prevent
+  reusing your jabber account with git-annex.
+* Most importantly, the git protocol lets the sender and receiver communicate with each other to work out what data to transfer. If one if offline, the best that could be done is to upload the whole git repository!
+
+If you want the ability for off-line clients to sync up, you should add a shared bare git repository.
+"""]]
diff --git a/doc/forum/Does_Jabber_syncing_work_when_the_buddy_is_offline__63__/comment_2_c358eb51047f333e582bd824be5e0e65._comment b/doc/forum/Does_Jabber_syncing_work_when_the_buddy_is_offline__63__/comment_2_c358eb51047f333e582bd824be5e0e65._comment
new file mode 100644
index 0000000000..b3efced0b8
--- /dev/null
+++ b/doc/forum/Does_Jabber_syncing_work_when_the_buddy_is_offline__63__/comment_2_c358eb51047f333e582bd824be5e0e65._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="annexuser"
+ ip="24.16.193.140"
+ subject="comment 2"
+ date="2013-03-11T19:40:30Z"
+ content="""
+How would I add a shared bare git repository in the web app?
+"""]]
diff --git a/doc/forum/Does_Jabber_syncing_work_when_the_buddy_is_offline__63__/comment_3_a2332c0e7b29110b9aed2ab69ce9d8c4._comment b/doc/forum/Does_Jabber_syncing_work_when_the_buddy_is_offline__63__/comment_3_a2332c0e7b29110b9aed2ab69ce9d8c4._comment
new file mode 100644
index 0000000000..c9a8779808
--- /dev/null
+++ b/doc/forum/Does_Jabber_syncing_work_when_the_buddy_is_offline__63__/comment_3_a2332c0e7b29110b9aed2ab69ce9d8c4._comment
@@ -0,0 +1,12 @@
+[[!comment format=mdwn
+ username="http://joeyh.name/"
+ nickname="joey"
+ subject="comment 3"
+ date="2013-03-11T19:45:34Z"
+ content="""
+That's done by the \"Remote server\" option on the Repositories page.
+
+You need to have git-annex (any old version will do) installed on the server for this to work.
+
+Of course, a regular git remote will do as well, although the assistant does not set this up. You could even use github for that remote, or a similar service; any remote added with `git remote add` will be used by the assistant.
+"""]]
diff --git a/doc/forum/Does_git-annex_version_big_files__63__.mdwn b/doc/forum/Does_git-annex_version_big_files__63__.mdwn
new file mode 100644
index 0000000000..5cbe7d50ac
--- /dev/null
+++ b/doc/forum/Does_git-annex_version_big_files__63__.mdwn
@@ -0,0 +1,5 @@
+Hi
+
+I am trying to understand how git-annex works. Does it version big files at all? 
+
+thanks
diff --git a/doc/forum/Does_git-annex_version_big_files__63__/comment_1_0b44003c1dc53adb807298ae452f8004._comment b/doc/forum/Does_git-annex_version_big_files__63__/comment_1_0b44003c1dc53adb807298ae452f8004._comment
new file mode 100644
index 0000000000..7f403f0cde
--- /dev/null
+++ b/doc/forum/Does_git-annex_version_big_files__63__/comment_1_0b44003c1dc53adb807298ae452f8004._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="http://joeyh.name/"
+ ip="4.152.108.27"
+ subject="comment 1"
+ date="2012-11-09T18:56:59Z"
+ content="""
+Yes. Have you read [[How_It_Works]]?
+"""]]
diff --git a/doc/forum/Does_git-annex_version_big_files__63__/comment_2_ca40b67abd7bd36155d16d0396d7472c._comment b/doc/forum/Does_git-annex_version_big_files__63__/comment_2_ca40b67abd7bd36155d16d0396d7472c._comment
new file mode 100644
index 0000000000..a753d7e82e
--- /dev/null
+++ b/doc/forum/Does_git-annex_version_big_files__63__/comment_2_ca40b67abd7bd36155d16d0396d7472c._comment
@@ -0,0 +1,14 @@
+[[!comment format=mdwn
+ username="karhun"
+ ip="99.20.248.32"
+ subject="comment 2"
+ date="2012-11-09T21:37:19Z"
+ content="""
+Hi
+
+Thanks for the pointer.
+
+The reason I asked about is that it says that \"git-annex allows managing files with git, without checking the file contents into git. \". So I was just trying to understand the mechanic behind the description. If it is not check into the git then probably it is not versioning the files. I assumed that it must be  merely a sync tool.
+
+
+"""]]
diff --git a/doc/forum/Does_git-annex_version_big_files__63__/comment_3_32de3501feedce51b43ed9dcc399c7a9._comment b/doc/forum/Does_git-annex_version_big_files__63__/comment_3_32de3501feedce51b43ed9dcc399c7a9._comment
new file mode 100644
index 0000000000..33c4ef7ba8
--- /dev/null
+++ b/doc/forum/Does_git-annex_version_big_files__63__/comment_3_32de3501feedce51b43ed9dcc399c7a9._comment
@@ -0,0 +1,15 @@
+[[!comment format=mdwn
+ username="karhun"
+ ip="99.20.248.32"
+ subject="comment 3"
+ date="2012-11-12T05:58:16Z"
+ content="""
+Hi
+
+I have one more follow up question.
+
+So git-annex versions files as I understand. Is it possible to use it without file versioning and just use it as a sync tool. I realize that there are example scenarios to demonstrate it however I would like to disable versioning if possible because I am intending to use it on large folders with big binary type files.
+
+thanks
+
+"""]]
diff --git a/doc/forum/Does_git-annex_version_big_files__63__/comment_4_8c65a7f8bda3c876971c2801fb6a76a1._comment b/doc/forum/Does_git-annex_version_big_files__63__/comment_4_8c65a7f8bda3c876971c2801fb6a76a1._comment
new file mode 100644
index 0000000000..f7f2e85dce
--- /dev/null
+++ b/doc/forum/Does_git-annex_version_big_files__63__/comment_4_8c65a7f8bda3c876971c2801fb6a76a1._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="http://joeyh.name/"
+ ip="4.152.108.27"
+ subject="comment 4"
+ date="2012-11-12T15:17:53Z"
+ content="""
+Let's suppose you delete a file from git, and sync the deletion. Later, you can run `git annex unused` and it will find content not needed by any branch of the git repository. And if you choose, you can use `git annex dropunused` to delete that content.
+"""]]
diff --git a/doc/forum/Does_git_annex_support_my_weird_workflow__63_____40__A_single__44___effectively___95__read_only__95___source__41__.mdwn b/doc/forum/Does_git_annex_support_my_weird_workflow__63_____40__A_single__44___effectively___95__read_only__95___source__41__.mdwn
new file mode 100644
index 0000000000..7899c62cd7
--- /dev/null
+++ b/doc/forum/Does_git_annex_support_my_weird_workflow__63_____40__A_single__44___effectively___95__read_only__95___source__41__.mdwn
@@ -0,0 +1,8 @@
+I tried using it once, but I couldn't get it to work for me, so I gave up. I'm hoping that if I describe my desired workflow to people who know more about it I can get a better sense of whether it can help me.
+
+Here's what I have:
+
+* A server, with a directory I have read-only access to; this directory is the one I would like to keep in git-annex. Obviously, I have read-write access to other directories.
+* A laptop, which is sometimes on the same LAN as the server and sometimes not; on this laptop, I would like to be able to easily maintain synced copies of certain subdirectories of the main directory (which is on the server). These copies should (ideally) not need to be in the same directory tree on the laptop.
+
+I don't know if this description is clear enough (I'm doubtful), but I can answer questions.
diff --git a/doc/forum/Does_git_annex_support_my_weird_workflow__63_____40__A_single__44___effectively___95__read_only__95___source__41__/comment_1_25572571debe2c91447b0ed952f45cb2._comment b/doc/forum/Does_git_annex_support_my_weird_workflow__63_____40__A_single__44___effectively___95__read_only__95___source__41__/comment_1_25572571debe2c91447b0ed952f45cb2._comment
new file mode 100644
index 0000000000..eb568646ef
--- /dev/null
+++ b/doc/forum/Does_git_annex_support_my_weird_workflow__63_____40__A_single__44___effectively___95__read_only__95___source__41__/comment_1_25572571debe2c91447b0ed952f45cb2._comment
@@ -0,0 +1,19 @@
+[[!comment format=mdwn
+ username="joey"
+ subject="""comment 1"""
+ date="2015-12-02T18:14:50Z"
+ content="""
+The two problems are: Read-only directory, and wanting a clone that doesn't
+have the same directory structure.
+
+For the read-only directory, you may be able to use a direct mode
+repository. You might need to set `GIT_DIR` in the environment to make
+the .git directory be placed somewhere outside the read-only directory.
+
+As to wanting different directory structure in your clone, it's an added
+complication I'd recommend avoiding. Not that it's impossible, you can move
+files around in the git repository and that could be automated via
+git-filter-branch or the like. But git-annex offers a simpler solution,
+which is to use the same directory structure and only get the contents of
+files in the subdirectories you want.
+"""]]
diff --git a/doc/forum/Does_git_annex_support_my_weird_workflow__63_____40__A_single__44___effectively___95__read_only__95___source__41__/comment_2_46c27b242c6df76f9df198bbd6f14030._comment b/doc/forum/Does_git_annex_support_my_weird_workflow__63_____40__A_single__44___effectively___95__read_only__95___source__41__/comment_2_46c27b242c6df76f9df198bbd6f14030._comment
new file mode 100644
index 0000000000..9f7dfb6a29
--- /dev/null
+++ b/doc/forum/Does_git_annex_support_my_weird_workflow__63_____40__A_single__44___effectively___95__read_only__95___source__41__/comment_2_46c27b242c6df76f9df198bbd6f14030._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="rpglover64@acd6753c831cc132736943bec350391fcb2ff77d"
+ nickname="rpglover64"
+ subject="Thanks!"
+ date="2015-12-02T20:15:57Z"
+ content="""
+Last time I tried, I managed to get somewhere with the readonly directory, I think using the same idea you suggested. As for the different directory structure, you make a very good point. Thanks for taking the time to respond. I'll take another shot at using git-annex when I have some time.
+"""]]
diff --git a/doc/forum/Does_git_annex_support_my_weird_workflow__63_____40__A_single__44___effectively___95__read_only__95___source__41__/comment_3_2d9baeb81857b354931251a8b8a756e5._comment b/doc/forum/Does_git_annex_support_my_weird_workflow__63_____40__A_single__44___effectively___95__read_only__95___source__41__/comment_3_2d9baeb81857b354931251a8b8a756e5._comment
new file mode 100644
index 0000000000..8687f9eae9
--- /dev/null
+++ b/doc/forum/Does_git_annex_support_my_weird_workflow__63_____40__A_single__44___effectively___95__read_only__95___source__41__/comment_3_2d9baeb81857b354931251a8b8a756e5._comment
@@ -0,0 +1,12 @@
+[[!comment format=mdwn
+ username="rpglover64@acd6753c831cc132736943bec350391fcb2ff77d"
+ nickname="rpglover64"
+ subject="Third time's the charm?"
+ date="2016-02-08T20:44:11Z"
+ content="""
+So I tried again (I got read-write access to the top level of the directory I want to share, so creating a .git was straightforward), and I had two things get in my way. I have a lot of relatively large files (~10,000), so I decided to use the WORM backend rather than waiting for the SHA to complete. Even so, adding everything was slow-going.
+
+Since the most interesting feature of git-annex is queueing up downloads to sync on LAN, is there a way to add the files especially quickly by sacrificing some more safety? Essentially, I want to use git-annex as a glorified file server, letting my laptop do all the interesting work.
+
+Second, when I Ctrl-C during `git annex add`, progress is not preserved in a direct-mode repository, so the add needs to start all over again.
+"""]]
diff --git a/doc/forum/Does_migrate_ensure_data_integrity__63__.mdwn b/doc/forum/Does_migrate_ensure_data_integrity__63__.mdwn
new file mode 100644
index 0000000000..872e7d53da
--- /dev/null
+++ b/doc/forum/Does_migrate_ensure_data_integrity__63__.mdwn
@@ -0,0 +1,7 @@
+Out of simple curiosity, how does 'git annex migrate' work?  I'm mostly wondering how file integrity is ensured.
+
+Let's say you want to migrate foo.txt from (say) md5 to sha256.  Does git annex simply sha256sum foo.txt and rename it in the .git/objects folder to the new sum?  Or does it md5sum foo.txt, verify that it's the not corrupt, then sha256sum it and rename to the new sum?
+
+You could run into problems if you migrate without first verifying; if the file is corrupt and you simply sha256sum and rename it, then the file wouldn't seem corrupt at your next fsck.
+
+I'm sure you've considered this during the basic design phase of git-annex, but I'd just like to be sure.  I'm kind of paranoid when it comes to data integrity. =P
diff --git a/doc/forum/Does_migrate_ensure_data_integrity__63__/comment_1_cef50b32c46f4406c6f918c5866ddc15._comment b/doc/forum/Does_migrate_ensure_data_integrity__63__/comment_1_cef50b32c46f4406c6f918c5866ddc15._comment
new file mode 100644
index 0000000000..7c819a8ca9
--- /dev/null
+++ b/doc/forum/Does_migrate_ensure_data_integrity__63__/comment_1_cef50b32c46f4406c6f918c5866ddc15._comment
@@ -0,0 +1,11 @@
+[[!comment format=mdwn
+ username="http://joeyh.name/"
+ nickname="joey"
+ subject="comment 1"
+ date="2013-05-13T18:28:01Z"
+ content="""
+I actually didn't originally consider this, but it turns out I have wonderful users, and you're at least the second to consider this case.
+
+So the released git-annex version does already verify the checksum when doing a migration. However, as I was checking this, I noticed 
+I had left a window between the verification and the generation of the new key for the migration. It's pretty unlikely a file would get corrupted just as it was being migrated, but that's no excuse. I've committed a change that reverses the order; so it generates a new key and then verifies the old one.
+"""]]
diff --git a/doc/forum/Does_migrate_ensure_data_integrity__63__/comment_2_f389b924c8531b35fdf5dedd10fc8000._comment b/doc/forum/Does_migrate_ensure_data_integrity__63__/comment_2_f389b924c8531b35fdf5dedd10fc8000._comment
new file mode 100644
index 0000000000..c377d814c5
--- /dev/null
+++ b/doc/forum/Does_migrate_ensure_data_integrity__63__/comment_2_f389b924c8531b35fdf5dedd10fc8000._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="GLITTAH"
+ ip="154.51.136.37"
+ subject="comment 2"
+ date="2013-05-14T15:54:08Z"
+ content="""
+An even better solution.  Thanks for your time and effort joey!
+"""]]
diff --git a/doc/forum/Does_the_windows_version_install_iStartSurf_malware__63__.mdwn b/doc/forum/Does_the_windows_version_install_iStartSurf_malware__63__.mdwn
new file mode 100644
index 0000000000..21c03e758d
--- /dev/null
+++ b/doc/forum/Does_the_windows_version_install_iStartSurf_malware__63__.mdwn
@@ -0,0 +1,3 @@
+Hi, since I installed git-annex-windows through the link from the homepage I got iStartSurf, a really anoying malware. Could somebody ack the existence of iStartSurf in windows version? I use my windows seldom, usualy start only to receive updates. (and of coures to try out git-annex)
+
+Zsolt
diff --git a/doc/forum/Does_the_windows_version_install_iStartSurf_malware__63__/comment_1_0536085a1f969802a7e8607093eee97d._comment b/doc/forum/Does_the_windows_version_install_iStartSurf_malware__63__/comment_1_0536085a1f969802a7e8607093eee97d._comment
new file mode 100644
index 0000000000..9d2e4d29a5
--- /dev/null
+++ b/doc/forum/Does_the_windows_version_install_iStartSurf_malware__63__/comment_1_0536085a1f969802a7e8607093eee97d._comment
@@ -0,0 +1,24 @@
+[[!comment format=mdwn
+ username="joey"
+ subject="""Correlation does not imply causation"""
+ date="2015-06-15T17:27:40Z"
+ content="""
+
    +joey@kite:~/lib/downloads/git-annex/windows/current> clamscan git-annex-installer.exe
+git-annex-installer.exe: OK
+
+----------- SCAN SUMMARY -----------
+Known viruses: 3841819
+Engine version: 0.98.7
+Scanned directories: 0
+Scanned files: 1
+Infected files: 0
+Data scanned: 36.55 MB
+Data read: 23.07 MB (ratio 1.58:1)
+Time: 10.835 sec (0 m 10 s)
+
    
+
+FWIW, the git-annex Windows build happens on the same build cluster that is used
+for msysgit. If it were infected by a virus, that would be bad news,
+so if you have some actual evidence of a virus, please share it.
+"""]]
diff --git a/doc/forum/Don__39__t_understand_how_to_delete__47__recover_files.mdwn b/doc/forum/Don__39__t_understand_how_to_delete__47__recover_files.mdwn
new file mode 100644
index 0000000000..aae2bf2764
--- /dev/null
+++ b/doc/forum/Don__39__t_understand_how_to_delete__47__recover_files.mdwn
@@ -0,0 +1,25 @@
+My Use Case:
+
+I try basicly to use annex as a raid-like tool (at least thats the first step)
+
+so I added some big files to it, and then I synced it to a usb-remote. So far all did go well...
+
+No I startet annex watch daemon... because I thought it would then watch the files and checkin and out at least for the "origin" copy of the files.
+
+Then I thought lets delete some files I dont need anymore... so... rm.... ^^
+
+Maybe I just wanted to see what happens or if that would magicly do what I wanted him to do... ok I knew that it would not delete the file on the usb-drive (backup...) ok... but maybe at least localy...
+
+
+Now what did happen instead of that... the links are gone yes... the files it self are in the .git objects tree, so they did not get deleted, so ok not the way I wanted... have to unlock it first, would make maybe sense...
+
+So I tried first to get the links back... tried fix, tried unused, tried get... but the links doesnt show up again...
+
+is there a way to first bring back the links?
+is it save or the right way to just git rebase HEAD~3 to bring the links back?
+
+and then when I want to delete files from all places whats the way to do that... annex unlock -> then delete? or git drop filex --copies=0 or something?
+
+like I said I try to use is like a more flexible raid thing, this files are to big to really back em up with history... I watch them... and then soon I will often delete them (from everywhere). but other parts stay... I dont delete them then... ^^
+
+So maybe I missuse annex for that usecase... try to find that out ;)
diff --git a/doc/forum/Don__39__t_understand_how_to_delete__47__recover_files/comment_1_b307bfb0b70d649897f411eb753bd50a._comment b/doc/forum/Don__39__t_understand_how_to_delete__47__recover_files/comment_1_b307bfb0b70d649897f411eb753bd50a._comment
new file mode 100644
index 0000000000..739319b71e
--- /dev/null
+++ b/doc/forum/Don__39__t_understand_how_to_delete__47__recover_files/comment_1_b307bfb0b70d649897f411eb753bd50a._comment
@@ -0,0 +1,14 @@
+[[!comment format=mdwn
+ username="http://joeyh.name/"
+ ip="4.154.0.60"
+ subject="comment 1"
+ date="2012-09-16T00:14:49Z"
+ content="""
+You can use any git command you like like `git rebase` or `git revert` to operate on the links.
+
+git-annex, like git, keeps data for files that have been deleted.
+
+The `git annex unused` command will find that data and let you remove it. See [[walkthrough/unused_data]].
+
+
+"""]]
diff --git a/doc/forum/Don__39__t_understand_how_to_delete__47__recover_files/comment_2_58a6a1476274b8c4feb3d43ecd998759._comment b/doc/forum/Don__39__t_understand_how_to_delete__47__recover_files/comment_2_58a6a1476274b8c4feb3d43ecd998759._comment
new file mode 100644
index 0000000000..01ad2901d2
--- /dev/null
+++ b/doc/forum/Don__39__t_understand_how_to_delete__47__recover_files/comment_2_58a6a1476274b8c4feb3d43ecd998759._comment
@@ -0,0 +1,41 @@
+[[!comment format=mdwn
+ username="https://www.google.com/accounts/o8/id?id=AItOawn4bbuawnh-nSo9pAh8irYAcV4MQCcfdHo"
+ nickname="Stefan"
+ subject="thanx for fast help ;)"
+ date="2012-09-16T16:09:29Z"
+ content="""
+git-annex version: 3.20120629
+
+git annex unused did not show me anything... dont know why... I did only \"rm Directory\" not git rm... but \"git annex watch\" did check it in... I sadly have reverted and droped the file now... but if you want I can try to delete other files the same way to see if annex unused shows them... then I post the hole commands... k I try it now:
+
+oh I see the directory I wanted to drop did not get dropped because I set numcopies to 2 ;) so I try it again with that file:
+
+1. rm -r directory
+2. ls it is gone (should be ;) )
+
+user@host:~/annex/SOFTWARE$ git annex unused
+unused . (checking for unused data...) (checking master...) (checking synced/master...) (checking usbdrive/master...) ok
+user@host:~/annex/SOFTWARE$ 
+
+Is that like it should be? If so, should unused only show stuff that I delete with git rm?
+
+
+-----------------------------------
+
+Ok I did read the unused doku you pointed me. So I thought I got it, but I did not:
+
+If I
+
+1. rm File
+2. in the usb-repos, git pull
+both repositories dont have a link to the file.
+3. git unused shows nothing
+4. it seemed that the link AND the File it links to got deleted:
+
+-../.git/annex/objects/pM/vq/SHA256-s1066518528--00c5e1b1610c0c2dfab05c7a55aaad
+
+but the file size is 1gb, both directories dont get smaller... also if I use git gc after that it does not get smaller too.
+
+Is there a way to really delete files from both repositories.
+
+"""]]
diff --git a/doc/forum/Don__39__t_understand_how_to_delete__47__recover_files/comment_3_4b857f481db7b2437ac9f8137a8510e2._comment b/doc/forum/Don__39__t_understand_how_to_delete__47__recover_files/comment_3_4b857f481db7b2437ac9f8137a8510e2._comment
new file mode 100644
index 0000000000..a1e003ee77
--- /dev/null
+++ b/doc/forum/Don__39__t_understand_how_to_delete__47__recover_files/comment_3_4b857f481db7b2437ac9f8137a8510e2._comment
@@ -0,0 +1,11 @@
+[[!comment format=mdwn
+ username="http://joeyh.name/"
+ ip="4.154.0.60"
+ subject="comment 3"
+ date="2012-09-16T17:08:49Z"
+ content="""
+> unused . (checking for unused data...) (checking master...) (checking synced
+/master...) (checking usbdrive/master...) ok
+
+It still considers a file's content used while some branch contains the file. So one of the branches it's checking there still has your file in it. Once you get all the branches synced up, the file will become unused.
+"""]]
diff --git a/doc/forum/Don__39__t_understand_how_to_delete__47__recover_files/comment_4_828db3bf2863d98c0b0fb4074aa7f066._comment b/doc/forum/Don__39__t_understand_how_to_delete__47__recover_files/comment_4_828db3bf2863d98c0b0fb4074aa7f066._comment
new file mode 100644
index 0000000000..20e1801075
--- /dev/null
+++ b/doc/forum/Don__39__t_understand_how_to_delete__47__recover_files/comment_4_828db3bf2863d98c0b0fb4074aa7f066._comment
@@ -0,0 +1,33 @@
+[[!comment format=mdwn
+ username="https://www.google.com/accounts/o8/id?id=AItOawn4bbuawnh-nSo9pAh8irYAcV4MQCcfdHo"
+ nickname="Stefan"
+ subject="comment 4"
+ date="2012-09-16T22:51:00Z"
+ content="""
+hmm Ok I tried now something that seems to did it...  after I did checkin and pull the deletion of the soft-link of the file... I addionaly used git annex sync, and then again pushed it from the usb drive... now the unused command finds stuff...
+
+So am I right now to make that happen:
+
+1. delete the file in the \"main\" repos
+2. git pull from the other repository(s)
+3. git annex sync
+
+4. git annex unused (now it founds something to delete)
+
+5. git annex dropunused 1-1000 (or some other range) do that in both repositories to delete in on both... and do that maybe as cronjob (from point 2 to 5) or do the last last step only if a drive is nearly full (automaticly or manually...)
+
+ok so maybe I could use that.
+
+the advantages over something like rsync are...
+
+1. I have a history and can reverse some operations (till I do step 5 ^^)
+2. I am more flexible in repo/remote types
+3. checksums get saved
+4. I can have repo with all links but 0 data on a 3rd pc and can \"get\" stuff dynamicly (what is the cheapest transfeir) ot them... aka partial checkouts
+
+ok I think the 4. advantage buys me ;)
+
+hope I got it now right with the 5 step instruction.
+
+keep up your great work, would have tried kickstarter here maybe too for other stuff... but I am no american ;) and eu-guys are not allowed, and I also have no relative in usa ;) but that only as a site note ;)
+"""]]
diff --git a/doc/forum/Don__39__t_understand_how_to_delete__47__recover_files/comment_5_cb2063d6a4e08a5c12bf3723d0fa74e0._comment b/doc/forum/Don__39__t_understand_how_to_delete__47__recover_files/comment_5_cb2063d6a4e08a5c12bf3723d0fa74e0._comment
new file mode 100644
index 0000000000..14e5b3da39
--- /dev/null
+++ b/doc/forum/Don__39__t_understand_how_to_delete__47__recover_files/comment_5_cb2063d6a4e08a5c12bf3723d0fa74e0._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="https://www.google.com/accounts/o8/id?id=AItOawmBUR4O9mofxVbpb8JV9mEbVfIYv670uJo"
+ nickname="Justin"
+ subject="comment 5"
+ date="2012-09-17T14:12:13Z"
+ content="""
+That looks right except you shouldn't have to `git pull from the other repository(s)`.  The first thing `git annex sync` does is a `git pull`.
+"""]]
diff --git a/doc/forum/Don__39__t_understand_how_to_delete__47__recover_files/comment_6_1759bcd5708f591f91b9c410f6dc5c54._comment b/doc/forum/Don__39__t_understand_how_to_delete__47__recover_files/comment_6_1759bcd5708f591f91b9c410f6dc5c54._comment
new file mode 100644
index 0000000000..379d728ee4
--- /dev/null
+++ b/doc/forum/Don__39__t_understand_how_to_delete__47__recover_files/comment_6_1759bcd5708f591f91b9c410f6dc5c54._comment
@@ -0,0 +1,14 @@
+[[!comment format=mdwn
+ username="https://www.google.com/accounts/o8/id?id=AItOawn4bbuawnh-nSo9pAh8irYAcV4MQCcfdHo"
+ nickname="Stefan"
+ subject="comment 6"
+ date="2012-09-17T23:00:09Z"
+ content="""
+ok I now at least got it that it is nice to run watch on both repositories... 
+
+but what I dont get is, is the git annex watch commando not a automatic way so that this deamon makes sync command whenever needed? so I have to use git annex sync first on the repos where I delete the file and then again on the repos I want to receive that change... that seemed to work, but why did watch not do that for me???
+
+
+
+
+"""]]
diff --git a/doc/forum/Don__39__t_understand_how_to_delete__47__recover_files/comment_7_2a389f01eb5131042ea1e71a73c9787a._comment b/doc/forum/Don__39__t_understand_how_to_delete__47__recover_files/comment_7_2a389f01eb5131042ea1e71a73c9787a._comment
new file mode 100644
index 0000000000..ecc159641e
--- /dev/null
+++ b/doc/forum/Don__39__t_understand_how_to_delete__47__recover_files/comment_7_2a389f01eb5131042ea1e71a73c9787a._comment
@@ -0,0 +1,21 @@
+[[!comment format=mdwn
+ username="https://www.google.com/accounts/o8/id?id=AItOawn4bbuawnh-nSo9pAh8irYAcV4MQCcfdHo"
+ nickname="Stefan"
+ subject="comment 7"
+ date="2012-09-18T00:11:50Z"
+ content="""
+cd /mnt/data/annex/ &&
+/usr/bin/git annex sync &&
+/usr/bin/git annex copy --auto --to usbdrive &&
+cd /mnt/usb1/annex/ &&
+/usr/bin/git annex sync &&
+echo \"synced\" ||
+echo \"backup of annex failed\" | sendxmpp -t name@server.country
+
+
+ok suprisingly that did not only sync the files but also did remove the file, so you dont need a pull... dont know what I did wrong the other times I tried, does maybe the  copy command also \"copies\" the delete commit?
+
+but annyway that works for me... (for now I do the dropunused command manuelly...)
+
+that should work as cronjob...
+"""]]
diff --git a/doc/forum/Don__39__t_understand_local_vs._known_keys.mdwn b/doc/forum/Don__39__t_understand_local_vs._known_keys.mdwn
new file mode 100644
index 0000000000..39f8c1041e
--- /dev/null
+++ b/doc/forum/Don__39__t_understand_local_vs._known_keys.mdwn
@@ -0,0 +1,19 @@
+I just created a new Annex by doing the following:
+
+ 1. git init
+ 2. git annex init
+ 3. git annex add .
+ 4. git commit -m "Added files"
+ 5. git annex status
+
+I see the following:
+
+    local annex keys: 224
+    local annex size: 41 gigabytes
+    known annex keys: 235
+    known annex size: 49 gigabytes
+    bloom filter size: 16 mebibytes (0% full)
+    backend usage: 
+            SHA256: 459
+
+Why is there an 8 gigabyte difference here?  What/where are those files?  What is a bloom filter?
diff --git a/doc/forum/Don__39__t_understand_local_vs._known_keys/comment_1_10749c0d76e824217dd1ff8c8a6e42a5._comment b/doc/forum/Don__39__t_understand_local_vs._known_keys/comment_1_10749c0d76e824217dd1ff8c8a6e42a5._comment
new file mode 100644
index 0000000000..5ab1d55cc2
--- /dev/null
+++ b/doc/forum/Don__39__t_understand_local_vs._known_keys/comment_1_10749c0d76e824217dd1ff8c8a6e42a5._comment
@@ -0,0 +1,10 @@
+[[!comment format=mdwn
+ username="https://www.google.com/accounts/o8/id?id=AItOawmBUR4O9mofxVbpb8JV9mEbVfIYv670uJo"
+ nickname="Justin"
+ subject="comment 1"
+ date="2012-10-17T12:54:42Z"
+ content="""
+Those are duplicate files.
+
+see http://git-annex.branchable.com/tips/finding_duplicate_files/ for how to easily display them.
+"""]]
diff --git a/doc/forum/Don__39__t_understand_local_vs._known_keys/comment_2_db9f1b6d9638c2b0a7e241c2727e8cfb._comment b/doc/forum/Don__39__t_understand_local_vs._known_keys/comment_2_db9f1b6d9638c2b0a7e241c2727e8cfb._comment
new file mode 100644
index 0000000000..c1ca018f5c
--- /dev/null
+++ b/doc/forum/Don__39__t_understand_local_vs._known_keys/comment_2_db9f1b6d9638c2b0a7e241c2727e8cfb._comment
@@ -0,0 +1,13 @@
+[[!comment format=mdwn
+ username="http://joeyh.name/"
+ ip="2001:4978:f:21a::2"
+ subject="comment 2"
+ date="2012-10-17T17:32:52Z"
+ content="""
+The local keys are files whose content is locally present.  
+The known keys are annexed files in the current branch, whose content may or may not be present. 
+
+Justin is correct -- if you have the same file in the tree twice, it will be counted twice as known keys. Since git-annex deuplicates, only one local key is needed to store it.
+
+The bloom filter is a technical implementation detail that allows the potentially expensive status scan to run in constant space. You can read about it on Wikipedia if interested. :)
+"""]]
diff --git a/doc/forum/Don__39__t_understand_sync_with_ssh_remote.mdwn b/doc/forum/Don__39__t_understand_sync_with_ssh_remote.mdwn
new file mode 100644
index 0000000000..45f77d1651
--- /dev/null
+++ b/doc/forum/Don__39__t_understand_sync_with_ssh_remote.mdwn
@@ -0,0 +1,54 @@
+Hi,
+
+I don't understand how to sync an empty local repository on my local pc with ssh server.
+
+On webfaction distant ssh host, i create a repository in v6  which contain all my file (commited with git commit).
+
+        00000000-0000-0000-0000-000000000001 -- web
+        00000000-0000-0000-0000-000000000002 -- bittorrent
+        151c9c3c-7bf2-4a06-ab47-38dfd65b0d12 -- reyman64@web504.webfaction.com:~/webdav/bib/seb/pdf-repository [here]
+
+On local pc, i create an empty repository, also in v6, and i configure ssh remote, nammed "pdfwebfaction", so git annex info return this
+
+        00000000-0000-0000-0000-000000000001 -- web
+        00000000-0000-0000-0000-000000000002 -- bittorrent
+        151c9c3c-7bf2-4a06-ab47-38dfd65b0d12 -- reyman64@web504.webfaction.com:~/webdav/bib/seb/pdf-repository
+        9d08d7c0-55da-451e-8ca3-aaa612eefb46 -- reyman@Dunwich:~/Sync/PDF_Webfaction [here]
+
+
+I want to sync local with this remote branch, and sync content, so i try :
+ 
+    git annex sync pdfwebfaction
+
+Which return :
+
+    commit  git-annex: git ["--git-dir=.git","--work-tree=.","--literal-pathspecs","commit","-a","-m","git-annex in reyman@Dunwich:~/Sync/PDF_Webfaction"] exited 1
+
+And i also try without success : 
+
+    git-annex get --from pdfwebfaction
+
+which return an error about uuid, don't understand why because uuid exist for ssh repo :
+
+    git-annex: cannot determine uuid for pdfwebfaction (perhaps you need to run "git annex sync"?) (remote.pdfwebfaction.annex-ignore is set)
+
+My git config is : 
+
+    [core]
+        repositoryformatversion = 0
+        filemode = true
+        bare = false
+        logallrefupdates = true
+    [annex]
+        uuid = 9d08d7c0-55da-451e-8ca3-aaa612eefb46
+        version = 6
+    [filter "annex"]
+        smudge = git-annex smudge %f
+        clean = git-annex smudge --clean %f
+    [remote "pdfwebfaction"]
+        url = ssh://reyman64@reyman64.webfactional.com:/home/reyman64/webdav/bib/seb/pdf-repository
+        fetch = +refs/heads/*:refs/remotes/pdfwebfaction/*
+
+There is something i don't understand, and the documentation talk only about ssh associated with a git clone command.
+
+PS : the ssh connection is correct, i test it.
diff --git a/doc/forum/Don__39__t_understand_sync_with_ssh_remote/comment_1_be10d6b564e130f4a516d438831681d1._comment b/doc/forum/Don__39__t_understand_sync_with_ssh_remote/comment_1_be10d6b564e130f4a516d438831681d1._comment
new file mode 100644
index 0000000000..3a96e88472
--- /dev/null
+++ b/doc/forum/Don__39__t_understand_sync_with_ssh_remote/comment_1_be10d6b564e130f4a516d438831681d1._comment
@@ -0,0 +1,18 @@
+[[!comment format=mdwn
+ username="reyman64@740e43250854ada1b8484b3bc44b53be03367d70"
+ nickname="reyman64"
+ subject="comment 1"
+ date="2016-01-26T19:14:51Z"
+ content="""
+Ok i finally understand.
+
+I remove the ssh with `git add remove pdfwebfaction` and recreate with
+
+`git remote add pdfwebfaction ssh://reyman@webxxx.webfaction.com:/home/reyman/webdav/pdf`
+
+At this step, `git annex get --from pdfwebfaction` doesn't work, i need to create and add a file like `touch test` to my local empty git repository.
+
+After that, the previous command work.
+
+
+"""]]
diff --git a/doc/forum/Drop_with_assistant.mdwn b/doc/forum/Drop_with_assistant.mdwn
new file mode 100644
index 0000000000..f8847ab188
--- /dev/null
+++ b/doc/forum/Drop_with_assistant.mdwn
@@ -0,0 +1,5 @@
+Hi,
+
+how can I handle *drop* with the assistant? There are some files in my annex that I don't need on my laptop. so i just *drop* them on the command line bevcause I have copies on my server and an external usb drive. But after a while the assistant pulls all the files back in. Can I avoid this and make the *drop* somehow "sticky" until i *get* them over cli?
+
+Best, Michael
diff --git a/doc/forum/Drop_with_assistant/comment_1_048f5a31c549afb19b76a65bddd0cd24._comment b/doc/forum/Drop_with_assistant/comment_1_048f5a31c549afb19b76a65bddd0cd24._comment
new file mode 100644
index 0000000000..e59c285c2f
--- /dev/null
+++ b/doc/forum/Drop_with_assistant/comment_1_048f5a31c549afb19b76a65bddd0cd24._comment
@@ -0,0 +1,13 @@
+[[!comment format=mdwn
+ username="https://www.google.com/accounts/o8/id?id=AItOawn7gQ1zZDdWhXy9H51W2krZYShNmKL3qfM"
+ nickname="Karsten"
+ subject="comment 1"
+ date="2013-04-11T07:32:50Z"
+ content="""
+Hi,
+
+you can set your repository type to 'manual'.
+
+Best
+Karsten
+"""]]
diff --git a/doc/forum/Drop_with_assistant/comment_2_527d7b6a8efa85b904111f179912d926._comment b/doc/forum/Drop_with_assistant/comment_2_527d7b6a8efa85b904111f179912d926._comment
new file mode 100644
index 0000000000..d3ffed3f71
--- /dev/null
+++ b/doc/forum/Drop_with_assistant/comment_2_527d7b6a8efa85b904111f179912d926._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="https://www.google.com/accounts/o8/id?id=AItOawmU_2tE75oyG0h2ZPN4lcroIKEMC8G-otE"
+ nickname="Michael"
+ subject="comment 2"
+ date="2013-04-11T08:26:01Z"
+ content="""
+@Karsten But then I would loose all replication features? Or can I mark subfolders as manual? Or would I mix different annexes?
+"""]]
diff --git a/doc/forum/Drop_with_assistant/comment_3_c50857506869bb1cd306b66acf37fba8._comment b/doc/forum/Drop_with_assistant/comment_3_c50857506869bb1cd306b66acf37fba8._comment
new file mode 100644
index 0000000000..cd45baeaf1
--- /dev/null
+++ b/doc/forum/Drop_with_assistant/comment_3_c50857506869bb1cd306b66acf37fba8._comment
@@ -0,0 +1,14 @@
+[[!comment format=mdwn
+ username="http://edheil.wordpress.com/"
+ ip="173.162.44.162"
+ subject="comment 3"
+ date="2013-04-11T13:18:04Z"
+ content="""
+The trivial way to do this is to set your repository to \"client\" and then when you want to drop a file, create a folder called \"archive\" and drag the file into that folder.  The assistant will drop it (if numcopies is satisfied that there are enough copies elsewhere).  If you want it back, drag it out of the archive and the assistant will re-fetch it.
+
+This is how I use git-annex: I have an ssh remote and a usb drive which are both \"backup\" and my laptop is \"client\", and my numcopies is sent to 2.
+
+Files are always copied to the ssh remote and the usb drive because they're \"backup.\"  That fulfills my numcopies=2.  At that point, anything in an \"archive\" folder on my laptop will be dropped automatically.
+
+It means my annex is littered with \"archive\" folders, but it achieves the desired effect.
+"""]]
diff --git a/doc/forum/Drop_with_assistant/comment_4_1ea37445d5eb96c3efa182e88e07b867._comment b/doc/forum/Drop_with_assistant/comment_4_1ea37445d5eb96c3efa182e88e07b867._comment
new file mode 100644
index 0000000000..56fcfa2549
--- /dev/null
+++ b/doc/forum/Drop_with_assistant/comment_4_1ea37445d5eb96c3efa182e88e07b867._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="https://www.google.com/accounts/o8/id?id=AItOawmU_2tE75oyG0h2ZPN4lcroIKEMC8G-otE"
+ nickname="Michael"
+ subject="comment 4"
+ date="2013-04-11T13:39:13Z"
+ content="""
+A nice feature would be, to  be able to put a .annex-manual file into a subfolder and from that point the files would require manual git-annex management.
+"""]]
diff --git a/doc/forum/Drop_with_assistant/comment_5_c08908ea5232cbe067c73ecd12d0e218._comment b/doc/forum/Drop_with_assistant/comment_5_c08908ea5232cbe067c73ecd12d0e218._comment
new file mode 100644
index 0000000000..b13392a9cd
--- /dev/null
+++ b/doc/forum/Drop_with_assistant/comment_5_c08908ea5232cbe067c73ecd12d0e218._comment
@@ -0,0 +1,10 @@
+[[!comment format=mdwn
+ username="http://joeyh.name/"
+ nickname="joey"
+ subject="comment 5"
+ date="2013-04-11T15:00:10Z"
+ content="""
+Setting your repository to manual doesn't prevent it syncing changes out to other repositories, it just prevents it from downloading the contents of files it doesn't already have.
+
+
+"""]]
diff --git a/doc/forum/Drop_with_assistant/comment_6_015134228cb865f97326fbb7193636ea._comment b/doc/forum/Drop_with_assistant/comment_6_015134228cb865f97326fbb7193636ea._comment
new file mode 100644
index 0000000000..b57bfdda97
--- /dev/null
+++ b/doc/forum/Drop_with_assistant/comment_6_015134228cb865f97326fbb7193636ea._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="https://www.google.com/accounts/o8/id?id=AItOawmU_2tE75oyG0h2ZPN4lcroIKEMC8G-otE"
+ nickname="Michael"
+ subject="comment 6"
+ date="2013-04-12T06:31:17Z"
+ content="""
+@joey But a *manual* node will still be the middleman between say a remote host and a local usb connected to the node?
+"""]]
diff --git a/doc/forum/Drop_with_assistant/comment_7_950759930667588f21659cd6d7065fbb._comment b/doc/forum/Drop_with_assistant/comment_7_950759930667588f21659cd6d7065fbb._comment
new file mode 100644
index 0000000000..c4bd03a9c8
--- /dev/null
+++ b/doc/forum/Drop_with_assistant/comment_7_950759930667588f21659cd6d7065fbb._comment
@@ -0,0 +1,17 @@
+[[!comment format=mdwn
+ username="http://edheil.wordpress.com/"
+ ip="173.162.44.162"
+ subject="comment 7"
+ date="2013-04-12T13:30:24Z"
+ content="""
+As I understand it, nothing will \"be the middleman\" in the sense of, if annex A has remotes B and C, grabbing content from B in order to give it to C without regards for whether or not A wants it.  Annex A doesn't think in terms of moving content from one remote to another remote, only in terms of pulling and pushing to remotes individually.
+
+I get the \"move content from remote B to C\" effect by setting numcopies=2 and making both remotes \"backup.\"  If content is only on remote B, it will migrate to A in order to fulfill numcopies.  Then it will migrate to C because of C's preferred content settings.
+
+(Respecting minimum numcopies is higher priority than respecting preferred content settings, so even if A wouldn't normally \"want\" the content, it will receive it if it needs to in order to fulfill numcopies.)
+
+I've got A set to \"client\" so content in archive directories disappears once the content is on B and C (because numcopies is fulfilled, and so A's preferred content can take effect.)
+
+If you had A set to \"manual,\" then it wouldn't automatically drop content once it moved from B to A to C, though.  Because it would only drop things manually.
+
+"""]]
diff --git a/doc/forum/Drop_with_assistant/comment_8_773e540e46adc43487323e8d38ceb2d9._comment b/doc/forum/Drop_with_assistant/comment_8_773e540e46adc43487323e8d38ceb2d9._comment
new file mode 100644
index 0000000000..9f9e0855ce
--- /dev/null
+++ b/doc/forum/Drop_with_assistant/comment_8_773e540e46adc43487323e8d38ceb2d9._comment
@@ -0,0 +1,23 @@
+[[!comment format=mdwn
+ username="http://joeyh.name/"
+ nickname="joey"
+ subject="comment 8"
+ date="2013-04-16T20:46:09Z"
+ content="""
+@edheil, you used to be right about the middleman. But I'm continually improving things. :)
+
+So, consider this situation:
+
+> A (client) --- B (client) ---- C (archive)
+
+If a file is created on A, inside an `archive` directory, B wants a copy, since it's not archived yet. Once B gets the copy, it sends it on the C. At that point, B notices that hey, this file was archived and is in an archive directory, and so it no longer wants its copy and drops it.
+
+(At this point A will also want to drop the file. However, it cannot! This is because git-annex requires positive, direct verification that some other repository has a file before dropping it, and A cannot talk to C to check.
+However, if you set C to be trusted, this verification is bypassed, and then A will be able to drop the file as well.)
+
+This support for middlemen is a new feature, which will be in the next release. You can get it in any recent nightly build.
+
+Amusingly this feature was built without writing any haskell code.. just fine-tuning the preferred content expressions!
+
+We can also consider what happens if B is set to manual. In this case, it won't automatically get the file from A. But if you manually get it, then B will send it on to C. And A will drop the file once it hears that C has it. Due to the manual mode, you'll have to manually drop it from B of course.
+"""]]
diff --git a/doc/forum/Drop_with_assistant/comment_9_d85d120d7219ea6c179c2619a17bdae9._comment b/doc/forum/Drop_with_assistant/comment_9_d85d120d7219ea6c179c2619a17bdae9._comment
new file mode 100644
index 0000000000..9c9aa439c8
--- /dev/null
+++ b/doc/forum/Drop_with_assistant/comment_9_d85d120d7219ea6c179c2619a17bdae9._comment
@@ -0,0 +1,15 @@
+[[!comment format=mdwn
+ username="http://edheil.wordpress.com/"
+ ip="99.54.57.201"
+ subject="comment 9"
+ date="2013-04-17T03:37:48Z"
+ content="""
+Nice!  It sounds like there would be another way to accomplish what I'm currently accomplishing by using numcopies=2 and standard client/backup directories. (backup <--> client <--> backup)
+
+In order to get double backups, I *could* tweak the content settings for my laptop's annex, which is currently set to standard \"client\" mode, so that it's exactly like \"client\" mode except that \"archive\" directories want to drop content which is archived *twice*. (backup <--> tweaked-client <--> backup)
+
+Then I'd get content automatically copying itself out to two archives, and dropping it when it has done so, without touching numcopies.
+
+No reason not to keep doing it the way I'm doing it, since it's simpler, but it's cool to see the assistant getting smarter. :)
+
+"""]]
diff --git a/doc/forum/Duplicate_entries_in_location_tracking_logs.mdwn b/doc/forum/Duplicate_entries_in_location_tracking_logs.mdwn
new file mode 100644
index 0000000000..f5f1354440
--- /dev/null
+++ b/doc/forum/Duplicate_entries_in_location_tracking_logs.mdwn
@@ -0,0 +1,25 @@
+I’ve noticed something odd when inspecting the history of the
+git-annex branch today. Apparently, the branch had some merge
+conflicts during sync that involved two alternative location tracking
+entries that both were for one and the same remote. Both entries only
+differed in their timestamps, and the union merge kept both, so that I
+now have .log files in the annex branch that contain duplicate parts
+like this.
+
+
    +1404838274.151066s 1 a2401cfd-1f58-4441-a2b3-d9bef06220ad
+1406978406.24838s 1 a2401cfd-1f58-4441-a2b3-d9bef06220ad
+
    
+
+The UUID here is my local repository.
+
+The duplication also occurred in the uuid.log:
+
+
    +4316c3dc-5b6d-46eb-b780-948c717b7be5 server timestamp=1404839228.113473s
+4316c3dc-5b6d-46eb-b780-948c717b7be5 server timestamp=1404847241.863051s
+
    
+
+Is this something to be concerned about? The situation somehow arose
+in relation to unannexing a bunch of files and rebasing the master
+branch.
diff --git a/doc/forum/Duplicate_entries_in_location_tracking_logs/comment_1_3afb76397519b8ca8b55958a344f1871._comment b/doc/forum/Duplicate_entries_in_location_tracking_logs/comment_1_3afb76397519b8ca8b55958a344f1871._comment
new file mode 100644
index 0000000000..cc159bd30b
--- /dev/null
+++ b/doc/forum/Duplicate_entries_in_location_tracking_logs/comment_1_3afb76397519b8ca8b55958a344f1871._comment
@@ -0,0 +1,10 @@
+[[!comment format=mdwn
+ username="http://joeyh.name/"
+ ip="209.250.56.112"
+ subject="comment 1"
+ date="2014-08-03T18:59:58Z"
+ content="""
+This is perfectly normal. The next time that file in the git-annex branch is updated for any reason, git-annex will automatically compress the two entries down to a single one. In the meantime, it has no difficulty working out which entry is more recent. This is basically why it's called a log file. ;)
+
+It would be possible to make the union merge code compress as it merges, but this would slow down union merging some, and make it a more conceptually complicated operation. Also, whether the old entry is present in the file or not, git will be storing a copy of that old entry, so it doesn't actually tend to make the git repository any larger. For more on this, see 
+"""]]
diff --git a/doc/forum/Duplicate_entries_in_location_tracking_logs/comment_2_6f327444772ee1e660a12e7442162df5._comment b/doc/forum/Duplicate_entries_in_location_tracking_logs/comment_2_6f327444772ee1e660a12e7442162df5._comment
new file mode 100644
index 0000000000..6cce29757b
--- /dev/null
+++ b/doc/forum/Duplicate_entries_in_location_tracking_logs/comment_2_6f327444772ee1e660a12e7442162df5._comment
@@ -0,0 +1,10 @@
+[[!comment format=mdwn
+ username="zardoz"
+ ip="134.147.14.84"
+ subject="comment 2"
+ date="2014-08-04T08:05:37Z"
+ content="""
+Thanks for the info, Joey! As long as the git tracks the history anyway, this should not increase space consumption that much.
+
+Perhaps it would be useful to have something like «git annex gc» that can clean up these things manually in some situations, e. g. to compact everything before doing a «git annex forget».
+"""]]
diff --git a/doc/forum/Duplicate_files_not_detected.mdwn b/doc/forum/Duplicate_files_not_detected.mdwn
new file mode 100644
index 0000000000..c1fc323f15
--- /dev/null
+++ b/doc/forum/Duplicate_files_not_detected.mdwn
@@ -0,0 +1 @@
+Using v6 repository and `git add .` / `git commit`, it appears that copying files to new names results in new files being stored (even though they are obviously the same file).  Is there some other command that needs to be executed to have duplicate files detected/removed/linked?
diff --git a/doc/forum/Duplicate_files_not_detected/comment_1_d49a34aae5decafed804f45a573c0027._comment b/doc/forum/Duplicate_files_not_detected/comment_1_d49a34aae5decafed804f45a573c0027._comment
new file mode 100644
index 0000000000..04b0515ef4
--- /dev/null
+++ b/doc/forum/Duplicate_files_not_detected/comment_1_d49a34aae5decafed804f45a573c0027._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="jakemagee"
+ avatar="http://cdn.libravatar.org/avatar/85f708f18a2deeefad7719fa70d67c90"
+ subject="comment 1"
+ date="2017-08-02T15:34:19Z"
+ content="""
+Ok, so I figured out the issue with renamed files (needed to change the annex.backend to SHA256) being duplicated.  However, it still seems that appending small changes to large files results in the entire file being duplicated.  Maybe I've confused a feature of git-annex with another project... does git-annex not support chunking files to alleviate duplicating the entire contents of large files with small changes?
+"""]]
diff --git a/doc/forum/Duplicate_files_not_detected/comment_2_c1afe87e818ac3eea66d5216bb15d727._comment b/doc/forum/Duplicate_files_not_detected/comment_2_c1afe87e818ac3eea66d5216bb15d727._comment
new file mode 100644
index 0000000000..53067a0367
--- /dev/null
+++ b/doc/forum/Duplicate_files_not_detected/comment_2_c1afe87e818ac3eea66d5216bb15d727._comment
@@ -0,0 +1,10 @@
+[[!comment format=mdwn
+ username="https://launchpad.net/~liori"
+ avatar="http://cdn.libravatar.org/avatar/e1d0fdc746b3d21bb147160d40815e37b257b9119774d21784939b2d3ba95a91"
+ subject="comment 2"
+ date="2017-08-04T22:13:43Z"
+ content="""
+> does git-annex not support chunking files to alleviate duplicating the entire contents of large files with small changes?
+
+Indeed it doesn't.
+"""]]
diff --git a/doc/forum/Duplicate_files_not_detected/comment_3_c07b9634f571b5f12665b368c0ad3c80._comment b/doc/forum/Duplicate_files_not_detected/comment_3_c07b9634f571b5f12665b368c0ad3c80._comment
new file mode 100644
index 0000000000..58195e6129
--- /dev/null
+++ b/doc/forum/Duplicate_files_not_detected/comment_3_c07b9634f571b5f12665b368c0ad3c80._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="woffs"
+ avatar="http://cdn.libravatar.org/avatar/198790d74209efe4896fd4cfc37ec2a6"
+ subject="comment 3"
+ date="2017-08-16T13:02:56Z"
+ content="""
+Maybe you mixed it up with `bup`
+"""]]
diff --git a/doc/forum/Effectively_replicating_backup_files.mdwn b/doc/forum/Effectively_replicating_backup_files.mdwn
new file mode 100644
index 0000000000..85d8f4b52e
--- /dev/null
+++ b/doc/forum/Effectively_replicating_backup_files.mdwn
@@ -0,0 +1,25 @@
+I currently use duply/duplicity to back up my networked computers to my home server. I have two external HDDs and, every week or so, I bring one of these home and copy the backup files to the hard drive (I leave them on the server to easily restore files and because I have a large hard drive in that). I use some hand-written scripts to keep a copy these backup files in the cloud (Ubuntu One) until they have been copied to both hard drives, ensuring that there are always two copies of the files somewhere offsite. Out of paranoia, I also have some "standalone backups" that are just huge encrypted archives of important folders (say, my entire Photos directory) as at a certain date - in case for some reason duplicity ever stops working or I need to roll something back to a version years earlier. I am less worried about these standalone backups and manually keep one copy of each somewhere.
+
+It sounds like Git-Annex could automate things quite nicely (and give me some neat extras, like knowing where they were). This is how I understand I should do it, but please let me know if it is the right approach or if you have any suggestions:
+
+1. Create a folder on my server called "annex" and make a Git-Annex "large backup" repository in there.
+2. Create a folder within that called "archive" and put a "backup" folder within that. I understand that having the backups within an archive folder will mean that they aren't automatically copied to my desktop machines etc. 
+3. Within that "backup" folder, create two folders, one called "duplicity" and one called "standalone". Put the backups in the respective folders.
+4. Set up gcrypt Git-Annex repositories on my two external HDDs as "small backups". This seems to just start copying files across. That surprised me, as the files are in the archive folder and I thought the default was numcopies=1. Is there some autosync option that I need to turn off? Ideally, I would like it to encrypt/decrypt primarily with my server GPG key (which I'm not worried about copying around my computers), but also encrypt to my personal GPG key (where I'd only put my public key on the server, but I know I will not lose the secret key for that). Am I right that to do that I would need to set the repos up manually with:
+
+    git init --bare /mnt/externalHDD1
+
+    git annex initremote externalHDD1 type=gcrypt gitrepo=/mnt/externalHDD1 keyid=$serverkey keyid=$personalkey
+
+    git annex sync externalHDD1
+
+    Or should the gitrepo be the location of my main Git-Annex repository? How do I make it sync up with my other repos?
+
+5. I understand that I would then need to set numcopies=3 in a .gitattributes file in the "archive/backup/duplicity" directory and, say, a numcopies=2 in the "archive/backup/standalone".
+6. I could then add a cloud repository as a "transfer" repository and Git-Annex should only keep files on that that are not already in the right number of places (similar to what my scripts are doing now).
+7. I have recently upgraded my hard drive, so I have my old 1TB internal hard drive that I will be putting in a cupboard somewhere. I was thinking that I could make this an archive drive for things like one copy of my duplicity/standalone backups. I wouldn't want it to be the only copy of anything. If I just set it as an archive drive, would this work?
+8. Are there more clever ways of doing this? I consider my external HDDs and the cloud repo as "offsite" repositories and ideally there would always be one copy of my backups offsite (in addition to at least three overall). There would also ideally be one of each of my files "live" (in most cases my server) that could instantly push files into a cloud repo and then to wherever I am. Is there any ability to put repositories in groups and write rules like that?
+
+Any thoughts greatly appreciated!
+
+Aaron
diff --git a/doc/forum/Effectively_replicating_backup_files/comment_1_b1ab0da82db076c5244b0dcc95282ddd._comment b/doc/forum/Effectively_replicating_backup_files/comment_1_b1ab0da82db076c5244b0dcc95282ddd._comment
new file mode 100644
index 0000000000..54f7f782a3
--- /dev/null
+++ b/doc/forum/Effectively_replicating_backup_files/comment_1_b1ab0da82db076c5244b0dcc95282ddd._comment
@@ -0,0 +1,14 @@
+[[!comment format=mdwn
+ username="http://joeyh.name/"
+ ip="209.250.56.47"
+ subject="comment 1"
+ date="2013-11-07T16:42:14Z"
+ content="""
+This all sounds reasonable to me.
+
+gitrepo= is the location of the repository you are setting up with gcrypt, not your existing git-annex repository.
+
+numcopies configures the lower bound of the number of copies, not the upper bound. There's no \"small backup\" type, you must mean either small archive, or incremental backup. Either type of repository is going to want to have files that have not previously been archived, or backed up.
+
+You might eventually want to write your own [[preferred_content]] expressions to handle offsite repositories. I'd recommend starting simple and building up.
+"""]]
diff --git a/doc/forum/Effectively_replicating_backup_files/comment_2_472ab9c973b475f7f3ce7e3934f94281._comment b/doc/forum/Effectively_replicating_backup_files/comment_2_472ab9c973b475f7f3ce7e3934f94281._comment
new file mode 100644
index 0000000000..668416c047
--- /dev/null
+++ b/doc/forum/Effectively_replicating_backup_files/comment_2_472ab9c973b475f7f3ce7e3934f94281._comment
@@ -0,0 +1,19 @@
+[[!comment format=mdwn
+ username="https://www.google.com/accounts/o8/id?id=AItOawnFjuvfPpi1kf6l54bxfFUm0Aw_Gf_IO0o"
+ nickname="Aaron"
+ subject="comment 2"
+ date="2013-11-07T19:46:32Z"
+ content="""
+Thanks Joey!
+
+> gitrepo= is the location of the repository you are setting up with gcrypt, not your existing git-annex repository.
+
+Great, thanks. If I set these up outside of the WebApp, what is the best way to link them up to my main repositories?
+
+> numcopies configures the lower bound of the number of copies, not the upper bound. There's no \"small backup\" type, 
+> you must mean either small archive, or incremental backup. Either type of repository is going to want to have files 
+> that have not previously been archived, or backed up.
+
+Yes, I meant incremental backup. Thanks, understood. The page on preferred content ( http://git-annex.branchable.com/preferred_content/ ) made things a bit clearer. I didn't realise that the incremental backups etc want a copy even if it isn't necessary to satisfy numcopies. Does that mean that once it is on one backup drive, it will not be sent to the other unless I increase numcopies?
+
+"""]]
diff --git a/doc/forum/Effectively_replicating_backup_files/comment_3_826493bd59b81786c1f6a56f1c438004._comment b/doc/forum/Effectively_replicating_backup_files/comment_3_826493bd59b81786c1f6a56f1c438004._comment
new file mode 100644
index 0000000000..9088de9e73
--- /dev/null
+++ b/doc/forum/Effectively_replicating_backup_files/comment_3_826493bd59b81786c1f6a56f1c438004._comment
@@ -0,0 +1,10 @@
+[[!comment format=mdwn
+ username="http://joeyh.name/"
+ ip="108.236.230.124"
+ subject="comment 3"
+ date="2013-11-08T17:06:02Z"
+ content="""
+The webapp can use remotes that you have configured at the command line just as well as it can use remotes you configure using the webapp.
+
+Each file only gets backed up to one incremental backup repository. If you have a full backup repository it will (try to) get another copy of every single file.
+"""]]
diff --git a/doc/forum/Efficiently_move_files_from_one_repo_to_another.mdwn b/doc/forum/Efficiently_move_files_from_one_repo_to_another.mdwn
new file mode 100644
index 0000000000..3b9a9a74ab
--- /dev/null
+++ b/doc/forum/Efficiently_move_files_from_one_repo_to_another.mdwn
@@ -0,0 +1,26 @@
+Hi,
+
+## Context
+
+I use git-annex to work with my files on multiple computers, and also to collaborate with other people on my files. Since I don't want to share all my files to all the devices (especially not these of other people), I have several different annex-repos. Of course there is location tracking and file content is not required on all devices. Still, for some devices I want a clear separation in the sense that there should not be any trace that the files from the other repos even exist. However, sometimes I want to change the "permissions" for a file or directory (meaning I have to move it to another repo).
+
+## Problem
+
+Having multiple annex-repos is a reasonable workaround for the use case discussed above. One remaining issue is that moving files or folders from one repo to another is quite inefficient. This is especially true for the remotes.
+
+### Example:
+
+Having two annex-repos (Repo-A and Repo-B) on my laptop. Both repos also exist on my server. All content is synced among the devices. On the laptop I move a large subdir from Repo-A to Repo-B. If I sync both repos with the server afterwards, I have to re-upload all files even though they are already there, but in a different repo.
+
+## Question
+
+Is there an efficient way to move files from one repo to another that works in the example given above?
+
+## Possible approach
+
+I could imagine something like a device-wide location tracking. In this case Repo-B could check "Is this file available in another repo on the same device?" and get it locally. However, for my use case it is really important, that this device-wide location tracking is not synchronized among all the remotes.
+
+### Example:
+
+Again there are Repo-A and Repo-B on laptop and server. Another repo Repo-C is on my laptop and my phone. The phone knows nothing about Repo-A and Repo-B and also is not aware that the server even exists. And this should stay this way. This means that the laptop must not tell the phone that there are other remotes and other repos, as well.
+
diff --git a/doc/forum/Efficiently_move_files_from_one_repo_to_another/comment_1_0cc04ef5e511241a5152a0702f943538._comment b/doc/forum/Efficiently_move_files_from_one_repo_to_another/comment_1_0cc04ef5e511241a5152a0702f943538._comment
new file mode 100644
index 0000000000..62ec2124c1
--- /dev/null
+++ b/doc/forum/Efficiently_move_files_from_one_repo_to_another/comment_1_0cc04ef5e511241a5152a0702f943538._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="Ilya_Shlyakhter"
+ avatar="http://cdn.libravatar.org/avatar/1647044369aa7747829c38b9dcc84df0"
+ subject="comment 1"
+ date="2018-09-24T15:58:03Z"
+ content="""
+Location tracking tracks locations of keys, not of filenames or directory names (unless you use the WORM backend).  Are you sure you care if other people know what keys (checksums) exist on your non-shared system?   If you don't, you could keep the contents of different repos as unconnected/unrelated branches in the same git repo.  See also [[https://git-annex.branchable.com/tips/local_caching_of_annexed_files/]]
+"""]]
diff --git a/doc/forum/Efficiently_move_files_from_one_repo_to_another/comment_2_be029c0880e76d6b5672eed394140438._comment b/doc/forum/Efficiently_move_files_from_one_repo_to_another/comment_2_be029c0880e76d6b5672eed394140438._comment
new file mode 100644
index 0000000000..add32237cb
--- /dev/null
+++ b/doc/forum/Efficiently_move_files_from_one_repo_to_another/comment_2_be029c0880e76d6b5672eed394140438._comment
@@ -0,0 +1,22 @@
+[[!comment format=mdwn
+ username="mario"
+ avatar="http://cdn.libravatar.org/avatar/4c63b0935789d29210d0bd8cad8d7ac7"
+ subject="re: comment 1"
+ date="2018-09-24T18:02:38Z"
+ content="""
+Hi Ilya_Shlyakhter, thanks for your answer.
+
+I'm not sure about using different branches. What prevents other users from just checking out the branches, also the git-annex branch tracks the clear names of all remotes. Moreover, it's a scalability question. Assume the server has thousands (millions?) of files while the phone has only a few. Then it still would be \"bothered\" all the location tracking information of the server.
+
+But one thing in the »local caching« tip raised my attention: \"git config remote.cache.annex-speculate-present true\"
+
+The tip says: \"The annex-speculate-present setting is the essential part. It makes git-annex know that the cache repository may contain the content of any annexed file. So, when getting a file, git-annex will try the cache repository first.\"
+
+Together with \"git config remote.cache.annex-pull false; git config remote.cache.annex-push false\" this could be pretty close to my \"local location tracking\" idea.
+
+It further says: \"The cache repository will remain an empty git repository (except for the content of annexed files). This means that the same cache can be used with multiple different git-annex repositories, without intermingling their git data.\" Thus, there should be no »information leaks« between two repos that both use this cache.
+
+A thing I would have to work around is that I should not do \"git annex --copy --to server --not --in server\". Or, at least before doing this I should always log into the server and do \"git-annex get\" first.
+
+Thanks again for your suggestion. I will give this approach a try.
+"""]]
diff --git a/doc/forum/Efficiently_move_files_from_one_repo_to_another/comment_3_c2f4b10fa1789a4966aed298e00dbc75._comment b/doc/forum/Efficiently_move_files_from_one_repo_to_another/comment_3_c2f4b10fa1789a4966aed298e00dbc75._comment
new file mode 100644
index 0000000000..5e4b0f5eef
--- /dev/null
+++ b/doc/forum/Efficiently_move_files_from_one_repo_to_another/comment_3_c2f4b10fa1789a4966aed298e00dbc75._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="Ilya_Shlyakhter"
+ avatar="http://cdn.libravatar.org/avatar/1647044369aa7747829c38b9dcc84df0"
+ subject="comment 3"
+ date="2018-09-24T18:29:36Z"
+ content="""
+See also config settings remote..annex-readonly , remote..annex-push, remote..annex-pull etc.
+"""]]
diff --git a/doc/forum/Efficiently_move_files_from_one_repo_to_another/comment_4_65bbee3637d09bb6078ec7b118852fd8._comment b/doc/forum/Efficiently_move_files_from_one_repo_to_another/comment_4_65bbee3637d09bb6078ec7b118852fd8._comment
new file mode 100644
index 0000000000..c1000dc45a
--- /dev/null
+++ b/doc/forum/Efficiently_move_files_from_one_repo_to_another/comment_4_65bbee3637d09bb6078ec7b118852fd8._comment
@@ -0,0 +1,10 @@
+[[!comment format=mdwn
+ username="mario"
+ avatar="http://cdn.libravatar.org/avatar/4c63b0935789d29210d0bd8cad8d7ac7"
+ subject="comment 4"
+ date="2018-09-25T09:45:00Z"
+ content="""
+I discovered one potential problem with this approach. The \"server\" in my example typically has \"--bare\" repos created by gitolite. I'm not sure if I can convince such a repo to get files from the cache. But I will try and report my findings.
+
+Is there a way to tell a remote repo (e.g. the server) to get files from another remote?
+"""]]
diff --git a/doc/forum/Efficiently_move_files_from_one_repo_to_another/comment_5_45c1788b39953cd7237f014fa94e086b._comment b/doc/forum/Efficiently_move_files_from_one_repo_to_another/comment_5_45c1788b39953cd7237f014fa94e086b._comment
new file mode 100644
index 0000000000..798fbfea27
--- /dev/null
+++ b/doc/forum/Efficiently_move_files_from_one_repo_to_another/comment_5_45c1788b39953cd7237f014fa94e086b._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="Ilya_Shlyakhter"
+ avatar="http://cdn.libravatar.org/avatar/1647044369aa7747829c38b9dcc84df0"
+ subject="comment 5"
+ date="2018-09-25T15:23:35Z"
+ content="""
+Maybe http://git-annex.branchable.com/bare_repositories/ can help.  Also, for the approach suggested earlier of storing different repos as unrelated branches in the same repo, maybe could use refspecs and hierarchical branch names (with '/') to keep things sufficiently separate.
+"""]]
diff --git a/doc/forum/Efficiently_move_files_from_one_repo_to_another/comment_6_23b3a1a1412866fa06e736f0f5bab583._comment b/doc/forum/Efficiently_move_files_from_one_repo_to_another/comment_6_23b3a1a1412866fa06e736f0f5bab583._comment
new file mode 100644
index 0000000000..25b4989713
--- /dev/null
+++ b/doc/forum/Efficiently_move_files_from_one_repo_to_another/comment_6_23b3a1a1412866fa06e736f0f5bab583._comment
@@ -0,0 +1,9 @@
+[[!comment format=mdwn
+ username="joey"
+ subject="""comment 6"""
+ date="2018-09-25T18:14:19Z"
+ content="""
+If you perform the same move between repos on both the laptop and the
+server, git-annex will generate the same keys for the files on both of
+them, and so won't need to transfer the data again.
+"""]]
diff --git a/doc/forum/Efficiently_move_files_from_one_repo_to_another/comment_7_fba097cd3187d1d8b7517594d875c373._comment b/doc/forum/Efficiently_move_files_from_one_repo_to_another/comment_7_fba097cd3187d1d8b7517594d875c373._comment
new file mode 100644
index 0000000000..a6d2fa8f12
--- /dev/null
+++ b/doc/forum/Efficiently_move_files_from_one_repo_to_another/comment_7_fba097cd3187d1d8b7517594d875c373._comment
@@ -0,0 +1,14 @@
+[[!comment format=mdwn
+ username="mario"
+ avatar="http://cdn.libravatar.org/avatar/4c63b0935789d29210d0bd8cad8d7ac7"
+ subject="comment 7"
+ date="2018-09-25T21:33:17Z"
+ content="""
+Joey, that's true. However, one of the things I like most about git-annex is that I don't have to repeat operations (like renaming/moving of files) on each remote. Therefore, I try to also automate this behavior even if the move operation spans two repos (if both repos exist on the remote).
+
+Probably such \"actions\" (get files from another local repo) could be checked-in as \"todos\" into the repo. Then the remote could try to perform it after a sync. However, I imagine that its hard to design this in a straight forward way and could get pretty messy. Therefore, I thought some kind of \"local location tracking\" could be the cleaner solution.
+
+My preferred solution would be the following. But I guess this would require changes in git-annex: Multiple local repos can share a single .git/annex/objects directory. Typically a repo would only know about its own files in there (and would not care about files put there by other repos at all, especially not leaking them into the location tracking). But if a file is checked-in into a repo (\"sync\", not \"sync --content\") and the content of this file is already present (e.g. because it also exists in another repo), the repo would detect that the file is there and updates location tracking for this file.
+
+I presume the \"local cache\" approach with hardlinks is the next best thing to this idea, that not requires changes to git-annex. E.g. some scripting/hooks that performs a \"get\" operation after a sync..
+"""]]
diff --git a/doc/forum/Efficiently_move_files_from_one_repo_to_another/comment_8_b8f81522e519bab56036cc8628f29ce9._comment b/doc/forum/Efficiently_move_files_from_one_repo_to_another/comment_8_b8f81522e519bab56036cc8628f29ce9._comment
new file mode 100644
index 0000000000..25df0a8f06
--- /dev/null
+++ b/doc/forum/Efficiently_move_files_from_one_repo_to_another/comment_8_b8f81522e519bab56036cc8628f29ce9._comment
@@ -0,0 +1,12 @@
+[[!comment format=mdwn
+ username="mario"
+ avatar="http://cdn.libravatar.org/avatar/4c63b0935789d29210d0bd8cad8d7ac7"
+ subject="comment 8"
+ date="2018-09-25T21:59:10Z"
+ content="""
+Ilya_Shlyakhter, thanks for the link. Apparently \"git-annex get\" also works on bare repos (but I still have to find out how to use this with gitolite, maybe I can write some custom hook or something..)
+
+About the refspecs etc. I think my git-knowledge is not sufficient to fully grasp your idea. But there's one thing that makes me hesitate. I'm looking for a solution that is as less \"special\" as possible. If I have to remember what I was thinking when I designed all this, I'm pretty sure things will break eventually. Thus, in a good solution all thinks should work identically as with regular git-annex. If I have to remember something to do things (e.g. move operations) more efficiently that's okay, as long as things don't break when I just use my repo like a regular git-annex repo. I'm not sure if your approach would provide this.
+
+Also, I suppose I would have to \"design\" one mega-repository for all my current (and probably future) use cases. A loose coupling of otherwise separated repos seems not to be possible, as far as I understand the solution. This could especially become an issue for repos I want to share with others.
+"""]]
diff --git a/doc/forum/Encrypted_Content_Remote_Daemon_.mdwn b/doc/forum/Encrypted_Content_Remote_Daemon_.mdwn
new file mode 100644
index 0000000000..614d276884
--- /dev/null
+++ b/doc/forum/Encrypted_Content_Remote_Daemon_.mdwn
@@ -0,0 +1 @@
+I am currently using rsync.net to keep my notes in sync, both git repo and the content (gpg encrypted) is stored there. Instead of creating two other repos on a server that I have which has git-annex installed where I can actually use remote daemon. Is it possible to create a repo containing both the git repo and the content encrypted from the command line?
diff --git a/doc/forum/Encrypted_Content_Remote_Daemon_/comment_1_96f63e509e23c081c48302274e21db78._comment b/doc/forum/Encrypted_Content_Remote_Daemon_/comment_1_96f63e509e23c081c48302274e21db78._comment
new file mode 100644
index 0000000000..eb97c2680e
--- /dev/null
+++ b/doc/forum/Encrypted_Content_Remote_Daemon_/comment_1_96f63e509e23c081c48302274e21db78._comment
@@ -0,0 +1,10 @@
+[[!comment format=mdwn
+ username="http://joeyh.name/"
+ ip="209.250.56.114"
+ subject="comment 1"
+ date="2014-04-27T00:31:18Z"
+ content="""
+Why not go all the way and encrypt the git repository stored on the remote server too? [[special_remotes/gcrypt]]
+
+When you have git-annex-shell installed on the remote server, the remote daemon should work for gcrypt repositories as well as normal non-encrypted git repositories.
+"""]]
diff --git a/doc/forum/Encrypted_Content_Remote_Daemon_/comment_2_acd0ffdc3f5079265858073c2af81557._comment b/doc/forum/Encrypted_Content_Remote_Daemon_/comment_2_acd0ffdc3f5079265858073c2af81557._comment
new file mode 100644
index 0000000000..a699ad640c
--- /dev/null
+++ b/doc/forum/Encrypted_Content_Remote_Daemon_/comment_2_acd0ffdc3f5079265858073c2af81557._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="https://me.yahoo.com/a/FHnTlSBo1eCGJRwueeKeB6.RCaPbGMPr5jxx8A--#ce0d8"
+ nickname="Hamza"
+ subject="comment 2"
+ date="2014-04-28T09:14:12Z"
+ content="""
+I would love to use gcrypt and I tried it but the problem is at every sync it will ask for my pass phrase even though the key used to encrypt the repo is pass phrase less it goes through the keys in my keychain and asks me to unlock all my keys one by one.
+"""]]
diff --git a/doc/forum/Encrypted_ssh_remote__44___synced_folders.mdwn b/doc/forum/Encrypted_ssh_remote__44___synced_folders.mdwn
new file mode 100644
index 0000000000..ea54026f87
--- /dev/null
+++ b/doc/forum/Encrypted_ssh_remote__44___synced_folders.mdwn
@@ -0,0 +1,87 @@
+Hello,
+
+I hope my understanding of the git-annex webapp is correct, and if not, please correct my wrong thought-pattern.
+
+I have 2 VPS servers and 5 computers. On the 5 computers, I want to sync one folder (with subfolders and files). All the computers are full-disk encrypted, and it is important for me that when the data leaves my computer, it stays encrypted in one way or another. I want to use the two VPS's as an encrypted ssh full backup repository.
+
+The computers are rarely on at the same time, some are off for two weeks, others are on more frequent. However, the VPS's are always on.
+
+Now, is it possible to set up git-annex to support this behaviour? Using an encrypted ssh remote to make sure the folder is in sync on all computers? 
+
+I've set git-annex up on one box, with the two servers as encrypted full backup remotes (without git-annex installed there), and configured my jabber account. That all works, and the files get synced.
+
+Then I've set up a second computer, while the first one is still on. There I've added my jabber account, and created a local repository. Then I selected "Share with your other devices". It now shows my jabber account on both machines, and I saw a few syncing messages, but then there came errors, "Unable to download files from your other devices". I've attached the log at the bottom of this page.
+
+It gives me the option to add a Cloud Repository. If I add the encrypted rsync full backup remote VPS there, will it work and keep my files in sync? is there something I'm doing wrong?
+
+Note that the computers I've set up now are two OS X machines running Mountain Lion, and the servers are CentOS 6.
+
+Log:
+
+
+    [2013-07-15 07:44:17 CEST] main: starting assistant version 4.20130709-g18e5f43
+
+    (scanning...) [2013-07-15 07:44:17 CEST] Watcher: Performing startup scan
+    (started...) [2013-07-15 07:45:08 CEST] main: starting assistant version 4.20130709-g18e5f43
+
+    (scanning...) [2013-07-15 07:45:08 CEST] Watcher: Performing startup scan
+    (started...) [2013-07-15 07:54:31 CEST] XMPPClient: Pairing with stnl in progress
+    rercrrrreveeeec:ccccv vvvv:r:::: e    rsrrrreoeeeesussssorooooucuuuurerrrrc cccceveeee a    vnvvvvaiaaaansnnnnihiiiisesssshdhhhhe eeeed(dddd C    (o((((CnCCCConoooonennnnncnnnneteeeecicccctottttiniiiio oooonrnnnn e    rsrrrreeeeeestsssse eeeetbtttt y    b bbbbypyyyy e    peppppereeeee)eeeer
+    rrrr)))))
+
+
+
+
+    recv: resource vanished (Connection reset by peer)
+    recv:r errceevcs:vo :ur rercseeos uovruacrneci esv havenadin si(hsCehoden dn( eC(coCtnoinnoennce tcriteoisnoe ntr  erbseyes tep teb eybr y)p
+    epeere)r
+    )
+    [2013-07-15 07:55:00 CEST] XMPPSendPack: Syncing with janwxmpp
+    Already up-to-date.
+    recv: resource vanished (Connection reset by peer)
+    [2013-07-15 07:55:04 CEST] XMPPReceivePack: Syncing with janwxmpp
+    To xmpp::janwxmpp@gmail.com
+     * [new branch]      git-annex -> refs/synced/0a41c397-09e8-4957-a6f4-2a6846a4f9d8/cmVsc3RubEBnbWFpbC5jb20=/git-annex
+     * [new branch]      master -> refs/synced/0a41c397-09e8-4957-a6f4-2a6846a4f9d8/cmVsc3RubEBnbWFpbC5jb20=/master
+    [2013-07-15 07:55:25 CEST] XMPPSendPack: Unable to download files from your other devices.
+    [2013-07-15 07:55:25 CEST] XMPPSendPack: Syncing with janwxmpp
+    recv: resource vanished (Connection reset by peer)
+    fatal: Could not read from remote repository.
+
+    Please make sure you have the correct access rights
+    and the repository exists.
+    [2013-07-15 07:57:25 CEST] XMPPSendPack: Unable to download files from your other devices.
+    recv: resource vanished (Connection reset by peer)
+    recv: resource vanished (Connection reset by peer)
+    recv: resource vanished (Connection reset by peer)
+    recv: resource vanished (Connection reset by peer)
+    recv: resource vanished (Connection reset by peer)
+
+    (Recording state in git...)
+
+    (scanning...) [2013-07-15 08:01:36 CEST] Watcher: Performing startup scan
+    (started...) recvr:rrrrrrre eeeeeeecrcccccccvevvvvvvv:s::::::: o       rurrrrrrrereeeeeeescsssssssoeooooooou uuuuuuurvrrrrrrrcaccccccceneeeeeee i       vsvvvvvvvahaaaaaaanennnnnnnidiiiiiiis sssssssh(hhhhhhheCeeeeeeedoddddddd n       (n(((((((CeCCCCCCCocooooooontnnnnnnnninnnnnnneoeeeeeeecnccccccct tttttttiriiiiiiioeooooooonsnnnnnnn e       rtrrrrrrre eeeeeeesbssssssseyeeeeeeet ttttttt p       bebbbbbbbyeyyyyyyy r       p)pppppppe
+    eeeeeeeeeeeeeeerrrrrrrr))))))))
+
+
+
+
+
+
+
+    [2013-07-15 08:44:42 CEST] XMPPSendPack: Syncing with janwxmpp
+    recv: resource vanished (Connection reset by peer)
+    recrvre:ec cvrv:e: s roreuesrsocoueur rcvceae n vivasanhnieisdsh he(edCd o (n(CnCoeoncnntneiecoctnti ioronen s reretes sebetyt   bpbyey e prpe)ee
+    err))
+
+    fatal: Could not read from remote repository.
+
+    Please make sure you have the correct access rights
+    and the repository exists.
+    [2013-07-15 08:46:42 CEST] XMPPSendPack: Unable to download files from your other devices.
+    [2013-07-15 08:46:42 CEST] XMPPSendPack: Syncing with janwxmpp
+    fatal: Could not read from remote repository.
+
+    Please make sure you have the correct access rights
+    and the repository exists.
+    [2013-07-15 08:48:42 CEST] XMPPSendPack: Unable to download files from your other devices.
diff --git a/doc/forum/Encrypted_ssh_remote__44___synced_folders/comment_1_7b9b4ef614c90e0b222d24678d1b9026._comment b/doc/forum/Encrypted_ssh_remote__44___synced_folders/comment_1_7b9b4ef614c90e0b222d24678d1b9026._comment
new file mode 100644
index 0000000000..9ebee5f702
--- /dev/null
+++ b/doc/forum/Encrypted_ssh_remote__44___synced_folders/comment_1_7b9b4ef614c90e0b222d24678d1b9026._comment
@@ -0,0 +1,10 @@
+[[!comment format=mdwn
+ username="http://joeyh.name/"
+ ip="4.154.4.90"
+ subject="comment 1"
+ date="2013-07-16T19:40:20Z"
+ content="""
+Yes, you need to add the rsync remote to the second computer. Once it's set up, they will be able to exchange file contents using it, and using XMPP as a control channel.
+
+If the first computer already has that rsync remote configured, it should push some notes about the existance of that remote to the second one, over XMPP. Then in the webapp, you should just need to click on \"enable\" next to the rsync remote, and it'll prompt for the necessary info to set it up.
+"""]]
diff --git a/doc/forum/Equivalent_to_git_bundle__63__.mdwn b/doc/forum/Equivalent_to_git_bundle__63__.mdwn
new file mode 100644
index 0000000000..d16b91cad9
--- /dev/null
+++ b/doc/forum/Equivalent_to_git_bundle__63__.mdwn
@@ -0,0 +1,10 @@
+Hi,
+
+git provides a neat way to create archives of git repos (or parts thereof): git bundle.
+
+git bundle obviously works with git annex as well, BUT those bundles don't include the actual content (in other words, only the symlinks are bundled up).
+
+Is there a way to get the git bundle functionality with git annex?
+
+THX & Cheers,
+Toby.
diff --git a/doc/forum/Equivalent_to_git_bundle__63__/comment_1_e42936a9bc36fbee69f48e32df303dee._comment b/doc/forum/Equivalent_to_git_bundle__63__/comment_1_e42936a9bc36fbee69f48e32df303dee._comment
new file mode 100644
index 0000000000..af3cf569bf
--- /dev/null
+++ b/doc/forum/Equivalent_to_git_bundle__63__/comment_1_e42936a9bc36fbee69f48e32df303dee._comment
@@ -0,0 +1,9 @@
+[[!comment format=mdwn
+ username="Bram"
+ ip="81.20.68.186"
+ subject="tar"
+ date="2014-10-03T13:44:50Z"
+ content="""
+I would say you're just looking for 'tar cf' or 'tar czf' of the root directory of your repository.
+Unless the (delta) compression that 'git bundle' would perform is a must-have...
+"""]]
diff --git a/doc/forum/Equivalent_to_git_bundle__63__/comment_2_2b8b5c237d8572fdd27202f3502bea96._comment b/doc/forum/Equivalent_to_git_bundle__63__/comment_2_2b8b5c237d8572fdd27202f3502bea96._comment
new file mode 100644
index 0000000000..167353df3e
--- /dev/null
+++ b/doc/forum/Equivalent_to_git_bundle__63__/comment_2_2b8b5c237d8572fdd27202f3502bea96._comment
@@ -0,0 +1,13 @@
+[[!comment format=mdwn
+ username="tdussa"
+ ip="217.84.78.25"
+ subject="tar -- not really"
+ date="2014-10-03T20:40:45Z"
+ content="""
+Hi,
+
+THX for your suggestion.  Unfortunately, git bundle is able to carve out particular slices of a repo, which a simple tar obviously cannot do.  This functionality is much desired.
+
+Cheers,
+Toby.
+"""]]
diff --git a/doc/forum/Error__58___failed_to_push_some_refs.mdwn b/doc/forum/Error__58___failed_to_push_some_refs.mdwn
new file mode 100644
index 0000000000..45d36f51ca
--- /dev/null
+++ b/doc/forum/Error__58___failed_to_push_some_refs.mdwn
@@ -0,0 +1,25 @@
+Hello all, my situation is I have two laptops, one for work and one for home use.  I have a lot of simple text files that I use in v6 unlocked mode (used to use direct mode, but recently switched).  I run the assistant to sync between these hosts using an intermediate server which is an always-on VPS (visarend.solasistim.net in this example).
+
+At some point, these files will start to be desynced, and I'll see messages like these in the log files.
+
+    [2018-09-03 11:34:04.062717904] NetWatcherFallback: Syncing with origin 
+
+    error: Your local changes to the following files would be overwritten by merge:
+            howto/c++.md
+            howto/emacs.mkd
+            howto/git-annex.mkd
+            howto/windows.mkd
+            org/kanban.org
+    Please commit your changes or stash them before you merge.
+    Aborting
+    To visarend.solasistim.net:annex
+     ! [rejected]            master -> synced/master (non-fast-forward)
+    error: failed to push some refs to 'visarend.solasistim.net:annex'
+    hint: Updates were rejected because a pushed branch tip is behind its remote
+    hint: counterpart. Check out this branch and integrate the remote changes
+    hint: (e.g. 'git pull ...') before pushing again.
+    hint: See the 'Note about fast-forwards' in 'git push --help' for details.
+
+Using git-annex 6.20170101.1 on all hosts.
+
+What exactly does this error mean?  How can I get rid of it?  I solve it by rm'ing the repository and re-cloning from visarend.solasistim.net, and integrating any changes manually, but I'm wondering why it's happening.
diff --git a/doc/forum/Error_adding_ssh_remote_in_assistant.mdwn b/doc/forum/Error_adding_ssh_remote_in_assistant.mdwn
new file mode 100644
index 0000000000..e2ef42a19d
--- /dev/null
+++ b/doc/forum/Error_adding_ssh_remote_in_assistant.mdwn
@@ -0,0 +1,15 @@
+I'm trying to add a ssh remote in the web app, but I receive the following error:
+
+    Internal Server Error
+
+    ssh-keygen ["-F","router.eisenacher81.org"] exited 1
+
+The console complains about the known_hosts file:
+
+   /home/michael/.ssh/known_hosts is not a valid known_hosts file.
+   line 44 missing key:  AAAAB3NzaC1yc2EAAAABIwAAAIEAtnX75Qa8YVR...
+   key_read: uudecode AAAAB3NzaC1|1|veTakKhYY3OSqCepiq7WAUK8cxQ=|suoi0YU/lgg781Vz9O7yTao5exY= ssh-rsa                            AAAAB3NzaC1yc2EAAAABIwAAAIEA4bTnEFFiKz4+i3S9DWHJVoV8CD1DsVRJRodWdTA86+x58S0l0rUyJgXbGwScMI+xFCrqzpd4Hgoc4ElRykj5SL+7IuB5ZAe+4ILQPeiL9ck/1Q7uoh7vWiURXr92Hz5tjuEe3QI9H   iKauXj5yj5mGq1VVXLN1CdfQ99G4zSxK7c= failed
+   line 73 invalid key: |1|k9vzo7Ftuxf235It5LbrX36p2f0=|rNHdygip...
+   line 93: invalid hashed name: |1||1|aptgcuUg4ITWhZQLxzRHH8gUIOQ=|Qe2JAlAw++SZosE9ZhQW+fA3twE=...   
+   line 120 missing key:  AAAAB3NzaC1yc2EAAAABIwAAAQEAsHa75NfjyB1...
+   /home/michael/.ssh/known_hosts is not a valid known_hosts file.
diff --git a/doc/forum/Error_adding_ssh_remote_in_assistant/comment_1_eecc0660db4083cc91c5330587f74610._comment b/doc/forum/Error_adding_ssh_remote_in_assistant/comment_1_eecc0660db4083cc91c5330587f74610._comment
new file mode 100644
index 0000000000..c5ca912317
--- /dev/null
+++ b/doc/forum/Error_adding_ssh_remote_in_assistant/comment_1_eecc0660db4083cc91c5330587f74610._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="http://joeyh.name/"
+ ip="4.153.14.141"
+ subject="comment 1"
+ date="2012-09-22T12:40:55Z"
+ content="""
+Looks like you have a corrupt known_hosts file, but I'm pretty sure git-annex did not do it, as it only lets ssh update that as normal. If you delete the problem lines it'll probably work, though you might want to save a copy to investigate what happened.
+"""]]
diff --git a/doc/forum/Error_adding_ssh_remote_in_assistant/comment_2_3e6aad22e8020b12ff7ef914b75281d1._comment b/doc/forum/Error_adding_ssh_remote_in_assistant/comment_2_3e6aad22e8020b12ff7ef914b75281d1._comment
new file mode 100644
index 0000000000..85acfcc1e9
--- /dev/null
+++ b/doc/forum/Error_adding_ssh_remote_in_assistant/comment_2_3e6aad22e8020b12ff7ef914b75281d1._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="http://joeyh.name/"
+ ip="4.153.14.141"
+ subject="comment 2"
+ date="2012-09-27T15:29:56Z"
+ content="""
+I've made the assistant ignore ssh-keygen -F failing.
+"""]]
diff --git a/doc/forum/Error_adding_ssh_remote_in_assistant/comment_3_3ea529e16502071fc0980c6d5c60a036._comment b/doc/forum/Error_adding_ssh_remote_in_assistant/comment_3_3ea529e16502071fc0980c6d5c60a036._comment
new file mode 100644
index 0000000000..799050de06
--- /dev/null
+++ b/doc/forum/Error_adding_ssh_remote_in_assistant/comment_3_3ea529e16502071fc0980c6d5c60a036._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="https://www.google.com/accounts/o8/id?id=AItOawmU_2tE75oyG0h2ZPN4lcroIKEMC8G-otE"
+ nickname="Michael"
+ subject="comment 3"
+ date="2012-09-27T15:42:19Z"
+ content="""
+Okay, cleaning up the file worked for me, but I suspect that these invalid entries were created by a valid program. So ignoring errors in the file is \"good enough\". 
+"""]]
diff --git a/doc/forum/Error_from_git-annex-shell_on_creation_of_gcrypt_special_remote.mdwn b/doc/forum/Error_from_git-annex-shell_on_creation_of_gcrypt_special_remote.mdwn
new file mode 100644
index 0000000000..e38e57c159
--- /dev/null
+++ b/doc/forum/Error_from_git-annex-shell_on_creation_of_gcrypt_special_remote.mdwn
@@ -0,0 +1,28 @@
+I just created a gcrypt special remote and got an error from (I think) `git-annex-shell`, though my local git-annex didn't complain and seemed to think the creation went okay.
+
+    annex $ ga initremote ma type=gcrypt encryption=hybrid gitrepo=ssh://ma/meta/s/spw/local/annex keyid=3B6D411B                                                  
+    (merging synced/git-annex into git-annex...)
+    (Recording state in git...)
+    initremote ma (encryption setup) (hybrid cipher with gpg key 0F56D0553B6D411B) gcrypt: Development version -- Repository format MAY CHANGE
+    gcrypt: Repository not found: ssh://ma/meta/s/spw/local/annex
+    gcrypt: Development version -- Repository format MAY CHANGE
+    gcrypt: Repository not found: ssh://ma/meta/s/spw/local/annex
+    gcrypt: Setting up new repository
+    gcrypt: Remote ID is :id:icUy+qU392R5L5H7g8Yk
+    Counting objects: 111456, done.
+    Compressing objects: 100% (74045/74045), done.
+    Total 111456 (delta 59312), reused 85857 (delta 33717)
+    gcrypt: Encrypting to:  -r 0F56D0553B6D411B
+    gcrypt: Requesting manifest signature
+    To gcrypt::ssh://ma/meta/s/spw/local/annex
+     * [new branch]      git-annex -> git-annex
+    fatal: What do you think I am? A shell?
+    git-annex-shell: git-shell failed
+    ok
+    (Recording state in git...)
+
+Further `git annex sync ma` runs go off without a hitch and the master branch gets pushed a synced/master, though it didn't get pushed on this initial creation as can be seen from the above output.
+
+Is it a bug that my local git-annex said "ok" when there was an error, or is this just a case of my local git-annex falling back to rsync instead of git-annex-shell because I didn't set up my $PATH properly on the remote?  Should I be worried that my gcrypt repo wasn't set up correctly and thus can't be trusted to hold my annexed data and metadata?  Thanks.
+
+Local git-annex: 5.20141125.  Remote git-annex (standalone build): 5.20150113-gcf247cf.
diff --git a/doc/forum/Error_from_git-annex-shell_on_creation_of_gcrypt_special_remote/comment_1_2a69ce8607f97aabdf3cc402c1091d7a._comment b/doc/forum/Error_from_git-annex-shell_on_creation_of_gcrypt_special_remote/comment_1_2a69ce8607f97aabdf3cc402c1091d7a._comment
new file mode 100644
index 0000000000..f94690b35a
--- /dev/null
+++ b/doc/forum/Error_from_git-annex-shell_on_creation_of_gcrypt_special_remote/comment_1_2a69ce8607f97aabdf3cc402c1091d7a._comment
@@ -0,0 +1,30 @@
+[[!comment format=mdwn
+ username="joey"
+ subject="""comment 1"""
+ date="2015-02-04T18:09:49Z"
+ content="""
+It seems that some versions of git-shell will give this unhelpful
+error message. Newer versions instead say 
+"fatal: Interactive git shell is not enabled."
+
+The initial push of the git-annex branch is not what failed.
+It seems that the attempt to run "git-annex-shell gcryptsetup"
+was what failed.
+
+My guess is that the remote server has an older version of git-annex
+installed in /usr/bin, which doesn't understand the gcryptsetup
+command, and that your newer installation of the standalone
+version is not getting run when it sshes in. (A common problem; `~/.bashrc`
+and `~/.bash_profile` are both ignored.) The old version of
+git-annex-shell then passed the "gcryptsetup" command on to git-shell,
+which I guess rejected it like that. (Not sure why.)
+
+git-annex seems to have successfully fallen back to using rsync to
+configure the ssh://ma/meta/s/spw/local/annex repository's core.gcrypt-id.
+(You might want to check that was set correctly.)
+
+Also, since it detected an old/not-working git-annex-shell on the remote,
+it will have set git.remote.ma.gcrypt to "true", rather than
+"shell", and this will cause git-annex to bypass using git-annex-shell
+and instead use direct rsync. 
+"""]]
diff --git a/doc/forum/Error_from_git-annex-shell_on_creation_of_gcrypt_special_remote/comment_2_a112f0bc1a30fa41504e3f52f679053b._comment b/doc/forum/Error_from_git-annex-shell_on_creation_of_gcrypt_special_remote/comment_2_a112f0bc1a30fa41504e3f52f679053b._comment
new file mode 100644
index 0000000000..7679ef0cd4
--- /dev/null
+++ b/doc/forum/Error_from_git-annex-shell_on_creation_of_gcrypt_special_remote/comment_2_a112f0bc1a30fa41504e3f52f679053b._comment
@@ -0,0 +1,11 @@
+[[!comment format=mdwn
+ username="spwhitton"
+ subject="comment 2"
+ date="2015-02-04T21:33:41Z"
+ content="""
+The `core.gcrypt-id` on the remote and local is correctly set (to the same thing) and `git.remote.ma.annex-gcrypt` (I assume you meant this and not `git.remote.ma.gcrypt`) on my local machine is indeed set to `true`.  And the remote seems to work for all purposes I've tried to use it for.  So I guess I'm okay.
+
+There is an older version in `/usr/bin` as you suggest: 3.20120523.  I don't know why I didn't notice this myself before I installed the standalone build.
+
+Thank you for taking the time to reply.
+"""]]
diff --git a/doc/forum/Exclude_specific_files_from_a_special_remote.mdwn b/doc/forum/Exclude_specific_files_from_a_special_remote.mdwn
new file mode 100644
index 0000000000..898ebfdded
--- /dev/null
+++ b/doc/forum/Exclude_specific_files_from_a_special_remote.mdwn
@@ -0,0 +1,3 @@
+I have a special remote configured to/from which I can copy files correctly.
+I would like to be prevent some annexed files to be copied to this remote, automatically (in other words, without excluding them manually on the command line)
+How can this be done?
diff --git a/doc/forum/Exclude_specific_files_from_a_special_remote/comment_1_d5a9f3308e09a9b9460014904d23ef48._comment b/doc/forum/Exclude_specific_files_from_a_special_remote/comment_1_d5a9f3308e09a9b9460014904d23ef48._comment
new file mode 100644
index 0000000000..0375b9239f
--- /dev/null
+++ b/doc/forum/Exclude_specific_files_from_a_special_remote/comment_1_d5a9f3308e09a9b9460014904d23ef48._comment
@@ -0,0 +1,18 @@
+[[!comment format=mdwn
+ username="joey"
+ subject="""comment 1"""
+ date="2015-07-20T16:32:29Z"
+ content="""
+This is a job for [[preferred_content]]!
+
+For example, you could configure the remote to not want
+any flac files, so only mp3, ogg, etc files should be stored on that
+remote:
+
+	git annex wanted myremotename 'exclude=*.flac'
+
+This configuration will automatically be honored if you use
+`git annex sync --content` (or the assistant). If you manually
+use `git annex copy` to copy files to the remote, you'll need to
+add the `--auto` flag to make it honor the preferred content settings.
+"""]]
diff --git a/doc/forum/Exclude_specific_files_from_a_special_remote/comment_2_671790686bd4724aadcadd8d8aad2c98._comment b/doc/forum/Exclude_specific_files_from_a_special_remote/comment_2_671790686bd4724aadcadd8d8aad2c98._comment
new file mode 100644
index 0000000000..8ad63b91b9
--- /dev/null
+++ b/doc/forum/Exclude_specific_files_from_a_special_remote/comment_2_671790686bd4724aadcadd8d8aad2c98._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="Oliv5@43a03b767570a41ba84b9e5c72f3c9816adb22e2"
+ nickname="Oliv5"
+ subject="comment 2"
+ date="2015-07-23T19:07:58Z"
+ content="""
+Yes, this is exactly what I need. It works well indeed. Thks. I didnt understand how this worked before.
+"""]]
diff --git a/doc/forum/External_drive_syncs_git-annex_branch_but_not_master_branch.mdwn b/doc/forum/External_drive_syncs_git-annex_branch_but_not_master_branch.mdwn
new file mode 100644
index 0000000000..76294b6a61
--- /dev/null
+++ b/doc/forum/External_drive_syncs_git-annex_branch_but_not_master_branch.mdwn
@@ -0,0 +1 @@
+I have an external HDD which I setup as a remote manually before I started using the assistant.  When I plug it in files sync fine and the git-annex branch gets synced, but the master branch doesn't; git annex sync doesn't do the job and I have to run git pull.  This is annoying because it means that portable drive is useless as a backup, since it doesn't have the required metadata; I have to manually git pull to make it a useful backup.  How can I make the assistant sync the metadata in--if not the master branch, at least origin/master so I could merge later without the remote available.  Thanks.
diff --git a/doc/forum/External_drive_syncs_git-annex_branch_but_not_master_branch/comment_1_9a909e3d89061adacbd8ed370520250c._comment b/doc/forum/External_drive_syncs_git-annex_branch_but_not_master_branch/comment_1_9a909e3d89061adacbd8ed370520250c._comment
new file mode 100644
index 0000000000..43fe578e17
--- /dev/null
+++ b/doc/forum/External_drive_syncs_git-annex_branch_but_not_master_branch/comment_1_9a909e3d89061adacbd8ed370520250c._comment
@@ -0,0 +1,9 @@
+[[!comment format=mdwn
+ username="http://edheil.wordpress.com/"
+ ip="173.162.44.162"
+ subject="comment 1"
+ date="2012-12-08T21:25:49Z"
+ content="""
+apologies if I misunderstand your question, but I believe that git-annex puts the metadata in the repository's sync/master branch, which you can merge into the repository's local master either manually or by running \"git annex sync\" in the repository.
+
+"""]]
diff --git a/doc/forum/External_drive_syncs_git-annex_branch_but_not_master_branch/comment_2_0dd489b264374b7b1065b89e1ff7561b._comment b/doc/forum/External_drive_syncs_git-annex_branch_but_not_master_branch/comment_2_0dd489b264374b7b1065b89e1ff7561b._comment
new file mode 100644
index 0000000000..d26707c224
--- /dev/null
+++ b/doc/forum/External_drive_syncs_git-annex_branch_but_not_master_branch/comment_2_0dd489b264374b7b1065b89e1ff7561b._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="spwhitton"
+ ip="163.1.166.255"
+ subject="comment 2"
+ date="2012-12-08T21:57:37Z"
+ content="""
+It does, which is what one would have expected I guess, but I didn't look.  Thanks so much.
+"""]]
diff --git a/doc/forum/Extra___38___missing_folders_on_remote.mdwn b/doc/forum/Extra___38___missing_folders_on_remote.mdwn
new file mode 100644
index 0000000000..b78961ccc8
--- /dev/null
+++ b/doc/forum/Extra___38___missing_folders_on_remote.mdwn
@@ -0,0 +1,9 @@
+I created a local annex directory that's an adjusted branch used with the assistant. On another machine, I initialized an annex directory, then made this into a full-backup ssh remote for my local.
+
+After the assistant pushes to the remote, and the remote runs `git annex sync`, the remote is missing some directories and has some extra directories. For example, it has the extra directory `Documents/programs/Documents/programs/`, which has different contents than `Documents/programs/`. Both directories are missing the subdirectory `graphing_experiments/`.
+
+From my local, `git annex whereis Documents/programs/graphing_experiments` says the directory exists on the remote. But it's not there.
+
+I recreated the remote from scratch and the problem persists.
+
+The assistant says the remote is caught up, and is keeping up with new content changes. What could cause this?
diff --git a/doc/forum/Fails_to_get_file_from_s3._How_to_recover__63__.mdwn b/doc/forum/Fails_to_get_file_from_s3._How_to_recover__63__.mdwn
new file mode 100644
index 0000000000..e6bfd4a91f
--- /dev/null
+++ b/doc/forum/Fails_to_get_file_from_s3._How_to_recover__63__.mdwn
@@ -0,0 +1,16 @@
+I have an annex that has an s3 special remote. The s3 remote has been configured with shared encryption and it uses partsize (not chunking). Currently when I try to get a file from the s3 remote, it fails:
+
+    $ git annex get mybigfile.tbz.gpg
+    get mybigfile.tbz.gpg (from s3...)
+    76%         10.6MB/s 57sgpg: WARNING: encrypted message has been manipulated!
+
+      Unable to access these remotes: s3
+
+      Try making some of these repositories available:
+            15ac19e4-223a-4c81-b7f7-797b9b026b86 -- [s3]
+
+      (Note that these git remotes have annex-ignore set: origin)
+    failed
+    git-annex: get: 1 failed
+
+The file is about 3GB. This happens consistently at 76%. No other copy of the file exists. Is there some way I can get the file from s3, either without git annex or just have git annex ignore the error, so that I can inspect the file locally and see if there is anything wrong with it?
diff --git a/doc/forum/Fails_to_get_file_from_s3._How_to_recover__63__/comment_1_c769dcbb9d5f082d54b8a091a44c9de4._comment b/doc/forum/Fails_to_get_file_from_s3._How_to_recover__63__/comment_1_c769dcbb9d5f082d54b8a091a44c9de4._comment
new file mode 100644
index 0000000000..51348f462e
--- /dev/null
+++ b/doc/forum/Fails_to_get_file_from_s3._How_to_recover__63__/comment_1_c769dcbb9d5f082d54b8a091a44c9de4._comment
@@ -0,0 +1,21 @@
+[[!comment format=mdwn
+ username="annexuser"
+ subject="comment 1"
+ date="2016-04-30T17:30:18Z"
+ content="""
+This also happens with a second file in the same repo, although the download seems to complete on this one before the error. The file is about the same size.
+
+    gxg otherfile.tgz.gpg
+    get otherfile.tgz.gpg (from s3...)
+    100%         10.7MB/s 0sgpg: WARNING: encrypted message has been manipulated!
+
+      Unable to access these remotes: s3
+
+      Try making some of these repositories available:
+            15ac19e4-223a-4c81-b7f7-797b9b026b86 -- [s3]
+
+      (Note that these git remotes have annex-ignore set: origin)
+    failed
+    git-annex: get: 1 failed
+
+"""]]
diff --git a/doc/forum/Fails_to_get_file_from_s3._How_to_recover__63__/comment_2_50e35159f7e670ca6b7280121821b494._comment b/doc/forum/Fails_to_get_file_from_s3._How_to_recover__63__/comment_2_50e35159f7e670ca6b7280121821b494._comment
new file mode 100644
index 0000000000..40751e41fa
--- /dev/null
+++ b/doc/forum/Fails_to_get_file_from_s3._How_to_recover__63__/comment_2_50e35159f7e670ca6b7280121821b494._comment
@@ -0,0 +1,21 @@
+[[!comment format=mdwn
+ username="joey"
+ subject="""comment 2"""
+ date="2016-05-03T18:54:41Z"
+ content="""
+What you could try is:
+
+	git config remote.s3.annex-gnupg-options --ignore-mdc-error
+	git config remote.s3.annex-verify false
+
+That should make gpg ignore the error and proceed, and make git-annex
+not try to verify the download either. The resulting file will probably
+be corrupt in some way. It might have a bad chunk in the middle, or be
+truncated, or be garbage past a certian point.
+
+This sounds like something went wrong with the multipart upload to S3.
+What does `git annex info s3` say about the configuration of your S3
+remote? I'd like to reproduce this problem, if possible. Since it hit two
+of your files, it seems to not have been some kind of one-off data
+corruption problem.
+"""]]
diff --git a/doc/forum/Fails_to_get_file_from_s3._How_to_recover__63__/comment_3_471ff0cdb00b311f2a3d801b04e9edef._comment b/doc/forum/Fails_to_get_file_from_s3._How_to_recover__63__/comment_3_471ff0cdb00b311f2a3d801b04e9edef._comment
new file mode 100644
index 0000000000..38f6d97f96
--- /dev/null
+++ b/doc/forum/Fails_to_get_file_from_s3._How_to_recover__63__/comment_3_471ff0cdb00b311f2a3d801b04e9edef._comment
@@ -0,0 +1,26 @@
+[[!comment format=mdwn
+ username="annexuser"
+ subject="comment 3"
+ date="2016-05-04T19:15:16Z"
+ content="""
+Thanks. I will try that and see if I can recover the files. Here is the output of git annex info s3:
+
+    remote: s3
+    description: [s3]
+    uuid: 15ac19e4-223a-4c81-b7f7-797b9b026b86
+    trust: semitrusted
+    cost: 250.0
+    type: S3
+    creds: stored locally
+    bucket: s3-15ac19e4-223a-4c81-b7f7-797b9b026b86
+    endpoint: s3.amazonaws.com
+    port: 80
+    storage class: OtherStorageClass \"STANDARD_IA\"
+    partsize: 1.07 gigabytes
+    public: no
+    encryption: encrypted (encryption key stored in git repository)
+    chunking: none
+    remote annex keys: 121924
+    remote annex size: 680.42 gigabytes
+
+"""]]
diff --git a/doc/forum/Fails_to_get_file_from_s3._How_to_recover__63__/comment_4_e0f80d306a2b4d19d65d268cde25ac31._comment b/doc/forum/Fails_to_get_file_from_s3._How_to_recover__63__/comment_4_e0f80d306a2b4d19d65d268cde25ac31._comment
new file mode 100644
index 0000000000..01df912538
--- /dev/null
+++ b/doc/forum/Fails_to_get_file_from_s3._How_to_recover__63__/comment_4_e0f80d306a2b4d19d65d268cde25ac31._comment
@@ -0,0 +1,7 @@
+[[!comment format=mdwn
+ username="annexuser"
+ subject="comment 4"
+ date="2016-05-05T18:11:49Z"
+ content="""
+Those settings don't seem to have an effect. I still get the same `gpg: WARNING: encrypted message has been manipulated!` error when getting the files after I set those options.
+"""]]
diff --git a/doc/forum/Fails_to_get_file_from_s3._How_to_recover__63__/comment_4_f648049cc1a82f183bbc7dc84cd170e2._comment b/doc/forum/Fails_to_get_file_from_s3._How_to_recover__63__/comment_4_f648049cc1a82f183bbc7dc84cd170e2._comment
new file mode 100644
index 0000000000..1e781a718c
--- /dev/null
+++ b/doc/forum/Fails_to_get_file_from_s3._How_to_recover__63__/comment_4_f648049cc1a82f183bbc7dc84cd170e2._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="joey"
+ subject="""comment 4"""
+ date="2016-05-05T15:55:29Z"
+ content="""
+So you have a lot of data in there. Are all files larger than 1 gb failing
+to download, or only two of them?
+"""]]
diff --git a/doc/forum/Fails_to_get_file_from_s3._How_to_recover__63__/comment_6_c2686480c6c77ea7297f739780ea6d3b._comment b/doc/forum/Fails_to_get_file_from_s3._How_to_recover__63__/comment_6_c2686480c6c77ea7297f739780ea6d3b._comment
new file mode 100644
index 0000000000..748ec08e13
--- /dev/null
+++ b/doc/forum/Fails_to_get_file_from_s3._How_to_recover__63__/comment_6_c2686480c6c77ea7297f739780ea6d3b._comment
@@ -0,0 +1,11 @@
+[[!comment format=mdwn
+ username="annexuser"
+ subject="comment 6"
+ date="2016-05-05T23:57:13Z"
+ content="""
+Yeah, it's a big one :)
+
+Those are the only 2 that are failing. I have a ~50GB file that downloads from s3 without issue. Others in the ~15GB range that download fine also.
+
+There are probably only ~30 files in the repo that are multiple gigabytes. The rest are all small files.
+"""]]
diff --git a/doc/forum/Fails_to_get_file_from_s3._How_to_recover__63__/comment_7_6f3d056e449366746aef4f45673515d5._comment b/doc/forum/Fails_to_get_file_from_s3._How_to_recover__63__/comment_7_6f3d056e449366746aef4f45673515d5._comment
new file mode 100644
index 0000000000..1cc923a3a9
--- /dev/null
+++ b/doc/forum/Fails_to_get_file_from_s3._How_to_recover__63__/comment_7_6f3d056e449366746aef4f45673515d5._comment
@@ -0,0 +1,12 @@
+[[!comment format=mdwn
+ username="joey"
+ subject="""comment 7"""
+ date="2016-05-10T16:26:03Z"
+ content="""
+I see now why remote.s3.annex-gnupg-options is not working: That is
+currently only passed to gpg when it's encrypting, not when it's
+decrypting.
+
+So, instead, edit ~/.gnupg/gpg.conf and put "ignore-mdc-error" in
+a line of its own in that file.
+"""]]
diff --git a/doc/forum/Fails_to_get_file_from_s3._How_to_recover__63__/comment_8_785e012f478631f4b69fefbd0394f1b8._comment b/doc/forum/Fails_to_get_file_from_s3._How_to_recover__63__/comment_8_785e012f478631f4b69fefbd0394f1b8._comment
new file mode 100644
index 0000000000..d4838d0982
--- /dev/null
+++ b/doc/forum/Fails_to_get_file_from_s3._How_to_recover__63__/comment_8_785e012f478631f4b69fefbd0394f1b8._comment
@@ -0,0 +1,9 @@
+[[!comment format=mdwn
+ username="annexuser"
+ subject="comment 8"
+ date="2016-05-11T20:32:54Z"
+ content="""
+I set `git config remote.s3.annex-verify false`, and `ignore-mdc-error` in my `gpg.conf`. That allowed me to succesfully download the files. From there I was able to recover the bulk of the files in the archives. Thanks!
+
+Any ideas what caused the files to become corrupted in the first place? Is there anything I can do different when adding new files and copying them to the s3 special remote to prevent corruption?
+"""]]
diff --git a/doc/forum/Fails_to_get_file_from_s3._How_to_recover__63__/comment_9_e021705e0fae6645a0b6459757260084._comment b/doc/forum/Fails_to_get_file_from_s3._How_to_recover__63__/comment_9_e021705e0fae6645a0b6459757260084._comment
new file mode 100644
index 0000000000..ece2c012ca
--- /dev/null
+++ b/doc/forum/Fails_to_get_file_from_s3._How_to_recover__63__/comment_9_e021705e0fae6645a0b6459757260084._comment
@@ -0,0 +1,18 @@
+[[!comment format=mdwn
+ username="joey"
+ subject="""comment 9"""
+ date="2016-05-12T19:04:00Z"
+ content="""
+My best guess at the moment is that the corruption must have something to
+do with using a partsize.
+
+Since you were able to get at the corrupted files, can you characterize at
+all how they were corrupted?
+
+In particular, did the corruption start at some multiple of N GiB?
+into the file? If so, that would correlate with your 1 GiB part size.
+
+Or was it just a small area of corruption?
+
+Or is the corrupt file truncated?
+"""]]
diff --git a/doc/forum/Fast_cloning_a_repo_initially.mdwn b/doc/forum/Fast_cloning_a_repo_initially.mdwn
new file mode 100644
index 0000000000..52a275786c
--- /dev/null
+++ b/doc/forum/Fast_cloning_a_repo_initially.mdwn
@@ -0,0 +1,3 @@
+Is there any fast way to rsync a direct-mode (assistant) git/annex directory from one machine to another?
+
+Scenario: I was already using unison to synchronize 40GB between my desktop and laptop. I've added git annex to my desktop, but is there a way to bootstrap on my laptop without needing to re-transfer all the files? (I don't have a .git on my laptop yet but all the files are there already.)
diff --git a/doc/forum/Fast_cloning_a_repo_initially/comment_1_0db0caafe30dd1b6f15fb1679dec8f9f._comment b/doc/forum/Fast_cloning_a_repo_initially/comment_1_0db0caafe30dd1b6f15fb1679dec8f9f._comment
new file mode 100644
index 0000000000..5d4aa35090
--- /dev/null
+++ b/doc/forum/Fast_cloning_a_repo_initially/comment_1_0db0caafe30dd1b6f15fb1679dec8f9f._comment
@@ -0,0 +1,10 @@
+[[!comment format=mdwn
+ username="http://joeyh.name/"
+ ip="24.159.78.125"
+ subject="comment 1"
+ date="2014-05-23T15:30:23Z"
+ content="""
+I don't recommend moving direct mode repositories between filesystems, because git-annex relies on the inode as one way to detect if files in a direct mode repo have changed. It will notice that all the inode numbers have changed, and fall back to a mode where it only uses mtime.
+
+Anyway, your situation is better handled by initializing a repository on the desktop, and a separate one on the laptop. Then add remotes tying the repos together, and git-annex sync will just do the right thing, including not transferring files that have already been synced to both places.
+"""]]
diff --git a/doc/forum/Fast_cloning_a_repo_initially/comment_2_e17abc209c2b3ba20158be76801ce04a._comment b/doc/forum/Fast_cloning_a_repo_initially/comment_2_e17abc209c2b3ba20158be76801ce04a._comment
new file mode 100644
index 0000000000..0a07264cd0
--- /dev/null
+++ b/doc/forum/Fast_cloning_a_repo_initially/comment_2_e17abc209c2b3ba20158be76801ce04a._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="jamieson"
+ ip="71.11.161.246"
+ subject="Thanks!"
+ date="2014-05-23T16:11:35Z"
+ content="""
+I'll do it that way, thanks!!
+"""]]
diff --git a/doc/forum/Feature_Request__58___Specify_file_size_when_adding_files_to_URL_backend.mdwn b/doc/forum/Feature_Request__58___Specify_file_size_when_adding_files_to_URL_backend.mdwn
new file mode 100644
index 0000000000..bc7be2775c
--- /dev/null
+++ b/doc/forum/Feature_Request__58___Specify_file_size_when_adding_files_to_URL_backend.mdwn
@@ -0,0 +1,10 @@
+It seems that the only way to add a key with the URL backend and with an explicit file size is to use addurl --fast, which requires a a connection to be made to the remote server. However, it becomes impractical when I have tens of thousands of URLs, and I end up hammering the server, when I know ahead of time exactly what the file sizes are. I know it is possible to create URL keys with a size attribute. It would be great to be able to have something like addurl --size=342345 --raw, or be able to pass a size to registerurl.
+
+I have figured out how to construct a SHA256 key outside of git-annex, but I've looked at the source and I admit I'm a bit over my head with manually constructing a URL key.
+
+
+I've been doing a lot of experimenting with [pre-seeding](https://git-annex.branchable.com/forum/__34__Preseeding__34___a_special_remote/), which I would like to share. I think it could be a really cool application of git-annex.
+
+So far, I've been able to generate a distfiles repo automatically for Gentoo portage, which contains BLAKE2B512 signatures for all of their distfiles. I think there could potentially be some really cool applications to keeping track of large sets of files, and I want to eventually try to make it so that portage could fetch distfiles from other users over ipfs.
+
+But, there are many interesting collections of files that are in open directories, and all I have is the filenames and sizes, and being able to quickly generate a repo with knowledge of the exact size would make it really easy to divvy up downloading and mirroring with git. I have a script that converts an FTP server to a git-annex repo with a WORM backend, but I still think that using URL keys would be a better representation of where the resource is.
diff --git a/doc/forum/Feature_Request__58___Specify_file_size_when_adding_files_to_URL_backend/comment_1_9d2fa7d796fd40718d0df9feeea654aa._comment b/doc/forum/Feature_Request__58___Specify_file_size_when_adding_files_to_URL_backend/comment_1_9d2fa7d796fd40718d0df9feeea654aa._comment
new file mode 100644
index 0000000000..b3c3f261e4
--- /dev/null
+++ b/doc/forum/Feature_Request__58___Specify_file_size_when_adding_files_to_URL_backend/comment_1_9d2fa7d796fd40718d0df9feeea654aa._comment
@@ -0,0 +1,20 @@
+[[!comment format=mdwn
+ username="joey"
+ subject="""comment 1"""
+ date="2018-05-29T16:41:34Z"
+ content="""
+I wonder if you've tried addurl --fast with a recent version of git-annex?
+
+Since it recently stopped shelling out to curl for every url, it keeps a
+http connection open to the server, and is limited only by network
+latency. Combined with -J10, it might be fast enough now to not be a
+serious bottleneck.
+
+The mangling of urls in keys is ad-hoc and I'd rather not document that
+as part of git-annex's interface, so building your own url keys is not
+recommended.
+
+I could see adding a size value to git-annex registerurl, and
+I think you'd also want git-annex fromkey to have that too so you can
+add the keys to the working tree.
+"""]]
diff --git a/doc/forum/Feature_Request__58___add_filename_to_hash_objects.mdwn b/doc/forum/Feature_Request__58___add_filename_to_hash_objects.mdwn
new file mode 100644
index 0000000000..173cd2d949
--- /dev/null
+++ b/doc/forum/Feature_Request__58___add_filename_to_hash_objects.mdwn
@@ -0,0 +1,6 @@
+hi,  
+i just did git annex unused in 2 repos, one with WORM and one with SHA1.   
+with WORM i instantly could see which file it was, with SHA1 not.  
+
+so would it be possible to just add the first seen filename to the hashes identifier? the name could just be ignored but it would help with unused to see which file i am searching for.  
+thanks!
diff --git a/doc/forum/Feature_Request__58___add_filename_to_hash_objects/comment_1_73dc0a9cad486cf2d34faf064c6193b1._comment b/doc/forum/Feature_Request__58___add_filename_to_hash_objects/comment_1_73dc0a9cad486cf2d34faf064c6193b1._comment
new file mode 100644
index 0000000000..8a2f9fb009
--- /dev/null
+++ b/doc/forum/Feature_Request__58___add_filename_to_hash_objects/comment_1_73dc0a9cad486cf2d34faf064c6193b1._comment
@@ -0,0 +1,10 @@
+[[!comment format=mdwn
+ username="http://joeyh.name/"
+ ip="64.134.31.139"
+ subject="comment 1"
+ date="2013-10-16T15:19:25Z"
+ content="""
+Adding a filename would defeat deduplication. You suggest adding the \"first seen\" filename, but then git-annex would need to maintain a lookup between hashes and hashes with the first seen filename to use when adding new files.
+
+There is already [[todo/wishlist: option to print more info with 'unused']].
+"""]]
diff --git a/doc/forum/Feature_Request__58___add_filename_to_hash_objects/comment_2_f818b3ecfeb1d1dd83df4668c061718a._comment b/doc/forum/Feature_Request__58___add_filename_to_hash_objects/comment_2_f818b3ecfeb1d1dd83df4668c061718a._comment
new file mode 100644
index 0000000000..638492822e
--- /dev/null
+++ b/doc/forum/Feature_Request__58___add_filename_to_hash_objects/comment_2_f818b3ecfeb1d1dd83df4668c061718a._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="orb"
+ ip="178.11.228.69"
+ subject="comment 2"
+ date="2013-10-17T21:12:09Z"
+ content="""
+thanks! (i hope a thanks is wanted, i know people who are just distracted and consider \"thanks\" posts unnecessary)
+"""]]
diff --git a/doc/forum/Feature_request__58___Multiple_concurrent_transfers.mdwn b/doc/forum/Feature_request__58___Multiple_concurrent_transfers.mdwn
new file mode 100644
index 0000000000..a9d9fb162d
--- /dev/null
+++ b/doc/forum/Feature_request__58___Multiple_concurrent_transfers.mdwn
@@ -0,0 +1,19 @@
+I'm sitting on a pretty fast connection, and often git-annex isn't using my entire upstream to fill the remotes.
+
+Lets say i have an annex with 10 x 100mb files. and my git-annex has 5 special remotes that needs to be filled.
+
+I would like to have git-annex do 5 concurrent uploads(one to each remote). Currently git-annex only uploads to one special-remote at a time (meaning the 4 remaining servers are just waiting).
+
+It would also be nice to have a per remote number of threads. Especially if adding a directory with 1000 small files in it(say 100bytes each), having 5 transfer threads to each remote would make the sync complete much faster.
+
+
+For now i've made a shell script that i call:
+
+    # for j in `seq -w 0 10`; do echo DOING $j; for i in `curl "http://127.0.0.1:$1/?auth=$2" | grep "continue" | gawk -F\"  ' { print $8 } '`; do curl "http://127.0.0.1:$1$i"; sleep 0; done; done
+
+But it is very rough, and basically just starts all transfers on the page. Which means i currently have 315 active transfers running. whoops.
+
+I could improve the shell script. But it really would be quite a bit niftier to have it as settings in git-annex.
+
+Sincerely
+Tobias
diff --git a/doc/forum/Feature_request__58___Multiple_concurrent_transfers/comment_1_aa666349e9b524fd5394a259ad622c3b._comment b/doc/forum/Feature_request__58___Multiple_concurrent_transfers/comment_1_aa666349e9b524fd5394a259ad622c3b._comment
new file mode 100644
index 0000000000..4ff1504ff9
--- /dev/null
+++ b/doc/forum/Feature_request__58___Multiple_concurrent_transfers/comment_1_aa666349e9b524fd5394a259ad622c3b._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="unqueued"
+ avatar="http://cdn.libravatar.org/avatar/3bcbe0c9e9825637ad7efa70f458640d"
+ subject="comment 1"
+ date="2018-05-28T14:46:04Z"
+ content="""
+I have been toying with \"external\" transfer handlers. Sometimes when I am dealing with very large files, it makes more sense to just copy the keys outside of git-annex, and then after they are copied, run fsck. Perhaps at some point there could be a way to have a transfer \"shim\" of some kind, for handling transfers.
+"""]]
diff --git a/doc/forum/Feature_request__58___git_annex_copy_--auto_does_the_right_thing.mdwn b/doc/forum/Feature_request__58___git_annex_copy_--auto_does_the_right_thing.mdwn
new file mode 100644
index 0000000000..0a069340c0
--- /dev/null
+++ b/doc/forum/Feature_request__58___git_annex_copy_--auto_does_the_right_thing.mdwn
@@ -0,0 +1,5 @@
+I'm not sure of the right place for feature requests; sorry if this is incorrect.
+
+It occurred to me that it would be neat if running git annex copy --auto copied enough copies of the file to connected remotes that don't have it to fulfil numcopies (if possible). Further coolness could come from choosing the remotes smartly (based, for example, on availability, access cost, remote disk space, trust level... you get the idea)
+
+Having an option like this would allow the user to use numcopies to backup files to wherever possible, if they don't really care about where those copies go.
diff --git a/doc/forum/Feature_request__58___git_annex_copy_--auto_does_the_right_thing/comment_1_bbac7d0810a79eb1f42a01e1b31d5c4c._comment b/doc/forum/Feature_request__58___git_annex_copy_--auto_does_the_right_thing/comment_1_bbac7d0810a79eb1f42a01e1b31d5c4c._comment
new file mode 100644
index 0000000000..cd834e08d4
--- /dev/null
+++ b/doc/forum/Feature_request__58___git_annex_copy_--auto_does_the_right_thing/comment_1_bbac7d0810a79eb1f42a01e1b31d5c4c._comment
@@ -0,0 +1,12 @@
+[[!comment format=mdwn
+ username="http://joeyh.name/"
+ ip="4.152.108.183"
+ subject="comment 1"
+ date="2013-02-09T02:32:23Z"
+ content="""
+You can kind of do this now by:
+
+for remote in $(git remote); do git annex copy --auto --to $remote; done
+
+Although without some of the potential smarts of course.
+"""]]
diff --git a/doc/forum/Feature_request__58___webapp_support_for_centralized_bare_repos.mdwn b/doc/forum/Feature_request__58___webapp_support_for_centralized_bare_repos.mdwn
new file mode 100644
index 0000000000..2fdab28a77
--- /dev/null
+++ b/doc/forum/Feature_request__58___webapp_support_for_centralized_bare_repos.mdwn
@@ -0,0 +1 @@
+I would like it if the webapp could support setting up [bare remotes](http://git-annex.branchable.com/tips/centralized_git_repository_tutorial/) to assist with syncing between two machines that are never online at the same time.
diff --git a/doc/forum/Feature_request__58___webapp_support_for_centralized_bare_repos/comment_1_0522e3df450efb6b2d264857e07997dc._comment b/doc/forum/Feature_request__58___webapp_support_for_centralized_bare_repos/comment_1_0522e3df450efb6b2d264857e07997dc._comment
new file mode 100644
index 0000000000..16c58c4692
--- /dev/null
+++ b/doc/forum/Feature_request__58___webapp_support_for_centralized_bare_repos/comment_1_0522e3df450efb6b2d264857e07997dc._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="ovidiu@66ace8a8d99ce938b0538ffa0f26d30db02a9626"
+ nickname="ovidiu"
+ subject="comment 1"
+ date="2016-04-04T11:24:38Z"
+ content="""
+seconded
+"""]]
diff --git a/doc/forum/Feature_request__58___webapp_support_for_centralized_bare_repos/comment_2_e0fc126b0d500d41a563b076a4f6ed6d._comment b/doc/forum/Feature_request__58___webapp_support_for_centralized_bare_repos/comment_2_e0fc126b0d500d41a563b076a4f6ed6d._comment
new file mode 100644
index 0000000000..bbcd0e792d
--- /dev/null
+++ b/doc/forum/Feature_request__58___webapp_support_for_centralized_bare_repos/comment_2_e0fc126b0d500d41a563b076a4f6ed6d._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="ovidiu@66ace8a8d99ce938b0538ffa0f26d30db02a9626"
+ nickname="ovidiu"
+ subject="comment 2"
+ date="2016-04-04T11:25:33Z"
+ content="""
+sorry, forgot to check the box for: \"email replies to me\" which was the sole purpose of commenting.
+"""]]
diff --git a/doc/forum/File_downloaded_again_and_again.mdwn b/doc/forum/File_downloaded_again_and_again.mdwn
new file mode 100644
index 0000000000..c4e06be80e
--- /dev/null
+++ b/doc/forum/File_downloaded_again_and_again.mdwn
@@ -0,0 +1,53 @@
+Hello,
+
+I have two repos, asaru in group client, marduk in groups archive and backup. Both are wanted=standard.
+
+Doing a ```git annex sync --content``` at asaru takes a long time and does the same action every time I run it (even if ran consecutively):
+
+```
+florian@asaru ~/Documents (git)-[master] % git annex sync --content
+commit  
+Auf Branch master
+Ihr Branch ist auf dem selben Stand wie 'marduk/master'.
+nichts zu committen, Arbeitsverzeichnis unverändert
+ok
+pull marduk 
+ok
+pull marduk.dynv6 
+ok
+get .localized (from marduk...) 
+SHA256E-s0--e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855
+              0 100%    0.00kB/s    0:00:00 (xfr#1, to-chk=0/1)
+(checksum...) ok
+get Bewerbung/bmw.aux (from marduk...) 
+SHA256E-s8--3345a075a22fb3bbc04567a1e5660ed0240d53f5881f458a2234dcd42a78d335.aux
+              8 100%    7.81kB/s    0:00:00 (xfr#1, to-chk=0/1)
+(checksum...) ok
+drop archive/Programmierung/zope/lib/python/Homefolder/.svn/empty-file (locking marduk.dynv6...) ok
+drop archive/Schule/Geschichte/Biographie 3/biographie.aux (locking marduk.dynv6...) ok
+pull marduk 
+ok
+pull marduk.dynv6 
+ok
+(recording state in git...)
+push marduk 
+Zähle Objekte: 8, Fertig.
+Delta compression using up to 4 threads.
+Komprimiere Objekte: 100% (8/8), Fertig.
+Schreibe Objekte: 100% (8/8), 679 bytes | 0 bytes/s, Fertig.
+Total 8 (delta 6), reused 0 (delta 0)
+To ssh://marduk.local/home/florian/Documents
+   bb9e640..b0fa6a1  git-annex -> synced/git-annex
+ok
+push marduk.dynv6 
+Everything up-to-date
+ok
+git annex sync --content  50,98s user 3,60s system 40% cpu 2:16,17 total
+```
+
+Taking a long time, as you can see in the last line. marduk.dynv6 and marduk are both (internet and local) remotes to Marduk.
+
+What is wrong here?
+
+Thanks,
+Florian
diff --git a/doc/forum/File_downloaded_again_and_again/comment_1_666b907935d467d5d7c0e57831d76631._comment b/doc/forum/File_downloaded_again_and_again/comment_1_666b907935d467d5d7c0e57831d76631._comment
new file mode 100644
index 0000000000..f4beaf50cc
--- /dev/null
+++ b/doc/forum/File_downloaded_again_and_again/comment_1_666b907935d467d5d7c0e57831d76631._comment
@@ -0,0 +1,28 @@
+[[!comment format=mdwn
+ username="joey"
+ subject="""comment 1"""
+ date="2016-07-19T18:53:04Z"
+ content="""
+Notice that the file it gets is empty, and the file it drops also appears
+to be an empty file, at least based on its filename. So, there is a single
+key used for both empty files.
+
+(There's also the two .aux files, which probably also have the same 8 byte
+content.)
+
+Your preferred content settings make one of the pair of files be wanted,
+and the other one not be wanted. In this situation, `get --auto` will get
+the key used for both files, and `drop --auto` will drop it. `sync
+--content` does both things, so repeatedly gets and then drops the key.
+
+One way to deal with this is to delete one of the copies of the file.
+Then it won't be in this mixed up state. But you may not want to do that.
+
+Another way to deal with it is to unlock one of the files, and add it back
+using `git annex add --backend=WORM $file`. This will make git-annex
+allocate a new key, only for that copy of the file.
+
+I do think this is a bug in git-annex, but not one that can immediately be
+fixed. I've opened a bug report at
+[[bugs/indeterminite_preferred_content_state_for_duplicated_file]].
+"""]]
diff --git a/doc/forum/File_too_big_for_file_system_of_git_remote.mdwn b/doc/forum/File_too_big_for_file_system_of_git_remote.mdwn
new file mode 100644
index 0000000000..6a1330d5c9
--- /dev/null
+++ b/doc/forum/File_too_big_for_file_system_of_git_remote.mdwn
@@ -0,0 +1 @@
+I have a file which is 5.4 GB big and so it won't fit on my fat32 formated flash drive. Since it's a git remote I can't enable chunking. Is there another way to go about this, or will that file simply not fit?
diff --git a/doc/forum/File_too_big_for_file_system_of_git_remote/comment_1_937df2f51506c42ddd2345735efd8f21._comment b/doc/forum/File_too_big_for_file_system_of_git_remote/comment_1_937df2f51506c42ddd2345735efd8f21._comment
new file mode 100644
index 0000000000..ae88326728
--- /dev/null
+++ b/doc/forum/File_too_big_for_file_system_of_git_remote/comment_1_937df2f51506c42ddd2345735efd8f21._comment
@@ -0,0 +1,12 @@
+[[!comment format=mdwn
+ username="joey"
+ subject="""comment 1"""
+ date="2017-06-06T15:33:17Z"
+ content="""
+You can add a directory special remote to the same flash drive,
+and enable chunking in that special remote, and store the file in there.
+
+Alternatively, there might be a less limited filesystem that you could
+reformat the drive with. Depending on the operating systems that this
+needs to work with..
+"""]]
diff --git a/doc/forum/Filesystem_recommendation___40__ext4__44___btrfs__44___xfs__44___you_name_it__41____63__.mdwn b/doc/forum/Filesystem_recommendation___40__ext4__44___btrfs__44___xfs__44___you_name_it__41____63__.mdwn
new file mode 100644
index 0000000000..95c745d49a
--- /dev/null
+++ b/doc/forum/Filesystem_recommendation___40__ext4__44___btrfs__44___xfs__44___you_name_it__41____63__.mdwn
@@ -0,0 +1,74 @@
+# Context: git annex "rules"
+
+First, I have to tell that `git-annex` is a big win so far (provided you have required git and other knowledge).
+
+My main use case for `git-annex` contains around 260,000 annexed files, for a total of 1.3 terabytes.
+
+Tried regular git on a subset of it and extrapolated.  Count 6-30 hours for simple operations like `git status`.  Plus huge space used for compressed copies of files.  All on current hardware: Intel i7 2.5GHz, 16GB RAM, hard disk raw performance 50-100+ MBytes/s.
+
+Using `git-annex` is a big win :
+
+* `git status` take not hours but about one to a few minutes (mostly thanks to decoupling voluntary data change and accidental data corruption and handling them at different times)
+* No space wasted.
+* "Just ask" push of data to remotes to maintain the required number of copies.
+* Easy fetch of missing data from remotes.
+* Corruption detection turns bad data into missing data which can just be fetched again.
+* Data is still there and readable without `git-annex` or even git.
+
+# Question
+
+**One to a few minutes for a `git status` is still long.  It is faster the second time (seconds) but still.  Can we reduce time for `git status` ?**
+
+
+
+This questions looks not `git-annex` specific.
+
+"Making `git status` fast is a git-level question".  In a sense it is, though `git-annex` repos are an extreme case of git-repository as they contain in most cases a lot of symlinks which look like small files at the filesystem level.
+
+Which makes the question more filesystem-level anyway, yet relevant to ask here.
+
+# Required features of a filesystem
+
+`git-annex` basically needs a filesystem that allows:
+
+* long file and directory names (hash in `.git/annex/objects` directory and file names)
+* long total paths
+* symlinks
+* hard links
+* unix permissions (to make hashed files immutable)
+
+More details e.g. on [day 188 crippled filesystem support](https://git-annex.branchable.com/design/assistant/blog/day_188__crippled_filesystem_support/)
+
+# Wished features of a filesystem
+
+Fast operations!
+
+Reiserfs, reiser4, btrs are said to be very efficient whe dealing with small files and symlinks thanks to [Block suballocation](https://en.wikipedia.org/wiki/Block_suballocation).
+
+# Question, restated.
+
+* Some users of `git-annex` will dedicate whole hard drives to `git-annex` repos, like I do.
+* Reading big files (from megabytes to gigabytes) from any decent filesystem implementation will yield similar performance.
+* Which leaves us to choose the filesystem based on safety and performance of reading a git repository with 100k to 1M symlinks.
+
+**Can anyone recommend a filesystem to use for fast git-annex level operations ?**
+
+## Anticipations
+
+Based on previous experience:
+
+* ext4: default choice, good.  Why chase for better?
+* "challenger filesystem X": might get better performance today (X=btrfs, X=reiser4)... or not. Might get dropped in the future (X=reiser4,X=btrfs). Might have bugs? All this might not be actually important, just do another git clone and reformat your drive to the new filesystem of the day.
+* btrfs: might waste a lot of space and actually have slower performance 
+* xfs?
+* a small and lightweight partition for metadata with a high performance filesystem and `.git/annex/objects` symlink to the big data-dedicated filesystem.  Might be better because of smaller head movements back and forth.  Size has to be decided in advance.
+
+Or perhaps all this is just nitpicking.
+
+Any thoughts?
+
+# References:
+
+* [Fast and slow symlinks - Unix & Linux Stack Exchange](http://unix.stackexchange.com/questions/147535/fast-and-slow-symlinks)
+* [Block suballocation - Wikipedia](https://en.wikipedia.org/wiki/Block_suballocation)
+* [git-annex across two filesystems](http://git-annex.branchable.com/forum/git-annex_across_two_filesystems/)
diff --git a/doc/forum/Filesystem_recommendation___40__ext4__44___btrfs__44___xfs__44___you_name_it__41____63__/comment_1_fcf7f132f771de28553bd20f4fb014cb._comment b/doc/forum/Filesystem_recommendation___40__ext4__44___btrfs__44___xfs__44___you_name_it__41____63__/comment_1_fcf7f132f771de28553bd20f4fb014cb._comment
new file mode 100644
index 0000000000..a671586f9a
--- /dev/null
+++ b/doc/forum/Filesystem_recommendation___40__ext4__44___btrfs__44___xfs__44___you_name_it__41____63__/comment_1_fcf7f132f771de28553bd20f4fb014cb._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="yarikoptic"
+ avatar="http://cdn.libravatar.org/avatar/f11e9c84cb18d26a1748c33b48c924b4"
+ subject="comment 1"
+ date="2018-01-24T03:10:53Z"
+ content="""
+Just ran into this question.  Might still be of interest: for our datalad.org project I did some benchmarking of various filesystems a few years back. Here you can find the final report: [http://old.datalad.org/test_fs_analysis.html](http://old.datalad.org/test_fs_analysis.html) -- I settled on a bunch of software raid5s with BTRFS on top of them.  So far (for years), quite good.  snapshots and [btrbk](http://digint.ch/btrbk/) are also a blessing.  I use the same btrfs also as docker backend.
+"""]]
diff --git a/doc/forum/Find_files_that_lack_a_certain_field_in_metadata.mdwn b/doc/forum/Find_files_that_lack_a_certain_field_in_metadata.mdwn
new file mode 100644
index 0000000000..b0fe9ddaab
--- /dev/null
+++ b/doc/forum/Find_files_that_lack_a_certain_field_in_metadata.mdwn
@@ -0,0 +1,5 @@
+Is there any way to find all files that do not have a certain field assigned in metadata. E.g. I want to find all files that do not have an author field set and
+
+    git-annex find --not --metadata "author=*"
+
+doesn't give any results.
diff --git a/doc/forum/Find_files_that_lack_a_certain_field_in_metadata/comment_1_476e52563ccd3ad1b43e3a2da4dfaa82._comment b/doc/forum/Find_files_that_lack_a_certain_field_in_metadata/comment_1_476e52563ccd3ad1b43e3a2da4dfaa82._comment
new file mode 100644
index 0000000000..fbfb6ebb69
--- /dev/null
+++ b/doc/forum/Find_files_that_lack_a_certain_field_in_metadata/comment_1_476e52563ccd3ad1b43e3a2da4dfaa82._comment
@@ -0,0 +1,10 @@
+[[!comment format=mdwn
+ username="http://joeyh.name/"
+ ip="209.250.56.172"
+ subject="comment 1"
+ date="2014-02-21T22:36:51Z"
+ content="""
+--metadata does not support globs, so your example is asking for all files that don't have an author field with a literal \"*\" value. When I try that command, it lists all files ... as expected.
+
+It seems that adding glob support to it would get to the result you want, and makes sense to parallel git annex view. Change made in git!
+"""]]
diff --git a/doc/forum/Find_out_if_file_is_in_annex__63__.mdwn b/doc/forum/Find_out_if_file_is_in_annex__63__.mdwn
new file mode 100644
index 0000000000..9d46808400
--- /dev/null
+++ b/doc/forum/Find_out_if_file_is_in_annex__63__.mdwn
@@ -0,0 +1,8 @@
+Hi,
+
+I am currently trying to move nearly all of my files into git-annex, and I was wondering if there is a (plumbing-level?) way to easily figure out if a file I find on a random hard disk is already contained in my annex repository?
+
+I.e., is there an easy way to compute the key for a file (which is not in an annex) and look up if the file is already contained in a given annex repository? I've looked a bit at the plumbing-level commands but none seem to exactly fit the bill :-/
+
+Cheers,
+Alex
diff --git a/doc/forum/Find_out_if_file_is_in_annex__63__/comment_1_3936ff53d5c9f7e5795b15beadf437c7._comment b/doc/forum/Find_out_if_file_is_in_annex__63__/comment_1_3936ff53d5c9f7e5795b15beadf437c7._comment
new file mode 100644
index 0000000000..f70436ab42
--- /dev/null
+++ b/doc/forum/Find_out_if_file_is_in_annex__63__/comment_1_3936ff53d5c9f7e5795b15beadf437c7._comment
@@ -0,0 +1,9 @@
+[[!comment format=mdwn
+ username="joey"
+ subject="""comment 1"""
+ date="2015-12-11T15:20:55Z"
+ content="""
+It's not plumbing level, but `git-annex import --deduplicate`
+or `git-annex import --skip-duplicates` are meant to handle this sort
+of thing.
+"""]]
diff --git a/doc/forum/Find_unlocked__47__locked_files.mdwn b/doc/forum/Find_unlocked__47__locked_files.mdwn
new file mode 100644
index 0000000000..922c8c5603
--- /dev/null
+++ b/doc/forum/Find_unlocked__47__locked_files.mdwn
@@ -0,0 +1,23 @@
+Hello, I would like to know if there is any way to specifically list the locked or unlocked annexed files in a git annex.
+I looked at the [[git-annex-lock]], [[git-annex-find]], and [[git-annex-matching-options]] pages and even the discussion about the [[tips/unlocked_files]] but I didn't find anything.
+
+I know it wouldn't make any sens for the older versions, but in the v6 mode, I think it might be useful to add such a shortcut search.
+I mean, we can already look for local content with:
+```git annex find --in=here```
+so why not create something like
+```git annex find --locked/unlocked=yes/no```
+?
+
+Sure it is already more or less doable by looking at symlinks:
+[[!format sh """
+#list all broken symlinks (locked absent files ?)
+find . -xtype l 
+#list all symlinks (locked present files ?)
+find -L . -xtype l
+#list all files that aren't symlinks (unlocked files ?)
+find . -type -f
+"""]]
+But it is also possible for any symlink or file not to be part of the annex.
+So, in order to find the locked/unlocked files, it would require to intersect the previous sets of files with the set of annexed ones.
+
+Am I missing any easy tip or command argument to do this ?
diff --git a/doc/forum/Find_unlocked__47__locked_files/comment_1_a34ce29b644c80789bf8cbfd575ffe1a._comment b/doc/forum/Find_unlocked__47__locked_files/comment_1_a34ce29b644c80789bf8cbfd575ffe1a._comment
new file mode 100644
index 0000000000..27bd6c89e5
--- /dev/null
+++ b/doc/forum/Find_unlocked__47__locked_files/comment_1_a34ce29b644c80789bf8cbfd575ffe1a._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="mario"
+ avatar="http://cdn.libravatar.org/avatar/4c63b0935789d29210d0bd8cad8d7ac7"
+ subject="comment 1"
+ date="2018-09-23T16:46:47Z"
+ content="""
+Usually I use \"git status\" to check if I have unlocked files. But, I'd say that's a workaround for the feature that you suggest.
+"""]]
diff --git a/doc/forum/First_attempt_at_an_OSX_launcher___40__.app__41__.mdwn b/doc/forum/First_attempt_at_an_OSX_launcher___40__.app__41__.mdwn
new file mode 100644
index 0000000000..ded094ec13
--- /dev/null
+++ b/doc/forum/First_attempt_at_an_OSX_launcher___40__.app__41__.mdwn
@@ -0,0 +1,3 @@
+I've made a first attempt at a .app launcher for git-annex webapp for OSX, please see  the icon needs to be resized and made to look nicer at somepoint, the launcher assumes that git-annex is in the run time path.
+
+I would imagine it might be possible to fiddle with the paths and get all the needed components into the .app file for OSX users.
diff --git a/doc/forum/First_attempt_at_an_OSX_launcher___40__.app__41__/comment_1_97c261b9080c5ecc5424683066bbe05b._comment b/doc/forum/First_attempt_at_an_OSX_launcher___40__.app__41__/comment_1_97c261b9080c5ecc5424683066bbe05b._comment
new file mode 100644
index 0000000000..aa2757054f
--- /dev/null
+++ b/doc/forum/First_attempt_at_an_OSX_launcher___40__.app__41__/comment_1_97c261b9080c5ecc5424683066bbe05b._comment
@@ -0,0 +1,14 @@
+[[!comment format=mdwn
+ username="http://joeyh.name/"
+ ip="4.153.14.141"
+ subject="comment 1"
+ date="2012-09-23T15:14:20Z"
+ content="""
+Aha! I had been fumbling trying to find the right way to do this. Looks very promising. Found the documentation for the file format: 
+
+Where do I install this thing? If I put the git-annex.app directory inside ~/Desktop/, will that put it on the user's desktop?
+
+Could you make a screenshot?
+
+I'm guessing that the NSAppleScriptEnabled is not needed, and perhaps the LSMinimumSystemVersionByArchitecture block could also be removed.
+"""]]
diff --git a/doc/forum/First_attempt_at_an_OSX_launcher___40__.app__41__/comment_2_ae45f9703b635c235409682cf252d36c._comment b/doc/forum/First_attempt_at_an_OSX_launcher___40__.app__41__/comment_2_ae45f9703b635c235409682cf252d36c._comment
new file mode 100644
index 0000000000..edb8e50c7a
--- /dev/null
+++ b/doc/forum/First_attempt_at_an_OSX_launcher___40__.app__41__/comment_2_ae45f9703b635c235409682cf252d36c._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="http://joeyh.name/"
+ ip="4.153.14.141"
+ subject="comment 2"
+ date="2012-09-23T15:17:42Z"
+ content="""
+Also, is there any global location I can install it when installing as root?
+"""]]
diff --git a/doc/forum/First_attempt_at_an_OSX_launcher___40__.app__41__/comment_3_066ca31a2e5dfe55a58092ba85231c7c._comment b/doc/forum/First_attempt_at_an_OSX_launcher___40__.app__41__/comment_3_066ca31a2e5dfe55a58092ba85231c7c._comment
new file mode 100644
index 0000000000..4a48117ee1
--- /dev/null
+++ b/doc/forum/First_attempt_at_an_OSX_launcher___40__.app__41__/comment_3_066ca31a2e5dfe55a58092ba85231c7c._comment
@@ -0,0 +1,10 @@
+[[!comment format=mdwn
+ username="https://www.google.com/accounts/o8/id?id=AItOawkSq2FDpK2n66QRUxtqqdbyDuwgbQmUWus"
+ nickname="Jimmy"
+ subject="comment 3"
+ date="2012-09-23T15:52:31Z"
+ content="""
+The .app file can be placed anywhere, typically you would want to bundle it in a .dmg file for the user and the user just \"drags\" it to the /Applications folder. It should be possible to bundle statically linked binaries inside the .app bundle, you might end up having to ship parts of coreutils and a few other applications to make it self contained.
+
+Anyhow, here is a quick screencast  and a screengrab of the app 
+"""]]
diff --git a/doc/forum/First_attempt_at_an_OSX_launcher___40__.app__41__/comment_4_a0a9f7f44cadb8036fcddfc21bb0781f._comment b/doc/forum/First_attempt_at_an_OSX_launcher___40__.app__41__/comment_4_a0a9f7f44cadb8036fcddfc21bb0781f._comment
new file mode 100644
index 0000000000..a18b341f71
--- /dev/null
+++ b/doc/forum/First_attempt_at_an_OSX_launcher___40__.app__41__/comment_4_a0a9f7f44cadb8036fcddfc21bb0781f._comment
@@ -0,0 +1,10 @@
+[[!comment format=mdwn
+ username="https://www.google.com/accounts/o8/id?id=AItOawkSq2FDpK2n66QRUxtqqdbyDuwgbQmUWus"
+ nickname="Jimmy"
+ subject="comment 4"
+ date="2012-09-23T16:00:35Z"
+ content="""
+Thinking this out a little, I reckon the script _MacOS/git-annex_ could be a bit more clever and get the basedir/basename of where it app is and execute git-annex from the .app bundle, then you wouldn't need to install it as root at all. It would require a bit of work to collect the executables needed.
+
+Also, I just quickly checked my Lion install, it seems to come with git and uuidgen once xcode and the command line tools are installed. 
+"""]]
diff --git a/doc/forum/First_attempt_at_an_OSX_launcher___40__.app__41__/comment_5_92240b3f8629f1f2bbe1829700082a79._comment b/doc/forum/First_attempt_at_an_OSX_launcher___40__.app__41__/comment_5_92240b3f8629f1f2bbe1829700082a79._comment
new file mode 100644
index 0000000000..e8a6d550f3
--- /dev/null
+++ b/doc/forum/First_attempt_at_an_OSX_launcher___40__.app__41__/comment_5_92240b3f8629f1f2bbe1829700082a79._comment
@@ -0,0 +1,10 @@
+[[!comment format=mdwn
+ username="http://joeyh.name/"
+ ip="4.153.14.141"
+ subject="comment 5"
+ date="2012-09-23T16:29:19Z"
+ content="""
+For now I have it generating the shell script, so it contains the path to whereever git-annex's binary is actually installed.
+
+For root, the app is installed in /Applications/ , for non-root installs I put it in ~/Desktop/
+"""]]
diff --git a/doc/forum/Fix_duplicate_UUID.mdwn b/doc/forum/Fix_duplicate_UUID.mdwn
new file mode 100644
index 0000000000..f1f7ee89c7
--- /dev/null
+++ b/doc/forum/Fix_duplicate_UUID.mdwn
@@ -0,0 +1,5 @@
+Hi,
+
+For some reason, two of my repositories share the same UUID. Honestly I don't remember how this happened, as I set up these remotes years ago and not used them for a long time; maybe I just accidentally made a copy instead of git clone… no idea. Anyway, I'd like to use these remotes now. What's the best way to correct this problem? Is just editing UUID manually enough?
+
+Thank you,
diff --git a/doc/forum/Fix_duplicate_UUID/comment_1_1186caf5aa6830c2412f8584a43d8d86._comment b/doc/forum/Fix_duplicate_UUID/comment_1_1186caf5aa6830c2412f8584a43d8d86._comment
new file mode 100644
index 0000000000..e224f1dcd7
--- /dev/null
+++ b/doc/forum/Fix_duplicate_UUID/comment_1_1186caf5aa6830c2412f8584a43d8d86._comment
@@ -0,0 +1,18 @@
+[[!comment format=mdwn
+ username="https://launchpad.net/~liori"
+ nickname="liori"
+ avatar="http://cdn.libravatar.org/avatar/e1d0fdc746b3d21bb147160d40815e37b257b9119774d21784939b2d3ba95a91"
+ subject="comment 1"
+ date="2017-07-16T13:08:41Z"
+ content="""
+So, the procedure that worked for me was:
+
+ 1. edit the annex.uuid configuration setting in one of the repositories that had a duplicate
+
+ 2. edit the remote.*.annex.uuid configuration setting in all repositories that had the repository edited in (1) as a remote
+
+ 3. `git annex fsck` in both repositories that had the duplicate uuid—this is because these repositories did not have correct information as to which files they contained, due to my previous syncing efforts
+
+ 4. `git annex sync` till convergence
+
+"""]]
diff --git a/doc/forum/Fix_duplicate_UUID/comment_2_b0f06b58ef63dbb770773f28dfa1a29d._comment b/doc/forum/Fix_duplicate_UUID/comment_2_b0f06b58ef63dbb770773f28dfa1a29d._comment
new file mode 100644
index 0000000000..19323dc144
--- /dev/null
+++ b/doc/forum/Fix_duplicate_UUID/comment_2_b0f06b58ef63dbb770773f28dfa1a29d._comment
@@ -0,0 +1,9 @@
+[[!comment format=mdwn
+ username="https://launchpad.net/~stephane-gourichon-lpad"
+ nickname="stephane-gourichon-lpad"
+ avatar="http://cdn.libravatar.org/avatar/02d4a0af59175f9123720b4481d55a769ba954e20f6dd9b2792217d9fa0c6089"
+ subject="comment 2"
+ date="2017-07-20T16:33:02Z"
+ content="""
+Thanks for sharing.  I stumbled upon a similar problem and (IIRC) ended up doing at least one new clone.  I don't know if your solution is fully clean but it seems to have lower cost.
+"""]]
diff --git a/doc/forum/Fix_duplicate_UUID/comment_3_525b2838985392eb65e705ad354d02c4._comment b/doc/forum/Fix_duplicate_UUID/comment_3_525b2838985392eb65e705ad354d02c4._comment
new file mode 100644
index 0000000000..bae8551a0d
--- /dev/null
+++ b/doc/forum/Fix_duplicate_UUID/comment_3_525b2838985392eb65e705ad354d02c4._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="https://launchpad.net/~liori"
+ avatar="http://cdn.libravatar.org/avatar/e1d0fdc746b3d21bb147160d40815e37b257b9119774d21784939b2d3ba95a91"
+ subject="comment 3"
+ date="2017-08-12T15:31:24Z"
+ content="""
+Indeed, I'm also not sure if this procedure fully solves the matter (some confirmation from developers would be nice!), but so far I haven't had any problems.
+"""]]
diff --git a/doc/forum/Fix_duplicate_UUID/comment_4_ce58bf264e63a07a214956436d96f1a5._comment b/doc/forum/Fix_duplicate_UUID/comment_4_ce58bf264e63a07a214956436d96f1a5._comment
new file mode 100644
index 0000000000..c0fb410aa5
--- /dev/null
+++ b/doc/forum/Fix_duplicate_UUID/comment_4_ce58bf264e63a07a214956436d96f1a5._comment
@@ -0,0 +1,7 @@
+[[!comment format=mdwn
+ username="joey"
+ subject="""comment 4"""
+ date="2017-10-02T16:47:31Z"
+ content="""
+The described procedure sounds ok to me.
+"""]]
diff --git a/doc/forum/Fix_wrong_rsync_with_git_annex_reinit__63__.mdwn b/doc/forum/Fix_wrong_rsync_with_git_annex_reinit__63__.mdwn
new file mode 100644
index 0000000000..ea4e30c9f3
--- /dev/null
+++ b/doc/forum/Fix_wrong_rsync_with_git_annex_reinit__63__.mdwn
@@ -0,0 +1,38 @@
+# Context
+
+* Had two repo A and B. 
+* A had a lot of content not yet tracked and thus not available on B.
+* Filesystem problem on A: (relatively small) part of big repo content is lost.
+* In emergency, used rsync to propagate yet untracked content from failing A to sane B.
+* But forgot `ignore .git`, so rsync also partly rsync'ed `.git/annex` .
+* Now sane repo B believes it is the source one (same UUID A) as shown by `git annex info --fast`. 
+
+Both A and B show:
+
+    UUID-of-A -- A [here]
+    UUID-of-B -- B
+
+# Actions done
+
+Search on Google, see [[https://git-annex.branchable.com/git-annex-reinit/]] .
+
+On B:
+
+    git annex reinit UUID-of-B
+    git annex fsck
+
+Now `git annex info -fast` on B shows as expected:
+
+    UUID-of-A -- A
+    UUID-of-B -- B [here]
+
+
+I'm currently doing `git annex copy --from B` to repropagate lost parts of A.
+
+# Question
+
+* Were actions done best thing to do?
+* Is there something more to do?
+* Something to check?
+
+Thanks.
diff --git a/doc/forum/Fix_wrong_rsync_with_git_annex_reinit__63__/comment_1_26cb6d46a408c1002aa2fbc6618242f3._comment b/doc/forum/Fix_wrong_rsync_with_git_annex_reinit__63__/comment_1_26cb6d46a408c1002aa2fbc6618242f3._comment
new file mode 100644
index 0000000000..33a480621d
--- /dev/null
+++ b/doc/forum/Fix_wrong_rsync_with_git_annex_reinit__63__/comment_1_26cb6d46a408c1002aa2fbc6618242f3._comment
@@ -0,0 +1,11 @@
+[[!comment format=mdwn
+ username="joey"
+ subject="""comment 1"""
+ date="2017-05-25T17:51:26Z"
+ content="""
+You must have rsynced over the .git/config from A. That is where the UUID
+of the repository is stored. So B now has whatever git config settings you
+had in A. The reinit fixed the UUID; there could be other things, like
+remotes, that are configured wrong. Otherwise, the way you recovered seems
+fine.
+"""]]
diff --git a/doc/forum/Fix_wrong_rsync_with_git_annex_reinit__63__/comment_2_14ddf5ddf2b85c9d5ed5ca070744f1fe._comment b/doc/forum/Fix_wrong_rsync_with_git_annex_reinit__63__/comment_2_14ddf5ddf2b85c9d5ed5ca070744f1fe._comment
new file mode 100644
index 0000000000..aee6435c42
--- /dev/null
+++ b/doc/forum/Fix_wrong_rsync_with_git_annex_reinit__63__/comment_2_14ddf5ddf2b85c9d5ed5ca070744f1fe._comment
@@ -0,0 +1,10 @@
+[[!comment format=mdwn
+ username="https://launchpad.net/~stephane-gourichon-lpad"
+ nickname="stephane-gourichon-lpad"
+ avatar="http://cdn.libravatar.org/avatar/02d4a0af59175f9123720b4481d55a769ba954e20f6dd9b2792217d9fa0c6089"
+ subject="comment 2"
+ date="2017-07-20T17:55:03Z"
+ content="""
+More specific information is in https://git-annex.branchable.com/forum/Fix_duplicate_UUID/
+
+"""]]
diff --git a/doc/forum/Fixing_up_corrupt_annexes.mdwn b/doc/forum/Fixing_up_corrupt_annexes.mdwn
new file mode 100644
index 0000000000..be6beeca8f
--- /dev/null
+++ b/doc/forum/Fixing_up_corrupt_annexes.mdwn
@@ -0,0 +1,10 @@
+I was wondering how does one recover from...
+
+
    +(Recording state in git...)
+error: invalid object 100644 8f154c946adc039af5240cc650a0a95c840e6fa6 for '041/5a4/SHA256-s6148--7ddcf853e4b16e77ab8c3c855c46867e6ed61c7089c334edf98bbdd3fb3a89ba.log'
+fatal: git-write-tree: error building trees
+git-annex: failed to read sha from git write-tree
+
    
+
+The above was caught when i ran a "git annex fsck --fast" to check stash of files"
diff --git a/doc/forum/Fixing_up_corrupt_annexes/comment_1_cea21f96bcfb56aaab7ea03c1c804d2d._comment b/doc/forum/Fixing_up_corrupt_annexes/comment_1_cea21f96bcfb56aaab7ea03c1c804d2d._comment
new file mode 100644
index 0000000000..335cbb51d2
--- /dev/null
+++ b/doc/forum/Fixing_up_corrupt_annexes/comment_1_cea21f96bcfb56aaab7ea03c1c804d2d._comment
@@ -0,0 +1,7 @@
+[[!comment format=mdwn
+ username="http://joeyh.name/"
+ subject="comment 1"
+ date="2012-07-24T22:00:35Z"
+ content="""
+This is a corrupt git repository. See [[tips/what_to_do_when_a_repository_is_corrupted]]
+"""]]
diff --git a/doc/forum/Fixing_up_corrupt_annexes/comment_2_5cdd2fcfa61b3f6255e5ad63a3ab00ce._comment b/doc/forum/Fixing_up_corrupt_annexes/comment_2_5cdd2fcfa61b3f6255e5ad63a3ab00ce._comment
new file mode 100644
index 0000000000..4692338af6
--- /dev/null
+++ b/doc/forum/Fixing_up_corrupt_annexes/comment_2_5cdd2fcfa61b3f6255e5ad63a3ab00ce._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="https://www.google.com/accounts/o8/id?id=AItOawkSq2FDpK2n66QRUxtqqdbyDuwgbQmUWus"
+ nickname="Jimmy"
+ subject="comment 2"
+ date="2012-07-25T06:52:32Z"
+ content="""
+Ah I was looking at the walkthrough on how to fix the issue, I had not thought about looking at the tips section. That tip fixed the issue for me, thanks.
+"""]]
diff --git a/doc/forum/Folders_for___34__actions__34___-_now_that_views_have_disrupted_the_file_structure__63__.mdwn b/doc/forum/Folders_for___34__actions__34___-_now_that_views_have_disrupted_the_file_structure__63__.mdwn
new file mode 100644
index 0000000000..61a8c7905d
--- /dev/null
+++ b/doc/forum/Folders_for___34__actions__34___-_now_that_views_have_disrupted_the_file_structure__63__.mdwn
@@ -0,0 +1,20 @@
+The _views_ functionality has made the file structure dynamic. 
+
+With this in mind would it be possible/suitable to make drag and drop targets for `git-annex get` and `git-annex drop` commands?
+This would make available git annex most interesting feature to assistant users.
+
+I'm not entirely sure it's a good idea but it avoids file manager scripts and should be discoverable and easy to use.
+
+I see two possibilities:
+
+1. A folder in the annex root that contains a mirror of the tree but with only the unavailable files. Unavaliable files would then not be displayed in the live tree. Perhaps the folder can be called `remote`? Files can then be dragged from this tree to the live tree to trigger transfer of files. Any error/info gets displayed in the assistant
+2. Flat droptargets in the annex root for `drop` and `get` commands.
+
+
+Number one above makes discovering available files a bit cumbersome as you have to navigate down a parallell tree. It does make the available/remote distinction clearer which may be good for assistant users.
+
+The mechanism could be extended to include a folder or droptarget for each remote.
+
+The creation of droptargets/action folders could be triggered by a big visible button in the assistant interface **Manage Files**
+
+Is it messy?
diff --git a/doc/forum/Forcing_offline_copies_seem_available.mdwn b/doc/forum/Forcing_offline_copies_seem_available.mdwn
new file mode 100644
index 0000000000..0789726971
--- /dev/null
+++ b/doc/forum/Forcing_offline_copies_seem_available.mdwn
@@ -0,0 +1,3 @@
+I have a large archive repository copied on an LTO tape, plus a proxy version of it with data dropped on local HDD. I've added this stripped proxy as a remote to a central repository to keep track of what is stored on the tape. Unfortunately, since the proxy has no contents, the central repo thinks there are no copies available anywhere, even though the offline tape contains all the data.
+
+How could I convince the central repo that the copies do in fact exist, even if they are very much offline?
diff --git a/doc/forum/Forcing_offline_copies_seem_available/comment_1_75e8597a24c925981edf83c517d5a799._comment b/doc/forum/Forcing_offline_copies_seem_available/comment_1_75e8597a24c925981edf83c517d5a799._comment
new file mode 100644
index 0000000000..dc37d67c96
--- /dev/null
+++ b/doc/forum/Forcing_offline_copies_seem_available/comment_1_75e8597a24c925981edf83c517d5a799._comment
@@ -0,0 +1,32 @@
+[[!comment format=mdwn
+ username="joey"
+ subject="""comment 1"""
+ date="2018-08-14T14:50:51Z"
+ content="""
+You can use `git annex setpresentkey` to tell git-annex that a remote with
+a given uuid contains a given content.
+
+For example, if the proxy remote is named proxy
+and you know it contains all annexed files in the current directory
+and below, you could run this to tell git-annex that the proxy contains
+all the files it thought it didn't contain:
+
+	uuid=$(git config remote.proxy.annex-uuid)
+	git annex find --not --in proxy --format "\${key} $uuid 1\n" | \
+		git annex setpresentkey --batch
+
+There will be some problems using this empty proxy remote, eg if you
+run `git annex move somefile --from proxy`, git-annex will try to
+delete the content from it, see the content is not there, and update its
+location tracking to say that the proxy does not contain the content any
+longer. `git annex fsck --from proxy` will do similar so you'll need to
+avoid it.
+
+And, you'll probably want to use `git annex trust proxy` so that `git-annex
+drop` assumes it contains the content you said it has; by default git-annex
+will double-check and that check will fail.
+
+To avoid all these kind of issues with the proxy, a better approach might
+be to make a custom special remote that actually accesses the data on the
+tape drive. See [[special remote implementation howto|special_remotes/external]]
+"""]]
diff --git a/doc/forum/Forcing_offline_copies_seem_available/comment_2_7d0e219628ad19093c6b6410d24c5f9a._comment b/doc/forum/Forcing_offline_copies_seem_available/comment_2_7d0e219628ad19093c6b6410d24c5f9a._comment
new file mode 100644
index 0000000000..a63102f62e
--- /dev/null
+++ b/doc/forum/Forcing_offline_copies_seem_available/comment_2_7d0e219628ad19093c6b6410d24c5f9a._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="jarno"
+ avatar="http://cdn.libravatar.org/avatar/da157278ab330a9c936dd733c43740f8"
+ subject="comment 2"
+ date="2018-08-14T16:03:39Z"
+ content="""
+Nice, thank you! I think I'll just delete or move the proxy away after feeding the (fake) availability info to the main repo. That should prevent git-annex from accidentally discovering that the data doesn't actually exist on the proxy, right?
+"""]]
diff --git a/doc/forum/Forcing_offline_copies_seem_available/comment_3_3d57e187679b36be62f3918eb996c3df._comment b/doc/forum/Forcing_offline_copies_seem_available/comment_3_3d57e187679b36be62f3918eb996c3df._comment
new file mode 100644
index 0000000000..9ede92c73e
--- /dev/null
+++ b/doc/forum/Forcing_offline_copies_seem_available/comment_3_3d57e187679b36be62f3918eb996c3df._comment
@@ -0,0 +1,10 @@
+[[!comment format=mdwn
+ username="joey"
+ subject="""comment 3"""
+ date="2018-08-14T17:40:10Z"
+ content="""
+Indeed, making the proxy remote not accessible will avoid such probles.
+
+You will certianly need to make git-annex trust the remote if you want it
+to count the tape as a copy of a file.
+"""]]
diff --git a/doc/forum/Forcing_one_repo_to_contain_a_copy_of_all_files.mdwn b/doc/forum/Forcing_one_repo_to_contain_a_copy_of_all_files.mdwn
new file mode 100644
index 0000000000..827383f9f4
--- /dev/null
+++ b/doc/forum/Forcing_one_repo_to_contain_a_copy_of_all_files.mdwn
@@ -0,0 +1 @@
+With numcopies it is possible to control how many copies there should be of a file overall in all of the remotes. Is it also possible, using .gitattributes or something else, to force one of the repos (e.g. a server) to contain a copy of all files, or does this have to be done manually?
diff --git a/doc/forum/Forcing_one_repo_to_contain_a_copy_of_all_files/comment_1_702b1b94c735f1b9cde16daa77a80c12._comment b/doc/forum/Forcing_one_repo_to_contain_a_copy_of_all_files/comment_1_702b1b94c735f1b9cde16daa77a80c12._comment
new file mode 100644
index 0000000000..d964a253bb
--- /dev/null
+++ b/doc/forum/Forcing_one_repo_to_contain_a_copy_of_all_files/comment_1_702b1b94c735f1b9cde16daa77a80c12._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="http://edheil.wordpress.com/"
+ ip="99.54.57.201"
+ subject="comment 1"
+ date="2013-09-14T20:36:53Z"
+ content="""
+That is what the \"backup\" preferred content group is for.  Depending on how your repos connect to each other it might take some more trickery to make all content migrate to it, but that will make it \"want\" all the files if it can get them.
+"""]]
diff --git a/doc/forum/Forcing_one_repo_to_contain_a_copy_of_all_files/comment_2_3df7fcbcd482bb9377ead238b314995b._comment b/doc/forum/Forcing_one_repo_to_contain_a_copy_of_all_files/comment_2_3df7fcbcd482bb9377ead238b314995b._comment
new file mode 100644
index 0000000000..e59af71fbd
--- /dev/null
+++ b/doc/forum/Forcing_one_repo_to_contain_a_copy_of_all_files/comment_2_3df7fcbcd482bb9377ead238b314995b._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="http://joeyh.name/"
+ ip="4.153.14.105"
+ subject="Ed's right"
+ date="2013-09-19T18:56:39Z"
+ content="""
+What we don't have is something to enforce a particular preferred content setting. Ie, refuse to allow dropping a file from a repo if it's preferred content there. Currently you have to remember to use `git annex drop --auto` to cause the preferred content to be honored.
+"""]]
diff --git a/doc/forum/Fsck_by_refs_or_keys__63__.mdwn b/doc/forum/Fsck_by_refs_or_keys__63__.mdwn
new file mode 100644
index 0000000000..ff2d02df2f
--- /dev/null
+++ b/doc/forum/Fsck_by_refs_or_keys__63__.mdwn
@@ -0,0 +1,5 @@
+Hello,
+
+I'm trying to wrote a cronjob to fsck my git-annex bare repository on a server. My problem is that there are a lot of keys with no known copies because they were file that weren't meant to stay. So I want to only check files that are live in HEAD of master. Is there anyway to tell `git-annex fsck` to only check file that are used in some refs?
+
+Thanks.
diff --git a/doc/forum/Fsck_by_refs_or_keys__63__/comment_1_345daa76cdd13538054e886a755bf29d._comment b/doc/forum/Fsck_by_refs_or_keys__63__/comment_1_345daa76cdd13538054e886a755bf29d._comment
new file mode 100644
index 0000000000..bd5f3f754c
--- /dev/null
+++ b/doc/forum/Fsck_by_refs_or_keys__63__/comment_1_345daa76cdd13538054e886a755bf29d._comment
@@ -0,0 +1,14 @@
+[[!comment format=mdwn
+ username="joey"
+ subject="""comment 1"""
+ date="2016-07-12T17:41:35Z"
+ content="""
+Currently the only way to do that is to make a clone of the bare repsitory
+that's not bare. Then run `git annex fsck --from origin` inside the clone.
+This is more efficient when the clone is on the same disk as the bare
+repository.
+
+Another way to deal with it is to use `git annex dead --key` to
+make fsk stop complaining about the specific keys that you don't mind are
+gone.
+"""]]
diff --git a/doc/forum/Fsck_by_refs_or_keys__63__/comment_2_1d74ad47cf3bef9bcc0ccdff4ccb69b9._comment b/doc/forum/Fsck_by_refs_or_keys__63__/comment_2_1d74ad47cf3bef9bcc0ccdff4ccb69b9._comment
new file mode 100644
index 0000000000..b5fba44147
--- /dev/null
+++ b/doc/forum/Fsck_by_refs_or_keys__63__/comment_2_1d74ad47cf3bef9bcc0ccdff4ccb69b9._comment
@@ -0,0 +1,7 @@
+[[!comment format=mdwn
+ username="Rémi"
+ subject="comment 2"
+ date="2016-07-12T20:28:44Z"
+ content="""
+Thanks, I opted for the clone solution, as there are fare too many lost key in this repository to manually suppress them all.
+"""]]
diff --git a/doc/forum/Fsck_by_refs_or_keys__63__/comment_3_b00a9522bb6dd91fed5d1c790153a230._comment b/doc/forum/Fsck_by_refs_or_keys__63__/comment_3_b00a9522bb6dd91fed5d1c790153a230._comment
new file mode 100644
index 0000000000..b2ee98f73f
--- /dev/null
+++ b/doc/forum/Fsck_by_refs_or_keys__63__/comment_3_b00a9522bb6dd91fed5d1c790153a230._comment
@@ -0,0 +1,9 @@
+[[!comment format=mdwn
+ username="joey"
+ subject="""comment 3"""
+ date="2016-07-19T17:01:03Z"
+ content="""
+It would be possible to add something like `git annex fsck --branch foo `
+which would operate on all files in the specified branch. I've opened
+[[todo/operate_on_branch_contents]] requesting that.
+"""]]
diff --git a/doc/forum/Full_restore_from_an_encrypted_special_remote.mdwn b/doc/forum/Full_restore_from_an_encrypted_special_remote.mdwn
new file mode 100644
index 0000000000..d724929b05
--- /dev/null
+++ b/doc/forum/Full_restore_from_an_encrypted_special_remote.mdwn
@@ -0,0 +1,3 @@
+If I lose all my git-annex repositories is it possible to restore the content of a repo from a special remote in which encryption was enabled? I know git-annex isn't designed to be a backup solution but more for the archive and/or nomad use case. I do like the idea of being able to restore the files from an S3 special remote though. I imagine part of the answer to this question has to do with what type of encryption is used (hybrid and shared key rely on information stored in the repo but pubkey doesn't rely on the repo at all) and would probably be easier if there were an option to disable hashing the filenames of the files (less secure, but maybe ok in some cases).
+
+Thoughts?
diff --git a/doc/forum/Full_restore_from_an_encrypted_special_remote/comment_1_f3a7a766ae1fe8e586cd67148761f156._comment b/doc/forum/Full_restore_from_an_encrypted_special_remote/comment_1_f3a7a766ae1fe8e586cd67148761f156._comment
new file mode 100644
index 0000000000..e56829ec5b
--- /dev/null
+++ b/doc/forum/Full_restore_from_an_encrypted_special_remote/comment_1_f3a7a766ae1fe8e586cd67148761f156._comment
@@ -0,0 +1,9 @@
+[[!comment format=mdwn
+ username="Marco"
+ subject="You need a copy of the git repo"
+ date="2015-07-29T12:20:06Z"
+ content="""
+It is not possible to recover just from a special remote. You need at least one copy of the git repository. This drive doesn't need to carry around the actual payload.
+Consider using an encrypted thumb drive as an additional destination. You can even fully encrypt this repo and sync it up once in a while to keep the information up to date.
+You can have a look at [[tips/fully_encrypted_git_repositories_with_gcrypt]] for detailed information.
+"""]]
diff --git a/doc/forum/Full_workflow_guide.mdwn b/doc/forum/Full_workflow_guide.mdwn
new file mode 100644
index 0000000000..a9b22e53a7
--- /dev/null
+++ b/doc/forum/Full_workflow_guide.mdwn
@@ -0,0 +1,16 @@
+I try to use git annex, but I frequently don't know if I'm doing things correctly, and my files are getting messed up. 
+
+An expert guide to the full workflow would go a long way toward user-friendliness for me. The walkthrough currently has guides to a number discrete items in the workflow, but it doesn't give me a clear sense of the process. 
+
+I'm always confused about when I'm supposed to be using pure git commands and when they should be git annex commands, when to commit, add, and sync --content, and when each of these is redundant.
+
+If possible, most helpful would be a guide to how you imagine the workflow from the beginning and including each step of the process, in the order you'd do it. 
+
+1. I want to start keeping track of some files I have in a directory
+2. I want to copy them to a second computer. 
+3. From a third place, I want to get them from the second computer. 
+4. I change the files on one computer, and I want to make sure the changes get synced to the others. 
+
+What are the commands you'd run at each step?
+
+Many thanks.
diff --git a/doc/forum/Full_workflow_guide/comment_1_14664efebb1218a1449577eaa4653a77._comment b/doc/forum/Full_workflow_guide/comment_1_14664efebb1218a1449577eaa4653a77._comment
new file mode 100644
index 0000000000..320dd6b453
--- /dev/null
+++ b/doc/forum/Full_workflow_guide/comment_1_14664efebb1218a1449577eaa4653a77._comment
@@ -0,0 +1,45 @@
+[[!comment format=mdwn
+ username="https://anarc.at/openid/"
+ nickname="anarcat"
+ avatar="http://cdn.libravatar.org/avatar/b36dcf65657dd36128161355d8920a99503def9461c1bb212410980fe6f07125"
+ subject="hopefully an answer..."
+ date="2017-01-17T16:54:22Z"
+ content="""
+looking at the page title here, i'm tempted to simply point you to [[todo/Workflow_guide]] and therefore the new [[workflow]] page, but that may not answer your question directly. so let's see...
+
+the the examples below, i'll assume you are familiar with the commandline and show examples using git and git-annex commands, since you mentioned that. but a lot of those things can be done with the web assistant, if not everything. but i am not very familiar with it, so you'll excuse my geekiness...
+
+1. I want to start keeping track of some files I have in a directory
+
+        cd directory
+        git init
+        git annex init
+        git annex add \"some\" \"files\"
+        git commit -m\"start keeping track of some files\"
+
+    Note that you could track all files with `git annex add .` or `git annex add *`... regular glob patterns apply.
+
+2. I want to copy them to a second computer.
+
+    This is the harder part. Here I'll assume that you have an SSH server on the first computer, which i'll name `firstcomputer`. If that's not your case, you can use the new [[special_remotes/tor]] to pair arbitrary machines with the [[tips/peer_to_peer_network_with_tor]] instructions instead of this.
+
+    On the second computer:
+
+        git clone firstcomputer:directory
+        cd directory
+        git annex get
+
+3. From a third place, I want to get them from the second computer.
+
+    Same comments as above apply.
+
+    On the third computer:
+
+        git clone secondcomputer:directory
+        cd directory
+        git annex get
+
+4. I change the files on one computer, and I want to make sure the changes get synced to the others.
+
+    This can be done manually with [[git-annex-sync]] (with the `--content` flag to copy actual content), or automated with the [[git-annex-assistant]] or [[git-annex-watch]].
+"""]]
diff --git a/doc/forum/Future_proofing___47___disaster_recovery_with_an_encrypted_special_remote.mdwn b/doc/forum/Future_proofing___47___disaster_recovery_with_an_encrypted_special_remote.mdwn
new file mode 100644
index 0000000000..2c229d211b
--- /dev/null
+++ b/doc/forum/Future_proofing___47___disaster_recovery_with_an_encrypted_special_remote.mdwn
@@ -0,0 +1,9 @@
+I am interested in using `git annex` to manage encrypted backups to Amazon S3/Glacier. So `git annex` will be used with the main file directory in direct mode and an encrypted S3 or Glacier remote set up in archive mode and then `git annex add .` and `git annex sync` will be run periodically. The intent is for this set up to be a backup for catastrophic failure, so I want to make sure I take care of future-proofing and disaster recovery properly. So my basic question is what would I need to have backed up and what would I have to do if the computer with the main repository died. I try to break that out into more specific questions below.
+
+0. S3/Glacier remotes store the contents of `.git/annex/objects` in encrypted form with hashes for file names and nothing else (other than a uuid). The hashes do not match the keys in the main repo. Are they the same keys encrypted? Is there a way to look up the S3 file name corresponding to a file in the repo?
+
+1. For `shared` encryption, I see the cipher text in `remote.log` in the `git-annex` branch. Assuming I didn't have access to `git annex`, what would I need to do to convert that cipher text into a form that I could use with `gpg` to decrypt files?
+   
+2. Same question but for `hybrid` encryption rather than `shared`. I assume the answer is similar but I need to decrypt the cipher first with my gpg key? How do I do that?
+
+3. Assuming I did have access to `git annex`, what would I need to create a new repo on a new computer with access to all of the files in the S3/Glacier bucket? I think I would need my Amazon credentials (possibly already embedded in the git repo), my gpg key if using hybrid or public key encryption, and the `.git` folder as it was the last time files were pushed to the S3/Glacier remote (which would have the necessary decryption information for shared encryption). Is that right? I guess mainly I am checking that the remote does not store any metadata about the repo, so for `git annex` to be able to pull files back out I would need a backup of the `.git` directory and that back up would need to be up to date (can't just copy remote.log and have `git annex` work out the rest from the remote's contents). So for a full backup, my script would need to `tar` the `.git` directory, encrypt it, and push it to S3/Glacier separately after `git annex` does a sync. Then I could recover everything as long as I had a secure backup of my Amazon credentials and my encryption key(s).
diff --git a/doc/forum/Future_proofing___47___disaster_recovery_with_an_encrypted_special_remote/comment_10_2eb2f05477a86f1352e734d163301ffa._comment b/doc/forum/Future_proofing___47___disaster_recovery_with_an_encrypted_special_remote/comment_10_2eb2f05477a86f1352e734d163301ffa._comment
new file mode 100644
index 0000000000..467ab44c3f
--- /dev/null
+++ b/doc/forum/Future_proofing___47___disaster_recovery_with_an_encrypted_special_remote/comment_10_2eb2f05477a86f1352e734d163301ffa._comment
@@ -0,0 +1,44 @@
+[[!comment format=mdwn
+ username="joey"
+ subject="""comment 10"""
+ date="2018-07-19T16:16:14Z"
+ content="""
+@oliv5 sharedpubkey's cipher has the same newline problem as pubkey does,
+as discussed above. Unlike pubkey, it has to be base64-decoded first,
+and then the extra newline appended to that.
+
+	                # Pull out MAC cipher from beginning of cipher
+	                if [ \"$encryption\" = \"hybrid\" ] ; then
+	                        cipher=\"$(echo -n \"$cipher\" | head  -c 256 )\"
+	                elif [ \"$encryption\" = \"shared\" ] ; then
+	                        cipher=\"$(echo -n \"$cipher\" | base64 -d | head  -c 256 )\"
+	                elif [ \"$encryption\" = \"pubkey\" ] ; then
+	                        cipher=\"$cipher
+	\"
+	                elif [ \"$encryption\" = \"sharedpubkey\" ] ; then
+	                        cipher=\"$(echo -n \"$cipher\" | base64 -d)\"
+				cipher=\"$cipher
+	\"
+			fi
+
+Note that base64 -d does emit the newline (verified with hexdump);
+again the shell is shooting you in the foot by eliminating it.
+
+BTW, a very simple code hack that makes it easy to dump out the cipher git-annex
+is using:
+
+	diff --git a/Remote/Helper/Encryptable.hs b/Remote/Helper/Encryptable.hs
+	index 97e55a415..c4d252912 100644
+	--- a/Remote/Helper/Encryptable.hs
+	+++ b/Remote/Helper/Encryptable.hs
+	@@ -192,7 +192,7 @@ describeCipher :: StorableCipher -> String
+	 describeCipher c = case c of
+ 		(SharedCipher _) -> \"encryption key stored in git repository\"
+	 	(EncryptedCipher _ _ ks) -> showkeys ks
+	-	(SharedPubKeyCipher _ ks) -> showkeys ks
+	+	(SharedPubKeyCipher c ks) -> show c ++ \" \" ++ showkeys ks
+	   where
+	 	showkeys (KeyIds { keyIds = ks }) = \"to gpg keys: \" ++ unwords ks
+
+Then git-annex info remote will display it. Obviously, this patch is insecure.
+"""]]
diff --git a/doc/forum/Future_proofing___47___disaster_recovery_with_an_encrypted_special_remote/comment_1_c2f7542ac9a4c423c5ae06c16d42a24f._comment b/doc/forum/Future_proofing___47___disaster_recovery_with_an_encrypted_special_remote/comment_1_c2f7542ac9a4c423c5ae06c16d42a24f._comment
new file mode 100644
index 0000000000..b8c9d33c2a
--- /dev/null
+++ b/doc/forum/Future_proofing___47___disaster_recovery_with_an_encrypted_special_remote/comment_1_c2f7542ac9a4c423c5ae06c16d42a24f._comment
@@ -0,0 +1,17 @@
+[[!comment format=mdwn
+ username="joey"
+ subject="""comment 1"""
+ date="2015-12-11T14:42:32Z"
+ content="""
+That's what's "special" about special remotes vs regular git remotes: They
+only store the content of annexed files and not the git repository. Back up
+the git repository separately (and your gpg key if it's being used, and the
+credentials if you didn't use embedcreds=yes)
+
+To use your backup, you can make a clone of the backed up git repository and
+use `git annex enableremote` to enable it to use the special remote.
+
+See [[design/encryption]] for details of how the encryption is implemented.
+I've seen people follow that and manually use the data from the git repo to
+decrypt files, but I don't have a pointer to an example at the moment.
+"""]]
diff --git a/doc/forum/Future_proofing___47___disaster_recovery_with_an_encrypted_special_remote/comment_2_9a9da0ad1538ba59e76a5c4f9fc7fcf7._comment b/doc/forum/Future_proofing___47___disaster_recovery_with_an_encrypted_special_remote/comment_2_9a9da0ad1538ba59e76a5c4f9fc7fcf7._comment
new file mode 100644
index 0000000000..ff35541ec1
--- /dev/null
+++ b/doc/forum/Future_proofing___47___disaster_recovery_with_an_encrypted_special_remote/comment_2_9a9da0ad1538ba59e76a5c4f9fc7fcf7._comment
@@ -0,0 +1,18 @@
+[[!comment format=mdwn
+ username="wsha.code+ga@b38779424f41c5701bbe5937340be43ff1474b2d"
+ subject="comment 2"
+ date="2015-12-12T10:42:46Z"
+ content="""
+Thanks for the response.
+
+With `pubkey` encryption, I am able to decrypt the remote's files normally using just `gpg --decrypt` with the public key used to encrypt them in my keyring. One thing I don't understand about `pubkey` encryption: what is in the `cipher=` entry in `remote.log` after the 256 bytes representing the HMAC cipher? The `gpg` key pair is used for encryption, so there is no encryption cipher to put in `remote.log` after the HMAC cipher, I would have thought.
+
+I understand the basic encryption set up, but I don't know how to use `gpg` to work with a raw cipher or raw encrypted text. For example with `pubkey` encryption, my interpretation is that the `cipher=` entry in `remote.log` is encrypted with the public key, but I can't just pass teh entry to `gpg --decrypt` because `gpg` expects the encrypted input to be formatted in a way that specifies which key to use to decrypt. Otherwise it says `gpg: no valid OpenPGP data found.` Similarly for `shared` encryption, I don't see how to pass the second half of the `remote.log` `cipher=` entry to `gpg` to decrypt the remote's files.
+
+I am also having trouble generating the special remote's keys. I created a `directory` remote with `shared` encryption (so that the `cipher=` entry in `remote.log` would not be encrypted). Then I added one file called `tmp.txt` which was stored in the annex as `SHA256E-s9--9c73fdec185d79405f58fc8b4e0ac22fa5ed2de7b7611a61b37606c905509650.txt` I did `git checkout git-annex -- remote.log` and then tried the following:
+
+        echo -n \"SHA256E-s9--9c73fdec185d79405f58fc8b4e0ac22fa5ed2de7b7611a61b37606c905509650.txt\" | openssl dgst -sha1 -hmac $(echo -n $(grep -oP 'cipher\=.*? ' remote.log | sed 's/cipher=//') | base64 -d | xxd -p -l 256 -c 256) -macopt hexkey:string`
+
+but the output does not match the GPGHMACSHA1 file name in the remote, and I don't understand why. I tried other variations as well (dropping the `.txt` or the SHA256E-s9--` prefix) but they did not work either.
+
+"""]]
diff --git a/doc/forum/Future_proofing___47___disaster_recovery_with_an_encrypted_special_remote/comment_3_27901f1c2c5de211f697972d9b21c104._comment b/doc/forum/Future_proofing___47___disaster_recovery_with_an_encrypted_special_remote/comment_3_27901f1c2c5de211f697972d9b21c104._comment
new file mode 100644
index 0000000000..fe0187145e
--- /dev/null
+++ b/doc/forum/Future_proofing___47___disaster_recovery_with_an_encrypted_special_remote/comment_3_27901f1c2c5de211f697972d9b21c104._comment
@@ -0,0 +1,11 @@
+[[!comment format=mdwn
+ username="wsha.code+ga@b38779424f41c5701bbe5937340be43ff1474b2d"
+ subject="comment 3"
+ date="2015-12-14T12:03:22Z"
+ content="""
+Small update: I have gotten the HMAC and decryption steps to work for shared encryption. I didn't figure out what was wrong with my HMAC command above yet, but I was able to reproduce the special remote's keys in Python using the `hmac` and `hashlib` libraries. I was trying to hash the correct string in the comment above (the full key with the `SHA...` prefix and the file extension suffix). For decryption, I used `gpg` on the encrypted file in the special remote and passed it the cipher from `remote.log` with the first 256 bytes removed as the passphrase (in the format returned by `base64.b64decode()` in Python).
+
+I still need to figure out how to decrypt the ciphers for `pubkey` and `hybrid`.
+
+I will try to put together a tip with the steps needed to reproduce special remote keys and to decrypt special remote files using only command line tools after that (assuming I can translate the Python steps back to command line tools; otherwise part of the steps will be in Python).
+"""]]
diff --git a/doc/forum/Future_proofing___47___disaster_recovery_with_an_encrypted_special_remote/comment_4_56994d9252e8651b3f33044f5c8599b3._comment b/doc/forum/Future_proofing___47___disaster_recovery_with_an_encrypted_special_remote/comment_4_56994d9252e8651b3f33044f5c8599b3._comment
new file mode 100644
index 0000000000..fbfa254ffa
--- /dev/null
+++ b/doc/forum/Future_proofing___47___disaster_recovery_with_an_encrypted_special_remote/comment_4_56994d9252e8651b3f33044f5c8599b3._comment
@@ -0,0 +1,7 @@
+[[!comment format=mdwn
+ username="wsha.code+ga@b38779424f41c5701bbe5937340be43ff1474b2d"
+ subject="comment 4"
+ date="2015-12-16T11:21:48Z"
+ content="""
+Decrypting the cipher for `hybrid` and `pubkey` remotes is actually pretty straightforward: `echo -n  | base64 -d | gpg --decrypt`. I think with that I have enough to put together a short summary of decrypting by hand the contents of `hybrid`, `pubkey`, and `shared` special remotes.
+"""]]
diff --git a/doc/forum/Future_proofing___47___disaster_recovery_with_an_encrypted_special_remote/comment_5_5aecb8c58d501e929e746d5536032c81._comment b/doc/forum/Future_proofing___47___disaster_recovery_with_an_encrypted_special_remote/comment_5_5aecb8c58d501e929e746d5536032c81._comment
new file mode 100644
index 0000000000..8c3478d978
--- /dev/null
+++ b/doc/forum/Future_proofing___47___disaster_recovery_with_an_encrypted_special_remote/comment_5_5aecb8c58d501e929e746d5536032c81._comment
@@ -0,0 +1,13 @@
+[[!comment format=mdwn
+ username="joey"
+ subject="""comment 5"""
+ date="2015-12-19T17:59:38Z"
+ content="""
+When using `pubkey`, the second 256 bytes of ciphertext are currently
+not used for anything.
+
+For `hybrid` and `shared`, the 256 bytes of ciphertext are used
+as a symmetric cipher. So the gpg option to use for both encrypting
+and decrypting is --symmetric. gpg then prompts for a passphrase,
+and the ciphertext is what's used.
+"""]]
diff --git a/doc/forum/Future_proofing___47___disaster_recovery_with_an_encrypted_special_remote/comment_6_ab0b660e5a3d7fc7d3d3a4b460022da5._comment b/doc/forum/Future_proofing___47___disaster_recovery_with_an_encrypted_special_remote/comment_6_ab0b660e5a3d7fc7d3d3a4b460022da5._comment
new file mode 100644
index 0000000000..eda99e459c
--- /dev/null
+++ b/doc/forum/Future_proofing___47___disaster_recovery_with_an_encrypted_special_remote/comment_6_ab0b660e5a3d7fc7d3d3a4b460022da5._comment
@@ -0,0 +1,139 @@
+[[!comment format=mdwn
+ username="wsha.code+ga@b38779424f41c5701bbe5937340be43ff1474b2d"
+ subject="comment 6"
+ date="2015-12-26T12:31:13Z"
+ content="""
+joey, thanks for answering my questions. I have put together a bash script to check my understanding of how the different encryption methods work. The script is below. With it, I am able to generate the encrypted keys for items for `hybrid` and `shared` encrypted `directory` remotes and to decrypt items for `hybrid`, `shared`, and `pubkey` encrypted `directory` remotes (and presumably other special remotes but I tested with `directory` remotes since those were simplest). The only thing I have not been able to get to work is the generation of the encrypted item keys for `pubkey` remotes. Do you see what I am doing wrong for that case? Once I figure that out, I can make a tip entry on the wiki with the script.
+
+	#!/usr/bin/env bash
+
+	# args: file_path remote decrypt decrypt_path
+
+	# For shared, pubkey, and hybrid encryption:
+	# 1. Take in symlink and output encrypted key
+	# 2. Take in path to encrypted file and print decrypted file to stdout
+	#
+	# args: -r remote [-k symlink] [-d encrypted_file_path]
+
+	usage() {
+		echo \"Usage: ga_decrypt.sh -r REMOTE [-k SYMLINK] [-d FILE]\"
+		echo \"\"
+		echo \"    Either lookups up key on REMOTE for annex file linked with SYMLINK\"
+		echo \"    or decrypts FILE encrypted for REMOTE.\"
+		echo \"\"
+		echo \"    -r: REMOTE is special remote to use\"
+		echo \"    -k: SYMLINK is symlink in annex to print encrypted special remote key for\"
+		echo \"    -d: FILE is path to special remote file to decrypt to STDOUT\"
+		echo \"\"
+		echo \"NOTES: \"
+		echo \"    * Run in an indirect git annex repo.\"
+		echo \"    * Must specify -k or -d.\"
+		echo \"    * -k prints the key including the leading directory names used for a \"
+		echo \"       directory remote (even if REMOTE is not a directory remote)\"
+		echo \"    * -d works on a locally accessible file. It does not fetch a remote file\"
+		echo \"    * Must have gpg and openssl\"
+	}
+
+	decrypt_cipher() {
+		cipher=\"$1\"
+		echo \"$(echo -n \"$cipher\" | base64 -d | gpg --decrypt --quiet)\"
+	}
+
+	lookup_key() {
+		encryption=\"$1\"
+		cipher=\"$2\"
+		symlink=\"$3\"
+
+		if [ \"$encryption\" == \"hybrid\" ] || [ \"$encryption\" == \"pubkey\" ]; then
+			cipher=\"$(decrypt_cipher \"$cipher\")\"
+		fi
+
+		# Pull out MAC cipher from beginning of cipher
+		if [ \"$encryption\" = \"hybrid\" ] ; then
+			cipher=\"$(echo -n \"$cipher\" | head  -c 256 )\"
+		elif [ \"$encryption\" = \"shared\" ] ; then
+			cipher=\"$(echo -n \"$cipher\" | base64 -d | tr -d '\n' | head  -c 256 )\"
+		elif [ \"$encryption\" = \"pubkey\" ] ; then
+			# ???
+			: # Use entire base64 decrypted cipher
+			# cipher=\"$(echo -n \"$cipher\" | head  -c 256 )\"
+			# cipher=\"$(echo -n \"$cipher\" | base64 -d | tr -d '\n' | head  -c 256 )\"
+			# cipher=\"$(echo -n \"$cipher\" | base64 -d | tr -d '\n' )\"
+		fi
+
+		annex_key=\"$(basename \"$(readlink \"$symlink\")\")\"
+		hash=\"$(echo -n \"$annex_key\" | openssl dgst -sha1 -hmac \"$cipher\" | sed 's/(stdin)= //')\"
+		key=\"GPGHMACSHA1--$hash\"
+		checksum=\"$(echo -n $key | md5sum)\"
+		echo \"${checksum:0:3}/${checksum:3:3}/$key\"
+	}
+
+	decrypt_file() {
+		encryption=\"$1\"
+		cipher=\"$2\"
+		file_path=\"$3\"
+
+		if [ \"$encryption\" = \"pubkey\" ] ; then
+			gpg --quiet --decrypt \"${file_path}\"
+		else
+			if [ \"$encryption\" = \"hybrid\" ] ; then
+				cipher=\"$(decrypt_cipher \"$cipher\" | tail -c +257)\"
+			elif [ \"$encryption\" = \"shared\" ] ; then
+				cipher=\"$(echo -n \"$cipher\" | base64 -d | tr -d '\n' | tail  -c +257 )\"
+			fi
+			gpg --quiet --batch --passphrase \"$cipher\" --output - \"${file_path}\"
+		fi
+	}
+
+	main() {
+		OPTIND=1
+
+		mode=\"\"
+		remote=\"\"
+
+		while getopts \"r:k:d:\" opt; do
+			case \"$opt\" in
+				r)  remote=\"$OPTARG\"
+					;;
+				k)  if [ -z \"$mode\" ] ; then
+						mode=\"lookup key\"
+					else
+						usage
+						exit 2
+					fi
+					symlink=\"$OPTARG\"
+					;;
+				d)  if [ -z \"$mode\" ] ; then
+						mode=\"decrypt file\"
+					else
+						usage
+						exit 2
+					fi
+					file_path=\"$OPTARG\"
+					;;
+			esac
+		done
+
+		if [ -z \"$mode\" ] || [ -z \"$remote\" ] ; then
+			usage
+			exit 2
+		fi
+
+		shift $((OPTIND-1))
+
+		# Pull out config for desired remote name
+		remote_config=\"$(git show git-annex:remote.log | grep 'name='\"$remote \")\"
+
+		# Get encryption type and cipher from config
+		encryption=\"$(echo \"$remote_config\" | grep -oP 'encryption\=.*? ' | tr -d ' \n' | sed 's/encryption=//')\"
+		cipher=\"$(echo \"$remote_config\" | grep -oP 'cipher\=.*? ' | tr -d ' \n' | sed 's/cipher=//')\"
+
+		if [ \"$mode\" = \"lookup key\" ] ; then
+			lookup_key \"$encryption\" \"$cipher\" \"$symlink\"
+		elif [ \"$mode\" = \"decrypt file\" ] ; then
+			decrypt_file \"$encryption\" \"$cipher\" \"${file_path}\"
+		fi
+	}
+
+	main \"$@\"
+"""]]
diff --git a/doc/forum/Future_proofing___47___disaster_recovery_with_an_encrypted_special_remote/comment_7_b96077c98792997cdf344725374abb5c._comment b/doc/forum/Future_proofing___47___disaster_recovery_with_an_encrypted_special_remote/comment_7_b96077c98792997cdf344725374abb5c._comment
new file mode 100644
index 0000000000..d43e118880
--- /dev/null
+++ b/doc/forum/Future_proofing___47___disaster_recovery_with_an_encrypted_special_remote/comment_7_b96077c98792997cdf344725374abb5c._comment
@@ -0,0 +1,27 @@
+[[!comment format=mdwn
+ username="joey"
+ subject="""comment 7"""
+ date="2016-01-01T20:42:34Z"
+ content="""
+I think it's great you're working on this, and [[future_proofing]] is
+probably a good place to put the result.
+
+I did some digging, and the cipher used for pubkey includes a trailing
+new line. The trailing newline seems to be lost in your script, and
+adding it makes it work.
+
+	cipher="$cipher
+	"
+
+I think the actual problem in the script is in this line:
+
+	cipher="$(decrypt_cipher "$cipher")"
+
+I verified that decrypt_cipher outputs the trailing newline, but
+in capturing its output, the shell chomps it. Can't remember ATM if there's
+a way to avoid that in shell scripting.
+
+Arguably git-annex should not be treating that newline as part of the
+cipher text, although it's probably too late to change that, certainly for
+existing repositories..
+"""]]
diff --git a/doc/forum/Future_proofing___47___disaster_recovery_with_an_encrypted_special_remote/comment_8_e9597062bbc530920343e5159a7b7ea1._comment b/doc/forum/Future_proofing___47___disaster_recovery_with_an_encrypted_special_remote/comment_8_e9597062bbc530920343e5159a7b7ea1._comment
new file mode 100644
index 0000000000..921d75e1c8
--- /dev/null
+++ b/doc/forum/Future_proofing___47___disaster_recovery_with_an_encrypted_special_remote/comment_8_e9597062bbc530920343e5159a7b7ea1._comment
@@ -0,0 +1,10 @@
+[[!comment format=mdwn
+ username="wsha.code+ga@b38779424f41c5701bbe5937340be43ff1474b2d"
+ nickname="wsha.code+ga"
+ subject="comment 8"
+ date="2016-01-08T06:56:02Z"
+ content="""
+Thanks for spending the time to check the script. It would have taken me quite a while to catch that issue. I was also a little bit surprised that the hybrid encryption mode uses the base64 encoded cipher string while the shared encryption mode uses the cipher after decoding from base64.
+
+I can now reproduce the HMAC'ed keys and decrypt the files all the supported forms of encryption using openssl and gpg which makes me feel comfortable using them for long term storage.
+"""]]
diff --git a/doc/forum/Future_proofing___47___disaster_recovery_with_an_encrypted_special_remote/comment_9_756aeeee7be8cb67c5cc9dfa6ea7b82c._comment b/doc/forum/Future_proofing___47___disaster_recovery_with_an_encrypted_special_remote/comment_9_756aeeee7be8cb67c5cc9dfa6ea7b82c._comment
new file mode 100644
index 0000000000..b58d3ce94b
--- /dev/null
+++ b/doc/forum/Future_proofing___47___disaster_recovery_with_an_encrypted_special_remote/comment_9_756aeeee7be8cb67c5cc9dfa6ea7b82c._comment
@@ -0,0 +1,25 @@
+[[!comment format=mdwn
+ username="oliv5"
+ avatar="http://cdn.libravatar.org/avatar/d7f0d33c51583bbd8578e4f1f9f8cf4b"
+ subject="comment 9"
+ date="2018-07-15T23:59:42Z"
+ content="""
+I'm trying to add support for the sharedpubkey encryption scheme, and while decrypting the files is straight-forward, I cannot get the \"lookup_key\" function to work with this scheme. I tried the following:
+
+    # Pull out MAC cipher from beginning of cipher
+    if [ \"$encryption\" = \"hybrid\" ] ; then
+        cipher=\"$(echo -n \"$cipher\" | head  -c 256 )\"
+    elif [ \"$encryption\" = \"shared\" ] ; then
+        cipher=\"$(echo -n \"$cipher\" | base64 -d | tr -d '\n' | head  -c 256 )\"
+    elif [ \"$encryption\" = \"pubkey\" ] ; then
+        # pubkey cipher includes a trailing newline which was stripped in
+        # decrypt_cipher process substitution step above
+        IFS= read -rd '' cipher < <( printf \"$cipher\n\" )
+    elif [ \"$encryption\" = \"sharedpubkey\" ] ; then
+        cipher=\"$(echo -n \"$cipher\" | base64 -d | tr -d '\n' | head -c 256)\"
+    fi
+
+as well as many other variations (full cipher, no base 64 etc...) without success. The cipher is said to be unencrypted, so I guess it is base-64 encoded only. Would it be possible that an extra character has been added to the cipher like in the \"pubkey\" encryption scheme ?
+
+My tests are done in a directory special remote, sharedpubkey encryption, no chunks, a single test file in the repository.
+"""]]
diff --git a/doc/forum/GIT__95__SSH.mdwn b/doc/forum/GIT__95__SSH.mdwn
new file mode 100644
index 0000000000..968385d084
--- /dev/null
+++ b/doc/forum/GIT__95__SSH.mdwn
@@ -0,0 +1,3 @@
+Is there any way to get git-annex to respect the GIT_SSH environment variable just like git?  This would be super helpful for specifying ssh options (keys in my application)
+
+Hacks like a ~/.ssh/config with a fake hostname to specify the key are not appropriate in my case, because keys are generated on the fly by a server-side application that may be distributed across multiple servers.  So I really need to be able to specify a key file at run time and then use the normal remote URI.  I can do this with git and GIT_SSH, but have not found a solution for git-annex yet.
diff --git a/doc/forum/GIT__95__SSH/comment_1_c2e827eeac7524845cad42671d77af95._comment b/doc/forum/GIT__95__SSH/comment_1_c2e827eeac7524845cad42671d77af95._comment
new file mode 100644
index 0000000000..baced499e7
--- /dev/null
+++ b/doc/forum/GIT__95__SSH/comment_1_c2e827eeac7524845cad42671d77af95._comment
@@ -0,0 +1,14 @@
+[[!comment format=mdwn
+ username="joey"
+ subject="""comment 1"""
+ date="2017-03-17T17:26:56Z"
+ content="""
+I've added support for this now.
+
+However, since git's interface to this doesn't let any options other than
+-p be passed to the command run by this, git-annex can't either. Which
+means it can't pass options for ssh connection caching. So, I'm afraid that
+using `GIT_SSH` will slow things down somewhat. Of course, you can always
+enable ssh connection caching yourself in either the `GIT_SSH` script or
+the ssh configuration.
+"""]]
diff --git a/doc/forum/GPG_passphrase_handling.mdwn b/doc/forum/GPG_passphrase_handling.mdwn
new file mode 100644
index 0000000000..b0f82cb890
--- /dev/null
+++ b/doc/forum/GPG_passphrase_handling.mdwn
@@ -0,0 +1,76 @@
+[[!meta title="GPG passphrase handling on OSX"]]
+
+Hello!
+I'm using OSX 10.9 and have installed gpg (and gpg2, if it matters) through
+homebrew and git-annex through cabal. I also installed
+https://github.com/joeyh/git-remote-gcrypt like the UI told me.
+
+Whenever I'm trying to add an encrypted remote through the web UI I get a
+lot of "You need a passphrase to unlock the secret key for user:" on stdout
+and, obviously, I can't enter my passphrase (If I could I wouldn't make this
+post to begin with :))
+Is this behavior normal? What should I do to work around it?
+I did also try to not use the web UI by using this command:
+git annex initremote rsync.net type=gcrypt gitrepo=user@host:directory encryption=pubkey keyid=X
+
+Because of this I can't copy files to my remotes. All I get is:
+-----
+$ git annex copy --to rsync.net
+copy MySecretFile (gpg) 
+You need a passphrase to unlock the secret key for
+user: "user"
+4096-bit RSA key, ID X, created 2013-10-01 (main key ID Y)
+
+(checking rsync.net...) (to rsync.net...) gpg: no valid addressees
+gpg: [stdin]: encryption failed: No user ID
+failed
+-----
+
+Yes, I am using gpg-agent. When other applications ask for my passphrase I get
+the pinentry dialog from GPGTools, just like I've configured it in
+~/.gnupg/gpg-agent.conf, but this isn't the case with git-annex.
+
+If I remove GPGTools from /usr/local/bin with: ``brew link --overwrite gnupg &&
+brew link --overwrite gnupg2'' it works *slightly* better. 
+I get that pinentry dialog I want but when I do a copy I get:
+-----
+$ git annex copy --to rsync.net
+copy MySecretFile (gpg) (checking rsync.net...) (to rsync.net...) gpg: no valid addressees
+gpg: [stdin]: encryption failed: no such user id
+failed
+-----
+
+--debug shows me it is executing gpg llke so:
+-----
+gpg ["--batch","--no-tty","--use-agent","--quiet","--trust-model","always","--batch","--encrypt","--no-encrypt-to","--no-default-recipient","--force-mdc","--no-textmode"]
+-----
+
+$ git annex version
+git-annex version: 4.20131024
+build flags: Assistant Webapp Pairing Testsuite S3 WebDAV FsEvents XMPP DNS Feeds Quvi TDFA CryptoHash
+key/value backends: SHA256E SHA1E SHA512E SHA224E SHA384E SKEIN256E SKEIN512E SHA256 SHA1 SHA512 SHA224 SHA384 SKEIN256 SKEIN512 WORM URL
+remote types: git gcrypt S3 bup directory rsync web webdav glacier hook
+
+$ gpg --version
+gpg (GnuPG) 2.0.22
+libgcrypt 1.5.3
+Copyright (C) 2013 Free Software Foundation, Inc.
+License GPLv3+: GNU GPL version 3 or later 
+This is free software: you are free to change and redistribute it.
+There is NO WARRANTY, to the extent permitted by law.
+
+Home: ~/.gnupg
+Supported algorithms:
+Pubkey: RSA, ELG, DSA, ?, ?
+Cipher: IDEA, 3DES, CAST5, BLOWFISH, AES, AES192, AES256, TWOFISH,
+        CAMELLIA128, CAMELLIA192, CAMELLIA256
+Hash: MD5, SHA1, RIPEMD160, SHA256, SHA384, SHA512, SHA224
+Compression: Uncompressed, ZIP, ZLIB, BZIP2
+
+ $ gpg-agent --version
+gpg-agent (GnuPG/MacGPG2) 2.0.22
+libgcrypt 1.5.3
+Copyright (C) 2013 Free Software Foundation, Inc.
+License GPLv3+: GNU GPL version 3 or later 
+This is free software: you are free to change and redistribute it.
+There is NO WARRANTY, to the extent permitted by law.
diff --git a/doc/forum/GPG_passphrase_handling/comment_1_11ba130e8bea6698858d0a1a5b01830f._comment b/doc/forum/GPG_passphrase_handling/comment_1_11ba130e8bea6698858d0a1a5b01830f._comment
new file mode 100644
index 0000000000..f2709f18cc
--- /dev/null
+++ b/doc/forum/GPG_passphrase_handling/comment_1_11ba130e8bea6698858d0a1a5b01830f._comment
@@ -0,0 +1,15 @@
+[[!comment format=mdwn
+ username="https://www.google.com/accounts/o8/id?id=AItOawkzwmw_zyMpZC9_J7ey--woeYPoZkAOgGw"
+ nickname="dxtrish"
+ subject="comment 1"
+ date="2013-10-31T00:06:22Z"
+ content="""
+In fact I just tried this with a key without a passphrase and I STILL get:
+
+[2013-10-31 01:00:50 CET] Pusher: Syncing with rsync.net 
+gpg: no valid addressees
+gpg: [stdin]: encryption failed: no such user id
+
+In addition I also noticed a lot of:
+git-annex: fd:28: hPutBuf: resource vanished (Broken pipe)
+"""]]
diff --git a/doc/forum/GPG_passphrase_handling/comment_2_ef9d58d15b7bbe0b3c7140bb01d73a31._comment b/doc/forum/GPG_passphrase_handling/comment_2_ef9d58d15b7bbe0b3c7140bb01d73a31._comment
new file mode 100644
index 0000000000..d15248ab7a
--- /dev/null
+++ b/doc/forum/GPG_passphrase_handling/comment_2_ef9d58d15b7bbe0b3c7140bb01d73a31._comment
@@ -0,0 +1,15 @@
+[[!comment format=mdwn
+ username="https://www.google.com/accounts/o8/id?id=AItOawkzwmw_zyMpZC9_J7ey--woeYPoZkAOgGw"
+ nickname="dxtrish"
+ subject="comment 2"
+ date="2013-10-31T02:42:47Z"
+ content="""
+I think I have found two ways I can make this happen.
+
+1) If a filename contains UTF-8 it seems to happen (At least with this specific filename I have right here)
+
+2) Finder.app does not seem to play nice with git-annex (I got the same error when I tried using Finder to copy files to my git-annex thing)
+
+
+
+"""]]
diff --git a/doc/forum/GPG_passphrase_handling/comment_3_84eb129c8483b87b3ae6ecaf8b4a8309._comment b/doc/forum/GPG_passphrase_handling/comment_3_84eb129c8483b87b3ae6ecaf8b4a8309._comment
new file mode 100644
index 0000000000..298e2f13d2
--- /dev/null
+++ b/doc/forum/GPG_passphrase_handling/comment_3_84eb129c8483b87b3ae6ecaf8b4a8309._comment
@@ -0,0 +1,12 @@
+[[!comment format=mdwn
+ username="http://joeyh.name/"
+ ip="209.250.56.47"
+ subject="comment 3"
+ date="2013-11-02T20:39:36Z"
+ content="""
+There seem to be a lot of different ways to install gpg on OSX, with different results in what works and not. I don't know what's going on with that. Advice from those who understand Mac OS would be appreciated.
+
+As far as the \"gpg: no valid addressees\" bug goes, I can reproduce it on Linux, and have fixed it. Only affected using gcrypt with encryption=pubkey.
+
+(Please file bug reports in [[bugs]] in the future, not in the forum!)
+"""]]
diff --git a/doc/forum/GPG_passphrase_handling/comment_4_8724297f6d7ac140ab395a940bab0d7d._comment b/doc/forum/GPG_passphrase_handling/comment_4_8724297f6d7ac140ab395a940bab0d7d._comment
new file mode 100644
index 0000000000..983c35c3ab
--- /dev/null
+++ b/doc/forum/GPG_passphrase_handling/comment_4_8724297f6d7ac140ab395a940bab0d7d._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="https://www.google.com/accounts/o8/id?id=AItOawkzwmw_zyMpZC9_J7ey--woeYPoZkAOgGw"
+ nickname="dxtrish"
+ subject="comment 4"
+ date="2013-11-03T13:10:11Z"
+ content="""
+I would have filed it in the bug section if I knew it was a bug. I just thought I had done something wrong :)
+"""]]
diff --git a/doc/forum/Generating_a_Temp_View_of_Available_Files.mdwn b/doc/forum/Generating_a_Temp_View_of_Available_Files.mdwn
new file mode 100644
index 0000000000..bdb8ece692
--- /dev/null
+++ b/doc/forum/Generating_a_Temp_View_of_Available_Files.mdwn
@@ -0,0 +1 @@
+Is it possible to generate a view of files currently available on the annex? My use case is that I have pretty large repo (couple of TBs) and I have partial checkouts on multiple machines instead of seeing 100s of broken symlinks I would like to just filter filter files that are present on the machine?
diff --git a/doc/forum/Get___39__source__39___group_to_automatically_drop_with_assistant.mdwn b/doc/forum/Get___39__source__39___group_to_automatically_drop_with_assistant.mdwn
new file mode 100644
index 0000000000..be87682225
--- /dev/null
+++ b/doc/forum/Get___39__source__39___group_to_automatically_drop_with_assistant.mdwn
@@ -0,0 +1,88 @@
+Hi,
+
+I'm trying to get my head around groups, wanted, etc. for a particular use case.
+
+**Problem:** I can't work out how to get a source(?) repository to automatically drop files when they hit a transfer repository.
+
+
+I have a machine (`Machine 1`) that is used for data acquisition but it is behind a strict firewall (both physical and virtual).  I usually physically carry a USB drive over, set up a rsync ssh -> local-USB-drive from the one machine (`Machine 2`) that is able to connect over the network to `Machine 1`.  As it is a pain to lug the drive over, I only do this rsync maybe weekly, so the rsync takes many hours (~24) to complete.  Then (when I remember) I visit and I carry the USB drive back...  Naturally, this slows down my work process.
+
+What I was hoping to do was set up git-annex with the assistant to help me.  I am able to run the assistant, but not the webapp on `Machines 1 and 2`.  :-(
+
+My thought was - as these have to be disconnected network transfers...
+
+- `Repository 1 -> Repository 2` (when space permits)
+- `Repository 2 -> Repository 3` (when space permits) `-> Repository 4` (USB drive(s))
+
+
+Another limitation is that `Repos/Machines 2 & 3` have limited storage space.
+
+
+As a test case I can set up (`Repo1 -> Repo2`) and (`Repo2 -> Repo3`) (on other machines, but the commands should be the same...)
+
+After reading a bit I made a changed [preferred content](/preferred_content/standard_groups/) for a transfer repo to:
+
+```
+not (inallgroup=client and copies=client:1) and ($client)
+```
+
+i.e. `copies` from `2` to `1`.
+ 
+---
+
+
+Finally...The question
+----------------------
+
+**BUT** I can't work out how to get `Repo1` (the source) to automatically drop the files when they hit `Repo2` (what I'm guessing should be a transfer repository).
+
+Can anyone suggest how to automagically do this with the assistant?
+
+
+---
+
+
+If it would help I can share the git-annex commands I've been using, but as I'm only doing testing up at the moment, I'm happy to start from scratch if there is a RTFM page out there.  :-)
+
+
+I've put some details about my thoughts on the repositories and restrictions below.
+
+
+Thanks - Olaf
+
+
+
+
+Repository 1
+------------
+- Type: source (Data collection)
+- Human readable directory structure
+- Physically: Machine 1
+- Strict firewall only incoming network connections from Machine 2
+- Storage: 50Gb
+
+
+Repository 2
+------------
+- Type: transfer
+- Physically: Machine 2
+- Reasonably relaxed firewall, can talk to Repository 3
+- Limited storage: 10Gb
+
+
+Repository 3
+------------
+- Type: transfer
+- Pysically: Machine 3
+- Reasonably relaxed firewall, can talk to Repository 2
+- Limited storage: 10Gb
+- Connected to USB drive(s)
+
+
+Repository 4, 5, ...
+--------------------
+- Type: ? Client ?
+- Human readable directory structure
+- Physically: USB drive
+- Usually (but not always) connected to machine 3
+- Large storage (2Tb) + Additional drives
diff --git a/doc/forum/Get___39__source__39___group_to_automatically_drop_with_assistant/comment_1_ead4b5b7ec0d085d4fa87fb4e29c4d55._comment b/doc/forum/Get___39__source__39___group_to_automatically_drop_with_assistant/comment_1_ead4b5b7ec0d085d4fa87fb4e29c4d55._comment
new file mode 100644
index 0000000000..dec09cf28c
--- /dev/null
+++ b/doc/forum/Get___39__source__39___group_to_automatically_drop_with_assistant/comment_1_ead4b5b7ec0d085d4fa87fb4e29c4d55._comment
@@ -0,0 +1,49 @@
+[[!comment format=mdwn
+ username="joey"
+ subject="""comment 1"""
+ date="2017-05-11T16:06:03Z"
+ content="""
+Sounds like an interesting data-gathering application, I have to say
+I'm curious what it is. ;)
+
+If Repo1 is configured like this:
+
+	git-annex group here source
+	git annex wanted here standard
+
+Then it should want to drop the contents of files from Repo1 once it knows
+they have reached any other repository. (Sometimes people put a repository
+in a group but forget to set wanted to "standard" ...)
+
+Looks like Repo1 cannot make outgoing connections to Repo2?
+
+So, you need to run the assistant on Repo2 and probably on Repo1.
+Then it works like this:
+
+1. The assistant on Repo1 will commit files as they're put in the repository.
+2. The assistant on Repo2 will notice changes to Repo1, pull down the
+   files, see that Repo1 is configured as a source repository, and drop
+   the files from Repo1.
+3. The assistant on Repo1 will also notice when it's sent a file to Repo2
+   and can drop it from Repo1 then -- but this may take a while for it to
+   notice and #2 will probably happen first.
+
+Note that if you get the files added and committed by some other process,
+you don't really need to run the assistant on Repo1.
+
+The USB drives need to be client, so that once content reaches one of them,
+the content will be dropped from the transfer repositories. The way that
+part should work:
+
+1. Content reaches Repo2.
+2. The assistant in Repo3 notices the change to Repo2, pulls down
+   the files.
+3. The assistant in Repo3 pushes the files to a connected USB drive.
+4. The assistant in Repo3 drops content from itself and Repo2, since
+   it knows both are transfer repos and the content has reached a client
+   (USB).
+
+If you're having trouble getting any of this to work, I recommend
+running `git annex sync --content` manually while testing it, and make sure
+it does what you would expect to happen at each step.
+"""]]
diff --git a/doc/forum/Get___39__source__39___group_to_automatically_drop_with_assistant/comment_2_835dba9de7a5eea868635dc793d4e75b._comment b/doc/forum/Get___39__source__39___group_to_automatically_drop_with_assistant/comment_2_835dba9de7a5eea868635dc793d4e75b._comment
new file mode 100644
index 0000000000..34d3b46ec7
--- /dev/null
+++ b/doc/forum/Get___39__source__39___group_to_automatically_drop_with_assistant/comment_2_835dba9de7a5eea868635dc793d4e75b._comment
@@ -0,0 +1,15 @@
+[[!comment format=mdwn
+ username="leavingchicago@c04c893e78d1c4c76cb3e32b5c227cf42bbf7682"
+ nickname="leavingchicago"
+ avatar="http://cdn.libravatar.org/avatar/4ae498d3d6ee558d6b65caa658f72572"
+ subject="comment 2"
+ date="2017-05-11T23:23:45Z"
+ content="""
+<ASIDE>
    
+*Laughing* I wish the data collection was interesting.  Just a machine with locked down networking so that it's a tad more secure.  Closest example is an EC2 instance sitting in a VPC...
     
+</ASIDE>
+
+Thanks for the feedback Joey.  I tried again and this approach works **except** where the downstream repo is direct mode - in this case the transfer repo doesn't drop.  I'm not sure if it's a bug or a feature I didn't read about.
+
+Happy to submit a bug report with steps if it's unexpected.
+"""]]
diff --git a/doc/forum/Get___39__source__39___group_to_automatically_drop_with_assistant/comment_3_553542a4a2b2530863a006a1afe91637._comment b/doc/forum/Get___39__source__39___group_to_automatically_drop_with_assistant/comment_3_553542a4a2b2530863a006a1afe91637._comment
new file mode 100644
index 0000000000..3f3dae5673
--- /dev/null
+++ b/doc/forum/Get___39__source__39___group_to_automatically_drop_with_assistant/comment_3_553542a4a2b2530863a006a1afe91637._comment
@@ -0,0 +1,19 @@
+[[!comment format=mdwn
+ username="olaf"
+ avatar="http://cdn.libravatar.org/avatar/4ae498d3d6ee558d6b65caa658f72572"
+ subject="comment 3"
+ date="2017-05-11T23:45:36Z"
+ content="""
+Oops.  I committed the greatest sin...
+
+```
+git-annex version: 6.20170321-gf3dee9d65
+build flags: Assistant Webapp Pairing Testsuite S3(multipartupload)(storageclasses) WebDAV Inotify DBus DesktopNotify ConcurrentOutput TorrentParser MagicMime Feeds Quvi
+key/value backends: SHA256E SHA256 SHA512E SHA512 SHA224E SHA224 SHA384E SHA384 SHA3_256E SHA3_256 SHA3_512E SHA3_512 SHA3_224E SHA3_224 SHA3_384E SHA3_384 SKEIN256E SKEIN256 SKEIN512E SKEIN512 SHA1E SHA1 MD5E MD5 WORM URL
+remote types: git gcrypt p2p S3 bup directory rsync web bittorrent webdav tahoe glacier ddar hook external
+local repository version: 5
+supported repository versions: 3 5 6
+upgrade supported from repository versions: 0 1 2 3 4 5
+operating system: linux i386
+```
+"""]]
diff --git a/doc/forum/Get___39__source__39___group_to_automatically_drop_with_assistant/comment_4_4540a15672f9b48b0f9310c97f1a0b86._comment b/doc/forum/Get___39__source__39___group_to_automatically_drop_with_assistant/comment_4_4540a15672f9b48b0f9310c97f1a0b86._comment
new file mode 100644
index 0000000000..0c69fd562c
--- /dev/null
+++ b/doc/forum/Get___39__source__39___group_to_automatically_drop_with_assistant/comment_4_4540a15672f9b48b0f9310c97f1a0b86._comment
@@ -0,0 +1,11 @@
+[[!comment format=mdwn
+ username="joey"
+ subject="""comment 4"""
+ date="2017-06-07T16:30:29Z"
+ content="""
+I would not expect the behavior with a direct mode repo to be any
+different. Unless the file in direct mode is getting modified, or
+perhaps git-annex gets confused and thinks it's modified. Either would
+prevent dropping the content from the transfer repository to avoid losing
+the unmodified content.
+"""]]
diff --git a/doc/forum/Getting__58___hGetChar__58___illegal_operation___40__handle_is_closed__41__.mdwn b/doc/forum/Getting__58___hGetChar__58___illegal_operation___40__handle_is_closed__41__.mdwn
new file mode 100644
index 0000000000..c167a66b1f
--- /dev/null
+++ b/doc/forum/Getting__58___hGetChar__58___illegal_operation___40__handle_is_closed__41__.mdwn
@@ -0,0 +1,13 @@
+Hello,
+
+using git-annex 6.20180807-23 on Arch I get this message from time to time:
+
+     copy archive/Studium/Diplomarbeit/Daten/Vortrag/vortrag.snm (to xgm...) 
+     git-annex-shell: : hGetChar: illegal operation (handle is closed)ok
+     copy archive/Studium/Diplomarbeit/Daten/Wheels/SingleWheelCalcs/5SpokeWheel/5Spoke_MRF/6000/Lambda2.gz 
+       Lost connection (fd:21: hFlush: resource vanished (Broken pipe))
+     (fd:21: hClose: resource vanished (Broken pipe)) failed
+
+The peer is git-annex, downloaded at Aug 7 at Debian Stable. Repeating the file transfer makes the message goes away, but I have to initiate the transfer multiple times, to be able to transfer all files without this error message.
+
+Best Thanks!
diff --git a/doc/forum/Getting_metadata.mdwn b/doc/forum/Getting_metadata.mdwn
new file mode 100644
index 0000000000..1f6f91fc71
--- /dev/null
+++ b/doc/forum/Getting_metadata.mdwn
@@ -0,0 +1,30 @@
+Hi!
+
+what's wrong here with getting metadata back:
+
+    # setup
+    $ git init .
+    $ git annex init
+    init  ok
+    (Recording state in git...)
+    $ touch testfile
+    $ git add testfile 
+    $ git commit -m "imp"
+    [master (Basis-Commit) 55c385e] imp
+     1 file changed, 0 insertions(+), 0 deletions(-)
+     create mode 100644 testfile
+
+    # set metadata
+    $ git annex metadata testfile --set key=val
+
+    # retrieval doesn't work
+    $ git annex metadata testfile --get key
+    $ git annex metadata testfile
+    $ 
+
+What I would expect here is to get "val" back.
+
+What am I doing wrong?
+
+
+Thanks!
diff --git a/doc/forum/Getting_metadata/comment_1_24bef8f59f370897d6fd6b2b43ed1f39._comment b/doc/forum/Getting_metadata/comment_1_24bef8f59f370897d6fd6b2b43ed1f39._comment
new file mode 100644
index 0000000000..125b852694
--- /dev/null
+++ b/doc/forum/Getting_metadata/comment_1_24bef8f59f370897d6fd6b2b43ed1f39._comment
@@ -0,0 +1,13 @@
+[[!comment format=mdwn
+ username="joey"
+ subject="""comment 1"""
+ date="2015-11-10T19:12:57Z"
+ content="""
+`git-annex metadata` only works on annexed files. Your `testfile` is
+checked into git directly, so these git-annex commands silently skip
+it, the same as `git annex whereis` will.
+
+There's some chance this could change.. It would be kind of nice,
+for git-annex's metadata to be able to be applied to regular files
+in the git repository.
+"""]]
diff --git a/doc/forum/Getting_started_with_Amazon_S3.mdwn b/doc/forum/Getting_started_with_Amazon_S3.mdwn
new file mode 100644
index 0000000000..1ee86b57fa
--- /dev/null
+++ b/doc/forum/Getting_started_with_Amazon_S3.mdwn
@@ -0,0 +1,28 @@
+I'm just getting started with git-annex and trying to wrap my head around using it with Amazon S3. I am familiar with using git, but things are a bit different as we can't init a repo at S3 directly.
+
+I've followed http://git-annex.branchable.com/tips/using_Amazon_S3/, and performed:
+
+`git init`
    
+Initialized empty Git repository in /home/
    
+`git annex init`
    
+init  ok
    
+`git annex initremote s3 type=S3 encryption=FOOBAR bucket=foo`
    
+initremote s3 (encryption setup with gpg key YGTVT51715TFR) (checking bucket...) (gpg) ok
    
+`git annex describe s3 "Amazon S3"`
    
+describe s3 ok
    
+`git annexx add foo/`
    
+add foo/bar.txt
    
+add foo/bar.png
    
+...etc
    
+`git annex sync`
    
+51 files changed, 51 insertions(+)
    
+create mode 120000 foo/bar.txt
    
+create mode 120000 foo/bar.png
    
+...etc
    
+
+
+Looking at http://git-annex.branchable.com/git-annex/, I thought the files added would then be pushed to S3 by git annex sync, but that doesn't seem to be the case. I've also tried variations of got annex copy, like `git annex copy . --to s3`, without any luck.
+
+Is there a way to push to s3?
+
+Any help is appreciated!
diff --git a/doc/forum/Getting_started_with_Amazon_S3/comment_1_f50883133d5d4903cc95c0dcaa52d052._comment b/doc/forum/Getting_started_with_Amazon_S3/comment_1_f50883133d5d4903cc95c0dcaa52d052._comment
new file mode 100644
index 0000000000..b2211fa6c1
--- /dev/null
+++ b/doc/forum/Getting_started_with_Amazon_S3/comment_1_f50883133d5d4903cc95c0dcaa52d052._comment
@@ -0,0 +1,10 @@
+[[!comment format=mdwn
+ username="http://joeyh.name/"
+ ip="4.153.81.112"
+ subject="comment 1"
+ date="2012-05-29T19:09:50Z"
+ content="""
+`git annex sync` only syncs git metadata, not file contents, and metadata is not stored on S3, so it does notthing (much).
+
+`git annex move . --to s3` or `git annex copy . --to s3` is the right way to send the files to S3. I'm not sure why you say it's not working. I'd try it but Amazon is not letting me sign up for S3 again right now. Can you show what goes wrong with copy?
+"""]]
diff --git a/doc/forum/Getting_started_with_Amazon_S3/comment_2_e90aa3259d9a12cd67daa27d42d69ab5._comment b/doc/forum/Getting_started_with_Amazon_S3/comment_2_e90aa3259d9a12cd67daa27d42d69ab5._comment
new file mode 100644
index 0000000000..742e8d4469
--- /dev/null
+++ b/doc/forum/Getting_started_with_Amazon_S3/comment_2_e90aa3259d9a12cd67daa27d42d69ab5._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="https://www.google.com/accounts/o8/id?id=AItOawnoUOqs_lbuWyZBqyU6unHgUduJwDDgiKY"
+ nickname="Matt"
+ subject="comment 2"
+ date="2012-05-30T00:40:45Z"
+ content="""
+It's strange. I've done some testing on another machine, and this one, and the issue seems to be with adding only certain sub-directories of the git-annex directory. Would it cause an issue with git-annex if a sub-directory was a git repo?
+"""]]
diff --git a/doc/forum/Getting_started_with_Amazon_S3/comment_3_c3adce7c0f29e71ed9dd07103ede2c1a._comment b/doc/forum/Getting_started_with_Amazon_S3/comment_3_c3adce7c0f29e71ed9dd07103ede2c1a._comment
new file mode 100644
index 0000000000..450a1513ce
--- /dev/null
+++ b/doc/forum/Getting_started_with_Amazon_S3/comment_3_c3adce7c0f29e71ed9dd07103ede2c1a._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="http://joeyh.name/"
+ ip="4.153.81.112"
+ subject="comment 3"
+ date="2012-05-30T00:54:38Z"
+ content="""
+If the subdirectory has a .git, then it's a separate git repo, and inside the directory, all git (and git-annex) commands in it will operate on that nested repo and ignore the outside one.
+"""]]
diff --git a/doc/forum/Getting_the_status_of_a_remotely_changed_annex_file.mdwn b/doc/forum/Getting_the_status_of_a_remotely_changed_annex_file.mdwn
new file mode 100644
index 0000000000..9a2c807992
--- /dev/null
+++ b/doc/forum/Getting_the_status_of_a_remotely_changed_annex_file.mdwn
@@ -0,0 +1,75 @@
+Hello,
+
+I am trying to wrap my head around annex still.  I'm creating a source local git repo, editing an annex file, and then syncing in a second repo.  In that second repo I'm trying to figure out how I can get a status notifying me that a file is out of date.
+
+If I use 'annex sync --content' the files are all up to date, as expected, but what I kind of expected is 'annex status' to say something like 'files out of date, blah blah'.  I am spelling out my example below.  
+
+Annex version is 5.20140613
+
+
+## I create a source and target repo, and I add a git managed file and an annex managed file
+
+    $ mkdir source target
+    $ cd source
+    $ git init
+    $ git annex init
+    $ cat > gitfile
+    hi this is my git file
+    $ cat > annexfile
+    hi this is my annex file
+    $ git add gitfile 
+    $ git annex add annexfile 
+    $ git commit -m 'init commit'
+    $ cd ../target
+    $ git clone ../source/ .
+    $ ls -al
+    total 16
+    drwxr-xr-x  170 Jul 15 15:55 .
+    drwxr-xr-x  136 Jul 15 15:54 ..
+    drwxr-xr-x  442 Jul 15 15:55 .git
+    lrwxr-xr-x  180 Jul 15 15:55 annexfile -> .git/annex/objects/Qp/F0/SHA256E-s25--045cf30cb201c6723cb6fad9ca539f639de7f242b87775b876ef9ccb1f577ccf/SHA256E-s25--045cf30cb201c6723cb6fad9ca539f639de7f242b87775b876ef9ccb1f577ccf
+    -rw-r--r--  23 Jul 15 15:55 gitfile
+    $ git annex sync
+    ...
+    To [base path]/target/../source/
+     * [new branch]      git-annex -> synced/git-annex
+     * [new branch]      master -> synced/master
+    ok
+    $ cat gitfile 
+    hi this is my git file
+    $ cat annexfile 
+    cat: annexfile: No such file or directory
+    $ git annex sync --content
+    $ cat annexfile 
+    hi this is my annex file
+
+## So far so good.  Now I'm going to edit my annexed file in source repo and see if I can find out that the file was edited in the target
+
+    $ cd ..
+    $ cd source
+    $ git annex edit annexfile 
+    $ cat > annexfile 
+    wow I changed my annex file
+    $ git annex add annexfile 
+    $ git commit -m 'changed an annex file'
+    $ cd ..
+    $ cd target
+    $ git pull
+    $ cat annexfile 
+    cat: annexfile: No such file or directory
+    $ git annex sync
+    $ ls
+    annexfile gitfile
+    $ cat annexfile 
+    cat: annexfile: No such file or directory
+    $ git annex status
+    [no output]
+
+## Here is where I'd expect something saying 'annexfile is out of date', etc.
+## I can infer it because the link is missing, but I'm sure there's a more logical way.
+
+    $ git annex sync --content
+    $ cat annexfile 
+    wow I changed my annex file
+
+## After I synced content all is well.
diff --git a/doc/forum/Getting_the_status_of_a_remotely_changed_annex_file/comment_1_e323c21d27bb0946993ba1438429c457._comment b/doc/forum/Getting_the_status_of_a_remotely_changed_annex_file/comment_1_e323c21d27bb0946993ba1438429c457._comment
new file mode 100644
index 0000000000..78b0a9c1e5
--- /dev/null
+++ b/doc/forum/Getting_the_status_of_a_remotely_changed_annex_file/comment_1_e323c21d27bb0946993ba1438429c457._comment
@@ -0,0 +1,14 @@
+[[!comment format=mdwn
+ username="http://joeyh.name/"
+ ip="209.250.56.2"
+ subject="comment 1"
+ date="2014-07-16T18:18:00Z"
+ content="""
+As soon as git-annex syncs with a remote that has made changes to the repository, any new files or files that have been modified, and the content is not yet available in the local repository will be broken symlinks. You can generally easily see a broken symlink by running `ls`, at least with a good `ls` on eg Linux that supports colorization. (Look for the red filenames.) Many file managers also represent broken symlinks in a visual way. 
+
+Or, you can run a command like `git annex whereis $file` and see if it includes \"here\" in the list of locations.
+
+Or, you can run `git annex find --not --in here` to find all files whose current content is not present.
+
+The reason `git annex status` doesn't say is that it's focused on showing you which files in the local repository have not yet been committed to git. The `git annex list` command has a similar one line per file output, but puts a \"_\" in the first column (under \"here\") if the file is not locally present.
+"""]]
diff --git a/doc/forum/Git-Annex_Android_sync_files_are_missing_on_Linuxx.mdwn b/doc/forum/Git-Annex_Android_sync_files_are_missing_on_Linuxx.mdwn
new file mode 100644
index 0000000000..4f144d51e4
--- /dev/null
+++ b/doc/forum/Git-Annex_Android_sync_files_are_missing_on_Linuxx.mdwn
@@ -0,0 +1,4 @@
+Hi
+I created shares between Linux and Android(using nightly 4.4). I used Assistant on both. It seems like the files from Android to Linux are all missing, at least the symlinks are broken maybe. However files from Linux to Android are fine.
+
+thanks
diff --git a/doc/forum/Git-Annex_Android_sync_files_are_missing_on_Linuxx/comment_1_72d7811990e78fba0b7fc2e1c7ee515f._comment b/doc/forum/Git-Annex_Android_sync_files_are_missing_on_Linuxx/comment_1_72d7811990e78fba0b7fc2e1c7ee515f._comment
new file mode 100644
index 0000000000..9a0f2880f8
--- /dev/null
+++ b/doc/forum/Git-Annex_Android_sync_files_are_missing_on_Linuxx/comment_1_72d7811990e78fba0b7fc2e1c7ee515f._comment
@@ -0,0 +1,15 @@
+[[!comment format=mdwn
+ username="htun"
+ ip="5.104.224.15"
+ subject="comment 1"
+ date="2014-10-17T04:57:08Z"
+ content="""
+
+
+Here is what the file is looking for symlink 
+.git/annex/objects/MF/kG/SHA256E-s1589297--0ec542aabec66b1877699ef27549fe355339224680d7923078d03f375028fca1.apk/SHA256E-s1589297--0ec542aabec66b1877699ef27549fe355339224680d7923078d03f375028fca1.apk
+
+
+and here is what is in the objects SHA256E-s1589297--0ec542aabec66b1877699ef27549fe355339224680d7923078d03f375028fca1.apk/SHA256E-s1589297--0ec542aabec66b1877699ef27549fe355339224680d7923078d03f375028fca1.apk.map  
+
+"""]]
diff --git a/doc/forum/Git-Annex_Android_sync_files_are_missing_on_Linuxx/comment_2_cfb7139958ed24c1c64f8b133c99f336._comment b/doc/forum/Git-Annex_Android_sync_files_are_missing_on_Linuxx/comment_2_cfb7139958ed24c1c64f8b133c99f336._comment
new file mode 100644
index 0000000000..8001f5cc9a
--- /dev/null
+++ b/doc/forum/Git-Annex_Android_sync_files_are_missing_on_Linuxx/comment_2_cfb7139958ed24c1c64f8b133c99f336._comment
@@ -0,0 +1,11 @@
+[[!comment format=mdwn
+ username="joey"
+ subject="""comment 2"""
+ date="2015-07-07T18:56:51Z"
+ content="""
+Sounds like the Android repository is not uploading the file contents to
+someplace where the Linux repository can download them from. So the first
+step is to look at what remotes the Android repository is configured to
+use, and see if it's successfully sending files to those. And then check if
+the Linux repository is able to get files from those same remotes.
+"""]]
diff --git a/doc/forum/Git-Annex_with_Asustor_NAS.mdwn b/doc/forum/Git-Annex_with_Asustor_NAS.mdwn
new file mode 100644
index 0000000000..8d29ce0235
--- /dev/null
+++ b/doc/forum/Git-Annex_with_Asustor_NAS.mdwn
@@ -0,0 +1,4 @@
+I'd like to use git-annex with my Asustor 302t NAS. It should be treated like a "simple" removable drive. I don't really know how to do this however, when scanning for "removable drives", it can't be found, when I try connecting via SSH, I receive an Error: Permission denied (publickey,password). As there is no way to insert my password before connecting.
+
+How do I setup a repository on my NAS to be synced with my laptop?
+I am using MAC OS X and the git-annex web interface currently.
diff --git a/doc/forum/Git-Annex_with_Asustor_NAS/comment_1_44445200e5b716caeec225972a5d5dce._comment b/doc/forum/Git-Annex_with_Asustor_NAS/comment_1_44445200e5b716caeec225972a5d5dce._comment
new file mode 100644
index 0000000000..ae5ea41e23
--- /dev/null
+++ b/doc/forum/Git-Annex_with_Asustor_NAS/comment_1_44445200e5b716caeec225972a5d5dce._comment
@@ -0,0 +1,10 @@
+[[!comment format=mdwn
+ username="http://joeyh.name/"
+ ip="108.236.230.124"
+ subject="comment 1"
+ date="2014-05-16T16:32:05Z"
+ content="""
+If your NAS is accessed over SSH, it is not a removably drive, clearly.
+
+Try starting git-annex webapp from a console, you will probably see the ssh password prompt there. Or, install ssh-askpass, or, install a current daily build of git-annex, which will prompt for the ssh password from within the webapp.
+"""]]
diff --git a/doc/forum/Git-annex_and_Microsoft_Office_files_on_OS_X.mdwn b/doc/forum/Git-annex_and_Microsoft_Office_files_on_OS_X.mdwn
new file mode 100644
index 0000000000..ab26e2b512
--- /dev/null
+++ b/doc/forum/Git-annex_and_Microsoft_Office_files_on_OS_X.mdwn
@@ -0,0 +1 @@
+On OS X 10.11.6, if you save any MS office file (.pptx, .docx, .xlsx) in a git-annex directory it won't be synced unless you restart the daemon.  If you look in the logs it says "/Users/me/annex/test.pptx still has writers, not adding"  This probably has something to do with temporary/lockfiles used by MS office apps.  Any ideas on a workaround, other than restarting the daemon constantly?
diff --git a/doc/forum/Git-annex_and_Microsoft_Office_files_on_OS_X/comment_1_71fdd5e7061d6deb357290057804cc27._comment b/doc/forum/Git-annex_and_Microsoft_Office_files_on_OS_X/comment_1_71fdd5e7061d6deb357290057804cc27._comment
new file mode 100644
index 0000000000..3cd1c5943c
--- /dev/null
+++ b/doc/forum/Git-annex_and_Microsoft_Office_files_on_OS_X/comment_1_71fdd5e7061d6deb357290057804cc27._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="joey"
+ subject="""comment 1"""
+ date="2017-06-09T17:51:35Z"
+ content="""
+A bug was opened about this:
+[[bugs/Git-annex_and_Microsoft_Office_files_on_OS_X]]
+"""]]
diff --git a/doc/forum/Git-annex_assistant__58___where_are_deleted_files_and_older_versions_saved__63__.mdwn b/doc/forum/Git-annex_assistant__58___where_are_deleted_files_and_older_versions_saved__63__.mdwn
new file mode 100644
index 0000000000..7771403541
--- /dev/null
+++ b/doc/forum/Git-annex_assistant__58___where_are_deleted_files_and_older_versions_saved__63__.mdwn
@@ -0,0 +1,9 @@
+Hello,
+
+first of all thank you very much for Git-annex: I think it is a great tool to keep files in sync. The feature I like most is the possibility of setting multiple remote servers and execute syncs in a decentralized fashion; this is great for me since I have multiple computers spreaded around but not all of them are always up and almost none is "99.99% guaranteed" and I need high availability.
+
+Getting to the poing: I set up git-annex manually on my computers (most remote are only accessible via ssh tunnels) and I am using it in direct mode, with the assistant running in background on every node. From time to time I open the webapp to check if everything looks fine. Finally, I set a couple of nodes (with plenty of storage) as 'full backup' even if I don't really understand what that setting is really for.
+
+My question is: in my setup, where do deleted files and older versions go? If I ever need one in the future where should I look for it?
+
+Thanks a million!
diff --git a/doc/forum/Git-annex_assistant__58___where_are_deleted_files_and_older_versions_saved__63__/comment_10_645af86feb8506c3ea2134badac68e0b._comment b/doc/forum/Git-annex_assistant__58___where_are_deleted_files_and_older_versions_saved__63__/comment_10_645af86feb8506c3ea2134badac68e0b._comment
new file mode 100644
index 0000000000..8029f07c0b
--- /dev/null
+++ b/doc/forum/Git-annex_assistant__58___where_are_deleted_files_and_older_versions_saved__63__/comment_10_645af86feb8506c3ea2134badac68e0b._comment
@@ -0,0 +1,7 @@
+[[!comment format=mdwn
+ username="Agenta"
+ subject="comment 10"
+ date="2015-11-19T09:23:55Z"
+ content="""
+Yes I might try that as well, using Bup as a special remote instead of using nightly backups! Thanks for the heads up!
+"""]]
diff --git a/doc/forum/Git-annex_assistant__58___where_are_deleted_files_and_older_versions_saved__63__/comment_1_d9acd44c00676d814e19613d6fd276a2._comment b/doc/forum/Git-annex_assistant__58___where_are_deleted_files_and_older_versions_saved__63__/comment_1_d9acd44c00676d814e19613d6fd276a2._comment
new file mode 100644
index 0000000000..a4915729af
--- /dev/null
+++ b/doc/forum/Git-annex_assistant__58___where_are_deleted_files_and_older_versions_saved__63__/comment_1_d9acd44c00676d814e19613d6fd276a2._comment
@@ -0,0 +1,13 @@
+[[!comment format=mdwn
+ username="Marco"
+ subject="Repository groups"
+ date="2015-06-23T12:21:58Z"
+ content="""
+Have a look at the [[preferred_content/standard_groups]] page. Backup Repositories want to get hold of anything.
+
+> anything
+>
+>Matches any version of any file.
+
+So if you have the assistant running it will take care of pushing the old versions in the backup.
+"""]]
diff --git a/doc/forum/Git-annex_assistant__58___where_are_deleted_files_and_older_versions_saved__63__/comment_2_45928886d1ae4a76e2daefb355967729._comment b/doc/forum/Git-annex_assistant__58___where_are_deleted_files_and_older_versions_saved__63__/comment_2_45928886d1ae4a76e2daefb355967729._comment
new file mode 100644
index 0000000000..1b0f118343
--- /dev/null
+++ b/doc/forum/Git-annex_assistant__58___where_are_deleted_files_and_older_versions_saved__63__/comment_2_45928886d1ae4a76e2daefb355967729._comment
@@ -0,0 +1,19 @@
+[[!comment format=mdwn
+ username="joey"
+ subject="""comment 2"""
+ date="2015-07-06T19:03:57Z"
+ content="""
+While Marco is right about backup repositories, note that the assistant can
+only move the content of deleted/old versions of files to a backup repository 
+if that repository is set up as a remote of the repository where the
+assistant is running.
+
+So, it's quite possible to have some backup repositories that are not well
+enough connected for deleted/old files to reach them. The deleted/old files
+will then be kept in whatever repositories happen to contain them.
+
+Where are the deleted files stored? Underneath .git/annex/objects/
+
+Also, the webapp has a configuration page to control what to do with unused
+files, and can be configured to delete them periodically if you'd prefer.
+"""]]
diff --git a/doc/forum/Git-annex_assistant__58___where_are_deleted_files_and_older_versions_saved__63__/comment_3_1249bf921ea7709b4f7ae25e613bf60a._comment b/doc/forum/Git-annex_assistant__58___where_are_deleted_files_and_older_versions_saved__63__/comment_3_1249bf921ea7709b4f7ae25e613bf60a._comment
new file mode 100644
index 0000000000..4fb44b1602
--- /dev/null
+++ b/doc/forum/Git-annex_assistant__58___where_are_deleted_files_and_older_versions_saved__63__/comment_3_1249bf921ea7709b4f7ae25e613bf60a._comment
@@ -0,0 +1,11 @@
+[[!comment format=mdwn
+ username="r@9dd621f83d4900d70d4ed2319f20160749f1343e"
+ nickname="r"
+ subject="How to get deleted files and older versions out of a full backup repository?"
+ date="2015-11-16T21:27:33Z"
+ content="""
+I'm sorry if I have overlooked something obvious, but after playing with a \"simple\" setup (2 desktops and 1 laptop all syncing their ~/annex directory with the assistant/webapp with direct ssh access and XMPP, including one \"full backup\" repository on an ssh server accessible by all) and moving a few directories in and out of ~/annex to verify that it syncs between the clients, the deleted files seem to be present only in the backup repository - simply judging by the size of the repositories on the different machines. However, how can I get them back after deleting them and letting that sync out by the assistant? 
+On the clients, git annex list/log will only print the files still there, not the deleted ones. On the backup repository (which seems to be a bare repository), git annex list/log is mostly empty. The backup repository was automatically created by the webapp on the first client I set up, I only created the (empty) directory and assigned appropriate user ACLs on the ssh server. I have not manually created a (non-bare) git repository there.
+
+Is this a supported/sane configuration for using assistant/webapp? The idea is to have the ~/annex directories behave like Dropbox (simplified point of view), but still be able to get to deleted or old versions of files. I am aware that there is no UI support for this yet, but I understood this to be supported by the \"full backup\" repositories.
+"""]]
diff --git a/doc/forum/Git-annex_assistant__58___where_are_deleted_files_and_older_versions_saved__63__/comment_4_616cb76a510fa32c2aa48c25d24c6044._comment b/doc/forum/Git-annex_assistant__58___where_are_deleted_files_and_older_versions_saved__63__/comment_4_616cb76a510fa32c2aa48c25d24c6044._comment
new file mode 100644
index 0000000000..cad2b2493b
--- /dev/null
+++ b/doc/forum/Git-annex_assistant__58___where_are_deleted_files_and_older_versions_saved__63__/comment_4_616cb76a510fa32c2aa48c25d24c6044._comment
@@ -0,0 +1,10 @@
+[[!comment format=mdwn
+ username="Agenta"
+ subject="comment 4"
+ date="2015-11-17T10:32:41Z"
+ content="""
+Dear r,
+
+I did lots of testing about recovering deleted files or previous versions with git-annex but I didn't have much success. I really found it difficult to recover files from git-annex, too time consuming.
+At the end I found a chep and dirty solution. I write it here in case it may be useful to you or to somone else. I now run a daily backup of my git-annex repo with Bup (http://bup.github.io/) on a couple of my computers, so I can easily restore to a previous state if needed. Bup implements data deduplication so you can have daily backups without consuming too much space. Bup is extremely easy to use too, you can mount backups as fuse filesystems or view them via a web browser. The only problem with this approach is that if you change a file multiple times during a day you won't be able to recover those changes.
+"""]]
diff --git a/doc/forum/Git-annex_assistant__58___where_are_deleted_files_and_older_versions_saved__63__/comment_5_100fc29a95923e5ef92eaf87d76bbc6f._comment b/doc/forum/Git-annex_assistant__58___where_are_deleted_files_and_older_versions_saved__63__/comment_5_100fc29a95923e5ef92eaf87d76bbc6f._comment
new file mode 100644
index 0000000000..c527276c6d
--- /dev/null
+++ b/doc/forum/Git-annex_assistant__58___where_are_deleted_files_and_older_versions_saved__63__/comment_5_100fc29a95923e5ef92eaf87d76bbc6f._comment
@@ -0,0 +1,37 @@
+[[!comment format=mdwn
+ username="joey"
+ subject="""comment 5"""
+ date="2015-11-18T17:35:10Z"
+ content="""
+All you need is a clone of the repository that is not using direct mode.
+(The assistant normally sets up repositories using direct mode.)
+So either stop the assistant and run "git annex direct" inside ~/annex,
+or `git clone` the backup repository to a temporary location.
+
+In that clone, you can use regular git commands to check out past versions
+of the tree. Inside such a checkout, you can use `git annex get` to get any
+of the files in that checkout from your backup repository or wherever the
+content of the file has been stored.
+
+For example, using the git-annex repository that I use to publish builds of
+git-annex:
+
+	# git clone https://downloads.kitenet.net/.git/ downloads
+	# cd downloads
+	# git log --stat git-annex/linux/current/
+	(shows a whole lot of past versions of files in that directory)
+	(I decide I want one from a few months ago)
+	# git checkout 81a0d7d
+	HEAD is now at 81a0d7d... publishing git-annex 5.20150731
+	# git annex get git-annex/linux/current/git-annex-standalone-i386.tar.gz
+	get git-annex/linux/current/git-annex-standalone-i386.tar.gz (from origin...) ok
+
+Your backup repository can be used the same way. 
+
+Deleted and old versions of files will be retained in it, as long as the
+assistant was able to send them to the backup repo in the first place
+before they got deleted or modified. When direct mode is used, if you
+change a file before it gets copied to the backup, its old contents are gone
+for good. To avoid that and make sure that all annexed files always get
+backed up, switch your ~/annex to use indirect mode.
+"""]]
diff --git a/doc/forum/Git-annex_assistant__58___where_are_deleted_files_and_older_versions_saved__63__/comment_6_9b19ae8e8c0c70e3827c9f362c1c1dea._comment b/doc/forum/Git-annex_assistant__58___where_are_deleted_files_and_older_versions_saved__63__/comment_6_9b19ae8e8c0c70e3827c9f362c1c1dea._comment
new file mode 100644
index 0000000000..fbec80f927
--- /dev/null
+++ b/doc/forum/Git-annex_assistant__58___where_are_deleted_files_and_older_versions_saved__63__/comment_6_9b19ae8e8c0c70e3827c9f362c1c1dea._comment
@@ -0,0 +1,9 @@
+[[!comment format=mdwn
+ username="Agenta"
+ subject="comment 6"
+ date="2015-11-18T18:07:51Z"
+ content="""
+Hello joey,
+
+but that doesn't use any kind of data deduplication, am I correct? In my particular use case the indirect backup repo could grow huge so I may be better of avoiding that.
+"""]]
diff --git a/doc/forum/Git-annex_assistant__58___where_are_deleted_files_and_older_versions_saved__63__/comment_7_0c1c76bb082c108f3074b725bcc272c9._comment b/doc/forum/Git-annex_assistant__58___where_are_deleted_files_and_older_versions_saved__63__/comment_7_0c1c76bb082c108f3074b725bcc272c9._comment
new file mode 100644
index 0000000000..d437b916d5
--- /dev/null
+++ b/doc/forum/Git-annex_assistant__58___where_are_deleted_files_and_older_versions_saved__63__/comment_7_0c1c76bb082c108f3074b725bcc272c9._comment
@@ -0,0 +1,9 @@
+[[!comment format=mdwn
+ username="joey"
+ subject="""comment 7"""
+ date="2015-11-18T19:09:48Z"
+ content="""
+git-annex automatically deduplicates files. The backup repo will store one
+copy of each version of each file, which may indeed get large depending on
+what you're doing.
+"""]]
diff --git a/doc/forum/Git-annex_assistant__58___where_are_deleted_files_and_older_versions_saved__63__/comment_8_1d7c8c8c5b34cbc7166f8d4d52b91761._comment b/doc/forum/Git-annex_assistant__58___where_are_deleted_files_and_older_versions_saved__63__/comment_8_1d7c8c8c5b34cbc7166f8d4d52b91761._comment
new file mode 100644
index 0000000000..41462a16b2
--- /dev/null
+++ b/doc/forum/Git-annex_assistant__58___where_are_deleted_files_and_older_versions_saved__63__/comment_8_1d7c8c8c5b34cbc7166f8d4d52b91761._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="Agenta"
+ subject="comment 8"
+ date="2015-11-18T19:19:37Z"
+ content="""
+It is my understanding the Bup deduplicates even \"within\" files, i.e. if a file was modified only the changes are stored, not the entire new version. For my use case that would save a lot of space.
+
+"""]]
diff --git a/doc/forum/Git-annex_assistant__58___where_are_deleted_files_and_older_versions_saved__63__/comment_9_2bfb2eaf1a72e0822341cfeba9058aa7._comment b/doc/forum/Git-annex_assistant__58___where_are_deleted_files_and_older_versions_saved__63__/comment_9_2bfb2eaf1a72e0822341cfeba9058aa7._comment
new file mode 100644
index 0000000000..086dce7528
--- /dev/null
+++ b/doc/forum/Git-annex_assistant__58___where_are_deleted_files_and_older_versions_saved__63__/comment_9_2bfb2eaf1a72e0822341cfeba9058aa7._comment
@@ -0,0 +1,13 @@
+[[!comment format=mdwn
+ username="joey"
+ subject="""comment 9"""
+ date="2015-11-18T19:46:07Z"
+ content="""
+That's right, bup can store multiple versions of a file as efficiently as
+git itself can (or maybe a little more).
+
+You can use [bup as a special remote of git-annex](https://git-annex.branchable.com/special_remotes/bup/),
+and then your backup repository would be a bup repository.
+Restoring old versions of files from it would work the same way
+I described above.
+"""]]
diff --git a/doc/forum/Git-annex_assistant_configuration_file__40__s__41___location__40__s__41__.mdwn b/doc/forum/Git-annex_assistant_configuration_file__40__s__41___location__40__s__41__.mdwn
new file mode 100644
index 0000000000..36cfd4d768
--- /dev/null
+++ b/doc/forum/Git-annex_assistant_configuration_file__40__s__41___location__40__s__41__.mdwn
@@ -0,0 +1,6 @@
+Hi,
+
+I accidentally removed the .git/ folder inside a git annex assistant-monitored folder. When I open the assistant web app it still shows the repository but when I try to select it the page request never returns. What can I do at this point? I was thinking I can remove the repository from the git annex assitant configuration file if it is a plain-text file, but I don't know of its location. Please advise.
+
+Regards,
+Blake
diff --git a/doc/forum/Git-annex_assistant_configuration_file__40__s__41___location__40__s__41__/comment_1_5baffd4d6994bbcb23614b17777a0ffe._comment b/doc/forum/Git-annex_assistant_configuration_file__40__s__41___location__40__s__41__/comment_1_5baffd4d6994bbcb23614b17777a0ffe._comment
new file mode 100644
index 0000000000..f48337a423
--- /dev/null
+++ b/doc/forum/Git-annex_assistant_configuration_file__40__s__41___location__40__s__41__/comment_1_5baffd4d6994bbcb23614b17777a0ffe._comment
@@ -0,0 +1,10 @@
+[[!comment format=mdwn
+ username="http://joeyh.name/"
+ ip="209.250.56.87"
+ subject="comment 1"
+ date="2013-12-17T16:07:00Z"
+ content="""
+The file you need is .config/git-annex/autostart in your home directory.
+
+Once deleted, re-running the assistant should let you re-create the git repository and it'll re-add your files.
+"""]]
diff --git a/doc/forum/Git-annex_deleted_all_my_files__44___how_can_I_recover_them__63__.mdwn b/doc/forum/Git-annex_deleted_all_my_files__44___how_can_I_recover_them__63__.mdwn
new file mode 100644
index 0000000000..c5a0d70fb8
--- /dev/null
+++ b/doc/forum/Git-annex_deleted_all_my_files__44___how_can_I_recover_them__63__.mdwn
@@ -0,0 +1,19 @@
+I am trying to follow this guide:
+
+https://git-annex.branchable.com/walkthrough/
+
+I:
+
+* created a repository
+* added a remote
+* added all the files via `git annex add .`
+* commited my changes
+* synced my repositories
+
+
+By the time I tied to actually get the content from repo one into repo two (via `git annex get folder`) I noticed that all of my stuff had been deleted from repo one (and not backed up by git annex anywhere else!).
+
+How can I get my stuff back?
+
+Also, why would any software which is meant for backup or archiving (or any sane software in general) delete stuff without me actually typing `rm` or `delete` into the terminal and without giving me a big warning message and a confirmation prompt? 
+ 
diff --git a/doc/forum/Git-annex_deleted_all_my_files__44___how_can_I_recover_them__63__/comment_1_65b41a6eb28261e04e4fe8732f97a1f1._comment b/doc/forum/Git-annex_deleted_all_my_files__44___how_can_I_recover_them__63__/comment_1_65b41a6eb28261e04e4fe8732f97a1f1._comment
new file mode 100644
index 0000000000..c06b202f29
--- /dev/null
+++ b/doc/forum/Git-annex_deleted_all_my_files__44___how_can_I_recover_them__63__/comment_1_65b41a6eb28261e04e4fe8732f97a1f1._comment
@@ -0,0 +1,9 @@
+[[!comment format=mdwn
+ username="CandyAngel"
+ subject="comment 1"
+ date="2015-04-18T07:16:01Z"
+ content="""
+Are you sure the files are actually deleted? You don't mention using direct mode or anything, so you should be able to reset the git repository to the last commit you did (restoring the symlinks) and be back on track.
+
+Sounds like you missed something either in your description of the problem, or while following the tutorial.
+"""]]
diff --git a/doc/forum/Git-annex_deleted_all_my_files__44___how_can_I_recover_them__63__/comment_2_200a869f335909566b9ddab3032fd5a2._comment b/doc/forum/Git-annex_deleted_all_my_files__44___how_can_I_recover_them__63__/comment_2_200a869f335909566b9ddab3032fd5a2._comment
new file mode 100644
index 0000000000..bb55a04eda
--- /dev/null
+++ b/doc/forum/Git-annex_deleted_all_my_files__44___how_can_I_recover_them__63__/comment_2_200a869f335909566b9ddab3032fd5a2._comment
@@ -0,0 +1,10 @@
+[[!comment format=mdwn
+ username="https://www.google.com/accounts/o8/id?id=AItOawkW9u-8uqR62QBZjeTNCXsL7Ds55dAMGbA"
+ nickname="Horea"
+ subject="comment 2"
+ date="2015-04-28T10:15:24Z"
+ content="""
+Apparently what happened is that whenever you want to set up git annex, you have to create just one repository, them *move* your files to a temporary folder on all your other media, *clone* the repository to the other media, and then copy all your files back to the clone and add them to git annex. This is a horrible workflow and not mentioned anywhere. If you do the intuitive thing and start by making a bunch of repos, because all repos are created equal, git annex will happily delete all your stuff on some of them. 
+
+Indeed, as you correctly mention, this can go horribly wrong if you use direct mode and cannot recover your files. A shame, really, that direct mode is the only mode in which you actually have write and copy access to your files, and thus, for most - including myself - the only sane way of managing your files. For the record, I was able to retrieve my files, but to this day git annex is the only software I ever used that ever managed to lose my data. 
+"""]]
diff --git a/doc/forum/Git-annex_init_gets_stuck_on_exFAT.mdwn b/doc/forum/Git-annex_init_gets_stuck_on_exFAT.mdwn
new file mode 100644
index 0000000000..4910d80e92
--- /dev/null
+++ b/doc/forum/Git-annex_init_gets_stuck_on_exFAT.mdwn
@@ -0,0 +1,30 @@
+I  have just started using git annex, and I cloned the annex onto an external usb formatted as exFAT. I am using Mac OS X.
+
+The clone went fine, but when running `git annex init` the following happened
+
+```
+  Detected a filesystem without fifo support.
+
+  Disabling ssh connection caching.
+
+  Detected a crippled filesystem.
+
+  Disabling core.symlinks.
+(merging origin/git-annex into git-annex...)
+(recording state in git...)
+
+  Enabling direct mode.
+```
+then git annex appears to get stuck according to `top`:  
+
+```
+PID    COMMAND      %CPU TIME     #TH   #WQ  #PORT MEM    PURG   CMPRS  PGRP  PPID  STATE    BOOSTS  
+
+86934  git-annex    0.0  01:07.09 4     0    13    4152K  0B     5684K  86933 86933 stuck    *0[1]
+```
+
+I am running git-annex 6.20170101, so direct mode should no longer be used. Is that correct? If so what should I do to figure out what's going on?
+
+ The most relevant related thread I could find was this one [Bare repo on USB ...](http://git-annex.branchable.com/forum/Bare_repo_on_USB_drive_not_providing_files/) which suggests exFAT may not work well on Linux. However, I mainly use this USB on my Macbook to supplement the SSD. Another possibly relevant thread is this one [Android table ... only exFAT...](http://git-annex.branchable.com/forum/Using_git_annex_with_Android_tablet_which_only_has_exFAT_and_no_symlinks/).
+
+I appreciate any assitance.
diff --git a/doc/forum/Git-annex_init_gets_stuck_on_exFAT/comment_1_48da28dbc7da82d4e06fd3ce7962dfc5._comment b/doc/forum/Git-annex_init_gets_stuck_on_exFAT/comment_1_48da28dbc7da82d4e06fd3ce7962dfc5._comment
new file mode 100644
index 0000000000..7e44bab07e
--- /dev/null
+++ b/doc/forum/Git-annex_init_gets_stuck_on_exFAT/comment_1_48da28dbc7da82d4e06fd3ce7962dfc5._comment
@@ -0,0 +1,16 @@
+[[!comment format=mdwn
+ username="joey"
+ subject="""comment 1"""
+ date="2017-01-31T16:42:54Z"
+ content="""
+Despite v6 repo format being available now, git-annex still defaults to v5,
+and so needs to use direct mode when run in a FAT filesystem. If you want
+to use v6 repo mode, you will need to `git annex init --version=6`
+
+Based on some googling, "stuck" state in OSX top means that it's stuck in a
+kernel call. This probably means that git-annex is waiting for a disk
+access, or from output from one of its child processes (eg git commands).
+
+It would be helpful to look at the process tree and see what child
+processes of git-annex are running at this point.
+"""]]
diff --git a/doc/forum/Git-annex_link_to_different_file_names.mdwn b/doc/forum/Git-annex_link_to_different_file_names.mdwn
new file mode 100644
index 0000000000..a7a7c2727c
--- /dev/null
+++ b/doc/forum/Git-annex_link_to_different_file_names.mdwn
@@ -0,0 +1,41 @@
+This is a recreation of a stackexchange question, in case the community here is more knowledgeable.
+
+Link to stackexchange question : http://unix.stackexchange.com/questions/325753/git-annex-link-to-different-file-names
+
+Content :
+"Maybe this is just a crazy use case that doesn't work, but I was wondering if there's a way to build a file's history from files with different file names. I'm exploring this idea because I'd like to have a git-annex system but I can't force my coworkers to adapt.
+
+Here's what I have in mind :
+
+    Folder 1, managed by coworkers (On a shared disk) :
+
+        drawing_shop_12_nov_2015.pdf
+        drawing_shop_13_nov_2015.pdf
+        drawing_asbuilt_14_nov_2015.pdf
+        drawing_asbuilt_rev1_15_nov_2015.pdf
+
+And
+
+    Git-annex, managed by me :
+
+        drawing.pdf
+
+    (with a shop branch and a asbuilt branch)
+
+The git-annex's drawing.pdf would have an history like this :
+
+    [shop]
+    |
+    Commit A "Initial shop drawing"
+    |
+    Commit B "Add corrections from Wizzbasket"
+     \
+      |
+      [asbuilt]
+      Commit C "Reflect as built"
+      |
+      Commit D "Change dweezelbox block for simplicity"
+
+But somehow the "managed by coworkers" repo would be a direct mode repo with Commit A pointing to drawing_shop_12_nov_2015.pdf, Commit B to drawing_shop_13_nov_2015.pdf etc.
+
+Can this be done?"
diff --git a/doc/forum/Git-annex_link_to_different_file_names/comment_1_17ab85276bcf495a656c7091753c086f._comment b/doc/forum/Git-annex_link_to_different_file_names/comment_1_17ab85276bcf495a656c7091753c086f._comment
new file mode 100644
index 0000000000..8085253ad5
--- /dev/null
+++ b/doc/forum/Git-annex_link_to_different_file_names/comment_1_17ab85276bcf495a656c7091753c086f._comment
@@ -0,0 +1,60 @@
+[[!comment format=mdwn
+ username="joey"
+ subject="""comment 1"""
+ date="2016-12-20T19:27:31Z"
+ content="""
+Yes, you can do this. Effectively, you have two branches. In the master
+branch, you have drawing.pdf with a single name and changes commited to it.
+In the coworkers branch, you have the multiple different versions. Git has
+no difficulty representing this, but it's up to you to maintain the
+different branches.
+
+For example:
+
+	joey@darkstar:~/tmp/shop>git commit -m 'updated drawing some more' 
+	[master 1403dd4] updated drawing some more
+	 1 file changed, 1 insertion(+), 1 deletion(-)
+	joey@darkstar:~/tmp/shop>git checkout coworkers 
+	Switched to branch 'coworkers'
+	joey@darkstar:~/tmp/shop#coworkers>git show master
+	commit 1403dd49b2c378e78b8b8ec82d73e295c486697b
+	Author: Joey Hess 
+	Date:   Tue Dec 20 15:31:17 2016 -0400
+	
+	    updated drawing some more
+	
+	diff --git a/drawing.pdf b/drawing.pdf
+	index b59371e..c05ed95 120000
+	--- a/drawing.pdf
+	+++ b/drawing.pdf
+	@@ -1 +1 @@
+	-.git/annex/objects/55/MZ/SHA256E-s13--c5f6529491f9e6d40e893d2ffc008bc297bcc56a680040c124e4019fb5c1a94d.pdf/SHA256E-s13--c5f6529491f9e6d40e893d2ffc008bc297bcc56a680040c124e4019fb5c1a94d.pdf
+	\ No newline at end of file
+	+.git/annex/objects/xj/XF/SHA256E-s17--7786e857a89634ff9242d899245cbcc5e009736af6b0553cb7283b2daef77d16.pdf/SHA256E-s17--7786e857a89634ff9242d899245cbcc5e009736af6b0553cb7283b2daef77d16.pdf
+	\ No newline at end of file
+	joey@darkstar:~/tmp/shop#coworkers>ln -s .git/annex/objects/xj/XF/SHA256E-s17--7786e857a89634ff9242d899245cbcc5e009736af6b0553cb7283b2daef77d16.pdf/SHA256E-s17--7786e857a89634ff9242d899245cbcc5e009736af6b0553cb7283b2daef77d16.pdf drawing_rev2.pdf
+	joey@darkstar:~/tmp/shop#coworkers>git add drawing_rev2.pdf 
+	joey@darkstar:~/tmp/shop#coworkers>ls
+	drawing.pdf@  drawing_rev2.pdf@
+	joey@darkstar:~/tmp/shop#coworkers>git commit -m 'added rev2 of drawing'
+	[coworkers cf27781] added rev2 of drawing
+	 1 file changed, 1 insertion(+)
+	  create mode 120000 drawing_rev2.pdf
+
+In the example, I looked at what was committed to master, and copied and
+pasted the git-annex symlink into a new drawing_rev2.pdf file.
+
+That's the basic idea. There might be a better way to do that. Another way,
+for example, would be to have 2 clones of the repo, one with master checked
+out and one with coworkers checked out. You could then run, in the
+coworkers checkout:
+
+	cp -a ../master/drawing.pdf drawing_rev2.pdf
+	git add drawing_rev2.pdf
+	git commit -m 'added rev2 of drawing'
+
+That results in the same commit as the method I showed.
+
+With some scripting, you should be able to automate keeping the two
+branches in sync.
+"""]]
diff --git a/doc/forum/GitBlit_as_a_centralized_repository.mdwn b/doc/forum/GitBlit_as_a_centralized_repository.mdwn
new file mode 100644
index 0000000000..884392d586
--- /dev/null
+++ b/doc/forum/GitBlit_as_a_centralized_repository.mdwn
@@ -0,0 +1,2 @@
+Is it possible to use Gitblit as a centralized repository using git-annex? 
+thanks Gary
diff --git a/doc/forum/GitBlit_as_a_centralized_repository/comment_1_06bedf5fbac45415e0cc81f41b25a5a0._comment b/doc/forum/GitBlit_as_a_centralized_repository/comment_1_06bedf5fbac45415e0cc81f41b25a5a0._comment
new file mode 100644
index 0000000000..91dac6305a
--- /dev/null
+++ b/doc/forum/GitBlit_as_a_centralized_repository/comment_1_06bedf5fbac45415e0cc81f41b25a5a0._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="http://joeyh.name/"
+ ip="108.236.230.124"
+ subject="comment 1"
+ date="2014-05-16T17:40:20Z"
+ content="""
+I assume gitblit is something like gitosis or gitolite. These let a user ssh in to a server to access a git repository, but they limit the commands they can run to `git-shell`. In order to use git-annex get/copy to such a repository, it needs to be able to run `git-annex-shell`, which is like git-shell but with more commands available. If gitblit can be made allow git-annex-shell to be run, it will work.
+"""]]
diff --git a/doc/forum/Git_Annex_Assistant_Occassionally_Loses_Files_and_Gives_Dangling_Symlink.mdwn b/doc/forum/Git_Annex_Assistant_Occassionally_Loses_Files_and_Gives_Dangling_Symlink.mdwn
new file mode 100644
index 0000000000..18f2e88b8d
--- /dev/null
+++ b/doc/forum/Git_Annex_Assistant_Occassionally_Loses_Files_and_Gives_Dangling_Symlink.mdwn
@@ -0,0 +1,9 @@
+I am using the git annex assistant between two linux machines. Sometimes, it seems to "lose" a file, having a dangling symlink of it. Fsck shows that there are no known copies of the file in the repositories, but if I do a search in .git/annex/objects, I'm able to find the file.
+
+
+I'm using git annex 6.20160923+gitgd1dabb3-1~ndall+1 and direct mode repositories thanks to the assistant.
+
+Any advice would be highly appreciated. Right now, I can't get it to consistently do this behavior, but any help would be appreciated, including what I should post, look for, etc. for more troubleshooting.
+
+Thanks again,
+Ryan
diff --git a/doc/forum/Git_Annex_Assistant_Occassionally_Loses_Files_and_Gives_Dangling_Symlink/comment_1_7b3cb1d52767b89cf46d57005bf57b06._comment b/doc/forum/Git_Annex_Assistant_Occassionally_Loses_Files_and_Gives_Dangling_Symlink/comment_1_7b3cb1d52767b89cf46d57005bf57b06._comment
new file mode 100644
index 0000000000..66d9a24a4e
--- /dev/null
+++ b/doc/forum/Git_Annex_Assistant_Occassionally_Loses_Files_and_Gives_Dangling_Symlink/comment_1_7b3cb1d52767b89cf46d57005bf57b06._comment
@@ -0,0 +1,11 @@
+[[!comment format=mdwn
+ username="joey"
+ subject="""comment 1"""
+ date="2016-10-17T19:39:21Z"
+ content="""
+You say you can find the file in .git/annex/objects. Have you tried
+running `git annex fsck` on the file? If it can also find the file
+there, it should fix the working tree up to have a copy of it.
+
+Direct mode is known to have the odd bug in this area.
+"""]]
diff --git a/doc/forum/Git_Annex_Assistant_Occassionally_Loses_Files_and_Gives_Dangling_Symlink/comment_2_52ad16f63ec47b34485a0d0f4db9e2c8._comment b/doc/forum/Git_Annex_Assistant_Occassionally_Loses_Files_and_Gives_Dangling_Symlink/comment_2_52ad16f63ec47b34485a0d0f4db9e2c8._comment
new file mode 100644
index 0000000000..8255ba778a
--- /dev/null
+++ b/doc/forum/Git_Annex_Assistant_Occassionally_Loses_Files_and_Gives_Dangling_Symlink/comment_2_52ad16f63ec47b34485a0d0f4db9e2c8._comment
@@ -0,0 +1,13 @@
+[[!comment format=mdwn
+ username="ryan@d4f0c2d3daacb5ec3a2945bca06f66decad4bfb5"
+ nickname="ryan"
+ avatar="http://cdn.libravatar.org/avatar/7732d2474bcd13998986a87a9b17273c"
+ subject="re: comment 1"
+ date="2016-10-17T19:54:45Z"
+ content="""
+I did try running fsck, which gave a message about not being able to find the file.
+
+That's too bad that there are these types of bugs in direct mode. I find it really nice to be able to just throw all my stuff in the git annex and not worry about the state and have it synced everywhere.
+
+Thanks again for making such a wonderful tool and helping me troubleshoot.
+"""]]
diff --git a/doc/forum/Git_Annex_Assistant_Occassionally_Loses_Files_and_Gives_Dangling_Symlink/comment_3_a60bc9a6e766c0f29835c088c0863cf5._comment b/doc/forum/Git_Annex_Assistant_Occassionally_Loses_Files_and_Gives_Dangling_Symlink/comment_3_a60bc9a6e766c0f29835c088c0863cf5._comment
new file mode 100644
index 0000000000..64c48602a9
--- /dev/null
+++ b/doc/forum/Git_Annex_Assistant_Occassionally_Loses_Files_and_Gives_Dangling_Symlink/comment_3_a60bc9a6e766c0f29835c088c0863cf5._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="joey"
+ subject="""comment 3"""
+ date="2016-10-17T20:38:45Z"
+ content="""
+If fsck couldn't find the file, but you could, that begs the question:
+What was the filename of the file that you found?
+"""]]
diff --git a/doc/forum/Git_Annex_Assistant_Occassionally_Loses_Files_and_Gives_Dangling_Symlink/comment_4_c8088fd093312685f130af07e5ca2e22._comment b/doc/forum/Git_Annex_Assistant_Occassionally_Loses_Files_and_Gives_Dangling_Symlink/comment_4_c8088fd093312685f130af07e5ca2e22._comment
new file mode 100644
index 0000000000..58b141306e
--- /dev/null
+++ b/doc/forum/Git_Annex_Assistant_Occassionally_Loses_Files_and_Gives_Dangling_Symlink/comment_4_c8088fd093312685f130af07e5ca2e22._comment
@@ -0,0 +1,11 @@
+[[!comment format=mdwn
+ username="ryan@d4f0c2d3daacb5ec3a2945bca06f66decad4bfb5"
+ nickname="ryan"
+ avatar="http://cdn.libravatar.org/avatar/7732d2474bcd13998986a87a9b17273c"
+ subject="fsck filename"
+ date="2016-10-18T04:05:51Z"
+ content="""
+Something like SHA256E-s146706--ee3814f2a8c017d1a8c2af39124f107a8a663ed00a13757f6e328bd652ecba1b.txt in /.git/annex/objects/...
+
+I didn't take very good notes, but it seemed to have the same filename as everything else. I was able to find it by doing an ag (grep) search for a string I knew was in the file.
+"""]]
diff --git a/doc/forum/Git_Annex_Assistant__58___How_to_add_a_remote__63__.mdwn b/doc/forum/Git_Annex_Assistant__58___How_to_add_a_remote__63__.mdwn
new file mode 100644
index 0000000000..0a3aac9116
--- /dev/null
+++ b/doc/forum/Git_Annex_Assistant__58___How_to_add_a_remote__63__.mdwn
@@ -0,0 +1,11 @@
+Hi,
+
+I am trying to use Git Annex Assistant on my Fedora Linux computer. I currently have a local repository that assistant monitors, but now I want to use my personal server as a git remote for the repository. I click on Configuration > Remote server and it prompts me for my server login credentials and then click the "Check this server" button. After this I am shown a screen that assistant is "Ready to add remote server" and then states:
+> The server  can be used as is, but installing git-annex on it would make it work better, and provide more options below.
+>
+> If you're able to install software on the server, do so and click Retry
+
+The information shown below this message is regarding encrypting the data (if I have git-annex installed on the server, which I don't). So my question is, what do I do now? There is no button to just add the remote from within git annex assistant? I created a bare repository on my server and added a remote manually inside the repository on my local machine, but git annex assistant doesn't notice it. I'm clueless on what git annex assistant is expecting from me at this point. I would think what I'm trying to do is one of the most common use cases for git annex, but I can't find any documentation or materials on the correct procedure for this. Any help is appreciated.
+
+Regards,
+Blake
diff --git a/doc/forum/Git_Annex_Assistant__58___How_to_add_a_remote__63__/comment_1_d0a3d0090928790d5a05e9f8e5f05320._comment b/doc/forum/Git_Annex_Assistant__58___How_to_add_a_remote__63__/comment_1_d0a3d0090928790d5a05e9f8e5f05320._comment
new file mode 100644
index 0000000000..5c3a596a5c
--- /dev/null
+++ b/doc/forum/Git_Annex_Assistant__58___How_to_add_a_remote__63__/comment_1_d0a3d0090928790d5a05e9f8e5f05320._comment
@@ -0,0 +1,12 @@
+[[!comment format=mdwn
+ username="http://joeyh.name/"
+ ip="64.134.31.139"
+ subject="comment 1"
+ date="2013-10-21T22:20:53Z"
+ content="""
+You have two choices:
+
+1. Install git-annex on your server. You can then use the webapp to set up a git repository, or use the git repository you've already set up.
+
+2. Don't install git-annex on your server. Select the option in the webapp to use an encrypred rsync repository on your server. (The screen you described the webapp showing had a button to use this option, but you seem to have misread it.)
+"""]]
diff --git a/doc/forum/Git_Annex_Assistant_won__39__t_backup_files_to_removable_drive.mdwn b/doc/forum/Git_Annex_Assistant_won__39__t_backup_files_to_removable_drive.mdwn
new file mode 100644
index 0000000000..20e435fcff
--- /dev/null
+++ b/doc/forum/Git_Annex_Assistant_won__39__t_backup_files_to_removable_drive.mdwn
@@ -0,0 +1,3 @@
+I'm using Git Annex Assistant for the first time, trying to get it to mirror a directory of files from my computer to an external USB drive. I've tried setting up the second repo as a "Removable Drive" in the "full backup" repository group. I've also tried putting it in the "client" group. The result is the same, and Assistant doesn't backup any of my files to the external drive.
+
+What I get in the external drive in the directory that should contain the repo are the set of files "annex, objects, refs, config, HEAD" and they are NOT in a .git subdirectory. Aside from this I don't see any of the files Git Annex Assistant says it has synced. As this is my first time using the Assistant, I don't know if this is a bug, or if I'm just using it incorrectly. If anyone knows what is going on here, please share.
diff --git a/doc/forum/Git_Annex_Assistant_won__39__t_backup_files_to_removable_drive/comment_1_a1d1ae9488924b08682b355aff51130d._comment b/doc/forum/Git_Annex_Assistant_won__39__t_backup_files_to_removable_drive/comment_1_a1d1ae9488924b08682b355aff51130d._comment
new file mode 100644
index 0000000000..31ed0999ef
--- /dev/null
+++ b/doc/forum/Git_Annex_Assistant_won__39__t_backup_files_to_removable_drive/comment_1_a1d1ae9488924b08682b355aff51130d._comment
@@ -0,0 +1,17 @@
+[[!comment format=mdwn
+ username="https://www.google.com/accounts/o8/id?id=AItOawk3HGoDpnOPob5jOjvIootmkve1-nCpRiI"
+ nickname="Kalle"
+ subject="This is the expected default behavior"
+ date="2014-05-17T20:49:06Z"
+ content="""
+Your files should be on the removable drive but the file structure and file names are following git annex logic and is not meant to be human readable.
+
+The default assumption (which I question) is that you will use the removable drive to transfer files between git-annex repositories. So by default you cannot read the files on devices that do not have git-annex installed.
+
+You can configure git-annex to do what you want using the `post-receive` hook. It's a bit complicated and might leave git-annex hogging your drive when you wan't to eject it but see the following:
+
+ - [[forum/USB_backup_with_files_visible/]]
+ - [[forum/Can't_get_git-annex_merge_to_work_from_git_hook/#comment-d93181e69b8bea54bc03cb335dd1d7c8]]
+
+The last post on that second link tells you exactly what to put in the `post-receive` hook in the .git/hooks folder on your removable drive.
+"""]]
diff --git a/doc/forum/Git_Annex_Assistant_won__39__t_backup_files_to_removable_drive/comment_2_6649077583bc14730a08aaaca7ccb62e._comment b/doc/forum/Git_Annex_Assistant_won__39__t_backup_files_to_removable_drive/comment_2_6649077583bc14730a08aaaca7ccb62e._comment
new file mode 100644
index 0000000000..059d446e7b
--- /dev/null
+++ b/doc/forum/Git_Annex_Assistant_won__39__t_backup_files_to_removable_drive/comment_2_6649077583bc14730a08aaaca7ccb62e._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="https://www.google.com/accounts/o8/id?id=AItOawnov5q9_Cl4Ps5NoYE08yE01NLSvBANnY8"
+ nickname="Eric"
+ subject="comment 2"
+ date="2014-05-18T09:35:45Z"
+ content="""
+Okay, I understand now. Probably Assistant should only treat the external drive that way if it's in the \"transfer\" group.
+"""]]
diff --git a/doc/forum/Git_Annex_Assistant_won__39__t_backup_files_to_removable_drive/comment_3_f359d9b9356de7ee10b9e725a011cc43._comment b/doc/forum/Git_Annex_Assistant_won__39__t_backup_files_to_removable_drive/comment_3_f359d9b9356de7ee10b9e725a011cc43._comment
new file mode 100644
index 0000000000..caec25ffd1
--- /dev/null
+++ b/doc/forum/Git_Annex_Assistant_won__39__t_backup_files_to_removable_drive/comment_3_f359d9b9356de7ee10b9e725a011cc43._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="https://www.google.com/accounts/o8/id?id=AItOawnov5q9_Cl4Ps5NoYE08yE01NLSvBANnY8"
+ nickname="Eric"
+ subject="comment 3"
+ date="2014-07-10T05:50:32Z"
+ content="""
+Actually, I understand the logic behind encouraging bare repositories.
+"""]]
diff --git a/doc/forum/Git_Annex_Sync_Delinks_Files.mdwn b/doc/forum/Git_Annex_Sync_Delinks_Files.mdwn
new file mode 100644
index 0000000000..1ce18dd0b9
--- /dev/null
+++ b/doc/forum/Git_Annex_Sync_Delinks_Files.mdwn
@@ -0,0 +1,11 @@
+So, I'm using git-annex to manage a group of external hard drives that I use to store a bunch of big files.
+
+When syncing tonight, though, it decided to take a whole subdirectory and replace the symlinks with the content directly when merging, or something.
+
+The remote side is not like that, and I didn't tell it to do anything like that.
+It just seems to feel like that's the thing to do, then it syncs and fails because suddenly git has many gigs of objects, which is what git-annex is supposed to solve.
+
+I've reset master a bunch of times, and it keeps choosing that path, but I don't know why.
+I've also reset master and synced/master a few times. I'm not fully sure what that one actually encodes, so I didn't want to touch it too much, but setting it to the same thing as master didn't seem to fix things either.
+
+Any idea why?
diff --git a/doc/forum/Git_Annex_Sync_Delinks_Files/comment_1_8b03707777a9d8e38715cb77d2a0addc._comment b/doc/forum/Git_Annex_Sync_Delinks_Files/comment_1_8b03707777a9d8e38715cb77d2a0addc._comment
new file mode 100644
index 0000000000..ad0605f494
--- /dev/null
+++ b/doc/forum/Git_Annex_Sync_Delinks_Files/comment_1_8b03707777a9d8e38715cb77d2a0addc._comment
@@ -0,0 +1,12 @@
+[[!comment format=mdwn
+ username="http://joeyh.name/"
+ ip="209.250.56.227"
+ subject="comment 1"
+ date="2013-12-31T18:06:15Z"
+ content="""
+Are you using direct mode?
+
+Before the direct mode guard was put in place, it was not uncommon for users to mistakenly run some command like \"git commit -a\" in a direct mode repository. This happily checks the full large files into git.
+
+It sounds to me like you've done that. But, you have not provided enough information to make anything more than a vague guess.
+"""]]
diff --git a/doc/forum/Git_Annex_Sync_Delinks_Files/comment_2_a625e7d88e321532ace103548b31b67b._comment b/doc/forum/Git_Annex_Sync_Delinks_Files/comment_2_a625e7d88e321532ace103548b31b67b._comment
new file mode 100644
index 0000000000..9dd931053c
--- /dev/null
+++ b/doc/forum/Git_Annex_Sync_Delinks_Files/comment_2_a625e7d88e321532ace103548b31b67b._comment
@@ -0,0 +1,18 @@
+[[!comment format=mdwn
+ username="https://www.google.com/accounts/o8/id?id=AItOawk_W9oOMDiBaU_xLIbeaTF8wW2wJzJ7bfw"
+ nickname="Christopher"
+ subject="comment 2"
+ date="2013-12-31T18:56:58Z"
+ content="""
+I am not in direct mode.
+I don't believe I did anything in the repo. I reset back to a commit and look in the folder and it's all symlinks and nothing in git status says anything about it.
+
+Then I run `git annex sync` and one of the thing it does is \"Checking out files\" which takes a long time, since it seems to be copying the data into the working directory and commiting it.
+I don't know why it's decided to do that.
+
+What steps can I take to either get you more information or fix things?
+
+Perhaps relatedly, if I make a mistake with git-annex, is git-reset master and git-reset synced/master the right approach?
+Are there other things I should try, etc?
+
+"""]]
diff --git a/doc/forum/Git_Annex_Sync_Delinks_Files/comment_3_5278164dab570755ed58afe466dfad42._comment b/doc/forum/Git_Annex_Sync_Delinks_Files/comment_3_5278164dab570755ed58afe466dfad42._comment
new file mode 100644
index 0000000000..cadf944cd4
--- /dev/null
+++ b/doc/forum/Git_Annex_Sync_Delinks_Files/comment_3_5278164dab570755ed58afe466dfad42._comment
@@ -0,0 +1,12 @@
+[[!comment format=mdwn
+ username="http://joeyh.name/"
+ ip="209.250.56.227"
+ subject="comment 3"
+ date="2013-12-31T19:19:15Z"
+ content="""
+\"Checking out files\" is a message printed by git (not by git-annex) when it is updating the work tree.
+
+It still seems that you have somehow committed large files directly to git. Perhaps you accidentally ran \"git add\" on a large file.
+
+git annex sync is probably merging this commit from one of your other repositories.
+"""]]
diff --git a/doc/forum/Git_Annex_Sync_Delinks_Files/comment_4_e43ede0bdc20de9aa10ab6ce387d8582._comment b/doc/forum/Git_Annex_Sync_Delinks_Files/comment_4_e43ede0bdc20de9aa10ab6ce387d8582._comment
new file mode 100644
index 0000000000..9b16ea2b06
--- /dev/null
+++ b/doc/forum/Git_Annex_Sync_Delinks_Files/comment_4_e43ede0bdc20de9aa10ab6ce387d8582._comment
@@ -0,0 +1,92 @@
+[[!comment format=mdwn
+ username="https://www.google.com/accounts/o8/id?id=AItOawk_W9oOMDiBaU_xLIbeaTF8wW2wJzJ7bfw"
+ nickname="Christopher"
+ subject="comment 4"
+ date="2014-01-01T15:23:17Z"
+ content="""
+I know that checking of files is a git message, I was just providing it for context.
+
+I know that my other repos do not have these things checked in, because one of them is offline and not involved in this, and the other repo literally can't contain that much data, which is the biggest reason this is an issue.
+
+Here are the commits I don't like:
+
+    commit 1ae2410a4bc3e35edbfd9d48a577bcfc7e3a8836
+    Author: Christopher Vollick <0@psycoti.ca>
+    Date:   Mon Dec 30 23:22:47 2013 -0500
+    
+        git-annex automatic sync
+    
+    :120000 100644 185b311... fd03ae5... T	subdir1/other_file0309
+    :120000 100644 515e6df... 44f8383... T	subdir1/other_file0309
+    
+    commit 7403e3116eb34e2b192dfa3ea4c16e35f82210ba
+    Author: Christopher Vollick <0@psycoti.ca>
+    Date:   Sat Dec 21 19:12:56 2013 -0500
+    
+        git-annex automatic sync
+    
+    :120000 100644 185b311... fd03ae5... T	subdir1/other_file0309
+    :120000 100644 515e6df... 357d9a7... T	subdir1/other_file0309
+    :120000 100644 079f69f... 97c785a... T	subdir1/file01
+    :120000 100644 c69574a... 1589721... T	subdir1/file02
+    :120000 100644 01d98a4... 154a059... T	subdir1/file03
+    :120000 100644 6ee02d5... 6857f73... T	subdir1/file04
+    :120000 100644 aab78e7... 19bae04... T	subdir1/file05
+    :120000 100644 9f8d11b... bd5b501... T	subdir1/file06
+    :120000 100644 ca824c1... a9e0b9e... T	subdir1/file07
+    :120000 100644 c85f526... a6383fc... T	subdir1/file08
+    :120000 100644 a6d957d... 9b99c99... T	subdir1/file09
+    :120000 100644 d3de258... d51da9a... T	subdir1/file10
+    :120000 100644 3cc803e... 8f7945f... T	subdir1/other_file0005
+    :120000 100644 cd94cca... 1c06cf8... T	subdir1/other_file0006
+    :120000 100644 1442c98... fee3c1d... T	subdir1/other_file0201
+    :120000 100644 4131025... f732332... T	subdir1/other_file0202
+    :120000 100644 865214f... 1178931... T	subdir1/other_file0203
+    :120000 100644 a0f84e2... 6c39a4a... T	subdir1/other_file0204
+    :120000 100644 21bf0c1... bbb3c93... T	subdir1/other_file0205
+    :120000 100644 5f6a5f3... d298a97... T	subdir1/other_file0206
+    :120000 100644 3d7f034... 4cac0d1... T	subdir1/other_file0207
+    :120000 100644 ee75d44... d769e8e... T	subdir1/other_file0208
+    :120000 100644 f381401... 5f1a7f3... T	subdir1/other_file0209
+    :120000 100644 8b6a409... 1db8c37... T	subdir1/other_file0210
+    :120000 100644 c6a23f1... 5a9bbf5... T	subdir1/other_file0211
+    :120000 100644 17a34c7... 068c619... T	subdir1/other_file0212
+    :120000 100644 3c7fe0a... 356315b... T	subdir1/other_file0213
+    :120000 100644 4b61d45... e9747e5... T	subdir1/other_file0214
+    :120000 100644 1c0d845... f7a11d3... T	subdir1/other_file0301
+    :120000 100644 cdd57b2... fb5512f... T	subdir1/other_file0302
+    :120000 100644 9bb8ffc... 9b3897d... T	subdir1/other_file0303
+    :120000 100644 007cfa6... 93e36d7... T	subdir1/other_file0304
+    :120000 100644 24cf5d3... 9eecff4... T	subdir1/other_file0305
+    :120000 100644 d9be965... 6c612f4... T	subdir1/other_file0306
+    :120000 100644 20cd474... 9d1c714... T	subdir1/other_file0307
+    :120000 100644 3f24532... 9ff7770... T	subdir1/other_file0308
+    :120000 100644 3939cbe... de9fefd... T	subdir1/other_file0308
+    :120000 100644 9de87f9... 2242892... T	subdir1/other_file0309
+    :120000 100644 7250b17... a544af6... T	subdir1/other_file0310
+    :120000 100644 9de87f9... 2242892... T	subdir1/other_file0312
+    :120000 100644 14fbe43... f643564... T	subdir1/other_file0401
+    :120000 100644 0f28f22... e4b5e2c... T	subdir1/other_file0402
+    :120000 100644 6704647... 4529fb0... T	subdir1/other_file0403
+    :120000 100644 65b3742... dacc2ce... T	subdir1/other_file0404
+    :120000 100644 14ad1d6... a4f7dab... T	subdir1/other_file0405
+    :120000 100644 87bda81... a3b2a0a... T	subdir1/other_file0406
+    :120000 100644 23fd37b... 7ea3182... T	subdir1/other_file0407
+    :120000 100644 a82d025... aa0e1f6... T	subdir1/other_file0408
+    :120000 100644 cd350d3... c225b3d... T	subdir1/other_file0409
+    :120000 100644 2ce6c3b... 1efffa6... T	subdir1/other_file0410
+    :120000 100644 3f780d5... 207f8af... T	subdir1/other_file0411
+    :120000 100644 27ecf10... 4a34b2d... T	subdir1/other_file0412
+    :120000 100644 5cf1ba0... ab0b15d... T	subdir1/other_file0413
+    :120000 100644 a1cccd7... 274dbd1... T	subdir1/other_file0004
+
+Now, the most interesting part here is that commit 7403e3116eb is dated Sat Dec 21.
+That is not when I was attempting this. The 30th, shown in 1ae24 is.
+
+That makes me think that I have failed to reset properly, and so when I'm telling it to sync it's not recomputing the thing, it's just grabbing the thing it did last time.
+
+Now, I still don't know why it chose to do that, but perhaps it wouldn't choose to do that again if I could figure out how to get it to forget that path.
+
+Thoughts?
+
+"""]]
diff --git a/doc/forum/Git_Annex_Sync_Delinks_Files/comment_5_dc71987f0e19f04a920561201f9552b4._comment b/doc/forum/Git_Annex_Sync_Delinks_Files/comment_5_dc71987f0e19f04a920561201f9552b4._comment
new file mode 100644
index 0000000000..03e4d2a9bf
--- /dev/null
+++ b/doc/forum/Git_Annex_Sync_Delinks_Files/comment_5_dc71987f0e19f04a920561201f9552b4._comment
@@ -0,0 +1,15 @@
+[[!comment format=mdwn
+ username="https://www.google.com/accounts/o8/id?id=AItOawk_W9oOMDiBaU_xLIbeaTF8wW2wJzJ7bfw"
+ nickname="Christopher"
+ subject="comment 5"
+ date="2014-01-01T16:37:38Z"
+ content="""
+Ok, so, looking into it a little more I think I've found that I screwed up updating a ref, so I didn't reset synced/master like I thought I did.
+
+So, I don't know why it chose to make this choice in the first place, but I was able to reset and manually change my way around it.
+
+That being said, I don't want to do this a lot, because I'm not fully clear on what the synced branches represent.
+Is it that, on repo1, repo2/synced/master is the last thing that pushed over, and that after a sync local synced/master is always master, and before that it's the last sync to anywhere?
+
+How safe is it to screw around with these things?
+"""]]
diff --git a/doc/forum/Git_Annex_Sync_Delinks_Files/comment_6_257a89f81858659c4dac4d116e7cf0a3._comment b/doc/forum/Git_Annex_Sync_Delinks_Files/comment_6_257a89f81858659c4dac4d116e7cf0a3._comment
new file mode 100644
index 0000000000..f293a60308
--- /dev/null
+++ b/doc/forum/Git_Annex_Sync_Delinks_Files/comment_6_257a89f81858659c4dac4d116e7cf0a3._comment
@@ -0,0 +1,32 @@
+[[!comment format=mdwn
+ username="http://joeyh.name/"
+ ip="209.250.56.227"
+ subject="comment 6"
+ date="2014-01-01T20:59:32Z"
+ content="""
+The ` T ` in your `git show --raw` indicates that what was a symlink has been replaced with a regular file, and this has been committed to git. 
+
+It's certainly possible to do that when using git-annex, but you have to go a bit out of your way to so shoot yourself in the foot, because normally the pre-commit hook will detect that, and fix up your commit to not change the type of the file. Something like this:
+
+[[!format sh \"\"\"
+joey@darkstar:~/tmp/r>git annex unlock foo
+unlock foo (copying...) ok
+joey@darkstar:~/tmp/r>git commit -a --no-verify -m oops
+[master 9b63e4e] oops
+ 1 file changed, 1 deletion(-)
+ rewrite foo (100%)
+ mode change 120000 => 100644
+joey@darkstar:~/tmp/r>git show --raw
+commit 9b63e4efad1f229ae7713749e9ac2d0f9c286008
+Author: Joey Hess 
+Date:   Wed Jan 1 16:55:19 2014 -0400
+
+    oops
+
+:120000 100644 ea46194... e69de29... T	foo
+\"\"\"]]
+
+The fact that you seem to have made 2 commits that did this, on the 21st and 30th, makes me wonder if your .git/hooks/pre-commit does not exist, or perhaps you are making some other mistake repeatedly.
+
+The synced/* branches never have any data that is not stored somewhere else (another branch, possibly in the remote repository), so it should always be ok to delete them.
+"""]]
diff --git a/doc/forum/Git_Annex_Transfer_Protocols.mdwn b/doc/forum/Git_Annex_Transfer_Protocols.mdwn
new file mode 100644
index 0000000000..d6a660a2d8
--- /dev/null
+++ b/doc/forum/Git_Annex_Transfer_Protocols.mdwn
@@ -0,0 +1,9 @@
+Hi,
+
+May I know, which processes git-annex is using to move/copy/get file content between repositories?
+Is it using same processes which git uses, Like send-pack, receive-pack.
+I want to use Aspera to move file contents between repositories.
+Is it enough if I customize send-pack, receive-pack of git code to do Aspera file transfer or git-annex uses any other transfer mechanism.
+
+Many Thanks,
+Royal Pinto
diff --git a/doc/forum/Git_Annex_Transfer_Protocols/comment_1_a870ec991078c95a6bb683d6962ab56e._comment b/doc/forum/Git_Annex_Transfer_Protocols/comment_1_a870ec991078c95a6bb683d6962ab56e._comment
new file mode 100644
index 0000000000..31c463470c
--- /dev/null
+++ b/doc/forum/Git_Annex_Transfer_Protocols/comment_1_a870ec991078c95a6bb683d6962ab56e._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="http://joeyh.name/"
+ ip="4.153.2.186"
+ subject="rsync over ssh"
+ date="2012-05-10T18:18:01Z"
+ content="""
+Some other protocols such as S3 for special remotes.
+"""]]
diff --git a/doc/forum/Git_Annex_Transfer_Protocols/comment_2_71419376ef50a679ea8f0f9e16991c17._comment b/doc/forum/Git_Annex_Transfer_Protocols/comment_2_71419376ef50a679ea8f0f9e16991c17._comment
new file mode 100644
index 0000000000..2e07aef088
--- /dev/null
+++ b/doc/forum/Git_Annex_Transfer_Protocols/comment_2_71419376ef50a679ea8f0f9e16991c17._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="https://www.google.com/accounts/o8/id?id=AItOawlYu7QmD7wrbHWkoxuriaA9XcijM-g5vrQ"
+ nickname="Royal"
+ subject="Protocols to transfer file content"
+ date="2012-05-10T18:48:40Z"
+ content="""
+Thanks, Is git annex is using same protocols as normal git to transfer content between normal git repositories?
+"""]]
diff --git a/doc/forum/Git_Annex_Transfer_Protocols/comment_3_fea43664a500111ca99f4043e0dadb14._comment b/doc/forum/Git_Annex_Transfer_Protocols/comment_3_fea43664a500111ca99f4043e0dadb14._comment
new file mode 100644
index 0000000000..6e7b36e311
--- /dev/null
+++ b/doc/forum/Git_Annex_Transfer_Protocols/comment_3_fea43664a500111ca99f4043e0dadb14._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="http://joeyh.name/"
+ ip="4.153.2.186"
+ subject="comment 3"
+ date="2012-05-10T18:51:56Z"
+ content="""
+git-annex doesn't transfer git content between git repositories. You use git for that. Well, git-annex sync can run a few git commands for you to do it.
+"""]]
diff --git a/doc/forum/Git_Annex_Transfer_Protocols/comment_4_56fb2dab1d4030c9820be32b495afdf0._comment b/doc/forum/Git_Annex_Transfer_Protocols/comment_4_56fb2dab1d4030c9820be32b495afdf0._comment
new file mode 100644
index 0000000000..fef10cd813
--- /dev/null
+++ b/doc/forum/Git_Annex_Transfer_Protocols/comment_4_56fb2dab1d4030c9820be32b495afdf0._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="https://www.google.com/accounts/o8/id?id=AItOawlYu7QmD7wrbHWkoxuriaA9XcijM-g5vrQ"
+ nickname="Royal"
+ subject="Git annex content transfer protocols"
+ date="2012-05-10T19:13:48Z"
+ content="""
+Sorry if I am not clear. Actually i meant to ask, if i have 2 git repositories which are not special remotes and I am transferring annexed file content between these repositories using git annex command (move or copy) then, which protocol it uses to transfer content? Is it uses git-send-pack git-recieve-pack or some other protocols. 
+"""]]
diff --git a/doc/forum/Git_Annex_Transfer_Protocols/comment_5_a6ec9c5a4a3c0bac1df87f1df9be140b._comment b/doc/forum/Git_Annex_Transfer_Protocols/comment_5_a6ec9c5a4a3c0bac1df87f1df9be140b._comment
new file mode 100644
index 0000000000..be25737c1d
--- /dev/null
+++ b/doc/forum/Git_Annex_Transfer_Protocols/comment_5_a6ec9c5a4a3c0bac1df87f1df9be140b._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="http://joeyh.name/"
+ ip="4.153.2.186"
+ subject="comment 5"
+ date="2012-05-10T19:17:22Z"
+ content="""
+rsync over ssh is used to transfer file contents between repositories. (You can use the -d option to see the commands git-annex runs.)
+"""]]
diff --git a/doc/forum/Git_Annex_Transfer_Protocols/comment_6_1678452fb7114aeabcf0cc3d5f6c69b0._comment b/doc/forum/Git_Annex_Transfer_Protocols/comment_6_1678452fb7114aeabcf0cc3d5f6c69b0._comment
new file mode 100644
index 0000000000..b7ef8f33c4
--- /dev/null
+++ b/doc/forum/Git_Annex_Transfer_Protocols/comment_6_1678452fb7114aeabcf0cc3d5f6c69b0._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="https://www.google.com/accounts/o8/id?id=AItOawlYu7QmD7wrbHWkoxuriaA9XcijM-g5vrQ"
+ nickname="Royal"
+ subject="comment 6"
+ date="2012-05-10T19:21:08Z"
+ content="""
+Ok. This helped me a lot. Thank you
+"""]]
diff --git a/doc/forum/Git_Annex_not_dropping_unused_content.mdwn b/doc/forum/Git_Annex_not_dropping_unused_content.mdwn
new file mode 100644
index 0000000000..ffdebcdbae
--- /dev/null
+++ b/doc/forum/Git_Annex_not_dropping_unused_content.mdwn
@@ -0,0 +1,9 @@
+I've deleted some files, but their content remains in the objects directory, and *git annex unused* does not list them.
+
+I've read in this post  that if other branches contain the files, then it won't count them as unused. My repo appears to have a few branches left over from views I've used.
+
+    remotes/sdrive/views/(added=14_09);(tag=Shared)
+    remotes/sdrive/views/(added=14_09);Images_=_
+    remotes/sdrive/views/added=_
+
+How can I delete these? Is git annex going to create a new branch for every new view I create?
diff --git a/doc/forum/Git_Annex_not_dropping_unused_content/comment_1_da47b6af512b19cba077499f41455189._comment b/doc/forum/Git_Annex_not_dropping_unused_content/comment_1_da47b6af512b19cba077499f41455189._comment
new file mode 100644
index 0000000000..d21648c9d6
--- /dev/null
+++ b/doc/forum/Git_Annex_not_dropping_unused_content/comment_1_da47b6af512b19cba077499f41455189._comment
@@ -0,0 +1,23 @@
+[[!comment format=mdwn
+ username="ghen1"
+ subject="Solved"
+ date="2014-12-19T12:56:55Z"
+ content="""
+I've figured it out:
+
+In the annex in which the view was made (in my case *sdrive*), the views must be deleted with single-quotes:
+
+    git branch -D 'views/(added=14_09);(tag=Shared)'
+    git branch -D 'views/(added=14_09);Images_=_'
+    git branch -D 'views/added=_'
+
+In the connected annexes, for which *sdrive* is a remote, this command removes the remote branches:
+
+    git fetch -p sdrive
+
+     x [deleted]         (none)     -> sdrive/views/(added=14_09);(tag=Shared)
+     x [deleted]         (none)     -> sdrive/views/(added=14_09);Images_=_
+     x [deleted]         (none)     -> sdrive/views/added=_
+
+After these steps *git annex* recognized old files as unused.
+"""]]
diff --git a/doc/forum/Git_annex__44___bup__44___windows_and_remote_linux_issue..mdwn b/doc/forum/Git_annex__44___bup__44___windows_and_remote_linux_issue..mdwn
new file mode 100644
index 0000000000..ee024eca43
--- /dev/null
+++ b/doc/forum/Git_annex__44___bup__44___windows_and_remote_linux_issue..mdwn
@@ -0,0 +1 @@
+I would like to make some thing. On windows one repo (R1) with files for work. On linux server the other (R2) is for syncing and the third (R3) on linux is for backup with bup. Is it any way to run remote backup command on R2 from R1 but not always when i sync them - only when i send special command? I am looking simple native solution avoiding cygwin or third scripts. Thanks.
diff --git a/doc/forum/Git_annex__44___bup__44___windows_and_remote_linux_issue./comment_1_c9239f9c67b2441ab0e1ac42c53db531._comment b/doc/forum/Git_annex__44___bup__44___windows_and_remote_linux_issue./comment_1_c9239f9c67b2441ab0e1ac42c53db531._comment
new file mode 100644
index 0000000000..3a5bc1c0d8
--- /dev/null
+++ b/doc/forum/Git_annex__44___bup__44___windows_and_remote_linux_issue./comment_1_c9239f9c67b2441ab0e1ac42c53db531._comment
@@ -0,0 +1,9 @@
+[[!comment format=mdwn
+ username="joey"
+ subject="""comment 1"""
+ date="2016-07-12T18:34:13Z"
+ content="""
+Well, you can ssh into the linux box and cd to R2 and run `git annex sync
+---content` and that should cause it to download any new files that it can
+(if it can contact R1), and send them along to R3.
+"""]]
diff --git a/doc/forum/Git_annex___39__corrupting__39___itself.mdwn b/doc/forum/Git_annex___39__corrupting__39___itself.mdwn
new file mode 100644
index 0000000000..fe1edc17dc
--- /dev/null
+++ b/doc/forum/Git_annex___39__corrupting__39___itself.mdwn
@@ -0,0 +1,34 @@
+Hi,
+
+Since the I updated to `2013-10-24` `git annex` has corrupted itself twice on my setup.  I'm not claiming causality here and thus I haven't filled it as a bug (probably the mistake is mine).
+
+I use three laptops and one remote ssh server.  The motherboard on laptop *T* seems to have broken and I haven't fixed it yet.  I have disabled sync to *T*  on the webapp dashboard.  
+
+First, one computer, called *W*, broke on Friday.  I got the dreaded [serious problem](http://git-annex.branchable.com/devblog/day_41__onward/) screen. `git fsck` reported lots of missing blobs and tree problem, though my data seemed OK.  I git cloned the repo from computer *X*.  Now, last night both X and W reported the same problem (log below).  *W*  recovered itself somehow, but *X*  is now in a cycle of re-adding all files in the repo and the posting the below error that has to do with computer *T* (IP ending with 107).  
+
+I would appreciate hints on why this suddenly started happen.  Computer T has been out of sync for almost 14 days (since the 24th of October I think).
+
+    error: refs/remotes/192.168.1.107_annex/git-annex does not point to a valid object!
+    error: refs/remotes/192.168.1.107_annex/master does not point to a valid object!
+    error: refs/remotes/192.168.1.107_annex/synced/git-annex does not point to a valid object!
+    error: refs/remotes/192.168.1.107_annex/synced/master does not point to a valid object!
+    error: refs/synced/2ed58ecf-8e8c-44b8-ad34-d42ddfb35315/cmFzbXVzQGNvZGVyb2xsZXJzLmNvbQ==/git-annex does not point to a valid object!
+    error: refs/synced/2ed58ecf-8e8c-44b8-ad34-d42ddfb35315/cmFzbXVzQGNvZGVyb2xsZXJzLmNvbQ==/master does not point to a valid object!
+    error: refs/synced/f54b8200-8ec1-43d2-8710-b73ec118addc/cmFzbXVzQGNvZGVyb2xsZXJzLmNvbQ==/git-annex does not point to a valid object!
+    error: refs/synced/f54b8200-8ec1-43d2-8710-b73ec118addc/cmFzbXVzQGNvZGVyb2xsZXJzLmNvbQ==/master does not point to a valid object!
+    error: refs/synced/f54b8200-8ec1-43d2-8710-b73ec118addc/git-annex does not point to a valid object!
+    error: refs/synced/f54b8200-8ec1-43d2-8710-b73ec118addc/master does not point to a valid object!
+    error: refs/remotes/192.168.1.107_annex/git-annex does not point to a valid object!
+    error: refs/remotes/192.168.1.107_annex/master does not point to a valid object!
+    error: refs/remotes/192.168.1.107_annex/synced/git-annex does not point to a valid object!
+    error: refs/remotes/192.168.1.107_annex/synced/master does not point to a valid object!
+    error: refs/synced/2ed58ecf-8e8c-44b8-ad34-d42ddfb35315/cmFzbXVzQGNvZGVyb2xsZXJzLmNvbQ==/git-annex does not point to a valid object!
+    error: refs/synced/2ed58ecf-8e8c-44b8-ad34-d42ddfb35315/cmFzbXVzQGNvZGVyb2xsZXJzLmNvbQ==/master does not point to a valid object!
+    error: refs/synced/f54b8200-8ec1-43d2-8710-b73ec118addc/cmFzbXVzQGNvZGVyb2xsZXJzLmNvbQ==/git-annex does not point to a valid object!
+    error: refs/synced/f54b8200-8ec1-43d2-8710-b73ec118addc/cmFzbXVzQGNvZGVyb2xsZXJzLmNvbQ==/master does not point to a valid object!
+    error: refs/synced/f54b8200-8ec1-43d2-8710-b73ec118addc/git-annex does not point to a valid object!
+    error: refs/synced/f54b8200-8ec1-43d2-8710-b73ec118addc/master does not point to a valid object!
+    error: invalid object 100644 1ca66de3cdd9c79cde26a7555cf3b8d26d0e371d for '000/147/SHA256E-s347--1ab8084bf9ae06407ce0a7260a83638ea6e9a028dc59b4815fd60aec61dbd747.txt.log.log'
+    fatal: git-write-tree: error building trees
+    TransferScanner crashed: failed to read sha from git write-tree
+    [2013-11-04 09:45:41 CET] TransferScanner: warning TransferScanner crashed: failed to read sha from git write-tree
diff --git a/doc/forum/Git_annex___39__corrupting__39___itself/comment_1_bcf50a215e2f8771e098aadfff4c300c._comment b/doc/forum/Git_annex___39__corrupting__39___itself/comment_1_bcf50a215e2f8771e098aadfff4c300c._comment
new file mode 100644
index 0000000000..2fe95ab335
--- /dev/null
+++ b/doc/forum/Git_annex___39__corrupting__39___itself/comment_1_bcf50a215e2f8771e098aadfff4c300c._comment
@@ -0,0 +1,43 @@
+[[!comment format=mdwn
+ username="https://www.google.com/accounts/o8/id?id=AItOawlEXAlu8qKv_FHYOv-uEohdGgMY_Lw-Ges"
+ nickname="Jonas"
+ subject="Just happened for me as well."
+ date="2013-11-12T08:10:56Z"
+ content="""
+This was the first search result when I googled the error message, I'm not sure what has happened (or if it is a problem since I don't change my files that often).
+
+git-annex version: 4.20131101
+
+Also running git annex with one ssh remote ( laptop1 & laptop2 synching through a ssh remote server ).
+
+Got a bunch of 
+    error: refs/synced/dbb70cea-4b83-416f-93af-e59082c4d633/git-annex does not point to a valid object!
+    error: refs/synced/dbb70cea-4b83-416f-93af-e59082c4d633/master does not point to a valid object!
+    error: refs/synced/dbb70cea-4b83-416f-93af-e59082c4d633/git-annex does not point to a valid object!
+    error: refs/synced/dbb70cea-4b83-416f-93af-e59082c4d633/master does not point to a valid object!
+when I tried a manual git annex sync, and noticed that I had a bunch of those in the log in the webapp as well... 
+
+The gui doesn't show that anything is wrong, and looking at the log it seems that things are atleast uploading
+
+    [2013-11-12 09:05:46 CET] Committer: Committing changes to git
+    [2013-11-12 09:05:46 CET] Pusher: Syncing with born 
+    error: refs/synced/dbb70cea-4b83-416f-93af-e59082c4d633/git-annex does not point to a valid object!
+    error: refs/synced/dbb70cea-4b83-416f-93af-e59082c4d633/master does not point to a valid object!
+    Everything up-to-date
+
+    FILE
+
+       32768  42%    0.00kB/s    0:00:00  
+       76972 100%   42.16MB/s    0:00:00 (xfer#1, to-check=0/1)
+
+    sent 77056 bytes  received 31 bytes  51391.33 bytes/sec
+    total size is 76972  speedup is 1.00
+    [2013-11-12 09:05:48 CET] Transferrer: Uploaded FILE
+    [2013-11-12 09:05:48 CET] Pusher: Syncing with born 
+    error: refs/synced/dbb70cea-4b83-416f-93af-e59082c4d633/git-annex does not point to a valid object!
+    error: refs/synced/dbb70cea-4b83-416f-93af-e59082c4d633/master does not point to a valid object!
+    error: refs/synced/dbb70cea-4b83-416f-93af-e59082c4d633/git-annex does not point to a valid object!
+    error: refs/synced/dbb70cea-4b83-416f-93af-e59082c4d633/master does not point to a valid object!
+    To ssh://me@born/annex
+      54e04cf..333cf10  git-annex -> synced/git-annex
+"""]]
diff --git a/doc/forum/Git_annex___39__corrupting__39___itself/comment_3_75f957e7be6c1ad8936c0a2a5374db3e._comment b/doc/forum/Git_annex___39__corrupting__39___itself/comment_3_75f957e7be6c1ad8936c0a2a5374db3e._comment
new file mode 100644
index 0000000000..80f778bb16
--- /dev/null
+++ b/doc/forum/Git_annex___39__corrupting__39___itself/comment_3_75f957e7be6c1ad8936c0a2a5374db3e._comment
@@ -0,0 +1,20 @@
+[[!comment format=mdwn
+ username="https://www.google.com/accounts/o8/id?id=AItOawlEXAlu8qKv_FHYOv-uEohdGgMY_Lw-Ges"
+ nickname="Jonas"
+ subject="comment 3"
+ date="2013-11-13T09:03:24Z"
+ content="""
+I did a change of one of the files and that totally broke all other repositories, git fsck reports a lot of missing trees (I'm not sure if it is normal), and git annex fsck started reporting invalid object for another file for some reason, also got a (git annex repair still reports everything is fine though.)
+
+    fatal: git-write-tree: error building trees
+    git-annex: failed to read sha from git write-tree
+
+
+
+I tried following those instructions but I don't think I had a good repository, git annex repair reported everything was ok but git fsck said a lot of trees where missing and it stopped updating.
+I noticed I was running different git version (git version 1.7.9.5 on the server and git version 1.8.4.2 on one of the clients), so maybe that was part of the reason.
+
+I couldn't find out what remote dbb70cea-4b83-416f-93af-e59082c4d633 was supposed to be either, it wasn't any of the active ones and it didn't show up with git annex status either (I've lost a few repositories when I reinstalled my computer).
+
+I ended up recreating the repository (by copying the raw files) from what I think is the most recent version from one of my computers and hope that it was a error that happened due to upgrades.
+"""]]
diff --git a/doc/forum/Git_annex___39__corrupting__39___itself/comment_3_ab062b1df3b55fd49852a6220c98249e._comment b/doc/forum/Git_annex___39__corrupting__39___itself/comment_3_ab062b1df3b55fd49852a6220c98249e._comment
new file mode 100644
index 0000000000..76e9d00de4
--- /dev/null
+++ b/doc/forum/Git_annex___39__corrupting__39___itself/comment_3_ab062b1df3b55fd49852a6220c98249e._comment
@@ -0,0 +1,10 @@
+[[!comment format=mdwn
+ username="http://id.clacke.se/"
+ nickname="clacke"
+ subject="comment 3"
+ date="2013-11-12T17:19:32Z"
+ content="""
+I have been getting this too. Here is what I have done to fix it:
+
+[[tips/recovering_from_a_corrupt_git_repository]]
+"""]]
diff --git a/doc/forum/Git_annex___39__corrupting__39___itself/comment_4_45974f60a81ed2d00b87ffb1a7963c6f._comment b/doc/forum/Git_annex___39__corrupting__39___itself/comment_4_45974f60a81ed2d00b87ffb1a7963c6f._comment
new file mode 100644
index 0000000000..4328f5d244
--- /dev/null
+++ b/doc/forum/Git_annex___39__corrupting__39___itself/comment_4_45974f60a81ed2d00b87ffb1a7963c6f._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="http://joeyh.name/"
+ ip="209.250.56.43"
+ subject="comment 4"
+ date="2014-01-13T20:27:34Z"
+ content="""
+Many problems of this sort can happen if the disk that the git reposotory is on is losing data that's written to it, or is just removed (or the computer crashes) at the wrong time. `git-annex repair` can be run in the repository to recover from many problems like this, and it's getting better all the time.
+"""]]
diff --git a/doc/forum/Git_annex_assistant_can__39__t_find_rsync_nor_git-annex_on_server.mdwn b/doc/forum/Git_annex_assistant_can__39__t_find_rsync_nor_git-annex_on_server.mdwn
new file mode 100644
index 0000000000..66d224b3a0
--- /dev/null
+++ b/doc/forum/Git_annex_assistant_can__39__t_find_rsync_nor_git-annex_on_server.mdwn
@@ -0,0 +1,14 @@
+Hi, I'm trying to setup git annex assistant (my first time).
+When I add the server (in "transfert" mode, if that matters) I get the following error:
+
+    "Neither rsync nor git-annex are installed on the server. Perhaps you should go install them?"
+
+I manually verified that both rsync and git/git-annex are installed and available from PATH in the "annex" account and all seems to be ok.
+
+Can you suggest a way to get a more specific information on the source of the error?
+
+My first guess was that this is due to the fact that rsync and git-annex are installed in "non-standard locations".  My server run NixOS (http://nixos.org) which has a completely different convention about directory hierarchy from traditional linux/unix OS (that is, no /usr/bin /usr/lib etc.).  However, I tried to "cheat" by manually adding symbolic links into a /usr/bin but this didn't work either, so I might be looking in the wrong direction.
+
+Any suggestion appreciated, thank you in advance,
+
+Marco
diff --git a/doc/forum/Git_annex_assistant_can__39__t_find_rsync_nor_git-annex_on_server/comment_1_75c599cc26e7d3645f69173861d4f8be._comment b/doc/forum/Git_annex_assistant_can__39__t_find_rsync_nor_git-annex_on_server/comment_1_75c599cc26e7d3645f69173861d4f8be._comment
new file mode 100644
index 0000000000..20b83372e7
--- /dev/null
+++ b/doc/forum/Git_annex_assistant_can__39__t_find_rsync_nor_git-annex_on_server/comment_1_75c599cc26e7d3645f69173861d4f8be._comment
@@ -0,0 +1,9 @@
+[[!comment format=mdwn
+ username="https://www.google.com/accounts/o8/id?id=AItOawn26A25mnLHRtWAP587-NPwEFKzolmENL4"
+ nickname="Marco"
+ subject="Also with standalone git-annex"
+ date="2014-09-24T14:14:43Z"
+ content="""
+Update: I also tried to install the standalone distribution in the home of the annex user on the server as shown in the video (BTW, nice illustration!), but I get the same error.
+(On the client side I installed the osx app instead.)
+"""]]
diff --git a/doc/forum/Git_annex_assistant_can__39__t_find_rsync_nor_git-annex_on_server/comment_2_496e2f3a61b609ebb28ab55e5c30022b._comment b/doc/forum/Git_annex_assistant_can__39__t_find_rsync_nor_git-annex_on_server/comment_2_496e2f3a61b609ebb28ab55e5c30022b._comment
new file mode 100644
index 0000000000..f0e63837c0
--- /dev/null
+++ b/doc/forum/Git_annex_assistant_can__39__t_find_rsync_nor_git-annex_on_server/comment_2_496e2f3a61b609ebb28ab55e5c30022b._comment
@@ -0,0 +1,12 @@
+[[!comment format=mdwn
+ username="http://joeyh.name/"
+ ip="209.250.56.144"
+ subject="comment 2"
+ date="2014-09-25T15:42:41Z"
+ content="""
+You need to be able to `ssh yourserver which rsync` and have it succeed. That's what git-annex uses to probe if rsync etc is present.
+
+Note that, since that does not start a login shell, bash doesn't source ~/.bash* at all, or even /etc/profile. So none of the ways people add nonstandard directories to PATH will work.
+
+So, use this to check the PATH that is available on the system: `ssh yourserver 'echo $PATH'`
+"""]]
diff --git a/doc/forum/Git_annex_assistant_can__39__t_find_rsync_nor_git-annex_on_server/comment_3_e202a6dd2803f733bd30ec8b6ddbe7bf._comment b/doc/forum/Git_annex_assistant_can__39__t_find_rsync_nor_git-annex_on_server/comment_3_e202a6dd2803f733bd30ec8b6ddbe7bf._comment
new file mode 100644
index 0000000000..412a370a57
--- /dev/null
+++ b/doc/forum/Git_annex_assistant_can__39__t_find_rsync_nor_git-annex_on_server/comment_3_e202a6dd2803f733bd30ec8b6ddbe7bf._comment
@@ -0,0 +1,19 @@
+[[!comment format=mdwn
+ username="pweemeeuw@86491f921da15d6a4dc6e1878fd42750b33f6963"
+ nickname="pweemeeuw"
+ subject="Got Nixos server configured as a ssh remote (FYI)"
+ date="2015-11-30T16:11:26Z"
+ content="""
+It needed some plumbing, this is what I did:
+
+_~/.ssh/rc_:
+
+    PATH=$HOME/bin:$HOME/.nix-profile/bin:/run/current-system/sw/bin
+
+Since _which_ is an alias (and hence still doesn't work for 'ssh foo which rsync'), I added a shell script _$HOME/bin/which_:
+
+    #!/run/current-system/sw/bin/bash
+    type -P $@
+
+Verified to work (syncs both ways) with a cloned repository on arch linux.
+"""]]
diff --git a/doc/forum/Git_annex_assistant_in_command_line.mdwn b/doc/forum/Git_annex_assistant_in_command_line.mdwn
new file mode 100644
index 0000000000..9323d2137f
--- /dev/null
+++ b/doc/forum/Git_annex_assistant_in_command_line.mdwn
@@ -0,0 +1,2 @@
+How can I clone a git annex remote (bare repository) created with the webapp in command line ? My goal is to be able to access the files using an ssh connection.
+
diff --git a/doc/forum/Git_annex_assistant_in_command_line/comment_1_ce05226307ade8db90ada2dbf290bd58._comment b/doc/forum/Git_annex_assistant_in_command_line/comment_1_ce05226307ade8db90ada2dbf290bd58._comment
new file mode 100644
index 0000000000..e02a4298c4
--- /dev/null
+++ b/doc/forum/Git_annex_assistant_in_command_line/comment_1_ce05226307ade8db90ada2dbf290bd58._comment
@@ -0,0 +1,10 @@
+[[!comment format=mdwn
+ username="http://joeyh.name/"
+ ip="4.153.254.222"
+ subject="comment 1"
+ date="2013-07-07T17:16:27Z"
+ content="""
+There is no difference between a repository created using the assistant and one created by hand. So you can just `git clone ssh://server/path` or `git remote add foo  ssh://server/path` and use it at the command line like any other repository.
+
+The assistant interoperates perfectly with the command line; you can switch between them at will at any time.
+"""]]
diff --git a/doc/forum/Git_annex_assistant_on_EC2.mdwn b/doc/forum/Git_annex_assistant_on_EC2.mdwn
new file mode 100644
index 0000000000..27f20ae462
--- /dev/null
+++ b/doc/forum/Git_annex_assistant_on_EC2.mdwn
@@ -0,0 +1,5 @@
+With Amazon EC2, it is possible to take a snapshot of the virtual machine and then bring up multiple instances based on the snapshot. 
+
+If I configure git annex assistant to sync a folder to S3, will it work correctly if I spin up multiple instances, all identical git annex assistant configurations?
+
+My use case is to allow users of my forums to upload attachments with their posts. The attachments are stored locally on disk, but I'm hoping that git annex assistant can help synchroise the attachments across all forum instances (I usually run only 1 instance, but their could be as many as 3 under heavy load).
diff --git a/doc/forum/Git_annex_assistant_on_EC2/comment_1_bbdb4611373117a2176c225378110a05._comment b/doc/forum/Git_annex_assistant_on_EC2/comment_1_bbdb4611373117a2176c225378110a05._comment
new file mode 100644
index 0000000000..cab80f976c
--- /dev/null
+++ b/doc/forum/Git_annex_assistant_on_EC2/comment_1_bbdb4611373117a2176c225378110a05._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="https://www.google.com/accounts/o8/id?id=AItOawmBUR4O9mofxVbpb8JV9mEbVfIYv670uJo"
+ nickname="Justin"
+ subject="comment 1"
+ date="2013-09-06T22:57:05Z"
+ content="""
+If you are using ec2 and s3 why don't you just serve the attachments directly from s3?  Not sure how git-annex adds anything here.
+"""]]
diff --git a/doc/forum/Git_annex_assistant_on_EC2/comment_2_614ed11f7134137d6376d36a61c293f5._comment b/doc/forum/Git_annex_assistant_on_EC2/comment_2_614ed11f7134137d6376d36a61c293f5._comment
new file mode 100644
index 0000000000..dfb629d62b
--- /dev/null
+++ b/doc/forum/Git_annex_assistant_on_EC2/comment_2_614ed11f7134137d6376d36a61c293f5._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="http://joeyh.name/"
+ ip="4.153.251.174"
+ subject="comment 2"
+ date="2013-09-07T17:08:27Z"
+ content="""
+You are going to need to make each repository have a distinct UUID. Otherwise, the assistant will get confused.
+"""]]
diff --git a/doc/forum/Git_annex_hangs.mdwn b/doc/forum/Git_annex_hangs.mdwn
new file mode 100644
index 0000000000..a7bb359054
--- /dev/null
+++ b/doc/forum/Git_annex_hangs.mdwn
@@ -0,0 +1,2 @@
+http://stackoverflow.com/questions/26305691/git-annex-hangs
+Does anyone know what might be causing this?
diff --git a/doc/forum/Git_annex_hangs/comment_10_ce5355b358f1acc63270331f3749fdcd._comment b/doc/forum/Git_annex_hangs/comment_10_ce5355b358f1acc63270331f3749fdcd._comment
new file mode 100644
index 0000000000..e07ab0403c
--- /dev/null
+++ b/doc/forum/Git_annex_hangs/comment_10_ce5355b358f1acc63270331f3749fdcd._comment
@@ -0,0 +1,25 @@
+[[!comment format=mdwn
+ username="mario"
+ avatar="http://cdn.libravatar.org/avatar/4c63b0935789d29210d0bd8cad8d7ac7"
+ subject="p.s."
+ date="2017-05-17T14:34:40Z"
+ content="""
+Actually the situation is even more complex.
+
+\"The server\" where my repo is stored are actually 4 servers for load balancing that share a single file system.
+
+I just found out that on one of the servers the were still some git-annex-shell processes running. Probably the move command with -J10 logged in on different servers; maybe this confused git-annex.
+
+Now, I logged-in on all four servers and killed any remaining processes. Still the problem persists.
+
+    git-annex info --debug
+    [2017-05-17 16:29:07.197350961] read: git [\"--git-dir=.git\",\"--work-tree=.\",\"--literal-pathspecs\",\"show-ref\",\"git-annex\"]
+    [2017-05-17 16:29:07.216435634] process done ExitSuccess
+    [2017-05-17 16:29:07.216539092] read: git [\"--git-dir=.git\",\"--work-tree=.\",\"--literal-pathspecs\",\"show-ref\",\"--hash\",\"refs/heads/git-annex\"]
+    [2017-05-17 16:29:07.2214318] process done ExitSuccess
+    [2017-05-17 16:29:07.2220868] read: git [\"--git-dir=.git\",\"--work-tree=.\",\"--literal-pathspecs\",\"log\",\"refs/heads/git-annex..76cf27fbf3697af8843ddb82f1fffde35aaba0b2\",\"--pretty=%H\",\"-n1\"]
+    [2017-05-17 16:29:07.23077964] process done ExitSuccess
+
+What's interesting is that these killing actions apparently have changed the hash number on which the hang happens (76cf..). What I also did was git-annex sync at another host, this actually seemed to work. Maybe this changed the hash.. I'm not entirely sure..
+
+"""]]
diff --git a/doc/forum/Git_annex_hangs/comment_11_e0b4616c25405fa3c496f7e85800ac43._comment b/doc/forum/Git_annex_hangs/comment_11_e0b4616c25405fa3c496f7e85800ac43._comment
new file mode 100644
index 0000000000..7c11079ec4
--- /dev/null
+++ b/doc/forum/Git_annex_hangs/comment_11_e0b4616c25405fa3c496f7e85800ac43._comment
@@ -0,0 +1,9 @@
+[[!comment format=mdwn
+ username="joey"
+ subject="""comment 11"""
+ date="2017-05-25T17:47:53Z"
+ content="""
+So there is some kind of networked filesystem involved? What kind?
+
+Do you have annex.pidlock enabled in the repository's `git config`?
+"""]]
diff --git a/doc/forum/Git_annex_hangs/comment_12_053666d256e26803b911d68622257cd5._comment b/doc/forum/Git_annex_hangs/comment_12_053666d256e26803b911d68622257cd5._comment
new file mode 100644
index 0000000000..abe99479c5
--- /dev/null
+++ b/doc/forum/Git_annex_hangs/comment_12_053666d256e26803b911d68622257cd5._comment
@@ -0,0 +1,11 @@
+[[!comment format=mdwn
+ username="mario"
+ avatar="http://cdn.libravatar.org/avatar/4c63b0935789d29210d0bd8cad8d7ac7"
+ subject="comment 12"
+ date="2017-05-25T18:25:59Z"
+ content="""
+I checked the file system, it's nfs.
+
+`git config --get annex.pidlock` gives nothing, so I guess it's not enabled. Do I have to enable it, if the repo is on an nfs mount?
+
+"""]]
diff --git a/doc/forum/Git_annex_hangs/comment_13_f28528c9b9766fe8e2d344e5f27ade65._comment b/doc/forum/Git_annex_hangs/comment_13_f28528c9b9766fe8e2d344e5f27ade65._comment
new file mode 100644
index 0000000000..a528672103
--- /dev/null
+++ b/doc/forum/Git_annex_hangs/comment_13_f28528c9b9766fe8e2d344e5f27ade65._comment
@@ -0,0 +1,19 @@
+[[!comment format=mdwn
+ username="mario"
+ avatar="http://cdn.libravatar.org/avatar/4c63b0935789d29210d0bd8cad8d7ac7"
+ subject="Working again"
+ date="2017-05-29T15:19:18Z"
+ content="""
+Hi,
+
+I'm not sure what exactly has changed, but it's working again!
+
+Until now, I was using git-annex version: 6.20170321-gf3dee9d65 from the standalone-amd64.tar.gz. In the meantime I convinced the administrator to install git-annex system wide. She installed git-annex version: 5.20140221 from the packet-system.
+
+Now, the repository works again with both, git-annex 5 and 6.
+
+However, I cannot exactly tell if anything else has changed in the meantime, since I am not the administrator of the system.
+
+Thanks for your help, anyway.
+
+"""]]
diff --git a/doc/forum/Git_annex_hangs/comment_1_e6b854d4625ae3015aea9c5de71a28ef._comment b/doc/forum/Git_annex_hangs/comment_1_e6b854d4625ae3015aea9c5de71a28ef._comment
new file mode 100644
index 0000000000..34df8fb971
--- /dev/null
+++ b/doc/forum/Git_annex_hangs/comment_1_e6b854d4625ae3015aea9c5de71a28ef._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="http://joeyh.name/"
+ ip="209.250.56.54"
+ subject="comment 1"
+ date="2014-10-10T18:51:57Z"
+ content="""
+I have followed up there, but the basic answer is, pass --debug to see what it's doing.
+"""]]
diff --git a/doc/forum/Git_annex_hangs/comment_2_4f848771e60c38321a97361b0d1b33dd._comment b/doc/forum/Git_annex_hangs/comment_2_4f848771e60c38321a97361b0d1b33dd._comment
new file mode 100644
index 0000000000..cc1b23c083
--- /dev/null
+++ b/doc/forum/Git_annex_hangs/comment_2_4f848771e60c38321a97361b0d1b33dd._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="https://www.google.com/accounts/o8/id?id=AItOawktbkKjilg70XC9XBFpIgVhtfLYH-0UMHY"
+ nickname="Tad"
+ subject="comment 2"
+ date="2014-10-10T19:08:49Z"
+ content="""
+--debug is useful. Thanks! I have tried from scratch and added backend after creating file and that worked. I will use --debug in the future
+"""]]
diff --git a/doc/forum/Git_annex_hangs/comment_3_a07abdd1dc21a69ad6be0526edaeffc1._comment b/doc/forum/Git_annex_hangs/comment_3_a07abdd1dc21a69ad6be0526edaeffc1._comment
new file mode 100644
index 0000000000..452f175a58
--- /dev/null
+++ b/doc/forum/Git_annex_hangs/comment_3_a07abdd1dc21a69ad6be0526edaeffc1._comment
@@ -0,0 +1,13 @@
+[[!comment format=mdwn
+ username="http://joeyh.name/"
+ ip="209.250.56.54"
+ subject="comment 3"
+ date="2014-10-12T20:14:36Z"
+ content="""
+Probably the same problem reported here:
+
+
+I guess this is a mismatch between the version of git that git-annex was built with and the version it's using. In particular, git check-attr behavior varies between git older than 1.7.7 and newer, and git-annex picks the version at build time.
+
+So, I think this is a broken Ubuntu PPA, and if that's the current one, the maintainer of the PPA needs to be contacted to update the git-annex to match the git version, or depend on an appropriate git version.
+"""]]
diff --git a/doc/forum/Git_annex_hangs/comment_4_2ba5992c32753ed03ddd5c12264e9acf._comment b/doc/forum/Git_annex_hangs/comment_4_2ba5992c32753ed03ddd5c12264e9acf._comment
new file mode 100644
index 0000000000..d46dea1cb5
--- /dev/null
+++ b/doc/forum/Git_annex_hangs/comment_4_2ba5992c32753ed03ddd5c12264e9acf._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="https://www.google.com/accounts/o8/id?id=AItOawnjssENdbjzvblxOm4Qr8x2C-BdIV_k_y4"
+ nickname="Tadeusz"
+ subject="comment 4"
+ date="2014-10-15T15:25:27Z"
+ content="""
+I will look into PPA. I hope it won't be this slow:) How long would it take for git annex to add/modify/sync files (~100) taking up 100GB or so? Days? Hours? Minutes?
+"""]]
diff --git a/doc/forum/Git_annex_hangs/comment_5_5fd749f92343079b3916a4d32ddf39c7._comment b/doc/forum/Git_annex_hangs/comment_5_5fd749f92343079b3916a4d32ddf39c7._comment
new file mode 100644
index 0000000000..892f2abf2f
--- /dev/null
+++ b/doc/forum/Git_annex_hangs/comment_5_5fd749f92343079b3916a4d32ddf39c7._comment
@@ -0,0 +1,12 @@
+[[!comment format=mdwn
+ username="joey"
+ subject="""comment 5"""
+ date="2014-10-20T15:28:01Z"
+ content="""
+Adding the files will take as long as it takes to read and hash 
+the conents of those files from disk. If that's too slow, `--backend=WORM`
+will bypass the hashing, so it will take seconds.
+
+Time required to sync files depends on the bandwidth to wherever it's
+syncing with, obviously.
+"""]]
diff --git a/doc/forum/Git_annex_hangs/comment_6_4abe6aa966feea4e1881f6a2fbe7a007._comment b/doc/forum/Git_annex_hangs/comment_6_4abe6aa966feea4e1881f6a2fbe7a007._comment
new file mode 100644
index 0000000000..2d84b08e5f
--- /dev/null
+++ b/doc/forum/Git_annex_hangs/comment_6_4abe6aa966feea4e1881f6a2fbe7a007._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="https://me.yahoo.com/a/yx5Y6EI1t.759Jsu63ZWqYclCmpOmxxd.ramtw--#7114a"
+ nickname="Ioanas"
+ subject="Same issue here "
+ date="2015-05-26T17:12:59Z"
+ content="""
+I am also experiencing the same issue. How do I solve this? Re-installing git and git-annex (as some linked GitHub thread suggested) did not help.
+"""]]
diff --git a/doc/forum/Git_annex_hangs/comment_7_f3ea42dc6699275143ba44e7f57e7d5e._comment b/doc/forum/Git_annex_hangs/comment_7_f3ea42dc6699275143ba44e7f57e7d5e._comment
new file mode 100644
index 0000000000..336a4ee057
--- /dev/null
+++ b/doc/forum/Git_annex_hangs/comment_7_f3ea42dc6699275143ba44e7f57e7d5e._comment
@@ -0,0 +1,15 @@
+[[!comment format=mdwn
+ username="parmour@1304ead3ef9008fa9fe66af7c8678dec867328da"
+ nickname="parmour"
+ subject="Similar sounding issue"
+ date="2016-01-28T15:02:31Z"
+ content="""
+I have seen a similar sounding version on Centos 6.5. I had git version 2.6.4 and installed git annex using:
+wget http://download.fedoraproject.org/pub/epel/6/x86_64/epel-release-6-8.noarch.rpm
+rpm -ivh epel-release-6-8.noarch.rpm
+yum install git-annex
+
+git annex would hang when running 'git annex add'
+
+When I changed the git version to be 1.7.1 this problem went away. So it is something to do with the combination of client git version and git annex version.
+"""]]
diff --git a/doc/forum/Git_annex_hangs/comment_8_1bad17001364d192e0d9affe5f0ea554._comment b/doc/forum/Git_annex_hangs/comment_8_1bad17001364d192e0d9affe5f0ea554._comment
new file mode 100644
index 0000000000..e0d360f371
--- /dev/null
+++ b/doc/forum/Git_annex_hangs/comment_8_1bad17001364d192e0d9affe5f0ea554._comment
@@ -0,0 +1,12 @@
+[[!comment format=mdwn
+ username="joey"
+ subject="""comment 8"""
+ date="2016-01-29T17:47:02Z"
+ content="""
+This was fixed in git-annex back in January 2015. It now checks the git
+version at runtime, not assuming it will be used with as recent a version
+of git as was available at compile time.
+
+So if you're still seeing this problem, upgrading git-annex or git is the only
+solution.
+"""]]
diff --git a/doc/forum/Git_annex_hangs/comment_9_07ae4582a4403b4f0185dbc1830642ad._comment b/doc/forum/Git_annex_hangs/comment_9_07ae4582a4403b4f0185dbc1830642ad._comment
new file mode 100644
index 0000000000..55de021f06
--- /dev/null
+++ b/doc/forum/Git_annex_hangs/comment_9_07ae4582a4403b4f0185dbc1830642ad._comment
@@ -0,0 +1,54 @@
+[[!comment format=mdwn
+ username="mario"
+ avatar="http://cdn.libravatar.org/avatar/4c63b0935789d29210d0bd8cad8d7ac7"
+ subject="Hanging problem still there"
+ date="2017-05-17T14:05:41Z"
+ content="""
+Hi,
+
+it seems my git-annex repository is also affected by the \"hanging\" bug.
+
+Any operation of git-annex just hangs and doesn't do anything. Problem occurs with: sync, whereis, fsck (and with everything I tested).
+
+    git-annex whereis file --debug
+    [2017-05-17 15:57:34.895221948] read: git [\"--git-dir=.git\",\"--work-tree=.\",\"--literal-pathspecs\",\"show-ref\",\"git-annex\"]
+    [2017-05-17 15:57:34.901449143] process done ExitSuccess
+    [2017-05-17 15:57:34.901536719] read: git [\"--git-dir=.git\",\"--work-tree=.\",\"--literal-pathspecs\",\"show-ref\",\"--hash\",\"refs/heads/git-annex\"]
+    [2017-05-17 15:57:34.905894352] process done ExitSuccess
+    [2017-05-17 15:57:34.906485313] read: git [\"--git-dir=.git\",\"--work-tree=.\",\"--literal-pathspecs\",\"log\",\"refs/heads/git-annex..6ccd8cbac5fffa4763294ed88cb1ebc0920c3812\",\"--pretty=%H\",\"-n1\"]
+    [2017-05-17 15:57:34.917489743] process done ExitSuccess
+
+Then it just hangs..
+
+    git-annex sync --debug
+    [2017-05-17 15:58:26.366522677] read: git [\"--git-dir=.git\",\"--work-tree=.\",\"--literal-pathspecs\",\"show-ref\",\"git-annex\"]
+    [2017-05-17 15:58:26.371989858] process done ExitSuccess
+    [2017-05-17 15:58:26.372083975] read: git [\"--git-dir=.git\",\"--work-tree=.\",\"--literal-pathspecs\",\"show-ref\",\"--hash\",\"refs/heads/git-annex\"]
+    [2017-05-17 15:58:26.384517659] process done ExitSuccess
+    [2017-05-17 15:58:26.385061668] read: git [\"--git-dir=.git\",\"--work-tree=.\",\"--literal-pathspecs\",\"log\",\"refs/heads/git-annex..6ccd8cbac5fffa4763294ed88cb1ebc0920c3812\",\"--pretty=%H\",\"-n1\"]
+    [2017-05-17 15:58:26.390054368] process done ExitSuccess
+
+Also here, hangs..
+
+I \"installed\" git-annex from the pre-compiled binaries (i.e., copied it somewhere and put a simlink to git-annex and git-annex-shell into $HOME/bin).
+
+Versions are:
+
+    git --version
+    git version 1.8.3.1
+
+    git-annex version
+    git-annex version: 6.20170321-gf3dee9d65
+    build flags: Assistant Webapp Pairing Testsuite S3(multipartupload)(storageclasses) WebDAV Inotify DBus DesktopNotify ConcurrentOutput TorrentParser MagicMime Feeds Quvi
+    key/value backends: SHA256E SHA256 SHA512E SHA512 SHA224E SHA224 SHA384E SHA384 SHA3_256E SHA3_256 SHA3_512E SHA3_512 SHA3_224E SHA3_224 SHA3_384E SHA3_384 SKEIN256E SKEIN256 SKEIN512E SKEIN512 SHA1E SHA1 MD5E MD5 WORM URL
+    remote types: git gcrypt p2p S3 bup directory rsync web bittorrent webdav tahoe glacier ddar hook external
+    local repository version: 5
+    supported repository versions: 3 5 6
+    upgrade supported from repository versions: 0 1 2 3 4 5
+    operating system: linux x86_64
+
+The setup was working quite fine for a while and now the problem just appeared. Actually, right before I noticed the problem I did git-annex move [...] -J10 from another machine to this repo but it didn't seem to work correctly, so I pressed CTRL-C..
+
+I cannot reboot the server, since I don't have root access.
+
+"""]]
diff --git a/doc/forum/Git_annex_on_Windows.mdwn b/doc/forum/Git_annex_on_Windows.mdwn
new file mode 100644
index 0000000000..b5ca80bee0
--- /dev/null
+++ b/doc/forum/Git_annex_on_Windows.mdwn
@@ -0,0 +1,11 @@
+Hi,
+I want to download some files in git and git annex in Windows OS.
+I have installed msysgit for git-scm in D:\Git directory,
+Now I download git-annex assistant for Windows, and install it
+in D:\Git\cmd directory.
+But when I run the command "git annex" in git-bash, it says it can not
+found the "git annex" command.
+My question is that, how should I use git annex in Windows to download files?
+For example, git annex get .
+Doea anyone has any idear to help me figure it out?
+Thanks in advance!
diff --git a/doc/forum/Git_annex_on_Windows/comment_1_da24ba0219a164f9ab93fe75dd85127e._comment b/doc/forum/Git_annex_on_Windows/comment_1_da24ba0219a164f9ab93fe75dd85127e._comment
new file mode 100644
index 0000000000..be50ece070
--- /dev/null
+++ b/doc/forum/Git_annex_on_Windows/comment_1_da24ba0219a164f9ab93fe75dd85127e._comment
@@ -0,0 +1,11 @@
+[[!comment format=mdwn
+ username="http://joeyh.name/"
+ ip="4.153.251.174"
+ subject="comment 1"
+ date="2013-09-07T17:00:03Z"
+ content="""
+Are you able to run git commands? IIRC msysgit prompts when it's installed about whether you want to put it in the path or not. git-annex just piggybacks on this path setting. It is installed in the same folder as the git.exe and should work as long as that is in path.
+Otherwise, consult windows documentation to add git-annex to the path, or you can copy the git-annex.exe to somewhere else I suppose.
+
+I built a clean Windows system yesterday and have just re-tested it and it does work.
+"""]]
diff --git a/doc/forum/Git_annex_on_Windows/comment_2_c0880ce3ee13d388ab5b46a740170845._comment b/doc/forum/Git_annex_on_Windows/comment_2_c0880ce3ee13d388ab5b46a740170845._comment
new file mode 100644
index 0000000000..bdf30b3b7e
--- /dev/null
+++ b/doc/forum/Git_annex_on_Windows/comment_2_c0880ce3ee13d388ab5b46a740170845._comment
@@ -0,0 +1,50 @@
+[[!comment format=mdwn
+ username="https://www.google.com/accounts/o8/id?id=AItOawkQrsXK-icnYXg6kJNd-jDjMgOixnhwE34"
+ nickname="x"
+ subject="reply to Comment 1"
+ date="2013-09-10T13:46:45Z"
+ content="""
+Thanks for your helpful comment.I add the path to windows, and the
+problem disappears now.
+But I meet another problem when I download the data using git annex.
+I use the command \"git annex get foo.nc\" to download data. The foo.nc
+is actually symlink. For example:
+foo.nc -> ../../.git/annex/objects/jK/v7/WORM-s16009120-m1337754158--
+foo.nc/WORM-s16009120-m1337754158--foo.nc
+Could you help me to figure it out?
+Thank you.
+
+The erro information when using git annex in Windows, is as below:
+
+$ git annex get tau.nc
+ 
+   Detected a crippled filesystem.
+
+   Enabling direct mode.
+
+   Detected a filesystem without fifo support.
+
+   Disabling ssh connection caching.
+
+*** Please tell me who you are.
+ 
+ 
+
+Run 
+
+    git config --global user.email \"you@example.com\"
+
+    git config --user.name \"Your Name\"
+ 
+to set your account's default identity.
+
+Omit --global to set the identity only in this repository.
+
+ 
+
+fatal: unable to auto-detect email address (got 'xshao@DELL-PC.(none')
+ 
+git-annex :tau.nc not found
+
+(Recording state in git ...)
+"""]]
diff --git a/doc/forum/Git_annex_on_Windows/comment_3_70c22716fde60d14fd0c7e74acf4a224._comment b/doc/forum/Git_annex_on_Windows/comment_3_70c22716fde60d14fd0c7e74acf4a224._comment
new file mode 100644
index 0000000000..772d9348a0
--- /dev/null
+++ b/doc/forum/Git_annex_on_Windows/comment_3_70c22716fde60d14fd0c7e74acf4a224._comment
@@ -0,0 +1,11 @@
+[[!comment format=mdwn
+ username="https://www.google.com/accounts/o8/id?id=AItOawkQrsXK-icnYXg6kJNd-jDjMgOixnhwE34"
+ nickname="x"
+ subject="Comment3"
+ date="2013-09-10T15:36:57Z"
+ content="""
+In addition, because the file is actually symlink direct to the data stored in Amazon S3
+How could I get the URL, given the symlink for git annex. For example:
+foo.nc -> ../../.git/annex/objects/jK/v7/WORM-s16009120-m1337754158--foo.nc/WORM-s16009120-m1337754158--foo.nc
+Thank you.
+"""]]
diff --git a/doc/forum/Git_annex_on_Windows/comment_4_b9232deab6bc5036d7339aa202013218._comment b/doc/forum/Git_annex_on_Windows/comment_4_b9232deab6bc5036d7339aa202013218._comment
new file mode 100644
index 0000000000..6fd2f8c3d2
--- /dev/null
+++ b/doc/forum/Git_annex_on_Windows/comment_4_b9232deab6bc5036d7339aa202013218._comment
@@ -0,0 +1,12 @@
+[[!comment format=mdwn
+ username="http://joeyh.name/"
+ ip="4.154.4.51"
+ subject="comment 4"
+ date="2013-09-12T20:58:30Z"
+ content="""
+Once `git annex get` has downloaded the file, it will replace the symlink with the contents of the file.
+
+If that is not happening, you might try running `git annex fsck`
+
+
+"""]]
diff --git a/doc/forum/Git_annex_on_Windows/comment_5_27af3c431b50b540d2bd1d3af3f21080._comment b/doc/forum/Git_annex_on_Windows/comment_5_27af3c431b50b540d2bd1d3af3f21080._comment
new file mode 100644
index 0000000000..5db96f08ed
--- /dev/null
+++ b/doc/forum/Git_annex_on_Windows/comment_5_27af3c431b50b540d2bd1d3af3f21080._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="http://joeyh.name/"
+ ip="4.154.2.134"
+ subject="comment 5"
+ date="2013-09-13T18:03:17Z"
+ content="""
+Turns out that there was a bug in the Windows port that could lead to the behavior described, and another bug that prevented `git annex fsck` from fixing it. I have fixed both bugs, so you should upgrade and then run `git annex fsck`.
+"""]]
diff --git a/doc/forum/Git_annex_on_Windows/comment_6_79fb5ec1b47593ab3355543c5499284a._comment b/doc/forum/Git_annex_on_Windows/comment_6_79fb5ec1b47593ab3355543c5499284a._comment
new file mode 100644
index 0000000000..c3f2ca2a81
--- /dev/null
+++ b/doc/forum/Git_annex_on_Windows/comment_6_79fb5ec1b47593ab3355543c5499284a._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="http://alerque.com/"
+ nickname="Caleb"
+ subject="path issue"
+ date="2014-03-20T13:05:34Z"
+ content="""
+It seems the use of the cmd folder in the Windows git-annex installer no longer meshes with the git install from msysgit. Changing \"cmd\" to \"bin\" during the install process made it work for me.
+"""]]
diff --git a/doc/forum/Git_annex_on_Windows/comment_7_75d4450b4608ad0b453bc69159e708de._comment b/doc/forum/Git_annex_on_Windows/comment_7_75d4450b4608ad0b453bc69159e708de._comment
new file mode 100644
index 0000000000..2d0b3ea3ce
--- /dev/null
+++ b/doc/forum/Git_annex_on_Windows/comment_7_75d4450b4608ad0b453bc69159e708de._comment
@@ -0,0 +1,10 @@
+[[!comment format=mdwn
+ username="http://joeyh.name/"
+ ip="209.250.56.102"
+ subject="comment 7"
+ date="2014-03-20T16:04:36Z"
+ content="""
+Hmm, putting git-annex.exe in to bin and not cmd with mysysgit 1.8.5.2 will make \"git annex\" work, but \"git-annex\" will not.
+
+What version of msysgit has the problem with bin?
+"""]]
diff --git a/doc/forum/Git_annex_on_Windows/comment_8_e4e5ad0cda34bb597fe1bb804acc15e9._comment b/doc/forum/Git_annex_on_Windows/comment_8_e4e5ad0cda34bb597fe1bb804acc15e9._comment
new file mode 100644
index 0000000000..3a649c5013
--- /dev/null
+++ b/doc/forum/Git_annex_on_Windows/comment_8_e4e5ad0cda34bb597fe1bb804acc15e9._comment
@@ -0,0 +1,10 @@
+[[!comment format=mdwn
+ username="http://joeyh.name/"
+ ip="209.250.56.102"
+ subject="comment 8"
+ date="2014-03-20T19:50:26Z"
+ content="""
+I upgraded to msysgit 1.9.0 and git annex works in cmd. 
+
+Did you tell the msysgit installer to \"Run git from the windows command prompt\"? This is the default. If you chose \"Use git bash only\" then git-annex will only work from within git bash.
+"""]]
diff --git a/doc/forum/Git_annex_status_always_showing_annexed_files_as_modified.mdwn b/doc/forum/Git_annex_status_always_showing_annexed_files_as_modified.mdwn
new file mode 100644
index 0000000000..fc4c3ed037
--- /dev/null
+++ b/doc/forum/Git_annex_status_always_showing_annexed_files_as_modified.mdwn
@@ -0,0 +1,33 @@
+Hi,
+
+Whenever I sync files with windows, it shows the every binary file as modified. I've just cloned "fresh" repo from a linux host and ```git annex get```ed the 15-11-03 folder.
+Running git annex status 15-11-03 shows
+
+```
+M ..\15-11-03/ADL Update 11-03-15.pptx
+M ..\15-11-03/Gaming-Rage-Face.jpg
+M ..\15-11-03/eluted-absorption-spectra.png
+M ..\15-11-03/elution.png
+M ..\15-11-03/prep-diagram.png
+M ..\15-11-03/rc-elution-01 (2015 Nov 03).csv
+M ..\15-11-03/rc-elution-02 (2015 Nov 03).csv
+M ..\15-11-03/rc-elution-03 (2015 Nov 03).csv
+M ..\15-11-03/rc-prep-01.h5
+M ..\15-11-03/rc-test-1 (2015 Nov 03).csv
+M ..\15-11-03/rc-test-2 (2015 Nov 03).csv
+M ..\15-11-03/wash-absorption.png
+```
+
+Edit: I should add that this repo is untrusted. Doing sync on it just ends up pulling from remotes, not pushing anywhere:
+
+```
+λ git annex sync
+commit  (recording state in git...)
+ok
+pull origin
+ok
+pull github
+ok
+```
+
+How would I check if this is a problem with my installation or a bug in git annex?
diff --git a/doc/forum/Git_annex_status_always_showing_annexed_files_as_modified/comment_1_b6e8f634187d1f5840641633cb82301a._comment b/doc/forum/Git_annex_status_always_showing_annexed_files_as_modified/comment_1_b6e8f634187d1f5840641633cb82301a._comment
new file mode 100644
index 0000000000..46b259e712
--- /dev/null
+++ b/doc/forum/Git_annex_status_always_showing_annexed_files_as_modified/comment_1_b6e8f634187d1f5840641633cb82301a._comment
@@ -0,0 +1,7 @@
+[[!comment format=mdwn
+ username="aloukian"
+ subject="comment 1"
+ date="2015-11-06T17:33:32Z"
+ content="""
+In particular, I have set up Gpg4Win to use a smart card for ssh-ing. It provides its own sha executables that are on the path and I wonder if that is the reason that the files always show up as modified.
+"""]]
diff --git a/doc/forum/Git_annex_status_always_showing_annexed_files_as_modified/comment_2_ab24dacf101a1a7e33e5976f72f6fe05._comment b/doc/forum/Git_annex_status_always_showing_annexed_files_as_modified/comment_2_ab24dacf101a1a7e33e5976f72f6fe05._comment
new file mode 100644
index 0000000000..6b4cdd13c2
--- /dev/null
+++ b/doc/forum/Git_annex_status_always_showing_annexed_files_as_modified/comment_2_ab24dacf101a1a7e33e5976f72f6fe05._comment
@@ -0,0 +1,25 @@
+[[!comment format=mdwn
+ username="joey"
+ subject="""comment 2"""
+ date="2015-11-10T19:31:18Z"
+ content="""
+git annex status doesn't checksum files, so it can't be the sha executables
+in PATH.
+
+Instead, git-annex status, when in a direct mode repository like on
+windows, checks the recorded timestamps etc of the files to determine if
+they're modified.
+
+I suppose one possibility is a time zone change or DST change. Did you
+recently "fall back" from DST in your location?
+
+Due to the braindead way Windows deals with timezone changes, 
+the timestamps of files appear to *change* when the timezone has changed.
+git-annex tries to detect this situation and offset to get back a stable
+timestamp that can be used to detect file modifications, but I can't say
+that I've tested this situation very carefully, beyond the one very painful
+day implementing that workaround.
+
+Anyway, running `git annex sync` may result in it noticing that nothing has
+actually changed and get the status display to work better.
+"""]]
diff --git a/doc/forum/Git_annex_status_always_showing_annexed_files_as_modified/comment_3_cade18a9487c583329249c2d3e72ca46._comment b/doc/forum/Git_annex_status_always_showing_annexed_files_as_modified/comment_3_cade18a9487c583329249c2d3e72ca46._comment
new file mode 100644
index 0000000000..8a8fc64e78
--- /dev/null
+++ b/doc/forum/Git_annex_status_always_showing_annexed_files_as_modified/comment_3_cade18a9487c583329249c2d3e72ca46._comment
@@ -0,0 +1,7 @@
+[[!comment format=mdwn
+ username="aloukian"
+ subject="comment 3"
+ date="2015-11-10T20:35:58Z"
+ content="""
+Well, sync'ing didn't help, though I didn't do sync --content. I'll try that and maybe it will work.
+"""]]
diff --git a/doc/forum/Git_annex_status_always_showing_annexed_files_as_modified/comment_4_1d2281f79cd0c2a1a3599cae42df0187._comment b/doc/forum/Git_annex_status_always_showing_annexed_files_as_modified/comment_4_1d2281f79cd0c2a1a3599cae42df0187._comment
new file mode 100644
index 0000000000..0683f39585
--- /dev/null
+++ b/doc/forum/Git_annex_status_always_showing_annexed_files_as_modified/comment_4_1d2281f79cd0c2a1a3599cae42df0187._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="neocryptek@659edac901ffbc8e541a974f8f18987eeafc63bd"
+ nickname="neocryptek"
+ subject="comment 4"
+ date="2015-11-21T17:52:26Z"
+ content="""
+I've been having the same problem on windows direct repository and now that I think about it, it could have started after daylight savings time (I also had the windows ssh upgrade problems that took a few days to figure out around the same time). No matter what I do I can't get it to stop showing up as modified in the status view. I've tried sync, sync --content, adding them again, etc. My status output is getting huge as everything I've added on windows is showing up there as modified now. I'm not sure if it's affecting the performance of status or sync (status is just generally very slow on windows direct repos with increasing number of files).
+"""]]
diff --git a/doc/forum/Git_annex_status_always_showing_annexed_files_as_modified/comment_5_5f2f2275ee9de47273671a8009217fb9._comment b/doc/forum/Git_annex_status_always_showing_annexed_files_as_modified/comment_5_5f2f2275ee9de47273671a8009217fb9._comment
new file mode 100644
index 0000000000..b266f442f7
--- /dev/null
+++ b/doc/forum/Git_annex_status_always_showing_annexed_files_as_modified/comment_5_5f2f2275ee9de47273671a8009217fb9._comment
@@ -0,0 +1,10 @@
+[[!comment format=mdwn
+ username="neocryptek@659edac901ffbc8e541a974f8f18987eeafc63bd"
+ nickname="neocryptek"
+ subject="comment 5"
+ date="2015-12-18T18:09:54Z"
+ content="""
+Aloukian were you able to get anything to help? Nothing I've tried has helped.
+
+Joey any more ideas (besides ditching windows :) )?
+"""]]
diff --git a/doc/forum/Git_annex_status_always_showing_annexed_files_as_modified/comment_6_54155343eef8c6f95574dffb3910fc61._comment b/doc/forum/Git_annex_status_always_showing_annexed_files_as_modified/comment_6_54155343eef8c6f95574dffb3910fc61._comment
new file mode 100644
index 0000000000..6f179bcc1e
--- /dev/null
+++ b/doc/forum/Git_annex_status_always_showing_annexed_files_as_modified/comment_6_54155343eef8c6f95574dffb3910fc61._comment
@@ -0,0 +1,46 @@
+[[!comment format=mdwn
+ username="joey"
+ subject="""comment 6"""
+ date="2015-12-19T16:30:55Z"
+ content="""
+Sorry I haven't had a chance to think about this problem any more.
+The v6 repository format I've been working on should eliminate class problem
+anyway. It will let git itself be able to tell if annexed files are modified
+or not.
+
+But there is something you can try to debug what's going on.
+
+0. Run the git bash shell and cd to your repository's directory.
+1. `cat .git/annex/sentinal.cache` and paste its content.
+2. `stat .git/annex/sentinal --terse` and paste the output of that.
+2. Pick one of the files wrongly showing as modified. Run `stat $file --terse`
+  and paste the output of that.
+3. And then this command should output the inode cache for the file.
+   Be sure to replace $file with the name of the file:
+   `cat $(git annex find $file --format='.git/annex/objects/${hashdirlower}${key}/${key}.cache`)
+
+But hmm, as I was running windows to get these instructions, I seem to have
+reproduced the problem myself! In my case:
+
+* The sentinal.cache contained data matching the stat of the sentinal file.
+* The annexed file's mtime and size (and even inode) matched the cached
+  values.
+* `git annex status` showed the file as modifed; `git annex sync` found
+  nothing to commit and didn't change that.
+
+Some more debugging and.. It seems this is not a horrible windows-specific
+time zone problem. Thank goodness. Instead, what's going on is that `git -c
+core.bare=false status` does not show these files as typechanged, but as
+modified instead. Since `git annex status` only has special case handling
+for typechanged files, it just passes the M through and displays it.
+
+So, this is only a display problem, and thus nothing to worry about really.
+Ie, the rest of git-annex's behavior should not be impacted at all.
+
+It's not windows specific.. Same happens on FAT on Linux. I think git's
+behavior probably changed since an earlier version; I'm pretty sure its
+status showed typechanged before. Anyway, I've fixed the status display,
+on these systems it will now treat files git says are modified the same
+as typechanged, and so will use git-annex's inode cache info to diplay
+an accurate status for them.
+"""]]
diff --git a/doc/forum/Git_annex_status_always_showing_annexed_files_as_modified/comment_7_cbf7035b218566a9fa1cb656290a9cf0._comment b/doc/forum/Git_annex_status_always_showing_annexed_files_as_modified/comment_7_cbf7035b218566a9fa1cb656290a9cf0._comment
new file mode 100644
index 0000000000..25ab25c1e3
--- /dev/null
+++ b/doc/forum/Git_annex_status_always_showing_annexed_files_as_modified/comment_7_cbf7035b218566a9fa1cb656290a9cf0._comment
@@ -0,0 +1,9 @@
+[[!comment format=mdwn
+ username="neocryptek"
+ subject="comment 7"
+ date="2016-01-21T03:38:56Z"
+ content="""
+version 6.20160114 fixes the problem on windows and status seems to be faster now (probably due to all the overhead of printing text to the console).
+
+Thanks!
+"""]]
diff --git a/doc/forum/Git_annex_syncing_speed__44___possible__63__.mdwn b/doc/forum/Git_annex_syncing_speed__44___possible__63__.mdwn
new file mode 100644
index 0000000000..d1fcc74aaa
--- /dev/null
+++ b/doc/forum/Git_annex_syncing_speed__44___possible__63__.mdwn
@@ -0,0 +1,21 @@
+Hello
+
+I'm trying Git annex latest builds available in Linux & Android
+Here is the scenario :-
+
+Syncing 2 peer devices running Linux & Android
+Devices can access each other directly over Cisco VPN encrypted link
+
+Currently, syncing files via rsync over webdav (over Cisco VPN)
+Everything is working, manual file tracking is an issue.
+
+Here's where Git annex steps in,
+I've configures external jabber account so they can talk to each other.
+
+But the transfers are super-slow over internet,
+I'm guessing Cisco VPN adding 15-20% overhead + SSH adding another 20-30% overhead.
+
+Since the network link is trusted, is there any way to speed things up assuming Cisco solution
+remains in place?
+
+looking to solve this issue
diff --git a/doc/forum/Git_annex_syncing_speed__44___possible__63__/comment_1_8aa224b3016dc38e4cea8ee1865a3ab6._comment b/doc/forum/Git_annex_syncing_speed__44___possible__63__/comment_1_8aa224b3016dc38e4cea8ee1865a3ab6._comment
new file mode 100644
index 0000000000..eb0e65f801
--- /dev/null
+++ b/doc/forum/Git_annex_syncing_speed__44___possible__63__/comment_1_8aa224b3016dc38e4cea8ee1865a3ab6._comment
@@ -0,0 +1,12 @@
+[[!comment format=mdwn
+ username="https://www.google.com/accounts/o8/id?id=AItOawmBUR4O9mofxVbpb8JV9mEbVfIYv670uJo"
+ nickname="Justin"
+ subject="comment 1"
+ date="2013-07-12T20:09:32Z"
+ content="""
+Not sure if it applies to your devices, but I considerably speed up SSH transfers on my raspberry pi by using
+
+    Ciphers arcfour
+
+in the ssh configuration.
+"""]]
diff --git a/doc/forum/Git_remote__63_____40__bitbucket__44___github__41__.mdwn b/doc/forum/Git_remote__63_____40__bitbucket__44___github__41__.mdwn
new file mode 100644
index 0000000000..245b298ff4
--- /dev/null
+++ b/doc/forum/Git_remote__63_____40__bitbucket__44___github__41__.mdwn
@@ -0,0 +1 @@
+Is it possible to use bitbucket as a remote?
diff --git a/doc/forum/Git_remote__63_____40__bitbucket__44___github__41__/comment_1_8a6de753ac0aa56f470b2aefca628388._comment b/doc/forum/Git_remote__63_____40__bitbucket__44___github__41__/comment_1_8a6de753ac0aa56f470b2aefca628388._comment
new file mode 100644
index 0000000000..c6c7f6e1c8
--- /dev/null
+++ b/doc/forum/Git_remote__63_____40__bitbucket__44___github__41__/comment_1_8a6de753ac0aa56f470b2aefca628388._comment
@@ -0,0 +1,10 @@
+[[!comment format=mdwn
+ username="http://joeyh.name/"
+ ip="108.236.230.124"
+ subject="comment 1"
+ date="2014-09-18T17:43:44Z"
+ content="""
+You can use any git repository as a git remote in git-annex, since git-annex uses plain old git repos.
+
+You will need to add some kind of [[special_remote|special_remotes]] to hold the content of the files stored in git-annex however.
+"""]]
diff --git a/doc/forum/Git_repos_in_git_annex__63__.mdwn b/doc/forum/Git_repos_in_git_annex__63__.mdwn
new file mode 100644
index 0000000000..2d16a66592
--- /dev/null
+++ b/doc/forum/Git_repos_in_git_annex__63__.mdwn
@@ -0,0 +1,7 @@
+Hello,
+
+A mode of operation that I find comfortable, and from various online posts it seems that so do some others, is putting whole git repositories in dropbox.
+
+It seems that currently this mode of operations if not supported in a simple manner. (i.e. just add and let assistant do the work). Is there any workaround or possibility to do this?
+
+Thanks!
diff --git a/doc/forum/Git_repos_in_git_annex__63__/comment_1_8aaa0d83e8fcd5997f6b0097f3b21622._comment b/doc/forum/Git_repos_in_git_annex__63__/comment_1_8aaa0d83e8fcd5997f6b0097f3b21622._comment
new file mode 100644
index 0000000000..fac5ad5b11
--- /dev/null
+++ b/doc/forum/Git_repos_in_git_annex__63__/comment_1_8aaa0d83e8fcd5997f6b0097f3b21622._comment
@@ -0,0 +1,14 @@
+[[!comment format=mdwn
+ username="EvanDeaubl"
+ ip="68.230.56.163"
+ subject="comment 1"
+ date="2013-07-28T02:40:24Z"
+ content="""
+I'll give this warning upfront: this works for me, but it requires manual intervention, and I wouldn't be surprised if this spectacularly broke with the assistant running in full auto mode.
+
+I'm currently storing git repos inside of my annex by using a separated git dir, so that git doesn't detect the nested repository and balk at adding it. Doing a git clone or git init with a --separate-git-dir= argument puts what would normally be in .git in that directory, and creates a simple .git file in its place which references the separated git dir.
+
+When I'm not using the repo, I move the .git file out of the directory, and everything looks like regular files to git-annex. When I want to use it, I move it back in place, and any git operations inside the repo directory use the inner git repo itself.
+
+Another option I used for a while was to store bare repos in the annex, and doing checkouts from those repos.
+"""]]
diff --git a/doc/forum/Git_repos_in_git_annex__63__/comment_2_8546341a561a5f55216c2f437f8ec0c2._comment b/doc/forum/Git_repos_in_git_annex__63__/comment_2_8546341a561a5f55216c2f437f8ec0c2._comment
new file mode 100644
index 0000000000..90d40d8aa6
--- /dev/null
+++ b/doc/forum/Git_repos_in_git_annex__63__/comment_2_8546341a561a5f55216c2f437f8ec0c2._comment
@@ -0,0 +1,32 @@
+[[!comment format=mdwn
+ username="https://www.google.com/accounts/o8/id?id=AItOawmNu4V5fvpLlBhaCUfXXOB0MI5NXwh8SkU"
+ nickname="Adam"
+ subject="Many months later..."
+ date="2014-05-08T16:49:26Z"
+ content="""
+Here is the relevant bug: 
+
+Here's my scenario: I have been using Dropbox for several years.  I want to move to git-annex.
+
+For some time I used Bazaar repos inside Dropbox.  A couple of times I accidentally modified files on one computer before another finished syncing, and the repo became mildly corrupted, but I recovered it okay.
+
+Recently I switched to git, and so far, so good.  I'm careful not to make any changes to my files before Dropbox is finished syncing, so everything works fine.
+
+But when I add a directory containing a git repo to an annex, git-annex ignores the .git directory, so the repo isn't synced.
+
+I read this forum thread, and I read the bug report, but there don't appear to be any real solutions.
+
+This seems like what should be a fairly common use case: a user has a directory of simple text files, like shell scripts or org-mode files.  He uses git to version-control them, and he uses Dropbox to sync the files and the git repo automatically between computers.  Now the user wants to use git-annex instead, but if he does this, it will mean having to choose between:
+
+* No longer using git to version-control the files
+* Only using git on a single system, which means that he can't make commits or do any other git operations when using other systems
+* Keeping separate git repos on each system, and making the same commits manually on each one...which would be a mess
+* Keeping separate git repos on each system, and using a script to make commits automatically--which would mean commits wouldn't be any more useful than a simple timed backup script
+
+I guess I could just use git instead of Dropbox or git-annex, but then I must choose between either:
+
+* Only being able to sync repos when both systems are online
+* Having to store repos in plaintext on remote servers
+
+This is one of my primary use cases for Dropbox/git-annex.  I'd really appreciate any help in figuring this out.  Thanks.
+"""]]
diff --git a/doc/forum/Git_repos_in_git_annex__63__/comment_3_88e37911a0280d817559b2d51ddb1d4e._comment b/doc/forum/Git_repos_in_git_annex__63__/comment_3_88e37911a0280d817559b2d51ddb1d4e._comment
new file mode 100644
index 0000000000..efd86d8f8f
--- /dev/null
+++ b/doc/forum/Git_repos_in_git_annex__63__/comment_3_88e37911a0280d817559b2d51ddb1d4e._comment
@@ -0,0 +1,10 @@
+[[!comment format=mdwn
+ username="memeplex"
+ avatar="http://cdn.libravatar.org/avatar/84a611000e819ef825421de06c9bca90"
+ subject="comment 3"
+ date="2017-04-14T22:35:49Z"
+ content="""
+Another use case is when you use annex for backup. For example I keep most of my files (dotfiles, scripts, music, books, etc) in a \"home\" git annex repo. Part of them is stored in google drive, part in a pendrive, part in my home computer, part in my work computer. Every once in a while I sync everything into my home computer and into an additional external hard drive remote that I keep as a backup. Sadly, my archived git projects can't be managed like that. Nevertheless, it's not a big deal since they are already in the cloud (gitlab or github) and in my local filesystem. Besides, since they are archived, I can just create tarballs and add them to the annex (and in case annex allowed to store .git directories, I'd not be really comfortable with the huge amount of symlinks that would produce).
+
+That said, it's kind of a bathos that the illusion of a distributed, decentralized, redundant, versioned, annotated, etc. filesystem over git is broken by git itself.   
+"""]]
diff --git a/doc/forum/Git_repositories_in_the_annex__63__.mdwn b/doc/forum/Git_repositories_in_the_annex__63__.mdwn
new file mode 100644
index 0000000000..a18af8c2c7
--- /dev/null
+++ b/doc/forum/Git_repositories_in_the_annex__63__.mdwn
@@ -0,0 +1,5 @@
+First of all thanks for the great work. git-annex looks very nice already!
+
+I wonder if it's possible to add Git repositories to the annex. In my current Dropbox setup I drop some small Git repositories into the Dropbox to keep them in sync on two computers. 
+
+This is obviously not possible with git-annex, as the annex itself is a Git repository and sub-repositories (directories with .git folders) are ignored. But in some cases people might want to be able to drop "anything" into the annex, maybe even Git repositories (e.g. the "nomad" usecase)...
diff --git a/doc/forum/Github_pull_request_with_git-annex.mdwn b/doc/forum/Github_pull_request_with_git-annex.mdwn
new file mode 100644
index 0000000000..b0ae1da5d3
--- /dev/null
+++ b/doc/forum/Github_pull_request_with_git-annex.mdwn
@@ -0,0 +1,11 @@
+I would like to use git annex with github. Here what I did:
+
+1. Create github directory "a". Added all the special remote and git annex files. Pushed them to the "a" using "git annex sync."
+2. Fork the repository on gihub to "b". Clone it and enable remote and I could pull annexed files.
+3. Added repository "a" as "upstream" tracking remote to "b".
+3. Created a second special remote for "b". Added new git annex files to it. Used "git annex sync origin" to push the changed files to the forked repository "b".
+4. Now I would like to create a pull request. What are branches I should create pull request for?  a:synced/git-annex vs b:synced/git-annex? Or, a:synced/git-annex vs b:git-annex? Do I need to create separate pull request for the "master"?
+
+Thanks for the help.
+
+ 
diff --git a/doc/forum/Github_pull_request_with_git-annex/comment_1_03023d2fba47e84d03573d996e8366cf._comment b/doc/forum/Github_pull_request_with_git-annex/comment_1_03023d2fba47e84d03573d996e8366cf._comment
new file mode 100644
index 0000000000..ef165fe94f
--- /dev/null
+++ b/doc/forum/Github_pull_request_with_git-annex/comment_1_03023d2fba47e84d03573d996e8366cf._comment
@@ -0,0 +1,22 @@
+[[!comment format=mdwn
+ username="joey"
+ subject="""comment 1"""
+ date="2015-08-19T18:17:17Z"
+ content="""
+Your pull request should be for the branch you changed. In this case, the
+`master` branch. This is the same as adding files to a regular git
+repository and making a pull request for that.
+
+The changes that git-annex makes to the `git-annex branch` can either be
+ignored by the person who merges your pull request, or they could opt to
+fetch your git-annex branch and let git-annex auto-merge it for them. A
+manual review and pull request of the `git-annex` branch is not likely to be
+productive. Whether they need to merge your `git-annex` branch into their
+`git-annex` branch depends on whether git-annex's information about eg,
+locations of file contents needs to be passed along with your pull
+request.
+
+Note that GitHub does not support using git-annex to upload the content of
+annexed files to it. (GitLab does support this.) This GitHub limitation
+may cause problems with this kind of workflow.
+"""]]
diff --git a/doc/forum/Gitolite_problems.mdwn b/doc/forum/Gitolite_problems.mdwn
new file mode 100644
index 0000000000..15b7d62698
--- /dev/null
+++ b/doc/forum/Gitolite_problems.mdwn
@@ -0,0 +1,11 @@
+I am having some issues trying to make git-annex work with the latest version of Gitolite, 
+When running git annex sync, or git annex copy --to I get.
+"""
+Failed to get annex.uuid configuration of repository origin
+
+Instead, got: "annex.uuid=\ncore.gcrypt-id=\n"
+"""
+
+Has anyone found any solution to this? Also, I am using the latest version of Gitolite, because I wanted to see whether it would work.
+Thanks, Zack.
+EDIT: It might also be worthy to note that I did have a go at following the guide on this site.
diff --git a/doc/forum/Gitolite_problems/comment_1_3a41f9b6bddc060b1fa9e35b9ce8b55f._comment b/doc/forum/Gitolite_problems/comment_1_3a41f9b6bddc060b1fa9e35b9ce8b55f._comment
new file mode 100644
index 0000000000..b29fdf2a3e
--- /dev/null
+++ b/doc/forum/Gitolite_problems/comment_1_3a41f9b6bddc060b1fa9e35b9ce8b55f._comment
@@ -0,0 +1,10 @@
+[[!comment format=mdwn
+ username="Renaud"
+ ip="126.10.66.235"
+ subject="comment 1"
+ date="2014-01-11T14:29:08Z"
+ content="""
+I had that problem when I tried to access a newly created remote repository.
+It seems git-annex doesn't recognize anymore if the remote repository has been initialized or not.
+The workaround I used is create a local repository, initialize it with git annex, `clone --bare` it, add the bare repository as a remote and `git annex sync` it to initialize the bare repository. Then, you can put the bare repository where gitolite usualy keep its repositories and use it normally.
+"""]]
diff --git a/doc/forum/Gitolite_problems/comment_2_ef156bf7a1e17496c5fc1f592d45f2ad._comment b/doc/forum/Gitolite_problems/comment_2_ef156bf7a1e17496c5fc1f592d45f2ad._comment
new file mode 100644
index 0000000000..7c4e8662f4
--- /dev/null
+++ b/doc/forum/Gitolite_problems/comment_2_ef156bf7a1e17496c5fc1f592d45f2ad._comment
@@ -0,0 +1,9 @@
+[[!comment format=mdwn
+ username="https://www.google.com/accounts/o8/id?id=AItOawmqaNwDQ367zpW6cIRviLz6zJZZFODgoEI"
+ nickname="Zack"
+ subject="Almost there!"
+ date="2014-01-12T19:13:29Z"
+ content="""
+Thanks for the help (I can now clone the repo.), but I now get this.
+get file FATAL: bad git-annex-shell command: git-annex-shell 'configlist' '/annex2.git' at /opt/gitolite2/bin/commands/git-annex-shell line 25,  line 1.
+"""]]
diff --git a/doc/forum/Handling_a_large_number_of_files.mdwn b/doc/forum/Handling_a_large_number_of_files.mdwn
new file mode 100644
index 0000000000..ccf8360a55
--- /dev/null
+++ b/doc/forum/Handling_a_large_number_of_files.mdwn
@@ -0,0 +1,3 @@
+I have noticed performance getting really slow when adding files (git annex add . ) to a directory already containing several hundred thousand files. When using git annex, is it more recommended to split large numbers of files into multiple directories containing fewer files? Is there a particular recommended way of handling large numbers of files (say getting into the millions) in git annex? 
+
+Thanks
diff --git a/doc/forum/Handling_a_large_number_of_files/comment_1_33d2e19fbfdf5a0bfb7b4fbc840824a2._comment b/doc/forum/Handling_a_large_number_of_files/comment_1_33d2e19fbfdf5a0bfb7b4fbc840824a2._comment
new file mode 100644
index 0000000000..055f69f26f
--- /dev/null
+++ b/doc/forum/Handling_a_large_number_of_files/comment_1_33d2e19fbfdf5a0bfb7b4fbc840824a2._comment
@@ -0,0 +1,13 @@
+[[!comment format=mdwn
+ username="CandyAngel"
+ subject="comment 1"
+ date="2015-06-17T07:51:53Z"
+ content="""
+I've posted a few tips on this site about dealing with large (file count) repositories.. I should probably collate them into a single post in tips (and I shall go so!).
+
+In terms of directories containing hundreds of thousands of files, my research indicates that the slowness stems from how the filesystem (at least, for ext3) fetches the listing, as it requires multiple \"trips\" to query it (it only returns 32KB of metadata at a time or something like that).
+
+So you can really speed it up by partitioning your files in such a way that a directory listing returns quickly. Personally, I use fpart and 5000 files per directory, but I think you can go up to about 20000.
+
+If you do 'find', and it takes a really long time to *start* printing paths, you have too many files :)
+"""]]
diff --git a/doc/forum/Handling_a_large_number_of_files/comment_2_a46bde96e3be5a105eb1c87bf637f8b6._comment b/doc/forum/Handling_a_large_number_of_files/comment_2_a46bde96e3be5a105eb1c87bf637f8b6._comment
new file mode 100644
index 0000000000..73d7668525
--- /dev/null
+++ b/doc/forum/Handling_a_large_number_of_files/comment_2_a46bde96e3be5a105eb1c87bf637f8b6._comment
@@ -0,0 +1,7 @@
+[[!comment format=mdwn
+ username="CandyAngel"
+ subject="comment 2"
+ date="2015-06-17T08:30:55Z"
+ content="""
+Here we go, I started a [tips page](/tips/Repositories_with_large_number_of_files) for this sort of usage :)
+"""]]
diff --git a/doc/forum/Handling_a_large_number_of_files/comment_3_c6564d8cfc217c5b70b2a7a8fe60039b._comment b/doc/forum/Handling_a_large_number_of_files/comment_3_c6564d8cfc217c5b70b2a7a8fe60039b._comment
new file mode 100644
index 0000000000..e3fe246363
--- /dev/null
+++ b/doc/forum/Handling_a_large_number_of_files/comment_3_c6564d8cfc217c5b70b2a7a8fe60039b._comment
@@ -0,0 +1,9 @@
+[[!comment format=mdwn
+ username="anarcat"
+ subject="possible large directory scaling..."
+ date="2015-06-17T12:36:36Z"
+ content="""
+awesome tips page, thanks!
+
+i was wondering ... i seem to recall newer changes to ext4 that made it more scalable with large directories, the `dir_index` flag i think. did you try that and, if so, what was the effect? --[[anarcat]]
+"""]]
diff --git a/doc/forum/Handling_a_large_number_of_files/comment_4_f9e25d63417d0354de158c47235cb3be._comment b/doc/forum/Handling_a_large_number_of_files/comment_4_f9e25d63417d0354de158c47235cb3be._comment
new file mode 100644
index 0000000000..5995243685
--- /dev/null
+++ b/doc/forum/Handling_a_large_number_of_files/comment_4_f9e25d63417d0354de158c47235cb3be._comment
@@ -0,0 +1,20 @@
+[[!comment format=mdwn
+ username="CandyAngel"
+ subject="comment 4"
+ date="2015-06-18T14:03:08Z"
+ content="""
+From what I have gleaned about dir_index, it speeds up finding particular files in a directory, but slows down enumeration of directory contents. It could very well end up as 6 of one, half a dozen of another as git-annex does a bit of both.
+
+So yeah, I've benchmarked this a little (just with a loopback ext3 partitions, dropping caches each time, so *caveat emptor*) and my findings match the above.
+
+Without dir_index, enumerating 102400 files takes 56% more time than 125 files (0.69 seconds vs 1.08 seconds), but with dir_index, it takes 715% more time (0.6 vs. 4.86). However, the initial creation of the files was much faster with dir_index than without (though I wasn't timing that bit).
+
+The results I've gotten so far show that (with dir_index):
+min time: Increases after 64000 files
+max time: Increases after 128000 files
+mean time: Increases after 16000 files
+
+Pretty easy to see where the 20000 files number may have come from.
+
+I'll post the numbers after I have run this longer one (100 iterations) in case anyone is interested. My conclusion is that dir_index being enabled and 16000 files per directory is optimal.
+"""]]
diff --git a/doc/forum/Handling_web_special_remote_when_content_changes__63__.mdwn b/doc/forum/Handling_web_special_remote_when_content_changes__63__.mdwn
new file mode 100644
index 0000000000..5b7be71575
--- /dev/null
+++ b/doc/forum/Handling_web_special_remote_when_content_changes__63__.mdwn
@@ -0,0 +1,19 @@
+I am in the process of organising all of my documents (pdf/ps/etc.) and putting them into an annex.  It seems like the easiest (and smartest?) way for me to manage my 15,000+ document library...  (Yeah, I've not read them all!)
+
+However, one of the issues I have run into is when I want to include something like a software manual...
+
+I'm not even sure if git-annex is the correct way to handle these sort of document, but...
+
+What would you do if the URL stays the same, but the file content changes over time?
+
+For example, the administration manual for R gets updated and re-released for each release of R, but the URL stays the same.
+
+    http://cran.r-project.org/doc/manuals/R-admin.pdf
+
+I'm not particularly worried about whether my annex version is the most recent, just that if I want to 'annex get' it, it will pull *a* version from somewhere.
+
+Any thoughts?
+
+I'd bet that the SHA-hash would change between releases(!) so would a WORM backend be the best approach?  It would mess up my one-file-in-multiple-directories (i.e. multiple simlinks to the one SHA-...-.....) approach.
+
+ 
diff --git a/doc/forum/Handling_web_special_remote_when_content_changes__63__/comment_1_05ee6a1b1943ef3c90634e52233bde1c._comment b/doc/forum/Handling_web_special_remote_when_content_changes__63__/comment_1_05ee6a1b1943ef3c90634e52233bde1c._comment
new file mode 100644
index 0000000000..0f0bf2f327
--- /dev/null
+++ b/doc/forum/Handling_web_special_remote_when_content_changes__63__/comment_1_05ee6a1b1943ef3c90634e52233bde1c._comment
@@ -0,0 +1,12 @@
+[[!comment format=mdwn
+ username="http://joey.kitenet.net/"
+ nickname="joey"
+ subject="comment 1"
+ date="2012-01-03T00:57:55Z"
+ content="""
+The web special remote will happily download files when you `git annex get` even if they don't have the same content that they did before.
+
+`git annex fsck` will detect such mismatched content to the best ability of the backend (so checking the SHA1, or verifying the file size at least matches if you use WORM), and complain and move such mismatched content aside. `git annex addurl --fast` deserves a special mention; it uses a backend that only records the URL, and so if it's used, fsck cannot later detect such changes. Which might be what you want..
+
+For most users, this is one of the reasons `git annex untrust web` is a recommended configuration. Once you untrust the web, any content you download from the web will be kept around in one of your own git-annex repositories, rather than the untrustworthy web being the old copy.
+"""]]
diff --git a/doc/forum/Handling_web_special_remote_when_content_changes__63__/comment_2_48d82e391812d8ec0d4e6562d0607fe7._comment b/doc/forum/Handling_web_special_remote_when_content_changes__63__/comment_2_48d82e391812d8ec0d4e6562d0607fe7._comment
new file mode 100644
index 0000000000..93c43137d3
--- /dev/null
+++ b/doc/forum/Handling_web_special_remote_when_content_changes__63__/comment_2_48d82e391812d8ec0d4e6562d0607fe7._comment
@@ -0,0 +1,10 @@
+[[!comment format=mdwn
+ username="http://a-or-b.myopenid.com/"
+ ip="203.45.2.230"
+ subject="comment 2"
+ date="2012-01-03T02:49:18Z"
+ content="""
+Thanks!  `git annex addurl --fast` does exactly what I want it to do.
+
+Wow.  Yet another special backend for me to play with.  :-)
+"""]]
diff --git a/doc/forum/Hashes_instead_of_content_in_files.mdwn b/doc/forum/Hashes_instead_of_content_in_files.mdwn
new file mode 100644
index 0000000000..86614cf40e
--- /dev/null
+++ b/doc/forum/Hashes_instead_of_content_in_files.mdwn
@@ -0,0 +1 @@
+I just cloned from a bare repo. The files I got aren't symlinks, but when I look inside them, they're just text SHA numbers, not the real content of the files. How can I get the real content?
diff --git a/doc/forum/Hashes_instead_of_content_in_files/comment_1_800d875c0d980614792207641cedb3ea._comment b/doc/forum/Hashes_instead_of_content_in_files/comment_1_800d875c0d980614792207641cedb3ea._comment
new file mode 100644
index 0000000000..23ce342860
--- /dev/null
+++ b/doc/forum/Hashes_instead_of_content_in_files/comment_1_800d875c0d980614792207641cedb3ea._comment
@@ -0,0 +1,11 @@
+[[!comment format=mdwn
+ username="joey"
+ subject="""comment 1"""
+ date="2016-09-15T04:53:16Z"
+ content="""
+Have you tried `git annex get`?
+
+The most common way this can happen is if using Windows or a FAT
+filesystem, or something else that does not support symbolic links.
+It also happens when using v6 repository mode with unlocked files.
+"""]]
diff --git a/doc/forum/Hashes_instead_of_content_in_files/comment_2_a111296eecf7d69ff30f6a82e9f1e6d4._comment b/doc/forum/Hashes_instead_of_content_in_files/comment_2_a111296eecf7d69ff30f6a82e9f1e6d4._comment
new file mode 100644
index 0000000000..d9f4d5d0dc
--- /dev/null
+++ b/doc/forum/Hashes_instead_of_content_in_files/comment_2_a111296eecf7d69ff30f6a82e9f1e6d4._comment
@@ -0,0 +1,7 @@
+[[!comment format=mdwn
+ username="Adam000"
+ subject="comment 2"
+ date="2016-09-15T15:57:59Z"
+ content="""
+Linux boxes. The repository is v6 and has files unlocked. I like to have my files unlocked so I can edit them. Is there a way to keep them unlocked, but transfer the real file content? Or, edit, then, as a batch operation, lock, sync, unlock?
+"""]]
diff --git a/doc/forum/Hashes_instead_of_content_in_files/comment_3_574964b83563df3afea9049754b1cfa2._comment b/doc/forum/Hashes_instead_of_content_in_files/comment_3_574964b83563df3afea9049754b1cfa2._comment
new file mode 100644
index 0000000000..9f2b988f19
--- /dev/null
+++ b/doc/forum/Hashes_instead_of_content_in_files/comment_3_574964b83563df3afea9049754b1cfa2._comment
@@ -0,0 +1,7 @@
+[[!comment format=mdwn
+ username="Adam000"
+ subject="comment 3"
+ date="2016-09-15T16:03:42Z"
+ content="""
+Locking the files in the first repo did the trick. Ideally a solution is available permitting transfer of unlocked files.
+"""]]
diff --git a/doc/forum/Hashes_instead_of_content_in_files/comment_4_67733580e8f8806b5036406e6712b6a1._comment b/doc/forum/Hashes_instead_of_content_in_files/comment_4_67733580e8f8806b5036406e6712b6a1._comment
new file mode 100644
index 0000000000..72b0cab7f4
--- /dev/null
+++ b/doc/forum/Hashes_instead_of_content_in_files/comment_4_67733580e8f8806b5036406e6712b6a1._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="joey"
+ subject="""comment 4"""
+ date="2016-09-21T16:51:02Z"
+ content="""
+Like I said, use `git annex get`. This will populate the unlocked files
+with the content instead of the hashes.
+"""]]
diff --git a/doc/forum/Hello_everyone.mdwn b/doc/forum/Hello_everyone.mdwn
new file mode 100644
index 0000000000..6735c2499b
--- /dev/null
+++ b/doc/forum/Hello_everyone.mdwn
@@ -0,0 +1,4 @@
+Hi Everyone
+I'm a new member and my name is Kullboys.
+I'm happy to be familiar with you.
+Thanks[.](https://lab.louiz.org/snippets/421) 
diff --git a/doc/forum/Help_Windows_walkthrough.mdwn b/doc/forum/Help_Windows_walkthrough.mdwn
new file mode 100644
index 0000000000..dbe61b556b
--- /dev/null
+++ b/doc/forum/Help_Windows_walkthrough.mdwn
@@ -0,0 +1,177 @@
+Hello,
+
+i am trying to run the walkthrough on Windows 7. When i try to get the contents of a file, i only get a some git annex text string and not the real file. Both repositories are on the same ntfs filesystem. 
+
+C:\tmp>git annex version  
+git-annex version: 4.20130827-g4f18612  
+build flags: Pairing Testsuite S3 WebDAV DNS  
+local repository version: 4  
+default repository version: 3  
+supported repository versions: 3 4  
+upgrade supported from repository versions: 2  
+  
+C:\tmp\server>git --version  
+git version 1.8.3.msysgit.0  
+   
+
+# walkthrough.bat
+    
+    doskey /history > commands.log
+    mkdir laptop
+    cd laptop
+    git init
+    git annex init laptop
+    cd ..
+    
+    git clone laptop server
+    cd server
+    git annex init server
+    git remote add laptop c:\tmp\laptop
+    
+    cd ..\laptop
+    git remote add server c:\tmp\server
+    copy ..\1.pdf .
+    git annex add 1.pdf
+    git commit -m add
+    dir 
+    
+    cd ..\server
+    dir
+    git fetch laptop
+    git merge laptop/master
+    git annex get 1.pdf
+    dir
+    type 1.pdf
+    
+
+# walkthrough.log
+
+    C:\tmp>walkthrough.bat
+    
+    C:\tmp>doskey /history  1>commands.log
+    
+    C:\tmp>mkdir laptop
+    
+    C:\tmp>cd laptop
+    
+    C:\tmp\laptop>git init
+    Initialized empty Git repository in C:/tmp/laptop/.git/
+    
+    C:\tmp\laptop>git annex init laptop
+    init laptop
+      Detected a crippled filesystem.
+    
+      Enabling direct mode.
+    
+      Detected a filesystem without fifo support.
+    
+      Disabling ssh connection caching.
+    ok
+    (Recording state in git...)
+    
+    C:\tmp\laptop>cd ..
+    
+    C:\tmp>git clone laptop server
+    Cloning into 'server'...
+    done.
+    warning: remote HEAD refers to nonexistent ref, unable to checkout.
+    
+    
+    C:\tmp>cd server
+    
+    C:\tmp\server>git annex init server
+    init server
+      Detected a crippled filesystem.
+    
+      Enabling direct mode.
+    
+      Detected a filesystem without fifo support.
+    
+      Disabling ssh connection caching.
+    ok
+    (Recording state in git...)
+    
+    C:\tmp\server>git remote add laptop c:\tmp\laptop
+    
+    C:\tmp\server>cd ..\laptop
+    
+    C:\tmp\laptop>git remote add server c:\tmp\server
+    
+    C:\tmp\laptop>copy ..\1.pdf .
+            1 file(s) copied.
+    
+    C:\tmp\laptop>git annex add 1.pdf
+    add 1.pdf (checksum...) ok
+    (Recording state in git...)
+    
+    C:\tmp\laptop>git commit -m add
+    [master (root-commit) 7ad1514] add
+     1 file changed, 1 insertion(+)
+     create mode 120000 1.pdf
+    
+    C:\tmp\laptop>dir
+     Volume in drive C has no label.
+     Volume Serial Number is x
+    
+     Directory of C:\tmp\laptop
+    
+    09/01/2013  11:03 AM              .
+    09/01/2013  11:03 AM              ..
+    08/30/2013  12:43 PM            37,500 1.pdf
+                   1 File(s)         37,500 bytes
+                   2 Dir(s)   7,698,817,024 bytes free
+    
+    C:\tmp\laptop>cd ..\server
+    
+    C:\tmp\server>dir
+     Volume in drive C has no label.
+     Volume Serial Number is x
+    
+     Directory of C:\tmp\server
+    
+    09/01/2013  11:03 AM              .
+    09/01/2013  11:03 AM              ..
+                   0 File(s)              0 bytes
+                   2 Dir(s)   7,698,817,024 bytes free
+    
+    C:\tmp\server>git fetch laptop
+    remote: Counting objects: 9, done.
+    remote: Compressing objects: 100% (6/6), done.
+    remote: Total 8 (delta 1), reused 0 (delta 0)
+    Unpacking objects: 100% (8/8), done.
+    From c:\tmp\laptop
+     * [new branch]      git-annex  -> laptop/git-annex
+     * [new branch]      master     -> laptop/master
+    
+    C:\tmp\server>git merge laptop/master
+    
+    C:\tmp\server>git annex get 1.pdf
+    get 1.pdf (merging laptop/git-annex origin/git-annex into git-annex...)
+    (Recording state in git...)
+    (from laptop...)
+    1.pdf
+           37500 100%    4.51MB/s    0:00:00 (xfer#1, to-check=0/1)
+    
+    sent 37573 bytes  received 31 bytes  75208.00 bytes/sec
+    total size is 37500  speedup is 1.00
+    ok
+    (Recording state in git...)
+    
+    C:\tmp\server>dir
+     Volume in drive C has no label.
+     Volume Serial Number is x
+    
+     Directory of C:\tmp\server
+    
+    09/01/2013  11:03 AM              .
+    09/01/2013  11:03 AM              ..
+    09/01/2013  11:03 AM               194 1.pdf
+                   1 File(s)            194 bytes
+                   2 Dir(s)   7,698,767,872 bytes free
+    
+    C:\tmp\server>type 1.pdf
+    .git/annex/objects/kM/0q/SHA256E-s37500--32d8190c7e189d45f48245a100e4cc981ea1bbc
+    02ac8bfa6188db73e41ce06f3.pdf/SHA256E-s37500--32d8190c7e189d45f48245a100e4cc981e
+    a1bbc02ac8bfa6188db73e41ce06f3.pdfC:\tmp\server>
+    C:\tmp\server>
+    
diff --git a/doc/forum/Help_Windows_walkthrough/comment_1_5fc22393a1b28235eabb2871ad83d0a7._comment b/doc/forum/Help_Windows_walkthrough/comment_1_5fc22393a1b28235eabb2871ad83d0a7._comment
new file mode 100644
index 0000000000..95623a6454
--- /dev/null
+++ b/doc/forum/Help_Windows_walkthrough/comment_1_5fc22393a1b28235eabb2871ad83d0a7._comment
@@ -0,0 +1,12 @@
+[[!comment format=mdwn
+ username="http://joeyh.name/"
+ ip="4.153.8.7"
+ subject="comment 1"
+ date="2013-09-03T17:59:03Z"
+ content="""
+The walkthrough assumes a system that uses indirect mode by default, so it won't work quite right on Windows, which is forced to use direct mode.
+
+Running `git annex fsck` in the server repository will fix up this situation, but the right thing on Windows is to use `git annex sync` rather than the manual `git fetch + git merge` the walkthrough shows.
+
+Guess I'll make the walkthrough use sync, although it may make it harder for people to understand what's going on internally.
+"""]]
diff --git a/doc/forum/Help_fixing_S3_mistake.mdwn b/doc/forum/Help_fixing_S3_mistake.mdwn
new file mode 100644
index 0000000000..42c6934c93
--- /dev/null
+++ b/doc/forum/Help_fixing_S3_mistake.mdwn
@@ -0,0 +1,7 @@
+Hello, I'd like some help fixing a mistake I made setting up an S3 remote.
+
+I created an S3 remote A, then at some later time accidentally created an S3 remote B with the same settings as A, and moved some files over to B.
+
+I have fixed it by removing/marking as dead B, but I am missing some files now.
+
+I believe that all of these files exist in A since it is the same bucket as B, but git annex doesn't know that they are in A.  Is there a way to somehow "reinject" or refresh git annex so it knows that the files are there?  I'm using chunking as well, so I don't know how to download all the chunks and use git annex's reinject command.
diff --git a/doc/forum/Help_fixing_S3_mistake/comment_1_dc60d08da4c0b90705b0b88b4c834f05._comment b/doc/forum/Help_fixing_S3_mistake/comment_1_dc60d08da4c0b90705b0b88b4c834f05._comment
new file mode 100644
index 0000000000..54150eed99
--- /dev/null
+++ b/doc/forum/Help_fixing_S3_mistake/comment_1_dc60d08da4c0b90705b0b88b4c834f05._comment
@@ -0,0 +1,9 @@
+[[!comment format=mdwn
+ username="darkfeline"
+ subject="comment 1"
+ date="2015-10-13T05:24:25Z"
+ content="""
+A and B have different UUIDs, and they point to the same S3 bucket (I realize I may have done something bad here).
+
+I still have the UUID for B (but not attached to a remote); is it possible to merge git annex's knowledge of B into A, or otherwise re-initialize B?
+"""]]
diff --git a/doc/forum/Help_fixing_S3_mistake/comment_2_f105714b9bac7f45fdcb9a47e77255ac._comment b/doc/forum/Help_fixing_S3_mistake/comment_2_f105714b9bac7f45fdcb9a47e77255ac._comment
new file mode 100644
index 0000000000..8ae5e7ce0d
--- /dev/null
+++ b/doc/forum/Help_fixing_S3_mistake/comment_2_f105714b9bac7f45fdcb9a47e77255ac._comment
@@ -0,0 +1,24 @@
+[[!comment format=mdwn
+ username="joey"
+ subject="""comment 2"""
+ date="2015-10-15T18:30:27Z"
+ content="""
+Depends.. If one or both special remotes used encryption then no,
+one can't see the encrypted files that were put in the other one.
+
+If neither used encryption, and they're otherwise configured the same,
+then you can just use `git annex fsck --from A`. This will check files
+to see if their content is located on remote A, and if so, and git-annex
+had thought the file was only located on remote B, it will update the location
+tracking log to reflect the reality that the file is present on A.
+
+If either remote used encryption, then A can't see files that were added
+to B. So instead, you need this approach , which involves data transfer:
+
+	git annex enableremote B
+	git annex copy --from B
+	git annex copy --to A
+	git annex drop --from B
+	git annex dead B # if it wasn't already dead
+	git remote remove B
+"""]]
diff --git a/doc/forum/Help_fixing_S3_mistake/comment_3_a00bca3b6b0d7b6c20ca1ca749c93469._comment b/doc/forum/Help_fixing_S3_mistake/comment_3_a00bca3b6b0d7b6c20ca1ca749c93469._comment
new file mode 100644
index 0000000000..acf1d0bb5e
--- /dev/null
+++ b/doc/forum/Help_fixing_S3_mistake/comment_3_a00bca3b6b0d7b6c20ca1ca749c93469._comment
@@ -0,0 +1,16 @@
+[[!comment format=mdwn
+ username="darkfeline"
+ subject="comment 3"
+ date="2015-10-18T20:15:56Z"
+ content="""
+Thanks for the help.  I'm currently stuck partway through fixing this.
+
+1. I changed the UUID of the remote \"amazon\" in .git/config to the UUID of B.
+2. I changed contents of the annex-uuid file in the S3 bucket from the UUID of A to the UUID of B (making sure to not include a newline).
+
+After this, I think I can follow your steps to fix the problem.  However, Git annex is now reporting the error whenever I try to run a command:
+
+    trusted repositories: git-annex: S3 bucket not configured
+
+My understanding of this message is that Git annex is not seeing the UUID it is expecting in the S3 bucket, and that it will be fixed when some cache expires.  Is this correct?
+"""]]
diff --git a/doc/forum/Help_fixing_S3_mistake/comment_4_34333d0f3eeaf2929089535632759ba1._comment b/doc/forum/Help_fixing_S3_mistake/comment_4_34333d0f3eeaf2929089535632759ba1._comment
new file mode 100644
index 0000000000..ddb5c0fb69
--- /dev/null
+++ b/doc/forum/Help_fixing_S3_mistake/comment_4_34333d0f3eeaf2929089535632759ba1._comment
@@ -0,0 +1,11 @@
+[[!comment format=mdwn
+ username="darkfeline"
+ subject="comment 4"
+ date="2015-10-18T23:58:08Z"
+ content="""
+Actually, I suspect that I may have done more damage than I had initially thought.
+
+Is there a way to check whether the repo still has information about a file's whereabouts, especially how it was chunked on an S3 remote?  I'm not sure if that information still exists or not.  If it doesn't exist anymore, then recovery is likely impossible.
+
+If this information still exists in the repo, I can reconstitute the files by hand (script) if necessary and reinject them.  I'm assuming that I can decrypt them using my private key?
+"""]]
diff --git a/doc/forum/Help_fixing_S3_mistake/comment_5_7060bcd3424caab3570dcc7cbd3df93e._comment b/doc/forum/Help_fixing_S3_mistake/comment_5_7060bcd3424caab3570dcc7cbd3df93e._comment
new file mode 100644
index 0000000000..5cfe3edd2b
--- /dev/null
+++ b/doc/forum/Help_fixing_S3_mistake/comment_5_7060bcd3424caab3570dcc7cbd3df93e._comment
@@ -0,0 +1,12 @@
+[[!comment format=mdwn
+ username="joey"
+ subject="""comment 5"""
+ date="2015-10-26T17:31:54Z"
+ content="""
+Since this information about remotes is stored in your git repository, I
+don't see how the repository could lose it. I mean, you might have to look
+in the history if you've changed some value, like the chunking setting
+from what it's supposed to be, in order to find the historical value, but
+like anything checked into a git repository, it's there until you throw the
+repository away or git-filter-branch the information out of existence.
+"""]]
diff --git a/doc/forum/Help_on_my_usecase.mdwn b/doc/forum/Help_on_my_usecase.mdwn
new file mode 100644
index 0000000000..ef670aa254
--- /dev/null
+++ b/doc/forum/Help_on_my_usecase.mdwn
@@ -0,0 +1,22 @@
+I am using git-annex assistant for a few months now and still don't think I have it all setup correctly. My setup is:
+
+* Laptop Computer with low storage. Content creator, assistant: file source
+* Desktop Computer with medium storage capacity. Consumer, assistant: manual Mode
+* 2TB USB HD (EXT4) Backup, assistant: full backup
+* 2.5" portable 500GB drive NTFS formatted. transfer to backup: assistant: tried transfer and partial backup
+* Root Server on the web, not yet in the mix. Planned: Index only / content creator
+
+My primary goal is getting the stuff from the Laptop to the 2TB USB drive, syncing the annex with the desktop so I can get files on the desktop from the backup.
+What works:
+
+* Checking stuff into the annex on the Laptop
+* When the 2TB drive is connected to the Laptop, stuff is automatically moved there too
+* Bare Repo on the 2.5" configured as partial backup automatically backups the data
+
+What doesn't work:
+
+* Connecting the 2.5" bare repo to the Desktop to move stuff to the backup drive automatically
+* Having a human readable folder Structure on the 2.5" NTFS drive. I'd be ok to have a bare repo plus a special remote on the drive, but I don't know how to configure it
+* How do I drop stuff from the bare repo on the 2.5" drive? I think a partial backup should drop content that is in a full backup, but it doesn't
+
+Thanks for reading this long post. If anyone has any tips on how to configure the assistant, thats much appreciated.
diff --git a/doc/forum/Help_on_my_usecase/comment_1_a35b35c7927640f21d47c3df4f91dabb._comment b/doc/forum/Help_on_my_usecase/comment_1_a35b35c7927640f21d47c3df4f91dabb._comment
new file mode 100644
index 0000000000..25116a7c43
--- /dev/null
+++ b/doc/forum/Help_on_my_usecase/comment_1_a35b35c7927640f21d47c3df4f91dabb._comment
@@ -0,0 +1,12 @@
+[[!comment format=mdwn
+ username="http://joeyh.name/"
+ ip="209.250.56.246"
+ subject="comment 1"
+ date="2013-11-09T18:18:49Z"
+ content="""
+An incremental backup is supposed to drop content once it reaches a full backup. But, the assistant can only do that if it's able to verify that the full backup drive (or some other repository it can access) has a file, at the same time it's dropping it from the incremental backup drive. If you don't have both the incremental and full backup drives plugged in at the same time, you could consider making git-annex [[trust]] the full backup drive, which would avoid this explicit check.
+
+It sounds like you want a non-bare git repository on the 2.5\" drive. The assistant does not currently support setting that up, but you can do it by just using `git clone` as shown in the [[walkthrough]].
+
+I'm not clear on what doesn't work when you connect the 2.5\" drive to your desktop.
+"""]]
diff --git a/doc/forum/Help_with_assistant_and_Adobe_Lightroom.mdwn b/doc/forum/Help_with_assistant_and_Adobe_Lightroom.mdwn
new file mode 100644
index 0000000000..d3cf4e7a1d
--- /dev/null
+++ b/doc/forum/Help_with_assistant_and_Adobe_Lightroom.mdwn
@@ -0,0 +1,3 @@
+I am trying to use the git-annex assistant to automatically sync photos from my Lightroom catalog on Mac OS X to a remote archive server. I hit a snag due to a race condition combined with some peculiar behavior of Adobe Lightroom: it dereferences symlinks and uses the name of the target when importing (see https://forums.adobe.com/thread/568863?tstart=0) I observe the following problematic workflow: 1. Files are imported using the original filename. 2. The assistant eventually moves the content to .git/annex/objects and creates symlinks to replace the originals. 3. The photos no longer appear in the Library, however Lightroom will not allow me to re-import as it thinks the files have already been imported and are duplicates.
+
+I know a few other folks are using git-annex and Lightroom together, but I cannot tell if they are using the assistant or manually managing the annex. This implementation is for my wife who is not technical enough to use git on the command line so that's out of the question. It seems like direct mode is my best option for now, however, I'd like to avoid it since it's being deprecated. Is there some other option that I'm missing? Is anyone else out there successfully using the assistant and Lightroom together? I'd love some advice.
diff --git a/doc/forum/Help_with_assistant_and_Adobe_Lightroom/comment_1_3adf9f5c60558b3c70a8aa041dbbe966._comment b/doc/forum/Help_with_assistant_and_Adobe_Lightroom/comment_1_3adf9f5c60558b3c70a8aa041dbbe966._comment
new file mode 100644
index 0000000000..ea5e6e927c
--- /dev/null
+++ b/doc/forum/Help_with_assistant_and_Adobe_Lightroom/comment_1_3adf9f5c60558b3c70a8aa041dbbe966._comment
@@ -0,0 +1,13 @@
+[[!comment format=mdwn
+ username="joey"
+ subject="""comment 1"""
+ date="2016-04-14T19:42:33Z"
+ content="""
+The usual fix for this kind of problem is to use direct mode for your
+git-annex repository.
+
+Direct mode is actually the default if you use the webapp to create your
+repository. Otherwise, you can stop the assistant running, and run "git
+annex direct" to convert the repository to direct mode, then start the
+assistant back up.
+"""]]
diff --git a/doc/forum/Help_with_syncing_file_contents.mdwn b/doc/forum/Help_with_syncing_file_contents.mdwn
new file mode 100644
index 0000000000..28ac22b2e1
--- /dev/null
+++ b/doc/forum/Help_with_syncing_file_contents.mdwn
@@ -0,0 +1,68 @@
+Hi everyone,
+
+everyday I understand more and more how git and git-annex work but I need help with this one.
+I guess I have two questions but let me describe the scenario first:
+
+I have one local repository of mp3s (assume just one file: file1.mp3).
+I clone that repository into a remote git-annex repository over ssh and "git annex get" the file contents for file1.mp3.
+
+I unlock some mp3's locally and modify some mp3 tags. 
+Then I notify git-annex of these changes
+with "git annex add *"
+and commit them with "git commit -m 'mp3 tags changed'". 
+[git annex locks them again and changes the symlinks to point to the changed file in the annex, git commits the changed symlink]
+
+At this point in time there are two objects in my git annex repository:
+hash(file1.mp3) 
+hash(file1.mp3|with modified tags)
+The symlink points now to hash(file1.mp3|with modified tags)
+
+At this point the remote still does not now of this commit and of the new file contents.
+Thus I do "git push" to send the changes to the remote.
+The remote now has a BROKEN symlink because it already points to hash(file1.mp3|with modified tags)
+but the remote annex's object directory only contains hash(file1.mp3).
+
+Then I want my remote repository to also have the updated mp3 tags.
+The only way I see (without scripting) to have the updated mp3 tags in the remote repository is to do an "git annex get file1.mp3" on the remote repository or an "git annex copy --to remote file1.mp3" at my local repository. However, although the binary differences between both files
+file1.mp3 and file1.mp3|with modified tags are small the latter is transferred completely from the local repository to the remote repository.
+
+This is not a problem when just changing one file, but a problem when I have 10GB's of files and when it takes 2days to upload them to the remote because of a low bandwidth. 
+
+First question: Did I miss something? Does git-annex already provide means to only transmit the diff between the two objects? 
+
+Second question regarding disk space.
+I now have a complete history of all changes to file1.mp3 in my git-annex repository. I have the objects that represent every state of file1.mp3 and I can go back to these states when I checkout the respective commits and thereby the symlinks that link to these "old" objects. This history can take up a lot of space. What is the clean way to forget the past? AFAIK "git drop unused" only drops file contents that are not referenced in any commit? 
+
+If one wanted to preserve the entire history but save disk space one could also only store the current content and the patches that allow to reconstruct older versions from the current one. I understand that applying several patches consecutively takes more cpu time but for me going back to an older commit with my binary files only happens rarely. 
+
+This is the algorithm I have in mind for an optimized "git annex get file1":
+
+On that repository where the file is missing:
+
+1. Find the newest object that represented the contents of file1 in file1's commit history.
+
+2. Transmit this object identification hash(object) to the remote that has the current version (the one I am getting from).
+
+At that remote: 
+
+If the history contains full versions of file contents:
+
+Create a binary diff between the object identified by hash(object) and the current content of file1.
+
+If the history contains only the current version and patches to older versions:
+Collect all patches that represent the change from hash(object) to the current content of file1.
+
+If the list of patches is bigger than the current content of file 1 transmit the current content of file1. Otherwise transmit the patch(es) 
+
+On that repository where the file is missing:
+
+Apply the patch(es) to the latest object to obtain the current object.
+
+What's your opinion on this?
+
+Marek
+
+
+
+
+
diff --git a/doc/forum/Help_with_syncing_file_contents/comment_1_7ec34de3140983739080115c82966bf5._comment b/doc/forum/Help_with_syncing_file_contents/comment_1_7ec34de3140983739080115c82966bf5._comment
new file mode 100644
index 0000000000..87ef6d1306
--- /dev/null
+++ b/doc/forum/Help_with_syncing_file_contents/comment_1_7ec34de3140983739080115c82966bf5._comment
@@ -0,0 +1,18 @@
+[[!comment format=mdwn
+ username="https://www.google.com/accounts/o8/id?id=AItOawmp1ThsNNAbSn46ju-gwFELfStlhl8usJo"
+ nickname="donkeyicydragon"
+ subject="Possible quick solution with rsync"
+ date="2013-05-28T10:52:30Z"
+ content="""
+One could achieve the effect of only transmitting the changes of file contents using rsync.
+
+On the repository that lacks the current version:
+
+cp latestversionavailable currentversion
+rsync remoterepository/currentversion ./currentversion
+
+
+
+
+
+"""]]
diff --git a/doc/forum/Help_with_syncing_file_contents/comment_2_7dba58d3c62d6f64a270298e4e4329a4._comment b/doc/forum/Help_with_syncing_file_contents/comment_2_7dba58d3c62d6f64a270298e4e4329a4._comment
new file mode 100644
index 0000000000..2d427cccb8
--- /dev/null
+++ b/doc/forum/Help_with_syncing_file_contents/comment_2_7dba58d3c62d6f64a270298e4e4329a4._comment
@@ -0,0 +1,10 @@
+[[!comment format=mdwn
+ username="https://www.google.com/accounts/o8/id?id=AItOawmp1ThsNNAbSn46ju-gwFELfStlhl8usJo"
+ nickname="donkeyicydragon"
+ subject="I should have read git annex unused first"
+ date="2013-05-28T22:14:46Z"
+ content="""
+Apparently \"git annex unused\" finds previous versions of the object that are not symlinked anymore. However, if a file is renamed then the oldest version before the rename is not marked as unused.
+
+
+"""]]
diff --git a/doc/forum/Help_with_syncing_file_contents/comment_3_b26cfa20dc81517d93e760f4809bdc24._comment b/doc/forum/Help_with_syncing_file_contents/comment_3_b26cfa20dc81517d93e760f4809bdc24._comment
new file mode 100644
index 0000000000..6293ada24b
--- /dev/null
+++ b/doc/forum/Help_with_syncing_file_contents/comment_3_b26cfa20dc81517d93e760f4809bdc24._comment
@@ -0,0 +1,12 @@
+[[!comment format=mdwn
+ username="http://joeyh.name/"
+ nickname="joey"
+ subject="comment 3"
+ date="2013-05-29T16:50:35Z"
+ content="""
+Treating each version of a file as a separate object that is stored and transmitted separately is one of the core simplifying assumptions of git-annex. Among other things it makes it easy to have dumb special remotes that are mere key/value stores. And it keeps it easy to understand.
+
+I don't consider this a problem as far as storage goes. Disk space is cheap; git-annex allows dropping unused versions of files from repositories that are not on large cheap storage.
+
+For file transfer, it would be nice to find optimisations that don't require changing this simplifying assumption. This can be done at least for rsync I think. The [[design/assistant/deltas]] page is where the design of this is or will be worked on.
+"""]]
diff --git a/doc/forum/HowTo_Conflict_Resolution.mdwn b/doc/forum/HowTo_Conflict_Resolution.mdwn
new file mode 100644
index 0000000000..2e43efe69e
--- /dev/null
+++ b/doc/forum/HowTo_Conflict_Resolution.mdwn
@@ -0,0 +1,66 @@
+Hello,
+
+I have two git annex repos, AA and BB in direct mode. Both have edited a binary file before syncing
+
+```
+~/Desktop/BB (git)-[annex/direct/master] % git annex sync
+commit  ok
+pull origin 
+remote: Counting objects: 21, done.
+remote: Compressing objects: 100% (17/17), done.
+remote: Total 21 (delta 3), reused 0 (delta 0)
+Unpacking objects: 100% (21/21), done.
+From /home/florian/Desktop/AA
+   d55cfa2..b86e708  annex/direct/master -> origin/annex/direct/master
+   83ff4c6..5cfbd34  git-annex  -> origin/git-annex
+   d7b79e8..b86e708  master     -> origin/master
+   d7b79e8..b86e708  synced/master -> origin/synced/master
+
+Auto-merging bin2
+CONFLICT (content): Merge conflict in bin2
+Auto-merging bin1
+CONFLICT (content): Merge conflict in bin1
+Automatic merge failed; fix conflicts and then commit the result.
+bin2: needs merge
+bin1: needs merge
+(recording state in git...)
+
+  Merge conflict was automatically resolved; you may want to examine the result.
+
+ok
+(merging origin/git-annex into git-annex...)
+(recording state in git...)
+push origin 
+Counting objects: 31, done.
+Delta compression using up to 4 threads.
+Compressing objects: 100% (25/25), done.
+Writing objects: 100% (31/31), 3.04 KiB | 0 bytes/s, done.
+Total 31 (delta 6), reused 0 (delta 0)
+To /home/florian/Desktop/AA
+   83ff4c6..7df9834  git-annex -> synced/git-annex
+   b86e708..59a1240  annex/direct/master -> synced/master
+ok
+```
+
+```
+florian@horus ~/Desktop/BB (git)-[annex/direct/master] % git annex get .
+get bin1.variant-c4b6 (from origin...) ok
+get bin2.variant-f16a (from origin...) ok
+(recording state in git...)
+florian@horus ~/Desktop/BB (git)-[annex/direct/master] % ll
+insgesamt 5824
+-rw-r----- 1 florian florian 2863795 24. Aug 21:32 bin1.variant-1696
+-rw-r----- 1 florian florian 2841749 24. Aug 21:30 bin1.variant-c4b6
+-rw-r----- 1 florian florian  125612 24. Aug 21:32 bin2.variant-0efa
+-rw-r----- 1 florian florian  126067 24. Aug 21:31 bin2.variant-f16a
+
+```
+How can I resolve this conflict now? Is there a way to tell which bin1 / bin2 is from AA, which from BB? Is there a way to tell git annex to completely take the data from AA or BB?
+
+This case might appear somehow artifical but I was caught in that various times when syncing my music database. Actually I didn't care which version of the database it took, but I was unable to produce a coherent data set, so that the database files only come from one sync partner.
+
+I did the git annex get after syncing, becaue usually I sync using --content.
+
+Thanks,
+Florian
+
diff --git a/doc/forum/HowTo_Conflict_Resolution/comment_1_81c06a0590ee828749b8d07e008028bb._comment b/doc/forum/HowTo_Conflict_Resolution/comment_1_81c06a0590ee828749b8d07e008028bb._comment
new file mode 100644
index 0000000000..1a1405ccc0
--- /dev/null
+++ b/doc/forum/HowTo_Conflict_Resolution/comment_1_81c06a0590ee828749b8d07e008028bb._comment
@@ -0,0 +1,19 @@
+[[!comment format=mdwn
+ username="joey"
+ subject="""comment 1"""
+ date="2015-09-21T16:49:43Z"
+ content="""
+The "variant" names are based on a short hash of the key
+of the file, so there's no way to get from such a name back to the
+repository that the file was added to.
+
+What you can do is, look at the `git log --stat`, and find
+the two conflicting commits, and use git to either revert back to one of
+the two, or reset back to before the automatic merge and manually
+merge the two. 
+
+(Of course, this all works best when the repository is
+in indirect mode and you can use any git commands you like to slice and
+dice it. When the repo is direct mode, you might be able to run those
+same git commands via `git annex proxy`)
+"""]]
diff --git a/doc/forum/HowTo_Conflict_Resolution/comment_2_e0ec78f09e5aba83ad3a7252695e4d96._comment b/doc/forum/HowTo_Conflict_Resolution/comment_2_e0ec78f09e5aba83ad3a7252695e4d96._comment
new file mode 100644
index 0000000000..930253db32
--- /dev/null
+++ b/doc/forum/HowTo_Conflict_Resolution/comment_2_e0ec78f09e5aba83ad3a7252695e4d96._comment
@@ -0,0 +1,16 @@
+[[!comment format=mdwn
+ username="Horus"
+ subject="comment 2"
+ date="2016-04-17T14:10:15Z"
+ content="""
+A very belayed reply...
+
+I would really love to see a feature to make this easier! Like
+
+* Interactive merge: During merge the user is asked for each file that conflicts, if it wants to keep the local or the remote
+* A set merge tool: A tool that allows to either select the local or remote file, renames it accordingly and commits it.
+
+Currently, I really fear a merge conflict, because I work with sets of binary files that needs to be in a coherent state and a merge conflict gives me very much trouble restoring that state.
+
+Thanks!
+"""]]
diff --git a/doc/forum/HowTo_Conflict_Resolution/comment_3_495aea6e73fd7a2d7a387f539496ac3a._comment b/doc/forum/HowTo_Conflict_Resolution/comment_3_495aea6e73fd7a2d7a387f539496ac3a._comment
new file mode 100644
index 0000000000..05e958f767
--- /dev/null
+++ b/doc/forum/HowTo_Conflict_Resolution/comment_3_495aea6e73fd7a2d7a387f539496ac3a._comment
@@ -0,0 +1,16 @@
+[[!comment format=mdwn
+ username="Horus"
+ subject="comment 3"
+ date="2016-04-17T14:10:37Z"
+ content="""
+A very belayed reply...
+
+I would really love to see a feature to make this easier! Like
+
+* Interactive merge: During merge the user is asked for each file that conflicts, if it wants to keep the local or the remote
+* A set merge tool: A tool that allows to either select the local or remote file, renames it accordingly and commits it.
+
+Currently, I really fear a merge conflict, because I work with sets of binary files that needs to be in a coherent state and a merge conflict gives me very much trouble restoring that state.
+
+Thanks!
+"""]]
diff --git a/doc/forum/HowTo_Conflict_Resolution/comment_4_b209d4527bec4620e1014fe759cbdde7._comment b/doc/forum/HowTo_Conflict_Resolution/comment_4_b209d4527bec4620e1014fe759cbdde7._comment
new file mode 100644
index 0000000000..1c8e0a5fb6
--- /dev/null
+++ b/doc/forum/HowTo_Conflict_Resolution/comment_4_b209d4527bec4620e1014fe759cbdde7._comment
@@ -0,0 +1,14 @@
+[[!comment format=mdwn
+ username="joey"
+ subject="""comment 4"""
+ date="2016-04-20T16:43:10Z"
+ content="""
+git's default handling of merge conflict resolution is to require you to
+manually decide which of the two versions of the file to use. So, if you
+just don't use git-annex sync, and instead use git pull, it seems to me
+you'll get what you want.
+
+(You could also use `git annex sync --no-pull --content` to let everything
+except for the pull and merge be done by that command, and run the git pull 
+manually first.)
+"""]]
diff --git a/doc/forum/HowTo_Conflict_Resolution/comment_5_446186626e5b2d08d078e3b48be06bfa._comment b/doc/forum/HowTo_Conflict_Resolution/comment_5_446186626e5b2d08d078e3b48be06bfa._comment
new file mode 100644
index 0000000000..a38aaf3615
--- /dev/null
+++ b/doc/forum/HowTo_Conflict_Resolution/comment_5_446186626e5b2d08d078e3b48be06bfa._comment
@@ -0,0 +1,9 @@
+[[!comment format=mdwn
+ username="Horus"
+ subject="comment 5"
+ date="2016-05-05T10:00:20Z"
+ content="""
+Ok, that something I can understand.
+
+But still, I think, when using git-annex from the command line the situation of a merge conflict is still extremeley hard to handle. What is the reason against adding from which remote the conflicted data comes from?
+"""]]
diff --git a/doc/forum/HowTo_Conflict_Resolution/comment_6_83cafddb480fa9e5b5d87d27b994b2bc._comment b/doc/forum/HowTo_Conflict_Resolution/comment_6_83cafddb480fa9e5b5d87d27b994b2bc._comment
new file mode 100644
index 0000000000..008bc623b2
--- /dev/null
+++ b/doc/forum/HowTo_Conflict_Resolution/comment_6_83cafddb480fa9e5b5d87d27b994b2bc._comment
@@ -0,0 +1,7 @@
+[[!comment format=mdwn
+ username="Horus"
+ subject="comment 6"
+ date="2016-05-05T10:01:21Z"
+ content="""
+Normal git also has merge markers that the tell me about which part is from HEAD, which from the branch to merge.
+"""]]
diff --git a/doc/forum/How_To_Permanently_Delete_a_File__63__.mdwn b/doc/forum/How_To_Permanently_Delete_a_File__63__.mdwn
new file mode 100644
index 0000000000..fd654079fc
--- /dev/null
+++ b/doc/forum/How_To_Permanently_Delete_a_File__63__.mdwn
@@ -0,0 +1,13 @@
+Hi,
+
+We have several large git annex repos where all of the files are on remotes and we want to got through and clean up the repositories by deleting some subset of files.
+
+What is the fastest way to permanently delete files from a git annex repository with remotes?
+
+I guess I can to ``git annex drop --numcopies=0 ; git rm ``. Does that actually delete the file permanently?
+
+Is there a faster way?
+
+Thanks,
+
+Mike
diff --git a/doc/forum/How_To_Permanently_Delete_a_File__63__/comment_1_7f2cefb0991789be5a960eb9c0a9df3f._comment b/doc/forum/How_To_Permanently_Delete_a_File__63__/comment_1_7f2cefb0991789be5a960eb9c0a9df3f._comment
new file mode 100644
index 0000000000..c0eb14174b
--- /dev/null
+++ b/doc/forum/How_To_Permanently_Delete_a_File__63__/comment_1_7f2cefb0991789be5a960eb9c0a9df3f._comment
@@ -0,0 +1,22 @@
+[[!comment format=mdwn
+ username="https://www.google.com/accounts/o8/id?id=AItOawmwjQzWgiD7_I3zw-_91rMRf_6qoThupis"
+ nickname="Mike"
+ subject="comment 1"
+ date="2014-10-09T17:53:04Z"
+ content="""
+I experimented with this by making an empty directory with two empty files and one file with some content. I added them all, then ran ``git annex drop --numcopies=0 ; git rm `` on one of the empty files.
+
+Interestingly, what happened is that git annex deleted the empty file from .git/annex/objects, but left the directory structure. In this case the link pointed to:
+
+.git/annex/objects/pX/ZJ/SHA256E-s0--e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855/SHA256E-s0--e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855
+
+After the drop command what was left was the following empty directory:
+
+.git/annex/objects/pX/ZJ/SHA256E-s0--e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855/
+
+Also interestingly and terrifyingly, because there were two empty files, both pointed to the same object, the ``git annex drop`` command deleted the file in the objects directory, and now the second link points to nothing. The file is done.
+
+This means that if you have a git annex repository and you have two copies of a file, and you think to yourself, \"oh, let me just delete one, I don't need two\", and you use the method above, you will permanently and irrevocably delete both files.  Not good.
+
+Any better ideas on how to do this?
+"""]]
diff --git a/doc/forum/How_To_Permanently_Delete_a_File__63__/comment_2_d13b456c5b3990082c16e78a50f5db91._comment b/doc/forum/How_To_Permanently_Delete_a_File__63__/comment_2_d13b456c5b3990082c16e78a50f5db91._comment
new file mode 100644
index 0000000000..959ac282ff
--- /dev/null
+++ b/doc/forum/How_To_Permanently_Delete_a_File__63__/comment_2_d13b456c5b3990082c16e78a50f5db91._comment
@@ -0,0 +1,14 @@
+[[!comment format=mdwn
+ username="https://www.google.com/accounts/o8/id?id=AItOawmwjQzWgiD7_I3zw-_91rMRf_6qoThupis"
+ nickname="Mike"
+ subject="comment 2"
+ date="2014-10-09T18:02:59Z"
+ content="""
+Tried another approach:
+
+``git annex unannex ; rm ``
+
+This does not delete the original, and it only works if you do ``git annex get `` first. It won't update the remote, unless you cd into that remote and run ``git annex sync`` there. After that there is the illusion the file is done, but its content is still in .git/annex/objects. In my test case I could vim into the file in question in the objects directory and it was still there.
+
+So ``git annex drop`` deletes both copies of duplicate files and so is too dangerous to use and ``git annex unannex`` doesn't delete the file anywhere. I am a little stuck here, what do I do?
+"""]]
diff --git a/doc/forum/How_To_Permanently_Delete_a_File__63__/comment_3_854c17ff8cb38486c4bef618d1e94919._comment b/doc/forum/How_To_Permanently_Delete_a_File__63__/comment_3_854c17ff8cb38486c4bef618d1e94919._comment
new file mode 100644
index 0000000000..404d12ae49
--- /dev/null
+++ b/doc/forum/How_To_Permanently_Delete_a_File__63__/comment_3_854c17ff8cb38486c4bef618d1e94919._comment
@@ -0,0 +1,24 @@
+[[!comment format=mdwn
+ username="https://www.google.com/accounts/o8/id?id=AItOawmwjQzWgiD7_I3zw-_91rMRf_6qoThupis"
+ nickname="Mike"
+ subject="comment 3"
+ date="2014-10-09T18:16:34Z"
+ content="""
+OK, I should have read more before writing.
+
+It seems like the procedure is described here:
+http://git-annex.branchable.com/walkthrough/unused_data/
+
+The process is:
+1. rm files or directories
+2. git annex sync in all remotes (that is a pain, I wish I only had to do it once)
+3. git annex unused
+4. git annex dropunused
+5. Repeat 4 and 5 on any repository where the data is stored
+
+That does work for me, it is just slightly cumbersome. If there is another way or if I am missing something, please let me know.
+
+Thanks,
+
+Mike
+"""]]
diff --git a/doc/forum/How_To_Permanently_Delete_a_File__63__/comment_4_9572ad02bbf6845b1ab6d7c612c12a2a._comment b/doc/forum/How_To_Permanently_Delete_a_File__63__/comment_4_9572ad02bbf6845b1ab6d7c612c12a2a._comment
new file mode 100644
index 0000000000..6e8ddc5959
--- /dev/null
+++ b/doc/forum/How_To_Permanently_Delete_a_File__63__/comment_4_9572ad02bbf6845b1ab6d7c612c12a2a._comment
@@ -0,0 +1,19 @@
+[[!comment format=mdwn
+ username="http://joeyh.name/"
+ ip="209.250.56.54"
+ subject="comment 4"
+ date="2014-10-09T18:23:58Z"
+ content="""
+This post is misplaced, it is not a tip about how to use git-annex, but a question. I will be moving it to the forum after posting this comment.
+
+The right answer is probably to run: `git annex drop $file`, with no --numcopies, no --force, etc. Just let git-annex do its job; it will check the remotes to ensure that enough copies of the file exist to make it safe to drop the content of the file from the local repository. (
+Note that --numcopies=0 is very unsafe, you're asking git-annex to delete even the last copy of your data without checking when you do that.)
+
+If your goal is to get rid of every copy of this file from every repository that has a copy, I suggest just `git rm $file; git commit`, followed by running `git annex unused` in the various repositories to clean them up.
+
+There is a faster way, which is to run `git annex drop --from $remote` for each remote that has the file. If you want to get rid of every copy of the file, for sure, you could add a --force to that.
+
+git-annex deduplicates data, so it's completely expected that if two files have the same content, dropping one will remove the content of the other.
+
+I cannot reproduce any .git/annex/objects/foo empty directories being left behind by git-annex after doing that. Perhaps you are not using a current version of git-annex?
+"""]]
diff --git a/doc/forum/How_To_Permanently_Delete_a_File__63__/comment_5_9c28faabb7d7bd1e83d551e2938d3532._comment b/doc/forum/How_To_Permanently_Delete_a_File__63__/comment_5_9c28faabb7d7bd1e83d551e2938d3532._comment
new file mode 100644
index 0000000000..5f6b0bf089
--- /dev/null
+++ b/doc/forum/How_To_Permanently_Delete_a_File__63__/comment_5_9c28faabb7d7bd1e83d551e2938d3532._comment
@@ -0,0 +1,14 @@
+[[!comment format=mdwn
+ username="https://www.google.com/accounts/o8/id?id=AItOawmwjQzWgiD7_I3zw-_91rMRf_6qoThupis"
+ nickname="Mike"
+ subject="comment 5"
+ date="2014-10-13T14:51:47Z"
+ content="""
+Sorry about the misplacement, that was a complete accident.
+
+What I am trying to do is to delete files as quickly as possible from every repository. In this case we are using git annex to move non-critical data from our main RAID drive to an external drive while still maintaining the full directory structure on the RAID drive. This is very valuable because we sometimes won't need the data for months or years, but then we may suddenly need a few files, and git annex makes it very easy to get them back. But we are talking about many terabytes and thousands and thousands of files here, and sometimes we just want to completely get rid of that data, it just takes up too much drive space. I wanted to make it as easy and safe as possible for people to just delete files from every repository, hence the question.
+
+I am nervous about using ``git annex drop --force`` because it seems to me that if there are two identical copies of a file in a repository, that command will kill the content of both... or does that only happen with ``git annex drop --numcopies=0``?
+
+I think the best solution for me seems to be the ``git rm ; git annex unused; git annex dropunused; git annex sync`` series of commands. It would just be nice if it were possible to achieve the same results in every repository with a simple command such as ``git annex rm --all ``. I recognise that this would be a dangerous command, but frankly I feel like in linux, everyone should be aware just how dangerous ``rm`` is in every context :-)
+"""]]
diff --git a/doc/forum/How_big_can_a_git-annex_repo_get_and_still_be_reliable_and_fast__63__.mdwn b/doc/forum/How_big_can_a_git-annex_repo_get_and_still_be_reliable_and_fast__63__.mdwn
new file mode 100644
index 0000000000..1349ad95ec
--- /dev/null
+++ b/doc/forum/How_big_can_a_git-annex_repo_get_and_still_be_reliable_and_fast__63__.mdwn
@@ -0,0 +1,8 @@
+As title says, how big can a git-annex repo be?
+
+My use case is this:
+I have a few external hard disks (like 2TB, 3TB etc) and each of them have a bunch of files. Ideally I would like to keep all of the data organized in a _single_ git annex repo. However, there are probably a handful of really big files and a LARGE number of really small files, and I doubt its a good idea to put all of them in together.
+
+What should be my biggest concern? I assume file size is not a problem, and I should either tar/zip smaller files into much bigger ones and annex that, or split it up into multiple small annex repositories.
+
+Any suggestions? What do the rest of you with similar amounts of data do?
diff --git a/doc/forum/How_big_can_a_git-annex_repo_get_and_still_be_reliable_and_fast__63__/comment_1_99e7e1b178bbb17fffea0892538d4049._comment b/doc/forum/How_big_can_a_git-annex_repo_get_and_still_be_reliable_and_fast__63__/comment_1_99e7e1b178bbb17fffea0892538d4049._comment
new file mode 100644
index 0000000000..84252ba5b1
--- /dev/null
+++ b/doc/forum/How_big_can_a_git-annex_repo_get_and_still_be_reliable_and_fast__63__/comment_1_99e7e1b178bbb17fffea0892538d4049._comment
@@ -0,0 +1,9 @@
+[[!comment format=mdwn
+ username="CandyAngel"
+ subject="comment 1"
+ date="2015-06-30T07:17:48Z"
+ content="""
+It can actually get really big and still be okay to use, if you follow the [tips page for annexs with lots of files](/tips/Repositories with large number of files).
+
+Mine is still reasonably responsive at 10 million files! Iterating over it all (git annex info, deduping) is really painful though, but it'd be like that without git-annex..
+"""]]
diff --git a/doc/forum/How_big_can_a_git-annex_repo_get_and_still_be_reliable_and_fast__63__/comment_2_6a7a3c372599eed7c52d5f54e5287577._comment b/doc/forum/How_big_can_a_git-annex_repo_get_and_still_be_reliable_and_fast__63__/comment_2_6a7a3c372599eed7c52d5f54e5287577._comment
new file mode 100644
index 0000000000..fc05eddf10
--- /dev/null
+++ b/doc/forum/How_big_can_a_git-annex_repo_get_and_still_be_reliable_and_fast__63__/comment_2_6a7a3c372599eed7c52d5f54e5287577._comment
@@ -0,0 +1,18 @@
+[[!comment format=mdwn
+ username="joey"
+ subject="""comment 2"""
+ date="2015-07-02T20:44:09Z"
+ content="""
+Depending on the total size of the small files, you might consider a mixed
+repo, with the small files checked into git normally, and the larger files
+annexed. 
+
+The advantage is that you then don't need to use git-annex commands to
+manage the many small files. This will probably be faster, for except you
+won't need to `git annex get` a ton of small files, which will avoid a lot
+of overhead.
+
+Of course, if you have gigabytes of small files, that will result
+in a git repo gigabytes in size, and you will start to run into some of the
+scalability problems that git-annex addresses.
+"""]]
diff --git a/doc/forum/How_big_can_a_git-annex_repo_get_and_still_be_reliable_and_fast__63__/comment_3_cc62f722b3e1db7d75d6976ef3854f94._comment b/doc/forum/How_big_can_a_git-annex_repo_get_and_still_be_reliable_and_fast__63__/comment_3_cc62f722b3e1db7d75d6976ef3854f94._comment
new file mode 100644
index 0000000000..da8067ab29
--- /dev/null
+++ b/doc/forum/How_big_can_a_git-annex_repo_get_and_still_be_reliable_and_fast__63__/comment_3_cc62f722b3e1db7d75d6976ef3854f94._comment
@@ -0,0 +1,7 @@
+[[!comment format=mdwn
+ username="Mattt"
+ subject="comment 3"
+ date="2015-07-05T19:40:30Z"
+ content="""
+Thanks CandyAngel and Joey. That helped. I have transferred pretty much my entire repo into git annex now and have it sorted the way I want. Yay! 
+"""]]
diff --git a/doc/forum/How_can_I_recover_from_misadventure__63__.mdwn b/doc/forum/How_can_I_recover_from_misadventure__63__.mdwn
new file mode 100644
index 0000000000..624ab38300
--- /dev/null
+++ b/doc/forum/How_can_I_recover_from_misadventure__63__.mdwn
@@ -0,0 +1,10 @@
+Hi, 
+
+  I have been attempting to use `git annex` for archiving about 300gb of data (a usb disk that is a copy of a laptop drive). I have an S3 special remote setup as a backup remote right now (the plan is once I feel like the data is safe on s3 I will change the remote type to archive and get rid of the local copies). I am using the WORM backend. I think something has gone wrong:
+
+1. As of right now I have about 200mb of data on S3, but 'git annex copy --to S3 .' runs for a while, then exits, without any indication that it has uploaded any additional files.
+2. .git/ is around 200gb in size
+3. 'git annex add .' adds files for a while, then exits, and does the same thing when run again.
+4. At this point I decided to try and restart from scratch so I ran 'git annex direct' to get the files replaced in the working directory, this runs for a while then hangs.
+
+This has been my first outing with `git annex`, so I am sort of at a loss as to what there is I can poke at to try and determine what is going on. It would be nice to be able to get the data back to the original state, but more than that, I would like to understand what I did wrong, and I would appreciate any pointers or advice. 
diff --git a/doc/forum/How_can_I_recover_from_misadventure__63__/comment_1_1e8ce32e1e8adf32acf7af28c7ab0d0b._comment b/doc/forum/How_can_I_recover_from_misadventure__63__/comment_1_1e8ce32e1e8adf32acf7af28c7ab0d0b._comment
new file mode 100644
index 0000000000..e96476d791
--- /dev/null
+++ b/doc/forum/How_can_I_recover_from_misadventure__63__/comment_1_1e8ce32e1e8adf32acf7af28c7ab0d0b._comment
@@ -0,0 +1,10 @@
+[[!comment format=mdwn
+ username="joey"
+ subject="""comment 1"""
+ date="2016-01-19T20:32:30Z"
+ content="""
+When you say that commands run for a while and then exit, are they exiting
+successfully? With a nonzero exit status? With an error message?
+
+What OS? What git-annex version?
+"""]]
diff --git a/doc/forum/How_can_I_recover_from_misadventure__63__/comment_2_94f5435529802ea9d1980ee07025c6eb._comment b/doc/forum/How_can_I_recover_from_misadventure__63__/comment_2_94f5435529802ea9d1980ee07025c6eb._comment
new file mode 100644
index 0000000000..dede1fc7ac
--- /dev/null
+++ b/doc/forum/How_can_I_recover_from_misadventure__63__/comment_2_94f5435529802ea9d1980ee07025c6eb._comment
@@ -0,0 +1,36 @@
+[[!comment format=mdwn
+ username="hiredman@b5864f8a30c0cafbc990313b56c39586ca7a21ec"
+ nickname="hiredman"
+ subject="comment 2"
+ date="2016-01-19T23:11:25Z"
+ content="""
+```
+$ git annex version
+git-annex version: 5.20151218-g5008846
+build flags: Assistant Webapp Webapp-secure Pairing Testsuite S3(multipartupload) WebDAV Inotify DBus DesktopNotify XMPP ConcurrentOutput DNS Feeds Quvi TDFA TorrentParser Database
+key/value backends: SHA256E SHA256 SHA512E SHA512 SHA224E SHA224 SHA384E SHA384 SHA3_256E SHA3_256 SHA3_512E SHA3_512 SHA3_224E SHA3_224 SHA3_384E SHA3_384 SKEIN256E SKEIN256 SKEIN512E SKEIN512 SHA1E SHA1 MD5E MD5 WORM URL  
+remote types: git gcrypt S3 bup directory rsync web bittorrent webdav tahoe glacier ddar hook external
+$ lsb_release -a
+No LSB modules are available.
+Distributor ID: Ubuntu
+Description:    Ubuntu 15.04
+Release:        15.04
+Codename:       vivid  
+$ uname -a
+Linux aktaios 4.2.0-040200rc2-generic #201507160938 SMP Thu Jul 16 09:49:19 UTC 2015 i686 i686 i686 GNU/Linux
+```
+
+The commands would exit successfully.
+
+
+I realize now that if I wanted to start from scratch the correct thing to do is likely 'git annex uninit' so I am running that now, but it fails often with an error like:
+
+```
+git-annex: Users/hiredman/.gem/specs/rubygems.org%80/quick/Marshal.4.8/bundler-1.0.15.gemspec points to annexed content, but is not checked into git.
+Perhaps this was left behind by an interrupted git annex add?
+Not continuing with uninit; either delete or git annex add the file and retry.
+```
+
+So I wrote in little shell loop to run 'git annex uninit', replace symlink with the contents out of .git/, and run 'git annex uinit' again. This is going slowly and seems to be getting slower, so I guess uninit does some kind of scan of all the files that is interrupted every time it hits the above error, and has to be done again from scratch every time, so as I remove those errors one by one uninit takes longer and longer to hit the next error. So what I need to do is write a script to fix the links up in a single scan. 
+
+"""]]
diff --git a/doc/forum/How_can_I_recover_from_misadventure__63__/comment_3_484fc154c68ba59939aae4452c663ce6._comment b/doc/forum/How_can_I_recover_from_misadventure__63__/comment_3_484fc154c68ba59939aae4452c663ce6._comment
new file mode 100644
index 0000000000..25732bb4b8
--- /dev/null
+++ b/doc/forum/How_can_I_recover_from_misadventure__63__/comment_3_484fc154c68ba59939aae4452c663ce6._comment
@@ -0,0 +1,18 @@
+[[!comment format=mdwn
+ username="joey"
+ subject="""comment 3"""
+ date="2016-01-20T19:08:34Z"
+ content="""
+So, this seems to suggest that `git annex add` exited before it got around
+to checking all the files it had added into the git repository.
+
+I don't see how that could happen if the command exited successfully.
+If it were interrupted, sure.
+
+Normally I'd recommend that `git annex add` just be re-run to recover from
+an interrupted add. But if it's just going to exit for some reason again,
+I guess that won't work.
+
+I think you need to strace a git-annex command like `git-annex copy --to S3`
+and see if there's some clue why it's exiting unexpectedly.
+"""]]
diff --git a/doc/forum/How_can_I_use_git-annex_get_a_list_of_files_in_directory_that_have_content__63___.mdwn b/doc/forum/How_can_I_use_git-annex_get_a_list_of_files_in_directory_that_have_content__63___.mdwn
new file mode 100644
index 0000000000..78b10b47a4
--- /dev/null
+++ b/doc/forum/How_can_I_use_git-annex_get_a_list_of_files_in_directory_that_have_content__63___.mdwn
@@ -0,0 +1,6 @@
+Hello.
+How can I use git-annex get a list of files in directory that have content? For each such file the result of command
+
+     git-annex get [file] 
+   
+will be empty.
diff --git a/doc/forum/How_can_I_use_git-annex_get_a_list_of_files_in_directory_that_have_content__63___/comment_1_e897d8fc10474cf865279dc22f22ecb7._comment b/doc/forum/How_can_I_use_git-annex_get_a_list_of_files_in_directory_that_have_content__63___/comment_1_e897d8fc10474cf865279dc22f22ecb7._comment
new file mode 100644
index 0000000000..8a0697e093
--- /dev/null
+++ b/doc/forum/How_can_I_use_git-annex_get_a_list_of_files_in_directory_that_have_content__63___/comment_1_e897d8fc10474cf865279dc22f22ecb7._comment
@@ -0,0 +1,10 @@
+[[!comment format=mdwn
+ username="arand"
+ ip="130.243.226.21"
+ subject="comment 1"
+ date="2013-12-30T13:15:20Z"
+ content="""
+    git annex find 
+
+that's what you meant?
+"""]]
diff --git a/doc/forum/How_can_I_use_git-annex_get_a_list_of_files_in_directory_that_have_content__63___/comment_2_3af326205db6ee04f2a8644baa1dd566._comment b/doc/forum/How_can_I_use_git-annex_get_a_list_of_files_in_directory_that_have_content__63___/comment_2_3af326205db6ee04f2a8644baa1dd566._comment
new file mode 100644
index 0000000000..6fe18b5cb2
--- /dev/null
+++ b/doc/forum/How_can_I_use_git-annex_get_a_list_of_files_in_directory_that_have_content__63___/comment_2_3af326205db6ee04f2a8644baa1dd566._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="Alexcei"
+ ip="92.255.239.77"
+ subject="comment 2"
+ date="2013-12-30T13:25:45Z"
+ content="""
+Yes, this is what I need. I have not seen this command before. Thank you.
+"""]]
diff --git a/doc/forum/How_do_I_backup_my_data_to_blue_ray_disks__63__.mdwn b/doc/forum/How_do_I_backup_my_data_to_blue_ray_disks__63__.mdwn
new file mode 100644
index 0000000000..b94111e943
--- /dev/null
+++ b/doc/forum/How_do_I_backup_my_data_to_blue_ray_disks__63__.mdwn
@@ -0,0 +1 @@
+I have several TB of media on a Debian ZFS server. If I created a git-annex repo for the data, how hard would it be get git-annex (using bup I assume) to back up the files onto a set of Blu Ray disks? I realize that 8TB of data would take about 320 BR 25GB disks, but it seems like git annex would be great for identifying what disk(s) I needed to recover a file. I'm good at bash scripting, and use git daily. I have no experience with git-annex or bup however. Any links to information or suggestions is very appreciated. Thanks!
diff --git a/doc/forum/How_do_I_backup_my_data_to_blue_ray_disks__63__/comment_1_ea2f6ca0768c55fa136606cf091471cd._comment b/doc/forum/How_do_I_backup_my_data_to_blue_ray_disks__63__/comment_1_ea2f6ca0768c55fa136606cf091471cd._comment
new file mode 100644
index 0000000000..d004c06a08
--- /dev/null
+++ b/doc/forum/How_do_I_backup_my_data_to_blue_ray_disks__63__/comment_1_ea2f6ca0768c55fa136606cf091471cd._comment
@@ -0,0 +1,25 @@
+[[!comment format=mdwn
+ username="joey"
+ subject="""comment 1"""
+ date="2015-05-01T21:36:59Z"
+ content="""
+There was a previous thread about using DVDS:
+
+
+If the bluerays are rewritable, I'd probably just slap a Real Filesystem
+(ext2 not isofs) on there and put a regular git-annex repo on it. I'd probably
+run git-annex with the option "-c annex.alwayscommit=false" to prevent it
+making many commits to the repo on the blueray, which would rewrite parts of
+this disk, perhaps too often.
+
+Or, to avoid any rewrites at all (except to directory metadata),
+I might use a directory special remote on the blueray.
+
+I don't see much benefit to using bup over the directory special remote.
+
+If the bluerays are not rewritable, I might try making the git-annex repo
+in a temporary directory on the hard disk, and then generating the ISO
+from that once I've filled it up. Should work fine, I might set "remote..annex-readonly"
+to true in git repos that had such a disk as a remote to let git-annex
+know to not try to write to it.
+"""]]
diff --git a/doc/forum/How_do_I_backup_my_data_to_blue_ray_disks__63__/comment_2_d79387905af9f8dce77f9aa85f2d6a03._comment b/doc/forum/How_do_I_backup_my_data_to_blue_ray_disks__63__/comment_2_d79387905af9f8dce77f9aa85f2d6a03._comment
new file mode 100644
index 0000000000..4d7caf7496
--- /dev/null
+++ b/doc/forum/How_do_I_backup_my_data_to_blue_ray_disks__63__/comment_2_d79387905af9f8dce77f9aa85f2d6a03._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="https://me.yahoo.com/a/e1w.8yMTnpUh.5fOXneP92pTdy23XJPwx84uLSM-#aca7c"
+ nickname="Michael"
+ subject="re: backup my data to blue ray disks"
+ date="2015-05-11T21:13:19Z"
+ content="""
+Thank you for your response. The disks are BD-R, so they are not rewritable. My experience in the past with rewritable disks has been that they work about as well as the average TV infomercial product. Do you have any opinions or suggestions on using par2 files on the disks?
+"""]]
diff --git a/doc/forum/How_do_I_cleanly_remove_an_Android_git-annex_installation__63__.mdwn b/doc/forum/How_do_I_cleanly_remove_an_Android_git-annex_installation__63__.mdwn
new file mode 100644
index 0000000000..a4bcc4843c
--- /dev/null
+++ b/doc/forum/How_do_I_cleanly_remove_an_Android_git-annex_installation__63__.mdwn
@@ -0,0 +1 @@
+I had accidentally chosen the wrong directory for the repository. What should I do to clean all of git-annex?
diff --git a/doc/forum/How_do_I_cleanly_remove_an_Android_git-annex_installation__63__/comment_1_e14757c2c106770c2d7069ace4987b3b._comment b/doc/forum/How_do_I_cleanly_remove_an_Android_git-annex_installation__63__/comment_1_e14757c2c106770c2d7069ace4987b3b._comment
new file mode 100644
index 0000000000..b051aa704a
--- /dev/null
+++ b/doc/forum/How_do_I_cleanly_remove_an_Android_git-annex_installation__63__/comment_1_e14757c2c106770c2d7069ace4987b3b._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="http://joeyh.name/"
+ ip="4.154.0.63"
+ subject="comment 1"
+ date="2013-08-24T18:43:08Z"
+ content="""
+It should be sufficient to delete `/sdcard/git-annex.home` and delete the .git directory inside whatever directory you set up as the repository.
+"""]]
diff --git a/doc/forum/How_do_I_do_with_.gitrefs__47_____63__.mdwn b/doc/forum/How_do_I_do_with_.gitrefs__47_____63__.mdwn
new file mode 100644
index 0000000000..d4a2a71b14
--- /dev/null
+++ b/doc/forum/How_do_I_do_with_.gitrefs__47_____63__.mdwn
@@ -0,0 +1 @@
+There is .gitrefs directory in my annexed repo while I am unaware. Should I do `git add .gitrefs/` or `git annex add .gitrefs/` or just ignore?
diff --git a/doc/forum/How_do_I_do_with_.gitrefs__47_____63__/comment_1_5e235af2ea13fd4f6a226c842f69965e._comment b/doc/forum/How_do_I_do_with_.gitrefs__47_____63__/comment_1_5e235af2ea13fd4f6a226c842f69965e._comment
new file mode 100644
index 0000000000..9e8d044a1e
--- /dev/null
+++ b/doc/forum/How_do_I_do_with_.gitrefs__47_____63__/comment_1_5e235af2ea13fd4f6a226c842f69965e._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="http://joeyh.name/"
+ ip="209.250.56.35"
+ subject="comment 1"
+ date="2014-01-06T15:44:15Z"
+ content="""
+AFAIK neither git nor git-annex creates a .gitrefs directory.
+"""]]
diff --git a/doc/forum/How_do_I_do_with_.gitrefs__47_____63__/comment_2_50d0c643537175b514d5eae604fb5bea._comment b/doc/forum/How_do_I_do_with_.gitrefs__47_____63__/comment_2_50d0c643537175b514d5eae604fb5bea._comment
new file mode 100644
index 0000000000..a812e99b0f
--- /dev/null
+++ b/doc/forum/How_do_I_do_with_.gitrefs__47_____63__/comment_2_50d0c643537175b514d5eae604fb5bea._comment
@@ -0,0 +1,51 @@
+[[!comment format=mdwn
+ username="EskildHustvedt"
+ ip="80.203.53.96"
+ subject="comment 2"
+ date="2014-01-11T07:51:57Z"
+ content="""
+I have noticed a .gitrefs/ directory being added to my git-annex repositories as well.
+Two direct-mode git-annex assistant-managed repos have turned up a .gitrefs directory the past month or so, with the directory often returning after I remove it. First occurrance according to git logs is on the 3rd of December.
+
+The directory looks a lot like .git/refs, as if some tool is trying to do something to .git/refs but has a bug that removes the first /, and then proceeds to copy or recreate the files.
+
+    [0 zerodogg@firefly annexed]$ du -hs .gitrefs .git/refs
+    172K    .gitrefs
+    192K    .git/refs
+    [0 zerodogg@firefly annexed]$ find .gitrefs > /tmp/gitr1
+    [0 zerodogg@firefly annexed]$ find .git/refs > /tmp/gitr2
+    [0 zerodogg@firefly annexed]$ perl -pi -e 's/^\.git.?refs//' /tmp/gitr*                                                                                                   
+    [0 zerodogg@firefly annexed]$ diff -BrayN --suppress-common-lines /tmp/gitr1 /tmp/gitr2                                                                                   
+                                                                  > /tags
+                                                                  > /remotes/browncoats
+                                                                  > /remotes/browncoats/master
+                                                                  > /remotes/browncoats/synced
+                                                                  > /remotes/browncoats/synced/master
+                                                                  > /remotes/browncoats/synced/git-annex
+                                                                  > /remotes/browncoats/git-annex
+                                                                  > /remotes/browncoats/annex
+                                                                  > /remotes/browncoats/annex/direct
+                                                                  > /remotes/browncoats/annex/direct/master
+    /remotes/serenity/master                                      <
+    /remotes/zoe                                                  | /remotes/faerun/annex
+    /remotes/zoe/master                                           | /remotes/faerun/annex/direct
+    /remotes/zoe/synced                                           | /remotes/faerunpi
+    /remotes/zoe/synced/master                                    | /remotes/faerunpi/synced
+    /remotes/zoe/synced/git-annex                                 | /remotes/faerunpi/annex
+    /remotes/firefly                                              | /remotes/faerunpi/annex/direct
+    /remotes/firefly/master                                       <
+    /remotes/firefly/synced                                       <
+    /remotes/firefly/synced/master                                <
+    /remotes/firefly/synced/git-annex                             <
+    /remotes/firefly/git-annex                                    <
+    /remotes/firefly/annex                                        <
+    /remotes/firefly/annex/direct                                 <
+    /remotes/firefly/annex/direct/master                          <
+    /remotes/river/master                                         <
+                                                                  > /remotes/origin
+                                                                  > /remotes/origin/synced
+                                                                  > /remotes/hufsa
+                                                                  > /remotes/hufsa/synced
+                                                                  > /synced
+
+"""]]
diff --git a/doc/forum/How_do_I_do_with_.gitrefs__47_____63__/comment_3_3d342c32b14c7edbece596ba970a8415._comment b/doc/forum/How_do_I_do_with_.gitrefs__47_____63__/comment_3_3d342c32b14c7edbece596ba970a8415._comment
new file mode 100644
index 0000000000..8902e004a6
--- /dev/null
+++ b/doc/forum/How_do_I_do_with_.gitrefs__47_____63__/comment_3_3d342c32b14c7edbece596ba970a8415._comment
@@ -0,0 +1,10 @@
+[[!comment format=mdwn
+ username="http://joeyh.name/"
+ ip="209.250.56.43"
+ subject="comment 3"
+ date="2014-01-15T20:40:35Z"
+ content="""
+A bug in the git repository repair code caused packed-refs files to be exploded to .gitrefs instead of .git/refs. I have fixed that bug.
+
+To fix up a repository that this happened to, I'd recommend checking if there are any tags and branches in the .gitrefs directory that have gone missing from .git/refs.. If so, you could move the files back into place (unannexing them first if necessary). Otherwise, deleting .gitrefs would seem to be the thing to do.
+"""]]
diff --git a/doc/forum/How_do_I_dropunused_with_an_rsync_remote__63__.mdwn b/doc/forum/How_do_I_dropunused_with_an_rsync_remote__63__.mdwn
new file mode 100644
index 0000000000..c9a5a6c0a4
--- /dev/null
+++ b/doc/forum/How_do_I_dropunused_with_an_rsync_remote__63__.mdwn
@@ -0,0 +1,3 @@
+`git annex dropunused` is simple enough.
+
+But how do I perform the equivalent procedure on an rsync remote?  I'd presume that `git annex copy --to remote` is not sufficient.
diff --git a/doc/forum/How_do_I_dropunused_with_an_rsync_remote__63__/comment_1_8db3cb5348b845eb99c2c829957db9ea._comment b/doc/forum/How_do_I_dropunused_with_an_rsync_remote__63__/comment_1_8db3cb5348b845eb99c2c829957db9ea._comment
new file mode 100644
index 0000000000..026d393bb5
--- /dev/null
+++ b/doc/forum/How_do_I_dropunused_with_an_rsync_remote__63__/comment_1_8db3cb5348b845eb99c2c829957db9ea._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="https://www.google.com/accounts/o8/id?id=AItOawmUJBh1lYmvfCCiGr3yrdx-QhuLCSRnU5c"
+ nickname="Justin"
+ subject="comment 1"
+ date="2013-05-12T23:06:34Z"
+ content="""
+Oh, sheesh, it's right there at the bottom of http://git-annex.branchable.com/special_remotes/.  Oops.
+"""]]
diff --git a/doc/forum/How_do_I_dropunused_with_an_rsync_remote__63__/comment_2_6cc909d9d74bc1ccb8a7b0d7d234c7cd._comment b/doc/forum/How_do_I_dropunused_with_an_rsync_remote__63__/comment_2_6cc909d9d74bc1ccb8a7b0d7d234c7cd._comment
new file mode 100644
index 0000000000..2df92a8a36
--- /dev/null
+++ b/doc/forum/How_do_I_dropunused_with_an_rsync_remote__63__/comment_2_6cc909d9d74bc1ccb8a7b0d7d234c7cd._comment
@@ -0,0 +1,10 @@
+[[!comment format=mdwn
+ username="http://joeyh.name/"
+ nickname="joey"
+ subject="comment 2"
+ date="2013-05-13T18:03:23Z"
+ content="""
+Well, in your defense it's documented there because I can't think of a better place to put it. Maybe you looked in some other places first before finding it there, and I should add a pointer to those places?
+
+(For the curious, the magic run is just: git annex unused --from remote)
+"""]]
diff --git a/doc/forum/How_do_I_dropunused_with_an_rsync_remote__63__/comment_3_f24d678e4192a70322aa164ed9b71fc8._comment b/doc/forum/How_do_I_dropunused_with_an_rsync_remote__63__/comment_3_f24d678e4192a70322aa164ed9b71fc8._comment
new file mode 100644
index 0000000000..2b53984c37
--- /dev/null
+++ b/doc/forum/How_do_I_dropunused_with_an_rsync_remote__63__/comment_3_f24d678e4192a70322aa164ed9b71fc8._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="https://www.google.com/accounts/o8/id?id=AItOawmUJBh1lYmvfCCiGr3yrdx-QhuLCSRnU5c"
+ nickname="Justin"
+ subject="comment 3"
+ date="2013-05-13T18:56:32Z"
+ content="""
+I think in general the `git annex command --help` bits could be more thorough.  If `git annex dropunused --help` indicated that it took `--from`, that would probably have helped me.
+"""]]
diff --git a/doc/forum/How_do_I_dropunused_with_an_rsync_remote__63__/comment_4_9233decd0aaf9211447f36e0d9346445._comment b/doc/forum/How_do_I_dropunused_with_an_rsync_remote__63__/comment_4_9233decd0aaf9211447f36e0d9346445._comment
new file mode 100644
index 0000000000..f326cd3400
--- /dev/null
+++ b/doc/forum/How_do_I_dropunused_with_an_rsync_remote__63__/comment_4_9233decd0aaf9211447f36e0d9346445._comment
@@ -0,0 +1,15 @@
+[[!comment format=mdwn
+ username="http://joeyh.name/"
+ nickname="joey"
+ subject="comment 4"
+ date="2013-05-13T18:59:08Z"
+ content="""
+You might be using an older version. Here's the output of that command in the current version:
+
+
    +Usage: git-annex dropunused NUM|RANGE ... [option ...]
+  -f REMOTE  --from=REMOTE  drop content from a remote
+
+To see additional options common to all commands, run: git annex help options
+
    
+"""]]
diff --git a/doc/forum/How_do_I_dropunused_with_an_rsync_remote__63__/comment_5_e1deb110f752e5495d5c77ec444abac5._comment b/doc/forum/How_do_I_dropunused_with_an_rsync_remote__63__/comment_5_e1deb110f752e5495d5c77ec444abac5._comment
new file mode 100644
index 0000000000..e9ca6bc138
--- /dev/null
+++ b/doc/forum/How_do_I_dropunused_with_an_rsync_remote__63__/comment_5_e1deb110f752e5495d5c77ec444abac5._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="https://www.google.com/accounts/o8/id?id=AItOawmUJBh1lYmvfCCiGr3yrdx-QhuLCSRnU5c"
+ nickname="Justin"
+ subject="comment 5"
+ date="2013-05-13T19:07:11Z"
+ content="""
+Ah, rebuilding got me that.  That's pretty good!
+"""]]
diff --git a/doc/forum/How_do_I_get_rid_of_a_wrong_remote_uuid__63__.mdwn b/doc/forum/How_do_I_get_rid_of_a_wrong_remote_uuid__63__.mdwn
new file mode 100644
index 0000000000..209e0f6a78
--- /dev/null
+++ b/doc/forum/How_do_I_get_rid_of_a_wrong_remote_uuid__63__.mdwn
@@ -0,0 +1,16 @@
+Hello,
+
+I had some trouble adding a remote (the files would not appear when I was copying them to the remote), so I started over and cloned an existing repository.
+
+Of course, as I started over, I had a duplicate uuid for the remote, which would cause problems when trying to copy (I would have an error "git-annex-shell: expected repository UUID 70582c7b-0b57-4087-a9d1-77b5f5f3c75e but found UUID 335699ea-d5b8-49ff-b207-1571b5969afe").
+
+I finally managed to find the wrong uuid in the .git/config file (there was a duplicated entry for the remote) and I'm now able to copy things to the remote repository, and "git annex sync" works well. However I still see a mention of this repository when I do a "git annex whereis":
+
+    cody:games schmitta$ git annex whereis
+    whereis dungeon_keeper_1.1.0.11.dmg (3 copies) 
+      	1cdfb490-0660-41fb-b7ce-74b89abb9aac -- top
+       	335699ea-d5b8-49ff-b207-1571b5969afe -- here (cody)
+       	70582c7b-0b57-4087-a9d1-77b5f5f3c75e
+
+
+Where can I find where this last line come from, and how can I get rid of it? I tried saying that this uuid is dead, but git annex tells me it's not a remote name.
diff --git a/doc/forum/How_do_I_get_rid_of_a_wrong_remote_uuid__63__/comment_1_b3c215cedba51fb47992ef10c60d6acc._comment b/doc/forum/How_do_I_get_rid_of_a_wrong_remote_uuid__63__/comment_1_b3c215cedba51fb47992ef10c60d6acc._comment
new file mode 100644
index 0000000000..80b664dced
--- /dev/null
+++ b/doc/forum/How_do_I_get_rid_of_a_wrong_remote_uuid__63__/comment_1_b3c215cedba51fb47992ef10c60d6acc._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="http://alan.petitepomme.net/"
+ nickname="Alan Schmitt"
+ subject="comment 1"
+ date="2014-03-08T06:42:29Z"
+ content="""
+If it helps, I had a look at \"git annex vicfg\", and there is no mention of the extra UUID there.
+"""]]
diff --git a/doc/forum/How_do_I_get_rid_of_a_wrong_remote_uuid__63__/comment_2_85415e1fceb737919cc1cd9f37242458._comment b/doc/forum/How_do_I_get_rid_of_a_wrong_remote_uuid__63__/comment_2_85415e1fceb737919cc1cd9f37242458._comment
new file mode 100644
index 0000000000..a251dade38
--- /dev/null
+++ b/doc/forum/How_do_I_get_rid_of_a_wrong_remote_uuid__63__/comment_2_85415e1fceb737919cc1cd9f37242458._comment
@@ -0,0 +1,10 @@
+[[!comment format=mdwn
+ username="http://joeyh.name/"
+ ip="209.250.56.154"
+ subject="comment 2"
+ date="2014-03-12T19:35:26Z"
+ content="""
+Probably this will fix it:
+
+`git annex dead 70582c7b-0b57-4087-a9d1-77b5f5f3c75e`
+"""]]
diff --git a/doc/forum/How_do_I_get_rid_of_a_wrong_remote_uuid__63__/comment_3_fb3a591dc60182f7922fc2b5c24f50f1._comment b/doc/forum/How_do_I_get_rid_of_a_wrong_remote_uuid__63__/comment_3_fb3a591dc60182f7922fc2b5c24f50f1._comment
new file mode 100644
index 0000000000..bebd8f290d
--- /dev/null
+++ b/doc/forum/How_do_I_get_rid_of_a_wrong_remote_uuid__63__/comment_3_fb3a591dc60182f7922fc2b5c24f50f1._comment
@@ -0,0 +1,11 @@
+[[!comment format=mdwn
+ username="http://alan.petitepomme.net/"
+ nickname="Alan Schmitt"
+ subject="comment 3"
+ date="2014-03-13T20:05:53Z"
+ content="""
+Unfortunately git annex tells me there is no such remote:
+
+    cody:games schmitta$ git annex dead 70582c7b-0b57-4087-a9d1-77b5f5f3c75e
+    dead 70582c7b-0b57-4087-a9d1-77b5f5f3c75e git-annex: there is no available git remote named \"70582c7b-0b57-4087-a9d1-77b5f5f3c75e\"
+"""]]
diff --git a/doc/forum/How_do_I_get_rid_of_a_wrong_remote_uuid__63__/comment_4_aed0be32e579c7a39c63aa7e3ec5f67b._comment b/doc/forum/How_do_I_get_rid_of_a_wrong_remote_uuid__63__/comment_4_aed0be32e579c7a39c63aa7e3ec5f67b._comment
new file mode 100644
index 0000000000..99ac0d5a6a
--- /dev/null
+++ b/doc/forum/How_do_I_get_rid_of_a_wrong_remote_uuid__63__/comment_4_aed0be32e579c7a39c63aa7e3ec5f67b._comment
@@ -0,0 +1,29 @@
+[[!comment format=mdwn
+ username="http://joeyh.name/"
+ ip="209.250.56.154"
+ subject="comment 4"
+ date="2014-03-14T17:46:23Z"
+ content="""
+It seems that the git-annex branch's uuid.log must somehow not list this uuid, but it's used in the location tracking log files. 
+
+The only way I can think of that this could happen is if you had set up a repository, run git-annex init, and then went in and changed the annex.uuid setting to this other uuid, and added files with that misconfiguration. Does that sound like what happened?
+
+The fix is just as evil as the cause -- you can edit .git/config to add a new, dummy git remote that has annex-uuid set to the problem uuid, and then `git-annex dead` can be used to kill the uuid via that remote. For example:
+
+[[!format sh \"\"\"
+joey@darkstar:~/tmp/x>git annex whereis eep
+whereis eep (1 copy) 
+  	00000a6d-e770-4ab9-a640-7d6272e9ffff
+ok
+joey@darkstar:~/tmp/x>git annex dead 00000a6d-e770-4ab9-a640-7d6272e9ffff
+dead 00000a6d-e770-4ab9-a640-7d6272e9ffff git-annex: there is no available git remote named \"00000a6d-e770-4ab9-a640-7d6272e9ffff\"
+- exit 1
+joey@darkstar:~/tmp/x>git remote add dummy dummy
+joey@darkstar:~/tmp/x>git config remote.dummy.annex-uuid 00000a6d-e770-4ab9-a640-7d6272e9ffff
+joey@darkstar:~/tmp/x>git annex dead dummy
+dead dummy ok
+(Recording state in git...)
+joey@darkstar:~/tmp/x>git annex whereis eep
+whereis eep (0 copies) failed
+\"\"\"]]
+"""]]
diff --git a/doc/forum/How_do_I_get_rid_of_a_wrong_remote_uuid__63__/comment_5_0c9a6c8a92d6c6e04ae3a8349b799c60._comment b/doc/forum/How_do_I_get_rid_of_a_wrong_remote_uuid__63__/comment_5_0c9a6c8a92d6c6e04ae3a8349b799c60._comment
new file mode 100644
index 0000000000..9807fb7d9f
--- /dev/null
+++ b/doc/forum/How_do_I_get_rid_of_a_wrong_remote_uuid__63__/comment_5_0c9a6c8a92d6c6e04ae3a8349b799c60._comment
@@ -0,0 +1,22 @@
+[[!comment format=mdwn
+ username="http://alan.petitepomme.net/"
+ nickname="Alan Schmitt"
+ subject="comment 5"
+ date="2014-03-14T18:11:52Z"
+ content="""
+Thanks, it worked!
+
+Regarding how I got into that state, here is what happened:
+- I configured a new repository (git init, git annex init) on machine B
+- I added that repository as a remote from machine A
+- I started pushing files from machine A to machine B, starting with the file that later showed this problem
+- I then noticed that, although the files were said to be successfully sent, I could not see them on machine B (no symbolic link was created), even after a \"git annex sync\" there
+- I decided to start over, deleting the annex, and creating it from a clone of an existing annex
+- This resulted in a duplicated UUID for the same path, which led to this problem
+
+Maybe some documentation on how to cleanly start over when things get in a bad state would be very useful.
+
+Thanks again,
+
+Alan
+"""]]
diff --git a/doc/forum/How_do_I_hide_files_not_present_in_the_local_annex__63__.mdwn b/doc/forum/How_do_I_hide_files_not_present_in_the_local_annex__63__.mdwn
new file mode 100644
index 0000000000..72e39cc360
--- /dev/null
+++ b/doc/forum/How_do_I_hide_files_not_present_in_the_local_annex__63__.mdwn
@@ -0,0 +1,2 @@
+Files only present in remotes show up as broken symlinks. That's great for knowing what files exist, but sometimes I just want to browse the files that are actually present. In this case, the many broken symlinks are just clutter.
+Is there a straightforward way to switch to a view that shows only locally present files?
diff --git a/doc/forum/How_do_I_hide_files_not_present_in_the_local_annex__63__/comment_1_005e9254a1239164df34ab5fbf2115a8._comment b/doc/forum/How_do_I_hide_files_not_present_in_the_local_annex__63__/comment_1_005e9254a1239164df34ab5fbf2115a8._comment
new file mode 100644
index 0000000000..b15296b33b
--- /dev/null
+++ b/doc/forum/How_do_I_hide_files_not_present_in_the_local_annex__63__/comment_1_005e9254a1239164df34ab5fbf2115a8._comment
@@ -0,0 +1,27 @@
+[[!comment format=mdwn
+ username="https://www.google.com/accounts/o8/id?id=AItOawlZ-6dtxJY4cP7shhvV8E6YyuV0Rak8it4"
+ nickname="Giovanni"
+ subject="direct mode: avoid broken symlinks; indirect mode: matching-options in views (not woring)"
+ date="2015-04-18T07:56:32Z"
+ content="""
+I would like to see only locally present files too, for some of my use case (e.g.: movies exported via NFS to a XBMS/Kodi media player)
+
+I'm using git annex (debian) version 5.20141024~bpo70+1
+
+# [direct mode](http://git-annex.branchable.com/direct_mode/)
+
+as far as I understand, in direct mode broken symlinks (or text files placeholders for filesystems not supporting symlinks) are there by [design](
+https://git-annex.branchable.com/design/assistant/desymlink/), in particular in [partial content](
+https://git-annex.branchable.com/design/assistant/partial_content/) use case where \"there needs to be a way for the user to browse files not on the gadget and request they be transferred to it\"
+
+if there are no other reasons to have broken symlinks (or placeholders) I think git-annex should avoid them (at least in direct mode): if a user needs a file not listed by regular filesystem tools she can simply `git-annex find` it and `git-annex get` it: or do I miss something here?
+
+please consider avoiding broken symlinks or placeholders
+
+# indirect mode
+
+in indirect mode we could live with broken symlinks using views; I tried `git annex view /=* --include=\"*\" --in=here` but the resulting tree was not filtered by matching-options
+
+are matching-options intended to work with the view command or not?
+
+"""]]
diff --git a/doc/forum/How_do_I_hide_files_not_present_in_the_local_annex__63__/comment_2_f376604560c36a0aa5afa4619797b396._comment b/doc/forum/How_do_I_hide_files_not_present_in_the_local_annex__63__/comment_2_f376604560c36a0aa5afa4619797b396._comment
new file mode 100644
index 0000000000..27197fc82a
--- /dev/null
+++ b/doc/forum/How_do_I_hide_files_not_present_in_the_local_annex__63__/comment_2_f376604560c36a0aa5afa4619797b396._comment
@@ -0,0 +1,9 @@
+[[!comment format=mdwn
+ username="GiovanniBiscuolo"
+ subject="hide files not present in the local annex: related forum post"
+ date="2015-04-23T11:00:56Z"
+ content="""
+[How to hide broken symlinks](http://git-annex.branchable.com/forum/How_to_hide_broken_symlinks/)
+
+using views (but see comments)
+"""]]
diff --git a/doc/forum/How_do_I_list_links_with_0_copies__63__.mdwn b/doc/forum/How_do_I_list_links_with_0_copies__63__.mdwn
new file mode 100644
index 0000000000..663549ee28
--- /dev/null
+++ b/doc/forum/How_do_I_list_links_with_0_copies__63__.mdwn
@@ -0,0 +1,5 @@
+I've been searching around and can't find this. I don't need `git annex unused`, but something that'll list the files I should "git rm".
+
+The best solution I've found so far is:
+
+    git annex whereis|grep '0 copies.*failed'|perl -pe 's/^whereis (.*?) \(0 copies\) failed$/$1/'
diff --git a/doc/forum/How_do_I_list_links_with_0_copies__63__/comment_1_ee222f0e227f1c65f84f73af399239e5._comment b/doc/forum/How_do_I_list_links_with_0_copies__63__/comment_1_ee222f0e227f1c65f84f73af399239e5._comment
new file mode 100644
index 0000000000..fdebc58a80
--- /dev/null
+++ b/doc/forum/How_do_I_list_links_with_0_copies__63__/comment_1_ee222f0e227f1c65f84f73af399239e5._comment
@@ -0,0 +1,10 @@
+[[!comment format=mdwn
+ username="joey"
+ subject="""comment 1"""
+ date="2016-12-28T00:15:34Z"
+ content="""
+	git annex find --not --copies 1
+
+This works because --copies 1 finds all files with 1 or more copies,
+and so inverting it with --not finds all files with fewer copies.
+"""]]
diff --git a/doc/forum/How_do_I_list_links_with_0_copies__63__/comment_2_1528b3c96d6ad5637c2fdda33057c12f._comment b/doc/forum/How_do_I_list_links_with_0_copies__63__/comment_2_1528b3c96d6ad5637c2fdda33057c12f._comment
new file mode 100644
index 0000000000..ca54006289
--- /dev/null
+++ b/doc/forum/How_do_I_list_links_with_0_copies__63__/comment_2_1528b3c96d6ad5637c2fdda33057c12f._comment
@@ -0,0 +1,10 @@
+[[!comment format=mdwn
+ username="avar"
+ avatar="http://cdn.libravatar.org/avatar/57332d67a86eb51e06bf78d2baf42c3c"
+ subject="comment 2"
+ date="2016-12-28T08:28:35Z"
+ content="""
+Great, this one-liner is then exactly what I needed:
+
+    git annex find --not --copies 1 --print0 | xargs -0 git rm
+"""]]
diff --git a/doc/forum/How_do_I_list_links_with_0_copies__63__/comment_3_e629a5393e56755a42581be9f6942adb._comment b/doc/forum/How_do_I_list_links_with_0_copies__63__/comment_3_e629a5393e56755a42581be9f6942adb._comment
new file mode 100644
index 0000000000..5470332252
--- /dev/null
+++ b/doc/forum/How_do_I_list_links_with_0_copies__63__/comment_3_e629a5393e56755a42581be9f6942adb._comment
@@ -0,0 +1,12 @@
+[[!comment format=mdwn
+ username="ghen1"
+ avatar="http://cdn.libravatar.org/avatar/efd0e92b6198291138f0cd7aedbf86b6"
+ subject="comment 3"
+ date="2016-12-28T11:16:17Z"
+ content="""
+When I first read this:
+
+    git annex find --not --copies 1
+
+I expected it would also select files with *more* than 1 copy. Nothing wrong with it if it works; just seems semantically ambiguous to me.
+"""]]
diff --git a/doc/forum/How_do_I_revert_back_to_specific_commit_with_git_annex.mdwn b/doc/forum/How_do_I_revert_back_to_specific_commit_with_git_annex.mdwn
new file mode 100644
index 0000000000..90856383a8
--- /dev/null
+++ b/doc/forum/How_do_I_revert_back_to_specific_commit_with_git_annex.mdwn
@@ -0,0 +1,8 @@
+I want to go back to a specific commit in git annex as I lost most recent copies of a file and want to revert back to a previous revision, few months old. All remotes are dead but after reading few article, there might be a way to get the file back from the objects directory:
+
+git annex whereis image 
+whereis image (0 copies) failed
+git-annex: whereis: 1 failed
+
+Thanks,
+
diff --git a/doc/forum/How_do_I_revert_back_to_specific_commit_with_git_annex/comment_1_ab8896c4f0e20fb351477e3dc92a35d8._comment b/doc/forum/How_do_I_revert_back_to_specific_commit_with_git_annex/comment_1_ab8896c4f0e20fb351477e3dc92a35d8._comment
new file mode 100644
index 0000000000..fa99665fd8
--- /dev/null
+++ b/doc/forum/How_do_I_revert_back_to_specific_commit_with_git_annex/comment_1_ab8896c4f0e20fb351477e3dc92a35d8._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="joey"
+ subject="""comment 1"""
+ date="2017-08-28T17:37:11Z"
+ content="""
+This works the same with git-annex as it does with regular git. You can
+use `git revert` or `git reset` or whatever.
+"""]]
diff --git a/doc/forum/How_do_I_sync_files_from_mobile_to_a_repo__63__.mdwn b/doc/forum/How_do_I_sync_files_from_mobile_to_a_repo__63__.mdwn
new file mode 100644
index 0000000000..d63af388ff
--- /dev/null
+++ b/doc/forum/How_do_I_sync_files_from_mobile_to_a_repo__63__.mdwn
@@ -0,0 +1,13 @@
+I have a local repo on a computer with some folder on the path
+/computer1/annex/
+
+Then I tried to run git annex on my phone and added a remote ssh repo syncing to
+/computer1/annex/mobile/ from my mobile's picture folder.
+
+But the files ending up there are:
+ls /computer1/annex/mobile/
+annex  branches  config  description  HEAD  hooks  info  objects  refs 
+
+I chose "full backup"
+
+Why dont I see my pictures there, even if they are hiding in the metadata ?
diff --git a/doc/forum/How_do_I_sync_files_from_mobile_to_a_repo__63__/comment_1_fc914b5998a09943fc8c1917a0e36096._comment b/doc/forum/How_do_I_sync_files_from_mobile_to_a_repo__63__/comment_1_fc914b5998a09943fc8c1917a0e36096._comment
new file mode 100644
index 0000000000..ac49670b9e
--- /dev/null
+++ b/doc/forum/How_do_I_sync_files_from_mobile_to_a_repo__63__/comment_1_fc914b5998a09943fc8c1917a0e36096._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="Petter_petterson"
+ ip="89.160.15.173"
+ subject="addition"
+ date="2014-09-13T07:54:58Z"
+ content="""
+I understand that the copy of the cellphones' photos are stored on the server too, when typing git annex whereis  I see that it exists on the server, but I need to be able to, at will copy out the jpg files for editing and using in other places.
+"""]]
diff --git a/doc/forum/How_do_I_sync_files_from_mobile_to_a_repo__63__/comment_2_fd5257eff7f94971557c031a94ac2766._comment b/doc/forum/How_do_I_sync_files_from_mobile_to_a_repo__63__/comment_2_fd5257eff7f94971557c031a94ac2766._comment
new file mode 100644
index 0000000000..97724fd18a
--- /dev/null
+++ b/doc/forum/How_do_I_sync_files_from_mobile_to_a_repo__63__/comment_2_fd5257eff7f94971557c031a94ac2766._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="https://www.google.com/accounts/o8/id?id=AItOawmBUR4O9mofxVbpb8JV9mEbVfIYv670uJo"
+ nickname="Justin"
+ subject="comment 2"
+ date="2014-09-14T18:57:12Z"
+ content="""
+If `/computer1/annex/` was already an annex repository you should have synced the phone to that, not to a new bare repository at `/computer1/annex/mobile/`
+"""]]
diff --git a/doc/forum/How_do_I_sync_files_from_mobile_to_a_repo__63__/comment_3_5a11c45f92bae1328a5120945bee1fa0._comment b/doc/forum/How_do_I_sync_files_from_mobile_to_a_repo__63__/comment_3_5a11c45f92bae1328a5120945bee1fa0._comment
new file mode 100644
index 0000000000..8a9f6cd181
--- /dev/null
+++ b/doc/forum/How_do_I_sync_files_from_mobile_to_a_repo__63__/comment_3_5a11c45f92bae1328a5120945bee1fa0._comment
@@ -0,0 +1,36 @@
+[[!comment format=mdwn
+ username="Petter_petterson"
+ ip="89.160.15.173"
+ subject="comment 3"
+ date="2014-09-16T08:13:59Z"
+ content="""
+Thanks Justin, but that wont work. Even pointing out a normal, non-bare repo and then adding it as a ssh remote will convert it into a bare repo. I confirmed that, and then I read this post:
+
+    http://git-annex.branchable.com/forum/Local_and_remote_in_direct_mode/
+
+that states that
+> The \"Remote server using ssh\" option in the webapp is intended to set up a bare git repository on a server, not a non-bare git repository on a client.\"
+
+I even tried to do 
+    git remote add B ssh://machineB:/~/annex
+but to no avail, the created annex on machine B becomes a bare repo.
+
+The only way to do it for me was to do the following,
+Assume my cellphone is device A, and my desktop is device B:
+
+On machine B:
+
+    cd ~/DCIM
+    git init
+    git annex init \"B\"
+    git annex direct
+    echo '*/5 * * * * * cd ~/DCIM; git annex sync' > crontab
+
+On machine A:
+
+    git clone ssh://user@machineB:/home/user/DCIM
+    git annex sync
+    git annex webapp
+
+now pictures are synced to the computer in direct, non-bare format every 5 minutes. I have spent literally days on this and now I finally nailed it in a crude but working fashion. 
+"""]]
diff --git a/doc/forum/How_do_I_sync_files_from_mobile_to_a_repo__63__/comment_4_404a8f9daa86c20a046b4c9f9051dfc0._comment b/doc/forum/How_do_I_sync_files_from_mobile_to_a_repo__63__/comment_4_404a8f9daa86c20a046b4c9f9051dfc0._comment
new file mode 100644
index 0000000000..cbe73a00bf
--- /dev/null
+++ b/doc/forum/How_do_I_sync_files_from_mobile_to_a_repo__63__/comment_4_404a8f9daa86c20a046b4c9f9051dfc0._comment
@@ -0,0 +1,16 @@
+[[!comment format=mdwn
+ username="http://joeyh.name/"
+ ip="108.236.230.124"
+ subject="comment 4"
+ date="2014-09-18T18:00:42Z"
+ content="""
+I have double-checked, and when I already have an existing, non-bare repository, pointing the webapp at it over ssh keeps it as a non-bare repository. As I would expect.
+
+> I even tried to do git remote add B ssh://machineB:/~/annex but to no avail, the created annex on machine B becomes a bare repo.
+
+I didn't try this because it's such a violation of the way git actually works that I just can't believe it could happen. If it does, you've found the git bug of the year.
+
+But, I think you just got confused about whether the repository existed before, or gave the wrong path to the existing repository which would result in a new, bare repository being created in the location you told it.
+
+If you really think this happens, show a transcript with enough details for me, or the git developers, to reproduce the problem.
+"""]]
diff --git a/doc/forum/How_do_I_sync_files_from_mobile_to_a_repo__63__/comment_5_3dcdaef370d0df38e7285f1fa11c6bb3._comment b/doc/forum/How_do_I_sync_files_from_mobile_to_a_repo__63__/comment_5_3dcdaef370d0df38e7285f1fa11c6bb3._comment
new file mode 100644
index 0000000000..7bcce97062
--- /dev/null
+++ b/doc/forum/How_do_I_sync_files_from_mobile_to_a_repo__63__/comment_5_3dcdaef370d0df38e7285f1fa11c6bb3._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="http://joeyh.name/"
+ ip="108.236.230.124"
+ subject="comment 5"
+ date="2014-09-18T18:05:33Z"
+ content="""
+It occurs to me that another way you could have gotten confused is, if ssh://machineB:/~/annex was eg, created in the first place by running the git-annex webapp on machineB, then it would be a direct mode repo. In this case, yes core.bare=true, but so does annex.direct=true. And that repository will not be a bare repo really; it will contain the same file tree as you have on your mobile.
+"""]]
diff --git a/doc/forum/How_do_you_know_when_something_fails_a_fsck__63__.mdwn b/doc/forum/How_do_you_know_when_something_fails_a_fsck__63__.mdwn
new file mode 100644
index 0000000000..d574956bf1
--- /dev/null
+++ b/doc/forum/How_do_you_know_when_something_fails_a_fsck__63__.mdwn
@@ -0,0 +1,4 @@
+Hi Joey,
+    I was wondering how you know when a file in your repo fails a fsck.  I'm having trouble finding out since I can't change any of the objects (which is great!).  I'd just like to know since I have thousands of files in one of my annexes, and scrolling through my term buffer is unwieldy.  I'd assume it would show up when the fsck was done, or in 'git annex status'...
+
+Thanks!
diff --git a/doc/forum/How_do_you_know_when_something_fails_a_fsck__63__/comment_1_1c14981916dd55376d5e9f95023556cb._comment b/doc/forum/How_do_you_know_when_something_fails_a_fsck__63__/comment_1_1c14981916dd55376d5e9f95023556cb._comment
new file mode 100644
index 0000000000..042a1c46a2
--- /dev/null
+++ b/doc/forum/How_do_you_know_when_something_fails_a_fsck__63__/comment_1_1c14981916dd55376d5e9f95023556cb._comment
@@ -0,0 +1,32 @@
+[[!comment format=mdwn
+ username="https://www.google.com/accounts/o8/id?id=AItOawmBUR4O9mofxVbpb8JV9mEbVfIYv670uJo"
+ nickname="Justin"
+ subject="comment 1"
+ date="2013-06-10T14:23:25Z"
+ content="""
+    justin@box:/tmp/f$ git init
+    Initialized empty Git repository in /tmp/f/.git/
+    justin@box:/tmp/f$ git-annex init 
+    init  ok
+    (Recording state in git...)
+    justin@box:/tmp/f$ cp /etc/motd .
+    justin@box:/tmp/f$ git-annex add .
+    add motd (checksum...) ok
+    (Recording state in git...)
+    justin@box:/tmp/f$ git-annex fsck
+    fsck motd (checksum...) ok
+    justin@box:/tmp/f$ chmod +w motd 
+    justin@box:/tmp/f$ echo hi >> motd 
+    justin@box:/tmp/f$ git-annex fsck
+    fsck motd 
+      Bad file size (3 B larger); moved to /tmp/f/.git/annex/bad/SHA256-s354--2e724dde1a5dc33bc15580b2aef1ee541ca8047d746fff9bb7917062b871c0bf
+
+      ** No known copies exist of motd
+    failed
+    (Recording state in git...)
+    git-annex: fsck: 1 failed
+
+
+You can also use fsck -q which will only show errors.
+
+"""]]
diff --git a/doc/forum/How_does_git-annex_handle_rsyncing_between_different_OSes_with_regards_to_UTF-8__63__.mdwn b/doc/forum/How_does_git-annex_handle_rsyncing_between_different_OSes_with_regards_to_UTF-8__63__.mdwn
new file mode 100644
index 0000000000..5bbbd59700
--- /dev/null
+++ b/doc/forum/How_does_git-annex_handle_rsyncing_between_different_OSes_with_regards_to_UTF-8__63__.mdwn
@@ -0,0 +1,4 @@
+I'm interested in using git-annex on a freebsd system and an OSX system. 
+I have previously had problems with manually rsyncing due to the fact that apparently different types of UTF-8 are used namely:  UTF-8 NFC & UTF-8 NFD so one option is to use  rsync -a --iconv=utf-8-mac,utf-8 localdir/ mynas:remotedir/ or alternatively, I can mount the files on the NAS via a cifs share and rsync to it...
+
+just curios how git-annex deals with this issue or if i will run into issues due to filenames containing special characters?
diff --git a/doc/forum/How_does_git-annex_handle_rsyncing_between_different_OSes_with_regards_to_UTF-8__63__/comment_1_f4b439f7d9ce20d4b5dc8d728cb2e180._comment b/doc/forum/How_does_git-annex_handle_rsyncing_between_different_OSes_with_regards_to_UTF-8__63__/comment_1_f4b439f7d9ce20d4b5dc8d728cb2e180._comment
new file mode 100644
index 0000000000..a261a0dc01
--- /dev/null
+++ b/doc/forum/How_does_git-annex_handle_rsyncing_between_different_OSes_with_regards_to_UTF-8__63__/comment_1_f4b439f7d9ce20d4b5dc8d728cb2e180._comment
@@ -0,0 +1,13 @@
+[[!comment format=mdwn
+ username="joey"
+ subject="""comment 1"""
+ date="2015-11-10T19:19:47Z"
+ content="""
+Both git and git-annex both treat filenames as strings of bytes, with no
+particular encoding, to the extent supported by the OS[1]. This may result
+in a checkout of git repository having filenames that don't display well on
+a different system.
+
+[1] Of course any reasonable Unix treats filenames like that too.. but
+Windows only supports unicode filenames.
+"""]]
diff --git a/doc/forum/How_does_git-annex_handle_rsyncing_between_different_OSes_with_regards_to_UTF-8__63__/comment_2_7393a4b2e94f9d36c3c9ca977a8f67b6._comment b/doc/forum/How_does_git-annex_handle_rsyncing_between_different_OSes_with_regards_to_UTF-8__63__/comment_2_7393a4b2e94f9d36c3c9ca977a8f67b6._comment
new file mode 100644
index 0000000000..eb12e51536
--- /dev/null
+++ b/doc/forum/How_does_git-annex_handle_rsyncing_between_different_OSes_with_regards_to_UTF-8__63__/comment_2_7393a4b2e94f9d36c3c9ca977a8f67b6._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="grawity@2ea26be48562f66fcb9b66307da72b1e2e37453f"
+ nickname="grawity"
+ subject="comment 2"
+ date="2016-01-05T11:09:07Z"
+ content="""
+Git does have a `core.precomposeUnicode` option for converting NFD to NFC; would that work?
+"""]]
diff --git a/doc/forum/How_does_git-annex_handle_rsyncing_between_different_OSes_with_regards_to_UTF-8__63__/comment_3_a38e2bbe229cb7e1f4f334bd93d91656._comment b/doc/forum/How_does_git-annex_handle_rsyncing_between_different_OSes_with_regards_to_UTF-8__63__/comment_3_a38e2bbe229cb7e1f4f334bd93d91656._comment
new file mode 100644
index 0000000000..a303be65ea
--- /dev/null
+++ b/doc/forum/How_does_git-annex_handle_rsyncing_between_different_OSes_with_regards_to_UTF-8__63__/comment_3_a38e2bbe229cb7e1f4f334bd93d91656._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="ovidiu@66ace8a8d99ce938b0538ffa0f26d30db02a9626"
+ nickname="ovidiu"
+ subject="comment 3"
+ date="2016-01-06T17:00:38Z"
+ content="""
+I can't add much to the conversation, I only raised the issue as I had run into it with rsync as described above.
+"""]]
diff --git a/doc/forum/How_does_git_annex_handle_swp_files.mdwn b/doc/forum/How_does_git_annex_handle_swp_files.mdwn
new file mode 100644
index 0000000000..0c07731202
--- /dev/null
+++ b/doc/forum/How_does_git_annex_handle_swp_files.mdwn
@@ -0,0 +1 @@
+Sometime I edit a file with lets say Emacs in the work directory which is synced with all my other computers. Then there is this .something.swp file. This is put into the queue for sync, but if I close this before it starts the sync I get an error since the file is now gone. Then I get a failed message in the webapp. This is slightly annoying. Is there a way to fix this in some way. I can of course disable the swp use, but this is default for different programs. Is there a way to exclude files? For instance all .swp files from ever getting synced?
diff --git a/doc/forum/How_does_git_annex_handle_swp_files/comment_1_8f0b81139fb66f28f9262e919692ecbd._comment b/doc/forum/How_does_git_annex_handle_swp_files/comment_1_8f0b81139fb66f28f9262e919692ecbd._comment
new file mode 100644
index 0000000000..df946a7a1f
--- /dev/null
+++ b/doc/forum/How_does_git_annex_handle_swp_files/comment_1_8f0b81139fb66f28f9262e919692ecbd._comment
@@ -0,0 +1,7 @@
+[[!comment format=mdwn
+ username="joey"
+ subject="""comment 1"""
+ date="2014-11-05T16:14:06Z"
+ content="""
+You can set up a .gitignore file containing `*.swp` or whatever.
+"""]]
diff --git a/doc/forum/How_does_one_change_git-annex_assistant__39__s_web_browser__63__.mdwn b/doc/forum/How_does_one_change_git-annex_assistant__39__s_web_browser__63__.mdwn
new file mode 100644
index 0000000000..a2ac090ad5
--- /dev/null
+++ b/doc/forum/How_does_one_change_git-annex_assistant__39__s_web_browser__63__.mdwn
@@ -0,0 +1,7 @@
+When starting git-annex assistant with:
+
+    git-annex webapp
+
+an unwanted web browser is opened.
+
+How does one change the web browser with which the web interface is displayed?
diff --git a/doc/forum/How_does_one_change_git-annex_assistant__39__s_web_browser__63__/comment_1_f4402eabda2327da3a0bbc64ed3baf9a._comment b/doc/forum/How_does_one_change_git-annex_assistant__39__s_web_browser__63__/comment_1_f4402eabda2327da3a0bbc64ed3baf9a._comment
new file mode 100644
index 0000000000..6369dbd0bb
--- /dev/null
+++ b/doc/forum/How_does_one_change_git-annex_assistant__39__s_web_browser__63__/comment_1_f4402eabda2327da3a0bbc64ed3baf9a._comment
@@ -0,0 +1,12 @@
+[[!comment format=mdwn
+ username="http://joeyh.name/"
+ ip="4.154.0.60"
+ subject="comment 1"
+ date="2012-09-17T15:32:07Z"
+ content="""
+The web browser is started by running the `xdg-open` program (or just `open` on Mac OS X).
+
+xdg-open, in turn, runs the desktop environment's kde-open, gvfs-open, exo-open. Which browser these run is configured in the desktop environment's settings. Which is my goal for the web app; it should open the same browser that other applications in the desktop use.
+
+When there's no desktop environment in use, xdg-open falls back to some default browser, or to what's configured by the BROWSER environment variable.
+"""]]
diff --git a/doc/forum/How_does_one_change_git-annex_assistant__39__s_web_browser__63__/comment_2_cdb41f2c7b6bc5bf40d88582dcbf45aa._comment b/doc/forum/How_does_one_change_git-annex_assistant__39__s_web_browser__63__/comment_2_cdb41f2c7b6bc5bf40d88582dcbf45aa._comment
new file mode 100644
index 0000000000..ce626fa9f5
--- /dev/null
+++ b/doc/forum/How_does_one_change_git-annex_assistant__39__s_web_browser__63__/comment_2_cdb41f2c7b6bc5bf40d88582dcbf45aa._comment
@@ -0,0 +1,10 @@
+[[!comment format=mdwn
+ username="https://www.google.com/accounts/o8/id?id=AItOawlZvtBCVyJw4O71OPsdwGpVh6iJ1W-xaPc"
+ nickname="Kilian"
+ subject="comment 2"
+ date="2012-10-11T21:56:26Z"
+ content="""
+On my Ubuntu 10.04 machine, ./git-annex-webapp starts Firefox whereas xdg-open starts my configured standard browser, Chrome.
+
+-ke
+"""]]
diff --git a/doc/forum/How_does_one_change_git-annex_assistant__39__s_web_browser__63__/comment_3_ca75e928c245eb23a02b5f40ec69cbb1._comment b/doc/forum/How_does_one_change_git-annex_assistant__39__s_web_browser__63__/comment_3_ca75e928c245eb23a02b5f40ec69cbb1._comment
new file mode 100644
index 0000000000..b2b605492c
--- /dev/null
+++ b/doc/forum/How_does_one_change_git-annex_assistant__39__s_web_browser__63__/comment_3_ca75e928c245eb23a02b5f40ec69cbb1._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="http://joeyh.name/"
+ ip="4.152.246.8"
+ subject="comment 3"
+ date="2012-10-11T23:41:36Z"
+ content="""
+Recently the webapp has started honoring git's web.browser setting, so if that's set in ~/.gitconfig, it'll use a different browser than xdg-open does.
+"""]]
diff --git a/doc/forum/How_does_one_change_git-annex_assistant__39__s_web_browser__63__/comment_4_1635f136909711295b9b70d1255e0378._comment b/doc/forum/How_does_one_change_git-annex_assistant__39__s_web_browser__63__/comment_4_1635f136909711295b9b70d1255e0378._comment
new file mode 100644
index 0000000000..e6d12ca5d0
--- /dev/null
+++ b/doc/forum/How_does_one_change_git-annex_assistant__39__s_web_browser__63__/comment_4_1635f136909711295b9b70d1255e0378._comment
@@ -0,0 +1,11 @@
+[[!comment format=mdwn
+ username="https://www.google.com/accounts/o8/id?id=AItOawnBJ6Dv1glxzzi4qIzGFNa6F-mfHIvv9Ck"
+ nickname="Jim"
+ subject="Doesn't use the configured web browser app..."
+ date="2012-12-04T23:10:43Z"
+ content="""
+On my system, `xdg-open http://google.com/` opens firefox as desired, but `xdg-open file:///tmp/foo.html` tries to load **wine**!
+I'm using xfce, and it seems like exo-open is noticing the `file://` URL, finding the mime type, and looking it up in `~/.local/share/applications/wine-extension-html.desktop`, which was (annoyingly) created by wine.  Deleting those files fixes it.
+
+Not sure how you'd make this better.  Maybe give xdg-open a URL like `http://localhost:54896/?authfile=...` that redirects to `file:///tmp/webapp12345.html` that redirects to `http://localhost:54896/?auth=...` ?
+"""]]
diff --git a/doc/forum/How_does_one_change_git-annex_assistant__39__s_web_browser__63__/comment_5_ee0cbe9498c518de98480a2ad229f685._comment b/doc/forum/How_does_one_change_git-annex_assistant__39__s_web_browser__63__/comment_5_ee0cbe9498c518de98480a2ad229f685._comment
new file mode 100644
index 0000000000..8dc9d74ecb
--- /dev/null
+++ b/doc/forum/How_does_one_change_git-annex_assistant__39__s_web_browser__63__/comment_5_ee0cbe9498c518de98480a2ad229f685._comment
@@ -0,0 +1,10 @@
+[[!comment format=mdwn
+ username="http://joeyh.name/"
+ ip="4.152.108.72"
+ subject="comment 5"
+ date="2012-12-05T01:21:03Z"
+ content="""
+@Jim, that would be a good technique.. except many browsers prohibit redirecting to file:// urls.
+
+
+"""]]
diff --git a/doc/forum/How_does_one_change_git-annex_assistant__39__s_web_browser__63__/comment_6_799b9d9d3ffbc2c14eca8d442e2aff8c._comment b/doc/forum/How_does_one_change_git-annex_assistant__39__s_web_browser__63__/comment_6_799b9d9d3ffbc2c14eca8d442e2aff8c._comment
new file mode 100644
index 0000000000..767426fefa
--- /dev/null
+++ b/doc/forum/How_does_one_change_git-annex_assistant__39__s_web_browser__63__/comment_6_799b9d9d3ffbc2c14eca8d442e2aff8c._comment
@@ -0,0 +1,11 @@
+[[!comment format=mdwn
+ username="https://www.google.com/accounts/o8/id?id=AItOawnZEanlyzay_QlEAL0CWpyZcRTyN7vay8U"
+ nickname="Carlo"
+ subject="comment 6"
+ date="2013-11-15T13:43:06Z"
+ content="""
+Since git web--browse defaults to firefox on my system (Ubuntu/XMonad), I did this on Ubuntu to get the system default browser.
+
+  git config --global web.browser xdg-open
+
+"""]]
diff --git a/doc/forum/How_does_one_change_git-annex_assistant__39__s_web_browser__63__/comment_7_71ff45948487e9ac8de809a5ccc3d874._comment b/doc/forum/How_does_one_change_git-annex_assistant__39__s_web_browser__63__/comment_7_71ff45948487e9ac8de809a5ccc3d874._comment
new file mode 100644
index 0000000000..115d402ad5
--- /dev/null
+++ b/doc/forum/How_does_one_change_git-annex_assistant__39__s_web_browser__63__/comment_7_71ff45948487e9ac8de809a5ccc3d874._comment
@@ -0,0 +1,29 @@
+[[!comment format=mdwn
+ username="rupi"
+ ip="80.110.152.13"
+ subject="xdg-open"
+ date="2014-02-28T17:48:23Z"
+ content="""
+Since I discovered the same behavior mentioned above (git annex webapp
+tries to use 'winebrowser'). There is an easy way to work with the difference
+between http
+url and html files: enter preferences for both:
+
+In ~/.local/share/applications/mimeapps.list:
+
+		[Added Associations]
+		..
+		x-scheme-handler/http=iceweasel.desktop;
+		x-scheme-handler/https=iceweasel.desktop;
+		text/html=icewesel.desktop;
+
+
+		[Default Applications]
+		x-scheme-handler/http=iceweasel.desktop
+		x-scheme-handler/https=iceweasel.desktop
+		text/html=iceweasel.desktop
+
+
+Your preferred browser might be a different one off course. And I agree that this has to be the most logical place to store that information ever.. 
+
+"""]]
diff --git a/doc/forum/How_does_one_change_the_number_of_simultaneous_uploads.mdwn b/doc/forum/How_does_one_change_the_number_of_simultaneous_uploads.mdwn
new file mode 100644
index 0000000000..8de9cc7692
--- /dev/null
+++ b/doc/forum/How_does_one_change_the_number_of_simultaneous_uploads.mdwn
@@ -0,0 +1,3 @@
+I want to upload and download more than one file at a time to and from the various remotes.
+
+How do I do that?
diff --git a/doc/forum/How_does_one_change_the_number_of_simultaneous_uploads/comment_1_d5559994ee45a5c185a55c9a4d824aa4._comment b/doc/forum/How_does_one_change_the_number_of_simultaneous_uploads/comment_1_d5559994ee45a5c185a55c9a4d824aa4._comment
new file mode 100644
index 0000000000..41999cf0e9
--- /dev/null
+++ b/doc/forum/How_does_one_change_the_number_of_simultaneous_uploads/comment_1_d5559994ee45a5c185a55c9a4d824aa4._comment
@@ -0,0 +1,12 @@
+[[!comment format=mdwn
+ username="http://joeyh.name/"
+ ip="71.80.94.56"
+ subject="comment 1"
+ date="2014-02-07T19:12:10Z"
+ content="""
+This is not currently supported. See [[todo/Slow_transfer_for_a_lot_of_small_files.]]
+
+ You can click on the button next to individual files in the transfer queue to force concurrent transfers.
+
+
+"""]]
diff --git a/doc/forum/How_to_add_dotfiles_in_direct_mode__63__.mdwn b/doc/forum/How_to_add_dotfiles_in_direct_mode__63__.mdwn
new file mode 100644
index 0000000000..bd79dcdbd1
--- /dev/null
+++ b/doc/forum/How_to_add_dotfiles_in_direct_mode__63__.mdwn
@@ -0,0 +1,6 @@
+I have my Darktable photo library in git-annex and I need to use
+
+- .gitignore to ignore some files like *.lock
+- .gitattributes to make the sidecar .xmp files to be non-annexed normal files (faster and saves the editing history)
+
+What is the correct way to add those dotfiles to git in direct mode? Do I need to switch the repository to indirect mode and then back to direct mode? (I rather not do that very often since the repo has tens of thousands of files.)
diff --git a/doc/forum/How_to_add_dotfiles_in_direct_mode__63__/comment_1_d62d4b9ae5f1ec825fc58090689f7571._comment b/doc/forum/How_to_add_dotfiles_in_direct_mode__63__/comment_1_d62d4b9ae5f1ec825fc58090689f7571._comment
new file mode 100644
index 0000000000..b38fba2282
--- /dev/null
+++ b/doc/forum/How_to_add_dotfiles_in_direct_mode__63__/comment_1_d62d4b9ae5f1ec825fc58090689f7571._comment
@@ -0,0 +1,16 @@
+[[!comment format=mdwn
+ username="joey"
+ subject="""comment 1"""
+ date="2017-08-17T15:52:29Z"
+ content="""
+The best way is to make your .gitattributes file configure annex.largefiles
+to not match the files or file extensions that you want to not be annexed.
+(Including the .gitattributes file itself.)
+
+There are also ways to temporarily override annex.largefiles if you want to
+add a single file to git. Although this can be messy to need to remember if
+you later modify that file and are adding it again. Configuring
+annex.largefiles is better.
+
+See 
+"""]]
diff --git a/doc/forum/How_to_add_dotfiles_in_direct_mode__63__/comment_2_c7269cde1db02069149268b6c6a50786._comment b/doc/forum/How_to_add_dotfiles_in_direct_mode__63__/comment_2_c7269cde1db02069149268b6c6a50786._comment
new file mode 100644
index 0000000000..9c14759d4e
--- /dev/null
+++ b/doc/forum/How_to_add_dotfiles_in_direct_mode__63__/comment_2_c7269cde1db02069149268b6c6a50786._comment
@@ -0,0 +1,13 @@
+[[!comment format=mdwn
+ username="pitsa@5f5d24a1fb7ba0d050ae7cf95b4c3032002e8e38"
+ nickname="pitsa"
+ avatar="http://cdn.libravatar.org/avatar/d1de087e3bdaf7c2288b89b8ea56e6a5"
+ subject="Thanks"
+ date="2017-08-17T16:35:33Z"
+ content="""
+Ok, this seems to work:
+
+    git annex add -c annex.largefiles='not include=*' .gitattributes .gitignore
+    git annex sync
+
+"""]]
diff --git a/doc/forum/How_to_cancel_an_add__63__.mdwn b/doc/forum/How_to_cancel_an_add__63__.mdwn
new file mode 100644
index 0000000000..e0b0f1c8fe
--- /dev/null
+++ b/doc/forum/How_to_cancel_an_add__63__.mdwn
@@ -0,0 +1,5 @@
+What is the safest way to cancel a `git annex add` before the first commit?
+
+Say I first added a directory with a lot of small files to the annex and then (while watching the endless list of checksum operations) decide I should actually put them in a tar file and annex that instead.
+
+The problem seems to be that `git annex add` immediately replaces the files with symlinks instead of waiting for a commit, so I can't just `git reset largedir/; tar cvfz largedir.tgz largedir/; git annex add largedir.tgz`.
diff --git a/doc/forum/How_to_cancel_an_add__63__/comment_1_f768ce5dc7c76f96ee6eb352f167be44._comment b/doc/forum/How_to_cancel_an_add__63__/comment_1_f768ce5dc7c76f96ee6eb352f167be44._comment
new file mode 100644
index 0000000000..cef9384691
--- /dev/null
+++ b/doc/forum/How_to_cancel_an_add__63__/comment_1_f768ce5dc7c76f96ee6eb352f167be44._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="http://joeyh.name/"
+ ip="209.250.56.47"
+ subject="comment 1"
+ date="2013-11-03T00:50:48Z"
+ content="""
+Run `git annex unannex` on the files.
+"""]]
diff --git a/doc/forum/How_to_change_the_name_of_a_repo_on_S3__63__.mdwn b/doc/forum/How_to_change_the_name_of_a_repo_on_S3__63__.mdwn
new file mode 100644
index 0000000000..70e3eada30
--- /dev/null
+++ b/doc/forum/How_to_change_the_name_of_a_repo_on_S3__63__.mdwn
@@ -0,0 +1,2 @@
+git-annex created a gitannexxxxxxxxxxxxxxxxxxxxx folder on S3 and I guess I'm a bit OCD but I'd like to rename it to git-annex. 
+Of course I can rename it in my S3 console but where does git-annex store that name so I can also change it on my local install of git-annex.
diff --git a/doc/forum/How_to_change_the_name_of_a_repo_on_S3__63__/comment_1_be74d63e1951f515948d232e096b4862._comment b/doc/forum/How_to_change_the_name_of_a_repo_on_S3__63__/comment_1_be74d63e1951f515948d232e096b4862._comment
new file mode 100644
index 0000000000..3aecea0517
--- /dev/null
+++ b/doc/forum/How_to_change_the_name_of_a_repo_on_S3__63__/comment_1_be74d63e1951f515948d232e096b4862._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="http://joeyh.name/"
+ ip="108.236.230.124"
+ subject="comment 1"
+ date="2014-05-16T16:28:40Z"
+ content="""
+If you look at the [[S3_documentation|special_remotes/S3]] you will see a `bucket` setting that does what you want. You can use `git annex enableremote` to change it.
+"""]]
diff --git a/doc/forum/How_to_change_the_name_of_a_repo_on_S3__63__/comment_2_d54a7163cfe9a94b7ba337860958e5c5._comment b/doc/forum/How_to_change_the_name_of_a_repo_on_S3__63__/comment_2_d54a7163cfe9a94b7ba337860958e5c5._comment
new file mode 100644
index 0000000000..d563d061ac
--- /dev/null
+++ b/doc/forum/How_to_change_the_name_of_a_repo_on_S3__63__/comment_2_d54a7163cfe9a94b7ba337860958e5c5._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="https://www.google.com/accounts/o8/id?id=AItOawkAUMhKOSkh9JaBA6xst3XxQIIsDEq5Zd4"
+ nickname="Ovidiu"
+ subject="comment 2"
+ date="2014-05-16T18:08:11Z"
+ content="""
+Thanks but I realized there's no way to rename an S3 bucket so giving up on this idea...
+"""]]
diff --git a/doc/forum/How_to_completely_remove_tracking_of_a_deleted_file_in_direct_mode.mdwn b/doc/forum/How_to_completely_remove_tracking_of_a_deleted_file_in_direct_mode.mdwn
new file mode 100644
index 0000000000..82fc666a08
--- /dev/null
+++ b/doc/forum/How_to_completely_remove_tracking_of_a_deleted_file_in_direct_mode.mdwn
@@ -0,0 +1,7 @@
+I am in direct mode (managing music) and have no other repositories than the one I'm in.  I would like to delete a file and have git no longer track it at all.
+
+So far I've manually deleted the file and run "git annex add .", but when I run "git annex status" the file is still marked as deleted.  I would like to receive no message whatsoever.
+
+I tried "git annex drop path" , "git annex unannex path", and everything else I can think of but nothing will get git annex to forget the file.
+
+I'm sure this is easy, but how do you remove a deleted file in direct mode?
diff --git a/doc/forum/How_to_completely_remove_tracking_of_a_deleted_file_in_direct_mode/comment_1_c6bd2ef90516dde928ff18ded36df625._comment b/doc/forum/How_to_completely_remove_tracking_of_a_deleted_file_in_direct_mode/comment_1_c6bd2ef90516dde928ff18ded36df625._comment
new file mode 100644
index 0000000000..9b5881ad6b
--- /dev/null
+++ b/doc/forum/How_to_completely_remove_tracking_of_a_deleted_file_in_direct_mode/comment_1_c6bd2ef90516dde928ff18ded36df625._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="http://joeyh.name/"
+ ip="108.236.230.124"
+ subject="comment 1"
+ date="2014-07-02T16:57:46Z"
+ content="""
+git annex sync
+"""]]
diff --git a/doc/forum/How_to_copy__47__duplicate_all_data_from_rsync__47__ssh_backend_to_other_backend__63__.mdwn b/doc/forum/How_to_copy__47__duplicate_all_data_from_rsync__47__ssh_backend_to_other_backend__63__.mdwn
new file mode 100644
index 0000000000..0e72582d32
--- /dev/null
+++ b/doc/forum/How_to_copy__47__duplicate_all_data_from_rsync__47__ssh_backend_to_other_backend__63__.mdwn
@@ -0,0 +1,14 @@
+Hello,
+
+I want to be safe and have two copies of my files on two different backend. Currently I only have a SSH backend, that stores all my data. I have full(root) access to that machine/backend. On my laptop I have only a few bytes of data, because all is moved/copied to that SSH backend. Now, I want to duplicate the data on the SSH backend to a Google Drive account (or any other). How could I do that (without downloading all data from the SSH backend)??? Encryption is not a must.
+
+I looked into the annex/objects folder on the SSH backend, but there are 3 char length directories compared to what I see on a test Google Drive backend, where only 2 char length directory names are.
+
+Example SSH backend: [git-annex root]/annex/objects/c10/90a/SHA256E-s445227--14c3f85d6dd3464f116f6a5bbd411012781d36794549d136b18d1914c4158820.jpg/SHA256E-s445227--14c3f85d6dd3464f116f6a5bbd411012781d36794549d136b18d1914c4158820.jpg
+
+Example Google Drive: [Google Drive root]/annex/W7/xQ/SHA256E-s913904--29f9800b0dd34d4200c4e9ee152b79c3556a9a473848720be7cf83d20eff65a4.JPG
+
+Is there a way to convert these directory names and do a simpe copy??? 
+
+Thank you,
+Bence
diff --git a/doc/forum/How_to_copy__47__duplicate_all_data_from_rsync__47__ssh_backend_to_other_backend__63__/comment_1_7973928b1aa9e0fcfeb6bf80885441f5._comment b/doc/forum/How_to_copy__47__duplicate_all_data_from_rsync__47__ssh_backend_to_other_backend__63__/comment_1_7973928b1aa9e0fcfeb6bf80885441f5._comment
new file mode 100644
index 0000000000..3c7d072588
--- /dev/null
+++ b/doc/forum/How_to_copy__47__duplicate_all_data_from_rsync__47__ssh_backend_to_other_backend__63__/comment_1_7973928b1aa9e0fcfeb6bf80885441f5._comment
@@ -0,0 +1,10 @@
+[[!comment format=mdwn
+ username="http://joeyh.name/"
+ ip="4.153.251.174"
+ subject="comment 1"
+ date="2013-09-07T17:11:03Z"
+ content="""
+Just run `git annex copy --all --to remote` 
+
+(Needs git-annex 4.20130709 or newer.)
+"""]]
diff --git a/doc/forum/How_to_deal_with_renamed_files_in_direct_mode__63__.mdwn b/doc/forum/How_to_deal_with_renamed_files_in_direct_mode__63__.mdwn
new file mode 100644
index 0000000000..576d598ddb
--- /dev/null
+++ b/doc/forum/How_to_deal_with_renamed_files_in_direct_mode__63__.mdwn
@@ -0,0 +1,3 @@
+I use git annex in direct mode and I have renamed a file. What should I do to make git annex understand that the file has just been renamed and not deleted/added?
+
+The file I renamed is quite big and I would like to avoid having to copy it again to the remotes and I would also like to avoid having to redownload it again in all the repositories.
diff --git a/doc/forum/How_to_deal_with_renamed_files_in_direct_mode__63__/comment_1_fe38fedbbc9e4a9e13bf19950e63c7ac._comment b/doc/forum/How_to_deal_with_renamed_files_in_direct_mode__63__/comment_1_fe38fedbbc9e4a9e13bf19950e63c7ac._comment
new file mode 100644
index 0000000000..6f452acb8c
--- /dev/null
+++ b/doc/forum/How_to_deal_with_renamed_files_in_direct_mode__63__/comment_1_fe38fedbbc9e4a9e13bf19950e63c7ac._comment
@@ -0,0 +1,10 @@
+[[!comment format=mdwn
+ username="http://joeyh.name/"
+ ip="4.153.254.222"
+ subject="comment 1"
+ date="2013-07-07T17:14:18Z"
+ content="""
+You don't have to do anything special, git-annex automatically detects when a file has been renamed.
+
+Just `git annex` add the new file, and commit and sync like usual.
+"""]]
diff --git a/doc/forum/How_to_debug_failing_sync.mdwn b/doc/forum/How_to_debug_failing_sync.mdwn
new file mode 100644
index 0000000000..ad18aeb55c
--- /dev/null
+++ b/doc/forum/How_to_debug_failing_sync.mdwn
@@ -0,0 +1,5 @@
+I've got a remote I setup while connected to my home network that is no longer syncing when I'm on the same network. I'm not sure how to debug the problem. Any tips?
+
+I will say that when I edit the repository in the assistant, the url it gives for the "git repository location at" field looks incorrect. It's got an extra "git-annex-" prepended to the hostname. I'm not sure how to alter that.
+
+I'm using version 5.20140517~bpo70+1, from the wheezy backports.
diff --git a/doc/forum/How_to_debug_failing_sync/comment_1_a597b868182e55e5f39394f154740534._comment b/doc/forum/How_to_debug_failing_sync/comment_1_a597b868182e55e5f39394f154740534._comment
new file mode 100644
index 0000000000..2d89ecda37
--- /dev/null
+++ b/doc/forum/How_to_debug_failing_sync/comment_1_a597b868182e55e5f39394f154740534._comment
@@ -0,0 +1,10 @@
+[[!comment format=mdwn
+ username="http://joeyh.name/"
+ ip="209.250.56.176"
+ subject="comment 1"
+ date="2014-05-29T18:15:12Z"
+ content="""
+That extra git-annex- prefix is used in ~/.ssh/config to set up the ssh keys for the repository. So it's normal.
+
+I suggest that you try running git-annex sync at the command line and paste the output so we can see how it fails.
+"""]]
diff --git a/doc/forum/How_to_debug_failing_sync/comment_2_db0a5652d76e4e568a4d7808195bc59c._comment b/doc/forum/How_to_debug_failing_sync/comment_2_db0a5652d76e4e568a4d7808195bc59c._comment
new file mode 100644
index 0000000000..bf707ecb14
--- /dev/null
+++ b/doc/forum/How_to_debug_failing_sync/comment_2_db0a5652d76e4e568a4d7808195bc59c._comment
@@ -0,0 +1,10 @@
+[[!comment format=mdwn
+ username="go8ose"
+ ip="59.167.191.61"
+ subject="comment 2"
+ date="2014-05-30T10:28:38Z"
+ content="""
+I was shown a password prompt for the remote. Then I realised I was controlling my ~/.ssh/authorized_keys file with puppet, which would have removed the public key that annex would have added to that file. I'm sure that once I fix that up, I'll get it working again.
+
+Thanks for the CLI suggestion.
+"""]]
diff --git a/doc/forum/How_to_define_a_balanced_prefered_content.mdwn b/doc/forum/How_to_define_a_balanced_prefered_content.mdwn
new file mode 100644
index 0000000000..1bfbe848b9
--- /dev/null
+++ b/doc/forum/How_to_define_a_balanced_prefered_content.mdwn
@@ -0,0 +1,5 @@
+I have 6 drives attached to my box and I am trying to split them in to 2 groups I would like to have 2 balanced groups [1] so each file lands on two of these disks. So assuming i have disks A B C D E F, I want to create two balanced groups A B C and D E F. But I can not figure out how to defined rules using  balanced_amoung(group) expression in vicfg. The only place it allows me to use the expression is the group section but how do i define a group that uses a balanced group.
+
+
+
+[1] https://git-annex.branchable.com/design/balanced_preferred_content/
diff --git a/doc/forum/How_to_define_a_balanced_prefered_content/comment_1_d4c94791a5fc7d2e618a7e11bc5db569._comment b/doc/forum/How_to_define_a_balanced_prefered_content/comment_1_d4c94791a5fc7d2e618a7e11bc5db569._comment
new file mode 100644
index 0000000000..b53c615dad
--- /dev/null
+++ b/doc/forum/How_to_define_a_balanced_prefered_content/comment_1_d4c94791a5fc7d2e618a7e11bc5db569._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="https://www.google.com/accounts/o8/id?id=AItOawnmF_9CAtfqdZkC4e-_dCX-rK5bqh4RWkw"
+ nickname="Carl"
+ subject="Only a design suggestion"
+ date="2015-09-07T17:24:41Z"
+ content="""
+I am pretty sure that balanced preferred content expressions are not implemented, but only a suggested model on how it could be done.
+"""]]
diff --git a/doc/forum/How_to_define_an_alternative_remote_url_for_a_git_remote_repository__63__.mdwn b/doc/forum/How_to_define_an_alternative_remote_url_for_a_git_remote_repository__63__.mdwn
new file mode 100644
index 0000000000..7b8e1a0bad
--- /dev/null
+++ b/doc/forum/How_to_define_an_alternative_remote_url_for_a_git_remote_repository__63__.mdwn
@@ -0,0 +1,7 @@
+I'm using git-annex to manage my files. Some of my remotes are available using 2 or 3 methods, for example ssh and nfs, but the nfs access is only possible on my local network, of course. Then, git-annex chooses the fastest method to sync or get the files (nfs, if available, ssh instead, using scores to choose). For now, I define a remote per repository and per access method (example: server-by-ssh, server-by-nfs), but that's not a really git-annex way to do the job, as it is not compliant with the numcopies option, which ckecks that the files in the annex are kept with enough replications (with my current method, many remotes are in fact the same folders on a hard drive).
+
+Then, my question applies to both git and git-annex, since the remotes are added in git, not git-annex: I wonder if it is possible to define alternative url for a given remote.
+
+I have tried to use the git remote set --url --add command, but it doesn't really work as I would expect. Indeed, if the nfs url is not reachable, git hangs up and waits.
+
+Any idea?
diff --git a/doc/forum/How_to_define_an_alternative_remote_url_for_a_git_remote_repository__63__/comment_1_52918b5ec25e55837215439fe1bb1a14._comment b/doc/forum/How_to_define_an_alternative_remote_url_for_a_git_remote_repository__63__/comment_1_52918b5ec25e55837215439fe1bb1a14._comment
new file mode 100644
index 0000000000..5bafa57d95
--- /dev/null
+++ b/doc/forum/How_to_define_an_alternative_remote_url_for_a_git_remote_repository__63__/comment_1_52918b5ec25e55837215439fe1bb1a14._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="https://www.google.com/accounts/o8/id?id=AItOawmBUR4O9mofxVbpb8JV9mEbVfIYv670uJo"
+ nickname="Justin"
+ subject="comment 1"
+ date="2012-08-21T12:42:16Z"
+ content="""
+I don't think having multiple remotes for the same repository confuses git-annex since it uses the UUID not the url to identify them.
+"""]]
diff --git a/doc/forum/How_to_define_an_alternative_remote_url_for_a_git_remote_repository__63__/comment_2_3a1567c9f484b5e12e5560cdcc2cfddd._comment b/doc/forum/How_to_define_an_alternative_remote_url_for_a_git_remote_repository__63__/comment_2_3a1567c9f484b5e12e5560cdcc2cfddd._comment
new file mode 100644
index 0000000000..859804678c
--- /dev/null
+++ b/doc/forum/How_to_define_an_alternative_remote_url_for_a_git_remote_repository__63__/comment_2_3a1567c9f484b5e12e5560cdcc2cfddd._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="http://joeyh.name/"
+ ip="4.153.8.29"
+ subject="Justin is right"
+ date="2012-08-21T15:57:46Z"
+ content="""
+git-annex doesn't care how many git remotes you have pointing to a given repo, that repo's uuid still only counts as one copy.
+"""]]
diff --git a/doc/forum/How_to_define_an_alternative_remote_url_for_a_git_remote_repository__63__/comment_3_48c3a80c14a85f27d742482b2ccbe628._comment b/doc/forum/How_to_define_an_alternative_remote_url_for_a_git_remote_repository__63__/comment_3_48c3a80c14a85f27d742482b2ccbe628._comment
new file mode 100644
index 0000000000..7a0054c493
--- /dev/null
+++ b/doc/forum/How_to_define_an_alternative_remote_url_for_a_git_remote_repository__63__/comment_3_48c3a80c14a85f27d742482b2ccbe628._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="https://me.yahoo.com/speredenn#aaf38"
+ nickname="Jean-Baptiste Carré"
+ subject="comment 3"
+ date="2012-08-21T18:15:48Z"
+ content="""
+You're totally right: The UUIDs are the same. So it shouldn't matter if there are many repositories pointing to the same folder, as you state it. Thanks a lot!
+"""]]
diff --git a/doc/forum/How_to_delete_a_file_in_direct_mode__63__.mdwn b/doc/forum/How_to_delete_a_file_in_direct_mode__63__.mdwn
new file mode 100644
index 0000000000..7e95224f9e
--- /dev/null
+++ b/doc/forum/How_to_delete_a_file_in_direct_mode__63__.mdwn
@@ -0,0 +1,31 @@
+My use case is that I'm organising my media in a staging area called **uploadme/**.
+
+At this staging area, I add all the media into git-annex and then I go to work pruning, editing & [adding metadata](https://github.com/muennich/sxiv/blob/master/exec/key-handler#L29).
+
+# Pruning
+
+If I find an image I don't like I move it to `/tmp`.
+
+I expected to maybe type `git status` to see the missing file and later confirm that I don't want it. But when I type `git status` I get:
+
+    X1C3:~/media/uploadme$ git status
+    fatal: This operation must be run in a work tree
+
+What should I be doing?
+
+
+
+My use case is that I'm organising my media in a staging area called **uploadme/**.
+
+At this staging area, I add all the media into git-annex and then I go to work pruning, editing & [adding metadata](https://github.com/muennich/sxiv/blob/master/exec/key-handler#L29).
+
+# Pruning
+
+If I find an image I don't like I move it to `/tmp`.
+
+I expected to maybe type `git status` to see the missing file and later confirm that I don't want it. But when I type `git status` I get:
+
+    X1C3:~/media/uploadme$ git status
+    fatal: This operation must be run in a work tree
+
+What should I be doing?
diff --git a/doc/forum/How_to_delete_a_file_in_direct_mode__63__/comment_1_995f3b8cf511921719774f8911e24907._comment b/doc/forum/How_to_delete_a_file_in_direct_mode__63__/comment_1_995f3b8cf511921719774f8911e24907._comment
new file mode 100644
index 0000000000..f04d38bc1a
--- /dev/null
+++ b/doc/forum/How_to_delete_a_file_in_direct_mode__63__/comment_1_995f3b8cf511921719774f8911e24907._comment
@@ -0,0 +1,15 @@
+[[!comment format=mdwn
+ username="joey"
+ subject="""comment 1"""
+ date="2015-03-26T15:45:22Z"
+ content="""
+**direct** mode gets its name, because you can **direct**ly operate on files
+with commands like "rm" or "mv /tmp".
+
+So, to delete a file, you use such a command as you normally would.
+
+`git annex status` can be used in direct mode since `git status` does not
+work.
+
+Perhaps you should read the documentation: [[/direct_mode]]
+"""]]
diff --git a/doc/forum/How_to_delete_a_remote__63__.mdwn b/doc/forum/How_to_delete_a_remote__63__.mdwn
new file mode 100644
index 0000000000..6aebaf5f51
--- /dev/null
+++ b/doc/forum/How_to_delete_a_remote__63__.mdwn
@@ -0,0 +1 @@
+I have a repository with an S3 remote named `cloud`. I now no longer want to use that remote. I've dropped all the data from it (`git annex drop . --from cloud`). How do I tell my repository to forget about the remote?
diff --git a/doc/forum/How_to_delete_a_remote__63__/comment_1_8cba186bb67079ff41bf6d0b04613f4a._comment b/doc/forum/How_to_delete_a_remote__63__/comment_1_8cba186bb67079ff41bf6d0b04613f4a._comment
new file mode 100644
index 0000000000..e60eb3d5d9
--- /dev/null
+++ b/doc/forum/How_to_delete_a_remote__63__/comment_1_8cba186bb67079ff41bf6d0b04613f4a._comment
@@ -0,0 +1,10 @@
+[[!comment format=mdwn
+ username="https://www.google.com/accounts/o8/id?id=AItOawnldTTAP8PAifJUmqhRar6RAWNWlRcencw"
+ nickname="Marco"
+ subject="You can set the trust to "dead""
+ date="2013-01-23T19:02:22Z"
+ content="""
+Simple answer: You can't fully delete a remote - with annex commands. You can mark it as dead.
+
+You can give this a try as a workaround: [[Truly_purging_dead_repositories/#comment-51ab797094f4c07bf5327fd21cad5835]]
+"""]]
diff --git a/doc/forum/How_to_delete_a_remote__63__/comment_2_33c429ffa7e9e2ed9c5fac760ee8e82c._comment b/doc/forum/How_to_delete_a_remote__63__/comment_2_33c429ffa7e9e2ed9c5fac760ee8e82c._comment
new file mode 100644
index 0000000000..2866710c8b
--- /dev/null
+++ b/doc/forum/How_to_delete_a_remote__63__/comment_2_33c429ffa7e9e2ed9c5fac760ee8e82c._comment
@@ -0,0 +1,12 @@
+[[!comment format=mdwn
+ username="http://joeyh.name/"
+ ip="2001:4830:1600:187::2"
+ subject="comment 2"
+ date="2013-09-04T06:43:26Z"
+ content="""
+Recently git-annex has gotten the ability to do this: `git annex forget --drop-dead`
+
+That prunes all history relating to all dead remotes. You need to be running a git-annex that supports this on all computers you use the repos on, or the pruned history will get merged back in.
+
+I don't recommend doing this just because you want to \"clean history\". Think of it as something you can do at some point in the future if the .git/objects somehow gets too large or too slow. Put off deleting data until tomorrow if you don't absolutely need to do it today.
+"""]]
diff --git a/doc/forum/How_to_delete_a_remote__63__/comment_3_e9c5508092ca2983f458b16bf1e07082._comment b/doc/forum/How_to_delete_a_remote__63__/comment_3_e9c5508092ca2983f458b16bf1e07082._comment
new file mode 100644
index 0000000000..337ef2efe1
--- /dev/null
+++ b/doc/forum/How_to_delete_a_remote__63__/comment_3_e9c5508092ca2983f458b16bf1e07082._comment
@@ -0,0 +1,14 @@
+[[!comment format=mdwn
+ username="konubinix"
+ ip="82.243.233.186"
+ subject="Dropping dead repositories"
+ date="2013-09-04T07:40:22Z"
+ content="""
+Actually, it may be a good idea to remove repositories made for tests purposes.
+
+I now have 2 dead repositories that are USB_test1 and USB_test2 that I created before knowing I could reuse the annex uuid.
+
+They are now there and it is difficult to remove them.
+
+For that special case, the --drop-dead feature is very welcome.
+"""]]
diff --git a/doc/forum/How_to_delete_a_remote__63__/comment_4_688a8473d074830303133ba939438084._comment b/doc/forum/How_to_delete_a_remote__63__/comment_4_688a8473d074830303133ba939438084._comment
new file mode 100644
index 0000000000..ef7b820bca
--- /dev/null
+++ b/doc/forum/How_to_delete_a_remote__63__/comment_4_688a8473d074830303133ba939438084._comment
@@ -0,0 +1,12 @@
+[[!comment format=mdwn
+ username="Cyberthal"
+ avatar="http://cdn.libravatar.org/avatar/1c619d65ee07d2343295c8f70f23c9df"
+ subject="Deleting a repository UUID from vicfg"
+ date="2017-05-03T16:50:53Z"
+ content="""
+I followed these instructions, but a single entry for the targeted UUID persists in vicfg.
+
+For those of us with OCD, this is mildly irksome. Later I might have dead UUID's I actually want to remember, and that bit of spam will always be sitting there at the top of the file.
+
+However, deleting it is still worthwhile, because otherwise it is repeated in every list of remotes.
+"""]]
diff --git a/doc/forum/How_to_delete_a_remote__63__/comment_5_de7dff2dc4357d8cc617d86a06e35b1b._comment b/doc/forum/How_to_delete_a_remote__63__/comment_5_de7dff2dc4357d8cc617d86a06e35b1b._comment
new file mode 100644
index 0000000000..b7792c8e4b
--- /dev/null
+++ b/doc/forum/How_to_delete_a_remote__63__/comment_5_de7dff2dc4357d8cc617d86a06e35b1b._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="joey"
+ subject="""comment 5"""
+ date="2017-05-09T18:03:54Z"
+ content="""
+It may help suppress the OCD to consider that vicfg is listing dead
+remotes to give you the option to make them not dead anymore.
+"""]]
diff --git a/doc/forum/How_to_destroy_a_master_annex_and_all_remotes_with_git_annex_assistant_and_ext4.mdwn b/doc/forum/How_to_destroy_a_master_annex_and_all_remotes_with_git_annex_assistant_and_ext4.mdwn
new file mode 100644
index 0000000000..c67d4e77f8
--- /dev/null
+++ b/doc/forum/How_to_destroy_a_master_annex_and_all_remotes_with_git_annex_assistant_and_ext4.mdwn
@@ -0,0 +1,28 @@
+Git annex is really amazing software, and this cute little scenario is actually Linux's fault. But it's a nasty situation nonetheless, and worth passing on.
+
+...
+
+Create a client repository on your laptop and two backup repositories on external USB drives. Keep all repositories mounted and connected. Now drop 60GB of data spread over 30,000 files into your annex, and watch git annex assistant start adding and syncing. So far, so good.
+
+Now wait a few hours, and watch some kernel crypto code—probably ecryptfs—fall over with a segfault.
+
+Since your laptop and USB drives are all running ext4, the sudden kernel panic will leave you with hundreds of 0-length files. Because git annex assistant was busily adding and syncing files, those 0-length files are spread randomly throughout all your git repositories (typically in `.git/objects`) and throughout all the associated annexes. Unfortunately, because `git annex assistant` generates _tons_ of commits, this is pretty much unrecoverable using standard git tools unless you're willing to get deep into the repositories' internals.
+
+So what should you do, if you want to add 10s of 1000s of files and there's some risk of kernel panic or accidentally bumping a USB cable? Here's my recommendation to limit the damage:
+
+1. Use the command line if possible.
+2. Add all your files with your remotes offline.
+3. Run `git gc` on your central repository, just on general principals.
+4. Mount one repository at a time.
+5. Sync the pure git data first, and then make sure that all disk I/O is flushed (`sync; sleep 10` is a good approximation).
+6. Use `git annex copy --to` to move the annex data.
+7. Unmount the USB repository cleanly and move onto the next one.
+
+If you _do_ bump a USB cable in the middle of step (6), then:
+
+1. Run 'git annex fsck' to clean up any garbage files.
+2. Try another 'git annex copy --to' where you left off.
+
+Wiser minds than I are encouraged to suggest optimizations for the recovery steps.
+
+The theory behind these steps is to only do one thing at a time, and to expose as few remotes as possible to a power failure or crash. A secondary goal is to make sure that pure git operations complete very quickly, limiting the risk that they will be interrupted, because they're the hardest operations to recover from after a crash.
diff --git a/doc/forum/How_to_destroy_a_master_annex_and_all_remotes_with_git_annex_assistant_and_ext4/comment_1_42ca6cfbbb79fe63514805b8119ac16b._comment b/doc/forum/How_to_destroy_a_master_annex_and_all_remotes_with_git_annex_assistant_and_ext4/comment_1_42ca6cfbbb79fe63514805b8119ac16b._comment
new file mode 100644
index 0000000000..f6d8396b7a
--- /dev/null
+++ b/doc/forum/How_to_destroy_a_master_annex_and_all_remotes_with_git_annex_assistant_and_ext4/comment_1_42ca6cfbbb79fe63514805b8119ac16b._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="http://joeyh.name/"
+ ip="2001:4978:f:21a::2"
+ subject="comment 1"
+ date="2013-07-23T14:45:05Z"
+ content="""
+It seems to me that it would be better to avoid using kernel features that crash. And, set up a git repository on another machine, which will serve as a real backup if something goes catostrophically wrong. The only potential failure mode then would be if a kernel/hardware issue zeroed some of your files, in direct mode the assistant might commit those new corrupt files and ship them over to the backup. But, if your offsite backup is accumulating old versions of files, you can recover from this by reverting the bad commit.
+"""]]
diff --git a/doc/forum/How_to_destroy_a_master_annex_and_all_remotes_with_git_annex_assistant_and_ext4/comment_2_c94ce6a9767c624e2445a7d9eea40396._comment b/doc/forum/How_to_destroy_a_master_annex_and_all_remotes_with_git_annex_assistant_and_ext4/comment_2_c94ce6a9767c624e2445a7d9eea40396._comment
new file mode 100644
index 0000000000..347a95539a
--- /dev/null
+++ b/doc/forum/How_to_destroy_a_master_annex_and_all_remotes_with_git_annex_assistant_and_ext4/comment_2_c94ce6a9767c624e2445a7d9eea40396._comment
@@ -0,0 +1,29 @@
+[[!comment format=mdwn
+ username="http://emk.myopenid.com/"
+ ip="24.2.150.84"
+ subject="Well, this is the first time it ever panicked"
+ date="2013-07-23T16:32:10Z"
+ content="""
+Well, it's not like my machine kernel panics on a regular basis or anything. :-) This is the first time I ever saw the kernel encryption code do this. I'm running a boring stock install of Ubuntu 12.04 LTS that was preloaded by ZaReason, and I'm using the ecryptfs home directory encryption supplied by Ubuntu. So in this case, \"stop using kernel features that crash\" means \"stop using Ubuntu on supported hardware.\"
+
+The underlying problem is that ext4 allocates file contents lazily and out-of-order, and it may wait a surprisingly long time before actually flushing data to disk:
+
+https://bugs.launchpad.net/ubuntu/+source/linux/+bug/317781
+http://linux.bihlman.com/2010/learn-linux-help/how-to-solve-zero-length-file-problem-in-linuxs-ext4-file-system/
+
+The problem is that `git annex assistant` and `git` don't sync the disks all that often, and that the assistant can generate huge amounts of complicated disk I/O across multiple volumes for hours on end. All you need to do is go through the walkthrough, create the specified repositories including one on a USB drive, and throw 50GB of data into the annex directory. `git annex assistant` will happily grind away overnight, and it anything prevents ext4 from flushing data, there's a good chance you'll wind up with multiple corrupted repositories with hundreds of `git fsck` errors.
+
+There are some potential workarounds:
+
+1. Reconfigure ext4 to run in data write-back mode. This isn't really possible for non-technical users, but it's an excellent idea for large external USB drives.
+2. Somehow convince `git` and `git annex assistant` to call `sync` or `fsync` more aggressively on local volumes.
+3. Use a remote server for at least one remote, as you suggested.
+4. Operate on the repository manually, so that you can ensure that `git`'s data is in a known-good state before trying to copy the annex files. The 
+annex files are much easier to recover than git's state.
+
+Anyway, I doubt this is really fixable. And it's not really `git annex`'s fault, in any case. But I'm really glad I had recent backups of all my data last night, which allowed me to checksum everything and start from scratch.
+
+...
+
+Leaving aside this incident, `git annex` is one of the nicest pieces of open source software I've seen in a long time, and it's clearly going to change how I use my computer. And thank you for posting the crowd-funding campaign so we can say \"Thanks!\"
+"""]]
diff --git a/doc/forum/How_to_destroy_a_master_annex_and_all_remotes_with_git_annex_assistant_and_ext4/comment_3_bcda51053b62bbb20ce71a59469e1b26._comment b/doc/forum/How_to_destroy_a_master_annex_and_all_remotes_with_git_annex_assistant_and_ext4/comment_3_bcda51053b62bbb20ce71a59469e1b26._comment
new file mode 100644
index 0000000000..5cf8c93f38
--- /dev/null
+++ b/doc/forum/How_to_destroy_a_master_annex_and_all_remotes_with_git_annex_assistant_and_ext4/comment_3_bcda51053b62bbb20ce71a59469e1b26._comment
@@ -0,0 +1,10 @@
+[[!comment format=mdwn
+ username="http://joeyh.name/"
+ ip="2001:4978:f:21a::2"
+ subject="comment 3"
+ date="2013-07-23T17:17:43Z"
+ content="""
+`git config core.fsyncobjectfiles true`
+
+This will make git fsync all the data it writes. Whether it's a good default, I don't know.
+"""]]
diff --git a/doc/forum/How_to_destroy_a_master_annex_and_all_remotes_with_git_annex_assistant_and_ext4/comment_4_48e5b9eae920e5f13812de8d6f6bc640._comment b/doc/forum/How_to_destroy_a_master_annex_and_all_remotes_with_git_annex_assistant_and_ext4/comment_4_48e5b9eae920e5f13812de8d6f6bc640._comment
new file mode 100644
index 0000000000..2bac2ca1bf
--- /dev/null
+++ b/doc/forum/How_to_destroy_a_master_annex_and_all_remotes_with_git_annex_assistant_and_ext4/comment_4_48e5b9eae920e5f13812de8d6f6bc640._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="http://joeyh.name/"
+ ip="2001:4978:f:21a::2"
+ subject="comment 4"
+ date="2013-07-23T17:38:50Z"
+ content="""
+I have made the assistant enable the fsync option when it creates a repository on a removable drive. That is, at least in my experience, the thing that most likely needs it. I don't know if it would have saved you in your situation or not.
+"""]]
diff --git a/doc/forum/How_to_destroy_a_master_annex_and_all_remotes_with_git_annex_assistant_and_ext4/comment_5_787c0bfdc1d309db1486c3a37723a957._comment b/doc/forum/How_to_destroy_a_master_annex_and_all_remotes_with_git_annex_assistant_and_ext4/comment_5_787c0bfdc1d309db1486c3a37723a957._comment
new file mode 100644
index 0000000000..dd40495ca9
--- /dev/null
+++ b/doc/forum/How_to_destroy_a_master_annex_and_all_remotes_with_git_annex_assistant_and_ext4/comment_5_787c0bfdc1d309db1486c3a37723a957._comment
@@ -0,0 +1,13 @@
+[[!comment format=mdwn
+ username="https://www.google.com/accounts/o8/id?id=AItOawmkBwMWvNKZZCge_YqobCSILPMeK6xbFw8"
+ nickname="develop"
+ subject="comment 5"
+ date="2013-07-23T17:39:42Z"
+ content="""
+Oh i've had that type of crash from ecryptfs many times over many different versions of Ubuntu, also before i even encountered git-annex.
+
+Git-annex might provoke this issue, but it is by no means the only thing that can crash your system when running ecryptfs.
+
+I highly suggest you try LUKS for a stable experience.
+
+"""]]
diff --git a/doc/forum/How_to_destroy_a_master_annex_and_all_remotes_with_git_annex_assistant_and_ext4/comment_6_8894beb06443f234e9200b03b5f3badf._comment b/doc/forum/How_to_destroy_a_master_annex_and_all_remotes_with_git_annex_assistant_and_ext4/comment_6_8894beb06443f234e9200b03b5f3badf._comment
new file mode 100644
index 0000000000..149998aaf0
--- /dev/null
+++ b/doc/forum/How_to_destroy_a_master_annex_and_all_remotes_with_git_annex_assistant_and_ext4/comment_6_8894beb06443f234e9200b03b5f3badf._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="http://joeyh.name/"
+ ip="2001:4978:f:21a::2"
+ subject="comment 6"
+ date="2013-07-23T17:54:00Z"
+ content="""
+If there's an easy way to detect when a directory is inside an ecryptfs volume, I'd be happy to make the assistant automatically enable fsync on it. ;)
+"""]]
diff --git a/doc/forum/How_to_destroy_a_master_annex_and_all_remotes_with_git_annex_assistant_and_ext4/comment_7_457f62ee3e58f68a55f66c5bde6002fd._comment b/doc/forum/How_to_destroy_a_master_annex_and_all_remotes_with_git_annex_assistant_and_ext4/comment_7_457f62ee3e58f68a55f66c5bde6002fd._comment
new file mode 100644
index 0000000000..8f545e2ebd
--- /dev/null
+++ b/doc/forum/How_to_destroy_a_master_annex_and_all_remotes_with_git_annex_assistant_and_ext4/comment_7_457f62ee3e58f68a55f66c5bde6002fd._comment
@@ -0,0 +1,10 @@
+[[!comment format=mdwn
+ username="http://emk.myopenid.com/"
+ ip="24.2.150.84"
+ subject="Thank you!"
+ date="2013-07-23T18:36:22Z"
+ content="""
+What good idea! I've turned on `core.fsyncobjectfiles`. This should definitely help avoid the worst damage when ext4 filesystems get unmounted while dirty. Thank you very much for addressing this issue so quickly, even if though has more to do with ext4 than git annex.
+
+develop: Thank you for the suggestion to use LUKS; that's good to know.
+"""]]
diff --git a/doc/forum/How_to_destroy_a_master_annex_and_all_remotes_with_git_annex_assistant_and_ext4/comment_8_bd2b412116a66107bc0ff0efd7e39a58._comment b/doc/forum/How_to_destroy_a_master_annex_and_all_remotes_with_git_annex_assistant_and_ext4/comment_8_bd2b412116a66107bc0ff0efd7e39a58._comment
new file mode 100644
index 0000000000..0d09a9652b
--- /dev/null
+++ b/doc/forum/How_to_destroy_a_master_annex_and_all_remotes_with_git_annex_assistant_and_ext4/comment_8_bd2b412116a66107bc0ff0efd7e39a58._comment
@@ -0,0 +1,10 @@
+[[!comment format=mdwn
+ username="http://emk.myopenid.com/"
+ ip="24.2.150.84"
+ subject="Detecting ecryptfs"
+ date="2013-07-23T18:49:08Z"
+ content="""
+If the string `\" ecryptfs \"` (with surrounding spaces) appears in `/etc/mtab`, then there's at least one ecryptfs volume on the system. More specific parsing will tell you where it's mounted.
+
+Ubuntu released a new kernels fixing the worst ecryptfs bugs in 12.04 LTS. The current version is OK, but perhaps not exactly great. Some older versions are public menaces with awful data corruption bugs.
+"""]]
diff --git a/doc/forum/How_to_emulate___34__one-way_sync__34__s_in_a_direct_repo__63__.mdwn b/doc/forum/How_to_emulate___34__one-way_sync__34__s_in_a_direct_repo__63__.mdwn
new file mode 100644
index 0000000000..92bf09ae27
--- /dev/null
+++ b/doc/forum/How_to_emulate___34__one-way_sync__34__s_in_a_direct_repo__63__.mdwn
@@ -0,0 +1,9 @@
+I'd like to provide a team of some non-technical Windows (=direct mode) users with the following git-shell scripts to make collaboration safer than with plain "sync":
+
+1) **update-changes-from-others.sh** -- Like sync, but don't commit any local changes. Merge them like sync, don't discard.
+
+2) **sync-all-but-deletes.sh** -- Like sync but don't commit any deletions (-> renames may become additions).
+
+3) **discard-all-my-changes.sh** -- Forcibly discard all local changes, then checkout and get latest master head.
+
+Any pointers on how to approach implementing these safely?
diff --git a/doc/forum/How_to_emulate___34__one-way_sync__34__s_in_a_direct_repo__63__/comment_1_c5226964b4b64bd47d888b5616ff970b._comment b/doc/forum/How_to_emulate___34__one-way_sync__34__s_in_a_direct_repo__63__/comment_1_c5226964b4b64bd47d888b5616ff970b._comment
new file mode 100644
index 0000000000..91edd264d0
--- /dev/null
+++ b/doc/forum/How_to_emulate___34__one-way_sync__34__s_in_a_direct_repo__63__/comment_1_c5226964b4b64bd47d888b5616ff970b._comment
@@ -0,0 +1,22 @@
+[[!comment format=mdwn
+ username="joey"
+ subject="""comment 1"""
+ date="2014-12-01T23:08:56Z"
+ content="""
+Well, if you set `remote.foo.annex-readonly` to true, then git-annex
+sync won't push changes to the remote named "foo".
+
+You could set that in .git/config, or you could do something like
+this to set it only when running the sync:
+
+git -c remote.foo.annex-readonly=true annex sync
+
+I am not sure about a good solution for sync-all-but-deletes.sh,
+although it seems like the new `git annex proxy` could perhaps
+be used to build it.
+
+discard-all-my-changes.sh could probably be implemented using the new
+`git annex proxy` command. It would be pretty unsafe though! Maybe
+instead make it first commit all local changes, and then use 
+`git annex proxy -- git reset --hard origin/master`
+"""]]
diff --git a/doc/forum/How_to_emulate___34__one-way_sync__34__s_in_a_direct_repo__63__/comment_2_748d243bac14a8d55be4ac324c581c1d._comment b/doc/forum/How_to_emulate___34__one-way_sync__34__s_in_a_direct_repo__63__/comment_2_748d243bac14a8d55be4ac324c581c1d._comment
new file mode 100644
index 0000000000..edef7c1470
--- /dev/null
+++ b/doc/forum/How_to_emulate___34__one-way_sync__34__s_in_a_direct_repo__63__/comment_2_748d243bac14a8d55be4ac324c581c1d._comment
@@ -0,0 +1,62 @@
+[[!comment format=mdwn
+ username="https://www.google.com/accounts/o8/id?id=AItOawmWBvsZvSsAL8P2ye3F0OBStjFCVnOImzM"
+ nickname="Jarno"
+ subject="comment 2"
+ date="2014-12-03T20:04:42Z"
+ content="""
+Thank you, `remote.foo.annex-readonly` looks like a nice solution for the update-changes-from-other.sh!
+
+For discard-all-my-changes.sh, `reset --hard` through proxy worked otherwise nicely, but it doesn't seem to restore deletions:
+
+    $ git annex proxy -- git reset --hard origin/master
+    HEAD is now at 37aed97 git-annex automatic sync
+
+    $ echo \"A\" > test.txt
+    
+    $ git annex add test.txt
+    add test.txt ok
+    (Recording state in git...)
+    
+    $ git annex sync
+    commit  ok
+    pull origin
+    ok
+    push origin
+    Counting objects: 17, done.
+    Delta compression using up to 8 threads.
+    Compressing objects: 100% (7/7), done.
+    Writing objects: 100% (8/8), 783 bytes | 0 bytes/s, done.
+    Total 8 (delta 4), reused 1 (delta 0)
+    To ssh://gitannex@serv-gitannex:/home/gitannex/git-annex-test.git
+       88a25b5..0a8281d  git-annex -> synced/git-annex
+       37aed97..2978dcd  annex/direct/master -> synced/master
+    ok
+    
+    $ rm test.txt
+    
+    $ git annex proxy -- git reset --hard origin/master
+    warning: packfile .git/objects/pack/pack-42568744cdbba46b2dd71a7f37546a52bb26684
+    4.pack cannot be accessed
+    HEAD is now at 2978dcd git-annex automatic sync
+    
+    $ ls test.txt
+    ls: test.txt: No such file or directory
+    
+    $ git annex status
+    D test.txt
+
+    $ git annex version
+    git-annex version: 5.20141128-g70f997e
+    build flags: Assistant Webapp Webapp-secure Pairing Testsuite S3 WebDAV DNS Feed
+    s Quvi TDFA CryptoHash
+    key/value backends: SHA256E SHA1E SHA512E SHA224E SHA384E SKEIN256E SKEIN512E SH
+    A256 SHA1 SHA512 SHA224 SHA384 SKEIN256 SKEIN512 WORM URL
+    remote types: git gcrypt S3 bup directory rsync web webdav tahoe glacier ddar ho
+    ok external
+    local repository version: 5
+    supported repository version: 5
+    upgrade supported from repository versions: 2 3 4
+
+Note the \"pack cannot be accessed\" after the second reset.
+
+"""]]
diff --git a/doc/forum/How_to_emulate___34__one-way_sync__34__s_in_a_direct_repo__63__/comment_3_cf9234339bad14ad9a9d3027f9066936._comment b/doc/forum/How_to_emulate___34__one-way_sync__34__s_in_a_direct_repo__63__/comment_3_cf9234339bad14ad9a9d3027f9066936._comment
new file mode 100644
index 0000000000..3cf4545621
--- /dev/null
+++ b/doc/forum/How_to_emulate___34__one-way_sync__34__s_in_a_direct_repo__63__/comment_3_cf9234339bad14ad9a9d3027f9066936._comment
@@ -0,0 +1,49 @@
+[[!comment format=mdwn
+ username="https://www.google.com/accounts/o8/id?id=AItOawmWBvsZvSsAL8P2ye3F0OBStjFCVnOImzM"
+ nickname="Jarno"
+ subject="Properly saving the changes before hard reset..?"
+ date="2014-12-03T21:46:23Z"
+ content="""
+BTW, my complete discard-all-my-changes.sh is, at the moment:
+
+    git annex add .
+    git annex proxy -- git commit -m \"Temp commit: changes to be discarded on $HOSTNAME.\"
+    git annex proxy -- git reset --hard origin/master
+    git annex get
+    git -c core.bare=false clean -dfi
+
+Is that `proxy -- git commit` even the proper way to save the changes locally before discarding, or will the branch change of the proxied commit always lose the annexed data? While testing, I've noticed that `add .` followed by `proxy -- git commit` at least sometimes replaces the contents of the changed file with a hash, with no apparent way to get them back.
+
+Example:
+
+    $ echo \"test string\" >> show-changes.sh
+    
+    $ git annex add show-changes.sh
+    add show-changes.sh ok
+    (Recording state in git...)
+    
+    $ git annex proxy -- git commit -m \"test\"
+    [annex/direct/master a2a594f] test
+     1 file changed, 1 insertion(+), 1 deletion(-)
+
+    $ cat show-changes.sh
+    ..\..\..\..\.git\annex\objects\QG\3J\SHA256E-s313--a05d6ed6cb73f6a95d98606d37e01
+    4e5b65e86b9fdb8e9fedded7d36a06ea90a.sh\SHA256E-s313--a05d6ed6cb73f6a95d98606d37e
+    014e5b65e86b9fdb8e9fedded7d36a06ea90a.sh    
+
+    $ git annex get show-changes.sh
+    get show-changes.sh (not available)
+      No other repository is known to contain the file.
+    failed
+    git-annex: get: 1 failed
+
+    $ git annex fsck show-changes.sh
+    fsck show-changes.sh (fixing link) ok
+    (Recording state in git...)
+
+    $ cat show-changes.sh
+    .git\annex\objects\QG\3J\SHA256E-s313--a05d6ed6cb73f6a95d98606d37e014e5b65e86b9f
+    db8e9fedded7d36a06ea90a.sh\SHA256E-s313--a05d6ed6cb73f6a95d98606d37e014e5b65e86b
+    9fdb8e9fedded7d36a06ea90a.sh
+
+"""]]
diff --git a/doc/forum/How_to_emulate___34__one-way_sync__34__s_in_a_direct_repo__63__/comment_4_d899cf35801636e618e4675d91d6104a._comment b/doc/forum/How_to_emulate___34__one-way_sync__34__s_in_a_direct_repo__63__/comment_4_d899cf35801636e618e4675d91d6104a._comment
new file mode 100644
index 0000000000..146156b2df
--- /dev/null
+++ b/doc/forum/How_to_emulate___34__one-way_sync__34__s_in_a_direct_repo__63__/comment_4_d899cf35801636e618e4675d91d6104a._comment
@@ -0,0 +1,24 @@
+[[!comment format=mdwn
+ username="https://www.google.com/accounts/o8/id?id=AItOawnbPKLjBONawBd74MKJZo05juCqdsP1jAU"
+ nickname="Ramon"
+ subject="comment 4"
+ date="2015-03-01T23:58:19Z"
+ content="""
+But is there a way to have the changes in one node (the \"read only\") discarded when there are changes in the remote, so that the new version in the remote is propagated to the read only node?
+So I guess what I'd like is something like *update-changes-from-others.sh* (Like sync, but don't commit any local changes. Merge them like sync, don't discard) but without the merging, so discarding any changes as soon as a new version appears in the remote. 
+
+
+I've tried the ```remote.foo.annex-readonly = true``` in Android but I am not getting my intended behavior.
+
+For instance, suppose a computer and a Android device (but with direct mode, since working with assistant)
+
+- Create file in computer
+- Let it appear in Android
+- Modify in Android; this change apparently does not propagate back to the computer.
+- Modify again in computer; sometime later one gets a conflict (file.variant-xxxx and file-variant-yyyy, with the computer getting one of the conflicts as a link that points nowhere).
+
+
+Adding ```autocommit = false``` does not help either.
+
+
+"""]]
diff --git a/doc/forum/How_to_expire_old_versions_of_files_that_have_been_edited__63__.mdwn b/doc/forum/How_to_expire_old_versions_of_files_that_have_been_edited__63__.mdwn
new file mode 100644
index 0000000000..f06135c24e
--- /dev/null
+++ b/doc/forum/How_to_expire_old_versions_of_files_that_have_been_edited__63__.mdwn
@@ -0,0 +1,7 @@
+My annex contains several large files that I have unlocked, edited, and committed again, i.e. the annex contains the version history of those files. However, I don't want the history -- keeping the latest version is good enough for me. Running `git annex unused` won't detect those old versions, though, because they aren't unused as old Git revisions still refer to them. So I wonder:
+
+1. What is the best way to get rid of the old versions of files in the annex?
+
+2. What is the best way to detect old versions of files in the annex?
+
+I guess, I could run `git rebase -i` to squash commits to those files into one commit, thereby getting rid of the references to the old copies, but that approach feels awkward and error prone. Is anyone aware of a better way?
diff --git a/doc/forum/How_to_expire_old_versions_of_files_that_have_been_edited__63__/comment_1_dccf4dc4483d08e5e2936b2cadeafeaf._comment b/doc/forum/How_to_expire_old_versions_of_files_that_have_been_edited__63__/comment_1_dccf4dc4483d08e5e2936b2cadeafeaf._comment
new file mode 100644
index 0000000000..ee4fe2e6ce
--- /dev/null
+++ b/doc/forum/How_to_expire_old_versions_of_files_that_have_been_edited__63__/comment_1_dccf4dc4483d08e5e2936b2cadeafeaf._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="http://peter-simons.myopenid.com/"
+ ip="77.186.179.173"
+ subject="comment 1"
+ date="2012-02-09T18:53:00Z"
+ content="""
+Sorry for commmenting on my own question ... I think I just figured out that `git annex unused` *does* in fact do what I want. When I tried it, it just didn't show the obsolete versions of the files I edited because I hadn't yet synchronized all repositories, so that was why the obsolete versions were still considered used.
+"""]]
diff --git a/doc/forum/How_to_expire_old_versions_of_files_that_have_been_edited__63__/comment_2_5710294c1c8652c12b6df2233255a45e._comment b/doc/forum/How_to_expire_old_versions_of_files_that_have_been_edited__63__/comment_2_5710294c1c8652c12b6df2233255a45e._comment
new file mode 100644
index 0000000000..576093a87f
--- /dev/null
+++ b/doc/forum/How_to_expire_old_versions_of_files_that_have_been_edited__63__/comment_2_5710294c1c8652c12b6df2233255a45e._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="http://joey.kitenet.net/"
+ nickname="joey"
+ subject="comment 2"
+ date="2012-02-09T19:42:28Z"
+ content="""
+Yes, contents are still considered used while tags or refs refer to them. Including remote tracking branches like `remotes/origin/master`
+"""]]
diff --git a/doc/forum/How_to_find_deleted_files_that_I_know_have_been_backed_up__63__.mdwn b/doc/forum/How_to_find_deleted_files_that_I_know_have_been_backed_up__63__.mdwn
new file mode 100644
index 0000000000..998c608dd4
--- /dev/null
+++ b/doc/forum/How_to_find_deleted_files_that_I_know_have_been_backed_up__63__.mdwn
@@ -0,0 +1,9 @@
+
+
+I removed some archived directories perhaps foolishly with `rm -rf`. How do I find the files that I've had deleted?
+
+
+
+I also have an issue where by I want one command to sync between two hardrives and [github](https://github.com/kaihendry/uploadme). Or do I have to: `git-annex move --to {foo,bar}; git-annex drop; git-annex sync`? Basically I want copies everywhere except on my laptop (X1C3).
+
+I also expected my git dir to be much smaller than 1.4GB after dropping everything. Thanks!
diff --git a/doc/forum/How_to_find_deleted_files_that_I_know_have_been_backed_up__63__/comment_1_8332f71241335a31e270407477bd84f3._comment b/doc/forum/How_to_find_deleted_files_that_I_know_have_been_backed_up__63__/comment_1_8332f71241335a31e270407477bd84f3._comment
new file mode 100644
index 0000000000..d142d742c1
--- /dev/null
+++ b/doc/forum/How_to_find_deleted_files_that_I_know_have_been_backed_up__63__/comment_1_8332f71241335a31e270407477bd84f3._comment
@@ -0,0 +1,18 @@
+[[!comment format=mdwn
+ username="joey"
+ subject="""comment 1"""
+ date="2015-04-18T19:06:27Z"
+ content="""
+Is your titular question a git-annex question, or a generic git question?
+Because I think the answer would be the same in either case. Ie, `git log`
+or `git status` will tell you what changes you've made to the work tree.
+
+If you've deleted files from your git working tree with rm -rf, then
+their content is still stored in the .git directory. This is also true
+when using git-annex (unless you're using direct mode). 
+
+The size of your .git directory might be a clue: If you've deleted
+files from the working tree, you may not have dropped their content
+from git annex. You can use `git annex unused` to find and clean up
+such file contents.
+"""]]
diff --git a/doc/forum/How_to_find_deleted_files_that_I_know_have_been_backed_up__63__/comment_2_984286a35ec828f1e8dda928ea577571._comment b/doc/forum/How_to_find_deleted_files_that_I_know_have_been_backed_up__63__/comment_2_984286a35ec828f1e8dda928ea577571._comment
new file mode 100644
index 0000000000..15a48bfe41
--- /dev/null
+++ b/doc/forum/How_to_find_deleted_files_that_I_know_have_been_backed_up__63__/comment_2_984286a35ec828f1e8dda928ea577571._comment
@@ -0,0 +1,35 @@
+[[!comment format=mdwn
+ username="http://hendry.iki.fi/"
+ nickname="Kai Hendry"
+ subject="git annex unused doesn't help"
+ date="2015-04-19T13:24:50Z"
+ content="""
+This is a git-annex question. I want to find files that are not in my working tree but are on the backups on my two USB hard drives.
+
+
+I see in this commit the images that I deleted: 
+
+How can I simply just get a listing of where the copies are?
+
+For example I deleted IMG_4110.JPG.
+
+But when I run:
+
+    X1C3:~/media/uploadme$ git-annex whereis IMG_4110.JPG
+    git-annex: IMG_4110.JPG not found
+
+I am pretty confident I have a copy of this on my external USB drives.
+
+Also tried another file:
+
+    X1C3:~/media/uploadme$ git-annex whereis IMG_4558.JPG
+    git-annex: IMG_4558.JPG not found
+    X1C3:~/media/uploadme$ git-annex whereis 2014-10-05/IMG_4558.JPG
+    git-annex: 2014-10-05/IMG_4558.JPG not found
+
+So I am still in the dark how to see how git-annex tracks deleted files across my remotes.
+
+
+
+
+"""]]
diff --git a/doc/forum/How_to_find_deleted_files_that_I_know_have_been_backed_up__63__/comment_3_29f6f9df1ad22113e9690b0d1da36ba0._comment b/doc/forum/How_to_find_deleted_files_that_I_know_have_been_backed_up__63__/comment_3_29f6f9df1ad22113e9690b0d1da36ba0._comment
new file mode 100644
index 0000000000..3a39314a98
--- /dev/null
+++ b/doc/forum/How_to_find_deleted_files_that_I_know_have_been_backed_up__63__/comment_3_29f6f9df1ad22113e9690b0d1da36ba0._comment
@@ -0,0 +1,13 @@
+[[!comment format=mdwn
+ username="CandyAngel"
+ subject="comment 3"
+ date="2015-04-19T15:51:45Z"
+ content="""
+Have you tried checking out a commit which it was present in and then using whereis?
+
+Not sure if you can do:
+
+    git annex whereis --key $KEY
+
+But if not, may be worth a feature request!
+"""]]
diff --git a/doc/forum/How_to_find_deleted_files_that_I_know_have_been_backed_up__63__/comment_4_43549b3d231f52cf53a66c477c34a708._comment b/doc/forum/How_to_find_deleted_files_that_I_know_have_been_backed_up__63__/comment_4_43549b3d231f52cf53a66c477c34a708._comment
new file mode 100644
index 0000000000..7d29bec3a7
--- /dev/null
+++ b/doc/forum/How_to_find_deleted_files_that_I_know_have_been_backed_up__63__/comment_4_43549b3d231f52cf53a66c477c34a708._comment
@@ -0,0 +1,15 @@
+[[!comment format=mdwn
+ username="joey"
+ subject="""comment 4"""
+ date="2015-04-21T20:22:25Z"
+ content="""
+Yeah, you need to `git checkout` a tree from before you deleted the files,
+and then you'll be able to use `git annex whereis` in there on the deleted
+files. This will tell you where the files are currently located (not some historical data).
+
+`git annex whereis --key` is indeed an alternative approach, if you know
+the key corresponding to the deleted file. You can see the keys in the git
+diff, if you know where to look.
+
+[[internals]] will let you understand how this all really works.
+"""]]
diff --git a/doc/forum/How_to_find_deleted_files_that_I_know_have_been_backed_up__63__/comment_5_94aca10f84783f35d927dbefbeba263a._comment b/doc/forum/How_to_find_deleted_files_that_I_know_have_been_backed_up__63__/comment_5_94aca10f84783f35d927dbefbeba263a._comment
new file mode 100644
index 0000000000..77cf1c2895
--- /dev/null
+++ b/doc/forum/How_to_find_deleted_files_that_I_know_have_been_backed_up__63__/comment_5_94aca10f84783f35d927dbefbeba263a._comment
@@ -0,0 +1,16 @@
+[[!comment format=mdwn
+ username="http://hendry.iki.fi/"
+ nickname="Kai Hendry"
+ subject="Two more questions"
+ date="2015-05-07T07:40:41Z"
+ content="""
+How do I undelete in my case then? I.e. not `git checkout ` to find where they are.
+
+`git revert c53dbdd9bd7879d68635a2adc81a7bc59a84c5ea`
+
+
+
+Assuming I make no effort to undelete or recover the files:
+
+The (deleted) files I have copies of on my other two hard drives. Do they detect I've deleted the files and just clean up? How does that process work?
+"""]]
diff --git a/doc/forum/How_to_find_out_if_assistant_is_still_repairing__63__.mdwn b/doc/forum/How_to_find_out_if_assistant_is_still_repairing__63__.mdwn
new file mode 100644
index 0000000000..a959424348
--- /dev/null
+++ b/doc/forum/How_to_find_out_if_assistant_is_still_repairing__63__.mdwn
@@ -0,0 +1 @@
+I have a corrupted repository, that git annex assistant wanted to repair. Before clicing on the button to start the repairing process, I had to deactivate my internet connection for another reason. The daemon.log is showing some connection errors to my remotes. But since pressing the button to repair my repo, the webapp is not showing the dashboard and there's no indicator that's trustworthy enough, that stuff is happening. There are some git and git annex processes running, but is there really now better way to know, if the repairing process is making progress, or if I need to restart the assistant because I needed to reset my internet connection.
diff --git a/doc/forum/How_to_find_out_if_assistant_is_still_repairing__63__/comment_1_9dd192a8e0bfa9e533b22070b87792af._comment b/doc/forum/How_to_find_out_if_assistant_is_still_repairing__63__/comment_1_9dd192a8e0bfa9e533b22070b87792af._comment
new file mode 100644
index 0000000000..988d23374f
--- /dev/null
+++ b/doc/forum/How_to_find_out_if_assistant_is_still_repairing__63__/comment_1_9dd192a8e0bfa9e533b22070b87792af._comment
@@ -0,0 +1,7 @@
+[[!comment format=mdwn
+ username="openmedi"
+ subject="comment 1"
+ date="2016-05-10T14:42:21Z"
+ content="""
+The assistant was indeed still repairing, but there has to be a way to better indicate this. Also it would be nice to let the user know, that, while in repair mode, you can not change syncing settings of your remotes. [See also.](http://git-annex.branchable.com/forum/Remote_in_webapp_shows_syncing_disabled_but_is_enabled___40__maybe__41__/)
+"""]]
diff --git a/doc/forum/How_to_get_detailed_information_on_special_remotes__63__.mdwn b/doc/forum/How_to_get_detailed_information_on_special_remotes__63__.mdwn
new file mode 100644
index 0000000000..415b7570e3
--- /dev/null
+++ b/doc/forum/How_to_get_detailed_information_on_special_remotes__63__.mdwn
@@ -0,0 +1 @@
+How can I get information on already-configured special remotes in a repository?  I'd like to find out what type a given special remote is, and which URL/bucket, encryption mode, etc. was used to set it up.
diff --git a/doc/forum/How_to_get_detailed_information_on_special_remotes__63__/comment_1_f1eca38ca0c1edb66e64ac64f0321829._comment b/doc/forum/How_to_get_detailed_information_on_special_remotes__63__/comment_1_f1eca38ca0c1edb66e64ac64f0321829._comment
new file mode 100644
index 0000000000..84b67f75e1
--- /dev/null
+++ b/doc/forum/How_to_get_detailed_information_on_special_remotes__63__/comment_1_f1eca38ca0c1edb66e64ac64f0321829._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="joey"
+ subject="""comment 1"""
+ date="2014-10-28T20:39:41Z"
+ content="""
+The latest version of git-annex adds the ability to `git annex info
+$remote` and get all the info you asked for. Enjoy!
+"""]]
diff --git a/doc/forum/How_to_get_detailed_information_on_special_remotes__63__/comment_2_73f8b7ebde5870f92bec02d15a533ef3._comment b/doc/forum/How_to_get_detailed_information_on_special_remotes__63__/comment_2_73f8b7ebde5870f92bec02d15a533ef3._comment
new file mode 100644
index 0000000000..cc174870c3
--- /dev/null
+++ b/doc/forum/How_to_get_detailed_information_on_special_remotes__63__/comment_2_73f8b7ebde5870f92bec02d15a533ef3._comment
@@ -0,0 +1,7 @@
+[[!comment format=mdwn
+ username="gernot"
+ subject="comment 2"
+ date="2014-11-09T20:30:39Z"
+ content="""
+That's perfect.  Thanks, Joey!
+"""]]
diff --git a/doc/forum/How_to_get_git-annex_to_forget_a_commit__63__.mdwn b/doc/forum/How_to_get_git-annex_to_forget_a_commit__63__.mdwn
new file mode 100644
index 0000000000..d5aa71b448
--- /dev/null
+++ b/doc/forum/How_to_get_git-annex_to_forget_a_commit__63__.mdwn
@@ -0,0 +1,54 @@
+I've done a `git annex add` and `git commit` on my annex which included some files I don't want to add to the annex. I've tried to reverse it all out, but whenever I `git annex add` something, the unwanted files show up in the git-annex branch.
+
+    git init forgetmenot
+    cd forgetmenot
+    git commit -m "create" --allow-empty
+    git annex init fmn
+    
+    echo 'foo' > foo
+    echo 'bar' > bar
+    
+    git annex add
+    git commit -m "add foo"
+    
+    git log --oneline --name-only
+    # 28634c0 add foo
+    # bar
+    # foo
+    # 4a87050 create
+
+    git log --oneline --name-only git-annex
+    74e6969 update
+    41d/a26/SHA256E-s4--b5bb9d8014a0f9b1d61e21e796d78dccdf1352f23cd32812f4850b878ae4944c.log
+    a70/4a5/SHA256E-s4--7d865e959b2466918c9863afca942d0fb89d7c9ac0c99bafc3749504ded97730.log
+    95285ed update
+    uuid.log
+    2135e07 branch created
+
+If you now try to get git-annex to forget by reverting *master* and *git-annex* and only adding/commiting *foo*, the master branch ends up correct, but git-annex magically remembers *bar*!
+
+    git reset --hard HEAD^
+    git branch -f git-annex git-annex^
+
+    echo 'foo' > foo
+    git annex add
+    git commit -m "add foo"
+
+    git log --oneline --name-only
+    # 1b4889e add foo
+    # foo
+    # 4a87050 create
+
+    git log --oneline --name-only git-annex
+    # 3d0b9bc update
+    # 41d/a26/SHA256E-s4--b5bb9d8014a0f9b1d61e21e796d78dccdf1352f23cd32812f4850b878ae4944c.log
+    # a70/4a5/SHA256E-s4--7d865e959b2466918c9863afca942d0fb89d7c9ac0c99bafc3749504ded97730.log
+    # 2e17a19 update
+    # uuid.log
+    # 646776b branch created
+
+How is git-annex remembering this and how can I get it to completely forget?
+
+I have tried `git gc --aggressive --prune=all`, `git annex fsck --all` and `git annex drop unused` but somehow, git-annex is remembering bar existed.
+
+This is an exercise in micro-managing the git-annex branch a bit, but this situation does also cause git-annex to complain about the missing files on fsck (0 out of 2 copies) so it isn't just being a control freak! Honest! :)
diff --git a/doc/forum/How_to_get_git-annex_to_forget_a_commit__63__/comment_1_65471c42e163ac8ee6ec109f1397271b._comment b/doc/forum/How_to_get_git-annex_to_forget_a_commit__63__/comment_1_65471c42e163ac8ee6ec109f1397271b._comment
new file mode 100644
index 0000000000..44b36ba604
--- /dev/null
+++ b/doc/forum/How_to_get_git-annex_to_forget_a_commit__63__/comment_1_65471c42e163ac8ee6ec109f1397271b._comment
@@ -0,0 +1,12 @@
+[[!comment format=mdwn
+ username="CandyAngel"
+ ip="81.111.193.130"
+ subject="comment 1"
+ date="2014-07-17T15:43:03Z"
+ content="""
+Joey has pointed me to the solution.
+
+git-annex was remembering these files due to them being present in *.git/annex/index*.
+
+A simple `rm .git/annex/index` after moving the git-annex branch to the earlier commit prevents the \"ghost\" log files from being recreated and Joey confirmed this is safe to do (git-annex automatically recreates it).
+"""]]
diff --git a/doc/forum/How_to_get_git-annex_to_forget_a_commit__63__/comment_2_cc991abd9155a8e4d935b04ba432744f._comment b/doc/forum/How_to_get_git-annex_to_forget_a_commit__63__/comment_2_cc991abd9155a8e4d935b04ba432744f._comment
new file mode 100644
index 0000000000..2d15cf35e0
--- /dev/null
+++ b/doc/forum/How_to_get_git-annex_to_forget_a_commit__63__/comment_2_cc991abd9155a8e4d935b04ba432744f._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="http://joeyh.name/"
+ ip="209.250.56.2"
+ subject="comment 2"
+ date="2014-07-17T16:05:33Z"
+ content="""
+Of course, if the repository has any git remotes, there will also be remote-tracking branches for the git-annex branch that get automatically merged back in, etc. 
+"""]]
diff --git a/doc/forum/How_to_handle_the_git-annex_branch__63__.mdwn b/doc/forum/How_to_handle_the_git-annex_branch__63__.mdwn
new file mode 100644
index 0000000000..49c732e226
--- /dev/null
+++ b/doc/forum/How_to_handle_the_git-annex_branch__63__.mdwn
@@ -0,0 +1,5 @@
+Hi!
+
+When I update one repository to/from another, am I to push/pull only the master branch (and other branches I'd have created myself) or also the git-annex branch?
+
+From what I understand, the git-annex branch is local to one repository and has no interest in being merged in another, but some examples make use of git pull --all (which merges distant git-annex branch into local).
diff --git a/doc/forum/How_to_handle_the_git-annex_branch__63__/comment_1_800bd55b322e72f229882d7fd3888b14._comment b/doc/forum/How_to_handle_the_git-annex_branch__63__/comment_1_800bd55b322e72f229882d7fd3888b14._comment
new file mode 100644
index 0000000000..6c5a5144f3
--- /dev/null
+++ b/doc/forum/How_to_handle_the_git-annex_branch__63__/comment_1_800bd55b322e72f229882d7fd3888b14._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="http://joey.kitenet.net/"
+ nickname="joey"
+ subject="comment 1"
+ date="2012-04-10T16:05:40Z"
+ content="""
+The git-annex branch is how the annex information for the different repositories is communicated around, so yes, you need to push/pull it. 
+"""]]
diff --git a/doc/forum/How_to_hide_broken_symlinks.mdwn b/doc/forum/How_to_hide_broken_symlinks.mdwn
new file mode 100644
index 0000000000..7c92f32514
--- /dev/null
+++ b/doc/forum/How_to_hide_broken_symlinks.mdwn
@@ -0,0 +1,45 @@
+This is a method for hiding broken links using git-annex views.
+
+Each annex will need it's own name for this system to work. For this example I'll use "localdrive." After getting file content, run:
+
+    git-annex metadata --not --in=here --metadata in=localdrive . -s in-=localdrive
+    git-annex metadata --in=here --not --metadata in=localdrive . -s in+=localdrive
+    git-annex view /=*
+    git-annex vfilter in=localdrive
+
+Unused links will be hidden. Folder structures will remain the same.
+
+To switch back use:
+
+    git-annex vpop 2
+
+Because this is a lot to type, I've placed these in a bash script in the base folder (ignored with .gitignore so it isn't sent to other repos). The local repo name can be changed by editing THISREPO:
+
+    #!/bin/bash
+    
+    THISREPO='localdrive'
+    
+    git-annex metadata --not --in=here --metadata in=$THISREPO . -s in-=$THISREPO
+    git-annex metadata --in=here --not --metadata in=$THISREPO . -s in+=$THISREPO
+    git-annex view /=*
+    git-annex vfilter in=$THISREPO
+    
+    exit 0
+
+## Hiding Broken Links in Preferred Content Repos
+
+If you have a repo with preferred content settings, this can be shortened to a single script which can be run to "refresh" the view:
+
+    #!/bin/bash
+    
+    THISREPO='pcrepo'
+    
+    git-annex vpop 2
+    git-annex sync
+    git-annex get --auto
+    git-annex metadata --not --in=here --metadata in=$THISREPO . -s in-=$THISREPO
+    git-annex metadata --in=here --not --metadata in=$THISREPO . -s in+=$THISREPO
+    git-annex view /=*
+    git-annex vfilter in=$THISREPO
+    
+    exit 0
diff --git a/doc/forum/How_to_hide_broken_symlinks/comment_1_07a70090a9ac8f675233b08b24f55544._comment b/doc/forum/How_to_hide_broken_symlinks/comment_1_07a70090a9ac8f675233b08b24f55544._comment
new file mode 100644
index 0000000000..c4f55b85ba
--- /dev/null
+++ b/doc/forum/How_to_hide_broken_symlinks/comment_1_07a70090a9ac8f675233b08b24f55544._comment
@@ -0,0 +1,22 @@
+[[!comment format=mdwn
+ username="joey"
+ subject="""comment 1"""
+ date="2015-01-05T20:52:11Z"
+ content="""
+This works, but it's very inneficient. You're
+adding metadata (which is committed to git!) to represent location information. 
+git-annex already knows about location information,
+so it could just look at it and use it to build the view.
+
+The reason I haven't implemented this yet is I'm not clear about what to do
+if a file is dropped while in a view. It seems like git-annex should update
+the view to no longer show the dropped file. But this means that any change
+to a file's presence needs to update the view, and so it entangles the
+implementation of views with a core part of git-annex. And it might be
+pretty inneficient to need to build a new view branch each time a file is
+dropped too.
+
+With your hack, the view doesn't automatically update when a file is
+dropped. Since you're using it this way, maybe you have some advice or
+thoughts about this question?
+"""]]
diff --git a/doc/forum/How_to_hide_broken_symlinks/comment_2_011f6b97b1eaee1ae513a4a0e445d105._comment b/doc/forum/How_to_hide_broken_symlinks/comment_2_011f6b97b1eaee1ae513a4a0e445d105._comment
new file mode 100644
index 0000000000..21a1c9288d
--- /dev/null
+++ b/doc/forum/How_to_hide_broken_symlinks/comment_2_011f6b97b1eaee1ae513a4a0e445d105._comment
@@ -0,0 +1,12 @@
+[[!comment format=mdwn
+ username="https://www.google.com/accounts/o8/id?id=AItOawm_O5eskJUCHFrhtP9jm5atZQlBkxV3g9U"
+ nickname="Markus"
+ subject="comment 2"
+ date="2015-03-26T16:18:08Z"
+ content="""
+I thought about the same thing today.
+
+Well view aren't changing automatically when you change metadata, you'd have to re-create the view by leaving and entering.
+
+It would be great to have some \"internal metadata\" like if the file is present.
+"""]]
diff --git a/doc/forum/How_to_hide_broken_symlinks/comment_3_e3606aa746f516fc771d5d9bf93d70af._comment b/doc/forum/How_to_hide_broken_symlinks/comment_3_e3606aa746f516fc771d5d9bf93d70af._comment
new file mode 100644
index 0000000000..2103ec35e3
--- /dev/null
+++ b/doc/forum/How_to_hide_broken_symlinks/comment_3_e3606aa746f516fc771d5d9bf93d70af._comment
@@ -0,0 +1,7 @@
+[[!comment format=mdwn
+ username="GiovanniBiscuolo"
+ subject="Related forum post"
+ date="2015-04-23T11:03:39Z"
+ content="""
+[How do I hide files not present in the local annex?](http://git-annex.branchable.com/forum/How_do_I_hide_files_not_present_in_the_local_annex__63__/)
+"""]]
diff --git a/doc/forum/How_to_hide_broken_symlinks/comment_4_c6e3ef3ba5f6e9e54d998bcbe3035650._comment b/doc/forum/How_to_hide_broken_symlinks/comment_4_c6e3ef3ba5f6e9e54d998bcbe3035650._comment
new file mode 100644
index 0000000000..206333db73
--- /dev/null
+++ b/doc/forum/How_to_hide_broken_symlinks/comment_4_c6e3ef3ba5f6e9e54d998bcbe3035650._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="https://www.google.com/accounts/o8/id?id=AItOawnVnsqEy82M-MuS2gLri-az83wSQ6lXSrc"
+ nickname="Jean"
+ subject="comment 4"
+ date="2015-04-26T01:19:21Z"
+ content="""
+If a file is dropped while in a view, how about letting the relevant symlinks become broken until the view is refreshed?
+"""]]
diff --git a/doc/forum/How_to_improve_performance_with_v6_repos__63__.mdwn b/doc/forum/How_to_improve_performance_with_v6_repos__63__.mdwn
new file mode 100644
index 0000000000..6a4f5e8cf5
--- /dev/null
+++ b/doc/forum/How_to_improve_performance_with_v6_repos__63__.mdwn
@@ -0,0 +1,3 @@
+I've been using git-annex to manage a repo of ~4000 files (an MP3 collection). I used to use direct mode, but switched to v6 and am using unlocked files instead. However, even small commits have been taking 10-20 minutes. Looking at `ps` output shows `git-annex smudge` processes running against all of the files in the repo, one by one.
+
+Is there any way I can speed things up?
diff --git a/doc/forum/How_to_improve_performance_with_v6_repos__63__/comment_1_56e0fead15a88ea12262e6091972c6d6._comment b/doc/forum/How_to_improve_performance_with_v6_repos__63__/comment_1_56e0fead15a88ea12262e6091972c6d6._comment
new file mode 100644
index 0000000000..c5f1866b13
--- /dev/null
+++ b/doc/forum/How_to_improve_performance_with_v6_repos__63__/comment_1_56e0fead15a88ea12262e6091972c6d6._comment
@@ -0,0 +1,7 @@
+[[!comment format=mdwn
+ username="page"
+ subject="Some additional notes"
+ date="2016-02-28T11:44:39Z"
+ content="""
+Just to add some info here. I just started using git-annex and v6 sounded like the way to go. I tried to add my photo collection (around 50k files) and all operations take a huge amount of time. I did 'git add photos/' (to add them in unlocked mode), which took hours. Then I run 'git annex sync', which took >10h again to do the commit. After that i run 'git annex sync' from another remote, and it's syncing items at ~2/s (smudge processes as the op mentions, not blocked either by cpu nor io). I'm not sure if I'm doing something wrong, but this is completely unusable for me. I'm all ears if I can try something, otherwise I'm gonna give v5 a try, I guess.
+"""]]
diff --git a/doc/forum/How_to_improve_performance_with_v6_repos__63__/comment_2_93a6a0779f52bdd3de6e4d2500cfd223._comment b/doc/forum/How_to_improve_performance_with_v6_repos__63__/comment_2_93a6a0779f52bdd3de6e4d2500cfd223._comment
new file mode 100644
index 0000000000..7e7d55028d
--- /dev/null
+++ b/doc/forum/How_to_improve_performance_with_v6_repos__63__/comment_2_93a6a0779f52bdd3de6e4d2500cfd223._comment
@@ -0,0 +1,10 @@
+[[!comment format=mdwn
+ username="joey"
+ subject="""comment 2"""
+ date="2016-07-06T18:57:15Z"
+ content="""
+This problem is dicussed in detail in [[todo/smudge]] and is the main
+reason that v6 mode is not production ready.
+
+I have patches to git that avoid this problem.
+"""]]
diff --git a/doc/forum/How_to_know_why_is_git-annex_uploading_a_file_to_a_transfer_repository.mdwn b/doc/forum/How_to_know_why_is_git-annex_uploading_a_file_to_a_transfer_repository.mdwn
new file mode 100644
index 0000000000..48ad2d1d83
--- /dev/null
+++ b/doc/forum/How_to_know_why_is_git-annex_uploading_a_file_to_a_transfer_repository.mdwn
@@ -0,0 +1,3 @@
+I have git-annex set up with three clients and a transfer repository and everything was working fine, but a couple of days ago I noticed that git-annex was starting to upload to the transfer repository files that where already present on the three clients, and it hasn't stopped since. It's uploading all my files and I don't really know why.
+
+Is there a way to know why does git-annex think it needs to upload this files?
diff --git a/doc/forum/How_to_know_why_is_git-annex_uploading_a_file_to_a_transfer_repository/comment_1_17db96492e6bc0e243fc7cb62565c4c4._comment b/doc/forum/How_to_know_why_is_git-annex_uploading_a_file_to_a_transfer_repository/comment_1_17db96492e6bc0e243fc7cb62565c4c4._comment
new file mode 100644
index 0000000000..9cbe0939c8
--- /dev/null
+++ b/doc/forum/How_to_know_why_is_git-annex_uploading_a_file_to_a_transfer_repository/comment_1_17db96492e6bc0e243fc7cb62565c4c4._comment
@@ -0,0 +1,17 @@
+[[!comment format=mdwn
+ username="http://joeyh.name/"
+ ip="209.250.56.41"
+ subject="comment 1"
+ date="2014-03-26T17:46:30Z"
+ content="""
+A few likely reasons:
+
+* If a 4th client repository had popped up. 
+* If you have configured a high number of copies, it might only be able to be met by keeping files on the transfer repository.
+* Similarly, if a repository that used to have the files has been marked as dead or deleted, more copies might be needed to make up for that.
+* For completeness, if the transfer repository accidentally had its type changed to some other kind of repository, like a full backup.
+
+You can enable debugging (start with --debug or go into the webapp's preferences) and it might say a little more, but the debugging info is not very good.
+
+The best thing is probably to look at one single file, use `git annex whereis` on the file to see what repositories contain it, and then think about how that interacts with the [[preferred_content_expression_of_the_transfer_repository|preferred_content/standard_groups]].
+"""]]
diff --git a/doc/forum/How_to_know_why_is_git-annex_uploading_a_file_to_a_transfer_repository/comment_2_e772ea0383ac690cbcbcf125258986cf._comment b/doc/forum/How_to_know_why_is_git-annex_uploading_a_file_to_a_transfer_repository/comment_2_e772ea0383ac690cbcbcf125258986cf._comment
new file mode 100644
index 0000000000..27f158ea1e
--- /dev/null
+++ b/doc/forum/How_to_know_why_is_git-annex_uploading_a_file_to_a_transfer_repository/comment_2_e772ea0383ac690cbcbcf125258986cf._comment
@@ -0,0 +1,16 @@
+[[!comment format=mdwn
+ username="https://www.google.com/accounts/o8/id?id=AItOawk9nck8WX8-ADF3Fdh5vFo4Qrw1I_bJcR8"
+ nickname="Jon Ander"
+ subject="comment 2"
+ date="2014-04-01T08:01:00Z"
+ content="""
+I thought I had already checked all does possibilities, the repositories where in the correct group and no new repositories had been added. *git annex info* didn't show anything weird and nor did *git annex whereis*. I finally found out *git annex vicfg* and I found two new repositories with no name in the repository groups. It looked something like this:
+
+    # (for )
+    group repository-hash = client
+    # (for )
+    group repository-hash = client
+
+
+No idea of how they got there, but setting both of them to unwanted solved the issue.
+"""]]
diff --git a/doc/forum/How_to_list_all_existing_metadata_types__63__.mdwn b/doc/forum/How_to_list_all_existing_metadata_types__63__.mdwn
new file mode 100644
index 0000000000..43b3c1cfc1
--- /dev/null
+++ b/doc/forum/How_to_list_all_existing_metadata_types__63__.mdwn
@@ -0,0 +1,15 @@
+Is there any way to list all of the existing tag and metadata field types? What I mean is, I have files tagged with several different tags, files with several metadata fields; is there any way to list all the tag and field names being used (not all the files WITH those tags)?
+
+For example, something like:
+
+    git annex metadata --listfields
+    lastchanged
+    month
+    month-lastchanged
+    year
+    year-lastchanged
+
+    git annex metadata --listtags
+    Public
+    Personal
+    Work
diff --git a/doc/forum/How_to_list_all_existing_metadata_types__63__/comment_1_a8c30f697f32a3807661a59482d79b18._comment b/doc/forum/How_to_list_all_existing_metadata_types__63__/comment_1_a8c30f697f32a3807661a59482d79b18._comment
new file mode 100644
index 0000000000..3405ea4d6c
--- /dev/null
+++ b/doc/forum/How_to_list_all_existing_metadata_types__63__/comment_1_a8c30f697f32a3807661a59482d79b18._comment
@@ -0,0 +1,19 @@
+[[!comment format=mdwn
+ username="http://joeyh.name/"
+ ip="209.250.56.54"
+ subject="comment 1"
+ date="2014-10-09T19:57:54Z"
+ content="""
+git-annex doesn't currently have a way to generate those lists itself, but you could use `git annex metadata --json` to get the metadata of all files, and pipe that json into a parser to get the data you want.
+
+The output could also be parsed in non-json mode. For example, this will list the tags:
+
+    git annex metadata  | grep '^  tag=' | cut -d '=' -f 2 | sort | uniq
+
+Although it's possible for metadata to contain newlines, and so parsing the json is a more reliable approach.
+
+Another nice way to see all the tags is to switch to a view of all tags:
+
+    git annex view 'tag=*'
+    ls
+"""]]
diff --git a/doc/forum/How_to_make_Maven_releases_work_with_git_annex___63__.mdwn b/doc/forum/How_to_make_Maven_releases_work_with_git_annex___63__.mdwn
new file mode 100644
index 0000000000..b39e69d7b5
--- /dev/null
+++ b/doc/forum/How_to_make_Maven_releases_work_with_git_annex___63__.mdwn
@@ -0,0 +1,5 @@
+Hi,
+
+In normal maven release procedure, it re-clones the gitrepo from the tag to do the release from either git or local file, But as git-annex uses symlinks when the repo is clones by maven, it misses the files. Any recommendations how i can make this work ? 
+
+Also would love some data on who is using git-annex in production environment would be good.
diff --git a/doc/forum/How_to_make_Maven_releases_work_with_git_annex___63__/comment_1_9298aa55771b68873de02e6a7964bbdc._comment b/doc/forum/How_to_make_Maven_releases_work_with_git_annex___63__/comment_1_9298aa55771b68873de02e6a7964bbdc._comment
new file mode 100644
index 0000000000..ca62bd43e0
--- /dev/null
+++ b/doc/forum/How_to_make_Maven_releases_work_with_git_annex___63__/comment_1_9298aa55771b68873de02e6a7964bbdc._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="http://joeyh.name/"
+ ip="4.154.0.21"
+ subject="comment 1"
+ date="2013-07-30T18:20:53Z"
+ content="""
+If you can find a way to run `git annex get` in the cloned repo, it will copy over the files. Perhaps there is some way to make Maven do that via a hook script or similar?
+"""]]
diff --git a/doc/forum/How_to_make_a_server_store_the_files.mdwn b/doc/forum/How_to_make_a_server_store_the_files.mdwn
new file mode 100644
index 0000000000..1af29381f2
--- /dev/null
+++ b/doc/forum/How_to_make_a_server_store_the_files.mdwn
@@ -0,0 +1 @@
+How can I make one of my servers keep a copy of the files on the server, so I can easily SFTP in and download the files. I don't want to see the .git folder, I want to see the files synced there.
diff --git a/doc/forum/How_to_make_a_server_store_the_files/comment_1_20196067475918e788afa0debc4d5ce5._comment b/doc/forum/How_to_make_a_server_store_the_files/comment_1_20196067475918e788afa0debc4d5ce5._comment
new file mode 100644
index 0000000000..e653881637
--- /dev/null
+++ b/doc/forum/How_to_make_a_server_store_the_files/comment_1_20196067475918e788afa0debc4d5ce5._comment
@@ -0,0 +1,9 @@
+[[!comment format=mdwn
+ username="http://joeyh.name/"
+ ip="108.236.230.124"
+ subject="comment 1"
+ date="2014-07-02T17:18:00Z"
+ content="""
+Make a non-bare git repository on the server, and make it have a git post-receive hook that runs 
+\"git annex merge\"
+"""]]
diff --git a/doc/forum/How_to_organise_backup_to_S3__63__.mdwn b/doc/forum/How_to_organise_backup_to_S3__63__.mdwn
new file mode 100644
index 0000000000..25ead06aae
--- /dev/null
+++ b/doc/forum/How_to_organise_backup_to_S3__63__.mdwn
@@ -0,0 +1,10 @@
+Hello,
+
+what is the best way to use git-annex as a backup tool to S3? As far as I know, even if all data is copied to S3, I need a checkout of the git repo, correct? And of course the private key used.
+
+- Is there a way to also save the repository data to S3? To have something like ```git annex restore-from-external-remote ...```
+
+- If not, how do you organise the backup so that it is disaster proof? Upload the repo to some other external server regularly?
+
+Thanks,
+Florian
diff --git a/doc/forum/How_to_organise_backup_to_S3__63__/comment_1_13c6421a3637e30193dc77b54dec4b8d._comment b/doc/forum/How_to_organise_backup_to_S3__63__/comment_1_13c6421a3637e30193dc77b54dec4b8d._comment
new file mode 100644
index 0000000000..0679b05d7d
--- /dev/null
+++ b/doc/forum/How_to_organise_backup_to_S3__63__/comment_1_13c6421a3637e30193dc77b54dec4b8d._comment
@@ -0,0 +1,11 @@
+[[!comment format=mdwn
+ username="joey"
+ subject="""comment 1"""
+ date="2017-12-05T17:30:31Z"
+ content="""
+You can use any git addon that allows git repositories to be pushed to S3
+to store your git repository there, and then everything is on S3. 
+
+Or yes, you can push the git repository to any other git hosting provider
+you want to, to back it up.
+"""]]
diff --git a/doc/forum/How_to_organise_backup_to_S3__63__/comment_2_e0ae05276ef7e1dcb21ef0986b6f7531._comment b/doc/forum/How_to_organise_backup_to_S3__63__/comment_2_e0ae05276ef7e1dcb21ef0986b6f7531._comment
new file mode 100644
index 0000000000..e176eababe
--- /dev/null
+++ b/doc/forum/How_to_organise_backup_to_S3__63__/comment_2_e0ae05276ef7e1dcb21ef0986b6f7531._comment
@@ -0,0 +1,16 @@
+[[!comment format=mdwn
+ username="Horus"
+ avatar="http://cdn.libravatar.org/avatar/8f0ee08b98ea5bba76c3fe112c08851c"
+ subject="comment 2"
+ date="2017-12-06T11:45:33Z"
+ content="""
+Hi,
+
+thanks for your reply. The repo server I used to store just the repository does not need to have git-annex installed, correct? As long as I do not intend to use it for exchanging files... 
+
+This message can be ignored?
+
+    Remote xgm does not have git-annex installed; setting annex-ignore
+
+    This could be a problem with the git-annex installation on the remote. Please make sure that git-annex-shell is available in PATH when you ssh into the remote. Once you have fixed the git-annex installation, run: git annex enableremote xgm
+"""]]
diff --git a/doc/forum/How_to_organise_backup_to_S3__63__/comment_3_7697112a04cdc9a53551257d71143eb5._comment b/doc/forum/How_to_organise_backup_to_S3__63__/comment_3_7697112a04cdc9a53551257d71143eb5._comment
new file mode 100644
index 0000000000..5c30dc647e
--- /dev/null
+++ b/doc/forum/How_to_organise_backup_to_S3__63__/comment_3_7697112a04cdc9a53551257d71143eb5._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="joey"
+ subject="""comment 3"""
+ date="2017-12-08T19:35:51Z"
+ content="""
+Right, you've split the cloud storage into annex and git parts, so it's
+fine for the git part to not support annex then.
+"""]]
diff --git a/doc/forum/How_to_overwrite_local_changes__63__.mdwn b/doc/forum/How_to_overwrite_local_changes__63__.mdwn
new file mode 100644
index 0000000000..904bc086be
--- /dev/null
+++ b/doc/forum/How_to_overwrite_local_changes__63__.mdwn
@@ -0,0 +1,7 @@
+Hello,
+
+I have a repo Desktop (client), a repo Server (transfer) and a repo Notebook (Client). Usually first thing I do in the morning is go git annex sync --content the changes to the laptop. Today I accidently modified some files on the laptop before I synced. After I syncing I know have some files foo.variant-afd1 and foo.variant-a6d3 with which I don't know what to do.
+
+How can I discard local changes and copy all changes from the transfer repo to the laptop? Local changes should be overwritten.
+
+Thanks!
diff --git a/doc/forum/How_to_overwrite_local_changes__63__/comment_1_1753d8c7a0daf04b64d639567a349190._comment b/doc/forum/How_to_overwrite_local_changes__63__/comment_1_1753d8c7a0daf04b64d639567a349190._comment
new file mode 100644
index 0000000000..05dbbed761
--- /dev/null
+++ b/doc/forum/How_to_overwrite_local_changes__63__/comment_1_1753d8c7a0daf04b64d639567a349190._comment
@@ -0,0 +1,20 @@
+[[!comment format=mdwn
+ username="joey"
+ subject="""comment 1"""
+ date="2015-07-06T18:51:43Z"
+ content="""
+Sorry that nobody answered your question until now..
+
+These .variant files preserve both versions of the files that were
+committed in the two different repositories. If you can identify which
+variant you don't want, you can just delete that one, and rename the other
+.variant file back to the original file name.
+
+There's no really great way to tell which variant is which. 
+I'd look at the contents of the variants of the file, unless there are
+too many files to look at.
+
+Another approach is to look at the git log, find the commit that merged
+the unwanted changed (and created these variant files), and `git revert` that
+merge, and the earlier commit that was made accidentally.
+"""]]
diff --git a/doc/forum/How_to_prevent_the_assistant_from_downloading_all_data__63__.mdwn b/doc/forum/How_to_prevent_the_assistant_from_downloading_all_data__63__.mdwn
new file mode 100644
index 0000000000..1b1ddff469
--- /dev/null
+++ b/doc/forum/How_to_prevent_the_assistant_from_downloading_all_data__63__.mdwn
@@ -0,0 +1,9 @@
+I have a repository on my laptop with a single remote on a USB drive. Both were created without the assistant.
+
+The remote holds all the content. My understanding is that this would be a "backup" repository, using the group definitions.
+
+The local repository generally holds no content. When I want a file, I make sure the remote is available, and then I `git annex get` it.
+
+This works fine in git annex, but I'm now experimenting with the assistant. As soon as I fire up the assistant, it starts downloading all of the data from the remote. I don't ever want it to automatically get any content from the remote. If it sees new content in the repository, I would like it to push that out to the remote. But never get anything from the remote, and never drop anything that is in the repository.
+
+How can I setup this behaviour in the assistant? I have set the remote's repository group to "backup". The local repository is not in any group, since none of the groups fit my model for this repo.
diff --git a/doc/forum/How_to_prevent_the_assistant_from_downloading_all_data__63__/comment_1_fd8b287758ad77b3527ae71017cffabf._comment b/doc/forum/How_to_prevent_the_assistant_from_downloading_all_data__63__/comment_1_fd8b287758ad77b3527ae71017cffabf._comment
new file mode 100644
index 0000000000..326158d95a
--- /dev/null
+++ b/doc/forum/How_to_prevent_the_assistant_from_downloading_all_data__63__/comment_1_fd8b287758ad77b3527ae71017cffabf._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="https://www.google.com/accounts/o8/id?id=AItOawn7gQ1zZDdWhXy9H51W2krZYShNmKL3qfM"
+ nickname="Karsten"
+ subject="comment 1"
+ date="2013-01-20T08:11:37Z"
+ content="""
+Run 'git annex vicfg', locate the preferred content setting for your repository, uncomment it and set it to 'present'. Then it shouldn't move content automatically.
+"""]]
diff --git a/doc/forum/How_to_prevent_the_assistant_from_downloading_all_data__63__/comment_2_e8e75b4451aaf55461edf2f3d68797ed._comment b/doc/forum/How_to_prevent_the_assistant_from_downloading_all_data__63__/comment_2_e8e75b4451aaf55461edf2f3d68797ed._comment
new file mode 100644
index 0000000000..790567ecaa
--- /dev/null
+++ b/doc/forum/How_to_prevent_the_assistant_from_downloading_all_data__63__/comment_2_e8e75b4451aaf55461edf2f3d68797ed._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="http://joeyh.name/"
+ ip="2001:4978:f:21a::2"
+ subject="comment 2"
+ date="2013-01-24T02:01:00Z"
+ content="""
+A recent commit allows you to just choose \"manual mode\" in the webapp to get the same effect.
+"""]]
diff --git a/doc/forum/How_to_rename_a_remote__63__.mdwn b/doc/forum/How_to_rename_a_remote__63__.mdwn
new file mode 100644
index 0000000000..be2355a079
--- /dev/null
+++ b/doc/forum/How_to_rename_a_remote__63__.mdwn
@@ -0,0 +1 @@
+I recently created WebDAV remote to my annex which I assumed to be just for testing purposes at the time, so I gave it a strange and long-wided name. It turned out, though, that the remote is stable, works fine, and that I'm going to keep it. Now, I would like to give it a snappier and more descriptive name than the one I originally chose. Is that possible somehow?
diff --git a/doc/forum/How_to_rename_a_remote__63__/comment_1_a9bfbd82f7bb47661f0d9e0e0d904332._comment b/doc/forum/How_to_rename_a_remote__63__/comment_1_a9bfbd82f7bb47661f0d9e0e0d904332._comment
new file mode 100644
index 0000000000..31f44c8f6f
--- /dev/null
+++ b/doc/forum/How_to_rename_a_remote__63__/comment_1_a9bfbd82f7bb47661f0d9e0e0d904332._comment
@@ -0,0 +1,28 @@
+[[!comment format=mdwn
+ username="http://joeyh.name/"
+ ip="4.154.3.194"
+ subject="comment 1"
+ date="2013-01-18T18:20:43Z"
+ content="""
+You can rename a git remote with just \"git remote rename  \". However, for a special remote, git-remote will fail to rename it unless you first add a dummy remote..fetch value in .git/config. You can remove it later.
+
+So, for example:
+
+
    +git config remote.$uglyname.fetch dummy
+git remote rename $uglyname $nicename
+git config remote.$nicename.fetch \"\"
+
    
+
+(The webapp handles doing this when you edit a remote's name, BTW.)
+
+---
+
+There's actually another place the name of a special remote is recorded, in `remote.log`. That name is only used when you use `git annex initremote` though. It actually is possible to change it, by using `initremote` to change the vale of the name field. Example:
+
+
    +git annex initremote $uglyname name=$newname
+
    
+
+Note that for some types of special remotes, this will require you to re-specify some other configuration. For example, with a directory special remote, it wanted me to include a directory= parameter.
+"""]]
diff --git a/doc/forum/How_to_resolve_a_interrupted_merge_in_direct_mode__63__.mdwn b/doc/forum/How_to_resolve_a_interrupted_merge_in_direct_mode__63__.mdwn
new file mode 100644
index 0000000000..a21d6d6d29
--- /dev/null
+++ b/doc/forum/How_to_resolve_a_interrupted_merge_in_direct_mode__63__.mdwn
@@ -0,0 +1,65 @@
+I have just done a sync in a direct mode repo:
+
+    git-annex sync sshremote
+
+It has quite a few conflicts (both sides adding the same file with same content). But somehow the merge won't conclude. Here's some log:
+
+    $ git-annex sync sshremote
+    commit  ok
+    pull sshremote 
+    warning: no common commits
+    remote: Counting objects: 32162, done.
+    remote: Compressing objects: 100% (31192/31192), done.
+    remote: Total 32162 (delta 3), reused 32162 (delta 3)
+    Receiving objects: 100% (32162/32162), 3.39 MiB | 1.61 MiB/s, done.
+    Resolving deltas: 100% (3/3), done.
+    From sshremote:/repo
+     * [new branch]      annex/direct/master -> sshremote/annex/direct/master
+     * [new branch]      git-annex  -> sshremote/git-annex
+     * [new branch]      master     -> sshremote/master
+     * [new branch]      synced/master -> sshremote/synced/master
+
+    Auto-merging file1
+    CONFLICT (add/add): Merge conflict in file1
+    Auto-merging file2
+    CONFLICT (add/add): Merge conflict in file2
+    Auto-merging file3
+    CONFLICT (add/add): Merge conflict in file3
+    Auto-merging file4
+    CONFLICT (add/add): Merge conflict in file4
+    Automatic merge failed; fix conflicts and then commit the result.
+    file1: needs merge
+    file2: needs merge
+    file3: needs merge
+    file4: needs merge
+    ^C
+
+I waited for a few minutes, and checked that both cpu and disk activity is near zero on both server. So I interrupted it, thinking it might be some locking issue. 
+
+But now I cannot sync with the remote anymore because the merge is still in process. How should I resolve the conflicts? I could do it manually if I cannot use annex's auto merge.
+
+    $ git-annex sync sshremote
+    commit  ok
+    pull sshremote 
+
+    fatal: You have not concluded your merge (MERGE_HEAD exists).
+    Please, commit your changes before you can merge.
+
+    fatal: You have not concluded your merge (MERGE_HEAD exists).
+    Please, commit your changes before you can merge.
+    failed
+
+    $ git merge --abort
+    fatal: This operation must be run in a work tree
+
+Update: this is running on OSX with:
+
+    git-annex version: 5.20131221-g00d1673
+    build flags: Assistant Webapp Pairing Testsuite S3 WebDAV FsEvents XMPP DNS Feeds Quvi TDFA CryptoHash
+    key/value backends: SHA256E SHA1E SHA512E SHA224E SHA384E SKEIN256E SKEIN512E SHA256 SHA1 SHA512 SHA224 SHA384 SKEIN256 SKEIN512 WORM URL
+    remote types: git gcrypt S3 bup directory rsync web webdav glacier hook
+    local repository version: 3
+    default repository version: 3
+    supported repository versions: 3 5
+    upgrade supported from repository versions: 0 1 2 4
+
diff --git a/doc/forum/How_to_resolve_a_interrupted_merge_in_direct_mode__63__/comment_1_8e2a14842b44844f90c80b862a1b3a6d._comment b/doc/forum/How_to_resolve_a_interrupted_merge_in_direct_mode__63__/comment_1_8e2a14842b44844f90c80b862a1b3a6d._comment
new file mode 100644
index 0000000000..3b1383376b
--- /dev/null
+++ b/doc/forum/How_to_resolve_a_interrupted_merge_in_direct_mode__63__/comment_1_8e2a14842b44844f90c80b862a1b3a6d._comment
@@ -0,0 +1,10 @@
+[[!comment format=mdwn
+ username="http://joeyh.name/"
+ ip="209.250.56.227"
+ subject="comment 1"
+ date="2014-01-01T20:31:21Z"
+ content="""
+I think you can fix at least the immediate problem by deleting the .git/MERGE_HEAD file.
+
+
+"""]]
diff --git a/doc/forum/How_to_resolve_a_interrupted_merge_in_direct_mode__63__/comment_2_031ab6b5a2765ed9e2b185b24a8cbd78._comment b/doc/forum/How_to_resolve_a_interrupted_merge_in_direct_mode__63__/comment_2_031ab6b5a2765ed9e2b185b24a8cbd78._comment
new file mode 100644
index 0000000000..d8c4ded898
--- /dev/null
+++ b/doc/forum/How_to_resolve_a_interrupted_merge_in_direct_mode__63__/comment_2_031ab6b5a2765ed9e2b185b24a8cbd78._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="https://www.google.com/accounts/o8/id?id=AItOawkYmMFDdf3GJ9Oba6NCVkzGc4JyB9WavMs"
+ nickname="Xinruo"
+ subject="Thanks"
+ date="2014-01-02T00:41:27Z"
+ content="""
+That did the trick! 
+"""]]
diff --git a/doc/forum/How_to_resolve_a_interrupted_merge_in_direct_mode__63__/comment_3_93f20519483837c59a75821621e22dee._comment b/doc/forum/How_to_resolve_a_interrupted_merge_in_direct_mode__63__/comment_3_93f20519483837c59a75821621e22dee._comment
new file mode 100644
index 0000000000..7cfc2a7f21
--- /dev/null
+++ b/doc/forum/How_to_resolve_a_interrupted_merge_in_direct_mode__63__/comment_3_93f20519483837c59a75821621e22dee._comment
@@ -0,0 +1,10 @@
+[[!comment format=mdwn
+ username="http://joeyh.name/"
+ ip="209.250.56.227"
+ subject="comment 3"
+ date="2014-01-02T02:05:05Z"
+ content="""
+AFAICS, git-merge was still running when you ctrl-c'd it. So this is coming before git-annex processes the merge to update the work tree.
+
+I don't know why it would take so long to run, especially if it was not using CPU or disk. You might try to reproduce the problem, passing --debug to git-annex, and perhaps strace the git-merge process to see what it's doing.
+"""]]
diff --git a/doc/forum/How_to_restore_symlinks.mdwn b/doc/forum/How_to_restore_symlinks.mdwn
new file mode 100644
index 0000000000..30fb07001b
--- /dev/null
+++ b/doc/forum/How_to_restore_symlinks.mdwn
@@ -0,0 +1 @@
+Somehow the symlinks have vanished from one directory of my repository. How can I restore them?
diff --git a/doc/forum/How_to_restore_symlinks/comment_1_c67e752cf7d5431096fab4b3304790a7._comment b/doc/forum/How_to_restore_symlinks/comment_1_c67e752cf7d5431096fab4b3304790a7._comment
new file mode 100644
index 0000000000..d9e14dd164
--- /dev/null
+++ b/doc/forum/How_to_restore_symlinks/comment_1_c67e752cf7d5431096fab4b3304790a7._comment
@@ -0,0 +1,11 @@
+[[!comment format=mdwn
+ username="https://www.google.com/accounts/o8/id?id=AItOawnldTTAP8PAifJUmqhRar6RAWNWlRcencw"
+ nickname="Marco"
+ subject="git annex fix"
+ date="2012-11-24T08:51:19Z"
+ content="""
+You can try
+    git annex fix yourDirectory
+
+Hope that helps.
+"""]]
diff --git a/doc/forum/How_to_restore_symlinks/comment_2_f9ec6096595e2c149c48924e3b54542f._comment b/doc/forum/How_to_restore_symlinks/comment_2_f9ec6096595e2c149c48924e3b54542f._comment
new file mode 100644
index 0000000000..689f321261
--- /dev/null
+++ b/doc/forum/How_to_restore_symlinks/comment_2_f9ec6096595e2c149c48924e3b54542f._comment
@@ -0,0 +1,14 @@
+[[!comment format=mdwn
+ username="http://joeyh.name/"
+ ip="4.154.6.49"
+ subject="comment 2"
+ date="2012-11-25T18:35:56Z"
+ content="""
+The key thing to realize is that the symlinks used by git-annex are checked into git like any other file would be. So you can use the entire git toolkit to manage them. 
+
+For example, you could run `git status` to see if git shows them as recently deleted, and then use `git checkout $file` to restore the deleted files. 
+
+Or perhaps the deletion has been committed to git, and then you'd use `git log --stat` to find the commit that deleted your files, and `git revert` could be used to undo it.
+
+(`git annex fix` is not related to this and won't help.)
+"""]]
diff --git a/doc/forum/How_to_restore_symlinks/comment_3_4ff80729787a2a4e2baf05dd1db37da3._comment b/doc/forum/How_to_restore_symlinks/comment_3_4ff80729787a2a4e2baf05dd1db37da3._comment
new file mode 100644
index 0000000000..ce0be74b00
--- /dev/null
+++ b/doc/forum/How_to_restore_symlinks/comment_3_4ff80729787a2a4e2baf05dd1db37da3._comment
@@ -0,0 +1,12 @@
+[[!comment format=mdwn
+ username="jbee"
+ ip="178.25.68.205"
+ subject="Thanks"
+ date="2012-11-28T11:52:24Z"
+ content="""
+> deletion has been committed to git, and then you'd use git log --stat to find the commit that deleted your files, and git revert could be used to undo it.
+
+That did the trick.
+
+Thanks Joey, for the answer and the magnificence that is git-annex.
+"""]]
diff --git a/doc/forum/How_to_retroactively_annex_a_file_already_in_a_git_repo.mdwn b/doc/forum/How_to_retroactively_annex_a_file_already_in_a_git_repo.mdwn
new file mode 100644
index 0000000000..981a53ba0b
--- /dev/null
+++ b/doc/forum/How_to_retroactively_annex_a_file_already_in_a_git_repo.mdwn
@@ -0,0 +1,5 @@
+I worked out how to retroactively annex a large file that had been checked into a git repo some time ago.  I thought this might be useful for others, so I am posting it here.
+
+> This is a great tip, so I've moved it to
+> [[tips|tips/How_to_retroactively_annex_a_file_already_in_a_git_repo]].
+> --[[Joey]] 
diff --git a/doc/forum/How_to_revert_metadata_changes.mdwn b/doc/forum/How_to_revert_metadata_changes.mdwn
new file mode 100644
index 0000000000..429640effb
--- /dev/null
+++ b/doc/forum/How_to_revert_metadata_changes.mdwn
@@ -0,0 +1,25 @@
+Hi,
+
+I accidentally changed some metadata tags for all annexed files in a directory (by omitting the filename to the git annex metadata call). How can I revert that change?
+
+I tried a git reset --hard on the git-annex branch, but after checking out master and adding a new file the changes magically reappeared.
+
+git-annex version: 5.20141125
+
+To reproduce do:
+
+    git init; git annex init
+    echo Hi > a.txt; git annex add a.txt
+    git commit -m 'Initial add'
+    git annex metadata --set name=bla
+    git annex metadata --set name=blu
+    git checkout git-annex
+    git reset --hard HEAD^
+    git checkout master
+    git annex metadata a.txt # name=bla
+    echo Huch > b.txt; git annex add b.txt
+    git annex metadata a.txt # name=blu again
+
+Cheers,
+
+Markus
diff --git a/doc/forum/How_to_revert_metadata_changes/comment_1_de40f1a3f364ca6d2f9bb489f4d64cb3._comment b/doc/forum/How_to_revert_metadata_changes/comment_1_de40f1a3f364ca6d2f9bb489f4d64cb3._comment
new file mode 100644
index 0000000000..da47be164d
--- /dev/null
+++ b/doc/forum/How_to_revert_metadata_changes/comment_1_de40f1a3f364ca6d2f9bb489f4d64cb3._comment
@@ -0,0 +1,39 @@
+[[!comment format=mdwn
+ username="joey"
+ subject="""comment 1"""
+ date="2016-01-01T19:23:29Z"
+ content="""
+`git-annex metadata` used to affect all files under the current directory
+if you left off the files to update. That changed a while ago to prevent
+that fairly common mistake, pity your git-annex is too old to have that
+fixed.
+
+You're on sort of the right track with the git-annex branch, but there are some
+complications:
+
+1. .git/annex/index is the index file for the git-annex branch. When you
+   change the git-annex branch manually, you also need to delete that
+   file. This will cause git-annex to re-generate its index from the new
+   contents of the branch.
+2. Resetting the git-annex branch to a previous rev won't "stick"
+   if the problematic rev has already been pushed to other repositories.
+   git-annex will automatically re-merge the git-annex branches from other
+   repos at some point and get the problem rev back. Instead you'll need to
+   make a commit to the git-annex branch that undoes the changes made by the
+   problem rev. (eg `git revert`)
+3. The contents of the git-annex branch are merged by essentially
+   taking the union of the local and remote contents.
+   So if some other clone of the repository still has the
+   problematic data in its git-annex branch, when git-annex union
+   merges that in, the problem data will come back again, even if you've
+   made a local commit that reverts its addition.
+
+So, you can easily deal with #1; it wouldn't be hard to do #2, but #3
+will be a real sticking point, if the bad data has been pushed out to any
+other repos.
+
+In that case, the only thing to do is to record new information on the
+git-annex branch, so git-annex knows that even if this metadata was set
+before, it's not set any longer. The way to do that is just use
+`git annex metadata --set name-=bla`
+"""]]
diff --git a/doc/forum/How_to_set_backend_based_on_file_size_in_.gitattributes.mdwn b/doc/forum/How_to_set_backend_based_on_file_size_in_.gitattributes.mdwn
new file mode 100644
index 0000000000..75a4ca7c06
--- /dev/null
+++ b/doc/forum/How_to_set_backend_based_on_file_size_in_.gitattributes.mdwn
@@ -0,0 +1,16 @@
+Hi there,
+
+In my repos, I want files larger than 500MB to use the WORM backend. So when I'm adding large files, I do it in 2 passes. I first run this:
+    
+    git annex add --largerthan=500MB --backend=WORM .
+
+to add only files larger than 500MB. The remaining files that are smaller than 500MB I then add using default backend:
+
+    git annex add
+
+Is it possible to set annex.backend in .gitattributes so that adding files larger than 500MB automatically use the WORM backend? Can I use expressions such as largerthan or smallerthan? From the example [here](https://git-annex.branchable.com/backends/), it seems that it can only be set based on file type. 
+
+
+Regards,
+
+Eric
diff --git a/doc/forum/How_to_set_backend_based_on_file_size_in_.gitattributes/comment_1_c8ac6b924d4446d4a1a12fb34532fd69._comment b/doc/forum/How_to_set_backend_based_on_file_size_in_.gitattributes/comment_1_c8ac6b924d4446d4a1a12fb34532fd69._comment
new file mode 100644
index 0000000000..878d589ad3
--- /dev/null
+++ b/doc/forum/How_to_set_backend_based_on_file_size_in_.gitattributes/comment_1_c8ac6b924d4446d4a1a12fb34532fd69._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="joey"
+ subject="""comment 1"""
+ date="2018-01-15T16:15:39Z"
+ content="""
+annex.backend only matches on file name, so can't be used for this. I think
+your current two pass approach is the only way to do it currently.
+"""]]
diff --git a/doc/forum/How_to_set_up_two_assistants_with_one_shared_transfer_repository__63__.mdwn b/doc/forum/How_to_set_up_two_assistants_with_one_shared_transfer_repository__63__.mdwn
new file mode 100644
index 0000000000..8ee5eeea56
--- /dev/null
+++ b/doc/forum/How_to_set_up_two_assistants_with_one_shared_transfer_repository__63__.mdwn
@@ -0,0 +1,15 @@
+This seems like something that should be completely obvious, but I've been trying to get it working for a while without success.
+
+I have a few machines (at least 3) at different locations. I want them all to be synced with a remote server that I have. I've installed git-annex on the server, and setting up the assistant on one of the machines, I can create the remote repository, and all seems to work (it spits out info about syncing... though I don't have any easy way of checking if it's working, as all I have is the one client). 
+
+It says in the webapp that setting up a remote git repository in transfer mode will make it easy to have other clients. But I'm wondering how to set that up. I tried just adding the same server with the same path on another machine, but it doesn't seem to be syncing, and I imagine that it is trying to create a fresh repo there instead of syncing with an existing one.
+
+So, how do I set this up? I don't mind adding git remotes, ssh keys, etc, manually, but I haven't been able to figure out what I should be doing! My end goal is to have one annex that is synced between the many computers (ala Dropbox, as the assistant is supposed to be).
+
+Note that the configuration of the test machines:
+client 1 - debian, git-annex built from current cabal
+server - debian, git-annex from apt
+client2 - mac osx, git-annex built from current cabal
+
+Thanks,
+Daniel
diff --git a/doc/forum/How_to_set_up_two_assistants_with_one_shared_transfer_repository__63__/comment_1_bedaf308cfc70b9e751914a400ebcbc2._comment b/doc/forum/How_to_set_up_two_assistants_with_one_shared_transfer_repository__63__/comment_1_bedaf308cfc70b9e751914a400ebcbc2._comment
new file mode 100644
index 0000000000..b6a884aa54
--- /dev/null
+++ b/doc/forum/How_to_set_up_two_assistants_with_one_shared_transfer_repository__63__/comment_1_bedaf308cfc70b9e751914a400ebcbc2._comment
@@ -0,0 +1,10 @@
+[[!comment format=mdwn
+ username="http://joeyh.name/"
+ ip="4.152.108.27"
+ subject="comment 1"
+ date="2012-11-08T21:29:26Z"
+ content="""
+Just adding the same server with the same path is the right thing to do, it will use the existing repo.
+
+Probably the problem you're having is that one client doesn't know when the other client has sent data to the server. I've recently been adding XMPP (Jabber) support to deal with that. That is not yet in a released version of git-annex, but it is available in git master.
+"""]]
diff --git a/doc/forum/How_to_set_up_two_assistants_with_one_shared_transfer_repository__63__/comment_2_d665b1514253c8aa487ebf8b2728e3b1._comment b/doc/forum/How_to_set_up_two_assistants_with_one_shared_transfer_repository__63__/comment_2_d665b1514253c8aa487ebf8b2728e3b1._comment
new file mode 100644
index 0000000000..9cb54839f1
--- /dev/null
+++ b/doc/forum/How_to_set_up_two_assistants_with_one_shared_transfer_repository__63__/comment_2_d665b1514253c8aa487ebf8b2728e3b1._comment
@@ -0,0 +1,10 @@
+[[!comment format=mdwn
+ username="https://www.google.com/accounts/o8/id?id=AItOawlsfh8P7pN2OBr24ztT_7AyYQLCcVqqa-4"
+ nickname="Daniel"
+ subject="comment 2"
+ date="2012-11-09T17:49:32Z"
+ content="""
+The master is segfaulting on one of my machines (the mac), but XMMP sounds good in general.
+
+In the meantime - does git annex sync do what the assistant does? Or is there a command to tell the assistant to check for remote changes? (or a way to get it to auto-poll)
+"""]]
diff --git a/doc/forum/How_to_set_up_two_assistants_with_one_shared_transfer_repository__63__/comment_3_aef42387a3673ab6710fb23e878d7e17._comment b/doc/forum/How_to_set_up_two_assistants_with_one_shared_transfer_repository__63__/comment_3_aef42387a3673ab6710fb23e878d7e17._comment
new file mode 100644
index 0000000000..2249c9964b
--- /dev/null
+++ b/doc/forum/How_to_set_up_two_assistants_with_one_shared_transfer_repository__63__/comment_3_aef42387a3673ab6710fb23e878d7e17._comment
@@ -0,0 +1,10 @@
+[[!comment format=mdwn
+ username="http://joeyh.name/"
+ ip="4.252.11.120"
+ subject="comment 3"
+ date="2012-11-13T17:57:49Z"
+ content="""
+I recently dealt with a segfault caused by a bug in the haskell gnutls library. It could be that's what you were seeing. It's fixed in gnutls (>= 0.1.4)
+
+Yes, you can run `git annex sync` by hand. It does the same syncing of the git repository that the assistant does. I don't think making the assistant poll is a good idea.
+"""]]
diff --git a/doc/forum/How_to_set_up_two_assistants_with_one_shared_transfer_repository__63__/comment_4_bfbcc041db472f4808979e6b3d7c4be2._comment b/doc/forum/How_to_set_up_two_assistants_with_one_shared_transfer_repository__63__/comment_4_bfbcc041db472f4808979e6b3d7c4be2._comment
new file mode 100644
index 0000000000..7bd4870a3c
--- /dev/null
+++ b/doc/forum/How_to_set_up_two_assistants_with_one_shared_transfer_repository__63__/comment_4_bfbcc041db472f4808979e6b3d7c4be2._comment
@@ -0,0 +1,10 @@
+[[!comment format=mdwn
+ username="https://www.google.com/accounts/o8/id?id=AItOawniCRkhl_W87gOK5eElfsef3FoUsUFpAr4"
+ nickname="Alexandre"
+ subject="Simplifying this kind of setup"
+ date="2012-12-10T14:33:08Z"
+ content="""
+Maybe it is possible to avoid the XMPP account setup and transferring via XMPP, maybe getting notifications through the SSH connection is possible.
+
+I'm thinking about a \"git-annex-shell server\" unix socket to which clients would connect using the SSH connection and get update notifications from other clients.
+"""]]
diff --git a/doc/forum/How_to_shrink_transfer_repo__63__.mdwn b/doc/forum/How_to_shrink_transfer_repo__63__.mdwn
new file mode 100644
index 0000000000..8e368087f1
--- /dev/null
+++ b/doc/forum/How_to_shrink_transfer_repo__63__.mdwn
@@ -0,0 +1,27 @@
+Hello,
+
+I have two repositories (Asaru and Horus) that are both ```group=client``` and ```wanted=standard```. The other one, Astarte is ```group=transfer``` and ```wanted=standard```. Pretty standard I think.
+
+```
+repository mode: direct
+trusted repositories: 0
+semitrusted repositories: 5
+        00000000-0000-0000-0000-000000000001 -- web
+        00000000-0000-0000-0000-000000000002 -- bittorrent
+        58001764-966d-4076-ae99-4ef6de25df39 -- Asaru [here]
+        8165bdf1-907e-4bbe-9c35-22fbf6f8cb00 -- Astarte [astarte]
+        cca0c3c8-593a-4395-936c-1093f0f762e8 -- Horus
+untrusted repositories: 0
+```
+
+I always sync on the two client repos like that ```git annex add . && git annex sync --content```. The transfer repo is growing larger and larger. ```git annex dropunused N``` says, that it ```could only verify the existence of 0 out of 1 necessary copies```.
+
+What is the best way to clean up the transfer repo?
+
+1. Make the two client repos trusted? The three repos have been created manually, not through the assistant. Is that what the assistant does, too?
+2. Try to get the two client repos into touch with each other and try to use ```dropunsed --from=astarte```?
+
+What is the recommended way for that?
+
+Thanks,
+Florian
diff --git a/doc/forum/How_to_shrink_transfer_repo__63__/comment_1_6b110311f4c147dc315c4f610cf56afa._comment b/doc/forum/How_to_shrink_transfer_repo__63__/comment_1_6b110311f4c147dc315c4f610cf56afa._comment
new file mode 100644
index 0000000000..5c1c71475d
--- /dev/null
+++ b/doc/forum/How_to_shrink_transfer_repo__63__/comment_1_6b110311f4c147dc315c4f610cf56afa._comment
@@ -0,0 +1,19 @@
+[[!comment format=mdwn
+ username="Horus"
+ subject="comment 1"
+ date="2016-03-03T18:17:35Z"
+ content="""
+Ok, I try to do 2) but it still fails:
+
+```
+dropunused 265 (from astarte...) (unsafe) 
+  Could only verify the existence of 0 out of 1 necessary copies
+
+  Rather than dropping this file, try using: git annex move
+
+  (Use --force to override this check, or adjust numcopies.)
+failed
+```
+
+all repositories are reachable directly.
+"""]]
diff --git a/doc/forum/How_to_shrink_transfer_repo__63__/comment_2_8e596cd606935d82bd6604f5c9c500a2._comment b/doc/forum/How_to_shrink_transfer_repo__63__/comment_2_8e596cd606935d82bd6604f5c9c500a2._comment
new file mode 100644
index 0000000000..edccafb1ec
--- /dev/null
+++ b/doc/forum/How_to_shrink_transfer_repo__63__/comment_2_8e596cd606935d82bd6604f5c9c500a2._comment
@@ -0,0 +1,21 @@
+[[!comment format=mdwn
+ username="joey"
+ subject="""comment 2"""
+ date="2016-03-07T17:52:21Z"
+ content="""
+Normally, files should be dropped from the transfer repository once they
+have reached all known client repositories. The drop should be done by
+the client repositories, so the transfer repo doesn't need access to the
+client repos to verify that they have a copy.
+
+But your problem is with *unused* files that are hanging around in
+the transfer repo. This can happen if a file reaches the transfer repo and
+then gets deleted from a client, and so the other clients never download it. 
+
+To clean out those files, run: `git annex dropunused --from astarte --force`
+
+You need to "use the force" because the unused file is only present in the
+transfer repo. If you want to get rid of its content for good, that's fine.
+A safer option is to move the unused file to the local repo: `git annex
+move --unused --from astarte`
+"""]]
diff --git a/doc/forum/How_to_shrink_transfer_repo__63__/comment_3_c536a946b717e9bdcb883b58cd0336ae._comment b/doc/forum/How_to_shrink_transfer_repo__63__/comment_3_c536a946b717e9bdcb883b58cd0336ae._comment
new file mode 100644
index 0000000000..4d0cee3318
--- /dev/null
+++ b/doc/forum/How_to_shrink_transfer_repo__63__/comment_3_c536a946b717e9bdcb883b58cd0336ae._comment
@@ -0,0 +1,7 @@
+[[!comment format=mdwn
+ username="Horus"
+ subject="comment 3"
+ date="2016-03-07T20:11:03Z"
+ content="""
+Thanks, that worked!
+"""]]
diff --git a/doc/forum/How_to_solve__problem_with_diverging_repositories_handled_by_the_assistant__63__.mdwn b/doc/forum/How_to_solve__problem_with_diverging_repositories_handled_by_the_assistant__63__.mdwn
new file mode 100644
index 0000000000..91d7a886b6
--- /dev/null
+++ b/doc/forum/How_to_solve__problem_with_diverging_repositories_handled_by_the_assistant__63__.mdwn
@@ -0,0 +1,40 @@
+I have one Annex meant to be kept in sync across two computers by the assistant. These computers can't connect directly, so I use a transfer repository (at rsync.net) and xmpp. Recently the two repositories have started to diverge: files added here do not appear there and vice versa, files modified here are not updated there and vice versa. It may be related to this bug: [[bugs/Jabber__47__xmpp_not_supported_on_Debian_Wheezy_backport/]] since one of the two installed versions of git-annex is 5.20140117~bpo70+1, but I do not know whether the divergence started the day I upgraded to this version.
+
+I tried this:
+
+    git-annex fsck --from rsync.net_annex
+
+on both machines. No error is reported, but files are missing here and there and files differ here and there.
+
+I stopped the assistant on both machines and tried to sync with:
+
+    git-annex sync 
+
+and got:
+
+    commit  
+    ok
+    pull 'XXX'
+    fatal: Unable to find remote helper for 'xmpp'
+    failed
+    push 'XXX'
+        Pushing to XXX failed 
+        (non-fast-forward problems can be solved by setting receive.denyNonFastforwards to false in the remote's git config)
+    failed
+    git-annex: sync: 2 failed
+
+where 'XXX' stands for my jabber account. I then tried:
+
+    git-annex sync rsync.net_annex
+
+but got:
+
+    git-annex: cannot sync special remotes: rsync.net_annex
+
+    
+
+This situation is unfortunate because the user is not made aware of anything failing. The assistant is running, the webapp shows files being uploaded to the transfer repository, but the two repositories are actually diverging.
+
+How can I synchronize these two repositories?
+
+Thanks,
diff --git a/doc/forum/How_to_solve__problem_with_diverging_repositories_handled_by_the_assistant__63__/comment_1_1c913395f076ee203caaab057da8afbe._comment b/doc/forum/How_to_solve__problem_with_diverging_repositories_handled_by_the_assistant__63__/comment_1_1c913395f076ee203caaab057da8afbe._comment
new file mode 100644
index 0000000000..7bf4585054
--- /dev/null
+++ b/doc/forum/How_to_solve__problem_with_diverging_repositories_handled_by_the_assistant__63__/comment_1_1c913395f076ee203caaab057da8afbe._comment
@@ -0,0 +1,13 @@
+[[!comment format=mdwn
+ username="https://www.google.com/accounts/o8/id?id=AItOawnNqLKszWk9EoD4CDCqNXJRIklKFBCN1Ao"
+ nickname="maurizio"
+ subject="downgrading is not an easy option"
+ date="2014-01-30T05:44:48Z"
+ content="""
+In order to check whether this is related to  [[https://git-annex.branchable.com/bugs/Jabber__47__xmpp_not_supported_on_Debian_Wheezy_backport/]] I tried to downgrade git-annex to the previous version that was available on wheezy-backports (git-annex_4.20131106~bpo70+1_i386.deb) but then I get:
+
+    git-annex: Repository version 5 is not supported. Upgrade git-annex.
+    failed
+
+
+"""]]
diff --git a/doc/forum/How_to_solve__problem_with_diverging_repositories_handled_by_the_assistant__63__/comment_2_081793c52bf15c74a7f48a67c49ff818._comment b/doc/forum/How_to_solve__problem_with_diverging_repositories_handled_by_the_assistant__63__/comment_2_081793c52bf15c74a7f48a67c49ff818._comment
new file mode 100644
index 0000000000..8d4d49a778
--- /dev/null
+++ b/doc/forum/How_to_solve__problem_with_diverging_repositories_handled_by_the_assistant__63__/comment_2_081793c52bf15c74a7f48a67c49ff818._comment
@@ -0,0 +1,12 @@
+[[!comment format=mdwn
+ username="https://www.google.com/accounts/o8/id?id=AItOawnNqLKszWk9EoD4CDCqNXJRIklKFBCN1Ao"
+ nickname="maurizio"
+ subject="clean up"
+ date="2014-01-31T19:55:30Z"
+ content="""
+ok, now that git-annex has been updated to 5.20140117~bpo70+2 things start flowing again, so it was actually caused by [[http://git-annex.branchable.com/bugs/Jabber__47__xmpp_not_supported_on_Debian_Wheezy_backport/]]. Now there is a certain number of files called 'filename.variant.06b8' for instance. That is a bit messy.
+
+I do not know wheter/how it would be possible to inform the user that something is going wrong, but certainely it would be better to avoid failing completely silently.
+
+It would be useful if the webapp could give more precise information about the files the assistant is manipulating. For instance: \"now uploading file 'bla', updating version last uploaded from 'remote' on 'date'\".
+"""]]
diff --git a/doc/forum/How_to_solve__problem_with_diverging_repositories_handled_by_the_assistant__63__/comment_3_f8e0376beb486cf8ce52384ff511ecf2._comment b/doc/forum/How_to_solve__problem_with_diverging_repositories_handled_by_the_assistant__63__/comment_3_f8e0376beb486cf8ce52384ff511ecf2._comment
new file mode 100644
index 0000000000..99bfbba0a8
--- /dev/null
+++ b/doc/forum/How_to_solve__problem_with_diverging_repositories_handled_by_the_assistant__63__/comment_3_f8e0376beb486cf8ce52384ff511ecf2._comment
@@ -0,0 +1,11 @@
+[[!comment format=mdwn
+ username="http://joeyh.name/"
+ ip="71.80.94.56"
+ subject="comment 3"
+ date="2014-02-07T19:07:11Z"
+ content="""
+Normally the webapp will tell you whenever it fails to sync with a remote. However, in this case, it had XMPP support compiled right out of it, so had no idea that remote even existed, and never tried to sync with it, so had no failure to report.
+
+
+The \".variant\" files are because you must have made files with the same names in the 2 repositories while they were disconnected.
+"""]]
diff --git a/doc/forum/How_to_sync_data_down_from_a___39__full_backup__39____63__.mdwn b/doc/forum/How_to_sync_data_down_from_a___39__full_backup__39____63__.mdwn
new file mode 100644
index 0000000000..52a277a6e5
--- /dev/null
+++ b/doc/forum/How_to_sync_data_down_from_a___39__full_backup__39____63__.mdwn
@@ -0,0 +1,4 @@
+
+I'm trying to find a way of syncing data across multiple platforms, including an android device. The latter I want to sync the data so that it's on the devices SD filesystem and independent of network. As far as I can tell git-annex's client is the only one that can do this, all other self hosted storage clients I've found are only network viewers (owncloud, seafile), which is no good to me. I want to sync my ABC sheet music collection onto it, viewable independent of network access, updating changes automatically. Other parts of the data set (i.e. large collection of cannon cr2 raws), I want synced between my laptop and desktop.
+
+I've set up git annex with a central git repo and another client directory which syncs fine to the server (set as a 'full backup'). Now I want to download this whole archive onto the android device. I've set up the client on the device and it works and will upload to the server, however I can't see any way of getting it to download. Surely git-annex has the capability to pull data?
diff --git a/doc/forum/How_to_sync_data_down_from_a___39__full_backup__39____63__/comment_1_c4e972c8635f2fa460cd6621f3d993c6._comment b/doc/forum/How_to_sync_data_down_from_a___39__full_backup__39____63__/comment_1_c4e972c8635f2fa460cd6621f3d993c6._comment
new file mode 100644
index 0000000000..d01ab86310
--- /dev/null
+++ b/doc/forum/How_to_sync_data_down_from_a___39__full_backup__39____63__/comment_1_c4e972c8635f2fa460cd6621f3d993c6._comment
@@ -0,0 +1,23 @@
+[[!comment format=mdwn
+ username="joey"
+ subject="""comment 1"""
+ date="2015-02-09T20:37:49Z"
+ content="""
+Yes, it absolutely does. On Android, git-annex defaults to making
+its repository be a "source" repository, which means it doesn't try
+to get a copy of every file. You can edit the repository in the webapp,
+and select "client" and it will go ahead and download every file it can
+get. See [[preferred_content/standard_groups]].
+
+It sounds like you might want to have the android device only
+get copies of certian files, and ignore others (the raws).
+Doing that is a little more complicated; it involves writing a
+[[preferred_content]] expression. You can do this by first exiting
+the webapp, and the at the git-annex app's console:
+
+	cd /sdcard/annex
+	git annex wanted . "include=sheetmusic/* and exclude=raws/*"
+
+Then when you start the webapp back up, it will get the sheet music 
+and not the raws. Adjust paths in that to suite..
+"""]]
diff --git a/doc/forum/How_to_sync_data_down_from_a___39__full_backup__39____63__/comment_2_e06abc9a2f7cf055523f3d35e4eed9bc._comment b/doc/forum/How_to_sync_data_down_from_a___39__full_backup__39____63__/comment_2_e06abc9a2f7cf055523f3d35e4eed9bc._comment
new file mode 100644
index 0000000000..c8ba6b1f0e
--- /dev/null
+++ b/doc/forum/How_to_sync_data_down_from_a___39__full_backup__39____63__/comment_2_e06abc9a2f7cf055523f3d35e4eed9bc._comment
@@ -0,0 +1,11 @@
+[[!comment format=mdwn
+ username="robert_hickman"
+ subject="comment 2"
+ date="2015-02-10T01:40:47Z"
+ content="""
+Thank you Joey, The setting should be added to the local, with the remote set to 'full backup'?
+
+I've tried updating that setting to client but it still doesn’t seem to be working. Looking at the log, there is a line 'fatal: not a valid object name refs/remotes/adesktop_annex/annex/direct/master'. Also it appears to have attempted to sync but quit on an invalid FAT file name. Should this stop syncing altogether? Or can it just ignore/report these problem files.
+
+Also thanks for saying how to do part-downloads, I was wandering if it could do that.
+"""]]
diff --git a/doc/forum/How_to_sync_data_down_from_a___39__full_backup__39____63__/comment_3_81d128be0fa60475062e795f34d920f7._comment b/doc/forum/How_to_sync_data_down_from_a___39__full_backup__39____63__/comment_3_81d128be0fa60475062e795f34d920f7._comment
new file mode 100644
index 0000000000..086f28e738
--- /dev/null
+++ b/doc/forum/How_to_sync_data_down_from_a___39__full_backup__39____63__/comment_3_81d128be0fa60475062e795f34d920f7._comment
@@ -0,0 +1,9 @@
+[[!comment format=mdwn
+ username="robert_hickman"
+ subject="comment 3"
+ date="2015-02-10T01:49:39Z"
+ content="""
+I'm also seeing a duplicate parent error at the end of the log.
+
+How can I clear the log? It took quite a while to get this running so there is quite a bit of irreverent stuff in it.
+"""]]
diff --git a/doc/forum/How_to_sync_data_down_from_a___39__full_backup__39____63__/comment_4_43615e11f1c6d08f5850458d540f5bac._comment b/doc/forum/How_to_sync_data_down_from_a___39__full_backup__39____63__/comment_4_43615e11f1c6d08f5850458d540f5bac._comment
new file mode 100644
index 0000000000..59afac274c
--- /dev/null
+++ b/doc/forum/How_to_sync_data_down_from_a___39__full_backup__39____63__/comment_4_43615e11f1c6d08f5850458d540f5bac._comment
@@ -0,0 +1,9 @@
+[[!comment format=mdwn
+ username="robert_hickman"
+ subject="comment 4"
+ date="2015-02-10T01:55:06Z"
+ content="""
+Last log content:
+
+(started...) Already up-to-date. error: duplicate parent c2aa8406693d1a3cdec09fb3be8002865cbc8d36 ignored warning: no threads support, ignoring --threads [2015-02-10 01:44:51 GMT] Pusher: Syncing with adesktop_annex warning: no threads support, ignoring --threads fatal: Unable to create '/home/annex/annex/gc.pid.lock': File exists. If no other git process is currently running, this probably means a git process crashed in this repository earlier. Make sure no other git process is running and remove the file manually to continue. To ssh://annex@git-annex-a-desktop-annex_22_annex/~/annex/ e99e004..5b09ad8 git-annex -> synced/git-annex c2aa840..f64d9e3 annex/direct/master -> synced/master error: Ref refs/heads/synced/git-annex is at 5b09ad85ff8d0ba013e4e135e386818eec0cc810 but expected e99e0040176275beb7553f89774f1b14697fccd8 error: Ref refs/heads/synced/master is at e3482dc007f109d0773cf50fd0de2ca2c213cc24 but expected c2aa8406693d1a3cdec09fb3be8002865cbc8d36 remote: error: failed to lock refs/heads/synced/git-annex remote: error: failed to lock refs/heads/synced/master To ssh://annex@git-annex-a-desktop-annex_22_annex/~/annex/ ! [remote rejected] git-annex -> synced/git-annex (failed to lock) ! [remote rejected] annex/direct/master -> synced/master (failed to lock) error: failed to push some refs to 'ssh://annex@git-annex-a-desktop-annex_22_annex/~/annex/' Already up-to-date. error: duplicate parent e3482dc007f109d0773cf50fd0de2ca2c213cc24 ignored warning: no threads support, ignoring --threads To ssh://annex@git-annex-a-desktop-annex_22_annex/~/annex/ e3482dc..e97d44a annex/direct/master -> synced/master
+"""]]
diff --git a/doc/forum/How_to_sync_without_enumerating_files__63__.mdwn b/doc/forum/How_to_sync_without_enumerating_files__63__.mdwn
new file mode 100644
index 0000000000..b67da98d8a
--- /dev/null
+++ b/doc/forum/How_to_sync_without_enumerating_files__63__.mdwn
@@ -0,0 +1,7 @@
+I regularly get this:
+
+    It took 148.17 seconds to enumerate untracked files. 'status -uno'
+    may speed it up, but you have to be careful not to forget to add
+    new files yourself (see 'git help status').
+
+How can I run git-annex sync without enumerating untracked files? Can I add the -uno switch somehow?
diff --git a/doc/forum/How_to_sync_without_enumerating_files__63__/comment_1_ad78c67ae1bb4dc8ed800779c0298158._comment b/doc/forum/How_to_sync_without_enumerating_files__63__/comment_1_ad78c67ae1bb4dc8ed800779c0298158._comment
new file mode 100644
index 0000000000..f79b3592fc
--- /dev/null
+++ b/doc/forum/How_to_sync_without_enumerating_files__63__/comment_1_ad78c67ae1bb4dc8ed800779c0298158._comment
@@ -0,0 +1,16 @@
+[[!comment format=mdwn
+ username="joey"
+ subject="""comment 1"""
+ date="2017-10-31T17:15:33Z"
+ content="""
+Well, `git annex sync` does not itself run `git status`. 
+
+Try running git-annex with the --debug option, it will say what git
+commands it runs, and that should help find out which git command is
+complaining.
+
+My guess is it's `git commit`, which  is probably getting the status
+unncessarily for a commit message template, despite a commit message
+being specified. If so, `git annex sync --no-commit` would avoid
+that.
+"""]]
diff --git a/doc/forum/How_to_work_with_transfer_repos_manually__63__.mdwn b/doc/forum/How_to_work_with_transfer_repos_manually__63__.mdwn
new file mode 100644
index 0000000000..8ec42dba42
--- /dev/null
+++ b/doc/forum/How_to_work_with_transfer_repos_manually__63__.mdwn
@@ -0,0 +1,18 @@
+Hello,
+
+I have 3 repos, desktop, external and server. desktop and server are sometimes connected, sometimes they should sync using the server. I want to do it manually without the assistent, since I love to learn it that way before I let the assistent do the work.
+
+client and desktop are "wanted standard" and "group client". server is "group transfer".
+
+desktop and server have each other and server in their remotes. server has no remotes.
+
+Is this setup fine that way?
+
+How to use it with the transfer repo?
+
+"git annex sync && git annex copy --to server --auto" after changing files?
+"git annex sync && git annex copy --from server --auto" to update?
+
+Will the on the server automatically be dropped? Or do the server needs to have a active role, i.e. called via ssh?
+
+Thanks!
diff --git a/doc/forum/How_to_work_with_transfer_repos_manually__63__/comment_1_3dec369405e6b6a4a6e5121546c03712._comment b/doc/forum/How_to_work_with_transfer_repos_manually__63__/comment_1_3dec369405e6b6a4a6e5121546c03712._comment
new file mode 100644
index 0000000000..c6d3f9dae3
--- /dev/null
+++ b/doc/forum/How_to_work_with_transfer_repos_manually__63__/comment_1_3dec369405e6b6a4a6e5121546c03712._comment
@@ -0,0 +1,11 @@
+[[!comment format=mdwn
+ username="joey"
+ subject="""comment 1"""
+ date="2014-10-20T15:29:41Z"
+ content="""
+If you use `git annex sync --content`, it will do a full sync,
+including uploading any necessary files to the transfer repo,
+downloading any files that are on the transfer repo, and
+dropping files from the transfer repo once they've been
+transferred to the client repos.
+"""]]
diff --git a/doc/forum/How_to_work_with_transfer_repos_manually__63__/comment_1_b8f3c09b470d99578a4a17064498dd39._comment b/doc/forum/How_to_work_with_transfer_repos_manually__63__/comment_1_b8f3c09b470d99578a4a17064498dd39._comment
new file mode 100644
index 0000000000..51a473ed1f
--- /dev/null
+++ b/doc/forum/How_to_work_with_transfer_repos_manually__63__/comment_1_b8f3c09b470d99578a4a17064498dd39._comment
@@ -0,0 +1,12 @@
+[[!comment format=mdwn
+ username="https://www.google.com/accounts/o8/id?id=AItOawk9SYh6N-JUMkYkW4aOk55zC3Vr9KonDV4"
+ nickname="Florian"
+ subject="comment 1"
+ date="2014-10-20T10:51:16Z"
+ content="""
+Ok, git annex copy --to server --auto and git annex get on the receiver did the trick.
+
+But how can I drop files from the transfer server that both clients have? I tried git annex drop . or git annex drop --auto . but it just did nothing.
+
+Thx!
+"""]]
diff --git a/doc/forum/How_to_work_with_transfer_repos_manually__63__/comment_3_be2c594bc1d162cfb1acc3a01fc284f2._comment b/doc/forum/How_to_work_with_transfer_repos_manually__63__/comment_3_be2c594bc1d162cfb1acc3a01fc284f2._comment
new file mode 100644
index 0000000000..393cd44973
--- /dev/null
+++ b/doc/forum/How_to_work_with_transfer_repos_manually__63__/comment_3_be2c594bc1d162cfb1acc3a01fc284f2._comment
@@ -0,0 +1,7 @@
+[[!comment format=mdwn
+ username="joey"
+ subject="""comment 3"""
+ date="2014-10-20T18:53:31Z"
+ content="""
+git annex drop --auto --from server
+"""]]
diff --git a/doc/forum/Howto_remove_a_repository__63__.mdwn b/doc/forum/Howto_remove_a_repository__63__.mdwn
new file mode 100644
index 0000000000..4eb4e68699
--- /dev/null
+++ b/doc/forum/Howto_remove_a_repository__63__.mdwn
@@ -0,0 +1,4 @@
+Hello,
+
+I have added a few repositories for testing purpose and would like to remove them.
+Is it possible to remove a repository through assistant? 
diff --git a/doc/forum/Howto_remove_a_repository__63__/comment_1_b55fa4e92bb457ecaa5ca8f5cee7be1d._comment b/doc/forum/Howto_remove_a_repository__63__/comment_1_b55fa4e92bb457ecaa5ca8f5cee7be1d._comment
new file mode 100644
index 0000000000..642da6575f
--- /dev/null
+++ b/doc/forum/Howto_remove_a_repository__63__/comment_1_b55fa4e92bb457ecaa5ca8f5cee7be1d._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="http://joeyh.name/"
+ nickname="joey"
+ subject="comment 1"
+ date="2013-03-17T19:05:27Z"
+ content="""
+There's no UI for it in the assistant. [[Relevant bug report|bugs/The_webapp_doesn't_allow_deleting_repositories]]
+"""]]
diff --git a/doc/forum/Howto_remove_unused_files.mdwn b/doc/forum/Howto_remove_unused_files.mdwn
new file mode 100644
index 0000000000..db770457b0
--- /dev/null
+++ b/doc/forum/Howto_remove_unused_files.mdwn
@@ -0,0 +1,31 @@
+Hello.
+
+My case: I have somehow managed to get my repo, with quite much stuff inside it messed up.
+
+There is single directory 'cities' containing quite many files (>10k) that i would rather see gone (and keep the tar.bz version of it ..). I tried doing 
+
+    git drop --force
+
+that works fine, but the files are still there after sync. 
+
+git annex unused says just 'ok' so i guess it thinks they are still used somewhere. I tried to look where, but i ended up just doing git annex drop -f my_remote_here test_file_name for each of my remotes. This doesnt help. 
+How can i get rid of these files? Doing git annex fsck shows that 
+
+
+So, try to search what is the key:
+
+     % ls -lah cities/diskcache/config.cfg
+     lrwxrwxrwx 1 XX XXX 190 Nov 29 05:52 cities/diskcache/config.cfg -> ../../../../.git/annex/objects/qm/M6/SHA256-s32--4f5ce34d1b0b8d854a315530b2fdcbfa9c3067636a2aa5433a04402db4151dce/SHA256-s32--4f5ce34d1b0b8d854a315530b2fdcbfa9c3067636a2aa5433a04402db4151dce
+     sundberg@sundberg-MS-7680 ~/git-repository/ubuntu.iso/Archive/Maps
+     % git log --stat --all -SSHA256-s32--4f5ce34d1b0b8d854a315530b2fdcbfa9c3067636a2aa5433a04402db4151dce/SHA256-s32--4f5ce34d1b0b8d854a315530b2fdcbfa9c3067636a2aa5433a04402db4151dce
+     commit 51a57a023774ff80408210828f298f5c42a7e0be
+     Author: XXXX
+     Date:   Sun Dec 9 13:42:40 2012 +0200
+        git-annex automatic sync
+     Archive/Maps/cities/diskcache/config.cfg |    1 +
+     1 file changed, 1 insertion(+)
+
+So how can i deduce what is the remote i should try to clean up ?
+
+
+Thanks!
diff --git a/doc/forum/Howto_remove_unused_files/comment_1_f2a7948268ce3cb3967a9fdd8ccc570a._comment b/doc/forum/Howto_remove_unused_files/comment_1_f2a7948268ce3cb3967a9fdd8ccc570a._comment
new file mode 100644
index 0000000000..b2bf64397b
--- /dev/null
+++ b/doc/forum/Howto_remove_unused_files/comment_1_f2a7948268ce3cb3967a9fdd8ccc570a._comment
@@ -0,0 +1,16 @@
+[[!comment format=mdwn
+ username="https://www.google.com/accounts/o8/id?id=AItOawkbv1oKTKhbBp0Ljh_WGU7BFSWWxBr7D3U"
+ nickname="Pauli"
+ subject="Just to add"
+ date="2013-04-02T19:01:09Z"
+ content="""
+
+When i do 'git annex get .' on my usb-remote i get these kind of prints:
+
+    get Archive/Maps/cities/diskcache/voices/slovenian_f/slovenian_female/g5ext_000 (not available) 
+    No other repository is known to contain the file.
+    failed
+
+
+No repository contains the file? Well why cannot i then get it removed from the listing .. 
+"""]]
diff --git a/doc/forum/Howto_remove_unused_files/comment_2_9b4d198c2d8a52adef3d166a8196fc0d._comment b/doc/forum/Howto_remove_unused_files/comment_2_9b4d198c2d8a52adef3d166a8196fc0d._comment
new file mode 100644
index 0000000000..ef4219e2fb
--- /dev/null
+++ b/doc/forum/Howto_remove_unused_files/comment_2_9b4d198c2d8a52adef3d166a8196fc0d._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="https://www.google.com/accounts/o8/id?id=AItOawkbv1oKTKhbBp0Ljh_WGU7BFSWWxBr7D3U"
+ nickname="Pauli"
+ subject="comment 2"
+ date="2013-04-02T19:10:18Z"
+ content="""
+Just to continue my monolog: doing 'git rm -r cities' followed with git annex sync did the trick. Why didn't i find this anywhere ..
+"""]]
diff --git a/doc/forum/Howto_remove_unused_files/comment_3_441d10901d5c055ac3ed2a6cb61c075c._comment b/doc/forum/Howto_remove_unused_files/comment_3_441d10901d5c055ac3ed2a6cb61c075c._comment
new file mode 100644
index 0000000000..ffb941ac88
--- /dev/null
+++ b/doc/forum/Howto_remove_unused_files/comment_3_441d10901d5c055ac3ed2a6cb61c075c._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="http://joeyh.name/"
+ nickname="joey"
+ subject="comment 3"
+ date="2013-04-02T21:21:21Z"
+ content="""
+git-annex adds large file capabilities to git, but it leaves all the regular git commands there for you to use. So the answer to \"how do I do <something I'd normally do with a git command>\" is always \"run the git command\". You only need to use git-annex to add large files to git, and to move their content around between repositories. Hope that helps clear that up.
+"""]]
diff --git a/doc/forum/Huge_differences_in_sync_times_in_different_directories_of_same_machines.mdwn b/doc/forum/Huge_differences_in_sync_times_in_different_directories_of_same_machines.mdwn
new file mode 100644
index 0000000000..c936fbd65d
--- /dev/null
+++ b/doc/forum/Huge_differences_in_sync_times_in_different_directories_of_same_machines.mdwn
@@ -0,0 +1,127 @@
+I am confused by the differences in the time between updates of different
+repositories in the same machines and of the same repository in different
+machines. I am using the assistant (in all machines, 
+'git annex assitant --autostart').
+
+
+
+For instance, I have three machines synced as
+
+A <-> B <-> C
+
+and I have two directories being synced. The configuration (except for
+directory names) is the same and looks like
+
+
+```
+[annex]
+        uuid = something
+        version = 5
+        direct = true
+        autoupgrade = true
+        autocommit = true
+        startupscan = true
+[remote "machine_repo"]
+        url = ssh://xxxxxx/~/repo/
+        fetch = +refs/heads/*:refs/remotes/machine_repo/*
+        annex-uuid = xxxxxxxxxxxxx
+
+```
+
+
+I modify or add a file in one of the directories in A, and it quickly
+(less than 30 seconds) appears in B and shortly after that in C.
+
+In another directory, however, more than 30 minutes can go by without C
+ever changing (even if B gets the change almost immediately). Looking at
+the logs I cannot understand what is happening. This is the case of a
+file, ```A_sentinel_file_A ```, created at 01:46.
+
+In B (where it appears almost immediately)  I can see
+
+
+```
+[2015-03-02 01:46:17 CET] Pusher: Syncing with 123.456.789.101_somedir 
+To ssh://ramon@git-annex-Machine_C-ramon_22_annex.2Done.2Dway.2F/~/some-dir/
+   794df9f..ae8c3f0  git-annex -> synced/git-annex
+Automatic merge went well; stopped before committing as requested
+[2015-03-02 01:47:03 CET] Committer: Committing changes to git
+[2015-03-02 01:47:03 CET] Pusher: Syncing with 123.456.789.101_somedir 
+
+A_sentinel_file_A
+             18 100%    0.00kB/s    0:00:00 (xfr#1, to-chk=0/1)
+[2015-03-02 01:47:04 CET] Transferrer: Uploaded A_sentinel_file_A
+To ssh://ramon@git-annex-Machine_C-ramon_22_annex.2Done.2Dway.2F/~/some-dir/
+   ae8c3f0..b043335  git-annex -> synced/git-annex
+   75269bc..b61c7a5  annex/direct/master -> synced/master
+[2015-03-02 01:47:04 CET] Committer: Committing changes to git
+[2015-03-02 01:47:06 CET] Pusher: Syncing with 123.456.789.101_somedir 
+To ssh://ramon@git-annex-Machine_C-ramon_22_annex.2Done.2Dway.2F/~/some-dir/
+   b043335..fc9279c  git-annex -> synced/git-annex
+```
+
+
+And in C (the machine with IP 123.456.789.101, above)  I can see
+
+
+```
+[2015-03-02 01:46:13 CET] RemoteControl: Syncing with Machine_B_somedir 
+From ssh://git-annex-Machine_B-ramon_22_annex.2Done.2Dway/~/some-dir
+   593d908..794df9f  git-annex  -> Machine_B_somedir/git-annex
+   593d908..235149d  synced/git-annex -> Machine_B_somedir/synced/git-annex
+   a1596b3..75269bc  synced/master -> Machine_B_somedir/synced/master
+[2015-03-02 01:46:17 CET] RemoteControl: Syncing with Machine_B_somedir 
+From ssh://git-annex-Machine_B-ramon_22_annex.2Done.2Dway/~/some-dir
+   a1596b3..75269bc  annex/direct/master -> Machine_B_somedir/annex/direct/master
+   794df9f..ae8c3f0  git-annex  -> Machine_B_somedir/git-annex
+   a1596b3..75269bc  master     -> Machine_B_somedir/master
+[2015-03-02 01:46:27 CET] RemoteControl: Syncing with Machine_B_somedir 
+From ssh://git-annex-Machine_B-ramon_22_annex.2Done.2Dway/~/some-dir
+   ae8c3f0..632caa4  git-annex  -> Machine_B_somedir/git-annex
+   235149d..632caa4  synced/git-annex -> Machine_B_somedir/synced/git-annex
+[2015-03-02 01:47:02 CET] RemoteControl: Syncing with Machine_B_somedir 
+From ssh://git-annex-Machine_B-ramon_22_annex.2Done.2Dway/~/some-dir
+   632caa4..56ac968  synced/git-annex -> Machine_B_somedir/synced/git-annex
+   75269bc..b61c7a5  synced/master -> Machine_B_somedir/synced/master
+[2015-03-02 01:47:02 CET] RemoteControl: Syncing with Machine_B_somedir 
+From ssh://git-annex-Machine_B-ramon_22_annex.2Done.2Dway/~/some-dir
+   632caa4..b043335  git-annex  -> Machine_B_somedir/git-annex
+[2015-03-02 01:47:07 CET] RemoteControl: Syncing with Machine_B_somedir 
+From ssh://git-annex-Machine_B-ramon_22_annex.2Done.2Dway/~/some-dir
+   75269bc..b61c7a5  annex/direct/master -> Machine_B_somedir/annex/direct/master
+   b043335..fc9279c  git-annex  -> Machine_B_somedir/git-annex
+   75269bc..b61c7a5  master     -> Machine_B_somedir/master
+[2015-03-02 01:47:15 CET] RemoteControl: Syncing with Machine_B_somedir 
+From ssh://git-annex-Machine_B-ramon_22_annex.2Done.2Dway/~/some-dir
+   fc9279c..a4642a8  git-annex  -> Machine_B_somedir/git-annex
+   56ac968..a4642a8  synced/git-annex -> Machine_B_somedir/synced/git-annex
+(merging synced/git-annex Machine_B_somedir/git-annex into git-annex...)
+(recording state in git...)
+[2015-03-02 02:17:23 CET] RemoteControl: Syncing with Machine_B_somedir 
+From ssh://git-annex-Machine_B-ramon_22_annex.2Done.2Dway/~/some-dir
+   a4642a8..bfaba72  git-annex  -> Machine_B_somedir/git-annex
+   a4642a8..bfaba72  synced/git-annex -> Machine_B_somedir/synced/git-annex
+   b61c7a5..29ff91e  synced/master -> Machine_B_somedir/synced/master   
+   
+```
+
+Why did Machine C did not show the updated file at 01:47, which is when, I
+think, machine B pushes it there? Or at 02:17 when both machines again
+seem to talk to each other? Regardless, at 02:27 the file still was not
+there. Yes, of course, if I issue a
+
+```
+git annex sync --content
+```
+
+then the file is shown in Machine C. But I should not need to do that.
+
+
+
+In contrast, as I said, there is another directory, with identical
+configuration, where these things do not happen and changes show
+immediately (both directories, for now, only have tiny test files).
+
+What am I doing wrong? This is too big a difference to make sense to me.
+
+
diff --git a/doc/forum/Huge_differences_in_sync_times_in_different_directories_of_same_machines/comment_1_445766c0f646e9cb5b45b863ad87e85a._comment b/doc/forum/Huge_differences_in_sync_times_in_different_directories_of_same_machines/comment_1_445766c0f646e9cb5b45b863ad87e85a._comment
new file mode 100644
index 0000000000..5c1ddf127b
--- /dev/null
+++ b/doc/forum/Huge_differences_in_sync_times_in_different_directories_of_same_machines/comment_1_445766c0f646e9cb5b45b863ad87e85a._comment
@@ -0,0 +1,136 @@
+[[!comment format=mdwn
+ username="https://www.google.com/accounts/o8/id?id=AItOawnbPKLjBONawBd74MKJZo05juCqdsP1jAU"
+ nickname="Ramon"
+ subject="comment 1"
+ date="2015-03-02T01:47:32Z"
+ content="""
+Ooops, the formatting got all messed up. Here it is again:
+
+I am confused by the differences in the time between updates of different
+repositories in the same machines and of the same repository in different
+machines. I am using the assistant (in all machines, 
+'git annex assitant --autostart').
+
+
+
+For instance, I have three machines synced as
+
+A <-> B <-> C
+
+and I have two directories being synced. The configuration (except for
+directory names) is the same and looks like
+
+
+
+    [annex]
+            uuid = something
+            version = 5
+            direct = true
+            autoupgrade = true
+            autocommit = true
+            startupscan = true
+    [remote \"machine_repo\"]
+            url = ssh://xxxxxx/~/repo/
+            fetch = +refs/heads/*:refs/remotes/machine_repo/*
+            annex-uuid = xxxxxxxxxxxxx
+    
+
+
+
+I modify or add a file in one of the directories in A, and it quickly
+(less than 30 seconds) appears in B and shortly after that in C.
+
+In another directory, however, more than 30 minutes can go by without C
+ever changing (even if B gets the change almost immediately). Looking at
+the logs I cannot understand what is happening. This is the case of a
+file, ```A_sentinel_file_A ```, created at 01:46.
+
+In B (where it appears almost immediately)  I can see
+
+
+    [2015-03-02 01:46:17 CET] Pusher: Syncing with 123.456.789.101_somedir 
+    To ssh://ramon@git-annex-Machine_C-ramon_22_annex.2Done.2Dway.2F/~/some-dir/
+       794df9f..ae8c3f0  git-annex -> synced/git-annex
+    Automatic merge went well; stopped before committing as requested
+    [2015-03-02 01:47:03 CET] Committer: Committing changes to git
+    [2015-03-02 01:47:03 CET] Pusher: Syncing with 123.456.789.101_somedir 
+    
+    A_sentinel_file_A
+                 18 100%    0.00kB/s    0:00:00 (xfr#1, to-chk=0/1)
+    [2015-03-02 01:47:04 CET] Transferrer: Uploaded A_sentinel_file_A
+    To ssh://ramon@git-annex-Machine_C-ramon_22_annex.2Done.2Dway.2F/~/some-dir/
+       ae8c3f0..b043335  git-annex -> synced/git-annex
+       75269bc..b61c7a5  annex/direct/master -> synced/master
+    [2015-03-02 01:47:04 CET] Committer: Committing changes to git
+    [2015-03-02 01:47:06 CET] Pusher: Syncing with 123.456.789.101_somedir 
+    To ssh://ramon@git-annex-Machine_C-ramon_22_annex.2Done.2Dway.2F/~/some-dir/
+       b043335..fc9279c  git-annex -> synced/git-annex
+
+
+
+And in C (the machine with IP 123.456.789.101, above)  I can see
+
+
+
+    [2015-03-02 01:46:13 CET] RemoteControl: Syncing with Machine_B_somedir 
+    From ssh://git-annex-Machine_B-ramon_22_annex.2Done.2Dway/~/some-dir
+       593d908..794df9f  git-annex  -> Machine_B_somedir/git-annex
+       593d908..235149d  synced/git-annex -> Machine_B_somedir/synced/git-annex
+       a1596b3..75269bc  synced/master -> Machine_B_somedir/synced/master
+    [2015-03-02 01:46:17 CET] RemoteControl: Syncing with Machine_B_somedir 
+    From ssh://git-annex-Machine_B-ramon_22_annex.2Done.2Dway/~/some-dir
+       a1596b3..75269bc  annex/direct/master -> Machine_B_somedir/annex/direct/master
+       794df9f..ae8c3f0  git-annex  -> Machine_B_somedir/git-annex
+       a1596b3..75269bc  master     -> Machine_B_somedir/master
+    [2015-03-02 01:46:27 CET] RemoteControl: Syncing with Machine_B_somedir 
+    From ssh://git-annex-Machine_B-ramon_22_annex.2Done.2Dway/~/some-dir
+       ae8c3f0..632caa4  git-annex  -> Machine_B_somedir/git-annex
+       235149d..632caa4  synced/git-annex -> Machine_B_somedir/synced/git-annex
+    [2015-03-02 01:47:02 CET] RemoteControl: Syncing with Machine_B_somedir 
+    From ssh://git-annex-Machine_B-ramon_22_annex.2Done.2Dway/~/some-dir
+       632caa4..56ac968  synced/git-annex -> Machine_B_somedir/synced/git-annex
+       75269bc..b61c7a5  synced/master -> Machine_B_somedir/synced/master
+    [2015-03-02 01:47:02 CET] RemoteControl: Syncing with Machine_B_somedir 
+    From ssh://git-annex-Machine_B-ramon_22_annex.2Done.2Dway/~/some-dir
+       632caa4..b043335  git-annex  -> Machine_B_somedir/git-annex
+    [2015-03-02 01:47:07 CET] RemoteControl: Syncing with Machine_B_somedir 
+    From ssh://git-annex-Machine_B-ramon_22_annex.2Done.2Dway/~/some-dir
+       75269bc..b61c7a5  annex/direct/master -> Machine_B_somedir/annex/direct/master
+       b043335..fc9279c  git-annex  -> Machine_B_somedir/git-annex
+       75269bc..b61c7a5  master     -> Machine_B_somedir/master
+    [2015-03-02 01:47:15 CET] RemoteControl: Syncing with Machine_B_somedir 
+    From ssh://git-annex-Machine_B-ramon_22_annex.2Done.2Dway/~/some-dir
+       fc9279c..a4642a8  git-annex  -> Machine_B_somedir/git-annex
+       56ac968..a4642a8  synced/git-annex -> Machine_B_somedir/synced/git-annex
+    (merging synced/git-annex Machine_B_somedir/git-annex into git-annex...)
+    (recording state in git...)
+    [2015-03-02 02:17:23 CET] RemoteControl: Syncing with Machine_B_somedir 
+    From ssh://git-annex-Machine_B-ramon_22_annex.2Done.2Dway/~/some-dir
+       a4642a8..bfaba72  git-annex  -> Machine_B_somedir/git-annex
+       a4642a8..bfaba72  synced/git-annex -> Machine_B_somedir/synced/git-annex
+       b61c7a5..29ff91e  synced/master -> Machine_B_somedir/synced/master   
+       
+
+
+Why did Machine C did not show the updated file at 01:47, which is when, I
+think, machine B pushes it there? Or at 02:17 when both machines again
+seem to talk to each other? Regardless, at 02:27 the file still was not
+there. Yes, of course, if I issue a
+
+```
+    git annex sync --content
+```
+
+then the file is shown in Machine C. But I should not need to do that.
+
+
+
+In contrast, as I said, there is another directory, with identical
+configuration, where these things do not happen and changes show
+immediately (both directories, for now, only have tiny test files).
+
+
+
+
+
+"""]]
diff --git a/doc/forum/Huge_differences_in_sync_times_in_different_directories_of_same_machines/comment_2_b3953f30fb817ee3753f888a871de51f._comment b/doc/forum/Huge_differences_in_sync_times_in_different_directories_of_same_machines/comment_2_b3953f30fb817ee3753f888a871de51f._comment
new file mode 100644
index 0000000000..58d019be98
--- /dev/null
+++ b/doc/forum/Huge_differences_in_sync_times_in_different_directories_of_same_machines/comment_2_b3953f30fb817ee3753f888a871de51f._comment
@@ -0,0 +1,12 @@
+[[!comment format=mdwn
+ username="joey"
+ subject="""comment 2"""
+ date="2015-07-06T20:07:08Z"
+ content="""
+I think there might be some bug where the assistant does not notice that
+the synced/master branch has been updated, and doesn't immediately merge it
+into the local master branch. This seems to be borne out by your log.
+
+[[bugs/assistant_sometimes_does_not_merge_changes_from_remote]] might be
+related.
+"""]]
diff --git a/doc/forum/Ignoring_folders__63__.mdwn b/doc/forum/Ignoring_folders__63__.mdwn
new file mode 100644
index 0000000000..55c7884a2d
--- /dev/null
+++ b/doc/forum/Ignoring_folders__63__.mdwn
@@ -0,0 +1 @@
+What is the correct way to ignore subfolders or files?  Just use .gitignore?  Or is there a git-annex specific way?  For example, partial download files.
diff --git a/doc/forum/Ignoring_folders__63__/comment_1_5fbf4d1ae75434118876470b04e2fd72._comment b/doc/forum/Ignoring_folders__63__/comment_1_5fbf4d1ae75434118876470b04e2fd72._comment
new file mode 100644
index 0000000000..f699f19d24
--- /dev/null
+++ b/doc/forum/Ignoring_folders__63__/comment_1_5fbf4d1ae75434118876470b04e2fd72._comment
@@ -0,0 +1,7 @@
+[[!comment format=mdwn
+ username="joey"
+ subject="""comment 1"""
+ date="2016-05-10T16:17:47Z"
+ content="""
+Use .gitignore, there's no need of a git-annex specific way.
+"""]]
diff --git a/doc/forum/Ignoring_folders__63__/comment_2_1adafdf1fddf8e1256e51b0f56d3a89e._comment b/doc/forum/Ignoring_folders__63__/comment_2_1adafdf1fddf8e1256e51b0f56d3a89e._comment
new file mode 100644
index 0000000000..e0095b077e
--- /dev/null
+++ b/doc/forum/Ignoring_folders__63__/comment_2_1adafdf1fddf8e1256e51b0f56d3a89e._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="browneej@96fb43ae83ff25c968e65939805b60f3e42f1e06"
+ nickname="browneej"
+ subject="comment 2"
+ date="2016-05-10T22:44:46Z"
+ content="""
+Awesome, thanks.
+"""]]
diff --git a/doc/forum/Import_availability_info___40__logs__47__refs__47__remotes__41___from_another_repo__63__.mdwn b/doc/forum/Import_availability_info___40__logs__47__refs__47__remotes__41___from_another_repo__63__.mdwn
new file mode 100644
index 0000000000..33a8c085f1
--- /dev/null
+++ b/doc/forum/Import_availability_info___40__logs__47__refs__47__remotes__41___from_another_repo__63__.mdwn
@@ -0,0 +1,3 @@
+Is it possible to transfer list of remotes and their file availability information (which files are found on which remotes) from another repository?
+
+My use case would be to combine several repositories that have no copies of the actual files, but which do know of remotes that have them, so that the new composite repository would also know where to get each file.
diff --git a/doc/forum/Import_availability_info___40__logs__47__refs__47__remotes__41___from_another_repo__63__/comment_1_32586d9e43dc0c5d0a9e94671586f763._comment b/doc/forum/Import_availability_info___40__logs__47__refs__47__remotes__41___from_another_repo__63__/comment_1_32586d9e43dc0c5d0a9e94671586f763._comment
new file mode 100644
index 0000000000..23ab14189c
--- /dev/null
+++ b/doc/forum/Import_availability_info___40__logs__47__refs__47__remotes__41___from_another_repo__63__/comment_1_32586d9e43dc0c5d0a9e94671586f763._comment
@@ -0,0 +1,9 @@
+[[!comment format=mdwn
+ username="joey"
+ subject="""comment 1"""
+ date="2017-06-06T19:48:20Z"
+ content="""
+Sure, you can just add one repository as a remote of the other, 
+and `git fetch` and then git-annex will automatically combine
+all the information about remotes and locations of files.
+"""]]
diff --git a/doc/forum/Import_options.mdwn b/doc/forum/Import_options.mdwn
new file mode 100644
index 0000000000..543d1a4ec8
--- /dev/null
+++ b/doc/forum/Import_options.mdwn
@@ -0,0 +1,14 @@
+Thank you for adding import options to handle duplicates. Very handy when consolidating data from various sources.
+
+Can deletion of the source files be decoupled from annex duplication/deduplication options? For example, I would like to import source files without deleting them and at the same time do not import duplicates.
+
+Better yet, since deletion of source files is potentially dangerous, a delete option could be required for deletion to be performed. Example:
+
+git annex import --deduplicate --delete_all_source_files
+git annex import --deduplicate --delete_source_duplicates
+
+Also, it would be great to have import "status" option which goes over files to be imported and logs their status ( to be imported, duplicate etc. ) without actually performing any changes. It would be great for testing and trial runs.
+
+I hope the above make sense. It would make import feature more flexible.
+
+Cheers, 
diff --git a/doc/forum/Import_options/comment_1_118a5f978090a3909299876a01c0adec._comment b/doc/forum/Import_options/comment_1_118a5f978090a3909299876a01c0adec._comment
new file mode 100644
index 0000000000..3475a0d734
--- /dev/null
+++ b/doc/forum/Import_options/comment_1_118a5f978090a3909299876a01c0adec._comment
@@ -0,0 +1,21 @@
+[[!comment format=mdwn
+ username="https://www.google.com/accounts/o8/id?id=AItOawkeJKC5Sy0stmcTWyePOLEVv0G-x1yaT_w"
+ nickname="Josef"
+ subject="wishlist"
+ date="2013-09-26T11:11:19Z"
+ content="""
+Posted the above yesterday before realizing that it should probably go to wishlist requests. I am sorry about that.
+
+Basically it is a request to extend import options and perhaps make the options easier to use/understand.
+
+Suggested Import Options:
+
+- source directory,
+- destination directory,
+- deduplicate in annex ( yes, no )
+- delete source files ( yes, no )
+- trial run ( screen output only )
+
+Many thanks for a great product!
+
+"""]]
diff --git a/doc/forum/Import_options/comment_2_21da91f08cb6b28ae3e79ade033db516._comment b/doc/forum/Import_options/comment_2_21da91f08cb6b28ae3e79ade033db516._comment
new file mode 100644
index 0000000000..a3e2596248
--- /dev/null
+++ b/doc/forum/Import_options/comment_2_21da91f08cb6b28ae3e79ade033db516._comment
@@ -0,0 +1,17 @@
+[[!comment format=mdwn
+ username="https://www.google.com/accounts/o8/id?id=AItOawkeJKC5Sy0stmcTWyePOLEVv0G-x1yaT_w"
+ nickname="Josef"
+ subject="Additional Comments"
+ date="2013-09-30T21:33:31Z"
+ content="""
+Imported several thousand files to annex and would like to add the following comments:
+
+- it would be great to have an option to exclude hidden dot files from import,
+
+- empty directories should be deleted when files located in the directories are deleted,
+
+- \"git annex add\" seems to process directories and files alphabetically, unfortunately import processes files in a different order, which makes it hard to predict which files are deleted when deduplicating,
+
+Cheers,
+
+"""]]
diff --git a/doc/forum/Improving_the_git-annex_forum.mdwn b/doc/forum/Improving_the_git-annex_forum.mdwn
new file mode 100644
index 0000000000..30d7a29737
--- /dev/null
+++ b/doc/forum/Improving_the_git-annex_forum.mdwn
@@ -0,0 +1,23 @@
+Dear git-annex community,
+
+git-annex has a great documentation, consisting of the wiki articles and the converted man page. On the other hand, in my opinion there is room for improvement when it comes to the forum. It's lacking quite some features you expect from a modern forum, e.g.
+
+- Slick UI
+- User friendly way to (un)subscribe to threads
+- Bump threads on new replies (that may be a matter of taste)
+- Reply via mail
+- Ability to pin posts (at least I've never seen it)
+- Weird markdown formatting, e.g. marking code by indenting by 4 characters, which is hard to achieve for many lines at once inside a browsers text field (Joey already told me pre does job, too)
+- no private messages
+and more...
+
+Therefore I think, that a better forum would significantly improve git-annex's user experience!
+
+I'm hosting a Discourse forum for some time now and think it's quite a decent, user- and admin friendly software. I would be happy to host a forum for git-annex on my own server, e.g. on a subdomain of git-annex.branchable.com.
+
+I had exchanged some mails with Joey about that, he asked me to take this discussion to the forum...
+
+What are your opinions on that matter?
+
+Best,
+Florian
diff --git a/doc/forum/Improving_the_git-annex_forum/comment_1_63c52c977689a5da34fec65391c25fd6._comment b/doc/forum/Improving_the_git-annex_forum/comment_1_63c52c977689a5da34fec65391c25fd6._comment
new file mode 100644
index 0000000000..0231987f63
--- /dev/null
+++ b/doc/forum/Improving_the_git-annex_forum/comment_1_63c52c977689a5da34fec65391c25fd6._comment
@@ -0,0 +1,34 @@
+[[!comment format=mdwn
+ username="joey"
+ subject="""comment 1"""
+ date="2018-01-15T17:33:12Z"
+ content="""
+Thanks, Florian for raising this topic.
+
+Granting the missing features you list (and perhaps adding that people seem
+to sometimes find it annoying to log in successfully here), I wonder how
+much those features are limiting use of this forum, if at all. I suppose
+one way to find out would be to move to another forum and see if traffic
+increases (or SNR or engagement improves).
+
+For myself, having the forum integrated in the git-annex repo is quite
+helpful to me, since I automatically track it along with all the other
+changes to the website, and am sure to see every single post here,
+and it meets my offline/low bandwidth needs. Also I sometimes promote
+forum posts to bug reports or make a commit following up to a forum
+post with a requested feature or fix.
+
+I suspect that the git-annex community is kind of split between people who
+are more comfortable in web forums, and people who are used to mailing
+lists, who are currently being left entirely out in the cold (and probably
+using the irc channel more). It could be that mailman3 with its mix of
+forum and mailing list could better unify the two groups. Or perhaps
+there's no point in trying to do that, and a simple mailing list would be a
+bigger win than changing the forum?
+
+I'm ok with however git-annex users choose to get together and
+talk about it, including ways that I can't/won't use myself (including
+non-free stuff like Slack..). To that end, if people want to get a
+git-annex community going somewhere, be it another forum, or reddit, or
+whatever, I'm fine with the website directing users to it.
+"""]]
diff --git a/doc/forum/Improving_the_git-annex_forum/comment_2_7b4b8d78148f41df332d8f9125ca9842._comment b/doc/forum/Improving_the_git-annex_forum/comment_2_7b4b8d78148f41df332d8f9125ca9842._comment
new file mode 100644
index 0000000000..6557c3be96
--- /dev/null
+++ b/doc/forum/Improving_the_git-annex_forum/comment_2_7b4b8d78148f41df332d8f9125ca9842._comment
@@ -0,0 +1,15 @@
+[[!comment format=mdwn
+ username="Horus"
+ avatar="http://cdn.libravatar.org/avatar/8f0ee08b98ea5bba76c3fe112c08851c"
+ subject="comment 2"
+ date="2018-02-28T09:09:20Z"
+ content="""
+Thanks, Joey for being open to suggestions!
+
+I was holding my reply back for while to see if more feedbacks comes in. It seems everybody is ok with the current forum. I'm still open to setting up a test forum. Discourse, the forum I have in mind let you couple the forum with a mailing list quite easily.
+
+Best,
+Florian
+
+
+"""]]
diff --git a/doc/forum/Improving_the_git-annex_forum/comment_3_71d5568d2ac64f83dba11100124addd1._comment b/doc/forum/Improving_the_git-annex_forum/comment_3_71d5568d2ac64f83dba11100124addd1._comment
new file mode 100644
index 0000000000..fc025a39bf
--- /dev/null
+++ b/doc/forum/Improving_the_git-annex_forum/comment_3_71d5568d2ac64f83dba11100124addd1._comment
@@ -0,0 +1,30 @@
+[[!comment format=mdwn
+ username="andrew"
+ avatar="http://cdn.libravatar.org/avatar/acc0ece1eedf07dd9631e7d7d343c435"
+ subject="comment 3"
+ date="2018-05-03T15:16:58Z"
+ content="""
+For me, the most difficult thing is keeping informed of responses to my questions.
+
+It looks like adding a comment, or creating a new thread, does have the “email comments to me” checkbox but this is un-checked by default. Maybe it should be checked by default? I must be forgetting to check this box. It seems a lot of people ask a question on the forum then never return to read the response, maybe they aren't subscribing? Or maybe there can be a new user preference subscribe to threads I comment on by default, which would check this box for us. That way users who want to stay informed via git checkouts on the forum don't have to worry about un-checking the email comments to me box all the time.
+
+   +++ User friendly way to (un)subscribe to threads
+
+Agreed, it would be nice to have an unsubscribe/subscribe button on every forum thread. I don't know how to do this from Preferences, my PageSpecs don't seem to be working right now.
+
+  ++++ Bump threads on new replies (that may be a matter of taste)
+
+Yes, I think that would be nice. I do care about seeing active threads first. It would also be good to see some visual notification for threads with no responses.
+
+  ++++ Weird markdown formatting
+
+My main issue is being unable to quote people with nice formatting. The default for Markdown quoting, Right bracket, doesn't format well in this forum, so I end up using space plus-plus-plus…
+
+
+
+
+
+
+
+
+"""]]
diff --git a/doc/forum/Improving_the_git-annex_forum/comment_4_847be68ea1992e6c2ceb2c79cfb65f5f._comment b/doc/forum/Improving_the_git-annex_forum/comment_4_847be68ea1992e6c2ceb2c79cfb65f5f._comment
new file mode 100644
index 0000000000..46db4ca821
--- /dev/null
+++ b/doc/forum/Improving_the_git-annex_forum/comment_4_847be68ea1992e6c2ceb2c79cfb65f5f._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="andrew"
+ avatar="http://cdn.libravatar.org/avatar/acc0ece1eedf07dd9631e7d7d343c435"
+ subject="test PageSpec"
+ date="2018-05-03T15:32:16Z"
+ content="""
+Also, related to Subscribing. I have created a PageSpec in my user preferences. How can I test this? IE I want to past my PageSpec string somewhere and see which results on https://git-annex.branchable.com/ that PageSpec would return.
+"""]]
diff --git a/doc/forum/Init_a_directory_special_remote_with_already_existing_files.mdwn b/doc/forum/Init_a_directory_special_remote_with_already_existing_files.mdwn
new file mode 100644
index 0000000000..40aa41a4f0
--- /dev/null
+++ b/doc/forum/Init_a_directory_special_remote_with_already_existing_files.mdwn
@@ -0,0 +1,17 @@
+Hi,
+
+I've been a git-annex user for few years now, and I'm progressively migrating old rsync backups into git-annex. Most of the time I create new fresh repos or special remotes and re-upload all data into it. But I'm now facing an issue with this workflow: one of the remote disk I use has a very low connection. Since it has already a complete up-to-date plain copy of all files, I'd like to avoid the "re-upload" phase.
+
+I was thinking of using a directory/rsync special remote, and feed it with the existing local content. But the file names & paths are not the usual plain ones. 
+
+- is this a good idea ?
+
+- if yes, what is the way to retrieve the "special remote" path of each file ? I'm not against scripting a little if necessary.
+
+- if no, what can I do ?
+
+Few additional notes:
+
+- I can't ssh to the remote, this is a windows share with a FAT fs underneath.
+
+- I think I can assume all the remote files are good, they are transfered and checked by rsync.
diff --git a/doc/forum/Init_a_directory_special_remote_with_already_existing_files/comment_1_f2a2f7309145d26d124c73ce6fcb32a5._comment b/doc/forum/Init_a_directory_special_remote_with_already_existing_files/comment_1_f2a2f7309145d26d124c73ce6fcb32a5._comment
new file mode 100644
index 0000000000..47be769a49
--- /dev/null
+++ b/doc/forum/Init_a_directory_special_remote_with_already_existing_files/comment_1_f2a2f7309145d26d124c73ce6fcb32a5._comment
@@ -0,0 +1,26 @@
+[[!comment format=mdwn
+ username="andrew"
+ avatar="http://cdn.libravatar.org/avatar/acc0ece1eedf07dd9631e7d7d343c435"
+ subject="comment 1"
+ date="2018-03-28T00:59:41Z"
+ content="""
+I think if you don't mind scripting checkout Joey's comment in this thread: [\"Preseeding\" a special remote](https://git-annex.branchable.com/forum/__34__Preseeding__34___a_special_remote/).
+
+Otherwise maybe the following will work. You'll still need to download all of the files to hash them.
+
+UNTESTED:
+Lets say your Windows share data is at mnt/share/data
+
+ * create a new repo on your Windows share at mnt/share/repo1
+ * [git-annex-import](https://git-annex.branchable.com/git-annex-import/) all of your files into the new repo. This probably would download all of the files to generate the hash, then hopefully do a file rename avoiding the re-upload step.
+ * then create a new [directory remote](http://git-annex.branchable.com/special_remotes/directory/) for repo1 at say, mnt/share/remote1
+ * then do a [git-annex-move](https://git-annex.branchable.com/git-annex-move/) of all the files from mnt/share/repo1 to your new empty remote
+ * you could then make a clone of that repo locally, now you have a nice looking working tree locally
+ * then add mnt/share/remote1 as a new remote to your local clone
+
+If you don't care about hashing you could use the [WORM](http://git-annex.branchable.com/backends/) backend avoiding the initial download.
+
+I think, If you don't care about having a nice working tree you could probably use the web special remote and just leave all the files in their messy rsynced state. IE create a local repo, then do `git annex addurl --fast ` for each file on your Windows share. See joey's comment on addurl in this [DVD thread](http://git-annex.branchable.com/forum/Managing_a_large_number_of_files_archived_on_many_pieces_of_read-only_medium___40__E.G._DVDs__41__/)
+
+
+"""]]
diff --git a/doc/forum/Init_a_directory_special_remote_with_already_existing_files/comment_2_c1740ce07be483d198880f5844fea7e0._comment b/doc/forum/Init_a_directory_special_remote_with_already_existing_files/comment_2_c1740ce07be483d198880f5844fea7e0._comment
new file mode 100644
index 0000000000..7c0708eb2c
--- /dev/null
+++ b/doc/forum/Init_a_directory_special_remote_with_already_existing_files/comment_2_c1740ce07be483d198880f5844fea7e0._comment
@@ -0,0 +1,52 @@
+[[!comment format=mdwn
+ username="oliv5kta@248adc7a643dc1f0ecef26925e357de176d0f6f3"
+ nickname="oliv5kta"
+ avatar="http://cdn.libravatar.org/avatar/d7f0d33c51583bbd8578e4f1f9f8cf4b"
+ subject="comment 2"
+ date="2018-04-25T17:49:48Z"
+ content="""
+Thks for the answer, I found many interesting info. I ended up scripting my own crappy stuff, please see below:
+
+-------------
+    # Populate a special remote directory with files from the input source
+    # The current repository is used to find out keys & file names,
+    # but is not used directly to copy/move the files from
+    # WHERE selects which files & repo to look for
+    # MOVE=1 moves files instead of copying them
+
+    alias annex_populate='MOVE= _annex_populate'
+
+    alias annex_populatem='MOVE=1 _annex_populate'
+
+    _annex_populate() {
+      local DST=\"${1:?No dst directory specified...}\"
+      local SRC=\"${2:-$PWD}\"
+      local WHERE=\"${3:-${WHERE:---include '*'}}\"
+      eval git annex find \"$WHERE\" --format='\${file}\\000\${hashdirlower}\${key}/\${key}\\000' | xargs -r0 -n2 sh -c '
+        MOVE=\"$1\"; SRC=\"$2/$4\"; DST=\"$3/$5\"
+        if [ -r \"$SRC\" ]; then
+          echo \"$SRC -> $DST\"
+          mkdir -p \"$(dirname \"$DST\")\"
+          if [ -n \"$MOVE\" -a ! -h \"$SRC\" ]; then
+            mv -f -T \"$SRC\" \"$DST\"
+          else
+            rsync -K -L --protect-args \"$SRC\" \"$DST\"
+          fi
+        fi
+      ' _ \"$MOVE\" \"$SRC\" \"$DST\"
+    }
+
+-------------
+Basic usage: annex_populate(m) /my/destination/folder/ /my/source/folder
+
+Principle: we get each file key from the current repository, then build the special remote backend filename and launch the copy/move operation.
+
+Warning:
+
+  1. before using it, make a backup of your data !
+
+  2. it was very little tested: it worked for me and I stopped there. Like many scripts in my toolbox, it may malfunction in different cases than mine.
+
+  3. it should copy files unless you define MOVE=X (or have it defined in your shell ! beware).
+
+"""]]
diff --git a/doc/forum/Install_git_annex_v6_in_Ubuntu_14.04_Trusty.mdwn b/doc/forum/Install_git_annex_v6_in_Ubuntu_14.04_Trusty.mdwn
new file mode 100644
index 0000000000..27d53bda6b
--- /dev/null
+++ b/doc/forum/Install_git_annex_v6_in_Ubuntu_14.04_Trusty.mdwn
@@ -0,0 +1 @@
+How can I get v6 for ubuntu trusty?
diff --git a/doc/forum/Install_git_annex_v6_in_Ubuntu_14.04_Trusty/comment_1_9c715cbb0550cad29402dd2fca1917f9._comment b/doc/forum/Install_git_annex_v6_in_Ubuntu_14.04_Trusty/comment_1_9c715cbb0550cad29402dd2fca1917f9._comment
new file mode 100644
index 0000000000..5223290e95
--- /dev/null
+++ b/doc/forum/Install_git_annex_v6_in_Ubuntu_14.04_Trusty/comment_1_9c715cbb0550cad29402dd2fca1917f9._comment
@@ -0,0 +1,13 @@
+[[!comment format=mdwn
+ username="joey"
+ subject="""comment 1"""
+ date="2016-09-05T19:40:48Z"
+ content="""
+I'd suggest using the Linux standalone build if your distribution
+has not yet packaged a new enough version of git-annex.
+
+Or file a bug on your distribution of course, to get them to be more
+proactive. git-annex v6 is at this point 9 months old, and has been
+in Debian since at least May, so I don't know why Ubuntu would be lagging
+on it.
+"""]]
diff --git a/doc/forum/Integrating_annex_get_into_git_checkout_with_hooks.mdwn b/doc/forum/Integrating_annex_get_into_git_checkout_with_hooks.mdwn
new file mode 100644
index 0000000000..5482f78dac
--- /dev/null
+++ b/doc/forum/Integrating_annex_get_into_git_checkout_with_hooks.mdwn
@@ -0,0 +1,7 @@
+I am considering using git annex for an existing work project that is getting bloated with compressed tarballs. We have a team of 7, so making the annex invisible to day to day use is important.
+
+Are there any git hooks that will make the process of git checkout/merge etc automatically do a git annex get ..., our do I have to write these myself?
+
+
+
+
diff --git a/doc/forum/Integrating_annex_get_into_git_checkout_with_hooks/comment_1_7258d0203a0578d0f4e0221295511237._comment b/doc/forum/Integrating_annex_get_into_git_checkout_with_hooks/comment_1_7258d0203a0578d0f4e0221295511237._comment
new file mode 100644
index 0000000000..ed8b0dbe3d
--- /dev/null
+++ b/doc/forum/Integrating_annex_get_into_git_checkout_with_hooks/comment_1_7258d0203a0578d0f4e0221295511237._comment
@@ -0,0 +1,13 @@
+[[!comment format=mdwn
+ username="joey"
+ subject="""comment 1"""
+ date="2016-01-01T20:38:09Z"
+ content="""
+git-annex doesn't set up such hooks, but you can easily make a
+post-checkout and/or post-merge hook that runs, eg "git-annex get"
+
+You might also want a pre-push hook that does "git annex copy --to origin"
+to make sure they upload their annexed files. 
+
+Or, change workflows to use `git annex sync --content`
+"""]]
diff --git a/doc/forum/Integrating_annex_get_into_git_checkout_with_hooks/comment_2_9ca7b7dfc039c443c4afa9b6ea7c78bd._comment b/doc/forum/Integrating_annex_get_into_git_checkout_with_hooks/comment_2_9ca7b7dfc039c443c4afa9b6ea7c78bd._comment
new file mode 100644
index 0000000000..588d82272f
--- /dev/null
+++ b/doc/forum/Integrating_annex_get_into_git_checkout_with_hooks/comment_2_9ca7b7dfc039c443c4afa9b6ea7c78bd._comment
@@ -0,0 +1,15 @@
+[[!comment format=mdwn
+ username="ColinW"
+ subject="Re Comment 1"
+ date="2016-01-01T22:39:14Z"
+ content="""
+Yes, it looks simple enough to setup. Just have to test all the git use-cases to ensure it all works before migrating everyone to (transparently) use it. There are only a few big files (~80Mb), so most people won't need to worry about the annex.
+The way I'm going to do it is:
+1. Put hook scripts into git under ./myhooks, so that a git pull will automatically update to latest hooks.
+2. After cloning the git repo, run a setup script to create symlinks from ./.git/hooks/post-checkout etc to the ./myhooks and also setup the annex with something like:
+   git annex init dummy-annex
+   git remote add dummy-annex {ip}:/dummy-annex.git
+
+So the developer does not have to notice the annex.
+Thanks Joey. 
+"""]]
diff --git a/doc/forum/Integrating_git-annex_with_Windows_Git_Extensions__63__.mdwn b/doc/forum/Integrating_git-annex_with_Windows_Git_Extensions__63__.mdwn
new file mode 100644
index 0000000000..2bb88ab0ef
--- /dev/null
+++ b/doc/forum/Integrating_git-annex_with_Windows_Git_Extensions__63__.mdwn
@@ -0,0 +1,10 @@
+I installed the Windows git-annex, and had to run the following command from Git Bash (in Admin mode) in order to get git to see git-annex:
+
+$ ln -s /C/Program\ Files\ \(x86\)/Git/usr/bin/git-annex.exe git-annex.exe
+
+Was this 
+1) a known install issue that I missed somewhere?
+2) a failure in my particular installation?
+3) or was I supposed to put C:\Program Files (x86)\Git\usr\bin in my Windows path?
+
+Wasn't sure if we needed a docs update...
diff --git a/doc/forum/Integrating_git-annex_with_Windows_Git_Extensions__63__/comment_1_2dac6e3ccf559d98f74c456c0ca7aafc._comment b/doc/forum/Integrating_git-annex_with_Windows_Git_Extensions__63__/comment_1_2dac6e3ccf559d98f74c456c0ca7aafc._comment
new file mode 100644
index 0000000000..0cb400e0e2
--- /dev/null
+++ b/doc/forum/Integrating_git-annex_with_Windows_Git_Extensions__63__/comment_1_2dac6e3ccf559d98f74c456c0ca7aafc._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="browneej@96fb43ae83ff25c968e65939805b60f3e42f1e06"
+ nickname="browneej"
+ subject="git-annex on windows additional"
+ date="2016-05-10T23:26:48Z"
+ content="""
+PS Program Files (x86)\Git\cmd did get added to the path during install, but git-annex.cmd simply ran 'git annex %*' at the cli.
+"""]]
diff --git a/doc/forum/Integrating_git-annex_with_Windows_Git_Extensions__63__/comment_2_181c6c87c64e4fbe798c108537c6a597._comment b/doc/forum/Integrating_git-annex_with_Windows_Git_Extensions__63__/comment_2_181c6c87c64e4fbe798c108537c6a597._comment
new file mode 100644
index 0000000000..e6ed07f9c7
--- /dev/null
+++ b/doc/forum/Integrating_git-annex_with_Windows_Git_Extensions__63__/comment_2_181c6c87c64e4fbe798c108537c6a597._comment
@@ -0,0 +1,12 @@
+[[!comment format=mdwn
+ username="joey"
+ subject="""comment 2"""
+ date="2016-05-12T19:24:39Z"
+ content="""
+Git for Windows has an option at install time to add it to the path.
+What you pick there should control whether git-annex is in the path or not,
+outside of Git Bash.
+
+In Git Bash, /usr/bin/ is in PATH, so the git-annex.exe in there should be
+seen by git.
+"""]]
diff --git a/doc/forum/Invalid_cross-device_link.mdwn b/doc/forum/Invalid_cross-device_link.mdwn
new file mode 100644
index 0000000000..4dd104c150
--- /dev/null
+++ b/doc/forum/Invalid_cross-device_link.mdwn
@@ -0,0 +1,9 @@
+I want to import from an SD card to a mounted annex:
+
+    user@laptop:/media/.../annex/Pictures/(master)$ git annex import /media/.../sdcard/Download/QuickTransfer
+    [...]
+    git-annex: /media/.../sdcard/Download/QuickTransfer/20150428_192110.jpg: rename: unsupported operation (Invalid cross-device link)
+    [...]
+
+Why is it trying to make a cross-device link? As long as it can read the files, 
+it should be OK, no?
diff --git a/doc/forum/Invalid_cross-device_link/comment_1_46c35e04c8f8cb7205f08aa71cc4912a._comment b/doc/forum/Invalid_cross-device_link/comment_1_46c35e04c8f8cb7205f08aa71cc4912a._comment
new file mode 100644
index 0000000000..96b88cad64
--- /dev/null
+++ b/doc/forum/Invalid_cross-device_link/comment_1_46c35e04c8f8cb7205f08aa71cc4912a._comment
@@ -0,0 +1,15 @@
+[[!comment format=mdwn
+ username="joey"
+ subject="""comment 1"""
+ date="2015-05-22T17:19:38Z"
+ content="""
+I do this all the time (most recently 12 hours ago), and have never seen
+this problem. At least not in recent memory.
+
+You didn't say which version of git-annex you are using. Perhaps you
+need to upgrade from some old and broken version...
+
+Looking at the current code, it first tries a rename, and if that
+fails for a reason such as the two locations being on different devices,
+it falls back to using `cp`.
+"""]]
diff --git a/doc/forum/Is_S3___34__Standard-Infrequent_Access__34___storage_class_supported__63__.mdwn b/doc/forum/Is_S3___34__Standard-Infrequent_Access__34___storage_class_supported__63__.mdwn
new file mode 100644
index 0000000000..f8760be5de
--- /dev/null
+++ b/doc/forum/Is_S3___34__Standard-Infrequent_Access__34___storage_class_supported__63__.mdwn
@@ -0,0 +1 @@
+Does `git-annex` support retrieving files from S3 if the remote objects have been moved to the "Standard-Infrequent Access" storage class via a AWS lifecyle rule?
diff --git a/doc/forum/Is_S3___34__Standard-Infrequent_Access__34___storage_class_supported__63__/comment_1_ddc2a1f2b1d810a40b459896993d2e8d._comment b/doc/forum/Is_S3___34__Standard-Infrequent_Access__34___storage_class_supported__63__/comment_1_ddc2a1f2b1d810a40b459896993d2e8d._comment
new file mode 100644
index 0000000000..44d2f5cd76
--- /dev/null
+++ b/doc/forum/Is_S3___34__Standard-Infrequent_Access__34___storage_class_supported__63__/comment_1_ddc2a1f2b1d810a40b459896993d2e8d._comment
@@ -0,0 +1,11 @@
+[[!comment format=mdwn
+ username="joey"
+ subject="""comment 1"""
+ date="2015-09-21T16:11:30Z"
+ content="""
+Retrieval will work just fine.
+
+As for storing files with this storage class already set, it's implemented
+in the `s3-classes` branch, but merging that is blocked on a patch being
+made to the aws library.
+"""]]
diff --git a/doc/forum/Is_an_automagic_upgrade_of_the_object_directory_safe__63__.mdwn b/doc/forum/Is_an_automagic_upgrade_of_the_object_directory_safe__63__.mdwn
new file mode 100644
index 0000000000..5643f6b7a0
--- /dev/null
+++ b/doc/forum/Is_an_automagic_upgrade_of_the_object_directory_safe__63__.mdwn
@@ -0,0 +1,9 @@
+Consider the following two use cases:
+
+* I have a git-annex repo on a portable medium and carry it around between several machines. I use it on a non-important system with the most current git-annex installed, automagic upgrade happens. I am now forced to upgrade git-annex on all other machines. Bonus points if this happens in the background and I don't even notice it until it's too late.
+
+* My system crashes and I use a rescue CD to access local data, including git-annex. The rescue CD includes a newer version of git-annex and once my system is restored, I am forced to upgrade git-annex locally.
+
+My suggestion would be not to upgrade automatically, but to either ask the user if this is OK or to error out and request that they run git annex update by hand.
+
+Optionally, this could be done via a local config variable which should default to error or ask, not upgrade.
diff --git a/doc/forum/Is_an_automagic_upgrade_of_the_object_directory_safe__63__/comment_1_c25900b9d2d62cc0b8c77150bcfebadf._comment b/doc/forum/Is_an_automagic_upgrade_of_the_object_directory_safe__63__/comment_1_c25900b9d2d62cc0b8c77150bcfebadf._comment
new file mode 100644
index 0000000000..8420d7bb3a
--- /dev/null
+++ b/doc/forum/Is_an_automagic_upgrade_of_the_object_directory_safe__63__/comment_1_c25900b9d2d62cc0b8c77150bcfebadf._comment
@@ -0,0 +1,13 @@
+[[!comment format=mdwn
+ username="http://joey.kitenet.net/"
+ nickname="joey"
+ subject="comment 1"
+ date="2011-03-18T00:38:51Z"
+ content="""
+These are good examples; I think you've convinced me at least for upgrades going forward after v2. I'm not sure we have enough users and outdated git-annex installations to worry about it for v1.
+
+(Hoping such upgrades are rare anyway.. Part of the point of changes made in v2 was to allow lots of changes to be made later w/o needing a v3.)
+
+Update: Upgrades from v1 to v2 will no longer be handled automatically
+now.
+"""]]
diff --git a/doc/forum/Is_git-annex_in_a_precarious_state_before_the_initial_commit__63__.mdwn b/doc/forum/Is_git-annex_in_a_precarious_state_before_the_initial_commit__63__.mdwn
new file mode 100644
index 0000000000..4fb3d05af9
--- /dev/null
+++ b/doc/forum/Is_git-annex_in_a_precarious_state_before_the_initial_commit__63__.mdwn
@@ -0,0 +1,18 @@
+So suppose you put an existing folder under git-annex control, like this:
+
+    cd Archive_of_all_my_files
+    git init .
+    git annex init "Archive Folder"
+    git annex add .
+
+And now you forget the last step:
+
+    git commit -am "init"
+
+Instead, you fool around with git. After all, isn't git designed to keep your data safe and make sure there's no way you can accidentally lose it? The only branch that exists is 'git-annex', so you switch to that branch to see what it is:
+
+    git checkout git-annex
+
+This command switches to the branch git-annex and makes a mess in the working directory. And you can't switch back to master since master does not have any commits. Is there a clean way to go back? I tried 'git checkout --orphan master', but the messy files stayed in the working directory. In the meantime, there should probably be a warning on the walkthrough page.
+
+I think git / git annex should warn you before switching the branch in this rare situation.
diff --git a/doc/forum/Is_git-annex_in_a_precarious_state_before_the_initial_commit__63__/comment_1_f9decde3955f10148febc4646fba5a68._comment b/doc/forum/Is_git-annex_in_a_precarious_state_before_the_initial_commit__63__/comment_1_f9decde3955f10148febc4646fba5a68._comment
new file mode 100644
index 0000000000..ed1c52d449
--- /dev/null
+++ b/doc/forum/Is_git-annex_in_a_precarious_state_before_the_initial_commit__63__/comment_1_f9decde3955f10148febc4646fba5a68._comment
@@ -0,0 +1,12 @@
+[[!comment format=mdwn
+ username="http://joeyh.name/"
+ ip="4.154.4.193"
+ subject="comment 1"
+ date="2013-07-02T05:28:00Z"
+ content="""
+Yes, any git repository where there has not been an initial commit made is in an unusal situation. I've often felt this is a design flaw of git; it could start off with an empty root 00000 commit and avoid this complexity.
+
+There are probably all kinds of fancy git commands that can be used to get out of the situation you describe, but finessing a perfect exit from a situation that has happened in the first 5 minutes of using a new repository seems like overkill. Just run `git annex unannex` to move all your files out of git-annex; run `git rm -rf .` to delete the files from the git-annex branch; run `rm -rf .git`, and you're back where you started with some files and no git repository.
+
+... And no, neither git nor git-annex make any promises about keeping your data secure until you have both committed it and sent it somewhere else.
+"""]]
diff --git a/doc/forum/Is_git-annex_in_a_precarious_state_before_the_initial_commit__63__/comment_2_ed32a48edce4f150bedf24cfe91de254._comment b/doc/forum/Is_git-annex_in_a_precarious_state_before_the_initial_commit__63__/comment_2_ed32a48edce4f150bedf24cfe91de254._comment
new file mode 100644
index 0000000000..946fcefa7f
--- /dev/null
+++ b/doc/forum/Is_git-annex_in_a_precarious_state_before_the_initial_commit__63__/comment_2_ed32a48edce4f150bedf24cfe91de254._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="http://joeyh.name/"
+ ip="4.154.4.193"
+ subject="comment 2"
+ date="2013-07-02T05:29:00Z"
+ content="""
+Oh and no, I don't think it's useful to bog down the walkthrough with warnings and caveats about this type of thing.
+"""]]
diff --git a/doc/forum/Is_git-annex_in_a_precarious_state_before_the_initial_commit__63__/comment_3_ef9618850e5e688bac3c646983f00ed8._comment b/doc/forum/Is_git-annex_in_a_precarious_state_before_the_initial_commit__63__/comment_3_ef9618850e5e688bac3c646983f00ed8._comment
new file mode 100644
index 0000000000..f29fc8f813
--- /dev/null
+++ b/doc/forum/Is_git-annex_in_a_precarious_state_before_the_initial_commit__63__/comment_3_ef9618850e5e688bac3c646983f00ed8._comment
@@ -0,0 +1,10 @@
+[[!comment format=mdwn
+ username="https://www.google.com/accounts/o8/id?id=AItOawnHRhCe3qwVKQ8_NOGGSYJnAMW6FFyKbOc"
+ nickname="Holger"
+ subject="comment 3"
+ date="2013-07-02T05:52:10Z"
+ content="""
+Thanks for your helpful response. The cleaning procedure that you propose is perfectly fine.
+
+I wasn't suggesting to put up a big warning sign; I just feel like the walkthrough would be a little smoother if it explained something like: hey, 'git annex add .' just moved all of your precious files to '.git/annex/' and renamed them in a way that may scare you. But don't worry: if, at any point, you want to get them out again and put them back in their original place, just run 'git annex unannex'.
+"""]]
diff --git a/doc/forum/Is_git-annex_in_a_precarious_state_before_the_initial_commit__63__/comment_4_4bf460c5826c36b205e418c4f3f7d770._comment b/doc/forum/Is_git-annex_in_a_precarious_state_before_the_initial_commit__63__/comment_4_4bf460c5826c36b205e418c4f3f7d770._comment
new file mode 100644
index 0000000000..9d5dc042c5
--- /dev/null
+++ b/doc/forum/Is_git-annex_in_a_precarious_state_before_the_initial_commit__63__/comment_4_4bf460c5826c36b205e418c4f3f7d770._comment
@@ -0,0 +1,15 @@
+[[!comment format=mdwn
+ username="Xyem"
+ ip="87.194.19.134"
+ subject="comment 4"
+ date="2013-10-22T11:28:52Z"
+ content="""
+I've had this a few times (or situations very similar).
+
+    git init
+is now always followed by
+
+    git commit -m \"CREATE GIT REPOSITORY\" --allow-empty
+
+.. which I should really set as an alias :)
+"""]]
diff --git a/doc/forum/Is_it_possible_to_create_a_view_on_files_that___42__dont__42___have_a_field_set__63__.mdwn b/doc/forum/Is_it_possible_to_create_a_view_on_files_that___42__dont__42___have_a_field_set__63__.mdwn
new file mode 100644
index 0000000000..1d19bf27e4
--- /dev/null
+++ b/doc/forum/Is_it_possible_to_create_a_view_on_files_that___42__dont__42___have_a_field_set__63__.mdwn
@@ -0,0 +1,3 @@
+I'm slowly adding metadata to https://github.com/ocharles/papers. For now, I'm mainly interested in setting `year` and `author` fields on files. There are a lot of files here though, so it's difficult to to work out which need metadata and which don't.
+
+Would it be possible to have a `git annex metadata -s author='*'` also have all files in the top level if they don't have an author set at all, or something? Basically, I'm looking for a way to get a view where files don't have `author` set at all, so I can fix that.
diff --git a/doc/forum/Is_it_possible_to_create_a_view_on_files_that___42__dont__42___have_a_field_set__63__/comment_1_f452c911b24ba40aef3be5bdaac9f9ee._comment b/doc/forum/Is_it_possible_to_create_a_view_on_files_that___42__dont__42___have_a_field_set__63__/comment_1_f452c911b24ba40aef3be5bdaac9f9ee._comment
new file mode 100644
index 0000000000..032811d3c2
--- /dev/null
+++ b/doc/forum/Is_it_possible_to_create_a_view_on_files_that___42__dont__42___have_a_field_set__63__/comment_1_f452c911b24ba40aef3be5bdaac9f9ee._comment
@@ -0,0 +1,11 @@
+[[!comment format=mdwn
+ username="CandyAngel"
+ subject="comment 1"
+ date="2016-04-29T10:04:55Z"
+ content="""
+You can do this, but it involves setting that field's value to something.
+
+    git annex metadata -s field?=untagged
+
+This will tag all files that don't have any 'field' values with 'untagged', so when you switch to the view, they will show up under 'untagged' (and you move/commit them to tag them).
+"""]]
diff --git a/doc/forum/Is_it_possible_to_create_a_view_on_files_that___42__dont__42___have_a_field_set__63__/comment_2_7373ec002ca78f4f382ef9297f39639e._comment b/doc/forum/Is_it_possible_to_create_a_view_on_files_that___42__dont__42___have_a_field_set__63__/comment_2_7373ec002ca78f4f382ef9297f39639e._comment
new file mode 100644
index 0000000000..e8b2f29408
--- /dev/null
+++ b/doc/forum/Is_it_possible_to_create_a_view_on_files_that___42__dont__42___have_a_field_set__63__/comment_2_7373ec002ca78f4f382ef9297f39639e._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="ollie@6d66b5a4b0992998efe3e4c244a66708bf5d96f3"
+ nickname="ollie"
+ subject="comment 2"
+ date="2016-04-29T15:04:17Z"
+ content="""
+Thanks! That will do the job :)
+"""]]
diff --git a/doc/forum/Is_it_possible_to_create_a_view_on_files_that___42__dont__42___have_a_field_set__63__/comment_3_cda315119bd3ccce74bc3df8dd56bf1a._comment b/doc/forum/Is_it_possible_to_create_a_view_on_files_that___42__dont__42___have_a_field_set__63__/comment_3_cda315119bd3ccce74bc3df8dd56bf1a._comment
new file mode 100644
index 0000000000..46b724ec92
--- /dev/null
+++ b/doc/forum/Is_it_possible_to_create_a_view_on_files_that___42__dont__42___have_a_field_set__63__/comment_3_cda315119bd3ccce74bc3df8dd56bf1a._comment
@@ -0,0 +1,22 @@
+[[!comment format=mdwn
+ username="joey"
+ subject="""comment 3"""
+ date="2016-05-03T17:48:07Z"
+ content="""
+Actually, you can do it without setting the field to a dummy value.
+
+For example, to find all files without an author metadata:
+
+	git annex find --not --metadata 'author=*'
+
+Same switches can be used with other commands, including the metadata
+command.
+
+You can also use the git-annex view command to enter a branch that only
+contains the files without metadata set:
+
+	git annex view 'author!=*'
+
+(Unfortunately, adding metadata to a file does not currently update this
+view branch.)
+"""]]
diff --git a/doc/forum/Is_it_possible_to_make_git-sync_not_nullify_symlinks__63__.mdwn b/doc/forum/Is_it_possible_to_make_git-sync_not_nullify_symlinks__63__.mdwn
new file mode 100644
index 0000000000..9c41acb707
--- /dev/null
+++ b/doc/forum/Is_it_possible_to_make_git-sync_not_nullify_symlinks__63__.mdwn
@@ -0,0 +1,23 @@
+Hey,
+
+I've found that git annex works great as a way to publish websites to a web server.  I can edit my website on my computer, `git annex sync` my working directory to the VPS, and then `git annex get files-I-want-to-publish`.  This works great.  I can maintain my normal working directory structure on the VPS and I don't have to worry about people seeing files I DIDN'T want to publish, since the dead symlinks just show up as 404s.
+
+There's one small problem.  
+
+Say:
+
+ 1) I've already published a file using `git annex get file-to-update`
+
+ 2) I update that file on my computer
+
+ 3) I do `git annex sync`
+
+ 4) I do `git annex get file-to-update`
+
+Between steps 3 and 4, file-to-update goes from being an accessible web resource to being a dead symlink.  It's not really a problem for me, as hardly anyone visits my site. But it would be nice if I could make `sync` leave the old symlink to the old file until I `get`ed the new one.
+
+Is this possible?
+
+PS: For those who might follow in my footsteps, remember that you probably don't want people reading the contents of your .git dir, so make a re-write rule for this!
+
+Timothy
diff --git a/doc/forum/Is_it_possible_to_make_git-sync_not_nullify_symlinks__63__/comment_1_d6f2d2cdc5f4ffde9eee9f3a8c215a06._comment b/doc/forum/Is_it_possible_to_make_git-sync_not_nullify_symlinks__63__/comment_1_d6f2d2cdc5f4ffde9eee9f3a8c215a06._comment
new file mode 100644
index 0000000000..0c20b9dd69
--- /dev/null
+++ b/doc/forum/Is_it_possible_to_make_git-sync_not_nullify_symlinks__63__/comment_1_d6f2d2cdc5f4ffde9eee9f3a8c215a06._comment
@@ -0,0 +1,10 @@
+[[!comment format=mdwn
+ username="http://joeyh.name/"
+ ip="2001:4978:f:21a::2"
+ subject="comment 1"
+ date="2013-08-07T16:18:52Z"
+ content="""
+I'd recommend that you just copy your files to the repository to publish them before running git-annex sync.
+
+Alternatively, unless your web pages are very large, you can just check them into git directly, still use git-annex sync if you want (and still using git-annex for large files), and avoid the complication of being able to have files that are listed in the repository but whose content is not present.
+"""]]
diff --git a/doc/forum/Is_parallel_fsck_for_two_or_more_different_directories_possible__63__.mdwn b/doc/forum/Is_parallel_fsck_for_two_or_more_different_directories_possible__63__.mdwn
new file mode 100644
index 0000000000..41eef70205
--- /dev/null
+++ b/doc/forum/Is_parallel_fsck_for_two_or_more_different_directories_possible__63__.mdwn
@@ -0,0 +1,31 @@
+I can put `git-annex fsck` in a loop to check a large directory like this:
+
+`-S` starts an incremental check, `-m` continues the started incremental check, `&>>` appends all output (both `stdout` and `stderr`) into the `fsck.log` file. 
+
+```
+$ git-annex fsck -S large-directory --from remote-repo --time-limit=60s &>>~/log/fsck.log
+#...
+#...
+#...
+$ while (sleep 10); do
+  git-annex fsck -m large-directory --from remote-repo --time-limit=1h &>>~/log/fsck.log
+#...
+#...
+#...
+done;
+```
+
+I need the loop because the connection to `remote-repo` fails after some time (or because remote server error) and needs a reconnect, after that, everything is ok.
+
+Suppose, I have many large directories and it would be faster to check them if I could run them parallelly. Many small files, they do not take too much bandwidth but more I/O and network communication.
+
+I know that the progress of `fsck` is stored in a database (now after every 1000 files or 5 minutes or `--time-limit`) but is the checked directory (large-directory) is taken into account when starting/storing the progress? 
+
+**Is the checked directory/path in the primary-key?** Or is it much more complicated?
+
+If I could start checking many directories in the same time, `fsck` would finish much faster (think about thousands of small icon files). Is it just me or somebody else could profit from this?
+
+(This is _not_ a feature request, I would like to know if anybody needs this, if possible at all.)
+
+Thanks,
+parhuzamos
diff --git a/doc/forum/Is_parallel_fsck_for_two_or_more_different_directories_possible__63__/comment_1_34af73f211cce966927757f7deec88eb._comment b/doc/forum/Is_parallel_fsck_for_two_or_more_different_directories_possible__63__/comment_1_34af73f211cce966927757f7deec88eb._comment
new file mode 100644
index 0000000000..b456480357
--- /dev/null
+++ b/doc/forum/Is_parallel_fsck_for_two_or_more_different_directories_possible__63__/comment_1_34af73f211cce966927757f7deec88eb._comment
@@ -0,0 +1,24 @@
+[[!comment format=mdwn
+ username="joey"
+ subject="""comment 1"""
+ date="2015-08-04T20:14:00Z"
+ content="""
+This is actually something that got worse when fsck changed to using a
+database, rather than its old sticky-bit based hack. Parallel fsck used to
+work entirely perfectly, they could even be run on the same directories w/o
+processing files redundantly.
+
+Now, it's safe to run multiple fsck processes in parallel. However, since
+the database is only occasionally updated, if the two fscks are working on
+the same directory, one won't know that the other has already fscked a
+file, and they'll tend to do redundant work.
+
+It'll work fine if you give concurrent fscks different directories
+or sets of files to work on. The git-annex key (ie, symlink target of a
+file) is the primary-key.
+
+Also, I'd at some point like to make git-annex fsck -J work. With
+concurrent fsck jobs running in the same process, it could easily divide
+work up amoung them. The only tricky part is the output of the concurrent
+jobs would be scrambled and interleaved..
+"""]]
diff --git a/doc/forum/Is_setting_core.preloadindex_safe_with_git_annex__63__.mdwn b/doc/forum/Is_setting_core.preloadindex_safe_with_git_annex__63__.mdwn
new file mode 100644
index 0000000000..09ca5fe360
--- /dev/null
+++ b/doc/forum/Is_setting_core.preloadindex_safe_with_git_annex__63__.mdwn
@@ -0,0 +1 @@
+I found [this suggestion](http://stackoverflow.com/a/2873039) on Stack Overflow. Is this safe to use in git annex? I don't know how the internals work. I have a very large annex and I don't want to screw it up.
diff --git a/doc/forum/Is_setting_core.preloadindex_safe_with_git_annex__63__/comment_1_c250362236a33d8dc9bf5ed825318c40._comment b/doc/forum/Is_setting_core.preloadindex_safe_with_git_annex__63__/comment_1_c250362236a33d8dc9bf5ed825318c40._comment
new file mode 100644
index 0000000000..073a91d35f
--- /dev/null
+++ b/doc/forum/Is_setting_core.preloadindex_safe_with_git_annex__63__/comment_1_c250362236a33d8dc9bf5ed825318c40._comment
@@ -0,0 +1,23 @@
+[[!comment format=mdwn
+ username="joey"
+ subject="""comment 1"""
+ date="2015-07-17T15:59:16Z"
+ content="""
+I don't fully understand what core.preloadindex does, but according to
+git's documentation, it's enabled by default. So, it seems pretty safe. :)
+
+Let's anwser the general question: When is a git config tweak not safe to
+use with a git-annex repository? Well, git-annex just stores symlinks in
+git. And, it maintains a separate git-annex branch with its own data, which
+it makes commits to behind the scenes, using its own .git/annex/index file.
+Otherwise, a git-annex repo is like any other git repo.
+
+It follows that, unless the git config tweak somehow breaks symlink
+support, or somehow breaks git-annex's use of the .git/annex/index file,
+the git config tweak should be safe to use with git-annex.
+
+An example of a config tweak that's not entirely safe is to configure
+gc.pruneexpire to a very short time period. That could result in git
+objects that .git/annex/index refers to being pruned before git-annex can
+commit them to the git-annex branch.
+"""]]
diff --git a/doc/forum/Is_there_a___34__git_annex_cat-file__34___type_command__63__.mdwn b/doc/forum/Is_there_a___34__git_annex_cat-file__34___type_command__63__.mdwn
new file mode 100644
index 0000000000..898cb0a2ff
--- /dev/null
+++ b/doc/forum/Is_there_a___34__git_annex_cat-file__34___type_command__63__.mdwn
@@ -0,0 +1,20 @@
+Out of curiosity, is there an equivalent to `git cat-file` with `git annex`?
+
+The motivation is our usage of Bazel as a build system, which during test enforces hermiticity, and thus is very persnickity about modifying your workspace (e.g., the Git repository) while the test is being run, and usually isolates execution to a chroot'd sandbox of sorts.
+
+Ideally, the workflows I'd like are:
+
+A. Developer
+
+- 1. Clones repository.
+- 2. Inits `git annex`, and does `git annex get .` to fetch all required files.
+- 3. Runs `bazel test //repo:my_test`, which will symlink the existing large file into the sandbox, and run without a hitch.
+
+B. Tentative Contributor
+
+- 1. Clones repository. Pokes around.
+- 2. Runs `bazel test //repo:my_test`. Since the large file does not exist, under the hood `git annex cat-file` is called to directly add the file to sandbox (possibly caching it somewhere, such that `git annex get` will use the already fetch'd file).
+
+
+May I ask if this doable with simple visible commands?
+If not, is there a way to achieve this that is special remote-agnostic?
diff --git a/doc/forum/Is_there_a___34__git_annex_cat-file__34___type_command__63__/comment_1_fe3dc778b35119d3ede2d341cac7a15b._comment b/doc/forum/Is_there_a___34__git_annex_cat-file__34___type_command__63__/comment_1_fe3dc778b35119d3ede2d341cac7a15b._comment
new file mode 100644
index 0000000000..f087e8de87
--- /dev/null
+++ b/doc/forum/Is_there_a___34__git_annex_cat-file__34___type_command__63__/comment_1_fe3dc778b35119d3ede2d341cac7a15b._comment
@@ -0,0 +1,11 @@
+[[!comment format=mdwn
+ username="joey"
+ subject="""comment 1"""
+ date="2017-09-29T19:49:29Z"
+ content="""
+Think of `git annex get` as more like `git pull` than `git cat-file`. 
+It necessarily needs to modify the repository to get the contents of files.
+
+What about making a temporary clone of the git repository that's
+writable in the sandbox?
+"""]]
diff --git a/doc/forum/Is_there_a_way_to_get_back_to_clean-state_after_unworking_annex_assistant_configuration_attempt__63__.mdwn b/doc/forum/Is_there_a_way_to_get_back_to_clean-state_after_unworking_annex_assistant_configuration_attempt__63__.mdwn
new file mode 100644
index 0000000000..cf5a5c8be4
--- /dev/null
+++ b/doc/forum/Is_there_a_way_to_get_back_to_clean-state_after_unworking_annex_assistant_configuration_attempt__63__.mdwn
@@ -0,0 +1,28 @@
+I am trying to synchronize "Linux Workstation" and "Android Tablet" with annex assistant through RSync remote and my google account.
+
+I keep failing horribly :-) I followed http://git-annex.branchable.com/assistant/remote_sharing_walkthrough/,
+but I seem to either do the steps in wrong order, because I never managed to materialize the files on my Android device.
+
+Steps went thusly:
+
+1. L) install annex on linux and run assistant and select repository dir
+2. L) add rsync remote, set folder to android_docs
+3. L) add jabber account -> I see progress bars and it seems to sync just fine
+4. A) install annex on android, specify repository
+5. A) add the "share with other devices repository"
+6. A) login with my google account
+7. A) sync fails because I am on failing wifi network
+8. A) try to add RSync remote, accidentaly specify annex folder
+9. A) realize mistake, try to remove rsync repo
+10. A) hangs on "cleaning step"
+11. A) battery dies
+12. A) charge battery, start android, move to functional wifi network
+13. A) start annex assistant, hangs in terminal (don't remember error message :-/)
+14. A) remove repo folder from android, remove annex,download apk again, reinstall
+15. A) run annex, select folder, add the "share with other devices repository", login with google account
+16. L) See fail syncing on linux "fatal: Could not read from remote repository. Please make sure you have the correct access rights and the repository exists."
+17. A) android seems that it finished syncig, asks for remote repository
+18. A) specify my rsync account with correct folder "android_docs"
+19. A) Could not resolve hostname git-annex-$servername-$username_$port_$directory
+
+I am starting to thing, that randomly creating and removing repositories is not the way forward, so I'd like to get to clean state and redo my setup :)
diff --git a/doc/forum/Is_there_a_way_to_get_back_to_clean-state_after_unworking_annex_assistant_configuration_attempt__63__/comment_1_d77fbbbe3a7438a1e79f175df1f69ef3._comment b/doc/forum/Is_there_a_way_to_get_back_to_clean-state_after_unworking_annex_assistant_configuration_attempt__63__/comment_1_d77fbbbe3a7438a1e79f175df1f69ef3._comment
new file mode 100644
index 0000000000..8dc0692e63
--- /dev/null
+++ b/doc/forum/Is_there_a_way_to_get_back_to_clean-state_after_unworking_annex_assistant_configuration_attempt__63__/comment_1_d77fbbbe3a7438a1e79f175df1f69ef3._comment
@@ -0,0 +1,17 @@
+[[!comment format=mdwn
+ username="https://www.google.com/accounts/o8/id?id=AItOawldUCypSR21BObbzC0Uf8NVd1vZnjYlXZc"
+ nickname="Adam"
+ subject="Ok, found it :-)"
+ date="2014-10-04T10:11:55Z"
+ content="""
+I think I found reliable although a bit excessive way to get back to clean state:
+
+1. stop all the running daemons
+2. on android, remove folder git-annex.home
+3. on linux, remove ~/.config/git-annex and ~/.ssh/git-annex
+4. on both attempted repositories I removed .git folder  
+5. I removed everything I have stored on rsync.net
+
+Then I re-did the steps on good wifi network an plugged in to power source on both devices :-)
+It seems that important step is to wait for the annex to share the remote repo with the other device (took ~5 minutes)
+"""]]
diff --git a/doc/forum/Is_there_a_way_to_log_what_changed_in_files_committed__44___similar_to_how_git_normally_works_for_coding_files__63__.mdwn b/doc/forum/Is_there_a_way_to_log_what_changed_in_files_committed__44___similar_to_how_git_normally_works_for_coding_files__63__.mdwn
new file mode 100644
index 0000000000..5844e7cf44
--- /dev/null
+++ b/doc/forum/Is_there_a_way_to_log_what_changed_in_files_committed__44___similar_to_how_git_normally_works_for_coding_files__63__.mdwn
@@ -0,0 +1,11 @@
+Hello all,
+
+I work for a company that produces books using InDesign and marketing assets for these books in Photoshop/Illustrator.
+
+What we're ultimately looking for is a system that allows us to make changes (primarily to the books) and log the changes to the file so we can keep track of what has been edited when.
+
+Ideally, we'd also be able to pull up an older version of the file if needed. So if we have 10 commits and are on commit #10, we could pull up commit #7 if needed and see that file as it was back when it was committed the 7th time.
+
+I hope I explained what I'm looking for clearly. Is this something git-annex could help wtih?
+
+Thanks
diff --git a/doc/forum/Is_there_a_way_to_log_what_changed_in_files_committed__44___similar_to_how_git_normally_works_for_coding_files__63__/comment_1_e30310472df6ed2928dfd63464c6bae7._comment b/doc/forum/Is_there_a_way_to_log_what_changed_in_files_committed__44___similar_to_how_git_normally_works_for_coding_files__63__/comment_1_e30310472df6ed2928dfd63464c6bae7._comment
new file mode 100644
index 0000000000..af153fd295
--- /dev/null
+++ b/doc/forum/Is_there_a_way_to_log_what_changed_in_files_committed__44___similar_to_how_git_normally_works_for_coding_files__63__/comment_1_e30310472df6ed2928dfd63464c6bae7._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="CandyAngel"
+ avatar="http://cdn.libravatar.org/avatar/15c0aade8bec5bf004f939dd73cf9ed8"
+ subject="comment 1"
+ date="2018-02-27T10:59:02Z"
+ content="""
+You can write the changes in the commit message as you would for using git with coding, if that is what you mean..?
+"""]]
diff --git a/doc/forum/Is_there_a_way_to_log_what_changed_in_files_committed__44___similar_to_how_git_normally_works_for_coding_files__63__/comment_2_f093b16307bd369399574fc651569b73._comment b/doc/forum/Is_there_a_way_to_log_what_changed_in_files_committed__44___similar_to_how_git_normally_works_for_coding_files__63__/comment_2_f093b16307bd369399574fc651569b73._comment
new file mode 100644
index 0000000000..208146ef1a
--- /dev/null
+++ b/doc/forum/Is_there_a_way_to_log_what_changed_in_files_committed__44___similar_to_how_git_normally_works_for_coding_files__63__/comment_2_f093b16307bd369399574fc651569b73._comment
@@ -0,0 +1,10 @@
+[[!comment format=mdwn
+ username="Jeff"
+ avatar="http://cdn.libravatar.org/avatar/99a3c8987afede93f4e5d3212b43ad5f"
+ subject="comment 2"
+ date="2018-02-28T14:43:52Z"
+ content="""
+Yep, that's what I mean! And would I be able to revert to previous commits like if I were using Github to track code? If it's for large files, does it just make a hidden copy of each version of a file every time I add a new commit?
+
+Thanks!
+"""]]
diff --git a/doc/forum/Is_there_a_way_to_log_what_changed_in_files_committed__44___similar_to_how_git_normally_works_for_coding_files__63__/comment_3_1edab7e44e7eae6d8f06a917704b17ec._comment b/doc/forum/Is_there_a_way_to_log_what_changed_in_files_committed__44___similar_to_how_git_normally_works_for_coding_files__63__/comment_3_1edab7e44e7eae6d8f06a917704b17ec._comment
new file mode 100644
index 0000000000..0e32b81708
--- /dev/null
+++ b/doc/forum/Is_there_a_way_to_log_what_changed_in_files_committed__44___similar_to_how_git_normally_works_for_coding_files__63__/comment_3_1edab7e44e7eae6d8f06a917704b17ec._comment
@@ -0,0 +1,12 @@
+[[!comment format=mdwn
+ username="joey"
+ subject="""comment 3"""
+ date="2018-02-28T18:02:04Z"
+ content="""
+Yes, git-annex retains all past versions of files, just like git does. You
+can check out a past version of the repository, the same as you would in a
+git repository (because it is a git repository..)
+
+(There are exceptions, like then `git annex unused` has been used to
+save space and delete some past revisions.)
+"""]]
diff --git a/doc/forum/Is_there_a_way_to_override_the_import_destination__63__.mdwn b/doc/forum/Is_there_a_way_to_override_the_import_destination__63__.mdwn
new file mode 100644
index 0000000000..0fa5c51422
--- /dev/null
+++ b/doc/forum/Is_there_a_way_to_override_the_import_destination__63__.mdwn
@@ -0,0 +1,5 @@
+I want to import a file that's on my filesystem as a/b/c, but have it end up in the repo at $REPO/d/e/f -- but only if it's not already present, i.e. using `--skip-duplicates`.  There are two requirements here; `git annex import` will meet one of them (dedup), and copy+`git annex add` will meet the other (arbitrary destination).  Is there a way to meet both at once?
+
+(Actually, for my current use case, what I really want is to import a/b/c to $REPO/a/b/c -- but _not_ to import all the rest of the tree under a/.  Can't get there either.  If I say `git annex import a/b/c`, the directory structure isn't preserved; the file ends up at $REPO/c.  For this specific use case, a `--preserve-directories` option to `import` would be splendid, but that's rather specific; being able to specify an arbitrary destination would be a more general solution.)
+
+Thanks.
diff --git a/doc/forum/Is_there_a_way_to_override_the_import_destination__63__/comment_1_5f094cc606eebd2aeeaaa4b861cf2c21._comment b/doc/forum/Is_there_a_way_to_override_the_import_destination__63__/comment_1_5f094cc606eebd2aeeaaa4b861cf2c21._comment
new file mode 100644
index 0000000000..27d17f99e4
--- /dev/null
+++ b/doc/forum/Is_there_a_way_to_override_the_import_destination__63__/comment_1_5f094cc606eebd2aeeaaa4b861cf2c21._comment
@@ -0,0 +1,9 @@
+[[!comment format=mdwn
+ username="joey"
+ subject="""comment 1"""
+ date="2018-08-03T17:37:59Z"
+ content="""
+`git-annex import` imports files into the current working directory,
+so make whatever subdirectory structure you want, cd into the place
+and import to there.
+"""]]
diff --git a/doc/forum/Is_there_a_way_to_unannex_some_file___63__.mdwn b/doc/forum/Is_there_a_way_to_unannex_some_file___63__.mdwn
new file mode 100644
index 0000000000..78aa6744a4
--- /dev/null
+++ b/doc/forum/Is_there_a_way_to_unannex_some_file___63__.mdwn
@@ -0,0 +1,3 @@
+Having by mistake annex a full repo i look for a way of unannex some file to make them
+managed by the "standard" git proccess again - mostly some source code file -
+Is there a way  to do that ?
diff --git a/doc/forum/Is_there_a_way_to_unannex_some_file___63__/comment_1_82095cef37dd815bcd1d101a1715e3c7._comment b/doc/forum/Is_there_a_way_to_unannex_some_file___63__/comment_1_82095cef37dd815bcd1d101a1715e3c7._comment
new file mode 100644
index 0000000000..0f87f1460c
--- /dev/null
+++ b/doc/forum/Is_there_a_way_to_unannex_some_file___63__/comment_1_82095cef37dd815bcd1d101a1715e3c7._comment
@@ -0,0 +1,12 @@
+[[!comment format=mdwn
+ username="Horus"
+ avatar="http://cdn.libravatar.org/avatar/8f0ee08b98ea5bba76c3fe112c08851c"
+ subject="comment 1"
+ date="2017-09-07T09:30:53Z"
+ content="""
+Well, there is ```git unannex ...``` 
+
+
+
+Is that what you're looking for?
+"""]]
diff --git a/doc/forum/Is_there_a_way_to_unannex_some_file___63__/comment_2_3b05e9eaa554b70594f0195dfcef831d._comment b/doc/forum/Is_there_a_way_to_unannex_some_file___63__/comment_2_3b05e9eaa554b70594f0195dfcef831d._comment
new file mode 100644
index 0000000000..d911c5efec
--- /dev/null
+++ b/doc/forum/Is_there_a_way_to_unannex_some_file___63__/comment_2_3b05e9eaa554b70594f0195dfcef831d._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="karel-de-macil"
+ avatar="http://cdn.libravatar.org/avatar/46688332185c941113a9c9827cb093e9"
+ subject="comment 2"
+ date="2017-09-08T08:31:36Z"
+ content="""
+yes exactly , thanks alot...
+"""]]
diff --git a/doc/forum/Is_there_an___39__annex-cost__39___to_NEVER_access_remote__63__.mdwn b/doc/forum/Is_there_an___39__annex-cost__39___to_NEVER_access_remote__63__.mdwn
new file mode 100644
index 0000000000..0f9b9924b8
--- /dev/null
+++ b/doc/forum/Is_there_an___39__annex-cost__39___to_NEVER_access_remote__63__.mdwn
@@ -0,0 +1,19 @@
+Depending on how I am getting my network connection there are cases where I want to never access a remote.  BUT it depends on the network connection.
+
+For example, when I'm roaming (and paying excessively for each byte)
+
+- I really don't want my music annex to *ever* hit some site (like S3 / Glacier), BUT
+- I do want to be able to get from a remote (like Glacier) for my work annex.
+
+But if I'm home, then all's fair.
+
+That means I can't do something simple like blocking a particular address/IP with DNS (or whatever).  Or even not having the remote listed.
+
+So... I was thinking of using `annex-cost-command` to allow me to set the cost of remotes, based on (say) my IP, but also the annex.  Thus it would result in
+
+- [roaming] + [work-annex] == veryExpensiveRemoteCost
+- [roaming] + [music-annex] == Blocked
+
+However I still need a way to say "never, ever, ever" for the cost of some remotes.
+
+Is this possible?  (With my cursory looking at the code, I can't see anything.)
diff --git a/doc/forum/Is_there_an___39__annex-cost__39___to_NEVER_access_remote__63__/comment_1_3760d5b8c8012405eef17e4d07cf151c._comment b/doc/forum/Is_there_an___39__annex-cost__39___to_NEVER_access_remote__63__/comment_1_3760d5b8c8012405eef17e4d07cf151c._comment
new file mode 100644
index 0000000000..1d6373527d
--- /dev/null
+++ b/doc/forum/Is_there_an___39__annex-cost__39___to_NEVER_access_remote__63__/comment_1_3760d5b8c8012405eef17e4d07cf151c._comment
@@ -0,0 +1,30 @@
+[[!comment format=mdwn
+ username="olaf"
+ avatar="http://cdn.libravatar.org/avatar/4ae498d3d6ee558d6b65caa658f72572"
+ subject="comment 1"
+ date="2017-08-15T23:28:58Z"
+ content="""
+Just to clarify...
+
+There will be situations where I will want to access other remotes (either networked or special, like an external drive) when I'm blocking access to others (like Glacier).
+
+So, for the music annex example when I'm roaming:
+
+- Remote: external drive  -->  low cost
+- Remote: other machine on my local network  -->  medium cost
+- Remote: youtube --> high cost  (if, for example, my data provider gives me a certain amount of free youtube streaming...)
+- Remote: Glacier  -->  NO
+
+...and in this case if there are files that are only available on Glacier, I'm happy to live without them.
+
+And for the work example, again when roaming:
+
+- Remote: external drive  -->  low cost
+- Remote: other machine on my local network  -->  medium cost
+- Remote: Glacier  -->  high cost
+
+...but if there is a file I need on Glacier and no where else, then just get it.
+
+
+
+"""]]
diff --git a/doc/forum/Is_there_an___39__annex-cost__39___to_NEVER_access_remote__63__/comment_2_b95ce6b13ae45a2fca05260a7f77148f._comment b/doc/forum/Is_there_an___39__annex-cost__39___to_NEVER_access_remote__63__/comment_2_b95ce6b13ae45a2fca05260a7f77148f._comment
new file mode 100644
index 0000000000..d5f78140ce
--- /dev/null
+++ b/doc/forum/Is_there_an___39__annex-cost__39___to_NEVER_access_remote__63__/comment_2_b95ce6b13ae45a2fca05260a7f77148f._comment
@@ -0,0 +1,19 @@
+[[!comment format=mdwn
+ username="joey"
+ subject="""comment 2"""
+ date="2017-08-17T15:59:13Z"
+ content="""
+Cost settings only affect sort order, they don't allow completely blocking
+use of a remote.
+
+I feel that it would be perhaps the wrong design to extend cost settings
+into blocking a remote.
+
+There's also the `remote..annex-start-command` hook. However,
+failure to run its command does not prevent git-annex from trying to access
+a remote.
+
+I think it would be reasonable to add a hook
+`remote..annex-check-command`. If set, the command would have
+to succed, or git-annex would refuse to use the remote.
+"""]]
diff --git a/doc/forum/Is_there_an___39__annex-cost__39___to_NEVER_access_remote__63__/comment_3_1c8f023b8d7a50cfce6d3464b12e446b._comment b/doc/forum/Is_there_an___39__annex-cost__39___to_NEVER_access_remote__63__/comment_3_1c8f023b8d7a50cfce6d3464b12e446b._comment
new file mode 100644
index 0000000000..0278e8637e
--- /dev/null
+++ b/doc/forum/Is_there_an___39__annex-cost__39___to_NEVER_access_remote__63__/comment_3_1c8f023b8d7a50cfce6d3464b12e446b._comment
@@ -0,0 +1,17 @@
+[[!comment format=mdwn
+ username="joey"
+ subject="""comment 3"""
+ date="2017-08-17T16:06:44Z"
+ content="""
+Another good reason to have a separate config is it should probably
+block `git annex sync` from pulling/pushing the remote. Cost does not
+affect pulling and pushing.
+
+The two config settings `annex-ignore` and
+`annex-sync` are the non-dynamic ways to control this. 
+So perhaps the way that annex-cost-command is the dynamic form of
+annex-cost, there ought to be annex-ignore-command and annex-sync-command.
+
+That also allows for situations like a network where you want git-annex
+sync to only push the git repository but not annexed contents.
+"""]]
diff --git a/doc/forum/Is_there_an___39__annex-cost__39___to_NEVER_access_remote__63__/comment_4_f1e234e80a6873b33986bec8d51c1001._comment b/doc/forum/Is_there_an___39__annex-cost__39___to_NEVER_access_remote__63__/comment_4_f1e234e80a6873b33986bec8d51c1001._comment
new file mode 100644
index 0000000000..3bcce95be3
--- /dev/null
+++ b/doc/forum/Is_there_an___39__annex-cost__39___to_NEVER_access_remote__63__/comment_4_f1e234e80a6873b33986bec8d51c1001._comment
@@ -0,0 +1,7 @@
+[[!comment format=mdwn
+ username="joey"
+ subject="""comment 4"""
+ date="2017-08-17T17:53:59Z"
+ content="""
+I've implemented annex-ignore-command and annex-sync-command. Enjoy!
+"""]]
diff --git a/doc/forum/Is_there_an___39__annex-cost__39___to_NEVER_access_remote__63__/comment_5_2c43830a95cd8955da20cc6dba27a9b6._comment b/doc/forum/Is_there_an___39__annex-cost__39___to_NEVER_access_remote__63__/comment_5_2c43830a95cd8955da20cc6dba27a9b6._comment
new file mode 100644
index 0000000000..2f1286ca5d
--- /dev/null
+++ b/doc/forum/Is_there_an___39__annex-cost__39___to_NEVER_access_remote__63__/comment_5_2c43830a95cd8955da20cc6dba27a9b6._comment
@@ -0,0 +1,10 @@
+[[!comment format=mdwn
+ username="olaf"
+ avatar="http://cdn.libravatar.org/avatar/4ae498d3d6ee558d6b65caa658f72572"
+ subject="comment 5"
+ date="2017-08-18T04:18:13Z"
+ content="""
+Brilliant - Thanks Joey!
+
+...and that's why I donated.  :-)
+"""]]
diff --git a/doc/forum/Is_there_any_delta_handling_for_file_modifications__63__.mdwn b/doc/forum/Is_there_any_delta_handling_for_file_modifications__63__.mdwn
new file mode 100644
index 0000000000..30463a496d
--- /dev/null
+++ b/doc/forum/Is_there_any_delta_handling_for_file_modifications__63__.mdwn
@@ -0,0 +1,20 @@
+Hi!
+
+I have few large files, which will be slightly modified from time to time. Consider the following example:
+
+    $ git init
+    $ git annex init "somerepo"
+    $ dd if=/dev/urandom of=foo bs=50M count=1
+    $ git annex add foo
+    $ git commit -m "foo added"
+    $ git annex unlock foo
+    $ echo "modification" >> foo
+    $ git annex add foo
+    $ git -m commit "modification 1 of foo"
+    $ echo "another modification" >> foo
+    $ git annex add foo
+    $ git -m commit "modification 2 of foo"
+
+I would have expected ending up with a full copy of foo, and a delta storage for the first version and the first modification. Instead, three full-sized copies are created within the .git/annex/objects directory, even though the modifications only added a few characters.
+
+Is there any way to not always store a full copy of each file for each modification, but only a delta?
diff --git a/doc/forum/Is_there_any_delta_handling_for_file_modifications__63__/comment_1_d901ac998c9fdb844c593d782c31cc73._comment b/doc/forum/Is_there_any_delta_handling_for_file_modifications__63__/comment_1_d901ac998c9fdb844c593d782c31cc73._comment
new file mode 100644
index 0000000000..bbc7df6d2c
--- /dev/null
+++ b/doc/forum/Is_there_any_delta_handling_for_file_modifications__63__/comment_1_d901ac998c9fdb844c593d782c31cc73._comment
@@ -0,0 +1,9 @@
+[[!comment format=mdwn
+ username="joey"
+ subject="""comment 1"""
+ date="2015-02-04T17:53:50Z"
+ content="""
+git-annex does not do deltas. However, the bup and ddar special remotes
+can both be used to store annexed files and will automatically do delta
+compression.
+"""]]
diff --git a/doc/forum/Is_there_any_delta_handling_for_file_modifications__63__/comment_2_e46cb391ce377c3a8eb40155a5d0f001._comment b/doc/forum/Is_there_any_delta_handling_for_file_modifications__63__/comment_2_e46cb391ce377c3a8eb40155a5d0f001._comment
new file mode 100644
index 0000000000..2883d29e32
--- /dev/null
+++ b/doc/forum/Is_there_any_delta_handling_for_file_modifications__63__/comment_2_e46cb391ce377c3a8eb40155a5d0f001._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="git-annex@c063eec406e857662ac6bbb814defa6aebde6417"
+ nickname="git-annex"
+ subject="any plans to add this?"
+ date="2016-02-21T12:12:43Z"
+ content="""
+Presumably this is a desirable feature for faster syncing of modifications to large files, so are there any plans to add it?
+"""]]
diff --git a/doc/forum/Joey__44___when_would_you_guess_v6_to_be_ready__63__.mdwn b/doc/forum/Joey__44___when_would_you_guess_v6_to_be_ready__63__.mdwn
new file mode 100644
index 0000000000..2c70e371ae
--- /dev/null
+++ b/doc/forum/Joey__44___when_would_you_guess_v6_to_be_ready__63__.mdwn
@@ -0,0 +1,3 @@
+A question for Joey: What's the expected schedule for v6 repos becoming solid enough to make them the default? Is it next on the priority list or are you still waiting for some git patches to propagate through or something like that?
+
+I'm asking because I make extensive use of v5 direct mode repos. They sometimes give me a bit of trouble in ways which I hope indirect, unlocked, thin v6 repos could fix. However, I don't want to switch to v6 as long as Joey considers it too experimental/unfinished to be made the default repo format. So I'm trying to decide whether I should put in the effort to investigate the issues I'm having (which are annoying but not catastrophic), or just wait until v6 is ready for prime time.
diff --git a/doc/forum/Joey__44___when_would_you_guess_v6_to_be_ready__63__/comment_1_08d3fd14a57b5d4dfb2cac411aba757f._comment b/doc/forum/Joey__44___when_would_you_guess_v6_to_be_ready__63__/comment_1_08d3fd14a57b5d4dfb2cac411aba757f._comment
new file mode 100644
index 0000000000..84d0ca100f
--- /dev/null
+++ b/doc/forum/Joey__44___when_would_you_guess_v6_to_be_ready__63__/comment_1_08d3fd14a57b5d4dfb2cac411aba757f._comment
@@ -0,0 +1,10 @@
+[[!comment format=mdwn
+ username="spwhitton"
+ avatar="http://cdn.libravatar.org/avatar/9c3f08f80e67733fd506c353239569eb"
+ subject="current status of v6"
+ date="2018-04-21T19:47:10Z"
+ content="""
+I too would be grateful to know Joey's view on the state of v6.  I would like to replace my direct mode repos.
+
+I'm not sure that the criteria for v6 being suitable for production is that Joey is willing to set it as the default -- there might be reasons other than stability/bugginess not to set it as the default.
+"""]]
diff --git a/doc/forum/Joey__44___when_would_you_guess_v6_to_be_ready__63__/comment_2_ff5b55e81c0d48968e59e1783943f286._comment b/doc/forum/Joey__44___when_would_you_guess_v6_to_be_ready__63__/comment_2_ff5b55e81c0d48968e59e1783943f286._comment
new file mode 100644
index 0000000000..00e87120c0
--- /dev/null
+++ b/doc/forum/Joey__44___when_would_you_guess_v6_to_be_ready__63__/comment_2_ff5b55e81c0d48968e59e1783943f286._comment
@@ -0,0 +1,10 @@
+[[!comment format=mdwn
+ username="qiang.fang@ddaed0de00c2925f8036e6c61ce6e12654263ada"
+ nickname="qiang.fang"
+ avatar="http://cdn.libravatar.org/avatar/5a55c6bcb3eaacde1355f9aca0d9384a"
+ subject="comment 2"
+ date="2018-04-26T09:14:54Z"
+ content="""
+The todo list is here
+https://git-annex.branchable.com/todo/smudge/
+"""]]
diff --git a/doc/forum/Joey__44___when_would_you_guess_v6_to_be_ready__63__/comment_3_f63a66196db731fb04d8e078136eeb3d._comment b/doc/forum/Joey__44___when_would_you_guess_v6_to_be_ready__63__/comment_3_f63a66196db731fb04d8e078136eeb3d._comment
new file mode 100644
index 0000000000..ec929fa6dc
--- /dev/null
+++ b/doc/forum/Joey__44___when_would_you_guess_v6_to_be_ready__63__/comment_3_f63a66196db731fb04d8e078136eeb3d._comment
@@ -0,0 +1,10 @@
+[[!comment format=mdwn
+ username="spwhitton"
+ avatar="http://cdn.libravatar.org/avatar/9c3f08f80e67733fd506c353239569eb"
+ subject="comment 3"
+ date="2018-04-27T17:15:24Z"
+ content="""
+Seems like the main thing is waiting for patches to git to migrate to systems like Debian stable.
+
+Joey, have your patches been merged?
+"""]]
diff --git a/doc/forum/Joey__44___when_would_you_guess_v6_to_be_ready__63__/comment_4_a7ea99c0b8bdc257c0c3453bdb5ce1ca._comment b/doc/forum/Joey__44___when_would_you_guess_v6_to_be_ready__63__/comment_4_a7ea99c0b8bdc257c0c3453bdb5ce1ca._comment
new file mode 100644
index 0000000000..6d496530d9
--- /dev/null
+++ b/doc/forum/Joey__44___when_would_you_guess_v6_to_be_ready__63__/comment_4_a7ea99c0b8bdc257c0c3453bdb5ce1ca._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="spwhitton"
+ avatar="http://cdn.libravatar.org/avatar/9c3f08f80e67733fd506c353239569eb"
+ subject="comment 4"
+ date="2018-05-10T17:51:35Z"
+ content="""
+I just checked -- they have not been merged.
+"""]]
diff --git a/doc/forum/Keybase_encrypted_git_as_a_remote__63__.mdwn b/doc/forum/Keybase_encrypted_git_as_a_remote__63__.mdwn
new file mode 100644
index 0000000000..d474986265
--- /dev/null
+++ b/doc/forum/Keybase_encrypted_git_as_a_remote__63__.mdwn
@@ -0,0 +1,6 @@
+Hello,
+
+Does anybody have experience of the new keybase encrypted git as a remote for git annex?
+
+Details can be found here: https://keybase.io/blog/encrypted-git-for-everyone
+
diff --git a/doc/forum/Lacking_webapp_on_Trisquel__47__Ubuntu_Precise.mdwn b/doc/forum/Lacking_webapp_on_Trisquel__47__Ubuntu_Precise.mdwn
new file mode 100644
index 0000000000..d85eefc2cb
--- /dev/null
+++ b/doc/forum/Lacking_webapp_on_Trisquel__47__Ubuntu_Precise.mdwn
@@ -0,0 +1,7 @@
+I'd like to have git-annex assistant running so I can use it to sync files, but when I downloaded and ran it the webapp was missing.  
+
+I've recently begun using Trisquel, or rather the KDE version Triskel.  The current Trisquel is built from Ubuntu Precise, 12.04, the current LTS version.  Thus it seems that I'm a bit behind the current Ubuntu software, and my impression is that this relates to my problem.
+
+In [a post](http://git-annex.branchable.com/forum/Cannot_launch_webapp_on_ubuntu_12.04_using_ppa/) on this forum I read that a version of git-annex was built without the webapp, and that this had since been corrected.  It appears from my git-annex version data that I have an older version, though I downloaded it only two days ago, and it's older than the corrected version.  Is this because my Ubuntu stuff is out of date? 
+
+More importantly, is there any way I can get webapp running on my current OS, or must I wait until Trisquel gets to 13.04?
diff --git a/doc/forum/Lacking_webapp_on_Trisquel__47__Ubuntu_Precise/comment_1_6bd27bd31833336c1df783253378ccae._comment b/doc/forum/Lacking_webapp_on_Trisquel__47__Ubuntu_Precise/comment_1_6bd27bd31833336c1df783253378ccae._comment
new file mode 100644
index 0000000000..43c8ae525f
--- /dev/null
+++ b/doc/forum/Lacking_webapp_on_Trisquel__47__Ubuntu_Precise/comment_1_6bd27bd31833336c1df783253378ccae._comment
@@ -0,0 +1,10 @@
+[[!comment format=mdwn
+ username="http://joeyh.name/"
+ ip="4.154.4.90"
+ subject="comment 1"
+ date="2013-07-17T19:36:27Z"
+ content="""
+There is a PPA for Ubuntu Precise, which includes the webapp.
+
+Or you can use the standalone tarball, which works nearly everywhere.
+"""]]
diff --git a/doc/forum/Large_Uploads_to_S3__63__.mdwn b/doc/forum/Large_Uploads_to_S3__63__.mdwn
new file mode 100644
index 0000000000..a72df425fb
--- /dev/null
+++ b/doc/forum/Large_Uploads_to_S3__63__.mdwn
@@ -0,0 +1,12 @@
+I set up a new git annex repo with an S3 remote. Uploading small files works file, but the process fails on larger files (>1 GB) with the following error. 
+
+    copy prosper/loaninfo.p (checking s3...) (to s3...) 
+    99%          10.7MB/s 0s
+    S3Error {s3StatusCode = Status {statusCode = 400, statusMessage = "Bad Request"}, s3ErrorCode = "RequestTimeout", s3ErrorMessage = "Your socket connection to the server was not read from or written to within the timeout period. Idle connections     will be closed.", s3ErrorResource = Nothing, s3ErrorHostId = Just "< a base64 encoded string>", s3ErrorAccessKeyId = Nothing, s3ErrorStringToSign = Nothing, s3ErrorBucket = Nothing, s3ErrorEndpointRaw = Nothing, s3ErrorEndpoint = Nothing}
+
+I tried these different options while setting up remote, but nothing worked.
+partsize=1GiB
+partsize=400MiB
+chunk=100MiB
+
+What am I doing wrong? Should I try an even smaller chunk sie
diff --git a/doc/forum/Large_Uploads_to_S3__63__/comment_1_c1fd2ed0b74ce58f818ab53158e581f3._comment b/doc/forum/Large_Uploads_to_S3__63__/comment_1_c1fd2ed0b74ce58f818ab53158e581f3._comment
new file mode 100644
index 0000000000..f09812ec30
--- /dev/null
+++ b/doc/forum/Large_Uploads_to_S3__63__/comment_1_c1fd2ed0b74ce58f818ab53158e581f3._comment
@@ -0,0 +1,26 @@
+[[!comment format=mdwn
+ username="joey"
+ subject="""comment 1"""
+ date="2016-09-05T16:52:24Z"
+ content="""
+Googling for that message suggests it's pretty common for large file
+uploads amoung different AWS implementations for different programming
+languages. The error message is coming from AWS not git-annex.
+
+If this is happening with a single file transfer, I'm pretty sure git-annex
+is not keeping the S3 connection idle. (If one file transfer succeeded, and
+then a later one in the same git-annex run failed, that might indicate that
+the S3 connection was being reused and timed out in between.)
+
+Based on things like ,
+this seems to be down to a network connection problem, especially on
+residential internet connections, such as the link
+getting saturated by something else and so the transfer stalling out.
+
+I think that finding a chunk size that works is your best bet. That
+will let uploads be resumed more or less where they left off.
+
+It might make sense for git-annex to retry an upload that fails this way,
+but imagine if it were a non-chunked 1 gb file and it failed part way
+through every time. That would waste a lot of bandwidth.
+"""]]
diff --git a/doc/forum/Layout_mismatch_between_symlinks_and_annex_objects.mdwn b/doc/forum/Layout_mismatch_between_symlinks_and_annex_objects.mdwn
new file mode 100644
index 0000000000..fe9167e975
--- /dev/null
+++ b/doc/forum/Layout_mismatch_between_symlinks_and_annex_objects.mdwn
@@ -0,0 +1,14 @@
+I have an old repository, and I'm not sure what I have done with it, but it seems to be broken in an interesting fashion.
+
+    $ ls -l pix/image001.jpg 
+    lrwxrwxrwx 1 la la 187 Sep  5 11:31 pix/image001.jpg -> ../.git/annex/objects/G7/J7/SHA256-s99372--0458b0b72b394a719b72032971b880a8dfafa65e8048ec6fc6c861fc3ea5e702/SHA256-s99372--0458b0b72b394a719b72032971b880a8dfafa65e8048ec6fc6c861fc3ea5e702
+    $ ls -lL pix/image001.jpg 
+    ls: cannot access pix/image001.jpg: No such file or directory
+    $ find .git/annex/objects/ -type f -name '*0458b0b72*'
+    .git/annex/objects/477/75c/SHA256-s99372--0458b0b72b394a719b72032971b880a8dfafa65e8048ec6fc6c861fc3ea5e702/SHA256-s99372--0458b0b72b394a719b72032971b880a8dfafa65e8048ec6fc6c861fc3ea5e702
+    $ git config annex.version
+    5
+
+So the symlinks use a layout where index directories have two-letter names, but the actual layout in the annex uses directory names with three hexadecimal digits.
+
+Is there some straightforward way to fix this or do I need to figure the index format and hack up a script to convert the annex repository?
diff --git a/doc/forum/Layout_mismatch_between_symlinks_and_annex_objects/comment_1_854786810dc4246cdaa38ff10c55bbca._comment b/doc/forum/Layout_mismatch_between_symlinks_and_annex_objects/comment_1_854786810dc4246cdaa38ff10c55bbca._comment
new file mode 100644
index 0000000000..d4565f25b6
--- /dev/null
+++ b/doc/forum/Layout_mismatch_between_symlinks_and_annex_objects/comment_1_854786810dc4246cdaa38ff10c55bbca._comment
@@ -0,0 +1,9 @@
+[[!comment format=mdwn
+ username="lealanko"
+ subject="comment 1"
+ date="2015-09-06T13:11:44Z"
+ content="""
+I think I figured it out: the repository was originally [[bare|bare repositories]], and I converted it to non-bare manually. But that doesn't work, because bare annex repositories use [[a different directory layout|internals/hashing]].
+
+I can understand why the hex-based layout is preferred (it's more portable), but deliberate incompatibility with the symlinks isn't very nice. I wish there was a way to upgrade the entire repository to a format that used the hex paths for symlinks, so bare and non-bare repositories used the same layout.
+"""]]
diff --git a/doc/forum/Layout_mismatch_between_symlinks_and_annex_objects/comment_2_9134154ba2ef3575c39c7848136b79a9._comment b/doc/forum/Layout_mismatch_between_symlinks_and_annex_objects/comment_2_9134154ba2ef3575c39c7848136b79a9._comment
new file mode 100644
index 0000000000..46ddb5f167
--- /dev/null
+++ b/doc/forum/Layout_mismatch_between_symlinks_and_annex_objects/comment_2_9134154ba2ef3575c39c7848136b79a9._comment
@@ -0,0 +1,21 @@
+[[!comment format=mdwn
+ username="joey"
+ subject="""comment 2"""
+ date="2015-09-09T18:18:17Z"
+ content="""
+I wish this incompatibility didn't exist either, but the transition to use
+consistently 3 letter hash directories would be too much to ask of all the
+users. And there are ways to convert a bare to non-bare that don't have
+this problem, like making a clone and using `git annex move --all --from
+origin`
+
+There's an easy way to finish your conversion from bare to non-bare.
+Just move .git/annex/objects to old-objects, and run `git annex add
+old-objects --backend=SHA256`. That will rename all the files into the right places for a
+non-bare repository, so your existing symlinks will work. You can then
+`rm -rf old-objects` to clean up.
+
+Note that method assumes that SHA256 is the backend of everything in that
+repo. If not, it gets a bit trickier; you'd need to use `git-annex
+setkey'
+"""]]
diff --git a/doc/forum/Let_watch_selectively_annex_files.mdwn b/doc/forum/Let_watch_selectively_annex_files.mdwn
new file mode 100644
index 0000000000..f56b19452a
--- /dev/null
+++ b/doc/forum/Let_watch_selectively_annex_files.mdwn
@@ -0,0 +1,27 @@
+Hello,
+
+First of all, thanks to Joey for developing git-annex, good job!
+
+I have a small feature request: when running git annex watch, new files are automatically added to the annex. It would be nice to let this depend on an attribute: add a file to annex if an attribute is set, otherwise do a regular git add.
+
+My use-case is the following: I have a repository containing documents I'm working on (mostly LaTeX), which I'd like to be regular files in git (no annex), and a bunch of extra documentation (pdfs) and images, which I'd like to go to the annex. I currently set a git-attribute (addtoannex), and use a shell script to selectively add files to annex as follows:
+
+Content of .gitattributes:
+
+    *.png addtoannex
+    *.jpg addtoannex
+
+Snippet of add script:
+
+    git check-attr addtoannex "$FILE" | grep -q ": set$"
+    if [ $? -eq 0 ]; then
+        git annex add "$FILE"
+    else
+        git add "$FILE"
+    fi
+
+It would be great if the watcher could honour an attribute.
+
+best regards,
+
+Tom
diff --git a/doc/forum/Let_watch_selectively_annex_files/comment_1_8379de87d16502d9aadf252da01e4d9a._comment b/doc/forum/Let_watch_selectively_annex_files/comment_1_8379de87d16502d9aadf252da01e4d9a._comment
new file mode 100644
index 0000000000..415b088941
--- /dev/null
+++ b/doc/forum/Let_watch_selectively_annex_files/comment_1_8379de87d16502d9aadf252da01e4d9a._comment
@@ -0,0 +1,10 @@
+[[!comment format=mdwn
+ username="http://joeyh.name/"
+ ip="4.154.3.125"
+ subject="comment 1"
+ date="2013-02-07T18:10:14Z"
+ content="""
+This has been requested a couple of times in different places.
+
+The problem with using a git attribute is it only allows basic matching on filenames. Some users would prefer to use the size of the file. So if/when this gets implemented it will probably not use a git attribute, but some other config setting.
+"""]]
diff --git a/doc/forum/Let_watch_selectively_annex_files/comment_2_2219ff6b4dc927eb2a299cd1af90aed8._comment b/doc/forum/Let_watch_selectively_annex_files/comment_2_2219ff6b4dc927eb2a299cd1af90aed8._comment
new file mode 100644
index 0000000000..d6cee26d5c
--- /dev/null
+++ b/doc/forum/Let_watch_selectively_annex_files/comment_2_2219ff6b4dc927eb2a299cd1af90aed8._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="http://joeyh.name/"
+ nickname="joey"
+ subject="comment 2"
+ date="2013-06-11T14:28:56Z"
+ content="""
+See [[tips/replacing_Sparkleshare_or_dvcs-autosync_with_the_assistant]]
+"""]]
diff --git a/doc/forum/Link_to_local_remote_instead_of_broken_link_when_possible__63__.mdwn b/doc/forum/Link_to_local_remote_instead_of_broken_link_when_possible__63__.mdwn
new file mode 100644
index 0000000000..dfe8afe2c9
--- /dev/null
+++ b/doc/forum/Link_to_local_remote_instead_of_broken_link_when_possible__63__.mdwn
@@ -0,0 +1,4 @@
+Hi,
+
+Suppose I have an annexed file whose content is stored on an extenal hard drive.
+When the hard drive is mounted, is it possible to have immediately access to this file without transfering it, by modifying the symlink to point to the file content on the hard drive instead of having a broken link?
diff --git a/doc/forum/Link_to_local_remote_instead_of_broken_link_when_possible__63__/comment_1_ce0464d5fca6ada9f1477831fd47ce09._comment b/doc/forum/Link_to_local_remote_instead_of_broken_link_when_possible__63__/comment_1_ce0464d5fca6ada9f1477831fd47ce09._comment
new file mode 100644
index 0000000000..ee6ec690e9
--- /dev/null
+++ b/doc/forum/Link_to_local_remote_instead_of_broken_link_when_possible__63__/comment_1_ce0464d5fca6ada9f1477831fd47ce09._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="http://joeyh.name/"
+ ip="209.250.56.244"
+ subject="comment 1"
+ date="2014-04-02T18:25:16Z"
+ content="""
+See [[todo/union_mounting]]
+"""]]
diff --git a/doc/forum/Links_or_actual_files__63___Confused__33__.mdwn b/doc/forum/Links_or_actual_files__63___Confused__33__.mdwn
new file mode 100644
index 0000000000..8d7ef2c172
--- /dev/null
+++ b/doc/forum/Links_or_actual_files__63___Confused__33__.mdwn
@@ -0,0 +1,21 @@
+I don't think it's a bug or a todo, just a request for clarification.
+
+So, I installed git-annex first on a Mac, and created the initial repository, and started dropping stuff into it. All fine.
+
+Created a second repository on an external (but never disconnected, also does Time Machine) drive. All fine. Everything's there.
+
+Went across the room to my Linux desktop, installed git-annex, and tried to pair it to my Mac's git-annex repo. Hit this bug, installed latest git-annex from tarball and got past that and got the pairing done. Syncing happened, all looked well...
+
+But when I look in the annex dir on that Linux machine, I now just see symlinks to the actual files in the repo's .git/annex stuff. It kinda works, but it's a bit distracting, especially if you open such a file in any desktop app and you can see it's got the target of the link (some sha-value filename.ext) rather than the sensibly-named link. Including files created and coped into that annex on that machine. (In Nautilus they all show with symlink & lock emblems.)
+
+Look again at some documentation, and apparently it's supposed to do this
+
+Which makes me wonder why it *isn't* doing it on the Mac I started the repository on?
+
+Went to another mac (laptop) and installed git-annex, and set up pairing to the first mac - as far as I can tell, doing the exact same procedure as I did under Linux. TBH through the webapp there's not much scope for doing it a different way. Files sync across...
+
+And there they are, the actual files, directly in the annex dir, not symlinks.
+
+(I am by the way confirming this at the commandline, it's not just what Finder/Nautilus may be displaying to me graphically.)
+
+Now, seeing the actual files in the annex directory is what I *prefer*; but I'm a: confused as to why the Macs are doing it that way but Linux isn't, and b: given the "how it works" link, why everything isn't using the links.
diff --git a/doc/forum/Links_or_actual_files__63___Confused__33__/comment_1_779cee2448d7070b1dd636d01296c01e._comment b/doc/forum/Links_or_actual_files__63___Confused__33__/comment_1_779cee2448d7070b1dd636d01296c01e._comment
new file mode 100644
index 0000000000..0122cf45bc
--- /dev/null
+++ b/doc/forum/Links_or_actual_files__63___Confused__33__/comment_1_779cee2448d7070b1dd636d01296c01e._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="http://joeyh.name/"
+ ip="4.154.4.51"
+ subject="comment 1"
+ date="2013-09-12T17:58:24Z"
+ content="""
+When you create a git-annex repository using the webapp, it uses [[direct_mode]]. That is not the default if you create a repository by hand at the command line, although you can enable it if you prefer it.
+"""]]
diff --git a/doc/forum/Links_or_actual_files__63___Confused__33__/comment_2_bccda88697ab7beec0b9fe9ee0230688._comment b/doc/forum/Links_or_actual_files__63___Confused__33__/comment_2_bccda88697ab7beec0b9fe9ee0230688._comment
new file mode 100644
index 0000000000..c6a851eff0
--- /dev/null
+++ b/doc/forum/Links_or_actual_files__63___Confused__33__/comment_2_bccda88697ab7beec0b9fe9ee0230688._comment
@@ -0,0 +1,18 @@
+[[!comment format=mdwn
+ username="https://www.google.com/accounts/o8/id?id=AItOawlEhzszkzOIy8-Rx8b2mcr75QcnIc6O_OA"
+ nickname="Rachel"
+ subject="webapp/direct"
+ date="2013-09-12T18:17:51Z"
+ content="""
+well, i did it using the webapp both times; i've barely scratched at the git-annex commandline so far.
+
+However... I then tried to repeat it on another linux box (running ubuntu saucy and the version of git-annex in that repo) and it worked as expected: Real files in the annex directory, ie: as I understand it, direct mode.
+
+So, went back to the first linux box (running ubuntu raring), and cleared everything off, quit and started again, doing the same thing again, and *it* did the direct mode thing itself this time.
+
+Which I think leaves me with it the way I want it. :-)
+
+My only guess is, maybe it got set up in non-direct mode when I first started using the older git-annex in the raring repos, before failing with the bad comment in ssh public key bug, at which point I switched to the downloaded-today tarball and just tried to resume. Starting from scratch with the newer version worked.
+
+I'm sure more questions to come later; am a Dropbox fugitive by way of ownCloud (won't-fix bug on version upgrades) basically trying to replicate that kind of convenience. But those are for other postings.
+"""]]
diff --git a/doc/forum/Links_or_actual_files__63___Confused__33__/comment_3_c2a9da3f03b55ff294dc0d2010380119._comment b/doc/forum/Links_or_actual_files__63___Confused__33__/comment_3_c2a9da3f03b55ff294dc0d2010380119._comment
new file mode 100644
index 0000000000..5df67c6057
--- /dev/null
+++ b/doc/forum/Links_or_actual_files__63___Confused__33__/comment_3_c2a9da3f03b55ff294dc0d2010380119._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="http://joeyh.name/"
+ ip="4.154.4.51"
+ subject="comment 3"
+ date="2013-09-12T20:22:44Z"
+ content="""
+Yep, relatively ancient (from last year) versions of the webapp didn't use direct mode by default, and have various bugs as well.
+"""]]
diff --git a/doc/forum/List_annexed_files_in_chronological_order__63__.mdwn b/doc/forum/List_annexed_files_in_chronological_order__63__.mdwn
new file mode 100644
index 0000000000..f4003c6563
--- /dev/null
+++ b/doc/forum/List_annexed_files_in_chronological_order__63__.mdwn
@@ -0,0 +1 @@
+How can I get git-annex to list the annexed files in the order they have been added?
diff --git a/doc/forum/List_annexed_files_in_chronological_order__63__/comment_1_825c950574b3a2f083554f474c9f7b50._comment b/doc/forum/List_annexed_files_in_chronological_order__63__/comment_1_825c950574b3a2f083554f474c9f7b50._comment
new file mode 100644
index 0000000000..30387e78d5
--- /dev/null
+++ b/doc/forum/List_annexed_files_in_chronological_order__63__/comment_1_825c950574b3a2f083554f474c9f7b50._comment
@@ -0,0 +1,12 @@
+[[!comment format=mdwn
+ username="joey"
+ subject="""comment 1"""
+ date="2015-01-05T16:45:08Z"
+ content="""
+This is a general git question, not a git-annex question. That is, any
+command that can list the files in a git repo, in whatever order you want,
+will also work in a git-annex repo.
+
+You probably want to use `git log --stat`, possibly with some more options
+to control the output format.
+"""]]
diff --git a/doc/forum/Local_and_remote_in_direct_mode.mdwn b/doc/forum/Local_and_remote_in_direct_mode.mdwn
new file mode 100644
index 0000000000..128d8eab61
--- /dev/null
+++ b/doc/forum/Local_and_remote_in_direct_mode.mdwn
@@ -0,0 +1,7 @@
+I have two machines (A and B ) both have a directory called "inbox"
+I want to sync these two directories. So I ran git annex assistant on machine A and set up a remote to B with **ssh** and selected **client: a repository on your computer**.
+
+The end result is that I have something that looks like a git bare repo on machine B and I have no files on A from machine B.
+
+What am I doing wrong? What shall I do to achieve my goal?
+It is kind of dropbox functionality what I want. (Which I never used for security/policy reasons.)
diff --git a/doc/forum/Local_and_remote_in_direct_mode/comment_1_45f89ebcb6092d1b2582feebc8a5e9d7._comment b/doc/forum/Local_and_remote_in_direct_mode/comment_1_45f89ebcb6092d1b2582feebc8a5e9d7._comment
new file mode 100644
index 0000000000..f7244cabc8
--- /dev/null
+++ b/doc/forum/Local_and_remote_in_direct_mode/comment_1_45f89ebcb6092d1b2582feebc8a5e9d7._comment
@@ -0,0 +1,15 @@
+[[!comment format=mdwn
+ username="http://joeyh.name/"
+ nickname="joey"
+ subject="comment 1"
+ date="2013-03-19T18:05:58Z"
+ content="""
+The \"Remote server using ssh\" option in the webapp is intended to set up a bare git repository on a server, not a non-bare git repository on a client.
+
+For what you want, both your computers need to be running the git-annex assistant.
+
+* If the computers are on the same network, pick the \"Local computer\" option on one to start a pairing process with the other.
+* If the computers are not on the same network, but can (apparently) access one-another by ssh, then you're a lucky guy. :) But the webapp doesn't cater to this unusual configuration (unless the local pairing option above works). All you need to do though, it manually add a git remote on each that points to the already existing, git-annex assistant managed repository on the other. Eg: `git remote add B ssh://machineB:/~/annex`
+* If the computers are not on the same network, and cannot access each other directly using ssh, you will need
+to use the XMPP option, and a transfer remote that both can access.
+"""]]
diff --git a/doc/forum/Local_and_remote_in_direct_mode/comment_2_90eeb2bffdb2db8032f9a0eac630ed56._comment b/doc/forum/Local_and_remote_in_direct_mode/comment_2_90eeb2bffdb2db8032f9a0eac630ed56._comment
new file mode 100644
index 0000000000..539ad1357f
--- /dev/null
+++ b/doc/forum/Local_and_remote_in_direct_mode/comment_2_90eeb2bffdb2db8032f9a0eac630ed56._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="Petter_petterson"
+ ip="89.160.15.173"
+ subject="comment 2"
+ date="2014-09-16T08:15:05Z"
+ content="""
+Doing git remote add B ssh://machineB:/~/annex still makes the repository on machineB a bare one, just try it and check git config -l | grep core.bare...
+"""]]
diff --git a/doc/forum/Local_and_remote_in_direct_mode/comment_3_859ec2b3a8e938073b2099fdc5781109._comment b/doc/forum/Local_and_remote_in_direct_mode/comment_3_859ec2b3a8e938073b2099fdc5781109._comment
new file mode 100644
index 0000000000..f39c95c037
--- /dev/null
+++ b/doc/forum/Local_and_remote_in_direct_mode/comment_3_859ec2b3a8e938073b2099fdc5781109._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="http://joeyh.name/"
+ ip="108.236.230.124"
+ subject="comment 3"
+ date="2014-09-18T17:55:21Z"
+ content="""
+Petter_petterson, I think you're mistaken about that. If you were right, that would be a massive bug in git -- nothing git-annex specific about that command after all.
+"""]]
diff --git a/doc/forum/Locating_errors_on_fsck.mdwn b/doc/forum/Locating_errors_on_fsck.mdwn
new file mode 100644
index 0000000000..3e1fcd8bc5
--- /dev/null
+++ b/doc/forum/Locating_errors_on_fsck.mdwn
@@ -0,0 +1,7 @@
+When I run `git-annex fsck` I see a lot of output passing by the screen. In the end, after many hours, I see an error count.
+
+Now, how do I see what files had problems and what is wrong with them? I can't see any option mentioning a log file or similar mechanism in the man page. Would I need to re-run the command and discard STDOUT? At this point I'm not even sure that would work.
+
+What is the proper way to run `fsck` so that I can examine the problems it finds?
+
+Thank you.
diff --git a/doc/forum/Locking_and_then_unlocking_a_file_results_in_file_changed_warning.mdwn b/doc/forum/Locking_and_then_unlocking_a_file_results_in_file_changed_warning.mdwn
new file mode 100644
index 0000000000..e7062ff29c
--- /dev/null
+++ b/doc/forum/Locking_and_then_unlocking_a_file_results_in_file_changed_warning.mdwn
@@ -0,0 +1,20 @@
+I have a git annex repository which I use to store versioned binaries in a regular git repository. We also use submodules in the repository (the sources of the versioned binaries) but the binaries are not in the submodules. This has worked great for us.
+
+Recently, there have been a number of changes (I got a new laptop, moved to fedora 20, renamed the repository, etc...) that happened at once. Now when I checkout a clean working version of the repository and then run git annex get . and then do an unlock and a lock on the file I get the "Locking this file would discard any changes you have made to it. Use 'git annex add' to stage your changes. (Or, use --force to override)" warning.
+
+I have tried to isolate this as much as possible but can't find what caused this. On a co-worker's laptop it seems to work fine so far. A clean repository didn't fix it. An old repository didn't fix it (though I recall it working there previously on this laptop). It did work and just stopped recently.
+
+    $ git annex version
+    git-annex version: 5.20140107
+    build flags: Assistant Inotify DBus Quvi TDFA
+    key/value backends: SHA256E SHA1E SHA512E SHA224E SHA384E SHA256 SHA1 SHA512 SHA224 SHA384 WORM URL
+    remote types: git gcrypt bup directory rsync web glacier hook external
+    local repository version: 5
+    supported repository version: 5
+    upgrade supported from repository versions: 0 1 2 4
+
+    $ git annex lock --debug --verbose vendors/unittest-xml-reporting-1.5.0.tar.gz  # (some names changed to protect the innocent)
+    [2014-02-05 12:54:45 IST] read: git ["--git-dir=/home/user/loppa_deuce/loppa/.git","--work-tree=/home/user/loppa_deuce/loppa","diff","--name-only","--diff-filter=T","-z","--","vendors/unittest-xml-reporting-1.5.0.tar.gz"]
+    [2014-02-05 12:54:45 IST] chat: git ["--git-dir=/home/user/loppa_deuce/loppa/.git","--work-tree=/home/user/loppa_deuce/loppa","cat-file","--batch"]
+    [2014-02-05 12:54:45 IST] read: git ["--git-dir=/home/user/loppa_deuce/loppa/.git","--work-tree=/home/user/loppa_deuce/loppa","diff","--name-only","--diff-filter=T","-z","--cached","--","vendors/unittest-xml-reporting-1.5.0.tar.gz"]
+    lock vendors/unittest-xml-reporting-1.5.0.tar.gz git-annex: Locking this file would discard any changes you have made to it. Use 'git annex add' to stage your changes. (Or, use --force to override)
diff --git a/doc/forum/Locking_and_then_unlocking_a_file_results_in_file_changed_warning/comment_1_25a04c7345f5b626aa71524603c833ed._comment b/doc/forum/Locking_and_then_unlocking_a_file_results_in_file_changed_warning/comment_1_25a04c7345f5b626aa71524603c833ed._comment
new file mode 100644
index 0000000000..84f556d654
--- /dev/null
+++ b/doc/forum/Locking_and_then_unlocking_a_file_results_in_file_changed_warning/comment_1_25a04c7345f5b626aa71524603c833ed._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="https://www.google.com/accounts/o8/id?id=AItOawmfVOIl06X7NKio-NBYZ-cfw5LFVqHMX8o"
+ nickname="Daniel"
+ subject="Hash is the same"
+ date="2014-02-05T11:13:12Z"
+ content="""
+I forgot to mention that I also checked the hash of the file to make sure that nothing changed the file and nothing changed. Just quick unlock/lock triggers it.
+"""]]
diff --git a/doc/forum/Locking_and_then_unlocking_a_file_results_in_file_changed_warning/comment_2_7146a3c69749b9b1001fffc6e7a8bcda._comment b/doc/forum/Locking_and_then_unlocking_a_file_results_in_file_changed_warning/comment_2_7146a3c69749b9b1001fffc6e7a8bcda._comment
new file mode 100644
index 0000000000..cc5e853169
--- /dev/null
+++ b/doc/forum/Locking_and_then_unlocking_a_file_results_in_file_changed_warning/comment_2_7146a3c69749b9b1001fffc6e7a8bcda._comment
@@ -0,0 +1,12 @@
+[[!comment format=mdwn
+ username="https://www.google.com/accounts/o8/id?id=AItOawmfVOIl06X7NKio-NBYZ-cfw5LFVqHMX8o"
+ nickname="Daniel"
+ subject="New git-annex package to blame"
+ date="2014-02-05T14:23:40Z"
+ content="""
+I downgraded my git-annex version, rechecked out the repository and now everything works.
+    $ git annex version
+    git-annex version: 4.20130827
+    build flags: Assistant Inotify
+
+"""]]
diff --git a/doc/forum/Locking_and_then_unlocking_a_file_results_in_file_changed_warning/comment_3_fd39e6ceffd9bf0709658c34945d8699._comment b/doc/forum/Locking_and_then_unlocking_a_file_results_in_file_changed_warning/comment_3_fd39e6ceffd9bf0709658c34945d8699._comment
new file mode 100644
index 0000000000..5b23df8707
--- /dev/null
+++ b/doc/forum/Locking_and_then_unlocking_a_file_results_in_file_changed_warning/comment_3_fd39e6ceffd9bf0709658c34945d8699._comment
@@ -0,0 +1,16 @@
+[[!comment format=mdwn
+ username="http://joeyh.name/"
+ ip="206.74.132.139"
+ subject="comment 3"
+ date="2014-02-06T17:05:45Z"
+ content="""
+Recent versions of git-annex have tried to extend the --force option to be needed in any operation that can possibly cause data loss. This includes locking a file, since that throws away any changes.
+
+Note that `git annex lock` does not check if the file is unmodified. For a few reasons including 
+
+* some backends don't include a checksum
+* it would be expensive to check a checksum
+* the file could get modified after or during a checksum check, and those modifications would be missed
+
+If you are sure you want to throw away any changes, use --force as suggested. If not, use `git annex add $file`, and assuming you're using a checksumming backend, it will notice the file has not changed and do what you want `git annex lock $file` to have done in this case.
+"""]]
diff --git a/doc/forum/Looking_at_the_webapp_on_OSX.mdwn b/doc/forum/Looking_at_the_webapp_on_OSX.mdwn
new file mode 100644
index 0000000000..b2c514d835
--- /dev/null
+++ b/doc/forum/Looking_at_the_webapp_on_OSX.mdwn
@@ -0,0 +1,18 @@
+Not logging this in the bugs section, but poking at the new webapp feature...
+
+
    +laplace:annex jtang$ git annex webapp -d
+[2012-07-26 19:02:50 IST] read: git ["--git-dir=/Users/jtang/annex/.git","--work-tree=/Users/jtang/annex","show-ref","git-annex"] 
+[2012-07-26 19:02:50 IST] read: git ["--git-dir=/Users/jtang/annex/.git","--work-tree=/Users/jtang/annex","show-ref","--hash","refs/heads/git-annex"] 
+[2012-07-26 19:02:50 IST] read: git ["--git-dir=/Users/jtang/annex/.git","--work-tree=/Users/jtang/annex","log","refs/heads/git-annex..731005d121426a38b206c4544da02cdb3b974974","--oneline","-n1"] 
+[2012-07-26 19:02:50 IST] read: git ["--git-dir=/Users/jtang/annex/.git","--work-tree=/Users/jtang/annex","log","refs/heads/git-annex..d36d8d88847decc2320f0be22892ad94a8abe594","--oneline","-n1"] 
+[2012-07-26 19:02:50 IST] read: git ["--git-dir=/Users/jtang/annex/.git","--work-tree=/Users/jtang/annex","log","refs/heads/git-annex..57bcddc14d03b61028f7002e2dabcc5181d74f3d","--oneline","-n1"] 
+[2012-07-26 19:02:50 IST] read: git ["--git-dir=/Users/jtang/annex/.git","--work-tree=/Users/jtang/annex","log","refs/heads/git-annex..372aceaf49b60ebe31cc3fe2e52ba4fbe99c134f","--oneline","-n1"] 
+[2012-07-26 19:02:50 IST] chat: git ["--git-dir=/Users/jtang/annex/.git","--work-tree=/Users/jtang/annex","cat-file","--batch"] 
+[2012-07-26 19:02:50 IST] read: git ["config","--null","--list"] 
+[2012-07-26 19:02:50 IST] call: open ["file:///Users/jtang/annex/.git/annex/webapp.html"] 
+The file /Users/jtang/annex/.git/annex/webapp.html does not exist.
+git-annex: failed to start web browser on url file:///Users/jtang/annex/.git/annex/webapp.html
+
    
+
+I would have expected the open command to open up http://localhost:port/ instead of a file on my machine. Anyway, its just an observation on the current state of the webapp feature, not expecting it to work enough for me to test it ;)
diff --git a/doc/forum/Looking_at_the_webapp_on_OSX/comment_1_68820f2f469356633c1abb18a47e0c59._comment b/doc/forum/Looking_at_the_webapp_on_OSX/comment_1_68820f2f469356633c1abb18a47e0c59._comment
new file mode 100644
index 0000000000..9aa05c8f0a
--- /dev/null
+++ b/doc/forum/Looking_at_the_webapp_on_OSX/comment_1_68820f2f469356633c1abb18a47e0c59._comment
@@ -0,0 +1,12 @@
+[[!comment format=mdwn
+ username="http://joeyh.name/"
+ ip="4.153.2.189"
+ subject="comment 1"
+ date="2012-07-26T18:17:40Z"
+ content="""
+This could happen if you already had an old version of git-annex assistant running in that repository.
+
+Try:
+
+git annex assistant --stop ; git annex webapp
+"""]]
diff --git a/doc/forum/Looking_at_the_webapp_on_OSX/comment_2_4ce86546d8a135df9cfab46b4612fa0b._comment b/doc/forum/Looking_at_the_webapp_on_OSX/comment_2_4ce86546d8a135df9cfab46b4612fa0b._comment
new file mode 100644
index 0000000000..b24956b8c7
--- /dev/null
+++ b/doc/forum/Looking_at_the_webapp_on_OSX/comment_2_4ce86546d8a135df9cfab46b4612fa0b._comment
@@ -0,0 +1,23 @@
+[[!comment format=mdwn
+ username="jtang"
+ ip="79.97.135.214"
+ subject="comment 2"
+ date="2012-07-26T20:42:37Z"
+ content="""
+The last few commits fixed the webapp startup quirk, it doesn't work yet for me as the watch command is probably failing
+
+
    +laplace:annex jtang$ git annex webapp  -d
+[2012-07-26 21:41:15 IST] read: git [\"--git-dir=/Users/jtang/annex/.git\",\"--work-tree=/Users/jtang/annex\",\"show-ref\",\"git-annex\"] 
+[2012-07-26 21:41:15 IST] read: git [\"--git-dir=/Users/jtang/annex/.git\",\"--work-tree=/Users/jtang/annex\",\"show-ref\",\"--hash\",\"refs/heads/git-annex\"] 
+[2012-07-26 21:41:15 IST] read: git [\"--git-dir=/Users/jtang/annex/.git\",\"--work-tree=/Users/jtang/annex\",\"log\",\"refs/heads/git-annex..731005d121426a38b206c4544da02cdb3b974974\",\"--oneline\",\"-n1\"] 
+[2012-07-26 21:41:15 IST] read: git [\"--git-dir=/Users/jtang/annex/.git\",\"--work-tree=/Users/jtang/annex\",\"log\",\"refs/heads/git-annex..d36d8d88847decc2320f0be22892ad94a8abe594\",\"--oneline\",\"-n1\"] 
+[2012-07-26 21:41:15 IST] read: git [\"--git-dir=/Users/jtang/annex/.git\",\"--work-tree=/Users/jtang/annex\",\"log\",\"refs/heads/git-annex..57bcddc14d03b61028f7002e2dabcc5181d74f3d\",\"--oneline\",\"-n1\"] 
+[2012-07-26 21:41:15 IST] read: git [\"--git-dir=/Users/jtang/annex/.git\",\"--work-tree=/Users/jtang/annex\",\"log\",\"refs/heads/git-annex..372aceaf49b60ebe31cc3fe2e52ba4fbe99c134f\",\"--oneline\",\"-n1\"] 
+[2012-07-26 21:41:15 IST] chat: git [\"--git-dir=/Users/jtang/annex/.git\",\"--work-tree=/Users/jtang/annex\",\"cat-file\",\"--batch\"] 
+[2012-07-26 21:41:15 IST] read: git [\"config\",\"--null\",\"--list\"] 
+git-annex: failed to start git-annex assistant
+
    
+
+it doesn't segfault or anything, it just states that the assistant command failed to start :P
+"""]]
diff --git a/doc/forum/Looking_at_the_webapp_on_OSX/comment_3_6d398a2cceff14a1b774b85ee1725073._comment b/doc/forum/Looking_at_the_webapp_on_OSX/comment_3_6d398a2cceff14a1b774b85ee1725073._comment
new file mode 100644
index 0000000000..a139537def
--- /dev/null
+++ b/doc/forum/Looking_at_the_webapp_on_OSX/comment_3_6d398a2cceff14a1b774b85ee1725073._comment
@@ -0,0 +1,12 @@
+[[!comment format=mdwn
+ username="http://joeyh.name/"
+ ip="4.153.2.189"
+ subject="comment 3"
+ date="2012-07-26T22:07:25Z"
+ content="""
+I've been improving this code path, so make sure you're current. I just made it wait for longer than the 10 seconds it was waiting before giving up. 
+
+If it still fails, try running `git-annex assistant --debug --foreground`
+in one terminal, wait for it to start up, and then run `git annex webapp` in another.
+If that still fails, check if `.git/annex/webapp.html` exists.
+"""]]
diff --git a/doc/forum/Looking_at_the_webapp_on_OSX/comment_4_5e503787a4b1d3534c5e20da5480b763._comment b/doc/forum/Looking_at_the_webapp_on_OSX/comment_4_5e503787a4b1d3534c5e20da5480b763._comment
new file mode 100644
index 0000000000..fde217e63c
--- /dev/null
+++ b/doc/forum/Looking_at_the_webapp_on_OSX/comment_4_5e503787a4b1d3534c5e20da5480b763._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="http://joeyh.name/"
+ ip="4.153.2.189"
+ subject="comment 4"
+ date="2012-07-26T22:08:53Z"
+ content="""
+Oh, wait, you're right, if the assistant process that watch forks off is segfaulting, that error message is about the best we can hope for. You can try the manual startup of the assistant to confirm it's crashing..
+"""]]
diff --git a/doc/forum/Looking_at_the_webapp_on_OSX/comment_5_c735841bc230efc61594ea013fc2902b._comment b/doc/forum/Looking_at_the_webapp_on_OSX/comment_5_c735841bc230efc61594ea013fc2902b._comment
new file mode 100644
index 0000000000..47f7be6033
--- /dev/null
+++ b/doc/forum/Looking_at_the_webapp_on_OSX/comment_5_c735841bc230efc61594ea013fc2902b._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="https://www.google.com/accounts/o8/id?id=AItOawkSq2FDpK2n66QRUxtqqdbyDuwgbQmUWus"
+ nickname="Jimmy"
+ subject="comment 5"
+ date="2012-07-27T15:53:50Z"
+ content="""
+I can confirm that the watch command is crashing when i start it up manually.
+"""]]
diff --git a/doc/forum/Looking_at_the_webapp_on_OSX/comment_6_0e489fbfc89d282e9eb47f1b814ff70c._comment b/doc/forum/Looking_at_the_webapp_on_OSX/comment_6_0e489fbfc89d282e9eb47f1b814ff70c._comment
new file mode 100644
index 0000000000..9014923867
--- /dev/null
+++ b/doc/forum/Looking_at_the_webapp_on_OSX/comment_6_0e489fbfc89d282e9eb47f1b814ff70c._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="jtang"
+ ip="79.97.135.214"
+ subject="comment 6"
+ date="2012-07-27T18:21:17Z"
+ content="""
+FYI, the webapp starts up and sends me to a web browser with the correct page ;) this is after the other issue of [[Watch command as of commit 6cecc26206c4a539999b04664136c6f785211a41 segfaults]] got fixed.
+"""]]
diff --git a/doc/forum/Lots_of_4k_symlinks.mdwn b/doc/forum/Lots_of_4k_symlinks.mdwn
new file mode 100644
index 0000000000..7f90f874db
--- /dev/null
+++ b/doc/forum/Lots_of_4k_symlinks.mdwn
@@ -0,0 +1,24 @@
+Hi, 
+
+this is a minor issue and probably there is no better solution, but nevertheless I would like to point it out and maybe discuss a little about the issue.
+
+Given that the symlinks generated by annex are pretty large in size (they point to a file named by a large hash number), ext4 is using an entire block (4K) of storage instead of [embedding the symlink into the inode][inode] itself. For the "archivist use case" of annex, this might lead to tens or hundreds of MBs of disk occupied by symlinks which actually don't add up to more than a few MBs.
+
+Here is a real world example:
+
+```
+(ins)carlos@carlos home$ du -hs music/
+56M	music/
+(ins)carlos@carlos home$ du -bhs music/
+3.3M	music/
+(ins)carlos@carlos home$ ln -s /tmp/x x
+(ins)carlos@carlos home$ du x
+0	x
+(ins)carlos@carlos home$ ln -s /tmp/xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx xx
+(ins)carlos@carlos home$ du xx
+4	xx
+```
+
+Cheers, Carlos
+
+[inode]: https://kernelnewbies.org/Linux_3.8#head-372b38979138cf2006bd0114ae97f889f67ef46a
diff --git a/doc/forum/Lots_of_4k_symlinks/comment_1_96384eaeef1d067a24678c7aa3613063._comment b/doc/forum/Lots_of_4k_symlinks/comment_1_96384eaeef1d067a24678c7aa3613063._comment
new file mode 100644
index 0000000000..10a9bbdc23
--- /dev/null
+++ b/doc/forum/Lots_of_4k_symlinks/comment_1_96384eaeef1d067a24678c7aa3613063._comment
@@ -0,0 +1,21 @@
+[[!comment format=mdwn
+ username="CandyAngel"
+ avatar="http://cdn.libravatar.org/avatar/15c0aade8bec5bf004f939dd73cf9ed8"
+ subject="comment 1"
+ date="2017-04-24T13:55:02Z"
+ content="""
+> For the \"archivist use case\" of annex, this might lead to tens or hundreds of MBs of disk occupied by symlinks which actually don't add up to more than a few MBs.
+
+    $ pwd
+    /home/sorting_annex/mnt/keyfile
+    $ du -shc *-*
+    ...
+    33M     fd0dc9d3-ad62-429e-ba1b-acc26a453ca4
+    33M     fd2fc989-bea7-4ffb-bbc8-2e34cd0e5be5
+    33M     fd79bbd4-d41e-4ea8-acc8-86437c5eed7c
+    33M     ffbd042e-f6d9-4450-9a57-8ed1086f587c
+    2.7G    total
+    $
+
+Just a bit :P (yes, that is 2.7G of symlinks *so far*)
+"""]]
diff --git a/doc/forum/Lots_of_4k_symlinks/comment_2_be9617e8cbc231069c44bc9f077ce673._comment b/doc/forum/Lots_of_4k_symlinks/comment_2_be9617e8cbc231069c44bc9f077ce673._comment
new file mode 100644
index 0000000000..1b2d91eed1
--- /dev/null
+++ b/doc/forum/Lots_of_4k_symlinks/comment_2_be9617e8cbc231069c44bc9f077ce673._comment
@@ -0,0 +1,32 @@
+[[!comment format=mdwn
+ username="joey"
+ subject="""comment 2"""
+ date="2017-05-09T21:07:11Z"
+ content="""
+You also get better seek speed with packed inodes.
+
+With default 256 byte inodes, there seems to be 59 bytes to play with.
+(Determined experimentally.) 
+
+Note that disks over 4 tb default to 32 kilobyte inodes, so probably most
+spinning hard disks these days *do* pack regular git-annex symlinks
+efficiently. (I don't have a 4 tb disk online to check this.. And I doubt
+CandyAngel was counting only the sizes of symlinks and not git repos
+or at least directory inodes to hold all the symlinks.)
+
+With a prefix like ".git/annex/objects/zX/Wx/S-s1000000000-"
+that leaves 20 bytes out of the 59 for the hash.
+
+That's not enough data to be cryptographically secure, but if
+we use SHA1 or MD5 as the base hash, it wouldn't be anyway. 15 bytes
+of hash state will base64 encode to 20 bytes. SHA1 is a 20 byte hash;
+MD5 is a 16 byte hash. So even MD5 would need to be truncated a little bit.
+Chances of (non-malicious) collision would still be small, only 256
+times as likely as a (non-malicious) MD5 collision. It could easily be made
+harder than MD5/SHA1 to maliciously collide by using truncated SHA2.
+
+(Files larger than 9.3 gb would still have too long symlinks due to the size
+field. The size field could also be omitted or encoded more efficiently,
+but omitting it would reduce git-annex's ability to not overfill disk
+and I don't think re-encoding buys enough to bother.)
+"""]]
diff --git a/doc/forum/Lots_of_4k_symlinks/comment_3_80b3a84e6d79871bbbef13d4bc87cc37._comment b/doc/forum/Lots_of_4k_symlinks/comment_3_80b3a84e6d79871bbbef13d4bc87cc37._comment
new file mode 100644
index 0000000000..d350dc2f5a
--- /dev/null
+++ b/doc/forum/Lots_of_4k_symlinks/comment_3_80b3a84e6d79871bbbef13d4bc87cc37._comment
@@ -0,0 +1,14 @@
+[[!comment format=mdwn
+ username="joey"
+ subject="""comment 3"""
+ date="2017-05-09T22:49:01Z"
+ content="""
+My analysis above assumes no subdirectories.
+
+To leave space for even a single "../" would need to drop to 13 bytes
+of hash state. 1/79228162514264337593543950336 chance of 2 files colliding.
+Not comfortable with something so worse than md5, and that still doesn't help
+when files are 2 directories deep. Droping to 11 bytes for that,
+1/1208925819614629174706176 chance is starting to get into could really
+happen territory.
+"""]]
diff --git a/doc/forum/Lots_of_4k_symlinks/comment_4_be12d26936b3502445e880be997b8877._comment b/doc/forum/Lots_of_4k_symlinks/comment_4_be12d26936b3502445e880be997b8877._comment
new file mode 100644
index 0000000000..8737c0c1b4
--- /dev/null
+++ b/doc/forum/Lots_of_4k_symlinks/comment_4_be12d26936b3502445e880be997b8877._comment
@@ -0,0 +1,14 @@
+[[!comment format=mdwn
+ username="CandyAngel"
+ avatar="http://cdn.libravatar.org/avatar/15c0aade8bec5bf004f939dd73cf9ed8"
+ subject="comment 4"
+ date="2017-05-10T09:21:34Z"
+ content="""
+> And I doubt CandyAngel was counting only the sizes of symlinks and not git repos or at least directory inodes to hold all the symlinks.)
+
+In that repository, it is only top level directories (no sub directories) and each directory in it only has symlinks (up to 8000 of them). Directories are **mkdir $(uuidgen -r)**, hence the wildcard for du.
+
+It would be including the directory size to hold all the inodes, but it definitely *isn't counting .git* as this annex spans 3 drives with 6TB of content so far. Well, 6 drives because of \"numcopies 2\" :P
+
+I will calculate this a different way and only count symlinks, when I have access to it again.
+"""]]
diff --git a/doc/forum/Lots_of_4k_symlinks/comment_5_ab3884748f3271a77ba3320f25d74414._comment b/doc/forum/Lots_of_4k_symlinks/comment_5_ab3884748f3271a77ba3320f25d74414._comment
new file mode 100644
index 0000000000..c1f4289755
--- /dev/null
+++ b/doc/forum/Lots_of_4k_symlinks/comment_5_ab3884748f3271a77ba3320f25d74414._comment
@@ -0,0 +1,25 @@
+[[!comment format=mdwn
+ username="CandyAngel"
+ avatar="http://cdn.libravatar.org/avatar/15c0aade8bec5bf004f939dd73cf9ed8"
+ subject="comment 5"
+ date="2017-05-10T12:44:08Z"
+ content="""
+    $ find -name .git -prune -o -type l | wc -l
+    1034886
+
+Just over a million symlinks.. very convenient :)
+
+    $ find -name .git -prune -o -type l -printf '%s\n' | awk '{sum+=$1} END {print sum/1024**3}'
+    195.9 # 195MB actual size
+    $ find -name .git -prune -o -type l -print0 | du -ch --files0-from=- | tail -n1
+    4.0G    total # 4GB disk usage
+
+And in comparison to my earlier comment 2 weeks ago:
+
+    $ du -shc *-* | tail -n3
+    33M     fd79bbd4-d41e-4ea8-acc8-86437c5eed7c
+    33M     ffbd042e-f6d9-4450-9a57-8ed1086f587c
+    4.1G    total
+
+So directory inode sizes are dwarfed by the 4K disk usage but ~198b actual usage of the symlinks (~96% wasted space?).
+"""]]
diff --git a/doc/forum/Lots_of_4k_symlinks/comment_6_fa72056ee788ac0f2c15bb57d9876cf6._comment b/doc/forum/Lots_of_4k_symlinks/comment_6_fa72056ee788ac0f2c15bb57d9876cf6._comment
new file mode 100644
index 0000000000..6e4c89631d
--- /dev/null
+++ b/doc/forum/Lots_of_4k_symlinks/comment_6_fa72056ee788ac0f2c15bb57d9876cf6._comment
@@ -0,0 +1,16 @@
+[[!comment format=mdwn
+ username="CandyAngel"
+ avatar="http://cdn.libravatar.org/avatar/15c0aade8bec5bf004f939dd73cf9ed8"
+ subject="comment 6"
+ date="2017-05-10T12:45:59Z"
+ content="""
+Oops,
+
+    find -name .git -prune -o -type l -printf '%s\n' | awk '{sum+=$1} END {print sum/1024**3}'
+
+should have been
+
+    find -name .git -prune -o -type l -printf '%s\n' | awk '{sum+=$1} END {print sum/1024**2}'
+
+That'll teach me to prematurely copy it :P
+"""]]
diff --git a/doc/forum/Lots_of_4k_symlinks/comment_7_bd6b81815e5888c26d7a8a96e565519e._comment b/doc/forum/Lots_of_4k_symlinks/comment_7_bd6b81815e5888c26d7a8a96e565519e._comment
new file mode 100644
index 0000000000..148778f02b
--- /dev/null
+++ b/doc/forum/Lots_of_4k_symlinks/comment_7_bd6b81815e5888c26d7a8a96e565519e._comment
@@ -0,0 +1,23 @@
+[[!comment format=mdwn
+ username="joey"
+ subject="""comment 7"""
+ date="2017-05-11T15:57:10Z"
+ content="""
+Note that the analysis in my earlier comment assumes that the
+.git/annex/objects/xx/yy/key/ directory is removed. As long as those
+per-key directories are used, the symlinks cannot possibly be made short enough
+to pack.
+
+There have been some
+other requests for that (datalad requested it because all those per-key
+directories use disk space, add to the size of the git repo, and slow down
+traversal). However, git-annex relies on those directories to prevent
+accidential rm -rf deleting the annexed objects and prevent some symlink
+following programs from editing/corrupting the annexed objects (the
+per-key directories are left mode 400 most of the time). So it would be
+fairly complicated to add a tuning that eliminated those while locking
+down the permissions some other way (eg, making the `yy` directories mode
+400 except when one or more thread/process needs to write to them), and since
+it would have to be a tuning, it would introduce a lot of conditional
+complexity into the code.
+"""]]
diff --git a/doc/forum/Lyve_Home.mdwn b/doc/forum/Lyve_Home.mdwn
new file mode 100644
index 0000000000..b9e0917695
--- /dev/null
+++ b/doc/forum/Lyve_Home.mdwn
@@ -0,0 +1,6 @@
+Lyve Home is an interesting device to store personal photos and videos. It is a 2TB hard drive that stores all your photos. It tries to replicate the files on every device you connect (computer, phone, etc.) and keeps at least two copies of each photo. The photos are not stored in the cloud, Lyve only stores metadata in the cloud.
+
+http://www.youtube.com/watch?v=sOE5uWqS6YQ
+
+This is something that could be easily made with git-annex and a small computer, like a Raspberry Pi, and a hard drive. 
+
diff --git a/doc/forum/MD5_and_MD5E_backends_no_longer_available__63__.mdwn b/doc/forum/MD5_and_MD5E_backends_no_longer_available__63__.mdwn
new file mode 100644
index 0000000000..9927743d42
--- /dev/null
+++ b/doc/forum/MD5_and_MD5E_backends_no_longer_available__63__.mdwn
@@ -0,0 +1,5 @@
+I set the backend to MD5E with git config, but when I try to import files I get this error:
+
+    git-annex: unknown backend MD5E
+
+MD5 doesn't work either. Have these been removed as backend options? They are listed [here](http://git-annex.branchable.com/backends/).
diff --git a/doc/forum/MD5_and_MD5E_backends_no_longer_available__63__/comment_1_e21fa57a2f2dc4e99737877c6a48cbfe._comment b/doc/forum/MD5_and_MD5E_backends_no_longer_available__63__/comment_1_e21fa57a2f2dc4e99737877c6a48cbfe._comment
new file mode 100644
index 0000000000..549b1153d0
--- /dev/null
+++ b/doc/forum/MD5_and_MD5E_backends_no_longer_available__63__/comment_1_e21fa57a2f2dc4e99737877c6a48cbfe._comment
@@ -0,0 +1,11 @@
+[[!comment format=mdwn
+ username="joey"
+ subject="""comment 1"""
+ date="2015-07-15T15:59:10Z"
+ content="""
+These have not been removed.
+
+Since they were first added in git-annex version 5.20150205,
+the most likely reason for your problem is if you're using an older version
+of git-annex than that. `git annex version` will tell you.
+"""]]
diff --git a/doc/forum/MD5_and_MD5E_backends_no_longer_available__63__/comment_2_5025d16f59f314711e49537e542b7c83._comment b/doc/forum/MD5_and_MD5E_backends_no_longer_available__63__/comment_2_5025d16f59f314711e49537e542b7c83._comment
new file mode 100644
index 0000000000..0756c3f3ae
--- /dev/null
+++ b/doc/forum/MD5_and_MD5E_backends_no_longer_available__63__/comment_2_5025d16f59f314711e49537e542b7c83._comment
@@ -0,0 +1,7 @@
+[[!comment format=mdwn
+ username="ghen1"
+ subject="comment 2"
+ date="2015-07-17T09:52:02Z"
+ content="""
+Yes, I am using an older version. Thank you for the help.
+"""]]
diff --git a/doc/forum/Make_whereis_output_more_compact.mdwn b/doc/forum/Make_whereis_output_more_compact.mdwn
new file mode 100644
index 0000000000..5503e716f3
--- /dev/null
+++ b/doc/forum/Make_whereis_output_more_compact.mdwn
@@ -0,0 +1,13 @@
+Hi there,
+
+first of all, great job done on annex! I think I finally found what I was looking for to organize all my data.
+
+My question: when I do git annex whereis Music, I get a detailed list of every single song and its remote locations. I have tens of thousands of songs, so just typing git annex whereis Music gives me way too much information. 
+Here is my proposition: in my case, every file of the Music folder resides in the same remote. So, wouldn't it be better to show just one line of output telling me that Music is on remote bla and blub, and then I know that all files in Music are in remotes bla and blub?
+
+Is that possible now already or would that be a feature request? What do you think?
+
+Cheers,
+Moritz.
+
+PS: I know that I can specify filters, e.g. git annex whereis --in bla --and --not --in blub. That is great!
diff --git a/doc/forum/Making_Firefox_not_dereference_symlinks_on_open.mdwn b/doc/forum/Making_Firefox_not_dereference_symlinks_on_open.mdwn
new file mode 100644
index 0000000000..5680cbd026
--- /dev/null
+++ b/doc/forum/Making_Firefox_not_dereference_symlinks_on_open.mdwn
@@ -0,0 +1,3 @@
+Firefox has the nasty habit that it will force-dereference symlinks when locally opening files (i. e., opening an annexed document will cause it to be opened in .git/annex/objects/…). Since this will break relative links within HTML files, this would make Firefox pretty useless when working with a git annex containing HTML files. (Apparently this behavior is [desired](https://bugzilla.mozilla.org/show_bug.cgi?id=803999) upstream and might not be fixed.)
+
+Seems I’m not the only one who would like to work with annexed HTML files, though. On the [Debian bugtracker](https://bugs.debian.org/cgi-bin/bugreport.cgi?bug=691099), another user shared a handy shim which can be used in LD_PRELOAD and which will force Firefox to open symlinks in-place. Thought I’d share this here in case it’s of use to anyone.
diff --git a/doc/forum/Making_Firefox_not_dereference_symlinks_on_open/comment_1_a7b092f2291fa515279cf7dce23df20d._comment b/doc/forum/Making_Firefox_not_dereference_symlinks_on_open/comment_1_a7b092f2291fa515279cf7dce23df20d._comment
new file mode 100644
index 0000000000..b4addb92dc
--- /dev/null
+++ b/doc/forum/Making_Firefox_not_dereference_symlinks_on_open/comment_1_a7b092f2291fa515279cf7dce23df20d._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="zardoz"
+ ip="78.48.163.229"
+ subject="comment 1"
+ date="2014-07-31T11:43:16Z"
+ content="""
+Sorry, it escaped my attention there’s a dedicated tips forum. Maybe this should be moved there.
+"""]]
diff --git a/doc/forum/Making_a_git-annexy_symlink___34__known__34____63__.mdwn b/doc/forum/Making_a_git-annexy_symlink___34__known__34____63__.mdwn
new file mode 100644
index 0000000000..8d298ae830
--- /dev/null
+++ b/doc/forum/Making_a_git-annexy_symlink___34__known__34____63__.mdwn
@@ -0,0 +1,7 @@
+Currently, if you git-add a symlink copied from another git-annex, git-annex will 'fix' it so it points to where the files would be in its new annex object store, but doesn't create the corresponding file for the key on the git-annex branch, so git-annex doesn't actually "know" about the file.
+
+This means running *git annex reinject --known* won't reinject the content for the symlink (e.g. "Not known content; skipping").
+
+I tried running git-annex-fsck hoping it would create the file (with the information that 0 copies exist) but it doesn't do that..?
+
+Any advice on how to go about resolving this? Preferably a "lightweight" way as this repository has a lot of such transplanted symlinks.. :)
diff --git a/doc/forum/Making_a_git-annexy_symlink___34__known__34____63__/comment_1_e04edbe950a41887c4c11a226ca77dce._comment b/doc/forum/Making_a_git-annexy_symlink___34__known__34____63__/comment_1_e04edbe950a41887c4c11a226ca77dce._comment
new file mode 100644
index 0000000000..ad898c5e89
--- /dev/null
+++ b/doc/forum/Making_a_git-annexy_symlink___34__known__34____63__/comment_1_e04edbe950a41887c4c11a226ca77dce._comment
@@ -0,0 +1,20 @@
+[[!comment format=mdwn
+ username="joey"
+ subject="""comment 1"""
+ date="2016-05-10T17:04:20Z"
+ content="""
+fsck sees that the (lack of) location
+log accurately states that the content is not present, and so avoids
+writing to the log.
+
+If fsck always wrote to the log when content was not present, running
+fsck in sparse repos would bloat the location logs unncessarily.
+
+But I suppose that it makes sense for fsck to notice that the key
+was not known and write to the log in this particular case.
+I've gone ahead and made that change.
+
+It's not clear to me though, why this workflow of copying over symlinks,
+and adding them and reinjecting is better than just moving over content and
+adding it.
+"""]]
diff --git a/doc/forum/Making_a_git-annexy_symlink___34__known__34____63__/comment_2_de77cc2bee45706c4bbe407d8846778e._comment b/doc/forum/Making_a_git-annexy_symlink___34__known__34____63__/comment_2_de77cc2bee45706c4bbe407d8846778e._comment
new file mode 100644
index 0000000000..1d84df8825
--- /dev/null
+++ b/doc/forum/Making_a_git-annexy_symlink___34__known__34____63__/comment_2_de77cc2bee45706c4bbe407d8846778e._comment
@@ -0,0 +1,13 @@
+[[!comment format=mdwn
+ username="CandyAngel"
+ subject="comment 2"
+ date="2016-05-10T17:44:38Z"
+ content="""
+It protects against adding in corrupted files without noticing.
+
+If I move the symlink as a real file, the receiving git-annex will rehash it and, if it is corrupted, effectively change the symlink to a corrupt file with no way to tell this has occured.
+
+This method leaves the transplanted symlink broken (and may be fixed by a reinject of a good copy of the file from another drive) and the corrupt content would be left behind by the reinject. This makes it very obvious something has happened.
+
+Thank you for making these changes to support my use case. I really hope it doesn't break anyone else's!
+"""]]
diff --git a/doc/forum/Making_git-annex_a_self-funded_project__63__.mdwn b/doc/forum/Making_git-annex_a_self-funded_project__63__.mdwn
new file mode 100644
index 0000000000..29eddaa0cb
--- /dev/null
+++ b/doc/forum/Making_git-annex_a_self-funded_project__63__.mdwn
@@ -0,0 +1,10 @@
+With the fundraiser having met its stretch goal of $15,000, I was thinking of how to keep git-annex going beyond another year.  What about selling stuff?  Not just merch (though being able to buy stickers would be neat), but useful things that would complement the software, like selling hardware for a small remote that you could plug into your home network, or even set up at a friend's house to have online, but offsite.
+
+
+Something like a CubieBoard2 and an HDD for annexes preloaded with all the necessary software and a git-annex apt repo for keeping it updated.  Everything would come in a nice little enclosure with a git-annex logo on it; you plug in the power, the ethernet, and boom it's ready to go, with no terminal magic (but it's there if you want it!).  You could create them on-demand without having to keep much of a stock (except maybe the enclosure), so there's practically no risk.
+
+
+
+Aesthetically, I was thinking of something that exposed as little of the insides as possible.  Basically just power, ethernet, and USB ports.
+
+Maybe material for another Kickstarter?
diff --git a/doc/forum/Making_git-annex_a_self-funded_project__63__/comment_1_4a1ba95b7231ba973ddb672d2419e28c._comment b/doc/forum/Making_git-annex_a_self-funded_project__63__/comment_1_4a1ba95b7231ba973ddb672d2419e28c._comment
new file mode 100644
index 0000000000..03ddae4832
--- /dev/null
+++ b/doc/forum/Making_git-annex_a_self-funded_project__63__/comment_1_4a1ba95b7231ba973ddb672d2419e28c._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="GLITTAH"
+ ip="94.126.178.1"
+ subject="comment 1"
+ date="2013-07-27T01:56:47Z"
+ content="""
+Other ideas welcome!
+"""]]
diff --git a/doc/forum/Making_git-annex_a_self-funded_project__63__/comment_2_7c476ae92e63c991f229708678874ca2._comment b/doc/forum/Making_git-annex_a_self-funded_project__63__/comment_2_7c476ae92e63c991f229708678874ca2._comment
new file mode 100644
index 0000000000..70e470c85c
--- /dev/null
+++ b/doc/forum/Making_git-annex_a_self-funded_project__63__/comment_2_7c476ae92e63c991f229708678874ca2._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="http://joeyh.name/"
+ ip="4.152.246.110"
+ subject="comment 2"
+ date="2013-07-28T00:21:39Z"
+ content="""
+A few people have suggested that idea. I don't know if I want to get into the HW business.
+"""]]
diff --git a/doc/forum/Making_git-annex_less_necessary.mdwn b/doc/forum/Making_git-annex_less_necessary.mdwn
new file mode 100644
index 0000000000..086c051398
--- /dev/null
+++ b/doc/forum/Making_git-annex_less_necessary.mdwn
@@ -0,0 +1,5 @@
+http://git-annex.branchable.com/walkthrough/ says "Git wants to first stage the entire contents of the file in its index. That can be slow for big files (sorta why git-annex exists in the first place)."
    
+What is git doing that git-annex isn't, other than copying the file to .git/objects rather than just moving it to .git/annex/objects, prepending it with "blob"+length, and compressing it? If git were changed to store the "blob"+length as part of the object filename rather than as part of the object file content, have a config option to use uncompressed objects for large files (and not try to pack them when creating pack files), and were used on a filesystem such as zfs or btrfs which does COW so the copy would be as fast as a move, then what speed advantage would git-annex still have over git? I realize git-annex has more features than just big file handling, and has the worm backend for even faster handling, but I'm just talking about the case with the default sha backend.
    
+Have such changes been proposed for git? It seems that for anybody already familiar with the git codebase, adding the config option for uncompressed objects and moving the storage location for "blob"+length would be easy changes to make, and I see no downside to them. It wouldn't break backwards compatibility because the object filename being hash."blob".length rather than just hash would indicate that the new object format is in use, and a ".raw" filename extension could be used for uncompressed objects (or more sensibly, in the new format, no additional extension for uncompressed, and ".compressed" for compressed).
    
+This would also eliminate the need for a git-annex object store separate from the git object store, and the complexities involved with having them separate, and the need for symlinks, and the complexities they cause. I don't think that relying on COW for speed is unreasonable once btrfs becomes the default in major Linux distros (the bsds already have zfs and hammerfs); right now part of what git-annex is doing is just working around the functional deficiency of non-COW filesystems.
    
+P.S. I recommend a "plain" option for the page type when submitting comments on your wiki, so I don't have to put HTML line break markup at the end of my lines.
diff --git a/doc/forum/Making_git-annex_less_necessary/comment_1_03faaa3866778d24cd03887b85dc9954._comment b/doc/forum/Making_git-annex_less_necessary/comment_1_03faaa3866778d24cd03887b85dc9954._comment
new file mode 100644
index 0000000000..3396643f66
--- /dev/null
+++ b/doc/forum/Making_git-annex_less_necessary/comment_1_03faaa3866778d24cd03887b85dc9954._comment
@@ -0,0 +1,12 @@
+[[!comment format=mdwn
+ username="http://joeyh.name/"
+ ip="4.153.2.245"
+ subject="comment 1"
+ date="2012-05-08T18:22:12Z"
+ content="""
+git's code base makes lots of assumptions hardcoding the size of the hash, etc. (grep its source for magic numbers 40 and 42...) I'd like to see git get parameratised hashes. SHA1 insecurity may evenutally push it in that direction. However, when I asked the git developers about this at the Gittogether last year, there were several ideas floated that would avoid parameterisation, and a lot of good thoughts about problems parameterised hashes would cause.
+
+Moving data into git proper would still leave the problems unique to large data of not being able to store it all on every clone. Which means a git-annex like thing is needed to track where the data resides and move it around.
+
+(BTW, in markdown, you separate paragraphs with blank lines. Like in email.)
+"""]]
diff --git a/doc/forum/Making_git-annex_less_necessary/comment_2_2db02a94dffd525885c9d7fc6c5fa464._comment b/doc/forum/Making_git-annex_less_necessary/comment_2_2db02a94dffd525885c9d7fc6c5fa464._comment
new file mode 100644
index 0000000000..4b1dbd271b
--- /dev/null
+++ b/doc/forum/Making_git-annex_less_necessary/comment_2_2db02a94dffd525885c9d7fc6c5fa464._comment
@@ -0,0 +1,12 @@
+[[!comment format=mdwn
+ username="https://me.yahoo.com/a/IAg3idYGk.joxsJb2WCxl20gig.0.8hS#d5165"
+ nickname="Kelly"
+ subject="comment 2"
+ date="2012-05-10T15:01:15Z"
+ content="""
+I think my comment a couple days ago got caught in the spam filter, so I'm reposting. 
+What were the ideas to avoid parameterisation? What were the problems of parameterisation, other than just the current hardcoded assumptions?
+
+Speaking of hash insecurity, http://static.usenix.org/events/hotos03/tech/full_papers/henson/henson_html/node8.html says compare-by-hash is a bad idea. As I understand, git doesn't have an option of verifying content matches when the hash matches when adding data to the object store (like zfs's \"dedup=verify\" option, which you can use even when using sha256), because the assumption is that the risk of collision (or at least just the risk of accidental collision) is negligible. Would it be worthwhile to add this option to git-annex?
+
+"""]]
diff --git a/doc/forum/Making_git-annex_less_necessary/comment_3_429ec656e0ac02f98843f8d7f3c02d6a._comment b/doc/forum/Making_git-annex_less_necessary/comment_3_429ec656e0ac02f98843f8d7f3c02d6a._comment
new file mode 100644
index 0000000000..41b7570e12
--- /dev/null
+++ b/doc/forum/Making_git-annex_less_necessary/comment_3_429ec656e0ac02f98843f8d7f3c02d6a._comment
@@ -0,0 +1,11 @@
+[[!comment format=mdwn
+ username="https://me.yahoo.com/a/IAg3idYGk.joxsJb2WCxl20gig.0.8hS#d5165"
+ nickname="Kelly"
+ subject="comment 2"
+ date="2012-05-09T01:22:13Z"
+ content="""
+What were the ideas to avoid parameterisation? What were the problems of parameterisation, other than just the current hardcoded assumptions?
+
+Speaking of hash insecurity, http://static.usenix.org/events/hotos03/tech/full_papers/henson/henson_html/node8.html says compare-by-hash is a bad idea. As I understand, git doesn't have an option of verifying content matches when the hash matches when adding data to the object store (like zfs's \"dedup=verify\" option, which you can use even when using sha256), because the assumption is that the risk of collision (or at least just the risk of accidental collision) is negligible. Would it be worthwhile to add this option to git-annex?
+
+"""]]
diff --git a/doc/forum/Making_git-annex_less_necessary/comment_4_384813dd022dfd9c1ef14e0f1479a123._comment b/doc/forum/Making_git-annex_less_necessary/comment_4_384813dd022dfd9c1ef14e0f1479a123._comment
new file mode 100644
index 0000000000..40f0e02b0a
--- /dev/null
+++ b/doc/forum/Making_git-annex_less_necessary/comment_4_384813dd022dfd9c1ef14e0f1479a123._comment
@@ -0,0 +1,18 @@
+[[!comment format=mdwn
+ username="http://christian.amsuess.com/"
+ nickname="chrysn"
+ subject="comment 2"
+ date="2012-05-11T17:40:20Z"
+ content="""
+from my layman's standpoint, i think it would be feasible. i've suggested this previously, but not pushed it too much. quoting from [[my user page|users/chrysn]]:
+
+* **would like git-annex to**: not be required any more as git itself learns to use cow filesystems to avoid abundant disk usage and gets better with sparser checkouts (git-annex might then still be a simpler tool that watches over what can be safely dropped for a sparser checkout)
+
+*concerning hash sizes or parameterized hashes*: the problems with hash sizes could be avoided if instead of putting the objects in the \"normal\" object dir, barefiles would be managed in a similar way as packs are. when a new files gets added, they'd be cow-copied to ``.git/objects/bare/${HA}/${SH}``, and ``.git/objects/bareprefix/${HA}/${SH}`` would contain the \"blob ${SIZE}\0\" prefix that gets concatenated to the object body to form the object itself.
+
+(maybe it'd even be sufficient to *just store the size* in the bareprefix, as all those objects would be blobs, but then again, some flexibility won't hurt.)
+
+if the *pack file format* is flexible enough, the bareprefix files can get packed too. for the adventerous user who modifies bigfiles, the pack file mechanisms should be made aware of their presence, and be able to store deltas between them. the operations for applying those deltas would be difficult to optimize, and could be added at a later stage. a typical example could be storing a pdf file -- the pdf file format is designed for appending, so chances are the new version is just the old version plus several k at the end.
+
+neither of that would affect git's *wire protocol*, so no compatibility problems. (it would be advisable to find a reasonable way to do sparse checkouts, though; something like \"server, pack and send your master, but make it sparse and don't include blobs >1mb\").
+"""]]
diff --git a/doc/forum/Malicious_autoenabled_remotes.mdwn b/doc/forum/Malicious_autoenabled_remotes.mdwn
new file mode 100644
index 0000000000..7e48e5d502
--- /dev/null
+++ b/doc/forum/Malicious_autoenabled_remotes.mdwn
@@ -0,0 +1,9 @@
+I've been trying to figure out whether git-annex can be used to make a user unknowingly download data from a malicious source. The general question here is, assuming a git/git-annex server that I can fully trust to be safe and secure (let's call it `trustedserver`):
+
+*Is it possible, when performing (for example) `git clone git@trustedserver:user/repo && cd repo && git annex init` for annex to set up and enable a remote that is **not** on `trustedserver`?*
+
+I'm trying to imagine a scenario where someone with access to the repository (a person who I share files with) can set up a remote to a different server (e.g., `badremote`), set it to `autoenable=true`, and sync changes. Would this enable the other user to put files on `badremote` that are not on `trustedserver` but are tracked by annex? More importantly, if this happens and I perform a `git clone` → `git annex init` → `git annex sync --content`, would I be downloading files from `badremote` without specifically enabling it?
+
+
+Thanks,
+Achilleas
diff --git a/doc/forum/Malicious_autoenabled_remotes/comment_1_c82464f40eb1442aadd0f1e89b72f41e._comment b/doc/forum/Malicious_autoenabled_remotes/comment_1_c82464f40eb1442aadd0f1e89b72f41e._comment
new file mode 100644
index 0000000000..0a1d230ed8
--- /dev/null
+++ b/doc/forum/Malicious_autoenabled_remotes/comment_1_c82464f40eb1442aadd0f1e89b72f41e._comment
@@ -0,0 +1,42 @@
+[[!comment format=mdwn
+ username="joey"
+ subject="""comment 1"""
+ date="2017-05-09T20:45:17Z"
+ content="""
+`git-annex init` will try to auto-enable special remotes that have
+been configured with autoenable=true.
+
+So if someone can push to the repository on trustedserver, they can
+set up such a special remote and cause your later clones of it to enable
+the special remote. Sync will then push content to their special remote.
+They could also check in additional annexed file to the git repository,
+and put their contents on their special remote, and sync would then
+download the contents from there.
+
+Of course, someone who can do this has to have write access to the
+git repository on trustedserver, and if they can write to the git repository,
+they can also send annexed file to there, unless you've prevented that
+somehow.
+
+I had not really considered the autoenable=true as a potential security
+problem, so it's good to think about it that way. I don't know if we have a
+real security problem here though. It seems to rely on the attacker
+having write access to the trustedserver so far.
+
+I suppose the attacker could instead convince you to pull from a clone that
+they control, and after you've pulled once, clones made from your
+repository (or trustedserver after you push to there) will then autoenable
+their special remote unexpectedly. Perhaps the goal then is to get git
+annex sync to unexpectedly send file contents there, so they can collect
+all your annexed files. Pulling from their repository once thus turns into
+sending them all your annexed files going forward.
+
+So I am starting to see this as a security problem.. 
+
+Note that pulling from someone untrusted can also change other settings in
+the git-annex branch (since it's automatically merged), which can probably
+screw up the repository fairly well in other ways, like setting numcopies
+to 0 and messing with preferred content expressions such that git-annex
+wants to drop all files, or copy files to repositories where you don't want
+them to go, etc.
+"""]]
diff --git a/doc/forum/Malicious_autoenabled_remotes/comment_2_37456c0a159453fa984c5a003578d1eb._comment b/doc/forum/Malicious_autoenabled_remotes/comment_2_37456c0a159453fa984c5a003578d1eb._comment
new file mode 100644
index 0000000000..6de3506f86
--- /dev/null
+++ b/doc/forum/Malicious_autoenabled_remotes/comment_2_37456c0a159453fa984c5a003578d1eb._comment
@@ -0,0 +1,21 @@
+[[!comment format=mdwn
+ username="achilleas.k@14be77d42a1252fab5ec9dbf4e5ea03c5833e8c8"
+ nickname="achilleas.k"
+ avatar="http://cdn.libravatar.org/avatar/ed6c67c4d8e6c6850930e16eaf85a771"
+ subject="comment 2"
+ date="2017-05-22T14:40:21Z"
+ content="""
+Hey, thanks for the feedback and your thoughts. Should have gotten back to you sooner on this.
+
+I wanted to share with you my thoughts about getting around this issue, from the point of view of the `trustedserver` administrator, and get your input on this.
+
+I want to run a server that uses git and git annex for data storage. I want users of this server to feel safe that when they clone a repository and sync content, they're not pulling things from an untrusted server. I was thinking of modifying annex configurations serverside, perhaps as a *post-receive* hook. The idea would be to go through the remotes on the serveride, bare git repository, and mark all unknown (ssh, rsync, etc) remotes as a `dead`.
+
+Would this cause any issues for the receiver or the sender? Other than potentially making files for the receiver unavailable (which is what I want), would it possibly put the repository in a state where the original sender can't push more changes, because of a disagreement between configurations?
+
+I've played around with the idea a bit and I think the idea is pretty safe, but I might be missing something.
+
+Thanks!
+
+Achilleas
+"""]]
diff --git a/doc/forum/Malicious_autoenabled_remotes/comment_3_608213f5d0df482b731ae1502cdd87af._comment b/doc/forum/Malicious_autoenabled_remotes/comment_3_608213f5d0df482b731ae1502cdd87af._comment
new file mode 100644
index 0000000000..702be5fbb4
--- /dev/null
+++ b/doc/forum/Malicious_autoenabled_remotes/comment_3_608213f5d0df482b731ae1502cdd87af._comment
@@ -0,0 +1,20 @@
+[[!comment format=mdwn
+ username="joey"
+ subject="""comment 3"""
+ date="2017-05-24T17:27:20Z"
+ content="""
+The server can certainly do filtering or blocking of changes to the
+git-annex branch to prevent this kind of abuse.
+
+Marking a repository as dead will indeed prevent it from being
+auto-enabled. It will not cause later synchronisation problems. It seems
+like a perhaps too big hammer though. Cloning from such a server, and then
+pushing back to it would make your clone be marked as dead on the next
+pull!
+
+And marking dead doesn't prevent malicious changes to preferred
+content settings etc.
+
+Filtering in the `pre-receive` hook should be very doable. See
+[[internals]] for the git-annex branch documentation.
+"""]]
diff --git a/doc/forum/Managing_a_large_number_of_files_archived_on_many_pieces_of_read-only_medium___40__E.G._DVDs__41__.mdwn b/doc/forum/Managing_a_large_number_of_files_archived_on_many_pieces_of_read-only_medium___40__E.G._DVDs__41__.mdwn
new file mode 100644
index 0000000000..acd9066ec8
--- /dev/null
+++ b/doc/forum/Managing_a_large_number_of_files_archived_on_many_pieces_of_read-only_medium___40__E.G._DVDs__41__.mdwn
@@ -0,0 +1,118 @@
+I have come up with a moderately complex solution to a particular use case that I have and am posting it here in case it is useful to someone else, and to get suggestions on how to improve it.
+
+#The problem:
+
+I have a large number of files that are accessed infrequently and stored off-line on DVD-Rs.  I need to keep track of which files are on which disc so that when I want a file I can find it.
+
+#The solution:
+
+I currently keep a text file to track which files are on which discs.  I would like to organize all the files in a proper filesystem using git annex, allowing me better organization and the ability to keep some smaller related files online near the annexed large files.
+
+#Requirements:
+
+1) Easily locate the DVD-R containing any specific offline file
+
+This is easily taken care of with git annex whereis
+
+2) Automatically de-duplicate stored files with the same contents
+
+This is taken care of with one of the hash backends (E.G. SHA256)
+
+3) The DVD-Rs still need to be usable without git or git-annex (E.G. The stored files should retain their normal human readable names)
+
+This requirement rules out dir and rsync special remotes, they store the files named according to their hash.  I have settled on making each disc a separate repo which will satisfy this requirement.
+
+
+#Future goals:
+
+4) Easily incorporate the current DVD-Rs into the new system
+
+I haven't found a way to fulfill this goal yet.  I have some convoluted ideas, but nothing so easy as mount disc, run git annex command.
+
+
+#The solution in detail
+Suppose you have the following tree:
+
+
    +~/mainrepo/thing1/file1.bin
+~/mainrepo/thing1/description1.txt
+~/mainrepo/thing2/file2.bin
+~/mainrepo/thing2/description2.txt
+
    
+
+You want to store thing1 on disc1 and thing2 on disc2, but you'd like to keep the descriptions online because they are small and useful for figuring out which thing you want later.
+
+1) Create the main repo and annex the files:
+
+
    +cd ~/mainrepo
+git init
+git annex init mainrepo
+git annex add .
+git commit -m 'added files'
+
    
+
+2) Create two new unrelated repos and populate them with their respective data and annex:
+
+
    +cd /tmp
+mkdir disc1repo disc2repo
+cd disc1repo
+cp ~/mainrepo/thing1/* .
+git init
+git annex init disc1
+git annex add .
+git commit -m 'added files'
+cd ../disc2repo
+cp ~/mainrepo/thing2/* .
+git init
+git annex init disc2
+git annex add .
+git commit -m 'added files'
+
    
+
+3) This is optional, but after annexing the files in these new repos, I replace the symlinks pointing into to the .git/annex/objects/ directory with hard links.  This makes the DVD-Rs usable from operating systems that can't deal with symlinks.  (mkisofs handles hard links correctly)
+
+
    +cd /tmp
+find disc1repo/ disc2repo/ -type l -execdir sh -c "mv -iv {} {}.symlink && ln -L {}.symlink {} && rm {}.symlink" \;
+
    
+
+4) Burn these repos onto DVD-Rs:
+
+
    +cd /tmp
+#make isos
+mkisofs -volid disc1 -rational-rock -joliet -joliet-long -udf -full-iso9660-filenames -iso-level 3 -o disc1.iso disc1repo/
+mkisofs -volid disc2 -rational-rock -joliet -joliet-long -udf -full-iso9660-filenames -iso-level 3 -o disc2.iso disc2repo/
+#burn the isos (untested command)
+cdrecord -v -dao disc1.iso
+cdrecord -v -dao disc2.iso
+
    
+
+5) Mount the DVD-Rs and add as a remote and fetch, then drop from the mainrepo:
+
+
    +cd ~/mainrepo
+#disc1
+mount /mnt/cdrom
+git remote add disc1 /mnt/cdrom
+git fetch disc1
+git annex drop thing1/thing1.bin
+umount /mnt/cdrom
+#disc2
+mount /mnt/cdrom
+git remote add disc2 /mnt/cdrom
+git fetch disc2
+git annex drop thing2/thing2.bin
+umount /mnt/cdrom
+
    
+
+6) Enjoy!  You can now find out what disc things are on simply using git annex whereis, and you can git annex get them or simply use them directly from the disc.
+
+
+I'd appreciate any comments and helpful suggestions.  Especially how to simplify the process or easily integrate all the things I already have stored on discs.
+
+Maybe it would be possible to create a special remote using the hooks for the DVD-Rs.
+
+Even though it is a bit tedious and complicated, the current process could be automated using a script.
diff --git a/doc/forum/Managing_a_large_number_of_files_archived_on_many_pieces_of_read-only_medium___40__E.G._DVDs__41__/comment_10_a061d300b718ad943c940e122cc57220._comment b/doc/forum/Managing_a_large_number_of_files_archived_on_many_pieces_of_read-only_medium___40__E.G._DVDs__41__/comment_10_a061d300b718ad943c940e122cc57220._comment
new file mode 100644
index 0000000000..488e9aa7bf
--- /dev/null
+++ b/doc/forum/Managing_a_large_number_of_files_archived_on_many_pieces_of_read-only_medium___40__E.G._DVDs__41__/comment_10_a061d300b718ad943c940e122cc57220._comment
@@ -0,0 +1,23 @@
+[[!comment format=mdwn
+ username="Steve"
+ ip="92.104.175.136"
+ subject="Web remote works"
+ date="2012-11-25T22:30:46Z"
+ content="""
+Thanks for the suggestion Joey, I found a way to make the web remotes work for adding the files from existing discs.  I wound up adding a symlink farm to the repo with a link for each disk pointing at the mount point.  This way when I try to retrieve a file, I see the URL which contains the name of the disc:
+
+    $ git annex get bigfile.bin
+    get bigfile.bin (from web...) 
+    curl: (37) Couldn't open file /var/tmp/repo/storage/dvd13/bigfile.bin
+    
+      Unable to access these remotes: web
+    
+      Try making some of these repositories available:
+            00000000-0000-0000-0000-000000000001 -- web
+    failed
+    git-annex: get: 1 failed
+
+It took me a while as the version of git-annex in portage was rather old and I just didn't get around to updating git-annex on Gentoo for while.  If anybody wants to get git-annex 3.20121112 running under Gentoo I detailed the process I used at 
+
+Now I'll have to try out the assistant!  (Though I didn't get the webapp to compile due to a shakespeare-js error)
+"""]]
diff --git a/doc/forum/Managing_a_large_number_of_files_archived_on_many_pieces_of_read-only_medium___40__E.G._DVDs__41__/comment_11_76529080054407570611b4357ce4f3ed._comment b/doc/forum/Managing_a_large_number_of_files_archived_on_many_pieces_of_read-only_medium___40__E.G._DVDs__41__/comment_11_76529080054407570611b4357ce4f3ed._comment
new file mode 100644
index 0000000000..8823ecf416
--- /dev/null
+++ b/doc/forum/Managing_a_large_number_of_files_archived_on_many_pieces_of_read-only_medium___40__E.G._DVDs__41__/comment_11_76529080054407570611b4357ce4f3ed._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="Steve"
+ ip="92.104.175.136"
+ subject="Bad ebuilds"
+ date="2012-12-01T02:45:37Z"
+ content="""
+Just an update for anybody that used the ebuilds I created from the link above.  They did not create the git-annex-shell symlink which can cause git-annex to start ignoring your remotes.  More details and fixed ebuilds are now on the page.
+"""]]
diff --git a/doc/forum/Managing_a_large_number_of_files_archived_on_many_pieces_of_read-only_medium___40__E.G._DVDs__41__/comment_12_9acf5ce41a023f3848a51891cceeb51b._comment b/doc/forum/Managing_a_large_number_of_files_archived_on_many_pieces_of_read-only_medium___40__E.G._DVDs__41__/comment_12_9acf5ce41a023f3848a51891cceeb51b._comment
new file mode 100644
index 0000000000..aa742361da
--- /dev/null
+++ b/doc/forum/Managing_a_large_number_of_files_archived_on_many_pieces_of_read-only_medium___40__E.G._DVDs__41__/comment_12_9acf5ce41a023f3848a51891cceeb51b._comment
@@ -0,0 +1,21 @@
+[[!comment format=mdwn
+ username="https://launchpad.net/~arand"
+ nickname="arand"
+ subject="comment 12"
+ date="2013-03-11T10:34:42Z"
+ content="""
+Without having read this, I've reported a very similar wishlist item at:
+[http://git-annex.branchable.com/todo/wishlist:_recursive_directory_remote_setup__47__addurl](http://git-annex.branchable.com/todo/wishlist:_recursive_directory_remote_setup__47__addurl)
+
+combining a recursive addurl (in my case using --fast) script with the suggestions regarding symlinks here, it's somewhat workable:
+
+    ln -s /media/cdrom /var/tmp/mycdrom123
+    ~/utv/scripts/annex-importdir /var/tmp/mycdrom123
+
+Ideally though, for optical media it would have a couple of more features (some already noted above):
+
+* Ability to form a (reasonably) unique identifier from a disc, using the label and the date of creation
+  * Ability for Annex to identify discs using this and ask for the correct disc if the file does not match (accomodating RW discs where label and date might change, or simply disc copies)
+    * Example: `not the original disc... trying anyway... file hash mismatch... please enable the remote disc with \"MYLABEL\" and creation date \"2001-01-01\"`
+* Option to checksum without importing the actual objects into the annex
+"""]]
diff --git a/doc/forum/Managing_a_large_number_of_files_archived_on_many_pieces_of_read-only_medium___40__E.G._DVDs__41__/comment_13_0a343a8fdad66765371ca22581b35b84._comment b/doc/forum/Managing_a_large_number_of_files_archived_on_many_pieces_of_read-only_medium___40__E.G._DVDs__41__/comment_13_0a343a8fdad66765371ca22581b35b84._comment
new file mode 100644
index 0000000000..2b3cf5839f
--- /dev/null
+++ b/doc/forum/Managing_a_large_number_of_files_archived_on_many_pieces_of_read-only_medium___40__E.G._DVDs__41__/comment_13_0a343a8fdad66765371ca22581b35b84._comment
@@ -0,0 +1,24 @@
+[[!comment format=mdwn
+ username="https://launchpad.net/~stephane-gourichon-lpad"
+ nickname="stephane-gourichon-lpad"
+ avatar="http://cdn.libravatar.org/avatar/02d4a0af59175f9123720b4481d55a769ba954e20f6dd9b2792217d9fa0c6089"
+ subject="Not a priority in itself, still feels like a missing piece."
+ date="2016-10-26T12:29:50Z"
+ content="""
+> I have a large number of files that are accessed infrequently and stored off-line on DVD-Rs. I need to keep track of which files are on which disc so that when I want a file I can find it.
+> (...)
+> 4) Easily incorporate the current DVD-Rs into the new system
+
+This last item would make `git-annex` suitable to catalog existing WORM media.
+
+In the past I have used some programs but was never satisfied with their graphical-UI-first approach or closed format.  For example: gtktalog, cdcat, cdcollect, where is it, virtual volume view, gnome catalog, basenji.  Ref: https://alternativeto.net/software/cdcollect/?platform=linux . 
+
+I also used at some point a plain old `find|{stat;md5}|gzip > ~/catalogs/my_volume_id.gz` then `grep mystring ~/catalogs/*gz` which, at the end of the day, has an overall good cost/benefit ratio.
+
+IMHO git-annex has a sane foundation and the potential to do better than those tools.
+
+Technically this looks indeed similar to a web special remote, but needs to accommodate for arbitrary mount point and keep count of copies.
+
+To be honest, the DVD use case is not a priority for me at the moment, but it feels like a missing piece in an otherwise good puzzle.  As if handling this case nicely would actually benefit other, more modern use cases.
+
+"""]]
diff --git a/doc/forum/Managing_a_large_number_of_files_archived_on_many_pieces_of_read-only_medium___40__E.G._DVDs__41__/comment_1_25e65ee3949e7d918376298cf11585f2._comment b/doc/forum/Managing_a_large_number_of_files_archived_on_many_pieces_of_read-only_medium___40__E.G._DVDs__41__/comment_1_25e65ee3949e7d918376298cf11585f2._comment
new file mode 100644
index 0000000000..e406f1b6f8
--- /dev/null
+++ b/doc/forum/Managing_a_large_number_of_files_archived_on_many_pieces_of_read-only_medium___40__E.G._DVDs__41__/comment_1_25e65ee3949e7d918376298cf11585f2._comment
@@ -0,0 +1,10 @@
+[[!comment format=mdwn
+ username="https://me.yahoo.com/a/2grhJvAC049fJnvALDXek.6MRZMTlg--#eec89"
+ nickname="John"
+ subject="Have you seen the dar utility?"
+ date="2012-10-20T19:03:37Z"
+ content="""
+http://dar.linux.free.fr/doc/index.html
+
+Would be nice to have this as another remote option for git-annex, since I too would like to have static (and possibly incrementally extended) remotes that span multiple DVDs
+"""]]
diff --git a/doc/forum/Managing_a_large_number_of_files_archived_on_many_pieces_of_read-only_medium___40__E.G._DVDs__41__/comment_2_8a71ca048f9de29a198a6afb17d5315e._comment b/doc/forum/Managing_a_large_number_of_files_archived_on_many_pieces_of_read-only_medium___40__E.G._DVDs__41__/comment_2_8a71ca048f9de29a198a6afb17d5315e._comment
new file mode 100644
index 0000000000..abff8702ec
--- /dev/null
+++ b/doc/forum/Managing_a_large_number_of_files_archived_on_many_pieces_of_read-only_medium___40__E.G._DVDs__41__/comment_2_8a71ca048f9de29a198a6afb17d5315e._comment
@@ -0,0 +1,11 @@
+[[!comment format=mdwn
+ username="Steve"
+ ip="92.104.175.136"
+ subject=""free-form" special remote / dar utility"
+ date="2012-10-20T22:11:23Z"
+ content="""
+dar looks familiar, I'm sure I have run across it in the past.  However, it is not suitable in this case; see requirement #3 above that the DVD-Rs be usable without git or git-annex.
+
+What would work we be some sort of special remote that allows free-form data.  Imagine that you create the DVD-R with the files on it, then you mount it and add the mount directory as a free-form special remote.  git-annex checksums all the files under the specified directory and stores the relative path to each file somewhere.  Then, when you want to fetch a specific hash from the remote it looks up the relative path, adds it to the base directory and transfers it into the local .git/annex/objects/ store.
+
+"""]]
diff --git a/doc/forum/Managing_a_large_number_of_files_archived_on_many_pieces_of_read-only_medium___40__E.G._DVDs__41__/comment_3_e3d1d3a3d3d831432ec940a8ab6f31e9._comment b/doc/forum/Managing_a_large_number_of_files_archived_on_many_pieces_of_read-only_medium___40__E.G._DVDs__41__/comment_3_e3d1d3a3d3d831432ec940a8ab6f31e9._comment
new file mode 100644
index 0000000000..973e97319e
--- /dev/null
+++ b/doc/forum/Managing_a_large_number_of_files_archived_on_many_pieces_of_read-only_medium___40__E.G._DVDs__41__/comment_3_e3d1d3a3d3d831432ec940a8ab6f31e9._comment
@@ -0,0 +1,14 @@
+[[!comment format=mdwn
+ username="http://lj.rossia.org/users/imz/"
+ ip="79.165.57.104"
+ subject="Yes, I agree, such a special remote for free-form read-only media would be convnient."
+ date="2012-10-20T23:58:45Z"
+ content="""
+I have already stored a lot of large files on DVDs. I did that for arhiving, so I cared that there are several copies. But I want this to be more automated.
+
+I take my disc (or one created by someone else, without any knowledge of Git), checksum its contents in git-annex, and in the projects where I'm using this content, I can check that the file is archived on at least N discs.
+
+Also, I might enhance the content -- this would be refected in a Git commit, so then I want also to be able to check that the new version has also ben archived on severeal discs.
+
+A special remote for such free-form read-only media would be very convenient.
+"""]]
diff --git a/doc/forum/Managing_a_large_number_of_files_archived_on_many_pieces_of_read-only_medium___40__E.G._DVDs__41__/comment_4_26a33eae98b4faaf6baf6635e3d28a8f._comment b/doc/forum/Managing_a_large_number_of_files_archived_on_many_pieces_of_read-only_medium___40__E.G._DVDs__41__/comment_4_26a33eae98b4faaf6baf6635e3d28a8f._comment
new file mode 100644
index 0000000000..44d547dd9c
--- /dev/null
+++ b/doc/forum/Managing_a_large_number_of_files_archived_on_many_pieces_of_read-only_medium___40__E.G._DVDs__41__/comment_4_26a33eae98b4faaf6baf6635e3d28a8f._comment
@@ -0,0 +1,27 @@
+[[!comment format=mdwn
+ username="Steve"
+ ip="92.104.175.136"
+ subject="Some free-form remote ideas"
+ date="2012-10-21T02:07:40Z"
+ content="""
+This is starting to get interesting.  A free-form remote would definitely simplify my use case, and also solve the \"future goal\" of easily incorporating my already existing DVD-Rs.
+
+I haven't really looked into the git-annex internals up to this point, but looking at the [[special_remotes/hook]] page there doesn't seem to be a hook for init which would be needed to populate git-annex's index of files in the remote.  (git-annex seems to assume that new special remotes are empty)
+
+Another problem is where to store the hash to path relation information.  On a RW remote it would be stored in the remote, but here we need to keep it in the repo somehow.  This could be in the git-annex branch, or possibly another branch created specifically for this purpose.
+
+1) initremote needs to:
+
+* hash the contents of all the remote's files
+* update git-annex's index of the remote's contents
+* store the paths to the hashes in the repo
+
+2) store and remove should just fail.
+
+3) retrieve and check present seem straight forward.
+
+The assistant blog mentions adding support for read only remotes but I don't know anything about it: [[design/assistant/blog/day_65__transfer_polish]] (I'm still on 3.20120605)
+
+Let me know if there is anything I haven't thought of yet.
+
+"""]]
diff --git a/doc/forum/Managing_a_large_number_of_files_archived_on_many_pieces_of_read-only_medium___40__E.G._DVDs__41__/comment_5_49ac298d39c824b0e52a239961463e09._comment b/doc/forum/Managing_a_large_number_of_files_archived_on_many_pieces_of_read-only_medium___40__E.G._DVDs__41__/comment_5_49ac298d39c824b0e52a239961463e09._comment
new file mode 100644
index 0000000000..b06891cd83
--- /dev/null
+++ b/doc/forum/Managing_a_large_number_of_files_archived_on_many_pieces_of_read-only_medium___40__E.G._DVDs__41__/comment_5_49ac298d39c824b0e52a239961463e09._comment
@@ -0,0 +1,14 @@
+[[!comment format=mdwn
+ username="http://joeyh.name/"
+ ip="4.154.0.23"
+ subject="comment 5"
+ date="2012-10-21T05:36:36Z"
+ content="""
+I encourage playing around with the hook special remote and see how far you can make it go.
+
+I may be doing something vaguely like this for [[design/assistant/desymlink]], although I'm pretty sure it would still have a git repository associated with the directory of regular files.
+
+One option is to use the web special remote, with file:// urls. Assuming a given disc will always end up mounted somewhere stable, such as /media/dvd1, /media/dvd2, etc, you could then just `git annex addurl file:///media/dvd1/$file`. `git annex whereis` will show the url, which has enough info to work out the disk to mount.
+
+The web special remote did not support file:// urls, but I've just fixed that. The only downside is that, while it will identify files duplicated across disks, and `whereis` will show multiple urls for such files, there's only one web special remote, and so it only counts as 1 copy. This could perhaps be improved; git-annex may eventually get support for remotes reporting how many copies of a file they contain.
+"""]]
diff --git a/doc/forum/Managing_a_large_number_of_files_archived_on_many_pieces_of_read-only_medium___40__E.G._DVDs__41__/comment_6_55a4a3616ea59654da1c2f9902561e3b._comment b/doc/forum/Managing_a_large_number_of_files_archived_on_many_pieces_of_read-only_medium___40__E.G._DVDs__41__/comment_6_55a4a3616ea59654da1c2f9902561e3b._comment
new file mode 100644
index 0000000000..ab971dab5e
--- /dev/null
+++ b/doc/forum/Managing_a_large_number_of_files_archived_on_many_pieces_of_read-only_medium___40__E.G._DVDs__41__/comment_6_55a4a3616ea59654da1c2f9902561e3b._comment
@@ -0,0 +1,13 @@
+[[!comment format=mdwn
+ username="http://www.openid.albertlash.com/openid/"
+ ip="74.96.185.87"
+ subject="It works!"
+ date="2012-10-24T22:00:31Z"
+ content="""
+This works great! I first tried it with WORM, no-go. I can see why the SHA backends are so powerful, they appear to circumvent the commits which git usually uses for merging. When I first do the merge, it reports this:
+
+warning: no common commits
+
+Compared to how I've managed CD/DVD backups in the past, this is a quantum leap forward, and I don't find it convoluted in comparison. Yes, there is dar, but I prefer this method. In my case, its the perfect solution for original files, which in generally are treated as immutable, and not accessed very often. They are usually large, too! I'm using them for digital pictures.
+
+"""]]
diff --git a/doc/forum/Managing_a_large_number_of_files_archived_on_many_pieces_of_read-only_medium___40__E.G._DVDs__41__/comment_7_92a2af3e0e328bb48bcc67a69187ee57._comment b/doc/forum/Managing_a_large_number_of_files_archived_on_many_pieces_of_read-only_medium___40__E.G._DVDs__41__/comment_7_92a2af3e0e328bb48bcc67a69187ee57._comment
new file mode 100644
index 0000000000..736fc6ce20
--- /dev/null
+++ b/doc/forum/Managing_a_large_number_of_files_archived_on_many_pieces_of_read-only_medium___40__E.G._DVDs__41__/comment_7_92a2af3e0e328bb48bcc67a69187ee57._comment
@@ -0,0 +1,13 @@
+[[!comment format=mdwn
+ username="Steve"
+ ip="92.104.175.136"
+ subject="web and hook special remotes"
+ date="2012-10-24T23:26:53Z"
+ content="""
+Hi Joey,
+
+Thanks for the advice.  I had thought of the web special remote; but as you may have noticed from my example, I don't use automount so my DVDs and CDs all get mounted in the same place. (/mnt/cdrom) so the web special remote won't work for me.
+
+I'll try to play around with the hook special remote this weekend.  I had a thought it might be interesting to have it search for the DVDs in some common places or even by parsing the mounted file systems, and allow an override or augmentation through git config.
+
+"""]]
diff --git a/doc/forum/Managing_a_large_number_of_files_archived_on_many_pieces_of_read-only_medium___40__E.G._DVDs__41__/comment_8_f6e39e71882d55cdc061166aea3e2bd3._comment b/doc/forum/Managing_a_large_number_of_files_archived_on_many_pieces_of_read-only_medium___40__E.G._DVDs__41__/comment_8_f6e39e71882d55cdc061166aea3e2bd3._comment
new file mode 100644
index 0000000000..84705d4cd7
--- /dev/null
+++ b/doc/forum/Managing_a_large_number_of_files_archived_on_many_pieces_of_read-only_medium___40__E.G._DVDs__41__/comment_8_f6e39e71882d55cdc061166aea3e2bd3._comment
@@ -0,0 +1,26 @@
+[[!comment format=mdwn
+ username="Steve"
+ ip="92.104.175.136"
+ subject="no need to merge"
+ date="2012-10-24T23:52:30Z"
+ content="""
+Albert,
+
+Thanks for feedback!  I'm glad that somebody else found the method I worked out useful.  As I'm going to try and turn it into a proper special remote, let me know if there is any particular use case or feature you'd like me to address.
+
+Note that in my testing, I found that you don't actually need to merge the DVD's branch into the local branch you are using for git annex to be able to find the files on it that are identical to files in your local branch.
+
+I haven't played around with cloning the repo, but I will try that this weekend.  I'm thinking it *might* be necessary to create local branches from the DVD remotes so that they'll get carried along when you clone the repo.
+
+As far as the repos on the DVD's not having a shared ancestry with main repo, that was a conscious choice that I made.  I wanted to add as little extra data to the DVDs as possible since I usually fill them to the brim anyway.  I didn't feel that it would be beneficial for the DVD's to know about the history of the main repo and other files that they don't contain.  Furthermore, besides all the links and history, you'd be replicating all the files in the main repo that aren't annexed.
+
+If you want to avoid the error, but still have a local branch for the DVD repos you should be able to do something like the following:
+
+WARNING: these commands are untested!
+
    +git checkout -b disc1 disc1/master
+git checkout -b disc2 disc2/master
+
    
+
+Working from the original example, you should then get local branches for the DVDs that don't have a common ancestor with your master local repo.  I haven't actually tested that though.  Testing will have to wait for this weekend.
+"""]]
diff --git a/doc/forum/Managing_a_large_number_of_files_archived_on_many_pieces_of_read-only_medium___40__E.G._DVDs__41__/comment_9_6c45a6264d69e22800c329a0f8a2d470._comment b/doc/forum/Managing_a_large_number_of_files_archived_on_many_pieces_of_read-only_medium___40__E.G._DVDs__41__/comment_9_6c45a6264d69e22800c329a0f8a2d470._comment
new file mode 100644
index 0000000000..5a3f2885c1
--- /dev/null
+++ b/doc/forum/Managing_a_large_number_of_files_archived_on_many_pieces_of_read-only_medium___40__E.G._DVDs__41__/comment_9_6c45a6264d69e22800c329a0f8a2d470._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="http://joeyh.name/"
+ ip="4.152.108.138"
+ subject="comment 9"
+ date="2012-10-25T03:33:29Z"
+ content="""
+@Steve, it seems to me you could still use the web special remote, just pointing it at an url that goes through a symlink to the mount point.
+"""]]
diff --git a/doc/forum/Managing_multiple_annexes_with_assistant__63__.mdwn b/doc/forum/Managing_multiple_annexes_with_assistant__63__.mdwn
new file mode 100644
index 0000000000..30e059743f
--- /dev/null
+++ b/doc/forum/Managing_multiple_annexes_with_assistant__63__.mdwn
@@ -0,0 +1,13 @@
+Is it possible to run more than one instance of the assistant in an account?
+
+A particular example is that I might have two annexes:
+
+1. for my music, and
+2. for some personal documents.
+
+I would like to have my music annex paired with my work laptop and have all the automagical power of git-annex-assistant working for me.  However I don't want to put my personal documents on a work machine, hence the second annex.
+
+(I think) An ideal world would have the assistant managing both annexes.  Is this possible?
+
+
+I know my question is similar to [one annex versus many annexes?](http://git-annex.branchable.com/forum/one_annex_versus_many_annexes__63__/) but I think it is different enough to warrant a new thread...
diff --git a/doc/forum/Managing_multiple_annexes_with_assistant__63__/comment_1_ba8c70e4a46441b48ad910625636eee5._comment b/doc/forum/Managing_multiple_annexes_with_assistant__63__/comment_1_ba8c70e4a46441b48ad910625636eee5._comment
new file mode 100644
index 0000000000..675b5b71c2
--- /dev/null
+++ b/doc/forum/Managing_multiple_annexes_with_assistant__63__/comment_1_ba8c70e4a46441b48ad910625636eee5._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="http://joeyh.name/"
+ ip="4.154.0.23"
+ subject="comment 1"
+ date="2012-10-19T17:08:39Z"
+ content="""
+If you go to the upper-right menu in the webapp, and select \"New repository\", you can add a separate independant repository. The assistant's autostart will run it on both of them when you log in.
+"""]]
diff --git a/doc/forum/Managing_multiple_annexes_with_assistant__63__/comment_2_4b4f0a7d84a51ae92536e2c190256069._comment b/doc/forum/Managing_multiple_annexes_with_assistant__63__/comment_2_4b4f0a7d84a51ae92536e2c190256069._comment
new file mode 100644
index 0000000000..86fa9ebabc
--- /dev/null
+++ b/doc/forum/Managing_multiple_annexes_with_assistant__63__/comment_2_4b4f0a7d84a51ae92536e2c190256069._comment
@@ -0,0 +1,10 @@
+[[!comment format=mdwn
+ username="https://www.google.com/accounts/o8/id?id=AItOawkk3K0AUduAybbBO_LRRGKOe2zcGeezbzI"
+ nickname="Nathan"
+ subject="Webapp spins when adding a second annex"
+ date="2012-11-20T04:55:05Z"
+ content="""
+I was trying to add a second annex (repository) via the webapp, but it seems like the process never completes: the browser just spins (\"connecting...\"). If I kill the webapp process and restart, I can see the second annex in the list of repositories, but clicking on it gives the same behaviour. Interestingly, the path *is* added to ~/.config/git-annex/autostart, and the webapp does create the path, if needed, and initializes it as a git annex. I've reproduced this on two different machines, both running Ubuntu 12.10 and git annex 3.20121112.
+
+Is there anything I can do to help debug?
+"""]]
diff --git a/doc/forum/Managing_multiple_annexes_with_assistant__63__/comment_3_86daadc565f96db5db13b6dbcbc66db3._comment b/doc/forum/Managing_multiple_annexes_with_assistant__63__/comment_3_86daadc565f96db5db13b6dbcbc66db3._comment
new file mode 100644
index 0000000000..6ad879d76d
--- /dev/null
+++ b/doc/forum/Managing_multiple_annexes_with_assistant__63__/comment_3_86daadc565f96db5db13b6dbcbc66db3._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="http://joeyh.name/"
+ ip="4.154.6.49"
+ subject="comment 3"
+ date="2012-11-25T18:59:37Z"
+ content="""
+@Nathan at this point, you should have two separate git-annex processes running, one for each repository. You can try opening a shell, changing to the new repository, and running `git annex webapp --debug` to get debugging information on the new one.
+"""]]
diff --git a/doc/forum/Managing_multiple_annexes_with_assistant__63__/comment_4_e43d71ddfdfdb7bcb13bfb894de6a5ec._comment b/doc/forum/Managing_multiple_annexes_with_assistant__63__/comment_4_e43d71ddfdfdb7bcb13bfb894de6a5ec._comment
new file mode 100644
index 0000000000..78570a08d3
--- /dev/null
+++ b/doc/forum/Managing_multiple_annexes_with_assistant__63__/comment_4_e43d71ddfdfdb7bcb13bfb894de6a5ec._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="http://joeyh.name/"
+ ip="4.154.6.49"
+ subject="comment 4"
+ date="2012-11-28T20:13:11Z"
+ content="""
+@Nathan, I have just noticed and fixed a problem that could cause this to happen if you were using the standalone tarball (or OSX app) builds. It would not affect git-annex installed by other methods.
+"""]]
diff --git a/doc/forum/Managing_multiple_annexes_with_assistant__63__/comment_5_e94d33be83b45918d1a39d6e16fba4b4._comment b/doc/forum/Managing_multiple_annexes_with_assistant__63__/comment_5_e94d33be83b45918d1a39d6e16fba4b4._comment
new file mode 100644
index 0000000000..ae9c5c5b60
--- /dev/null
+++ b/doc/forum/Managing_multiple_annexes_with_assistant__63__/comment_5_e94d33be83b45918d1a39d6e16fba4b4._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="https://www.google.com/accounts/o8/id?id=AItOawkk3K0AUduAybbBO_LRRGKOe2zcGeezbzI"
+ nickname="Nathan"
+ subject="comment 5"
+ date="2012-12-11T04:15:49Z"
+ content="""
+Thanks, Joey; I was using the standalone build, and it seems to be behaving better now.
+"""]]
diff --git a/doc/forum/Managing_multiple_repositories_concurrently__63__.mdwn b/doc/forum/Managing_multiple_repositories_concurrently__63__.mdwn
new file mode 100644
index 0000000000..60942584b8
--- /dev/null
+++ b/doc/forum/Managing_multiple_repositories_concurrently__63__.mdwn
@@ -0,0 +1,5 @@
+I have the webapp setup to manage 4 different repositories.
+
+One repository had many gigabytes to upload to a remote. As this was moving along, I wanted to check on the status of one of the other repositories. I knew that this second repository had a lot of data to download. When I switched to the second repo in the webapp, I noticed in my network monitor that the amount of upstream bandwidth I was using dropped and the amount of downstream bandwidth increased. I switched back to the first repo, and my downstream bandwidth dropped and the upstream bandwidth increased.
+
+It looks like git-annex stopped uploading data from the first repo when I switched to the second, and stopped downloading data to the second repo when I switched to the first. Is this correct? I was under the impression that switching repositories in the webapp simply changed the view that I was looking at, but that the assistant would still be managing all the repositories I had setup -- uploading/downloading/syncing concurrently.
diff --git a/doc/forum/Managing_multiple_repositories_concurrently__63__/comment_1_ebec1ddad71e961cdc9b21cbddfbcdaf._comment b/doc/forum/Managing_multiple_repositories_concurrently__63__/comment_1_ebec1ddad71e961cdc9b21cbddfbcdaf._comment
new file mode 100644
index 0000000000..c1c5cd9179
--- /dev/null
+++ b/doc/forum/Managing_multiple_repositories_concurrently__63__/comment_1_ebec1ddad71e961cdc9b21cbddfbcdaf._comment
@@ -0,0 +1,10 @@
+[[!comment format=mdwn
+ username="http://joeyh.name/"
+ nickname="joey"
+ subject="comment 1"
+ date="2013-03-17T18:53:26Z"
+ content="""
+Each repository gets its own git-annex assistant daemon, which is entirely separate from the daemons used by other repositories. 
+
+The only affect that switching which repository is in view in the webapp can have is it can start a daemon running on a repository that for some reason did not already have the daemon running. This could happen if it was not set to autostart on login, for example.
+"""]]
diff --git a/doc/forum/Manipulate_a_git-annex_repo_from_an_Android_app.mdwn b/doc/forum/Manipulate_a_git-annex_repo_from_an_Android_app.mdwn
new file mode 100644
index 0000000000..bafd5d9e9b
--- /dev/null
+++ b/doc/forum/Manipulate_a_git-annex_repo_from_an_Android_app.mdwn
@@ -0,0 +1,21 @@
+I use the command line almost every hour, I'm familiar with it. But it's a pita on the phone.
+
+I'd like to create an "graphical" Android application that lists/browses the repo's content and allows the user to manipulate the files/directories.
+
+git-annex for Android (5.0) is installed and working fine. I was messing around (on my rooted phone) with the rwx permissions to run the git-annex binary from my app as it's a different user and has no access to those binaries by default. But no success yet: 
+
+```
+shell@jfltexx:/storage/sdcard1/repo $ /data/data/ga.androidterm/lib/lib.git-annex.so list
+
+lib.git-annex.so: git: createProcess: runInteractiveProcess: exec: permission denied (Permission denied)
+```
+
+Question 1: How should I set up the permissions to be able to successfully run any git-annex command from my (non-commercial, private) Android app?
+
+Question 2: What if I include all the git-annex binaries in my app as a resource and write them to disk after install and execute them? (Permissions should work.)
+
+
+Use case: family videos are stored in a repo at home. This repo is cloned to my phone's sdcard. I want to browse the repo on my phone and mark some directories to be transferred from home to phone. My app would be a simple frontend for git-annex where you can browse and do some basic stuff: get, drop, manipulate metadata (and of course start playing a video).
+
+Thank you for any ideas,
+Bence
diff --git a/doc/forum/Manipulate_a_git-annex_repo_from_an_Android_app/comment_1_c59ec715eb02fbdd63780b2450ff2de1._comment b/doc/forum/Manipulate_a_git-annex_repo_from_an_Android_app/comment_1_c59ec715eb02fbdd63780b2450ff2de1._comment
new file mode 100644
index 0000000000..8a09376027
--- /dev/null
+++ b/doc/forum/Manipulate_a_git-annex_repo_from_an_Android_app/comment_1_c59ec715eb02fbdd63780b2450ff2de1._comment
@@ -0,0 +1,12 @@
+[[!comment format=mdwn
+ username="joey"
+ subject="""comment 1"""
+ date="2015-12-02T17:05:03Z"
+ content="""
+Android's security model may not be friendly to code reuse. You might need
+to put both git-annex and your program in the same app.
+
+I know this is something others want too. Maybe you can find or make a file
+manager type app that integrates with git-annex get/drop and could be
+included in the git-annex android app?
+"""]]
diff --git a/doc/forum/Manipulate_a_git-annex_repo_from_an_Android_app/comment_2_20ee5f900a2ff6154c0d90263579723e._comment b/doc/forum/Manipulate_a_git-annex_repo_from_an_Android_app/comment_2_20ee5f900a2ff6154c0d90263579723e._comment
new file mode 100644
index 0000000000..e95d090e54
--- /dev/null
+++ b/doc/forum/Manipulate_a_git-annex_repo_from_an_Android_app/comment_2_20ee5f900a2ff6154c0d90263579723e._comment
@@ -0,0 +1,13 @@
+[[!comment format=mdwn
+ username="cbaines"
+ subject="comment 2"
+ date="2015-12-21T12:40:52Z"
+ content="""
+I am also interested in this.
+
+It might be better to use intents [1], such that git-annex on Android can provide an interface for any app to ask it to perform actions on repositories.
+
+Joey, is there any kind of api in the current Android app?
+
+1: https://developer.android.com/reference/android/content/Intent.html
+"""]]
diff --git a/doc/forum/Manual_Setup_of_a_Central_Repo.mdwn b/doc/forum/Manual_Setup_of_a_Central_Repo.mdwn
new file mode 100644
index 0000000000..82a1c70bae
--- /dev/null
+++ b/doc/forum/Manual_Setup_of_a_Central_Repo.mdwn
@@ -0,0 +1 @@
+My current setup involves 3 computers, one desktop one laptop and a vps. In my current setup I have created annex repos on the server and cloned from it, both machines sync to the server. All three use non base repos. Now I have another annex folder on the desktop that I would like to sync between the three. Both machines are behind NAT so server can not communicate with the machines. In order to init the repo on the vps, I was thinking of setting up a temporary VPN/port forward between the desktop and the VPS then clone from the desktop finally remove the remote section in .git/config on the server so VPS becomes the master again. First of all is there an easier way to do this? if not is it safe to do this? or is it going to cause problems down the line.
diff --git a/doc/forum/Manual_Setup_of_a_Central_Repo/comment_1_3a163fd5629dc40423f1290a78ae1c07._comment b/doc/forum/Manual_Setup_of_a_Central_Repo/comment_1_3a163fd5629dc40423f1290a78ae1c07._comment
new file mode 100644
index 0000000000..e5630b3319
--- /dev/null
+++ b/doc/forum/Manual_Setup_of_a_Central_Repo/comment_1_3a163fd5629dc40423f1290a78ae1c07._comment
@@ -0,0 +1,12 @@
+[[!comment format=mdwn
+ username="http://joeyh.name/"
+ ip="4.154.0.63"
+ subject="comment 1"
+ date="2013-08-26T18:46:06Z"
+ content="""
+You could certainly do that. I don't think it's the easiest way.
+
+Note that this is essentially a git question. It really has nothing to do with git-annex, unless you want to use the git-annex assistant, which can sync a repository over XMPP without needing a central git repository at all.
+
+If I had this problem with git in general, I would make a new empty repository on the server, and push the local repository I have on the one machine to it. Then on the other machine, I would clone from the server. Problem solved, I think?
+"""]]
diff --git a/doc/forum/Manual_commit_message_in_direct_mode.mdwn b/doc/forum/Manual_commit_message_in_direct_mode.mdwn
new file mode 100644
index 0000000000..d74c366073
--- /dev/null
+++ b/doc/forum/Manual_commit_message_in_direct_mode.mdwn
@@ -0,0 +1,3 @@
+I am just returning to git-annex after some time. The last time I was using it, the [[/todo/direct_mode_guard]] was not yet in place, and I could supply my own commit messages even in a direct mode repository using `git commit`. Now that the guard is in place, is there a built-in way to provide a commit message that will be used instead of "git-annex automatic sync" by `git annex sync`? 
+
+I have tried doing a manual commit with `git --git-dir=/path/to/local/.git --work-tree=/path/to/local -c core.bare=false commit -m 'Manual Commit Message'`, and it seems to work. Does this do the same thing as the part of `git annex sync` that commits any outstanding changes before syncing with remotes?
diff --git a/doc/forum/Manual_commit_message_in_direct_mode/comment_1_32f95eefec25bb127ed96248446c21b1._comment b/doc/forum/Manual_commit_message_in_direct_mode/comment_1_32f95eefec25bb127ed96248446c21b1._comment
new file mode 100644
index 0000000000..bb53d4458e
--- /dev/null
+++ b/doc/forum/Manual_commit_message_in_direct_mode/comment_1_32f95eefec25bb127ed96248446c21b1._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="http://joeyh.name/"
+ ip="209.250.56.176"
+ subject="comment 1"
+ date="2014-05-27T16:10:29Z"
+ content="""
+Yes, you can use the `-c core.bare=false` trick to bypass the direct mode guard if you want to. Just bear in mind that it's very easy to shoot yourself in the foot by passing -a or filenames to git commit, which then checks the whole large file into git..
+"""]]
diff --git a/doc/forum/Manual_commit_message_in_direct_mode/comment_2_bf1d10067379c802ac5020d8becd6d35._comment b/doc/forum/Manual_commit_message_in_direct_mode/comment_2_bf1d10067379c802ac5020d8becd6d35._comment
new file mode 100644
index 0000000000..facb5ce334
--- /dev/null
+++ b/doc/forum/Manual_commit_message_in_direct_mode/comment_2_bf1d10067379c802ac5020d8becd6d35._comment
@@ -0,0 +1,10 @@
+[[!comment format=mdwn
+ username="https://www.google.com/accounts/o8/id?id=AItOawnxvt4mc32O0ctvPGhhII_ZKsHUiaD2o7I"
+ nickname="William"
+ subject="comment 2"
+ date="2014-05-28T04:40:27Z"
+ content="""
+Joey, thanks for clarifying that. I learned about the perils of using regular git commands in direct mode before the guard was implemented, so I will use with care. Specifying a commit message is really the only case where I am tempted to bypass the the `git annex` subcommands in direct mode. 
+
+
+"""]]
diff --git a/doc/forum/Manual_mode_option_in_assistant_auto-syncs.mdwn b/doc/forum/Manual_mode_option_in_assistant_auto-syncs.mdwn
new file mode 100644
index 0000000000..49b58e529e
--- /dev/null
+++ b/doc/forum/Manual_mode_option_in_assistant_auto-syncs.mdwn
@@ -0,0 +1,11 @@
+Hi,
+
+I've recently set-up a server which uses Southpaw's Tactic DAM system and I've initialised a git-annex directory using the assistant which will manage any files which Tactic puts into the git-annex. I plan to make some remote repositories to Amazon S3, friends and some local machines on my home network. The server is running Ubuntu 64-bit and so I've written an upstart job which runs 'git-annex assistant --autostart' as the user 'git-annex' as this user doesn't log-in and run the xdg autostart .desktop at all.
+
+I saw that you can set the purpose of each repository which it will sync to and noticed 'Manual Mode.' From the description, it seems it will only work if I do explicit git-annex commands to it which would be perfect for me as I'd like to write tools which run git-annex add/get/drop/etc. manually on some remotes like ones to friends as I don't want them to sync to everything or any files that they produce, only files which they request with a special tool that I'll write and have Tactic marshal the file changes/names/etc.
+
+I've set those remotes to manual-mode via the assistant and tried copying a file to the remote's directory, but it auto-synced the file anyway. Maybe I'm getting confused at how manual mode works but I'd like to only explicitly set which files to add to the repo by a command and not just 'any' file which gets placed into that directory. If this is more of a wishlist request I guess just move this post into there and I'll reword it as a wish request.
+
+I don't need to use the assistant if that makes more sense, but I would like to be able to still monitor things as the webapp makes for a great GUI to check for that stuff. I can't wait for the https version  of webapp too, I currently run git-annex webapp --listen=: and then run the link it outputs on my desktop machine to manage it from the server.
+
+Thanks, and this is such a great bit of software, especially as my Internet connection is really bad for 2013 standards, and having the option for friends/remote servers to sync up via an encrypted S3 or box.com account is great!
diff --git a/doc/forum/Manual_mode_option_in_assistant_auto-syncs/comment_1_4a0468b6ca2ffff8ef8f19800597567d._comment b/doc/forum/Manual_mode_option_in_assistant_auto-syncs/comment_1_4a0468b6ca2ffff8ef8f19800597567d._comment
new file mode 100644
index 0000000000..56e68f779b
--- /dev/null
+++ b/doc/forum/Manual_mode_option_in_assistant_auto-syncs/comment_1_4a0468b6ca2ffff8ef8f19800597567d._comment
@@ -0,0 +1,10 @@
+[[!comment format=mdwn
+ username="http://joeyh.name/"
+ ip="4.154.0.63"
+ subject="comment 1"
+ date="2013-08-24T16:20:29Z"
+ content="""
+In the assistant, putting a repository in manual mode prevents the assistant from automatically downloading every file that is added to other clones of the repository. As you've noticed, it does not prevent the assistant from automatically adding new files that are put into the local repository, or from uploading those files to other clients that want them (ie, clients not themselves set to use manual mode).
+
+You can prevent the assistant from noticing when you add new files to a repository by clicking on \"syncing enabled\" in the first repository in the list. (The repository labeled as \"here\".) You can then `git annex add` the files you want to add by hand, and manually `copy annex copy` them to other repositories, and manually `git annex sync`.
+"""]]
diff --git a/doc/forum/Manual_webapp_behaviour_on_ARM.mdwn b/doc/forum/Manual_webapp_behaviour_on_ARM.mdwn
new file mode 100644
index 0000000000..8b7b8fe278
--- /dev/null
+++ b/doc/forum/Manual_webapp_behaviour_on_ARM.mdwn
@@ -0,0 +1,15 @@
+Hello,
+
+I have a Samsung ARM Chromebook running Ubuntu with crouton (chroot in chromeos). I want to use git-annex, and add the chromebook as a second "device" to my already set up git-annex repo on my main pc. That repo is one folder set up with the assistant, and it has one encrypted ssh(rsync) remote.
+
+I've read that the webapp is not available on ARM, and I've also found a topic with a simple script to replicate git-annex-assistant steps: http://git-annex.branchable.com/forum/Running_assistant_steps_manually/. So, I've used the `cabal` steps to install git-annex manually, which is working.
+
+I want to use git-annex in a dropbox-like fashion, folder and file sync between these two devices (and maybe later on another encrypted ssh remote). However I'm in need of some guidance regarding the command line usage and such.
+
+My questions are:
+
+- How do I "add" the chromebook repo to the existing git-annex setup?
+- The machines are not on at the same time. One is my main workstation and the chromebook is my "mobile friend". The encrypted ssh remote is on a VPS, that is always on.
+  - How do I configure the syncing between the main machine, the chromebook and the encrypted ssh remote on the vps? If possible, bi-directional.
+
+If it is possible, both explanation and command line examples would help a lot. 
diff --git a/doc/forum/Manual_webapp_behaviour_on_ARM/comment_1_5431a57aa7d659faf0aee12e638dd1ba._comment b/doc/forum/Manual_webapp_behaviour_on_ARM/comment_1_5431a57aa7d659faf0aee12e638dd1ba._comment
new file mode 100644
index 0000000000..f045b5c3ec
--- /dev/null
+++ b/doc/forum/Manual_webapp_behaviour_on_ARM/comment_1_5431a57aa7d659faf0aee12e638dd1ba._comment
@@ -0,0 +1,56 @@
+[[!comment format=mdwn
+ username="https://www.google.com/accounts/o8/id?id=AItOawnudiRFAyVAwehTACPhuNV_dhhWsHKIACw"
+ nickname="Jose"
+ subject="git annex on chromebook"
+ date="2015-03-12T18:32:18Z"
+ content="""
+I know I'm commenting on something old but I was looking this topic up and felt this may help any future questions. :S
+
+* its just like any other machine with git annex. You can connect your instance of git annex that runs in the chroot (crouton) with your existing set up like normal. usb, ssh, xmpp+cloud
+
+* you will need a transfer repository, that vps should do nicely (I use a few USB drives)
+
+    * this depends if the VPS also has git annex installed or not. 
+if it does, great! If you want it configured for you, you can run git annex webapp to add a remote server and give it the details to connect to your VPS via ssh. Then tell it where your transfer repository will be and that should be it. If you plan on sharing this repo with your friend, or if you want more control over the data, you may want to set up your ssh repo via the command line instead. you may be able to add encryption and share your repository without necessarily letting your friend have access to everything on your account (if you have separate accounts)
+
+
+if it does not, also doable. Again if you want simple configuration git annex webapp will prompt you for credentials; in this case it may as you to encrypt the data (http://git-annex.branchable.com/encryption/). 
+
+* how to add an ssh remote via commandline (please don't just copy paste, a simple google of the commands will help you understand what they are doing)
+
+$ cd ~/Downloads/Documents
+
+$ git init
+
+$ git annex init \"chromebook\"
+
+$ git remote add my-ssh ssh://user@google.com/home/user/annex/Documents #I recommend you use an 'ssh alias' so you can just do 'ssh://alias1/home/user/annex/Documents'
+
+$ git annex add
+
+$ git annex sync
+
+$ git annex sync --content #I do not remember but I believe it may be necessary to do this for some versions of git annex.
+
+git annex may or may not complain about git annex being installed on your machine. (it does for me each time I do this... perhaps I'm missing a step?)
+If you know git annex is installed in the $PATH of your server (also your machine) then you can just run
+
+$  git config remote.my-ssh.annex-ignore false
+
+$  git annex sync
+
+$  git annex sync --content
+
+
+* how come I can not see my annexed files on chromeos but I can on crouton?
+because you are using git annex in 'indirect' mode
+
+$ cd ~/Downloads/Documents
+
+$ git annex direct
+
+You can also 'unlock' the files too but you will need to run git annex unlock every time you need to read/write a file using. 
+
+$ git annex unlock 
+
+"""]]
diff --git a/doc/forum/Massive_drop_in_performance_with_--jobs_option.mdwn b/doc/forum/Massive_drop_in_performance_with_--jobs_option.mdwn
new file mode 100644
index 0000000000..bccb0d9dc9
--- /dev/null
+++ b/doc/forum/Massive_drop_in_performance_with_--jobs_option.mdwn
@@ -0,0 +1,9 @@
+Old version:  5.20150916-1
+New version:  5.20151208-1
+
+
+I have noticed that with the addition of the progress display, the actual overall performance of the transfer drop very noticeably from these two versions.  I am very tempted to keep the old version around to just handle the transfers.
+
+Also, I decided to build a newer version from git, using debuild.  I noticed that the libghc-persistant dependencies have no trailing commas.  I added the commas, built the package and had two errors in one of the tests.  I removed these dependencies from the control file and I am currently performing a rebuild (they were not present in 12/08 version).
+
+Build complete.  Tests passed.  Parallel transfers seem to be back up to speed.  Thanks!
diff --git a/doc/forum/Massive_drop_in_performance_with_--jobs_option/comment_2_8a9b5bc3b8a8ef947f2620422307c465._comment b/doc/forum/Massive_drop_in_performance_with_--jobs_option/comment_2_8a9b5bc3b8a8ef947f2620422307c465._comment
new file mode 100644
index 0000000000..b00eb61a0b
--- /dev/null
+++ b/doc/forum/Massive_drop_in_performance_with_--jobs_option/comment_2_8a9b5bc3b8a8ef947f2620422307c465._comment
@@ -0,0 +1,27 @@
+[[!comment format=mdwn
+ username="joey"
+ subject="""comment 2"""
+ date="2016-01-07T22:13:14Z"
+ content="""
+Version 5.20151019 is right between the two versions given, and probably
+contains the reason for the drop in performance: That version started
+checksumming files after receiving them to verify that their content is
+good.
+
+If so, this will get back the old level of performance:
+
+	git config annex.verify false
+
+If the performance problem were actually related to the new progress
+display, passing --quiet will disable that display. If you can show that
+simply passing --quiet speeds up transfers significantly somehow,
+I'd be happy to investigate further.
+
+
+----
+
+@umeboshi I don't think your comment is really related to the original question.
+If you run gets of a bunch of files in parallel, and two of the files
+have the same content, you can get this situation where a transfer of
+the content is already in progress.
+"""]]
diff --git a/doc/forum/Massive_drop_in_performance_with_--jobs_option/comment_2_c9fea9db7116e7ee82a09c23f22b54e9._comment b/doc/forum/Massive_drop_in_performance_with_--jobs_option/comment_2_c9fea9db7116e7ee82a09c23f22b54e9._comment
new file mode 100644
index 0000000000..cfc5f2684c
--- /dev/null
+++ b/doc/forum/Massive_drop_in_performance_with_--jobs_option/comment_2_c9fea9db7116e7ee82a09c23f22b54e9._comment
@@ -0,0 +1,30 @@
+[[!comment format=mdwn
+ username="umeboshi"
+ subject="comment 2"
+ date="2016-01-02T23:13:15Z"
+ content="""
+I have been getting more failures than I used to when using -J with get and copy commands.
+
+The dirannex remote is a local directory remote.
+
+```
+get 74/024842a88350573fa70e33f6dea9d274.jpg (from web...) ok
+get 74/02c67a1850ee54848d0203c48a613d74.jpg (from dirannex...) ok
+get 75/024b4a9e700c51aeb9a51d7566a19c75.jpg (transfer already in progress, or unable to take transfer lock) 
+  Unable to access these remotes: web
+
+  Try making some of these repositories available:
+        00000000-0000-0000-0000-000000000001 -- web
+failed
+```
+
+```
+git-annex copy -J5 --to dirannex
+
+(transfer already in progress, or unable to take transfer lock) failed
+git-annex: copy: 5 failed
+```
+
+
+
+"""]]
diff --git a/doc/forum/MegaAnnex_not_working.mdwn b/doc/forum/MegaAnnex_not_working.mdwn
new file mode 100644
index 0000000000..3eff4b7414
--- /dev/null
+++ b/doc/forum/MegaAnnex_not_working.mdwn
@@ -0,0 +1,32 @@
+When copying to a megaannex remote, it hangs after a while, and I get this:
+
+    copy Boomarks/Android (gpg) Traceback (most recent call last):
+      File "/usr/local/bin//git-annex-remote-mega", line 511, in 
+        common.startRemote()
+      File "/home/zack/megaannex/lib/CommonFunctions.py", line 557, in startRemote
+        sys.modules["__main__"].checkpresent(line)
+      File "/usr/local/bin//git-annex-remote-mega", line 483, in checkpresent
+        folder = setFolder(conf["folder"], common.ask("DIRHASH " + line[1]))
+      File "/usr/local/bin//git-annex-remote-mega", line 401, in setFolder
+        folder = createFolder(conf["folder"], 2)
+      File "/usr/local/bin//git-annex-remote-mega", line 378, in createFolder
+        res = m.create_folder(subject, folder)
+      File "/usr/lib/python2.7/site-packages/mega/mega.py", line 617, in create_folder
+        'i': self.request_id})
+      File "/usr/lib/python2.7/site-packages/mega/mega.py", line 110, in _api_request
+        timeout=self.timeout)
+      File "/usr/lib/python2.7/site-packages/requests/api.py", line 88, in post
+        return request('post', url, data=data, **kwargs)
+      File "/usr/lib/python2.7/site-packages/requests/api.py", line 44, in request
+        return session.request(method=method, url=url, **kwargs)
+      File "/usr/lib/python2.7/site-packages/requests/sessions.py", line 354, in request
+        resp = self.send(prep, **send_kwargs)
+      File "/usr/lib/python2.7/site-packages/requests/sessions.py", line 460, in send
+        r = adapter.send(request, **kwargs)
+      File "/usr/lib/python2.7/site-packages/requests/adapters.py", line 250, in send
+        raise SSLError(e)
+    requests.exceptions.SSLError: The read operation timed out
+    (external special remote protocol error, unexpectedly received "" (unable to parse command)) failed
+
+
+Any help would be appreciated, thanks!
diff --git a/doc/forum/MegaAnnex_not_working/comment_1_5aa3fd366d4c78ca79bb58005a49791c._comment b/doc/forum/MegaAnnex_not_working/comment_1_5aa3fd366d4c78ca79bb58005a49791c._comment
new file mode 100644
index 0000000000..c2e7353dbc
--- /dev/null
+++ b/doc/forum/MegaAnnex_not_working/comment_1_5aa3fd366d4c78ca79bb58005a49791c._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="http://joeyh.name/"
+ ip="209.250.56.2"
+ subject="comment 1"
+ date="2014-07-11T19:42:24Z"
+ content="""
+This looks to me like the connection to Mega fails, so either they have a server down, or a network problem, or perhaps the API that the megaannex remote was relying on might have been removed..
+"""]]
diff --git a/doc/forum/MegaAnnex_not_working/comment_2_f3eaf1ee06ebac951514d865f298f9d3._comment b/doc/forum/MegaAnnex_not_working/comment_2_f3eaf1ee06ebac951514d865f298f9d3._comment
new file mode 100644
index 0000000000..e9fd01f147
--- /dev/null
+++ b/doc/forum/MegaAnnex_not_working/comment_2_f3eaf1ee06ebac951514d865f298f9d3._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="https://www.google.com/accounts/o8/id?id=AItOawmqaNwDQ367zpW6cIRviLz6zJZZFODgoEI"
+ nickname="Zack"
+ subject="comment 2"
+ date="2014-07-19T12:48:08Z"
+ content="""
+I believe it may have been removed, unfortunately. :(
+"""]]
diff --git a/doc/forum/Merging_two_git-annex_repositories_together__63__.mdwn b/doc/forum/Merging_two_git-annex_repositories_together__63__.mdwn
new file mode 100644
index 0000000000..fb3c9d01ed
--- /dev/null
+++ b/doc/forum/Merging_two_git-annex_repositories_together__63__.mdwn
@@ -0,0 +1,7 @@
+Hello!
+
+I’ve got two repos, both are using git-annex.
+
+Can I somehow merge them together, into one?
+
+Thank you!
diff --git a/doc/forum/Merging_two_git-annex_repositories_together__63__/comment_1_b96bb95cdbbc78c0b4b348861e2264bd._comment b/doc/forum/Merging_two_git-annex_repositories_together__63__/comment_1_b96bb95cdbbc78c0b4b348861e2264bd._comment
new file mode 100644
index 0000000000..46be89f739
--- /dev/null
+++ b/doc/forum/Merging_two_git-annex_repositories_together__63__/comment_1_b96bb95cdbbc78c0b4b348861e2264bd._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="openmedi"
+ avatar="http://cdn.libravatar.org/avatar/563ffaff3b492c579bd8f094472e4506"
+ subject="comment 1"
+ date="2017-02-25T20:35:52Z"
+ content="""
+I think you could add those two repositories as remotes of each other and merge them with the `--allow-unrelated-histories` option. But I'm not sure if this is a good idea, or not. But some googling in that direction might yield some useful information. Good luck!
+"""]]
diff --git a/doc/forum/Merging_two_git-annex_repositories_together__63__/comment_2_539a33b268be5c6954f3112d89dd34ef._comment b/doc/forum/Merging_two_git-annex_repositories_together__63__/comment_2_539a33b268be5c6954f3112d89dd34ef._comment
new file mode 100644
index 0000000000..5059bc45df
--- /dev/null
+++ b/doc/forum/Merging_two_git-annex_repositories_together__63__/comment_2_539a33b268be5c6954f3112d89dd34ef._comment
@@ -0,0 +1,10 @@
+[[!comment format=mdwn
+ username="michalrus"
+ avatar="http://cdn.libravatar.org/avatar/83c0b6e7f9d20f09a892263c4903bbae"
+ subject="comment 2"
+ date="2017-02-26T00:59:21Z"
+ content="""
+No, actually everything just works. ♥
+
+It’s amazing! Thank you!
+"""]]
diff --git a/doc/forum/Migrate_from_one_connection_mechanism_for_special_remote_to_another.mdwn b/doc/forum/Migrate_from_one_connection_mechanism_for_special_remote_to_another.mdwn
new file mode 100644
index 0000000000..ac06b99900
--- /dev/null
+++ b/doc/forum/Migrate_from_one_connection_mechanism_for_special_remote_to_another.mdwn
@@ -0,0 +1,44 @@
+Hi there,
+
+I want to migrate from git-annex-remote-hubic to git-annex-remote-rclone to access the same hubic remote, same container, same data. So ideally I just want to switch to rclone but reuse the old content (since uploading to hubic is so sloooooow). 
+
+My old remote is called hubic, my new one hubic2 in my git annex repo.
+Both are reported as similar by git annex info -F hubic/hubic2 :
+
+    remote: hubic
+    description: [hubic]
+    uuid: 396e481c-5fc9-4933-b6a4-b1398fb7c61b
+    trust: trusted
+    cost: 225.0
+    type: external
+    externaltype: hubic
+    encryption: pubkey (to gpg keys: XXX)
+    chunking: none
+
+    remote: hubic2
+    description: [hubic2]
+    uuid: adaea02d-920c-48fa-811d-c27e51bda85f
+    trust: semitrusted
+    cost: 250.0
+    type: external
+    externaltype: rclone
+    encryption: pubkey (to gpg keys: XXX)
+    chunking: none
+
+Yes, the GPG key is the same. I only have one configured on this machine.
+
+However, when I fsck for a file that is on hubic or on hubic2 I only get a positive result for hubic. The chatlogs of both fsck reveal that the key that fsck is looking for is different for the different remotes.
+ 
+    git-annex-remote-hubic[1] <-- CHECKPRESENT GPGHMACSHA1--64e155e3010f69e909e629c6d9fdacae5c0c3bef
+respectively:
+
+    git-annex-remote-rclone[1] <-- CHECKPRESENT GPGHMACSHA1--5cf9edd1a413c561775f8920fc7b21410be6c5a4
+
+There is one thing I noticed though:
+Although in the remote-hubic case I have not chunking configured, I see with *rclone lsl* that I have some files in chunks up there.
+However, the file I am using for this test is only 2MB and it is not chunked on the remote.
+
+What am I missing, what am I doing wrong? While writing this I thought of the following: Is there an intermediate key used for the actual encryption or for the encryption of the filenames? If yes how can I access it or make it available to the new remote?
+
+Thanks,
+Marek
diff --git a/doc/forum/Migrate_from_one_connection_mechanism_for_special_remote_to_another/comment_1_6e8e3350ae26b6c59daa56b7e86cc46a._comment b/doc/forum/Migrate_from_one_connection_mechanism_for_special_remote_to_another/comment_1_6e8e3350ae26b6c59daa56b7e86cc46a._comment
new file mode 100644
index 0000000000..e3a63b3f91
--- /dev/null
+++ b/doc/forum/Migrate_from_one_connection_mechanism_for_special_remote_to_another/comment_1_6e8e3350ae26b6c59daa56b7e86cc46a._comment
@@ -0,0 +1,14 @@
+[[!comment format=mdwn
+ username="joey"
+ subject="""comment 1"""
+ date="2018-05-30T15:21:57Z"
+ content="""
+Yes, there is an intermediate encryption key.
+
+What may work is to use `git annex enableremote` on the original remote
+with `externaltype=rclone`, which will make it use the other program.
+
+Whether this will work depends on implementation details of both special
+remotes. If they use the same names/urls/whatever to access the data stored
+in them, it should work.
+"""]]
diff --git a/doc/forum/Migrate_from_one_connection_mechanism_for_special_remote_to_another/comment_2_db4b772371ee49c743605c728e23c91a._comment b/doc/forum/Migrate_from_one_connection_mechanism_for_special_remote_to_another/comment_2_db4b772371ee49c743605c728e23c91a._comment
new file mode 100644
index 0000000000..36fccdd80f
--- /dev/null
+++ b/doc/forum/Migrate_from_one_connection_mechanism_for_special_remote_to_another/comment_2_db4b772371ee49c743605c728e23c91a._comment
@@ -0,0 +1,19 @@
+[[!comment format=mdwn
+ username="marek@33e8ba4fbc201af14a2badcc0656024401f5c916"
+ nickname="marek"
+ avatar="http://cdn.libravatar.org/avatar/65a60e8f5183feeeef8cef815bf73e61"
+ subject="Fixed a first problem and then ran into bigger problems"
+ date="2018-05-31T09:29:30Z"
+ content="""
+By using 
+    git show git-annex:remote.log
+
+one can extract the cipher from the old remote and set it with
+
+    git annex enableremote $newremote cipher=$CIPHER
+
+However, then I ran into a new problem. The git-annex-remote-hubic special remote created files with leading slashes (/GPGHMACSHA1--00021a41c3143f1e52f16af93625d1072e861543).
+Rclone, and I don`t even mean git-annex-remote-rclone, cannot access these files. It interprets the slash as directory separator which leads to those files not being found.
+
+I guess I will just reupload to fix the slash error in the first place. Or find a Hubic client that can deal with slashes and rename the objects remotely.
+"""]]
diff --git a/doc/forum/Missing_git-annex.linux__47__runshell.mdwn b/doc/forum/Missing_git-annex.linux__47__runshell.mdwn
new file mode 100644
index 0000000000..3ad7e85c13
--- /dev/null
+++ b/doc/forum/Missing_git-annex.linux__47__runshell.mdwn
@@ -0,0 +1,44 @@
+Hi,
+
+I've said up two clients to sync locally as well as over ssh-rsync.
+
+However, locally one client is complaining about missing `runshell` at a location where `git annex` is not installed.  The log is below.  On the assistent overview it just says "unfinished repository".  Note that I am able to ssh into the other machine and from the other machine I'm also able to ssh into this machine.
+
+
+I'm using the latest prebuild binary package (git-annex-bin at Archlinux AUR).  So my `git annex` setup contains 
+
+    $> pacman -Ql git-annex-bin 
+    git-annex-bin /usr/
+    git-annex-bin /usr/bin/
+    git-annex-bin /usr/bin/git-annex
+    git-annex-bin /usr/bin/git-annex-shell
+
+    $> pacman -Qi git-annex-bin # with chops
+    Name           : git-annex-bin
+    Version        : 4.20130909-1
+    Description    : Precompiled version of git-annex, webapp and assistant
+    included.
+    Architecture   : x86_64
+
+Here is a log of one event where it fails to transfer.  
+
+    [2013-09-28 18:06:38 CEST] TransferScanner: queued Upload NoUUID config/emacs.d/ac-l-dict/amsmath-c-a-* Nothing : expensive scan found missing object
+    [2013-09-28 18:06:38 CEST] Transferrer: Transferring: Upload NoUUID config/dotfiles/stardict/dic/stardict-oxford-2.4.2/oxford.ifo Nothing
+    [2013-09-28 18:06:38 CEST] call: git-annex ["transferkeys","--readfd","96","--writefd","94"]
+    [2013-09-28 18:06:38 CEST] read: git ["--git-dir=/home/rasmus/annex/.git","--work-tree=/home/rasmus/annex","show-ref","git-annex"]
+    [2013-09-28 18:06:38 CEST] read: git ["--git-dir=/home/rasmus/annex/.git","--work-tree=/home/rasmus/annex","show-ref","--hash","refs/heads/git-annex"]
+    [2013-09-28 18:06:38 CEST] read: git ["--git-dir=/home/rasmus/annex/.git","--work-tree=/home/rasmus/annex[","log2","0r1e3f-s/0h9e-ad2s8/ g1i8t-annex..90df78a0910a6f2998655e6:06:38 CEST] 127.0.0.1 GET /transfers/NotificationId%201 Mozilla/5.0 (X11; Linux x86_64; rv:26.0) Gecko/20100101 Firefox/26.0 Aurora/26.0a2
+    ea47d478ea0589b54","--oneline","-n1"]
+    [2013-09-28 18:06:38 CEST] read: git ["--git-dir=/home/rasmus/annex/.git","--work-tree=/home/rasmus/annex","log","refs/heads/git-annex..5f6870ed24e5ded1764765bbfef2b85aff046569","--oneline","-n1"]
+    [2013-09-28 18:06:38 CEST] chat: git ["--git-dir=/home/rasmus/annex/.git","--work-tree=/home/rasmus/annex","cat-file","--batch"]
+    [2013-09-28 18:06:38 CEST] read: ssh ["-S","/home/rasmus/annex/.git/annex/ssh/00b8ef4cb08290718ba625d9bd86ca0b","-o","ControlMaster=auto","-o","ControlPersist=yes","-T","rasmus@git-annex-192.168.1.107-rasmus_annex","git-annex-shell 'configlist' '/~/annex/'"]
+    /home/rasmus/.ssh/git-annex-shell: line 4: /opt/git-annex.linux/runshell: No such file or directory
+    [2013-09-28 18:06:38 CEST] call: git ["--git-dir=/home/rasmus/annex/.git","--work-tree=/home/rasmus/annex","fetch","--quiet","192.168.1.107_annex"]
+    [2013-09-28 18:06:38 CEST] 127.0.0.1 GET /transfers/NotificationId%201 Mozilla/5.0 (X11; Linux x86_64; rv:26.0) Gecko/20100101 Firefox/26.0 Aurora/26.0a2
+    [2013-09-28 18:06:38 CEST] 127.0.0.1 GET /log Mozilla/5.0 (X11; Linux x86_64; rv:26.0) Gecko/20100101 Firefox/26.0 Aurora/26.0a2
+    /home/rasmus/.ssh/git-annex-shell: line 4: /opt/git-annex.linux/runshell: No such file or directory
+    fatal: Could not read from remote repository.
+
+    Please make sure you have the correct access rights
+    and the repository exists.
+    git-annex: Unknown UUID
diff --git a/doc/forum/Missing_git-annex.linux__47__runshell/comment_1_f29a5105649579ef15e79d983c4e1f8e._comment b/doc/forum/Missing_git-annex.linux__47__runshell/comment_1_f29a5105649579ef15e79d983c4e1f8e._comment
new file mode 100644
index 0000000000..a10c4aaf0f
--- /dev/null
+++ b/doc/forum/Missing_git-annex.linux__47__runshell/comment_1_f29a5105649579ef15e79d983c4e1f8e._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="http://joeyh.name/"
+ ip="4.153.8.80"
+ subject="comment 1"
+ date="2013-09-30T16:08:40Z"
+ content="""
+You apparently at one point installed git-annex from the prebuilt tarball, in `/opt/git-annex.linux/`. It set up a ~/.ssh/git-annex-shell that points to this location. If you no longer have it installed that way, you can remove that file.
+"""]]
diff --git a/doc/forum/Modification_time_of_files_retained_in_synchronized_remote_copies__63__.mdwn b/doc/forum/Modification_time_of_files_retained_in_synchronized_remote_copies__63__.mdwn
new file mode 100644
index 0000000000..3cd519a6db
--- /dev/null
+++ b/doc/forum/Modification_time_of_files_retained_in_synchronized_remote_copies__63__.mdwn
@@ -0,0 +1,5 @@
+I've noticed how, when files are synchronized/transferred from one repository (annex?) to another by the assistant, the files in the «remote» repository do no retain the modification time of the original files.
+From what I know git does not save file-specific metadata (ownership/timestamps/etc.).
+Since I've just started using git-annex, and I'd very much like for backup copies to retain the mtime of the originals, I would like to ask whether I am experiencing a malfunction or whether this is expected behaviour.
+
+Regards
diff --git a/doc/forum/Modification_time_of_files_retained_in_synchronized_remote_copies__63__/comment_1_2b13584998108af0522b898c5d396ba4._comment b/doc/forum/Modification_time_of_files_retained_in_synchronized_remote_copies__63__/comment_1_2b13584998108af0522b898c5d396ba4._comment
new file mode 100644
index 0000000000..1d5bf00a5f
--- /dev/null
+++ b/doc/forum/Modification_time_of_files_retained_in_synchronized_remote_copies__63__/comment_1_2b13584998108af0522b898c5d396ba4._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="https://www.google.com/accounts/o8/id?id=AItOawmK0703vNSIQsP1mGf-4MAPnsBZiSc6yVo"
+ nickname="Emre"
+ subject="comment 1"
+ date="2014-09-19T21:31:23Z"
+ content="""
+I just saw your post and agree with you. Maybe we shall create a bug report. I would also need to keep my files in original timestamps. (may not mind about permissions though).
+"""]]
diff --git a/doc/forum/Modus_operandi_for_local_full_backup.mdwn b/doc/forum/Modus_operandi_for_local_full_backup.mdwn
new file mode 100644
index 0000000000..587c188529
--- /dev/null
+++ b/doc/forum/Modus_operandi_for_local_full_backup.mdwn
@@ -0,0 +1,20 @@
+Hello, 
+I've been for several hours soaking up on git annex and I don't see how to efficiently achieve the following. Your thoughts welcome.
+
+Let's say I have a computer with a main disk and a backup disk. I currently use rsnapshot to rsync from main to backup and take periodic snapshots. Once the backup disk is filled up I archive it and I start with a fresh one.
+
+This is neat but for the lack of intelligent history. I'm contemplating moving to a VCS solution so changes through time can be tracked more easily.
+
+Let's say I want to move to git-annex as a way of keeping history, maybe even across backup disks. The common part would be to rsync from main to backup and then commit. I see the following showstoppers:
+
+1) If I put the backup repo in direct mode, past versions of files are lost since they never enter either plain git nor annex control.
+
+2) If I put the backup repo in indirect mode, rsync destroys all soft links resulting in unnecessary copying over. It also seems to cause further breakage when a regular file appears where there was a symlink. I know no rsync option to sync through links to the target files (rsync -K only works for folders).
+
+3) I could put the repo in direct mode, rsync, commit, put in indirect to save versions, and repeat at every backup. Seems kinda inefficient, there are hundreds of gigas in files.
+
+4) I can't have the main disk under annex and use the back disk as a remote because the main disk contains many git repos (whose history I don't mind losing with rsync -C, or I wouldn't consider git-annex for backup at all). Also I'm not partial to populate the main disk with something only related to backup.
+
+If rsync -K worked with files there would be no problem. I'm missing the right flag here? Or something else that can be done on the git-annex side?
+
+Thanks in advance.
diff --git a/doc/forum/Modus_operandi_for_local_full_backup/comment_1_fa8ac55ebd94312a36b284047b6cf932._comment b/doc/forum/Modus_operandi_for_local_full_backup/comment_1_fa8ac55ebd94312a36b284047b6cf932._comment
new file mode 100644
index 0000000000..b6cc7e9896
--- /dev/null
+++ b/doc/forum/Modus_operandi_for_local_full_backup/comment_1_fa8ac55ebd94312a36b284047b6cf932._comment
@@ -0,0 +1,9 @@
+[[!comment format=mdwn
+ username="https://id.koumbit.net/anarcat"
+ subject="why use rsync at all?"
+ date="2015-11-11T01:13:00Z"
+ content="""
+i basically exclude git-annex repositories from my backups now. i have a `numcopies` setting to a satisfactory number (say 2 or 3) that ensures that there are at least 2 copies of every file. then i send copies off to external drives, when they are full, i create a new repository on a new drive.
+
+old versions are kept as long as you don't `dropunused`. 
+"""]]
diff --git a/doc/forum/Modus_operandi_for_local_full_backup/comment_2_fc2ac75608e91018f67c3753e9b09f76._comment b/doc/forum/Modus_operandi_for_local_full_backup/comment_2_fc2ac75608e91018f67c3753e9b09f76._comment
new file mode 100644
index 0000000000..5556f6697a
--- /dev/null
+++ b/doc/forum/Modus_operandi_for_local_full_backup/comment_2_fc2ac75608e91018f67c3753e9b09f76._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="alejandro@2e9e229fdf45d44cd300a5681432552693d458ab"
+ nickname="alejandro"
+ subject="comment 2"
+ date="2015-11-11T08:29:09Z"
+ content="""
+Hi anarcat, if I understand you correctly you're using method 4) in my original post. That is, having the source of backups as a git annex repo. However, all my readings on this site seem to indicate poor interaction with nested git repositories, that I would have. Isn't that so?
+"""]]
diff --git a/doc/forum/Modus_operandi_for_local_full_backup/comment_3_6089f53d8af5746f2a410d0214567943._comment b/doc/forum/Modus_operandi_for_local_full_backup/comment_3_6089f53d8af5746f2a410d0214567943._comment
new file mode 100644
index 0000000000..f388d57b85
--- /dev/null
+++ b/doc/forum/Modus_operandi_for_local_full_backup/comment_3_6089f53d8af5746f2a410d0214567943._comment
@@ -0,0 +1,15 @@
+[[!comment format=mdwn
+ username="https://id.koumbit.net/anarcat"
+ subject="comment 3"
+ date="2015-11-11T14:53:11Z"
+ content="""
+that is so: git annex doesn't \"backup\" very well other git repositories. it's probably possible, but that's not really what git-annex is designed for. git-annex is [[not]] a backup system: it's more like an archival system. so if you want to backup your whole system and keep history, the [[not]] page recommends [[bup]], but i would also recommend [borg](http://borgbackup.readthedocs.org/).
+
+if you have git repos: backup those using your regular backup system, git-annex won't help you there. you can clone the repos using git, use rsync and whatnot. git itself keeps history and unless you royally mess up, git is pretty forgiving.
+
+for large files: create topical git-annex repositories. i have one for \"music\", \"video\", \"podcasts\", and so on. those are managed on a per-repo basis.
+
+so when i do my \"backups\" (which include \"archiving\" as well now), i connect the hard drives in my computer, run the `borg` (or `bup`) backup and then run `git annex sync --content` on all my git-annex repositories. that way new files in the git-annex repositories are synced to the external drive and files not managed by git-annex (yes, including non-git-annex git repos) are backed up as well.
+
+hope that helps
+"""]]
diff --git a/doc/forum/Modus_operandi_for_local_full_backup/comment_4_4c27aae1361465993fb60997bb0eadf8._comment b/doc/forum/Modus_operandi_for_local_full_backup/comment_4_4c27aae1361465993fb60997bb0eadf8._comment
new file mode 100644
index 0000000000..7daf4a7a23
--- /dev/null
+++ b/doc/forum/Modus_operandi_for_local_full_backup/comment_4_4c27aae1361465993fb60997bb0eadf8._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="alejandro@2e9e229fdf45d44cd300a5681432552693d458ab"
+ nickname="alejandro"
+ subject="comment 4"
+ date="2015-11-11T15:22:05Z"
+ content="""
+I see, so your comments basically match my grasp on the situation. Thanks for the borg pointer, I have been reading on bup too and from posts in the mailing list I'm afraid it might not be mature enough for my particular use case.
+"""]]
diff --git a/doc/forum/Move_files_in_git_annex___40__direct_mode__41__.mdwn b/doc/forum/Move_files_in_git_annex___40__direct_mode__41__.mdwn
new file mode 100644
index 0000000000..7974a3e42f
--- /dev/null
+++ b/doc/forum/Move_files_in_git_annex___40__direct_mode__41__.mdwn
@@ -0,0 +1,22 @@
+Hi guys, I have a git annex repo under `/path`.
+
+I ran:
+
+```
+cd /path/subdir2
+mv ../subdir1/stuff* .
+git annex add
+git annex sync
+```
+
+Then I went to another repo (clone of the first) and did:
+
+```
+cd /paht1
+git annex sync
+git annex get .
+```
+
+Now I got the stuff from under `subdir2` in my second repo, just that the same stuff is also still under `subdir1` - how do I make git-annex aware of this change?
+
+Also, someone suggested running `git annex watch`
diff --git a/doc/forum/Move_files_in_git_annex___40__direct_mode__41__/comment_1_6426aca2296b5e03b333d5253ae7d88c._comment b/doc/forum/Move_files_in_git_annex___40__direct_mode__41__/comment_1_6426aca2296b5e03b333d5253ae7d88c._comment
new file mode 100644
index 0000000000..4482ac4096
--- /dev/null
+++ b/doc/forum/Move_files_in_git_annex___40__direct_mode__41__/comment_1_6426aca2296b5e03b333d5253ae7d88c._comment
@@ -0,0 +1,12 @@
+[[!comment format=mdwn
+ username="joey"
+ subject="""comment 1"""
+ date="2016-01-29T17:39:13Z"
+ content="""
+Well, that should work; the files should get moved in the second clone
+automatically when it's synced.
+
+Just tested it, seems to work ok.
+
+What does `git annex status` say in the second repo?
+"""]]
diff --git a/doc/forum/Move_unsynced_file_in_direct_mode.mdwn b/doc/forum/Move_unsynced_file_in_direct_mode.mdwn
new file mode 100644
index 0000000000..015b5f58e7
--- /dev/null
+++ b/doc/forum/Move_unsynced_file_in_direct_mode.mdwn
@@ -0,0 +1,97 @@
+When I rename unsynced files in a direct mode repo, the original symlink gets removed from git, but the new symlink doesn't get added back by autocommit or by explicitly using `git annex add`.
+
+First, I create a file in a git-annex repo:
+
+        $ mkdir annex1
+        $ cd annex1
+        $ git init
+        Initialized empty Git repository in /home/cwarden/annex1/.git/
+        $ git annex init
+        init  ok
+        (Recording state in git...)
+        $ echo test > test1
+        $ git annex add test1
+        add test1 ok
+        (Recording state in git...)
+        $ git annex sync
+        commit  ok
+        $ ls -l
+        total 4
+        lrwxrwxrwx 1 cwarden cwarden 178 Sep 12 10:14 test1 -> .git/annex/objects/w8/pv/SHA256E-s5--f2ca1bb6c7e907d06dafe4687e579fce76b37e4e93b7605022da52e6ccc26fd2/SHA256E-s5--f2ca1bb6c7e907d06dafe4687e579fce76b37e4e93b7605022da52e6ccc26fd2
+        $ cat test1
+        test
+
+Now, I clone the repo and enable autocommit and direct mode in the second repo:
+
+        $ cd ..
+        $ git clone annex1 annex2
+        Cloning into 'annex2'...
+        done.
+        $ cd annex2
+        $ git config annex.autocommit true
+        $ git annex direct
+        commit
+        On branch master
+        Your branch is up-to-date with 'origin/master'.
+
+        nothing to commit, working directory clean
+        ok
+        direct  ok
+
+I drop the file, then rename it:
+
+        $ git annex drop test1
+        (merging origin/git-annex into git-annex...)
+        (Recording state in git...)
+        $ mv test1 test2
+        $ ls -l
+        total 4
+        lrwxrwxrwx 1 cwarden cwarden 178 Sep 12 10:14 test2 -> .git/annex/objects/w8/pv/SHA256E-s5--f2ca1bb6c7e907d06dafe4687e579fce76b37e4e93b7605022da52e6ccc26fd2/SHA256E-s5--f2ca1bb6c7e907d06dafe4687e579fce76b37e4e93b7605022da52e6ccc26fd2
+        $ git annex sync
+        commit  (Recording state in git...)
+        ok
+        pull origin
+        ok
+        push origin
+        Counting objects: 6, done.
+        Delta compression using up to 4 threads.
+        Compressing objects: 100% (5/5), done.
+        Writing objects: 100% (6/6), 674 bytes | 0 bytes/s, done.
+        Total 6 (delta 1), reused 0 (delta 0)
+        To /home/cwarden/annex1
+                2772756..ffcb7a1  annex/direct/master -> synced/master
+         * [new branch]      git-annex -> synced/git-annex
+        ok
+        (Recording state in git...)
+
+Now, I want to get the renamed file:
+
+        $ git annex get test2
+        $ ls -l
+        total 4
+        lrwxrwxrwx 1 cwarden cwarden 178 Sep 12 10:14 test2 -> .git/annex/objects/w8/pv/SHA256E-s5--f2ca1bb6c7e907d06dafe4687e579fce76b37e4e93b7605022da52e6ccc26fd2/SHA256E-s5--f2ca1bb6c7e907d06dafe4687e579fce76b37e4e93b7605022da52e6ccc26fd2
+        $ cat test2
+        cat: test2: No such file or directory
+
+Explicitly adding test2 doesn't work:
+
+        $ git annex add test2
+        $ git annex get test2
+        $ cat test2
+        cat: test2: No such file or directory
+
+`git annex fsck` doesn't help:
+
+        $ git annex fsck
+        $ cat test2
+        cat: test2: No such file or directory
+
+The only way I've found to get the renamed file back into git is to use `core.bare=false`, but the [documentation](http://git-annex.branchable.com/direct_mode/) says that "there should be no good reason to need to do this, ever".
+
+        $ git -c core.bare=false add test2
+        $ git -c core.bare=false commit -m'force renamed file back into git'
+        $ git annex get test2
+        $ cat test2
+        test
+
+Is there a better solution?
diff --git a/doc/forum/Move_unsynced_file_in_direct_mode/comment_1_12a797cba753168dfde9e6339c00f481._comment b/doc/forum/Move_unsynced_file_in_direct_mode/comment_1_12a797cba753168dfde9e6339c00f481._comment
new file mode 100644
index 0000000000..73a164da26
--- /dev/null
+++ b/doc/forum/Move_unsynced_file_in_direct_mode/comment_1_12a797cba753168dfde9e6339c00f481._comment
@@ -0,0 +1,10 @@
+[[!comment format=mdwn
+ username="http://joeyh.name/"
+ ip="108.236.230.124"
+ subject="comment 1"
+ date="2014-09-18T18:27:37Z"
+ content="""
+Well, you can run `git annex assistant` or `git annex watch` and it will automatically notice the moved file and commit it. I think this is what you were trying to do when you set annex.autocommit to true (which is the default so accomplished nothing).
+
+But your example does show a bug: `git annex add` should add the dangling symlink to git in direct mode, as it already does in indirect mode. Fixed in [[!commit 44e7d6e1fe6e13091adbd572f66412e3601df3c5]].
+"""]]
diff --git a/doc/forum/Move_unsynced_file_in_direct_mode/comment_2_f3aec24668c35780a033f2b035df10ee._comment b/doc/forum/Move_unsynced_file_in_direct_mode/comment_2_f3aec24668c35780a033f2b035df10ee._comment
new file mode 100644
index 0000000000..971e70cdca
--- /dev/null
+++ b/doc/forum/Move_unsynced_file_in_direct_mode/comment_2_f3aec24668c35780a033f2b035df10ee._comment
@@ -0,0 +1,20 @@
+[[!comment format=mdwn
+ username="xn"
+ ip="71.59.214.243"
+ subject="comment 2"
+ date="2014-09-18T19:03:30Z"
+ content="""
+Thanks for tracking down that bug and for clearing up my confusion about `annex.autocommit`, Joey.
+
+I didn't realize `annex.autocommit=true` is only used by `git annex assistant` and `git annex watch`.  I thought that running `git annex sync` with `annex.autocommit=true` would also commit the change.
+
+A few small changes to `git-annex(1)` could clarify:
+
+    sync [remote ...]
+         ...
+         The sync process involves first committing all local *staged* changes...
+
+    annex.autocommit
+       Set to false to prevent git-annex assistant and *git-annex watch* from automatically committing changes to files in the repository.
+
+"""]]
diff --git a/doc/forum/Moving_large_files_within_the_repo_without_copying___63__.mdwn b/doc/forum/Moving_large_files_within_the_repo_without_copying___63__.mdwn
new file mode 100644
index 0000000000..55d51f21df
--- /dev/null
+++ b/doc/forum/Moving_large_files_within_the_repo_without_copying___63__.mdwn
@@ -0,0 +1,19 @@
+Is there a way to move a large file without "git annex unlock"ing it (Which takes very long for copying and then rehashing the file)?
+Using a simple "mv" or "git mv" results in broken symlinks, if the target directory is on a different hierarchy level:
+
+Example:
+
+Initial State:
+
+    $ ls -Hl Pictures:
+    Pictures/showImages.jpg -> ../.git/annex/objects/90/32/SHA1-s8737--a8bfb285d0ae394cb75c86f1eb9f703fb678a51e/SHA1-s8737--a8bfb285d0ae394cb
+
+Move: 
+
+    $ mv Pictures/showImages.jpg .
+
+Result: 
+
+    $ ls -H showImages.jpg
+    ls: cannot access showImages.jpg: No such file or directory
+    
diff --git a/doc/forum/Moving_large_files_within_the_repo_without_copying___63__/comment_1_9e3290138133d5a23a80f72342f47ec4._comment b/doc/forum/Moving_large_files_within_the_repo_without_copying___63__/comment_1_9e3290138133d5a23a80f72342f47ec4._comment
new file mode 100644
index 0000000000..d1df9cf514
--- /dev/null
+++ b/doc/forum/Moving_large_files_within_the_repo_without_copying___63__/comment_1_9e3290138133d5a23a80f72342f47ec4._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="https://www.google.com/accounts/o8/id?id=AItOawn7gQ1zZDdWhXy9H51W2krZYShNmKL3qfM"
+ nickname="Karsten"
+ subject="comment 1"
+ date="2012-11-28T13:28:21Z"
+ content="""
+Just commit, git-annex will fix the symlinks. Or do it manually with 'git annex fix'.
+"""]]
diff --git a/doc/forum/Moving_large_files_within_the_repo_without_copying___63__/comment_2_232b77894dda51d02cbc34bd25d3213b._comment b/doc/forum/Moving_large_files_within_the_repo_without_copying___63__/comment_2_232b77894dda51d02cbc34bd25d3213b._comment
new file mode 100644
index 0000000000..0559e79543
--- /dev/null
+++ b/doc/forum/Moving_large_files_within_the_repo_without_copying___63__/comment_2_232b77894dda51d02cbc34bd25d3213b._comment
@@ -0,0 +1,13 @@
+[[!comment format=mdwn
+ username="https://www.google.com/accounts/o8/id?id=AItOawlgyVag95OnpvSzQofjyX0WjW__MOMKsl0"
+ nickname="Sehr"
+ subject="Does not work"
+ date="2012-11-28T14:50:16Z"
+ content="""
+    $ git annex fix 
+
+Does not change anything. and
+
+    $ git annex fix showImages.jpg
+    git-annex: showImages.jpg not found
+"""]]
diff --git a/doc/forum/Moving_large_files_within_the_repo_without_copying___63__/comment_3_d35ac1bdb3fa6e303ad92348ba174158._comment b/doc/forum/Moving_large_files_within_the_repo_without_copying___63__/comment_3_d35ac1bdb3fa6e303ad92348ba174158._comment
new file mode 100644
index 0000000000..d4d25a7056
--- /dev/null
+++ b/doc/forum/Moving_large_files_within_the_repo_without_copying___63__/comment_3_d35ac1bdb3fa6e303ad92348ba174158._comment
@@ -0,0 +1,11 @@
+[[!comment format=mdwn
+ username="https://www.google.com/accounts/o8/id?id=AItOawn7gQ1zZDdWhXy9H51W2krZYShNmKL3qfM"
+ nickname="Karsten"
+ subject="comment 3"
+ date="2012-11-28T15:03:42Z"
+ content="""
+In my tests 'git annex fix' works only on symlinks in directories alredy known to git. So if you created a new directory inside your annex, be sure to 'git add' the directory before running 'git annex fix'.
+
+HTH,
+Karsten
+"""]]
diff --git a/doc/forum/Moving_large_files_within_the_repo_without_copying___63__/comment_4_4b443ec6b47eaabe214d0c2222083e4a._comment b/doc/forum/Moving_large_files_within_the_repo_without_copying___63__/comment_4_4b443ec6b47eaabe214d0c2222083e4a._comment
new file mode 100644
index 0000000000..cf09b261ae
--- /dev/null
+++ b/doc/forum/Moving_large_files_within_the_repo_without_copying___63__/comment_4_4b443ec6b47eaabe214d0c2222083e4a._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="https://www.google.com/accounts/o8/id?id=AItOawlgyVag95OnpvSzQofjyX0WjW__MOMKsl0"
+ nickname="Sehr"
+ subject="comment 4"
+ date="2012-11-28T15:14:49Z"
+ content="""
+Thanks. It seems that the file itself has to be added to git annex to be able to bet git annex fixed!
+"""]]
diff --git a/doc/forum/Moving_older_version__39__s_file_content_without_doing_checkout.mdwn b/doc/forum/Moving_older_version__39__s_file_content_without_doing_checkout.mdwn
new file mode 100644
index 0000000000..3ed022b481
--- /dev/null
+++ b/doc/forum/Moving_older_version__39__s_file_content_without_doing_checkout.mdwn
@@ -0,0 +1,4 @@
+Hi,
+Is there any way I can move or copy file content of older version without doing checkout to that version, by passing commit hash as parameter in move command itself? 
+
+Thank you
diff --git a/doc/forum/Moving_older_version__39__s_file_content_without_doing_checkout/comment_1_f114b75b29123453758b493fae7f5167._comment b/doc/forum/Moving_older_version__39__s_file_content_without_doing_checkout/comment_1_f114b75b29123453758b493fae7f5167._comment
new file mode 100644
index 0000000000..a53c6bbd69
--- /dev/null
+++ b/doc/forum/Moving_older_version__39__s_file_content_without_doing_checkout/comment_1_f114b75b29123453758b493fae7f5167._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="https://www.google.com/accounts/o8/id?id=AItOawnpdM9F8VbtQ_H5PaPMpGSxPe_d5L1eJ6w"
+ nickname="Rafael"
+ subject="comment 1"
+ date="2012-05-15T07:59:57Z"
+ content="""
+I had a similiar question in forum/new_microfeatures/. I would like to fetch/copy all the annexed content from a repo, be it on the current branch, another branch, or corresponds to an old version of a file. A command like \"git annex copy --all --from=source [path]\" would then ensure I have access to all the content I need even if I have later no longer access to source. Sure I could use rsync.
+"""]]
diff --git a/doc/forum/Moving_older_version__39__s_file_content_without_doing_checkout/comment_2_e377b7614c2961b460a10e285f3db274._comment b/doc/forum/Moving_older_version__39__s_file_content_without_doing_checkout/comment_2_e377b7614c2961b460a10e285f3db274._comment
new file mode 100644
index 0000000000..18ff153ad0
--- /dev/null
+++ b/doc/forum/Moving_older_version__39__s_file_content_without_doing_checkout/comment_2_e377b7614c2961b460a10e285f3db274._comment
@@ -0,0 +1,10 @@
+[[!comment format=mdwn
+ username="http://joeyh.name/"
+ ip="4.153.2.51"
+ subject="comment 2"
+ date="2012-05-15T17:00:10Z"
+ content="""
+Yes, I think that [[todo/add_-all_option]] is the right approach for this. Seems unlikely you'd have some files' hashes handy without having them checked out, but operating on all content makes sense.
+
+That page discusses some problems implementing it for some commands, but should not pose a problem for `move`. It would also be possible to support `get` and `copy`, except `--auto` couldn't be used with `--all`. Even `fsck` could support it.
+"""]]
diff --git a/doc/forum/Moving_older_version__39__s_file_content_without_doing_checkout/comment_3_d251958795ab0867c65cf182e54a6ffe._comment b/doc/forum/Moving_older_version__39__s_file_content_without_doing_checkout/comment_3_d251958795ab0867c65cf182e54a6ffe._comment
new file mode 100644
index 0000000000..63d1e68006
--- /dev/null
+++ b/doc/forum/Moving_older_version__39__s_file_content_without_doing_checkout/comment_3_d251958795ab0867c65cf182e54a6ffe._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="http://joeyh.name/"
+ ip="4.154.4.193"
+ subject="comment 3"
+ date="2013-07-03T18:10:29Z"
+ content="""
+Now the --all option is built. You can, for example, run `git annex copy --all --to backup`
+"""]]
diff --git a/doc/forum/Multiple_group_groupwanted_settings_help.mdwn b/doc/forum/Multiple_group_groupwanted_settings_help.mdwn
new file mode 100644
index 0000000000..49781d91fe
--- /dev/null
+++ b/doc/forum/Multiple_group_groupwanted_settings_help.mdwn
@@ -0,0 +1,34 @@
+Help me figure this out. I can't get it to work.
+
+Here's my hypothetical repo setup: a1 a2 b1 b2
+
+All of these are connected to each other.
+
+Settings are as follows:
+
+    group a1 = a 1
+    group a2 = a 2
+    group b1 = b 1
+    group b2 = b 2
+
+    wanted a1 = groupwanted
+    wanted a2 = groupwanted
+    wanted b1 = groupwanted
+    wanted b2 = groupwanted
+
+    groupwanted 1 = not copies=1:1
+    groupwanted 2 = not copies=2:1
+    groupwanted a = not copies=a:1
+    groupwanted b = not copies=b:1
+
+    numcopies default = 2
+    config annex.synccontent = true
+
+The behavior I want is: Group "a" repos will only have 1 copy of a file, and group "b" repos will only have 1 copy of a file.
+Similarly, group "1" repos will only have 1 copy of a file, and group "2" repos will only have 1 copy of a file.
+
+So, for example, a file could be stored in a1 and b2, or a2 and b1, but NOT a1 and a2 (both are group "a") or a1 and b1 (both are group "1").
+
+But it doesn't work. When I add a file to a1, it is not copied to b2 on sync. The documentation says repos can have multiple groups, but I'm not sure how the groupwanted settings of multiple groups work together.
+
+Is this even possible?
diff --git a/doc/forum/Multiple_group_groupwanted_settings_help/comment_1_e78d0aa6b23225c509b9e97549fe915b._comment b/doc/forum/Multiple_group_groupwanted_settings_help/comment_1_e78d0aa6b23225c509b9e97549fe915b._comment
new file mode 100644
index 0000000000..134f074a36
--- /dev/null
+++ b/doc/forum/Multiple_group_groupwanted_settings_help/comment_1_e78d0aa6b23225c509b9e97549fe915b._comment
@@ -0,0 +1,34 @@
+[[!comment format=mdwn
+ username="joey"
+ subject="""comment 1"""
+ date="2018-06-12T16:17:30Z"
+ content="""
+groupwanted expressions are only used when there is only one that the
+repository could use. The repository can be in two groups and if only one
+of the groups has a groupwanted expression, it will be used, but if both
+do, neither expression will be used. 
+
+This limitation to exactly one expression also holds for standard
+preferred content expressions (which is documented).
+git-annex does not try to combine together two expressions with OR.
+Why not? Well, consider the standard preferred content expressions for the 
+client and transfer groups. The one for transfer looks at whether client
+repositories have the content:
+`not (inallgroup=client and copies=client:2) ...`
+
+If a repository is both a transfer repository and a client at the same
+time, that would make it want to have content as long as not all clients
+contain the content -- so once all other clients get the content, the
+repository would want to drop its copy. But then, it would see that
+not all client repositories now have the content (since it doesn't!) and
+want to get it. And potentially round and round.. 
+
+git-annex actually detects and prevents this kind of drop/get cycle
+(probably), but the right behavior of combining those two groupwanted
+expressions is at best undefined. So it's better to have the user write
+down an expression with what they really want to happen than try to OR
+expressions.
+
+I've updated the docs for groupwanted to note that it's only used when one 
+of a repos's groups has it.
+"""]]
diff --git a/doc/forum/Multiple_group_groupwanted_settings_help/comment_2_b3d95d5c03c7b985d39a4d5a45ba3dec._comment b/doc/forum/Multiple_group_groupwanted_settings_help/comment_2_b3d95d5c03c7b985d39a4d5a45ba3dec._comment
new file mode 100644
index 0000000000..b981e34b32
--- /dev/null
+++ b/doc/forum/Multiple_group_groupwanted_settings_help/comment_2_b3d95d5c03c7b985d39a4d5a45ba3dec._comment
@@ -0,0 +1,27 @@
+[[!comment format=mdwn
+ username="ghen1"
+ avatar="http://cdn.libravatar.org/avatar/efd0e92b6198291138f0cd7aedbf86b6"
+ subject="comment 2"
+ date="2018-06-13T17:18:49Z"
+ content="""
+Thank you for clarifying. I've worked out a setup that seems to do what I want. The rules are a little messier, but not too bad:
+
+    group a1 = a 1 a1store
+    group a2 = a 2 a2store
+    group b1 = b 1 b1store
+    group b2 = b 2 b2store
+    
+    wanted a1 = groupwanted
+    wanted a2 = groupwanted
+    wanted b1 = groupwanted
+    wanted b2 = groupwanted
+    
+    groupwanted a1store = not copies=1:1 and not copies=a:1
+    groupwanted a2store = not copies=2:1 and not copies=a:1
+    groupwanted b1store = not copies=1:1 and not copies=b:1
+    groupwanted b2store = not copies=2:1 and not copies=b:1
+    
+    numcopies default = 2
+    config annex.synccontent = true
+
+"""]]
diff --git a/doc/forum/Multiple_interface_to_the_same_annex.mdwn b/doc/forum/Multiple_interface_to_the_same_annex.mdwn
new file mode 100644
index 0000000000..54f4ae2d91
--- /dev/null
+++ b/doc/forum/Multiple_interface_to_the_same_annex.mdwn
@@ -0,0 +1,29 @@
+I would like to get available the access to the annex of my repository as remote with rsync protocol (read only).
+The idea is not to have a redundant copy of the annex but to add an other option to connect to the annex.
+
+If I create an rsync remote I obtain something like
+
+	00000000-0000-0000-0000-000000000001 -- web
    
+
+ 	00000000-0000-0000-0000-000000000002 -- bit torrent
+
+ 	044cf7ac-d2b1-4f84-8bbc-747ac4659c8f -- rsync
+
+ 	a0e14705-ef07-4b8a-9a00-8724fd936a91 -- origin /my-repo/.git/annex
+
+The remote "rsync" points to the same location of the annex /my-repo/.git/annex
+
+If I add a file
+
+ 	git annex add file1
+
+and then I check where it is available
+
+ 	git annex whereis file 1
+
+I obtain that it is not available on 'rsync'
+
+Therefore if someone has a copy of the repository, without access to the origin annex,
+there is no way to use the additional access through 'rsync'
+
+What is the right way to configure the 'rsync' remote in such a way behave like an alias for the origin annex?
diff --git a/doc/forum/Multiple_interface_to_the_same_annex/comment_1_ea9e3a987112d8bf6421be234bf61d3c._comment b/doc/forum/Multiple_interface_to_the_same_annex/comment_1_ea9e3a987112d8bf6421be234bf61d3c._comment
new file mode 100644
index 0000000000..df21114bcb
--- /dev/null
+++ b/doc/forum/Multiple_interface_to_the_same_annex/comment_1_ea9e3a987112d8bf6421be234bf61d3c._comment
@@ -0,0 +1,15 @@
+[[!comment format=mdwn
+ username="joey"
+ subject="""comment 1"""
+ date="2016-12-13T16:55:21Z"
+ content="""
+Creating a separate special remote pointing to the same content is not a
+good idea. This will confuse git-annex's location tracking, and in some
+cases can lead to data loss, since git-annex will assume it can safely
+delete a file from one of the "two" repositories, since it thinks the
+"other" one will still have the content of the file.
+
+Instead, you need a way to configure a regular git remote,
+pointing at the repository, that is read-only, or is accessed over rsync,
+or whatever your requirements are. 
+"""]]
diff --git a/doc/forum/Multiple_machine_remotes_on_external_drive.mdwn b/doc/forum/Multiple_machine_remotes_on_external_drive.mdwn
new file mode 100644
index 0000000000..70c910d380
--- /dev/null
+++ b/doc/forum/Multiple_machine_remotes_on_external_drive.mdwn
@@ -0,0 +1,7 @@
+I have a question about remotes behavior on external drives.
+
+Say I have two machines A and B each with a git annex repository at `~/repo`, and an external drive C with the same repository at `/repo`.
+
+I can add C remotes for A and B at, for example, `/media/hdd/repo`, but I'm not certain how it works for C.
+
+Do I stick C into A and add a remote A at path `~/repo`, then stick C into B and add a remote B with the same path `~/repo`?  Does git annex handle this sanely?
diff --git a/doc/forum/Multiple_prefered_content_groups.mdwn b/doc/forum/Multiple_prefered_content_groups.mdwn
new file mode 100644
index 0000000000..f3b00fe91c
--- /dev/null
+++ b/doc/forum/Multiple_prefered_content_groups.mdwn
@@ -0,0 +1,66 @@
+Hello,
+
+I have a repo alpha
+
+```
+florian@asaru ~/git-annex % tree
+.
+└── alpha
+    ├── archive
+    │   └── some_archived_file
+    └── some_file
+```
+
+which is standard/client.
+
+Another repo beta, cloned from alpha is also standard/client. ```git annex sync --content``` makes beta get all the content. Perfect!
+
+Another repo delta, also cloned from alpha is:
+
+```
+% git annex wanted . && git annex group .
+standard
+archive backup
+```
+Manpage of git-annex-group says a repo could be in multiple groups.
+
+I expect the groups to be combined. archive makes the other repos drop content in archive/ when it has reached this repo, backup to also retain deleted files.
+
+But ```git annex sync --content``` gets no files:
+
+```
+florian@asaru ~/git-annex/delta (git)-[master] % git annex sync --content
+commit  
+Auf Branch master
+Ihr Branch ist auf dem selben Stand wie 'origin/master'.
+nichts zu committen, Arbeitsverzeichnis unverändert
+ok
+pull origin 
+ok
+```
+```
+florian@asaru ~/git-annex/delta (git)-[master] % git annex info          
+repository mode: indirect
+trusted repositories: 0
+semitrusted repositories: 5
+        00000000-0000-0000-0000-000000000001 -- web
+        00000000-0000-0000-0000-000000000002 -- bittorrent
+        19d4aafe-0d6d-4bb8-913d-6f0541a6ee11 -- Alpha [origin]
+        6da720cc-4583-429d-bde0-724b7f9e494a -- Delta [here]
+        8523418a-0c1b-4987-a048-c967fc5ec1e7 -- Beta
+untrusted repositories: 0
+transfers in progress: none
+available local disk space: 123.11 gigabytes (+1 megabyte reserved)
+local annex keys: 0
+local annex size: 0 bytes
+annexed files in working tree: 2
+size of annexed files in working tree: 2.59 megabytes
+bloom filter size: 32 mebibytes (0% full)
+backend usage: 
+        SHA256E: 2
+```
+
+What is wrong there? My mental model or some git-annex issue?
+
+Thanks,
+Florian
diff --git a/doc/forum/Multiple_prefered_content_groups/comment_1_b401d234c0730412053f0f6a6f9cf2d6._comment b/doc/forum/Multiple_prefered_content_groups/comment_1_b401d234c0730412053f0f6a6f9cf2d6._comment
new file mode 100644
index 0000000000..f47fe5eefc
--- /dev/null
+++ b/doc/forum/Multiple_prefered_content_groups/comment_1_b401d234c0730412053f0f6a6f9cf2d6._comment
@@ -0,0 +1,32 @@
+[[!comment format=mdwn
+ username="joey"
+ subject="""comment 1"""
+ date="2016-07-11T16:39:44Z"
+ content="""
+What git-annex actually does when a repository is in multiple groups
+is that the "standard" preferred content expression has no effect.
+Because this is an ambiguous configuration and it's not clear what you
+want.
+
+I don't think that you can combine preferred content expressions for
+multiple groups and always get behavior that makes sense. In particular,
+it seems likely that in some situations you'd want to combine them with AND
+but in other situations with OR.
+
+So, the best thing for you to do is probably to manually write a preferred
+content expression that clearly expresses what you want. Note that you
+can have a repository that's in the archive group, but instead of "standard"
+has its own custom preferred content expression. In your case, I think you
+want to do that; keeping the repository in the archive group will retain
+the behavior of other repositories dropping archived files once they reach
+it.
+
+Seems that for the preferred content setting for this repositry, you
+want the same one used for backup repositories, "anything".
+
+So:
+
+	git annex group . archive
+	git annex ungroup . backup
+	git anenx wanted . anything
+"""]]
diff --git a/doc/forum/Multiple_prefered_content_groups/comment_2_a91177f4f02500e8fa6594933e128ef0._comment b/doc/forum/Multiple_prefered_content_groups/comment_2_a91177f4f02500e8fa6594933e128ef0._comment
new file mode 100644
index 0000000000..84aef3011e
--- /dev/null
+++ b/doc/forum/Multiple_prefered_content_groups/comment_2_a91177f4f02500e8fa6594933e128ef0._comment
@@ -0,0 +1,11 @@
+[[!comment format=mdwn
+ username="Horus"
+ subject="comment 2"
+ date="2016-07-14T08:32:11Z"
+ content="""
+Ah, ok, then it makes sense.
+
+Maybe it's an idea to expand the standard expression for client groups in a way that they drop a file if it has reached a backup repo (additionally to a archive repo)?
+
+Thanks!
+"""]]
diff --git a/doc/forum/Multiple_remotes_with_the_same_path.mdwn b/doc/forum/Multiple_remotes_with_the_same_path.mdwn
new file mode 100644
index 0000000000..55d9ee1df6
--- /dev/null
+++ b/doc/forum/Multiple_remotes_with_the_same_path.mdwn
@@ -0,0 +1,67 @@
+This is a followup to  – I'm actually testing the "two repos at same URL" situation now, with git-annex 6.20160211 and 6.20160221.
+
+So I have a git-annex repo on two hosts at `rain:~/Attic/Software`, `frost:~/Attic/Software`, and a third clone at `/mnt/portable_HD/Attic/Software`. This means that the repo on the portable HD has two remotes with identical paths, but corresponding to different repositories:
+
+    rain    /home/grawity/Attic/Software
+    frost   /home/grawity/Attic/Software
+
+[I was told earlier](/forum/basic_usage_questions) that this configuration would work fine and git-annex would not get confused. However, that doesn't seem to be the case (unless I misunderstood what it considers to be the "right thing"?) I see that whenever I run a command like `git annex info` on the portable\_HD repo, it overrides `remote.{rain,frost}.annex-uuid` with whatever UUID it sees right now – resulting in `git annex info` output such as:
+
+    ┌ frost /run/media/grawity/vol4_grimoire/Attic/Software master
+    ┘ git config -l | grep annex-uuid
+    remote.origin.annex-uuid=0ebc2083-f95e-4637-bd3e-09db8471daf3
+    remote.rain.annex-uuid=3e342a37-6c35-40e5-99a1-1f140e6c363d    <--
+    remote.frost.annex-uuid=524b8690-9b3e-48d4-b1a8-c0edb35c1ccf   <--
+    remote.fs1.annex-uuid=ed83c81c-1a95-4acb-b76c-b6724fe88873
+    
+    ┌ frost /run/media/grawity/vol4_grimoire/Attic/Software master
+    ┘ git annex info --fast --verbose --debug
+    repository mode: indirect
+    trusted repositories: [2016-02-29 08:09:29.284136] read: git ["--git-dir=.git","--work-tree=.","--literal-pathspecs","show-ref","git-annex"]
+    [2016-02-29 08:09:29.286872] process done ExitSuccess
+    [2016-02-29 08:09:29.286954] read: git ["--git-dir=.git","--work-tree=.","--literal-pathspecs","show-ref","--hash","refs/heads/git-annex"]
+    [2016-02-29 08:09:29.289167] process done ExitSuccess
+    [2016-02-29 08:09:29.289316] read: git ["--git-dir=.git","--work-tree=.","--literal-pathspecs","log","refs/heads/git-annex..fc873fc472a8c5d0db3c91a3868d9adc072f7076","-n1","--pretty=%H"]
+    [2016-02-29 08:09:29.291836] process done ExitSuccess
+    [2016-02-29 08:09:29.291941] read: git ["--git-dir=.git","--work-tree=.","--literal-pathspecs","log","refs/heads/git-annex..23c27db2f2a02515ba39e7d0bb8653fa786b6aea","-n1","--pretty=%H"]
+    [2016-02-29 08:09:29.294762] process done ExitSuccess
+    [2016-02-29 08:09:29.294869] read: git ["--git-dir=.git","--work-tree=.","--literal-pathspecs","log","refs/heads/git-annex..690a90da7fc28be67d5053c0a6c3050cca614eaa","-n1","--pretty=%H"]
+    [2016-02-29 08:09:29.301018] process done ExitSuccess
+    [2016-02-29 08:09:29.30112] read: git ["--git-dir=.git","--work-tree=.","--literal-pathspecs","log","refs/heads/git-annex..94375fb892d1c9f5b70bdd5917c85c5c18a13168","-n1","--pretty=%H"]
+    [2016-02-29 08:09:29.3031] process done ExitSuccess
+    [2016-02-29 08:09:29.303182] read: git ["--git-dir=.git","--work-tree=.","--literal-pathspecs","log","refs/heads/git-annex..4a7f17e644e86776b510606beb6f3cb3819d8256","-n1","--pretty=%H"]
+    [2016-02-29 08:09:29.304916] process done ExitSuccess
+    [2016-02-29 08:09:29.305455] chat: git ["--git-dir=.git","--work-tree=.","--literal-pathspecs","cat-file","--batch"]
+    [2016-02-29 08:09:29.308946] read: git ["config","--null","--list"]
+    [2016-02-29 08:09:29.312347] process done ExitSuccess
+    [2016-02-29 08:09:29.312939] read: git ["config","--null","--list"]
+    [2016-02-29 08:09:29.3179] process done ExitSuccess
+    [2016-02-29 08:09:29.318494] call: git ["--git-dir=.git","--work-tree=.","--literal-pathspecs","config","remote.rain.annex-uuid","524b8690-9b3e-48d4-b1a8-c0edb35c1ccf"]
+    [2016-02-29 08:09:29.320648] process done ExitSuccess
+    [2016-02-29 08:09:29.320735] read: git ["config","--null","--list"]
+    [2016-02-29 08:09:29.327697] process done ExitSuccess
+    0
+    semitrusted repositories: 9
+    	00000000-0000-0000-0000-000000000001 -- web
+     	00000000-0000-0000-0000-000000000002 -- bittorrent
+     	0ebc2083-f95e-4637-bd3e-09db8471daf3 -- origin
+     	3e14eb23-1b16-4b52-8798-56efb550ab00 -- [vol4_grimoire]:/Attic/Software [here]
+     	3e342a37-6c35-40e5-99a1-1f140e6c363d -- grawity@rain:~/Attic/Software              <--
+     	524b8690-9b3e-48d4-b1a8-c0edb35c1ccf -- grawity@frost:~/Downloads/Software [rain]  <--
+     	b9c0c485-07e5-4166-b5ac-ba971faab98a -- [vol3_tombstone]:/Attic/Software
+     	c3c6dd39-ebc7-475a-9992-f99ca01a7f3a -- grawity@wolke:~/Attic/Software
+     	ed83c81c-1a95-4acb-b76c-b6724fe88873 -- fs1:/Attic/Software [fs1]
+    untrusted repositories: 0
+    transfers in progress: none
+    available local disk space: 920.96 gigabytes (+1 megabyte reserved)
+    
+    ┌ frost /run/media/grawity/vol4_grimoire/Attic/Software master
+    ┘ git config -l | grep annex-uuid
+    remote.origin.annex-uuid=0ebc2083-f95e-4637-bd3e-09db8471daf3
+    remote.rain.annex-uuid=524b8690-9b3e-48d4-b1a8-c0edb35c1ccf    <--
+    remote.frost.annex-uuid=524b8690-9b3e-48d4-b1a8-c0edb35c1ccf   <--
+    remote.fs1.annex-uuid=ed83c81c-1a95-4acb-b76c-b6724fe88873
+
+Notice how the `[rain]` remote tag now shows up next to the wrong remote. Among other things, **this means I cannot use `git annex find --in frost --not --in rain`** and similar commands (unless I specify all the UUIDs by hand). And even though I trust git-annex to not lose any data, this behavior is a bit confusing.
+
+Is it possible that git-annex could detect such situations and avoid updating an UUID if it's _already attached_ to another remote?
diff --git a/doc/forum/Multiple_remotes_with_the_same_path/comment_1_78f2d8e3906c11ba5753478d08973bd2._comment b/doc/forum/Multiple_remotes_with_the_same_path/comment_1_78f2d8e3906c11ba5753478d08973bd2._comment
new file mode 100644
index 0000000000..5f9f7426ae
--- /dev/null
+++ b/doc/forum/Multiple_remotes_with_the_same_path/comment_1_78f2d8e3906c11ba5753478d08973bd2._comment
@@ -0,0 +1,18 @@
+[[!comment format=mdwn
+ username="joey"
+ subject="""comment 1"""
+ date="2016-02-29T15:47:39Z"
+ content="""
+git-annex is not getting confused; it's looking at the current content of
+the remote and using that uuid. Failing to do so would constitute
+confusion.
+
+If you want the remote to only work when the drive is plugged into a single
+host, you need to make the remote path only reach a repository on that
+host. One way would be a symlink, eg `/home/grawity/Attic/rain-Software ->
+Software` on rain.
+
+Or, just use the repository description, eg
+`--in grawity@rain:~/Attic/Software`. Note that the description can be
+changed to something easier to type.
+"""]]
diff --git a/doc/forum/Multiple_remotes_with_the_same_path/comment_2_8cd3edf2e71e904f1b651abdfd7a4499._comment b/doc/forum/Multiple_remotes_with_the_same_path/comment_2_8cd3edf2e71e904f1b651abdfd7a4499._comment
new file mode 100644
index 0000000000..f18404a6d4
--- /dev/null
+++ b/doc/forum/Multiple_remotes_with_the_same_path/comment_2_8cd3edf2e71e904f1b651abdfd7a4499._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="grawity@2ea26be48562f66fcb9b66307da72b1e2e37453f"
+ nickname="grawity"
+ subject="comment 2"
+ date="2016-02-29T17:25:17Z"
+ content="""
+Hmm, I still think that avoiding duplicating uuids would be smarter behavior, but the host symlinks will do just fine. Thanks for the suggestion.
+"""]]
diff --git a/doc/forum/Multiple_repos_on_same_path.mdwn b/doc/forum/Multiple_repos_on_same_path.mdwn
new file mode 100644
index 0000000000..d90a9f0fb0
--- /dev/null
+++ b/doc/forum/Multiple_repos_on_same_path.mdwn
@@ -0,0 +1,12 @@
+I want to use git-annex to manage some data which I keep at, say, /home/my_user/data .
+I have multiple external storage devices for this repo, such as /run/media/my_user/data1 e
+
+BUT, I also have multiple machines, e.g. my desktop and my laptop. 
+These all have the data under /home/my_user/data, as I like to keep my paths consistent. 
+So, when I connect my external media to these machines, practically they see the same repo path, though these are different repos.
+
+1. Should I try to create multiple repos with the same path, or should I just add one
+
+2. Can I expect any major issues from this set-up?
+
+  
diff --git a/doc/forum/Multiple_repos_on_same_path/comment_1_67ed823c4c40f5acf12b48eb75d7afa8._comment b/doc/forum/Multiple_repos_on_same_path/comment_1_67ed823c4c40f5acf12b48eb75d7afa8._comment
new file mode 100644
index 0000000000..5ab93d9ed5
--- /dev/null
+++ b/doc/forum/Multiple_repos_on_same_path/comment_1_67ed823c4c40f5acf12b48eb75d7afa8._comment
@@ -0,0 +1,32 @@
+[[!comment format=mdwn
+ username="https://sunny256.wordpress.com/"
+ nickname="sunny256"
+ subject="Create symlink in the root directory"
+ date="2015-05-12T13:04:18Z"
+ content="""
+My way of dealing with this is to create a symlink in the root directory that point to the root directory, like
+
+[[!format sh \"\"\"
+$ cd /
+$ sudo ln -sv . compname
+[sudo] password for sunny: hunter2
+'compname' -> '.'
+$ ls -l compname
+lrwxrwxrwx 1 root root 1 May 12 14:52 compname -> .
+$
+\"\"\"]]
+
+where `compname` is the hostname for the computer. Now you can create paths like
+
+[[!format sh \"\"\"
+git remote add comp-a /comp-a/home/my_user/data/repo
+git remote add comp-b /comp-b/home/my_user/data/repo
+\"\"\"]]
+
+This is also useful in other scripts where fetching data from the right directory on the wrong computer is bad. Also, this is a cheap way for a script to determine which computer it's running on:
+
+[[!format sh \"\"\"
+test -L \"/comp-a\" && echo Running on computer comp-a
+\"\"\"]]
+
+"""]]
diff --git a/doc/forum/Multiple_repos_on_same_path/comment_2_e1c4ac71eb0f9ff6ae1701b3a6d738dd._comment b/doc/forum/Multiple_repos_on_same_path/comment_2_e1c4ac71eb0f9ff6ae1701b3a6d738dd._comment
new file mode 100644
index 0000000000..65a233c725
--- /dev/null
+++ b/doc/forum/Multiple_repos_on_same_path/comment_2_e1c4ac71eb0f9ff6ae1701b3a6d738dd._comment
@@ -0,0 +1,26 @@
+[[!comment format=mdwn
+ username="joey"
+ subject="""comment 2"""
+ date="2015-07-20T17:38:10Z"
+ content="""
+sunny256 has a nice approach with the symlinks to make the paths
+to the remores explicitly contain the machine name.
+
+However, if you want to keep it simple, it's perfectly fine for a
+remote's path to point to a directory which has different
+repositories mounted on it at different times. I do this in my own
+removable media drives, so the "host" remote uses /home/joey/sound,
+for example.
+
+This is safe to do because git-annex always checks the UUID of the
+remote before using it. For a local path like this, it will automatically
+update the cached annex-uuid of the remote when it finds a repo with a
+different UUID mounted. (For a path to a repo on a remote server, it uses
+other methods to verify that it's accessing the repo with the UUID it
+expected.)
+
+The only thing I'd be careful about doing is switching the repository that
+is mounted at a path with another one while git-annex is running, since it
+only checks at startup and won't notice the substitution and could get
+confused.
+"""]]
diff --git a/doc/forum/Multisession_compatible__63__.mdwn b/doc/forum/Multisession_compatible__63__.mdwn
new file mode 100644
index 0000000000..6c117a5ba5
--- /dev/null
+++ b/doc/forum/Multisession_compatible__63__.mdwn
@@ -0,0 +1,8 @@
+Thanks for the great tool!
+
+Is git-annex multisession compatible? I'm assuming that an "add" for example, will switch to the git-annex branch, commit data, and switch back to the previous branch. Is this protected by a Lock of some sort so that if two separate processes are interacting with the same repo/annex simultaneously there won't be any corruption?
+
+If not, is this something I could add within my python class that is implementing the interface to the Annex?
+
+Thanks,
+Mark
diff --git a/doc/forum/Multisession_compatible__63__/comment_1_dab5a92416314a7d4c02663d4e46cd60._comment b/doc/forum/Multisession_compatible__63__/comment_1_dab5a92416314a7d4c02663d4e46cd60._comment
new file mode 100644
index 0000000000..9dc228316a
--- /dev/null
+++ b/doc/forum/Multisession_compatible__63__/comment_1_dab5a92416314a7d4c02663d4e46cd60._comment
@@ -0,0 +1,11 @@
+[[!comment format=mdwn
+ username="joey"
+ subject="""comment 1"""
+ date="2016-02-04T16:30:30Z"
+ content="""
+git-annex has quite extensive locking, so it's entirely safe to run any
+combination of any git-annex commands at the same time.
+
+(It doesn't actually check out the git-annex branch when making commits to
+it BTW.)
+"""]]
diff --git a/doc/forum/Multisession_compatible__63__/comment_2_1c8d0f97d353c3e1d290edbe0a48b359._comment b/doc/forum/Multisession_compatible__63__/comment_2_1c8d0f97d353c3e1d290edbe0a48b359._comment
new file mode 100644
index 0000000000..1eedafdd54
--- /dev/null
+++ b/doc/forum/Multisession_compatible__63__/comment_2_1c8d0f97d353c3e1d290edbe0a48b359._comment
@@ -0,0 +1,16 @@
+[[!comment format=mdwn
+ username="mark@6b90344cdab3158eacb94a3944460d138afc9bef"
+ nickname="mark"
+ subject="comment 2"
+ date="2016-02-04T18:23:07Z"
+ content="""
+Thanks Joey for the response.
+
+This brings up two questions. 
+
+1) In my setup I have my own \"sync\" of just the git-annex branch with the remote after git annex operations. (I didn't want the master branch synch of \"git annex sync\". In this sync of mine I switch branches to do a pull/push on the git-annex branch. I found that I had to put file locks around all other git annex operations in order for things to keep from getting messed up in git. Is this because if I switch to another branch the git annex add (for example) won't work correctly even though it doesn't switch branches?
+
+2) I could avoid the whole issue if I could \"pull -rebase\" and push the git-annex branch without switching branches. Is there a way to do that?
+
+Thanks! -Mark
+"""]]
diff --git a/doc/forum/Multisession_compatible__63__/comment_2_c1dc4294cc2a43cfecfb7279d6411924._comment b/doc/forum/Multisession_compatible__63__/comment_2_c1dc4294cc2a43cfecfb7279d6411924._comment
new file mode 100644
index 0000000000..0d4c1505f3
--- /dev/null
+++ b/doc/forum/Multisession_compatible__63__/comment_2_c1dc4294cc2a43cfecfb7279d6411924._comment
@@ -0,0 +1,21 @@
+[[!comment format=mdwn
+ username="joey"
+ subject="""comment 2"""
+ date="2016-02-05T16:37:48Z"
+ content="""
+Well obviously if you have the git-annex branch checked out, any operation
+involving the work tree is not going to operate on the usual work tree.
+So obviously `git add` and `git annex add` are not going to do what you
+want them to do (unless you want them to add files to the checked-out
+git-annex branch). I'm not sure what other problems you might run into
+doing that.
+
+git-annex uses the .git/annex/index file to stage commits to the git-annex
+branch. Using that index file might help with rebasing or other operations
+on the branch. 
+
+It's certianly
+theoretically possible to rebase without touching the work tree and only
+using such an index file, but I don't know if git's rebase works that way.
+I also don't see any particular benefit to rebasing the git-annex branch.
+"""]]
diff --git a/doc/forum/Multisession_compatible__63__/comment_4_b7148c3afd31ce1effd135a4f7c92dbc._comment b/doc/forum/Multisession_compatible__63__/comment_4_b7148c3afd31ce1effd135a4f7c92dbc._comment
new file mode 100644
index 0000000000..6d0129fe6e
--- /dev/null
+++ b/doc/forum/Multisession_compatible__63__/comment_4_b7148c3afd31ce1effd135a4f7c92dbc._comment
@@ -0,0 +1,16 @@
+[[!comment format=mdwn
+ username="mark@6b90344cdab3158eacb94a3944460d138afc9bef"
+ nickname="mark"
+ subject="comment 4"
+ date="2016-02-05T17:32:22Z"
+ content="""
+Thanks again. Good information.
+
+So does \"git annex sync\" do any branch checkout?
+
+I'm using git annex to manage regression test outputs for audio. I don't want \"git annex sync\" because I want annex gets, etc. to be basically automatic. But I need to make sure when a test starts that the git-annex branch is up-to-date in case updated reference outputs were checked in elsewhere. So I want to sync the git-annex branch (but not the master because someone maybe locally testing code changes to vet the impact). I'm not a git expert, but it appears that you can't pull (or more specifically merge) outside of your working directory, so I'm simply switching branches to pull/push the git annex branch, but as we've discussed that isn't multisession safe.
+
+So, if you're able to do the git annex sync without a branch checkout, then there is hope that I can do a subset (just a sync of the git-annex branch) without a branch checkout.
+
+If I'm displaying a fundamental misunderstanding or missing something obvious, I'm all ears.
+"""]]
diff --git a/doc/forum/Multisession_compatible__63__/comment_5_d0ddac05268d978e950c91e76b023ffe._comment b/doc/forum/Multisession_compatible__63__/comment_5_d0ddac05268d978e950c91e76b023ffe._comment
new file mode 100644
index 0000000000..65a5694036
--- /dev/null
+++ b/doc/forum/Multisession_compatible__63__/comment_5_d0ddac05268d978e950c91e76b023ffe._comment
@@ -0,0 +1,11 @@
+[[!comment format=mdwn
+ username="joey"
+ subject="""comment 5"""
+ date="2016-02-05T17:34:37Z"
+ content="""
+`git annex sync` does not check out the git-annex branch; nothing in git-annex does.
+
+git-annex checks for any remote/git-annex branches that have not been
+merged yet and auto-merges them, whenever it's run. So, probably all you
+need to do is `git fetch` and possibly `git push` of the branch.
+"""]]
diff --git a/doc/forum/Multisession_compatible__63__/comment_6_e2319f774eda62600c27ab9d2d0d1e50._comment b/doc/forum/Multisession_compatible__63__/comment_6_e2319f774eda62600c27ab9d2d0d1e50._comment
new file mode 100644
index 0000000000..d85de4c977
--- /dev/null
+++ b/doc/forum/Multisession_compatible__63__/comment_6_e2319f774eda62600c27ab9d2d0d1e50._comment
@@ -0,0 +1,10 @@
+[[!comment format=mdwn
+ username="mark@6b90344cdab3158eacb94a3944460d138afc9bef"
+ nickname="mark"
+ subject="comment 6"
+ date="2016-02-05T18:18:58Z"
+ content="""
+Ah, that is very helpful information. I'll explore that. Any reason the fetch should not use the \"+\" to work with non-fast-forward updates?
+
+I greatly appreciate your time. Thanks!
+"""]]
diff --git a/doc/forum/Multisession_compatible__63__/comment_7_0e6c8ab32c589f0d971d4c208a8d1645._comment b/doc/forum/Multisession_compatible__63__/comment_7_0e6c8ab32c589f0d971d4c208a8d1645._comment
new file mode 100644
index 0000000000..49d21874f4
--- /dev/null
+++ b/doc/forum/Multisession_compatible__63__/comment_7_0e6c8ab32c589f0d971d4c208a8d1645._comment
@@ -0,0 +1,10 @@
+[[!comment format=mdwn
+ username="mark@6b90344cdab3158eacb94a3944460d138afc9bef"
+ nickname="mark"
+ subject="comment 7"
+ date="2016-02-05T22:35:19Z"
+ content="""
+Seems to be working with \"+\" fetch. I did have to put the fetch before the git annex commands and the push afterward or some of the changed git state from the git annex operations was lost (for example a drop and subsequent get wouldn't work because the drop wasn't probably tracked).
+
+Mark
+"""]]
diff --git a/doc/forum/My_first_impressions_after_some_weeks_with_git-annex_assistant.mdwn b/doc/forum/My_first_impressions_after_some_weeks_with_git-annex_assistant.mdwn
new file mode 100644
index 0000000000..0614554add
--- /dev/null
+++ b/doc/forum/My_first_impressions_after_some_weeks_with_git-annex_assistant.mdwn
@@ -0,0 +1,57 @@
+This post is a personal story, a bug report, and a feature request.
+
+I discovered `git-annex assistant` by accident a couple of weeks (months?) ago. I was very impressed by the technical smartness and decided it was worth a try. My current configuration (everything done through the assistant web GUI) is:
+
+* Two machines that act as client repositories and are usually in the same LAN. Both of them run OS X Mountain Lion.
+* One transfer repository which was Box.com, but then I switched to Amazon S3 (today, I’ll explain why).
+* The repository consists of mixed files and is ~4 GB big (very small plain text files, PDFs, some MP3 files, the biggest files are about 20–50 MB). The Finder reports that the repository is ~40 GB big while `du -sh` reports 4.4 GB. It’s probably just a silly Finder bug, but it would be great to know why this happens.
+* I have version 4.20130324 on both clients. The OS X versions seems to be always a little bit behind. It would be great if it could match with other platforms.
+
+Here are some issues which I think are worth mentioning:
+
+1. I had some issues with the assistant, because I initially launched it from the mounted DMG. `git-annex assistant` set the environment up, but it configured the wrong paths (to `/Volumes` instead `/Applications` where the bundle later went). This has caused issues with authentication and some ssh keys. `git-annex` was confused which keys to use and I had to manually remove the first ones from the `.ssh/` directory. This could be very confusing for technically not so savvy users, because the error messages in the GUI were confusing.
+
+2. I’m still very confused about the behavior of `git-annex assistant` when it launches. The application icon bounces in the Dock for quite a long time, then it looks like the process would’ve crashed (I can only *Force Quit* it from the Dock). Then the icon disappears after some minutes / half an hour. I was unsure if the assistant would run if I would *Force Quit* it (yes). The behavior is very strange on OS X. It would be great to have at least a normal icon which bounces a couple of times, stays in the Dock without appearing like a non-responding app, and that launches the web GUI when a user clicks on it. Optimal would be just a menu bar icon which would show the current progress, let me launch the web GUI, check for updates, pause & resume all transfers, and restart & quit `git-annex assistant`.
+
+3. The initial sync went great and pretty fast. I was also impressed by the speed `git-annex` picks up new changes and synchronizes them to other repositories. After adding the Box.com transfer repository everything got worse. Files that were added to one of the client repositories were never synced to the other machine, despite making sure that the client repositories appeared above the Box.com repository in `git-annex assistant`. The transfer was very slow (that’s probably due to the API rate limits of Box.com) and I got the impression that the files were uploaded multiple times (probably true, because the assistant couldn’t upload them in the first place and then just tried it again giving me the impression that it was stuck in some kind of loop). I tried to manually click on a couple of hundred play buttons in the queue in the hope that there would somewhere appear the item which should’ve been synced to the other machine in the first place. I also attached some log messages to the end of this post which could be helpful. At this point `git-annex assistant` basically stopped working for me. I lived somehow with it in the hope that it would upload everything to Box.com in the upcoming days, but it never got better, even after a couple of weeks (in the meantime we copied files manually over an attached network storage which lived outside of the repository). Today I bit the bullet and switched to Amazon S3. All files were uploaded in a couple of minutes and it looks like this setup would be more reliable. It’ll cost us a couple of $ each month, but it’s OK, because it actually works.
+
+4. I read somewhere that transfer repositories where meant to keep only files that are not present on all client repositories. With my configuration `git-annex` just uploads everything to the transfer repository (no matter if it’s Box.com or Amazon S3). Is this the correct behavior? Did I get something wrong or is this just not implemented yet? It’s OK in the current version, but it would be great to store only the missing files in the future.
+
+5. After I switched to Amazon S3 some encrypted file names appear in my queue. They are gone after I restart `git-annex assistant` or when I manually remove them with the `X` button. Is this a bug?
+
+6. It is confusing that the web GUI shows that the transfer failed when the second machine is offline. While technically this is true, it would be much more assuring to know that the client is just offline. If something fails I get the impression that it is broken, but the client was only unreachable and that’s just a temporary state that’ll get automatically fixed when the client goes online again. Adding green / red lights to the repository with a mouse over description would be probably a good idea. Reporting that the client is unreachable instead of showing the error message would also be more appropriate.
+
+7. The biggest issue I have now is that `git-annex assistant` randomly crashes on both machines. I don’t know if this is a known issue or if this an OS X only issue, but it is slightly annoying. If you want to make sure that files really synchronized, you’ve to start `git-annex assistant` just to check if it still runs. I hope that this will get fixed as soon as possible. I would love to help you with debug logs, but I don’t know where to look. There’s nothing meaningful in the system log or the debug log.
+
+8. If the issues with Box.com are because of their API rate limits, I would make this clearer when adding new Box.com repositories or I would remove the feature altogether, because they actually make `git-annex assistant` unusable.
+
+To sum everything up: my biggest issues so far are the constant crashes and the strange behavior of the Dock icon. I’m still pretty impressed by the technical smartness and that the software is available at all. Thank you very much—Joey—for letting us use this software on a daily basis. I’m glad that you launched the Kickstarter campaign. I don’t know if it’s much of a hassle, but a second Kickstarter to make sure that development will continue would be probably a good idea. I would like to help with logs, but I don’t know how. Please let me know if there’s something I can do.
+
+**Box.com logs:**
+
+    87%          16.0KB/s 2s[2013-04-22 15:14:35 CEST] Pusher: Syncing with 10.0.1.4_jcshared
+    To ssh://rafael@git-annex-10.0.1.4-rafael/~/jc-shared/
+       7428090..f0d2f9e  git-annex -> synced/git-annex
+       9d2cb35..1f7b678  master -> synced/master
+    Already up-to-date.
+    Already up-to-date.
+    gpg: [stdout]: write error: Broken pipe
+    gpg: DBG: deflate: iobuf_write failed
+    gpg: build_packet failed: file write error
+    gpg: [stdout]: write error: Broken pipe
+    gpg: iobuf_flush failed on close: file write error
+    gpg: symmetric encryption of `[stdin]' failed: file write error
+    ResponseTimeout
+    git-annex: fd:41: hPutBuf: resource vanished (Broken pipe)
+    ResponseTimeout
+    (gpg)
+    87%          16.0KB/s 2sgpg: [stdout]: write error: Broken pipe
+    gpg: DBG: deflate: iobuf_write failed
+    gpg: build_packet failed: file write error
+    gpg: [stdout]: write error: Broken pipe
+    gpg: iobuf_flush failed on close: file write error
+    gpg: symmetric encryption of `[stdin]' failed: file write error
+    ResponseTimeout
+    git-annex: fd:54: hPutBuf: resource vanished (Broken pipe)
+    send: resource vanished (Connection reset by peer)
+    ResponseTimeout
diff --git a/doc/forum/My_first_impressions_after_some_weeks_with_git-annex_assistant/comment_1_9d4019a54fb508e286a5d6d2660361d9._comment b/doc/forum/My_first_impressions_after_some_weeks_with_git-annex_assistant/comment_1_9d4019a54fb508e286a5d6d2660361d9._comment
new file mode 100644
index 0000000000..10e10e06c4
--- /dev/null
+++ b/doc/forum/My_first_impressions_after_some_weeks_with_git-annex_assistant/comment_1_9d4019a54fb508e286a5d6d2660361d9._comment
@@ -0,0 +1,26 @@
+[[!comment format=mdwn
+ username="http://joeyh.name/"
+ nickname="joey"
+ subject="comment 1"
+ date="2013-04-23T21:41:31Z"
+ content="""
+This is a lot of stuff. It would be better to file individual detailed bug reports if you want any of this looked at in depth. Otherwise, I just read it to get an impression of how things are working, or not, for you, and try to hit some of the main points:
+
+* The OSX app icon behavior is probably because OSX does not realize that this is supposed to be a daemon. I don't know how to fix that. Clicking on the app icon is supposed to launch the web app, and has been reported to do so by others. I have no access to an OSX desktop.
+
+* Transfer repositories only keep files that have not yet reached all your client repositories. However, it's possible that files
+  may be uploaded to a transfer repository unnecessarily (if there is some other path for the file to get to the other client
+  repositories, ie if you've locally paired them). In this case, the file will be uploaded and then later removed from the transfer
+  repository, without being used.
+
+* I have never seen encrypted filenames appear in the webapp. I don't see how that could happen TBH.
+
+* git-annex logs information to `.git/annex/debug.log` within your repository. You can go to Configuration -> Preferences to enable
+  additional debug logging. I absolutely need these logs to help debug almost any problems with the assistant.
+
+* \"Clicking a couple of hundred play buttons\" could result in it trying to make a couple of hundred uploads to box.com
+  at the same time. This is probably not a good idea.
+
+* The fragment of log you posted shows me some problem that caused gpg to fail to encrypt a file before sending it to somewhere,
+  but it's lacking all context (and starts when the file transfer is already 87% complete!) so I can't say more. I need to see the whole log to make any informed guesses about what might be going on.
+"""]]
diff --git a/doc/forum/My_first_impressions_after_some_weeks_with_git-annex_assistant/comment_2_109534a45881ce94a4586c8a83945f9f._comment b/doc/forum/My_first_impressions_after_some_weeks_with_git-annex_assistant/comment_2_109534a45881ce94a4586c8a83945f9f._comment
new file mode 100644
index 0000000000..33b2aa393e
--- /dev/null
+++ b/doc/forum/My_first_impressions_after_some_weeks_with_git-annex_assistant/comment_2_109534a45881ce94a4586c8a83945f9f._comment
@@ -0,0 +1,85 @@
+[[!comment format=mdwn
+ username="EmanueleAina"
+ ip="93.38.211.231"
+ subject="comment 2"
+ date="2013-09-11T17:55:14Z"
+ content="""
+I seem to hit the same issue (`ResponseTimeout`) on box.com, but I don't have any log file under .git/annex.
+
+The timeout is quite unpredictable, sometimes I'm able to transfer less than 20% of the file while sometimes it fails after reaching 51%. The file is quite big, 6.2G.
+
+Unfortunately, after the first upload fails, `git-annex` seems to think that the file has been uploaded successfully and will refuse to copy it again. Even `whereis` will list the box.com location.
+
+I think there are really two bugs: the one triggering the timeouts, and the fact that `git-annex` thinks that the failing upload succeeded and is unable to recover the interrupted upload.
+
+Here's the log I got on the shell. Let me know how I can provide any additional information needed. Thanks!
+
+	$ git annex copy --verbose --debug home.tar.gz --to box.com
+	[2013-09-11 17:42:52 CEST] read: git [\"--git-dir=/data/backup/.git\",\"--work-tree=/data/backup\",\"show-ref\",\"git-annex\"]
+	[2013-09-11 17:42:52 CEST] read: git [\"--git-dir=/data/backup/.git\",\"--work-tree=/data/backup\",\"show-ref\",\"--hash\",\"refs/heads/git-annex\"]
+	[2013-09-11 17:42:52 CEST] read: git [\"--git-dir=/data/backup/.git\",\"--work-tree=/data/backup\",\"log\",\"refs/heads/git-annex..1e029f92c5c82b094cbe70b55927996c31579e2e\",\"--oneline\",\"-n1\"]
+	[2013-09-11 17:42:52 CEST] read: git [\"--git-dir=/data/backup/.git\",\"--work-tree=/data/backup\",\"log\",\"refs/heads/git-annex..1090c21b121fd8b6e2ab49a772c8eab5235a3930\",\"--oneline\",\"-n1\"]
+	[2013-09-11 17:42:52 CEST] chat: git [\"--git-dir=/data/backup/.git\",\"--work-tree=/data/backup\",\"cat-file\",\"--batch\"]
+	[2013-09-11 17:42:52 CEST] read: git [\"--git-dir=/data/backup/.git\",\"--work-tree=/data/backup\",\"ls-files\",\"--cached\",\"-z\",\"--\",\"home.tar.gz\"]
+	copy home.tar.gz (gpg) [2013-09-11 17:42:52 CEST] chat: gpg [\"--quiet\",\"--trust-model\",\"always\",\"--decrypt\"]
+	(checking box.com...) (to box.com...) 
+	[2013-09-11 17:42:57 CEST] chat: gpg [\"--quiet\",\"--trust-model\",\"always\",\"--batch\",\"--passphrase-fd\",\"11\",\"--symmetric\",\"--force-mdc\",\"--no-textmode\"]
+	35%      660.8KB/s 1h44mResponseTimeout
+	gpg: [stdout]: write error: Broken pipe
+	gpg: DBG: deflate: iobuf_write failed
+	gpg: build_packet failed: file write error
+	gpg: [stdout]: write error: Broken pipe
+	gpg: iobuf_flush failed on close: file write error
+	gpg: [stdout]: write error: Broken pipe
+	gpg: iobuf_flush failed on close: file write error
+	gpg: symmetric encryption of `[stdin]' failed: file write error
+	git-annex: fd:13: hPutBuf: resource vanished (Broken pipe)
+	failed
+	git-annex: copy: 1 failed
+	$ git annex copy --verbose --debug home.tar.gz --to box.com
+	[2013-09-11 19:38:26 CEST] read: git [\"--git-dir=/data/backup/.git\",\"--work-tree=/data/backup\",\"show-ref\",\"git-annex\"]
+	[2013-09-11 19:38:26 CEST] read: git [\"--git-dir=/data/backup/.git\",\"--work-tree=/data/backup\",\"show-ref\",\"--hash\",\"refs/heads/git-annex\"]
+	[2013-09-11 19:38:26 CEST] read: git [\"--git-dir=/data/backup/.git\",\"--work-tree=/data/backup\",\"log\",\"refs/heads/git-annex..1e029f92c5c82b094cbe70b55927996c31579e2e\",\"--oneline\",\"-n1\"]
+	[2013-09-11 19:38:26 CEST] read: git [\"--git-dir=/data/backup/.git\",\"--work-tree=/data/backup\",\"log\",\"refs/heads/git-annex..1090c21b121fd8b6e2ab49a772c8eab5235a3930\",\"--oneline\",\"-n1\"]
+	[2013-09-11 19:38:26 CEST] chat: git [\"--git-dir=/data/backup/.git\",\"--work-tree=/data/backup\",\"cat-file\",\"--batch\"]
+	[2013-09-11 19:38:26 CEST] read: git [\"--git-dir=/data/backup/.git\",\"--work-tree=/data/backup\",\"ls-files\",\"--cached\",\"-z\",\"--\",\"home.tar.gz\"]
+	copy home.tar.gz (gpg) [2013-09-11 19:38:26 CEST] chat: gpg [\"--quiet\",\"--trust-model\",\"always\",\"--decrypt\"]
+	(checking box.com...) ok
+	$ git annex whereis --verbose --debug home.tar.gz             
+	[2013-09-11 19:38:57 CEST] read: git [\"--git-dir=/data/backup/.git\",\"--work-tree=/data/backup\",\"show-ref\",\"git-annex\"]
+	[2013-09-11 19:38:57 CEST] read: git [\"--git-dir=/data/backup/.git\",\"--work-tree=/data/backup\",\"show-ref\",\"--hash\",\"refs/heads/git-annex\"]
+	[2013-09-11 19:38:57 CEST] read: git [\"--git-dir=/data/backup/.git\",\"--work-tree=/data/backup\",\"log\",\"refs/heads/git-annex..1e029f92c5c82b094cbe70b55927996c31579e2e\",\"--oneline\",\"-n1\"]
+	[2013-09-11 19:38:57 CEST] read: git [\"--git-dir=/data/backup/.git\",\"--work-tree=/data/backup\",\"log\",\"refs/heads/git-annex..1090c21b121fd8b6e2ab49a772c8eab5235a3930\",\"--oneline\",\"-n1\"]
+	[2013-09-11 19:38:57 CEST] chat: git [\"--git-dir=/data/backup/.git\",\"--work-tree=/data/backup\",\"cat-file\",\"--batch\"]
+	[2013-09-11 19:38:57 CEST] read: git [\"--git-dir=/data/backup/.git\",\"--work-tree=/data/backup\",\"ls-files\",\"--cached\",\"-z\",\"--\",\"home.tar.gz\"]
+	whereis home.tar.gz (2 copies) 
+		d7db543e-5463-11e2-b7dd-9f423f798cc4 -- here (em@ocracy:/data/backup)
+		e71fa45e-5463-11e2-a14e-93ca09c272da -- box.com
+	ok
+	$ git annex fsck --verbose --debug home.tar.gz
+	[2013-09-11 19:39:05 CEST] read: git [\"--git-dir=/data/backup/.git\",\"--work-tree=/data/backup\",\"ls-files\",\"--cached\",\"-z\",\"--\",\"home.tar.gz\"]
+	[2013-09-11 19:39:05 CEST] chat: git [\"--git-dir=/data/backup/.git\",\"--work-tree=/data/backup\",\"check-attr\",\"-z\",\"--stdin\",\"annex.backend\",\"annex.numcopies\",\"--\"]
+	fsck home.tar.gz [2013-09-11 19:39:05 CEST] read: git [\"--git-dir=/data/backup/.git\",\"--work-tree=/data/backup\",\"show-ref\",\"git-annex\"]
+	[2013-09-11 19:39:05 CEST] read: git [\"--git-dir=/data/backup/.git\",\"--work-tree=/data/backup\",\"show-ref\",\"--hash\",\"refs/heads/git-annex\"]
+	[2013-09-11 19:39:05 CEST] read: git [\"--git-dir=/data/backup/.git\",\"--work-tree=/data/backup\",\"log\",\"refs/heads/git-annex..1e029f92c5c82b094cbe70b55927996c31579e2e\",\"--oneline\",\"-n1\"]
+	[2013-09-11 19:39:05 CEST] read: git [\"--git-dir=/data/backup/.git\",\"--work-tree=/data/backup\",\"log\",\"refs/heads/git-annex..1090c21b121fd8b6e2ab49a772c8eab5235a3930\",\"--oneline\",\"-n1\"]
+	[2013-09-11 19:39:05 CEST] chat: git [\"--git-dir=/data/backup/.git\",\"--work-tree=/data/backup\",\"cat-file\",\"--batch\"]
+	(checksum...) [2013-09-11 19:39:05 CEST] read: sha256sum [\"/data/backup/.git/annex/objects/mG/kp/SHA256E-s6640317400--dcf0a535728f3f3f787db6339b740a4a6f6529e5ce1d238f28574499a8172670.tar.gz/SHA256E-s6640317400--dcf0a535728f3f3f787db6339b740a4a6f6529e5ce1d238f28574499a8172670.tar.gz\"]
+	ok
+	$ git annex fsck --verbose --debug home.tar.gz --from box.com
+	[2013-09-11 19:42:15 CEST] read: git [\"--git-dir=/data/backup/.git\",\"--work-tree=/data/backup\",\"show-ref\",\"git-annex\"]
+	[2013-09-11 19:42:15 CEST] read: git [\"--git-dir=/data/backup/.git\",\"--work-tree=/data/backup\",\"show-ref\",\"--hash\",\"refs/heads/git-annex\"]
+	[2013-09-11 19:42:15 CEST] read: git [\"--git-dir=/data/backup/.git\",\"--work-tree=/data/backup\",\"log\",\"refs/heads/git-annex..1e029f92c5c82b094cbe70b55927996c31579e2e\",\"--oneline\",\"-n1\"]
+	[2013-09-11 19:42:15 CEST] read: git [\"--git-dir=/data/backup/.git\",\"--work-tree=/data/backup\",\"log\",\"refs/heads/git-annex..1090c21b121fd8b6e2ab49a772c8eab5235a3930\",\"--oneline\",\"-n1\"]
+	[2013-09-11 19:42:15 CEST] chat: git [\"--git-dir=/data/backup/.git\",\"--work-tree=/data/backup\",\"cat-file\",\"--batch\"]
+	[2013-09-11 19:42:15 CEST] read: git [\"--git-dir=/data/backup/.git\",\"--work-tree=/data/backup\",\"ls-files\",\"--cached\",\"-z\",\"--\",\"home.tar.gz\"]
+	[2013-09-11 19:42:15 CEST] chat: git [\"--git-dir=/data/backup/.git\",\"--work-tree=/data/backup\",\"check-attr\",\"-z\",\"--stdin\",\"annex.backend\",\"annex.numcopies\",\"--\"]
+	fsck home.tar.gz (gpg) [2013-09-11 19:42:15 CEST] chat: gpg [\"--quiet\",\"--trust-model\",\"always\",\"--decrypt\"]
+	(checking box.com...) 
+	[2013-09-11 19:42:28 CEST] chat: gpg [\"--quiet\",\"--trust-model\",\"always\",\"--batch\",\"--passphrase-fd\",\"10\",\"--decrypt\"]
+	gpg: decrypt_message failed: eof
+	ok
+
+
+
+"""]]
diff --git a/doc/forum/NTFS_usb_on_linux_unable_to_connect_to_ssh_remote.mdwn b/doc/forum/NTFS_usb_on_linux_unable_to_connect_to_ssh_remote.mdwn
new file mode 100644
index 0000000000..d7867f9870
--- /dev/null
+++ b/doc/forum/NTFS_usb_on_linux_unable_to_connect_to_ssh_remote.mdwn
@@ -0,0 +1,29 @@
+There are SSH keys in ~/.ssh to a remote server that I added as a git annex remote. On my Debian box, I am able to `git annex copy` and `git annex move` to and from the remote to a local repo at `~/archive` with no problems. I also have no problems with external USB drives formatted ext4. 
+
+I have an external usb drive formatted NTFS that I connect to a Debian box. I added the remote server like the others and when I try to copy or move to and from it, this error code shows up. 
+
+
+
+    pull archive 
+    Control socket connect(.git/annex/ssh/f7be67fcc0a6f016ba90edcdd8e02e1f): Connection refused
+    Failed to connect to new control master
+    fatal: Could not read from remote repository.
+
+    Please make sure you have the correct access rights
+    and the repository exists.
+    failed
+    push archive 
+    Control socket connect(.git/annex/ssh/f7be67fcc0a6f016ba90edcdd8e02e1f): Connection refused
+    Failed to connect to new control master
+    fatal: Could not read from remote repository.
+
+    Please make sure you have the correct access rights
+    and the repository exists.
+
+    Pushing to archive failed.
+
+
+
+I think it is because of NTFS and the ssh key stored in `~/.ssh`, but I don't know what is going on. 
+
+Thanks!
diff --git a/doc/forum/NTFS_usb_on_linux_unable_to_connect_to_ssh_remote/comment_1_b2d2bf6be441dc2e4443d06ef8db7134._comment b/doc/forum/NTFS_usb_on_linux_unable_to_connect_to_ssh_remote/comment_1_b2d2bf6be441dc2e4443d06ef8db7134._comment
new file mode 100644
index 0000000000..d5a0d6460a
--- /dev/null
+++ b/doc/forum/NTFS_usb_on_linux_unable_to_connect_to_ssh_remote/comment_1_b2d2bf6be441dc2e4443d06ef8db7134._comment
@@ -0,0 +1,12 @@
+[[!comment format=mdwn
+ username="https://www.google.com/accounts/o8/id?id=AItOawksOCeakibYmGDt3wLLo4nkY0FkB72I2Uo"
+ nickname="Source"
+ subject="comment 1"
+ date="2015-03-07T02:50:19Z"
+ content="""
+I think I have tracked down part of the issue. It due to the options that are called by `git-annex` when trying to `rsync` between the NTFS local hard drive and the SSH remote. 
+
+I got `rsync` to work by using `rsync -vrc source dest` as options (outside of git-annex). Found [here](http://askubuntu.com/questions/112863/rsync-not-working-between-ntfs-fat-and-ext).
+
+Is there any way to change the rsync options put out by git-annex when copying/moving to and from local ntfs drives on linux?
+"""]]
diff --git a/doc/forum/NTFS_usb_on_linux_unable_to_connect_to_ssh_remote/comment_2_75abe6da74cd0432e616e391f5be6fe3._comment b/doc/forum/NTFS_usb_on_linux_unable_to_connect_to_ssh_remote/comment_2_75abe6da74cd0432e616e391f5be6fe3._comment
new file mode 100644
index 0000000000..49dd5f4ef2
--- /dev/null
+++ b/doc/forum/NTFS_usb_on_linux_unable_to_connect_to_ssh_remote/comment_2_75abe6da74cd0432e616e391f5be6fe3._comment
@@ -0,0 +1,14 @@
+[[!comment format=mdwn
+ username="joey"
+ subject="""comment 2"""
+ date="2015-03-16T17:14:50Z"
+ content="""
+This control socket message sounds like the actual problem is
+that unix sockets cannot be used on this filesystem. git-annex
+puts the socket there to make ssh connection caching work.
+
+So, I think you need to disable ssh connection caching: 
+`git config annex.sshcaching false`
+
+I was able to reproduce a similar failure on nfts on linux.
+"""]]
diff --git a/doc/forum/Need_new_build_instructions_for_Debian_stable.mdwn b/doc/forum/Need_new_build_instructions_for_Debian_stable.mdwn
new file mode 100644
index 0000000000..7db19697c3
--- /dev/null
+++ b/doc/forum/Need_new_build_instructions_for_Debian_stable.mdwn
@@ -0,0 +1,5 @@
+The instructions for building git-annex on [[install/Debian]] stable don't seem to be valid anymore.
+
+1. `dpkg-checkbuilddeps` is looking for the wrong packages, e.g. libghc-missingh-dev instead of libghc6-missingh-dev.
+
+2. Not all dependencies are available in the Squeeze repositories anymore (at least not Crypto and hS3), if I am not mistaken.
diff --git a/doc/forum/Need_new_build_instructions_for_Debian_stable/comment_1_8c1eea6dfec8b7e1c7a371b6e9c26118._comment b/doc/forum/Need_new_build_instructions_for_Debian_stable/comment_1_8c1eea6dfec8b7e1c7a371b6e9c26118._comment
new file mode 100644
index 0000000000..e464c84da1
--- /dev/null
+++ b/doc/forum/Need_new_build_instructions_for_Debian_stable/comment_1_8c1eea6dfec8b7e1c7a371b6e9c26118._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="http://joey.kitenet.net/"
+ nickname="joey"
+ subject="comment 1"
+ date="2011-04-26T15:27:49Z"
+ content="""
+I have updated the instructions.
+"""]]
diff --git a/doc/forum/Need_new_build_instructions_for_Debian_stable/comment_2_f6ff8306c946219dbe39bb8938a349ab._comment b/doc/forum/Need_new_build_instructions_for_Debian_stable/comment_2_f6ff8306c946219dbe39bb8938a349ab._comment
new file mode 100644
index 0000000000..40f570610a
--- /dev/null
+++ b/doc/forum/Need_new_build_instructions_for_Debian_stable/comment_2_f6ff8306c946219dbe39bb8938a349ab._comment
@@ -0,0 +1,21 @@
+[[!comment format=mdwn
+ username="gernot"
+ ip="213.196.216.21"
+ subject="comment 2"
+ date="2011-04-26T18:56:44Z"
+ content="""
+Thanks for the update, Joey. I think you forgot to change libghc-missingh-dev to libghc6-missingh-dev for the copy & paste instructions though.
+
+Also, after having checked that I have everything installed I'm still getting this error:
+
+	...
+	[15 of 77] Compiling Annex            ( Annex.hs, Annex.o )
+
+	Annex.hs:19:35:
+		Module `Control.Monad.State' does not export `state'
+	make[1]: *** [git-annex] Error 1
+	make[1]: Leaving directory `/home/gernot/dev/git-annex'
+	dh_auto_build: make -j1 returned exit code 2
+	make: *** [binary] Error 2
+
+"""]]
diff --git a/doc/forum/Need_new_build_instructions_for_Debian_stable/comment_3_bcda70cbfc7c1a14fa82da70f9f876e2._comment b/doc/forum/Need_new_build_instructions_for_Debian_stable/comment_3_bcda70cbfc7c1a14fa82da70f9f876e2._comment
new file mode 100644
index 0000000000..8b41116431
--- /dev/null
+++ b/doc/forum/Need_new_build_instructions_for_Debian_stable/comment_3_bcda70cbfc7c1a14fa82da70f9f876e2._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="http://joey.kitenet.net/"
+ nickname="joey"
+ subject="comment 3"
+ date="2011-04-26T23:40:33Z"
+ content="""
+Both problems fixed.
+"""]]
diff --git a/doc/forum/Need_some_help_installing_on_freeBSD.mdwn b/doc/forum/Need_some_help_installing_on_freeBSD.mdwn
new file mode 100644
index 0000000000..8a2e7a13ae
--- /dev/null
+++ b/doc/forum/Need_some_help_installing_on_freeBSD.mdwn
@@ -0,0 +1,48 @@
+Hi there,
+
+**I'm trying to install git-annex on my NAS which is freeNAS running freeBSD. 
+I've created a jail for it and installed via: pkg install hs-git-annex**
+
+I've seen on this page: http://www.freshports.org/devel/hs-git-annex/ that there are Configuration Options. 
+I assume to use those I'd have to have compiled it myself from the ports like this: 
+To install the port: cd /usr/ports/devel/hs-git-annex/ && make install clean
+
+So in this case of using the pkg, how can I check the way git-annex was compiled? 
+Anyone else ever used it on freeBSD and can give some input please? 
+
+**Figured out how to see this info:**
+
+git-annex version
+git-annex version: 5.20150727
+build flags: Assistant Webapp Webapp-secure Pairing S3 WebDAV Kqueue XMPP DNS Feeds Quvi TDFA Database
+key/value backends: SHA256E SHA1E SHA512E SHA224E SHA384E SKEIN256E SKEIN512E MD5E SHA256 SHA1 SHA512 SHA224 SHA384 SKEIN256 SKEIN512 MD5 WORM URL
+remote types: git gcrypt S3 bup directory rsync web bittorrent webdav tahoe glacier ddar hook external
+
+**So apparently box.com isn't supported in this build?**
+
+I installed via a web terminal, not via a normal ssh connection so when I came back the installation with all dependencies was finished but I couldn't scroll back to see all warning, only caught this last part of the screen; anyone got any idea if git-annex is supposed to run fine on FreeBSD 9.3-RELEASE-p26 amd64? 
+
+**part of the text I caught:**
+  IN_CLOSE_WRITE                                                               
+                                                                                
+Symbolic Link notifications:                                                    
+   IN_DONT_FOLLOW                                                               
+   IN_ATTRIB                                                                    
+   IN_MOVE_SELF                                                                 
+   IN_DELETE_SELF                                                               
+                                                                                
+Kernel patches to address the missing directory and symbolic link               
+notifications are available from:                                               
+                                                                                
+https://github.com/dmatveev/libinotify-kqueue/tree/master/patches               
+                                                                                
+=============================================================================   
+You might want to consider increasing the kern.maxfiles tunable if you plan     
+to use this library for applications that need to monitor activity of a lot     
+of files.                                                                       
+                                                                                
+If the default on your system is too low, add the following line to             
+/boot/loader.conf, then reboot the system:                                      
+                                                                                
+    kern.maxfiles="25000"                                                       
+============================================================================= 
diff --git a/doc/forum/Need_some_help_installing_on_freeBSD/comment_1_3962871e646280e2c4fdff8a08bed485._comment b/doc/forum/Need_some_help_installing_on_freeBSD/comment_1_3962871e646280e2c4fdff8a08bed485._comment
new file mode 100644
index 0000000000..58186a9b16
--- /dev/null
+++ b/doc/forum/Need_some_help_installing_on_freeBSD/comment_1_3962871e646280e2c4fdff8a08bed485._comment
@@ -0,0 +1,14 @@
+[[!comment format=mdwn
+ username="ovidiu@66ace8a8d99ce938b0538ffa0f26d30db02a9626"
+ nickname="ovidiu"
+ subject="comment 1"
+ date="2015-10-05T18:53:20Z"
+ content="""
+it also seems like the git annex webapp command needs lsof, had to install it separately.
+
+I then figured out one has to start the webapp with a listen command to prevent it from opening a browser.
+*git annex webapp --listen=internal_IP*
+
+Not sure how this option will work: Start the git-annex assistant at boot or on login. - as I assume I get a different auth URL with every start?
+Any help available to get *git annex webapp --listen=internal_IP* to auto-start with every reboot?
+"""]]
diff --git a/doc/forum/Need_some_help_installing_on_freeBSD/comment_2_528fdd2c385809b971eceee1eb70ee38._comment b/doc/forum/Need_some_help_installing_on_freeBSD/comment_2_528fdd2c385809b971eceee1eb70ee38._comment
new file mode 100644
index 0000000000..414d4518f1
--- /dev/null
+++ b/doc/forum/Need_some_help_installing_on_freeBSD/comment_2_528fdd2c385809b971eceee1eb70ee38._comment
@@ -0,0 +1,11 @@
+[[!comment format=mdwn
+ username="ovidiu@66ace8a8d99ce938b0538ffa0f26d30db02a9626"
+ nickname="ovidiu"
+ subject="comment 2"
+ date="2015-10-05T19:07:51Z"
+ content="""
+oh, and once I closed the webapp, I saw on the console that it needed more info:
+
+git config --global user.email \"you@example.com\"
+git config --global user.name \"Your Name\"
+"""]]
diff --git a/doc/forum/Need_some_help_installing_on_freeBSD/comment_3_5ae3cd6e24a25d6dc803ec2adea0414f._comment b/doc/forum/Need_some_help_installing_on_freeBSD/comment_3_5ae3cd6e24a25d6dc803ec2adea0414f._comment
new file mode 100644
index 0000000000..080c22a4de
--- /dev/null
+++ b/doc/forum/Need_some_help_installing_on_freeBSD/comment_3_5ae3cd6e24a25d6dc803ec2adea0414f._comment
@@ -0,0 +1,9 @@
+[[!comment format=mdwn
+ username="joey"
+ subject="""comment 3"""
+ date="2015-10-06T20:05:24Z"
+ content="""
+The WebDAV support means it does support box.com. Looking at the
+version info, this freebsd build seems to have more or less every possible
+option enabled. Nice job freebsd porters!
+"""]]
diff --git a/doc/forum/Need_some_help_installing_on_freeBSD/comment_4_1dcc43aaad2099bb1684873d24da668f._comment b/doc/forum/Need_some_help_installing_on_freeBSD/comment_4_1dcc43aaad2099bb1684873d24da668f._comment
new file mode 100644
index 0000000000..3b8f06f05d
--- /dev/null
+++ b/doc/forum/Need_some_help_installing_on_freeBSD/comment_4_1dcc43aaad2099bb1684873d24da668f._comment
@@ -0,0 +1,9 @@
+[[!comment format=mdwn
+ username="joey"
+ subject="""comment 4"""
+ date="2015-10-06T20:07:43Z"
+ content="""
+To start the assistant, use "git annex assistant". You
+can then run "git annex webapp" as desired to open the webapp,
+communicating with the assistant if it was already running.
+"""]]
diff --git a/doc/forum/Need_some_help_installing_on_freeBSD/comment_5_527bf6df2485c5e906f1d50930ec01fe._comment b/doc/forum/Need_some_help_installing_on_freeBSD/comment_5_527bf6df2485c5e906f1d50930ec01fe._comment
new file mode 100644
index 0000000000..5351794710
--- /dev/null
+++ b/doc/forum/Need_some_help_installing_on_freeBSD/comment_5_527bf6df2485c5e906f1d50930ec01fe._comment
@@ -0,0 +1,9 @@
+[[!comment format=mdwn
+ username="ovidiu@66ace8a8d99ce938b0538ffa0f26d30db02a9626"
+ nickname="ovidiu"
+ subject="comment 5"
+ date="2015-10-07T20:12:20Z"
+ content="""
+thanks, I'll try and use only the assistant, not the webapp on the NAS, any links on how to get started apart from typing git annex? 
+any first steps to take, a particular order in which to do things?
+"""]]
diff --git a/doc/forum/Need_some_help_to_fix_my_repository.mdwn b/doc/forum/Need_some_help_to_fix_my_repository.mdwn
new file mode 100644
index 0000000000..e4f6fb593f
--- /dev/null
+++ b/doc/forum/Need_some_help_to_fix_my_repository.mdwn
@@ -0,0 +1,31 @@
+Hi, 
+first sorry for my poor english it's not my native language.
+
+I have one repository on my laptop and two repository on usb disk. Made with following  walkthrough (creating a repository and adding a remote).
+
+Yesterday I have a backup of my repository on usb disk before add some file with
+
+cd /media/usb/annex;git fetch laptop; git merge laptop/master&&git annex get .&&git annex sync
+
+Now the repository on my usb disk is a mess.
+Every file before the commit are lost. 
+For example : 
+ After the sync : file Z.7z 
+Z.7z: broken symbolic link to `../../../../.git/annex/objects/2K/49/SHA256-s0--e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855/SHA256-s0--e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855'
+
+The file ../../../../.git/annex/objects/2K/49/SHA256-s0--e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855/SHA256-s0--e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 doesn't exist.
+
+On the repository before the sync (inside the Backup) :
+file Z.7z 
+Z.7z: symbolic link to `../../../../.git/annex/objects/J1/f4/SHA256-s696365035--d2dcc67bf2f05fcfc7f42723b2d415d4f057a2eeadc282b40f5bc3724534f2f4/SHA256-s696365035--d2dcc67bf2f05fcfc7f42723b2d415d4f057a2eeadc282b40f5bc3724534f2f4'
+
+The file ../../../../.git/annex/objects/J1/f4/SHA256-s696365035--d2dcc67bf2f05fcfc7f42723b2d415d4f057a2eeadc282b40f5bc3724534f2f4/SHA256-s696365035--d2dcc67bf2f05fcfc7f42723b2d415d4f057a2eeadc282b40f5bc3724534f2f4  still exist in the repository after the sync.
+
+3 questions, if somebody could help me :
+
+ - what I do wrong ?
+ - why the- symlink for every file had change after "cd /media/usb/annex;git fetch laptop; git merge laptop/master&&git annex get .&&git annex sync"
+ - how could I fix my repository ? recover file from the backup ? how ? Copy every file to start my repository from a new clean state ?
+ 
+
+
diff --git a/doc/forum/Need_some_help_to_fix_my_repository/comment_1_f0d279c530b796b2c93d793f85d147e8._comment b/doc/forum/Need_some_help_to_fix_my_repository/comment_1_f0d279c530b796b2c93d793f85d147e8._comment
new file mode 100644
index 0000000000..6f9a8078f9
--- /dev/null
+++ b/doc/forum/Need_some_help_to_fix_my_repository/comment_1_f0d279c530b796b2c93d793f85d147e8._comment
@@ -0,0 +1,13 @@
+[[!comment format=mdwn
+ username="http://joeyh.name/"
+ nickname="joey"
+ subject="comment 1"
+ date="2013-04-22T20:00:12Z"
+ content="""
+1. Use `git log --stat` to find the commit that changed your file `Z.7z`
+2. Use `git revert` to revert that commit
+
+That will put things back the way they were.
+
+You should then investigate what caused the bad commit to be made. It seems to have involved the file being reset to be 0 bytes in size.
+"""]]
diff --git a/doc/forum/Need_some_help_to_fix_my_repository/comment_2_a3fcfa1f8eadec5fa8a9efacca174048._comment b/doc/forum/Need_some_help_to_fix_my_repository/comment_2_a3fcfa1f8eadec5fa8a9efacca174048._comment
new file mode 100644
index 0000000000..bded393bf0
--- /dev/null
+++ b/doc/forum/Need_some_help_to_fix_my_repository/comment_2_a3fcfa1f8eadec5fa8a9efacca174048._comment
@@ -0,0 +1,10 @@
+[[!comment format=mdwn
+ username="https://www.google.com/accounts/o8/id?id=AItOawkyQL8VPwrSf8LA0k8PKDWFCEKFH3O5Jds"
+ nickname="marco"
+ subject="Brvo"
+ date="2013-04-22T23:33:10Z"
+ content="""
+Bravo. Mon Hero.
+
+You save my data ! I can't find what happen but thanks a lot.
+"""]]
diff --git a/doc/forum/Need_some_help_to_fix_my_repository/comment_3_7878f9b76ddfa3392c9ec6a1810cb745._comment b/doc/forum/Need_some_help_to_fix_my_repository/comment_3_7878f9b76ddfa3392c9ec6a1810cb745._comment
new file mode 100644
index 0000000000..67657110a4
--- /dev/null
+++ b/doc/forum/Need_some_help_to_fix_my_repository/comment_3_7878f9b76ddfa3392c9ec6a1810cb745._comment
@@ -0,0 +1,10 @@
+[[!comment format=mdwn
+ username="http://joeyh.name/"
+ nickname="joey"
+ subject="glad I could help"
+ date="2013-04-23T21:16:47Z"
+ content="""
+I'm curious if your repository is using direct mode, and is perhaps on a FAT filesystem, or other filesystem that does not support symlinks?
+
+One way this could have happened is due to a bug in `git annex sync` when used on such a filesystem. I've just fixed this bug in commit 07580dc3dfb2f4d9af9a20bc172cbc32953c49d6
+"""]]
diff --git a/doc/forum/Need_to_recover_unused_files_because_of_bad_sync__63__.mdwn b/doc/forum/Need_to_recover_unused_files_because_of_bad_sync__63__.mdwn
new file mode 100644
index 0000000000..3584569b64
--- /dev/null
+++ b/doc/forum/Need_to_recover_unused_files_because_of_bad_sync__63__.mdwn
@@ -0,0 +1,23 @@
+I've two huge annex repos (about 100G) with all my photos.
+One is on my laptop and it uses indirect mode; the other one is on my NAS and I mount it into a CIFS folder, then it uses direct mode because of the crippled fs.
+Each repo is remote of the other one. The NAS repo belongs to backup group with the standard preferred content.
+I changed about 1000 photos in my PC repo (mainly renamed them) and I wanted to sync changes to the NAS repo, so I went through this sequence:
+
+* "git annex add" and "git annex sync" on my PC => OK;
+
+* "git annex sync" on the NAS repo => it started copying all the content BUT I stopped it because I had not enough time to wait at that very moment;
+
+* again "git anne sync" on the NAS repo => I had more time to wait BUT it didn't resume the copy from the PC repo, it just said something like "all done, nothing to do"
+
+* I checked that several files were missing on the NAS that I had modified/renamed on the PC repo so I did "git annex add" again on the PC => nothing new to add, nothing to do;
+
+* I did again "git annex sync" on the PC repo => it deleted also from my PC repo all the photos that were also missing on the NAS repo;
+
+* I've also checked for unused files on the PC repo and it now gives about 1000 files.
+
+So here are my questions:
+
+1. where was I wrong?
+
+2. can I restore my photos on my PC repo (which uses indirect mode) ?
+
diff --git a/doc/forum/Need_to_recover_unused_files_because_of_bad_sync__63__/comment_1_af0ed0730645c1e3c9a4946acd48c18a._comment b/doc/forum/Need_to_recover_unused_files_because_of_bad_sync__63__/comment_1_af0ed0730645c1e3c9a4946acd48c18a._comment
new file mode 100644
index 0000000000..8f2905c6f3
--- /dev/null
+++ b/doc/forum/Need_to_recover_unused_files_because_of_bad_sync__63__/comment_1_af0ed0730645c1e3c9a4946acd48c18a._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="http://joeyh.name/"
+ ip="108.236.230.124"
+ subject="comment 1"
+ date="2014-06-06T18:18:36Z"
+ content="""
+It's not clear from your description when the files got deleted or why, but you can use `git log --stat` to find out. Then you can `git revert` the problem commit.
+"""]]
diff --git a/doc/forum/Need_to_recover_unused_files_because_of_bad_sync__63__/comment_2_12f3c2bb2458b69e6355c8f94bab868f._comment b/doc/forum/Need_to_recover_unused_files_because_of_bad_sync__63__/comment_2_12f3c2bb2458b69e6355c8f94bab868f._comment
new file mode 100644
index 0000000000..4c30d0f4c5
--- /dev/null
+++ b/doc/forum/Need_to_recover_unused_files_because_of_bad_sync__63__/comment_2_12f3c2bb2458b69e6355c8f94bab868f._comment
@@ -0,0 +1,13 @@
+[[!comment format=mdwn
+ username="feulif"
+ ip="151.27.145.89"
+ subject="Thank you"
+ date="2014-06-08T08:36:40Z"
+ content="""
+Thank you so much!
+That solved my problem and restored my precious photos!
+
+Now I see there are similar issues in the forum already, but I was searching for the bad keywords (like \"restore unused\") and I could not find them.
+
+By the way, your software is outstanding.
+"""]]
diff --git a/doc/forum/Need_to_recover_unused_files_because_of_bad_sync__63__/comment_3_b233b4daac32c452776e1e3d9a29f2cc._comment b/doc/forum/Need_to_recover_unused_files_because_of_bad_sync__63__/comment_3_b233b4daac32c452776e1e3d9a29f2cc._comment
new file mode 100644
index 0000000000..faec248bfb
--- /dev/null
+++ b/doc/forum/Need_to_recover_unused_files_because_of_bad_sync__63__/comment_3_b233b4daac32c452776e1e3d9a29f2cc._comment
@@ -0,0 +1,12 @@
+[[!comment format=mdwn
+ username="http://joeyh.name/"
+ ip="108.236.230.124"
+ subject="comment 3"
+ date="2014-06-09T19:44:49Z"
+ content="""
+What did the bad commit that you ended up reverting look like?
+
+It seems to me that something must have went wrong due to your interrupting the first sync on the NAS. Perhaps this left it with newly added files not yet moved into place in the tree. So the next time you synced, this would cause it to see that the files seemed to have been deleted and make a commit reflecting that.
+
+This, then is a bug in how direct mode handles merges. [[bugs/direct_mode_merge_interrupt]]
+"""]]
diff --git a/doc/forum/Need_to_recover_unused_files_because_of_bad_sync__63__/comment_4_8e079e92929dcfdf19f6adec16f800e5._comment b/doc/forum/Need_to_recover_unused_files_because_of_bad_sync__63__/comment_4_8e079e92929dcfdf19f6adec16f800e5._comment
new file mode 100644
index 0000000000..0e079a6944
--- /dev/null
+++ b/doc/forum/Need_to_recover_unused_files_because_of_bad_sync__63__/comment_4_8e079e92929dcfdf19f6adec16f800e5._comment
@@ -0,0 +1,23 @@
+[[!comment format=mdwn
+ username="feulif"
+ ip="151.27.145.89"
+ subject="comment 4"
+ date="2014-06-10T18:43:34Z"
+ content="""
+Here is the log of the \"bad\" commit:
+
+    commit 38e0e1ef009b820f008b46f53c24d9835ca0f9a6
+    Author: me 
+    Date:   Wed Jun 4 00:04:37 2014 +0200
+    
+    git-annex automatic sync
+
+     folder/subfolder1/subfolder2/photo.jpg          | 1 +
+     [...]
+     [...]
+     [...]
+     folder/subfolder1/subfolderN/video.MOV          | 1 +
+     2266 files changed, 2266 insertions(+)
+
+I stopped (ctrl-c) when it was adding the video.
+"""]]
diff --git a/doc/forum/New_Arch_Linux_package_for_git-annex.mdwn b/doc/forum/New_Arch_Linux_package_for_git-annex.mdwn
new file mode 100644
index 0000000000..30deda362a
--- /dev/null
+++ b/doc/forum/New_Arch_Linux_package_for_git-annex.mdwn
@@ -0,0 +1,7 @@
+I have added a new git-annex package in Arch Linux's official repository [1]. I plan to keep it tracking latest release, and the delay should be within a few days after a new release appears.
+
+It would be great to update the installation instructions for Arch,  also please let me know if there is any problem with the package.
+
+Thank you!
+
+[1] https://www.archlinux.org/packages/community/x86_64/git-annex/
diff --git a/doc/forum/New_Arch_Linux_package_for_git-annex/comment_1_5881a5974ec1c4f701775c30b814ee2f._comment b/doc/forum/New_Arch_Linux_package_for_git-annex/comment_1_5881a5974ec1c4f701775c30b814ee2f._comment
new file mode 100644
index 0000000000..39bed8dabe
--- /dev/null
+++ b/doc/forum/New_Arch_Linux_package_for_git-annex/comment_1_5881a5974ec1c4f701775c30b814ee2f._comment
@@ -0,0 +1,7 @@
+[[!comment format=mdwn
+ username="joey"
+ subject="""comment 1"""
+ date="2016-01-15T17:06:09Z"
+ content="""
+This looks nice.. You can edit the install page yourself; this is a wiki.
+"""]]
diff --git a/doc/forum/New_Arch_Linux_package_for_git-annex/comment_2_f41f9248f4d5039309e8fd98524e86bc._comment b/doc/forum/New_Arch_Linux_package_for_git-annex/comment_2_f41f9248f4d5039309e8fd98524e86bc._comment
new file mode 100644
index 0000000000..ac0a1db71c
--- /dev/null
+++ b/doc/forum/New_Arch_Linux_package_for_git-annex/comment_2_f41f9248f4d5039309e8fd98524e86bc._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="https://launchpad.net/~felixonmars"
+ nickname="felixonmars"
+ subject="comment 2"
+ date="2016-01-16T11:35:22Z"
+ content="""
+Thanks. I have updated the page :)
+"""]]
diff --git a/doc/forum/New_Arch_Linux_package_for_git-annex/comment_3_ab1f6e45625e6ab80770d4ff65ec4242._comment b/doc/forum/New_Arch_Linux_package_for_git-annex/comment_3_ab1f6e45625e6ab80770d4ff65ec4242._comment
new file mode 100644
index 0000000000..69fbbc8785
--- /dev/null
+++ b/doc/forum/New_Arch_Linux_package_for_git-annex/comment_3_ab1f6e45625e6ab80770d4ff65ec4242._comment
@@ -0,0 +1,7 @@
+[[!comment format=mdwn
+ username="CandyAngel"
+ subject="comment 3"
+ date="2016-01-18T23:23:48Z"
+ content="""
+As someone who uses Arch a lot.. thank you so much!
+"""]]
diff --git a/doc/forum/New_git-annex_integration_mode_for_Emacs_users.mdwn b/doc/forum/New_git-annex_integration_mode_for_Emacs_users.mdwn
new file mode 100644
index 0000000000..0513e0df8f
--- /dev/null
+++ b/doc/forum/New_git-annex_integration_mode_for_Emacs_users.mdwn
@@ -0,0 +1,3 @@
+I've started a another project to provide Emacs integration for git-annex: [[https://github.com/jwiegley/git-annex-el]]
+
+My problem with the existing mode is that it didn't feel at all Emacsy, while my mode just piggy backs on existing paradigms, like using `C-x C-q` in a locked file to make it editable, and allowing you to browse and lock/unlock annexed files from Dired.
diff --git a/doc/forum/New_user_misunderstandings.mdwn b/doc/forum/New_user_misunderstandings.mdwn
new file mode 100644
index 0000000000..d1b1df180b
--- /dev/null
+++ b/doc/forum/New_user_misunderstandings.mdwn
@@ -0,0 +1,24 @@
+New user? Can't figure out the basics? Add it here - what you wanted, what you tried.
+
+#### I wanted to keep track of some files I had all organized in a directory outside of my ~/annex:
+
+    $ cd ~/annex
+    $ git annex add /path/to/some/photos
+    fatal: '/path/to/some/photos' is outside repository
+
+ But git-annex doesn't work that way. I had to do this instead
+
+    $ rsync -a /path/to/some/photos
+    $ git annex add photos
+    (Recording state in git...)
+    $ git annex status
+    ... lots of helpful info...
+
+#### I just have the OS/X app, can I do commandline stuff?
+
+yes
+
+    $ /Applications/git-annex.app/Contents/MacOS/git-annex add photos/
+    (Recording state in git...)
+
+but perhaps there is a better way.
diff --git a/doc/forum/New_user_misunderstandings/comment_1_c1785924109b5d5cde9aa3d3460cf955._comment b/doc/forum/New_user_misunderstandings/comment_1_c1785924109b5d5cde9aa3d3460cf955._comment
new file mode 100644
index 0000000000..b15f5cfb76
--- /dev/null
+++ b/doc/forum/New_user_misunderstandings/comment_1_c1785924109b5d5cde9aa3d3460cf955._comment
@@ -0,0 +1,10 @@
+[[!comment format=mdwn
+ username="http://joeyh.name/"
+ ip="4.153.8.117"
+ subject="comment 1"
+ date="2012-12-17T16:21:36Z"
+ content="""
+For importing your photos, you can use `git annex import /path/to/some/photos` .. this will move them from the given location into the current directory, and add them to the annex. It preserves subdirectory structure too.
+
+For using the OSX app at the command line, the only thing that is guaranteed to work properly is to run `/Applications/git-annex.app/Contents/MacOS/runshell`, which will modify your PATH and other environment appropriately.
+"""]]
diff --git a/doc/forum/Newb_question_on_sneekernet_sync.mdwn b/doc/forum/Newb_question_on_sneekernet_sync.mdwn
new file mode 100644
index 0000000000..7703bbec18
--- /dev/null
+++ b/doc/forum/Newb_question_on_sneekernet_sync.mdwn
@@ -0,0 +1,7 @@
+Hi,
+
+I have set up git-annex to sync a folder located on laptop A, USB Key, and laptop B.   After reading the walkthrough, I don't quite understand the proper procedure to keep in sync.  Should I run git-annex sync from within the sync directory on the USB Key on laptop A and then from within the sync folder on the USB Key when plugged into laptop B?  Or do I have to run git-annex sync from within the sync directory on the laptops, then the USB Key?  And/or do I have to run git-annex get on the sync directory on the USB key before syncing to the second laptop?
+
+Also I should not normally be making changes on laptops A and B at the same time, but if that were the case, is there any difference in what I would run to sync the changes?
+
+Thanks!
diff --git a/doc/forum/Newb_question_on_sneekernet_sync/comment_1_409cc5ba605d63331b386066d32f797b._comment b/doc/forum/Newb_question_on_sneekernet_sync/comment_1_409cc5ba605d63331b386066d32f797b._comment
new file mode 100644
index 0000000000..691d401372
--- /dev/null
+++ b/doc/forum/Newb_question_on_sneekernet_sync/comment_1_409cc5ba605d63331b386066d32f797b._comment
@@ -0,0 +1,31 @@
+[[!comment format=mdwn
+ username="https://www.google.com/accounts/o8/id?id=AItOawkutSE8_3fFAETmO_E598zja4gKwYXbb8E"
+ nickname="Сергей"
+ subject="comment 1"
+ date="2015-04-08T12:31:33Z"
+ content="""
+Well, working with git annex, you should think of two things:
+
+1. location of annex information (git commit tree), let's call it INFO
+2. location of files contents, let's call it CONTENT
+
+So, for your sneekernet scenario these general steps are required:
+
+1. sync INFO between Laptop A and USB
+2. transfer required CONTENT from LAPTOP A to USB
+3. sync INFO between Laptop B and USB
+4. transfer required CONTENT from USB to Laptop B
+
+Let's describe your options on every step:
+
+1. issue `git annex sync` on Laptop A. Either from within USB or Laptop repo, I prefer Laptop because of automatic commit.
+2. 
+   * issue `git annex copy  --to=` from within Laptop A repo
+   * issue `git annex get ` from within USB repo. This is not possible if you have bare repo on USB.
+3. issue `git annex sync` on Laptop B
+4. same as 2, but transfer to Laptop B
+
+Also, using `git annex sync --content`, you don't need to manually transfer CONTENT, see [--content](https://git-annex.branchable.com/git-annex-sync/). But in this case you somewhat loose control of what is transferred.
+
+If you make changes on both laptops at the same time no extra steps required. In case of conflict both versions saved in working directory, see [automatic conflict resolution](http://git-annex.branchable.com/automatic_conflict_resolution/).
+"""]]
diff --git a/doc/forum/Newbie_question_for_a_simple_task.mdwn b/doc/forum/Newbie_question_for_a_simple_task.mdwn
new file mode 100644
index 0000000000..49db807956
--- /dev/null
+++ b/doc/forum/Newbie_question_for_a_simple_task.mdwn
@@ -0,0 +1,11 @@
+Hi,
+
+I am pretty confident, that this is a newbie question. Nevertheless, I did not find the answer (or the solution) event after looking at the screencasts and reading through the files.
+
+I want to sync 2 clients on different networks though a ssh cloud server begin a full backup.
+Also, I would like to do this for 2 different folders on the clients and server.
+I mean : client 1, 2 and the server sync a repo named "DATA" and also sync another repo named "IMAGES"
+
+Setting the client1 as local client repo and also ssh remote repo works quite well, it uploads everything to the server. Then I make the same on the second client, and it does not start to download everything from the server...
+
+Any clue ?
diff --git a/doc/forum/Newbie_question_for_a_simple_task/comment_1_51cf643c8d843dd2ab49c83aaf4fd218._comment b/doc/forum/Newbie_question_for_a_simple_task/comment_1_51cf643c8d843dd2ab49c83aaf4fd218._comment
new file mode 100644
index 0000000000..3f34ca14a2
--- /dev/null
+++ b/doc/forum/Newbie_question_for_a_simple_task/comment_1_51cf643c8d843dd2ab49c83aaf4fd218._comment
@@ -0,0 +1,9 @@
+[[!comment format=mdwn
+ username="Régis"
+ subject="comment 1"
+ date="2015-02-23T09:20:54Z"
+ content="""
+Ok, I found PART of the answer here : 
+
+It is mentioned to use Jabber to make both clients message each other when there is something new to sync. What about my setup with 2 groups of repos syncing together each for itself ? I read somewhere, that the jabber account will sync ALL repos. Is the solution to create 2 Jabber accounts ?
+"""]]
diff --git a/doc/forum/Newbie_question_for_a_simple_task/comment_2_a9d18e6fb1c6b064f50467c23bc89d2e._comment b/doc/forum/Newbie_question_for_a_simple_task/comment_2_a9d18e6fb1c6b064f50467c23bc89d2e._comment
new file mode 100644
index 0000000000..8002298a23
--- /dev/null
+++ b/doc/forum/Newbie_question_for_a_simple_task/comment_2_a9d18e6fb1c6b064f50467c23bc89d2e._comment
@@ -0,0 +1,9 @@
+[[!comment format=mdwn
+ username="joey"
+ subject="""comment 2"""
+ date="2015-02-25T19:18:58Z"
+ content="""
+Since you have a ssh server, it's better to avoid jabber. Install
+git-annex 5.20140405 or newer on the server, and the clients will
+use it to communicate with one-another.
+"""]]
diff --git a/doc/forum/Newbie_stuck_at___34__Unable_to_connect_to_the_Jabber_server__34__.mdwn b/doc/forum/Newbie_stuck_at___34__Unable_to_connect_to_the_Jabber_server__34__.mdwn
new file mode 100644
index 0000000000..eb1aa3b50b
--- /dev/null
+++ b/doc/forum/Newbie_stuck_at___34__Unable_to_connect_to_the_Jabber_server__34__.mdwn
@@ -0,0 +1,5 @@
+I'm brand new to git-annex and am having trouble getting going.  I've got an annex and I get messages like "Ran daily sanity check" and "Synced with web", but I can't get connected to a Jabber account.  I've been using my Gmail account, which works fine for logging into this forum but gives me the message "Unable to connect to the Jabber server. Maybe you entered the wrong password?" when I try to configure the Jabber account.
+
+I'm using version 3.20121112ubuntu2.  I think a later version might exist but I don't know how to reach it; I've done the standard install for Ubuntu-style OSs (I'm running Kubuntu right now), and that's the git-annex version that arrives.
+
+Any suggestions?  Sorry for the simplistic startup question, but it looks like this will be a great tool once I get it running.
diff --git a/doc/forum/Newbie_stuck_at___34__Unable_to_connect_to_the_Jabber_server__34__/comment_1_59158afcedac18a7285d57491b2a468a._comment b/doc/forum/Newbie_stuck_at___34__Unable_to_connect_to_the_Jabber_server__34__/comment_1_59158afcedac18a7285d57491b2a468a._comment
new file mode 100644
index 0000000000..45b4a9bdff
--- /dev/null
+++ b/doc/forum/Newbie_stuck_at___34__Unable_to_connect_to_the_Jabber_server__34__/comment_1_59158afcedac18a7285d57491b2a468a._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="http://joeyh.name/"
+ ip="4.152.246.110"
+ subject="comment 1"
+ date="2013-07-27T22:13:49Z"
+ content="""
+Since 2012 git-annex has had massive improvements to its Jabber support. See [[/install/Ubuntu]] to get a current version.
+"""]]
diff --git a/doc/forum/Newbie_stuck_at___34__Unable_to_connect_to_the_Jabber_server__34__/comment_2_2a70ac08bb95774415b09dab7d7f8605._comment b/doc/forum/Newbie_stuck_at___34__Unable_to_connect_to_the_Jabber_server__34__/comment_2_2a70ac08bb95774415b09dab7d7f8605._comment
new file mode 100644
index 0000000000..777a238510
--- /dev/null
+++ b/doc/forum/Newbie_stuck_at___34__Unable_to_connect_to_the_Jabber_server__34__/comment_2_2a70ac08bb95774415b09dab7d7f8605._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="https://www.google.com/accounts/o8/id?id=AItOawlutNrg7ajiNAfi5gqJfD8crv0aimZa27k"
+ nickname="Chuck"
+ subject="Now I'm really confused 8-/"
+ date="2013-07-28T05:26:02Z"
+ content="""
+Yes, I followed the directions on that page and executed \"sudo apt-get install git-annex\" on my 13.04 (Raring) system.  I did it again just now to verify, and I got the same result from \"git-annex version\", namely \"3.20121112ubuntu2\".  Do I need to add a repo for Raring as for Precise? 
+"""]]
diff --git a/doc/forum/Newbie_stuck_at___34__Unable_to_connect_to_the_Jabber_server__34__/comment_3_92a52b523ed4c68b70ddcabc2a050b76._comment b/doc/forum/Newbie_stuck_at___34__Unable_to_connect_to_the_Jabber_server__34__/comment_3_92a52b523ed4c68b70ddcabc2a050b76._comment
new file mode 100644
index 0000000000..9b1d95e788
--- /dev/null
+++ b/doc/forum/Newbie_stuck_at___34__Unable_to_connect_to_the_Jabber_server__34__/comment_3_92a52b523ed4c68b70ddcabc2a050b76._comment
@@ -0,0 +1,12 @@
+[[!comment format=mdwn
+ username="https://www.google.com/accounts/o8/id?id=AItOawmKKg3Vmzk7KwRGRKjHVdtyoj1JfxLX6NM"
+ nickname="Tom"
+ subject="comment 3"
+ date="2013-10-01T18:33:05Z"
+ content="""
+I've got the same issue on Xubuntu 13.04. I installed using this script: https://github.com/zerodogg/scriptbucket/blob/master/gitannex-install
+
+`git-annex version` makes no mention of DNS or ADNS
+
+`host` command is installed on my machine. any suggestions on how best to fix for this setup?
+"""]]
diff --git a/doc/forum/Newbie_stuck_at___34__Unable_to_connect_to_the_Jabber_server__34__/comment_4_c52a75761ea107f6d69c09bac64f0f0a._comment b/doc/forum/Newbie_stuck_at___34__Unable_to_connect_to_the_Jabber_server__34__/comment_4_c52a75761ea107f6d69c09bac64f0f0a._comment
new file mode 100644
index 0000000000..b15edfa5a6
--- /dev/null
+++ b/doc/forum/Newbie_stuck_at___34__Unable_to_connect_to_the_Jabber_server__34__/comment_4_c52a75761ea107f6d69c09bac64f0f0a._comment
@@ -0,0 +1,12 @@
+[[!comment format=mdwn
+ username="https://www.google.com/accounts/o8/id?id=AItOawmKKg3Vmzk7KwRGRKjHVdtyoj1JfxLX6NM"
+ nickname="Tom"
+ subject="comment 4"
+ date="2013-10-02T21:27:36Z"
+ content="""
+Update to my comment above - it seems my problem is related to ejabberd as discussed here: 
+
+Think I might change my server to [Prosody](https://prosody.im/) and see what happens then.
+
+
+"""]]
diff --git a/doc/forum/Newbie_stuck_at___34__Unable_to_connect_to_the_Jabber_server__34__/comment_5_2685e3a87464ccd37d593516d94ba5cf._comment b/doc/forum/Newbie_stuck_at___34__Unable_to_connect_to_the_Jabber_server__34__/comment_5_2685e3a87464ccd37d593516d94ba5cf._comment
new file mode 100644
index 0000000000..84d6832ff3
--- /dev/null
+++ b/doc/forum/Newbie_stuck_at___34__Unable_to_connect_to_the_Jabber_server__34__/comment_5_2685e3a87464ccd37d593516d94ba5cf._comment
@@ -0,0 +1,10 @@
+[[!comment format=mdwn
+ username="https://www.google.com/accounts/o8/id?id=AItOawkr_EUIHZ8CTtOGm-j-guMBzaYuOzrXJwg"
+ nickname="Christoph"
+ subject="be aware of the 2-step verification"
+ date="2013-10-15T20:03:12Z"
+ content="""
+if you enabled the \"2-step verification\" in our google account (Account -> Security -> 2-step verification: Status: ON) you have to make a \"Application-specific passwords\" for your Jabber login.
+
+NoXorius
+"""]]
diff --git a/doc/forum/No_SSL_traffic_for_S3__63__.mdwn b/doc/forum/No_SSL_traffic_for_S3__63__.mdwn
new file mode 100644
index 0000000000..c2f4e3717f
--- /dev/null
+++ b/doc/forum/No_SSL_traffic_for_S3__63__.mdwn
@@ -0,0 +1,8 @@
+As far I can tell, the encryption mentioned on the wiki for S3 [1] refers to file-level encryption. That is, it encrypts files with GPG before storing them (or after retrieving them) from S3. However, even if I have GPG encryption off, I still want S3 requests to use SSL encryption. As far as I can tell, HTTPS isn't used with S3 remotes. Is there any way to enable it (if so, it should be the default)?
+
+I am using git-annex version: 3.20120406
+
+Cheers,
+--acrefoot
+
+[1] http://git-annex.branchable.com/special_remotes/S3/
diff --git a/doc/forum/No_SSL_traffic_for_S3__63__/comment_1_f509bf273896180e6df8c771438dd093._comment b/doc/forum/No_SSL_traffic_for_S3__63__/comment_1_f509bf273896180e6df8c771438dd093._comment
new file mode 100644
index 0000000000..808d4c0351
--- /dev/null
+++ b/doc/forum/No_SSL_traffic_for_S3__63__/comment_1_f509bf273896180e6df8c771438dd093._comment
@@ -0,0 +1,11 @@
+[[!comment format=mdwn
+ username="http://joeyh.name/"
+ ip="4.154.7.238"
+ subject="comment 1"
+ date="2013-01-15T20:25:42Z"
+ content="""
+ is a Haskell library for S3, which git-annex uses. It does not support HTTPS. I'm sure its author would appreciate help, or maybe even just gentle motivation.
+
+FWIW, I think that S3's authorization is designed to be pretty secure even over an un-encrypted transport.
+It uses HMAC to sign the request with your AWS credentials securely, and includes a date that is hopefully used to avoid replay attacks.
+"""]]
diff --git a/doc/forum/No_SSL_traffic_for_S3__63__/comment_2_358635d19c82202c63014ca84de7fc3b._comment b/doc/forum/No_SSL_traffic_for_S3__63__/comment_2_358635d19c82202c63014ca84de7fc3b._comment
new file mode 100644
index 0000000000..edf1b63381
--- /dev/null
+++ b/doc/forum/No_SSL_traffic_for_S3__63__/comment_2_358635d19c82202c63014ca84de7fc3b._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="https://www.google.com/accounts/o8/id?id=AItOawnRIwPkmDOZSpul-Lx61_jUGHh9f_F3od8"
+ nickname="Michael"
+ subject="comment 2"
+ date="2013-01-15T20:47:50Z"
+ content="""
+The authorization is fine, but it's scary to see your data floating along the wire in plain-text :)
+"""]]
diff --git a/doc/forum/No_output_from_git-annex_status_v5.20131224-g692aa01.mdwn b/doc/forum/No_output_from_git-annex_status_v5.20131224-g692aa01.mdwn
new file mode 100644
index 0000000000..b13cab9bf9
--- /dev/null
+++ b/doc/forum/No_output_from_git-annex_status_v5.20131224-g692aa01.mdwn
@@ -0,0 +1,16 @@
+I'm running version 5.20131224-g692aa01 standalone tarball on linux x86_64 (Linux Mint 15). When I run `git-annex status` in the terminal I get no output. 
+
+I've tried:
+
+* system linked (`ln -s`) the `git-annex`, `git-annex-shell`, and `git-annex-webapp` to `~/bin/` which is on my PATH
+* run the `runshell` program from the terminal 
+
+Other things to consider:
+
+* I've just upgraded the repo from version 3 to version 5
+* Repo was created using git-annex version 3 
+* I don't use assistant or the webapp
+* Only used from my terminal
+* Repo is in indirect mode
+
+What do I need to do to see the results of `git-annex status`? in my terminal?
diff --git a/doc/forum/No_output_from_git-annex_status_v5.20131224-g692aa01/comment_1_b014f1edcb7ce39da9b582683d3b80c0._comment b/doc/forum/No_output_from_git-annex_status_v5.20131224-g692aa01/comment_1_b014f1edcb7ce39da9b582683d3b80c0._comment
new file mode 100644
index 0000000000..cffcbab75e
--- /dev/null
+++ b/doc/forum/No_output_from_git-annex_status_v5.20131224-g692aa01/comment_1_b014f1edcb7ce39da9b582683d3b80c0._comment
@@ -0,0 +1,10 @@
+[[!comment format=mdwn
+ username="http://joeyh.name/"
+ ip="209.250.56.227"
+ subject="comment 1"
+ date="2013-12-26T23:06:34Z"
+ content="""
+Run `git annex info` instead. That is what older versions of git-annex called `git annex status`
+
+Meanwhile, `git annex status` is now like `git status` -- it shows an overview of uncommitted changes in your work tree.
+"""]]
diff --git a/doc/forum/No_output_from_git-annex_status_v5.20131224-g692aa01/comment_2_24602de6cfe1f3d988c5105e7266a518._comment b/doc/forum/No_output_from_git-annex_status_v5.20131224-g692aa01/comment_2_24602de6cfe1f3d988c5105e7266a518._comment
new file mode 100644
index 0000000000..b09cf9757f
--- /dev/null
+++ b/doc/forum/No_output_from_git-annex_status_v5.20131224-g692aa01/comment_2_24602de6cfe1f3d988c5105e7266a518._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="https://openid.stackexchange.com/user/e65e6d0e-58ba-41de-84cc-1f2ba54cf574"
+ nickname="Mica Semrick"
+ subject="Thanks!"
+ date="2013-12-27T01:37:36Z"
+ content="""
+Not sure how I missed that :D
+"""]]
diff --git a/doc/forum/Not_sure_how_to_get_my_s3_remote_back.mdwn b/doc/forum/Not_sure_how_to_get_my_s3_remote_back.mdwn
new file mode 100644
index 0000000000..86f8da4548
--- /dev/null
+++ b/doc/forum/Not_sure_how_to_get_my_s3_remote_back.mdwn
@@ -0,0 +1,31 @@
+My situation goes something like this:
+
+I have a machine with an annex and a number of special remotes (s3, box, and an rsync'd nas).  I also have a git remote that's doesn't have git annex on it, so it's just got the git branches.  That machine has some problems so I start getting the annex set up on a different machine.  These are the steps that I went through:
+
+* clone from the git remote
+* git annex doesn't think this is an actual annex at this point, so `git annex init` it
+* Now I can start the webapp `git annex webapp`
+* My rsync'd remote works and the assistant starts downloading the files from there (which is great, it's the local network one) but the box and s3 remotes are disabled (sure, not a huge deal).
+* Click enable on the box remote, I have to specify that it's a full backup remote
+* Things start copying from box as well, so I disable it until everything from the local network is done
+* Click enable on the s3 remote and it wants my AWS creds
+* Download those and add it
+* Set the remote up the same way, as a full backup.
+
+At this point, all my files have copied from the rsync remote, so I enable the other remotes.
+
+Now I want to make sure that all the remotes are set up and working properly.
+
+I turn off the webapp, turn off direct mode (I think it was indirect mode by default, but I'd been playing with things before then), and `git annex drop ` a file that I don't particularly care about.  Everything drops successfully.
+
+I'm able to `git annex get -f  ` and from `` successfully, but when I try to get from the s3 remote, it doesn't give me any output and doesn't download the file.
+
+Having used regular git annex without the assistant before, I try re-initing the remote `git annex initremote `.  It complains that there's no type, so I `git annex initremote  type=S3`, then it complains about encryption.  `git annex initremote  type=S3 encryption=shared`.  It says it worked, but I `git annex get -f  ` still doesn't do anything.
+
+After more looking around, it turns out that I may have created a second remote with the same name as the original s3 remote... (figure that out).  I use the webapp to rename the remotes so they're different, but neither of them will `get -f` successfully.
+
+At this point, I turn to you and ask what the heck I did wrong.  I tried editing the remote.log and uuid.log files to remove the new s3 remote (and I figured out which one was which) from the git-annex branch.  I also marked the new s3 remote as dead, but I still can't get access to s3.
+
+`git annex fsck -f ` doesn't actually seem to hit s3 (I seem to recall that it used to, it calculated checksums), it just checks some git local information.
+
+I don't mind deleting my current checkout and starting from the clone step again if you think I've made too much of a mess.  At least I know I can get my files off my rsync remote and box :)
diff --git a/doc/forum/Not_sure_how_to_get_my_s3_remote_back/comment_10_ed35a6ec605e8f79ec107856af6d1a46._comment b/doc/forum/Not_sure_how_to_get_my_s3_remote_back/comment_10_ed35a6ec605e8f79ec107856af6d1a46._comment
new file mode 100644
index 0000000000..eda042c832
--- /dev/null
+++ b/doc/forum/Not_sure_how_to_get_my_s3_remote_back/comment_10_ed35a6ec605e8f79ec107856af6d1a46._comment
@@ -0,0 +1,18 @@
+[[!comment format=mdwn
+ username="http://joeyh.name/"
+ nickname="joey"
+ subject="comment 10"
+ date="2013-04-26T20:33:03Z"
+ content="""
+Oh.. I got confused by you talking about the box remote. Lines you pasted look ok anyway.
+
+Ok, looking at the S3 remote then...
+
+> I did notice that initially the s3 remote was named \"annex\". That was probably the web interface's doing, way back when I added it.
+
+So, you can never change the names used to refer to remotes in remote.log. These names can be different from the names used to refer to the same remotes in .git/config. (Which can vary from repository to repository anyway..) So, if you originally added a s3 remote and called it \"annex\", you still need to use that name when running initremote elsewhere to add that remote to your repository.
+
+The remote with name \"s3\" added in the 11:26 is a separate s3 remote, and I think one you don't want. (And have marked dead?)
+
+I think all you need to do is \"git annex initremote annex\" to add the s3 remote you want to your new repository.
+"""]]
diff --git a/doc/forum/Not_sure_how_to_get_my_s3_remote_back/comment_11_e48b6efa42159dc83e1be11bfb54abcd._comment b/doc/forum/Not_sure_how_to_get_my_s3_remote_back/comment_11_e48b6efa42159dc83e1be11bfb54abcd._comment
new file mode 100644
index 0000000000..41da88a8d3
--- /dev/null
+++ b/doc/forum/Not_sure_how_to_get_my_s3_remote_back/comment_11_e48b6efa42159dc83e1be11bfb54abcd._comment
@@ -0,0 +1,14 @@
+[[!comment format=mdwn
+ username="https://www.google.com/accounts/o8/id?id=AItOawkZktNHFhxC1kYA9KKdKpYJO4clq9WDsjE"
+ nickname="Jason"
+ subject="comment 11"
+ date="2013-04-26T20:37:51Z"
+ content="""
+Ah, I see.  It looks like that did solve my problem.
+
+Yes, I did mark the old s3 remote as dead.
+
+At least now I know how to fix it if it ever happens again.  I wonder if I'll ever be able to recreate it...
+
+Thanks!
+"""]]
diff --git a/doc/forum/Not_sure_how_to_get_my_s3_remote_back/comment_12_b58232d0e3fa4649565c0c7d4ce2e82e._comment b/doc/forum/Not_sure_how_to_get_my_s3_remote_back/comment_12_b58232d0e3fa4649565c0c7d4ce2e82e._comment
new file mode 100644
index 0000000000..8b13675fe7
--- /dev/null
+++ b/doc/forum/Not_sure_how_to_get_my_s3_remote_back/comment_12_b58232d0e3fa4649565c0c7d4ce2e82e._comment
@@ -0,0 +1,31 @@
+[[!comment format=mdwn
+ username="http://joeyh.name/"
+ nickname="joey"
+ subject="comment 12"
+ date="2013-04-26T20:52:36Z"
+ content="""
+It's easy to recreate. As I understand it, the entire process went something like this:
+
+git annex initremote annex type=S3 encryption=blahblah # possibly this was done in the webapp?
+
+git remote rename annex s3 # also possibly done in the webapp
+
+# clone to different computer, and on the new clone:
+
+git annex initremote s3
+
+git-annex: Specify the type of remote with type=
+
+git annex initremote s3 type=S3 encryption=blahblah
+
+The last line creates a *new* remote.
+
+I'm inclined to think the main confusing thing here is that initremote is used to both create a new special remote, and to configure the repository to use an already existing special remote that was created elsewhere. If you had to use `enableremote` for the latter,
+things could be less confusing:
+
+# clone to different computer, and on the new clone:
+
+git annex enableremote s3
+
+git-annex: No existing special remote named s3. Choose from one of these existing special remotes: annex
+"""]]
diff --git a/doc/forum/Not_sure_how_to_get_my_s3_remote_back/comment_13_85368b60091dc3ce2efb58013ffe9f83._comment b/doc/forum/Not_sure_how_to_get_my_s3_remote_back/comment_13_85368b60091dc3ce2efb58013ffe9f83._comment
new file mode 100644
index 0000000000..c51377dbbe
--- /dev/null
+++ b/doc/forum/Not_sure_how_to_get_my_s3_remote_back/comment_13_85368b60091dc3ce2efb58013ffe9f83._comment
@@ -0,0 +1,10 @@
+[[!comment format=mdwn
+ username="https://www.google.com/accounts/o8/id?id=AItOawkZktNHFhxC1kYA9KKdKpYJO4clq9WDsjE"
+ nickname="Jason"
+ subject="comment 13"
+ date="2013-04-26T21:16:30Z"
+ content="""
+I tend to agree with you.  At first I liked the idea that initremote could be used to re-initialize a remote, but then I got confused about what the name of that remote was.  I suppose git annex status could have told me.  I kept wanting to have something like \"git annex remote\" (which would list them) and then \"git annex remote init\" to initialize them.  That way the remote actions would follow the same sort of interface as \"git remote\", where you could list, init, create, edit, rename, enable, disable, kill (dead?), etc.  The main drawback I see with that is having too many levels to type.
+
+I really like the idea of having the ability to \"git annex remote show s3\" and it will tell me what the type, uuid, options, etc are for that remote.
+"""]]
diff --git a/doc/forum/Not_sure_how_to_get_my_s3_remote_back/comment_14_e65281bef23e0076936c508728a87897._comment b/doc/forum/Not_sure_how_to_get_my_s3_remote_back/comment_14_e65281bef23e0076936c508728a87897._comment
new file mode 100644
index 0000000000..31fd9d60b0
--- /dev/null
+++ b/doc/forum/Not_sure_how_to_get_my_s3_remote_back/comment_14_e65281bef23e0076936c508728a87897._comment
@@ -0,0 +1,25 @@
+[[!comment format=mdwn
+ username="http://joeyh.name/"
+ nickname="joey"
+ subject="comment 14"
+ date="2013-04-26T22:25:04Z"
+ content="""
+Have now split out an enableremote command.
+
+
    +joey@gnu:~/tmp/annex>git annex initremote foo                                 
+git-annex: There is already a special remote named \"foo\". (Use enableremote to enable an existing special remote.)
+joey@gnu:~/tmp/annex>git annex enableremote
+git-annex: Specify the name of the special remote to enable. Known special remotes: foo
+
    
+
+Also, I wrote something wrong before. It *is* possible to change the name used by initremote (now enableremote).
+
+With the current release of git-annex:
+
+`git annex initremote annex name=mys3`
+
+With the next release:
+
+`git annex enableremote annex name=mys3`
+"""]]
diff --git a/doc/forum/Not_sure_how_to_get_my_s3_remote_back/comment_1_fffb59ad5a197d2980dd0ec35cf4aafa._comment b/doc/forum/Not_sure_how_to_get_my_s3_remote_back/comment_1_fffb59ad5a197d2980dd0ec35cf4aafa._comment
new file mode 100644
index 0000000000..d8d754309c
--- /dev/null
+++ b/doc/forum/Not_sure_how_to_get_my_s3_remote_back/comment_1_fffb59ad5a197d2980dd0ec35cf4aafa._comment
@@ -0,0 +1,10 @@
+[[!comment format=mdwn
+ username="http://joeyh.name/"
+ nickname="joey"
+ subject="comment 1"
+ date="2013-04-24T23:56:54Z"
+ content="""
+What you describe, `git annex get $file --from remote` silently not doing anything, is the expected behavior if the remote doesn't have the file. This allows you to eg, run `git annex get . --from remote` and get all files that the remote does have while skipping the rest.
+
+Did you ever try looking at `git annex whereis $file` ? 
+"""]]
diff --git a/doc/forum/Not_sure_how_to_get_my_s3_remote_back/comment_2_0cfcc2075bff556b9fde5acc3dc1d599._comment b/doc/forum/Not_sure_how_to_get_my_s3_remote_back/comment_2_0cfcc2075bff556b9fde5acc3dc1d599._comment
new file mode 100644
index 0000000000..c673737c1b
--- /dev/null
+++ b/doc/forum/Not_sure_how_to_get_my_s3_remote_back/comment_2_0cfcc2075bff556b9fde5acc3dc1d599._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="https://www.google.com/accounts/o8/id?id=AItOawkZktNHFhxC1kYA9KKdKpYJO4clq9WDsjE"
+ nickname="Jason"
+ subject="comment 2"
+ date="2013-04-25T00:20:59Z"
+ content="""
+When I run `git annex whereis $file` it tells me it's available at box, s3, my rsync remote, the original repo (from the machine I created the annex on), and my temporary usb stick repo (nothing wrong with redundancy...).  So it seems that git annex thinks the file is available in the s3 remote, even though it's refusing to download it.
+"""]]
diff --git a/doc/forum/Not_sure_how_to_get_my_s3_remote_back/comment_3_6fe2ff1282fb14a4ce26ef8dc775d07e._comment b/doc/forum/Not_sure_how_to_get_my_s3_remote_back/comment_3_6fe2ff1282fb14a4ce26ef8dc775d07e._comment
new file mode 100644
index 0000000000..187993a5ab
--- /dev/null
+++ b/doc/forum/Not_sure_how_to_get_my_s3_remote_back/comment_3_6fe2ff1282fb14a4ce26ef8dc775d07e._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="http://joeyh.name/"
+ nickname="joey"
+ subject="comment 3"
+ date="2013-04-25T00:50:01Z"
+ content="""
+Does the uuid that whereis prints next to the name of the s3 remote match the annex..uuid setting for that remote in .git/config?
+"""]]
diff --git a/doc/forum/Not_sure_how_to_get_my_s3_remote_back/comment_4_64338d2d77dcbabd16b55eb145f40dc6._comment b/doc/forum/Not_sure_how_to_get_my_s3_remote_back/comment_4_64338d2d77dcbabd16b55eb145f40dc6._comment
new file mode 100644
index 0000000000..d4a05ce8e2
--- /dev/null
+++ b/doc/forum/Not_sure_how_to_get_my_s3_remote_back/comment_4_64338d2d77dcbabd16b55eb145f40dc6._comment
@@ -0,0 +1,12 @@
+[[!comment format=mdwn
+ username="https://www.google.com/accounts/o8/id?id=AItOawkZktNHFhxC1kYA9KKdKpYJO4clq9WDsjE"
+ nickname="Jason"
+ subject="comment 4"
+ date="2013-04-25T01:18:00Z"
+ content="""
+Oh! That might be it!  During the whole \"I have two remotes with the name s3\" situation, it seems that both of them in my .git/config ended up with the same uuid, even though the original one had a different uuid.  If I change it back, I end up getting an access denied when I try to `git annex get ...`.  Progress!
+
+I thought that you were supposed to do a `git annex initremote s3` from a clone to enable a remote with the credentials stored in the repo.  It seems that internally something still thinks that the \"s3\" remote has the new uuid.  When I run that command it changes the uuid back to the new (invalid) one.
+
+Is there a way I can totally remove the bad s3 (which I've partially renamed to s3thefirst) remote from my history/repo (I'm pretty sure it's been synced back up to origin at this point) or properly rename it so it doesn't keep getting confused?  Hopefully that will address my problem.
+"""]]
diff --git a/doc/forum/Not_sure_how_to_get_my_s3_remote_back/comment_5_dd66c9ea0c83388f6826751944330d10._comment b/doc/forum/Not_sure_how_to_get_my_s3_remote_back/comment_5_dd66c9ea0c83388f6826751944330d10._comment
new file mode 100644
index 0000000000..58fba2ca8f
--- /dev/null
+++ b/doc/forum/Not_sure_how_to_get_my_s3_remote_back/comment_5_dd66c9ea0c83388f6826751944330d10._comment
@@ -0,0 +1,16 @@
+[[!comment format=mdwn
+ username="http://joeyh.name/"
+ nickname="joey"
+ subject="comment 5"
+ date="2013-04-25T02:14:18Z"
+ content="""
+Yes, when you run `git annex initremote $remotename` with no other parameters, it enables a remote from the stored configuration.
+Which does not include `AWS_SECRET_ACCESS_KEY` and `AWS_ACCESS_KEY_ID`; you need to set those and then
+you should not get access denied.
+
+You seem to say your .git/config contains two remotes with the same name, but I don't think that's possible.
+
+I don't know how you could end up with two remotes with the same name in `git show git-annex:remote.log`, unless the two were added in separate repositories which were then synced together. Since this is not a usual situation there's not any UI to deal with it. I've just committed a change that will make `initremote` prefer remotes that have not been marked dead when there's a naming comflict.
+
+However, I'm more curious how this situation came about.  I have not been able to reproduce the problem when enabling a S3 remote using the webapp.
+"""]]
diff --git a/doc/forum/Not_sure_how_to_get_my_s3_remote_back/comment_6_dc0c5e395e4c443b7227afdb157194e5._comment b/doc/forum/Not_sure_how_to_get_my_s3_remote_back/comment_6_dc0c5e395e4c443b7227afdb157194e5._comment
new file mode 100644
index 0000000000..c9c352c352
--- /dev/null
+++ b/doc/forum/Not_sure_how_to_get_my_s3_remote_back/comment_6_dc0c5e395e4c443b7227afdb157194e5._comment
@@ -0,0 +1,10 @@
+[[!comment format=mdwn
+ username="http://joeyh.name/"
+ nickname="joey"
+ subject="comment 6"
+ date="2013-04-25T02:18:56Z"
+ content="""
+What you could do to help track down how this occurred is to check out the `git-annex` branch, and use `git blame` to find out when the second remote with the same name was first added to the `remote.log` file.
+
+Then you should be able to tell, either from the email address used for that commit, or at least the date of the commit, whether this occurred recently when you enabled the S3 remote in the webapp, or perhaps at some time in the past.
+"""]]
diff --git a/doc/forum/Not_sure_how_to_get_my_s3_remote_back/comment_7_3c0ea4c76cdd889707f7308576e3efa0._comment b/doc/forum/Not_sure_how_to_get_my_s3_remote_back/comment_7_3c0ea4c76cdd889707f7308576e3efa0._comment
new file mode 100644
index 0000000000..b325328947
--- /dev/null
+++ b/doc/forum/Not_sure_how_to_get_my_s3_remote_back/comment_7_3c0ea4c76cdd889707f7308576e3efa0._comment
@@ -0,0 +1,65 @@
+[[!comment format=mdwn
+ username="https://www.google.com/accounts/o8/id?id=AItOawkZktNHFhxC1kYA9KKdKpYJO4clq9WDsjE"
+ nickname="Jason"
+ subject="comment 7"
+ date="2013-04-26T19:10:15Z"
+ content="""
+http://pastebin.com/CM2EfQ21
+
+This is what the commit log looks like for the remote.log file.  There is some interesting stuff in here.  I'll try to highlight the changes without giving too much of the important bits away.
+
+The I commit at 2013-04-22 11:57 is when I added the box remote:
+
+0490d177-78e2-421b-a004-47d88ee7a2e3 chunksize=10mb cipher=... davcreds=... embedcreds=yes name=box.com type=webdav url=https://www.box.com/dav/annex timestamp=1366657062.972357s
+1d0ab67c-6a43-11e2-9feb-df22c6d1e308 bucket=annex-1d0ab67c-6a43-11e2-9feb-df22c6d1e308 cipher=... datacenter=US host=s3.amazonaws.com name=annex port=80 storageclass=REDUCED_REDUNDANCY type=S3 timestamp=1359484726.520727s
+
+The contents also includes my nas remote, but I will omit that for brevity's sake.  I did notice that initially the s3 remote was named \"annex\".  That was probably the web interface's doing, way back when I added it.
+
+The next commit at 2013-04-24 10:55 seems to have added encryption=shared and highRandomQuality=false to the nas remote (I think this was when I re-enabled the nas remote through the webapp).
+
+The commit at 2013-04-24 11:05 looks like it added similar stuff to the box remote (added highRandomQuality=false).  Probably this was from enabling it then as well.
+
+At 2013-04-24 11:12 the s3 remote had highRandomQuality=false added also.
+
+At 2013-04-24 11:26, a new remote was added:
+
+4d86972d-9b0a-4095-bc50-f9bec8144c30 bucket=s3-4d86972d-9b0a-4095-bc50-f9bec8144c30 cipher=... datacenter=US host=s3.amazonaws.com name=s3 port=80 storageclass=STANDARD type=S3 timestamp=1366828017.8792s
+
+Very possibly this was me doing a `git annex initremote ...` thinking that the s3 remote was actually named s3 (somehow, I feel like I would have checked that, but I'm going to chalk that up to my own stupidity).
+
+Then at 2013-04-24 11:35, the new s3 remote was changed... but it seems like only the timestamp was altered.  I suspect this was from another command line change, but I don't remember exactly what I did at that point.  Probably a reference in a different file was also modified, but I'm not looking at those.
+
+At 2013-04-24 11:37, again the new s3 remote was changed, but again it was just the timestamp.
+
+In the merge at 2013-04-24 15:15, a bunch of things happened.  This may be where stuff went wrong.  I do find it weird because it should have just been a fast forward, given what the history looks like.  I suspect that this was caused by a `git annex sync`, but I'm not 100% sure.
+
+In this commit the following happened:
+
+* The box remote was duplicated (with different davcreds and one having highRandomQuality=false)
+* The annex remote was duplicated (with highRandomQuality=false in one)
+* The nas remote was duplicated (one with encryption=shared and highRandomQuality=false and the other without)
+
+In addition, within that commit, my uuid.log file also had duplication that seems to be where part of the confusion comes from:
+
+* The 1d0ab67c-6a43-11e2-9feb-df22c6d1e308 remote shows up twice, once named \"annex\" and the other time named \"s3\".
+* The 4d86972d-9b0a-4095-bc50-f9bec8144c30 remote is only include once in there, but its name is also \"s3\".
+* Other remotes are duplicated, with different timestamps, but no overlapping uuids.
+
+Then at 2013-04-24 18:13, I think things try to fix themselves:
+
+* The older box remote (I guess based on timestamp) is removed.  Now there's only one.
+* The older 1d0ab67c-6a43-11e2-9feb-df22c6d1e308 remote (still named annex) is removed.  Now there's only one there too.
+* The single 4d86972d-9b0a-4095-bc50-f9bec8144c30 remote is updated with a new timestamp.
+* The older nas remote is also removed.
+
+No duplicates exist in this file and no cross-references exist either.
+
+The uuid.log file seems to be the place where the annex remote is renamed to s3.  I have no idea what caused that, but it was probably me.
+
+* In 2013-04-24 11:12, everything is fine in the uuid.log file.  The annex timestamp is updated, but no problems.
+* In 2013-04-24 11:13 (which doesn't show up when I look at the remote.log changes, because it didn't change that), a file's location log is updated and the 1d0ab67c-6a43-11e2-9feb-df22c6d1e308 remote is renamed from annex to s3 in uuid.log, but not in remotes.log.
+* In 2013-04-24 11:26, 4d86972d-9b0a-4095-bc50-f9bec8144c30 is added to remotes.log with the name s3 and to uuid.log with the name s3 (which is now a duplicate of the renamed 1d0ab67c-6a43-11e2-9feb-df22c6d1e308, but only in uuid.log).
+
+All of this seems horribly confusing and I don't envy your trying to unwind it.
+
+"""]]
diff --git a/doc/forum/Not_sure_how_to_get_my_s3_remote_back/comment_8_36519ee4499a19f0864e4fcd264e9933._comment b/doc/forum/Not_sure_how_to_get_my_s3_remote_back/comment_8_36519ee4499a19f0864e4fcd264e9933._comment
new file mode 100644
index 0000000000..39a1ce51d5
--- /dev/null
+++ b/doc/forum/Not_sure_how_to_get_my_s3_remote_back/comment_8_36519ee4499a19f0864e4fcd264e9933._comment
@@ -0,0 +1,20 @@
+[[!comment format=mdwn
+ username="http://joeyh.name/"
+ nickname="joey"
+ subject="comment 8"
+ date="2013-04-26T20:06:34Z"
+ content="""
+Most of this is perfectly normal. The duplication of lines are normal; when two git-annex branches are union merged, it's as if it runs `cat branch1:file branch2:file | uniq > file`. When there are conflicting lines for the same uuid, the one with the newest timestamp is used.
+
+The description of the remote in uuid.log is also not relevant to this bug.
+
+This is the key part:
+
+> The box remote was duplicated (with different davcreds and one having highRandomQuality=false)
+
+As you note, 2013-04-24 15:15 was a merge. So there must have been two branches before, which had different box remotes with different davcreds. 
+
+It would probably help if you can paste those lines as they looked after that merge (omitting most of the davcreds).
+
+Also, I'd like to see the box line from the 11:05 commit.
+"""]]
diff --git a/doc/forum/Not_sure_how_to_get_my_s3_remote_back/comment_9_85b23f375e53469fb09b24b945b3aba9._comment b/doc/forum/Not_sure_how_to_get_my_s3_remote_back/comment_9_85b23f375e53469fb09b24b945b3aba9._comment
new file mode 100644
index 0000000000..be9bd3c0df
--- /dev/null
+++ b/doc/forum/Not_sure_how_to_get_my_s3_remote_back/comment_9_85b23f375e53469fb09b24b945b3aba9._comment
@@ -0,0 +1,17 @@
+[[!comment format=mdwn
+ username="https://www.google.com/accounts/o8/id?id=AItOawkZktNHFhxC1kYA9KKdKpYJO4clq9WDsjE"
+ nickname="Jason"
+ subject="comment 9"
+ date="2013-04-26T20:21:05Z"
+ content="""
+Two box lines after 15:15 merge:
+
+0490d177-78e2-421b-a004-47d88ee7a2e3 chunksize=10mb cipher=... davcreds=... embedcreds=yes highRandomQuality=false name=box.com type=webdav url=https://www.box.com/dav/annex timestamp=1366826729.945023s
+0490d177-78e2-421b-a004-47d88ee7a2e3 chunksize=10mb cipher=... davcreds=... embedcreds=yes name=box.com type=webdav url=https://www.box.com/dav/annex timestamp=1366657062.972357s
+
+After the 11:05 commit, the box line looked like this:
+
+0490d177-78e2-421b-a004-47d88ee7a2e3 chunksize=10mb cipher=... davcreds=... embedcreds=yes highRandomQuality=false name=box.com type=webdav url=https://www.box.com/dav/annex timestamp=1366826729.945023s
+
+I am curious why you want to know about box, when s3 is the one that I'm having trouble with...
+"""]]
diff --git a/doc/forum/OSX_Finder_extension.mdwn b/doc/forum/OSX_Finder_extension.mdwn
new file mode 100644
index 0000000000..2b0e6c374f
--- /dev/null
+++ b/doc/forum/OSX_Finder_extension.mdwn
@@ -0,0 +1,5 @@
+OSX 10.10 / Yosemite introduces the [Finder Sync extension][] which allows any app / extension to display informational badges in Finder. This could be used to indicate the sync status (there are some stories which claim the extensions got added because Dropbox used an undocumented API previously). I had a quick look and it would not be too difficult to implement a simple extension which reads the metadata from the `.git/annex` directory and shows a sync / transfer status icon.
+
+Just wanted to throw the idea out there, I might even start building it.
+
+[Finder Sync Extension]: https://developer.apple.com/library/prerelease/ios/documentation/General/Conceptual/ExtensibilityPG/Finder.html
diff --git a/doc/forum/OSX_Finder_extension/comment_1_4aa37c3fbe3c260d71f2ed16fd4f20bc._comment b/doc/forum/OSX_Finder_extension/comment_1_4aa37c3fbe3c260d71f2ed16fd4f20bc._comment
new file mode 100644
index 0000000000..4b018ce116
--- /dev/null
+++ b/doc/forum/OSX_Finder_extension/comment_1_4aa37c3fbe3c260d71f2ed16fd4f20bc._comment
@@ -0,0 +1,7 @@
+[[!comment format=mdwn
+ username="SchemaCzar"
+ subject="PLEASE!!! PLEASE BUILD IT!!!"
+ date="2015-11-13T16:10:44Z"
+ content="""
+I am completely dying for this
+"""]]
diff --git a/doc/forum/OSX_Finder_extension/comment_2_e16092041890f81dc65a6ab28b2224a6._comment b/doc/forum/OSX_Finder_extension/comment_2_e16092041890f81dc65a6ab28b2224a6._comment
new file mode 100644
index 0000000000..0949047982
--- /dev/null
+++ b/doc/forum/OSX_Finder_extension/comment_2_e16092041890f81dc65a6ab28b2224a6._comment
@@ -0,0 +1,12 @@
+[[!comment format=mdwn
+ username="andrew"
+ avatar="http://cdn.libravatar.org/avatar/acc0ece1eedf07dd9631e7d7d343c435"
+ subject="released Finder integration"
+ date="2018-03-07T23:23:47Z"
+ content="""
+I just released an app, still in beta that provides Finder integration for git-annex. It relies on Core Data so it actually requires macOS 10.12 or later. If there is enough need/requests it could probably be updated to support OS-X 10.10 by switching from Core Data to sqllite directly.
+
+The app is called [git-annex-turtle](https://github.com/andrewringler/git-annex-turtle) and available on github.
+
+—Andrew
+"""]]
diff --git a/doc/forum/OSX_Mavericks_anyone__63__.mdwn b/doc/forum/OSX_Mavericks_anyone__63__.mdwn
new file mode 100644
index 0000000000..b19133680c
--- /dev/null
+++ b/doc/forum/OSX_Mavericks_anyone__63__.mdwn
@@ -0,0 +1,3 @@
+Was anyone able to run git annex on Mavericks?
+
+Didn't have time for qualified troubleshooting, sorry
diff --git a/doc/forum/OSX_Mavericks_anyone__63__/comment_1_3075b02aeb57adcbf4addf9fb4c123ba._comment b/doc/forum/OSX_Mavericks_anyone__63__/comment_1_3075b02aeb57adcbf4addf9fb4c123ba._comment
new file mode 100644
index 0000000000..1657d676d8
--- /dev/null
+++ b/doc/forum/OSX_Mavericks_anyone__63__/comment_1_3075b02aeb57adcbf4addf9fb4c123ba._comment
@@ -0,0 +1,11 @@
+[[!comment format=mdwn
+ username="http://mike.magin.org/"
+ nickname="mmagin"
+ subject="a little"
+ date="2013-10-24T17:02:18Z"
+ content="""
+Some syncing and copying, nothing other than normal remotes and one S3 remote.
+Haven't even done an \"add\" yet.
+
+Not an assistant user.  Still running 4.20130422-gb9341fd on this computer.
+"""]]
diff --git a/doc/forum/OSX_Mavericks_anyone__63__/comment_2_c2b6110fc4a3d3481ed8a4b48efb9635._comment b/doc/forum/OSX_Mavericks_anyone__63__/comment_2_c2b6110fc4a3d3481ed8a4b48efb9635._comment
new file mode 100644
index 0000000000..bfc43d682a
--- /dev/null
+++ b/doc/forum/OSX_Mavericks_anyone__63__/comment_2_c2b6110fc4a3d3481ed8a4b48efb9635._comment
@@ -0,0 +1,17 @@
+[[!comment format=mdwn
+ username="Alex"
+ ip="217.8.62.35"
+ subject="not at all"
+ date="2013-10-24T19:50:11Z"
+ content="""
+I'd love to see an updated OS X build for Mavericks.
+
+Currently I'm getting this error:
+
+    dyld: Symbol not found: _objc_debug_taggedpointer_mask
+      Referenced from: /System/Library/Frameworks/CoreFoundation.framework/Versions/A/CoreFoundation
+      Expected in: /Applications/git-annex.app/Contents/MacOS/bundle/I
+     in /System/Library/Frameworks/CoreFoundation.framework/Versions/A/CoreFoundation
+    Trace/BPT trap: 5
+
+"""]]
diff --git a/doc/forum/OSX_Mavericks_anyone__63__/comment_3_7df9ba63cb1f385681242b4b58d6a87c._comment b/doc/forum/OSX_Mavericks_anyone__63__/comment_3_7df9ba63cb1f385681242b4b58d6a87c._comment
new file mode 100644
index 0000000000..691fd9064c
--- /dev/null
+++ b/doc/forum/OSX_Mavericks_anyone__63__/comment_3_7df9ba63cb1f385681242b4b58d6a87c._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="http://joeyh.name/"
+ ip="67.223.19.96"
+ subject="comment 3"
+ date="2013-10-24T19:53:08Z"
+ content="""
+Mavericks is being discussed [[here|bugs/git annex doesn't work in Max OS X 10.9]]
+"""]]
diff --git a/doc/forum/OSX_Mavericks_anyone__63__/comment_4_740fee31c4ca9d84428f97f63ffc075a._comment b/doc/forum/OSX_Mavericks_anyone__63__/comment_4_740fee31c4ca9d84428f97f63ffc075a._comment
new file mode 100644
index 0000000000..3ad5c23446
--- /dev/null
+++ b/doc/forum/OSX_Mavericks_anyone__63__/comment_4_740fee31c4ca9d84428f97f63ffc075a._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="Alex"
+ ip="217.8.62.35"
+ subject="comment 4"
+ date="2013-10-24T19:55:56Z"
+ content="""
+Thanks, Joey!
+"""]]
diff --git a/doc/forum/OSX__39__s_default_sshd_behaviour_has_limited_paths_set.mdwn b/doc/forum/OSX__39__s_default_sshd_behaviour_has_limited_paths_set.mdwn
new file mode 100644
index 0000000000..5e417b14c9
--- /dev/null
+++ b/doc/forum/OSX__39__s_default_sshd_behaviour_has_limited_paths_set.mdwn
@@ -0,0 +1,12 @@
+This is a tip for users who wish to use remotes which are based on OSX systems and have used macports to install some of the required utilities for git-annex to work.
+
+The default behaviour of OSX's sshd is to have a "highly restricted" restricted environment. The defaults that it allows is
+
+    jtang@x00:~ $ ssh x00 echo \$PATH
+    /usr/bin:/bin:/usr/sbin:/sbin
+
+One solution is to enable *PermitUserEnvironment yes* in `/etc/sshd_config` and then in your own `~/.ssh/environment` file you could add something like (the below is an example)
+
+    PATH=/Users/jtang/bin:/opt/local/bin:/usr/bin:/bin:/usr/sbin:/sbin:/usr/local/bin:/opt/X11/bin:/usr/X11/bin:/Users/jtang/.cabal/bin:/opt/local/libexec/gnubin
+
+If the above is not done, cloning from the OSX host will fail if git is not installed in /usr/bin (which it probably won't be).
diff --git a/doc/forum/OSX__39__s_default_sshd_behaviour_has_limited_paths_set/comment_1_a136ff877389f0930c066ba118edd9fd._comment b/doc/forum/OSX__39__s_default_sshd_behaviour_has_limited_paths_set/comment_1_a136ff877389f0930c066ba118edd9fd._comment
new file mode 100644
index 0000000000..0cbf79c8d5
--- /dev/null
+++ b/doc/forum/OSX__39__s_default_sshd_behaviour_has_limited_paths_set/comment_1_a136ff877389f0930c066ba118edd9fd._comment
@@ -0,0 +1,10 @@
+[[!comment format=mdwn
+ username="http://joeyh.name/"
+ ip="209.250.56.114"
+ subject="comment 1"
+ date="2014-04-24T18:08:00Z"
+ content="""
+If you use git-annex from the OSX .dmg, it will set up a ~/.ssh/git-annex-shell, which is the only command that is needed when git-annex is using an OSX server as a remote. Since version 5.20140421, the webapp will also use ~/.ssh/git-annex-wrapper, which the .dmg also sets up, to run some commands like git. 
+
+The upshot is that this should not affect git-annex when installed from the .dmg on the OSX server. If you build git-annex from source yourself, you do need to make sure that it and git end up in PATH.
+"""]]
diff --git a/doc/forum/OSX__39__s_default_sshd_behaviour_has_limited_paths_set/comment_2_5fcc52a62770fb932a4a2101f5badbc0._comment b/doc/forum/OSX__39__s_default_sshd_behaviour_has_limited_paths_set/comment_2_5fcc52a62770fb932a4a2101f5badbc0._comment
new file mode 100644
index 0000000000..bb8f3ee8d7
--- /dev/null
+++ b/doc/forum/OSX__39__s_default_sshd_behaviour_has_limited_paths_set/comment_2_5fcc52a62770fb932a4a2101f5badbc0._comment
@@ -0,0 +1,17 @@
+[[!comment format=mdwn
+ username="ewen"
+ subject="Linux to OS X ~/.ssh/git-annex-shell"
+ date="2014-11-15T00:02:02Z"
+ content="""
+I installed git-annex on OS/X 10.9 (Mavericks) from the DMG file (the 10.10 DMG file from 2014-11-11, since there was no longer one linked for Mavericks -- it does seem to work locally on 10.9).  Separately I installed git-annex 5.20141024~bpo70+1 on Debian Wheezy, from Backports.  I created a git-annex on the OS X system, and was then trying to set up another git-annex of that on the Linux system, initiated from the Linux system.  After some ssh tunnel magic (due to a firewall stopping git-annex's dream that everything can ssh into everything else without problems), I was able to \"git clone\" and \"git annex init\" on the Linux system.  But ran into problems trying to initiate a \"git annex sync\" from the Linux end.
+
+In particular I got this \"bash: git-annex-shell: command not found\" report, despite the fact that the OS X side *does* have \"~/.ssh/git-annex-shell\" (apparently set up when I ran git-annex on the OS X side first), and on the OS/X side running \"~/.ssh/git-annex-shell\" does work (well it says \"bad parameters\" and gives a list of commands).
+
+It's not clear to me how git-annex is expecting that ~/.ssh/git-annex-shell will end up being found by the ssh connection initiated from the Linux side.  AFAICT from \"git annex sync --debug REMOTE\" (and strace), all that the client end is running is a bare \"git-annex-shell\", so (a) it'd only be searching the path (as per original post), (b) it shouldn't be affected by the different expansion of ~ on Linux and OS/X (/home/ewen and /Users/ewen respectively -- the OS defaults), and (c) AFAICT nothing is adding ~/.ssh to the PATH.  Maybe running something more from the DMG than just the command line \"git-annex\" sets something else up, which I'm expected to set up by hand, that makes \"~/.ssh/git-annex-shell\" be found?
+
+I ended up working around it by noticing that the ssh shell was reading ~/.bashrc (which adds some of my local directories to the PATH), so I was able to add yet another helper script in that directory (~/.bin/, in my case).  Beware it does have to be a direct symlink to the directory with the rest of the git-annex files though, not via another symlink, because the helper scripts only do one level of readlink.  After that it seems to work.
+
+(Possibly these \"make this command available on the PATH at this time\" dependencies could stand to be better documented on the OS X install page?)
+
+Ewen
+"""]]
diff --git a/doc/forum/OSX__39__s_default_sshd_behaviour_has_limited_paths_set/comment_3_521f34c340ad54cef6458a8b6cfff8a9._comment b/doc/forum/OSX__39__s_default_sshd_behaviour_has_limited_paths_set/comment_3_521f34c340ad54cef6458a8b6cfff8a9._comment
new file mode 100644
index 0000000000..088aa75511
--- /dev/null
+++ b/doc/forum/OSX__39__s_default_sshd_behaviour_has_limited_paths_set/comment_3_521f34c340ad54cef6458a8b6cfff8a9._comment
@@ -0,0 +1,13 @@
+[[!comment format=mdwn
+ username="joey"
+ subject="""comment 3"""
+ date="2014-12-02T17:42:36Z"
+ content="""
+When the git-annex webapp is used to add a ssh repository, it
+installs a ssh key in `~/.ssh/authorized_keys`, which makes
+ssh run `~/.ssh/git-annex-shell` if necessary.
+
+This is not done for you if you set up a ssh remote using git at the
+commend line. In that case, arranging for git-annex-shell to be in PATH
+on the remote host somehow is left up to you.
+"""]]
diff --git a/doc/forum/OSX__39__s_haskell-platform_statically_links_things.mdwn b/doc/forum/OSX__39__s_haskell-platform_statically_links_things.mdwn
new file mode 100644
index 0000000000..537c85d019
--- /dev/null
+++ b/doc/forum/OSX__39__s_haskell-platform_statically_links_things.mdwn
@@ -0,0 +1,17 @@
+This isn't really a bug of git-annex, but a problem with haskell-platform/ghc6.12.x so this post might need to be moved to a better place (maybe tips).
+
+OSX's haskell-platform doesn't have the dynamic libraries available, as far as I know it just isn't supported therefore git-annex will always be statically built on OSX, so wrappers like  or [[!google dsocks]] for preloading connect() calls won't work. 
+
+
    +jtang@x00:~/annex $ tsocks git annex get .
+dyld: could not load inserted library: /opt/local/lib/libtsocks.dylib
+
+error: git-annex died of signal 5
+
    
+
+The side effect of this means that users who are behind restrictive firewalls that allow only ssh via a socks proxy, they will need to configure ssh to use something like .
+
+
    +host remotemyhost
+        ProxyCommand connect -S proxy.mydomain:1080 -R local %h %p
+
    
diff --git a/doc/forum/OSX__58___Upgrader__58___Failed_to_check_if_upgrade_is_available.mdwn b/doc/forum/OSX__58___Upgrader__58___Failed_to_check_if_upgrade_is_available.mdwn
new file mode 100644
index 0000000000..1f02b568c3
--- /dev/null
+++ b/doc/forum/OSX__58___Upgrader__58___Failed_to_check_if_upgrade_is_available.mdwn
@@ -0,0 +1,36 @@
+I noticed this in my log file...
+
+
    +[2015-05-21 18:10:02 CDT] main: starting assistant version 5.20150508-gf71c23f
+(started...) gpg: keyring `/Applications/git-annex.app/Contents/MacOS/trustedkeys.gpg' created
+...
+[2015-05-21 18:10:02 CDT] Upgrader: Checking if an upgrade is available.
+...
+[2015-05-21 18:10:02 CDT] call: curl ["-s","-f","-L","-C","-","-#","-o","/var/folders/zk/tk15c40n6h1bjl94ntk5md8h0000gn/T/git-annex.tmp.0/info.sig","http://downloads.kitenet.net/git-annex/OSX/current/10.10_Yosemite/git-annex.dmg.info.sig","--user-agent","git-annex/5.20150419-g900e1b6"]
+[2015-05-21 18:10:03 CDT] call: gpg ["--no-default-keyring","--no-auto-check-trustdb","--no-options","--homedir","/var/folders/zk/tk15c40n6h1bjl94ntk5md8h0000gn/T/git-annex-gpg.tmp.0","--keyring","/Applications/git-annex.app/Contents/MacOS/trustedkeys.gpg","--verify","/var/folders/zk/tk15c40n6h1bjl94ntk5md8h0000gn/T/git-annex.tmp.0/info.sig"]
+gpg: Signature made Fri May  8 15:10:28 2015 CDT using DSA key ID 89C809CB
+gpg: Can't check signature: public key not found
+[2015-05-21 18:10:03 CDT] Upgrader: Failed to check if upgrade is available.
+
    
+
+So I copied trustedkeys.gpg from the bundle folder to the MacOS folder, replacing the newly created trustedkeys.gpg and now I get this... which is better... ;-)
+
+
    +[2015-05-21 18:17:24 CDT] main: starting assistant version 5.20150508-gf71c23f
+...
+[2015-05-21 18:17:25 CDT] Upgrader: Checking if an upgrade is available.
+...
+[2015-05-21 18:17:25 CDT] call: curl ["-s","-f","-L","-C","-","-#","-o","/var/folders/zk/tk15c40n6h1bjl94ntk5md8h0000gn/T/git-annex.tmp.0/info","http://downloads.kitenet.net/git-annex/OSX/current/10.10_Yosemite/git-annex.dmg.info","--user-agent","git-annex/5.20150508-gf71c23f"]
+...
+[2015-05-21 18:17:25 CDT] call: curl ["-s","-f","-L","-C","-","-#","-o","/var/folders/zk/tk15c40n6h1bjl94ntk5md8h0000gn/T/git-annex.tmp.0/info.sig","http://downloads.kitenet.net/git-annex/OSX/current/10.10_Yosemite/git-annex.dmg.info.sig","--user-agent","git-annex/5.20150508-gf71c23f"]
+[2015-05-21 18:17:25 CDT] call: gpg ["--no-default-keyring","--no-auto-check-trustdb","--no-options","--homedir","/var/folders/zk/tk15c40n6h1bjl94ntk5md8h0000gn/T/git-annex-gpg.tmp.0","--keyring","/Applications/git-annex.app/Contents/MacOS/trustedkeys.gpg","--verify","/var/folders/zk/tk15c40n6h1bjl94ntk5md8h0000gn/T/git-annex.tmp.0/info.sig"]
+gpg: Signature made Fri May  8 15:10:28 2015 CDT using DSA key ID 89C809CB
+gpg: /var/folders/zk/tk15c40n6h1bjl94ntk5md8h0000gn/T/git-annex-gpg.tmp.0/trustdb.gpg: trustdb created
+gpg: Good signature from "git-annex distribution signing key (for Joey Hess) "
+gpg: WARNING: This key is not certified with a trusted signature!
+gpg:          There is no indication that the signature belongs to the owner.
+Primary key fingerprint: 4005 5C6A FD2D 526B 2961  E78F 5EE1 DBA7 89C8 09CB
+[2015-05-21 18:17:25 CDT] Upgrader: No new version found.
+
    
+
+FYI
diff --git a/doc/forum/OSX__58___Upgrader__58___Failed_to_check_if_upgrade_is_available/comment_1_ac7c1f39e723cbcd49d6497649e7e44f._comment b/doc/forum/OSX__58___Upgrader__58___Failed_to_check_if_upgrade_is_available/comment_1_ac7c1f39e723cbcd49d6497649e7e44f._comment
new file mode 100644
index 0000000000..39eae02a8b
--- /dev/null
+++ b/doc/forum/OSX__58___Upgrader__58___Failed_to_check_if_upgrade_is_available/comment_1_ac7c1f39e723cbcd49d6497649e7e44f._comment
@@ -0,0 +1,13 @@
+[[!comment format=mdwn
+ username="joey"
+ subject="""comment 1"""
+ date="2015-05-22T17:35:11Z"
+ content="""
+Aha, this is a path propel to the trustedkeys.gpg file, it's in bundle/
+in the OSX app, and git-annex tells gpg to look in the parent directory
+instead.
+
+Fixed in git. Unfortunately I can't fix old versions of git-annex so
+OSX users will need to manually upgrade to version 5.20150522 to get this
+fix.
+"""]]
diff --git a/doc/forum/Odd_Hybrid_Symlinks_To_Content.mdwn b/doc/forum/Odd_Hybrid_Symlinks_To_Content.mdwn
new file mode 100644
index 0000000000..5e3b4d8b4a
--- /dev/null
+++ b/doc/forum/Odd_Hybrid_Symlinks_To_Content.mdwn
@@ -0,0 +1,27 @@
+I've somehow managed to get my indirect repository to symlink to literal content instead of object files. 
+
+By this I mean literally the symlink is pointing at the contents of the file as the filename.
+
+So if I have a blah.txt file with this content:
+
+* First line
+* second line
+
+And I ls -al to view the symlink pointer, it shows up as this:
+
+* blah.txt -> First line?second line
+
+It literally has the contents of the file as the destination filename.
+
+I've tried a couple things I could think of to re-symlink the files, but they don't seem to do anything as they think everything is fine:
+
+* git annex indirect //returns nothing
+* git annex lock blah.txt //returns nothing
+* git annex fix blah.txt //returns nothing
+* git annex fsck //returns nothing
+
+I'm actually able to find several of these files hanging around by searching for all symlinks that don't point to something in the .git directory.
+
+Is there a way for me to replace the symlinks with correct symlinks to the objects in .git/annex? Can it even figure out which ones it was supposed to point to if the symlinks are messed up (are filenames -> content hashes stored anywhere else)?
+
+Else I might have to go do some manual rebasing and history editing to try to undo the bad commits manually. I've synced this repo to another direct repo so I'll need to figure out how to manually fix that repo too (using proxy).  From what I can tell the annex/direct/master seems to be same as master and synced/master branches? Is there an [[internals]] page for direct branches besides [[direct_mode]] so I know what should be fixed where?
diff --git a/doc/forum/One_repo_with_multiple_remotes.mdwn b/doc/forum/One_repo_with_multiple_remotes.mdwn
new file mode 100644
index 0000000000..a9d306cb16
--- /dev/null
+++ b/doc/forum/One_repo_with_multiple_remotes.mdwn
@@ -0,0 +1,20 @@
+I have three machines with the same repository in all of the machines.
+  
+  - cat-laptop:~/personal
+  - dog:~/personal
+  - turtle:~/personal
+ 
+All of them are connected to each other in `client mode`, and were
+paired via the `Assistant` as `Local Computers`.
+
+`dog:~/personal` is accessible through the LAN and externally to the
+internet.
+
+Will there be a problem if I add `dog:~/personal` as a `Remote Server`
+through the assistant  with a different hostname and port to
+`cat-laptop:~/personal`? And does git-annex know what remote to use if
+`cat-laptop` is connected to the same LAN as `dog`?
+
+My  worries are file collisions, missing commits, or any lose of
+updates because of one repository treated as multiple remotes in other
+client repositories.
diff --git a/doc/forum/One_repo_with_multiple_remotes/comment_1_4b9b3137396a3e830fe96726401270b8._comment b/doc/forum/One_repo_with_multiple_remotes/comment_1_4b9b3137396a3e830fe96726401270b8._comment
new file mode 100644
index 0000000000..e4e7fb321f
--- /dev/null
+++ b/doc/forum/One_repo_with_multiple_remotes/comment_1_4b9b3137396a3e830fe96726401270b8._comment
@@ -0,0 +1,9 @@
+[[!comment format=mdwn
+ username="lykos@d125a37d89b1cfac20829f12911656c40cb70018"
+ nickname="lykos"
+ avatar="http://cdn.libravatar.org/avatar/085df7b04d3408ba23c19f9c49be9ea2"
+ subject="comment 1"
+ date="2018-03-21T12:17:45Z"
+ content="""
+That's not a problem at all. The repositories are identified by their UUID, so there's no danger of annex counting too many copies. Local remotes have a lower cost, so git-annex should prefer them, but you're probably going to encounter connection errors when you're not inside the LAN, so have a look at the annex-ignore-command option (see [manpage](https://git-annex.branchable.com/git-annex)).
+"""]]
diff --git a/doc/forum/OpenOffice___47___Libre_Office.mdwn b/doc/forum/OpenOffice___47___Libre_Office.mdwn
new file mode 100644
index 0000000000..f28ad62598
--- /dev/null
+++ b/doc/forum/OpenOffice___47___Libre_Office.mdwn
@@ -0,0 +1,5 @@
+I'm trying to use git-annex for keeping my company invoices data. It seems a good idea because it's encrypted and whatnot, I'd never upload them to Dropbox. However, Libre Office refuses to work with files copied to annex-tracked folder. It's likely a flaw in LO/OOo, but maybe someone here knows a way to make the two work together.
+
+When opening a spreadsheet, I'm getting a "Document in use - document locked by: Unknown User - Open read-only or open a copy" dialog. I tried opening a copy and saving under a different filename , but I can only save once - any subsequent saves to the same filename result in an error.
+
+A graphic version of my story: http://f.gdr.name/annex-ooo-1.png http://f.gdr.name/annex-ooo-2.png
diff --git a/doc/forum/OpenOffice___47___Libre_Office/comment_1_98ed542fedd820d47bf8deb7d3232725._comment b/doc/forum/OpenOffice___47___Libre_Office/comment_1_98ed542fedd820d47bf8deb7d3232725._comment
new file mode 100644
index 0000000000..68dbee1661
--- /dev/null
+++ b/doc/forum/OpenOffice___47___Libre_Office/comment_1_98ed542fedd820d47bf8deb7d3232725._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="http://joeyh.name/"
+ ip="2001:4978:f:21a::2"
+ subject="comment 1"
+ date="2013-01-02T19:10:28Z"
+ content="""
+Files in git-annex are locked normally. You can use `git annex unlock` to unlock a file for editing. Or you can use the new [[direct_mode]].
+"""]]
diff --git a/doc/forum/OpenOffice___47___Libre_Office/comment_2_f313fdaa23863c2ae99cfbfe9ec2e1e0._comment b/doc/forum/OpenOffice___47___Libre_Office/comment_2_f313fdaa23863c2ae99cfbfe9ec2e1e0._comment
new file mode 100644
index 0000000000..394f6fb012
--- /dev/null
+++ b/doc/forum/OpenOffice___47___Libre_Office/comment_2_f313fdaa23863c2ae99cfbfe9ec2e1e0._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="https://launchpad.net/~gdr-go2"
+ nickname="gdr-go2"
+ subject="Thanks"
+ date="2013-01-02T19:58:44Z"
+ content="""
+Direct mode has solved the problem. Great new addition.
+"""]]
diff --git a/doc/forum/Out_of_memory_error_in_fsck__44___whereis__44___find_and_status_cmds.mdwn b/doc/forum/Out_of_memory_error_in_fsck__44___whereis__44___find_and_status_cmds.mdwn
new file mode 100644
index 0000000000..8c54f49da5
--- /dev/null
+++ b/doc/forum/Out_of_memory_error_in_fsck__44___whereis__44___find_and_status_cmds.mdwn
@@ -0,0 +1,5 @@
+Please file bug reports in [[bugs]], not the forum, so they can be closed
+when fixed.
+
+Moved to [[bug report|bugs/Out_of_memory_error_in_fsck_whereis_find_and_status_cmds]].
+--[[Joey]]
diff --git a/doc/forum/Out_of_memory_on_NFS4_with_files_bigger_than_available_memory.mdwn b/doc/forum/Out_of_memory_on_NFS4_with_files_bigger_than_available_memory.mdwn
new file mode 100644
index 0000000000..94953b60e7
--- /dev/null
+++ b/doc/forum/Out_of_memory_on_NFS4_with_files_bigger_than_available_memory.mdwn
@@ -0,0 +1,34 @@
+I do the following:
+
+    # create repo
+    git init repo
+    git -C ./repo annex init --version=6
+    git -C ./repo config annex.thin true
+    # add big file
+    cd repo
+    cp /tmp/bigfile . # 12GB file on system with 8GB RAM
+    git annex add bigfile
+    # commit
+    git commit -m 'msg'
+
+On the commit, I get "fatal: Out of memory, realloc failed". I found out that "git add" leads to the same error with big files, so I use "git annex add".
+
+I do this on an NFS4 disk. The same on the local EXT4 disk works.
+
+I have noticed that annex creates a link for the file on EXT4 but not on NFS4. This seems suspicious.
+
+Version information:
+
+    git version
+    git version 2.16.2
+
+    git annex version
+    build flags: Assistant Webapp Pairing S3(multipartupload)(storageclasses) WebDAV Inotify DBus DesktopNotify ConcurrentOutput TorrentParser MagicMime Feeds Testsuite
+    dependency versions: aws-0.16 bloomfilter-2.0.1.0 cryptonite-0.23 DAV-1.3.1 feed-0.3.12.0 ghc-8.0.2 http-client-0.5.7.1 persistent-sqlite-2.6.3 torrent-10000.1.1 uuid-1.3.13 yesod-1.4.5
+    key/value backends: SHA256E SHA256 SHA512E SHA512 SHA224E SHA224 SHA384E SHA384 SHA3_256E SHA3_256 SHA3_512E SHA3_512 SHA3_224E SHA3_224 SHA3_384E SHA3_384 SKEIN256E SKEIN256 SKEIN512E SKEIN512 BLAKE2B256E BLAKE2B256 BLAKE2B512E BLAKE2B512 BLAKE2B160E BLAKE2B160 BLAKE2B224E BLAKE2B224 BLAKE2B384E BLAKE2B384 BLAKE2S256E BLAKE2S256 BLAKE2S160E BLAKE2S160 BLAKE2S224E BLAKE2S224 BLAKE2SP256E BLAKE2SP256 BLAKE2SP224E BLAKE2SP224 SHA1E SHA1 MD5E MD5 WORM URL
+    remote types: git gcrypt p2p S3 bup directory rsync web bittorrent webdav adb tahoe glacier ddar hook external
+    local repository version: 6
+    supported repository versions: 3 5 6
+    upgrade supported from repository versions: 0 1 2 3 4 5
+    operating system: linux x86_64
+
diff --git a/doc/forum/Out_of_memory_on_NFS4_with_files_bigger_than_available_memory/comment_1_2e89bcc3fbd069e5d9eb12c1c19993ad._comment b/doc/forum/Out_of_memory_on_NFS4_with_files_bigger_than_available_memory/comment_1_2e89bcc3fbd069e5d9eb12c1c19993ad._comment
new file mode 100644
index 0000000000..95c0388366
--- /dev/null
+++ b/doc/forum/Out_of_memory_on_NFS4_with_files_bigger_than_available_memory/comment_1_2e89bcc3fbd069e5d9eb12c1c19993ad._comment
@@ -0,0 +1,14 @@
+[[!comment format=mdwn
+ username="yves.noirjean@3f9b06d19a920fbf5c82340c362e5971b00d4af2"
+ nickname="yves.noirjean"
+ avatar="http://cdn.libravatar.org/avatar/2f1ad9d443c037337d91f29781560344"
+ subject="comment 1"
+ date="2018-06-08T15:17:16Z"
+ content="""
+I wanted to try it with a repository using v5 indirect mode. This tells me:
+
+      Filesystem allows writing to files whose write bit is not set.
+    git-annex: This repository seems to be on a crippled filesystem, you must use direct mode.
+
+Is that the case with NFS4 in general or might the disk not be mounted properly?
+"""]]
diff --git a/doc/forum/Out_of_memory_on_NFS4_with_files_bigger_than_available_memory/comment_2_cf8065e697a0a83cf3461672cb6421e0._comment b/doc/forum/Out_of_memory_on_NFS4_with_files_bigger_than_available_memory/comment_2_cf8065e697a0a83cf3461672cb6421e0._comment
new file mode 100644
index 0000000000..51edcafd5a
--- /dev/null
+++ b/doc/forum/Out_of_memory_on_NFS4_with_files_bigger_than_available_memory/comment_2_cf8065e697a0a83cf3461672cb6421e0._comment
@@ -0,0 +1,9 @@
+[[!comment format=mdwn
+ username="yves.noirjean@3f9b06d19a920fbf5c82340c362e5971b00d4af2"
+ nickname="yves.noirjean"
+ avatar="http://cdn.libravatar.org/avatar/2f1ad9d443c037337d91f29781560344"
+ subject="comment 2"
+ date="2018-06-18T11:25:18Z"
+ content="""
+The issue has been resolved. The unix file permissions were not applied, but the NFS4 permissions. So git annex did not trust that files without write flags could not be changed. Therefore, it didn't move the files and create symlinks. This somehow lead to git allocating a lot of memory during commit.
+"""]]
diff --git a/doc/forum/Out_of_memory_on_NFS4_with_files_bigger_than_available_memory/comment_3_e6bd4eb070a094116e13064dca6d5bb5._comment b/doc/forum/Out_of_memory_on_NFS4_with_files_bigger_than_available_memory/comment_3_e6bd4eb070a094116e13064dca6d5bb5._comment
new file mode 100644
index 0000000000..94ae963a3f
--- /dev/null
+++ b/doc/forum/Out_of_memory_on_NFS4_with_files_bigger_than_available_memory/comment_3_e6bd4eb070a094116e13064dca6d5bb5._comment
@@ -0,0 +1,11 @@
+[[!comment format=mdwn
+ username="yves.noirjean@3f9b06d19a920fbf5c82340c362e5971b00d4af2"
+ nickname="yves.noirjean"
+ avatar="http://cdn.libravatar.org/avatar/2f1ad9d443c037337d91f29781560344"
+ subject="comment 3"
+ date="2018-06-21T16:15:05Z"
+ content="""
+Actually, I am still interested in a response.
+
+I would prefer to use annex.addunlocked = true, which would use hardlinks.
+"""]]
diff --git a/doc/forum/Out_of_memory_on_NFS4_with_files_bigger_than_available_memory/comment_4_42adc29a2c3216910a3bcda04d6f6a6b._comment b/doc/forum/Out_of_memory_on_NFS4_with_files_bigger_than_available_memory/comment_4_42adc29a2c3216910a3bcda04d6f6a6b._comment
new file mode 100644
index 0000000000..7efefbffb1
--- /dev/null
+++ b/doc/forum/Out_of_memory_on_NFS4_with_files_bigger_than_available_memory/comment_4_42adc29a2c3216910a3bcda04d6f6a6b._comment
@@ -0,0 +1,17 @@
+[[!comment format=mdwn
+ username="andrew"
+ avatar="http://cdn.libravatar.org/avatar/acc0ece1eedf07dd9631e7d7d343c435"
+ subject="comment 4"
+ date="2018-07-09T13:44:15Z"
+ content="""
+Yes, see Joey's notes at [smudge](http://git-annex.branchable.com/todo/smudge/) *“When git add is run with a large file, it allocates memory for the whole file content, even though it's only going to stream it to the clean filter. My proposed smudge/clean interface patch also fixed this problem, since it made git not read the file at all.”* I believe that `git annex add` will avoid this issue as you have seen.
+
+See also [unlocked files](http://git-annex.branchable.com/tips/unlocked_files/):
+
+   - `git config annex.addunlocked true` tells git annex to add all files as unlocked
+   - `git config annex.thin true` tells git annex to only keep one copy of unlocked files by using hardlinks
+
+I believe both of these settings only have an effect on a V6 repo. I assume for your desires you will want to use both settings. Did you try setting both of these on your V6 repo then doing a `git annex add`?
+
+The unlocked page and smudge page also talk about various limitations of the thin setting, for example from the smudge page: *“…with annex.thin no attempt is made to protect the object from being modified. If a user wants to protect object contents from modification, they should use git annex add, not git add, or they can git annex lock after adding, or not enable annex.thin.”*
+"""]]
diff --git a/doc/forum/Overwriting_data_without_getting_it.mdwn b/doc/forum/Overwriting_data_without_getting_it.mdwn
new file mode 100644
index 0000000000..ccd23b74e7
--- /dev/null
+++ b/doc/forum/Overwriting_data_without_getting_it.mdwn
@@ -0,0 +1,3 @@
+My collaborators and I use git annex to track various large data files (among some smaller metadata files managed by ordinary git).  Some of these data files need to change completely -- the old ones were just wrong.  So I do a git checkout, but don't `git annex get` because it would just be a waste of time and bandwidth.  This means that my "data files" are just broken symlinks.  Now, I find that by making the necessary directories under `.git/annex/objects/`, I can write to these files in the usual way.  The symlinks are preserved, and the files they link to now exist and are full of my corrected data.  This seems like it's a problem because the hash has presumably changed.  (I'm still a little fuzzy on how exactly git-annex works.)  Also, git/git-annex doesn't seem to realize that anything has changed.  Is this recoverable?
+
+Would it have been better to just `git rm` (or something) the original version of the file, commit that, and then add the new data?  And if so, how should I go about this now that I've created these many very large files?  If not, what would be the preferred way to do this?
diff --git a/doc/forum/Overwriting_data_without_getting_it/comment_1_f1c0199ee9bffcc84287370b89361294._comment b/doc/forum/Overwriting_data_without_getting_it/comment_1_f1c0199ee9bffcc84287370b89361294._comment
new file mode 100644
index 0000000000..a6092b1f21
--- /dev/null
+++ b/doc/forum/Overwriting_data_without_getting_it/comment_1_f1c0199ee9bffcc84287370b89361294._comment
@@ -0,0 +1,26 @@
+[[!comment format=mdwn
+ username="http://joeyh.name/"
+ nickname="joey"
+ subject="comment 1"
+ date="2013-06-12T16:57:52Z"
+ content="""
+I think you're making this more complicated than it needs to be. You don't need to mess around with .git/annex/objects at all. You can replace git-annex symlinks with new files and git annex add the new content.
+
+For example:
+
+[[!format sh \"\"\"
+joey@gnu:~/tmp/repo>git annex drop --force foo
+drop foo ok
+(Recording state in git...)
+joey@gnu:~/tmp/repo>ls
+foo@
+joey@gnu:~/tmp/repo>rm foo
+joey@gnu:~/tmp/repo>echo \"new good contents\" > foo
+joey@gnu:~/tmp/repo>git annex add foo
+add foo (checksum...) ok
+(Recording state in git...)
+joey@gnu:~/tmp/repo>git commit -m add
+[master ec3ed14] add
+ 1 file changed, 1 insertion(+), 1 deletion(-)
+\"\"\"]]
+"""]]
diff --git a/doc/forum/Overwriting_data_without_getting_it/comment_2_6a1d08dbca206129ef6cf8aa97daeee1._comment b/doc/forum/Overwriting_data_without_getting_it/comment_2_6a1d08dbca206129ef6cf8aa97daeee1._comment
new file mode 100644
index 0000000000..698d00c35b
--- /dev/null
+++ b/doc/forum/Overwriting_data_without_getting_it/comment_2_6a1d08dbca206129ef6cf8aa97daeee1._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="https://www.google.com/accounts/o8/id?id=AItOawkxmke7K8gEXleVRuQvCK5LHPLIzQA6s0E"
+ nickname="Michael"
+ subject="comment 2"
+ date="2013-06-12T17:14:11Z"
+ content="""
+Yes, it seems I was making this more complicated than it needed to be.  Just a plain rm seems to work.  But just to be clear, I never had the data so I don't need to drop it, right?  (By which I mean, is there some other function of drop that I don't understand?)
+"""]]
diff --git a/doc/forum/Overwriting_data_without_getting_it/comment_3_52958e76e506fdbb6b533681ab619b3b._comment b/doc/forum/Overwriting_data_without_getting_it/comment_3_52958e76e506fdbb6b533681ab619b3b._comment
new file mode 100644
index 0000000000..19292b8707
--- /dev/null
+++ b/doc/forum/Overwriting_data_without_getting_it/comment_3_52958e76e506fdbb6b533681ab619b3b._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="http://joeyh.name/"
+ nickname="joey"
+ subject="comment 3"
+ date="2013-06-12T17:15:26Z"
+ content="""
+Right. I just put the drop in there to get my repository to the state you described.
+"""]]
diff --git a/doc/forum/Per_directory_numcopies.mdwn b/doc/forum/Per_directory_numcopies.mdwn
new file mode 100644
index 0000000000..a151c937a8
--- /dev/null
+++ b/doc/forum/Per_directory_numcopies.mdwn
@@ -0,0 +1,5 @@
+I have some photo backups that I would like to have 2 copies but the rest can be one. 
+
+Is there any way to create numcopies rules per directory, rather than by filetype?
+
+I really enjoy this program Joey, thanks!
diff --git a/doc/forum/Per_directory_numcopies/comment_1_33497e3a46926595920eb3dcdb58447c._comment b/doc/forum/Per_directory_numcopies/comment_1_33497e3a46926595920eb3dcdb58447c._comment
new file mode 100644
index 0000000000..cb6a9579d8
--- /dev/null
+++ b/doc/forum/Per_directory_numcopies/comment_1_33497e3a46926595920eb3dcdb58447c._comment
@@ -0,0 +1,9 @@
+[[!comment format=mdwn
+ username="markusk"
+ subject="comment 1"
+ date="2015-03-03T21:04:55Z"
+ content="""
+Look for \".gitattributes\" files in http://git-annex.branchable.com/walkthrough/backups/ and the \"annex.numcopies\" option to use inside these files. You can place the files at any location. 
+
+
+"""]]
diff --git a/doc/forum/Per_directory_numcopies/comment_2_00572e3c15e5a9d253fb4ea57b45ff0d._comment b/doc/forum/Per_directory_numcopies/comment_2_00572e3c15e5a9d253fb4ea57b45ff0d._comment
new file mode 100644
index 0000000000..d75f37ec05
--- /dev/null
+++ b/doc/forum/Per_directory_numcopies/comment_2_00572e3c15e5a9d253fb4ea57b45ff0d._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="https://www.google.com/accounts/o8/id?id=AItOawksOCeakibYmGDt3wLLo4nkY0FkB72I2Uo"
+ nickname="Source"
+ subject="comment 2"
+ date="2015-03-05T01:27:52Z"
+ content="""
+That's exactly what I was looking for, Thanks!
+"""]]
diff --git a/doc/forum/Performance_implications_of_triply_nested_objects_directory.mdwn b/doc/forum/Performance_implications_of_triply_nested_objects_directory.mdwn
new file mode 100644
index 0000000000..7a6e548b1f
--- /dev/null
+++ b/doc/forum/Performance_implications_of_triply_nested_objects_directory.mdwn
@@ -0,0 +1,23 @@
+Posting this in case anyone might find it interesting.
+
+I had noticed impractical and abysmally slow performance when tracking a huge number of files (150k) in a git-annex. In direct mode, the repo was outright unusable, but even in indirect mode, many operations where painfully slow; even operations beyond the well-known offender, «git annex findunused», e.g., the seemingly harmless «git annex info».
+
+I also noticed that the performance was hugely improved on my (otherwise comparable) machine running btrfs, and I wondered how this might be. From previous benchmarks, I had gained the impression that ext4 and btrfs are on par, performance-wise, and you choose btrfs for the features rather than performance. Now, after trying to update my back-ups via rsync, I have had an idea how the contrast between the two machines might might be accounted for. Specifically, I noted that, after converting my 150k folder into a git-annex repo, ascertaining that the back-up is current via rsync would take ~15 minutes, where it used to take mere seconds before. This could then only be due to the demands on directory traversal introduced by the annex-layout.
+
+Accordingly, I wanted to see whether the traversal would be something that explains the difference in performance between btrfs and ext4, so I ran a tiny benchmark, traversing the .git folder on my home drive (ext4, SATA) and the backup drive (btrfs, USB3), and I was astounded by the difference:
+
+zardoz [ /mnt/bak/m-annex ]$ time tree .git >/dev/null
+
+tree .git >/dev/null  14,23s user 2,78s system 24% cpu 1:09,69 total
+
+zardoz [ ~/m-annex ]$ time tree .git >/dev/null
+
+tree .git >/dev/null  26,40s user 0,96s system 1% cpu 23:37,94 total
+
+While running, I peeked into the io using iotop, and observed around 500K/s during traversal on ext4 vs. 5M/s on btrfs.
+
+While I was aquainted with the dogma that file-systems hate to have a single folder with a bazillion files, sources on the net seem to indicate that having lots and lots of sparse folders is even worse, and given the one-file-per-folder structure of the annex objects store, this would then potentially explain the heavy thrashing on ext4.
+
+Something I am wondering now: Which operations in git-annex (or plain git) incite that sort of directory traversal? One candidate which occurred to me is «git annex unused», and the differences in performance between ext4 and btrfs are on the same order as in the above benchmark. Originally, Joey related somewhere that the search in «unused» is expensive; but if traversal is involved, it could actually be that this has even more impact than searching git history.
+
+In any case, if anyone wants to track a very large number of files via git-annex, ext4 seems to be not the ideal file-system for this.
diff --git a/doc/forum/Performance_implications_of_triply_nested_objects_directory/comment_1_068a8f120d188b8fa5d3e5b687fd02dc._comment b/doc/forum/Performance_implications_of_triply_nested_objects_directory/comment_1_068a8f120d188b8fa5d3e5b687fd02dc._comment
new file mode 100644
index 0000000000..0a368b3a31
--- /dev/null
+++ b/doc/forum/Performance_implications_of_triply_nested_objects_directory/comment_1_068a8f120d188b8fa5d3e5b687fd02dc._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="http://joeyh.name/"
+ ip="209.250.56.2"
+ subject="comment 1"
+ date="2014-07-14T18:33:25Z"
+ content="""
+I'm afraid that you're comparing apples (SATA drives) and oranges (USB drives).
+"""]]
diff --git a/doc/forum/Performance_implications_of_triply_nested_objects_directory/comment_2_cc0f5be21fd1523bdddc7bcf6ff04435._comment b/doc/forum/Performance_implications_of_triply_nested_objects_directory/comment_2_cc0f5be21fd1523bdddc7bcf6ff04435._comment
new file mode 100644
index 0000000000..e47565d676
--- /dev/null
+++ b/doc/forum/Performance_implications_of_triply_nested_objects_directory/comment_2_cc0f5be21fd1523bdddc7bcf6ff04435._comment
@@ -0,0 +1,10 @@
+[[!comment format=mdwn
+ username="zardoz"
+ ip="78.49.247.112"
+ subject="comment 2"
+ date="2014-07-14T19:34:31Z"
+ content="""
+I suppose you misread something. Note that the 60s traversal was on the (slower) /external/ drive, and the 24minute traversal on the (faster) internal one, so if anything, the results become more pronounced because of the comparison.
+
+In the meantime I converted the internal drive to btrfs, and the traversal time on SATA is now 30s, as opposed to the >20min on ext.
+"""]]
diff --git a/doc/forum/Performance_implications_of_triply_nested_objects_directory/comment_3_1133795276371c86cdd52b25a8b20c52._comment b/doc/forum/Performance_implications_of_triply_nested_objects_directory/comment_3_1133795276371c86cdd52b25a8b20c52._comment
new file mode 100644
index 0000000000..6b875932d0
--- /dev/null
+++ b/doc/forum/Performance_implications_of_triply_nested_objects_directory/comment_3_1133795276371c86cdd52b25a8b20c52._comment
@@ -0,0 +1,28 @@
+[[!comment format=mdwn
+ username="zardoz"
+ ip="134.147.14.84"
+ subject="comment 3"
+ date="2014-07-15T07:29:37Z"
+ content="""
+The ext-community seems to
+[corroborate](http://www.redhat.com/archives/ext3-users/2013-March/msg00007.html)
+the observation that having many sparse directories scales extremely
+poorly, though this still leaves me curious as to how btrfs deals
+with.
+
+One thing I read is that btrfs stores a secondary index on directories
+and uses this index in the readdir() syscall; this secondary index
+works in such a way that inodes are likely to be traversed in
+sequential on-disk order, while for ext, the readdir() results will be
+ordered by inode number, yielding a random access pattern.
+
+I must confess I’ve always liked to think of the file-system as a
+cheap data-base, but apparently that is not such a good idea (i. e.,
+it’s not cheap at all, in the long run). On the other hand (supposing
+that operations like «git annex unused» do indeed work by traversing
+the object tree), it probably wouldn’t be easy coming up with a scheme
+that scales better. For traversal-bound operations, one might maintain
+a database, but it would be a hassle to ensure that this in always in
+sync with the file-system.
+
+"""]]
diff --git a/doc/forum/Performance_implications_of_triply_nested_objects_directory/comment_4_4e44289e6913797844e103f9cdf4c5a2._comment b/doc/forum/Performance_implications_of_triply_nested_objects_directory/comment_4_4e44289e6913797844e103f9cdf4c5a2._comment
new file mode 100644
index 0000000000..ea0b82f715
--- /dev/null
+++ b/doc/forum/Performance_implications_of_triply_nested_objects_directory/comment_4_4e44289e6913797844e103f9cdf4c5a2._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="http://joeyh.name/"
+ ip="108.236.230.124"
+ subject="comment 4"
+ date="2014-07-21T17:19:42Z"
+ content="""
+ suggests using -I 512 -O inline_data (\"bignode\")
+"""]]
diff --git a/doc/forum/Permission_denied_-_what_is_wrong__63__.mdwn b/doc/forum/Permission_denied_-_what_is_wrong__63__.mdwn
new file mode 100644
index 0000000000..689ec7a0de
--- /dev/null
+++ b/doc/forum/Permission_denied_-_what_is_wrong__63__.mdwn
@@ -0,0 +1,28 @@
+I just set up one local repo and one remote server repo (accessible via SSH) that are synced with each other. However, when I add a file to the local repo it never appears on the remote repo. From the log:
+
+***************************************************************************
+    (scanning...) [2016-01-18 14:55:26.566983] Watcher: Performing startup scan
+    [2016-01-18 14:55:26.584387] Committer: Committing changes to git
+    (recording state in git...)
+    (started...) 
+    Permission denied, please try again.
+    Permission denied, please try again.
+    Permission denied (publickey,password,keyboard-interactive).
+    rsync: connection unexpectedly closed (0 bytes received so far) [sender]
+    rsync error: unexplained error (code 255) at /SourceCache/rsync/rsync-45/rsync/io.c(453) [sender=2.6.9]
+    
+      rsync failed -- run git annex again to resume file transfer
+    [2016-01-18 14:55:30.429824] main: Syncing with lacieannex 
+    
+    Permission denied, please try again.
+    Permission denied, please try again.
+    Permission denied (publickey,password,keyboard-interactive).
+    rsync: connection unexpectedly closed (0 bytes received so far) [sender]
+    rsync error: error in rsync protocol data stream
+      rsync failed -- run git annex again to resume file transfer
+     (code 12) at /SourceCache/rsync/rsync-45/rsync/io.c(453) [sender=2.6.9]
+***************************************************************************
+
+I suspect there is something with the ssh authentication. I tried to remove all keys from my known_hosts that was related to the server (a handful since I had accessed it at server.local, server.lan, 192.168.1.3, fqdn:2222 etc). My plan was to re-run the setup screen where you enter the details about the remote server, including your username and password, but that step seems unavailable after the initial setup.
+
+How do I fix the issues above?
diff --git a/doc/forum/Permission_denied_-_what_is_wrong__63__/comment_1_b850a32cce4c9782e59fc91076b6cae7._comment b/doc/forum/Permission_denied_-_what_is_wrong__63__/comment_1_b850a32cce4c9782e59fc91076b6cae7._comment
new file mode 100644
index 0000000000..91a09860cd
--- /dev/null
+++ b/doc/forum/Permission_denied_-_what_is_wrong__63__/comment_1_b850a32cce4c9782e59fc91076b6cae7._comment
@@ -0,0 +1,14 @@
+[[!comment format=mdwn
+ username="joey"
+ subject="""comment 1"""
+ date="2016-01-19T20:26:57Z"
+ content="""
+The remote server is rejecting your ssh key to log into it.
+
+If you used the web app to set this up, it will have generated a ssh key
+just for this server, and put it in `~/.ssh/git-annex/` on the client.
+
+You need to check your `~/.ssh/authorized_keys` file on the server
+and make sure that the ssh key is listed in it. Checking the server's
+ssh log file may also help track down why it's rejecting the login.
+"""]]
diff --git a/doc/forum/Permission_denied_-_what_is_wrong__63__/comment_2_e8d2170e5f830d0f98bc8251caf8bc5e._comment b/doc/forum/Permission_denied_-_what_is_wrong__63__/comment_2_e8d2170e5f830d0f98bc8251caf8bc5e._comment
new file mode 100644
index 0000000000..766b3dbf02
--- /dev/null
+++ b/doc/forum/Permission_denied_-_what_is_wrong__63__/comment_2_e8d2170e5f830d0f98bc8251caf8bc5e._comment
@@ -0,0 +1,14 @@
+[[!comment format=mdwn
+ username="db"
+ subject="comment 2"
+ date="2016-01-23T23:06:03Z"
+ content="""
+In fact, I have 2x2 keys on my local machine
+
+key.git-annex-fqdn-db_2222_db.2Fannex
+key.git-annex-fqdn-db_2222_db.2Fannex.pub
+key.git-annex-fqdn-db_22_annex
+key.git-annex-fqdn-db_22_annex.pub
+
+Why duplicates? The \"public 2222\" key is in my  ~/.ssh/authorized_keys but not the \"public 22\"-key.
+"""]]
diff --git a/doc/forum/Permission_denied___40__publickey__41___On_second_Step_...___63__.mdwn b/doc/forum/Permission_denied___40__publickey__41___On_second_Step_...___63__.mdwn
new file mode 100644
index 0000000000..caa9fc85a3
--- /dev/null
+++ b/doc/forum/Permission_denied___40__publickey__41___On_second_Step_...___63__.mdwn
@@ -0,0 +1,36 @@
+Hello,
+
+I've a strange (for me ;-)) problem.
+
    I'm trying to use git-annex with 2 debian wheezy (packages from backports)
+
    I've put my ssh pub key on the server in the autorized_keys for my username account (and for the root account to be sure)
+my username account is member of the sudo group (not sure it's required)
+
+I can create an ssh session directly to the server with :
+
+    ssh username@remoteserver.lan -p 62322
+
+and work inside this session.
+
+    Sep 14 03:42:35 file sshd[8849]: Accepted publickey for username from 80.201.25.4 port 32969 ssh2
+    Sep 14 03:42:35 file sshd[8849]: pam_unix(sshd:session): session opened for user username by (uid=0)
+    ...
+    Sep 14 05:13:14 file sshd[8851]: Received disconnect from 80.201.25.4: 11: disconnected by user
+    Sep 14 05:13:14 file sshd[8849]: pam_unix(sshd:session): session closed for user username
+
+But when I use the git-annex assistant on the same workstation with the same remote server and port,
+
    the 1st step is OK,
+
    but when I've to choose between git or rsync encrypted, I'm choosing git and receive :
+
+    "Permission denied (publickey)."
+
+On the server I can only see that the ssh session is immediately closed after opened and before the choice git or rsync is made.
+
+    Sep 14 05:13:42 file sshd[8882]: Accepted publickey for username from 80.201.25.4 port 32984 ssh2
+    Sep 14 05:13:42 file sshd[8882]: pam_unix(sshd:session): session opened for user username by (uid=0)
+    Sep 14 05:13:42 file sshd[8886]: Received disconnect from 80.201.25.4: 11: disconnected by user
+    Sep 14 05:13:42 file sshd[8882]: pam_unix(sshd:session): session closed for user username
+
+Do you think it's the problem ? or whatelse ...
+
    and could you guide me to a possible solution ?
+
+Best regards
diff --git a/doc/forum/Permission_denied___40__publickey__41___On_second_Step_...___63__/comment_1_6c74f0b43c457fe97b2d8630ca4fde29._comment b/doc/forum/Permission_denied___40__publickey__41___On_second_Step_...___63__/comment_1_6c74f0b43c457fe97b2d8630ca4fde29._comment
new file mode 100644
index 0000000000..8d22157726
--- /dev/null
+++ b/doc/forum/Permission_denied___40__publickey__41___On_second_Step_...___63__/comment_1_6c74f0b43c457fe97b2d8630ca4fde29._comment
@@ -0,0 +1,10 @@
+[[!comment format=mdwn
+ username="http://joeyh.name/"
+ ip="108.236.230.124"
+ subject="comment 1"
+ date="2013-11-08T18:24:27Z"
+ content="""
+The webapp sets up a dedicated ssh public key when you add a ssh repository in there.
+
+You might see some useful debugging info your your problem in ~/annex/.git/annex/daemon.log
+"""]]
diff --git a/doc/forum/Permission_denied___40__publickey__41___On_second_Step_...___63__/comment_2_b7a384e853e1756a684774348fad29e6._comment b/doc/forum/Permission_denied___40__publickey__41___On_second_Step_...___63__/comment_2_b7a384e853e1756a684774348fad29e6._comment
new file mode 100644
index 0000000000..ecf26daa10
--- /dev/null
+++ b/doc/forum/Permission_denied___40__publickey__41___On_second_Step_...___63__/comment_2_b7a384e853e1756a684774348fad29e6._comment
@@ -0,0 +1,22 @@
+[[!comment format=mdwn
+ username="Auroch"
+ ip="87.65.174.80"
+ subject="OK,thanx for this"
+ date="2013-11-09T09:10:42Z"
+ content="""
+The problem is not solve, but with your informations, I think i've found somehting.
+
+Is it possible that the problem is linked to my situation of port-forwarding ?
+The remoteserver is a host (proxmox) with iptable port forwarding 62322 to the real port 22 of the VM hosting the git where I try to connect.
+
+So could the problem seems to be that some actions are taken on the host and not the VM ... correct ?
+
+    [2013-11-09 09:51:33 CET] read: ssh-keygen [\"-F\",\"bla.remote.tld\"]
+    [2013-11-09 09:51:33 CET] read: ssh [\"-oNumberOfPasswordPrompts=0\",\"-n\",\"-p\",\"62322\",\"user@bla.remote.tld\",\"sh -c 'echo git-annex-probe loggedin;if which git-annex-shell; then echo git-annex-probe git-annex-shell; fi;if which rsync; then echo git-annex-probe rsync; fi;if which ~/.ssh/git-annex-shell; then echo git-annex-probe ~/.ssh/git-annex-shell; fi'\"]
+    [2013-11-09 09:51:35 CET] chat: ssh [\"user@bla.remote.tld\",\"sh -c 'mkdir -p '\\"'\\"'annex'\\"'\\"'&&cd '\\"'\\"'annex'\\"'\\"'&&if [ ! -d .git ]; then git init --bare --shared; fi&&git annex init'\"]
+
+If it's correct, have you an idea for solving this ?
+
+best regards
+
+"""]]
diff --git a/doc/forum/Permission_denied___40__publickey__41___On_second_Step_...___63__/comment_3_3a8a7f51cb04a92c576549d379b57248._comment b/doc/forum/Permission_denied___40__publickey__41___On_second_Step_...___63__/comment_3_3a8a7f51cb04a92c576549d379b57248._comment
new file mode 100644
index 0000000000..546f647edb
--- /dev/null
+++ b/doc/forum/Permission_denied___40__publickey__41___On_second_Step_...___63__/comment_3_3a8a7f51cb04a92c576549d379b57248._comment
@@ -0,0 +1,12 @@
+[[!comment format=mdwn
+ username="http://joeyh.name/"
+ ip="209.250.56.246"
+ subject="comment 3"
+ date="2013-11-09T18:07:56Z"
+ content="""
+Well, it does look like your git-annex did not pass the port to the second command. 
+
+What version are you using? I've just tested with a ssh server only listening on 2222 and it used the port throughout.
+
+You might try upgrading to a recent version. I know that version 4.20130627, at least, had a bug with the ssh port.
+"""]]
diff --git a/doc/forum/Permission_denied___40__publickey__41___On_second_Step_...___63__/comment_4_582ad3ba0c62a77b08a10b37a780c670._comment b/doc/forum/Permission_denied___40__publickey__41___On_second_Step_...___63__/comment_4_582ad3ba0c62a77b08a10b37a780c670._comment
new file mode 100644
index 0000000000..c8d8c8b520
--- /dev/null
+++ b/doc/forum/Permission_denied___40__publickey__41___On_second_Step_...___63__/comment_4_582ad3ba0c62a77b08a10b37a780c670._comment
@@ -0,0 +1,15 @@
+[[!comment format=mdwn
+ username="Auroch"
+ ip="87.65.174.80"
+ subject="comment 4"
+ date="2013-11-09T19:24:54Z"
+ content="""
+I've the wheezy-backports version
+
+    i   4.20131002~bpo70+1                                                        wheezy-backports                                      100
+
+I'll have to wait an update and I'll come back to you.
+
+Thanx this amazing work !
+
+"""]]
diff --git a/doc/forum/Please_fix_compatibility_with_ghc_7.0.mdwn b/doc/forum/Please_fix_compatibility_with_ghc_7.0.mdwn
new file mode 100644
index 0000000000..d49081bf34
--- /dev/null
+++ b/doc/forum/Please_fix_compatibility_with_ghc_7.0.mdwn
@@ -0,0 +1 @@
+I'm having trouble installing the latest git-annex. It depends on 'base >= 4.5 && base < 5', but ghc 7.0 only ships base 3.0. I've tried upgrading ghc to 7.4 but that breaks a whole bunch of other things; for example, the Crypto module fails to compile, preventing me from installing ghc. Please fix compatibility with ghc 7.0.
diff --git a/doc/forum/Please_fix_compatibility_with_ghc_7.0/comment_1_d1d10217ebd0151e947b3a6cd37399ce._comment b/doc/forum/Please_fix_compatibility_with_ghc_7.0/comment_1_d1d10217ebd0151e947b3a6cd37399ce._comment
new file mode 100644
index 0000000000..04b8ce6750
--- /dev/null
+++ b/doc/forum/Please_fix_compatibility_with_ghc_7.0/comment_1_d1d10217ebd0151e947b3a6cd37399ce._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="http://joey.kitenet.net/"
+ nickname="joey"
+ subject="comment 1"
+ date="2012-03-11T15:50:11Z"
+ content="""
+There is a `ghc7.0` branch in [[git|download]] that is being maintained to work with that version.
+"""]]
diff --git a/doc/forum/Please_help__44___half_my_files_disappeared.mdwn b/doc/forum/Please_help__44___half_my_files_disappeared.mdwn
new file mode 100644
index 0000000000..b8a1a0186b
--- /dev/null
+++ b/doc/forum/Please_help__44___half_my_files_disappeared.mdwn
@@ -0,0 +1,14 @@
+Hello, I was writing a message to ask for help but I found a solution for my problem so I deleted it but now I think it could be useful to someone else so hereafter is my initial message and my workaround.
+
+
+> Hello, this time I am very very frightened by the odds but I am still looking for your help. Long story short: windows update.
+
+> Yesterday, everything was fine with my local git-annex using WSL. This morning, windows had updated itself to version 1803 automatically, (by the way, thanks to Microsoft for their forced updates), and half my locked files are gone. I tried git annex fix ., nothing, I tried to find the associated keys with find .git/annex/objects -name 'sha...', nothing. git annex whereis  says the file is here whereas git annex fsck  fails telling the file's content is missing.
+
+> Now, it seems like the only remaining files have links pointing to annex/objects directories that are exclusively composed of uppercase letters or numbers. Looking more closely to the objects folders, it seems like they are all strangely duplicated depending on the spelling: for example the variations zZ, Zz, ZZ, zz which all have the same subdirectories and files. Is this the way those directories should be named or did Windows break everything?
+
+> The last thing that keeps me hoping is that the hard drive usage seems to be the same as before but with the apparent duplication of object directories, my hopes are fading away pretty fast... If anyone can help me, or even finish to turn my hopes down, I am listening.
+
+I solved this by using the shiny feature that lets a Windows user revert the OS to previous version once yet another a big flaw has been found in the 1803 version.
+Strangely, after the revert, all my files were back to their normal state, as I left them before the upgrade.
+I hope none of you will need this ;)
diff --git a/doc/forum/Please_help__44___half_my_files_disappeared/comment_1_ba7c79dd7805aa9ec1b90d643f6bf555._comment b/doc/forum/Please_help__44___half_my_files_disappeared/comment_1_ba7c79dd7805aa9ec1b90d643f6bf555._comment
new file mode 100644
index 0000000000..2cf06bda26
--- /dev/null
+++ b/doc/forum/Please_help__44___half_my_files_disappeared/comment_1_ba7c79dd7805aa9ec1b90d643f6bf555._comment
@@ -0,0 +1,16 @@
+[[!comment format=mdwn
+ username="joey"
+ subject="""comment 1"""
+ date="2018-09-24T15:14:49Z"
+ content="""
+Glad you recovered ok.
+
+It sounds like windows may be doing something horrible with case
+insensitivity in the file system. It's normal for .git/annex/objects/
+to contain subdirectories with names that only differ by case.
+
+If Windows used to respond to a request for .git/annex/object/XY with
+the contents of that, but now instead responds with the contents of the xy
+directory, it should be possible to rearrange things so git-annex still
+works, but I would not recommend doing it with live data without backups.
+"""]]
diff --git a/doc/forum/Please_publish_new_releases_not_shorter_than_11_days.mdwn b/doc/forum/Please_publish_new_releases_not_shorter_than_11_days.mdwn
new file mode 100644
index 0000000000..f67d2d797d
--- /dev/null
+++ b/doc/forum/Please_publish_new_releases_not_shorter_than_11_days.mdwn
@@ -0,0 +1,6 @@
+Hi,
+i am following debian testing. The latest git-annex publication has manged to replace the former version on the last possible moment. Now another 10 days waiting.
+Would be nice if you can coordinate with testing.
+
+Thank you for git-annex. 
+Jürgen
diff --git a/doc/forum/Please_publish_new_releases_not_shorter_than_11_days/comment_1_da3d39de5be47ebe8b25a42ed1f36510._comment b/doc/forum/Please_publish_new_releases_not_shorter_than_11_days/comment_1_da3d39de5be47ebe8b25a42ed1f36510._comment
new file mode 100644
index 0000000000..1fb259c98c
--- /dev/null
+++ b/doc/forum/Please_publish_new_releases_not_shorter_than_11_days/comment_1_da3d39de5be47ebe8b25a42ed1f36510._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="http://joeyh.name/"
+ ip="209.250.56.47"
+ subject="comment 1"
+ date="2013-11-03T19:57:24Z"
+ content="""
+I do coordinate with testing. That version had no possibility of ever reaching testing, since it failed to build on several architectures.
+"""]]
diff --git a/doc/forum/Podcast_syncing_use-case.mdwn b/doc/forum/Podcast_syncing_use-case.mdwn
new file mode 100644
index 0000000000..6b3c81cabc
--- /dev/null
+++ b/doc/forum/Podcast_syncing_use-case.mdwn
@@ -0,0 +1,34 @@
+I've been trying to use git-annex with the following strategy.
+
+* Download podcasts into the annex `gpodder-downloads`
+* Check the podcasts into the annex using `git annex add`.
+* Copy the podcasts over to my mp3 player in the annex `usb-ariaz`.
+  This is a FAT-formatted mp3 player, so I have been using a bare
+  repository.  
+* Move the podcasts to a different annex called `gpodder-on-usbdisk`
+  to indicate that they have been successfully put on the mp3 player.
+* `chmod` the files on the mp3 player to `0600` so that I can delete
+  them from the player when I am done listening to them.
+  
+Then I go for a run or something and listen to a bunch of podcasts,
+deleting them after I have listened to them.  When I get back, I would
+like to find the files that I have listened to and remove them from
+the annexes that are not on the mp3 player.  What I have been hoping
+is that something like
+
+    ~/gpodder-on-usbdisk $ git annex find --not --in usb-ariaz --print0 | xargs -0 git rm
+    ~/gpodder-on-usbdisk $ git annex unused
+    ~/gpodder-on-usbdisk $ git annex dropunused `seq X`
+	
+would work.  However, it appears that `git-annex find` does not
+actually check to see that the file contents are present, but only
+looks at the `git-annex` branch of the `usb-ariaz` repository.  Since
+I have not changed that with my sneaky deletions, it has no way of
+knowing that the files have been deleted.
+
+Is there any way to do this properly?  (And by properly, I don't mean
+"don't delete the files".  That is really the only way I have of
+marking that I have listened to podcasts on this particular mp3 player.)
+
+I tried setting the `usb-ariaz` repository to be untrusted, but that
+did not change the behavior of `git annex find`.
diff --git a/doc/forum/Podcast_syncing_use-case/comment_1_ace6f9d3a950348a3ac0ff592b62e786._comment b/doc/forum/Podcast_syncing_use-case/comment_1_ace6f9d3a950348a3ac0ff592b62e786._comment
new file mode 100644
index 0000000000..fe396c39f3
--- /dev/null
+++ b/doc/forum/Podcast_syncing_use-case/comment_1_ace6f9d3a950348a3ac0ff592b62e786._comment
@@ -0,0 +1,10 @@
+[[!comment format=mdwn
+ username="http://joey.kitenet.net/"
+ nickname="joey"
+ subject="comment 1"
+ date="2011-11-27T17:56:31Z"
+ content="""
+Right, --in goes by git-annex's [[location_tracking]] information; actually checking if a remote still has the files would make --in too expensive in many cases.
+
+So you need to give `gpodder-on-usbdisk` current information. You can do that by going to `usb-ariaz` and doing a `git annex fsck`. That will find the deleted files and update the location information. Then, back on `gpodder-on-usbdisk`, `git pull usb-ariaz`, and then you can proceed with the commands you showed.
+"""]]
diff --git a/doc/forum/Podcast_syncing_use-case/comment_2_930a6620b4d516e69ed952f9da5371bb._comment b/doc/forum/Podcast_syncing_use-case/comment_2_930a6620b4d516e69ed952f9da5371bb._comment
new file mode 100644
index 0000000000..97eb3c681c
--- /dev/null
+++ b/doc/forum/Podcast_syncing_use-case/comment_2_930a6620b4d516e69ed952f9da5371bb._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="http://cgray.myopenid.com/"
+ nickname="cgray"
+ subject="comment 2"
+ date="2011-11-27T22:10:44Z"
+ content="""
+Thanks, that works perfectly!
+"""]]
diff --git a/doc/forum/Poor_man__39__s_IMAP.mdwn b/doc/forum/Poor_man__39__s_IMAP.mdwn
new file mode 100644
index 0000000000..af53ddc197
--- /dev/null
+++ b/doc/forum/Poor_man__39__s_IMAP.mdwn
@@ -0,0 +1,6 @@
+I have an e-mail server configured to save my mail in ~/Maildir on an account that is available over ssh. I'd like to keep emailserver:~/Maildir in a two-way sync with laptop:~/Mail/private essentially creating a poor man's IMAP — without setting up and maintaining an actual IMAP server. **Is it an appropriate use of git-annex or would another tool be more fitting? And how do I go about doing it?** I'd like to sync the files, the content, not just information about the files or other meta-data. 
+
+I tried setting it up with the webUI to the assistant but it only offers encrypted storage[1] on the remote server. I looked into setting it up manually but "git-annex does not notice when files are added to remote rsync repositories."[2]
+
+[1] http://git-annex.branchable.com/bugs/Remote_repositories_have_to_be_setup_encrypted/  
+[2] from comments on http://git-annex.branchable.com/special_remotes/rsync/
diff --git a/doc/forum/Poor_man__39__s_IMAP/comment_1_258ff23c462dc88b88ced405c4f5040f._comment b/doc/forum/Poor_man__39__s_IMAP/comment_1_258ff23c462dc88b88ced405c4f5040f._comment
new file mode 100644
index 0000000000..8965f18be3
--- /dev/null
+++ b/doc/forum/Poor_man__39__s_IMAP/comment_1_258ff23c462dc88b88ced405c4f5040f._comment
@@ -0,0 +1,11 @@
+[[!comment format=mdwn
+ username="http://a-or-b.myopenid.com/"
+ ip="220.244.41.108"
+ subject="comment 1"
+ date="2013-08-14T06:45:56Z"
+ content="""
+I'd say that git-annex is the wrong tool.
+
+OfflineIMAP ([[http://offlineimap.org]]) is simple to set up, is easy to secure over ssh and does exactly what you want.
+
+"""]]
diff --git a/doc/forum/Poor_man__39__s_IMAP/comment_2_c88d1abdda4cb526a6ee45a710c75bc4._comment b/doc/forum/Poor_man__39__s_IMAP/comment_2_c88d1abdda4cb526a6ee45a710c75bc4._comment
new file mode 100644
index 0000000000..4ab7af46ba
--- /dev/null
+++ b/doc/forum/Poor_man__39__s_IMAP/comment_2_c88d1abdda4cb526a6ee45a710c75bc4._comment
@@ -0,0 +1,10 @@
+[[!comment format=mdwn
+ username="http://a-or-b.myopenid.com/"
+ ip="220.244.41.108"
+ subject="comment 2"
+ date="2013-08-14T06:53:04Z"
+ content="""
+From my massive amount of googling, i.e. 2min, apparently mbsync ([[http://isync.sourceforge.net/mbsync.html]]) is good too.
+
+I've never used it, so YMMV.
+"""]]
diff --git a/doc/forum/Poor_man__39__s_IMAP/comment_3_3847e371db1c2788c075e7dca1fbd33e._comment b/doc/forum/Poor_man__39__s_IMAP/comment_3_3847e371db1c2788c075e7dca1fbd33e._comment
new file mode 100644
index 0000000000..a8ed314756
--- /dev/null
+++ b/doc/forum/Poor_man__39__s_IMAP/comment_3_3847e371db1c2788c075e7dca1fbd33e._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="http://spindritf.myopenid.com/"
+ nickname="spindritf"
+ subject="comment 3"
+ date="2013-08-14T07:30:30Z"
+ content="""
+Offlineimap connects to an IMAP server and I was hoping to avoid running one. Thanks though. 
+"""]]
diff --git a/doc/forum/Poor_man__39__s_IMAP/comment_4_cf6cc21f2cf2aa5c949844e24a7b4075._comment b/doc/forum/Poor_man__39__s_IMAP/comment_4_cf6cc21f2cf2aa5c949844e24a7b4075._comment
new file mode 100644
index 0000000000..bcb6982164
--- /dev/null
+++ b/doc/forum/Poor_man__39__s_IMAP/comment_4_cf6cc21f2cf2aa5c949844e24a7b4075._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="https://me.yahoo.com/a/FHnTlSBo1eCGJRwueeKeB6.RCaPbGMPr5jxx8A--#ce0d8"
+ nickname="Hamza"
+ subject="comment 4"
+ date="2013-08-14T11:11:59Z"
+ content="""
+IMHO It is not the wrong tool for the job, after all the whole point of assistant is to sync folder across machines. I have a similar setup. I have two folders on two machines plus an encrypted repo on an ssh server. On both folders (local/server) i keep webapp running they sync using the encrypted repo/xmpp. The only downside to this approach is that VPS has two copies of every file one in the folder on in the encrypted git repo.
+"""]]
diff --git a/doc/forum/Poor_man__39__s_IMAP/comment_5_d861fa69475ce526841b3195be8ee356._comment b/doc/forum/Poor_man__39__s_IMAP/comment_5_d861fa69475ce526841b3195be8ee356._comment
new file mode 100644
index 0000000000..d5621bb4a9
--- /dev/null
+++ b/doc/forum/Poor_man__39__s_IMAP/comment_5_d861fa69475ce526841b3195be8ee356._comment
@@ -0,0 +1,10 @@
+[[!comment format=mdwn
+ username="http://spindritf.myopenid.com/"
+ nickname="spindritf"
+ subject="comment 5"
+ date="2013-08-14T18:08:22Z"
+ content="""
+> IMHO It is not the wrong tool for the job, after all the whole point of assistant is to sync folder across machines.
+
+That's what I thought. OK, so how do I do it? Should I set up this rsync special remote and run sync every once in a while. When starting the e-mail client for example? 
+"""]]
diff --git a/doc/forum/Poor_man__39__s_IMAP/comment_6_1e81bd4bb62652bc674cdcd7ed57ac5c._comment b/doc/forum/Poor_man__39__s_IMAP/comment_6_1e81bd4bb62652bc674cdcd7ed57ac5c._comment
new file mode 100644
index 0000000000..7e07204561
--- /dev/null
+++ b/doc/forum/Poor_man__39__s_IMAP/comment_6_1e81bd4bb62652bc674cdcd7ed57ac5c._comment
@@ -0,0 +1,12 @@
+[[!comment format=mdwn
+ username="https://me.yahoo.com/a/FHnTlSBo1eCGJRwueeKeB6.RCaPbGMPr5jxx8A--#ce0d8"
+ nickname="Hamza"
+ subject="comment 6"
+ date="2013-08-16T09:14:52Z"
+ content="""
+You can either keep running 2 webapps paired using xmpp running all the time that gives you push like notifications or if you are ok with syncing every once in a while you can have a check mail script that adds files on the server commits them then calls git annex sync locally,
+
+    ssh $1 \"cd /path/to/annex/;git add .;git commit 'Update'\";git annex sync
+
+should do the trick. The latter method you just need to annex repos no encrypted third repo. Just init your git annex repo on the server and clone on the laptop thats it.
+"""]]
diff --git a/doc/forum/Poor_man__39__s_IMAP/comment_7_b3929281dff6078d77f1b9ae42e25bb6._comment b/doc/forum/Poor_man__39__s_IMAP/comment_7_b3929281dff6078d77f1b9ae42e25bb6._comment
new file mode 100644
index 0000000000..aba4cd0de9
--- /dev/null
+++ b/doc/forum/Poor_man__39__s_IMAP/comment_7_b3929281dff6078d77f1b9ae42e25bb6._comment
@@ -0,0 +1,16 @@
+[[!comment format=mdwn
+ username="http://joeyh.name/"
+ ip="4.154.0.63"
+ subject="comment 7"
+ date="2013-08-23T18:30:04Z"
+ content="""
+I don't feel that git-annex is the best thing to use for this. Maildir has some specific semantics for the filenames used in it that let imap clients resolve inconsistencies, such as a message that was read on machine A, and deleted on machine B. git-annex is unlikely to work as well.
+
+However, I have to say that the very beginning of this thread has a wrong statement in it. 
+
+> I tried setting it up with the webUI to the assistant but it only offers encrypted storage[1] on the remote server.
+
+If you install git and git-annex on your remote server, the git-annex assistant will detect this, and offer the choice between a regular git repository, and encrypted storage. If you don't have them installed, it tells you you don't, offers to let you retry once you do install them, and offers encrypted storage as the only option that works given what's installed on the server.
+
+(Also, the bug you linked to in [1] has nothing at all to do with what you were talking about.)
+"""]]
diff --git a/doc/forum/Poor_man__39__s_IMAP/comment_8_69506e8c519196f44b9ed15b32f00106._comment b/doc/forum/Poor_man__39__s_IMAP/comment_8_69506e8c519196f44b9ed15b32f00106._comment
new file mode 100644
index 0000000000..18b99a880c
--- /dev/null
+++ b/doc/forum/Poor_man__39__s_IMAP/comment_8_69506e8c519196f44b9ed15b32f00106._comment
@@ -0,0 +1,22 @@
+[[!comment format=mdwn
+ username="konubinix"
+ ip="82.243.233.186"
+ subject="And with a lots of mails?"
+ date="2013-08-27T06:44:56Z"
+ content="""
+Hi,
+
+I have a question quite similar, but for a different purpose. I use OfflineImap for imap synchronisation, but in my current situation, I travel a lot between two places: one connected to the Internet and the other not connected.
+
+When I am connected to the Internet, I may synchronize mails, then I rsync my ~/Mail directory to a usb key so that I have access to them in the place without connection. The mails filenames may be changed as Joeyh mentioned while I read or delete them. I rsync them back to the usb key before going back to the place with Internet connection, where the OfflineImap synchronization may occur.
+
+This solution, with rsynced key, works well. But I would love a history of what was done with file names and may be able to retrieve an old mail.
+
+I figured out that that the git annex was a really good solution for that.
+
+The main drawback from my point of view is that I have around 100 000 mails (some would say that's \"[Not much mail](http://notmuchmail.org/)\"), and I am afraid that git will be quite slow with that amount of files.
+
+Did anyone experience an annexed repository with so many items it in?
+
+Best
+"""]]
diff --git a/doc/forum/Portable_version_of_git-annex_for_windows.mdwn b/doc/forum/Portable_version_of_git-annex_for_windows.mdwn
new file mode 100644
index 0000000000..e4c0bb7374
--- /dev/null
+++ b/doc/forum/Portable_version_of_git-annex_for_windows.mdwn
@@ -0,0 +1,22 @@
+Problem: I don't have admin access on all the computers I want to plug my USB with git-annex on.
+Solution: Make git & git-annex portable.
+
+The only catch is I'm not familiar enough with git to be able to actually tell if the more specialized git-annex working properly. The commands execute without crashing so I suspect it's fine but I was hoping someone else more competent could try this out and let me know if I it's working so I can add it to the tips section. Thanks in advance!
+
+Ingredients:
+
+* Portable version of msysgit: 
+
+* Windows git-annex installer: 
+
+* 7-zip (portable version optional): 
+
+Steps:
+
+1. Use 7-zip to extract the contents of the git-annex installer into their own folder.
+
+2. Extract the portable version of git and copy all files extracted from git-annex installer into the "cmd" folder.
+
+3. Run it!
+
+Edit: the markdown is eating my attempts to create a numbered list and making them a paragraph instead. Apologies!
diff --git a/doc/forum/Portable_version_of_git-annex_for_windows/comment_1_e5e60fa8d104a09152a8164d5a906aec._comment b/doc/forum/Portable_version_of_git-annex_for_windows/comment_1_e5e60fa8d104a09152a8164d5a906aec._comment
new file mode 100644
index 0000000000..2d5b22ee87
--- /dev/null
+++ b/doc/forum/Portable_version_of_git-annex_for_windows/comment_1_e5e60fa8d104a09152a8164d5a906aec._comment
@@ -0,0 +1,10 @@
+[[!comment format=mdwn
+ username="rtwolf"
+ ip="69.165.155.234"
+ subject="comment 1"
+ date="2014-01-07T17:38:19Z"
+ content="""
+Seems windows version of git-annex only runs in direct mode so I got the dreaded \"fatal: this operation must be run in a work tree\" error. Ran \"git config core.bare false\" which fixed it.
+
+More info on direct mode: 
+"""]]
diff --git a/doc/forum/Portable_version_of_git-annex_for_windows/comment_2_d8d1aa0920351e880ba6678bb97585de._comment b/doc/forum/Portable_version_of_git-annex_for_windows/comment_2_d8d1aa0920351e880ba6678bb97585de._comment
new file mode 100644
index 0000000000..3602e88c6a
--- /dev/null
+++ b/doc/forum/Portable_version_of_git-annex_for_windows/comment_2_d8d1aa0920351e880ba6678bb97585de._comment
@@ -0,0 +1,10 @@
+[[!comment format=mdwn
+ username="rtwolf"
+ ip="69.165.155.234"
+ subject="comment 2"
+ date="2014-01-07T17:41:25Z"
+ content="""
+Correction to above. Don't run that command, just use git annex sync instead.
+
+Also, git annex drop gives just a blank line. Maybe it doesn't work in direct mode either?
+"""]]
diff --git a/doc/forum/Possible_bug_in_setting_up_Rsync.net_remote.mdwn b/doc/forum/Possible_bug_in_setting_up_Rsync.net_remote.mdwn
new file mode 100644
index 0000000000..21921cd291
--- /dev/null
+++ b/doc/forum/Possible_bug_in_setting_up_Rsync.net_remote.mdwn
@@ -0,0 +1,24 @@
+Here is a line from the debug log...
+
+    [2016-09-08 13:08:37.01053] chat: ssh
+      ["-oNumberOfPasswordPrompts=0","-oStrictHostKeyChecking=no",
+       "9553@git-annex-.usw.2Ds009.2Ersync.2Enet-9553_22_annex",
+       "mkdir -p .ssh;touch .ssh/authorized_keys;dd of=.ssh/authorized_keys oflag=append conv=notrunc;mkdir -p annex"]
+
+The hostname I entered was ordinary: `usw-s009.rsync.net`... but as you can see, the `user@host:port` string is mangled.
+
+I'm using git tag `6.20160907` with changes to `git-annex.cabal` and `stack.yaml` to force use of `concurrent-output-1.7.7` since `1.7.6` had a bug that kept it from building on Windows (I guess?).
+
+Oh, this is on Windows, in case that wasn't clear...
+
+I think the bug is in `${git-annex-root}/Assistant/Ssh.hs` or `${git-annex-root}/Assistant/Pairing/MakeRemote.hs`.  The `.2D` and `.2E` bits in the mangled string make me think that the `-` and `.` characters in my hostname are being replaced by some Haskell representation of those values (`2D` in hexadecimal in ASCII is `-`, `2E` is `.`).
+
+But I've never even written hello world in Haskell so my path ends there.
+
+I'm happy to pull some tag or branch from github and run `stack install` over again and try adding the rsync.net remote again.
+
+I hope this helps!
+
+Cheers,
+
+--Dave
diff --git a/doc/forum/Possible_bug_in_setting_up_Rsync.net_remote/comment_1_dba203d6c701365685f1a0d4269633a3._comment b/doc/forum/Possible_bug_in_setting_up_Rsync.net_remote/comment_1_dba203d6c701365685f1a0d4269633a3._comment
new file mode 100644
index 0000000000..129c430862
--- /dev/null
+++ b/doc/forum/Possible_bug_in_setting_up_Rsync.net_remote/comment_1_dba203d6c701365685f1a0d4269633a3._comment
@@ -0,0 +1,18 @@
+[[!comment format=mdwn
+ username="dave@2ab82f485adf7e2ce787066e35f5f9789bff430b"
+ nickname="dave"
+ subject="output of git annex version"
+ date="2016-09-08T18:48:43Z"
+ content="""
+    $ git annex version
+    git-annex version: 6.20160907-gad0a7f6
+    build flags: Assistant Webapp Pairing Testsuite S3(multipartupload)(storageclasses) WebDAV ConcurrentOutput TorrentParser Feeds Quvi
+    key/value backends: SHA256E SHA256 SHA512E SHA512 SHA224E SHA224 SHA384E SHA384 SHA3_256E SHA3_256 SHA3_512E SHA3_512 SHA3_224E
+                        SHA3_224 SHA3_384E SHA3_384 SKEIN256E SKEIN256 SKEIN512E SKEIN512 SHA1E SHA1 MD5E MD5 WORM URL
+    remote types: git gcrypt S3 bup directory rsync web bittorrent webdav tahoe glacier ddar hook external
+    local repository version: unknown
+    supported repository versions: 5 6
+    upgrade supported from repository versions: 2 3 4 5
+    operating system: mingw32 x86_64
+
+"""]]
diff --git a/doc/forum/Possible_bug_in_setting_up_Rsync.net_remote/comment_2_47b14e71ce0dd653a300dabe8ab0ac3a._comment b/doc/forum/Possible_bug_in_setting_up_Rsync.net_remote/comment_2_47b14e71ce0dd653a300dabe8ab0ac3a._comment
new file mode 100644
index 0000000000..180be858c0
--- /dev/null
+++ b/doc/forum/Possible_bug_in_setting_up_Rsync.net_remote/comment_2_47b14e71ce0dd653a300dabe8ab0ac3a._comment
@@ -0,0 +1,33 @@
+[[!comment format=mdwn
+ username="dave@2ab82f485adf7e2ce787066e35f5f9789bff430b"
+ nickname="dave"
+ subject="Similar issue on Debian machine"
+ date="2016-09-08T20:04:45Z"
+ content="""
+OK, let's try this on a GNU machine (debian, sid) running a git-annex from the official debian repos:
+
+    $ git annex version
+    git-annex version: 6.20160808
+    build flags: Assistant Webapp Pairing Testsuite S3(multipartupload)(storageclasses) WebDAV Inotify DBus DesktopNotify
+                 XMPP ConcurrentOutput TorrentParser MagicMime Feeds Quvi
+    key/value backends: SHA256E SHA256 SHA512E SHA512 SHA224E SHA224 SHA384E SHA384 SHA3_256E SHA3_256 SHA3_512E SHA3_512
+                        SHA3_224E SHA3_224 SHA3_384E SHA3_384 SKEIN256E SKEIN256 SKEIN512E SKEIN512 SHA1E SHA1 MD5E MD5 WORM URL
+    remote types: git gcrypt S3 bup directory rsync web bittorrent webdav tahoe glacier ddar hook external
+    local repository version: 5
+    supported repository versions: 5 6
+    upgrade supported from repository versions: 0 1 2 3 4 5
+    operating system: linux x86_64
+
+Now, this time, the error message in the web app says something ordinary: `Permission denied (publickey,password,keyboard-interactive).`.  However, I see this in the log:
+
+    [2016-09-08 14:56:36.125939] read: ssh-keygen [\"-F\",\"usw-s009.rsync.net\"]
+    [2016-09-08 14:56:36.134026] process done ExitSuccess
+    [2016-09-08 14:56:36.1344] chat: ssh [
+        \"-oNumberOfPasswordPrompts=0\",
+        \"9553@git-annex-.usw.2Ds009.2Ersync.2Enet-9553_22_annex\",
+        \"mkdir -p .ssh;touch .ssh/authorized_keys;dd of=.ssh/authorized_keys oflag=append conv=notrunc;mkdir -p annex\"
+    ]
+    [2016-09-08 14:56:36.71948] process done ExitFailure 255
+
+As you can see, that string mangling is present there too.
+"""]]
diff --git a/doc/forum/Possible_bug_in_setting_up_Rsync.net_remote/comment_3_753ff00c43be5f9bef6bcb529ad68bb9._comment b/doc/forum/Possible_bug_in_setting_up_Rsync.net_remote/comment_3_753ff00c43be5f9bef6bcb529ad68bb9._comment
new file mode 100644
index 0000000000..ddb6db24fd
--- /dev/null
+++ b/doc/forum/Possible_bug_in_setting_up_Rsync.net_remote/comment_3_753ff00c43be5f9bef6bcb529ad68bb9._comment
@@ -0,0 +1,11 @@
+[[!comment format=mdwn
+ username="joey"
+ subject="""comment 3"""
+ date="2016-09-21T17:08:41Z"
+ content="""
+That mangling of the hostname is completely normal.
+
+Your problem description lacked a description of an actual problem.
+Is something not working, or were you put off by this ugly looking
+hostname? If something is not working, what is the error message?
+"""]]
diff --git a/doc/forum/Possible_bug_in_setting_up_Rsync.net_remote/comment_4_43f04bfbaeb4f04a5980ade59a36ae91._comment b/doc/forum/Possible_bug_in_setting_up_Rsync.net_remote/comment_4_43f04bfbaeb4f04a5980ade59a36ae91._comment
new file mode 100644
index 0000000000..6b317e8ea3
--- /dev/null
+++ b/doc/forum/Possible_bug_in_setting_up_Rsync.net_remote/comment_4_43f04bfbaeb4f04a5980ade59a36ae91._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="joey"
+ subject="""comment 4"""
+ date="2016-09-21T17:18:35Z"
+ content="""
+The error message "Permission denied (publickey,password,keyboard-interactive)"
+seems to be saying you entered the wrong password for rsync.net.
+"""]]
diff --git a/doc/forum/Possible_bug_in_setting_up_Rsync.net_remote/comment_5_6dd6819103814056d8b71356804be56a._comment b/doc/forum/Possible_bug_in_setting_up_Rsync.net_remote/comment_5_6dd6819103814056d8b71356804be56a._comment
new file mode 100644
index 0000000000..a27494ca3d
--- /dev/null
+++ b/doc/forum/Possible_bug_in_setting_up_Rsync.net_remote/comment_5_6dd6819103814056d8b71356804be56a._comment
@@ -0,0 +1,12 @@
+[[!comment format=mdwn
+ username="dave@2ab82f485adf7e2ce787066e35f5f9789bff430b"
+ nickname="dave"
+ subject="re: mangling is normal"
+ date="2016-09-21T21:12:16Z"
+ content="""
+... ok, here's the original message from the (web) UI:
+
+    ssh: Could not resolve hostname git-annex-.usw.2ds009.2ersync.2enet-9553_22_annex: Name or service not known
+
+I had figured the debug log might be helpful... Pardon me... :)
+"""]]
diff --git a/doc/forum/Possible_bug_in_setting_up_Rsync.net_remote/comment_6_b90f349f00ea283d018681505b4feaf5._comment b/doc/forum/Possible_bug_in_setting_up_Rsync.net_remote/comment_6_b90f349f00ea283d018681505b4feaf5._comment
new file mode 100644
index 0000000000..1232f3195a
--- /dev/null
+++ b/doc/forum/Possible_bug_in_setting_up_Rsync.net_remote/comment_6_b90f349f00ea283d018681505b4feaf5._comment
@@ -0,0 +1,35 @@
+[[!comment format=mdwn
+ username="dave@2ab82f485adf7e2ce787066e35f5f9789bff430b"
+ nickname="dave"
+ subject="re: regular ssh remote"
+ date="2016-09-21T21:30:24Z"
+ content="""
+I tried adding my rsync.net space as a regular (ssh) remote instead of as an rsync.net remote.  I got this in the webapp:
+
+    Failed to ssh to the server. Transcript: 
+
+(sic)
+
+This time the debug log shows this:
+
+    [2016-09-21 16:19:15.0106417] read: ssh-keygen [\"-F\",\"usw-s009.rsync.net\"]
+    [2016-09-21 16:19:15.2376417] process done ExitSuccess
+    [2016-09-21 16:19:15.2376417] chat: ssh [
+      \"-oNumberOfPasswordPrompts=0\",
+      \"-oStrictHostKeyChecking=yes\",
+      \"-n\",
+      \"-p\",
+      \"22\",
+      \"9553@usw-s009.rsync.net\",
+      \"sh -c 'echo '\\"'\\"'git-annex-probe loggedin'\\"'\\"';if which git-annex-shell; then echo '\\"'\\"'git-annex-probe git-annex-shell'\\"'\\"'; fi;if which git; then echo '\\"'\\"'git-annex-probe git'\\"'\\"'; fi;if which rsync; then echo '\\"'\\"'git-annex-probe rsync'\\"'\\"'; fi;if which ~/.ssh/git-annex-shell; then echo '\\"'\\"'git-annex-probe ~/.ssh/git-annex-shell'\\"'\\"'; fi;if which ~/.ssh/git-annex-wrapper; then echo '\\"'\\"'git-annex-probe ~/.ssh/git-annex-wrapper'\\"'\\"'; fi;cd '\\"'\\"'annex'\\"'\\"' && git config --list'\"]
+    [2016-09-21 16:19:17.4001417] process done ExitFailure 1
+
+
+For what it is worth:
+
+    $ ssh -oNumberOfPasswordPrompts=0 -oStrictHostKeyChecking=yes -n -p 22 9553@usw-s009.rsync.net sh -c 'echo hello'
+
+THAT fails with exit code 1.  But that's just because it's rsync.net, a restricted shell...  (The same command with just 'ls' works!)
+
+I'm back to thinking that it's actually trying to resolve the mangled string.  However, I've been known to be completely wrong before!
+"""]]
diff --git a/doc/forum/Post-Kickstarter.mdwn b/doc/forum/Post-Kickstarter.mdwn
new file mode 100644
index 0000000000..d774ca25b1
--- /dev/null
+++ b/doc/forum/Post-Kickstarter.mdwn
@@ -0,0 +1,5 @@
+We're nearing the end of the year of development funded by Kickstarter. I'm curious to know what the future of the project looks like.
+
+I assume that development will decrease from its current levels as you focus more of your energy on activities that put bread on the table, but is git-annex still something that you foresee actively working on or will it be in more of a feature-freeze-bug-fix stage?
+
+As a backer my only regret is that during the campaign I wasn't able to donate as much as I would have liked. The project has exceeded my expectations since then. I don't know that another Kickstarter campaign is worth the effort, but if you were to ask for donations to fund another month or two or three I would gladly donate. Or perhaps implement a feature-bounty program where users could donate money toward development of a particular feature they would like to see added. Or perhaps you're sick of working for the Internet!
diff --git a/doc/forum/Preferred_content_settings_for_sets_of_backup_drives__63__.mdwn b/doc/forum/Preferred_content_settings_for_sets_of_backup_drives__63__.mdwn
new file mode 100644
index 0000000000..6ddb1e011d
--- /dev/null
+++ b/doc/forum/Preferred_content_settings_for_sets_of_backup_drives__63__.mdwn
@@ -0,0 +1,3 @@
+I have one main 4TB annex and 4 2TB drives that I would like to use for long-term backups. The plan is to have two complete backups, one at home and one in a different state, and swap + update them occasionally. If I put an annex on each drive, how can I tell git annex that every file should be duplicated to one or the other from each set? Like this:
+
+main and (backup1a or backup1b) and (backup2a or backup2b)
diff --git a/doc/forum/Preferred_content_settings_for_sets_of_backup_drives__63__/comment_1_54cdea3a60d61dad6332af31b2968bdd._comment b/doc/forum/Preferred_content_settings_for_sets_of_backup_drives__63__/comment_1_54cdea3a60d61dad6332af31b2968bdd._comment
new file mode 100644
index 0000000000..7f02010dbf
--- /dev/null
+++ b/doc/forum/Preferred_content_settings_for_sets_of_backup_drives__63__/comment_1_54cdea3a60d61dad6332af31b2968bdd._comment
@@ -0,0 +1,16 @@
+[[!comment format=mdwn
+ username="joey"
+ subject="""comment 1"""
+ date="2017-06-06T17:38:55Z"
+ content="""
+Repository groups are a good way to do this. Eg: 
+
+	git annex group backup1a backup1
+	git annex group backup1b backup1
+	git annex group backup2a backup2
+	git annex group backup2b backup2
+	git annex wanted backup1a "not copies=backup1:1"
+	git annex wanted backup1b "not copies=backup1:1"
+	git annex wanted backup2a "not copies=backup2:1"
+	git annex wanted backup2b "not copies=backup2:1"
+"""]]
diff --git a/doc/forum/Preferred_content_settings_for_sets_of_backup_drives__63__/comment_2_c21e128f6bdd5408607b51c963854d4d._comment b/doc/forum/Preferred_content_settings_for_sets_of_backup_drives__63__/comment_2_c21e128f6bdd5408607b51c963854d4d._comment
new file mode 100644
index 0000000000..d85c266099
--- /dev/null
+++ b/doc/forum/Preferred_content_settings_for_sets_of_backup_drives__63__/comment_2_c21e128f6bdd5408607b51c963854d4d._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="jeff"
+ avatar="http://cdn.libravatar.org/avatar/6f3f360b92a588e1b0999bbd4707ce12"
+ subject="comment 2"
+ date="2017-06-07T21:24:36Z"
+ content="""
+Thanks! I missed that you can put group names in the \"copies=\" part.
+"""]]
diff --git a/doc/forum/Preserving_Directories_in_Metadata_Views.mdwn b/doc/forum/Preserving_Directories_in_Metadata_Views.mdwn
new file mode 100644
index 0000000000..dfc45cb4b6
--- /dev/null
+++ b/doc/forum/Preserving_Directories_in_Metadata_Views.mdwn
@@ -0,0 +1,47 @@
+I want to use metadata views to sort files into top-level directories based on a tag, but then preserve the directory structure underneath that. I'm having trouble with this.
+
+Say I have an annex at `~/annex` with a structure like this:
+
+    $ tree
+    .
+    ├── foo
+    │   └── bar
+    │       ├── one.txt
+    │       ├── three.txt
+    │       └── two.txt
+    └── waldo
+        └── fred
+            ├── a.txt
+            ├── b.txt
+            └── c.txt
+
+I tag some of the files with `blah`:
+
+    $ git annex metadata -t blah foo/bar/*
+
+Now I want to change my view to only see those files with a certain tag, but I want to maintain their directory structure, ie I want to end up with something like this:
+
+    $ tree
+    .
+    ├── blah
+    │   └── foo
+    │       └── bar
+    │           ├── one.txt
+    │           ├── three.txt
+    │           └── two.txt
+
+If I do `git annex view blah` I see the files `one.txt`, `two.txt` and `three.txt` but they are in the top level of `~/annex`. The `foo` and `bar` directories are not present.
+
+If I do `git annex view blah "/=*"` then the files I present under the `foo` directory, but the `bar` subdirectory is not there.
+
+It would also be fine if I could just hide the files that did not have the `blah` tag, so that I ended up with this:
+
+    $ tree
+    .
+    ├── foo
+    │   └── bar
+    │       ├── one.txt
+    │       ├── three.txt
+    │       └── two.txt
+
+Is something like this possible?
diff --git a/doc/forum/Preserving_extended_attributes.mdwn b/doc/forum/Preserving_extended_attributes.mdwn
new file mode 100644
index 0000000000..5533daeea9
--- /dev/null
+++ b/doc/forum/Preserving_extended_attributes.mdwn
@@ -0,0 +1,5 @@
+Hey,
+
+I was wondering if it is currently possible to let the assistant (or git-annex in general) preserve extended attributes. I didn't find any options hinting at this, although it should be possible at least in theory by using the metadata system of git-annex...
+
+Considering that some applications use extended attributes to store custom meta data (like tags etc.), I think it would be valuable to have such an option...
diff --git a/doc/forum/Preserving_file_access_rights_in_directory_tree_below_objects__47__.mdwn b/doc/forum/Preserving_file_access_rights_in_directory_tree_below_objects__47__.mdwn
new file mode 100644
index 0000000000..d89bcee290
--- /dev/null
+++ b/doc/forum/Preserving_file_access_rights_in_directory_tree_below_objects__47__.mdwn
@@ -0,0 +1,77 @@
+Hello,
+
+I have the problem that, while git-annex preserves the file access rights (user, group, others) for the actual file, it does not make sure that others can access this file through the directory tree above said file:
+
+    /tmp $ mkdir test
+    /tmp $ chown claudius:media test
+    /tmp $ chmod 750 test
+    /tmp $ ls -dl test
+    drwxr-x--- 2 claudius media 40 2012-01-23 19:27 test/
+    /tmp $ cd test
+    /tmp/test $ git init --shared=all
+    Initialized empty shared Git repository in /tmp/test/.git/
+    /tmp/test $ git annex init "test"
+    init test ok
+    /tmp/test $ echo 123 > abc
+    /tmp/test $ chmod 640 abc
+    /tmp/test $ chown claudius:media abc
+    /tmp/test $ ls -l
+    total 4
+    -rw-r----- 1 claudius media 4 2012-01-23 19:27 abc
+    /tmp/test $ git annex add .
+    add abc (checksum...) ok
+    (Recording state in git...)
+    /tmp/test $ ls -l
+    total 4
+    lrwxrwxrwx 1 claudius claudius 176 2012-01-23 19:27 abc -> .git/annex/objects/8F/pj/SHA256-s4--181210f8f9c779c26da1d9b2075bde0127302ee0e3fca38c9a83f5b1dd8e5d3b/SHA256-s4--181210f8f9c779c26da1d9b2075bde0127302ee0e3fca38c9a83f5b1dd8e5d3b
+    /tmp/test $ ls -l .git/annex/objects/8F/pj/SHA256-s4--181210f8f9c779c26da1d9b2075bde0127302ee0e3fca38c9a83f5b1dd8e5d3b/SHA256-s4--181210f8f9c779c26da1d9b2075bde0127302ee0e3fca38c9a83f5b1dd8e5d3b
+    -r--r----- 1 claudius media 4 2012-01-23 19:27 .git/annex/objects/8F/pj/SHA256-s4--181210f8f9c779c26da1d9b2075bde0127302ee0e3fca38c9a83f5b1dd8e5d3b/SHA256-s4--181210f8f9c779c26da1d9b2075bde0127302ee0e3fca38c9a83f5b1dd8e5d3b
+    /tmp/test $ ls -lR .git/annex/objects/
+    .git/annex/objects/:
+    total 0
+    drwx--S--- 3 claudius claudius 60 2012-01-23 19:28 8F/
+    
+    .git/annex/objects/8F:
+    total 0
+    drwx--S--- 3 claudius claudius 60 2012-01-23 19:28 pj/
+    
+    .git/annex/objects/8F/pj:
+    total 0
+    dr-x--S--- 2 claudius claudius 60 2012-01-23 19:28 SHA256-s4--181210f8f9c779c26da1d9b2075bde0127302ee0e3fca38c9a83f5b1dd8e5d3b/
+    
+    .git/annex/objects/8F/pj/SHA256-s4--181210f8f9c779c26da1d9b2075bde0127302ee0e3fca38c9a83f5b1dd8e5d3b:
+    total 4
+    -r--r----- 1 claudius media 4 2012-01-23 19:27 SHA256-s4--181210f8f9c779c26da1d9b2075bde0127302ee0e3fca38c9a83f5b1dd8e5d3b
+    /tmp/test $ stat .git/annex/objects/
+      File: `.git/annex/objects/'
+      Size: 60        	Blocks: 0          IO Block: 4096   directory
+    Device: 11h/17d	Inode: 2365970     Links: 3
+    Access: (2700/drwx--S---)  Uid: ( 1000/claudius)   Gid: ( 1000/claudius)
+    Access: 2012-01-23 19:28:10.614948386 +0100
+    Modify: 2012-01-23 19:28:10.614948386 +0100
+    Change: 2012-01-23 19:28:10.614948386 +0100
+     Birth: -
+
+The use case is that I have a rather large collection of music I would like to manage with git-annex in various locations (all of it on my external hard drive, some on my notebook etc. This music is played by MPD, which can access the collection because it is in the "media" group. After changing to git-annex, however, this fails.
+
+I tried to avoid this specific problem by declaring the git repository to be shared, which does appear to have some effect on the other files in .git:
+
+    /tmp/test $ ls -l .git
+    total 16
+    drwx--S---  5 claudius claudius 160 2012-01-23 19:28 annex/
+    drwxrwsr-x  2 claudius claudius  40 2012-01-23 19:27 branches/
+    -rw-rw-r--  1 claudius claudius 218 2012-01-23 19:27 config
+    -rw-rw-r--  1 claudius claudius  73 2012-01-23 19:27 description
+    -rw-rw-r--  1 claudius claudius  23 2012-01-23 19:27 HEAD
+    drwxrwsr-x  2 claudius claudius 220 2012-01-23 19:27 hooks/
+    -rw-rw-r--  1 claudius claudius 104 2012-01-23 19:28 index
+    drwxrwsr-x  2 claudius claudius  60 2012-01-23 19:27 info/
+    drwxrwsr-x  3 claudius claudius  60 2012-01-23 19:27 logs/
+    drwxrwsr-x 15 claudius claudius 300 2012-01-23 19:28 objects/
+    drwxrwsr-x  4 claudius claudius  80 2012-01-23 19:27 refs/
+
+I could obviously try to change the rights of annex/, annex/objects etc., but I would like to avoid having to adapt them each time a new folder is added somewhere below annex/objects/.
+
+My knowledge of git and especially git-annex is not too good, so it might well be that I missed something obvious. Any hints? :)
+
+(And thank you, of course, for taking the time to read all this)
diff --git a/doc/forum/Preserving_file_access_rights_in_directory_tree_below_objects__47__/comment_1_5dd978f9b5a0771f44ab9e086bf5a07f._comment b/doc/forum/Preserving_file_access_rights_in_directory_tree_below_objects__47__/comment_1_5dd978f9b5a0771f44ab9e086bf5a07f._comment
new file mode 100644
index 0000000000..d2da5e94df
--- /dev/null
+++ b/doc/forum/Preserving_file_access_rights_in_directory_tree_below_objects__47__/comment_1_5dd978f9b5a0771f44ab9e086bf5a07f._comment
@@ -0,0 +1,14 @@
+[[!comment format=mdwn
+ username="http://joey.kitenet.net/"
+ nickname="joey"
+ subject="comment 1"
+ date="2012-01-23T19:00:40Z"
+ content="""
+You say you started the repo with \"git init --shared\" .. but what that's really meant for is bare repositories, which can have several users pushing into it, not a non-bare repository.
+
+The strange mode on the directories \"dr-x--S---\" and files \"-r--r-----\" must be due to your umask setting though. My umask is 022 and the directories and files under `.git/annex/objects` are \"drwxr-xr-x\" and \"-r--r--r--\", which allows anyone to read them unless an upper directory blocks it -- and with this umask, none do unless I explicitly remove permissions from one to lock down a repository.
+
+About mpd, the obvious fix is to run mpd not as a system user but as yourself. I put \"@reboot mpd\" in my crontab to do this.
+
+
+"""]]
diff --git a/doc/forum/Preserving_file_access_rights_in_directory_tree_below_objects__47__/comment_2_9f51947b35ee04e473655e20d56c740a._comment b/doc/forum/Preserving_file_access_rights_in_directory_tree_below_objects__47__/comment_2_9f51947b35ee04e473655e20d56c740a._comment
new file mode 100644
index 0000000000..5d632ab860
--- /dev/null
+++ b/doc/forum/Preserving_file_access_rights_in_directory_tree_below_objects__47__/comment_2_9f51947b35ee04e473655e20d56c740a._comment
@@ -0,0 +1,16 @@
+[[!comment format=mdwn
+ username="https://www.google.com/accounts/o8/id?id=AItOawlB7-aXsqwzOi2BIR_Q4sUF8sjj24H6F3c"
+ nickname="Claudius"
+ subject="comment 2"
+ date="2012-01-23T19:39:17Z"
+ content="""
+Thank you for your comment! Indeed, setting the umask to, for example, 022 has the desired effect that annex/objects etc. are executable (and in this special case also writable), my previous umask setting was 077; the \"strange\" permissions on the git directories was probably due to --shared=all, and the mode of \"440\" on the files within the git-annex tree is correct (the original file was 640 and stripped of its write permission).
+
+Using this umask setting and newgrp to switch the default group, I was successfully able to set up the repositories.
+
+However, I would like to suggest adding the execute bit to the directories below .git/annex/objects/ per default, even if the umask of the current shell differs. As the correct rights are already preserved in the actual files (minus their write permission) together with correct owner and group, the files are still protected the same way as previously, and because +x does not allow directory listings, no additional information can leak out either. Not having to set the umask to something \"sensible\" before operating git-annex would be a huge plus, too :)
+
+The reason why I am not running MPD as my user is that I am a bit wary of running an application even exposed to the local network as my main user, and I see nothing wrong with running it as its own user.
+
+Thank you again for your help and the time you put into this project!
+"""]]
diff --git a/doc/forum/Previous_versions_in_direct_mode__63__.mdwn b/doc/forum/Previous_versions_in_direct_mode__63__.mdwn
new file mode 100644
index 0000000000..2958346e73
--- /dev/null
+++ b/doc/forum/Previous_versions_in_direct_mode__63__.mdwn
@@ -0,0 +1,3 @@
+I am in Windows and therefore forced to use direct mode. After reading for a long while I still have not found a way of "git annex get" previous versions. Is this supported? is it only the last version in the repository?
+
+any chance to use indirect mode in Windows? In theory Windows supports NTFS symlinks, why is this not supported?
diff --git a/doc/forum/Previous_versions_in_direct_mode__63__/comment_1_352d460acd5500587e679d934180eee4._comment b/doc/forum/Previous_versions_in_direct_mode__63__/comment_1_352d460acd5500587e679d934180eee4._comment
new file mode 100644
index 0000000000..b81e6c85e4
--- /dev/null
+++ b/doc/forum/Previous_versions_in_direct_mode__63__/comment_1_352d460acd5500587e679d934180eee4._comment
@@ -0,0 +1,10 @@
+[[!comment format=mdwn
+ username="http://joeyh.name/"
+ ip="209.250.56.227"
+ subject="comment 1"
+ date="2013-12-24T20:56:00Z"
+ content="""
+git-annex for windows is targeting being used with msysgit, and msysgit does not seem to support symlinks. That's one good reason. Since cywin's git does support symlinks, it might be possible to get git-annex to use it and support indirect mode.
+
+Doing anything about this is probably \"patches accepted\" territory for me. I'm mostly interested in porting git-annex to Windows so it can be used on computers owned by non-technical users, to interoperate with git-annex on more capable operating systems.
+"""]]
diff --git a/doc/forum/Problem_compiling_current_master.mdwn b/doc/forum/Problem_compiling_current_master.mdwn
new file mode 100644
index 0000000000..d79dcd0b4f
--- /dev/null
+++ b/doc/forum/Problem_compiling_current_master.mdwn
@@ -0,0 +1,12 @@
+While trying to compile the current master I run in the following error:
+
+    Utility/Yesod.hs:21:14:
+        Couldn't match expected type `String'
+                    with actual type `WidgetFileSettings'
+        Expected type: String -> Q Exp
+          Actual type: WidgetFileSettings -> FilePath -> Q Exp
+        In the expression: widgetFileNoReload
+        In an equation for `widgetFile': widgetFile = widgetFileNoReload
+    make: *** [git-annex] Fehler 1
+
+I installed all dependencies from the INSTALL document. What is wrong?
diff --git a/doc/forum/Problem_compiling_current_master/comment_1_135df61ec850c06e3b48ccfef7b5b031._comment b/doc/forum/Problem_compiling_current_master/comment_1_135df61ec850c06e3b48ccfef7b5b031._comment
new file mode 100644
index 0000000000..676cc697d6
--- /dev/null
+++ b/doc/forum/Problem_compiling_current_master/comment_1_135df61ec850c06e3b48ccfef7b5b031._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="http://joeyh.name/"
+ ip="4.152.246.16"
+ subject="comment 1"
+ date="2012-08-29T15:51:15Z"
+ content="""
+I think you're building with the Makefile, not with cabal, and have the recent yesod 1.1.0 release installed. The Makefile contains a -DWITH_OLD_YESOD that you need to remove in that situation.
+"""]]
diff --git a/doc/forum/Problem_compiling_current_master/comment_2_fb3e27b6014e84bd919a7a4a95e39ef9._comment b/doc/forum/Problem_compiling_current_master/comment_2_fb3e27b6014e84bd919a7a4a95e39ef9._comment
new file mode 100644
index 0000000000..61c6f3be67
--- /dev/null
+++ b/doc/forum/Problem_compiling_current_master/comment_2_fb3e27b6014e84bd919a7a4a95e39ef9._comment
@@ -0,0 +1,20 @@
+[[!comment format=mdwn
+ username="https://www.google.com/accounts/o8/id?id=AItOawmU_2tE75oyG0h2ZPN4lcroIKEMC8G-otE"
+ nickname="Michael"
+ subject="comment 2"
+ date="2012-08-30T11:32:24Z"
+ content="""
+That helped now the compile stops at:
+
+    Utility/WebApp.hs:106:9:
+        Ambiguous occurrence `liftIO'
+        It could refer to either `Yesod.liftIO',
+                                 imported from `Yesod' at Utility/WebApp.hs:14:1-12
+                                 (and originally defined in `transformers-0.2.2.0:Control.Monad.IO.Class')
+                              or `Control.Monad.IO.Class.liftIO',
+                                 imported from `Control.Monad.IO.Class' at Utility/WebApp.hs:18:1-29
+    make: *** [git-annex] Fehler 1
+
+
+
+"""]]
diff --git a/doc/forum/Problem_compiling_current_master/comment_3_b737b3945103c5e2aa798b4e65fbce06._comment b/doc/forum/Problem_compiling_current_master/comment_3_b737b3945103c5e2aa798b4e65fbce06._comment
new file mode 100644
index 0000000000..12c4b1af08
--- /dev/null
+++ b/doc/forum/Problem_compiling_current_master/comment_3_b737b3945103c5e2aa798b4e65fbce06._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="http://joeyh.name/"
+ ip="4.152.246.16"
+ subject="comment 3"
+ date="2012-08-30T17:07:44Z"
+ content="""
+I've fixed this.. I think. I don't have this problem with the version of yesod I have here.
+"""]]
diff --git a/doc/forum/Problem_compiling_current_master/comment_4_28c1b335ae388d4e1f22b711ac1c001f._comment b/doc/forum/Problem_compiling_current_master/comment_4_28c1b335ae388d4e1f22b711ac1c001f._comment
new file mode 100644
index 0000000000..d16e17c139
--- /dev/null
+++ b/doc/forum/Problem_compiling_current_master/comment_4_28c1b335ae388d4e1f22b711ac1c001f._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="https://www.google.com/accounts/o8/id?id=AItOawmU_2tE75oyG0h2ZPN4lcroIKEMC8G-otE"
+ nickname="Michael"
+ subject="comment 4"
+ date="2012-08-31T08:04:18Z"
+ content="""
+Success! Now it compiles fine on Ubuntu 12.04 with make. Thank you.
+"""]]
diff --git a/doc/forum/Problem_with_corrupt_SQLite_DB.mdwn b/doc/forum/Problem_with_corrupt_SQLite_DB.mdwn
new file mode 100644
index 0000000000..7cc2a5ddfb
--- /dev/null
+++ b/doc/forum/Problem_with_corrupt_SQLite_DB.mdwn
@@ -0,0 +1,17 @@
+Apparently, I managed to corrupt an SQLite DB when `git-annex get` ran out of space.  This prevents git-annex operations from
+working at all now.
+
+For example:
+
+    % git annex find --in .
+    sqlite worker thread crashed: SQLite3 returned ErrorError while attempting to perform prepare "SELECT null from content limit 1": no such table: content
+    git-annex: sqlite query crashed
+
+One of the few mentions of SQLite being used is in incremental fsck
+([[design/caching_database]]).  I did run incremental fsck in
+another repository a few days before this.  The fsck finished
+without issues, so I'd be happy with a solution that involves simply
+deleting the DB or something.
+
+- git-annex version: 6.20160527-gb7d4774
+- All repositories/remotes: version 5, indirect
diff --git a/doc/forum/Problem_with_corrupt_SQLite_DB/comment_1_4e10e6d1882feba8d0c1ad058e01143c._comment b/doc/forum/Problem_with_corrupt_SQLite_DB/comment_1_4e10e6d1882feba8d0c1ad058e01143c._comment
new file mode 100644
index 0000000000..ecb3206ee4
--- /dev/null
+++ b/doc/forum/Problem_with_corrupt_SQLite_DB/comment_1_4e10e6d1882feba8d0c1ad058e01143c._comment
@@ -0,0 +1,9 @@
+[[!comment format=mdwn
+ username="joey"
+ subject="""comment 1"""
+ date="2016-05-31T15:53:32Z"
+ content="""
+Delete .git/annex/keys/db
+
+This database is not really needed unless your repository is in v6 mode.
+"""]]
diff --git a/doc/forum/Problem_with_corrupt_SQLite_DB/comment_2_98020b857e5b52ebbbaaeba1313f2697._comment b/doc/forum/Problem_with_corrupt_SQLite_DB/comment_2_98020b857e5b52ebbbaaeba1313f2697._comment
new file mode 100644
index 0000000000..d85b6a16e7
--- /dev/null
+++ b/doc/forum/Problem_with_corrupt_SQLite_DB/comment_2_98020b857e5b52ebbbaaeba1313f2697._comment
@@ -0,0 +1,7 @@
+[[!comment format=mdwn
+ username="gernot"
+ subject="comment 2"
+ date="2016-06-02T18:33:59Z"
+ content="""
+Perfect, thank you!
+"""]]
diff --git a/doc/forum/Problem_with_corrupt_SQLite_DB/comment_3_079359513b6b9fcf0030f49df0eaf64d._comment b/doc/forum/Problem_with_corrupt_SQLite_DB/comment_3_079359513b6b9fcf0030f49df0eaf64d._comment
new file mode 100644
index 0000000000..1bc48c8586
--- /dev/null
+++ b/doc/forum/Problem_with_corrupt_SQLite_DB/comment_3_079359513b6b9fcf0030f49df0eaf64d._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="avar"
+ avatar="http://cdn.libravatar.org/avatar/57332d67a86eb51e06bf78d2baf42c3c"
+ subject="comment 3"
+ date="2017-06-16T20:37:47Z"
+ content="""
+What about if you are in v6 mode? I just read the wiki about that version upgrade, sounds exciting, but I got this with 
+# this is really for the run time, you can set these settings in /etc/sysctl.conf
+sudo sysctl -w kern.maxproc=2048
+sudo sysctl -w kern.maxprocperuid=1024
+
+# tell launchd about having higher limits
+sudo echo \"limit maxfiles 1024 unlimited\" >> /etc/launchd.conf
+sudo echo \"limit maxproc 1024 2048\" >> /etc/launchd.conf
+
    
+
+There are other system limits which you can check by doing a \"ulimit -a\", once you make the above changes, you will need to reboot to make the changes take affect. I am unsure if the above will help as it is an example of what I did on 10.6.6 a few months ago to fix some forking issues. From the error you got you will probably need to increase the stacksize to something bigger or even make it unlimited if you feel lucky, the default stacksize on OSX is 8192, try making it say 10times that size first and see what happens.
+"""]]
diff --git a/doc/forum/Problems_with_large_numbers_of_files/comment_2_0392a11219463e40c53bae73c8188b69._comment b/doc/forum/Problems_with_large_numbers_of_files/comment_2_0392a11219463e40c53bae73c8188b69._comment
new file mode 100644
index 0000000000..8ea5531f43
--- /dev/null
+++ b/doc/forum/Problems_with_large_numbers_of_files/comment_2_0392a11219463e40c53bae73c8188b69._comment
@@ -0,0 +1,25 @@
+[[!comment format=mdwn
+ username="http://joey.kitenet.net/"
+ nickname="joey"
+ subject="comment 2"
+ date="2011-04-05T17:46:03Z"
+ content="""
+This message comes from ghc's runtime memory manager. Apparently your ghc defaults to limiting the stack to 80 mb.
+Mine seems to limit it slightly higher -- I have seen haskell programs successfully grow as large as 350 mb, although generally not intentionally. :)
+
+Here's how to adjust the limit at runtime, obviously you'd want a larger number:
+
+
    +# git-annex +RTS -K100 -RTS find
+Stack space overflow: current size 100 bytes.
+Use `+RTS -Ksize -RTS' to increase it.
+
    
+
+I've tried to avoid git-annex using quantities of memory that scale with the number of files in the repo, and I think in general successfully -- I run it on 32 mb and 128 mb machines, FWIW. There are some tricky cases, and haskell makes it easy to accidentally write code that uses much more memory than would be expected.
+
+One well known case is `git annex unused`, which *has* to build a structure of every annexed file. I have been considering using a bloom filter or something to avoid that.
+
+Another possible case is when running a command like `git annex add`, and passing it a lot of files/directories. Some code tries to preserve the order of your input after passing it through `git ls-files` (which destroys ordering), and to do so it needs to buffer both the input and the result in ram.
+
+It's possible to build git-annex with memory profiling and generate some quite helpful profiling data. Edit the Makefile and add this to GHCFLAGS: `-prof -auto-all -caf-all -fforce-recomp` then when running git-annex, add the parameters: `+RTS -p -RTS` , and look for the git-annex.prof file.
+"""]]
diff --git a/doc/forum/Problems_with_large_numbers_of_files/comment_3_537e9884c1488a7a4bcf131ea63b71f7._comment b/doc/forum/Problems_with_large_numbers_of_files/comment_3_537e9884c1488a7a4bcf131ea63b71f7._comment
new file mode 100644
index 0000000000..8e4101e37e
--- /dev/null
+++ b/doc/forum/Problems_with_large_numbers_of_files/comment_3_537e9884c1488a7a4bcf131ea63b71f7._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="http://joey.kitenet.net/"
+ nickname="joey"
+ subject="comment 3"
+ date="2011-04-05T18:02:05Z"
+ content="""
+Oh, you'll need profiling builds of various haskell libraries to build with profiling support. If that's not easily accomplished, if you could show me the form of the command you're running, and also how git annex unannex fails, that would be helpful for investigating.
+"""]]
diff --git a/doc/forum/Problems_with_large_numbers_of_files/comment_4_7cb65d013e72bd2b7e90452079d42ac9._comment b/doc/forum/Problems_with_large_numbers_of_files/comment_4_7cb65d013e72bd2b7e90452079d42ac9._comment
new file mode 100644
index 0000000000..bac9fd7cad
--- /dev/null
+++ b/doc/forum/Problems_with_large_numbers_of_files/comment_4_7cb65d013e72bd2b7e90452079d42ac9._comment
@@ -0,0 +1,29 @@
+[[!comment format=mdwn
+ username="https://www.google.com/accounts/o8/id?id=AItOawkhdKAhe3l_UyGt5SdfRBPYVwe-9f8P2dM"
+ nickname="Justin"
+ subject="comment 4"
+ date="2011-04-05T21:14:12Z"
+ content="""
+@joey
+
+OK, I'll try increasing the stack size and see if that helps.
+
+For reference, I was running:
+
+git annex add .
+
+on a directory containing about 100k files spread over many nested subdirectories. I actually have more than a dozen projects like this that I plan to keep in git annex, possibly in separate repositories if necessary. I could probably tar the data and then archive that, but I like the idea of being able to see the structure of my data even though the contents of the files are on a different machine.
+
+After the crash, running:
+
+git annex unannex
+
+does nothing and returns instantly. What exactly is 'git annex add' doing? I know that it's moving files into the key-value store and adding symlinks, but I don't know what else it does.
+
+--Justin
+
+
+
+If 
+
+"""]]
diff --git a/doc/forum/Problems_with_large_numbers_of_files/comment_5_86a42ee3173a5d38f803e64b79496ab3._comment b/doc/forum/Problems_with_large_numbers_of_files/comment_5_86a42ee3173a5d38f803e64b79496ab3._comment
new file mode 100644
index 0000000000..7dcccef2e5
--- /dev/null
+++ b/doc/forum/Problems_with_large_numbers_of_files/comment_5_86a42ee3173a5d38f803e64b79496ab3._comment
@@ -0,0 +1,14 @@
+[[!comment format=mdwn
+ username="http://joey.kitenet.net/"
+ nickname="joey"
+ subject="comment 5"
+ date="2011-04-07T16:41:00Z"
+ content="""
+I think what is happening with \"git annex unannex\" is that \"git annex add\" crashes before it can \"git add\" the symlinks. unannex only looks at files that \"git ls-files\" shows, and so files that are not added to git are not seen. So, this can be recovered from by looking at git status and manually adding the symlinks to git, and then unannex.
+
+That also suggests that \"git annex add .\" has done something before crashing. That's consistent with you passing it < 2 parameters; it's not just running out of memory trying to expand and preserve order of its parameters (like it might if you ran \"git annex add experiment-1/ experiment-2/\")
+
+I'm pretty sure I know where the space leak is now. git-annex builds up a queue of git commands, so that it can run git a minimum number of times. Currently, this queue is only flushed at the end. I had been meaning to work on having it flush the queue periodically to avoid it growing without bounds, and I will prioritize doing that.
+
+(The only other thing that \"git annex add\" does is record location log information.)
+"""]]
diff --git a/doc/forum/Problems_with_large_numbers_of_files/comment_6_4551274288383c9cc27cbf85b122d307._comment b/doc/forum/Problems_with_large_numbers_of_files/comment_6_4551274288383c9cc27cbf85b122d307._comment
new file mode 100644
index 0000000000..fff8f7cdde
--- /dev/null
+++ b/doc/forum/Problems_with_large_numbers_of_files/comment_6_4551274288383c9cc27cbf85b122d307._comment
@@ -0,0 +1,11 @@
+[[!comment format=mdwn
+ username="http://joey.kitenet.net/"
+ nickname="joey"
+ subject="comment 6"
+ date="2011-04-07T18:09:13Z"
+ content="""
+I've committed the queue flush improvements, so it will buffer up to 10240 git actions, and then flush the queue.
+
+There may be other memory leaks at scale (besides the two I mentioned earlier), but this seems promising. I'm well into running `git annex add` on a half million files and it's using 18 mb ram and has flushed the queue several times. This run
+will fail due to running out of inodes for the log files, not due to memory. :)
+"""]]
diff --git a/doc/forum/Problems_with_large_numbers_of_files/comment_7_d18cf944352f8303799c86f2c0354e8e._comment b/doc/forum/Problems_with_large_numbers_of_files/comment_7_d18cf944352f8303799c86f2c0354e8e._comment
new file mode 100644
index 0000000000..7d2ad5eba7
--- /dev/null
+++ b/doc/forum/Problems_with_large_numbers_of_files/comment_7_d18cf944352f8303799c86f2c0354e8e._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="https://www.google.com/accounts/o8/id?id=AItOawl9sYlePmv1xK-VvjBdN-5doOa_Xw-jH4U"
+ nickname="Richard"
+ subject="comment 7"
+ date="2011-04-08T21:55:36Z"
+ content="""
+http://xfs.org/index.php/XFS_FAQ#Q:_Performance:_mkfs.xfs_-n_size.3D64k_option
+"""]]
diff --git a/doc/forum/Proper_usage_of_git_annex_proxy_to_mimc_undo_--depth.mdwn b/doc/forum/Proper_usage_of_git_annex_proxy_to_mimc_undo_--depth.mdwn
new file mode 100644
index 0000000000..1f15d66553
--- /dev/null
+++ b/doc/forum/Proper_usage_of_git_annex_proxy_to_mimc_undo_--depth.mdwn
@@ -0,0 +1,15 @@
+[Thanks Joey for correcting the docs on `git annex undo --depth`, and thanks for the `git-revert` suggestion as a replacement!](http://git-annex.branchable.com/direct_mode/#comment-b6dcfc80842008e7f9f5b8f612b27867)
+
+**Context: Creating an OSX GUI for assistant managed direct mode repos to help with restoring old file versions.**
+
+I saw this in the `git revert` docs and thought that `git annex proxy -- git checkout annex/direct/master~$depth -- $filename` might best suit my needs of restoring a previous version of a file. (I liked the idea of presenting the user with a depth rather than a hash.)
+
+>Note: git revert is used to record some new commits to reverse the effect of some earlier commits (often only a faulty one). If you want to throw away all uncommitted changes in your working directory, you should see git-reset[1], particularly the --hard option. If you want to extract specific files as they were in another commit, you should see git-checkout[1], specifically the git checkout -- syntax. Take care with these alternatives as both will discard uncommitted changes in your working directory.
+
+What I've found is that your suggestion of `git revert` is nice because it wouldn't create a conflict, as `git checkout` does. 
+
+So annex, thorough as it is, creates a `$filename.variant-local.$ext` file after the auto conflict resolution to preserve the original. `git revert` is neater, history wise, because there is no conflict as git knows exactly what's changing and from whence it came, rather than just some new file content showing up from who knows where with `git checkout`. 
+
+The issue it seems is that `git revert` works on a commit basis, while `git checkout` can operate on files. If I'm correct in this it would be good to know if annex uses one commit per file, for sure, every time? If this is the case, there would be no problem using the better in most every other way `git revert`.
+
+Though, I'm still not clear how to use the "depth" referencing with `git revert` rather than hashes, any suggestions?
diff --git a/doc/forum/Pruning_out_unwanted_Git_objects.mdwn b/doc/forum/Pruning_out_unwanted_Git_objects.mdwn
new file mode 100644
index 0000000000..36397b2670
--- /dev/null
+++ b/doc/forum/Pruning_out_unwanted_Git_objects.mdwn
@@ -0,0 +1,3 @@
+I have a backups repository with a few files in it, that at one point had some huge filesets erroneously added to it.  As a result, even though there are only 23,334 annexed files, the number of non-dangling Git objects in the repository comes to 593,584.
+
+Normally I would use `git filter-branch` to clear out the deadwood in situations like this, since it is a completely private repository.  What I'm wondering is, is any such thing possible with git-annex, or is the best option just to start over, copy all the files into the new repository, and then `git-add` them all?
diff --git a/doc/forum/Pruning_out_unwanted_Git_objects/comment_1_0cf7a12bfa2957260f4b2f79b0cadf2f._comment b/doc/forum/Pruning_out_unwanted_Git_objects/comment_1_0cf7a12bfa2957260f4b2f79b0cadf2f._comment
new file mode 100644
index 0000000000..fbf538afa5
--- /dev/null
+++ b/doc/forum/Pruning_out_unwanted_Git_objects/comment_1_0cf7a12bfa2957260f4b2f79b0cadf2f._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="https://me.yahoo.com/a/2grhJvAC049fJnvALDXek.6MRZMTlg--#eec89"
+ nickname="John"
+ subject="comment 1"
+ date="2013-08-29T07:23:13Z"
+ content="""
+Maybe one way to solve this that would be general is to have some kind of `prune-history` command, which keeps only the HEAD and drops everything else.  Because there are some repositories that I want to manage with `git-annex` for many reasons, but I don't care about keep history around at all.
+"""]]
diff --git a/doc/forum/Pruning_out_unwanted_Git_objects/comment_2_7472943c02cfe2808b0d566e06caa1a5._comment b/doc/forum/Pruning_out_unwanted_Git_objects/comment_2_7472943c02cfe2808b0d566e06caa1a5._comment
new file mode 100644
index 0000000000..0c26463514
--- /dev/null
+++ b/doc/forum/Pruning_out_unwanted_Git_objects/comment_2_7472943c02cfe2808b0d566e06caa1a5._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="https://me.yahoo.com/a/2grhJvAC049fJnvALDXek.6MRZMTlg--#eec89"
+ nickname="John"
+ subject="comment 2"
+ date="2013-08-30T06:18:42Z"
+ content="""
+This was answered quite thoroughly in:http://git-annex.branchable.com/forum/safely_dropping_git-annex_history/
+"""]]
diff --git a/doc/forum/Pruning_out_unwanted_Git_objects/comment_3_6a1e7a83d94394454fc085f6d2728cd7._comment b/doc/forum/Pruning_out_unwanted_Git_objects/comment_3_6a1e7a83d94394454fc085f6d2728cd7._comment
new file mode 100644
index 0000000000..90951961f3
--- /dev/null
+++ b/doc/forum/Pruning_out_unwanted_Git_objects/comment_3_6a1e7a83d94394454fc085f6d2728cd7._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="http://joeyh.name/"
+ ip="2001:4830:1600:187::2"
+ subject="comment 3"
+ date="2013-09-04T06:36:15Z"
+ content="""
+`git annex forget` automates this now. Needs a version of git-annex supporting it installed on *all* the computers you use the repo on.
+"""]]
diff --git a/doc/forum/Purge_a_remote.mdwn b/doc/forum/Purge_a_remote.mdwn
new file mode 100644
index 0000000000..f1b04ead2d
--- /dev/null
+++ b/doc/forum/Purge_a_remote.mdwn
@@ -0,0 +1,2 @@
+How could I delete and purge a remote? I want to remove all traces of it
+but I can't find out how to achieve this.
diff --git a/doc/forum/Purge_a_remote/comment_1_78b3b77f457c65d31fd8a5abf714905d._comment b/doc/forum/Purge_a_remote/comment_1_78b3b77f457c65d31fd8a5abf714905d._comment
new file mode 100644
index 0000000000..a7eff3a99c
--- /dev/null
+++ b/doc/forum/Purge_a_remote/comment_1_78b3b77f457c65d31fd8a5abf714905d._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="http://joeyh.name/"
+ ip="209.250.56.47"
+ subject="comment 1"
+ date="2013-11-05T16:17:22Z"
+ content="""
+In the git-annex webapp, each repository has a settings menu with \"delete repository\".
+"""]]
diff --git a/doc/forum/Purge_a_remote/comment_2_dc65719157dee63b3979563ed57ee0ce._comment b/doc/forum/Purge_a_remote/comment_2_dc65719157dee63b3979563ed57ee0ce._comment
new file mode 100644
index 0000000000..86827c529a
--- /dev/null
+++ b/doc/forum/Purge_a_remote/comment_2_dc65719157dee63b3979563ed57ee0ce._comment
@@ -0,0 +1,10 @@
+[[!comment format=mdwn
+ username="https://www.google.com/accounts/o8/id?id=AItOawkzwmw_zyMpZC9_J7ey--woeYPoZkAOgGw"
+ nickname="dxtrish"
+ subject="comment 2"
+ date="2013-11-05T17:46:02Z"
+ content="""
+When I use that Delete button it seems to still have records of it
+because when I delete the repo on the other end and create a new bare
+one it complains about some UUID mismatch.
+"""]]
diff --git a/doc/forum/Purge_a_remote/comment_3_63e0280273b816fa4b837724e102f813._comment b/doc/forum/Purge_a_remote/comment_3_63e0280273b816fa4b837724e102f813._comment
new file mode 100644
index 0000000000..621bda5085
--- /dev/null
+++ b/doc/forum/Purge_a_remote/comment_3_63e0280273b816fa4b837724e102f813._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="http://joeyh.name/"
+ ip="209.250.56.47"
+ subject="comment 3"
+ date="2013-11-05T18:03:26Z"
+ content="""
+Hmm well, it's true that deleting a repository does not clean out any git remotes that other repositories might have configured using it, and the uuid of the old remote is cached in there. This is only a problem if you put back another repository in the same location as the old one. `git remote remove` should clean that up for you.
+"""]]
diff --git a/doc/forum/Purge_a_remote/comment_4_7fad1c4798ca03a4095ac3241c279f6d._comment b/doc/forum/Purge_a_remote/comment_4_7fad1c4798ca03a4095ac3241c279f6d._comment
new file mode 100644
index 0000000000..afb4dc92e3
--- /dev/null
+++ b/doc/forum/Purge_a_remote/comment_4_7fad1c4798ca03a4095ac3241c279f6d._comment
@@ -0,0 +1,11 @@
+[[!comment format=mdwn
+ username="https://www.google.com/accounts/o8/id?id=AItOawkzwmw_zyMpZC9_J7ey--woeYPoZkAOgGw"
+ nickname="dxtrish"
+ subject="comment 4"
+ date="2013-11-06T14:24:21Z"
+ content="""
+That is exactly what I have wanted to do a couple of times now and 'git remote remove' does not seem to remove it all either.
+When I try to create a new repository in the same place it still complains about that darn UUID :)
+
+What is working is to manually add the new UUID to .git/config
+"""]]
diff --git a/doc/forum/Purge_whereis.mdwn b/doc/forum/Purge_whereis.mdwn
new file mode 100644
index 0000000000..d5b614898b
--- /dev/null
+++ b/doc/forum/Purge_whereis.mdwn
@@ -0,0 +1,9 @@
+Hey there,
+
+I am playing around with git-annex to handle my files and backup's. Sometimes, I wish to delete a file entirely from my local repo and any other repo that contains the file.
+
+First, I drop the file from my local repo and run git-annex sync --content. Then I run git annex whereis and drop the file from other repo's. Subsequently, I run git annex whereis to ensure that the file has been dropped and does not exist in any repo.
+
+However, git annex still tracks the file, but I just have 0 copies of it. How do I remove/delete/disable tracking of a single file or multiple files?
+
+The reason why I have deleted the file is that I don't want it in any of my repo's any longer. 
diff --git a/doc/forum/Purge_whereis/comment_1_89d2446e3c9a58ab7b2a85ff941f7c8d._comment b/doc/forum/Purge_whereis/comment_1_89d2446e3c9a58ab7b2a85ff941f7c8d._comment
new file mode 100644
index 0000000000..848d1b75bb
--- /dev/null
+++ b/doc/forum/Purge_whereis/comment_1_89d2446e3c9a58ab7b2a85ff941f7c8d._comment
@@ -0,0 +1,9 @@
+[[!comment format=mdwn
+ username="cbaines"
+ subject="comment 1"
+ date="2015-12-21T12:32:55Z"
+ content="""
+If you want to stop tracking a file, just delete it, and then run git annex sync.
+
+There are some other commands (and settings in the assistant) for managing unused content. That is, files in the annex that are no longer referenced/used.
+"""]]
diff --git a/doc/forum/Purge_whereis/comment_2_0307ce47de9b719adc2096600d94dff6._comment b/doc/forum/Purge_whereis/comment_2_0307ce47de9b719adc2096600d94dff6._comment
new file mode 100644
index 0000000000..66e7bd6f15
--- /dev/null
+++ b/doc/forum/Purge_whereis/comment_2_0307ce47de9b719adc2096600d94dff6._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="frost.kristian@75a6b6a25121f985cd8708f98c691d41716ac720"
+ nickname="frost.kristian"
+ subject="comment 2"
+ date="2015-12-21T13:08:49Z"
+ content="""
+It seems I have to enable syncing from the webapp. I thought doing git annex sync would do the same?
+"""]]
diff --git a/doc/forum/Purge_whereis/comment_3_e87b53ebbf8053bb17caeb2aed871212._comment b/doc/forum/Purge_whereis/comment_3_e87b53ebbf8053bb17caeb2aed871212._comment
new file mode 100644
index 0000000000..bdfe711f22
--- /dev/null
+++ b/doc/forum/Purge_whereis/comment_3_e87b53ebbf8053bb17caeb2aed871212._comment
@@ -0,0 +1,7 @@
+[[!comment format=mdwn
+ username="cbaines"
+ subject="comment 3"
+ date="2015-12-21T13:33:34Z"
+ content="""
+I just did a quick test, running git annex sync does commit the removal of files for me.
+"""]]
diff --git a/doc/forum/Purge_whereis/comment_4_72b6c28d8a4865ad8c84b1f7dfefb78a._comment b/doc/forum/Purge_whereis/comment_4_72b6c28d8a4865ad8c84b1f7dfefb78a._comment
new file mode 100644
index 0000000000..0e0cab11ce
--- /dev/null
+++ b/doc/forum/Purge_whereis/comment_4_72b6c28d8a4865ad8c84b1f7dfefb78a._comment
@@ -0,0 +1,10 @@
+[[!comment format=mdwn
+ username="frost.kristian@75a6b6a25121f985cd8708f98c691d41716ac720"
+ nickname="frost.kristian"
+ subject="comment 4"
+ date="2015-12-21T17:47:13Z"
+ content="""
+I did some digging. When creating a repo from the webapp, it is per default in direct mode. In direct mode, I am not able to run any git commands such as git commit. Subsequently, when running git annex sync. Files added when sync is disabled from the webapp are not committed.
+
+Is git-annex behaving as expected here?
+"""]]
diff --git a/doc/forum/Purge_whereis/comment_5_966e0285639a41ee9db3e9fb337e2699._comment b/doc/forum/Purge_whereis/comment_5_966e0285639a41ee9db3e9fb337e2699._comment
new file mode 100644
index 0000000000..7dbe3d3a9a
--- /dev/null
+++ b/doc/forum/Purge_whereis/comment_5_966e0285639a41ee9db3e9fb337e2699._comment
@@ -0,0 +1,11 @@
+[[!comment format=mdwn
+ username="joey"
+ subject="""comment 5"""
+ date="2016-01-01T20:35:18Z"
+ content="""
+Described behavior sounds right to me. If you've clicked on "syncing
+enabled" in the webapp for the local repository, so it's changed to
+"syncing disabled" then the webapp won't automatically commit file
+deletions/adds. Click on it again and it'll catch up with any changes you
+made while it was disabled.
+"""]]
diff --git a/doc/forum/Push__47__Pull_with_the_Assistant.mdwn b/doc/forum/Push__47__Pull_with_the_Assistant.mdwn
new file mode 100644
index 0000000000..315ad38d85
--- /dev/null
+++ b/doc/forum/Push__47__Pull_with_the_Assistant.mdwn
@@ -0,0 +1 @@
+If I use git-annex with a centralized bare git repository as [described here](http://git-annex.branchable.com/tips/centralized_git_repository_tutorial/), will the Assistant automatically `git push` and `git pull` the master and git-annex branches? Or does it basically just do a `git annex sync`?
diff --git a/doc/forum/Push__47__Pull_with_the_Assistant/comment_1_f7b63d379c2d21794adf8658f546f8a7._comment b/doc/forum/Push__47__Pull_with_the_Assistant/comment_1_f7b63d379c2d21794adf8658f546f8a7._comment
new file mode 100644
index 0000000000..0477218bc6
--- /dev/null
+++ b/doc/forum/Push__47__Pull_with_the_Assistant/comment_1_f7b63d379c2d21794adf8658f546f8a7._comment
@@ -0,0 +1,10 @@
+[[!comment format=mdwn
+ username="http://joeyh.name/"
+ ip="4.252.11.120"
+ subject="comment 1"
+ date="2012-11-17T20:25:03Z"
+ content="""
+The assistant will push, pull, and merge the master and git-annex branches (which is exactly what git annex sync does). 
+
+Additionally, the assistant will transfer file contents to the bare repository, either to make it have a copy of everything, or as needed to transfer to other repositories that also use that bare repository. This behavior is configurable using the webapp.
+"""]]
diff --git a/doc/forum/Push__47__Pull_with_the_Assistant/comment_2_aec8cc20576e7ffd5a8be4348d1a0073._comment b/doc/forum/Push__47__Pull_with_the_Assistant/comment_2_aec8cc20576e7ffd5a8be4348d1a0073._comment
new file mode 100644
index 0000000000..20be4c6ce3
--- /dev/null
+++ b/doc/forum/Push__47__Pull_with_the_Assistant/comment_2_aec8cc20576e7ffd5a8be4348d1a0073._comment
@@ -0,0 +1,21 @@
+[[!comment format=mdwn
+ username="annexuser"
+ ip="85.159.236.214"
+ subject="comment 2"
+ date="2012-11-17T20:44:53Z"
+ content="""
+When I make a change in the repository and run a plain old `git annex sync`, it pushes out to any other available annex, but not to my bare hub. If I run `git annex sync` a second time it tells me that I'm ahead of the hub:
+
+    commit
+    # On branch master
+    # Your branch is ahead of 'hub/master' by 1 commit.
+    #
+    nothing to commit, working directory clean
+    ok
+
+In order to push the changes out to the hub I have to do `git push --all`.
+
+The same thing with pulling. If I make a change on the annex, push it to the hub, and then go to a second annex that knows about the hub but not the first annex, a `git annex sync` does not pull the changes in from the hub. I have to do `git pull` to get them.
+
+Do I have something setup incorrectly?
+"""]]
diff --git a/doc/forum/Pusher_crashed__44___attempt_to_repair_hangs__44___broken_symlinks.mdwn b/doc/forum/Pusher_crashed__44___attempt_to_repair_hangs__44___broken_symlinks.mdwn
new file mode 100644
index 0000000000..4c1bcc129b
--- /dev/null
+++ b/doc/forum/Pusher_crashed__44___attempt_to_repair_hangs__44___broken_symlinks.mdwn
@@ -0,0 +1,77 @@
+Sorry that I put all this in the same thread but I don't know what happened and how it is related.
+
+I have just a simple setup: git-annex client with assistant (Windows 7) and on a server (Debian, no assistant).
+
+Suddenly weird things started to happen
+
+1.) On Windows, when I start the assistant, it writes "Attempting to repair THINKTANK:c:\data\annex [here]" but it runs forever and never stops
+
+2.) On Windows, when I get "Pusher crashed: failed to read sha from git write-tree [Restart Thread]". When I click "Restart Thread" nothing happens but the message from (1) persists.
+
+3.) When I run "git annex fsck" on the client I get thousands of messages like
+
+    fsck Fotos/2014/DSC_0303.JPG
+      ** No known copies exist of Fotos/2014/DSC_0303.JPG
+    failed
+
+Here the same:
+
+    $ git annex whereis "Fotos/2014/DSC_0303.JPG"
+    whereis Fotos/2014/DSC_0303.JPG (0 copies) failed
+    git-annex: whereis: 1 failed
+
+4.) When I do "git annex status" a whole bunch of files are displayed with "M" (modified) although they are not, they are not even checked out and should be only at the server ...
+
+5.) On the server, files that should ALWAYS be on the server (configured as "full backup") suddenly wiped data that was also made available on the client. The symlinks are dangling symlinks and contain just binary data:
+
+    ls -l
+    lrwxrwxrwx  1 4 Aug  2 08:55 DSC_0011.JPG -> ????
+    lrwxrwxrwx  1 4 Aug  2 08:55 DSC_0012.JPG -> ????
+    lrwxrwxrwx  1 4 Aug  2 08:55 DSC_0013.JPG -> ????
+    lrwxrwxrwx  1 4 Aug  2 08:55 DSC_0014.JPG -> ????
+    lrwxrwxrwx  1 4 Aug  2 08:55 DSC_0015.JPG -> ????
+    lrwxrwxrwx  1 4 Aug  2 08:55 DSC_0018.JPG -> ????
+    lrwxrwxrwx  1 4 Aug  2 08:55 DSC_0019.JPG -> ????
+    lrwxrwxrwx  1 4 Aug  2 08:55 DSC_0020.JPG -> ????
+
+6.) "git annex fsck" on the server is still successful, returning no errors!
+
+7.) Manually executing "git annex sync --content" on both sides does not change anything and does not output any error messages.
+
+8.) On the client:
+
+    $ git annex group here
+    error: invalid object 100644 3b3767ae65e5c6d2e3835af3d55fbf2f9e145c8b for '000/0e6/SHA256Es193806--b6d4689fba8e15acd6497f9a7e584c93ea0c8c2199ad32eadac79d59b9f49814.JPG.log'
+    fatal: git-write-tree: error building trees
+    manual
+    (Recording state in git...)
+    git-annex: failed to read sha from git write-tree
+
+    $ git annex wanted here
+    error: invalid object 100644 3b3767ae65e5c6d2e3835af3d55fbf2f9e145c8b for '000/0e6/SHA256Es193806--b6d4689fba8e15acd6497f9a7e584c93ea0c8c2199ad32eadac79d59b9f49814.JPG.log'
+    fatal: git-write-tree: error building trees
+    exclude="*" and present
+    git-annex: failed to read sha from git write-tree
+
+9.) Ok I don't know what happened I did nothing special but it seems that the repository is broken :( :(
+
+    $ git annex --verbose --debug repair
+    [...]
+    [2014-08-02 13:27:38 Pacific Daylight Time] read: git ["--git-dir=C:\\Data\\annex\\.git","--work-tree=C:\\Data\\annex","-c","core.bare=false","show","ef3fe549f457783dbbd877b467b4e54b0ebc813c"]
+    Running git fsck ...
+    
+    git-annex: DeleteFile "C:\\Data\\annex\\.git\\objects\\2a\\54bb281c80c91ea7a732c0d48db0c5acc0ca2c": permission denied (Access is denied.)
+    failed
+    git-annex: repair: 1 failed
+
+But this file exists, I can read, write and delete to this file manually, there is definitely no permission denied ...
+
+
+
+Oh no, so desparate :-( Any ideas?
+
+As it seems the client repository is broken but how can it be then that also files on the server repository get deleted which shouldn't be deleted?
+And how can it be that there are not only broken symlinks but symlinks that have just binary garbage as target and "fsck" returns success?
+
+(I am happy to share all log files privately but I do not want to publish them here because they contain sensitive data)
+
diff --git a/doc/forum/Pusher_crashed__44___attempt_to_repair_hangs__44___broken_symlinks/comment_1_c89b64b0dc7c5a760a84a9d2cfd8982c._comment b/doc/forum/Pusher_crashed__44___attempt_to_repair_hangs__44___broken_symlinks/comment_1_c89b64b0dc7c5a760a84a9d2cfd8982c._comment
new file mode 100644
index 0000000000..38bd4daf8c
--- /dev/null
+++ b/doc/forum/Pusher_crashed__44___attempt_to_repair_hangs__44___broken_symlinks/comment_1_c89b64b0dc7c5a760a84a9d2cfd8982c._comment
@@ -0,0 +1,11 @@
+[[!comment format=mdwn
+ username="divB"
+ ip="128.12.90.218"
+ subject="comment 1"
+ date="2014-09-12T02:47:37Z"
+ content="""
+I try it once again - any hints for me?
+I haven't done anything since then because I do not know how to proceed here (except creating everything new from scratch).
+
+Is more information needed? If yes, what?
+"""]]
diff --git a/doc/forum/Pusher_crashed__44___attempt_to_repair_hangs__44___broken_symlinks/comment_2_c511691c4ea6c77e63ef85c28c921be9._comment b/doc/forum/Pusher_crashed__44___attempt_to_repair_hangs__44___broken_symlinks/comment_2_c511691c4ea6c77e63ef85c28c921be9._comment
new file mode 100644
index 0000000000..3ca5ec6d2c
--- /dev/null
+++ b/doc/forum/Pusher_crashed__44___attempt_to_repair_hangs__44___broken_symlinks/comment_2_c511691c4ea6c77e63ef85c28c921be9._comment
@@ -0,0 +1,12 @@
+[[!comment format=mdwn
+ username="joey"
+ subject="""comment 2"""
+ date="2015-02-09T20:56:15Z"
+ content="""
+You see "-> ???" for symlink targets, and you are unable to delete
+files that you own, and git is unable to read objects that should be in the
+repository.
+
+This all suggests that your disk is broken. You need to check your
+filesystem for corruption, etc.
+"""]]
diff --git a/doc/forum/Pushing_an_existing_git_annex_repo_to_a_remote_ssh_host.mdwn b/doc/forum/Pushing_an_existing_git_annex_repo_to_a_remote_ssh_host.mdwn
new file mode 100644
index 0000000000..f453c59e29
--- /dev/null
+++ b/doc/forum/Pushing_an_existing_git_annex_repo_to_a_remote_ssh_host.mdwn
@@ -0,0 +1 @@
+I have an existing repo on my local machine, with a mirror that I sync on the flash drive as well, using a remote, as described in the walkthrough. I'd like to create another repo on my remote server and have that work as a mirror as well, using ssh and rsync too. The only resources I've found regarding creating a remote over ssh imply that the repo is already on the host, is this a necessary step in creating a ssh remote? I was thinking in the terms of creating a git annex repository, adding a remote, then pushing it much like how git works, yet I can't find anything like that in the documentation.
diff --git a/doc/forum/Pushing_an_existing_git_annex_repo_to_a_remote_ssh_host/comment_1_4008b735809161d5950b7991d69d3f68._comment b/doc/forum/Pushing_an_existing_git_annex_repo_to_a_remote_ssh_host/comment_1_4008b735809161d5950b7991d69d3f68._comment
new file mode 100644
index 0000000000..90a212f360
--- /dev/null
+++ b/doc/forum/Pushing_an_existing_git_annex_repo_to_a_remote_ssh_host/comment_1_4008b735809161d5950b7991d69d3f68._comment
@@ -0,0 +1,12 @@
+[[!comment format=mdwn
+ username="joey"
+ subject="""comment 1"""
+ date="2017-10-03T17:16:34Z"
+ content="""
+It works just like git works, since a git-annex repository is a git
+repository. Set up the git repository on the remote host and then add it as
+a remote.
+
+In order for git-annex to be able to store the content of annexed files on
+the remote host, git-annex will need to be installed on that host.
+"""]]
diff --git a/doc/forum/Pushing_git_repo_to_AWS_S3_from_behind_proxy.mdwn b/doc/forum/Pushing_git_repo_to_AWS_S3_from_behind_proxy.mdwn
new file mode 100644
index 0000000000..03676502ba
--- /dev/null
+++ b/doc/forum/Pushing_git_repo_to_AWS_S3_from_behind_proxy.mdwn
@@ -0,0 +1,9 @@
+I want to setup my remote git repo on S3
+Referred to
+http://git-annex.branchable.com/tips/using_Amazon_S3/
+http://git-annex.branchable.com/special_remotes/S3/
+
+I need support for AWS.config param proxy_uri
+http://docs.amazonwebservices.com/AWSRubySDK/latest/AWS.html
+
+Can some one help?
diff --git a/doc/forum/Question__58___git-annex-only_repo___63__.mdwn b/doc/forum/Question__58___git-annex-only_repo___63__.mdwn
new file mode 100644
index 0000000000..e6b5ab6636
--- /dev/null
+++ b/doc/forum/Question__58___git-annex-only_repo___63__.mdwn
@@ -0,0 +1,3 @@
+Is there a way to declare (init) a git-annex-only repository?
+
+I mean if the repository will only contain large (or maybe binary) files without any source-code... In that case all files added to the repo should be managed by git-annex automatically. Is this currently possible?
diff --git a/doc/forum/Question__58___git-annex-only_repo___63__/comment_10_ada27f7b8c7dc3c388e6c56ef095e682._comment b/doc/forum/Question__58___git-annex-only_repo___63__/comment_10_ada27f7b8c7dc3c388e6c56ef095e682._comment
new file mode 100644
index 0000000000..b0fe83dd62
--- /dev/null
+++ b/doc/forum/Question__58___git-annex-only_repo___63__/comment_10_ada27f7b8c7dc3c388e6c56ef095e682._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="boustanihani@93604388a6a2b8df16f3e34157232801a67c1021"
+ nickname="boustanihani"
+ subject="comment 10"
+ date="2015-07-06T22:00:36Z"
+ content="""
+Thanks :) Just one last question, how what happens when 2 users edit the same file in parallel, how does git-annex deal with this by default? Does the newer one overwrite the old one or do we get some info like \"unable to auto-merge\"...
+"""]]
diff --git a/doc/forum/Question__58___git-annex-only_repo___63__/comment_11_531e6a091c8d005626e4e8290ba317d7._comment b/doc/forum/Question__58___git-annex-only_repo___63__/comment_11_531e6a091c8d005626e4e8290ba317d7._comment
new file mode 100644
index 0000000000..7f485a418f
--- /dev/null
+++ b/doc/forum/Question__58___git-annex-only_repo___63__/comment_11_531e6a091c8d005626e4e8290ba317d7._comment
@@ -0,0 +1,12 @@
+[[!comment format=mdwn
+ username="joey"
+ subject="""comment 11"""
+ date="2015-07-06T22:13:24Z"
+ content="""
+If using git-annex sync or git-annex merge to do the merging, it will
+automatically resolve the merge conflict and put both conflicting versions
+of the file in the tree under different filenames.
+
+Of course, you can also use regular git commands to pull and merge your
+master branch; then you'd get a regular git merge conflict.
+"""]]
diff --git a/doc/forum/Question__58___git-annex-only_repo___63__/comment_1_321632cc794c00181932f5737db9aff0._comment b/doc/forum/Question__58___git-annex-only_repo___63__/comment_1_321632cc794c00181932f5737db9aff0._comment
new file mode 100644
index 0000000000..b69abb3bdf
--- /dev/null
+++ b/doc/forum/Question__58___git-annex-only_repo___63__/comment_1_321632cc794c00181932f5737db9aff0._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="Marco"
+ subject="Yes"
+ date="2015-06-23T12:12:52Z"
+ content="""
+Yes, you can do this. Simply set up a git annex repo and you are ready to go.
+If you have the assistant running in the background it will take care of committing and distributing the files around.
+"""]]
diff --git a/doc/forum/Question__58___git-annex-only_repo___63__/comment_2_941b62a63c11caa1ada5d544c2e926ca._comment b/doc/forum/Question__58___git-annex-only_repo___63__/comment_2_941b62a63c11caa1ada5d544c2e926ca._comment
new file mode 100644
index 0000000000..fdde45be61
--- /dev/null
+++ b/doc/forum/Question__58___git-annex-only_repo___63__/comment_2_941b62a63c11caa1ada5d544c2e926ca._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="boustanihani@93604388a6a2b8df16f3e34157232801a67c1021"
+ nickname="boustanihani"
+ subject="comment 2"
+ date="2015-06-23T12:42:46Z"
+ content="""
+you mean there is no need for track then ?
+"""]]
diff --git a/doc/forum/Question__58___git-annex-only_repo___63__/comment_3_0c0dad1ffa11f2f8708906d310a189db._comment b/doc/forum/Question__58___git-annex-only_repo___63__/comment_3_0c0dad1ffa11f2f8708906d310a189db._comment
new file mode 100644
index 0000000000..d61393dc8f
--- /dev/null
+++ b/doc/forum/Question__58___git-annex-only_repo___63__/comment_3_0c0dad1ffa11f2f8708906d310a189db._comment
@@ -0,0 +1,11 @@
+[[!comment format=mdwn
+ username="Marco"
+ subject="comment 3"
+ date="2015-06-24T07:23:28Z"
+ content="""
+What do you mean by track?
+
+If you are using the assistant, it will take care for you. It will add, commit and distribute files.
+
+If you are using the command line without the assistant, you should have a look at the [[walkthrough]]. You can play around with repositories in /tmp before you push all your important data into it ;)
+"""]]
diff --git a/doc/forum/Question__58___git-annex-only_repo___63__/comment_4_426160c2237db790b133b9c9f0abd4b2._comment b/doc/forum/Question__58___git-annex-only_repo___63__/comment_4_426160c2237db790b133b9c9f0abd4b2._comment
new file mode 100644
index 0000000000..ff771d16cc
--- /dev/null
+++ b/doc/forum/Question__58___git-annex-only_repo___63__/comment_4_426160c2237db790b133b9c9f0abd4b2._comment
@@ -0,0 +1,10 @@
+[[!comment format=mdwn
+ username="boustanihani@93604388a6a2b8df16f3e34157232801a67c1021"
+ nickname="boustanihani"
+ subject="comment 4"
+ date="2015-06-24T10:14:18Z"
+ content="""
+Thanks I'll take a look at git-annex assistant, all new to me :)
+
+Never mind about track, I thought git-annex is similar to git-lfs where you should manually call \"track...\" on files whose content should not be added to git...
+"""]]
diff --git a/doc/forum/Question__58___git-annex-only_repo___63__/comment_5_36bd0a398b385e65df7eda26fa5c50d0._comment b/doc/forum/Question__58___git-annex-only_repo___63__/comment_5_36bd0a398b385e65df7eda26fa5c50d0._comment
new file mode 100644
index 0000000000..fe9c9714f8
--- /dev/null
+++ b/doc/forum/Question__58___git-annex-only_repo___63__/comment_5_36bd0a398b385e65df7eda26fa5c50d0._comment
@@ -0,0 +1,7 @@
+[[!comment format=mdwn
+ username="anarcat"
+ subject="comment 5"
+ date="2015-06-24T12:14:02Z"
+ content="""
+you are correct. \"git add\" *will* add a file into git directly. \"git annex add\" is like \"git track\"...
+"""]]
diff --git a/doc/forum/Question__58___git-annex-only_repo___63__/comment_6_32fb6930bcaac36d00b088d8ee09face._comment b/doc/forum/Question__58___git-annex-only_repo___63__/comment_6_32fb6930bcaac36d00b088d8ee09face._comment
new file mode 100644
index 0000000000..ce10012ee9
--- /dev/null
+++ b/doc/forum/Question__58___git-annex-only_repo___63__/comment_6_32fb6930bcaac36d00b088d8ee09face._comment
@@ -0,0 +1,14 @@
+[[!comment format=mdwn
+ username="boustanihani@93604388a6a2b8df16f3e34157232801a67c1021"
+ nickname="boustanihani"
+ subject="comment 6"
+ date="2015-06-24T14:06:35Z"
+ content="""
+cool thanks :)
+
+Is it also possible to version control large binary files using git-annex assistant ??
+
+I mean to be able to switch back to earlier versions when needed...
+
+Is there something like a way to specify how  many older versions should be kept in the repo? This would be a nice feature for preserving disk space...
+"""]]
diff --git a/doc/forum/Question__58___git-annex-only_repo___63__/comment_7_8278ee493da14aac86e928d2349d8173._comment b/doc/forum/Question__58___git-annex-only_repo___63__/comment_7_8278ee493da14aac86e928d2349d8173._comment
new file mode 100644
index 0000000000..56aae7d1d3
--- /dev/null
+++ b/doc/forum/Question__58___git-annex-only_repo___63__/comment_7_8278ee493da14aac86e928d2349d8173._comment
@@ -0,0 +1,9 @@
+[[!comment format=mdwn
+ username="anarcat"
+ subject="somewhat"
+ date="2015-06-24T16:28:42Z"
+ content="""
+git-annex doesn't deliberately keep all versions of the files. however, if you change a file (using [[git-annex-edit]]), it will keep the old copy, which will become [[walkthrough/unused_data/]]. that data will be purged by the assistant after a while (if configured to do so) but you can also archive it, etc... 
+
+so in a way, yes, it is versionned, but the unused data is somewhat more 'brittle' than the current dataset.
+"""]]
diff --git a/doc/forum/Question__58___git-annex-only_repo___63__/comment_8_3e24c9f9042d6abdafaae45015645b11._comment b/doc/forum/Question__58___git-annex-only_repo___63__/comment_8_3e24c9f9042d6abdafaae45015645b11._comment
new file mode 100644
index 0000000000..0011ebb203
--- /dev/null
+++ b/doc/forum/Question__58___git-annex-only_repo___63__/comment_8_3e24c9f9042d6abdafaae45015645b11._comment
@@ -0,0 +1,10 @@
+[[!comment format=mdwn
+ username="boustanihani@93604388a6a2b8df16f3e34157232801a67c1021"
+ nickname="boustanihani"
+ subject="comment 8"
+ date="2015-06-25T21:13:01Z"
+ content="""
+Is it possible to force git-annex to keep all versions for some or all files??
+
+Or (maybe better) specify how many old versions to keep (for example keep the last 20 versions or so...) ?
+"""]]
diff --git a/doc/forum/Question__58___git-annex-only_repo___63__/comment_9_182c66e98086dbfde29b615d0cf0e8e4._comment b/doc/forum/Question__58___git-annex-only_repo___63__/comment_9_182c66e98086dbfde29b615d0cf0e8e4._comment
new file mode 100644
index 0000000000..b9432bb03c
--- /dev/null
+++ b/doc/forum/Question__58___git-annex-only_repo___63__/comment_9_182c66e98086dbfde29b615d0cf0e8e4._comment
@@ -0,0 +1,19 @@
+[[!comment format=mdwn
+ username="joey"
+ subject="""comment 9"""
+ date="2015-07-06T18:47:23Z"
+ content="""
+If you use indirect mode, and avoid passing --force to any git-annex
+commands, then git-annex will ensure that it always keeps all versions of
+all files.
+
+Any command that would delete any data requires the --force option.
+
+If your repo uses direct mode, we can't make this guarantee, because
+the files are left directly in your work tree and you can edit or delete
+them manually and there's nothing git-annex can do to stop that.
+
+There is not currently a way to keep a particular number of old versions of
+a file. This comes down to picking the versions to keep or drop when
+running `git annex dropunused` (with --force of course).
+"""]]
diff --git a/doc/forum/Reappearing_repos_in_webapp_and_vicfg.mdwn b/doc/forum/Reappearing_repos_in_webapp_and_vicfg.mdwn
new file mode 100644
index 0000000000..41a64630d4
--- /dev/null
+++ b/doc/forum/Reappearing_repos_in_webapp_and_vicfg.mdwn
@@ -0,0 +1,43 @@
+Hi,
+
+Whenever I do a git annex vicfg (as pasted below) the long gone repo 40da403e-6f74-4705-aae0-433aa656b55e keeps automatically being added back into the configuration.  Worse in the webapp this manifests itself as a repository list item that's empty of text (and when buttons are clicked there's a complaint about missing UUIDs).
+
+In addition to removing the lines via git-annex vicfg, I've switched my repository into indirect mode, manually checked out the the git-annex branch and deleted the offending key from the preferred content and group files (it oddly didn't appear in the uuid file).  However they keep coming back :-(
+
+Everything seems to be okay bar this rogue entry which in the web-app irrationally bothers me.
+
+I'm running the very latest git-annex release.
+
+Any thoughts?
+
+Cheers,
+
+Matt.
+
+
+    # git-annex configuration
+    #
+    # Changes saved to this file will be recorded in the git-annex branch.
+    #
+    # Lines in this file have the format:
+    #   setting uuid = value
+    
+    # Repository trust configuration
+    # (Valid trust levels: trusted semitrusted untrusted dead)
+    # (for mattford63 (matt@descartes.rss.mhs.man.ac.uk:~/current))
+    trust 88733342-45ef-420f-8f1b-9a5ed8d8e070 = semitrusted
+    
+    # Repository groups
+    # (Standard groups: client transfer backup incrementalbackup smallarchive archive source manual public unwanted)
+    # (Separate group names with spaces)
+    # (for )
+    group 40da403e-6f74-4705-aae0-433aa656b55e = client
+    # (for mattford63 (matt@descartes.rss.mhs.man.ac.uk:~/current))
+    group 88733342-45ef-420f-8f1b-9a5ed8d8e070 = client
+
+    # Repository preferred contents
+    # (for )
+    content 40da403e-6f74-4705-aae0-433aa656b55e = standard
+    # (for mattford63 (matt@descartes.rss.mhs.man.ac.uk:~/current))
+    content 88733342-45ef-420f-8f1b-9a5ed8d8e070 = standard
+    
diff --git a/doc/forum/Reappearing_repos_in_webapp_and_vicfg/comment_1_bd977e864ae89816fa7f4ff69879b15f._comment b/doc/forum/Reappearing_repos_in_webapp_and_vicfg/comment_1_bd977e864ae89816fa7f4ff69879b15f._comment
new file mode 100644
index 0000000000..983dd8478c
--- /dev/null
+++ b/doc/forum/Reappearing_repos_in_webapp_and_vicfg/comment_1_bd977e864ae89816fa7f4ff69879b15f._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="http://joeyh.name/"
+ ip="4.154.4.193"
+ subject="comment 1"
+ date="2013-07-02T16:56:42Z"
+ content="""
+Run: `git annex dead 40da403e-6f74-4705-aae0-433aa656b55e`
+"""]]
diff --git a/doc/forum/Reappearing_repos_in_webapp_and_vicfg/comment_2_05749f9e75689d0111339b7126c12300._comment b/doc/forum/Reappearing_repos_in_webapp_and_vicfg/comment_2_05749f9e75689d0111339b7126c12300._comment
new file mode 100644
index 0000000000..bd56c6670a
--- /dev/null
+++ b/doc/forum/Reappearing_repos_in_webapp_and_vicfg/comment_2_05749f9e75689d0111339b7126c12300._comment
@@ -0,0 +1,15 @@
+[[!comment format=mdwn
+ username="https://www.google.com/accounts/o8/id?id=AItOawnXybLxkPMYpP3yw4b_I6IdC3cKTD-xEdU"
+ nickname="Matt"
+ subject="comment 2"
+ date="2013-07-02T19:48:19Z"
+ content="""
+Hi,
+
+I should have mentioned I can't do this either:
+
+    ~/current $ git annex dead 40da403e-6f74-4705-aae0-433aa656b55e
+    dead 40da403e-6f74-4705-aae0-433aa656b55e git-annex: there is no available git remote named \"40da403e-6f74-4705-aae0-433aa656b55e\"
+
+I can't quite remember how I removed the remote - it was a whilst back, I think maybe I just edited it out of the git config file....
+"""]]
diff --git a/doc/forum/Reappearing_repos_in_webapp_and_vicfg/comment_3_b1531994eea0fbbf4cb097e604378a53._comment b/doc/forum/Reappearing_repos_in_webapp_and_vicfg/comment_3_b1531994eea0fbbf4cb097e604378a53._comment
new file mode 100644
index 0000000000..a079e3b123
--- /dev/null
+++ b/doc/forum/Reappearing_repos_in_webapp_and_vicfg/comment_3_b1531994eea0fbbf4cb097e604378a53._comment
@@ -0,0 +1,12 @@
+[[!comment format=mdwn
+ username="http://joeyh.name/"
+ ip="4.154.4.193"
+ subject="comment 3"
+ date="2013-07-02T19:58:30Z"
+ content="""
+It seems that the uuid is present in the group.log and preferred-content.log files in the git-annex branch, but not in the uuid.log file. I missed that in your description before. That's weird, don't know how that could happen.
+
+However, when I manually configure things this way, I do not see the repository in the webapp. Which makes sense; uuid.log is the canonical list of repositories.
+
+I guess if I were you I'd manually edit the uuid into uuid.log and then mark it dead.
+"""]]
diff --git a/doc/forum/Reappearing_repos_in_webapp_and_vicfg/comment_4_f1eba3e8aa4116e3c20747ec1d6e24e5._comment b/doc/forum/Reappearing_repos_in_webapp_and_vicfg/comment_4_f1eba3e8aa4116e3c20747ec1d6e24e5._comment
new file mode 100644
index 0000000000..6c5e8835e8
--- /dev/null
+++ b/doc/forum/Reappearing_repos_in_webapp_and_vicfg/comment_4_f1eba3e8aa4116e3c20747ec1d6e24e5._comment
@@ -0,0 +1,12 @@
+[[!comment format=mdwn
+ username="https://www.google.com/accounts/o8/id?id=AItOawnXybLxkPMYpP3yw4b_I6IdC3cKTD-xEdU"
+ nickname="Matt"
+ subject="comment 4"
+ date="2013-07-02T20:45:29Z"
+ content="""
+I tried as you suggested.  It worked as far in so far that I was able to mark the repository dead.  However, I still see an empty line in the web app so perhaps this odd repo isn't the cause (but I'm deeply suspicious of it).  Looking at the HTML source the line in the web-app does not have any UUID at all (see the screen shot [here](http://dancingfrog.hopto.org/~matt/git-annex.png)).
+
+What causes the code to add back a previously deleted repository (I vaguely remember something about this being due to the merge algorithm...and why one can only mark repos as deleted)?  Why would it only add back to the group and trust logs and not the UUID?  The code that generates the web-app HTML must use something else or things in addition to the UUID log?
+
+The logs don't really say much - how can I help debug this further?
+"""]]
diff --git a/doc/forum/Rebuild_a_set_of_repos_from_checked-out_trees_and_.git__47__annex__47__objects__63__.mdwn b/doc/forum/Rebuild_a_set_of_repos_from_checked-out_trees_and_.git__47__annex__47__objects__63__.mdwn
new file mode 100644
index 0000000000..975746526b
--- /dev/null
+++ b/doc/forum/Rebuild_a_set_of_repos_from_checked-out_trees_and_.git__47__annex__47__objects__63__.mdwn
@@ -0,0 +1,69 @@
+# Situation/trouble
+
+I have a set of big repos.  Each full replica manages about 172.000 annexed files plus a number of small regular files in git history, for a total of about 1.8TB. At filesystem level, `find` reports about 924.000 entries (directories, files, symlinks).
+
+That worked rather well for a while except that a number of operations are slow (even outside git-annex, e.g. a plain `find` takes more than our hour the first time).  Also, the git part got rather heavy.  Hundreds of megabytes that should have been annexed were committed as regular files and vice versa.
+
+The whole setup even survived some catastrophe rather well, for example 6 months ago when the first 1.5GB of one hard drive was accidentally overwritten.  `fsck` with an alternate superblock fixed the lower level, while `git annex repair` fixed the rest nicely.
+
+Last night, though, the git parts got corrupted and I'm struggling to get things back to a sane state.  `git log` shows only recent history then fails.  Various attempts with `git annex repair` failed so far, I'll try again adding a new local "bare" git clone of the server as remote for `git annex repair` to use.
+
+Storage media and filesystems seem sane, still.  Software has been unchanged for a long time:
+
+* client run Ubuntu 16.04 with locally compiled git-annex version: 6.20161001-gade6ab4
+* server runs with locally compiled (in a Debian unstable chroot) git-annex version: 6.20161011-g3135d35 .
+
+
+# Considered solution
+
+I'm considering:
+
+* recreating a new set of replicas
+* each replica on same filesystem as old one
+* recreated only from the checked-out tree and the `.git/annex.objects` tree.
+* *without* copying data (re-reading on import is okay, but no room on a 2TB disk for duplicating 1.8TB)
+* non-constraint: this will lose detailed history which is an inconvenience we can live with.
+
+# Solution, practically
+
+## (1) Assuming `git annex fsck` can take into account objects manually placed into `.git/annex/objects`
+
+	mkdir $newrepo
+	cd $newrepo
+	git init
+	git annex init
+	cp -al $oldrepo/* $newrepo/ # ignores .git and other .* (dotfiles)
+	cp -al $oldrepo/.git/annex/objects/* $newrepo/.git/annex/objects/*
+	git annex fsck # will this find and use result of cp ?
+	# git annex fsck will also tell if some checked-out files lack their annexed data
+	git remote add ...(other replicas)...
+	git annex sync ...(other replicas)...
+	git annex unused # will tell if some files don't appear in checked-out tree?
+
+## (2) Else...
+
+	mkdir $newrepo
+	cd $newrepo
+	git init
+	git annex init
+	cp -al $oldrepo/* $newrepo/ # ignores .git and other .* (dotfiles)
+	cp -al $oldrepo/.git/annex/objects/* ${newrepo}.objectdup
+	git annex reinject --known ${newrepo}.objectdup  # will that perform a copy? I must not in this case.
+	# or something like find "${newrepo}.objectdup" -type f -exec git annex reinject --known {} \;
+	git annex fsck # will tell if some checked-out files lack their annexed data
+	git remote add ...(other replicas)...
+	git annex sync ...(other replicas)...
+	# if some files in $oldrepo/.git/annex/objects/* don't appear in checked-out tree, the won't be picked up by reinject and remain in ${newrepo}.objectdup
+
+# Questions
+
+No one wins when a lot of time is spent on dead ends. :-) Before I
+spend time testing if solutions 1 and 2 can work, is there any caveat
+to mention?
+
+For example, perhaps one must clone from a common empty ancestor
+instead of creating independent annexed then sync?
+
+What else? Is the whole approach sane? Doomed? Anything simpler/better?
+
+Thanks a lot.
diff --git a/doc/forum/Rebuild_a_set_of_repos_from_checked-out_trees_and_.git__47__annex__47__objects__63__/comment_1_456772b5b0deb0c121d7feb5dcf41979._comment b/doc/forum/Rebuild_a_set_of_repos_from_checked-out_trees_and_.git__47__annex__47__objects__63__/comment_1_456772b5b0deb0c121d7feb5dcf41979._comment
new file mode 100644
index 0000000000..a6389d995d
--- /dev/null
+++ b/doc/forum/Rebuild_a_set_of_repos_from_checked-out_trees_and_.git__47__annex__47__objects__63__/comment_1_456772b5b0deb0c121d7feb5dcf41979._comment
@@ -0,0 +1,12 @@
+[[!comment format=mdwn
+ username="CandyAngel"
+ avatar="http://cdn.libravatar.org/avatar/15c0aade8bec5bf004f939dd73cf9ed8"
+ subject="comment 1"
+ date="2017-07-21T09:25:25Z"
+ content="""
+My solution is very roundabout but preserves a lot of information, but did involve buying another drive (and exclusively using v5 indirect mode!).
+
+I create a new repository (on the new drive) which I import all the contents of the \"keyfiles\" (contents of .git/annex/objects). Then I create another repository with the filelinks (symlinks pointing to .git/annex/objects). After adding the keyfiles remote to this, this lets me see which content is present and valid, which got corrupted, is missing etc.
+
+Then I can move the valid content from this recovery annex into a proper annex and try and repair/find the corrupted/missing.
+"""]]
diff --git a/doc/forum/Rebuild_a_set_of_repos_from_checked-out_trees_and_.git__47__annex__47__objects__63__/comment_2_410f329fb47f13afc59b5b37b88a8c3a._comment b/doc/forum/Rebuild_a_set_of_repos_from_checked-out_trees_and_.git__47__annex__47__objects__63__/comment_2_410f329fb47f13afc59b5b37b88a8c3a._comment
new file mode 100644
index 0000000000..fc76a02c24
--- /dev/null
+++ b/doc/forum/Rebuild_a_set_of_repos_from_checked-out_trees_and_.git__47__annex__47__objects__63__/comment_2_410f329fb47f13afc59b5b37b88a8c3a._comment
@@ -0,0 +1,84 @@
+[[!comment format=mdwn
+ username="https://launchpad.net/~stephane-gourichon-lpad"
+ nickname="stephane-gourichon-lpad"
+ avatar="http://cdn.libravatar.org/avatar/02d4a0af59175f9123720b4481d55a769ba954e20f6dd9b2792217d9fa0c6089"
+ subject="Indeed git annex fsck can take into account objects manually placed into .git/annex/objects"
+ date="2017-07-28T05:03:30Z"
+ content="""
+Thanks @CandyAngel.  This is similar to wat I'm doing and somehow validates.
+I'm trying to repair on the same filesystem without long recopy.
+I don't understand why your solution is specific to v5 indirect mode.
+
+# Indeed git annex `fsck` can take into account objects manually placed into `.git/annex/objects`.
+
+Let's create a repo:
+
+    mkdir repo1
+    cd repo1/
+    ls -al
+    git init
+    git annex init repo1
+    ls -a
+    ls -al
+    echo 1 > 1
+    git annex add 1
+    git annex sync
+
+    git annex fsck
+    git annex repair --verbose
+
+Everything is fine.
+
+Let's say this repo has its git structures broken and we rebuild it from checked-out tree and `.git/annex/objects`.  We'll lose tree history and location tracking history but recover content.
+
+    cd ..
+    mkdir repo2
+    cd repo2
+    git init
+    git annex init repo2
+
+Ok we have an empty repo.  Let's import tree.
+
+    cp -al ../repo1/* .
+    ls -al
+    git annex add .
+    git annex repair --verbose
+
+    Running git fsck ...
+    No problems found.
+    ok
+
+Notice that `git annex repair` does not care about annexed objects, only history data.
+
+    git annex fsck
+
+But `fsck` notices about missing objects.
+
+    fsck 1
+      ** No known copies exist of 1
+    failed
+    (recording state in git...)
+    git-annex: fsck: 1 failed
+
+So, in a sense, `git annex fsck` and `git annex repair` operate on nearly independent things.
+
+Let's get annexed objects back.
+
+    ls -al # red shows symlink is broken
+    cp -al ../repo1/.git/annex/objects/ .git/annex/
+    find .git/annex/objects/
+    ls -al # grey shows symlink is okay
+    git annex fsck
+
+Hooray, `fsck` notices that objects are back.
+
+    fsck 1 (fixing location log) (checksum...) ok
+    (recording state in git...)
+
+# Conclusion
+
+I can use approach (1).  Extra benefit: it will notice if some files got corrupted on the filesystem.
+
+Approach (2) would mean, if any file was corrupted on the filesystem, it would have been considered the correct content, and I'd prefer to avoid that.
+
+"""]]
diff --git a/doc/forum/Recommended_number_of_repositories.mdwn b/doc/forum/Recommended_number_of_repositories.mdwn
new file mode 100644
index 0000000000..9e9f2838d6
--- /dev/null
+++ b/doc/forum/Recommended_number_of_repositories.mdwn
@@ -0,0 +1,4 @@
+With git it is easy to create one repository per project, and it almost always makes sense to do so.  When using git-annex, what is the recommended setup?
+
+Should one have a single annex containing all files, or is it recommended to create different repositories for things like 'photos', 'music', 'isos' ?
+
diff --git a/doc/forum/Recommended_number_of_repositories/comment_1_3ef256230756be8a9679b107cdbfd018._comment b/doc/forum/Recommended_number_of_repositories/comment_1_3ef256230756be8a9679b107cdbfd018._comment
new file mode 100644
index 0000000000..46ce0e8d53
--- /dev/null
+++ b/doc/forum/Recommended_number_of_repositories/comment_1_3ef256230756be8a9679b107cdbfd018._comment
@@ -0,0 +1,15 @@
+[[!comment format=mdwn
+ username="http://joey.kitenet.net/"
+ nickname="joey"
+ subject="depends ..."
+ date="2011-11-04T19:59:24Z"
+ content="""
+It makes sense to have separate repositories when you have well-defined uses for them. 
+
+I have a separate repository just for music and podcasts, which I can put various places where I have no need of the overhead of a tree of other files.
+
+If you're using it for whatever arbitrary large files you accumulate, I find it's useful to have them in one repository. This way I can rearrange things as makes sense. It might make sense to have \"photos\" and \"isos\" as categories today, but next year you might prefer to move those under 2011/{photos,isos}. It would certainly make sense to have different repositories for home, work, etc.
+
+How to split repositories up for a home directory is a general problem that the [vcs-home](http://vcs-home.branchable.com)
+project has surely considered at one time or another.
+"""]]
diff --git a/doc/forum/Recover_files__44___annex_stuck.mdwn b/doc/forum/Recover_files__44___annex_stuck.mdwn
new file mode 100644
index 0000000000..b3e46471f8
--- /dev/null
+++ b/doc/forum/Recover_files__44___annex_stuck.mdwn
@@ -0,0 +1,28 @@
+I have a directory with 6TB of data in it. I tried to use git annex to back it up to three 3TB drives, I didn't want to use RAID as it sucks, and I didn't want to use tar as I wanted my files easily available.
+
+I added my remotes successfully, then I ran ``git annex add .``
+
+That mostly worked, although it understandably took ages, although it missed several GB of files here and there.
+
+Next I tried to do ``git commit -a -m added``, hoping that this would copy all of my files to the remotes.  It didn't it just died with the error 
+
+    fatal: No HEAD commit to compare with (yet)
+    fatal: No HEAD commit to compare with (yet)
+    Stack space overflow: current size 8388608 bytes.
+    Use `+RTS -Ksize -RTS' to increase it.
+
+So I freaked out and decided to undo the mess and just go with tar instead, since at this point every git command takes multiple minutes and fails with the same error as above.
+
+I tried to run ``git anne unannex .``, but I got this error:
+
+``
+unannex GWAS/by-download-number/27081.log.gz fatal: No HEAD commit to compare with (yet)
+``
+
+So now I can't do anything without committing the files it seems, and I somehow need to grow the git cache, although when I search online for `+RTS -Ksize -RTS', I get nothing. 
+
+Does anyone know how to increase the cache size, or how to unannex the files without this HEAD error?
+
+Thanks,
+
+Mike
diff --git a/doc/forum/Recover_files__44___annex_stuck/comment_10_6d85c3ec73ddc0682d9643f4d5eeda70._comment b/doc/forum/Recover_files__44___annex_stuck/comment_10_6d85c3ec73ddc0682d9643f4d5eeda70._comment
new file mode 100644
index 0000000000..1a8cca6009
--- /dev/null
+++ b/doc/forum/Recover_files__44___annex_stuck/comment_10_6d85c3ec73ddc0682d9643f4d5eeda70._comment
@@ -0,0 +1,18 @@
+[[!comment format=mdwn
+ username="https://www.google.com/accounts/o8/id?id=AItOawmwjQzWgiD7_I3zw-_91rMRf_6qoThupis"
+ nickname="Mike"
+ subject="comment 10"
+ date="2014-06-18T20:15:42Z"
+ content="""
+``git ls-files --cached | wc -l`` returns: 1882028
+
+As far as I can tell, the largest objects in .git/objects are 65kb, there are just a bunch of them (257). Also, my repo contains 1,886,125 files and directories total, most in a single directory (after git annex add completed, that one directory contained 8.3GB of symlinks.
+
+``git-annex add .`` just completed successfully, I am now running ``git commit -a -m added``, it is chugging away and taking its time.
+
+Is there an obvious upper limit to the number of files or the total size of files that git annex can handle? For example, is 1 million files too many? How about 6TB? or 9TB? For this repo I think I have a little less than 2m files, and the total size of the repo is greater than 6TB. Is that too much? Should I split it into multiple repos?
+
+I also have a question just about the utility of git-annex for this purpose. I don't need to backup this data, I just want to have it off the big hard drive and onto a bunch of small drives. I have added 3 4TB drives as remotes and I want all of the data stored on them, I will take them offline and put them in a safe. Ideally my file and directory structure will remain intact as symlinks, and then when I want to access a file in the future, I can run ``git annex get ``, connect the drive that git annex tells me to, and then get that file, use it, and then drop it when I am done. From all of my reading it seems like that is a good usage for git annex, but I want to check with you and see if that makes sense to you. Also, can I just run ``git annex drop --auto --numcopies=1``  to get git annex to move all of the files to my remote repositories?
+
+Thanks for all of your help, and let me know if there are any other debug steps you would like me to run. I am still waiting for git commit to run, and for an exact repo size for you.
+"""]]
diff --git a/doc/forum/Recover_files__44___annex_stuck/comment_11_52e799bb6f24a1ebed58fad6cebd3a71._comment b/doc/forum/Recover_files__44___annex_stuck/comment_11_52e799bb6f24a1ebed58fad6cebd3a71._comment
new file mode 100644
index 0000000000..d5fc3e7ae6
--- /dev/null
+++ b/doc/forum/Recover_files__44___annex_stuck/comment_11_52e799bb6f24a1ebed58fad6cebd3a71._comment
@@ -0,0 +1,17 @@
+[[!comment format=mdwn
+ username="http://joeyh.name/"
+ ip="209.250.56.203"
+ subject="comment 11"
+ date="2014-06-18T20:54:31Z"
+ content="""
+Ok, my suspicion is that the root problem is having a large number of file in a single directory. That would cause the git tree objects to get big, and it may 
+be that git-annex somewhere buffers a whole tree object in memory, although I cannot think of where off the top of my head.
+
+git-annex scales to any size of files (limited only by checksumming time unless you use the WORM backend to avoid checksumming). git-annex tries to scale at least as good as git does to a large quantity of files in a repository. git doesn't handle a million files in a repository very fast, due to a number of issues including how the index works. I have never tested git-annex with more than 1 million files, and not all in the same directory.
+
+Other than the number of files, your use case seems reasonable. `git annex drop` will drop files that you have already copied to enough of the remotes (using eg, `git-annex copy`).
+
+Above you show a git-annex add failing after 5 files. I suspect you truncated that output, and it processed rather more files. git-annex only says \"(Recording state in git...)\" once it's added all the files, or after it's added around 10 thousand files and still has more to do). It seems to have failed at the point where the files are staged into the index.
+
+I'm building a 2 million file in one directory repo on a fast server now to see if I can reproduce this.
+"""]]
diff --git a/doc/forum/Recover_files__44___annex_stuck/comment_12_686a285bc7e950aae67856c47e7cb21e._comment b/doc/forum/Recover_files__44___annex_stuck/comment_12_686a285bc7e950aae67856c47e7cb21e._comment
new file mode 100644
index 0000000000..0428dfd309
--- /dev/null
+++ b/doc/forum/Recover_files__44___annex_stuck/comment_12_686a285bc7e950aae67856c47e7cb21e._comment
@@ -0,0 +1,18 @@
+[[!comment format=mdwn
+ username="https://www.google.com/accounts/o8/id?id=AItOawmwjQzWgiD7_I3zw-_91rMRf_6qoThupis"
+ nickname="Mike"
+ subject="comment 12"
+ date="2014-06-18T22:28:27Z"
+ content="""
+Yes, you are right. ``git-annex add`` got through almost all of the files in the first run, which I did a week ago, I am not sure how long it took, several days I think (which is fine, time isn't that important here).
+
+I re-ran ``git-annex add .`` yesterday after having trouble with ``git commit``, which is when I uncovered these problems. I think you are right that the problem appears when git-annex is staging files into the index. No problems occurred during checksumming or moving files.
+
+Also, the repo isn't as large as I thought, it if 4.1TB, so it makes sense that the issue is number of files, not files size.
+
+``git add`` and ``git commit`` are now working fine, all git operations (e.g. ``git status``) are now taking around 30s to 1 min, which is acceptable. 
+
+I am going to try and move the data to the remotes now. Is there anything special I need to do since the remotes are smaller than the current repo? The remotes are just single drives with ext4 filesystems and an empty repo on them. I ideally want to fill each drive as much as possible, and have the current repo contain no files, how do I do that? Can I just run ``git-annex move --to mito_backup1`` and then when it is full run a second command of ``git-annex move --to mito_backup2``.  Is it better to use ``git-annex copy`` instead of ``move`` and then use ``drop`` after?
+
+Thanks! 
+"""]]
diff --git a/doc/forum/Recover_files__44___annex_stuck/comment_13_a4d62d494b340458e6535d573bade965._comment b/doc/forum/Recover_files__44___annex_stuck/comment_13_a4d62d494b340458e6535d573bade965._comment
new file mode 100644
index 0000000000..e68933b5c7
--- /dev/null
+++ b/doc/forum/Recover_files__44___annex_stuck/comment_13_a4d62d494b340458e6535d573bade965._comment
@@ -0,0 +1,12 @@
+[[!comment format=mdwn
+ username="http://joeyh.name/"
+ ip="209.250.56.203"
+ subject="comment 13"
+ date="2014-06-18T22:43:08Z"
+ content="""
+It will be faster to use `git annex move`, assuming you want to only have 1 copy of each file, and not more. git-annex will stop storing files on a drive once it gets close to full (annex.diskreserve), and you can safely interrupt it and switch to the next drive.
+
+Do you have any special git configuration? In particular I'm curious about any annex.queuesize setting, which if set to something really high would make `git annex add` buffer a lot of filenames and stage them all at once. (However, I just noticed that annex.queuesize didn't cause as large a queue to be used as intended, so it would need to have been set to some really enormous value to run it out of stack space.)
+
+Also, see [[scalability]].
+"""]]
diff --git a/doc/forum/Recover_files__44___annex_stuck/comment_14_c10f0fe1440ccd170804a433db2267ee._comment b/doc/forum/Recover_files__44___annex_stuck/comment_14_c10f0fe1440ccd170804a433db2267ee._comment
new file mode 100644
index 0000000000..71c8d239db
--- /dev/null
+++ b/doc/forum/Recover_files__44___annex_stuck/comment_14_c10f0fe1440ccd170804a433db2267ee._comment
@@ -0,0 +1,12 @@
+[[!comment format=mdwn
+ username="https://www.google.com/accounts/o8/id?id=AItOawmwjQzWgiD7_I3zw-_91rMRf_6qoThupis"
+ nickname="Mike"
+ subject="comment 14"
+ date="2014-06-18T22:51:28Z"
+ content="""
+As far as I know I am using the defaults, I didn't customize anything. I am thinking of switching to the WORM backend though as I think it will make things a little faster, but I haven't done that yet.
+
+Also, I actually compiled cabal-install with the ghc flag ``-rtsopts`` and git-annex with flags ``-rtsopts -with-rtsopts=-K1000m``. Due to the amount of memory available, I am not worried if git-annex leaks memory and uses 1GB of memory during operations, but I have been watching it with htop, and its memory usage is usually very small.
+
+
+"""]]
diff --git a/doc/forum/Recover_files__44___annex_stuck/comment_15_14446cafac6c33a3f95b5344c42c0bef._comment b/doc/forum/Recover_files__44___annex_stuck/comment_15_14446cafac6c33a3f95b5344c42c0bef._comment
new file mode 100644
index 0000000000..bc1faa82cf
--- /dev/null
+++ b/doc/forum/Recover_files__44___annex_stuck/comment_15_14446cafac6c33a3f95b5344c42c0bef._comment
@@ -0,0 +1,14 @@
+[[!comment format=mdwn
+ username="https://www.google.com/accounts/o8/id?id=AItOawmwjQzWgiD7_I3zw-_91rMRf_6qoThupis"
+ nickname="Mike"
+ subject="comment 15"
+ date="2014-06-19T21:19:01Z"
+ content="""
+Everything seems to be working fine now, but I have another question:
+
+Is there any way to speed up the copying of many small files? It looks like git-annex is calling rsync for each individual file, which is very fast for large files, but on my directories with many small files, the total speed is working on to just a few MB an hour - it has only transferred 1GB in the last 4 hours.
+
+I am using the WORM backed with the ``-b WORM`` flag, but I wonder if there is a different move method implemented? For example many calls to ``mv`` will be much faster than many calls to ``rsync``. 
+
+Thanks!
+"""]]
diff --git a/doc/forum/Recover_files__44___annex_stuck/comment_16_63c19f58b7e95e39ba25a735bdcc0bcf._comment b/doc/forum/Recover_files__44___annex_stuck/comment_16_63c19f58b7e95e39ba25a735bdcc0bcf._comment
new file mode 100644
index 0000000000..c55edddf97
--- /dev/null
+++ b/doc/forum/Recover_files__44___annex_stuck/comment_16_63c19f58b7e95e39ba25a735bdcc0bcf._comment
@@ -0,0 +1,10 @@
+[[!comment format=mdwn
+ username="http://joeyh.name/"
+ ip="209.250.56.203"
+ subject="comment 16"
+ date="2014-06-19T22:16:21Z"
+ content="""
+git-annex could be made to always use cp for local transfers, see Remote/Git.hs rsyncOrCopyFile and change `ifM (sameDeviceIds src dest) (docopy, dorsync)` to just `docopy`
+
+However, I doubt that will be a significant speedup. It's more likely that the overhead around copying a file and updating the location tracking etc adds up with millions of small files.
+"""]]
diff --git a/doc/forum/Recover_files__44___annex_stuck/comment_17_8e5c7572ab8d1f0e41fedf6f805b942a._comment b/doc/forum/Recover_files__44___annex_stuck/comment_17_8e5c7572ab8d1f0e41fedf6f805b942a._comment
new file mode 100644
index 0000000000..4761bcf3ab
--- /dev/null
+++ b/doc/forum/Recover_files__44___annex_stuck/comment_17_8e5c7572ab8d1f0e41fedf6f805b942a._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="https://www.google.com/accounts/o8/id?id=AItOawmwjQzWgiD7_I3zw-_91rMRf_6qoThupis"
+ nickname="Mike"
+ subject="comment 17"
+ date="2014-06-21T01:56:48Z"
+ content="""
+Would git-annex be able to detect if I manually moved some files? At this point it looks like the transfer will take multiple weeks... if I just moved the files from .git/annex directly from one repo to the other I could make the transfer significantly faster, but would that corrupt the repo?
+"""]]
diff --git a/doc/forum/Recover_files__44___annex_stuck/comment_18_e5357c63107f79571bd3ff609b4406a7._comment b/doc/forum/Recover_files__44___annex_stuck/comment_18_e5357c63107f79571bd3ff609b4406a7._comment
new file mode 100644
index 0000000000..0398980b44
--- /dev/null
+++ b/doc/forum/Recover_files__44___annex_stuck/comment_18_e5357c63107f79571bd3ff609b4406a7._comment
@@ -0,0 +1,26 @@
+[[!comment format=mdwn
+ username="http://joeyh.name/"
+ ip="209.250.56.55"
+ subject="comment 18"
+ date="2014-07-04T18:03:54Z"
+ content="""
+You can manually move files and use `git annex fsck`, but it is not likely to be any faster.
+
+---
+
+After letting a 2 million file import run while I was away on vacation, I came back to it, and it indeed ran out of memory:
+
+
    +add 999996 ok
+add 999997 ok
+(Recording state in git...)
+[2014-06-21 11:49:28 JEST] feed: xargs [\"-0\",\"git\",\"--git-dir=/home/joey/tmp/r/.git\",\"--work-tree=/home/joey/tmp/r\",\"add\",\"--\"]
+add 999998 ok
+add 999999 ok
+[2014-06-21 11:49:49 JEST] read: git [\"--git-dir=/home/joey/tmp/r/.git\",\"--work-tree=/home/joey/tmp/r\",\"diff\",\"--name-only\",\"--diff-filter=T\",\"-z\",\"--\",\".\"]
+(Recording state in git...)
+[2014-06-21 11:52:24 JEST] feed: xargs [\"-0\",\"git\",\"--git-dir=/home/joey/tmp/r/.git\",\"--work-tree=/home/joey/tmp/r\",\"add\",\"--\"]
+Stack space overflow: current size 8388608 bytes.
+Use `+RTS -Ksize -RTS' to increase it.
+
    
+"""]]
diff --git a/doc/forum/Recover_files__44___annex_stuck/comment_19_3316652073710f39965cd49ceea5c4ff._comment b/doc/forum/Recover_files__44___annex_stuck/comment_19_3316652073710f39965cd49ceea5c4ff._comment
new file mode 100644
index 0000000000..828b07d41f
--- /dev/null
+++ b/doc/forum/Recover_files__44___annex_stuck/comment_19_3316652073710f39965cd49ceea5c4ff._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="http://joeyh.name/"
+ ip="209.250.56.55"
+ subject="comment 19"
+ date="2014-07-04T18:18:41Z"
+ content="""
+[[bugs/runs_of_of_memory_adding_2_million_files]]
+"""]]
diff --git a/doc/forum/Recover_files__44___annex_stuck/comment_1_d605f755c363d56cf5f1060ad06ee173._comment b/doc/forum/Recover_files__44___annex_stuck/comment_1_d605f755c363d56cf5f1060ad06ee173._comment
new file mode 100644
index 0000000000..18c1c0e9bb
--- /dev/null
+++ b/doc/forum/Recover_files__44___annex_stuck/comment_1_d605f755c363d56cf5f1060ad06ee173._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="http://joeyh.name/"
+ ip="209.250.56.203"
+ subject="comment 1"
+ date="2014-06-18T16:22:13Z"
+ content="""
+What version of git-annex are you using? And what version of git? Is your git-annex repository in direct mode or indirect mode?
+"""]]
diff --git a/doc/forum/Recover_files__44___annex_stuck/comment_2_f3ee184a4d3b8d82a8a362a6c03a54a3._comment b/doc/forum/Recover_files__44___annex_stuck/comment_2_f3ee184a4d3b8d82a8a362a6c03a54a3._comment
new file mode 100644
index 0000000000..7c46785bf2
--- /dev/null
+++ b/doc/forum/Recover_files__44___annex_stuck/comment_2_f3ee184a4d3b8d82a8a362a6c03a54a3._comment
@@ -0,0 +1,54 @@
+[[!comment format=mdwn
+ username="https://www.google.com/accounts/o8/id?id=AItOawmwjQzWgiD7_I3zw-_91rMRf_6qoThupis"
+ nickname="Mike"
+ subject="Frustrating"
+ date="2014-06-18T16:29:31Z"
+ content="""
+OK, this is a little frustrating. I found this post from three years ago: [[http://git-annex.branchable.com/forum/Problems_with_large_numbers_of_files/]] and I decided to try a newer version of git-annex.
+
+I uninstalled ghc and haskell from Scientific Linux because all of these Red Hat based distros have ancient packages.
+
+I installed the latest git from source, the latest ghc linux x86_64 binary, and then the latest haskell platform from source. Then I used cabal to install all dependencies for git annex with ``cabal install --only-dependencies git-annex``. Finally I installed git annex from source.
+
+I then tried to run ``git-annex add .`` in my directory and got the following error:
+
+    #> git annex add .
+    add AllStudies.txt.csv.gz ok
+    add GWAS/action.txt.gz ok
+    add GWAS/archive/dump2.txt.gz ok
+    add GWAS/archive/dump3.txt.gz ok
+    add GWAS/by-download-number/27080.log.gz ok
+    (Recording state in git...)
+    Stack space overflow: current size 8388608 bytes.
+    Use `+RTS -Ksize -RTS' to increase it.
+
+
+Ok, I was hoping that the latest version would just work, no luck. So I did what it told me to:
+
+``git-annex +RTS -K1000000 -RTS add .``
+
+That gave the error:
+
+``
+    git-annex: Most RTS options are disabled. Link with -rtsopts to enable them.
+``
+
+Grr.
+
+So I went into the Makefile and added the line ``-rtsopts -with-rtsopts=\"-K1000m\"`` after every call to ghc I could find. I also added ``ghc-options: -with-rtsopts=\"-K100000\"`` to my ~/.cabal/config file.
+
+Now when I run ``make`` I get this error:
+
+
+    if [ \"cabal \" = ./Setup ]; then ghc --make Setup; fi
+    cabal  configure
+    cabal: Most RTS options are disabled. Link with -rtsopts to enable them.
+    make: *** [Build/SysConfig.hs] Error 1
+
+
+Do I have to manually compile the entire Haskell Platform with the -rtsopts flag in order to get this to work?
+
+I can't find any easy-to-follow information anywhere that shows me how to just increase the memory limit. My server has 48 cores, 192GB of memory, over 1TB of scratch space, and over 60TB of storage. I really want to be able to use git-annex to easily move files from our large RAID arrays onto archive drives, and be able to intelligently get that data back whenever I want, I don't understand why I am being limited to 8MB of memory for this.
+
+Any advice would be fantastic, thank you.
+"""]]
diff --git a/doc/forum/Recover_files__44___annex_stuck/comment_3_341b47663d133411587ec70ef2b178c6._comment b/doc/forum/Recover_files__44___annex_stuck/comment_3_341b47663d133411587ec70ef2b178c6._comment
new file mode 100644
index 0000000000..77a7c93c8b
--- /dev/null
+++ b/doc/forum/Recover_files__44___annex_stuck/comment_3_341b47663d133411587ec70ef2b178c6._comment
@@ -0,0 +1,14 @@
+[[!comment format=mdwn
+ username="https://www.google.com/accounts/o8/id?id=AItOawmwjQzWgiD7_I3zw-_91rMRf_6qoThupis"
+ nickname="Mike"
+ subject="Version"
+ date="2014-06-18T16:32:32Z"
+ content="""
+Hi Joeyh,
+
+Thanks for the reply. I am using git version 2.0.0.390.gcb682f8, not sure what version of git-annex, but I downloaded it from github about 20 minutes ago.
+
+Thanks!
+
+-Mike
+"""]]
diff --git a/doc/forum/Recover_files__44___annex_stuck/comment_4_66c0d9284d5edbac189a64b03c4fe50a._comment b/doc/forum/Recover_files__44___annex_stuck/comment_4_66c0d9284d5edbac189a64b03c4fe50a._comment
new file mode 100644
index 0000000000..5d153e2481
--- /dev/null
+++ b/doc/forum/Recover_files__44___annex_stuck/comment_4_66c0d9284d5edbac189a64b03c4fe50a._comment
@@ -0,0 +1,10 @@
+[[!comment format=mdwn
+ username="http://joeyh.name/"
+ ip="209.250.56.203"
+ subject="comment 4"
+ date="2014-06-18T16:35:29Z"
+ content="""
+You can find out the version of git-annex by running: git-annex version
+
+You can find out if your repository is in direct or indirect mode by running: git config annex.direct
+"""]]
diff --git a/doc/forum/Recover_files__44___annex_stuck/comment_5_8b32f6597f447f88bee7a80698fb4df6._comment b/doc/forum/Recover_files__44___annex_stuck/comment_5_8b32f6597f447f88bee7a80698fb4df6._comment
new file mode 100644
index 0000000000..cf656f3eae
--- /dev/null
+++ b/doc/forum/Recover_files__44___annex_stuck/comment_5_8b32f6597f447f88bee7a80698fb4df6._comment
@@ -0,0 +1,18 @@
+[[!comment format=mdwn
+ username="https://www.google.com/accounts/o8/id?id=AItOawmwjQzWgiD7_I3zw-_91rMRf_6qoThupis"
+ nickname="Mike"
+ subject="Versions"
+ date="2014-06-18T16:40:14Z"
+ content="""
+``git-annex version`` returns:
+
+    git-annex version: 5.20140618-gc2f1c63 
+    build flags: Assistant Webapp Webapp-secure Pairing Testsuite S3 WebDAV Inotify DBus DesktopNotify XMPP DNS Feeds Quvi TDFA CryptoHash
+    key/value backends: SHA256E SHA1E SHA512E SHA224E SHA384E SKEIN256E SKEIN512E SHA256 SHA1 SHA512 SHA224 SHA384 SKEIN256 SKEIN512 WORM URL
+    remote types: git gcrypt S3 bup directory rsync web webdav tahoe glacier ddar hook external
+    local repository version: unknown
+    supported repository version: 5
+    upgrade supported from repository versions: 0 1 2 4
+
+``git config annex.direct`` exits with error code 1 and doesn't return any information, however I never explicitly set direct mode, and the repository is all symlinked, so my assumption is that it is in indirect mode. Would direct mode be better for such a large repo?
+"""]]
diff --git a/doc/forum/Recover_files__44___annex_stuck/comment_6_4cc81169e99a453cdb6e83e57e638f37._comment b/doc/forum/Recover_files__44___annex_stuck/comment_6_4cc81169e99a453cdb6e83e57e638f37._comment
new file mode 100644
index 0000000000..52e04dfc85
--- /dev/null
+++ b/doc/forum/Recover_files__44___annex_stuck/comment_6_4cc81169e99a453cdb6e83e57e638f37._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="https://www.google.com/accounts/o8/id?id=AItOawmwjQzWgiD7_I3zw-_91rMRf_6qoThupis"
+ nickname="Mike"
+ subject="Reinstall GHC or Cabal?"
+ date="2014-06-18T16:49:35Z"
+ content="""
+Do I need to reinstall ghc or cabal with rtsopts enabled somehow in order to be able to compile git-annex with -K1000m?
+"""]]
diff --git a/doc/forum/Recover_files__44___annex_stuck/comment_7_2d104cf4682e04906f8ca0ced7288cf1._comment b/doc/forum/Recover_files__44___annex_stuck/comment_7_2d104cf4682e04906f8ca0ced7288cf1._comment
new file mode 100644
index 0000000000..6ea7b45f64
--- /dev/null
+++ b/doc/forum/Recover_files__44___annex_stuck/comment_7_2d104cf4682e04906f8ca0ced7288cf1._comment
@@ -0,0 +1,17 @@
+[[!comment format=mdwn
+ username="http://joeyh.name/"
+ ip="209.250.56.203"
+ subject="comment 7"
+ date="2014-06-18T17:14:40Z"
+ content="""
+Ok, so the repository is in indirect mode, and this rules out a large quantity of problems that could have been caused by direct mode (no, I don't recommend using direct mode).
+
+If you want to build git-annex with the +RTS option enabled, you just need to pass -rtsopts to ghc when building git-annex. (Not -with-rtsopts ...) 
+That *might* let you pump up the memory and bypass whatever the problem is, or at least find out how much memory it's trying to allocate, which might be a useful clue. But I would be much more interested in debugging and fixing the actual problem, since git-annex should not normally need to allocate a 8+ mb chunk of memory.
+
+The  \"No HEAD commit to compare with (yet)\" failure mode was removed from git in 2011. You must have been using old versions of git and git-annex before you upgraded. Perhaps they have left the repository in some broken state.
+
+What size does `du -hsc .git/objects` report? How about `du -h .git/index`?
+
+Are git commands that do not involve git-annex still taking a long time to run or failing in some way? (Note that `git commit` has a hook that runs git-annex; you can bypass that with `git commit --no-verify`)
+"""]]
diff --git a/doc/forum/Recover_files__44___annex_stuck/comment_8_d356c4fce9f1197e5292f9dedf85bbc9._comment b/doc/forum/Recover_files__44___annex_stuck/comment_8_d356c4fce9f1197e5292f9dedf85bbc9._comment
new file mode 100644
index 0000000000..df1af4f06b
--- /dev/null
+++ b/doc/forum/Recover_files__44___annex_stuck/comment_8_d356c4fce9f1197e5292f9dedf85bbc9._comment
@@ -0,0 +1,22 @@
+[[!comment format=mdwn
+ username="https://www.google.com/accounts/o8/id?id=AItOawmwjQzWgiD7_I3zw-_91rMRf_6qoThupis"
+ nickname="Mike"
+ subject="comment 8"
+ date="2014-06-18T17:31:58Z"
+ content="""
+The git available through yum is git 1.7.1, which looks like it was released in 2010 or earlier. (I really wish I had a different version of linux on this server). It is possible that in some way screwed up the repo.
+
+I figured out how to compile cabal and git-annex with rtsopts, so I can now set higher memory levels, but I am happy to help debug the problem too, as I would really love a fully functional git-annex.
+
+git commands now run quickly, thanks to the new git I think.
+
+``du -hsc .git/objects`` returns: ``8.1G    .git/objects``
+
+``du -h .git/index`` returns: ``437M    .git/index``
+
+I am currently running the command ``git-annex +RTS -K1000m -RTS add .``, it is chugging away doing something, but is not printing any messages yet after 11 minutes of running, it is a 6TB directory though, and there are a lot of concurrent IO operations on that disk right now.
+
+I am also running ``du -h --max-depth=1`` on the root repo directory, and also ``find | wc -l``, so that I can tell you the exact size of the dir and the total number of files too.  These operations combined may take more than an hour though, I will send details when the commands complete.
+
+Let me know if you want me to stop the ``git-annex +RTS -K1000m -RTS add .`` command and run git-annex some other way.
+"""]]
diff --git a/doc/forum/Recover_files__44___annex_stuck/comment_9_856c7e1575f5d99530ecd54004315487._comment b/doc/forum/Recover_files__44___annex_stuck/comment_9_856c7e1575f5d99530ecd54004315487._comment
new file mode 100644
index 0000000000..c092d99f50
--- /dev/null
+++ b/doc/forum/Recover_files__44___annex_stuck/comment_9_856c7e1575f5d99530ecd54004315487._comment
@@ -0,0 +1,12 @@
+[[!comment format=mdwn
+ username="http://joeyh.name/"
+ ip="209.250.56.203"
+ subject="comment 9"
+ date="2014-06-18T17:36:44Z"
+ content="""
+Both of those du's look extremely large. How many files are listed by `git ls-files --cached | wc -l` ?
+
+I don't think that there's any point in running `git annex add` while you're still having some problem. I am curious though how much memory the git-annex add you have running has used.
+
+If I were you, I'd look in .git/objects for large files (> 100kb, say).
+"""]]
diff --git a/doc/forum/Recover_repository_from_bup_repository.mdwn b/doc/forum/Recover_repository_from_bup_repository.mdwn
new file mode 100644
index 0000000000..473bfb62e2
--- /dev/null
+++ b/doc/forum/Recover_repository_from_bup_repository.mdwn
@@ -0,0 +1 @@
+I screwed up one of my repositories, so I decided to start from clean slate. I have a remote bup repository that has all of my previous repository data. Now, I want to get it back. How I can do this?
diff --git a/doc/forum/Recover_repository_from_bup_repository/comment_1_6e3b1ff58fbdc10bcaca6d660eb1156f._comment b/doc/forum/Recover_repository_from_bup_repository/comment_1_6e3b1ff58fbdc10bcaca6d660eb1156f._comment
new file mode 100644
index 0000000000..9b95472e44
--- /dev/null
+++ b/doc/forum/Recover_repository_from_bup_repository/comment_1_6e3b1ff58fbdc10bcaca6d660eb1156f._comment
@@ -0,0 +1,35 @@
+[[!comment format=mdwn
+ username="joey"
+ subject="""comment 1"""
+ date="2016-09-21T15:59:55Z"
+ content="""
+If your bup repository is a git-annex special remote, it contains only 
+the contents of annexed files, but not the rest of the git repository data
+(no filenames, no pointers to git-annex keys, etc).
+
+Also, the files in it may be encrypted in a variety of ways depending on
+how you set up that git-annex special remote. If encryption was used, most
+likely it needs information that was stored in the git repository to
+decrypt it.
+
+So, your best bet is to restore your git-annex repository from a backup or
+a clone of that repository. Then you can just `git annex enableremote` the
+bup special remote and use `git-annex get` as usual. If you didn't have a
+backup or a clone of the git-annex repository, then important 
+information is lost.
+
+Without the git-annex repository, you can manually get the contents of the
+files from the bup repository. In the bup repository, run `git branch`;
+this will print out the names of all the git-annex keys that were stored
+there. Then you can use `bup join` to extract the content of each key:
+
+	bup join -r /path/to/bup/repository $key > $key.content
+
+Without the git-annex repository, there's no record of what the filename
+was, but you can extract the content this way.
+
+But, if the key names start with "GPG" the the data is stored in bup
+encrypted and you are probably out of luck (although if you used
+encryption=pubkey when setting up the bup special remote, you
+can use gpg to decrypt the files after `bup join`).
+"""]]
diff --git a/doc/forum/Recovering_data_from_rsync_special_remote_after_git-annex-forget.mdwn b/doc/forum/Recovering_data_from_rsync_special_remote_after_git-annex-forget.mdwn
new file mode 100644
index 0000000000..0df8689985
--- /dev/null
+++ b/doc/forum/Recovering_data_from_rsync_special_remote_after_git-annex-forget.mdwn
@@ -0,0 +1,11 @@
+Hello there!
+
+I sent a part of my files to a special rsync remote ```git annex move --all --to rsync_remote```
+Then I rewrote the history of my repository ```git annex forget```
+And some days later I try ```git annex whereis file``` which tells me there is no copy left and ````git annex info rsync_remote``` which outputs no annexed keys.
+Now, I know the data is still in the rsync special remote because looking for the files SHA keys, I am able to find their annex objects directories.
+The annex objects directories are different between the local git annex and the special rsync remote, supposedly because of my rewrite.
+
+Is there any simple way to repair that? To tell git-annex that the keys are accessible in the special rsync remote?
+I might be able to recover the data doing something like copying all the rsync directory locally and reinjecting files but it seems over complicated while the keys are valid.
+Moreover, the computer with the special rsync remote is a distant one whith limited support (I can't install git-annex on it) so it would be very slow to copy all the data through the network.
diff --git a/doc/forum/Recovering_data_from_rsync_special_remote_after_git-annex-forget/comment_1_265563f95a8ec5918314afc7e6226b56._comment b/doc/forum/Recovering_data_from_rsync_special_remote_after_git-annex-forget/comment_1_265563f95a8ec5918314afc7e6226b56._comment
new file mode 100644
index 0000000000..077398fc36
--- /dev/null
+++ b/doc/forum/Recovering_data_from_rsync_special_remote_after_git-annex-forget/comment_1_265563f95a8ec5918314afc7e6226b56._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="CandyAngel"
+ avatar="http://cdn.libravatar.org/avatar/15c0aade8bec5bf004f939dd73cf9ed8"
+ subject="comment 1"
+ date="2018-09-19T11:03:54Z"
+ content="""
+I think `git annex fsck --fast --from $remote` will make the local annex relearn all the files the remote has without transferring them.
+"""]]
diff --git a/doc/forum/Recovering_data_from_rsync_special_remote_after_git-annex-forget/comment_2_4da1d3f6a37b8772838bae762d4c43be._comment b/doc/forum/Recovering_data_from_rsync_special_remote_after_git-annex-forget/comment_2_4da1d3f6a37b8772838bae762d4c43be._comment
new file mode 100644
index 0000000000..7542ce4072
--- /dev/null
+++ b/doc/forum/Recovering_data_from_rsync_special_remote_after_git-annex-forget/comment_2_4da1d3f6a37b8772838bae762d4c43be._comment
@@ -0,0 +1,10 @@
+[[!comment format=mdwn
+ username="webanck"
+ avatar="http://cdn.libravatar.org/avatar/cd273f76ef8c4218510b4f50ef7e1f3d"
+ subject="comment 2"
+ date="2018-09-19T11:40:56Z"
+ content="""
+Indeed, this command did the trick!
+I didn't even think about using ```fsck``` because the ```git annex info``` was returning 0 references.
+Thanks a lot!
+"""]]
diff --git a/doc/forum/Recovering_deleted_file_in_direct_mode.mdwn b/doc/forum/Recovering_deleted_file_in_direct_mode.mdwn
new file mode 100644
index 0000000000..9dc1bdaf1f
--- /dev/null
+++ b/doc/forum/Recovering_deleted_file_in_direct_mode.mdwn
@@ -0,0 +1,13 @@
+I accidentally deleted a file from a git-annex repository with a plain "rm". How can I restore it from other repositories that have it?
+
+I tried using
+
+    git annex copy --from $REMOTE $REMOVED_FILE
+
+but git-annex complaints about
+
+    $REMOVED_FILE not found
+
+I suppose that I could switch to indirect mode and do checkout and older version, but I'd prefer not to. My repository contains thousands of quite big files; it would take a lot of time and the probability of something going wrong during the conversion is quite high (see other bugs reported during import).
+
+I'm using git-annex 5.20141024-g613f396.
diff --git a/doc/forum/Recovering_deleted_file_in_direct_mode/comment_1_a3e6fc93669ec76656c84793d1f59149._comment b/doc/forum/Recovering_deleted_file_in_direct_mode/comment_1_a3e6fc93669ec76656c84793d1f59149._comment
new file mode 100644
index 0000000000..0b23bbc784
--- /dev/null
+++ b/doc/forum/Recovering_deleted_file_in_direct_mode/comment_1_a3e6fc93669ec76656c84793d1f59149._comment
@@ -0,0 +1,11 @@
+[[!comment format=mdwn
+ username="joey"
+ subject="""comment 1"""
+ date="2014-10-28T19:22:54Z"
+ content="""
+> the probability of something going wrong during the conversion is quite high (see other bugs reported during import)
+
+I don't know what bugs you speak of. If the probability of something going
+wrong is quite high, then you must have a reproducible test case. Submit a
+bug with such a test case, and I can fix it.
+"""]]
diff --git a/doc/forum/Recovering_deleted_file_in_direct_mode/comment_2_98c01a756c5f2dda28cffa5dd1dea385._comment b/doc/forum/Recovering_deleted_file_in_direct_mode/comment_2_98c01a756c5f2dda28cffa5dd1dea385._comment
new file mode 100644
index 0000000000..007160ab25
--- /dev/null
+++ b/doc/forum/Recovering_deleted_file_in_direct_mode/comment_2_98c01a756c5f2dda28cffa5dd1dea385._comment
@@ -0,0 +1,16 @@
+[[!comment format=mdwn
+ username="http://joeyh.name/"
+ subject="comment 2"
+ date="2014-11-12T21:00:41Z"
+ content="""
+A new `git annex proxy` command was just added that allows safely running commands like `git revert` in a direct mode repsitory.
+
+So, you might do:
+
+    rm myfile # oops!
+    git annex sync
+    git annex proxy -- git revert HEAD
+    git annex get myfile
+
+The sync is needed to commit the deletion, and then that commit gets reverted. Finally, you can use git-annex get to download the file from some other repository that still has a copy. (Assuming you didn't delete the last copy.)
+"""]]
diff --git a/doc/forum/Recovery_after_freeze_while_importing_files.mdwn b/doc/forum/Recovery_after_freeze_while_importing_files.mdwn
new file mode 100644
index 0000000000..f67e7fb44b
--- /dev/null
+++ b/doc/forum/Recovery_after_freeze_while_importing_files.mdwn
@@ -0,0 +1,3 @@
+My PC froze while git-annex was in the process of importing a file. It was in the final stage of updating git status when the freeze occurred. git status shows the imported file under "changes to be committed" as "new file."
+
+I don't know how git-annex works under the hood, so I don't know how the freeze might've affected the repo. I don't want to lose my files. What do you recommend is the best way of recovering from this and making sure there are no errors?
diff --git a/doc/forum/Recovery_after_freeze_while_importing_files/comment_1_d033b77b9417468e13076ff9302f2963._comment b/doc/forum/Recovery_after_freeze_while_importing_files/comment_1_d033b77b9417468e13076ff9302f2963._comment
new file mode 100644
index 0000000000..c9d004c7aa
--- /dev/null
+++ b/doc/forum/Recovery_after_freeze_while_importing_files/comment_1_d033b77b9417468e13076ff9302f2963._comment
@@ -0,0 +1,7 @@
+[[!comment format=mdwn
+ username="joey"
+ subject="""comment 1"""
+ date="2014-12-02T17:09:24Z"
+ content="""
+Doesn't sound like a problem. Have you tried committing the file?
+"""]]
diff --git a/doc/forum/Recovery_after_freeze_while_importing_files/comment_2_e14f330de3e9ebfa9a99c32e65d59d11._comment b/doc/forum/Recovery_after_freeze_while_importing_files/comment_2_e14f330de3e9ebfa9a99c32e65d59d11._comment
new file mode 100644
index 0000000000..3a897cc027
--- /dev/null
+++ b/doc/forum/Recovery_after_freeze_while_importing_files/comment_2_e14f330de3e9ebfa9a99c32e65d59d11._comment
@@ -0,0 +1,7 @@
+[[!comment format=mdwn
+ username="ghen1"
+ subject="comment 2"
+ date="2014-12-18T11:28:35Z"
+ content="""
+I did, and everything seems fine.
+"""]]
diff --git a/doc/forum/Reduce_disk_space_by_dropping_older_versions_in_local_git_annex.mdwn b/doc/forum/Reduce_disk_space_by_dropping_older_versions_in_local_git_annex.mdwn
new file mode 100644
index 0000000000..1b7ebfcff9
--- /dev/null
+++ b/doc/forum/Reduce_disk_space_by_dropping_older_versions_in_local_git_annex.mdwn
@@ -0,0 +1,9 @@
+Looking at using git-annex, I wonder if there is a way of removing previous versions from a local git repository annex. We have a 'gold' copy of the repository and will also have of the annex, so dropping a local copy is fine.
+
+If I checkout older git tags, then the git repo annex fills up with those old annex file versions, and I would like a way of flushing out the old versions to minimise the disk space.
+
+I tried git annex drop, but that appears to only drop the current version.
+
+Does git-annex understand that all the versions of a file are related, or is this relationship only discoverable from the symlink in the git repo?
+
+It is not a deal-breaker, but I would like a single command I can give my team if they are running short on disk space.
diff --git a/doc/forum/Reduce_disk_space_by_dropping_older_versions_in_local_git_annex/comment_1_2a84bf661e0cbd5c8a558d3dbf5aef65._comment b/doc/forum/Reduce_disk_space_by_dropping_older_versions_in_local_git_annex/comment_1_2a84bf661e0cbd5c8a558d3dbf5aef65._comment
new file mode 100644
index 0000000000..553c2248a8
--- /dev/null
+++ b/doc/forum/Reduce_disk_space_by_dropping_older_versions_in_local_git_annex/comment_1_2a84bf661e0cbd5c8a558d3dbf5aef65._comment
@@ -0,0 +1,15 @@
+[[!comment format=mdwn
+ username="joey"
+ subject="""comment 1"""
+ date="2016-01-15T19:31:32Z"
+ content="""
+`git annex unused` is the way to find such old versions of files, and
+then you can use commands like `git annex move --unused --to gold` or
+`git annex dropunused`
+
+By default, `git annex unused` assumes that if any branch or tag points to
+a version of a file, that version is used. If you want it to consider tags
+as unused, run:
+
+	git annex unused --used-refspec='+refs/heads/*'
+"""]]
diff --git a/doc/forum/Reloading_.git__47__config_mid-sync.mdwn b/doc/forum/Reloading_.git__47__config_mid-sync.mdwn
new file mode 100644
index 0000000000..0c2ef14fd9
--- /dev/null
+++ b/doc/forum/Reloading_.git__47__config_mid-sync.mdwn
@@ -0,0 +1 @@
+There are some settings (rsync bandwidth settings are my use case) I'd like to reload after assistant starts, possibly mid-sync. Does assistant support SIGHUP, etc? What's the correct way to change bandwidth settings?
diff --git a/doc/forum/Reloading_.git__47__config_mid-sync/comment_1_69e8879e0fed0e1b1589a721f9c6e3c7._comment b/doc/forum/Reloading_.git__47__config_mid-sync/comment_1_69e8879e0fed0e1b1589a721f9c6e3c7._comment
new file mode 100644
index 0000000000..d39b21ffbf
--- /dev/null
+++ b/doc/forum/Reloading_.git__47__config_mid-sync/comment_1_69e8879e0fed0e1b1589a721f9c6e3c7._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="http://joeyh.name/"
+ ip="24.159.78.125"
+ subject="comment 1"
+ date="2014-07-23T15:02:04Z"
+ content="""
+Assistant does not support SIGHUP. There is a menu item (top right menu) that can restart it.
+"""]]
diff --git a/doc/forum/Relocating_annex_directory.mdwn b/doc/forum/Relocating_annex_directory.mdwn
new file mode 100644
index 0000000000..f2e271333e
--- /dev/null
+++ b/doc/forum/Relocating_annex_directory.mdwn
@@ -0,0 +1 @@
+I have around 70 GBs of data spread around 4 repositories on a Linux (Ubuntu) box. My problem is I need to reformat the drive they are on. I would like to move them to an external usb drive temporarily during reinstall then move them back to their original location. When I started with annex I did try to mv annexFolder/ toNewLoc/ which failed leaving behind a corrupt repo. What is a safe way to move an annex folder? My primary connection is a 3G modem so I am trying to avoid re downloading everything. Another thing I am trying to avoid is cloning the repos to the external drive, reinstall, clone it back to the internal drive and mark original and external repos as dead, but AFAIK those dead repos will show up in the output of whereis, so everytime I reformat I gonna have two extra dead repos.
diff --git a/doc/forum/Relocating_annex_directory/comment_1_13ff5438baa1db110beb6aab3a783def._comment b/doc/forum/Relocating_annex_directory/comment_1_13ff5438baa1db110beb6aab3a783def._comment
new file mode 100644
index 0000000000..5136941c50
--- /dev/null
+++ b/doc/forum/Relocating_annex_directory/comment_1_13ff5438baa1db110beb6aab3a783def._comment
@@ -0,0 +1,11 @@
+[[!comment format=mdwn
+ username="https://www.google.com/accounts/o8/id?id=AItOawmBUR4O9mofxVbpb8JV9mEbVfIYv670uJo"
+ nickname="Justin"
+ subject="comment 1"
+ date="2013-08-16T03:42:00Z"
+ content="""
+moving the annex should work fine, provided:
+
+* You are moving it to a proper unix filesystem - NOT fat32
+* The assistant is shutdown. If it is running, bad things will happen.
+"""]]
diff --git a/doc/forum/Relocating_annex_directory/comment_2_6d88ff03fcf00ae872442e8a86c968ed._comment b/doc/forum/Relocating_annex_directory/comment_2_6d88ff03fcf00ae872442e8a86c968ed._comment
new file mode 100644
index 0000000000..e7b361675f
--- /dev/null
+++ b/doc/forum/Relocating_annex_directory/comment_2_6d88ff03fcf00ae872442e8a86c968ed._comment
@@ -0,0 +1,10 @@
+[[!comment format=mdwn
+ username="http://joeyh.name/"
+ ip="4.154.0.63"
+ subject="comment 2"
+ date="2013-08-24T18:56:11Z"
+ content="""
+`git annex whereis` does not show dead repositories. 
+
+Anyway Justin is of course right: Provided the assistant is not running in a repository, the repository is just a collection of files in a directory, and can be moved around, tarred up, untarred, etc just like any other repository. If the assistant is running it may become unhappy if its repository vanishes out from underneath it.
+"""]]
diff --git a/doc/forum/Remote__39__s_git-annex-shell_not_detected.mdwn b/doc/forum/Remote__39__s_git-annex-shell_not_detected.mdwn
new file mode 100644
index 0000000000..5d9223ff98
--- /dev/null
+++ b/doc/forum/Remote__39__s_git-annex-shell_not_detected.mdwn
@@ -0,0 +1,20 @@
+When doing `git annex get fname` in a repo A with 2 remotes on LAN (B and C), it says they don't have git annex installed so it sets the `annex-ignore` flag. However, they do have it, and I can `annex-get` files from C to B. Versions of git and git-annex vary between the machines. Any idea what could be causing it? Perhaps A's git needs an update?
+
+
+A:
+
+* Ubuntu. 
+* git 1.9.1
+* git-annex 5.20140412ubuntu1
+
+B: 
+
+* Fedora 22
+* git 2.3.2 (installed through Nix package manager, so its in a nonstandard path)
+* git-annex 5.20140717 (installed normally, in /usr/bin)
+
+C: 
+
+* Fedora 22
+* git 2.3.2 (installed through Nix package manager, so its in a nonstandard path)
+* git-annex 5.20150219 (installed through Nix
diff --git a/doc/forum/Remote__39__s_git-annex-shell_not_detected/comment_1_8354782cf0fab248e607c7eae59d6bb2._comment b/doc/forum/Remote__39__s_git-annex-shell_not_detected/comment_1_8354782cf0fab248e607c7eae59d6bb2._comment
new file mode 100644
index 0000000000..ff17520ff4
--- /dev/null
+++ b/doc/forum/Remote__39__s_git-annex-shell_not_detected/comment_1_8354782cf0fab248e607c7eae59d6bb2._comment
@@ -0,0 +1,12 @@
+[[!comment format=mdwn
+ username="joey"
+ subject="""comment 1"""
+ date="2015-08-09T14:13:50Z"
+ content="""
+I'd guess that however you have installed git-annex does not result in
+git-annex-shell being in PATH until .bashrc or similar is loaded. 
+
+Since bash refuses to load any such files when a command like "ssh thehost
+git-annex-shell" is run, you can't install git-annex-shell that way. You
+have to install it somewhere in the default PATH.
+"""]]
diff --git a/doc/forum/Remote__39__s_git-annex-shell_not_detected/comment_2_35e5d9b427963fc7b9618eb522bd1273._comment b/doc/forum/Remote__39__s_git-annex-shell_not_detected/comment_2_35e5d9b427963fc7b9618eb522bd1273._comment
new file mode 100644
index 0000000000..825e749f1c
--- /dev/null
+++ b/doc/forum/Remote__39__s_git-annex-shell_not_detected/comment_2_35e5d9b427963fc7b9618eb522bd1273._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="joey"
+ subject="""comment 2"""
+ date="2015-08-09T21:20:05Z"
+ content="""
+I've added [[tips/get_git-annex-shell_into_PATH]] explaining this problem
+and solutions in detail.
+"""]]
diff --git a/doc/forum/Remote__39__s_git-annex-shell_not_detected/comment_2_7fb6168197c3f16de9775a60b7941cde._comment b/doc/forum/Remote__39__s_git-annex-shell_not_detected/comment_2_7fb6168197c3f16de9775a60b7941cde._comment
new file mode 100644
index 0000000000..2ee4e40349
--- /dev/null
+++ b/doc/forum/Remote__39__s_git-annex-shell_not_detected/comment_2_7fb6168197c3f16de9775a60b7941cde._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="https://openid.stackexchange.com/user/1938c4c8-6bd7-4fbe-8784-f00dc5dd9fcb"
+ nickname="johnny-tedesco"
+ subject="comment 2"
+ date="2015-08-09T19:56:52Z"
+ content="""
+Thanks! I'll look into it
+"""]]
diff --git a/doc/forum/Remote_in_webapp_shows_syncing_disabled_but_is_enabled___40__maybe__41__.mdwn b/doc/forum/Remote_in_webapp_shows_syncing_disabled_but_is_enabled___40__maybe__41__.mdwn
new file mode 100644
index 0000000000..c494dc9f88
--- /dev/null
+++ b/doc/forum/Remote_in_webapp_shows_syncing_disabled_but_is_enabled___40__maybe__41__.mdwn
@@ -0,0 +1,5 @@
+As the heading suggests. Here's a gif of my issue:
+
+View post on imgur.com
+
+The other remotes don't seem to be working either.
diff --git a/doc/forum/Remote_in_webapp_shows_syncing_disabled_but_is_enabled___40__maybe__41__/comment_1_04144280b69a7ba0cfa4fa3e1b5ef4ec._comment b/doc/forum/Remote_in_webapp_shows_syncing_disabled_but_is_enabled___40__maybe__41__/comment_1_04144280b69a7ba0cfa4fa3e1b5ef4ec._comment
new file mode 100644
index 0000000000..a39edfacdb
--- /dev/null
+++ b/doc/forum/Remote_in_webapp_shows_syncing_disabled_but_is_enabled___40__maybe__41__/comment_1_04144280b69a7ba0cfa4fa3e1b5ef4ec._comment
@@ -0,0 +1,7 @@
+[[!comment format=mdwn
+ username="openmedi"
+ subject="comment 1"
+ date="2016-05-10T14:40:18Z"
+ content="""
+This was solved by letting the assistant repair the repo (which took almost a whole night!). [See also.](http://git-annex.branchable.com/forum/How_to_find_out_if_assistant_is_still_repairing__63__/)
+"""]]
diff --git a/doc/forum/Remote_in_webapp_shows_syncing_disabled_but_is_enabled___40__maybe__41__/comment_2_2c0340e5eac6d712045614886a8403a5._comment b/doc/forum/Remote_in_webapp_shows_syncing_disabled_but_is_enabled___40__maybe__41__/comment_2_2c0340e5eac6d712045614886a8403a5._comment
new file mode 100644
index 0000000000..7498b1635d
--- /dev/null
+++ b/doc/forum/Remote_in_webapp_shows_syncing_disabled_but_is_enabled___40__maybe__41__/comment_2_2c0340e5eac6d712045614886a8403a5._comment
@@ -0,0 +1,22 @@
+[[!comment format=mdwn
+ username="joey"
+ subject="""comment 2"""
+ date="2016-05-12T19:36:19Z"
+ content="""
+Enormous animated gif shows:
+
+1. List of remotes. "Kapsel" remote has "Syncing disabled"
+2. Apparently user clicks on that, although it's not entirely clear.
+   Page seems to reload and still has "Syncing disabled" for Kapsel.
+3. User goed to Edit repository for Kapsel. That page has "Syncing enabled"
+   checked.
+4. User saves and remote list still has it disabled.
+
+I guess my suggestion is, first to tell us what version of git-annex you're
+using, and second, take a look at .git/annex/daemon.log inside the git
+repository. There's probaby something to help narrow the problem down in
+the log.
+
+It would probably also help to paste the .git/config stanza for
+the remote.
+"""]]
diff --git a/doc/forum/Remote_in_webapp_shows_syncing_disabled_but_is_enabled___40__maybe__41__/comment_3_2b1ac09cb8b5f5ec62a201785c2f6531._comment b/doc/forum/Remote_in_webapp_shows_syncing_disabled_but_is_enabled___40__maybe__41__/comment_3_2b1ac09cb8b5f5ec62a201785c2f6531._comment
new file mode 100644
index 0000000000..9e2745c107
--- /dev/null
+++ b/doc/forum/Remote_in_webapp_shows_syncing_disabled_but_is_enabled___40__maybe__41__/comment_3_2b1ac09cb8b5f5ec62a201785c2f6531._comment
@@ -0,0 +1,23 @@
+[[!comment format=mdwn
+ username="joey"
+ subject="""comment 3"""
+ date="2016-05-12T19:52:49Z"
+ content="""
+Looking at the code, the RepoList looks at syncRemotes to decide if a
+remote is syncing or not. The edit form instead looks at remoteAnnexSync
+from the remote's gitconfig. So, seems syncRemotes for some reason does
+not include this remote, despite its gitconfig saying to sync with it.
+
+Well, calcSyncRemotes starts with remoteAnnexSync but also excludes Dead
+repos. And indeed, when a repo is dead, it behaves as shown.
+
+So, `git annex semitrust Kapsel` should get it back to syncing with that
+remote. Assuming it's not dead for a reason.
+
+The webapp UI doesn't currently indicate when a remote is dead. I think you
+can only get into this state by running `git annex dead` at the command line
+though.
+
+Simple fix is to not make the webapp display dead remotes in the
+RepoList at all. Done.
+"""]]
diff --git a/doc/forum/Remote_in_webapp_shows_syncing_disabled_but_is_enabled___40__maybe__41__/comment_4_a8e68a41e069803d89f18d1645876298._comment b/doc/forum/Remote_in_webapp_shows_syncing_disabled_but_is_enabled___40__maybe__41__/comment_4_a8e68a41e069803d89f18d1645876298._comment
new file mode 100644
index 0000000000..b303c3afae
--- /dev/null
+++ b/doc/forum/Remote_in_webapp_shows_syncing_disabled_but_is_enabled___40__maybe__41__/comment_4_a8e68a41e069803d89f18d1645876298._comment
@@ -0,0 +1,7 @@
+[[!comment format=mdwn
+ username="openmedi"
+ subject="comment 4"
+ date="2016-05-14T13:21:04Z"
+ content="""
+Indeed it was a question of trust. Something that the assistant should notify me about. Maybe it should even be a setting in the assistant. No feedback here is just not a good idea.
+"""]]
diff --git a/doc/forum/Remote_listed_in_whereis_but_not_in_info.mdwn b/doc/forum/Remote_listed_in_whereis_but_not_in_info.mdwn
new file mode 100644
index 0000000000..d7f7f18e0d
--- /dev/null
+++ b/doc/forum/Remote_listed_in_whereis_but_not_in_info.mdwn
@@ -0,0 +1,29 @@
+I came across the following strange behaviour:
+
+    $ git annex whereis "2013-WS/ecl/Algorithms for Scoring Coreference Chains.pdf"
+    whereis 2013-WS/ecl/Algorithms for Scoring Coreference Chains.pdf (4 copies) 
+   	    04140d86-2ad5-4807-a789-f478dbf477c7 -- [mojzesz]
+   	    622fce61-6702-448f-8eee-9a31d8a67e14 -- here
+   	    8bb266ed-453d-4489-9d8a-de38b2bc77c2
+   	    d8149441-8b4d-4d37-bed4-c0f709165f32 -- [alonzo]
+    ok
+
+I have no idea what that remote without a name is. Is there a way to find that out?
+
+Plus, it is not shown by
+
+    $ git annex info
+    repository mode: indirect
+    trusted repositories: 0
+    semitrusted repositories: 3
+	    04140d86-2ad5-4807-a789-f478dbf477c7 -- [mojzesz]
+ 	    622fce61-6702-448f-8eee-9a31d8a67e14 -- here
+ 	    d8149441-8b4d-4d37-bed4-c0f709165f32 -- [alonzo]
+    untrusted repositories: 4
+	    00000000-0000-0000-0000-000000000001 -- web
+ 	    00000000-0000-0000-0000-000000000002 -- bittorrent
+ 	    11d4b299-0170-49b3-8b71-7ea2c47f212b -- nexus5
+ 	    dd22c018-65f8-4fa7-b880-48616016e272 -- miracle
+    ...
+
+Also, is there a way to mark that remote as dead?
diff --git a/doc/forum/Remote_listed_in_whereis_but_not_in_info/comment_1_85cc355213f894e8c1aeece432de67c3._comment b/doc/forum/Remote_listed_in_whereis_but_not_in_info/comment_1_85cc355213f894e8c1aeece432de67c3._comment
new file mode 100644
index 0000000000..1db8b0c196
--- /dev/null
+++ b/doc/forum/Remote_listed_in_whereis_but_not_in_info/comment_1_85cc355213f894e8c1aeece432de67c3._comment
@@ -0,0 +1,20 @@
+[[!comment format=mdwn
+ username="joey"
+ subject="""comment 1"""
+ date="2017-11-07T18:05:49Z"
+ content="""
+For this to happen, the repository must somehow not be listed in uuid.log,
+but the location log for the file includes its uuid. Normally, any time a
+git-annex repo is initialized, it gets recorded in uuid.log.
+
+One way that could have happened is if `git annex reinit` was run
+with some uuid that was not known in uuid.log. Or, if `git config
+annex.uuid` were manually set.
+
+I don't think it's a problem, other than you not knowing where this
+repository is -- unless some file is only present in that repository. 
+
+You can run `git annex dead` with the uuid as a parameter to mark it dead,
+or you could `git annex describe` with the uuid as a parameter to give
+it a description.
+"""]]
diff --git a/doc/forum/Remote_listed_in_whereis_but_not_in_info/comment_2_ddede3236f1d91ac4f0117b76267a021._comment b/doc/forum/Remote_listed_in_whereis_but_not_in_info/comment_2_ddede3236f1d91ac4f0117b76267a021._comment
new file mode 100644
index 0000000000..736f98ef6a
--- /dev/null
+++ b/doc/forum/Remote_listed_in_whereis_but_not_in_info/comment_2_ddede3236f1d91ac4f0117b76267a021._comment
@@ -0,0 +1,12 @@
+[[!comment format=mdwn
+ username="robert.schuetz@7942237bf71a2ae4f5d3cb047d167612b8c9e311"
+ nickname="robert.schuetz"
+ avatar="http://cdn.libravatar.org/avatar/89879460a9e84b9c736d982d9489d3d9"
+ subject="comment 2"
+ date="2017-11-07T19:14:18Z"
+ content="""
+Surprisingly, I can't use the UUID:
+
+    $ git annex dead 8bb266ed-453d-4489-9d8a-de38b2bc77c2
+    dead 8bb266ed-453d-4489-9d8a-de38b2bc77c2 git-annex: there is no available git remote named \"8bb266ed-453d-4489-9d8a-de38b2bc77c2\"
+"""]]
diff --git a/doc/forum/Remote_listed_in_whereis_but_not_in_info/comment_3_607d5b01eb95906391337f7351e0193c._comment b/doc/forum/Remote_listed_in_whereis_but_not_in_info/comment_3_607d5b01eb95906391337f7351e0193c._comment
new file mode 100644
index 0000000000..f2ba0e6074
--- /dev/null
+++ b/doc/forum/Remote_listed_in_whereis_but_not_in_info/comment_3_607d5b01eb95906391337f7351e0193c._comment
@@ -0,0 +1,15 @@
+[[!comment format=mdwn
+ username="joey"
+ subject="""comment 3"""
+ date="2017-11-07T19:35:17Z"
+ content="""
+Ah right, it looks in `uuid.log` to make sure the UUID provided is
+valid. Bit of a catch 22 there.
+
+Here's a way that will work:
+
+	git config remote.unknown.url foo
+	git config remote.unknown.annex-uuid 8bb266ed-453d-4489-9d8a-de38b2bc77c2
+	git annex dead unknown
+	git remote rm unknown
+"""]]
diff --git a/doc/forum/Remote_server_only_for_the_git_repository.mdwn b/doc/forum/Remote_server_only_for_the_git_repository.mdwn
new file mode 100644
index 0000000000..12f4c5c78f
--- /dev/null
+++ b/doc/forum/Remote_server_only_for_the_git_repository.mdwn
@@ -0,0 +1,3 @@
+Hi, I'm using git-annex 5.2014012 with the webapp and I've added two repositories: a remote server and a box.com account. When I add a file it is uploaded properly to both remotes as you would expect.
+
+The problem is that I would like the files to only be uploaded to box.com, and the remote server only store the git repository. Is there any way of saying to git-annex to not sync files to a remote server?
diff --git a/doc/forum/Remote_server_only_for_the_git_repository/comment_1_d4d8d8cfebf9a98ca8878c5684d5bb50._comment b/doc/forum/Remote_server_only_for_the_git_repository/comment_1_d4d8d8cfebf9a98ca8878c5684d5bb50._comment
new file mode 100644
index 0000000000..c97184cb82
--- /dev/null
+++ b/doc/forum/Remote_server_only_for_the_git_repository/comment_1_d4d8d8cfebf9a98ca8878c5684d5bb50._comment
@@ -0,0 +1,10 @@
+[[!comment format=mdwn
+ username="https://me.yahoo.com/a/FHnTlSBo1eCGJRwueeKeB6.RCaPbGMPr5jxx8A--#ce0d8"
+ nickname="Hamza"
+ subject="comment 1"
+ date="2014-02-06T13:30:33Z"
+ content="""
+You can tell git-annex to ignore the repo (assuming git repo is called origin)
+
+     git config remote.origin.annex-ignore true
+"""]]
diff --git a/doc/forum/Remote_server_only_for_the_git_repository/comment_2_a62dec8ab98ac7bd65059a9e425a01e2._comment b/doc/forum/Remote_server_only_for_the_git_repository/comment_2_a62dec8ab98ac7bd65059a9e425a01e2._comment
new file mode 100644
index 0000000000..d02994628c
--- /dev/null
+++ b/doc/forum/Remote_server_only_for_the_git_repository/comment_2_a62dec8ab98ac7bd65059a9e425a01e2._comment
@@ -0,0 +1,9 @@
+[[!comment format=mdwn
+ username="https://www.google.com/accounts/o8/id?id=AItOawk9nck8WX8-ADF3Fdh5vFo4Qrw1I_bJcR8"
+ nickname="Jon Ander"
+ subject="comment 2"
+ date="2014-02-07T11:36:32Z"
+ content="""
+Thanks, that seams to be exactly what I need.
+Any change of adding this option to the webapp?
+"""]]
diff --git a/doc/forum/Remote_type_to_make_portable_HDD_like_normal_but_indexed.mdwn b/doc/forum/Remote_type_to_make_portable_HDD_like_normal_but_indexed.mdwn
new file mode 100644
index 0000000000..152990af2c
--- /dev/null
+++ b/doc/forum/Remote_type_to_make_portable_HDD_like_normal_but_indexed.mdwn
@@ -0,0 +1,13 @@
+Hello,
+
+I am pretty paranoid when it comes to my backups. I have a couple of live backups, but also take snapshots now and again and store these on offline hard drives in different places. I would like to use git-annex to keep track of what backup files I have and where. I encrypt all of my backup files, so the hard drive/repository does not need to be encrypted and I would prefer that so that there were fewer things that can go wrong -- I ideally want the files just sitting on the disk as a list of .gpg files, but with git annex able to tell me what I put on there/that the file I need is on there/count it towards any “number of copies” calculations.
+
+I have had a quick play with git annex and it seems I only have a couple of options:
+1) an (encrypted or unencrypted) bare repository, which I’m not keen on because I can’t see my files happily sitting there; or
+2) a clone of the whole git repository, but I’m concerned that this means any access details to things like my box.com account or whatever would be sitting on the drive in plaintext.
+
+Is there something else that I’ve missed or misunderstood?
+
+Many thanks!
+
+Aaron
diff --git a/doc/forum/Remote_type_to_make_portable_HDD_like_normal_but_indexed/comment_1_7caf32756f799b67b0c0588ea5e17a20._comment b/doc/forum/Remote_type_to_make_portable_HDD_like_normal_but_indexed/comment_1_7caf32756f799b67b0c0588ea5e17a20._comment
new file mode 100644
index 0000000000..26b472ef77
--- /dev/null
+++ b/doc/forum/Remote_type_to_make_portable_HDD_like_normal_but_indexed/comment_1_7caf32756f799b67b0c0588ea5e17a20._comment
@@ -0,0 +1,9 @@
+[[!comment format=mdwn
+ username="Aaron"
+ subject="Any thoughts Joey?"
+ date="2016-09-10T15:15:44Z"
+ content="""
+Sorry to bump, but I'm trying to figure out what I'm going to use for a disk that I want to leave on the other side of the world in the next few weeks :).
+
+Any suggestions, Joey?
+"""]]
diff --git a/doc/forum/Remote_type_to_make_portable_HDD_like_normal_but_indexed/comment_2_36c202f5883b66a431ae5e72c5ecede1._comment b/doc/forum/Remote_type_to_make_portable_HDD_like_normal_but_indexed/comment_2_36c202f5883b66a431ae5e72c5ecede1._comment
new file mode 100644
index 0000000000..53f94b677e
--- /dev/null
+++ b/doc/forum/Remote_type_to_make_portable_HDD_like_normal_but_indexed/comment_2_36c202f5883b66a431ae5e72c5ecede1._comment
@@ -0,0 +1,12 @@
+[[!comment format=mdwn
+ username="joey"
+ subject="""comment 2"""
+ date="2016-09-21T19:54:43Z"
+ content="""
+Suggest you use git-remote-gcrypt to encrypt the git repository stored on
+the backup drive. This will encrypt the whole git repository with
+your gpg key, and git-annex can also use it as a remote, and will encrypt
+the annexed files stored there too.
+
+See [[tips/fully_encrypted_git_repositories_with_gcrypt]].
+"""]]
diff --git a/doc/forum/Remove_variants_from_all_repositories___40__and_possibly_restore_the_original_naming__41__.mdwn b/doc/forum/Remove_variants_from_all_repositories___40__and_possibly_restore_the_original_naming__41__.mdwn
new file mode 100644
index 0000000000..8a62a6e406
--- /dev/null
+++ b/doc/forum/Remove_variants_from_all_repositories___40__and_possibly_restore_the_original_naming__41__.mdwn
@@ -0,0 +1 @@
+I, by mistake, added some files directly to the remote locations and my local repository, afterwards, using `git annex get '*'` to get "all" the new files from the remotes I created some `file.variant-xxxx.ext` files. For me this is undesirable. I'm sure the files are almost the same (I'm presuming some metadata got changed, they are audio files) so I want to remove one keeping the other and removing the `variant-xxxx` string. Is there a way this is possible without manually drooping/renaming all these files?
diff --git a/doc/forum/Remove_variants_from_all_repositories___40__and_possibly_restore_the_original_naming__41__/comment_1_dfc64d8618e07177bb196d9af4474698._comment b/doc/forum/Remove_variants_from_all_repositories___40__and_possibly_restore_the_original_naming__41__/comment_1_dfc64d8618e07177bb196d9af4474698._comment
new file mode 100644
index 0000000000..7f31058901
--- /dev/null
+++ b/doc/forum/Remove_variants_from_all_repositories___40__and_possibly_restore_the_original_naming__41__/comment_1_dfc64d8618e07177bb196d9af4474698._comment
@@ -0,0 +1,19 @@
+[[!comment format=mdwn
+ username="joey"
+ subject="""comment 1"""
+ date="2015-01-05T20:12:56Z"
+ content="""
+Probably the best thing to do is to use `git log --stat` 
+to find the commits that were made on the remote that added the conflicting
+files. Also find the merge commit that created the variant
+files. You can then `git revert` the merge commit, and follow up by `git
+revert` the commits that added the conflicting files. This will bring your
+working tree back to the state it was in originally.
+
+Alternatively, if your current repo has the contents present for 
+the variants of the files you want to keep, while the remote repos contain
+the contents of the variants you want to delete, you could use this command
+to delete the variants that don't have their contents in the local repo:
+
+	git annex find --include='*.variant-*' --not --in here --print0 | xargs -0 rm
+"""]]
diff --git a/doc/forum/Removing_files_not_found_by_git_annex_unused.mdwn b/doc/forum/Removing_files_not_found_by_git_annex_unused.mdwn
new file mode 100644
index 0000000000..ce0aedc51d
--- /dev/null
+++ b/doc/forum/Removing_files_not_found_by_git_annex_unused.mdwn
@@ -0,0 +1,29 @@
+Hi,
+
+I've removed some large files with git remove, but seem to be unable to remove the corresponding annex content.
+
+Example:
+
+kheymann@corax:~/annex$ find -name "*s24576--10daa3d9007edad720dc057e4272a9c6cda930bef34a83e3bc1d93f1c55b9cac" 
+
+./.git/annex/objects/jF/j7/SHA256-s24576--10daa3d9007edad720dc057e4272a9c6cda930bef34a83e3bc1d93f1c55b9cac
+
+./.git/annex/objects/jF/j7/SHA256-s24576--10daa3d9007edad720dc057e4272a9c6cda930bef34a83e3bc1d93f1c55b9cac/SHA256-s24576--10daa3d9007edad720dc057e4272a9c6cda930bef34a83e3bc1d93f1c55b9cac
+
+kheymann@corax:~/annex$ git annex dropkey -vvv --backend SHA256 s24576--10daa3d9007edad720dc057e4272a9c6cda930bef34a83e3bc1d93f1c55b9cac
+
+No output but the files remain in the annex. Git annex fsck and git annex unused run without listing files to be removes. What can I do apart from deleting the files manually from the annex?
+
+Some info:
+
+    kheymann@corax:~/annex$ git annex version
+    git-annex version: 3.20121017
+    local repository version: 3
+    default repository version: 3
+    supported repository versions: 3
+    upgrade supported from repository versions: 0 1 2
+
+Any hints?
+
+Best, 
+Karsten
diff --git a/doc/forum/Removing_files_not_found_by_git_annex_unused/comment_1_420c6230e68de0a0ac7d7da91ac60801._comment b/doc/forum/Removing_files_not_found_by_git_annex_unused/comment_1_420c6230e68de0a0ac7d7da91ac60801._comment
new file mode 100644
index 0000000000..f932f779cf
--- /dev/null
+++ b/doc/forum/Removing_files_not_found_by_git_annex_unused/comment_1_420c6230e68de0a0ac7d7da91ac60801._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="https://www.google.com/accounts/o8/id?id=AItOawn7gQ1zZDdWhXy9H51W2krZYShNmKL3qfM"
+ nickname="Karsten"
+ subject="comment 1"
+ date="2012-11-09T19:37:24Z"
+ content="""
+Please ignore my previous post, by following the instructions in the comment in http://git-annex.branchable.com/walkthrough/unused_data/ I was able to get rid of the data.
+"""]]
diff --git a/doc/forum/Removing_git-annex_repo/comment_1_58fcceb96647a8c7f33d188ae908f3bd._comment b/doc/forum/Removing_git-annex_repo/comment_1_58fcceb96647a8c7f33d188ae908f3bd._comment
new file mode 100644
index 0000000000..d03ca6b4e0
--- /dev/null
+++ b/doc/forum/Removing_git-annex_repo/comment_1_58fcceb96647a8c7f33d188ae908f3bd._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="http://joeyh.name/"
+ ip="209.250.56.144"
+ subject="comment 1"
+ date="2014-09-25T15:44:16Z"
+ content="""
+chmod u+w -R ~/annex/.git
+"""]]
diff --git a/doc/forum/Rename_local_repository_for_git-annex-info.mdwn b/doc/forum/Rename_local_repository_for_git-annex-info.mdwn
new file mode 100644
index 0000000000..1c26974314
--- /dev/null
+++ b/doc/forum/Rename_local_repository_for_git-annex-info.mdwn
@@ -0,0 +1,32 @@
+I initialized local repository with
+
+    git-annex init $HOSTNAME --version=6
+
+unfortunately I didn't change HOSTNAME on a new machine and it was 'localhost.localdomain'. I didn't notice that before I cloned a git-annex repository.
+
+Now in the remote repository when I run `git-annex info` (and same in local repository), I see
+
+    $ git-annex info
+    ...
+    semitrusted repositories: 6
+	00000000-0000-0000-0000-000000000001 -- web
+ 	00000000-0000-0000-0000-000000000002 -- bittorrent
+ 	0085(maybe it's not secure to write it on the internet)-e8f803a -- localhost.localdomain
+    ...
+
+(and other repositories. By the way, I never initialized 'web' and 'bittorent', where did they get from?)
+
+I would like 'localhost.localdomain' to become my real $HOSTNAME, so that I would distinguish that machine. How could I do that?
+
+I found [How to rename a remote](https://git-annex.branchable.com/forum/How_to_rename_a_remote__63__/), but my 'localhost' is not listed in git-remotes. 
+
+I grep-ed .git for 'localhost.localdomain', and changed `.git/COMMIT_EDITMSG`. However, after running git-annex sync it returns to 'localhost.localdomain'.
+
+    $ more .git/COMMIT_EDITMSG
+    git-annex in Acer
+    $ git-annex sync
+    ...
+    $ more .git/COMMIT_EDITMSG
+    git-annex in localhost.localdomain
+
+I would like to change 'localhost' to my real machine name both on the remote repository from which I cloned and on local repository. Thank you.
diff --git a/doc/forum/Rename_local_repository_for_git-annex-info/comment_1_6df9b666f7d948b9e03b2ecf9858f6c5._comment b/doc/forum/Rename_local_repository_for_git-annex-info/comment_1_6df9b666f7d948b9e03b2ecf9858f6c5._comment
new file mode 100644
index 0000000000..5c08afa2a9
--- /dev/null
+++ b/doc/forum/Rename_local_repository_for_git-annex-info/comment_1_6df9b666f7d948b9e03b2ecf9858f6c5._comment
@@ -0,0 +1,12 @@
+[[!comment format=mdwn
+ username="joey"
+ subject="""comment 1"""
+ date="2017-12-05T17:32:58Z"
+ content="""
+That's not the name of the repository, it's a description of the
+repository.
+
+You can use `git annex describe` to adjust the description of that
+or any repository. This will affect all clones of the repository
+once the description change is synced out to them.
+"""]]
diff --git a/doc/forum/Replace_glacier-cli__63__.mdwn b/doc/forum/Replace_glacier-cli__63__.mdwn
new file mode 100644
index 0000000000..3557cc2529
--- /dev/null
+++ b/doc/forum/Replace_glacier-cli__63__.mdwn
@@ -0,0 +1,13 @@
+I was looking for a good way to archive files to Glacier, and git-annex seems like an excellent approach.
+
+After struggling for a while with the glacier-cli project, I just don't feel that it is well maintained.
+
+ - I have to manually symlink it in place of its dependency's glacier binary.
+ - There are patches to boto that are required for it to work, which are have never been merged. And it seems from the errors I am getting that similar edits are now required in many more places.
+ - It is locked into supporting only an older version of boto.
+
+That just seems like not a great place to be starting out with an archiving project.
+
+I was wondering if you thought it might be feasible to get rid of that dependency with something else, perhaps this haskell aws project that seems to be actively maintained?
+
+https://github.com/aristidb/aws
diff --git a/doc/forum/Replace_glacier-cli__63__/comment_1_318cd0ca9f4e997161ed22fffa5d5da3._comment b/doc/forum/Replace_glacier-cli__63__/comment_1_318cd0ca9f4e997161ed22fffa5d5da3._comment
new file mode 100644
index 0000000000..f87c8e4f6d
--- /dev/null
+++ b/doc/forum/Replace_glacier-cli__63__/comment_1_318cd0ca9f4e997161ed22fffa5d5da3._comment
@@ -0,0 +1,18 @@
+[[!comment format=mdwn
+ username="joey"
+ subject="""comment 1"""
+ date="2018-02-08T17:17:10Z"
+ content="""
+There's been an open feature request on the haskell aws
+for 5 years to support glacier 
+at this point I am doubtful anything will ever happen there.
+
+The most likely approach seems to be to use the amazonka library, which
+supports S3 with a Glacier lifecycle. git-annex's S3 remote could be
+rewritten to use that.
+
+I think it should be possible to set up a S3 bucket and configure it with
+some web tool to have a Glacier lifecycle, and then use the existing
+git-annex S3 support to access it. Except for the problem documented at
+[[todo/wishlist__58___Restore_s3_files_moved_to_Glacier]].
+"""]]
diff --git a/doc/forum/Replace_glacier-cli__63__/comment_2_c91e393951825cdcceac845e234d7497._comment b/doc/forum/Replace_glacier-cli__63__/comment_2_c91e393951825cdcceac845e234d7497._comment
new file mode 100644
index 0000000000..ac7596a4b3
--- /dev/null
+++ b/doc/forum/Replace_glacier-cli__63__/comment_2_c91e393951825cdcceac845e234d7497._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="moaxey"
+ avatar="http://cdn.libravatar.org/avatar/06abae5b84ce9bbcd39c1cbf49817d29"
+ subject="Thanks"
+ date="2018-02-19T20:57:09Z"
+ content="""
+I have it working smoothly now with glacier-cli.
+"""]]
diff --git a/doc/forum/Replace_glacier-cli__63__/comment_3_405aad8dd28ea035226f7fda470fde33._comment b/doc/forum/Replace_glacier-cli__63__/comment_3_405aad8dd28ea035226f7fda470fde33._comment
new file mode 100644
index 0000000000..bcabcb13e2
--- /dev/null
+++ b/doc/forum/Replace_glacier-cli__63__/comment_3_405aad8dd28ea035226f7fda470fde33._comment
@@ -0,0 +1,10 @@
+[[!comment format=mdwn
+ username="andrew"
+ avatar="http://cdn.libravatar.org/avatar/acc0ece1eedf07dd9631e7d7d343c435"
+ subject="switching to amazonka"
+ date="2018-02-21T18:15:05Z"
+ content="""
+Switching to amazonka would have the benefit of supporting SSL traffic for S3, which is a feature I would enjoy (see git-annex forum thread [No SSL traffic for S3?](http://git-annex.branchable.com/forum/No_SSL_traffic_for_S3__63__/)). I have no experience with the amazonka library though, so I can't comment on it.
+
+—Andrew
+"""]]
diff --git a/doc/forum/Replace_glacier-cli__63__/comment_4_a863eb91006df199fcdbc98f42b2d40b._comment b/doc/forum/Replace_glacier-cli__63__/comment_4_a863eb91006df199fcdbc98f42b2d40b._comment
new file mode 100644
index 0000000000..1a30822dee
--- /dev/null
+++ b/doc/forum/Replace_glacier-cli__63__/comment_4_a863eb91006df199fcdbc98f42b2d40b._comment
@@ -0,0 +1,9 @@
+[[!comment format=mdwn
+ username="abimelech@632162f535b3bbc90e3336b14ceef969c41c05a2"
+ nickname="abimelech"
+ avatar="http://cdn.libravatar.org/avatar/057e618fbf211a3733e9fef66030c0b9"
+ subject="I've been using amazonka-glacier to stream uploads to glacier"
+ date="2018-06-27T21:50:45Z"
+ content="""
+Could write a command-line app for it. I also have a couple open PRs against amazonka to fix the glacier support.
+"""]]
diff --git a/doc/forum/Replace_glacier-cli__63__/comment_5_fda6c754437385afa0851e680bf17cf0._comment b/doc/forum/Replace_glacier-cli__63__/comment_5_fda6c754437385afa0851e680bf17cf0._comment
new file mode 100644
index 0000000000..3da6bc450b
--- /dev/null
+++ b/doc/forum/Replace_glacier-cli__63__/comment_5_fda6c754437385afa0851e680bf17cf0._comment
@@ -0,0 +1,15 @@
+[[!comment format=mdwn
+ username="joey"
+ subject="""comment 5"""
+ date="2018-07-02T16:39:16Z"
+ content="""
+@abimelech, why don't you work up a branch that makes the glacier special
+remote be implemented with amazonka. If you want to start with a
+separate utility that git-annex runs, that would be ok, but I think that
+the goal should be for git-annex to end up using amazonka itself.
+
+Once that happens, switching the S3
+special remote from aws to amazonka would be a natural next step, which I'd
+be inclined to work on myself. Seems easier to take this path than an
+immediate leap to amazonka for everything.
+"""]]
diff --git a/doc/forum/Replace_glacier-cli__63__/comment_6_2e7b5b6cc7c2e33be36c3c510adec01c._comment b/doc/forum/Replace_glacier-cli__63__/comment_6_2e7b5b6cc7c2e33be36c3c510adec01c._comment
new file mode 100644
index 0000000000..074731304f
--- /dev/null
+++ b/doc/forum/Replace_glacier-cli__63__/comment_6_2e7b5b6cc7c2e33be36c3c510adec01c._comment
@@ -0,0 +1,10 @@
+[[!comment format=mdwn
+ username="abimelech@632162f535b3bbc90e3336b14ceef969c41c05a2"
+ nickname="abimelech"
+ avatar="http://cdn.libravatar.org/avatar/057e618fbf211a3733e9fef66030c0b9"
+ subject="comment 6"
+ date="2018-07-07T00:09:48Z"
+ content="""
+I've never used git-annex, so I'm not particularly enthusiastic about developing for it. If someone could describe what glacier actions are needed I could maybe code something up.
+In any case, wouldn't we want to wait until a version of amazonka with functioning glacier support is released? Unless you'd want to add my git fork of the repo as a dependency...
+"""]]
diff --git a/doc/forum/Repo_accessible_from___34__dumb__34___client_without_git-annex.mdwn b/doc/forum/Repo_accessible_from___34__dumb__34___client_without_git-annex.mdwn
new file mode 100644
index 0000000000..bcf2a87a39
--- /dev/null
+++ b/doc/forum/Repo_accessible_from___34__dumb__34___client_without_git-annex.mdwn
@@ -0,0 +1,11 @@
+Hi,
+
+I'm trying to figure out if it's possible to have a remote that allows access from _clients without git-annex_. That is a remote type that presents itself as a client repo. I'm probably missing something obvious.
+
+Ideally I'd like to use my Nokia N900 phone as a ssh remote from which I can open files on any computer when the device is connected via usb in mass storage mode. This would allow me to access my files when I'm using a computer at work or at a friends house without installing git-annex. The phone has an ext3 partition so links are possible. Naturally I'd have to ensure the files I want are on the device before I leave the house.
+
+I'd also like to be able to scp files from my ssh remote when on git-annex less devices. Unfortunately the ssh/rsync repo presents no links or normal filenames.
+
+Git annex, and the assistant is working really smoothly between my local computers and my server on the internet. Setting things up with the webapp is super simple. Great work!
+
+
diff --git a/doc/forum/Repo_accessible_from___34__dumb__34___client_without_git-annex/comment_1_077c492fd37d335f74a5c886ff0d524f._comment b/doc/forum/Repo_accessible_from___34__dumb__34___client_without_git-annex/comment_1_077c492fd37d335f74a5c886ff0d524f._comment
new file mode 100644
index 0000000000..4e724ff55e
--- /dev/null
+++ b/doc/forum/Repo_accessible_from___34__dumb__34___client_without_git-annex/comment_1_077c492fd37d335f74a5c886ff0d524f._comment
@@ -0,0 +1,32 @@
+[[!comment format=mdwn
+ username="http://edheil.wordpress.com/"
+ ip="99.54.57.201"
+ subject="comment 1"
+ date="2013-01-20T20:20:06Z"
+ content="""
+This is off the top of my head, so I may be wrong on details:
+
+An ssh remote, or a USB remote, when you create it with the assistant, is a bare repo.  The files in it are not visible or accessible except through a clone of it (such as your client repo).
+
+You *could* make an SSH remote or USB remote as a *non-bare repo*, in which case the files within it would be visible and accessible just like on a client repo.  You could set that up using the command line; the assistant doesn't do it, but once you got it set up on the command line the assistant would use it just like normal.
+
+There is a catch though.  Though potentially available, the files in that repo don't actually show up until you run \"git annex sync\" in that repo.  Because git annex does not update the checked out branch of a remote.  So you'd have to cd into that repo manually and run \"git annex sync\" to make the latest-and-greatest files available.
+
+Here's how I'd do this...
+
+If the git annex assistant is running, turn it off.
+
+follow the instructions here: http://git-annex.branchable.com/walkthrough/  under \"adding a remote\" on your usb drive.  You now have a non-bare remote.
+
+Fire up the assistant and you'll see that that is now one of your remotes.  Configure it to \"backup\" or whatever you want.  Let the assistant fill it up with content.
+
+When you want to take it someplace, get on the command line and \"cd /wherever/my/usb/drive/is/annex\" and then do: \"git annex sync\" -- that will update the links to the content so you can actually see it.
+
+You now have all your content available on your USB drive.
+
+You can do a similar process to manually create a non-bare SSH remote, and you can ssh in and run \"git annex sync\" to update its links and make the content available, which you can then retrieve va scp or sftp, or for that matter, you could configure a web server or WebDAV to serve it up if you like.
+
+The only complication is running git-annex sync to update your links.
+
+Again, this is all off the top of my head, I haven't looked up or tested anything here so I welcome correction.  But I think what you want to do can be pretty straightforward if you're willing to go behind Assistant's back to do the setup work, and run \"git annex sync\" when you need to.
+"""]]
diff --git a/doc/forum/Repo_accessible_from___34__dumb__34___client_without_git-annex/comment_2_00e6576e3e60d2650461eeb0f918e6e5._comment b/doc/forum/Repo_accessible_from___34__dumb__34___client_without_git-annex/comment_2_00e6576e3e60d2650461eeb0f918e6e5._comment
new file mode 100644
index 0000000000..64f111921a
--- /dev/null
+++ b/doc/forum/Repo_accessible_from___34__dumb__34___client_without_git-annex/comment_2_00e6576e3e60d2650461eeb0f918e6e5._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="https://www.google.com/accounts/o8/id?id=AItOawn7gQ1zZDdWhXy9H51W2krZYShNmKL3qfM"
+ nickname="Karsten"
+ subject="comment 2"
+ date="2013-01-21T07:02:17Z"
+ content="""
+An alternative to running git annex sync in the non-bare remote is running an assistant there.
+"""]]
diff --git a/doc/forum/Repo_accessible_from___34__dumb__34___client_without_git-annex/comment_3_c36a9562c53ac683b62fc4471405aa2a._comment b/doc/forum/Repo_accessible_from___34__dumb__34___client_without_git-annex/comment_3_c36a9562c53ac683b62fc4471405aa2a._comment
new file mode 100644
index 0000000000..db7dec37f2
--- /dev/null
+++ b/doc/forum/Repo_accessible_from___34__dumb__34___client_without_git-annex/comment_3_c36a9562c53ac683b62fc4471405aa2a._comment
@@ -0,0 +1,15 @@
+[[!comment format=mdwn
+ username="https://www.google.com/accounts/o8/id?id=AItOawk3HGoDpnOPob5jOjvIootmkve1-nCpRiI"
+ nickname="Kalle"
+ subject="Thanks for the help"
+ date="2013-01-21T20:40:39Z"
+ content="""
+I take it all the above requires mounting an sshfs filsystem of the remote client to run the git-annex or the assistant \"locally\" on the desktop or laptop. Or am I missing something.
+
+For the usb usecase a normal mount would suffice of course.
+
+The lack of automation means running a rsync script from (or to) the phone is easier. Not as clever though :( and means running a separate \"system\" for distributing files to the phone.
+
+Thanks again!
+
+"""]]
diff --git a/doc/forum/Repository_backup.mdwn b/doc/forum/Repository_backup.mdwn
new file mode 100644
index 0000000000..39501f7005
--- /dev/null
+++ b/doc/forum/Repository_backup.mdwn
@@ -0,0 +1,5 @@
+What's the best way of backing up the git repository itself? I feel fairly comfortable using a special remote to create an offsite backup in S3 or Glacier or whatnot, but restoring from those still requires I have a working repository somewhere that can map chunks back to files, no? 
+
+I can probably just create a repo that has no content, tar it up, and store it on my backup medium (or on bitbucket or wherever), but that seems kinda hackish--it'd be nice to handle this within git-annex. In fact, it'd be really neat if I could handle this with any existing special remote--for example, if there were a way to commit and restore the git repo state (and symlink tree) to the remote. But that doesn't seem to exist. Is there a recommended approach for this? Am I just missing something obvious?
+
+Thanks. 
diff --git a/doc/forum/Repository_backup/comment_1_4d3197a1fefeb744e8b1e1b5fcf9b104._comment b/doc/forum/Repository_backup/comment_1_4d3197a1fefeb744e8b1e1b5fcf9b104._comment
new file mode 100644
index 0000000000..c668a4b6d7
--- /dev/null
+++ b/doc/forum/Repository_backup/comment_1_4d3197a1fefeb744e8b1e1b5fcf9b104._comment
@@ -0,0 +1,12 @@
+[[!comment format=mdwn
+ username="joey"
+ subject="""comment 1"""
+ date="2015-02-25T18:14:42Z"
+ content="""
+The best way is to make a git remote and let `git-annex sync`
+sync with it. You can put the git remote on github, gitorious, any
+server with ssh, any server with rsync, a local backup drive, etc.
+
+Special remotes are key/value storage; this is not a very good
+match for the access patterns needed for a git remote.
+"""]]
diff --git a/doc/forum/Repository_backup/comment_2_e362221ef3d13bd417b2b6b0c51738a8._comment b/doc/forum/Repository_backup/comment_2_e362221ef3d13bd417b2b6b0c51738a8._comment
new file mode 100644
index 0000000000..e84be7f22a
--- /dev/null
+++ b/doc/forum/Repository_backup/comment_2_e362221ef3d13bd417b2b6b0c51738a8._comment
@@ -0,0 +1,14 @@
+[[!comment format=mdwn
+ username="https://www.google.com/accounts/o8/id?id=AItOawnb2yfWzJ2lYQw1UTm6XVZ4y8qashNagZA"
+ nickname="Daniel"
+ subject="Re: comment1"
+ date="2015-02-26T12:48:56Z"
+ content="""
+Thanks for the tip!
+
+Yeah, I get that special remotes aren't meant for this, and backing up a bare (i.e. contentless) git repo somewhere else is pretty trivial. My only point was that it'd be nice to have a single unified offsite backup--if you backup the content to S3, say, and the git repo itself to Github, you now have two points of failure and, in some sense, twice the management hassle. (I now need both my Amazon and Github credentials to restore, for example.) 
+
+This seems somewhat undesirable in a backup system. 
+
+An easy way around this may be for me to just have a wrapper script that first backs up content to the special remote and then copies the .git repo (minus the objects) to the same place. So it's something I can easily enough work around. 
+"""]]
diff --git a/doc/forum/Repository_backup/comment_3_ead30baf0a6d718569feb4002ba19b9e._comment b/doc/forum/Repository_backup/comment_3_ead30baf0a6d718569feb4002ba19b9e._comment
new file mode 100644
index 0000000000..3db7f7d18d
--- /dev/null
+++ b/doc/forum/Repository_backup/comment_3_ead30baf0a6d718569feb4002ba19b9e._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="git-annex.branchable.com@69b3f2da2837a3d9de8eab0b93771008294b7d69"
+ nickname="git-annex.branchable.com"
+ subject="Does a tar of `.git` solve the problem"
+ date="2016-01-28T19:03:08Z"
+ content="""
+Just wondering if tarring .git (excluding .git/annex of course}) into a file in the root of the repo will serve this purpose? Seems like it would to me... How would one restore from this... it'd be (practically) a bare repo right?
+"""]]
diff --git a/doc/forum/Repository_backup/comment_4_6f20653dff70aaaea4460a5efb28081f._comment b/doc/forum/Repository_backup/comment_4_6f20653dff70aaaea4460a5efb28081f._comment
new file mode 100644
index 0000000000..fa44e679bf
--- /dev/null
+++ b/doc/forum/Repository_backup/comment_4_6f20653dff70aaaea4460a5efb28081f._comment
@@ -0,0 +1,13 @@
+[[!comment format=mdwn
+ username="joey"
+ subject="""comment 4"""
+ date="2016-01-29T18:02:14Z"
+ content="""
+Storing a tarball of the .git repository as a file inside the repository
+presents a chicken and egg problem when it comes to restoring it. You'll
+have to find the object file that git-annex has stored in the special
+remote. If the special remote is encrypted, you may not be able to decrypt
+it without the key stored in the repository. Etc.
+
+Better to make a clone of the git repository elsewhere.
+"""]]
diff --git a/doc/forum/Repository_backup/comment_5_287510b4e8b9043fe361b7c5dc1f4e9f._comment b/doc/forum/Repository_backup/comment_5_287510b4e8b9043fe361b7c5dc1f4e9f._comment
new file mode 100644
index 0000000000..2fac3bccc1
--- /dev/null
+++ b/doc/forum/Repository_backup/comment_5_287510b4e8b9043fe361b7c5dc1f4e9f._comment
@@ -0,0 +1,7 @@
+[[!comment format=mdwn
+ username="sts"
+ subject="comment 5"
+ date="2016-06-27T19:36:58Z"
+ content="""
+I always backup my git-annex repos using 'git bundle create ... --all' which is similar to a git clone, but much more convenient for backups, because it is just a single file :).
+"""]]
diff --git a/doc/forum/Require_a_local_copy_of_all_new_files_without_explicitly_calling___34__get__34___or___34__sync__34__.mdwn b/doc/forum/Require_a_local_copy_of_all_new_files_without_explicitly_calling___34__get__34___or___34__sync__34__.mdwn
new file mode 100644
index 0000000000..9c19b9b8c7
--- /dev/null
+++ b/doc/forum/Require_a_local_copy_of_all_new_files_without_explicitly_calling___34__get__34___or___34__sync__34__.mdwn
@@ -0,0 +1,9 @@
+Greetings.  Here is my desired usage scenario:
+
+I have setup a server and three laptops which I am syncing over ssh. Each laptop may access the server, but because different people are using each of the laptops, they may not sync between laptops for security/privacy.
+
+Each of the laptops will be adding content to the repository, and each of the laptops will be using the new content that the others have added.
+
+Currently, in order to make all content available to all users, I'm having to ssh into the server and use `git annex get .` every time new content is added from one of the laptops, which is a pain. Is there a way to require a local copy of all content in a given repository so that is pulled/pushed automatically? I would like for each laptop to be able to add new content to their copies of the repository and sync with the server, after which each other laptop can access the content through the server.
+
+Thanks!
diff --git a/doc/forum/Require_a_local_copy_of_all_new_files_without_explicitly_calling___34__get__34___or___34__sync__34__/comment_1_8adf9c6d2a3ef29120703bfa1b8f9ae2._comment b/doc/forum/Require_a_local_copy_of_all_new_files_without_explicitly_calling___34__get__34___or___34__sync__34__/comment_1_8adf9c6d2a3ef29120703bfa1b8f9ae2._comment
new file mode 100644
index 0000000000..12aef24737
--- /dev/null
+++ b/doc/forum/Require_a_local_copy_of_all_new_files_without_explicitly_calling___34__get__34___or___34__sync__34__/comment_1_8adf9c6d2a3ef29120703bfa1b8f9ae2._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="http://joeyh.name/"
+ ip="108.236.230.124"
+ subject="comment 1"
+ date="2014-07-21T19:46:44Z"
+ content="""
+This is generally handled by using either `git annex sync --content`, which will make the laptops upload new files to the server before they push the git repository to it, or by using the git-annex assistant (on the laptops or perhaps on the server) to automatically copy files. Either way, you configure [[preferred_content]] settings to control which repositories want which  files.
+"""]]
diff --git a/doc/forum/Restoring_files__63__.mdwn b/doc/forum/Restoring_files__63__.mdwn
new file mode 100644
index 0000000000..d0bf86d90b
--- /dev/null
+++ b/doc/forum/Restoring_files__63__.mdwn
@@ -0,0 +1,11 @@
+One of my users created a git annex repository a while ago and has since left the lab, and I think they messed it up badly.
+
+It looks like all of the data is present in the annex directory, but there are no files in main directory, other than the .git folder. The master branch seems to be gone, and the only branch left is called git-annex.
+
+git log gives the error "fatal: bad default revision 'HEAD'" when I first do git log, because the branch that I am on doesn't exist (the master branch). Changing to the git-annex branch works, but then I can't go back to the master branch because it doesn't exist.
+
+git annex unannex doesn't seem to do anything. Is there any way to get the data back following this kind of mess up?
+
+Thanks for your help
+
+-Mike
diff --git a/doc/forum/Restoring_files__63__/comment_1_735b98419b3ba6207cc364426b03ce74._comment b/doc/forum/Restoring_files__63__/comment_1_735b98419b3ba6207cc364426b03ce74._comment
new file mode 100644
index 0000000000..81cf3612b6
--- /dev/null
+++ b/doc/forum/Restoring_files__63__/comment_1_735b98419b3ba6207cc364426b03ce74._comment
@@ -0,0 +1,40 @@
+[[!comment format=mdwn
+ username="Chel"
+ subject="comment 1"
+ date="2015-06-30T06:32:59Z"
+ content="""
+If there are no branches, other than git-annex, then you do not have git history.
+
+If it is really the old repository with deleted branches and not a newly created
+one, then there is a possibility, that the git history has not been fully
+deleted/garbage-collected yet (i.e. there are old objects and packs in
+`.git/objects` and `.git/objects/pack`). It that case:
+
+1. *Do not run* git commands until you create a backup of the .git directory,
+   because some usual git commands automatically launch `git gc --auto`, which
+   removes some old unreachable objects (and maybe reflog entries).
+
+2. See if there are some reflogs of deleted branches or HEAD left in `.git/logs`.
+   Reflogs will give you commit ids that branches’ tips pointed to. But usually
+   reflogs are deleted with their branches.
+
+3. As the last resort, use `git fsck --dangling` to find objects, that may be
+   the commits of deleted branches. See also other options of `git fsck` command.
+
+Of course, all that is not necessary if you have a clone of the repo somewhere.
+Then just fetch the history from it.
+
+Git history will give you the history of modifications in the repository, the
+content of not annexed files (that were stored directly in git) and the names
+of annexed files (represented as symlinks).
+
+If all you need is just the contents of annexed files, then look at
+`.git/annex/objects`.  
+**But**: if the repository was in direct mode, then `.git/annex/objects` *may*
+contain only *old* versions of files. The current versions of annexed files
+in direct mode are stored in the working directory, which is empty in your case.
+
+The git-annex branch contains just the location log of the content of annexed
+files, i.e. which git-annex repositories and when stored the contents.
+
+"""]]
diff --git a/doc/forum/Restricting_SSH_+_supply_key.mdwn b/doc/forum/Restricting_SSH_+_supply_key.mdwn
new file mode 100644
index 0000000000..59dd72fdc6
--- /dev/null
+++ b/doc/forum/Restricting_SSH_+_supply_key.mdwn
@@ -0,0 +1,7 @@
+Just two very simple questions:
+
+1.) Is there a way to restrict the SSH key for git annex by supplying a command= ? Even better, is it also possible to supply a directory in which the repository is? (I do not want chroot - too complicated but a soft check would be sufficient for me).
+
+2.) Can I tell git and git-ssh which pubkey to use WITHOUT changing system-/user wide config (e.g.., .ssh/*)? If it is indeed not possible, what's the best way to do it in Windows?
+
+Thanks!
diff --git a/doc/forum/Restricting_SSH_+_supply_key/comment_10_8bbd0b6488c23ce8b182bd6b1765c94b._comment b/doc/forum/Restricting_SSH_+_supply_key/comment_10_8bbd0b6488c23ce8b182bd6b1765c94b._comment
new file mode 100644
index 0000000000..2bcf658b96
--- /dev/null
+++ b/doc/forum/Restricting_SSH_+_supply_key/comment_10_8bbd0b6488c23ce8b182bd6b1765c94b._comment
@@ -0,0 +1,11 @@
+[[!comment format=mdwn
+ username="http://joeyh.name/"
+ ip="209.250.56.2"
+ subject="ooooh"
+ date="2014-07-16T22:21:36Z"
+ content="""
+Thank *you* for taking the time to figure that out!
+ 
+I agree on moving the .vbs files. I have done so. (Well, you have to run the git-annex-uninstall.exe to remove the old ones, but that shouldn't matter.)
+Testing of an autobuild would be appreciated.
+"""]]
diff --git a/doc/forum/Restricting_SSH_+_supply_key/comment_11_3b946519c9f52569ce60ac04bdc3380a._comment b/doc/forum/Restricting_SSH_+_supply_key/comment_11_3b946519c9f52569ce60ac04bdc3380a._comment
new file mode 100644
index 0000000000..6283053771
--- /dev/null
+++ b/doc/forum/Restricting_SSH_+_supply_key/comment_11_3b946519c9f52569ce60ac04bdc3380a._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="joey"
+ subject="""fixed"""
+ date="2014-10-28T20:19:04Z"
+ content="""
+I seem to have forgotten to follow up here, but I think I fixed
+this problem some time ago, in [[!commit 5afc8b28e03f4d242fa81a9a93384714d12d4e5c]].
+"""]]
diff --git a/doc/forum/Restricting_SSH_+_supply_key/comment_1_cac35ac1ac0b300ddfac5ffc74291bce._comment b/doc/forum/Restricting_SSH_+_supply_key/comment_1_cac35ac1ac0b300ddfac5ffc74291bce._comment
new file mode 100644
index 0000000000..edcb5307e6
--- /dev/null
+++ b/doc/forum/Restricting_SSH_+_supply_key/comment_1_cac35ac1ac0b300ddfac5ffc74291bce._comment
@@ -0,0 +1,12 @@
+[[!comment format=mdwn
+ username="http://joeyh.name/"
+ ip="209.250.56.2"
+ subject="comment 1"
+ date="2014-07-10T19:41:54Z"
+ content="""
+1. Yes, use [[git-annex-shell]].
+
+2. The best way to do this is to use a dummy hostname in the git url for the remote. Then in .ssh/config, you can add a Host stanza that sets the real Hostname and also specifies the IdentityFile to use for that host.
+
+Incidentially, the git-annex webapp takes care of both of these things for you automatically when setting up a remote on a ssh server.
+"""]]
diff --git a/doc/forum/Restricting_SSH_+_supply_key/comment_2_e9803dd1794b4d078efa9435ff5ba295._comment b/doc/forum/Restricting_SSH_+_supply_key/comment_2_e9803dd1794b4d078efa9435ff5ba295._comment
new file mode 100644
index 0000000000..e3ae82116e
--- /dev/null
+++ b/doc/forum/Restricting_SSH_+_supply_key/comment_2_e9803dd1794b4d078efa9435ff5ba295._comment
@@ -0,0 +1,19 @@
+[[!comment format=mdwn
+ username="divB"
+ ip="171.67.173.91"
+ subject="comment 2"
+ date="2014-07-11T21:43:11Z"
+ content="""
+Hey Joey,
+
+Cool, that's great! Thanks!
+
+As for the client side. I assume you probably don't know too much about the git setup on Windows ... but maybe (or someone else has an idea :)
+
+1.) The ssh client config for git is \"c:\Program Files (x86)\Git\etc\ssh\ssh_config\" which is *very* unhandy since it's system wide! If it would at least be in the user profile ... (as mentioned, within the \".git\" directory of the repository would be the best)
+
+2.) Even if I create a \"Host\" and use \"IdentityFile\", ssh still queries the agent! This is absolutely unwanted and slows down things! I have a particular key \"git-annex.key\" and *only* this should be tried - no agent. The reason why the agent is especially problematic for me is that I use a special agent which \"locks\" itself after some inactivity and requires to re-enter the passwords.
+
+Thank you!
+
+"""]]
diff --git a/doc/forum/Restricting_SSH_+_supply_key/comment_3_1c3beb859e76cb69d2bacd2473ec72b7._comment b/doc/forum/Restricting_SSH_+_supply_key/comment_3_1c3beb859e76cb69d2bacd2473ec72b7._comment
new file mode 100644
index 0000000000..5412405e7e
--- /dev/null
+++ b/doc/forum/Restricting_SSH_+_supply_key/comment_3_1c3beb859e76cb69d2bacd2473ec72b7._comment
@@ -0,0 +1,10 @@
+[[!comment format=mdwn
+ username="http://joeyh.name/"
+ ip="209.250.56.2"
+ subject="comment 3"
+ date="2014-07-14T20:20:28Z"
+ content="""
+Re the ssh config on Windows, my windows VM has a per-user .ssh/config inside c:\Documents and Settings\$user. This is where the assistant stores configurations so I know it works.
+
+The soution to ssh still trying to use the agent is probably to set \"IdentitiesOnly yes\" in the stanza for a host. This is what the assistant does, anyway.
+"""]]
diff --git a/doc/forum/Restricting_SSH_+_supply_key/comment_4_1c541fc9a44e5cfb13c7d3ef0eeba2c7._comment b/doc/forum/Restricting_SSH_+_supply_key/comment_4_1c541fc9a44e5cfb13c7d3ef0eeba2c7._comment
new file mode 100644
index 0000000000..cba554860f
--- /dev/null
+++ b/doc/forum/Restricting_SSH_+_supply_key/comment_4_1c541fc9a44e5cfb13c7d3ef0eeba2c7._comment
@@ -0,0 +1,23 @@
+[[!comment format=mdwn
+ username="divB"
+ ip="128.12.90.218"
+ subject="comment 4"
+ date="2014-07-15T03:11:18Z"
+ content="""
+Hi Joey,
+
+Sorry that I bother so much with this.
+Thank you so much for your answer. At least with the command line it works now. I had it in %USERPROFILE%/.ssh/ssh_config (where I think it's supposed to be) rather than %USERPROFILE%/.ssh/config. In this file I have a stanza \"Host annex\" with Hostname, Port and IdentityFile set. When I call \"ssh annex\" from the command line everything works. It seems that it also works when I use e.g. \"git annex sync\" from the command line.
+
+However, if I use the webapp, the daemon.log is full of:
+
+    Please make sure you have the correct access rights
+    and the repository exists.
+    ssh: Could not resolve hostname annex: hostname nor servname provided, or not known
+    fatal: Could not read from remote repository.
+
+Is it possible that the assistant ignores the ssh config or does something differently?
+
+Thanks again!
+
+"""]]
diff --git a/doc/forum/Restricting_SSH_+_supply_key/comment_5_4dbd5605f2638de0a3edfb3886a47938._comment b/doc/forum/Restricting_SSH_+_supply_key/comment_5_4dbd5605f2638de0a3edfb3886a47938._comment
new file mode 100644
index 0000000000..26792873d1
--- /dev/null
+++ b/doc/forum/Restricting_SSH_+_supply_key/comment_5_4dbd5605f2638de0a3edfb3886a47938._comment
@@ -0,0 +1,22 @@
+[[!comment format=mdwn
+ username="divB"
+ ip="128.12.90.218"
+ subject="comment 5"
+ date="2014-07-15T04:03:16Z"
+ content="""
+After some debugging I found another weird thing which is I think the reason. If I execute
+    git annex get file.jpg
+
+everything works. But if I do
+
+    \"c:\program files (x86)\git\bin\git.exe\" annex get file.jpg
+
+it fails, claiming it can't connect to host \"annex\". I found that there are 3 (!) git.exe installed. When I choose
+
+    \"c:\program files (x86)\git\cmd\git.exe\" annex get file.jpg
+
+it works again. When I use \"which git\" in cygwin, it also points me to the \"cmd/git.exe\" version.
+So I think this is a bug and git annex assistant should call the executeable in \"cmd\" rather than \"bin\".
+Why are there three versions of git.exe at all (one more is in directory libexec/git-core)?
+
+"""]]
diff --git a/doc/forum/Restricting_SSH_+_supply_key/comment_6_a9c5b424a6acb2da152bf87b2e7617bb._comment b/doc/forum/Restricting_SSH_+_supply_key/comment_6_a9c5b424a6acb2da152bf87b2e7617bb._comment
new file mode 100644
index 0000000000..ef15c43a28
--- /dev/null
+++ b/doc/forum/Restricting_SSH_+_supply_key/comment_6_a9c5b424a6acb2da152bf87b2e7617bb._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="http://joeyh.name/"
+ ip="209.250.56.2"
+ subject="comment 6"
+ date="2014-07-15T19:20:13Z"
+ content="""
+msgit seems to install it in both places, I am not sure why. Either one works ok when I try to use them. The msgit installer puts \"c:\program files (x86)\git\cmd\\" into PATH so I assume that's the one you're supposed to use.
+"""]]
diff --git a/doc/forum/Restricting_SSH_+_supply_key/comment_7_93b7c2a5947fb6904c88cd5c120e404c._comment b/doc/forum/Restricting_SSH_+_supply_key/comment_7_93b7c2a5947fb6904c88cd5c120e404c._comment
new file mode 100644
index 0000000000..caeec29e59
--- /dev/null
+++ b/doc/forum/Restricting_SSH_+_supply_key/comment_7_93b7c2a5947fb6904c88cd5c120e404c._comment
@@ -0,0 +1,16 @@
+[[!comment format=mdwn
+ username="divB"
+ ip="128.12.90.218"
+ subject="comment 7"
+ date="2014-07-15T20:33:07Z"
+ content="""
+Thanks Joey,
+
+The problem is more that git-annex assistant takes the wrong (which is in \"bin\" rather than \"cmd\"). I think this is a bug. Because this way the connection does not work in git-annex assistant the same way it does not work with the version in \"bin\" ...
+
+I think when git-annex assistant just calls the git.exe from path (which should be cmd/git.exe) then it should work.
+
+Regards
+Niki
+
+"""]]
diff --git a/doc/forum/Restricting_SSH_+_supply_key/comment_8_beaa350751eca4642545d1b83e528dd7._comment b/doc/forum/Restricting_SSH_+_supply_key/comment_8_beaa350751eca4642545d1b83e528dd7._comment
new file mode 100644
index 0000000000..0d236d5e58
--- /dev/null
+++ b/doc/forum/Restricting_SSH_+_supply_key/comment_8_beaa350751eca4642545d1b83e528dd7._comment
@@ -0,0 +1,10 @@
+[[!comment format=mdwn
+ username="http://joeyh.name/"
+ ip="209.250.56.2"
+ subject="comment 8"
+ date="2014-07-16T18:10:09Z"
+ content="""
+git-annex only ever runs git from PATH; I never hardcode paths to programs.
+
+You can verify this by running it with the --debug flag to see the exact commands it runs.
+"""]]
diff --git a/doc/forum/Restricting_SSH_+_supply_key/comment_9_2faceeaf0d39f82e5d624eae19e4ca53._comment b/doc/forum/Restricting_SSH_+_supply_key/comment_9_2faceeaf0d39f82e5d624eae19e4ca53._comment
new file mode 100644
index 0000000000..8ed240a818
--- /dev/null
+++ b/doc/forum/Restricting_SSH_+_supply_key/comment_9_2faceeaf0d39f82e5d624eae19e4ca53._comment
@@ -0,0 +1,31 @@
+[[!comment format=mdwn
+ username="divB"
+ ip="204.17.143.10"
+ subject="comment 9"
+ date="2014-07-16T20:54:54Z"
+ content="""
+Hi Joey,
+
+Thanks for taking care about all these Windows troubles (Windows and POSIX is unfortunately a big mess).
+
+I finally found the issue now and maybe the bug is related to msysgit. I guess you don't know too much about it so I will report to the msysgit folks.
+
+However, there is still an (easy to fix) \"bug\" related to git-annex I think. I shortly describe the issue and a possible fix:
+
+1.) As mentioned above, from the 3 git.exe, only the one in \"cmd\" should be called! It seems to be a wrapper for the \"bin\"-version. If the git.exe from \"bin\" is called something with the environment is wrong (e.g., ssh_config can not be found)
+
+2.) cmd/git.exe is in %PATH% so usually no problem
+
+3.) However, git-annex-autostart.vbs is in \"bin\" folder. Therefore, when called from there PWD is the \"bin\" folder and when calling \"git.exe\" without absolute path, this overwrites %PATH% because it's the current directory (of course, such behavior does not appear on UNIX).
+
+4.) Now the git-annex assistant daemon always calls the wrong git.exe resulting in a broken config
+
+
+Short term fix for users: Create a shortcut to git-annex-autostart.vbs and change the working directory to anything else
+
+Long term fix for git-annex option 1: Do a chdir in the vbs file before calling git. This is not so good because where to?
+
+Long term fix for git-annex option 2: Just place the vbs files in the parent directory (where \"Git Bash.vbs\" is). This looks like the cleanest solution to me.
+
+
+"""]]
diff --git a/doc/forum/Restricting_git-annex-shell_to_a_specific_repository.mdwn b/doc/forum/Restricting_git-annex-shell_to_a_specific_repository.mdwn
new file mode 100644
index 0000000000..bed019e415
--- /dev/null
+++ b/doc/forum/Restricting_git-annex-shell_to_a_specific_repository.mdwn
@@ -0,0 +1,25 @@
+Is there a way to restrict git-annex-shell to a specific directory?
+Currently, if git-annex is paired to a remote repository, it adds this to the authorized_keys:
+
+
+    $ cat ~/.ssh/authorized_keys
+    command="~/.ssh/git-annex-shell",no-agent-forwarding,no-port-forwarding,no-X11-forwarding ssh-rsa AAAAB3...
+
+    $ cat ~/.ssh/git-annex-shell 
+    #!/bin/sh
+    set -e
+    exec git-annex-shell -c "$SSH_ORIGINAL_COMMAND"
+
+That gives whoever has the pubkey the right to access all repositories of one user.
+It would be nice to have a manual way to limit the access to a specific repository like 
+
+
+    $ cat ~/.ssh/git-annex-shell 
+    #!/bin/sh
+    set -e
+    export GIT_ANNEX_SHELL_REPO=~/annex
+    exec git-annex-shell -c "$SSH_ORIGINAL_COMMAND"
+
+
+Or maybe some chroot hackery is the way to go?
+
diff --git a/doc/forum/Restricting_git-annex-shell_to_a_specific_repository/comment_1_66544520bff71181e4a03ca583b0b458._comment b/doc/forum/Restricting_git-annex-shell_to_a_specific_repository/comment_1_66544520bff71181e4a03ca583b0b458._comment
new file mode 100644
index 0000000000..141f964945
--- /dev/null
+++ b/doc/forum/Restricting_git-annex-shell_to_a_specific_repository/comment_1_66544520bff71181e4a03ca583b0b458._comment
@@ -0,0 +1,12 @@
+[[!comment format=mdwn
+ username="http://joeyh.name/"
+ ip="4.152.108.194"
+ subject="comment 1"
+ date="2012-11-05T16:40:39Z"
+ content="""
+I've added a `GIT_ANNEX_SHELL_DIRECTORY` environment variable, that, if set, prevents git-annex-shell from operating on any other directory.
+
+I've made the assistant include that setting in `authorized_keys` that it sets up. For example:
+
+    command=\"GIT_ANNEX_SHELL_DIRECTORY=/home/me/annex ~/.ssh/git-annex-shell\" ...
+"""]]
diff --git a/doc/forum/Restricting_git-annex-shell_to_a_specific_repository/comment_2_2a210255e8535712c71fa183e56ab600._comment b/doc/forum/Restricting_git-annex-shell_to_a_specific_repository/comment_2_2a210255e8535712c71fa183e56ab600._comment
new file mode 100644
index 0000000000..42b6c1758e
--- /dev/null
+++ b/doc/forum/Restricting_git-annex-shell_to_a_specific_repository/comment_2_2a210255e8535712c71fa183e56ab600._comment
@@ -0,0 +1,13 @@
+[[!comment format=mdwn
+ username="http://olg.myid.net/"
+ ip="93.218.131.119"
+ subject="comment 2"
+ date="2012-11-05T23:34:20Z"
+ content="""
+Wow, thank you for implementing this so quickly!
+
+
+Just one question: As far as I understood git-annex-shell relays all unknown commands to git-shell. In this case are there the same restrictions active, too?
+
+Thanks again...
+"""]]
diff --git a/doc/forum/Restricting_git-annex-shell_to_a_specific_repository/comment_3_52cd4bd9694b2100b0e0dd2eafa9e828._comment b/doc/forum/Restricting_git-annex-shell_to_a_specific_repository/comment_3_52cd4bd9694b2100b0e0dd2eafa9e828._comment
new file mode 100644
index 0000000000..e198f8e09a
--- /dev/null
+++ b/doc/forum/Restricting_git-annex-shell_to_a_specific_repository/comment_3_52cd4bd9694b2100b0e0dd2eafa9e828._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="http://joeyh.name/"
+ ip="4.152.108.27"
+ subject="comment 3"
+ date="2012-11-05T23:42:10Z"
+ content="""
+All commands usually run by `git-shell` have the repository directory as their last parameter, and git-annex-shell depends on that being the case and checks them. It's possible to add some commands to git-shell by putting them in a special directory, and if those commands don't take the directory last, they wouldn't work.
+"""]]
diff --git a/doc/forum/Restricting_git-annex-shell_to_a_specific_repository/comment_4_4712a74774f60c29704d1e0f598caa78._comment b/doc/forum/Restricting_git-annex-shell_to_a_specific_repository/comment_4_4712a74774f60c29704d1e0f598caa78._comment
new file mode 100644
index 0000000000..b50f13757b
--- /dev/null
+++ b/doc/forum/Restricting_git-annex-shell_to_a_specific_repository/comment_4_4712a74774f60c29704d1e0f598caa78._comment
@@ -0,0 +1,7 @@
+[[!comment format=mdwn
+ username="anarcat"
+ subject="comment 4"
+ date="2015-05-26T16:59:02Z"
+ content="""
+seems to me this should be migrated to a tip, no?
+"""]]
diff --git a/doc/forum/Restricting_git-annex-shell_to_a_specific_repository/comment_5_27aa66428410cb8e77fcebccd2c6d1f4._comment b/doc/forum/Restricting_git-annex-shell_to_a_specific_repository/comment_5_27aa66428410cb8e77fcebccd2c6d1f4._comment
new file mode 100644
index 0000000000..b6c4af1991
--- /dev/null
+++ b/doc/forum/Restricting_git-annex-shell_to_a_specific_repository/comment_5_27aa66428410cb8e77fcebccd2c6d1f4._comment
@@ -0,0 +1,10 @@
+[[!comment format=mdwn
+ username="joey"
+ subject="""comment 5"""
+ date="2015-05-27T18:59:11Z"
+ content="""
+It might make sense to be added to some tip that's about setting up
+git-annex-shell with untrusted users. 
+
+The git-annex-shell man pages documents this already of course.
+"""]]
diff --git a/doc/forum/Restricting_git-annex-shell_to_a_specific_repository/comment_6_b3a0db1c2f11770b7c6f13964f2d1784._comment b/doc/forum/Restricting_git-annex-shell_to_a_specific_repository/comment_6_b3a0db1c2f11770b7c6f13964f2d1784._comment
new file mode 100644
index 0000000000..bd0fdd0d02
--- /dev/null
+++ b/doc/forum/Restricting_git-annex-shell_to_a_specific_repository/comment_6_b3a0db1c2f11770b7c6f13964f2d1784._comment
@@ -0,0 +1,7 @@
+[[!comment format=mdwn
+ username="anarcat"
+ subject="clarified manpage"
+ date="2016-04-04T20:00:39Z"
+ content="""
+i have (hopefully) clarified the [[git-annex-shell]] manpage to clearly state how to setup a restricted repository. hopefully, that will avoid further mistakes. :) i am still unclear as to why the wrapper script is necessary, but that's a different issue. --[[anarcat]]
+"""]]
diff --git a/doc/forum/Retrieve_previous_version_in_direct_mode.mdwn b/doc/forum/Retrieve_previous_version_in_direct_mode.mdwn
new file mode 100644
index 0000000000..c74b55b368
--- /dev/null
+++ b/doc/forum/Retrieve_previous_version_in_direct_mode.mdwn
@@ -0,0 +1,5 @@
+I use the assistant with a local directory in direct mode. I have read in different places that direct mode does not ensure that past versions of files are preserved.
+
+What precisely happens to past versions? Under which conditions are past versions kept?
+
+How would I go about retrieving the past version of a file in direct mode?
diff --git a/doc/forum/Retrieve_previous_version_in_direct_mode/comment_1_ca3a999ed64c42b8df810115de205d2f._comment b/doc/forum/Retrieve_previous_version_in_direct_mode/comment_1_ca3a999ed64c42b8df810115de205d2f._comment
new file mode 100644
index 0000000000..dce26832b4
--- /dev/null
+++ b/doc/forum/Retrieve_previous_version_in_direct_mode/comment_1_ca3a999ed64c42b8df810115de205d2f._comment
@@ -0,0 +1,12 @@
+[[!comment format=mdwn
+ username="http://joeyh.name/"
+ ip="64.134.31.139"
+ subject="comment 1"
+ date="2013-10-21T22:16:16Z"
+ content="""
+If you change a direct mode file and that file has not yet been transferred to some other repository, you've changed the only copy of the file, and so you cannot get back the old version.
+
+Other than that, git-annex always preserves old versions of direct mode files, the same as it preserves old versions of indirect mode files.
+
+To get back an old version of a file, use `git annex indirect` to get out of direct mode, use regular git commands to check out the version of the repository that has the file you need, and use `git annex get` to retrieve the file if it's not available in the local repository.
+"""]]
diff --git a/doc/forum/Retrieve_previous_version_in_direct_mode/comment_2_1292b34ff6d9976b2bd08748e1ba4e7a._comment b/doc/forum/Retrieve_previous_version_in_direct_mode/comment_2_1292b34ff6d9976b2bd08748e1ba4e7a._comment
new file mode 100644
index 0000000000..47ab2b742d
--- /dev/null
+++ b/doc/forum/Retrieve_previous_version_in_direct_mode/comment_2_1292b34ff6d9976b2bd08748e1ba4e7a._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="https://www.google.com/accounts/o8/id?id=AItOawmNu4V5fvpLlBhaCUfXXOB0MI5NXwh8SkU"
+ nickname="Adam"
+ subject="Disk space?"
+ date="2013-11-02T01:33:10Z"
+ content="""
+Sorry, I'm confused: does using regular client repositories in direct mode still store old versions of files?  If so, for how long?  I don't have unlimited disk space, and if it stored in git every copy of every file forever, especially large ones, I'd run out of space...and then what?
+"""]]
diff --git a/doc/forum/Retrieve_previous_version_in_direct_mode/comment_3_699e816c0397f6db924feeab906f1151._comment b/doc/forum/Retrieve_previous_version_in_direct_mode/comment_3_699e816c0397f6db924feeab906f1151._comment
new file mode 100644
index 0000000000..77fa9b0e7c
--- /dev/null
+++ b/doc/forum/Retrieve_previous_version_in_direct_mode/comment_3_699e816c0397f6db924feeab906f1151._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="http://joeyh.name/"
+ ip="209.250.56.47"
+ subject="comment 3"
+ date="2013-11-02T17:36:23Z"
+ content="""
+You can use `git annex unused` and `git annex dropunused` to remove old versions of files.
+"""]]
diff --git a/doc/forum/Retrieve_previous_version_in_direct_mode/comment_4_d900388753de5870b7b9c0e8b8c06ed7._comment b/doc/forum/Retrieve_previous_version_in_direct_mode/comment_4_d900388753de5870b7b9c0e8b8c06ed7._comment
new file mode 100644
index 0000000000..deb26163eb
--- /dev/null
+++ b/doc/forum/Retrieve_previous_version_in_direct_mode/comment_4_d900388753de5870b7b9c0e8b8c06ed7._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="https://www.google.com/accounts/o8/id?id=AItOawmNu4V5fvpLlBhaCUfXXOB0MI5NXwh8SkU"
+ nickname="Adam"
+ subject="comment 4"
+ date="2013-11-02T22:30:07Z"
+ content="""
+Thank you.  Just curious, are there plans to put that in the webapp?
+"""]]
diff --git a/doc/forum/Retrieve_previous_version_in_direct_mode/comment_5_1360b936aa389a0ab5e5e453824b2ece._comment b/doc/forum/Retrieve_previous_version_in_direct_mode/comment_5_1360b936aa389a0ab5e5e453824b2ece._comment
new file mode 100644
index 0000000000..c921758b42
--- /dev/null
+++ b/doc/forum/Retrieve_previous_version_in_direct_mode/comment_5_1360b936aa389a0ab5e5e453824b2ece._comment
@@ -0,0 +1,9 @@
+[[!comment format=mdwn
+ username="https://www.google.com/accounts/o8/id?id=AItOawkAUMhKOSkh9JaBA6xst3XxQIIsDEq5Zd4"
+ nickname="Ovidiu"
+ subject="How to do this on Mac OSX"
+ date="2014-05-08T22:46:49Z"
+ content="""
+So I have a local repository on my MBP and a backup repository on S3. 
+How do I now restore OLD or deleted file versions on my MBP with git-annex assistant?
+"""]]
diff --git a/doc/forum/Reusing_existing_annex.uuid.mdwn b/doc/forum/Reusing_existing_annex.uuid.mdwn
new file mode 100644
index 0000000000..2d438e8a28
--- /dev/null
+++ b/doc/forum/Reusing_existing_annex.uuid.mdwn
@@ -0,0 +1,27 @@
+I recently replaced a disk that was failing with a new disk. Rather than just clone and create a new remote, I decided I'd reuse the existing anex.uuid, since I wanted to mount the replacement disk in the same spot/using the same name as before. The annex.uuid was the one I saw [here](http://git-annex.branchable.com/forum/Truly_purging_dead_repositories/#comment-b89d6a7ab50180c901f53761f8a1a99d). 
+
+I have multiple disks that I store full copies of each of my repositories on; several are just regular git-annex remotes, and some are gcrypt remotes. All of them are local disks. 
+
+When I ran a `git annex sync` in the repo with the reused UUID, the repo, and subsequently the other non-encrypted repos that I synced to, have lost knowledge of the encrypted repos, e.g. when I run `git-annex info`, I do not see the encrypted repos in that list. Even more strangely, this only happened in 2 out of 7 repos; 5 of the repos retained the knowledge of the gcrypt repos. 
+
+The steps I used to recreate all of the repos was:
+
+```
+mkdir resued-uuid
+cd reused-uuid
+git init
+git config annex.uuid xxxxxxxxxxx-xxxxxxxx-xxxxxxxxx
+git annex init "My reused uuid repo"
+git annex fsck (as directed to run in the linked comment, but this didn't do anything because there were no files in the repo and the repo did not have other repos' remotes added to it)
+git remote add existing-repo ~/some/repo/path
+git annex sync
+git annex sync --content (all content copied over; subsequent fscks' don't reveal any damaged files)
+```
+
+I'm curious as to any insight as to why this might have happened and what went wrong.
+
+How can I add existing gcrypt remotes to the repos that are missing them.
+
+Does git annex copy some part of .git/config around to different repos?  
+
+Can I just copy part of a .git/config with the grcrypt remotes listed there and sync them up? 
diff --git a/doc/forum/Reusing_existing_annex.uuid/comment_1_dc3d6efbf5c076bedd0c7a75ff5314bc._comment b/doc/forum/Reusing_existing_annex.uuid/comment_1_dc3d6efbf5c076bedd0c7a75ff5314bc._comment
new file mode 100644
index 0000000000..5347a42e86
--- /dev/null
+++ b/doc/forum/Reusing_existing_annex.uuid/comment_1_dc3d6efbf5c076bedd0c7a75ff5314bc._comment
@@ -0,0 +1,11 @@
+[[!comment format=mdwn
+ username="joey"
+ subject="""comment 1"""
+ date="2015-12-19T17:48:48Z"
+ content="""
+Take a look at remote.log in the git-annex branch; it should list all your
+gcrypt and other special remotes. If some seem to be missing, you could
+examine the git log of that file and see what happened.
+
+I don't see how forcing reuse of a uuid could lead to this.
+"""]]
diff --git a/doc/forum/Reusing_existing_annex.uuid/comment_2_cdb21c9c47c8a116e82062c9354d9339._comment b/doc/forum/Reusing_existing_annex.uuid/comment_2_cdb21c9c47c8a116e82062c9354d9339._comment
new file mode 100644
index 0000000000..e100e5952f
--- /dev/null
+++ b/doc/forum/Reusing_existing_annex.uuid/comment_2_cdb21c9c47c8a116e82062c9354d9339._comment
@@ -0,0 +1,10 @@
+[[!comment format=mdwn
+ username="https://openid.stackexchange.com/user/e65e6d0e-58ba-41de-84cc-1f2ba54cf574"
+ nickname="Mica"
+ subject="comment 2"
+ date="2015-12-21T06:02:21Z"
+ content="""
+The gcrypt remotes are present in the remotes.log file, but I can't seem to push or sync to them.
+
+How do I get them added back to the repo such that using `git annex sync` syncs the content?
+"""]]
diff --git a/doc/forum/Reusing_existing_annex.uuid/comment_3_5544f712897e06024342aa1bdd68df1a._comment b/doc/forum/Reusing_existing_annex.uuid/comment_3_5544f712897e06024342aa1bdd68df1a._comment
new file mode 100644
index 0000000000..ae763b1e8f
--- /dev/null
+++ b/doc/forum/Reusing_existing_annex.uuid/comment_3_5544f712897e06024342aa1bdd68df1a._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="https://openid.stackexchange.com/user/e65e6d0e-58ba-41de-84cc-1f2ba54cf574"
+ nickname="Mica"
+ subject="Copy from .git/config"
+ date="2016-01-09T06:44:59Z"
+ content="""
+Copying the values for the encrypted remotes from a back-up copy of .git/config seems to have worked. 
+"""]]
diff --git a/doc/forum/Revert_file_linkage_to_original_files.mdwn b/doc/forum/Revert_file_linkage_to_original_files.mdwn
new file mode 100644
index 0000000000..ba83cfd150
--- /dev/null
+++ b/doc/forum/Revert_file_linkage_to_original_files.mdwn
@@ -0,0 +1,9 @@
+I've recently found the following problem:
+
+I really really want to get back my original folder structure - which includes the real files, not the symlinks. I've searched for quite a while, but I simply could not find an acceptable solution...
+
+So I thought I would like to ask you guys here, if anybody experienced similar problems (or at least knows a solution for my problem)?
+
+Greetings
+
+Pethor
diff --git a/doc/forum/Revert_file_linkage_to_original_files/comment_1_898ca2c9976e92d22470c7404aa9813f._comment b/doc/forum/Revert_file_linkage_to_original_files/comment_1_898ca2c9976e92d22470c7404aa9813f._comment
new file mode 100644
index 0000000000..a72a2cfdd4
--- /dev/null
+++ b/doc/forum/Revert_file_linkage_to_original_files/comment_1_898ca2c9976e92d22470c7404aa9813f._comment
@@ -0,0 +1,10 @@
+[[!comment format=mdwn
+ username="http://joeyh.name/"
+ ip="2001:4978:f:21a::2"
+ subject="comment 1"
+ date="2013-08-15T09:02:38Z"
+ content="""
+Sounds like you want to switch to [[direct mode]].
+
+(Or possibly to run `git annex unannex`, if you don't want to use git-annex for these files.)
+"""]]
diff --git a/doc/forum/Revert_to_a_precedent_state_in_direct_mode.mdwn b/doc/forum/Revert_to_a_precedent_state_in_direct_mode.mdwn
new file mode 100644
index 0000000000..08aa843cfd
--- /dev/null
+++ b/doc/forum/Revert_to_a_precedent_state_in_direct_mode.mdwn
@@ -0,0 +1,3 @@
+I have made some mistakes while using `git annex import` in direct mode. Now I see that some files have been erroneously added and there are other problems. I have not yet used `git annex sync`.
+
+How can I tell git-annex in direct mode (or bare git) to forget about all these changes and revert back to the last known good (pre-import) state? This means also removing the few imported files and recreate their links.
diff --git a/doc/forum/Revert_to_a_precedent_state_in_direct_mode/comment_1_1ae9f7defbab44621c3108973a4f683a._comment b/doc/forum/Revert_to_a_precedent_state_in_direct_mode/comment_1_1ae9f7defbab44621c3108973a4f683a._comment
new file mode 100644
index 0000000000..88f7fc895d
--- /dev/null
+++ b/doc/forum/Revert_to_a_precedent_state_in_direct_mode/comment_1_1ae9f7defbab44621c3108973a4f683a._comment
@@ -0,0 +1,26 @@
+[[!comment format=mdwn
+ username="http://joeyh.name/"
+ ip="108.236.230.124"
+ subject="comment 1"
+ date="2014-06-04T19:03:05Z"
+ content="""
+I wish that I had a good answer for you, but since you're using direct mode, I really do not!
+
+You can run this to see the status:
+
+    git -c core.bare=false status
+
+That's safe. Everything below has a good chance of losing data if not used carefully.
+
+You can run a command like this to *delete* an unwanted file (that is shown as a new file in the above).
+
+    git -c core.bare=false rm -f unwanted_file
+
+If the status shows a file as being modified, you can run this to throw away the modified version. This does not put back the old version! It will leave you with a broken symlink.
+
+    git -c core.bare=false reset modified_file
+    git -c core.bare=false checkout modified_file
+    git annex fsck
+
+If you're lucky, you can then `git annex get` to get the old version from some other repository that still has it.
+"""]]
diff --git a/doc/forum/Revert_to_a_precedent_state_in_direct_mode/comment_2_0bcfc0e89de7072bfdf1e3cdeaa16a1b._comment b/doc/forum/Revert_to_a_precedent_state_in_direct_mode/comment_2_0bcfc0e89de7072bfdf1e3cdeaa16a1b._comment
new file mode 100644
index 0000000000..36102071bf
--- /dev/null
+++ b/doc/forum/Revert_to_a_precedent_state_in_direct_mode/comment_2_0bcfc0e89de7072bfdf1e3cdeaa16a1b._comment
@@ -0,0 +1,11 @@
+[[!comment format=mdwn
+ username="https://www.google.com/accounts/o8/id?id=AItOawmWBvsZvSsAL8P2ye3F0OBStjFCVnOImzM"
+ nickname="Jarno"
+ subject="How about with the 'proxy' command?"
+ date="2014-11-28T18:47:17Z"
+ content="""
+Why does this not work?
+
+    git annex proxy -- git reset HEAD
+
+"""]]
diff --git a/doc/forum/Revert_to_a_precedent_state_in_direct_mode/comment_3_e49a37f92a3dd7d386e1e5f37ab37df4._comment b/doc/forum/Revert_to_a_precedent_state_in_direct_mode/comment_3_e49a37f92a3dd7d386e1e5f37ab37df4._comment
new file mode 100644
index 0000000000..fc3e9c6ba3
--- /dev/null
+++ b/doc/forum/Revert_to_a_precedent_state_in_direct_mode/comment_3_e49a37f92a3dd7d386e1e5f37ab37df4._comment
@@ -0,0 +1,12 @@
+[[!comment format=mdwn
+ username="joey"
+ subject="""comment 3"""
+ date="2014-12-01T22:40:05Z"
+ content="""
+Yes, the new `git annex proxy` command was added partly thanks to this
+thread, and you can use it like that.
+
+There's also a new `git annex undo` command that's easier to use than the
+proxy command. It just undoes the last change you made to the file or
+directory you specify.
+"""]]
diff --git a/doc/forum/Robustness_on_failing_hardware.mdwn b/doc/forum/Robustness_on_failing_hardware.mdwn
new file mode 100644
index 0000000000..ec32cbfe70
--- /dev/null
+++ b/doc/forum/Robustness_on_failing_hardware.mdwn
@@ -0,0 +1,30 @@
+Hi everyone,
+
+
+I've had a bad git-annex day and I left some musings on my first use of it. There's a question though at the end if you don't need to read that.
+
+
+today I've spend some time trying to deal with a failing SHA256E backend. Or more precise, I'm patienting a refurbished Athlon board[*] without case and that seems to give a lot of errors. I've put up some shielding at the SDRAM which helped a lot on the checksumming, but now its the network department is having a hard time. 
+
+This is basicly an effort to recover an old 1.5TB USB NTFS disks with bit rot, a failed ext4 and recover collections from other aging USB/desktop disks. There is some new hardware on the way, to restore another (proper) box with some ECC memory. That should fix my problems, I think. I've never dealt with a problem like this before. Before git-annex I would have been using rsync -azc or mostly -azu, but I've never had problems like this before. 
+
+It has me pondering a bit though. To help git-annex i've done manual rsync -azc transfers, with some although a very low success--I wonder if its the lack of casing, lack of ground, or simply the non-ecc that is the cause. But that is not really the question.
+
+rsync does not at first glance have an option to configure the amount of retries, but -azc does try and gives a checksum failure another try before giving up. I had never seen rsync do that before even though I've been using it for a decade. I guess that shows how rare this problem should be, but its still creeping me a bit. I've never had checksummed media file collections, so I really cant be sure about integrity, can I?
+
+The behaviour of the SHA256E backend is a bit hysterical in this environment. Running a full annex fsck always results in some bad files. The bigger the file the harder it is to get a checksum match. I guess if I wanted to continue I should wrap the fsck step in a shell script that can reinject and fsck again and does do not needless re-checks in one run. It should figure it out in the end with a bit of more time.
+
+My other reaction would be to throw more checksums at it, from simple to complex. Something between filesize and sha256. I guess the whole would then have some different modes of retries and behaviours of failure maybe. Or levels of trust in the integrity. Maybe using simpler checksums would make the situation more bearable, since the problem is not disk-related iow. if the file is there and has checked out ok, then the re-fsck itself and bad memory/shielding are the culprits. 
+
+tldr;
+
+git-annex has no retry at all. I have seen it can take a -i  somewhere. For robustness, it would be nice if there was a rsync -c mode. It would confirm the object was transferred correctly too. My workflow is to 'annex fsck' at the remote after copying files there. Is it correct to presume that using 'annex move' in my case, would have made the sending repo dropping files even though the remote still has only corrupted copies?
+
+I'm new to git-annex but been wanting something like git annex for a long time. Made some attempts myself, and too pondered how to use venerable GIT but never figured something out. What I'm saying though is I may want to dig around to see what I can do with Haskell. 
+
+I'm thinking a backend with some more checksums, maybe then some option to fsck to request only partial check of the key. Has something like a faster/less-thorough fsck using smaller checksums ever come up? And might this be a way to go at this? What do you think.
+
+
+
+
+[*] Asus M2A7A-AM SE w/ AMD Athlon 64 X2 5200+ Dual Core 2.7 
diff --git a/doc/forum/Robustness_on_failing_hardware/comment_1_d89ffbebe59d05fb2c47d957792cf97e._comment b/doc/forum/Robustness_on_failing_hardware/comment_1_d89ffbebe59d05fb2c47d957792cf97e._comment
new file mode 100644
index 0000000000..baac15e81f
--- /dev/null
+++ b/doc/forum/Robustness_on_failing_hardware/comment_1_d89ffbebe59d05fb2c47d957792cf97e._comment
@@ -0,0 +1,21 @@
+[[!comment format=mdwn
+ username="joey"
+ subject="""comment 1"""
+ date="2015-07-06T16:09:22Z"
+ content="""
+If the remote that git-annex is moving files to uses rsync as its transport
+(ie, is a regular git remote, or a rsync special remote), then
+the receiving rsync process verifies a checksum of the file it's receiving.
+So, if the sender sends incorrect data, or calculates the wrong checksum
+for the right data, the rsync will fail, and git-annex won't delete the
+local copy of the file. I guess that if the sending rsync reads bad data
+from disk, these checksums won't prevent the bad data being sent to the
+remote though. (Some other special remotes don't have a transfer checksum
+at all.)
+
+If I had hardware this broken, I'd be glad that `git annex fsck` detected
+it, and would move the storage to good hardware and run my fsck there to
+learn the actual state of my repository. Re-running checksums or using
+multiple checksum types to work around badly broken hardware seems like a
+losing idea to me.
+"""]]
diff --git a/doc/forum/Running_assistant_on_a_server___40__no_X_available__41__.mdwn b/doc/forum/Running_assistant_on_a_server___40__no_X_available__41__.mdwn
new file mode 100644
index 0000000000..4116577bb2
--- /dev/null
+++ b/doc/forum/Running_assistant_on_a_server___40__no_X_available__41__.mdwn
@@ -0,0 +1,2 @@
+How can I run the assistant on a server and access the webinterface over the network from my client pc?
+The use case I have in mind is that the server can automatically synchronize files, pushed over XMPP.
diff --git a/doc/forum/Running_assistant_on_a_server___40__no_X_available__41__/comment_1_dd75d78ef63f2689199a302ed1846017._comment b/doc/forum/Running_assistant_on_a_server___40__no_X_available__41__/comment_1_dd75d78ef63f2689199a302ed1846017._comment
new file mode 100644
index 0000000000..ec31212e54
--- /dev/null
+++ b/doc/forum/Running_assistant_on_a_server___40__no_X_available__41__/comment_1_dd75d78ef63f2689199a302ed1846017._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="https://www.google.com/accounts/o8/id?id=AItOawn7gQ1zZDdWhXy9H51W2krZYShNmKL3qfM"
+ nickname="Karsten"
+ subject="comment 1"
+ date="2013-01-17T08:30:54Z"
+ content="""
+You don't need the webapp to run the assistant, but if you want, you can forward it to your local system. You will find the URL in .git/annex/url and have to extract the port from it (http://127.0.0.1:/.../), afterwards forward that port either via ssh port forwarding or via a webserver acting as a reverse proxy.
+"""]]
diff --git a/doc/forum/Running_assistant_on_a_server___40__no_X_available__41__/comment_2_df654df60c5fa6a84d786d248928a352._comment b/doc/forum/Running_assistant_on_a_server___40__no_X_available__41__/comment_2_df654df60c5fa6a84d786d248928a352._comment
new file mode 100644
index 0000000000..ff43a083d8
--- /dev/null
+++ b/doc/forum/Running_assistant_on_a_server___40__no_X_available__41__/comment_2_df654df60c5fa6a84d786d248928a352._comment
@@ -0,0 +1,11 @@
+[[!comment format=mdwn
+ username="http://joeyh.name/"
+ ip="4.154.7.238"
+ subject="comment 2"
+ date="2013-01-17T16:29:39Z"
+ content="""
+If this is a true server, with a static IP (or some dynamic DNS), there is no need to run the assistant on it; clients using the assistant can ssh in and sync data to it directly.
+
+Anyway, you can git config web.browser to run some script with the url it will be passed to the webapp when the webapp is started. I sometimes configure it to just echo the url out; it'd also be possible to automate
+setting up a redirection for a remote browser, I suppose.
+"""]]
diff --git a/doc/forum/Running_assistant_steps_manually.mdwn b/doc/forum/Running_assistant_steps_manually.mdwn
new file mode 100644
index 0000000000..b3b683433b
--- /dev/null
+++ b/doc/forum/Running_assistant_steps_manually.mdwn
@@ -0,0 +1,20 @@
+Unfortunately one of the machines that I am running git-annex on is too old to handle the watcher thread.
+
+The error message is:
+
+    Watcher crashed: Need at least OSX 10.7.0 for file-level FSEvents
+
+
+I am however interested in duplicating the assistant steps manually.  I'm happy to accept that I won't have the great, auto-magic, coverage, but, how I run the steps that the assistant runs?
+
+I'm guessing there is a:
+
+    git annex add .
+    git commit -m "[blah]"
+    git annex sync
+
+but how do I get the pushing to backup/transfer/whatever remotes to work?  And what order should I do these?
+
+What I would *like* to do is get a script that runs all of the needed annex steps and then kick it off in a cron job 2-5 times a day.
+
+Any thoughts?
diff --git a/doc/forum/Running_assistant_steps_manually/comment_1_e14e0a1d55d01cb4f67a94bbe349b872._comment b/doc/forum/Running_assistant_steps_manually/comment_1_e14e0a1d55d01cb4f67a94bbe349b872._comment
new file mode 100644
index 0000000000..6629ff024c
--- /dev/null
+++ b/doc/forum/Running_assistant_steps_manually/comment_1_e14e0a1d55d01cb4f67a94bbe349b872._comment
@@ -0,0 +1,20 @@
+[[!comment format=mdwn
+ username="http://joeyh.name/"
+ nickname="joey"
+ subject="comment 1"
+ date="2013-05-19T20:18:11Z"
+ content="""
+Essentially, you want to run this:
+
+
    +git annex add
+git annex sync
+for remote in $(git remote); do
+    git annex copy --auto --to $remote
+    git annex get --auto --from $remote
+done
+git annex drop --auto
+
    
+
+Of course this cannot sync with XMPP remotes. Otherwise it will get you to essentially the same place as the assistant.
+"""]]
diff --git a/doc/forum/Running_assistant_steps_manually/comment_2_3192f614c929b8060d4fbde56a7adec1._comment b/doc/forum/Running_assistant_steps_manually/comment_2_3192f614c929b8060d4fbde56a7adec1._comment
new file mode 100644
index 0000000000..b6c10ee880
--- /dev/null
+++ b/doc/forum/Running_assistant_steps_manually/comment_2_3192f614c929b8060d4fbde56a7adec1._comment
@@ -0,0 +1,14 @@
+[[!comment format=mdwn
+ username="https://www.google.com/accounts/o8/id?id=AItOawnRai_qFYPVvEgC6i1nlM1bh-C__jbhqS0"
+ nickname="Matthew"
+ subject="Looks great"
+ date="2013-08-29T12:45:10Z"
+ content="""
+This looks great as I have:
+
+ * A preference for multiple small repositories.
+ * Old versions for `git-annex` due to being on Ubuntu LTS for my server.
+ * A Samsung Galaxy Nexus which somehow seems too slow to run the assistant.
+
+So these steps combined with some locking and maybe `inotify` seem perfect
+"""]]
diff --git a/doc/forum/Running_out_of__inodes.mdwn b/doc/forum/Running_out_of__inodes.mdwn
new file mode 100644
index 0000000000..417de18de0
--- /dev/null
+++ b/doc/forum/Running_out_of__inodes.mdwn
@@ -0,0 +1,17 @@
+When syncing with a huge git annex repository on usb disk, my small laptop partition runs out of inodes.
+
+Any workaround for this?
+
+- Use bare repository?
+
+    Some git annex command are not supported.
+    This makes managing (particularly adding) files difficult.
+
+- Use a loop file partition with tiny block size and large inode numbers?
+
+    Operations on a huge git repository are slow.
+    On a loop file partition will be slower.
+
+- Maybe shrink the partition and make room for a specific partition for git annex repository?
+
+Any opinions?   
diff --git a/doc/forum/Running_out_of__inodes/comment_1_abc73d9ad662ef642337b683bf0a0253._comment b/doc/forum/Running_out_of__inodes/comment_1_abc73d9ad662ef642337b683bf0a0253._comment
new file mode 100644
index 0000000000..11f2f7b7eb
--- /dev/null
+++ b/doc/forum/Running_out_of__inodes/comment_1_abc73d9ad662ef642337b683bf0a0253._comment
@@ -0,0 +1,12 @@
+[[!comment format=mdwn
+ username="http://joeyh.name/"
+ ip="4.154.4.51"
+ subject="comment 1"
+ date="2013-09-12T20:37:05Z"
+ content="""
+Define \"huge\"? Checking out a git repository necessarily requires one inode per file in the repository, plus a smaller quantity for the things in .git. A git-annex repository  is much the same as any other git repository.
+
+Even when I make a really tiny 100 mb ext4 filesystem, it defaults to 25000 inodes, which would be enough to contain a checkout of my second largest git-annex repository. 
+
+Anyway, using git branches seems like a reasonable workaround, to the extent I understand your problem. Make a branch with the files in it you want to have available on the small drive, or check out an empty branch on the small drive and `git annex add` files in there. You can merge the branch back into your master branch on the large drive.
+"""]]
diff --git a/doc/forum/Running_out_of__inodes/comment_2_30d58cb046b13d6ddd435ad0ae2f0ae1._comment b/doc/forum/Running_out_of__inodes/comment_2_30d58cb046b13d6ddd435ad0ae2f0ae1._comment
new file mode 100644
index 0000000000..4ec867ec53
--- /dev/null
+++ b/doc/forum/Running_out_of__inodes/comment_2_30d58cb046b13d6ddd435ad0ae2f0ae1._comment
@@ -0,0 +1,44 @@
+[[!comment format=mdwn
+ username="https://launchpad.net/~stephane-gourichon-lpad"
+ nickname="stephane-gourichon-lpad"
+ avatar="http://cdn.libravatar.org/avatar/02d4a0af59175f9123720b4481d55a769ba954e20f6dd9b2792217d9fa0c6089"
+ subject="git-annex+git consumes lots of inodes. They are cheap, don't limit their amount when creating a filesystem."
+ date="2016-11-04T09:56:45Z"
+ content="""
+TL;DR: don't play with `inode_ratio` when creating a filesystem. You will certainly get more storage space, up to 1.5%. But the numerous symlinks maintained by git-annex will consume a lot of inodes. You don't want your filesystem to fail to create any new file when doing a big `git annex add` or worse, `git repack` or `git gc` or `git repair`, right ?
+
+## Git annex heavily uses symlinks and dirs, which consumes several inodes per file.
+
+I ran out of inodes on two of my disks, with different git-annex repositories.
+
+Joeyh wrote:
+
+> Checking out a git repository necessarily requires one inode per file in the repository, plus a smaller quantity for the things in .git. A git-annex repository is much the same as any other git repository.
+
+I believe that in `.git/annex/objects`, the structure consumes two inodes, one for the directory and one for the actual file storing data.  Is it right?
+
+So, when a file is checked out and managed by `git-annex`, is consumes 3 inodes, right?
+
+Plus possibly many inodes used temporarily when doing some git operations.  E.g. `git repair`. Man page says: \"Since this command unpacks all packs in the repository, you may want to run git gc afterwards.\"
+
+So, at least 4 inodes per file ?
+
+## Optimising space by reducing inodes is no longer worth the trouble.
+
+External disks were formatted years ago maximizing disk space (for details see [http://serverfault.com/a/523210](http://serverfault.com/a/523210) ).
+The assumption was that nearly all files were more than 4 megabytes in size, so the ext2 `inode_ratio` was 4194304.
+This assumption was very true and all went well until `git-annex` entered the game.
+
+In the case of a git-annex repository, if there is at least 4 inodes per actual image file, `inode_ratio` must be at most the average file size divided by 4.
+
+I considered an inode ratio at 64k. With most files far above 1 megabyte, that leaves room for much more than 16 inodes for housekeeping.  You don't know in advance how the usage will evolve.  Perhaps you will need to do one or more git clones of the same annexed data, especially in a recovery situation, which will quickly multiply the number of consumed inodes.
+
+`mkfs.ext4` chooses by itself `inode_ratio=16384`. Filesystem capacity is 1.5% smaller than with the minimum number of inode.  I'll just stick with that.
+
+```
+time mkfs.ext4 -E lazy_itable_init -L \"MyWonderFulLabel\" -O sparse_super -m 0 /dev/sdb1
+```
+
+(Yes, `-m 0` removes the 5% reserved space for root, and yes, filesystem performance drops dramatically then full beyond 90% then 95%. That parameter can be tuned with tune2fs at any time. In doubt, remove it.)
+
+"""]]
diff --git a/doc/forum/S3_Host_Question.mdwn b/doc/forum/S3_Host_Question.mdwn
new file mode 100644
index 0000000000..f114d7c930
--- /dev/null
+++ b/doc/forum/S3_Host_Question.mdwn
@@ -0,0 +1,11 @@
+I have two machines A & B.  Each have their our repository.
+
+On machine A I have created a remotehost (S3) and have synced successfully.
+
+I am trying to link machine B to the same remote host.  I keep getting an error on the initremote command on machine B.
+
+I tried using the same command as I did on A and it is not working.   Is there a different command to link machine B to an existing remote repository?
+
+Any help is appreciated.
+
+Rob
diff --git a/doc/forum/S3_Host_Question/comment_1_8c8ecea703405753e47e0da5e8325929._comment b/doc/forum/S3_Host_Question/comment_1_8c8ecea703405753e47e0da5e8325929._comment
new file mode 100644
index 0000000000..b8d63e5fdd
--- /dev/null
+++ b/doc/forum/S3_Host_Question/comment_1_8c8ecea703405753e47e0da5e8325929._comment
@@ -0,0 +1,11 @@
+[[!comment format=mdwn
+ username="https://me.yahoo.com/a/FHnTlSBo1eCGJRwueeKeB6.RCaPbGMPr5jxx8A--#ce0d8"
+ nickname="Hamza"
+ subject="comment 1"
+ date="2013-12-13T08:37:24Z"
+ content="""
+On machine B you don't initremote you enableremote,
+
+
+    git annex enableremote mys3
+"""]]
diff --git a/doc/forum/S3_Host_Question/comment_2_3d6fa3147d59a5e0d10b005388b23c7a._comment b/doc/forum/S3_Host_Question/comment_2_3d6fa3147d59a5e0d10b005388b23c7a._comment
new file mode 100644
index 0000000000..cef33d3542
--- /dev/null
+++ b/doc/forum/S3_Host_Question/comment_2_3d6fa3147d59a5e0d10b005388b23c7a._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="https://www.google.com/accounts/o8/id?id=AItOawnKhw0-Tu5oHjeoz_gAP4Nu5V9uo8ta3hM"
+ nickname="Robert"
+ subject="unknown special remote"
+ date="2013-12-13T15:12:37Z"
+ content="""
+How does machine B know about the remote?  Machines A & B are separate and don't share references. Am I missing a step? 
+"""]]
diff --git a/doc/forum/S3_Host_Question/comment_3_797edf3ad41561ab8960f3b28d20611e._comment b/doc/forum/S3_Host_Question/comment_3_797edf3ad41561ab8960f3b28d20611e._comment
new file mode 100644
index 0000000000..a0dbae5bdc
--- /dev/null
+++ b/doc/forum/S3_Host_Question/comment_3_797edf3ad41561ab8960f3b28d20611e._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="https://me.yahoo.com/a/FHnTlSBo1eCGJRwueeKeB6.RCaPbGMPr5jxx8A--#ce0d8"
+ nickname="Hamza"
+ subject="comment 3"
+ date="2013-12-14T09:05:26Z"
+ content="""
+remote info is recorded in git once you clone the git repo it knows about machine a and s3 repo.
+"""]]
diff --git a/doc/forum/S3_remote_not_available.mdwn b/doc/forum/S3_remote_not_available.mdwn
new file mode 100644
index 0000000000..e89a8ead71
--- /dev/null
+++ b/doc/forum/S3_remote_not_available.mdwn
@@ -0,0 +1,31 @@
+One of my remotes is S3 (named s3). I used to be able to run
+
+    $ git annex sync s3 --content
+
+But now I'm getting the following error:
+
+    commit  
+    On branch master
+    nothing to commit, working directory clean
+    ok
+    git-annex: there is no available git remote named "s3"
+
+My `.git/config` file contains following lines which I would assume mean that the remote is there:
+
+    [remote "s3"]
+            annex-s3 = true
+            annex-uuid = bd9af194-b736-4114-a689-9a8bf212fb18
+            fetch = 
+            annex-sync = false
+
+As far as I can tell only change I did was to install newer version of git-annex, currently I have:
+
+    git-annex version: 6.20160418
+    build flags: Assistant Webapp Pairing Testsuite WebDAV FsEvents XMPP ConcurrentOutput TorrentParser MagicMime Feeds Quvi
+    key/value backends: SHA256E SHA256 SHA512E SHA512 SHA224E SHA224 SHA384E SHA384 SHA3_256E SHA3_256 SHA3_512E SHA3_512 SHA3_224E SHA3_224 SHA3_384E SHA3_384 SKEIN256E SKEIN256 SKEIN512E SKEIN512 SHA1E SHA1 MD5E MD5 WORM URL
+    remote types: git gcrypt bup directory rsync web bittorrent webdav tahoe glacier ddar hook external
+    local repository version: 6
+    supported repository versions: 5 6
+    upgrade supported from repository versions: 0 1 2 4 5
+
+How do I recover access to that remote?
diff --git a/doc/forum/S3_remote_not_available/comment_1_0caf92239fb1475cf03de45f0d25291d._comment b/doc/forum/S3_remote_not_available/comment_1_0caf92239fb1475cf03de45f0d25291d._comment
new file mode 100644
index 0000000000..1f6fcd8fca
--- /dev/null
+++ b/doc/forum/S3_remote_not_available/comment_1_0caf92239fb1475cf03de45f0d25291d._comment
@@ -0,0 +1,15 @@
+[[!comment format=mdwn
+ username="joey"
+ subject="""comment 1"""
+ date="2016-05-04T17:17:00Z"
+ content="""
+The version info you posted shows that your git-annex is not built with
+support for S3. The "remote types" line should list S3 and does not.
+
+If you got this from a distribution, could you let the distribution know
+that they are building git-annex without S3 support?
+
+If you built it yourself, you probably built it without the haskell aws
+library installed, or with some dependency issue that prevented that
+library from being used.
+"""]]
diff --git a/doc/forum/S3_remote_not_available/comment_2_5895bd050237d08314e3595d93455830._comment b/doc/forum/S3_remote_not_available/comment_2_5895bd050237d08314e3595d93455830._comment
new file mode 100644
index 0000000000..d2397cb15a
--- /dev/null
+++ b/doc/forum/S3_remote_not_available/comment_2_5895bd050237d08314e3595d93455830._comment
@@ -0,0 +1,9 @@
+[[!comment format=mdwn
+ username="Lukasz"
+ subject="Thank you, indeed that was the problem"
+ date="2016-05-04T17:47:36Z"
+ content="""
+Indeed after getting git-annex from https://downloads.kitenet.net/git-annex/OSX/current/10.10_Yosemite/git-annex.dmg my problem was solved. Turns out the version from Homebrew-cask doesn't support S3 and I vaguely remember that it was the case first time I tried.
+
+Thank you for the help and for the great piece of software
+"""]]
diff --git a/doc/forum/S3_remote_not_available/comment_3_b685f1d172cca4c488a517024c86ba21._comment b/doc/forum/S3_remote_not_available/comment_3_b685f1d172cca4c488a517024c86ba21._comment
new file mode 100644
index 0000000000..1b8682baed
--- /dev/null
+++ b/doc/forum/S3_remote_not_available/comment_3_b685f1d172cca4c488a517024c86ba21._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="joey"
+ subject="""comment 3"""
+ date="2016-05-04T18:13:39Z"
+ content="""
+Hmm, there should be no reason for homebrew to not built it with S3
+support. Could you file a bug report there?
+"""]]
diff --git a/doc/forum/S3_remote_not_available/comment_4_ff5ce14bd27b32fc1707426f8c2a5487._comment b/doc/forum/S3_remote_not_available/comment_4_ff5ce14bd27b32fc1707426f8c2a5487._comment
new file mode 100644
index 0000000000..c050d67fca
--- /dev/null
+++ b/doc/forum/S3_remote_not_available/comment_4_ff5ce14bd27b32fc1707426f8c2a5487._comment
@@ -0,0 +1,7 @@
+[[!comment format=mdwn
+ username="Lukasz"
+ subject="comment 4"
+ date="2016-05-04T18:29:54Z"
+ content="""
+I must have done something wrong. It works perfectly after reinstalling it from Homebrew-cask.
+"""]]
diff --git a/doc/forum/SSH_remote_transfers_queued_but_no_movement.mdwn b/doc/forum/SSH_remote_transfers_queued_but_no_movement.mdwn
new file mode 100644
index 0000000000..de5001f007
--- /dev/null
+++ b/doc/forum/SSH_remote_transfers_queued_but_no_movement.mdwn
@@ -0,0 +1,3 @@
+I created a local repository using the assistant, followed by setting up a remote ssh 'full backup' repository. While the assistant is running, changes in the local repository are detected and transfers are initiated, but they stay at 0%. How can I go about trouble-shooting this? I've made the remote annex directories 777 permissions for testing, but no change.
+
+Thanks
diff --git a/doc/forum/SSH_remote_transfers_queued_but_no_movement/comment_1_fea4e2317f850d6166480cddba088ae5._comment b/doc/forum/SSH_remote_transfers_queued_but_no_movement/comment_1_fea4e2317f850d6166480cddba088ae5._comment
new file mode 100644
index 0000000000..d9bf5a97f1
--- /dev/null
+++ b/doc/forum/SSH_remote_transfers_queued_but_no_movement/comment_1_fea4e2317f850d6166480cddba088ae5._comment
@@ -0,0 +1,10 @@
+[[!comment format=mdwn
+ username="http://joeyh.name/"
+ ip="108.236.230.124"
+ subject="comment 1"
+ date="2014-09-18T19:41:39Z"
+ content="""
+First, take a look at the output of `ps fax` .. you should see a `git-annex assistant` process and near it there ought to be a `git annex transferkeys` process. See if that process has any children under it, like perhaps a rsync. If so, it might just be stalled talking to the host for some reason.
+
+The best way to debug it further is probably to run `git annex copy --to $remote` at the command line, passing the name of your remote repository. See if it also stalls there. If so, add a --debug and you can see the actual rsync commands it's using, and perhaps work out the problem from there.
+"""]]
diff --git a/doc/forum/Same_Jabber_account_for_different_annexes.mdwn b/doc/forum/Same_Jabber_account_for_different_annexes.mdwn
new file mode 100644
index 0000000000..f58d4abd06
--- /dev/null
+++ b/doc/forum/Same_Jabber_account_for_different_annexes.mdwn
@@ -0,0 +1 @@
+Is it possible to use the same jabber account for different annexes? I think I've mixed up several repositories by using the same jabber account for synchronizing...
diff --git a/doc/forum/Same_Jabber_account_for_different_annexes/comment_1_90c3954fe11980eef42b5f5d34f83488._comment b/doc/forum/Same_Jabber_account_for_different_annexes/comment_1_90c3954fe11980eef42b5f5d34f83488._comment
new file mode 100644
index 0000000000..e39b7b52be
--- /dev/null
+++ b/doc/forum/Same_Jabber_account_for_different_annexes/comment_1_90c3954fe11980eef42b5f5d34f83488._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="http://joeyh.name/"
+ nickname="joey"
+ subject="comment 1"
+ date="2013-04-11T15:02:40Z"
+ content="""
+This is a limitation of how jabber is currently used. You need different accounts to sync different repositories over jabber. Supporting using 1 account is the main open todo item on the [[design/assistant/xmpp]] page.
+"""]]
diff --git a/doc/forum/Same_Jabber_account_for_different_annexes/comment_2_802600b3568e5f94d0550092b22975db._comment b/doc/forum/Same_Jabber_account_for_different_annexes/comment_2_802600b3568e5f94d0550092b22975db._comment
new file mode 100644
index 0000000000..6be7841efd
--- /dev/null
+++ b/doc/forum/Same_Jabber_account_for_different_annexes/comment_2_802600b3568e5f94d0550092b22975db._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="paulvt"
+ ip="2001:1af8:ff19:0:a288:b4ff:fe2c:f600"
+ subject="Resources"
+ date="2013-07-13T08:14:35Z"
+ content="""
+XMPP resources (carrying annex.uuid) cannot be of any help here?
+"""]]
diff --git a/doc/forum/Same_files_are_synced_over_and_over_again.mdwn b/doc/forum/Same_files_are_synced_over_and_over_again.mdwn
new file mode 100644
index 0000000000..73c6696439
--- /dev/null
+++ b/doc/forum/Same_files_are_synced_over_and_over_again.mdwn
@@ -0,0 +1,58 @@
+Hello,
+
+I have posted a similiar (same?) problem already some weeks ago:  But since there has been another problem which was fixed, I dare to repost.
+
+I have three repos: marduk (anything / archive) and horus (standard / client) and asaru (standard / client).
+
+Whenever I do a ```git annex sync --content`` the same files are synced over and over again:
+
+```
+florian@horus ~/Documents (git)-[master] % git annex sync --content
+commit  
+On branch master
+nothing to commit, working directory clean
+ok
+pull marduk 
+ok
+get .localized (from marduk...) 
+SHA256E-s0--e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855
+              0 100%    0.00kB/s    0:00:00 (xfr#1, to-chk=0/1)
+(checksum...) ok
+get Bewerbung/thermoplast.aux (from marduk...) 
+SHA256E-s8--3345a075a22fb3bbc04567a1e5660ed0240d53f5881f458a2234dcd42a78d335.aux
+              8 100%    7.81kB/s    0:00:00 (xfr#1, to-chk=0/1)
+(checksum...) ok
+drop archive/Programmierung/zope/lib/python/Homefolder/.svn/empty-file (locking marduk...) ok
+drop archive/Schule/Geschichte/Biographie 3/biographie.aux (locking marduk...) ok
+pull marduk 
+ok
+(recording state in git...)
+push marduk 
+Counting objects: 8, done.
+Delta compression using up to 4 threads.
+Compressing objects: 100% (8/8), done.
+Writing objects: 100% (8/8), 661 bytes | 0 bytes/s, done.
+Total 8 (delta 6), reused 0 (delta 0)
+To ssh://marduk.local/home/florian/Documents
+   d151eb6..c31839b  git-annex -> synced/git-annex
+ok
+git annex sync --content  81,50s user 7,19s system 50% cpu 2:54,56 total
+```
+
+When I execute the same command again, the same files are synced again. On the other repos it's the same behavior.
+
+git annex does not seem to record presence of that file:
+
+```
+florian@horus ~/Documents (git)-[master] % git annex whereis Bewerbung/thermoplast.aux
+whereis Bewerbung/thermoplast.aux (1 copy) 
+        f30dd4f0-5a03-4022-8f45-7c2620c4c058 -- Marduk [marduk]
+ok
+```
+
+Is it a bug? Any more information I can provide?
+
+EDIT: A pastebin of the output of a sync run without and with ```--debug```: 
+
+Thanks!
+Florian
diff --git a/doc/forum/Same_files_are_synced_over_and_over_again/comment_1_1260b5ede5a88a95e98855363455ca41._comment b/doc/forum/Same_files_are_synced_over_and_over_again/comment_1_1260b5ede5a88a95e98855363455ca41._comment
new file mode 100644
index 0000000000..87b51550b2
--- /dev/null
+++ b/doc/forum/Same_files_are_synced_over_and_over_again/comment_1_1260b5ede5a88a95e98855363455ca41._comment
@@ -0,0 +1,7 @@
+[[!comment format=mdwn
+ username="Horus"
+ subject="comment 1"
+ date="2016-08-26T08:32:05Z"
+ content="""
+Nobody having any clue on that?
+"""]]
diff --git a/doc/forum/Searching_metadata_and_file_content__63__.mdwn b/doc/forum/Searching_metadata_and_file_content__63__.mdwn
new file mode 100644
index 0000000000..c13ff36607
--- /dev/null
+++ b/doc/forum/Searching_metadata_and_file_content__63__.mdwn
@@ -0,0 +1,11 @@
+Hi
+
+Is there any (built-in or otherwise) way to search git-annex metadata and file content?  Ideally I think something that knows about git-annex would be helpful because of files moving around / going away due to metadata driven views, dangling symlinks etc.
+
+I'm imagining:
+
+* Something based on Lucene (solr/elasticsearch) or Xapian for fast searches
+* Probably ideally using git-annex metadata to track which files move where on disk
+* Maybe use of inotify to let it know when git annex has moved file content around or added/removed it from a working tree
+
+Thanks everybody, and Joey for making git annex and being an inspiration
diff --git a/doc/forum/Searching_metadata_and_file_content__63__/comment_1_25ac180194a59b659e0b02bb95dd26aa._comment b/doc/forum/Searching_metadata_and_file_content__63__/comment_1_25ac180194a59b659e0b02bb95dd26aa._comment
new file mode 100644
index 0000000000..8b72cc5854
--- /dev/null
+++ b/doc/forum/Searching_metadata_and_file_content__63__/comment_1_25ac180194a59b659e0b02bb95dd26aa._comment
@@ -0,0 +1,45 @@
+[[!comment format=mdwn
+ username="konubinix"
+ subject="recoll"
+ date="2015-08-18T13:30:21Z"
+ content="""
+I personally use recoll to do that. It is not perfect but works well.
+
+To extract the metadata, I use a script called git-annex_dump_tags.sh whose content is:
+	set -eu
+	FILE=\"${1}\"
+	DIR=\"$(dirname \"${FILE}\")\"
+	FILE_NO_DIR=\"$(basename \"${FILE}\")\"
+	cd \"${DIR}\"
+	git annex metadata \"${FILE_NO_DIR}\"|\
+	    tail -n+2|\
+	    head -n-1|\
+	    sed -r 's/^ +//'|\
+	    sed -r 's/^([^=]+)=(.+)$/\1 = \2/'
+
+Then in the recoll configuration, I added
+	[~/perso]
+	metadatacmds = ; rclmulti_gitannex = git-annex_dump_tags.sh %f
+
+More information can be found in [link the recoll manual](http://www.lesbonscomptes.com/recoll/usermanual/usermanual.html#RCL.INSTALL.CONFIG.RECOLLCONF).
+
+Then you'll have to indicate what key to use in the indexing by updating the \"fields\" file. For instance, you could add:
+	[prefixes]
+	ack = XYACK
+	year = XYMONTH
+	month = XYMONTH
+	day = XYDAY
+
+	[stored]
+	ack=
+	year=
+	month=
+	day=
+
+I generally use the ack metadata in my bibliography to indicate whether I read the paper or not (ack=no or ack=yes). I can get access to all the paper I did not read yet, that were added in August 2015, and that deal with gaussian processes with the query
+	recoll -t -q ack:no year:2015 month:8 gaussian processes
+
+It was not easy to setup, but it does the job.
+
+Hope that helps.
+"""]]
diff --git a/doc/forum/Searching_metadata_and_file_content__63__/comment_2_5dd29566dacb5ca70340d54b469c40a5._comment b/doc/forum/Searching_metadata_and_file_content__63__/comment_2_5dd29566dacb5ca70340d54b469c40a5._comment
new file mode 100644
index 0000000000..9bd3c27424
--- /dev/null
+++ b/doc/forum/Searching_metadata_and_file_content__63__/comment_2_5dd29566dacb5ca70340d54b469c40a5._comment
@@ -0,0 +1,12 @@
+[[!comment format=mdwn
+ username="konubinix"
+ subject="link to the files"
+ date="2015-08-18T13:37:12Z"
+ content="""
+The previous comment did not display the files correctly.
+
+[link the script to dump the tags](https://raw.githubusercontent.com/Konubinix/Devel/master/bin/konix_git-annex_dump_tags.sh)
+
+[link the fields file](https://raw.githubusercontent.com/Konubinix/Devel/master/config/recoll/fields)
+
+"""]]
diff --git a/doc/forum/Searching_metadata_and_file_content__63__/comment_3_67715b73349dc92e9746b234024057ef._comment b/doc/forum/Searching_metadata_and_file_content__63__/comment_3_67715b73349dc92e9746b234024057ef._comment
new file mode 100644
index 0000000000..56986b55c1
--- /dev/null
+++ b/doc/forum/Searching_metadata_and_file_content__63__/comment_3_67715b73349dc92e9746b234024057ef._comment
@@ -0,0 +1,18 @@
+[[!comment format=mdwn
+ username="joey"
+ subject="""comment 3"""
+ date="2015-08-19T17:11:41Z"
+ content="""
+Nice script to use recoil, and I imagine similar approaches could be used
+to integrate with other search engines.
+
+Note that `git annex metadata --json` yields a format better suited to
+machine parsing. Especially because metadata values can contain arbitrary
+content, even potentially newlines.
+
+There are plans to eventually make git-annex use a caching database for
+things, including metadata. This would automatically get updated whenever there
+are changes to the metadata, and SQL queries could then be run against it.
+Or, `git annex metadata` queries could boil down to SQL queries and so run
+a lot faster than they do now.
+"""]]
diff --git a/doc/forum/Searching_metadata_and_file_content__63__/comment_4_b8fbd129664c9680cd77f89185974741._comment b/doc/forum/Searching_metadata_and_file_content__63__/comment_4_b8fbd129664c9680cd77f89185974741._comment
new file mode 100644
index 0000000000..fc94528faa
--- /dev/null
+++ b/doc/forum/Searching_metadata_and_file_content__63__/comment_4_b8fbd129664c9680cd77f89185974741._comment
@@ -0,0 +1,9 @@
+[[!comment format=mdwn
+ username="jean.jordaan@4bb3bd508a9eb0a4bab5d1b587dadd2b6c4a7edc"
+ nickname="jean.jordaan"
+ subject="recoll: there's a Unity Lens for it"
+ date="2015-08-26T13:04:12Z"
+ content="""
+I see there is an Ubuntu Unity search lens for recoll: http://www.webupd8.org/2012/03/recoll-lens-full-text-search-unity-lens.html
+It should be possible to integrate git-annex metadata with that ..
+"""]]
diff --git a/doc/forum/Securing_a_shared_ssh_server.mdwn b/doc/forum/Securing_a_shared_ssh_server.mdwn
new file mode 100644
index 0000000000..3e0006cb36
--- /dev/null
+++ b/doc/forum/Securing_a_shared_ssh_server.mdwn
@@ -0,0 +1,3 @@
+Hi, I'd like to share files with friends, to do so we need a shared server to exchange the files. I could setup an ssh access on one of my servers but I don't want others to have a full access. I assume I can restrict this using a `command='wrapper-around-git'` prefix in `.ssh/authorized_keys`, where `wrapper-around-git` is simply a script that checks if `SSH_ORIGINAL_COMMAND` is an authorised command and runs it. This is exactly the approach used with Mercurial (hg-ssh) and presumably git also relies on this mechanism.
+
+But to do so, I need to know what commands can be executed over ssh when using git-annex assistant. Could you document about this? Thanks!
diff --git a/doc/forum/Securing_a_shared_ssh_server/comment_1_ea971b57d94db5b8d487f728faa5e9a8._comment b/doc/forum/Securing_a_shared_ssh_server/comment_1_ea971b57d94db5b8d487f728faa5e9a8._comment
new file mode 100644
index 0000000000..f82a57295b
--- /dev/null
+++ b/doc/forum/Securing_a_shared_ssh_server/comment_1_ea971b57d94db5b8d487f728faa5e9a8._comment
@@ -0,0 +1,10 @@
+[[!comment format=mdwn
+ username="http://joeyh.name/"
+ nickname="joey"
+ subject="comment 1"
+ date="2013-05-24T15:35:16Z"
+ content="""
+The git-annex assistant automatically sets up a ssh key that is locked down this way when you select \"ssh server\" in the webapp.
+
+The command you need to allow to run is git-annex-shell. This has been designed to be secure.
+"""]]
diff --git a/doc/forum/Securing_a_shared_ssh_server/comment_2_421a19f6e1fb40db6ee205daf8e3f867._comment b/doc/forum/Securing_a_shared_ssh_server/comment_2_421a19f6e1fb40db6ee205daf8e3f867._comment
new file mode 100644
index 0000000000..3d92c8b273
--- /dev/null
+++ b/doc/forum/Securing_a_shared_ssh_server/comment_2_421a19f6e1fb40db6ee205daf8e3f867._comment
@@ -0,0 +1,15 @@
+[[!comment format=mdwn
+ username="https://www.google.com/accounts/o8/id?id=AItOawkwjBDXkP9HAQKhjTgThGOxUa1B99y_WRA"
+ nickname="Franck"
+ subject="comment 2"
+ date="2013-05-24T15:47:36Z"
+ content="""
+Thanks for your quick answer! But this is true only for servers where git-annex is installed. On a server that just does SSH, things must be different. So, I've started to use the SSH `command=` mechanism to log the commands and discovered so far four of them:
+
+ * `sh -c 'echo git-annex-probe...` and `sh -c 'mkdir -p ...` when the server is first connected by a client
+ * `rsync --server -vre.iLsf --partial-dir .rsync-partial . DIR/` and `rsync --server --sender -e31.14 --inplace . DIR//bd1/469/TEXT` when files are transfered between clients
+
+I want to derive patterns from this but if you could give them to me (ie, tell me which parts are fixed and which are variable) this will be safer. Moreover, I'm quite sure there are somme commands missing from my logs... By the way, parameter `-e31.14` to `rsync` surprises me because `-e` is supposed to set the remote shell (like `--rsh`).
+
+Cheers, Franck
+"""]]
diff --git a/doc/forum/Securing_a_shared_ssh_server/comment_3_acdbf92f646dbbf691621f08b3d94c26._comment b/doc/forum/Securing_a_shared_ssh_server/comment_3_acdbf92f646dbbf691621f08b3d94c26._comment
new file mode 100644
index 0000000000..ea48a2caf0
--- /dev/null
+++ b/doc/forum/Securing_a_shared_ssh_server/comment_3_acdbf92f646dbbf691621f08b3d94c26._comment
@@ -0,0 +1,12 @@
+[[!comment format=mdwn
+ username="http://joeyh.name/"
+ nickname="joey"
+ subject="comment 3"
+ date="2013-05-27T19:14:37Z"
+ content="""
+I am confident that it will be *easier* to install git-annex-shell on your ssh server than it will be to lock down rsync. 
+
+All you need to do is go get the standalone linux tarball of git-annex, untar it, and add its directory to PATH.
+
+You can google for perl scripts that lock down rsync, but I have never been happy with any of the ones I found.
+"""]]
diff --git a/doc/forum/Securing_a_shared_ssh_server/comment_4_67533d08e1b8706b844262e9c483d982._comment b/doc/forum/Securing_a_shared_ssh_server/comment_4_67533d08e1b8706b844262e9c483d982._comment
new file mode 100644
index 0000000000..107c7b3cce
--- /dev/null
+++ b/doc/forum/Securing_a_shared_ssh_server/comment_4_67533d08e1b8706b844262e9c483d982._comment
@@ -0,0 +1,15 @@
+[[!comment format=mdwn
+ username="https://www.google.com/accounts/o8/id?id=AItOawkwjBDXkP9HAQKhjTgThGOxUa1B99y_WRA"
+ nickname="Franck"
+ subject="comment 4"
+ date="2013-05-28T06:34:04Z"
+ content="""
+Thanks, but my server is a synology nas and as you know from another thread of comments, having git-annex work on it is not that simple. ;-)
+Moreover, I'd like to be able to use ssh accounts where I don't have a root access and not necessarily git. So, a general method to restrict ssh would interest me. 
+
+But your answer seems to suggest that almost arbitrary rsync commands may be given. If so, I agree that there are few hopes to build a secured jail around this... But if really a limited subset of commands is used, I think it should be possible to check them securely. 
+
+Now on I'm focused on having git-annex work because this looks like the most promising way. But I'll have another question regarding it: I noticed that we can restrict access to a specific repository using an appropriate environnement variable. But it's it possible to provide a list of repositories instead of just one? My collaborators will typically have access to several shares but not to all of them. 
+
+Thanks for your responsiveness, after trying tens of candidates git-annex appears to be the only serious solution to replace Dropbox and I'm really glad that you actively help your users! 
+"""]]
diff --git a/doc/forum/Securing_a_shared_ssh_server/comment_5_bf193e02b388b4358632a169d2425b5c._comment b/doc/forum/Securing_a_shared_ssh_server/comment_5_bf193e02b388b4358632a169d2425b5c._comment
new file mode 100644
index 0000000000..d0f78da92e
--- /dev/null
+++ b/doc/forum/Securing_a_shared_ssh_server/comment_5_bf193e02b388b4358632a169d2425b5c._comment
@@ -0,0 +1,10 @@
+[[!comment format=mdwn
+ username="http://joeyh.name/"
+ nickname="joey"
+ subject="comment 5"
+ date="2013-05-29T16:56:37Z"
+ content="""
+I can't say what options rsync will choose to pass to the server, when the rsync special remote uses rsync. It's up to that command.
+
+`GIT_ANNEX_SHELL_DIRECTORY` currently only supports specifying one directory. 
+"""]]
diff --git a/doc/forum/Securing_a_shared_ssh_server/comment_6_50d391992cd444080ebc70db30b215c5._comment b/doc/forum/Securing_a_shared_ssh_server/comment_6_50d391992cd444080ebc70db30b215c5._comment
new file mode 100644
index 0000000000..fce7ab3dfb
--- /dev/null
+++ b/doc/forum/Securing_a_shared_ssh_server/comment_6_50d391992cd444080ebc70db30b215c5._comment
@@ -0,0 +1,9 @@
+[[!comment format=mdwn
+ username="https://www.google.com/accounts/o8/id?id=AItOawkwjBDXkP9HAQKhjTgThGOxUa1B99y_WRA"
+ nickname="Franck"
+ subject="comment 6"
+ date="2013-05-29T17:41:25Z"
+ content="""
+OK, I understand now why it is complicated to secure rsync, the best I can hope is to chroot it on a restricted account. Thanks! And the \"currently\" regarding ˋGIT_ANNEX_SHELL_DIRECTORYˋ gives me hope about a future change.  :-)
+Cheers, Franck
+"""]]
diff --git a/doc/forum/Sending_requests_across_the_network.mdwn b/doc/forum/Sending_requests_across_the_network.mdwn
new file mode 100644
index 0000000000..db53224776
--- /dev/null
+++ b/doc/forum/Sending_requests_across_the_network.mdwn
@@ -0,0 +1,15 @@
+Hi,
+
+Is it possible to have git-annex send requests across the repository network? Say I have a network topology like this:  
+Home (annex) <-> Cloud (S3) <-> Laptop (annex)  
+
+Home has all files, cloud has zero, and laptop has subset of files. Let's also assume Laptop can't talk to Home directly (maybe it's behind a NAT), but both Home and Laptop are connected to the internet.  
+If I'm away on my laptop, can I retrieve a file from Home "through" Cloud?
+
+That is, Laptop checking its remotes and none of them have the file I want, so it checks remotes of remotes, etc. I'm not sure if git-annex knows the topology (seems likely considering it can generate the graphviz image). It also seems there is a communication medium of Jabber/XMPP where it could communicate between internet-connected git-annex servers (which might not otherwise be able to talk to each other directly)? So the fact that Cloud isn't a git-annex server, just a dumb key/value store would be okay?
+
+I realize that I could set it up so Home mirrors everything to Cloud and then that eventually mirrors over to Laptop, but let's assume both Cloud and Laptop have small storage capacities, so on-demand fetching would be needed.
+
+This is basically the same usecase as the USB transfer drive to sync two annexes not on the same network, but automated.
+
+Thanks!
diff --git a/doc/forum/Sending_requests_across_the_network/comment_1_8ff713d4c968705061bf2044ea0fe5a0._comment b/doc/forum/Sending_requests_across_the_network/comment_1_8ff713d4c968705061bf2044ea0fe5a0._comment
new file mode 100644
index 0000000000..8cb4575c10
--- /dev/null
+++ b/doc/forum/Sending_requests_across_the_network/comment_1_8ff713d4c968705061bf2044ea0fe5a0._comment
@@ -0,0 +1,10 @@
+[[!comment format=mdwn
+ username="http://ypid.wordpress.com/"
+ ip="213.153.84.215"
+ subject="preferred content …"
+ date="2014-07-02T07:26:59Z"
+ content="""
+Hi
+
+This can be accomplished with the use of [[/preferred_content/]].
+"""]]
diff --git a/doc/forum/Sending_requests_across_the_network/comment_2_cb29e5346a8775d87d30b18b7fc005a7._comment b/doc/forum/Sending_requests_across_the_network/comment_2_cb29e5346a8775d87d30b18b7fc005a7._comment
new file mode 100644
index 0000000000..87e6976fe9
--- /dev/null
+++ b/doc/forum/Sending_requests_across_the_network/comment_2_cb29e5346a8775d87d30b18b7fc005a7._comment
@@ -0,0 +1,10 @@
+[[!comment format=mdwn
+ username="http://joeyh.name/"
+ ip="108.236.230.124"
+ subject="comment 2"
+ date="2014-07-02T17:25:51Z"
+ content="""
+The network topology shown has only a S3 special remote connecting the two repositories. This only provides a place to store file contents, it does not allow syncing the contents of git repositories. 
+
+You need either XMPP(Jabber) for that, or a git remote on some other server. Once there is a way for the git repository to sync, the Home repository will see when Laptop is missing a file it wants (configured via the preferred content mentioned above), and upload it to S3.
+"""]]
diff --git a/doc/forum/Sending_requests_across_the_network/comment_3_9859c46db3527ad329c8e0df06edd153._comment b/doc/forum/Sending_requests_across_the_network/comment_3_9859c46db3527ad329c8e0df06edd153._comment
new file mode 100644
index 0000000000..c63404e0e8
--- /dev/null
+++ b/doc/forum/Sending_requests_across_the_network/comment_3_9859c46db3527ad329c8e0df06edd153._comment
@@ -0,0 +1,11 @@
+[[!comment format=mdwn
+ username="neocryptek@659edac901ffbc8e541a974f8f18987eeafc63bd"
+ nickname="neocryptek"
+ avatar="http://cdn.libravatar.org/avatar/d9bfdefa9b503f1ac4844a686618374e"
+ subject="comment 3"
+ date="2016-11-13T22:39:44Z"
+ content="""
+Thanks, that makes sense.
+
+All git annex repositories using the same branch will have the same (symlink) working directory right (assuming entire network has been synced eventually)?
+"""]]
diff --git a/doc/forum/Setup_of_rsync_special_remote_with_non-standard_ssh_port.mdwn b/doc/forum/Setup_of_rsync_special_remote_with_non-standard_ssh_port.mdwn
new file mode 100644
index 0000000000..85755e982d
--- /dev/null
+++ b/doc/forum/Setup_of_rsync_special_remote_with_non-standard_ssh_port.mdwn
@@ -0,0 +1,13 @@
+I want to setup a rsync special remote on my server with a non-standard ssh port.
+
+I tried the following steps:
+
+    git annex initremote rsync-encrypted type=rsync rsyncurl=1.2.3.4:/encrypted-annex encryption=AAAAAAAAA
+    git config remote.rsync-encrypted.annex-rsync-options "-e \'ssh -p 443\'"
+
+But I just get this error:
+
+    [2012-11-22 21:04:30 CET] read: rsync ["-e","'ssh","-p","443'","--progress","--recursive","--partial","-- partial-dir=.rsync-partial","/home/marco/annex/.git/annex/tmp/rsynctmp/15309/","1.2.3.4:/encrypted-annex"]
+    Missing trailing-' in remote-shell command.
+
+I tried some ways to escape the config but I don't have a clue. Anybody?
diff --git a/doc/forum/Setup_of_rsync_special_remote_with_non-standard_ssh_port/comment_1_1eb6990e93ec92cb6fd7dbee59f31072._comment b/doc/forum/Setup_of_rsync_special_remote_with_non-standard_ssh_port/comment_1_1eb6990e93ec92cb6fd7dbee59f31072._comment
new file mode 100644
index 0000000000..5af84acc07
--- /dev/null
+++ b/doc/forum/Setup_of_rsync_special_remote_with_non-standard_ssh_port/comment_1_1eb6990e93ec92cb6fd7dbee59f31072._comment
@@ -0,0 +1,13 @@
+[[!comment format=mdwn
+ username="https://www.google.com/accounts/o8/id?id=AItOawmLB39PC89rfGaA8SwrsnB6tbumezj-aC0"
+ nickname="Tobias"
+ subject="comment 1"
+ date="2012-11-22T22:44:11Z"
+ content="""
+Add the host to your .ssh/config like this
+
+Host myotherpc
+     HostName 10.0.0.10
+     Port 2222
+
+"""]]
diff --git a/doc/forum/Setup_of_rsync_special_remote_with_non-standard_ssh_port/comment_2_c85d5167e7ccce1ecf1de396e72ce7bc._comment b/doc/forum/Setup_of_rsync_special_remote_with_non-standard_ssh_port/comment_2_c85d5167e7ccce1ecf1de396e72ce7bc._comment
new file mode 100644
index 0000000000..c336be0d1c
--- /dev/null
+++ b/doc/forum/Setup_of_rsync_special_remote_with_non-standard_ssh_port/comment_2_c85d5167e7ccce1ecf1de396e72ce7bc._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="https://www.google.com/accounts/o8/id?id=AItOawnldTTAP8PAifJUmqhRar6RAWNWlRcencw"
+ nickname="Marco"
+ subject="~/.ssh/config"
+ date="2012-11-23T05:08:47Z"
+ content="""
+Thank you very much. Works like a charm.
+"""]]
diff --git a/doc/forum/Several_concurrent_operations.mdwn b/doc/forum/Several_concurrent_operations.mdwn
new file mode 100644
index 0000000000..1b7a383fde
--- /dev/null
+++ b/doc/forum/Several_concurrent_operations.mdwn
@@ -0,0 +1 @@
+Is it safe to have several `git annex` processes running concurrently in the same repo or in multiple repositories? For example, I have several repositories syncing to each other. Now I run `git annex sync` on them sequentially. Is it safe to run them concurrently?
diff --git a/doc/forum/Several_concurrent_operations/comment_1_885edabc67687e727ab1a111d7d10d1f._comment b/doc/forum/Several_concurrent_operations/comment_1_885edabc67687e727ab1a111d7d10d1f._comment
new file mode 100644
index 0000000000..4c1f739d14
--- /dev/null
+++ b/doc/forum/Several_concurrent_operations/comment_1_885edabc67687e727ab1a111d7d10d1f._comment
@@ -0,0 +1,7 @@
+[[!comment format=mdwn
+ username="joey"
+ subject="""comment 1"""
+ date="2018-08-15T15:43:00Z"
+ content="""
+git-annex is concurrency safe
+"""]]
diff --git a/doc/forum/Several_concurrent_operations/comment_2_581e53306e47ddb826c030333f9ae9c2._comment b/doc/forum/Several_concurrent_operations/comment_2_581e53306e47ddb826c030333f9ae9c2._comment
new file mode 100644
index 0000000000..9bee88f4c8
--- /dev/null
+++ b/doc/forum/Several_concurrent_operations/comment_2_581e53306e47ddb826c030333f9ae9c2._comment
@@ -0,0 +1,9 @@
+[[!comment format=mdwn
+ username="https://launchpad.net/~liori"
+ nickname="liori"
+ avatar="http://cdn.libravatar.org/avatar/e1d0fdc746b3d21bb147160d40815e37b257b9119774d21784939b2d3ba95a91"
+ subject="comment 2"
+ date="2018-08-17T21:58:03Z"
+ content="""
+So, indeed it works. However, syncing a large archive (2TB, ~1M objects) over several USB HDDs at the same time ended up a bad idea: inevitably at some point every process tried syncing with the same, slowest, repository, and lack of good I/O concurrency on a USB-connected HDD killed all performance. I'm guessing that introducing some targetted locking could probably help, or even maybe just randomizing the order in which repositories are pulled from/pushed to.
+"""]]
diff --git a/doc/forum/Share_only_certain_files_of_a_repo___40__Assistant__41__.mdwn b/doc/forum/Share_only_certain_files_of_a_repo___40__Assistant__41__.mdwn
new file mode 100644
index 0000000000..53d0e0165c
--- /dev/null
+++ b/doc/forum/Share_only_certain_files_of_a_repo___40__Assistant__41__.mdwn
@@ -0,0 +1,6 @@
+Hi,
+
+When using git-annex assistant, is there a possibility to share only certain files or directories from a repository (similar to what Dropbox can do), or do we have to share whole repositories so far?
+Could this be a nice wishlist item, or is there a strong reason against adding this feature to git-annex?
+
+Thanks a lot!
diff --git a/doc/forum/Share_only_certain_files_of_a_repo___40__Assistant__41__/comment_1_ec0d56cb31b918023a9184cee168b406._comment b/doc/forum/Share_only_certain_files_of_a_repo___40__Assistant__41__/comment_1_ec0d56cb31b918023a9184cee168b406._comment
new file mode 100644
index 0000000000..8fa6f7b40b
--- /dev/null
+++ b/doc/forum/Share_only_certain_files_of_a_repo___40__Assistant__41__/comment_1_ec0d56cb31b918023a9184cee168b406._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="http://joeyh.name/"
+ ip="4.153.255.110"
+ subject="comment 1"
+ date="2013-09-09T19:44:57Z"
+ content="""
+Everyone can see the names of every file that is put into the git repository. Whether the contents of those files is transferred from the machine where they were created to other clones of the repository is configurable using [[preferred_content]], but it's not really intended as an access control mechanism.
+"""]]
diff --git a/doc/forum/Share_with_friend_copies_only_sym_links.mdwn b/doc/forum/Share_with_friend_copies_only_sym_links.mdwn
new file mode 100644
index 0000000000..8f31b63372
--- /dev/null
+++ b/doc/forum/Share_with_friend_copies_only_sym_links.mdwn
@@ -0,0 +1,38 @@
+I just setup git-annex across three macs, using git-annex assistant.  I have one repository working fine, which successfully syncs between two of my own accounts on two different machines, via S3.
+
+Now I'm trying to get the "share with a friend" feature to work.  The problem is that only Mac Alias's / sym links are created, for example:
+
+Machine 1:
+- Copy file into ~/annex/
+
+Machine 2:
+- The file is synced as a sym-link, with a link into a file in .git that doesn't exist.
+
+The config is:
+- Version: 4.20130909-ga29f960 - Build flags: Assistant Webapp Pairing Testsuite S3 WebDAV FsEvents XMPP DNS Feeds Quvi
+- Two macs
+- Share with friend over Jabber
+- Use Box as the cloud sync service with encryption enabled.  Box is setup as a transfer repository.
+
+I've looked through the logs in .git/annex/daemon.log* and there doesn't look to be any problems.
+
+This is what the log looks like:
+
+    [2013-09-14 19:55:47 PDT] Committer: Adding 20120101-..084-1.jpg
+    failed
+    add /Users/nick/Shared/20120101-GOPR0084-1.jpg (checksum...) ok 
+    add /Users/nick/Shared/20120101-GOPR0084-1.jpg (checksum...) ok 
+    add /Users/nick/Shared/20120101-GOPR0084-1.jpg (checksum...) ok 
+    add /Users/nick/Shared/20120101-GOPR0084-1.jpg (checksum...) ok 
+    add /Users/nick/Shared/20120101-GOPR0084-1.jpg (checksum...) [2013-09-14 19:55:47 PDT] Committer: Committing changes to git
+    [2013-09-14 19:55:48 PDT] XMPPSendPack: Syncing with USER2 
+    Already up-to-date.
+    To xmpp::USER2@gmail.com
+       7b65b15..183b91c  git-annex -> refs/synced/6163ef8e-c36a-4b09-919b-9c18ade55234/bmlja2JsYWNrMUBnbWFpbC5jb20=/git-annex
+       3104275..96afa09  master -> refs/synced/6163ef8e-c36a-4b09-919b-9c18ade55234/bmlja2JsYWNrMUBnbWFpbC5jb20=/master
+    [2013-09-14 19:55:49 PDT] XMPPSendPack: Syncing with USER2 
+    Everything up-to-date
+    recv: resource vanished (Connection reset by peer)
+
+
+Any ideas what I should look at next?  Thanks in advance :-)
diff --git a/doc/forum/Share_with_friend_copies_only_sym_links/comment_1_a8d22dfefb219f0c9130cc294364b198._comment b/doc/forum/Share_with_friend_copies_only_sym_links/comment_1_a8d22dfefb219f0c9130cc294364b198._comment
new file mode 100644
index 0000000000..84217711aa
--- /dev/null
+++ b/doc/forum/Share_with_friend_copies_only_sym_links/comment_1_a8d22dfefb219f0c9130cc294364b198._comment
@@ -0,0 +1,17 @@
+[[!comment format=mdwn
+ username="http://joeyh.name/"
+ ip="4.153.14.105"
+ subject="comment 1"
+ date="2013-09-19T17:41:47Z"
+ content="""
+git-annex can only share the contents of files with a friend when you both have access to a transfer repository. Your friend's git-annex webapp should detect that it does not have such a repository, and should be displaying an alert like this:
+
+[[assistant/cloudnudge.png]]
+(But in this case it should say \"Unable to download files from YourName\")
+
+So, you need to give your friend access to the S3 repository that you're already using to keep your 2 computers in sync. Or, set up some other transfer repository that you both can access.
+
+There is a way in S3 to create another key that can read and write to a S3 bucket. You can then give that key to your friend.
+
+Or, you could set up a shared git repository on a ssh server you both have access to. Or, set up a box.com remote -- the webapp will offer to store the box.com credentials and share them with your friend.
+"""]]
diff --git a/doc/forum/Shared_Encryption_does_not_Work_as_Expected.mdwn b/doc/forum/Shared_Encryption_does_not_Work_as_Expected.mdwn
new file mode 100644
index 0000000000..660006a5ba
--- /dev/null
+++ b/doc/forum/Shared_Encryption_does_not_Work_as_Expected.mdwn
@@ -0,0 +1,11 @@
+I recently started to set up a share that would be synced between multiple machines.  Since I control all of them I thought it might be cool to try using the suggested "shared encryption".  This seems to have been a mistake.
+
+My project folder is now a big mess of symlinks to read-only files.  
+
+I can fix this with a "cp -RH" and get my data back and I can muddle through fixing the permissions but this leads me to two conclusions:
+
+1.  I did something wrong but I don't know what.  Should I have created the ssh remote first and the local repo second?  
+
+2.  This feature should either be hidden or the UI should be much more explicit about how to use it properly.  This cost me quite a bit of time and frustration.
+
+I really like git-annex so I would appreciate any comments or suggestions.
diff --git a/doc/forum/Shared_Encryption_does_not_Work_as_Expected/comment_1_793c71a1efcf160e1829bec3ef0b1be6._comment b/doc/forum/Shared_Encryption_does_not_Work_as_Expected/comment_1_793c71a1efcf160e1829bec3ef0b1be6._comment
new file mode 100644
index 0000000000..25424d4a0c
--- /dev/null
+++ b/doc/forum/Shared_Encryption_does_not_Work_as_Expected/comment_1_793c71a1efcf160e1829bec3ef0b1be6._comment
@@ -0,0 +1,10 @@
+[[!comment format=mdwn
+ username="https://openid.stackexchange.com/user/e65e6d0e-58ba-41de-84cc-1f2ba54cf574"
+ nickname="Mica Semrick"
+ subject="details"
+ date="2014-11-16T04:42:36Z"
+ content="""
+Hello, 
+
+It is hard to say what happened without an error message or description of what you did. Perhaps you can share those details.
+"""]]
diff --git a/doc/forum/Shared_Encryption_does_not_Work_as_Expected/comment_2_6b603265a2fc78bd2df54102bf94234f._comment b/doc/forum/Shared_Encryption_does_not_Work_as_Expected/comment_2_6b603265a2fc78bd2df54102bf94234f._comment
new file mode 100644
index 0000000000..bec7954c8e
--- /dev/null
+++ b/doc/forum/Shared_Encryption_does_not_Work_as_Expected/comment_2_6b603265a2fc78bd2df54102bf94234f._comment
@@ -0,0 +1,12 @@
+[[!comment format=mdwn
+ username="joey"
+ subject="""comment 2"""
+ date="2014-12-02T17:37:49Z"
+ content="""
+Symlinks to read-only files in .git/annex/objects is how a git-annex
+repository normally looks.
+
+Perhaps you want to use [[direct_mode]]?
+
+This has nothing to do with shared encryption that I can see.
+"""]]
diff --git a/doc/forum/Shared_directory_with_non_git-annex_users.mdwn b/doc/forum/Shared_directory_with_non_git-annex_users.mdwn
new file mode 100644
index 0000000000..b6e9d5aaaa
--- /dev/null
+++ b/doc/forum/Shared_directory_with_non_git-annex_users.mdwn
@@ -0,0 +1,15 @@
+Hello,
+
+I want to use git-annex on a directory that also other persons, without git-annex use, i.e. modify, add and remove files to.
+
+What need I consider when doing that?
+
+ * Use indirect mode
+ * The the .git directory somewhere else?
+ * Be prepared that a ```git annex get``` can always bring up checksum problems because someone modified the file
+
+I assume git-annex checksums before pushing to the repo, so that I don't otherwrite someone elses changes.
+
+What else?
+
+Thanks!
diff --git a/doc/forum/Shared_directory_with_non_git-annex_users/comment_1_a56cb4993982e030eb9fd4cdb3b0c368._comment b/doc/forum/Shared_directory_with_non_git-annex_users/comment_1_a56cb4993982e030eb9fd4cdb3b0c368._comment
new file mode 100644
index 0000000000..698967e7ab
--- /dev/null
+++ b/doc/forum/Shared_directory_with_non_git-annex_users/comment_1_a56cb4993982e030eb9fd4cdb3b0c368._comment
@@ -0,0 +1,50 @@
+[[!comment format=mdwn
+ username="http://xgm.de/oid/"
+ nickname="Florian"
+ avatar="http://cdn.libravatar.org/avatar/4c5c0e290374d76c713f482e41f60a3cbee0fa64bb94c6da94e5a61a50824811"
+ subject="comment 1"
+ date="2017-09-24T04:21:11Z"
+ content="""
+Ok, I tried it using ```GIT_DIR``` and ```GIT_WORK_TREE```:
+
+    Current directory is ~/git-annex, ./work exists and is populated with some files.
+
+    % GIT_DIR=~/git-annex/git GIT_WORK_TREE=~/git-annex/work git init
+    % GIT_DIR=~/git-annex/git GIT_WORK_TREE=~/git-annex/work git annex init \"server\"
+    % GIT_DIR=~/git-annex/git GIT_WORK_TREE=~/git-annex/work git annex direct
+    % GIT_DIR=~/git-annex/git GIT_WORK_TREE=~/git-annex/work git annex add .
+    [... file are addded ...]
+    % GIT_DIR=~/git-annex/git GIT_WORK_TREE=~/git-annex/work git annex sync
+    [... file are synced ...]
+    
+    % git clone git remote
+    % cd remote
+    % git annex init \"remote\"
+    % git annex sync
+    
+    
+    % git annex get a.out
+    get a.out
+      Unable to access these remotes: origin
+    
+      Try making some of these repositories available:
+            ed208c9f-a963-4000-a505-c3fe9dab0042 -- server [origin]
+    failed
+    git-annex: get: 1 failed
+    
+    
+    % git annex whereis a.out
+    whereis a.out (1 copy)
+            ed208c9f-a963-4000-a505-c3fe9dab0042 -- server [origin]
+    ok
+
+
+The remote is of course available, it's all local.
+
+What is still wrong?
+
+Thanks,
+Florian
+
+
+"""]]
diff --git a/doc/forum/Shared_pubkeys__58___decrypting_files_in_special_remotes_without_git-annex.mdwn b/doc/forum/Shared_pubkeys__58___decrypting_files_in_special_remotes_without_git-annex.mdwn
new file mode 100644
index 0000000000..6001cdd50a
--- /dev/null
+++ b/doc/forum/Shared_pubkeys__58___decrypting_files_in_special_remotes_without_git-annex.mdwn
@@ -0,0 +1,16 @@
+Hi,
+
+I'm trying to update the script provided in topic "Decrypting files in special remotes without git-annex" to add support for different MAC algorithms, chunks and sharedpubkey encryption scheme.
+
+I can already decrypt the files using my GPG keys directly, and I'm trying to make the key lookup function to work.
+
+I have difficulties to parse git-annex source code, I cannot find the exact way the special remote keys are computed. I used the script from "yibe" as a starting point (refer to comment #1) because it is in pure bash I can understand:
+
+- From the doc, sharedpubkey cipher is unencrypted in the remote, it is only base64-ed and limited to 256 characters. So I added:
+cipher="$(echo -n "$cipher" | base64 -d | head -c 256)" like it is done for the "shared" encryption scheme.
+
+- From Yibe's script, chunks keys have an extra "-S-C--" values inside the annex key. But I doubt this part. I tried with and without it: no success.
+
+- Yibe's script handles MAC algorithms correctly. I see no issue there.
+
+In the end, I cannot get the right remote keys for my test file. Question: is there a documentation somewhere about all this encoding chain?
diff --git a/doc/forum/Shared_pubkeys__58___decrypting_files_in_special_remotes_without_git-annex/comment_10_d214446e2b5a95a6e14b288457e53305._comment b/doc/forum/Shared_pubkeys__58___decrypting_files_in_special_remotes_without_git-annex/comment_10_d214446e2b5a95a6e14b288457e53305._comment
new file mode 100644
index 0000000000..47d6d32375
--- /dev/null
+++ b/doc/forum/Shared_pubkeys__58___decrypting_files_in_special_remotes_without_git-annex/comment_10_d214446e2b5a95a6e14b288457e53305._comment
@@ -0,0 +1,20 @@
+[[!comment format=mdwn
+ username="oliv5"
+ avatar="http://cdn.libravatar.org/avatar/d7f0d33c51583bbd8578e4f1f9f8cf4b"
+ subject="comment 10"
+ date="2018-07-20T22:48:47Z"
+ content="""
+I managed to get the lookup_key() working with sharedpubkey, chunks and mac SHA512, by combining Joey's cipher and yibe's script.
+
+I have to clean the script before posting, but it is basically Yibe's script [(at the bottom of this page)](https://git-annex.branchable.com/tips/Decrypting_files_in_special_remotes_without_git-annex/) with the sharedpubkey cipher above, and nothing more.
+
+    +    elif [ \"$encryption\" = \"sharedpubkey\" ] ; then
+    +        cipher=\"$(echo -n \"$cipher\" | base64 -d)
+    +\"
+        fi
+
+It is working good for my test file, but this file is made of 1 chunk only. Before relying on the script, more tests should be done with bigger files, e.g more chunks.
+
+Anyway, thks to all of you for your help.
+
+"""]]
diff --git a/doc/forum/Shared_pubkeys__58___decrypting_files_in_special_remotes_without_git-annex/comment_11_d7cf8754a3e923d3f90be44ebd224d18._comment b/doc/forum/Shared_pubkeys__58___decrypting_files_in_special_remotes_without_git-annex/comment_11_d7cf8754a3e923d3f90be44ebd224d18._comment
new file mode 100644
index 0000000000..264dfd892b
--- /dev/null
+++ b/doc/forum/Shared_pubkeys__58___decrypting_files_in_special_remotes_without_git-annex/comment_11_d7cf8754a3e923d3f90be44ebd224d18._comment
@@ -0,0 +1,10 @@
+[[!comment format=mdwn
+ username="oliv5"
+ avatar="http://cdn.libravatar.org/avatar/d7f0d33c51583bbd8578e4f1f9f8cf4b"
+ subject="comment 11"
+ date="2018-07-20T23:03:37Z"
+ content="""
+The original script with sharedpubkey support is available [here](https://raw.githubusercontent.com/oliv5/profile/master/pbin/git-annex-decrypt.sh)
+
+Yibe's enhanced script with sharedpubkey, different mac and chunk support is available [here](https://raw.githubusercontent.com/oliv5/profile/master/pbin/git-annex-decrypt-2.sh). Chunk management should be verified before relying on it, I haven't done it yet except for a basic case (1 chunk per file). Mac support seems ok.
+"""]]
diff --git a/doc/forum/Shared_pubkeys__58___decrypting_files_in_special_remotes_without_git-annex/comment_1_cf81a5c0d055875a858dd1e6a137b53c._comment b/doc/forum/Shared_pubkeys__58___decrypting_files_in_special_remotes_without_git-annex/comment_1_cf81a5c0d055875a858dd1e6a137b53c._comment
new file mode 100644
index 0000000000..a43883348c
--- /dev/null
+++ b/doc/forum/Shared_pubkeys__58___decrypting_files_in_special_remotes_without_git-annex/comment_1_cf81a5c0d055875a858dd1e6a137b53c._comment
@@ -0,0 +1,17 @@
+[[!comment format=mdwn
+ username="andrew"
+ avatar="http://cdn.libravatar.org/avatar/acc0ece1eedf07dd9631e7d7d343c435"
+ subject="comment 1"
+ date="2018-07-09T01:25:22Z"
+ content="""
+[internals](https://git-annex.branchable.com/internals/) is the place to start for all of this documentation.
+
+I believe that the special remote key names (written by Joey) are the same as the keys you see in `.git` which are documented here: 
+[key format](https://git-annex.branchable.com/internals/key_format/), you can see this is independent of encryption scheme, and [hashing](https://git-annex.branchable.com/internals/hashing/) talks about the nested directories structure, which you would also need to know.
+
+Some more context on the [Decrypting files in special remotes without git-annex](https://git-annex.branchable.com/tips/Decrypting_files_in_special_remotes_without_git-annex/) tip is at the related forum post [Future proofing / disaster recovery with an encrypted special remote](http://git-annex.branchable.com/forum/Future_proofing___47___disaster_recovery_with_an_encrypted_special_remote/). Encryption documentation is at [encryption](https://git-annex.branchable.com/encryption/).
+
+But, since you don't actually want to experiment with how keys are stored, and you don't mind relying on `git-annex` I think an easier way to experiment with different encryption schemes would be to [implement your own special remote](https://git-annex.branchable.com/special_remotes/external/), that page contains an `example.sh` code block which is well documented. With `example.sh` you would just need to change `doretrieve` to decrypt before retrieving and `dostore` encrypt before storing.
+
+
+"""]]
diff --git a/doc/forum/Shared_pubkeys__58___decrypting_files_in_special_remotes_without_git-annex/comment_2_82be119d86818f63da519fa1669d8cdd._comment b/doc/forum/Shared_pubkeys__58___decrypting_files_in_special_remotes_without_git-annex/comment_2_82be119d86818f63da519fa1669d8cdd._comment
new file mode 100644
index 0000000000..c723f63bc8
--- /dev/null
+++ b/doc/forum/Shared_pubkeys__58___decrypting_files_in_special_remotes_without_git-annex/comment_2_82be119d86818f63da519fa1669d8cdd._comment
@@ -0,0 +1,82 @@
+[[!comment format=mdwn
+ username="oliv5"
+ avatar="http://cdn.libravatar.org/avatar/d7f0d33c51583bbd8578e4f1f9f8cf4b"
+ subject="comment 2"
+ date="2018-07-15T22:46:05Z"
+ content="""
+Thks. My apologies for the long answer delay.
+
+Let's clarify what I'm trying to do: from the files of an encrypted special remote, with their data and filenames encrypted, I'd like to recover the original files back, without using git-annex at all, just in case one day I 'm not able to use git-annex anymore. This is the purpose of the script presented in [Future proofing / disaster recovery with an encrypted special remote](https://git-annex.branchable.com/forum/Future_proofing___47___disaster_recovery_with_an_encrypted_special_remote/)
+
+*\"I believe that the special remote key names (written by Joey) are the same as the keys you see in .git\"*
+
+This is wrong, and the confusion is this: the local repository key does not change (it is not encrypted), but the file sent to the special remote uses an encrypted key (filename if you prefer). Yes, the file sent to the special remote has its data encrypted and its filename hashed and encrypted too.
+
+*\"I think an easier way to experiment with different encryption schemes would be to implement your own special remote\"*
+
+I did almost this actually: I'm using git-annex \"--debug\" command switch which shows all git commands under the hood. In these, I can see the final encrypted key, which is different than the original one.
+
+An exemple is better than a long speech: I'm using a rclone special remote with the \"shared pubkey\" encryption scheme (see [Encryption](https://git-annex.branchable.com/encryption/), section sharedpubkey). In my local test repo, I have a single file. I can upload and download the file from the special remote as expected.
+
+Local filename: ./test.pdf
+
+Local file: .git/annex/objects/F3/pf/SHA256E-s127597--abc14a6cf4ebb79fdc2eb0d1bf9c304cfce30959661e72e98536faf1bb1b393b.pdf/SHA256E-s127597--abc14a6cf4ebb79fdc2eb0d1bf9c304cfce30959661e72e98536faf1bb1b393b.pdf
+
+Local key: SHA256E-s127597--abc14a6cf4ebb79fdc2eb0d1bf9c304cfce30959661e72e98536faf1bb1b393b.pdf
+
+Special remote debug log: git annex get ./test.pdf --debug
+
+    [2018-07-08 11:37:17.92299562] read: git [\"--git-dir=.git\",\"--work-tree=.\",\"--literal-pathspecs\",\"show-ref\",\"git-annex\"]
+    [2018-07-08 11:37:17.926091484] process done ExitSuccess
+    [2018-07-08 11:37:17.926263261] read: git [\"--git-dir=.git\",\"--work-tree=.\",\"--literal-pathspecs\",\"show-ref\",\"--hash\",\"refs/heads/git-annex\"]
+    [2018-07-08 11:37:17.929194789] process done ExitSuccess
+    [2018-07-08 11:37:17.929657888] read: git [\"--git-dir=.git\",\"--work-tree=.\",\"--literal-pathspecs\",\"log\",\"refs/heads/git-annex..bc15cac806cf0e178b4a4edb29d59ec34a8124cd\",\"--pretty=%H\",\"-n1\"]
+    [2018-07-08 11:37:17.931914542] process done ExitSuccess
+    [2018-07-08 11:37:17.932438022] chat: git [\"--git-dir=.git\",\"--work-tree=.\",\"--literal-pathspecs\",\"cat-file\",\"--batch\"]
+    [2018-07-08 11:37:17.933180259] chat: git [\"--git-dir=.git\",\"--work-tree=.\",\"--literal-pathspecs\",\"cat-file\",\"--batch-check=%(objectname) %(objecttype) %(objectsize)\"]
+    [2018-07-08 11:37:17.93938515] read: git [\"--git-dir=.git\",\"--work-tree=.\",\"--literal-pathspecs\",\"ls-files\",\"--cached\",\"-z\",\"--\",\"./test.pdf\"]
+    get test.pdf (from myremote...) 
+    [2018-07-08 11:37:17.947328106] chat: /home/olivier/bin/git-annex-remote-rclone []
+    [2018-07-08 11:37:17.949286944] git-annex-remote-rclone[1] --> VERSION 1
+    [2018-07-08 11:37:17.949439446] git-annex-remote-rclone[1] <-- EXTENSIONS INFO
+    [2018-07-08 11:37:17.949976023] git-annex-remote-rclone[1] --> UNSUPPORTED-REQUEST
+    [2018-07-08 11:37:17.950062015] git-annex-remote-rclone[1] <-- PREPARE
+    [2018-07-08 11:37:17.950285846] git-annex-remote-rclone[1] --> GETCONFIG prefix
+    [2018-07-08 11:37:17.950360356] git-annex-remote-rclone[1] <-- VALUE mystore
+    [2018-07-08 11:37:17.953250415] git-annex-remote-rclone[1] --> GETCONFIG target
+    [2018-07-08 11:37:17.953381013] git-annex-remote-rclone[1] <-- VALUE storage
+    [2018-07-08 11:37:17.956128895] git-annex-remote-rclone[1] --> GETCONFIG rclone_layout
+    [2018-07-08 11:37:17.956253609] git-annex-remote-rclone[1] <-- VALUE lower
+    [2018-07-08 11:37:17.958960866] git-annex-remote-rclone[1] --> PREPARE-SUCCESS
+    [2018-07-08 11:37:17.959086461] git-annex-remote-rclone[1] <-- TRANSFER RETRIEVE GPGHMACSHA512--9cbf6fe8def32a6b434c8bfc8991916ff425a0c990be48fffe647c5ab7a6b294ba38e96fa58aebd59eb5b14d0e98475b241cd6098f08f2c10953b999d1bcd01c .git/annex/tmp/GPGHMACSHA512--9cbf6fe8def32a6b434c8bfc8991916ff425a0c990be48fffe647c5ab7a6b294ba38e96fa58aebd59eb5b14d0e98475b241cd6098f08f2c10953b999d1bcd01c
+    [2018-07-08 11:37:17.9597168] git-annex-remote-rclone[1] --> DIRHASH-LOWER GPGHMACSHA512--9cbf6fe8def32a6b434c8bfc8991916ff425a0c990be48fffe647c5ab7a6b294ba38e96fa58aebd59eb5b14d0e98475b241cd6098f08f2c10953b999d1bcd01c
+    [2018-07-08 11:37:17.959807775] git-annex-remote-rclone[1] <-- VALUE 00a/620/
+    [2018-07-08 11:37:19.129883435] git-annex-remote-rclone[1] --> TRANSFER-SUCCESS RETRIEVE GPGHMACSHA512--9cbf6fe8def32a6b434c8bfc8991916ff425a0c990be48fffe647c5ab7a6b294ba38e96fa58aebd59eb5b14d0e98475b241cd6098f08f2c10953b999d1bcd01c
+    [2018-07-08 11:37:19.130224384] chat: gpg [\"--batch\",\"--no-tty\",\"--use-agent\",\"--quiet\",\"--trust-model\",\"always\",\"--batch\",\"--decrypt\"]
+    [2018-07-08 11:37:19.248666938] process done ExitSuccess
+
+    (checksum...) ok
+    [2018-07-08 11:37:19.2521772] chat: git [\"--git-dir=.git\",\"--work-tree=.\",\"--literal-pathspecs\",\"hash-object\",\"-w\",\"--stdin-paths\",\"--no-filters\"]
+    [2018-07-08 11:37:19.252634291] feed: git [\"--git-dir=.git\",\"--work-tree=.\",\"--literal-pathspecs\",\"update-index\",\"-z\",\"--index-info\"]
+    [2018-07-08 11:37:19.259545841] process done ExitSuccess
+    [2018-07-08 11:37:19.259672048] read: git [\"--git-dir=.git\",\"--work-tree=.\",\"--literal-pathspecs\",\"show-ref\",\"--hash\",\"refs/heads/git-annex\"]
+    [2018-07-08 11:37:19.262905258] process done ExitSuccess
+    (recording state in git...)
+    [2018-07-08 11:37:19.26348496] read: git [\"--git-dir=.git\",\"--work-tree=.\",\"--literal-pathspecs\",\"write-tree\"]
+    [2018-07-08 11:37:19.280879681] process done ExitSuccess
+    [2018-07-08 11:37:19.281192446] chat: git [\"--git-dir=.git\",\"--work-tree=.\",\"--literal-pathspecs\",\"commit-tree\",\"9fa3141c6ae10188ede17ecd10b71b9f679c32f9\",\"--no-gpg-sign\",\"-p\",\"refs/heads/git-annex\"]
+    [2018-07-08 11:37:19.285386915] process done ExitSuccess
+    [2018-07-08 11:37:19.285622336] call: git [\"--git-dir=.git\",\"--work-tree=.\",\"--literal-pathspecs\",\"update-ref\",\"refs/heads/git-annex\",\"81296ea05b728e9440c9d9927121c9b02aca33d4\"]
+    [2018-07-08 11:37:19.289082082] process done ExitSuccess
+    [2018-07-08 11:37:19.290328762] process done ExitSuccess
+    [2018-07-08 11:37:19.290683887] process done ExitSuccess
+    [2018-07-08 11:37:19.290954281] process done ExitSuccess
+    [2018-07-08 11:37:19.291245651] process done ExitSuccess
+
+As you can see, the hashed key is:
+
+00a/620/GPGHMACSHA512--9cbf6fe8def32a6b434c8bfc8991916ff425a0c990be48fffe647c5ab7a6b294ba38e96fa58aebd59eb5b14d0e98475b241cd6098f08f2c10953b999d1bcd01c
+
+It has been hashed (openssl HMACSHA512) and encrypted with GPG. I'd like to be able to recover the original key (SHA256E-s127597--abc14a6cf4ebb79fdc2eb0d1bf9c304cfce30959661e72e98536faf1bb1b393b.pdf) without git-annex. Then I have backup-ed the map between local keys and original filenames, so I'm able to get my file back without using git-annex.
+
+"""]]
diff --git a/doc/forum/Shared_pubkeys__58___decrypting_files_in_special_remotes_without_git-annex/comment_3_9852ed3def9f7c73924d8be88b8581e5._comment b/doc/forum/Shared_pubkeys__58___decrypting_files_in_special_remotes_without_git-annex/comment_3_9852ed3def9f7c73924d8be88b8581e5._comment
new file mode 100644
index 0000000000..82ceebb07b
--- /dev/null
+++ b/doc/forum/Shared_pubkeys__58___decrypting_files_in_special_remotes_without_git-annex/comment_3_9852ed3def9f7c73924d8be88b8581e5._comment
@@ -0,0 +1,10 @@
+[[!comment format=mdwn
+ username="andrew"
+ avatar="http://cdn.libravatar.org/avatar/acc0ece1eedf07dd9631e7d7d343c435"
+ subject="remote encrypted keys"
+ date="2018-07-16T12:09:22Z"
+ content="""
+Aaah. I see you have commented on the [Future proofing / disaster recovery with an encrypted special remote](http://git-annex.branchable.com/forum/Future_proofing___47___disaster_recovery_with_an_encrypted_special_remote/) as well. I'll try to test with `sharedpubkey` if I get a chance.
+
+Have you tried looking at the generated cipher? `git show git-annex:remote.log | grep 'name='\"your-remote-name \"`? Did you try adding `IFS= read -rd '' cipher < <( printf \"$cipher\n\" )`?
+"""]]
diff --git a/doc/forum/Shared_pubkeys__58___decrypting_files_in_special_remotes_without_git-annex/comment_4_cd56ee6915acb9389876b7a4b7c0e728._comment b/doc/forum/Shared_pubkeys__58___decrypting_files_in_special_remotes_without_git-annex/comment_4_cd56ee6915acb9389876b7a4b7c0e728._comment
new file mode 100644
index 0000000000..a063c28025
--- /dev/null
+++ b/doc/forum/Shared_pubkeys__58___decrypting_files_in_special_remotes_without_git-annex/comment_4_cd56ee6915acb9389876b7a4b7c0e728._comment
@@ -0,0 +1,10 @@
+[[!comment format=mdwn
+ username="oliv5"
+ avatar="http://cdn.libravatar.org/avatar/d7f0d33c51583bbd8578e4f1f9f8cf4b"
+ subject="comment 4"
+ date="2018-07-16T17:04:06Z"
+ content="""
+Yes, I had a look at the cipher, and I cannot tell much about it. It is ... random :) And yes, I tried the removal of the '\n' characters like in pubkey encryption scheme, with no success so far.
+
+I'll post my test script, which is a basic variation of the one given in [Decrypting_files_in_special_remotes_without_git-annex](https://git-annex.branchable.com/tips/Decrypting_files_in_special_remotes_without_git-annex/) to make multiple calls to lookup_key().
+"""]]
diff --git a/doc/forum/Shared_pubkeys__58___decrypting_files_in_special_remotes_without_git-annex/comment_5_feda43ef61bc7676c3fd8c9308605650._comment b/doc/forum/Shared_pubkeys__58___decrypting_files_in_special_remotes_without_git-annex/comment_5_feda43ef61bc7676c3fd8c9308605650._comment
new file mode 100644
index 0000000000..8f8f936b06
--- /dev/null
+++ b/doc/forum/Shared_pubkeys__58___decrypting_files_in_special_remotes_without_git-annex/comment_5_feda43ef61bc7676c3fd8c9308605650._comment
@@ -0,0 +1,16 @@
+[[!comment format=mdwn
+ username="andrew"
+ avatar="http://cdn.libravatar.org/avatar/acc0ece1eedf07dd9631e7d7d343c435"
+ subject="comment 5"
+ date="2018-07-16T22:58:30Z"
+ content="""
+OK. Thanks for all of the patience with the back-and-forth. After trying to follow these steps and re-reading your posts I believe that what you are trying to do is unfortunately not possible with your current setup.
+
+Although you are using public keys to encrypt your content, `git-annex` is hashing the original key names (using a one-way hash) aka SHA1 Digest signed with your public key before storing on the special remote. The public key is just used by the digest algorithm for signing and does not enable you to recover the hashed key. The `git-annex` special remote protocol does not require special remotes to actually store `git-annex` keys, it only requires that special remotes can retrieve content *given* a key.
+
+In Joey's source code `Crypto.hs` in reference to this key generation process he does have a comment to this effect, \"The encryption does not need to be reversable\". I assume Joey used hashes for simplicity and so the filenames could stay short.
+
+In [Future proofing / disaster recovery with an encrypted special remote](http://git-annex.branchable.com/forum/Future_proofing___47___disaster_recovery_with_an_encrypted_special_remote/) Joey also mentions “That's what's \"special\" about special remotes vs regular git remotes: They only store the content of annexed files and not the git repository. Back up the git repository separately”
+
+So, it seems, in order to recover the original key names you will either have to keep a backup of the original repository or create a new special remote that stores these in a recoverable fashion (instead of using a digest). Perhaps some git commit hook that zips up the `.git` directory and adds it to your repository could be of use?
+"""]]
diff --git a/doc/forum/Shared_pubkeys__58___decrypting_files_in_special_remotes_without_git-annex/comment_6_da33ab62284728f832257014c351f151._comment b/doc/forum/Shared_pubkeys__58___decrypting_files_in_special_remotes_without_git-annex/comment_6_da33ab62284728f832257014c351f151._comment
new file mode 100644
index 0000000000..bccc57fb6d
--- /dev/null
+++ b/doc/forum/Shared_pubkeys__58___decrypting_files_in_special_remotes_without_git-annex/comment_6_da33ab62284728f832257014c351f151._comment
@@ -0,0 +1,18 @@
+[[!comment format=mdwn
+ username="oliv5"
+ avatar="http://cdn.libravatar.org/avatar/d7f0d33c51583bbd8578e4f1f9f8cf4b"
+ subject="comment 6"
+ date="2018-07-18T22:35:44Z"
+ content="""
+Hi,
+I understand your points, but I think there is a little misunderstanding.
+
+I agree there may be no way to revert the hashing, even given the public key and its corresponding private key (that I have of course, to be able to decrypt the files content). But this is not what I'm trying to do; actually, I want to do the opposite: from the local key, compute the hashed key. In other words, I want to do exactly what git-annex does already internally, but in a little shell script, independent of git-annex (in case git-annex is unusable one day, a.k.a the \"disaster\").
+
+This is the whole purpose of the function \"lookup_key()\" of the shell script of page [Future proofing / disaster recovery with an encrypted special remote](https://git-annex.branchable.com/forum/Future_proofing___47___disaster_recovery_with_an_encrypted_special_remote/): the functions tries to hash/use the keys the same way git-annex does. It prints the final hashed key used in the special remote. This function allows a user to find which file in the special remote corresponds to a given file in the local repository.
+
+This mapping \"local file name\" / \"local key\" / \"special remote hashed key\" is what I want to backup. In case of trouble to run git-annex one day, this mapping would allow me to rename my special remote files to their original filename, after downloading them using third party-tools (sftp, scp, rsync, whatever) and decrypting their content using my private key. All this without git-annex and its special remote third-party script.
+
+Do you see what I'm trying to do ? I rely much on this lookup_key() function, which is basically already implemented inside git-annex. My main pb is I don't understand Haskell, I can only lurk around in the code, but I didn't identify the sequence of operations about these hashed keys.
+
+"""]]
diff --git a/doc/forum/Shared_pubkeys__58___decrypting_files_in_special_remotes_without_git-annex/comment_7_eb448e997e4dc77c845bc4a9462ee128._comment b/doc/forum/Shared_pubkeys__58___decrypting_files_in_special_remotes_without_git-annex/comment_7_eb448e997e4dc77c845bc4a9462ee128._comment
new file mode 100644
index 0000000000..4bd78cfe77
--- /dev/null
+++ b/doc/forum/Shared_pubkeys__58___decrypting_files_in_special_remotes_without_git-annex/comment_7_eb448e997e4dc77c845bc4a9462ee128._comment
@@ -0,0 +1,19 @@
+[[!comment format=mdwn
+ username="joey"
+ subject="""comment 7"""
+ date="2018-07-19T16:57:22Z"
+ content="""
+I have addressed the sharedpubkey thing in the other thread.
+
+Chunk keys may have a -S as well as the -C, if the special remote was
+set up with new-style chunking enabled.
+
+A remote can have several different chunk sizes over its
+lifetime; the chunk size used for a given key is in the .log.cnk file
+in the git-annex branch, documented in [[internals]].
+
+The easy way to test if you are generating
+the right key, prior to HMAC encrypting it, is to set up a non-encrypted
+special remote with the same chunking configuration, and look at the chunk
+keys used when files are stored in it.
+"""]]
diff --git a/doc/forum/Shared_pubkeys__58___decrypting_files_in_special_remotes_without_git-annex/comment_8_7001fb7530f2ee1591b400195f702cd9._comment b/doc/forum/Shared_pubkeys__58___decrypting_files_in_special_remotes_without_git-annex/comment_8_7001fb7530f2ee1591b400195f702cd9._comment
new file mode 100644
index 0000000000..4793a181db
--- /dev/null
+++ b/doc/forum/Shared_pubkeys__58___decrypting_files_in_special_remotes_without_git-annex/comment_8_7001fb7530f2ee1591b400195f702cd9._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="andrew"
+ avatar="http://cdn.libravatar.org/avatar/acc0ece1eedf07dd9631e7d7d343c435"
+ subject="comment 8"
+ date="2018-07-19T18:18:39Z"
+ content="""
+This does not seem to resolve the issue. I've filed a bug report with an easy to reproduce case using just local directory remotes. `sharedpubkey` seems to be using a different method than `shared` to created encrypted filenames.
+"""]]
diff --git a/doc/forum/Shared_pubkeys__58___decrypting_files_in_special_remotes_without_git-annex/comment_9_a27a661e8b589cffb4acad3591586433._comment b/doc/forum/Shared_pubkeys__58___decrypting_files_in_special_remotes_without_git-annex/comment_9_a27a661e8b589cffb4acad3591586433._comment
new file mode 100644
index 0000000000..0143d601f3
--- /dev/null
+++ b/doc/forum/Shared_pubkeys__58___decrypting_files_in_special_remotes_without_git-annex/comment_9_a27a661e8b589cffb4acad3591586433._comment
@@ -0,0 +1,12 @@
+[[!comment format=mdwn
+ username="joey"
+ subject="""comment 9"""
+ date="2018-07-19T20:30:18Z"
+ content="""
+I tested the script fragement that I posted.
+
+The obvious difference between shared and sharedpubkey, if you look
+at that script fragement, is that shared only uses the first 256 bytes
+of the cipher to encrypt filenames, while sharedpubkey uses the entire
+cipher. 
+"""]]
diff --git a/doc/forum/Shared_repository_without_git-annex_on_central_server.mdwn b/doc/forum/Shared_repository_without_git-annex_on_central_server.mdwn
new file mode 100644
index 0000000000..424174a28e
--- /dev/null
+++ b/doc/forum/Shared_repository_without_git-annex_on_central_server.mdwn
@@ -0,0 +1,10 @@
+Hi,
+
+I went through the documentation but could not find a clear statement about central repositories (or I did not understand it).
+
+I plan to use a central server to store and sync documents from my local PC and my laptop. I do not trust this server so I want to use gcrypt to only save encrypted information remote. The server provides git and rsync access but does not have git-annex installed. My main source of information is: [fully encrypted git repositories with gcrypt](https://git-annex.branchable.com/tips/fully_encrypted_git_repositories_with_gcrypt/). On that page is a hint that you can use a server without git-annex but I am not sure if I really understand the documentation as the [centralized git repository tutorial](https://git-annex.branchable.com/tips/centralized_git_repository_tutorial/on_your_own_server/) states you need git-annex on the centralized server.
+
+Is there an option to use a centralized git repository without git-annex installed on the central server?
+
+Thank you very much and kind regards,
+Frank
diff --git a/doc/forum/Shared_repository_without_git-annex_on_central_server/comment_1_286c963b30eee46b261d64937cf66d06._comment b/doc/forum/Shared_repository_without_git-annex_on_central_server/comment_1_286c963b30eee46b261d64937cf66d06._comment
new file mode 100644
index 0000000000..97026460a7
--- /dev/null
+++ b/doc/forum/Shared_repository_without_git-annex_on_central_server/comment_1_286c963b30eee46b261d64937cf66d06._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="xloem"
+ avatar="http://cdn.libravatar.org/avatar/b8c087f7c5e6a9358748f0727c077f3b"
+ subject="comment 1"
+ date="2017-01-16T12:16:37Z"
+ content="""
+You can set the server up as a normal git remote to store file changelogs, and then also as an rsync special remote to store file data.  See https://git-annex.branchable.com/special_remotes/rsync/
+"""]]
diff --git a/doc/forum/Shared_repository_without_git-annex_on_central_server/comment_2_d1f803b68c3789854e483da1318d043d._comment b/doc/forum/Shared_repository_without_git-annex_on_central_server/comment_2_d1f803b68c3789854e483da1318d043d._comment
new file mode 100644
index 0000000000..b8200fc482
--- /dev/null
+++ b/doc/forum/Shared_repository_without_git-annex_on_central_server/comment_2_d1f803b68c3789854e483da1318d043d._comment
@@ -0,0 +1,17 @@
+[[!comment format=mdwn
+ username="joey"
+ subject="""comment 2"""
+ date="2017-06-09T17:47:45Z"
+ content="""
+The gcrypt special remote will work without git-annex being installed
+on the server. The server only needs git and rsync. I think the
+tip is fairly clear about that when it says
+
+"While this will work without git-annex being installed on the server, it
+is recommended to have it installed."
+
+Without git-annex being available on the remote server, some things
+like `git annex notifychanges` and content locking during dropping won't
+work with that remote. The basics should work well enough to use it that
+way.
+"""]]
diff --git a/doc/forum/Sharing_annex_with_local_clones.mdwn b/doc/forum/Sharing_annex_with_local_clones.mdwn
new file mode 100644
index 0000000000..756075ef90
--- /dev/null
+++ b/doc/forum/Sharing_annex_with_local_clones.mdwn
@@ -0,0 +1 @@
+Hi, is there any particular problem with symlinking one .git/annex to share between multiple repos?
diff --git a/doc/forum/Sharing_annex_with_local_clones/comment_1_2b60e13e5f7b8cee56cf2ddc6c47f64d._comment b/doc/forum/Sharing_annex_with_local_clones/comment_1_2b60e13e5f7b8cee56cf2ddc6c47f64d._comment
new file mode 100644
index 0000000000..892bc50f52
--- /dev/null
+++ b/doc/forum/Sharing_annex_with_local_clones/comment_1_2b60e13e5f7b8cee56cf2ddc6c47f64d._comment
@@ -0,0 +1,12 @@
+[[!comment format=mdwn
+ username="http://joey.kitenet.net/"
+ nickname="joey"
+ subject="don't do that"
+ date="2012-03-19T18:23:13Z"
+ content="""
+Suppose you do that to repos A and B. Now, in A, you `git annex drop` a file that is only present in those repositories. A checks B to make sure it still has a copy of the file. It sees the (same) file there, so assumes it's safe to drop. The file is removed from A, also removing it from B, and losing data.
+
+It is possible to configure A and B to mutually distrust one-another and avoid this problem, but there will be other problems too.
+
+Instead, git-annex supports using `cp --reflink=auto`, which on filesystems supporting Copy On Write (eg, btrfs), avoids duplicating contents when A and B are on the same filesystem.
+"""]]
diff --git a/doc/forum/Sharing_annex_with_local_clones/comment_2_24ff2c1eb643077daa37c01644cebcd2._comment b/doc/forum/Sharing_annex_with_local_clones/comment_2_24ff2c1eb643077daa37c01644cebcd2._comment
new file mode 100644
index 0000000000..88f495da63
--- /dev/null
+++ b/doc/forum/Sharing_annex_with_local_clones/comment_2_24ff2c1eb643077daa37c01644cebcd2._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="https://www.google.com/accounts/o8/id?id=AItOawnhsaESlYphzLTzpJy5IxxGFxxctIhWYfo"
+ nickname="Bryon"
+ subject="comment 2"
+ date="2012-03-19T18:46:13Z"
+ content="""
+Ah, OK. Is there a configuration step to set this up, or is this included magic in a new enough git-annex client?
+"""]]
diff --git a/doc/forum/Sharing_annex_with_local_clones/comment_3_5359b8eada24d27be83214ac0ae62f23._comment b/doc/forum/Sharing_annex_with_local_clones/comment_3_5359b8eada24d27be83214ac0ae62f23._comment
new file mode 100644
index 0000000000..3e5fd11545
--- /dev/null
+++ b/doc/forum/Sharing_annex_with_local_clones/comment_3_5359b8eada24d27be83214ac0ae62f23._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="https://www.google.com/accounts/o8/id?id=AItOawnhsaESlYphzLTzpJy5IxxGFxxctIhWYfo"
+ nickname="Bryon"
+ subject="comment 3"
+ date="2012-03-19T18:55:03Z"
+ content="""
+Nevermind, found it. (git-annex 0.08)
+"""]]
diff --git a/doc/forum/Sharing_annex_with_local_clones/comment_4_5c870c49d8093e1a2895224cc6e91ca0._comment b/doc/forum/Sharing_annex_with_local_clones/comment_4_5c870c49d8093e1a2895224cc6e91ca0._comment
new file mode 100644
index 0000000000..5bdaa17003
--- /dev/null
+++ b/doc/forum/Sharing_annex_with_local_clones/comment_4_5c870c49d8093e1a2895224cc6e91ca0._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="https://www.google.com/accounts/o8/id?id=AItOawkr_KjpfjbrD4ln15NsBBn7c9j90U6uwqY"
+ nickname="Nicolas"
+ subject="comment 4"
+ date="2014-11-04T23:05:46Z"
+ content="""
+On file system not supporting reflink is there another solution for sharing annex with local clones ?
+"""]]
diff --git a/doc/forum/Sharing_annex_with_local_clones/comment_5_0bf4c0aa3258b156a3df794d5fabb6a2._comment b/doc/forum/Sharing_annex_with_local_clones/comment_5_0bf4c0aa3258b156a3df794d5fabb6a2._comment
new file mode 100644
index 0000000000..5fb2eaec60
--- /dev/null
+++ b/doc/forum/Sharing_annex_with_local_clones/comment_5_0bf4c0aa3258b156a3df794d5fabb6a2._comment
@@ -0,0 +1,23 @@
+[[!comment format=mdwn
+ username="joey"
+ subject="""comment 5"""
+ date="2014-11-06T19:50:54Z"
+ content="""
+Yes, there's another option since git-annex version 5.20140915.
+
+annex.hardlink can be set to true, and then `git annex get` will
+simply hardlink the files into place. 
+
+Note the caution about using this option:
+
+              Use with caution --  This  can  invalidate  numcopies  counting,
+              since  with hard links, fewer copies of a file can exist. So, it
+              is a good idea to  mark  a  repository  using  this  setting  as
+              untrusted.
+
+And, there's a nice easy way to set up local clone that is configured this way:
+
+              When  a repository is set up using git clone --shared, git-annex
+              init will automatically set annex.hardlink and mark the  reposi‐
+              tory as untrusted.
+"""]]
diff --git a/doc/forum/Simple_check_out_with_assistant__63__.mdwn b/doc/forum/Simple_check_out_with_assistant__63__.mdwn
new file mode 100644
index 0000000000..755b237e49
--- /dev/null
+++ b/doc/forum/Simple_check_out_with_assistant__63__.mdwn
@@ -0,0 +1,2 @@
+I want to use the annex assistant instead of the command line. Which is the recommended method to check out a file that I want to edit?
+
diff --git a/doc/forum/Simple_check_out_with_assistant__63__/comment_1_ade8a0743ef1ec933c8a40ed64eeac2d._comment b/doc/forum/Simple_check_out_with_assistant__63__/comment_1_ade8a0743ef1ec933c8a40ed64eeac2d._comment
new file mode 100644
index 0000000000..dbb018b53f
--- /dev/null
+++ b/doc/forum/Simple_check_out_with_assistant__63__/comment_1_ade8a0743ef1ec933c8a40ed64eeac2d._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="https://www.google.com/accounts/o8/id?id=AItOawmaXxj2aPZrtRkWzwswTNep8hFif1cXykw"
+ nickname="Erik"
+ subject="Follow-up: check-in"
+ date="2012-11-08T08:59:14Z"
+ content="""
+And the obvious follow-up question: How to I check in and sync the file that I just edited? (Note: I want to avoid the command line)
+"""]]
diff --git a/doc/forum/Simple_question_about_web_remote.mdwn b/doc/forum/Simple_question_about_web_remote.mdwn
new file mode 100644
index 0000000000..1805405076
--- /dev/null
+++ b/doc/forum/Simple_question_about_web_remote.mdwn
@@ -0,0 +1,27 @@
+Hello. I have been having some trouble with downloading podcasts. I have read the guide, but I was getting the following error message:
+
+```
+  verification of content failed
+
+  Unable to access these remotes: web
+
+  Try making some of these repositories available:
+  	00000000-0000-0000-0000-000000000001 -- web
+failed
+git-annex: get: 1 failed
+```
+I have tried several times. Some times it seemed to work, others it did not — it was not very structured, so I do not recall the details. Recently I recreated the entire stuff from scratch and was downloading files, and after I upgraded to 6.20160318 it stopped working and just gives me the error message above, after downloading the file.
+
+I tried looking up information about the web remote, but it mentions nothing about "making the web remote available" or something that I found to address this subject. So I am confused. What is wrong with my web repository? Why did it stop working? How can I fix it? How can I prevent this from happening in the future? Thank you.
+
+Here is the version information:
+
+```
+git-annex version: 6.20160318
+build flags: Assistant Webapp Pairing Testsuite S3(multipartupload)(storageclasses) WebDAV Inotify DBus DesktopNotify XMPP ConcurrentOutput TorrentParser MagicMime Feeds Quvi
+key/value backends: SHA256E SHA256 SHA512E SHA512 SHA224E SHA224 SHA384E SHA384 SHA3_256E SHA3_256 SHA3_512E SHA3_512 SHA3_224E SHA3_224 SHA3_384E SHA3_384 SKEIN256E SKEIN256 SKEIN512E SKEIN512 SHA1E SHA1 MD5E MD5 WORM URL
+remote types: git gcrypt S3 bup directory rsync web bittorrent webdav tahoe glacier ddar hook external
+local repository version: 6
+supported repository versions: 5 6
+upgrade supported from repository versions: 0 1 2 4 5
+```
diff --git a/doc/forum/Simple_question_about_web_remote/comment_1_a732a7264b60faed29782a034c0bc592._comment b/doc/forum/Simple_question_about_web_remote/comment_1_a732a7264b60faed29782a034c0bc592._comment
new file mode 100644
index 0000000000..4b39005f2f
--- /dev/null
+++ b/doc/forum/Simple_question_about_web_remote/comment_1_a732a7264b60faed29782a034c0bc592._comment
@@ -0,0 +1,24 @@
+[[!comment format=mdwn
+ username="joey"
+ subject="""comment 1"""
+ date="2016-04-12T17:33:08Z"
+ content="""
+The web remote is automatically available (so long as you have an internet
+connection) and does not need any configuration to use.
+
+What's going on here is, after git-annex downloads a file from the web
+remote, as from any remote, it verifies its checksum. Seems that the
+checksum of the content at an url on the web has changed. So, git-annex
+rejects the bad data.
+
+(Note that git-annex only started verifying checksums of downloads in
+version 5.20151019.)
+
+Now, sometimes you don't care if eg, a podcast file might get modified
+after `git-annex importfeed` initially adds it, and you want to just
+download whatever file is on the web now, and treat any contents coming
+from the web at different times as equivilant.
+
+The way to do that is to use --relaxed when running git-annex addurl or
+importfeed.
+"""]]
diff --git a/doc/forum/Simple_question_about_web_remote/comment_2_274810bdc176984c056304e17668c63b._comment b/doc/forum/Simple_question_about_web_remote/comment_2_274810bdc176984c056304e17668c63b._comment
new file mode 100644
index 0000000000..2e6db099d1
--- /dev/null
+++ b/doc/forum/Simple_question_about_web_remote/comment_2_274810bdc176984c056304e17668c63b._comment
@@ -0,0 +1,16 @@
+[[!comment format=mdwn
+ username="Gus"
+ subject="Branching questions"
+ date="2016-04-29T12:11:35Z"
+ content="""
+Thank you for your help. I would still like to ask a few related questions.
+
+1. I was unaware that RSS feeds provide a form of checksum. Are they stored as the filename? I still find it unlikely that the old file was changed in the few minutes between git-annex importing the feed and me asking for the file. However, maybe the file was changed, the feed was not updated and podcast downloading software are not as strict as git-annex.
+
+2. How should I proceed in order to update my podcasts. This is something I do not understand and I should play a bit with git-annex to figure out: there are multiple ways to remove files. As I understand it:
+  - I could `rm` the file. This removes the link. git still remembers the file and keeps it.
+  - I could `git rm` the file and commit. This could keep the file or remove it. git remembers the file's history and keeps its last state. However, in git-annex I am not sure what is kept in .git/. Can I re-add the file?
+  - I could `git-annex drop` the file. git-annex ensures the minimum number of copies, so this makes me believe that git totally forgets about the file (no last state copy). However, in the podcast tips you say that dropped files are not redownloaded. So this leaves me confused.
+
+I tried to `git rm -r` the podcast directory, and now I cannot importfeed it again. :-S Hmm... It is nothing important, but could you explain me what is going on, please?
+"""]]
diff --git a/doc/forum/Simple_question_about_web_remote/comment_3_984a3b05bcc6b24430210d9ae039a026._comment b/doc/forum/Simple_question_about_web_remote/comment_3_984a3b05bcc6b24430210d9ae039a026._comment
new file mode 100644
index 0000000000..93ac6c4026
--- /dev/null
+++ b/doc/forum/Simple_question_about_web_remote/comment_3_984a3b05bcc6b24430210d9ae039a026._comment
@@ -0,0 +1,30 @@
+[[!comment format=mdwn
+ username="joey"
+ subject="""comment 3"""
+ date="2016-05-03T18:14:51Z"
+ content="""
+The checksum is not coming from the RSS feed. If you ran `git-annex
+importfeed` without `--fast`, it downloaded the content and checksummed it
+locally, and this checksum becomes the one it expects to have when
+downloading the file again. If you ran `git annex importfeed` with `--fast`, 
+it doesn't get a checksum, but it does record the current size of the file
+as reported by the web server. 
+
+Either the size or the checksum changing could be what caused the
+verification problem. It's possible that the web server reports a bogus
+size somehow (I've seen this happen before), or the web server might
+be some kind of CDN that is serving up different file contents at
+different times for whatever reason.
+
+If you use `git rm` or commit a `rm`, then any content of the file is
+retained in the git-annex repository, at least until you use `git annex
+unused` to remove it. You can check out old versions of the branch and the
+file content will still be there.
+
+If you use `git annex drop`, it drops the content from the local
+repository. git-annex still knows about the file, and `git annex get` can
+get the content again, perhaps by downloading it from the web again.
+
+Since git-annex still knows about the file even if it's removed or dropped,
+`git annex importfeed` avoids re-importing such files from a rss feed.
+"""]]
diff --git a/doc/forum/Simple_question_about_web_remote/comment_4_510abbb4deca924020b87f1eb7d61005._comment b/doc/forum/Simple_question_about_web_remote/comment_4_510abbb4deca924020b87f1eb7d61005._comment
new file mode 100644
index 0000000000..a44d75dd0f
--- /dev/null
+++ b/doc/forum/Simple_question_about_web_remote/comment_4_510abbb4deca924020b87f1eb7d61005._comment
@@ -0,0 +1,15 @@
+[[!comment format=mdwn
+ username="Gus"
+ subject="Restarting"
+ date="2016-05-17T10:36:55Z"
+ content="""
+My attempt to redo the podcast repository adding the `--fast` flag seems to be successful. Thank you.
+
+However, I would still like to ask you one thing about the delete operations.
+When would you use `git rm` instead of `git-annex drop` (in a repository that uses only git-annex to store the files)?
+
+Oh, and I'll also sneak in a small question that is unrelated to the above, since I don't think it deserves its own topic:
+Is git-annex safe to operate in parallel tasks? Can I be adding file A and move file B and rename file C from the same repository at the same time, without the integrity of the repository being compromised?
+
+
+"""]]
diff --git a/doc/forum/Slightly_finer_control_over_file_whereabouts.mdwn b/doc/forum/Slightly_finer_control_over_file_whereabouts.mdwn
new file mode 100644
index 0000000000..2caf332af4
--- /dev/null
+++ b/doc/forum/Slightly_finer_control_over_file_whereabouts.mdwn
@@ -0,0 +1,18 @@
+I have previously been confused by how to have a bit more manual control over where files are (as in [[bugs/git-annex_immediately_re-gets_dropped_files/]]).
+
+I thought perhaps something like an archive directory would work, but as I might want different computers to have different content, perhaps a per-computer archive repository.
+
+To that end, I have
+
+
    walter@dionysus:~/annex$ git annex content .
+(exclude=archive/dionysus/* or (not (copies=archive:1 or copies=smallarchive:1))) or (not copies=semitrusted+:1)
+ok
    
+
+which is a modified version of the archive [[preferred_content]], but with the intention that I change the part after `exclude` to be different for each client.
+I also set the group to client for this client (dionysus).
+
+However, it does not seem to drop files when I move them into `archive/dionysus/`, and I cannot see anything in the logs to suggest why.
+
+What am I doing wrong here? Or, is this the wrong approach?
+
+--Walter
diff --git a/doc/forum/Slightly_finer_control_over_file_whereabouts/comment_10_bcb883d46a637dd1a8ef9a92733d202a._comment b/doc/forum/Slightly_finer_control_over_file_whereabouts/comment_10_bcb883d46a637dd1a8ef9a92733d202a._comment
new file mode 100644
index 0000000000..5064de49dc
--- /dev/null
+++ b/doc/forum/Slightly_finer_control_over_file_whereabouts/comment_10_bcb883d46a637dd1a8ef9a92733d202a._comment
@@ -0,0 +1,10 @@
+[[!comment format=mdwn
+ username="http://joeyh.name/"
+ ip="108.236.230.124"
+ subject="comment 10"
+ date="2013-11-08T18:32:12Z"
+ content="""
+If I could comprehensively fix that bug I would.
+
+But, it's supposed to already be fixed when using direct mode. Were you using indirect mode when you saw the problem?
+"""]]
diff --git a/doc/forum/Slightly_finer_control_over_file_whereabouts/comment_11_b7a8b9eaf114f883866fbf2be51b622f._comment b/doc/forum/Slightly_finer_control_over_file_whereabouts/comment_11_b7a8b9eaf114f883866fbf2be51b622f._comment
new file mode 100644
index 0000000000..7155b560f4
--- /dev/null
+++ b/doc/forum/Slightly_finer_control_over_file_whereabouts/comment_11_b7a8b9eaf114f883866fbf2be51b622f._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="https://www.google.com/accounts/o8/id?id=AItOawnSenxKyE_2Z6Wb-EBMO8FciyRywjx1ZiQ"
+ nickname="Walter"
+ subject="comment 11"
+ date="2013-11-09T06:02:36Z"
+ content="""
+No, this was using direct mode.
+"""]]
diff --git a/doc/forum/Slightly_finer_control_over_file_whereabouts/comment_1_6236bcfa9beba705ead3ec2141c5d835._comment b/doc/forum/Slightly_finer_control_over_file_whereabouts/comment_1_6236bcfa9beba705ead3ec2141c5d835._comment
new file mode 100644
index 0000000000..b9f598a1f0
--- /dev/null
+++ b/doc/forum/Slightly_finer_control_over_file_whereabouts/comment_1_6236bcfa9beba705ead3ec2141c5d835._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="http://joeyh.name/"
+ ip="4.153.14.105"
+ subject="comment 1"
+ date="2013-09-19T17:57:22Z"
+ content="""
+That sound like it should work. I suggest you play around with `git annex drop --auto` at the command line. It will probably tell you why it is unable to drop a file if the problem is something like not enough copies located elsewhere. Or, if it doesn't try to drop the file at all, you'll know your preferred content expression makes it want to keep the file.
+"""]]
diff --git a/doc/forum/Slightly_finer_control_over_file_whereabouts/comment_2_ea935b37ca93e73c85d04df7c9bf6057._comment b/doc/forum/Slightly_finer_control_over_file_whereabouts/comment_2_ea935b37ca93e73c85d04df7c9bf6057._comment
new file mode 100644
index 0000000000..50f113aa33
--- /dev/null
+++ b/doc/forum/Slightly_finer_control_over_file_whereabouts/comment_2_ea935b37ca93e73c85d04df7c9bf6057._comment
@@ -0,0 +1,12 @@
+[[!comment format=mdwn
+ username="https://www.google.com/accounts/o8/id?id=AItOawnSenxKyE_2Z6Wb-EBMO8FciyRywjx1ZiQ"
+ nickname="Walter"
+ subject="comment 2"
+ date="2013-09-19T22:22:03Z"
+ content="""
+So, it sort of works...
+
+I realised that I don't have `archive` repository but instead `backup` ones, so adding that makes `git annex drop --auto` want to drop files in archive/dionysus.
+
+However, the webapp then gets them. This I don't understand.
+"""]]
diff --git a/doc/forum/Slightly_finer_control_over_file_whereabouts/comment_3_f89a8e38283ac4c8c4a3b74c413d67a1._comment b/doc/forum/Slightly_finer_control_over_file_whereabouts/comment_3_f89a8e38283ac4c8c4a3b74c413d67a1._comment
new file mode 100644
index 0000000000..41bb0cb58e
--- /dev/null
+++ b/doc/forum/Slightly_finer_control_over_file_whereabouts/comment_3_f89a8e38283ac4c8c4a3b74c413d67a1._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="http://joeyh.name/"
+ ip="4.153.14.105"
+ subject="comment 3"
+ date="2013-09-20T00:15:07Z"
+ content="""
+does `git annex get --auto` also get them?
+"""]]
diff --git a/doc/forum/Slightly_finer_control_over_file_whereabouts/comment_4_07a0a754a089c46ff69dc97ea7ba9384._comment b/doc/forum/Slightly_finer_control_over_file_whereabouts/comment_4_07a0a754a089c46ff69dc97ea7ba9384._comment
new file mode 100644
index 0000000000..30e3965af0
--- /dev/null
+++ b/doc/forum/Slightly_finer_control_over_file_whereabouts/comment_4_07a0a754a089c46ff69dc97ea7ba9384._comment
@@ -0,0 +1,22 @@
+[[!comment format=mdwn
+ username="https://www.google.com/accounts/o8/id?id=AItOawnSenxKyE_2Z6Wb-EBMO8FciyRywjx1ZiQ"
+ nickname="Walter"
+ subject="comment 4"
+ date="2013-09-20T02:41:22Z"
+ content="""
+I am testing on my laptop, and the content expression there is
+`(exclude=archive/kronos/* ) or (not copies=semitrusted+:1)`
+to simplify things; I interpret this to mean it wants everything not in archive/kronos, and it wants things there where that's the only copy.
+
+The problem is that `git annex drop --auto` does indeed drop things, but `git annex get --auto` gets them; surely it should not be possible for that to happen?
+
+I also tried the preferred content expression of `(exclude=archive/kronos/* )`, and the same thing happens.
+
+The version of git-annex that I am using is
+
    git-annex version: 4.20130919-g9be7762
+build flags: Assistant Webapp Pairing Testsuite S3 WebDAV Inotify DBus XMPP DNS Feeds Quvi
+local repository version: 4
+default repository version: 3
+supported repository versions: 3 4
+upgrade supported from repository versions: 0 1 2
    
+"""]]
diff --git a/doc/forum/Slightly_finer_control_over_file_whereabouts/comment_5_e884c001a556a0c693d1cc9a97c068ac._comment b/doc/forum/Slightly_finer_control_over_file_whereabouts/comment_5_e884c001a556a0c693d1cc9a97c068ac._comment
new file mode 100644
index 0000000000..50e5cf9afe
--- /dev/null
+++ b/doc/forum/Slightly_finer_control_over_file_whereabouts/comment_5_e884c001a556a0c693d1cc9a97c068ac._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="http://joeyh.name/"
+ ip="2001:4830:1600:187::2"
+ subject="comment 5"
+ date="2013-09-20T15:31:18Z"
+ content="""
+Ok. This seems to be a bug in git-annex then. While it's surprisingly difficult to do so, it tries to interpet preferred content expressions in a stable way -- that is, they should not want to get a file when it's missing and then want to drop it when it's present.
+"""]]
diff --git a/doc/forum/Slightly_finer_control_over_file_whereabouts/comment_6_3e8674b5857e4994dfbc26be4f4b2855._comment b/doc/forum/Slightly_finer_control_over_file_whereabouts/comment_6_3e8674b5857e4994dfbc26be4f4b2855._comment
new file mode 100644
index 0000000000..eff5d9339e
--- /dev/null
+++ b/doc/forum/Slightly_finer_control_over_file_whereabouts/comment_6_3e8674b5857e4994dfbc26be4f4b2855._comment
@@ -0,0 +1,25 @@
+[[!comment format=mdwn
+ username="http://joeyh.name/"
+ ip="2001:4830:1600:187::2"
+ subject="comment 6"
+ date="2013-09-20T15:36:25Z"
+ content="""
+I tried to reproduce this behavior, but failed. My configuration was 2 repos, A and B, with B the origin of A. A was configured with \"exclude=archive/kronos/*\"
+(also tried \"(exclude=archive/kronos/* ) or (not copies=semitrusted+:1)\")
+
+
    +joey@darkstar:~/tmp/A>git annex get --auto
+joey@darkstar:~/tmp/A>git annex get
+get archive/kronos/foo (from origin...) ok
+(Recording state in git...)
+joey@darkstar:~/tmp/A>git annex drop --auto
+drop archive/kronos/foo ok
+(Recording state in git...)
+joey@darkstar:~/tmp/A>git annex get --auto
+joey@darkstar:~/tmp/A>
+
    
+
+... Which is how you want it to behave.
+
+So I wonder if it's something else in your configuration. The best thing to do would be if you can come up with a series of commands I can follow to build repositories that exhibit the problem.
+"""]]
diff --git a/doc/forum/Slightly_finer_control_over_file_whereabouts/comment_7_7aeabc2e52a39423e83fbd04560e8f91._comment b/doc/forum/Slightly_finer_control_over_file_whereabouts/comment_7_7aeabc2e52a39423e83fbd04560e8f91._comment
new file mode 100644
index 0000000000..145a913e90
--- /dev/null
+++ b/doc/forum/Slightly_finer_control_over_file_whereabouts/comment_7_7aeabc2e52a39423e83fbd04560e8f91._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="http://joeyh.name/"
+ ip="2001:4830:1600:187::2"
+ subject="comment 7"
+ date="2013-09-20T15:37:48Z"
+ content="""
+Also, please check if you're running into [[bugs/Handling_of_files_inside_and_outside_archive_directory_at_the_same_time]]
+"""]]
diff --git a/doc/forum/Slightly_finer_control_over_file_whereabouts/comment_8_53b95449cfad2fe0f72d2ad642822c03._comment b/doc/forum/Slightly_finer_control_over_file_whereabouts/comment_8_53b95449cfad2fe0f72d2ad642822c03._comment
new file mode 100644
index 0000000000..d866380f20
--- /dev/null
+++ b/doc/forum/Slightly_finer_control_over_file_whereabouts/comment_8_53b95449cfad2fe0f72d2ad642822c03._comment
@@ -0,0 +1,15 @@
+[[!comment format=mdwn
+ username="https://www.google.com/accounts/o8/id?id=AItOawnSenxKyE_2Z6Wb-EBMO8FciyRywjx1ZiQ"
+ nickname="Walter"
+ subject="comment 8"
+ date="2013-09-20T20:30:52Z"
+ content="""
+Hmm, that could be it; the same content is both inside and outside the archive directory.
+
+What I really want to do (because maybe I'm not going about this the best way) is to use the assistant not in manual mode, but allow for some repos not having some files.
+I thought by placing them in an archive directory, then they would be dropped, and to get them again, I can just delete from the archive dir, and they will be got (as the file is also outside the archive dir.
+
+But, if that is not going to work, is there a better way to manage that?
+
+By the way, I really appreciate all the work you put into this awesome project.
+"""]]
diff --git a/doc/forum/Slightly_finer_control_over_file_whereabouts/comment_9_a17c102a45e4fc3f101a79acb8eb4081._comment b/doc/forum/Slightly_finer_control_over_file_whereabouts/comment_9_a17c102a45e4fc3f101a79acb8eb4081._comment
new file mode 100644
index 0000000000..56d22c7fdf
--- /dev/null
+++ b/doc/forum/Slightly_finer_control_over_file_whereabouts/comment_9_a17c102a45e4fc3f101a79acb8eb4081._comment
@@ -0,0 +1,18 @@
+[[!comment format=mdwn
+ username="https://www.google.com/accounts/o8/id?id=AItOawnSenxKyE_2Z6Wb-EBMO8FciyRywjx1ZiQ"
+ nickname="Walter"
+ subject="comment 9"
+ date="2013-09-23T21:58:42Z"
+ content="""
+Thinking about this further, I'm not entirely sure what I want. I think the confusion arises from git-annex sometimes (mostly) caring about file *contents* (ie dependent on the hash of the file), but sometimes (preferred content, not sure of anywhere else) caring about file *location*.
+
+What I think I actually want is a way of specifying locations that are not synced, such that if the file is changed somewhere (on another computer), the new version should not be downloaded. But, if the same content is in another location as well, the behaviour should be stable, I don't know how that should work though.
+
+So, perhaps the best strategy is to have an explicit list of locations that I don't want, in the preferred content expression, if it could cope with files being in and out of some location at the same time. I think this would be easiest if I could avoid manually editing the expression all the time, maybe make a file with a list of file locations, if it would be possible for git-annex to handle that? I think it isn't at the moment, and my haskell is non-existent. That way, I could write some helper to add and remove files from this list. For example `dont-want file1 file2 dir1` would add these locations to a file, and `want file1 file2 dir1` would remove them from this list. Actually, I suppose I could make it just create an appropriate preferred-content expression, and then it doesn't need to support some file of locations.
+
+So, after that ramble, I guess I'm envisaging a preferred content expression like `content=(exclude=path/to/file1 and exclude=path/to/file1 and exclude=path/to/dir1/*) or (some statement about numcopies)`, which I imagine updating whenever I decide I do/don't want some file. The only obstacle to this working is [[bugs/Handling of files inside and outside archive directory at the same time]] (as I understand that bug, could be wrong on the implications of it), meaning (of course) if there are two files with the same content, and I exclude one of them, and not the other, then it both wants and doesn't want the file, and it (and I) get really confused.
+
+I suppose a short-term (well, slow) solution is to find duplicates of files I don't want, and if that exists either add the duplicate to my content expression (to say I don't want it), or remove the one I don't want from the expression (to say I do). This doesn't work well for when the content of one of the files changes (and so they are no longer duplicates), but I think I would search for them each time I generate the expression, so at that time it would no longer find the duplicate.
+
+So, @joey, I guess my question is, what are the chances of that bug being resolved somehow? Or if that is not likely to happen soon, I might try to implement my solution outline from the previous two paragraphs.
+"""]]
diff --git a/doc/forum/Slow_direct_mode_repository.mdwn b/doc/forum/Slow_direct_mode_repository.mdwn
new file mode 100644
index 0000000000..7a175c2c1b
--- /dev/null
+++ b/doc/forum/Slow_direct_mode_repository.mdwn
@@ -0,0 +1,48 @@
+Hello.
+
+I have been using a portable USB disc for storing media. After several weeks of inactivity, when I used it with git-annex again I noticed it was running very slowly. I mean, it is an USB 2.0 connection, `git-annex sync` would take a few minutes, but now it takes *many hours*.
+
+When adding files to it (`git-annex add`), I would see the list of files passing by at the speed I would expect, but then, at the end, I would have to wait; the next day I saw the operation had completed. `git-annex status` makes me wait a similar amount of time, and `git-annex sync` also makes me wait, even when it has nothing to do — I saw `top` reporting 100% CPU usage, `iotop` saying `git` and `fuse` were saturating the USB connection to its maximum.
+
+File operations outside of `git-annex` seem to work fine, so I can't blame the hardware or the filesystem (NTFS mounted with `fuse`).
+
+I would like to know your thought about this problem, that I was not noticing before. Here is the regular information to help you form an oppinion:
+
+    % time git annex status --debug
+    [2016-09-07 20:20:46.093990203] read: git ["--git-dir=.git","--work-tree=.","--literal-pathspecs","-c","core.bare=false","status","-uall","-z"]
+    [2016-09-08 15:05:23.239573491] chat: git ["--git-dir=.git","--work-tree=.","--literal-pathspecs","-c","core.bare=false","cat-file","--batch"]
+    [2016-09-08 15:08:25.605689622] process done ExitSuccess
+    git annex status --debug  33802.97s user 1162.38s system 51% cpu 18:47:39.54 total
+
+
+    % git-annex info
+    repository mode: direct
+    trusted repositories: 0
+    semitrusted repositories: 4
+    	00000000-0000-0000-0000-000000000001 -- web
+     	00000000-0000-0000-0000-000000000002 -- bittorrent
+     	069de9a2-dc53-4c0a-82e0-a61a1f29e6b3 -- stratos PC [stratos]
+     	49b5b3a4-56ac-4cf2-aed9-1c23d3181c97 -- Toshiba USB HDD [here]
+    untrusted repositories: 0
+    transfers in progress: none
+    available local disk space: 120.41 gigabytes (+1 megabyte reserved)
+    local annex keys: 6303
+    local annex size: 1.87 terabytes
+    annexed files in working tree: 7412
+    size of annexed files in working tree: 2.24 terabytes
+    bloom filter size: 32 mebibytes (1.3% full)
+    backend usage: 
+    	SHA256E: 7412
+
+
+    % git-annex version
+    git-annex version: 6.20160613-g1e4e6f4
+    build flags: Assistant Webapp Pairing Testsuite S3(multipartupload)(storageclasses) WebDAV Inotify DBus DesktopNotify XMPP ConcurrentOutput TorrentParser MagicMime Feeds Quvi
+    key/value backends: SHA256E SHA256 SHA512E SHA512 SHA224E SHA224 SHA384E SHA384 SHA3_256E SHA3_256 SHA3_512E SHA3_512 SHA3_224E SHA3_224 SHA3_384E SHA3_384 SKEIN256E SKEIN256 SKEIN512E SKEIN512 SHA1E SHA1 MD5E MD5 WORM URL
+    remote types: git gcrypt S3 bup directory rsync web bittorrent webdav tahoe glacier ddar hook external
+    local repository version: 5
+    supported repository versions: 5 6
+    upgrade supported from repository versions: 0 1 2 3 4 5
+    operating system: linux x86_64
+
+Thank you for your time.
diff --git a/doc/forum/Slow_direct_mode_repository/comment_1_63656f103d418b35712a68f771ed559c._comment b/doc/forum/Slow_direct_mode_repository/comment_1_63656f103d418b35712a68f771ed559c._comment
new file mode 100644
index 0000000000..9c57bf1824
--- /dev/null
+++ b/doc/forum/Slow_direct_mode_repository/comment_1_63656f103d418b35712a68f771ed559c._comment
@@ -0,0 +1,9 @@
+[[!comment format=mdwn
+ username="Gus"
+ subject="Never mind"
+ date="2016-09-10T09:15:56Z"
+ content="""
+I tried to reboot the system. I am getting normal speeds again. That was a relief!
+
+I'm sorry to have bothered you.
+"""]]
diff --git a/doc/forum/Slow_transfer_speeds_on_copy_in_Windows.mdwn b/doc/forum/Slow_transfer_speeds_on_copy_in_Windows.mdwn
new file mode 100644
index 0000000000..3cbd7f4459
--- /dev/null
+++ b/doc/forum/Slow_transfer_speeds_on_copy_in_Windows.mdwn
@@ -0,0 +1,3 @@
+Copying a large file from windows (msysgit 1.9.5.msysgit.1, git-annex 20150406) to a bare repository on debian wheezy (git 1.9.1 git-annex 20141024)  I can only get around 2 MB/s transfer speed.  I tested using normal windows copy (smb) and got around 10 MB/s, while the git-annex copy was still going on.  I tried plain ssh and got 9 MB/s.  So, something to do with git-annex is being slow it seems.  Any ideas on how to remedy this, or is it a known issue?
+
+Note: Problem not seen going from linux to linux... waaaayyy faster (40 MB/s) to put it lightly (10 min vs all day)  The destination server is the same.
diff --git a/doc/forum/Slow_transfer_speeds_on_copy_in_Windows/comment_1_dd4ebb10ac87e3ee6158b7e7b1273a81._comment b/doc/forum/Slow_transfer_speeds_on_copy_in_Windows/comment_1_dd4ebb10ac87e3ee6158b7e7b1273a81._comment
new file mode 100644
index 0000000000..e6bdd2c4e7
--- /dev/null
+++ b/doc/forum/Slow_transfer_speeds_on_copy_in_Windows/comment_1_dd4ebb10ac87e3ee6158b7e7b1273a81._comment
@@ -0,0 +1,12 @@
+[[!comment format=mdwn
+ username="joey"
+ subject="""comment 1"""
+ date="2015-04-09T20:30:19Z"
+ content="""
+There's not a lot of places where git-annex could make this slower.
+git-annex copy is just running rsync to the ssh server.
+
+Have you tried benchmarking rsync of a large file to the server w/o
+git-annex? rsync does do considerably more client-side work than does
+scp, in order to support resuming, so that might be it.
+"""]]
diff --git a/doc/forum/Slow_transfer_speeds_on_copy_in_Windows/comment_2_9ebd40fe286f6c13f1021bf360e9c48e._comment b/doc/forum/Slow_transfer_speeds_on_copy_in_Windows/comment_2_9ebd40fe286f6c13f1021bf360e9c48e._comment
new file mode 100644
index 0000000000..9ad9acb7da
--- /dev/null
+++ b/doc/forum/Slow_transfer_speeds_on_copy_in_Windows/comment_2_9ebd40fe286f6c13f1021bf360e9c48e._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="https://www.google.com/accounts/o8/id?id=AItOawlh1G1u_AMJEyADqlfuzV2cePniocDyK6A"
+ nickname="Adam"
+ subject="comment 2"
+ date="2015-04-13T14:21:12Z"
+ content="""
+rsync is indeed slow...  The version bundled with msysgit is being used, and I read it has performance issues.  Will try a different version of rsync, perhaps in cygwin.
+"""]]
diff --git a/doc/forum/Slow_transfer_speeds_on_copy_in_Windows/comment_3_c169ad6205c998f3d44f9c0859071b2d._comment b/doc/forum/Slow_transfer_speeds_on_copy_in_Windows/comment_3_c169ad6205c998f3d44f9c0859071b2d._comment
new file mode 100644
index 0000000000..13a8541886
--- /dev/null
+++ b/doc/forum/Slow_transfer_speeds_on_copy_in_Windows/comment_3_c169ad6205c998f3d44f9c0859071b2d._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="https://www.google.com/accounts/o8/id?id=AItOawlh1G1u_AMJEyADqlfuzV2cePniocDyK6A"
+ nickname="Adam"
+ subject="comment 3"
+ date="2015-04-13T17:23:15Z"
+ content="""
+Verified to be rsync 3.0.9 that is bundled with git annex which is causing the slowdown. Updated to cwRsync 3.1.1 and it was fast again.
+"""]]
diff --git a/doc/forum/Slow_transfer_speeds_on_copy_in_Windows/comment_4_aee3fc6be01bb75709451eea0decf112._comment b/doc/forum/Slow_transfer_speeds_on_copy_in_Windows/comment_4_aee3fc6be01bb75709451eea0decf112._comment
new file mode 100644
index 0000000000..ed2f3cc358
--- /dev/null
+++ b/doc/forum/Slow_transfer_speeds_on_copy_in_Windows/comment_4_aee3fc6be01bb75709451eea0decf112._comment
@@ -0,0 +1,13 @@
+[[!comment format=mdwn
+ username="joey"
+ subject="""comment 4"""
+ date="2015-04-18T19:12:19Z"
+ content="""
+Well, git-annex bundles a copy of rsync, and the one that it includes 
+is the one from cygwin.
+
+It sounds like there's a newer version, also from cygwin, that is faster?
+
+I've pinged the git-annex Windows autobuilder admin to see if we can
+upgrade that.
+"""]]
diff --git a/doc/forum/Slow_transfer_speeds_on_copy_in_Windows/comment_5_b1841fc129a9ce6d1c22840ee648f958._comment b/doc/forum/Slow_transfer_speeds_on_copy_in_Windows/comment_5_b1841fc129a9ce6d1c22840ee648f958._comment
new file mode 100644
index 0000000000..147ca76828
--- /dev/null
+++ b/doc/forum/Slow_transfer_speeds_on_copy_in_Windows/comment_5_b1841fc129a9ce6d1c22840ee648f958._comment
@@ -0,0 +1,13 @@
+[[!comment format=mdwn
+ username="joey"
+ subject="""comment 5"""
+ date="2015-04-18T19:26:25Z"
+ content="""
+
+
+Hmm, ok, so that's not the cygwin rsync and it has a gui and bundles its
+own ssh that won't behave the same as the one bundled with git-annex.
+
+Maybe at least some of the speed improvements could be had by just
+upgrading the rsync and ssh from cygwin though..
+"""]]
diff --git a/doc/forum/Slow_transfer_speeds_on_copy_in_Windows/comment_6_1ce9bb47dadd2b1c500b2a20fd669907._comment b/doc/forum/Slow_transfer_speeds_on_copy_in_Windows/comment_6_1ce9bb47dadd2b1c500b2a20fd669907._comment
new file mode 100644
index 0000000000..43f5c3f2ee
--- /dev/null
+++ b/doc/forum/Slow_transfer_speeds_on_copy_in_Windows/comment_6_1ce9bb47dadd2b1c500b2a20fd669907._comment
@@ -0,0 +1,9 @@
+[[!comment format=mdwn
+ username="https://www.google.com/accounts/o8/id?id=AItOawlh1G1u_AMJEyADqlfuzV2cePniocDyK6A"
+ nickname="Adam"
+ subject="comment 6"
+ date="2015-04-20T17:40:38Z"
+ content="""
+There are pathing issues with other applications if I add cygwin to the path or are you saying copy over cygwin's copy?  I did have issues with ssh and had to copy all the files to the bin dir rather than the cmd dir to get it to work.  This also screwed up my home dir, making c:\ my home.  I went with it, since I'm the only user, and copied my .ssh folder to C:\... not the most secure option, but it worked.  (Note: it seemed to ignore the $HOME env variable)
+
+"""]]
diff --git a/doc/forum/Slow_transfer_speeds_on_copy_in_Windows/comment_7_4b78b200b884c4ac7c052055b3e26784._comment b/doc/forum/Slow_transfer_speeds_on_copy_in_Windows/comment_7_4b78b200b884c4ac7c052055b3e26784._comment
new file mode 100644
index 0000000000..a49eacd140
--- /dev/null
+++ b/doc/forum/Slow_transfer_speeds_on_copy_in_Windows/comment_7_4b78b200b884c4ac7c052055b3e26784._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="https://www.google.com/accounts/o8/id?id=AItOawlh1G1u_AMJEyADqlfuzV2cePniocDyK6A"
+ nickname="Adam"
+ subject="comment 7"
+ date="2015-04-20T17:43:11Z"
+ content="""
+I'm not using cygwin for git-annex.  I'm using msysgit and git's command window.
+"""]]
diff --git a/doc/forum/Slow_transfer_speeds_on_copy_in_Windows/comment_8_97ef11581c5dc6eeeabb4b244bdc6c30._comment b/doc/forum/Slow_transfer_speeds_on_copy_in_Windows/comment_8_97ef11581c5dc6eeeabb4b244bdc6c30._comment
new file mode 100644
index 0000000000..9363c0bfa7
--- /dev/null
+++ b/doc/forum/Slow_transfer_speeds_on_copy_in_Windows/comment_8_97ef11581c5dc6eeeabb4b244bdc6c30._comment
@@ -0,0 +1,7 @@
+[[!comment format=mdwn
+ username="http://joeyh.name/"
+ subject="comment 8"
+ date="2015-04-20T17:43:48Z"
+ content="""
+We've updated the autobuilder, so the daily build now has the most recent rsync and ssh from cygwin included in it. I'd be curious if that improves the speed any.
+"""]]
diff --git a/doc/forum/Some_files_missing_extension_in_annex.mdwn b/doc/forum/Some_files_missing_extension_in_annex.mdwn
new file mode 100644
index 0000000000..bae149b3c1
--- /dev/null
+++ b/doc/forum/Some_files_missing_extension_in_annex.mdwn
@@ -0,0 +1,11 @@
+I hadn't noticed this before, but I added some files in the past and, finally looking at them today, the file extension is missing in the annex. This confuses OS X something awful, so it opens the file in TextEdit instead of VLC. If I open the file directly using VLC, it works.
+
+I just dropped and re-added the file that has an extension. It lacked it before:
+
+    ls -l 10*
+    lrwxr-xr-x  1 jasonb  staff  215 Sep 16 15:23 10 Minute Ab Chisel.mkv -> ../.git/annex/objects/zw/mG/WORM-s103155193-m1516830000--Masters,32Hammer,32and,32Chisel-a67ed6d2a3dfd5fe29be62146fbd0c04/WORM-s103155193-m1516830000--Masters,32Hammer,32and,32Chisel-a67ed6d2a3dfd5fe29be62146fbd0c04
+    lrwxr-xr-x  1 jasonb  staff  209 Sep 16 15:04 10 Minute Ab Hammer.mkv -> ../.git/annex/objects/jx/5p/WORM-s117692368-m1537124682--Masters,32Hammer,32and,32Chisel%10,32Minute,32Ab,32Hammer.mkv/WORM-s117692368-m1537124682--Masters,32Hammer,32and,32Chisel%10,32Minute,32Ab,32Hammer.mkv
+
+Is there something I can do to recover my file extensions besides manually dropping and re-adding each file?
+
+Thanks.
diff --git a/doc/forum/Some_files_missing_extension_in_annex/comment_1_12ab578a4390c0f5896106906997de4b._comment b/doc/forum/Some_files_missing_extension_in_annex/comment_1_12ab578a4390c0f5896106906997de4b._comment
new file mode 100644
index 0000000000..09f12fbccd
--- /dev/null
+++ b/doc/forum/Some_files_missing_extension_in_annex/comment_1_12ab578a4390c0f5896106906997de4b._comment
@@ -0,0 +1,9 @@
+[[!comment format=mdwn
+ username="jasonb@ab4484d9961a46440958fa1a528e0fc435599057"
+ nickname="jasonb"
+ avatar="http://cdn.libravatar.org/avatar/c7330f4da122c671b935fc1d58bb02b1"
+ subject="comment 1"
+ date="2018-09-16T20:25:39Z"
+ content="""
+Doing a git annex unannex and then a git annex add fixes it. Now sure how this happened originally.
+"""]]
diff --git a/doc/forum/Some_files_missing_extension_in_annex/comment_2_7fa25274fb00ba823b112d587c52df76._comment b/doc/forum/Some_files_missing_extension_in_annex/comment_2_7fa25274fb00ba823b112d587c52df76._comment
new file mode 100644
index 0000000000..4c5f186563
--- /dev/null
+++ b/doc/forum/Some_files_missing_extension_in_annex/comment_2_7fa25274fb00ba823b112d587c52df76._comment
@@ -0,0 +1,11 @@
+[[!comment format=mdwn
+ username="jasonb@ab4484d9961a46440958fa1a528e0fc435599057"
+ nickname="jasonb"
+ avatar="http://cdn.libravatar.org/avatar/c7330f4da122c671b935fc1d58bb02b1"
+ subject="comment 2"
+ date="2018-09-16T20:32:08Z"
+ content="""
+Unfortunately, my remotes don't know anything about the change, so when I push it, the symlinks point to files that don't exist, even though the data is still in the annex with the original mysterious filenames.
+
+How to get git-annex to recover the existing data?
+"""]]
diff --git a/doc/forum/Some_mounted_devices_not_detected.mdwn b/doc/forum/Some_mounted_devices_not_detected.mdwn
new file mode 100644
index 0000000000..6146ef14fb
--- /dev/null
+++ b/doc/forum/Some_mounted_devices_not_detected.mdwn
@@ -0,0 +1,3 @@
+Automounted USB devices are not detected by the git-annex webapp on my Debian testing/squeeze installation, only drives with entries in /etc/fstab show up in the device list. Is there any way to tweak/get around this?
+
+I'm running version 4.20131106 (couldn't manage to build the package for sid). 
diff --git a/doc/forum/Some_mounted_devices_not_detected/comment_1_0ba07b95f12f57ea63bb450b88430c45._comment b/doc/forum/Some_mounted_devices_not_detected/comment_1_0ba07b95f12f57ea63bb450b88430c45._comment
new file mode 100644
index 0000000000..5cd472c8e6
--- /dev/null
+++ b/doc/forum/Some_mounted_devices_not_detected/comment_1_0ba07b95f12f57ea63bb450b88430c45._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="http://joeyh.name/"
+ ip="209.250.56.64"
+ subject="comment 1"
+ date="2013-11-24T16:04:23Z"
+ content="""
+can you post an example line from /proc/mounts for one of these devices?
+"""]]
diff --git a/doc/forum/Some_mounted_devices_not_detected/comment_2_4f8c7bcd0f20dafa5635a3580ec8d1f6._comment b/doc/forum/Some_mounted_devices_not_detected/comment_2_4f8c7bcd0f20dafa5635a3580ec8d1f6._comment
new file mode 100644
index 0000000000..60512d418b
--- /dev/null
+++ b/doc/forum/Some_mounted_devices_not_detected/comment_2_4f8c7bcd0f20dafa5635a3580ec8d1f6._comment
@@ -0,0 +1,13 @@
+[[!comment format=mdwn
+ username="https://www.google.com/accounts/o8/id?id=AItOawmyEv6SesaYeke389mHdiHlavirPkGHzHI"
+ nickname="Albin"
+ subject="comment 2"
+ date="2013-11-26T14:35:58Z"
+ content="""
+I missed your reply on IRC. Sorry for that!
+
+Here's an example from /proc/mounts:
+
+   /dev/sdf1 /media/SoderUSB ext4 rw,nosuid,nodev,noexec,relatime,errors=remount-ro,data=ordered 0 0
+
+"""]]
diff --git a/doc/forum/Some_mounted_devices_not_detected/comment_3_06c0db7d670d9b82823102d22db15a36._comment b/doc/forum/Some_mounted_devices_not_detected/comment_3_06c0db7d670d9b82823102d22db15a36._comment
new file mode 100644
index 0000000000..53c76e210b
--- /dev/null
+++ b/doc/forum/Some_mounted_devices_not_detected/comment_3_06c0db7d670d9b82823102d22db15a36._comment
@@ -0,0 +1,10 @@
+[[!comment format=mdwn
+ username="http://joeyh.name/"
+ ip="209.250.56.64"
+ subject="comment 3"
+ date="2013-11-26T19:59:42Z"
+ content="""
+Hmm, that looks fine.
+
+Perhaps the user you're running git-annex as cannot write to `/media/SoderUSB` ? The webapp avoids listing places you can't write, since it would fail to make a repository there. (Also because this is an easy way for it to avoid listing `/`, `/usr`, or whatever other OS-level mounts a system might have..)
+"""]]
diff --git a/doc/forum/Some_mounted_devices_not_detected/comment_4_80820a29361c5be4a94672dacfdefa6f._comment b/doc/forum/Some_mounted_devices_not_detected/comment_4_80820a29361c5be4a94672dacfdefa6f._comment
new file mode 100644
index 0000000000..03b917a69a
--- /dev/null
+++ b/doc/forum/Some_mounted_devices_not_detected/comment_4_80820a29361c5be4a94672dacfdefa6f._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="https://www.google.com/accounts/o8/id?id=AItOawmyEv6SesaYeke389mHdiHlavirPkGHzHI"
+ nickname="Albin"
+ subject="comment 4"
+ date="2013-11-27T17:33:22Z"
+ content="""
+Actually, that (write permissions) seems to have been the problem. Thanks! Can't imagine why pmount would not mount a device writable by my user, though.
+"""]]
diff --git a/doc/forum/Somehow_have_lots_of_directories_in_root__58___000...ffff.mdwn b/doc/forum/Somehow_have_lots_of_directories_in_root__58___000...ffff.mdwn
new file mode 100644
index 0000000000..3d93d80269
--- /dev/null
+++ b/doc/forum/Somehow_have_lots_of_directories_in_root__58___000...ffff.mdwn
@@ -0,0 +1,22 @@
+I'm using git-annex to manage my photos on my laptop, and have my NAS (which is just a debian linux system with lots of disks) as one of my remotes. I use my NAS as a backup, and to free up space on my laptop.
+
+But somehow I now have lots of directories in my main root directory of the photos directory. it looks like [0-9a-f][0-9a-f][0-9a-f], so I have 000, ... a00, ... ddd, ..., all the way to fff. Thousands of them. Each one has a few directories that also match [0-9a-f][0-9a-f][0-9a-f], and each one has a file with names of the git annex object filename.
+
+e.g.:
+
+    $ cat e1d/760/SHA256-s4061375--7909e266f340ff8bc692e073e776ceda267042adf790275a2d851f2c7eb1ca83.log 
+    1338931894.759394s 1 1adfa84c-af51-11e1-b075-0b1206d553bc
+    1338992164.202818s 1 721d82ae-afbb-11e1-b78f-f396ebf52c05
+    1339862991.346546s 1 721d82ae-afbb-11e1-b78f-f396ebf52c05
+    1339946933.224915s 1 224d85b4-b86f-11e1-b11e-3333ce526126
+    1339964057.205993s 1 224d85b4-b86f-11e1-b11e-3333ce526126
+    1343495865.083951s 1 721d82ae-afbb-11e1-b78f-f396ebf52c05
+    1344788941.403528s 1 721d82ae-afbb-11e1-b78f-f396ebf52c05
+    1351802706.679542s 1 721d82ae-afbb-11e1-b78f-f396ebf52c05
+    1362423729.820711s 1 721d82ae-afbb-11e1-b78f-f396ebf52c05
+
+My remote (NAS) has these files & directories as well as my laptop. The regular photo directory hierarchy is there as well, the files work, can be copied etc. "git status" say nothing has changed. Those files are tracked by git (they have lot messages). Everything just works, it's just that there's loads of these directories? Can I remove them? How can I clear it up? I can't remember when these directories/files were added, I only noticed it today.
+
+I don't know if I had my remote (NAS) as a bare git repo and used it as that, then somehow used it as a non-bare one, would that cause this?
+
+Can I remove them? Where did they come from? What's going on?
diff --git a/doc/forum/Somehow_have_lots_of_directories_in_root__58___000...ffff/comment_1_20147b287fd995fa8ac9e868b5974d8a._comment b/doc/forum/Somehow_have_lots_of_directories_in_root__58___000...ffff/comment_1_20147b287fd995fa8ac9e868b5974d8a._comment
new file mode 100644
index 0000000000..1f5018e34a
--- /dev/null
+++ b/doc/forum/Somehow_have_lots_of_directories_in_root__58___000...ffff/comment_1_20147b287fd995fa8ac9e868b5974d8a._comment
@@ -0,0 +1,12 @@
+[[!comment format=mdwn
+ username="http://joeyh.name/"
+ ip="209.250.56.2"
+ subject="comment 1"
+ date="2014-07-10T19:50:38Z"
+ content="""
+The files that you describe are normally tucked away on the git-annex branch of the git repository, where they're neither seen nor heard.
+
+The most likely cause of your problem would be if you have done a \"git checkout git-annex\"  ... or worse, a \"git merge git-annex\".
+
+So, check what branch you have checked out, and if it's the git-annex branch, you'll want to change back to master. If you've got master checked out and have these files, you can use `git log --stat` and see if there's a commit that added a bunch of these files to your master branch. Then you can revert that commit.
+"""]]
diff --git a/doc/forum/Somehow_have_lots_of_directories_in_root__58___000...ffff/comment_2_f6d977a534264b4368401e1b13628931._comment b/doc/forum/Somehow_have_lots_of_directories_in_root__58___000...ffff/comment_2_f6d977a534264b4368401e1b13628931._comment
new file mode 100644
index 0000000000..e902380633
--- /dev/null
+++ b/doc/forum/Somehow_have_lots_of_directories_in_root__58___000...ffff/comment_2_f6d977a534264b4368401e1b13628931._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="https://launchpad.net/~rorymcc"
+ nickname="rorymcc"
+ subject="comment 2"
+ date="2014-08-11T18:37:46Z"
+ content="""
+Thanks for your reply. I think I might have done a \"git merge git-annex\" at some point (or many times), because I thought that was what you were supposed to do... :( PEBKAC I'll try to fix up my repository. Thanks.
+"""]]
diff --git a/doc/forum/Somehow_have_lots_of_directories_in_root__58___000...ffff/comment_3_d534da276b79a40fdb7d8d158f6eae26._comment b/doc/forum/Somehow_have_lots_of_directories_in_root__58___000...ffff/comment_3_d534da276b79a40fdb7d8d158f6eae26._comment
new file mode 100644
index 0000000000..214444c61a
--- /dev/null
+++ b/doc/forum/Somehow_have_lots_of_directories_in_root__58___000...ffff/comment_3_d534da276b79a40fdb7d8d158f6eae26._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="https://launchpad.net/~rorymcc"
+ nickname="rorymcc"
+ subject="comment 3"
+ date="2014-08-11T18:41:05Z"
+ content="""
+Would just standard \"git rm ./000/\" etc. in master be OK? Instead of hunting down and reverting all the commits? 
+"""]]
diff --git a/doc/forum/Somehow_have_lots_of_directories_in_root__58___000...ffff/comment_4_8c817d08ca9d94a1228fb21cd0b15744._comment b/doc/forum/Somehow_have_lots_of_directories_in_root__58___000...ffff/comment_4_8c817d08ca9d94a1228fb21cd0b15744._comment
new file mode 100644
index 0000000000..74a3a2a756
--- /dev/null
+++ b/doc/forum/Somehow_have_lots_of_directories_in_root__58___000...ffff/comment_4_8c817d08ca9d94a1228fb21cd0b15744._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="http://joeyh.name/"
+ ip="209.250.56.7"
+ subject="comment 4"
+ date="2014-08-12T17:50:15Z"
+ content="""
+Sure, it's fine to delete the files. The same info will be committed to git either way.
+"""]]
diff --git a/doc/forum/Special_remote_public_key_encryption_issue.mdwn b/doc/forum/Special_remote_public_key_encryption_issue.mdwn
new file mode 100644
index 0000000000..aed35ad31b
--- /dev/null
+++ b/doc/forum/Special_remote_public_key_encryption_issue.mdwn
@@ -0,0 +1,18 @@
+I'm unable to set up a standard architecture: 2 computers sharing some files, talking via a jabber account and transferring files via e.g. box.com.
+Steps:
+
+Computer 1: 
+a) Install git-annex, open up the assistant, create a new repository, add a jabber account
+b) Add a box.com account using
+WEBDAV_USERNAME=[username] WEBDAV_PASSWORD=[password] git annex initremote box.com type=webdav url=https://dav.box.com/dav/git-annex chunksize=75mb encryption=pubkey keyid=[key id].
+All seemed fine.
+
+Computer 2:
+a) same
+b) at this point I saw the option of adding box.com as a cloud storage. I clicked enable and got the message
+
+user error (gpg ["--quiet","--trust-model","always","--decrypt"] exited 2)
+
+In the meantime, on Computer 1 I saw the same option. I clicked enable and got the same message.
+
+It all works fine if I use shared encryption.
diff --git a/doc/forum/Special_remote_public_key_encryption_issue/comment_1_a9caafea017a3c148f89d4ddeee15a4c._comment b/doc/forum/Special_remote_public_key_encryption_issue/comment_1_a9caafea017a3c148f89d4ddeee15a4c._comment
new file mode 100644
index 0000000000..6c5a35c3a3
--- /dev/null
+++ b/doc/forum/Special_remote_public_key_encryption_issue/comment_1_a9caafea017a3c148f89d4ddeee15a4c._comment
@@ -0,0 +1,10 @@
+[[!comment format=mdwn
+ username="http://joeyh.name/"
+ ip="108.236.230.124"
+ subject="comment 1"
+ date="2014-05-16T18:22:26Z"
+ content="""
+Which is why the assistant uses shared encryption by default..
+
+You have set up a webdav repository using a gpg key that is on computer 1. To be able to access this repository on computer 2 you'll need to either copy the gpg private key from computer 1, or give computer 2 its own key and, on computer 1, add that key as one that can access the webdav repository.
+"""]]
diff --git a/doc/forum/Special_remote_public_key_encryption_issue/comment_2_adfa582d611ca501e21110282df07315._comment b/doc/forum/Special_remote_public_key_encryption_issue/comment_2_adfa582d611ca501e21110282df07315._comment
new file mode 100644
index 0000000000..8e27535307
--- /dev/null
+++ b/doc/forum/Special_remote_public_key_encryption_issue/comment_2_adfa582d611ca501e21110282df07315._comment
@@ -0,0 +1,18 @@
+[[!comment format=mdwn
+ username="hyperio"
+ ip="62.31.92.216"
+ subject="Still not working"
+ date="2014-05-17T12:22:17Z"
+ content="""
+Thank you for your reply. I copied over the public key from computer 2 to computer 1 and tried to add it using git annex enableremote box.com keyid+=[keyid]. I am then asked for the passphrase for my private key on computer 1. Afterwards I am asked for the passphrase for the private key on computer 2... why? Obviously I don't have it so I get 
+
+gpg: can't query passphrase in batch mode
+gpg: decryption failed: secret key not available
+
+git-annex: user error (gpg [\"--quiet\",\"--trust-model\",\"always\",\"--batch\",\"--decrypt\"] exited 2)
+
+Why would I need computer 2's secret key? Or I might just get it completely wrong.
+Also, can the passphrase be remembered somewhere, so that I can use the assistant? (otherwise I think it won't work, because I would have to type the passphrase at every synchronisation attempt).
+
+Sorry for my confusion, thanks for your support.
+"""]]
diff --git a/doc/forum/Special_remote_public_key_encryption_issue/comment_3_eac16bf98a6e87461ba2f3ab7e990b2b._comment b/doc/forum/Special_remote_public_key_encryption_issue/comment_3_eac16bf98a6e87461ba2f3ab7e990b2b._comment
new file mode 100644
index 0000000000..692b307b10
--- /dev/null
+++ b/doc/forum/Special_remote_public_key_encryption_issue/comment_3_eac16bf98a6e87461ba2f3ab7e990b2b._comment
@@ -0,0 +1,10 @@
+[[!comment format=mdwn
+ username="http://joeyh.name/"
+ ip="216.145.95.162"
+ subject="comment 3"
+ date="2014-05-17T17:02:15Z"
+ content="""
+I dont think it's possible for gpg to prompt for a passphrase when all it has is a public key, so I think you copied the private key too.
+
+gpg agents can cache your passphrase.
+"""]]
diff --git a/doc/forum/Special_remote_public_key_encryption_issue/comment_6_097f52aaf178340b3abb5bfc80f0d447._comment b/doc/forum/Special_remote_public_key_encryption_issue/comment_6_097f52aaf178340b3abb5bfc80f0d447._comment
new file mode 100644
index 0000000000..93fa8b5b9c
--- /dev/null
+++ b/doc/forum/Special_remote_public_key_encryption_issue/comment_6_097f52aaf178340b3abb5bfc80f0d447._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="hyperio"
+ ip="62.31.92.216"
+ subject="Still not working"
+ date="2014-05-17T17:25:39Z"
+ content="""
+I exported the public key on computer 2 with gpg --armor --export [keyid] and just imported it on computer 1...
+"""]]
diff --git a/doc/forum/Special_remote_without_chmod.mdwn b/doc/forum/Special_remote_without_chmod.mdwn
new file mode 100644
index 0000000000..46ace443b1
--- /dev/null
+++ b/doc/forum/Special_remote_without_chmod.mdwn
@@ -0,0 +1,12 @@
+Apparently, the tablet computer I'm using (Galaxy Tab 2, non-rooted) does not export the sd cards via mass storage protocol (UMS) any more. In order to be able to use a special remote for my mp3-files,  I installed an ftp/ssh server on the tablet device and used curlftpfs/sshfs (fuse file systems) to mount the sd cards on my local Linux machine. In this way, it is is quite easy to setup the special remote (e.g. "git-annex initremote galaxy-tab type=directory directory=$HOME/mnt/galaxy-tab encryption=none"). 
+
+Problems arise, when files are transferred to the special remote. From the logs of the ftp server, I can see that the actual copy operation is successful (the data is written to the file system), but the subsequent "chmod" that changes the permissions to read-only fails. The output on the console of a "git-annex copy -t galaxy-tab" is 
+
+    copy 01 some.mp3 (to galaxy-tab...) 
+    failed                  
+    git-annex: copy: 1 failed
+
+
+Therefore my question: is it possible to perform a file transfer to a special remote (type=directory) without the final "chmod"-operation?
+
+Thank you.
diff --git a/doc/forum/Special_remote_without_chmod/comment_1_4f5f9506cae72a1f321296fc5a5f339a._comment b/doc/forum/Special_remote_without_chmod/comment_1_4f5f9506cae72a1f321296fc5a5f339a._comment
new file mode 100644
index 0000000000..225cfa6cb5
--- /dev/null
+++ b/doc/forum/Special_remote_without_chmod/comment_1_4f5f9506cae72a1f321296fc5a5f339a._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="http://joeyh.name/"
+ ip="4.154.6.49"
+ subject="comment 1"
+ date="2012-11-25T22:00:12Z"
+ content="""
+That could be hacked into directory as a configuration setting, but I think it makes more sense to add a proper sftp special remote, and avoid the rather complex stack you're using.
+"""]]
diff --git a/doc/forum/Speed_up_remote_transfers.mdwn b/doc/forum/Speed_up_remote_transfers.mdwn
new file mode 100644
index 0000000000..04ebdc8f9d
--- /dev/null
+++ b/doc/forum/Speed_up_remote_transfers.mdwn
@@ -0,0 +1,5 @@
+I am currently trying to transfer git-annex managed files to my server.
+
+I am getting them with `git annex /path`. However, these are very many small files, and git annex seems to take a tiny break in between them, so the process takes considerably longer than uploading a single file of the same size would.
+
+Do you have any suggestions for improving my transfer speed in this case?
diff --git a/doc/forum/Speed_up_remote_transfers/comment_1_bd97e1c47338322c5232cea7cdbc26fc._comment b/doc/forum/Speed_up_remote_transfers/comment_1_bd97e1c47338322c5232cea7cdbc26fc._comment
new file mode 100644
index 0000000000..f4e9194dd4
--- /dev/null
+++ b/doc/forum/Speed_up_remote_transfers/comment_1_bd97e1c47338322c5232cea7cdbc26fc._comment
@@ -0,0 +1,18 @@
+[[!comment format=mdwn
+ username="CandyAngel"
+ subject="comment 1"
+ date="2015-06-06T14:57:35Z"
+ content="""
+I did this using a couple of tools (off the top of my head):
+
+git annex find path/ --format '${key}\n' > /tmp/keyslist (outputs a list of keys)
+find .git/annex/objects -type f | grep -wFf /tmp/keyslist > /tmp/filelist (outputs a list of files)
+
+Then I use 'resync -avhP --files-from=/tmp/filelist . othermachine:some/tmp/dir' to transfer the files to the other machine.
+
+Then I 'git annex import some/tmp/dir' to inject the content, then just delete the additional symlinks (and reset the index).
+
+This speeds things up a bit. Even more speed can be had using tar/netcat.
+
+Hope this points you in the right direction.
+"""]]
diff --git a/doc/forum/Speed_up_remote_transfers/comment_2_723b3b69d47c947e5cbe8e94b84df022._comment b/doc/forum/Speed_up_remote_transfers/comment_2_723b3b69d47c947e5cbe8e94b84df022._comment
new file mode 100644
index 0000000000..bca1f6eccd
--- /dev/null
+++ b/doc/forum/Speed_up_remote_transfers/comment_2_723b3b69d47c947e5cbe8e94b84df022._comment
@@ -0,0 +1,9 @@
+[[!comment format=mdwn
+ username="joey"
+ subject="""comment 2"""
+ date="2015-06-09T19:45:26Z"
+ content="""
+I'd suggest running 2 or more concurrent transfers. Either by just running
+the same "git annex get" command multiple times, or by using "git annex get -J2"
+if you've got a new enough version of git-annex that has that feature.
+"""]]
diff --git a/doc/forum/Split_annex_into_multiple.mdwn b/doc/forum/Split_annex_into_multiple.mdwn
new file mode 100644
index 0000000000..2ebdf63d53
--- /dev/null
+++ b/doc/forum/Split_annex_into_multiple.mdwn
@@ -0,0 +1,16 @@
+Lets say I have an annex, called Global in some location A. The plan is to mirror it to some location B.
+
+I suddenly decide that Global has become too unwieldy, and I wish to split it up. 
+
+I "rsync -Lhav" the entire annex over to B, so it turns back into raw files (as I'm a little worried about the broken symlinks some have mentioned when using uninit).
+I then reorganize all the files in B into five new annexes. Yay.
+
+Now, I clone those five repos back onto A. How can I use the content of the original repo "Global" to populate the new annexes?
+
+**Approach 1**: delete Global, use git-annex copy (slow, wasteful use of precious disk life)
+
+**Approach 2**: uninit Global, deal with whatever broken symlink mess *may* occur, and then use reinject. Reinject doesn't seem to work recursively afaik (ok, I can use find -type f -exec I guess...), and doesn't seem to want to work on files that are sitting inside an annex already (i.e. the uninit is not optional)
+
+So besides these two, are there any approaches I'm overlooking?
+
+Context: entire photo collection. split into e.g. parents digitized photos, my photos, etc
diff --git a/doc/forum/Split_annex_into_multiple/comment_1_e934b404f738cbd67df928fd6769e3ff._comment b/doc/forum/Split_annex_into_multiple/comment_1_e934b404f738cbd67df928fd6769e3ff._comment
new file mode 100644
index 0000000000..84451b86a8
--- /dev/null
+++ b/doc/forum/Split_annex_into_multiple/comment_1_e934b404f738cbd67df928fd6769e3ff._comment
@@ -0,0 +1,17 @@
+[[!comment format=mdwn
+ username="joey"
+ subject="""comment 1"""
+ date="2017-04-05T19:09:03Z"
+ content="""
+Recent git-annex's support `git annex reinject --known`
+
+You can do this -- but do note that it leaves Global in an unusable state,
+so only do it if you don't plan to use that repository again:
+
+	chmod -R +w $Global/.git/annex/objects/
+	git annex reinject --known $(find $Global/.git/annex/objects/ -type f)
+
+BTW, splitting the files amoung several git branches is also a useful
+thing to do in this situation. Splitting amoung branches avoids most of
+git's problems with a repository having too many files in it.
+"""]]
diff --git a/doc/forum/Split_annex_into_multiple/comment_2_eec6bc8c49c08411a4bb2cf7cc1c4697._comment b/doc/forum/Split_annex_into_multiple/comment_2_eec6bc8c49c08411a4bb2cf7cc1c4697._comment
new file mode 100644
index 0000000000..1957eb2ac8
--- /dev/null
+++ b/doc/forum/Split_annex_into_multiple/comment_2_eec6bc8c49c08411a4bb2cf7cc1c4697._comment
@@ -0,0 +1,13 @@
+[[!comment format=mdwn
+ username="archimedes"
+ avatar="http://cdn.libravatar.org/avatar/5b2ed40aabedcb1b8c2c0ce69f55a1e1"
+ subject="comment 2"
+ date="2017-04-05T20:01:44Z"
+ content="""
+So actually \"import --reinject-duplicates\" solves the issue of reinject not being recursive.
+
+
+That trick on .git/annex/objects/ is cunning, but the destruction of Global before everything has been finished is discomforting...if only there was a \"--reinject-duplicates --duplicate\" option for import.
+
+No chance that I can make a recursive copy of .git/annex/objects with hardlinks can I? To then use with reinject-duplicates? 
+"""]]
diff --git a/doc/forum/Ssh_remote_on_NAS.mdwn b/doc/forum/Ssh_remote_on_NAS.mdwn
new file mode 100644
index 0000000000..27bca5a510
--- /dev/null
+++ b/doc/forum/Ssh_remote_on_NAS.mdwn
@@ -0,0 +1,34 @@
+Hello,
+
+I finally have git annex working on my NAS (QNAP TS-119P II) thanks to the stand-alone arm pre-build package.
+
+I've just extracted all the files on my NAS and I've linked all the exec files from the git-annex.linux folder to links placed in a path folder, so that I can run git annex everywhere on the NAS.
+
+    [~] # git annex version
+    git-annex version: 5.20140528-g92a0591
+    build flags: Assistant Webapp Webapp-secure Pairing Testsuite S3 WebDAV Inotify DBus DesktopNotify XMPP DNS Feeds Quvi TDFA CryptoHash
+    key/value backends: SHA256E SHA1E SHA512E SHA224E SHA384E SKEIN256E SKEIN512E SHA256 SHA1 SHA512 SHA224 SHA384 SKEIN256 SKEIN512 WORM URL
+    remote types: git gcrypt S3 bup directory rsync web webdav tahoe glacier ddar hook external
+
+Now I'm trying to set it up as an SSH remote of my laptop repository, but I get this error:
+
+    git clone ssh://admin@nas:/share/HDA_DATA/myDir ./
+    Cloning into '.'...
+    admin@nas's password: 
+    sh: git-upload-pack: command not found
+    fatal: Could not read from remote repository.
+    
+    Please make sure you have the correct access rights
+    and the repository exists.
+
+I've checked that git-upload-pack is available both on my laptop and on the NAS and the "myDir" folder is supposed to be readable for the admin user on the NAS, even if the user I use on the laptop and on the NAS for git annex is different from the one I use to login. In fact, if I try to scp files from the annex folder then I don't get any permissions error.
+
+    scp admin@192.168.132.66:/share/HDA_DATA/myDir/* ./
+    admin@192.168.132.66's password: 
+    doctest                                                100%    5     0.0KB/s   0.0KB/s   00:00    
+    doctest2                                               100%   51     0.1KB/s   0.1KB/s   00:01
+
+Is there something else I should look at, in order to fix it and make it work?
+
+Thanks,
+Fabio
diff --git a/doc/forum/Ssh_remote_on_NAS/comment_1_1dd8a0d0e70a1fb36fce62e89c99b404._comment b/doc/forum/Ssh_remote_on_NAS/comment_1_1dd8a0d0e70a1fb36fce62e89c99b404._comment
new file mode 100644
index 0000000000..6563a9bb60
--- /dev/null
+++ b/doc/forum/Ssh_remote_on_NAS/comment_1_1dd8a0d0e70a1fb36fce62e89c99b404._comment
@@ -0,0 +1,10 @@
+[[!comment format=mdwn
+ username="http://joeyh.name/"
+ ip="209.250.56.2"
+ subject="comment 1"
+ date="2014-07-11T19:46:05Z"
+ content="""
+How did you set your PATH? Note that the bash shell provides a lot of dotfiles which you can set the PATH in -- and reuses to read any single one of them when a noninteractive login is made to run a command.
+
+It might help to install git on the NAS. It's included in the git-annex tarball, but not in a way that will put it on PATH; only in a way that will let git-annex use it.
+"""]]
diff --git a/doc/forum/Ssh_remote_on_NAS/comment_2_261601313d8825c52322949b8509bc74._comment b/doc/forum/Ssh_remote_on_NAS/comment_2_261601313d8825c52322949b8509bc74._comment
new file mode 100644
index 0000000000..1cc8fb4443
--- /dev/null
+++ b/doc/forum/Ssh_remote_on_NAS/comment_2_261601313d8825c52322949b8509bc74._comment
@@ -0,0 +1,16 @@
+[[!comment format=mdwn
+ username="feulif"
+ ip="20.133.1.1"
+ subject="comment 2"
+ date="2014-07-15T18:03:56Z"
+ content="""
+> How did you set your PATH? Note that the bash shell provides a lot of dotfiles which you can set the PATH in -- and reuses to read any single one of them when a noninteractive login is made to run a command.
+
+Actually I didn't set the PATH: I've just sim-linked executable files from git-annex stand-alone folder to a directory that was in my PATH already.
+
+> It might help to install git on the NAS. It's included in the git-annex tarball, but not in a way that will put it on PATH; only in a way that will let git-annex use it.
+
+It was installed, bud I've removed it because it was \"conflicting\" with the one used by git-annex. That is, when I ran any \"git annex ...\" command, git was complaining it could not find git-annex (because my nas was running the installed git binary file).
+Without git, I don't have this conflict anymore and I can effectively use git-annex when I'm logged in my NAS, but I still can't add it as an SSH remote.
+
+"""]]
diff --git a/doc/forum/Ssh_remote_on_NAS/comment_3_ed602f4f972b78bce4f62bdfca8cfe47._comment b/doc/forum/Ssh_remote_on_NAS/comment_3_ed602f4f972b78bce4f62bdfca8cfe47._comment
new file mode 100644
index 0000000000..7ddb9dc8de
--- /dev/null
+++ b/doc/forum/Ssh_remote_on_NAS/comment_3_ed602f4f972b78bce4f62bdfca8cfe47._comment
@@ -0,0 +1,11 @@
+[[!comment format=mdwn
+ username="http://joeyh.name/"
+ ip="209.250.56.2"
+ subject="comment 3"
+ date="2014-07-15T18:55:58Z"
+ content="""
+I don't think it was a good idea to remove git from the NAS. To set up a git remote, you necessarily need to have git installed on the remote.
+
+`git annex` should work as long as git-annex is somewhere in PATH -- unless your build of git is very strange and does not check PATH for git-foo commands.
+Even if this was the case, you could run `git-annex` instead.
+"""]]
diff --git a/doc/forum/Ssh_remote_on_NAS/comment_4_423244f174123318f1ece7b5794aeea5._comment b/doc/forum/Ssh_remote_on_NAS/comment_4_423244f174123318f1ece7b5794aeea5._comment
new file mode 100644
index 0000000000..9b0a03ecf1
--- /dev/null
+++ b/doc/forum/Ssh_remote_on_NAS/comment_4_423244f174123318f1ece7b5794aeea5._comment
@@ -0,0 +1,28 @@
+[[!comment format=mdwn
+ username="feulif"
+ ip="151.27.171.22"
+ subject="comment 4"
+ date="2014-07-17T22:24:45Z"
+ content="""
+First of all, I understand this is more a nas-related issue rather than a git-annex-related one, so thank you for your support and valuable software.
+
+But I'm still struggling with this error.
+
+Now I've reinstalled git on the NAS and, as previously, I can run every git command when I'm logged in.
+But when I try to sync my PC with the git remote on the NAS, then I get this error:
+
+
+    git annex sync
+    sh: git-annex-shell: command not found
+    sh: git-upload-pack: command not found
+    fatal: Could not read from remote repository.
+    
+    Please make sure you have the correct access rights
+    and the repository exists.
+    commit  
+    ok
+    git-annex: no branch is checked out
+
+Can this be really related to some access right? What user is git annex using when syncing with an SSH remote (e.g. admin) ? the one I specified when setting up the remote? where is it complaining about git-upload-pack non found, on the nas or on the PC I'm syncing from?
+
+"""]]
diff --git a/doc/forum/Ssh_remote_on_NAS/comment_5_07348ed61ccdd93417365f1525a05bf2._comment b/doc/forum/Ssh_remote_on_NAS/comment_5_07348ed61ccdd93417365f1525a05bf2._comment
new file mode 100644
index 0000000000..c8e631226f
--- /dev/null
+++ b/doc/forum/Ssh_remote_on_NAS/comment_5_07348ed61ccdd93417365f1525a05bf2._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="joey"
+ subject="""comment 5"""
+ date="2014-10-28T20:26:08Z"
+ content="""
+git-upload-pack is a git command, which is not present in PATH
+on your NAS when git sshes in and tries to run it. 
+"""]]
diff --git a/doc/forum/Ssh_remote_on_NAS/comment_6_26d444afe2edea3500f80bb5c62ba491._comment b/doc/forum/Ssh_remote_on_NAS/comment_6_26d444afe2edea3500f80bb5c62ba491._comment
new file mode 100644
index 0000000000..af180a980c
--- /dev/null
+++ b/doc/forum/Ssh_remote_on_NAS/comment_6_26d444afe2edea3500f80bb5c62ba491._comment
@@ -0,0 +1,14 @@
+[[!comment format=mdwn
+ username="torpidus"
+ subject="comment 6"
+ date="2016-03-16T22:23:26Z"
+ content="""
+Hello Fabio,
+I came across this posting from you one year ago. Since I'm also using a QNAP NAS for hosting several of my git-annex repos, it would be very valuable to hear some details about your setup. My setup currently consists of a virtual machine hosted on the NAS with an ordinary linux, nothing special. Would love to have git-annex installed on the NAS operating system rather, which is unfortunately not that easy.
+
+
+Even in case you may not have been able to solve the issue described in this thread, can you maybe tell for example how you solved the issue of the QNAP operating system purging self-installed software? Or how you managed to let the pre-installed SSH daemon let other users than \"admin\" log-in?
+
+
+Probably we're the first ones ever trying this, so it would be very useful to collect all about QNAP NAS and the usage of git-annex on these devices.
+"""]]
diff --git a/doc/forum/Ssh_remote_on_NAS/comment_7_3c70f8b9ac2a77462c6f21929c02d9d3._comment b/doc/forum/Ssh_remote_on_NAS/comment_7_3c70f8b9ac2a77462c6f21929c02d9d3._comment
new file mode 100644
index 0000000000..6f6c687ca7
--- /dev/null
+++ b/doc/forum/Ssh_remote_on_NAS/comment_7_3c70f8b9ac2a77462c6f21929c02d9d3._comment
@@ -0,0 +1,11 @@
+[[!comment format=mdwn
+ username="Fabio"
+ avatar="http://cdn.libravatar.org/avatar/2bf56b78ae28b64d6131ba3115df238b"
+ subject="comment 7"
+ date="2017-01-30T22:26:46Z"
+ content="""
+Hello Torpidus,
+I've never managed to get SSH working properly on my NAS because of the issues you have reported and so I eventually gave up with using git-annex on it.
+But I'm back here every now and then to check how this wonderful software is going on and if anyone has made it work on my NAS :)
+
+"""]]
diff --git a/doc/forum/Stale_keys_and_.cache_files_left_in_.git__47__annex__47__objects.mdwn b/doc/forum/Stale_keys_and_.cache_files_left_in_.git__47__annex__47__objects.mdwn
new file mode 100644
index 0000000000..043a3f2db6
--- /dev/null
+++ b/doc/forum/Stale_keys_and_.cache_files_left_in_.git__47__annex__47__objects.mdwn
@@ -0,0 +1,39 @@
+I recently migrated some of my repositories from WORM to SHA1E, and
+noticed after finishing conversion and dropping all WORM keys that
+there are some stale files and directories left over in the
+.git/annex/objects directory. These seem to fall into two categories:
+
+1. There are some empty directories meant for WORM-keys. Strangely, I
+don’t believe the content of these keys has ever been present on this
+machine, and the corresponding .log files do not contain the local
+UUID. What might be the deal with those?
+
+Another set of empty WORM directories housed content I unannexed and
+checked into regular git on my other remote, and then pulled locally.
+A subsequent «dropunused» seems to have left the empty directories
+after dropping their content.
+
+I suppose the stale directories can be safely pruned away?
+
+One thing I noticed is that, while the terminal leaf in the objects
+directory is usually sticky (mode +t), the stale directories to
+content I unannexed are all non-sticky. Maybe this gives some
+indication where things got stuck? A few (though not all) of the other
+terminal directories are non-sticky, as well.
+
+2. There are some .map and .cache files leftover in
+.git/annex/objects. This is an indirect repository, but I briefly
+switched it to direct once. The stale files seem to belong to content
+with which I had some difficulties when «git annex add»’ing the files
+(I recall I had to add them multiple times before they were correctly
+indexed). I now examined these files again and noticed they were
+tracked via regular git. I «git rm»’ed them and readded them into the
+annex. Can the stale .cache and .map files be safely deleted?
+
+I noticed some of these keys have the format
+«WORM-s123-m-123456789--name», with an additional dash before the
+mtime. Has there been a format change, or does this indicate a
+problem?
+
+Best regards,
+Z.
diff --git a/doc/forum/Stale_keys_and_.cache_files_left_in_.git__47__annex__47__objects/comment_1_2aa80b317863a99e676a375d907d0e84._comment b/doc/forum/Stale_keys_and_.cache_files_left_in_.git__47__annex__47__objects/comment_1_2aa80b317863a99e676a375d907d0e84._comment
new file mode 100644
index 0000000000..34cbb011a3
--- /dev/null
+++ b/doc/forum/Stale_keys_and_.cache_files_left_in_.git__47__annex__47__objects/comment_1_2aa80b317863a99e676a375d907d0e84._comment
@@ -0,0 +1,13 @@
+[[!comment format=mdwn
+ username="http://joeyh.name/"
+ ip="209.250.56.54"
+ subject="comment 1"
+ date="2014-10-02T16:23:35Z"
+ content="""
+I cannot see a way that eg, `dropunused` could leave empty object directories. The few functions that remove content from a repository call `cleanObjectLoc` to prune the empty directories that result. Of course, the empty directories can be safely removed.
+
+IIRC older versions of direct mode might have made stale .cache and .map before. They can be deleted if you're not using direct mode.
+
+The only way I can see that a dash could come before the 'm' and the mtime is if the mtime were negative. I don't know how that would happen, but the code has certainly never put a dash there, and 
+WORM-s123-m-123456789--name is not a valid git-annex key; git-annex cannot parse it.
+"""]]
diff --git a/doc/forum/Standard_groups__47__preferred_contents.mdwn b/doc/forum/Standard_groups__47__preferred_contents.mdwn
new file mode 100644
index 0000000000..89e2b848c7
--- /dev/null
+++ b/doc/forum/Standard_groups__47__preferred_contents.mdwn
@@ -0,0 +1,14 @@
+Sorry that I am not stop asking questions - I am trying to wrap my head around annex still - and I do not fully understand the standard groups/preferred contents yet.
+
+
+First, basic: Is the "preferred content group" an attribute of a repository or of a remote? E.g., when I use assistant, I can set this property for a *remote* repository. Will this property be synced to all other clients then or is it only valid for this particular client?
+
+Second, I think the standard groups do still not cover what I want and I wonder if this could be done with preferred contents somehow? What I want:
+
+* Generally only the structure should be synced, no contents (i.e. "manual" is the closest)
+
+* When I (manually) do "git annex get" on a file this file should be synced whenever it changes locally *or* remotely (until I drop it again). Currently it seems that when it changes remotely, it is "dropped" again so I need to manually use "git annex get" again
+
+* When I (manually) do "git annex get" on a directory, it should be (recursively) synced whenever it or any file changes locally or remotely (until I drop it again). Currently nothing seems to sync at all in this case (except the meta data)
+
+Thanks!
diff --git a/doc/forum/Standard_groups__47__preferred_contents/comment_1_143aa3095d7222c869c36a5039282e35._comment b/doc/forum/Standard_groups__47__preferred_contents/comment_1_143aa3095d7222c869c36a5039282e35._comment
new file mode 100644
index 0000000000..343a049e7a
--- /dev/null
+++ b/doc/forum/Standard_groups__47__preferred_contents/comment_1_143aa3095d7222c869c36a5039282e35._comment
@@ -0,0 +1,16 @@
+[[!comment format=mdwn
+ username="sts"
+ ip="134.147.73.126"
+ subject="comment 1"
+ date="2014-07-18T21:16:50Z"
+ content="""
+> First, basic: Is the \"preferred content group\" an attribute of a repository or of a remote? E.g., when I use assistant, I can set this property for a remote repository. Will this property be synced to all other clients then or is it only valid for this particular client?
+
+Yeah, it syncs the content group between the remotes, so you have to set it only in one repository and the information gets synced across the network.
+
+> Second, I think the standard groups do still not cover what I want and I wonder if this could be done with preferred contents somehow? What I want:
+> Generally only the structure should be synced, no contents (i.e. \"manual\" is the closest)
+
+Well, I guess you have to manually create a git repo (but no annex repo!) on your remote. Once you add the remote the assistant should tell you that he can only sync the metadata, but no contents.
+
+"""]]
diff --git a/doc/forum/Standard_groups__47__preferred_contents/comment_2_65bef012eb113986a2e9b02470a6cbd1._comment b/doc/forum/Standard_groups__47__preferred_contents/comment_2_65bef012eb113986a2e9b02470a6cbd1._comment
new file mode 100644
index 0000000000..df204b0582
--- /dev/null
+++ b/doc/forum/Standard_groups__47__preferred_contents/comment_2_65bef012eb113986a2e9b02470a6cbd1._comment
@@ -0,0 +1,15 @@
+[[!comment format=mdwn
+ username="https://www.google.com/accounts/o8/id?id=AItOawmMLeU-zCzx2mc5pL2XT8a1UNkQwHAHjg8"
+ nickname="daniele"
+ subject="comment 2"
+ date="2014-07-19T00:38:58Z"
+ content="""
+    Generally only the structure should be synced, no contents (i.e. \"manual\" is the closest)
+    Currently it seems that when it changes remotely, it is \"dropped\" again so I need to manually use \"git annex get\" again
+
+Maybe this could work? 
+
+(in preferred content) exclude=\"*\" and present
+
+So that what is present is preferred and not dropped, everything else is not downloaded unless the user explicitly asks to get it.
+"""]]
diff --git a/doc/forum/Standard_groups__47__preferred_contents/comment_3_202a7e2820e0adf079ccd14a7993ad25._comment b/doc/forum/Standard_groups__47__preferred_contents/comment_3_202a7e2820e0adf079ccd14a7993ad25._comment
new file mode 100644
index 0000000000..523aa742a5
--- /dev/null
+++ b/doc/forum/Standard_groups__47__preferred_contents/comment_3_202a7e2820e0adf079ccd14a7993ad25._comment
@@ -0,0 +1,21 @@
+[[!comment format=mdwn
+ username="divB"
+ ip="204.17.143.10"
+ subject="comment 3"
+ date="2014-07-22T00:11:34Z"
+ content="""
+Unfortunately the same :(
+
+Testcase:
+1.) Create a file 'file' on the server, git annex add/sync etc.
+2.) On the client: git annex wanted here 'exclude=\"*\" and present'
+3.) On the client: git annex get file . The file is now present on the client
+4.) Change the file on the server, git annex sync
+5.) git annex sync --content on the client
+
+Result: File is dropped again on client
+
+
+
+
+"""]]
diff --git a/doc/forum/Standard_groups__47__preferred_contents/comment_4_e0faf9ebd3162e0de860eba0fd28c67c._comment b/doc/forum/Standard_groups__47__preferred_contents/comment_4_e0faf9ebd3162e0de860eba0fd28c67c._comment
new file mode 100644
index 0000000000..50acdbc4a7
--- /dev/null
+++ b/doc/forum/Standard_groups__47__preferred_contents/comment_4_e0faf9ebd3162e0de860eba0fd28c67c._comment
@@ -0,0 +1,15 @@
+[[!comment format=mdwn
+ username="divB"
+ ip="204.17.143.10"
+ subject="comment 4"
+ date="2014-07-22T00:24:46Z"
+ content="""
+... and for the directories the same:
+
+1.) Create a (sub-)directory 'subdir' with files and sync everything
+2.) On the client: git annex get subdir . The subdirectory is now present, all files under it downloaded.
+3.) On the server create a new file in 'subdir' and git annex add; git annex sync --content
+4.) git annex sync --content on the client
+
+Result: Content of the files is not synced to client 
+"""]]
diff --git a/doc/forum/Standard_groups__47__preferred_contents/comment_5_eb47696244931173bddcbeb8d5f78637._comment b/doc/forum/Standard_groups__47__preferred_contents/comment_5_eb47696244931173bddcbeb8d5f78637._comment
new file mode 100644
index 0000000000..fa5c47ee2e
--- /dev/null
+++ b/doc/forum/Standard_groups__47__preferred_contents/comment_5_eb47696244931173bddcbeb8d5f78637._comment
@@ -0,0 +1,13 @@
+[[!comment format=mdwn
+ username="https://www.google.com/accounts/o8/id?id=AItOawmMLeU-zCzx2mc5pL2XT8a1UNkQwHAHjg8"
+ nickname="daniele"
+ subject="comment 5"
+ date="2014-07-22T00:29:20Z"
+ content="""
+wait, is the group working together with the preferred content?
+
+Usually you set a repository in a group and then you tell annex that particular group has this preferred content expression (which by default is set to 'standard'). So you could add the repo to 'client' group then set the group 'client' to that preferred content expression. If you only want that particular client to have that expression you can play around with \"groupwanted\", or even define your own group I guess.
+
+Use \"git annex vicfg\" to quickly check both \"group\" and \"wanted\" settings together.
+
+"""]]
diff --git a/doc/forum/Standard_groups__47__preferred_contents/comment_6_c70c9fa97bce8e4eb9b3880d8f843aef._comment b/doc/forum/Standard_groups__47__preferred_contents/comment_6_c70c9fa97bce8e4eb9b3880d8f843aef._comment
new file mode 100644
index 0000000000..435d7f7d2e
--- /dev/null
+++ b/doc/forum/Standard_groups__47__preferred_contents/comment_6_c70c9fa97bce8e4eb9b3880d8f843aef._comment
@@ -0,0 +1,25 @@
+[[!comment format=mdwn
+ username="divB"
+ ip="204.17.143.10"
+ subject="comment 6"
+ date="2014-07-23T05:20:47Z"
+ content="""
+Hi daniele,
+
+I am not completely sure what you mean but I as far as I understand as soon as I set \"wanted\" to anything other than \"default\" the standard groups do not apply at all.
+I can't use vicfg because I use windows and this command hangs ... probably because there is no vi.
+
+However, I can still check the expressions:
+
+    $ git annex group here
+    manual
+    $ git annex wanted here
+    exclude=\"*\" and present
+    $
+
+Have you tried this? Does it work in your case?
+Because the standard preferred contents also contains \"present\" so I assume it would behave the same.
+Is maybe \"present\" broken or the behavior different than expected?
+
+
+"""]]
diff --git a/doc/forum/Standard_groups__47__preferred_contents/comment_7_413593ea0663874196305c9eb602e7c4._comment b/doc/forum/Standard_groups__47__preferred_contents/comment_7_413593ea0663874196305c9eb602e7c4._comment
new file mode 100644
index 0000000000..32bfa6bc00
--- /dev/null
+++ b/doc/forum/Standard_groups__47__preferred_contents/comment_7_413593ea0663874196305c9eb602e7c4._comment
@@ -0,0 +1,7 @@
+[[!comment format=mdwn
+ username="joey"
+ subject="""comment 7"""
+ date="2014-10-28T20:23:55Z"
+ content="""
+See [[bugs/present_files__47__directories_are_dropped_after_a_sync]]
+"""]]
diff --git a/doc/forum/Standards_usage_pattern__63__.mdwn b/doc/forum/Standards_usage_pattern__63__.mdwn
new file mode 100644
index 0000000000..280e909a5b
--- /dev/null
+++ b/doc/forum/Standards_usage_pattern__63__.mdwn
@@ -0,0 +1,8 @@
+Hi,
+
+I have few devices: 2x android, 2x windows, 2x linux and I would like to share at least one folder on all of them.
+None of the devices have public IPs and some regularly change location (work laptops). Because the XMPP support is missing on windows I couldn't find a way to synchronize the repositories (the keys) or did I miss something in the documentation?
+How can I archieve "seamless" synchronization when I start the webapp on the devices?
+
+Thank you,
+Karol
diff --git a/doc/forum/Standards_usage_pattern__63__/comment_1_1ccbfaefe3bbaf767e999cd1c61debfb._comment b/doc/forum/Standards_usage_pattern__63__/comment_1_1ccbfaefe3bbaf767e999cd1c61debfb._comment
new file mode 100644
index 0000000000..d96585f9b3
--- /dev/null
+++ b/doc/forum/Standards_usage_pattern__63__/comment_1_1ccbfaefe3bbaf767e999cd1c61debfb._comment
@@ -0,0 +1,9 @@
+[[!comment format=mdwn
+ username="https://www.google.com/accounts/o8/id?id=AItOawk9SYh6N-JUMkYkW4aOk55zC3Vr9KonDV4"
+ nickname="Florian"
+ subject="comment 1"
+ date="2014-11-19T22:01:19Z"
+ content="""
+I think you need at least one server that is reachable from every device and serves as a transfer repository. XMPP is only needed to inform peers about changes. Usual the assistant performs a startup check and detects changes that way. You can't sync content over XMPP. 
+All that is very afaik, I hope for corrections if wrong. 
+"""]]
diff --git a/doc/forum/Starting_assistant_from_CLI.mdwn b/doc/forum/Starting_assistant_from_CLI.mdwn
new file mode 100644
index 0000000000..8a4bc3d1bf
--- /dev/null
+++ b/doc/forum/Starting_assistant_from_CLI.mdwn
@@ -0,0 +1,9 @@
+I am unable to start the git-annex assistant/webapp.
+
+I use OpenBox as desktop manager and the assistant/webapp is not available through the menu.
+
+Trying to use the CLI, all my attempts fail with a message saying that it(?) is not a git repository!? Since the video show that on first start the assistant/webapp allows a choice of a directory and then creates it, I am not sure as to what git initialized directory does the assistant/webapp requires in this instance. And I also guess that means invoking the webapp from that directory rather than from the directory that contains the standalone git-annex.
+
+Any help would be appreciated as git-annex really seems to be the app I am looking for.  :)
+
+Thanks
diff --git a/doc/forum/Starting_assistant_from_CLI/comment_1_afd51ddb0f1bb3cac528e1d96829ef83._comment b/doc/forum/Starting_assistant_from_CLI/comment_1_afd51ddb0f1bb3cac528e1d96829ef83._comment
new file mode 100644
index 0000000000..05c7a12d40
--- /dev/null
+++ b/doc/forum/Starting_assistant_from_CLI/comment_1_afd51ddb0f1bb3cac528e1d96829ef83._comment
@@ -0,0 +1,12 @@
+[[!comment format=mdwn
+ username="https://me.yahoo.com/a/FHnTlSBo1eCGJRwueeKeB6.RCaPbGMPr5jxx8A--#ce0d8"
+ nickname="Hamza"
+ subject="comment 1"
+ date="2014-04-14T15:35:17Z"
+ content="""
+running,
+
+    git annex webapp
+
+should launch the web app.
+"""]]
diff --git a/doc/forum/Starting_assistant_from_CLI/comment_2_76c34c00cf2065809b15a594023a688b._comment b/doc/forum/Starting_assistant_from_CLI/comment_2_76c34c00cf2065809b15a594023a688b._comment
new file mode 100644
index 0000000000..d0ce4b28fe
--- /dev/null
+++ b/doc/forum/Starting_assistant_from_CLI/comment_2_76c34c00cf2065809b15a594023a688b._comment
@@ -0,0 +1,11 @@
+[[!comment format=mdwn
+ username="https://www.google.com/accounts/o8/id?id=AItOawlqL3QbH6hrzf4XT-OW5IcMj6zSrMWl2dg"
+ nickname="Michel"
+ subject="comment 2"
+ date="2014-04-14T22:45:23Z"
+ content="""
+Unfortunately, no, I tried this and many other variants, (I tried again just now, to be sure, and I still get the \"git-annex: Not in a git repository.\" message.
+
+Thanks for trying to help. It is very much appreciated.
+
+"""]]
diff --git a/doc/forum/Starting_assistant_from_CLI/comment_3_f7826867f78b1adbfc2dad2fad4d6720._comment b/doc/forum/Starting_assistant_from_CLI/comment_3_f7826867f78b1adbfc2dad2fad4d6720._comment
new file mode 100644
index 0000000000..fbe70c4038
--- /dev/null
+++ b/doc/forum/Starting_assistant_from_CLI/comment_3_f7826867f78b1adbfc2dad2fad4d6720._comment
@@ -0,0 +1,17 @@
+[[!comment format=mdwn
+ username="http://joeyh.name/"
+ ip="209.250.56.191"
+ subject="comment 3"
+ date="2014-04-17T20:48:56Z"
+ content="""
+Yes, the webapp is supposed to be able to be started from anywhere. 
+
+1. If you make a git annex repository and run `git annex webapp` from inside it, it'll start up the webapp on that repository. 
+2. If `~/.config/git-annex/autostart` lists some git repositories, the webapp will start up in the first listed one.
+3. Otherwise, the webapp will walk you through making a new repository.
+
+Most likely problem then is #2. If you used the webapp once, say in ~/annex and then deleted ~/annex/.git directory, it would try to start up in ~/annex, but it's no longer a git repository so it cannot start. This was a bug, so I've fixed it. 
+You could work around that problem by deleting `~/.config/git-annex/autostart` too.
+
+If it's some other problem, you can work around it by going the #1 route and making a git repository by hand (\"git init annex; cd annex; git annex init\") and running the webapp in there.
+"""]]
diff --git a/doc/forum/Starting_assistant_from_CLI/comment_4_fa7055a232a1dcb743db47308f7acf0b._comment b/doc/forum/Starting_assistant_from_CLI/comment_4_fa7055a232a1dcb743db47308f7acf0b._comment
new file mode 100644
index 0000000000..4eea29c5fd
--- /dev/null
+++ b/doc/forum/Starting_assistant_from_CLI/comment_4_fa7055a232a1dcb743db47308f7acf0b._comment
@@ -0,0 +1,14 @@
+[[!comment format=mdwn
+ username="https://www.google.com/accounts/o8/id?id=AItOawlqL3QbH6hrzf4XT-OW5IcMj6zSrMWl2dg"
+ nickname="Michel"
+ subject="comment 4"
+ date="2014-04-18T00:20:47Z"
+ content="""
+Thank you.
+
+The problem was indeed with having a file containing a non-existent ~/.config/git-annex/autostart.
+
+I can now continue my evaluation of what looks like a very fine and useful piece of software.
+
+Thanks again for the help.
+"""]]
diff --git a/doc/forum/Storing_copies_on_LTO_tapes__63__.mdwn b/doc/forum/Storing_copies_on_LTO_tapes__63__.mdwn
new file mode 100644
index 0000000000..175bc2b4ce
--- /dev/null
+++ b/doc/forum/Storing_copies_on_LTO_tapes__63__.mdwn
@@ -0,0 +1,13 @@
+I'm looking for a way to count/track annex data copied to LTO tapes.
+
+In my use case I'm splitting many terabytes of data to separate repositories on several hard drives, backing them up on tapes, and keeping track of all this with remotes in a central repository (with lots of dropped content). All good with the HDs, but I'm at a loss about how to handle the copies on tapes.
+
+Three alternatives I've come up with so far:
+
+1. Simply tar the repositories from the HDs to the tapes. Problem: no way to notify git annex of the existence of these manual copies. Or is there?
+2. Remote (special or normal) on LTFS (linear posix compatible file system on top of tape). Problems:
+    1. `git annex get`ing a dropped directory from there would cause files to be accessed in random order, right? Or is the retrieve guaranteed to happen in the same order as the files in the directory were written by `git annex copy`?
+    2. LTFS has a big block size (512KB) => wasted space when lots of small files. (Not a major problem, though.)
+3. Write a special (read-only) remote hook for `tar`. Problem: `git annex get` would make one hook *RETRIEVE* request per file, leading to random access again, while the only effective way would be to get a list of all files to be retrieved, and then returning them in the order they turn up from the tar package (or even ingest the whole tar file to .git/annex/).
+
+Thoughts?
diff --git a/doc/forum/Storing_copies_on_LTO_tapes__63__/comment_1_9ae9ca3983f7d4209fd1de65d982731d._comment b/doc/forum/Storing_copies_on_LTO_tapes__63__/comment_1_9ae9ca3983f7d4209fd1de65d982731d._comment
new file mode 100644
index 0000000000..805042e0c5
--- /dev/null
+++ b/doc/forum/Storing_copies_on_LTO_tapes__63__/comment_1_9ae9ca3983f7d4209fd1de65d982731d._comment
@@ -0,0 +1,31 @@
+[[!comment format=mdwn
+ username="joey"
+ subject="""comment 1"""
+ date="2015-07-02T21:00:00Z"
+ content="""
+All your solutions are reasonable, but you're right that none of them
+avoid random-access. And git-annex is generally not built in a way that
+makes it easy to avoid random-access.
+
+But, I think it's tractable with a special remote.
+
+Consider a special remote that has two retrieval modes.
+In one mode, it always fails to retrieve keys, but keeps a list. In the
+second mode, it starts up by going through its list, retreives everything
+in order to a temporary directory, and then when asked to retrieve a key,
+just moves it from the temp dir into place. This is somewhat similar to how
+Amazon Glacier works, and like git-annex's glacier support, it would result
+in the first `git-annex get` failing, and a second `git annex get` being
+needed to finish the retrival.
+
+That could be improved.. Make the special remote fail to retrive keys,
+and keep a list. On shutdown, it then sorts the list, retrieves the keys
+in order, and runs `git annex setkey` to move the content into the annex.
+Still a little bit weird, because `git annex get` would seem to fail
+and then pause at the end for a long time, after which the files would
+actually end up being present. 
+
+(Also, I er, removed `git annex setkey` in
+2011, because it didn't seem very useful, but this is in fact a use case
+for it, so I've added it back now.)
+"""]]
diff --git a/doc/forum/Storing_copies_on_LTO_tapes__63__/comment_2_6d16d00c7ef8d846e370e1b298a7bc7a._comment b/doc/forum/Storing_copies_on_LTO_tapes__63__/comment_2_6d16d00c7ef8d846e370e1b298a7bc7a._comment
new file mode 100644
index 0000000000..ed00112b17
--- /dev/null
+++ b/doc/forum/Storing_copies_on_LTO_tapes__63__/comment_2_6d16d00c7ef8d846e370e1b298a7bc7a._comment
@@ -0,0 +1,18 @@
+[[!comment format=mdwn
+ username="joey"
+ subject="""another approach"""
+ date="2015-07-02T22:25:25Z"
+ content="""
+You could make a special remote that streams the whole tar file from the
+tape, and uses `git annex setkey` to add each file from the tarball to the
+annex.
+
+Done this way, the first file that `git annex get` processed would actually
+cause *every* file to be gotten from the tape. As it continued on to
+subsequent files, the `git annex get` would see their content was already
+present and skip them.
+
+Of course, the downside is it works on a whole tape at a time, so if you
+don't want to load the whole tape into the filesystem, you wouldn't want to
+use this approach.
+"""]]
diff --git a/doc/forum/Storing_git_repos_in_git-annex.mdwn b/doc/forum/Storing_git_repos_in_git-annex.mdwn
new file mode 100644
index 0000000000..964040cf0a
--- /dev/null
+++ b/doc/forum/Storing_git_repos_in_git-annex.mdwn
@@ -0,0 +1,27 @@
+I tried to use the [old thread](http://git-annex.branchable.com/forum/Git_repos_in_git_annex__63__/?updated#comment-9fca5cf31ccfd3d78c78cb65f7672340) for this, but threads don't get bumped when they are updated, so I guess I'll have to start a new one...
+
+Here is the relevant [bug report](http://git-annex.branchable.com/bugs/Can__39__t_add_a_git_repo_to_git_annex:___34__Invalid_path_repo__47__.git__47__X__34___for_many_X).
+
+Here's my scenario: I have been using Dropbox for several years.  I want to move to git-annex.
+
+For some time I used Bazaar repos inside Dropbox.  A couple of times I accidentally modified files on one computer before another finished syncing, and the repo became mildly corrupted, but I recovered it okay.
+
+Recently I switched from bzr to git, and so far, so good.  I'm careful not to make any changes to my files before Dropbox is finished syncing, so everything works fine.
+
+Now I'm testing git-annex as a replacement for Dropbox again.  But when I add a directory containing a git repo to an annex, git-annex ignores the .git directory, so the repo isn't synced.
+
+I read this forum thread, and I read the bug report, but there don't appear to be any real solutions.
+
+This seems like what should be a fairly common use case: a user has a directory of simple text files, like shell scripts or org-mode files.  He uses git to version-control them, and he uses Dropbox to sync the files and the git repo automatically between computers.  Now the user wants to use git-annex instead, but if he does this, it will mean having to choose between:
+
+* No longer using git to version-control the files
+* Only using git on a single system, which means that he can't make commits or do any other git operations when using other systems
+* Keeping separate git repos on each system, and making the same commits manually on each one...which would be a mess
+* Keeping separate git repos on each system, and using a script to make commits automatically--which would mean commits wouldn't be any more useful than a simple timed backup script
+
+I guess I could just use git instead of Dropbox or git-annex, but then I must choose between either:
+
+* Only being able to sync repos when both systems are online
+* Having to store repos in plaintext on remote servers
+
+This is one of my primary use cases for Dropbox/git-annex.  I'd really appreciate any help in figuring this out.  Thanks.
diff --git a/doc/forum/Storing_git_repos_in_git-annex/comment_1_ac7b52c0b0f75d79760ffe6a9b5c8759._comment b/doc/forum/Storing_git_repos_in_git-annex/comment_1_ac7b52c0b0f75d79760ffe6a9b5c8759._comment
new file mode 100644
index 0000000000..cb5b39ffc2
--- /dev/null
+++ b/doc/forum/Storing_git_repos_in_git-annex/comment_1_ac7b52c0b0f75d79760ffe6a9b5c8759._comment
@@ -0,0 +1,20 @@
+[[!comment format=mdwn
+ username="http://joeyh.name/"
+ ip="108.236.230.124"
+ subject="comment 1"
+ date="2014-05-16T17:29:15Z"
+ content="""
+Why not just use git the way it's designed: Make a git repository on one computer. Clone to another computer. Commit changes on either whenever you like, and pull/push whenever you like. No need for any dropbox to help git sync, it can do so perfectly well on its own.
+
+If you have a lot of repositories, see 
+
+I continue to be unimpressed and unmotived by this use case. Posting more forum threads with the same lame reasons won't help. But at least you did say:
+
+> For some time I used Bazaar repos inside Dropbox.  A couple of times I accidentally modified files on one computer before another finished syncing
+, and the repo became mildly corrupted, but I recovered it okay.
+
+> Recently I switched to git, and so far, so good.  I'm careful not to make any changes to my files before Dropbox is finished syncing, so everythin
+g works fine.
+
+So, you used dropbox, and corrupted data.  But that was ok. And now you're very careful to avoid doing things that will corrupt data. But that's ok. Yeah, that seems like a really good decision, given all the excellent possibilities out there.``
+"""]]
diff --git a/doc/forum/Storing_git_repos_in_git-annex/comment_3_3bec1f02ff1a61791e3cbb428c7acb4c._comment b/doc/forum/Storing_git_repos_in_git-annex/comment_3_3bec1f02ff1a61791e3cbb428c7acb4c._comment
new file mode 100644
index 0000000000..69da79a89c
--- /dev/null
+++ b/doc/forum/Storing_git_repos_in_git-annex/comment_3_3bec1f02ff1a61791e3cbb428c7acb4c._comment
@@ -0,0 +1,12 @@
+[[!comment format=mdwn
+ username="https://www.google.com/accounts/o8/id?id=AItOawmNu4V5fvpLlBhaCUfXXOB0MI5NXwh8SkU"
+ nickname="Adam"
+ subject="comment 3"
+ date="2014-05-17T00:23:02Z"
+ content="""
+Well, like Pere, I am rather surprised by your response, Joey.  I've been an enthusiastic supporter and follower of git-annex for a long time now, and have been looking forward to using it.  I didn't expect a snarky reply...and you said yourself that it was snarky.  Maybe you had a bad day or something?  :)
+
+I don't understand why this request or idea is so controversial to you.  I have several computers, and I use git to store basic text files like shell scripts and config files--a very common situation.  I use Dropbox to sync them automatically between computers.  I could use just git, and push/pull manually, but the whole point of having computers is for them to do things for me, automatically.  The purpose of the git-annex assistant, as I understand it, is to do exactly that.  Doing the push/pull manually would be a step backwards.  And giving up manual control of the git repository, losing the ability to track changes to my files, would be a huge step backwards.
+
+This seems like it must be a very common use case among Linux users, especially ones who use the shell, git, etc.  If git-annex isn't good for this situation, what is?  Surely there's a better way than opening a shell and running \"git pull\" every time I walk from one computer to the other.
+"""]]
diff --git a/doc/forum/Storing_git_repos_in_git-annex/comment_4_76ddbd27cc2f3785bb5aaebb0bb6e087._comment b/doc/forum/Storing_git_repos_in_git-annex/comment_4_76ddbd27cc2f3785bb5aaebb0bb6e087._comment
new file mode 100644
index 0000000000..9d0b024c1e
--- /dev/null
+++ b/doc/forum/Storing_git_repos_in_git-annex/comment_4_76ddbd27cc2f3785bb5aaebb0bb6e087._comment
@@ -0,0 +1,14 @@
+[[!comment format=mdwn
+ username="https://www.google.com/accounts/o8/id?id=AItOawmNu4V5fvpLlBhaCUfXXOB0MI5NXwh8SkU"
+ nickname="Adam"
+ subject="comment 4"
+ date="2014-05-17T00:41:04Z"
+ content="""
+I should have also mentioned, the network topology is an issue that git-annex helps solve.  When my computers are on the same LAN, they could obviously sync directly with each other using git.  But when they are not on the same network, or when one of them is not online, a transfer repo is needed.  Dropbox and git-annex make this simple by handling it for me.  
+
+But if I did all my syncing with git manually, it could end up being quite a mess.  If I took my laptop with me and left the house without syncing it first, I'd have to sync with my server on the Internet.  But if I forgot to push from my desktop computer before I left, the server would be out-of-date, and I'd be stuck.
+
+Dropbox handles this for me by automatically syncing as soon as I make changes.  git-annex does the same thing, but doesn't work with nested git repos.  But giving up the nested git repo would mean giving up version control of my files.  As a developer, you understand why that's not an option!  :)
+
+Again, this seems like a very natural problem to run into, and I don't understand why it is controversial to want git-annex to handle this the way Dropbox or other sync software (e.g. SpiderOak, Wuala) can.
+"""]]
diff --git a/doc/forum/Storing_git_repos_in_git-annex/comment_5_f9520cbc6669622aa342acad35581943._comment b/doc/forum/Storing_git_repos_in_git-annex/comment_5_f9520cbc6669622aa342acad35581943._comment
new file mode 100644
index 0000000000..95bf6a0d4d
--- /dev/null
+++ b/doc/forum/Storing_git_repos_in_git-annex/comment_5_f9520cbc6669622aa342acad35581943._comment
@@ -0,0 +1,14 @@
+[[!comment format=mdwn
+ username="https://me.yahoo.com/a/FHnTlSBo1eCGJRwueeKeB6.RCaPbGMPr5jxx8A--#ce0d8"
+ nickname="Hamza"
+ subject="comment 5"
+ date="2014-05-18T12:55:47Z"
+ content="""
+My, 2 cents. I used to do the same keep my git repos in Dropbox. I do not push/pull to dropbox I just keep them in there so they sync between my machines. My use case is that I am in the middle of a change and I need to switch computers (home/office) I can continue where I left of on another machine otherwise you have to make wierd commits with broken code. So IMHO it is perfectly reasonable for a single person to keep his git repo in git-annex assuming I don't use it as a central repo where other people pull push.
+
+Plus one giant problem with git annex is that there is no way to revert a file in direct repo. It only sync and reverting to old files take a lot of effort (convert to indirect checkout commit save file to a temp location convert back to direct mode place the file back its place.) with a git repo in annex I can just use git to revert files.
+
+On the other hand, git likes to create a lot of small files and sync them can take a lot of time. annex copies on file at a time.
+
+That said, at one point I did experiment putting a git repo in annex for testing (using --git-dir option) it did work so it should not be that big of a deal to add the support.
+"""]]
diff --git a/doc/forum/Storing_git_repos_in_git-annex/comment_9_d5676400e7148b7d3408f2bdb3d54b7d._comment b/doc/forum/Storing_git_repos_in_git-annex/comment_9_d5676400e7148b7d3408f2bdb3d54b7d._comment
new file mode 100644
index 0000000000..23610fa715
--- /dev/null
+++ b/doc/forum/Storing_git_repos_in_git-annex/comment_9_d5676400e7148b7d3408f2bdb3d54b7d._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="https://www.google.com/accounts/o8/id?id=AItOawnaYy6kTuKAHmsa4BtGls2oqa42Jo2w2v0"
+ nickname="Pere"
+ subject="So, you are more clever than your users, isn't?"
+ date="2014-05-16T18:13:34Z"
+ content="""
+Joey, your comment is really disappointing. I also use Dropbox as a central git repo and it works great for me. Maybe you find this stupid, but there is a lot of people with good reasons to do so. I agree that this is your software project, thus you have the right not to support this use case, but you don't have the right to treat the users that want this feature as you did in your comment. Next time you do a new crowdfunding campaign, I not sure I'll be between the donors again.
+"""]]
diff --git a/doc/forum/Storing_uncontrolled_files_in_an_annex.mdwn b/doc/forum/Storing_uncontrolled_files_in_an_annex.mdwn
new file mode 100644
index 0000000000..e2111f0b95
--- /dev/null
+++ b/doc/forum/Storing_uncontrolled_files_in_an_annex.mdwn
@@ -0,0 +1,3 @@
+Is there a way to store a file in an annex repo without the assistant trying to commit it? My particular issue is that git annex watch tries to add my .thunderbird folder, and I don't want it in annex. Is that a supported use case?
+
+Also, for my better understanding, does watch keep track of what files it saw last time and only look for new things on startup? Or does it try to commit anything that's not already in git (whether symlink or not)?
diff --git a/doc/forum/Storing_uncontrolled_files_in_an_annex/comment_1_175645a90be0c79221c129308adf643e._comment b/doc/forum/Storing_uncontrolled_files_in_an_annex/comment_1_175645a90be0c79221c129308adf643e._comment
new file mode 100644
index 0000000000..2932efa399
--- /dev/null
+++ b/doc/forum/Storing_uncontrolled_files_in_an_annex/comment_1_175645a90be0c79221c129308adf643e._comment
@@ -0,0 +1,27 @@
+[[!comment format=mdwn
+ username="http://edheil.wordpress.com/"
+ ip="173.162.44.162"
+ subject="comment 1"
+ date="2012-12-16T00:10:39Z"
+ content="""
+wow, are you git annexing your whole home directory?  that sounds extreme and problematic, since everything in git annex becomes read-only!!
+
+This how to make git annex ignore something though: make git ignore it in the first place.
+
+Stop the git assistant if it's running.
+
+create a file called \".gitignore\" in the root of your annex directory; add a line to that file for everything you want git (and therefore git annex) to ignore, e.g. \".thunderbird\"
+
+
+e.g.:
+
+    echo '.thunderbird' > '.gitignore'
+
+check that file into git:
+
+
+    git add .gitignore
+    git commit -m 'check in .gitignore'
+
+Now you should be able to start up git-annex again and have it ignore .thunderbird, or any other path you put in .gitignore
+"""]]
diff --git a/doc/forum/Storing_uncontrolled_files_in_an_annex/comment_2_d29f214eadfe3bfd098bbc3bcf07129a._comment b/doc/forum/Storing_uncontrolled_files_in_an_annex/comment_2_d29f214eadfe3bfd098bbc3bcf07129a._comment
new file mode 100644
index 0000000000..395f08d245
--- /dev/null
+++ b/doc/forum/Storing_uncontrolled_files_in_an_annex/comment_2_d29f214eadfe3bfd098bbc3bcf07129a._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="andy"
+ ip="108.92.50.62"
+ subject="Thanks!"
+ date="2012-12-16T00:31:09Z"
+ content="""
+That does what I needed. I'd forgotten about that. :)
+"""]]
diff --git a/doc/forum/Storing_uncontrolled_files_in_an_annex/comment_3_286b502e7906cca50e9e747db735bc88._comment b/doc/forum/Storing_uncontrolled_files_in_an_annex/comment_3_286b502e7906cca50e9e747db735bc88._comment
new file mode 100644
index 0000000000..f23854cbf3
--- /dev/null
+++ b/doc/forum/Storing_uncontrolled_files_in_an_annex/comment_3_286b502e7906cca50e9e747db735bc88._comment
@@ -0,0 +1,10 @@
+[[!comment format=mdwn
+ username="http://joeyh.name/"
+ ip="4.153.8.117"
+ subject="comment 3"
+ date="2012-12-17T16:32:31Z"
+ content="""
+AFAIK, the assistant does *not* currently honor .gitgnore completely. There are parts of it that do, and places where running git commands to check .gitignore would be very expensive, and so I've TODOed it for later, as I'll need to write my own .gitignore parser. 
+
+So, use caution..
+"""]]
diff --git a/doc/forum/Structure_needs_cleaning_git-annex__58___fd__58__15__58___hGetLine__58___end_of_file.mdwn b/doc/forum/Structure_needs_cleaning_git-annex__58___fd__58__15__58___hGetLine__58___end_of_file.mdwn
new file mode 100644
index 0000000000..0621f303b4
--- /dev/null
+++ b/doc/forum/Structure_needs_cleaning_git-annex__58___fd__58__15__58___hGetLine__58___end_of_file.mdwn
@@ -0,0 +1,24 @@
+I get this error after a hard reboot of the device when running git annex sync
+
+
+    fatal: Cannot open '/media/usbplatte/Podcasts/.git/annex/journal/e2f_ae4_SHA256E-s8388114--55d0a8881d06e24584ad9a94a6c5a369ea301309e0626497ea8c67bc65c3d0b2.mp3.log.web': Die Struktur muss bereinigt werden
+
+    git-annex: fd:15: hGetLine: end of file
+    fatal: Cannot open '/media/usbplatte/Podcasts/.git/annex/journal/e2f_ae4_SHA256E-s8388114--55d0a8881d06e24584ad9a94a6c5a369ea301309e0626497ea8c67bc65c3d0b2.mp3.log.web': Die Struktur muss bereinigt werden
+
+    git-annex: fd:15: hGetLine: end of file
+
+I ran 
+
+    git annex fsck
+
+and
+
+    git annex repair
+
+Both commands finished with Status Ok.
+
+How can I save the repo?
+
+TIA
+juh
diff --git a/doc/forum/Structure_needs_cleaning_git-annex__58___fd__58__15__58___hGetLine__58___end_of_file/comment_1_7bcc0111015985bde398a2a1a02b28ba._comment b/doc/forum/Structure_needs_cleaning_git-annex__58___fd__58__15__58___hGetLine__58___end_of_file/comment_1_7bcc0111015985bde398a2a1a02b28ba._comment
new file mode 100644
index 0000000000..02c18f876f
--- /dev/null
+++ b/doc/forum/Structure_needs_cleaning_git-annex__58___fd__58__15__58___hGetLine__58___end_of_file/comment_1_7bcc0111015985bde398a2a1a02b28ba._comment
@@ -0,0 +1,7 @@
+[[!comment format=mdwn
+ username="juh"
+ subject="SOLVED"
+ date="2016-06-06T15:45:12Z"
+ content="""
+I solved the problem with fsck the disk.
+"""]]
diff --git a/doc/forum/Structure_needs_cleaning_git-annex__58___fd__58__15__58___hGetLine__58___end_of_file/comment_2_b1dfc6168bb7127312d3e06cc0939b8d._comment b/doc/forum/Structure_needs_cleaning_git-annex__58___fd__58__15__58___hGetLine__58___end_of_file/comment_2_b1dfc6168bb7127312d3e06cc0939b8d._comment
new file mode 100644
index 0000000000..9fbac07dd9
--- /dev/null
+++ b/doc/forum/Structure_needs_cleaning_git-annex__58___fd__58__15__58___hGetLine__58___end_of_file/comment_2_b1dfc6168bb7127312d3e06cc0939b8d._comment
@@ -0,0 +1,11 @@
+[[!comment format=mdwn
+ username="joey"
+ subject="""comment 2"""
+ date="2016-06-09T19:19:07Z"
+ content="""
+	LANG=de_DE.UTF-8 errno -s 'Die Struktur muss bereinigt werden'
+	EUCLEAN 117 Die Struktur muss bereinigt werden
+
+This seems to be a failure mode of XFS filesystems, where when it needs to
+be fscked, it throws EUCLEAN.
+"""]]
diff --git a/doc/forum/Stuck_in_a_bad_state.mdwn b/doc/forum/Stuck_in_a_bad_state.mdwn
new file mode 100644
index 0000000000..9b83f26888
--- /dev/null
+++ b/doc/forum/Stuck_in_a_bad_state.mdwn
@@ -0,0 +1,10 @@
+So I managed to muck stuff up pretty bad with git-annex assistant and wondering if there is a reasonable way to fix the situation without blowing everything away and starting over at this point.  So my setup is kind of laid out like this:
+I have the main annex folder in my home folder at this location: /home//annex  I have a full backup on a separate drive on the same system which is mounted to /mnt/store and there is a sym link to it in my home folder so ~/store and annex folder goes to ~/store/annex on that drive.  I sync with one other computer that comes and goes online on the local folder (I don't use a transfer repository).  Then I have an rsync.net account that is also set to full backup, I originally setup annex to use this with a self generated key that it would keep in git and it had synced and everything seemed good.  I later determined that I would rather use my own encryption key that I have backed up in a different way.  It seems the only way to change that is to delete the rsync.net remote and re-add it with the key that I want to use.  So I told it to delete it in the web-gui, it has been stuck cleaning out for about a month now.  A thread keeps dying and I keep restarting it according to the gui.  There is an error in the logs, but it looks like it is trying to connect to both the computer that isn't always up and rsync.net and dying with it:
+
+```
+[2016-07-20 09:40:26.563762726] NetWatcherFallback: Syncing with 192.168.1.118_annex, rsync.net_annex 
+ssh: connect to host 192.168.1.118 port 22: No route to host
+fatal: Could not read from remote repository.
+```
+
+So at this point, I am contemplating just copying all the files out of the annex folders and deleting all the annex stuff and configuration and all the stores and starting over basically unless there is another approach that I can kill the rsync.net annex remote, delete the stuff directly on rsync, setup a new rsync.net remote.
diff --git a/doc/forum/Stupid_mistake__58___recoverable__63__.mdwn b/doc/forum/Stupid_mistake__58___recoverable__63__.mdwn
new file mode 100644
index 0000000000..8e25d70812
--- /dev/null
+++ b/doc/forum/Stupid_mistake__58___recoverable__63__.mdwn
@@ -0,0 +1,31 @@
+Hi,
+
+I was a bit hasty the other day and did something stupid.  I
+added a new folder to git annex.  Something like
+
+    git annex add my-important folder
+
+my-important folder contains a lot of files and it took a couple
+of minutes to add.  When I then tried to do
+
+    git commit -am 'added files'
+
+per the walkthrough I got an error (9, as I recall).  I thought
+I'd added too many files or something so I wanted to start over
+and perhaps I didn't fully understand the mechanisms of annex I
+did the following
+
+    git reset --hard .
+
+Unfortunately, did replaced my files with a bunch of symlinks,
+rather than making git annex forget and go back to the previous
+stage as I had hoped.
+
+I have managed to recover most of my files from backup, but some
+of them I still can't recover.  Is there any way back?  It seems
+I still have the files in my git folder.
+
+Thanks,
+Rasmus
+
+PS: Sorry, I shouldn't have made this text rather than Markdown.
diff --git a/doc/forum/Stupid_mistake__58___recoverable__63__/comment_1_00ceb3a5e37825c4bbc806f532893706._comment b/doc/forum/Stupid_mistake__58___recoverable__63__/comment_1_00ceb3a5e37825c4bbc806f532893706._comment
new file mode 100644
index 0000000000..ebc8073f7a
--- /dev/null
+++ b/doc/forum/Stupid_mistake__58___recoverable__63__/comment_1_00ceb3a5e37825c4bbc806f532893706._comment
@@ -0,0 +1,23 @@
+[[!comment format=mdwn
+ username="http://joeyh.name/"
+ nickname="joey"
+ subject="comment 1"
+ date="2013-04-01T18:24:20Z"
+ content="""
+The file contents are still there in `.git/annex/objects/`. 
+
+When I run `git reset --hard`, it deletes any files that have been added, but not committed. You say you still have symlinks to the content. If so, you can just `git add` the symlinks and be back how git-annex wants things to be.
+
+If, instead, you are missing the symlinks, then since you never committed the symlinks to git, there is no record anywhere in git of the filenames that go with the file contents. 
+
+What you can do is ask git-annex to put back symlinks to the file contents:
+
+
    +git annex unused
+git annex addunused 1-10000
+
    
+
+This will make you have a bunch of files in the repository with names starting with \"unused\". You can then rename them manually.
+
+BTW, I would of course be interested in any error message from `git commit`.
+"""]]
diff --git a/doc/forum/Stupid_mistake__58___recoverable__63__/comment_2_cbedc29678d9b6af3b3c0bb1915d2391._comment b/doc/forum/Stupid_mistake__58___recoverable__63__/comment_2_cbedc29678d9b6af3b3c0bb1915d2391._comment
new file mode 100644
index 0000000000..f552cfb186
--- /dev/null
+++ b/doc/forum/Stupid_mistake__58___recoverable__63__/comment_2_cbedc29678d9b6af3b3c0bb1915d2391._comment
@@ -0,0 +1,12 @@
+[[!comment format=mdwn
+ username="Rasmus"
+ ip="213.120.148.111"
+ subject="comment 2"
+ date="2013-04-01T23:23:55Z"
+ content="""
+Joey,
+
+Thanks for your useful tips and for providing this software.  It seems incredibly powerful and useful.  I made a copy of the folder which I messed up.  I will try the tips and report back (I'm on the road right now).
+
+The other day all the Haskell deps worked out and I managed to build git annex.  Hopefully, this will give more reliable results as it seemed that some of my bugs were caused by the 32bit prebuild on 64bit.  If I succeed in recreating my files I will try to recreate the error :)
+"""]]
diff --git a/doc/forum/Stupid_mistake__58___recoverable__63__/comment_3_86aa4d92a1330811862da1ba568b3037._comment b/doc/forum/Stupid_mistake__58___recoverable__63__/comment_3_86aa4d92a1330811862da1ba568b3037._comment
new file mode 100644
index 0000000000..b642cde987
--- /dev/null
+++ b/doc/forum/Stupid_mistake__58___recoverable__63__/comment_3_86aa4d92a1330811862da1ba568b3037._comment
@@ -0,0 +1,42 @@
+[[!comment format=mdwn
+ username="Rasmus"
+ ip="213.120.148.111"
+ subject="comment 3"
+ date="2013-04-02T23:25:45Z"
+ content="""
+Joey,
+
+Sorry for asking again, but I haven't been able to resolve this using the symlinks.
+
+Consider this file 
+
+    ElgarCEH.pdf -> ../.git/annex/objects/g6/q8/SHA256E-759787--3ec5f8d64cc4f84dec74ed7f0f7a640b3e672a03faf0f5fc3daf0b823d596f03.pdf/SHA256E-759787--3ec5f8d64cc4f84dec74ed7f0f7a640b3e672a03faf0f5fc3daf0b823d596f03.pdf
+
+The file that the symlink points to doesn't exist.  I'm not sure it should.  The file with {.map,.cahce} extensions does exist, in this case 
+
+    sh>ls ../.git/annex/objects/g6/q8/SHA256E-s759787--3ec5f8d64cc4f84dec74ed7f0f7a640b3e672a03faf0f5fc3daf0b823d596f03.pdf/
+       SHA256E-s759787--3ec5f8d64cc4f84dec74ed7f0f7a640b3e672a03faf0f5fc3daf0b823d596f03.pdf.cache
+       SHA256E-s759787--3ec5f8d64cc4f84dec74ed7f0f7a640b3e672a03faf0f5fc3daf0b823d596f03.pdf.map
+
+The content of the .cache file is 
+
+    1703824 411810 1350480334
+
+and the content of the .map file is 
+
+    documents/ElgarCEH.pdf
+
+However, for all of the following
+
+    git add ElgarCEH.pdf
+    git add --force ElgarCEH.pdf
+    git annex add ElgarCEH.pdf
+
+the files are not staged, and I get the message usual
+
+    no changes added to commit.
+
+with `git log`.
+
+So do I need to something more in order to get `git` to add my symlinks?
+"""]]
diff --git a/doc/forum/Stupid_mistake__58___recoverable__63__/comment_4_6d15bf8a3c3c27cc92957070161675a9._comment b/doc/forum/Stupid_mistake__58___recoverable__63__/comment_4_6d15bf8a3c3c27cc92957070161675a9._comment
new file mode 100644
index 0000000000..97a4cd0db1
--- /dev/null
+++ b/doc/forum/Stupid_mistake__58___recoverable__63__/comment_4_6d15bf8a3c3c27cc92957070161675a9._comment
@@ -0,0 +1,12 @@
+[[!comment format=mdwn
+ username="http://joeyh.name/"
+ nickname="joey"
+ subject="comment 4"
+ date="2013-04-02T23:39:48Z"
+ content="""
+If you `git add` a file and `git commit` says \"no changes added to commit\", what would you think would be the reason?
+
+I'd think it was because the file was already committed to git. So that must be the case with your symlink.
+
+You seem to have told git-annex to do something to the content of that file. You could use `git annex whereis` to find where it is, or `git annex get` to get it. (Assuming you have not used `git annex drop --force` on the file to nuke its content earlier.)
+"""]]
diff --git a/doc/forum/Stupid_mistake__58___recoverable__63__/comment_5_f836b9b1d03d94c49e3798961790b2ba._comment b/doc/forum/Stupid_mistake__58___recoverable__63__/comment_5_f836b9b1d03d94c49e3798961790b2ba._comment
new file mode 100644
index 0000000000..d28c247e1f
--- /dev/null
+++ b/doc/forum/Stupid_mistake__58___recoverable__63__/comment_5_f836b9b1d03d94c49e3798961790b2ba._comment
@@ -0,0 +1,21 @@
+[[!comment format=mdwn
+ username="Rasmus"
+ ip="213.120.148.111"
+ subject="comment 5"
+ date="2013-04-03T00:45:05Z"
+ content="""
+> [links not being added].
+Right.  Obliviously.  Sorry.
+
+
+I think the problem is more severe as all of the files have somehow ended up in the .git/objects folder.  I have no idea how, but I have been unsuccessful with both methods (the second methods gives me 
+    
+    getFileStatus: does not exist (No such file or directory)
+
+which I guess is related to the fact that my files somehow ended up in the git objects folder rather than the git annex objects folder.
+
+I appreciated the help.
+
+Cheers,
+Rasmus
+"""]]
diff --git a/doc/forum/Suggestion__58___Put_ssh_server_back_into_android_version.mdwn b/doc/forum/Suggestion__58___Put_ssh_server_back_into_android_version.mdwn
new file mode 100644
index 0000000000..1f8730514b
--- /dev/null
+++ b/doc/forum/Suggestion__58___Put_ssh_server_back_into_android_version.mdwn
@@ -0,0 +1,9 @@
+Hi!
+
+Just tried various android ssh servers which do work, but have no access to the command line tools shipped with git annex.
+
+From what I gather from the planning docs, sshd was stripped from the android tools to save space. If this were the case, it (naively) seems to me it would have to be simple to re-enable.
+
+Rationale: It would make it super convenient to use command line git annex on my phone alongside the assistant, through letting me type on my desktop. Not to mention that the command line tools shipped with git annex are by far the best around for android work.
+
+Carlo
diff --git a/doc/forum/Suggestion__58___Put_ssh_server_back_into_android_version/comment_1_5c2f376a82458c6387560355940419d3._comment b/doc/forum/Suggestion__58___Put_ssh_server_back_into_android_version/comment_1_5c2f376a82458c6387560355940419d3._comment
new file mode 100644
index 0000000000..9a39a2e4c9
--- /dev/null
+++ b/doc/forum/Suggestion__58___Put_ssh_server_back_into_android_version/comment_1_5c2f376a82458c6387560355940419d3._comment
@@ -0,0 +1,12 @@
+[[!comment format=mdwn
+ username="http://joeyh.name/"
+ ip="209.250.56.47"
+ subject="comment 1"
+ date="2013-11-02T19:42:13Z"
+ content="""
+sshd was removed from the bundle because it's not needed for git-annex to be used on Android, and it would be very complicated to get it to work. (And would require rooting the device at least for port 22 and probably also since sshd needs to run as root generally.)
+
+If there are android ssh servers that let you choose which user to log in as, you could log in as the app_NN user corresponding to the git-annex android app, and have all the tools available, once you ran runshell. I belive there's also an equivilant to su on android that lets you change to an app_NN user, although you probably already need root.
+
+I think it's more usual to use adb in these situations.
+"""]]
diff --git a/doc/forum/Suggestion__58___Put_ssh_server_back_into_android_version/comment_2_6321dec0b2f22f841f3cb986e063113f._comment b/doc/forum/Suggestion__58___Put_ssh_server_back_into_android_version/comment_2_6321dec0b2f22f841f3cb986e063113f._comment
new file mode 100644
index 0000000000..b1ce5a1a5b
--- /dev/null
+++ b/doc/forum/Suggestion__58___Put_ssh_server_back_into_android_version/comment_2_6321dec0b2f22f841f3cb986e063113f._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="https://www.google.com/accounts/o8/id?id=AItOawnZEanlyzay_QlEAL0CWpyZcRTyN7vay8U"
+ nickname="Carlo"
+ subject="comment 2"
+ date="2013-11-03T09:48:53Z"
+ content="""
+Ah, the \"very complicated to get it to work\" part is what I didn't know, thanks for explaining and or the tips.
+"""]]
diff --git a/doc/forum/Symantec_endpoint_protection_raises_WS.Reputation.1_on_git-annex_windows_installer.mdwn b/doc/forum/Symantec_endpoint_protection_raises_WS.Reputation.1_on_git-annex_windows_installer.mdwn
new file mode 100644
index 0000000000..16be75923e
--- /dev/null
+++ b/doc/forum/Symantec_endpoint_protection_raises_WS.Reputation.1_on_git-annex_windows_installer.mdwn
@@ -0,0 +1,13 @@
+Hello,
+
+I downloaded git-annex-installer.exe from https://downloads.kitenet.net/git-annex/windows/current/git-annex-installer.exe . 
+When I try to run it on my work windows 7 64-bit laptop, Symantec Encpoint Protection complains with a WS.Reputation.1 flag that the file has a low reputation and quarantines the file. 
+
+I searched online trying to find why git-annex is flagged as such by Symantec but was unable to find anything.
+I prefer not to build from source if possible as this is my work laptop and it would require that I install Haskell, MingW, and Cygwin. 
+
+Any pointers/suggestions would be greatly appreciated.
+
+Thanks,
+
+Ahmad
diff --git a/doc/forum/Symantec_endpoint_protection_raises_WS.Reputation.1_on_git-annex_windows_installer/comment_1_3b1da8ce0dcf43efcdc9b264ef21fb3a._comment b/doc/forum/Symantec_endpoint_protection_raises_WS.Reputation.1_on_git-annex_windows_installer/comment_1_3b1da8ce0dcf43efcdc9b264ef21fb3a._comment
new file mode 100644
index 0000000000..7cbed378f5
--- /dev/null
+++ b/doc/forum/Symantec_endpoint_protection_raises_WS.Reputation.1_on_git-annex_windows_installer/comment_1_3b1da8ce0dcf43efcdc9b264ef21fb3a._comment
@@ -0,0 +1,9 @@
+[[!comment format=mdwn
+ username="joey"
+ subject="""comment 1"""
+ date="2015-03-16T17:29:39Z"
+ content="""
+Does "low reputation" mean "few users", or "we have a reason to distrust
+this"? In the former case, I obviously can't do anything, and in the latter
+case, I'd need to know what problem they think it has.
+"""]]
diff --git a/doc/forum/Symlink_points_to_old_version.mdwn b/doc/forum/Symlink_points_to_old_version.mdwn
new file mode 100644
index 0000000000..257a7d5aa3
--- /dev/null
+++ b/doc/forum/Symlink_points_to_old_version.mdwn
@@ -0,0 +1,13 @@
+Hello everyone,
+
+I have some PDF documents in a git annex repository. I appended a page to (several) of these PDF documents the following way:
+
+- `git annex edit file.pdf`
+- Append page to file.pdf
+- `git annex add file.pdf`
+- `git commit`
+
+If I now look at file.pdf it will not have the appended page. But if do `git annex edit file.pdf` again I will get the version with the appended page. `git annex add file.pdf` and the page "disappears" again. Anyone got any tips on how to solve this "mystery"?
+
+All the best,
+Per
diff --git a/doc/forum/Symlink_points_to_old_version/comment_1_c9c777333a01865e95332ea7ff1e6cf7._comment b/doc/forum/Symlink_points_to_old_version/comment_1_c9c777333a01865e95332ea7ff1e6cf7._comment
new file mode 100644
index 0000000000..febe4c9a8b
--- /dev/null
+++ b/doc/forum/Symlink_points_to_old_version/comment_1_c9c777333a01865e95332ea7ff1e6cf7._comment
@@ -0,0 +1,41 @@
+[[!comment format=mdwn
+ username="joey"
+ subject="""comment 1"""
+ date="2015-05-13T15:46:00Z"
+ content="""
+The symlink should point to the new version, unless git-annex is catastropically
+broken, and this can be easily demonstrated that git-annex is not catastropically
+broken:
+
+	joey@darkstar:~/tmp/demo>cat foo.pdf 
+	original pdf
+	joey@darkstar:~/tmp/demo>git annex edit foo.pdf
+	unlock foo.pdf (copying...) ok
+	joey@darkstar:~/tmp/demo>echo "added page" >> foo.pdf 
+	joey@darkstar:~/tmp/demo>git annex add foo.pdf
+	add foo.pdf ok
+	(recording state in git...)
+	joey@darkstar:~/tmp/demo>git commit -m foo
+	[master 8d7f117] foo
+	 1 file changed, 1 insertion(+), 1 deletion(-)
+	joey@darkstar:~/tmp/demo>cat foo.pdf 
+	original pdf
+	added page
+
+So, what's really going on for you to see what you see when you look at the
+pdf? My guess is that your pdf editor/viewer is doing something bad/smart when
+it encounters the symlink. Perhaps it's loading an old cached version of the
+pdf rather than following the symlink or something.
+
+When you use `git annex edit`, the file stops being a symlink, and
+so whatever smart/bad behavior is causing the problem is avoided.
+
+If this is the case, switching to direct mode would avoid the problem.
+(`git annex direct`)
+
+If I were using a program and verfied that it has such bad/smart behavior
+on symlinks, I'd complain vigorously to its creators; it should't matter
+if a program is asked to open a symlink or not, it should behave the same either way.
+Unfortunately, it seems that some programs go out of their way to get
+this wrong and all we can do about it is filed bugs and/or use direct mode.
+"""]]
diff --git a/doc/forum/Symlink_points_to_old_version/comment_2_9bf097d27a64743a420cba0136787165._comment b/doc/forum/Symlink_points_to_old_version/comment_2_9bf097d27a64743a420cba0136787165._comment
new file mode 100644
index 0000000000..7563a1ed55
--- /dev/null
+++ b/doc/forum/Symlink_points_to_old_version/comment_2_9bf097d27a64743a420cba0136787165._comment
@@ -0,0 +1,10 @@
+[[!comment format=mdwn
+ username="https://me.yahoo.com/a/StKYI.ZuofVB3xNCCzjJo.V7Fg--#11600"
+ nickname="Per K"
+ subject="comment 2"
+ date="2015-05-14T09:31:09Z"
+ content="""
+Thank you for the response. Sorry I didn't think of the possibility that the PDF reader could be the problem. I have done the same operation many times and never encountered this problem before though. The culprit is Apple's Preview.app. It now shows the new version and to my knowledge I haven't done anything in the meantime. Guess it is just time passed that has fixed \"the problem\".
+
+Thank you!
+"""]]
diff --git a/doc/forum/Sync_All_Missing_Content__63__.mdwn b/doc/forum/Sync_All_Missing_Content__63__.mdwn
new file mode 100644
index 0000000000..326afa3038
--- /dev/null
+++ b/doc/forum/Sync_All_Missing_Content__63__.mdwn
@@ -0,0 +1 @@
+I have a git annex repo I'm using to sync to a couple of computers, plus backup to Glacier.  git annex sync --content normally works fine, but recently I've added some large files with git annex that also have multiple versions, and I see that when I do git annex sync --content, only the current version of the file content gets transferred.  I see that the --all option exists as well, which seems like it should do what I need, but when I run git annex sync --content --all it appears to start trying to sync _all_ content, even if it's already present on both machines.  And if I try to use it on my glacier remote, it seems to be trying to pull from Glacier as well, which of course fails because you can't immediately access a Glacier file.  Is there any combination of options that can do what I'm looking for here, to transfer all content but only a delta between what's present on the current machine and the remote?
diff --git a/doc/forum/Sync_All_Missing_Content__63__/comment_1_fbe27e8dc9dd85e860b3382f99426c14._comment b/doc/forum/Sync_All_Missing_Content__63__/comment_1_fbe27e8dc9dd85e860b3382f99426c14._comment
new file mode 100644
index 0000000000..a2e7974150
--- /dev/null
+++ b/doc/forum/Sync_All_Missing_Content__63__/comment_1_fbe27e8dc9dd85e860b3382f99426c14._comment
@@ -0,0 +1,18 @@
+[[!comment format=mdwn
+ username="lykos@d125a37d89b1cfac20829f12911656c40cb70018"
+ nickname="lykos"
+ avatar="http://cdn.libravatar.org/avatar/085df7b04d3408ba23c19f9c49be9ea2"
+ subject="comment 1"
+ date="2018-03-23T18:41:20Z"
+ content="""
+Did you set a [preferred content](https://git-annex.branchable.com/git-annex-preferred-content/) for this remote? `git annex wanted  anything` should tell git annex to sync all file versions when running `git annex sync --content`.
+
+Or (better) use the backup group:
+
+
+    git annex wanted  standard
+    git annex group  backup
+
+
+
+"""]]
diff --git a/doc/forum/Sync_All_Missing_Content__63__/comment_2_d77b7f56dcb8bf384f5a19bf54fe629e._comment b/doc/forum/Sync_All_Missing_Content__63__/comment_2_d77b7f56dcb8bf384f5a19bf54fe629e._comment
new file mode 100644
index 0000000000..040576f425
--- /dev/null
+++ b/doc/forum/Sync_All_Missing_Content__63__/comment_2_d77b7f56dcb8bf384f5a19bf54fe629e._comment
@@ -0,0 +1,9 @@
+[[!comment format=mdwn
+ username="misc@e661ec6eb62aa0134e4ac4ca16f98640acf5c627"
+ nickname="misc"
+ avatar="http://cdn.libravatar.org/avatar/78f317f1cdf5dfdaa295207a6dbf1311"
+ subject="comment 2"
+ date="2018-03-27T05:53:43Z"
+ content="""
+That's exactly what I needed, thanks!
+"""]]
diff --git a/doc/forum/Sync_Enabled__47__Disabled_Status.mdwn b/doc/forum/Sync_Enabled__47__Disabled_Status.mdwn
new file mode 100644
index 0000000000..196a67899d
--- /dev/null
+++ b/doc/forum/Sync_Enabled__47__Disabled_Status.mdwn
@@ -0,0 +1 @@
+In the webapp I can see which repos have syncing enabled or disabled, and toggle the property by clicking the current status. How can I do this in the terminal? I want to be able to see the current syncing status of the repository, and to change it, without opening the webapp.
diff --git a/doc/forum/Sync_Enabled__47__Disabled_Status/comment_1_cfe98bdd50cea0fd0e36a42892262084._comment b/doc/forum/Sync_Enabled__47__Disabled_Status/comment_1_cfe98bdd50cea0fd0e36a42892262084._comment
new file mode 100644
index 0000000000..2649c7d575
--- /dev/null
+++ b/doc/forum/Sync_Enabled__47__Disabled_Status/comment_1_cfe98bdd50cea0fd0e36a42892262084._comment
@@ -0,0 +1,15 @@
+[[!comment format=mdwn
+ username="joey"
+ subject="""comment 1"""
+ date="2018-04-04T16:18:11Z"
+ content="""
+The webapp is changing the git configuration 
+"remote..annex-sync" to false to disable and back to true to enable.
+
+You can use `git config` to change that yourself. 
+
+However, if you have the assistant running in the background and make such
+a git configuration change, it won't notice the change, and will keep
+running with the old configuration. So you'll want to restart it by running
+`git annex assistant --stop; git annex assistant`
+"""]]
diff --git a/doc/forum/Sync_files_from_remote_to_remote.mdwn b/doc/forum/Sync_files_from_remote_to_remote.mdwn
new file mode 100644
index 0000000000..a723c33f33
--- /dev/null
+++ b/doc/forum/Sync_files_from_remote_to_remote.mdwn
@@ -0,0 +1,5 @@
+Not sure if this use case is supported, but I have the following setup - I use git-annex to store my Lightroom catalog and photos. My local machine does not contain all files, `.mov` files for example are excluded via the preferred content mechanism. The full backup is on a remote server using ssh.
+
+Now I want to create another copy of the data (encrypted) from my local machine to a USB stick or similar. I found that the locally excluded files are not synced. In order to copy all the data I first need to change the preferred content config, execute `git-annex get` to fetch the data, then copy via `git-annex sync myusbremote --content`, then change the local preferred content pattern, then run `git annex drop --auto` to delete the local unwanted content again.
+
+I was wondering if there's a way to sync from one remote to another, without storing any data locallly ? Something like `git-annex sync --from ... --to ...`.
diff --git a/doc/forum/Sync_files_from_remote_to_remote/comment_1_c5247657a5b3cd68fd55112efe97ef99._comment b/doc/forum/Sync_files_from_remote_to_remote/comment_1_c5247657a5b3cd68fd55112efe97ef99._comment
new file mode 100644
index 0000000000..66ee6ce6ae
--- /dev/null
+++ b/doc/forum/Sync_files_from_remote_to_remote/comment_1_c5247657a5b3cd68fd55112efe97ef99._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="https://openid.stackexchange.com/user/e65e6d0e-58ba-41de-84cc-1f2ba54cf574"
+ nickname="Mica Semrick"
+ subject="assistant or cron"
+ date="2015-01-11T04:00:04Z"
+ content="""
+I'd either (1) run the assistant on your server or (2) write a simple cron script that runs git-annex sync --content on the server.
+"""]]
diff --git a/doc/forum/Sync_files_from_remote_to_remote/comment_2_634e399d21271f26f7f77c15c5fb132b._comment b/doc/forum/Sync_files_from_remote_to_remote/comment_2_634e399d21271f26f7f77c15c5fb132b._comment
new file mode 100644
index 0000000000..4a0038f9d4
--- /dev/null
+++ b/doc/forum/Sync_files_from_remote_to_remote/comment_2_634e399d21271f26f7f77c15c5fb132b._comment
@@ -0,0 +1,7 @@
+[[!comment format=mdwn
+ username="Jan"
+ subject="comment 2"
+ date="2015-01-11T08:54:52Z"
+ content="""
+this would work, but the server is on a remote machine and the USB drive I want to sync to is connected to my local machine.
+"""]]
diff --git a/doc/forum/Sync_files_from_remote_to_remote/comment_3_b89701582b40e2c337a1a49733877210._comment b/doc/forum/Sync_files_from_remote_to_remote/comment_3_b89701582b40e2c337a1a49733877210._comment
new file mode 100644
index 0000000000..73c9cc0bcd
--- /dev/null
+++ b/doc/forum/Sync_files_from_remote_to_remote/comment_3_b89701582b40e2c337a1a49733877210._comment
@@ -0,0 +1,10 @@
+[[!comment format=mdwn
+ username="https://www.google.com/accounts/o8/id?id=AItOawnm19dBCRphmtjXfopm_NpvnRwz-qIJ2Tw"
+ nickname="Remi"
+ subject="May be use a prefered content expression about what is in the usb stick"
+ date="2015-01-11T09:14:47Z"
+ content="""
+Is there a full git repos on the usb stick? if yes You could add a remote on it and run `git annex --content` from there.
+
+You could also try to change the preferred content of your local machine to include something like \"--not --in=usb\". With this you local repos will download file not in the usb stick, copy it to the usb stick and then remove it from the local repos. (I didn't check exactly the syntax of the preferred content, so my expression could be incorrect, but I hope you get the idea)
+"""]]
diff --git a/doc/forum/Sync_files_from_remote_to_remote/comment_4_b3ad93e032c3c9d2312970711540bf92._comment b/doc/forum/Sync_files_from_remote_to_remote/comment_4_b3ad93e032c3c9d2312970711540bf92._comment
new file mode 100644
index 0000000000..ad730a3ead
--- /dev/null
+++ b/doc/forum/Sync_files_from_remote_to_remote/comment_4_b3ad93e032c3c9d2312970711540bf92._comment
@@ -0,0 +1,7 @@
+[[!comment format=mdwn
+ username="Jan"
+ subject="comment 4"
+ date="2015-01-13T09:35:10Z"
+ content="""
+Thanks for your answers. But I want to encrypt the content, so initialising a repo on the stick is not a possibility if I understand it correctly. And wouldn't `--not --in=usb` exclude a lot of content I actually want ? (Haven't tried it yet)
+"""]]
diff --git a/doc/forum/Sync_files_from_remote_to_remote/comment_5_bdf293d30692311983d0586231535bd4._comment b/doc/forum/Sync_files_from_remote_to_remote/comment_5_bdf293d30692311983d0586231535bd4._comment
new file mode 100644
index 0000000000..4ea668070c
--- /dev/null
+++ b/doc/forum/Sync_files_from_remote_to_remote/comment_5_bdf293d30692311983d0586231535bd4._comment
@@ -0,0 +1,20 @@
+[[!comment format=mdwn
+ username="https://www.google.com/accounts/o8/id?id=AItOawnm19dBCRphmtjXfopm_NpvnRwz-qIJ2Tw"
+ nickname="Remi"
+ subject="comment 5"
+ date="2015-01-13T12:45:47Z"
+ content="""
+Well my idea wasn't to have only this in the preferred content. You
+can use \"or\" expression.
+
+That said, I've just check the manual, and it seem that there is no --in for
+preferred content... But you could add your usb repos say to the
+archival group (git annex group usb archival), Then use something like
+like `\"include=* and ( exclude=*.mov or not inallgroup=archival )\"`
+(this is untested, but I hope you get the idea).
+
+With such a preferred content, you local repository should get (and
+keep) the .mov not in usb, and should drop them when they are send
+there.
+
+"""]]
diff --git a/doc/forum/Sync_files_from_remote_to_remote/comment_6_fda62d735035213f2a0b5b2fc95991ab._comment b/doc/forum/Sync_files_from_remote_to_remote/comment_6_fda62d735035213f2a0b5b2fc95991ab._comment
new file mode 100644
index 0000000000..9ab809aea4
--- /dev/null
+++ b/doc/forum/Sync_files_from_remote_to_remote/comment_6_fda62d735035213f2a0b5b2fc95991ab._comment
@@ -0,0 +1,11 @@
+[[!comment format=mdwn
+ username="joey"
+ subject="""comment 6"""
+ date="2015-01-20T18:51:06Z"
+ content="""
+Remi's idea of using [[preferred_content]] can be made to work.
+
+There's no `git annex copy --from A --to B`. It would need to locally spool
+each file into the current repo, and being an unusual corner case has not
+seemed worth doing so far.
+"""]]
diff --git a/doc/forum/Sync_files_from_remote_to_remote/comment_7_77c6550397435e9601fbae467be90976._comment b/doc/forum/Sync_files_from_remote_to_remote/comment_7_77c6550397435e9601fbae467be90976._comment
new file mode 100644
index 0000000000..bbacebbdcc
--- /dev/null
+++ b/doc/forum/Sync_files_from_remote_to_remote/comment_7_77c6550397435e9601fbae467be90976._comment
@@ -0,0 +1,12 @@
+[[!comment format=mdwn
+ username="Jan"
+ subject="comment 7"
+ date="2016-02-03T02:49:01Z"
+ content="""
+I can confirm that Remi's suggestion does work as expected, thanks!
+
+    $ git annex sync --content
+    get 2014/2014-06-14/IMG_1448.JPG (from origin...) (checksum...) ok
+    copy 2014/2014-06-14/IMG_1448.JPG copy 2014/2014-06-14/IMG_1448.JPG (to usb...) ok
+    drop 2014/2014-06-14/IMG_1448.JPG ok
+"""]]
diff --git a/doc/forum/Sync_files_from_remote_to_remote/comment_8_4d3c05acc310a21f6c8ab77fb2638a8c._comment b/doc/forum/Sync_files_from_remote_to_remote/comment_8_4d3c05acc310a21f6c8ab77fb2638a8c._comment
new file mode 100644
index 0000000000..279929f2ac
--- /dev/null
+++ b/doc/forum/Sync_files_from_remote_to_remote/comment_8_4d3c05acc310a21f6c8ab77fb2638a8c._comment
@@ -0,0 +1,11 @@
+[[!comment format=mdwn
+ username="konubinix"
+ subject="git-annex-getcopy.sh"
+ date="2016-02-03T12:32:55Z"
+ content="""
+To do that, I created a simple bash script that get locally the files and then copies them to the remote
+
+https://github.com/Konubinix/Devel/blob/master/bin/git-annex-getcopy.sh
+
+I hope that helps.
+"""]]
diff --git a/doc/forum/Sync_only_what_is_not_available_from_my_central_git_repo.mdwn b/doc/forum/Sync_only_what_is_not_available_from_my_central_git_repo.mdwn
new file mode 100644
index 0000000000..223e5eaeda
--- /dev/null
+++ b/doc/forum/Sync_only_what_is_not_available_from_my_central_git_repo.mdwn
@@ -0,0 +1,7 @@
+I work in a distributed startup.  We have a master repository on GitHub.  At the office we have modern hunky Linux desktops (seriously more powerful than any of our laptops).
+
+I (and others) have hard deadlines at the end of the day (typically child commitments).  If I leave from a meeting or brainstorming session without having pushed all of my work into the cloud then I cannot continue working from home.  This is because our office is behind multiple levels of NAT over which we have no control.  Hence I am unable to remote into my machine in order to push work up to GitHub or scp it to my laptop.
+
+Is there a way to get git-annex to sync only files that do not correspond to git objects available from our GitHub repository?  Because our source tree is quite large (2.5GB including tests and test data) I would not want git-annex to sync the entire tree.  And when I checkout a different branch I would not want git-annex to sync all changes between the previously checked-out branch and the new branch.  Rather I would like git-annex to discard any previously synced objects, note the identity of the new branch and the values of all files (object) not available from the upstream state of that branch on GitHub.
+
+Thanks in advance for any suggestions.
diff --git a/doc/forum/Sync_only_what_is_not_available_from_my_central_git_repo/comment_1_b31d02c97447996495de73705ac39f71._comment b/doc/forum/Sync_only_what_is_not_available_from_my_central_git_repo/comment_1_b31d02c97447996495de73705ac39f71._comment
new file mode 100644
index 0000000000..b7f343b133
--- /dev/null
+++ b/doc/forum/Sync_only_what_is_not_available_from_my_central_git_repo/comment_1_b31d02c97447996495de73705ac39f71._comment
@@ -0,0 +1,12 @@
+[[!comment format=mdwn
+ username="gjost"
+ ip="75.140.92.244"
+ subject="comment 1"
+ date="2014-05-20T15:19:35Z"
+ content="""
+> If I leave from a meeting or brainstorming session without having pushed all of my work into the cloud then I cannot continue working from home. This is because our office is behind multiple levels of NAT over which we have no control. Hence I am unable to remote into my machine in order to push work up to GitHub or scp it to my laptop.
+
+This is not a git-annex solution, but you could use Pagekite to SSH to your work machine from home.
+
+
+"""]]
diff --git a/doc/forum/Sync_only_what_is_not_available_from_my_central_git_repo/comment_2_098465ae8af32931779d2cd63750d5dc._comment b/doc/forum/Sync_only_what_is_not_available_from_my_central_git_repo/comment_2_098465ae8af32931779d2cd63750d5dc._comment
new file mode 100644
index 0000000000..95a8721dff
--- /dev/null
+++ b/doc/forum/Sync_only_what_is_not_available_from_my_central_git_repo/comment_2_098465ae8af32931779d2cd63750d5dc._comment
@@ -0,0 +1,10 @@
+[[!comment format=mdwn
+ username="http://joeyh.name/"
+ ip="209.250.56.3"
+ subject="comment 2"
+ date="2014-05-21T18:11:45Z"
+ content="""
+It seems to me that simply running `git annex sync --content` before you go home would do what you need. Have you tried using git-annex? 
+
+Some of the things you're saying don't entirely make sense to me, and so I suspect you're perhaps worrying about problems that don't exist, or that git-annex deals with nicely.
+"""]]
diff --git a/doc/forum/Sync_with_one_offline_peer.mdwn b/doc/forum/Sync_with_one_offline_peer.mdwn
new file mode 100644
index 0000000000..6ee05c6e67
--- /dev/null
+++ b/doc/forum/Sync_with_one_offline_peer.mdwn
@@ -0,0 +1,11 @@
+Hello,
+
+I use the assistant to set up two repositories A and B synced using jabber. A third repository C on my server is used as rsync transfer. Syncing works fine between both repos when both are online.
+
+But when either A or B is offline the sync does not happen when it comes online again, though the file was synced to C.
+
+Is this because C is only a rsync repository and can't hold metadata? How can I achieve that the sync happens also when one of the repositories is offline?
+
+I also tried using the static build of git annex on my server. It seemed to run fine but during the setup the assistant got an error about too many command line arguments. On A und B I use the ArchLinux AUR build (https://aur.archlinux.org/packages/git-annex-standalone/), on C I use the static build. Could it be a version mismatch?
+
+Thanks!
diff --git a/doc/forum/Sync_with_one_offline_peer/comment_1_3859d842d4f7e2ef44877b05ebe881fb._comment b/doc/forum/Sync_with_one_offline_peer/comment_1_3859d842d4f7e2ef44877b05ebe881fb._comment
new file mode 100644
index 0000000000..5fe670d35c
--- /dev/null
+++ b/doc/forum/Sync_with_one_offline_peer/comment_1_3859d842d4f7e2ef44877b05ebe881fb._comment
@@ -0,0 +1,20 @@
+[[!comment format=mdwn
+ username="https://www.google.com/accounts/o8/id?id=AItOawnsuhFUIfWNT-Q-C02FDaSQqceFDge5M9w"
+ nickname="Florian"
+ subject="comment 1"
+ date="2013-09-09T21:49:17Z"
+ content="""
+Another try on the layout:
+
+Hello,
+
+I use the assistant to set up two repositories A and B synced using jabber. A third repository C on my server is used as rsync transfer. Syncing works fine between both repos when both are online.
+
+But when either A or B is offline the sync does not happen when it comes online again, though the file was synced to C.
+
+Is this because C is only a rsync repository and can't hold metadata? How can I achieve that the sync happens also when one of the repositories is offline?
+
+I also tried using the static build of git annex on my server. It seemed to run fine but during the setup the assistant got an error about too many command line arguments. On A und B I use the ArchLinux AUR build (https://aur.archlinux.org/packages/git-annex-standalone/), on C I use the static build. Could it be a version mismatch?
+
+Thanks!
+"""]]
diff --git a/doc/forum/Sync_with_one_offline_peer/comment_2_c9ba3983b37b0c1868269616fd81e518._comment b/doc/forum/Sync_with_one_offline_peer/comment_2_c9ba3983b37b0c1868269616fd81e518._comment
new file mode 100644
index 0000000000..20b82dd3e5
--- /dev/null
+++ b/doc/forum/Sync_with_one_offline_peer/comment_2_c9ba3983b37b0c1868269616fd81e518._comment
@@ -0,0 +1,14 @@
+[[!comment format=mdwn
+ username="http://joeyh.name/"
+ ip="4.154.4.51"
+ subject="comment 2"
+ date="2013-09-12T21:13:36Z"
+ content="""
+Yes, the rsync repository is not a git repository and so does not hold metadata. If you can install git-annex on your server, you can add a regular git repository there, and then the clients can sync to it even when the other is offline.
+
+This is also why I an working on adding support for [[special_remotes/gcrypt]] repositories, so you can have a fully encrypted git repository on the server and sync through that.
+
+>   during the setup the assistant got an error about too many command line arguments
+
+I have never heard about such a problem. You should file a bug report with the details.
+"""]]
diff --git a/doc/forum/Sync_with_one_offline_peer/comment_3_28b9c003b4560c3ce90c9ebf808b091b._comment b/doc/forum/Sync_with_one_offline_peer/comment_3_28b9c003b4560c3ce90c9ebf808b091b._comment
new file mode 100644
index 0000000000..332cd83c35
--- /dev/null
+++ b/doc/forum/Sync_with_one_offline_peer/comment_3_28b9c003b4560c3ce90c9ebf808b091b._comment
@@ -0,0 +1,10 @@
+[[!comment format=mdwn
+ username="https://www.google.com/accounts/o8/id?id=AItOawnsuhFUIfWNT-Q-C02FDaSQqceFDge5M9w"
+ nickname="Florian"
+ subject="comment 3"
+ date="2013-09-13T10:52:33Z"
+ content="""
+Thanks for your reply. The bug report is at 
+
+I'll be happy to provide any more help if I can!
+"""]]
diff --git a/doc/forum/Sync_without_jabber_account.mdwn b/doc/forum/Sync_without_jabber_account.mdwn
new file mode 100644
index 0000000000..b459f33103
--- /dev/null
+++ b/doc/forum/Sync_without_jabber_account.mdwn
@@ -0,0 +1,9 @@
+Hi,
+
+It is possible to keep devices sync only using a box.com account as a transfer remote, without Jabber? I can't use Jabber because of firewall restrictions.
+
+It would be nice to be able to use the transfer remote, in my case box.net, as the sync "provider", for example by polling a file with the needed changes in a pre-defined interval.
+
+Regards,
+
+Stone
diff --git a/doc/forum/Sync_without_jabber_account/comment_1_3e95ac2e67451f953cf0538094109f8b._comment b/doc/forum/Sync_without_jabber_account/comment_1_3e95ac2e67451f953cf0538094109f8b._comment
new file mode 100644
index 0000000000..1a601848c1
--- /dev/null
+++ b/doc/forum/Sync_without_jabber_account/comment_1_3e95ac2e67451f953cf0538094109f8b._comment
@@ -0,0 +1,10 @@
+[[!comment format=mdwn
+ username="http://joeyh.name/"
+ ip="4.154.4.90"
+ subject="comment 1"
+ date="2013-07-17T19:06:59Z"
+ content="""
+We don't currently have a way to store a git repository on box.com, and you need such a git repo on a server somewhere if you're not using Jabber.
+
+Of course you can mount the box.com, using either davfs2 or something else, and put a bare git repository in its directory, and if you set this up on multiple computers, it might just work (or they might both try to write to it at the same time and fail.. I have not tried).
+"""]]
diff --git a/doc/forum/Synchronize_large_files___40__VM_images__41__.mdwn b/doc/forum/Synchronize_large_files___40__VM_images__41__.mdwn
new file mode 100644
index 0000000000..71de3ee7cf
--- /dev/null
+++ b/doc/forum/Synchronize_large_files___40__VM_images__41__.mdwn
@@ -0,0 +1,10 @@
+Hi, 
+
+i'm thinking to use git-annex to synchronize my virtual machine directory (Virtualbox) between 3 pc. It's quite big: more than 200GB and some of the images are 40Gb in size.
+
+The synchronization will be over a lan (obviously). It is already in place with 2pc and unison but the configuration of the 3rd pc is cumbersome. 
+Does anybody have experiences with git-annex and such amount of data?
+
+Thanks in advance
+
+Gabriele
diff --git a/doc/forum/Synchronize_large_files___40__VM_images__41__/comment_1_619f6ed2d7da5832ab253d61b6dd8044._comment b/doc/forum/Synchronize_large_files___40__VM_images__41__/comment_1_619f6ed2d7da5832ab253d61b6dd8044._comment
new file mode 100644
index 0000000000..292ee3b7bf
--- /dev/null
+++ b/doc/forum/Synchronize_large_files___40__VM_images__41__/comment_1_619f6ed2d7da5832ab253d61b6dd8044._comment
@@ -0,0 +1,10 @@
+[[!comment format=mdwn
+ username="http://joeyh.name/"
+ ip="4.154.6.49"
+ subject="comment 1"
+ date="2012-11-28T18:01:31Z"
+ content="""
+This volume of data should be no problem for git-annex.
+
+The only catch would be if you're running those VM images and want to sync them as they're changed. With git-annex, you'd need to `git annex unlock` a file to allow it to be modified, and then `git annex add` it back and commit changes made to it.
+"""]]
diff --git a/doc/forum/Synchronize_large_files___40__VM_images__41__/comment_2_bbd98d0b5d77dc7efc55ef8c2a18d612._comment b/doc/forum/Synchronize_large_files___40__VM_images__41__/comment_2_bbd98d0b5d77dc7efc55ef8c2a18d612._comment
new file mode 100644
index 0000000000..bc04122df2
--- /dev/null
+++ b/doc/forum/Synchronize_large_files___40__VM_images__41__/comment_2_bbd98d0b5d77dc7efc55ef8c2a18d612._comment
@@ -0,0 +1,13 @@
+[[!comment format=mdwn
+ username="https://me.yahoo.com/a/hVbIabkhqO11.DpKUWBoztFSLD5q#8cbe8"
+ nickname="Murat"
+ avatar="http://cdn.libravatar.org/avatar/52d95e40aca820c1993077ef9aa676c75700a072511c143f6db6b78be6b1b212"
+ subject="Why is it takins too long?"
+ date="2017-04-18T16:26:06Z"
+ content="""
+Hi,
+due to my requirement I need to revert vm image every time before running it via \"git reset --hard\" which is really fast on the other hand \"git annex unlock\" takes really long, I run git-annex on Centos 6 and git-annex version git-annex-3.20120522-2.1.el6.x86_64, if I update git-annex version can it help to fasten \"unlock\"?
+
+thanks a lot
+
+"""]]
diff --git a/doc/forum/Synchronize_large_files___40__VM_images__41__/comment_3_6488bbe6f39e7154f950530498e9b548._comment b/doc/forum/Synchronize_large_files___40__VM_images__41__/comment_3_6488bbe6f39e7154f950530498e9b548._comment
new file mode 100644
index 0000000000..9c55e48f7c
--- /dev/null
+++ b/doc/forum/Synchronize_large_files___40__VM_images__41__/comment_3_6488bbe6f39e7154f950530498e9b548._comment
@@ -0,0 +1,22 @@
+[[!comment format=mdwn
+ username="joey"
+ subject="""comment 3"""
+ date="2017-05-11T17:06:23Z"
+ content="""
+The reason `git annex unlock` is slow is because it makes a copy
+of the entire file. The file is left as-is in the annex so the
+old version is available later, and the unlocked copy is made available for
+modification.
+
+More recent versions of git-annex support v6 mode, which has a annex.thin
+configuration that makes `git annex unlock` not do this copy, so it's very
+fast. But then no copy of the old version of the file will be made,
+and so you won't be able to revert to the old version. Which seems to be
+an important part of your workflow.
+
+Another way to make `git annex unlock` fast is to use a file system that
+supports Copy On Write (CoW). git-annex will use CoW automatically when
+available, and then unlocking doesn't need to actually copy the file,
+but the old version will still be preserved. Btrfs is the only filesystem
+I know of that supports CoW, although there may be others.
+"""]]
diff --git a/doc/forum/Synchronize_two_latops_with_a_ssh_remote.mdwn b/doc/forum/Synchronize_two_latops_with_a_ssh_remote.mdwn
new file mode 100644
index 0000000000..1cf693b5fd
--- /dev/null
+++ b/doc/forum/Synchronize_two_latops_with_a_ssh_remote.mdwn
@@ -0,0 +1,3 @@
+Using the command `git-annex webapp` I was able to configure a local repository on my laptop A that is synchronized with a remote repository using ssh. This latter has sync enabled and it is encrypted. I also put it in the *full backup* group.
+
+Now, I would like to synchronize my laptop B with the remote repository with ssh. Using the webapp, I tried to do the same thing I did with my laptop A. However now, my both laptop are synchronized with the remote repository but they do not share any file between them. How can I achieve that ?
diff --git a/doc/forum/Syncing_Compressed_files.mdwn b/doc/forum/Syncing_Compressed_files.mdwn
new file mode 100644
index 0000000000..dd2f5adea7
--- /dev/null
+++ b/doc/forum/Syncing_Compressed_files.mdwn
@@ -0,0 +1 @@
+I don't see anywhere in the documentation that specifies this, but git-annex hasn't been syncing any kind of compressed file for me like zip files or 7z compressed files.  Haven't tried gzip files.  It has been working fine with other files like PDFs, video, images etc.  Was wanting to know if that is expected behaviour or if something is wrong?
diff --git a/doc/forum/Syncing_Compressed_files/comment_1_e44598cd735830a8df9a29e0031d34f1._comment b/doc/forum/Syncing_Compressed_files/comment_1_e44598cd735830a8df9a29e0031d34f1._comment
new file mode 100644
index 0000000000..bc20f3eb0f
--- /dev/null
+++ b/doc/forum/Syncing_Compressed_files/comment_1_e44598cd735830a8df9a29e0031d34f1._comment
@@ -0,0 +1,10 @@
+[[!comment format=mdwn
+ username="joey"
+ subject="""comment 1"""
+ date="2016-01-29T17:44:15Z"
+ content="""
+git-annex doesn't differentiate between different sorts of files in any
+way.
+
+Are you sure you're adding these compressed files to the repo?
+"""]]
diff --git a/doc/forum/Syncing_Compressed_files/comment_2_db1ea446ae9720fe1651c56da38e7088._comment b/doc/forum/Syncing_Compressed_files/comment_2_db1ea446ae9720fe1651c56da38e7088._comment
new file mode 100644
index 0000000000..595ccc7b25
--- /dev/null
+++ b/doc/forum/Syncing_Compressed_files/comment_2_db1ea446ae9720fe1651c56da38e7088._comment
@@ -0,0 +1,158 @@
+[[!comment format=mdwn
+ username="moird"
+ subject="comment 2"
+ date="2016-01-29T18:42:12Z"
+ content="""
+Using assistant on a folder.  Any files that are dropped in that folder will immediately be synced as far as I can tell except for a couple of zip files.  I turned on debug logging to look at it, and it looks like the watcher is not being called when the zip file is being copied into the folder, but is when the file is being removed from the folder.  Compared to when a regular file is dropped into the folder where it will run without issue:
+
+    [2016-01-27 06:51:26.988169] feed: git [\"--git-dir=.git\",\"--work-tree=.\",\"--literal-pathspecs\",\"-c\",\"core.bare=false\",\"update-index\",\"-z\",\"--index-info\"]
+    [2016-01-27 06:51:26.990623] process done ExitSuccess
+    [2016-01-27 06:51:26.990908] read: git [\"--git-dir=.git\",\"--work-tree=.\",\"--literal-pathspecs\",\"-c\",\"core.bare=false\",\"symbolic-ref\",\"-q\",\"HEAD\"]
+    [2016-01-27 06:51:26.992654] process done ExitSuccess
+    [2016-01-27 06:51:26.992981] read: git [\"--git-dir=.git\",\"--work-tree=.\",\"--literal-pathspecs\",\"-c\",\"core.bare=false\",\"show-ref\",\"--hash\",\"refs/heads/annex/direct/master\"]
+    [2016-01-27 06:51:26.995703] process done ExitSuccess
+    [2016-01-27 06:51:26.995998] read: git [\"--git-dir=.git\",\"--work-tree=.\",\"--literal-pathspecs\",\"-c\",\"core.bare=false\",\"write-tree\"]
+    [2016-01-27 06:51:26.997896] process done ExitSuccess
+    [2016-01-27 06:51:26.998196] read: git [\"--git-dir=.git\",\"--work-tree=.\",\"--literal-pathspecs\",\"-c\",\"core.bare=false\",\"rev-parse\",\"db03e812e512e24afe7f4b440a6ca174aa33cbdd:\"]
+    [2016-01-27 06:51:27.000111] process done ExitSuccess
+    [2016-01-27 06:51:27.00038] read: git [\"--git-dir=.git\",\"--work-tree=.\",\"--literal-pathspecs\",\"-c\",\"core.bare=false\",\"symbolic-ref\",\"-q\",\"HEAD\"]
+    [2016-01-27 06:51:27.001926] process done ExitSuccess
+    [2016-01-27 06:51:27.002223] read: git [\"--git-dir=.git\",\"--work-tree=.\",\"--literal-pathspecs\",\"-c\",\"core.bare=false\",\"show-ref\",\"refs/heads/annex/direct/master\"]
+    [2016-01-27 06:51:27.00406] process done ExitSuccess
+    [2016-01-27 06:51:27.004382] call: git [\"--git-dir=.git\",\"--work-tree=.\",\"--literal-pathspecs\",\"-c\",\"core.bare=false\",\"branch\",\"-f\",\"synced/master\"]
+    [2016-01-27 06:51:27.006269] process done ExitSuccess
+    [2016-01-27 06:51:27.006567] call: git [\"--git-dir=.git\",\"--work-tree=.\",\"--literal-pathspecs\",\"-c\",\"core.bare=false\",\"branch\",\"-f\",\"master\"]
+    [2016-01-27 06:51:27.008293] process done ExitSuccess
+    [2016-01-27 06:51:27.008992] Pusher: Syncing with annex 
+    [2016-01-27 06:51:27.009551] chat: git [\"--git-dir=.git\",\"--work-tree=.\",\"--literal-pathspecs\",\"-c\",\"core.bare=false\",\"hash-object\",\"-w\",\"--stdin-paths\",\"--no-filters\"]
+    [2016-01-27 06:51:27.010828] feed: git [\"--git-dir=.git\",\"--work-tree=.\",\"--literal-pathspecs\",\"-c\",\"core.bare=false\",\"update-index\",\"-z\",\"--index-info\"]
+    [2016-01-27 06:51:27.013085] process done ExitSuccess
+    [2016-01-27 06:51:27.013608] process done ExitSuccess
+    [2016-01-27 06:51:27.014011] read: git [\"--git-dir=.git\",\"--work-tree=.\",\"--literal-pathspecs\",\"-c\",\"core.bare=false\",\"show-ref\",\"--hash\",\"refs/heads/git-annex\"]
+    [2016-01-27 06:51:27.01584] process done ExitSuccess
+    (recording state in git...)
+    [2016-01-27 06:51:27.016212] read: git [\"--git-dir=.git\",\"--work-tree=.\",\"--literal-pathspecs\",\"-c\",\"core.bare=false\",\"write-tree\"]
+    [2016-01-27 06:51:27.018706] process done ExitSuccess
+    [2016-01-27 06:51:27.018978] chat: git [\"--git-dir=.git\",\"--work-tree=.\",\"--literal-pathspecs\",\"-c\",\"core.bare=false\",\"commit-tree\",\"29075d2d02995138a1774809a34c5220f54cea52\",\"--no-gpg-sign\",\"-p\",\"refs/heads/git-annex\"]
+    [2016-01-27 06:51:27.021681] process done ExitSuccess
+    [2016-01-27 06:51:27.021966] call: git [\"--git-dir=.git\",\"--work-tree=.\",\"--literal-pathspecs\",\"-c\",\"core.bare=false\",\"update-ref\",\"refs/heads/git-annex\",\"c833bef56cf282eefb957347e2a1938ef8568da6\"]
+    [2016-01-27 06:51:27.024413] process done ExitSuccess
+    [2016-01-27 06:51:27.025057] read: git [\"--git-dir=.git\",\"--work-tree=.\",\"--literal-pathspecs\",\"-c\",\"core.bare=false\",\"symbolic-ref\",\"-q\",\"HEAD\"]
+    [2016-01-27 06:51:27.027394] process done ExitSuccess
+    [2016-01-27 06:51:27.027759] read: git [\"--git-dir=.git\",\"--work-tree=.\",\"--literal-pathspecs\",\"-c\",\"core.bare=false\",\"show-ref\",\"refs/heads/annex/direct/master\"]
+    [2016-01-27 06:51:27.030137] process done ExitSuccess
+    [2016-01-27 06:51:27.030507] Pusher: pushing to [Remote { name =\"annex\" }]
+    [2016-01-27 06:51:27.030554] read: git [\"--git-dir=.git\",\"--work-tree=.\",\"--literal-pathspecs\",\"-c\",\"core.bare=false\",\"show-ref\",\"git-annex\"]
+    [2016-01-27 06:51:27.033305] process done ExitSuccess
+    [2016-01-27 06:51:27.033621] read: git [\"--git-dir=.git\",\"--work-tree=.\",\"--literal-pathspecs\",\"-c\",\"core.bare=false\",\"show-ref\",\"--hash\",\"refs/heads/git-annex\"]
+    [2016-01-27 06:51:27.035758] process done ExitSuccess
+    [2016-01-27 06:51:27.036238] read: git [\"--git-dir=.git\",\"--work-tree=.\",\"--literal-pathspecs\",\"-c\",\"core.bare=false\",\"log\",\"refs/heads/git-annex..c833bef56cf282eefb957347e2a1938ef8568da6\",\"-n1\",\"--pretty=%H\"]
+    [2016-01-27 06:51:27.039257] process done ExitSuccess
+    [2016-01-27 06:51:27.039752] read: git [\"--git-dir=.git\",\"--work-tree=.\",\"--literal-pathspecs\",\"-c\",\"core.bare=false\",\"log\",\"refs/heads/git-annex..7e6f7d4c9ff0bdb8f942c8edcb8a5a2581e7adb2\",\"-n1\",\"--pretty=%H\"]
+    [2016-01-27 06:51:27.043657] process done ExitSuccess
+    [2016-01-27 06:51:27.044082] read: git [\"--git-dir=.git\",\"--work-tree=.\",\"--literal-pathspecs\",\"-c\",\"core.bare=false\",\"ls-tree\",\"--full-tree\",\"-z\",\"--\",\"refs/heads/git-annex\",\"uuid.log\",\"remote.log\",\"trust.log\",\"group.log\",\"numcopies.log\",\"schedule.log\",\"preferred-content.log\",\"required-content.log\",\"group-preferred-content.log\"]
+    [2016-01-27 06:51:27.047843] process done ExitSuccess
+    [2016-01-27 06:51:27.048343] ConfigMonitor: reloading config numcopies.log
+    [2016-01-27 06:51:27.04847] call: git [\"--git-dir=.git\",\"--work-tree=.\",\"--literal-pathspecs\",\"-c\",\"core.bare=false\",\"push\",\"annex\",\"+git-annex:synced/git-annex\",\"annex/direct/master:synced/master\"]
+    [2016-01-27 06:51:27.050624] Cronner: reloading changed activities
+    To /home/daniel/store/annex
+       7e6f7d4..c833bef  git-annex -> synced/git-annex
+    [2016-01-27 06:51:27.290878] read: git [\"--git-dir=.git\",\"--work-tree=.\",\"--literal-pathspecs\",\"-c\",\"core.bare=false\",\"show-ref\",\"git-annex\"]
+    [2016-01-27 06:51:27.291094] process done ExitSuccess
+    [2016-01-27 06:51:27.292194] read: git [\"--git-dir=.git\",\"--work-tree=.\",\"--literal-pathspecs\",\"-c\",\"core.bare=false\",\"push\",\"annex\",\"git-annex\",\"master\"]
+    [2016-01-27 06:51:27.293901] process done ExitSuccess
+    [2016-01-27 06:51:27.294222] read: git [\"--git-dir=.git\",\"--work-tree=.\",\"--literal-pathspecs\",\"-c\",\"core.bare=false\",\"show-ref\",\"--hash\",\"refs/heads/git-annex\"]
+    [2016-01-27 06:51:27.297276] process done ExitSuccess
+    [2016-01-27 06:51:27.297782] read: git [\"--git-dir=.git\",\"--work-tree=.\",\"--literal-pathspecs\",\"-c\",\"core.bare=false\",\"log\",\"refs/heads/git-annex..c833bef56cf282eefb957347e2a1938ef8568da6\",\"-n1\",\"--pretty=%H\"]
+    [2016-01-27 06:51:27.300978] process done ExitSuccess
+    [2016-01-27 06:51:27.301324] read: git [\"--git-dir=.git\",\"--work-tree=.\",\"--literal-pathspecs\",\"-c\",\"core.bare=false\",\"log\",\"refs/heads/git-annex..7e6f7d4c9ff0bdb8f942c8edcb8a5a2581e7adb2\",\"-n1\",\"--pretty=%H\"]
+    [2016-01-27 06:51:27.304226] process done ExitSuccess
+    [2016-01-27 06:51:27.309109] read: git [\"--git-dir=.git\",\"--work-tree=.\",\"--literal-pathspecs\",\"-c\",\"core.bare=false\",\"show-ref\",\"git-annex\"]
+    [2016-01-27 06:51:27.30931] process done ExitSuccess
+    [2016-01-27 06:51:27.3116] process done ExitSuccess
+    [2016-01-27 06:51:27.311946] read: git [\"--git-dir=.git\",\"--work-tree=.\",\"--literal-pathspecs\",\"-c\",\"core.bare=false\",\"show-ref\",\"--hash\",\"refs/heads/git-annex\"]
+    [2016-01-27 06:51:27.314424] process done ExitSuccess
+    [2016-01-27 06:51:27.314803] read: git [\"--git-dir=.git\",\"--work-tree=.\",\"--literal-pathspecs\",\"-c\",\"core.bare=false\",\"log\",\"refs/heads/git-annex..c833bef56cf282eefb957347e2a1938ef8568da6\",\"-n1\",\"--pretty=%H\"]
+    [2016-01-27 06:51:27.317614] process done ExitSuccess
+    [2016-01-27 06:51:29.311876] Pusher: Syncing with annex 
+    [2016-01-27 06:51:29.312427] read: git [\"--git-dir=.git\",\"--work-tree=.\",\"--literal-pathspecs\",\"-c\",\"core.bare=false\",\"symbolic-ref\",\"-q\",\"HEAD\"]
+    [2016-01-27 06:51:29.315265] process done ExitSuccess
+    [2016-01-27 06:51:29.315673] read: git [\"--git-dir=.git\",\"--work-tree=.\",\"--literal-pathspecs\",\"-c\",\"core.bare=false\",\"show-ref\",\"refs/heads/annex/direct/master\"]
+    [2016-01-27 06:51:29.318964] process done ExitSuccess
+    [2016-01-27 06:51:29.319639] Pusher: pushing to [Remote { name =\"annex\" }]
+    [2016-01-27 06:51:29.32004] call: git [\"--git-dir=.git\",\"--work-tree=.\",\"--literal-pathspecs\",\"-c\",\"core.bare=false\",\"push\",\"annex\",\"+git-annex:synced/git-annex\",\"annex/direct/master:synced/master\"]
+    Everything up-to-date
+    [2016-01-27 06:51:29.329677] process done ExitSuccess
+    [2016-01-27 06:51:29.330047] read: git [\"--git-dir=.git\",\"--work-tree=.\",\"--literal-pathspecs\",\"-c\",\"core.bare=false\",\"push\",\"annex\",\"git-annex\",\"master\"]
+    [2016-01-27 06:51:29.339748] process done ExitSuccess
+    [2016-01-27 06:52:27.059869] read: git [\"--git-dir=.git\",\"--work-tree=.\",\"--literal-pathspecs\",\"-c\",\"core.bare=false\",\"ls-tree\",\"--full-tree\",\"-z\",\"--\",\"refs/heads/git-annex\",\"uuid.log\",\"remote.log\",\"trust.log\",\"group.log\",\"numcopies.log\",\"schedule.log\",\"preferred-content.log\",\"required-content.log\",\"group-preferred-content.log\"]
+    [2016-01-27 06:52:27.064751] process done ExitSuccess
+    [2016-01-27 06:55:25.738944] Watcher: file deleted Documents/Innovation.zip
+    [2016-01-27 06:55:25.75008] Committer: committing 1 changes
+    [2016-01-27 06:55:25.75047] Committer: Committing changes to git
+    (recording state in git...)
+    [2016-01-27 06:55:25.750868] feed: git [\"--git-dir=.git\",\"--work-tree=.\",\"--literal-pathspecs\",\"-c\",\"core.bare=false\",\"update-index\",\"-z\",\"--index-info\"]
+    [2016-01-27 06:55:25.753501] process done ExitSuccess
+    [2016-01-27 06:55:25.753793] read: git [\"--git-dir=.git\",\"--work-tree=.\",\"--literal-pathspecs\",\"-c\",\"core.bare=false\",\"symbolic-ref\",\"-q\",\"HEAD\"]
+    [2016-01-27 06:55:25.755492] process done ExitSuccess
+    [2016-01-27 06:55:25.755827] read: git [\"--git-dir=.git\",\"--work-tree=.\",\"--literal-pathspecs\",\"-c\",\"core.bare=false\",\"show-ref\",\"--hash\",\"refs/heads/annex/direct/master\"]
+    [2016-01-27 06:55:25.75798] process done ExitSuccess
+    [2016-01-27 06:55:25.758268] read: git [\"--git-dir=.git\",\"--work-tree=.\",\"--literal-pathspecs\",\"-c\",\"core.bare=false\",\"write-tree\"]
+    [2016-01-27 06:55:25.760247] process done ExitSuccess
+    [2016-01-27 06:55:25.760524] read: git [\"--git-dir=.git\",\"--work-tree=.\",\"--literal-pathspecs\",\"-c\",\"core.bare=false\",\"rev-parse\",\"db03e812e512e24afe7f4b440a6ca174aa33cbdd:\"]
+    [2016-01-27 06:55:25.762615] process done ExitSuccess
+    [2016-01-27 06:55:25.762953] read: git [\"--git-dir=.git\",\"--work-tree=.\",\"--literal-pathspecs\",\"-c\",\"core.bare=false\",\"symbolic-ref\",\"-q\",\"HEAD\"]
+    [2016-01-27 06:55:25.764649] process done ExitSuccess
+    [2016-01-27 06:55:25.764895] read: git [\"--git-dir=.git\",\"--work-tree=.\",\"--literal-pathspecs\",\"-c\",\"core.bare=false\",\"show-ref\",\"refs/heads/annex/direct/master\"]
+    [2016-01-27 06:55:25.766833] process done ExitSuccess
+    [2016-01-27 06:55:25.767095] call: git [\"--git-dir=.git\",\"--work-tree=.\",\"--literal-pathspecs\",\"-c\",\"core.bare=false\",\"branch\",\"-f\",\"synced/master\"]
+    [2016-01-27 06:55:25.769018] process done ExitSuccess
+    [2016-01-27 06:55:25.769381] call: git [\"--git-dir=.git\",\"--work-tree=.\",\"--literal-pathspecs\",\"-c\",\"core.bare=false\",\"branch\",\"-f\",\"master\"]
+    [2016-01-27 06:55:25.771227] process done ExitSuccess
+    [2016-01-27 06:55:25.771526] Pusher: Syncing with annex 
+    [2016-01-27 06:55:25.771855] read: git [\"--git-dir=.git\",\"--work-tree=.\",\"--literal-pathspecs\",\"-c\",\"core.bare=false\",\"symbolic-ref\",\"-q\",\"HEAD\"]
+    [2016-01-27 06:55:25.773533] process done ExitSuccess
+    [2016-01-27 06:55:25.773778] read: git [\"--git-dir=.git\",\"--work-tree=.\",\"--literal-pathspecs\",\"-c\",\"core.bare=false\",\"show-ref\",\"refs/heads/annex/direct/master\"]
+    [2016-01-27 06:55:25.775489] process done ExitSuccess
+    [2016-01-27 06:55:25.775781] Pusher: pushing to [Remote { name =\"annex\" }]
+    [2016-01-27 06:55:25.776187] call: git [\"--git-dir=.git\",\"--work-tree=.\",\"--literal-pathspecs\",\"-c\",\"core.bare=false\",\"push\",\"annex\",\"+git-annex:synced/git-annex\",\"annex/direct/master:synced/master\"]
+    Everything up-to-date
+    [2016-01-27 06:55:25.783416] process done ExitSuccess
+    [2016-01-27 06:55:25.78379] read: git [\"--git-dir=.git\",\"--work-tree=.\",\"--literal-pathspecs\",\"-c\",\"core.bare=false\",\"push\",\"annex\",\"git-annex\",\"master\"]
+    [2016-01-27 06:55:25.792144] process done ExitSuccess
+    [2016-01-27 06:56:55.488921] read: git [\"--git-dir=.git\",\"--work-tree=.\",\"--literal-pathspecs\",\"-c\",\"core.bare=false\",\"symbolic-ref\",\"-q\",\"HEAD\"]
+    [2016-01-27 06:56:55.491198] process done ExitSuccess
+    [2016-01-27 06:56:55.491606] read: git [\"--git-dir=.git\",\"--work-tree=.\",\"--literal-pathspecs\",\"-c\",\"core.bare=false\",\"show-ref\",\"refs/heads/annex/direct/master\"]
+    [2016-01-27 06:56:55.494862] process done ExitSuccess
+    [2016-01-27 06:56:55.495272] read: git [\"--git-dir=.git\",\"--work-tree=.\",\"--literal-pathspecs\",\"-c\",\"core.bare=false\",\"show-ref\",\"git-annex\"]
+    [2016-01-27 06:56:55.49766] process done ExitSuccess
+    [2016-01-27 06:56:55.498193] read: git [\"--git-dir=.git\",\"--work-tree=.\",\"--literal-pathspecs\",\"-c\",\"core.bare=false\",\"show-ref\",\"--hash\",\"refs/heads/git-annex\"]
+    [2016-01-27 06:56:55.50149] process done ExitSuccess
+    [2016-01-27 06:56:55.501967] read: git [\"--git-dir=.git\",\"--work-tree=.\",\"--literal-pathspecs\",\"-c\",\"core.bare=false\",\"log\",\"refs/heads/git-annex..c833bef56cf282eefb957347e2a1938ef8568da6\",\"-n1\",\"--pretty=%H\"]
+    [2016-01-27 06:56:55.505598] process done ExitSuccess
+    [2016-01-27 06:56:55.506265] read: git [\"--git-dir=.git\",\"--work-tree=.\",\"--literal-pathspecs\",\"-c\",\"core.bare=false\",\"symbolic-ref\",\"-q\",\"HEAD\"]
+    [2016-01-27 06:56:55.508634] process done ExitSuccess
+    [2016-01-27 06:56:55.50908] read: git [\"--git-dir=.git\",\"--work-tree=.\",\"--literal-pathspecs\",\"-c\",\"core.bare=false\",\"show-ref\",\"refs/heads/annex/direct/master\"]
+    [2016-01-27 06:56:55.512201] process done ExitSuccess
+    [2016-01-27 07:01:34.745946] Watcher: add Videos/SPOT0001.avi
+    [2016-01-27 07:01:34.757341] read: lsof [\"-F0can\",\"+d\",\".git/annex/misctmp/\"]
+    lsof: WARNING: can't stat() fuse.gvfsd-fuse file system /run/user/120/gvfs
+          Output information may be incomplete.
+    [2016-01-27 07:01:34.922312] process done ExitFailure 1
+
+      Videos/SPOT0001.avi still has writers, not adding
+    [2016-01-27 07:01:34.92397] Watcher: add Videos/SPOT0001.avi
+    [2016-01-27 07:01:35.93566] read: lsof [\"-F0can\",\"+d\",\".git/annex/misctmp/\"]
+    lsof: WARNING: can't stat() fuse.gvfsd-fuse file system /run/user/120/gvfs
+          Output information may be incomplete.
+    [2016-01-27 07:01:36.076194] process done ExitFailure 1
+    [2016-01-27 07:01:36.076529] Committer: Adding SPOT0001.avi
+    add Videos/SPOT0001.avi [2016-01-27 07:01:36.077018] read: sha256sum [\".git/annex/misctmp/SPOT00017523927541474612399.avi\"]
+    [2016-01-27 07:01:36.173532] process done ExitSuccess
+    [2016-01-27 07:01:36.18156] chat: git [\"--git-dir=.git\",\"--work-tree=.\",\"--literal-pathspecs\",\"-c\",\"core.bare=false\",\"hash-object\",\"-t\",\"blob\",\"-w\",\"--stdin\",\"--no-filters\"]
+    [2016-01-27 07:01:36.183854] process done ExitSuccess
+    ok
+    << trimmed because it worked >>
+
+I did end up leaving the zip in the folder, but I have tried in different subfolders and different zips, just not tar.gzs...
+"""]]
diff --git a/doc/forum/Syncing_Compressed_files/comment_3_885c045c8ddfccbd426a6bc1bc053435._comment b/doc/forum/Syncing_Compressed_files/comment_3_885c045c8ddfccbd426a6bc1bc053435._comment
new file mode 100644
index 0000000000..32e67c3f00
--- /dev/null
+++ b/doc/forum/Syncing_Compressed_files/comment_3_885c045c8ddfccbd426a6bc1bc053435._comment
@@ -0,0 +1,7 @@
+[[!comment format=mdwn
+ username="joey"
+ subject="""comment 3"""
+ date="2016-01-29T18:50:57Z"
+ content="""
+Is this on Linux? It may be some kind of problem with inotify..
+"""]]
diff --git a/doc/forum/Syncing_Compressed_files/comment_4_d619d9b7ad4ca3f2fa9505ea928d5bf7._comment b/doc/forum/Syncing_Compressed_files/comment_4_d619d9b7ad4ca3f2fa9505ea928d5bf7._comment
new file mode 100644
index 0000000000..65f44a0f6c
--- /dev/null
+++ b/doc/forum/Syncing_Compressed_files/comment_4_d619d9b7ad4ca3f2fa9505ea928d5bf7._comment
@@ -0,0 +1,17 @@
+[[!comment format=mdwn
+ username="moird"
+ subject="comment 4"
+ date="2016-01-29T19:43:21Z"
+ content="""
+yes, this is arch linux.  It seems to be more of an issue of the extension, I was toying with a zip file and renaming it and like if the extension was zipper then it would sync the file.  It would not sync the file if it was named with gz.  Was messing with inotifywatch and it is seeing the zip file creation:
+
+    total  create  filename
+    2      2       /home/daniel/annex/.git/refs/remotes/annex/synced/
+    2      2       /home/daniel/annex/.git/
+    1      1       /home/daniel/annex/.git/refs/heads/synced/
+    1      1       /home/daniel/annex/.git/refs/heads/
+    1      1       /home/daniel/annex/.git/refs/remotes/annex/
+    1      1       /home/daniel/annex/
+
+On successful files syncing will show the hashed annex folders 
+"""]]
diff --git a/doc/forum/Syncing_Compressed_files/comment_5_0c7af752cc35013273a07a5fa1f71f6e._comment b/doc/forum/Syncing_Compressed_files/comment_5_0c7af752cc35013273a07a5fa1f71f6e._comment
new file mode 100644
index 0000000000..574ba4e811
--- /dev/null
+++ b/doc/forum/Syncing_Compressed_files/comment_5_0c7af752cc35013273a07a5fa1f71f6e._comment
@@ -0,0 +1,7 @@
+[[!comment format=mdwn
+ username="moird"
+ subject="comment 5"
+ date="2016-01-30T19:01:28Z"
+ content="""
+well that figures... I forgot I had a global .gitignore file and it by default filtered out compressed files like zips and such.  Removed that restarted the agents, everything is good.
+"""]]
diff --git a/doc/forum/Syncing_machines_on_different_networks.mdwn b/doc/forum/Syncing_machines_on_different_networks.mdwn
new file mode 100644
index 0000000000..6851a6c016
--- /dev/null
+++ b/doc/forum/Syncing_machines_on_different_networks.mdwn
@@ -0,0 +1,9 @@
+I've been using git-annex locally for a couple months. So far I've only used it to keep track of files on my laptop and local usb hard drives. Now I would like to add a network into the picture, and hopefully start to move away from Dropbox.
+
+I have Dropbox on two computers: my home machine and my work machine. The home machine is only on when I'm at home and the work machine is only on when I'm at work, so the computers are never on at the same time and thus can never communicate directly. What are my options for keeping annexes on these two machines in sync?
+
+Initially I was hoping that I could use an S3 special remote for this, but I see that special remotes only hold the actual file data, not any of the git stuff. So I can't push my changes to S3 at work and then pull those changes in at home.
+
+From what I can tell from the documentation, the only way I can handle this problem is to have an annex sitting on a VPS or someplace that both my home and work machines can talk to. Is that correct? That would be ok, but if I'm going to put my annex out there in the cloud somewhere, I want the files to be encrypted. It looks like git-annex only supports encryption of file data with special remotes, not a full annex. Is there no way to have some sort of encrypted git-annex hub?
+
+I backed the assistant and have been following the development blog, but I haven't tried it out yet. Am I correct in thinking that nothing in the assistant will address this particular issue?
diff --git a/doc/forum/Syncing_machines_on_different_networks/comment_1_1c3523c722c178a96b096a68b9be4165._comment b/doc/forum/Syncing_machines_on_different_networks/comment_1_1c3523c722c178a96b096a68b9be4165._comment
new file mode 100644
index 0000000000..dcc59017b4
--- /dev/null
+++ b/doc/forum/Syncing_machines_on_different_networks/comment_1_1c3523c722c178a96b096a68b9be4165._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="https://www.google.com/accounts/o8/id?id=AItOawn7gQ1zZDdWhXy9H51W2krZYShNmKL3qfM"
+ nickname="Karsten"
+ subject="comment 1"
+ date="2012-11-02T07:21:15Z"
+ content="""
+I might be thinking too simple, but can't you just put another annex repository on an usbdrive and use it to carry the metadata around? Add it as a remote to both compuers annex repositories and sync when you come/leave. As it does not have to carry the actual data some 100M will usually suffice. Just don't use any special remotes, but simply a cloned git repository.
+"""]]
diff --git a/doc/forum/Syncing_machines_on_different_networks/comment_2_d7b14ffee65072329cfe9ab08a0dba50._comment b/doc/forum/Syncing_machines_on_different_networks/comment_2_d7b14ffee65072329cfe9ab08a0dba50._comment
new file mode 100644
index 0000000000..6f0f044967
--- /dev/null
+++ b/doc/forum/Syncing_machines_on_different_networks/comment_2_d7b14ffee65072329cfe9ab08a0dba50._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="annexuser"
+ ip="50.125.40.225"
+ subject="comment 2"
+ date="2012-11-02T19:41:27Z"
+ content="""
+That's what I've started to do. It gets the job done, but I don't like that I have to remember to plug in the stick and sync to it at the end of the work day, and then plug in the stick and sync from it when I get home. I'm hoping that there is some way to accomplish the syncing automagically, more akin to the Dropbox experience.
+"""]]
diff --git a/doc/forum/Syncing_machines_on_different_networks/comment_3_65d1dae9b76fccb5f2b8fd8c69b60075._comment b/doc/forum/Syncing_machines_on_different_networks/comment_3_65d1dae9b76fccb5f2b8fd8c69b60075._comment
new file mode 100644
index 0000000000..2fa7ac055c
--- /dev/null
+++ b/doc/forum/Syncing_machines_on_different_networks/comment_3_65d1dae9b76fccb5f2b8fd8c69b60075._comment
@@ -0,0 +1,15 @@
+[[!comment format=mdwn
+ username="http://joeyh.name/"
+ ip="4.152.108.194"
+ subject="comment 3"
+ date="2012-11-04T19:44:11Z"
+ content="""
+You need two things:
+
+1. A special remote in the cloud. S3, or rsync.net, or whatever.
+2. A git repository hosted in the cloud. You could just use github (paying them if you want to avoid it being public).
+
+Is it possible to have an encrypted git remote repository? It's not directly supported by git. The way git needs to be able to request arbitrary objects by SHA, kind of prevents encrypting things to any useful level. The only way I can think of to do that would be to use some cloud storage that can be mounted as a filesystem, and store an encrypted filesystem containing the git repo in a file inside that. Not easy.
+
+The git-annex assistant is going to get around this by not requiring a central git repository and transferring git data peer-to-peer, but that will require both the peers be on the network at the same time.
+"""]]
diff --git a/doc/forum/Syncing_machines_on_different_networks/comment_4_2ec67428af69d6c0ea051c6a67d58905._comment b/doc/forum/Syncing_machines_on_different_networks/comment_4_2ec67428af69d6c0ea051c6a67d58905._comment
new file mode 100644
index 0000000000..34d37ce4fd
--- /dev/null
+++ b/doc/forum/Syncing_machines_on_different_networks/comment_4_2ec67428af69d6c0ea051c6a67d58905._comment
@@ -0,0 +1,10 @@
+[[!comment format=mdwn
+ username="annexuser"
+ ip="50.125.40.225"
+ subject="comment 4"
+ date="2012-11-05T02:07:00Z"
+ content="""
+Thanks for the feedback.
+
+I'm thinking the smartest move will be for me to invest in some sort of wall-wart, like a SheevaPlug or RasberryPi. I can use that as the centralized hub and still avoid putting my files on the big bad interwebs.
+"""]]
diff --git a/doc/forum/Syncing_machines_on_different_networks/comment_5_5ce093f82a2aad3fd8d7ccd5fdcab94f._comment b/doc/forum/Syncing_machines_on_different_networks/comment_5_5ce093f82a2aad3fd8d7ccd5fdcab94f._comment
new file mode 100644
index 0000000000..c3d9d52b22
--- /dev/null
+++ b/doc/forum/Syncing_machines_on_different_networks/comment_5_5ce093f82a2aad3fd8d7ccd5fdcab94f._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="https://www.google.com/accounts/o8/id?id=AItOawn7gQ1zZDdWhXy9H51W2krZYShNmKL3qfM"
+ nickname="Karsten"
+ subject="comment 5"
+ date="2012-11-05T06:43:25Z"
+ content="""
+Just a small add-up: I've already used a truecrypt volume file on a dropbox successfully, although not for a git repository. Maybe that's worth a try.
+"""]]
diff --git a/doc/forum/Syncing_machines_on_different_networks/comment_6_a55982c28d7b90e0b70ec2bb5e594e08._comment b/doc/forum/Syncing_machines_on_different_networks/comment_6_a55982c28d7b90e0b70ec2bb5e594e08._comment
new file mode 100644
index 0000000000..d5ccdcf87c
--- /dev/null
+++ b/doc/forum/Syncing_machines_on_different_networks/comment_6_a55982c28d7b90e0b70ec2bb5e594e08._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="annexuser"
+ ip="24.16.193.140"
+ subject="comment 6"
+ date="2012-11-05T17:20:50Z"
+ content="""
+Right now I use Dropbox with encfs. I could do the same thing with git-annex, but it's kind of a hassle.
+"""]]
diff --git a/doc/forum/Syncing_machines_on_different_networks/comment_7_c519d546e1a2a4e834609f3de3a605b0._comment b/doc/forum/Syncing_machines_on_different_networks/comment_7_c519d546e1a2a4e834609f3de3a605b0._comment
new file mode 100644
index 0000000000..ea4aeeaf2b
--- /dev/null
+++ b/doc/forum/Syncing_machines_on_different_networks/comment_7_c519d546e1a2a4e834609f3de3a605b0._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="https://www.google.com/accounts/o8/id?id=AItOawnZEanlyzay_QlEAL0CWpyZcRTyN7vay8U"
+ nickname="Carlo"
+ subject="Quick Cloud Tip"
+ date="2012-11-21T13:16:54Z"
+ content="""
+If you have broadband, get a raspberry pi and hook up a thumb drive. Then allow ssh access through your router for really cheap cloud storage.
+"""]]
diff --git a/doc/forum/Syncing_machines_on_different_networks/comment_8_84a822238ddbaf211cce5f527c3559d3._comment b/doc/forum/Syncing_machines_on_different_networks/comment_8_84a822238ddbaf211cce5f527c3559d3._comment
new file mode 100644
index 0000000000..78c3190dbd
--- /dev/null
+++ b/doc/forum/Syncing_machines_on_different_networks/comment_8_84a822238ddbaf211cce5f527c3559d3._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="spwhitton"
+ ip="163.1.167.50"
+ subject="s3ql"
+ date="2012-11-29T17:58:35Z"
+ content="""
+I am achieving success on this issue using afuse to automatically mount an s3ql volume.  Though of course you have to manually push and pull from the git repo.
+"""]]
diff --git a/doc/forum/Syncing_with___39__include__39___rule_for_duplicate_files.mdwn b/doc/forum/Syncing_with___39__include__39___rule_for_duplicate_files.mdwn
new file mode 100644
index 0000000000..6121691b25
--- /dev/null
+++ b/doc/forum/Syncing_with___39__include__39___rule_for_duplicate_files.mdwn
@@ -0,0 +1,17 @@
+I have the following problem. I have file in git annex repo which is in two places in this repo.
+So there are two links to the file in the working tree.
+Let say (for clarity) that its path is: a/foo and b/foo.
+
+And when I do:
+
+    git annex wanted . "include=a/*"
+    git annex sync --content
+
+git-annex downloads the file and then drops it
+(i.e. it try to get a/foo and then drop b/foo).
+
+What should I do to avoid droping the file when include only
+one link to the file?
+
+The only one solution to the problem I've found is to use
+deprecated 'direct' mode, but let's say I want to do this the right way.
diff --git a/doc/forum/Syncing_with___39__include__39___rule_for_duplicate_files/comment_1_5d776e90c24c4eed14ae01c6019c76b8._comment b/doc/forum/Syncing_with___39__include__39___rule_for_duplicate_files/comment_1_5d776e90c24c4eed14ae01c6019c76b8._comment
new file mode 100644
index 0000000000..db2f4ea643
--- /dev/null
+++ b/doc/forum/Syncing_with___39__include__39___rule_for_duplicate_files/comment_1_5d776e90c24c4eed14ae01c6019c76b8._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="joey"
+ subject="""comment 1"""
+ date="2017-10-16T18:52:52Z"
+ content="""
+Unfortunately this is an open bug in git-annex:
+[[bugs/indeterminite_preferred_content_state_for_duplicated_file]]
+"""]]
diff --git a/doc/forum/Syncing_with_an_encrypted_remote_from_a_different_computer__63__.mdwn b/doc/forum/Syncing_with_an_encrypted_remote_from_a_different_computer__63__.mdwn
new file mode 100644
index 0000000000..9315d92180
--- /dev/null
+++ b/doc/forum/Syncing_with_an_encrypted_remote_from_a_different_computer__63__.mdwn
@@ -0,0 +1,4 @@
+I created an "full archive" repo on my local pc and an encrypted "full backup" repo on Box.com. I 'm copying files on the local repo and they are getting encrypted and uploaded to Box. Superb so far :)
+
+What I am wondering though is, suppose my local pc dies. How do I get the data out of Box unencrypted from a new pc?
+
diff --git a/doc/forum/Syncing_with_an_encrypted_remote_from_a_different_computer__63__/comment_1_cd55d06a4065b9d3f14d50674c3fcaf7._comment b/doc/forum/Syncing_with_an_encrypted_remote_from_a_different_computer__63__/comment_1_cd55d06a4065b9d3f14d50674c3fcaf7._comment
new file mode 100644
index 0000000000..5a5ec35894
--- /dev/null
+++ b/doc/forum/Syncing_with_an_encrypted_remote_from_a_different_computer__63__/comment_1_cd55d06a4065b9d3f14d50674c3fcaf7._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="https://me.yahoo.com/a/FHnTlSBo1eCGJRwueeKeB6.RCaPbGMPr5jxx8A--#ce0d8"
+ nickname="Hamza"
+ subject="comment 1"
+ date="2013-09-22T21:18:17Z"
+ content="""
+Just clone the repository on another computer or usb drive and enable box.com remote as long as you have the clone of the repo you can download your files back.
+"""]]
diff --git a/doc/forum/Syncing_with_an_encrypted_remote_from_a_different_computer__63__/comment_2_25cbdf478091af9923090e049c432a7d._comment b/doc/forum/Syncing_with_an_encrypted_remote_from_a_different_computer__63__/comment_2_25cbdf478091af9923090e049c432a7d._comment
new file mode 100644
index 0000000000..69db183e28
--- /dev/null
+++ b/doc/forum/Syncing_with_an_encrypted_remote_from_a_different_computer__63__/comment_2_25cbdf478091af9923090e049c432a7d._comment
@@ -0,0 +1,22 @@
+[[!comment format=mdwn
+ username="John"
+ ip="109.242.130.160"
+ subject="comment 2"
+ date="2013-09-22T22:20:22Z"
+ content="""
+Thank you Hamza!
+
+I 'm new on git, so please excuse my trivial questions:
+
+a) I am using the git-annex assistant, is it something I can do from there or is it command line only?
+I googled a bit and from what I can tell, I should make a directory on the usb drive, go there and do 
+> $ git clone /path/to/fullArchiveRepo
+
+Would that be correct?
+
+b) Assuming I 've done it correctly, then I put the USB on a drawer and leave it there for a month. In the meantime, I 've been using the repo on my pc and more files have been archived encrypted on Box.com. Then my local pc dies. When I plug the usb on the new pc, will I be able to recover all the encrypted files, or only those up to 1 month ago?
+
+c) What is the proper process to use the cloned repo on a new pc? Plug the usb drive, open the git-annex assistant and go through the \"create new repo\" but use the path for the existing repo on the usb? Then add another repo from Box (with the same account and the same directory there? Would that work?
+
+Thank you for your time & knowledge! :)
+"""]]
diff --git a/doc/forum/Syncing_with_an_encrypted_remote_from_a_different_computer__63__/comment_3_7e71d355457d6b1a0391d4cdae6895e6._comment b/doc/forum/Syncing_with_an_encrypted_remote_from_a_different_computer__63__/comment_3_7e71d355457d6b1a0391d4cdae6895e6._comment
new file mode 100644
index 0000000000..81de3fc459
--- /dev/null
+++ b/doc/forum/Syncing_with_an_encrypted_remote_from_a_different_computer__63__/comment_3_7e71d355457d6b1a0391d4cdae6895e6._comment
@@ -0,0 +1,16 @@
+[[!comment format=mdwn
+ username="https://me.yahoo.com/a/FHnTlSBo1eCGJRwueeKeB6.RCaPbGMPr5jxx8A--#ce0d8"
+ nickname="Hamza"
+ subject="comment 3"
+ date="2013-09-23T00:34:24Z"
+ content="""
+I do not use the assistant I prefer command line but try adding an USB drive (last I used it it had the option) It should do the clone and init it check the resulting folder if it contains a .git/ folder you have a clone of your git repo.
+
+a) For manual cloning follow http://git-annex.branchable.com/walkthrough/adding_a_remote/
+
+b) you need to keep syncing to the clone too. asistant should automatically sync to that repo. AFAIK asisstant detects when the usb repo is plugged and automatically syncs to it (again I do not use it but I seem to remember one of joey's talks showing that. YMMV)
+
+If you lose all your repos then you lose the keys to un encrypt files they are gone!, if you have a outdated repo you can get the files back using the key stored in it but without the directory structure.
+
+Correct workflow depends on how you use annex. I sync 3 computers with annex so if one dies I can clone the repo from another one. But if you are only using it on a single computer I would use a clone on an external usb drive that is always connected, so you have two clones one on the internal disk and one on the external disk so you can survive one of the drives crashing.
+"""]]
diff --git a/doc/forum/Syncing_with_an_encrypted_remote_from_a_different_computer__63__/comment_4_a73f67f2fcf0762fbd7c8366b3844af6._comment b/doc/forum/Syncing_with_an_encrypted_remote_from_a_different_computer__63__/comment_4_a73f67f2fcf0762fbd7c8366b3844af6._comment
new file mode 100644
index 0000000000..7e649f858c
--- /dev/null
+++ b/doc/forum/Syncing_with_an_encrypted_remote_from_a_different_computer__63__/comment_4_a73f67f2fcf0762fbd7c8366b3844af6._comment
@@ -0,0 +1,11 @@
+[[!comment format=mdwn
+ username="http://joeyh.name/"
+ ip="4.152.108.220"
+ subject="comment 4"
+ date="2013-09-23T20:18:38Z"
+ content="""
+Other good options to supplement an offline backup drive:
+
+* A clone of the repository on another computer of yours, or on an Adroid tablet or phone.
+* An encrypted git repository stored on a remote ssh server. (Supported by recent git-annex releases, although the assistant does not yet have a UI to set this up it's not very hard to do it manually at the command line and then the assistant will use it.)
+"""]]
diff --git a/doc/forum/Syncronisation_of_syncronisation_between_3_repositories__63__.mdwn b/doc/forum/Syncronisation_of_syncronisation_between_3_repositories__63__.mdwn
new file mode 100644
index 0000000000..988ec30295
--- /dev/null
+++ b/doc/forum/Syncronisation_of_syncronisation_between_3_repositories__63__.mdwn
@@ -0,0 +1,11 @@
+Hello Joey,
+
+I just want to know if file transfers between three inter-connected repositories somehow gets syncronized. I have a laptop, a local file server in my home and a virtual server on the internet.
+
+Both my laptop and my file server are configured with a full git repository and connected through pairing and xmpp. The server on the internet is configured as a rsync special remote.
+
+I want the local file server to hold a copy of everything in my annex, the rsync remote should get everything except the folder "media" (too large to upload and not that important) and the laptop whatever I manually decide (preferred content is set to "present").
+
+I have added some files to the repository on the local file server and transferred their content to my laptop by using "git annex get". But when I started the web interface, I saw that those files who are present on both the file server and on the laptop get uploaded to the remote server from both computers, often the same file at the same time. Previously I though the two assistants would somehow talk to each other via xmpp so that doubled effort like that would be avoided. I'm also worried about data consistency because two apparently separate processes rsync to the same remote repository at the same time.
+
+So my question would be: Is your xmpp protocol designed to deal with situations like this and I did not set it up correctly or is what I'm trying to accomplish simply not (yet) possible?
diff --git a/doc/forum/Syncronisation_of_syncronisation_between_3_repositories__63__/comment_1_ca5192a26950627a1c2efcb55d6d2fa3._comment b/doc/forum/Syncronisation_of_syncronisation_between_3_repositories__63__/comment_1_ca5192a26950627a1c2efcb55d6d2fa3._comment
new file mode 100644
index 0000000000..dccd1d8d5e
--- /dev/null
+++ b/doc/forum/Syncronisation_of_syncronisation_between_3_repositories__63__/comment_1_ca5192a26950627a1c2efcb55d6d2fa3._comment
@@ -0,0 +1,10 @@
+[[!comment format=mdwn
+ username="http://joeyh.name/"
+ ip="4.154.7.238"
+ subject="comment 1"
+ date="2013-01-14T16:22:33Z"
+ content="""
+git-annex does not currently prevent multiple uploads of the same file to a rsync special remote. It is able to guard against this when uploading to a git remote, since then the remote runs git-annex-shell, which can detect when an upload is already running. You might want to convert your rsync remote to a git remote.
+
+I hope that rsyncing the same file twice at the same time is safe, but it's certainly excessively expensive.
+"""]]
diff --git a/doc/forum/Termux_Android_8.1_Proot_Error.mdwn b/doc/forum/Termux_Android_8.1_Proot_Error.mdwn
new file mode 100644
index 0000000000..91e7cf4538
--- /dev/null
+++ b/doc/forum/Termux_Android_8.1_Proot_Error.mdwn
@@ -0,0 +1,17 @@
+Hi,
+
+I'm trying to get git-annex working on my Google Pixel running Android version 8.1 through Termux but I'm having some difficulty.
+
+I copy-pasta'd the instructions found here:  until I got the following error when trying to run git-annex.linux/runshell
+
+    Running on Android.. Adding git-annex to PATH for you, and tuning for optimal behavior
+    proot error: execve("/data/data/com.termux/files/home/git-annex.linux/bin/sh"): No such file or directory
+    proot info: possible causes
+      * the program is a script but its interpreter (eg. /bin/sh) was not found;
+      * the program is an ELF but its interpreter (eg. ld-linux.so) was not found;
+      * the program is a foreign binary but qemu was not specified;
+      * qemu does not work correctly (if specified);
+      * the loader was not found or doesn't work.
+    fatal error: see `proot --help`.
+
+If I could get some help troubleshooting this that would be great. Thanks!
diff --git a/doc/forum/Termux_Android_8.1_Proot_Error/comment_1_1ef5f1b83e1bc0061d9fb1bba601a692._comment b/doc/forum/Termux_Android_8.1_Proot_Error/comment_1_1ef5f1b83e1bc0061d9fb1bba601a692._comment
new file mode 100644
index 0000000000..2ea415da1c
--- /dev/null
+++ b/doc/forum/Termux_Android_8.1_Proot_Error/comment_1_1ef5f1b83e1bc0061d9fb1bba601a692._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="joey"
+ subject="""comment 1"""
+ date="2018-07-02T16:10:36Z"
+ content="""
+termux-fix-shebang has been reported to fix that problem, some of the
+comments to the page you linked to discuss it.
+"""]]
diff --git a/doc/forum/The_ability_to_leave_a_file_unlocked_for_a_bit_while_committing_it_repeatedly__63__.mdwn b/doc/forum/The_ability_to_leave_a_file_unlocked_for_a_bit_while_committing_it_repeatedly__63__.mdwn
new file mode 100644
index 0000000000..627e3b7360
--- /dev/null
+++ b/doc/forum/The_ability_to_leave_a_file_unlocked_for_a_bit_while_committing_it_repeatedly__63__.mdwn
@@ -0,0 +1,3 @@
+Sometimes I'm doing something crazy like working in a large GIMP file, and I just want to be able to keep "committing" the changes to the annex but I hate running into the problem of having to keep unlocking the file over and over again while saving.  I guess one solution to this might be to use git-annex assistant?  But I'm not sure I really want a daemon running, and I don't want this auto-added... I just want to be able to leave the file unlocked while I'm actively working on it, then eventually say "I'm done with this!" and set it back to a state where it's locked.
+
+Basically, I hate running into "permission denied" errors all the time when I'm saving!  I guess I could do some sort of command where I commit to the annex and then immediately unlock again... is there a better way?
diff --git a/doc/forum/The_ability_to_leave_a_file_unlocked_for_a_bit_while_committing_it_repeatedly__63__/comment_1_3cbe520b184d323219cb402ff046c3b4._comment b/doc/forum/The_ability_to_leave_a_file_unlocked_for_a_bit_while_committing_it_repeatedly__63__/comment_1_3cbe520b184d323219cb402ff046c3b4._comment
new file mode 100644
index 0000000000..fac3628923
--- /dev/null
+++ b/doc/forum/The_ability_to_leave_a_file_unlocked_for_a_bit_while_committing_it_repeatedly__63__/comment_1_3cbe520b184d323219cb402ff046c3b4._comment
@@ -0,0 +1,29 @@
+[[!comment format=mdwn
+ username="http://identi.ca/cwebber/"
+ nickname="cwebber"
+ subject="Reasonable solution?"
+ date="2012-11-27T21:24:49Z"
+ content="""
+I think I have built a reasonable solution... this function checks a specific file into git and then immediately unlocks it.
+
+    function git-annex-unlocked-commit {
+        if [ $# -ne 2 ]; then
+            echo \"Wrong number of args.\"
+            return 1
+        fi
+    
+        if [ ! -e \"$1\" ]; then
+            echo \"Need a filename!\"
+            return 1
+        fi
+    
+        read -p \"Really do an unlocked commit? (y/n): \"
+        if [ \"$REPLY\" == \"y\" ]; then
+            git commit $1 -m \"$2\" && git annex unlock $1
+        fi
+    }
+
+Use it like:
+
+    git-annex-unlocked-commit mediagoblin_postcard.xcf \"Starting to fill in spencer\"
+"""]]
diff --git a/doc/forum/The_ability_to_leave_a_file_unlocked_for_a_bit_while_committing_it_repeatedly__63__/comment_2_6afe7f593e955db2eefe87d9fa01882b._comment b/doc/forum/The_ability_to_leave_a_file_unlocked_for_a_bit_while_committing_it_repeatedly__63__/comment_2_6afe7f593e955db2eefe87d9fa01882b._comment
new file mode 100644
index 0000000000..c515372b13
--- /dev/null
+++ b/doc/forum/The_ability_to_leave_a_file_unlocked_for_a_bit_while_committing_it_repeatedly__63__/comment_2_6afe7f593e955db2eefe87d9fa01882b._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="http://joeyh.name/"
+ ip="4.154.6.49"
+ subject="comment 2"
+ date="2012-11-27T21:27:33Z"
+ content="""
+The only thing I would do differently is \"git annex add\" the file before \"git commit\". This is more efficient for large files since it avoids git looking at the whole file content.
+"""]]
diff --git a/doc/forum/The_ability_to_leave_a_file_unlocked_for_a_bit_while_committing_it_repeatedly__63__/comment_3_209399487fc4f76b29f03ad82dbc2d6f._comment b/doc/forum/The_ability_to_leave_a_file_unlocked_for_a_bit_while_committing_it_repeatedly__63__/comment_3_209399487fc4f76b29f03ad82dbc2d6f._comment
new file mode 100644
index 0000000000..c59c7fe6f8
--- /dev/null
+++ b/doc/forum/The_ability_to_leave_a_file_unlocked_for_a_bit_while_committing_it_repeatedly__63__/comment_3_209399487fc4f76b29f03ad82dbc2d6f._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="http://identi.ca/cwebber/"
+ nickname="cwebber"
+ subject="comment 3"
+ date="2012-11-27T21:30:11Z"
+ content="""
+Just in case, since I don't see licensing stuff on this wiki: copyright to that code waived under CC0. :) http://creativecommons.org/publicdomain/zero/1.0/
+"""]]
diff --git a/doc/forum/The_ability_to_leave_a_file_unlocked_for_a_bit_while_committing_it_repeatedly__63__/comment_4_f33fd6f72cb9ad7dd20a04c82199413b._comment b/doc/forum/The_ability_to_leave_a_file_unlocked_for_a_bit_while_committing_it_repeatedly__63__/comment_4_f33fd6f72cb9ad7dd20a04c82199413b._comment
new file mode 100644
index 0000000000..ba6baad91e
--- /dev/null
+++ b/doc/forum/The_ability_to_leave_a_file_unlocked_for_a_bit_while_committing_it_repeatedly__63__/comment_4_f33fd6f72cb9ad7dd20a04c82199413b._comment
@@ -0,0 +1,26 @@
+[[!comment format=mdwn
+ username="http://identi.ca/cwebber/"
+ nickname="cwebber"
+ subject="comment 4"
+ date="2012-11-27T23:58:10Z"
+ content="""
+Ah, thanks Joey... here's an updated version:
+
+    function git-annex-unlocked-commit {
+        if [ $# -ne 2 ]; then
+            echo \"Wrong number of args.\"
+            return 1
+        fi
+    
+        if [ ! -e \"$1\" ]; then
+            echo \"Need a filename!\"
+            return 1
+        fi
+    
+        read -p \"Really do an unlocked commit? (y/n): \"
+        if [ \"$REPLY\" == \"y\" ]; then
+            git add $1 && git commit $1 -m \"$2\" && git annex unlock $1
+        fi
+    }
+
+"""]]
diff --git a/doc/forum/The_future_of_Direct_Mode.mdwn b/doc/forum/The_future_of_Direct_Mode.mdwn
new file mode 100644
index 0000000000..8c295eb1f4
--- /dev/null
+++ b/doc/forum/The_future_of_Direct_Mode.mdwn
@@ -0,0 +1,32 @@
+Hello everyone. I could use some pointers from experienced users. Here is my use-case:
+
+I have a number of portable storage devices where I store media files. They serve a number of purposes:
+
+  1. as archives and backup where I put away the data I am not using to allow for more free space on my work PC;
+
+  2. as storage for the media I play on a media playing device (that only supports Windows filesystems);
+
+  3. as transport medium to share files with other people (that are Windows-users).
+
+I wanted to use git-annex to manage/organize/locate all these devices, but without breaking too much my system. Most files are not super-important, and I intend to keep multiple copies of those that are.
+The main issue is that points 2 and 3 require that the real files need to be accessible directly. I *could* work around this using an extra device, but that would be much less practical, as I would need to go to the PC and transfer the large files over USB, so I am trying to avoid that.
+
+Here is what I learned so far:
+
+  - Indirect mode is OK on my PC. No problem there. However, it won't work on VFAT or NTFS, so I can't use it on the portable devices.
+
+  - Direct mode seems to work fine on the portable devices, but is now deprecated. It won't work on the V6 repositories.
+
+  - The new V6 repositories allows for the unlocking of files, replacing the link-like files (on Windows filesystems) with the real files, that allows me to browse them on the media player (by path and name). I did not find a way to keep them always unlocked by default, so I guess I would need to unlock them manually. Also, I would be keeping an extra copy of each file, since `annex.thin` won't work. In addition, as I don't expect to be editing any these files (not on these devices, at least), the loss of half the storage capacity is wasteful and undesirable.
+
+  - Special remotes `directory` and `rsync` do not keep the original structure of the system, and for this reason do not fit for my purposes.
+
+For now I can create V5 repositories and use direct mode on the external devices. I think this could work quite well, but I have the following questions:
+
+  1. Am I missing something that could help me?
+
+  2. Will using V5 repositories be somewhat future-proof? I mean, will I be able to create V5 repositories in the future? How much of a bad idea would it be to start using git-annex depending on "old versions"? I understand that direct mode will continue to be supported, but deprecation indicates it is a bad idea.
+
+  3. I am guessing that "direct mode" will not be making a comeback, but it does have its uses — however, I did not expect that my case was an unusual one. Is there anything on V6 repositories that would make new version more adequate to my requirements? I would not mind having the files read-only or facing similar restrictions if it helped simplify things.
+
+Thank you for your time.
diff --git a/doc/forum/The_future_of_Direct_Mode/comment_1_f16f6e41f01d59ccef9666a2ec46d294._comment b/doc/forum/The_future_of_Direct_Mode/comment_1_f16f6e41f01d59ccef9666a2ec46d294._comment
new file mode 100644
index 0000000000..64f7373c27
--- /dev/null
+++ b/doc/forum/The_future_of_Direct_Mode/comment_1_f16f6e41f01d59ccef9666a2ec46d294._comment
@@ -0,0 +1,32 @@
+[[!comment format=mdwn
+ username="joey"
+ subject="""comment 1"""
+ date="2016-02-12T18:21:38Z"
+ content="""
+I guess that fully getting rid of direct mode is a couple of years out.
+
+First, [[design/adjusted_branches]] needs to be implemented, to allow
+locked files to appear as unlocked automatically when a repository is
+checked out on a device that doesn't support symlinks.
+
+Then, a git-annex version with good enough support for v6 unlocked files
+needs to have been around long enough that basically everyone has upgraded.
+So in various stable and long term support version of linux distros, etc.
+Only at that point could direct mode repositories be automatically updated
+to v6 repositories.
+
+And yeah, it's potentially a problem that annex.thin doesn't work on
+Windows or FAT, and so converting such a repository doubles the disk use.
+I've added a note to [[todo/smudge]] to remember this is a potential thing
+to be improved before getting rid of direct mode. I think there are a
+couple of approaches that could avoid the problem.
+
+So, I recommend not being put off from using direct mode today out of
+worries that it might be removed 3 years from now.
+
+(You mentioned repeatedly unlocking. You shouldn't need to do that.
+In a V6 repository, when you unlock a file, that changes how it's
+represented in the git repository, so you can commit that change and it
+will be unlocked in other clones too. And if you `git add` large files,
+they'll be unlocked by default and you don't need to unlock them.)
+"""]]
diff --git a/doc/forum/The_future_of_Direct_Mode/comment_2_c7408d45d0a2a854020e4169fe757d51._comment b/doc/forum/The_future_of_Direct_Mode/comment_2_c7408d45d0a2a854020e4169fe757d51._comment
new file mode 100644
index 0000000000..e0448a9615
--- /dev/null
+++ b/doc/forum/The_future_of_Direct_Mode/comment_2_c7408d45d0a2a854020e4169fe757d51._comment
@@ -0,0 +1,7 @@
+[[!comment format=mdwn
+ username="Gus"
+ subject="Thanks"
+ date="2016-02-12T21:47:53Z"
+ content="""
+Thank you for your help.
+"""]]
diff --git a/doc/forum/This_account_is_restricted_by_rssh._Allowed_commands__58___scp_rsync__160__.mdwn b/doc/forum/This_account_is_restricted_by_rssh._Allowed_commands__58___scp_rsync__160__.mdwn
new file mode 100644
index 0000000000..932c78a9a8
--- /dev/null
+++ b/doc/forum/This_account_is_restricted_by_rssh._Allowed_commands__58___scp_rsync__160__.mdwn
@@ -0,0 +1,5 @@
+Hello,
+
+I'm using the git-annex assistant (in Mac Mavericks) and I'm trying to create a new remote rsync repo, I have the details and everything but I can't do it over ssh, I need the call to use rsync otherwise I get: **This account is restricted by rssh. Allowed commands: scp rsync ** . Can this be changed manually? How can I create a remote rsync repo?
+
+Thanks
diff --git a/doc/forum/This_account_is_restricted_by_rssh._Allowed_commands__58___scp_rsync__160__/comment_1_68e911629da672473bd6188407a68be2._comment b/doc/forum/This_account_is_restricted_by_rssh._Allowed_commands__58___scp_rsync__160__/comment_1_68e911629da672473bd6188407a68be2._comment
new file mode 100644
index 0000000000..b6fa2d1a18
--- /dev/null
+++ b/doc/forum/This_account_is_restricted_by_rssh._Allowed_commands__58___scp_rsync__160__/comment_1_68e911629da672473bd6188407a68be2._comment
@@ -0,0 +1,13 @@
+[[!comment format=mdwn
+ username="http://joeyh.name/"
+ ip="209.250.56.54"
+ subject="comment 1"
+ date="2014-10-02T15:33:58Z"
+ content="""
+You can set it up manually at the command line, and one it's set up, the assistant will be able to transfer files to it using rsync. See [[special_remotes/rsync]] for some setup examples. Note that the rsync server will need to be configured, somehow, to let you log in without giving a password -- one typical way this might be set up is using .ssh/authorized_keys on the rsync server.
+
+The assistant doesn't try to set up rsync special remotes because this is not very common, and there's no one way it can use to set up a ssh key so it can log in without a password
+that will work across different rsync server setups. It does support setting up rsync.net just because that's one I know how to handle.
+
+
+"""]]
diff --git a/doc/forum/Timeline_of_git_reinject__63__.mdwn b/doc/forum/Timeline_of_git_reinject__63__.mdwn
new file mode 100644
index 0000000000..af6761de4b
--- /dev/null
+++ b/doc/forum/Timeline_of_git_reinject__63__.mdwn
@@ -0,0 +1,50 @@
+# Context
+
+I'm currently using `git annex reinject --known` to deduplicate directories containing dozens number of huge (up to 4-13 Gb) files.
+Let's focus on one example big file.
+
+The file being reinjected is not already available in `.git/annex/objects`. It will be after `git annex reinject --known` completes.
+The file being reinjected is on a different filesystem on the same disk. This might be important.
+
+# Time taken to process one file.
+
+It's done in the background on a server and yields a log that shows how much time passes.
+
+It looks like:
+
+```
+reinject my_big_file.dv    (7 minutes pass)
+(checksum...)    (20 minutes pass)
+ok
+```
+
+`my_big_file.dv` is 8.7G big. 
+
+With the USB2 bandwith available, reading that file can take between 7 and 12 minutes.
+
+# What happens?
+
+* 7 minutes is a reasonable time to read the whole file
+* after "checksum..." appears, 20 minutes pass which is a reasonable time to move the file to the partition containing git-annex repository ... or to read it twice?
+
+This looks "mostly reasonable", perhaps a little long.
+
+Source code in Hash.hs says:
+
+	mstat <- liftIO $ catchMaybeIO $ getFileStatus file
+	case (mstat, fast) of
+		(Just stat, False) -> do
+			filesize <- liftIO $ getFileSize' file stat
+			showAction "checksum"
+			check <$> hashFile hash file filesize
+		_ -> return True
+
+
+I expected "checksum..." to appear *before* the checksum is actually computed, and source code appears to confirm that (trying to compensate ignorance of Haskell with knowledge of OCaml, pure functions, closures, functional programming, including C# and reactive programming).
+
+# Questions
+
+* Is it true that checksum is computed after "checksum..." appears?
+* Why do 7 minute pass before "checksum..." appear? What happens?
+* What happens in the 20 minutes after "checksum..." appear and before "ok"?
+
diff --git a/doc/forum/Timeline_of_git_reinject__63__/comment_1_e88d612590d3cd0c2ea824c1e8a38678._comment b/doc/forum/Timeline_of_git_reinject__63__/comment_1_e88d612590d3cd0c2ea824c1e8a38678._comment
new file mode 100644
index 0000000000..50d9aa68fb
--- /dev/null
+++ b/doc/forum/Timeline_of_git_reinject__63__/comment_1_e88d612590d3cd0c2ea824c1e8a38678._comment
@@ -0,0 +1,19 @@
+[[!comment format=mdwn
+ username="joey"
+ subject="""comment 1"""
+ date="2016-11-07T16:02:28Z"
+ content="""
+What turns out to have been going on here is the file was first checksummed
+silently to get the key and see if it is --known, 
+and then checksummed a second time (with the message displayed) as part of
+the reinject process.
+
+So, the second checksum is not needed in --known mode and I've made it not
+be done.
+
+It might be that the "(checksum)" message should be displayed during the
+intial checksum of the file. git-annex used to always say when it
+checksummed, but [[!commit 64160a96795d03ee791faa4757057200934687bc]] got
+rid of that in most cases. I guess that "reinject bigfile <13 minute wait> ok"
+is acceptable output though.
+"""]]
diff --git a/doc/forum/Tracking_a_directory_with_some_hg_and_git_repositories.mdwn b/doc/forum/Tracking_a_directory_with_some_hg_and_git_repositories.mdwn
new file mode 100644
index 0000000000..898d4870ae
--- /dev/null
+++ b/doc/forum/Tracking_a_directory_with_some_hg_and_git_repositories.mdwn
@@ -0,0 +1 @@
+In my documents folder I have some Mercurial and Git repositories. Are there any side effects if I track the whole directory with git annex?
diff --git a/doc/forum/Tracking_a_directory_with_some_hg_and_git_repositories/comment_1_9fc3f6c2f7379755e0084a850fa9acd4._comment b/doc/forum/Tracking_a_directory_with_some_hg_and_git_repositories/comment_1_9fc3f6c2f7379755e0084a850fa9acd4._comment
new file mode 100644
index 0000000000..c61ea9a4d5
--- /dev/null
+++ b/doc/forum/Tracking_a_directory_with_some_hg_and_git_repositories/comment_1_9fc3f6c2f7379755e0084a850fa9acd4._comment
@@ -0,0 +1,10 @@
+[[!comment format=mdwn
+ username="http://joeyh.name/"
+ ip="209.250.56.227"
+ subject="comment 1"
+ date="2013-12-27T19:05:59Z"
+ content="""
+I don't recommend using git-annex that way.
+
+.git directories cannot be checked into git, so will just be skipped. A hg repository will be checked in, but you can get into all kinds of bad situations if making commits to the \"same\" hg repository in two different locations. You will end up having to manually resolve conflicted merges to the files used by your distributed version control system, which defeats the point of using distributed version control.
+"""]]
diff --git a/doc/forum/Tracking_a_directory_with_some_hg_and_git_repositories/comment_2_f024d6a105143af1e06aafe49661ee06._comment b/doc/forum/Tracking_a_directory_with_some_hg_and_git_repositories/comment_2_f024d6a105143af1e06aafe49661ee06._comment
new file mode 100644
index 0000000000..50b94ffaf6
--- /dev/null
+++ b/doc/forum/Tracking_a_directory_with_some_hg_and_git_repositories/comment_2_f024d6a105143af1e06aafe49661ee06._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="https://www.google.com/accounts/o8/id?id=AItOawnX1msQxnLoSeu7q-i-c9BWghonsN7Qmns"
+ nickname="Jan Ulrich"
+ subject="Thanks!"
+ date="2013-12-27T20:02:20Z"
+ content="""
+Ok, I feared that answer. I'll have to change my directory structure. 
+"""]]
diff --git a/doc/forum/Tracking_a_directory_with_some_hg_and_git_repositories/comment_3_aff0093c38bda8b093f05e8cbe8775e9._comment b/doc/forum/Tracking_a_directory_with_some_hg_and_git_repositories/comment_3_aff0093c38bda8b093f05e8cbe8775e9._comment
new file mode 100644
index 0000000000..7108ddea3a
--- /dev/null
+++ b/doc/forum/Tracking_a_directory_with_some_hg_and_git_repositories/comment_3_aff0093c38bda8b093f05e8cbe8775e9._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="http://joeyh.name/"
+ ip="209.250.56.227"
+ subject="comment 3"
+ date="2013-12-27T20:08:07Z"
+ content="""
+Well, you can always use .gitignore to ignore the whole content of the directory that you don't want git-annex to handle.
+"""]]
diff --git a/doc/forum/Tracking_a_directory_with_some_hg_and_git_repositories/comment_4_c889050d3079edefc4633451bd5baff8._comment b/doc/forum/Tracking_a_directory_with_some_hg_and_git_repositories/comment_4_c889050d3079edefc4633451bd5baff8._comment
new file mode 100644
index 0000000000..55f9f2534e
--- /dev/null
+++ b/doc/forum/Tracking_a_directory_with_some_hg_and_git_repositories/comment_4_c889050d3079edefc4633451bd5baff8._comment
@@ -0,0 +1,11 @@
+[[!comment format=mdwn
+ username="https://www.google.com/accounts/o8/id?id=AItOawnvVfFLW4CTKs7UjdiLIsOn_cxj1Jnh64I"
+ nickname="Charl"
+ subject="use case: changing workstations mid-way through development session"
+ date="2014-03-23T21:22:15Z"
+ content="""
+An important use case for me with automatic synchronization, is being able to change computers mid-way through a development session. I used to do this with dropbox for years: I'd work in a git checkout, then take another laptop to go work elsewhere, and just continue in the \"same\" synchronized git repo. Having to remember to commit and push a long list of repos before changing workstations is error-prone. Also, I prefer to commit when a discrete atom of work has been completed, and to push when a topic branch is good and ready to be merged (for example), not every time that I feel like changing workstations. :) Dropbox never dropped a stitch in all this time. As long as all laptops are in a 1) connected and 2) synced up state, switching is possible.
+
+The limitation of not being able to sync the .git directories in git checkouts means that this use case can't currently be supported by git-annex (assistant). Is a work-around possible, or is this something that can't ever be supported due to the limitations of git itself?
+
+"""]]
diff --git a/doc/forum/Transfer_remotes.mdwn b/doc/forum/Transfer_remotes.mdwn
new file mode 100644
index 0000000000..eb03f3166e
--- /dev/null
+++ b/doc/forum/Transfer_remotes.mdwn
@@ -0,0 +1,3 @@
+Hi!
+
+I have a laptop A on which I setup a box.com special remote as a transfer encrypted repository. It is quite unclear to me how I should configure laptop B to use the same box.com remote and share files between the two machines.
diff --git a/doc/forum/Transfer_remotes/comment_1_c08cf3bda00d7f20a3ca3d0fdba19c9c._comment b/doc/forum/Transfer_remotes/comment_1_c08cf3bda00d7f20a3ca3d0fdba19c9c._comment
new file mode 100644
index 0000000000..7b5de17a0d
--- /dev/null
+++ b/doc/forum/Transfer_remotes/comment_1_c08cf3bda00d7f20a3ca3d0fdba19c9c._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="https://www.google.com/accounts/o8/id?id=AItOawniayrgSdVLUc3c6bf93VbO-_HT4hzxmyo"
+ nickname="Tobias"
+ subject="Screencast"
+ date="2013-04-11T09:59:24Z"
+ content="""
+Maybe this screencasts helps: [git-annex assistant remote sharing](http://git-annex.branchable.com/videos/git-annex_assistant_remote_sharing/)
+"""]]
diff --git a/doc/forum/Transfer_remotes/comment_2_98930629d398329f1161135464a966a5._comment b/doc/forum/Transfer_remotes/comment_2_98930629d398329f1161135464a966a5._comment
new file mode 100644
index 0000000000..1ce26e5b20
--- /dev/null
+++ b/doc/forum/Transfer_remotes/comment_2_98930629d398329f1161135464a966a5._comment
@@ -0,0 +1,12 @@
+[[!comment format=mdwn
+ username="http://cyprio.net/"
+ nickname="oz"
+ subject="versions?"
+ date="2013-04-15T22:50:31Z"
+ content="""
+Well, in the screencast it seems to work alright.
+
+I only get a \"Syncing with laptop B\" on one side, nothing more happens. I'm not sure if there is a recommended version to follow this screencast, or if the versions need to match (I'm syncing between macos & linux/debian, and the mac packages are lagging behind).
+
+My guess is that the assistant is simply \"not there yet\": using git-annex manually works just fine, but the assistant... ew. :/
+"""]]
diff --git a/doc/forum/Trouble_adding_ssh_remote_using_assistant_on_windows.mdwn b/doc/forum/Trouble_adding_ssh_remote_using_assistant_on_windows.mdwn
new file mode 100644
index 0000000000..08da616a86
--- /dev/null
+++ b/doc/forum/Trouble_adding_ssh_remote_using_assistant_on_windows.mdwn
@@ -0,0 +1,19 @@
+Hi, just trying to set up git-annex-assistant on freshly installed windows 10. I've run the web app and created a local repo. I also grabbed the tarball on my server and ran "git-annex" with no arguments (?). I'm now trying to add remote server using ssh. I add the correct details and click "Check this server", at which point it displays "Testing Server, Checking ssh connection to the server. This could take a minute". indefinitely.
+
+I've checked the log and don't see any ssh errors, but I do see some weirdness:
+
+```
+(scanning...) [2015-06-29 19:02:36 GMT Summer Time] Watcher: Performing startup scan
+(started...) rerrrrrrceereeeevccecccc:vvcvvvv ::v::::f  :    aff ffffiaafaaaaliiaiiiiellilllldeeleeee ddedddd(  d    N(( ((((oNN(NNNN ooNooooe  o    ree eeeerrrerrrrorrrrrrrrooroooo)rrorrrr
+))r))))
+
+)
+
+rerrrceeevccc:vvv :::f   afffiaaaliiiellldeee ddd(   N(((oNNN oooe   reeerrrrorrrrooo)rrr
+)))
+
+rerrrrrrrrrrrrrceeeeeeeeeeeeevccccccccccccc:vvvvvvvvvvvvv :::::::::::::f             afffffffffffffiaaaaaaaaaaaaaliiiiiiiiiiiiiellllllllllllldeeeeeeeeeeeee ddddddddddddd(             N(((((((((((((oNNNNNNNNNNNNN oooooooooooooe             reeeeeeeeeeeeerrrrrrrrrrrrrrorrrrrrrrrrrrrrooooooooooooo)rrrrrrrrrrrrr
+)))))))))))))
+```
+
+Not sure what that is but it might be relevant? Any help would be appreciated.
diff --git a/doc/forum/Trouble_adding_ssh_remote_using_assistant_on_windows/comment_1_0055392bfead0dd1d9dc1a196a6e5269._comment b/doc/forum/Trouble_adding_ssh_remote_using_assistant_on_windows/comment_1_0055392bfead0dd1d9dc1a196a6e5269._comment
new file mode 100644
index 0000000000..74a4ae025b
--- /dev/null
+++ b/doc/forum/Trouble_adding_ssh_remote_using_assistant_on_windows/comment_1_0055392bfead0dd1d9dc1a196a6e5269._comment
@@ -0,0 +1,14 @@
+[[!comment format=mdwn
+ username="joey"
+ subject="""comment 1"""
+ date="2015-07-02T20:53:54Z"
+ content="""
+This particular feature is unfortunately currently broken on Windows.
+
+I have developed a fix, but am waiting on it getting into a library I use.
+See [[bugs/windows_ssh_webapp_password_entry_broken]].
+
+As a workaround, if you can manually set up a passwordless ssh key on
+Windows, and configure the remote server to let you log in using that key,
+the webapp can then be used to add a repository on the server.
+"""]]
diff --git a/doc/forum/Trouble_adding_ssh_remote_using_assistant_on_windows/comment_2_3ec8af32b8148d31aaa3ba781b4dcf89._comment b/doc/forum/Trouble_adding_ssh_remote_using_assistant_on_windows/comment_2_3ec8af32b8148d31aaa3ba781b4dcf89._comment
new file mode 100644
index 0000000000..c16f7e3775
--- /dev/null
+++ b/doc/forum/Trouble_adding_ssh_remote_using_assistant_on_windows/comment_2_3ec8af32b8148d31aaa3ba781b4dcf89._comment
@@ -0,0 +1,7 @@
+[[!comment format=mdwn
+ username="acous"
+ subject="comment 2"
+ date="2015-07-07T22:25:40Z"
+ content="""
+Thanks, that did the trick. Am I right to assume that git-remote-gcrypt is not compatible with windows? I have cygwin set up which I could try to install it into, but I expect git-annex-assistant only has access to stuff available from the standard command prompt.
+"""]]
diff --git a/doc/forum/Trouble_adding_ssh_remote_using_assistant_on_windows/comment_3_8ddec87534a9025c26394fecd5456b84._comment b/doc/forum/Trouble_adding_ssh_remote_using_assistant_on_windows/comment_3_8ddec87534a9025c26394fecd5456b84._comment
new file mode 100644
index 0000000000..07fefd2166
--- /dev/null
+++ b/doc/forum/Trouble_adding_ssh_remote_using_assistant_on_windows/comment_3_8ddec87534a9025c26394fecd5456b84._comment
@@ -0,0 +1,9 @@
+[[!comment format=mdwn
+ username="joey"
+ subject="""comment 3"""
+ date="2015-07-15T16:05:01Z"
+ content="""
+git-remote-gcrpyt unfortunately is written as a shell script, so getting it
+to work on windows would probably be very hard unless it can be run with
+cygwin's shell somehow.
+"""]]
diff --git a/doc/forum/Trouble_adding_ssh_remote_using_assistant_on_windows/comment_4_9c12c508f62f2e76d09f4ed15af9c74e._comment b/doc/forum/Trouble_adding_ssh_remote_using_assistant_on_windows/comment_4_9c12c508f62f2e76d09f4ed15af9c74e._comment
new file mode 100644
index 0000000000..e24dab7812
--- /dev/null
+++ b/doc/forum/Trouble_adding_ssh_remote_using_assistant_on_windows/comment_4_9c12c508f62f2e76d09f4ed15af9c74e._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="graboluk@f6de53961ab0f884e203f602f65eb5cdc0fb7513"
+ nickname="graboluk"
+ subject="is it also broken in debian testing?"
+ date="2015-09-25T17:50:32Z"
+ content="""
+I'm using debian testing version 5.20150731. I've set up an account git-annex on my private server, I've checked that I can login to it just fine, but the box \"Testing server...\" is there indefinitely (I'm trying a login with password)
+"""]]
diff --git a/doc/forum/Trouble_adding_ssh_remote_using_assistant_on_windows/comment_5_b496f6938bd463de501eaa5c1850ce5c._comment b/doc/forum/Trouble_adding_ssh_remote_using_assistant_on_windows/comment_5_b496f6938bd463de501eaa5c1850ce5c._comment
new file mode 100644
index 0000000000..b1fdaacee1
--- /dev/null
+++ b/doc/forum/Trouble_adding_ssh_remote_using_assistant_on_windows/comment_5_b496f6938bd463de501eaa5c1850ce5c._comment
@@ -0,0 +1,7 @@
+[[!comment format=mdwn
+ username="joey"
+ subject="""comment 5"""
+ date="2015-11-18T19:12:02Z"
+ content="""
+@graboluk no, this problem was specific to Windows.
+"""]]
diff --git a/doc/forum/Trouble_installing_from_cabal_on_debian-testing.mdwn b/doc/forum/Trouble_installing_from_cabal_on_debian-testing.mdwn
new file mode 100644
index 0000000000..28c3a1749a
--- /dev/null
+++ b/doc/forum/Trouble_installing_from_cabal_on_debian-testing.mdwn
@@ -0,0 +1,15 @@
+I'm having trouble install from cabal on Debian-testing since the new beta released.
+
+    % sudo aptitude remove --purge cabal-install
+    % rm -rf $HOME/.cabal
+    % sudo aptitude install cabal-install
+    % cabal update
+    % cabal install git-annex
+
+The output all follows this general syntax:
+
+     depends on  which failed to install
+
+On the flip side, I upgrade my Debian to sid and it installed just fine through aptitude.
+
+(Apologies for my English, and if this is simply a user error)
diff --git a/doc/forum/Trouble_installing_from_cabal_on_debian-testing/comment_1_0d3e9d7cffafc34bc212557e8bbb987d._comment b/doc/forum/Trouble_installing_from_cabal_on_debian-testing/comment_1_0d3e9d7cffafc34bc212557e8bbb987d._comment
new file mode 100644
index 0000000000..d312ba7b17
--- /dev/null
+++ b/doc/forum/Trouble_installing_from_cabal_on_debian-testing/comment_1_0d3e9d7cffafc34bc212557e8bbb987d._comment
@@ -0,0 +1,12 @@
+[[!comment format=mdwn
+ username="http://joeyh.name/"
+ ip="4.153.14.141"
+ subject="comment 1"
+ date="2012-09-25T16:13:08Z"
+ content="""
+The difficulty with using cabal on Debian testing (and unstable) is that Debian has an older version of Yesod, and I have not managed to get cabal to work with both versions at once.
+
+If you build using the Makefile, it currently uses the old version of Yesod, so that'll work.
+
+The [package from unstable](http://packages.debian.org/sid/git-annex) should install ok on testing.
+"""]]
diff --git a/doc/forum/Trouble_setting_up_public_repo_cloneable_from_a_web_site.mdwn b/doc/forum/Trouble_setting_up_public_repo_cloneable_from_a_web_site.mdwn
new file mode 100644
index 0000000000..3c81f6ec94
--- /dev/null
+++ b/doc/forum/Trouble_setting_up_public_repo_cloneable_from_a_web_site.mdwn
@@ -0,0 +1,35 @@
+I'm trying to set up a public repository that is cloneable from my web site,
+similar to what is mentioned at
+https://git-annex.branchable.com/tips/setup_a_public_repository_on_a_web_site/
+
+I've got a repo from my laptop that I can push to my web server, and I can list
+the contents of the `.git` directory:
+
+    $ curl https://example.com/annex/.git/
+    
+    Index of /annex/.git/
+    
+    
    Index of /annex/.git/
    ../
+    annex/                 12-Mar-2017 17:42                   -
+    branches/           12-Mar-2017 17:38                   -
+    hooks/                 12-Mar-2017 17:38                   -
+    info/                   12-Mar-2017 17:38                   -
+    logs/                   12-Mar-2017 17:38                   -
+    objects/             12-Mar-2017 17:40                   -
+    refs/                   12-Mar-2017 17:38                   -
+    HEAD                     12-Mar-2017 17:39                  23
+    config                 12-Mar-2017 17:40                 269
+    description       12-Mar-2017 17:38                  73
+    index                   12-Mar-2017 17:41                1200
+    
    
+    
+
+However, when I try to clone it to simulate what I expect people to do, I get:
+
+    $ git clone https://exmaple.com/annex/.git
+    Cloning into 'annex...
+    fatal: repository 'https://example.com/annex/.git/' not found
+
+I've configured with `git config core.sharedrepository world` and `git config
+receive.denyCurrentBranch updateInstead`, but it doesn't seem to be working as I
+can't clone the repository over https.
diff --git a/doc/forum/Trouble_setting_up_public_repo_cloneable_from_a_web_site/comment_1_34a858688c46faf624fa7d98b05e6519._comment b/doc/forum/Trouble_setting_up_public_repo_cloneable_from_a_web_site/comment_1_34a858688c46faf624fa7d98b05e6519._comment
new file mode 100644
index 0000000000..e42a45af3c
--- /dev/null
+++ b/doc/forum/Trouble_setting_up_public_repo_cloneable_from_a_web_site/comment_1_34a858688c46faf624fa7d98b05e6519._comment
@@ -0,0 +1,11 @@
+[[!comment format=mdwn
+ username="joey"
+ subject="""comment 1"""
+ date="2017-04-07T20:31:26Z"
+ content="""
+Hmm, that all looks ok. I wonder if you forgot to run `git
+update-server-info`?
+
+ is an example of such a repo, which
+you can clone. So it certianly works if you set it up right.
+"""]]
diff --git a/doc/forum/Trouble_setting_up_public_repo_cloneable_from_a_web_site/comment_2_409103144cf803ced9d81d3380ca76fb._comment b/doc/forum/Trouble_setting_up_public_repo_cloneable_from_a_web_site/comment_2_409103144cf803ced9d81d3380ca76fb._comment
new file mode 100644
index 0000000000..bd2e5fb4b6
--- /dev/null
+++ b/doc/forum/Trouble_setting_up_public_repo_cloneable_from_a_web_site/comment_2_409103144cf803ced9d81d3380ca76fb._comment
@@ -0,0 +1,13 @@
+[[!comment format=mdwn
+ username="lee@7614f42c1a6cc84dbc813df25d2f75ed54948e17"
+ nickname="lee"
+ avatar="http://cdn.libravatar.org/avatar/bcacd00a7f05c4772329cf9f446c7987"
+ subject="comment 2"
+ date="2017-04-07T21:10:34Z"
+ content="""
+> I wonder if you forgot to run git update-server-info?
+
+That was it! Thanks!
+
+I actually didn't forget to run this, but I did not realize that it needed to be run on the *server*, so I was running it on my local annex instead. Now that I've run it everything works!
+"""]]
diff --git a/doc/forum/Truly_purging_dead_repositories.mdwn b/doc/forum/Truly_purging_dead_repositories.mdwn
new file mode 100644
index 0000000000..8a231439c4
--- /dev/null
+++ b/doc/forum/Truly_purging_dead_repositories.mdwn
@@ -0,0 +1 @@
+Since I'm just starting out with git-annex, I've had several false starts in getting things setup nicely between three machines.  In the course of so doing, I've ended up with several repositories which no longer exist, and which I've marked dead.  However, this is making the "git annex status" report a bit ugly, since these repositories no longer exist and can never exist again. Is there a way to truly purge dead repositories from my annex?  I'd be fine with a command that must be ran in tandem on all annexes before doing a sync...
diff --git a/doc/forum/Truly_purging_dead_repositories/comment_1_a4c75d49714b3543a9f1617a15d4a2d1._comment b/doc/forum/Truly_purging_dead_repositories/comment_1_a4c75d49714b3543a9f1617a15d4a2d1._comment
new file mode 100644
index 0000000000..6b778ed567
--- /dev/null
+++ b/doc/forum/Truly_purging_dead_repositories/comment_1_a4c75d49714b3543a9f1617a15d4a2d1._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="http://joeyh.name/"
+ ip="4.154.0.23"
+ subject="comment 1"
+ date="2012-10-18T02:20:25Z"
+ content="""
+If you really want to, check out the git-annex branch and delete the line from uuid.log, and do this on every repo in tandem. But it seems a lot of bother for something that is entirely hidden except for this one place.
+"""]]
diff --git a/doc/forum/Truly_purging_dead_repositories/comment_2_3da60a02e7323a204c5c5dd02ba04d6c._comment b/doc/forum/Truly_purging_dead_repositories/comment_2_3da60a02e7323a204c5c5dd02ba04d6c._comment
new file mode 100644
index 0000000000..8a89eee961
--- /dev/null
+++ b/doc/forum/Truly_purging_dead_repositories/comment_2_3da60a02e7323a204c5c5dd02ba04d6c._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="http://joeyh.name/"
+ ip="4.154.0.23"
+ subject="comment 2"
+ date="2012-10-18T02:23:54Z"
+ content="""
+Another possibilty is to reuse the dead uuids for later repos. You can do this by just setting annex.uuid in .git/config before you run `git annex init`, and then run `git annex fsck`.
+"""]]
diff --git a/doc/forum/Truly_purging_dead_repositories/comment_3_2576e45436008ff5a7ae5a38cade658e._comment b/doc/forum/Truly_purging_dead_repositories/comment_3_2576e45436008ff5a7ae5a38cade658e._comment
new file mode 100644
index 0000000000..2b78ab0e02
--- /dev/null
+++ b/doc/forum/Truly_purging_dead_repositories/comment_3_2576e45436008ff5a7ae5a38cade658e._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="https://me.yahoo.com/a/2grhJvAC049fJnvALDXek.6MRZMTlg--#eec89"
+ nickname="John"
+ subject="Another idea"
+ date="2012-10-18T05:38:07Z"
+ content="""
+What if git annex status didnn't show dead repository unless I specified \"git annex status --dead\"?  If you're OK with that, I'll try adding it.
+"""]]
diff --git a/doc/forum/Truly_purging_dead_repositories/comment_4_477e3c213c5a5d4a33afd42a5b94c718._comment b/doc/forum/Truly_purging_dead_repositories/comment_4_477e3c213c5a5d4a33afd42a5b94c718._comment
new file mode 100644
index 0000000000..3f39803b24
--- /dev/null
+++ b/doc/forum/Truly_purging_dead_repositories/comment_4_477e3c213c5a5d4a33afd42a5b94c718._comment
@@ -0,0 +1,10 @@
+[[!comment format=mdwn
+ username="http://joeyh.name/"
+ ip="2001:4830:1600:187::2"
+ subject="comment 4"
+ date="2013-09-04T06:44:42Z"
+ content="""
+Status no longer shows dead repositories.
+
+See also, answer here: 
+"""]]
diff --git a/doc/forum/Trying_to_delete_an_S3_repo_fails.mdwn b/doc/forum/Trying_to_delete_an_S3_repo_fails.mdwn
new file mode 100644
index 0000000000..6544fd1f3d
--- /dev/null
+++ b/doc/forum/Trying_to_delete_an_S3_repo_fails.mdwn
@@ -0,0 +1,3 @@
+[2015-10-03 22:10:53.948377] TransferScanner: warning TransferScanner crashed: unknown response from git cat-file ("refs/heads/git-annex:706/2a9/SHA256E-s13840914--38cdcf384aee777000693f58f1302684bf18f577045c1340d7aeb72b6d6f2233.dmg.log.cnk missing","refs/heads/git-annex:b8f/4d2/SHA256E-s26301635--ce2f215486df83c6ada74a81679ff78f717404d0637bfd51ecbf0330c21db12b.dmg.log")
+
+What exactly is this telling me?
diff --git a/doc/forum/Trying_to_delete_an_S3_repo_fails/comment_1_d8ab7757cd224439d55a18bfec405772._comment b/doc/forum/Trying_to_delete_an_S3_repo_fails/comment_1_d8ab7757cd224439d55a18bfec405772._comment
new file mode 100644
index 0000000000..88e301ac7a
--- /dev/null
+++ b/doc/forum/Trying_to_delete_an_S3_repo_fails/comment_1_d8ab7757cd224439d55a18bfec405772._comment
@@ -0,0 +1,9 @@
+[[!comment format=mdwn
+ username="joey"
+ subject="""comment 1"""
+ date="2015-10-04T19:33:36Z"
+ content="""
+This message suggests that your git repository has somehow lost an object
+that was committed to it. Suggest you run `git annex repair` to fix
+whatever the problem might be.
+"""]]
diff --git a/doc/forum/Trying_to_delete_an_S3_repo_fails/comment_2_94653e0c2c9e3e3be45c383442c7ec9b._comment b/doc/forum/Trying_to_delete_an_S3_repo_fails/comment_2_94653e0c2c9e3e3be45c383442c7ec9b._comment
new file mode 100644
index 0000000000..c120784df0
--- /dev/null
+++ b/doc/forum/Trying_to_delete_an_S3_repo_fails/comment_2_94653e0c2c9e3e3be45c383442c7ec9b._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="ovidiu@66ace8a8d99ce938b0538ffa0f26d30db02a9626"
+ nickname="ovidiu"
+ subject="comment 2"
+ date="2015-10-05T06:28:17Z"
+ content="""
+Thanks, is there a docs page somewhere about all the different  command line switches or can I determine those via \"git annex help\" maybe?
+"""]]
diff --git a/doc/forum/Trying_to_setup_2_full_backup_repos_that_sync_with_eachother.mdwn b/doc/forum/Trying_to_setup_2_full_backup_repos_that_sync_with_eachother.mdwn
new file mode 100644
index 0000000000..95d7483480
--- /dev/null
+++ b/doc/forum/Trying_to_setup_2_full_backup_repos_that_sync_with_eachother.mdwn
@@ -0,0 +1,30 @@
+So I'm liking git-annex alot but trying to setup something up
+by using the webapp wasn't very successful.
+
+I have serverA machineB laptopC laptopD
+
+Now what I want to setup is:
+
+serverA       machineB
+repo    repo
+
+Basically I want A and B to constantly sync with each
+other for a fullbackup repo on each. But I also want both A and B to
+have the full checkout local.
+
+Then I have C and D just be clients and pull whatever they need from A or B.
+
+What I've gotten working atm:
+On A I setup the repo(not bare - I want access to the contents)). On B I ran
+a clone from A. That way B always syncs to A.
+
+Setup both repos to be fullbackup. I've tried to setup A to also sync
+from B but the webapp fails at doing it - and when I retry it fails
+again but this time with an error in the log that the remote already exists.
+
+Would I need to setup things manually for this scenario? Or is it not even possible?
+
+Do I just setup a normal remote? Or will this loop break things?
+
+When I'm done setting it all up I'd like to have the assistant
+running on both systems to do the syncs for me.
diff --git a/doc/forum/Trying_to_wrap_my_head_around_git-annex__44___need_some_feedback.mdwn b/doc/forum/Trying_to_wrap_my_head_around_git-annex__44___need_some_feedback.mdwn
new file mode 100644
index 0000000000..934617888f
--- /dev/null
+++ b/doc/forum/Trying_to_wrap_my_head_around_git-annex__44___need_some_feedback.mdwn
@@ -0,0 +1,39 @@
+So after more than a year I think I am slowly beginning grasping git-annex. I'd like to put all my questions and ideas into this one thread, maybe you guys can help me sort things out. I'll number everything so you're more than welcome to chime in with regards to a particular item only. Happy to get links to read up stuff, don't hesitate to just paste a link if you think it is useful.
+
+	*1.* Right or wrong: I can only "work" with files in "CLIENT" repositories, not with backup, archive or anything else?
+
+	*2.* How does the copying or synching work? Is it comparable to rsync? How good is it at solving conflicts?
+
+	*3.* Can I get some feedback if git-annex is a solution for my scenario?
+
+*I have 1 Macbook, 1 NAS (Freenas running freeBSD) and access to S3 and box.com as well as a few external HDs.*
+My current setup involves a lot of manual copying, (very careful usage of) rsync and basically ends up me having a copy/backup of everything on the NAS and the files I currently work with on my Macbook.
+
+
+*Ideally I'd want to achieve something like this:*
+
+	- Have all files on the NAS
+	- Only have current work files on my Macbook
+	- Use external HDs only when necessary or transferign stuff
+	- S3 and box.com can be used however needed.
+
+	*4.* DOCUMENTS
+So in case I am travelling without Macbook, I use another PC or my Android phone to connect to my router at home via VPN then access a share on my NAS, work on a file, print it, save it back so whenever my MACbook is online again, it syncs the latest version.
+To achieve this with got-annex I guess I need the repository on the NAS to be a client repo, right? But the problem I see is that if I move a sub-folder from Docs into Archive, i.e. a folder of manuals I don't need on my MACbook all the time, it also gets moved out of the client repo on the NAS into an archive repo so how would I access it remotely if necessary?
+Also, talking about archiving, doesn't this get messy if you have a complicated folder structure inside a repo? How would you put stuff back from archives exactly where it was? 
+Sorry if this sounds a bit silly but I rely on a very precise folder system, everything is properly placed where I know it will be so if I now drag/drop all sorts of files/folders into archive I'll never figure out what's what again.
+
+	*5.* MUSIC
+This should be fairly easy, all my music is on my NAS and on my macbook, whenever I add music to my macbook it'll automatically sync to the NAS.
+
+	*6.* PICTURES
+This is where it gets complicated.
+I have a folder: Pictures with subfolders for each year, aka 2010, 2011 ... 2015 and inside is a sub folder for each photographed event. Now these pics get too big for my macbook so I'd like all images to be on my NAS, available for access, and to be able to simply archive say 2014. Now the question is if later I remember I need access to a particular event in 2014, can I browse that folder and un-archive that particular folder?
+
+	*7.* Also, say I'd like to be able to take a folder/sub-folder of images with me for external editing on a HD, and when coming back with them I'd like them to be synced back when I plug the HD back in, what type of repository would i have to set this HD to?
+
+	*8.* Obviously I'd need to setup a different repository for each Documents, Pictures, Music with their own settings, right?
+
+        *9.* At the moment it is not possible to have separate number of minimum number of copies per repository?
+
+So, what do you think, is git-annex suitable for my needs? Partially at least?
diff --git a/doc/forum/Trying_to_wrap_my_head_around_git-annex__44___need_some_feedback/comment_1_86a86f1f3ea2e778e4022689391b887b._comment b/doc/forum/Trying_to_wrap_my_head_around_git-annex__44___need_some_feedback/comment_1_86a86f1f3ea2e778e4022689391b887b._comment
new file mode 100644
index 0000000000..4ef121bf9d
--- /dev/null
+++ b/doc/forum/Trying_to_wrap_my_head_around_git-annex__44___need_some_feedback/comment_1_86a86f1f3ea2e778e4022689391b887b._comment
@@ -0,0 +1,16 @@
+[[!comment format=mdwn
+ username="https://www.google.com/accounts/o8/id?id=AItOawnmF_9CAtfqdZkC4e-_dCX-rK5bqh4RWkw"
+ nickname="Carl"
+ subject="comment 1"
+ date="2015-10-04T17:58:48Z"
+ content="""
+I think it can be a good fit. Let me try to answer some of your questions in the order that seems most reasonable to me
+
+*8* It is not a strict requirement to hav different repos for the different things, but I guess it might simplify things. Plus You can start with one of them and when that is properly set up continue with the next one.
+
+*5* your music solution seems simple enough. Numcopies=2 and everything should work. 
+
+*1*, *4*, *9* There is nothing magic with client remotes. It is just a group, that has some default rules. You can create your own groups and also change their rules. For my documents, I have two custom groups, \"offsite\" and \"cloud\". I have set up rules so that files prefer to be in one offsite remote and one cloud remote. However until I had possibility to plug my computer into an off site disc and sync it all files want to be in two cloud remotes. 
+
+*4*, *6* I personally do not run the assistant, but do everything on the command line. For me the setup where old things that I will probably never access again lies in an archive folder, if I do need it I can get the file back wit a simple \"git annex get $filename\". For my documents tree I only add files and folder or move folders into an archive folder. (Also note that archive folders can be located anywhere in your file tree, and that you can have arbitrarily many
+"""]]
diff --git a/doc/forum/Trying_to_wrap_my_head_around_git-annex__44___need_some_feedback/comment_2_77d61eceec24d2f41f044a643d8542cc._comment b/doc/forum/Trying_to_wrap_my_head_around_git-annex__44___need_some_feedback/comment_2_77d61eceec24d2f41f044a643d8542cc._comment
new file mode 100644
index 0000000000..42e7ab19f5
--- /dev/null
+++ b/doc/forum/Trying_to_wrap_my_head_around_git-annex__44___need_some_feedback/comment_2_77d61eceec24d2f41f044a643d8542cc._comment
@@ -0,0 +1,25 @@
+[[!comment format=mdwn
+ username="ovidiu@66ace8a8d99ce938b0538ffa0f26d30db02a9626"
+ nickname="ovidiu"
+ subject="comment 2"
+ date="2015-10-07T20:22:16Z"
+ content="""
+Hi Carl, thanks for the feedback.
+
+I'll be very slowly in replying, taking some late night classes this week but I WILL ask more questions once I get around to try some tests with dummy data.
+
+Meanwhile here is one question that has been bugging me for days now:
+
+Take the following scenario:
+
+- client repo setup on NAS
+- client repo setup on MAcbook
+- backup and archive repos elsewhere (doesn't matter for this question)
+
+So NAS and MACbook are in sync when I decide I don't need a specific folder on the MACbook and archive it.
+
+The big question:
+Will it be archived on the NAS too or not? (assistant running on both devices and please assume there are enough copies of this folder in backup and archive repos)
+
+This is really an important part of understanding git-annex for me as it would simply be a sync tool if it archives on both devices but if it does not it solves my problem of being able to decide waht data to carry with me on my MACbook and still be able to use a VPN and access a share on the NAS (the client repo folder).
+"""]]
diff --git a/doc/forum/Trying_to_wrap_my_head_around_git-annex__44___need_some_feedback/comment_3_529cada8df4888c12254818c26231835._comment b/doc/forum/Trying_to_wrap_my_head_around_git-annex__44___need_some_feedback/comment_3_529cada8df4888c12254818c26231835._comment
new file mode 100644
index 0000000000..3803624c81
--- /dev/null
+++ b/doc/forum/Trying_to_wrap_my_head_around_git-annex__44___need_some_feedback/comment_3_529cada8df4888c12254818c26231835._comment
@@ -0,0 +1,10 @@
+[[!comment format=mdwn
+ username="https://www.google.com/accounts/o8/id?id=AItOawnmF_9CAtfqdZkC4e-_dCX-rK5bqh4RWkw"
+ nickname="Carl"
+ subject="comment 3"
+ date="2015-10-09T08:45:25Z"
+ content="""
+As I understand your question, you will move some files into an \"archive-folder\" The client repo in your mac will then check if it allowed to drop the file. It seem that it will as there are enough copies on S3 et al. After syncing the file structure on the git repo will be updated on your NAS, as it is a client it will also check if it allowed to drop, and as enough copies are available it will. 
+
+If this is not what you want, then you do not want to have your NAS as a client. My setup is not dissimilar from yours, and my home server (corresponding to your NAS) is in the backup group. It thus wants to hold on to everything.
+"""]]
diff --git a/doc/forum/Trying_to_wrap_my_head_around_git-annex__44___need_some_feedback/comment_4_1561613ff2f0f3d7b42159d09ae1c7f7._comment b/doc/forum/Trying_to_wrap_my_head_around_git-annex__44___need_some_feedback/comment_4_1561613ff2f0f3d7b42159d09ae1c7f7._comment
new file mode 100644
index 0000000000..c7c32dc62a
--- /dev/null
+++ b/doc/forum/Trying_to_wrap_my_head_around_git-annex__44___need_some_feedback/comment_4_1561613ff2f0f3d7b42159d09ae1c7f7._comment
@@ -0,0 +1,10 @@
+[[!comment format=mdwn
+ username="ovidiu@66ace8a8d99ce938b0538ffa0f26d30db02a9626"
+ nickname="ovidiu"
+ subject="comment 4"
+ date="2015-10-10T10:16:14Z"
+ content="""
+Hi Carl,
+
+thanks for clarifying so the question is, if I don't use my NAS as a client, what other function can I assign, I'd really want to access its files via shares too. Can I make it a backup but keep the files \"in the clear\" so I am able to access them via a share?
+"""]]
diff --git a/doc/forum/Trying_to_wrap_my_head_around_git-annex__44___need_some_feedback/comment_5_59991756981681648070d03e2277ccbb._comment b/doc/forum/Trying_to_wrap_my_head_around_git-annex__44___need_some_feedback/comment_5_59991756981681648070d03e2277ccbb._comment
new file mode 100644
index 0000000000..ca0a79c1cb
--- /dev/null
+++ b/doc/forum/Trying_to_wrap_my_head_around_git-annex__44___need_some_feedback/comment_5_59991756981681648070d03e2277ccbb._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="https://www.google.com/accounts/o8/id?id=AItOawnmF_9CAtfqdZkC4e-_dCX-rK5bqh4RWkw"
+ nickname="Carl"
+ subject="comment 5"
+ date="2015-10-10T18:53:58Z"
+ content="""
+The type of group you have does not change if you have the files accesable from the filenames or not. As long as it is a git remote, that in not bare, or a direct git-annex remote you are good to go. 
+"""]]
diff --git a/doc/forum/Trying_to_wrap_my_head_around_git-annex__44___need_some_feedback/comment_6_eeafd62249d44b89d1091f0be79b3086._comment b/doc/forum/Trying_to_wrap_my_head_around_git-annex__44___need_some_feedback/comment_6_eeafd62249d44b89d1091f0be79b3086._comment
new file mode 100644
index 0000000000..c9f91a709d
--- /dev/null
+++ b/doc/forum/Trying_to_wrap_my_head_around_git-annex__44___need_some_feedback/comment_6_eeafd62249d44b89d1091f0be79b3086._comment
@@ -0,0 +1,18 @@
+[[!comment format=mdwn
+ username="ovidiu@66ace8a8d99ce938b0538ffa0f26d30db02a9626"
+ nickname="ovidiu"
+ subject="comment 6"
+ date="2015-10-11T16:51:06Z"
+ content="""
+you lost me there. so I'm using the command line on the NAS not the webapp but on the MACbook I do use the webapp.
+so when creating a repo through the webapp I chose:  Remote server - Set up a repository on a remote server using ssh. and then I can chose between: Unencrypted repository, Simple shared encryption and Encrypt with GnuPG key
+
+Unfortunately the descriptions aren't quite that self-explanatory. So far I have tested the 3rd option but here files get stored inside .git/annex/objects so I cannot access the files straight away. 
+I'll be testing a few more option awaiting a reply here, maybe I'll figure it out on my own. 
+The config of hte created repo on the NAS contained this:
+[core]
+        repositoryformatversion = 0
+        filemode = true
+        bare = true
+        sharedrepository = 1
+"""]]
diff --git a/doc/forum/Trying_to_wrap_my_head_around_git-annex__44___need_some_feedback/comment_7_24a20071d43ded3be9c4c68a637ca4f5._comment b/doc/forum/Trying_to_wrap_my_head_around_git-annex__44___need_some_feedback/comment_7_24a20071d43ded3be9c4c68a637ca4f5._comment
new file mode 100644
index 0000000000..ce05261124
--- /dev/null
+++ b/doc/forum/Trying_to_wrap_my_head_around_git-annex__44___need_some_feedback/comment_7_24a20071d43ded3be9c4c68a637ca4f5._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="https://www.google.com/accounts/o8/id?id=AItOawnmF_9CAtfqdZkC4e-_dCX-rK5bqh4RWkw"
+ nickname="Carl"
+ subject="comment 7"
+ date="2015-10-12T18:21:21Z"
+ content="""
+I am not fluent in the webapp. Usually I use the commandline and follow the walktrough on the homepage. Your problem in not being able to see the files is that the NAS has a bare repo. You want a non bare.
+"""]]
diff --git a/doc/forum/Trying_to_wrap_my_head_around_git-annex__44___need_some_feedback/comment_8_8ad6bd5fce8f9bc1622f225c162ae167._comment b/doc/forum/Trying_to_wrap_my_head_around_git-annex__44___need_some_feedback/comment_8_8ad6bd5fce8f9bc1622f225c162ae167._comment
new file mode 100644
index 0000000000..190cf74339
--- /dev/null
+++ b/doc/forum/Trying_to_wrap_my_head_around_git-annex__44___need_some_feedback/comment_8_8ad6bd5fce8f9bc1622f225c162ae167._comment
@@ -0,0 +1,43 @@
+[[!comment format=mdwn
+ username="ovidiu@66ace8a8d99ce938b0538ffa0f26d30db02a9626"
+ nickname="ovidiu"
+ subject="comment 8"
+ date="2015-10-17T14:54:12Z"
+ content="""
+Thanks for bearing with me Carl, you mention the walkthrough and the command line so lets see:
+
+on my nas I follow the walkthrough:
+
+# mkdir ~/annex
+# cd ~/annex
+# git init
+# git annex init
+
+which yields in a BARE repo:
+
+cat .git/config 
+[core]
+        repositoryformatversion = 0
+        filemode = true
+        bare = false
+        logallrefupdates = true
+[user]
+        name = gitannex
+        email = gitannex
+[annex]
+        uuid = 36bca403-027b-4e90-9504-123456789012
+        version = 5
+
+So where do I stand now? You mention I need a non-bare repo and suggested to follow the walkthrough. Kinda stuck here now and I really would like to get to use gitannex.
+
+
+Maybe joey will give some advice?
+
+Would you mind sharing more of your setup? Theory or practical advice welcome :-)
+
+
+regarding an earlier question of mine, about how the syncing works: 
+I understand its not possible to have git-annex running somewhere centrally and have two clients who only access parts of a repo?
+
+i.e. on \"server\" have Documents with subfolder a) and b)  then have 2 clients, i.e. Macbooka and Macbookb which would each only sync the a) respectively b) folders out of the complete repo?
+"""]]
diff --git a/doc/forum/Trying_to_wrap_my_head_around_git-annex__44___need_some_feedback/comment_9_6406fcf0021ac8266b4f2ef850be0fa0._comment b/doc/forum/Trying_to_wrap_my_head_around_git-annex__44___need_some_feedback/comment_9_6406fcf0021ac8266b4f2ef850be0fa0._comment
new file mode 100644
index 0000000000..bc2a6685b4
--- /dev/null
+++ b/doc/forum/Trying_to_wrap_my_head_around_git-annex__44___need_some_feedback/comment_9_6406fcf0021ac8266b4f2ef850be0fa0._comment
@@ -0,0 +1,12 @@
+[[!comment format=mdwn
+ username="joey"
+ subject="""comment 9"""
+ date="2015-10-19T17:37:32Z"
+ content="""
+git init will never create a bare git repository (unless you specify
+--bare), and a normal bare repository doesn't have a .git directory.
+
+So I suspect that your repository is using git-annex's direct
+mode, which only sets core.bare as an implementation detail. Likely
+you trimmed off the end of the .git/config, where it said "direct = true"
+"""]]
diff --git a/doc/forum/Two_computer_setup__58_____34__transfer__34___or___34__full_backup__34___repository_groups__63__.mdwn b/doc/forum/Two_computer_setup__58_____34__transfer__34___or___34__full_backup__34___repository_groups__63__.mdwn
new file mode 100644
index 0000000000..5495e26498
--- /dev/null
+++ b/doc/forum/Two_computer_setup__58_____34__transfer__34___or___34__full_backup__34___repository_groups__63__.mdwn
@@ -0,0 +1,18 @@
+Hi,
+
+Here is my setup:
+
+* I have a locally networked server Server-A with a large storage hard drive auto-mounted.
+* I have two personal client computers: Client-A and Client-B.
+* Client-A is my primary (desktop) computer. 
+* Client-B is my secondary (laptop) computer that I use less often.
+* Git-annex is installed on all 3 computers and I use git-annex assistant on both Client-A and Client-B.
+* Both Client-A and Client-B need to share the same repository/files.
+
+
+What is the best or recommended git-annex assistant repository configuration for this setup? Ideally I want Server-A to have a full backup of the shared repository. I currently have a client repository for both Client-A and Client-B. Also, both Client-A and Client-B have a "full backup" repository pointing to the same repository hosted on Server-A. Is this OK? Should only one of the the client's have this repository as a "full backup"? Will this current setup also have git annex assistant auto-sync the repositories between the clients? I have both clients setup right now and they have both successfully cloned the shared repository, but I have noticed that they have yet to sync up modifications made on the other client. An alternative setup I can think of would be to have Client-A keep its current configuration and have Client-B switch the "full backup" repository to be just a "transfer" repository.
+
+I hope my intentions are clear. Please give me your recommendations.
+
+Regards,
+Blake
diff --git a/doc/forum/Two_computer_setup__58_____34__transfer__34___or___34__full_backup__34___repository_groups__63__/comment_1_b8702892280447193e6e80be22a580a0._comment b/doc/forum/Two_computer_setup__58_____34__transfer__34___or___34__full_backup__34___repository_groups__63__/comment_1_b8702892280447193e6e80be22a580a0._comment
new file mode 100644
index 0000000000..915e7209bd
--- /dev/null
+++ b/doc/forum/Two_computer_setup__58_____34__transfer__34___or___34__full_backup__34___repository_groups__63__/comment_1_b8702892280447193e6e80be22a580a0._comment
@@ -0,0 +1,11 @@
+[[!comment format=mdwn
+ username="https://www.google.com/accounts/o8/id?id=AItOawlkA6XinbeOdnEDxEGQUWyjqPGh0kdMXr4"
+ nickname="Blake"
+ subject="update"
+ date="2013-10-30T07:19:56Z"
+ content="""
+It seems both Client-A and Client-B ARE actually syncing with this current setup. I think Client-B was just delayed due to the initial startup scan. So I guess at this point I am more or less asking if having both Client-A and Client-B have the remote repository being a \"full backup\" type is appropriate or if there is a better configuration that achieves what I am hoping to have.
+
+Regards,
+Blake
+"""]]
diff --git a/doc/forum/Two_computer_setup__58_____34__transfer__34___or___34__full_backup__34___repository_groups__63__/comment_2_50cafde7e30b928480d1f142ddd763d2._comment b/doc/forum/Two_computer_setup__58_____34__transfer__34___or___34__full_backup__34___repository_groups__63__/comment_2_50cafde7e30b928480d1f142ddd763d2._comment
new file mode 100644
index 0000000000..4495401b21
--- /dev/null
+++ b/doc/forum/Two_computer_setup__58_____34__transfer__34___or___34__full_backup__34___repository_groups__63__/comment_2_50cafde7e30b928480d1f142ddd763d2._comment
@@ -0,0 +1,10 @@
+[[!comment format=mdwn
+ username="http://joeyh.name/"
+ ip="209.250.56.47"
+ subject="comment 2"
+ date="2013-11-01T16:47:18Z"
+ content="""
+git-annex keeps track of the repository group on a per repository basis. That is, if the repository on Server-A is a full backup, both Client-A and Client-B will consider it to be one, and use it appropriately. You can't have just Client-A think it's a full backup; this setting will be automatically synced to Client-B.
+
+Your setup sounds ok, although you might need to set up a XMPP account in order for Client-A and Client-B to immediately message one-another when they have new changes to sync. Otherwise, they might only poll Server-A occasionally for changes.
+"""]]
diff --git a/doc/forum/Two_computers__44___one_networked_repository__44___no_Jabber_-_how_to_setup_sync__63___.mdwn b/doc/forum/Two_computers__44___one_networked_repository__44___no_Jabber_-_how_to_setup_sync__63___.mdwn
new file mode 100644
index 0000000000..64426eb4d1
--- /dev/null
+++ b/doc/forum/Two_computers__44___one_networked_repository__44___no_Jabber_-_how_to_setup_sync__63___.mdwn
@@ -0,0 +1,7 @@
+Hello there,
+
+I've got two computers, let's call them Alice and Bob, which should share the same repository. Alice and Bob are never online at the same time, and the remote repository setup as "transfer file repository" is an account on Box.com. So Jabber is obviously no option.
+
+I've tried setting up this with the webapp, but it's not working as intended: Bob never pulls the content from Alice from the remote repository. 
+
+How can I setup this to make it work with the remote repository in the web app? I am kinda at a loss right now. 
diff --git a/doc/forum/Two_computers__44___one_networked_repository__44___no_Jabber_-_how_to_setup_sync__63___/comment_1_b90aef011d853ad9feb6f8edc08bf7d9._comment b/doc/forum/Two_computers__44___one_networked_repository__44___no_Jabber_-_how_to_setup_sync__63___/comment_1_b90aef011d853ad9feb6f8edc08bf7d9._comment
new file mode 100644
index 0000000000..11e06cd406
--- /dev/null
+++ b/doc/forum/Two_computers__44___one_networked_repository__44___no_Jabber_-_how_to_setup_sync__63___/comment_1_b90aef011d853ad9feb6f8edc08bf7d9._comment
@@ -0,0 +1,16 @@
+[[!comment format=mdwn
+ username="joey"
+ subject="""comment 1"""
+ date="2015-01-20T19:15:50Z"
+ content="""
+Since Alice and Bob are never online at the same time, you need
+a central git repository somewhere. 
+
+This could be on a linux server, perhaps even with git-annex installed.
+Or, you could just make a repository on github or gitorious. No need
+for git-annex.
+
+Wherever it is, add the central repository as a git remote on Alice and
+Bob, and they'll each use it to drop off the current state of their
+git repo, and sync up with the changes from each other.
+"""]]
diff --git a/doc/forum/USB_backup_with_files_visible.mdwn b/doc/forum/USB_backup_with_files_visible.mdwn
new file mode 100644
index 0000000000..747de85ffe
--- /dev/null
+++ b/doc/forum/USB_backup_with_files_visible.mdwn
@@ -0,0 +1,7 @@
+Quick question: I want to create a repository on my removable USB drive that will acquire all files (eg. "full backup" repository group), but where the files will be visible and usable if you load the drive on another computer.
+
+How do I do that from within the assistant?
+
+Thanks,
+
+Zellyn
diff --git a/doc/forum/USB_backup_with_files_visible/comment_10_b90b18ca57f9299f1920254bb2aa699b._comment b/doc/forum/USB_backup_with_files_visible/comment_10_b90b18ca57f9299f1920254bb2aa699b._comment
new file mode 100644
index 0000000000..763a3ce544
--- /dev/null
+++ b/doc/forum/USB_backup_with_files_visible/comment_10_b90b18ca57f9299f1920254bb2aa699b._comment
@@ -0,0 +1,7 @@
+[[!comment format=mdwn
+ username="http://joeyh.name/"
+ subject="comment 10"
+ date="2014-11-25T20:41:56Z"
+ content="""
+When you actually think about how users would try to use this proposed system, you get here:  Ie, if a removable drive shows all of a user's files, then they'll  try to edit them, and since the assistant can't run on removable drives (makes them kinda hard to remove), such changes by the user won't get committed.
+"""]]
diff --git a/doc/forum/USB_backup_with_files_visible/comment_1_2832f8ae24dfb0f101e06f7c18283028._comment b/doc/forum/USB_backup_with_files_visible/comment_1_2832f8ae24dfb0f101e06f7c18283028._comment
new file mode 100644
index 0000000000..d70d859bf5
--- /dev/null
+++ b/doc/forum/USB_backup_with_files_visible/comment_1_2832f8ae24dfb0f101e06f7c18283028._comment
@@ -0,0 +1,14 @@
+[[!comment format=mdwn
+ username="http://joeyh.name/"
+ ip="4.154.4.193"
+ subject="comment 1"
+ date="2013-07-02T17:00:41Z"
+ content="""
+This is not something the assistant really handles yet. The issue is that if you have a non-bare repository on the USB drive, something has to run `git annex sync` in it when changes are pushed to it, to update it to display the files that are in the repository.
+
+One way to do this is to set up a non-bare repository by hand and add a git post-receive hook that runs `git annex sync`
+
+Update: Another way, recently added is
+
+This new approach even supports FAT filesystems!
+"""]]
diff --git a/doc/forum/USB_backup_with_files_visible/comment_2_6163e01aa441f8435091f026cc6da337._comment b/doc/forum/USB_backup_with_files_visible/comment_2_6163e01aa441f8435091f026cc6da337._comment
new file mode 100644
index 0000000000..591dc847d9
--- /dev/null
+++ b/doc/forum/USB_backup_with_files_visible/comment_2_6163e01aa441f8435091f026cc6da337._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="https://www.google.com/accounts/o8/id?id=AItOawkQafKy7hNSEolLs6TvbgUnkklTctUY9LI"
+ nickname="Zellyn"
+ subject="comment 2"
+ date="2013-07-02T17:10:59Z"
+ content="""
+Aah, perfect. I did in fact set up a non-bare git annex on the drive. I was just lacking the `git annex sync` command. Thanks!
+"""]]
diff --git a/doc/forum/USB_backup_with_files_visible/comment_3_ee92ff320eb5d9a031bdd1896aee0d86._comment b/doc/forum/USB_backup_with_files_visible/comment_3_ee92ff320eb5d9a031bdd1896aee0d86._comment
new file mode 100644
index 0000000000..98a3c1cc5d
--- /dev/null
+++ b/doc/forum/USB_backup_with_files_visible/comment_3_ee92ff320eb5d9a031bdd1896aee0d86._comment
@@ -0,0 +1,11 @@
+[[!comment format=mdwn
+ username="https://www.google.com/accounts/o8/id?id=AItOawln3ckqKx0x_xDZMYwa9Q1bn4I06oWjkog"
+ nickname="Michael"
+ subject="post-receive hook looks troublesome"
+ date="2013-07-03T16:11:13Z"
+ content="""
+I just tried setting up a post-receive hook with just \"git annex sync\" in it.
+The end result is that files (git annex symlinks, really) are getting removed somehow as a result after a couple of git annex sync's done manually.
+It would be nice if this actually worked.
+
+"""]]
diff --git a/doc/forum/USB_backup_with_files_visible/comment_4_437c8342c0b65e3a89129800313eb73c._comment b/doc/forum/USB_backup_with_files_visible/comment_4_437c8342c0b65e3a89129800313eb73c._comment
new file mode 100644
index 0000000000..a45b05b25c
--- /dev/null
+++ b/doc/forum/USB_backup_with_files_visible/comment_4_437c8342c0b65e3a89129800313eb73c._comment
@@ -0,0 +1,10 @@
+[[!comment format=mdwn
+ username="http://joeyh.name/"
+ ip="4.154.4.193"
+ subject="comment 4"
+ date="2013-07-03T19:54:21Z"
+ content="""
+Seems to me that any such problems would be caused by the environment variables git sets in the hook, or possibly the current working directory in use when the hook runs. So it might take some finessing to get right.
+
+Meanwhile, I've committed a change that makes `git annex merge` merge in changes synced to the repo from other repositories. So that can be used the same way `git annex sync` can, but without the added overhead of committing, pushing, and pulling.
+"""]]
diff --git a/doc/forum/USB_backup_with_files_visible/comment_5_5e10cffe8465ea4ecaa71c03a4c29ea4._comment b/doc/forum/USB_backup_with_files_visible/comment_5_5e10cffe8465ea4ecaa71c03a4c29ea4._comment
new file mode 100644
index 0000000000..20853a4a0d
--- /dev/null
+++ b/doc/forum/USB_backup_with_files_visible/comment_5_5e10cffe8465ea4ecaa71c03a4c29ea4._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="http://edheil.wordpress.com/"
+ ip="99.54.57.201"
+ subject="comment 5"
+ date="2013-07-04T05:11:46Z"
+ content="""
+Oh!  git annex merge is very nice.
+"""]]
diff --git a/doc/forum/USB_backup_with_files_visible/comment_6_add048a16837f7940a859f21426cdbe9._comment b/doc/forum/USB_backup_with_files_visible/comment_6_add048a16837f7940a859f21426cdbe9._comment
new file mode 100644
index 0000000000..95f7495741
--- /dev/null
+++ b/doc/forum/USB_backup_with_files_visible/comment_6_add048a16837f7940a859f21426cdbe9._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="https://www.google.com/accounts/o8/id?id=AItOawmNu4V5fvpLlBhaCUfXXOB0MI5NXwh8SkU"
+ nickname="Adam"
+ subject="Confused"
+ date="2013-11-03T00:13:47Z"
+ content="""
+Sorry, I'm confused.  I set up a Full Backup repo on a second hard drive, but the files are visible in its directory just like in my Client repos' directories.  Are the files not supposed to be visible?  Is something wrong?
+"""]]
diff --git a/doc/forum/USB_backup_with_files_visible/comment_7_de227ca9911fe57d7a6d3e037f574fe9._comment b/doc/forum/USB_backup_with_files_visible/comment_7_de227ca9911fe57d7a6d3e037f574fe9._comment
new file mode 100644
index 0000000000..0f74efac08
--- /dev/null
+++ b/doc/forum/USB_backup_with_files_visible/comment_7_de227ca9911fe57d7a6d3e037f574fe9._comment
@@ -0,0 +1,14 @@
+[[!comment format=mdwn
+ username="http://joeyh.name/"
+ ip="209.250.56.47"
+ subject="comment 7"
+ date="2013-11-03T01:09:16Z"
+ content="""
+You can set it up that way, but the often confusing behavior (including not being able to umount a drive that has the git-annex assistant deamon running in it!) is why the webapp attempt to steer users toward making bare reposotories on USB drives.
+
+If you click on \"Add another repository\" -> \"Removable drive\", you get a bare repository.
+
+If you click on upper-right menu -> \"Add another local repository\" and enter a path, and tell it to combine the repositories, you get a synced non-bare repository, and potentially problems unmounting the drive since you now have 2 assistant daemons running.
+
+I have now added a further link to the nice removable drive path to that second UI path.
+"""]]
diff --git a/doc/forum/USB_backup_with_files_visible/comment_8_0c0ed0e038f7f0e2d2d4ed69b7b29fbc._comment b/doc/forum/USB_backup_with_files_visible/comment_8_0c0ed0e038f7f0e2d2d4ed69b7b29fbc._comment
new file mode 100644
index 0000000000..6511343e86
--- /dev/null
+++ b/doc/forum/USB_backup_with_files_visible/comment_8_0c0ed0e038f7f0e2d2d4ed69b7b29fbc._comment
@@ -0,0 +1,10 @@
+[[!comment format=mdwn
+ username="https://www.google.com/accounts/o8/id?id=AItOawmNu4V5fvpLlBhaCUfXXOB0MI5NXwh8SkU"
+ nickname="Adam"
+ subject="comment 8"
+ date="2013-11-03T01:13:26Z"
+ content="""
+Thanks for the explanation, Joey.  I just posted a [comment](http://git-annex.branchable.com/bugs/Unnecessary_remote_transfers/#comment-f7110e694a9951d8ab546b8dec568bfa) about this on the bug report too.  Maybe the problem is simply that the Removable Drive option doesn't allow using internal, non-removable drives.
+
+BTW, I don't know if this is relevant, but I only see one \"git-annex assistant\" process running.
+"""]]
diff --git a/doc/forum/USB_backup_with_files_visible/comment_9_dfb830fb00c10b797c3d26179e6dfefd._comment b/doc/forum/USB_backup_with_files_visible/comment_9_dfb830fb00c10b797c3d26179e6dfefd._comment
new file mode 100644
index 0000000000..f73f186717
--- /dev/null
+++ b/doc/forum/USB_backup_with_files_visible/comment_9_dfb830fb00c10b797c3d26179e6dfefd._comment
@@ -0,0 +1,14 @@
+[[!comment format=mdwn
+ username="https://www.google.com/accounts/o8/id?id=AItOawmicVKRM8vJX4wPuAwlLEoS2cjmFXQkjkE"
+ nickname="Thomas"
+ subject="post-receive hook is not an option with FAT drives"
+ date="2014-11-20T15:53:29Z"
+ content="""
+Hi Joey,
+
+I tried to solve this use case with a post-receive hook and had to realize that FAT does not support the executable bit and thus the hook doesn't run. I've found several requests for the use case described here and I think it would be desirable, if the assistant/webapp could solve it without commandline-hacks.
+
+So I want to stick my thumbdrive in my work desktop and the assistant notices that it's available and starts copying stuff on it and runs git annex merge afterwards.
+
+I also had to manually set annex.diskreserve to a smaller value since annex wants to reserve more space (10G) than is available on most thumb drives. I think git-annex init should be more clever about the default diskreserve value, e.g. set it to a percentage of the total disk size.
+"""]]
diff --git a/doc/forum/USB_drive_in_transfer_group_keeps_growing_-_assistant.mdwn b/doc/forum/USB_drive_in_transfer_group_keeps_growing_-_assistant.mdwn
new file mode 100644
index 0000000000..6b1c03d9a4
--- /dev/null
+++ b/doc/forum/USB_drive_in_transfer_group_keeps_growing_-_assistant.mdwn
@@ -0,0 +1,22 @@
+1. Set up two computers with a client repository.
+2. Add a removable drive repository and set it to transfer group.
+3. Start adding files to computer #1 repository. See how the files get synced to the usb drive.
+4. Connect the usb drive to computer #2 and see the files getting transferred to computer #2. Everything is looking good.
+
+
+5. Connect the usb drive to computer #1 again. 
+6. Add a file that is larger than the remaining size of the usb drive, BUT smaller than the original size of the usb drive.
+7. The file does not get transferred to the usb drive due to lack of disk space.
+
+I would expect the assistant to make some space on the usb drive. Removing the files it knows has been transferred to computer #2, and then transfer the new file to the usb drive. But this does not seem to happen. 
+
+Have I missed something about how the transfer group is supposed to work?
+
+ 
+Using version:
+git-annex version: 4.20130802-g0a52f02
+build flags: Assistant Webapp Pairing Testsuite S3 WebDAV Inotify DBus XMPP
+
+Pre built tar file.
+
+
diff --git a/doc/forum/USB_drive_in_transfer_group_keeps_growing_-_assistant/comment_1_0a6f6054d70009979f4a036e24b7c500._comment b/doc/forum/USB_drive_in_transfer_group_keeps_growing_-_assistant/comment_1_0a6f6054d70009979f4a036e24b7c500._comment
new file mode 100644
index 0000000000..f298b56cde
--- /dev/null
+++ b/doc/forum/USB_drive_in_transfer_group_keeps_growing_-_assistant/comment_1_0a6f6054d70009979f4a036e24b7c500._comment
@@ -0,0 +1,10 @@
+[[!comment format=mdwn
+ username="http://joeyh.name/"
+ ip="4.154.0.63"
+ subject="comment 1"
+ date="2013-08-24T19:22:05Z"
+ content="""
+The assistant is supposed to remove files from transfer repositories once the file has been transferred to all known clients.  This generally seems to work, with all the transfer repositories I've tested it with. Nothing is special about USB drives compared with other transfer repositories.
+
+Perhaps you have not configured the USB drive as a transfer repository? Or perhaps you have another client repository (even a client repository you had once but have now deleted).
+"""]]
diff --git a/doc/forum/USB_drive_in_transfer_group_keeps_growing_-_assistant/comment_2_f9eef3019fe690e90c1228d62a16f70a._comment b/doc/forum/USB_drive_in_transfer_group_keeps_growing_-_assistant/comment_2_f9eef3019fe690e90c1228d62a16f70a._comment
new file mode 100644
index 0000000000..7f761419c5
--- /dev/null
+++ b/doc/forum/USB_drive_in_transfer_group_keeps_growing_-_assistant/comment_2_f9eef3019fe690e90c1228d62a16f70a._comment
@@ -0,0 +1,10 @@
+[[!comment format=mdwn
+ username="https://www.google.com/accounts/o8/id?id=AItOawnNqLKszWk9EoD4CDCqNXJRIklKFBCN1Ao"
+ nickname="maurizio"
+ subject="even a client repository you had once but have now deleted"
+ date="2014-07-09T22:08:44Z"
+ content="""
+if this client repository has been deleted, why does it still have an impact? What is the user supposed to do to use a transfer repository if some day another client which was deleted has existed? 
+
+Old repositories seem to remain for ever in the webapp. You can try to delete them with the webapp, they will stay there. You can disable them, they will stay there. If you want to try to delete them again in the webapp, you first have to enable them again and nothing ever happens. There seems to be basically no way to delete repositories which have some day existed...
+"""]]
diff --git a/doc/forum/USB_drive_in_transfer_group_keeps_growing_-_assistant/comment_3_7fb74f7fab6c1baff4ffc270cf15ef0a._comment b/doc/forum/USB_drive_in_transfer_group_keeps_growing_-_assistant/comment_3_7fb74f7fab6c1baff4ffc270cf15ef0a._comment
new file mode 100644
index 0000000000..df8333cb3c
--- /dev/null
+++ b/doc/forum/USB_drive_in_transfer_group_keeps_growing_-_assistant/comment_3_7fb74f7fab6c1baff4ffc270cf15ef0a._comment
@@ -0,0 +1,15 @@
+[[!comment format=mdwn
+ username="http://joeyh.name/"
+ ip="209.250.56.2"
+ subject="comment 3"
+ date="2014-07-11T19:29:06Z"
+ content="""
+The webapp certianly allows deleting repositories.
+
+But, you can always go in and manually do it:
+
+
    +git annex dead $uuid
+git annex group $uuid unwanted
+
    
+"""]]
diff --git a/doc/forum/USB_drive_in_transfer_group_keeps_growing_-_assistant/comment_4_5925b609aff0954f2860269cff107284._comment b/doc/forum/USB_drive_in_transfer_group_keeps_growing_-_assistant/comment_4_5925b609aff0954f2860269cff107284._comment
new file mode 100644
index 0000000000..31a7e63d80
--- /dev/null
+++ b/doc/forum/USB_drive_in_transfer_group_keeps_growing_-_assistant/comment_4_5925b609aff0954f2860269cff107284._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="https://www.google.com/accounts/o8/id?id=AItOawnNqLKszWk9EoD4CDCqNXJRIklKFBCN1Ao"
+ nickname="maurizio"
+ subject="comment 4"
+ date="2014-07-17T19:48:30Z"
+ content="""
+I may be facing one unexpected behavior, but your statement that the \"webapp certainely allows to delete repositories\" does not match my experience. It does offer this option, but this option has failed on me so many times that I have learnt to live with a couple of repositories disabled, sitting here. Trying to delete them starts some process (emptying the repository, starting deletion process) but ends up in nothing. At least one of them does not seem to have a name anymore (probably what you refer to as $uuid) since only the part in square bracket is visible in the webapp. I concluded that this was what the webapp was supposed to do. 
+"""]]
diff --git a/doc/forum/USB_drive_in_transfer_group_keeps_growing_-_assistant/comment_5_45062d5776f8aefb0850d70f2a517c75._comment b/doc/forum/USB_drive_in_transfer_group_keeps_growing_-_assistant/comment_5_45062d5776f8aefb0850d70f2a517c75._comment
new file mode 100644
index 0000000000..3dbae84a07
--- /dev/null
+++ b/doc/forum/USB_drive_in_transfer_group_keeps_growing_-_assistant/comment_5_45062d5776f8aefb0850d70f2a517c75._comment
@@ -0,0 +1,12 @@
+[[!comment format=mdwn
+ username="https://www.google.com/accounts/o8/id?id=AItOawnNqLKszWk9EoD4CDCqNXJRIklKFBCN1Ao"
+ nickname="maurizio"
+ subject="comment 5"
+ date="2014-07-18T08:13:08Z"
+ content="""
+by the way: the commands you suggest do not seem to match what I read here:
+
+http://git-annex.branchable.com/forum/How_to_delete_a_remote__63__/
+
+
+"""]]
diff --git a/doc/forum/Ubuntu_PPA.mdwn b/doc/forum/Ubuntu_PPA.mdwn
new file mode 100644
index 0000000000..b97cc1aec5
--- /dev/null
+++ b/doc/forum/Ubuntu_PPA.mdwn
@@ -0,0 +1,3 @@
+Is there a PPA that Ubuntu people are using to track the latest git-annex?
+
+The [Ubuntu install page](http://git-annex.branchable.com/install/Ubuntu/) mentions [https://launchpad.net/~rubiojr/+archive/git-annex](https://launchpad.net/~rubiojr/+archive/git-annex) which hasn't been updated for a while.
diff --git a/doc/forum/Ubuntu_PPA/comment_1_b55535258b1b4bcfc802235f0cba075d._comment b/doc/forum/Ubuntu_PPA/comment_1_b55535258b1b4bcfc802235f0cba075d._comment
new file mode 100644
index 0000000000..d91a0d1eb8
--- /dev/null
+++ b/doc/forum/Ubuntu_PPA/comment_1_b55535258b1b4bcfc802235f0cba075d._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="https://www.google.com/accounts/o8/id?id=AItOawmeki7KlfJpAAN9CUmi9EdwhwBmIhDMYuE"
+ nickname="Marvin"
+ subject="comment 1"
+ date="2013-03-11T09:40:43Z"
+ content="""
+im also looking for a PPA with newer git-annex versions. mostly because of the glacier support...
+"""]]
diff --git a/doc/forum/Ubuntu_PPA/comment_2_adc4d644fed058d1811acf0b35db9c18._comment b/doc/forum/Ubuntu_PPA/comment_2_adc4d644fed058d1811acf0b35db9c18._comment
new file mode 100644
index 0000000000..bc91500c76
--- /dev/null
+++ b/doc/forum/Ubuntu_PPA/comment_2_adc4d644fed058d1811acf0b35db9c18._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="https://www.google.com/accounts/o8/id?id=AItOawmVICFY2CDP08xdsPr3cgmScomy9HA-1sk"
+ nickname="Andrew"
+ subject="comment 2"
+ date="2013-03-13T02:31:52Z"
+ content="""
+It's perfectly possible (though slightly messy) to install git-annex on Ubuntu via [[/install/cabal]], though you need to find and install all the required dependencies yourself (haskell, libidn, libgnutls, libgsasl etc). If you want to get crazy with this look at 
+"""]]
diff --git a/doc/forum/Ubuntu_PPA/comment_3_fc9cd51558c47718f243437202a11803._comment b/doc/forum/Ubuntu_PPA/comment_3_fc9cd51558c47718f243437202a11803._comment
new file mode 100644
index 0000000000..4c77308d05
--- /dev/null
+++ b/doc/forum/Ubuntu_PPA/comment_3_fc9cd51558c47718f243437202a11803._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="fmarier"
+ ip="121.98.93.240"
+ subject="My PPA (Ubuntu Precise) has git-annex 4.20130417"
+ date="2013-04-27T11:12:26Z"
+ content="""
+I just went through the pain of backporting 90+ Haskell packages from Debian unstable: https://launchpad.net/~fmarier/+archive/ppa
+"""]]
diff --git a/doc/forum/Ubuntu_PPA/comment_4_3a8bbd0a7450a7f5323cd13144824aea._comment b/doc/forum/Ubuntu_PPA/comment_4_3a8bbd0a7450a7f5323cd13144824aea._comment
new file mode 100644
index 0000000000..335cc57ba4
--- /dev/null
+++ b/doc/forum/Ubuntu_PPA/comment_4_3a8bbd0a7450a7f5323cd13144824aea._comment
@@ -0,0 +1,12 @@
+[[!comment format=mdwn
+ username="http://grossmeier.net/"
+ nickname="greg"
+ subject="All PPAs are outdated"
+ date="2013-05-08T17:38:04Z"
+ content="""
+My work preinstalled Ubuntu machine is now on Raring, and even there the version of git-annex is only 3.20121112ubuntu3; too old to have the webapp option enabled :/ The two PPAs mentioned here aren't configured to build for Raring, so even though the one from François has a build from May 1st (7 days ago), it doesn't help me ;)
+
+François: mind updating your PPA to support Raring? Thanks much if you do!
+
+Either that, or I'll take the time some day to do a proper install on this machine (read: Debian). ;-)
+"""]]
diff --git a/doc/forum/Ubuntu_PPA/comment_5_2e1beaeebda0201c635db8b276cedf20._comment b/doc/forum/Ubuntu_PPA/comment_5_2e1beaeebda0201c635db8b276cedf20._comment
new file mode 100644
index 0000000000..0d94ba6f22
--- /dev/null
+++ b/doc/forum/Ubuntu_PPA/comment_5_2e1beaeebda0201c635db8b276cedf20._comment
@@ -0,0 +1,12 @@
+[[!comment format=mdwn
+ username="https://www.google.com/accounts/o8/id?id=AItOawmBUR4O9mofxVbpb8JV9mEbVfIYv670uJo"
+ nickname="Justin"
+ subject="comment 5"
+ date="2013-05-08T23:25:37Z"
+ content="""
+If you are on raring you can still use
+
+    deb http://ppa.launchpad.net/fmarier/ppa/ubuntu precise main
+
+It's worked fine for me so far.
+"""]]
diff --git a/doc/forum/Ubuntu_PPA/comment_6_bd99fb70399fc58d98781a89c6d38428._comment b/doc/forum/Ubuntu_PPA/comment_6_bd99fb70399fc58d98781a89c6d38428._comment
new file mode 100644
index 0000000000..8252fe5d93
--- /dev/null
+++ b/doc/forum/Ubuntu_PPA/comment_6_bd99fb70399fc58d98781a89c6d38428._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="https://www.google.com/accounts/o8/id?id=AItOawkx5V3MTbzCXS3J7Mn9FEq8M9bPPYMkAHY"
+ nickname="Pedro"
+ subject="comment 6"
+ date="2013-05-20T15:52:40Z"
+ content="""
+Be careful that the fmarier ppa includes more than just git-annex.
+"""]]
diff --git a/doc/forum/Ubuntu_PPA/comment_7_c3f7ec8573934c59d70a48e36e321c13._comment b/doc/forum/Ubuntu_PPA/comment_7_c3f7ec8573934c59d70a48e36e321c13._comment
new file mode 100644
index 0000000000..849381ec29
--- /dev/null
+++ b/doc/forum/Ubuntu_PPA/comment_7_c3f7ec8573934c59d70a48e36e321c13._comment
@@ -0,0 +1,12 @@
+[[!comment format=mdwn
+ username="https://openid.fmarier.org/"
+ nickname="fmarier"
+ subject="New location for my git-annex PPA"
+ date="2013-06-15T07:42:13Z"
+ content="""
+My Ubuntu Precise PPA has moved here: https://launchpad.net/~fmarier/+archive/git-annex
+
+It contains version 20130601 of git-annex.
+
+As Pedro suggested, it's now in a separate PPA with only git-annex and the 220 Haskell packages it depends on.
+"""]]
diff --git a/doc/forum/Ubuntu_PPA/comment_8_feecd478a4d44cb5ffc7020c646174ba._comment b/doc/forum/Ubuntu_PPA/comment_8_feecd478a4d44cb5ffc7020c646174ba._comment
new file mode 100644
index 0000000000..51fde1f039
--- /dev/null
+++ b/doc/forum/Ubuntu_PPA/comment_8_feecd478a4d44cb5ffc7020c646174ba._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="https://www.google.com/accounts/o8/id?id=AItOawntodrSgODU27WUCyN2PV7TC14YMkyaoxQ"
+ nickname="Dennis"
+ subject="Ubuntu Trusty 14.04"
+ date="2015-01-24T03:19:41Z"
+ content="""
+I am running into the same issue with Ubuntu Trusty 14.04. The version shipped is 5.20140412ubuntu1, but I would like to try a more recent version to see if an issue I experience still persists. Is there any up-to-date PPA available for Trusty?
+"""]]
diff --git a/doc/forum/Un-git-annex__63__.mdwn b/doc/forum/Un-git-annex__63__.mdwn
new file mode 100644
index 0000000000..68e8068fc1
--- /dev/null
+++ b/doc/forum/Un-git-annex__63__.mdwn
@@ -0,0 +1,6 @@
+Hi,
+
+I am using git-annex assistant on my Fedora Linux computer. Everything seems to be working perfectly, however I have noticed that almost all of my files are now symbolic links to cryptic file names located in the .git/ directory. Oddly though it seems files inside git-annex assistant does not move/symbolic link files within git repositories inside my git-annex assist directories. My question is: how can I safely uninstall git-annex from my machine and restore all files to their non-symbolic link states? When I click on the settings drop-down for a repository inside git-annex assistant and choose disable it goes to the same page as the delete option. Any ideas?
+
+Regards,
+Blake
diff --git a/doc/forum/Un-git-annex__63__/comment_1_6059265afb66190d325083e0f28bcf33._comment b/doc/forum/Un-git-annex__63__/comment_1_6059265afb66190d325083e0f28bcf33._comment
new file mode 100644
index 0000000000..907293f99d
--- /dev/null
+++ b/doc/forum/Un-git-annex__63__/comment_1_6059265afb66190d325083e0f28bcf33._comment
@@ -0,0 +1,10 @@
+[[!comment format=mdwn
+ username="http://edheil.wordpress.com/"
+ ip="99.54.57.201"
+ subject="comment 1"
+ date="2013-10-06T06:45:14Z"
+ content="""
+\"git annex uninit\" is the command which will revert an entire git annex repository back to a bunch of ordinary files, untracked by git annex (or git).  I'm not sure whether there is a way to issue that command from the assistant though.
+
+
+"""]]
diff --git a/doc/forum/Un-git-annex__63__/comment_2_fac4bfb81dbbf0dc82059aace261eb51._comment b/doc/forum/Un-git-annex__63__/comment_2_fac4bfb81dbbf0dc82059aace261eb51._comment
new file mode 100644
index 0000000000..2b1a7381ae
--- /dev/null
+++ b/doc/forum/Un-git-annex__63__/comment_2_fac4bfb81dbbf0dc82059aace261eb51._comment
@@ -0,0 +1,10 @@
+[[!comment format=mdwn
+ username="https://www.google.com/accounts/o8/id?id=AItOawlOc-EOD5ZyggsAp6lOnU7x5MxizwLtUXA"
+ nickname="Hendrik"
+ subject="comment 2"
+ date="2013-10-06T18:35:14Z"
+ content="""
+I had uninit'd my annex several time for testing git-annex. What I discovered was that is worked reliably but for all files that are duplicates. For duplicates only 1 of the files got restored properly.  All that was 1 year ago - so your results my differ. I had found a piece of script in the forum / tips sections that will show all duplicates.
+
+Hope this helps...
+"""]]
diff --git a/doc/forum/Unable_to_change_RSS_feeds.mdwn b/doc/forum/Unable_to_change_RSS_feeds.mdwn
new file mode 100644
index 0000000000..02bccd6c3a
--- /dev/null
+++ b/doc/forum/Unable_to_change_RSS_feeds.mdwn
@@ -0,0 +1,8 @@
+Hello. It seems that one of the RSS feed I was using for a podcast is now pointing to a missing server, and so I started getting the error `Unable to access these remotes: web`.
+I found another feed that does not have this problem, but I have not been able to update my git annex repository with this new URL.
+
+I have tried to add the new feed to the feed list and `importfeed` and have tried to use `addurl` to provide an alternate source, but that did not change anything. I tried to remove files and run `importfeed` but `info` always gives the same URL and `get` fails.
+
+What is the proper way to go about this? I am using `importfeed --relaxed --fast --template=...` to pull in the information.
+
+Thank you for any help you can provide.
diff --git a/doc/forum/Understanding___34__deletion__34___and___34__dropping__34_____38___cleanup_of_history.mdwn b/doc/forum/Understanding___34__deletion__34___and___34__dropping__34_____38___cleanup_of_history.mdwn
new file mode 100644
index 0000000000..9c6c3df73e
--- /dev/null
+++ b/doc/forum/Understanding___34__deletion__34___and___34__dropping__34_____38___cleanup_of_history.mdwn
@@ -0,0 +1,9 @@
+Hi!
+
+I have a few questions according to deletion and dropping with git-annex:
+
+1. I couldn't figure out what unused files really are. What is *unused* related to? (Head of) branches? How does a file become unused?  
+2. If I'm working in indirect mode, I can safely `git rm` files and restore them later, by checking out the corresponding commit. Consider several git-annex repos being in sync, where the repository contains a nested directory structure. Is there a way to completely remove all deletes files within some directory (which might but not must be the root of the repo), that have been deleted for more than X days? For example I would want to run something like `git annex drop-deleted --from-folder some/subdirectory/ --older-than 5`. Is there a way to achieve this?
+3. Just like 2, is there a way to remove (the content of) all non-current *versions* of all file that are older than Y days?
+
+Thanks in advance!
diff --git a/doc/forum/Understanding___34__deletion__34___and___34__dropping__34_____38___cleanup_of_history/comment_1_08dbad67df1d8f07d3883c6fed2fe0c4._comment b/doc/forum/Understanding___34__deletion__34___and___34__dropping__34_____38___cleanup_of_history/comment_1_08dbad67df1d8f07d3883c6fed2fe0c4._comment
new file mode 100644
index 0000000000..84ec31fd87
--- /dev/null
+++ b/doc/forum/Understanding___34__deletion__34___and___34__dropping__34_____38___cleanup_of_history/comment_1_08dbad67df1d8f07d3883c6fed2fe0c4._comment
@@ -0,0 +1,16 @@
+[[!comment format=mdwn
+ username="joey"
+ subject="""comment 1"""
+ date="2015-02-04T19:55:54Z"
+ content="""
+`git-annex unused` finds files that are not used by the heads of any
+branches or tags. These files may still be referred to in individual
+commits in git history, or there could be no commits at all that 
+refer to them. It's up to you to decide whether these are really
+"unused" or not.
+
+git-annex doesn't currently have an concept of file age. You can
+look at git history to find that out, but it's quite expensive 
+to do that for a lot of files, and so a --older-than option
+would make it pretty slow.
+"""]]
diff --git a/doc/forum/Undo_Git_Annex_Changes_To_Linked_Files.mdwn b/doc/forum/Undo_Git_Annex_Changes_To_Linked_Files.mdwn
new file mode 100644
index 0000000000..c299041f02
--- /dev/null
+++ b/doc/forum/Undo_Git_Annex_Changes_To_Linked_Files.mdwn
@@ -0,0 +1,7 @@
+Hello,
+
+I was recently evaluating git-annex as a file sync tool, but I've got my files in a state where they are a mess.  The short version is that I have a local repo, and then an Encrypted S3 remote.  Once I encrypted the files, they got renamed and so forth and the files are now just symbolic links to files within the .git directory somewhere.   This has made some of them unusable for me and I need to revert the files back to not being symlinks any longer.  How would I go about doing that?
+
+Thanks,
+
+Brandon
diff --git a/doc/forum/Undo_Git_Annex_Changes_To_Linked_Files/comment_1_568dde820c2608d86d05b07444146a26._comment b/doc/forum/Undo_Git_Annex_Changes_To_Linked_Files/comment_1_568dde820c2608d86d05b07444146a26._comment
new file mode 100644
index 0000000000..21cf455229
--- /dev/null
+++ b/doc/forum/Undo_Git_Annex_Changes_To_Linked_Files/comment_1_568dde820c2608d86d05b07444146a26._comment
@@ -0,0 +1,13 @@
+[[!comment format=mdwn
+ username="http://joeyh.name/"
+ ip="4.152.246.110"
+ subject="comment 1"
+ date="2013-07-28T00:09:45Z"
+ content="""
+Run:
+
+`git annex get $file`
+`git annex unannex $file`
+
+$file will now be back how it was before you used git-annex. Can also run above on whole directories.
+"""]]
diff --git a/doc/forum/Undo_Git_Annex_Changes_To_Linked_Files/comment_2_a8cf71cdf1217d9c8596cd9006eb83f5._comment b/doc/forum/Undo_Git_Annex_Changes_To_Linked_Files/comment_2_a8cf71cdf1217d9c8596cd9006eb83f5._comment
new file mode 100644
index 0000000000..2e9c387ad4
--- /dev/null
+++ b/doc/forum/Undo_Git_Annex_Changes_To_Linked_Files/comment_2_a8cf71cdf1217d9c8596cd9006eb83f5._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="https://www.google.com/accounts/o8/id?id=AItOawmRFsVANDSwNHzCWk-iP2_nXnfYKPJWziE"
+ nickname="Brandon"
+ subject="comment 2"
+ date="2013-07-28T03:26:23Z"
+ content="""
+Thanks, much appreciated.  Got all of my files back successfully.
+"""]]
diff --git a/doc/forum/Undo_git_merge_git-annex.mdwn b/doc/forum/Undo_git_merge_git-annex.mdwn
new file mode 100644
index 0000000000..bc299174cb
--- /dev/null
+++ b/doc/forum/Undo_git_merge_git-annex.mdwn
@@ -0,0 +1,3 @@
+After accidentally typing git merge git-annex, I am now wondering how to clean up the resulting chaos...
+
+Any tips?
diff --git a/doc/forum/Undo_git_merge_git-annex/comment_1_0b15f7a8c3bca87dcbf748a229e13a4b._comment b/doc/forum/Undo_git_merge_git-annex/comment_1_0b15f7a8c3bca87dcbf748a229e13a4b._comment
new file mode 100644
index 0000000000..bbc6117f7b
--- /dev/null
+++ b/doc/forum/Undo_git_merge_git-annex/comment_1_0b15f7a8c3bca87dcbf748a229e13a4b._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="frederik@ffbea6a549cb3f460d110386c0f634c1ddc6a68a"
+ nickname="frederik"
+ subject="comment 1"
+ date="2016-03-04T14:06:17Z"
+ content="""
+git revert -m 1 [sha_of_merge] seems to have done the trick
+"""]]
diff --git a/doc/forum/Undo_git_merge_git-annex/comment_2_c5a8839c53145a3b0d44950096c5180f._comment b/doc/forum/Undo_git_merge_git-annex/comment_2_c5a8839c53145a3b0d44950096c5180f._comment
new file mode 100644
index 0000000000..4db2007616
--- /dev/null
+++ b/doc/forum/Undo_git_merge_git-annex/comment_2_c5a8839c53145a3b0d44950096c5180f._comment
@@ -0,0 +1,9 @@
+[[!comment format=mdwn
+ username="joey"
+ subject="""comment 2"""
+ date="2016-03-07T17:25:08Z"
+ content="""
+`git reflog` is a handy command to get to know, for getting out of
+situations along the lines of "I did $something wrong and want to get back
+to a previous version of a branch"
+"""]]
diff --git a/doc/forum/Unknown_remote_type_S3.mdwn b/doc/forum/Unknown_remote_type_S3.mdwn
new file mode 100644
index 0000000000..ae8432286c
--- /dev/null
+++ b/doc/forum/Unknown_remote_type_S3.mdwn
@@ -0,0 +1,5 @@
+I have an annex set up with an s3 special remote. It works fine on one computer. I cloned the annex to a USB stick, plugged the stick into a second computer and cloned the annex to that machine. When I try to `git annex initremote cloud` on the second machine, I get the following error:
+
+    git-annex: Unknown remote type S3
+
+The second machine is running git-annex 3.20121017.
diff --git a/doc/forum/Unknown_remote_type_S3/comment_1_2aea2cd51286c809427d16519606cd37._comment b/doc/forum/Unknown_remote_type_S3/comment_1_2aea2cd51286c809427d16519606cd37._comment
new file mode 100644
index 0000000000..80e1b11aa1
--- /dev/null
+++ b/doc/forum/Unknown_remote_type_S3/comment_1_2aea2cd51286c809427d16519606cd37._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="http://joeyh.name/"
+ ip="4.152.108.27"
+ subject="comment 1"
+ date="2012-11-07T18:57:17Z"
+ content="""
+It's possible to build git-annex without S3 support. I think a few Linux distros ship it this way due to not having all the necessary haskell libraries.
+"""]]
diff --git a/doc/forum/Unknown_remote_type_S3/comment_2_06f775062cd30767979fe56bcb3cf7bf._comment b/doc/forum/Unknown_remote_type_S3/comment_2_06f775062cd30767979fe56bcb3cf7bf._comment
new file mode 100644
index 0000000000..75864aedc5
--- /dev/null
+++ b/doc/forum/Unknown_remote_type_S3/comment_2_06f775062cd30767979fe56bcb3cf7bf._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="annexuser"
+ ip="24.16.193.140"
+ subject="comment 2"
+ date="2012-11-07T20:12:27Z"
+ content="""
+That was the issue. Thank you!
+"""]]
diff --git a/doc/forum/Unlock_files_when_assistant_is_running__63__.mdwn b/doc/forum/Unlock_files_when_assistant_is_running__63__.mdwn
new file mode 100644
index 0000000000..0c514368ab
--- /dev/null
+++ b/doc/forum/Unlock_files_when_assistant_is_running__63__.mdwn
@@ -0,0 +1,13 @@
+I just started using the assistant on an existing annex. I fire up the assistant like so:
+
+    $ git annex webapp
+
+Everything syncs and looks to be working fine. In another terminal, I then create a new file:
+
+    $ touch testfile.txt
+
+The assistant sees that file, immediately adds it and syncs. Ok. So now I want to edit that file.
+
+    $ git annex unlock testfile.txt
+
+As soon as I unlock the file, the assistant re-adds it to the annex and syncs, preventing me from editing the file. How can I edit files with the assistant running?
diff --git a/doc/forum/Unlock_files_when_assistant_is_running__63__/comment_1_3f4aadf0c856c81e15c6f5ae7f1992b4._comment b/doc/forum/Unlock_files_when_assistant_is_running__63__/comment_1_3f4aadf0c856c81e15c6f5ae7f1992b4._comment
new file mode 100644
index 0000000000..f17fc56345
--- /dev/null
+++ b/doc/forum/Unlock_files_when_assistant_is_running__63__/comment_1_3f4aadf0c856c81e15c6f5ae7f1992b4._comment
@@ -0,0 +1,10 @@
+[[!comment format=mdwn
+ username="http://joeyh.name/"
+ ip="4.152.108.27"
+ subject="comment 1"
+ date="2012-11-11T16:06:42Z"
+ content="""
+I suspect you have an old build of the assistant. Version 3.20121009 was supposed to fix this.
+
+I re-tested with current git head today, and found I was able to unlock a file, repeatedly edit it in place without the assistant re-adding it, and then run \"git annex add\" and the assistant auto-committed the changed file. Better behavior than I was expecting in fact -- the ability to repeatedly edit is a pleasant surprise.
+"""]]
diff --git a/doc/forum/Unlock_files_when_assistant_is_running__63__/comment_2_a76797ee9e05e43af7947508cadd7bed._comment b/doc/forum/Unlock_files_when_assistant_is_running__63__/comment_2_a76797ee9e05e43af7947508cadd7bed._comment
new file mode 100644
index 0000000000..5567205235
--- /dev/null
+++ b/doc/forum/Unlock_files_when_assistant_is_running__63__/comment_2_a76797ee9e05e43af7947508cadd7bed._comment
@@ -0,0 +1,9 @@
+[[!comment format=mdwn
+ username="annexuser"
+ ip="24.16.193.140"
+ subject="comment 2"
+ date="2012-11-12T20:09:20Z"
+ content="""
+    $ git annex version                                                   
+    git-annex version: 3.20121017
+"""]]
diff --git a/doc/forum/Unlocked_directory__63___Having_all_its_new_files_unlocked.mdwn b/doc/forum/Unlocked_directory__63___Having_all_its_new_files_unlocked.mdwn
new file mode 100644
index 0000000000..7c8e55fc4e
--- /dev/null
+++ b/doc/forum/Unlocked_directory__63___Having_all_its_new_files_unlocked.mdwn
@@ -0,0 +1,13 @@
+Hello!
+
+How can I have just a single directory unlocked at all times?
+
+• I’m running the Assistant.
+
+• If a new file is created in it, I’d like it to be added to annex BUT automatically unlocked (kept editable).
+
+• If such an unlocked file is changed, I’d like the Assistant to sync the change.
+
+How can this be configured?
+
+Thank you!
diff --git a/doc/forum/Unlocked_directory__63___Having_all_its_new_files_unlocked/comment_1_cd08064d171bea606df1cb7dafbdd381._comment b/doc/forum/Unlocked_directory__63___Having_all_its_new_files_unlocked/comment_1_cd08064d171bea606df1cb7dafbdd381._comment
new file mode 100644
index 0000000000..36c6b89f0b
--- /dev/null
+++ b/doc/forum/Unlocked_directory__63___Having_all_its_new_files_unlocked/comment_1_cd08064d171bea606df1cb7dafbdd381._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="michalrus"
+ avatar="http://cdn.libravatar.org/avatar/83c0b6e7f9d20f09a892263c4903bbae"
+ subject="comment 1"
+ date="2017-08-16T11:03:03Z"
+ content="""
+So something along the lines of direct mode but *just for one* subdirectory.
+"""]]
diff --git a/doc/forum/Unlocked_directory__63___Having_all_its_new_files_unlocked/comment_2_6da3347c2d9243a9fe61fd75999d6f36._comment b/doc/forum/Unlocked_directory__63___Having_all_its_new_files_unlocked/comment_2_6da3347c2d9243a9fe61fd75999d6f36._comment
new file mode 100644
index 0000000000..5efa8f82ab
--- /dev/null
+++ b/doc/forum/Unlocked_directory__63___Having_all_its_new_files_unlocked/comment_2_6da3347c2d9243a9fe61fd75999d6f36._comment
@@ -0,0 +1,12 @@
+[[!comment format=mdwn
+ username="michalrus"
+ avatar="http://cdn.libravatar.org/avatar/83c0b6e7f9d20f09a892263c4903bbae"
+ subject="comment 2"
+ date="2017-08-16T12:47:48Z"
+ content="""
+Whoa.
+
+I just upgraded my repo to v6 (6.20170520), and it seems that a running assistant is adding every new file unlocked. :-o
+
+How can I control which file will get added unlocked and which locked by the assistant?
+"""]]
diff --git a/doc/forum/Unlocked_directory__63___Having_all_its_new_files_unlocked/comment_3_78dfedd93393709d3d7e417b779109d4._comment b/doc/forum/Unlocked_directory__63___Having_all_its_new_files_unlocked/comment_3_78dfedd93393709d3d7e417b779109d4._comment
new file mode 100644
index 0000000000..22a5974c3f
--- /dev/null
+++ b/doc/forum/Unlocked_directory__63___Having_all_its_new_files_unlocked/comment_3_78dfedd93393709d3d7e417b779109d4._comment
@@ -0,0 +1,12 @@
+[[!comment format=mdwn
+ username="michalrus"
+ avatar="http://cdn.libravatar.org/avatar/83c0b6e7f9d20f09a892263c4903bbae"
+ subject="comment 3"
+ date="2017-08-16T12:58:24Z"
+ content="""
+But it’s better. As a dirty workaround, I could add a cron job/timer that would periodically git-annex-lock all files *except* that single directory I want unlocked.
+
+But a static configuration—say, in .gitattributes—would be way better. ♥
+
+Or is there a way to configure that already?
+"""]]
diff --git a/doc/forum/Unlocked_directory__63___Having_all_its_new_files_unlocked/comment_4_bdd6cb9cdf2366e5d930c0cdd80b3984._comment b/doc/forum/Unlocked_directory__63___Having_all_its_new_files_unlocked/comment_4_bdd6cb9cdf2366e5d930c0cdd80b3984._comment
new file mode 100644
index 0000000000..a2efa3955d
--- /dev/null
+++ b/doc/forum/Unlocked_directory__63___Having_all_its_new_files_unlocked/comment_4_bdd6cb9cdf2366e5d930c0cdd80b3984._comment
@@ -0,0 +1,13 @@
+[[!comment format=mdwn
+ username="joey"
+ subject="""comment 4"""
+ date="2017-08-17T15:56:25Z"
+ content="""
+There's the annex.addunlocked setting which controls this in v6
+repositories, but it's currently limited to true/false, can't configure
+different behavior in different directories.
+
+I'd probably need a motivating example to make it configurable in
+.gitattributes. Having a hard time currently understanding how that would
+be useful.
+"""]]
diff --git a/doc/forum/Unlocked_directory__63___Having_all_its_new_files_unlocked/comment_5_7a2109bbe16de4c05db48763d00bfaf1._comment b/doc/forum/Unlocked_directory__63___Having_all_its_new_files_unlocked/comment_5_7a2109bbe16de4c05db48763d00bfaf1._comment
new file mode 100644
index 0000000000..df3883bd22
--- /dev/null
+++ b/doc/forum/Unlocked_directory__63___Having_all_its_new_files_unlocked/comment_5_7a2109bbe16de4c05db48763d00bfaf1._comment
@@ -0,0 +1,18 @@
+[[!comment format=mdwn
+ username="michalrus"
+ avatar="http://cdn.libravatar.org/avatar/83c0b6e7f9d20f09a892263c4903bbae"
+ subject="comment 5"
+ date="2017-08-17T17:53:24Z"
+ content="""
+My particular example is that I’m using git-annex:
+
+• mostly as an archive (esp. in family files context),
+
+• but also as a sort-of FLOSS Dropbox for some specific files, that we’re editing.
+
+I’d like to prevent easy (accidental, really) editing of those archived files.
+
+Does this make sense?
+
+Thank you!
+"""]]
diff --git a/doc/forum/Update_on_remote_repo.mdwn b/doc/forum/Update_on_remote_repo.mdwn
new file mode 100644
index 0000000000..4977c41597
--- /dev/null
+++ b/doc/forum/Update_on_remote_repo.mdwn
@@ -0,0 +1,6 @@
+Hello,
+
+using the git-annex assistant I can activate auto update on the client machines. What about the server that serves as transfer repository? It needs to have git-annex installed. Is there any possibility of auto update there? What is the recommended procedure?
+
+Thanks,
+Florian
diff --git a/doc/forum/Update_on_remote_repo/comment_1_d58cfe914d6c36d8bff8d1342e1c2102._comment b/doc/forum/Update_on_remote_repo/comment_1_d58cfe914d6c36d8bff8d1342e1c2102._comment
new file mode 100644
index 0000000000..d4a7a62e6f
--- /dev/null
+++ b/doc/forum/Update_on_remote_repo/comment_1_d58cfe914d6c36d8bff8d1342e1c2102._comment
@@ -0,0 +1,11 @@
+[[!comment format=mdwn
+ username="joey"
+ subject="""comment 1"""
+ date="2014-12-02T17:05:05Z"
+ content="""
+You can run the git-annex assistant in any (non-bare) git repository,
+including one on the server.
+
+A git post-receive hook is another way to update a repository on
+a server. See [[tips/setup_a_public_repository_on_a_web_site]]
+"""]]
diff --git a/doc/forum/Upspin.mdwn b/doc/forum/Upspin.mdwn
new file mode 100644
index 0000000000..0ef764130a
--- /dev/null
+++ b/doc/forum/Upspin.mdwn
@@ -0,0 +1,5 @@
+As soon as I read about Upspin, I thought about git-annex.  Could Upspin's interfaces be beneficial to the project and provide a standardization for naming, permissions and access control?
+
+I realize some of these concepts are not yet part of git-annex, but I thought I'd drop the note and give you the opportunity to evaluate the extent in which Upspin makes sense in the context of git-annex.
+
+https://upspin.io/
diff --git a/doc/forum/Upspin/comment_1_30adffff50a043c14b765ebdc6d637d2._comment b/doc/forum/Upspin/comment_1_30adffff50a043c14b765ebdc6d637d2._comment
new file mode 100644
index 0000000000..7281f29298
--- /dev/null
+++ b/doc/forum/Upspin/comment_1_30adffff50a043c14b765ebdc6d637d2._comment
@@ -0,0 +1,9 @@
+[[!comment format=mdwn
+ username="unicell@9c0b0afd4176d5933d4b5c41350ebe61488c1df0"
+ nickname="unicell"
+ avatar="http://cdn.libravatar.org/avatar/ac836d39c7be1503f636843bdcd7b199"
+ subject="comment 1"
+ date="2017-02-23T23:05:10Z"
+ content="""
+Wondering exactly the same thing
+"""]]
diff --git a/doc/forum/Use_case__58___Main_machine__44___portable_drive__44___backup_machine.mdwn b/doc/forum/Use_case__58___Main_machine__44___portable_drive__44___backup_machine.mdwn
new file mode 100644
index 0000000000..92885a01b6
--- /dev/null
+++ b/doc/forum/Use_case__58___Main_machine__44___portable_drive__44___backup_machine.mdwn
@@ -0,0 +1,26 @@
+Hello everybody, I need some setup advice.
+
+I want to set up a portability and data security (in the sense of redundancy) scenario for a user with some, but not exulting, tech skills.
+A directory that contains progressing work should be synced with incoming data and backed up.
+
+## Locations
+
+- Main machine (laptop), containing all software to process data and write result files
+- USB thumb drive, will occassionally receive additional data at the faculty (no root access/possibility to install annex or even git there)
+- Offline file server (ssh/sftp through wired lan), backing up all data
+
+## Goals
+
+- The laptop and USB drive should stay in sync in both directions; processed work must appear on the USB, data copied onto the USB must appear on the main machine (copies of files, not links).
+- All data should be backed up on the file server.
+- The user should not be required to stage/commit files from the command line, this should be handled automatically (annex assistant?)
+
+
+## Questions
+
+- This sounds a bit like the "nomad" scenario described, but not totally like it. Is git annex the proper tool for the job?
+- How do I set that scenario up? Especially, who is a remote of what and why, and how do I make sure I got a copy of every file in every location without doing it all manually?
+
+At the moment this is all done with a number of grsync sessions. It works, but if anything is done in the wrong order, current data is overwritten by older one (this could be resolved using git, but would require manual staging/commiting, what we want to avoid).
+
+Can you help me on that?
diff --git a/doc/forum/Use_case__58___Main_machine__44___portable_drive__44___backup_machine/comment_1_d2991b94eb067f0a6e47cd77f922411d._comment b/doc/forum/Use_case__58___Main_machine__44___portable_drive__44___backup_machine/comment_1_d2991b94eb067f0a6e47cd77f922411d._comment
new file mode 100644
index 0000000000..5f915607a5
--- /dev/null
+++ b/doc/forum/Use_case__58___Main_machine__44___portable_drive__44___backup_machine/comment_1_d2991b94eb067f0a6e47cd77f922411d._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="https://me.yahoo.com/a/hi7yosAfnufqtL0Mb123WhWS.Yel7LOeB6A-#1512c"
+ nickname="-"
+ subject="One more requirement"
+ date="2016-08-02T06:58:04Z"
+ content="""
+Oh, I forgot: data must be writable at least on the main machine, possibly on the thumb drive, too.
+"""]]
diff --git a/doc/forum/Use_case__58___Main_machine__44___portable_drive__44___backup_machine/comment_2_0b778710b23eab1413256e289f0aa180._comment b/doc/forum/Use_case__58___Main_machine__44___portable_drive__44___backup_machine/comment_2_0b778710b23eab1413256e289f0aa180._comment
new file mode 100644
index 0000000000..9a4530b41a
--- /dev/null
+++ b/doc/forum/Use_case__58___Main_machine__44___portable_drive__44___backup_machine/comment_2_0b778710b23eab1413256e289f0aa180._comment
@@ -0,0 +1,32 @@
+[[!comment format=mdwn
+ username="joey"
+ subject="""comment 2"""
+ date="2016-09-21T19:59:25Z"
+ content="""
+Use git-annex repositories on both the laptop and the thumb drive.
+
+You can start by setting up the repository on the thumb drive,
+and then git clone it to the laptop.
+
+Use `git annex direct` to make them use direct mode, so that
+files can be edited.
+
+You can run the git-annex assistant in the repository on the laptop
+(~/annex is the usual place for such a repository). This way
+changes to files there will automatically be committed to git.
+
+There's not currently a good way to have the assistant also commit
+any files that were added to the thumb drive, or merge in changes
+from the laptop to the thumb drive. Running the assistant in the repository
+on the thumb drive would take care of both, but fails when the thumb
+drive gets unplugged.
+
+What you can do is write a simple shell script to sync up the thumb drive.
+Something like this:
+
+	#!/bin/sh
+	set -e
+	cd /media/thumbdrive
+	git annex add
+	git annex sync -m "syncing thumb drive"
+"""]]
diff --git a/doc/forum/Use_case_for_addurl.mdwn b/doc/forum/Use_case_for_addurl.mdwn
new file mode 100644
index 0000000000..ee18107bd3
--- /dev/null
+++ b/doc/forum/Use_case_for_addurl.mdwn
@@ -0,0 +1,8 @@
+Hi,
+
+
+question. What is exactly the use-case for addurl. To add unique files as if they where never changing resources?
+
+There is a --relaxed flag to enable some leniency in redownloading the file it seems. But how would one go about updating a file from the web, drop it first?
+Or do I have to add pseudo queries or hashes to the URLs to keep them unique and manage any versioning that way perhaps?
+
diff --git a/doc/forum/Use_case_for_addurl/comment_1_2205b1857158cca4f6e2225c53bec1d5._comment b/doc/forum/Use_case_for_addurl/comment_1_2205b1857158cca4f6e2225c53bec1d5._comment
new file mode 100644
index 0000000000..608370c58c
--- /dev/null
+++ b/doc/forum/Use_case_for_addurl/comment_1_2205b1857158cca4f6e2225c53bec1d5._comment
@@ -0,0 +1,16 @@
+[[!comment format=mdwn
+ username="joey"
+ subject="""comment 1"""
+ date="2015-11-10T18:59:13Z"
+ content="""
+The use case is to be able to `git annex addurl` in one repository
+and `git annex get` in another checkout, without needing to remember 
+the url yourself. Or to use `git annex whereis` to look up the url
+associated with a file.
+
+When --relaxed is used, you're telling git-annex you want to treat
+the current and future content of the url as close enough to the same
+as to not matter. I wouldn't use it if I expected the content of an url to 
+change significantly, and git-annex is not able to help with manageing such
+differences.
+"""]]
diff --git a/doc/forum/Use_case_with_syncing_only_a_subset_possible__63__.mdwn b/doc/forum/Use_case_with_syncing_only_a_subset_possible__63__.mdwn
new file mode 100644
index 0000000000..0d102b0d40
--- /dev/null
+++ b/doc/forum/Use_case_with_syncing_only_a_subset_possible__63__.mdwn
@@ -0,0 +1,13 @@
+I'm trying to figure out if git-annex can be used for the following use case:
+
+- 2 repos in direct mode A and B
+- A is on a local filesystem, B could be a NFS mountpoint or ssh
+- B contains a number of toplevel directories, A contains a subset of those
+- For the subset in A all changes in those directories that happen on A or B should be synced automatically between both repos, including file removals.
+- A can decide to not carry a toplevel dir any more, but it must still exist in B.
+- A can copy new toplevel dirs from B and those should be synced from then on.
+
+I've been looking at the docs and played with two test repos, but I cannot seem to make the above work. To me it looks like I would need a possibility include/exclude paths from syncing via a regex.
+
+Thanks,
+Felix
diff --git a/doc/forum/Use_case_with_syncing_only_a_subset_possible__63__/comment_1_a0a272a0931b27e5c94b93e42656b62c._comment b/doc/forum/Use_case_with_syncing_only_a_subset_possible__63__/comment_1_a0a272a0931b27e5c94b93e42656b62c._comment
new file mode 100644
index 0000000000..3e930b5424
--- /dev/null
+++ b/doc/forum/Use_case_with_syncing_only_a_subset_possible__63__/comment_1_a0a272a0931b27e5c94b93e42656b62c._comment
@@ -0,0 +1,10 @@
+[[!comment format=mdwn
+ username="http://joeyh.name/"
+ ip="4.154.4.51"
+ subject="comment 1"
+ date="2013-09-12T20:39:51Z"
+ content="""
+I guess you might be using the git-annex assistant. If you were just using git-annex at the command line, you could simply only run `git annex get` on the files or directories you wanted to get.
+
+You can control which directories the git-annex assistant downloads/uploads files for using [[preferred_content]] expressions. Or the easy solution is to use the webapp to put the repository into \"manual mode\" and then manually run `git annex get` as described above to get just the files you want. (And `git annex copy --to remote` to share new files you make.)
+"""]]
diff --git a/doc/forum/Use_existing_S3_bucket.mdwn b/doc/forum/Use_existing_S3_bucket.mdwn
new file mode 100644
index 0000000000..5f215b5cb8
--- /dev/null
+++ b/doc/forum/Use_existing_S3_bucket.mdwn
@@ -0,0 +1,8 @@
+Hello,
+
+is it possible to use an existing AWS S3 bucket (one I created in the AWS web interface) when executing initremote? I tried but git annex failed with s3ErrorCode = "BucketAlreadyOwnedByYou".
+
+Background is that http://git-annex.branchable.com/special_remotes/S3/ only mentions EU for datacenter. But there are three possible EU datacenters now. So I created a bucket in EU (Frankfurt) and tried to use this one with bucket=mybucket
+
+Best Thanks!
+
diff --git a/doc/forum/Use_existing_S3_bucket/comment_1_ed0ead0b09ddbe52d42150a281d84f5c._comment b/doc/forum/Use_existing_S3_bucket/comment_1_ed0ead0b09ddbe52d42150a281d84f5c._comment
new file mode 100644
index 0000000000..87c4c1345c
--- /dev/null
+++ b/doc/forum/Use_existing_S3_bucket/comment_1_ed0ead0b09ddbe52d42150a281d84f5c._comment
@@ -0,0 +1,16 @@
+[[!comment format=mdwn
+ username="joey"
+ subject="""comment 1"""
+ date="2017-02-13T18:01:40Z"
+ content="""
+It's not supposed to fail when the bucket already exists. There is an
+explicit check to see whether the bucket exists in current versions.
+
+Unfortunately, git-annex can't support the Frankfurt datacenter,
+until this API problem gets fixed 
+
+In my tests, it doesn't fail that way when the bucket already exists.
+I didn't create the bucket in Frankfurt though, and it could be
+that due to the API problem it fails to notice that the bucket in Frankfurt
+already exists.
+"""]]
diff --git a/doc/forum/Use_existing_S3_bucket/comment_2_6b108846fca4a514d507f56d1052fdde._comment b/doc/forum/Use_existing_S3_bucket/comment_2_6b108846fca4a514d507f56d1052fdde._comment
new file mode 100644
index 0000000000..5e9fa1fd51
--- /dev/null
+++ b/doc/forum/Use_existing_S3_bucket/comment_2_6b108846fca4a514d507f56d1052fdde._comment
@@ -0,0 +1,13 @@
+[[!comment format=mdwn
+ username="http://xgm.de/oid/"
+ nickname="Horus"
+ avatar="http://cdn.libravatar.org/avatar/4c5c0e290374d76c713f482e41f60a3cbee0fa64bb94c6da94e5a61a50824811"
+ subject="comment 2"
+ date="2017-02-14T18:03:27Z"
+ content="""
+Ok, I can confirm it works with an existing bucket in Europe/Ireland.
+
+Looks like I shouldn't count on being able to use Frankfurt anytime soon, since this PR has not been worked on since april last year.
+
+Best Thanks!
+"""]]
diff --git a/doc/forum/Use_local_files_instead_of_re-downloading_from_S3_remote.mdwn b/doc/forum/Use_local_files_instead_of_re-downloading_from_S3_remote.mdwn
new file mode 100644
index 0000000000..67e7971d2c
--- /dev/null
+++ b/doc/forum/Use_local_files_instead_of_re-downloading_from_S3_remote.mdwn
@@ -0,0 +1,3 @@
+I have a question on setting up a new repo. Is it possible to initialize a new repo using files which are already local on say my laptop, instead of downloading them from a S3 remote repository? Sorry if this is unclear. But I'm trying to make use of the files that are local on my laptop so I don's have to go through the ordeal of downloading them from S3.
+
+I should also mention that I don't have a complete set of all the files locally on the laptop vs. what's already in the S3 repository.
diff --git a/doc/forum/Use_local_files_instead_of_re-downloading_from_S3_remote/comment_1_cfb6021a36eee087705967a69967f327._comment b/doc/forum/Use_local_files_instead_of_re-downloading_from_S3_remote/comment_1_cfb6021a36eee087705967a69967f327._comment
new file mode 100644
index 0000000000..0d1f74e833
--- /dev/null
+++ b/doc/forum/Use_local_files_instead_of_re-downloading_from_S3_remote/comment_1_cfb6021a36eee087705967a69967f327._comment
@@ -0,0 +1,10 @@
+[[!comment format=mdwn
+ username="https://www.google.com/accounts/o8/id?id=AItOawn7gQ1zZDdWhXy9H51W2krZYShNmKL3qfM"
+ nickname="Karsten"
+ subject="comment 1"
+ date="2013-05-27T18:13:43Z"
+ content="""
+Hi,
+
+just copy the local File into the annex again, replacing the symlinks. If you have the assistant running, it should re-add the files automatically, otherwise you can 'git annex add' them manually. 
+"""]]
diff --git a/doc/forum/Use_local_files_instead_of_re-downloading_from_S3_remote/comment_2_7268b194ba72331858bc3274996b780e._comment b/doc/forum/Use_local_files_instead_of_re-downloading_from_S3_remote/comment_2_7268b194ba72331858bc3274996b780e._comment
new file mode 100644
index 0000000000..b066c85db9
--- /dev/null
+++ b/doc/forum/Use_local_files_instead_of_re-downloading_from_S3_remote/comment_2_7268b194ba72331858bc3274996b780e._comment
@@ -0,0 +1,11 @@
+[[!comment format=mdwn
+ username="http://joeyh.name/"
+ nickname="joey"
+ subject="comment 2"
+ date="2013-05-27T18:47:17Z"
+ content="""
+You can also use `git annex reinject` to feed in contents of specific files, without needing to directly touch the files in the repository.
+
+Or, you can check all the local files into a temporary directory. Either the assistant or a manual `git annex add` will 
+notice if these files have the same content as files already in the repository. The files will then be available in two locations, the temp directory and wherever it was checked into your repository before. You can then delete the temp directory.
+"""]]
diff --git a/doc/forum/Use_password_protected_gpg_keypair_without_password_prompt__63__.mdwn b/doc/forum/Use_password_protected_gpg_keypair_without_password_prompt__63__.mdwn
new file mode 100644
index 0000000000..1ef6345bb9
--- /dev/null
+++ b/doc/forum/Use_password_protected_gpg_keypair_without_password_prompt__63__.mdwn
@@ -0,0 +1,7 @@
+Hi.
+
+I couldn't find a way to pass the passphrase, required to open the gpg keypair for an encrypted special remote, to "git annex sync --content".
+I need this for my backup script. Is it possible to use an environment variable or a passphrase file?
+
+Best regards,
+sfowijowa
diff --git a/doc/forum/Use_password_protected_gpg_keypair_without_password_prompt__63__/comment_1_b177f16daeab7f02022f18154cfa1c3c._comment b/doc/forum/Use_password_protected_gpg_keypair_without_password_prompt__63__/comment_1_b177f16daeab7f02022f18154cfa1c3c._comment
new file mode 100644
index 0000000000..b5dc5b4ebf
--- /dev/null
+++ b/doc/forum/Use_password_protected_gpg_keypair_without_password_prompt__63__/comment_1_b177f16daeab7f02022f18154cfa1c3c._comment
@@ -0,0 +1,14 @@
+[[!comment format=mdwn
+ username="joey"
+ subject="""comment 1"""
+ date="2015-07-15T15:55:31Z"
+ content="""
+Any method that can be used to provide a passphrase to gpg should work. 
+
+For example, gpg-agent can cache the passphrase.
+
+Or, you could use gpg's passphrase-file option to make it read the
+passphrase from a file. This could be put in ~/.gnupg/gpg.conf.
+Although this configuration is probably no more secure than just removing
+the passphrase from the gpg key..
+"""]]
diff --git a/doc/forum/Use_password_protected_gpg_keypair_without_password_prompt__63__/comment_2_6d88e5a48de78e6add0e4e4b6ebc3f1b._comment b/doc/forum/Use_password_protected_gpg_keypair_without_password_prompt__63__/comment_2_6d88e5a48de78e6add0e4e4b6ebc3f1b._comment
new file mode 100644
index 0000000000..99b64002ff
--- /dev/null
+++ b/doc/forum/Use_password_protected_gpg_keypair_without_password_prompt__63__/comment_2_6d88e5a48de78e6add0e4e4b6ebc3f1b._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="sfowijowa"
+ subject="comment 2"
+ date="2015-07-17T18:20:06Z"
+ content="""
+Hi, I am using the \"--passphrase-file\" option outside git-annex to sign the files that have to be backed up:
+\"gpg --batch --passphrase-file /home/user/.gnupg/pass.txt --no-tty --yes --detach-sign /tmp/root.tar.gz\" Unfortunately, I couldn't find a way to pass this option to gpg with git-annex. In duplicity, for example, you can pass gpg options with \"dupclitiy --gpg-options '--passphrase-file /home/user/.gnupg/pass.txt'\" Best regards
+"""]]
diff --git a/doc/forum/Use_password_protected_gpg_keypair_without_password_prompt__63__/comment_3_a2eb4244788eebba5167cd32a73ff204._comment b/doc/forum/Use_password_protected_gpg_keypair_without_password_prompt__63__/comment_3_a2eb4244788eebba5167cd32a73ff204._comment
new file mode 100644
index 0000000000..96a14aaba8
--- /dev/null
+++ b/doc/forum/Use_password_protected_gpg_keypair_without_password_prompt__63__/comment_3_a2eb4244788eebba5167cd32a73ff204._comment
@@ -0,0 +1,11 @@
+[[!comment format=mdwn
+ username="joey"
+ subject="""comment 3"""
+ date="2015-07-20T16:30:10Z"
+ content="""
+I gave you one way in my previous comment; you can put any gpg option in
+~/.gnupg/gpg.conf, just leave off the leading dashes.
+
+Another way is the annex.gnupg-options git configuration setting,
+see git-annex's man page.
+"""]]
diff --git a/doc/forum/Use_reflinks_on_BTRFS_instead_of_symlinks___63__.mdwn b/doc/forum/Use_reflinks_on_BTRFS_instead_of_symlinks___63__.mdwn
new file mode 100644
index 0000000000..d33fda6008
--- /dev/null
+++ b/doc/forum/Use_reflinks_on_BTRFS_instead_of_symlinks___63__.mdwn
@@ -0,0 +1 @@
+On BTRFS it is possible to use [reflinks / clones](https://en.wikipedia.org/wiki/Btrfs#Cloning). This is the equivalent of hardlinks except that when the file is modified, a copy is performed (copy on write). Perhaps git-annex could detect the availability of this feature and use it in case it is available (and the repository and the file are in the same filesystem).
diff --git a/doc/forum/Use_reflinks_on_BTRFS_instead_of_symlinks___63__/comment_1_85806316ed28d7a891f04fab4027141b._comment b/doc/forum/Use_reflinks_on_BTRFS_instead_of_symlinks___63__/comment_1_85806316ed28d7a891f04fab4027141b._comment
new file mode 100644
index 0000000000..b175204e55
--- /dev/null
+++ b/doc/forum/Use_reflinks_on_BTRFS_instead_of_symlinks___63__/comment_1_85806316ed28d7a891f04fab4027141b._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="http://thkoch2001.myopenid.com/"
+ nickname="thkoch"
+ subject="It would also be nice, if git-annex would use btrfs reflinks on unlock"
+ date="2013-08-24T12:46:40Z"
+ content="""
+I don't know whether git-annex already does use btrfs reflinks, but I suspect, that I double the size of my git-annex repo folder on disk when I unlink all files.
+"""]]
diff --git a/doc/forum/Use_reflinks_on_BTRFS_instead_of_symlinks___63__/comment_2_ecb411a2c4d67917b734a90bd460d44b._comment b/doc/forum/Use_reflinks_on_BTRFS_instead_of_symlinks___63__/comment_2_ecb411a2c4d67917b734a90bd460d44b._comment
new file mode 100644
index 0000000000..a4ade57926
--- /dev/null
+++ b/doc/forum/Use_reflinks_on_BTRFS_instead_of_symlinks___63__/comment_2_ecb411a2c4d67917b734a90bd460d44b._comment
@@ -0,0 +1,10 @@
+[[!comment format=mdwn
+ username="http://joeyh.name/"
+ ip="4.154.0.63"
+ subject="comment 2"
+ date="2013-08-24T15:54:02Z"
+ content="""
+git-annex has always used cp --reflink=auto, including when it cp's the file in `git annex unlock`.
+
+You can verify this with --debug.
+"""]]
diff --git a/doc/forum/Use_reflinks_on_BTRFS_instead_of_symlinks___63__/comment_3_83dc719be1883d5369d27a787a5b5705._comment b/doc/forum/Use_reflinks_on_BTRFS_instead_of_symlinks___63__/comment_3_83dc719be1883d5369d27a787a5b5705._comment
new file mode 100644
index 0000000000..f5c132449d
--- /dev/null
+++ b/doc/forum/Use_reflinks_on_BTRFS_instead_of_symlinks___63__/comment_3_83dc719be1883d5369d27a787a5b5705._comment
@@ -0,0 +1,14 @@
+[[!comment format=mdwn
+ username="sylvain.joyeux@5c33d7abae4df5f574fc4b8791d02054d245bf62"
+ nickname="sylvain.joyeux"
+ subject="not on the directory special remote"
+ date="2016-01-20T11:39:35Z"
+ content="""
+I'm trying to share videos across my (many) presentations using annex. My idea was to have a local annex directory remote that is used by all the presentation gits.
+
+However, it seems that copying from a directory special remote does not use reflink. Is that right ?
+
+Thanks for git-annex !
+
+Sylvain
+"""]]
diff --git a/doc/forum/Use_reflinks_on_BTRFS_instead_of_symlinks___63__/comment_4_bbe2e6359f7b754bde9d61b5a8d5e672._comment b/doc/forum/Use_reflinks_on_BTRFS_instead_of_symlinks___63__/comment_4_bbe2e6359f7b754bde9d61b5a8d5e672._comment
new file mode 100644
index 0000000000..a269048833
--- /dev/null
+++ b/doc/forum/Use_reflinks_on_BTRFS_instead_of_symlinks___63__/comment_4_bbe2e6359f7b754bde9d61b5a8d5e672._comment
@@ -0,0 +1,18 @@
+[[!comment format=mdwn
+ username="joey"
+ subject="""comment 4"""
+ date="2016-01-20T18:43:13Z"
+ content="""
+It's true; the directory special remote copies the files in and out. The
+main reason this is done is to get a useful progress display, which using
+`cp` wouldn't provide. Of course, if cp were using reflinks, it'd be so
+fast that a progress display would not be necessary! But often, reflinks
+cannot be used.
+
+It might be a good enhancement for the directory special remote to be
+configurable to use cp --reflink, or even autodetect if reflinks can be
+used (although I don't know how it would detect that).
+
+OTOH, you might be better off using `git clone --shared`. Recent versions
+of git-annex can hard-link objects between shared clones.
+"""]]
diff --git a/doc/forum/Usecase__58___Tree_of_files_on_a_remote_SMB_server_that_i_need_to_leave_there__44____while_also_cloning_and_syncing_too.mdwn b/doc/forum/Usecase__58___Tree_of_files_on_a_remote_SMB_server_that_i_need_to_leave_there__44____while_also_cloning_and_syncing_too.mdwn
new file mode 100644
index 0000000000..37dd1bc24c
--- /dev/null
+++ b/doc/forum/Usecase__58___Tree_of_files_on_a_remote_SMB_server_that_i_need_to_leave_there__44____while_also_cloning_and_syncing_too.mdwn
@@ -0,0 +1,11 @@
+I use git everywhere i can to manage versions, but I have a usecase that is probably very common, but i could not see in the howto lists.
+
+I have a shared fileserver(FS) that is managed by others and is full of the equivalent of binary blobs (media, word docs etc). Binary blobs of course are something that git cannot normally do anything meaningful with. That is diff/patch for managing the evolution of any given file. Binary blobs have a different lifecycle of course, and i see annex as the solution for managing that.
+
+The remote FS is being used as an SMB server to windows and linux clients. The FS has other capabilities that i can use other than SMB mount including rsync and ssh. As it is running samba it also uses symlinks underneath the the SMB to manage shared folders, which can be seen when mounted in unix mode. This means the FS landscape is different when using samba as compared to rsync and ssh, as some paths may be symlinked in. The symlinks should not effect this use case, but I mention them because I don't know how annex would manage this.
+
+The basic scenario is that i have a set of project files and folders in tree on the remote FS. I want to use git to pull these binary files from the FS to edit locally while also seein/capturing changes made to files in that tree by other people not using git. Then i want to push my edits back, without effecting the files that i have no responsibility for (but have write access to). 
+
+Think a team of people with various technical capabilities working on changing binary blobs in this remote directory tree. I want to manage my interactions with portions of that remote FS tree via git (annex) with a local mirror (preferably actually containing only the files i care about) so that i can manipulate them locally, clone to other work machines and generally play about with and then push back while watching the whole tree and capturing changes others make (during sync/merge/push/pull?). Because i do not manage it, or own the contents, i also need to be able to maintain the some of the permissions (guid/uid) of the remote and untouched files. 
+
+Can you give me any suggestions?
diff --git a/doc/forum/Usecase__58___Tree_of_files_on_a_remote_SMB_server_that_i_need_to_leave_there__44____while_also_cloning_and_syncing_too/comment_1_30205c1ba18e5dca2314f593e1a0e236._comment b/doc/forum/Usecase__58___Tree_of_files_on_a_remote_SMB_server_that_i_need_to_leave_there__44____while_also_cloning_and_syncing_too/comment_1_30205c1ba18e5dca2314f593e1a0e236._comment
new file mode 100644
index 0000000000..7f170975a6
--- /dev/null
+++ b/doc/forum/Usecase__58___Tree_of_files_on_a_remote_SMB_server_that_i_need_to_leave_there__44____while_also_cloning_and_syncing_too/comment_1_30205c1ba18e5dca2314f593e1a0e236._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="https://www.google.com/accounts/o8/id?id=AItOawkI9pq1WH6MWeExXHVQVEsniT3DdFv4AB8"
+ nickname="Roberto"
+ subject="My usecase too"
+ date="2014-01-12T23:43:18Z"
+ content="""
+Just wanted to thank your for describing my very same use case. Unfortunately I still haven't wrapped my head around git annex to accommodate such scenario.
+"""]]
diff --git a/doc/forum/Usecase__58___Tree_of_files_on_a_remote_SMB_server_that_i_need_to_leave_there__44____while_also_cloning_and_syncing_too/comment_2_f8df728de28218a6b060ae9f08adac79._comment b/doc/forum/Usecase__58___Tree_of_files_on_a_remote_SMB_server_that_i_need_to_leave_there__44____while_also_cloning_and_syncing_too/comment_2_f8df728de28218a6b060ae9f08adac79._comment
new file mode 100644
index 0000000000..576007b62b
--- /dev/null
+++ b/doc/forum/Usecase__58___Tree_of_files_on_a_remote_SMB_server_that_i_need_to_leave_there__44____while_also_cloning_and_syncing_too/comment_2_f8df728de28218a6b060ae9f08adac79._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="https://launchpad.net/~timo-linux"
+ nickname="Tim O'Callaghan"
+ subject="Would GIT_DIR or GIT_WORK_TREE be valid for this?"
+ date="2014-01-14T07:29:31Z"
+ content="""
+For example if i mounted this filesystem and used annex and GIT_ env variables to watch it? Presumably in direct mode, because i don't want to remove the source files, but i do want to manage my source files. 
+"""]]
diff --git a/doc/forum/Usecase__58___Tree_of_files_on_a_remote_SMB_server_that_i_need_to_leave_there__44____while_also_cloning_and_syncing_too/comment_3_ba438b3a371261ee841665f1ae57eba2._comment b/doc/forum/Usecase__58___Tree_of_files_on_a_remote_SMB_server_that_i_need_to_leave_there__44____while_also_cloning_and_syncing_too/comment_3_ba438b3a371261ee841665f1ae57eba2._comment
new file mode 100644
index 0000000000..9184495b51
--- /dev/null
+++ b/doc/forum/Usecase__58___Tree_of_files_on_a_remote_SMB_server_that_i_need_to_leave_there__44____while_also_cloning_and_syncing_too/comment_3_ba438b3a371261ee841665f1ae57eba2._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="https://launchpad.net/~timo-linux"
+ nickname="Tim O'Callaghan"
+ subject="What about annex mirror? "
+ date="2014-01-14T08:00:42Z"
+ content="""
+I cannot find any documentation on it, is it appropriate for this situation? 
+"""]]
diff --git a/doc/forum/Usecase__58___Tree_of_files_on_a_remote_SMB_server_that_i_need_to_leave_there__44____while_also_cloning_and_syncing_too/comment_4_9f72e6c7c14a77330297526aef260762._comment b/doc/forum/Usecase__58___Tree_of_files_on_a_remote_SMB_server_that_i_need_to_leave_there__44____while_also_cloning_and_syncing_too/comment_4_9f72e6c7c14a77330297526aef260762._comment
new file mode 100644
index 0000000000..d43019ce04
--- /dev/null
+++ b/doc/forum/Usecase__58___Tree_of_files_on_a_remote_SMB_server_that_i_need_to_leave_there__44____while_also_cloning_and_syncing_too/comment_4_9f72e6c7c14a77330297526aef260762._comment
@@ -0,0 +1,11 @@
+[[!comment format=mdwn
+ username="http://cstork.org/"
+ nickname="Chris Stork"
+ subject="Many peoples' use case"
+ date="2014-01-14T09:39:05Z"
+ content="""
+Well, a description of this use case is posted every few weeks in different forms. Here are two, including my own: ;-)
+
+* [[Using git annex with a SMB  47  FTP TV NAS with preconfigured dirs]]
+* [[Direct special remotes]]
+"""]]
diff --git a/doc/forum/Usecase__58___Tree_of_files_on_a_remote_SMB_server_that_i_need_to_leave_there__44____while_also_cloning_and_syncing_too/comment_5_a06e8c9b4e30c1cd6cbed40d2db50abc._comment b/doc/forum/Usecase__58___Tree_of_files_on_a_remote_SMB_server_that_i_need_to_leave_there__44____while_also_cloning_and_syncing_too/comment_5_a06e8c9b4e30c1cd6cbed40d2db50abc._comment
new file mode 100644
index 0000000000..321fa9c53f
--- /dev/null
+++ b/doc/forum/Usecase__58___Tree_of_files_on_a_remote_SMB_server_that_i_need_to_leave_there__44____while_also_cloning_and_syncing_too/comment_5_a06e8c9b4e30c1cd6cbed40d2db50abc._comment
@@ -0,0 +1,10 @@
+[[!comment format=mdwn
+ username="https://launchpad.net/~timo-linux"
+ nickname="Tim O'Callaghan"
+ subject="Still have not found a reasonable solution for this myself."
+ date="2014-02-03T14:34:14Z"
+ content="""
+And it seems i'm not the only one, So I'd like to bump the topic. 
+
+Can someone at least tell me if it is possible with annex/annex-gui or not? 
+"""]]
diff --git a/doc/forum/User_name_with_whitespace_not_working_with___34____40__+__41___Local_computer__34__.mdwn b/doc/forum/User_name_with_whitespace_not_working_with___34____40__+__41___Local_computer__34__.mdwn
new file mode 100644
index 0000000000..45880d27e0
--- /dev/null
+++ b/doc/forum/User_name_with_whitespace_not_working_with___34____40__+__41___Local_computer__34__.mdwn
@@ -0,0 +1,7 @@
+When using the "(+) Local computer" feature, I get the error message
+
+Internal Server Error: wrong number of words in ssh public key
+
+This is due to the use of pattern matching in this line https://github.com/RichiH/git-annex/blob/master/Assistant/Ssh.hs#L73 and because the key comment contains spaces. And of course it cotains spaces, on windows people often use their full name as user name. I will fill a bug later.
+
+My question is, how can I work around this bug? The client with this public key is running windows. How can I influence or at least see how this public key looks like? I could not find any documentation on this.
diff --git a/doc/forum/User_name_with_whitespace_not_working_with___34____40__+__41___Local_computer__34__/comment_1_f803fc1fb340587fc4b55640fbae6604._comment b/doc/forum/User_name_with_whitespace_not_working_with___34____40__+__41___Local_computer__34__/comment_1_f803fc1fb340587fc4b55640fbae6604._comment
new file mode 100644
index 0000000000..e0f28a20d8
--- /dev/null
+++ b/doc/forum/User_name_with_whitespace_not_working_with___34____40__+__41___Local_computer__34__/comment_1_f803fc1fb340587fc4b55640fbae6604._comment
@@ -0,0 +1,9 @@
+[[!comment format=mdwn
+ username="joey"
+ subject="""comment 1"""
+ date="2015-02-04T18:04:51Z"
+ content="""
+Please report bugs at [[bugs]], not in the forum.
+
+I've fixed this one.
+"""]]
diff --git a/doc/forum/Using_Git-Annex___40__Assistant__41___to_manage_photos_with_Shotwell.mdwn b/doc/forum/Using_Git-Annex___40__Assistant__41___to_manage_photos_with_Shotwell.mdwn
new file mode 100644
index 0000000000..d3e2cef009
--- /dev/null
+++ b/doc/forum/Using_Git-Annex___40__Assistant__41___to_manage_photos_with_Shotwell.mdwn
@@ -0,0 +1,13 @@
+Hello,
+
+I'm very excited about Git Annex and Git Annex Assistant.
+
+One big issue that I have is that I currently have all of my photos (70-80GB) on a HTPC because it has a large hard drive. I have a decent laptop/screen set-up where I would prefer to manage the photos, but the hard drive in there is a small SSD. 
+
+I use Shotwell to manage my photos (currently on the HTPC). I would be happy to keep things simple and only access them on the laptop. I have 100GB of box.com storage (and another 100GB of Mega.co.nz storage, but I don't think that is supported by GAA yet).
+
+Ideally, what I would like is for a cache of, say, 10GB of my recently-accessed photos to live on my laptop, with the remaining 70GB living on both my HTPC and box.com. If I ever needed the underlying files for non-cached files (say if I was trying to view full-quality versions of them or send them to someone), it could try to pull these from my HTPC over the local network or, if I was "on the road", box.com. As I understand it, Shotwell has its own database and a set of thumbnails etc that you can use to manage and browse your photos while very rarely accessing the underlying photos.
+
+Is this possible with the existing GAA/Shotwell, or does one or the other require changes? Could I solve the problem by using something like the sharebox Git-Annex FUSE filesystem, so that Shotwell thinks that the files are local, but runs away and grabs them if they are remote?
+
+I would appreciate any thoughts that anyone has - particularly if anyone has made this work. 
diff --git a/doc/forum/Using_Git-Annex___40__Assistant__41___to_manage_photos_with_Shotwell/comment_1_5e8d54daf6b7ff357619ac65fe39a2d7._comment b/doc/forum/Using_Git-Annex___40__Assistant__41___to_manage_photos_with_Shotwell/comment_1_5e8d54daf6b7ff357619ac65fe39a2d7._comment
new file mode 100644
index 0000000000..7ec8ac2fc0
--- /dev/null
+++ b/doc/forum/Using_Git-Annex___40__Assistant__41___to_manage_photos_with_Shotwell/comment_1_5e8d54daf6b7ff357619ac65fe39a2d7._comment
@@ -0,0 +1,12 @@
+[[!comment format=mdwn
+ username="http://joeyh.name/"
+ nickname="joey"
+ subject="comment 1"
+ date="2013-05-27T19:09:50Z"
+ content="""
+Mega can be used via [[/tips/megaannex]]. I don't have personal experience with it, but if you set up a repository manually using that, the assistant can use that repository just as it uses any other repository.
+
+git-annex at the command line is great for small local repositories that pull files from various larger remotes as needed. You just run \"git annex get\" when you want a file and \"git annex drop\" when you want to free disk space. You can also use this mode with the assistant, by configuring the local repository to be in \"manual mode\".
+
+By default though, the way the assistant handles this kind of use case is with `archive` directories. It tries to move any files in an archive directory away from your local disk, and it tries to get any files not in an archive directory to be locally available. So you can just move files around between directories to control where they are stored. See [[assistant/archival_walkthrough]] for details an an example video.
+"""]]
diff --git a/doc/forum/Using_Linux_static_builds.mdwn b/doc/forum/Using_Linux_static_builds.mdwn
new file mode 100644
index 0000000000..ee856f72c1
--- /dev/null
+++ b/doc/forum/Using_Linux_static_builds.mdwn
@@ -0,0 +1,25 @@
+Hello,
+
+I want to use git-annex on a couple of machines that are not under my control (and one that is). For that I use the provided static build, but encountered several problems:
+
+I suppose the first step for command line use is to execute runshell:
+
+On one machine:
+
+flindner@yoko:~> software/git-annex.linux/runshell 
+/bin/bash: /lib64/libc.so.6: version `GLIBC_2.11' not found (required by /home/flindner/software/git-annex.linux//lib/x86_64-linux-gnu/libreadline.so.6)
+
+Is there anything I can do about it?
+
+On the other machine:
+
+flindner@bladefoam2:~$ software/git-annex.linux/runshell 
+bash-4.1$ 
+
+It starts a bare, non-configured shell instance. Can I modifiy and source the script to only set the environment and not start up a new shell?
+
+Another question: If using ssh:// remotes, how can I tell git-annex the first command to set up the environment need in order to execute command via SSH?
+
+Thanks,
+
+Florian
diff --git a/doc/forum/Using_Linux_static_builds/comment_1_22fd266cbe68af3e754a10f1f1295e9b._comment b/doc/forum/Using_Linux_static_builds/comment_1_22fd266cbe68af3e754a10f1f1295e9b._comment
new file mode 100644
index 0000000000..b40ae76952
--- /dev/null
+++ b/doc/forum/Using_Linux_static_builds/comment_1_22fd266cbe68af3e754a10f1f1295e9b._comment
@@ -0,0 +1,13 @@
+[[!comment format=mdwn
+ username="http://joeyh.name/"
+ ip="4.154.0.149"
+ subject="comment 1"
+ date="2012-10-09T15:10:45Z"
+ content="""
+Yes, it seems glibc 2.11 is the oldest version you can get away with. At least until we get a build machine with some truely ancient libc, but modern versions of everything else. Statically linking libc, or including it in the tarball are unfortunatly not technically possible.
+
+You don't need to use runshell if you just want to use the assistant. Which is what this static build is really targeted for. If you want to use all of git-annex, you're probably better off installing it properly.
+It is possible to use \"runshell git annex\" to run a single command inside the shell. Or even make shell script wrappers that do that.. but it seems more difficult than installing git-annex normally.
+
+The static build can be used with ssh. The git-annex assistant should set things up properly. If you want to do it by hand, when it does is configure the ssh key to run ~/.ssh/git-annex-shell, and that file is then made a shell script that runs `runshell git-annex-shell -c \"$SSH_ORIGINAL_COMMAND\"`
+"""]]
diff --git a/doc/forum/Using_Linux_static_builds/comment_2_36f69f30117ff8696425a754ab19a08b._comment b/doc/forum/Using_Linux_static_builds/comment_2_36f69f30117ff8696425a754ab19a08b._comment
new file mode 100644
index 0000000000..4f43034a32
--- /dev/null
+++ b/doc/forum/Using_Linux_static_builds/comment_2_36f69f30117ff8696425a754ab19a08b._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="https://www.google.com/accounts/o8/id?id=AItOawnsuhFUIfWNT-Q-C02FDaSQqceFDge5M9w"
+ nickname="Florian"
+ subject="comment 2"
+ date="2012-10-10T10:48:04Z"
+ content="""
+Installing it properly seems to be quite a bit of effort, given the long list of Haskell dependencies. Most of the machines I plan to use it on are not under my control and seem to be somewhat outdated. What way of installing it properly would you suggest?
+"""]]
diff --git a/doc/forum/Using_Linux_static_builds/comment_3_64506833dad0202626239e00d1eb6490._comment b/doc/forum/Using_Linux_static_builds/comment_3_64506833dad0202626239e00d1eb6490._comment
new file mode 100644
index 0000000000..af301bd7e4
--- /dev/null
+++ b/doc/forum/Using_Linux_static_builds/comment_3_64506833dad0202626239e00d1eb6490._comment
@@ -0,0 +1,23 @@
+[[!comment format=mdwn
+ username="http://colberg.org/"
+ ip="2001:470:1d:4d4:2d4b:7654:2c00:b95c"
+ subject="comment 3"
+ date="2012-11-12T05:05:39Z"
+ content="""
+Hi Florian,
+
+I maintain a makefile for GNU/Linux systems that builds git-annex from scratch using a single command.
+
+    git clone http://git.colberg.org/packages.git
+
+    nice make -f packages.mk CONCURRENCY_LEVEL=16 PREFIX=$HOME/usr/rhel6-x64_64 install-git-annex
+
+The makefile fetches pre-compiled GHC 6.x (x86_64), which works on old systems (e.g. RHEL 5), builds a recent GHC 7.x from source, a minimal Haskell environment with cabal-install, and finally git-annex. I have succesfully compiled git-annex on RHEL 5.x and 6.x, and CentOS 6.x.
+
+If the build fails using the system's GCC, there is also a newer GCC in the same makefile:
+
+    nice make -f packages.mk CONCURRENCY_LEVEL=16 PREFIX=$HOME/usr/rhel6-x64_64 install-gcc
+
+Regards,
+Peter
+"""]]
diff --git a/doc/forum/Using___34__sync__34___to_sink_all_branches__63__.mdwn b/doc/forum/Using___34__sync__34___to_sink_all_branches__63__.mdwn
new file mode 100644
index 0000000000..99ddcc3281
--- /dev/null
+++ b/doc/forum/Using___34__sync__34___to_sink_all_branches__63__.mdwn
@@ -0,0 +1,9 @@
+Is there a way to sync all (or a subset of) local branches, not just the currently checked out branch?
+
+Does this make sense at all, or does this show I am missing some important point in git-annex?
+
+I am asking because I would like to use git-annex to keep git repositories with normal git files (with versioned and branched content) in sync.
+
+If it's not currently possible, could you provide some pointers on where to start, if I wanted to change to Haskell source?
+
+Thnx
diff --git a/doc/forum/Using___34__sync__34___to_sink_all_branches__63__/comment_1_ef3d5c5e2600ffa36dd933c8a42cdf96._comment b/doc/forum/Using___34__sync__34___to_sink_all_branches__63__/comment_1_ef3d5c5e2600ffa36dd933c8a42cdf96._comment
new file mode 100644
index 0000000000..2b060197ba
--- /dev/null
+++ b/doc/forum/Using___34__sync__34___to_sink_all_branches__63__/comment_1_ef3d5c5e2600ffa36dd933c8a42cdf96._comment
@@ -0,0 +1,16 @@
+[[!comment format=mdwn
+ username="https://www.google.com/accounts/o8/id?id=AItOawlgyVag95OnpvSzQofjyX0WjW__MOMKsl0"
+ nickname="Sehr"
+ subject="comment 1"
+ date="2012-10-29T17:04:30Z"
+ content="""
+Ok, this is what I have come up with:
+
+Merges can only be done on checked out branches. That's why the only way to do this without breaking the whole idea of git-annex (sync) is to check out each branch and run it separately. 
+
+What would be the easiest way to do this? Considering possible unsaved changes in worktree and index?
+
+Maybe simply using git-clone /tmp locally, which uses hardlinks for .git/objects/ which should make it perform ok?
+
+
+"""]]
diff --git a/doc/forum/Using___34__sync__34___to_sink_all_branches__63__/comment_2_424b0c6fdfe87ca08f5d408b7684ab08._comment b/doc/forum/Using___34__sync__34___to_sink_all_branches__63__/comment_2_424b0c6fdfe87ca08f5d408b7684ab08._comment
new file mode 100644
index 0000000000..1c4b9445b6
--- /dev/null
+++ b/doc/forum/Using___34__sync__34___to_sink_all_branches__63__/comment_2_424b0c6fdfe87ca08f5d408b7684ab08._comment
@@ -0,0 +1,10 @@
+[[!comment format=mdwn
+ username="https://www.google.com/accounts/o8/id?id=AItOawlgyVag95OnpvSzQofjyX0WjW__MOMKsl0"
+ nickname="Sehr"
+ subject="comment 2"
+ date="2012-10-29T18:52:59Z"
+ content="""
+It turns out, this is even easier: 
+
+As the behavior of \"git-annex sync\" includes performing a \"git commit -a\", any pending changes in the work tree will be committed anyway. This means, that it is safe to simply checkout and sync any branch, one wants to sync.
+"""]]
diff --git a/doc/forum/Using___34__sync__34___to_sink_all_branches__63__/comment_3_adaf9114c69f1268330adcebd8018fa0._comment b/doc/forum/Using___34__sync__34___to_sink_all_branches__63__/comment_3_adaf9114c69f1268330adcebd8018fa0._comment
new file mode 100644
index 0000000000..146785ad3b
--- /dev/null
+++ b/doc/forum/Using___34__sync__34___to_sink_all_branches__63__/comment_3_adaf9114c69f1268330adcebd8018fa0._comment
@@ -0,0 +1,10 @@
+[[!comment format=mdwn
+ username="http://joeyh.name/"
+ ip="4.152.108.194"
+ subject="comment 3"
+ date="2012-11-04T20:08:02Z"
+ content="""
+Yes, that's right, `git annex sync` handles the current branch, and you can use it on different branches if desired.
+
+(It is actually possible to merge non-checked out branches, but I have not implemented that for sync.)
+"""]]
diff --git a/doc/forum/Using_a_glacier_remote_as_a_backup.mdwn b/doc/forum/Using_a_glacier_remote_as_a_backup.mdwn
new file mode 100644
index 0000000000..2baaa7f914
--- /dev/null
+++ b/doc/forum/Using_a_glacier_remote_as_a_backup.mdwn
@@ -0,0 +1,28 @@
+Hallo everybody,
+
+I have an ARM based Synology NAS and I would like to use git annex to replace the "backup" solution provided by Synology. The basic idea is that I want files in a safe place when the house burns down or they get removed by accident.
+
+Since I only care about the latest version and want to make really sure local programs (cifs service, photostation and so on) do not run into trouble caused by symlinks, I guess direct mode is what I want. I have been tinkering around and things seem to be working for the most part. A few questions remain:
+
+Assuming I have all files synced to glacier and then accidentally remove all content and try to recover with the bare repo - with metadata but without content. the situation looks like this. 
+
+    ➜  syno-archive git:(annex/direct/master) git annex status test.txt
+    D test.txt
+    ➜  syno-archive git:(annex/direct/master)
+    
+I try to get my files back out of glacier:
+
+    ➜  syno-archive git:(annex/direct/master) git annex get test.txt   
+    get test.txt (from glacier...) 
+    ok                      
+    (Recording state in git...)
+    ➜  syno-archive git:(annex/direct/master)
+
+Contrary to my expectation, text.txt did not appear on disk.
+
+Given the bare repo, you would one recover all content (thousands of files)? I expected "git annex get --all" to do the trick. 
+
+PS: This is from git-annex version 5.20141125
+
+regards
+Andreas
diff --git a/doc/forum/Using_a_glacier_remote_as_a_backup/comment_1_bb6022943ec739b1e80351c76f259e89._comment b/doc/forum/Using_a_glacier_remote_as_a_backup/comment_1_bb6022943ec739b1e80351c76f259e89._comment
new file mode 100644
index 0000000000..c6899e4222
--- /dev/null
+++ b/doc/forum/Using_a_glacier_remote_as_a_backup/comment_1_bb6022943ec739b1e80351c76f259e89._comment
@@ -0,0 +1,17 @@
+[[!comment format=mdwn
+ username="joey"
+ subject="""comment 1"""
+ date="2015-05-11T16:58:01Z"
+ content="""
+Your status shows that test.txt is deleted.
+`git annex get` does not un-delete files, it just gets the *content*
+of a file (whether that file is deleted or not).
+
+You can use normal git commands to un-delete the file. Ie, "git checkout
+text.txt". If you're using direct mode, you can't use such commands, but
+can use "git annex undo" to undo a deletion.
+
+Normally, if you have a bare repo, you'll want to clone it to get a non-bare
+repo. I suspect you did something else that resulted in your repo being in
+this state were files are deleted.
+"""]]
diff --git a/doc/forum/Using_a_glacier_remote_as_a_backup/comment_2_41d47f0ca2e05e3296face5f89b819da._comment b/doc/forum/Using_a_glacier_remote_as_a_backup/comment_2_41d47f0ca2e05e3296face5f89b819da._comment
new file mode 100644
index 0000000000..66b08b2fee
--- /dev/null
+++ b/doc/forum/Using_a_glacier_remote_as_a_backup/comment_2_41d47f0ca2e05e3296face5f89b819da._comment
@@ -0,0 +1,15 @@
+[[!comment format=mdwn
+ username="https://www.google.com/accounts/o8/id?id=AItOawnwNDA50ZupMvOgpgDqzDRyu5B-mYlVwa4"
+ nickname="Andreas"
+ subject="comment 2"
+ date="2015-05-12T06:52:04Z"
+ content="""
+Actually, I just removed all annexed files to try out disaster recovery. I stumbled across the `undo` command. Unfortunately, it does not seem available on Ubuntu running version 5.20141125 shipped by vivid (`git-annex: Unknown command 'undo'`).
+
+Am I missing something, or how do I get a version supporting `undo`?
+
+thanks
+Andreas
+
+
+"""]]
diff --git a/doc/forum/Using_a_glacier_remote_as_a_backup/comment_3_e572b7bd25b4e7f6c6f83c7354d7407f._comment b/doc/forum/Using_a_glacier_remote_as_a_backup/comment_3_e572b7bd25b4e7f6c6f83c7354d7407f._comment
new file mode 100644
index 0000000000..ac8530baa6
--- /dev/null
+++ b/doc/forum/Using_a_glacier_remote_as_a_backup/comment_3_e572b7bd25b4e7f6c6f83c7354d7407f._comment
@@ -0,0 +1,7 @@
+[[!comment format=mdwn
+ username="joey"
+ subject="""comment 3"""
+ date="2015-07-06T20:01:45Z"
+ content="""
+git-annex undo first appeared in version 5.20141203 of git-annex.
+"""]]
diff --git a/doc/forum/Using_a_single_backup_repo_for_multiple_independent_client_repos__63__.mdwn b/doc/forum/Using_a_single_backup_repo_for_multiple_independent_client_repos__63__.mdwn
new file mode 100644
index 0000000000..985bff87c9
--- /dev/null
+++ b/doc/forum/Using_a_single_backup_repo_for_multiple_independent_client_repos__63__.mdwn
@@ -0,0 +1,36 @@
+I don't know if this is an "unusual setup" or something that should be supported, but it seems like a reasonable use case to me, so here goes:
+
+I have three systems: a desktop, a laptop, and a little netbook.  I have one main annex that is currently synced between the desktop and the laptop at ~/annex on both, and to a secondary, Full Backup repo on the desktop, under /mnt.
+
+I have some music on my laptop, and I like to sync ~/Music between the laptop and the netbook.  So I thought I'd make a new git-annex repo at ~/Music on both and sync them.  I want this to be kept separate from ~/annex.
+
+I also thought it would be good to backup the music, so I thought I'd connect laptop:~/Music to the Full Backup repo at desktop:/mnt.  So I paired the repo at laptop:~/Music to the repo at desktop:/mnt.
+
+So I had set up these repos:
+
+* desktop:
+  * /mnt  (Full Backup)
+  * ~/annex  (Client)
+* laptop:
+  * ~/annex  (Client)
+  * ~/Music  (Client)
+* netbook:
+  * ~/Music  (Client)
+
+As I feared--but I wasn't sure, since I'm new to git-annex--since the desktop:/mnt repo was connected to laptop:~/Music, and desktop:~/mnt was also connected to desktop:~/annex,  this caused everything in laptop:~/Music to be synced to desktop:~/annex, because they were part of the same "set" of repos; they were all in the same .git/config files.
+
+So now I have a few problems:
+
+1.  How do I get all these music files out of my ~/annex repos?  I tried removing the remote repos from ~/annex/.git/config and then running 'git-annex unused', but that didn't result in it thinking all the music files were unused.  Do I have to delete them all manually?  (There are a *lot* of directories from ~/Music now in ~/annex).
+
+2.  It seems that desktop:~/annex/.git is...now over 600 MB, even though few of the actual music files have synced over.  Is it possible to clean out ~/annex/.git and restore it to the pre-mess state without starting from scratch?
+
+3.  Is it possible--or reasonable--to use a single Full Backup repo for multiple independent client repos?  That is, is it possible to do that without the independent client repos "cross-pollinating"?  Or do I need to use separate Full Backup repos for each "set" of client repos?
+
+4.  I'm not sure if this is a separate problem, but I just want to make sure I understand the best way to handle things like this.  I wanted to disconnect desktop:~/annex from laptop:~/Music, but I thought that if I used the webapp to "Delete" the laptop:~/Music repo, it would actually start deleting the music files in laptop:~/Music, after moving them to desktop:~/annex.  Am I correct that that would have happened?  If so, "Deleting" a repo seems like a dangerous operation that should warn and prompt the user.  Maybe an "unlink repo" option needs to be available to simply disconnect two repos and leave their respective contents intact.
+
+I apologize if these are newbie problems.  Despite reading every dev blog entry--both before and after the switchover--I'm still new to actually using git-annex.  I'm not sure if it's supposed to work this way or if I'm using it wrong.
+
+If I am using it wrong...well, I can imagine other git-annex newbies doing similar things and ending up with similar messes, so maybe this needs to...be considered or something.  :)  Since I'm still experimenting with git-annex, I haven't put any "live" data in it, so I can blow away all the annexes and start over, but I would like to learn how to properly fix a situation like this in case I goof up in the future.
+
+Thanks for your help.
diff --git a/doc/forum/Using_a_single_backup_repo_for_multiple_independent_client_repos__63__/comment_1_c61c28600f1079fb03ddabc950307f27._comment b/doc/forum/Using_a_single_backup_repo_for_multiple_independent_client_repos__63__/comment_1_c61c28600f1079fb03ddabc950307f27._comment
new file mode 100644
index 0000000000..b7119aa2d0
--- /dev/null
+++ b/doc/forum/Using_a_single_backup_repo_for_multiple_independent_client_repos__63__/comment_1_c61c28600f1079fb03ddabc950307f27._comment
@@ -0,0 +1,11 @@
+[[!comment format=mdwn
+ username="http://joeyh.name/"
+ ip="209.250.56.47"
+ subject="comment 1"
+ date="2013-11-05T16:14:34Z"
+ content="""
+1. Turn off the assistant, switch to indirect mode (`git annex indirect`), use `git log --stat` to find the commit(s) that added the music files, and `git revert` the commits, in reverse chronological order. Or delete the files manually.
+2. You have merged two git repositories. Without some serious git repository surgery (ie, resetting to an old version and deleting the reflog), the full history of both is going to remain stored in git going forward. However, a simple `git gc` might pack that down to a much smaller amount of disk space.
+3. Combining two repositories merges their git histories. You then have two clones of one repository containing the full contents and git history of both. Don't do that if that's not what you want. The webapp is always careful to tell you when an action will combine two repositories.
+4. Once you have combined two repositories, you have two synced repositories that contain the same data. Deleting one of them will just delete one copy of the data. It won't result in data loss, but neither will it untangle the two git trees that were previously combined.
+"""]]
diff --git a/doc/forum/Using_for_Music_repo.mdwn b/doc/forum/Using_for_Music_repo.mdwn
new file mode 100644
index 0000000000..25f1365fb8
--- /dev/null
+++ b/doc/forum/Using_for_Music_repo.mdwn
@@ -0,0 +1,13 @@
+Hi, 
+
+I am looking into using git-annex to sync my Music between 3 sources: desktop, laptop, backup, but I'm not sure if this will work ok.
+
+I notice on the wiki that:  "Normally, the content of files in the annex is prevented from being modified. That's a good thing, because it might be the only copy, you wouldn't want to lose it in a fumblefingered mistake."
+
+But most music players will download artwork, and some of course will dynamically modify/rename files according to meta data.  And I may be downloading and/or playing music on both the desktop and laptop.  So obviously I wouldn't want to be manually unlocking individual files here.  It would kinda interrupt the music listening experience.  So how would this work?
+
+I am assuming that I would always have to do a git annex add *.mp3 *.flac *.jpg before doing a git annex sync.  But can I just keep everything unlocked?  Any side effects here?  Will this work or should I just use rsync?
+
+Regards,
+
+bk
diff --git a/doc/forum/Using_for_Music_repo/comment_1_3488ed85ad98f14cb17f229225ece26e._comment b/doc/forum/Using_for_Music_repo/comment_1_3488ed85ad98f14cb17f229225ece26e._comment
new file mode 100644
index 0000000000..6176f755dd
--- /dev/null
+++ b/doc/forum/Using_for_Music_repo/comment_1_3488ed85ad98f14cb17f229225ece26e._comment
@@ -0,0 +1,10 @@
+[[!comment format=mdwn
+ username="http://joeyh.name/"
+ nickname="joey"
+ subject="comment 1"
+ date="2013-04-18T20:01:50Z"
+ content="""
+You can use [[direct_mode]] to keep all files unlocked all the time.
+
+You can use the [[assistant]] to automatically commit whatever changes are made to the directory, as well as automatically sync them to your other computers.
+"""]]
diff --git a/doc/forum/Using_for_Music_repo/comment_2_c794648878cfc77558f8db862271f997._comment b/doc/forum/Using_for_Music_repo/comment_2_c794648878cfc77558f8db862271f997._comment
new file mode 100644
index 0000000000..9ce8743076
--- /dev/null
+++ b/doc/forum/Using_for_Music_repo/comment_2_c794648878cfc77558f8db862271f997._comment
@@ -0,0 +1,25 @@
+[[!comment format=mdwn
+ username="https://www.google.com/accounts/o8/id?id=AItOawmgXUaPZHcAY8Wv4sTnX88CbJfYdYdKO-Y"
+ nickname="Boyd"
+ subject="comment 2"
+ date="2013-04-18T23:33:10Z"
+ content="""
+Thanks Joey!  Anyway I see that with Fedora, I don't have the 'assistant'.  I don't even seem to have the 'direct mode'.  Also tried to do a cabal install git-annex, but ran into dependency hell.  Oh well, this is goin on the back burner for a while.  Looks very promising though!
+
+Boyd
+
+[root@zbox Music]# git annex direct
+git-annex: Unknown command 'direct'
+
+Did you mean one of these?
+	reinject
+
+[root@zbox Music]# git annex version
+git-annex version: 3.20120615
+local repository version: 3
+default repository version: 3
+supported repository versions: 3
+upgrade supported from repository versions: 0 1 2
+[root@zbox Music]# 
+
+"""]]
diff --git a/doc/forum/Using_for_Music_repo/comment_3_8c5e820f5ff7d717d64b1fd66927941b._comment b/doc/forum/Using_for_Music_repo/comment_3_8c5e820f5ff7d717d64b1fd66927941b._comment
new file mode 100644
index 0000000000..842a83052d
--- /dev/null
+++ b/doc/forum/Using_for_Music_repo/comment_3_8c5e820f5ff7d717d64b1fd66927941b._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="http://joeyh.name/"
+ nickname="joey"
+ subject="comment 3"
+ date="2013-04-19T18:05:49Z"
+ content="""
+Fedora has an old version of git-annex. You can use the [[install/Linux_standalone]] tarball.
+"""]]
diff --git a/doc/forum/Using_git-annex.mdwn b/doc/forum/Using_git-annex.mdwn
new file mode 100644
index 0000000000..0e2be4e81b
--- /dev/null
+++ b/doc/forum/Using_git-annex.mdwn
@@ -0,0 +1,73 @@
+Hi,
+
+I have created a git annex repo, added data. I then went to check it out in another location in the following way (my goal is to checkout origin, add a test file, push it back to origin).
+ git clone ../test_repo/
+
+   282  17:19   cd test_repo/
+
+   283  17:19   ls
+
+   284  17:19   git status
+
+   285  17:22   git annex init DEV
+
+   286  17:22   touch test.txt
+
+   287  17:22   vi test.txt
+
+   288  17:22   git annex merge
+
+   289  17:22   git annex add test.txt
+
+   290  17:22   git commit -am "test"
+
+   291  17:23   git push origin master git-annex 
+
+However I am getting the following error
+
+Counting objects: 3, done.
+
+Delta compression using up to 48 threads.
+
+Compressing objects: 100% (3/3), done.
+
+Writing objects: 100% (3/3), 364 bytes | 0 bytes/s, done.
+
+Total 3 (delta 1), reused 0 (delta 0)
+
+remote: error: refusing to update checked out branch: refs/heads/master
+
+remote: error: By default, updating the current branch in a non-bare repository
+
+remote: error: is denied, because it will make the index and work tree inconsistent
+
+remote: error: with what you pushed, and will require 'git reset --hard' to match
+
+remote: error: the work tree to HEAD.
+
+remote: error: 
+
+remote: error: You can set 'receive.denyCurrentBranch' configuration variable to
+
+remote: error: 'ignore' or 'warn' in the remote repository to allow pushing into
+
+remote: error: its current branch; however, this is not recommended unless you
+
+remote: error: arranged to update its work tree to match what you pushed in some
+
+remote: error: other way.
+
+remote: error: 
+
+remote: error: To squelch this message and still keep the default behaviour, set
+
+remote: error: 'receive.denyCurrentBranch' configuration variable to 'refuse'.
+
+To /test_repo/
+
+ ! [remote rejected] master -> master (branch is currently checked out)
+
+error: failed to push some refs to '/test_repo/'
+
+
+What am I missing?
diff --git a/doc/forum/Using_git-annex/comment_1_ac386ca07e9aaf009c264f3deed83832._comment b/doc/forum/Using_git-annex/comment_1_ac386ca07e9aaf009c264f3deed83832._comment
new file mode 100644
index 0000000000..9e0e8ff665
--- /dev/null
+++ b/doc/forum/Using_git-annex/comment_1_ac386ca07e9aaf009c264f3deed83832._comment
@@ -0,0 +1,16 @@
+[[!comment format=mdwn
+ username="joey"
+ subject="""comment 1"""
+ date="2014-10-31T20:40:18Z"
+ content="""
+git does not allow pushing into a non-bare repsitory. This is why
+bare repositories exist.
+
+You have basically 3 choices:
+
+1. Go to the first repo you made, and `git pull` from the other repo
+   into it. Pulling in this direction will work.
+2. Set up a bare repo that both the non-bare repos use.
+3. Use `git annex sync`, which avoids this problem. You need to run
+   it in both repos in order to get them to sync with one-another.
+"""]]
diff --git a/doc/forum/Using_git-annex/comment_2_bf5e13a490e16943acd2732f093695fc._comment b/doc/forum/Using_git-annex/comment_2_bf5e13a490e16943acd2732f093695fc._comment
new file mode 100644
index 0000000000..97ec4fb98e
--- /dev/null
+++ b/doc/forum/Using_git-annex/comment_2_bf5e13a490e16943acd2732f093695fc._comment
@@ -0,0 +1,16 @@
+[[!comment format=mdwn
+ username="https://www.google.com/accounts/o8/id?id=AItOawktbkKjilg70XC9XBFpIgVhtfLYH-0UMHY"
+ nickname="Tad"
+ subject="comment 2"
+ date="2014-11-06T15:39:12Z"
+ content="""
+Ok so when I create my origin and after I add files to it, when I run git annex sync I get
+
+Invalid command: 'git-annex-shell 'configlist' '/~/gitfile.git''
+  You appear to be using ssh to clone a git:// URL.
+  Make sure your core.gitProxy config option and the
+  GIT_PROXY_COMMAND environment variable are NOT set.
+warning: no common commits
+
+and then it hangs. Is git-annex not accepting git via ssh?
+"""]]
diff --git a/doc/forum/Using_git-annex/comment_3_471c9268b028f47ef3fac3f0be14b2b1._comment b/doc/forum/Using_git-annex/comment_3_471c9268b028f47ef3fac3f0be14b2b1._comment
new file mode 100644
index 0000000000..4dfa8916d0
--- /dev/null
+++ b/doc/forum/Using_git-annex/comment_3_471c9268b028f47ef3fac3f0be14b2b1._comment
@@ -0,0 +1,15 @@
+[[!comment format=mdwn
+ username="joey"
+ subject="""comment 3"""
+ date="2014-11-06T19:42:33Z"
+ content="""
+git-annex certianly works with git over ssh. You seem to not have git-annex
+installed on your git server, which will limit what you can do with it.
+
+I cannot find the message "You appear to be using ssh to clone a git:// URL." 
+in either git or git-annex's code base. Perhaps the message is coming from
+something like gitosis or gitolite on your git server?
+
+It's hard to give advice when you're not said anything about how you've set
+things up.
+"""]]
diff --git a/doc/forum/Using_git-annex_as_a_library.mdwn b/doc/forum/Using_git-annex_as_a_library.mdwn
new file mode 100644
index 0000000000..3727679dd0
--- /dev/null
+++ b/doc/forum/Using_git-annex_as_a_library.mdwn
@@ -0,0 +1 @@
+I am looking to use git-annex as a library/tool to bring syncing to another application. Is there any documentation or open examples of doing something like this?
diff --git a/doc/forum/Using_git-annex_as_a_library/comment_1_1f8e74c5856f21c53d5a91892cbef0c6._comment b/doc/forum/Using_git-annex_as_a_library/comment_1_1f8e74c5856f21c53d5a91892cbef0c6._comment
new file mode 100644
index 0000000000..6ba664aead
--- /dev/null
+++ b/doc/forum/Using_git-annex_as_a_library/comment_1_1f8e74c5856f21c53d5a91892cbef0c6._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="https://www.google.com/accounts/o8/id?id=AItOawlaQ9Hd5o_wjCIcfC9MAx8GRli5-LyBang"
+ nickname="George"
+ subject="comment 1"
+ date="2013-06-29T16:07:45Z"
+ content="""
+I should say specifically interfacing with git-annex
+"""]]
diff --git a/doc/forum/Using_git-annex_as_a_library/comment_2_11a243fa7d8ac947aa9a798228dbd191._comment b/doc/forum/Using_git-annex_as_a_library/comment_2_11a243fa7d8ac947aa9a798228dbd191._comment
new file mode 100644
index 0000000000..4177d7d208
--- /dev/null
+++ b/doc/forum/Using_git-annex_as_a_library/comment_2_11a243fa7d8ac947aa9a798228dbd191._comment
@@ -0,0 +1,12 @@
+[[!comment format=mdwn
+ username="http://joeyh.name/"
+ ip="4.154.4.193"
+ subject="comment 2"
+ date="2013-06-30T18:08:52Z"
+ content="""
+I think there are two approaches to doing this.
+
+1. Use the git-annex command line interface from your program. You can use the --json switch to enable machine-parsable output of many git-annex commands. If something needs work to be more suitable to be used as \"plumbing\" in this way, we can improve it to meet your needs.
+
+2. Use the git-annex Haskell code as a library for your program. The git-annex assistant is a great example of how far you can take this. It has the benefit that by accessing git-annex's internals, you can sometimes do things more efficiently than by using the CLI. Much of git-annex's code is already well modularized and suitable for use as a library in this way. The build system doesn't currently spit out git-annex libraries, but it would not be hard to make it do so. Of course this would entail writing at least some of your program in Haskell.
+"""]]
diff --git a/doc/forum/Using_git-annex_as_a_library/comment_3_ac52304a096ebc66967352efaffb060a._comment b/doc/forum/Using_git-annex_as_a_library/comment_3_ac52304a096ebc66967352efaffb060a._comment
new file mode 100644
index 0000000000..cda6e0799a
--- /dev/null
+++ b/doc/forum/Using_git-annex_as_a_library/comment_3_ac52304a096ebc66967352efaffb060a._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="Alexcei"
+ ip="92.255.239.77"
+ subject="comment 3"
+ date="2014-03-25T05:43:36Z"
+ content="""
+With version 5.20140127 first approach became impossible. Why did you decide to remove json supported by the majority of command?
+"""]]
diff --git a/doc/forum/Using_git-annex_as_a_library/comment_4_d502fea60bf3a82f8a50f72a90a80c25._comment b/doc/forum/Using_git-annex_as_a_library/comment_4_d502fea60bf3a82f8a50f72a90a80c25._comment
new file mode 100644
index 0000000000..642fda2fef
--- /dev/null
+++ b/doc/forum/Using_git-annex_as_a_library/comment_4_d502fea60bf3a82f8a50f72a90a80c25._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="http://joeyh.name/"
+ ip="209.250.56.41"
+ subject="comment 4"
+ date="2014-03-26T18:31:30Z"
+ content="""
+I didn't. I removed the --json options from commands that didn't necessarily output valid json when used with it. There are numerous commands that support --json (find, whereis, info, status, metadata, examinekey), and I'm open to adding it to other commands -- but it's probably not feasible to make commands like `get` output json, since their output includes progress info printed by rsync etc.
+"""]]
diff --git a/doc/forum/Using_git-annex_as_a_library/comment_5_a4ab4173620b72ac0a24d575fa9c810c._comment b/doc/forum/Using_git-annex_as_a_library/comment_5_a4ab4173620b72ac0a24d575fa9c810c._comment
new file mode 100644
index 0000000000..70f2ad9353
--- /dev/null
+++ b/doc/forum/Using_git-annex_as_a_library/comment_5_a4ab4173620b72ac0a24d575fa9c810c._comment
@@ -0,0 +1,13 @@
+[[!comment format=mdwn
+ username="https://www.google.com/accounts/o8/id?id=AItOawmkuFJVGp6WVvJtIV5JYb8IqN8mRvSGQdI"
+ nickname="Emilio Jesús"
+ subject="Would you accept a patch?"
+ date="2014-08-03T01:18:54Z"
+ content="""
+Dear Joey,
+
+I am also interested in using git-annex as a Haskell library, would you accept a patch to the .cabal file then?
+
+Thanks,
+Emilio
+"""]]
diff --git a/doc/forum/Using_git-annex_as_a_library/comment_6_45d9520ebc13d1b4fd88c25abc61f1b4._comment b/doc/forum/Using_git-annex_as_a_library/comment_6_45d9520ebc13d1b4fd88c25abc61f1b4._comment
new file mode 100644
index 0000000000..8591e79aa0
--- /dev/null
+++ b/doc/forum/Using_git-annex_as_a_library/comment_6_45d9520ebc13d1b4fd88c25abc61f1b4._comment
@@ -0,0 +1,11 @@
+[[!comment format=mdwn
+ username="http://joeyh.name/"
+ ip="209.250.56.7"
+ subject="comment 6"
+ date="2014-08-12T17:48:34Z"
+ content="""
+There are a couple of problems with using the haskell code as a library that would need to be addressed:
+
+* I can't guarantee much about providing every interface in a compatible way going forward. It might make sense to pick out some key interfaces and make those stable, but I don't know what the right choices would be.
+* If all of git-annex is a library, `cabal build` will build everything a second time. This is annoying when trying to do a fast edit/build/test cycle, but I don't know a way to make cabal not do it. AFAIK cabal build flags cannot be used to disable building a library.
+"""]]
diff --git a/doc/forum/Using_git-annex_to_manage_git_repositories.mdwn b/doc/forum/Using_git-annex_to_manage_git_repositories.mdwn
new file mode 100644
index 0000000000..a2733c0db7
--- /dev/null
+++ b/doc/forum/Using_git-annex_to_manage_git_repositories.mdwn
@@ -0,0 +1,5 @@
+I am interested in using git-annex to manage git repositories, and I am wondering if it is possible and if anyone has experience with it?
+
+I have done some searching, and I know that many people have asked for support for a Dropbox-like workflow, where Git repositories are mirrored everywhere.  I also know that no such support seems forthcoming, however this is not my goal.  Rather, I would like to use git-annex to track the location of many repositories.  I keep a lot of repositories and would like to offload them onto other storage devices and keep track of where each repository is stored.
+
+Perhaps entire Git repositories can be added as a single unit for tracking in git-annex?
diff --git a/doc/forum/Using_git-annex_to_manage_git_repositories/comment_1_f221717814f62e0779d691d2f601ab12._comment b/doc/forum/Using_git-annex_to_manage_git_repositories/comment_1_f221717814f62e0779d691d2f601ab12._comment
new file mode 100644
index 0000000000..593496b630
--- /dev/null
+++ b/doc/forum/Using_git-annex_to_manage_git_repositories/comment_1_f221717814f62e0779d691d2f601ab12._comment
@@ -0,0 +1,27 @@
+[[!comment format=mdwn
+ username="https://openid.stackexchange.com/user/e65e6d0e-58ba-41de-84cc-1f2ba54cf574"
+ nickname="Mica"
+ subject="Yes and no..."
+ date="2015-04-04T04:13:48Z"
+ content="""
+
+> I am interested in using git-annex to manage git repositories, and I am wondering if it is possible and if anyone has experience with it?
+
+That is not what git-annex is for. git-annex adds large file support to git.
+
+> I have done some searching, and I know that many people have asked for support for a Dropbox-like workflow, where Git repositories are mirrored everywhere. 
+
+The git-annex assistant will automatically sync your files for you. It is similar to dropbox but not entirely the same.
+
+> I also know that no such support seems forthcoming, however this is not my goal. 
+
+I don't know what would give you that idea, the git-annex assistant has already been released.
+
+> Rather, I would like to use git-annex to track the location of many repositories. I keep a lot of repositories and would like to offload them onto other storage devices and keep track of where each repository is stored.
+
+git-annex is not the right tool for that. Maybe you want something like myrepos, which is always written by Joey.
+
+> Perhaps entire Git repositories can be added as a single unit for tracking in git-annex?
+
+Seems unlikely as it is out of scope for git-annex.
+"""]]
diff --git a/doc/forum/Using_git-annex_to_manage_git_repositories/comment_2_ab24098c2f4519cbbfdcd00ba3e21d2b._comment b/doc/forum/Using_git-annex_to_manage_git_repositories/comment_2_ab24098c2f4519cbbfdcd00ba3e21d2b._comment
new file mode 100644
index 0000000000..87fe1768bf
--- /dev/null
+++ b/doc/forum/Using_git-annex_to_manage_git_repositories/comment_2_ab24098c2f4519cbbfdcd00ba3e21d2b._comment
@@ -0,0 +1,10 @@
+[[!comment format=mdwn
+ username="joey"
+ subject="""comment 2"""
+ date="2015-04-06T17:01:32Z"
+ content="""
+This is not what git-annex was designed to do, so I don't think it's the
+right tool for the job.
+
+I suggest 
+"""]]
diff --git a/doc/forum/Using_git-annex_to_manage_git_repositories/comment_3_5cf9e2ff83bb76650469444c80fee4fb._comment b/doc/forum/Using_git-annex_to_manage_git_repositories/comment_3_5cf9e2ff83bb76650469444c80fee4fb._comment
new file mode 100644
index 0000000000..115cea9061
--- /dev/null
+++ b/doc/forum/Using_git-annex_to_manage_git_repositories/comment_3_5cf9e2ff83bb76650469444c80fee4fb._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="https://www.google.com/accounts/o8/id?id=AItOawk_STgv_0fB-y1x_fgzUI10Bg-aajJyMyc"
+ nickname="猫"
+ subject="comment 3"
+ date="2015-04-08T06:30:06Z"
+ content="""
+I'll take a closer look at myrepos, but it doesn't seem to do location tracking, which is the main feature of git-annex that I'm looking for.
+"""]]
diff --git a/doc/forum/Using_git-annex_via_command_line_in_OS_X.mdwn b/doc/forum/Using_git-annex_via_command_line_in_OS_X.mdwn
new file mode 100644
index 0000000000..5eba701bdd
--- /dev/null
+++ b/doc/forum/Using_git-annex_via_command_line_in_OS_X.mdwn
@@ -0,0 +1,3 @@
+After installing the binary for Mac OS X and including some larger directories, my computer now seems quite busy (for the last days) with the import process. I see a process called git-annex causing the load, so it seems OK. As the git-annex assistant seems to hang quite a bit, I would like to see the progress using a command line interface. However, I cannot use the "git annex" command as I get the message "git: 'annex' is not a git command. See 'git --help'."
+
+I guess this is my normal git version, installed by homebrew (the OS X version for apt-get) that does not know anything about the installed binary. Can I still use the CLI with this version? Or better, is there or will there be a way to install git-annex from source, or even better using "brew install git-annex"?
diff --git a/doc/forum/Using_git-annex_via_command_line_in_OS_X/comment_1_1c9e121f60fb6868c07f1a53b03c4ed0._comment b/doc/forum/Using_git-annex_via_command_line_in_OS_X/comment_1_1c9e121f60fb6868c07f1a53b03c4ed0._comment
new file mode 100644
index 0000000000..1f51e6c971
--- /dev/null
+++ b/doc/forum/Using_git-annex_via_command_line_in_OS_X/comment_1_1c9e121f60fb6868c07f1a53b03c4ed0._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="https://www.google.com/accounts/o8/id?id=AItOawnaZYaspvzx1lQiM56UQo-X82BnPiAaiEY"
+ nickname="Martin"
+ subject="comment 1"
+ date="2013-08-19T18:07:04Z"
+ content="""
+One more question. When running the git-annex binary the webapp opens. After closing the window I cannot access the webapp anymore. Using the IP + port number (http://127.0.0.1:53027/) does not work. Would be handy to have a bookmark for the webapp that works as long as the assistant is running.
+"""]]
diff --git a/doc/forum/Using_git-annex_via_command_line_in_OS_X/comment_2_52d8ffba82e29ac2722a8e43e469cc47._comment b/doc/forum/Using_git-annex_via_command_line_in_OS_X/comment_2_52d8ffba82e29ac2722a8e43e469cc47._comment
new file mode 100644
index 0000000000..b30773cd22
--- /dev/null
+++ b/doc/forum/Using_git-annex_via_command_line_in_OS_X/comment_2_52d8ffba82e29ac2722a8e43e469cc47._comment
@@ -0,0 +1,9 @@
+[[!comment format=mdwn
+ username="https://www.google.com/accounts/o8/id?id=AItOawnaZYaspvzx1lQiM56UQo-X82BnPiAaiEY"
+ nickname="Martin"
+ subject="git-annex via comand line"
+ date="2013-08-19T21:53:11Z"
+ content="""
+Just found the page how to install the CLI on OS X: http://git-annex.branchable.com/install/OSX/
+That solved all my questions for now.
+"""]]
diff --git a/doc/forum/Using_git_annex_with_Android_tablet_which_only_has_exFAT_and_no_symlinks.mdwn b/doc/forum/Using_git_annex_with_Android_tablet_which_only_has_exFAT_and_no_symlinks.mdwn
new file mode 100644
index 0000000000..3420c129f7
--- /dev/null
+++ b/doc/forum/Using_git_annex_with_Android_tablet_which_only_has_exFAT_and_no_symlinks.mdwn
@@ -0,0 +1,8 @@
+Hallo,
+
+I have got a Samsung Galaxy Note 10.1 I want to sync with a remote ssh-Server, but the tablet only supports exFAT and FAT file Systems, so I cannot use symlinks.
+
+Does any solution exist for this? 
+
+Mebus
+
diff --git a/doc/forum/Using_git_annex_with_Android_tablet_which_only_has_exFAT_and_no_symlinks/comment_1_b9f202a30ba7e3bc264064d24454c099._comment b/doc/forum/Using_git_annex_with_Android_tablet_which_only_has_exFAT_and_no_symlinks/comment_1_b9f202a30ba7e3bc264064d24454c099._comment
new file mode 100644
index 0000000000..f5d4b3cdd5
--- /dev/null
+++ b/doc/forum/Using_git_annex_with_Android_tablet_which_only_has_exFAT_and_no_symlinks/comment_1_b9f202a30ba7e3bc264064d24454c099._comment
@@ -0,0 +1,12 @@
+[[!comment format=mdwn
+ username="http://joeyh.name/"
+ ip="64.134.31.139"
+ subject="comment 1"
+ date="2013-10-22T17:37:44Z"
+ content="""
+The sad state of affairs that in the supposedly modern 21st century, popular operating systems are regressing in functionality? 
+
+Yes, git-annex can deal with this. `git annex init` will detect when it's run on a crippled filesystem and enable various workarounds, including [[direct_mode]].
+
+The git-annex android app deals with all this automatically when creating a repository.
+"""]]
diff --git a/doc/forum/Using_git_annex_with_Android_tablet_which_only_has_exFAT_and_no_symlinks/comment_2_1334a8d9f4bb60f3bf3ebabc656d98d9._comment b/doc/forum/Using_git_annex_with_Android_tablet_which_only_has_exFAT_and_no_symlinks/comment_2_1334a8d9f4bb60f3bf3ebabc656d98d9._comment
new file mode 100644
index 0000000000..493aaf65e3
--- /dev/null
+++ b/doc/forum/Using_git_annex_with_Android_tablet_which_only_has_exFAT_and_no_symlinks/comment_2_1334a8d9f4bb60f3bf3ebabc656d98d9._comment
@@ -0,0 +1,18 @@
+[[!comment format=mdwn
+ username="Mebus"
+ ip="2001:4dd0:ff00:29::2"
+ subject="comment 2"
+ date="2013-10-31T23:39:42Z"
+ content="""
+Hallo,
+
+I tried it again.
+
+On the Android tablet I get many files, which have got a size of 200 Bytes. And they should at least be a view kB.
+
+Why?
+
+Mebus
+
+
+"""]]
diff --git a/doc/forum/Using_git_annex_with_Android_tablet_which_only_has_exFAT_and_no_symlinks/comment_3_076f22d05fad140068a540e4d835106f._comment b/doc/forum/Using_git_annex_with_Android_tablet_which_only_has_exFAT_and_no_symlinks/comment_3_076f22d05fad140068a540e4d835106f._comment
new file mode 100644
index 0000000000..0fb4f61e2b
--- /dev/null
+++ b/doc/forum/Using_git_annex_with_Android_tablet_which_only_has_exFAT_and_no_symlinks/comment_3_076f22d05fad140068a540e4d835106f._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="http://joeyh.name/"
+ ip="209.250.56.47"
+ subject="comment 3"
+ date="2013-11-01T16:42:33Z"
+ content="""
+Files in the git repository whose content has not yet been downloaded will appear as these short placeholder files when using a filesystem that does not support symlinks. Once the files have been downloaded, that is replaced with their actual content. You may need to set up a way to your android tablet to download the files from the place they're being created.
+"""]]
diff --git a/doc/forum/Using_git_annex_with_Android_tablet_which_only_has_exFAT_and_no_symlinks/comment_4_c8446ee1b817f1824fa0df07e742015c._comment b/doc/forum/Using_git_annex_with_Android_tablet_which_only_has_exFAT_and_no_symlinks/comment_4_c8446ee1b817f1824fa0df07e742015c._comment
new file mode 100644
index 0000000000..cd42b6c158
--- /dev/null
+++ b/doc/forum/Using_git_annex_with_Android_tablet_which_only_has_exFAT_and_no_symlinks/comment_4_c8446ee1b817f1824fa0df07e742015c._comment
@@ -0,0 +1,12 @@
+[[!comment format=mdwn
+ username="Mebus"
+ ip="2001:4dd0:ff00:29::2"
+ subject="comment 4"
+ date="2013-11-11T20:24:11Z"
+ content="""
+Does it have to be the place, where they were created or can this also be a central server, I pushed everything to?
+
+Mebus
+
+
+"""]]
diff --git a/doc/forum/Using_git_annex_with_Android_tablet_which_only_has_exFAT_and_no_symlinks/comment_5_f746c1b85ee8e4b57b6819ccceabd28b._comment b/doc/forum/Using_git_annex_with_Android_tablet_which_only_has_exFAT_and_no_symlinks/comment_5_f746c1b85ee8e4b57b6819ccceabd28b._comment
new file mode 100644
index 0000000000..676e7b6f10
--- /dev/null
+++ b/doc/forum/Using_git_annex_with_Android_tablet_which_only_has_exFAT_and_no_symlinks/comment_5_f746c1b85ee8e4b57b6819ccceabd28b._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="http://joeyh.name/"
+ ip="209.250.56.246"
+ subject="comment 5"
+ date="2013-11-13T16:37:46Z"
+ content="""
+The git-annex app will download the files from any remote you set up that has a copy of the file. A transfer remote in the cloud would work well.
+"""]]
diff --git a/doc/forum/Using_git_annex_with_a_SMB__47__FTP_TV_NAS_with_preconfigured_dirs.mdwn b/doc/forum/Using_git_annex_with_a_SMB__47__FTP_TV_NAS_with_preconfigured_dirs.mdwn
new file mode 100644
index 0000000000..1e64430385
--- /dev/null
+++ b/doc/forum/Using_git_annex_with_a_SMB__47__FTP_TV_NAS_with_preconfigured_dirs.mdwn
@@ -0,0 +1,5 @@
+I've got a Freebox Revolution set-top box / TV NAS at home, with some preconfigured layou for videos, photos and music.
+
+I'd like to use it as a git-annex remote, but am afraid there's no support for FTP (authenticated) or SMB shares exported by the NAS (I don't think it supports other protocols even though runs Linux internally, AFAIK).
+
+Is there any option to store much on that NAS and sync it with git-annex ?
diff --git a/doc/forum/Using_git_annex_with_a_SMB__47__FTP_TV_NAS_with_preconfigured_dirs/comment_1_bd73c8d10028e1b45da9ea8f657e5064._comment b/doc/forum/Using_git_annex_with_a_SMB__47__FTP_TV_NAS_with_preconfigured_dirs/comment_1_bd73c8d10028e1b45da9ea8f657e5064._comment
new file mode 100644
index 0000000000..d5e8d54043
--- /dev/null
+++ b/doc/forum/Using_git_annex_with_a_SMB__47__FTP_TV_NAS_with_preconfigured_dirs/comment_1_bd73c8d10028e1b45da9ea8f657e5064._comment
@@ -0,0 +1,10 @@
+[[!comment format=mdwn
+ username="http://cstork.org/"
+ nickname="Chris Stork"
+ subject="Similar to "direct special remotes""
+ date="2013-08-16T17:57:26Z"
+ content="""
+You have a very similar problem as I do.
+
+See my post where I called this [direct special remotes](http://git-annex.branchable.com/forum/Direct_special_remotes/).
+"""]]
diff --git a/doc/forum/Using_git_annex_with_a_SMB__47__FTP_TV_NAS_with_preconfigured_dirs/comment_2_16c3c994ee8fcb466e52ca0e812e5915._comment b/doc/forum/Using_git_annex_with_a_SMB__47__FTP_TV_NAS_with_preconfigured_dirs/comment_2_16c3c994ee8fcb466e52ca0e812e5915._comment
new file mode 100644
index 0000000000..93eef73220
--- /dev/null
+++ b/doc/forum/Using_git_annex_with_a_SMB__47__FTP_TV_NAS_with_preconfigured_dirs/comment_2_16c3c994ee8fcb466e52ca0e812e5915._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="http://olivier.berger.myopenid.com/"
+ nickname="obergix"
+ subject="Except I don't have rsync/sftp on the NAS AFAICT"
+ date="2013-08-17T08:56:37Z"
+ content="""
+There seemed to be some hope with rsync in your case, but not mine. Thanks anyway for the pointer.
+"""]]
diff --git a/doc/forum/Using_git_annex_with_a_SMB__47__FTP_TV_NAS_with_preconfigured_dirs/comment_3_ac60f6edb76bdd541711e472eec9591a._comment b/doc/forum/Using_git_annex_with_a_SMB__47__FTP_TV_NAS_with_preconfigured_dirs/comment_3_ac60f6edb76bdd541711e472eec9591a._comment
new file mode 100644
index 0000000000..8729a8481e
--- /dev/null
+++ b/doc/forum/Using_git_annex_with_a_SMB__47__FTP_TV_NAS_with_preconfigured_dirs/comment_3_ac60f6edb76bdd541711e472eec9591a._comment
@@ -0,0 +1,10 @@
+[[!comment format=mdwn
+ username="http://olivier.berger.myopenid.com/"
+ nickname="obergix"
+ subject="Experimenting with a CIFS mount point of the NAS and direct mode"
+ date="2013-08-17T22:11:19Z"
+ content="""
+I've been experimenting with a direct mode repo on a CIFS mount of the SMB share of the NAS.
+
+Unfortunately, it seems I can't propagate changes made on the laptop to the mount point by issueing a git pull or merge, as it is part of the unsafe commands that don't support direct mode (see details in )... direct mode wouldn't be very useful then : I intend to mirror on the NAS the files I've been managing locally. For instance I will sort my photos in subdirs, on the laptop, and intend to mirror that on the NAS.
+"""]]
diff --git a/doc/forum/Using_git_annex_with_a_SMB__47__FTP_TV_NAS_with_preconfigured_dirs/comment_4_2194f0600d9a90f0d9c947ea9cc213a3._comment b/doc/forum/Using_git_annex_with_a_SMB__47__FTP_TV_NAS_with_preconfigured_dirs/comment_4_2194f0600d9a90f0d9c947ea9cc213a3._comment
new file mode 100644
index 0000000000..4a59acdeb7
--- /dev/null
+++ b/doc/forum/Using_git_annex_with_a_SMB__47__FTP_TV_NAS_with_preconfigured_dirs/comment_4_2194f0600d9a90f0d9c947ea9cc213a3._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="http://joeyh.name/"
+ ip="4.154.0.63"
+ subject="comment 4"
+ date="2013-08-23T17:55:39Z"
+ content="""
+Just run `git annex sync` to update the direct mode repository on your NAS.
+"""]]
diff --git a/doc/forum/Using_git_annex_with_a_SMB__47__FTP_TV_NAS_with_preconfigured_dirs/comment_5_eb7d13f6b6fa674a2536bde51bfc3fd1._comment b/doc/forum/Using_git_annex_with_a_SMB__47__FTP_TV_NAS_with_preconfigured_dirs/comment_5_eb7d13f6b6fa674a2536bde51bfc3fd1._comment
new file mode 100644
index 0000000000..e014efb86c
--- /dev/null
+++ b/doc/forum/Using_git_annex_with_a_SMB__47__FTP_TV_NAS_with_preconfigured_dirs/comment_5_eb7d13f6b6fa674a2536bde51bfc3fd1._comment
@@ -0,0 +1,48 @@
+[[!comment format=mdwn
+ username="http://olivier.berger.myopenid.com/"
+ nickname="obergix"
+ subject="So it seems I have found a sequence that seems to be operating fine"
+ date="2013-08-23T20:28:45Z"
+ content="""
+Here's a script, which I think makes it work, using git clone, git annex copy and git annex sync :
+    # The \"master\" remote which is with default indirect mode, on a Linux FS
+    BASE1=~/tmp
+    REPO1=$BASE1/annex-test
+    
+    # The \"slave\" remote on the NAS (a Samba server), which has been mounted with cifs, and thus will be in direct mode
+    BASE2=/mnt/freebox-server/
+    REPO2=$BASE2/annex-test
+    
+    cd $BASE1
+    mkdir $REPO1
+    cd $REPO1
+    git init
+    git annex init \"my laptop\"
+    
+    cd $REPO1
+    cp -Lr ~/some_large_files ./
+    git annex add some_large_files
+    git commit -m \"added\"
+    
+    cd $BASE2    
+    git clone $REPO1 $REPO2
+    cd $REPO2
+    
+    git annex init \"freebox server\"
+    # This is not really needed, but if you want to replicate on a non cifs mount
+    git annex direct
+    
+    cd $REPO1
+    git remote add freebox-server $REPO2
+    
+    git annex copy --to freebox-server
+    git annex sync
+    
+    cd $REPO2
+    git remote add laptop $REPO1
+    #git annex sync
+
+After this, you should have your .git and plain \"direct\" files on the NAS, mirrored from what's on the laptop.
+
+Hope this helps.
+"""]]
diff --git a/doc/forum/Using_git_annex_with_a_SMB__47__FTP_TV_NAS_with_preconfigured_dirs/comment_6_ae323b16ddb9342e91be955408eca3b1._comment b/doc/forum/Using_git_annex_with_a_SMB__47__FTP_TV_NAS_with_preconfigured_dirs/comment_6_ae323b16ddb9342e91be955408eca3b1._comment
new file mode 100644
index 0000000000..44d3f5a85c
--- /dev/null
+++ b/doc/forum/Using_git_annex_with_a_SMB__47__FTP_TV_NAS_with_preconfigured_dirs/comment_6_ae323b16ddb9342e91be955408eca3b1._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="http://joeyh.name/"
+ ip="4.154.0.63"
+ subject="comment 6"
+ date="2013-08-24T15:58:28Z"
+ content="""
+You can run any of these operations in any order and it will work. Ie, this is far more complicated and detailed than it needs to be. Just run git-annex commands in any order to do what you want to do. If they don't, file a (detailed) bug report.
+"""]]
diff --git a/doc/forum/Using_integer_ranges_with_metadata.mdwn b/doc/forum/Using_integer_ranges_with_metadata.mdwn
new file mode 100644
index 0000000000..1cdbe50671
--- /dev/null
+++ b/doc/forum/Using_integer_ranges_with_metadata.mdwn
@@ -0,0 +1,40 @@
+I keep my movie collection in an annex. I recently wrote a python script that pulls information about each movie down from IMDB and stores it as metadata on the annexed file. One of the attributes I'm storing is `rating`. For instance, the metadata for my copy of Blade Runner looks like this:
+
+    $ git annex metadata blade_runner.mkv
+    metadata blade_runner.mkv
+      director="Ridley Scott"
+      director-lastchanged=2016-04-20@04-21-33
+      genre="Sci-Fi"
+      genre="Thriller"
+      genre-lastchanged=2016-04-20@04-21-33
+      lastchanged=2016-04-20@04-21-33
+      rating=8.2
+      rating-lastchanged=2016-04-20@04-21-33
+      runtime=117
+      runtime-lastchanged=2016-04-20@04-21-33
+      title="Blade Runner"
+      title-lastchanged=2016-04-20@04-21-33
+      year=1982
+      year-lastchanged=2016-04-20@04-21-33
+    ok
+
+I can now use the metadata to ask git annex to show me all movies with a rating of 8.2.
+
+    $ git annex find --metadata rating=8.2
+    blade_runner.mkv
+
+However, that isn't very useful. What I want to do is specify a range. For example, I want to ask git annex to show me all movies with a rating above 8:
+
+    $ git annex find --metadata rating=>8
+
+Or, show me all movies with a rating between 6 and 9
+
+    $ git annex find --metadata rating=>6 rating=<9
+
+Is something like this possible?
+
+I'd like to do something similar with the `year` attribute. Right now I can use metadata views to group movies by their release year, which is pretty neat.
+
+    $ git annex view "year=*"
+
+But I would also like to be able to give a range so that I could group movies by release decade, for example.
diff --git a/doc/forum/Using_integer_ranges_with_metadata/comment_1_36298a679379e807892cd6af18ec1dd6._comment b/doc/forum/Using_integer_ranges_with_metadata/comment_1_36298a679379e807892cd6af18ec1dd6._comment
new file mode 100644
index 0000000000..11795051a9
--- /dev/null
+++ b/doc/forum/Using_integer_ranges_with_metadata/comment_1_36298a679379e807892cd6af18ec1dd6._comment
@@ -0,0 +1,13 @@
+[[!comment format=mdwn
+ username="joey"
+ subject="""comment 1"""
+ date="2016-04-20T17:51:43Z"
+ content="""
+This is a great idea! So great that I got in the ol' time machine,
+set the dial for February 29th 2016, and convinced past-me to implement it
+then.
+
+(For some reason, past-me decided to make the syntax `field>=number` and
+`field<=number` instead of the syntax you suggested. He can be a bit of a
+stick in the mud with his own outdated ideas that he holds onto tightly.)
+"""]]
diff --git a/doc/forum/Using_integer_ranges_with_metadata/comment_2_064e2c5706d1acc0f9e4576ff80407f7._comment b/doc/forum/Using_integer_ranges_with_metadata/comment_2_064e2c5706d1acc0f9e4576ff80407f7._comment
new file mode 100644
index 0000000000..cc95e48166
--- /dev/null
+++ b/doc/forum/Using_integer_ranges_with_metadata/comment_2_064e2c5706d1acc0f9e4576ff80407f7._comment
@@ -0,0 +1,9 @@
+[[!comment format=mdwn
+ username="pigmonkey"
+ subject="comment 2"
+ date="2016-04-20T18:20:43Z"
+ content="""
+Perfect. It works great! Thanks.
+
+Is this documented anywhere? I see it in the [release notes](https://git-annex.branchable.com/news/version_6.20160229/) now, but I didn't see anything mentioned in the metadata, metadata driven views or metadata design pages, or the git-annex-metadata man page. I'm not sure where I should look to see how else I can query the metadata.
+"""]]
diff --git a/doc/forum/Using_integer_ranges_with_metadata/comment_3_c3ce727f93cce19db1f76de9f7216cee._comment b/doc/forum/Using_integer_ranges_with_metadata/comment_3_c3ce727f93cce19db1f76de9f7216cee._comment
new file mode 100644
index 0000000000..bdd36a4341
--- /dev/null
+++ b/doc/forum/Using_integer_ranges_with_metadata/comment_3_c3ce727f93cce19db1f76de9f7216cee._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="joey"
+ subject="""comment 3"""
+ date="2016-04-20T18:25:20Z"
+ content="""
+It's documented in [[git-annex-matching-options]] and
+[[git-annex-preferred-content]].
+"""]]
diff --git a/doc/forum/Using_integer_ranges_with_metadata/comment_4_c3301d1b2b57994666947fd518fb9922._comment b/doc/forum/Using_integer_ranges_with_metadata/comment_4_c3301d1b2b57994666947fd518fb9922._comment
new file mode 100644
index 0000000000..e08fb7d0ec
--- /dev/null
+++ b/doc/forum/Using_integer_ranges_with_metadata/comment_4_c3301d1b2b57994666947fd518fb9922._comment
@@ -0,0 +1,9 @@
+[[!comment format=mdwn
+ username="cstork"
+ subject="comment 4"
+ date="2016-04-21T14:35:46Z"
+ content="""
+Could you share your IMDB -> metadata script?  It would be interesting to see what works in practice.
+
+I was about to write something similar and why duplicate work? :-)
+"""]]
diff --git a/doc/forum/Using_integer_ranges_with_metadata/comment_5_9d310fac003f1cd9e6ca9bc96a210def._comment b/doc/forum/Using_integer_ranges_with_metadata/comment_5_9d310fac003f1cd9e6ca9bc96a210def._comment
new file mode 100644
index 0000000000..619eb3ce02
--- /dev/null
+++ b/doc/forum/Using_integer_ranges_with_metadata/comment_5_9d310fac003f1cd9e6ca9bc96a210def._comment
@@ -0,0 +1,11 @@
+[[!comment format=mdwn
+ username="pigmonkey"
+ subject="comment 5"
+ date="2016-04-21T16:57:03Z"
+ content="""
+Absolutely!
+
+
+
+It's a quick hack job, but has been working so far. I haven't played around with [[tips/automatically adding metadata]] yet. The filename based search does the right thing 99% of the time (at least for my file names), but I still always want to confirm that it is using the right movie data, so I'm running the script manually and keeping it interactive.
+"""]]
diff --git a/doc/forum/Using_integer_ranges_with_metadata/comment_6_ead2b08bf4631547c34486ce58559017._comment b/doc/forum/Using_integer_ranges_with_metadata/comment_6_ead2b08bf4631547c34486ce58559017._comment
new file mode 100644
index 0000000000..785b0e4eb3
--- /dev/null
+++ b/doc/forum/Using_integer_ranges_with_metadata/comment_6_ead2b08bf4631547c34486ce58559017._comment
@@ -0,0 +1,7 @@
+[[!comment format=mdwn
+ username="cstork"
+ subject="comment 6"
+ date="2016-04-23T16:43:11Z"
+ content="""
+Thanks a lot!
+"""]]
diff --git a/doc/forum/Using_standard_groups_and_sync_to_preserve_history__58___--all_not_recognised.mdwn b/doc/forum/Using_standard_groups_and_sync_to_preserve_history__58___--all_not_recognised.mdwn
new file mode 100644
index 0000000000..f191503e3c
--- /dev/null
+++ b/doc/forum/Using_standard_groups_and_sync_to_preserve_history__58___--all_not_recognised.mdwn
@@ -0,0 +1,33 @@
+My copy of *git-annex* refuses to sync all, namely when I try it I get the following error
+
+    $ git annex sync --content --all
+    git-annex: unrecognized option `--all'
+
+    Usage: git-annex sync [REMOTE ...] [option ...]
+        --content  also transfer file contents
+
+    To see additional options common to all commands, run: git annex help options
+
+This contradicts the advice on [preferred content](http://git-annex.branchable.com/preferred_content/) set out under **difference: unused**, 
+and I cannot see any other options in my man page that would address the lack of this option.
+
+The problem I am trying to solve is that I wish to preserve all history on the backup drives.  Namely, if I do the following
+
+    touch test-of-annex-backup.txt
+    git annex add test-of-annex-backup.txt
+    git commit --message='test: Create empty test-of-annex-backup.txt file'
+    git annex edit test-of-annex-backup.txt
+    echo "This line creates version 2 of this file" > test-of-annex-backup.txt
+    git annex add test-of-annex-backup.txt
+    git commit --message='test: Create version 2 of test-of-annex-backup.txt'
+    git annex sync --content --all
+
+I expect to see 2 copies of `test-of-annex-backup.txt` be copied to each accessible annex repository in the `backup` group
+
+I tried googling for `"git annex sync --content --all"`, but I only find pages telling me that this is what I should use, and none saying the option has been deprecated.
+
+I am very confused, as this seems to me an almost stereotypical use of *git-annex*, and yet I cannot see how to do it
+
+thanks
+
+Andrew
diff --git a/doc/forum/Using_standard_groups_and_sync_to_preserve_history__58___--all_not_recognised/comment_1_c3dd239d4b5b372229f2cec42a8a6eb4._comment b/doc/forum/Using_standard_groups_and_sync_to_preserve_history__58___--all_not_recognised/comment_1_c3dd239d4b5b372229f2cec42a8a6eb4._comment
new file mode 100644
index 0000000000..935b46a01b
--- /dev/null
+++ b/doc/forum/Using_standard_groups_and_sync_to_preserve_history__58___--all_not_recognised/comment_1_c3dd239d4b5b372229f2cec42a8a6eb4._comment
@@ -0,0 +1,10 @@
+[[!comment format=mdwn
+ username="joey"
+ subject="""comment 1"""
+ date="2015-02-25T19:28:03Z"
+ content="""
+It looks like the documentation is wrong; `git annex sync` has never
+supported `--all` or `--unused`. I've fixed the documentation. If you do
+need this, please file a todo item so I can remember to try to implement
+it. Should be doable, but not trivially so.
+"""]]
diff --git a/doc/forum/Using_standard_groups_and_sync_to_preserve_history__58___--all_not_recognised/comment_2_b76dd79aabe87ec717eea164aa965d74._comment b/doc/forum/Using_standard_groups_and_sync_to_preserve_history__58___--all_not_recognised/comment_2_b76dd79aabe87ec717eea164aa965d74._comment
new file mode 100644
index 0000000000..2b5a8672b2
--- /dev/null
+++ b/doc/forum/Using_standard_groups_and_sync_to_preserve_history__58___--all_not_recognised/comment_2_b76dd79aabe87ec717eea164aa965d74._comment
@@ -0,0 +1,12 @@
+[[!comment format=mdwn
+ username="https://www.google.com/accounts/o8/id?id=AItOawkWHj0RxNMfuwvFzo2d-V6vBKOYwW_Fnfk"
+ nickname="Andrew"
+ subject="Thanks for the clarification"
+ date="2015-03-07T03:00:14Z"
+ content="""
+I will post the todo, and in the meantime I can script a `git annex copy --unused` followed by a `git annex sync --content` to capture the full history in the archive
+
+regards
+
+Andrew
+"""]]
diff --git a/doc/forum/Using_the_Git-Annex_Assistant_as_a_Backup_and_Syncing_Service.mdwn b/doc/forum/Using_the_Git-Annex_Assistant_as_a_Backup_and_Syncing_Service.mdwn
new file mode 100644
index 0000000000..8d3c840246
--- /dev/null
+++ b/doc/forum/Using_the_Git-Annex_Assistant_as_a_Backup_and_Syncing_Service.mdwn
@@ -0,0 +1,5 @@
+I'm looking to use the Git-Annex Assistant to backup a single repository that is present on and synced between two computers (a home and a working computer). Ideally, each computer uses rsync.net for both of these service, while at the same time treating the cloud storage service as untrusted (so anything stored or tranferred through there is encrypted). Is it possible to do this using solely rsync.net (without the addition of some XMPP service)? According to the software, using shared encryption allows anyone with a clone of the repository to decrypt files on a remote, but the simplest way to make this clone seems to be to first clone to a removable drive, and then from the drive to the second computer (and then deleting the records of the clone to the drive). I'm unsure if by then setting up the backup at rsync.net for the second computer, whether the software will create a second backup that acts independently of the first, neglecting any syncing, or if it will recognize the backup as one of the same repository. I'm also unsure as to whether the software will even sync if the backup is recognized, or whether a tranfer repository at rsync.net is also necessary to complete the setup. Could you perhaps give me some advice on how to achieve this setup, or point me to some information that may help me along?
+
+(If the setup is unclear, I'm essentially trying to replicate something like SpiderOak with the Git-Annex Assistant, without using an XMPP service)
+
+[Edit: It may be easier to just use Git-Annex (no assistant), so that works too]
diff --git a/doc/forum/Using_the_Git-Annex_Assistant_as_a_Backup_and_Syncing_Service/comment_1_7070f6e7e05fba7686d8620d62906a83._comment b/doc/forum/Using_the_Git-Annex_Assistant_as_a_Backup_and_Syncing_Service/comment_1_7070f6e7e05fba7686d8620d62906a83._comment
new file mode 100644
index 0000000000..07cc634340
--- /dev/null
+++ b/doc/forum/Using_the_Git-Annex_Assistant_as_a_Backup_and_Syncing_Service/comment_1_7070f6e7e05fba7686d8620d62906a83._comment
@@ -0,0 +1,12 @@
+[[!comment format=mdwn
+ username="http://joeyh.name/"
+ ip="209.250.56.54"
+ subject="comment 1"
+ date="2014-10-02T15:54:44Z"
+ content="""
+rsync.net alone is not sufficient, because it's only used for storing the (encrypted) contents of files, not for syncing the git repository, or finding out when there other computer has made a change to the repository.
+
+You can use rsync.net plus xmpp. However, xmpp is not encrypted when it passes through the xmpp server.
+
+A fully encrypted option is to install git-anex on a server accessible by ssh, and then use the assistant to set up an encrypted git repository on the server. No xmpp needed in this configuration.
+"""]]
diff --git a/doc/forum/Verification.mdwn b/doc/forum/Verification.mdwn
new file mode 100644
index 0000000000..cbce0b4085
--- /dev/null
+++ b/doc/forum/Verification.mdwn
@@ -0,0 +1,9 @@
+Hi,
+
+I have a simple Client Mac <> FullBackup on remote SSH <> Client On Mac setup which seems to work great.
+
+I have 2 questions :
+- Whenever a new file is created or changed it can takes a few hours until the second client gets the file. Is this normal ?
+- How Can I make sure, that all the files are on the FullBackup ?
+
+Many thank !
diff --git a/doc/forum/Verification/comment_1_d56818a8f5b3a94ecf5159c76f24768c._comment b/doc/forum/Verification/comment_1_d56818a8f5b3a94ecf5159c76f24768c._comment
new file mode 100644
index 0000000000..d9684d922d
--- /dev/null
+++ b/doc/forum/Verification/comment_1_d56818a8f5b3a94ecf5159c76f24768c._comment
@@ -0,0 +1,18 @@
+[[!comment format=mdwn
+ username="joey"
+ subject="""comment 1"""
+ date="2015-02-27T18:51:52Z"
+ content="""
+Well, it depends on how big the files are, but taking a few hours to sync
+might indicate that your repositories are not immediately notifying
+one-another of changes. It might be falling back to polling every half
+an hour for new changes.
+
+You could fix that by setting up XMPP, or better, by installing git-annex
+5.20140421 or newer on the SSH server; then the clients would immediately
+notify when there are changes.
+
+You can find out if all files are present on the server by running
+`git annex find --not --in $server` on one of the clients. Any files
+it prints out have not been stored in the server.
+"""]]
diff --git a/doc/forum/Verification/comment_2_7df45d1e20a32458791603d5b9fe3dc4._comment b/doc/forum/Verification/comment_2_7df45d1e20a32458791603d5b9fe3dc4._comment
new file mode 100644
index 0000000000..c98a210b6a
--- /dev/null
+++ b/doc/forum/Verification/comment_2_7df45d1e20a32458791603d5b9fe3dc4._comment
@@ -0,0 +1,15 @@
+[[!comment format=mdwn
+ username="Régis"
+ subject="comment 2"
+ date="2015-02-27T21:38:05Z"
+ content="""
+Thanks for this speedy answer !
+
+I have version 5.20150219 installed on all the clients and on the server.
+The ssh server is a gcrypt repository. Could this be the reason of the clients not being notificated of the  changes ??
+
+Also, in theory, some file on one of the repositories could get corrupted, deleted, whatever. How can I make the repo check if everything is like intended on the other repos ? 
+For example, I made a test renaming a file within the objects directory on the remote ssh server and ran git annex fchk, but it reported nothing...
+
+Many thanks again !
+"""]]
diff --git a/doc/forum/Verification/comment_3_74db3ec8b03f48912306e48b8d5f7242._comment b/doc/forum/Verification/comment_3_74db3ec8b03f48912306e48b8d5f7242._comment
new file mode 100644
index 0000000000..962707277e
--- /dev/null
+++ b/doc/forum/Verification/comment_3_74db3ec8b03f48912306e48b8d5f7242._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="Régis"
+ subject="comment 3"
+ date="2015-02-28T18:33:27Z"
+ content="""
+Also, I noted a difference between encrypted (gcrypt) repos and unencrypted repos. The second ones have the other icon saying that it is live messaging. The encrypted one have a standard icon.
+Is it a technical limitation that encrypted repos can not live message ? And could it be the reason why the sync is not happening itself until i manually choose \"sync\" from the menu ?
+"""]]
diff --git a/doc/forum/Verification/comment_4_eb4d936a9bd577f58483b278ae5dc5f6._comment b/doc/forum/Verification/comment_4_eb4d936a9bd577f58483b278ae5dc5f6._comment
new file mode 100644
index 0000000000..f1d04e83f1
--- /dev/null
+++ b/doc/forum/Verification/comment_4_eb4d936a9bd577f58483b278ae5dc5f6._comment
@@ -0,0 +1,15 @@
+[[!comment format=mdwn
+ username="joey"
+ subject="""comment 4"""
+ date="2015-03-16T18:36:45Z"
+ content="""
+You can use `git annex fsck` to verify your repository contents.
+If you want to verify a local repository, the best thing to do is
+to run `git annex fsck` there. If you cannot do that, you can use
+`git annex fsck --from remoterepo --fast` to verify a remote. If you leave
+off the --fast it will download all file contents to completely verify
+them.
+
+I suggest you read git-annex's documentation, there is plenty of it about
+using git-annex fsck to verify repositories.
+"""]]
diff --git a/doc/forum/Verification/comment_5_c327c72ceced27920681d5f93bc256c8._comment b/doc/forum/Verification/comment_5_c327c72ceced27920681d5f93bc256c8._comment
new file mode 100644
index 0000000000..e07a572547
--- /dev/null
+++ b/doc/forum/Verification/comment_5_c327c72ceced27920681d5f93bc256c8._comment
@@ -0,0 +1,9 @@
+[[!comment format=mdwn
+ username="joey"
+ subject="""comment 5"""
+ date="2015-03-16T18:39:01Z"
+ content="""
+The lack of "live messaging" for gcrypt repos is a bug. I'm fixing
+it now and the next version of git-annex will have remotedaemon
+properly supporting gcrypt repos.
+"""]]
diff --git a/doc/forum/View_performance_with_7__44__000_files.mdwn b/doc/forum/View_performance_with_7__44__000_files.mdwn
new file mode 100644
index 0000000000..21491c77f5
--- /dev/null
+++ b/doc/forum/View_performance_with_7__44__000_files.mdwn
@@ -0,0 +1,26 @@
+I've imported about half of my photos into an annex on an external HDD,
+using metadata extensively for EXIF info, including place names.
+
+Checking out a new view is slower than I expected, at ~6 minutes.
+Is this expected behavior, am I pushing the limits of file count already?
+
+Is there anything I can do to speed things up?
+
+    % ls -1 | wc -l
+        7050
+    
+    % git branch -vv
+      git-annex                      4e590d4 update
+    * master                         985ba54 add jen's phone backups
+      views/Year=_;Month=_;Address=_ 795a58b refs/heads/views/Year=_;Month=_;Address=_
+    
+    % /usr/bin/time -p git annex view "Year=*" "Address=*" 
+    view  (searching...)
+    
+    Checking out files: 100% (12789/12789), done.
+    Switched to branch 'views/Year=_;Address=_'
+    ok
+    real       376.80
+    user        17.98
+    sys          9.66
+
diff --git a/doc/forum/View_performance_with_7__44__000_files/comment_1_e45ea752100d09d29efb6136a722eab3._comment b/doc/forum/View_performance_with_7__44__000_files/comment_1_e45ea752100d09d29efb6136a722eab3._comment
new file mode 100644
index 0000000000..cf6f140b2a
--- /dev/null
+++ b/doc/forum/View_performance_with_7__44__000_files/comment_1_e45ea752100d09d29efb6136a722eab3._comment
@@ -0,0 +1,34 @@
+[[!comment format=mdwn
+ username="https://www.google.com/accounts/o8/id?id=AItOawmvzzyDA8uXFz8yokeCrepbh8PwWe_WrjE"
+ nickname="Michael"
+ subject="debug output"
+ date="2014-06-16T14:17:06Z"
+ content="""
+So after looking for a debug flag, I see that it's spending all its time in cat-file, which I guess is probably not a surprise:
+
+    % /usr/bin/time -p git annex view \"Year=*\" \"Address=*\" -d
+    view  [2014-06-16 06:29:12 PDT] read: git [\"--git-dir=/Volumes/Four TB Backup/Photos/.git\",\"--work-tree=/Volumes/Four TB Backup/Photos\",\"symbolic-ref\",\"HEAD\"]
+    [2014-06-16 06:29:12 PDT] read: git [\"--git-dir=/Volumes/Four TB Backup/Photos/.git\",\"--work-tree=/Volumes/Four TB Backup/Photos\",\"show-ref\",\"refs/heads/master\"]
+    [2014-06-16 06:29:12 PDT] read: git [\"--git-dir=/Volumes/Four TB Backup/Photos/.git\",\"--work-tree=/Volumes/Four TB Backup/Photos\",\"symbolic-ref\",\"HEAD\"]
+    [2014-06-16 06:29:12 PDT] read: git [\"--git-dir=/Volumes/Four TB Backup/Photos/.git\",\"--work-tree=/Volumes/Four TB Backup/Photos\",\"show-ref\",\"refs/heads/master\"]
+    (searching...)
+    [2014-06-16 06:29:12 PDT] read: git [\"--git-dir=/Volumes/Four TB Backup/Photos/.git\",\"--work-tree=/Volumes/Four TB Backup/Photos\",\"ls-files\",\"--cached\",\"-z\",\"--\",\"/Volumes/Four TB Backup/Photos\"]
+    [2014-06-16 06:29:12 PDT] feed: git [\"--git-dir=/Volumes/Four TB Backup/Photos/.git\",\"--work-tree=/Volumes/Four TB Backup/Photos\",\"update-index\",\"-z\",\"--index-info\"]
+    [2014-06-16 06:29:12 PDT] chat: git [\"--git-dir=/Volumes/Four TB Backup/Photos/.git\",\"--work-tree=/Volumes/Four TB Backup/Photos\",\"hash-object\",\"-w\",\"--stdin-paths\",\"--no-filters\"]
+    [2014-06-16 06:29:12 PDT] read: git [\"--git-dir=/Volumes/Four TB Backup/Photos/.git\",\"--work-tree=/Volumes/Four TB Backup/Photos\",\"show-ref\",\"git-annex\"]
+    [2014-06-16 06:29:12 PDT] read: git [\"--git-dir=/Volumes/Four TB Backup/Photos/.git\",\"--work-tree=/Volumes/Four TB Backup/Photos\",\"show-ref\",\"--hash\",\"refs/heads/git-annex\"]
+    [2014-06-16 06:29:12 PDT] read: git [\"--git-dir=/Volumes/Four TB Backup/Photos/.git\",\"--work-tree=/Volumes/Four TB Backup/Photos\",\"log\",\"refs/heads/git-annex..4e590d433e01886e2823c3316d18b7e3cbafe227\",\"--oneline\",\"-n1\"]
+    [2014-06-16 06:29:12 PDT] chat: git [\"--git-dir=/Volumes/Four TB Backup/Photos/.git\",\"--work-tree=/Volumes/Four TB Backup/Photos\",\"cat-file\",\"--batch\"]
+    [2014-06-16 06:34:54 PDT] read: git [\"--git-dir=/Volumes/Four TB Backup/Photos/.git\",\"--work-tree=/Volumes/Four TB Backup/Photos\",\"write-tree\"]
+    [2014-06-16 06:34:55 PDT] chat: git [\"--git-dir=/Volumes/Four TB Backup/Photos/.git\",\"--work-tree=/Volumes/Four TB Backup/Photos\",\"commit-tree\",\"c0fd1ea9ccae70a7353a2e787089b88115a42b7a\"]
+    [2014-06-16 06:34:55 PDT] call: git [\"--git-dir=/Volumes/Four TB Backup/Photos/.git\",\"--work-tree=/Volumes/Four TB Backup/Photos\",\"update-ref\",\"refs/heads/views/Year=_;Address=_\",\"249c37d78e45d1fccfcb00f40337e05cb69fb64c\"]
+    
+    [2014-06-16 06:34:55 PDT] call: git [\"--git-dir=/Volumes/Four TB Backup/Photos/.git\",\"--work-tree=/Volumes/Four TB Backup/Photos\",\"checkout\",\"views/Year=_;Address=_\"]
+    Checking out files: 100% (12789/12789), done.
+    Switched to branch 'views/Year=_;Address=_'
+    ok
+    real       362.14
+    user        18.16
+    sys          7.70
+
+"""]]
diff --git a/doc/forum/View_performance_with_7__44__000_files/comment_2_b1942eed65e9b5c046095a094191a38c._comment b/doc/forum/View_performance_with_7__44__000_files/comment_2_b1942eed65e9b5c046095a094191a38c._comment
new file mode 100644
index 0000000000..91259f3351
--- /dev/null
+++ b/doc/forum/View_performance_with_7__44__000_files/comment_2_b1942eed65e9b5c046095a094191a38c._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="http://joeyh.name/"
+ ip="108.236.230.124"
+ subject="comment 2"
+ date="2014-06-16T17:52:34Z"
+ content="""
+View's don't scale to lots of files yet. Discussed in [[design/caching_database]]
+"""]]
diff --git a/doc/forum/View_performance_with_7__44__000_files/comment_3_e6e19339c9d72cf8eaae32ef4269e850._comment b/doc/forum/View_performance_with_7__44__000_files/comment_3_e6e19339c9d72cf8eaae32ef4269e850._comment
new file mode 100644
index 0000000000..01ac676cbf
--- /dev/null
+++ b/doc/forum/View_performance_with_7__44__000_files/comment_3_e6e19339c9d72cf8eaae32ef4269e850._comment
@@ -0,0 +1,10 @@
+[[!comment format=mdwn
+ username="https://www.google.com/accounts/o8/id?id=AItOawmvzzyDA8uXFz8yokeCrepbh8PwWe_WrjE"
+ nickname="Michael"
+ subject="comment 3"
+ date="2014-06-16T18:19:20Z"
+ content="""
+@joeyh, thanks - is that code all up to date here: https://github.com/joeyh/git-annex/tree/database ?
+
+I'd be interested in helping test it.
+"""]]
diff --git a/doc/forum/View_special_remote_information__63__.mdwn b/doc/forum/View_special_remote_information__63__.mdwn
new file mode 100644
index 0000000000..31382040bf
--- /dev/null
+++ b/doc/forum/View_special_remote_information__63__.mdwn
@@ -0,0 +1 @@
+How is it possible to view the URL etc of a special remote with git annex? I checked out a git annex repository and would like to know where the files where fetched from. 
diff --git a/doc/forum/View_special_remote_information__63__/comment_1_7c106e0679890f7279c73b5c52d648e2._comment b/doc/forum/View_special_remote_information__63__/comment_1_7c106e0679890f7279c73b5c52d648e2._comment
new file mode 100644
index 0000000000..0a3914b3e4
--- /dev/null
+++ b/doc/forum/View_special_remote_information__63__/comment_1_7c106e0679890f7279c73b5c52d648e2._comment
@@ -0,0 +1,10 @@
+[[!comment format=mdwn
+ username="joey"
+ subject="""comment 1"""
+ date="2015-03-16T18:00:33Z"
+ content="""
+Recent versions of git-annex have expanded `git annex info $remote`
+to be able to show some info for special remotes. What is
+shown depends on the type of remote. For example, it will show
+the bucket used for a S3 remote, or the url used for a WebDAV remote.
+"""]]
diff --git a/doc/forum/WARNING__58___linker__58__git-annex_has_text_relocations.mdwn b/doc/forum/WARNING__58___linker__58__git-annex_has_text_relocations.mdwn
new file mode 100644
index 0000000000..2d9ee1324f
--- /dev/null
+++ b/doc/forum/WARNING__58___linker__58__git-annex_has_text_relocations.mdwn
@@ -0,0 +1,7 @@
+I have configured git-annex on my Nexus 5. Even though it works fantastic, it shows some warning messages.
+
+1: 'git-annex sync' :  WARNING: linker: git-annex has text relocations. This is wasting memory and is a security risk. Please fix.
+
+2: 'git log': error: cannot run less. No such file or directory.
+
+Both the commands will work has expected with these warnings. What could be the issue here.
diff --git a/doc/forum/WARNING__58___linker__58__git-annex_has_text_relocations/comment_1_fee360353f0b46aab6ee7a902c0837bb._comment b/doc/forum/WARNING__58___linker__58__git-annex_has_text_relocations/comment_1_fee360353f0b46aab6ee7a902c0837bb._comment
new file mode 100644
index 0000000000..8d9af9624e
--- /dev/null
+++ b/doc/forum/WARNING__58___linker__58__git-annex_has_text_relocations/comment_1_fee360353f0b46aab6ee7a902c0837bb._comment
@@ -0,0 +1,11 @@
+[[!comment format=mdwn
+ username="http://joeyh.name/"
+ ip="209.250.56.146"
+ subject="comment 1"
+ date="2014-03-05T20:38:03Z"
+ content="""
+  
+Says that should have been fixed in ndk-r8c's gcc, but git-annex is already built with ndk-r9b. It might be though, that the llvm compiler still has the problem. In any case, I doubt there is a real security issue here, it's probably just a binary exploit hardening thing.
+
+As for the git log, git users the less pager by default, but of course it's not necessary and so not included on Android. You can `git config core.pager cat` to get around this.
+"""]]
diff --git a/doc/forum/Walkthrough_does_not_work_for_me__44___what_am_i_doing_wrong__63__.mdwn b/doc/forum/Walkthrough_does_not_work_for_me__44___what_am_i_doing_wrong__63__.mdwn
new file mode 100644
index 0000000000..c6cb35912f
--- /dev/null
+++ b/doc/forum/Walkthrough_does_not_work_for_me__44___what_am_i_doing_wrong__63__.mdwn
@@ -0,0 +1,16 @@
+I'm trying to use git-annex as a dropbox-replacement, but I'm hitting unexpected things although I'm doing exactly what I see in the walkthrough screencast and/or the walkthrough commands. I'm using Ubuntu 13.10, installed git-annex with apt-get. It is version 4.20130815.
+
+1) I launch the webapp and create an ~/annex folder as my initial repo. I copy some small text files in it, I see in webapp that they are added. 
+2) I add a new "Removable Storage" repo, in my usb drive. I do not touch any other settings, leave it as is. (ie in transfer group for example). It is successfully added to the repo list and I see sync complete and files synced on the webapp.
+3) When I check the contents of the usb disk annex folder, I see the following, while I was expecting to see the files I synced instead.
+
+emre@emrenb:~$ ls /media/emre/348B-78F0/annex/
+annex  branches  config  description  HEAD  hooks  info  objects  refs
+
+However, I expect the below which is the content of my ~/annex folder:
+emre@emrenb:~$ ls annex/
+unison-sync.log  unison-1.log  unison-2.log  unison.log
+
+In the walkthrough video, when Joey adds a USB storage, the filenames & thir contents sync to the usb drive. Why doesn't it work for me? 
+
+Note that I tried also android app and also the zipped tarball build. Same result, I never see actual files. 
diff --git a/doc/forum/Walkthrough_does_not_work_for_me__44___what_am_i_doing_wrong__63__/comment_1_cdac15fec6fc41d5487b7f653fa718a4._comment b/doc/forum/Walkthrough_does_not_work_for_me__44___what_am_i_doing_wrong__63__/comment_1_cdac15fec6fc41d5487b7f653fa718a4._comment
new file mode 100644
index 0000000000..437e2ecdc1
--- /dev/null
+++ b/doc/forum/Walkthrough_does_not_work_for_me__44___what_am_i_doing_wrong__63__/comment_1_cdac15fec6fc41d5487b7f653fa718a4._comment
@@ -0,0 +1,14 @@
+[[!comment format=mdwn
+ username="https://www.google.com/accounts/o8/id?id=AItOawk3HGoDpnOPob5jOjvIootmkve1-nCpRiI"
+ nickname="Kalle"
+ subject="Default removable storage repo not readable without git annex"
+ date="2013-10-21T08:50:50Z"
+ content="""
+Hi,
+
+Sounds to me you are expecting the files to be readable by a filemanager? By default a removable drive repo is only readable by git-annex.
+
+The standard use case is to move files between computers with git-annex installed. I agree this is not what you´d expect.
+
+Read  and the last post of  for more info.
+"""]]
diff --git a/doc/forum/Walkthrough_does_not_work_for_me__44___what_am_i_doing_wrong__63__/comment_2_82050b7dc367ca5968ab0306db9bd7e3._comment b/doc/forum/Walkthrough_does_not_work_for_me__44___what_am_i_doing_wrong__63__/comment_2_82050b7dc367ca5968ab0306db9bd7e3._comment
new file mode 100644
index 0000000000..edcc49a35d
--- /dev/null
+++ b/doc/forum/Walkthrough_does_not_work_for_me__44___what_am_i_doing_wrong__63__/comment_2_82050b7dc367ca5968ab0306db9bd7e3._comment
@@ -0,0 +1,10 @@
+[[!comment format=mdwn
+ username="http://joeyh.name/"
+ ip="64.134.31.139"
+ subject="comment 2"
+ date="2013-10-21T22:29:14Z"
+ content="""
+Kalle is right (I seem to keep saying that, Kalle!)
+
+Following the walkthrough commands will get you a non-bare git repository on a removable drive, and you can run \"git annex sync\" in there periodically and see all the files in your repository on the removable drive. The webapp does not make that kind of repository though, and in the screencast I'm showing the normal behavior of the webapp.
+"""]]
diff --git a/doc/forum/Walkthrough_for_direct_mode__63__.mdwn b/doc/forum/Walkthrough_for_direct_mode__63__.mdwn
new file mode 100644
index 0000000000..b3fae29447
--- /dev/null
+++ b/doc/forum/Walkthrough_for_direct_mode__63__.mdwn
@@ -0,0 +1 @@
+Hello Joey, I would be very much interested in a walkthrough for direct mode, as detailed as the one currently published. I see the comments in the current walkthrough on some differences to direct mode, but to me it is not obvious what best practices for git-annex use would be in direct mode, with and without the assistant. For a mix of Linux, OS X and Windows installations in the homes, it may also be interesting to see how to best set up the individual machines. Many thanks - 
diff --git a/doc/forum/Want_to_stop_using_Git-Annex.mdwn b/doc/forum/Want_to_stop_using_Git-Annex.mdwn
new file mode 100644
index 0000000000..a076cf1b06
--- /dev/null
+++ b/doc/forum/Want_to_stop_using_Git-Annex.mdwn
@@ -0,0 +1,9 @@
+Hi
+
+I created a git annex repo in one of my drives. But now all my files have turned into symbolic links.
+I need to remove the git repo and get my files back. How can I do that.
+
+Some of the softwares am using will not follow symbolic links, so need put the files back properly as soon as  possible.
+
+Please help.
+Thank you in advance :)
diff --git a/doc/forum/Want_to_stop_using_Git-Annex/comment_1_32e37515bd4f5d22ff9aedd3c9d98046._comment b/doc/forum/Want_to_stop_using_Git-Annex/comment_1_32e37515bd4f5d22ff9aedd3c9d98046._comment
new file mode 100644
index 0000000000..9937e5d0ca
--- /dev/null
+++ b/doc/forum/Want_to_stop_using_Git-Annex/comment_1_32e37515bd4f5d22ff9aedd3c9d98046._comment
@@ -0,0 +1,10 @@
+[[!comment format=mdwn
+ username="http://ypid.wordpress.com/"
+ ip="213.153.84.215"
+ subject="Direct mode"
+ date="2014-06-27T07:36:40Z"
+ content="""
+Hi
+
+[[direct_mode/]] is your friend.
+"""]]
diff --git a/doc/forum/Want_to_stop_using_Git-Annex/comment_2_e29e6d052ef3677ad7d5615721f3fe33._comment b/doc/forum/Want_to_stop_using_Git-Annex/comment_2_e29e6d052ef3677ad7d5615721f3fe33._comment
new file mode 100644
index 0000000000..0eddeb201c
--- /dev/null
+++ b/doc/forum/Want_to_stop_using_Git-Annex/comment_2_e29e6d052ef3677ad7d5615721f3fe33._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="http://joeyh.name/"
+ ip="108.236.230.124"
+ subject="comment 2"
+ date="2014-07-02T17:04:29Z"
+ content="""
+git annex uninit
+"""]]
diff --git a/doc/forum/Wanted_content_based_on_date_added.mdwn b/doc/forum/Wanted_content_based_on_date_added.mdwn
new file mode 100644
index 0000000000..12c12e892e
--- /dev/null
+++ b/doc/forum/Wanted_content_based_on_date_added.mdwn
@@ -0,0 +1,12 @@
+Hi,
+
+I was wondering whether it was possible to configure git annex in a way, that new content is held in a specific repository until it is considered old.
+
+I found that "git config annex.genmetadata true" creates year and month tags in the metadata. However, creating the wanted expression poses a problem. Afaik there is no way to access the current year/month in a wanted expression and no way to do arithmetic in a wanted expression. Furthermore, I do not know, when the assistant checks wanted expressions for matches? I assume it does so only when something changes in git or when the wanted expressions change but not in regular intervals?
+
+I guess I could whip up a cronjob that changes the wanted expression every month and do the calculations outside of git annex but this seems somehow wrong. 
+
+Is this a problem other users share or maybe have already solved?
+
+Cheers,
+Marek
diff --git a/doc/forum/Wanted_content_based_on_date_added/comment_1_af2a6e6cba4cdfe24e9ee2c819a007eb._comment b/doc/forum/Wanted_content_based_on_date_added/comment_1_af2a6e6cba4cdfe24e9ee2c819a007eb._comment
new file mode 100644
index 0000000000..a10b909b30
--- /dev/null
+++ b/doc/forum/Wanted_content_based_on_date_added/comment_1_af2a6e6cba4cdfe24e9ee2c819a007eb._comment
@@ -0,0 +1,17 @@
+[[!comment format=mdwn
+ username="joey"
+ subject="""comment 1"""
+ date="2017-02-20T16:07:52Z"
+ content="""
+The assistant re-checks preferred content as rarely as it can get away with,
+since that can be an expensive process. Basically, it checks it when
+starting up, and when a remote is added or edited, and when it detects
+certian kinds of divergences between the local repository and a remote.
+
+I agree that it's not possible to express an expiry using a preferred
+content expression.
+
+What would work better is if you can move the old files to some other
+directory via a cron job or whatever means. Then it's easy to configure it
+not to want files in that directory.
+"""]]
diff --git a/doc/forum/Watch__47__assistant__47__webapp_documentation.mdwn b/doc/forum/Watch__47__assistant__47__webapp_documentation.mdwn
new file mode 100644
index 0000000000..3d185ba800
--- /dev/null
+++ b/doc/forum/Watch__47__assistant__47__webapp_documentation.mdwn
@@ -0,0 +1,12 @@
+Hello,
+
+I'm not sure about the differences and interactions between watch / assistant / webapp / direct mode. I think I figured the following out, can someone confirm this, and perhaps a few words to the documentation / man page?
+
+- git annex watch uses inotify to find new files, and runs git annex add on them (it does not do regular git add)
+- git annex assistant does the same as watch, but also runs git annex sync for each new file (does it also enable direct mode?)
+- git annex webapp does the same as assistant, and also starts a webapp (in my case it immediately started sending files to origin, without asking for confirmation, which was surprising, I guess this is because I have * annex.numcopies=2 set, and there was only one copy. Still I interpreted the documentation as if it would only show me an interface, not start doing things right away.)
+
+Do these commands do anything else than what I described above?
+
+best regards,
+Tom
diff --git a/doc/forum/Watch__47__assistant__47__webapp_documentation/comment_1_adb377589dbae7fc91001df235c6b48e._comment b/doc/forum/Watch__47__assistant__47__webapp_documentation/comment_1_adb377589dbae7fc91001df235c6b48e._comment
new file mode 100644
index 0000000000..556cf4f9b8
--- /dev/null
+++ b/doc/forum/Watch__47__assistant__47__webapp_documentation/comment_1_adb377589dbae7fc91001df235c6b48e._comment
@@ -0,0 +1,14 @@
+[[!comment format=mdwn
+ username="http://joeyh.name/"
+ ip="4.154.3.125"
+ subject="comment 1"
+ date="2013-02-07T18:12:17Z"
+ content="""
+Most of this is explained pretty well on the [[man page|git-annex]]. 
+
+`git annex webapp` doesn't do anything on its own, other than start the assistant if it is not already running, and point your browser at its built-in webapp. Starting the assistant manually will always have the same effect as opening the webapp.
+
+The assistant syncs files as configured by the preferred content settings of repositories. There's an interface to disable this syncing in the webapp, or change the setting to something that doesn't prefer all content be synced to a repository. This syncing of content is different from `git annex sync`, which only syncs git repository data. It's more like running `git annex copy --auto`
+
+The assistant enables direct mode when a new repository is created (using the webapp). It does not enable direct mode when run in an existing repository.
+"""]]
diff --git a/doc/forum/Web_app_on_server_without_X11__63__.mdwn b/doc/forum/Web_app_on_server_without_X11__63__.mdwn
new file mode 100644
index 0000000000..cb880cf292
--- /dev/null
+++ b/doc/forum/Web_app_on_server_without_X11__63__.mdwn
@@ -0,0 +1,3 @@
+When I run git-annex-webapp, a browser is opened and I am redirected to the assistant. However, how can I run the web app and just have it start the server process without opening a browser, and navigating to the page from a remote computer?
+
+Thanks!
diff --git a/doc/forum/Web_app_on_server_without_X11__63__/comment_1_bee409ad389e5fb5f3e8de4b5c68278d._comment b/doc/forum/Web_app_on_server_without_X11__63__/comment_1_bee409ad389e5fb5f3e8de4b5c68278d._comment
new file mode 100644
index 0000000000..f428ece1e1
--- /dev/null
+++ b/doc/forum/Web_app_on_server_without_X11__63__/comment_1_bee409ad389e5fb5f3e8de4b5c68278d._comment
@@ -0,0 +1,10 @@
+[[!comment format=mdwn
+ username="Xyem"
+ ip="81.111.193.130"
+ subject="comment 1"
+ date="2014-09-10T14:32:21Z"
+ content="""
+[Running the assistant headless](http://git-annex.branchable.com/design/assistant/blog/day_232__headless_webapp/).
+
+Hope this helps.
+"""]]
diff --git a/doc/forum/Webapp_not_watching_repos.mdwn b/doc/forum/Webapp_not_watching_repos.mdwn
new file mode 100644
index 0000000000..b751b65637
--- /dev/null
+++ b/doc/forum/Webapp_not_watching_repos.mdwn
@@ -0,0 +1 @@
+I am using the webapp to watch 3 repositories. When I run git annex webapp it starts watching one of the repos but unless I switch to other repos it does not start noticing changes after switching to each repo once it works as intended.
diff --git a/doc/forum/Webapp_not_watching_repos/comment_1_02735050122afdb6498d91b462d32767._comment b/doc/forum/Webapp_not_watching_repos/comment_1_02735050122afdb6498d91b462d32767._comment
new file mode 100644
index 0000000000..9b0d61af10
--- /dev/null
+++ b/doc/forum/Webapp_not_watching_repos/comment_1_02735050122afdb6498d91b462d32767._comment
@@ -0,0 +1,12 @@
+[[!comment format=mdwn
+ username="http://joeyh.name/"
+ ip="209.250.56.227"
+ subject="comment 1"
+ date="2013-12-27T19:02:28Z"
+ content="""
+Each separate repository needs to have a separate `git annex assistant` process handling it.
+
+Normally, these would all be started when `git annex assistant --autostart` is run, typically when you log into a desktop environment.
+
+So, on Linux, this is done via a .desktop file (or you can arrange for the command to be run as the right user at boot if not using a desktop); on OSX it's done via a .plist file. On Windows and Android, nothing handles the autostarting yet.
+"""]]
diff --git a/doc/forum/Webapp_not_watching_repos/comment_2_8ab2d4c18ddc99774e44c4f4401bfa57._comment b/doc/forum/Webapp_not_watching_repos/comment_2_8ab2d4c18ddc99774e44c4f4401bfa57._comment
new file mode 100644
index 0000000000..63573b0231
--- /dev/null
+++ b/doc/forum/Webapp_not_watching_repos/comment_2_8ab2d4c18ddc99774e44c4f4401bfa57._comment
@@ -0,0 +1,12 @@
+[[!comment format=mdwn
+ username="https://me.yahoo.com/a/FHnTlSBo1eCGJRwueeKeB6.RCaPbGMPr5jxx8A--#ce0d8"
+ nickname="Hamza"
+ subject="comment 2"
+ date="2013-12-27T22:40:06Z"
+ content="""
+I don't have a desktop env. just plain xmonad, so the correct sequence to start the webapp is,
+
+    git annex assistant --autostart && git annex webapp
+
+right?
+"""]]
diff --git a/doc/forum/Webapp_not_watching_repos/comment_3_b448aa5a95a57a5228b361390e5fc838._comment b/doc/forum/Webapp_not_watching_repos/comment_3_b448aa5a95a57a5228b361390e5fc838._comment
new file mode 100644
index 0000000000..92fd3cb459
--- /dev/null
+++ b/doc/forum/Webapp_not_watching_repos/comment_3_b448aa5a95a57a5228b361390e5fc838._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="http://joeyh.name/"
+ ip="209.250.56.227"
+ subject="comment 3"
+ date="2013-12-29T17:59:53Z"
+ content="""
+In this case, I would probably put `git annex assistant --autostart` in `.xinitrc`, or even make a @reboot cron job that starts it. The webapp can then be opened with `git annex webapp` at any time. (The webapp is not a separate process and does not need to be started at login.)
+"""]]
diff --git a/doc/forum/Webapp_on_ARM.mdwn b/doc/forum/Webapp_on_ARM.mdwn
new file mode 100644
index 0000000000..c9766dd8cf
--- /dev/null
+++ b/doc/forum/Webapp_on_ARM.mdwn
@@ -0,0 +1,6 @@
+Webapp on ARM?
+==============
+
+Since the webapp is apparently now available on Android (not tested yet, but I plan to do it soon ;)), I was wondering what was the status of the webapp on ARM. Does it build, does it work, and if it does would it be possible to enable it in the Debian package for the next release?
+
+For the record, I'm using git-annex on my NAS (Synology DS413j). I'm using the Debian armel package and running it in an Arch Linux chroot (a very simple setup). It works really well, and I'm extremely satisfied with git-annex. Thanks a lot for all your work, Joey *et al.*!
diff --git a/doc/forum/Webapp_on_ARM/comment_1_82ac40cef5b59070136527b8d81a5ce2._comment b/doc/forum/Webapp_on_ARM/comment_1_82ac40cef5b59070136527b8d81a5ce2._comment
new file mode 100644
index 0000000000..00e3fed262
--- /dev/null
+++ b/doc/forum/Webapp_on_ARM/comment_1_82ac40cef5b59070136527b8d81a5ce2._comment
@@ -0,0 +1,10 @@
+[[!comment format=mdwn
+ username="http://joeyh.name/"
+ ip="4.154.1.10"
+ subject="comment 1"
+ date="2013-07-22T19:13:38Z"
+ content="""
+To build the WebApp on arm, you need a ghc that supports template haskell on arm. While I have heard persistent rumors of such a thing existing, I have yet to see it.
+
+It's possible to build the WebApp on arm without that, but it's a complicated process, involving first building on x86 with the same versions of the haskell libraries that you have on the arm system, and then using the EvilSplicer to generate a source tree with the template haskell expanded, which can then be built on arm. This build process is not really suitable for a Debian package.
+"""]]
diff --git a/doc/forum/Weird_behavior_with_OS_X_Finder_and_Preview.app.mdwn b/doc/forum/Weird_behavior_with_OS_X_Finder_and_Preview.app.mdwn
new file mode 100644
index 0000000000..055a09e2ac
--- /dev/null
+++ b/doc/forum/Weird_behavior_with_OS_X_Finder_and_Preview.app.mdwn
@@ -0,0 +1,12 @@
+I am having some weird issues with git annex under OS X.  I am new to git annex, and am generally familiar with git.  I really like the concept of git-annex and am trying to lean how to fit it into my workflow, but Im running into problems.  
+
+I have a folder that I would like to keep in sync across multiple computers and thumb drives.  Right now this folder consists of other folders and PDF files.  After initializing git and git-annex, all of the files turn into symlinks, as expected.  
+
+Looking at this folder in OSX's finder, none of the symlinks have preview images unfortunately, as expected.  Also, when trying to open the file (or symlink, and in this case PDFs) in finder, it opens the console which outputs a bunch of crap and then nothing happens.  Right clicking on the symlink in finder and using Open with... the default application to open the symlink to the PDF is OSX's Preview.app, as expected, however for some reason this isn't being used when simple double clicking.  If you click on the Preview.app option under the right click menu, Preview.app launches, but never opens the file.  Selecting an alternative such as the Skim.app pdf viewer (if installed) successfully launches the application and opens the file.  
+
+My questions are:
+1. What is the cause of not being able to launch the file in the correct app by simply double clicking it?  Do I have something mis-configued?  Is it a bug with git-annex?  Is git-annex simply not set up to work with finder yet?
+2. How come preview can't open the file but skim can?  This might be directly related to the above question, but maybe not.
+3. Is it possible to get file previews for the symlinks in your annex folder?  Are there plans to enable this kind of thing in the future, if even possible? 
+
+Details about my setup.  I am running on a 32-bit Core Duo Macbook pro from 2006, so enviably I have to run 10.6.8 and its the highest OS version I can reach right now.  All my CLI packages are installed and up to date with Mac Homebrew.  I am running git version 1.7.12.  I freshly installed haskell-platform: stable 2012.2.0.0 via homebrew.  From there I installed git-annex-3.20120825.
diff --git a/doc/forum/Weird_behavior_with_OS_X_Finder_and_Preview.app/comment_1_8c8d86790a9d31518f9bb96a2d2dafee._comment b/doc/forum/Weird_behavior_with_OS_X_Finder_and_Preview.app/comment_1_8c8d86790a9d31518f9bb96a2d2dafee._comment
new file mode 100644
index 0000000000..4234eac2bc
--- /dev/null
+++ b/doc/forum/Weird_behavior_with_OS_X_Finder_and_Preview.app/comment_1_8c8d86790a9d31518f9bb96a2d2dafee._comment
@@ -0,0 +1,18 @@
+[[!comment format=mdwn
+ username="http://joeyh.name/"
+ ip="4.153.8.30"
+ subject="comment 1"
+ date="2012-09-09T17:22:58Z"
+ content="""
+It is unfortunate that certain badly-coded programs fall over on symlinks. In theory, any program that doesn't try to be too smart for its own good will see a symlink as just like the file it links to, and should behave the same. This is the theory that led me to use symlinks in git-annex. But some programs say \"oh, a symlink! Let's see what file that's pointing at, and behave differently!\" I don't understand the mindset that leads to this kind of program being written, unless it's a program like `cp`, that has very good reasons for (at your command) following symlinks or not.
+
+I know absolutely nothing about Mac OSX, having never used it (except for in a ssh session while porting git-annex to it). But I have seen a similar problem with the FBReader ebook viewer. [Bug report](http://bugs.debian.org/685433)
+
+I was able to work around that bug by migrating my files to the SHA256E backend. This ensures that the keys stored in git-annex have extensions like .pdf etc, similar to the symlinks that point to them. You might try migrating some of your files too, and see if it makes OSX behave better. Although I thought OSX didn't rely on file extensions to determine file type.. 
+
+Anyway, the command is:
+
+    git annex migrate --backend=SHA256E
+
+I have been considering switching the default backend to SHA256E to avoid this type of problem. Your testing will be important grist for that decision.
+"""]]
diff --git a/doc/forum/Weird_behavior_with_OS_X_Finder_and_Preview.app/comment_2_b538dc2c6f122b9ce5f7569de1b03f3e._comment b/doc/forum/Weird_behavior_with_OS_X_Finder_and_Preview.app/comment_2_b538dc2c6f122b9ce5f7569de1b03f3e._comment
new file mode 100644
index 0000000000..ee09be502c
--- /dev/null
+++ b/doc/forum/Weird_behavior_with_OS_X_Finder_and_Preview.app/comment_2_b538dc2c6f122b9ce5f7569de1b03f3e._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="http://mike.magin.org/"
+ nickname="mmagin"
+ subject="comment 2"
+ date="2012-09-11T15:27:58Z"
+ content="""
+I've been using git-annex mostly on Mac OS X, and it rapidly caused me to switch to SHA256E.  I'd certainly be in favor of it being the default.  Aside from [[bugs/fsck thinks file content is bad when it isn't]] (fixed by the time I encountered it), I have had no issues with it so far.
+"""]]
diff --git a/doc/forum/Weird_behavior_with_OS_X_Finder_and_Preview.app/comment_3_16e6724fa184392d4decbe0c4eb6efe6._comment b/doc/forum/Weird_behavior_with_OS_X_Finder_and_Preview.app/comment_3_16e6724fa184392d4decbe0c4eb6efe6._comment
new file mode 100644
index 0000000000..0315feb642
--- /dev/null
+++ b/doc/forum/Weird_behavior_with_OS_X_Finder_and_Preview.app/comment_3_16e6724fa184392d4decbe0c4eb6efe6._comment
@@ -0,0 +1,14 @@
+[[!comment format=mdwn
+ username="https://www.google.com/accounts/o8/id?id=AItOawkurjhi0CRJvgm7QNaZDWS9hitBtavqIpc"
+ nickname="Bret"
+ subject="comment 3"
+ date="2012-09-12T09:23:51Z"
+ content="""
+Sorry for the delayed response.  I am in the middle of an apartment move.
+
+I tried changing my default PDF viewer to Skim to see if that improved anything.  Nada.  Same thing.  Finder opens the terminal and outputs a bunch of nonsense.  
+
+I tried `git annex migrate --backend=SHA256E` and it seemed to really improve things from finders point of view.  Still not icon previews, but you can launch every file by double clicking it and preview is able to follow the symlink.  
+
+Note, I am on OS X 10.6.  I cant upgrade to 10.7+ due to my laptops aging processor (10.7+ requires 64-bit intel chips) and thus can't test git-annex on a more modern OS version.  Someone should test it out on a newer version of OS X before chaining any defaults, however, it seems to really improve things for the setup I have.
+"""]]
diff --git a/doc/forum/Weird_behavior_with_OS_X_Finder_and_Preview.app/comment_4_e514fe2d4d0ad6a10e281939e6ab4266._comment b/doc/forum/Weird_behavior_with_OS_X_Finder_and_Preview.app/comment_4_e514fe2d4d0ad6a10e281939e6ab4266._comment
new file mode 100644
index 0000000000..aa50ab57c8
--- /dev/null
+++ b/doc/forum/Weird_behavior_with_OS_X_Finder_and_Preview.app/comment_4_e514fe2d4d0ad6a10e281939e6ab4266._comment
@@ -0,0 +1,15 @@
+[[!comment format=mdwn
+ username="https://www.google.com/accounts/o8/id?id=AItOawmBUR4O9mofxVbpb8JV9mEbVfIYv670uJo"
+ nickname="Justin"
+ subject="comment 4"
+ date="2012-09-12T20:32:52Z"
+ content="""
+I see a migration to SHA256E currently outputs:
+
+migrate issue.txt (checksum...) (Recording state in git...)
+ok
+
+Is that verifying the existing checksum or re-computing it to re-name the file?  If the latter, couldn't the SHA256 to SHA256E migration just rename the file?
+
+I would be worried about this process hiding data corruption.. If it isn't using the old checksum but instead re-computing a new one then it would be easy to miss the fact that a files checksum changed during this process.
+"""]]
diff --git a/doc/forum/Weird_behavior_with_OS_X_Finder_and_Preview.app/comment_5_e0eec765f72f7bf6f5a2a92c9b5dacad._comment b/doc/forum/Weird_behavior_with_OS_X_Finder_and_Preview.app/comment_5_e0eec765f72f7bf6f5a2a92c9b5dacad._comment
new file mode 100644
index 0000000000..254a524aa3
--- /dev/null
+++ b/doc/forum/Weird_behavior_with_OS_X_Finder_and_Preview.app/comment_5_e0eec765f72f7bf6f5a2a92c9b5dacad._comment
@@ -0,0 +1,10 @@
+[[!comment format=mdwn
+ username="http://joeyh.name/"
+ ip="4.153.8.30"
+ subject="comment 5"
+ date="2012-09-14T04:20:27Z"
+ content="""
+Good catch, I've made it verify the old checksum first.
+
++/- E migrations are not currently optimised.
+"""]]
diff --git a/doc/forum/What_am_I_doing_wrong_with_move__63__.mdwn b/doc/forum/What_am_I_doing_wrong_with_move__63__.mdwn
new file mode 100644
index 0000000000..0bf0980c71
--- /dev/null
+++ b/doc/forum/What_am_I_doing_wrong_with_move__63__.mdwn
@@ -0,0 +1,48 @@
+I still have little experience with git-annex, so I may be missing something fairly obvious.
+
+I have been moving files between two repositories. Things *seemed* to be going well, but today I noticed that I was missing some content that I have just moved. I would like some help to figure out where I went wrong to avoid doing worse mistakes in the future.
+
+What I did was to run the command `git-annex move --from=toshiba` to move a bunch of files from my USB unit to my current repository and then I ran `git-annex sync` on both ends. Afterwards I noticed that the content of the files was not available, so I tried to track one of them.
+What I could see in my local indirect repository was a broken symbolic link pointing to `../.git/annex/objects/ZQ/WF/SHA256E-s241--f3d7e5d1f788235b8eec0af58cc0c526b112b9e834a47ba7a475876c49dce343.jpg/SHA256E-s241--f3d7e5d1f788235b8eec0af58cc0c526b112b9e834a47ba7a475876c49dce343.jpg`.
+
+`git-annex info` gave me similar information:
+
+    file: Heavy Metal 1981.jpg
+    size: 241 bytes
+    key: SHA256E-s241--f3d7e5d1f788235b8eec0af58cc0c526b112b9e834a47ba7a475876c49dce343.jpg
+
+`git-annex whereis` locates the file still on its origin:
+
+    whereis Heavy Metal 1981.jpg (1 copy) 
+  	    49b5b3a4-56ac-4cf2-aed9-1c23d3181c97 -- Toshiba USB HDD [toshiba]
+    ok
+
+On my external HDD drive, where I have an indirect mode repository, the file has already been replaced by a reference (241 bytes).
+
+    $ cat "Heavy Metal 1981.jpg"
+    ../../../../../../../../../media/TOSHIBA EXT/annex/.git/annex/objects/4z/gf/SHA256E-s245--ae647e7ad31089255413a9290ca9344542f3cd15ecef66884613bf776387633d.jpg/SHA256E-s245--ae647e7ad31089255413a9290ca9344542f3cd15ecef66884613bf776387633d.jpg
+
+`git-annex log` remembers something about the operation:
+
+    + Tue, 17 May 2016 12:22:43 WEST Heavy Metal 1981.jpg | 49b5b3a4-56ac-4cf2-aed9-1c23d3181c97 -- Toshiba USB HDD [toshiba]
+
+I tried to `git-annex get "Heavy Metal 1981.jpg"`, and now I have a working symlink on my PC. However it does not point to the image, but to the same 241 bytes reference file that I have on my external HDD.
+
+`git-annex whereis` now mentions 2 locations to my file, but none of the working dirs holds its contents. So, where are the contents of the file? Lost somewhere? It appears that git-annex took the indirect mode reference file and took it for the real file contents -- and that is not good.
+
+I looked at the output of `git-annex unused`, cross-referenced with `git log --stat -S` and I managed to find it somewhere in the list of unused files in my PC's repository, but not on the external drive.
+
+Still, at this point I'm a little worried. I would like to understand what I could have done to cause this mess. Also, how I can clean it up (the rest of the files remain as broken links at the moment).
+
+I have a couple more drives I wanted to add to this setup, but you can understand that I hesitate a little bit at the moment. Maybe I have "lost" more data than I realize.
+
+The version details:
+
+    git-annex version: 6.20160511
+    build flags: Assistant Webapp Pairing Testsuite S3(multipartupload)(storageclasses) WebDAV Inotify DBus DesktopNotify XMPP ConcurrentOutput TorrentParser MagicMime Feeds Quvi
+    key/value backends: SHA256E SHA256 SHA512E SHA512 SHA224E SHA224 SHA384E SHA384 SHA3_256E SHA3_256 SHA3_512E SHA3_512 SHA3_224E SHA3_224 SHA3_384E SHA3_384 SKEIN256E SKEIN256 SKEIN512E SKEIN512 SHA1E SHA1 MD5E MD5 WORM URL
+    remote types: git gcrypt S3 bup directory rsync web bittorrent webdav tahoe glacier ddar hook external
+    local repository version: 5
+    supported repository versions: 5 6
+    upgrade supported from repository versions: 0 1 2 4 5
+    operating system: linux x86_64
diff --git a/doc/forum/What_am_I_doing_wrong_with_move__63__/comment_1_a49d98426ef8dd17e6cca6b5fba5c9bf._comment b/doc/forum/What_am_I_doing_wrong_with_move__63__/comment_1_a49d98426ef8dd17e6cca6b5fba5c9bf._comment
new file mode 100644
index 0000000000..d1c96f4f38
--- /dev/null
+++ b/doc/forum/What_am_I_doing_wrong_with_move__63__/comment_1_a49d98426ef8dd17e6cca6b5fba5c9bf._comment
@@ -0,0 +1,21 @@
+[[!comment format=mdwn
+ username="joey"
+ subject="""comment 1"""
+ date="2016-05-17T18:04:53Z"
+ content="""
+It sounds like the symlink in the git repository has been changed to point
+to some other content that the content it had originally. Since git tracks
+past versions of files, you should be able to use `git log` on the file to
+find out when this hapened and you can use `git checkout` to check out the
+original version of the file, or `git revert` the commit that changed it to
+point to the wrong content. The content of the file should still be stored
+in the git annex, so you should be able to access it this way.
+
+Now, it kind of sounds like something added the git-annex link for a
+file, as it would appear on the FAT filesytem, to git-annex as the new 
+content of the file. It would certianly be a bug if a git-annex command
+did that.
+
+What does `git annex info` (with no other parameters) report when you run it in the repository on
+the USB drive? What filesystem is in use on the USB drive?
+"""]]
diff --git a/doc/forum/What_am_I_doing_wrong_with_move__63__/comment_2_eeae2f172b4481fcb08745fc28aa4d4e._comment b/doc/forum/What_am_I_doing_wrong_with_move__63__/comment_2_eeae2f172b4481fcb08745fc28aa4d4e._comment
new file mode 100644
index 0000000000..2ec35684c7
--- /dev/null
+++ b/doc/forum/What_am_I_doing_wrong_with_move__63__/comment_2_eeae2f172b4481fcb08745fc28aa4d4e._comment
@@ -0,0 +1,42 @@
+[[!comment format=mdwn
+ username="Gus"
+ subject="Thank you"
+ date="2016-05-19T11:27:15Z"
+ content="""
+The external unit seems to hold a NTFS. Here is the `git-annex info` output:
+
+    repository mode: direct
+    trusted repositories: 0
+    semitrusted repositories: 4
+    	00000000-0000-0000-0000-000000000001 -- web
+     	00000000-0000-0000-0000-000000000002 -- bittorrent
+     	069de9a2-dc53-4c0a-82e0-a61a1f29e6b3 -- stratus PC [stratus]
+     	49b5b3a4-56ac-4cf2-aed9-1c23d3181c97 -- Toshiba USB HDD [here]
+    untrusted repositories: 0
+    transfers in progress: none
+    available local disk space: 4.42 gigabytes (+1 megabyte reserved)
+    local annex keys: 5556
+    local annex size: 1.66 terabytes
+    annexed files in working tree: 6618
+    size of annexed files in working tree: 1.1 terabytes
+    bloom filter size: 32 mebibytes (1.1% full)
+    backend usage: 
+    	SHA256E: 6618
+
+However, `git status` says:
+
+    fatal: This operation must be run in a work tree
+
+Which is not the same message as \"no git found here\", but is also not what I expected to see. `git log` seems to work but says nothing about the file at hand.
+
+On the PC side, however, I can see three commits on the file (I wish the commit message contained the command line with arguments, rather than the less descriptive \"git-annex in Toshiba USB HDD\").
+Using `git show` and `git cat-file` I managed to determine the following:
+
+March 4: the initial version of the file was committed.
+
+May 17 11:51: the file's content changed to `../../../../../../../../../media/TOSHIBA EXT/annex/.git/annex/objects/kx/3W/SHA256E-s96418--c6164e17d88914b2e6781e2cb8e7b91e9669ddf2d9ee6f5cbb17f3212bccfba4.jpg/SHA256E-s96418--c6164e17d88914b2e6781e2cb8e7b91e9669ddf2d9ee6f5cbb17f3212bccfba4.jpg`. This is blob `../.git/annex/objects/4z/gf/SHA256E-s245--ae647e7ad31089255413a9290ca9344542f3cd15ecef66884613bf776387633d.jpg/SHA256E-s245--ae647e7ad31089255413a9290ca9344542f3cd15ecef66884613bf776387633d.jpg`.
+
+May 17 12:22 the file's content changed again to `../../../../../../../../../media/TOSHIBA EXT/annex/.git/annex/objects/4z/gf/SHA256E-s245--ae647e7ad31089255413a9290ca9344542f3cd15ecef66884613bf776387633d.jpg/SHA256E-s245--ae647e7ad31089255413a9290ca9344542f3cd15ecef66884613bf776387633d.jpg`, i.e., a reference to the previous object. This is blob `../.git/annex/objects/ZQ/WF/SHA256E-s241--f3d7e5d1f788235b8eec0af58cc0c526b112b9e834a47ba7a475876c49dce343.jpg/SHA256E-s241--f3d7e5d1f788235b8eec0af58cc0c526b112b9e834a47ba7a475876c49dce343.jpg`.
+
+What else can I do in order to work out what went wrong? Is having concurrent commands manipulating the same repository a bad idea?
+"""]]
diff --git a/doc/forum/What_am_I_doing_wrong_with_move__63__/comment_3_f2770f86138d9b8489fbf9a25462b4ce._comment b/doc/forum/What_am_I_doing_wrong_with_move__63__/comment_3_f2770f86138d9b8489fbf9a25462b4ce._comment
new file mode 100644
index 0000000000..e0c0d3aaf4
--- /dev/null
+++ b/doc/forum/What_am_I_doing_wrong_with_move__63__/comment_3_f2770f86138d9b8489fbf9a25462b4ce._comment
@@ -0,0 +1,10 @@
+[[!comment format=mdwn
+ username="Gus"
+ subject="I think I have figured out something"
+ date="2016-06-05T22:42:17Z"
+ content="""
+When I `git-annex drop --force` a file from my direct repository, it gets replaced by a symbolic-link-like file, containing a path.
+Then, when I `git-annex sync` the repository to propagate the changes I have made, the file's content gets updated as if the file has been replaced.
+
+My question is then: why does the original file gets replaced by the link-like file when I drop it?
+"""]]
diff --git a/doc/forum/What_am_I_doing_wrong_with_move__63__/comment_4_76aa27cdeefb1413baaa1c891ccd4d0d._comment b/doc/forum/What_am_I_doing_wrong_with_move__63__/comment_4_76aa27cdeefb1413baaa1c891ccd4d0d._comment
new file mode 100644
index 0000000000..a801357916
--- /dev/null
+++ b/doc/forum/What_am_I_doing_wrong_with_move__63__/comment_4_76aa27cdeefb1413baaa1c891ccd4d0d._comment
@@ -0,0 +1,27 @@
+[[!comment format=mdwn
+ username="joey"
+ subject="""comment 4"""
+ date="2016-06-09T19:51:10Z"
+ content="""
+The symbolic-link-like file is in fact, a symlink, which is what git-annex
+uses to represent an annexed file in git. If your filesystem does
+not support symlinks, git writes the link location to a regular file
+instead.
+
+git annex drop removes the content of a file from the local repository, but
+its symlink remains checked into git. So, the content of the file is
+replaced by the symlink in your working tree.
+
+That symlink should be the same thing git already had recorded for the
+file.
+
+Based on your earlier comment, it does seem that the symlink standin file
+that git uses is being treated as new content for the file, and getting
+annexed. That would be a bug.
+
+Is that happening when you run `git annex sync` on Linux, or is it on Windows?
+
+What else can you tell or show me to help me reproduce your problem? 
+I've tried setting up an NTFS filesystem, putting a git-annex repository on it, and dropping a file;
+git-annex sync did not do the wrong thing when I tried it.
+"""]]
diff --git a/doc/forum/What_can_be_done_in_case_of_conflict.mdwn b/doc/forum/What_can_be_done_in_case_of_conflict.mdwn
new file mode 100644
index 0000000000..42167a6091
--- /dev/null
+++ b/doc/forum/What_can_be_done_in_case_of_conflict.mdwn
@@ -0,0 +1,7 @@
+Hi,
+
+How can I resolve the conflict when it occurs?
+
+Suppose I have 2 branches (master, current), When I merge these branches or while doing cherry-pick, if I get conflict how can I resolve it?
+
+Thank You
diff --git a/doc/forum/What_can_be_done_in_case_of_conflict/comment_1_5ca86b099dfa08a50f656ea03bf1dcd9._comment b/doc/forum/What_can_be_done_in_case_of_conflict/comment_1_5ca86b099dfa08a50f656ea03bf1dcd9._comment
new file mode 100644
index 0000000000..6f1d241300
--- /dev/null
+++ b/doc/forum/What_can_be_done_in_case_of_conflict/comment_1_5ca86b099dfa08a50f656ea03bf1dcd9._comment
@@ -0,0 +1,12 @@
+[[!comment format=mdwn
+ username="http://joey.kitenet.net/"
+ nickname="joey"
+ subject="comment 1"
+ date="2012-04-23T14:29:03Z"
+ content="""
+You handle conflicts in annexed files the same as you would handle them in other binary files checked into git.
+
+For example, you might choose to `git rm` or `git add` the file to resolve the conflict.
+
+[[Previous_discussion|forum/A_really_stupid_question]]
+"""]]
diff --git a/doc/forum/What_can_be_done_in_case_of_conflict/comment_2_69ee17959a92bb8359c0fd7b2a9d8dfb._comment b/doc/forum/What_can_be_done_in_case_of_conflict/comment_2_69ee17959a92bb8359c0fd7b2a9d8dfb._comment
new file mode 100644
index 0000000000..f4293d9c11
--- /dev/null
+++ b/doc/forum/What_can_be_done_in_case_of_conflict/comment_2_69ee17959a92bb8359c0fd7b2a9d8dfb._comment
@@ -0,0 +1,10 @@
+[[!comment format=mdwn
+ username="https://www.google.com/accounts/o8/id?id=AItOawlYu7QmD7wrbHWkoxuriaA9XcijM-g5vrQ"
+ nickname="Royal"
+ subject="Resolving conflict"
+ date="2012-04-24T03:59:31Z"
+ content="""
+Hi,
+Now I am able to resolve the conflict.
+Thank you.
+"""]]
diff --git a/doc/forum/What_can_be_done_in_case_of_conflict/comment_3_017f4bac57a040c496e0c9d068dcfd9e._comment b/doc/forum/What_can_be_done_in_case_of_conflict/comment_3_017f4bac57a040c496e0c9d068dcfd9e._comment
new file mode 100644
index 0000000000..a0d3ded394
--- /dev/null
+++ b/doc/forum/What_can_be_done_in_case_of_conflict/comment_3_017f4bac57a040c496e0c9d068dcfd9e._comment
@@ -0,0 +1,41 @@
+[[!comment format=mdwn
+ username="https://www.google.com/accounts/o8/id?id=AItOawlYu7QmD7wrbHWkoxuriaA9XcijM-g5vrQ"
+ nickname="Royal"
+ subject="Resolving conflict"
+ date="2012-04-23T15:49:30Z"
+ content="""
+Thanks for the reply.
+
+I am executing the following commands.
+
+git init main
+cd main
+git annex init main
+echo a > a
+git annex add a
+git commit -m Initial
+git annex unlock a
+echo aa > a
+git annex add a
+git commit -m first
+git annex unlock a
+echo aaa > a
+git annex add a
+git commit -m second
+git log
+git cherry-pick 
+
+--------------------
+Error: 
+
+error: could not apply 2be8f38... first
+hint: after resolving the conflicts, mark the corrected paths
+hint: with 'git add ' or 'git rm '
+hint: and commit the result with 'git commit'
+
+How can resolve the the above conflict.
+If I see the content of the file I will get the content of second commit.
+Is there any way I can get the content for first commit(Like in git we have 'theirs' option.)
+
+Thank you.
+"""]]
diff --git a/doc/forum/What_does___34__disable__47__enable_expiry_after_X_days__34___mean__63__.mdwn b/doc/forum/What_does___34__disable__47__enable_expiry_after_X_days__34___mean__63__.mdwn
new file mode 100644
index 0000000000..e95e27a755
--- /dev/null
+++ b/doc/forum/What_does___34__disable__47__enable_expiry_after_X_days__34___mean__63__.mdwn
@@ -0,0 +1 @@
+Git-annex assistant configuration has an item saying "disable/enable expiry after X days".  What does it mean?  Am I the only one confused by this?
diff --git a/doc/forum/What_does___34__disable__47__enable_expiry_after_X_days__34___mean__63__/comment_1_face45c85e4713325bb23bbca7b07502._comment b/doc/forum/What_does___34__disable__47__enable_expiry_after_X_days__34___mean__63__/comment_1_face45c85e4713325bb23bbca7b07502._comment
new file mode 100644
index 0000000000..adae1861a1
--- /dev/null
+++ b/doc/forum/What_does___34__disable__47__enable_expiry_after_X_days__34___mean__63__/comment_1_face45c85e4713325bb23bbca7b07502._comment
@@ -0,0 +1,28 @@
+[[!comment format=mdwn
+ username="joey"
+ subject="""comment 1"""
+ date="2014-10-31T20:21:52Z"
+ content="""
+Well, let's see.. You've gone into the Configuration and picked "Unused
+files". The full text in front of you is:
+
+
    +Managing unused files
+
+Old versions of files and deleted files can be preserved inside this repository.
+
+This might be useful, if you ever need to access those old or deleted files. But they'll also use up disk space. There are three ways to deal with this
+
+1. Set up a backup or archive repository, on a removable drive or in the cloud, and the unused files will be moved to it, freeing up space. 
+   [Add a new repository]
+2. Or, you can let unused files expire after a period of time. 
+   [Disable expiry / Enable expiry] after [7] days. [Save changes]
+
    
+
+So yes, if I read only item #2, and possibly skip the first sentence of it
+and look at only the "Disable expiry / Enable expiry",
+it seems a litte confusing, possibly, but if I read the 2 paragraphs
+of explaination above it, not so much?
+
+What do you find confusing about this form as a whole?
+"""]]
diff --git a/doc/forum/What_does_a___91__RepoName__93___means__63__.mdwn b/doc/forum/What_does_a___91__RepoName__93___means__63__.mdwn
new file mode 100644
index 0000000000..fa1841f69f
--- /dev/null
+++ b/doc/forum/What_does_a___91__RepoName__93___means__63__.mdwn
@@ -0,0 +1,5 @@
+Hello,
+
+sometimes, ```git annex info``` shows a (special remote) repository's name in [Brackets], sometimes not. Every theory (depending on enabled / not enabled) I had so far, is disproved by my next step. What does it means when the name is in brackets?
+
+Thanks!
diff --git a/doc/forum/What_does_a___91__RepoName__93___means__63__/comment_1_9b1cb4beea0f67d7e7375e7ec189b802._comment b/doc/forum/What_does_a___91__RepoName__93___means__63__/comment_1_9b1cb4beea0f67d7e7375e7ec189b802._comment
new file mode 100644
index 0000000000..c8dc7ce906
--- /dev/null
+++ b/doc/forum/What_does_a___91__RepoName__93___means__63__/comment_1_9b1cb4beea0f67d7e7375e7ec189b802._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="joey"
+ subject="""comment 1"""
+ date="2017-12-05T16:44:45Z"
+ content="""
+The name in brackets is the name of the git remote, if any, corresponding
+to the repository.
+"""]]
diff --git a/doc/forum/What_does_a___91__RepoName__93___means__63__/comment_2_a233086f100fec27cd26f4d39c25a385._comment b/doc/forum/What_does_a___91__RepoName__93___means__63__/comment_2_a233086f100fec27cd26f4d39c25a385._comment
new file mode 100644
index 0000000000..cac61f40d8
--- /dev/null
+++ b/doc/forum/What_does_a___91__RepoName__93___means__63__/comment_2_a233086f100fec27cd26f4d39c25a385._comment
@@ -0,0 +1,26 @@
+[[!comment format=mdwn
+ username="http://xgm.de/oid/"
+ nickname="Flo"
+ avatar="http://cdn.libravatar.org/avatar/4c5c0e290374d76c713f482e41f60a3cbee0fa64bb94c6da94e5a61a50824811"
+ subject="comment 2"
+ date="2017-12-06T08:39:10Z"
+ content="""
+Sure, that I know. But what does it mean for a S3 type special remote? 
+
+
+    repository mode: indirect
+    trusted repositories: 0
+    semitrusted repositories: 6
+           00000000-0000-0000-0000-000000000001 -- web
+           00000000-0000-0000-0000-000000000002 -- bittorrent
+           4cc98291-9ee9-4331-b089-b57d3d03ab0e -- Horus [horus]
+           8c201bd1-9089-4a4c-a795-7b5def64e470 -- [S3]
+           a2d60179-3e1e-49d9-90bf-f98b8a77590d -- Asaru [local]
+           ebb96a27-c997-46b5-b896-bb4f313f6e76 -- External_2TB [here]
+    untrusted repositories: 0
+    transfers in progress: none
+    available local disk space: 536.31 gigabytes (+1 megabyte reserved)
+
+I have seen the S3 remote without brackets at some repo, with at others.
+
+"""]]
diff --git a/doc/forum/What_does_a___91__RepoName__93___means__63__/comment_3_4b62bf0c03fed86270a7873a8835dc7f._comment b/doc/forum/What_does_a___91__RepoName__93___means__63__/comment_3_4b62bf0c03fed86270a7873a8835dc7f._comment
new file mode 100644
index 0000000000..b642d060f9
--- /dev/null
+++ b/doc/forum/What_does_a___91__RepoName__93___means__63__/comment_3_4b62bf0c03fed86270a7873a8835dc7f._comment
@@ -0,0 +1,12 @@
+[[!comment format=mdwn
+ username="joey"
+ subject="""comment 3"""
+ date="2017-12-08T19:36:59Z"
+ content="""
+The "[S3]" means that the special remote's name is "S3".
+So, you can run `git annex copy --to S3` or something like that and it will
+use that special remote.
+
+This particular remote has no description set, which is why it only
+displays the name.
+"""]]
diff --git a/doc/forum/What_happened_to_the_walkthrough__63__.mdwn b/doc/forum/What_happened_to_the_walkthrough__63__.mdwn
new file mode 100644
index 0000000000..e8098d29a1
--- /dev/null
+++ b/doc/forum/What_happened_to_the_walkthrough__63__.mdwn
@@ -0,0 +1 @@
+As of right now (2012-05-24 at 18:00 UTC), the [[Walkthrough]] page is basically empty. Its entire contents are "A walkthrough of the basic features of git-annex." No links (other than the autogenerated "what links to this page" list at the bottom) and no contents. Any idea what happened?
diff --git a/doc/forum/What_happened_to_the_walkthrough__63__/comment_1_70db0e3cfb1318e95671c23726e5541d._comment b/doc/forum/What_happened_to_the_walkthrough__63__/comment_1_70db0e3cfb1318e95671c23726e5541d._comment
new file mode 100644
index 0000000000..cbf852dbfb
--- /dev/null
+++ b/doc/forum/What_happened_to_the_walkthrough__63__/comment_1_70db0e3cfb1318e95671c23726e5541d._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="https://www.google.com/accounts/o8/id?id=AItOawl9sYlePmv1xK-VvjBdN-5doOa_Xw-jH4U"
+ nickname="Richard"
+ subject="comment 1"
+ date="2012-05-24T19:18:55Z"
+ content="""
+It seems the pages that are supposed to be inlined are not being found even though they are in `doc/walkthrough/`.
+"""]]
diff --git a/doc/forum/What_happened_to_the_walkthrough__63__/comment_2_f9305dd19b9b5f35e66d915b8c30374b._comment b/doc/forum/What_happened_to_the_walkthrough__63__/comment_2_f9305dd19b9b5f35e66d915b8c30374b._comment
new file mode 100644
index 0000000000..9720adfbc6
--- /dev/null
+++ b/doc/forum/What_happened_to_the_walkthrough__63__/comment_2_f9305dd19b9b5f35e66d915b8c30374b._comment
@@ -0,0 +1,7 @@
+[[!comment format=mdwn
+ username="http://joeyh.name/"
+ subject="comment 2"
+ date="2012-05-24T20:15:19Z"
+ content="""
+Broken last night during upgrade, fixed now, thanks for noticing.
+"""]]
diff --git a/doc/forum/What_happens_after_terminated_add_of_huge_picture_folder__63__.mdwn b/doc/forum/What_happens_after_terminated_add_of_huge_picture_folder__63__.mdwn
new file mode 100644
index 0000000000..8bcc1dc781
--- /dev/null
+++ b/doc/forum/What_happens_after_terminated_add_of_huge_picture_folder__63__.mdwn
@@ -0,0 +1,77 @@
+Step by step:
+
+git annex add ./hugePictureFolder
    
+// no it's to big and taking to long, let's not do this
    
+CRTL+D
    
+git annex --force drop ./hugePictureFolder
    
+git status
    
+fatal: bad default revision 'HEAD'
    
+git reset --hard git-annex
    
+git status // ok
    
+ls 
    
+drwxr-xr-x 10 rolas rolas      4096 Vas 16 11:40 000
    
+drwxr-xr-x  9 rolas rolas      4096 Vas 16 11:40 001
    
+drwxr-xr-x 11 rolas rolas      4096 Vas 16 11:40 002
    
+drwxr-xr-x 12 rolas rolas      4096 Vas 16 11:40 003
    
+drwxr-xr-x  6 rolas rolas      4096 Vas 16 11:40 004
    
+drwxr-xr-x 12 rolas rolas      4096 Vas 16 11:40 005
    
+drwxr-xr-x 11 rolas rolas      4096 Vas 16 11:40 006
    
+drwxr-xr-x 13 rolas rolas      4096 Vas 16 11:40 007
    
+drwxr-xr-x  6 rolas rolas      4096 Vas 16 11:40 008
    
+drwxr-xr-x 13 rolas rolas      4096 Vas 16 11:40 009
    
+drwxr-xr-x 14 rolas rolas      4096 Vas 16 11:40 00a
    
+drwxr-xr-x 16 rolas rolas      4096 Vas 16 11:40 00b
    
+drwxr-xr-x 11 rolas rolas      4096 Vas 16 11:40 00c
    
+drwxr-xr-x  9 rolas rolas      4096 Vas 16 11:40 00d
    
+drwxr-xr-x 20 rolas rolas      4096 Vas 16 11:40 00e
    
+drwxr-xr-x 18 rolas rolas      4096 Vas 16 11:40 00f
    
+drwxr-xr-x 14 rolas rolas      4096 Vas 16 11:40 010
    
+drwxr-xr-x 11 rolas rolas      4096 Vas 16 11:40 011
    
+drwxr-xr-x 10 rolas rolas      4096 Vas 16 11:40 012
    
+drwxr-xr-x 12 rolas rolas      4096 Vas 16 11:40 013
    
+drwxr-xr-x  7 rolas rolas      4096 Vas 16 11:40 014
    
+drwxr-xr-x 16 rolas rolas      4096 Vas 16 11:40 015
    
+drwxr-xr-x  7 rolas rolas      4096 Vas 16 11:40 016
    
+drwxr-xr-x 10 rolas rolas      4096 Vas 16 11:40 017
    
+drwxr-xr-x  9 rolas rolas      4096 Vas 16 11:40 018
    
+drwxr-xr-x 10 rolas rolas      4096 Vas 16 11:40 019
    
+drwxr-xr-x  8 rolas rolas      4096 Vas 16 11:40 01a
    
+drwxr-xr-x 10 rolas rolas      4096 Vas 16 11:40 01b
    
+drwxr-xr-x 10 rolas rolas      4096 Vas 16 11:40 01c
    
+drwxr-xr-x  8 rolas rolas      4096 Vas 16 11:40 01d
    
+drwxr-xr-x 10 rolas rolas      4096 Vas 16 11:40 01e
    
+drwxr-xr-x 11 rolas rolas      4096 Vas 16 11:40 01f
    
+drwxr-xr-x 15 rolas rolas      4096 Vas 16 11:40 020
    
+drwxr-xr-x 13 rolas rolas      4096 Vas 16 11:40 021
    
+drwxr-xr-x  5 rolas rolas      4096 Vas 16 11:40 022
    
+drwxr-xr-x 10 rolas rolas      4096 Vas 16 11:40 023
    
+drwxr-xr-x  9 rolas rolas      4096 Vas 16 11:40 024
    
+drwxr-xr-x 12 rolas rolas      4096 Vas 16 11:40 025
    
+drwxr-xr-x  8 rolas rolas      4096 Vas 16 11:40 026
    
+drwxr-xr-x 12 rolas rolas      4096 Vas 16 11:40 027
    
+drwxr-xr-x 10 rolas rolas      4096 Vas 16 11:40 028
    
+drwxr-xr-x 10 rolas rolas      4096 Vas 16 11:40 029
    
+drwxr-xr-x  9 rolas rolas      4096 Vas 16 11:40 02a
    
+drwxr-xr-x  9 rolas rolas      4096 Vas 16 11:40 02b
    
+drwxr-xr-x  6 rolas rolas      4096 Vas 16 11:40 02c
    
+drwxr-xr-x  7 rolas rolas      4096 Vas 16 11:40 02d
    
+drwxr-xr-x 10 rolas rolas      4096 Vas 16 11:40 02e
    
+drwxr-xr-x  5 rolas rolas      4096 Vas 16 11:40 02f
    
+drwxr-xr-x 13 rolas rolas      4096 Vas 16 11:40 030
    
+drwxr-xr-x 10 rolas rolas      4096 Vas 16 11:40 031
    
+drwxr-xr-x 10 rolas rolas      4096 Vas 16 11:40 032
    
+...
    
+...
    
+
    
+What did I Do? Can I do CTRL+D? If yes, what should I do to recover?
    
+
    
+Thanks
    
+Rolandas
    
+ 
    
+$ git --version
    
+git version 2.3.0
    
+
    
+$ git annex version
    
+git-annex version: 5.20140412ubuntu1
    
+
    
+
diff --git a/doc/forum/What_happens_after_terminated_add_of_huge_picture_folder__63__/comment_1_0ba60f9625ccda45d59adbd385f5fe98._comment b/doc/forum/What_happens_after_terminated_add_of_huge_picture_folder__63__/comment_1_0ba60f9625ccda45d59adbd385f5fe98._comment
new file mode 100644
index 0000000000..931bab540a
--- /dev/null
+++ b/doc/forum/What_happens_after_terminated_add_of_huge_picture_folder__63__/comment_1_0ba60f9625ccda45d59adbd385f5fe98._comment
@@ -0,0 +1,12 @@
+[[!comment format=mdwn
+ username="https://www.google.com/accounts/o8/id?id=AItOawnPgn611P6ym5yyL0BS8rUzO0_ZKRldMt0"
+ nickname="Samuel"
+ subject="Reseting to the git-annex branch"
+ date="2015-02-17T09:21:12Z"
+ content="""
+Well, it appears you explicitely asked for reseting to the git-annex branch with the following command
+  git annex reset --hard git-annex
+To go back to the master branch, containing the symlinks, just do
+  git annex checkout master
+
+"""]]
diff --git a/doc/forum/What_happens_after_terminated_add_of_huge_picture_folder__63__/comment_3_f0a2ec3d0b05d84fb05a1708313a6762._comment b/doc/forum/What_happens_after_terminated_add_of_huge_picture_folder__63__/comment_3_f0a2ec3d0b05d84fb05a1708313a6762._comment
new file mode 100644
index 0000000000..d06b509651
--- /dev/null
+++ b/doc/forum/What_happens_after_terminated_add_of_huge_picture_folder__63__/comment_3_f0a2ec3d0b05d84fb05a1708313a6762._comment
@@ -0,0 +1,12 @@
+[[!comment format=mdwn
+ username="https://www.google.com/accounts/o8/id?id=AItOawldTFGlKsKUyq6F6CJ22CNBnmYJ0LQbLUQ"
+ nickname="Rolandas"
+ subject="comment 3"
+ date="2015-02-18T06:58:59Z"
+ content="""
+Jes, it was an entirely new git repo and I removed .git folder and start over.
    
+The question was is there some sort of „UNDO“ in this case.
    
+I get it now.
    
+I will read internals documentation.
    
+Thank You very much!
    
+"""]]
diff --git a/doc/forum/What_happens_after_terminated_add_of_huge_picture_folder__63__/comment_3_f628f146a0c652f812c09f78bd574b77._comment b/doc/forum/What_happens_after_terminated_add_of_huge_picture_folder__63__/comment_3_f628f146a0c652f812c09f78bd574b77._comment
new file mode 100644
index 0000000000..b5814c7c7c
--- /dev/null
+++ b/doc/forum/What_happens_after_terminated_add_of_huge_picture_folder__63__/comment_3_f628f146a0c652f812c09f78bd574b77._comment
@@ -0,0 +1,24 @@
+[[!comment format=mdwn
+ username="joey"
+ subject="""comment 2"""
+ date="2015-02-17T21:31:42Z"
+ content="""
+There is never a reason to run "git reset --hard git-annex"! For that matter,
+don't mess with the git-annex branch if you have not read and understand
+the [[internals]] documentation. Even if you have, it's entirely the wrong
+thing to be messing with in this situation. It has nothing at all to do
+with your problem, except that after running that completely random reset
+command, you now have two problems..
+
+The right answer to your interrupted add is something like:
+
+* `git reset --hard master`
+* Or, run the `git-annex add` command again and let it resume
+* Or, run `git commit` to commit any changes the add made,
+  followed by `git annex unannex` to back out adding those files.
+
+Or, if this is an entirely new git repo that you have
+never committed to before 
+(my guess based on the "bad default revision 'HEAD'"),
+just `rm -rf .git` and start over.
+"""]]
diff --git a/doc/forum/What_happens_when_in_the_git-annex_assistant_you___34__Add_another_local_repository__34___on_an_existing_repository__63__.mdwn b/doc/forum/What_happens_when_in_the_git-annex_assistant_you___34__Add_another_local_repository__34___on_an_existing_repository__63__.mdwn
new file mode 100644
index 0000000000..870f8913eb
--- /dev/null
+++ b/doc/forum/What_happens_when_in_the_git-annex_assistant_you___34__Add_another_local_repository__34___on_an_existing_repository__63__.mdwn
@@ -0,0 +1 @@
+I want to add an existing repository using the assistant. Is that the way?
diff --git a/doc/forum/What_happens_when_in_the_git-annex_assistant_you___34__Add_another_local_repository__34___on_an_existing_repository__63__/comment_1_d844cfe5f9907a766e871b64d68966c2._comment b/doc/forum/What_happens_when_in_the_git-annex_assistant_you___34__Add_another_local_repository__34___on_an_existing_repository__63__/comment_1_d844cfe5f9907a766e871b64d68966c2._comment
new file mode 100644
index 0000000000..0c70448895
--- /dev/null
+++ b/doc/forum/What_happens_when_in_the_git-annex_assistant_you___34__Add_another_local_repository__34___on_an_existing_repository__63__/comment_1_d844cfe5f9907a766e871b64d68966c2._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="http://joeyh.name/"
+ ip="71.80.94.56"
+ subject="comment 1"
+ date="2014-02-07T19:03:30Z"
+ content="""
+Sure, you can use that menu item. If you try it out and enter in the path of a repository, you'll see that it offers to either let you combine the two repositories together, or leave them as two unconnected repositories with different files managed by the assistant.
+"""]]
diff --git a/doc/forum/What_is_the_assistant_up_to__63__.mdwn b/doc/forum/What_is_the_assistant_up_to__63__.mdwn
new file mode 100644
index 0000000000..623268f7e4
--- /dev/null
+++ b/doc/forum/What_is_the_assistant_up_to__63__.mdwn
@@ -0,0 +1,5 @@
+Is there a way to see what the assistant is doing right now, what failed, etc. ?
+I am running the assistant on a remote server so the Webapp's interface is not easily available.
+
+I was hoping to have something that is easier to read than the daemon.log
+
diff --git a/doc/forum/What_is_the_assistant_up_to__63__/comment_1_9baa0e54c19105c7cce946c19c587866._comment b/doc/forum/What_is_the_assistant_up_to__63__/comment_1_9baa0e54c19105c7cce946c19c587866._comment
new file mode 100644
index 0000000000..f007f5d301
--- /dev/null
+++ b/doc/forum/What_is_the_assistant_up_to__63__/comment_1_9baa0e54c19105c7cce946c19c587866._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="marekj"
+ avatar="http://cdn.libravatar.org/avatar/65a60e8f5183feeeef8cef815bf73e61"
+ subject="git annex info"
+ date="2016-12-21T12:58:37Z"
+ content="""
+I found that git annex info provides information on current transfers. Using -F it provides what I want.
+"""]]
diff --git a/doc/forum/What_is_the_best_way_to___34__git_annex_mv__34___file__63__.mdwn b/doc/forum/What_is_the_best_way_to___34__git_annex_mv__34___file__63__.mdwn
new file mode 100644
index 0000000000..6dd5c16df3
--- /dev/null
+++ b/doc/forum/What_is_the_best_way_to___34__git_annex_mv__34___file__63__.mdwn
@@ -0,0 +1 @@
+Since all the annexed (in indirect mode) files are symlinks to topdir/.git/annex/...   moving files among directories at different levels is not that straightforward since symlinks would get broken.  And since there is not 'annex mv' command -- what is the best way?  (unlock is not the resolution since it copies the file, which might be prohibitively large and inefficient)
diff --git a/doc/forum/What_is_the_best_way_to___34__git_annex_mv__34___file__63__/comment_1_02d305f307b4d2ff7acd98cb36508a2f._comment b/doc/forum/What_is_the_best_way_to___34__git_annex_mv__34___file__63__/comment_1_02d305f307b4d2ff7acd98cb36508a2f._comment
new file mode 100644
index 0000000000..9eae8ef7ab
--- /dev/null
+++ b/doc/forum/What_is_the_best_way_to___34__git_annex_mv__34___file__63__/comment_1_02d305f307b4d2ff7acd98cb36508a2f._comment
@@ -0,0 +1,10 @@
+[[!comment format=mdwn
+ username="http://joeyh.name/"
+ nickname="joey"
+ subject="comment 1"
+ date="2013-06-12T16:42:54Z"
+ content="""
+You can move the symlinks however you like (git mv, or regular mv and git add).
+
+To fix up broken symlinks, you can either run `git annex fix`, or just commit the move. The pre-commit hook fixes up links automatically.
+"""]]
diff --git a/doc/forum/What_is_the_difference_between___34__local_computer__34___and___34__remote_server__34__.mdwn b/doc/forum/What_is_the_difference_between___34__local_computer__34___and___34__remote_server__34__.mdwn
new file mode 100644
index 0000000000..7222e07b58
--- /dev/null
+++ b/doc/forum/What_is_the_difference_between___34__local_computer__34___and___34__remote_server__34__.mdwn
@@ -0,0 +1,3 @@
+What is the difference between "local computer" and "remote server" in the assistant's repository configuration menu? How does git-annex treat the targets differently? 
+
+
diff --git a/doc/forum/What_is_the_difference_between___34__local_computer__34___and___34__remote_server__34__/comment_1_68734a118b7dc0c88ba67eca20953a55._comment b/doc/forum/What_is_the_difference_between___34__local_computer__34___and___34__remote_server__34__/comment_1_68734a118b7dc0c88ba67eca20953a55._comment
new file mode 100644
index 0000000000..167d008f9a
--- /dev/null
+++ b/doc/forum/What_is_the_difference_between___34__local_computer__34___and___34__remote_server__34__/comment_1_68734a118b7dc0c88ba67eca20953a55._comment
@@ -0,0 +1,10 @@
+[[!comment format=mdwn
+ username="http://joeyh.name/"
+ ip="2001:4978:f:21a::2"
+ subject="comment 1"
+ date="2013-01-25T22:39:20Z"
+ content="""
+Local computer pairs with another computer on your LAN. You don't need an account on that computer, it can belong to someone else and has to be running the webapp as well for them to verify the pairing. 
+
+Remote server is for people with a ssh server out there, and does not require it be running the assistant, or even have git-annex installed. (Though it works better if at least git-annex is installed on it.)
+"""]]
diff --git a/doc/forum/What_operations_are_safe_to_interrupt__63__.mdwn b/doc/forum/What_operations_are_safe_to_interrupt__63__.mdwn
new file mode 100644
index 0000000000..ea89f8e277
--- /dev/null
+++ b/doc/forum/What_operations_are_safe_to_interrupt__63__.mdwn
@@ -0,0 +1,34 @@
+# Context
+
+* Git annex does a lot under the hood: replace files with symlinks, check and adjust moved links, etc.
+* Git annex allows to mix and match with plain git commands.
+* Operations may be interrupted by signal (Ctrl-C), by network failure (especially in mobile situation), by power failure (laptop on battery).
+
+Git itself is mostly transactional: most actions can be interrupted and everything looks like they were never started.
+
+I'm confident that you Joey have an eye for details and do the safe thing.  Yet parameters above open up for a lot of combinations, some of which might not actually do what's intended, or are perhaps safe only for someone who knows where *not* to put a foot.
+
+# Example
+
+For example, nearly 5 years ago you wrote about interrupted `git annex add`: in [git annex add crash and subsequent recovery](https://git-annex.branchable.com/forum/git_annex_add_crash_and_subsequent_recovery/) 
+
+> Thought I'd mention how to clean up from interrupting git annex add. When you do that, it doesn't get a chance to git add the files it's added (this is normally done at the end, or sometimes at points in the middle when you're adding a lot of files). Which is also why fsck, whereis, and unannex wouldn't operate on them, since they only deal with files in git.
+> So the first step is to manually use git add on any symlinks.
+>  I've made git annex add recover when ran a second time. 
+
+# Questions
+
+* Which operations are safe to interrupt? What will be the result? No change? Partial but valid result? Messy result? With what consequences?
+* What operations should *not* be performed after an interrupted one?
+
+For example, let's say I've started a `git annex sync`. While it works, I realize there are unwanted changes. If I interrupt what is the result? Can I safely use manual git commands to add or remove some files to index, make separated commits, then git sync again?
+
+I'm using a v5 repository, indirect mode, mixed content. Is it different in v6? In direct mode? In mixed vs non-mixed content repository?
+
+# Wish a dedicated page in git-annex documentation
+
+IMHO this is an important, difficult and changing topic, which would deserve its own "when things go wrong" topic pages, like [fsck: when things go wrong](https://git-annex.branchable.com/walkthrough/fsck__58___when_things_go_wrong/) and [transferring files: When things go wrong](https://git-annex.branchable.com/walkthrough/transferring_files__58___When_things_go_wrong/).
+
+Perhaps just starting with broad rules like "Commands foo and bar are ok, baz is mostly ok just repeat it" or "in any on-trivial case, do `git annex sync` last after you check twice that any necessary commit is done" would be very appreciated.
+
+Thank you.
diff --git a/doc/forum/What_operations_are_safe_to_interrupt__63__/comment_1_03d8ddc049cf073e5579ca829b352367._comment b/doc/forum/What_operations_are_safe_to_interrupt__63__/comment_1_03d8ddc049cf073e5579ca829b352367._comment
new file mode 100644
index 0000000000..432db2626c
--- /dev/null
+++ b/doc/forum/What_operations_are_safe_to_interrupt__63__/comment_1_03d8ddc049cf073e5579ca829b352367._comment
@@ -0,0 +1,24 @@
+[[!comment format=mdwn
+ username="joey"
+ subject="""comment 1"""
+ date="2016-11-07T14:43:46Z"
+ content="""
+Note that git is not entirely robust against being interrupted.
+In particular, interrupting a commit can leave a stale lock file in place,
+and committing will then fail until the user manually goes in and removes
+the lock file.
+
+I'd say that git-annex is overall more robust against interruptions
+than git is, except that git-annex uses git so of course inherits 
+its limitations.
+
+git-annex tries to handle all interruptions sanely. Ie, files are always
+moved into place atomically so partial writes are not a problem, stale lock
+files are not a problem, and any recordkeeping that might be lost by an
+interruption should be able to be recovered by running the interrupted
+command again, or perhaps `git annex fsck`.
+
+The only problem that I can think of with interuptions is that
+since git-annex uses temp files in `.git/annex/tmp` extensively, 
+if it's interrupted nothing will clean up those temp files.
+"""]]
diff --git a/doc/forum/What_to_do_if_special_remotes_refuses_drops__63__.mdwn b/doc/forum/What_to_do_if_special_remotes_refuses_drops__63__.mdwn
new file mode 100644
index 0000000000..512e89528c
--- /dev/null
+++ b/doc/forum/What_to_do_if_special_remotes_refuses_drops__63__.mdwn
@@ -0,0 +1,9 @@
+I have a special remote that I would like to delete and have marked it as such in the assistant. Although this was before my myriad of problems with git annex itself wanting to repair the repo all the time. Right now if I take a loog into my daemon.log I see the following error over and over again:
+
+```
+drop skydrive foo.bar 
+  This file could not be removed
+failed
+```
+
+I checked if I can login into my account and it works just fine. So I assume that this might be a bug? Is it somehow possible to forego the cleaning out of the special remote and just mark it as deleted for good? Thanks in advance!
diff --git a/doc/forum/What_to_do_if_special_remotes_refuses_drops__63__/comment_1_0b523b2b6c361346c36ad456bbbac645._comment b/doc/forum/What_to_do_if_special_remotes_refuses_drops__63__/comment_1_0b523b2b6c361346c36ad456bbbac645._comment
new file mode 100644
index 0000000000..cf808aac4b
--- /dev/null
+++ b/doc/forum/What_to_do_if_special_remotes_refuses_drops__63__/comment_1_0b523b2b6c361346c36ad456bbbac645._comment
@@ -0,0 +1,15 @@
+[[!comment format=mdwn
+ username="joey"
+ subject="""comment 1"""
+ date="2016-11-18T15:39:20Z"
+ content="""
+It could certianly be a bug in the special remote implementation. It's also
+possible for some special remotes to intentionally not be able to remove
+content (this is the case with the web special remote, and the bup special
+remote at least).
+
+You can manually remove the special remote, by editing .git/config and
+deleting the stanza for that remote. You may want to run `git annex dead
+$remotename` first, if you don't intend to ever use that special remote
+again.
+"""]]
diff --git a/doc/forum/What_to_do_when_computer_is_lost_and_only_an_encrypted_ssh_remote_is_left__63__.mdwn b/doc/forum/What_to_do_when_computer_is_lost_and_only_an_encrypted_ssh_remote_is_left__63__.mdwn
new file mode 100644
index 0000000000..1504d10cd6
--- /dev/null
+++ b/doc/forum/What_to_do_when_computer_is_lost_and_only_an_encrypted_ssh_remote_is_left__63__.mdwn
@@ -0,0 +1 @@
+What do I do when I have set up git-annex to back up a folder to an encrypted remote on a vps, and my laptop was stolen? This recently happened, and luckally I had another backup, but what would I do in the case that I have 1 computer using git-annex, backung ip a folder to an encrypted ssh remote on a vps, and I loose the computer? How do I get the data back then?
diff --git a/doc/forum/What_to_do_when_computer_is_lost_and_only_an_encrypted_ssh_remote_is_left__63__/comment_1_67ee446ca6d66e2c259ea771c2c9a2b2._comment b/doc/forum/What_to_do_when_computer_is_lost_and_only_an_encrypted_ssh_remote_is_left__63__/comment_1_67ee446ca6d66e2c259ea771c2c9a2b2._comment
new file mode 100644
index 0000000000..832c931b42
--- /dev/null
+++ b/doc/forum/What_to_do_when_computer_is_lost_and_only_an_encrypted_ssh_remote_is_left__63__/comment_1_67ee446ca6d66e2c259ea771c2c9a2b2._comment
@@ -0,0 +1,12 @@
+[[!comment format=mdwn
+ username="http://joeyh.name/"
+ ip="2001:4978:f:21a::2"
+ subject="comment 1"
+ date="2013-07-26T03:57:06Z"
+ content="""
+I'm glad you had another backup. That is wise.
+
+When you have a special remote, that does not store your git repository, but only the contents of files. So if you lose all copies of your git repositories, you have lost all record of the file name associated with a file. It's the same as if `fsck` rescued inodes to `lost+found`; you'd have to manually rename the files to new names.
+
+Then there's the added complication of the encryption. This depends on how it was set up. If you set up the remote using a GPG key (and didn't lose the key too), you can manually use that key to decrypt the files. On the other hand, if it generated its own key and stored it in the git repo (default with the assistant), then when you lose all copies of the git repo, the key is gone, and your encrypted files are useless.
+"""]]
diff --git a/doc/forum/What_to_do_when_computer_is_lost_and_only_an_encrypted_ssh_remote_is_left__63__/comment_2_6d3cce3c8048e4aea8f0ed76473f6af1._comment b/doc/forum/What_to_do_when_computer_is_lost_and_only_an_encrypted_ssh_remote_is_left__63__/comment_2_6d3cce3c8048e4aea8f0ed76473f6af1._comment
new file mode 100644
index 0000000000..308df9a182
--- /dev/null
+++ b/doc/forum/What_to_do_when_computer_is_lost_and_only_an_encrypted_ssh_remote_is_left__63__/comment_2_6d3cce3c8048e4aea8f0ed76473f6af1._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="https://www.google.com/accounts/o8/id?id=AItOawln3ckqKx0x_xDZMYwa9Q1bn4I06oWjkog"
+ nickname="Michael"
+ subject="comment 2"
+ date="2013-07-26T15:04:31Z"
+ content="""
+I thought the key was checked into .git in any case, but encrypted in the case of GPG.  So .git gone => everything gone in any case.
+"""]]
diff --git a/doc/forum/What_to_do_when_computer_is_lost_and_only_an_encrypted_ssh_remote_is_left__63__/comment_3_bd506e1ca7307660b3b9769eb97beddb._comment b/doc/forum/What_to_do_when_computer_is_lost_and_only_an_encrypted_ssh_remote_is_left__63__/comment_3_bd506e1ca7307660b3b9769eb97beddb._comment
new file mode 100644
index 0000000000..4591a559e0
--- /dev/null
+++ b/doc/forum/What_to_do_when_computer_is_lost_and_only_an_encrypted_ssh_remote_is_left__63__/comment_3_bd506e1ca7307660b3b9769eb97beddb._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="http://joeyh.name/"
+ ip="2001:4978:f:21a::2"
+ subject="comment 3"
+ date="2013-07-26T16:45:26Z"
+ content="""
+Michael, you're right. Thanks for fact checking my late night comment! ;)
+"""]]
diff --git a/doc/forum/Where_are_my_remote_ssh_files__63__.mdwn b/doc/forum/Where_are_my_remote_ssh_files__63__.mdwn
new file mode 100644
index 0000000000..704f95ff09
--- /dev/null
+++ b/doc/forum/Where_are_my_remote_ssh_files__63__.mdwn
@@ -0,0 +1,10 @@
+
+
+I managed to sync files to a remote ssh store "bible" with `git annex sync --content` however, where I ssh to bible, I was surprised not to see any of the JPG files that were copied there.
+
+
+What am I missing?
+
+# Solution
+
+I need to run `git annex sync` on the host bible too!
diff --git a/doc/forum/Where_did_my_files_go__63__.mdwn b/doc/forum/Where_did_my_files_go__63__.mdwn
new file mode 100644
index 0000000000..f7d811d8fa
--- /dev/null
+++ b/doc/forum/Where_did_my_files_go__63__.mdwn
@@ -0,0 +1,20 @@
+I import some files that I've seen before somewhere:
+
+    $ git annex import --deduplicate .../download
+    ...
+    import download/What_is_The_Digital_Fiction_Factory-Conker_Media.pdf (duplicate) ok
+    ...
+
+But the resulting download directory is empty, and `list` doesn't show any of the files:
+
+    $ git annex list --allrepos What_is_The_Digital_Fiction_Factory-Conker_Media.pdf
+    here
+    |...
+    |...
+    |...
+    |...
+    |...
+    |...
+    git-annex: What_is_The_Digital_Fiction_Factory-Conker_Media.pdf not found
+
+How do I find out where this file can be found?
diff --git a/doc/forum/Where_did_my_files_go__63__/comment_1_3ff3ffa95eb2745ff9ec2a903e071d97._comment b/doc/forum/Where_did_my_files_go__63__/comment_1_3ff3ffa95eb2745ff9ec2a903e071d97._comment
new file mode 100644
index 0000000000..f0b2ae3e7e
--- /dev/null
+++ b/doc/forum/Where_did_my_files_go__63__/comment_1_3ff3ffa95eb2745ff9ec2a903e071d97._comment
@@ -0,0 +1,9 @@
+[[!comment format=mdwn
+ username="https://www.google.com/accounts/o8/id?id=AItOawnVnsqEy82M-MuS2gLri-az83wSQ6lXSrc"
+ nickname="Jean"
+ subject="comment 1"
+ date="2015-05-06T09:21:19Z"
+ content="""
+I did a dump of all known files: `$ git annex whereis > ../git-annex-whereis`
+No sign of the imported file. 
+"""]]
diff --git a/doc/forum/Where_did_my_files_go__63__/comment_2_9d902e66ca19b3332f4454f694d4a12e._comment b/doc/forum/Where_did_my_files_go__63__/comment_2_9d902e66ca19b3332f4454f694d4a12e._comment
new file mode 100644
index 0000000000..b1b186ab98
--- /dev/null
+++ b/doc/forum/Where_did_my_files_go__63__/comment_2_9d902e66ca19b3332f4454f694d4a12e._comment
@@ -0,0 +1,16 @@
+[[!comment format=mdwn
+ username="joey"
+ subject="""comment 2"""
+ date="2015-05-06T18:04:35Z"
+ content="""
+I don't think you're using a recent version of git-annex, the "(duplicate)"
+message has changed to something else a while ago.
+
+Also, that message is only shown if you pass the --deduplicate flag, which
+you don't show in your transcript.
+
+That flag does what it says on the tin: If the file is a duplicate of one
+git-annex has seen before, the file is deleted from the import location.
+(The next version of git-annex does add an additional check that the
+content of the file is present in the annex.)
+"""]]
diff --git a/doc/forum/Where_is_the_content__63__.mdwn b/doc/forum/Where_is_the_content__63__.mdwn
new file mode 100644
index 0000000000..0d79bbb093
--- /dev/null
+++ b/doc/forum/Where_is_the_content__63__.mdwn
@@ -0,0 +1,8 @@
+On Android 4.4.4, I can install the most recent git annex. I want to join a git annex running on my linux server. Git annex on android seems to be able to contact remote, and it asks if I want to 'merge' with it. I hope that did not mean erase the remote.  Anyway, the sync says it is proceeding, but when it is done, I cannot find a folder with remote copies of files. 
+
+I figure either it worked, or it destroyed my remote. 
+
+After new install, I have folders /sdcard/annex and /sdcard/git-annex.home. I don't find a new folder with name of remote repo, don't understand why.
+
+Help?
+
diff --git a/doc/forum/Where_is_the_content__63__/comment_1_812e1cf740cbfa449ab3ef4dd5f2df08._comment b/doc/forum/Where_is_the_content__63__/comment_1_812e1cf740cbfa449ab3ef4dd5f2df08._comment
new file mode 100644
index 0000000000..423bd82d28
--- /dev/null
+++ b/doc/forum/Where_is_the_content__63__/comment_1_812e1cf740cbfa449ab3ef4dd5f2df08._comment
@@ -0,0 +1,132 @@
+[[!comment format=mdwn
+ username="https://www.google.com/accounts/o8/id?id=AItOawkpIIYg6Fl7OFsOHVPEchZYj68A3dk4lVg"
+ nickname="Paul"
+ subject="install log"
+ date="2014-07-15T12:02:59Z"
+ content="""
+Installation starting to /data/data/ga.androidterm
+ac90d4f57076a59cac3a8017b89ac9b5ab4e5021
+installing busybox
+installing git-annex
+installing git-shell
+installing git-upload-pack
+installing git
+installing gpg
+installing rsync
+installing ssh
+installing ssh-keygen
+linking ./libexec/git-core/git-show to git
+linking ./libexec/git-core/git-cherry-pick to git
+linking ./libexec/git-core/git-revert to git
+linking ./libexec/git-core/git-rerere to git
+linking ./libexec/git-core/git to git
+linking ./libexec/git-core/git-cat-file to git
+linking ./libexec/git-core/git-archive to git
+linking ./libexec/git-core/git-apply to git
+linking ./libexec/git-core/git-repack to git
+linking ./libexec/git-core/git-replace to git
+linking ./libexec/git-core/git-verify-pack to git
+linking ./libexec/git-core/git-name-rev to git
+linking ./libexec/git-core/git-grep to git
+linking ./libexec/git-core/git-format-patch to git
+linking ./libexec/git-core/git-log to git
+linking ./libexec/git-core/git-read-tree to git
+linking ./libexec/git-core/git-fast-export to git
+linking ./libexec/git-core/git-mktag to git
+linking ./libexec/git-core/git-cherry to git
+linking ./libexec/git-core/git-stage to git
+linking ./libexec/git-core/git-blame to git
+linking ./libexec/git-core/git-help to git
+linking ./libexec/git-core/git-tag to git
+linking ./libexec/git-core/git-column to git
+linking ./libexec/git-core/git-clone to git
+linking ./libexec/git-core/git-pack-redundant to git
+linking ./libexec/git-core/git-rev-list to git
+linking ./libexec/git-core/git-ls-files to git
+linking ./libexec/git-core/git-update-index to git
+linking ./libexec/git-core/git-merge-tree to git
+linking ./libexec/git-core/git-shortlog to git
+linking ./libexec/git-core/git-pack-refs to git
+linking ./libexec/git-core/git-fsck to git
+linking ./libexec/git-core/git-mv to git
+linking ./libexec/git-core/git-index-pack to git
+linking ./libexec/git-core/git-upload-archive to git
+linking ./libexec/git-core/git-unpack-file to git
+linking ./libexec/git-core/git-merge-recursive to git
+linking ./libexec/git-core/git-checkout to git
+linking ./libexec/git-core/git-mailsplit to git
+linking ./libexec/git-core/git-whatchanged to git
+linking ./libexec/git-core/git-remote-fd to git
+linking ./libexec/git-core/git-fsck-objects to git
+linking ./libexec/git-core/git-diff to git
+linking ./libexec/git-core/git-diff-tree to git
+linking ./libexec/git-core/git-check-ref-format to git
+linking ./libexec/git-core/git-status to git
+linking ./libexec/git-core/git-annotate to git
+linking ./libexec/git-core/git-bisect--helper to git
+linking ./libexec/git-core/git-mktree to git
+linking ./libexec/git-core/git-check-ignore to git
+linking ./libexec/git-core/git-clean to git
+linking ./libexec/git-core/git-merge to git
+linking ./libexec/git-core/git-var to git
+linking ./libexec/git-core/git-get-tar-commit-id to git
+linking ./libexec/git-core/git-remote-ext to git
+linking ./libexec/git-core/git-stripspace to git
+linking ./libexec/git-core/git-send-pack to git
+linking ./libexec/git-core/git-fetch to git
+linking ./libexec/git-core/git-ls-remote to git
+linking ./libexec/git-core/git-init to git
+linking ./libexec/git-core/git-notes to git
+linking ./libexec/git-core/git-commit to git
+linking ./libexec/git-core/git-receive-pack to git
+linking ./libexec/git-core/git-prune to git
+linking ./libexec/git-core/git-update-server-info to git
+linking ./libexec/git-core/git-gc to git
+linking ./libexec/git-core/git-show-ref to git
+linking ./libexec/git-core/git-merge-subtree to git
+linking ./libexec/git-core/git-describe to git
+linking ./libexec/git-core/git-branch to git
+linking ./libexec/git-core/git-checkout-index to git
+linking ./libexec/git-core/git-prune-packed to git
+linking ./libexec/git-core/git-ls-tree to git
+linking ./libexec/git-core/git-hash-object to git
+linking ./libexec/git-core/git-push to git
+linking ./libexec/git-core/git-for-each-ref to git
+linking ./libexec/git-core/git-verify-tag to git
+linking ./libexec/git-core/git-diff-files to git
+linking ./libexec/git-core/git-check-attr to git
+linking ./libexec/git-core/git-mailinfo to git
+linking ./libexec/git-core/git-check-mailmap to git
+linking ./libexec/git-core/git-fetch-pack to git
+linking ./libexec/git-core/git-diff-index to git
+linking ./libexec/git-core/git-add to git
+linking ./libexec/git-core/git-config to git
+linking ./libexec/git-core/git-merge-ours to git
+linking ./libexec/git-core/git-symbolic-ref to git
+linking ./libexec/git-core/git-init-db to git
+linking ./libexec/git-core/git-pack-objects to git
+linking ./libexec/git-core/git-rm to git
+linking ./libexec/git-core/git-write-tree to git
+linking ./libexec/git-core/git-show-branch to git
+linking ./libexec/git-core/git-fmt-merge-msg to git
+linking ./libexec/git-core/git-count-objects to git
+linking ./libexec/git-core/git-reset to git
+linking ./libexec/git-core/git-reflog to git
+linking ./libexec/git-core/git-rev-parse to git
+linking ./libexec/git-core/git-update-ref to git
+linking ./libexec/git-core/git-patch-id to git
+linking ./libexec/git-core/git-commit-tree to git
+linking ./libexec/git-core/git-merge-file to git
+linking ./libexec/git-core/git-credential to git
+linking ./libexec/git-core/git-unpack-objects to git
+linking ./libexec/git-core/git-bundle to git
+linking ./libexec/git-core/git-merge-index to git
+linking ./libexec/git-core/git-remote to git
+linking ./libexec/git-core/git-merge-base to git
+linking ./bin/git-upload-archive to git
+linking ./bin/git-receive-pack to git
+linking ./libexec/git-core/git-shell to git-shell
+linking ./libexec/git-core/git-upload-pack to git-upload-pack
+Installation complete
+
+"""]]
diff --git a/doc/forum/Where_is_the_content__63__/comment_2_5e2cfdfab6c4f84fe7a19447b417b5a7._comment b/doc/forum/Where_is_the_content__63__/comment_2_5e2cfdfab6c4f84fe7a19447b417b5a7._comment
new file mode 100644
index 0000000000..45b7b024e4
--- /dev/null
+++ b/doc/forum/Where_is_the_content__63__/comment_2_5e2cfdfab6c4f84fe7a19447b417b5a7._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="http://joeyh.name/"
+ ip="209.250.56.2"
+ subject="comment 2"
+ date="2014-07-15T18:37:34Z"
+ content="""
+It'll be in `/sdcard/annex` by defualt, unless you told the webapp to put it somewhere else when it asked where to put the repository. `/sdcard/annex` is a git repository which should contain all the files from the remote (not in any separate subdirectory, that's not how git works). If it does not, you can run commands like `git log --stat` in that repository to see what files have been added/removed, or `git annex status` to check the status of the files.
+"""]]
diff --git a/doc/forum/Where_is_the_content__63__/comment_3_bd4cbc8f256a94ffde4f57d2c406a9ec._comment b/doc/forum/Where_is_the_content__63__/comment_3_bd4cbc8f256a94ffde4f57d2c406a9ec._comment
new file mode 100644
index 0000000000..5b736b4100
--- /dev/null
+++ b/doc/forum/Where_is_the_content__63__/comment_3_bd4cbc8f256a94ffde4f57d2c406a9ec._comment
@@ -0,0 +1,16 @@
+[[!comment format=mdwn
+ username="https://www.google.com/accounts/o8/id?id=AItOawkpIIYg6Fl7OFsOHVPEchZYj68A3dk4lVg"
+ nickname="Paul"
+ subject="text relocations refusal"
+ date="2014-07-16T02:19:27Z"
+ content="""
+I'm afraid I've ruined the remote repo. For directory, I wrote \"mediashare\" because that's where I want copy to go.  i don't want it in default \"annex\". In git assistant, it is unclearbwhst directory means.   Mount point for cooy of remote? 
+
+Well, this is all I get now...
+
+1|u0_a127@manta:/sdcard $ cd annex
+u0_a127@manta:/sdcard/annex $ git annex status
+WARNING: linker: git-annex has text relocations. This is wasting memory and is a security risk. Please fix.
+
+
+"""]]
diff --git a/doc/forum/Where_is_the_content__63__/comment_4_a36b35d47472b5db779b0489bf3d4893._comment b/doc/forum/Where_is_the_content__63__/comment_4_a36b35d47472b5db779b0489bf3d4893._comment
new file mode 100644
index 0000000000..69b929810e
--- /dev/null
+++ b/doc/forum/Where_is_the_content__63__/comment_4_a36b35d47472b5db779b0489bf3d4893._comment
@@ -0,0 +1,10 @@
+[[!comment format=mdwn
+ username="http://joeyh.name/"
+ ip="209.250.56.2"
+ subject="comment 4"
+ date="2014-07-16T20:41:17Z"
+ content="""
+That warning about text relocations always happens on Android and is not relevant to whatever problem you're having.
+
+If git-annex status does not print anything else, then it seems you have a git-annex repository in /sdcard/annex. If you entered \"mediashare\" in the webapp, you might have another repository in another location. When you open the webapp, it tells you the location of the repository in the upper-right corner -- \"Repository: $location\"
+"""]]
diff --git a/doc/forum/Which_branches_are___34__required__34__.mdwn b/doc/forum/Which_branches_are___34__required__34__.mdwn
new file mode 100644
index 0000000000..576d583529
--- /dev/null
+++ b/doc/forum/Which_branches_are___34__required__34__.mdwn
@@ -0,0 +1,31 @@
+Hi,
+
+I have the problem where in one repository I should not have any files anymore (according to git annex list --in .) but the annex directory still is very big (several GB). Git unused does not return any unused files so I guess they must be referenced somewhere. I do not use tags so it must be a branch. Could you tell me which of these branches are required by git annex and which are superfluous?
+
+I have two repositories (at some point I had more): origin and exp and two special remotes (hubic and acd). exp is connected to hubic and acd.
+
+on exp I have more space usage than indicated by git annex list
+ exp has the following branches:
+
+    ~/annex$ git branch -a
+      git-annex
+    * master
+      synced/git-annex
+      synced/master
+      remotes/origin/HEAD -> origin/master
+      remotes/origin/git-annex
+      remotes/origin/master
+      remotes/origin/synced/git-annex
+      remotes/origin/synced/master
+
+I have read the page on the internal workings and so far pretty much of all the wiki but if someone could point me to a description (if one exists) that explains which branch is used for what that would be great!
+
+I understand that upon "git annex sync" a repo pulls the $REMOTE/master into remotes/$REMOTE/master and $REMOTE/synced/master into remotes/$REMOTE/synced/master. 
+"git annex sync" on $REMOTE will push master to another repo into synced/master.
+
+Cheers,
+Marek
+
+
+ 
+
diff --git a/doc/forum/Which_branches_are___34__required__34__/comment_1_78d9ac05c66b7ccec031e0c57095af39._comment b/doc/forum/Which_branches_are___34__required__34__/comment_1_78d9ac05c66b7ccec031e0c57095af39._comment
new file mode 100644
index 0000000000..779f0f6571
--- /dev/null
+++ b/doc/forum/Which_branches_are___34__required__34__/comment_1_78d9ac05c66b7ccec031e0c57095af39._comment
@@ -0,0 +1,10 @@
+[[!comment format=mdwn
+ username="joey"
+ subject="""comment 1"""
+ date="2017-08-28T17:18:43Z"
+ content="""
+`git-annex unused` looks at all branches, both local branches and branches
+pulled from remotes. You could try the `--used-refspec` option to make
+that command only look at certian branches. Or delete old branches/remotes
+you're not using any longer.
+"""]]
diff --git a/doc/forum/Which_cloud_providers_are_supported__63___.mdwn b/doc/forum/Which_cloud_providers_are_supported__63___.mdwn
new file mode 100644
index 0000000000..9f8afc3fc0
--- /dev/null
+++ b/doc/forum/Which_cloud_providers_are_supported__63___.mdwn
@@ -0,0 +1,3 @@
+So Box.net and Amazon is supported, any way to get google drive in there?
+
+thanks for cool software!
diff --git a/doc/forum/Which_cloud_providers_are_supported__63___/comment_1_1f9398840144e0452a2fed9336046547._comment b/doc/forum/Which_cloud_providers_are_supported__63___/comment_1_1f9398840144e0452a2fed9336046547._comment
new file mode 100644
index 0000000000..9d8f0ecf75
--- /dev/null
+++ b/doc/forum/Which_cloud_providers_are_supported__63___/comment_1_1f9398840144e0452a2fed9336046547._comment
@@ -0,0 +1,10 @@
+[[!comment format=mdwn
+ username="http://joeyh.name/"
+ ip="4.154.0.140"
+ subject="comment 1"
+ date="2013-07-18T17:40:36Z"
+ content="""
+[[tips/googledriveannex]]
+
+See [[special_remotes]] for a full list of things people have gotten to work.
+"""]]
diff --git a/doc/forum/Why_are_ignored_files_being_deleted__63__.mdwn b/doc/forum/Why_are_ignored_files_being_deleted__63__.mdwn
new file mode 100644
index 0000000000..b7b857e941
--- /dev/null
+++ b/doc/forum/Why_are_ignored_files_being_deleted__63__.mdwn
@@ -0,0 +1,22 @@
+How do we get into the following situation?
+First we complain about an ignored file (why not just ignore it?),
+and then apparently we delete the ignored file.
+
+    [...]
+    import Pictures/2005/11/16/.IMG_0819.tmpwrite.JPG (duplicate) ok
+    (Recording state in git...)
+    The following paths are ignored by one of your .gitignore files:
+    Pictures/2008/11/27/.img_1315.tmpwrite.jpg
+    Use -f if you really want to add them.
+    fatal: no files added
+    git-annex: user error (xargs ["-0","git","--git-dir=/.../annex/.git","--work-tree=/.../annex","add","--"] exited 123)
+
+    # eek, the file that we complained about has vanished!
+    $ rm ../Pictures/2008/12/27/.img_1315.tmpwrite.jpg
+    rm: cannot remove ‘../Pictures/2008/11/27/.img_1315.tmpwrite.jpg’: No such file or directory
+
+Expected:
+- leave ignored files untouched. Maybe report "Skipped ignored files."
+
+Actual:
+- Stop import, but delete the ignored file as side effect.
diff --git a/doc/forum/Why_are_ignored_files_being_deleted__63__/comment_1_3081bda5bb600fc564185499ddfb5cf9._comment b/doc/forum/Why_are_ignored_files_being_deleted__63__/comment_1_3081bda5bb600fc564185499ddfb5cf9._comment
new file mode 100644
index 0000000000..1693b286af
--- /dev/null
+++ b/doc/forum/Why_are_ignored_files_being_deleted__63__/comment_1_3081bda5bb600fc564185499ddfb5cf9._comment
@@ -0,0 +1,35 @@
+[[!comment format=mdwn
+ username="https://www.google.com/accounts/o8/id?id=AItOawnVnsqEy82M-MuS2gLri-az83wSQ6lXSrc"
+ nickname="Jean"
+ subject="Ignored file imported, in spite of error message"
+ date="2015-04-08T09:08:53Z"
+ content="""
+Today I'm seeing this:
+
+    git-annex: user error (xargs [\"-0\",\"git\",\"--git-dir=/.../annex/.git\",\"--work-tree=/.../annex\",\"add\",\"--\"] exited 123)
+    failed
+    (Recording state in git...)
+    The following paths are ignored by one of your .gitignore files:
+    btsync/DCIM (1)/Camera 1/20150101090842.jpg.tmp
+    Use -f if you really want to add them.
+    fatal: no files added
+
+    git-annex: user error (xargs [\"-0\",\"git\",\"--git-dir=/.../annex/.git\",\"--work-tree=/.../annex\",\"add\",\"--\"] exited 123)
+    failed
+    (Recording state in git...)
+    ^C
+
+This repeats until I kill the import.
+
+Subsequently I see that the ignored file was in fact imported:
+
+    $ ls -lh \"btsync/DCIM (1)/Camera 1/20150101090842.jpg.tmp\"
+    lrwxrwxrwx 1 user user 203 Jan 11 14:11 btsync/DCIM (1)/Camera 1/20150101090842.jpg.tmp -> ../../../.git/annex/objects/0K/GX/SHA256E-s0--e3b0c44298ec1c149aebe4c8996eb92427ae41e4649b934ca495991b7852b855.jpg.tmp/SHA256E-s0--e3b0c44298ec1c149aebe4c8996eb92427ae41e4649b934ca495991b7852b855.jpg.tmp
+
+In this case the original file was not deleted, because I used `import --duplicate`:
+
+    $ ls -lh \"../btsync/DCIM (1)/Camera 1/20150101090842.jpg.tmp\"
+    -rw-rw-r-- 1 user user 0 Jan 11 14:11 ../btsync/DCIM (1)/Camera 1/20150101090842.jpg.tmp
+
+
+"""]]
diff --git a/doc/forum/Why_are_ignored_files_being_deleted__63__/comment_2_178aa574855a3bfffab4b21f90a84092._comment b/doc/forum/Why_are_ignored_files_being_deleted__63__/comment_2_178aa574855a3bfffab4b21f90a84092._comment
new file mode 100644
index 0000000000..d23ad6ff4d
--- /dev/null
+++ b/doc/forum/Why_are_ignored_files_being_deleted__63__/comment_2_178aa574855a3bfffab4b21f90a84092._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="joey"
+ subject="""comment 2"""
+ date="2015-04-29T17:48:12Z"
+ content="""
+It seems this was also filed as a bug report
+ so I'll deal with it there.
+"""]]
diff --git a/doc/forum/Why_are_new_repos_v5__63___Should_I_be_concerned__63__.mdwn b/doc/forum/Why_are_new_repos_v5__63___Should_I_be_concerned__63__.mdwn
new file mode 100644
index 0000000000..22edabb887
--- /dev/null
+++ b/doc/forum/Why_are_new_repos_v5__63___Should_I_be_concerned__63__.mdwn
@@ -0,0 +1,5 @@
+I'm fairly new to git-annex. Just spent the better part of an hour scratching my head about locked and unlocked files, until I realized that *my repository is v5, even though git-annex on all my machines is v6!*
+
+Is there a good reason repos created using v6 default to v5 rather than v6? I.e., not-yet-implemented features that will cause me a headache? Just want to make sure I'm not inviting more trouble if I update the repository to v6. Otherwise, I'd like to move ahead and do that, since I want to have an easier workflow for maintaining frequently edited binary files across my machines.
+
+Thanks.
diff --git a/doc/forum/Why_are_new_repos_v5__63___Should_I_be_concerned__63__/comment_1_b7fc905780f941314c99b94a87e0438a._comment b/doc/forum/Why_are_new_repos_v5__63___Should_I_be_concerned__63__/comment_1_b7fc905780f941314c99b94a87e0438a._comment
new file mode 100644
index 0000000000..4cbe446058
--- /dev/null
+++ b/doc/forum/Why_are_new_repos_v5__63___Should_I_be_concerned__63__/comment_1_b7fc905780f941314c99b94a87e0438a._comment
@@ -0,0 +1,11 @@
+[[!comment format=mdwn
+ username="RonnyPfannschmidt"
+ avatar="http://cdn.libravatar.org/avatar/c5379a3fe2188b7571858c49f9db63c6"
+ subject="comment 1"
+ date="2018-07-30T21:09:11Z"
+ content="""
+as per [[upgrades/#index1h2]] the upgrade to v6 repos is currently manual due to the finer details of git smudge handling and direct mode vs lock/unlock
+
+since v6 repos do behave differently the updates are manual
+
+"""]]
diff --git a/doc/forum/Why_are_we_stopping_at_a_duplicate__63__.mdwn b/doc/forum/Why_are_we_stopping_at_a_duplicate__63__.mdwn
new file mode 100644
index 0000000000..2f296c2820
--- /dev/null
+++ b/doc/forum/Why_are_we_stopping_at_a_duplicate__63__.mdwn
@@ -0,0 +1,7 @@
+What does the following mean? I would expect that if we encounter a duplicate while using the `--deduplicate` option, the duplicate file would be removed from the import location and import would continue. Does that mean that the newly encountered `img_0405.jpg` is different from the one previously seen? 
+Could the error message perhaps be improved?
+
+    .../usbdisk/annex(master)$ git annex import --deduplicate ../Pictures/
+    import Pictures/2008/10/12/img_0405.jpg git-annex: not overwriting existing Pictures/2008/10/12/img_0405.jpg (use --force to override)
+
+Many other duplicates are being tidied away.
diff --git a/doc/forum/Why_are_we_stopping_at_a_duplicate__63__/comment_1_dd3b610032cd3091effdb3f0828f45a8._comment b/doc/forum/Why_are_we_stopping_at_a_duplicate__63__/comment_1_dd3b610032cd3091effdb3f0828f45a8._comment
new file mode 100644
index 0000000000..0c2a1659c8
--- /dev/null
+++ b/doc/forum/Why_are_we_stopping_at_a_duplicate__63__/comment_1_dd3b610032cd3091effdb3f0828f45a8._comment
@@ -0,0 +1,9 @@
+[[!comment format=mdwn
+ username="joey"
+ subject="""comment 1"""
+ date="2015-04-29T17:47:08Z"
+ content="""
+The --deduplicate option deals with duplicated file contents. This is 
+a filename that is already in your git repo, git annex import avoids
+overwriting that with a different imported file unless you tell it --force.
+"""]]
diff --git a/doc/forum/Why_can__39__t_encryption_be_enabled_for_removable_drives__63__.mdwn b/doc/forum/Why_can__39__t_encryption_be_enabled_for_removable_drives__63__.mdwn
new file mode 100644
index 0000000000..9ff6f11fb6
--- /dev/null
+++ b/doc/forum/Why_can__39__t_encryption_be_enabled_for_removable_drives__63__.mdwn
@@ -0,0 +1,7 @@
+I currently have a pair of hard drives that I store backups etc on and that I would like to use as archive repositories for Git-Annex. I want everything on these to be encrypted, because I can't guarantee physical security of them.
+
+When I am setting up a removable drive as a repository, there is no option to encrypt to it. I understand that there is a "Directory" special remote, but it seems odd to have to use a different special remote that is not built into the webapp. Wouldn't it be good to have a checkbox for encryption when setting up a removable drive as a repository?
+
+Is there something that I'm missing here?
+
+Thanks for a great piece of software!
diff --git a/doc/forum/Why_can__39__t_encryption_be_enabled_for_removable_drives__63__/comment_1_4341898d5ae4f09a5b06d24f5fe6192d._comment b/doc/forum/Why_can__39__t_encryption_be_enabled_for_removable_drives__63__/comment_1_4341898d5ae4f09a5b06d24f5fe6192d._comment
new file mode 100644
index 0000000000..0ddf26666b
--- /dev/null
+++ b/doc/forum/Why_can__39__t_encryption_be_enabled_for_removable_drives__63__/comment_1_4341898d5ae4f09a5b06d24f5fe6192d._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="http://joeyh.name/"
+ ip="4.154.4.90"
+ subject="comment 1"
+ date="2013-07-16T19:45:42Z"
+ content="""
+git-annex stores a clone of the git repository on the removable drive. To encrypt a git remote, you can use the [git-remote-gcrypt](https://github.com/blake2-ppc/git-remote-gcrypt) tool. This will work with git-annex, but it needs integration into the webapp to support setting it up, and further integration so git-annex can detect when a git remote is encrypted and also encrypt the contents of file is stores there.
+"""]]
diff --git a/doc/forum/Why_does_git_annex_does_not_want_that_file__63__.mdwn b/doc/forum/Why_does_git_annex_does_not_want_that_file__63__.mdwn
new file mode 100644
index 0000000000..3dac8e4ae5
--- /dev/null
+++ b/doc/forum/Why_does_git_annex_does_not_want_that_file__63__.mdwn
@@ -0,0 +1,33 @@
+Hello,
+
+I have an repo that uses standard groups / wanted expressions:
+
+    % git annex wanted . && git annex group .               
+    standard
+    archive backup
+
+I use the two groups so that the repo wants anything (backup) and is recognized as an archive repo, so other repos drop files in archive/.
+
+    % git annex drop fortune
+    drop fortune (checking S3...) ok
+    (recording state in git...)
+
+fine. But:
+
+    % git annex get --auto
+
+does nothing. I do not understand, why the repo does not want the file. Same for git annex sync --content.
+
+    % git annex whereis fortune
+    whereis fortune (3 copies) 
+            19317658-7d40-4412-94ba-a2525e625283 -- Asaru
+            2f41eb74-5f33-4737-bb6b-1cd31dd67ee5 -- External
+            728648bf-5276-4a86-9df9-9a30f4783027 -- [S3]
+    ok
+
+Asaru is standard/client, S3 is standard/backup and External has no group / wanted set.
+
+
+
+Thanks,
+Florian
diff --git a/doc/forum/Why_does_git_annex_does_not_want_that_file__63__/comment_1_44cd6ca1f7215395e19ce654513197a9._comment b/doc/forum/Why_does_git_annex_does_not_want_that_file__63__/comment_1_44cd6ca1f7215395e19ce654513197a9._comment
new file mode 100644
index 0000000000..709bc80fdf
--- /dev/null
+++ b/doc/forum/Why_does_git_annex_does_not_want_that_file__63__/comment_1_44cd6ca1f7215395e19ce654513197a9._comment
@@ -0,0 +1,38 @@
+[[!comment format=mdwn
+ username="joey"
+ subject="""comment 1"""
+ date="2018-01-15T18:20:20Z"
+ content="""
+The problem is that you have the repository in two groups.
+
+The way that `standard` works is that the repository has to be in
+exactly one group and then it will use the preferred content for that
+group. It does not know what to do when there are two groups being
+combined.
+
+Solution for you is to run: 
+
+	git annex wanted . anything
+
+Since backup repositories have a copy of every file, matching anything is
+what you want it to do. Since it's also in the archive group, other archive
+repositories will treat it as an archive too, so once a file reaches your
+combo backup and archive repository, it will be removed from other
+archives.
+
+Of course, you could also OR the preferred content expression for backup
+with the preferred content expression for archive, but that would have the
+same result since "anything OR archive" matches anything.
+
+But we can't just OR the two preferred content expressions for two groups
+in general. Consider the client and archive preferred content expressions:
+
+	(include=* and ((exclude=*/archive/* and exclude=archive/*) or (not (copies=archive:1 or copies=smallarchive:1)))) or approxlackingcopies=1
+
+	(not (copies=archive:1 or copies=smallarchive:1)) or approxlackingcopies=1
+
+These are contradictory when a repository is in both the client and archive
+groups; the client doesn't want anything that's in an archive, but if the
+repository is both a client and an archive, it doesn't want any content
+that it has!
+"""]]
diff --git a/doc/forum/Why_does_the_bup_remote_use___126____47__.bup__63__.mdwn b/doc/forum/Why_does_the_bup_remote_use___126____47__.bup__63__.mdwn
new file mode 100644
index 0000000000..c250ef7c1e
--- /dev/null
+++ b/doc/forum/Why_does_the_bup_remote_use___126____47__.bup__63__.mdwn
@@ -0,0 +1,5 @@
+I created a test remote using the command:
+
+    git annex initremote mybup type=bup encryption=none buprepo=/tmp/mybup
+
+I can copy files to and from the remote just fine.  However, every time I do so it makes changes inside `~/.bup`.  If I delete `~/.bup` it will recreate it.  Is this expected?  Are the files in `~/.bup` of any consequence?
diff --git a/doc/forum/Why_does_the_bup_remote_use___126____47__.bup__63__/comment_1_da9c7c0e93aefc2da7409de5b138d86f._comment b/doc/forum/Why_does_the_bup_remote_use___126____47__.bup__63__/comment_1_da9c7c0e93aefc2da7409de5b138d86f._comment
new file mode 100644
index 0000000000..cbb3c8d6d9
--- /dev/null
+++ b/doc/forum/Why_does_the_bup_remote_use___126____47__.bup__63__/comment_1_da9c7c0e93aefc2da7409de5b138d86f._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="https://me.yahoo.com/a/2grhJvAC049fJnvALDXek.6MRZMTlg--#eec89"
+ nickname="John"
+ subject="Ah"
+ date="2012-10-18T23:52:03Z"
+ content="""
+I get now why ~/.bup is being used: -r DIR specifies a local DIR as the bup directory, but ~/.bup is still used for the local temp files.  It would be nice, therefore, if BUP_DIR were set in the case that -r specifies a local directory, so that only that directory is used.  Unless, of course, that leads to space limitations in the bup directory.  Hmm... maybe this should just be left as is in that case.
+"""]]
diff --git a/doc/forum/Why_git_commit_fails_from_within_a_newly_git-annexed_subdir__63__.mdwn b/doc/forum/Why_git_commit_fails_from_within_a_newly_git-annexed_subdir__63__.mdwn
new file mode 100644
index 0000000000..ccf059f0c1
--- /dev/null
+++ b/doc/forum/Why_git_commit_fails_from_within_a_newly_git-annexed_subdir__63__.mdwn
@@ -0,0 +1,42 @@
+If I `git-annex add` a directory, with some files inside, and then move into the directory to execute `git commit -m message`, then `git commit` fails with: `git-annex: git: createProcess: runInteractiveProcess: chdir: does not exist (No such file or directory)`. Why?
+
+Notice that if I do the same operations within a standard git repository (not annex), then `git commit` works with no problems, as expected.
+
+As it is described above, this difference between git-annex and git seems a minor issue, but I got stuck into that for a while when I stumbled into it in a more complex scenario. 
+
+Steps to reproduce the issue:
+
+    > mkdir annex
+    > cd annex
+    /annex> git init
+    Initialized empty Git repository in /tmp/zzz/annex/.git/
+    /annex (master)> git-annex init
+    init  ok
+    (recording state in git...)
+    /annex (master)> mkdir a
+    /annex (master)> cd a
+    /annex/a (master)> echo foo > bar
+    /annex/a (master)> git annex add bar
+    add bar ok
+    (recording state in git...)
+    /annex/a (master)> git commit -m "bar added"
+    git-annex: git: createProcess: runInteractiveProcess: chdir: does not exist (No such file or directory)
+
+
+Differently, within a standard git repository, the equivalent `git commit` does not fail:
+
+    > mkdir repo
+    > cd repo
+    /repo> git init
+    Initialized empty Git repository in /tmp/zzz/repo/.git/
+    /repo (master)> mkdir a
+    /repo (master)> cd a
+    /repo/a (master)> echo foo > bar
+    /repo/a (master)> git add bar
+    /repo/a (master)> git commit -m "bar added"
+    [master (root-commit) 203c996] bar added
+     1 file changed, 1 insertion(+)
+     create mode 100644 a/bar
+
+
+In any case, a big thank for making `git-annex`, it is a really great piece of software!
diff --git a/doc/forum/Why_git_commit_fails_from_within_a_newly_git-annexed_subdir__63__/comment_10_abae8a95b60f17052a5e2ac30a7ece32._comment b/doc/forum/Why_git_commit_fails_from_within_a_newly_git-annexed_subdir__63__/comment_10_abae8a95b60f17052a5e2ac30a7ece32._comment
new file mode 100644
index 0000000000..a0e6e86e65
--- /dev/null
+++ b/doc/forum/Why_git_commit_fails_from_within_a_newly_git-annexed_subdir__63__/comment_10_abae8a95b60f17052a5e2ac30a7ece32._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="AndyPerry"
+ avatar="http://cdn.libravatar.org/avatar/42114bc1ad6d16de30842b297a91ec45"
+ subject="comment 10"
+ date="2018-09-14T13:20:42Z"
+ content="""
+I have built from source and it works. Can the MacOS repository be updated soon to have a version with the fix?
+"""]]
diff --git a/doc/forum/Why_git_commit_fails_from_within_a_newly_git-annexed_subdir__63__/comment_1_dfd69ac96065f8146f5ea87b957cca6e._comment b/doc/forum/Why_git_commit_fails_from_within_a_newly_git-annexed_subdir__63__/comment_1_dfd69ac96065f8146f5ea87b957cca6e._comment
new file mode 100644
index 0000000000..5e0c73ba53
--- /dev/null
+++ b/doc/forum/Why_git_commit_fails_from_within_a_newly_git-annexed_subdir__63__/comment_1_dfd69ac96065f8146f5ea87b957cca6e._comment
@@ -0,0 +1,10 @@
+[[!comment format=mdwn
+ username="inecas@fe552cc962c331a5cb1499e80bd5b2aa0808d987"
+ nickname="inecas"
+ avatar="http://cdn.libravatar.org/avatar/08cb45efffe7995a6bcbb904e9066f79"
+ subject="comment 1"
+ date="2018-09-07T07:25:53Z"
+ content="""
+I see this issue as well, it started appearing after updating git-annex-6.20180626-1 to git-annex-6.20180807-1 (Fedora distribution).
+Downgrading helped resolving this issue for me for now.
+"""]]
diff --git a/doc/forum/Why_git_commit_fails_from_within_a_newly_git-annexed_subdir__63__/comment_2_8d36f559d7f8b61f5633f0d24b424cec._comment b/doc/forum/Why_git_commit_fails_from_within_a_newly_git-annexed_subdir__63__/comment_2_8d36f559d7f8b61f5633f0d24b424cec._comment
new file mode 100644
index 0000000000..29f643f8c1
--- /dev/null
+++ b/doc/forum/Why_git_commit_fails_from_within_a_newly_git-annexed_subdir__63__/comment_2_8d36f559d7f8b61f5633f0d24b424cec._comment
@@ -0,0 +1,9 @@
+[[!comment format=mdwn
+ username="emanuele.olivetti@47d88ed185b03191e25329caa6fabc2efb3118b2"
+ nickname="emanuele.olivetti"
+ avatar="http://cdn.libravatar.org/avatar/f51cc5c6c3a0eb28faa6491c3cbcfcce"
+ subject="comment 2"
+ date="2018-09-07T12:27:28Z"
+ content="""
+Shall I file it as a bug/regression then?
+"""]]
diff --git a/doc/forum/Why_git_commit_fails_from_within_a_newly_git-annexed_subdir__63__/comment_3_a318582d30f5483f3738e8f0f1a2e1df._comment b/doc/forum/Why_git_commit_fails_from_within_a_newly_git-annexed_subdir__63__/comment_3_a318582d30f5483f3738e8f0f1a2e1df._comment
new file mode 100644
index 0000000000..2cf8809c13
--- /dev/null
+++ b/doc/forum/Why_git_commit_fails_from_within_a_newly_git-annexed_subdir__63__/comment_3_a318582d30f5483f3738e8f0f1a2e1df._comment
@@ -0,0 +1,11 @@
+[[!comment format=mdwn
+ username="joey"
+ subject="""comment 3"""
+ date="2018-09-11T17:45:41Z"
+ content="""
+This seems like the same or related to the problem discussed here:
+
+
+It seems you are not using v6 mode, but perhaps you also have a broken old
+version of git?
+"""]]
diff --git a/doc/forum/Why_git_commit_fails_from_within_a_newly_git-annexed_subdir__63__/comment_4_bc2e255f8732f6ec02e90a6623ffc8af._comment b/doc/forum/Why_git_commit_fails_from_within_a_newly_git-annexed_subdir__63__/comment_4_bc2e255f8732f6ec02e90a6623ffc8af._comment
new file mode 100644
index 0000000000..1326e58709
--- /dev/null
+++ b/doc/forum/Why_git_commit_fails_from_within_a_newly_git-annexed_subdir__63__/comment_4_bc2e255f8732f6ec02e90a6623ffc8af._comment
@@ -0,0 +1,9 @@
+[[!comment format=mdwn
+ username="joey"
+ subject="""comment 4"""
+ date="2018-09-11T19:54:45Z"
+ content="""
+It seems likely I've just fixed this in git-annex [[!commit fdbdf64d87e2980e8399d1182ccb43bd0f013b16]]
+but since I can't reproduce the problem from your transcript I can't say
+for sure.
+"""]]
diff --git a/doc/forum/Why_git_commit_fails_from_within_a_newly_git-annexed_subdir__63__/comment_5_691dc07b192f077f50611544d488b394._comment b/doc/forum/Why_git_commit_fails_from_within_a_newly_git-annexed_subdir__63__/comment_5_691dc07b192f077f50611544d488b394._comment
new file mode 100644
index 0000000000..df0ba9d451
--- /dev/null
+++ b/doc/forum/Why_git_commit_fails_from_within_a_newly_git-annexed_subdir__63__/comment_5_691dc07b192f077f50611544d488b394._comment
@@ -0,0 +1,10 @@
+[[!comment format=mdwn
+ username="emanuele.olivetti@47d88ed185b03191e25329caa6fabc2efb3118b2"
+ nickname="emanuele.olivetti"
+ avatar="http://cdn.libravatar.org/avatar/f51cc5c6c3a0eb28faa6491c3cbcfcce"
+ subject="comment 5"
+ date="2018-09-12T10:45:03Z"
+ content="""
+Thanks Joey for looking into this one. I use git v2.7.4, currently shipped with Ubuntu 16.04, which may not be very recent. I don't use v6 because it is not the default in git-annex, at least till now. Should I?
+I'll try the recent commit you mention, as soon as it will be available in the Linux standalone version or in Neurodebian (currently 6.20180807-g48d11a5df and 6.20180807+git230-gaa291acfe-1~ndall+1) and I will report here shortly after.
+"""]]
diff --git a/doc/forum/Why_git_commit_fails_from_within_a_newly_git-annexed_subdir__63__/comment_6_71d7a08600dcba07dd895ddf6ad1331a._comment b/doc/forum/Why_git_commit_fails_from_within_a_newly_git-annexed_subdir__63__/comment_6_71d7a08600dcba07dd895ddf6ad1331a._comment
new file mode 100644
index 0000000000..83a38e2227
--- /dev/null
+++ b/doc/forum/Why_git_commit_fails_from_within_a_newly_git-annexed_subdir__63__/comment_6_71d7a08600dcba07dd895ddf6ad1331a._comment
@@ -0,0 +1,29 @@
+[[!comment format=mdwn
+ username="emanuele.olivetti@47d88ed185b03191e25329caa6fabc2efb3118b2"
+ nickname="emanuele.olivetti"
+ avatar="http://cdn.libravatar.org/avatar/f51cc5c6c3a0eb28faa6491c3cbcfcce"
+ subject="comment 6"
+ date="2018-09-13T14:56:51Z"
+ content="""
+My mistake. The last Linux standalone version, 6.20180807-g48d11a5df, works correctly and does not show the issue anymore:
+
+    $ mkdir annex
+    $ cd annex
+    $ git init
+    Initialized empty Git repository in /tmp/annex/.git/
+    $ git annex init
+    init  ok
+    (recording state in git...)
+    $ mkdir a
+    $ cd a
+    $ echo foo > bar
+    $ git annex add bar
+    add bar ok
+    (recording state in git...)
+    $ git commit -m \"bar added\"
+    [master (root-commit) 4cf38d9] bar added
+     1 file changed, 1 insertion(+)
+     create mode 120000 a/bar
+
+Thank you for fixing the issue and thanks again for git-annex!
+"""]]
diff --git a/doc/forum/Why_git_commit_fails_from_within_a_newly_git-annexed_subdir__63__/comment_7_a5d60323c2c6b2d263514b33ad774aa9._comment b/doc/forum/Why_git_commit_fails_from_within_a_newly_git-annexed_subdir__63__/comment_7_a5d60323c2c6b2d263514b33ad774aa9._comment
new file mode 100644
index 0000000000..a6fcb4a2f0
--- /dev/null
+++ b/doc/forum/Why_git_commit_fails_from_within_a_newly_git-annexed_subdir__63__/comment_7_a5d60323c2c6b2d263514b33ad774aa9._comment
@@ -0,0 +1,9 @@
+[[!comment format=mdwn
+ username="AndyPerry"
+ avatar="http://cdn.libravatar.org/avatar/42114bc1ad6d16de30842b297a91ec45"
+ subject="comment 7"
+ date="2018-09-14T09:49:57Z"
+ content="""
+I get the same error/failure on MacOS. I have git-annex 6.20180807
+I made a very simple change, deleting an old symlink, but the commit fails as mentioned in this thread.
+"""]]
diff --git a/doc/forum/Why_git_commit_fails_from_within_a_newly_git-annexed_subdir__63__/comment_8_83e52bd3d66ce87ba7d2913d21c76b08._comment b/doc/forum/Why_git_commit_fails_from_within_a_newly_git-annexed_subdir__63__/comment_8_83e52bd3d66ce87ba7d2913d21c76b08._comment
new file mode 100644
index 0000000000..d7d319a1d7
--- /dev/null
+++ b/doc/forum/Why_git_commit_fails_from_within_a_newly_git-annexed_subdir__63__/comment_8_83e52bd3d66ce87ba7d2913d21c76b08._comment
@@ -0,0 +1,9 @@
+[[!comment format=mdwn
+ username="AndyPerry"
+ avatar="http://cdn.libravatar.org/avatar/42114bc1ad6d16de30842b297a91ec45"
+ subject="comment 8"
+ date="2018-09-14T11:39:41Z"
+ content="""
+I installed the *latest* via `brew install git-annex`. Is there another repository I can use to get a newer version? 
+I'd rather not have top build from source as the pre-reqs are not tools I'd use for anything else.
+"""]]
diff --git a/doc/forum/Why_git_commit_fails_from_within_a_newly_git-annexed_subdir__63__/comment_9_0bd122e2e664d29dca027719721b1523._comment b/doc/forum/Why_git_commit_fails_from_within_a_newly_git-annexed_subdir__63__/comment_9_0bd122e2e664d29dca027719721b1523._comment
new file mode 100644
index 0000000000..8be5f1fa7c
--- /dev/null
+++ b/doc/forum/Why_git_commit_fails_from_within_a_newly_git-annexed_subdir__63__/comment_9_0bd122e2e664d29dca027719721b1523._comment
@@ -0,0 +1,9 @@
+[[!comment format=mdwn
+ username="emanuele.olivetti@47d88ed185b03191e25329caa6fabc2efb3118b2"
+ nickname="emanuele.olivetti"
+ avatar="http://cdn.libravatar.org/avatar/f51cc5c6c3a0eb28faa6491c3cbcfcce"
+ subject="comment 9"
+ date="2018-09-14T12:24:14Z"
+ content="""
+Check the app bundle (pre-built) that you can download from . If it is the analogous of the Linux standalone file, then it should be version 6.20180807-g48d11a5df, which solves the bug discussed in this page. It would be nice if you send feedback here after trying.
+"""]]
diff --git a/doc/forum/Why_is_git_annex_status_slow__63__.mdwn b/doc/forum/Why_is_git_annex_status_slow__63__.mdwn
new file mode 100644
index 0000000000..5c659b61d4
--- /dev/null
+++ b/doc/forum/Why_is_git_annex_status_slow__63__.mdwn
@@ -0,0 +1,20 @@
+It seems that `git annex status` is much slower than `git status`, at least in direct mode. The man page does not give any hint about why it should be slower.
+
+Does `git annex status` do something that `git status` does not?
+
+Here is an example in a repo with 8000+ files in direct mode and with no modified files:
+
+
+    $ time git -c core.bare=false status --porcelain > /dev/null
+
+    real   0m0.096s
+    user   0m0.042s
+    sys    0m0.071s
+
+    $ time git annex status
+
+    real   0m17.144s
+    user   0m10.555s
+    sys    0m1.934s
+
+It is strange to see that `git annex status` is ~200 times slower than the bare `git status`.
diff --git a/doc/forum/Why_is_git_annex_status_slow__63__/comment_1_a5139c5a9fc94486cf62fc9a9fa9f8d9._comment b/doc/forum/Why_is_git_annex_status_slow__63__/comment_1_a5139c5a9fc94486cf62fc9a9fa9f8d9._comment
new file mode 100644
index 0000000000..f21fcac488
--- /dev/null
+++ b/doc/forum/Why_is_git_annex_status_slow__63__/comment_1_a5139c5a9fc94486cf62fc9a9fa9f8d9._comment
@@ -0,0 +1,18 @@
+[[!comment format=mdwn
+ username="joey"
+ subject="""comment 1"""
+ date="2014-11-03T17:08:50Z"
+ content="""
+`git status` looks at the index and work tree. In an indirect mode
+repository, `git annex status` does too, and is not significantly slower.
+
+In direct mode, `git annex status` has to look up from git the key
+that corresponds to each file in the work tree. This is the main
+thing that slows it down.
+
+(See the code for details, it's quite clear.)
+
+The best workaround is proably to pass git-annex status a subdirectory
+that you're interested in, so it can only look at the contents of that one
+directory.
+"""]]
diff --git a/doc/forum/Why_is_git_annex_status_slow__63__/comment_2_2f61c2cdf9d1ffd298fd906207012818._comment b/doc/forum/Why_is_git_annex_status_slow__63__/comment_2_2f61c2cdf9d1ffd298fd906207012818._comment
new file mode 100644
index 0000000000..02b158b703
--- /dev/null
+++ b/doc/forum/Why_is_git_annex_status_slow__63__/comment_2_2f61c2cdf9d1ffd298fd906207012818._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="https://www.google.com/accounts/o8/id?id=AItOawkutSE8_3fFAETmO_E598zja4gKwYXbb8E"
+ nickname="Сергей"
+ subject="Slow on windows but not on linux"
+ date="2015-04-07T18:20:23Z"
+ content="""
+If this is by design and `git annex status` should take more time in direct mode then what I'm experiencing is strange. On windows every 100M file adds approximately 1 second to `status` duration (on my laptop), but on linux it does not. On linux `git annex status` even in direct mode takes milliseconds. What is wrong with my setup?
+"""]]
diff --git a/doc/forum/Why_is_git_annex_status_slow__63__/comment_3_8d9e5a1aef2648d7a844623e6237d551._comment b/doc/forum/Why_is_git_annex_status_slow__63__/comment_3_8d9e5a1aef2648d7a844623e6237d551._comment
new file mode 100644
index 0000000000..54d2b24c49
--- /dev/null
+++ b/doc/forum/Why_is_git_annex_status_slow__63__/comment_3_8d9e5a1aef2648d7a844623e6237d551._comment
@@ -0,0 +1,14 @@
+[[!comment format=mdwn
+ username="joey"
+ subject="""comment 3"""
+ date="2015-04-09T17:44:01Z"
+ content="""
+The sizes of the files should not affect how fast git-annex status runs. 
+
+But, direct mode certianly does. git-annex has to do significantly more
+work in direct mode to figure out the status of a file. Including querying
+git. In indirect mode, it can just stat the symlink and see if its content
+is present, which is much faster.
+
+(There's probably also some other inneficiencies in Windows.)
+"""]]
diff --git a/doc/forum/Will_git-annex_solve_my_problem__63__.mdwn b/doc/forum/Will_git-annex_solve_my_problem__63__.mdwn
new file mode 100644
index 0000000000..0aa2ded520
--- /dev/null
+++ b/doc/forum/Will_git-annex_solve_my_problem__63__.mdwn
@@ -0,0 +1,7 @@
+Here's my current situation:
+
+I have a box which creates about a dozen files periodically. All files add up to about 1GB in size. The files are text and sorted. I then rsync the files to n servers. The rsync diff algorithm transfers way less than n * 1GB because the files are largely the same. However, this distribution technique is inefficient because I must run n rsync processes in parallel and the rsync diff algorithm takes a lot of CPU.
+
+How could I use git-annex instead of rsync?
+
+Because the box producing the new files also has the old files, then presumably git could calculate the diffs for each file once instead of n times as with the rsync solution? Then only the diffs need be distributed to the n servers... using git-annex? And finally the newly updated version of the dozen files needs to be available on each of the n servers. Ideally, the diffs would not mount up over time on either the publishing server or the n servers, thus causing out of disk problems etc. How to deploy git-annex to solve my problem?
diff --git a/doc/forum/Will_git-annex_solve_my_problem__63__/comment_1_35acbdd1a7727df204d776c2e8f02b53._comment b/doc/forum/Will_git-annex_solve_my_problem__63__/comment_1_35acbdd1a7727df204d776c2e8f02b53._comment
new file mode 100644
index 0000000000..a4f27f46ed
--- /dev/null
+++ b/doc/forum/Will_git-annex_solve_my_problem__63__/comment_1_35acbdd1a7727df204d776c2e8f02b53._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="http://joeyh.name/"
+ nickname="joey"
+ subject="comment 1"
+ date="2013-03-27T22:48:16Z"
+ content="""
+git-annex will not perform better than rsync. It uses rsync. It also does not yet use old versions of files as a rsync basis when transferring a new version of a file, so is not as efficient in this scenario.
+"""]]
diff --git a/doc/forum/Will_git-annex_solve_my_problem__63__/comment_2_230256c19ac139dea207d89c06f70782._comment b/doc/forum/Will_git-annex_solve_my_problem__63__/comment_2_230256c19ac139dea207d89c06f70782._comment
new file mode 100644
index 0000000000..2b7b7e5f7b
--- /dev/null
+++ b/doc/forum/Will_git-annex_solve_my_problem__63__/comment_2_230256c19ac139dea207d89c06f70782._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="https://www.google.com/accounts/o8/id?id=AItOawkaBh9VNJ-RZ26wJZ4BEhMN1IlPT-DK6JA"
+ nickname="Alex"
+ subject="Bup"
+ date="2013-04-03T05:47:41Z"
+ content="""
+I think bup will solve your use case. Make a local bup repo and backup there then clone that repo n times. If you run git annex on top of bup you can then retrieve the subset of files you need at each destination. This will work best if you don't need every file on every remote or if your network is much slower than local disk access. 
+"""]]
diff --git a/doc/forum/Will_git_annex_work_on_a_FAT32_formatted_key__63__.mdwn b/doc/forum/Will_git_annex_work_on_a_FAT32_formatted_key__63__.mdwn
new file mode 100644
index 0000000000..f458ba72e0
--- /dev/null
+++ b/doc/forum/Will_git_annex_work_on_a_FAT32_formatted_key__63__.mdwn
@@ -0,0 +1,3 @@
+FAT32 does not support symlinks, so I wonder if there's going to be a problem with that.
+
+Generally speaking, I am wondering about portability of git annex on windows and on android...
diff --git a/doc/forum/Will_git_annex_work_on_a_FAT32_formatted_key__63__/comment_1_426482e6eb3a27687a48f24f6ef2332f._comment b/doc/forum/Will_git_annex_work_on_a_FAT32_formatted_key__63__/comment_1_426482e6eb3a27687a48f24f6ef2332f._comment
new file mode 100644
index 0000000000..119c9e535a
--- /dev/null
+++ b/doc/forum/Will_git_annex_work_on_a_FAT32_formatted_key__63__/comment_1_426482e6eb3a27687a48f24f6ef2332f._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="http://joey.kitenet.net/"
+ nickname="joey"
+ subject="comment 1"
+ date="2011-03-07T19:13:14Z"
+ content="""
+See [[bugs/fat_support]]. A bare git repo will have to be used to avoid symlink problems, at least for now. The other problem is that git-annex key files have colons in their filenames.
+"""]]
diff --git a/doc/forum/Will_git_annex_work_on_a_FAT32_formatted_key__63__/comment_2_af4f8b52526d8bea2904c95406fd2796._comment b/doc/forum/Will_git_annex_work_on_a_FAT32_formatted_key__63__/comment_2_af4f8b52526d8bea2904c95406fd2796._comment
new file mode 100644
index 0000000000..ca599b2857
--- /dev/null
+++ b/doc/forum/Will_git_annex_work_on_a_FAT32_formatted_key__63__/comment_2_af4f8b52526d8bea2904c95406fd2796._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="http://joey.kitenet.net/"
+ nickname="joey"
+ subject="comment 2"
+ date="2011-03-19T15:37:22Z"
+ content="""
+Now it's fully supported, so long as you put a bare git repo on your key.
+"""]]
diff --git a/doc/forum/Windows_-_You_don__39__t_have_access.mdwn b/doc/forum/Windows_-_You_don__39__t_have_access.mdwn
new file mode 100644
index 0000000000..a12d46ad9f
--- /dev/null
+++ b/doc/forum/Windows_-_You_don__39__t_have_access.mdwn
@@ -0,0 +1,42 @@
+git version: 1.9.5.msysgit.1
+git-annex version: 5.20150710-g8fd7052
+
+I have a repo up on GitLab. I have annex’d files in that repo. On a Linux server I can “git annex sync” and then “git annex get” just fine. On Windows when I try to run “git annex sync” I get:
+
+GitLab: You don't have access
+
+  Remote origin does not have git-annex installed; setting annex-ignore
+
+  This could be a problem with the git-annex installation on the remote. Please make sure that git-a
+nnex-shell is available in PATH when you ssh into the remote. Once you have fixed the git-annex inst
+allation, run: git config remote.origin.annex-ignore false
+commit  ok
+pull origin
+git-annex.exe: unknown command git@gitlab.server.com
+…
+fatal: Could not read from remote repository.
+
+Please make sure you have the correct access rights
+and the repository exists.
+failed
+git-annex: sync: 1 failed
+
+I have access to the repo however. I can git pull/push…whatever. It’s just annex that is having problems with access and I’m not sure why. Here is my git config:
+
+[core]
+        repositoryformatversion = 0
+        filemode = false
+        bare = false
+        logallrefupdates = true
+        symlinks = false
+        ignorecase = true
+        hideDotFiles = dotGitOnly
+[remote "origin"]
+        url = git@gitlab.company.com:repo.git
+        fetch = +refs/heads/*:refs/remotes/origin/*
+        annex-ignore = false
+[annex]
+        uuid = 2noa1e70-9f88-4did-843c-3f8sdf3495990
+        sshcaching = false
+        crippledfilesystem = true
+        version = 5
diff --git a/doc/forum/Windows_-_You_don__39__t_have_access/comment_1_5c1da63922cc71483c9519e8670d532b._comment b/doc/forum/Windows_-_You_don__39__t_have_access/comment_1_5c1da63922cc71483c9519e8670d532b._comment
new file mode 100644
index 0000000000..ad11ef8dfc
--- /dev/null
+++ b/doc/forum/Windows_-_You_don__39__t_have_access/comment_1_5c1da63922cc71483c9519e8670d532b._comment
@@ -0,0 +1,17 @@
+[[!comment format=mdwn
+ username="joey"
+ subject="""comment 1"""
+ date="2015-07-20T17:51:33Z"
+ content="""
+The "GitLab: You don't have access" seems to be gitlab refusing to run
+git-annex-shell when asked to. Only the gitlab people could help you with
+that. (Gitlab does seem to work with git-annex today when I try it,
+I had some similar problems with their implementation refusing to
+start git-annex-shell earlier.)
+
+OTOH, the "git-annex.exe: unknown command git@gitlab.server.com"
+looks like you ran into this bug:
+
+Since I have still not managed to reproduce or get to the bottom of that
+bug report, any information about it would be useful.
+"""]]
diff --git a/doc/forum/Windows_-_You_don__39__t_have_access/comment_2_aa6ea60465df9fab7990bd6f510b74c5._comment b/doc/forum/Windows_-_You_don__39__t_have_access/comment_2_aa6ea60465df9fab7990bd6f510b74c5._comment
new file mode 100644
index 0000000000..dd64dd7f00
--- /dev/null
+++ b/doc/forum/Windows_-_You_don__39__t_have_access/comment_2_aa6ea60465df9fab7990bd6f510b74c5._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="fusionx86@2cab672ef75a8502e153ceb90177dde38985dba9"
+ nickname="fusionx86"
+ subject="comment 2"
+ date="2015-07-20T18:59:29Z"
+ content="""
+GitLab is working fine with annex. I'm able to annex sync and sync --content on Linux and OSX without any problems. It's just Windows client I'm struggling with. I'll look at the bug you linked and post back with anything interesting.
+"""]]
diff --git a/doc/forum/Windows_-_You_don__39__t_have_access/comment_3_593a8a54fa80258d5048f5c061992664._comment b/doc/forum/Windows_-_You_don__39__t_have_access/comment_3_593a8a54fa80258d5048f5c061992664._comment
new file mode 100644
index 0000000000..798aa125a7
--- /dev/null
+++ b/doc/forum/Windows_-_You_don__39__t_have_access/comment_3_593a8a54fa80258d5048f5c061992664._comment
@@ -0,0 +1,10 @@
+[[!comment format=mdwn
+ username="joey"
+ subject="""comment 3"""
+ date="2015-07-20T19:03:58Z"
+ content="""
+Pretty sure that the only difference gitlab can see between your Windows
+and Linux boxes there is the ssh key that they're using. So, maybe you need
+to add the Windows box's ssh key to the gitlab repo, or there's a problem
+in that area.
+"""]]
diff --git a/doc/forum/Windows_-_You_don__39__t_have_access/comment_4_aa28602eb0b7fa7103eaf90dc1d5cf8f._comment b/doc/forum/Windows_-_You_don__39__t_have_access/comment_4_aa28602eb0b7fa7103eaf90dc1d5cf8f._comment
new file mode 100644
index 0000000000..89778431ca
--- /dev/null
+++ b/doc/forum/Windows_-_You_don__39__t_have_access/comment_4_aa28602eb0b7fa7103eaf90dc1d5cf8f._comment
@@ -0,0 +1,54 @@
+[[!comment format=mdwn
+ username="fusionx86@2cab672ef75a8502e153ceb90177dde38985dba9"
+ nickname="fusionx86"
+ subject="comment 4"
+ date="2015-07-20T20:14:07Z"
+ content="""
+The bug you linked does look like the same thing I'm seeing. I decided to install cygwin on the same server to see if it made a difference. It started adding files this time and I thought I had a solution, but then it threw the same error about access.
+----
+$ git annex sync --verbose
+
+  Detected a filesystem without fifo support.
+
+  Disabling ssh connection caching.
+
+  Detected a crippled filesystem.
+(merging origin/git-annex into git-annex...)
+(recording state in git...)
+
+  Enabling direct mode.
+commit  (recording state in git...)
+add Bin/FastMM_FullDebugMode.dll ok
+add Bin/Res/bar_blank.gif ok
+add Bin/Res/bar_blank_gray.gif ok
+add Bin/Res/bar_gray.gif ok
+add Bin/Res/bar_left.gif ok
+add Bin/Res/bar_middle.gif ok
+add Bin/Res/bar_right.gif ok
+add Bin/Res/btn_back_0.gif ok
+add Bin/Res/btn_back_1.gif ok
+add Bin/Res/btn_back_2.gif ok
+add Bin/Res/btn_back_3.gif ok
+...
+(recording state in git...)
+ok
+pull origin git-annex.exe: unknown command git@gitlab.company.com
+
+Usage: git-annex command [option ...]
+...
+fatal: Could not read from remote repository.
+
+Please make sure you have the correct access rights
+and the repository exists.
+
+  Pushing to origin failed.
+
+  (non-fast-forward problems can be solved by setting receive.denyNonFastforwards to false in the remote's git config)
+failed
+(recording state in git...)
+git-annex.exe: sync: 2 failed
+----
+
+So it seemed to get a little further, but still fails. The key $HOME/.ssh/id_rsa has full access to the repo and I'm able to push/pull with git just fine. Not sure what else to try for annex sync.
+
+"""]]
diff --git a/doc/forum/Windows_-_You_don__39__t_have_access/comment_5_f9041e48b911dbdd37ad3c1bbc801709._comment b/doc/forum/Windows_-_You_don__39__t_have_access/comment_5_f9041e48b911dbdd37ad3c1bbc801709._comment
new file mode 100644
index 0000000000..b77d09c122
--- /dev/null
+++ b/doc/forum/Windows_-_You_don__39__t_have_access/comment_5_f9041e48b911dbdd37ad3c1bbc801709._comment
@@ -0,0 +1,12 @@
+[[!comment format=mdwn
+ username="fusionx86@2cab672ef75a8502e153ceb90177dde38985dba9"
+ nickname="fusionx86"
+ subject="comment 5"
+ date="2015-07-20T20:41:00Z"
+ content="""
+Also, I created git-annex-shell.exe as a copy of git-annex.exe which you recommended here: https://git-annex.branchable.com/bugs/no_git-annex_shell_on_Windows/.
+
+I also made sure UAC and Windows Firewall were disabled and they already were.
+
+What environment variables should exist to support annex? I see references to GIT_ANNEX_SSHOPTION and GIT_SSH, but those aren't currently configured on my Windows server. The git, git-annex and git-annex-shell files are in the global path and accessible from anywhere.
+"""]]
diff --git a/doc/forum/Windows_-_You_don__39__t_have_access/comment_6_7656eb72bb9e39db59092557c57a2489._comment b/doc/forum/Windows_-_You_don__39__t_have_access/comment_6_7656eb72bb9e39db59092557c57a2489._comment
new file mode 100644
index 0000000000..c78501a873
--- /dev/null
+++ b/doc/forum/Windows_-_You_don__39__t_have_access/comment_6_7656eb72bb9e39db59092557c57a2489._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="fusionx86@2cab672ef75a8502e153ceb90177dde38985dba9"
+ nickname="fusionx86"
+ subject="comment 6"
+ date="2015-07-24T18:10:20Z"
+ content="""
+Any other ideas on this problem? I was planning to use annex for large/binary files, but we have a lot of developers that would need to run it on Windows as well and I'm not sure how to roll it out when I can't get it working myself on a Windows build server. I think I saw somewhere that you have tested it on WinXP. Any chance you could download an evaluation copy of a later version (Win7/8.1/2008R2/2012R2) and try again? Annex is a great tool and would like to incorporate it into our SCM and deployment workflow if possible. Thanks.
+"""]]
diff --git a/doc/forum/Windows_S3_host_issue.mdwn b/doc/forum/Windows_S3_host_issue.mdwn
new file mode 100644
index 0000000000..02cd879073
--- /dev/null
+++ b/doc/forum/Windows_S3_host_issue.mdwn
@@ -0,0 +1,11 @@
+I am trying to connect to dreamhost's objects (S3) platform.  I am using the windows version.  
+Am I specifying the host correctly? Any help is appreciated.
+
+$ git-annex initremote host="objects.dreamhost.com" cloud keyid=XXXXX type=S3
+
+initremote host=objects.dreamhost.com (encryption setup) (hybrid cipher with gpg
+ key 70827ADCDE25DA0F) (checking bucket...)
+git-annex: user error (openTCPConnection: host lookup failure for "host=objects.
+dreamhost.com-f5573b49-3668-4f94-a1da-aa55085c45e8.s3.amazonaws.com")
+failed
+git-annex.exe: initremote: 1 failed
diff --git a/doc/forum/Windows_S3_host_issue/comment_1_8c0a1e84713a04a25fdc1f74919d34aa._comment b/doc/forum/Windows_S3_host_issue/comment_1_8c0a1e84713a04a25fdc1f74919d34aa._comment
new file mode 100644
index 0000000000..7382cafa6e
--- /dev/null
+++ b/doc/forum/Windows_S3_host_issue/comment_1_8c0a1e84713a04a25fdc1f74919d34aa._comment
@@ -0,0 +1,10 @@
+[[!comment format=mdwn
+ username="http://joeyh.name/"
+ ip="209.250.56.87"
+ subject="comment 1"
+ date="2013-12-12T17:38:21Z"
+ content="""
+No, you cannot quote part of a parameter like that. git-annex thinks you are trying to use a host that includes the quote marks you put around it. 
+
+Since you're using Windows, I can't give any advice on how to use quoting at the shell/cmd prompt. However, I don't see any need for quoting here anyway.
+"""]]
diff --git a/doc/forum/Windows_S3_host_issue/comment_2_06ecc76797c430b27a2e24776761d043._comment b/doc/forum/Windows_S3_host_issue/comment_2_06ecc76797c430b27a2e24776761d043._comment
new file mode 100644
index 0000000000..3e918120ee
--- /dev/null
+++ b/doc/forum/Windows_S3_host_issue/comment_2_06ecc76797c430b27a2e24776761d043._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="https://www.google.com/accounts/o8/id?id=AItOawk_bIB1Vadfc_GfPLKX_ScfWFCcaGyk8Ac"
+ nickname="Rob"
+ subject="Remove quotes"
+ date="2013-12-13T04:44:22Z"
+ content="""
+I tried the command with and without quotes it results in the same error.  I think the issue is that the host is trying to resolve a concatenation of the hostname and amazon.com when in fact it should be resolving to the host name only.
+"""]]
diff --git a/doc/forum/Windows_S3_host_issue/comment_3_1715557daa15b9e9e17b4850141e62af._comment b/doc/forum/Windows_S3_host_issue/comment_3_1715557daa15b9e9e17b4850141e62af._comment
new file mode 100644
index 0000000000..e4645d07b0
--- /dev/null
+++ b/doc/forum/Windows_S3_host_issue/comment_3_1715557daa15b9e9e17b4850141e62af._comment
@@ -0,0 +1,10 @@
+[[!comment format=mdwn
+ username="http://joeyh.name/"
+ ip="209.250.56.87"
+ subject="comment 3"
+ date="2013-12-16T17:07:17Z"
+ content="""
+Looking at this again, you forgot to tell it the name of the remote, so it took the first parameter (host=\"objects.dreamhost.com\" ) as the name. The rest probably follows from that mistake.
+
+use `git annex initremote mys3 host=...`
+"""]]
diff --git a/doc/forum/Windows_error__58_____34__git__58_____39__annex__39___is_not_a_git_command.__34__.mdwn b/doc/forum/Windows_error__58_____34__git__58_____39__annex__39___is_not_a_git_command.__34__.mdwn
new file mode 100644
index 0000000000..99dcc799b0
--- /dev/null
+++ b/doc/forum/Windows_error__58_____34__git__58_____39__annex__39___is_not_a_git_command.__34__.mdwn
@@ -0,0 +1,6 @@
+Hi, I'm trying to use git annex through git bash on windows, I've installed git (32 bit) and git annex yet I get the error mentioned above.
+
+I've already had a search and have tried sorting out my windows environment paths as well as moving git annex .exe files from cmd to bin, neither of which have fixed the issue. Any ideas?
+
+Thanks
+Joe
diff --git a/doc/forum/Windows_error__58_____34__git__58_____39__annex__39___is_not_a_git_command.__34__/comment_1_b051dc30bf5a0fc9f5710c664f487ea9._comment b/doc/forum/Windows_error__58_____34__git__58_____39__annex__39___is_not_a_git_command.__34__/comment_1_b051dc30bf5a0fc9f5710c664f487ea9._comment
new file mode 100644
index 0000000000..eba1455f1f
--- /dev/null
+++ b/doc/forum/Windows_error__58_____34__git__58_____39__annex__39___is_not_a_git_command.__34__/comment_1_b051dc30bf5a0fc9f5710c664f487ea9._comment
@@ -0,0 +1,14 @@
+[[!comment format=mdwn
+ username="joey"
+ subject="""comment 1"""
+ date="2015-09-21T16:39:54Z"
+ content="""
+Please make sure you have Git For Windows, and the current version
+5.20150916 
+of git-annex installed. msysgit is no longer supported, and any
+older version of git-annex won't work with Git For Windows.
+
+Since you mention git-annex.exe in the cmd directory, I guess
+you probably have an old version installed. The current version
+does not put that file there.
+"""]]
diff --git a/doc/forum/Windows_error__58_____34__git__58_____39__annex__39___is_not_a_git_command.__34__/comment_2_8775a1483b29e2cc8f0456473c24e51e._comment b/doc/forum/Windows_error__58_____34__git__58_____39__annex__39___is_not_a_git_command.__34__/comment_2_8775a1483b29e2cc8f0456473c24e51e._comment
new file mode 100644
index 0000000000..d757ef7f1b
--- /dev/null
+++ b/doc/forum/Windows_error__58_____34__git__58_____39__annex__39___is_not_a_git_command.__34__/comment_2_8775a1483b29e2cc8f0456473c24e51e._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="dfiorentino@645c87e5e4e39b138b9717d01441dc2e3914fe01"
+ nickname="dfiorentino"
+ subject="“Windows error: "git: 'annex' is not a git command."” "
+ date="2015-09-22T18:54:38Z"
+ content="""
+I am having the same issue… “Windows error: \"git: 'annex' is not a git command.\"” I downloaded Git for Windows from https://downloads.kitenet.net/git-annex/windows/current/ and ran git-annex-installer.exe - 2015-09-16 11:43 	18M
+"""]]
diff --git a/doc/forum/Windows_error__58_____34__git__58_____39__annex__39___is_not_a_git_command.__34__/comment_3_cbaa3e0fa4e2347332b3929411f60d1d._comment b/doc/forum/Windows_error__58_____34__git__58_____39__annex__39___is_not_a_git_command.__34__/comment_3_cbaa3e0fa4e2347332b3929411f60d1d._comment
new file mode 100644
index 0000000000..e68f9b0809
--- /dev/null
+++ b/doc/forum/Windows_error__58_____34__git__58_____39__annex__39___is_not_a_git_command.__34__/comment_3_cbaa3e0fa4e2347332b3929411f60d1d._comment
@@ -0,0 +1,11 @@
+[[!comment format=mdwn
+ username="joey"
+ subject="""comment 3"""
+ date="2015-09-22T19:33:09Z"
+ content="""
+@dfiorentino um, are you sure you have Git For Windows installed, and not the old
+msysgit? What does git --version say?
+
+(You don't download Git For Windows from where you said you downloaded it,
+so I think you're confused about something.)
+"""]]
diff --git a/doc/forum/Windows_installation_notes.mdwn b/doc/forum/Windows_installation_notes.mdwn
new file mode 100644
index 0000000000..3fb25ec050
--- /dev/null
+++ b/doc/forum/Windows_installation_notes.mdwn
@@ -0,0 +1,146 @@
+[[!toc]]
+
+# Introduction
+
+Hello.
+
+I've installed git-annex and git-annex assistant on Windows 7 in a corp environment (hello gotchas!).
+
+In this post I'll describe issues I encountered, how I fixed them, recommendations I have for the installer, and some results from a couple `git annex test` runs.
+
+# Background
+
+My regular domain user doesn't have permissions to write to `C:\Program Files (x86)`, so I use a secondary domain user which is in the Administrators group.  I use "Run as different user" to run installers, etc.  (cf. "Run as Administrator")
+
+During msysgit installation I checked "only bash, don't add to path, don't integrate with Explorer" etc, since I like my third-party applications isolated.
+
+# The installer
+
+## Where to install `git-annex.exe`
+
+The nightly build of git-annex/assistant from NEST (20140908) only prompts for the base path of the msysgit location and it installs files in `$BASE/bin` and `$BASE/cmd`...  I'll try manually copying files post-install to mitigate the path issues described in other posts on this forum.
+
+The msysgit installer (1.9.4-preview20140815) presents a certain screen with three radio options:
+
+1. git bash only
+2. just git in `cmd.exe`
+3. git + unix tools in `cmd.exe`.
+
+I *think* this is the meaning of each:
+
+1. cmd.exe's PATH is not touched.
+2. `$GITBASE/cmd` is added to PATH
+3. `$GITBASE/bin` is added to PATH
+
+Therefore, I think that if you do something so that `git-annex.exe` is added to both $GITBASE/cmd and $GITBASE/bin (perhaps a symlink or even a .lnk file) then all three user preference options will be covered.
+
+All I did was copy `$BASE/cmd/git-annex.exe` to `$BASE/bin/git-annex.exe` and now both `git annex` and `git-annex` work in my msysgit "git bash" console.  I didn't test `cmd.exe` since I selected option 1 in the msysgit installer.
+
+## Installer locations: user profile or system-wide?
+
+I found a shortcut for the webapp in Start Menu/Startup ... for the wrong user.  Please prompt the user during the installation: "Install startup link system-wide or for current user?"
+
+# git annex test results
+
+## `$HOME` defaulted to some mapped drive, whoops!
+
+The test suite has been running since before I started this post.  Is that normal? :)
+
+I notice that it emits "Detected a crippled filesystem", "Enabling direct mode." and other messages again and again.  If those checks are expensive, maybe the result should be memoized/cached.
+
+Oh goodness, the test is reading and writing to my "home directory": a remote filesystem I never use.  It's slow.  I'll have to configure msysgit to use a different, more local `$HOME`.  This a common problem on this workstation.  I'll let the test finish in case it reveals something useful to you, but this will not be how I use it going forward...
+
+I am unable to attach `testWithMappedDriveHomeDirConsoleOutput.txt` to this post.  1 out of 84 tests failed.  Here is the only case sensitive occurrence of FAIL in the console output, with some lines of context.
+
+    OK
+        info:                                             Detected a filesystem without fifo support.
+      Disabling ssh connection caching.
+      Detected a crippled filesystem.
+      Enabling direct mode.
+    git-annex: Data.BloomFilter.Util.suggestSizing: capacity too large to represent
+    FAIL
+          Exception: user error (git-annex ["info","--json"] exited 1)
+        version:                                          Detected a filesystem without fifo support.
+      Disabling ssh connection caching.
+      Detected a crippled filesystem.
+      Enabling direct mode.
+    git-annex version: 5.20140908-g378fbb1
+    build flags: Assistant Webapp Webapp-secure Pairing Testsuite S3 WebDAV DNS Feeds Quvi TDFA CryptoHash
+
+## test results with local NTFS `$HOME`
+
+...The console output is scrolling by much more quickly.
+
+2 out of 84 tests failed.
+
+        prop_past_sane:                                 OK
+          +++ OK, passed 1000 tests.
+        prop_duration_roundtrips:                       OK
+          +++ OK, passed 1000 tests.
+        prop_metadata_sane:                             OK
+          +++ OK, passed 1000 tests.
+        prop_metadata_serialize:                        OK
+          +++ OK, passed 1000 tests.
+        prop_branchView_legal:                          OK
+          +++ OK, passed 1000 tests.
+        prop_view_roundtrips:                           OK
+          +++I nOiKt,  Tpeasstsse
+    d  1 0i0n0i tt:e sts.
+        prop_viewedFile_rountrips:                      FAIL
+          *** Failed! Falsifiable (after 51 tests and 1 shrink):
+          "a:"
+          Use --quickcheck-replay '50 592211036 1831676953' to reproduce.
+      Unit Tests
+        add sha1dup:                                    init test repo
+      Detected a filesystem without fifo support.
+    
+      Disabling ssh connection caching.
+
+and
+
+    OK
+        info:                                             Detected a filesystem without fifo support.
+      Disabling ssh connection caching.
+      Detected a crippled filesystem.
+      Enabling direct mode.
+    git-annex: Data.BloomFilter.Util.suggestSizing: capacity too large to represent
+    FAIL
+          Exception: user error (git-annex ["info","--json"] exited 1)
+        version:                                          Detected a filesystem without fifo support.
+      Disabling ssh connection caching.
+      Detected a crippled filesystem.
+      Enabling direct mode.
+    git-annex version: 5.20140908-g378fbb1
+    build flags: Assistant Webapp Webapp-secure Pairing Testsuite S3 WebDAV DNS Feeds Quvi TDFA CryptoHash
+
+...Note the corruption.  I think this happens when I drag the scroll bar while console output is being emitted.  (msysgit's problem?)  I would presume and hope that this is a "display only" issue. UPDATE: see section Corruption below.
+
+# .vbs failure
+
+I copied the `git-annex.lnk` out of my admin user's start menu onto my desktop and double clicked it.  `wscript.exe` got stuck in a loop where new copies were being spawned over and over again (and old copies dieing at the same rate).
+
+I think I know why.  `git-annex.exe` isn't on the path... but `git-annex.lnk` is in the CWD (Desktop in this case).  Yeah, that is the problem.  The vbs attempts to run "git-annex webapp", and this .lnk points to a valid "executable": `git-annex-webapp.vbs`...  So it just calls itself with an argument over and over again.
+
+Workaround: invoke `git annex webapp` from the normal git bash console.
+
+# Corruption?
+
+In some section above I speculated that the "jittery" corruption I was seeing in my console was a "display only" problem caused by scrolling around while new characters were being printed to the console.  Now, I don't think so.
+
+The corruption can be seen in the Log in the webapp.  Here's an example from the top of the log:
+
+    [2014-09-08 13:37:45 Central Daylight Time] main: starting assistant version 5.20140908-g378fbb1
+    Launching web browser on file://d:\annex\.git\annex\webapp.html
+    [2014-09-08 13:37:45 Central Daylight Time] Cronner: You should enable consistency checking to protect your data. 
+    (scanning...) [2014-09-08 13:37:45 Central Daylight Time] Watcher: Performing startup scan
+    (started...) rreerrcceevvcc::vv  ::ff  aaffiiaalliieellddee  dd((  NN((ooNN  ooee  rreerrrroorrrroo))rr
+    
+    ))
+
+I have no clue about this!  (Well...  "I think it's trying to communicate!") 
+
+# Conclusion
+
+I hope this information is helpful.  I've enabled the 'email comments to me' option on this post and I'd be happy to perform further tests upon request.
+
+Cheers!
diff --git a/doc/forum/Windows_installation_notes/comment_1_54c253ab61fea4f5aaefa04bb078c7b0._comment b/doc/forum/Windows_installation_notes/comment_1_54c253ab61fea4f5aaefa04bb078c7b0._comment
new file mode 100644
index 0000000000..7b26547a09
--- /dev/null
+++ b/doc/forum/Windows_installation_notes/comment_1_54c253ab61fea4f5aaefa04bb078c7b0._comment
@@ -0,0 +1,12 @@
+[[!comment format=mdwn
+ username="joey"
+ subject="""comment 1"""
+ date="2015-04-09T18:26:30Z"
+ content="""
+Not following the instructions to have git put itself into the path seems
+like a sure-fire way to shoot yourself in the foot.
+
+The loop problem is being discussed at
+
+perhaps you'll be able to answer my questions about it there?
+"""]]
diff --git a/doc/forum/Windows_installation_notes/comment_2_c8c2744e30f2950d9830936d2bd51617._comment b/doc/forum/Windows_installation_notes/comment_2_c8c2744e30f2950d9830936d2bd51617._comment
new file mode 100644
index 0000000000..cab6b1dfd8
--- /dev/null
+++ b/doc/forum/Windows_installation_notes/comment_2_c8c2744e30f2950d9830936d2bd51617._comment
@@ -0,0 +1,14 @@
+[[!comment format=mdwn
+ username="https://www.google.com/accounts/o8/id?id=AItOawkQOUUx4LVAk6EnstSLvdv7gZc0NsRlHXw"
+ nickname="Dave"
+ subject="comment 2"
+ date="2015-04-09T20:37:51Z"
+ content="""
+Sorry, joey, I haven't played with git annex on windows since my original post.
+
+I only have one new piece of information: some git annex related auto-start entry was added to the user profile of the account I used to perform the installation.  I noticed this when I logged in interactively as that user...
+
+See what I'm saying?  I have DOMAIN\dave.loyall and WORKSTATION\local.admin.  I have to use the latter to carry out installation, via \"Run as...\", but I need stuff to be installed into DOMAIN\dave.loyall's profile or the AllUsers profile (or whatever it is called).
+
+Meanwhile, more and more of my daily work is carried out in my GNU/Linux virtual machines.  I don't personally want/need anyone to prioritize windows-only deficiencies.
+"""]]
diff --git a/doc/forum/Windows_support.mdwn b/doc/forum/Windows_support.mdwn
new file mode 100644
index 0000000000..0e9e8dcb6e
--- /dev/null
+++ b/doc/forum/Windows_support.mdwn
@@ -0,0 +1,6 @@
+Hi,
+
+Do you have any news about Windows support?
+Is this something you're currently working on?
+
+Thanks!
diff --git a/doc/forum/Windows_support/comment_1_23fa9aa3b00940a1c1b3876c35eef019._comment b/doc/forum/Windows_support/comment_1_23fa9aa3b00940a1c1b3876c35eef019._comment
new file mode 100644
index 0000000000..95323ff997
--- /dev/null
+++ b/doc/forum/Windows_support/comment_1_23fa9aa3b00940a1c1b3876c35eef019._comment
@@ -0,0 +1,9 @@
+[[!comment format=mdwn
+ username="http://joey.kitenet.net/"
+ nickname="joey"
+ subject="comment 1"
+ date="2012-03-12T06:43:02Z"
+ content="""
+[[todo/windows_support]] has everything I know about making a windows port. This badly needs someone who understand Windows to dive into it. The question of how to create a symbolic link (or the relevant Windows equivilant) from haskell on Windows
+is a good starting point..
+"""]]
diff --git a/doc/forum/Windows_usage_instructions.mdwn b/doc/forum/Windows_usage_instructions.mdwn
new file mode 100644
index 0000000000..0b7b946251
--- /dev/null
+++ b/doc/forum/Windows_usage_instructions.mdwn
@@ -0,0 +1,25 @@
+Having a spot of bother in setting up for windows usage.
+
+I'm attempting to have a windows box syncing to a server (over ssh) and a linux box also syncing against that*
+
+So, on each machine I do
+
+ git init
+ git annex init
+
+
+On the windows and linux desktops I then do a 
+ 
+ git remote add server serverdetails.
+
+Now the problem is that if I don't add files to the repos on the machines, they won't sync as there is no branch checked out; and if I do then the first one is fine but the second will fail as it doesn't allow fast-forwards. What am I doing wrong? I've tried making the server repo bare / not bare.
+
+I'm using the latest nightly windows build, and a build from git from today (29d5bb94b4512cfe3072c9ff840cb0ce9f2af744)
+
+
+
+
+
+
+
+*Actually I'm trying to do something a little more complex than that, but this is the simplest version I can come up with.
diff --git a/doc/forum/Windows_usage_instructions/comment_1_d43dbd9406da3b9747b147715eca94ac._comment b/doc/forum/Windows_usage_instructions/comment_1_d43dbd9406da3b9747b147715eca94ac._comment
new file mode 100644
index 0000000000..6162b9540e
--- /dev/null
+++ b/doc/forum/Windows_usage_instructions/comment_1_d43dbd9406da3b9747b147715eca94ac._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="https://www.google.com/accounts/o8/id?id=AItOawmhfodZquCI_EEl-f3h7HkROTszlsQL6yA"
+ nickname="Joe"
+ subject="comment 1"
+ date="2013-06-16T11:29:36Z"
+ content="""
+This may be related to [this](http://git-annex.branchable.com/bugs/windows_port_-_repo_can__39__t_pull_newly_added_files_/#comment-a023dca06c2bcb9636d8a332f8e95ae4)
+"""]]
diff --git a/doc/forum/Wishlist__58___Bittorrent-like_transfers.mdwn b/doc/forum/Wishlist__58___Bittorrent-like_transfers.mdwn
new file mode 100644
index 0000000000..7ec6632169
--- /dev/null
+++ b/doc/forum/Wishlist__58___Bittorrent-like_transfers.mdwn
@@ -0,0 +1,5 @@
+**EDIT: Mistakenly posted this thread in the forum.  I created a new post in [[todo|todo/Bittorrent-like_features]]
+
+Do you think it would be possible to have bittorrent-like transfers between remotes, so that no one remote gets pegged too hard with transfers? It would be great if you distribute your files between multiple bandwidth-capped remotes, and want fast down speed.  Obviously, this isn't a simple task, but the protocol is already there, it just needs to be adapted for the purpose (and re-written in Haskell...).  Maybe some day in the future after the more important stuff gets taken care of?  It could be an enticing stretch goal.
+
+PS: still working on getting BTC, will be donating soon!
diff --git a/doc/forum/Wishlist__58___Bittorrent-like_transfers/comment_1_13544d54fb0418af4ca9200cdb045d91._comment b/doc/forum/Wishlist__58___Bittorrent-like_transfers/comment_1_13544d54fb0418af4ca9200cdb045d91._comment
new file mode 100644
index 0000000000..73295478f9
--- /dev/null
+++ b/doc/forum/Wishlist__58___Bittorrent-like_transfers/comment_1_13544d54fb0418af4ca9200cdb045d91._comment
@@ -0,0 +1,15 @@
+[[!comment format=mdwn
+ username="http://a-or-b.myopenid.com/"
+ ip="203.45.2.230"
+ subject="comment 1"
+ date="2013-07-19T02:33:36Z"
+ content="""
+I agree.  I mentioned it briefly here:
+
+[[http://git-annex.branchable.com/design/assistant/polls/what_is_preventing_me_from_using_git-annex_assistant/#comment-5a5d46967b826f602c423d7f72ac6f5e]]
+
+
+I did *briefly* look into torrents and the like, but couldn't work out a way to lock it down so only authorized users could use either the tracker or connect to pull the data down...
+
+The security aspect is the kicker in my mind.
+"""]]
diff --git a/doc/forum/Wishlist__58___Bittorrent-like_transfers/comment_2_9a7dad35bf80c684ad97892420d7370c._comment b/doc/forum/Wishlist__58___Bittorrent-like_transfers/comment_2_9a7dad35bf80c684ad97892420d7370c._comment
new file mode 100644
index 0000000000..affd07efd9
--- /dev/null
+++ b/doc/forum/Wishlist__58___Bittorrent-like_transfers/comment_2_9a7dad35bf80c684ad97892420d7370c._comment
@@ -0,0 +1,16 @@
+[[!comment format=mdwn
+ username="GLITTAH"
+ ip="37.130.227.133"
+ subject="comment 2"
+ date="2013-07-19T07:38:47Z"
+ content="""
+Okay I've got some ideas, but they need refining.  I should be posting more tomorrow.
+
+One thing I thought of though, there could be two types of torrent support:
+
+1.  You use a torrent file or magnet link as a special remote URL.  This remote is read-only, naturally, and you'd only be able to leech (and seed, I suppose) the file in the torrent.  Neat feature maybe.  I wasn't previously thinking of this.
+
+2. This isn't really bittorrent, just inspired by it.  You use a bittorrent-type protocol to transfer between your private remotes.  You could download from multiple remotes at once to get a file (like leeching, without a tracker, just pre-defined peers...aka remotes), or you could superseed to remotes to push a file, and have the remotes then share the parts of the file amongst themselves to each get a complete copy.  Remote groups could possibly be built on, or special groups could be coded in to define the swarm.  You update (add/remove) the swarm members locally, and push the changes to each of the old swarm members so they can add/remove other remotes (or themselves) from the swarm.  Possibly, you could just push the updated swarm list to one remote already in the swarm, it adopts the new list, and shares it with the rest of the swarm.  Each remote does the same.  Maybe the swarm list should be GPG-signed and verified before adopted and passed forward?
+
+The more I think about it, the more awesome I think it could be, but holy shit it would be a LOT of work.
+"""]]
diff --git a/doc/forum/Wishlist__58___Bittorrent-like_transfers/comment_3_e5de748bc5da12a4a01e08cde2407dd1._comment b/doc/forum/Wishlist__58___Bittorrent-like_transfers/comment_3_e5de748bc5da12a4a01e08cde2407dd1._comment
new file mode 100644
index 0000000000..ba8da25f03
--- /dev/null
+++ b/doc/forum/Wishlist__58___Bittorrent-like_transfers/comment_3_e5de748bc5da12a4a01e08cde2407dd1._comment
@@ -0,0 +1,14 @@
+[[!comment format=mdwn
+ username="https://www.google.com/accounts/o8/id?id=AItOawmLB39PC89rfGaA8SwrsnB6tbumezj-aC0"
+ nickname="Tobias"
+ subject="comment 3"
+ date="2013-07-19T07:50:09Z"
+ content="""
+Personally i think the easiest implementation to make would be smart transfers.
+
+You have a repo on your laptop, and a repo at work. And you have(say) 5 special remotes in between them that they need to use to transfer data.
+
+The laptop could upload 5 different files to those repositories. Wouldn't quite be bittorent transfer(not even close). But it would probably be much simpler to implement. And when syncing a lot of files(instead of just one really big one) the speed gain should really be about the same as if all the special remotes used some kind of bittorent thing.
+
+Just a thought.
+"""]]
diff --git a/doc/forum/Wishlist__58___Bittorrent-like_transfers/comment_4_e51530178f1e034c0fdd5c9aa9945567._comment b/doc/forum/Wishlist__58___Bittorrent-like_transfers/comment_4_e51530178f1e034c0fdd5c9aa9945567._comment
new file mode 100644
index 0000000000..d3e0d33145
--- /dev/null
+++ b/doc/forum/Wishlist__58___Bittorrent-like_transfers/comment_4_e51530178f1e034c0fdd5c9aa9945567._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="https://www.google.com/accounts/o8/id?id=AItOawmLB39PC89rfGaA8SwrsnB6tbumezj-aC0"
+ nickname="Tobias"
+ subject="comment 4"
+ date="2013-07-19T07:51:39Z"
+ content="""
+Of course all files should be uploaded to all remote(to match with numcopies). but the initial push could be spread out over all existing remotes, to facility faster initial download/transfer.
+"""]]
diff --git a/doc/forum/Wishlist__58___Bittorrent-like_transfers/comment_5_81ea9c129d8c02097f09ef8c68f1bb11._comment b/doc/forum/Wishlist__58___Bittorrent-like_transfers/comment_5_81ea9c129d8c02097f09ef8c68f1bb11._comment
new file mode 100644
index 0000000000..1ef2b25ac3
--- /dev/null
+++ b/doc/forum/Wishlist__58___Bittorrent-like_transfers/comment_5_81ea9c129d8c02097f09ef8c68f1bb11._comment
@@ -0,0 +1,27 @@
+[[!comment format=mdwn
+ username="GLITTAH"
+ ip="37.130.227.133"
+ subject="comment 5"
+ date="2013-07-19T21:23:35Z"
+ content="""
+Disclaimer: I'm thinking out loud of what could make git-annex even more awesome.  I don't expect this to be implemented any time soon.  Please pardon any dumbassery.
+
+Much easier to implement, but having your remotes (optionally!) act like a swarm would be an awesome feature to have because you bring in a lot of new features that optimize storage, bandwidth, and overall traffic usage.  This would be made a lot easier if parts of it were implemented in small steps that added a nifty feature.  The best part is, each of these could be implemented by themselves, and they're all features that would be really useful.
+
+Step 1. Concurrent downloads of a file from remotes.
+
+This would make sense to have, it saves upload traffic on your remotes, and you also get faster DL speeds on the receiving end.
+
+
+Step 2. Implementing part of the super-seeding capabilities.
+
+You upload pieces of a file to different remotes from your laptop, and on your desktop you can download all those pieces and put them together again to get a complete file.  If you *really* wanted to get fancy, you could build in redundancy (ala RAID) so if a remote or two gets lost, you don't lose the entire file.  This would be a very efficient use of storage if you have a bunch of free cloud storage accounts (~1GB each) and some big files you want to back up.
+
+
+Step 3. Setting it up so that those remotes could talk to one another and share those pieces.  
+
+This is where it gets more like bittorrent.  Useful because you upload one copy and in a few hours, have say, 5 complete copies spread across your remotes.  You could add or remove remotes from a swarm locally, and push those changes to those remotes, which then adapt themselves to suit the new rules and share those with other remotes in the swarm (rules should be GPG-signed as a safety precaution).  Also, if/when deltas get implemented, you could push that delta to the swarm and have all the remotes adopt it.  This is cooler than regular bittorrent because the shared file can be updated.  As a safety precaution, the delta could be GPG signed so a corrupt file doesn't contaminate the entire swarm.  Each remote could have bandwidth/storage limits set in a dotfile.
+
+This is a high-level idea of how it might work, and it's also a HUGE set of features to add, but if implemented, you'd be saving a ton of resources, adding new use cases, and making git-annex more flexible.
+
+"""]]
diff --git a/doc/forum/Wishlist__58___Bittorrent-like_transfers/comment_6_3b5798414f89686526da3dfa72c0c4f2._comment b/doc/forum/Wishlist__58___Bittorrent-like_transfers/comment_6_3b5798414f89686526da3dfa72c0c4f2._comment
new file mode 100644
index 0000000000..7a027c909c
--- /dev/null
+++ b/doc/forum/Wishlist__58___Bittorrent-like_transfers/comment_6_3b5798414f89686526da3dfa72c0c4f2._comment
@@ -0,0 +1,10 @@
+[[!comment format=mdwn
+ username="GLITTAH"
+ ip="37.130.227.133"
+ subject="comment 6"
+ date="2013-07-19T21:28:58Z"
+ content="""
+Obviously, Step 3 would only work on remotes that you have control of processes on, but if given login credentials to cloud storage remotes (potentially dangerous!) they could read/write to something like dropbox or rsync.
+
+Another thing, this would be completely trackerless.  You just use remote groups (or create swarm definitions) and share those with your remotes.  It's completely decentralized!
+"""]]
diff --git a/doc/forum/Wishlist__58___Don__39__t_make_files_readonly.mdwn b/doc/forum/Wishlist__58___Don__39__t_make_files_readonly.mdwn
new file mode 100644
index 0000000000..4d67f93b1a
--- /dev/null
+++ b/doc/forum/Wishlist__58___Don__39__t_make_files_readonly.mdwn
@@ -0,0 +1,3 @@
+Maybe this explained somewhere else, but what is the purpose of marking files readonly. To me this is an annoyance that doesn't make sense. I want to be able to change my files, having to go through an unlock step, to do that seems like unnecessary work. Interestingly, Microsoft's Team Foundation Server source control does the same thing and I don't like it there either.
+
+In addition why replace files with symlinks? Why not just leave the real files in place, or do the reverse and put the symlink to the file in the repository.
diff --git a/doc/forum/Wishlist__58___Don__39__t_make_files_readonly/comment_1_7148527961e2d27793810966588c8d35._comment b/doc/forum/Wishlist__58___Don__39__t_make_files_readonly/comment_1_7148527961e2d27793810966588c8d35._comment
new file mode 100644
index 0000000000..b7a6f55f47
--- /dev/null
+++ b/doc/forum/Wishlist__58___Don__39__t_make_files_readonly/comment_1_7148527961e2d27793810966588c8d35._comment
@@ -0,0 +1,10 @@
+[[!comment format=mdwn
+ username="https://www.google.com/accounts/o8/id?id=AItOawmL8pteP2jbYJUn1M3CbeLDvz2SWAA1wtg"
+ nickname="Kristian"
+ subject="comment 1"
+ date="2012-11-22T23:38:33Z"
+ content="""
+By using symlinks gits problems with big files goes away. You simply can't put 600 GB in a git repo - that's the way I use git-annex. Also having files read only by default eliminates the need to store two copies of all files in one form or another.
+
+
+"""]]
diff --git a/doc/forum/Wishlist__58___Is_it_possible_to___34__unlock__34___files_without_copying_the_file_data__63__.mdwn b/doc/forum/Wishlist__58___Is_it_possible_to___34__unlock__34___files_without_copying_the_file_data__63__.mdwn
new file mode 100644
index 0000000000..1a7930fec4
--- /dev/null
+++ b/doc/forum/Wishlist__58___Is_it_possible_to___34__unlock__34___files_without_copying_the_file_data__63__.mdwn
@@ -0,0 +1,6 @@
+I have a DLink Boxee media player and it can not play content from symbolic links, it needs to access regular media files. Unfortunately unlocking/locking is quite slow for such a large amount of data due to the required data copying, but it should not even be needed since I do not need write access to any file to watch the movie or to play the song.
+
+Is it currently possible or would it be possible to add a commands like "unlock" which would not copy the file data but simply move files out from the data store into the tree while still keeping the files read only? A corresponding "lock" command would also be needed to restore the normal symbolic link tree structure.
+
+Update:
+I tried the rsync special remote http://git-annex.branchable.com/special_remotes/rsync/ and it works but the file structure created reflects the data store not the view given by the symbolic links.
diff --git a/doc/forum/Wishlist__58___Is_it_possible_to___34__unlock__34___files_without_copying_the_file_data__63__/comment_1_1cf4ab29dfa2cff59b86305fc0018251._comment b/doc/forum/Wishlist__58___Is_it_possible_to___34__unlock__34___files_without_copying_the_file_data__63__/comment_1_1cf4ab29dfa2cff59b86305fc0018251._comment
new file mode 100644
index 0000000000..3ab518714d
--- /dev/null
+++ b/doc/forum/Wishlist__58___Is_it_possible_to___34__unlock__34___files_without_copying_the_file_data__63__/comment_1_1cf4ab29dfa2cff59b86305fc0018251._comment
@@ -0,0 +1,10 @@
+[[!comment format=mdwn
+ username="http://joey.kitenet.net/"
+ nickname="joey"
+ subject="comment 1"
+ date="2011-07-07T15:27:28Z"
+ content="""
+The rsync or directory special remotes would work if the media player uses metadata in the files, rather than directory locations.
+
+Beyond that there is the [[todo/smudge]] idea, which is hoped to be supported sometime.
+"""]]
diff --git a/doc/forum/Wishlist__58___Is_it_possible_to___34__unlock__34___files_without_copying_the_file_data__63__/comment_2_f5ebb7f43dcef861ecc13373fb1e263f._comment b/doc/forum/Wishlist__58___Is_it_possible_to___34__unlock__34___files_without_copying_the_file_data__63__/comment_2_f5ebb7f43dcef861ecc13373fb1e263f._comment
new file mode 100644
index 0000000000..9601003798
--- /dev/null
+++ b/doc/forum/Wishlist__58___Is_it_possible_to___34__unlock__34___files_without_copying_the_file_data__63__/comment_2_f5ebb7f43dcef861ecc13373fb1e263f._comment
@@ -0,0 +1,15 @@
+[[!comment format=mdwn
+ username="https://www.google.com/accounts/o8/id?id=AItOawmL8pteP2jbYJUn1M3CbeLDvz2SWAA1wtg"
+ nickname="Kristian"
+ subject="Solution"
+ date="2011-07-31T15:24:25Z"
+ content="""
+Yes, it can read id3-tags and guess titles from movie filenames but it sometimes gets confused by the filename metadata provided by the WORM-backend.
+
+I think I have a good enough solution to this problem. It's not efficient when it comes to renames but handles adding and deletion just fine
+
+    rsync -vaL --delete source dest
+
+The -L flag looks at symbolic links and copies the actual data they are pointing to. Of course \"source\" must have all data locally for this to work.
+
+"""]]
diff --git a/doc/forum/Wishlist__58___Ways_of_selecting_files_based_on_meta-information.mdwn b/doc/forum/Wishlist__58___Ways_of_selecting_files_based_on_meta-information.mdwn
new file mode 100644
index 0000000000..1de06f7cda
--- /dev/null
+++ b/doc/forum/Wishlist__58___Ways_of_selecting_files_based_on_meta-information.mdwn
@@ -0,0 +1,15 @@
+It would be extremely useful to have some additional ways to select files (for git annex copy/move/get and maybe others) based on the meta-information available to git-annex, rather than just by file or directory name.
+
+An example of what I'd like to do is this:
+
+    host1$ git annex copy --to usb-drive --missing-on host2
+
+This would check location tracking information and copy each file from host1's annex which is not present on host2 onto the usb-drive annex -- i.e. it's what I want when I need to do a sneakernet synchronisation of host1 and host2 (for backup purposes, for example). Note that of course I could copy --to host2, assuming network connectivity, but that would take a long time.
+
+There's probably other selectors that we can imagine; an obvious one could be --present-on  -- useful for judiciously dropping only those files that you have easily available in a local annex (as you may want to keep files that are hard to make available even if --numcopies would nominally be satisfied).
+
+Other similar ideas for file content selectors:
+
+ * Files that have less than n, exactly n or more than n copies -- for when you need to satisfy your --numcopies policy over sneakernet.
+ * Files that are present (or not present) on some trusted annex -- for making sure you have trusted copies of everything.
+ * Boolean combinations of these filters -- "git annex drop --present-on lanserver1 --or --present-on lanserver2" or similar syntax, although obviously doing this in full generality may be quite fiddly.
diff --git a/doc/forum/Wishlist__58___Ways_of_selecting_files_based_on_meta-information/comment_1_818f38aa988177d3a9415055e084f0fb._comment b/doc/forum/Wishlist__58___Ways_of_selecting_files_based_on_meta-information/comment_1_818f38aa988177d3a9415055e084f0fb._comment
new file mode 100644
index 0000000000..11b44b8094
--- /dev/null
+++ b/doc/forum/Wishlist__58___Ways_of_selecting_files_based_on_meta-information/comment_1_818f38aa988177d3a9415055e084f0fb._comment
@@ -0,0 +1,15 @@
+[[!comment format=mdwn
+ username="http://christian.amsuess.com/chrysn"
+ nickname="chrysn"
+ subject="filtering based on git-commits"
+ date="2011-06-23T13:56:35Z"
+ content="""
+additional filter criteria could come from the git history:
+
+* `git annex get --touched-in HEAD~5..` to fetch what has recently been worked on
+* `git annex get --touched-by chrysn --touched-in version-1.0..HEAD` to fetch what i've been workin on recently (based on regexp or substring match in author; git experts could probably craft much more meaningful expressions)
+
+these options could also apply to `git annex find` -- actually, looking at the normal file system tools for such tasks, that might even be sufficient (think `git annex find --numcopies-gt 3 --present-on lanserver1 --drop` like `find -iname '*foo*' -delete`
+
+(i was about to open a new forum discussion for commit-based getting, but this is close enough to be usefully joint in a discussion)
+"""]]
diff --git a/doc/forum/Wishlist__58___Ways_of_selecting_files_based_on_meta-information/comment_2_97e2ed48bd552d02918c4f98f963e6e1._comment b/doc/forum/Wishlist__58___Ways_of_selecting_files_based_on_meta-information/comment_2_97e2ed48bd552d02918c4f98f963e6e1._comment
new file mode 100644
index 0000000000..787cf8f5d7
--- /dev/null
+++ b/doc/forum/Wishlist__58___Ways_of_selecting_files_based_on_meta-information/comment_2_97e2ed48bd552d02918c4f98f963e6e1._comment
@@ -0,0 +1,9 @@
+[[!comment format=mdwn
+ username="http://joey.kitenet.net/"
+ nickname="joey"
+ subject="comment 2"
+ date="2011-09-19T18:46:35Z"
+ content="""
+This is now almost completely implemented. See [[walkthrough/powerful_file_matching]].
+
+"""]]
diff --git a/doc/forum/Wishlist__58___automatic_reinject.mdwn b/doc/forum/Wishlist__58___automatic_reinject.mdwn
new file mode 100644
index 0000000000..f975c75214
--- /dev/null
+++ b/doc/forum/Wishlist__58___automatic_reinject.mdwn
@@ -0,0 +1,14 @@
+I think it would be useful to supplement the `reinject` command with an automatic
+mode which calculates the checksum of the source file and injects the file if it
+is known to the repository (without the need to provide a destination filename).
+In addition, this could be done recursively if the user provides a directory to
+inject. All this can probably be done already with some plumbing, but a simple
+`reinject --auto` (or `scour`, or `scavenge`, if you like) would be a nice addition.
+Of course this would only work for the checksum backends.
+
+Example use cases would be:
+
+* Recovering data from lost+found easily
+* Making use of old (pre-git-annex) archival volumes with useful files
+  scattered among non-useful files
+* Sneaker-netting files between disconnected git-annex repositories
diff --git a/doc/forum/Wishlist__58___getting_the_disk_used_by_a_subtree_of_files.mdwn b/doc/forum/Wishlist__58___getting_the_disk_used_by_a_subtree_of_files.mdwn
new file mode 100644
index 0000000000..7bdd93654f
--- /dev/null
+++ b/doc/forum/Wishlist__58___getting_the_disk_used_by_a_subtree_of_files.mdwn
@@ -0,0 +1,10 @@
+I'm not sure if this _feature_ exists already wrapped or provided as a recipe for users or not yet. But it would be nice to be able to do a
+
+    git annex du [PATH]
+
+Such that the output that git annex would return is the total disk used locally in the PATH and the theoretical disk used by the PATH if it was fully populated locally. e.g.
+
+    $ git annex du FSL0001_ANALYSIS
+    $ Local: 1000kb, Annex: 2000kb
+
+or something along the lines of that?
diff --git a/doc/forum/Wishlist__58___getting_the_disk_used_by_a_subtree_of_files/comment_1_7abb1155081a23ce4829ee69b2064541._comment b/doc/forum/Wishlist__58___getting_the_disk_used_by_a_subtree_of_files/comment_1_7abb1155081a23ce4829ee69b2064541._comment
new file mode 100644
index 0000000000..bff5b2ea7e
--- /dev/null
+++ b/doc/forum/Wishlist__58___getting_the_disk_used_by_a_subtree_of_files/comment_1_7abb1155081a23ce4829ee69b2064541._comment
@@ -0,0 +1,9 @@
+[[!comment format=mdwn
+ username="http://joeyh.name/"
+ ip="4.153.2.25"
+ subject="comment 1"
+ date="2012-06-27T12:36:08Z"
+ content="""
+Use `du -L` for the disk space used locally. The other number is not currently available, but it would be nice to have. I also sometimes would like to have data on which backends are used how much, so making this `git annex status --subdir` is tempting. Unfortunatly, it's current implementation scans `.git/annex/objects`
+and not the disk tree (better for accurate numbers due to copies), so it would not be a very easy thing to add. Not massively hard, but not something I can pound out before I start work today..
+"""]]
diff --git a/doc/forum/Wishlist__58___getting_the_disk_used_by_a_subtree_of_files/comment_2_b4c6ebada7526263e04c70eac312fda9._comment b/doc/forum/Wishlist__58___getting_the_disk_used_by_a_subtree_of_files/comment_2_b4c6ebada7526263e04c70eac312fda9._comment
new file mode 100644
index 0000000000..551c685d4e
--- /dev/null
+++ b/doc/forum/Wishlist__58___getting_the_disk_used_by_a_subtree_of_files/comment_2_b4c6ebada7526263e04c70eac312fda9._comment
@@ -0,0 +1,18 @@
+[[!comment format=mdwn
+ username="https://www.google.com/accounts/o8/id?id=AItOawmBUR4O9mofxVbpb8JV9mEbVfIYv670uJo"
+ nickname="Justin"
+ subject="comment 2"
+ date="2012-06-27T12:45:42Z"
+ content="""
+I have a hacked up version of sharebox that does this.. I need to fix it up and push it to github..
+
+the short of it is that you can do
+
+    def calculate_size(path):
+        annexfile = os.path.basename(os.readlink(path))
+        #SHA256-s2007550713--....
+        size = annexfile.split(\"-\")[1]
+        return int(size[1:])
+
+to get the size of files.. a 'git-annex du' should be pretty straightforward...
+"""]]
diff --git a/doc/forum/Wishlist__58___getting_the_disk_used_by_a_subtree_of_files/comment_3_ded71b270b94617a8ebb3a713d46a274._comment b/doc/forum/Wishlist__58___getting_the_disk_used_by_a_subtree_of_files/comment_3_ded71b270b94617a8ebb3a713d46a274._comment
new file mode 100644
index 0000000000..ddc9ae1475
--- /dev/null
+++ b/doc/forum/Wishlist__58___getting_the_disk_used_by_a_subtree_of_files/comment_3_ded71b270b94617a8ebb3a713d46a274._comment
@@ -0,0 +1,19 @@
+[[!comment format=mdwn
+ username="http://joeyh.name/"
+ nickname="joey"
+ subject="comment 3"
+ date="2013-03-11T05:27:38Z"
+ content="""
+You can now use \"git annex status .\"
+
+Example:
+
+
    +git annex status roms
+directory: roms
+local annex keys: 1
+local annex size: 248 megabytes
+known annex keys: 277
+known annex size: 12 gigabytes
+
    
+"""]]
diff --git a/doc/forum/Wishlist__58___logging_to_file_when_running_as_a_daemon___40__for_the_assistant__41__.mdwn b/doc/forum/Wishlist__58___logging_to_file_when_running_as_a_daemon___40__for_the_assistant__41__.mdwn
new file mode 100644
index 0000000000..c3f915e7dd
--- /dev/null
+++ b/doc/forum/Wishlist__58___logging_to_file_when_running_as_a_daemon___40__for_the_assistant__41__.mdwn
@@ -0,0 +1 @@
+Logging to file would be nice when running git-annex as a daemon so when something fails or when certain events happens one can look at the logs to see what has failed, unless I'm missing something this probably should be on the wishlist or roadmap.
diff --git a/doc/forum/Wishlist__58___logging_to_file_when_running_as_a_daemon___40__for_the_assistant__41__/comment_1_42aa2b61b880f4048d874210212aa63b._comment b/doc/forum/Wishlist__58___logging_to_file_when_running_as_a_daemon___40__for_the_assistant__41__/comment_1_42aa2b61b880f4048d874210212aa63b._comment
new file mode 100644
index 0000000000..c0746a5a7c
--- /dev/null
+++ b/doc/forum/Wishlist__58___logging_to_file_when_running_as_a_daemon___40__for_the_assistant__41__/comment_1_42aa2b61b880f4048d874210212aa63b._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="http://joeyh.name/"
+ ip="4.153.14.60"
+ subject="comment 1"
+ date="2012-06-23T14:30:22Z"
+ content="""
+The logging format could be improved, but the daemon already logs to .git/annex/daemon.log. It also automatically rotates the log file.
+"""]]
diff --git a/doc/forum/Wishlist__58___logging_to_file_when_running_as_a_daemon___40__for_the_assistant__41__/comment_2_3e201039fa0e611554171ee30e69a414._comment b/doc/forum/Wishlist__58___logging_to_file_when_running_as_a_daemon___40__for_the_assistant__41__/comment_2_3e201039fa0e611554171ee30e69a414._comment
new file mode 100644
index 0000000000..b1a76ce2d3
--- /dev/null
+++ b/doc/forum/Wishlist__58___logging_to_file_when_running_as_a_daemon___40__for_the_assistant__41__/comment_2_3e201039fa0e611554171ee30e69a414._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="https://www.google.com/accounts/o8/id?id=AItOawkSq2FDpK2n66QRUxtqqdbyDuwgbQmUWus"
+ nickname="Jimmy"
+ subject="comment 1"
+ date="2012-06-23T08:00:12Z"
+ content="""
+actually, scratch that, i found it. it was in _.git/annex/daemon.log_ along with the other bits and pieces
+"""]]
diff --git a/doc/forum/Wishlist__58___logging_to_file_when_running_as_a_daemon___40__for_the_assistant__41__/comment_3_d1074724c44f3296cb438b2d526d8728._comment b/doc/forum/Wishlist__58___logging_to_file_when_running_as_a_daemon___40__for_the_assistant__41__/comment_3_d1074724c44f3296cb438b2d526d8728._comment
new file mode 100644
index 0000000000..eb18628290
--- /dev/null
+++ b/doc/forum/Wishlist__58___logging_to_file_when_running_as_a_daemon___40__for_the_assistant__41__/comment_3_d1074724c44f3296cb438b2d526d8728._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="https://www.google.com/accounts/o8/id?id=AItOawkSq2FDpK2n66QRUxtqqdbyDuwgbQmUWus"
+ nickname="Jimmy"
+ subject="comment 2"
+ date="2012-07-03T14:48:16Z"
+ content="""
+Adding a date and timestamp would be a nice start to improving things.
+"""]]
diff --git a/doc/forum/Wishlist__58___mark_remotes_offline.mdwn b/doc/forum/Wishlist__58___mark_remotes_offline.mdwn
new file mode 100644
index 0000000000..046c62210f
--- /dev/null
+++ b/doc/forum/Wishlist__58___mark_remotes_offline.mdwn
@@ -0,0 +1,12 @@
+I have several remotes which are not always accessible. For example they can
+be on hosts only accessible by LAN or on a portable hard drive which is not
+plugged in. When running sync these remotes are checked as well, leading to
+unnecessary error messages and possibly git-annex waiting for a few minutes
+on each remote for a timeout.
+
+In this situation it would be useful to mark some remotes as offline
+(`git annex offline `), so that git-annex would not even attempt
+to contact them. Then, I could configure my system to automatically, for example,
+mark a portable hard disk remote online when plugging it in, and offline when
+unplugging it, and similarly marking remotes offline and online depending on
+whether I have an internet connection or a connection to a specific network.
diff --git a/doc/forum/Wishlist__58___mark_remotes_offline/comment_1_9e3901f0123abb66034cce95cc5a941a._comment b/doc/forum/Wishlist__58___mark_remotes_offline/comment_1_9e3901f0123abb66034cce95cc5a941a._comment
new file mode 100644
index 0000000000..c24a786c96
--- /dev/null
+++ b/doc/forum/Wishlist__58___mark_remotes_offline/comment_1_9e3901f0123abb66034cce95cc5a941a._comment
@@ -0,0 +1,14 @@
+[[!comment format=mdwn
+ username="http://joeyh.name/"
+ subject="comment 1"
+ date="2012-07-06T13:04:07Z"
+ content="""
+You can already do this:
+
+    git config remote.foo.annex-ignore true
+
+There's no need to do anything for portable drives that are sometimes mounted and sometimes not -- git-annex will automatically avoid using repositories in directories that do not currently exist.
+
+I thought git-annex also had a way to run a command and use its exit status to control whether a repo was
+ignored or not, but it seems I never actually implemented that. It might be worth adding, although the command would necessarily run whenever git-annex is transferring data around.
+"""]]
diff --git a/doc/forum/Wishlist__58___mark_remotes_offline/comment_2_d10e3d90cf421ae425e64ab266ea811b._comment b/doc/forum/Wishlist__58___mark_remotes_offline/comment_2_d10e3d90cf421ae425e64ab266ea811b._comment
new file mode 100644
index 0000000000..e14cfa8225
--- /dev/null
+++ b/doc/forum/Wishlist__58___mark_remotes_offline/comment_2_d10e3d90cf421ae425e64ab266ea811b._comment
@@ -0,0 +1,12 @@
+[[!comment format=mdwn
+ username="https://www.google.com/accounts/o8/id?id=AItOawncBlzaDI248OZGjKQMXrLVQIx4XrZrzFo"
+ nickname="Perttu"
+ subject="comment 2"
+ date="2012-07-07T17:45:43Z"
+ content="""
+Ah, I didn't read the man page carefully enough. My apologies.
+
+Setting the ignore status based on an exit status would be
+even better, since this avoids re-writing a new config file for
+each repository each time I enter or exit my LAN.
+"""]]
diff --git a/doc/forum/Wishlist__58___options_for_syncing_meta-data_and_data.mdwn b/doc/forum/Wishlist__58___options_for_syncing_meta-data_and_data.mdwn
new file mode 100644
index 0000000000..d1df6628ea
--- /dev/null
+++ b/doc/forum/Wishlist__58___options_for_syncing_meta-data_and_data.mdwn
@@ -0,0 +1,13 @@
+Since _transfer queueing_  and syncing of data works now in the assistant branch (been playing with it), there are times when I really don't want to sync the data, I would like to just sync meta-data and manually do a _get_ on files that I would want or selectively sync data in a subtree.
+
+It would be nice to have the syncing/watch feature to have the option of syncing only *meta-data* or *meta-data and data*, I think this sort of option was already planned? It would also be nice to be able to automatically sync data for only a subtree.
+
+My use case is, I have a big stash of files somewhere at home or work, and I want to keep what I am actually using on my laptop and be able to selectively just take a subtree or a set of subtree's of files. I would not always want to suck down all the data but still have the functionally to add files and push them upstream and sync meta-data.
+
+that is...
+
+> * Site A: big master annex in a server room with lots of disk (or machines), watches a directory and syncs both data and meta-data, it should always try and pull data from all it's child repos. That way I will always have a master copy of my data somewhere, it would be even nicer if I could have clones of the annex, where each annex is on a different machine which is configured to only sync a subtree of files so I can distribute my annex across different systems and disks.
+>   * Site A: machine A: syncs Folder A
+>   * Site A: machine B: syncs Folder B
+>   * and so on with selectively syncing sites and directories
+> * Laptop: has a clone of the annex, and watches a directory, syncs meta-data as usual and only uploads files to a remote (all or a designated one) but it never downloads files automatically or it should only occur inside a selected subtree.
diff --git a/doc/forum/Wishlist__58___rename_files__47__dirs_w__47___special_characters_if_filesystem_is_FAT.mdwn b/doc/forum/Wishlist__58___rename_files__47__dirs_w__47___special_characters_if_filesystem_is_FAT.mdwn
new file mode 100644
index 0000000000..847ff01bd5
--- /dev/null
+++ b/doc/forum/Wishlist__58___rename_files__47__dirs_w__47___special_characters_if_filesystem_is_FAT.mdwn
@@ -0,0 +1,5 @@
+Hi Joey,
+    I recently tried cloning a music annex to a microSD card to play on rockbox (FAT only) with my Sansa Clip, but 'git clone' stops the first time it runs into something with a special character in a file or directory, like a colon in an album title.  After this happens, there doesn't seem to be a way to get a full annex file/directory listing for the annex; it only has up to where git failed since FAT can't handle the name.  Is it possible for git-annex to replace special characters in directory/filenames with a space or underscore if it's on a crippled filesystem?  Also, would it be possible to then clone from that annex to a non-crippled FS, preserving the original filenames with special characters?  I've been meaning to ask about this for a while, so I'm a teeny bit fuzzy on the specifics...if you want, I'll reproduce the problem and cough up some more details.
+
+
+PS: I love git-annex!
diff --git a/doc/forum/Wishlist__58___rename_files__47__dirs_w__47___special_characters_if_filesystem_is_FAT/comment_1_5d33bcbd862537f53edd91dcff2b8977._comment b/doc/forum/Wishlist__58___rename_files__47__dirs_w__47___special_characters_if_filesystem_is_FAT/comment_1_5d33bcbd862537f53edd91dcff2b8977._comment
new file mode 100644
index 0000000000..42fad62b9b
--- /dev/null
+++ b/doc/forum/Wishlist__58___rename_files__47__dirs_w__47___special_characters_if_filesystem_is_FAT/comment_1_5d33bcbd862537f53edd91dcff2b8977._comment
@@ -0,0 +1,13 @@
+[[!comment format=mdwn
+ username="http://joeyh.name/"
+ nickname="joey"
+ subject="comment 1"
+ date="2013-04-06T21:13:44Z"
+ content="""
+I think this is a general git question, not specific to git-annex. At the point in time that you run `git clone` on the FAT filesystem, git-annex is not running at all.
+
+You could ask the git community about this. I know that git already has a core.precomposeunicode setting that is used to work around a somewhat similar problem on Mac OS.
+
+What git-annex *does* let you do is set up a [[directory_special_remote|special_remote/directory]] on the FAT filesystem. This way,
+there is no git repository there, and the filenames stored in git are not used at all on FAT. If your media player looks at mp3 metadata to get song titles, etc and doesn't care about the filenames at all, this would be a good option.
+"""]]
diff --git a/doc/forum/Workflow_for_adding_files.mdwn b/doc/forum/Workflow_for_adding_files.mdwn
new file mode 100644
index 0000000000..cb631543fb
--- /dev/null
+++ b/doc/forum/Workflow_for_adding_files.mdwn
@@ -0,0 +1,32 @@
+What is the correct way to add/modify files such that the changes are synchronized?
+
+Suppose I have a server "server" and a windows client "client" (more after some time); everything direct mode.
+
+I created the repos on the server:
+
+     cd bin
+     git init
+     git annex init server
+     git annex direct
+     git annex add .
+     git annex sync
+
+On the (Windows) client using cygwin:
+
+     git annex clone ssh://me@server:/srv/bin
+     cd bin
+     git annex init client
+
+Then I want to add files on the client:
+
+     git annex add tools
+     git annex sync
+
+At this point I wonder that the data is not copied to the server but only metadata!
+I then used
+
+     git annex sync --content
+
+but then everything from the server is leeched as well (as if I could have called "git annex get .")
+
+What is the intented workflow such that added/modified files on the client always go to the server, modified/new files from the server are always pulled but ONLY if I previously got the file/directory via "get annex get"?
diff --git a/doc/forum/Workflow_for_adding_files/comment_1_a60dae97db827bc641d6256d1f382b5f._comment b/doc/forum/Workflow_for_adding_files/comment_1_a60dae97db827bc641d6256d1f382b5f._comment
new file mode 100644
index 0000000000..9774e69bf7
--- /dev/null
+++ b/doc/forum/Workflow_for_adding_files/comment_1_a60dae97db827bc641d6256d1f382b5f._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="https://www.google.com/accounts/o8/id?id=AItOawnJO6OKamxo1HDLwdV-W3KV4GhJ5Qitl9M"
+ nickname="Frederik"
+ subject="comment 1"
+ date="2014-05-15T00:57:23Z"
+ content="""
+Check the description for manual out on [[http://git-annex.branchable.com/preferred_content/standard_groups/]]. I believe that's what you'll need.
+"""]]
diff --git a/doc/forum/Workflow_for_adding_files/comment_2_28dd15ac50f79fb07bacf8b8326c7edc._comment b/doc/forum/Workflow_for_adding_files/comment_2_28dd15ac50f79fb07bacf8b8326c7edc._comment
new file mode 100644
index 0000000000..c11be26673
--- /dev/null
+++ b/doc/forum/Workflow_for_adding_files/comment_2_28dd15ac50f79fb07bacf8b8326c7edc._comment
@@ -0,0 +1,19 @@
+[[!comment format=mdwn
+ username="https://www.google.com/accounts/o8/id?id=AItOawk9SYh6N-JUMkYkW4aOk55zC3Vr9KonDV4"
+ nickname="Florian"
+ subject="comment 2"
+ date="2014-06-18T07:35:31Z"
+ content="""
+Just to be sure I understood the topic, this would be achieved in putting the repo in the manual transfer group?
+
+manual
+
+This gives you nearly full manual control over what content is stored in the repository. This allows using the assistant without it trying to keep a local copy of every file. Instead, you can manually run git annex get, git annex drop, etc to manage content. Only content that is already present is wanted.
+
+The exception to this manual control is that content that a client repository would not want is not wanted. So, files in archive directories are not wanted once their content has reached an archive repository.
+
+present and ($client)
+
+(Where $client is a copy of the preferred content expression used for clients.)
+<<<
+"""]]
diff --git a/doc/forum/Workflow_for_using_git-annex_only_on_external_drives.mdwn b/doc/forum/Workflow_for_using_git-annex_only_on_external_drives.mdwn
new file mode 100644
index 0000000000..734be10ea1
--- /dev/null
+++ b/doc/forum/Workflow_for_using_git-annex_only_on_external_drives.mdwn
@@ -0,0 +1,8 @@
+I am starting to use annex and feel a bit overwhelmed. I would appreciate guidance on this use case.
+
+1. Content must exist on multiple external drives only. I do not want those files in my internal HDD.
+2. Not all drives will have all files. Probably none will. Drives might be smaller than the full set.
+3. I should always have two copies of each file somewhere (and be warned if not).
+3. Some files already exist in some drives I intend to use annex with in this way. When setting annex in them, I had rather avoid having to transfer in files from another set up drive if those files already exist in the new drive.
+4. Some drives will necessarily be using FAT32.
+5. I will be adding new content over time.
diff --git a/doc/forum/Workflow_for_using_git-annex_only_on_external_drives/comment_1_c3c495efb7cdd242b51cc7fa7719d95f._comment b/doc/forum/Workflow_for_using_git-annex_only_on_external_drives/comment_1_c3c495efb7cdd242b51cc7fa7719d95f._comment
new file mode 100644
index 0000000000..95154d3d5e
--- /dev/null
+++ b/doc/forum/Workflow_for_using_git-annex_only_on_external_drives/comment_1_c3c495efb7cdd242b51cc7fa7719d95f._comment
@@ -0,0 +1,21 @@
+[[!comment format=mdwn
+ username="https://launchpad.net/~stephane-gourichon-lpad"
+ nickname="stephane-gourichon-lpad"
+ avatar="http://cdn.libravatar.org/avatar/02d4a0af59175f9123720b4481d55a769ba954e20f6dd9b2792217d9fa0c6089"
+ subject="comment 1"
+ date="2017-07-20T17:53:04Z"
+ content="""
+(Another user here.)
+
+I can't take you by the hand but all this seems like regular use of git-annex.
+
+I was overwhelmed at the beginning, followed http://git-annex.branchable.com/walkthrough , visited a number of pages.
+I finally had something that worked.
+
+In some cases you might consider, instead of moving files then using git annex add, using https://git-annex.branchable.com/git-annex-import/ with or without some of the options.
+
+If you have specific problems, ask a more specific question.
+
+Happy journey!
+
+"""]]
diff --git a/doc/forum/Workflow_for_using_git-annex_only_on_external_drives/comment_2_f29471761e95b25e35e4cb09ad89737a._comment b/doc/forum/Workflow_for_using_git-annex_only_on_external_drives/comment_2_f29471761e95b25e35e4cb09ad89737a._comment
new file mode 100644
index 0000000000..6dbe05f3af
--- /dev/null
+++ b/doc/forum/Workflow_for_using_git-annex_only_on_external_drives/comment_2_f29471761e95b25e35e4cb09ad89737a._comment
@@ -0,0 +1,15 @@
+[[!comment format=mdwn
+ username="CandyAngel"
+ avatar="http://cdn.libravatar.org/avatar/15c0aade8bec5bf004f939dd73cf9ed8"
+ subject="comment 2"
+ date="2017-07-21T09:15:40Z"
+ content="""
+1) Check out the source repository group, which will drop files once enough numcopies are available elsewhere
+2) This is pretty much why git-annex exists :)
+3) Set numcopies to 2 and use 'git annex fsck' to find out when there aren't enough copies
+4) You can use 'import' or 'reinject --known' to clean up known content outside of git-annex
+5) git-annex will run on 'crippled' filesystems like FAT32. Would recommend avoiding this if possible though
+6) This is presumed :)
+
+Sorry for brevity, but this should give some direction/keywords.
+"""]]
diff --git a/doc/forum/Writing_a_book_about_git-annex__58___I_need_a_sparring_partner.mdwn b/doc/forum/Writing_a_book_about_git-annex__58___I_need_a_sparring_partner.mdwn
new file mode 100644
index 0000000000..85395869e9
--- /dev/null
+++ b/doc/forum/Writing_a_book_about_git-annex__58___I_need_a_sparring_partner.mdwn
@@ -0,0 +1,22 @@
+Hi all,
+
+the last few days I wrote the first 1200 words of a Git annex book (in German), but I am unsure whether I shall continue. 
+
+I've written two books about non mainstream software, namely about the documentation generator Sphinx and the document converter Pandoc. I am using these programs myself on a daily basis so that I was quite sure to cover most of their features in the books. I even narrowed the scope of the Pandoc book to "book production" as this is what I do with it, leaving out some use cases such as slides or HTML production.
+
+Before I really start writing the book, I would like to get a clear and robust understanding of the program. As you can see, I don't write books about very popular software in order to sell a lot of copies, I write about software I really like. But I would not write about software that isn't used at all. 
+
+So to get a clear understand of Git annex I need a sparring partner who is willing to discuss usecases, viewpoints and the outline of the book. The following questions and problems are blocking my work:
+
+Some time ago I used Git annex to sync my desktop and my laptop. I used the webapp. My experiences were very bad. Git annex had a very bad performance slowing down my computer making it nearly unusable. I had numerous Unicode errors which let to a duplication of files. So I stopped using Git annex and switched to Syncthing, which does the job without any problems. Syncing computers with the webapp obviously is not be the usecase I would like to cover in the book. Today I use Git annex as a feedreader. 
+
+So why do I want to write a book about a software I do not really use? First of all I like the concept. I like the idea to manage big files in one tree while the content of many files is stored in remote archives. So this could be the idea of the book: sustainable management of big files. And after reading https://git-annex.branchable.com/design/iabackup/ I think that this could be the main use case of Git annex. 
+
+When I wrote about Sphinx and Pandoc I talked to people who used the software to a great extent. Sphinx is the the de facto standard to document Python code. So I was able to evaluate the software before I invested half a year of writing a book. I am not able to say whether Git annex is good for doing x or y, because my setup is way too small. 
+
+So I am looking for power users, for people who use Git annex to manage at least 1TB of data or more. Would you recommend Git annex to film professionals, to music professionals, to data miners, to universities, to your parents? Does it really makes your life easier? What are your experiences with the webapp? 
+
+I would be happy if you are willing to discuss these questions here, privately by mail or by chat eg. via Firefox Hello. 
+
+TIA
+juh
diff --git a/doc/forum/Writing_a_book_about_git-annex__58___I_need_a_sparring_partner/comment_1_ee783a968a5948e2f50bd2d090a2238b._comment b/doc/forum/Writing_a_book_about_git-annex__58___I_need_a_sparring_partner/comment_1_ee783a968a5948e2f50bd2d090a2238b._comment
new file mode 100644
index 0000000000..3eaa59b6b6
--- /dev/null
+++ b/doc/forum/Writing_a_book_about_git-annex__58___I_need_a_sparring_partner/comment_1_ee783a968a5948e2f50bd2d090a2238b._comment
@@ -0,0 +1,19 @@
+[[!comment format=mdwn
+ username="anarcat"
+ subject="comment 1"
+ date="2015-06-10T12:29:46Z"
+ content="""
+hi!
+
+I understand you may have had unicode problems with git-annex in the past: i had my [[own share|bugs/forget_corrupts_non-ascii_chars/]], but most bugs i filed were fixed (see [[users/anarcat/]] for a full list).
+
+I regularly use git-annex to manage my music, video and book collection. The total fileset is around 1-2TB, with thousands of files. Sometimes the music collection operations are a little slow because there are so many files, but in general i am *very* happy with the project. I don't use the webapp so much, because I feel the interface is too limited, and do a lot of things on the commandline.
+
+git-annex not only makes my life easier, it makes possible some things that were impossible before. I used to have a really hackish shell script to rsync part of my music collection to my laptop because it's too big to fit there completely. It wasn't working so well and there was no way to sync *new* music *back* in the main collection. Now I regularly can make changes to the music collection on the laptop without the files even being present. I can also import new files on the laptop easily when i meet people that want to share with me in my travels.
+
+i also work on the Isuma project, for which i will eventually do a longer write up here. For now, look at [[devblog/day_290__/]]. The project aims to manage over a terabyte of data across multiple devices spread over remote areas with limited bandwidth connexions. It's a distributed, two-way, CDN. So far it works well, and issues we have found have been quickly addressed by [[joeyh]].
+
+I would definitely recommend git-annex to film, music and university enthousiast. It is certainly worth the technical learning curve. For my parents, I am less sure. I mentioned the project, but they haven't found the use case for it yet, and they run Windows and OSX all over the place, which makes integration a little harder. With iPads, in particular, it seems that local storage is a thing of the past and everything goes \"in the cloud\". Which is in contradiction with git-annex a little, as it tries to take back control of your files.
+
+Anyways, happy to spar more if you have any more questions. Good luck with the book!
+"""]]
diff --git a/doc/forum/Writing_a_book_about_git-annex__58___I_need_a_sparring_partner/comment_2_259b67a887dd6aa327b752f455c24bff._comment b/doc/forum/Writing_a_book_about_git-annex__58___I_need_a_sparring_partner/comment_2_259b67a887dd6aa327b752f455c24bff._comment
new file mode 100644
index 0000000000..e7697132af
--- /dev/null
+++ b/doc/forum/Writing_a_book_about_git-annex__58___I_need_a_sparring_partner/comment_2_259b67a887dd6aa327b752f455c24bff._comment
@@ -0,0 +1,17 @@
+[[!comment format=mdwn
+ username="CandyAngel"
+ subject="comment 2"
+ date="2015-06-10T13:09:08Z"
+ content="""
+I'm happy to help if I can. Though I don't use the assistant (personally, though people \"at the other end\" of annexes I use do though) or special remotes, I do have at least one unusual use case which entails several terabytes over millions of files.
+
+In this case, I have a collection of harddrives and I am annexing all their contents into one large annex for quick backup/sorting later. This is well into git's problem with scaling to lots of files (I've posted a few tips on the site to minimise this impact).
+
+My other uses are more normal, such as managing photos and music and other digital assets such as games. I use the metadata quite a lot (e.g. I tag files I want to have available in Kodi, or tag game installer/archives to denote what operating system they are for).
+
+Being really picky about commits, several annexes I have have auto-commits on the git-annex branch disabled so I can control them. I'm even picky about doing things in such a way that they look nice in gitk! For example, I avoid cloning, because it makes the new annex appear at the bottom of gitk with its merge commit really far away, instead of a close merge (I can screenshot it if that isn't making any sense).
+
+I'm not sure if I would recommend it to someone who doesn't have someone who understands it nearby to help (just in case), but I would say it is definitely worth learning and using.
+
+In the book, I would say that you should mention that git-annex builds on git, and now people are building things on git-annex. For example, a project I am dabbling in is a VR environment which uses git and git-annex to manage its data: room definitions are stored as text files in git, with references to asset data (such as textures) which are 'git annex get'd as required. This is interesting as it means my co-developer can create/change rooms while I am in the VR environment and they change almost immediately. He changes the texture for the walls? It is downloaded and updated automatically. Imagine Second Life but with a (fully revertable) git-annex backend!
+"""]]
diff --git a/doc/forum/Writing_a_book_about_git-annex__58___I_need_a_sparring_partner/comment_3_a072a5c8f0798094d9ab0b7dc6c34f98._comment b/doc/forum/Writing_a_book_about_git-annex__58___I_need_a_sparring_partner/comment_3_a072a5c8f0798094d9ab0b7dc6c34f98._comment
new file mode 100644
index 0000000000..ff649fe609
--- /dev/null
+++ b/doc/forum/Writing_a_book_about_git-annex__58___I_need_a_sparring_partner/comment_3_a072a5c8f0798094d9ab0b7dc6c34f98._comment
@@ -0,0 +1,15 @@
+[[!comment format=mdwn
+ username="juh"
+ subject="Thanks"
+ date="2015-06-11T15:51:53Z"
+ content="""
+Thanks for your comments. I am very interested in things people build on top of git-annex. I am looking forward to read more about these projects. 
+
+Interesting that both of you don't use the webapp or the assistant. It was the first thing I used and it was disappointing, so now I try out the commandline. 
+
+I understand that managing an amount of files way too big for ones notebook or desktop is one of the main use cases. And this is a use case I definitely will cover in the book, if I write it at all. 
+
+Still evaluating... 
+
+
+"""]]
diff --git a/doc/forum/Wrong_symlink_target_on_usb_drive.mdwn b/doc/forum/Wrong_symlink_target_on_usb_drive.mdwn
new file mode 100644
index 0000000000..661dc5bd3a
--- /dev/null
+++ b/doc/forum/Wrong_symlink_target_on_usb_drive.mdwn
@@ -0,0 +1,55 @@
+As I'm not sure whether this is my fault or a bug of git-annex, I'll post it in forum first.
+
+I have a normal v6 git-annex setup, with a desktop, laptop and usb stick.
+
+All repositories are in sync, also the usb drive.
+There are some unlocked files, which are fine.
+But there are also some locked files, which appear to be broken.
+Checking the target, there is a big difference between the paths. On the computers, it's `.git/annex/objects/0W/vj/SHA256E-s10230770--dcbf7418ff3c9698e9d98418fd07a52afec172aeef80aee2a84dcfdaeef859ed/SHA256E-s10230770--dcbf7418ff3c9698e9d98418fd07a52afec172aeef80aee2a84dcfdaeef859ed`.
+
+Also the usb drive links to this location, but the file does not exist.
+Searching with the firs part of the hash, I got some interesting result: the file was stored in `.git/annex/objects/1cc/840/SHA256E-s10230770--dcbf7418ff3c9698e9d98418fd07a52afec172aeef80aee2a84dcfdaeef859ed`.
+The difference is that the two first directories have three instead of two letters and only one layer.
+The hash remains the same, and the file is valid.
+
+```
+$ cd desktop
+$ git annex sync --content
+(success)
+$ readlink -f locked-file
+.git/annex/objects/0W/vj/SHA256E-s10230770--dcbf7418ff3c9698e9d98418fd07a52afec172aeef80aee2a84dcfdaeef859ed/SHA256E-s10230770--dcbf7418ff3c9698e9d98418fd07a52afec172aeef80aee2a84dcfdaeef859ed
+$ cd usb-drive
+$ readlink -f locked-file
+(no output → broken)
+$ find .git/annex/objects/ -name *SHA256E-s10230770*
+.git/annex/objects/1cc/840/SHA256E-s10230770--dcbf7418ff3c9698e9d98418fd07a52afec172aeef80aee2a84dcfdaeef859ed
+(file is at different place)
+```
+
+I did some troubleshooting and unspecific fixing.
+
+```
+$ git annex whereis locked-file
+whereis locked-file (3 copies) 
+   	d0a64b4b-054e-4c42-a64f-58a08591abab -- laptop
+   	e3013e6b-a06a-46a5-a67a-ce9ec0520d21 -- desktop
+   	ebd16ddb-548c-4078-b35e-087132523924 -- usb-drive [here]
+ok
+$ git annex fsck
+(everything fine, also unlocked-file was checked)
+$ git annex fix
+(no error, but no solution)
+$ git annex unlock locked-file
+(file is there and editable, great!)
+$ git annex lock locked-file
+(symlink is there and points… right, to the wrong location)
+$ git annex drop locked-file && git annex get locked-file
+(no change visible in work directory)
+```
+
+Specially on this drive disk space is important, so it would be for me a bad solution to keep them unlocked.
+
+As I mentioned earlier, this could be also my fault, as I have done some not-often used methods
+
+- I rebased the history and checked manually (without `git annex sync`) the `master` and `synced/master` branch to the same commit on all repositories, no errors reported
+- I used a v5 repository on laptop before syncing with the drive, then upgraded. I don't think it is the problem here as it works on desktop after the sync as wanted
diff --git a/doc/forum/Wrong_symlink_target_on_usb_drive/comment_1_9a50df886c0c23ee6f5586bd9f6b61c3._comment b/doc/forum/Wrong_symlink_target_on_usb_drive/comment_1_9a50df886c0c23ee6f5586bd9f6b61c3._comment
new file mode 100644
index 0000000000..a82da1b4f1
--- /dev/null
+++ b/doc/forum/Wrong_symlink_target_on_usb_drive/comment_1_9a50df886c0c23ee6f5586bd9f6b61c3._comment
@@ -0,0 +1,11 @@
+[[!comment format=mdwn
+ username="joey"
+ subject="""comment 1"""
+ date="2016-05-10T16:16:36Z"
+ content="""
+What OS? 
+
+What filesystem is the drive formatted with? 
+
+What version of git-annex?
+"""]]
diff --git a/doc/forum/Wrong_symlink_target_on_usb_drive/comment_2_75df13154d3ea662119e9d701a0d9c97._comment b/doc/forum/Wrong_symlink_target_on_usb_drive/comment_2_75df13154d3ea662119e9d701a0d9c97._comment
new file mode 100644
index 0000000000..976d66f536
--- /dev/null
+++ b/doc/forum/Wrong_symlink_target_on_usb_drive/comment_2_75df13154d3ea662119e9d701a0d9c97._comment
@@ -0,0 +1,29 @@
+[[!comment format=mdwn
+ username="pixunil"
+ subject="more information"
+ date="2016-05-10T17:27:46Z"
+ content="""
+```
+$ lsb_release -a
+Distributor ID:	LinuxMint
+Description:	Linux Mint 17.3 Rosa
+Release:	17.3
+Codename:	rosa
+$ git annex version
+git-annex version: 6.20160505-gddbe008
+build flags: Assistant Webapp Pairing Testsuite S3(multipartupload)(storageclasses) WebDAV Inotify ConcurrentOutput TorrentParser MagicMime Feeds Quvi
+key/value backends: SHA256E SHA256 SHA512E SHA512 SHA224E SHA224 SHA384E SHA384 SHA3_256E SHA3_256 SHA3_512E SHA3_512 SHA3_224E SHA3_224 SHA3_384E SHA3_384 SKEIN256E SKEIN256 SKEIN512E SKEIN512 SHA1E SHA1 MD5E MD5 WORM URL
+remote types: git gcrypt S3 bup directory rsync web bittorrent webdav tahoe glacier ddar hook external
+local repository version: 6
+supported repository versions: 5 6
+upgrade supported from repository versions: 0 1 2 4 5
+operating system: linux x86_64
+```
+
+OS is on both computers the same, on ext4.
+The usb drive has 8GB and is NTFS formatted. About NTFS, I wasn't sure, so I left the partition as it was.
+
+I built git-annex with `stack` three times on top of the latest tag, the version number differed in minor afaik (but can't remember the old ones).
+I'm aware that I'm using an unstable version (forgot to mention it, sry about that), but it was on the two latest builds.
+I also don't know where the information is what the stable release is…
+"""]]
diff --git a/doc/forum/Wrong_symlink_target_on_usb_drive/comment_3_7f2fb4e7f0c0db9f9d9cbb637f02c9ff._comment b/doc/forum/Wrong_symlink_target_on_usb_drive/comment_3_7f2fb4e7f0c0db9f9d9cbb637f02c9ff._comment
new file mode 100644
index 0000000000..50356feea4
--- /dev/null
+++ b/doc/forum/Wrong_symlink_target_on_usb_drive/comment_3_7f2fb4e7f0c0db9f9d9cbb637f02c9ff._comment
@@ -0,0 +1,10 @@
+[[!comment format=mdwn
+ username="joey"
+ subject="""comment 2"""
+ date="2016-05-10T17:36:53Z"
+ content="""
+I suppose it must have something to do with NTFS, but I can't reproduce it
+on NTFS.. Normally git-annex will refuse to use symlinks on NTFS entirely.
+
+Can you please paste in the whole .git/config file of the repository?
+"""]]
diff --git a/doc/forum/Wrong_symlink_target_on_usb_drive/comment_4_f6ee6bfd5d82f0925f0e174b4c75e592._comment b/doc/forum/Wrong_symlink_target_on_usb_drive/comment_4_f6ee6bfd5d82f0925f0e174b4c75e592._comment
new file mode 100644
index 0000000000..b80d950620
--- /dev/null
+++ b/doc/forum/Wrong_symlink_target_on_usb_drive/comment_4_f6ee6bfd5d82f0925f0e174b4c75e592._comment
@@ -0,0 +1,27 @@
+[[!comment format=mdwn
+ username="pixunil"
+ subject="git config"
+ date="2016-05-10T17:59:10Z"
+ content="""
+```
+[core]
+	repositoryformatversion = 0
+	filemode = false
+	bare = false
+	logallrefupdates = true
+	symlinks = true
+[annex]
+	uuid = ebd16ddb-548c-4078-b35e-087132523924
+	crippledfilesystem = true
+	version = 6
+[filter \"annex\"]
+	smudge = git-annex smudge %f
+	clean = git-annex smudge --clean %f
+# remote and user omitted
+```
+
+Gotcha! `core.symlinks` is `true`!
+I set it to `false` and running `unlock` works fine, but after `lock` the file contents gets replaced with the two digits directories (not the three digits directories which are the place where the object is saved). Also, I can't unlock the file afterwards anymore.
+
+However, I would be fine if symlinks work fine on other file system (as mentioned, would be great for saving space and have the access to the files) or other method.
+"""]]
diff --git a/doc/forum/Wrong_symlink_target_on_usb_drive/comment_5_e5a152cef3c48ef05cb183907377f128._comment b/doc/forum/Wrong_symlink_target_on_usb_drive/comment_5_e5a152cef3c48ef05cb183907377f128._comment
new file mode 100644
index 0000000000..8a5dd5530d
--- /dev/null
+++ b/doc/forum/Wrong_symlink_target_on_usb_drive/comment_5_e5a152cef3c48ef05cb183907377f128._comment
@@ -0,0 +1,24 @@
+[[!comment format=mdwn
+ username="joey"
+ subject="""comment 5"""
+ date="2016-05-10T18:08:10Z"
+ content="""
+Ok, I reproduced the problem. Normally core.symlinks will be false
+by on NTFS. You have to manually set it to true to experience
+this problem AFAICS.
+
+I was able to lock a file (resulting in a broken symlink) and then successfully 
+unlock it and the content was back in place.
+
+git-annex always uses the lower case hash directory names when on a
+crippled filesystem, since that's more portable and avoids lots of
+potential problems. By configuring core.symlinks=true, you make git-annex
+support locking files using symlinks, but these symlinks can't point to the
+actual content location.
+
+I think it makes sense for git-annex to use the the mixed case hash
+directory names when core.symlinks=true even if the filesystem is crippled.
+There's some foot-shooting potential, since some crippled fileystems
+don't support the mixed-case hash directories. But, you have to manually
+set up this configuration. I've made a change along these lines.
+"""]]
diff --git a/doc/forum/Wrong_symlink_target_on_usb_drive/comment_6_be59fcc5633b352eefe993871df1ec4a._comment b/doc/forum/Wrong_symlink_target_on_usb_drive/comment_6_be59fcc5633b352eefe993871df1ec4a._comment
new file mode 100644
index 0000000000..83fda1a207
--- /dev/null
+++ b/doc/forum/Wrong_symlink_target_on_usb_drive/comment_6_be59fcc5633b352eefe993871df1ec4a._comment
@@ -0,0 +1,14 @@
+[[!comment format=mdwn
+ username="pixunil"
+ subject="comment 6"
+ date="2016-05-10T20:45:56Z"
+ content="""
+Thanks for clearing it up, so I guess it was me.
+Otherwise, I have some questions:
+
+- Could I set `annex.crippledfilesystem` to `false` or is that dangerous on NTFS?
+- Do symlinks also on Window work? (direction of the slash)
+- Wouldn't it be better to modify the link target than the location of the objects?
+
+I'm also fine with redirecting me to a FAQ, thanks again for your time and efforts.
+"""]]
diff --git a/doc/forum/Wrong_symlink_target_on_usb_drive/comment_7_64e1cdf144a17319831a58b04b22b7ea._comment b/doc/forum/Wrong_symlink_target_on_usb_drive/comment_7_64e1cdf144a17319831a58b04b22b7ea._comment
new file mode 100644
index 0000000000..a8ef69cc62
--- /dev/null
+++ b/doc/forum/Wrong_symlink_target_on_usb_drive/comment_7_64e1cdf144a17319831a58b04b22b7ea._comment
@@ -0,0 +1,17 @@
+[[!comment format=mdwn
+ username="joey"
+ subject="""comment 7"""
+ date="2016-05-10T21:00:06Z"
+ content="""
+You might be able to unset annex.crippledfilesystem. Or it might cause some
+stuff to break. Especially if you access this drive from Windows.
+
+AFAIK symlinks don't work in Windows; at least git-annex does not know how
+to create them on Windows. (Windows has symlink-ish things and git might be
+able to use them.)
+
+The link target is committed to the master branch, so you need to be able
+to check out that same branch in other clones and have it point at the
+content. So there can only be one link target used, and by default that's
+the mixed case one. See [[tuning]] for how to change that.
+"""]]
diff --git a/doc/forum/XBMC__44___NFS___38___git-annex_.mdwn b/doc/forum/XBMC__44___NFS___38___git-annex_.mdwn
new file mode 100644
index 0000000000..fc5cd702ad
--- /dev/null
+++ b/doc/forum/XBMC__44___NFS___38___git-annex_.mdwn
@@ -0,0 +1,27 @@
+Hi,
+
+this is not a git-annex problem, but more likely a way some software handles symlinks and/or do file-type detection.
+
+my setup is a following:
+- multiple pc/laptop (linux)
+- one NAS (debian) - main repo for music/films/photos - all in git-annex
+- a Raspberry Pi (Raspbmc) - connected to the TV
+
+I want to share all my films/music/.. to the Pi.
+So, i setup a r/o NFS and mounted that in the XBMC.
+So far this works, I see directories but XBMC do not recognize the media files.
+It works, if I "git-annex edit ..." the file.
+
+My _assumption_ is, that it follows the symlink, finds a file with no extension - and ignores that.
+In XBMC-config there is a list of supported filetypes by extension. ( .avi,.mpg,.foo,.bar )
+
+What I have think of so far:
+- tell XBMC somehow to load all the files (did not work)
+- having some kind of (FUSE?) filter, which hides the symlink in a transparent way
+- creating via script hard-links in a seperate folder with same structure, mount that.
+- using some alternative to NFS ( like ftp, smb ) or a other kind of media-server (server-side)
+
+any comments, ideas ? 
+If i find a solution, I'll post it here.
+
+.ka
diff --git a/doc/forum/XBMC__44___NFS___38___git-annex_/comment_1_86480f31d410e903766f82e6ecf83e1c._comment b/doc/forum/XBMC__44___NFS___38___git-annex_/comment_1_86480f31d410e903766f82e6ecf83e1c._comment
new file mode 100644
index 0000000000..e825225487
--- /dev/null
+++ b/doc/forum/XBMC__44___NFS___38___git-annex_/comment_1_86480f31d410e903766f82e6ecf83e1c._comment
@@ -0,0 +1,12 @@
+[[!comment format=mdwn
+ username="https://www.google.com/accounts/o8/id?id=AItOawmBUR4O9mofxVbpb8JV9mEbVfIYv670uJo"
+ nickname="Justin"
+ subject="Samba works"
+ date="2013-05-30T14:22:46Z"
+ content="""
+I use samba. Just need to set
+
+    unix extensions = no
+
+which causes samba to handle the symlinks internally.
+"""]]
diff --git a/doc/forum/XBMC__44___NFS___38___git-annex_/comment_2_d8ed4dd51d3050db691a8abdec24cd42._comment b/doc/forum/XBMC__44___NFS___38___git-annex_/comment_2_d8ed4dd51d3050db691a8abdec24cd42._comment
new file mode 100644
index 0000000000..0b3cb22c14
--- /dev/null
+++ b/doc/forum/XBMC__44___NFS___38___git-annex_/comment_2_d8ed4dd51d3050db691a8abdec24cd42._comment
@@ -0,0 +1,10 @@
+[[!comment format=mdwn
+ username="http://joeyh.name/"
+ nickname="joey"
+ subject="comment 2"
+ date="2013-05-30T15:20:51Z"
+ content="""
+The default git-annex backend for over a year now is SHA256E, which includes the filename extension in the key to deal with programs that rely on them. If you're not using that backend, you can `git annex migrate` to it.
+
+Or you can turn on direct mode, which will certainly solve the problem.
+"""]]
diff --git a/doc/forum/XBMC__44___NFS___38___git-annex_/comment_3_42b80ee51ce25775bf4532f53a8ecfe3._comment b/doc/forum/XBMC__44___NFS___38___git-annex_/comment_3_42b80ee51ce25775bf4532f53a8ecfe3._comment
new file mode 100644
index 0000000000..00fbf6a208
--- /dev/null
+++ b/doc/forum/XBMC__44___NFS___38___git-annex_/comment_3_42b80ee51ce25775bf4532f53a8ecfe3._comment
@@ -0,0 +1,11 @@
+[[!comment format=mdwn
+ username="ka7"
+ ip="2001:7b8:155d:0:222:64ff:fe16:dc52"
+ subject="comment 3"
+ date="2013-05-31T15:45:51Z"
+ content="""
+ah, great.
+I'll try the samba way.
+And i guess my git-annex is too old (using the git-annex version: 3.20120629~bpo60+2 from debian-backports)
+thanks !
+"""]]
diff --git a/doc/forum/XBMC__44___NFS___38___git-annex_/comment_4_01767f3f864954cf8080274e206da9d4._comment b/doc/forum/XBMC__44___NFS___38___git-annex_/comment_4_01767f3f864954cf8080274e206da9d4._comment
new file mode 100644
index 0000000000..58f00af15e
--- /dev/null
+++ b/doc/forum/XBMC__44___NFS___38___git-annex_/comment_4_01767f3f864954cf8080274e206da9d4._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="ka7"
+ ip="2001:7b8:155d:0:222:64ff:fe16:dc52"
+ subject="comment 4"
+ date="2013-05-31T18:46:29Z"
+ content="""
+after a quick check I realized that sha256E is available -- migrate and yepey!
+"""]]
diff --git a/doc/forum/XMPP_authentication_failure.mdwn b/doc/forum/XMPP_authentication_failure.mdwn
new file mode 100644
index 0000000000..3c4c41bc2a
--- /dev/null
+++ b/doc/forum/XMPP_authentication_failure.mdwn
@@ -0,0 +1,15 @@
+Hi,
+
+Not sure if this is a bug or not, so posting it here rather than in the bug tracker.
+
+Basically, I am unable to add an XMPP identity via the webapp, I continually  get the error message: "Unable to connect to the Jabber server. Maybe you entered the wrong password? (Error message: AuthenticationFailure)"
+
+I didn't enter the wrong password, I copy/pasted it from my password manager and the same account/password work fine with Pidgin. The log doesn't show anything of interest that I can see.
+
+Using git annex version 4.20130405 from Debain Unstable. The server is ejabberd running on my VPS with a self signed certificate.
+
+Anyone got any idea what might be going on?
+
+Cheers,
+
+Rob
diff --git a/doc/forum/XMPP_authentication_failure/comment_1_19c7c3aa79d209d613d2e061e3129690._comment b/doc/forum/XMPP_authentication_failure/comment_1_19c7c3aa79d209d613d2e061e3129690._comment
new file mode 100644
index 0000000000..a5862e58fb
--- /dev/null
+++ b/doc/forum/XMPP_authentication_failure/comment_1_19c7c3aa79d209d613d2e061e3129690._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="http://joeyh.name/"
+ nickname="joey"
+ subject="comment 1"
+ date="2013-04-09T18:31:01Z"
+ content="""
+I installed ejabberd and was able to trivially reproduce this bug. Pidgin works, git-annex fails, and I just created the user with `ejabberdctl register`. Filed an upstream bug report against the haskell network-protocol-xmpp library, by email, since there's no upstream BTS:
+"""]]
diff --git a/doc/forum/XMPP_authentication_failure/comment_2_870059fed451e8377e5d382464ecc34b._comment b/doc/forum/XMPP_authentication_failure/comment_2_870059fed451e8377e5d382464ecc34b._comment
new file mode 100644
index 0000000000..bcf4e22ac6
--- /dev/null
+++ b/doc/forum/XMPP_authentication_failure/comment_2_870059fed451e8377e5d382464ecc34b._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="robconnolly"
+ ip="203.167.218.84"
+ subject="comment 2"
+ date="2013-04-09T23:01:44Z"
+ content="""
+Excellent, thanks for the quick response/action!
+"""]]
diff --git a/doc/forum/XMPP_authentication_failure/comment_3_1a7ff955e9173f13d10b75f203792384._comment b/doc/forum/XMPP_authentication_failure/comment_3_1a7ff955e9173f13d10b75f203792384._comment
new file mode 100644
index 0000000000..587daa0edc
--- /dev/null
+++ b/doc/forum/XMPP_authentication_failure/comment_3_1a7ff955e9173f13d10b75f203792384._comment
@@ -0,0 +1,10 @@
+[[!comment format=mdwn
+ username="http://joeyh.name/"
+ nickname="joey"
+ subject="comment 3"
+ date="2013-04-12T22:25:12Z"
+ content="""
+John has gotten back to me with some analysis. The authentication failure is when SCRAM-SHA-1 is used. Ejabberd is rejecting the client's authentication request with a reason of \"bad-protocol\". It's not clear if ejabberd is broken or if gsasl is generating a bad SCRAM-SHA-1 authentication. We're inclined toward the latter, and will be forwarding this on to ejabberd.
+
+The only way to turn off this authentication in ejabberd is to configure it to store passwords in plain text, or downgrade to a version older than 2.1.9, which first added it. Note that debian stable contains 2.1.5.3, so is not currently affected, for example.
+"""]]
diff --git a/doc/forum/XMPP_authentication_failure/comment_4_d59031ebc0dd3abc1f4c96878328362c._comment b/doc/forum/XMPP_authentication_failure/comment_4_d59031ebc0dd3abc1f4c96878328362c._comment
new file mode 100644
index 0000000000..e3e927818a
--- /dev/null
+++ b/doc/forum/XMPP_authentication_failure/comment_4_d59031ebc0dd3abc1f4c96878328362c._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="http://joeyh.name/"
+ nickname="joey"
+ subject="comment 4"
+ date="2013-04-17T13:26:45Z"
+ content="""
+Upstream bug report, with patch https://support.process-one.net/browse/EJAB-1632
+"""]]
diff --git a/doc/forum/XMPP_authentication_failure/comment_5_c37ef477bef7efdb79dd05dce90dfde6._comment b/doc/forum/XMPP_authentication_failure/comment_5_c37ef477bef7efdb79dd05dce90dfde6._comment
new file mode 100644
index 0000000000..37a3365d01
--- /dev/null
+++ b/doc/forum/XMPP_authentication_failure/comment_5_c37ef477bef7efdb79dd05dce90dfde6._comment
@@ -0,0 +1,10 @@
+[[!comment format=mdwn
+ username="https://www.google.com/accounts/o8/id?id=AItOawkF8_uQjLYm5Mf5F_JuVW-BxlvzpWjvR_o"
+ nickname="Andrew"
+ subject="Wheezy is affected"
+ date="2013-05-25T12:00:21Z"
+ content="""
+Now that Wheezy is out, Debian Stable appears to be affected by this issue. It has ejabberd v2.1.10 .
+
+The patch on the linked bug report doesn't apply cleanly to v2.1.10. I'll try and look at this again tomorrow.
+"""]]
diff --git a/doc/forum/XMPP_authentication_failure/comment_6_48cabea4c2caf5b3bd854df3aaa17d3d._comment b/doc/forum/XMPP_authentication_failure/comment_6_48cabea4c2caf5b3bd854df3aaa17d3d._comment
new file mode 100644
index 0000000000..fe4ab07443
--- /dev/null
+++ b/doc/forum/XMPP_authentication_failure/comment_6_48cabea4c2caf5b3bd854df3aaa17d3d._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="http://joeyh.name/"
+ nickname="joey"
+ subject="comment 6"
+ date="2013-05-25T17:57:43Z"
+ content="""
+Debian has it fixed in 2.1.10-5 but this has unfortunately not made it to the wheezy release.
+"""]]
diff --git a/doc/forum/XMPP_authentication_failure/comment_7_14cd9b67806db93c3af055d88c9a910a._comment b/doc/forum/XMPP_authentication_failure/comment_7_14cd9b67806db93c3af055d88c9a910a._comment
new file mode 100644
index 0000000000..38a8733923
--- /dev/null
+++ b/doc/forum/XMPP_authentication_failure/comment_7_14cd9b67806db93c3af055d88c9a910a._comment
@@ -0,0 +1,10 @@
+[[!comment format=mdwn
+ username="https://www.google.com/accounts/o8/id?id=AItOawkF8_uQjLYm5Mf5F_JuVW-BxlvzpWjvR_o"
+ nickname="Andrew"
+ subject="comment 7"
+ date="2013-05-25T20:27:32Z"
+ content="""
+Ah ha, so it is. I've installed ejabberd from Jessie onto my Wheezy box and now git-annex can authenticate.
+
+Thank you!
+"""]]
diff --git a/doc/forum/XMPP_authentication_failure/comment_8_151d3fd7d3cceb30fd20a8f3bd54036c._comment b/doc/forum/XMPP_authentication_failure/comment_8_151d3fd7d3cceb30fd20a8f3bd54036c._comment
new file mode 100644
index 0000000000..2700343ca2
--- /dev/null
+++ b/doc/forum/XMPP_authentication_failure/comment_8_151d3fd7d3cceb30fd20a8f3bd54036c._comment
@@ -0,0 +1,24 @@
+[[!comment format=mdwn
+ username="https://www.google.com/accounts/o8/id?id=AItOawnVTPaZaIfnZqDbkmW97pqxwi2QuZIJAF4"
+ nickname="Boris"
+ subject="jabberd also affected"
+ date="2013-06-25T13:00:46Z"
+ content="""
+I tried logging in to my local 'jabberd' server and got the same response:
+
+AuthenticationError \"No supported authentication mechanism\"
+
+checking the version gives:
+
+    jabberd -v
+    jabberd14 version 1.6.1.1
+
+    The following optional features have been enabled:
+    - support for IPv6.
+    - support for TLS using GNU TLS
+    - support for MySQL
+    - support for PostgreSQL
+
+--------------
+Can I do anything, investigate something?
+"""]]
diff --git a/doc/forum/XMPP_authentication_failure/comment_9_fbb9eba65fbb72201f08511945fbcf8c._comment b/doc/forum/XMPP_authentication_failure/comment_9_fbb9eba65fbb72201f08511945fbcf8c._comment
new file mode 100644
index 0000000000..e019056de6
--- /dev/null
+++ b/doc/forum/XMPP_authentication_failure/comment_9_fbb9eba65fbb72201f08511945fbcf8c._comment
@@ -0,0 +1,9 @@
+[[!comment format=mdwn
+ username="http://joeyh.name/"
+ ip="4.152.108.193"
+ subject="comment 9"
+ date="2013-06-25T17:18:56Z"
+ content="""
+Boris, that seems like an unrelated problem to the one affecting ejabberd. Different error message. 
+
+"""]]
diff --git a/doc/forum/XMPP_email_setup_says_wrong_password_but_it__39__s_correct._Can_I_provide_some_kind_of_debug_data__63__.mdwn b/doc/forum/XMPP_email_setup_says_wrong_password_but_it__39__s_correct._Can_I_provide_some_kind_of_debug_data__63__.mdwn
new file mode 100644
index 0000000000..f29637c0df
--- /dev/null
+++ b/doc/forum/XMPP_email_setup_says_wrong_password_but_it__39__s_correct._Can_I_provide_some_kind_of_debug_data__63__.mdwn
@@ -0,0 +1,9 @@
+git-annex-assistant is so exciting...
+
+Can't set up XMPP sharing. The web app says "Wrong Password" for my gmail/home-google-apps/work-google-apps addresses, but they are correct.
+
+I'd love to send some debug output, can anyone tell me how to get it? Of course if I'm just doing something wrong and you have an FAQ link or similar, feel free to RTFM me :)
+
+I'm on Ubuntu 12.04 running https://downloads.kitenet.net/git-annex/linux/3.20121112/git-annex-standalone-amd64.tar.gz
+
+Carlo
diff --git a/doc/forum/XMPP_email_setup_says_wrong_password_but_it__39__s_correct._Can_I_provide_some_kind_of_debug_data__63__/comment_1_1ba0735141fc6a21ac15913f4cacefae._comment b/doc/forum/XMPP_email_setup_says_wrong_password_but_it__39__s_correct._Can_I_provide_some_kind_of_debug_data__63__/comment_1_1ba0735141fc6a21ac15913f4cacefae._comment
new file mode 100644
index 0000000000..d3f408690c
--- /dev/null
+++ b/doc/forum/XMPP_email_setup_says_wrong_password_but_it__39__s_correct._Can_I_provide_some_kind_of_debug_data__63__/comment_1_1ba0735141fc6a21ac15913f4cacefae._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="https://www.google.com/accounts/o8/id?id=AItOawn7gQ1zZDdWhXy9H51W2krZYShNmKL3qfM"
+ nickname="Karsten"
+ subject="comment 1"
+ date="2012-11-13T14:38:28Z"
+ content="""
+Just an idea: Do you have two-factor-authorisation set up for your google account, you need to generate an application specific password, as the 'normal' password will not work.
+"""]]
diff --git a/doc/forum/XMPP_email_setup_says_wrong_password_but_it__39__s_correct._Can_I_provide_some_kind_of_debug_data__63__/comment_2_16994dc86b87592fc62799e2d206d172._comment b/doc/forum/XMPP_email_setup_says_wrong_password_but_it__39__s_correct._Can_I_provide_some_kind_of_debug_data__63__/comment_2_16994dc86b87592fc62799e2d206d172._comment
new file mode 100644
index 0000000000..2c5f408e7b
--- /dev/null
+++ b/doc/forum/XMPP_email_setup_says_wrong_password_but_it__39__s_correct._Can_I_provide_some_kind_of_debug_data__63__/comment_2_16994dc86b87592fc62799e2d206d172._comment
@@ -0,0 +1,10 @@
+[[!comment format=mdwn
+ username="http://joeyh.name/"
+ ip="4.252.11.120"
+ subject="comment 2"
+ date="2012-11-13T17:21:35Z"
+ content="""
+@Karsten that seems likely.
+
+I've made a change so it shows the error message from the XMPP library I'm using. Which may or may not be useful..
+"""]]
diff --git a/doc/forum/XMPP_email_setup_says_wrong_password_but_it__39__s_correct._Can_I_provide_some_kind_of_debug_data__63__/comment_3_6afd424edc4095b8f71b136de2a9e64d._comment b/doc/forum/XMPP_email_setup_says_wrong_password_but_it__39__s_correct._Can_I_provide_some_kind_of_debug_data__63__/comment_3_6afd424edc4095b8f71b136de2a9e64d._comment
new file mode 100644
index 0000000000..eab2e796bc
--- /dev/null
+++ b/doc/forum/XMPP_email_setup_says_wrong_password_but_it__39__s_correct._Can_I_provide_some_kind_of_debug_data__63__/comment_3_6afd424edc4095b8f71b136de2a9e64d._comment
@@ -0,0 +1,10 @@
+[[!comment format=mdwn
+ username="https://www.google.com/accounts/o8/id?id=AItOawnZEanlyzay_QlEAL0CWpyZcRTyN7vay8U"
+ nickname="Carlo"
+ subject="XMPP"
+ date="2012-11-13T23:26:50Z"
+ content="""
+Just tried a fresh jabber.org address, same error.
+
+Will see if I can get the changed version built. Or are there nightlies?
+"""]]
diff --git a/doc/forum/XMPP_email_setup_says_wrong_password_but_it__39__s_correct._Can_I_provide_some_kind_of_debug_data__63__/comment_4_1381b6a927410642c6a93aa8354be791._comment b/doc/forum/XMPP_email_setup_says_wrong_password_but_it__39__s_correct._Can_I_provide_some_kind_of_debug_data__63__/comment_4_1381b6a927410642c6a93aa8354be791._comment
new file mode 100644
index 0000000000..9cdca6fc1e
--- /dev/null
+++ b/doc/forum/XMPP_email_setup_says_wrong_password_but_it__39__s_correct._Can_I_provide_some_kind_of_debug_data__63__/comment_4_1381b6a927410642c6a93aa8354be791._comment
@@ -0,0 +1,17 @@
+[[!comment format=mdwn
+ username="https://www.google.com/accounts/o8/id?id=AItOawnZEanlyzay_QlEAL0CWpyZcRTyN7vay8U"
+ nickname="Carlo"
+ subject="Now Works! For jabber.org, gmail, but NOT google apps"
+ date="2012-11-27T11:40:43Z"
+ content="""
+Hi again!
+
+With the latest version, 3.20121126, pairing works for both something@gmail.com and something@jabber.org! Yay!
+
+What still fails is my google apps address, something@somedomain.com, seems to behave same as before. the assistant says
+
+  Unable to connect to the Jabber server. Maybe you entered the wrong password? (Error message: connect: timeout (Connection timed out))
+
+If you would like a google apps address on my domain, so you don't have to set up your own, just let me know.
+
+"""]]
diff --git a/doc/forum/XMPP_email_setup_says_wrong_password_but_it__39__s_correct._Can_I_provide_some_kind_of_debug_data__63__/comment_5_c5b33c7a8aa8e6d0f9349510dac2366d._comment b/doc/forum/XMPP_email_setup_says_wrong_password_but_it__39__s_correct._Can_I_provide_some_kind_of_debug_data__63__/comment_5_c5b33c7a8aa8e6d0f9349510dac2366d._comment
new file mode 100644
index 0000000000..58fb0242b3
--- /dev/null
+++ b/doc/forum/XMPP_email_setup_says_wrong_password_but_it__39__s_correct._Can_I_provide_some_kind_of_debug_data__63__/comment_5_c5b33c7a8aa8e6d0f9349510dac2366d._comment
@@ -0,0 +1,12 @@
+[[!comment format=mdwn
+ username="http://joeyh.name/"
+ ip="4.154.6.49"
+ subject="comment 5"
+ date="2012-11-27T20:32:47Z"
+ content="""
+Hmm, I can't think of anything I've changed that would make it work where it didn't before.
+
+I suspect the google apps problem is because it looks for a SRV record in DNS to find the jabber server, and something is wrong there. Without knowing what \"something@somedomain.com\" is standing for, it's hard to tell. :)
+
+If you'd like to set me up something to test with, my email address is id@joeyh.name
+"""]]
diff --git a/doc/forum/XMPP_email_setup_says_wrong_password_but_it__39__s_correct._Can_I_provide_some_kind_of_debug_data__63__/comment_6_9913d2983ba2744ed24911f74988e4c7._comment b/doc/forum/XMPP_email_setup_says_wrong_password_but_it__39__s_correct._Can_I_provide_some_kind_of_debug_data__63__/comment_6_9913d2983ba2744ed24911f74988e4c7._comment
new file mode 100644
index 0000000000..cc633e8fb5
--- /dev/null
+++ b/doc/forum/XMPP_email_setup_says_wrong_password_but_it__39__s_correct._Can_I_provide_some_kind_of_debug_data__63__/comment_6_9913d2983ba2744ed24911f74988e4c7._comment
@@ -0,0 +1,11 @@
+[[!comment format=mdwn
+ username="https://www.google.com/accounts/o8/id?id=AItOawnZEanlyzay_QlEAL0CWpyZcRTyN7vay8U"
+ nickname="Carlo"
+ subject="comment 6"
+ date="2012-11-29T15:37:46Z"
+ content="""
+Weird, 20121112 works too now.
+
+Looks like some kind of edge case... I'll let you know if I find it.
+
+"""]]
diff --git a/doc/forum/XMPP_email_setup_says_wrong_password_but_it__39__s_correct._Can_I_provide_some_kind_of_debug_data__63__/comment_7_ad6f385a2b95803eb9d81dfe76359551._comment b/doc/forum/XMPP_email_setup_says_wrong_password_but_it__39__s_correct._Can_I_provide_some_kind_of_debug_data__63__/comment_7_ad6f385a2b95803eb9d81dfe76359551._comment
new file mode 100644
index 0000000000..de8403c52e
--- /dev/null
+++ b/doc/forum/XMPP_email_setup_says_wrong_password_but_it__39__s_correct._Can_I_provide_some_kind_of_debug_data__63__/comment_7_ad6f385a2b95803eb9d81dfe76359551._comment
@@ -0,0 +1,10 @@
+[[!comment format=mdwn
+ username="https://www.google.com/accounts/o8/id?id=AItOawlutNrg7ajiNAfi5gqJfD8crv0aimZa27k"
+ nickname="Chuck"
+ subject="I'm having this problem too"
+ date="2013-07-21T19:14:21Z"
+ content="""
+I'm a git-annex newbie looking for Dropbox-like functionality.  I've got git-annex and the webapp running on my desktop and laptop, and I believe I've got a Jabber account working since Kopete seems happy with my account info.  But when I enter the correct Gmail UID and password I get the above error.
+
+I'm running the version of git-annex that's current in the standard Ubuntu repos (I'm actually running Kubuntu if that matters), and that version is 20121112ubuntu2.  If this is an issue that was fixed in the 20121126 release, how would I get that?  Doing the standard install gets me the version I have now...
+"""]]
diff --git a/doc/forum/XMPP_problem_behind_router.mdwn b/doc/forum/XMPP_problem_behind_router.mdwn
new file mode 100644
index 0000000000..5eec76a07d
--- /dev/null
+++ b/doc/forum/XMPP_problem_behind_router.mdwn
@@ -0,0 +1,3 @@
+I'm trying to configure a jabber account for use with git-annex, but it seems that something's wrong as soon as I try to go through my router (wired or wireless).  Compared to directly connecting to my modem, wireshark shows a lot of TCP retransmissions, eventually resulting in "Unable to connect to the Jabber server. Maybe you entered the wrong password? (Error message: host gmail.com:5222 failed: connect: timeout (Connection timed out))" in the webapp.
+
+I've tried to configure the account both in the webapp and manually in the .git/annex/creds/xmpp file, but it doesn't seem to make a difference.  It's able to connect if I directly connect it to my modem, so I'm fairly sure it's not a problem at my computer or with the credentials.  It doesn't appear to be a problem at the firewall on the router, but I could certainly be missing something.  Are there some other tests I could try to narrow down the problem?
diff --git a/doc/forum/XMPP_problem_behind_router/comment_1_25a7f8dc5cf14cda4d76b2f8c6ca77d5._comment b/doc/forum/XMPP_problem_behind_router/comment_1_25a7f8dc5cf14cda4d76b2f8c6ca77d5._comment
new file mode 100644
index 0000000000..04b1965c00
--- /dev/null
+++ b/doc/forum/XMPP_problem_behind_router/comment_1_25a7f8dc5cf14cda4d76b2f8c6ca77d5._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="https://me.yahoo.com/a/wbh0dY54mcPwTpeOweuPQ8JiZrH3hg--#9b726"
+ nickname="Joe"
+ subject="comment 1"
+ date="2014-09-23T01:15:23Z"
+ content="""
+I forgot to mention that XMPP via other clients (e.g., empathy) works fine.
+"""]]
diff --git a/doc/forum/XMPP_problem_behind_router/comment_2_3186ebe32c30764b9fd53625dd3e4eda._comment b/doc/forum/XMPP_problem_behind_router/comment_2_3186ebe32c30764b9fd53625dd3e4eda._comment
new file mode 100644
index 0000000000..63387ae3dd
--- /dev/null
+++ b/doc/forum/XMPP_problem_behind_router/comment_2_3186ebe32c30764b9fd53625dd3e4eda._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="http://joeyh.name/"
+ ip="209.250.56.144"
+ subject="comment 2"
+ date="2014-09-25T15:48:39Z"
+ content="""
+I don't think that google XMPP server is at gmail.com. This suggests that the SRV record lookup to get from the email server to the xmpp server failed somehow. So, I'd look for a issue on the router's DNS server, or perhaps reconfigure DNS to bypass that server.
+"""]]
diff --git a/doc/forum/XMPP_problem_behind_router/comment_3_7fa8fe8cb92993c935ba2dbfb2aef728._comment b/doc/forum/XMPP_problem_behind_router/comment_3_7fa8fe8cb92993c935ba2dbfb2aef728._comment
new file mode 100644
index 0000000000..d8ee076f64
--- /dev/null
+++ b/doc/forum/XMPP_problem_behind_router/comment_3_7fa8fe8cb92993c935ba2dbfb2aef728._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="https://me.yahoo.com/a/wbh0dY54mcPwTpeOweuPQ8JiZrH3hg--#9b726"
+ nickname="Joe"
+ subject="comment 3"
+ date="2014-09-29T01:38:14Z"
+ content="""
+Thanks, that seemed to be the problem!  My router's dnsmasq had the 'filterwin2k' option set, which apparently [blocks queries for SRV records](http://wiki.openwrt.org/doc/howto/dhcp.dnsmasq#sip-phones.and.dnsmasq).  This may be fixed in newer OpenWRT versions, but my device is no longer supported.
+"""]]
diff --git a/doc/forum/XMPP_problem_behind_router/comment_4_6497ca72904dfdefe4e02f8a55127a8e._comment b/doc/forum/XMPP_problem_behind_router/comment_4_6497ca72904dfdefe4e02f8a55127a8e._comment
new file mode 100644
index 0000000000..959b298c7d
--- /dev/null
+++ b/doc/forum/XMPP_problem_behind_router/comment_4_6497ca72904dfdefe4e02f8a55127a8e._comment
@@ -0,0 +1,9 @@
+[[!comment format=mdwn
+ username="mdcotonopu@d25539d868f269dc6d1b838c21bb86205a40dab9"
+ nickname="mdcotonopu"
+ avatar="http://cdn.libravatar.org/avatar/7acb7ca4679a80a3ea544df55ae4959b"
+ subject="comment 4"
+ date="2017-01-11T09:35:48Z"
+ content="""
+Running high debug verbosity with DEBUG=* npm start indicates that the socket connection with the target XMPP server was started, but still the online event is never fired, and online is never logged. (The server times out at 150 seconds before anything can be logged.best 4 wireless routers
+"""]]
diff --git a/doc/forum/_Does_git_annex_find___40____38___friends__41___batch_queries_to_the_location_log__63__.mdwn b/doc/forum/_Does_git_annex_find___40____38___friends__41___batch_queries_to_the_location_log__63__.mdwn
new file mode 100644
index 0000000000..a9db915dab
--- /dev/null
+++ b/doc/forum/_Does_git_annex_find___40____38___friends__41___batch_queries_to_the_location_log__63__.mdwn
@@ -0,0 +1,45 @@
+Hi,
+
+Some time ago I asked [[here|git_annex_copy_--fast_--to_blah_much_slower_than_--from_blah]]
+about possible improvements in git `copy --fast --to`, since it was painfully slow
+on moderately large repos.
+
+Now I found a way to make it much faster for my particular use case, by
+accessing some annex internals. And I realized that maybe commands like `git
+annex find --in=repo` do not batch queries to the location log. This is based on
+the following timings, on a new repo (only a few commits) and about 30k files.
+
+    > time git annex find --in=skynet > /dev/null
+
+    real	0m55.838s
+    user	0m30.000s
+    sys	0m1.583s
+
+
+    > time git ls-tree -r git-annex | cut -d ' ' -f 3 | cut -f 1 | git cat-file --batch > /dev/null
+
+    real	0m0.334s
+    user	0m0.517s
+    sys	0m0.030s
+
+Those numbers are on linux (with an already warm file cache) and an ext4 filesystem on a SSD.
+
+The second command above is feeding a list of objects to a single `git cat-file`
+process that cats them all to stdout, preceeding every file dump by the object
+being cat-ed. It is a trivial matter to parse this output and use it for
+whatever annex needs.
+
+Above I wrote a `git ls-tree` on the git-annex branch for simplicity, but we could
+just as well do a `ls-tree | ... | git cat-file` on HEAD to get the keys for the
+annexed files matching some path and then feed those keys to a cat-file on
+the git-annex branch. And this still would be an order of magnitude faster than
+what currently annex seems to do.
+
+I'm assuming the bottleneck is in that annex does not batch the `cat-file`, as the rest of logic needed for a find will be fast. Is that right? 
+
+Now, if the queries to the location log for `copy --to` and `find` could be batched this way, the
+performance of several useful things to do, like checking how many annexed files
+we are missing, would be bastly improved. Hell, I could even put that number on
+the command line prompt!
+
+I'm not yet very fluent in Haskell, but I'm willing to help if this is something that makes sense and can be done.
diff --git a/doc/forum/_Does_git_annex_find___40____38___friends__41___batch_queries_to_the_location_log__63__/comment_1_01cbfc513c790faef3a3ede5315d3589._comment b/doc/forum/_Does_git_annex_find___40____38___friends__41___batch_queries_to_the_location_log__63__/comment_1_01cbfc513c790faef3a3ede5315d3589._comment
new file mode 100644
index 0000000000..6b4a6b2a05
--- /dev/null
+++ b/doc/forum/_Does_git_annex_find___40____38___friends__41___batch_queries_to_the_location_log__63__/comment_1_01cbfc513c790faef3a3ede5315d3589._comment
@@ -0,0 +1,10 @@
+[[!comment format=mdwn
+ username="http://joeyh.name/"
+ ip="209.250.56.47"
+ subject="comment 1"
+ date="2013-11-06T22:21:33Z"
+ content="""
+git-annex uses cat-file --batch, yes. You can verify this with --debug. Or you can read Annex/CatFile.hs and Git/CatFile.hs
+
+git-annex has to ensure that the git-annex branch is up-to-date and that any info synced into the repository is merged into it. This can require several calls to git log. Your command does not do that. git-annex find also runs `git ls-files --cached`, which has to examine the state of the index and of files on disk, in order to only show files that are in the working tree. Your command also omits that.
+"""]]
diff --git a/doc/forum/_Does_git_annex_find___40____38___friends__41___batch_queries_to_the_location_log__63__/comment_2_fe28dfb360caa12d5d5bc186def3eb45._comment b/doc/forum/_Does_git_annex_find___40____38___friends__41___batch_queries_to_the_location_log__63__/comment_2_fe28dfb360caa12d5d5bc186def3eb45._comment
new file mode 100644
index 0000000000..4ba4c82646
--- /dev/null
+++ b/doc/forum/_Does_git_annex_find___40____38___friends__41___batch_queries_to_the_location_log__63__/comment_2_fe28dfb360caa12d5d5bc186def3eb45._comment
@@ -0,0 +1,35 @@
+[[!comment format=mdwn
+ username="https://www.google.com/accounts/o8/id?id=AItOawkkyBDsfOB7JZvPZ4a8F3rwv0wk6Nb9n48"
+ nickname="Abdó"
+ subject="comment 2"
+ date="2013-11-06T23:14:03Z"
+ content="""
+Ok, then I don't understand where annex spends its time. git annex takes 55
+seconds! vs less than a second for a batched query on all the keys in the
+location log. Checking that branches are in sync, or traversing the working dir
+shouldn't amount the extra 54 seconds! At least not on a recently synced repo
+with up to date index and clean working dir.
+
+> git-annex has to ensure that the git-annex branch is up-to-date and that any info synced into the repository is merged into it. This can require several calls to git log
+
+Ok, I understand that. Checking that should be typically fast though, isn't it? On a repo that has just been synced, it doesn't need to go very far on the log.
+
+> git-annex find also runs git ls-files --cached, which has to examine the state of the index and of files on disk, in order to only show files that are in the working tree
+
+I understand that too. For my particular use case, I know I do the `git copy` when the
+repo is in sync and the working dir has no uncommited changes. So I use HEAD to retrieve the keys for
+the files in the working tree. I do something like that:
+
+    time git ls-tree -r HEAD | grep -e '^120000' | cut -d ' ' -f 3 | cut -f 1 | git cat-file --batch > /dev/null
+
+    real	0m0.178s
+    user	0m0.277s
+    sys 	0m0.037s
+
+That plus some fast parsing of the output gets the list of keys for the files in HEAD in less than a second. Where do the 54 extra seconds hide, then?
+
+Mm... how does annex retrieve the keys for files in the working tree? Does it follow
+the actual symlinks on the filesystem? I can believe that following 30k symlinks may be slow (although not 55 second slow).
+
+Sorry for being so insistent on this... It is just that I do think the same can be done much faster, and such an improvement in performance would be very interesting, not only for me.
+"""]]
diff --git a/doc/forum/_Does_git_annex_find___40____38___friends__41___batch_queries_to_the_location_log__63__/comment_3_9bb30ab62febe4ef63bed49f831a473a._comment b/doc/forum/_Does_git_annex_find___40____38___friends__41___batch_queries_to_the_location_log__63__/comment_3_9bb30ab62febe4ef63bed49f831a473a._comment
new file mode 100644
index 0000000000..c92be8ad53
--- /dev/null
+++ b/doc/forum/_Does_git_annex_find___40____38___friends__41___batch_queries_to_the_location_log__63__/comment_3_9bb30ab62febe4ef63bed49f831a473a._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="http://joeyh.name/"
+ ip="209.250.56.47"
+ subject="comment 3"
+ date="2013-11-07T18:12:09Z"
+ content="""
+It's hard to say until some profiling has actually been done. Comparing apples and oranges, or perhaps better to say, orange blossoms and oranges, is not very useful. 
+"""]]
diff --git a/doc/forum/_Does_git_annex_find___40____38___friends__41___batch_queries_to_the_location_log__63__/comment_4_7832f0347a41b8204538c01b72487803._comment b/doc/forum/_Does_git_annex_find___40____38___friends__41___batch_queries_to_the_location_log__63__/comment_4_7832f0347a41b8204538c01b72487803._comment
new file mode 100644
index 0000000000..3e7eb01dba
--- /dev/null
+++ b/doc/forum/_Does_git_annex_find___40____38___friends__41___batch_queries_to_the_location_log__63__/comment_4_7832f0347a41b8204538c01b72487803._comment
@@ -0,0 +1,19 @@
+[[!comment format=mdwn
+ username="https://www.google.com/accounts/o8/id?id=AItOawkkyBDsfOB7JZvPZ4a8F3rwv0wk6Nb9n48"
+ nickname="Abdó"
+ subject="comment 4"
+ date="2013-11-07T18:40:27Z"
+ content="""
+Ok, thanks.
+
+I understand that annex is more sophisticated than what I'm proposing. I'm also
+not comparing apples to oranges, what I'm saying is this: All the
+logic and checking and git calls that I thought annex needs to do, take
+much much less time than those 55 seconds. So either I'm missing something big (about 98% of it in execution time), or
+annex is doing something very inefficient. That's all I'm saying, and I was
+hoping that you could clarify that, since you already know the code.
+
+Also I'm sorry for being so annoying. If annex were writen in C, I would either have understood those 55 seconds or sent you a patch making annex faster. Being in
+haskell means that it will take me considerably more time and effort. But if I manage to
+improve things and if it is ok with you I'll send you patches... eventually.
+"""]]
diff --git a/doc/forum/__171__Locking__187___files_until_synced.mdwn b/doc/forum/__171__Locking__187___files_until_synced.mdwn
new file mode 100644
index 0000000000..129927bdc1
--- /dev/null
+++ b/doc/forum/__171__Locking__187___files_until_synced.mdwn
@@ -0,0 +1,7 @@
+I’m currently trying to set up a sync which involves an SQLite3 database file that should always be in the same state on all systems at all times (since there is no readily available way of merging the data). Basically, I’m looking for a practical way that gives me some help in making sure the files never drift apart between my remotes. Since I’m forgetful and might forget syncing the repo before going home from the office, I was wondering whether there might be a good way to assist me in this. Has anyone had some good ideas in this direction, or is there a canonical solution?
+
+It occurred to me one way would be to instate a hook somewhere that links the database to /dev/null in all other remotes (so the software will fail to work if I start it without having synced), but it seems tricky. I guess would have to involve per-remote branches, which will be hard to do, since I need to use direct mode.
+
+A less complex method might be forcing a sync before shutting down the system resp. when booting up.
+
+Maybe other folks have had ideas for a practical, yet robust solution.
diff --git a/doc/forum/__171__Locking__187___files_until_synced/comment_1_8bf59f47fee0a8d5741fe209b5899863._comment b/doc/forum/__171__Locking__187___files_until_synced/comment_1_8bf59f47fee0a8d5741fe209b5899863._comment
new file mode 100644
index 0000000000..d327a58b4c
--- /dev/null
+++ b/doc/forum/__171__Locking__187___files_until_synced/comment_1_8bf59f47fee0a8d5741fe209b5899863._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="http://joeyh.name/"
+ ip="209.250.56.55"
+ subject="comment 1"
+ date="2014-07-08T17:31:53Z"
+ content="""
+This is something git's distributed nature preludes. You might frankly be better with one of the centralized VCSs that supports locking in this specific case. Or, you can do some kind of \"lock server\" that the clients contact, layering a centralized approach on top of git.
+"""]]
diff --git a/doc/forum/__171__Locking__187___files_until_synced/comment_2_0c683547a6178e4303f0b1ed1f5605a5._comment b/doc/forum/__171__Locking__187___files_until_synced/comment_2_0c683547a6178e4303f0b1ed1f5605a5._comment
new file mode 100644
index 0000000000..b76f35da5e
--- /dev/null
+++ b/doc/forum/__171__Locking__187___files_until_synced/comment_2_0c683547a6178e4303f0b1ed1f5605a5._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="zardoz"
+ ip="134.147.14.84"
+ subject="comment 2"
+ date="2014-07-09T07:07:27Z"
+ content="""
+Yea, it’s basically just the isolated one file that needs «locking», so one could just try a different route. The idea with keeping state on a server sounds like a pretty good idea. I could just «echo» via SSH to my webserver some info on who has the file locked; then I could wrap the program that uses the SQLite3 DB in a script that checks whether it’s safe.
+"""]]
diff --git a/doc/forum/__34__Lost__34___data__63___Maybe_a_bug__63__.mdwn b/doc/forum/__34__Lost__34___data__63___Maybe_a_bug__63__.mdwn
new file mode 100644
index 0000000000..20f6d696a5
--- /dev/null
+++ b/doc/forum/__34__Lost__34___data__63___Maybe_a_bug__63__.mdwn
@@ -0,0 +1,14 @@
+Hey,
+
+I lost some symlinks to my data and I do not know how to recover them. I was in view mode  with some tag folders already there. I added _new_ files from outside annex into some folder and 'git annex add' those files.
+
+What I expected: Git-Annex should add those files to the annex, move the symlinks to the root of the annex (because there is no other way to tell where to put them) and tag them with the specific tag. That is the way I would like to work, first tag, then organize in folder structure.
+
+Now that seems not to be a scenario which has been respected? Because I don't see my files... anywhere. Not in master branch nor in the view branch (I already did 'git annex vpop'). If that is not supported and never will be git-annex should not accept data from the outside world if it is currently in view mode.
+
+Now, how do I get my symlinks back? I guess the content is still there, but the links are missing and I don't find any reference or history log to revert that. 'git annex unused' does not show them either.
+
+I hope somebody can help me :)
+
+Cheers,
+Stephan
diff --git a/doc/forum/__34__Lost__34___data__63___Maybe_a_bug__63__/comment_1_52e4453432184524d84d88f6382cac9d._comment b/doc/forum/__34__Lost__34___data__63___Maybe_a_bug__63__/comment_1_52e4453432184524d84d88f6382cac9d._comment
new file mode 100644
index 0000000000..2ecde6390d
--- /dev/null
+++ b/doc/forum/__34__Lost__34___data__63___Maybe_a_bug__63__/comment_1_52e4453432184524d84d88f6382cac9d._comment
@@ -0,0 +1,16 @@
+[[!comment format=mdwn
+ username="sts"
+ ip="134.147.240.107"
+ subject="comment 1"
+ date="2014-08-11T11:39:44Z"
+ content="""
+OK, I could find the commit where I have added the data. I can 'git show' the commit and see the keys. I can also checkout the commit and I can see my data. Now I tried to create symlinks based on the keys I found in the commit, so whats the right way?
+
+    git annex examinekey SHA256E-s1390161393--1dcba6e914ad6a9133d374e3c55fbf9a58f036e64298262692c7f8e7cdb65852.mkv
+    SHA256E-s1390161393--1dcba6e914ad6a9133d374e3c55fbf9a58f036e64298262692c7f8e7cdb65852.mkv
+
+    git annex fromkey SHA256E-s1390161393--1dcba6e914ad6a9133d374e3c55fbf9a58f036e64298262692c7f8e7cdb65852.mkv e01.mkv
+    git-annex: key (SHA256E-s1390161393--1dcba6e914ad6a9133d374e3c55fbf9a58f036e64298262692c7f8e7cdb65852.mkv) is not present in backend
+
+I am not sure what to do now :-/.
+"""]]
diff --git a/doc/forum/__34__Lost__34___data__63___Maybe_a_bug__63__/comment_2_4d80b96e788d233706fa8f3e363d2f76._comment b/doc/forum/__34__Lost__34___data__63___Maybe_a_bug__63__/comment_2_4d80b96e788d233706fa8f3e363d2f76._comment
new file mode 100644
index 0000000000..e9122c069c
--- /dev/null
+++ b/doc/forum/__34__Lost__34___data__63___Maybe_a_bug__63__/comment_2_4d80b96e788d233706fa8f3e363d2f76._comment
@@ -0,0 +1,12 @@
+[[!comment format=mdwn
+ username="http://joeyh.name/"
+ ip="209.250.56.7"
+ subject="comment 2"
+ date="2014-08-12T17:57:26Z"
+ content="""
+views are a somewhat new feature that still needs work, and you will find this question of what to do about a file added while in a view in the todo list [[here|design/metadata]].
+
+Since views are just git branches, you can check out the view branch where you added the file, and it'll still be there. You could merge the branch (probably not a good idea since the filenames are moved around in the view). 
+
+Using `fromkey` will also work, if you have the right key and the content is present in the annex -- I just tested it. 
+"""]]
diff --git a/doc/forum/__34__Pairing__34___more_than_two_computers.mdwn b/doc/forum/__34__Pairing__34___more_than_two_computers.mdwn
new file mode 100644
index 0000000000..099c7a6631
--- /dev/null
+++ b/doc/forum/__34__Pairing__34___more_than_two_computers.mdwn
@@ -0,0 +1,11 @@
+I need some help understanding here.
+
+We use AeroFS at work to sync the normal user files across computers. I'll quite likely be replacing that with git annex as soon as Windows port is stable enough.
+
+How it works is that you create a repo, and share it with one or more users. Then AeroFS discovers what other repos are online and if they're on the local network, and syncs from wherever is most convenient.
+
+This sounds a little like pairing, but with pairing you need to arrange more than 2 devices in a "star" or "chain". This is fine for my own devices, but it becomes brittle if you have lots of devices syncing from each other, which may or may not be online at any given time. The only way around it seems to be to pair each device with each other device, with the labor rising exponentially.
+
+Is this correct? Best compromise for my use case seems to be to just do a star setup and pair everything with an always-on machine. Do you agree?
+
+Thanks! Carlo
diff --git a/doc/forum/__34__Pairing__34___more_than_two_computers/comment_1_80f7a4bb3c66b11e566043407b611bbf._comment b/doc/forum/__34__Pairing__34___more_than_two_computers/comment_1_80f7a4bb3c66b11e566043407b611bbf._comment
new file mode 100644
index 0000000000..026681f79c
--- /dev/null
+++ b/doc/forum/__34__Pairing__34___more_than_two_computers/comment_1_80f7a4bb3c66b11e566043407b611bbf._comment
@@ -0,0 +1,10 @@
+[[!comment format=mdwn
+ username="http://joeyh.name/"
+ ip="64.134.31.139"
+ subject="comment 1"
+ date="2013-10-22T17:41:58Z"
+ content="""
+You're basically correct. Typically if you have a central machine, it's a server, so at that point you're not pairing, but are just adding a ssh server, or some other type of remote.
+
+I think that systems like AeroFS are pretty neat, but my goal is not really to build another one of those. I'd rather make git-annex be able to use such things as [[special_remotes]].
+"""]]
diff --git a/doc/forum/__34__Preseeding__34___a_special_remote.mdwn b/doc/forum/__34__Preseeding__34___a_special_remote.mdwn
new file mode 100644
index 0000000000..4b1342598c
--- /dev/null
+++ b/doc/forum/__34__Preseeding__34___a_special_remote.mdwn
@@ -0,0 +1,5 @@
+I have a remote repository that I have yet to add as a remote into git-annex, but which already contains data that I want git-annex to manage. I already know what the SHA256 hashes and sizes of all the files are, and I can arrange them to match what git-annex will expect.
+
+Is there a way that I can tell git-annex about the presence of the data, to save me having to download and re-upload everything, in a way that is safe? What I want seems to be similar to "git annex reinject" but for special remotes (and I'll take care of the renaming), but I don't see anything in the manpage that looks likely.
+
+I can quite easily create and commit the symlinks with correctly predicted names in my master branch. Will git-annex will treat these correctly?
diff --git a/doc/forum/__34__Preseeding__34___a_special_remote/comment_1_b0c46d0eba900d0f6169a2c698d7a222._comment b/doc/forum/__34__Preseeding__34___a_special_remote/comment_1_b0c46d0eba900d0f6169a2c698d7a222._comment
new file mode 100644
index 0000000000..6fae5c62e3
--- /dev/null
+++ b/doc/forum/__34__Preseeding__34___a_special_remote/comment_1_b0c46d0eba900d0f6169a2c698d7a222._comment
@@ -0,0 +1,12 @@
+[[!comment format=mdwn
+ username="http://joeyh.name/"
+ ip="209.250.56.3"
+ subject="comment 1"
+ date="2014-05-21T17:41:22Z"
+ content="""
+If you also have the files present locally, you can simply do `git annex copy --fast --to remote`. git-annex copy will first check to see if the remote has the file; seeing that it does it will update the location log.
+
+Another option, if you have shell access on the remote is to simply set up a git repository there, move the files into it and `git annex add` them, and merge that into your local repository.
+
+There is not currently any way to set the [[location_tracking]] information to tell git-annex that a file has appeared on a remote. Of course, you can modify the git-annex branch manually to do so. See [[internals]].
+"""]]
diff --git a/doc/forum/__34__Preseeding__34___a_special_remote/comment_2_5e63f5e6f45c11cc86b293ce8acad77f._comment b/doc/forum/__34__Preseeding__34___a_special_remote/comment_2_5e63f5e6f45c11cc86b293ce8acad77f._comment
new file mode 100644
index 0000000000..cdbf1e76de
--- /dev/null
+++ b/doc/forum/__34__Preseeding__34___a_special_remote/comment_2_5e63f5e6f45c11cc86b293ce8acad77f._comment
@@ -0,0 +1,65 @@
+[[!comment format=mdwn
+ username="https://www.google.com/accounts/o8/id?id=AItOawkexhIpGcYa22aPQtLm-StpHiF-MHYPh5w"
+ nickname="Robie"
+ subject="Solved"
+ date="2014-06-01T13:55:58Z"
+ content="""
+Thanks to Joey for pointing me in the right direction. I got this working now.
+
+There are approximately three steps:
+
+1. Obtain a mapping of git-annex key to friendly name, and rename all entries
+in the special remote to their git-annex keys.
+
+2. Create and commit symlinks in the `master` branch (or wherever you want them).
+
+3. Add location tracking entries to the `git-annex` branch for all entries.
+
+First, I created an \"index\" file describing the contents of my special remote,
+in the form \"KEY NAME\" where KEY is the git-annex key (I used SHA256) and NAME
+is the name I want to use for each file.
+
+## Step 1: Map and rename
+
+In my case I was \"importing\" a ddar remote, so I wrote a quick script
+(https://github.com/basak/ddar/blob/master/contrib/git-annex-convert.py) to
+generate this index as well as rename all ddar archive members to their
+git-annex keys instead.
+
+## Step 2: Create and commit symlinks
+
+Then, I created symlinks in my master branch using:
+
+    exec 3> \"$log\";git add \"$log\";done
+    exec 3<&-
+    git commit -m'Import knowledge of ddar repository contents'
+
+## Verifying
+
+`git-annex fsck --from ddar --fast` checks that the keys expected in the
+special remote can be found (replace the special remote name as needed).
+
+Skipping `--fast` will download all data to verify it. I didn't do this -
+instead I just sampled one entry which seemed to be OK.
+
+
+
+"""]]
diff --git a/doc/forum/__34__Scanning_for_files_to_transfer__34__.mdwn b/doc/forum/__34__Scanning_for_files_to_transfer__34__.mdwn
new file mode 100644
index 0000000000..e08be0b39d
--- /dev/null
+++ b/doc/forum/__34__Scanning_for_files_to_transfer__34__.mdwn
@@ -0,0 +1,9 @@
+I have a Git Annex repository on my laptop with currently 141094 files.
+
+I used "git config annex.startupscan false" to disable the startup scan.
+
+Yet the assistant still runs at high CPU consumption for extended periods of time after starting.
+
+In the webapp, the startup scan is shown as completed, but under "Transfers" it says "Scanning for files to transfer".
+
+What is the difference between "Performing startup scan" and "Scanning for files to transfer"?
diff --git a/doc/forum/__34__Scanning_for_files_to_transfer__34__/comment_1_4fa6d0d5264707886e1e9b5184090386._comment b/doc/forum/__34__Scanning_for_files_to_transfer__34__/comment_1_4fa6d0d5264707886e1e9b5184090386._comment
new file mode 100644
index 0000000000..cab618d639
--- /dev/null
+++ b/doc/forum/__34__Scanning_for_files_to_transfer__34__/comment_1_4fa6d0d5264707886e1e9b5184090386._comment
@@ -0,0 +1,36 @@
+[[!comment format=mdwn
+ username="joey"
+ subject="""comment 1"""
+ date="2017-03-06T17:03:12Z"
+ content="""
+The scan that is skipped is one of the files on disk in order to find
+changes that were made while the assistant was not running.
+
+What you are seeing is the full transfer scan. While annex.startupscan
+could be made to also skip that scan, a full transfer scan is not only run
+at startup, but after merging git-annex branch changes from a remote. So
+disabling it only at startup does not seem very useful.
+
+There could be an option to disable the full transfer scan ever running.
+However, this would make the assistant not notice certian transfers/drops
+that you would normally want it to do. For example, if a remote got a bunch
+of files in an archive/ directory from somewhere else, and the local
+repository contains those files, the full transfer scan is needed to notice
+that the archived files can now be removed from the local repository.
+In other situations, the local repository would not get files that it
+ought to contain.
+
+So, I think it might be better to make the expensive transfer scan run a
+little bit slower so it doesn't peg your CPU. I've added a 1/200th second
+delay after each file it checks. 
+
+That will make it use something like 
+5-10% of the CPU, instead of 100%. At the same time it doesn't slow down the
+total scan very much. In a repository with 5k files, it makes the scan 25
+seconds slower, which makes the assistant react that much slower -- but
+the expensive scan is only needed to make sure things turn out consistent,
+so its overall speed is not super important.
+
+Check it out, let me know if it's still using too much CPU. We could always
+make that 1/200th second tunable, or find a better value for it.
+"""]]
diff --git a/doc/forum/__34__du__34___equivalent_on_an_annex__63__.mdwn b/doc/forum/__34__du__34___equivalent_on_an_annex__63__.mdwn
new file mode 100644
index 0000000000..7485958c0b
--- /dev/null
+++ b/doc/forum/__34__du__34___equivalent_on_an_annex__63__.mdwn
@@ -0,0 +1,5 @@
+One reason to use git-annex is to save disk space by tossing files you don't use that often.
+
+I can find big files in the repository with git annex find --largerthan=100M, but is there a way to find large *directories*?  In an ordinary filesystem I'd use "du -h" with a maxdepth to get an idea of what parts of a directory are taking up my disk space, but obviously that won't work with git annex because all the content is in .git/annex.  Any ideas?
+
+(I can get a listing of file sizes in a directory with the handy -L flag of ls -- "ls -lL" shows me the sizes of the link targets -- but that won't summarize all the sizes of subdirectories.  Unless my ls-fu is just weak.)
diff --git a/doc/forum/__34__du__34___equivalent_on_an_annex__63__/comment_1_a41bd02361aa961e5285aeaf1ea062be._comment b/doc/forum/__34__du__34___equivalent_on_an_annex__63__/comment_1_a41bd02361aa961e5285aeaf1ea062be._comment
new file mode 100644
index 0000000000..0b69a41424
--- /dev/null
+++ b/doc/forum/__34__du__34___equivalent_on_an_annex__63__/comment_1_a41bd02361aa961e5285aeaf1ea062be._comment
@@ -0,0 +1,10 @@
+[[!comment format=mdwn
+ username="http://sunny256.sunbase.org/"
+ nickname="sunny256"
+ subject="comment 1"
+ date="2012-11-28T23:21:11Z"
+ content="""
+du(1) also accepts the -L option, so if you for example want to find what directories occupies most storage:
+
+    $ du -L | sort -n
+"""]]
diff --git a/doc/forum/__34__du__34___equivalent_on_an_annex__63__/comment_2_28ba62a546f5cc8f416491423d743d8a._comment b/doc/forum/__34__du__34___equivalent_on_an_annex__63__/comment_2_28ba62a546f5cc8f416491423d743d8a._comment
new file mode 100644
index 0000000000..019cb8b81c
--- /dev/null
+++ b/doc/forum/__34__du__34___equivalent_on_an_annex__63__/comment_2_28ba62a546f5cc8f416491423d743d8a._comment
@@ -0,0 +1,10 @@
+[[!comment format=mdwn
+ username="http://sunny256.sunbase.org/"
+ nickname="sunny256"
+ subject="comment 2"
+ date="2012-11-28T23:24:11Z"
+ content="""
+And if you want to find the biggest files in a directory tree:
+
+    $ find -type l -print0 | xargs -0 du -L | sort -n | tail -500
+"""]]
diff --git a/doc/forum/__34__du__34___equivalent_on_an_annex__63__/comment_3_8d97f40c1d14b7230f3656a00a99cf80._comment b/doc/forum/__34__du__34___equivalent_on_an_annex__63__/comment_3_8d97f40c1d14b7230f3656a00a99cf80._comment
new file mode 100644
index 0000000000..0a11f7cc93
--- /dev/null
+++ b/doc/forum/__34__du__34___equivalent_on_an_annex__63__/comment_3_8d97f40c1d14b7230f3656a00a99cf80._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="http://edheil.wordpress.com/"
+ ip="173.162.44.162"
+ subject="comment 3"
+ date="2012-11-29T02:03:24Z"
+ content="""
+Sweet!  I should have RTFM a bit more.  Thanks. :)
+"""]]
diff --git a/doc/forum/__34__du__34___equivalent_on_an_annex__63__/comment_4_baa8fbbdd5c449a0dc2bb622cb4a47ce._comment b/doc/forum/__34__du__34___equivalent_on_an_annex__63__/comment_4_baa8fbbdd5c449a0dc2bb622cb4a47ce._comment
new file mode 100644
index 0000000000..227c74b026
--- /dev/null
+++ b/doc/forum/__34__du__34___equivalent_on_an_annex__63__/comment_4_baa8fbbdd5c449a0dc2bb622cb4a47ce._comment
@@ -0,0 +1,18 @@
+[[!comment format=mdwn
+ username="Steve"
+ ip="92.104.175.136"
+ subject="comment 4"
+ date="2012-11-29T23:51:21Z"
+ content="""
+I've been thinking about writing a sort of git-annex du.  I'm surprised to find someone else looking for such a thing.  While \"du -L\" will tell you how much space is used by files you actually have, I was interested in knowing (approximately) how much space would be used if you were to git-annex get everything you don't yet have.
+
+There are many options and variations to think about, such as:
+
+* do you want to count duplicate files once or as many times as they appear (as if you 'git-annex lock'd them all)
+* maybe you want to know how much space is used by files that reside only on a certain remote or set of remotes
+* you might want to know how much space would be used by all the files you don't yet have, but not count the files you already have
+
+All of the backends so so far seem to store the size of the files in the filename, so my plan was to read it out of the links.  If anybody has a better idea about how to get the sizes of annexed files or options that would be handy for a git-annex du, let me know.  I'll see if I can get the start of something useful this weekend.  I'll post here when I have something to share.
+
+I'm also open to suggestions for the executable name.  Right now I'm thinking \"gadu\" for git-annex disk usage.
+"""]]
diff --git a/doc/forum/__34__du__34___equivalent_on_an_annex__63__/comment_5_2ee6cbbfe54a2e7b6e8eb539c18e663d._comment b/doc/forum/__34__du__34___equivalent_on_an_annex__63__/comment_5_2ee6cbbfe54a2e7b6e8eb539c18e663d._comment
new file mode 100644
index 0000000000..021614405d
--- /dev/null
+++ b/doc/forum/__34__du__34___equivalent_on_an_annex__63__/comment_5_2ee6cbbfe54a2e7b6e8eb539c18e663d._comment
@@ -0,0 +1,10 @@
+[[!comment format=mdwn
+ username="http://sunny256.sunbase.org/"
+ nickname="sunny256"
+ subject="comment 5"
+ date="2012-11-30T00:29:44Z"
+ content="""
+Steve, that would be a very useful utility. I've been thinking of such a tool, but haven't gotten around to write it yet. It would be practical to have before copying big/many files from another drive. If I've been short of free space, I've executed `du -L` in the source directory, but that's a bit cumbersome.
+
+And \"gadu\" is a fine name, yes. Goes well along with my \"ga\" shortcut for \"git annex\", which I created two hours after I started using git-annex. I've probably saved thousands of keystrokes because of that. ☺
+"""]]
diff --git a/doc/forum/__34__du__34___equivalent_on_an_annex__63__/comment_6_48f6a2761a34b7f991325f1d24e2c5ff._comment b/doc/forum/__34__du__34___equivalent_on_an_annex__63__/comment_6_48f6a2761a34b7f991325f1d24e2c5ff._comment
new file mode 100644
index 0000000000..1c456e24b3
--- /dev/null
+++ b/doc/forum/__34__du__34___equivalent_on_an_annex__63__/comment_6_48f6a2761a34b7f991325f1d24e2c5ff._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="Steve"
+ ip="92.104.175.136"
+ subject="gadu 0.01 is up"
+ date="2012-12-08T06:22:50Z"
+ content="""
+I've got an initial try at gadu up over at   I created a separate thread for it: 
+"""]]
diff --git a/doc/forum/__34__du__34___equivalent_on_an_annex__63__/comment_7_d632baff41b8582f1a79bc5018c68545._comment b/doc/forum/__34__du__34___equivalent_on_an_annex__63__/comment_7_d632baff41b8582f1a79bc5018c68545._comment
new file mode 100644
index 0000000000..6fa2659f93
--- /dev/null
+++ b/doc/forum/__34__du__34___equivalent_on_an_annex__63__/comment_7_d632baff41b8582f1a79bc5018c68545._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="https://me.yahoo.com/a/2grhJvAC049fJnvALDXek.6MRZMTlg--#eec89"
+ nickname="John"
+ subject="Also try sizes"
+ date="2013-01-02T22:05:39Z"
+ content="""
+The \"sizes\" tool on Hackage is a git-annex aware du-like utility.  Just give it the \"-A\" flag to have it interpret annex symlinks as if they were normal files, and to also ignore files inside a .git/annex.
+"""]]
diff --git a/doc/forum/__34__du__34___equivalent_on_an_annex__63__/comment_8_6a51c22d9893fa5b1503f5a14b1eb8ce._comment b/doc/forum/__34__du__34___equivalent_on_an_annex__63__/comment_8_6a51c22d9893fa5b1503f5a14b1eb8ce._comment
new file mode 100644
index 0000000000..349c145cd5
--- /dev/null
+++ b/doc/forum/__34__du__34___equivalent_on_an_annex__63__/comment_8_6a51c22d9893fa5b1503f5a14b1eb8ce._comment
@@ -0,0 +1,7 @@
+[[!comment format=mdwn
+ username="https://id.koumbit.net/anarcat"
+ subject="comment 8"
+ date="2015-03-31T03:11:06Z"
+ content="""
+i have used the [symlink patch of ncdu](http://dev.yorhel.nl/ncdu/bug/16) with good results here...
+"""]]
diff --git a/doc/forum/__34__du__34___equivalent_on_an_annex__63__/comment_9_a2bda4c74ef09a58709c0c5f6ee1b726._comment b/doc/forum/__34__du__34___equivalent_on_an_annex__63__/comment_9_a2bda4c74ef09a58709c0c5f6ee1b726._comment
new file mode 100644
index 0000000000..0e344a44d4
--- /dev/null
+++ b/doc/forum/__34__du__34___equivalent_on_an_annex__63__/comment_9_a2bda4c74ef09a58709c0c5f6ee1b726._comment
@@ -0,0 +1,7 @@
+[[!comment format=mdwn
+ username="anarcat"
+ subject="comment 9"
+ date="2015-06-24T21:40:07Z"
+ content="""
+... but nowadays, i use `git annex info --fast *`.
+"""]]
diff --git a/doc/forum/__34__git_add__34___vs___34__git_annex_add__34___in_v6.mdwn b/doc/forum/__34__git_add__34___vs___34__git_annex_add__34___in_v6.mdwn
new file mode 100644
index 0000000000..493344e52d
--- /dev/null
+++ b/doc/forum/__34__git_add__34___vs___34__git_annex_add__34___in_v6.mdwn
@@ -0,0 +1,5 @@
+Prior to v6, I used `git annex add` for files that should go into the annex, and `git add` for files that should be tracked in git normally.  With v6, `git add` silently adds to the annex instead, which surprised me because I didn't expect the file contents to still only reside only on the source computer after a push!  I don't have `annex.largefiles` set, but it seems like this is intended behavior.  Is there a way to return to v5 behavior for those two commands?  Is it safe to just remove the smudge/clean filter?  If so, is there a way to remove it in such a way that it propagates to all repositories?  (It's currently set in `.git/info/attributes`, which overrides `.gitattributes`).
+
+Having to come up a generic rule for `annex.largefiles` to cover all of my distinctions is difficult, and it's tedious to re-specify it for each new file.  I'd much rather just be able to continue using git as usual, and explicitly choose when to invoke git-annex.
+
+Jim
diff --git a/doc/forum/__34__git_add__34___vs___34__git_annex_add__34___in_v6/comment_1_285032a01ca754c539ce0634823db23c._comment b/doc/forum/__34__git_add__34___vs___34__git_annex_add__34___in_v6/comment_1_285032a01ca754c539ce0634823db23c._comment
new file mode 100644
index 0000000000..f27c0602ae
--- /dev/null
+++ b/doc/forum/__34__git_add__34___vs___34__git_annex_add__34___in_v6/comment_1_285032a01ca754c539ce0634823db23c._comment
@@ -0,0 +1,29 @@
+[[!comment format=mdwn
+ username="joey"
+ subject="""comment 1"""
+ date="2016-09-21T20:25:10Z"
+ content="""
+This is a documented behavior change in v6 mode. I'm willing to listen to
+arguments that the behavior change is a bad idea, but see below.
+
+To get the old behavior, you can use:
+
+	git -c 'annex.largefiles=exclude=*' add
+
+Unfortunately git won't let you alias git add to always pass that
+switch. But you could alias `git sadd` or something to use that switch.
+
+----
+
+The same behavior change also makes `git commit -a` use git-annex when file
+contents have changed, rather than the old hacky method which added the
+files to git and then undid that in a pre-commit hook. So there's quite a
+nice benefit there.
+
+And for that reason it doesn't make sense to add a configuration option to
+disable it, because such an option would break `git commit -a` of
+modified annexed files.
+
+Changing the gitattributes won't work because then the v6 repository won't
+get annexed files checked out properly.
+"""]]
diff --git a/doc/forum/__34__git_add__34___vs___34__git_annex_add__34___in_v6/comment_2_93eec60060bc7bca8dd1633b670f9a53._comment b/doc/forum/__34__git_add__34___vs___34__git_annex_add__34___in_v6/comment_2_93eec60060bc7bca8dd1633b670f9a53._comment
new file mode 100644
index 0000000000..721a1d0c1f
--- /dev/null
+++ b/doc/forum/__34__git_add__34___vs___34__git_annex_add__34___in_v6/comment_2_93eec60060bc7bca8dd1633b670f9a53._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="spwhitton"
+ avatar="http://cdn.libravatar.org/avatar/9c3f08f80e67733fd506c353239569eb"
+ subject="comment 2"
+ date="2018-04-27T16:39:49Z"
+ content="""
+Is the `git annex -a` benefit one of performance?  I.e. the reason why the pre-commit hook approach was less good was that could be slower?
+"""]]
diff --git a/doc/forum/__34__git_annex_assistant_--stop__34___doesn__39__t_work_in_OSX_10.11.4.mdwn b/doc/forum/__34__git_annex_assistant_--stop__34___doesn__39__t_work_in_OSX_10.11.4.mdwn
new file mode 100644
index 0000000000..c57a50c358
--- /dev/null
+++ b/doc/forum/__34__git_annex_assistant_--stop__34___doesn__39__t_work_in_OSX_10.11.4.mdwn
@@ -0,0 +1 @@
+As the title suggests, stopping the assistant through the terminal doesn't work as expected and I had to kill the assitant by hand, which in turn means, that git annex has to repair the repo. A workaround seems to be to start the assistant through "git annex webapp" and use the shutdown daemon button in the webapp. Before this workaround does work, you have to "git annex assistant --autostop" and restart, so that launchctl won't keep the daemon running.
diff --git a/doc/forum/__34__git_annex_assistant_--stop__34___doesn__39__t_work_in_OSX_10.11.4/comment_1_34bd938e8cd0cba6ec48020932858485._comment b/doc/forum/__34__git_annex_assistant_--stop__34___doesn__39__t_work_in_OSX_10.11.4/comment_1_34bd938e8cd0cba6ec48020932858485._comment
new file mode 100644
index 0000000000..cb94b7225e
--- /dev/null
+++ b/doc/forum/__34__git_annex_assistant_--stop__34___doesn__39__t_work_in_OSX_10.11.4/comment_1_34bd938e8cd0cba6ec48020932858485._comment
@@ -0,0 +1,7 @@
+[[!comment format=mdwn
+ username="openmedi"
+ subject="comment 1"
+ date="2016-05-12T06:57:15Z"
+ content="""
+As it turns out, even this doesn't work hundred percent of the time, which is terrible! I rely on stoping the assistant once to at least three times a day to be able to skype without having to worry about my upload. My connection simply cannot handle both things at the same time. So what should I do?
+"""]]
diff --git a/doc/forum/__34__git_annex_assistant_--stop__34___doesn__39__t_work_in_OSX_10.11.4/comment_2_3ec07e7f0e713e6fa925abb99c646bc7._comment b/doc/forum/__34__git_annex_assistant_--stop__34___doesn__39__t_work_in_OSX_10.11.4/comment_2_3ec07e7f0e713e6fa925abb99c646bc7._comment
new file mode 100644
index 0000000000..294c5042fd
--- /dev/null
+++ b/doc/forum/__34__git_annex_assistant_--stop__34___doesn__39__t_work_in_OSX_10.11.4/comment_2_3ec07e7f0e713e6fa925abb99c646bc7._comment
@@ -0,0 +1,7 @@
+[[!comment format=mdwn
+ username="openmedi"
+ subject="comment 2"
+ date="2016-05-12T07:49:33Z"
+ content="""
+I finally could reliably prevent git annex from fuzzinh with my skype sessions by using [little snitch](http://obdev.at/products/littlesnitch). I created a profile that prevents outgoing connections from git annex etc. and now I can leave git annex assistant running in the background. This is at the same time a good and a terrible solution, since I now need to run commercial software to prevent git annex from dictating how well my skype sessions are going and at the same time not relying on killing git annex processes by hand which in turn leads to git annex wanting repair itself all the time.
+"""]]
diff --git a/doc/forum/__34__git_annex_assistant_--stop__34___doesn__39__t_work_in_OSX_10.11.4/comment_3_3dce000eb6db22e07ed2f53c4f62a1e0._comment b/doc/forum/__34__git_annex_assistant_--stop__34___doesn__39__t_work_in_OSX_10.11.4/comment_3_3dce000eb6db22e07ed2f53c4f62a1e0._comment
new file mode 100644
index 0000000000..8eaab758b3
--- /dev/null
+++ b/doc/forum/__34__git_annex_assistant_--stop__34___doesn__39__t_work_in_OSX_10.11.4/comment_3_3dce000eb6db22e07ed2f53c4f62a1e0._comment
@@ -0,0 +1,27 @@
+[[!comment format=mdwn
+ username="joey"
+ subject="""comment 3"""
+ date="2016-05-12T18:28:09Z"
+ content="""
+It seems to me that you could just click on "syncing enabled" in the
+webapp, to toggle it to "syncing disabled". This stops all data transfers
+without killing the program.
+
+The problem you describe with stopping doesn't quite make sense to me.
+
+For one things, `git annex assistant --autostop` does the exact same
+thing as changing directory into the repo and `git annex assistant --stop`.
+But, you seem to say that one works and the other doesn't.
+
+For another, `git annex assistant --stop` does exactly the same thing as
+killing the process by hand! That is, it sends it a TERM signal. 
+Perhaps you killed the process by hand with a stronger signal than TERM,
+but I don't see how even that would cause it to need to repair the repo.
+
+Why is launchd restarting the daemon when it exits?
+`~/Library/LaunchAgents/com.branchable.git-annex.assistant.plist`
+has RunAtLoad set, which AFAIK is supposed to make it run it once
+when you log in. It does not have KeepAlive set, so unless you've changed
+the configuration to include KeepAlive, I don't know why launchd would
+behave that way.
+"""]]
diff --git a/doc/forum/__34__git_annex_assistant_--stop__34___doesn__39__t_work_in_OSX_10.11.4/comment_4_5bed42e467859d2310f3da35e55acd38._comment b/doc/forum/__34__git_annex_assistant_--stop__34___doesn__39__t_work_in_OSX_10.11.4/comment_4_5bed42e467859d2310f3da35e55acd38._comment
new file mode 100644
index 0000000000..2036ad6fff
--- /dev/null
+++ b/doc/forum/__34__git_annex_assistant_--stop__34___doesn__39__t_work_in_OSX_10.11.4/comment_4_5bed42e467859d2310f3da35e55acd38._comment
@@ -0,0 +1,7 @@
+[[!comment format=mdwn
+ username="openmedi"
+ subject="comment 4"
+ date="2016-05-14T13:24:06Z"
+ content="""
+I assume now that the problem is related to the [b2 special remote](https://github.com/encryptio/git-annex-remote-b2) which might be preventing git annex from shutting down. I don't know. What should I do and post here to help you triage this problem?
+"""]]
diff --git a/doc/forum/__34__git_annex_assistant_--stop__34___doesn__39__t_work_in_OSX_10.11.4/comment_5_5cb379490b0e85c24bff48db8201c7d4._comment b/doc/forum/__34__git_annex_assistant_--stop__34___doesn__39__t_work_in_OSX_10.11.4/comment_5_5cb379490b0e85c24bff48db8201c7d4._comment
new file mode 100644
index 0000000000..2bf5237f04
--- /dev/null
+++ b/doc/forum/__34__git_annex_assistant_--stop__34___doesn__39__t_work_in_OSX_10.11.4/comment_5_5cb379490b0e85c24bff48db8201c7d4._comment
@@ -0,0 +1,9 @@
+[[!comment format=mdwn
+ username="joey"
+ subject="""comment 5"""
+ date="2016-05-16T21:23:09Z"
+ content="""
+Why do you think that the b2 special remote is causing the problem?
+
+(Seems unlikely to me..)
+"""]]
diff --git a/doc/forum/__34__git_annex_assistant_--stop__34___doesn__39__t_work_in_OSX_10.11.4/comment_6_46eb0f4dcbccea5140a26dca36c6f1d1._comment b/doc/forum/__34__git_annex_assistant_--stop__34___doesn__39__t_work_in_OSX_10.11.4/comment_6_46eb0f4dcbccea5140a26dca36c6f1d1._comment
new file mode 100644
index 0000000000..97caaaff46
--- /dev/null
+++ b/doc/forum/__34__git_annex_assistant_--stop__34___doesn__39__t_work_in_OSX_10.11.4/comment_6_46eb0f4dcbccea5140a26dca36c6f1d1._comment
@@ -0,0 +1,11 @@
+[[!comment format=mdwn
+ username="openmedi"
+ subject="comment 6"
+ date="2016-05-16T23:11:34Z"
+ content="""
+I did assume so, because b2s process seems to be running when the assistant fails to shut down. But a new, maybe related problem might have come to light: I have/had a gitlab remote, that grew so big (30 gig) that they shut it down, in the sense that I can't access it anymore (even reading from it seems to have stoped working). I believe that this gitlab remote has some objects that aren't available anymore and git annex seems to try to repair stuff that is not repairable. As always: This is all said with a big \"maybe\", since I wouldn't know how to test my repo against that hypothesis. I have now done `git annex untrust gitlab` and will try if that helps. All this seem very mysterious to me, since I can't believe that I failed remote should prevent the repo to find an acceptable state.
+
+- git annex fsck states that there are 60 failed files (\"Only 1 of 2 trustworthy copies exist of…\" for all of them as far as I can see in the log)
+- git annex version 6.20160418  installed through homebrew
+- git annex repo version is 6
+"""]]
diff --git a/doc/forum/__34__git_annex_copy_--to___60__REMOTE__62___.__34___doesn__39__t_send_it_to_working_directory.mdwn b/doc/forum/__34__git_annex_copy_--to___60__REMOTE__62___.__34___doesn__39__t_send_it_to_working_directory.mdwn
new file mode 100644
index 0000000000..ea0ad80ba8
--- /dev/null
+++ b/doc/forum/__34__git_annex_copy_--to___60__REMOTE__62___.__34___doesn__39__t_send_it_to_working_directory.mdwn
@@ -0,0 +1,11 @@
+I tried to "git annex copy --to SSH-REMOTE" it says it sends it:
+copy Test (checking SSH-REMOTE...) (to SSH-REMOTE...) 
+SHA256E-s6--8283daccd02d2b2797da53446d367e39dc0fb1615bcae4274460f44455bd9822
+           6 100%    0.00kB/s    0:00:00 (xfer#1, to-check=0/1)
+
+sent 146 bytes  received 31 bytes  354.00 bytes/sec
+total size is 6  speedup is 0.03
+ok
+(Recording state in git...)
+
+But I can't find it in the working directory. Can anyone help me?
diff --git a/doc/forum/__34__git_annex_copy_--to___60__REMOTE__62___.__34___doesn__39__t_send_it_to_working_directory/comment_1_0c0a5999a92bf5880f2113177dc67cc2._comment b/doc/forum/__34__git_annex_copy_--to___60__REMOTE__62___.__34___doesn__39__t_send_it_to_working_directory/comment_1_0c0a5999a92bf5880f2113177dc67cc2._comment
new file mode 100644
index 0000000000..18c41f1c97
--- /dev/null
+++ b/doc/forum/__34__git_annex_copy_--to___60__REMOTE__62___.__34___doesn__39__t_send_it_to_working_directory/comment_1_0c0a5999a92bf5880f2113177dc67cc2._comment
@@ -0,0 +1,11 @@
+[[!comment format=mdwn
+ username="http://www.rfc1149.net/"
+ nickname="Sam"
+ subject="comment 1"
+ date="2013-08-05T09:49:56Z"
+ content="""
+`git annex copy` will only push objects (content) into git annex private directory. You have to issue a `git annex merge` (or `git annex sync`) on the receiving end, or run `git annex assistant` there to update the working directory.
+
+Note that you can run `git annex merge` as a post-update hook if you want this to be done automatically.
+
+"""]]
diff --git a/doc/forum/__34__git_annex_copy_--to___60__REMOTE__62___.__34___doesn__39__t_send_it_to_working_directory/comment_2_c18083d9054f66f0bd51d63452af07eb._comment b/doc/forum/__34__git_annex_copy_--to___60__REMOTE__62___.__34___doesn__39__t_send_it_to_working_directory/comment_2_c18083d9054f66f0bd51d63452af07eb._comment
new file mode 100644
index 0000000000..da0a869298
--- /dev/null
+++ b/doc/forum/__34__git_annex_copy_--to___60__REMOTE__62___.__34___doesn__39__t_send_it_to_working_directory/comment_2_c18083d9054f66f0bd51d63452af07eb._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="https://www.google.com/accounts/o8/id?id=AItOawmqaNwDQ367zpW6cIRviLz6zJZZFODgoEI"
+ nickname="Zack"
+ subject="It worked!"
+ date="2013-08-05T09:57:25Z"
+ content="""
+Thanks for the help, it worked.
+"""]]
diff --git a/doc/forum/__34__git_annex_drop__34___freezes_while___34__locking_origin...__34__.mdwn b/doc/forum/__34__git_annex_drop__34___freezes_while___34__locking_origin...__34__.mdwn
new file mode 100644
index 0000000000..b4e9b087df
--- /dev/null
+++ b/doc/forum/__34__git_annex_drop__34___freezes_while___34__locking_origin...__34__.mdwn
@@ -0,0 +1,12 @@
+Hi,
+
+I am using git annex 5.20151208-1build1 on my local machine and 5.20151208-g5ef6ed8 on "origin". I can not drop files anymore. "git annex sync --content" has the same problem when it tries to drop files:
+
+    $ git annex drop xyz
+    drop xyz (locking origin...)   <--  waits here for ever
+
+    $ git annex sync --content
+    [git annex is pulling and merging stuff]
+    drop xyz (locking origin...)   <--  waits here for ever
+
+What can I do? Thanks!
diff --git a/doc/forum/__34__git_annex_drop__34___freezes_while___34__locking_origin...__34__/comment_1_09fbe4362fea8f2a53a1f955b0519019._comment b/doc/forum/__34__git_annex_drop__34___freezes_while___34__locking_origin...__34__/comment_1_09fbe4362fea8f2a53a1f955b0519019._comment
new file mode 100644
index 0000000000..cfe473562b
--- /dev/null
+++ b/doc/forum/__34__git_annex_drop__34___freezes_while___34__locking_origin...__34__/comment_1_09fbe4362fea8f2a53a1f955b0519019._comment
@@ -0,0 +1,18 @@
+[[!comment format=mdwn
+ username="joey"
+ subject="""comment 1"""
+ date="2016-09-05T19:43:23Z"
+ content="""
+You're using a version of git-annex that is nearly 2 years old.
+Upgrading to a new version should be the very first thing you try in such a situation..
+
+Sounds like probably this changelog entry is relevant, from version 
+6.20160418.
+
+  * Fix hang when dropping content needs to lock the content on a
+    ssh remote, which occurred when the remote has git-annex version
+    5.20151019 or newer. (The bug was in the client side; the remote
+    git-annex-shell does not need to be upgraded.)
+
+And it appears that hang first appeared in git-annex version 5.20151009.
+"""]]
diff --git a/doc/forum/__34__git_annex_get__34___on_windows_fails_with_rsync_error.mdwn b/doc/forum/__34__git_annex_get__34___on_windows_fails_with_rsync_error.mdwn
new file mode 100644
index 0000000000..5fb7492bad
--- /dev/null
+++ b/doc/forum/__34__git_annex_get__34___on_windows_fails_with_rsync_error.mdwn
@@ -0,0 +1,20 @@
+I tried cloning an annex repo between two drives "c:" and "d:". The part with "git clone" itself works, but when I try to execute "git annex get", rsync reports an error about a missing path starting with "/cygdrive".
+
+    Sameer@DESKTOP-6CJGO0T MINGW32 /d/a (annex/direct/master)
+    $ git annex get --not --in here
+    get world.txt (from origin...)
+    rsync: change_dir "/cygdrive/c/scratch/a" failed: No such file or directory (2)
+    rsync error: some files/attrs were not transferred (see previous errors) (code 23) at main.c(1183) [sender=3.1.1]
+    
+      rsync failed -- run git annex again to resume file transfer
+    
+      Unable to access these remotes: origin
+    
+      Try making some of these repositories available:
+            cda3a2c6-7ddc-4164-ad6d-fbb2720b24d7 -- DESKTOP-6CJGO0T:C:\scratch\a [origin]
+    failed
+    git-annex: get: 1 failed
+
+I am running git annex that was installed in the same directory as a 32-bit git version 2.6.4 for windows (mingw32).
+
+The question is, my windows drives are actually visible as "/c" and "/a" ... then why is rsync searching for "/cygdrive/c" etc? This is clearly not a Cygwin installation.
diff --git a/doc/forum/__34__git_annex_get__34___on_windows_fails_with_rsync_error/comment_1_a1a881d7b0f0c32b1bfdd3bc5ee4dd18._comment b/doc/forum/__34__git_annex_get__34___on_windows_fails_with_rsync_error/comment_1_a1a881d7b0f0c32b1bfdd3bc5ee4dd18._comment
new file mode 100644
index 0000000000..95bb2d6020
--- /dev/null
+++ b/doc/forum/__34__git_annex_get__34___on_windows_fails_with_rsync_error/comment_1_a1a881d7b0f0c32b1bfdd3bc5ee4dd18._comment
@@ -0,0 +1,7 @@
+[[!comment format=mdwn
+ username="sameerds"
+ subject="bump!"
+ date="2016-01-08T07:31:18Z"
+ content="""
+Am I the only one seeing this? :(
+"""]]
diff --git a/doc/forum/__34__git_annex_get__34___on_windows_fails_with_rsync_error/comment_3_50d0b68fcf0e239037a5d7950595b1d6._comment b/doc/forum/__34__git_annex_get__34___on_windows_fails_with_rsync_error/comment_3_50d0b68fcf0e239037a5d7950595b1d6._comment
new file mode 100644
index 0000000000..f9a003ced7
--- /dev/null
+++ b/doc/forum/__34__git_annex_get__34___on_windows_fails_with_rsync_error/comment_3_50d0b68fcf0e239037a5d7950595b1d6._comment
@@ -0,0 +1,9 @@
+[[!comment format=mdwn
+ username="pkitslaar"
+ subject="Same here"
+ date="2016-01-10T07:50:14Z"
+ content="""
+I have the same issue.
+
+git-annex version: 6.20160108-ge94ef9b
+"""]]
diff --git a/doc/forum/__34__git_annex_get__34___on_windows_fails_with_rsync_error/comment_3_9bc58c6c2333655870c5e42138f9749e._comment b/doc/forum/__34__git_annex_get__34___on_windows_fails_with_rsync_error/comment_3_9bc58c6c2333655870c5e42138f9749e._comment
new file mode 100644
index 0000000000..55d0699fe8
--- /dev/null
+++ b/doc/forum/__34__git_annex_get__34___on_windows_fails_with_rsync_error/comment_3_9bc58c6c2333655870c5e42138f9749e._comment
@@ -0,0 +1,15 @@
+[[!comment format=mdwn
+ username="pkitslaar"
+ subject="Possible fix for rsync windows paths"
+ date="2016-01-11T10:25:22Z"
+ content="""
+I looked at the code that creates the paths for the rsync command on Windows.
+And it is using Cygwin style paths. However, the rsync version that is bundled with
+git-annex on Windows is taken from the MSYS2 distribution. This uses different path
+convention for drive letters (simply `/c` and `/d` for `C:\` and `D:\` etc..)
+
+I created a quick fix by adding a toMSYS2Path function and using that for rsync on Windows.
+I fix can be found here: 
+
+For me this works when I locally build git-annex and I can now \"get\" files from repositories on another drive.
+"""]]
diff --git a/doc/forum/__34__git_annex_get__34___on_windows_fails_with_rsync_error/comment_4_958e887dd83b8b636df2d81ade0dbbf1._comment b/doc/forum/__34__git_annex_get__34___on_windows_fails_with_rsync_error/comment_4_958e887dd83b8b636df2d81ade0dbbf1._comment
new file mode 100644
index 0000000000..a345023215
--- /dev/null
+++ b/doc/forum/__34__git_annex_get__34___on_windows_fails_with_rsync_error/comment_4_958e887dd83b8b636df2d81ade0dbbf1._comment
@@ -0,0 +1,7 @@
+[[!comment format=mdwn
+ username="sameerds"
+ subject="cygwin/msys-independent fix?"
+ date="2016-01-12T04:07:39Z"
+ content="""
+Thank you, pkitslaar, for working this out! I will try it out ASAP and provide feedback. But generally, why can we not create Windows-style paths, and then rely on the underlying platform to do the right thing? I would expect that \"rsync\" or any other binary will accept \"C:\scratch\a\" as a valid path, and then translate it appropriately for its own host such as cygwin or msys. \"Clients\" of these binaries should not have to worry about such things.
+"""]]
diff --git a/doc/forum/__34__git_annex_get__34___on_windows_fails_with_rsync_error/comment_5_24782f9b1012a2935466ac358acea97b._comment b/doc/forum/__34__git_annex_get__34___on_windows_fails_with_rsync_error/comment_5_24782f9b1012a2935466ac358acea97b._comment
new file mode 100644
index 0000000000..d5d6f10993
--- /dev/null
+++ b/doc/forum/__34__git_annex_get__34___on_windows_fails_with_rsync_error/comment_5_24782f9b1012a2935466ac358acea97b._comment
@@ -0,0 +1,11 @@
+[[!comment format=mdwn
+ username="joey"
+ subject="""comment 5"""
+ date="2016-01-13T16:29:42Z"
+ content="""
+rsync thinks that "C:foo" is the file foo on host C. This makes it not
+usable cross-drive on windows without some hack.
+
+So, this hack broke when the bundled rsync build was changed from the cygwin
+build to the msys build. I'll apply pkitslaar's patch.
+"""]]
diff --git a/doc/forum/__34__git_annex_lock__34___very_slow_for_big_repo.mdwn b/doc/forum/__34__git_annex_lock__34___very_slow_for_big_repo.mdwn
new file mode 100644
index 0000000000..9bacf28dc0
--- /dev/null
+++ b/doc/forum/__34__git_annex_lock__34___very_slow_for_big_repo.mdwn
@@ -0,0 +1,7 @@
+I found the command "git annex lock" very slow (much slower than the initial "git annex add" with SHA1), for a not so big directory, when run in a big repo.
+It seems that each underlying git command is not fast, so I thought it would be better to run them once with all files as arguments.
+I had to stop the lock command, and ran "git checkout ." (I did not change any file), is this a correct alternative?
+
+Thanks a LOT for this software, one that I missed since a long time (but wasn't able to write)!
+
+Rafaël
diff --git a/doc/forum/__34__git_annex_lock__34___very_slow_for_big_repo/comment_1_044f1c5e5f7a939315c28087495a8ba8._comment b/doc/forum/__34__git_annex_lock__34___very_slow_for_big_repo/comment_1_044f1c5e5f7a939315c28087495a8ba8._comment
new file mode 100644
index 0000000000..0e2773bda3
--- /dev/null
+++ b/doc/forum/__34__git_annex_lock__34___very_slow_for_big_repo/comment_1_044f1c5e5f7a939315c28087495a8ba8._comment
@@ -0,0 +1,16 @@
+[[!comment format=mdwn
+ username="http://joey.kitenet.net/"
+ nickname="joey"
+ subject="fixed"
+ date="2011-05-31T18:51:13Z"
+ content="""
+Running `git checkout` by hand is fine, of course.
+
+Underlying problem is that git has some O(N) scalability of operations on the index with regards to the number of files in the repo. So a repo with a whole lot of files will have a big index, and any operation that changes the index, like the `git reset` this needs to do, has to read in the entire index, and write out a new, modified version. It seems that git could be much smarter about its index data structures here, but I confess I don't understand the index's data structures at all. I hope someone takes it on, as git's scalability to number of files in the repo is becoming a new pain point, now that scalability to large files is \"solved\". ;)
+
+Still, it is possible to speed this up at git-annex's level. Rather than doing a `git reset` followed by a git checkout, it can just `git checkout HEAD -- file`, and since that's one command, it can then be fed into the queueing machinery in git-annex (that exists mostly to work around this git malfescence), and so only a single git command will need to be run to lock multiple files.
+
+I've just implemented the above. In my music repo, this changed an lock of a CD's worth of files from taking ctrl-c long to 1.75 seconds. Enjoy!
+
+(Hey, this even speeds up the one file case greatly, since `git reset -- file` is slooooow -- it seems to scan the *entire* repository tree. Yipes.)
+"""]]
diff --git a/doc/forum/__34__git_annex_lock__34___very_slow_for_big_repo/comment_2_e854b93415d5ab80eda8e3be3b145ec2._comment b/doc/forum/__34__git_annex_lock__34___very_slow_for_big_repo/comment_2_e854b93415d5ab80eda8e3be3b145ec2._comment
new file mode 100644
index 0000000000..9e9e778ce9
--- /dev/null
+++ b/doc/forum/__34__git_annex_lock__34___very_slow_for_big_repo/comment_2_e854b93415d5ab80eda8e3be3b145ec2._comment
@@ -0,0 +1,13 @@
+[[!comment format=mdwn
+ username="https://www.google.com/accounts/o8/id?id=AItOawnpdM9F8VbtQ_H5PaPMpGSxPe_d5L1eJ6w"
+ nickname="Rafaël"
+ subject="comment 2"
+ date="2011-05-31T21:43:22Z"
+ content="""
+Nice!
+So if I understand correctly, 'git reset -- file' was there to discard staged (but not commited) changes made to 'file', before checking out, so that it is equivalent to directly 'git checkout HEAD -- file' ?
+I'm curious about the \"queueing machinery in git-annex\": does it end up calling the one git command with multiple files as arguments? does it correspond to the message \"(Recording state in git...)\" ?
+Thanks!
+
+
+"""]]
diff --git a/doc/forum/__34__git_annex_lock__34___very_slow_for_big_repo/comment_3_95c110500bc54013bc1969c1a9c8f842._comment b/doc/forum/__34__git_annex_lock__34___very_slow_for_big_repo/comment_3_95c110500bc54013bc1969c1a9c8f842._comment
new file mode 100644
index 0000000000..87da0c396d
--- /dev/null
+++ b/doc/forum/__34__git_annex_lock__34___very_slow_for_big_repo/comment_3_95c110500bc54013bc1969c1a9c8f842._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="http://joey.kitenet.net/"
+ nickname="joey"
+ subject="comment 3"
+ date="2011-05-31T21:54:23Z"
+ content="""
+@Rafaël , you're correct on all counts.
+"""]]
diff --git a/doc/forum/__34__git_annex_lock__34___very_slow_for_big_repo/comment_4_a4fd212cb066cd53d0d66eb09f3b39a8._comment b/doc/forum/__34__git_annex_lock__34___very_slow_for_big_repo/comment_4_a4fd212cb066cd53d0d66eb09f3b39a8._comment
new file mode 100644
index 0000000000..ecb5ba913b
--- /dev/null
+++ b/doc/forum/__34__git_annex_lock__34___very_slow_for_big_repo/comment_4_a4fd212cb066cd53d0d66eb09f3b39a8._comment
@@ -0,0 +1,18 @@
+[[!comment format=mdwn
+ username="josch"
+ ip="2001:638:709:5:2ad2:44ff:fe4b:56aa"
+ subject="thanks a lot!"
+ date="2014-06-23T15:13:34Z"
+ content="""
+Thank you for the answer to this problem!
+
+I put 150k emails in maildir format in git annex. Adding them took a couple of hours and so did the first full sync. But after that things didnt noticibly slow down - that was a surprise!
+
+What but then I wanted to do a one-time sync using rsync. I didnt find out how to tell rsync to follow symlinks at the destination instead of replacing symlinks with normal files. So I first unlocked the 150k emails (which was moderately quick and probably only I/O bound) and then did the rsync using the --checksum option which worked well as well. The problems started when I wanted to lock the whole thing again. This took ages and even after two days it did not output that it was adding files at all. So I used `find -type f | split -a3 -l100` and then `for f in x*; do echo $f; git annex add `echo \`cat $f\``; rm $f; done`. This started off well and I could finally see files being added. Unfortunately when I came back after a couple of hours, the progress slowed down to a crawl of only one file every few seconds.
+
+The solution was to just `git checkout -- Mail` everything. This finished in a matter of seconds and left the new mail which was copied over by rsync intact.
+
+Thanks a lot for the tip!
+
+Let me hereby also report that git annex seems to work without problems when using it with offlineimap and notmuch. At least I did not run into any problems with that setup yet.
+"""]]
diff --git a/doc/forum/__34__git_annex_sync_--content__34___with_special_remote_of_type___34__directory__34__.mdwn b/doc/forum/__34__git_annex_sync_--content__34___with_special_remote_of_type___34__directory__34__.mdwn
new file mode 100644
index 0000000000..bab3167e5e
--- /dev/null
+++ b/doc/forum/__34__git_annex_sync_--content__34___with_special_remote_of_type___34__directory__34__.mdwn
@@ -0,0 +1,17 @@
+Hello, I started using the "wanted" feature of git-annex.
+
+I have (besides others) one local repository ("neon"), and two special remotes "ldk" (rsync) and "storage" (directory).
+
+"wanted" and "group" are configured as (replaced UUIDs with names):
+
+    group "storage" = backup
+    wanted "storage" = standard
+    wanted "neon" = (exclude=pictures/* and exclude=video/*) or present
+
+Now, let's assume there is a file named "video/foo.mp4". It is only present in "ldk". I want it to be present in "storage", too.
+
+When I run "git annex sync --content" on "neon" the file "video/foo.mp4" is neither fetched to be placed in "neon" nor in "storage".
+
+Which command do I have to run to transfer the file "video/foo.mp4" from "ldk" to "storage" when run from "neon".
+
+Previously, I started with "git annex get \`git annex find --not --in storage\`" and then continued with "git annex copy \`git annex find --not --in storage\` --to storage". I was hoping that the wanted feature would simplify this.
diff --git a/doc/forum/__34__git_annex_sync_--content__34___with_special_remote_of_type___34__directory__34__/comment_1_c51363e109bcc5cd1df40c5d0ec993b3._comment b/doc/forum/__34__git_annex_sync_--content__34___with_special_remote_of_type___34__directory__34__/comment_1_c51363e109bcc5cd1df40c5d0ec993b3._comment
new file mode 100644
index 0000000000..b7d538ae9b
--- /dev/null
+++ b/doc/forum/__34__git_annex_sync_--content__34___with_special_remote_of_type___34__directory__34__/comment_1_c51363e109bcc5cd1df40c5d0ec993b3._comment
@@ -0,0 +1,38 @@
+[[!comment format=mdwn
+ username="joey"
+ subject="""comment 1"""
+ date="2015-04-21T20:01:26Z"
+ content="""
+What's going on here is that git-annex does not know that you want to
+use "neon" as a repository that transfers content between "ldk" and
+"storage". The configuration for "neon" only makes it want a specific
+set of files, so it doesn't get other files from "ldk", and so does
+not have them to send to "storage".
+
+The solution is to change "neon"'s preferred content settings so it wants
+files that are not yet present in "storage".
+
+The [[transfer repository expression|preferred_content/standard_groups]]
+is one way to configure preferred content settings so that a repo
+will want to download files that other repos want to have. 
+
+	not (inallgroup=client and copies=client:2) and ($client)
+
+But that expression only works for transfer repsitories in between client
+repositories. Your "storage" repo is set to be a backup repository.
+
+So, we need something a little bit different.
+
+	not inallgroup=backup and include=*
+
+I think that would work, but I don't see a way to reconcile it with
+the configuration you already have for "neon". If neon wants to
+"exclude=pictures/* and exclude=video/*" then it will never get those
+and so can never send them on to "storage". And if neon wants all
+"present" files, then anything it does get for whatever reason
+will stay in it, which is just not how a transfer repo needs to work.
+
+What might work better is to set up a separate repository that can talk
+to "ldk" and "storage" (as well as perhaps pulling files from "neon"
+when available), and make it have that preferred content expression.
+"""]]
diff --git a/doc/forum/__34__git_annex_sync_--content__34___with_special_remote_of_type___34__directory__34__/comment_2_be52a6d21df4732c9f83463bb5e6f612._comment b/doc/forum/__34__git_annex_sync_--content__34___with_special_remote_of_type___34__directory__34__/comment_2_be52a6d21df4732c9f83463bb5e6f612._comment
new file mode 100644
index 0000000000..c2566bf60f
--- /dev/null
+++ b/doc/forum/__34__git_annex_sync_--content__34___with_special_remote_of_type___34__directory__34__/comment_2_be52a6d21df4732c9f83463bb5e6f612._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="https://me.yahoo.com/a/wS4b1K9kooKjDS5nqjBuZsN4Czwko0ECcg--#2eff3"
+ nickname="Markus"
+ subject="comment 2"
+ date="2015-04-22T10:47:26Z"
+ content="""
+Thank you. I settled on using `present or (not inallgroup=backup) or (exclude=pictures/* and exclude=video/*)`, which seems to be doing what I want. The most important thing to me is that files are transferred to the backup group. Some files might be left in pictures/ or video/ on \"neon\", but I can drop those from time to time if disk space is low.
+"""]]
diff --git a/doc/forum/__34__git_annex_sync__34___synced_after_8_hours.mdwn b/doc/forum/__34__git_annex_sync__34___synced_after_8_hours.mdwn
new file mode 100644
index 0000000000..2804828f59
--- /dev/null
+++ b/doc/forum/__34__git_annex_sync__34___synced_after_8_hours.mdwn
@@ -0,0 +1,40 @@
+Hi,
+
+The git annex seem has problem with many files.
+
+For synchronize, the operation lasts 8 hours. Here the sample for synchronizing to my local remote server (sbackup)
+
+start at **20:12** / end at **04:13** / total time = ~ **8 hours**
+
+    git annex sync sbackup
+    
+    [2015-04-13 20:12:26 CEST] call: git ["--git-dir=.git","--work-tree=.","push","sbackup","+git-annex:synced/git-annex","master:synced/master"]
+    Counting objects: 792155, done.
+    Delta compression using up to 4 threads.
+    Compressing objects: 100% (789727/789727), done.
+    Writing objects: 100% (792155/792155), 75.73 MiB | 2.35 MiB/s, done.
+    Total 792155 (delta 449604), reused 1 (delta 0)
+    To partage@192.168.253.53:/data/samba/git-annex/docshare
+      ae182f0..fad3aca  git-annex -> synced/git-annex
+      e0e67fe..5226a6f  master -> synced/master
+    [2015-04-14 04:13:05 CEST] read: git ["--git-dir=.git","--work-tree=.","push","sbackup","git-annex","master"]
+    ok
+
+Another problem, I do not know exactly how many files I own (i use **find . | wc -l** )
+
+.git = 1250633
+
+documents = 61124
+
+medias = 199504
+
+it seem i own ~250000 files, but in the .git **1.2 millions files**.
+
+The following command also very slow
+
+    git annex info
+
+
+What the best pratices for use git annex with many files > 500 000 or maintenance & reduction/cleaning method
+
+Thanks
diff --git a/doc/forum/__34__git_annex_sync__34___synced_after_8_hours/comment_1_e815ac48a17cc4296473d61e712d95e0._comment b/doc/forum/__34__git_annex_sync__34___synced_after_8_hours/comment_1_e815ac48a17cc4296473d61e712d95e0._comment
new file mode 100644
index 0000000000..4cc665e044
--- /dev/null
+++ b/doc/forum/__34__git_annex_sync__34___synced_after_8_hours/comment_1_e815ac48a17cc4296473d61e712d95e0._comment
@@ -0,0 +1,13 @@
+[[!comment format=mdwn
+ username="CandyAngel"
+ subject="comment 1"
+ date="2015-04-14T08:40:33Z"
+ content="""
+If you want a file count:
+
+    git annex find | wc -l
+
+is probably the best measure.
+
+I have an annex with about several million files in it and it is slow, but not as slow as you are describing. Have you done a repack/gc cycle?
+"""]]
diff --git a/doc/forum/__34__git_annex_sync__34___synced_after_8_hours/comment_2_d13a0af48b8831c81276a0b2c8e25303._comment b/doc/forum/__34__git_annex_sync__34___synced_after_8_hours/comment_2_d13a0af48b8831c81276a0b2c8e25303._comment
new file mode 100644
index 0000000000..940922e6c2
--- /dev/null
+++ b/doc/forum/__34__git_annex_sync__34___synced_after_8_hours/comment_2_d13a0af48b8831c81276a0b2c8e25303._comment
@@ -0,0 +1,37 @@
+[[!comment format=mdwn
+ username="https://www.google.com/accounts/o8/id?id=AItOawlXt6nnNs-3uw61EGYtxr_AVhJqXybwLR8"
+ nickname="Bruno"
+ subject="comment 2"
+ date="2015-04-15T17:51:18Z"
+ content="""
+@CandyAngel Thank you for your **git annex find** tips. But for git gc, it seem not working fine :)
+After i have executed the **git gc**, the **git annex info** return the result after **1h 45m**
+
+    % time git annex info
+    repository mode: indirect
+    trusted repositories: 0
+    semitrusted repositories: 5
+	00000000-0000-0000-0000-000000000001 -- web
+ 	00000000-0000-0000-0000-000000000002 -- bittorrent
+ 	181d4dae-2131-435e-9c00-b8c7f1bfc332 -- [sbackup]
+ 	2db1f8e7-0b29-4d61-8875-a4a4a42a79dd -- [dellcomputer]
+ 	703df355-73a6-4487-97fd-a3a5d6ae034e -- usbhomebackup [here]
+    untrusted repositories: 0
+    transfers in progress: none
+    available local disk space: 135.24 gigabytes (+1 megabyte reserved)
+    local annex keys: 275416
+    local annex size: 780.55 gigabytes
+    annexed files in working tree: 265888
+    size of annexed files in working tree: 751.49 gigabytes
+    bloom filter size: 16 mebibytes (55.1% full)
+    backend usage: 
+	SHA256E: 541304
+
+git annex info  83,95s user 50,68s system 2% **cpu 1:45:01,70 total**
+
+
+Can you explain exactly the git gc or git repack parameters that you use for optimizing git annex performance  ?
+
+Thanks
+
+"""]]
diff --git a/doc/forum/__34__git_annex_sync__34___synced_after_8_hours/comment_3_c50b62e5a84b861117a4405c2a2f5cfb._comment b/doc/forum/__34__git_annex_sync__34___synced_after_8_hours/comment_3_c50b62e5a84b861117a4405c2a2f5cfb._comment
new file mode 100644
index 0000000000..20031b9529
--- /dev/null
+++ b/doc/forum/__34__git_annex_sync__34___synced_after_8_hours/comment_3_c50b62e5a84b861117a4405c2a2f5cfb._comment
@@ -0,0 +1,16 @@
+[[!comment format=mdwn
+ username="CandyAngel"
+ subject="comment 3"
+ date="2015-04-16T07:41:07Z"
+ content="""
+*git annex info* has check every file (not sure if it traverses *.git/annex/objects* specifically or not) to get \"local annex\" information. You can improve its performance by improving directory traversal in general (different filesystem or [changing the hashing method so it isn't Xx/Yy/KEY/FILE](https://github.com/datalad/datalad/issues/32)).
+
+The repack/gc speeds up operations for the git side of things, like syncing (pull/push), cloning and committing.
+
+Here's what I used:
+
+    git repack -ad
+    git gc
+
+This took git actions down from 1 hour+ to ~10 minutes (for a repo with 5.6 million objects).
+"""]]
diff --git a/doc/forum/__34__git_annex_sync__34___synced_after_8_hours/comment_4_0051e83196945b97e2f3ed14a58daaea._comment b/doc/forum/__34__git_annex_sync__34___synced_after_8_hours/comment_4_0051e83196945b97e2f3ed14a58daaea._comment
new file mode 100644
index 0000000000..2fc762f77b
--- /dev/null
+++ b/doc/forum/__34__git_annex_sync__34___synced_after_8_hours/comment_4_0051e83196945b97e2f3ed14a58daaea._comment
@@ -0,0 +1,14 @@
+[[!comment format=mdwn
+ username="https://www.google.com/accounts/o8/id?id=AItOawlXt6nnNs-3uw61EGYtxr_AVhJqXybwLR8"
+ nickname="Bruno"
+ subject="comment 4"
+ date="2015-04-16T11:47:50Z"
+ content="""
+Thanks @CandyAngle, 
+
+Effectively, your tips for reduce a time for some git-annex commands if works fine, i will see in the long term if that is work perfectly
+
+ex:, now **git annex sync** it work in **45s** ! :)
+
+Thanks
+"""]]
diff --git a/doc/forum/__34__git_annex_sync__34___synced_after_8_hours/comment_5_26de32ea240621e23717c55866ad9764._comment b/doc/forum/__34__git_annex_sync__34___synced_after_8_hours/comment_5_26de32ea240621e23717c55866ad9764._comment
new file mode 100644
index 0000000000..907f32b940
--- /dev/null
+++ b/doc/forum/__34__git_annex_sync__34___synced_after_8_hours/comment_5_26de32ea240621e23717c55866ad9764._comment
@@ -0,0 +1,9 @@
+[[!comment format=mdwn
+ username="CandyAngel"
+ subject="comment 5"
+ date="2015-04-16T21:08:32Z"
+ content="""
+My pleasure, glad it is working for you!
+
+Going forward, you should run *git repack* (without -ad) every now and again to pack new objects into pack files. You can use *git count-objects -v* to find out how many unpacked objects you have.
+"""]]
diff --git a/doc/forum/__34__git_annex_sync__34___synced_after_8_hours/comment_6_0714eca58513f1059fc246ee152078e7._comment b/doc/forum/__34__git_annex_sync__34___synced_after_8_hours/comment_6_0714eca58513f1059fc246ee152078e7._comment
new file mode 100644
index 0000000000..fbf7265ffb
--- /dev/null
+++ b/doc/forum/__34__git_annex_sync__34___synced_after_8_hours/comment_6_0714eca58513f1059fc246ee152078e7._comment
@@ -0,0 +1,16 @@
+[[!comment format=mdwn
+ username="candyangel@7b9260fff42e1f6f552e1348007f7409a9101e82"
+ nickname="candyangel"
+ subject="comment 6"
+ date="2015-05-20T14:14:32Z"
+ content="""
+I've just come across something else which has sped up git operations on the index a *lot*.
+
+My $annex/.git/index was over 500M which made operations on the index really slow (git add/rm etc.) as it gets rewritten out in its entirety, every time!
+
+    git update-index --index-version 4
+
+brought it down to 200M, making everything much, much faster!
+
+[There is a warning in the git documentation](http://git-scm.com/docs/git-update-index) that alternative git implementations might not understand it, so bear that in mind if you want to use it.
+"""]]
diff --git a/doc/forum/__34__git_annex_unused__34___doesn__39__t_find_unused_files_after_git_history_was_rewritten.mdwn b/doc/forum/__34__git_annex_unused__34___doesn__39__t_find_unused_files_after_git_history_was_rewritten.mdwn
new file mode 100644
index 0000000000..cbf9770c33
--- /dev/null
+++ b/doc/forum/__34__git_annex_unused__34___doesn__39__t_find_unused_files_after_git_history_was_rewritten.mdwn
@@ -0,0 +1,17 @@
+Hi,
+
+I recently cleaned my git history using:
+
+    git filter-branch --tree-filter 'rm -rf personal/Mail' --prune-empty HEAD
+    git for-each-ref --format="%(refname)" refs/original/ | xargs -n 1 git update-ref -d
+    git gc
+
+I then ran:
+
+    git annex unused
+
+But was surprised to find that it didn't report any unused files. Not surprisingly (given that I didn't touch the annex data at all so far), when looking into `.git/annex/objects/` I still find the content of all the files that are now not used anymore.
+
+Is there a way to clean these up besides creating a fresh clone of the repository?
+
+Thanks!
diff --git a/doc/forum/__34__git_annex_unused__34___doesn__39__t_find_unused_files_after_git_history_was_rewritten/comment_1_8f2ced0b9039968f485943e1525dcf6e._comment b/doc/forum/__34__git_annex_unused__34___doesn__39__t_find_unused_files_after_git_history_was_rewritten/comment_1_8f2ced0b9039968f485943e1525dcf6e._comment
new file mode 100644
index 0000000000..b07cab8e2f
--- /dev/null
+++ b/doc/forum/__34__git_annex_unused__34___doesn__39__t_find_unused_files_after_git_history_was_rewritten/comment_1_8f2ced0b9039968f485943e1525dcf6e._comment
@@ -0,0 +1,10 @@
+[[!comment format=mdwn
+ username="joey"
+ subject="""comment 1"""
+ date="2016-09-05T20:16:04Z"
+ content="""
+`git-annex unused` looks at all local branches, and all remote branches.
+If you only filter-branched a single branch, you could well have other
+branches that still refer to those files, so they would not be considered
+unused.
+"""]]
diff --git a/doc/forum/__34__next_time_Im_online_I_would_like_to_have_file_x_y_z__34__.mdwn b/doc/forum/__34__next_time_Im_online_I_would_like_to_have_file_x_y_z__34__.mdwn
new file mode 100644
index 0000000000..a5492dc861
--- /dev/null
+++ b/doc/forum/__34__next_time_Im_online_I_would_like_to_have_file_x_y_z__34__.mdwn
@@ -0,0 +1,5 @@
+git-annex is seriously cool, however I havent figured out an important detail:
+in the short presentation Richard Hartmann mentions the Nomad use case and says "... next time Im online I would like to have file x y z ... "
+How can this be achieved exactly? If I do a git annex copy/get then it will want to do it instantly, not queuing it.
+thanks a lot!
+Aron
diff --git a/doc/forum/__34__next_time_Im_online_I_would_like_to_have_file_x_y_z__34__/comment_1_bfeb1446dee4d2f52ef25fabfb8cc8f6._comment b/doc/forum/__34__next_time_Im_online_I_would_like_to_have_file_x_y_z__34__/comment_1_bfeb1446dee4d2f52ef25fabfb8cc8f6._comment
new file mode 100644
index 0000000000..0789316dc7
--- /dev/null
+++ b/doc/forum/__34__next_time_Im_online_I_would_like_to_have_file_x_y_z__34__/comment_1_bfeb1446dee4d2f52ef25fabfb8cc8f6._comment
@@ -0,0 +1,11 @@
+[[!comment format=mdwn
+ username="https://www.google.com/accounts/o8/id?id=AItOawmBUR4O9mofxVbpb8JV9mEbVfIYv670uJo"
+ nickname="Justin"
+ subject="comment 1"
+ date="2012-07-19T14:41:39Z"
+ content="""
+What I do for this is maintain a `todo` directory and `cp -a` + `git annex add` files I want to download into this directory.
+the `cp -a` maintains the symlinks and `git annex add` fixes them if needed.
+
+Also this works the same way if the situation is reversed and the machine you want to download the files onto is not online.  You can add files to the `todo` directory on the server, then once the client machine is online do a `git annex sync` + `git annex get todo`.
+"""]]
diff --git a/doc/forum/__34__next_time_Im_online_I_would_like_to_have_file_x_y_z__34__/comment_2_e60f2bbc1c058993472fd920edbc75fc._comment b/doc/forum/__34__next_time_Im_online_I_would_like_to_have_file_x_y_z__34__/comment_2_e60f2bbc1c058993472fd920edbc75fc._comment
new file mode 100644
index 0000000000..b0a8066ee8
--- /dev/null
+++ b/doc/forum/__34__next_time_Im_online_I_would_like_to_have_file_x_y_z__34__/comment_2_e60f2bbc1c058993472fd920edbc75fc._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="https://www.google.com/accounts/o8/id?id=AItOawnbBRfl5F8gKRr1ko8Ai6FbEZStXXNF1S4"
+ nickname="Áron"
+ subject="comment 2"
+ date="2012-07-19T22:41:53Z"
+ content="""
+hmm not bad workaround but still not very comfortable, thanks though.
+"""]]
diff --git a/doc/forum/__34__permission_denied__34___in_fsck_on_shared_repo.mdwn b/doc/forum/__34__permission_denied__34___in_fsck_on_shared_repo.mdwn
new file mode 100644
index 0000000000..f13aed2c2f
--- /dev/null
+++ b/doc/forum/__34__permission_denied__34___in_fsck_on_shared_repo.mdwn
@@ -0,0 +1,17 @@
+i'm getting errors in ``git annex fsck`` on a shared bare git repo with git-annex 3.20120418 local repo version 3:
+
+``git-annex: ${PATH}/${MYREPO}.git/annex/objects/${HA}/${SH}/SHA1-${HASH}/SHA1-${HASH}: setFileMode: permission denied (Operation not permitted)``
+
+the repository is shared among several users in a common group, and the repo is set up with sticky group, and with appropriate umasks, everything should work.
+
+however, even with the file having permissions -rw-rw-r-- in the directory with permissions drwxrwsr-x, owned by someone else but by a group i'm currently in (as verified by issuing `groups`), i get said error message.
+
+a strace reveals that the failing syscall is:
+
+``[pid 17626] chmod("${FILENAME}", 0100555) = -1 EPERM (Operation not permitted)``
+
+(maybe related: git annex looks for the file in another ${HA}/${SH} combination (of three digits instead of two digits each) before, i take it this is just a new feature not used by the data in my repo? also, i should add that the repository dates back to git-annex 0.13.)
+
+as a workaround, i'm currently ``sudo chown``ing all files to me before the check.
+
+why does fsck try to set permissions even if they are ok? is this a bug in my setup, and if yes, how is a shared repository set up correctly?
diff --git a/doc/forum/__34__permission_denied__34___in_fsck_on_shared_repo/comment_1_3a5202ef2116ebb5559b6f4d920755fc._comment b/doc/forum/__34__permission_denied__34___in_fsck_on_shared_repo/comment_1_3a5202ef2116ebb5559b6f4d920755fc._comment
new file mode 100644
index 0000000000..5a5cafa721
--- /dev/null
+++ b/doc/forum/__34__permission_denied__34___in_fsck_on_shared_repo/comment_1_3a5202ef2116ebb5559b6f4d920755fc._comment
@@ -0,0 +1,10 @@
+[[!comment format=mdwn
+ username="http://joey.kitenet.net/"
+ nickname="joey"
+ subject="comment 1"
+ date="2012-04-21T16:09:19Z"
+ content="""
+Well, the modes you show are wrong. Nothing in the annex should be writable. fsck needs to fix those. (It's true that it also always chmods even correct mode files/directories.. I've made a change avoiding that.)
+
+I have not thought or tried shared git annex repos with multiple unix users writing to them. ([[tips/Using_gitolite_with_git-annex]] would be an alternative.) Seems to me that removing content from the annex would also be a problem, since the directory will need to be chmodded to allow deleting the content from it, and that will fail if it's owned by someone else.  Perhaps git-annex needs to honor core.sharedRepository and avoid these nice safeguards on file modes then.
+"""]]
diff --git a/doc/forum/__34__permission_denied__34___in_fsck_on_shared_repo/comment_2_86663eeb75b0477f53c45f26c8e4b051._comment b/doc/forum/__34__permission_denied__34___in_fsck_on_shared_repo/comment_2_86663eeb75b0477f53c45f26c8e4b051._comment
new file mode 100644
index 0000000000..1c9bfbfe40
--- /dev/null
+++ b/doc/forum/__34__permission_denied__34___in_fsck_on_shared_repo/comment_2_86663eeb75b0477f53c45f26c8e4b051._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="http://joey.kitenet.net/"
+ nickname="joey"
+ subject="comment 2"
+ date="2012-04-21T23:46:42Z"
+ content="""
+All right, I've made all the changes so it supports `core.sharedRepository`.
+"""]]
diff --git a/doc/forum/__34__permission_denied__34___in_fsck_on_shared_repo/comment_3_c336b2b07cd006d378e5be9639ff17ec._comment b/doc/forum/__34__permission_denied__34___in_fsck_on_shared_repo/comment_3_c336b2b07cd006d378e5be9639ff17ec._comment
new file mode 100644
index 0000000000..fd75f2f856
--- /dev/null
+++ b/doc/forum/__34__permission_denied__34___in_fsck_on_shared_repo/comment_3_c336b2b07cd006d378e5be9639ff17ec._comment
@@ -0,0 +1,10 @@
+[[!comment format=mdwn
+ username="http://christian.amsuess.com/chrysn"
+ nickname="chrysn"
+ subject="comment 3"
+ date="2012-04-23T14:14:28Z"
+ content="""
+thanks, that's great. will there be a way to have sharedRepository work for shared remotes (rsync, directory) too, or is that better taken care of by acls?
+
+@not thought of shared repos: we're having our family photo archive spread over our laptops, and backed up on our home storage server and on an rsync+encryption off-site server, with everyone naturally having their own accounts on all systems -- just if you need a use case.
+"""]]
diff --git a/doc/forum/__34__permission_denied__34___in_fsck_on_shared_repo/comment_4_1339cd27ca2955f30b01ecf4da7d6fe8._comment b/doc/forum/__34__permission_denied__34___in_fsck_on_shared_repo/comment_4_1339cd27ca2955f30b01ecf4da7d6fe8._comment
new file mode 100644
index 0000000000..568f11330c
--- /dev/null
+++ b/doc/forum/__34__permission_denied__34___in_fsck_on_shared_repo/comment_4_1339cd27ca2955f30b01ecf4da7d6fe8._comment
@@ -0,0 +1,10 @@
+[[!comment format=mdwn
+ username="http://joey.kitenet.net/"
+ nickname="joey"
+ subject="comment 4"
+ date="2012-04-23T14:35:39Z"
+ content="""
+I'm not currently planning to support sharedRepository perms on special remotes. I suppose I could be convinced otherwise, it's perhaps doable for the ones you mention (rsync might be tricky). (bup special remote already supports it of course.)
+
+thanks for the use case!
+"""]]
diff --git a/doc/forum/__34__unable_to_resolve_reference_refs__47__heads__47__git-annex__34__.mdwn b/doc/forum/__34__unable_to_resolve_reference_refs__47__heads__47__git-annex__34__.mdwn
new file mode 100644
index 0000000000..19aacf0215
--- /dev/null
+++ b/doc/forum/__34__unable_to_resolve_reference_refs__47__heads__47__git-annex__34__.mdwn
@@ -0,0 +1,34 @@
+After a `git annex copy --auto -t m3` I got this error:
+
+    (Recording state in git...)
+    error: unable to resolve reference refs/heads/git-annex: No such file or directory
+    fatal: Cannot lock the ref 'refs/heads/git-annex'.
+    git-annex: git [Param "update-ref",Param "refs/heads/git-annex",Param "d768b20f76ce40157214a713fb0ccb9cfc9134c2"] failed
+
+I did a `git annex sync m3` and got
+
+    [...]
+    push m3
+    Counting objects: 2976, done.
+    Delta compression using up to 2 threads.
+    Compressing objects: 100% (2052/2052), done.
+    Writing objects: 100% (2379/2379), 392.33 KiB, done.
+    Total 2379 (delta 1652), reused 460 (delta 316)
+    Auto packing the repository for optimum performance.
+    error: bad ref for refs/heads/git-annex
+    error: bad ref for refs/heads/git-annex
+    error: bad ref for refs/heads/git-annex
+    To /media/m3/annex
+    87c82c5..06219eb  git-annex -> synced/git-annex
+    be7ff5e..6625634  master -> synced/master
+    ok
+
+And then I ran the command git-annex was trying to run:
+
+    $ git update-ref refs/heads/git-annex d768b20f76ce40157214a713fb0ccb9cfc9134c2
+    error: Trying to write ref refs/heads/git-annex with nonexistent object d768b20f76ce40157214a713fb0ccb9cfc9134c2
+    fatal: Cannot update the ref 'refs/heads/git-annex'.
+
+`git fsck --full` gives no errors.
+
+What does this error mean?  Should I be worried?  Thanks.
diff --git a/doc/forum/__34__unable_to_resolve_reference_refs__47__heads__47__git-annex__34__/comment_1_e50188896df347f1d92e20a52053aa14._comment b/doc/forum/__34__unable_to_resolve_reference_refs__47__heads__47__git-annex__34__/comment_1_e50188896df347f1d92e20a52053aa14._comment
new file mode 100644
index 0000000000..b7ad4a13af
--- /dev/null
+++ b/doc/forum/__34__unable_to_resolve_reference_refs__47__heads__47__git-annex__34__/comment_1_e50188896df347f1d92e20a52053aa14._comment
@@ -0,0 +1,10 @@
+[[!comment format=mdwn
+ username="http://joeyh.name/"
+ nickname="joey"
+ subject="comment 1"
+ date="2013-04-01T18:33:37Z"
+ content="""
+I'm afraid that this looks like a partially corrupted git repository to me. Something seems to have gone wrong with `.git/refs/heads/git-annex`. Also, git object d768b20f76ce40157214a713fb0ccb9cfc9134c2 seems to have been present before, and be gone now. 
+
+I would run `git fsck`, then look at what's happened in `.git/refs/heads`, and if the repository does seem to be damanged, re-clone it. (You can copy over `.git/config` and `.git/annex/` to keep your annexed data in the new clone.) You should probably also run `git annex fsck`, as it could have lost some location log changes.
+"""]]
diff --git a/doc/forum/__34__unable_to_resolve_reference_refs__47__heads__47__git-annex__34__/comment_2_d67793f7c969f64943d1fd54a1208c2b._comment b/doc/forum/__34__unable_to_resolve_reference_refs__47__heads__47__git-annex__34__/comment_2_d67793f7c969f64943d1fd54a1208c2b._comment
new file mode 100644
index 0000000000..1dff891217
--- /dev/null
+++ b/doc/forum/__34__unable_to_resolve_reference_refs__47__heads__47__git-annex__34__/comment_2_d67793f7c969f64943d1fd54a1208c2b._comment
@@ -0,0 +1,15 @@
+[[!comment format=mdwn
+ username="spwhitton"
+ ip="163.1.166.255"
+ subject="comment 2"
+ date="2013-04-06T19:06:24Z"
+ content="""
+Thanks for taking the time to reply.
+
+1) The corruption spread between repositories so I resorted to restoring from a backup from some days ago, and then I restored newly added files by manually copying the symlinks from the old repo into the new one.  Of course I can't update the git-annex branch by hand.  I've run `git annex fsck`: does that re-create the location log information for the symlinks that I re-added, at least for the local repository even if it doesn't know about copies elsewhere?
+
+2) I initially tried to git clone from my restored backup, rather than just moving the restore into place.  But then while this clone showed no errors on a `git fsck --full`, any git-annex command, such as `init` (following [this](http://git-annex.branchable.com/tips/what_to_do_when_a_repository_is_corrupted/)), gives the following sort of error:
+
+    error: invalid object 100644  for ''
+(unfortunately I lost my xterm so don't have one of my actual errors in full).  Can you think of any reason why doing a clone of the backup copy would cause this?  Instead I have just mv'd the backup copy into place and it works fine.
+"""]]
diff --git a/doc/forum/__34__unable_to_resolve_reference_refs__47__heads__47__git-annex__34__/comment_3_3523884833b5fd458a35f898797bf897._comment b/doc/forum/__34__unable_to_resolve_reference_refs__47__heads__47__git-annex__34__/comment_3_3523884833b5fd458a35f898797bf897._comment
new file mode 100644
index 0000000000..21511d2ecd
--- /dev/null
+++ b/doc/forum/__34__unable_to_resolve_reference_refs__47__heads__47__git-annex__34__/comment_3_3523884833b5fd458a35f898797bf897._comment
@@ -0,0 +1,10 @@
+[[!comment format=mdwn
+ username="http://joeyh.name/"
+ nickname="joey"
+ subject="comment 3"
+ date="2013-04-08T17:23:24Z"
+ content="""
+1. I don't understand how corruption could spread between git repositories. git is supposed to use checksums to prevent bad data propigating. I would be running a lot of `git fsck` if I were you. Yes, `git annex fsck` will update location log information for files it finds in the local repository.
+
+2. Again this looks like something is corrupting your repository when git writes it. Perhaps bad memory? I think you need to get to the root of the problem with the corrupting git repositories.
+"""]]
diff --git a/doc/forum/__34__unable_to_resolve_reference_refs__47__heads__47__git-annex__34__/comment_4_02c32c2521ba1a1eaa19eaca7281f2a6._comment b/doc/forum/__34__unable_to_resolve_reference_refs__47__heads__47__git-annex__34__/comment_4_02c32c2521ba1a1eaa19eaca7281f2a6._comment
new file mode 100644
index 0000000000..25d07e8ad3
--- /dev/null
+++ b/doc/forum/__34__unable_to_resolve_reference_refs__47__heads__47__git-annex__34__/comment_4_02c32c2521ba1a1eaa19eaca7281f2a6._comment
@@ -0,0 +1,10 @@
+[[!comment format=mdwn
+ username="spwhitton"
+ ip="163.1.167.50"
+ subject="comment 4"
+ date="2013-04-17T17:02:11Z"
+ content="""
+I didn't think that git would let corruption spread either, but it did: the repository m3 which I showed a push to in my original post showed a broken refs/heads/git-annex too.  You can see it happening when git tries to repack m3 after pushing to it in my original post.
+
+I am pretty sure that my HDD on the machine where this first occurred needs replacing; I've marked the remote as untrusted.
+"""]]
diff --git a/doc/forum/__91__Installation__93___base-3.0.3.2_requires_syb___61____61__0.1.0.2.mdwn b/doc/forum/__91__Installation__93___base-3.0.3.2_requires_syb___61____61__0.1.0.2.mdwn
new file mode 100644
index 0000000000..950b8ee193
--- /dev/null
+++ b/doc/forum/__91__Installation__93___base-3.0.3.2_requires_syb___61____61__0.1.0.2.mdwn
@@ -0,0 +1,16 @@
+Hi, another installation issue on Ubuntu Lucid:
+
+I started with a clean `~/.cabal` directory and did the following:
+
+    andreas@antares:~$ sudo aptitude install cabal-install
+    andreas@antares:~$ cabal update
+    andreas@antares:~$ cabal install git-annex -v --bindir=$HOME/
+
+However, I got some dpendancy error:
+
+    cabal: dependencies conflict: base-3.0.3.2 requires syb ==0.1.0.2 however
+    syb-0.1.0.2 was excluded because json-0.5 requires syb >=0.3.3
+
+Any ideas?
+
+Thanks for your help!
diff --git a/doc/forum/__91__Installation__93___base-3.0.3.2_requires_syb___61____61__0.1.0.2/comment_1_fae6e88115d175239fc55cef4c33fb2c._comment b/doc/forum/__91__Installation__93___base-3.0.3.2_requires_syb___61____61__0.1.0.2/comment_1_fae6e88115d175239fc55cef4c33fb2c._comment
new file mode 100644
index 0000000000..c04cc335b6
--- /dev/null
+++ b/doc/forum/__91__Installation__93___base-3.0.3.2_requires_syb___61____61__0.1.0.2/comment_1_fae6e88115d175239fc55cef4c33fb2c._comment
@@ -0,0 +1,13 @@
+[[!comment format=mdwn
+ username="http://joey.kitenet.net/"
+ nickname="joey"
+ subject="comment 1"
+ date="2012-01-15T19:53:35Z"
+ content="""
+This is now about different build failure than the bug you reported, which was already fixed. Conflating the two is just confusing.
+
+The error message about `syb` is because by using cabal-install on an Ubuntu system from 2010, you're mixing the very old versions of some haskell libraries in Ubuntu with the new versions cabal wants to install. The solution is to stop mixing two package management systems --
+
+* Either install git-annex without using cabal, and use apt-get to install all its dependencies from Ubuntu, assuming your distribution has all the necessary haskell libraries packaged.
+* Or `apt-get remove ghc`, and manually install a current version of [The Haskell Platform](http://hackage.haskell.org/platform/) and use cabal.
+"""]]
diff --git a/doc/forum/__91__Installation__93___base-3.0.3.2_requires_syb___61____61__0.1.0.2/comment_2_4c7a75638e8717132ccde949018d6008._comment b/doc/forum/__91__Installation__93___base-3.0.3.2_requires_syb___61____61__0.1.0.2/comment_2_4c7a75638e8717132ccde949018d6008._comment
new file mode 100644
index 0000000000..3a3106a635
--- /dev/null
+++ b/doc/forum/__91__Installation__93___base-3.0.3.2_requires_syb___61____61__0.1.0.2/comment_2_4c7a75638e8717132ccde949018d6008._comment
@@ -0,0 +1,10 @@
+[[!comment format=mdwn
+ username="https://openid.stackexchange.com/user/fd55c6e3-966a-4626-865f-5d0f73e1eb88"
+ nickname="Andreas H."
+ subject="Thanks and sorry"
+ date="2012-01-15T20:54:55Z"
+ content="""
+Joey, thanks for you quick help! I'll try the manual haskell-platform install once I have quicker internet again, i.e. tomorrow.
+
+And sorry for the mess-up; I splitted the post into two. Hope it's clearer now.
+"""]]
diff --git a/doc/forum/__91__NEED_HELP__93___manual_ssh_remote_setup_with_shared_key.mdwn b/doc/forum/__91__NEED_HELP__93___manual_ssh_remote_setup_with_shared_key.mdwn
new file mode 100644
index 0000000000..f6cdb4c668
--- /dev/null
+++ b/doc/forum/__91__NEED_HELP__93___manual_ssh_remote_setup_with_shared_key.mdwn
@@ -0,0 +1,19 @@
+
+So much time gone by after the kickstarter and now I wanted to give it a kick finally.
+I set up everything on my linux desktop with no problems at all.
+
+I configured a ssh remote with shared encryption key.
+
+Then I tried to use this to share files with a windows Desktop. This is where problems started.
+
+1. Since there is no jabber sync I need to exchange keys manually. But although I read and googled for hours I just found that it would be there in the git repo somewhere in plain text. I didnt find a command to actually display it on the command line.
+2. I need to setup the (existing) ssh remote on windows manually, because I could not make the webapp recognize the ssh-keys from pageant. Also, unfortunately, all examples refer to s3. I didnt even find out which type I had to specify. There seems to be no comprehensive list of repository types for the cmdline and their required arguments respectively at all. The man page says "type=..." for the three-line description of shared encryption setups which appeared like a mockery to me. Since the remote does not have git metadata I also can not use git clone.
+3. The docs say initremote would only be used on fresh remotes, not existing ones. But enableremote only enables remotes that are already configured. So what am I to do here?
+
+Could someone tell me please
+
+1. How to find the shared key
+2. How to add the ssh remote properly
+3. Where 1 and 2 are documented
+
+Thank you!
diff --git a/doc/forum/__91__NEED_HELP__93___manual_ssh_remote_setup_with_shared_key/comment_1_811cab17410ba6e07ae7af3249cd98df._comment b/doc/forum/__91__NEED_HELP__93___manual_ssh_remote_setup_with_shared_key/comment_1_811cab17410ba6e07ae7af3249cd98df._comment
new file mode 100644
index 0000000000..7a6eafa7a7
--- /dev/null
+++ b/doc/forum/__91__NEED_HELP__93___manual_ssh_remote_setup_with_shared_key/comment_1_811cab17410ba6e07ae7af3249cd98df._comment
@@ -0,0 +1,14 @@
+[[!comment format=mdwn
+ username="http://joeyh.name/"
+ ip="71.80.94.56"
+ subject="comment 1"
+ date="2014-02-07T19:28:00Z"
+ content="""
+Sorry for the delay getting to this post.
+
+The lack of jabber support on Windows makes this a bit hard to set up. You instead need to make a git repository some place that both the Windows and Linux machines can both connect to. Once git-annex on both systems is syncing with that common git repository, the Windows system will learn about the encrypted remotes you have set up, and then `git annex enableremote` will be able to use them with no problem.
+
+Of course, if you don't trust your ssh server where you already made an encrypted remote, you may not want to store a un-encrypted git repository on it, and since Windows also doesn't support encrypted git repositories yet, you'd be sort of out of luck. (For now; Windows support is being improved.) 
+
+OTOH, if you just set up that encrypted rsync remote on the ssh server because the ssh server didn't have git-annex installed on it, you can easily also put a git repository on the ssh server, and the combination will be enough to let you sync between the 2 machines.
+"""]]
diff --git a/doc/forum/__91__announce__93___metadata_extration_utility.mdwn b/doc/forum/__91__announce__93___metadata_extration_utility.mdwn
new file mode 100644
index 0000000000..3214f36d9b
--- /dev/null
+++ b/doc/forum/__91__announce__93___metadata_extration_utility.mdwn
@@ -0,0 +1,30 @@
+Let me announce 'metatag', a simple metadata extraction utility.
+
+The Design Idea is to make it completely event driven. There are string matching rules over added metadata,
+who invoke engines when matched, which in turn add more metadata and so on.
+Thus the whole metadata extraction process is controlled by those easily configurable rules. Processing a file or
+directory just starts by adding "/=filename" to the metadata, everything else bootstraps from that. After metadata
+got extracted there are exporters which implement different backends for storing this metadata (currently only a
+'print' and a 'gitannex' exporter are implemented)
+
+While still in a infancy state it already works for me. It now needs more rules and engines for metadata extraction
+and some more efforts to 'standardize' generated metadata. I'd like to welcome comments and contributions.
+
+A README about it can be found at
+
+ 
+
+The code is available under git from
+
+    git clone git://git.pipapo.org/metatag
+
+To make the contribution barrier as low as possible there is a public pushable 'mob' repository where everyone can
+send changes too at `git://git.pipapo.org/mob/metatag`
+
+after installing it, using it on a annexed directory is like
+
+    metatag -r -O gitannex,gitexclude -o gitannex:-stat ./
+
+There is a mailinglist for the project, you can subscribe at
+
+ 
diff --git a/doc/forum/__96__git_annex_copy__96____44___estimate_how_much_data_to_copy.mdwn b/doc/forum/__96__git_annex_copy__96____44___estimate_how_much_data_to_copy.mdwn
new file mode 100644
index 0000000000..dcc3377edf
--- /dev/null
+++ b/doc/forum/__96__git_annex_copy__96____44___estimate_how_much_data_to_copy.mdwn
@@ -0,0 +1,3 @@
+Hi,
+
+Is there any way to estimate how much data will be transferred and/or written down on the target repository of a `git annex copy` command? Something that would take into account that some data is already there, etc.
diff --git a/doc/forum/__96__git_annex_copy__96____44___estimate_how_much_data_to_copy/comment_1_785e4fe097ef2a2d51d84146fc91b2ca._comment b/doc/forum/__96__git_annex_copy__96____44___estimate_how_much_data_to_copy/comment_1_785e4fe097ef2a2d51d84146fc91b2ca._comment
new file mode 100644
index 0000000000..93a391e6cb
--- /dev/null
+++ b/doc/forum/__96__git_annex_copy__96____44___estimate_how_much_data_to_copy/comment_1_785e4fe097ef2a2d51d84146fc91b2ca._comment
@@ -0,0 +1,19 @@
+[[!comment format=mdwn
+ username="joey"
+ subject="""comment 1"""
+ date="2016-07-19T19:05:16Z"
+ content="""
+Using `git annex info .` in combination with the
+[[git-annex-matching-options]] is a good way to do such queries.
+
+The "size of annexed files in working tree" line gives the total size of
+the files that match that query.
+
+If you're copying files to a remote, run something like:
+
+	git annex info . --not --in $remote
+
+And if you're getting files, run something like:
+
+	git annex info . --not --in here
+"""]]
diff --git a/doc/forum/__96__git_annex_sync__96___hangs.mdwn b/doc/forum/__96__git_annex_sync__96___hangs.mdwn
new file mode 100644
index 0000000000..5a6a3d8ad7
--- /dev/null
+++ b/doc/forum/__96__git_annex_sync__96___hangs.mdwn
@@ -0,0 +1,113 @@
+I am on a direct remote. I just deleted a directory, and would like to commit that change.
+
+After I run `git annex sync` I see a line which says `commit`, and that is all. I tried to wait it out a few hours, but nothing. Why does git annex hang?
+
+Here is the output of `strace git annex sync`:
+
+```
+chymera@zenbookhost /run/media/chymera/data3 $ strace git annex sync
+execve("/usr/bin/git", ["git", "annex", "sync"], [/* 78 vars */]) = 0
+brk(0)                                  = 0x1afa000
+mmap(NULL, 4096, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS, -1, 0) = 0x7f17d12c4000
+access("/etc/ld.so.preload", R_OK)      = -1 ENOENT (No such file or directory)
+open("/etc/ld.so.cache", O_RDONLY|O_CLOEXEC) = 3
+fstat(3, {st_mode=S_IFREG|0644, st_size=207886, ...}) = 0
+mmap(NULL, 207886, PROT_READ, MAP_PRIVATE, 3, 0) = 0x7f17d1291000
+close(3)                                = 0
+open("/lib64/libpcre.so.1", O_RDONLY|O_CLOEXEC) = 3
+read(3, "\177ELF\2\1\1\0\0\0\0\0\0\0\0\0\3\0>\0\1\0\0\0P\27\0\0\0\0\0\0"..., 832) = 832
+fstat(3, {st_mode=S_IFREG|0755, st_size=452672, ...}) = 0
+mmap(NULL, 2548328, PROT_READ|PROT_EXEC, MAP_PRIVATE|MAP_DENYWRITE, 3, 0) = 0x7f17d0e36000
+mprotect(0x7f17d0ea4000, 2093056, PROT_NONE) = 0
+mmap(0x7f17d10a3000, 8192, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_FIXED|MAP_DENYWRITE, 3, 0x6d000) = 0x7f17d10a3000
+close(3)                                = 0
+open("/lib64/libz.so.1", O_RDONLY|O_CLOEXEC) = 3
+read(3, "\177ELF\2\1\1\0\0\0\0\0\0\0\0\0\3\0>\0\1\0\0\0`#\0\0\0\0\0\0"..., 832) = 832
+fstat(3, {st_mode=S_IFREG|0755, st_size=88456, ...}) = 0
+mmap(NULL, 2183720, PROT_READ|PROT_EXEC, MAP_PRIVATE|MAP_DENYWRITE, 3, 0) = 0x7f17d0c20000
+mprotect(0x7f17d0c35000, 2093056, PROT_NONE) = 0
+mmap(0x7f17d0e34000, 8192, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_FIXED|MAP_DENYWRITE, 3, 0x14000) = 0x7f17d0e34000
+close(3)                                = 0
+open("/lib64/libpthread.so.0", O_RDONLY|O_CLOEXEC) = 3
+read(3, "\177ELF\2\1\1\3\0\0\0\0\0\0\0\0\3\0>\0\1\0\0\0@p\0\0\0\0\0\0"..., 832) = 832
+fstat(3, {st_mode=S_IFREG|0755, st_size=125536, ...}) = 0
+mmap(NULL, 4096, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS, -1, 0) = 0x7f17d1290000
+mmap(NULL, 2204816, PROT_READ|PROT_EXEC, MAP_PRIVATE|MAP_DENYWRITE, 3, 0) = 0x7f17d0a05000
+mprotect(0x7f17d0a1b000, 2093056, PROT_NONE) = 0
+mmap(0x7f17d0c1a000, 8192, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_FIXED|MAP_DENYWRITE, 3, 0x15000) = 0x7f17d0c1a000
+mmap(0x7f17d0c1c000, 13456, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_FIXED|MAP_ANONYMOUS, -1, 0) = 0x7f17d0c1c000
+close(3)                                = 0
+open("/lib64/librt.so.1", O_RDONLY|O_CLOEXEC) = 3
+read(3, "\177ELF\2\1\1\3\0\0\0\0\0\0\0\0\3\0>\0\1\0\0\0`#\0\0\0\0\0\0"..., 832) = 832
+fstat(3, {st_mode=S_IFREG|0755, st_size=31536, ...}) = 0
+mmap(NULL, 2128920, PROT_READ|PROT_EXEC, MAP_PRIVATE|MAP_DENYWRITE, 3, 0) = 0x7f17d07fd000
+mprotect(0x7f17d0804000, 2093056, PROT_NONE) = 0
+mmap(0x7f17d0a03000, 8192, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_FIXED|MAP_DENYWRITE, 3, 0x6000) = 0x7f17d0a03000
+close(3)                                = 0
+open("/lib64/libc.so.6", O_RDONLY|O_CLOEXEC) = 3
+read(3, "\177ELF\2\1\1\3\0\0\0\0\0\0\0\0\3\0>\0\1\0\0\0\300\33\2\0\0\0\0\0"..., 832) = 832
+fstat(3, {st_mode=S_IFREG|0755, st_size=1660320, ...}) = 0
+mmap(NULL, 3766680, PROT_READ|PROT_EXEC, MAP_PRIVATE|MAP_DENYWRITE, 3, 0) = 0x7f17d0465000
+mprotect(0x7f17d05f3000, 2097152, PROT_NONE) = 0
+mmap(0x7f17d07f3000, 24576, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_FIXED|MAP_DENYWRITE, 3, 0x18e000) = 0x7f17d07f3000
+mmap(0x7f17d07f9000, 14744, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_FIXED|MAP_ANONYMOUS, -1, 0) = 0x7f17d07f9000
+close(3)                                = 0
+mmap(NULL, 4096, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS, -1, 0) = 0x7f17d128f000
+mmap(NULL, 4096, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS, -1, 0) = 0x7f17d128e000
+mmap(NULL, 4096, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS, -1, 0) = 0x7f17d128d000
+arch_prctl(ARCH_SET_FS, 0x7f17d128e700) = 0
+mprotect(0x7f17d07f3000, 16384, PROT_READ) = 0
+mprotect(0x7f17d0c1a000, 4096, PROT_READ) = 0
+mprotect(0x7f17d0a03000, 4096, PROT_READ) = 0
+mprotect(0x7f17d0e34000, 4096, PROT_READ) = 0
+mprotect(0x7f17d10a3000, 4096, PROT_READ) = 0
+mprotect(0x79b000, 4096, PROT_READ)     = 0
+mprotect(0x7f17d12c5000, 4096, PROT_READ) = 0
+munmap(0x7f17d1291000, 207886)          = 0
+set_tid_address(0x7f17d128e9d0)         = 2218
+set_robust_list(0x7f17d128e9e0, 24)     = 0
+rt_sigaction(SIGRTMIN, {0x7f17d0a0bb10, [], SA_RESTORER|SA_SIGINFO, 0x7f17d0a14d20}, NULL, 8) = 0
+rt_sigaction(SIGRT_1, {0x7f17d0a0bba0, [], SA_RESTORER|SA_RESTART|SA_SIGINFO, 0x7f17d0a14d20}, NULL, 8) = 0
+rt_sigprocmask(SIG_UNBLOCK, [RTMIN RT_1], NULL, 8) = 0
+getrlimit(RLIMIT_STACK, {rlim_cur=8192*1024, rlim_max=RLIM64_INFINITY}) = 0
+open("/dev/null", O_RDWR)               = 3
+close(3)                                = 0
+rt_sigprocmask(SIG_UNBLOCK, [PIPE], NULL, 8) = 0
+rt_sigaction(SIGPIPE, {SIG_DFL, [PIPE], SA_RESTORER|SA_RESTART, 0x7f17d0499ee0}, {SIG_DFL, [], 0}, 8) = 0
+brk(0)                                  = 0x1afa000
+brk(0x1b1b000)                          = 0x1b1b000
+open("/usr/lib64/locale/locale-archive", O_RDONLY|O_CLOEXEC) = 3
+fstat(3, {st_mode=S_IFREG|0644, st_size=106086208, ...}) = 0
+mmap(NULL, 106086208, PROT_READ, MAP_PRIVATE, 3, 0) = 0x7f17c9f39000
+close(3)                                = 0
+stat(".git", {st_mode=S_IFDIR|0755, st_size=4096, ...}) = 0
+access("/etc/gitconfig", R_OK)          = -1 ENOENT (No such file or directory)
+access("/home/chymera/.config/git/config", R_OK) = -1 ENOENT (No such file or directory)
+access("/home/chymera/.gitconfig", R_OK) = 0
+open("/home/chymera/.gitconfig", O_RDONLY) = 3
+fstat(3, {st_mode=S_IFREG|0644, st_size=147, ...}) = 0
+mmap(NULL, 4096, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS, -1, 0) = 0x7f17d12c3000
+read(3, "[user]\n\temail = h.chr@mail.ru\n\tn"..., 4096) = 147
+read(3, "", 4096)                       = 0
+close(3)                                = 0
+munmap(0x7f17d12c3000, 4096)            = 0
+access(".git/config", R_OK)             = 0
+open(".git/config", O_RDONLY)           = 3
+fstat(3, {st_mode=S_IFREG|0644, st_size=335, ...}) = 0
+mmap(NULL, 4096, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS, -1, 0) = 0x7f17d12c3000
+read(3, "[core]\n\trepositoryformatversion "..., 4096) = 335
+read(3, "", 4096)                       = 0
+close(3)                                = 0
+munmap(0x7f17d12c3000, 4096)            = 0
+pipe([3, 4])                            = 0
+clone(child_stack=0, flags=CLONE_CHILD_CLEARTID|CLONE_CHILD_SETTID|SIGCHLD, child_tidptr=0x7f17d128e9d0) = 2219
+rt_sigaction(SIGINT, {0x4f4d70, [INT], SA_RESTORER|SA_RESTART, 0x7f17d0499ee0}, {SIG_DFL, [], 0}, 8) = 0
+rt_sigaction(SIGHUP, {0x4f4d70, [HUP], SA_RESTORER|SA_RESTART, 0x7f17d0499ee0}, {SIG_DFL, [], 0}, 8) = 0
+rt_sigaction(SIGTERM, {0x4f4d70, [TERM], SA_RESTORER|SA_RESTART, 0x7f17d0499ee0}, {SIG_DFL, [], 0}, 8) = 0
+rt_sigaction(SIGQUIT, {0x4f4d70, [QUIT], SA_RESTORER|SA_RESTART, 0x7f17d0499ee0}, {SIG_DFL, [], 0}, 8) = 0
+rt_sigaction(SIGPIPE, {0x4f4d70, [PIPE], SA_RESTORER|SA_RESTART, 0x7f17d0499ee0}, {SIG_DFL, [PIPE], SA_RESTORER|SA_RESTART, 0x7f17d0499ee0}, 8) = 0
+close(4)                                = 0
+read(3, "", 1)                          = 0
+close(3)                                = 0
+wait4(2219, commit  
+```
diff --git a/doc/forum/__96__git_annex_sync__96___hangs/comment_1_ffd16cb94b39b93dfb0a2932b70938fc._comment b/doc/forum/__96__git_annex_sync__96___hangs/comment_1_ffd16cb94b39b93dfb0a2932b70938fc._comment
new file mode 100644
index 0000000000..e373a3c572
--- /dev/null
+++ b/doc/forum/__96__git_annex_sync__96___hangs/comment_1_ffd16cb94b39b93dfb0a2932b70938fc._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="https://me.yahoo.com/a/yx5Y6EI1t.759Jsu63ZWqYclCmpOmxxd.ramtw--#7114a"
+ nickname="Ioanas"
+ subject="comment 1"
+ date="2015-06-04T14:10:17Z"
+ content="""
+nobody?
+"""]]
diff --git a/doc/forum/__96__git_annex_sync__96___hangs/comment_2_fad59a54d92ed7cb230f0f365fddb0f4._comment b/doc/forum/__96__git_annex_sync__96___hangs/comment_2_fad59a54d92ed7cb230f0f365fddb0f4._comment
new file mode 100644
index 0000000000..19c3b0abb5
--- /dev/null
+++ b/doc/forum/__96__git_annex_sync__96___hangs/comment_2_fad59a54d92ed7cb230f0f365fddb0f4._comment
@@ -0,0 +1,10 @@
+[[!comment format=mdwn
+ username="joey"
+ subject="""comment 2"""
+ date="2015-07-20T17:35:42Z"
+ content="""
+Sorry didn't get to this until now.
+
+It would probably help to pass the --debug option to git-annex sync,
+and paste the output up to the point where it hangs.
+"""]]
diff --git a/doc/forum/__96__git_annex_sync__96___uses_too_much_space__63__/comment_1_29cd5e9acd78d8ac6b58fe535fee9650._comment b/doc/forum/__96__git_annex_sync__96___uses_too_much_space__63__/comment_1_29cd5e9acd78d8ac6b58fe535fee9650._comment
new file mode 100644
index 0000000000..bdf9861f14
--- /dev/null
+++ b/doc/forum/__96__git_annex_sync__96___uses_too_much_space__63__/comment_1_29cd5e9acd78d8ac6b58fe535fee9650._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="https://www.google.com/accounts/o8/id?id=AItOawnVnsqEy82M-MuS2gLri-az83wSQ6lXSrc"
+ nickname="Jean"
+ subject="comment 1"
+ date="2015-05-02T08:28:56Z"
+ content="""
+I tried reformatting the device with smaller blocksize and inode size, but this didn't help. 
+"""]]
diff --git a/doc/forum/__96__git_annex_sync__96___uses_too_much_space__63__/comment_2_7a24236bc511cbfa869aaeb431a003d2._comment b/doc/forum/__96__git_annex_sync__96___uses_too_much_space__63__/comment_2_7a24236bc511cbfa869aaeb431a003d2._comment
new file mode 100644
index 0000000000..60e1f22af8
--- /dev/null
+++ b/doc/forum/__96__git_annex_sync__96___uses_too_much_space__63__/comment_2_7a24236bc511cbfa869aaeb431a003d2._comment
@@ -0,0 +1,18 @@
+[[!comment format=mdwn
+ username="joey"
+ subject="""comment 2"""
+ date="2015-05-05T16:42:43Z"
+ content="""
+It's failing to pull the git repository from the remote because that
+git repository is apparently larger than 487 mb. That is not usual 
+when using git-annex, or even when using git unless you have millions
+of files in the git repository.
+
+Since you only have a few thousand files in the git repository,
+my guess is you have committed some large files directly to git,
+instead of using git-annex. So, you're seeing why git-annex exists...
+
+You should find the files in your git repository that are not git-annex
+symlinks, and are large files. You may need to use `git filter-branch`
+to remove the from your repository's history.
+"""]]
diff --git a/doc/forum/_illegal_control_characters_in_pairing_message__59___ignoring.mdwn b/doc/forum/_illegal_control_characters_in_pairing_message__59___ignoring.mdwn
new file mode 100644
index 0000000000..5d39ec77f8
--- /dev/null
+++ b/doc/forum/_illegal_control_characters_in_pairing_message__59___ignoring.mdwn
@@ -0,0 +1,40 @@
+Currently I am trying to synchronize two folders between my NAS and my main computer. On both machines I can access the web app. However, once I try to pair these machines with a secret phrase I get the following error:
+ 
+illegal control characters in pairing message; ignoring
+
+The log on one machine is:
+
+[2016-03-18 14:25:16.861132] main: Pairing in progress
+Generating public/private rsa key pair.
+Your identification has been saved in /var/folders/_v/hkf9zbqs0bgd2vcnmr_qnlgm0000gn/T/git-annex-keygenN92651/key.
+Your public key has been saved in /var/folders/_v/hkf9zbqs0bgd2vcnmr_qnlgm0000gn/T/git-annex-keygenN92651/key.pub.
+The key fingerprint is:
+SHA256:beyV1f5MKFGaaH9p1CRMPmaVcYAmOyc/lhp7HCw668M username@iMac.home
+The key's randomart image is:
++---[RSA 2048]----+
+|             +=o*|
+|          ..o=.B.|
+|          o++ B o|
+|         ++..B = |
+|        S +*=.= o|
+|         oo.B+ o.|
+|       . ..B o  o|
+|        E o o    |
+|       .o+ .     |
++----[SHA256]-----+
+[2016-03-18 14:25:26.442301] main: Pairing in progress
+ 
+  illegal control characters in pairing message; ignoring
+ 
+  illegal control characters in pairing message; ignoring
+ 
+  illegal control characters in pairing message; ignoring
+ 
+  illegal control characters in pairing message; ignoring
+
+While on the other machine I get:
+
+[2016-03-18 14:25:35 CET] main: Pairing with username@iMac.home:~/Desktop/annex in progress
+
+
+
diff --git a/doc/forum/_illegal_control_characters_in_pairing_message__59___ignoring/comment_1_99975a677f832c410050ba721be6d904._comment b/doc/forum/_illegal_control_characters_in_pairing_message__59___ignoring/comment_1_99975a677f832c410050ba721be6d904._comment
new file mode 100644
index 0000000000..0ebe49ec5b
--- /dev/null
+++ b/doc/forum/_illegal_control_characters_in_pairing_message__59___ignoring/comment_1_99975a677f832c410050ba721be6d904._comment
@@ -0,0 +1,15 @@
+[[!comment format=mdwn
+ username="joey"
+ subject="""comment 1"""
+ date="2016-03-18T15:31:17Z"
+ content="""
+This probably has something to do with the hostname of the other host than
+the one that displays the error. Or possibly with the name of the user on
+that host, or the directory on that host where the git-annex repository is.
+
+Can you share that information, or at least the parts of it that are not
+regular alphabet characters?
+
+(There's also an outside possibility it could have found a control
+character in the ssh public key, but I don't see how.)
+"""]]
diff --git a/doc/forum/_illegal_control_characters_in_pairing_message__59___ignoring/comment_2_f4dd76879e2b2bd0da557ee6a539533c._comment b/doc/forum/_illegal_control_characters_in_pairing_message__59___ignoring/comment_2_f4dd76879e2b2bd0da557ee6a539533c._comment
new file mode 100644
index 0000000000..6e732416f5
--- /dev/null
+++ b/doc/forum/_illegal_control_characters_in_pairing_message__59___ignoring/comment_2_f4dd76879e2b2bd0da557ee6a539533c._comment
@@ -0,0 +1,95 @@
+[[!comment format=mdwn
+ username="butirsky@86ff3803c45a8a1e5fc253d2728ce493c8addf96"
+ nickname="butirsky"
+ avatar="http://cdn.libravatar.org/avatar/bf8e56c57b179935c6f52ebc4690c2bf"
+ subject="comment 2"
+ date="2017-01-18T13:19:43Z"
+ content="""
+Having the same issue, the error only on one side, syncing doesn't start:
+
+```
+[2017-01-18 15:25:30.603909] main: starting assistant version 5.20151208-1build1
+[2017-01-18 15:25:30.615017] Cronner: You should enable consistency checking to protect your data. 
+
+  No known volume monitor available through dbus; falling back to mtab polling
+(scanning...) [2017-01-18 15:25:31.040087] Watcher: Performing startup scan
+(started...) 
+(scanning...) [2017-01-18 15:27:19.254315] Watcher: Performing startup scan
+(started...) git-annex: Daemon is already running.
+warning: no common commits
+From /home/abutirsky/Desktop/annex
+ * [new branch]      annex/direct/master -> annex/annex/direct/master
+ * [new branch]      git-annex  -> annex/git-annex
+ * [new branch]      master     -> annex/master
+
+(merging annex/git-annex into git-annex...)
+(recording state in git...)
+
+Already up-to-date!
+Merge made by the 'recursive' strategy.
+[2017-01-18 15:29:18.003333] main: Syncing with annex 
+[2017-01-18 15:29:18.003518] Pusher: Syncing with annex 
+To /home/abutirsky/Desktop/annex
+ * [new branch]      git-annex -> synced/git-annex
+ * [new branch]      master -> synced/master
+Everything up-to-date
+
+(nautilus:19391): GLib-GIO-CRITICAL **: g_dbus_interface_skeleton_unexport: assertion 'interface_->priv->connections != NULL' failed
+
+(nautilus:19391): GLib-GIO-CRITICAL **: g_dbus_interface_skeleton_unexport: assertion 'interface_->priv->connections != NULL' failed
+
+(nautilus:19391): Gtk-CRITICAL **: gtk_icon_theme_get_for_screen: assertion 'GDK_IS_SCREEN (screen)' failed
+
+(nautilus:19391): GLib-GObject-WARNING **: invalid (NULL) pointer instance
+
+(nautilus:19391): GLib-GObject-CRITICAL **: g_signal_connect_object: assertion 'G_TYPE_CHECK_INSTANCE (instance)' failed
+git-annex: Daemon is already running.
+(scanning...) [2017-01-18 15:34:47.539423] Watcher: Performing startup scan
+(started...) git-annex: Daemon is already running.
+git-annex: Daemon is already running.
+git-annex: Daemon is already running.
+
+
+  illegal control characters in pairing message; ignoring
+
+  illegal control characters in pairing message; ignoring
+
+  illegal control characters in pairing message; ignoring
+
+  illegal control characters in pairing message; ignoring
+
+  illegal control characters in pairing message; ignoring
+
+  illegal control characters in pairing message; ignoring
+
+  illegal control characters in pairing message; ignoring
+
+  illegal control characters in pairing message; ignoring
+
+  illegal control characters in pairing message; ignoring
+Generating public/private rsa key pair.
+Your identification has been saved in /tmp/git-annex-keygen.0/key.
+Your public key has been saved in /tmp/git-annex-keygen.0/key.pub.
+The key fingerprint is:
+SHA256:dJ9VU8CQ+hHL2fsta1E5PkxTvkqDThbyOainUXZvaY4 abutirsky@sml-pc-066
+The key's randomart image is:
++---[RSA 2048]----+
+|            .+.o+|
+|            o ..o|
+|        . .o =..o|
+|       . o.o=o.=o|
+|        So+o*.+o+|
+|        o..Boo+= |
+|       .. + o=ooo|
+|       ... .=.o o|
+|       .o  E o.o |
++----[SHA256]-----+
+[2017-01-18 15:48:44.423064] main: Pairing in progress
+
+  illegal control characters in pairing message; ignoring
+
+  illegal control characters in pairing message; ignoring
+
+  illegal control characters in pairing message; ignoring
+```
+"""]]
diff --git a/doc/forum/_illegal_control_characters_in_pairing_message__59___ignoring/comment_3_d437dc44114f59d5717c177831ae0d53._comment b/doc/forum/_illegal_control_characters_in_pairing_message__59___ignoring/comment_3_d437dc44114f59d5717c177831ae0d53._comment
new file mode 100644
index 0000000000..52c0d8134b
--- /dev/null
+++ b/doc/forum/_illegal_control_characters_in_pairing_message__59___ignoring/comment_3_d437dc44114f59d5717c177831ae0d53._comment
@@ -0,0 +1,9 @@
+[[!comment format=mdwn
+ username="butirsky@86ff3803c45a8a1e5fc253d2728ce493c8addf96"
+ nickname="butirsky"
+ avatar="http://cdn.libravatar.org/avatar/bf8e56c57b179935c6f52ebc4690c2bf"
+ subject="comment 3"
+ date="2017-01-18T13:21:45Z"
+ content="""
+Version of annex on the other side is 5.20140412ubuntu1, if it matters
+"""]]
diff --git a/doc/forum/_illegal_control_characters_in_pairing_message__59___ignoring/comment_4_dd29ac2b5615ff6f79458a36bce44306._comment b/doc/forum/_illegal_control_characters_in_pairing_message__59___ignoring/comment_4_dd29ac2b5615ff6f79458a36bce44306._comment
new file mode 100644
index 0000000000..3b2ba3a2fa
--- /dev/null
+++ b/doc/forum/_illegal_control_characters_in_pairing_message__59___ignoring/comment_4_dd29ac2b5615ff6f79458a36bce44306._comment
@@ -0,0 +1,55 @@
+[[!comment format=mdwn
+ username="butirsky@86ff3803c45a8a1e5fc253d2728ce493c8addf96"
+ nickname="butirsky"
+ avatar="http://cdn.libravatar.org/avatar/bf8e56c57b179935c6f52ebc4690c2bf"
+ subject="comment 4"
+ date="2017-01-18T16:52:21Z"
+ content="""
+Log from other side:
+
+```
+[2017-01-18 15:46:22 MSK] main: starting assistant version 5.20140412ubuntu1
+[2017-01-18 15:46:22 MSK] Cronner: You should enable consistency checking to protect your data. 
+(Recording state in git...)
+(scanning...) [2017-01-18 15:46:22 MSK] Watcher: Performing startup scan
+(started...) Generating public/private rsa key pair.
+Your identification has been saved in /tmp/git-annex-keygen.0/key.
+Your public key has been saved in /tmp/git-annex-keygen.0/key.pub.
+The key fingerprint is:
+aa:c0:c3:b8:ca:4d:5c:85:58:02:b6:a0:4a:64:18:dd echormonov@sml-pc-091
+The key's randomart image is:
++--[ RSA 2048]----+
+|+B.o .           |
+|B o E .          |
+|.o . . .         |
+|o     .          |
+|.    .  S        |
+| +. .  .         |
+|. =o  .          |
+|..oo .           |
+|+. ..            |
++-----------------+
+[2017-01-18 15:48:26 MSK] main: Pairing in progress
+[2017-01-18 15:48:44 MSK] PairListener: abutirsky@sml-pc-066:~/Desktop/gitrepo_test is sending a pair request.
+Generating public/private rsa key pair.
+Your identification has been saved in /tmp/git-annex-keygen.0/key.
+Your public key has been saved in /tmp/git-annex-keygen.0/key.pub.
+The key fingerprint is:
+28:e0:f4:a2:2c:a2:bc:7a:22:6e:4c:6d:8f:38:ac:b4 echormonov@sml-pc-091
+The key's randomart image is:
++--[ RSA 2048]----+
+|                 |
+|                 |
+|  o              |
+| o o   .         |
+|  + o . S        |
+|.o + .           |
+|Boo o            |
+|O*o. .           |
+|XEo              |
++-----------------+
+[2017-01-18 15:50:40 MSK] main: Pairing with abutirsky@sml-pc-066:~/Desktop/gitrepo_test in progress
+```
+
+By the way, any comment here is accessible for deletion by anyone. Seems like security hole
+"""]]
diff --git a/doc/forum/_illegal_control_characters_in_pairing_message__59___ignoring/comment_5_e07b69bb36cd8aa2c4fd689744f12df1._comment b/doc/forum/_illegal_control_characters_in_pairing_message__59___ignoring/comment_5_e07b69bb36cd8aa2c4fd689744f12df1._comment
new file mode 100644
index 0000000000..62e073336e
--- /dev/null
+++ b/doc/forum/_illegal_control_characters_in_pairing_message__59___ignoring/comment_5_e07b69bb36cd8aa2c4fd689744f12df1._comment
@@ -0,0 +1,15 @@
+[[!comment format=mdwn
+ username="joey"
+ subject="""comment 5"""
+ date="2017-01-31T16:50:18Z"
+ content="""
+Is "username@iMac.home" data that you obscured, or are you actually using
+an account called "username" with a hostname of "iMac.home"?
+
+As I said before, this is likely caused by some character that git-annex
+dooes not like in username or hostname of the other host than the one that
+displays the error.
+
+Hopefully one of you guys who can reproduce the problem can get this basic
+information to me, so I can try to get to the bottom of the problem.
+"""]]
diff --git a/doc/forum/_illegal_control_characters_in_pairing_message__59___ignoring/comment_6_f61a390f304e1f6d1e8c4a90cdfbd9b9._comment b/doc/forum/_illegal_control_characters_in_pairing_message__59___ignoring/comment_6_f61a390f304e1f6d1e8c4a90cdfbd9b9._comment
new file mode 100644
index 0000000000..05cc831836
--- /dev/null
+++ b/doc/forum/_illegal_control_characters_in_pairing_message__59___ignoring/comment_6_f61a390f304e1f6d1e8c4a90cdfbd9b9._comment
@@ -0,0 +1,9 @@
+[[!comment format=mdwn
+ username="butirsky@86ff3803c45a8a1e5fc253d2728ce493c8addf96"
+ nickname="butirsky"
+ avatar="http://cdn.libravatar.org/avatar/bf8e56c57b179935c6f52ebc4690c2bf"
+ subject="comment 6"
+ date="2017-01-31T18:30:00Z"
+ content="""
+@joey, in my case all the user/host names are actual, and they do not include bad characters obviously
+"""]]
diff --git a/doc/forum/_illegal_control_characters_in_pairing_message__59___ignoring/comment_7_625df33e70f2f959c574d918a706648d._comment b/doc/forum/_illegal_control_characters_in_pairing_message__59___ignoring/comment_7_625df33e70f2f959c574d918a706648d._comment
new file mode 100644
index 0000000000..e6303d1c46
--- /dev/null
+++ b/doc/forum/_illegal_control_characters_in_pairing_message__59___ignoring/comment_7_625df33e70f2f959c574d918a706648d._comment
@@ -0,0 +1,13 @@
+[[!comment format=mdwn
+ username="joey"
+ subject="""comment 7"""
+ date="2017-02-13T19:52:36Z"
+ content="""
+Seems we can rule out user/hostname/directory, which probably leaves
+the ssh public key as the culprit, but I'm not sure.
+
+So, I changed the error message to display the problem data,
+in [[!commit f36adc2dbc46ec50ee7c88962de0a0d1473681eb]]. Once you have a
+git-annex version with that commit, if you see the problem, paste in the
+error message and I should be able to get to the bottom of this.
+"""]]
diff --git a/doc/forum/_preferred_content__58___lastpresent.mdwn b/doc/forum/_preferred_content__58___lastpresent.mdwn
new file mode 100644
index 0000000000..e8730eb00c
--- /dev/null
+++ b/doc/forum/_preferred_content__58___lastpresent.mdwn
@@ -0,0 +1 @@
+Is there any kind of "lastpresent" in the preferred-content expression? If set, git-annex would see if "git log --follow $path -n 1" (or some configurable -n) was present.
diff --git a/doc/forum/_preferred_content__58___lastpresent/comment_1_7610cd866b256d36646b642eb5f8cae5._comment b/doc/forum/_preferred_content__58___lastpresent/comment_1_7610cd866b256d36646b642eb5f8cae5._comment
new file mode 100644
index 0000000000..d5910c6d03
--- /dev/null
+++ b/doc/forum/_preferred_content__58___lastpresent/comment_1_7610cd866b256d36646b642eb5f8cae5._comment
@@ -0,0 +1,14 @@
+[[!comment format=mdwn
+ username="http://joeyh.name/"
+ ip="209.250.56.246"
+ subject="comment 1"
+ date="2013-11-15T20:39:43Z"
+ content="""
+The idea seems to be to match files that have been deleted.
+
+I don't see how that could work; preferred content expressions are always matched against files in the working tree.
+
+Even if it were doable, it would be quite expensive if every preferred content query involved a `git log`...
+
+What are you trying to achieve with this?
+"""]]
diff --git a/doc/forum/_preferred_content__58___lastpresent/comment_2_d25666a173b78213d583f029fd166d06._comment b/doc/forum/_preferred_content__58___lastpresent/comment_2_d25666a173b78213d583f029fd166d06._comment
new file mode 100644
index 0000000000..4714034c36
--- /dev/null
+++ b/doc/forum/_preferred_content__58___lastpresent/comment_2_d25666a173b78213d583f029fd166d06._comment
@@ -0,0 +1,10 @@
+[[!comment format=mdwn
+ username="https://www.google.com/accounts/o8/id?id=AItOawn1QhtPvsRBV7pfaDW_ZTPFv_ZIxSzQ8Rg"
+ nickname="Paul Léo"
+ subject="comment 2"
+ date="2013-11-16T13:12:35Z"
+ content="""
+The aim is to be able to have some semi-automatic mode (using \"git-annex wanted . (lastpresent)\"), where file paths which you \"git-annex get\" one time always get updated to the latest version.
+
+Or, you could also use some variant of it the other way round (using \"git-annex wanted . (notlastpresent)\"): all files would be present by default, but paths which you \"git-annex drop\"ed once wouldn't become present again.
+"""]]
diff --git a/doc/forum/add_only_binary_files__63__.mdwn b/doc/forum/add_only_binary_files__63__.mdwn
new file mode 100644
index 0000000000..4b40db5dd7
--- /dev/null
+++ b/doc/forum/add_only_binary_files__63__.mdwn
@@ -0,0 +1 @@
+Is there a way to only add binary files with git annex add command?
diff --git a/doc/forum/add_only_binary_files__63__/comment_1_7ce3be5bafd62ce5ed78bcd9323039cc._comment b/doc/forum/add_only_binary_files__63__/comment_1_7ce3be5bafd62ce5ed78bcd9323039cc._comment
new file mode 100644
index 0000000000..e53cb1f0f3
--- /dev/null
+++ b/doc/forum/add_only_binary_files__63__/comment_1_7ce3be5bafd62ce5ed78bcd9323039cc._comment
@@ -0,0 +1,16 @@
+[[!comment format=mdwn
+ username="http://joeyh.name/"
+ ip="209.250.56.54"
+ subject="comment 1"
+ date="2014-10-06T15:36:54Z"
+ content="""
+If you can configure `annex.largefiles` to match only binary files, then `git annex add` will respect it, and only add those files.
+
+For example, if you were working on a game written in C, and wanted to use git-annex only for the game art, but not the source code, you could configure it:
+
+    git config annex.largefiles 'largerthan=100kb and not (include=*.c or include=*.h)'
+
+This doesn't currently support looking at the file content to determine eg, its MIME type. That's been suggested as an added feature before.
+
+More simply, if you `git add` the non-binary files yourself first, `git annex add` will skip over those files and only add the other files.
+"""]]
diff --git a/doc/forum/adding_files_without_hashing_them.mdwn b/doc/forum/adding_files_without_hashing_them.mdwn
new file mode 100644
index 0000000000..a9aef308f4
--- /dev/null
+++ b/doc/forum/adding_files_without_hashing_them.mdwn
@@ -0,0 +1 @@
+I would like to be able to add files without having to hash its contents (like WORM) but being able to modify them and record its changes. Is this possible? In other words, I would like to provide other not-that-expensive mechanism for identifying files of a particular version.
diff --git a/doc/forum/adding_files_without_hashing_them/comment_1_c3113d7aff6b64a325a32b8b281df605._comment b/doc/forum/adding_files_without_hashing_them/comment_1_c3113d7aff6b64a325a32b8b281df605._comment
new file mode 100644
index 0000000000..56577ca1a7
--- /dev/null
+++ b/doc/forum/adding_files_without_hashing_them/comment_1_c3113d7aff6b64a325a32b8b281df605._comment
@@ -0,0 +1,12 @@
+[[!comment format=mdwn
+ username="http://joeyh.name/"
+ ip="209.250.56.22"
+ subject="comment 1"
+ date="2014-09-16T19:50:41Z"
+ content="""
+You can edit files that use the WORM backend, as long as the editing changes their size or mtime, or both. If neither changes, git-annex won't be able to keep your edits separate when using WORM.
+
+A perhaps safer compromise can be to use the WORM backend initially, but them `git annex migrate --backend=SHA256E` when you have spare CPU cycles. 
+
+Or, when you commit a change to a file that had been using WORM, use `git annex add $wormy_file --backend=SHA256E` to make the new version use the better backend.
+"""]]
diff --git a/doc/forum/adding_remote_bup_repo_using_ssh.mdwn b/doc/forum/adding_remote_bup_repo_using_ssh.mdwn
new file mode 100644
index 0000000000..1c13d7f8e9
--- /dev/null
+++ b/doc/forum/adding_remote_bup_repo_using_ssh.mdwn
@@ -0,0 +1,42 @@
+I have two machines connected to my home network. machine A. Machine A has a git-annex repo which stores the data using a remote bup repo. The bup repo resides on Machine A itself. Now, I want to set up the same kind of environment in Machine B ( git annex repo backed down by bup remote). The only condition is that I need to re-use the bup repo in the machine A. So I went up like this:
+    
+    git clone ssh://192.168.1.33/path/to/git-annex-repo-on-machine-a
+
+This succeeded. The next step was to add a reference to the bup repo on machine A to this new git-annex repo in machine B. I went about like this:
+
+    git annex initremote mybup type=bup encryption=none buprepo=192.168.1.33:/path/to/bup-repo-on-machine-a
+
+This seems to work OK, except the fact that the following shows up:
+
+    Reinitialized existing Git repository in /Users/XXX/.bup/ 
+    Reinitialized existing Git repository in /Users/path/to/bup-repo-on-machine-a
+
+*Q1*: Why does git-annex create bup repos on the local folder while initializaing a remote bup repo?
+
+After that, I tried to get the file from the git repo using 
+
+    git annex get   --from mybup
+
+..And nothing happened. This is the debug output from the last command :
+
+    read: git ["--git-dir=/private/tmp/annexed-setups/.git","--work-tree=/private/tmp/annexed-setups","show-ref","git-annex"]
+    read: git ["--git-dir=/private/tmp/annexed-setups/.git","--work-tree=/private/tmp/annexed-setups","show-ref","--hash","refs/heads/git-annex"]
+    read: git ["--git-dir=/private/tmp/annexed-setups/.git","--work-tree=/private/tmp/annexed-setups","log","refs/heads/git-   annex..c9af31cb3e563657e83b8b8e1f9a8de1ff690e4f","-n1","--pretty=%H"]
+    read: git ["--git-dir=/private/tmp/annexed-setups/.git","--work-tree=/private/tmp/annexed-setups","log","refs/heads/git-annex..b7b2af7f94af4770d5b4da7231bc3e41c0a6129d","-n1","--pretty=%H"]
+    read: git ["--git-dir=/private/tmp/annexed-setups/.git","--work-tree=/private/tmp/annexed-setups","log","refs/heads/git-annex..7b1017c5e68ec5a2b298b899bf723d0093fa0c7c","-n1","--pretty=%H"]
+    chat: git ["--git-dir=/private/tmp/annexed-setups/.git","--work-tree=/private/tmp/annexed-setups","cat-file","--batch"]
+    read: git ["--git-dir=/private/tmp/annexed-setups/.git","--work-tree=/private/tmp/annexed-setups","ls-files","--cached","-z","--","file.iso"]
+
+The file was not obtained from the remote. However, when I did 
+
+    git annex fsck --from mybup
+
+This resolved all the files properly and then when I did a 
+
+    git annex list. 
+
+it showed all files to be reachable from the mybup repo. 
+
+*Q2* Is this the correct workflow? 
+*Q3* Do I need to do a [costly] fsck for bup remote to properly work over ssh?
+
diff --git a/doc/forum/adding_remote_bup_repo_using_ssh/comment_1_d1ab38bccc415a94fa3906f7f5e823f3._comment b/doc/forum/adding_remote_bup_repo_using_ssh/comment_1_d1ab38bccc415a94fa3906f7f5e823f3._comment
new file mode 100644
index 0000000000..bd9d014596
--- /dev/null
+++ b/doc/forum/adding_remote_bup_repo_using_ssh/comment_1_d1ab38bccc415a94fa3906f7f5e823f3._comment
@@ -0,0 +1,18 @@
+[[!comment format=mdwn
+ username="joey"
+ subject="""comment 1"""
+ date="2015-01-20T16:05:31Z"
+ content="""
+IIRC bup always initializes ~/.bup, even when using a remote repository.
+
+Your mistake is that you have run `git annex initremote` twice -- so now
+git-annex knows about two different remotes, which both happen to be backed
+by the same bup repository -- but git-annex doesn't know that.
+
+Instead, you should run `git annex initremote` in one repository, and then
+use `git annex enableremote` in clones of that repository to enable your
+bup remote.
+
+You will probably want to kill off the redundant bup remote with `git annex
+dead mybup` and then using `git annex enableremote` to enable the other one.
+"""]]
diff --git a/doc/forum/adding_remote_bup_repo_using_ssh/comment_2_6698ecb746e6edb25130c8dbceb36fc0._comment b/doc/forum/adding_remote_bup_repo_using_ssh/comment_2_6698ecb746e6edb25130c8dbceb36fc0._comment
new file mode 100644
index 0000000000..233ee88093
--- /dev/null
+++ b/doc/forum/adding_remote_bup_repo_using_ssh/comment_2_6698ecb746e6edb25130c8dbceb36fc0._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="joey"
+ subject="""comment 2"""
+ date="2015-01-20T16:44:21Z"
+ content="""
+I've added a new section to the walkthrough about this:
+
+"""]]
diff --git a/doc/forum/advantages_of_SHA__42___over_WORM.mdwn b/doc/forum/advantages_of_SHA__42___over_WORM.mdwn
new file mode 100644
index 0000000000..5b544593f5
--- /dev/null
+++ b/doc/forum/advantages_of_SHA__42___over_WORM.mdwn
@@ -0,0 +1,5 @@
+Thanks for creating git-annex.
+
+I am confused about the advantages of the SHA* backends over WORM.  The "backends" page in this wiki says that with WORM, files "can be moved around, but should never be added to or changed".  But I don't see any difference to SHA* files as long as the premise of WORM that "any file with the same basename, size, and modification time has the same content" is true.  Using "git annex unlock", WORM files can be modified in the same way as SHA* files.
+
+If the storage I use is dependable (i.e. I don't need SHA checksums for detection of corruption), and I don't need to optimize for the case that the modification date of a file is changed but the contents stay the same, and if it is unlikely that several files will be identical, is there actually any advantage in using SHA*?
diff --git a/doc/forum/advantages_of_SHA__42___over_WORM/comment_1_96c354cac4b5ce5cf6664943bc84db1d._comment b/doc/forum/advantages_of_SHA__42___over_WORM/comment_1_96c354cac4b5ce5cf6664943bc84db1d._comment
new file mode 100644
index 0000000000..218027ca53
--- /dev/null
+++ b/doc/forum/advantages_of_SHA__42___over_WORM/comment_1_96c354cac4b5ce5cf6664943bc84db1d._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="http://joey.kitenet.net/"
+ nickname="joey"
+ subject="comment 1"
+ date="2011-08-29T16:10:38Z"
+ content="""
+You're right -- as long as nothing changes a file without letting the modification time update, editing WORM files is safe.
+"""]]
diff --git a/doc/forum/alternativeto.net___34__Like__34__.mdwn b/doc/forum/alternativeto.net___34__Like__34__.mdwn
new file mode 100644
index 0000000000..95e03dc21e
--- /dev/null
+++ b/doc/forum/alternativeto.net___34__Like__34__.mdwn
@@ -0,0 +1,3 @@
+When I went to alternativeto.net I noticed that SpiderOak is a featured application. I decided to search git-annex and see how "Like"-ed it is in comparison... there were 0 "Like"-s.
+
+I suggest going to  and "Like" git-annex.
diff --git a/doc/forum/android_binary-only_download.mdwn b/doc/forum/android_binary-only_download.mdwn
new file mode 100644
index 0000000000..90204fbf42
--- /dev/null
+++ b/doc/forum/android_binary-only_download.mdwn
@@ -0,0 +1,9 @@
+It would be really neat if there was a way to get just the `git-annex` binary on Android, without getting the entire APK. This can be useful when one is already using their own shell and rootfs, for example I work with this.
+
+This way, one can just use parts of git-annex, without having to have a whole APK and its dependencies, and the space it takes up, etc.
+
+So would you be able to add a download link for this?
+
+
+
+Thanks for the wonderful project!
diff --git a/doc/forum/android_binary-only_download/comment_1_aab206e0bf0bb5ff47c7cc9795f12f92._comment b/doc/forum/android_binary-only_download/comment_1_aab206e0bf0bb5ff47c7cc9795f12f92._comment
new file mode 100644
index 0000000000..d8e6d81fce
--- /dev/null
+++ b/doc/forum/android_binary-only_download/comment_1_aab206e0bf0bb5ff47c7cc9795f12f92._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="http://joeyh.name/"
+ ip="4.154.4.193"
+ subject="comment 1"
+ date="2013-07-02T16:57:43Z"
+ content="""
+This seems a pretty unusual use case. Can't you just unpack the apk (it's a regular zip file) and get out the parts you want?
+"""]]
diff --git a/doc/forum/annex.largefiles__58___two_quesitons.mdwn b/doc/forum/annex.largefiles__58___two_quesitons.mdwn
new file mode 100644
index 0000000000..170aad7e63
--- /dev/null
+++ b/doc/forum/annex.largefiles__58___two_quesitons.mdwn
@@ -0,0 +1,9 @@
+Hi,
+
+I have two questions related to `annex.largefiles`, for which I was unable to find the answers.
+
+1. Is there a way to transfer my `annex.largefiles` from one repo to the other?  I noticed that the property was not adopted when cloning a repo.
+2. On one repo I noticed that I had not set a good `annex.largefiles`.  Can I tell `git annex` to *reevaluate* whether a file should be controlled as a "normal" file or a "largefile" give the updated `annex.largefiles`?
+
+Thanks,
+Rasmus
diff --git a/doc/forum/annex.largefiles__58___two_quesitons/comment_1_75fc6cd2315e9ad6897b94bcc527ddc1._comment b/doc/forum/annex.largefiles__58___two_quesitons/comment_1_75fc6cd2315e9ad6897b94bcc527ddc1._comment
new file mode 100644
index 0000000000..2be567c397
--- /dev/null
+++ b/doc/forum/annex.largefiles__58___two_quesitons/comment_1_75fc6cd2315e9ad6897b94bcc527ddc1._comment
@@ -0,0 +1,11 @@
+[[!comment format=mdwn
+ username="joey"
+ subject="""comment 1"""
+ date="2015-03-26T15:56:59Z"
+ content="""
+1. No, annex.largefiles is a local git repository configuration.
+
+2. If you have a specific file in mind, and it is currently stored in the
+annex, you can always `git annex unannex $file` and then `git annex add` it
+back, which will re-check the current annex.largefiles configuration.
+"""]]
diff --git a/doc/forum/annex.largefiles__58___two_quesitons/comment_2_f34cf28c3f8354e6fc055d4dba6ed555._comment b/doc/forum/annex.largefiles__58___two_quesitons/comment_2_f34cf28c3f8354e6fc055d4dba6ed555._comment
new file mode 100644
index 0000000000..ab02f0e4a2
--- /dev/null
+++ b/doc/forum/annex.largefiles__58___two_quesitons/comment_2_f34cf28c3f8354e6fc055d4dba6ed555._comment
@@ -0,0 +1,13 @@
+[[!comment format=mdwn
+ username="cstork"
+ subject="comment 2"
+ date="2016-04-21T06:46:40Z"
+ content="""
+Just to prevent a potential misunderstanding: There are two ways to configure annex.largefiles.
+
+1. in `.git/config`, e.g., via `git config annex.largefiles=...`
+2. in `.gitattributes`
+
+The first configuration is specific to your local copy of the repository and does not travel with the content,
+whereas the second one does carry the configuration over to the other repos.
+"""]]
diff --git a/doc/forum/annex_merge_creates___34__synced__47____42____34___branches.mdwn b/doc/forum/annex_merge_creates___34__synced__47____42____34___branches.mdwn
new file mode 100644
index 0000000000..b4f02bc565
--- /dev/null
+++ b/doc/forum/annex_merge_creates___34__synced__47____42____34___branches.mdwn
@@ -0,0 +1,7 @@
+Hi,
+
+when I fetch from a central repository and `git annex merge` a synced/master branch is created. How can I disable this behaviour?
+
+Or is there a better workflow for getting and pushing updates from/to a central git repo?
+
+bye
diff --git a/doc/forum/annex_merge_creates___34__synced__47____42____34___branches/comment_1_4667fadb05c594b0a212bf455ee65298._comment b/doc/forum/annex_merge_creates___34__synced__47____42____34___branches/comment_1_4667fadb05c594b0a212bf455ee65298._comment
new file mode 100644
index 0000000000..90643ca9b6
--- /dev/null
+++ b/doc/forum/annex_merge_creates___34__synced__47____42____34___branches/comment_1_4667fadb05c594b0a212bf455ee65298._comment
@@ -0,0 +1,12 @@
+[[!comment format=mdwn
+ username="http://joeyh.name/"
+ ip="209.250.56.22"
+ subject="comment 1"
+ date="2014-09-16T18:22:11Z"
+ content="""
+Why is this a problem? You can delete the branch at any time of course if it's in the way.
+
+It would be possible for `git-annex sync` to avoid creating the synced/master branch at all when syncing with a bare git repository, but this would actually make it less efficient and slower. Where currently it makes one push, updating the remote's master branch when possible, and forcing an update of its synced/master branch at the same time, it would instead need to first try to update remote's master, then check if that succeeded and if not force the update of synced/master. Also, it's not clear how to check if the push to master succeeded, since something else might update it further in a race.
+
+I suppose that `git annex sync/merge` could delete the local synced/* branches once it was done merging them. This wouldn't prevent `git pull` from pulling down those branches, though.
+"""]]
diff --git a/doc/forum/annex_merge_creates___34__synced__47____42____34___branches/comment_2_cb6971a766a28bd8c094d0b986272c65._comment b/doc/forum/annex_merge_creates___34__synced__47____42____34___branches/comment_2_cb6971a766a28bd8c094d0b986272c65._comment
new file mode 100644
index 0000000000..5db94a7a00
--- /dev/null
+++ b/doc/forum/annex_merge_creates___34__synced__47____42____34___branches/comment_2_cb6971a766a28bd8c094d0b986272c65._comment
@@ -0,0 +1,11 @@
+[[!comment format=mdwn
+ username="https://www.google.com/accounts/o8/id?id=AItOawlog_5wIICaMcrKTexlFNA6IO6UTp323aE"
+ nickname="Torkaly"
+ subject="comment 2"
+ date="2014-09-17T08:47:03Z"
+ content="""
+Thank you for the answer.
+
+There are no origin/synced/* branches, only a origin/git-annex and the local git-annex branch. What i need git annex merge to do, is to merge the fetched remote git-annex with the local git-annex. There is no need for creating or merging the synced-branches as there are no.
+
+"""]]
diff --git a/doc/forum/annex_merge_creates___34__synced__47____42____34___branches/comment_3_1a0384edd20cc379e53fe7d7f650f7e2._comment b/doc/forum/annex_merge_creates___34__synced__47____42____34___branches/comment_3_1a0384edd20cc379e53fe7d7f650f7e2._comment
new file mode 100644
index 0000000000..cb80c07f9f
--- /dev/null
+++ b/doc/forum/annex_merge_creates___34__synced__47____42____34___branches/comment_3_1a0384edd20cc379e53fe7d7f650f7e2._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="http://joeyh.name/"
+ ip="108.236.230.124"
+ subject="comment 3"
+ date="2014-09-18T17:39:19Z"
+ content="""
+`git annex merge` does not create any synced/* branches. These branches will be pulled down by `git pull` or `git annex sync`.
+"""]]
diff --git a/doc/forum/annex_merge_creates___34__synced__47____42____34___branches/comment_4_79219e920a6beb4bd3265571f59f51cb._comment b/doc/forum/annex_merge_creates___34__synced__47____42____34___branches/comment_4_79219e920a6beb4bd3265571f59f51cb._comment
new file mode 100644
index 0000000000..8e07649484
--- /dev/null
+++ b/doc/forum/annex_merge_creates___34__synced__47____42____34___branches/comment_4_79219e920a6beb4bd3265571f59f51cb._comment
@@ -0,0 +1,36 @@
+[[!comment format=mdwn
+ username="https://www.google.com/accounts/o8/id?id=AItOawlog_5wIICaMcrKTexlFNA6IO6UTp323aE"
+ nickname="Torkaly"
+ subject="comment 4"
+ date="2014-09-19T18:22:36Z"
+ content="""
+delete the *synced/master* branch:
+```
+$ git branch -d synced/master
+Branch synced/master entfernt (war 20ec8b3).
+```
+
+then call *annex merge*:
+```
+$ git annex merge
+merge git-annex ok
+```
+
+check branches:
+```
+$ git branch -a
+  git-annex
+* master
+  synced/master
+  remotes/origin/git-annex
+  remotes/origin/master
+```
+and there is the *synced/master* branch again.
+
+But that's not my problem. My problem was: how to use annex with a central repository.
+I done that by deleting all remote synced/* branches. And now I'm updating the git-annex branch by `git fetch`ing and
+`git annex merge`ing again.
+
+PS: the MD for code blocks is broken
+
+"""]]
diff --git a/doc/forum/annex_merge_creates___34__synced__47____42____34___branches/comment_5_39009651c3f65e2ecd225e17d157c8d8._comment b/doc/forum/annex_merge_creates___34__synced__47____42____34___branches/comment_5_39009651c3f65e2ecd225e17d157c8d8._comment
new file mode 100644
index 0000000000..488f3ae7b3
--- /dev/null
+++ b/doc/forum/annex_merge_creates___34__synced__47____42____34___branches/comment_5_39009651c3f65e2ecd225e17d157c8d8._comment
@@ -0,0 +1,10 @@
+[[!comment format=mdwn
+ username="joey"
+ subject="""comment 5"""
+ date="2015-07-07T19:20:17Z"
+ content="""
+So, I was wrong about `git annex merge` creating the synced/master branch.
+
+There's no good reason for it to do that, so I've fixed it not to create
+it.
+"""]]
diff --git a/doc/forum/annexed_file_key_for_web_remote_with_SHA256E_backend.mdwn b/doc/forum/annexed_file_key_for_web_remote_with_SHA256E_backend.mdwn
new file mode 100644
index 0000000000..6c428e3752
--- /dev/null
+++ b/doc/forum/annexed_file_key_for_web_remote_with_SHA256E_backend.mdwn
@@ -0,0 +1,12 @@
+First off, thanks so much for your hard work, git-annex is amazing.
+
+I just started using the [web as a special remote](http://git-annex.branchable.com/tips/using_the_web_as_a_special_remote/) feature with the SHA256E backend, and I noticed that although the annexed file has the correct backend prefix (SHA256E) it does not have the extension of the file in the URL. The URL is `https://...IMG_1234.JPG` but the annexed file is `SHA256E-...832c99` with no extension.
+
+This is fine for most use cases, but I actually access an S3 remote directly from another app (independent of git-annex) to render photos, and in that app I'm using the extensions to figure out file types, so not having that info is slightly inconvenient.
+
+Is there any way to either:
+
+1. tell git-annex to preserve the extension of a file on the web in the annexed file, or
+2. alternatively, change the annexed filename (add the extension manually) without screwing anything up?
+
+Any help would be much appreciated, thanks!
diff --git a/doc/forum/annexed_file_key_for_web_remote_with_SHA256E_backend/comment_1_d1605a6e3b4d6863f4089218994ce564._comment b/doc/forum/annexed_file_key_for_web_remote_with_SHA256E_backend/comment_1_d1605a6e3b4d6863f4089218994ce564._comment
new file mode 100644
index 0000000000..8b78618a94
--- /dev/null
+++ b/doc/forum/annexed_file_key_for_web_remote_with_SHA256E_backend/comment_1_d1605a6e3b4d6863f4089218994ce564._comment
@@ -0,0 +1,29 @@
+[[!comment format=mdwn
+ username="http://joeyh.name/"
+ nickname="joey"
+ subject="comment 1"
+ date="2013-05-06T15:43:06Z"
+ content="""
+You don't say what version of git-annex you're using. I tested with the current version, 4.20130405:
+
+[[!format sh \"\"\"
+joey@gnu:~/tmp/rr>git annex addurl http://localhost/~joey/header_background.png
+addurl localhost_~joey_header_background.png (downloading http://localhost/~joey/header_background.png ...) --2013-05-06 11:36:41--  http://localhost/~joey/header_background.png
+Resolving localhost (localhost)... ::1, 127.0.0.1, 127.0.1.1
+Connecting to localhost (localhost)|::1|:80... connected.
+HTTP request sent, awaiting response... 200 OK
+Length: 53693 (52K) [image/png]
+Saving to: ‘/home/joey/tmp/rr/.git/annex/tmp/URL--http&c%%localhost%~joey%header_background.png’
+
+100%[======================================>] 53,693      --.-K/s   in 0.001s  
+
+2013-05-06 11:36:41 (58.9 MB/s) - '/home/joey/tmp/rr/.git/annex/tmp/URL--http&c%%localhost%~joey%header_background.png' saved [53693/53693]
+
+(checksum...) ok
+(Recording state in git...)
+joey@gnu:~/tmp/rr>dir
+lrwxrwxrwx   1 joey joey  194 Sep 11  2010 localhost_~joey_header_background.png -> .git/annex/objects/M1/p0/SHA256E-s53693--3f065e8e2db1248765d0753cf483e40ae0eaf9bffe34b0cf738158815d0884e3.png/SHA256E-s53693--3f065e8e2db1248765d0753cf483e40ae0eaf9bffe34b0cf738158815d0884e3.png
+\"\"\"]]
+
+Looks like it's doing the right thing with the extension!
+"""]]
diff --git a/doc/forum/annexed_file_key_for_web_remote_with_SHA256E_backend/comment_2_d249ff27fa3d9ac3ca32485cdef49930._comment b/doc/forum/annexed_file_key_for_web_remote_with_SHA256E_backend/comment_2_d249ff27fa3d9ac3ca32485cdef49930._comment
new file mode 100644
index 0000000000..3d5814bc68
--- /dev/null
+++ b/doc/forum/annexed_file_key_for_web_remote_with_SHA256E_backend/comment_2_d249ff27fa3d9ac3ca32485cdef49930._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="https://www.google.com/accounts/o8/id?id=AItOawnkBYpLu_NOj7Uq0-acvLgWhxF8AUEIJbo"
+ nickname="Chris"
+ subject="comment 2"
+ date="2013-05-07T03:37:24Z"
+ content="""
+Thanks very much! That's great news. I'm using 3.20120629, I'll just update then.
+"""]]
diff --git a/doc/forum/archaeology_of_deleted_files.mdwn b/doc/forum/archaeology_of_deleted_files.mdwn
new file mode 100644
index 0000000000..a4ed66dc81
--- /dev/null
+++ b/doc/forum/archaeology_of_deleted_files.mdwn
@@ -0,0 +1,36 @@
+Earlier this week, I somehow lost a ton of files from my annex -- by switching on the command line from indirect to direct mode while the assistant was running, I think. I'm not sure.
+
+Anyway, by "lost" I mean "lost the symlinks to," because git-annex defaults to keeping content around till you tell it otherwise.  So I still had the content in the repos on my two backup drives.  All I needed was the symlinks back.
+
+But how to figure out exactly what I lost and get it back?
+
+I found that out here:
+
+http://stackoverflow.com/questions/953481/restore-a-deleted-file-in-a-git-repo
+
+Here's a magical formula you can use to find every single file deletion in the history of your repo:
+
+    git log --diff-filter=D --summary 
+
+That will give you every commit that deleted things, and what was deleted.
+
+To bring back all the files deleted in a given commit, where COMMITHASH is the commit hash, use this command:
+
+    git checkout COMMITHASH^1 -- .
+
+to bring back only a specific file:
+
+    git checkout COMMITHASH^1 -- path/to/file.txt
+
+to bring back only a subdirectory:
+
+    git checkout COMMITHASH^1 -- sub/directory
+
+that will bring them back into the staging area.  You can see which ones just reappeared by typing:
+
+    git status
+
+then you can actually make the restore permanent by typing: 
+
+    git commit -m "I just resurrected some files"
+
diff --git a/doc/forum/archaeology_of_deleted_files/comment_1_48f27df03ec18d2c27cf6b70dcf71dc5._comment b/doc/forum/archaeology_of_deleted_files/comment_1_48f27df03ec18d2c27cf6b70dcf71dc5._comment
new file mode 100644
index 0000000000..3647193b85
--- /dev/null
+++ b/doc/forum/archaeology_of_deleted_files/comment_1_48f27df03ec18d2c27cf6b70dcf71dc5._comment
@@ -0,0 +1,10 @@
+[[!comment format=mdwn
+ username="http://edheil.wordpress.com/"
+ ip="99.54.57.201"
+ subject="comment 1"
+ date="2013-01-25T14:07:36Z"
+ content="""
+seems I inadvertently got caught by Markdown -- I tried to write COMMITHASH-hat-1 and it turned into COMMITHASH superscript 1.
+
+Tilde (~) would also have worked instead of hat (^)
+"""]]
diff --git a/doc/forum/archaeology_of_deleted_files/comment_2_c698cd10c8038bac45bd1049506a27c3._comment b/doc/forum/archaeology_of_deleted_files/comment_2_c698cd10c8038bac45bd1049506a27c3._comment
new file mode 100644
index 0000000000..363d320791
--- /dev/null
+++ b/doc/forum/archaeology_of_deleted_files/comment_2_c698cd10c8038bac45bd1049506a27c3._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="https://www.google.com/accounts/o8/id?id=AItOawmBUR4O9mofxVbpb8JV9mEbVfIYv670uJo"
+ nickname="Justin"
+ subject="comment 2"
+ date="2013-01-25T14:27:47Z"
+ content="""
+I fixed it for you
+"""]]
diff --git a/doc/forum/archival_and_multiple_users.mdwn b/doc/forum/archival_and_multiple_users.mdwn
new file mode 100644
index 0000000000..da451d9fb9
--- /dev/null
+++ b/doc/forum/archival_and_multiple_users.mdwn
@@ -0,0 +1,8 @@
+The assistant archival walk-through describes how files get removed from local repo if moved into archive repos.
+What if the files in question aren't needed immediately (and can be archived) for one user of a shared repo, but needed by someone else on their own machine?
+git-annex directory arrangement propagation wouldn't work for this case, it seems.  Could assistant handle this case?
+
+It looks like having a browser of the directory tree in git-annex assistant webpage with say right-click menu with "get" and "drop" would be the most flexible.
+
+Or maybe it could be more about integration with Linux etc file browsers (but then it's system specific, unlike with webapp).
+
diff --git a/doc/forum/archival_and_multiple_users/comment_1_fc4ee256f03a7c189d687caf4a34e21e._comment b/doc/forum/archival_and_multiple_users/comment_1_fc4ee256f03a7c189d687caf4a34e21e._comment
new file mode 100644
index 0000000000..090091a8a6
--- /dev/null
+++ b/doc/forum/archival_and_multiple_users/comment_1_fc4ee256f03a7c189d687caf4a34e21e._comment
@@ -0,0 +1,9 @@
+[[!comment format=mdwn
+ username="http://edheil.wordpress.com/"
+ ip="173.162.44.162"
+ subject="comment 1"
+ date="2013-07-24T15:30:51Z"
+ content="""
+A feature I would love to see is a file manager integrated with the webapp, so that you could browse your files and retrieve or drop them manually as if you were using the command line.  Then you could have your repo set on \"manual\" and not worry about those archive directories.  Just manually drop content if you got low on space.
+
+"""]]
diff --git a/doc/forum/archival_and_multiple_users/comment_2_a96d57d4bb567ac9b0b9167d5b1be011._comment b/doc/forum/archival_and_multiple_users/comment_2_a96d57d4bb567ac9b0b9167d5b1be011._comment
new file mode 100644
index 0000000000..cf97f28fdb
--- /dev/null
+++ b/doc/forum/archival_and_multiple_users/comment_2_a96d57d4bb567ac9b0b9167d5b1be011._comment
@@ -0,0 +1,12 @@
+[[!comment format=mdwn
+ username="http://joeyh.name/"
+ ip="2001:4978:f:21a::2"
+ subject="comment 2"
+ date="2013-07-25T18:42:59Z"
+ content="""
+I believe that the world has enough (or too many) file managers, and do not want to build another one just for git-annex. In any case \"git annex get\" or any GUI wrapping around it is not going to help with the described use case, because the problem is that the files are archived away in my offline drive, while you want to access them in your repository, which does not have access to that drive.
+
+The way to handle this situation is to copy or move the files you want out of the archive directory, and back to the parent directory. This move of the file will be synced from your repository to all the other repositories. When the repository that does have access to the archive notices this, it will get the files that have been moved out of the archive. Normal assistant syncing will then arrange to get them transferred back to the repository of the user who had wanted them.
+
+Of course, if the archive is on an offline drive, this won't happen until that drive is plugged back in.
+"""]]
diff --git a/doc/forum/archival_and_multiple_users/comment_3_bd44634b04732ffb91154c61ef9cf828._comment b/doc/forum/archival_and_multiple_users/comment_3_bd44634b04732ffb91154c61ef9cf828._comment
new file mode 100644
index 0000000000..20b67976e0
--- /dev/null
+++ b/doc/forum/archival_and_multiple_users/comment_3_bd44634b04732ffb91154c61ef9cf828._comment
@@ -0,0 +1,14 @@
+[[!comment format=mdwn
+ username="https://www.google.com/accounts/o8/id?id=AItOawln3ckqKx0x_xDZMYwa9Q1bn4I06oWjkog"
+ nickname="Michael"
+ subject="comment 3"
+ date="2013-07-25T19:16:20Z"
+ content="""
+> The way to handle this situation is to copy or move the files you want out of the archive directory, and back to the parent directory. This move of the file will be synced from your repository to all the other repositories. When the repository that does have access to the archive notices this, it will get the files that have been moved out of the archive. Normal assistant syncing will then arrange to get them transferred back to the repository of the user who had wanted them.
+
+But won't it also transfer the same files into repos of all other users (who didn't want the files)?
+On a related note, allowing moving files for everyone could jeopardise integrity of the master archive.  Is there a way to limit what kind of ops the users can do (e.g. copy but not move)?
+
+Re offline drives, the case I was thinking of is actually where each user has access to e.g. special remotes directly.  So a manual get/put would solve the problem at hand.
+
+"""]]
diff --git a/doc/forum/archival_and_multiple_users/comment_4_b89a56a5f1cd641f87925c7a5f74bcec._comment b/doc/forum/archival_and_multiple_users/comment_4_b89a56a5f1cd641f87925c7a5f74bcec._comment
new file mode 100644
index 0000000000..332e0a0a00
--- /dev/null
+++ b/doc/forum/archival_and_multiple_users/comment_4_b89a56a5f1cd641f87925c7a5f74bcec._comment
@@ -0,0 +1,13 @@
+[[!comment format=mdwn
+ username="http://joeyh.name/"
+ ip="2001:4978:f:21a::2"
+ subject="comment 4"
+ date="2013-07-25T19:23:30Z"
+ content="""
+Moving files shouldn't jepardize anything. If a user makes a change they shouldn't have made, it can be reverted.
+
+If you have the archive drive available locally you can set your repository to manual mode and just use `git annex get` on content you want. Or you could set it to be a backup repository and the assistant would get all content.
+Or write a custom [[preferred_content]] expression for your repository to make it want files in `archive/stillusing/`
+
+
+"""]]
diff --git a/doc/forum/archival_and_multiple_users/comment_5_81293bf5dc8ad4552712c2083fd589c9._comment b/doc/forum/archival_and_multiple_users/comment_5_81293bf5dc8ad4552712c2083fd589c9._comment
new file mode 100644
index 0000000000..b7b93fdf4b
--- /dev/null
+++ b/doc/forum/archival_and_multiple_users/comment_5_81293bf5dc8ad4552712c2083fd589c9._comment
@@ -0,0 +1,19 @@
+[[!comment format=mdwn
+ username="https://www.google.com/accounts/o8/id?id=AItOawmkBwMWvNKZZCge_YqobCSILPMeK6xbFw8"
+ nickname="develop"
+ subject="comment 5"
+ date="2013-07-26T08:20:21Z"
+ content="""
+I think i agree that this should be implemented.
+
+Not only to be able to \"get/drop\" from the assistant without the use of archive folders.
+
+But mainly because i think this will be needed anyways, to do a \"recover previous version of file\" option. Which should definitely be in the web assistant.
+
+We can't expect family and friends to figure out all command line stuff, thus they need to use the assistant. And i really think it would be a shame to not have an easy way to recover old versions, when the old versions are actually stored.
+
+Always think about having a good WAF(Wife acceptance factor).
+
+Just my 2 bits (and a haircut).
+
+"""]]
diff --git a/doc/forum/armhf_binary.mdwn b/doc/forum/armhf_binary.mdwn
new file mode 100644
index 0000000000..442fb121d2
--- /dev/null
+++ b/doc/forum/armhf_binary.mdwn
@@ -0,0 +1,3 @@
+Does a armhf binary tarball exist anywhere? I'm running Ubuntu trusty on a armhf platform (beagleboard), and the repository package is out of date. I might try to get the standalone armel binary working using multiarch, but that seems only slightly less painful than compiling from scratch.
+
+Or am I better off changing to a debian boot image, and be done with it?
diff --git a/doc/forum/armhf_binary/comment_1_9ca7ff6cb1f5dfc1e5ce8527e7e0a45f._comment b/doc/forum/armhf_binary/comment_1_9ca7ff6cb1f5dfc1e5ce8527e7e0a45f._comment
new file mode 100644
index 0000000000..9a1e3a4af5
--- /dev/null
+++ b/doc/forum/armhf_binary/comment_1_9ca7ff6cb1f5dfc1e5ce8527e7e0a45f._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="http://joeyh.name/"
+ ip="209.250.56.7"
+ subject="comment 1"
+ date="2014-08-15T15:58:02Z"
+ content="""
+The standalone armel build should work fine on armhf, assuming that the kernel supports EABI, which I'm pretty sure it does (or multiarch armel would not work).
+"""]]
diff --git a/doc/forum/armhf_binary/comment_2_743ecf263d1ec1bc4f24e32c0a178f2b._comment b/doc/forum/armhf_binary/comment_2_743ecf263d1ec1bc4f24e32c0a178f2b._comment
new file mode 100644
index 0000000000..864d5f23c7
--- /dev/null
+++ b/doc/forum/armhf_binary/comment_2_743ecf263d1ec1bc4f24e32c0a178f2b._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="justinl"
+ ip="184.17.213.135"
+ subject="works"
+ date="2014-09-10T17:59:48Z"
+ content="""
+Yep, the standalone armel build worked perfectly. Thanks!
+"""]]
diff --git a/doc/forum/assistant__44___direct_mode__44___large_files__44___command_line_-_how_do_these_play_together__63__.mdwn b/doc/forum/assistant__44___direct_mode__44___large_files__44___command_line_-_how_do_these_play_together__63__.mdwn
new file mode 100644
index 0000000000..e48bf4cf6b
--- /dev/null
+++ b/doc/forum/assistant__44___direct_mode__44___large_files__44___command_line_-_how_do_these_play_together__63__.mdwn
@@ -0,0 +1,10 @@
+Hi,
+
+I am trying to understand git-annex, and the walkthroughs on the page did help a lot. I *feel* that git-annex is the solution for my use case: having a large archive, several computers, NAS both at work and at home, being offline during commuting, …
+
+Everything seems to be clear for me when using the command line version. Using annex assistant looks promising, and it is easy to create repositories that match my usage. But now I see that these repositories are in direct mode by default; and I am not sure how this impacts my ability to work with large files. I do not want to end up with git (without annex) handling huge files.
+
+So - can somebody explain how direct mode affects this? Can I switch to indirect mode with the assistant? Am I asking the wrong questions? 
+
+thanks,
+Ulli
diff --git a/doc/forum/assistant__44___direct_mode__44___large_files__44___command_line_-_how_do_these_play_together__63__/comment_1_d90d1d599ce557af03c6f0f2ea188212._comment b/doc/forum/assistant__44___direct_mode__44___large_files__44___command_line_-_how_do_these_play_together__63__/comment_1_d90d1d599ce557af03c6f0f2ea188212._comment
new file mode 100644
index 0000000000..d762791da1
--- /dev/null
+++ b/doc/forum/assistant__44___direct_mode__44___large_files__44___command_line_-_how_do_these_play_together__63__/comment_1_d90d1d599ce557af03c6f0f2ea188212._comment
@@ -0,0 +1,10 @@
+[[!comment format=mdwn
+ username="http://joeyh.name/"
+ ip="209.250.56.246"
+ subject="comment 1"
+ date="2013-11-09T17:36:01Z"
+ content="""
+Files in direct mode are stored in the git annex the same way as when using indirect mode. But you're perhaps right to be wary of using direct mode, it's been quite easy to mess up while using it at the command line until very recently when the [[todo/direct_mode_guard]] was implemented.
+
+You can switch repositories created using the assistant from direct to indirect mode if you like. The assistant works in either mode.
+"""]]
diff --git a/doc/forum/assistant__44___direct_mode__44___large_files__44___command_line_-_how_do_these_play_together__63__/comment_2_58b1af497cab132acb28cb5f9283ec2a._comment b/doc/forum/assistant__44___direct_mode__44___large_files__44___command_line_-_how_do_these_play_together__63__/comment_2_58b1af497cab132acb28cb5f9283ec2a._comment
new file mode 100644
index 0000000000..45cec4db56
--- /dev/null
+++ b/doc/forum/assistant__44___direct_mode__44___large_files__44___command_line_-_how_do_these_play_together__63__/comment_2_58b1af497cab132acb28cb5f9283ec2a._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="https://www.google.com/accounts/o8/id?id=AItOawmkXtBdMgE1d9nCz2iBc4f85xh4izZ_auU"
+ nickname="Ulrich"
+ subject="Thanks"
+ date="2013-11-12T08:57:36Z"
+ content="""
+Ok, thanks for the explanation. I still have to figure out what happens when I annex drop files on a direct repository, and how well all this plays with the Finder on Mac…
+"""]]
diff --git a/doc/forum/assistant__44___direct_mode__44___large_files__44___command_line_-_how_do_these_play_together__63__/comment_3_37d4fd8f69e8066b5aa19454b714e443._comment b/doc/forum/assistant__44___direct_mode__44___large_files__44___command_line_-_how_do_these_play_together__63__/comment_3_37d4fd8f69e8066b5aa19454b714e443._comment
new file mode 100644
index 0000000000..71a0919303
--- /dev/null
+++ b/doc/forum/assistant__44___direct_mode__44___large_files__44___command_line_-_how_do_these_play_together__63__/comment_3_37d4fd8f69e8066b5aa19454b714e443._comment
@@ -0,0 +1,11 @@
+[[!comment format=mdwn
+ username="https://www.google.com/accounts/o8/id?id=AItOawmkXtBdMgE1d9nCz2iBc4f85xh4izZ_auU"
+ nickname="Ulrich"
+ subject="So what does "git annex drop" do in direct mode?"
+ date="2013-11-15T15:07:57Z"
+ content="""
+I just tried a git annex drop on a file in a direct mode repository, and that just did not change anything at all, as far as I can see. Actually I don't know what to expect, but doesn't that mean that I should have to switch to indirect mode in order to be able to drop large files to save space? 
+
+Funny is that git annex drop did not complain at all but just reported \"ok\" after quite a while. And when I tried to drop a file that was not available in any other repository, it failed (which is expected behavior). So what is it what was \"ok\"?
+
+"""]]
diff --git a/doc/forum/assistant__44___direct_mode__44___large_files__44___command_line_-_how_do_these_play_together__63__/comment_4_a974e2105774d4f82ad286ff0792ba84._comment b/doc/forum/assistant__44___direct_mode__44___large_files__44___command_line_-_how_do_these_play_together__63__/comment_4_a974e2105774d4f82ad286ff0792ba84._comment
new file mode 100644
index 0000000000..587d704d38
--- /dev/null
+++ b/doc/forum/assistant__44___direct_mode__44___large_files__44___command_line_-_how_do_these_play_together__63__/comment_4_a974e2105774d4f82ad286ff0792ba84._comment
@@ -0,0 +1,10 @@
+[[!comment format=mdwn
+ username="http://joeyh.name/"
+ ip="209.250.56.246"
+ subject="comment 4"
+ date="2013-11-15T20:33:47Z"
+ content="""
+Drop in direct mode does the same thing as in indirect mode.
+
+If you are using git-annex on a crippled filesystem without symlinks, git-annex cannot represent a file whose content has been dropped using a broken symlink, so it instead represents it with a nearly empty file.
+"""]]
diff --git a/doc/forum/assistant_and_archive_folder.mdwn b/doc/forum/assistant_and_archive_folder.mdwn
new file mode 100644
index 0000000000..11f405736f
--- /dev/null
+++ b/doc/forum/assistant_and_archive_folder.mdwn
@@ -0,0 +1,13 @@
+I have a setup with a windows desktop and a linux laptop with both an annex repository, interconnected via a backup repository in the middle (connected via ssh).
+Both computers are setup as clients and all 3 machines are running the assistant.
+
+Today i tried creating an archive folder and a putting some files in it. To my understanding those files should have been dropped in the 2 machines but kept on the server.. instead I have them copied on each machine. Listing where the files are on both computers shows that both know that themselves and the server have a copy, but they don't know anything about each other.
+
+AFAIK it looks like the bug reported in https://git-annex.branchable.com/forum/question_about_assistant_and___47__archive__47__/
+
+laptop: git-annex version: 5.20151116+gitg5416a1a-1~ndall+1
+
+server: git-annex version: 5.20151109-g08bb3b1
+
+windows: the latest build on the website
+
diff --git a/doc/forum/assistant_and_archive_folder/comment_1_ab8808372b886fd1a9f55588d1cef602._comment b/doc/forum/assistant_and_archive_folder/comment_1_ab8808372b886fd1a9f55588d1cef602._comment
new file mode 100644
index 0000000000..c567bc3038
--- /dev/null
+++ b/doc/forum/assistant_and_archive_folder/comment_1_ab8808372b886fd1a9f55588d1cef602._comment
@@ -0,0 +1,12 @@
+[[!comment format=mdwn
+ username="joey"
+ subject="""comment 1"""
+ date="2016-04-20T16:53:04Z"
+ content="""
+How many copies have you configured git-annex to keep of each file?
+
+If it's 1 copy, the clients should be able to contact the server, see that
+it has the file, and drop it as it's in the achive folder. But if 2 copies
+are needed, neither client can verify that the other client has a copy, so
+both will hold onto their own copy for safety.
+"""]]
diff --git a/doc/forum/assistant_created_encrypted__backup_remote__58___Howto_restore__63__.mdwn b/doc/forum/assistant_created_encrypted__backup_remote__58___Howto_restore__63__.mdwn
new file mode 100644
index 0000000000..c18dfb6ca8
--- /dev/null
+++ b/doc/forum/assistant_created_encrypted__backup_remote__58___Howto_restore__63__.mdwn
@@ -0,0 +1,38 @@
+Hi,
+
+I have a question about encryption keys. 
+Basically I do not know how to use the data which was copied in case my local
+machine dies?
+
+
+## I have done the following:
+---
+1. copied my ssh public key to a remote server
+1. created a directory and started git annex assistant
+ 
+   mkdir ~/test_annex
+
+   cd ~/test_annex
+
+   git annex webapp
+
+1. clicked Repository->Add another local repository
+1. Assistant: Where do you want to put this new repository? Me: ~/test_annex "Make Repository"
+1. Assistant "Combine repositories?" Me: "Keep repositories separate"
+1. clicked "Add another repository" clicked "remote server"
+1. filled out "Adding a remote server using ssh" form and clicked "Check this server"
+1. choose "Use an encrypted rsync repository on the server"
+1. Assistant "Repository created" I select "Full backup" as repository group.
+
+So far so good. If I now add a file on my local machine into ~/test_annex something not human readable is created on the remote machine.
+
+What I do not understand is how I use this backup later. I am expecting something along the "hybrid encryption keys" scheme
+explained in this wiki. However I was not able to determine which of my gpg keys was used or how the data was encrypted.
+
+So my question is: "How do I use the encrypted backup remote on a second machine?"
+
+
+
+
+
+
diff --git a/doc/forum/assistant_created_encrypted__backup_remote__58___Howto_restore__63__/comment_1_70200f871b9df49261f32752a6bb0e67._comment b/doc/forum/assistant_created_encrypted__backup_remote__58___Howto_restore__63__/comment_1_70200f871b9df49261f32752a6bb0e67._comment
new file mode 100644
index 0000000000..9d457a9d2d
--- /dev/null
+++ b/doc/forum/assistant_created_encrypted__backup_remote__58___Howto_restore__63__/comment_1_70200f871b9df49261f32752a6bb0e67._comment
@@ -0,0 +1,14 @@
+[[!comment format=mdwn
+ username="http://joeyh.name/"
+ ip="209.250.56.47"
+ subject="comment 1"
+ date="2013-10-27T21:19:45Z"
+ content="""
+When you create an encrypted rsync repository using the webapp like that, its encryption key is stored in your git repository, using the [[shared encryption scheme|encryption#index2h2]]. No gpg key needs to be used to decrypt files from the rsync repository; anyone with a clone of your git repository can do so.  This has its plusses and its minuses; the webapp picks that type of encryption because it's easy to use.
+
+So, the answer is to just make a clone of your repository on the other machine, and then you can use it. There are lots of ways to do that; if you stay in the webapp, go to Add Another Repisitory and any of the \"Share with your other devices\", \"Share with a friend\", or \"Local computer\" options are easy ways to do it.
+
+----
+
+Now, if you had, manually, set up a rsync repository encrypted with the [[hybrid encryption key scheme|encryption#index1h2]], to access it from another computer with clone of the repository you would need to have a gpg key that has been given access to the repository. So you would either copy your gpg secret key to the other computer, or if you don't want to trust that other computer with the your main gpg key, you could make another gpg secret key for that computer, and add that key as one of the keys that can access the encrypted repository. (Or, if the computer belonged to a friend, you could just get their gpg key, and add it.)
+"""]]
diff --git a/doc/forum/assistant_created_encrypted__backup_remote__58___Howto_restore__63__/comment_2_173da510b45f0320ae8aa2df9f14ae7b._comment b/doc/forum/assistant_created_encrypted__backup_remote__58___Howto_restore__63__/comment_2_173da510b45f0320ae8aa2df9f14ae7b._comment
new file mode 100644
index 0000000000..dbf2178f5b
--- /dev/null
+++ b/doc/forum/assistant_created_encrypted__backup_remote__58___Howto_restore__63__/comment_2_173da510b45f0320ae8aa2df9f14ae7b._comment
@@ -0,0 +1,11 @@
+[[!comment format=mdwn
+ username="Frank"
+ ip="31.19.114.93"
+ subject="Comment 2"
+ date="2013-11-07T19:45:16Z"
+ content="""
+Thank you very much for the explanation! Have fun!
+
+
+
+"""]]
diff --git a/doc/forum/assistant_created_encrypted__backup_remote__58___Howto_restore__63__/comment_3_1ecea12a4be5ad09013cddb62df6ab20._comment b/doc/forum/assistant_created_encrypted__backup_remote__58___Howto_restore__63__/comment_3_1ecea12a4be5ad09013cddb62df6ab20._comment
new file mode 100644
index 0000000000..aaf3735899
--- /dev/null
+++ b/doc/forum/assistant_created_encrypted__backup_remote__58___Howto_restore__63__/comment_3_1ecea12a4be5ad09013cddb62df6ab20._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="https://me.yahoo.com/a/W6vhceUml87q0GB9MGH1qEeteW8-#ee26a"
+ nickname="Leandro"
+ subject="comment 3"
+ date="2014-05-17T17:39:04Z"
+ content="""
+So, just to clarify things a little bit: Suppose that you have a remote at box.com (shared encrypted), then the CEO at box.com would eventually be able to decrypt your files easily since the keys are store in the git repository without encryption. Is that right? 
+"""]]
diff --git a/doc/forum/assistant_created_encrypted__backup_remote__58___Howto_restore__63__/comment_4_af4bc222d1479482bd83952353c97f05._comment b/doc/forum/assistant_created_encrypted__backup_remote__58___Howto_restore__63__/comment_4_af4bc222d1479482bd83952353c97f05._comment
new file mode 100644
index 0000000000..7b7b86c306
--- /dev/null
+++ b/doc/forum/assistant_created_encrypted__backup_remote__58___Howto_restore__63__/comment_4_af4bc222d1479482bd83952353c97f05._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="http://joeyh.name/"
+ ip="216.145.95.162"
+ subject="comment 4"
+ date="2014-05-19T15:38:14Z"
+ content="""
+No, box.com cannot decrypt your files, because the git repository is not stored on box.com. 
+"""]]
diff --git a/doc/forum/assistant_created_encrypted__backup_remote__58___Howto_restore__63__/comment_5_c1d247fa128c0a0fc899284f5f95002c._comment b/doc/forum/assistant_created_encrypted__backup_remote__58___Howto_restore__63__/comment_5_c1d247fa128c0a0fc899284f5f95002c._comment
new file mode 100644
index 0000000000..474bd6673b
--- /dev/null
+++ b/doc/forum/assistant_created_encrypted__backup_remote__58___Howto_restore__63__/comment_5_c1d247fa128c0a0fc899284f5f95002c._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="https://www.google.com/accounts/o8/id?id=AItOawmK0703vNSIQsP1mGf-4MAPnsBZiSc6yVo"
+ nickname="Emre"
+ subject="Understanding"
+ date="2014-09-19T21:24:19Z"
+ content="""
+If box does not have the encryption keys in this \"shared encryption\" scenario and if you had only your computer and this remote repo, does that mean losing your computer (ie your git repository) would mean also losing access to those encrypted content? So, actually an encrypted remote, even if marked as full backup, is not actually a backup unless you have a 3rd computer that has the same git repo, in the case of losing your original computer or accidentally wiping it...
+"""]]
diff --git a/doc/forum/assistant_created_encrypted__backup_remote__58___Howto_restore__63__/comment_6_cf877a3502802492cd2bc3012cb2d779._comment b/doc/forum/assistant_created_encrypted__backup_remote__58___Howto_restore__63__/comment_6_cf877a3502802492cd2bc3012cb2d779._comment
new file mode 100644
index 0000000000..3765a67cc6
--- /dev/null
+++ b/doc/forum/assistant_created_encrypted__backup_remote__58___Howto_restore__63__/comment_6_cf877a3502802492cd2bc3012cb2d779._comment
@@ -0,0 +1,12 @@
+[[!comment format=mdwn
+ username="http://joeyh.name/"
+ ip="209.250.56.144"
+ subject="comment 6"
+ date="2014-09-25T15:59:34Z"
+ content="""
+Right.
+
+So, I think I should go change the description displayed by the webapp to \"full backup (file contents only)\" and \"full backup (entire git repository)\" or so. It's a little hard to word it precisely without making it hard to understand.
+
+Or, the webapp could display a nudge to make a clone when no other clones of the git repository exist. I think that's probably more valuable, so [[todo_added|todo/webapp_nudge_when_less_than_numcopies_clones]].
+"""]]
diff --git a/doc/forum/assistant_overzealously_moving_stuff_to_other_repos.mdwn b/doc/forum/assistant_overzealously_moving_stuff_to_other_repos.mdwn
new file mode 100644
index 0000000000..0cf3f2726a
--- /dev/null
+++ b/doc/forum/assistant_overzealously_moving_stuff_to_other_repos.mdwn
@@ -0,0 +1,5 @@
+Debian Squeeze, git version 1.7.10.4, git-annex version 3.20121211
+
+The machine has a clone of the annex with preferred content string: `present or include=calibre/* or include=img/* or include=mail/* or include=music/* or include=sounds/*`
+
+This, I believe, should mean the assistant should never drop anything.  However for the past two days it's been moving files to an encrypted rsync remote so that there are two copies (there is another copy on an external HDD) and then it's dropping them from the current annex.  I want to keep the files here; can anyone think of any reason why they would be moved away?
diff --git a/doc/forum/assistant_overzealously_moving_stuff_to_other_repos/comment_1_6bd240edf1868615024ff11c24c3d52c._comment b/doc/forum/assistant_overzealously_moving_stuff_to_other_repos/comment_1_6bd240edf1868615024ff11c24c3d52c._comment
new file mode 100644
index 0000000000..ad4b23f29a
--- /dev/null
+++ b/doc/forum/assistant_overzealously_moving_stuff_to_other_repos/comment_1_6bd240edf1868615024ff11c24c3d52c._comment
@@ -0,0 +1,13 @@
+[[!comment format=mdwn
+ username="http://edheil.wordpress.com/"
+ ip="173.162.44.162"
+ subject="comment 1"
+ date="2012-12-19T17:19:28Z"
+ content="""
+I had similar issues when I tried to use \"present\" settings a while back, except it was with adding files.  I would add a file and then it would be immediately dropped (because it was not yet \"present\" while it was being added?).  I gave up and just started using \"archive\" subdirectories for things I didn't want to be present.  Even so I'm having some confusing issues with the assistant.  It keeps dropping certain files which are in \"backup\" repositories and therefore should never have anything dropped.  I haven't isolated what's going on to the degree I can get a good bug report together.  But I'm having a hard time trusting assistant and its content settings to do what I want.  I have a couple scripts that do things like:
+
+git annex copy --not --in=usbdrive --to=usbdrive
+
+to populate my backup repos, and these seem much more reliable than the assistant.
+
+"""]]
diff --git a/doc/forum/assistant_overzealously_moving_stuff_to_other_repos/comment_2_37c5e9a7669b5b94fbadb8792a765316._comment b/doc/forum/assistant_overzealously_moving_stuff_to_other_repos/comment_2_37c5e9a7669b5b94fbadb8792a765316._comment
new file mode 100644
index 0000000000..58292aa14b
--- /dev/null
+++ b/doc/forum/assistant_overzealously_moving_stuff_to_other_repos/comment_2_37c5e9a7669b5b94fbadb8792a765316._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="http://joeyh.name/"
+ ip="4.153.8.117"
+ subject="comment 2"
+ date="2012-12-19T18:20:50Z"
+ content="""
+I've found and fixed a bug in transfer log parsing that could cause the assistant to get confused about the name of the file it had just transferred, and think it was not preferred content and drop it. I think this was causing both problems reported above.
+"""]]
diff --git a/doc/forum/assistant_overzealously_moving_stuff_to_other_repos/comment_3_87aa4c5942929be81ddc1e2795d56f0e._comment b/doc/forum/assistant_overzealously_moving_stuff_to_other_repos/comment_3_87aa4c5942929be81ddc1e2795d56f0e._comment
new file mode 100644
index 0000000000..f4d6240bce
--- /dev/null
+++ b/doc/forum/assistant_overzealously_moving_stuff_to_other_repos/comment_3_87aa4c5942929be81ddc1e2795d56f0e._comment
@@ -0,0 +1,9 @@
+[[!comment format=mdwn
+ username="http://edheil.wordpress.com/"
+ ip="173.162.44.162"
+ subject="comment 3"
+ date="2012-12-20T00:05:08Z"
+ content="""
+Thanks, I'll give it another shot!
+
+"""]]
diff --git a/doc/forum/assistant_periodical_unused_moves.mdwn b/doc/forum/assistant_periodical_unused_moves.mdwn
new file mode 100644
index 0000000000..5725f8d236
--- /dev/null
+++ b/doc/forum/assistant_periodical_unused_moves.mdwn
@@ -0,0 +1,11 @@
+I am trying a setup to move files from/to work using a usb key that also has a acopy of the whole repository (if I have to access it from somewhere else without git-annex).
+
+At home there's a direct client repository, the usb is an indirect incrementalbackup and at work I have a indirect backup and a direct client repositories.
+
+
+What I am trying to obtain is to have a lightweight repository at home that doesn't hold backups, and the usb key to also move the unused files from home to work as a backup.
+I am trying to make the assistant do it for me and as far as I could read the assistant should periodically move unused files out of repositories of the group client to the (incremental)backup ones.
+
+
+Having the assistant running on home repository and the two work repositories should do the trick? considering that the usb key doesn't spend much time connected to a computer I have no idea how the assistant would do the periodcal move.
+Also i hope i got it right but files from incrementalbackup should be moved to backup repositories when they can be synchronized, but again, I don't know if a 5 minutes mount with the assistant running on the backup repository will actually do it.
diff --git a/doc/forum/assistant_periodical_unused_moves/comment_1_98e30e1f3ca63790c4a8ce11d29634ae._comment b/doc/forum/assistant_periodical_unused_moves/comment_1_98e30e1f3ca63790c4a8ce11d29634ae._comment
new file mode 100644
index 0000000000..275c5e85f8
--- /dev/null
+++ b/doc/forum/assistant_periodical_unused_moves/comment_1_98e30e1f3ca63790c4a8ce11d29634ae._comment
@@ -0,0 +1,14 @@
+[[!comment format=mdwn
+ username="joey"
+ subject="""comment 1"""
+ date="2015-07-06T19:36:22Z"
+ content="""
+The assistant checks once per day for unused files, and tries to
+transfer them to remotes at that point. Exactly when this runs varies,
+just once per day sometime. It shouldn't much matter when it runs;
+if the drive is not connected at the time, the transfers will fail 
+and be remembered to be retried later next time it gets connected.
+
+The webapp will detect if too many unused files are piling up
+and will display a message about it.
+"""]]
diff --git a/doc/forum/assistant_without_watch__63__.mdwn b/doc/forum/assistant_without_watch__63__.mdwn
new file mode 100644
index 0000000000..34aa3e4415
--- /dev/null
+++ b/doc/forum/assistant_without_watch__63__.mdwn
@@ -0,0 +1,13 @@
+"watch" is described as
+
+> With this running as a daemon in the background, you no longer need to manually run git commands  when  manipulating your files.
+
+and "assistant" is described as
+
+> Like watch, but also automatically syncs changes to other remotes.
+
+I would like the behaviour of assistant, without the watch part:
+
+I have an archive of large files, which I think would be useful to manage manually using git.  But I don't want to manually enforce the "numcopies=" requirement, playing with the assistant and webapp it seems really nice to have it take care of that.
+
+Is there currently an "assistant-without-watch" option?   If not, is it planned?
diff --git a/doc/forum/assistant_without_watch__63__/comment_1_be1f7c038426e53209a85ae1119269d5._comment b/doc/forum/assistant_without_watch__63__/comment_1_be1f7c038426e53209a85ae1119269d5._comment
new file mode 100644
index 0000000000..dc64ef7d2d
--- /dev/null
+++ b/doc/forum/assistant_without_watch__63__/comment_1_be1f7c038426e53209a85ae1119269d5._comment
@@ -0,0 +1,15 @@
+[[!comment format=mdwn
+ username="http://joeyh.name/"
+ ip="4.153.253.75"
+ subject="comment 1"
+ date="2013-02-22T22:31:29Z"
+ content="""
+Not quite. 
+
+Recent versions allow you to disable the automatic committing while the assistant is running by setting annex.autocommit=false .. or just by going into the webapp repository list and pausing syncing for the \"here\" repository.
+
+But, with automatic committing disabled, the assistant doesn't know when new files are added, so will not transfer them.  The rest of it still works, so for example if another clone of the repository makes changes
+the assistant will merge them in and download the new files.
+
+To transfer new files you created, you could use `git annex copy --auto --to someremote`, which will only copy them if needed to satisfy numcopies.
+"""]]
diff --git a/doc/forum/assitant__58___special_remote_server___40__needs_ssh_tunnel__41__.mdwn b/doc/forum/assitant__58___special_remote_server___40__needs_ssh_tunnel__41__.mdwn
new file mode 100644
index 0000000000..45c43f4434
--- /dev/null
+++ b/doc/forum/assitant__58___special_remote_server___40__needs_ssh_tunnel__41__.mdwn
@@ -0,0 +1,10 @@
+To connect to a remote server I have to tunnel it through a server I have access to. My .ssh/config for this connection looks like this:
+
+
+    Host server-at-home
+        HostName foo.bar.de
+        User foo
+        Port 222
+        ProxyCommand /usr/bin/ssh me@server-i-have-access-to /bin/nc -w 3700 %h %p
+
+Is it possible to create a connection using a similar configuration via the git-annex assistant?
diff --git a/doc/forum/assitant__58___special_remote_server___40__needs_ssh_tunnel__41__/comment_1_817a3ed424e4fb76fcd33295f2953250._comment b/doc/forum/assitant__58___special_remote_server___40__needs_ssh_tunnel__41__/comment_1_817a3ed424e4fb76fcd33295f2953250._comment
new file mode 100644
index 0000000000..b3e495dc22
--- /dev/null
+++ b/doc/forum/assitant__58___special_remote_server___40__needs_ssh_tunnel__41__/comment_1_817a3ed424e4fb76fcd33295f2953250._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="https://me.yahoo.com/a/FHnTlSBo1eCGJRwueeKeB6.RCaPbGMPr5jxx8A--#ce0d8"
+ nickname="Hamza"
+ subject="comment 1"
+ date="2013-11-28T22:08:04Z"
+ content="""
+You can add/clone from the remote from the command line and webapp will use it. 
+"""]]
diff --git a/doc/forum/autobuilders_for_git-annex_to_aid_development.mdwn b/doc/forum/autobuilders_for_git-annex_to_aid_development.mdwn
new file mode 100644
index 0000000000..2c1280e514
--- /dev/null
+++ b/doc/forum/autobuilders_for_git-annex_to_aid_development.mdwn
@@ -0,0 +1,34 @@
+This is a continuation of the conversation from [[the comments|design/assistant/#comment-77e54e7ebfbd944c370173014b535c91]] section in the design of git-assistant. In summary, I've setup an auto builder which should help [[Joey]] have an easier time developing on git-annex on non-linux/debian platforms. This builder is currently running on OSX 10.7 with the 64bit version of Haskell Platform.
+
+The builder output can be found at , the CGI on this site does not work as my OSX workstation is pushing the output from another location.
+
+The builder currently tries to build all branches except 
+
+* debian-stable
+* pristine-tar
+* setup
+
+It also does not build any of the tags as well, Joey had suggested to ignore the bpo named tags, but for now it's easier for me to not build any tags. To continue on this discussion, if anyone wants to setup a gitbuilder instance, here is the build.sh script that I am using.
+
+
    +#!/bin/bash -x
+
+# Macports
+export PATH=/opt/local/bin:$PATH
+
+# Haskell userland
+export PATH=$PATH:$HOME/.cabal/bin
+
+# Macports gnu
+export PATH=/opt/local/libexec/gnubin:$PATH
+
+make || exit 3
+
+make -q test
+if [ "$?" = 1 ]; then
+        # run "make test", but give it a time limit in case a test gets stuck
+        ../maxtime 1800 make test || exit 4
+fi
+
    
+
+It's also using the branches-local script for sorting and prioritising the branches to build, this branches-local script can be found at the [autobuild-ceph](https://github.com/ceph/autobuild-ceph/blob/master/branches-local) repository. If there are other people interested in setting up their own instances of gitbuilder for git-annex, please let me know and I will setup an aggregator page to collect status of the builds. The builder runs and updates on a very regular basis.
diff --git a/doc/forum/autobuilders_for_git-annex_to_aid_development/comment_1_7e88f815e8d9652ef18ea6d54b118962._comment b/doc/forum/autobuilders_for_git-annex_to_aid_development/comment_1_7e88f815e8d9652ef18ea6d54b118962._comment
new file mode 100644
index 0000000000..82380cd602
--- /dev/null
+++ b/doc/forum/autobuilders_for_git-annex_to_aid_development/comment_1_7e88f815e8d9652ef18ea6d54b118962._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="https://www.google.com/accounts/o8/id?id=AItOawkSq2FDpK2n66QRUxtqqdbyDuwgbQmUWus"
+ nickname="Jimmy"
+ subject="comment 1"
+ date="2012-07-02T16:25:55Z"
+ content="""
+I've some binaries for OSX which can be found at  its just the master branch, and it's built on a system that runs macports. Binaries are built and updated whenever there are changes made to the master branch of git-annex.
+"""]]
diff --git a/doc/forum/autobuilders_for_git-annex_to_aid_development/comment_2_fef17a10226af5671495c2929653c337._comment b/doc/forum/autobuilders_for_git-annex_to_aid_development/comment_2_fef17a10226af5671495c2929653c337._comment
new file mode 100644
index 0000000000..da3cd6e671
--- /dev/null
+++ b/doc/forum/autobuilders_for_git-annex_to_aid_development/comment_2_fef17a10226af5671495c2929653c337._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="https://www.google.com/accounts/o8/id?id=AItOawkSq2FDpK2n66QRUxtqqdbyDuwgbQmUWus"
+ nickname="Jimmy"
+ subject="comment 2"
+ date="2012-09-29T09:01:11Z"
+ content="""
+I'm currently playing with the autobuilder to get it to do more sensible things so it may be broken for a day or two
+"""]]
diff --git a/doc/forum/avoid_rehashing_when_converting_existing_backups_into_new_remotes.mdwn b/doc/forum/avoid_rehashing_when_converting_existing_backups_into_new_remotes.mdwn
new file mode 100644
index 0000000000..34bae17fa5
--- /dev/null
+++ b/doc/forum/avoid_rehashing_when_converting_existing_backups_into_new_remotes.mdwn
@@ -0,0 +1,3 @@
+I have two copies of a large collection of photos on two computers. I have been using rsync to keep them synchronized, but I would like to use git annex instead. I finished initializing one repo, and now its time to initialize the other repo before I can sync them up. But it seems redundant that the other repo will compute its own hashes although I know that the two contents are identical. Can't I just copy the keys and tell git annex to assume that any files seen have already been hashed?
+
+As a start, I cloned the .git from the first machine to the second. But that wasn't enough ... I now have an empty git-annex repo in the second machine. What else is missing?
diff --git a/doc/forum/avoid_rehashing_when_converting_existing_backups_into_new_remotes/comment_1_7d0b479244fefd933193d921adcd897c._comment b/doc/forum/avoid_rehashing_when_converting_existing_backups_into_new_remotes/comment_1_7d0b479244fefd933193d921adcd897c._comment
new file mode 100644
index 0000000000..538a070edb
--- /dev/null
+++ b/doc/forum/avoid_rehashing_when_converting_existing_backups_into_new_remotes/comment_1_7d0b479244fefd933193d921adcd897c._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="https://me.yahoo.com/a/ZF7p46cPmpWtb9zvA8iTitPmiQ--#eb014"
+ nickname="Jason"
+ subject="I'm also having this issue"
+ date="2015-12-14T03:17:58Z"
+ content="""
+I have the exact same issue: two copies of photos and videos that I have been using rsync to maintain. Converted one disk to git-annex, cloned the repo and moved .git/ into the second but can't seem to find a fast way to move the files to the correct hash id in the annex dir and create the symlinks. I'm thinking it might be best to just do a `git init; git annex init; git annex add .; git commit` then add the remotes in both directions and \"sync\".
+"""]]
diff --git a/doc/forum/avoid_rehashing_when_converting_existing_backups_into_new_remotes/comment_2_d42222ef94f4d409ac40e523f8211ffc._comment b/doc/forum/avoid_rehashing_when_converting_existing_backups_into_new_remotes/comment_2_d42222ef94f4d409ac40e523f8211ffc._comment
new file mode 100644
index 0000000000..bfd8066ace
--- /dev/null
+++ b/doc/forum/avoid_rehashing_when_converting_existing_backups_into_new_remotes/comment_2_d42222ef94f4d409ac40e523f8211ffc._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="https://me.yahoo.com/a/ZF7p46cPmpWtb9zvA8iTitPmiQ--#eb014"
+ nickname="Jason"
+ subject="It worked"
+ date="2015-12-14T04:46:46Z"
+ content="""
+The above process just completed. Of course it took a while to hash and create symlinks for the 20K+ files. The initial sync caused an expected \"Warning: no common commits\" however since the symlinks all pointed at the exact same hashed filenames there was nothing to merge. The sync did take several minutes, but it was much shorter than copying 120GB. I hope this helps someone else in the future.
+"""]]
diff --git a/doc/forum/avoid_rehashing_when_converting_existing_backups_into_new_remotes/comment_3_dc93ae2a58de5620c440531f43f1237d._comment b/doc/forum/avoid_rehashing_when_converting_existing_backups_into_new_remotes/comment_3_dc93ae2a58de5620c440531f43f1237d._comment
new file mode 100644
index 0000000000..539f574f6a
--- /dev/null
+++ b/doc/forum/avoid_rehashing_when_converting_existing_backups_into_new_remotes/comment_3_dc93ae2a58de5620c440531f43f1237d._comment
@@ -0,0 +1,25 @@
+[[!comment format=mdwn
+ username="joey"
+ subject="""comment 3"""
+ date="2015-12-19T18:09:26Z"
+ content="""
+I'd say that method, or any similar set of steps, is the typical way to
+handle this.
+
+Sure, everything gets hashed twice. This is unlikely to waste enough time
+to make it worthwhile to develop a hack that only hashes once.
+
+If you really want to develop such a hack, the plumbing command that you
+can use to make it happen is `git annex setkey`. So, you'd add all the
+files to the first repository, and then use `git-annex find
+--format="${key} ${file}"` to list all the files and the keys that resulted
+from hashing them. Then in the second repository, you'd use that list to
+run `git annex setkey` and force the files into the annex without
+hashing them.
+
+This will probably turn out to be slower than just re-hashing the files
+would be, since you'll have to run `git annex setkey` once per file.
+Adding a `--batch` option that reads from stdin would probably be called
+for to get it fast enough to bother with. Although passing `-c
+annex.alwayscommit=false` might speed it up enough.
+"""]]
diff --git a/doc/forum/avoid_rehashing_when_converting_existing_backups_into_new_remotes/comment_4_e6bb8a33608908d02eeb921d039f22b9._comment b/doc/forum/avoid_rehashing_when_converting_existing_backups_into_new_remotes/comment_4_e6bb8a33608908d02eeb921d039f22b9._comment
new file mode 100644
index 0000000000..8560857a7a
--- /dev/null
+++ b/doc/forum/avoid_rehashing_when_converting_existing_backups_into_new_remotes/comment_4_e6bb8a33608908d02eeb921d039f22b9._comment
@@ -0,0 +1,9 @@
+[[!comment format=mdwn
+ username="CandyAngel"
+ subject="comment 4"
+ date="2015-12-23T12:46:36Z"
+ content="""
+> This is unlikely to waste enough time to make it worthwhile to develop a hack that only hashes once.
+
+In my super-heavy use case, the second hashing of the files is *dwarfed* by the 45 minute wait for git to update .git/index, so I would agree with this.
+"""]]
diff --git a/doc/forum/bainstorming__58___git_annex_push___38___pull.mdwn b/doc/forum/bainstorming__58___git_annex_push___38___pull.mdwn
new file mode 100644
index 0000000000..8a6c552b80
--- /dev/null
+++ b/doc/forum/bainstorming__58___git_annex_push___38___pull.mdwn
@@ -0,0 +1,28 @@
+Wouldn't it make sense to offer
+
+    git annex pull
+
+which would basically do
+
+    git pull
+    git annex get
+
+and
+
+    git annex push
+
+which would do
+
+    git annex commit .
+    git annex put # (the proposed "send to default annex" command)
+    git commit -a -m "$HOST $(date +%F-%H-%M-%S)" # or similar
+    git push
+
+Resulting in commands that are totally analogous to git push & pull: Sync all data from/to a remote.
+
+> Update:
+
+This is useful:
+
+    git config [--global] alias.annex-push '!git pull && git annex add . && git annex copy . --to $REMOTE --fast --quiet && git commit -a -m "$HOST $(date +%F--%H-%M-%S-%Z)" && git push'
+
diff --git a/doc/forum/bainstorming__58___git_annex_push___38___pull/comment_1_3a0bf74b51586354b7a91f8b43472376._comment b/doc/forum/bainstorming__58___git_annex_push___38___pull/comment_1_3a0bf74b51586354b7a91f8b43472376._comment
new file mode 100644
index 0000000000..3d69e8f290
--- /dev/null
+++ b/doc/forum/bainstorming__58___git_annex_push___38___pull/comment_1_3a0bf74b51586354b7a91f8b43472376._comment
@@ -0,0 +1,11 @@
+[[!comment format=mdwn
+ username="http://joey.kitenet.net/"
+ nickname="joey"
+ subject="comment 1"
+ date="2011-04-05T18:05:00Z"
+ content="""
+Maybe, otoh, part of the point of git-annex is that the data may be too large to pull down all of it.
+
+I find mr useful as a policy layer over top of git-annex, so \"mr update\" can pull down appropriate quantities of data from
+appropriate locations.
+"""]]
diff --git a/doc/forum/bainstorming__58___git_annex_push___38___pull/comment_2_b02ca09914e788393c01196686f95831._comment b/doc/forum/bainstorming__58___git_annex_push___38___pull/comment_2_b02ca09914e788393c01196686f95831._comment
new file mode 100644
index 0000000000..e0ecc1a819
--- /dev/null
+++ b/doc/forum/bainstorming__58___git_annex_push___38___pull/comment_2_b02ca09914e788393c01196686f95831._comment
@@ -0,0 +1,14 @@
+[[!comment format=mdwn
+ username="https://www.google.com/accounts/o8/id?id=AItOawl9sYlePmv1xK-VvjBdN-5doOa_Xw-jH4U"
+ nickname="Richard"
+ subject="comment 2"
+ date="2011-04-05T20:52:52Z"
+ content="""
+No-so-subtle sarcasm taken and acknowledged :)
+
+Arguably, git-annex should know about any local limits and not have them implemented via mr from the outside. I guess my concern boils down to having git-annex do the right thing all by itself with minimal user interaction. And while I really do appreciate the flexibility of chaining commands, I am a firm believer in exposing the common use cases as easily as possible.
+
+And yes, I am fully aware that not all annexes are created equal. Point in case, I would never use git annex pull on my laptop, but I would git annex push extensively.
+
+
+"""]]
diff --git a/doc/forum/bash_completion.mdwn b/doc/forum/bash_completion.mdwn
new file mode 100644
index 0000000000..649ad0457a
--- /dev/null
+++ b/doc/forum/bash_completion.mdwn
@@ -0,0 +1 @@
+I wrote more than 2 years ago a bash completion file for git-annex (did I miss an official version?) I just updated it today to match current version of git-annex, it is far from perfect, still incomplete, and probably buggy, but works for me. I tried to attach the file in case it could be useful to someone, but it seems that I can't; is it ok to paste the code directly?
diff --git a/doc/forum/bash_completion/comment_1_5c42c0c8e7fc3224bf5406880f9fd0c4._comment b/doc/forum/bash_completion/comment_1_5c42c0c8e7fc3224bf5406880f9fd0c4._comment
new file mode 100644
index 0000000000..7f2f0afd05
--- /dev/null
+++ b/doc/forum/bash_completion/comment_1_5c42c0c8e7fc3224bf5406880f9fd0c4._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="ringprince"
+ ip="134.76.140.110"
+ subject="comment 1"
+ date="2013-10-10T07:08:21Z"
+ content="""
+I hope it is.  I'd love to use it.
+"""]]
diff --git a/doc/forum/bash_completion/comment_2_6cbe3c825db99bf9188a0de8bb937d5b._comment b/doc/forum/bash_completion/comment_2_6cbe3c825db99bf9188a0de8bb937d5b._comment
new file mode 100644
index 0000000000..72d5e3fc41
--- /dev/null
+++ b/doc/forum/bash_completion/comment_2_6cbe3c825db99bf9188a0de8bb937d5b._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="https://id.koumbit.net/anarcat"
+ ip="72.0.72.144"
+ subject="comment 2"
+ date="2013-10-10T09:22:00Z"
+ content="""
+it is, please publish.
+"""]]
diff --git a/doc/forum/bash_completion/comment_3_948c40f1e46ca220d61365aebcd4f6d7._comment b/doc/forum/bash_completion/comment_3_948c40f1e46ca220d61365aebcd4f6d7._comment
new file mode 100644
index 0000000000..fbe659164e
--- /dev/null
+++ b/doc/forum/bash_completion/comment_3_948c40f1e46ca220d61365aebcd4f6d7._comment
@@ -0,0 +1,136 @@
+[[!comment format=mdwn
+ username="https://www.google.com/accounts/o8/id?id=AItOawkUwqII7LhbatqAQY1T5ZZOdPEFcQJKG6I"
+ nickname="Rafael"
+ subject="comment 3"
+ date="2013-10-13T16:14:34Z"
+ content="""
+This code needs the bash-completion file for git by Shawn O. Pearce, I think it is distributed by default on debian. Any feed-back or improvement is welcome! I'm far from expert, use at your own risks.
+
+
+    #!bash
+    #
+    # bash completion support for git-annex
+
+    # depends on git completion file (by Shawn O. Pearce):
+    [ -n \"$__git_whitespacelist\" ] || . $BASH_COMPLETION_DIR/git
+
+
+    # almost copy of __git_aliased_command
+    # requires 2 arguments: alias and main command (after git)
+    __git_aliased_subcommand ()
+    {
+        local word cmdline=$(git --git-dir=\"$(__gitdir)\" \
+            config --get \"alias.$1\")
+        for word in $cmdline; do
+            case \"$word\" in
+                \!*)	: shell command alias ;;
+                -*)	: option ;;
+                *=*)	: setting env ;;
+                git)	: git itself ;;
+                \"$2\")   : main command
+                    local found=1 ;;
+                *)
+                    [ -n \"${found-}\" ] && echo \"$word\"
+                    return
+            esac
+        done
+    }
+
+
+    _git_annex ()
+    {
+        _get_comp_words_by_ref -n =: cur words
+        # $ git annex 2>&1 |sed '1,6d '|grep -v '^$'|grep -v 'commands:$'|cut -c1-12
+        local subcommands=\"
+            add addurl assistant copy drop edit get import importfeed lock
+            mirror move rmurl sync unlock watch webapp content dead describe
+            direct enableremote group indirect init initremote semitrust
+            trust ungroup untrust vicfg addunused dropunused fix forget fsck
+            merge unused upgrade find help list log map status version
+            whereis migrate reinject unannex uninit dropkey
+            \"
+        # plumbing (to complete?): fromkey fuzztest pre-commit rekey test
+        # transferkey transferkeys xmppgit
+        local subcommand=\"$(__git_find_on_cmdline \"$subcommands\")\"
+        if [ -z \"$subcommand\" ]; then
+            ## search for aliased subcommand
+            ## works only if the alias is invoked just after git
+            ## (simpler, cf _git function)
+            local maybesubcom maybealias=${words[1]}
+            [ -n \"$maybealias\" ] && # false with a bash alias='git annex'
+            [ \"${maybealias:0:1}\" != '-' ] &&
+            maybesubcom=$(__git_aliased_subcommand \"$maybealias\" annex)
+            for subcommand in $subcommands \"\"; do
+                [ \"$maybesubcom\" = \"$subcommand\" ] && break
+            done
+            [ -z \"$subcommand\" ] &&
+            __gitcomp \"$subcommands\" &&
+            return
+        fi
+        case \"$cur\" in
+            --from=*|--to=*|--trust=*|--semitrust=*|--untrust=*|--in=*)
+                __gitcomp \"here $(__git_remotes)\" \"\" \"${cur##*=}\"
+                ;;
+            --*)
+                __gitcomp \"
+                    --force --fast --auto --all --unused --quiet --verbose --json --debug
+                    --no-debug --from= --to= --numcopies= --time-limit= --trust= --semitrust=
+                    --untrust= --trust-glacier-inventory --backend= --format= --user-agent=
+                    --exclude= --include= --in= --copies= --inbackend= --inallgroup=
+                    --smallerthan= --largerthan= --not --and --or
+                    \"
+                ;;
+            # -*)
+            # __gitcomp \"-( -) -c\"
+            # ;;
+            *)
+                case \"$subcommand\" in
+                    # subcommands with PATH (currently outputs wrongly \"uninit\",
+                    # and not \"reinject\")
+                    # $ tr ' ' '|' <<< $(git annex 2>&1 |sed '1,6d '|grep -v '^$' | \
+                    # grep -v 'commands:$'|grep PATH |cut -c1-12)
+                    add|copy|drop|edit|get|import|lock|mirror|move|unlock|fix| \
+                        fsck|find|list|log|status|whereis|migrate|unannex| \
+                        fromkey|pre-commit|rekey|reinject)
+                        COMPREPLY=() # complete with paths
+                        ;;
+                    addurl|importfeed|rmurl) # URL commands
+                        COMPREPLY=() # at least works with file://...
+                        ;;
+                    init|uninit|unused|merge|status|map|upgrade|version|assistant| \
+                        watch|webapp|direct|indirect|vicfg|forget|help) # | plumbing...
+                        COMPREPLY= # no more args expected
+                        # is 'COMPREPLY=' correct? (seems not standard practice...)
+                        ;;
+                    describe|trust|untrust|semitrust|sync|content|dead|group|ungroup)
+                        __gitcomp \"here $(__git_remotes)\"
+                        # TODO: handle git-annex special remotes
+                        ;;
+                    initremote|enableremote)
+                        case \"$cur\" in
+                            type=*)
+                                __gitcomp \"S3 bup directory rsync gcrypt webdav\" \"\" \"${cur##type=}\"
+                                ;;
+                            encryption=*)
+                                __gitcomp \"none hybrid shared pubkey\" \"\" \"${cur##encryption=}\"
+                                ;;
+
+                            buprepo=|directory=|rsyncurl=)
+                                COMPREPLY=()
+                                # use COMPREPLY (and compgen below) directly because
+                                #  __gitcomp does not handle '*=*' pattern, only '--*=*'
+                                # Writing a __gitannexcomp function may be worth it.
+                                ;;
+                            *)
+                                local initremoteopts=\"
+                                    type= encryption= buprepo= directory= rsyncurl= \"
+                                COMPREPLY=($(compgen -W \"$initremoteopts \" -- \"$cur\"))
+                                ;;
+                        esac
+                        ;;
+                esac
+                ;;
+        esac
+    }
+
+"""]]
diff --git a/doc/forum/bash_completion/comment_4_dbae348b230b780cda91ed8576b8f9fa._comment b/doc/forum/bash_completion/comment_4_dbae348b230b780cda91ed8576b8f9fa._comment
new file mode 100644
index 0000000000..525a613b0b
--- /dev/null
+++ b/doc/forum/bash_completion/comment_4_dbae348b230b780cda91ed8576b8f9fa._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="http://joeyh.name/"
+ ip="64.134.31.139"
+ subject="comment 4"
+ date="2013-10-15T17:53:21Z"
+ content="""
+@Rafael, you should put a license on that code..
+"""]]
diff --git a/doc/forum/bash_completion/comment_5_d45570fd750ae8d69dedc3edeef0cc10._comment b/doc/forum/bash_completion/comment_5_d45570fd750ae8d69dedc3edeef0cc10._comment
new file mode 100644
index 0000000000..d6797b5e14
--- /dev/null
+++ b/doc/forum/bash_completion/comment_5_d45570fd750ae8d69dedc3edeef0cc10._comment
@@ -0,0 +1,13 @@
+[[!comment format=mdwn
+ username="joey"
+ subject="""comment 5"""
+ date="2015-07-15T15:44:11Z"
+ content="""
+I've recently added built-in bash completion to git-annex, enabled by the
+`bash-completion.bash` file. It knows about all sub-commands,
+and all options, and will automatically always be up-to-date with the
+argument parser!
+
+Currently, it only works for completing "git-annex", not "git annex".
+I have submitted a patch to git's bash completion to make the latter work.
+"""]]
diff --git a/doc/forum/bash_completion/comment_6_2dbfa336ad8957ff0cafd8301ca1150f._comment b/doc/forum/bash_completion/comment_6_2dbfa336ad8957ff0cafd8301ca1150f._comment
new file mode 100644
index 0000000000..5cebfe08f3
--- /dev/null
+++ b/doc/forum/bash_completion/comment_6_2dbfa336ad8957ff0cafd8301ca1150f._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="wsha.code+ga@b38779424f41c5701bbe5937340be43ff1474b2d"
+ nickname="wsha.code+ga"
+ subject="comment 6"
+ date="2016-01-10T03:08:24Z"
+ content="""
+It seems that the `bash-completion.bash` file is not included in the standalone Linux tarball. Could it be added? It would be helpful at least for the Arch package in the AUR that is based on the tarball.
+"""]]
diff --git a/doc/forum/bash_completion/comment_7_2fd198746dc7351fe957e7a8150fcb89._comment b/doc/forum/bash_completion/comment_7_2fd198746dc7351fe957e7a8150fcb89._comment
new file mode 100644
index 0000000000..49b95e723a
--- /dev/null
+++ b/doc/forum/bash_completion/comment_7_2fd198746dc7351fe957e7a8150fcb89._comment
@@ -0,0 +1,13 @@
+[[!comment format=mdwn
+ username="joey"
+ subject="""comment 7"""
+ date="2016-01-11T16:22:02Z"
+ content="""
+It's not included in the standalone tarball because it wouldn't be
+installed in a usable location if it were included.
+
+I continue to think that the git-annex-bin package in Arch is not a good 
+choice. I get more complaints about problems with that package than any
+other distribution package of git-annex. Complicating the tarball to
+support that further is not appealing.
+"""]]
diff --git a/doc/forum/bash_completion/comment_8_7802f4bd08dddd24ac9d19ee7847735f._comment b/doc/forum/bash_completion/comment_8_7802f4bd08dddd24ac9d19ee7847735f._comment
new file mode 100644
index 0000000000..d73e0e7585
--- /dev/null
+++ b/doc/forum/bash_completion/comment_8_7802f4bd08dddd24ac9d19ee7847735f._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="andrew@1d92abee601373fe09908b30706c97ffce3da255"
+ nickname="andrew"
+ subject="comment 8"
+ date="2016-01-23T14:24:18Z"
+ content="""
+I think it might still be useful to have in the tarball -- we can symlink it into the right place.
+"""]]
diff --git a/doc/forum/basic_usage_questions.mdwn b/doc/forum/basic_usage_questions.mdwn
new file mode 100644
index 0000000000..cd6a5505b3
--- /dev/null
+++ b/doc/forum/basic_usage_questions.mdwn
@@ -0,0 +1,5 @@
+Seeking some clarification:
+
+ 1. I'm using direct mode. When I run `git annex sync --content` on repo A, all files get copied to repo B, but they remain hidden under `.git/annex/objects`. Is there a way to _automatically_ put them in repo B's working tree, without having to go to repo B and run `git annex sync` there as well? _(I'm sure I saw that happen earlier, but not anymore?)_
+
+ 1. I have two PCs and a portable HD. There are git-annex repos on PC_1 and USB_HD, with each other listed under `git remote`. Now I want to set up git-annex on PC_2. Is it okay to use the same repo path (~/Videos/) on both PCs? I'm concerned that it would confuse the USB_HD repo greatly, as it would end up having two "remotes" with identical paths.
diff --git a/doc/forum/basic_usage_questions/comment_1_c2a5fedc24f8c301d9042999a0371d9f._comment b/doc/forum/basic_usage_questions/comment_1_c2a5fedc24f8c301d9042999a0371d9f._comment
new file mode 100644
index 0000000000..91af0e3ac6
--- /dev/null
+++ b/doc/forum/basic_usage_questions/comment_1_c2a5fedc24f8c301d9042999a0371d9f._comment
@@ -0,0 +1,11 @@
+[[!comment format=mdwn
+ username="joey"
+ subject="""comment 1"""
+ date="2016-01-06T15:50:13Z"
+ content="""
+1. You have to run something on the receiving remote to update its working
+tree. Running the assistant in that repository would be one easy option.
+
+2. This is fine, git-annex will notice if a repository on a path changes
+and do the right thing.
+"""]]
diff --git a/doc/forum/basic_usage_questions/comment_2_67d047a107b20874cb71ba311681e181._comment b/doc/forum/basic_usage_questions/comment_2_67d047a107b20874cb71ba311681e181._comment
new file mode 100644
index 0000000000..e6e0225789
--- /dev/null
+++ b/doc/forum/basic_usage_questions/comment_2_67d047a107b20874cb71ba311681e181._comment
@@ -0,0 +1,16 @@
+[[!comment format=mdwn
+ username="grawity@2ea26be48562f66fcb9b66307da72b1e2e37453f"
+ nickname="grawity"
+ subject="comment 2"
+ date="2016-01-07T06:19:30Z"
+ content="""
+Alright, thanks. For #1, it sounds like something like Git's hooks/post-receive would be very useful here – can they be used with direct mode, or might git-annex have its own mechanism?
+
+---
+
+Also question 3 – what's the proper way to remove files and have the deletion propagate to all repos? I couldn't find it under \"Removing files\"... Is it `git annex drop --force`? Or should I just `rm` the files and hope that `git annex sync` will pick it up?
+
+The latter seems to work, although the files remain in `git annex unused`, and when I run `git annex dropunused` I again get warnings about \"0 out of 1 copies confirmed\" (so I have to use `--force` again), so it seems like a wrong way?
+
+(These are large read-only video files, I don't care about keeping revision history, only about syncing what I have. I haven't yet done any \"preferred content\" config either...)
+"""]]
diff --git a/doc/forum/basic_usage_questions/comment_3_94b4efa49efbee741c051227b3204562._comment b/doc/forum/basic_usage_questions/comment_3_94b4efa49efbee741c051227b3204562._comment
new file mode 100644
index 0000000000..604903aa0e
--- /dev/null
+++ b/doc/forum/basic_usage_questions/comment_3_94b4efa49efbee741c051227b3204562._comment
@@ -0,0 +1,13 @@
+[[!comment format=mdwn
+ username="joey"
+ subject="""comment 3"""
+ date="2016-01-15T19:14:07Z"
+ content="""
+You can use git's post-receive hook to run `git annex merge` to update a
+direct mode repository. See example in
+[[tips/setup_a_public_repository_on_a_web_site]].
+
+It's fine to use --force with dropunused if you really want to force
+deleting the only copy of a file. git-annex will never delete the last copy
+of a file without being forced as a safety measure.
+"""]]
diff --git a/doc/forum/batch_check_on_remote_when_using_copy.mdwn b/doc/forum/batch_check_on_remote_when_using_copy.mdwn
new file mode 100644
index 0000000000..633b61d6f9
--- /dev/null
+++ b/doc/forum/batch_check_on_remote_when_using_copy.mdwn
@@ -0,0 +1,34 @@
+When I copy my local repository with SHA* to a remote repo with SHA*, every single file is checked by itself which seems rather inefficient. When my remote is accessed via ssh, git-annex opens a new connections for every check. If you are not using a ssh key or key agent, this gets tedious...
+
+For all locked files, either git's built-in mechanisms should be used or, if that's not possible, a few hundred checksums (assuming SHA* backend) should be transfered at once and then checked locally before deciding that to transfer.
+
+Once all checks are done, one single transfer session should be started. Creating new sessions and waiting for TCP's slowstart to get going is a lot less than efficient.
+
+
+-- RichiH
+
+> (Use of SHA is irrelevant here, copy does not checksum anything.)
+> 
+> I think what you're seeing is
+> that `git annex copy --to remote` is slow, going to the remote repository
+> every time to see if it has the file, while `git annex copy --from remote`
+> is fast, since it looks at what files are locally present.
+> 
+> That is something I mean to improve. At least `git annex copy --fast --to remote`
+> could easily do a fast copy of all files that are known to be missing from
+> the remote repository. When local and remote git repos are not 100% in sync,
+> relying on that data could miss some files that the remote doesn't have anymore,
+> but local doesn't know it dropped. That's why it's a candidate for `--fast`.
+> 
+> I've just implemented that.
+> 
+> While I do hope to improve ssh usage so that it sshs once, and feeds
+> `git-annex-shell` a series of commands to run, that is a much longer-term
+> thing. --[[Joey]]
+
+>> FYI, in a repo with 1228 files, all small, repos _completely in sync_.
+
+    % git annex copy . --to foo # 1200 seconds
+    % git annex copy . --to foo --fast # 20 seconds
+
+>> RichiH
diff --git a/doc/forum/benefit_of_splitting_a_repository.mdwn b/doc/forum/benefit_of_splitting_a_repository.mdwn
new file mode 100644
index 0000000000..111feb2cf0
--- /dev/null
+++ b/doc/forum/benefit_of_splitting_a_repository.mdwn
@@ -0,0 +1,10 @@
+I am evaluating the best strategy to use git-annex to manage my media library. It consists of about 300.000 files totaling 1 TB of data.
+
+My question is, wheither it would be of advantage to split the repo into several smaller ones (Like Photos, Videos, Musik, Books, ...)?
+
+Would this affect performance of certain operations? I.e. Operations that have superlinear (O(n^a) with a > 1) complexity?
+
+I am thinking about "git annex unused", which takes 22 minutes on my machine performed on the full repo. 
+
+
+Do you have more interesting information on using git-annex in this scale?
diff --git a/doc/forum/benefit_of_splitting_a_repository/comment_1_93a86cb03b66e7ab5dd7146e7b86c9e8._comment b/doc/forum/benefit_of_splitting_a_repository/comment_1_93a86cb03b66e7ab5dd7146e7b86c9e8._comment
new file mode 100644
index 0000000000..72ea4f29bb
--- /dev/null
+++ b/doc/forum/benefit_of_splitting_a_repository/comment_1_93a86cb03b66e7ab5dd7146e7b86c9e8._comment
@@ -0,0 +1,15 @@
+[[!comment format=mdwn
+ username="http://joeyh.name/"
+ ip="4.154.6.49"
+ subject="comment 1"
+ date="2012-11-28T18:16:10Z"
+ content="""
+`git-annex unused` needs to scan the entire repository. But it uses a bloom filter, so its complexity is O(n) to the number of keys.
+
+`git annex fsck` scans the entire repository and also reads all available file content. But we have incremental fsck support now.
+
+The rest of git-annex is designed to have good locality.
+
+The main problem you are likely to run into is innefficiencies with git's index file. This file records the status of every file in the repository, and commands like `git add` rewrite the whole file. git-annex uses a journal to minimise operations that need to rewrite the git index file, but this won't help you when you're using raw git commands in the repository.
+
+"""]]
diff --git a/doc/forum/benefit_of_splitting_a_repository/comment_2_4e2fed247298d620fee7be883a1e86a6._comment b/doc/forum/benefit_of_splitting_a_repository/comment_2_4e2fed247298d620fee7be883a1e86a6._comment
new file mode 100644
index 0000000000..c102243f40
--- /dev/null
+++ b/doc/forum/benefit_of_splitting_a_repository/comment_2_4e2fed247298d620fee7be883a1e86a6._comment
@@ -0,0 +1,15 @@
+[[!comment format=mdwn
+ username="https://www.google.com/accounts/o8/id?id=AItOawlgyVag95OnpvSzQofjyX0WjW__MOMKsl0"
+ nickname="Sehr"
+ subject="comment 2"
+ date="2012-12-02T19:24:59Z"
+ content="""
+The biggest problem I am facing actually is the git pack-objects command, which takes forever. 
+
+I am not starting it manually, but \"git annex sync\" is. 
+
+Any advice on how to make this faster?
+Does it make sense at all to pack a bunch of uncompressible simlinks?
+
+I already tried the delta=false attrigute, without any (major) effect.
+"""]]
diff --git a/doc/forum/best_practices_for_importing_photos__63__.mdwn b/doc/forum/best_practices_for_importing_photos__63__.mdwn
new file mode 100644
index 0000000000..2f57f3b35e
--- /dev/null
+++ b/doc/forum/best_practices_for_importing_photos__63__.mdwn
@@ -0,0 +1,13 @@
+What are everyone's tips for importing photos to make best use of metadata and views?
+
+Let's assume there's no need to be compatible with a photo manager app, but we may be importing lots of duplicates, and while content deduplication is great, I'd like to avoid naming problems too.
+
+Do you bother to rename your photos?
+
+Do you use EXIF metadata as git-annex metadata? Selectively or wholesale, with all the redundant tags in EXIF?
+
+If you do use a photo manager app, do you need to do anything special to make that work?
+
+Thanks for your responses everyone!
+
+-mike
diff --git a/doc/forum/best_practices_for_importing_photos__63__/comment_1_37f0ae4b552ec2a4a144ddcdc17c8453._comment b/doc/forum/best_practices_for_importing_photos__63__/comment_1_37f0ae4b552ec2a4a144ddcdc17c8453._comment
new file mode 100644
index 0000000000..3d17874ad5
--- /dev/null
+++ b/doc/forum/best_practices_for_importing_photos__63__/comment_1_37f0ae4b552ec2a4a144ddcdc17c8453._comment
@@ -0,0 +1,19 @@
+[[!comment format=mdwn
+ username="Xyem"
+ ip="178.79.137.64"
+ subject="comment 1"
+ date="2014-04-15T14:25:04Z"
+ content="""
+git-annex's metadata and views made me stop hopping between programs (digikam, tagsistant etc.) to organise my photos (I had even just started working on my own FUSE tagging filesystem which was effectively going to be tagsistant, but with a git-annex'y backend).
+
+As usual, my method is probably a little odd :)
+
+Photos are 'git import'ed into a $(uuidgen) directory (so no worries about filename collisions) and tagged with media=Photograph and tag=untagged. Then I go through them and add relevant tags (one of which is \"xbmc\", no prizes for guessing how that works with the views :]) and move them into a more appropriate directory structure, using gqview and its \"sort manager\". This is really nice and fast, due to it only copying/moving symlinks!
+
+One thing I'm considering doing it putting a shim between git-annex and gqview, so that it generates entries in the sort manager which are appropriate for the current view. So, for example, if the view is location=*, the sort manager would have:
+
+    location=Malta
+    location=York
+
+While it wouldn't get updated if I create new tags (by creating directories in the view), it would save a lot of time creating them every time the view changes.
+"""]]
diff --git a/doc/forum/best_practices_for_importing_photos__63__/comment_2_7f96f0fe0fc073321bd7c5bbd9048425._comment b/doc/forum/best_practices_for_importing_photos__63__/comment_2_7f96f0fe0fc073321bd7c5bbd9048425._comment
new file mode 100644
index 0000000000..349ee432a3
--- /dev/null
+++ b/doc/forum/best_practices_for_importing_photos__63__/comment_2_7f96f0fe0fc073321bd7c5bbd9048425._comment
@@ -0,0 +1,11 @@
+[[!comment format=mdwn
+ username="http://joeyh.name/"
+ ip="209.250.56.191"
+ subject="comment 2"
+ date="2014-04-17T20:11:59Z"
+ content="""
+It seems to me that there is a lot of cruft in EXIF that I would not want to bloat my git-annex branch with storing. That's why [[tips/automatically_adding_metadata]] imports only the listed fields. It's easy to add fields later and re-run the metadata importer on your existing files.
+
+I have not gotten as far as having any best practices to share. :)
+
+"""]]
diff --git a/doc/forum/big_overhead.mdwn b/doc/forum/big_overhead.mdwn
new file mode 100644
index 0000000000..f16b95942b
--- /dev/null
+++ b/doc/forum/big_overhead.mdwn
@@ -0,0 +1,46 @@
+[[!meta title="unreachable git objects"]]
+
+Hi,
+
+I am been seeing quite big overheads using `git-annex`.  Is this is normal?
+
+The `.git/objects` folder is explosive in my system, often being larger
+than the content watched by git-annex.  Here's the actual statistics
+of my git-annex folders, where the fourth column is calculated as col3/(col2-col3).
+
+[[!table  data="""
+folder,size,size .git,relative size
+conf.annex,777536,720100,12.537433
+doc.annex,20351624,11260204,1.2385528
+images.annex,817064,435580,1.1418041
+misc.annex,803328,572476,2.4798399
+music.annex,23756116,9192740,0.63122314"""]]
+
+That is, four of five repos require more space for the `.git` folder than the actual files.  Most of this comes from the `objects` folder.  
+
+Number of files:
+
+[[!table data="""
+folder,no. files,no files .git,relative size
+conf.annex,11350,9539,5.2672557
+doc.annex,84954,66824,3.6858246
+images.annex,92787,91285,60.775632
+misc.annex,95461,95160,316.14618
+music.annex,16414,13520,4.6717346
+"""]]
+
+
+I use the assistant web interface, and direct
+mode.  I use two laptops running Linux that are synchronized
+directly over LAN at home or via a transfer repo on a ssh server
+where git-annex is installed.  The latter is set up using the web interface and the gcrypt repo.
+[Mostly, the transfer repo isn't working
+and I often end up with only symlinks on the computer where I did not edit the file in question, 
+but this is probably unrelated.]
+
+I have previously tried to fix it using `git gc` or `git annex forget`, but it doesn't seem to significantly reduce the sizes, and what it helps isn't persistent.
+
+Is this kind of 'overhead' something that one must accept when using
+`git-annex` or do such numbers indicate that something is wrong?
+
+Thanks.
diff --git a/doc/forum/big_overhead/comment_10_d5f4e353e7f711d8c38cdcc222339bca._comment b/doc/forum/big_overhead/comment_10_d5f4e353e7f711d8c38cdcc222339bca._comment
new file mode 100644
index 0000000000..2ebd56d30e
--- /dev/null
+++ b/doc/forum/big_overhead/comment_10_d5f4e353e7f711d8c38cdcc222339bca._comment
@@ -0,0 +1,14 @@
+[[!comment format=mdwn
+ username="http://joeyh.name/"
+ ip="108.236.230.124"
+ subject="comment 10"
+ date="2014-09-18T17:27:36Z"
+ content="""
+In the meantime, I've been looking over the Annex.Branch code. 
+
+`stageJournal` is only ever called in code paths that commit the updated index, so those code paths cannot result in dangling objects unless git-annex is interrupted before it can commit. (This may explain some of my own repos having a few dangling refs, that were not commits; I could have ctrl-c'd git-annex.)
+
+It's possible for a forced update of the local git-annex branch, done by eg a push from another repo, to overwrite a commit made to it. In this case, the git-annex index is merged with the branch, resulting in a new commit, and the old commit that was overwritten will indeed be dangling. However, `git annex sync` doesn't overwrite the git-annex branch; it pushes to synced/git-annex, or does a `taggedPush` to a private ref. It is the case that both those pushes are forced pushes, so can overwrite a branch ref and leave the old commit it pointed to dangling. In the case of `taggedPush`, the old commit should be a parent of the new, so it won't dangle. In the case of synced/git-annex being overwritten, the old commit could dangle, but only until whatever repo pushed it syncs again, at which time it should get incorporated as one of the parents of the new synced/git-annex it pushes. So, I don't see how long-term dangling commits could happen this way, except for in the case where a repository stops syncing/goes missing/rebases its git-annex branch (ie, git-annex forget is used). (This may explain the 2 dangling commits I found on elephant; we did delete some clones of that repository recently.)
+
+At this point I'm not convinced that the dangling objects I found in my own repos are due to some systematic problem, the above seems like it could explain them, and the above is not a problem on the class of the one Rasmus is having. Of course, it's hard to be sure you've spotted all possible ways that a resource leak can happen, and that's what these dangling objects basically are.
+"""]]
diff --git a/doc/forum/big_overhead/comment_11_cbf25217e4149f2cfad4e2bf94f2b4ca._comment b/doc/forum/big_overhead/comment_11_cbf25217e4149f2cfad4e2bf94f2b4ca._comment
new file mode 100644
index 0000000000..e2a74068d7
--- /dev/null
+++ b/doc/forum/big_overhead/comment_11_cbf25217e4149f2cfad4e2bf94f2b4ca._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="http://joeyh.name/"
+ ip="108.236.230.124"
+ subject="comment 11"
+ date="2014-09-18T17:32:09Z"
+ content="""
+I knew I *had* used \"Initial commit\" somewhere ... etckeeper uses that message. And commits as root. Could an etckeeper repo have somehow gotten merged into your git-annex repo? Seems strange, and the filenames and contents don't really look like /etc to me, but it otherwise somewhat fits.
+"""]]
diff --git a/doc/forum/big_overhead/comment_12_475d5af95adcfcd3a51e10f270205eb7._comment b/doc/forum/big_overhead/comment_12_475d5af95adcfcd3a51e10f270205eb7._comment
new file mode 100644
index 0000000000..53e15b7642
--- /dev/null
+++ b/doc/forum/big_overhead/comment_12_475d5af95adcfcd3a51e10f270205eb7._comment
@@ -0,0 +1,71 @@
+[[!comment format=mdwn
+ username="rasmus"
+ ip="146.185.23.178"
+ subject="comment 12"
+ date="2014-09-19T00:43:56Z"
+ content="""
+Hi Joey,
+
+Thanks for giving the thread a more appropriate title and thanks for the helpful messages. 
+
+Let me start with the easy points:
+
+
+* Looking at my log file of installed packages I have never used `etckeeper` on my system.  So unless it could have entered through `annex` then I think we can rule that one out. 
+* According to `git log` the repos are from January 2014 where I restarted my repos. 
+
+
+        commit 029a8e76ab5f66aa4390987130985550a1ccd69c
+        Author: Rasmus 
+        Date:   Thu Jan 23 21:06:13 2014 +0100
+
+        created repository
+
+
+* When I start git repos I typically just use \"init\" so I don't think I did the 2012 commits. 
+* I checked out one of the 74mb files.  When I do `file test.blob` it shows `test.blob: GPG symmetrically encrypted data (CAST5 cipher)`.  But none of my normal passwords worked.  Could such a gpg'ed file be from local network connections where the assistant asks for a passphrase?  I'm pretty sure that my transfer repo has only been using `gcrypt` and I believe I \"restarted\" my repos because I switched to `gcrypt` repos.  Also, my transfer repo is 10Gb as well which sounds big for transfer repo. 
+
+I performed a similar \"analysis\" on the `conf.annex` repo which should contain mostly no binary files (some 16x16 pngs etc).  
+
+`conf.annex` has 727 unreachable objects and 3477 commits in total.  Of these 338 are commits.  Here's an example of a larger commit message of an unreachable commit.
+
+    commit 601c10f9512e8d3502d9dd52ef409560ebb5b7e0
+    Author: root 
+    Date:   Mon Dec 31 19:00:01 2012 -0400
+
+         Initial commit
+
+     diff --git a/6fbbea493cdec9d912d256374199cc4c012022d35524c8789a7aceeb953442a5 b/6fbbea493cdec9d912d256374199cc4c012022d35524c8789a7aceeb953442a5
+     new file mode 100644
+     index 0000000..ea5fcc3
+     Binary files /dev/null and b/6fbbea493cdec9d912d256374199cc4c012022d35524c8789a7aceeb953442a5 differ
+     diff --git a/91bd0c092128cf2e60e1a608c31e92caf1f9c1595f83f2890ef17c0e4881aa0a b/91bd0c092128cf2e60e1a608c31e92caf1f9c1595f83f2890ef17c0e4881aa0a
+     new file mode 100644
+     index 0000000..a86c1a9
+     Binary files /dev/null and b/91bd0c092128cf2e60e1a608c31e92caf1f9c1595f83f2890ef17c0e4881aa0a differ
+     diff --git a/9da3fcfc1635c674012c35d90c21adce3c35440e629d64fe117fe349a6b3e194 b/9da3fcfc1635c674012c35d90c21adce3c35440e629d64fe117fe349a6b3e194
+     new file mode 100644
+     index 0000000..ef1d71c
+     Binary files /dev/null and b/9da3fcfc1635c674012c35d90c21adce3c35440e629d64fe117fe349a6b3e194 differ
+     diff --git a/ad4ae79c29b3756f7e41257db7454f3c319112d06385a8bc12d28209a82f2594 b/ad4ae79c29b3756f7e41257db7454f3c319112d06385a8bc12d28209a82f2594
+     new file mode 100644
+     index 0000000..61d3e5b
+     Binary files /dev/null and b/ad4ae79c29b3756f7e41257db7454f3c319112d06385a8bc12d28209a82f2594 differ
+     diff --git a/bd0e9cb492077e0c090bc62892c8de438c51a956c8215b2c68de7caa7e2431cc b/bd0e9cb492077e0c090bc62892c8de438c51a956c8215b2c68de7caa7e2431cc
+     new file mode 100644
+     index 0000000..92e9bd7
+     Binary files /dev/null and b/bd0e9cb492077e0c090bc62892c8de438c51a956c8215b2c68de7caa7e2431cc differ
+
+Across all commits 6006 objects are mentioned, but only 371 are unique. 
+
+I checked out one blob and again `file` reports `GPG symmetrically encrypted data (CAST5 cipher)`.  Interesting for `conf.annex` I get this line when trying to decrypt
+
+    gpg: DBG: cleared passphrase cached with ID: SBF83A0F822D0F664
+
+
+For `doc.annex` I get
+
+    gpg: DBG: cleared passphrase cached with ID: S32DEAD1E8DD06A4D
+
+And on my other computer I see a third ID.  I'm not sure if this means anything when files are symmetrically encrypted, though. 
+"""]]
diff --git a/doc/forum/big_overhead/comment_13_1c8cc992f04fc63179094c494bd25025._comment b/doc/forum/big_overhead/comment_13_1c8cc992f04fc63179094c494bd25025._comment
new file mode 100644
index 0000000000..229ef256a7
--- /dev/null
+++ b/doc/forum/big_overhead/comment_13_1c8cc992f04fc63179094c494bd25025._comment
@@ -0,0 +1,10 @@
+[[!comment format=mdwn
+ username="http://joeyh.name/"
+ ip="108.236.230.124"
+ subject="I know what it is now"
+ date="2014-09-19T02:43:22Z"
+ content="""
+These objects are the ones written by git-remote-gcrypt when pushing to a remote. That's why the weird dates, root pseudo-commit, crazy filenames, and big gpg encrypted blobs. All countermeasures that git-remote-gcrypt uses to keep your encrypted git remote safe and not leak information about what's in it.
+
+So, this is a bug in git-remote-gcrypt. It needs to clean these objects up after pushing them! (Also after failed pushes.)
+"""]]
diff --git a/doc/forum/big_overhead/comment_14_cbfb3d557915258e72c65a4e84df77a9._comment b/doc/forum/big_overhead/comment_14_cbfb3d557915258e72c65a4e84df77a9._comment
new file mode 100644
index 0000000000..87632c9093
--- /dev/null
+++ b/doc/forum/big_overhead/comment_14_cbfb3d557915258e72c65a4e84df77a9._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="http://joeyh.name/"
+ ip="108.236.230.124"
+ subject="comment 14"
+ date="2014-09-19T02:59:36Z"
+ content="""
+
+"""]]
diff --git a/doc/forum/big_overhead/comment_15_b973529bae549bcbaaae792f0403989b._comment b/doc/forum/big_overhead/comment_15_b973529bae549bcbaaae792f0403989b._comment
new file mode 100644
index 0000000000..a875cc34e1
--- /dev/null
+++ b/doc/forum/big_overhead/comment_15_b973529bae549bcbaaae792f0403989b._comment
@@ -0,0 +1,10 @@
+[[!comment format=mdwn
+ username="rasmus"
+ ip="217.130.110.20"
+ subject="comment 15"
+ date="2014-09-19T06:29:58Z"
+ content="""
+Brilliant!  Thanks for taking time to analyze the issue and taking the bug to `gcrypt`.
+
+[I'm surprised that a different key than my git-annex key is used and that it's a symmetric key, but I will explore the technology on my own].
+"""]]
diff --git a/doc/forum/big_overhead/comment_1_0c184520c30a89bd2604ab7c0eb7ac45._comment b/doc/forum/big_overhead/comment_1_0c184520c30a89bd2604ab7c0eb7ac45._comment
new file mode 100644
index 0000000000..20c2ff6c02
--- /dev/null
+++ b/doc/forum/big_overhead/comment_1_0c184520c30a89bd2604ab7c0eb7ac45._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="https://www.google.com/accounts/o8/id?id=AItOawm8wY171R5c4u_jPmB6LU6n6Px2xePM4sE"
+ nickname="Efraim"
+ subject="comment 1"
+ date="2014-09-07T14:09:05Z"
+ content="""
+have you tried from the command line `git annex unused` to see if you have unused files in your repo? From the assistant, the option under configuration -> Unused files gives you an option to expire old files after a period of time so they get deleted from your repo.
+"""]]
diff --git a/doc/forum/big_overhead/comment_2_13a017c6c84511894ded2d89d86eb541._comment b/doc/forum/big_overhead/comment_2_13a017c6c84511894ded2d89d86eb541._comment
new file mode 100644
index 0000000000..25d010fe11
--- /dev/null
+++ b/doc/forum/big_overhead/comment_2_13a017c6c84511894ded2d89d86eb541._comment
@@ -0,0 +1,10 @@
+[[!comment format=mdwn
+ username="rasmus"
+ ip="109.201.154.150"
+ subject="comment 2"
+ date="2014-09-07T15:17:25Z"
+ content="""
+Thanks for help, Efraim.
+
+I'm not sure this is it.  On my other laptop, where the above statics were not calculated the `.git` folder of `doc.annex` is 26Gb (contents is 8.6Gb).  Meanwhile, unused files are 0.6Gb.  In `conf.annex` the `.git` folder is 3.2Gb, content is 70Mb and unused files is 2.2Mb.  I used the web interface to find the size of unused files.
+"""]]
diff --git a/doc/forum/big_overhead/comment_3_b4761cacc02396b5bdf34bcf3457cb91._comment b/doc/forum/big_overhead/comment_3_b4761cacc02396b5bdf34bcf3457cb91._comment
new file mode 100644
index 0000000000..f090e5b258
--- /dev/null
+++ b/doc/forum/big_overhead/comment_3_b4761cacc02396b5bdf34bcf3457cb91._comment
@@ -0,0 +1,10 @@
+[[!comment format=mdwn
+ username="http://id.clacke.se/"
+ nickname="Claes"
+ subject="repack parameters"
+ date="2014-09-07T21:53:10Z"
+ content="""
+Because git-annex tracks all the events of an annexed file for each repo -- added, dropped, copied etc -- and it tracks these in one object per file in the git-annex branch, it does indeed create a lot of objects. To improve both space and performance I made sure to add `git gc --auto` as a post-commit hook, as the objects in my case can quickly reach the tens or even hundreds of thousands.
+
+To further improve performance and space, you can choose to set `pack.window` and `pack.depth` to vastly higher values than the defaults (10 and 50, respectively), because there is a large amount of objects with very similar content. I did a `git repack --window 2500 --depth 1000 -f -a d` and brought down my repo from 3 GiB (packed!) to 300 MiB. Make sure to have a lot of memory and CPU available when doing this, or it will take forever. You can set `pack.window` ridiculously high if you like, as long as you limit it with `pack.windowMemory`, so that it makes use of all your available memory for comparing objects and finding the optimal delta.
+"""]]
diff --git a/doc/forum/big_overhead/comment_4_633f8538f368220eaca82b8bbfe9b49d._comment b/doc/forum/big_overhead/comment_4_633f8538f368220eaca82b8bbfe9b49d._comment
new file mode 100644
index 0000000000..a0001f71e1
--- /dev/null
+++ b/doc/forum/big_overhead/comment_4_633f8538f368220eaca82b8bbfe9b49d._comment
@@ -0,0 +1,16 @@
+[[!comment format=mdwn
+ username="rasmus"
+ ip="109.201.154.209"
+ subject="Re: repack parameters"
+ date="2014-09-08T13:20:36Z"
+ content="""
+Thanks for your tips, Claes.  I was really aware of `git repack` and that set of parameters.
+
+I didn't mention, but sadly I'd run `git gc` on the repos just before collecting the above numbers.
+
+I tried to repack two repositories -- `doc.annex` and `config.annex` -- using the values you suggested.   However, it did not have any measurable effect (less than 100mb in both cases).  
+
+The number of unused files seem to be (much) less than 500 files in the repos.
+
+BTW: All of the extra size is in the `.git/objects/` folder.  `.git/annex/` is quite small (always much less than 1GB).  Would that indicate that large files are checked in with git sans annex somehow?
+"""]]
diff --git a/doc/forum/big_overhead/comment_5_599092b8d4f1fc8f378796328ef42931._comment b/doc/forum/big_overhead/comment_5_599092b8d4f1fc8f378796328ef42931._comment
new file mode 100644
index 0000000000..ee91a5b2bd
--- /dev/null
+++ b/doc/forum/big_overhead/comment_5_599092b8d4f1fc8f378796328ef42931._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="rasmus"
+ ip="109.201.154.183"
+ subject="comment 5"
+ date="2014-09-08T13:48:03Z"
+ content="""
+So `git prune` worked wonders on my repos, getting rid of GBs of stuff in the `.git/objects` folders.  I don't know why they weren't picked up by `git gc`.  In retrospect, it was perhaps a bit careless of me to run `git prune` directly, but hopefully I will be OK. . .
+"""]]
diff --git a/doc/forum/big_overhead/comment_6_adb4b4e7eb4dac7760f3425bae6cbbc2._comment b/doc/forum/big_overhead/comment_6_adb4b4e7eb4dac7760f3425bae6cbbc2._comment
new file mode 100644
index 0000000000..0bc1763a3d
--- /dev/null
+++ b/doc/forum/big_overhead/comment_6_adb4b4e7eb4dac7760f3425bae6cbbc2._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="rasmus"
+ ip="193.145.48.43"
+ subject="comment 6"
+ date="2014-09-10T12:07:53Z"
+ content="""
+Seems `git prune` only worked as a temporary fix.  My `doc.annex/.git/objects` is 3.6Gb after two days.  I don't get why `git` sans `annex` is checking in stuff -- which I assume is the reason it's stored in `.git/objects`.
+"""]]
diff --git a/doc/forum/big_overhead/comment_7_a762eb55addf81c1c5350c7968598d0f._comment b/doc/forum/big_overhead/comment_7_a762eb55addf81c1c5350c7968598d0f._comment
new file mode 100644
index 0000000000..1ec6a6eef6
--- /dev/null
+++ b/doc/forum/big_overhead/comment_7_a762eb55addf81c1c5350c7968598d0f._comment
@@ -0,0 +1,16 @@
+[[!comment format=mdwn
+ username="http://joeyh.name/"
+ ip="108.236.230.124"
+ subject="comment 7"
+ date="2014-09-17T20:20:40Z"
+ content="""
+There are a few things that can cause git to leave unreachable objects. These include: Rebasing; interrupting a pull before it updates the refs; running git add on a file and then changing the file's content and adding it a second time before committing.
+
+I can think of one case where this happens when using git-annex at the command line: `git annex add $file; git mv $file other-directory; git commit` will result in a dangling object storing the old symlink target before the file was moved.
+
+It'd be useful to investigate, by using `git fsck --unreachable` to get a list of currently unreachable objects, and then use `git show` to look at the objects and try to determine where they came from. Ie, are they symlink targets or are they git-annex location log files (formatted as columns of timestamps and uuids). Any unreachable commits would be the most useful to investigate.
+
+I see a few loose objects here and there in my annexes, but not very many, and git-gc has cleaned up old ones (> 1 month old). Some of them seem to be location log files. I see those in both repositories where I use the assistant, and repositories where I use only command line git-annex. I was able to find 2 unreachable commits in a repository that runs the assistant full-time; both commits were \"merging origin/synced/git-annex into git-annex\". This suggests to me that perhaps the assistant merged the git-annex branch but that merge was overwritten by another thread that committed changes to the branch at the same time.
+
+You should also check the size of inodes on your system; a thousand small loose objects in .git/objects does not normally take up gigabytes of space; with typical inode sizes it might use up a few megabytes. With 1 mb inodes, those same thousand files would use 1 gb..
+"""]]
diff --git a/doc/forum/big_overhead/comment_8_4a66f57c6c0bdc6123618cb69a719be5._comment b/doc/forum/big_overhead/comment_8_4a66f57c6c0bdc6123618cb69a719be5._comment
new file mode 100644
index 0000000000..fdbebf4e4d
--- /dev/null
+++ b/doc/forum/big_overhead/comment_8_4a66f57c6c0bdc6123618cb69a719be5._comment
@@ -0,0 +1,65 @@
+[[!comment format=mdwn
+ username="rasmus"
+ ip="217.130.110.20"
+ subject="comment 8"
+ date="2014-09-18T11:28:45Z"
+ content="""
+Hi Joey,
+
+Thanks for your careful reply.  
+
+Easy things first:
+
+I *never* add anything from the terminal, though I may do checks and `git annex get`, since sometimes the assistance actually grab the updated files.  Until recently I started git annex automatically on boot, but at the moment it simply renders my laptop useless for too long -- presumably due to the errors investigated here.
+
+I use btrfs (don't ask me why).  Searching online, I did not find a way to find the size of inodes, but I assume that it's sensible?  tune2fs doesn't work but as I understand it is designed for ext*.
+
+What takes up space in my `.git/objects` is files of several Mb.  So at the moment the `pack` folder is 700mb.  In the next biggest folder there's three files that are 73,4mb and 8 files that are 4kb.  This pattern repeats.  A couple of large files (73,4 shows up quite a bit as well as 45) and many small files. 
+
+I have an astonishing amount of dangling objects.  In the `doc.annex` `git rev-list HEAD --count` gives 27354.  In this repo I have 1108 unreachable blobs and commits, respectively 569 and 539.  This probably explains why `git prune` solves my problem but I don't understand why all these large files reappears when I sync -- even after having run `git prune` on both laptops.  Could they come from the `annex` on my remote server?
+
+`git show` isn't nice on blobs, but here is an example of a dangling commit
+
+
+    commit 478425bef867782e8ff22aca24316e9421288c49
+    Author: root 
+    Date:   Mon Dec 31 19:00:01 2012 -0400
+
+        Initial commit
+
+    diff --git a/6e5039464b41f39088a4aece64ced787aa2b04ec2dd5ac6f6c6ca4b9a06a99e5 b/6e5039464b41f39088a4aece64ced787aa2b04ec2dd5ac6f6c6ca4b9a06a99e5
+    new file mode 100644
+    index 0000000..af12763
+    Binary files /dev/null and b/6e5039464b41f39088a4aece64ced787aa2b04ec2dd5ac6f6c6ca4b9a06a99e5 differ
+    diff --git a/8ae4ee273eb540fb71b78152d10010ea2dd3d1bb82afe410ecf3d811cb72bd6d b/8ae4ee273eb540fb71b78152d10010ea2dd3d1bb82afe410ecf3d811cb72bd6d
+    new file mode 100644
+    index 0000000..0a6af91
+    Binary files /dev/null and b/8ae4ee273eb540fb71b78152d10010ea2dd3d1bb82afe410ecf3d811cb72bd6d differ
+    diff --git a/91bd0c092128cf2e60e1a608c31e92caf1f9c1595f83f2890ef17c0e4881aa0a b/91bd0c092128cf2e60e1a608c31e92caf1f9c1595f83f2890ef17c0e4881aa0a
+    new file mode 100644
+    index 0000000..26d921e
+    Binary files /dev/null and b/91bd0c092128cf2e60e1a608c31e92caf1f9c1595f83f2890ef17c0e4881aa0a differ
+    diff --git a/9f7728197cfcd9792eef1ff5930a4ab580e38e64291037130f1ad0914e34a1fc b/9f7728197cfcd9792eef1ff5930a4ab580e38e64291037130f1ad0914e34a1fc
+    new file mode 100644
+    index 0000000..2a92974
+    Binary files /dev/null and b/9f7728197cfcd9792eef1ff5930a4ab580e38e64291037130f1ad0914e34a1fc differ
+    diff --git a/ac801235d97275e761efa12a76ee009472cae8549a0835d5be8bd3f6657047fb b/ac801235d97275e761efa12a76ee009472cae8549a0835d5be8bd3f6657047fb
+    new file mode 100644
+    index 0000000..543430c
+    Binary files /dev/null and b/ac801235d97275e761efa12a76ee009472cae8549a0835d5be8bd3f6657047fb differ
+    diff --git a/d400d0f616a980ea5e3ef68a1f9d670d1eeccbd27f34d1cb7ea976e1f98e2fb7 b/d400d0f616a980ea5e3ef68a1f9d670d1eeccbd27f34d1cb7ea976e1f98e2fb7
+    new file mode 100644
+    index 0000000..7b7eadd
+    Binary files /dev/null and b/d400d0f616a980ea5e3ef68a1f9d670d1eeccbd27f34d1cb7ea976e1f98e2fb7 differ
+    diff --git a/e988a26fbabe3f498e2a564096948eafb289ccadfb186423c1f63c5a3b2c19db b/e988a26fbabe3f498e2a564096948eafb289ccadfb186423c1f63c5a3b2c19db
+    new file mode 100644
+    index 0000000..3bd1dfa
+    Binary files /dev/null and b/e988a26fbabe3f498e2a564096948eafb289ccadfb186423c1f63c5a3b2c19db differ
+
+There are several things I don't understand.  Why is the author root?  I never run `git annex` with `sudo` or as root.  I think the date is bogus.  I'm pretty sure I wasn't even running `git annex` in 2012 much less working with this repo. . . What is weird is that this is the date for *all* lost commits!  (Same for Author).  Over all lost commits there are 2352 binary files that differ.  Of these there are 284 unique hashes. . .  I don't know what this means other than my repo being seriously messed up.  I don't understand what I did wrong to end up in this state as I have been fairly careful in mainly using the `webapp`.
+
+I wonder if the best way to proceed is to start over, or whether this repo can be recovered.
+
+Thanks,
+Rasmus
+"""]]
diff --git a/doc/forum/big_overhead/comment_9_5fa681ea0d6bd0dcac7142d40df9d54f._comment b/doc/forum/big_overhead/comment_9_5fa681ea0d6bd0dcac7142d40df9d54f._comment
new file mode 100644
index 0000000000..856f326f4f
--- /dev/null
+++ b/doc/forum/big_overhead/comment_9_5fa681ea0d6bd0dcac7142d40df9d54f._comment
@@ -0,0 +1,12 @@
+[[!comment format=mdwn
+ username="http://joeyh.name/"
+ ip="108.236.230.124"
+ subject="comment 9"
+ date="2014-09-18T16:54:10Z"
+ content="""
+That is a very strange commit by every metric. Weird author, weird date, weird filenames in it (not files that git-annex uses!), with apparently some weird binary content (which git-annex would not be committing). Even a weird commit message -- git-annex never makes a commit with a message of \"Initial commit\", and as far as I can tell using `git log -S`, it never has. (OTOH, it's a pretty common example message used in eg, git documentation.) So, I feel pretty sure that dangling commit was not made by git-annex.
+
+I think you need to take a look at some of the 4+mb unreachable blobs, to get some idea of what these files are. One way is to use git-show on the hash of one of the blobs to get its content, and then, perhaps pass it to `file` or `strings`. Or, you could stop the assistant, `git checkout 478425bef867782e8ff22aca24316e9421288c49` and have a look at this strange tree that was apparently committed in 2012 to see what's in there.
+
+It might be possible that the dangling commits come somehow from the remote server. I'm not 100% sure, but I think that a git pack can end up with dangling objects in it, and then git can pull down that pack to get other, non-dangling objects. You should use `git show` on the server on some of the dangling shas to see if they are present there.
+"""]]
diff --git a/doc/forum/btsync_equivalent__63__.mdwn b/doc/forum/btsync_equivalent__63__.mdwn
new file mode 100644
index 0000000000..ce72b10c5d
--- /dev/null
+++ b/doc/forum/btsync_equivalent__63__.mdwn
@@ -0,0 +1,8 @@
+I am using bit torrent sync to sync folders and files between my server, my laptop and my smartphone.
+How similar, or different, is git-annex?
+
+Btsync allows me to sync files with no centralized server, encrypted, with some revision history (but I believe it is basic).
+
+
+
+/glattering
diff --git a/doc/forum/btsync_equivalent__63__/comment_1_6bfdc6f595a8aa8979a7c2a10925812f._comment b/doc/forum/btsync_equivalent__63__/comment_1_6bfdc6f595a8aa8979a7c2a10925812f._comment
new file mode 100644
index 0000000000..dd8b17f1a9
--- /dev/null
+++ b/doc/forum/btsync_equivalent__63__/comment_1_6bfdc6f595a8aa8979a7c2a10925812f._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="https://id.koumbit.net/anarcat"
+ ip="2001:1928:1:9::1"
+ subject="comment 1"
+ date="2014-01-16T15:55:05Z"
+ content="""
+Maybe [[telehash|devblog/day_97__exciting_telehash_possiblities/]] is the free and open source answer to btsync?
+"""]]
diff --git a/doc/forum/build_from_source_on_Linux_fails___40__out_of_memory__41__.mdwn b/doc/forum/build_from_source_on_Linux_fails___40__out_of_memory__41__.mdwn
new file mode 100644
index 0000000000..a492bd4c94
--- /dev/null
+++ b/doc/forum/build_from_source_on_Linux_fails___40__out_of_memory__41__.mdwn
@@ -0,0 +1,74 @@
+Hi
+
+When trying to do "make" in Linux x64_64, the process fails with an out-of-memory error. "top" shows that memory consumption reached 16GB during build.
+
+Grateful for any recommendations.
+
+Ciaron
+
+    which: no ikiwiki in 
+    if [ "cabal " = ./Setup ]; then ghc --make Setup; fi
+    cabal  configure
+    Resolving dependencies...
+    [ 1 of 31] Compiling Utility.FileSize ( Utility/FileSize.hs, dist/setup/Utility/FileSize.o )
+    [ 2 of 31] Compiling Utility.PosixFiles ( Utility/PosixFiles.hs, dist/setup/Utility/PosixFiles.o )
+    [ 3 of 31] Compiling Utility.FileSystemEncoding ( Utility/FileSystemEncoding.hs, dist/setup/Utility/FileSystemEncoding.o )
+    [ 4 of 31] Compiling Utility.Applicative ( Utility/Applicative.hs, dist/setup/Utility/Applicative.o )
+    [ 5 of 31] Compiling Utility.Data     ( Utility/Data.hs, dist/setup/Utility/Data.o )
+    [ 6 of 31] Compiling Utility.Monad    ( Utility/Monad.hs, dist/setup/Utility/Monad.o )
+    [ 7 of 31] Compiling Utility.Exception ( Utility/Exception.hs, dist/setup/Utility/Exception.o )
+    [ 8 of 31] Compiling Utility.Misc     ( Utility/Misc.hs, dist/setup/Utility/Misc.o )
+    [ 9 of 31] Compiling Utility.Tmp      ( Utility/Tmp.hs, dist/setup/Utility/Tmp.o )
+    [10 of 31] Compiling Utility.Env      ( Utility/Env.hs, dist/setup/Utility/Env.o )
+    [11 of 31] Compiling Utility.UserInfo ( Utility/UserInfo.hs, dist/setup/Utility/UserInfo.o )
+    [12 of 31] Compiling Utility.Path     ( Utility/Path.hs, dist/setup/Utility/Path.o )
+    [13 of 31] Compiling Utility.OSX      ( Utility/OSX.hs, dist/setup/Utility/OSX.o )
+    [14 of 31] Compiling Utility.Process  ( Utility/Process.hs, dist/setup/Utility/Process.o )
+    [15 of 31] Compiling Utility.SafeCommand ( Utility/SafeCommand.hs, dist/setup/Utility/SafeCommand.o )
+    [16 of 31] Compiling Utility.Directory ( Utility/Directory.hs, dist/setup/Utility/Directory.o )
+    [17 of 31] Compiling Utility.ExternalSHA ( Utility/ExternalSHA.hs, dist/setup/Utility/ExternalSHA.o )
+    [18 of 31] Compiling Utility.Network  ( Utility/Network.hs, dist/setup/Utility/Network.o )
+    [19 of 31] Compiling Utility.PartialPrelude ( Utility/PartialPrelude.hs, dist/setup/Utility/PartialPrelude.o )
+    [20 of 31] Compiling Common           ( Common.hs, dist/setup/Common.o )
+    [21 of 31] Compiling Utility.DottedVersion ( Utility/DottedVersion.hs, dist/setup/Utility/DottedVersion.o )
+    [22 of 31] Compiling Git.Version      ( Git/Version.hs, dist/setup/Git/Version.o )
+    [23 of 31] Compiling Utility.FreeDesktop ( Utility/FreeDesktop.hs, dist/setup/Utility/FreeDesktop.o )
+    [24 of 31] Compiling Config.Files     ( Config/Files.hs, dist/setup/Config/Files.o )
+    [25 of 31] Compiling Assistant.Install.AutoStart ( Assistant/Install/AutoStart.hs, dist/setup/Assistant/Install/AutoStart.o )
+    [26 of 31] Compiling Assistant.Install.Menu ( Assistant/Install/Menu.hs, dist/setup/Assistant/Install/Menu.o )
+    [27 of 31] Compiling Build.Version    ( Build/Version.hs, dist/setup/Build/Version.o )
+    [28 of 31] Compiling Build.TestConfig ( Build/TestConfig.hs, dist/setup/Build/TestConfig.o )
+    [29 of 31] Compiling Build.Configure  ( Build/Configure.hs, dist/setup/Build/Configure.o )
+    [30 of 31] Compiling Build.DesktopFile ( Build/DesktopFile.hs, dist/setup/Build/DesktopFile.o )
+    [31 of 31] Compiling Main             ( dist/setup/setup.hs, dist/setup/Main.o )
+    Linking ./dist/setup/setup ...
+      checking version... 5.20150612-gf176464
+      checking UPGRADE_LOCATION... not available
+      checking git... yes
+      checking git version... 1.7.9
+      checking cp -a... yes
+      checking cp -p... yes
+      checking cp --preserve=timestamps... yes
+      checking cp --reflink=auto... no
+      checking xargs -0... yes
+      checking rsync... yes
+      checking curl... yes
+      checking wget... yes
+      checking wget supports -q --show-progress... no
+      checking bup... no
+      checking nice... yes
+      checking ionice... yes
+      checking nocache... no
+      checking gpg... gpg
+      checking lsof... lsof
+      checking git-remote-gcrypt... not available
+      checking ssh connection caching... no
+      checking sha1... sha1sum
+      checking sha256... sha256sum
+      checking sha512... sha512sum
+      checking sha224... sha224sum
+      checking sha384... sha384sum
+    Configuring git-annex-5.20150528...
+    setup: out of memory (requested 1048576 bytes)
+    make: *** [Build/SysConfig.hs] Error 1
+
diff --git a/doc/forum/build_from_source_on_Linux_fails___40__out_of_memory__41__/comment_1_cd004b531ae384e134e16600b64473e0._comment b/doc/forum/build_from_source_on_Linux_fails___40__out_of_memory__41__/comment_1_cd004b531ae384e134e16600b64473e0._comment
new file mode 100644
index 0000000000..a6445c3e93
--- /dev/null
+++ b/doc/forum/build_from_source_on_Linux_fails___40__out_of_memory__41__/comment_1_cd004b531ae384e134e16600b64473e0._comment
@@ -0,0 +1,18 @@
+[[!comment format=mdwn
+ username="joey"
+ subject="""comment 1"""
+ date="2015-06-14T16:48:36Z"
+ content="""
+This is cabal trying to resolve the dependencies and failing to find any
+solution, and trying harder and harder as it leaks memory somewhere.
+
+IIRC newer versions of cabal behave better in these situations, but the
+underlying problem is that there is a problem in your set of installed
+haskell libraries that prevents cabal finding a solution.
+
+cabal configure -v3 will show you the dependency resolver at work, and can be examined
+to find the details. But, the easy solution is probably to use stackage
+which will provide cabal with a canned dependency solution. Just copy
+`standalone/linux/cabal.config` to the top of the source tree and re-run
+cabal configure.
+"""]]
diff --git a/doc/forum/bup_backups_to_amazon_cloud_drive.mdwn b/doc/forum/bup_backups_to_amazon_cloud_drive.mdwn
new file mode 100644
index 0000000000..04b0b6fe9b
--- /dev/null
+++ b/doc/forum/bup_backups_to_amazon_cloud_drive.mdwn
@@ -0,0 +1,12 @@
+I have the following setup:
+
+1. Removable hard drive (truecrypt)
+2. Removable hard drive (truecrypt)
+3. Amazon Cloud drive
+
+I usually use bup to make backups to (1) and (2) but now I would like to add (3). I would like to take advantage of git annex to sync/pull my backups easily, however git-annex doesn't like git inside git for some reason. Git annex give me the following features:
+
+1. Encryption using gnupg
+2. Pulling a backup is easy from remote then I run bup on the pulled backup.
+
+Any suggestion on how to accomplish this?
diff --git a/doc/forum/bup_backups_to_amazon_cloud_drive/comment_1_e8f6fdc70cedd5482da6a185c6c6b9b9._comment b/doc/forum/bup_backups_to_amazon_cloud_drive/comment_1_e8f6fdc70cedd5482da6a185c6c6b9b9._comment
new file mode 100644
index 0000000000..bd782a0e49
--- /dev/null
+++ b/doc/forum/bup_backups_to_amazon_cloud_drive/comment_1_e8f6fdc70cedd5482da6a185c6c6b9b9._comment
@@ -0,0 +1,7 @@
+[[!comment format=mdwn
+ username="Tafnzart"
+ subject="comment 1"
+ date="2016-10-08T16:43:54Z"
+ content="""
+Please ignore the previous message. I found my way. Actually I was confused by git annex list thinking it lists remotes, while it does list files too. So I was able to have a directory called dot-bup inside annex without problem. Thank you for great tool!
+"""]]
diff --git a/doc/forum/cabal_install_fails_on_uuid.mdwn b/doc/forum/cabal_install_fails_on_uuid.mdwn
new file mode 100644
index 0000000000..606f7954dc
--- /dev/null
+++ b/doc/forum/cabal_install_fails_on_uuid.mdwn
@@ -0,0 +1,23 @@
+Hey, so I am trying to compile git-annex through cabal but it fails on building uuid. It gives me this strange error:
+
+    Resolving dependencies...
+    Configuring uuid-1.3.1...
+    Building uuid-1.3.1...
+    Preprocessing library uuid-1.3.1...
+    [ 1 of 10] Compiling Data.UUID.Builder ( Data/UUID/Builder.hs, dist/build/Data/UUID/Builder.o )
+    [ 2 of 10] Compiling Data.Word.Util   ( Data/Word/Util.hs, dist/build/Data/Word/Util.o )
+    [ 3 of 10] Compiling Data.UUID.Internal ( Data/UUID/Internal.hs, dist/build/Data/UUID/Internal.o )
+    Data/UUID/Internal.hs:394:20: Not in scope: `BL.fromStrict'
+    Data/UUID/Internal.hs:399:48: Not in scope: `BL.toStrict'
+    cabal: Error: some packages failed to install:
+    git-annex-4.20131002 depends on uuid-1.3.1 which failed to install.
+    uuid-1.3.1 failed during the building phase. The exception was:
+    ExitFailure 1
+
+Any advice on how to proceed?
+
+System is: Debian Wheezy (CrunchBang Waldorf) 64bit
+
+versions are:
+    uuid-dev: 2.20.1-5.3
+    cabal: 1.14.0
diff --git a/doc/forum/cabal_install_fails_on_uuid/comment_1_2a3963e21bc7ff526124b902cb0b6ad2._comment b/doc/forum/cabal_install_fails_on_uuid/comment_1_2a3963e21bc7ff526124b902cb0b6ad2._comment
new file mode 100644
index 0000000000..47fe7e5ff8
--- /dev/null
+++ b/doc/forum/cabal_install_fails_on_uuid/comment_1_2a3963e21bc7ff526124b902cb0b6ad2._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="http://joeyh.name/"
+ ip="64.134.31.139"
+ subject="comment 1"
+ date="2013-10-22T17:52:10Z"
+ content="""
+That version of the haskell uuid library installs ok on my Debian sid system. Seems to me you have a too old version of the haskell bytestring library, which added `fromStrict` in version 0.10.0.0. Probably `cabal install bytestring` will fix it up for you, but building git-annex from source using cabal is a complicated thing that can go wrong in many ways. Have you considered one of the prebuilt binaries?
+"""]]
diff --git a/doc/forum/cabal_install_fails_on_uuid/comment_2_1609525998e2b36c04d67f4d988139c0._comment b/doc/forum/cabal_install_fails_on_uuid/comment_2_1609525998e2b36c04d67f4d988139c0._comment
new file mode 100644
index 0000000000..0f308df0b4
--- /dev/null
+++ b/doc/forum/cabal_install_fails_on_uuid/comment_2_1609525998e2b36c04d67f4d988139c0._comment
@@ -0,0 +1,10 @@
+[[!comment format=mdwn
+ username="https://www.google.com/accounts/o8/id?id=AItOawkj7tMEJKcZpNXIFkHAAcNi5qJFSFyjn6o"
+ nickname="thissideup"
+ subject="comment 2"
+ date="2013-10-23T11:15:20Z"
+ content="""
+I contacted the maintainer of the uuid-package and he fixed the issue in the current release, so everythings fine now. and: everything compiled beautifully. 
+
+But to answer your question: I tend to rather compile things myself if I don't find them (or a current enough version) in the repos - but that's purely personal. ;)
+"""]]
diff --git a/doc/forum/can_I_only_add_my_own_files__63__.mdwn b/doc/forum/can_I_only_add_my_own_files__63__.mdwn
new file mode 100644
index 0000000000..5554295824
--- /dev/null
+++ b/doc/forum/can_I_only_add_my_own_files__63__.mdwn
@@ -0,0 +1,27 @@
+Hi,
+
+[ sorry for the wrong bugreport in the last version of this entry ]
+
+It seems as if I can only add my own file to the annex, even if I have group write permissions.  Is that correct?  Can that be circumvented other than copying/deleting the large file?
+
+Here is some demonstration:
+
+    > git init
+    > git config user.name dtr
+    > git config user.email dtrn@dtrn.com
+    
+    > dd if/dev/zero of=test2.bin count=1000
+    
+    > ll
+    -rw-rw-r-- 1 dtr dtr 512000 Sep 16  2013 test2.bin
+    
+    > sudo chown someone test2.bin
+    > ll
+    -rw-rw-r-- 1 someone dtr 512000 Sep 16  2013 test2.bin
+    
+    > git annex init
+    > git annex add test2.bin
+    add test2.bin (checksum...) 
+    git-annex: /test-git-annex/.git/annex/objects/Fq/f6/SHA256-s512000--2d4da04b861bb9dbe77c871415931785a18138d6db035f1bbcd0cf8277c6fc23/SHA256-s512000--2d4da04b861bb9dbe77c871415931785a18138d6db035f1bbcd0cf8277c6fc23: setFileMode: permission denied (Operation not permitted)
+    failed
+    git-annex: add: 1 failed
diff --git a/doc/forum/can_I_only_add_my_own_files__63__/comment_1_767d647af9d0345f337338d6319071fa._comment b/doc/forum/can_I_only_add_my_own_files__63__/comment_1_767d647af9d0345f337338d6319071fa._comment
new file mode 100644
index 0000000000..80efaf04bd
--- /dev/null
+++ b/doc/forum/can_I_only_add_my_own_files__63__/comment_1_767d647af9d0345f337338d6319071fa._comment
@@ -0,0 +1,10 @@
+[[!comment format=mdwn
+ username="http://joeyh.name/"
+ ip="4.152.108.220"
+ subject="comment 1"
+ date="2013-09-25T18:42:00Z"
+ content="""
+git-annex needs to be able to lock down files to ensure that nobody can write to them, and to do this it needs to remove the write bit, and you can't remove the write bit from a file you don't own.
+
+Note that if you configure git's core.sharedRepository when making a repository (git init --shared), then all files in both git and git-annex will be group writable. Put you and the other person you wanted to be able to write to the file in a group, and you can both access the repository. So that's the right way to do it.
+"""]]
diff --git a/doc/forum/can_I_only_add_my_own_files__63__/comment_2_0c3306ffb38b97b54e1e0436d12c1876._comment b/doc/forum/can_I_only_add_my_own_files__63__/comment_2_0c3306ffb38b97b54e1e0436d12c1876._comment
new file mode 100644
index 0000000000..5a63a71760
--- /dev/null
+++ b/doc/forum/can_I_only_add_my_own_files__63__/comment_2_0c3306ffb38b97b54e1e0436d12c1876._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="ringprince"
+ ip="134.76.140.110"
+ subject="comment 2"
+ date="2013-09-26T06:32:48Z"
+ content="""
+Thanks a lot.  The solution with --shared was really helpful.
+"""]]
diff --git a/doc/forum/can__39__t_sync_content.mdwn b/doc/forum/can__39__t_sync_content.mdwn
new file mode 100644
index 0000000000..9e7a4cc49d
--- /dev/null
+++ b/doc/forum/can__39__t_sync_content.mdwn
@@ -0,0 +1,35 @@
+Hi,
+
+I'm completely new to git-annex, I'm trying to follow this guide:
+
+https://git-annex.branchable.com/tips/centralized_git_repository_tutorial/on_GitLab/
+
+All steps succeeded except the `git annex sync --content` which results in the following message:
+
+'Remote origin not usable by git-annex; setting annex-ignore'
+
+Here is an example of the output I receive:
+
+    $ git status
+    On branch master
+    Your branch is up-to-date with 'origin/master'.
+    nothing to commit, working directory clean
+    $
+    $ git push origin master git-annex
+    Everything up-to-date
+    $
+    $ git annex sync --content
+    
+      Remote origin not usable by git-annex; setting annex-ignore
+    commit  ok
+    pull origin
+    ok
+    pull origin
+    ok
+    $
+    $ git config remote.origin.url
+    https://user:pass@gitlab.com/user/repname.git
+
+Is it required to use the ssh protocol in order to get annex working?
+
+I'm using https since I'm behind a firewall, and the ssh protocol always time out. Is there anything I can do?
diff --git a/doc/forum/can__39__t_sync_content/comment_1_0d6c6f5967c91e7dda55fbf716dd88a0._comment b/doc/forum/can__39__t_sync_content/comment_1_0d6c6f5967c91e7dda55fbf716dd88a0._comment
new file mode 100644
index 0000000000..c851e51f58
--- /dev/null
+++ b/doc/forum/can__39__t_sync_content/comment_1_0d6c6f5967c91e7dda55fbf716dd88a0._comment
@@ -0,0 +1,15 @@
+[[!comment format=mdwn
+ username="joey"
+ subject="""comment 1"""
+ date="2015-08-05T14:04:39Z"
+ content="""
+It's possible to set up a http repo such that git-annex can use it, but
+gitlab's http repos are not set up this way. It might be worth asking them
+to do it. [[tips/setup_a_public_repository_on_a_web_site]] explains
+one way to set up a http repo for git-annex.
+
+But, even when git-annex is able to use a http repo, it can only
+download files from it. There's no support for uploading git-annex managed
+files to a remote http repo. This would need additional server-side support
+of some kind.
+"""]]
diff --git a/doc/forum/can__39__t_sync_content/comment_2_e9c476eeac7427950573c07313a8a175._comment b/doc/forum/can__39__t_sync_content/comment_2_e9c476eeac7427950573c07313a8a175._comment
new file mode 100644
index 0000000000..9b9d7a439d
--- /dev/null
+++ b/doc/forum/can__39__t_sync_content/comment_2_e9c476eeac7427950573c07313a8a175._comment
@@ -0,0 +1,10 @@
+[[!comment format=mdwn
+ username="dberebi@ca13ee3cf0ff9c61fdfea562ebf725e0b6dc40ef"
+ nickname="dberebi"
+ subject="comment 2"
+ date="2015-08-06T01:51:03Z"
+ content="""
+It doesn't solve my problem but at least now I know there is no way to do that behind such a firewall.
+
+Thanks for your reply!
+"""]]
diff --git a/doc/forum/can_git-annex_replace_ddm__63__.mdwn b/doc/forum/can_git-annex_replace_ddm__63__.mdwn
new file mode 100644
index 0000000000..8d49652c3d
--- /dev/null
+++ b/doc/forum/can_git-annex_replace_ddm__63__.mdwn
@@ -0,0 +1,13 @@
+Hi,
+a few years ago I wrote a tool called 'ddm'.  The code is overengineered and the script is more complicated then it should be,
+but I think it demonstrates some good use cases, and I wonder how well git-annex can fulfill the requirements for those use cases - maybe I should remove ddm and start hacking with git-annex instead.
+
+To answer this question, you should read the section about the possible dataset types on http://dieter.plaetinck.be/ddm_a_distributed_data_manager.html, and the example at the bottom of that page. it demonstrates the idea behind the "selection" dataset to always try to keep a subset (the most appropriate, based on the output of some script) of files "checked out".
+the introduction section on https://github.com/Dieterbe/ddm/raw/358f7cf92c0ba7b336dc97638351d4e324461afa/MANUAL should further clarify things, as well as give some more good use cases (as you can see it's a bit more about [semi-]automated workflows then purely tracking what's where)
+
+So I'm not sure, maybe the way to go for me is to make git-annex my "housekeeping about which data is where" backend and make ddm into a set of policies and tools on top of git-annex.
+
+Any input?
+
+Thanks,
+Dieter
diff --git a/doc/forum/can_git-annex_replace_ddm__63__/comment_1_aa05008dfe800474ff76678a400099e1._comment b/doc/forum/can_git-annex_replace_ddm__63__/comment_1_aa05008dfe800474ff76678a400099e1._comment
new file mode 100644
index 0000000000..eb824971f6
--- /dev/null
+++ b/doc/forum/can_git-annex_replace_ddm__63__/comment_1_aa05008dfe800474ff76678a400099e1._comment
@@ -0,0 +1,12 @@
+[[!comment format=mdwn
+ username="http://joey.kitenet.net/"
+ nickname="joey"
+ subject="comment 1"
+ date="2011-02-14T22:08:54Z"
+ content="""
+Yes, there is value in layering something over git-annex to use a policy to choose what goes where.
+
+I use [mr](http://kitenet.net/~joey/code/mr/) to update and manage all my repositories, and since mr can be made to run arbitrary commands when doing eg, an update, I use its config file as such a policy layer. For example, my podcasts are pulled into my sound repository in a subdirectory; boxes that consume podcasts run \"git pull; git annex get podcasts --exclude=\"*/out/*\"; git annex drop podcasts/*/out\". I move podcasts to \"out\" directories once done with them (I have yet to teach mpd to do that for me..), and the next time I run \"mr update\" to update everything, it pulls down new ones and removes old ones.
+
+I don't see any obstacle to doing what you want. May be that you'd need better querying facilities in git-annex (so the policy layer can know what is available where), or finer control (--exclude is a good enough hammer for me, but maybe not for you).
+"""]]
diff --git a/doc/forum/can_git-annex_replace_ddm__63__/comment_2_008554306dd082d7f543baf283510e92._comment b/doc/forum/can_git-annex_replace_ddm__63__/comment_2_008554306dd082d7f543baf283510e92._comment
new file mode 100644
index 0000000000..ab114bb1c8
--- /dev/null
+++ b/doc/forum/can_git-annex_replace_ddm__63__/comment_2_008554306dd082d7f543baf283510e92._comment
@@ -0,0 +1,19 @@
+[[!comment format=mdwn
+ username="http://dieter-be.myopenid.com/"
+ nickname="dieter"
+ subject="comment 2"
+ date="2011-02-16T21:32:04Z"
+ content="""
+thanks Joey,
+
+is it possible to run some git annex command that tells me, for a specific directory, which files are available in an other remote? (and which remote, and which filenames?)
+I guess I could run that, do my own policy thingie, and run `git annex get` for the files I want.
+
+For your podcast use case (and some of my use cases) don't you think git [annex] might actually be overkill?  For example your podcasts use case, what value does git annex give over a simple rsync/rm script?
+such a script wouldn't even need a data store to store its state, unlike git. it seems simpler and cleaner to me.
+
+for the mpd thing, check http://alip.github.com/mpdcron/ (bad project name, it's a plugin based \"event handler\")
+you should be able to write a simple plugin for mpdcron that does what you want (or even interface with mpd yourself from perl/python/.. to use its idle mode to get events)
+
+Dieter
+"""]]
diff --git a/doc/forum/can_git-annex_replace_ddm__63__/comment_3_4c69097fe2ee81359655e59a03a9bb8d._comment b/doc/forum/can_git-annex_replace_ddm__63__/comment_3_4c69097fe2ee81359655e59a03a9bb8d._comment
new file mode 100644
index 0000000000..5cdd6aa0c6
--- /dev/null
+++ b/doc/forum/can_git-annex_replace_ddm__63__/comment_3_4c69097fe2ee81359655e59a03a9bb8d._comment
@@ -0,0 +1,12 @@
+[[!comment format=mdwn
+ username="http://joey.kitenet.net/"
+ nickname="joey"
+ subject="comment 3"
+ date="2011-03-16T03:01:17Z"
+ content="""
+Whups, the comment above got stuck in moderation queue for 27 days. I will try to check that more frequently.
+
+In the meantime, I've implemented \"git annex whereis\" -- enjoy!
+
+I find keeping my podcasts in the annex useful because it allows me to download individual episodes or poscasts easily when low bandwidth is available (ie, dialup), or over sneakernet. And generally keeps everything organised.
+"""]]
diff --git a/doc/forum/canceling_wrong_repository_merge.mdwn b/doc/forum/canceling_wrong_repository_merge.mdwn
new file mode 100644
index 0000000000..e2d7009c0f
--- /dev/null
+++ b/doc/forum/canceling_wrong_repository_merge.mdwn
@@ -0,0 +1,3 @@
+so i mistakenly merged two unrelated repos together. i have "canceled" the merge as in reverted it by removing the created files, but then those zillion of small files will stick around the git repository forever.
+
+is there a way to use something like `git annex forget` for this? i know about things like [git rebase --onto](https://sethrobertson.github.io/GitFixUm/fixup.html#remove_deep) but that won't propagate across all repositories... can git-annex give me a hand here? --[[anarcat]]
diff --git a/doc/forum/canceling_wrong_repository_merge/comment_1_7230f2be014afcb40514ef521d53c170._comment b/doc/forum/canceling_wrong_repository_merge/comment_1_7230f2be014afcb40514ef521d53c170._comment
new file mode 100644
index 0000000000..4f82b38e67
--- /dev/null
+++ b/doc/forum/canceling_wrong_repository_merge/comment_1_7230f2be014afcb40514ef521d53c170._comment
@@ -0,0 +1,7 @@
+[[!comment format=mdwn
+ username="https://id.koumbit.net/anarcat"
+ subject="comment 1"
+ date="2015-02-11T15:05:32Z"
+ content="""
+it turns out that using [this procedure](https://sethrobertson.github.io/GitFixUm/fixup.html#remove_deep) worked for me. i just had to reproduce it on all remotes.
+"""]]
diff --git a/doc/forum/canceling_wrong_repository_merge/comment_2_87aabff41a1c6aec773b8f52ead51105._comment b/doc/forum/canceling_wrong_repository_merge/comment_2_87aabff41a1c6aec773b8f52ead51105._comment
new file mode 100644
index 0000000000..83d5c4834a
--- /dev/null
+++ b/doc/forum/canceling_wrong_repository_merge/comment_2_87aabff41a1c6aec773b8f52ead51105._comment
@@ -0,0 +1,11 @@
+[[!comment format=mdwn
+ username="https://id.koumbit.net/anarcat"
+ subject="watch out for direct mode"
+ date="2015-02-16T23:22:19Z"
+ content="""
+so while `git rebase` can do magic, it will not work out of the box on direct mode repositories, unless you use `-c core.bare=false`, in which case you will totally shoot yourself in the foot because git will happily remove all those real files sitting in the checkout. you will need to `git annex indirect` before you do any of that magic. working on a clone of the git repo is also a good idea, if only for testing.
+
+i personnally destroyed my whole music collection doing such a cleanup of the history. fortunately, i had a recent archived clone of the repo, so things weren't so bad.
+
+but watch out for direct mode, as always.
+"""]]
diff --git a/doc/forum/canceling_wrong_repository_merge/comment_3_d6e45d7e4f4bdf0a08ab91a08e0c1be6._comment b/doc/forum/canceling_wrong_repository_merge/comment_3_d6e45d7e4f4bdf0a08ab91a08e0c1be6._comment
new file mode 100644
index 0000000000..1fbec86033
--- /dev/null
+++ b/doc/forum/canceling_wrong_repository_merge/comment_3_d6e45d7e4f4bdf0a08ab91a08e0c1be6._comment
@@ -0,0 +1,37 @@
+[[!comment format=mdwn
+ username="https://id.koumbit.net/anarcat"
+ subject="the actual process i use"
+ date="2015-02-17T00:58:38Z"
+ content="""
+So it seems i am able to forget all of this within the matter of a few days, and since this is so error prone, here goes a more detailed explanation.
+
+What I do is:
+
+
    +git clone repo repo.test
+cd repo.test
+git annex indirect # be safe! this may take a while, but it's necessary!
+git tag bak # keep track of a good working state
+git log --stat --stat-count=3 # find the commits we want to trash
+git tag firstbad badbeef1 # the first commit we want to kill
+git tag keep dada1234 # the first commit we want to keep
+git rebase -p --onto firstbad^ keep # drop everything between firstbad (inclusive) and keep (exclusive)
+git diff --stat keep # make sure this did what we expected
+git branch -D annex/direct/master synced/master # destroy this old branch that still has refs to the old commits
+
    
+
+Then for each repo:
+
+
    +cd repo
+git tag bak
+git fetch origin # sync the master branch in
+git remote prune origin # make sure the dropped branches are gone
+git annex indirect # be safe
+git reset --hard origin/master
+git branch -D synced/master annex/direct/master
+git diff --stat bak # should change
+
    
+
+It would be useful to have that transition propagate properly everywhere so I don't have to do this in every repo, but at least the above should work fairly reliably.
+"""]]
diff --git a/doc/forum/cannot_fetch_from___40__gpg-encrypted__41___rsync_remote_on_MacOSX.mdwn b/doc/forum/cannot_fetch_from___40__gpg-encrypted__41___rsync_remote_on_MacOSX.mdwn
new file mode 100644
index 0000000000..59ca2e9fc7
--- /dev/null
+++ b/doc/forum/cannot_fetch_from___40__gpg-encrypted__41___rsync_remote_on_MacOSX.mdwn
@@ -0,0 +1,18 @@
+I have a remote rsync with gpg encryption and can restore without problems on my thinkpad (FreeBSD) - but not on my MacOSX:
+
+$ git annex whereis DSC_7615.JPG
+whereis DSC_7615.JPG (2 copies) 
+  	6855de17-c8fb-11e1-9948-f0def1c18073 -- thinkpad
+   	e388bcf6-c8fc-11e1-a96f-6ffcbceb4af4 -- backup (rsync xxxx)
+ok
+
+$ git annex get --from backup DSC_7615.JPG
+fatal: Could not switch to '../../../.git/annex/objects/Pw/XP/SHA256-s1028494--SHA256': No such file or directory
+
+git-annex: : hGetLine: end of file
+failed
+git-annex: get: 1 failed
+
+$ git annex version
+git-annex version: 3.20120629
+local repository version: 3
diff --git a/doc/forum/cannot_fetch_from___40__gpg-encrypted__41___rsync_remote_on_MacOSX/comment_1_21f0101447623f5a0cf9e72c3ff463bb._comment b/doc/forum/cannot_fetch_from___40__gpg-encrypted__41___rsync_remote_on_MacOSX/comment_1_21f0101447623f5a0cf9e72c3ff463bb._comment
new file mode 100644
index 0000000000..c6c962c403
--- /dev/null
+++ b/doc/forum/cannot_fetch_from___40__gpg-encrypted__41___rsync_remote_on_MacOSX/comment_1_21f0101447623f5a0cf9e72c3ff463bb._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="https://www.google.com/accounts/o8/id?id=AItOawmBUR4O9mofxVbpb8JV9mEbVfIYv670uJo"
+ nickname="Justin"
+ subject="comment 1"
+ date="2012-08-07T13:11:48Z"
+ content="""
+Do the remotes in `.git/config` look the same?
+"""]]
diff --git a/doc/forum/cannot_fetch_from___40__gpg-encrypted__41___rsync_remote_on_MacOSX/comment_2_6234ca64bd03a0e15efbe8f5c204338a._comment b/doc/forum/cannot_fetch_from___40__gpg-encrypted__41___rsync_remote_on_MacOSX/comment_2_6234ca64bd03a0e15efbe8f5c204338a._comment
new file mode 100644
index 0000000000..5ae18235b8
--- /dev/null
+++ b/doc/forum/cannot_fetch_from___40__gpg-encrypted__41___rsync_remote_on_MacOSX/comment_2_6234ca64bd03a0e15efbe8f5c204338a._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="hannes"
+ ip="130.226.133.37"
+ subject="they do"
+ date="2012-08-07T14:06:55Z"
+ content="""
+I was also able to copy from the macosx to the rsync backup...
+"""]]
diff --git a/doc/forum/cannot_fetch_from___40__gpg-encrypted__41___rsync_remote_on_MacOSX/comment_3_5ac2b520a907e232984eb513ce088054._comment b/doc/forum/cannot_fetch_from___40__gpg-encrypted__41___rsync_remote_on_MacOSX/comment_3_5ac2b520a907e232984eb513ce088054._comment
new file mode 100644
index 0000000000..d80a01a0b2
--- /dev/null
+++ b/doc/forum/cannot_fetch_from___40__gpg-encrypted__41___rsync_remote_on_MacOSX/comment_3_5ac2b520a907e232984eb513ce088054._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="hannes"
+ ip="130.226.133.37"
+ subject="also, fsck works"
+ date="2012-08-07T14:12:20Z"
+ content="""
+git annex fsck -fbackup some-local-and-remote-file works fine.
+"""]]
diff --git a/doc/forum/cannot_fetch_from___40__gpg-encrypted__41___rsync_remote_on_MacOSX/comment_4_183dd1c29f66539193e7c0b73f329430._comment b/doc/forum/cannot_fetch_from___40__gpg-encrypted__41___rsync_remote_on_MacOSX/comment_4_183dd1c29f66539193e7c0b73f329430._comment
new file mode 100644
index 0000000000..6ab9ebb259
--- /dev/null
+++ b/doc/forum/cannot_fetch_from___40__gpg-encrypted__41___rsync_remote_on_MacOSX/comment_4_183dd1c29f66539193e7c0b73f329430._comment
@@ -0,0 +1,9 @@
+[[!comment format=mdwn
+ username="http://joeyh.name/"
+ subject="comment 4"
+ date="2012-08-07T17:53:35Z"
+ content="""
+I recently encountered this problem on OSX, and I'm pretty sure I fixed it. But I don't remember the details of how. I'm sorry I don't have more detail, but I recommend trying a newer version of git-annex. 
+
+And if the problem persists, --debug will show the git commands and should help identify the one that's being passed bad parameters.
+"""]]
diff --git a/doc/forum/cannot_fetch_from___40__gpg-encrypted__41___rsync_remote_on_MacOSX/comment_5_c920d04ffe332caed9d223fa0ac42746._comment b/doc/forum/cannot_fetch_from___40__gpg-encrypted__41___rsync_remote_on_MacOSX/comment_5_c920d04ffe332caed9d223fa0ac42746._comment
new file mode 100644
index 0000000000..32420ac661
--- /dev/null
+++ b/doc/forum/cannot_fetch_from___40__gpg-encrypted__41___rsync_remote_on_MacOSX/comment_5_c920d04ffe332caed9d223fa0ac42746._comment
@@ -0,0 +1,7 @@
+[[!comment format=mdwn
+ username="http://joeyh.name/"
+ subject="remember now"
+ date="2012-08-07T19:37:22Z"
+ content="""
+This was a bug in determining the git version at compile time. Fixed in commit bafc50e05e098234d2d22886085d9844fc763e0e which was included in version 3.20120721.
+"""]]
diff --git a/doc/forum/cannot_fetch_from___40__gpg-encrypted__41___rsync_remote_on_MacOSX/comment_6_7a3cf0853a8ec7b996e19b5e80145d21._comment b/doc/forum/cannot_fetch_from___40__gpg-encrypted__41___rsync_remote_on_MacOSX/comment_6_7a3cf0853a8ec7b996e19b5e80145d21._comment
new file mode 100644
index 0000000000..9ffb376a71
--- /dev/null
+++ b/doc/forum/cannot_fetch_from___40__gpg-encrypted__41___rsync_remote_on_MacOSX/comment_6_7a3cf0853a8ec7b996e19b5e80145d21._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="hannes"
+ ip="130.226.133.37"
+ subject="awesome, thanks"
+ date="2012-08-08T12:25:19Z"
+ content="""
+works with an updated git-annex!
+"""]]
diff --git a/doc/forum/cannot_sync___39__unused__39___files.mdwn b/doc/forum/cannot_sync___39__unused__39___files.mdwn
new file mode 100644
index 0000000000..cf9ca1d2a2
--- /dev/null
+++ b/doc/forum/cannot_sync___39__unused__39___files.mdwn
@@ -0,0 +1,45 @@
+Hi,
+
+I have setup a remote git-annex repository (ssh into some machine in our local network) which I will call "common_repo".  Multiple contributors will then be cloning this repo into their laptops (I will call "client_repos").
+
+When these client_repos change and do a "git annex sync  --content", all the files from their laptops are successfully synchronised.  The problem arises when:
+
+
      
+  
    1. CLIENT_A creates somefile.csv
      2. 
+  
    2. CLIENT_A commits somefile.csv
+    
      i.e.:
+    
          git annex add somefile.csv
+    
          git commit -a -m "uploaded somefile.csv"
+  
      3. 
+  
    3. CLIENT_A modifies somefile.csv
      4. 
+  
    4.  CLIENT_A commits modified somefile.csv
+     
      i.e.:
+     
          git add somefile.csv
+     
          git commit -a -m "updated somefile.csv"
+  
      5. 
+  
    5. CLIENT_A synchronises with common_repo
+     
      i.e.: git annex sync  --content
+  
      6. 
+
    
+
+If I check the .git/annex/objects of common_repo, I can't seem to find a copy of the unmodified somefile.csv.  It only has a copy of the latest somefile.csv.
+
+This is problematic if one client tries to checkout a revision of the project that uses the original somefile.csv.
+
+I learned that I can change the preferred content of git-annex.  So, the appropriate preset for common_repo seemed to be "backup".  After running the commands in common_repo:
+
    
+git annex wanted . standard
    
+git annex group . backup
+
    
+
+I've done another test of the scenario above, and common_repo is still missing the previous revision of the file!!  The preferred file content of common_repo should be "include=* or unused".  In my case, the previous version of somefile.csv will probably fall in the "unused" category.  But I still cannot find it.
+
+A workaround is using two commands from the client(s):
+
    
+git annex copy --to  --all
    
+git annex sync  --content
+
    
+
+But I can imagine my users forgetting to run 'copy' and my repo will go to shit over time.
+
+Any ideas why I can't synchronise 'unused' files?
diff --git a/doc/forum/cannot_sync___39__unused__39___files/comment_1_d7c88a5cdf8ce8206795ff7ab215d372._comment b/doc/forum/cannot_sync___39__unused__39___files/comment_1_d7c88a5cdf8ce8206795ff7ab215d372._comment
new file mode 100644
index 0000000000..51fc740cf0
--- /dev/null
+++ b/doc/forum/cannot_sync___39__unused__39___files/comment_1_d7c88a5cdf8ce8206795ff7ab215d372._comment
@@ -0,0 +1,9 @@
+[[!comment format=mdwn
+ username="joey"
+ subject="""comment 1"""
+ date="2015-06-15T17:56:45Z"
+ content="""
+The reason is that `git annex sync --content` only syncs files in the
+current work tree.
+
+"""]]
diff --git a/doc/forum/central_non-bare_and_git_push.mdwn b/doc/forum/central_non-bare_and_git_push.mdwn
new file mode 100644
index 0000000000..ecc81096ed
--- /dev/null
+++ b/doc/forum/central_non-bare_and_git_push.mdwn
@@ -0,0 +1,9 @@
+hi,
+
+i have a usecase that i think many people have. a cental server which should be non-bare to be able to browse the files also via webdav.
+multiple clients behind nat so only pushes via xmpp are possible.
+
+i set everything up without xmpp, it works but if files are updated, none of the clients gets a git push of course because ssh works only unidirectional.
+i couldnt figure out how to set up xmpp push and have a non-bare central repo at the same time because i have to choose between ssh and xmpp git remote on the clients to the server.
+
+thanks!
diff --git a/doc/forum/central_non-bare_and_git_push/comment_1_76d0c73c8985e860eb86333c63be6340._comment b/doc/forum/central_non-bare_and_git_push/comment_1_76d0c73c8985e860eb86333c63be6340._comment
new file mode 100644
index 0000000000..6bf3d1a5e7
--- /dev/null
+++ b/doc/forum/central_non-bare_and_git_push/comment_1_76d0c73c8985e860eb86333c63be6340._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="http://joeyh.name/"
+ ip="2001:4978:f:21a::2"
+ subject="comment 1"
+ date="2013-07-25T17:05:00Z"
+ content="""
+Simply set up xmpp on your clients in addition to the central repository you have now. When one client pushes over ssh to the central repo, it will use XMPP to inform other clients of the change.
+"""]]
diff --git a/doc/forum/checkout_view_to_directory_outside_of_annex.mdwn b/doc/forum/checkout_view_to_directory_outside_of_annex.mdwn
new file mode 100644
index 0000000000..5a198b64ea
--- /dev/null
+++ b/doc/forum/checkout_view_to_directory_outside_of_annex.mdwn
@@ -0,0 +1,23 @@
+i am working on a project where i need to copy many subsets of files stored in a single annex repo to separate folders outside of the annex repo (each subset to a separate folder).
+
+to select each subset, i'm using `git annex metadata --set = `.
+
+what i would like to do, ideally, is to leverage each `git annex view =` invocation to directly check out the view's files to a specific directory outside of the annex repo.
+
+the use case is basically the "Copying objects" strategy of the [syncthing special remote discussion](http://git-annex.branchable.com/todo/syncthing_special_remote/), although rather than having the 'directory' special remote contain files in that remote's specific layout, i would like them to be checked out simply with their original file names.
+
+say i have these files in my main annex repo:
+
+    a.pdf (metadata: topic=haskell)
+    b.pdf (metadata: topic=haskell)
+    c.pdf (metadata: topic=iojs)
+    d.pdf (metadata: topic=python)
+    e.pdf (metadata: topic=haskell)
+
+if i issue `git annex view topic=haskell`, i have the files i want in my annex' root:
+
+    a.pdf
+    b.pdf
+    e.pdf
+
+obviously i could then simply run `rsync --exclude .git -aL --delete . ../other/dir`, which is totally fine, but maybe i'm just blindly missing something obvious and i could simply use something like `git annex --work-tree=../other/dir view topic=haskell` and see `a.pdf`, `b.pdf` and `e.pdf` appear in the target directory (i don't need any metadata in that directory, so only the plain files and no .git folder for a remote is fine).
diff --git a/doc/forum/checkout_view_to_directory_outside_of_annex/comment_1_a31a501bdc28eabc60588502f7219e50._comment b/doc/forum/checkout_view_to_directory_outside_of_annex/comment_1_a31a501bdc28eabc60588502f7219e50._comment
new file mode 100644
index 0000000000..a0d1f4dab7
--- /dev/null
+++ b/doc/forum/checkout_view_to_directory_outside_of_annex/comment_1_a31a501bdc28eabc60588502f7219e50._comment
@@ -0,0 +1,20 @@
+[[!comment format=mdwn
+ username="joey"
+ subject="""comment 1"""
+ date="2015-08-11T16:57:45Z"
+ content="""
+Well, you can certianly use --work-tree with git-annex. As a global git
+option, it has to be passed after the "git" and before the "annex". However,
+since the view is constructed by making symlinks to the annexed content,
+the result will be a directory full of symlinks, rather than the contents
+of the files. Also, it'll still update the repo  to think it has this view
+checked out, with confusing results.
+
+rsync seems to be a perfectly good way to do what you want, if you want
+to have copies of the files.
+
+Or, you might consider making a shared clone of the git repo wherever you 
+want that tree of files, and checking out the desired view in the clone.
+In a shared clone with a recent version of git-annex, running `git annex
+get` will quickly hard-link the files from the main repo (when possible).
+"""]]
diff --git a/doc/forum/checkout_view_to_directory_outside_of_annex/comment_2_b4abbb048ebcd3c4b59d20e103948fd8._comment b/doc/forum/checkout_view_to_directory_outside_of_annex/comment_2_b4abbb048ebcd3c4b59d20e103948fd8._comment
new file mode 100644
index 0000000000..fecf397411
--- /dev/null
+++ b/doc/forum/checkout_view_to_directory_outside_of_annex/comment_2_b4abbb048ebcd3c4b59d20e103948fd8._comment
@@ -0,0 +1,7 @@
+[[!comment format=mdwn
+ username="https://mey.vn/"
+ subject="comment 2"
+ date="2015-08-12T10:54:29Z"
+ content="""
+thanks @joey - the shared clone strategy looks perfect as a way to leave the main repo alone while enjoying all the benefits of views.
+"""]]
diff --git a/doc/forum/checkout_view_to_directory_outside_of_annex/comment_3_2139407a3a556023a2458148a4a2c009._comment b/doc/forum/checkout_view_to_directory_outside_of_annex/comment_3_2139407a3a556023a2458148a4a2c009._comment
new file mode 100644
index 0000000000..b6e48b4ece
--- /dev/null
+++ b/doc/forum/checkout_view_to_directory_outside_of_annex/comment_3_2139407a3a556023a2458148a4a2c009._comment
@@ -0,0 +1,12 @@
+[[!comment format=mdwn
+ username="konubinix"
+ subject="git-new-workdir"
+ date="2015-08-15T12:48:30Z"
+ content="""
+I am not sure whether it would help, but to show views of my git-annex repositories without messing with the current working directory, I generally use git-new-workdir
+https://github.com/git/git/blob/master/contrib/workdir/git-new-workdir
+
+It allows to checkout a separate branch in another directory. The new directory's .git directory contains symbolic links to the parent one, making sure that changes in the new work dir are propagated to the parent one.
+
+Hope that helps.
+"""]]
diff --git a/doc/forum/clear_box.com_repository.mdwn b/doc/forum/clear_box.com_repository.mdwn
new file mode 100644
index 0000000000..0de67cf623
--- /dev/null
+++ b/doc/forum/clear_box.com_repository.mdwn
@@ -0,0 +1 @@
+Using webapp I cannot get rid of box.com repository, it is locked in 'cleaning out' state for more than two weeks. How can I remove the repository from the list?
diff --git a/doc/forum/clear_box.com_repository/comment_1_2e839d8f974269c80a9fca183712350f._comment b/doc/forum/clear_box.com_repository/comment_1_2e839d8f974269c80a9fca183712350f._comment
new file mode 100644
index 0000000000..939e9b8938
--- /dev/null
+++ b/doc/forum/clear_box.com_repository/comment_1_2e839d8f974269c80a9fca183712350f._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="http://joeyh.name/"
+ nickname="joey"
+ subject="comment 1"
+ date="2013-06-14T16:21:54Z"
+ content="""
+We need to first understand why it's not finishing the process. Does the box.com repository still contain files? You can find out by running the command: `git annex find --in box.com` (substitute the name of your remote for box.com as needed).
+"""]]
diff --git a/doc/forum/clear_box.com_repository/comment_2_8f9c7248a148a24ae2aba39c4a79a6d1._comment b/doc/forum/clear_box.com_repository/comment_2_8f9c7248a148a24ae2aba39c4a79a6d1._comment
new file mode 100644
index 0000000000..9725c96e5a
--- /dev/null
+++ b/doc/forum/clear_box.com_repository/comment_2_8f9c7248a148a24ae2aba39c4a79a6d1._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="tomas"
+ ip="188.167.111.235"
+ subject="comment 2"
+ date="2013-06-16T13:32:31Z"
+ content="""
+The result of that command returned without writing any files.
+"""]]
diff --git a/doc/forum/clear_box.com_repository/comment_3_f64ad21e5abfbf4e1f925b3d651bdba3._comment b/doc/forum/clear_box.com_repository/comment_3_f64ad21e5abfbf4e1f925b3d651bdba3._comment
new file mode 100644
index 0000000000..6b19da5c7d
--- /dev/null
+++ b/doc/forum/clear_box.com_repository/comment_3_f64ad21e5abfbf4e1f925b3d651bdba3._comment
@@ -0,0 +1,10 @@
+[[!comment format=mdwn
+ username="http://joeyh.name/"
+ nickname="joey"
+ subject="comment 3"
+ date="2013-06-17T17:34:04Z"
+ content="""
+In that case, I'd expect the assistant will notice it can remove the remote on the next full scan, and prompt you to finish the removal. Full scans are run in a few situations, but the only guaranteed way to run one is to stop and restart the assistant.
+
+You could also enable debug mode when starting it, I'm sure the log would be useful if it somehow doesn't behave how it should.
+"""]]
diff --git a/doc/forum/clear_box.com_repository/comment_4_f8c06ac9b23b51cf18d362c260fc47a9._comment b/doc/forum/clear_box.com_repository/comment_4_f8c06ac9b23b51cf18d362c260fc47a9._comment
new file mode 100644
index 0000000000..a36ca12370
--- /dev/null
+++ b/doc/forum/clear_box.com_repository/comment_4_f8c06ac9b23b51cf18d362c260fc47a9._comment
@@ -0,0 +1,13 @@
+[[!comment format=mdwn
+ username="tomas"
+ ip="188.167.111.235"
+ subject="comment 4"
+ date="2013-06-17T20:17:30Z"
+ content="""
+Hi, I cannot enable debug logging via webapp, because after restart the checkbox is unchecked again. I saw nothing at all in logs after restart, the box repo is still hanging there. I would prefer using command line to remove the last traces of this repository. 
+Another similar question - how do I unassociate two repositories? The web app warns me that about deleting repo, not sure if it would also wipe it out. Is there a CLI command to add and remove the associations (git remote rm?)? Overall great work, but I am lost with the webapp. It is good for learning the concepts, but I would love to know which commands it actually executes on the backend to make the gui -> cli transition easier.
+
+Version: 4.20130516.1
+Build flags: Assistant Webapp Pairing Testsuite S3 WebDAV Inotify DBus XMPP
+
+"""]]
diff --git a/doc/forum/clear_box.com_repository/comment_5_61d401b29322802cb25896503f3e6514._comment b/doc/forum/clear_box.com_repository/comment_5_61d401b29322802cb25896503f3e6514._comment
new file mode 100644
index 0000000000..122e054a93
--- /dev/null
+++ b/doc/forum/clear_box.com_repository/comment_5_61d401b29322802cb25896503f3e6514._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="http://joeyh.name/"
+ nickname="joey"
+ subject="comment 5"
+ date="2013-06-17T23:53:24Z"
+ content="""
+Use the --debug switch to enable debugging on startup.
+"""]]
diff --git a/doc/forum/cloud_services_to_support.mdwn b/doc/forum/cloud_services_to_support.mdwn
new file mode 100644
index 0000000000..4d6bde1b54
--- /dev/null
+++ b/doc/forum/cloud_services_to_support.mdwn
@@ -0,0 +1,16 @@
+git-annex can already be used to store data in several cloud services:
+Amazon S3, rsync.net, Tahoe-LAFS, The Internet Archive.
+
+I would like to support as many other cloud services as possible/reasonable.
+
+* [[swift|todo/wishlist:_swift_backend]]
+* Dropbox (I had been reluctant to go there due to it using a non-free client,
+  which I have no interest in installing, but there is actually an API,
+  and already a
+  [haskell module to use it](http://hackage.haskell.org/package/dropbox-sdk).
+  Would need to register for an API key.
+  .
+  Annoyingly, Dropbox reviews each app before granting it production status.
+  Whoops my interest level dropped by 99%.)
+
+Post others in the comments. --[[Joey]]
diff --git a/doc/forum/cloudcmd.mdwn b/doc/forum/cloudcmd.mdwn
new file mode 100644
index 0000000000..a572abf1ca
--- /dev/null
+++ b/doc/forum/cloudcmd.mdwn
@@ -0,0 +1,6 @@
+Hi,
+
+Glad to see this project appears to be doing well.  I just wanted to mention that there's a similar project called cloudcmd that has many of the same use cases in spirit: https://github.com/briangu/cloudcmd
+
+Cheers,
+Brian
diff --git a/doc/forum/commit_current_workdir_state_in_direct_mode.mdwn b/doc/forum/commit_current_workdir_state_in_direct_mode.mdwn
new file mode 100644
index 0000000000..63945f7901
--- /dev/null
+++ b/doc/forum/commit_current_workdir_state_in_direct_mode.mdwn
@@ -0,0 +1,5 @@
+I'd like to commit the current workdir state in direct mode in a script. This is what git annex watch does but I can not reliably start and stop git annex watch in a cron script.
+
+Is this already possible with the current api or could we have something like git annex watch --once?
+
+Thank you, Thomas Koch
diff --git a/doc/forum/commit_current_workdir_state_in_direct_mode/comment_1_748481ff00374f570284bd4571584874._comment b/doc/forum/commit_current_workdir_state_in_direct_mode/comment_1_748481ff00374f570284bd4571584874._comment
new file mode 100644
index 0000000000..50bd6661ce
--- /dev/null
+++ b/doc/forum/commit_current_workdir_state_in_direct_mode/comment_1_748481ff00374f570284bd4571584874._comment
@@ -0,0 +1,10 @@
+[[!comment format=mdwn
+ username="http://joeyh.name/"
+ ip="4.154.0.21"
+ subject="comment 1"
+ date="2013-07-31T16:25:34Z"
+ content="""
+It should be sufficient to run: `git annex add; git annex sync`
+
+That adds any new files and then commits those along with any changed or deleted files.
+"""]]
diff --git a/doc/forum/compiling_on_MacOS.mdwn b/doc/forum/compiling_on_MacOS.mdwn
new file mode 100644
index 0000000000..c6732e3643
--- /dev/null
+++ b/doc/forum/compiling_on_MacOS.mdwn
@@ -0,0 +1,3 @@
+When compiling on MacOS using Stack, I get errors related to the iconv library:
+https://travis-ci.org/conda-forge/staged-recipes/builds/315444536?utm_source=github_status&utm_medium=notification
+Any suggestions on how to fix this?
diff --git a/doc/forum/compiling_on_MacOS/comment_1_143ae6d4f2704d4ab74e110a2d7a3c82._comment b/doc/forum/compiling_on_MacOS/comment_1_143ae6d4f2704d4ab74e110a2d7a3c82._comment
new file mode 100644
index 0000000000..93911b784a
--- /dev/null
+++ b/doc/forum/compiling_on_MacOS/comment_1_143ae6d4f2704d4ab74e110a2d7a3c82._comment
@@ -0,0 +1,17 @@
+[[!comment format=mdwn
+ username="joey"
+ subject="""comment 1"""
+ date="2017-12-31T16:41:06Z"
+ content="""
+I can only give the same anwser I gave in
+,
+that it seems to be using the wrong version of the iconv library, and that
+the `LIBRARY_PATH`, `LDFLAGS` etc settings your using seem likely to be
+causing the problem.
+
+I mean, you're simply using haskell's stack tool, you've not started to
+build git-annex at all at this point. If haskell's stack is really this
+broken on OSX, I'd think that a lot of people would have noticed. But if
+you can reproduce the problem without any strange environment variables
+being set, why not file a bug report at .
+"""]]
diff --git a/doc/forum/comprehension_question__58___repository_vs._working_copy_in_direct_mode.mdwn b/doc/forum/comprehension_question__58___repository_vs._working_copy_in_direct_mode.mdwn
new file mode 100644
index 0000000000..350bcd7bd2
--- /dev/null
+++ b/doc/forum/comprehension_question__58___repository_vs._working_copy_in_direct_mode.mdwn
@@ -0,0 +1,10 @@
+(I put this in a separate topic because it is a different question)
+
+I still do not fully understand when (in direct mode!) a file is in the work copy and when in the repository.
+Because what git-annex always refers to a "repository" is actually a repository (i.e., the server part for subversion) *and* a work copy (the client part in subversion), right? (except for bare repositories which have only the former)
+
+When I change a file in the work copy, I still need to "git annex sync", right? But do I also need to "git annex sync --content" to move the changes from the work copy to the repository?
+
+Because I have a headless server where the annex should also be accessed via samba. When files are added/changed, the annex clients do not see the changes until I manually do a couple of "git annex sync" and "git annex sync --content" (it's not deterministic yet for me how often).
+
+Is this the intended behavior? If yes, is there a better way to automatically sync the changes on the headless server other than a cron job?
diff --git a/doc/forum/comprehension_question__58___repository_vs._working_copy_in_direct_mode/comment_1_a6b4db0cefa439f72b97089d48dfacbd._comment b/doc/forum/comprehension_question__58___repository_vs._working_copy_in_direct_mode/comment_1_a6b4db0cefa439f72b97089d48dfacbd._comment
new file mode 100644
index 0000000000..327c057053
--- /dev/null
+++ b/doc/forum/comprehension_question__58___repository_vs._working_copy_in_direct_mode/comment_1_a6b4db0cefa439f72b97089d48dfacbd._comment
@@ -0,0 +1,16 @@
+[[!comment format=mdwn
+ username="http://joeyh.name/"
+ ip="108.236.230.124"
+ subject="comment 1"
+ date="2014-07-21T19:54:15Z"
+ content="""
+A git repository is a `.git` directory (or `git.bare` for a bare repository).
+
+A working tree is the directory that contains the `.git` directory.
+
+That is standard git terminology; git-annex does not change this at all really. The only difference is that a file added to git-annex is in both the repository and the working tree at the same time, rather than their being 2 local copies of the file (which would need twice the disk space so not good for large files).
+
+`git annex sync` commits any changes to files in the working tree, and pushes those changes to other remotes. You need to pass --content to also make git-annex upload the files to other remotes. Once a remote has been pushed to, you can run `git annex merge` in it to update its working tree to reflect the pushed changes (`git annex sync` also does that merge).
+
+To automatically sync changes to remotes, you can run the git-annex assistant.
+"""]]
diff --git a/doc/forum/confused_about_external_drives.mdwn b/doc/forum/confused_about_external_drives.mdwn
new file mode 100644
index 0000000000..9fdb5c979e
--- /dev/null
+++ b/doc/forum/confused_about_external_drives.mdwn
@@ -0,0 +1,30 @@
+First off, to get that off my chest: I am not using Gnome, KDE, XFCE or any "desktop manager" (well, not quite true, I use some bits of XFCE, but it's easier to assume I'm not).
+
+So automounting is always a little tricky for me, and I often use `pmount` to mount external drives and flash cards.
+
+Now, I have tried to add an external drive as a backup to one of my git-annex repositories. I zero'd the drive, added a GPT partition (I wanted labels), formatted it as `ext4` and then gave it to my user:
+
+    cfdisk /dev/sdd
+    mkfs -t ext4 /dev/sdd1
+    mount /dev/sdd1 /mnt
+    chown anarcat /mnt
+    umount /mnt
+
+Now, at this step I would have assumed the drive would be seen by the webapp, but that wasn't the case. Maybe I wasn't patient enough? In any case, at some point I got tired and went to the commandline to add the drive as a remote. I did the following:
+
+    mkdir -p /media/ata-WDC_WD15EADS-00P8B0_WD-WMAVU0748851-part1
+    mount /dev/disk/by-id/ata-WDC_WD15EADS-00P8B0_WD-WMAVU0748851-part1 /media/ata-WDC_WD15EADS-00P8B0_WD-WMAVU0748851-part1
+    cd /srv/mp3
+    git annex remote add backup /media/ata-WDC_WD15EADS-00P8B0_WD-WMAVU0748851-part1/mp3
+    git init /media/ata-WDC_WD15EADS-00P8B0_WD-WMAVU0748851-part1/mp3
+    git annex sync
+    git annex copy --to backup
+
+So that worked, and the webapp was now seeing `backup` as yet another repository, but not quite removeable. So failed. Scratching that, I stopped the assistant, and trashed the repository:
+
+    rm -rf /media/ata-WDC_WD15EADS-00P8B0_WD-WMAVU0748851-part1/mp3
+    git remote rm backup
+
+But now, oddly enough, the webapp would see the external drive after being restarted, and I was able to add that drive.
+
+What am I doing wrong? how is the assistant finding those drives? --[[anarcat]]
diff --git a/doc/forum/confused_about_external_drives/comment_1_8340e5ff17a4846b41789e4966fed70c._comment b/doc/forum/confused_about_external_drives/comment_1_8340e5ff17a4846b41789e4966fed70c._comment
new file mode 100644
index 0000000000..5b7ca01141
--- /dev/null
+++ b/doc/forum/confused_about_external_drives/comment_1_8340e5ff17a4846b41789e4966fed70c._comment
@@ -0,0 +1,12 @@
+[[!comment format=mdwn
+ username="http://joeyh.name/"
+ ip="209.250.56.227"
+ subject="separation of concerns"
+ date="2013-12-29T17:57:36Z"
+ content="""
+It doesn't make sense for the git-annex assistant to try to somehow detect and mount drives. Every OS has its own way to do that.
+
+The assistant simply detects drives that have been mounted by the OS.
+
+There is absolutely no problem with manually setting up a git repository on a removable drive and using it with the assistant. This is identical to what the assistant would set up for an external drive (except perhaps being a non-bare repository). The assistant simply detects if any git remote whose url is a directory is inside a mount point when it sees a new device be mounted.
+"""]]
diff --git a/doc/forum/confused_about_external_drives/comment_2_ad10bd0ec14c16bcad089b3ebe64580e._comment b/doc/forum/confused_about_external_drives/comment_2_ad10bd0ec14c16bcad089b3ebe64580e._comment
new file mode 100644
index 0000000000..c0eb3d8c29
--- /dev/null
+++ b/doc/forum/confused_about_external_drives/comment_2_ad10bd0ec14c16bcad089b3ebe64580e._comment
@@ -0,0 +1,10 @@
+[[!comment format=mdwn
+ username="https://id.koumbit.net/anarcat"
+ ip="2001:1928:1:9::1"
+ subject="comment 2"
+ date="2013-12-30T17:47:26Z"
+ content="""
+Fair enough, I guess. However, I must say that it seems that git-annex doesn't actually detect properly externally mounted hard drives. For example, here I have an annex in /srv/foo that is part of my system hard drive (but in a separate /srv partition). Then I made a backup annex in /backup/srv/foo that is on an external hard drive, yet the webapp sees those two repos as under its control - as in, i can choose both in the \"switch repository\" menu.
+
+Another drive I was able to add using the \"add external drive\" tool in the webapp doesn't show up in that list (and that's fine too!).
+"""]]
diff --git a/doc/forum/confusion_with_remotes__44___map.mdwn b/doc/forum/confusion_with_remotes__44___map.mdwn
new file mode 100644
index 0000000000..0ae75d4e99
--- /dev/null
+++ b/doc/forum/confusion_with_remotes__44___map.mdwn
@@ -0,0 +1,113 @@
+I'm starting out with git-annex and running into some confusion with setting up the remotes.
+
+I have three systems I'm trying to set up (domains edited):
+
+* psychosis: ssh://psychosis.foo.com/vid
+* bacon: ssh://bucket.foo.com/vid
+* bucket: ssh://bucket.bar.org/vid
+
+And one bare repository so that I can have a single place to push/pull:
+
+* origin: https://git.foo.com/jim/vid.git
+
+On psychosis:
+
+    psychosis$ git config --list | grep ^remote | sort
+    remote.bacon.annex-uuid=8f1f0898-f8c1-11e0-9bf2-b387af26ee63
+    remote.bacon.fetch=+refs/heads/*:refs/remotes/bacon/*
+    remote.bacon.url=ssh://bucket.foo.com/vid
+    remote.bucket.annex-uuid=82814942-f8e0-11e0-b053-e70a61e98e19
+    remote.bucket.fetch=+refs/heads/*:refs/remotes/bucket/*
+    remote.bucket.url=ssh://bucket.bar.org/vid
+    remote.origin.fetch=+refs/heads/*:refs/remotes/origin/*
+    remote.origin.url=https://git.foo.com/jim/vid.git
+    
+    psychosis$ git annex status
+    supported backends: WORM SHA1 SHA256 SHA512 SHA224 SHA384 SHA1E SHA256E SHA512E SHA224E SHA384E URL
+    supported remote types: git S3 bup directory rsync web hook
+    known repositories: 
+            09c0b436-f8de-11e0-842f-b7644539d57f -- here (psychosis)
+            82814942-f8e0-11e0-b053-e70a61e98e19 -- bucket
+    local annex keys: 2256
+    local annex size: 449 gigabytes
+    total annex keys: 2256
+    total annex size: 449 gigabytes
+    backend usage: 
+            WORM: 2256
+
+**First point of confusion**: Why doesn't "bacon" show up in "git annex status"?  I can "git annex copy --to bacon filename" and it will copy it there.  Is there some step of setting it up that I missed?  I basically just did "git remote add bacon ssh://bucket.foo.com/vid".
+
+Now I've started setting up the remotes on each host:
+
+On bacon:
+   
+    bacon$ git config --list | grep ^remote | sort
+    remote.origin.fetch=+refs/heads/*:refs/remotes/origin/*
+    remote.origin.url=https://git.foo.com/jim/vid.git
+    remote.psychosis.annex-uuid=09c0b436-f8de-11e0-842f-b7644539d57f
+    remote.psychosis.fetch=+refs/heads/*:refs/remotes/psychosis/*
+    remote.psychosis.url=ssh://psychosis.foo.com/vid
+    
+    bacon$ git annex status
+    supported backends: WORM SHA1 SHA256 SHA512 SHA224 SHA384 SHA1E SHA256E SHA512E SHA224E SHA384E URL
+    supported remote types: git S3 bup directory rsync web hook
+    known repositories: 
+            09c0b436-f8de-11e0-842f-b7644539d57f -- psychosis
+            8f1f0898-f8c1-11e0-9bf2-b387af26ee63 -- here (bacon)
+    temporary directory size: 366 megabytes (clean up with git-annex unused)
+    local annex keys: 1
+    local annex size: 308 bytes
+    total annex keys: 2256
+    total annex size: 449 gigabytes
+    backend usage: 
+            WORM: 2256
+
+On bucket:
+
+    bucket$ git config --list | grep ^remote | sort
+    remote.origin.fetch=+refs/heads/*:refs/remotes/origin/*
+    remote.origin.url=https://git.foo.com/jim/vid.git
+    remote.psychosis.annex-uuid=09c0b436-f8de-11e0-842f-b7644539d57f
+    remote.psychosis.fetch=+refs/heads/*:refs/remotes/psychosis/*
+    remote.psychosis.url=ssh://psychosis.foo.com/vid
+
+    bucket$ git annex status
+    supported backends: WORM SHA1 SHA256 SHA512 SHA224 SHA384 SHA1E SHA256E SHA512E SHA224E SHA384E URL
+    supported remote types: git S3 bup directory rsync web hook
+    known repositories: 
+            09c0b436-f8de-11e0-842f-b7644539d57f -- psychosis
+            82814942-f8e0-11e0-b053-e70a61e98e19 -- here (bucket)
+    temporary directory size: 183 megabytes (clean up with git-annex unused)
+    local annex keys: 3
+    local annex size: 550 megabytes
+    total annex keys: 2256
+    total annex size: 449 gigabytes
+    backend usage: 
+            WORM: 2256
+
+But I'm getting weird results if I try to show the map from psychosis:
+
+    psychosis$ git annex map
+    $ git annex map
+    map /vid/tv ok
+    map bacon (sshing...) 
+    ok
+    map bucket (sshing...) 
+    ok
+    map origin 
+    failed
+    map psychosis (sshing...) 
+    jim@psychosis.foo.com's password: 
+    ok
+    map psychosis (sshing...) 
+    jim@psychosis.foo.com's password: 
+    ok
+    
+      running: dot -Tx11 map.dot
+
+**Second confusion**: it's as if psychosis was considered a new remote each time?
+The generated map has psychosis listed with several redundant links:
+
+![Map](http://jim.sh/~jim/tmp/map.png)
+
+Is this some bug or do I just need to be hit with the clue bat?
diff --git a/doc/forum/confusion_with_remotes__44___map/comment_1_a38ded23b7f288292a843abcb1a56f38._comment b/doc/forum/confusion_with_remotes__44___map/comment_1_a38ded23b7f288292a843abcb1a56f38._comment
new file mode 100644
index 0000000000..97de93d9ec
--- /dev/null
+++ b/doc/forum/confusion_with_remotes__44___map/comment_1_a38ded23b7f288292a843abcb1a56f38._comment
@@ -0,0 +1,10 @@
+[[!comment format=mdwn
+ username="http://joey.kitenet.net/"
+ nickname="joey"
+ subject="comment 1"
+ date="2011-10-17T19:01:21Z"
+ content="""
+My guess is that psychosis has not pulled the git-annex branch since bacon was set up (or that bacon's git-annex branch has not been pushed to origin). git-annex status only shows remotes present in git-annex:uuid.log This may be a bug.
+
+The duplicate links in the map I don't quite understand. I only see duplicate links in my maps when I have the same repository configured as two different git remotes (for example, because the same repository can be accessed two different ways). You don't seem to have that in your config.
+"""]]
diff --git a/doc/forum/confusion_with_remotes__44___map/comment_2_cd1c98b1276444e859a22c3dbd6f2a79._comment b/doc/forum/confusion_with_remotes__44___map/comment_2_cd1c98b1276444e859a22c3dbd6f2a79._comment
new file mode 100644
index 0000000000..a61b126c0c
--- /dev/null
+++ b/doc/forum/confusion_with_remotes__44___map/comment_2_cd1c98b1276444e859a22c3dbd6f2a79._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="http://joey.kitenet.net/"
+ nickname="joey"
+ subject="comment 2"
+ date="2011-10-17T19:02:50Z"
+ content="""
+Actually, there is a hint that, while you ran the git annex map on psychosis, it decided to ssh to itself two times. That seems to be where the duplicate links came from, I guess you must have some git remotes you did not show.
+"""]]
diff --git a/doc/forum/confusion_with_remotes__44___map/comment_3_18531754089c991b6caefc57a5c17fe9._comment b/doc/forum/confusion_with_remotes__44___map/comment_3_18531754089c991b6caefc57a5c17fe9._comment
new file mode 100644
index 0000000000..4c77222619
--- /dev/null
+++ b/doc/forum/confusion_with_remotes__44___map/comment_3_18531754089c991b6caefc57a5c17fe9._comment
@@ -0,0 +1,24 @@
+[[!comment format=mdwn
+ username="https://www.google.com/accounts/o8/id?id=AItOawnBJ6Dv1glxzzi4qIzGFNa6F-mfHIvv9Ck"
+ nickname="Jim"
+ subject="comment 3"
+ date="2011-10-17T19:50:06Z"
+ content="""
+No extra remotes (that I'm aware of); that output was only edited to change hostnames.
+
+On all three hosts, \"git push origin\" and \"git pull origin\" say everything is up to date.
+
+I'm using git-annex 3.20111011 on all hosts (although some were running 3.20110928 when I created the repositories).
+
+Regarding the multiple links, I've put a copy of the dot file [here](http://jim.sh/~jim/tmp/map.dot).
+It shows psychosis in three separate subgraphs, that are just getting rendered together as one,
+if that helps clarify anything.
+
+Wait, I just realized you said \"the git-annex branch\".  My origin only has \"master\".
+Do you mean the one specifically named \"git-annex\"?  I thought that was something that
+gets managed automatically, or is it something I need to manually check out and deal with?
+
+Any other info I could provide?
+
+
+"""]]
diff --git a/doc/forum/confusion_with_remotes__44___map/comment_4_3b89b6d1518267fcbc050c9de038b9ca._comment b/doc/forum/confusion_with_remotes__44___map/comment_4_3b89b6d1518267fcbc050c9de038b9ca._comment
new file mode 100644
index 0000000000..f6e5993c8e
--- /dev/null
+++ b/doc/forum/confusion_with_remotes__44___map/comment_4_3b89b6d1518267fcbc050c9de038b9ca._comment
@@ -0,0 +1,11 @@
+[[!comment format=mdwn
+ username="https://www.google.com/accounts/o8/id?id=AItOawnBJ6Dv1glxzzi4qIzGFNa6F-mfHIvv9Ck"
+ nickname="Jim"
+ subject="comment 4"
+ date="2011-10-17T20:36:51Z"
+ content="""
+Ok, after pushing the \"git-annex\" branch to origin, then \"git annex status\" knows all repositories on all hosts, so that part makes sense now.  Thanks for the tip.  But the \"git annex map\" output hasn't changed.
+
+
+
+"""]]
diff --git a/doc/forum/confusion_with_remotes__44___map/comment_5_27801584325d259fa490f67273f2ff71._comment b/doc/forum/confusion_with_remotes__44___map/comment_5_27801584325d259fa490f67273f2ff71._comment
new file mode 100644
index 0000000000..77a2c4adbe
--- /dev/null
+++ b/doc/forum/confusion_with_remotes__44___map/comment_5_27801584325d259fa490f67273f2ff71._comment
@@ -0,0 +1,16 @@
+[[!comment format=mdwn
+ username="https://www.google.com/accounts/o8/id?id=AItOawnBJ6Dv1glxzzi4qIzGFNa6F-mfHIvv9Ck"
+ nickname="Jim"
+ subject="comment 5"
+ date="2011-10-18T04:59:13Z"
+ content="""
+I think:
+
+* The first extra edge is because bucket had \"ssh://psychosis.foo.com/vid/\", while
+bacon had \"ssh://psychosis.foo.com/vid\" with no trailing slash.  That got lost in the hostname/path editing I did, sorry.
+Maybe those should be considered matching?
+* The second extra edge is because, when running \"git annex map\" from psychosis, it doesn't recognize the remote's
+remote URL as pointing back to itself.
+
+For the second case, after the \"spurious\" SSH, it could still recognize that the repositories are the same by the duplicated annex uuid, which currently shows up in `map.dot` twice.  I wonder what it would take to avoid the spurious SSH -- maybe some config that lists \"alternate\" URLs that should be considered the same as the current repository?  Or actually list URLs in uuid.log?  Fortunately, I think this only affects the map, so it's not a big problem.
+"""]]
diff --git a/doc/forum/confusion_with_remotes__44___map/comment_6_496b0d9b86869bbac3a1356d53a3dda4._comment b/doc/forum/confusion_with_remotes__44___map/comment_6_496b0d9b86869bbac3a1356d53a3dda4._comment
new file mode 100644
index 0000000000..412937f3fc
--- /dev/null
+++ b/doc/forum/confusion_with_remotes__44___map/comment_6_496b0d9b86869bbac3a1356d53a3dda4._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="http://joey.kitenet.net/"
+ nickname="joey"
+ subject="comment 6"
+ date="2011-10-22T01:18:27Z"
+ content="""
+Hmm, I don't see the spurious ssh edge in the dot file -- that is, I don't see any ssh:// uris in it?
+"""]]
diff --git a/doc/forum/confusion_with_remotes__44___map/comment_7_9a456f61f956a3d5e81e723d5a90794c._comment b/doc/forum/confusion_with_remotes__44___map/comment_7_9a456f61f956a3d5e81e723d5a90794c._comment
new file mode 100644
index 0000000000..85ede3a89c
--- /dev/null
+++ b/doc/forum/confusion_with_remotes__44___map/comment_7_9a456f61f956a3d5e81e723d5a90794c._comment
@@ -0,0 +1,27 @@
+[[!comment format=mdwn
+ username="https://www.google.com/accounts/o8/id?id=AItOawnBJ6Dv1glxzzi4qIzGFNa6F-mfHIvv9Ck"
+ nickname="Jim"
+ subject="comment 7"
+ date="2011-10-22T05:25:47Z"
+ content="""
+I think that's because the SSH was successful (I entered the password and let it connect), so it got the UUID and put that in the .dot instead.  The same UUID (for psychosis) then ended up in two different \"subgraph\" stanzas, and Graphviz just plotted them together as one node.
+
+
    Maybe this will clarify:
+
+
    On psychosis, run \"git annex map\" and press ^C at the ssh password prompt: [map-nossh.dot](http://jim.sh/~jim/tmp/map-nossh.dot)
+![Map](http://jim.sh/~jim/tmp/map-nossh.png)
+
+
    On psychosis, run \"git annex map\" and type the correct password: [map-goodssh.dot](http://jim.sh/~jim/tmp/map-goodssh.dot)
+![Map](http://jim.sh/~jim/tmp/map-goodssh.png)
+
+As I see it:
+
+* psychosis (\"localhost\") connects to each of its remotes
+* some of them point back to ssh://psychosis
+* psychosis doesn't know that ssh://psychosis is itself, so it tries to connect
+* if successful:
+  * psychosis gets put twice in the .dot as if it was two different hosts, one \"local\" and one \"ssh://psychosis\"
+  * graphviz recognizes it as the same node because the UUID is the same, but graphviz still draws the extra connecting lines
+* if unsuccessful:
+  * ssh://psychosis is shown as an additional host that can't be reached
+"""]]
diff --git a/doc/forum/controlling_how___34__git_annex_sync__34___resolves_conflicts.mdwn b/doc/forum/controlling_how___34__git_annex_sync__34___resolves_conflicts.mdwn
new file mode 100644
index 0000000000..5c9b8f5115
--- /dev/null
+++ b/doc/forum/controlling_how___34__git_annex_sync__34___resolves_conflicts.mdwn
@@ -0,0 +1,6 @@
+Hi,
+I'd like to know if there's an easy way to control how "git annex sync" resolves conflicts. I use git annex (with wrappers I have written) to manage repos full of debs. The debs themselves never conflict, but the metadata does. So far this works great if people check out the repo, add some debs in, regenerate the metadata from the debs (using external scripts), and then shove it back into the central repo. Many people can do this for our shared project.
+
+This doesn't work well if two people have the repo checked out at the same time; when they "git annex sync" to shove the data back up, annex detects the merge conflict in the metadata and renames the files.
+
+What I'd like is a flag saying that while pushing the changes upstream, the sync will only perform fast-forwards and fail if that doesn't work so my wrappers can abort the user's workflow. Any advice would be appreciated.
diff --git a/doc/forum/controlling_how___34__git_annex_sync__34___resolves_conflicts/comment_1_48be720ff150112f30273960f677056c._comment b/doc/forum/controlling_how___34__git_annex_sync__34___resolves_conflicts/comment_1_48be720ff150112f30273960f677056c._comment
new file mode 100644
index 0000000000..37a0fa3c7c
--- /dev/null
+++ b/doc/forum/controlling_how___34__git_annex_sync__34___resolves_conflicts/comment_1_48be720ff150112f30273960f677056c._comment
@@ -0,0 +1,20 @@
+[[!comment format=mdwn
+ username="joey"
+ subject="""comment 1"""
+ date="2017-06-01T15:41:33Z"
+ content="""
+Failing at push seems too late; the user will already have a branch with
+the unwanted merge in it at that point and could have additional changes
+commited on top of it. And, only allowing fast-forward
+merges is probably not what you want either; regular non-conflicting git
+merges that are not fast-forwards will almost always be fine. If you did
+want to prevent any pushes containing non-fast-forward merges, you could
+do it with an `update` hook in the server's git repo.
+
+There is not currently a way to prevent `git annex sync` from
+auto-resolving a merge conflict, but I think there definitely should be.
+Something like `git config annex.resolvemerge false`.
+And, it probably makes sense to let that be configuration be propigated
+to all clones of a repository, by using
+`git-annex config annex.resolvemerge false`
+"""]]
diff --git a/doc/forum/controlling_how___34__git_annex_sync__34___resolves_conflicts/comment_2_c091e5a668b15aca7b2faa955beab403._comment b/doc/forum/controlling_how___34__git_annex_sync__34___resolves_conflicts/comment_2_c091e5a668b15aca7b2faa955beab403._comment
new file mode 100644
index 0000000000..f254f6e7d1
--- /dev/null
+++ b/doc/forum/controlling_how___34__git_annex_sync__34___resolves_conflicts/comment_2_c091e5a668b15aca7b2faa955beab403._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="joey"
+ subject="""comment 2"""
+ date="2017-06-01T16:16:44Z"
+ content="""
+I've implemented the annex.automerge configuration setting, for the next
+release. There's also a `git annex sync --no-resolvemerge`
+"""]]
diff --git a/doc/forum/controlling_how___34__git_annex_sync__34___resolves_conflicts/comment_2_d8dddf7b565273390ce4ba1c22dabf46._comment b/doc/forum/controlling_how___34__git_annex_sync__34___resolves_conflicts/comment_2_d8dddf7b565273390ce4ba1c22dabf46._comment
new file mode 100644
index 0000000000..c257a3debe
--- /dev/null
+++ b/doc/forum/controlling_how___34__git_annex_sync__34___resolves_conflicts/comment_2_d8dddf7b565273390ce4ba1c22dabf46._comment
@@ -0,0 +1,15 @@
+[[!comment format=mdwn
+ username="pgunn01@39c747700d10e9e9e4557a407cba2f88c22b202d"
+ nickname="pgunn01"
+ avatar="http://cdn.libravatar.org/avatar/0ca97d677b2cc6c5a262f3709ad93381"
+ subject="reply to joey"
+ date="2017-06-01T16:47:25Z"
+ content="""
+In our case (Not claiming a broader usage), failing at push is exactly what we want. We're using git-annex, with wrappers, as a transport mechanism for many-gigabyte debian repos, and in the end there's only one correct notion of what's in our repos. There's no local work that's of any significance, and we can catch problems clientside and redo as needed.
+
+A user will log onto a small number of hosts, pulldown the repo (with scripts I wrote that wrap annex), add some debs, regenerate the apt metadata, and then commit/push all that to S3/git. And then they'll remove their checkout.
+
+On the other side, on many repo hosts they'll get new delivered versions of the git repo, then refresh from S3, and then they're ready to serve the contents to the rest of our infra.
+
+(I realise this is probably a weird edge case for how people use annex; we use it because it lets us use git and see who's doing things, because it works well with our centralised git repos, and because those centralised git repos are not burdened with gigantic files)
+"""]]
diff --git a/doc/forum/controlling_how___34__git_annex_sync__34___resolves_conflicts/comment_4_c019c5b4d1c2a7941aed8791b6749f5b._comment b/doc/forum/controlling_how___34__git_annex_sync__34___resolves_conflicts/comment_4_c019c5b4d1c2a7941aed8791b6749f5b._comment
new file mode 100644
index 0000000000..8eb169ba93
--- /dev/null
+++ b/doc/forum/controlling_how___34__git_annex_sync__34___resolves_conflicts/comment_4_c019c5b4d1c2a7941aed8791b6749f5b._comment
@@ -0,0 +1,13 @@
+[[!comment format=mdwn
+ username="joey"
+ subject="""comment 4"""
+ date="2017-06-01T17:14:28Z"
+ content="""
+Well, a git `update` hook is the only way to utterly prevent non-ff's being
+pushed.
+
+Since you're checking out the repository fresh each time, your scripts
+could use `git annex sync --no-pull`. That would fail if someone else
+pushed work to the repo in the meantime, and you could then use whatever
+git commands make sense for you to fetch and rebase on top of the new changes.
+"""]]
diff --git a/doc/forum/converting_a_v5_repo_to_an_always_unlocked_v6_repo.mdwn b/doc/forum/converting_a_v5_repo_to_an_always_unlocked_v6_repo.mdwn
new file mode 100644
index 0000000000..40735383ae
--- /dev/null
+++ b/doc/forum/converting_a_v5_repo_to_an_always_unlocked_v6_repo.mdwn
@@ -0,0 +1,34 @@
+Hi,
+I'm trying to convert a v5 classic, symlink repository into a v6 always unloked repository.
+
+
+I'm trying to follow along with: 
+but something goes wrong in the process, so I'm sure I'm missing something.
+The repository is about 600G to start with on a ext4 filesystem, and I don't have another 600G of free space on disk, so I'm going with thin mode:
+
+1.       git annex upgrade 
+2.       git config annex.thin true 
+3.       git annex fix 
+4.       git annex unlock
+
+It's all good to this point. Everything gets unlocked and is fine. 
+Then I try to commit the changes with:
+
+     git annex sync
+
+The process seems to take 
+and incredibly long time (several hours) and then ends up running out of space. I check the repository with "du -sh"
+and it's almost double the size. Is there a reason for this? Is there a way to avoid this duplication of data. Shouldn't annex.thin do the trick?
+
+This is also strange: with "git annex info" I get
+
+     local annex keys: 3572
+     local annex size: 933.19 gigabytes
+     annexed files in working tree: 0
+     size of annexed files in working tree: 0 bytes
+
+the annex size should be about 540G and why are there no annexed files in the working tree?
+
+Is there a correct and faster way to migrate my repo to an always unlocked one which won't require hours of time and take all that disk space?
+
+thanks a lot, daniel
diff --git a/doc/forum/converting_a_v5_repo_to_an_always_unlocked_v6_repo/comment_1_e08eb54547bfd4dbfcc79a5a4a529647._comment b/doc/forum/converting_a_v5_repo_to_an_always_unlocked_v6_repo/comment_1_e08eb54547bfd4dbfcc79a5a4a529647._comment
new file mode 100644
index 0000000000..362609f524
--- /dev/null
+++ b/doc/forum/converting_a_v5_repo_to_an_always_unlocked_v6_repo/comment_1_e08eb54547bfd4dbfcc79a5a4a529647._comment
@@ -0,0 +1,35 @@
+[[!comment format=mdwn
+ username="joey"
+ subject="""comment 1"""
+ date="2016-03-14T16:48:33Z"
+ content="""
+Thank you for trying v6 unlocked mode; do note that it's still somewhat
+experimental.
+
+`git annex info` doesn't report correctly on unlocked files, which is why
+it has 0 in two places in the output you showed. I have just committed a
+fix for that problem.
+
+I don't know what would cause the unexpectedly small "local annex size".
+That should correspond to `du -hsc .git/annex/objects`; if it does then you
+seem to have fewer annexed objects than you expect for some reason.
+
+The behavior on sync sounds kind of like git commit is checking the whole
+contents of files into git, bypassing the annex. I don't know how that
+could happen, barring a misconfiguration, but it's at least the first thing
+to check. Check for gigabytes of data in .git/objects/ to see if that
+is the case. 
+
+If the above isn't the problem, can you see if the files in the work tree
+have a link count of 2? For example:
+
+	$ ls -l
+	-rw-r--r--  2 joey joey     30 Mar 14 12:49 foo
+	            ^
+
+If you don't see link count of 2, something has caused the annex.thin not
+to take effect. One possibility would be if the `git annex sync` merged in
+changes that moved a lot of files around. When that happens, git checks out
+the updated work tree, and git-annex currently is not able to preserve
+annex.thin in that case.
+"""]]
diff --git a/doc/forum/converting_a_v5_repo_to_an_always_unlocked_v6_repo/comment_2_85946916ac9c2a19c0d65e2b1a3e5832._comment b/doc/forum/converting_a_v5_repo_to_an_always_unlocked_v6_repo/comment_2_85946916ac9c2a19c0d65e2b1a3e5832._comment
new file mode 100644
index 0000000000..4c8f1b2387
--- /dev/null
+++ b/doc/forum/converting_a_v5_repo_to_an_always_unlocked_v6_repo/comment_2_85946916ac9c2a19c0d65e2b1a3e5832._comment
@@ -0,0 +1,21 @@
+[[!comment format=mdwn
+ username="susetux@ed6f4e20192c3eae018e1fc6442bf993d41b3848"
+ nickname="susetux"
+ subject="re"
+ date="2016-03-15T14:14:47Z"
+ content="""
+Hi, thanks for your quick feedback!
+
+So, I've investigated a bit and this is what I found:
+
+1. hardlinks are there: both \"ls -l\" and \"find -samefile\" report that most files (I haven't checked all of them though) are hardlinked, as expected, in the annex. So annex.thin is doing something.
+2. \"du -shc .git/annex/objects\" has the same size of the same command given on the whole annex, as expected  (du counts hard links only once, so it makes sense). It gives about 933G.
+3. 900G is well above what it should be: 530G of the indirect annex.
+4. This size is also reported in git annex info: \"local annex size: 933.19 gigabytes\"
+5. I tried locking the files again (and git annex fix), but the annex remained oversized.
+
+I currently deleted the annex and restored from a backup because I couldn't afford to keep it in an inconsistent state.
+It seems like this problem can be consistently reproduced. I had the same problem on a smaller annex (200G) which took a long time to commit and also inflated during the process. At the end of the process (it had enough space to finish) the size shrinked again, although it was still bigger than the original. I also tried with a test annex and random data a the problem seemed to still be there.
+
+
+"""]]
diff --git a/doc/forum/copy_--auto_copies_already_synced_files.mdwn b/doc/forum/copy_--auto_copies_already_synced_files.mdwn
new file mode 100644
index 0000000000..13a497a9da
--- /dev/null
+++ b/doc/forum/copy_--auto_copies_already_synced_files.mdwn
@@ -0,0 +1,24 @@
+Hello,
+
+I have two repos, both are standard/client and they are in sync:
+
+```
+% git annex sync marduk.dynv6 --content
+commit  ok
+pull marduk.dynv6 
+ok
+git annex sync marduk.dynv6 --content  64,35s user 4,54s system 88% cpu 1:18,15 total
+```
+I would ```copy --auto``` except to copy no files, but still, ```git annex copy --to=marduk.dynv6 --auto``` seems to copy all files:
+
+```
+% git annex copy --to=marduk.dynv6 --auto
+copy .gitignore (checking marduk.dynv6...) ok
+copy Alben/2Raumwohnung - Es wird Morgen/2Raumwohnung - 01 - Wolken ziehen vorbei.mp3 (checking marduk.dynv6...) ok
+copy Alben/2Raumwohnung - Es wird Morgen/2Raumwohnung - 02 - Spiel mit.mp3 (checking marduk.dynv6...) ok
+[... canceled]
+```
+Why is copy --auto copying these (presumably all) files here=
+
+Thanks!
+Florian
diff --git a/doc/forum/copy_--auto_copies_already_synced_files/comment_1_cc5e2e7630a33b73d1eb92eebe254cea._comment b/doc/forum/copy_--auto_copies_already_synced_files/comment_1_cc5e2e7630a33b73d1eb92eebe254cea._comment
new file mode 100644
index 0000000000..ae3b461513
--- /dev/null
+++ b/doc/forum/copy_--auto_copies_already_synced_files/comment_1_cc5e2e7630a33b73d1eb92eebe254cea._comment
@@ -0,0 +1,7 @@
+[[!comment format=mdwn
+ username="Horus"
+ subject="comment 1"
+ date="2016-07-14T09:28:32Z"
+ content="""
+Addendum: It seems it is not really copying the files, but checking. The solution is probably using ```git annex copy --to=marduk.dynv6 --auto --fast``` which behaves like I expect it (copies nothing).
+"""]]
diff --git a/doc/forum/copy_--auto_copies_already_synced_files/comment_2_3cfd11d8199a4d5e442d207a1edb38e0._comment b/doc/forum/copy_--auto_copies_already_synced_files/comment_2_3cfd11d8199a4d5e442d207a1edb38e0._comment
new file mode 100644
index 0000000000..5fe0b215de
--- /dev/null
+++ b/doc/forum/copy_--auto_copies_already_synced_files/comment_2_3cfd11d8199a4d5e442d207a1edb38e0._comment
@@ -0,0 +1,12 @@
+[[!comment format=mdwn
+ username="joey"
+ subject="""comment 2"""
+ date="2016-07-19T16:57:59Z"
+ content="""
+Right, it's checking if the file content is still present on the remote.
+Since that can take a while, or even lead to a ssh password prompt etc,
+it prints out the name of each file as it checks it. 
+
+--fast avoids that check, which is fine. I have gone back and forth on
+making --fast the default in this sitation.
+"""]]
diff --git a/doc/forum/copy_and_move__63__.mdwn b/doc/forum/copy_and_move__63__.mdwn
new file mode 100644
index 0000000000..88ce775034
--- /dev/null
+++ b/doc/forum/copy_and_move__63__.mdwn
@@ -0,0 +1,19 @@
+My use case is that I [import media](http://ix.io/h6m) from my iphone/gopro to my laptop. I then want to upload them to [Flickr](https://www.flickr.com/) (JPG/PNG) and S3 (MP4). However I want to keep a backup using git-annex onto a external USB drive.
+
+# How do I quickly copy and drop?
+
+I realise I can:
+
+1. git-annex add 2014-*
+* git-annex copy --to 2tbusbdrive
+* git-annex drop 2014-*
+
+But that's several steps. Is there a quicker way?
+
+# How do drop only files I've actually uploaded?
+
+In the past, I've just marked files as uploaded by `touch "$media.uploaded"` once my python Flickr script returns a successful error code. But I have no idea how to tell git-annex that this means the file can now be dropped, providing a copy exists on my **2tbusbdrive**.
+
+# Furthermore keeping a clean deck
+
+Tbh I don't want the old upload files/directories cluttering up my _uploadme_ directory. Can I just delete them or configure drop to remove the filename too?
diff --git a/doc/forum/copy_and_move__63__/comment_1_a24b1ab4608fcf8129c9890a5ae9e943._comment b/doc/forum/copy_and_move__63__/comment_1_a24b1ab4608fcf8129c9890a5ae9e943._comment
new file mode 100644
index 0000000000..3af1a21cee
--- /dev/null
+++ b/doc/forum/copy_and_move__63__/comment_1_a24b1ab4608fcf8129c9890a5ae9e943._comment
@@ -0,0 +1,7 @@
+[[!comment format=mdwn
+ username="joey"
+ subject="""the obvious answer"""
+ date="2015-03-26T15:17:05Z"
+ content="""
+`git annex move file --to remote`
+"""]]
diff --git a/doc/forum/correct_way_to_add_two_preexisting_datasets.mdwn b/doc/forum/correct_way_to_add_two_preexisting_datasets.mdwn
new file mode 100644
index 0000000000..bfc1a62721
--- /dev/null
+++ b/doc/forum/correct_way_to_add_two_preexisting_datasets.mdwn
@@ -0,0 +1,25 @@
+I've been syncronizing my data since long time, mainly using rsync or unison. Thus I had two 3.5Gb datasets set1 (usb drive, hfs+ partition) and set2 (hdd, ext4 ubuntu 13.04 box) which differed only in 50Mb (new on set1 ). This was double checked using diff -r before doing anything.
+
+I created a git annex repo in direct mode for set2 from command line, and after that I let the assistant scan it.
+After that created the repo for set1 and added it to the assistant. I think here comes my mistake (I think).
+
+Instead of keeping them apart, at told assistant to sync with set2.
+Why I think this was a mistake? Because set2 was indexed and set1 no, and I'm seeing a lot of file moving a copying, which in my humble opinion should not happen.
+What I expected it only the difference to be transferred from set1 to set2.
+What it seems to be doing is moving away all content in set1, and copying it back from set2. I think it will end correctly, but with a lot of unnecessary and risky operations.
+
+I think I should have independently added both datasets, let them be scanned and then connect to each other.
+So, now the questions:
+
+1. Is that the correct way to proceed?
+2. What if I have to identical files with different modifying times, I hope they are not synced, right?
+3. Is it posssible to achieve this behaviour of copying only the 50Mb?
+
+Thanks in advance and keep up the good work.
+Best regards,
+             Juan
+
+EDIT: a couple of questions more:
+
+4. after finishing, set2 ended with a lot of symlinks but only in one subfolder. To prevent this should I put numcopies in 2?
+5. This data is composed of input datasets and output simulations. Thus, I need to change them often, but not as often as code and in a very partial way (chunks of 50Mb). For me direct mode is the best (or plain git). However, I was wondering, it is possible to drop some files (even in direct mode) and use simlinks instead?
diff --git a/doc/forum/correct_way_to_add_two_preexisting_datasets/comment_1_c5c3ff25c9f5e34db222b5f4ae58b093._comment b/doc/forum/correct_way_to_add_two_preexisting_datasets/comment_1_c5c3ff25c9f5e34db222b5f4ae58b093._comment
new file mode 100644
index 0000000000..13e2a58d2b
--- /dev/null
+++ b/doc/forum/correct_way_to_add_two_preexisting_datasets/comment_1_c5c3ff25c9f5e34db222b5f4ae58b093._comment
@@ -0,0 +1,10 @@
+[[!comment format=mdwn
+ username="https://me.yahoo.com/a/FHnTlSBo1eCGJRwueeKeB6.RCaPbGMPr5jxx8A--#ce0d8"
+ nickname="Hamza"
+ subject="comment 1"
+ date="2013-08-23T16:17:58Z"
+ content="""
+I did something similar for my videos, I've created the repo on one machine add the video files then cloned it on the other machine then reinjected the files in to the cloned repo.
+
+http://joeyh.name/blog/entry/moving_my_email_archives_and_packages_to_git-annex/
+"""]]
diff --git a/doc/forum/correct_way_to_add_two_preexisting_datasets/comment_2_ee3ecc86990ac5a8d0c4fdfb482a7594._comment b/doc/forum/correct_way_to_add_two_preexisting_datasets/comment_2_ee3ecc86990ac5a8d0c4fdfb482a7594._comment
new file mode 100644
index 0000000000..d1e259afd9
--- /dev/null
+++ b/doc/forum/correct_way_to_add_two_preexisting_datasets/comment_2_ee3ecc86990ac5a8d0c4fdfb482a7594._comment
@@ -0,0 +1,16 @@
+[[!comment format=mdwn
+ username="http://joeyh.name/"
+ ip="4.154.0.63"
+ subject="comment 2"
+ date="2013-08-23T18:23:52Z"
+ content="""
+If you need to preserve mtimes, or differentiate between files with identical content but different mtimes, neither git nor git-annex is going to do what you want, since git doesn't care about preserving much file metadata.
+
+As far as importing two sets of files on two computers, the best thing to do is import each, and then let the two sync up. Otherwise, when you're running the assistant it will start downloading the first set you import to the second computer, before the second set is added there, and do extra work. Although once the duplicate files from the second set land in the second git repository, the assistant will avoid any additional redundant transfers.
+
+(The assistant never *moves* files, if both repositories are configured to be in the default client repository group. It only copies.)
+
+I don't understand question #1. \"set2 ended with a lot of symlinks but only in one subfolder\" doesn't make sense to me, or rather I could interpret it to mean any of dozen things (none of which seem likely)
+
+You can `git annex drop` files in direct mode. However, if you're running the assistant, it will try to get them back. You can configure your repository to be in manual mode to prevent the assistant doing that, or not use the assistant, or configure a [[preferred_content]] expression to make the assistant do something more custom like not try to get files located in a \"olddata\" directory.
+"""]]
diff --git a/doc/forum/correct_way_to_add_two_preexisting_datasets/comment_3_e29bf8b848da04c761dc601ac979ac14._comment b/doc/forum/correct_way_to_add_two_preexisting_datasets/comment_3_e29bf8b848da04c761dc601ac979ac14._comment
new file mode 100644
index 0000000000..d750e9581a
--- /dev/null
+++ b/doc/forum/correct_way_to_add_two_preexisting_datasets/comment_3_e29bf8b848da04c761dc601ac979ac14._comment
@@ -0,0 +1,11 @@
+[[!comment format=mdwn
+ username="https://www.google.com/accounts/o8/id?id=AItOawmTNrhkVQ26GBLaLD5-zNuEiR8syTj4mI8"
+ nickname="Juan"
+ subject="comment 3"
+ date="2013-08-25T13:00:04Z"
+ content="""
+Thanks. It is very clear now. I think I got it running. I have 2 direct mode copies in my ubuntu box and in the USB drive and one indirect in my ultrabook (small SSD).
+What I meant is that even in direct mode, after sync ended, the set I indexed first ended with the contents of a folder in the .git dir using symlinks. But it might have been a leftover of previous attempts.
+I think I got confused by the great amount of flexibility it provides.
+Thanks.
+"""]]
diff --git a/doc/forum/could_not_read_from_remote_repository.mdwn b/doc/forum/could_not_read_from_remote_repository.mdwn
new file mode 100644
index 0000000000..6103bfd36f
--- /dev/null
+++ b/doc/forum/could_not_read_from_remote_repository.mdwn
@@ -0,0 +1,36 @@
+hi, i'm trying to set up my notebook and desktop pc to sync my documents. so i used local computers in the web app as a sync option, but on my notebook i get the following error:
+
+    [2013-11-26 11:29:03 CET] main: Pairing in progress
+    X11 forwarding request failed
+    /home/max/.ssh/git-annex-shell: line 4: exec: git-annex-shell: not found
+    X11 forwarding request failed on channel 0
+    /home/max/.ssh/git-annex-shell: line 4: exec: git-annex-shell: not found
+    fatal: Could not read from remote repository.
+    
+    Please make sure you have the correct access rights
+    and the repository exists.
+    [2013-11-26 11:29:31 CET] PairListener: Syncing with desk_Private_Dokumente 
+    X11 forwarding request failed on channel 0
+    /home/max/.ssh/git-annex-shell: line 4: exec: git-annex-shell: not found
+    fatal: Could not read from remote repository.
+
+    Please make sure you have the correct access rights
+    and the repository exists.
+
+    merge: refs/remotes/desk_Private_Dokumente/master - not something we can merge
+
+    merge: refs/remotes/desk_Private_Dokumente/synced/master - not something we can merge
+    X11 forwarding request failed
+    /home/max/.ssh/git-annex-shell: line 4: exec: git-annex-shell: not found
+    X11 forwarding request failed on channel 0
+    /home/max/.ssh/git-annex-shell: line 4: exec: git-annex-shell: not found
+    fatal: Could not read from remote repository.
+
+    Please make sure you have the correct access rights
+    and the repository exists.
+
+my desktop succeeded in adding the repository. 
+
+any idea whats going wrong ?
+
+thanks
diff --git a/doc/forum/could_not_read_from_remote_repository/comment_1_27d4d1556a80c06505ed3d8a9422d082._comment b/doc/forum/could_not_read_from_remote_repository/comment_1_27d4d1556a80c06505ed3d8a9422d082._comment
new file mode 100644
index 0000000000..aaf0946931
--- /dev/null
+++ b/doc/forum/could_not_read_from_remote_repository/comment_1_27d4d1556a80c06505ed3d8a9422d082._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="http://joeyh.name/"
+ ip="209.250.56.64"
+ subject="comment 1"
+ date="2013-11-26T20:18:26Z"
+ content="""
+The problem seems to be on your desktop computer. /home/max/.ssh/git-annex-shell is a script that is created if you install git-annex from the standalone tarball (assuming you're using linux?). It seems that your desktop has that script, but you must have deleted that instalation of git-annex (perhaps you switched to installing it some other way), so the script doesn't work. As long as you have git-annex-shell in your PATH somehow, you can just remove /home/max/.ssh/git-annex-shell
+"""]]
diff --git a/doc/forum/could_not_read_from_remote_repository/comment_2_cf7d5e231675921c3d98faab3613c92f._comment b/doc/forum/could_not_read_from_remote_repository/comment_2_cf7d5e231675921c3d98faab3613c92f._comment
new file mode 100644
index 0000000000..6775d9278d
--- /dev/null
+++ b/doc/forum/could_not_read_from_remote_repository/comment_2_cf7d5e231675921c3d98faab3613c92f._comment
@@ -0,0 +1,49 @@
+[[!comment format=mdwn
+ username="https://www.google.com/accounts/o8/id?id=AItOawnX1msQxnLoSeu7q-i-c9BWghonsN7Qmns"
+ nickname="Jan Ulrich"
+ subject="Same error after deleting .ssh/git-annex-shell"
+ date="2014-06-11T15:53:17Z"
+ content="""
+Hi,
+
+I get the error before and after I deleted .ssh/git-annex-shell on my Ubuntu desktop:
+
+    janulrimacbook2:Movies juh$ git annex sync
+    commit  ok
+    pull sokrates.local_Videos
+    bash: /home/juh/.ssh/git-annex-shell: Datei oder Verzeichnis nicht gefunden
+    fatal: Could not read from remote repository.
+
+    Please make sure you have the correct access rights
+    and the repository exists.
+    failed  
+    push sokrates.local_Videos
+    bash: /home/juh/.ssh/git-annex-shell: Datei oder Verzeichnis nicht gefunden 
+    fatal: Could not read from remote repository.
+
+    Please make sure you have the correct access rights
+    and the repository exists.
+
+      Pushing to sokrates.local_Videos failed.
+
+       (non-fast-forward problems can be solved by setting receive.denyNonFastforwards to false in the remote's git config)
+    failed
+    git-annex: sync: 2 failed
+
+Git-annex version on macbook:
+
+    git-annex version: 5.20140605-gc2add53
+    build flags: Assistant Webapp Webapp-secure Pairing Testsuite S3 WebDAV FsEvents XMPP DNS Feeds Quvi TDFA CryptoHash
+    key/value backends: SHA256E SHA1E SHA512E SHA224E SHA384E SKEIN256E SKEIN512E SHA256 SHA1 SHA512 SHA224 SHA384 SKEIN256 SKEIN512 WORM URL
+    remote types: git gcrypt S3 bup directory rsync web webdav tahoe glacier ddar hook external
+    local repository version: 5
+    supported repository version: 5
+    upgrade supported from repository versions: 0 1 2 4
+
+Git-annex version on Ubuntu desktop:
+
+    git-annex version: 5.20140517
+    build flags: Assistant Webapp Webapp-secure Pairing Testsuite S3 WebDAV Inotify DBus DesktopNotify XMPP DNS Feeds Quvi TDFA CryptoHash
+    key/value backends: SHA256E SHA1E SHA512E SHA224E SHA384E SKEIN256E SKEIN512E SHA256 SHA1 SHA512 SHA224 SHA384 SKEIN256 SKEIN512 WORM URL
+    remote types: git gcrypt S3 bup directory rsync web webdav tahoe glacier ddar hook external
+"""]]
diff --git a/doc/forum/custom_URLs___40__use_case__58___dnanexus__41__.mdwn b/doc/forum/custom_URLs___40__use_case__58___dnanexus__41__.mdwn
new file mode 100644
index 0000000000..37d3b6fb4e
--- /dev/null
+++ b/doc/forum/custom_URLs___40__use_case__58___dnanexus__41__.mdwn
@@ -0,0 +1,2 @@
+I'd like to do something like git-annex-addurl, except the URL is not a web address but a domain-specific identifier.  Specifically, on the DNAnexus platform, each file is assigned an immutable ID like file-asdf432fdsa.   I want to configure git-annex so that it knows it can fetch the file using a specific command (not curl but 'dx download file-asdf432fdsa'); and so that it completely trusts that the file can always be fetched this way.  What would be the right combination of configurations/settings to make this work?  I was thinking of some combination of WORM backend configured just for DNAnexus files and a special remote, but not sure (this "special remote" would only support copying files from it, not to it; so it's more like "special web").
+Thanks for any suggestions!
diff --git a/doc/forum/custom_URLs___40__use_case__58___dnanexus__41__/comment_1_55c2a5c932ccc5a2a49a54239cfba139._comment b/doc/forum/custom_URLs___40__use_case__58___dnanexus__41__/comment_1_55c2a5c932ccc5a2a49a54239cfba139._comment
new file mode 100644
index 0000000000..8fdbba043d
--- /dev/null
+++ b/doc/forum/custom_URLs___40__use_case__58___dnanexus__41__/comment_1_55c2a5c932ccc5a2a49a54239cfba139._comment
@@ -0,0 +1,16 @@
+[[!comment format=mdwn
+ username="joey"
+ subject="""comment 1"""
+ date="2018-09-11T16:56:14Z"
+ content="""
+What you're looking for is the [[design/external_special_remote_protocol]].
+When you build your own special remote using that protocol,
+it can use SETSTATE and GETSTATE to store and retrieve
+the immutable IDs.
+
+It should not be hard to write your own special remote. See
+[[this page|special_remotes/external]].
+
+The Datalad project has a bunch of special remotes doing things like this
+BTW.
+"""]]
diff --git a/doc/forum/custom_URLs___40__use_case__58___dnanexus__41__/comment_2_db80224d61ca549557af558e9a5f099a._comment b/doc/forum/custom_URLs___40__use_case__58___dnanexus__41__/comment_2_db80224d61ca549557af558e9a5f099a._comment
new file mode 100644
index 0000000000..8f35da21e8
--- /dev/null
+++ b/doc/forum/custom_URLs___40__use_case__58___dnanexus__41__/comment_2_db80224d61ca549557af558e9a5f099a._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="Ilya_Shlyakhter"
+ avatar="http://cdn.libravatar.org/avatar/1647044369aa7747829c38b9dcc84df0"
+ subject="comment 2"
+ date="2018-09-12T19:10:46Z"
+ content="""
+Maybe remote..annex-speculate-present could support being set to a regexp, such that all URLs matching the regexp would be speculated-present in the remote, rather than all URLs?  Then could say that all URLs of the form dx://file-xxxxxx should be speculated present in the DNAnexus remote.
+"""]]
diff --git a/doc/forum/custom_URLs___40__use_case__58___dnanexus__41__/comment_3_8b69aa113bc35a042c90beddf8b0f3a9._comment b/doc/forum/custom_URLs___40__use_case__58___dnanexus__41__/comment_3_8b69aa113bc35a042c90beddf8b0f3a9._comment
new file mode 100644
index 0000000000..2ef0700ee8
--- /dev/null
+++ b/doc/forum/custom_URLs___40__use_case__58___dnanexus__41__/comment_3_8b69aa113bc35a042c90beddf8b0f3a9._comment
@@ -0,0 +1,17 @@
+[[!comment format=mdwn
+ username="joey"
+ subject="""comment 3"""
+ date="2018-09-12T19:26:31Z"
+ content="""
+annex-speculate-present is a boolean setting, I see that the docs didn't
+make that clear and have improved them.
+
+I don't see any value in speculating that urls are present in some specific
+remote; there's already sufficient support for remotes to register and
+claim urls. 
+
+See CLAIMURL in the external special remote protocol.
+The ipfs special remote is an example of a a remote that uses CLAIMURL to
+add support for a new type of url to git-annex.
+
+"""]]
diff --git a/doc/forum/custom_URLs___40__use_case__58___dnanexus__41__/comment_4_94e2fbb04c553b2c8c7992541e634017._comment b/doc/forum/custom_URLs___40__use_case__58___dnanexus__41__/comment_4_94e2fbb04c553b2c8c7992541e634017._comment
new file mode 100644
index 0000000000..897569c5c9
--- /dev/null
+++ b/doc/forum/custom_URLs___40__use_case__58___dnanexus__41__/comment_4_94e2fbb04c553b2c8c7992541e634017._comment
@@ -0,0 +1,10 @@
+[[!comment format=mdwn
+ username="Ilya_Shlyakhter"
+ avatar="http://cdn.libravatar.org/avatar/1647044369aa7747829c38b9dcc84df0"
+ subject="comment 4"
+ date="2018-09-12T19:33:04Z"
+ content="""
+If I run 'git-annex addurl dx://file-xxxxx', won't git-annex just try calling curl to fetch the URL, instead of querying special remotes?
+I guess I can add my own curl wrapper to the front of PATH, but that feels somewhat hacky.
+
+"""]]
diff --git a/doc/forum/dangling_blobs.mdwn b/doc/forum/dangling_blobs.mdwn
new file mode 100644
index 0000000000..dc3cfbd78d
--- /dev/null
+++ b/doc/forum/dangling_blobs.mdwn
@@ -0,0 +1,27 @@
+Revisiting an issue I reported a couple of months ago but never figured out. I am trying to use git annex assistant on two separate machines to automatically mirror files between them. But after I start the second assistant and add new files to the annex, I find that git fsck reports dangling blobs. Is there a conflict between the two assistants?
+
+On the server:
+
+    $ mkdir ~/annex
+    $ cd ~/annex
+    $ git init
+    $ git annex init u --version=6
+    $ echo This is test file 1. >testfile1.txt
+    $ git annex add testfile1.txt
+    $ git annex sync
+    $ git remote add ml2 ssh://laptop/Users/username/annex
+    $ git annex assistant
+
+After all that, I do this on the laptop:
+
+    $ cd ~/
+    $ git clone ssh://server/home/username/annex
+    $ cd annex
+    $ git annex init ml2 --version=6
+    $ git annex sync
+    $ git annex assistant
+
+At this point git fsck is happy. But when I add files to the annex on either machine and run git fsck, I get messages like:
+
+    Checking object directories: 100% (256/256), done.
+    dangling blob 31a30177d1e37faf8eac96524302a61713d3d522
diff --git a/doc/forum/dangling_blobs/comment_1_120681851a595909e19afcc0334383ec._comment b/doc/forum/dangling_blobs/comment_1_120681851a595909e19afcc0334383ec._comment
new file mode 100644
index 0000000000..df086c0b2c
--- /dev/null
+++ b/doc/forum/dangling_blobs/comment_1_120681851a595909e19afcc0334383ec._comment
@@ -0,0 +1,22 @@
+[[!comment format=mdwn
+ username="joey"
+ subject="""comment 1"""
+ date="2017-02-17T16:33:02Z"
+ content="""
+This message from git fsck does not indicate any kind of serious problem
+with the repository.
+
+A dangling blob can be added to a git repository when something gets
+staged but is then never included in a commit. For example, with the
+assistant, if a file is created, and then modified soon after, the
+original version would get staged, but before it was committed, the
+new version staged and included in the commit. 
+
+There are several other scenarios, involving the assistant and not, where
+that could happen.
+
+So, git is telling you that the git repository contains
+information that's not being used. Unless you are low on disk
+space this probably doesn't matter, and git will automatically
+clean up dangling blobs after a while (or see `git gc`).
+"""]]
diff --git a/doc/forum/default_repo_version_is_still_5__63__.mdwn b/doc/forum/default_repo_version_is_still_5__63__.mdwn
new file mode 100644
index 0000000000..f7d7e9a997
--- /dev/null
+++ b/doc/forum/default_repo_version_is_still_5__63__.mdwn
@@ -0,0 +1 @@
+git-annex version: 6.20180807-gebc1bb5 supports creating version 6 repos, but by default creates version 5.  Are version 6 repos still in beta / not reliable?
diff --git a/doc/forum/default_repo_version_is_still_5__63__/comment_1_5f63f10cf79ef8ca33214ceaefd1271e._comment b/doc/forum/default_repo_version_is_still_5__63__/comment_1_5f63f10cf79ef8ca33214ceaefd1271e._comment
new file mode 100644
index 0000000000..7164e824df
--- /dev/null
+++ b/doc/forum/default_repo_version_is_still_5__63__/comment_1_5f63f10cf79ef8ca33214ceaefd1271e._comment
@@ -0,0 +1,9 @@
+[[!comment format=mdwn
+ username="joey"
+ subject="""comment 1"""
+ date="2018-09-11T17:14:04Z"
+ content="""
+That's right, see [[todo/smudge]] for the current status of v6.
+There are known issues, it's been improved a lot in unreleated git-annex
+but even then, still known issues including memory leaks in git.
+"""]]
diff --git a/doc/forum/deploy.mdwn b/doc/forum/deploy.mdwn
new file mode 100644
index 0000000000..b9e0974122
--- /dev/null
+++ b/doc/forum/deploy.mdwn
@@ -0,0 +1,5 @@
+Greetings,
+
+I use the push-to-deploy pattern (as described in 4.1 http://gitolite.com/deploy.html). However, my git repo has large binary files that I'd like to annex. Is there an example of using git annex with a bare remote repository with the appropriate post-receive hook to accomplish the deploy?
+
+Thanks
diff --git a/doc/forum/deploy/comment_1_bd9855cc19f0f49620258564845f1f77._comment b/doc/forum/deploy/comment_1_bd9855cc19f0f49620258564845f1f77._comment
new file mode 100644
index 0000000000..1f4732b8df
--- /dev/null
+++ b/doc/forum/deploy/comment_1_bd9855cc19f0f49620258564845f1f77._comment
@@ -0,0 +1,16 @@
+[[!comment format=mdwn
+ username="joey"
+ subject="""comment 1"""
+ date="2017-05-11T17:13:35Z"
+ content="""
+I'm not sure that a bare repository is the best way to go about it.
+
+One way to do it is to have a non-bare repository, which gets updated when
+you push (by eg setting receive.denyNonFastforwards=false), and have that
+non-bare repository be the one you push to deploy to.
+
+While you could push first to a bare repository and have it then push on
+the the non-bare repository, that seems to mostly only complicate things,
+since you will need to find a way to copy the annexed file contents from
+the bare to the non-bare repository.
+"""]]
diff --git a/doc/forum/differenc_in_webapp_icons__63__.mdwn b/doc/forum/differenc_in_webapp_icons__63__.mdwn
new file mode 100644
index 0000000000..3b4f355a1d
--- /dev/null
+++ b/doc/forum/differenc_in_webapp_icons__63__.mdwn
@@ -0,0 +1,4 @@
+Hi,
+
+I'm wondering what the difference between the network signal and network sync icons are in the web-app?  Nothing stands out to me why some repos have one some the other?
+
diff --git a/doc/forum/differenc_in_webapp_icons__63__/comment_1_c38e2692b13a1b76777bf88312a03966._comment b/doc/forum/differenc_in_webapp_icons__63__/comment_1_c38e2692b13a1b76777bf88312a03966._comment
new file mode 100644
index 0000000000..69777eaff9
--- /dev/null
+++ b/doc/forum/differenc_in_webapp_icons__63__/comment_1_c38e2692b13a1b76777bf88312a03966._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="http://joeyh.name/"
+ ip="209.250.56.114"
+ subject="comment 1"
+ date="2014-04-26T23:03:46Z"
+ content="""
+The signal icon shows when there's a direct connection to another repositoriry. This leats your repository immediately learn when there is a change, triggering a sync. Without any signal icons, there's no fast syncing when changes are made, so the webapp will prompt for you to set up such a connection.
+"""]]
diff --git a/doc/forum/difference_between_full_backup_and_number_of_copies__63__.mdwn b/doc/forum/difference_between_full_backup_and_number_of_copies__63__.mdwn
new file mode 100644
index 0000000000..4223a2ded7
--- /dev/null
+++ b/doc/forum/difference_between_full_backup_and_number_of_copies__63__.mdwn
@@ -0,0 +1,9 @@
+If I have three repositories setup in the git annex webui as:
+
+full backup,
+
+and every file put into one of the repos seems to be propagated to the other two,
+
+what is the usage of the setting "number of copies" which is default to 1 but can be increased to 2 or 3? Does this setting matter in this context?
+
+I'm using assistant version 5.20140517.4
diff --git a/doc/forum/difference_between_full_backup_and_number_of_copies__63__/comment_1_df1850059a7a3006db7cb5c588dac3d7._comment b/doc/forum/difference_between_full_backup_and_number_of_copies__63__/comment_1_df1850059a7a3006db7cb5c588dac3d7._comment
new file mode 100644
index 0000000000..e58f5402b9
--- /dev/null
+++ b/doc/forum/difference_between_full_backup_and_number_of_copies__63__/comment_1_df1850059a7a3006db7cb5c588dac3d7._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="https://www.google.com/accounts/o8/id?id=AItOawm8wY171R5c4u_jPmB6LU6n6Px2xePM4sE"
+ nickname="Efraim"
+ subject="comment 1"
+ date="2014-08-31T07:09:52Z"
+ content="""
+number of copies is the minimum number of copies that can exist when you try to drop a file from a repository/without git-annex telling you that you don't have enough copies and should protect your data better.  A full backup by default tries to get every file it can get its hands on, including old versions.
+"""]]
diff --git a/doc/forum/difference_between_uninit_and_unlock.mdwn b/doc/forum/difference_between_uninit_and_unlock.mdwn
new file mode 100644
index 0000000000..6f4d6def2e
--- /dev/null
+++ b/doc/forum/difference_between_uninit_and_unlock.mdwn
@@ -0,0 +1,10 @@
+What's the difference between
+
+    git annex uninit
+
+and 
+
+    git annex unlock *
+    rm -rf .git
+
+?
diff --git a/doc/forum/difference_between_uninit_and_unlock/comment_1_6b89cffc10650c087dbbfbdd3e099b5f._comment b/doc/forum/difference_between_uninit_and_unlock/comment_1_6b89cffc10650c087dbbfbdd3e099b5f._comment
new file mode 100644
index 0000000000..4db07242df
--- /dev/null
+++ b/doc/forum/difference_between_uninit_and_unlock/comment_1_6b89cffc10650c087dbbfbdd3e099b5f._comment
@@ -0,0 +1,10 @@
+[[!comment format=mdwn
+ username="joey"
+ subject="""comment 1"""
+ date="2016-05-03T18:51:22Z"
+ content="""
+In a v5 repo in indirect mode, they're broadly equivilant.
+
+`git annex uninit` removes git config changes and hooks that `git annex
+init` sets up, and generally hand-holds the process.
+"""]]
diff --git a/doc/forum/do_not_use_git-annex_inside_your_Dropbox.mdwn b/doc/forum/do_not_use_git-annex_inside_your_Dropbox.mdwn
new file mode 100644
index 0000000000..2669903449
--- /dev/null
+++ b/doc/forum/do_not_use_git-annex_inside_your_Dropbox.mdwn
@@ -0,0 +1,5 @@
+Do not place git-annex enabled git repository inside your Dropbox folder! 
+
+You will encounter broken symlinks and git-annex complaining about lost files. Dropbox [doesn't preserve case of file names](https://www.dropbox.com/help/145/en) and this breaks storage of annexed files (.git/annex/objects/). For possible recovery see .
+
+
diff --git a/doc/forum/do_not_use_git-annex_inside_your_Dropbox/comment_1_5a1dc9da6e6861829e321446ec7991ee._comment b/doc/forum/do_not_use_git-annex_inside_your_Dropbox/comment_1_5a1dc9da6e6861829e321446ec7991ee._comment
new file mode 100644
index 0000000000..d5da4f0005
--- /dev/null
+++ b/doc/forum/do_not_use_git-annex_inside_your_Dropbox/comment_1_5a1dc9da6e6861829e321446ec7991ee._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="sascha"
+ ip="2a02:8108:5c0:3ec:615a:1ac3:c7b9:a3a9"
+ subject="so how can i keep in sync a dropbox subfolder with a usbdrive"
+ date="2014-01-24T04:32:38Z"
+ content="""
+my usecase is i want to be able to be working on other computers on files that i have on my thumb-usb-drive. Back home i want to sync changes back to the subfolder of my local Dropbox-directory. Any suggestions?
+"""]]
diff --git a/doc/forum/done_bugs_are_missing.mdwn b/doc/forum/done_bugs_are_missing.mdwn
new file mode 100644
index 0000000000..1121f431d0
--- /dev/null
+++ b/doc/forum/done_bugs_are_missing.mdwn
@@ -0,0 +1,5 @@
+Joey, did you mean to delete all of the [done bugs](http://git-annex.branchable.com/bugs/done/) during your [mass bug closures](http://git-annex.branchable.com/devblog/day_498__unexpected_release_prep/)?
+
+I usually like to check done bugs on the website to see what has been fixed in a release. But, it seems many of the fixed bugs are now missing. I had been looking to see if you fixed the webapp not launching on OS-X, but I couldn't find the bug report.
+
+—[Andrew](http://git-annex.branchable.com/users/andrew/)
diff --git a/doc/forum/done_bugs_are_missing/comment_1_400c15824ec84bd67583102a023edcf8._comment b/doc/forum/done_bugs_are_missing/comment_1_400c15824ec84bd67583102a023edcf8._comment
new file mode 100644
index 0000000000..5c65bfa60e
--- /dev/null
+++ b/doc/forum/done_bugs_are_missing/comment_1_400c15824ec84bd67583102a023edcf8._comment
@@ -0,0 +1,10 @@
+[[!comment format=mdwn
+ username="andrew"
+ avatar="http://cdn.libravatar.org/avatar/acc0ece1eedf07dd9631e7d7d343c435"
+ subject="comment 1"
+ date="2018-05-31T03:15:37Z"
+ content="""
+Ooops. Hit post, before I finished.
+
+…but I couldn't find the bug report. I was pleasantly surprised when I did discover you had fixed this, so quickly! But the fix was only mentioned in the release notes. I think it is great to see the entire list of bugs you have fixed on the website, it shows how much effort you put into this project and how actively maintained it is. 
+"""]]
diff --git a/doc/forum/done_bugs_are_missing/comment_2_1fdcfb4ed807e424fe73f134c4ad9526._comment b/doc/forum/done_bugs_are_missing/comment_2_1fdcfb4ed807e424fe73f134c4ad9526._comment
new file mode 100644
index 0000000000..880b012f72
--- /dev/null
+++ b/doc/forum/done_bugs_are_missing/comment_2_1fdcfb4ed807e424fe73f134c4ad9526._comment
@@ -0,0 +1,11 @@
+[[!comment format=mdwn
+ username="joey"
+ subject="""comment 2"""
+ date="2018-05-31T15:50:05Z"
+ content="""
+The page was simply only listing the most recently closed 10
+bugs, and not all the others. I've changed it to list them all.
+
+(I do archive old closed bugs, generally about a year after the last
+activity on the bug, to keep the size of the wiki in check.)
+"""]]
diff --git a/doc/forum/done_bugs_are_missing/comment_3_6eb74e1bcf6b062b61b25e59b0b1ddc6._comment b/doc/forum/done_bugs_are_missing/comment_3_6eb74e1bcf6b062b61b25e59b0b1ddc6._comment
new file mode 100644
index 0000000000..2558227bbd
--- /dev/null
+++ b/doc/forum/done_bugs_are_missing/comment_3_6eb74e1bcf6b062b61b25e59b0b1ddc6._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="andrew"
+ avatar="http://cdn.libravatar.org/avatar/acc0ece1eedf07dd9631e7d7d343c435"
+ subject="comment 3"
+ date="2018-05-31T17:43:21Z"
+ content="""
+Thanks!
+"""]]
diff --git a/doc/forum/dot_git_slash_annex_slash_tmp.mdwn b/doc/forum/dot_git_slash_annex_slash_tmp.mdwn
new file mode 100644
index 0000000000..5f1262c7aa
--- /dev/null
+++ b/doc/forum/dot_git_slash_annex_slash_tmp.mdwn
@@ -0,0 +1,5 @@
+When I was checking out disk usage on my home directory, I noticed a bunch of files stored in .git/annex/tmp under their own filenames, rather than in .git/annex/objects.  Checking a couple, they seem to be redundant with stuff in .git/annex/objects.
+
+Is there any way to clean that stuff up?  Will the assistant eventually clean it up?
+
+Thanks!
diff --git a/doc/forum/dot_git_slash_annex_slash_tmp/comment_10_14b74438bb1e3e02cff7926d774ba09a._comment b/doc/forum/dot_git_slash_annex_slash_tmp/comment_10_14b74438bb1e3e02cff7926d774ba09a._comment
new file mode 100644
index 0000000000..fe7a71dcbf
--- /dev/null
+++ b/doc/forum/dot_git_slash_annex_slash_tmp/comment_10_14b74438bb1e3e02cff7926d774ba09a._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="http://edheil.wordpress.com/"
+ ip="173.162.44.162"
+ subject="comment 10"
+ date="2013-06-06T13:16:50Z"
+ content="""
+Huh, at some point this happened again.  Ended up with 8 gigs of files in that tmp dir.  Cleaning out again.  Yay, disk space!
+"""]]
diff --git a/doc/forum/dot_git_slash_annex_slash_tmp/comment_11_b1f717342c1c8ea42a451caa2d936622._comment b/doc/forum/dot_git_slash_annex_slash_tmp/comment_11_b1f717342c1c8ea42a451caa2d936622._comment
new file mode 100644
index 0000000000..c882a85731
--- /dev/null
+++ b/doc/forum/dot_git_slash_annex_slash_tmp/comment_11_b1f717342c1c8ea42a451caa2d936622._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="stoile"
+ ip="2a01:198:242:0:219:66ff:fef2:c021"
+ subject="Same problem here"
+ date="2013-11-16T12:10:55Z"
+ content="""
+I got the same problem on linux. Under what circumstances does \"git annex unused\" not find files in .git/annex/tmp? Maybe it is possible to improve \"git annex unused\"...
+"""]]
diff --git a/doc/forum/dot_git_slash_annex_slash_tmp/comment_12_e2c6ad99333018c8c46e736da416b8ef._comment b/doc/forum/dot_git_slash_annex_slash_tmp/comment_12_e2c6ad99333018c8c46e736da416b8ef._comment
new file mode 100644
index 0000000000..153a617b57
--- /dev/null
+++ b/doc/forum/dot_git_slash_annex_slash_tmp/comment_12_e2c6ad99333018c8c46e736da416b8ef._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="https://www.google.com/accounts/o8/id?id=AItOawmkXtBdMgE1d9nCz2iBc4f85xh4izZ_auU"
+ nickname="Ulrich"
+ subject="Wow. Found >180 GB of data in annex/tmp"
+ date="2013-11-24T21:53:29Z"
+ content="""
+… no wonder my disk was filling up. On Mac OS X 10.9…
+"""]]
diff --git a/doc/forum/dot_git_slash_annex_slash_tmp/comment_13_35ae9d6bcb8d9762a92e3564b686ed72._comment b/doc/forum/dot_git_slash_annex_slash_tmp/comment_13_35ae9d6bcb8d9762a92e3564b686ed72._comment
new file mode 100644
index 0000000000..0cbeb8f5ea
--- /dev/null
+++ b/doc/forum/dot_git_slash_annex_slash_tmp/comment_13_35ae9d6bcb8d9762a92e3564b686ed72._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="http://joeyh.name/"
+ ip="209.250.56.87"
+ subject="comment 13"
+ date="2013-12-12T20:01:09Z"
+ content="""
+The people who are apparently still having this problem need to provide details about the names of the files in the temp directory. Otherwise, I cannot fix whatever problem you are encountering.
+"""]]
diff --git a/doc/forum/dot_git_slash_annex_slash_tmp/comment_14_92b1e8956513dbf52da31cec3f58e2c5._comment b/doc/forum/dot_git_slash_annex_slash_tmp/comment_14_92b1e8956513dbf52da31cec3f58e2c5._comment
new file mode 100644
index 0000000000..b84e663ccc
--- /dev/null
+++ b/doc/forum/dot_git_slash_annex_slash_tmp/comment_14_92b1e8956513dbf52da31cec3f58e2c5._comment
@@ -0,0 +1,104 @@
+[[!comment format=mdwn
+ username="http://alan.petitepomme.net/"
+ nickname="Alan Schmitt"
+ subject="comment 14"
+ date="2014-04-04T11:07:31Z"
+ content="""
+I just discovered there were 30GB in this tmp directory. Here are the file names. It seems that the file names are partial (they are missing some characters: it seems most file names are missing the last 16 characters and the file extension). As I'm in dire need of disk space, I'm going to delete this files now.
+
+    schmitta@top-wifi ~/D/a/.g/a/tmp (GIT_DIR!)> pwd
+    /Users/schmitta/Documents/annex/.git/annex/tmp
+    schmitta@top-wifi ~/D/a/.g/a/tmp (GIT_DIR!)> ls -al
+    total 73802184
+    drwxr-xr-x  91 schmitta  staff        3094 16 mar 11:15 .
+    drwxr-xr-x  34 schmitta  staff        1156  4 avr 12:58 ..
+    -r--r--r--@  6 schmitta  staff   132882914  8 déc 20:17 12426.
+    -r--r--r--@  6 schmitta  staff   113471066  8 déc 20:15 12426.2-
+    -r--r--r--@  6 schmitta  staff   148673586  8 déc 20:24 12427.
+    -r--r--r--@  6 schmitta  staff    96197733  8 déc 20:09 12427.2-
+    -r--r--r--@  6 schmitta  staff   132882914  8 déc 20:17 12489.
+    -r--r--r--@  6 schmitta  staff   113471066  8 déc 20:15 12489.2-
+    -r--r--r--@  6 schmitta  staff   148673586  8 déc 20:24 12490.
+    -r--r--r--@  6 schmitta  staff    96197733  8 déc 20:09 12490.2-
+    -r--r--r--@  6 schmitta  staff   132882914  8 déc 20:17 12498.
+    -r--r--r--@  6 schmitta  staff   113471066  8 déc 20:15 12498.2-
+    -r--r--r--@  6 schmitta  staff   148673586  8 déc 20:24 12499.
+    -r--r--r--@  6 schmitta  staff    96197733  8 déc 20:09 12499.2-
+    -r--r--r--@  6 schmitta  staff   132882914  8 déc 20:17 12504.
+    -r--r--r--@  6 schmitta  staff   113471066  8 déc 20:15 12504.2-
+    -r--r--r--@  6 schmitta  staff   148673586  8 déc 20:24 12505.
+    -r--r--r--@  6 schmitta  staff    96197733  8 déc 20:09 12505.2-
+    -r--r--r--@  6 schmitta  staff   132882914  8 déc 20:17 12562.
+    -r--r--r--@  6 schmitta  staff   113471066  8 déc 20:15 12562.2-
+    -r--r--r--@  6 schmitta  staff   148673586  8 déc 20:24 12563.
+    -r--r--r--@  6 schmitta  staff    96197733  8 déc 20:09 12563.2-
+    -r--r--r--   2 schmitta  staff  1237391994 21 déc 19:27 BTSM With Flowerchild Li562
+    -r--r--r--   2 schmitta  staff  1097868668 18 déc 13:24 BTSM with Flowerchild 562
+    -r--r--r--   3 schmitta  staff   181962494  1 déc 05:19 Cosmos - A Personal Voyage - Episod422
+    -r--r--r--   3 schmitta  staff   199341564 17 déc 14:28 Cosmos - A Personal Voyage - Episod423
+    -r--r--r--   3 schmitta  staff   196677905  9 jan 08:36 Cosmos - A Personal Voyage - Episod424
+    -r--r--r--   3 schmitta  staff   196671683 26 déc 19:36 Cosmos - A Personal Voyage - Episod425
+    -r--r--r--   3 schmitta  staff   199297657 19 jan 14:33 Cosmos - A Personal Voyage - Episod426
+    -r--r--r--   3 schmitta  staff   194250261 18 jan 17:10 Cosmos - A Personal Voyage - Episod427
+    -r--r--r--   3 schmitta  staff   216726281 16 jan 16:26 Cosmos - A Personal Voyage - Episod428
+    -r--r--r--   3 schmitta  staff   204634401 16 jan 12:17 Cosmos - A Personal Voyage - Episod429
+    -r--r--r--   3 schmitta  staff   179412519 11 jan 11:29 Cosmos - A Personal Voyage - Episod430
+    -r--r--r--   3 schmitta  staff   181962494  1 déc 05:19 Cosmos - A Personal Voyage - Episod572
+    -r--r--r--   3 schmitta  staff   199341564 17 déc 14:28 Cosmos - A Personal Voyage - Episod573
+    -r--r--r--   3 schmitta  staff   196677905  9 jan 08:36 Cosmos - A Personal Voyage - Episod574
+    -r--r--r--   3 schmitta  staff   196671683 26 déc 19:36 Cosmos - A Personal Voyage - Episod575
+    -r--r--r--   3 schmitta  staff   199297657 19 jan 14:33 Cosmos - A Personal Voyage - Episod576
+    -r--r--r--   3 schmitta  staff   194250261 18 jan 17:10 Cosmos - A Personal Voyage - Episod577
+    -r--r--r--   3 schmitta  staff   216726281 16 jan 16:26 Cosmos - A Personal Voyage - Episod578
+    -r--r--r--   3 schmitta  staff   204634401 16 jan 12:17 Cosmos - A Personal Voyage - Episod579
+    -r--r--r--   3 schmitta  staff   179412519 11 jan 11:29 Cosmos - A Personal Voyage - Episod580
+    -r--r--r--   3 schmitta  staff   237284500 15 mai  2012 Cosmos - A Personal Voyage - Episode422
+    -r--r--r--   3 schmitta  staff   207386296 28 jan 18:39 Cosmos - A Personal Voyage - Episode423
+    -r--r--r--   3 schmitta  staff   194659898 28 jan 20:35 Cosmos - A Personal Voyage - Episode424
+    -r--r--r--   3 schmitta  staff   221944244 11 jan 07:18 Cosmos - A Personal Voyage - Episode425
+    -r--r--r--   3 schmitta  staff   237284500 15 mai  2012 Cosmos - A Personal Voyage - Episode572
+    -r--r--r--   3 schmitta  staff   207386296 28 jan 18:39 Cosmos - A Personal Voyage - Episode573
+    -r--r--r--   3 schmitta  staff   194659898 28 jan 20:35 Cosmos - A Personal Voyage - Episode574
+    -r--r--r--   3 schmitta  staff   221944244 11 jan 07:18 Cosmos - A Personal Voyage - Episode575
+    -r--r--r--   1 schmitta  staff   166417358 25 nov 08:47 Icy Plays - Kerbal Space Program - Better Than Starting Mann456
+    -r--r--r--   2 schmitta  staff  1110410229 28 déc 00:40 Kerbal Space Program - BTSM With Flowerchild Li562
+    -r--r--r--   1 schmitta  staff   898936971 26 déc 06:22 Let's Play Minecraft Better Than Wolves   Ep 10 _ HARDCORE _ Ooooh le petit mat572
+    -r--r--r--   1 schmitta  staff   816306596 27 déc 05:08 Let's Play Minecraft Better Than Wolves   Ep 11 _ HARDCORE _ Un peu de cannabis pour le moral 572
+    -r--r--r--   1 schmitta  staff   897143936 27 déc 10:42 Let's Play Minecraft Better Than Wolves   Ep 12 _ HARDCORE _ Chaud l'enf572
+    -r--r--r--   1 schmitta  staff   922364739 29 déc 08:30 Let's Play Minecraft Better Than Wolves   Ep 13 _ HARDCORE _ Walking dea572
+    -r--r--r--   1 schmitta  staff  1321555505  1 jan 02:15 Let's Play Minecraft Better Than Wolves   Ep 14 l HARDCORE l Couvre572
+    -r--r--r--   1 schmitta  staff   889592223  5 jan 02:07 Let's Play Minecraft Better Than Wolves   Ep 15 l HARDCORE l Pa572
+    -r--r--r--   1 schmitta  staff   153571799  6 jan 03:56 Let's Play Minecraft Better Than Wolves   Ep 16 l HARDCORE l Point de s572
+    -r--r--r--   1 schmitta  staff  1036932250  9 jan 07:36 Let's Play Minecraft Better Than Wolves   Ep 17 l HARDCORE l Satané meuh572
+    -r--r--r--   1 schmitta  staff  1289798767 11 jan 22:08 Let's Play Minecraft Better Than Wolves   Ep 18 l HARDCORE l Un Moulin572
+    -r--r--r--   1 schmitta  staff  1582120875 19 jan 15:39 Let's Play Minecraft Better Than Wolves   Ep 19 l HARDCORE l Explosion de ca572
+    -r--r--r--   1 schmitta  staff   728469432 13 fév 07:02 Let's Play Minecraft Better Than Wolves   Ep 2 _ HARDCORE _ Préparation pour l'explorat422
+    -r--r--r--   1 schmitta  staff   911653518 22 jan 13:35 Let's Play Minecraft Better Than Wolves   Ep 20 l HARDCORE l Minesha572
+    -r--r--r--   1 schmitta  staff  1377428597 28 jan 08:43 Let's Play Minecraft Better Than Wolves   Ep 21 l HARDCORE l Terreur en sous-sol o_O !572
+    -r--r--r--   1 schmitta  staff  1376224927  4 fév 03:24 Let's Play Minecraft Better Than Wolves   Ep 22 l HARDCORE l Machinerie infern572
+    -r--r--r--   1 schmitta  staff  1102293665 13 fév 03:56 Let's Play Minecraft Better Than Wolves   Ep 23 l HARDCORE l Piston chenil572
+    -r--r--r--   1 schmitta  staff  1835677068 19 fév 09:14 Let's Play Minecraft Better Than Wolves   Ep 24 l HARDCORE l Piston chenille et o_Oce572
+    -r--r--r--   1 schmitta  staff   753740284 30 déc 19:31 Let's Play Minecraft Better Than Wolves   Ep 3 _ HARDCORE _ First pioche en fer422
+    -r--r--r--   1 schmitta  staff  1019022279 30 déc 18:38 Let's Play Minecraft Better Than Wolves   Ep 5 _ HARDCORE _ Fonderie super desi572
+    -r--r--r--   1 schmitta  staff   788714805 30 déc 17:47 Let's Play Minecraft Better Than Wolves   Ep 6 _ HARDCORE _ Minage de la mor572
+    -r--r--r--   1 schmitta  staff   778987710 23 déc 19:56 Let's Play Minecraft Better Than Wolves   Ep 7 _ HARDCORE _ Diamant572
+    -r--r--r--   1 schmitta  staff   644564981 23 déc 23:06 Let's Play Minecraft Better Than Wolves   Ep 8 _ HARDCORE _ Et le poulet ma572
+    -r--r--r--   1 schmitta  staff   871173289 24 déc 10:44 Let's Play Minecraft Better Than Wolves   Ep 9 _ HARDCORE _ Araignée verte de la jun572
+    -r--------@  4 schmitta  staff   149520850 14 aoû  2010 OPLSS10-Mc422
+    -r--------@  4 schmitta  staff    95020824 14 aoû  2010 OPLSS10-Mc423
+    -r--------@  4 schmitta  staff    98980596  1 sep  2010 OPLSS10-Mc424
+    -r--------@  4 schmitta  staff   207525348  1 sep  2010 OPLSS10-Mc425
+    -r--------@  4 schmitta  staff   149520850 14 aoû  2010 OPLSS10-Mc534
+    -r--------@  4 schmitta  staff    95020824 14 aoû  2010 OPLSS10-Mc535
+    -r--------@  4 schmitta  staff    98980596  1 sep  2010 OPLSS10-Mc536
+    -r--------@  4 schmitta  staff   207525348  1 sep  2010 OPLSS10-Mc537
+    -r--------@  4 schmitta  staff   149520850 14 aoû  2010 OPLSS10-Mc572
+    -r--------@  4 schmitta  staff    95020824 14 aoû  2010 OPLSS10-Mc573
+    -r--------@  4 schmitta  staff    98980596  1 sep  2010 OPLSS10-Mc574
+    -r--------@  4 schmitta  staff   207525348  1 sep  2010 OPLSS10-Mc575
+    -r--r--r--@  2 schmitta  staff   150781568 10 mar 13:34 XD300-23_68HighlightsAResearchCntAugHuma469
+    -r--r--r--@  2 schmitta  staff   145745487 10 mar 13:36 XD300-24_68HighlightsAResearchCntAugHuma469
+    -r--r--r--@  2 schmitta  staff   142414085 10 mar 13:36 XD300-25_68HighlightsAResearchCntAugHuma469
+    -r--------@  2 schmitta  staff   548973365  7 jan 20:21 laumond-20120119449.mp4
+    -r--r--r--@  2 schmitta  staff  1820482781  8 mar 18:15 salt-020131469
+"""]]
diff --git a/doc/forum/dot_git_slash_annex_slash_tmp/comment_15_839e29d41de9dcc8f01dfdc585a51d12._comment b/doc/forum/dot_git_slash_annex_slash_tmp/comment_15_839e29d41de9dcc8f01dfdc585a51d12._comment
new file mode 100644
index 0000000000..36ad14e57c
--- /dev/null
+++ b/doc/forum/dot_git_slash_annex_slash_tmp/comment_15_839e29d41de9dcc8f01dfdc585a51d12._comment
@@ -0,0 +1,12 @@
+[[!comment format=mdwn
+ username="http://joeyh.name/"
+ ip="216.145.95.162"
+ subject="comment 15"
+ date="2014-05-19T15:43:52Z"
+ content="""
+Since version 5.20140517, the git-annex assistant will automatically clean up stale tmp files on startup.
+
+If not using the assistant, you can do it yourself..
+
+So far, all the tmp files people have been kind enough to share the details about with me seem to be created by the assistant when it locks a file down. I know this can result in dangling files if the computer is rebooted while the assistant is in the middle of doing that.
+"""]]
diff --git a/doc/forum/dot_git_slash_annex_slash_tmp/comment_1_1a35ef8cb89e0cd392f6e9fcee1fb92c._comment b/doc/forum/dot_git_slash_annex_slash_tmp/comment_1_1a35ef8cb89e0cd392f6e9fcee1fb92c._comment
new file mode 100644
index 0000000000..d00f681443
--- /dev/null
+++ b/doc/forum/dot_git_slash_annex_slash_tmp/comment_1_1a35ef8cb89e0cd392f6e9fcee1fb92c._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="http://joeyh.name/"
+ ip="4.154.6.49"
+ subject="comment 1"
+ date="2012-11-29T19:35:41Z"
+ content="""
+These are partially downloaded files. You can use `git annex unused` to clean them up. The assistant doesn't yet explicitly clean these up, but it does keep retrying (and when possible, resuming) failed downloads until they succeed, at which point the temp file is moved into the objects directory.
+"""]]
diff --git a/doc/forum/dot_git_slash_annex_slash_tmp/comment_2_f4cc36c493d7c20fbaf949edd38e1252._comment b/doc/forum/dot_git_slash_annex_slash_tmp/comment_2_f4cc36c493d7c20fbaf949edd38e1252._comment
new file mode 100644
index 0000000000..8aeaf63cfd
--- /dev/null
+++ b/doc/forum/dot_git_slash_annex_slash_tmp/comment_2_f4cc36c493d7c20fbaf949edd38e1252._comment
@@ -0,0 +1,15 @@
+[[!comment format=mdwn
+ username="http://edheil.wordpress.com/"
+ ip="173.162.44.162"
+ subject="comment 2"
+ date="2012-11-29T19:53:33Z"
+ content="""
+that's odd... these files are on my client machine, and shouldn't be downloaded from anywhere, just uploaded to other places.  There are lots and lots of them.
+
+git-annex unused only found two files, and there are dozens and dozens of these files in my .git/annex/tmp directory.  
+
+Would it be safe to delete them by hand?
+
+(The only thing I can think of that might e relevant is, some of these have definitely been moved from place to place within my repo.  I use \"git mv\" when I do that.  Maybe I messed that up somehow, and git-annex thought they had disappeared and needed to be re-downloaded from a remote repo?)
+
+"""]]
diff --git a/doc/forum/dot_git_slash_annex_slash_tmp/comment_3_69268f8aa29e807a56248f1fac86aa41._comment b/doc/forum/dot_git_slash_annex_slash_tmp/comment_3_69268f8aa29e807a56248f1fac86aa41._comment
new file mode 100644
index 0000000000..d9863375ae
--- /dev/null
+++ b/doc/forum/dot_git_slash_annex_slash_tmp/comment_3_69268f8aa29e807a56248f1fac86aa41._comment
@@ -0,0 +1,10 @@
+[[!comment format=mdwn
+ username="http://joeyh.name/"
+ ip="4.154.6.49"
+ subject="comment 3"
+ date="2012-11-29T20:22:01Z"
+ content="""
+No, moving files within the repository is unlikely to be related.
+
+If you can show me what the filenames look like I'll have a much better chance of getting somewhere, since they do not seem to be filenames that `git annex unused` knows how to deal with, and so I don't know what they could be.
+"""]]
diff --git a/doc/forum/dot_git_slash_annex_slash_tmp/comment_4_0ffb0c803c232a1587f956f16113aeb7._comment b/doc/forum/dot_git_slash_annex_slash_tmp/comment_4_0ffb0c803c232a1587f956f16113aeb7._comment
new file mode 100644
index 0000000000..11da788418
--- /dev/null
+++ b/doc/forum/dot_git_slash_annex_slash_tmp/comment_4_0ffb0c803c232a1587f956f16113aeb7._comment
@@ -0,0 +1,27 @@
+[[!comment format=mdwn
+ username="http://edheil.wordpress.com/"
+ ip="173.162.44.162"
+ subject="comment 4"
+ date="2012-11-29T20:33:09Z"
+ content="""
+They're the full actual filenames (but not directories) of particular files in my repository, sometimes with numeric suffixes (or rather, numbers before the file suffixes), like this:
+
+746.DS_Store
+747.DS_Store
+748.DS_Store
+749.DS_Store
+Birdplane87712.mp4
+Birdplane87713.mp4
+Bodyrock87712.mp4
+Bodyrock87713.mp4
+Dubstep Violin- Lindsey Stirling- Crystallize746.mp4
+Dubstep Violin- Lindsey Stirling- Crystallize87712.mp4
+Dubstep Violin- Lindsey Stirling- Crystallize87713.mp4
+Handlebars (fan video)87712.mp4
+Handlebars (fan video)87713.mp4
+Handlebars (official)87712.mp4
+Handlebars (official)87713.mp4
+
+Most of them are videos, and I've moved videos around a lot in my repository, and made some mistakes doing so from time to time when I was first learning git-annex, and fooling around on the command line -- breaking symlinks and figuring out how to fix that with git-annex fix.
+
+"""]]
diff --git a/doc/forum/dot_git_slash_annex_slash_tmp/comment_5_c303e28825241733d69fca74f2015fc6._comment b/doc/forum/dot_git_slash_annex_slash_tmp/comment_5_c303e28825241733d69fca74f2015fc6._comment
new file mode 100644
index 0000000000..60896ab5c7
--- /dev/null
+++ b/doc/forum/dot_git_slash_annex_slash_tmp/comment_5_c303e28825241733d69fca74f2015fc6._comment
@@ -0,0 +1,13 @@
+[[!comment format=mdwn
+ username="http://edheil.wordpress.com/"
+ ip="173.162.44.162"
+ subject="comment 5"
+ date="2012-11-29T20:47:36Z"
+ content="""
+Oh, another stupid thing I might have done -- 
+
+* added files to the repo, and not realizing it might take assistant a while to recognize and add them all, gotten impatient
+* added them myself, but did it with \"import\" instead of \"add\" so I was git-annex importing files from *inside* git-annex, with an assistant process running at the same time.
+
+Might not be relevant, just trying to think of weird things I've done.
+"""]]
diff --git a/doc/forum/dot_git_slash_annex_slash_tmp/comment_6_3f0b376e37bd092b8d46c46bb1940e35._comment b/doc/forum/dot_git_slash_annex_slash_tmp/comment_6_3f0b376e37bd092b8d46c46bb1940e35._comment
new file mode 100644
index 0000000000..0fc7f5c159
--- /dev/null
+++ b/doc/forum/dot_git_slash_annex_slash_tmp/comment_6_3f0b376e37bd092b8d46c46bb1940e35._comment
@@ -0,0 +1,14 @@
+[[!comment format=mdwn
+ username="http://joeyh.name/"
+ ip="4.154.6.49"
+ subject="comment 6"
+ date="2012-11-29T20:53:34Z"
+ content="""
+Ah, I think I know what this is. When the assistant notices a new file appear, it makes a hard link in the temp directory to the new file, and waits for it to stop being written to. 
+
+If the hard link is broken before the assistant is done adding the file to the annex, it looks like it could leave the temp file behind. I've committed a change that should avoid that.
+
+Since some of these are OSX DS_Store files, and the rest have a strange number added (which git-annex did not do), I suspect you moved them into the annexed directory using the OSX Finder or something like that, and whatever was putting them into the directory first wrote them with the names we see here. Then, while git-annex was still processing them, they got renamed.
+
+I'd say you can safely delete them.
+"""]]
diff --git a/doc/forum/dot_git_slash_annex_slash_tmp/comment_7_615641b3dd176d4b3a5bbfb521098e38._comment b/doc/forum/dot_git_slash_annex_slash_tmp/comment_7_615641b3dd176d4b3a5bbfb521098e38._comment
new file mode 100644
index 0000000000..20e8b205f6
--- /dev/null
+++ b/doc/forum/dot_git_slash_annex_slash_tmp/comment_7_615641b3dd176d4b3a5bbfb521098e38._comment
@@ -0,0 +1,9 @@
+[[!comment format=mdwn
+ username="http://edheil.wordpress.com/"
+ ip="173.162.44.162"
+ subject="comment 7"
+ date="2012-11-29T22:15:11Z"
+ content="""
+Sounds likely!!  I'll get to deleting, and when I get a chance I'll grab your latest commit and recompile.
+
+"""]]
diff --git a/doc/forum/dot_git_slash_annex_slash_tmp/comment_8_4600fa9234a787004ea0e0dbb36184b9._comment b/doc/forum/dot_git_slash_annex_slash_tmp/comment_8_4600fa9234a787004ea0e0dbb36184b9._comment
new file mode 100644
index 0000000000..3325f598a5
--- /dev/null
+++ b/doc/forum/dot_git_slash_annex_slash_tmp/comment_8_4600fa9234a787004ea0e0dbb36184b9._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="http://edheil.wordpress.com/"
+ ip="173.162.44.162"
+ subject="comment 8"
+ date="2012-11-29T22:15:55Z"
+ content="""
+(thanks very much!)
+"""]]
diff --git a/doc/forum/dot_git_slash_annex_slash_tmp/comment_9_4f5cd0d0d4db0479c1ad86ffdc5ae434._comment b/doc/forum/dot_git_slash_annex_slash_tmp/comment_9_4f5cd0d0d4db0479c1ad86ffdc5ae434._comment
new file mode 100644
index 0000000000..cc996c825d
--- /dev/null
+++ b/doc/forum/dot_git_slash_annex_slash_tmp/comment_9_4f5cd0d0d4db0479c1ad86ffdc5ae434._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="https://www.google.com/accounts/o8/id?id=AItOawl2Jj8q2upJL4ZQAc2lp7ugTxJiGtcICv8"
+ nickname="Michael"
+ subject="comment 9"
+ date="2013-02-20T10:23:12Z"
+ content="""
+I'm seeing this too, and the number being added to the file name is the PID of the git-annex process.
+"""]]
diff --git a/doc/forum/download_from_multiple_remotes_in_parallel.mdwn b/doc/forum/download_from_multiple_remotes_in_parallel.mdwn
new file mode 100644
index 0000000000..d749929be1
--- /dev/null
+++ b/doc/forum/download_from_multiple_remotes_in_parallel.mdwn
@@ -0,0 +1,9 @@
+Hey everyone, 
+
+I have the situation that there are two remote repositories A and B, both of which have several files that I want to get. The problem is that both A and B have limited upstream, so launching several threads downloading from A would not speed up the download, but downloading from A and B in parallel would help significantly.
+
+When I git-annex-get these files, my dream would be that specifying --jobs=2 has the effect that thread #1 only downloads from A and thread #2 only downloads from B. If this is the case, great, but if it is not (and I expect it to not be the case), is there any way to get files in parallel from A and B, but only ever having one connection to each of these repositories?
+
+Thanks a lot!
+
+
diff --git a/doc/forum/download_from_multiple_remotes_in_parallel/comment_1_81e2612ddaffefec81455e9ff706c4c9._comment b/doc/forum/download_from_multiple_remotes_in_parallel/comment_1_81e2612ddaffefec81455e9ff706c4c9._comment
new file mode 100644
index 0000000000..bb603c819d
--- /dev/null
+++ b/doc/forum/download_from_multiple_remotes_in_parallel/comment_1_81e2612ddaffefec81455e9ff706c4c9._comment
@@ -0,0 +1,14 @@
+[[!comment format=mdwn
+ username="joey"
+ subject="""comment 1"""
+ date="2016-09-05T20:18:56Z"
+ content="""
+Currently the way to do this to run two git-annex get's in parallel, one
+--from A and the other --from B. They will avoid stepping on each other's
+toes.
+
+-J2 does not currently accomplish this; git-annex always defaults to
+using the remote with the lowest cost. It might be a nice improvement to
+have get round-robin when multiple remotes of the same cost contain files.
+[[todo/get_round_robin]]
+"""]]
diff --git a/doc/forum/downloading_from_moodle.mdwn b/doc/forum/downloading_from_moodle.mdwn
new file mode 100644
index 0000000000..ad38d7f1ff
--- /dev/null
+++ b/doc/forum/downloading_from_moodle.mdwn
@@ -0,0 +1,76 @@
+my school uses moodle for our classes.  We have to sign in and then manually click to download each file, assignment and video uploaded.  I asked the school's tech administrator if there was a direct way I could access the videos through the ssh access they've given us to one of the servers, but he said it wasn't possible.
+
+when I click on the link shown, the location I see is: http://moodle.jct.ac.il/mod/resource/view.php?id=135374
+Inspect element gives more information. the response from the server is:
+
+    Remote Address:147.161.6.59:80
+    Request URL:http://moodle.jct.ac.il/mod/resource/view.php?id=135374
+    Request Method:GET
+    Status Code:303 See Other
+    Request Headersview source
+    Accept:text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8
+    Accept-Encoding:gzip,deflate,sdch
+    Accept-Language:en-US,en;q=0.8,he;q=0.6
+    Connection:keep-alive
+    Cookie:visid_incap_97364=qJx2WaKqQfGidGf9VfM6QWrnlFIAAAAAQUIPAAAAAAC5EXcbt00vqNu9jdVDwEDN; __utma=98014340.1409421308.1381214363.1381214363.1390387318.2; __utmz=98014340.1381214363.1.1.utmcsr=(direct)|utmccn=(direct)|utmcmd=(none); MoodleSession5771=7s1cfqfo4ahdtmna5h7vserg97; MOODLEID1_5771=%257F%25D39%2522N%25B4%25AFY
+    DNT:1
+    Host:moodle.jct.ac.il
+    Referer:http://moodle.jct.ac.il/course/view.php?id=20151
+    User-Agent:Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/36.0.1985.67 Safari/537.36
+    Query String Parametersview sourceview URL encoded
+    id:135374
+    Response Headersview source
+    Cache-Control:no-store, no-cache, must-revalidate, post-check=0, pre-check=0
+    Connection:Keep-Alive
+    Content-Encoding:gzip
+    Content-Language:he
+    Content-Length:503
+    Content-Type:text/html
+    Date:Mon, 16 Jun 2014 12:31:22 GMT
+    Expires:Thu, 19 Nov 1981 08:52:00 GMT
+    Keep-Alive:timeout=15, max=100
+    Location:http://moodle.jct.ac.il/pluginfile.php/288409/mod_resource/content/0/movie%205773/150151.5773.week1.wmv?forcedownload=1
+    Pragma:no-cache
+    Server:Apache/2.2.14 (Ubuntu)
+    Vary:Accept-Encoding
+    X-Powered-By:PHP/5.3.2-1ubuntu4.24
+
+this then pulls the following: 
+
+    Remote Address:147.161.6.59:80
+    Request URL:http://moodle.jct.ac.il/pluginfile.php/288409/mod_resource/content/0/movie%205773/150151.5773.week1.wmv?forcedownload=1
+    Request Method:GET
+    Status Code:200 OK
+    Request Headersview source
+    Accept:text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8
+    Accept-Encoding:gzip,deflate,sdch
+    Accept-Language:en-US,en;q=0.8,he;q=0.6
+    Connection:keep-alive
+    Cookie:visid_incap_97364=qJx2WaKqQfGidGf9VfM6QWrnlFIAAAAAQUIPAAAAAAC5EXcbt00vqNu9jdVDwEDN; __utma=98014340.1409421308.1381214363.1381214363.1390387318.2; __utmz=98014340.1381214363.1.1.utmcsr=(direct)|utmccn=(direct)|utmcmd=(none); MoodleSession5771=7s1cfqfo4ahdtmna5h7vserg97; MOODLEID1_5771=%257F%25D39%2522N%25B4%25AFY
+    DNT:1
+    Host:moodle.jct.ac.il
+    Referer:http://moodle.jct.ac.il/course/view.php?id=20151
+    User-Agent:Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/36.0.1985.67 Safari/537.36
+    Query String Parametersview sourceview URL encoded
+    forcedownload:1
+    Response Headersview source
+    Accept-Ranges:bytes
+    Cache-Control:max-age=86400
+    Connection:Keep-Alive
+    Content-Disposition:attachment; filename="150151.5773.week1.wmv"
+    Content-Length:1353673203
+    Content-Type:application/x-forcedownload
+    Date:Mon, 16 Jun 2014 12:31:23 GMT
+    ETag:675e7d2cffd7a79afd8686c59ff2533f9e3508b7
+    Expires:Tue, 17 Jun 2014 12:31:23 GMT
+    Keep-Alive:timeout=15, max=99
+    Last-Modified:Fri, 19 Jul 2013 17:06:54 GMT
+    Pragma:
+    Server:Apache/2.2.14 (Ubuntu)
+    X-Powered-By:PHP/5.3.2-1ubuntu4.24
+
+when I right-click on the second one in the inspect element window, I can select "copy as cURL" i get:
+
+    curl 'http://moodle.jct.ac.il/pluginfile.php/288409/mod_resource/content/0/movie%205773/150151.5773.week1.wmv?forcedownload=1' -H 'DNT: 1' -H 'Accept-Encoding: gzip,deflate,sdch' -H 'Accept-Language: en-US,en;q=0.8,he;q=0.6' -H 'User-Agent: Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/36.0.1985.67 Safari/537.36' -H 'Accept: text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8' -H 'Referer: http://moodle.jct.ac.il/course/view.php?id=20151' -H 'Cookie: visid_incap_97364=qJx2WaKqQfGidGf9VfM6QWrnlFIAAAAAQUIPAAAAAAC5EXcbt00vqNu9jdVDwEDN; __utma=98014340.1409421308.1381214363.1381214363.1390387318.2; __utmz=98014340.1381214363.1.1.utmcsr=(direct)|utmccn=(direct)|utmcmd=(none); MoodleSession5771=7s1cfqfo4ahdtmna5h7vserg97; MOODLEID1_5771=%257F%25D39%2522N%25B4%25AFY' -H 'Connection: keep-alive' --compressed
+
+if I append " > week1.wmv" to the end of that output it downloads fine.  How can I add this file to a git-annex repo?
diff --git a/doc/forum/downloading_from_moodle/comment_1_3f677130d268de4288e87cfa86ea055c._comment b/doc/forum/downloading_from_moodle/comment_1_3f677130d268de4288e87cfa86ea055c._comment
new file mode 100644
index 0000000000..a489859adf
--- /dev/null
+++ b/doc/forum/downloading_from_moodle/comment_1_3f677130d268de4288e87cfa86ea055c._comment
@@ -0,0 +1,10 @@
+[[!comment format=mdwn
+ username="http://joeyh.name/"
+ ip="108.236.230.124"
+ subject="comment 1"
+ date="2014-06-16T17:55:13Z"
+ content="""
+`git annex addurl` does not support storing cookies needed to access files. I doubt that makes sense anyway, since most login cookies expire.
+
+So, your best bet seems to just be to `git annex add` the file after you download it.
+"""]]
diff --git a/doc/forum/downloading_from_moodle/comment_2_dcef60ec144f123dadd165fb602ab950._comment b/doc/forum/downloading_from_moodle/comment_2_dcef60ec144f123dadd165fb602ab950._comment
new file mode 100644
index 0000000000..1529004333
--- /dev/null
+++ b/doc/forum/downloading_from_moodle/comment_2_dcef60ec144f123dadd165fb602ab950._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="http://joeyh.name/"
+ ip="108.236.230.124"
+ subject="comment 2"
+ date="2014-06-16T17:59:20Z"
+ content="""
+Another option would be to set up some kind of http proxy that adds the login cookie to requests.. If you really wanted to use `git annex addurl`.
+"""]]
diff --git a/doc/forum/downloading_from_moodle/comment_3_d3efb767bf9b20f96242dcf64817bd4b._comment b/doc/forum/downloading_from_moodle/comment_3_d3efb767bf9b20f96242dcf64817bd4b._comment
new file mode 100644
index 0000000000..c9b1012254
--- /dev/null
+++ b/doc/forum/downloading_from_moodle/comment_3_d3efb767bf9b20f96242dcf64817bd4b._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="http://joeyh.name/"
+ ip="108.236.230.124"
+ subject="comment 3"
+ date="2014-06-16T18:04:48Z"
+ content="""
+Actually, a simpler option may be to set `annex.http-headers` in the git config, to the appropriate http headers that set the cookie.
+"""]]
diff --git a/doc/forum/downloading_mp4_by_default_with_importfeed.mdwn b/doc/forum/downloading_mp4_by_default_with_importfeed.mdwn
new file mode 100644
index 0000000000..88df821c18
--- /dev/null
+++ b/doc/forum/downloading_mp4_by_default_with_importfeed.mdwn
@@ -0,0 +1 @@
+Is it possible to download youtube videos in mp4 format by default with importfeed. Without any options git annex often downloads the webm format.  
diff --git a/doc/forum/downloading_mp4_by_default_with_importfeed/comment_1_c9e5a3700764faa33d2c68de6a8236dc._comment b/doc/forum/downloading_mp4_by_default_with_importfeed/comment_1_c9e5a3700764faa33d2c68de6a8236dc._comment
new file mode 100644
index 0000000000..07691d8233
--- /dev/null
+++ b/doc/forum/downloading_mp4_by_default_with_importfeed/comment_1_c9e5a3700764faa33d2c68de6a8236dc._comment
@@ -0,0 +1,15 @@
+[[!comment format=mdwn
+ username="joey"
+ subject="""comment 1"""
+ date="2015-05-30T15:27:57Z"
+ content="""
+git-annex uses quvi for these downloads. If quvi has a command-line 
+option to make it prefer mp4, you can pass it using the annex.quvi-options
+git configuration setting. Or maybe it has a config file. I don't
+know how to configure quvi to do that, but if it's possible to, you should
+be able to get git-annex to run it with that configuration.
+
+Update: Now git-annex uses youtube-dl, so you can use the
+annex.youtube-dl-options git configuration setting to configure youtube-dl
+to download your preferred format.
+"""]]
diff --git a/doc/forum/downloading_mp4_by_default_with_importfeed/comment_2_ade70b45f49eb3eaf127ba7b2bd37900._comment b/doc/forum/downloading_mp4_by_default_with_importfeed/comment_2_ade70b45f49eb3eaf127ba7b2bd37900._comment
new file mode 100644
index 0000000000..10ad1c4b90
--- /dev/null
+++ b/doc/forum/downloading_mp4_by_default_with_importfeed/comment_2_ade70b45f49eb3eaf127ba7b2bd37900._comment
@@ -0,0 +1,7 @@
+[[!comment format=mdwn
+ username="juh"
+ subject="comment 2"
+ date="2015-06-01T11:29:12Z"
+ content="""
+Ok, I look into that.
+"""]]
diff --git a/doc/forum/drop__47__whereis_not_showing_gcrypted_special_ssh_remote.mdwn b/doc/forum/drop__47__whereis_not_showing_gcrypted_special_ssh_remote.mdwn
new file mode 100644
index 0000000000..9445539b41
--- /dev/null
+++ b/doc/forum/drop__47__whereis_not_showing_gcrypted_special_ssh_remote.mdwn
@@ -0,0 +1,11 @@
+I have my laptop, my server and my usb drive. My server is a gcrypted remote via ssh. My laptop is just a repo that's referenced as a remote with a filepath.
+
+I do git annex copy --to server, and let it copy stuff. I repeat the same thing for the usb drive. 
+
+I run git annex sync and it does the whole sync dance successfully. it pushes stuff to both the usb drive and the server.
+
+Afterwards, I do git annex whereis, and I only get 2 copies showing - my laptop and my usb drive. Likewise, since I set numcopies to 2, it won't let me drop anything at all, because it doesn't know there's a copy on my server.
+
+Anything I can do about this? (What further info do you need?) 
+
+I should probably add that my server is a ubuntu machine, and so it runs version 5.20140412ubuntu1. My laptop runs a more recent 5.20140717-g5a7d4ff.
diff --git a/doc/forum/drop__47__whereis_not_showing_gcrypted_special_ssh_remote/comment_1_968bc2be595008790e9b93d82342714c._comment b/doc/forum/drop__47__whereis_not_showing_gcrypted_special_ssh_remote/comment_1_968bc2be595008790e9b93d82342714c._comment
new file mode 100644
index 0000000000..45699f4437
--- /dev/null
+++ b/doc/forum/drop__47__whereis_not_showing_gcrypted_special_ssh_remote/comment_1_968bc2be595008790e9b93d82342714c._comment
@@ -0,0 +1,10 @@
+[[!comment format=mdwn
+ username="annexuser123"
+ ip="88.65.52.206"
+ subject="comment 1"
+ date="2014-08-04T18:00:00Z"
+ content="""
+To add further to this, the gcrypted remote does not show up in either the usb drive's git annex info, nor the laptop's git annex info. 
+
+Despite that, I can push to it and copy to it. 
+"""]]
diff --git a/doc/forum/drop__47__whereis_not_showing_gcrypted_special_ssh_remote/comment_2_2c14d88e55ea7d4edc90ce0091025f32._comment b/doc/forum/drop__47__whereis_not_showing_gcrypted_special_ssh_remote/comment_2_2c14d88e55ea7d4edc90ce0091025f32._comment
new file mode 100644
index 0000000000..a181940d2f
--- /dev/null
+++ b/doc/forum/drop__47__whereis_not_showing_gcrypted_special_ssh_remote/comment_2_2c14d88e55ea7d4edc90ce0091025f32._comment
@@ -0,0 +1,10 @@
+[[!comment format=mdwn
+ username="http://joeyh.name/"
+ ip="209.250.56.54"
+ subject="comment 2"
+ date="2014-10-02T15:48:02Z"
+ content="""
+Sorry that it took me so long to get back to you.
+
+The git remote that corresponds to your server should have an annex-uuid setting. (Eg, remote.server.annex-uuid.) Look up the uuid. Then look in `git-annex info`. I'd expect the uuid to be listed there. It might be that somehow two repositories got the same uuid set (probability says no, but things find ways to break). If that did happen, I think it would replicate what you've reported, and in that case, the server's uuid would be listed by `git annex info`, but perhaps it would have the name of the usb drive.
+"""]]
diff --git a/doc/forum/drop_old_versions_of_a_file.mdwn b/doc/forum/drop_old_versions_of_a_file.mdwn
new file mode 100644
index 0000000000..95f103dcc9
--- /dev/null
+++ b/doc/forum/drop_old_versions_of_a_file.mdwn
@@ -0,0 +1,3 @@
+I have a music repository which has multiple versions of a music file(modified id3 tags etc,.) and in my music player same file is showing two times with two different id3 tags, one is from music directory and another is from .git-annex directory which is a older version(which I don't want to see).
+
+I was just wondering if there is a way I can drop old version of a file in android(direct mode).
diff --git a/doc/forum/drop_old_versions_of_a_file/comment_1_799a413248fb8f98efbf226b1bc4300d._comment b/doc/forum/drop_old_versions_of_a_file/comment_1_799a413248fb8f98efbf226b1bc4300d._comment
new file mode 100644
index 0000000000..09b01c8a8c
--- /dev/null
+++ b/doc/forum/drop_old_versions_of_a_file/comment_1_799a413248fb8f98efbf226b1bc4300d._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="http://joeyh.name/"
+ ip="108.236.230.124"
+ subject="comment 1"
+ date="2014-03-10T17:59:01Z"
+ content="""
+What you're looking for is `git annex unused` and `git annex dropunused`
+"""]]
diff --git a/doc/forum/duplicated_content___40__user_error__41__._How_to_fix__63__.mdwn b/doc/forum/duplicated_content___40__user_error__41__._How_to_fix__63__.mdwn
new file mode 100644
index 0000000000..2fd18f9888
--- /dev/null
+++ b/doc/forum/duplicated_content___40__user_error__41__._How_to_fix__63__.mdwn
@@ -0,0 +1,24 @@
+I have Ubuntu and Debian systems, playing with git-annex. This is not mission critical data, just testing. I ended up with an extra duplicate of the media files on the laptop.
+
+The whereis listing is lots and lots like this:
+
+whereis 00-Unsorted/2008-RobustStats-AmerPsyc.pdf (3 copies)
+        67e69242-d57c-4b50-aaf9-74876b899962
+        9e0bc9e4-f8bf-11e3-b9c1-9b4158540a9d -- pols110.pols.ku.edu_mediashare (
+pdf and mp3)
+        d82d2e6f-9200-49cf-86a3-1d674a768971 -- here (pauljohn@dellap14:~/medias
+hare)
+ok
+
+(I'm pretty sure) This happened because I copied the media files to /home/pauljohn/mediashare/manuscripts manually, and then I used the git-annex assistant to set up the remote linkage to same content, from  the workstation, and it apparently copied in a whole new set. 
+
+How to clean this up? Without erasing everything and starting over?
+
+Can I guess? 
+
+Open a terminal and git remove manually 67e69242-d57c-4b50-aaf9-74876b899962 ???
+
+Thanks in advance if you care to advise me :)
+
+Paul Johnson
+http://pj.freefaculty.org
diff --git a/doc/forum/duplicated_content___40__user_error__41__._How_to_fix__63__/comment_1_bb6d749b758b17178227929bf7327fe1._comment b/doc/forum/duplicated_content___40__user_error__41__._How_to_fix__63__/comment_1_bb6d749b758b17178227929bf7327fe1._comment
new file mode 100644
index 0000000000..d528331cef
--- /dev/null
+++ b/doc/forum/duplicated_content___40__user_error__41__._How_to_fix__63__/comment_1_bb6d749b758b17178227929bf7327fe1._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="https://openid.stackexchange.com/user/e65e6d0e-58ba-41de-84cc-1f2ba54cf574"
+ nickname="Mica Semrick"
+ subject="comment 1"
+ date="2014-06-24T04:24:12Z"
+ content="""
+You should first try `git-annex unused` and if that doesn't work just `git-annex drop` the unwanted files.
+"""]]
diff --git a/doc/forum/duplicated_content___40__user_error__41__._How_to_fix__63__/comment_2_d834df30633f7d5569797ee818cf38c3._comment b/doc/forum/duplicated_content___40__user_error__41__._How_to_fix__63__/comment_2_d834df30633f7d5569797ee818cf38c3._comment
new file mode 100644
index 0000000000..1f2cf2ff55
--- /dev/null
+++ b/doc/forum/duplicated_content___40__user_error__41__._How_to_fix__63__/comment_2_d834df30633f7d5569797ee818cf38c3._comment
@@ -0,0 +1,20 @@
+[[!comment format=mdwn
+ username="https://www.google.com/accounts/o8/id?id=AItOawkpIIYg6Fl7OFsOHVPEchZYj68A3dk4lVg"
+ nickname="Paul"
+ subject="please explain"
+ date="2014-07-01T13:45:08Z"
+ content="""
+Sorry. I just don't understand. 
+what am I doing wrong?  How to remove extra copy:
+
+
    +whereis WaldTest/Vaeth_WaldTest.pdf (3 copies)
+        67e69242-d57c-4b50-aaf9-74876b899962
+        9e0bc9e4-f8bf-11e3-b9c1-9b4158540a9d -- pols110.pols.ku.edu_mediashare (pdf and mp3)
+        d82d2e6f-9200-49cf-86a3-1d674a768971 -- here (pauljohn@dellap14:~/mediashare)
+ok
+pauljohn@dellap14:~/mediashare/manuscripts$ git annex drop 67e69242-d57c-4b50-aaf9-74876b899962
+git-annex: 67e69242-d57c-4b50-aaf9-74876b899962 not found
+pauljohn@dellap14:~/mediashare/manuscripts$ git annex drop WaldTest/67e69242-d57c-4b50-aaf9-74876b899962
+git-annex: WaldTest/67e69242-d57c-4b50-aaf9-74876b899962 not found
+"""]]
diff --git a/doc/forum/duplicated_content___40__user_error__41__._How_to_fix__63__/comment_3_1e02eff33c9fa7bea03aa6d58b910175._comment b/doc/forum/duplicated_content___40__user_error__41__._How_to_fix__63__/comment_3_1e02eff33c9fa7bea03aa6d58b910175._comment
new file mode 100644
index 0000000000..b0061a9d0d
--- /dev/null
+++ b/doc/forum/duplicated_content___40__user_error__41__._How_to_fix__63__/comment_3_1e02eff33c9fa7bea03aa6d58b910175._comment
@@ -0,0 +1,14 @@
+[[!comment format=mdwn
+ username="http://joeyh.name/"
+ ip="209.250.56.55"
+ subject="comment 3"
+ date="2014-07-03T19:20:49Z"
+ content="""
+\"67e69242-d57c-4b50-aaf9-74876b899962\" is the UUID of a git annnex repository that git-annex has on record as containing the file. Since it does not have a description set (which normally gets done automatically when setting up the repository), and is not one of the remotes of the repository where you ran `git annex whereis`, it's a bit hard to tell what repository that is.
+
+What I would do in this situation is:
+
+1. Look around my computers for a repository with that UUID. You can run this command in a repository to see its uuid: `git config annex.uuid`  
+2. If I found it, I'd run `git annex describe here \"something sensible\"` and maybe set it up as a remote of other repositories and then `git annex drop` the data from it if desired.
+3. If I was unable to find the repository, I might assume it was one I created before, and have removed. Then I'd tell git-annex that: `git annex dead 67e69242-d57c-4b50-aaf9-74876b899962` (if it turns out I was wrong and the repository turns up later, this can always be reversed by running `git annex semitrust 67e69242-d57c-4b50-aaf9-74876b899962`)
+"""]]
diff --git a/doc/forum/empty_directory_handling.mdwn b/doc/forum/empty_directory_handling.mdwn
new file mode 100644
index 0000000000..b806b2e2fe
--- /dev/null
+++ b/doc/forum/empty_directory_handling.mdwn
@@ -0,0 +1,3 @@
+I've just tried using git annex assistant for the first time, to synchronize a shared directory between machines. This shared directory contains a collection of bare git repositories for various projects. Unfortunately this doesn't seem to work, as to be recognised as valid bare repos, they need to have the ref/heads and ref/tags subdirectories. git annex assistant fails to synchronise these, as they are empty.
+
+What should I do? How can I make git annex reproduce the source directory structure when synchronising?
diff --git a/doc/forum/empty_directory_handling/comment_1_34ac97b9337b6230ed8a4748203fe543._comment b/doc/forum/empty_directory_handling/comment_1_34ac97b9337b6230ed8a4748203fe543._comment
new file mode 100644
index 0000000000..edf4984ea4
--- /dev/null
+++ b/doc/forum/empty_directory_handling/comment_1_34ac97b9337b6230ed8a4748203fe543._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="http://joeyh.name/"
+ ip="108.236.230.124"
+ subject="comment 1"
+ date="2014-07-02T17:02:31Z"
+ content="""
+This is not a good idea. See 
+"""]]
diff --git a/doc/forum/empty_directory_handling/comment_2_73a39e28d5a09ac342cb4195d263d91e._comment b/doc/forum/empty_directory_handling/comment_2_73a39e28d5a09ac342cb4195d263d91e._comment
new file mode 100644
index 0000000000..250c08adfd
--- /dev/null
+++ b/doc/forum/empty_directory_handling/comment_2_73a39e28d5a09ac342cb4195d263d91e._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="https://www.google.com/accounts/o8/id?id=AItOawmlO6w9_FAVe3v6MdnkePxaTJqp0RPbGhM"
+ nickname="Tim"
+ subject="comment 2"
+ date="2014-07-04T04:00:25Z"
+ content="""
+Thanks - that thread was helpful. I now realise that what I was trying to do was not sensible, and I'll have to work out another approach.
+"""]]
diff --git a/doc/forum/endless_password_prompt_loop.mdwn b/doc/forum/endless_password_prompt_loop.mdwn
new file mode 100644
index 0000000000..bfdd3c1dbc
--- /dev/null
+++ b/doc/forum/endless_password_prompt_loop.mdwn
@@ -0,0 +1,8 @@
+Hi,
+
+I setup a first repository on local disk, it went fine. Next I tried setting up repo on my nas to clone the files. I installed git-annex on the NAS (it's armel arch) and created the repository via webapp, I entered the password once. Git-annex started syncing the remote repo and after few seconds password pop-up hell broke loose. I was flooded with password-prompt pop-up windows. The synchronization was continuing regardless of correctly/incorrectly entered passwords. What's more, these popups grab the keyboard and you cannot do (appropriate word) with your system.
+
+I checked this topic: http://git-annex.branchable.com/forum/ssh_password/ and ssh config is in place on my computer and on the NAS.
+
+What is causing this and how can I get rid of this?
+
diff --git a/doc/forum/endless_password_prompt_loop/comment_1_cceba12ed25cd671c7cee5a28631163e._comment b/doc/forum/endless_password_prompt_loop/comment_1_cceba12ed25cd671c7cee5a28631163e._comment
new file mode 100644
index 0000000000..d162ff3e52
--- /dev/null
+++ b/doc/forum/endless_password_prompt_loop/comment_1_cceba12ed25cd671c7cee5a28631163e._comment
@@ -0,0 +1,10 @@
+[[!comment format=mdwn
+ username="http://joeyh.name/"
+ ip="4.152.108.193"
+ subject="comment 1"
+ date="2013-06-25T19:49:25Z"
+ content="""
+What version of git-annex? Did more than one password prompt window somehow appear at the same time? What is doing the ssh password prompting, is it ssh-askpass, or something provided by your desktop environment?
+
+What happens if you manually ssh from your computer to the NAS, using the special hostname that git-annex has configured (you can find this hostname in the `.git/config`). `ssh -v` is generally a good way to track these problems down.
+"""]]
diff --git a/doc/forum/endless_password_prompt_loop/comment_2_f0cb86b45eb289f35197c43f83660a8f._comment b/doc/forum/endless_password_prompt_loop/comment_2_f0cb86b45eb289f35197c43f83660a8f._comment
new file mode 100644
index 0000000000..45f87d136e
--- /dev/null
+++ b/doc/forum/endless_password_prompt_loop/comment_2_f0cb86b45eb289f35197c43f83660a8f._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="https://www.google.com/accounts/o8/id?id=AItOawmUcCZeEK9HKV40JrcHMXAXpPKQPERt7iM"
+ nickname="Włodek"
+ subject="comment 2"
+ date="2013-07-10T22:00:47Z"
+ content="""
+Thanks for the answer. I found out that the public key auth is not working at all and ssh falls back to password auth, this is the reason for password prompts. What I don't know yet is why popups appear despite giving correct password. I'm looking for a solution. This can take some time though because my disk died few days ago = I cannot check versions. Once I've got it solved I'll post back.
+"""]]
diff --git a/doc/forum/endless_password_prompt_loop/comment_3_eb1dce6d9af6e19cf77df63da1a68425._comment b/doc/forum/endless_password_prompt_loop/comment_3_eb1dce6d9af6e19cf77df63da1a68425._comment
new file mode 100644
index 0000000000..58914c58e3
--- /dev/null
+++ b/doc/forum/endless_password_prompt_loop/comment_3_eb1dce6d9af6e19cf77df63da1a68425._comment
@@ -0,0 +1,16 @@
+[[!comment format=mdwn
+ username="damien.courousse"
+ subject="same behaviour when the remote repo is not reachable"
+ date="2015-05-01T15:47:55Z"
+ content="""
+hello,
+
+I experience the same behaviour : once git-annex is started using the assistant, the ssh askpass window keeps asking me for a password. 
+The repo is not reachable at this time (the server is down).
+
+What's really annoying is that the askpass window is sometimes hidden behind other windows, but it keeps grabbing keyboard events: it looks like the keyboard \"does not work\" anymore. Hopefully pressing escape shuts closes askpass.
+
+The askpass window still pop ups from time to time even once the assistant has been closed.
+
+Running git-annex version 5.20141125  on Debian (amd64) on jessie.
+"""]]
diff --git a/doc/forum/error__58___refs__47__heads__47__git-annex_does_not_point_to_a_valid_object__33__.mdwn b/doc/forum/error__58___refs__47__heads__47__git-annex_does_not_point_to_a_valid_object__33__.mdwn
new file mode 100644
index 0000000000..a39951bb50
--- /dev/null
+++ b/doc/forum/error__58___refs__47__heads__47__git-annex_does_not_point_to_a_valid_object__33__.mdwn
@@ -0,0 +1,17 @@
+for the command:
+
+    $ git annex info
+
+I get the following output:
+
+   repository mode: direct
+   trusted repositories: error: refs/heads/git-annex does not point to a valid object!
+   error: refs/heads/git-annex does not point to a valid object!
+   error: invalid object 100644 993859515190743e9bed7fc8e697d5ac4e2d03a0 for '01d/56e/SHA256E-s209799168--cb8450b63c699c5f8fb121e51ed7e387258d57c851b256af0e5ceeae9542ffd8.me.avi.log'
+   fatal: git-write-tree: error building trees
+   git-annex: failed to read sha from git write-tree
+
+Please help,
+Yuval Langer.
+
+BTW, the code formatting is kinda weird. Could you help with that as well?
diff --git a/doc/forum/error__58___refs__47__heads__47__git-annex_does_not_point_to_a_valid_object__33__/comment_1_d370b044da3bfebf9e4c90ce1e243587._comment b/doc/forum/error__58___refs__47__heads__47__git-annex_does_not_point_to_a_valid_object__33__/comment_1_d370b044da3bfebf9e4c90ce1e243587._comment
new file mode 100644
index 0000000000..cf7601d408
--- /dev/null
+++ b/doc/forum/error__58___refs__47__heads__47__git-annex_does_not_point_to_a_valid_object__33__/comment_1_d370b044da3bfebf9e4c90ce1e243587._comment
@@ -0,0 +1,18 @@
+[[!comment format=mdwn
+ username="http://joeyh.name/"
+ ip="108.236.230.124"
+ subject="comment 1"
+ date="2014-05-15T20:22:40Z"
+ content="""
+This means that a git commit's data has gotten lost somehow. You can verify that by trying
+
+    git show 993859515190743e9bed7fc8e697d5ac4e2d03a0
+
+The most common reason for this to happen is if the system is shutdown unexpectedly while git-annex is running. Or, if this repository is on a removable drive, if it got removed before the data could be written.
+
+You can probably fix the problem by running:
+
+    git annex repair
+
+(Code formatting: Use 4 spaces.)
+"""]]
diff --git a/doc/forum/error_in_installation_of_base-4.5.0.0.mdwn b/doc/forum/error_in_installation_of_base-4.5.0.0.mdwn
new file mode 100644
index 0000000000..673222ed6d
--- /dev/null
+++ b/doc/forum/error_in_installation_of_base-4.5.0.0.mdwn
@@ -0,0 +1,14 @@
+Hi,
+
+I was trying to install git-annex, but then, I got a warning saying that I need to install base-4.5.0.0 first.
+
+So, I did "sudo cabal install base-4.5.0.0". Everything was going well, until I got this error:
+
+config.status: error: cannot find input file: `base.buildinfo.in'
+cabal: Error: some packages failed to install:
+base-4.5.0.0 failed during the configure step. The exception was:
+ExitFailure 1
+
+I tried to look for information on the internet, but I did not find anything useful.
+
+I know that this is not totally related to git-annex, but anyone has any thoughts on this?
diff --git a/doc/forum/error_in_installation_of_base-4.5.0.0/comment_1_0b2f79c014e0dd9badd52b8b6aa47e0c._comment b/doc/forum/error_in_installation_of_base-4.5.0.0/comment_1_0b2f79c014e0dd9badd52b8b6aa47e0c._comment
new file mode 100644
index 0000000000..5a52a28ab1
--- /dev/null
+++ b/doc/forum/error_in_installation_of_base-4.5.0.0/comment_1_0b2f79c014e0dd9badd52b8b6aa47e0c._comment
@@ -0,0 +1,19 @@
+[[!comment format=mdwn
+ username="http://joey.kitenet.net/"
+ nickname="joey"
+ subject="comment 1"
+ date="2012-04-22T05:39:28Z"
+ content="""
+git-annex needs ghc 7.4, that's why it depends on that base version that comes with it. So you either need to upgrade your ghc, or you can build from the `ghc7.0` branch in [[git|download]], like this:
+
+
    +git clone git://git-annex.branchable.com/ git-annex
+cd git-annex
+git checkout ghc7.0
+cabal update
+cabal install --only-dependencies
+cabal configure
+cabal build
+cabal install --bindir=$HOME/bin
+
    
+"""]]
diff --git a/doc/forum/error_in_installation_of_base-4.5.0.0/comment_2_3badd64e48fbb174cd7de1ac9589bedf._comment b/doc/forum/error_in_installation_of_base-4.5.0.0/comment_2_3badd64e48fbb174cd7de1ac9589bedf._comment
new file mode 100644
index 0000000000..3ad1a11388
--- /dev/null
+++ b/doc/forum/error_in_installation_of_base-4.5.0.0/comment_2_3badd64e48fbb174cd7de1ac9589bedf._comment
@@ -0,0 +1,31 @@
+[[!comment format=mdwn
+ username="https://www.google.com/accounts/o8/id?id=AItOawkaT0B6s9jQuMzQUYRVBgWqtO7BhT_ZSaE"
+ nickname="Fernando Seabra"
+ subject="comment 2"
+ date="2012-04-22T14:09:33Z"
+ content="""
+Thanks for the fast response!
+
+Unfortunately, I had another problem:
+
+==================================
+Building git-annex-3.20120419...
+Utility/libdiskfree.c: In function ‘diskfree’:
+
+Utility/libdiskfree.c:61:0:
+     warning: ‘statfs64’ is deprecated (declared at /usr/include/sys/mount.h:379)
+[  6 of 157] Compiling Build.SysConfig  ( Build/SysConfig.hs, dist/build/git-annex/git-annex-tmp/Build/SysConfig.o )
+[ 15 of 157] Compiling Utility.Touch    ( dist/build/git-annex/git-annex-tmp/Utility/Touch.hs, dist/build/git-annex/git-annex-tmp/Utility/Touch.o )
+
+Utility/Touch.hsc:118:21: Not in scope: `noop'
+cabal: Error: some packages failed to install:
+git-annex-3.20120419 failed during the building phase. The exception was:
+ExitFailure 1
+==================================
+
+I also tried to look for information on the internet, and I did not find anything useful.
+Any idea of what happened?
+
+Thanks again!
+
+"""]]
diff --git a/doc/forum/error_in_installation_of_base-4.5.0.0/comment_3_d8190061ac1c683a7b699cf42e9db694._comment b/doc/forum/error_in_installation_of_base-4.5.0.0/comment_3_d8190061ac1c683a7b699cf42e9db694._comment
new file mode 100644
index 0000000000..d9de5089e9
--- /dev/null
+++ b/doc/forum/error_in_installation_of_base-4.5.0.0/comment_3_d8190061ac1c683a7b699cf42e9db694._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="http://joey.kitenet.net/"
+ nickname="joey"
+ subject="comment 3"
+ date="2012-04-22T15:23:26Z"
+ content="""
+That's my fault, I made a change last night that caused the noop problem. Fixed now.
+"""]]
diff --git a/doc/forum/error_in_installation_of_base-4.5.0.0/comment_4_49a4fcd2dc4f97d4055b5051feea5e3b._comment b/doc/forum/error_in_installation_of_base-4.5.0.0/comment_4_49a4fcd2dc4f97d4055b5051feea5e3b._comment
new file mode 100644
index 0000000000..ba99334511
--- /dev/null
+++ b/doc/forum/error_in_installation_of_base-4.5.0.0/comment_4_49a4fcd2dc4f97d4055b5051feea5e3b._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="https://www.google.com/accounts/o8/id?id=AItOawkaT0B6s9jQuMzQUYRVBgWqtO7BhT_ZSaE"
+ nickname="Fernando Seabra"
+ subject="comment 4"
+ date="2012-04-22T16:08:55Z"
+ content="""
+Thanks, it worked now!
+"""]]
diff --git a/doc/forum/error_setting_up_centralized_server_on_OSX_-_failed_to_get_annex.uuid_configuration_of_repository_origin.mdwn b/doc/forum/error_setting_up_centralized_server_on_OSX_-_failed_to_get_annex.uuid_configuration_of_repository_origin.mdwn
new file mode 100644
index 0000000000..647557e7da
--- /dev/null
+++ b/doc/forum/error_setting_up_centralized_server_on_OSX_-_failed_to_get_annex.uuid_configuration_of_repository_origin.mdwn
@@ -0,0 +1,19 @@
+I'm attempting to set up a centralized git repository to use with git-annex as specified on https://git-annex.branchable.com/tips/centralized_git_repository_tutorial/on_your_own_server/
+
+I'm using macbooks as clients and servers. They are all running git-annex version 5.20150916 installed through homebrew. The server is running 10.10 Yosemite and the clients are running 10.9 Mavericks. There is a symlink on the server in ```/usr/bin/git-annex``` and ```/usr/bin/git-annex-shell``` to ```/usr/local/bin/git-annex``` and ```/usr/local/bin/git-annex-shell```, which then symlinks to the latest version in homebrew's directories. I did this because I was previously facing an error where git-annex was not found on the server (it would not work if you ran ssh tom@server git-annex).
+
+I'm currently at the step after cloning and cding into the repository on the client. When I attempt to run git annex init 'Tom Macbook', I get the following error:
+
+```
+git annex init 'Tom Macbook'
+init Tom Macbook
+Failed to get annex.uuid configuration of repository origin
+
+Instead, got: "annex.uuid=\ncore.gcrypt-id=\n"
+
+This is unexpected; please check the network transport!
+ok
+(recording state in git...)
+```
+
+I'm really stumped here. Any suggestions? Thanks. Let me know if I can provide any further info. 
diff --git a/doc/forum/error_setting_up_centralized_server_on_OSX_-_failed_to_get_annex.uuid_configuration_of_repository_origin/comment_1_943ff286824bd59a164b161553d65698._comment b/doc/forum/error_setting_up_centralized_server_on_OSX_-_failed_to_get_annex.uuid_configuration_of_repository_origin/comment_1_943ff286824bd59a164b161553d65698._comment
new file mode 100644
index 0000000000..eed9dfda96
--- /dev/null
+++ b/doc/forum/error_setting_up_centralized_server_on_OSX_-_failed_to_get_annex.uuid_configuration_of_repository_origin/comment_1_943ff286824bd59a164b161553d65698._comment
@@ -0,0 +1,15 @@
+[[!comment format=mdwn
+ username="joey"
+ subject="""comment 1"""
+ date="2015-10-15T18:55:42Z"
+ content="""
+Yeah, this is a bug, and one I was able to reproduce. It was introduced in
+the last release of git-annex.
+
+Luckily, it's only a cosmetic bug. The `git annex init` does
+succeed, and your repo is left in an ok state. If you continue
+along the walkthrough path, adding some files and syncing with origin, it
+will proceed to set up the origin appropriately and everything should work.
+
+I'm gonna fix the bug of course. Indeed, it's fixed in git master now.
+"""]]
diff --git a/doc/forum/example_of_massively_disconnected_operation.mdwn b/doc/forum/example_of_massively_disconnected_operation.mdwn
new file mode 100644
index 0000000000..78ed6784b2
--- /dev/null
+++ b/doc/forum/example_of_massively_disconnected_operation.mdwn
@@ -0,0 +1,33 @@
+I found this archival drive that had been offline since October 26th 2010. Since I released git-annex 0.02 on October 27th, this must have been made using the very first release of git-annex, ever.
+
+So, I synced it back up! :) --[[Joey]]
+
+
    +commit 4151f4595fe6205d4aed653617ab23eb3335130a
+Author: Joey Hess 
+Date:   Tue Oct 26 02:18:03 2010 -0400
+
+joey> git pull
+remote: Counting objects: 428782, done.
+remote: Compressing objects: 100% (280714/280714), done.
+remote: Total 416692 (delta 150923), reused 389593 (delta 125143)
+Receiving objects: 100% (416692/416692), 44.71 MiB | 495 KiB/s, done.
+Resolving deltas: 100% (150923/150923), completed with 818 local objects.
+ * [new branch]      git-annex  -> origin/git-annex
+   1893f9c..9ebcc0e  master     -> origin/master
+Updating 1893f9c..9ebcc0e
+Checking out files: 100% (76884/76884), done.
+joey> git annex version
+git-annex version: 3.20110611
+local repository version: unknown
+default repository version: 3
+supported repository versions: 3
+upgrade supported from repository versions: 0 1 2
+joey> git config annex.version 0
+joey> git annex upgrade
+upgrade . (v0 to v1...) (v1 to v2) (moving content...) (updating symlinks...)  (moving location logs...) (v2 to v3) (merging origin/git-annex into git-annex...)
+
+  git-annex branch created
+  Be sure to push this branch when pushing to remotes.
+ok
+
    
diff --git a/doc/forum/exclude_files_from_annex.mdwn b/doc/forum/exclude_files_from_annex.mdwn
new file mode 100644
index 0000000000..c1707afb13
--- /dev/null
+++ b/doc/forum/exclude_files_from_annex.mdwn
@@ -0,0 +1,10 @@
+I'm wanting to do essentially [[/tips/replacing_Sparkleshare_or_dvcs-autosync_with_the_assistant/]], that is prevent git-annex from annexing certain files (by extension), and rather manage them in git only.
+
+This seems to work fine, and the assistant auto-adds them to git, and commits when I make changes, when in indirect mode.
+
+However, in a direct mode repository, then the files are automatically annexed, and even if I create them in an indirect repo, and then change it, they are annexed.
+
+Is this expected behaviour? Does the annex.largefiles setting only make sense in indirect mode?
+
+
+--Walter
diff --git a/doc/forum/exclude_files_from_annex/comment_1_82e7de5e631bae3b347815586274a936._comment b/doc/forum/exclude_files_from_annex/comment_1_82e7de5e631bae3b347815586274a936._comment
new file mode 100644
index 0000000000..23ad6aac92
--- /dev/null
+++ b/doc/forum/exclude_files_from_annex/comment_1_82e7de5e631bae3b347815586274a936._comment
@@ -0,0 +1,25 @@
+[[!comment format=mdwn
+ username="http://joeyh.name/"
+ nickname="joey"
+ subject="comment 1"
+ date="2013-05-19T20:29:14Z"
+ content="""
+I don't seem to be seeing the problem you describe in direct mode:
+
+[[!format sh \"\"\"
+
+joey@gnu:~/tmp/r2>git init
+joey@gnu:~/tmp/r2>git annex init
+joey@gnu:~/tmp/r2>git config annex.largefiles \"largerthan=100kb and not (include=*.c or include=*.h)\"
+joey@gnu:~/tmp/r2>git annex direct
+joey@gnu:~/tmp/r2>git annex assistant 
+joey@gnu:~/tmp/r2>dd if=/dev/urandom of=bigfile bs=1M count=1
+1+0 records in
+1+0 records out
+joey@gnu:~/tmp/r2>echo \"main() { printf(\"hello, world!\n\") }\" > hello.c
+\"\"\"]]
+
+At that point, the bigfile was annexed, and the hello.c file was checked into git as a regular file.
+
+Are you sure you have annex.largefiles correctly configured to exclude the files you don't want to be annexed?
+"""]]
diff --git a/doc/forum/exclude_files_from_annex/comment_2_03d4599fdceb3dff184eed82824674bc._comment b/doc/forum/exclude_files_from_annex/comment_2_03d4599fdceb3dff184eed82824674bc._comment
new file mode 100644
index 0000000000..7d8f6ce772
--- /dev/null
+++ b/doc/forum/exclude_files_from_annex/comment_2_03d4599fdceb3dff184eed82824674bc._comment
@@ -0,0 +1,12 @@
+[[!comment format=mdwn
+ username="https://www.google.com/accounts/o8/id?id=AItOawnSenxKyE_2Z6Wb-EBMO8FciyRywjx1ZiQ"
+ nickname="Walter"
+ subject="comment 2"
+ date="2013-05-19T21:35:37Z"
+ content="""
+This seems to have stopped happening, after I installed a newer version (from git) of git-annex.
+
+I might try and recreate it later, but for now it seems to work as expected.
+
+Thanks for your help.
+"""]]
diff --git a/doc/forum/expire_files__44___move_to_other_hosts.mdwn b/doc/forum/expire_files__44___move_to_other_hosts.mdwn
new file mode 100644
index 0000000000..9ead2f899c
--- /dev/null
+++ b/doc/forum/expire_files__44___move_to_other_hosts.mdwn
@@ -0,0 +1,19 @@
+Before turning this into a 'todo' item i'd like to discuss the possibilities...
+
+The idea is following:
+
+Having a Laptop with a rather small SSD or some other mobile device i'd like to move files away which are not needed anymore.
+The first thing is that the --to destination should be semi-automatically choosen, including ensuring enough replicas
+
+    git annex move --away 
+
+should pick remotes which are suitable (either by configuration and/or other rules like disk utilization on the remote side).
+
+I am rather new to git-annex and wondering if there is currently already something which gives similar results, esp not need to hand pick the remotes where to move files.
+
+Further on there needs to be some way to find out which files are not needed anymore. On a first thought filtering by 'atime' would be nice, but nowadays mounting with noatime/relatime is common which would make this infeasible. To accomplish this, the assistant could (optionally) manage a lazy-atime by setting inotify or fanotify watches on all annexed files in a repository (close_nowrite) and queue/batch atime updates coarsely together. Then atimes on disk are only lazily updated (after some time expires, when the queue becomes full or at shutdown of the assistant), we can afford to loose some atime updates here in case of unexpected shutdowns (i rather wonder why the kernel has no lazy-atime option).
+
+Then the assistant (or by crontab) one can schedule some regular maintenance. There are certainly plenty of options to consider here, for example a mobile device might prefer only to send files if connected to Wlan, someone wants to move files away until a certain threshold of free disk space is reached etc...
+ 
+While at this, the assistant could also watch (fanotify) if someone tries to open a not available (dead symlinked) file, block that request, get the file and then proceed with the request.
+
diff --git a/doc/forum/expire_files__44___move_to_other_hosts/comment_1_ddcc2a00be1ae96a352d75a443458bcf._comment b/doc/forum/expire_files__44___move_to_other_hosts/comment_1_ddcc2a00be1ae96a352d75a443458bcf._comment
new file mode 100644
index 0000000000..a23a214948
--- /dev/null
+++ b/doc/forum/expire_files__44___move_to_other_hosts/comment_1_ddcc2a00be1ae96a352d75a443458bcf._comment
@@ -0,0 +1,14 @@
+[[!comment format=mdwn
+ username="http://joeyh.name/"
+ ip="4.154.0.21"
+ subject="comment 1"
+ date="2013-07-30T19:14:25Z"
+ content="""
+I think that you're conflating two different features.
+
+First, there's the question of determining which remotes should store a file. The [[preferred_content]] expressions are a way to let the user define this in a way that makes sense for their setup. There's also the annex.diskreserve setting, to allow a remote to reject a file if it doesn't have space. The git-annex assistant can automatically apply those, and arrange to transfer files to all remotes that want a copy. Or you can do it on the command line using --auto. Perhaps it would be nice to have a `git annex copy --auto --to any`, to avoid needing to run copy multiple times to send to different remotes.
+
+Second, there's the concept of expiring files. I don't think relatime would prevent using atime for this (git-annex could just set the atime to 0 when it first gets a file to work around relatime), and I don't like the idea of inotify watching all files just to work around noatime. I suppose that the preferred content expressions could have an atime check operation added to them. Although I guess you could just as well use `find`..
+
+(Finally, I don't think there's a good way to block processes that are trying to open files that are not there, without using FuSE. And I don't know that a system that could block any program indefinitely waiting on some large files being pulled down from wherever would be one I'd want to use. This has prevented me from going down that path so far.)
+"""]]
diff --git a/doc/forum/expire_files__44___move_to_other_hosts/comment_2_7a4c3858c5eae409d04de3f9da43b57e._comment b/doc/forum/expire_files__44___move_to_other_hosts/comment_2_7a4c3858c5eae409d04de3f9da43b57e._comment
new file mode 100644
index 0000000000..ea1e680745
--- /dev/null
+++ b/doc/forum/expire_files__44___move_to_other_hosts/comment_2_7a4c3858c5eae409d04de3f9da43b57e._comment
@@ -0,0 +1,17 @@
+[[!comment format=mdwn
+ username="cehteh"
+ ip="217.8.62.137"
+ subject="comment 2"
+ date="2013-07-30T20:29:11Z"
+ content="""
+I agree here are even more than 2 features involved.
+
+I'd appreciate a 'git annex copy --auto --to any' like feature. The point is that this should not only copy data until diskreserve is hit but distribute data in some (configureable) way around all remotes. Preferred content is one part of that, available disksize another. The user might also choose destinations depending on location and bandwidth and balance load over multiple servers until enough replicas are distributed. Details have to be worked out.
+
+Expiring content is another thing, i also thinking its most likely improper to add some atime watching thing to git-annex. Instead of that I am thinking to write a dedicated daemon which handles atime updates in userspace, this then could add some more rules to ignore accesses by other system tools (file indexers, users, etc). This makes such a expire facility completely independent from git annex and a user can choose if/what he likes.
+
+Filtering out files which are not accessed recently can then be done by 'find' or something similar and piped into 'git annex move/copy/drop'.
+
+And for your final note: fanotify can block accesses to files, it might be a bit ugly and certainly this is not for everyone, perhaps this could be externalized into some watching daemon too or if integrated to git-annex be treated very carefully and only be used if explicitly configured.
+
+"""]]
diff --git a/doc/forum/exporting_annexed_files.mdwn b/doc/forum/exporting_annexed_files.mdwn
new file mode 100644
index 0000000000..0b8d6f36b2
--- /dev/null
+++ b/doc/forum/exporting_annexed_files.mdwn
@@ -0,0 +1,4 @@
+Is there an easy way to export annexed files out of the repository? (e.g. to make a copy elsewhere, send a file by email...)
+
+Thanks,
+Denis.
diff --git a/doc/forum/exporting_annexed_files/comment_1_e08e4c79588e17fb2f1cdf53d9fab7ea._comment b/doc/forum/exporting_annexed_files/comment_1_e08e4c79588e17fb2f1cdf53d9fab7ea._comment
new file mode 100644
index 0000000000..69fc46245f
--- /dev/null
+++ b/doc/forum/exporting_annexed_files/comment_1_e08e4c79588e17fb2f1cdf53d9fab7ea._comment
@@ -0,0 +1,16 @@
+[[!comment format=mdwn
+ username="http://joeyh.name/"
+ ip="4.154.6.135"
+ subject="comment 1"
+ date="2012-06-15T19:25:59Z"
+ content="""
+Sure, you can simply:
+
+    cp annexedfile ~
+
+Or just attach the file right from the git repository to an email, like any other file. Should work fine.
+
+If you wanted to copy a whole directory to export, you'd need to use the -L flag to make cp follow the symlinks and copy the real contents:
+
+    cp -r -L annexeddirectory /media/usbdrive/
+"""]]
diff --git a/doc/forum/exporting_annexed_files/comment_2_15dc3024417b5b2ff3544a08beacab34._comment b/doc/forum/exporting_annexed_files/comment_2_15dc3024417b5b2ff3544a08beacab34._comment
new file mode 100644
index 0000000000..3621f9b895
--- /dev/null
+++ b/doc/forum/exporting_annexed_files/comment_2_15dc3024417b5b2ff3544a08beacab34._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="http://denis.laxalde.org/"
+ nickname="dlax"
+ subject="nautilus"
+ date="2012-06-15T19:57:31Z"
+ content="""
+Ah! I was fooled by nautilus which is not able to properly handle symlinks when copying. It copies links instead of target [[!gnomebug 623580]].
+"""]]
diff --git a/doc/forum/exporting_annexed_files/comment_3_86f0e0f767a84a0f583e121d36cb7d48._comment b/doc/forum/exporting_annexed_files/comment_3_86f0e0f767a84a0f583e121d36cb7d48._comment
new file mode 100644
index 0000000000..db6f90d881
--- /dev/null
+++ b/doc/forum/exporting_annexed_files/comment_3_86f0e0f767a84a0f583e121d36cb7d48._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="http://joeyh.name/"
+ ip="4.154.6.135"
+ subject="comment 3"
+ date="2012-06-16T03:26:37Z"
+ content="""
+That nautilous behavior is a bad thing when trying to export files out, but it's a good thing when just moving files around inside your repository...
+"""]]
diff --git a/doc/forum/external_special_remote_protocol_and_threads.mdwn b/doc/forum/external_special_remote_protocol_and_threads.mdwn
new file mode 100644
index 0000000000..93b370d4e2
--- /dev/null
+++ b/doc/forum/external_special_remote_protocol_and_threads.mdwn
@@ -0,0 +1,3 @@
+As the sync command just got a -J option I have started to use threads to transfer new files to multiple remotes in parallel. A question that arises is if I still can assume that only one instance of the special remote will be started. I tend to store quite some state in the special remote program that I use to connect to a cloud server, and running two instances in parallel might run in to all sorts of fun behavior.
+
+That said, I am of course aware that if I run multiple git annex commands I would run into these problems.
diff --git a/doc/forum/external_special_remote_protocol_and_threads/comment_1_f29c2b0f0f5e349e36eb47176e33498f._comment b/doc/forum/external_special_remote_protocol_and_threads/comment_1_f29c2b0f0f5e349e36eb47176e33498f._comment
new file mode 100644
index 0000000000..8448052da2
--- /dev/null
+++ b/doc/forum/external_special_remote_protocol_and_threads/comment_1_f29c2b0f0f5e349e36eb47176e33498f._comment
@@ -0,0 +1,20 @@
+[[!comment format=mdwn
+ username="joey"
+ subject="""comment 1"""
+ date="2015-10-02T16:14:20Z"
+ content="""
+Well, I can confirm that only one instance of any 
+external special remote will be run per git-annex process currently. 
+
+Actually, it's a bit more than that -- If multiple threads try to use the
+same external special remote at the same time, everything is necessarily
+serialized, and so using -J won't let multiple downloads happen at once
+from a single external special remote, although it may still usefuly
+parallelize amoung several remotes.
+
+Since that seems like something worth dealing with, perhaps by having a
+pool of external special remote processes, I don't feel comfortable making
+any promises about the behavior. And as you note, it's easy to get multiple
+procesess by running multiple git-annex commands, so that is something an
+external special remote needs to deal with anyway.
+"""]]
diff --git a/doc/forum/external_special_remote_protocol_and_threads/comment_2_df97223aad2c8c95ce109944d64a59fe._comment b/doc/forum/external_special_remote_protocol_and_threads/comment_2_df97223aad2c8c95ce109944d64a59fe._comment
new file mode 100644
index 0000000000..11ce16ac83
--- /dev/null
+++ b/doc/forum/external_special_remote_protocol_and_threads/comment_2_df97223aad2c8c95ce109944d64a59fe._comment
@@ -0,0 +1,10 @@
+[[!comment format=mdwn
+ username="https://www.google.com/accounts/o8/id?id=AItOawnmF_9CAtfqdZkC4e-_dCX-rK5bqh4RWkw"
+ nickname="Carl"
+ subject="comment 2"
+ date="2015-10-04T07:49:02Z"
+ content="""
+Thanks for a clear answer. It seems that I am still in the sweet spot for using -J4. I.e transport everything to multiple remotes in parallel. 
+
+I do however take it as a recommendation to be careful, and will give it another consideration to be careful in case of doing more development work and releasing it.
+"""]]
diff --git a/doc/forum/fail_to_git_annex_add_some_files__58___getFileStatus__58___does_not_exist__40__v_3.20111231__41__.mdwn b/doc/forum/fail_to_git_annex_add_some_files__58___getFileStatus__58___does_not_exist__40__v_3.20111231__41__.mdwn
new file mode 100644
index 0000000000..633ff6f9bc
--- /dev/null
+++ b/doc/forum/fail_to_git_annex_add_some_files__58___getFileStatus__58___does_not_exist__40__v_3.20111231__41__.mdwn
@@ -0,0 +1,26 @@
+First, I'm using a 2011 version because i'm getting this kind of errors from cabal install (on Fedora 16 and 17):
+
+    $ cabal install git-annex --bindir=$HOME/.local/bin
+    Resolving dependencies...
+    cabal: cannot configure git-annex-3.20120406. It requires base >=4.5 && <5
+    For the dependency on base >=4.5 && <5 there are these packages: base-4.5.0.0.
+    However none of them are available.
+    base-4.5.0.0 was excluded because of the top level dependency base -any
+
+So I installed a 2011 version and it worked.
+
+Now, when I add some files in the git annex repository I get an error:
+
+    $ git annex add Photo\ Library/2010/06/28/IMG_4926.JPG
+    add Photo Library/2010/06/28/IMG_4926.JPG (checksum...)
+    git-annex: Photo Library/2010/06/28/IMG_4926.JPG: getFileStatus: does not exist (No such file or directory)
+    failed
+    git-annex: add: 1 failed
+
+None of the other files in the same directory are a problem. The file content is not a problem either as I can move the file elsewhere and git annex add it w/o any problem. It's this file in this directory that causes the problem.
+
+Something interesting though. If I move the file elsewhere, and git annex add it, there is no problem. Now, if I git mv the file back into its original location, and git annex fix the file, the symbolic link is wrong: instead of pointing to `../../../../.git/annex/objects/somefile` it points to  `../../../annex/objects/somefile` (notice the missing `../.git/` part of the path).
+
+I can fix that by hand, and it works well, but that's very annoying. There are not much files having that bug though.
+
+[Mildred](http://mildred.fr)
diff --git a/doc/forum/fail_to_git_annex_add_some_files__58___getFileStatus__58___does_not_exist__40__v_3.20111231__41__/comment_1_990197bf01351dc1ccbe1940d5084adb._comment b/doc/forum/fail_to_git_annex_add_some_files__58___getFileStatus__58___does_not_exist__40__v_3.20111231__41__/comment_1_990197bf01351dc1ccbe1940d5084adb._comment
new file mode 100644
index 0000000000..7164de760d
--- /dev/null
+++ b/doc/forum/fail_to_git_annex_add_some_files__58___getFileStatus__58___does_not_exist__40__v_3.20111231__41__/comment_1_990197bf01351dc1ccbe1940d5084adb._comment
@@ -0,0 +1,7 @@
+[[!comment format=mdwn
+ username="http://mildred.pip.verisignlabs.com/"
+ subject="Manually fixing links doesn't even work"
+ date="2012-04-12T15:46:54Z"
+ content="""
+I'm just answering myself: manually fixing symlinks doesn't always works. Sometimes the pre-commit hook will just rewrite the link to some wrong path.
+"""]]
diff --git a/doc/forum/fail_to_git_annex_add_some_files__58___getFileStatus__58___does_not_exist__40__v_3.20111231__41__/comment_2_3bb1d21b7f0d0bd6d59190ae9d246d46._comment b/doc/forum/fail_to_git_annex_add_some_files__58___getFileStatus__58___does_not_exist__40__v_3.20111231__41__/comment_2_3bb1d21b7f0d0bd6d59190ae9d246d46._comment
new file mode 100644
index 0000000000..2fa6cc01a6
--- /dev/null
+++ b/doc/forum/fail_to_git_annex_add_some_files__58___getFileStatus__58___does_not_exist__40__v_3.20111231__41__/comment_2_3bb1d21b7f0d0bd6d59190ae9d246d46._comment
@@ -0,0 +1,12 @@
+[[!comment format=mdwn
+ username="http://joey.kitenet.net/"
+ nickname="joey"
+ subject="comment 2"
+ date="2012-04-12T16:29:58Z"
+ content="""
+This bug was fixed in git-annex 3.20120230. You have a few options to get the fix:
+
+* Upgrade to ghc 7.4, the need for which is the cause of the cabal error message you pasted.
+* Manually [[download]] from git, and `git checkout ghc7.0` -- that branch will build with your old ghc and has the fix.
+* cherry-pick commit 51338486dcf9ab86de426e41b1eb31af1d3a6c87
+"""]]
diff --git a/doc/forum/fail_to_git_annex_add_some_files__58___getFileStatus__58___does_not_exist__40__v_3.20111231__41__/comment_3_692f268218690437138ae0540c879425._comment b/doc/forum/fail_to_git_annex_add_some_files__58___getFileStatus__58___does_not_exist__40__v_3.20111231__41__/comment_3_692f268218690437138ae0540c879425._comment
new file mode 100644
index 0000000000..d5c84b431f
--- /dev/null
+++ b/doc/forum/fail_to_git_annex_add_some_files__58___getFileStatus__58___does_not_exist__40__v_3.20111231__41__/comment_3_692f268218690437138ae0540c879425._comment
@@ -0,0 +1,16 @@
+[[!comment format=mdwn
+ username="http://mildred.pip.verisignlabs.com/"
+ subject="Thank you a lot"
+ date="2012-04-13T07:28:10Z"
+ content="""
+Thank you,
+
+I imagined it was something like that.
+I 'm just sorry I posted that on the forum and not on the bugs section (I hadn't discovered it at that time). but now, if people search for this error, they should find this.
+
+Note for Fedora users: unfortunately GHC 7.4 will not be shipped with Fedora 17 (which is still not released). The [feature page](https://fedoraproject.org/wiki/Features/GHC74) mention it for Fedora 18. I feel like I am using debian ... outdated packages the day of the release.
+
+And many thanks for this wonderful piece of software.
+
+Mildred
+"""]]
diff --git a/doc/forum/failed_to_push_anonymously.mdwn b/doc/forum/failed_to_push_anonymously.mdwn
new file mode 100644
index 0000000000..196f6f71ca
--- /dev/null
+++ b/doc/forum/failed_to_push_anonymously.mdwn
@@ -0,0 +1,19 @@
+I have pushed changes to this wiki in the past using this URL as specified in the `Branchable` tab:
+
+    git://git-annex.branchable.com/
+
+Yet after writing the [[tips/hashdeep integration]] page, I wasn't able to push it to this git repo, and got the following error:
+
+    [1023]anarcat@curie:git-annex$ git push 
+    Décompte des objets: 5, fait.
+    Delta compression using up to 4 threads.
+    Compression des objets: 100% (5/5), fait.
+    Écriture des objets: 100% (5/5), 3.30 KiB | 3.30 MiB/s, fait.
+    Total 5 (delta 3), reused 0 (delta 0)
+    remote: fatal: Not a git repository (or any of the parent directories): .git
+    remote: 'git log --pretty=raw --raw --abbrev=40 --always -c -r d8de48ddee630f546bd71adff6024f875f6db4c2..5bce64b25e53849a6cd7f5973fed849898428efa --no-renames -- .' failed: 
+    To git://git-annex.branchable.com/
+     ! [remote rejected]     master -> master (pre-receive hook declined)
+    error: impossible de pousser des références vers 'git://git-annex.branchable.com/'
+
+Seems like there's something wrong on that end, or did I do something wrong? -- [[anarcat]]
diff --git a/doc/forum/failed_to_push_anonymously/comment_1_bdf93b0e9fb109b7a4052a99110f6079._comment b/doc/forum/failed_to_push_anonymously/comment_1_bdf93b0e9fb109b7a4052a99110f6079._comment
new file mode 100644
index 0000000000..5d8692632b
--- /dev/null
+++ b/doc/forum/failed_to_push_anonymously/comment_1_bdf93b0e9fb109b7a4052a99110f6079._comment
@@ -0,0 +1,7 @@
+[[!comment format=mdwn
+ username="joey"
+ subject="""comment 1"""
+ date="2018-07-02T16:45:07Z"
+ content="""
+
+"""]]
diff --git a/doc/forum/failing_to_add_an_usb_drive_made_me_lose_all_symlinks.mdwn b/doc/forum/failing_to_add_an_usb_drive_made_me_lose_all_symlinks.mdwn
new file mode 100644
index 0000000000..3ee7fae84d
--- /dev/null
+++ b/doc/forum/failing_to_add_an_usb_drive_made_me_lose_all_symlinks.mdwn
@@ -0,0 +1,27 @@
+Dear all,
+        I had (snif!) and indirect repo with 2.7 of pdf files with filenames were manually added by me with author and title, and took years of work. I had them in a git annex repo from quite some time.  
+Now I'm trying to sync them to an usb drive. Main repo lives in an ubuntu box with ext4 filesystem. I failed many times to do this, both on fat and exfat external drives (direct).  
+As I was failing, I tried to to a test on the same local filesystem. So I have my main repo in /home/juan/papers and the other in /tmp/papers.  
+While trying to sync them using the assistant, i noticed that most of the folders and symlinks from the main repo just dissapeared. 
+I still have them in .git/annex/objects but I don't know how to recover the links. 
+I must say I'm an everyday git user, and big fan of git-annex but I'm trying to use it for six months without luck. I think I will quit after recovering the data, if possible.
+
+EDIT 1
+-------
+After digging around a bit, I found that I had the keys, and could find out the original filename, for example:
+
+    git log --stat -S 1279
+    commit a6362df13d728037875320a457182acb073d08ea
+    Author: Juan Cardelino 
+    Date:   Fri Nov 21 11:25:31 2014 -0200
+
+     2013_august/feature_matching_by_searching_maximum_clique_on_high_order_association.pdf | 1 +
+     LevelSets/fedkiw/stanford2002-06.pdf                                                   | 1 +
+     new/nfa/00106994.pdf                                                                   | 1 +
+     3 files changed, 3 insertions(+)
+
+How can I recover them?
+
+EDIT 2
+-------
+Seems that I overwritten my repo with a new repo. Because git log only shows 4 entries from today. Is it possible that the webapp has made that?
diff --git a/doc/forum/failing_to_add_an_usb_drive_made_me_lose_all_symlinks/comment_1_3167052f67706797b00423a50f2ae7f1._comment b/doc/forum/failing_to_add_an_usb_drive_made_me_lose_all_symlinks/comment_1_3167052f67706797b00423a50f2ae7f1._comment
new file mode 100644
index 0000000000..dd07c1924c
--- /dev/null
+++ b/doc/forum/failing_to_add_an_usb_drive_made_me_lose_all_symlinks/comment_1_3167052f67706797b00423a50f2ae7f1._comment
@@ -0,0 +1,23 @@
+[[!comment format=mdwn
+ username="joey"
+ subject="""comment 1"""
+ date="2014-12-01T21:56:45Z"
+ content="""
+Well, git-annex just uses a regular git repo. It should not be possible to
+fully lose data. I suggest you run `git log
+--stat` and see if a commit deleted all your files. If so, you can revert
+that commit.
+
+If the git log really is only 4 commits deep, perhaps you got some other
+branch checked out somehow. Run `git reflog` to see the history of what
+branches have been checked out before, and you can use `git checkout` to go
+back to one of those past checkouts.
+
+Or, failing all that, you showed a commit
+a6362df13d728037875320a457182acb073d08ea which had one of your files,
+so `git checkout a6362df13d728037875320a457182acb073d08ea` will check out
+the tree as it appeared at the time of that commit.
+
+This is all git 101. AFAICS, you have not lost any data, and need to become
+more familiar with the basics of git.
+"""]]
diff --git a/doc/forum/failing_to_add_an_usb_drive_made_me_lose_all_symlinks/comment_2_0082ac7a148db69293a4e37fa6d6c891._comment b/doc/forum/failing_to_add_an_usb_drive_made_me_lose_all_symlinks/comment_2_0082ac7a148db69293a4e37fa6d6c891._comment
new file mode 100644
index 0000000000..06832ca8bd
--- /dev/null
+++ b/doc/forum/failing_to_add_an_usb_drive_made_me_lose_all_symlinks/comment_2_0082ac7a148db69293a4e37fa6d6c891._comment
@@ -0,0 +1,11 @@
+[[!comment format=mdwn
+ username="joey"
+ subject="""comment 2"""
+ date="2014-12-02T17:34:41Z"
+ content="""
+I'm glad you were able to solve it!
+
+Now that you know what the bad commit is, can you tell me what it looks
+like? The assistant should not be committing deletions of all files, 
+unless all files actually were deleted...
+"""]]
diff --git a/doc/forum/failing_to_add_an_usb_drive_made_me_lose_all_symlinks/comment_2_6411aa3e6f5c1515845f3223bbcdabf1._comment b/doc/forum/failing_to_add_an_usb_drive_made_me_lose_all_symlinks/comment_2_6411aa3e6f5c1515845f3223bbcdabf1._comment
new file mode 100644
index 0000000000..6e9c75aaf7
--- /dev/null
+++ b/doc/forum/failing_to_add_an_usb_drive_made_me_lose_all_symlinks/comment_2_6411aa3e6f5c1515845f3223bbcdabf1._comment
@@ -0,0 +1,14 @@
+[[!comment format=mdwn
+ username="https://www.google.com/accounts/o8/id?id=AItOawmTNrhkVQ26GBLaLD5-zNuEiR8syTj4mI8"
+ nickname="Juan"
+ subject="I figured it out"
+ date="2014-12-02T01:48:38Z"
+ content="""
+I'm not new to git, and I perfectly know how to revert commits, however, I was confused about how git annex and git play together. In particular I tried to use it as a black box with the assistant and that was a bad idea.
+After adding the usb drive as remote with the assitant, all symlinks got deleted. However, doing a checkout to the commit that messed up everything I got thing back to normal.
+Sadly, your message arrived a couple of hours late, I could have saved me a lot of time.
+Thanks for taking the time to answer.
+Sorry for the previous rant.
+Best regards,
+                      Juan
+"""]]
diff --git a/doc/forum/faking_location_information.mdwn b/doc/forum/faking_location_information.mdwn
new file mode 100644
index 0000000000..7c84e12e39
--- /dev/null
+++ b/doc/forum/faking_location_information.mdwn
@@ -0,0 +1,19 @@
+Hi
+
+I am using git-annex even if people I exchange data with (currently) don‘t use it for there data. My idea behind this is that I would like to know from where I got a file, whom I gave a file and who does (probably) still have a copy of the file. To do this you need to trick git-annex location tracking feature a bit. I successfully managed to achieve this in a simple data exchange which only consisted of me coping over files to one of my git-annex repos. I did the following to make git-annex believe that the files are in two repos without the need to *copy* them around the repos.
+
+This is what I did in this simple case:
+
+1. mounted the drive from someone
+2. made a clone of my git-annex repo on the filesystem which should hold the copy of the data
+3. initialized the cloned repo with `git annex init "Drive from person X"`
+4. imported the files to the cloned repo with `git annex import --duplicate $path_to_files_from_person_x`
+5. `git annex sync` in the cloned repo
+6. `git annex sync` in main repo
+7. `git annex move . --to origin` in the cloned repo
+
+The impotent part (and the limit) here was that you can not sync these two repos after you moved files to the main repo. The problem is that there will be situations where I will have to sync them also after moving files around (for example if I want to store new files in multiple repos (and not just one main repo), or if I also want to copy files over to drives from someone).
+
+Note: I have also worked out a solution to allow someone to choose which files he/she would like to get as described [on superuser.com](http://superuser.com/a/717689).
+
+Are there better ways to fake location information then the thing I came up with (except making multiple repos for one person/drive)? Can multiple remotes be merged to one remote?
diff --git a/doc/forum/fatal__58___Out_of_memory__63___mmap_failed__58___No_such_file_or_directory.mdwn b/doc/forum/fatal__58___Out_of_memory__63___mmap_failed__58___No_such_file_or_directory.mdwn
new file mode 100644
index 0000000000..ee0b49eb64
--- /dev/null
+++ b/doc/forum/fatal__58___Out_of_memory__63___mmap_failed__58___No_such_file_or_directory.mdwn
@@ -0,0 +1,112 @@
+I'm getting this error, not exactly sure why... Here's the output of running git annex sync in my pictures folder:
+
+	[2014-06-29 12:43:59 Mitteleuropäische Sommerzeit] read: git ["--git-dir=C:\\Users\\Name\\Pictures\\.git","--work-tree=C:\\Users\\Name\\Pictures","show-ref","git-annex"]
+	[2014-06-29 12:43:59 Mitteleuropäische Sommerzeit] read: git ["--git-dir=C:\\Users\\Name\\Pictures\\.git","--work-tree=C:\\Users\\Name\\Pictures","show-ref","--hash","refs/heads/git-annex"]
+	[2014-06-29 12:43:59 Mitteleuropäische Sommerzeit] read: git ["--git-dir=C:\\Users\\Name\\Pictures\\.git","--work-tree=C:\\Users\\Name\\Pictures","log","refs/heads/git-annex..7bf55ad4463cad9389cbb11e48334d151aacf45f","--oneline","-n1"]
+	[2014-06-29 12:43:59 Mitteleuropäische Sommerzeit] read: git ["--git-dir=C:\\Users\\Name\\Pictures\\.git","--work-tree=C:\\Users\\Name\\Pictures","log","refs/heads/git-annex..f3331818fc79b6e9523e483b9d97aaec5c8da845","--oneline","-n1"]
+	[2014-06-29 12:43:59 Mitteleuropäische Sommerzeit] read: git ["--git-dir=C:\\Users\\Name\\Pictures\\.git","--work-tree=C:\\Users\\Name\\Pictures","log","refs/heads/git-annex..6c3bee76aa98f5c789ddf89450b3bd52658171a6","--oneline","-n1"]
+	[2014-06-29 12:43:59 Mitteleuropäische Sommerzeit] read: git ["--git-dir=C:\\Users\\Name\\Pictures\\.git","--work-tree=C:\\Users\\Name\\Pictures","log","refs/heads/git-annex..9329bf7ae5609b19e81d9605fdc990c87c5816dc","--oneline","-n1"]
+	[2014-06-29 12:43:59 Mitteleuropäische Sommerzeit] read: git ["--git-dir=C:\\Users\\Name\\Pictures\\.git","--work-tree=C:\\Users\\Name\\Pictures","log","refs/heads/git-annex..ad7115c6108170dc54095f74632db339dbca4cdd","--oneline","-n1"]
+	[2014-06-29 12:43:59 Mitteleuropäische Sommerzeit] chat: git ["--git-dir=C:\\Users\\Name\\Pictures\\.git","--work-tree=C:\\Users\\Name\\Pictures","cat-file","--batch"]
+	[2014-06-29 12:43:59 Mitteleuropäische Sommerzeit] read: git ["config","--null","--list"]
+	commit 
+	[2014-06-29 12:43:59 Mitteleuropäische Sommerzeit] read: git ["--git-dir=C:\\Users\\Name\\Pictures\\.git","--work-tree=C:\\Users\\Name\\Pictures","ls-files","--stage","-z","--others","--exclude-standard","--","C:\\Users\\Name\\Pictures"]
+	[2014-06-29 12:43:59 Mitteleuropäische Sommerzeit] chat: git ["--git-dir=C:\\Users\\Name\\Pictures\\.git","--work-tree=C:\\Users\\Name\\Pictures","cat-file","--batch"]
+	(Recording state in git...)
+	[2014-06-29 12:49:57 Mitteleuropäische Sommerzeit] call: git ["--git-dir=C:\\Users\\Name\\Pictures\\.git","--work-tree=C:\\Users\\Name\\Pictures","add","-f","Hochzeit von NameB und NameC/2013-08-18_MVI_0583.MOV"]
+	fatal: Out of memory? mmap failed: No such file or directory
+	[2014-06-29 12:49:57 Mitteleuropäische Sommerzeit] call: git ["--git-dir=C:\\Users\\Name\\Pictures\\.git","--work-tree=C:\\Users\\Name\\Pictures","add","-f","Hochzeit von NameB und NameC/2013-08-18_MVI_0582.MOV"]
+	fatal: Out of memory? mmap failed: No such file or directory
+	[2014-06-29 12:49:57 Mitteleuropäische Sommerzeit] call: git ["--git-dir=C:\\Users\\Name\\Pictures\\.git","--work-tree=C:\\Users\\Name\\Pictures","add","-f","Hochzeit von NameB und NameC/2013-08-18_MVI_0417.MOV"]
+	fatal: Out of memory? mmap failed: No such file or directory
+	[2014-06-29 12:49:57 Mitteleuropäische Sommerzeit] call: git ["--git-dir=C:\\Users\\Name\\Pictures\\.git","--work-tree=C:\\Users\\Name\\Pictures","add","-f","Hochzeit von NameB und NameC/2013-08-17_MVI_9701.MOV"]
+	fatal: Out of memory? mmap failed: No such file or directory
+	[2014-06-29 12:49:57 Mitteleuropäische Sommerzeit] call: git ["--git-dir=C:\\Users\\Name\\Pictures\\.git","--work-tree=C:\\Users\\Name\\Pictures","add","-f","2014/Hochzeit von Jana und Stefan/MVI_7168.MOV"]
+	[2014-06-29 12:49:57 Mitteleuropäische Sommerzeit] call: git ["--git-dir=C:\\Users\\Name\\Pictures\\.git","--work-tree=C:\\Users\\Name\\Pictures","add","-f","2012/Event/IMG_2456.JPG"]
+	[2014-06-29 12:49:57 Mitteleuropäische Sommerzeit] call: git ["--git-dir=C:\\Users\\Name\\Pictures\\.git","--work-tree=C:\\Users\\Name\\Pictures","add","-f","2012/Event/IMG_2452.JPG"]
+	[2014-06-29 12:49:57 Mitteleuropäische Sommerzeit] call: git ["--git-dir=C:\\Users\\Name\\Pictures\\.git","--work-tree=C:\\Users\\Name\\Pictures","add","-f","2012/Event/IMG_2450.JPG"]
+	[2014-06-29 12:49:57 Mitteleuropäische Sommerzeit] call: git ["--git-dir=C:\\Users\\Name\\Pictures\\.git","--work-tree=C:\\Users\\Name\\Pictures","add","-f","2012/Event/IMG_2449.JPG"]
+	[2014-06-29 12:49:57 Mitteleuropäische Sommerzeit] call: git ["--git-dir=C:\\Users\\Name\\Pictures\\.git","--work-tree=C:\\Users\\Name\\Pictures","add","-f","2012/Event/IMG_2448.JPG"]
+	[2014-06-29 12:49:57 Mitteleuropäische Sommerzeit] call: git ["--git-dir=C:\\Users\\Name\\Pictures\\.git","--work-tree=C:\\Users\\Name\\Pictures","add","-f","2012/Event/IMG_2447.JPG"]
+	[2014-06-29 12:49:57 Mitteleuropäische Sommerzeit] call: git ["--git-dir=C:\\Users\\Name\\Pictures\\.git","--work-tree=C:\\Users\\Name\\Pictures","add","-f","2012/Event/IMG_2443.JPG"]
+	[2014-06-29 12:49:58 Mitteleuropäische Sommerzeit] call: git ["--git-dir=C:\\Users\\Name\\Pictures\\.git","--work-tree=C:\\Users\\Name\\Pictures","add","-f","2012/Event/IMG_2439.JPG"]
+	[2014-06-29 12:49:58 Mitteleuropäische Sommerzeit] call: git ["--git-dir=C:\\Users\\Name\\Pictures\\.git","--work-tree=C:\\Users\\Name\\Pictures","add","-f","2012/Event/IMG_2436.JPG"]
+	[2014-06-29 12:49:58 Mitteleuropäische Sommerzeit] call: git ["--git-dir=C:\\Users\\Name\\Pictures\\.git","--work-tree=C:\\Users\\Name\\Pictures","add","-f","2012/Event/IMG_2431.JPG"]
+	[2014-06-29 12:49:58 Mitteleuropäische Sommerzeit] call: git ["--git-dir=C:\\Users\\Name\\Pictures\\.git","--work-tree=C:\\Users\\Name\\Pictures","add","-f","2012/Event/IMG_2430.JPG"]
+	[2014-06-29 12:49:58 Mitteleuropäische Sommerzeit] call: git ["--git-dir=C:\\Users\\Name\\Pictures\\.git","--work-tree=C:\\Users\\Name\\Pictures","add","-f","2012/Event/IMG_2425.JPG"]
+	[2014-06-29 12:49:58 Mitteleuropäische Sommerzeit] call: git ["--git-dir=C:\\Users\\Name\\Pictures\\.git","--work-tree=C:\\Users\\Name\\Pictures","add","-f","2012/Event/IMG_2424.JPG"]
+	[2014-06-29 12:49:58 Mitteleuropäische Sommerzeit] call: git ["--git-dir=C:\\Users\\Name\\Pictures\\.git","--work-tree=C:\\Users\\Name\\Pictures","add","-f","2012/Event/IMG_2423.JPG"]
+	[2014-06-29 12:49:58 Mitteleuropäische Sommerzeit] call: git ["--git-dir=C:\\Users\\Name\\Pictures\\.git","--work-tree=C:\\Users\\Name\\Pictures","add","-f","2012/Event/IMG_2421.JPG"]
+	[2014-06-29 12:49:58 Mitteleuropäische Sommerzeit] call: git ["--git-dir=C:\\Users\\Name\\Pictures\\.git","--work-tree=C:\\Users\\Name\\Pictures","add","-f","2012/Event/IMG_2418.JPG"]
+	[2014-06-29 12:49:58 Mitteleuropäische Sommerzeit] call: git ["--git-dir=C:\\Users\\Name\\Pictures\\.git","--work-tree=C:\\Users\\Name\\Pictures","add","-f","2012/Event/IMG_2417.JPG"]
+	[2014-06-29 12:49:58 Mitteleuropäische Sommerzeit] call: git ["--git-dir=C:\\Users\\Name\\Pictures\\.git","--work-tree=C:\\Users\\Name\\Pictures","add","-f","2012/Event/IMG_2416.JPG"]
+	[2014-06-29 12:49:58 Mitteleuropäische Sommerzeit] call: git ["--git-dir=C:\\Users\\Name\\Pictures\\.git","--work-tree=C:\\Users\\Name\\Pictures","add","-f","2012/Event/IMG_2415.JPG"]
+	[2014-06-29 12:49:58 Mitteleuropäische Sommerzeit] call: git ["--git-dir=C:\\Users\\Name\\Pictures\\.git","--work-tree=C:\\Users\\Name\\Pictures","add","-f","2012/Event/IMG_2411.JPG"]
+	[2014-06-29 12:49:58 Mitteleuropäische Sommerzeit] call: git ["--git-dir=C:\\Users\\Name\\Pictures\\.git","--work-tree=C:\\Users\\Name\\Pictures","add","-f","2012/Event/IMG_2409.JPG"]
+	[2014-06-29 12:49:59 Mitteleuropäische Sommerzeit] call: git ["--git-dir=C:\\Users\\Name\\Pictures\\.git","--work-tree=C:\\Users\\Name\\Pictures","add","-f","2012/Event/IMG_2406.JPG"]
+	[2014-06-29 12:49:59 Mitteleuropäische Sommerzeit] call: git ["--git-dir=C:\\Users\\Name\\Pictures\\.git","--work-tree=C:\\Users\\Name\\Pictures","add","-f","2012/Event/IMG_2405.JPG"]
+	[2014-06-29 12:49:59 Mitteleuropäische Sommerzeit] call: git ["--git-dir=C:\\Users\\Name\\Pictures\\.git","--work-tree=C:\\Users\\Name\\Pictures","add","-f","2012/Event/IMG_2404.JPG"]
+	[2014-06-29 12:49:59 Mitteleuropäische Sommerzeit] call: git ["--git-dir=C:\\Users\\Name\\Pictures\\.git","--work-tree=C:\\Users\\Name\\Pictures","add","-f","2012/Event/IMG_2403.JPG"]
+	[2014-06-29 12:49:59 Mitteleuropäische Sommerzeit] call: git ["--git-dir=C:\\Users\\Name\\Pictures\\.git","--work-tree=C:\\Users\\Name\\Pictures","add","-f","2012/Event/IMG_2402.JPG"]
+	[2014-06-29 12:49:59 Mitteleuropäische Sommerzeit] call: git ["--git-dir=C:\\Users\\Name\\Pictures\\.git","--work-tree=C:\\Users\\Name\\Pictures","add","-f","2012/Event/IMG_2400.JPG"]
+	[2014-06-29 12:49:59 Mitteleuropäische Sommerzeit] call: git ["--git-dir=C:\\Users\\Name\\Pictures\\.git","--work-tree=C:\\Users\\Name\\Pictures","add","-f","2012/Event/IMG_2394.JPG"]
+	[2014-06-29 12:49:59 Mitteleuropäische Sommerzeit] call: git ["--git-dir=C:\\Users\\Name\\Pictures\\.git","--work-tree=C:\\Users\\Name\\Pictures","add","-f","2012/Event/IMG_2392.JPG"]
+	[2014-06-29 12:49:59 Mitteleuropäische Sommerzeit] call: git ["--git-dir=C:\\Users\\Name\\Pictures\\.git","--work-tree=C:\\Users\\Name\\Pictures","add","-f","2012/Event/IMG_2390.JPG"]
+	[2014-06-29 12:49:59 Mitteleuropäische Sommerzeit] call: git ["--git-dir=C:\\Users\\Name\\Pictures\\.git","--work-tree=C:\\Users\\Name\\Pictures","add","-f","2005/Amsterdam/CIMG0446.JPG"]
+	[2014-06-29 12:49:59 Mitteleuropäische Sommerzeit] call: git ["--git-dir=C:\\Users\\Name\\Pictures\\.git","--work-tree=C:\\Users\\Name\\Pictures","add","-f","2005/Amsterdam/CIMG0445.JPG"]
+	[2014-06-29 12:49:59 Mitteleuropäische Sommerzeit] call: git ["--git-dir=C:\\Users\\Name\\Pictures\\.git","--work-tree=C:\\Users\\Name\\Pictures","add","-f","2005/Amsterdam/CIMG0444.JPG"]
+	[2014-06-29 12:49:59 Mitteleuropäische Sommerzeit] call: git ["--git-dir=C:\\Users\\Name\\Pictures\\.git","--work-tree=C:\\Users\\Name\\Pictures","add","-f","2005/Amsterdam/CIMG0443.JPG"]
+	[2014-06-29 12:50:00 Mitteleuropäische Sommerzeit] call: git ["--git-dir=C:\\Users\\Name\\Pictures\\.git","--work-tree=C:\\Users\\Name\\Pictures","add","-f","2005/Amsterdam/CIMG0442.JPG"]
+	[2014-06-29 12:50:00 Mitteleuropäische Sommerzeit] call: git ["--git-dir=C:\\Users\\Name\\Pictures\\.git","--work-tree=C:\\Users\\Name\\Pictures","add","-f","2005/Amsterdam/CIMG0441.JPG"]
+	[2014-06-29 12:50:00 Mitteleuropäische Sommerzeit] call: git ["--git-dir=C:\\Users\\Name\\Pictures\\.git","--work-tree=C:\\Users\\Name\\Pictures","add","-f","2005/Amsterdam/CIMG0440.JPG"]
+	[2014-06-29 12:50:00 Mitteleuropäische Sommerzeit] call: git ["--git-dir=C:\\Users\\Name\\Pictures\\.git","--work-tree=C:\\Users\\Name\\Pictures","add","-f","2005/Amsterdam/CIMG0438.JPG"]
+	[2014-06-29 12:50:00 Mitteleuropäische Sommerzeit] call: git ["--git-dir=C:\\Users\\Name\\Pictures\\.git","--work-tree=C:\\Users\\Name\\Pictures","add","-f","2005/Amsterdam/CIMG0437.JPG"]
+	[2014-06-29 12:50:00 Mitteleuropäische Sommerzeit] call: git ["--git-dir=C:\\Users\\Name\\Pictures\\.git","--work-tree=C:\\Users\\Name\\Pictures","add","-f","2005/Amsterdam/CIMG0436.JPG"]
+	[2014-06-29 12:50:00 Mitteleuropäische Sommerzeit] call: git ["--git-dir=C:\\Users\\Name\\Pictures\\.git","--work-tree=C:\\Users\\Name\\Pictures","add","-f","2005/Amsterdam/CIMG0435.JPG"]
+	[2014-06-29 12:50:00 Mitteleuropäische Sommerzeit] call: git ["--git-dir=C:\\Users\\Name\\Pictures\\.git","--work-tree=C:\\Users\\Name\\Pictures","add","-f","2005/Amsterdam/CIMG0434.JPG"]
+	[2014-06-29 12:50:00 Mitteleuropäische Sommerzeit] call: git ["--git-dir=C:\\Users\\Name\\Pictures\\.git","--work-tree=C:\\Users\\Name\\Pictures","add","-f","2005/Amsterdam/CIMG0433.JPG"]
+	[2014-06-29 12:50:00 Mitteleuropäische Sommerzeit] call: git ["--git-dir=C:\\Users\\Name\\Pictures\\.git","--work-tree=C:\\Users\\Name\\Pictures","add","-f","2005/Amsterdam/CIMG0432.JPG"]
+	[2014-06-29 12:50:00 Mitteleuropäische Sommerzeit] call: git ["--git-dir=C:\\Users\\Name\\Pictures\\.git","--work-tree=C:\\Users\\Name\\Pictures","add","-f","2005/Amsterdam/CIMG0431.JPG"]
+	[2014-06-29 12:50:00 Mitteleuropäische Sommerzeit] call: git ["--git-dir=C:\\Users\\Name\\Pictures\\.git","--work-tree=C:\\Users\\Name\\Pictures","add","-f","2005/Amsterdam/CIMG0368.JPG"]
+	[2014-06-29 12:50:00 Mitteleuropäische Sommerzeit] call: git ["--git-dir=C:\\Users\\Name\\Pictures\\.git","--work-tree=C:\\Users\\Name\\Pictures","add","-f","2005/Amsterdam/CIMG0367.JPG"]
+	[2014-06-29 12:50:00 Mitteleuropäische Sommerzeit] call: git ["--git-dir=C:\\Users\\Name\\Pictures\\.git","--work-tree=C:\\Users\\Name\\Pictures","add","-f","2005/Amsterdam/CIMG0366.JPG"]
+	[2014-06-29 12:50:01 Mitteleuropäische Sommerzeit] call: git ["--git-dir=C:\\Users\\Name\\Pictures\\.git","--work-tree=C:\\Users\\Name\\Pictures","add","-f","2005/Amsterdam/CIMG0365.JPG"]
+	[2014-06-29 12:50:01 Mitteleuropäische Sommerzeit] call: git ["--git-dir=C:\\Users\\Name\\Pictures\\.git","--work-tree=C:\\Users\\Name\\Pictures","add","-f","2005/Amsterdam/CIMG0364.JPG"]
+	[2014-06-29 12:50:01 Mitteleuropäische Sommerzeit] read: git ["--git-dir=C:\\Users\\Name\\Pictures\\.git","--work-tree=C:\\Users\\Name\\Pictures","show-ref","--head"]
+	[2014-06-29 12:50:01 Mitteleuropäische Sommerzeit] read: git ["--git-dir=C:\\Users\\Name\\Pictures\\.git","--work-tree=C:\\Users\\Name\\Pictures","diff-index","-z","--raw","--no-renames","-l0","--cached","HEAD"]
+	[2014-06-29 12:50:01 Mitteleuropäische Sommerzeit] read: git ["--git-dir=C:\\Users\\Name\\Pictures\\.git","--work-tree=C:\\Users\\Name\\Pictures","symbolic-ref","HEAD"]
+	[2014-06-29 12:50:01 Mitteleuropäische Sommerzeit] read: git ["--git-dir=C:\\Users\\Name\\Pictures\\.git","--work-tree=C:\\Users\\Name\\Pictures","show-ref","--hash","refs/heads/annex/direct/master"]
+	[2014-06-29 12:50:01 Mitteleuropäische Sommerzeit] read: git ["--git-dir=C:\\Users\\Name\\Pictures\\.git","--work-tree=C:\\Users\\Name\\Pictures","write-tree"]
+	[2014-06-29 12:50:01 Mitteleuropäische Sommerzeit] read: git ["--git-dir=C:\\Users\\Name\\Pictures\\.git","--work-tree=C:\\Users\\Name\\Pictures","rev-parse","ec008c688692d3156b1aececf33a388f43e8c868:"]
+	ok
+	[2014-06-29 12:50:01 Mitteleuropäische Sommerzeit] read: git ["--git-dir=C:\\Users\\Name\\Pictures\\.git","--work-tree=C:\\Users\\Name\\Pictures","symbolic-ref","HEAD"]
+	[2014-06-29 12:50:01 Mitteleuropäische Sommerzeit] read: git ["--git-dir=C:\\Users\\Name\\Pictures\\.git","--work-tree=C:\\Users\\Name\\Pictures","show-ref","refs/heads/annex/direct/master"]
+	[2014-06-29 12:50:01 Mitteleuropäische Sommerzeit] call: git ["--git-dir=C:\\Users\\Name\\Pictures\\.git","--work-tree=C:\\Users\\Name\\Pictures","show-ref","--verify","-q","refs/heads/synced/master"]
+	[2014-06-29 12:50:01 Mitteleuropäische Sommerzeit] read: git ["--git-dir=C:\\Users\\Name\\Pictures\\.git","--work-tree=C:\\Users\\Name\\Pictures","log","refs/heads/annex/direct/master..refs/heads/synced/master","--oneline","-n1"]
+	pull HomeServerLocal
+	[2014-06-29 12:50:01 Mitteleuropäische Sommerzeit] call: git ["--git-dir=C:\\Users\\Name\\Pictures\\.git","--work-tree=C:\\Users\\Name\\Pictures","fetch","HomeServerLocal"]
+	[2014-06-29 12:50:02 Mitteleuropäische Sommerzeit] call: git ["--git-dir=C:\\Users\\Name\\Pictures\\.git","--work-tree=C:\\Users\\Name\\Pictures","show-ref","--verify","-q","refs/remotes/HomeServerLocal/annex/direct/master"]
+	[2014-06-29 12:50:02 Mitteleuropäische Sommerzeit] read: git ["--git-dir=C:\\Users\\Name\\Pictures\\.git","--work-tree=C:\\Users\\Name\\Pictures","log","refs/heads/annex/direct/master..refs/remotes/HomeServerLocal/annex/direct/master","--oneline","-n1"]
+	[2014-06-29 12:50:02 Mitteleuropäische Sommerzeit] call: git ["--git-dir=C:\\Users\\Name\\Pictures\\.git","--work-tree=C:\\Users\\Name\\Pictures","show-ref","--verify","-q","refs/remotes/HomeServerLocal/synced/master"]
+	[2014-06-29 12:50:02 Mitteleuropäische Sommerzeit] read: git ["--git-dir=C:\\Users\\Name\\Pictures\\.git","--work-tree=C:\\Users\\Name\\Pictures","log","refs/heads/synced/master..refs/remotes/HomeServerLocal/synced/master","--oneline","-n1"]
+	ok
+	[2014-06-29 12:50:02 Mitteleuropäische Sommerzeit] read: git ["--git-dir=C:\\Users\\Name\\Pictures\\.git","--work-tree=C:\\Users\\Name\\Pictures","show-ref","git-annex"]
+	[2014-06-29 12:50:02 Mitteleuropäische Sommerzeit] read: git ["--git-dir=C:\\Users\\Name\\Pictures\\.git","--work-tree=C:\\Users\\Name\\Pictures","show-ref","--hash","refs/heads/git-annex"]
+	[2014-06-29 12:50:02 Mitteleuropäische Sommerzeit] read: git ["--git-dir=C:\\Users\\Name\\Pictures\\.git","--work-tree=C:\\Users\\Name\\Pictures","log","refs/heads/git-annex..7bf55ad4463cad9389cbb11e48334d151aacf45f","--oneline","-n1"]
+	[2014-06-29 12:50:02 Mitteleuropäische Sommerzeit] read: git ["--git-dir=C:\\Users\\Name\\Pictures\\.git","--work-tree=C:\\Users\\Name\\Pictures","log","refs/heads/git-annex..f3331818fc79b6e9523e483b9d97aaec5c8da845","--oneline","-n1"]
+	[2014-06-29 12:50:02 Mitteleuropäische Sommerzeit] read: git ["--git-dir=C:\\Users\\Name\\Pictures\\.git","--work-tree=C:\\Users\\Name\\Pictures","log","refs/heads/git-annex..6c3bee76aa98f5c789ddf89450b3bd52658171a6","--oneline","-n1"]
+	[2014-06-29 12:50:03 Mitteleuropäische Sommerzeit] read: git ["--git-dir=C:\\Users\\Name\\Pictures\\.git","--work-tree=C:\\Users\\Name\\Pictures","log","refs/heads/git-annex..9329bf7ae5609b19e81d9605fdc990c87c5816dc","--oneline","-n1"]
+	[2014-06-29 12:50:03 Mitteleuropäische Sommerzeit] read: git ["--git-dir=C:\\Users\\Name\\Pictures\\.git","--work-tree=C:\\Users\\Name\\Pictures","log","refs/heads/git-annex..ad7115c6108170dc54095f74632db339dbca4cdd","--oneline","-n1"]
+	[2014-06-29 12:50:03 Mitteleuropäische Sommerzeit] call: git ["--git-dir=C:\\Users\\Name\\Pictures\\.git","--work-tree=C:\\Users\\Name\\Pictures","branch","-f","synced/master"]
+	[2014-06-29 12:50:03 Mitteleuropäische Sommerzeit] call: git ["--git-dir=C:\\Users\\Name\\Pictures\\.git","--work-tree=C:\\Users\\Name\\Pictures","branch","-f","master"]
+	[2014-06-29 12:50:03 Mitteleuropäische Sommerzeit] call: git ["--git-dir=C:\\Users\\Name\\Pictures\\.git","--work-tree=C:\\Users\\Name\\Pictures","show-ref","--verify","-q","refs/remotes/HomeServerLocal/synced/master"]
+	[2014-06-29 12:50:03 Mitteleuropäische Sommerzeit] read: git ["--git-dir=C:\\Users\\Name\\Pictures\\.git","--work-tree=C:\\Users\\Name\\Pictures","log","refs/remotes/HomeServerLocal/synced/master..refs/heads/synced/master","--oneline","-n1"]
+	[2014-06-29 12:50:03 Mitteleuropäische Sommerzeit] call: git ["--git-dir=C:\\Users\\Name\\Pictures\\.git","--work-tree=C:\\Users\\Name\\Pictures","show-ref","--verify","-q","refs/remotes/HomeServerLocal/git-annex"]
+	[2014-06-29 12:50:03 Mitteleuropäische Sommerzeit] read: git ["--git-dir=C:\\Users\\Name\\Pictures\\.git","--work-tree=C:\\Users\\Name\\Pictures","log","refs/remotes/HomeServerLocal/git-annex..git-annex","--oneline","-n1"]
+	push HomeServerLocal
+	[2014-06-29 12:50:03 Mitteleuropäische Sommerzeit] call: git ["--git-dir=C:\\Users\\Name\\Pictures\\.git","--work-tree=C:\\Users\\Name\\Pictures","push","HomeServerLocal","+git-annex:synced/git-annex","annex/direct/master:synced/master"]
+	Everything up-to-date
+	[2014-06-29 12:50:04 Mitteleuropäische Sommerzeit] read: git ["--git-dir=C:\\Users\\Name\\Pictures\\.git","--work-tree=C:\\Users\\Name\\Pictures","push","HomeServerLocal","master"]
+	ok
+
+Any idea? Thanks in advance...
diff --git a/doc/forum/fatal__58___Out_of_memory__63___mmap_failed__58___No_such_file_or_directory/comment_1_d98a155fa01d10ecff9058d79290156d._comment b/doc/forum/fatal__58___Out_of_memory__63___mmap_failed__58___No_such_file_or_directory/comment_1_d98a155fa01d10ecff9058d79290156d._comment
new file mode 100644
index 0000000000..989ab1a6a4
--- /dev/null
+++ b/doc/forum/fatal__58___Out_of_memory__63___mmap_failed__58___No_such_file_or_directory/comment_1_d98a155fa01d10ecff9058d79290156d._comment
@@ -0,0 +1,13 @@
+[[!comment format=mdwn
+ username="Idhup"
+ ip="91.58.248.157"
+ subject="System"
+ date="2014-06-29T19:26:42Z"
+ content="""
+Forgot to tell: This is a Win7 x64 system, running git 1.9.2.msysgit.0 and
+
+git-annex version: 5.20140517-gee56d21
+build flags: Assistant Webapp Webapp-secure Pairing Testsuite S3 WebDAV DNS Feeds Quvi TDFA CryptoHash
+key/value backends: SHA256E SHA1E SHA512E SHA224E SHA384E SKEIN256E SKEIN512E SHA256 SHA1 SHA512 SHA224 SHA384 SKEIN256 SKEIN512 WORM URL
+remote types: git gcrypt S3 bup directory rsync web webdav tahoe glacier ddar hook external
+"""]]
diff --git a/doc/forum/fatal__58___Out_of_memory__63___mmap_failed__58___No_such_file_or_directory/comment_2_3b9ea7a1254ac5b50a5ab59cd331ec3f._comment b/doc/forum/fatal__58___Out_of_memory__63___mmap_failed__58___No_such_file_or_directory/comment_2_3b9ea7a1254ac5b50a5ab59cd331ec3f._comment
new file mode 100644
index 0000000000..2a575e0599
--- /dev/null
+++ b/doc/forum/fatal__58___Out_of_memory__63___mmap_failed__58___No_such_file_or_directory/comment_2_3b9ea7a1254ac5b50a5ab59cd331ec3f._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="http://joeyh.name/"
+ ip="209.250.56.55"
+ subject="comment 2"
+ date="2014-07-04T17:55:45Z"
+ content="""
+
+"""]]
diff --git a/doc/forum/fatal__58___Out_of_memory__63___mmap_failed__58___No_such_file_or_directory/comment_3_5ee300034819c5825c676cd7e3af659f._comment b/doc/forum/fatal__58___Out_of_memory__63___mmap_failed__58___No_such_file_or_directory/comment_3_5ee300034819c5825c676cd7e3af659f._comment
new file mode 100644
index 0000000000..55345537e8
--- /dev/null
+++ b/doc/forum/fatal__58___Out_of_memory__63___mmap_failed__58___No_such_file_or_directory/comment_3_5ee300034819c5825c676cd7e3af659f._comment
@@ -0,0 +1,12 @@
+[[!comment format=mdwn
+ username="Idhup"
+ ip="91.58.248.157"
+ subject="Didn't work"
+ date="2014-07-06T17:46:01Z"
+ content="""
+Repacking in cygwin towards a smaller packsize did not help. I shrinked it down to 2 MB per packfile, I still got the errors.
+
+When checking the debug output, you can see that git crashes when adding some large-ish .mov files. I'm not even sure if git's supposed to add them or if something went wrong with Windows/DirectMode. I executed git reset to remove pending additions that weren't supposed to be there, but to no avail.
+
+Any ideas on how to fix that?
+"""]]
diff --git a/doc/forum/fatal__58___Out_of_memory__63___mmap_failed__58___No_such_file_or_directory/comment_4_cf7f5c91d3c15f72d2a714b7362c1197._comment b/doc/forum/fatal__58___Out_of_memory__63___mmap_failed__58___No_such_file_or_directory/comment_4_cf7f5c91d3c15f72d2a714b7362c1197._comment
new file mode 100644
index 0000000000..d0b89c59e5
--- /dev/null
+++ b/doc/forum/fatal__58___Out_of_memory__63___mmap_failed__58___No_such_file_or_directory/comment_4_cf7f5c91d3c15f72d2a714b7362c1197._comment
@@ -0,0 +1,30 @@
+[[!comment format=mdwn
+ username="http://joeyh.name/"
+ ip="209.250.56.2"
+ subject="comment 4"
+ date="2014-07-10T20:16:28Z"
+ content="""
+[2014-06-29 12:49:57 Mitteleuropäische Sommerzeit] call: git [\"--git-dir=C:\\Users\\Name\\Pictures\\.git\",\"--work-tree=C:\\Users\\Name\\Pictures\",\"add\",\"-f\",\"Hochzeit von NameB und NameC/2013-08-17_MVI_9701.MOV\"]
+
+When does git-annex in direct mode run \"git add -f\"? In stageDirect, when a file has been modified that is not an annexed file, but already has been committed  directly to git.
+
+This certainly points in the direction of what the problem is with the repository. I think you need to take a look at `git log --stat \"Hochzeit von NameB und NameC/2013-08-18_MVI_0583.MOV\"` and see if it has been committed to git as a symlink or if it is indeed being stored as a Bin file in git. 
+
+Here's how the log would look for a regular git-annexed symlink:
+
+
    + $somefile | 1 +
+ 1 file changed, 1 insertion(+)
+
    
+
+And here for a binary file stored in git:
+
+
    + $somefile | Bin 0 -> 1016920 bytes
+ 1 file changed, 0 insertions(+), 0 deletions(-)
+
    
+
+If you find the latter in the log, then the author and commit message of the commit adding it would be interesting.
+
+Hypothesis: Perhaps this repository started off on a Linux or OSX system, and you were using a git-annex older than 5.20131118, when the direct mode guard was added. You might have added this file back then and accidentally committed it directly to git.
+"""]]
diff --git a/doc/forum/files_being_dropped_undesirably.mdwn b/doc/forum/files_being_dropped_undesirably.mdwn
new file mode 100644
index 0000000000..2a4d1bf7c9
--- /dev/null
+++ b/doc/forum/files_being_dropped_undesirably.mdwn
@@ -0,0 +1,47 @@
+I currently am using 3 repositories for my personal annex, and on each of them I'm running the assistant (they are normal git annex repositories). However all my files are migrating to my work desktop. My other two repositories seem to keep dropping them.
+
+Last night on my laptop I did a "git annex get *" to pull _all_ the files onto it. I saw in the .git/annex/daemon.log that each file was being dropped as soon as it was gotten. The output of "git annex get" showed files being transfered across, and the .git/annex/daemon.log showed files being dropped straight away. Currently I'd like to keep all my files on all my repositories (and perhaps later I'll revise that).
+
+Could someone please help me understand why annex is dropping my files, and what I could do to keep them on all my repositories?
+
+Here is the output of a get for a single file:
+
+    ~/Documents/personal-annex $ git annex get 2014/09/15/IMG_1123.JPG
+    get 2014/09/15/IMG_1123.JPG (from pea15.local_Documents_annexpersonal...)
+    SHA256E-s1841221--deeaa13935907ad606f941397bb57432c1eccfd5c361b8c16d2b19bfbe8437a6.JPG
+          1,841,221 100%   11.40MB/s    0:00:00 (xfr#1, to-chk=0/1)
+    ok
+    (Recording state in git...)
+
+
+Here is the corresponding daemon.log output:
+
+    [2014-10-09 09:11:34 AEDT] Committer: Committing changes to git
+    [2ok
+    (Recording state in git...)
+    (Recording state in git...)
+    (Recording state in git...)
+    drop 2014/09/15/IMG_1123.JPG 01(checking pea15.local_Documents_annexpersonal...) 4-10-09 09:11:34 AEDT] Pusher: Syncing with pea15.local_Documents_annexpersonal 
+    [2014-10-09 09:11:35 AEDT] Committer: Committing changes to git
+    To ssh://geoffc@git-annex-pea-15-geoffc_Documents.2Fannex.2Dpersonal/~/Documents/annex-personal/
+       04742c0..d1a5a36  git-annex -> synced/git-annex
+    [2014-10-09 09:11:38 AEDT] Pusher: Syncing with pea15.local_Documents_annexpersonal 
+    Everything up-to-date
+
+
+And here is a snippet from my .git/config:
+
+    [annex]
+            uuid = 57c4e6d1-0c6b-4c49-a235-4119d3864c14
+            version = 5
+            direct = true
+            #diskreserve = 2 gigabyte
+            autoupgrade = ask
+            debug = false
+            expireunused = false
+            autocommit = true
+    [remote "pea15.local_Documents_annexpersonal"]
+            url = ssh://geoffc@git-annex-pea-15-geoffc_Documents.2Fannex.2Dpersonal/~/Documents/annex-personal/
+            fetch = +refs/heads/*:refs/remotes/pea15.local_Documents_annexpersonal/*
+            annex-uuid = 2ef6bbfe-662f-48ba-aa52-8e2f82bcfb15
+            annex-cost = 175.0
diff --git a/doc/forum/files_being_dropped_undesirably/comment_1_d03f8ed7d3f3da58612bf238c1790fb4._comment b/doc/forum/files_being_dropped_undesirably/comment_1_d03f8ed7d3f3da58612bf238c1790fb4._comment
new file mode 100644
index 0000000000..887b12ac96
--- /dev/null
+++ b/doc/forum/files_being_dropped_undesirably/comment_1_d03f8ed7d3f3da58612bf238c1790fb4._comment
@@ -0,0 +1,11 @@
+[[!comment format=mdwn
+ username="Bram"
+ ip="81.20.68.186"
+ subject="Group and wanted expression"
+ date="2014-10-09T12:09:58Z"
+ content="""
+What are the group and wanted expressions for your repository?
+
+    git annex group .
+    git annex wanted .
+"""]]
diff --git a/doc/forum/files_being_dropped_undesirably/comment_2_7d885abebfec789348639494b1bb1829._comment b/doc/forum/files_being_dropped_undesirably/comment_2_7d885abebfec789348639494b1bb1829._comment
new file mode 100644
index 0000000000..7bdb9243ce
--- /dev/null
+++ b/doc/forum/files_being_dropped_undesirably/comment_2_7d885abebfec789348639494b1bb1829._comment
@@ -0,0 +1,24 @@
+[[!comment format=mdwn
+ username="go8ose"
+ ip="203.26.118.202"
+ subject="Group and wanted expression"
+ date="2014-10-09T22:19:01Z"
+ content="""
+Here is the group and wanted output on my laptop (that is dropping all the files):
+
+    ~/Documents/personal-annex $ git annex group .
+    unwanted
+    ~/Documents/personal-annex $ git annex wanted .
+    standard
+
+Here is the output on my desktop (that seems to be keeping all the files):
+
+    ~/Documents/annex-personal $ git annex group .
+    
+    ~/Documents/annex-personal $ git annex wanted .
+    
+
+
+
+Reading the manpage suggests I might want to change this. However I haven't seen in the man page a list of the standard predefined groups, nor an explanation of how each predefined group behaves. Is that documented somewhere else?
+"""]]
diff --git a/doc/forum/files_being_dropped_undesirably/comment_3_7c70b58f89408304055eefb1b166ef2e._comment b/doc/forum/files_being_dropped_undesirably/comment_3_7c70b58f89408304055eefb1b166ef2e._comment
new file mode 100644
index 0000000000..00d5115962
--- /dev/null
+++ b/doc/forum/files_being_dropped_undesirably/comment_3_7c70b58f89408304055eefb1b166ef2e._comment
@@ -0,0 +1,10 @@
+[[!comment format=mdwn
+ username="go8ose"
+ ip="203.26.118.202"
+ subject="comment 3"
+ date="2014-10-09T22:31:49Z"
+ content="""
+To answer my question about documentation on the 'groups' and 'wanted' features, I've found [[preferred_content]] and [[preferred_content/standard_groups]].
+
+In my case I've now setup my laptop to use group 'client' for the '.' repository. Similarly I've set 'client' on my desktop. I found my home server was already set to 'backup'. I still don't understand \"git annex\" well enough to know what's going on. I thought with these settings that running \"git annex get --auto\" on my laptop would have fetched all the files, but it didn't. A \"git annex get *\" is fetching all the files though, so I'm achieving what I want.
+"""]]
diff --git a/doc/forum/files_got_replaced_by_broken_symblinks__44___how_to_get_them_back__63__.mdwn b/doc/forum/files_got_replaced_by_broken_symblinks__44___how_to_get_them_back__63__.mdwn
new file mode 100644
index 0000000000..5a90d15355
--- /dev/null
+++ b/doc/forum/files_got_replaced_by_broken_symblinks__44___how_to_get_them_back__63__.mdwn
@@ -0,0 +1,23 @@
+Basically, this is how it looks now (this is part of my music collection, but there are other directories with the same symptoms):
+
+    simon@simon-desktop /run/media/simon/TOSHIBA EXT/Stuff/Music/Amy Kuney - Breaking Bad Habits EP (2006) ±annex/direct/master » ls -l
+    insgesamt 4
+    -rwxrwxrwx 1 simon simon 196  7. Dez 13:37 01 So Help Me God.mp3
+    -rwxrwxrwx 1 simon simon 196  7. Dez 13:37 02 Sore Loser.mp3
+    -rwxrwxrwx 1 simon simon 196  7. Dez 13:37 03 How The Wind Must Feel.mp3
+    -rwxrwxrwx 1 simon simon 196  7. Dez 13:37 04 Sold Out.mp3
+    -rwxrwxrwx 1 simon simon 196  7. Dez 13:37 05 In The Dark.mp3
+    -rwxrwxrwx 1 simon simon 196  7. Dez 13:37 06 Breaking Bad Habits.mp3
+    -rwxrwxrwx 1 simon simon 196  7. Dez 13:37 AmyKuneyEP.jpg
+    simon@simon-desktop /run/media/simon/TOSHIBA EXT/Stuff/Music/Amy Kuney - Breaking Bad Habits EP (2006) ±annex/direct/master » cat 01\ So\ Help\ Me\ God.mp3 
+    ../../.git/annex/objects/M5/Pq/SHA256E-s213--96db10197b716998d35576c39fc2468bd197b5a4ba2bec14a78454b278d96498.mp3/SHA256E-s213--96db10197b716998d35576c39fc2468bd197b5a4ba2bec14a78454b278d96498.mp3%
+    simon@simon-desktop /run/media/simon/TOSHIBA EXT/Stuff/Music/Amy Kuney - Breaking Bad Habits EP (2006) ±annex/direct/master » ls ../../.git/annex/objects/M5
+    ls: Zugriff auf ../../.git/annex/objects/M5 nicht möglich: Datei oder Verzeichnis nicht gefunden
+    simon@simon-desktop /run/media/simon/TOSHIBA EXT/Stuff/Music/Amy Kuney - Breaking Bad Habits EP (2006) ±annex/direct/master » find ../../.git/annex | grep 96db10197b716998d3557                               2 ↵
+    ../../.git/annex/objects/a5b/e78/SHA256E-s213--96db10197b716998d35576c39fc2468bd197b5a4ba2bec14a78454b278d96498.mp3
+    ../../.git/annex/objects/a5b/e78/SHA256E-s213--96db10197b716998d35576c39fc2468bd197b5a4ba2bec14a78454b278d96498.mp3/SHA256E-s213--96db10197b716998d35576c39fc2468bd197b5a4ba2bec14a78454b278d96498.mp3.map
+    simon@simon-desktop /run/media/simon/TOSHIBA EXT/Stuff/Music/Amy Kuney - Breaking Bad Habits EP (2006) ±annex/direct/master » git annex info  
+    repository mode: direct
+
+So, I only have a bunch of text files containing file paths. The actual files are in .git/annex/objects somewhere, but not exactly in the path specified in those text files. I'm not sure how it ended up that way, I'm in direct mode after all.
+For starters, I would like to know how I can repair this mess.
diff --git a/doc/forum/files_got_replaced_by_broken_symblinks__44___how_to_get_them_back__63__/comment_1_4d07f87b433228fdaebce2805fc64492._comment b/doc/forum/files_got_replaced_by_broken_symblinks__44___how_to_get_them_back__63__/comment_1_4d07f87b433228fdaebce2805fc64492._comment
new file mode 100644
index 0000000000..d8046026b9
--- /dev/null
+++ b/doc/forum/files_got_replaced_by_broken_symblinks__44___how_to_get_them_back__63__/comment_1_4d07f87b433228fdaebce2805fc64492._comment
@@ -0,0 +1,17 @@
+[[!comment format=mdwn
+ username="joey"
+ subject="""comment 1"""
+ date="2016-01-15T19:01:22Z"
+ content="""
+git-annex uses broken symlinks for files in the git repository whose
+content is not currently present.
+
+On filesystems not supporting symlinks, git represents them by putting the
+symlink target in a regular file. This seems to be what you have here.
+
+Your `find` command shows that the content is *not* located in
+.git/annex/objects. It only found a .map file, not the content file.
+
+So, `git annex get` seems like the most likely thing that will get the
+content of these files from wherever it's stored.
+"""]]
diff --git a/doc/forum/files_got_replaced_by_broken_symblinks__44___how_to_get_them_back__63__/comment_2_97d24db6027270b9f2d759e98d5f428f._comment b/doc/forum/files_got_replaced_by_broken_symblinks__44___how_to_get_them_back__63__/comment_2_97d24db6027270b9f2d759e98d5f428f._comment
new file mode 100644
index 0000000000..0990352b97
--- /dev/null
+++ b/doc/forum/files_got_replaced_by_broken_symblinks__44___how_to_get_them_back__63__/comment_2_97d24db6027270b9f2d759e98d5f428f._comment
@@ -0,0 +1,38 @@
+[[!comment format=mdwn
+ username="simon.parzer@f837bbade0d93f560dc574b04e835b7875c4026f"
+ nickname="simon.parzer"
+ subject="comment 2"
+ date="2016-01-15T19:37:57Z"
+ content="""
+Well here is where it gets weird. I used \"git annex get\" to retrieve the file and it did something.
+
+    simon@simon-desktop /run/media/simon/TOSHIBA EXT/Stuff/Music/Amy Kuney - Breaking Bad Habits EP (2006) ±annex/direct/master » git annex get 01\ So\ Help\ Me\ God.mp3
+    get 01 So Help Me God.mp3 (from desktop-linux...) 
+    01 So Help Me God.mp3
+                 213 100%    0.00kB/s    0:00:00 (xfr#1, to-chk=0/1)
+    (checksum...) ok
+    (recording state in git...)
+
+    simon@simon-desktop /run/media/simon/TOSHIBA EXT/Stuff/Music/Amy Kuney - Breaking Bad Habits EP (2006) ±annex/direct/master » ls -l
+    insgesamt 4
+    -rwxrwxrwx 1 simon simon 213 15. Jän 20:30 01 So Help Me God.mp3
+    ....
+
+    simon@simon-desktop /run/media/simon/TOSHIBA EXT/Stuff/Music/Amy Kuney - Breaking Bad Habits EP (2006) ±annex/direct/master » cat 01\ So\ Help\ Me\ God.mp3 
+    ..\..\..\..\..\.git\annex\objects\K8\xx\SHA256E-s3460908--ad74d891fd20ef771f933d3fea6247eb640b2d8c0ca4848065735992a07a4eb7.mp3\SHA256E-s3460908--ad74d891fd20ef771f933d3fea6247eb640b2d8c0ca4848065735992a07a4eb7.mp3%                          
+
+Yet the file still is only a fake symlink.
+On the other computer (desktop-linux) where it got the file from it's also only a symlink, but the actual file is somewhere in annex/objects
+
+    simon@simon-desktop /d/Stuff/Music/Amy Kuney - Breaking Bad Habits EP (2006) ±annex/direct/master » git annex whereis 01\ So\ Help\ Me\ God.mp3 
+    whereis 01 So Help Me God.mp3 (1 copy) 
+      	5e0f083c-d81f-4a4b-b1a5-35b45e057dea -- desktop pc [here]
+    ok
+    simon@simon-desktop /d/Stuff/Music/Amy Kuney - Breaking Bad Habits EP (2006) ±annex/direct/master » cat 01\ So\ Help\ Me\ God.mp3 
+    ..\..\..\..\..\.git\annex\objects\K8\xx\SHA256E-s3460908--ad74d891fd20ef771f933d3fea6247eb640b2d8c0ca4848065735992a07a4eb7.mp3\SHA256E-s3460908--ad74d891fd20ef771f933d3fea6247eb640b2d8c0ca4848065735992a07a4eb7.mp3%
+    simon@simon-desktop /d/Stuff/Music/Amy Kuney - Breaking Bad Habits EP (2006) ±annex/direct/master » find /d/Stuff/.git/annex/objects | grep a6247eb640b2d8c0ca48480
+    /d/Stuff/.git/annex/objects/48d/eb4/SHA256E-s3460908--ad74d891fd20ef771f933d3fea6247eb640b2d8c0ca4848065735992a07a4eb7.mp3
+    /d/Stuff/.git/annex/objects/48d/eb4/SHA256E-s3460908--ad74d891fd20ef771f933d3fea6247eb640b2d8c0ca4848065735992a07a4eb7.mp3/SHA256E-s3460908--ad74d891fd20ef771f933d3fea6247eb640b2d8c0ca4848065735992a07a4eb7.mp3.map
+
+
+"""]]
diff --git a/doc/forum/files_got_replaced_by_broken_symblinks__44___how_to_get_them_back__63__/comment_3_e61107dd83efecc3266217acad9c943b._comment b/doc/forum/files_got_replaced_by_broken_symblinks__44___how_to_get_them_back__63__/comment_3_e61107dd83efecc3266217acad9c943b._comment
new file mode 100644
index 0000000000..dfc6b1fbd6
--- /dev/null
+++ b/doc/forum/files_got_replaced_by_broken_symblinks__44___how_to_get_them_back__63__/comment_3_e61107dd83efecc3266217acad9c943b._comment
@@ -0,0 +1,9 @@
+[[!comment format=mdwn
+ username="joey"
+ subject="""comment 3"""
+ date="2016-01-20T19:06:00Z"
+ content="""
+Try running `git-annex fsck` on the file; this may fix it up to make the
+content actually available. It might also output something useful to debug
+what went wrong.
+"""]]
diff --git a/doc/forum/files_got_replaced_by_broken_symblinks__44___how_to_get_them_back__63__/comment_4_362370f02ea77addf9efaed9cb4b433a._comment b/doc/forum/files_got_replaced_by_broken_symblinks__44___how_to_get_them_back__63__/comment_4_362370f02ea77addf9efaed9cb4b433a._comment
new file mode 100644
index 0000000000..c344399555
--- /dev/null
+++ b/doc/forum/files_got_replaced_by_broken_symblinks__44___how_to_get_them_back__63__/comment_4_362370f02ea77addf9efaed9cb4b433a._comment
@@ -0,0 +1,13 @@
+[[!comment format=mdwn
+ username="simon.parzer@f837bbade0d93f560dc574b04e835b7875c4026f"
+ nickname="simon.parzer"
+ subject="comment 4"
+ date="2016-01-21T12:54:57Z"
+ content="""
+I already tried ```git annex fsck```, but it does not report any errors and the files remain unchanged. Here is the output.
+
+    simon@simon-desktop /d/Stuff/Music/Amy Kuney - Breaking Bad Habits EP (2006) ±annex/direct/master » git annex fsck 01\ So\ Help\ Me\ God.mp3
+    fsck 01 So Help Me God.mp3 (checksum...) ok
+    (recording state in git...)
+
+"""]]
diff --git a/doc/forum/files_got_replaced_by_broken_symblinks__44___how_to_get_them_back__63__/comment_5_4333c2f189cea3430104ece6769143bc._comment b/doc/forum/files_got_replaced_by_broken_symblinks__44___how_to_get_them_back__63__/comment_5_4333c2f189cea3430104ece6769143bc._comment
new file mode 100644
index 0000000000..5c4c91fdfe
--- /dev/null
+++ b/doc/forum/files_got_replaced_by_broken_symblinks__44___how_to_get_them_back__63__/comment_5_4333c2f189cea3430104ece6769143bc._comment
@@ -0,0 +1,11 @@
+[[!comment format=mdwn
+ username="joey"
+ subject="""comment 5"""
+ date="2016-01-29T17:49:47Z"
+ content="""
+Does `git annex info` show this repository as being in direct mode, or
+indirect mode? (It's the first line of its output.)
+
+I'm thinking you might somehow have a repository that is not really in
+direct mode despite the current branch being annex/direct/master.
+"""]]
diff --git a/doc/forum/files_got_replaced_by_broken_symblinks__44___how_to_get_them_back__63__/comment_6_8c4c2e8dc4706ca824cbbc9b2ed62be4._comment b/doc/forum/files_got_replaced_by_broken_symblinks__44___how_to_get_them_back__63__/comment_6_8c4c2e8dc4706ca824cbbc9b2ed62be4._comment
new file mode 100644
index 0000000000..4df27e9999
--- /dev/null
+++ b/doc/forum/files_got_replaced_by_broken_symblinks__44___how_to_get_them_back__63__/comment_6_8c4c2e8dc4706ca824cbbc9b2ed62be4._comment
@@ -0,0 +1,12 @@
+[[!comment format=mdwn
+ username="simon.parzer@f837bbade0d93f560dc574b04e835b7875c4026f"
+ nickname="simon.parzer"
+ subject="comment 6"
+ date="2016-05-05T11:26:03Z"
+ content="""
+To sort of follow up on this after 3 months, ```git annex info``` displays ```repository mode: direct```
+
+I have given up on git annex in the meantime, it is a nightmare to sync between different operating systems/file systems.
+What might be \"wrong\" with my setup is that I have at least two machines that are dual-boot. So I'm using the same folder both in Windows and Linux. I also have an external HD with an annex repository that I plug in to different machines, and it apparently causes problems.
+Now I have to write a python script to recover my lost files from the bottomless abyss that is the .git/annex folder.
+"""]]
diff --git a/doc/forum/files_got_replaced_by_broken_symblinks__44___how_to_get_them_back__63__/comment_7_24b09f1263f6bc6b219085c17ced915c._comment b/doc/forum/files_got_replaced_by_broken_symblinks__44___how_to_get_them_back__63__/comment_7_24b09f1263f6bc6b219085c17ced915c._comment
new file mode 100644
index 0000000000..c648d125da
--- /dev/null
+++ b/doc/forum/files_got_replaced_by_broken_symblinks__44___how_to_get_them_back__63__/comment_7_24b09f1263f6bc6b219085c17ced915c._comment
@@ -0,0 +1,49 @@
+[[!comment format=mdwn
+ username="simon.parzer@f837bbade0d93f560dc574b04e835b7875c4026f"
+ nickname="simon.parzer"
+ subject="Python script to recover git annex' lost files"
+ date="2016-05-05T12:01:46Z"
+ content="""
+Maybe someone else needs this at some point :|
+
+Use it at your own risk of course.
+
+    #!python3
+    
+    import os
+    import re
+    from shutil import copyfile
+    
+    targets = {}
+    
+    for path, dirs, files in os.walk('.'):
+        for f in files:
+            fp = os.path.join(path, f)
+            if fp.endswith('.py'): continue
+            fp_size = os.stat(fp).st_size
+            if fp_size < 72 or fp_size > 256: continue
+            with open(fp, 'r', encoding='latin1') as stream:
+                l = stream.readline()
+                hashmatch = re.match('.*SHA256E\-\w+\-\-(\w{16})', l)
+                if hashmatch:
+                    targets[hashmatch.group(1)] = os.path.abspath(fp)
+    
+    #print(targets)
+    
+    target = os.path.abspath('.')
+    while not os.path.exists('.git/annex/objects'):
+        os.chdir('..')
+    
+    for path, dirs, files in os.walk('.git/annex/objects'):
+        for f in files:
+            fp = os.path.join(path, f)
+            fp_size = os.stat(fp).st_size
+            if fp_size > 512:
+                hashmatch = re.match('.*SHA256E\-\w+\-\-(\w{16})', fp)
+                if hashmatch:
+                    hash = hashmatch.group(1)
+                    if hash in targets:
+                        print(fp, '->', targets[hash])
+                        copyfile(fp, targets[hash])
+
+"""]]
diff --git a/doc/forum/files_only_visible_in_.git_annex.mdwn b/doc/forum/files_only_visible_in_.git_annex.mdwn
new file mode 100644
index 0000000000..e011ea5831
--- /dev/null
+++ b/doc/forum/files_only_visible_in_.git_annex.mdwn
@@ -0,0 +1,15 @@
+Dear all, 
+
+I'm a beginner in using git annex and got the following strange situation:
+
+On my PC I have stored all the data and put them to git annex (don' know the standard nomenclature used here, but I guess its called setting up a repository)
+
+On my Laptop I have now also git-annex and can have access to my stored data by git-annex get (so far so good thats git-annex)
+
+Now comes the strange thing: 
+On my laptop, I can see all the links to my data (where it shows me a path like ../../.git/annex/objects/...) but at my PC, where all the data should have been stored, the folder is empty. When I follow the link which is shown on the laptop, on my PC, I get access to the data. Some somehow, the data is still on my PC but not stored in its original folders but in .git/annex/objects/...
+
+How can I get back my original folder from the .git/annex/objects on my PC?
+
+Thank you very much for your help
+Stefan
diff --git a/doc/forum/files_only_visible_in_.git_annex/comment_1_a967dc488d4fb0fd6079b0a1b087679f._comment b/doc/forum/files_only_visible_in_.git_annex/comment_1_a967dc488d4fb0fd6079b0a1b087679f._comment
new file mode 100644
index 0000000000..8bdcdb077c
--- /dev/null
+++ b/doc/forum/files_only_visible_in_.git_annex/comment_1_a967dc488d4fb0fd6079b0a1b087679f._comment
@@ -0,0 +1,9 @@
+[[!comment format=mdwn
+ username="cbaines"
+ subject="comment 1"
+ date="2015-06-17T17:20:26Z"
+ content="""
+Could you post the output of \"git annex info\" on both your laptop and desktop?
+
+Also, what operating system are you using on both device?
+"""]]
diff --git a/doc/forum/files_only_visible_in_.git_annex/comment_2_e024e26b943f49a5e63c8fd1cd90a420._comment b/doc/forum/files_only_visible_in_.git_annex/comment_2_e024e26b943f49a5e63c8fd1cd90a420._comment
new file mode 100644
index 0000000000..d4285eee3e
--- /dev/null
+++ b/doc/forum/files_only_visible_in_.git_annex/comment_2_e024e26b943f49a5e63c8fd1cd90a420._comment
@@ -0,0 +1,51 @@
+[[!comment format=mdwn
+ username="graviton31415@3f1790ceea563b1bc76c318c7bfd6d4e817ecc5d"
+ nickname="graviton31415"
+ subject="comment 2"
+ date="2015-06-18T05:08:47Z"
+ content="""
+on my PC (where the data should be): git annex info gives:
+
+repository mode: indirect
+trusted repositories: 0
+semitrusted repositories: 2
+	00000000-0000-0000-0000-000000000001 -- web
+ 	e1664284-234f-4be1-8623-3fec7f406b57 -- here (Documents at myPC)
+untrusted repositories: 0
+transfers in progress: none
+available local disk space: 138.66 gigabytes (+1 megabyte reserved)
+local annex keys: 1616
+local annex size: 1.98 gigabytes
+annexed files in working tree: 125
+size of annexed files in working tree: 350.37 megabytes
+bloom filter size: 16 mebibytes (0.3% full)
+backend usage: 
+	SHA256E: 1741
+
+
+
+on my Laptop I get:
+
+repository mode: indirect
+trusted repositories: 0
+semitrusted repositories: 3
+	00000000-0000-0000-0000-000000000001 -- web
+ 	77106ecf-d208-4366-b70c-64def14c62bb -- here
+ 	e1664284-234f-4be1-8623-3fec7f406b57 -- origin (Documents at myPC)
+untrusted repositories: 0
+transfers in progress: none
+available local disk space: 207.67 gigabytes (+1 megabyte reserved)
+local annex keys: 7
+local annex size: 181.68 megabytes
+annexed files in working tree: 1567
+size of annexed files in working tree: 1.83 gigabytes
+bloom filter size: 16 mebibytes (0% full)
+backend usage: 
+	SHA256E: 1574
+
+
+On both systems, I'm using Ubuntu 14.04
+
+Thank you for your help,
+Stefan
+"""]]
diff --git a/doc/forum/files_only_visible_in_.git_annex/comment_3_2c3d5991056d5915225b2707ad7d25dd._comment b/doc/forum/files_only_visible_in_.git_annex/comment_3_2c3d5991056d5915225b2707ad7d25dd._comment
new file mode 100644
index 0000000000..ac6d42eebf
--- /dev/null
+++ b/doc/forum/files_only_visible_in_.git_annex/comment_3_2c3d5991056d5915225b2707ad7d25dd._comment
@@ -0,0 +1,14 @@
+[[!comment format=mdwn
+ username="joey"
+ subject="""comment 3"""
+ date="2015-07-06T18:58:17Z"
+ content="""
+Take a look at what git branch is checked out in your repository on the PC.
+It might have, eg, an old version of the "master" branch checked out,
+from before when the files were added.
+
+Checking out the right branch will get you your files visible.
+
+You might need to run `git annex sync` on the PC to pull in changes
+from the laptop.
+"""]]
diff --git a/doc/forum/files_vanishing.mdwn b/doc/forum/files_vanishing.mdwn
new file mode 100644
index 0000000000..f80adb51a2
--- /dev/null
+++ b/doc/forum/files_vanishing.mdwn
@@ -0,0 +1,12 @@
+Can anybody help me locate content that seems to have gone missing? 
+
+Setup is two repos, connected via ssh, both direct mode, both debian machines, repo A with 5.20150731-1 (testing), repo B with 5.20141125 (stable).
+
+Both repos where synchronising (in this case transfering files from A to B), when I was copying a few more hundred files (second set) into a folder in repo A. In the webapp I have seen the assistant pick them up and transfer them to repo B. This took an hour or so, after which I can still see the name of nine files from the second set (and "... and 626 more") in the status messages on the right as added and the same number of files as uploaded.
+
+Unfortunately, from the second set of files only the first two files can be found in A and B, none of the other few hundred.
+
+Running git-annex fsck in repo A did not return any errors.
+
+I would appreciate any pointers,
+Felix
diff --git a/doc/forum/files_vanishing/comment_1_eb69e88aee0005fca74dd540293e57ca._comment b/doc/forum/files_vanishing/comment_1_eb69e88aee0005fca74dd540293e57ca._comment
new file mode 100644
index 0000000000..8282b9e383
--- /dev/null
+++ b/doc/forum/files_vanishing/comment_1_eb69e88aee0005fca74dd540293e57ca._comment
@@ -0,0 +1,14 @@
+[[!comment format=mdwn
+ username="joey"
+ subject="""comment 1"""
+ date="2016-04-14T19:50:52Z"
+ content="""
+Suggest you run git log --stat to see if you can find a commit that deleted
+the missing files. Reverting the commit should then get the files back.
+
+There have been several bugs of this kind in direct mode. Using such an old
+git-annex from 2014 makes it more likely you'll hit an old bug like this.
+
+The new v6 repository mode should be much less likely to have this kind of
+bug.
+"""]]
diff --git a/doc/forum/finding_out_why_git_annex_import_failed.mdwn b/doc/forum/finding_out_why_git_annex_import_failed.mdwn
new file mode 100644
index 0000000000..664cf8542d
--- /dev/null
+++ b/doc/forum/finding_out_why_git_annex_import_failed.mdwn
@@ -0,0 +1,43 @@
+Hi,
+
+I keep importing data to my git annex, which gets bigger.
+
+Today while importing an old archive (with lots of files), git import failed with this unhelpful message:
+
+    $ git annex import /Users/public/Documents
+    import Documents/.DS_Store
+      not importing Documents/.DS_Store which is .gitignored (use --force to override)
+    failed
+    ...
+    import Documents/Recettes/recettes nicoises/20130511_201509.jpg ok
+    (recording state in git...)
+    git-annex: import: 10 failed
+    CallStack (from HasCallStack):
+      error, called at ./CmdLine/Action.hs:41:28 in main:CmdLine.Action
+
+It seems no data was lost: I could finish the import with "git commit" (and diff -R with the backup shows no difference). But I would like to know what did happen.
+And also what does the "10" mean in this context and how to get more information?
+
+Note that I might have a real problem on this system
+
+* many filenames have extend chars
+* I'm not sure the mac OS locale setup is OK
+* I'm quite low on disk space and inodes:
+
+      $ df
+      Filesystem    512-blocks       Used Available Capacity   iused   ifree %iused  Mounted on
+      /dev/disk1    1462744832 1441994016  20238816    99% 180313250 2529852   99%   /
+
+The version is
+
+    $ git annex version
+    git-annex version: 6.20161111
+    build flags: Assistant Webapp Pairing Testsuite S3(multipartupload)(storageclasses) WebDAV FsEvents XMPP ConcurrentOutput TorrentParser MagicMime Feeds Quvi
+    key/value backends: SHA256E SHA256 SHA512E SHA512 SHA224E SHA224 SHA384E SHA384 SHA3_256E SHA3_256 SHA3_512E SHA3_512 SHA3_224E SHA3_224 SHA3_384E SHA3_384 SKEIN256E SKEIN256 SKEIN512E SKEIN512 SHA1E SHA1 MD5E MD5 WORM URL
+    remote types: git gcrypt S3 bup directory rsync web bittorrent webdav tahoe glacier ddar hook external
+    local repository version: 5
+    supported repository versions: 3 5 6
+    upgrade supported from repository versions: 0 1 2 3 4 5
+    operating system: darwin x86_64
+
+Thank you!
diff --git a/doc/forum/finding_out_why_git_annex_import_failed/comment_1_276bbd2dab64e4bbc297e0e53e02215d._comment b/doc/forum/finding_out_why_git_annex_import_failed/comment_1_276bbd2dab64e4bbc297e0e53e02215d._comment
new file mode 100644
index 0000000000..b2bc2f403e
--- /dev/null
+++ b/doc/forum/finding_out_why_git_annex_import_failed/comment_1_276bbd2dab64e4bbc297e0e53e02215d._comment
@@ -0,0 +1,12 @@
+[[!comment format=mdwn
+ username="joey"
+ subject="""comment 1"""
+ date="2017-10-02T16:53:28Z"
+ content="""
+The `.DS_Store` file matched an ignore in your repositories `.gitignore`
+file, and so it was not imported.  Like the message says..
+
+The "10 failed" means that there were 10 things that failed to import. 
+The one you showed and 9 you didn't include in the paste. Probably all for
+similar reasons.
+"""]]
diff --git a/doc/forum/first-time_setup_git-annex.mdwn b/doc/forum/first-time_setup_git-annex.mdwn
new file mode 100644
index 0000000000..735d746212
--- /dev/null
+++ b/doc/forum/first-time_setup_git-annex.mdwn
@@ -0,0 +1,7 @@
+I have a git installation on my web server (web faction hosting). It allows me to run repositories from my hosting account. 
+
+What I want is to use git-annex as a dropbox replacement. Specifically, the central repository being my web server and then using local installations on my 2 mac laptops (primarily), access through browser on windows, and access on android as well (browser or some other method). 
+
+I'm looking at the install instructions, walkthrough, etc. What I'm initially not clear on is whether I need to install git-annex on my (linux centOS) webserver and any machine through which I'm accessing the repo? Seems like that would be the case of course, but I wanted to validate. 
+
+Am I going to run into issues that folks might already be able to warn me about? Anything I need to know? 
diff --git a/doc/forum/first-time_setup_git-annex/comment_1_a58d83ee3a7c2251d9a775847223f8ca._comment b/doc/forum/first-time_setup_git-annex/comment_1_a58d83ee3a7c2251d9a775847223f8ca._comment
new file mode 100644
index 0000000000..8b86d7b179
--- /dev/null
+++ b/doc/forum/first-time_setup_git-annex/comment_1_a58d83ee3a7c2251d9a775847223f8ca._comment
@@ -0,0 +1,13 @@
+[[!comment format=mdwn
+ username="http://joeyh.name/"
+ nickname="joey"
+ subject="comment 1"
+ date="2013-05-13T18:39:59Z"
+ content="""
+Sorry it took so long to get back to you.
+
+You do not necessarily have to have git-annex installed on your web server. But it will open up one of the nicest ways to use git-annex with a server, which is to put a bare git repository on the server, and let git-annex send the contents of large files to that repository. It's fine to install any old version of git-annex on the server, they're all forwards and backwards compatable.
+
+In any case, you need git-annex installed on any computers where you want to access the repository, certainly.
+
+"""]]
diff --git a/doc/forum/flickrannex_--_not_sure_I_get_it.mdwn b/doc/forum/flickrannex_--_not_sure_I_get_it.mdwn
new file mode 100644
index 0000000000..fa69d61587
--- /dev/null
+++ b/doc/forum/flickrannex_--_not_sure_I_get_it.mdwn
@@ -0,0 +1,7 @@
+flickrannex -- is it a way to upload your images to flickr by putting them in a git annex directory, and make them retrievable from it handily that way?
+
+Or is it a hack to exploit flickr's free storage, by allowing you to somehow wrap your files in images and uploading them?
+
+My python and knowledge of how git annex's hooks work is not good enough for me to be certain by reading the code.
+
+Thanks
diff --git a/doc/forum/flickrannex_--_not_sure_I_get_it/comment_1_57ea9f26760f970a70f09934d31a79b5._comment b/doc/forum/flickrannex_--_not_sure_I_get_it/comment_1_57ea9f26760f970a70f09934d31a79b5._comment
new file mode 100644
index 0000000000..3d5da7d374
--- /dev/null
+++ b/doc/forum/flickrannex_--_not_sure_I_get_it/comment_1_57ea9f26760f970a70f09934d31a79b5._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="http://joeyh.name/"
+ nickname="joey"
+ subject="comment 1"
+ date="2013-05-27T01:48:43Z"
+ content="""
+Little bit of both I think. Pictures uploaded by it are visible on flickr. You can configure a [[preferred_content]] expression so the flickr remote only wants *.jpg etc. Any other file is wrapped inside an image so flickr will accept it. It would probably be a good idea to configure git-annex to `untrust` your repository if using it this way.
+"""]]
diff --git a/doc/forum/flickrannex_--_not_sure_I_get_it/comment_2_ba93563b4ce1f6497a9f1d5e6eb0d1bb._comment b/doc/forum/flickrannex_--_not_sure_I_get_it/comment_2_ba93563b4ce1f6497a9f1d5e6eb0d1bb._comment
new file mode 100644
index 0000000000..cb9b65f6a0
--- /dev/null
+++ b/doc/forum/flickrannex_--_not_sure_I_get_it/comment_2_ba93563b4ce1f6497a9f1d5e6eb0d1bb._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="http://edheil.wordpress.com/"
+ ip="99.54.57.201"
+ subject="comment 2"
+ date="2013-05-27T01:57:25Z"
+ content="""
+Cool, thanks!  I ran into some issues setting it up, I'll take those to github (I hadn't been able to get as far as setting it up with an annex)
+"""]]
diff --git a/doc/forum/flickrannex_--_not_sure_I_get_it/comment_3_74f143965f48c89a3583acf1b6a7635a._comment b/doc/forum/flickrannex_--_not_sure_I_get_it/comment_3_74f143965f48c89a3583acf1b6a7635a._comment
new file mode 100644
index 0000000000..21fcde85d3
--- /dev/null
+++ b/doc/forum/flickrannex_--_not_sure_I_get_it/comment_3_74f143965f48c89a3583acf1b6a7635a._comment
@@ -0,0 +1,9 @@
+[[!comment format=mdwn
+ username="https://www.google.com/accounts/o8/id?id=AItOawmkBwMWvNKZZCge_YqobCSILPMeK6xbFw8"
+ nickname="develop"
+ subject="I noticed you stopped filing issues on github"
+ date="2013-06-02T10:38:04Z"
+ content="""
+Does that mean the hook is working properly for you properly now?
+
+"""]]
diff --git a/doc/forum/flickrannex_--_not_sure_I_get_it/comment_4_493bb86dedfa91ccc0c9be4045953ee4._comment b/doc/forum/flickrannex_--_not_sure_I_get_it/comment_4_493bb86dedfa91ccc0c9be4045953ee4._comment
new file mode 100644
index 0000000000..dd474ef593
--- /dev/null
+++ b/doc/forum/flickrannex_--_not_sure_I_get_it/comment_4_493bb86dedfa91ccc0c9be4045953ee4._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="http://edheil.wordpress.com/"
+ ip="99.54.57.201"
+ subject="comment 4"
+ date="2013-06-02T20:27:08Z"
+ content="""
+it is, thank you!  working perfectly.
+"""]]
diff --git a/doc/forum/flickrannex_--_not_sure_I_get_it/comment_5_2c410aa478b21c0e6eb0e4d54bc8c362._comment b/doc/forum/flickrannex_--_not_sure_I_get_it/comment_5_2c410aa478b21c0e6eb0e4d54bc8c362._comment
new file mode 100644
index 0000000000..f55a894a89
--- /dev/null
+++ b/doc/forum/flickrannex_--_not_sure_I_get_it/comment_5_2c410aa478b21c0e6eb0e4d54bc8c362._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="https://www.google.com/accounts/o8/id?id=AItOawmkBwMWvNKZZCge_YqobCSILPMeK6xbFw8"
+ nickname="develop"
+ subject="comment 5"
+ date="2013-06-02T20:29:48Z"
+ content="""
+Good :)
+"""]]
diff --git a/doc/forum/folder_size_question.mdwn b/doc/forum/folder_size_question.mdwn
new file mode 100644
index 0000000000..51d11ca8bc
--- /dev/null
+++ b/doc/forum/folder_size_question.mdwn
@@ -0,0 +1,3 @@
+For test purposes I copied a folder into my local repository to do some tests. 
+I now had a look at the original folder size and the size of the repository and compared it to the original folder and the repo is about 2x bigger. How come?
+=> http://screencast.com/t/Tvqe5P22Ux
diff --git a/doc/forum/folder_size_question/comment_1_782cbf836335d86ff29853c34f00fec3._comment b/doc/forum/folder_size_question/comment_1_782cbf836335d86ff29853c34f00fec3._comment
new file mode 100644
index 0000000000..36bb93df3e
--- /dev/null
+++ b/doc/forum/folder_size_question/comment_1_782cbf836335d86ff29853c34f00fec3._comment
@@ -0,0 +1,12 @@
+[[!comment format=mdwn
+ username="http://joeyh.name/"
+ ip="108.236.230.124"
+ subject="comment 1"
+ date="2014-05-16T17:02:29Z"
+ content="""
+I don't have bandwidth to watch screencasts, so I can only guess.
+
+My first guess would be that you used `git add` to check the files into a git repository. This will exactly double the disk space, since each file is now stored in .git/ as well as in the working tree. This is a problem that git-annex solves.
+
+If you'd like to provide more information (cut and paste of details is useful), I can perhaps make more informed guesses. Or you could use `du` so see which files are taking up more space than you expect.
+"""]]
diff --git a/doc/forum/folder_size_question/comment_2_391aa62e4d8c496a58be4707522d8edb._comment b/doc/forum/folder_size_question/comment_2_391aa62e4d8c496a58be4707522d8edb._comment
new file mode 100644
index 0000000000..f031deb818
--- /dev/null
+++ b/doc/forum/folder_size_question/comment_2_391aa62e4d8c496a58be4707522d8edb._comment
@@ -0,0 +1,22 @@
+[[!comment format=mdwn
+ username="https://www.google.com/accounts/o8/id?id=AItOawkAUMhKOSkh9JaBA6xst3XxQIIsDEq5Zd4"
+ nickname="Ovidiu"
+ subject="comment 2"
+ date="2014-05-16T17:15:59Z"
+ content="""
+I don't recall doing anything besides using the git annex assistant...
+
+That link is just a screen shot...
+
+anyway here is more info:
+
+du -h -s git-annex/*
+ 38M	git-annex/Finance
+8.0K	git-annex/archive
+
+BUT if I check the folder git-annex's properties it says: 
+
+45 980 806 bytes (80,4 MB on disk) for 8 420 items
+
+Is that normal?
+"""]]
diff --git a/doc/forum/folder_size_question/comment_3_1e850dbe36fafe0505b60dd2ce0bd5d7._comment b/doc/forum/folder_size_question/comment_3_1e850dbe36fafe0505b60dd2ce0bd5d7._comment
new file mode 100644
index 0000000000..978ffcf51e
--- /dev/null
+++ b/doc/forum/folder_size_question/comment_3_1e850dbe36fafe0505b60dd2ce0bd5d7._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="http://joeyh.name/"
+ ip="108.236.230.124"
+ subject="comment 3"
+ date="2014-05-16T19:07:10Z"
+ content="""
+Sounds to me like some GUI thing that is confused in some way. Which is, in my experience not unusual when it comes to GUI things and the actual size of files on disk. I'd trust `du`.
+"""]]
diff --git a/doc/forum/folder_size_question/comment_4_907f3b1cfe745abf94a6a8ba0dbd4396._comment b/doc/forum/folder_size_question/comment_4_907f3b1cfe745abf94a6a8ba0dbd4396._comment
new file mode 100644
index 0000000000..a32e9177a5
--- /dev/null
+++ b/doc/forum/folder_size_question/comment_4_907f3b1cfe745abf94a6a8ba0dbd4396._comment
@@ -0,0 +1,25 @@
+[[!comment format=mdwn
+ username="https://www.google.com/accounts/o8/id?id=AItOawkAUMhKOSkh9JaBA6xst3XxQIIsDEq5Zd4"
+ nickname="Ovidiu"
+ subject="comment 4"
+ date="2014-05-17T10:56:22Z"
+ content="""
+I thinks its all the small files. 
+
+    du -hs /Users/ovi/Sharing/git-annex/
+     77M	/Users/ovi/Sharing/git-annex/
+
+    du -hs /Users/ovi/Sharing/git-annex/Finance/
+     38M	/Users/ovi/Sharing/git-annex/Finance/
+
+    du -hs /Users/ovi/Sharing/git-annex/.git/
+     39M	/Users/ovi/Sharing/git-annex/.git/
+
+The GUI shows: 
+    46 136 859 bytes (80,6 MB on disk) for 8 002 items
+
+And the hidden .git folder shows:
+    7 373 616 bytes (41 MB on disk) for 7 625 items
+
+So apparently there are a lot of very small files, smaller than the blocksize hence the info is skewed? Anyway, I think its alright?
+"""]]
diff --git a/doc/forum/freebsd_git-annex_seems_to_be_in_an_upgrade_loop.mdwn b/doc/forum/freebsd_git-annex_seems_to_be_in_an_upgrade_loop.mdwn
new file mode 100644
index 0000000000..06ddda244d
--- /dev/null
+++ b/doc/forum/freebsd_git-annex_seems_to_be_in_an_upgrade_loop.mdwn
@@ -0,0 +1,16 @@
+I installed git-annex on my freenas running freebsd according to the instructions on this site. 
+every now and then it'll pop up (when using the webapp) telling me git-annex has been updated, do I want to restart. 
+if I say yes, the webapp dies, I have to manually stop git annex assistant --stop and after a few minutes of using the webapp it again tells me it has been updated...
+
+only saw this in the logs:
+UpgradeWatcher: A new version of git-annex has been installed. 
+
+
+Also, I connected 2 repos (on my NAS with freebsd and on my Macbook) but the NAS one can't connect to the MAcbook via SSH
+ssh: connect to host 192.168.178.33 port 22: Connection refused 
+
+How is this supposed to work? Obviously for now it shows its only syncing metadata with my MACbook: 
+syncing enabled (metadata only)
+
+Also, both repositories are bare, direct but when a file got pulled from the NAS to the MACbook it was a symlink and not the file itself, so what's wrong here?
+
diff --git a/doc/forum/freebsd_git-annex_seems_to_be_in_an_upgrade_loop/comment_1_ce8e7f58c3b81ec334436726be70bde5._comment b/doc/forum/freebsd_git-annex_seems_to_be_in_an_upgrade_loop/comment_1_ce8e7f58c3b81ec334436726be70bde5._comment
new file mode 100644
index 0000000000..d0548a5b3f
--- /dev/null
+++ b/doc/forum/freebsd_git-annex_seems_to_be_in_an_upgrade_loop/comment_1_ce8e7f58c3b81ec334436726be70bde5._comment
@@ -0,0 +1,14 @@
+[[!comment format=mdwn
+ username="ovidiu@66ace8a8d99ce938b0538ffa0f26d30db02a9626"
+ nickname="ovidiu"
+ subject="comment 1"
+ date="2015-10-19T07:09:53Z"
+ content="""
+Also found this in the logs:
+
+> swap_pager_getswapspace(16): failed
+> swap_pager_getswapspace(6): failed
+> pid 85783 (lsof), uid 1001, was killed: out of swap space
+
+And I didn't do anything except create 2 repos, connect them, no files inside, still testing...
+"""]]
diff --git a/doc/forum/freebsd_git-annex_seems_to_be_in_an_upgrade_loop/comment_2_43cdcdfff3aaa4482e6182d3dd77b61a._comment b/doc/forum/freebsd_git-annex_seems_to_be_in_an_upgrade_loop/comment_2_43cdcdfff3aaa4482e6182d3dd77b61a._comment
new file mode 100644
index 0000000000..370513ae58
--- /dev/null
+++ b/doc/forum/freebsd_git-annex_seems_to_be_in_an_upgrade_loop/comment_2_43cdcdfff3aaa4482e6182d3dd77b61a._comment
@@ -0,0 +1,17 @@
+[[!comment format=mdwn
+ username="joey"
+ subject="""comment 2"""
+ date="2015-10-19T17:42:25Z"
+ content="""
+I don't know of any freebsd build of git-annex that would use its
+self-upgrade facility. I don't provide one, and I don't see why the
+freebsd porters would enable that feature.
+
+I wonder if you're somehow using the linux standalone build on freebsd via
+some linux syscall compatability layer. Which is interesting if it works,
+but not suprising if it fails.. 
+
+In particular, it seems likely that the
+linux version of lsof wouldn't be very happy when run in a freebsd
+system, and perhaps that's why it's running out of memory.
+"""]]
diff --git a/doc/forum/fsck_gives_false_positives.mdwn b/doc/forum/fsck_gives_false_positives.mdwn
new file mode 100644
index 0000000000..2fae57c4ed
--- /dev/null
+++ b/doc/forum/fsck_gives_false_positives.mdwn
@@ -0,0 +1,6 @@
+Hi,
+
+I use git-annex 3.20120123 on a debian-testing amd-64 machine with software RAID6 and LVM2 on it. I needed to move the whole `/home` directory to another LV (the new LV is on encrypted PV, the old LV is encrypted and not properly aligned; I'm changing from encrypted `/home` only to encrypted everything except `/boot`), so I have used the `rsync -aAXH` from a `ro` mounted `/home` to a new LV mounted on `/mnt/home_2`. After the move was complete I run the `git annex fsck` on my (4TB of) data. The fsck finds some files bad, and moves them to the `..../bad` directory. So far so good, this is how it should be, right? But then- I have a file with sha1sum of all my files. So - I checked the 'bad' file against that. It was OK. Then I computed the SHA256 of the file - this is used by `git annex fsck`. It was OK, too. So how did it happen, that the file was marked as bad? Do I miss something here? Could it be related to the hardware (HDDs) and silent data corruption? Or is it the undesirable effect of rsync? Or maybe the fsck is at fault here?
+
+Any ideas?
+
diff --git a/doc/forum/fsck_gives_false_positives/comment_1_b91070218b9d5fb687eeee1f244237ad._comment b/doc/forum/fsck_gives_false_positives/comment_1_b91070218b9d5fb687eeee1f244237ad._comment
new file mode 100644
index 0000000000..c65eaf51d9
--- /dev/null
+++ b/doc/forum/fsck_gives_false_positives/comment_1_b91070218b9d5fb687eeee1f244237ad._comment
@@ -0,0 +1,15 @@
+[[!comment format=mdwn
+ username="http://joey.kitenet.net/"
+ nickname="joey"
+ subject="comment 1"
+ date="2012-02-14T16:58:33Z"
+ content="""
+Well, it should only move files to `.git/annex/bad/` if their filesize is wrong, or their checksum is wrong.
+
+You can try moving a file out of `.git/annex/bad/` and re-run fsck and see if it fails it again. (And if it does, paste in a log!) 
+
+To do that -- 
+Suppose you have a file `.git/annex/bad/SHA256-s33--5dc45521382f1c7974d9dbfcff1246370404b952` and you know that file `foobar` was supposed to have that content (you can check that `foobar` is a symlink to that SHA value). Then reinject it:
+
+`git annex reinject .git/annex/bad/SHA256-s33--5dc45521382f1c7974d9dbfcff1246370404b952 foobar`
+"""]]
diff --git a/doc/forum/fsck_gives_false_positives/comment_2_f51c53f3f6e6ee1ad463992657db5828._comment b/doc/forum/fsck_gives_false_positives/comment_2_f51c53f3f6e6ee1ad463992657db5828._comment
new file mode 100644
index 0000000000..1e0111e672
--- /dev/null
+++ b/doc/forum/fsck_gives_false_positives/comment_2_f51c53f3f6e6ee1ad463992657db5828._comment
@@ -0,0 +1,10 @@
+[[!comment format=mdwn
+ username="antymat"
+ ip="77.190.74.127"
+ subject="comment 2"
+ date="2012-02-14T22:48:37Z"
+ content="""
+Thanks, joey, but I still do not know, why the file that has been (and *is*) OK according to separate sha1 and sha256 checks, has been marked 'bad' by `fsck` and moved to `.git/annex/bad`. What could be a reason for that? Could have `rsync` caused it? I know too little about internal workings of `git-annex` to answer this question.
+
+But one thing I know for certain - the false positives should *not* happen, unless something *is* wrong with the file. Otherwise, if it is unreliable, if I have to check twice, it is useless. I might as well just keep checksums of all the files and do all checks by hand...
+"""]]
diff --git a/doc/forum/fsck_gives_false_positives/comment_3_692d6d4cd2f75a497e7d314041a768d2._comment b/doc/forum/fsck_gives_false_positives/comment_3_692d6d4cd2f75a497e7d314041a768d2._comment
new file mode 100644
index 0000000000..bd807f4048
--- /dev/null
+++ b/doc/forum/fsck_gives_false_positives/comment_3_692d6d4cd2f75a497e7d314041a768d2._comment
@@ -0,0 +1,10 @@
+[[!comment format=mdwn
+ username="http://joey.kitenet.net/"
+ nickname="joey"
+ subject="comment 3"
+ date="2012-02-14T22:57:29Z"
+ content="""
+All that git annex fsck does is checksum the file and move it away if the checksum fails. 
+
+If bad data was somehow read from the disk that one time, what you describe could occur. I cannot think of any other way it could happen.
+"""]]
diff --git a/doc/forum/fsck_gives_false_positives/comment_4_7ceb395bf8a2e6a041ccd8de63b1b6eb._comment b/doc/forum/fsck_gives_false_positives/comment_4_7ceb395bf8a2e6a041ccd8de63b1b6eb._comment
new file mode 100644
index 0000000000..038d41620c
--- /dev/null
+++ b/doc/forum/fsck_gives_false_positives/comment_4_7ceb395bf8a2e6a041ccd8de63b1b6eb._comment
@@ -0,0 +1,10 @@
+[[!comment format=mdwn
+ username="antymat"
+ ip="77.190.74.127"
+ subject="comment 4"
+ date="2012-02-15T07:13:12Z"
+ content="""
+OK, thanks. I was just wondering - since there are links in git(-annex), and a hard links too, that maybe the issue has been caused by `rsync`.
+
+I will keep my eye on that and run checks with my own checksum and `fsck` from time to time, and see what happens. I will post my results here, but the whole run (`fsck` or checksum) takes almost 2 days, so I will not do it too often... ;)
+"""]]
diff --git a/doc/forum/fsck_gives_false_positives/comment_5_86484a504c3bbcecd5876982b9c95688._comment b/doc/forum/fsck_gives_false_positives/comment_5_86484a504c3bbcecd5876982b9c95688._comment
new file mode 100644
index 0000000000..7f0fbf96a0
--- /dev/null
+++ b/doc/forum/fsck_gives_false_positives/comment_5_86484a504c3bbcecd5876982b9c95688._comment
@@ -0,0 +1,13 @@
+[[!comment format=mdwn
+ username="http://joey.kitenet.net/"
+ nickname="joey"
+ subject="comment 5"
+ date="2012-02-15T15:22:56Z"
+ content="""
+The symlinks are in the git repository. So if the rsync damanged one, git would see the change. And nothing that happens to the symlinks can affect fsck.
+
+git-annex does not use hard links at all.
+
+fsck corrects mangled file permissions. It is possible to screw up the permissions so badly that it cannot see the files at all (ie, chmod 000 on a file under .git/annex/objects), but then fsck will complain and give up, not move the files to bad.
+So I don't see how a botched rsync could result in fsck moving a file with correct content to bad.
+"""]]
diff --git a/doc/forum/fsck_gives_false_positives/comment_6_1d4fbbd212fa92967abda346323031f4._comment b/doc/forum/fsck_gives_false_positives/comment_6_1d4fbbd212fa92967abda346323031f4._comment
new file mode 100644
index 0000000000..294778cbfa
--- /dev/null
+++ b/doc/forum/fsck_gives_false_positives/comment_6_1d4fbbd212fa92967abda346323031f4._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="https://www.google.com/accounts/o8/id?id=AItOawmWQTrnPloMWiPFg8Y2Y5g-2IYe26D0KKw"
+ nickname="Jim"
+ subject="comment 6"
+ date="2012-07-07T16:18:06Z"
+ content="""
+It's also possible you got a one-time DRAM corruption.  You have to expect those to happen every so often unless you're using ECC memory.
+"""]]
diff --git a/doc/forum/ga-ncdu.mdwn b/doc/forum/ga-ncdu.mdwn
new file mode 100644
index 0000000000..d4b48c2097
--- /dev/null
+++ b/doc/forum/ga-ncdu.mdwn
@@ -0,0 +1,5 @@
+I've just written a quick script which provides an output which ncdu will use.
+
+It uses the size from the key for apparent size and the size of the file (if present) for the disk size so 'a' toggles between total size of the annex and the actually used disk space.
+
+Is there anyone else interested in having this? If so, I'll tidy it up and release it :)
diff --git a/doc/forum/ga-ncdu/comment_1_cde3f7bbd099b303bacdaa5e1588b71e._comment b/doc/forum/ga-ncdu/comment_1_cde3f7bbd099b303bacdaa5e1588b71e._comment
new file mode 100644
index 0000000000..6d7caee2c1
--- /dev/null
+++ b/doc/forum/ga-ncdu/comment_1_cde3f7bbd099b303bacdaa5e1588b71e._comment
@@ -0,0 +1,7 @@
+[[!comment format=mdwn
+ username="andy"
+ subject="comment 1"
+ date="2015-03-28T22:03:53Z"
+ content="""
+I'd take a look. I just downloaded ncdu prompted by your comment, and that would be helpful to get sane output regarding my annexed files.
+"""]]
diff --git a/doc/forum/ga-ncdu/comment_2_8d369dc264ccaf80490c1cb37a330239._comment b/doc/forum/ga-ncdu/comment_2_8d369dc264ccaf80490c1cb37a330239._comment
new file mode 100644
index 0000000000..ef5cb76f24
--- /dev/null
+++ b/doc/forum/ga-ncdu/comment_2_8d369dc264ccaf80490c1cb37a330239._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="markusk"
+ subject="Sounds nice !"
+ date="2015-03-28T22:41:45Z"
+ content="""
+
+I would be very glad if you would release it!
+"""]]
diff --git a/doc/forum/ga-ncdu/comment_3_c5ce3b663de76b50754de70b3fb23bf0._comment b/doc/forum/ga-ncdu/comment_3_c5ce3b663de76b50754de70b3fb23bf0._comment
new file mode 100644
index 0000000000..318441e7b5
--- /dev/null
+++ b/doc/forum/ga-ncdu/comment_3_c5ce3b663de76b50754de70b3fb23bf0._comment
@@ -0,0 +1,15 @@
+[[!comment format=mdwn
+ username="CandyAngel"
+ subject="comment 3"
+ date="2015-04-12T22:12:53Z"
+ content="""
+Whelp, didn't realise it had been over two weeks! Got caught up in other stuff (VR).
+
+[Here's the bitbucket repository!](https://bitbucket.org/CandyAngel/ga-ncdu)
+
+I've coded my own JSON output so it doesn't depend on any non-core Perl modules.
+
+Please let me know of any bugs, feature requests etc. Feedback would be appreciated, even just letting me know you are using it would be great!
+
+    ga-ncdu.pl ~/mah_annex | ncdu -f-
+"""]]
diff --git a/doc/forum/ga_dev_newbie_Q__58___pointers_to_start_playing_with_metadata_cache_plz.mdwn b/doc/forum/ga_dev_newbie_Q__58___pointers_to_start_playing_with_metadata_cache_plz.mdwn
new file mode 100644
index 0000000000..da8c030702
--- /dev/null
+++ b/doc/forum/ga_dev_newbie_Q__58___pointers_to_start_playing_with_metadata_cache_plz.mdwn
@@ -0,0 +1,13 @@
+Hello. Am a newbie to Git Annex(ga), but love it already. I kept trying to index own important files for the past long time, but ended up all tangled up. With ga I now see a light at the end of the tunnel! (Hope it's not a train heading my way :)
+
+So thanks a bucket for writing Git Annex!
+
+I am an "archiver": Every file I add to ga repo is a never-to-be-changed file (it's checksum stays same throughout eternity, only metadata keeps changin). All I need ga for atm is to tag all files. Unfortunately we are talking about few hundred thousand files and the performance with the master git-annex-6.20170519 is not quite what one might hope for.
+
+From your design/caching_database doc I gather that the outlook with metadata is positive ( "For metadata, the story is much nicer. Querying for 30000 keys that all have a particular tag in their metadata takes 0.65s. So fast enough to be used in views." ), but  is not in a db (sqlite) yet in the master (git-annex-6.20170519) . I tried to dig through some of the Links there to find out which commit could I checkout and build to try out a cached metadata, but no avail.
+
+Since I don't ever change any file once it gets checked into the ga repo, does that simplify my possible use of current metadata cache code, or will I have to try to learn haskell and will I need to code stuff to get performance (creating views and such).
+
+TIA for any pointers, tips and cavats and THANKS AGAIN FOR WRITING GIT-ANNEX.
+
+  ganewbie01
diff --git a/doc/forum/ga_dev_newbie_Q__58___pointers_to_start_playing_with_metadata_cache_plz/comment_1_0547fcabefe0b8e99a43977ffec3e28c._comment b/doc/forum/ga_dev_newbie_Q__58___pointers_to_start_playing_with_metadata_cache_plz/comment_1_0547fcabefe0b8e99a43977ffec3e28c._comment
new file mode 100644
index 0000000000..db608ffb8e
--- /dev/null
+++ b/doc/forum/ga_dev_newbie_Q__58___pointers_to_start_playing_with_metadata_cache_plz/comment_1_0547fcabefe0b8e99a43977ffec3e28c._comment
@@ -0,0 +1,10 @@
+[[!comment format=mdwn
+ username="ganewbie01"
+ avatar="http://cdn.libravatar.org/avatar/a3b7d6e560486cb87c51cb0cf3328c8e"
+ subject="development branches inaccessible?"
+ date="2017-11-26T13:12:16Z"
+ content="""
+To not sit idle, I've been looking for development branches (specifically the one containing code that gave the rise to Joey's claim \"Querying for 30000 keys that all have a particular tag in their metadata takes 0.65s.\"), but could find only repos with the one branch - the master branch, which doesn't (naturally seem to) include the code for SQLite metadata tinkering.
+
+Is there someplace I could find such development branches please?
+"""]]
diff --git a/doc/forum/ga_dev_newbie_Q__58___pointers_to_start_playing_with_metadata_cache_plz/comment_2_606281f102d8c3168cbc55142a996e05._comment b/doc/forum/ga_dev_newbie_Q__58___pointers_to_start_playing_with_metadata_cache_plz/comment_2_606281f102d8c3168cbc55142a996e05._comment
new file mode 100644
index 0000000000..f503270d17
--- /dev/null
+++ b/doc/forum/ga_dev_newbie_Q__58___pointers_to_start_playing_with_metadata_cache_plz/comment_2_606281f102d8c3168cbc55142a996e05._comment
@@ -0,0 +1,39 @@
+[[!comment format=mdwn
+ username="olaf"
+ avatar="http://cdn.libravatar.org/avatar/4ae498d3d6ee558d6b65caa658f72572"
+ subject="comment 2"
+ date="2017-11-27T05:39:04Z"
+ content="""
+Did you clone the repository?
+
+    $ git clone git://git-annex.branchable.com/ git-annex
+
+I see lots of branches (remember they are *remote* branches so you will need the `-a` flag):
+
+    $ git branch -a
+    * master
+      remotes/origin/HEAD -> origin/master
+      remotes/origin/atomic-store-test
+      remotes/origin/debian
+      remotes/origin/debian-jessie-backport
+      remotes/origin/debian-squeeze-backport
+      remotes/origin/debian-stable-security-fix
+      remotes/origin/debian-wheezy-backport
+      remotes/origin/ghc7.0
+      remotes/origin/improved-smudge-filters
+      remotes/origin/master
+      remotes/origin/newwinrelease
+      remotes/origin/no-direct-mode
+      remotes/origin/p2p-map
+      remotes/origin/setup
+      remotes/origin/smudge
+      remotes/origin/tweak-fetch
+      remotes/origin/uuid-type-rework
+      remotes/origin/winsplicehack
+
+You can checkout one of the branches like:
+
+    $ git checkout remotes/origin/setup
+
+Does that help?
+"""]]
diff --git a/doc/forum/ga_dev_newbie_Q__58___pointers_to_start_playing_with_metadata_cache_plz/comment_3_ca138f01184631135d07835d79a95112._comment b/doc/forum/ga_dev_newbie_Q__58___pointers_to_start_playing_with_metadata_cache_plz/comment_3_ca138f01184631135d07835d79a95112._comment
new file mode 100644
index 0000000000..a8933e1b4c
--- /dev/null
+++ b/doc/forum/ga_dev_newbie_Q__58___pointers_to_start_playing_with_metadata_cache_plz/comment_3_ca138f01184631135d07835d79a95112._comment
@@ -0,0 +1,15 @@
+[[!comment format=mdwn
+ username="ganewbie01"
+ avatar="http://cdn.libravatar.org/avatar/a3b7d6e560486cb87c51cb0cf3328c8e"
+ subject="found it! ( I think ... or should I be still looking for "database" branch? )"
+ date="2017-11-28T01:03:05Z"
+ content="""
+hi, thanks for your reply;
+I've spent several hours today looking through the git-annex repo. I think it was a great idea to place the forums and everything in one repo! It provides sort of a \"running commentary\" on what was going on and why ...
+
+After a couple of hours looking through the repo using tig, I checked out the key commit \"bb242bdd82a438ebfc937609d8d13b512cb49943\" and found the foo.hs and fooes.hs files which are most likely the ones that Joey was writing about when he expressed hopes for metadata in an sqlite file. ( I didn't find a way to see \"old branches\" though, e.g. the one named `database`. Maybe if I study git more ... )
+
+Thanks for your reply to a silly newbie question anyway! I'll study this some more and see if I have some on-topic questions (hopefully they will be more educated by then :) )
+
+g'day!
+"""]]
diff --git a/doc/forum/ga_dev_newbie_Q__58___pointers_to_start_playing_with_metadata_cache_plz/comment_4_750e5eaac44e53030be18858c03cbe6c._comment b/doc/forum/ga_dev_newbie_Q__58___pointers_to_start_playing_with_metadata_cache_plz/comment_4_750e5eaac44e53030be18858c03cbe6c._comment
new file mode 100644
index 0000000000..fa96166ec1
--- /dev/null
+++ b/doc/forum/ga_dev_newbie_Q__58___pointers_to_start_playing_with_metadata_cache_plz/comment_4_750e5eaac44e53030be18858c03cbe6c._comment
@@ -0,0 +1,22 @@
+[[!comment format=mdwn
+ username="joey"
+ subject="""comment 4"""
+ date="2017-11-28T21:47:54Z"
+ content="""
+Yeah, you found the stuff. That's as far as the metadata cache idea has
+gotten yet. I've restored the missing "database" branch, which was just
+that commit you found.
+
+I do hope to circle back around to this eventually to speed up generating
+views and other metadata queries.
+
+But, as a programmer, you could create your own sqlite database and put
+metadata about your git-annex repository in it. Using
+`git annex metadata --batch --json` you can query git-annex
+for metadata about your files as fast as it can pull it out of git,
+and shove it into your database, and then write your own sql queries.
+
+That would be a good first step, because working with real-world
+data would help develop the sql schema and see if it'll be fast enough to
+bother with putting into git-annex.
+"""]]
diff --git a/doc/forum/gadu_-_git-annex_disk_usage.mdwn b/doc/forum/gadu_-_git-annex_disk_usage.mdwn
new file mode 100644
index 0000000000..e4109a2cfe
--- /dev/null
+++ b/doc/forum/gadu_-_git-annex_disk_usage.mdwn
@@ -0,0 +1,7 @@
+Based on the thread over at [[forum/__34__du__34___equivalent_on_an_annex__63__]] I decided to finally write a du like utility for git-annex.  A 0.01 version is up over at .  It works, but I intend to make it smarter about handling git repos and annexed files, as well as adding more of the options available in the standard du utility.
+
+Currently it will tally up the sizes of links that look like they are annexed files.  I plan to make it actually interact with git and git-annex to verify the files are annexed and enable options like tallying files only from specific remotes, only missing files, not double counting files which are annexed multiple times but stored only once, etc...
+
+I'll have time to work on this on the weekends, and plan to get my git repo up soon.  After gadu is mostly complete I might work on some other tools.
+
+Releases are signed with a PGP key with fingerprint 5E1A 65D7 D5E9 56F1 C239  43DF C6C8 9A0B 6003 8953 (available on the website)
diff --git a/doc/forum/gadu_-_git-annex_disk_usage/comment_10_f632a62c4dbbf01b29f146893d7725f9._comment b/doc/forum/gadu_-_git-annex_disk_usage/comment_10_f632a62c4dbbf01b29f146893d7725f9._comment
new file mode 100644
index 0000000000..602cc43630
--- /dev/null
+++ b/doc/forum/gadu_-_git-annex_disk_usage/comment_10_f632a62c4dbbf01b29f146893d7725f9._comment
@@ -0,0 +1,15 @@
+[[!comment format=mdwn
+ username="Steve"
+ ip="92.104.175.136"
+ subject="comment 10"
+ date="2013-01-14T01:54:12Z"
+ content="""
+No problem, glad to see it is useful.  I'm not exactly a web guy, but I want to get some sort of comment/discussion system up there soon so we aren't filling up Joey's web site with semi-offtopic discussion.  (also a little beautification is in order)
+
+Yes, contributions are welcome.  GPG/PGP encrypted email is the preferred mode of communication.
+
+Currently I ask for copyright assignment in case I want to change licenses in the future.  I pledge not to go to a non-free license, but the GPL3 license choice was fairly arbitrary.  I might want to add the \"or any later version\" clause, for example.  There is also potential for a library to be split off which might benefit from something like LGPL licensing or similar.  I haven't really studied the licensing situation since GPL3 came around, so I need to take some time to look into it.
+
+I don't want to have a licensing discussion here though as it would be offtopic.  Feel free to email me and we can discuss.
+
+"""]]
diff --git a/doc/forum/gadu_-_git-annex_disk_usage/comment_11_73461da2d55d040cb43e0db286975821._comment b/doc/forum/gadu_-_git-annex_disk_usage/comment_11_73461da2d55d040cb43e0db286975821._comment
new file mode 100644
index 0000000000..5f9a8bbecc
--- /dev/null
+++ b/doc/forum/gadu_-_git-annex_disk_usage/comment_11_73461da2d55d040cb43e0db286975821._comment
@@ -0,0 +1,21 @@
+[[!comment format=mdwn
+ username="http://joeyh.name/"
+ nickname="joey"
+ subject="comment 11"
+ date="2013-03-11T05:31:04Z"
+ content="""
+I don't want to steal gadu's thunder, and I really quite like having an ecosystem of tools develop around git-annex.
+
+With that said, \"git annex status .\" now shows the disk used for all files in the current directory and below. It also shows the number of keys, and the total amount of disk those keys would use.
+
+Additionally, you can use all the standard git-annex file limiting options. For example, here I'm finding out how much disk space is used by files located on a *remote* system:
+
+
    +git annex status . --in turtle
+directory: .
+local annex keys: 0
+local annex size: 0 bytes
+known annex keys: 10
+known annex size: 3 gigabytes
+
    
+"""]]
diff --git a/doc/forum/gadu_-_git-annex_disk_usage/comment_12_6c4fb123091bde435c18ac3dfd5a9b77._comment b/doc/forum/gadu_-_git-annex_disk_usage/comment_12_6c4fb123091bde435c18ac3dfd5a9b77._comment
new file mode 100644
index 0000000000..d228e196de
--- /dev/null
+++ b/doc/forum/gadu_-_git-annex_disk_usage/comment_12_6c4fb123091bde435c18ac3dfd5a9b77._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="http://joeyh.name/"
+ nickname="joey"
+ subject="comment 12"
+ date="2013-03-11T05:33:09Z"
+ content="""
+BTW, I think gadu still has its own uses, due to having a du like output, that can list space used by subdirectories. You can do that with git annex status *, but it's much more verbose, and doesn't show a break down by deeper subdirectories.
+"""]]
diff --git a/doc/forum/gadu_-_git-annex_disk_usage/comment_13_8e0e86ae716ff018025808f417e1f7f6._comment b/doc/forum/gadu_-_git-annex_disk_usage/comment_13_8e0e86ae716ff018025808f417e1f7f6._comment
new file mode 100644
index 0000000000..b62b05ad89
--- /dev/null
+++ b/doc/forum/gadu_-_git-annex_disk_usage/comment_13_8e0e86ae716ff018025808f417e1f7f6._comment
@@ -0,0 +1,14 @@
+[[!comment format=mdwn
+ username="https://www.google.com/accounts/o8/id?id=AItOawkWHj0RxNMfuwvFzo2d-V6vBKOYwW_Fnfk"
+ nickname="Andrew"
+ subject="Update on determining disk usage"
+ date="2015-01-22T06:29:36Z"
+ content="""
+I just had a look at this question today as I learn git-annex.  I think the commands have changed since the last comment.  However, there remain several ways to determine disk usage, for example in the folder `Music`
+
+    git annex info Music
+
+but you could also use `du` with
+
+    du --human-readable --dereference Music
+"""]]
diff --git a/doc/forum/gadu_-_git-annex_disk_usage/comment_14_d8f69914b88feb3f3ed4f72c26dd74e5._comment b/doc/forum/gadu_-_git-annex_disk_usage/comment_14_d8f69914b88feb3f3ed4f72c26dd74e5._comment
new file mode 100644
index 0000000000..33fca774c4
--- /dev/null
+++ b/doc/forum/gadu_-_git-annex_disk_usage/comment_14_d8f69914b88feb3f3ed4f72c26dd74e5._comment
@@ -0,0 +1,231 @@
+[[!comment format=mdwn
+ username="anarcat"
+ subject="it's git annex info, not status"
+ date="2015-06-23T04:15:24Z"
+ content="""
+so the previous comments by joeyh were correct 2 years ago, but now git annex status behaves more like git-status than anything else, and will *not* give you disk usage.
+
+however, `git annex info` will, and if you use `--fast`, it works pretty fast as well. example, on my pictures collection:
+
+
    +[1044]anarcat@marcos:Photos$ time git annex info --fast *
+directory: 1969
+local annex keys: 5
+local annex size: 10.95 megabytes
+annexed files in working tree: 5
+size of annexed files in working tree: 10.95 megabytes
+directory: 1970
+local annex keys: 26
+local annex size: 827.5 megabytes
+annexed files in working tree: 26
+size of annexed files in working tree: 827.5 megabytes
+directory: 1998
+local annex keys: 10
+local annex size: 3.31 megabytes
+annexed files in working tree: 10
+size of annexed files in working tree: 3.31 megabytes
+directory: 2004
+local annex keys: 49
+local annex size: 42.01 megabytes
+annexed files in working tree: 49
+size of annexed files in working tree: 42.01 megabytes
+directory: 2005
+local annex keys: 561
+local annex size: 379.23 megabytes
+annexed files in working tree: 561
+size of annexed files in working tree: 379.23 megabytes
+directory: 2006
+local annex keys: 932
+local annex size: 995.95 megabytes
+annexed files in working tree: 932
+size of annexed files in working tree: 995.95 megabytes
+directory: 2007
+local annex keys: 1162
+local annex size: 2.33 gigabytes
+annexed files in working tree: 1162
+size of annexed files in working tree: 2.33 gigabytes
+directory: 2008
+local annex keys: 658
+local annex size: 934.88 megabytes
+annexed files in working tree: 658
+size of annexed files in working tree: 934.88 megabytes
+directory: 2009
+local annex keys: 500
+local annex size: 836.65 megabytes
+annexed files in working tree: 500
+size of annexed files in working tree: 836.65 megabytes
+directory: 2010
+local annex keys: 1049
+local annex size: 1.85 gigabytes
+annexed files in working tree: 1049
+size of annexed files in working tree: 1.85 gigabytes
+directory: 2011
+local annex keys: 1206
+local annex size: 1.54 gigabytes
+annexed files in working tree: 1206
+size of annexed files in working tree: 1.54 gigabytes
+directory: 2012
+local annex keys: 2767
+local annex size: 10.52 gigabytes
+annexed files in working tree: 2767
+size of annexed files in working tree: 10.52 gigabytes
+directory: 2013
+local annex keys: 4071
+local annex size: 32.49 gigabytes
+annexed files in working tree: 4071
+size of annexed files in working tree: 32.49 gigabytes
+directory: 2014
+local annex keys: 6930
+local annex size: 27.34 gigabytes
+annexed files in working tree: 6930
+size of annexed files in working tree: 27.34 gigabytes
+directory: 2015
+local annex keys: 2134
+local annex size: 8.07 gigabytes
+annexed files in working tree: 2134
+size of annexed files in working tree: 8.07 gigabytes
+directory: rando-velo
+local annex keys: 184
+local annex size: 537.58 megabytes
+annexed files in working tree: 184
+size of annexed files in working tree: 537.58 megabytes
+directory: RMLL2008-Koumbit
+local annex keys: 11
+local annex size: 25.58 megabytes
+annexed files in working tree: 11
+size of annexed files in working tree: 25.58 megabytes
+5.47user 1.75system 0:14.70elapsed 49%CPU (0avgtext+0avgdata 30524maxresident)k
+121136inputs+0outputs (2major+18418minor)pagefaults 0swaps
+
    
+
+whereas without `--fast` is much slower, presumably because it's fetching the tracking information:
+
+
    +[1045]anarcat@marcos:Photos$ time git annex info  *
+directory: 1969
+local annex keys: 5
+local annex size: 10.95 megabytes
+annexed files in working tree: 5
+size of annexed files in working tree: 10.95 megabytes
+numcopies stats:
+        numcopies +0: 5
+directory: 1970
+local annex keys: 26
+local annex size: 827.5 megabytes
+annexed files in working tree: 26
+size of annexed files in working tree: 827.5 megabytes
+numcopies stats:
+        numcopies +0: 26
+directory: 1998
+local annex keys: 10
+local annex size: 3.31 megabytes
+annexed files in working tree: 10
+size of annexed files in working tree: 3.31 megabytes
+numcopies stats:
+        numcopies +0: 10
+directory: 2004
+local annex keys: 49
+local annex size: 42.01 megabytes
+annexed files in working tree: 49
+size of annexed files in working tree: 42.01 megabytes
+numcopies stats:
+        numcopies +0: 49
+directory: 2005
+local annex keys: 561
+local annex size: 379.23 megabytes
+annexed files in working tree: 561
+size of annexed files in working tree: 379.23 megabytes
+numcopies stats:
+        numcopies +0: 561
+directory: 2006
+local annex keys: 932
+local annex size: 995.95 megabytes
+annexed files in working tree: 932
+size of annexed files in working tree: 995.95 megabytes
+numcopies stats:
+        numcopies +0: 932
+directory: 2007
+local annex keys: 1162
+local annex size: 2.33 gigabytes
+annexed files in working tree: 1162
+size of annexed files in working tree: 2.33 gigabytes
+numcopies stats:
+        numcopies +0: 1162
+directory: 2008
+local annex keys: 658
+local annex size: 934.88 megabytes
+annexed files in working tree: 658
+size of annexed files in working tree: 934.88 megabytes
+numcopies stats:
+        numcopies +0: 658
+directory: 2009
+local annex keys: 500
+local annex size: 836.65 megabytes
+annexed files in working tree: 500
+size of annexed files in working tree: 836.65 megabytes
+numcopies stats:
+        numcopies +0: 500
+directory: 2010
+local annex keys: 1049
+local annex size: 1.85 gigabytes
+annexed files in working tree: 1049
+size of annexed files in working tree: 1.85 gigabytes
+numcopies stats:
+        numcopies +0: 1049
+directory: 2011
+local annex keys: 1206
+local annex size: 1.54 gigabytes
+annexed files in working tree: 1206
+size of annexed files in working tree: 1.54 gigabytes
+numcopies stats:
+        numcopies +0: 1206
+directory: 2012
+local annex keys: 2767
+local annex size: 10.52 gigabytes
+annexed files in working tree: 2767
+size of annexed files in working tree: 10.52 gigabytes
+numcopies stats:
+        numcopies +0: 2767
+directory: 2013
+local annex keys: 4071
+local annex size: 32.49 gigabytes
+annexed files in working tree: 4071
+size of annexed files in working tree: 32.49 gigabytes
+numcopies stats:
+        numcopies +0: 4071
+directory: 2014
+local annex keys: 6930
+local annex size: 27.34 gigabytes
+annexed files in working tree: 6930
+size of annexed files in working tree: 27.34 gigabytes
+numcopies stats:
+        numcopies +0: 6930
+directory: 2015
+local annex keys: 2134
+local annex size: 8.07 gigabytes
+annexed files in working tree: 2134
+size of annexed files in working tree: 8.07 gigabytes
+numcopies stats:
+        numcopies +0: 2134
+directory: rando-velo
+local annex keys: 184
+local annex size: 537.58 megabytes
+annexed files in working tree: 184
+size of annexed files in working tree: 537.58 megabytes
+numcopies stats:
+        numcopies +0: 184
+directory: RMLL2008-Koumbit
+local annex keys: 11
+local annex size: 25.58 megabytes
+annexed files in working tree: 11
+size of annexed files in working tree: 25.58 megabytes
+numcopies stats:
+        numcopies +0: 11
+37.46user 5.70system 1:54.20elapsed 37%CPU (0avgtext+0avgdata 30704maxresident)k
+107912inputs+0outputs (2major+19426minor)pagefaults 0swaps
+
    
+
+14 seconds vs 114 seconds! almost an order of magnitude of difference...
+
+still, it seems to me `git annex info --fast $path` should be more clearly put forward as an alternative du solution for now. maybe this should be made into a tips page?
+"""]]
diff --git a/doc/forum/gadu_-_git-annex_disk_usage/comment_1_067d0ffe8900751bd2d2743254ac4d77._comment b/doc/forum/gadu_-_git-annex_disk_usage/comment_1_067d0ffe8900751bd2d2743254ac4d77._comment
new file mode 100644
index 0000000000..9699b6ee68
--- /dev/null
+++ b/doc/forum/gadu_-_git-annex_disk_usage/comment_1_067d0ffe8900751bd2d2743254ac4d77._comment
@@ -0,0 +1,11 @@
+[[!comment format=mdwn
+ username="Steve"
+ ip="92.104.175.136"
+ subject="0.02 is up"
+ date="2012-12-08T14:20:16Z"
+ content="""
+I fixed some bugs that gave the wrong answer occasionally, and made gadu much smarter now.
+
+It now searches for the .git dir an makes sure the git-annex links are well formed before counting them.
+I also added a few more du like options.
+"""]]
diff --git a/doc/forum/gadu_-_git-annex_disk_usage/comment_2_ec8b57426e4d82c3392eb7dd683f2ddc._comment b/doc/forum/gadu_-_git-annex_disk_usage/comment_2_ec8b57426e4d82c3392eb7dd683f2ddc._comment
new file mode 100644
index 0000000000..04c6c3f650
--- /dev/null
+++ b/doc/forum/gadu_-_git-annex_disk_usage/comment_2_ec8b57426e4d82c3392eb7dd683f2ddc._comment
@@ -0,0 +1,17 @@
+[[!comment format=mdwn
+ username="http://sunny256.sunbase.org/"
+ nickname="sunny256"
+ subject="comment 2"
+ date="2012-12-08T15:35:16Z"
+ content="""
+Have downloaded v0.02 and experimented a bit, and it seems to work nicely. A couple of things, though:
+
+- It displays the sizes in 512 byte blocks as default. I find that very confusing, and the standard `du`(1) from GNU coreutils uses 1024kB as default. AFAIK 512 byte blocks is an old way of measuring sizes from the really ancient UNIX days. Traditionally correct, maybe, but not very useful these days.
+- When not specifying a path, can it use \"`.`\" as default?
+- A human-readable format would've been nice, like 234M or 13G. The `du`(1) from GNU coreutils uses `-h` for this, but that option is already used for `--help`. And that's OK, I think `-h` should be reserved for that purpose. IMHO using `-h` as a synonym for `--human-readable` was a bad choice by coreutils, but it's too late to change that now.
+
+Is there any Git repository available for git-annex-utils somewhere? That's my preferred way of getting updates and follow the development.
+
+Anyway, thanks. :)
+
+"""]]
diff --git a/doc/forum/gadu_-_git-annex_disk_usage/comment_3_38296fef5a2dc5794c2dc09df676b8c1._comment b/doc/forum/gadu_-_git-annex_disk_usage/comment_3_38296fef5a2dc5794c2dc09df676b8c1._comment
new file mode 100644
index 0000000000..42e4f5a713
--- /dev/null
+++ b/doc/forum/gadu_-_git-annex_disk_usage/comment_3_38296fef5a2dc5794c2dc09df676b8c1._comment
@@ -0,0 +1,18 @@
+[[!comment format=mdwn
+ username="Steve"
+ ip="92.104.175.136"
+ subject="gadu 0.03 is up"
+ date="2012-12-09T13:05:10Z"
+ content="""
+* 1K blocksize is now the default
+* \".\" is now the default path
+* implemented -B/--block-size option
+* --help is no longer -h, it only has a long option like du
+* implemented -h/--human-readable option
+
+du will take up to yottabytes for the --block-size option.  I had been fudging the sizes with a size_t thinking 16 exabytes was plenty big enough for now, but since I was implementing --block-size I went ahead and converted everything to use the GNU MP.  So libgmp is now a dependency.
+
+--human-readable probably doesn't have exactly the same output, but I think it is good enough.  I tried to make the options work mostly the same as du from core-utils.  Let me know if you find other discrepancies.
+
+I'll see about making the git tree available soon, but it may have to wait until next weekend.  I may also look into a forum for the website, or a mailing list.
+"""]]
diff --git a/doc/forum/gadu_-_git-annex_disk_usage/comment_4_1bcc94f9982c6cfd0888f3dba0f9221e._comment b/doc/forum/gadu_-_git-annex_disk_usage/comment_4_1bcc94f9982c6cfd0888f3dba0f9221e._comment
new file mode 100644
index 0000000000..807dac1b38
--- /dev/null
+++ b/doc/forum/gadu_-_git-annex_disk_usage/comment_4_1bcc94f9982c6cfd0888f3dba0f9221e._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="http://sunny256.sunbase.org/"
+ nickname="sunny256"
+ subject="comment 4"
+ date="2012-12-09T20:13:47Z"
+ content="""
+Thanks a lot, Steve. Awesome, got everything on my wishlist. :) A very useful utility, and works perfectly. Will be using this a lot. git-annex-utils is a good name for this, I'm sure if you place it on GitHub or somewhere else you'll get lots of contributions and this could grow to be a project containing many useful utilities for git-annex.
+"""]]
diff --git a/doc/forum/gadu_-_git-annex_disk_usage/comment_5_4365cd3031456fac1b563ee72984638e._comment b/doc/forum/gadu_-_git-annex_disk_usage/comment_5_4365cd3031456fac1b563ee72984638e._comment
new file mode 100644
index 0000000000..e2611dbaaf
--- /dev/null
+++ b/doc/forum/gadu_-_git-annex_disk_usage/comment_5_4365cd3031456fac1b563ee72984638e._comment
@@ -0,0 +1,18 @@
+[[!comment format=mdwn
+ username="Steve"
+ ip="92.104.175.136"
+ subject="comment 5"
+ date="2012-12-10T04:07:53Z"
+ content="""
+I pay attention to feedback ;)
+
+I'm not done with it yet, I want to add in some options to limit what gets counted.
+
+For example:  If you have two annexed files that contain the same content using the same backend, they will be stored only once in the .git/annex/objects directory but be counted twice by gadu.
+
+I want to fix that, but I'll leave an option to keep that behavior if you want.  I also want to add options to count or not count files that exist in a certain repo.  It will be very easy to add options to only count files that you have or don't have locally as well.
+
+Making it pay attention to environment variables that git and git-annex do would also be a good idea. (like GIT_DIR, etc...)
+
+I'm open to good ideas that anybody has, unfortunately I can only work on it on the weekends for now.
+"""]]
diff --git a/doc/forum/gadu_-_git-annex_disk_usage/comment_6_2b03d7b857497cb811e992f85700cdcc._comment b/doc/forum/gadu_-_git-annex_disk_usage/comment_6_2b03d7b857497cb811e992f85700cdcc._comment
new file mode 100644
index 0000000000..9b8d049088
--- /dev/null
+++ b/doc/forum/gadu_-_git-annex_disk_usage/comment_6_2b03d7b857497cb811e992f85700cdcc._comment
@@ -0,0 +1,12 @@
+[[!comment format=mdwn
+ username="markus"
+ ip="79.243.252.165"
+ subject="Great util!"
+ date="2012-12-28T17:45:27Z"
+ content="""
+Hi 
+
+gadu is a great util! The speed increase compared to \"du -smL\" will make it my fav. util for size calc! 
+
+ciao markus 
+"""]]
diff --git a/doc/forum/gadu_-_git-annex_disk_usage/comment_7_03a4dfaf3bd73d41c6f3c3fab0a6a922._comment b/doc/forum/gadu_-_git-annex_disk_usage/comment_7_03a4dfaf3bd73d41c6f3c3fab0a6a922._comment
new file mode 100644
index 0000000000..05f20ddcc2
--- /dev/null
+++ b/doc/forum/gadu_-_git-annex_disk_usage/comment_7_03a4dfaf3bd73d41c6f3c3fab0a6a922._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="https://me.yahoo.com/a/2grhJvAC049fJnvALDXek.6MRZMTlg--#eec89"
+ nickname="John"
+ subject="sizes has git-annex support"
+ date="2012-12-30T22:47:42Z"
+ content="""
+Just to note that I already added git-annex support to my \"sizes\" utility on Hackage several months back.  With -A, it shows you storage totals with annex symlinks computed fully resolved.
+"""]]
diff --git a/doc/forum/gadu_-_git-annex_disk_usage/comment_8_fc6ddb4dc075ee42368863c1b026dbf7._comment b/doc/forum/gadu_-_git-annex_disk_usage/comment_8_fc6ddb4dc075ee42368863c1b026dbf7._comment
new file mode 100644
index 0000000000..642ad3d961
--- /dev/null
+++ b/doc/forum/gadu_-_git-annex_disk_usage/comment_8_fc6ddb4dc075ee42368863c1b026dbf7._comment
@@ -0,0 +1,13 @@
+[[!comment format=mdwn
+ username="Steve"
+ ip="92.104.175.136"
+ subject="git repo is now up"
+ date="2013-01-01T23:39:33Z"
+ content="""
+sunny256, the git repo is now accessible at 
+
+Markus, never used the -m option myself.  I added it in git it'll be in the next tarball.  (I plan to go through the du man page and add all appropriate options soon)
+
+John, I wasn't aware of your sizes utility.  I'll look into it.
+
+"""]]
diff --git a/doc/forum/gadu_-_git-annex_disk_usage/comment_9_f03254e518cbdda73e4b88e72476275d._comment b/doc/forum/gadu_-_git-annex_disk_usage/comment_9_f03254e518cbdda73e4b88e72476275d._comment
new file mode 100644
index 0000000000..d3ebbae61d
--- /dev/null
+++ b/doc/forum/gadu_-_git-annex_disk_usage/comment_9_f03254e518cbdda73e4b88e72476275d._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="http://sunny256.sunbase.org/"
+ nickname="sunny256"
+ subject="The git-annex-utils repo"
+ date="2013-01-09T09:39:16Z"
+ content="""
+Thanks for setting up the git-annex-utils repo, Steve. Will install the newest gadu(1) on my computers now. I'll probably contribute some patches for some utility scripts or programs, if that's OK. What's your preferred way to receive patches? GPG-encrypted mail to your address in the commit log?
+"""]]
diff --git a/doc/forum/gcrypt_os_x_app_vs_brew.mdwn b/doc/forum/gcrypt_os_x_app_vs_brew.mdwn
new file mode 100644
index 0000000000..98a2d5f5d3
--- /dev/null
+++ b/doc/forum/gcrypt_os_x_app_vs_brew.mdwn
@@ -0,0 +1,18 @@
+Gcrypt remotes work when using the git-annex command bundled in the git-annex.app. But gcrypt doesn't work when git-annex is installed via home-brew (brew install git-annex).
+
+The initial push will work, any subsequent commands (push/pull) will fail with:
+
+    gpg: anonymous recipient; trying secret key...
+    gpg: anonymous recipient; trying secret key...
+    gpg: anonymous recipient; trying secret key...
+    gpg: anonymous recipient; trying secret key...
+    gpg: decryption failed: No secret key
+    gcrypt: Failed to decrypt manifest!
+
+In both cases (app/brew) it tries the same keys. The app version will use its own version of gpg, which will trigger password prompts. With the brew version gpgtools is used, so I won't get any prompts. (Keychain)
+
+I tried "echo i | gpg -e -R XX -R XX | gpg -d" with the same recipients as the repo. It works well. 
+
+Has anybody hints or ideas what to try next?
+
+Best, Jean-Louis
diff --git a/doc/forum/gcrypt_os_x_app_vs_brew/comment_1_cce5e2c16720cc8e32a4a479f50ce6b3._comment b/doc/forum/gcrypt_os_x_app_vs_brew/comment_1_cce5e2c16720cc8e32a4a479f50ce6b3._comment
new file mode 100644
index 0000000000..bcf094073d
--- /dev/null
+++ b/doc/forum/gcrypt_os_x_app_vs_brew/comment_1_cce5e2c16720cc8e32a4a479f50ce6b3._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="Ganwell"
+ ip="178.174.3.221"
+ subject="Problem solved"
+ date="2014-08-14T23:53:05Z"
+ content="""
+It turns out gpgtools will save to wrong passphrase to the keychain without complaining. Thats why standard gpg worked and gpgtools didn't: There was a typo in the passphrase in the keychain.
+"""]]
diff --git a/doc/forum/gcrypt_os_x_app_vs_brew/comment_2_8df8ba1ccea0f68110593ed90a9cad6d._comment b/doc/forum/gcrypt_os_x_app_vs_brew/comment_2_8df8ba1ccea0f68110593ed90a9cad6d._comment
new file mode 100644
index 0000000000..fcdfba41de
--- /dev/null
+++ b/doc/forum/gcrypt_os_x_app_vs_brew/comment_2_8df8ba1ccea0f68110593ed90a9cad6d._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="http://joeyh.name/"
+ ip="209.250.56.7"
+ subject="comment 2"
+ date="2014-08-15T16:04:08Z"
+ content="""
+Note that you can avoid the trying of multiple keys by doing `git config gcrypt.publish-participants true` -- this is done by default by the assistant when setting up new gcrypt remotes. It needs my branch of git-remote-gcrypt, which is included in the osx app, I don't know which one is being used in brew.
+"""]]
diff --git a/doc/forum/gcrypt_os_x_app_vs_brew/comment_3_be4de1663a37f49a4e42d6b21c0178fe._comment b/doc/forum/gcrypt_os_x_app_vs_brew/comment_3_be4de1663a37f49a4e42d6b21c0178fe._comment
new file mode 100644
index 0000000000..99f3b7cbe9
--- /dev/null
+++ b/doc/forum/gcrypt_os_x_app_vs_brew/comment_3_be4de1663a37f49a4e42d6b21c0178fe._comment
@@ -0,0 +1,10 @@
+[[!comment format=mdwn
+ username="Ganwell"
+ ip="46.14.43.36"
+ subject="brew doesn't include git-remote-gcrypt"
+ date="2014-08-19T21:54:16Z"
+ content="""
+Brew doesn't include git-remote-gcrypt, so I installed your fork of it manually. That works well and if you use a symlink to include it in the PATH you can easily update (pull) the repo. If one can install git-remote-gcrypt using brew, I don't know how.
+
+Best, Jean-Louis
+"""]]
diff --git a/doc/forum/get_and_copy_with_bare_repositories.mdwn b/doc/forum/get_and_copy_with_bare_repositories.mdwn
new file mode 100644
index 0000000000..846887f96d
--- /dev/null
+++ b/doc/forum/get_and_copy_with_bare_repositories.mdwn
@@ -0,0 +1,7 @@
+is `git annex get` and `git annex copy --to somewhere` expected to work with bare repos? the [[bare repositories]] page doesn't indicate otherwise, but a `git annex get` does plain nothing in my setup.
+
+if it's supposed not to work, there should be an error message saying that and an indication on the [[bare repositories]], otherwise, how can i trace it down?
+
+in case it is just unimplemented for lack of use cases: my setup consists of several laptops using parts of a 200gb+ photo collection, a central trusted server that should host everything, and an external encrypted remote backup. clients *should* copy everything they add to both central locations, but i'd prefer the trusted server to sync the two of them too.
+
+`get` and `copy` usually operate on the current directory, which in case of a bare repo does not contain any relevant files, but i tried explicitly specifying files too. `git annex` should either look them up in master, or always operate on all files (as indexed in the `git-annex` branch) unconditionally.
diff --git a/doc/forum/get_and_copy_with_bare_repositories/comment_1_a6e4628c0770e3f5e81348a6f29dd845._comment b/doc/forum/get_and_copy_with_bare_repositories/comment_1_a6e4628c0770e3f5e81348a6f29dd845._comment
new file mode 100644
index 0000000000..a19dd86310
--- /dev/null
+++ b/doc/forum/get_and_copy_with_bare_repositories/comment_1_a6e4628c0770e3f5e81348a6f29dd845._comment
@@ -0,0 +1,10 @@
+[[!comment format=mdwn
+ username="http://joeyh.name/"
+ ip="4.252.11.120"
+ subject="comment 1"
+ date="2012-11-13T17:30:31Z"
+ content="""
+git-annex commands only operate on files in the currently checked out working copy. You can send files to/from bare repos from non-bare repos. But you cannot do anything with them inside the bare repo, just as regular git doesn't let you commit inside a bare repo.
+
+There is talk about [[todo/add_-all_option]], which would be usable in bare repos.
+"""]]
diff --git a/doc/forum/get_and_copy_with_bare_repositories/comment_2_652fa1bae5c2bb63dcffcbda97a567c4._comment b/doc/forum/get_and_copy_with_bare_repositories/comment_2_652fa1bae5c2bb63dcffcbda97a567c4._comment
new file mode 100644
index 0000000000..fdf9f91a2b
--- /dev/null
+++ b/doc/forum/get_and_copy_with_bare_repositories/comment_2_652fa1bae5c2bb63dcffcbda97a567c4._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="http://joeyh.name/"
+ ip="4.154.4.193"
+ subject="comment 2"
+ date="2013-07-03T18:13:25Z"
+ content="""
+`git annex get` and `git annex copy`, as well as `git annex move` can now be used in a bare repository. In this mode, they behave as if --all were specified, and so operate on every single version of every single file.
+"""]]
diff --git a/doc/forum/get_content_of_a_file_from_a_file.mdwn b/doc/forum/get_content_of_a_file_from_a_file.mdwn
new file mode 100644
index 0000000000..3965ed488e
--- /dev/null
+++ b/doc/forum/get_content_of_a_file_from_a_file.mdwn
@@ -0,0 +1,5 @@
+I have two annex repositories A and B, the connection between them is terribly slow. There is a file in B that A wants to get, but incidentally A already has an exact copy of the file outside the annex repository. Is it possible for A to retrieve the content of that annexed file from its local copy rather than from B?
+
+The only idea I could come up with is to annex the file and then unannexing it, which does not seem optimal. I would prefer to not pollute the git log with these kinds of shenanigans.
+
+Any help is greatly appreciated!
diff --git a/doc/forum/get_content_of_a_file_from_a_file/comment_1_174fe84e87b31419295af3397db19671._comment b/doc/forum/get_content_of_a_file_from_a_file/comment_1_174fe84e87b31419295af3397db19671._comment
new file mode 100644
index 0000000000..a88b08e2e3
--- /dev/null
+++ b/doc/forum/get_content_of_a_file_from_a_file/comment_1_174fe84e87b31419295af3397db19671._comment
@@ -0,0 +1,10 @@
+[[!comment format=mdwn
+ username="jd.schroeder@0c8632a8f2bffdd4b0de05a0a3660f32acdfeeca"
+ nickname="jd.schroeder"
+ subject="comment 1"
+ date="2016-07-29T19:09:44Z"
+ content="""
+Have a look at [[https://git-annex.branchable.com/git-annex-reinject/]]
+
+Good luck!
+"""]]
diff --git a/doc/forum/get_content_of_a_file_from_a_file/comment_2_9cc8905f867fedbf2b10c9ad84294543._comment b/doc/forum/get_content_of_a_file_from_a_file/comment_2_9cc8905f867fedbf2b10c9ad84294543._comment
new file mode 100644
index 0000000000..8a79b0fe5d
--- /dev/null
+++ b/doc/forum/get_content_of_a_file_from_a_file/comment_2_9cc8905f867fedbf2b10c9ad84294543._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="squid@d632da79105a546295e411392baaa70df380a4a2"
+ nickname="squid"
+ subject="comment 2"
+ date="2016-07-29T22:27:51Z"
+ content="""
+Hey, thank you very much! This is exactly what I was looking for. Unfortunately, FreeBSD sports version 5.20150727 where the --known option is not yet available, but that's a whole different problem that can be easily solved on a saturday.
+"""]]
diff --git a/doc/forum/getting_git_annex_to_do_a_force_copy_to_a_remote.mdwn b/doc/forum/getting_git_annex_to_do_a_force_copy_to_a_remote.mdwn
new file mode 100644
index 0000000000..cc9091ae5b
--- /dev/null
+++ b/doc/forum/getting_git_annex_to_do_a_force_copy_to_a_remote.mdwn
@@ -0,0 +1,11 @@
+I'm not sure if this is my stupidity or if it's a bug, but
+
+    git annex copy --force --to REMOTE . 
+
+just zip's through really quickly and doesn't actually force a copy to a
+remote location. This is just following up on the
+[[bugs/git-annex_directory_hashing_problems_on_osx]]. I want to just do a force copy of all my data to my portable disk to really make sure that the data is really there. I would similarly would want to make sure I can force a 
+
+    git annex copy --force --from REMOTE .
+
+to pull down files from a remote.
diff --git a/doc/forum/getting_git_annex_to_do_a_force_copy_to_a_remote/comment_1_3deb2c31cad37a49896f00d600253ee3._comment b/doc/forum/getting_git_annex_to_do_a_force_copy_to_a_remote/comment_1_3deb2c31cad37a49896f00d600253ee3._comment
new file mode 100644
index 0000000000..d2692f26f0
--- /dev/null
+++ b/doc/forum/getting_git_annex_to_do_a_force_copy_to_a_remote/comment_1_3deb2c31cad37a49896f00d600253ee3._comment
@@ -0,0 +1,14 @@
+[[!comment format=mdwn
+ username="http://joey.kitenet.net/"
+ nickname="joey"
+ subject="comment 1"
+ date="2011-04-03T16:49:01Z"
+ content="""
+How remote is REMOTE? If it's a directory on the same computer, then git-annex copy --to is actually quickly checking that each file is present on the remote, and when it is, skipping copying it again.
+
+If the remote is ssh, git-annex copy talks to the remote to see if it has the file. This makes copy --to slow, as Rich [[complained_before|forum/batch_check_on_remote_when_using_copy]]. :)
+
+So, copy --to does not trust location tracking information (unless --fast is specified), which means that it should be doing exactly what you want it to do in your situation -- transferring every file that is really not present in the destination repository already.
+
+Neither does copy --from, by the way. It always checks if each file is present in the current repository's annex before trying to download it.
+"""]]
diff --git a/doc/forum/getting_git_annex_to_do_a_force_copy_to_a_remote/comment_2_627f54d158d3ca4b72e45b4da70ff5cd._comment b/doc/forum/getting_git_annex_to_do_a_force_copy_to_a_remote/comment_2_627f54d158d3ca4b72e45b4da70ff5cd._comment
new file mode 100644
index 0000000000..1079303197
--- /dev/null
+++ b/doc/forum/getting_git_annex_to_do_a_force_copy_to_a_remote/comment_2_627f54d158d3ca4b72e45b4da70ff5cd._comment
@@ -0,0 +1,12 @@
+[[!comment format=mdwn
+ username="https://www.google.com/accounts/o8/id?id=AItOawkSq2FDpK2n66QRUxtqqdbyDuwgbQmUWus"
+ nickname="Jimmy"
+ subject="comment 2"
+ date="2011-04-03T16:59:47Z"
+ content="""
+Remote as in \"another physical machine\". I assumed that
+
+    git annex copy --force --to REMOTE .
+
+would have not trusted the contents in the current directory (or the remote that is being copied to) and then just go off and re-download/upload all the files and overwrite what is already there. I expected the combination of *--force* and copy *--to* that it would not bother to check if the files are there or not and just copy it regardless of the outcome.
+"""]]
diff --git a/doc/forum/getting_git_annex_to_do_a_force_copy_to_a_remote/comment_3_3f49dab11aae5df0c4eb5e4b8d741379._comment b/doc/forum/getting_git_annex_to_do_a_force_copy_to_a_remote/comment_3_3f49dab11aae5df0c4eb5e4b8d741379._comment
new file mode 100644
index 0000000000..c3df214988
--- /dev/null
+++ b/doc/forum/getting_git_annex_to_do_a_force_copy_to_a_remote/comment_3_3f49dab11aae5df0c4eb5e4b8d741379._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="https://www.google.com/accounts/o8/id?id=AItOawkSq2FDpK2n66QRUxtqqdbyDuwgbQmUWus"
+ nickname="Jimmy"
+ subject="comment 3"
+ date="2011-04-03T17:12:35Z"
+ content="""
+On second thought maybe the current behaviour is better than what I am suggesting that the force command should do. I guess it's better to be safe than sorry.
+"""]]
diff --git a/doc/forum/ghost_semitrusted_repositories.mdwn b/doc/forum/ghost_semitrusted_repositories.mdwn
new file mode 100644
index 0000000000..9e00178b75
--- /dev/null
+++ b/doc/forum/ghost_semitrusted_repositories.mdwn
@@ -0,0 +1,28 @@
+I had one repo on HDD and another in an USB drive. After mount point changed, I went to the HDD repo folder and tried to use git remote-set url to point it to the new location, however I think I ended in a weird state. git-annex info shows this:
+
+    git-annex info
+    repository mode: indirect
+    trusted repositories: 0
+    semitrusted repositories: 5
+        00000000-0000-0000-0000-000000000001 -- web
+        26c0c4ba-6489-4416-a054-670d373f09bd -- juan@invasor.local:/Volumes/sapo_hfs/live/papers
+        61158820-db14-45b9-b9f9-8619d956388e -- [usb_papers]
+        d4456c86-fa2b-43a7-a132-027915390cf6 -- usb_papers
+        fed56814-08c6-11e3-bf3c-af5da9f7f388 -- sapo [here]
+    untrusted repositories: 0
+    transfers in progress: none
+    available local disk space: 255.46 gigabytes (+1 megabyte reserved)
+    local annex keys: 1252
+    local annex size: 2.55 gigabytes
+    annexed files in working tree: 1297
+    size of annexed files in working tree: 2.7 gigabytes
+    bloom filter size: 16 mebibytes (0.3% full)
+    backend usage: 
+        SHA256E: 2549
+
+usb_papers was the original usb repo, and now a new one [usb_papers] shows. webapp shows only sapo[here] and [usb_papers], the other 2 repos are unknown. I was digging around but couldn't find a way to remove them.
+Seems that the webapp managed to use the new URL, but said something about "fixing repo".
+Any suggestions?
+Thanks in advance.
+Best regards,
+              Juan
diff --git a/doc/forum/ghost_semitrusted_repositories/comment_1_99bea1a964da9c5603b8cfbdc19bcde8._comment b/doc/forum/ghost_semitrusted_repositories/comment_1_99bea1a964da9c5603b8cfbdc19bcde8._comment
new file mode 100644
index 0000000000..ced8645e28
--- /dev/null
+++ b/doc/forum/ghost_semitrusted_repositories/comment_1_99bea1a964da9c5603b8cfbdc19bcde8._comment
@@ -0,0 +1,19 @@
+[[!comment format=mdwn
+ username="http://joeyh.name/"
+ ip="209.250.56.2"
+ subject="comment 1"
+ date="2014-07-11T19:40:32Z"
+ content="""
+    61158820-db14-45b9-b9f9-8619d956388e -- [usb_papers]
+
+This is the git remote you currently have set up.
+
+    d4456c86-fa2b-43a7-a132-027915390cf6 -- usb_papers
+
+This is the old remote from before.
+
+Take a look at the repository in the USB drive, and see what its annex.uuid is set to. git-annex doesn't change the UUID after a repository is set up, so the fact that you have two different UUIDs is puzzling. 
+
+If you really wanted to, you could remove the old repository from the list by running: `git annex dead d4456c86-fa2b-43a7-a132-027915390cf6`  
+But I would want to get to the bottom of what happened before doing that.
+"""]]
diff --git a/doc/forum/ghost_semitrusted_repositories/comment_2_fe5fe5539d06c6b1ef69f3ed805f1ab4._comment b/doc/forum/ghost_semitrusted_repositories/comment_2_fe5fe5539d06c6b1ef69f3ed805f1ab4._comment
new file mode 100644
index 0000000000..931783a0e1
--- /dev/null
+++ b/doc/forum/ghost_semitrusted_repositories/comment_2_fe5fe5539d06c6b1ef69f3ed805f1ab4._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="https://www.google.com/accounts/o8/id?id=AItOawmTNrhkVQ26GBLaLD5-zNuEiR8syTj4mI8"
+ nickname="Juan"
+ subject="web remote"
+ date="2014-07-15T13:19:42Z"
+ content="""
+Ok. Got it. And what's up with the web remote? Why is it there, I didn't add that one.
+"""]]
diff --git a/doc/forum/ghost_semitrusted_repositories/comment_3_588325ef52c80cfc67d1dd80a9d5bd13._comment b/doc/forum/ghost_semitrusted_repositories/comment_3_588325ef52c80cfc67d1dd80a9d5bd13._comment
new file mode 100644
index 0000000000..1cdceff0d5
--- /dev/null
+++ b/doc/forum/ghost_semitrusted_repositories/comment_3_588325ef52c80cfc67d1dd80a9d5bd13._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="http://joeyh.name/"
+ ip="209.250.56.2"
+ subject="comment 3"
+ date="2014-07-15T18:56:31Z"
+ content="""
+It's there to represent the world wide web when using eg, `git annex addurl`
+"""]]
diff --git a/doc/forum/git-annes_assistant_+_MAC_OSX_questions.mdwn b/doc/forum/git-annes_assistant_+_MAC_OSX_questions.mdwn
new file mode 100644
index 0000000000..d32e01a6e6
--- /dev/null
+++ b/doc/forum/git-annes_assistant_+_MAC_OSX_questions.mdwn
@@ -0,0 +1,16 @@
+After spending the whole last night playing with it, I think I have it figured out:
+
+it runs on my Macbook and I have added a free 50GB box.com account as archive and an S3 bucket as full-backup and set the min. number of file occurrences to keep to 2 so IF I delete anything on my Macbook, this should make sure I definitely have 2 more occurrences of each file on two separate storages.
+
+BUT here come the problems/questions
+
+- say I add a folder called: Documents, it has now apparently successfully synced to box.com and S3. I say apparently because there is no tool in the assistant to check
+- I can't seem to figure out how to delete the log of the assistant, its getting quite longish
+- I can't seem to figure out how to upgrade from 5.20140420 to 5.20140421
+- I move a file from a local repo to the archive folder but nothing happens except git-annex recording the move: (Recording state in git...)  /Users/ovi/Sharing/git-annex/Finance/1-564 Erkl. z. Gebiets- u. Steuereigenschaft 08_04.pdf still has writers, not adding [2014-05-09 17:22:43 SAST] Committer: Committing changes to git
+- the assistant doesn'T have any kind of gui for retrieving/deleting/finding stuff - I'm no stranger to the command line but I can't figure out how to add the /Applications/gitannex.app.5.20140421/Contents/MacOS to my path and if I go there and run runshell, most git-annex commands tell me: not in a git repository so all I can do right now, is look at things :-/
+- reading the description on the kickstarter page I must say I expected more. This is in no way comparable to Dropbox. 
+
+Don't get me wrong, this is the most promising project of its kind I came across if I can be pointed in the right direction :-)
+
+P.S. How does it work on the "inside"? i.e. I edit a file and git-annex picks it up and "syncs" - what is synced? The whole file? A diff? How can one restore older versions or even look to see what versions are available?
diff --git a/doc/forum/git-annes_assistant_+_MAC_OSX_questions/comment_1_e661f31acd08a6459842f7f95e5c062b._comment b/doc/forum/git-annes_assistant_+_MAC_OSX_questions/comment_1_e661f31acd08a6459842f7f95e5c062b._comment
new file mode 100644
index 0000000000..9b524ea3b7
--- /dev/null
+++ b/doc/forum/git-annes_assistant_+_MAC_OSX_questions/comment_1_e661f31acd08a6459842f7f95e5c062b._comment
@@ -0,0 +1,9 @@
+[[!comment format=mdwn
+ username="https://www.google.com/accounts/o8/id?id=AItOawkAUMhKOSkh9JaBA6xst3XxQIIsDEq5Zd4"
+ nickname="Ovidiu"
+ subject="partially solved"
+ date="2014-05-11T09:32:32Z"
+ content="""
+I managed to add the path and now I can use git annex command line.
+All my other questions and remarks are still open though :-)
+"""]]
diff --git a/doc/forum/git-annes_assistant_+_MAC_OSX_questions/comment_2_e71a9d1fcf1f945fec0b7834b6038e91._comment b/doc/forum/git-annes_assistant_+_MAC_OSX_questions/comment_2_e71a9d1fcf1f945fec0b7834b6038e91._comment
new file mode 100644
index 0000000000..d11f641a60
--- /dev/null
+++ b/doc/forum/git-annes_assistant_+_MAC_OSX_questions/comment_2_e71a9d1fcf1f945fec0b7834b6038e91._comment
@@ -0,0 +1,21 @@
+[[!comment format=mdwn
+ username="https://www.google.com/accounts/o8/id?id=AItOawkYmMFDdf3GJ9Oba6NCVkzGc4JyB9WavMs"
+ nickname="Xinruo"
+ subject="comment 2"
+ date="2014-05-11T15:15:50Z"
+ content="""
+> say I add a folder called: Documents, it has now apparently successfully synced to box.com and S3. I say apparently because there is no tool in the assistant to check
+
+From the command line, git annex whereis FILE will tell you which remotes have the file, or git annex list will list out this for a directory
+
+> I can't seem to figure out how to delete the log of the assistant, its getting quite longish
+
+They live in .git/annex/daemon.log*
+
+> I can't seem to figure out how to upgrade from 5.20140420 to 5.20140421
+
+> I move a file from a local repo to the archive folder but nothing happens except git-annex recording the move: (Recording state in git...) /Users/ovi/Sharing/git-annex/Finance/1-564 Erkl. z. Gebiets- u. Steuereigenschaft 08_04.pdf still has writers, not adding [2014-05-09 17:22:43 SAST] Committer: Committing changes to git
+
+It says the pdf still has writers.. Maybe the file is locked by another program? In this case the file is not going to be annexed, I believe, until the lock is revoked.
+
+"""]]
diff --git a/doc/forum/git-annes_assistant_+_MAC_OSX_questions/comment_3_372d9da8295d093b8b316f0a48b60ee1._comment b/doc/forum/git-annes_assistant_+_MAC_OSX_questions/comment_3_372d9da8295d093b8b316f0a48b60ee1._comment
new file mode 100644
index 0000000000..93f9ef414b
--- /dev/null
+++ b/doc/forum/git-annes_assistant_+_MAC_OSX_questions/comment_3_372d9da8295d093b8b316f0a48b60ee1._comment
@@ -0,0 +1,12 @@
+[[!comment format=mdwn
+ username="https://www.google.com/accounts/o8/id?id=AItOawkAUMhKOSkh9JaBA6xst3XxQIIsDEq5Zd4"
+ nickname="Ovidiu"
+ subject="comment 3"
+ date="2014-05-12T11:03:59Z"
+ content="""
+So basically, I need to check the log for a specific commit and then restore that one?
+No way to list i.e. all commits for a folder and restore the whole folder?
+Or list all commits for a fiel and then restore that one?
+
+Still learning all of this, any hints welcome.
+"""]]
diff --git a/doc/forum/git-annes_assistant_+_MAC_OSX_questions/comment_4_573537a49e082515bfb1be84c91b5d1b._comment b/doc/forum/git-annes_assistant_+_MAC_OSX_questions/comment_4_573537a49e082515bfb1be84c91b5d1b._comment
new file mode 100644
index 0000000000..01aac51119
--- /dev/null
+++ b/doc/forum/git-annes_assistant_+_MAC_OSX_questions/comment_4_573537a49e082515bfb1be84c91b5d1b._comment
@@ -0,0 +1,10 @@
+[[!comment format=mdwn
+ username="https://www.google.com/accounts/o8/id?id=AItOawkAUMhKOSkh9JaBA6xst3XxQIIsDEq5Zd4"
+ nickname="Ovidiu"
+ subject="comment 4"
+ date="2014-06-21T13:03:11Z"
+ content="""
+Any feedback on my last questions? 
+Is there a road map somewhere? Need to figure out if this is going somewhere soon or if I should look for another solution? 
+Any other kickstarter campaigns you're planning? Wouldn't mind chipping in if it means this goes somewhere fast(er) ;-)
+"""]]
diff --git a/doc/forum/git-annex_6.20170520_doesn__39__t_build_with_QuickCheck_2.10.mdwn b/doc/forum/git-annex_6.20170520_doesn__39__t_build_with_QuickCheck_2.10.mdwn
new file mode 100644
index 0000000000..ce19721034
--- /dev/null
+++ b/doc/forum/git-annex_6.20170520_doesn__39__t_build_with_QuickCheck_2.10.mdwn
@@ -0,0 +1,8 @@
+    [  9 of 561] Compiling Utility.QuickCheck ( Utility/QuickCheck.hs, dist/build/git-annex/git-annex-tmp/Utility/QuickCheck.dyn_o )
+    
+    Utility/QuickCheck.hs:38:10: error:
+        Duplicate instance declarations:
+          instance Arbitrary EpochTime
+            -- Defined at Utility/QuickCheck.hs:38:10
+          instance [safe] Arbitrary Foreign.C.Types.CTime
+            -- Defined in ‘Test.QuickCheck.Arbitrary’
diff --git a/doc/forum/git-annex_6.20170520_doesn__39__t_build_with_QuickCheck_2.10/comment_1_0e25ab25c0fa3d41336e6b7551ef9e47._comment b/doc/forum/git-annex_6.20170520_doesn__39__t_build_with_QuickCheck_2.10/comment_1_0e25ab25c0fa3d41336e6b7551ef9e47._comment
new file mode 100644
index 0000000000..00aa8e5109
--- /dev/null
+++ b/doc/forum/git-annex_6.20170520_doesn__39__t_build_with_QuickCheck_2.10/comment_1_0e25ab25c0fa3d41336e6b7551ef9e47._comment
@@ -0,0 +1,11 @@
+[[!comment format=mdwn
+ username="https://launchpad.net/~felixonmars"
+ nickname="felixonmars"
+ avatar="http://cdn.libravatar.org/avatar/17284a3bb2e4ad9d3be8fab31f49865be9c1dc22143c728de731fe800a335d38"
+ subject="comment 1"
+ date="2017-06-28T17:45:48Z"
+ content="""
+This is the same as https://git-annex.branchable.com/bugs/Build_failure__58___Utility__47__QuickCheck.hs__58__38__58__10__58___error__58___Duplicate_instance_declarations/
+
+Somehow I missed it. Sorry for the duplicated report!
+"""]]
diff --git a/doc/forum/git-annex_SSH_server_+_cloud_remote.mdwn b/doc/forum/git-annex_SSH_server_+_cloud_remote.mdwn
new file mode 100644
index 0000000000..873b75650c
--- /dev/null
+++ b/doc/forum/git-annex_SSH_server_+_cloud_remote.mdwn
@@ -0,0 +1,13 @@
+I'd like to accomplish something like this:
+
+* shared git-annex remote hosted by SSH server
+* file contents stored in S3
+* everything encrypted
+
+The use case here is to achieve the availability of an SSH server (that is, multiple clients will sync to one, main server, eliminating a dependency on each other) and a single source of truth for the repository (i.e. eliminating P2P repo synchronization), and having the bulk of the disk space consumption living on S3 hosting.
+
+Is this possible? If so, any suggestions on how to do it? I understand how to do everything, except configure the SSH server to store annexed files on S3 hosting.
+
+Also, any thoughts on how much space the SSH server's git repo (without annexed files) could possibly grow to, with average use? Is it reasonable to say that it would never get bigger than 'tens of megabytes'? 
+
+Thank you!
diff --git a/doc/forum/git-annex__58___getUserEntryForID__58___does_not_exist___40__no_such_user__41___.mdwn b/doc/forum/git-annex__58___getUserEntryForID__58___does_not_exist___40__no_such_user__41___.mdwn
new file mode 100644
index 0000000000..247407dfea
--- /dev/null
+++ b/doc/forum/git-annex__58___getUserEntryForID__58___does_not_exist___40__no_such_user__41___.mdwn
@@ -0,0 +1,24 @@
+Hi,
+
+I ran into this problem:
+$ git annex mirror --from origin
+git-annex: getUserEntryForID: does not exist (no such user)
+
+I found in the forum that, if the the machine uses LDAP etc. I should be able to use the HOME and USER environment variables.
+
+I'm not quite sure what the user authentication management on the machine I'm on is and I'd rather not deal with the sysadmins.
+But I know that  the /etc/passwd file does not have my user listed.
+I do have the the HOME and USER variables properly set and exported. 
+
+$ echo $HOME $USER
+/nethome/iiossifov iiossifov
+
+In the git-annex source code I found that the error occurs in Utility/UserInfo.hs. 
+I can't read Haskell but my guess is that the problem is with the myUserGecos for which there is no environment variable fall back.
+
+I'm using the pre-build git-annex version: 6.20160527-gb7d4774
+
+Would you be able to advice me how to deal with the problem?
+
+Thank you,
+Ivan
diff --git a/doc/forum/git-annex__58___getUserEntryForID__58___does_not_exist___40__no_such_user__41___/comment_1_94d92b7e64ad31db8837dcf3d131cf23._comment b/doc/forum/git-annex__58___getUserEntryForID__58___does_not_exist___40__no_such_user__41___/comment_1_94d92b7e64ad31db8837dcf3d131cf23._comment
new file mode 100644
index 0000000000..a72c67ea63
--- /dev/null
+++ b/doc/forum/git-annex__58___getUserEntryForID__58___does_not_exist___40__no_such_user__41___/comment_1_94d92b7e64ad31db8837dcf3d131cf23._comment
@@ -0,0 +1,28 @@
+[[!comment format=mdwn
+ username="joey"
+ subject="""comment 1"""
+ date="2016-06-08T17:31:41Z"
+ content="""
+IIRC it's supposed to be possible to configure a system such as LDAP 
+so that `getpwuid` queries it, instead of failing as it seems to on your
+system, using `/etc/nsswitch.conf`. But I'm not sure about that or how to
+do it and you'd need to be the admin of the system.
+
+git-annex has three calls to `getpwuid` (aka getUserEntryForID`).
+In two of them it first looks at commonly set environment variables
+(HOME and USER/LOGNAME) and only uses `getpwuid` as a fallback.
+It seems reasonable to crash when git-annex can't determine the user's home
+directory or username in code that needs it, especially since all three
+environment variables are almost always going to be set.
+
+The third use is a GECOS lookup, and here there's no corresponding
+environment variable and git-annex can deal without knowing the full name
+of the user. So, I've made that not crash if `getpwuid` fails.
+
+(There's also a little-used code path where `getpwnam` is used 
+to expand `~` in a git remote path. Seems best to leave this crashing
+too if used on a system that does not have a working `getpwnam`.)
+
+So, you can try upgrading to an autobuild that has the fix, and if it still
+crashes, check that HOME and USER are set.
+"""]]
diff --git a/doc/forum/git-annex__58___getUserEntryForID__58___does_not_exist___40__no_such_user__41___/comment_2_c9474c4e925948d0cabdfe7f988e1ebe._comment b/doc/forum/git-annex__58___getUserEntryForID__58___does_not_exist___40__no_such_user__41___/comment_2_c9474c4e925948d0cabdfe7f988e1ebe._comment
new file mode 100644
index 0000000000..4db7b992d8
--- /dev/null
+++ b/doc/forum/git-annex__58___getUserEntryForID__58___does_not_exist___40__no_such_user__41___/comment_2_c9474c4e925948d0cabdfe7f988e1ebe._comment
@@ -0,0 +1,15 @@
+[[!comment format=mdwn
+ username="ivan.iossifov@2673267f2e818d14a75f506d41738323cbc0bd04"
+ nickname="ivan.iossifov"
+ subject="how do I get the autobuild?"
+ date="2016-06-13T13:56:12Z"
+ content="""
+Hi Joy,
+
+Thanks a lot for the quick reply! It's seems that's exactly what will solve my problem.
+But, I couldn't find how to download the latest autobuilds.
+Would mind pointing me to it?
+
+Thanks again,
+Ivan
+"""]]
diff --git a/doc/forum/git-annex__58___getUserEntryForID__58___does_not_exist___40__no_such_user__41___/comment_3_3a9e1cb8f36a44fd1917b5fc191776a3._comment b/doc/forum/git-annex__58___getUserEntryForID__58___does_not_exist___40__no_such_user__41___/comment_3_3a9e1cb8f36a44fd1917b5fc191776a3._comment
new file mode 100644
index 0000000000..7071886860
--- /dev/null
+++ b/doc/forum/git-annex__58___getUserEntryForID__58___does_not_exist___40__no_such_user__41___/comment_3_3a9e1cb8f36a44fd1917b5fc191776a3._comment
@@ -0,0 +1,10 @@
+[[!comment format=mdwn
+ username="joey"
+ subject="""comment 3"""
+ date="2016-07-19T19:09:07Z"
+ content="""
+The June 13th release of git-annex included the fix for this problem.
+
+The Windows autobuild is linked to from
+
+"""]]
diff --git a/doc/forum/git-annex__58___map__58___1_failed.mdwn b/doc/forum/git-annex__58___map__58___1_failed.mdwn
new file mode 100644
index 0000000000..10be60ca5b
--- /dev/null
+++ b/doc/forum/git-annex__58___map__58___1_failed.mdwn
@@ -0,0 +1,12 @@
+What exactly does that mean?
+I got that while exploring the git annex command:
+
+shiny-2:git-annex ovi$ pwd
+/Users/ovi/Sharing/git-annex
+shiny-2:git-annex ovi$ git annex map
+map /Users/ovi/Sharing/git-annex ok
+
+  running: dot -Tx11 /Users/ovi/Sharing/git-annex/.git/annex/map.dot
+
+failed
+git-annex: map: 1 failed
diff --git a/doc/forum/git-annex__58___map__58___1_failed/comment_1_35ff3256e823ab8cfc53276a2123ad5f._comment b/doc/forum/git-annex__58___map__58___1_failed/comment_1_35ff3256e823ab8cfc53276a2123ad5f._comment
new file mode 100644
index 0000000000..d1beb59096
--- /dev/null
+++ b/doc/forum/git-annex__58___map__58___1_failed/comment_1_35ff3256e823ab8cfc53276a2123ad5f._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="http://joeyh.name/"
+ ip="108.236.230.124"
+ subject="comment 1"
+ date="2014-05-16T16:29:54Z"
+ content="""
+`dot` is a command from the graphviz package. Perhaps you do not have it installed.
+"""]]
diff --git a/doc/forum/git-annex__58___map__58___1_failed/comment_2_b0826073ebbd2847f4ab0d9bdba2dce5._comment b/doc/forum/git-annex__58___map__58___1_failed/comment_2_b0826073ebbd2847f4ab0d9bdba2dce5._comment
new file mode 100644
index 0000000000..b8841dd362
--- /dev/null
+++ b/doc/forum/git-annex__58___map__58___1_failed/comment_2_b0826073ebbd2847f4ab0d9bdba2dce5._comment
@@ -0,0 +1,14 @@
+[[!comment format=mdwn
+ username="https://www.google.com/accounts/o8/id?id=AItOawkAUMhKOSkh9JaBA6xst3XxQIIsDEq5Zd4"
+ nickname="Ovidiu"
+ subject="comment 2"
+ date="2014-05-16T17:21:09Z"
+ content="""
+indeed I didn't have it... was it stated somewhere that I need it?
+
+Busy installing it now... going to kill my 3g Connection :-/
+
+sudo port install graphviz
+--->  Computing dependencies for graphviz
+--->  Dependencies to be installed: autoconf m4 automake fontconfig freetype gd2 jpeg libvpx tiff xpm xorg-libXext xorg-libX11 xorg-kbproto xorg-libXau xorg-xproto xorg-libXdmcp xorg-libxcb xorg-libpthread-stubs xorg-xcb-proto xorg-xextproto xorg-libXt xorg-libsm xorg-libice gts netpbm jasper jbigkit libLASi pango Xft2 xrender xorg-renderproto cairo libpixman xorg-xcb-util gobject-introspection libtool harfbuzz graphite2 poppler curl lcms2 openjpeg15 poppler-data urw-fonts webp xorg-libXaw xorg-libXmu
+"""]]
diff --git a/doc/forum/git-annex__58___unknown_response_from_git_cat-file.mdwn b/doc/forum/git-annex__58___unknown_response_from_git_cat-file.mdwn
new file mode 100644
index 0000000000..b0bc4791fb
--- /dev/null
+++ b/doc/forum/git-annex__58___unknown_response_from_git_cat-file.mdwn
@@ -0,0 +1,10 @@
+I recently upgraded my annex from version 5 to version 6. In fact, there are two sister repos, one on an NTFS partition and the other on EXT4. I am getting the following error on the EXT4 repo:
+
+    git-annex: unknown response from git cat-file ("refs/heads/git-annex:79a/279/SHA256E-s1287921--0970d35c130c8f678fe9cd74508ee9944598aa211e410c0378833afa214ab0e9.jpg missing",Ref "refs/heads/git-annex:79a/279/SHA256E-s1287921--0970d35c130c8f678fe9cd74508ee9944598aa211e410c0378833afa214ab0e9.jpg\n.log")
+    CallStack (from HasCallStack):
+      error, called at ./Git/CatFile.hs:88:28 in main:Git.CatFile
+
+
+I see this message on the EXT4 repo if I run any usual commands like "git annex get" or "git annex list".
+
+I do not have any files with newlines in their names. I may have interrupted commands like "git add" and "git annex lock", so it's difficult to say how I ended up with the above situation. The most puzzling part is that under .git/annex/objects, the EXT4 repo has only two-letter directories while the NTFS repo has only three-letter directories, and there is no .git/annex/objects/79a/279 in either.
diff --git a/doc/forum/git-annex__58___unknown_response_from_git_cat-file/comment_1_96ad30294a6f18c0f73be1136de47632._comment b/doc/forum/git-annex__58___unknown_response_from_git_cat-file/comment_1_96ad30294a6f18c0f73be1136de47632._comment
new file mode 100644
index 0000000000..b9d423c0a3
--- /dev/null
+++ b/doc/forum/git-annex__58___unknown_response_from_git_cat-file/comment_1_96ad30294a6f18c0f73be1136de47632._comment
@@ -0,0 +1,22 @@
+[[!comment format=mdwn
+ username="joey"
+ subject="""comment 1"""
+ date="2018-08-03T17:41:16Z"
+ content="""
+I was able to reproduce the error message when I made a git-annex symlink
+that has a newline at the end of its link target. 
+
+So, it seems you must have one in your tree. Find it with:
+
+	find -ls | grep SHA256E-s1287921--0970d35c130c8f678fe9cd7
+
+Deleting the symlink that finds and replacing it with a new symlink without
+the newline at the end of its link target should fix the problem.
+
+It would be interesting to know how this symlink came to be. If you make
+another clone of the repository, do you get the same symlink? Was the file
+originally added to the NTFS repository and the bad symlink somehow came
+from there?
+
+Also, what git-annex version was used to add the bad symlink?
+"""]]
diff --git a/doc/forum/git-annex___38___ikiwiki_experiment.mdwn b/doc/forum/git-annex___38___ikiwiki_experiment.mdwn
new file mode 100644
index 0000000000..5b426a1808
--- /dev/null
+++ b/doc/forum/git-annex___38___ikiwiki_experiment.mdwn
@@ -0,0 +1,28 @@
+Hi,
+
+I've been experimenting with combining [ikiwiki](http://ikiwiki.info) with git-annex and it seems to work. Thought I'd post my process. I've [commented](http://ikiwiki.info/todo/git-annex_support/discussion/) on the ikiwiki website as well but perhaps it'd be of interest to git-annex folks.
+
+I have very little understanding of any of the tools involved and have just attempted to make it work using my limited knowledge. I don't use the web interface for ikiwiki which simplifies things.
+
+The [website in question](http://stockholm.kalleswork.net) just went online and is currently an archive of architectural photographs and the site relies heavily on the ikiwiki osm and album plugins.
+
+### Setting things up
+
+To start with I set up the wiki on the server and git clone to into `$wrkdir` on my laptop. I then initialize a git-annex repo in the `$srcdir` on the server. Leaving the `$gitdir` untouched. The `$scrdir` git-annex repo has to be in `direct` mode. Before doing any syncing I add `annex-ignore = true` and `annex-sync = false` to `.git/config` in the origin repo (`$gitdir`): this is to prevent polluting `$gitdir` with git-annex data. The same process is repeated in the `$wrkdir` on the laptop.
+
+### Pushing and syncing
+
+With this setup I can then `git add remote $srcdir`, `git add $file` and `git-push` mdwn files and other lightweight data from the laptop. While `git annex-add`, `git-annex sync` and `git-annex copy --to $srcdir` jpg's and other heavy files. All pure git commands work as expected with ikiwiki and the website recompiles etc.
+
+### Snags
+
+I'm frequently left with (non-dangling) symlinks in the `$srcdir` despite the annex repo being in direct mode. When this happens `git-annex fsck` sorts things out.
+
+Uploading image files does require a bit of manual work. But as this is done less frequently it's not much of an issue for me. I'm guessing that by doing things it the correct order (whatever that might be) I could avoid some of the manual work.
+
+The thing to keep in mind is to never `git-add` the typechanged annexed files in the $srcdir. In general I never use git commands in the $srcdir.
+
+The main problem is the symlinks though as they demand a manual `git-annex fsck`. I have no idea what causes the symlinks in a direct mode repo.
+
+Any comments? 
+
diff --git a/doc/forum/git-annex___38___ikiwiki_experiment/comment_1_9f74449ec91577dbf6095f4beafac293._comment b/doc/forum/git-annex___38___ikiwiki_experiment/comment_1_9f74449ec91577dbf6095f4beafac293._comment
new file mode 100644
index 0000000000..332f77bcc8
--- /dev/null
+++ b/doc/forum/git-annex___38___ikiwiki_experiment/comment_1_9f74449ec91577dbf6095f4beafac293._comment
@@ -0,0 +1,10 @@
+[[!comment format=mdwn
+ username="https://id.koumbit.net/anarcat"
+ ip="72.0.72.144"
+ subject="how about the web interface?"
+ date="2013-09-25T11:09:58Z"
+ content="""
+I understand you do not use the web interface - but what if you did? would it commit all those files into git?
+
+Could we add the git-annex files to a .gitignore file?
+"""]]
diff --git a/doc/forum/git-annex___38___ikiwiki_experiment/comment_2_e034585c8b51cc30b35c1f7ae68205bf._comment b/doc/forum/git-annex___38___ikiwiki_experiment/comment_2_e034585c8b51cc30b35c1f7ae68205bf._comment
new file mode 100644
index 0000000000..2201813fe4
--- /dev/null
+++ b/doc/forum/git-annex___38___ikiwiki_experiment/comment_2_e034585c8b51cc30b35c1f7ae68205bf._comment
@@ -0,0 +1,14 @@
+[[!comment format=mdwn
+ username="https://www.google.com/accounts/o8/id?id=AItOawk3HGoDpnOPob5jOjvIootmkve1-nCpRiI"
+ nickname="Kalle"
+ subject="comment 2"
+ date="2013-09-25T11:58:42Z"
+ content="""
+I have very poor understanding of what ikiwiki actually does behind the scenes including how it uses the $srcdir and $gitdir.
+
+The only way I could see the web interface working would be to use the git-annex content expressions and having the assistant running on the server. That still doesn't prevent large files from being checked into git though? That all depends on which order ikiwiki can be made to do things.
+
+I might be able to test on a local wiki but that would have to wait a while.
+
+
+"""]]
diff --git a/doc/forum/git-annex___38___ikiwiki_experiment/comment_3_fbbd47c3dbe8de24b0df664e4afd5cb8._comment b/doc/forum/git-annex___38___ikiwiki_experiment/comment_3_fbbd47c3dbe8de24b0df664e4afd5cb8._comment
new file mode 100644
index 0000000000..63919bd22d
--- /dev/null
+++ b/doc/forum/git-annex___38___ikiwiki_experiment/comment_3_fbbd47c3dbe8de24b0df664e4afd5cb8._comment
@@ -0,0 +1,14 @@
+[[!comment format=mdwn
+ username="http://joeyh.name/"
+ ip="4.152.108.220"
+ subject="comment 3"
+ date="2013-09-25T17:03:29Z"
+ content="""
+Interesting experiment.
+
+I don't know why you don't want to let git-annex sync its data to $gitdir.
+
+The symlinks could be occuring because of a bug in direct mode. (I have fixed many past bugs that caused that.) But just as likely it's because ikiwiki will run `git pull` in the srcdir.
+
+I think it would make more sense to use the underlay plugin and keep your annexed repository in a separate underlay. This would guarantee ikiwiki doesn't run git commands in there, and would ensure that nothing done with the web interface could mess with the annex.
+"""]]
diff --git a/doc/forum/git-annex___38___ikiwiki_experiment/comment_4_55da5c3c41c13b08590ce1ff8117cef6._comment b/doc/forum/git-annex___38___ikiwiki_experiment/comment_4_55da5c3c41c13b08590ce1ff8117cef6._comment
new file mode 100644
index 0000000000..6299326a89
--- /dev/null
+++ b/doc/forum/git-annex___38___ikiwiki_experiment/comment_4_55da5c3c41c13b08590ce1ff8117cef6._comment
@@ -0,0 +1,23 @@
+[[!comment format=mdwn
+ username="https://www.google.com/accounts/o8/id?id=AItOawk3HGoDpnOPob5jOjvIootmkve1-nCpRiI"
+ nickname="Kalle"
+ subject="comment 4"
+ date="2013-09-25T18:28:40Z"
+ content="""
+@Joey
+
+    > I don't know why you don't want to let git-annex sync its data to $gitdir.
+
+Well neither do I! :) It seemed to be the way to avoid duplicating data while still having the images picked up by the ikiwiki album plugin. Wouldn't the files in the $gitdir end up duplicated in $srcdir?
+
+    > The symlinks could be occuring because of a bug in direct mode. 
+    > (I have fixed many past bugs that caused that.) But just as likely
+    > it's because ikiwiki will run git pull in the srcdir.
+
+When you mention it I've had similar problems with my vfat usb annex repos. Using the post-receive merge hook to make files visible for non git-annex devices about town. Nothing I can reliably recreate but I will keep my eye out for bugs.
+
+    > I think it would make more sense to use the underlay plugin and keep 
+    > your annexed repository in a separate underlay. 
+
+Yep that would be ideal. For my usecase the album plugin is vital and I can't understand how album would pick up and deal with images in an underlay dir. This is a bit OT for this site though most of my questions are ikiwiki related.
+"""]]
diff --git a/doc/forum/git-annex___38___ikiwiki_experiment/comment_5_f67823351164ddfe7d595685c3679652._comment b/doc/forum/git-annex___38___ikiwiki_experiment/comment_5_f67823351164ddfe7d595685c3679652._comment
new file mode 100644
index 0000000000..2701bd4af1
--- /dev/null
+++ b/doc/forum/git-annex___38___ikiwiki_experiment/comment_5_f67823351164ddfe7d595685c3679652._comment
@@ -0,0 +1,10 @@
+[[!comment format=mdwn
+ username="https://www.google.com/accounts/o8/id?id=AItOawk3HGoDpnOPob5jOjvIootmkve1-nCpRiI"
+ nickname="Kalle"
+ subject="comment 5"
+ date="2013-09-26T06:56:49Z"
+ content="""
+Turns out using the underlay was a piece of cake! Just mirror the folder structure of the repo in your underlaydir put your jpegs there and off you go. Images are picked up by the album plugin and it all just works. No need to coax git-annex into doing odd stuff.
+
+Thought I'd post this for other ikiwikers.
+"""]]
diff --git a/doc/forum/git-annex___38___ikiwiki_experiment/comment_6_d5cc91164772849d027fed5f962d9000._comment b/doc/forum/git-annex___38___ikiwiki_experiment/comment_6_d5cc91164772849d027fed5f962d9000._comment
new file mode 100644
index 0000000000..1d3abdf5de
--- /dev/null
+++ b/doc/forum/git-annex___38___ikiwiki_experiment/comment_6_d5cc91164772849d027fed5f962d9000._comment
@@ -0,0 +1,10 @@
+[[!comment format=mdwn
+ username="https://id.koumbit.net/anarcat"
+ ip="72.0.72.144"
+ subject="step by step?"
+ date="2013-09-26T09:40:57Z"
+ content="""
+So how does this actually work in practice? Is the underlay directory a completely disconnected repository?
+
+Can we have step by step instructions on how to set this up?
+"""]]
diff --git a/doc/forum/git-annex___38___ikiwiki_experiment/comment_7_cb4ec7ed3c39d0649133191a85ea6ab3._comment b/doc/forum/git-annex___38___ikiwiki_experiment/comment_7_cb4ec7ed3c39d0649133191a85ea6ab3._comment
new file mode 100644
index 0000000000..8e387d89a5
--- /dev/null
+++ b/doc/forum/git-annex___38___ikiwiki_experiment/comment_7_cb4ec7ed3c39d0649133191a85ea6ab3._comment
@@ -0,0 +1,17 @@
+[[!comment format=mdwn
+ username="https://www.google.com/accounts/o8/id?id=AItOawk3HGoDpnOPob5jOjvIootmkve1-nCpRiI"
+ nickname="Kalle"
+ subject="comment 7"
+ date="2013-09-26T13:05:53Z"
+ content="""
+@anarcat
+
+    > Can we have step by step instructions on how to set this up?
+
+[I described the ikiwiki album setup](http://ikiwiki.info/forum/ikiwiki_with_album___38___underlay_plugins/) on the ikiwiki website as there are no git-annex tricks anymore. Just standard behaviour.
+
+    > So how does this actually work in practice? Is the underlay directory a 
+    > completely disconnected repository?
+
+Yes they are completely separate repos. Really the underlay dir could be managed in any way you like svn, rsync, ftp etc. I didn't expect the album plugin to pick everything up but apparently it does! Perhaps underlays are completely integrated and appear as 'normal' to all of ikiwiki?
+"""]]
diff --git a/doc/forum/git-annex___38___ikiwiki_experiment/comment_8_86565e5e1508ff1862f88975446650a2._comment b/doc/forum/git-annex___38___ikiwiki_experiment/comment_8_86565e5e1508ff1862f88975446650a2._comment
new file mode 100644
index 0000000000..473551f657
--- /dev/null
+++ b/doc/forum/git-annex___38___ikiwiki_experiment/comment_8_86565e5e1508ff1862f88975446650a2._comment
@@ -0,0 +1,12 @@
+[[!comment format=mdwn
+ username="https://id.koumbit.net/anarcat"
+ ip="72.0.72.144"
+ subject="comment 8"
+ date="2013-09-26T13:10:25Z"
+ content="""
+Yes, I believe that is the way the underlay works - the files are just picked up.
+
+As for the ikiwiki forum post - maybe you should move it to the \"tips\" section? :)
+
+Thanks for the documentation!!
+"""]]
diff --git a/doc/forum/git-annex_across_two_filesystems.mdwn b/doc/forum/git-annex_across_two_filesystems.mdwn
new file mode 100644
index 0000000000..1e95a793c6
--- /dev/null
+++ b/doc/forum/git-annex_across_two_filesystems.mdwn
@@ -0,0 +1,30 @@
+Hi everyone,
+
+I need some suggestions on how to operate git-annex best in my setup.
+
+I need git-annex mainly for its ability to have directories of all my data on all my nodes but not for the data redundancy it can provide.
+I have one node that contains 2 filesystems that I want to merge in one git-annex repository. One filesystem (lets call it SAFE) is on top of a RAID1 between two 1TB hds. The other (BIG) is on top of a 3TB hd. SAFE holds data I do not want to loose (like digital pictures). BIG holds data that I can loose.
+
+I do not have enough disk space on other nodes to get rid of the RAID1.
+
+This is how I mount my filesystems:
+
+SAFE at ~/AllData/
+
+BIG at ~/AllData/bigfiles/
+
+The root of the git repository is at ~/AllData/ however when I do:
+
+git-annex add ~/AllData/bigfiles/file1
+It says:
+add bigfiles/file1 failed
+
+I assume that is because of file1 being on a different filesystem.
+
+Do I have to create two repositories: one for each filesystem or do you have any ideas on how to use git-annex best in this scenario?
+Having two repositories also has the disadvantage that I need two repositories on all other nodes am I right?
+
+Thanks for you suggestions
+
+
+
diff --git a/doc/forum/git-annex_across_two_filesystems/comment_1_53167648b8b70b41d19ca662a5f3687e._comment b/doc/forum/git-annex_across_two_filesystems/comment_1_53167648b8b70b41d19ca662a5f3687e._comment
new file mode 100644
index 0000000000..cd363948ca
--- /dev/null
+++ b/doc/forum/git-annex_across_two_filesystems/comment_1_53167648b8b70b41d19ca662a5f3687e._comment
@@ -0,0 +1,26 @@
+[[!comment format=mdwn
+ username="http://joeyh.name/"
+ nickname="joey"
+ subject="comment 1"
+ date="2013-04-22T19:48:55Z"
+ content="""
+git-annex stores the contents of files inside `.git/annex/objects`. The `git annex add` is failing because it cannot `rename()` the file into that directory, because it is on a different filesystem. Even if it did a more expensive move of the file, it would not do what you want, because all the files would be moved to the `.git/annex/objects` directory, which is stored on your smaller drive.
+
+The way git-annex is intended to be used with multiple drives is this: 
+
+* Make a separate git repository on each drive.
+* Set up git remotes connecting these repositories together. You don't have to connect them all up, but at least make
+  the git repository on your main filesystem have a remote for each git repository on other drives.
+* Use `git annex sync` to keep the git repositories in sync. (Or do it manually with `git pull`)
+* When you want a file to be available in the local repository, use `git annex get $file` to get it.
+* When your local repository is getting too full, use `git annex drop` or `git annex move` to flush files
+  out to the other drive(s).
+
+The [[walkthrough]] goes through an example of adding a removable USB drive this way, but you can do the same thing for
+non-removable drives.
+
+> Having two repositories also has the disadvantage that I need two repositories on all other nodes am I right?
+
+No -- you combine the two repositories, so any clone of either one contains all the files in both. Other notes then only need one
+repository. However, for another node to be able to get files from both repositories on this node, it will need to have two git remotes configured, one for each repository.
+"""]]
diff --git a/doc/forum/git-annex_across_two_filesystems/comment_2_39adeebc1af9c437f1fc2e00c07509bf._comment b/doc/forum/git-annex_across_two_filesystems/comment_2_39adeebc1af9c437f1fc2e00c07509bf._comment
new file mode 100644
index 0000000000..ef57bdfec3
--- /dev/null
+++ b/doc/forum/git-annex_across_two_filesystems/comment_2_39adeebc1af9c437f1fc2e00c07509bf._comment
@@ -0,0 +1,11 @@
+[[!comment format=mdwn
+ username="https://pradermecker.myopenid.com/"
+ ip="81.244.215.244"
+ subject="comment 2"
+ date="2013-04-27T17:14:43Z"
+ content="""
+It works fine. After it is set up with the client as described, sync is automatic from the Assistant.
+
+What I find cumbersome is the fact that I need to manually call \"git annex sync\" on the remote (usb or local ssh) to view (generate) the link. Is there a way to avoid this extra step ?
+
+"""]]
diff --git a/doc/forum/git-annex_across_two_filesystems/comment_3_f4e3f28db005301adeef7ccd2c9998fb._comment b/doc/forum/git-annex_across_two_filesystems/comment_3_f4e3f28db005301adeef7ccd2c9998fb._comment
new file mode 100644
index 0000000000..52a8f55654
--- /dev/null
+++ b/doc/forum/git-annex_across_two_filesystems/comment_3_f4e3f28db005301adeef7ccd2c9998fb._comment
@@ -0,0 +1,18 @@
+[[!comment format=mdwn
+ username="https://www.google.com/accounts/o8/id?id=AItOawmp1ThsNNAbSn46ju-gwFELfStlhl8usJo"
+ nickname="Marek"
+ subject="I should have thought of this"
+ date="2013-04-30T19:55:19Z"
+ content="""
+Thanks joey, I thought of the object storage just yesterday and that the drive where it resides needs to be the big one. 
+
+I think that I will do it as you described with one addition:
+
+I set up the git-annex repository on my big drive. Lets call that BIG.
+Then I set up another on the smaller drive (or raid) and set both to sync. Lets call that RAID.
+
+BUT: Can I use num copies to tell BIG to only sync files in certain directories to RAID. Or will syncing sync everything regardless?
+
+Thanks for your help
+
+"""]]
diff --git a/doc/forum/git-annex_across_two_filesystems/comment_4_53fa7ac6f80e3281768a7bfd3d438b34._comment b/doc/forum/git-annex_across_two_filesystems/comment_4_53fa7ac6f80e3281768a7bfd3d438b34._comment
new file mode 100644
index 0000000000..bab1a8b02c
--- /dev/null
+++ b/doc/forum/git-annex_across_two_filesystems/comment_4_53fa7ac6f80e3281768a7bfd3d438b34._comment
@@ -0,0 +1,10 @@
+[[!comment format=mdwn
+ username="https://www.google.com/accounts/o8/id?id=AItOawmp1ThsNNAbSn46ju-gwFELfStlhl8usJo"
+ nickname="Marek"
+ subject="Another correction"
+ date="2013-04-30T20:11:57Z"
+ content="""
+sync just syncs the meta data but not the objects itself right? I have to do an explicit git-annex copy -to or copy -from?
+
+
+"""]]
diff --git a/doc/forum/git-annex_across_two_filesystems/comment_5_2e1be54c01970ef3456e8af4aaf00cbf._comment b/doc/forum/git-annex_across_two_filesystems/comment_5_2e1be54c01970ef3456e8af4aaf00cbf._comment
new file mode 100644
index 0000000000..83644a5548
--- /dev/null
+++ b/doc/forum/git-annex_across_two_filesystems/comment_5_2e1be54c01970ef3456e8af4aaf00cbf._comment
@@ -0,0 +1,10 @@
+[[!comment format=mdwn
+ username="http://joeyh.name/"
+ nickname="joey"
+ subject="comment 5"
+ date="2013-04-30T21:57:13Z"
+ content="""
+Yes, `git annex sync` only syncs the metadata. 
+
+If you have numcopies set to to, and run `git annex copy --to RAID --auto`, it will only copy files that have less than 2 copies. (Same with `git annex copy --from BIG --auto` or `git annex get --auto`)
+"""]]
diff --git a/doc/forum/git-annex_across_two_filesystems/comment_6_aa6b153478f79c78e778865a99f01373._comment b/doc/forum/git-annex_across_two_filesystems/comment_6_aa6b153478f79c78e778865a99f01373._comment
new file mode 100644
index 0000000000..4e05541ace
--- /dev/null
+++ b/doc/forum/git-annex_across_two_filesystems/comment_6_aa6b153478f79c78e778865a99f01373._comment
@@ -0,0 +1,21 @@
+[[!comment format=mdwn
+ username="thomas.l.clune@39f0ca5621ddc6cbe4550be3ca9a22ec70edc9a6"
+ nickname="thomas.l.clune"
+ avatar="http://cdn.libravatar.org/avatar/31edb934c0c30a68a44617fa0eea0f8f"
+ subject="git-annex - many filesystems"
+ date="2017-01-30T14:24:11Z"
+ content="""
+I believe I understood the earlier discussion about managing git-annex across multiple file-systems, but I'm facing a more extreme case and am hoping for some useful advice.
+
+We are exploring use of git-annex to manage the large boundary conditions used within our weather model.     For the most part, this seems to be an excellent fit.   But one of our goals is to limit data duplication among end users, and here there is a problem.   Even in the era of multi-petabyte file systems, needless duplication of data s a concern.
+
+In our computing facility there are many O(10-20) filesystems from which users run our model.    If each user creates their own clone from some master repo on filesystem A, many (but by no means all) files will have multiple copies across the other filesystems.   Worse, if multiple users running on filesystem X both cloned from A, we'd have multiple copies even on X as they would not know about the other local clone.    The current ad hoc system (i.e., without git-annex)  uses symbolic links to file system A and avoids the copies, but is otherwise quite limited in terms of managing variant input files and running in other computing environments.      
+
+One halfway solution would be to have a \"master\" repo on _each_ filesystem and then ensure that users made their repos set upstream remotes to _all_ of them.    We'd still have multiple copies across filesystems, but at least there would then be at most one copy of any given file on each filesystem.
+
+The other possibility is to somehow enforce a policy that all input files are to be accessed on filesystem A.      This is what we'll probably end up doing.   We'd kludge our existing scripts that set up symbolic links to point at clones on filesystem A.
+
+I would very much like there to be another option.
+
+Thanks in advance.
+"""]]
diff --git a/doc/forum/git-annex_across_two_filesystems/comment_7_b87281c6a8b068cec60d2d5b7d015a23._comment b/doc/forum/git-annex_across_two_filesystems/comment_7_b87281c6a8b068cec60d2d5b7d015a23._comment
new file mode 100644
index 0000000000..9de9286e3a
--- /dev/null
+++ b/doc/forum/git-annex_across_two_filesystems/comment_7_b87281c6a8b068cec60d2d5b7d015a23._comment
@@ -0,0 +1,25 @@
+[[!comment format=mdwn
+ username="joey"
+ subject="""comment 7"""
+ date="2017-01-30T17:54:47Z"
+ content="""
+Hmm, this is a rather old forum thread to be reactivating, maybe a new
+thread more about your specific use case, Thomas?
+
+Your master repository idea seems like a good one. If the master
+repository is cloned with `git clone --shared` then that clone will
+hard link files between it and the master repository (assuming
+a git-annex rather newer than the start date of this forum thread!), 
+so multiple repositories will only have one copy of the file. 
+Of course, since it uses hard links, master and clone need to
+be on the same drive.
+
+There are probably ways to improve git-annex to handle this kind of use
+case better. Maybe `git clone --shared` across filesystems should
+use symlinks rather than hard links or something like that.
+That might take some time to design and implement (it changes a core
+invariant of git-annex, that .git/annex/objects/ contains files, not
+symlinks).
+
+Happy to discuss other options..
+"""]]
diff --git a/doc/forum/git-annex_add_out_of_memory_error.mdwn b/doc/forum/git-annex_add_out_of_memory_error.mdwn
new file mode 100644
index 0000000000..594427d25e
--- /dev/null
+++ b/doc/forum/git-annex_add_out_of_memory_error.mdwn
@@ -0,0 +1,38 @@
+Greetings all!
+
+I have only been using git-annex for a few days, but I am loving it so far.
+
+I have run into one problem:
+
+At the top of a directory tree with some very large (GB+) blobs, I issue this command:
+
+find . -size +5M  -exec git-annex add --debug {} \;  2>&1 | tee -a ./annex-add.log
+
+When the command completes, I see a number of these types of errors in the log:
+
+    [2015-10-05 21:14:13.502615] chat: git ["--git-dir=.git","--work-tree=.","--literal-pathspecs","cat-file","--batch"]
+    [2015-10-05 21:14:13.521586] chat: git ["--git-dir=.git","--work-tree=.","--literal-pathspecs","cat-file","--batch"]
+    [2015-10-05 21:14:13.533095] read: git ["--git-dir=.git","--work-tree=.","--literal-pathspecs","diff","--name-only","--diff-filter=T","-z","--","./application/mozilla/firefox/linux/firefox-39.0/media/firefox-39.0.tar.bz2"]
+    [2015-10-05 21:14:13.545739] feed: xargs ["-0","git","--git-dir=.git","--work-tree=.","--literal-pathspecs","add","--"]
+    [2015-10-05 21:14:13.562727] process done ExitSuccess
+    [2015-10-05 21:14:13.564103] chat: git ["--git-dir=.git","--work-tree=.","--literal-pathspecs","hash-object","-w","--stdin-paths","--no-filters"]
+    [2015-10-05 21:14:13.56616] feed: git ["--git-dir=.git","--work-tree=.","--literal-pathspecs","update-index","-z","--index-info"]
+    [2015-10-05 21:14:13.58032] process done ExitSuccess
+    fatal: Out of memory, getdelim failed
+    [2015-10-05 21:14:13.582109] process done ExitFailure 128
+    ok
+    (recording state in git...)
+    git-annex: user error (git ["--git-dir=.git","--work-tree=.","--literal-pathspecs","hash-object","-w","--stdin-paths","--no-filters"] exited 128)
+
+I've googled around and searched the forums/bug reports here without much success.
+I have tried setting the queuesize to smaller values, but that does not seem to help.
+
+I am using git-annex version: 5.20150930-g40fdbe9 on RHEL 6.
+
+I am happy to help debug further...
+
+Thanks,
+
+JC
+
+P.S. - Is it normal that some links in the .git dir are created without user write access?
diff --git a/doc/forum/git-annex_add_out_of_memory_error/comment_10_d90d392db9a4962f9f8eb0ee95bf2914._comment b/doc/forum/git-annex_add_out_of_memory_error/comment_10_d90d392db9a4962f9f8eb0ee95bf2914._comment
new file mode 100644
index 0000000000..3aacef337a
--- /dev/null
+++ b/doc/forum/git-annex_add_out_of_memory_error/comment_10_d90d392db9a4962f9f8eb0ee95bf2914._comment
@@ -0,0 +1,24 @@
+[[!comment format=mdwn
+ username="yarikoptic"
+ avatar="http://cdn.libravatar.org/avatar/f11e9c84cb18d26a1748c33b48c924b4"
+ subject="relates to being on NFS"
+ date="2017-08-09T17:26:40Z"
+ content="""
+pretty sure in that 
+
+[[! format sh \"\"\"
+yhalchen@discovery:/mnt/scratch/yoh/datalad$ git pull --ff-only origin master
+fatal: Out of memory, getdelim failed
+error: git://github.com/datalad/datalad did not send all necessary objects
+
+yhalchen@discovery:/mnt/scratch/yoh/datalad$ git pull --ff-only origin master
+fatal: Out of memory, getdelim failed
+error: git://github.com/datalad/datalad did not send all necessary objects
+
+yhalchen@discovery:/mnt/scratch/yoh/datalad$ git pull --ff-only origin master
+From git://github.com/datalad/datalad
+ * branch              master     -> FETCH_HEAD
+fatal: Out of memory, getdelim failed
+
+\"\"\"]]
+"""]]
diff --git a/doc/forum/git-annex_add_out_of_memory_error/comment_11_459f810fd06aab3ddeeba8207d774dc0._comment b/doc/forum/git-annex_add_out_of_memory_error/comment_11_459f810fd06aab3ddeeba8207d774dc0._comment
new file mode 100644
index 0000000000..2907187a7b
--- /dev/null
+++ b/doc/forum/git-annex_add_out_of_memory_error/comment_11_459f810fd06aab3ddeeba8207d774dc0._comment
@@ -0,0 +1,21 @@
+[[!comment format=mdwn
+ username="joey"
+ subject="""comment 11"""
+ date="2017-08-15T18:00:42Z"
+ content="""
+Yes, I think that the git-annex-standalone.deb is potentially broken
+by mismatches between the system locales and the bundled version of glibc.
+
+The git-annex standalone tarball was designed to be unpacked by the user
+who's going to use it; having root install it is not using it as it was
+designed. If you're going to have root install something, why not have them
+install a docker container or some other container format designed to be
+installed by root, that can completely isolate the application inside it?
+
+(As to the getdelim error specifically, here's a link to the thread where
+that bug in git was tracked down and patched
+.
+I'm still not clear on how locale issues might contribute to the getdelim
+error. It seems like something before the getdelim call is failing in this
+situation for whatever reason, and that might be the actual root cause.)
+"""]]
diff --git a/doc/forum/git-annex_add_out_of_memory_error/comment_12_d7d3785b65b0608424b25056c9e3a139._comment b/doc/forum/git-annex_add_out_of_memory_error/comment_12_d7d3785b65b0608424b25056c9e3a139._comment
new file mode 100644
index 0000000000..0f5299be7a
--- /dev/null
+++ b/doc/forum/git-annex_add_out_of_memory_error/comment_12_d7d3785b65b0608424b25056c9e3a139._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="davicastro"
+ avatar="http://cdn.libravatar.org/avatar/4e708663cf4d5b9e8cfea74caf4307fc"
+ subject="Consequences of this error message"
+ date="2018-06-04T14:16:23Z"
+ content="""
+Hi, can someone comment what is the consequences when the \"fatal: Out of memory, getdelim failed\" occurs? It happened to me in some files when adding a bunch but I don't see any side effects. Git fsck and Git annex fsck are OK. git annex unused makes sense. Could this error cause a file to not be added to the repository? (I'm not using NFS file system, git annex version: 6.20180409-gfb0780266, git version: 2.14.1. It might be due to actual out of memory, I'm not sure, I'm just concerned about the side effects).
+"""]]
diff --git a/doc/forum/git-annex_add_out_of_memory_error/comment_1_a2331032781c2fa3cdf22c0dc6b5b869._comment b/doc/forum/git-annex_add_out_of_memory_error/comment_1_a2331032781c2fa3cdf22c0dc6b5b869._comment
new file mode 100644
index 0000000000..e4207f4f96
--- /dev/null
+++ b/doc/forum/git-annex_add_out_of_memory_error/comment_1_a2331032781c2fa3cdf22c0dc6b5b869._comment
@@ -0,0 +1,19 @@
+[[!comment format=mdwn
+ username="joey"
+ subject="""comment 1"""
+ date="2015-10-06T19:31:44Z"
+ content="""
+This message about "getdelim" comes from git, not git-annex. It seems
+git hash-object is what's running out of memory, and probably
+when it's processing its input from stdin.
+
+That's pretty strange. git-annex just passes some filenames to git
+hash-object. Unless the system is for some reason very low on memory,
+it shouldn't choke on such a small input as a filename.
+
+This seems likely to be a git bug, so what version of git? Perhaps
+try a newer one..
+
+(Re: P.S.: It's normal for files in .git/annex/objects to not
+have write access.)
+"""]]
diff --git a/doc/forum/git-annex_add_out_of_memory_error/comment_2_de9d956d95cb41e2372eabbf672e0f67._comment b/doc/forum/git-annex_add_out_of_memory_error/comment_2_de9d956d95cb41e2372eabbf672e0f67._comment
new file mode 100644
index 0000000000..0d8fb4bccf
--- /dev/null
+++ b/doc/forum/git-annex_add_out_of_memory_error/comment_2_de9d956d95cb41e2372eabbf672e0f67._comment
@@ -0,0 +1,18 @@
+[[!comment format=mdwn
+ username="jec@8edf298ac8f8152bffcd4fdc1a310949d3547fe4"
+ nickname="jec"
+ subject="Git Version"
+ date="2015-10-07T20:24:05Z"
+ content="""
+Thank for the response Joey...
+
+Here's my git version:
+
+    [xxx@infrastructure ~]$ which git
+    /opt/git-annex.linux/git
+    [xxx@infrastructure ~]$ /opt/git-annex.linux/git --version
+    git version 2.6.0
+
+I've doubled the amount of RAM on that VM to 8GB - I'll see what happens when I try another run.
+
+"""]]
diff --git a/doc/forum/git-annex_add_out_of_memory_error/comment_3_91af8300d640c34ff2466c89ec7a234c._comment b/doc/forum/git-annex_add_out_of_memory_error/comment_3_91af8300d640c34ff2466c89ec7a234c._comment
new file mode 100644
index 0000000000..603409b90a
--- /dev/null
+++ b/doc/forum/git-annex_add_out_of_memory_error/comment_3_91af8300d640c34ff2466c89ec7a234c._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="joey"
+ subject="""comment 3"""
+ date="2016-09-29T15:33:00Z"
+ content="""
+Had another report of this, and there NFS seemed to be involved in the
+circumstances of the crash.
+"""]]
diff --git a/doc/forum/git-annex_add_out_of_memory_error/comment_4_ef913a45e3d865d088cf11460cd88020._comment b/doc/forum/git-annex_add_out_of_memory_error/comment_4_ef913a45e3d865d088cf11460cd88020._comment
new file mode 100644
index 0000000000..e9b52c3c28
--- /dev/null
+++ b/doc/forum/git-annex_add_out_of_memory_error/comment_4_ef913a45e3d865d088cf11460cd88020._comment
@@ -0,0 +1,14 @@
+[[!comment format=mdwn
+ username="joey"
+ subject="""comment 4"""
+ date="2016-10-04T16:23:12Z"
+ content="""
+NFS was a red herring.
+
+We've tracked this down to the linux standalone tarball not including
+locale-archive, so it uses the system one, which may have a different
+format and so breaks glibc.
+
+I'm committing a fix that adds locale-archive to the linux standalone
+tarball.
+"""]]
diff --git a/doc/forum/git-annex_add_out_of_memory_error/comment_5_64008199df70f77ab64882339aa079f9._comment b/doc/forum/git-annex_add_out_of_memory_error/comment_5_64008199df70f77ab64882339aa079f9._comment
new file mode 100644
index 0000000000..85eb8c0fc5
--- /dev/null
+++ b/doc/forum/git-annex_add_out_of_memory_error/comment_5_64008199df70f77ab64882339aa079f9._comment
@@ -0,0 +1,11 @@
+[[!comment format=mdwn
+ username="yarikoptic"
+ avatar="http://cdn.libravatar.org/avatar/f11e9c84cb18d26a1748c33b48c924b4"
+ subject="issue seems to be out there"
+ date="2017-07-31T15:03:41Z"
+ content="""
+[https://github.com/datalad/datalad/issues/1678](https://github.com/datalad/datalad/issues/1678)
+git annex standalone is quite recent so those manpage workarounds should be in place
+
+
+"""]]
diff --git a/doc/forum/git-annex_add_out_of_memory_error/comment_5_6a8580087688b4bbfc3241292df5422e._comment b/doc/forum/git-annex_add_out_of_memory_error/comment_5_6a8580087688b4bbfc3241292df5422e._comment
new file mode 100644
index 0000000000..bd579e381a
--- /dev/null
+++ b/doc/forum/git-annex_add_out_of_memory_error/comment_5_6a8580087688b4bbfc3241292df5422e._comment
@@ -0,0 +1,10 @@
+[[!comment format=mdwn
+ username="joey"
+ subject="""comment 5"""
+ date="2017-07-31T18:42:38Z"
+ content="""
+The locale generation done by the standalone tarball needs write access to
+its directory. If datalad is using the git-annex-standalone.deb, non-root
+user's won't be able to write to it, and it will still use the system
+locales.
+"""]]
diff --git a/doc/forum/git-annex_add_out_of_memory_error/comment_8_76a65b4edf4f6e3896f1593a197f24eb._comment b/doc/forum/git-annex_add_out_of_memory_error/comment_8_76a65b4edf4f6e3896f1593a197f24eb._comment
new file mode 100644
index 0000000000..134a832b3c
--- /dev/null
+++ b/doc/forum/git-annex_add_out_of_memory_error/comment_8_76a65b4edf4f6e3896f1593a197f24eb._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="yarikoptic"
+ avatar="http://cdn.libravatar.org/avatar/f11e9c84cb18d26a1748c33b48c924b4"
+ subject="comment 8"
+ date="2017-08-01T02:59:30Z"
+ content="""
+so does it mean we need to do some \"locale\" operation on those files while generating standalone .deb package?  or that syadmin installing git-annex standalone package should do before allowing users to use that git-annex installation where they would not have R/W access to git-annex installation?
+"""]]
diff --git a/doc/forum/git-annex_add_out_of_memory_error/comment_8_ffbb059664541c92dcbf97c150d9b951._comment b/doc/forum/git-annex_add_out_of_memory_error/comment_8_ffbb059664541c92dcbf97c150d9b951._comment
new file mode 100644
index 0000000000..3b8a23c926
--- /dev/null
+++ b/doc/forum/git-annex_add_out_of_memory_error/comment_8_ffbb059664541c92dcbf97c150d9b951._comment
@@ -0,0 +1,42 @@
+[[!comment format=mdwn
+ username="yarikoptic"
+ avatar="http://cdn.libravatar.org/avatar/f11e9c84cb18d26a1748c33b48c924b4"
+ subject=""reproduces" in a complete git installation "
+ date="2017-08-09T17:07:17Z"
+ content="""
+Installed a stretch based singularity container on a Centos 6.9 with 2.6.32-696.6.3.el6.x86_64 kernel and within it, I reliably get the message
+
+[[!format sh \"\"\"
+yhalchen@discovery:~/datalad$ git pull --ff-only origin master
+From git://github.com/datalad/datalad
+ * branch              master     -> FETCH_HEAD
+fatal: Out of memory, getdelim failed
+yhalchen@discovery:~/datalad$ echo $?
+128
+yhalchen@discovery:~/datalad$ git --version
+git version 2.11.0
+yhalchen@discovery:~/datalad$ which git
+/usr/bin/git
+yhalchen@discovery:~/datalad$ apt-cache policy git
+git:
+  Installed: 1:2.11.0-3
+  Candidate: 1:2.11.0-3
+  Version table:
+ *** 1:2.11.0-3 500
+        500 http://smaug.datalad.org:3142/debian stretch/main amd64 Packages
+        100 /var/lib/dpkg/status
+
+\"\"\"]]
+also note that message might come not the last one in the output (initial run for the command was different):
+
+[[!format sh \"\"\"
+yhalchen@discovery:~/datalad$ git pull --ff-only
+remote: Counting objects: 19, done.
+remote: Compressing objects: 100% (6/6), done.
+remote: Total 19 (delta 13), reused 19 (delta 13), pack-reused 0
+Unpacking objects: 100% (19/19), done.
+fatal: Out of memory, getdelim failed
+error: git://github.com/datalad/datalad did not send all necessary objects
+
+\"\"\"]]
+"""]]
diff --git a/doc/forum/git-annex_add_out_of_memory_error/comment_9_7ef671c0036c77ee5707a6e00dd4dd6a._comment b/doc/forum/git-annex_add_out_of_memory_error/comment_9_7ef671c0036c77ee5707a6e00dd4dd6a._comment
new file mode 100644
index 0000000000..7e2a167910
--- /dev/null
+++ b/doc/forum/git-annex_add_out_of_memory_error/comment_9_7ef671c0036c77ee5707a6e00dd4dd6a._comment
@@ -0,0 +1,43 @@
+[[!comment format=mdwn
+ username="yarikoptic"
+ avatar="http://cdn.libravatar.org/avatar/f11e9c84cb18d26a1748c33b48c924b4"
+ subject="comes and goes!"
+ date="2017-08-09T17:11:37Z"
+ content="""
+ha -- and then later call succeeded! (damn... right before I was trying to debug it)
+
+[[!format sh \"\"\"
+yhalchen@discovery:~/datalad$ git pull --ff-only origin master
+From git://github.com/datalad/datalad
+ * branch              master     -> FETCH_HEAD
+Updating c0cd538d..39f80454
+
+Fast-forward
+ CHANGELOG.md                              |  16 ++++
+ CONTRIBUTING.md                           |   2 +-
+ COPYING                                   |   3 +-
+ datalad/__init__.py                       |  30 ++++---
+ datalad/distribution/tests/test_update.py |   4 +-
+ datalad/distribution/update.py            |   3 +
+ datalad/interface/test.py                 |  33 +++++++-
+ datalad/support/annexrepo.py              |  42 +++++++---
+ datalad/support/third/noseclasses.py      | 339 +++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++
+ datalad/support/third/nosetester.py       | 450 ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++
+ datalad/tests/test_utils.py               |  13 +++
+ datalad/utils.py                          |  15 ++++
+ datalad/version.py                        |   2 +-
+ 13 files changed, 923 insertions(+), 29 deletions(-)
+ create mode 100644 datalad/support/third/noseclasses.py
+ create mode 100644 datalad/support/third/nosetester.py
+\"\"\"]]
+but comes back anyways ;)
+[[!format sh \"\"\"
+yhalchen@discovery:~/datalad$ git branch
+* (HEAD detached from 0.8.0)
+  master
+yhalchen@discovery:~/datalad$ git pull --ff-only origin master
+From git://github.com/datalad/datalad
+ * branch              master     -> FETCH_HEAD
+fatal: Out of memory, getdelim failed
+\"\"\"]]
+"""]]
diff --git a/doc/forum/git-annex_and_bup_-_the_other_way_around___63____63____63__.mdwn b/doc/forum/git-annex_and_bup_-_the_other_way_around___63____63____63__.mdwn
new file mode 100644
index 0000000000..ba005e37a8
--- /dev/null
+++ b/doc/forum/git-annex_and_bup_-_the_other_way_around___63____63____63__.mdwn
@@ -0,0 +1,28 @@
+I've been wondering about using `git-annex` and `bup` together - but _not_ with `bup` as the backend, but rather backing up `bup` repos using `git-annex`.
+
+Let me try to explain...
+
+* `bup` is a a great deduplicating backup tool, but it does not have encryption
+* `git-annex` is a awesome in so many ways.  Including 1) multiple copies, 2) encryption
+
+(I know the following reads like the motivation for `git-annex`, but let me add the word **backup**) 
+
+* Recovering large backups over the internet can be costly and time consuming
+* Local copies are fast, but are risky
+
+So I was wondering about having my `bup` repos in `git-annex`, with multiple copies, including, say, an encrypted S3 bucket and some local copies.
+
+Then, if I had a problem and needed to restore I could use my local copies for as much as I could and then only pull part of the backup from the complete remote backup.
+
+If that all works, I then have:
+
+1. A more complicated process than a simple backup tool  :-(
+2. Multiple complete backups available :-D
+3. Encrypted, offsite backups :-D
+4. Small transfers (`bup` uses the awesomness of `git` to dedup the hell out of your data) :-D
+
+---
+
+I've not done this yet, but was thinking it through.
+
+Can anyone share some opinions, thoughts, concerns or high-5s for the awesomeness of my idea?  ;-)
diff --git a/doc/forum/git-annex_and_bup_-_the_other_way_around___63____63____63__/comment_1_0bb5e21b4cbaaaa61fbdb2e0066f0ebf._comment b/doc/forum/git-annex_and_bup_-_the_other_way_around___63____63____63__/comment_1_0bb5e21b4cbaaaa61fbdb2e0066f0ebf._comment
new file mode 100644
index 0000000000..ae1721e9db
--- /dev/null
+++ b/doc/forum/git-annex_and_bup_-_the_other_way_around___63____63____63__/comment_1_0bb5e21b4cbaaaa61fbdb2e0066f0ebf._comment
@@ -0,0 +1,12 @@
+[[!comment format=mdwn
+ username="m8r-568niv@9cefd6353b1102081f43c2f5bc53afdddc153274"
+ nickname="m8r-568niv"
+ subject="comment 1"
+ date="2016-06-17T07:07:10Z"
+ content="""
+I've posted to the bup list.
+
+Some positive comments:
+
+[https://groups.google.com/d/topic/bup-list/ReIZxGpTQCo/discussion](https://groups.google.com/d/topic/bup-list/ReIZxGpTQCo/discussion)
+"""]]
diff --git a/doc/forum/git-annex_and_bup_-_the_other_way_around___63____63____63__/comment_2_ab6420f7bc18e8143201c603c09a0d7e._comment b/doc/forum/git-annex_and_bup_-_the_other_way_around___63____63____63__/comment_2_ab6420f7bc18e8143201c603c09a0d7e._comment
new file mode 100644
index 0000000000..e63f26c495
--- /dev/null
+++ b/doc/forum/git-annex_and_bup_-_the_other_way_around___63____63____63__/comment_2_ab6420f7bc18e8143201c603c09a0d7e._comment
@@ -0,0 +1,10 @@
+[[!comment format=mdwn
+ username="m8r-568niv@9cefd6353b1102081f43c2f5bc53afdddc153274"
+ nickname="m8r-568niv"
+ subject="comment 2"
+ date="2016-06-19T22:56:17Z"
+ content="""
+Looks like I missed an obvious link/reference:
+
+[https://git-annex.branchable.com/tips/Bup_repositories_in_git-annex/](https://git-annex.branchable.com/tips/Bup_repositories_in_git-annex/)
+"""]]
diff --git a/doc/forum/git-annex_and_tagfs.mdwn b/doc/forum/git-annex_and_tagfs.mdwn
new file mode 100644
index 0000000000..1ef40aea1c
--- /dev/null
+++ b/doc/forum/git-annex_and_tagfs.mdwn
@@ -0,0 +1,14 @@
+Hi,
+
+Thanks for git-annex, really a great project.
+Another related feature which would be useful to have is described by tagfs [3][] [4][], experimental implementations such as [tagfs over fuse][1], and [tagsistant][2] exist, but having a solution within the power of git-annex would really be attractive.
+
+How hard would this be to implement within the existing infrastructure?
+Thanks.
+
+related post: [[multiple_sym_links___40__for_tagging_photos__41____63__]]
+
+[1]: http://code.google.com/p/tagfs/
+[2]: www.tagsistant.net/
+[3]: http://site.xam.de/2006/01-tagfs.pdf
+[4]: http://www.google.com/url?sa=t&rct=j&q=&esrc=s&source=web&cd=6&ved=0CE4QFjAF&url=http%3A%2F%2Fweb.mit.edu%2F6.033%2F2011%2Fwwwdocs%2Fwriting-samples%2Fsbezek_dp1.pdf&ei=ILQkUdqxMueR0QWgx4CwDw&usg=AFQjCNE1eWeFxmaxzOLZYVsb0tomqWNQaw&bvm=bv.42661473,d.d2k
diff --git a/doc/forum/git-annex_and_tagfs/comment_1_887c74cb61d30198322ef74ebc80f950._comment b/doc/forum/git-annex_and_tagfs/comment_1_887c74cb61d30198322ef74ebc80f950._comment
new file mode 100644
index 0000000000..95739bd93f
--- /dev/null
+++ b/doc/forum/git-annex_and_tagfs/comment_1_887c74cb61d30198322ef74ebc80f950._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="http://joeyh.name/"
+ ip="4.152.108.210"
+ subject="comment 1"
+ date="2013-02-24T21:52:41Z"
+ content="""
+I welcome people doing this kind of thing with git-annex. Since you can have multiple files linking to the same content easily, it should be a reasonable platform with which to build something like this.
+"""]]
diff --git a/doc/forum/git-annex_and_tagfs/comment_2_8769d6e57061023c0828af3e38faafba._comment b/doc/forum/git-annex_and_tagfs/comment_2_8769d6e57061023c0828af3e38faafba._comment
new file mode 100644
index 0000000000..2bb4ec273e
--- /dev/null
+++ b/doc/forum/git-annex_and_tagfs/comment_2_8769d6e57061023c0828af3e38faafba._comment
@@ -0,0 +1,7 @@
+[[!comment format=mdwn
+ username="exaos"
+ subject="Be eager for the TagFS features on git-annex."
+ date="2014-12-09T01:15:23Z"
+ content="""
+Cann't wait to see these implemented. But I don't know programming. :-(
+"""]]
diff --git a/doc/forum/git-annex_and_tagfs/comment_3_d96948c444f9f485a329420b7c9e3b16._comment b/doc/forum/git-annex_and_tagfs/comment_3_d96948c444f9f485a329420b7c9e3b16._comment
new file mode 100644
index 0000000000..4bc1ccc1f9
--- /dev/null
+++ b/doc/forum/git-annex_and_tagfs/comment_3_d96948c444f9f485a329420b7c9e3b16._comment
@@ -0,0 +1,7 @@
+[[!comment format=mdwn
+ username="Xyem"
+ subject="comment 3"
+ date="2014-12-15T08:38:58Z"
+ content="""
+Do the [metadata views](http://git-annex.branchable.com/tips/metadata_driven_views/) not do what you want?
+"""]]
diff --git a/doc/forum/git-annex_assistant_atempts_to_repair__44___but_never_finishes.mdwn b/doc/forum/git-annex_assistant_atempts_to_repair__44___but_never_finishes.mdwn
new file mode 100644
index 0000000000..75ba03dd32
--- /dev/null
+++ b/doc/forum/git-annex_assistant_atempts_to_repair__44___but_never_finishes.mdwn
@@ -0,0 +1 @@
+I'm using git-annex (git-annex-6.20160126 from homebrew) on osx and for a while now, it claims that it tries to repair my annex-dir on my osx machine, even though it never finishes (or fails) to do so. I'm not sure what's causing this and I also tried to use `git annex repair`, but this didn't help either (it ran forever but never finishes). I tried to run `git annex fsck` and saw that some larger video files were only in one other place and therefore couldn't be droped, which I think I fixed by doing `git annex add` then `git annex sync` and finally `git annex copy --auto -to $Remote`. I'm not sure how to proceed, the log file doesn't seem to tell me a whole lot either. It just tells me for some of my files "Unable to lock down 1 copy of file that is required to safely drop it." and tries to repair that stuff over and over and over… or at least that's what I guess happens.
diff --git a/doc/forum/git-annex_assistant_atempts_to_repair__44___but_never_finishes/comment_1_a12892b2881969ddd6faa92d860e1bc8._comment b/doc/forum/git-annex_assistant_atempts_to_repair__44___but_never_finishes/comment_1_a12892b2881969ddd6faa92d860e1bc8._comment
new file mode 100644
index 0000000000..92a3f70b7d
--- /dev/null
+++ b/doc/forum/git-annex_assistant_atempts_to_repair__44___but_never_finishes/comment_1_a12892b2881969ddd6faa92d860e1bc8._comment
@@ -0,0 +1,7 @@
+[[!comment format=mdwn
+ username="openmedi"
+ subject="comment 1"
+ date="2016-03-19T17:08:19Z"
+ content="""
+I added an old external hdd to my setup and now it seems to be working again. I assume, that a lot of my remotes stoped working, or only working sometimes. I guess I have to use `testremote` and go from there…
+"""]]
diff --git a/doc/forum/git-annex_assistant_with_2_dedicated_servers.mdwn b/doc/forum/git-annex_assistant_with_2_dedicated_servers.mdwn
new file mode 100644
index 0000000000..1544b14eff
--- /dev/null
+++ b/doc/forum/git-annex_assistant_with_2_dedicated_servers.mdwn
@@ -0,0 +1,11 @@
+Hello,
+
+I am looking for a solution to replicate lots of files between 2 servers (with dedicated IP address) over WAN. (I already looked at GlusterFS, DRBD, Bittorent-Sync and ownCloud, but comments are welcome). Now I am evaluating git-annex but it is (kind of) hard to get the concepts as I am not familiar with git.
+
+I could already connect the servers by following the steps of "remote sharing walkthrough" but I feel that using a 3rd special node (as "transfer repository" / rsync or ssh) is an overhead and should not be needed. But no matter how hard I tried, I couldn't make the 2 servers do a sync without a special node. 
+
+Could you please give me a hint how to do it? Or even better, some detailed steps. :)
+
+Thanks,
+
+David
diff --git a/doc/forum/git-annex_assistant_with_2_dedicated_servers/comment_10_533ade2215c879cd46782fd66a97b167._comment b/doc/forum/git-annex_assistant_with_2_dedicated_servers/comment_10_533ade2215c879cd46782fd66a97b167._comment
new file mode 100644
index 0000000000..ec7dc0f861
--- /dev/null
+++ b/doc/forum/git-annex_assistant_with_2_dedicated_servers/comment_10_533ade2215c879cd46782fd66a97b167._comment
@@ -0,0 +1,16 @@
+[[!comment format=mdwn
+ username="David"
+ ip="178.21.48.117"
+ subject="comment 10"
+ date="2013-11-16T23:15:49Z"
+ content="""
+Great. Understood. :) Thanks.
+
+Although there is one more thing that I don't really understand. Why do we need 2 repositories. In my mind (without knowing git) I thout it would work with one repository, where both server1 and server2 can upload/download files and they got synced through this. 
+
+    server1 <-> repository (somewhere, eother server1 or server2) <-> server2
+
+Now they use different repositories and I don't get why. Also I don't get how conflict-handling can be done if there are 2 repositories for the 2 transfer-ways.
+
+Sorry to bother you with this.
+"""]]
diff --git a/doc/forum/git-annex_assistant_with_2_dedicated_servers/comment_11_c9ae51d7b772cf7a91d90925f74d2b60._comment b/doc/forum/git-annex_assistant_with_2_dedicated_servers/comment_11_c9ae51d7b772cf7a91d90925f74d2b60._comment
new file mode 100644
index 0000000000..f614d066dc
--- /dev/null
+++ b/doc/forum/git-annex_assistant_with_2_dedicated_servers/comment_11_c9ae51d7b772cf7a91d90925f74d2b60._comment
@@ -0,0 +1,10 @@
+[[!comment format=mdwn
+ username="http://joeyh.name/"
+ ip="209.250.56.64"
+ subject="comment 11"
+ date="2013-11-16T23:25:26Z"
+ content="""
+When using git, each place you access your files is a separate repository. Thus, you have:
+
+server1 (repository) <--> server2 (repository) 
+"""]]
diff --git a/doc/forum/git-annex_assistant_with_2_dedicated_servers/comment_12_41fbee0ec9bc890e309bcd48a58c3851._comment b/doc/forum/git-annex_assistant_with_2_dedicated_servers/comment_12_41fbee0ec9bc890e309bcd48a58c3851._comment
new file mode 100644
index 0000000000..d707de6e10
--- /dev/null
+++ b/doc/forum/git-annex_assistant_with_2_dedicated_servers/comment_12_41fbee0ec9bc890e309bcd48a58c3851._comment
@@ -0,0 +1,18 @@
+[[!comment format=mdwn
+ username="David"
+ ip="178.21.48.117"
+ subject="comment 12"
+ date="2013-11-18T11:25:06Z"
+ content="""
+Hello,
+
+Thanks for your help and explanation. Now I had some time to test git-annex and I had a sad experience. I tried to sync the files from server1 to server2 (as described before). We are talking about 87000 small files (these are maildir folders and each file is an e-mail) with the total size of 19 Gbytes. After a few hours git-annex assistant consumed all memory on both servers ... and then all swap ... and then kernel killed git-annex. The servers have 512 MB RAM and 1 GB swap.
+
+According to the Scalability page (http://git-annex.branchable.com/scalability/) the \"memory usage should be constant\" and I am sure I didn't run \"git-annex unused\".
+
+Is this memory consumption normal or it might be a bug? What shall we do?
+
+Thanks,
+
+David
+"""]]
diff --git a/doc/forum/git-annex_assistant_with_2_dedicated_servers/comment_13_571cffc0beb8ba5fc936db6971cd3d62._comment b/doc/forum/git-annex_assistant_with_2_dedicated_servers/comment_13_571cffc0beb8ba5fc936db6971cd3d62._comment
new file mode 100644
index 0000000000..aefe988132
--- /dev/null
+++ b/doc/forum/git-annex_assistant_with_2_dedicated_servers/comment_13_571cffc0beb8ba5fc936db6971cd3d62._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="David"
+ ip="178.21.48.117"
+ subject="comment 13"
+ date="2013-11-19T09:17:26Z"
+ content="""
+Please tell me if/how I can help you. For example if you need access to servers to test...
+"""]]
diff --git a/doc/forum/git-annex_assistant_with_2_dedicated_servers/comment_14_0fb62af673a4bc8183e8fef048ceedd4._comment b/doc/forum/git-annex_assistant_with_2_dedicated_servers/comment_14_0fb62af673a4bc8183e8fef048ceedd4._comment
new file mode 100644
index 0000000000..7a4b80aa9b
--- /dev/null
+++ b/doc/forum/git-annex_assistant_with_2_dedicated_servers/comment_14_0fb62af673a4bc8183e8fef048ceedd4._comment
@@ -0,0 +1,10 @@
+[[!comment format=mdwn
+ username="http://joeyh.name/"
+ ip="209.250.56.87"
+ subject="comment 14"
+ date="2013-12-16T20:36:14Z"
+ content="""
+I see the assistant spike up to around 150 mb when adding a lot of files, then drop back down once it finishes one batch (limited to around 5000 files) and moves on to the next. 
+
+You might need to file a bug report with some more details so I can reproduce the problem.
+"""]]
diff --git a/doc/forum/git-annex_assistant_with_2_dedicated_servers/comment_1_53a79af2d8e3abe50b983bf91972b8f2._comment b/doc/forum/git-annex_assistant_with_2_dedicated_servers/comment_1_53a79af2d8e3abe50b983bf91972b8f2._comment
new file mode 100644
index 0000000000..f6f058988c
--- /dev/null
+++ b/doc/forum/git-annex_assistant_with_2_dedicated_servers/comment_1_53a79af2d8e3abe50b983bf91972b8f2._comment
@@ -0,0 +1,14 @@
+[[!comment format=mdwn
+ username="http://joeyh.name/"
+ ip="209.250.56.246"
+ subject="comment 1"
+ date="2013-11-16T17:42:48Z"
+ content="""
+The [[assistant/remote sharing walkthrough]] assumes that the computers are not servers, so they might not be turned on at the same time, and cannot directly contect each other.
+
+With 2 servers, it's much simpler. Just add a git remote on each pointing at the git repository on the other server, and run the git-annex assistant on both to keep them in sync.
+
+You can add the git remote with `git remote add otherserver ssh://otherserver/path/to/repo`, or you can use the webapp to do it (Add another repository -> Remote server)
+
+(If your WAN puts the servers on the same virtual subnet, ie a VPN, you can also probably use local pairing over the WAN to get to the same setup by a different route.)
+"""]]
diff --git a/doc/forum/git-annex_assistant_with_2_dedicated_servers/comment_2_c0ba3e8b7fbf8a5ed718001cec8df676._comment b/doc/forum/git-annex_assistant_with_2_dedicated_servers/comment_2_c0ba3e8b7fbf8a5ed718001cec8df676._comment
new file mode 100644
index 0000000000..8c4bb5922c
--- /dev/null
+++ b/doc/forum/git-annex_assistant_with_2_dedicated_servers/comment_2_c0ba3e8b7fbf8a5ed718001cec8df676._comment
@@ -0,0 +1,35 @@
+[[!comment format=mdwn
+ username="David"
+ ip="178.21.48.117"
+ subject="still need some clarification"
+ date="2013-11-16T22:02:46Z"
+ content="""
+Hello,
+
+Thanks for your reply. I guessed that there is an easier, direct way to do it. But unfortunately I still need some clarification. This is how I tried:
+
+1. Cleared everything to start from scratch.
+2. I started git-annex on server1: screen git annex webapp --listen=server1_ip, opened the webapp and created the local repository /media/mail
+3. I configured the XMPP account on server1.
+4. Started git-annex on server2: screen git annex webapp --listen=server2_ip, opened the webapp and created the local repository /media/mail
+5. I configured the XMPP account on server2.
+
+And here comes the confusing part:
+
+6. On server1 I clicked Add another repository / Remote server: and configured ssh server2 to the same folder: /media/mail. I chose client repository group instead of transfer. (I am not sure if I did it right) ... but server1 started sync-ing.
+7. On server2 I clicked Add another repository / Remote server: and configured ssh server1 to the same folder: /media/mail. Then I received an error message:
+
+    Failed to make repository
+    Something went wrong setting up the repository on the remote server.
+    
+    Transcript:
+    
+    init  error: could not lock config file /media/mail-test/mail/.git/config: Permission denied
+    git-annex: git [Param \"config\",Param \"annex.version\",Param \"3\"] failed
+
+Cold you please help me where I did wrong?
+
+Thanks,
+
+David
+"""]]
diff --git a/doc/forum/git-annex_assistant_with_2_dedicated_servers/comment_3_60c39bc8ef74e80e72381d514b6dd223._comment b/doc/forum/git-annex_assistant_with_2_dedicated_servers/comment_3_60c39bc8ef74e80e72381d514b6dd223._comment
new file mode 100644
index 0000000000..f9e405b70b
--- /dev/null
+++ b/doc/forum/git-annex_assistant_with_2_dedicated_servers/comment_3_60c39bc8ef74e80e72381d514b6dd223._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="http://joeyh.name/"
+ ip="209.250.56.246"
+ subject="comment 3"
+ date="2013-11-16T22:18:20Z"
+ content="""
+It seems to me that you probably told git-annex to log into server2 as a user that does not have write access to sever2:/media/mail. Check the username field in the remote repository form.
+"""]]
diff --git a/doc/forum/git-annex_assistant_with_2_dedicated_servers/comment_4_6241120b4325c905661ef72881f4d7af._comment b/doc/forum/git-annex_assistant_with_2_dedicated_servers/comment_4_6241120b4325c905661ef72881f4d7af._comment
new file mode 100644
index 0000000000..bb7c193895
--- /dev/null
+++ b/doc/forum/git-annex_assistant_with_2_dedicated_servers/comment_4_6241120b4325c905661ef72881f4d7af._comment
@@ -0,0 +1,20 @@
+[[!comment format=mdwn
+ username="David"
+ ip="178.21.48.117"
+ subject="still need some clarification"
+ date="2013-11-16T22:31:07Z"
+ content="""
+Hey,
+
+Yes, you were right. The ssh user did not have privileges to read/write the folder. Now I have corrected this issue and things started sync-ing.
+
+But I am afraid, this is not exactly the thing I wanted to achieve. My goal was to sync from folder to folder ... and not from folder to repository both ways.
+
+So, what I would like to do is that if I create a file at server1 (e.g. /media/mail/test01.txt) then it should appear at server2 at /media/mail/test01.txt.
+
+How can I achieve this?
+
+Thanks a lot.
+
+David
+"""]]
diff --git a/doc/forum/git-annex_assistant_with_2_dedicated_servers/comment_5_cab00b8fa195340f4d3fdaf5af975b57._comment b/doc/forum/git-annex_assistant_with_2_dedicated_servers/comment_5_cab00b8fa195340f4d3fdaf5af975b57._comment
new file mode 100644
index 0000000000..179a65a99a
--- /dev/null
+++ b/doc/forum/git-annex_assistant_with_2_dedicated_servers/comment_5_cab00b8fa195340f4d3fdaf5af975b57._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="http://joeyh.name/"
+ ip="209.250.56.64"
+ subject="comment 5"
+ date="2013-11-16T22:55:34Z"
+ content="""
+Well, that's what should happen. What behavior are you seeing? What version of git-annex are you using?
+"""]]
diff --git a/doc/forum/git-annex_assistant_with_2_dedicated_servers/comment_6_e24df9a1c68724a912b8ac6533d9bd25._comment b/doc/forum/git-annex_assistant_with_2_dedicated_servers/comment_6_e24df9a1c68724a912b8ac6533d9bd25._comment
new file mode 100644
index 0000000000..a28e947dda
--- /dev/null
+++ b/doc/forum/git-annex_assistant_with_2_dedicated_servers/comment_6_e24df9a1c68724a912b8ac6533d9bd25._comment
@@ -0,0 +1,16 @@
+[[!comment format=mdwn
+ username="David"
+ ip="178.21.48.117"
+ subject="comment 6"
+ date="2013-11-16T22:58:15Z"
+ content="""
+Hello,
+
+Yes, meanwhile files started appearing. :) Maybe I just did not wait enough time.
+
+So, I have just one more question. Shall I set the remote ssh repositories to client or transfer? I read the docs and understand the difference, but in my case I simply cannot decide. 
+
+Thanks a lot for your hard work and great support. 
+
+David
+"""]]
diff --git a/doc/forum/git-annex_assistant_with_2_dedicated_servers/comment_7_ace3dc7c60c710a04b0a587206b341c4._comment b/doc/forum/git-annex_assistant_with_2_dedicated_servers/comment_7_ace3dc7c60c710a04b0a587206b341c4._comment
new file mode 100644
index 0000000000..c41fe492f4
--- /dev/null
+++ b/doc/forum/git-annex_assistant_with_2_dedicated_servers/comment_7_ace3dc7c60c710a04b0a587206b341c4._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="http://joeyh.name/"
+ ip="209.250.56.64"
+ subject="comment 7"
+ date="2013-11-16T23:04:29Z"
+ content="""
+In your situation, you do not have or need a repository whose only job is to transfer files between two other client repositories. So the right choice for your repositories is client -- or something else possibly -- but almost certainly not transfer.
+"""]]
diff --git a/doc/forum/git-annex_assistant_with_2_dedicated_servers/comment_8_9a96bc970a17ed62b0ceb7aa3f0a6f8b._comment b/doc/forum/git-annex_assistant_with_2_dedicated_servers/comment_8_9a96bc970a17ed62b0ceb7aa3f0a6f8b._comment
new file mode 100644
index 0000000000..1d0f1db52c
--- /dev/null
+++ b/doc/forum/git-annex_assistant_with_2_dedicated_servers/comment_8_9a96bc970a17ed62b0ceb7aa3f0a6f8b._comment
@@ -0,0 +1,14 @@
+[[!comment format=mdwn
+ username="David"
+ ip="178.21.48.117"
+ subject="comment 8"
+ date="2013-11-16T23:09:32Z"
+ content="""
+Okay. One last question. Could you please help me understand how this sync works now? 
+
+So for example if I save a file on server1, it gets pushed to server2's repository. But how is it translated back to a normal file from the repository?
+
+And in this case will this file be on server2's repository as well (so are there 2 copies on server2, one in the repository and one human-readabl)?
+
+Thanks!
+"""]]
diff --git a/doc/forum/git-annex_assistant_with_2_dedicated_servers/comment_9_ea88e0696f6e25e6904248a323f6cc36._comment b/doc/forum/git-annex_assistant_with_2_dedicated_servers/comment_9_ea88e0696f6e25e6904248a323f6cc36._comment
new file mode 100644
index 0000000000..cafac10772
--- /dev/null
+++ b/doc/forum/git-annex_assistant_with_2_dedicated_servers/comment_9_ea88e0696f6e25e6904248a323f6cc36._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="http://joeyh.name/"
+ ip="209.250.56.64"
+ subject="comment 9"
+ date="2013-11-16T23:11:58Z"
+ content="""
+Since you created these repositories with the webapp, they default to using [[direct_mode]]. git-annex handles putting the files in place when they are transferred to the repository. Only 1 copy is made, there is no separate copy in the repository.
+"""]]
diff --git a/doc/forum/git-annex_build_for_Nokia_N9___40__Meego_Harmattan__41___and_Sailfish_OS.mdwn b/doc/forum/git-annex_build_for_Nokia_N9___40__Meego_Harmattan__41___and_Sailfish_OS.mdwn
new file mode 100644
index 0000000000..e2be4fd22b
--- /dev/null
+++ b/doc/forum/git-annex_build_for_Nokia_N9___40__Meego_Harmattan__41___and_Sailfish_OS.mdwn
@@ -0,0 +1,7 @@
+Hi,
+
+Is there a chance to get a build of git-annex for the Nokia N9 smartphone with Meego Harmattan and later for Sailfish OS (Jolla phone)?
+Git is already available for Meego Harmattan: http://www.who.is.free.fr/wiki/doku.php?id=harmattan
+
+Cheers,
+Tobias
diff --git a/doc/forum/git-annex_build_for_Nokia_N9___40__Meego_Harmattan__41___and_Sailfish_OS/comment_1_301a51c48c3d54f9d37feace26a772f8._comment b/doc/forum/git-annex_build_for_Nokia_N9___40__Meego_Harmattan__41___and_Sailfish_OS/comment_1_301a51c48c3d54f9d37feace26a772f8._comment
new file mode 100644
index 0000000000..d6464b679a
--- /dev/null
+++ b/doc/forum/git-annex_build_for_Nokia_N9___40__Meego_Harmattan__41___and_Sailfish_OS/comment_1_301a51c48c3d54f9d37feace26a772f8._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="paulvt"
+ ip="2001:1af8:ff19:0:a288:b4ff:fe2c:f600"
+ subject="comment 1"
+ date="2013-07-13T08:21:29Z"
+ content="""
+I've just checked and almost all of the depends that Debian Sid on my laptop has, the N9 has.  But in the case of build-depends, they are nowhere near since the entire Haskell stack is not available...
+"""]]
diff --git a/doc/forum/git-annex_communication_channels.mdwn b/doc/forum/git-annex_communication_channels.mdwn
new file mode 100644
index 0000000000..8c56ac36a2
--- /dev/null
+++ b/doc/forum/git-annex_communication_channels.mdwn
@@ -0,0 +1,10 @@
+Thought I'd ask how y'all are finding the current communication by this forum/website/git repo only.
+
+Would there be a benefit to having an irc channel for git-annex?
+
+Maybe a mailing list? (Any persuasive reason why it would be better than this forum?)
+
+Are the existing RSS feeds on this site, for eg, new [[comments]] and posts to this forum, sufficient to keep up with
+things?
+
+--[[Joey]]
diff --git a/doc/forum/git-annex_communication_channels/comment_1_198325d2e9337c90f026396de89eec0e._comment b/doc/forum/git-annex_communication_channels/comment_1_198325d2e9337c90f026396de89eec0e._comment
new file mode 100644
index 0000000000..163aae02cc
--- /dev/null
+++ b/doc/forum/git-annex_communication_channels/comment_1_198325d2e9337c90f026396de89eec0e._comment
@@ -0,0 +1,17 @@
+[[!comment format=mdwn
+ username="https://www.google.com/accounts/o8/id?id=AItOawl9sYlePmv1xK-VvjBdN-5doOa_Xw-jH4U"
+ nickname="Richard"
+ subject="comment 1"
+ date="2011-03-28T15:48:08Z"
+ content="""
+No matter what you end up doing, I would appreciate a git-annex-announce@ list.
+
+I really like the persistence of ikiwiki, but it's not ideal for quick communication. I would be fine with IRC and/or ML. The advantage of a ML over ikiwiki is that it doesn't seem to be as \"wasteful\" to mix normal chat with actual problem-solving.  But maybe that's merely my own perception.
+
+Speaking of RSS: I thought I had added a wishlist item to ikiwiki about providing per-subsite RSS feeds. For example there is no (obvious) way to subscribe to changes in http://git-annex.branchable.com/forum/git-annex_communication_channels/ .
+
+FWIW, I resorted to tagging my local clone of git-annex to keep track of what I've read, already.
+
+
+-- RichiH
+"""]]
diff --git a/doc/forum/git-annex_communication_channels/comment_2_c7aeefa6ef9a2e75d8667b479ade1b7f._comment b/doc/forum/git-annex_communication_channels/comment_2_c7aeefa6ef9a2e75d8667b479ade1b7f._comment
new file mode 100644
index 0000000000..09a2b8c1a5
--- /dev/null
+++ b/doc/forum/git-annex_communication_channels/comment_2_c7aeefa6ef9a2e75d8667b479ade1b7f._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="https://www.google.com/accounts/o8/id?id=AItOawkSq2FDpK2n66QRUxtqqdbyDuwgbQmUWus"
+ nickname="Jimmy"
+ subject="comment 2"
+ date="2011-03-28T18:35:50Z"
+ content="""
+I think the forums/website currently is sufficient, I do at times wish there was a mailing list or anonymous git push to the wiki as I find editing posts through the web browser is some times tedious (the lack of !fmt or alt-q bugs me at times ;) ). The main advantage of keeping stuff on the site/forum is that everything gets saved and passed on to anyone who checks out the git repo of the code base.
+"""]]
diff --git a/doc/forum/git-annex_communication_channels/comment_3_1ff08a3e0e63fa0e560cbc9602245caa._comment b/doc/forum/git-annex_communication_channels/comment_3_1ff08a3e0e63fa0e560cbc9602245caa._comment
new file mode 100644
index 0000000000..72a48445ed
--- /dev/null
+++ b/doc/forum/git-annex_communication_channels/comment_3_1ff08a3e0e63fa0e560cbc9602245caa._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="https://www.google.com/accounts/o8/id?id=AItOawl9sYlePmv1xK-VvjBdN-5doOa_Xw-jH4U"
+ nickname="Richard"
+ subject="comment 3"
+ date="2011-03-28T20:47:23Z"
+ content="""
+Push access to the non-code bits of git-annex' ikiwiki would be very welcome indeed. Given the choice, I would rather edit everything in Vim than in a browser. -- RichiH
+"""]]
diff --git a/doc/forum/git-annex_communication_channels/comment_4_1ba6ddf54843c17c7d19a9996f2ab712._comment b/doc/forum/git-annex_communication_channels/comment_4_1ba6ddf54843c17c7d19a9996f2ab712._comment
new file mode 100644
index 0000000000..d6bba93651
--- /dev/null
+++ b/doc/forum/git-annex_communication_channels/comment_4_1ba6ddf54843c17c7d19a9996f2ab712._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="https://www.google.com/accounts/o8/id?id=AItOawnx8kHW66N3BqmkVpgtXDlYMvr8TJ5VvfY"
+ nickname="Yaroslav"
+ subject="comment 4"
+ date="2011-04-13T17:53:26Z"
+ content="""
+.1 cents: Having IRC would be really nice for seeking quick help.   E.g. like I was trying to do now, google lead me to this page.
+"""]]
diff --git a/doc/forum/git-annex_communication_channels/comment_5_404b723a681eb93fee015cea8024b6bc._comment b/doc/forum/git-annex_communication_channels/comment_5_404b723a681eb93fee015cea8024b6bc._comment
new file mode 100644
index 0000000000..042dcc1f38
--- /dev/null
+++ b/doc/forum/git-annex_communication_channels/comment_5_404b723a681eb93fee015cea8024b6bc._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="https://www.google.com/accounts/o8/id?id=AItOawkptNW1PzrVjYlJWP_9e499uH0mjnBV6GQ"
+ nickname="Christian"
+ subject="comment 5"
+ date="2011-04-14T11:24:59Z"
+ content="""
+I would also like an git-annex channel. Would be #git-annex@OFTC ok?
+"""]]
diff --git a/doc/forum/git-annex_communication_channels/comment_6_0d87d0e26461494b1d7f8a701a924729._comment b/doc/forum/git-annex_communication_channels/comment_6_0d87d0e26461494b1d7f8a701a924729._comment
new file mode 100644
index 0000000000..8dfd0f8203
--- /dev/null
+++ b/doc/forum/git-annex_communication_channels/comment_6_0d87d0e26461494b1d7f8a701a924729._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="https://www.google.com/accounts/o8/id?id=AItOawl9sYlePmv1xK-VvjBdN-5doOa_Xw-jH4U"
+ nickname="Richard"
+ subject="comment 6"
+ date="2011-04-15T19:32:08Z"
+ content="""
+We seem to be using #vcs-home @ OFTC for now. madduck is fine with it and joeyh pokes his head in there, as well. I just added a CIA bot to #vcs-home and this comment is a test if pushing works. -- RichiH
+"""]]
diff --git a/doc/forum/git-annex_communication_channels/comment_7_2c87c7a0648fe87c2bf6b4391f1cc468._comment b/doc/forum/git-annex_communication_channels/comment_7_2c87c7a0648fe87c2bf6b4391f1cc468._comment
new file mode 100644
index 0000000000..830d678ca1
--- /dev/null
+++ b/doc/forum/git-annex_communication_channels/comment_7_2c87c7a0648fe87c2bf6b4391f1cc468._comment
@@ -0,0 +1,10 @@
+[[!comment format=mdwn
+ username="http://joey.kitenet.net/"
+ nickname="joey"
+ subject="anonymous git push"
+ date="2011-05-19T19:21:51Z"
+ content="""
+@Jimmy mentioned anonymous git push -- that is now enabled for this wiki. Enjoy!
+
+I may try to spend more time on #vcs-home -- or I can be summoned there from my other lurking places on irc, I guess.
+"""]]
diff --git a/doc/forum/git-annex_confused_by_a_case-preserving_filesystem__63__.mdwn b/doc/forum/git-annex_confused_by_a_case-preserving_filesystem__63__.mdwn
new file mode 100644
index 0000000000..8fc51db50d
--- /dev/null
+++ b/doc/forum/git-annex_confused_by_a_case-preserving_filesystem__63__.mdwn
@@ -0,0 +1,73 @@
+Hello,
+
+I use git annex to sync files between my home linux server and my mac os laptop (macbook).
+This is a report about some troubles I had recovering git-annexed files from the macbook hard-drive.
+
+The laptop had some hardware failure, so I bought another laptop (this time running linux), 
+and plugged the macbook hard drive on yet another linux computer to recover the data (some files which were
+added to the macbook annex but never synchronized to the server):
+
+* mount the hfsplus FS as read-only
+* cp -a (as root) the annex to /home/seb/macbook
+* use git annex as usual (from the linux computer) to push the files to the linux server.
+
+Later on, I noticed that some files were missing (for instance "Le retour du printemps.mp4").
+I eventually understood that it was related to HFS+ being case-insensitive (case-preserving. actually), and that by copying the annex to the linux partition I lost that property.
+
+here are commands which I run from the linux computer to reproduce the issue:
+
+    $ cd /home/seb/macbook
+    $ git annex version
+    git-annex version: 5.20140412ubuntu1
+    $ git annex whereis  "public/Videos/Documentaries/Nature/Le retour du printemps.mp4"
+    whereis public/Videos/Documentaries/Nature/Le retour du printemps.mp4 (1 copy) 
+            18b9c2cc-e3c1-4002-a0a2-7e07782fae2c -- here (navimac/annex)
+
+Trying to copy the file to the linux server
+
+    $ git annex copy  "public/Videos/Documentaries/Nature/Le retour du printemps.mp4" --to origin ; echo $?
+    0
+
+Here git-annex did not copy anything, but reported success, maybe a bug.
+Fsck does account the issue:
+
+    $ git annex fsck  "public/Videos/Documentaries/Nature/Le retour du printemps.mp4" 
+    fsck public/Videos/Documentaries/Nature/Le retour du printemps.mp4 (fixing location log) 
+      ** Based on the location log, public/Videos/Documentaries/Nature/Le retour du printemps.mp4
+      ** was expected to be present, but its content is missing.
+      ** No known copies exist of public/Videos/Documentaries/Nature/Le retour du printemps.mp4
+    failed
+    git-annex: fsck: 1 failed
+
+It turn out the file is there, but in gm/29 instead of gM/29:
+
+    $ ls -l  "public/Videos/Documentaries/Nature/Le retour du printemps.mp4"
+    lrwxrwxrwx 1 seb root 214 Apr 17  2017 public/Videos/Documentaries/Nature/Le retour du printemps.mp4 -> ../../../../.git/annex/objects/gM/29/SHA256E-s915126032--79eb20cad1b56adf4831b8a665b02c7de68aaaa4c9ca424a5a90e288abcec2aa.mp4/SHA256E-s915126032--79eb20cad1b56adf4831b8a665b02c7de68aaaa4c9ca424a5a90e288abcec2aa.mp4
+    $ ls -l .git/annex/objects/g{M,m}/29/SHA256E-s915126032--79eb20cad1b56adf4831b8a665b02c7de68aaaa4c9ca424a5a90e288abcec2aa.mp4/SHA256E-s915126032--79eb20cad1b56adf4831b8a665b02c7de68aaaa4c9ca424a5a90e288abcec2aa.mp4
+    ls: cannot access .git/annex/objects/gM/29/SHA256E-s915126032--79eb20cad1b56adf4831b8a665b02c7de68aaaa4c9ca424a5a90e288abcec2aa.mp4/SHA256E-s915126032--79eb20cad1b56adf4831b8a665b02c7de68aaaa4c9ca424a5a90e288abcec2aa.mp4: No such file or directory
+    -r--r--r-- 1 seb root 915126032 Apr 17  2017 .git/annex/objects/gm/29/SHA256E-s915126032--79eb20cad1b56adf4831b8a665b02c7de68aaaa4c9ca424a5a90e288abcec2aa.mp4/SHA256E-s915126032--79eb20cad1b56adf4831b8a665b02c7de68aaaa4c9ca424a5a90e288abcec2aa.mp4
+
+I think that if ran the git annex from the hfsplus partition directly (ie. not copying the annex to /home/seb/macbook), it would have worked.
+
+Moving the file where git annex expects to find it solves it all:
+
+    $ mkdir .git/annex/objects/gM
+    $ mv  .git/annex/objects/gm/29 .git/annex/objects/gM/
+    $ git annex fsck  "public/Videos/Documentaries/Nature/Le retour du printemps.mp4" 
+    fsck public/Videos/Documentaries/Nature/Le retour du printemps.mp4 (fixing location log) (checksum...)
+    ok
+    (Recording state in git...)
+
+    $ git annex copy  "public/Videos/Documentaries/Nature/Le retour du printemps.mp4" --to origin ; echo $?                                                                                                                                                 
+    copy public/Videos/Documentaries/Nature/Le retour du printemps.mp4 (checking origin...)
+    seb@192.168.0.1's password: 
+    (to origin...) 
+    SHA256E-s915126032--79eb20cad1b56adf4831b8a665b02c7de68aaaa4c9ca424a5a90e288abcec2aa.mp4
+        915,126,032 100%    3.44MB/s    0:04:13 (xfr#1, to-chk=0/1)
+    ok
+    (Recording state in git...)
+    0
+
+Maybe I could instead have used cpiofs to avoid renaming files. (But I have less than 10 files to rename anyway).
+
+I learned a lot about git annex today! Thank you again fot git-annex.
diff --git a/doc/forum/git-annex_does_not_protect_files_on_NTFS-Fuse.mdwn b/doc/forum/git-annex_does_not_protect_files_on_NTFS-Fuse.mdwn
new file mode 100644
index 0000000000..e8084f6ff5
--- /dev/null
+++ b/doc/forum/git-annex_does_not_protect_files_on_NTFS-Fuse.mdwn
@@ -0,0 +1,8 @@
+I've read that git-annex probes the host filesystem to determine whether it has the necessary features for an indirect annex. Indirect annexes are supposed to protect annexed files from accidental editing:
+
+    # echo oops > my_cool_big_file
+    bash: my_cool_big_file: Permission denied
+
+I have an NTFS drive to share files between my Windows and Linux systems (dual boot). On Linux fuse sets the file permissions to rwx for the user and nothing for the rest, and this cannot be changed. Files in the annex can be modified as in the above example.
+
+If git-annex detects whether a fs can handle indirect annexes or not, I suggest checking for this case if possible.
diff --git a/doc/forum/git-annex_does_not_protect_files_on_NTFS-Fuse/comment_1_4aef65f680e4801eb110fefa1f947f02._comment b/doc/forum/git-annex_does_not_protect_files_on_NTFS-Fuse/comment_1_4aef65f680e4801eb110fefa1f947f02._comment
new file mode 100644
index 0000000000..039d237406
--- /dev/null
+++ b/doc/forum/git-annex_does_not_protect_files_on_NTFS-Fuse/comment_1_4aef65f680e4801eb110fefa1f947f02._comment
@@ -0,0 +1,11 @@
+[[!comment format=mdwn
+ username="joey"
+ subject="""comment 1"""
+ date="2015-07-30T17:54:11Z"
+ content="""
+Good idea. Also, putting an indirect mode repo on a NTFS filesystem would
+probably yield confusing results when trying to use it on windows..
+
+I've put a check for this in with the other crippled FS checks that init
+does.
+"""]]
diff --git a/doc/forum/git-annex_does_not_respect_preferred_content_settings.mdwn b/doc/forum/git-annex_does_not_respect_preferred_content_settings.mdwn
new file mode 100644
index 0000000000..bb719e08d4
--- /dev/null
+++ b/doc/forum/git-annex_does_not_respect_preferred_content_settings.mdwn
@@ -0,0 +1,18 @@
+I have the following vicfg,
+
+    group UUID1 = PodA
+    group UUID2 = PodA
+    group UUID3 = PodA
+    group UUID4 = PodB
+
+    wanted UUID1 = groupwanted
+    wanted UUID2 = groupwanted
+    wanted UUID3 = groupwanted
+    wanted UUID4 = groupwanted
+
+    groupwanted PodA = present and not copies=PodA:1
+    groupwanted PodB = present and not copies=PodB:1
+
+at this point repos 1 2 3 combined has 1 copy of each file and repo 4 has full copy of all files. (used to move files around manually before grouping repos). 
+
+What I am trying to achieve is to have a copy of a file in one of the 3 repos (1 2 3) and one copy in 4. running git annex get --auto starts getting all files, files with copies on other disks. testing using git annex find --want-get --not --in . returns all remaining files not in the repo but has enough copies on other repos both PodA and PodB has copies. numofcopies is set to 2.
diff --git a/doc/forum/git-annex_does_not_respect_preferred_content_settings/comment_1_ea347ea6ebdfe5ccfb0ed8786638f279._comment b/doc/forum/git-annex_does_not_respect_preferred_content_settings/comment_1_ea347ea6ebdfe5ccfb0ed8786638f279._comment
new file mode 100644
index 0000000000..647a9d1e23
--- /dev/null
+++ b/doc/forum/git-annex_does_not_respect_preferred_content_settings/comment_1_ea347ea6ebdfe5ccfb0ed8786638f279._comment
@@ -0,0 +1,17 @@
+[[!comment format=mdwn
+ username="joey"
+ subject="""comment 1"""
+ date="2015-09-11T17:49:32Z"
+ content="""
+Um, I'm pretty sure git-annex respects your preferred content settings.
+Whether the settings do what you think they do is another matter.
+
+When I tried replicating the same settings you show, "git annex get --auto"
+didn't want to get any files. That makes perfect sense, since
+"present" only succeeds if the file is already in the repo, so for files
+that are not, it fails, and so "present AND foo" fails, and so the file is
+not wanted.
+
+Lose the "present". All you need to make the files land on only 1 of the
+PodA repos is "not copies=PodA:1"
+"""]]
diff --git a/doc/forum/git-annex_does_not_respect_preferred_content_settings/comment_2_5fac7c41c9841cea4234afa0a98b1636._comment b/doc/forum/git-annex_does_not_respect_preferred_content_settings/comment_2_5fac7c41c9841cea4234afa0a98b1636._comment
new file mode 100644
index 0000000000..90c6303915
--- /dev/null
+++ b/doc/forum/git-annex_does_not_respect_preferred_content_settings/comment_2_5fac7c41c9841cea4234afa0a98b1636._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="https://me.yahoo.com/a/FHnTlSBo1eCGJRwueeKeB6.RCaPbGMPr5jxx8A--#ce0d8"
+ nickname="Hamza"
+ subject="comment 2"
+ date="2015-09-11T20:46:00Z"
+ content="""
+joey thanks for the reply, I've updated the expression to \"not copies=PodA:1\" but running \"git annex get --auto\" still wants to get all files that are already present in other repos in PodA. Say I have file asd.txt present in PodA repo 2 and PodB repo 4 content is already there. running get on repo 1 still gets the file in repo 1. 
+"""]]
diff --git a/doc/forum/git-annex_does_not_respect_preferred_content_settings/comment_3_e2ea1843bbc9580ca7b4ec494f53b552._comment b/doc/forum/git-annex_does_not_respect_preferred_content_settings/comment_3_e2ea1843bbc9580ca7b4ec494f53b552._comment
new file mode 100644
index 0000000000..1870bc743f
--- /dev/null
+++ b/doc/forum/git-annex_does_not_respect_preferred_content_settings/comment_3_e2ea1843bbc9580ca7b4ec494f53b552._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="https://me.yahoo.com/a/FHnTlSBo1eCGJRwueeKeB6.RCaPbGMPr5jxx8A--#ce0d8"
+ nickname="Hamza"
+ subject="comment 3"
+ date="2015-09-13T12:52:51Z"
+ content="""
+As a test I've created 4 new test repos (different test content) with your suggested settings, you are correct that with the settings everything works as it should but for some reason my actual repos does not honor those settings.
+"""]]
diff --git a/doc/forum/git-annex_does_not_respect_preferred_content_settings/comment_4_59ede467a902f27b81f14e4706a7503a._comment b/doc/forum/git-annex_does_not_respect_preferred_content_settings/comment_4_59ede467a902f27b81f14e4706a7503a._comment
new file mode 100644
index 0000000000..504afc907d
--- /dev/null
+++ b/doc/forum/git-annex_does_not_respect_preferred_content_settings/comment_4_59ede467a902f27b81f14e4706a7503a._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="joey"
+ subject="""comment 4"""
+ date="2015-09-15T16:00:16Z"
+ content="""
+Are these repositories all in sync, so that git-annex in repo 1
+knows that repo 2 and repo 4 have already gotten the file?
+"""]]
diff --git a/doc/forum/git-annex_doesn__39__t_build_with_feed-1.0.0.0.mdwn b/doc/forum/git-annex_doesn__39__t_build_with_feed-1.0.0.0.mdwn
new file mode 100644
index 0000000000..bdcb7e0f22
--- /dev/null
+++ b/doc/forum/git-annex_doesn__39__t_build_with_feed-1.0.0.0.mdwn
@@ -0,0 +1,209 @@
+    [470 of 574] Compiling Command.ImportFeed ( Command/ImportFeed.hs, dist/build/git-annex/git-annex-tmp/Command/ImportFeed.dyn_o )
+
+    Command/ImportFeed.hs:139:61: error:
+        • Couldn't match type ‘Data.Text.Internal.Text’ with ‘[Char]’
+        Expected type: URLString
+            Actual type: Text.Atom.Feed.URI
+        • In the first argument of ‘Enclosure’, namely ‘enclosureurl’
+        In the second argument of ‘($)’, namely ‘Enclosure enclosureurl’
+        In the second argument of ‘($)’, namely
+            ‘ToDownload f u i $ Enclosure enclosureurl’
+        |
+    139 |                         Just $ ToDownload f u i $ Enclosure enclosureurl
+        |                                                             ^^^^^^^^^^^^
+
+    Command/ImportFeed.hs:142:49: error:
+        • Couldn't match type ‘Data.Text.Internal.Text’ with ‘[Char]’
+        Expected type: URLString
+            Actual type: Data.Text.Internal.Text
+        • In the first argument of ‘quviSupported’, namely ‘link’
+        In the first argument of ‘ifM’, namely ‘(quviSupported link)’
+        In the expression:
+            ifM
+            (quviSupported link)
+            (return $ Just $ ToDownload f u i $ QuviLink link, return Nothing)
+        |
+    142 |                 Just link -> ifM (quviSupported link)
+        |                                                 ^^^^
+
+    Command/ImportFeed.hs:143:71: error:
+        • Couldn't match type ‘Data.Text.Internal.Text’ with ‘[Char]’
+        Expected type: URLString
+            Actual type: Data.Text.Internal.Text
+        • In the first argument of ‘QuviLink’, namely ‘link’
+        In the second argument of ‘($)’, namely ‘QuviLink link’
+        In the second argument of ‘($)’, namely
+            ‘ToDownload f u i $ QuviLink link’
+        |
+    143 |                         ( return $ Just $ ToDownload f u i $ QuviLink link
+        |                                                                       ^^^^
+
+    Command/ImportFeed.hs:214:54: error:
+        • Couldn't match type ‘[Char]’ with ‘Data.Text.Internal.Text’
+        Expected type: S.Set Data.Text.Internal.Text
+            Actual type: S.Set ItemId
+        • In the second argument of ‘S.member’, namely ‘(knownitems cache)’
+        In the expression: S.member itemid (knownitems cache)
+        In a case alternative:
+            Just (_, itemid) -> S.member itemid (knownitems cache)
+        |
+    214 |                 Just (_, itemid) -> S.member itemid (knownitems cache)
+        |                                                      ^^^^^^^^^^^^^^^^
+
+    Command/ImportFeed.hs:279:42: error:
+        • Couldn't match type ‘Data.Text.Internal.Text’ with ‘[Char]’
+        Expected type: Maybe [Char]
+            Actual type: Maybe Text.RSS.Syntax.DateString
+        • In the second argument of ‘(<$>)’, namely
+            ‘getItemPublishDateString itm’
+        In the expression: replace "/" "-" <$> getItemPublishDateString itm
+        In a case alternative:
+            _ -> replace "/" "-" <$> getItemPublishDateString itm
+        |
+    279 |                 _ -> replace "/" "-" <$> getItemPublishDateString itm
+        |                                          ^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+
+    Command/ImportFeed.hs:293:44: error:
+        • Couldn't match type ‘Data.Text.Internal.Text’ with ‘[Char]’
+        Expected type: String
+            Actual type: Data.Text.Internal.Text
+        • In the first argument of ‘toMetaValue’, namely ‘itemid’
+        In the second argument of ‘($)’, namely ‘toMetaValue itemid’
+        In the second argument of ‘M.singleton’, namely
+            ‘(S.singleton $ toMetaValue itemid)’
+        |
+    293 |                 (S.singleton $ toMetaValue itemid)
+        |                                            ^^^^^^
+
+    Command/ImportFeed.hs:299:26: error:
+        • Couldn't match type ‘Data.Text.Internal.Text’ with ‘[Char]’
+        Expected type: Maybe String
+            Actual type: Maybe Data.Text.Internal.Text
+        • In the expression: feedtitle
+        In the expression: [feedtitle]
+        In the expression: ("feedtitle", [feedtitle])
+        |
+    299 |         [ ("feedtitle", [feedtitle])
+        |                          ^^^^^^^^^
+
+    Command/ImportFeed.hs:300:26: error:
+        • Couldn't match type ‘Data.Text.Internal.Text’ with ‘[Char]’
+        Expected type: Maybe String
+            Actual type: Maybe Data.Text.Internal.Text
+        • In the expression: itemtitle
+        In the expression: [itemtitle]
+        In the expression: ("itemtitle", [itemtitle])
+        |
+    300 |         , ("itemtitle", [itemtitle])
+        |                          ^^^^^^^^^
+
+    Command/ImportFeed.hs:301:27: error:
+        • Couldn't match type ‘Data.Text.Internal.Text’ with ‘[Char]’
+        Expected type: Maybe String
+            Actual type: Maybe Data.Text.Internal.Text
+        • In the expression: feedauthor
+        In the expression: [feedauthor]
+        In the expression: ("feedauthor", [feedauthor])
+        |
+    301 |         , ("feedauthor", [feedauthor])
+        |                           ^^^^^^^^^^
+
+    Command/ImportFeed.hs:302:27: error:
+        • Couldn't match type ‘Data.Text.Internal.Text’ with ‘[Char]’
+        Expected type: Maybe String
+            Actual type: Maybe Data.Text.Internal.Text
+        • In the expression: itemauthor
+        In the expression: [itemauthor]
+        In the expression: ("itemauthor", [itemauthor])
+        |
+    302 |         , ("itemauthor", [itemauthor])
+        |                           ^^^^^^^^^^
+
+    Command/ImportFeed.hs:303:28: error:
+        • Couldn't match type ‘Data.Text.Internal.Text’ with ‘[Char]’
+        Expected type: Maybe String
+            Actual type: Maybe Data.Text.Internal.Text
+        • In the expression: getItemSummary $ item i
+        In the expression: [getItemSummary $ item i]
+        In the expression: ("itemsummary", [getItemSummary $ item i])
+        |
+    303 |         , ("itemsummary", [getItemSummary $ item i])
+        |                            ^^^^^^^^^^^^^^^^^^^^^^^
+
+    Command/ImportFeed.hs:304:32: error:
+        • Couldn't match type ‘Data.Text.Internal.Text’ with ‘[Char]’
+        Expected type: Maybe String
+            Actual type: Maybe Data.Text.Internal.Text
+        • In the expression: getItemDescription $ item i
+        In the expression: [getItemDescription $ item i]
+        In the expression:
+            ("itemdescription", [getItemDescription $ item i])
+        |
+    304 |         , ("itemdescription", [getItemDescription $ item i])
+        |                                ^^^^^^^^^^^^^^^^^^^^^^^^^^^
+
+    Command/ImportFeed.hs:305:27: error:
+        • Couldn't match type ‘Data.Text.Internal.Text’ with ‘[Char]’
+        Expected type: Maybe String
+            Actual type: Maybe Data.Text.Internal.Text
+        • In the expression: getItemRights $ item i
+        In the expression: [getItemRights $ item i]
+        In the expression: ("itemrights", [getItemRights $ item i])
+        |
+    305 |         , ("itemrights", [getItemRights $ item i])
+        |                           ^^^^^^^^^^^^^^^^^^^^^^
+
+    Command/ImportFeed.hs:306:23: error:
+        • Couldn't match type ‘Data.Text.Internal.Text’ with ‘[Char]’
+        Expected type: Maybe String
+            Actual type: Maybe Data.Text.Internal.Text
+        • In the expression: snd <$> getItemId (item i)
+        In the expression: [snd <$> getItemId (item i)]
+        In the expression: ("itemid", [snd <$> getItemId (item i)])
+        |
+    306 |         , ("itemid", [snd <$> getItemId (item i)])
+        |                       ^^^^^^^^^^^^^^^^^^^^^^^^^^
+
+    Command/ImportFeed.hs:307:22: error:
+        • Couldn't match type ‘Data.Text.Internal.Text’ with ‘[Char]’
+        Expected type: Maybe String
+            Actual type: Maybe Data.Text.Internal.Text
+        • In the expression: itemtitle
+        In the expression: [itemtitle, feedtitle]
+        In the expression: ("title", [itemtitle, feedtitle])
+        |
+    307 |         , ("title", [itemtitle, feedtitle])
+        |                      ^^^^^^^^^
+
+    Command/ImportFeed.hs:307:33: error:
+        • Couldn't match type ‘Data.Text.Internal.Text’ with ‘[Char]’
+        Expected type: Maybe String
+            Actual type: Maybe Data.Text.Internal.Text
+        • In the expression: feedtitle
+        In the expression: [itemtitle, feedtitle]
+        In the expression: ("title", [itemtitle, feedtitle])
+        |
+    307 |         , ("title", [itemtitle, feedtitle])
+        |                                 ^^^^^^^^^
+
+    Command/ImportFeed.hs:308:23: error:
+        • Couldn't match type ‘Data.Text.Internal.Text’ with ‘[Char]’
+        Expected type: Maybe String
+            Actual type: Maybe Data.Text.Internal.Text
+        • In the expression: itemauthor
+        In the expression: [itemauthor, feedauthor]
+        In the expression: ("author", [itemauthor, feedauthor])
+        |
+    308 |         , ("author", [itemauthor, feedauthor])
+        |                       ^^^^^^^^^^
+
+    Command/ImportFeed.hs:308:35: error:
+        • Couldn't match type ‘Data.Text.Internal.Text’ with ‘[Char]’
+        Expected type: Maybe String
+            Actual type: Maybe Data.Text.Internal.Text
+        • In the expression: feedauthor
+        In the expression: [itemauthor, feedauthor]
+        In the expression: ("author", [itemauthor, feedauthor])
+        |
+    308 |         , ("author", [itemauthor, feedauthor])
+        |                                   ^^^^^^^^^^
diff --git a/doc/forum/git-annex_doesn__39__t_build_with_feed-1.0.0.0/comment_1_0be4545b1d93232a721c5ae65030334c._comment b/doc/forum/git-annex_doesn__39__t_build_with_feed-1.0.0.0/comment_1_0be4545b1d93232a721c5ae65030334c._comment
new file mode 100644
index 0000000000..2c630f2676
--- /dev/null
+++ b/doc/forum/git-annex_doesn__39__t_build_with_feed-1.0.0.0/comment_1_0be4545b1d93232a721c5ae65030334c._comment
@@ -0,0 +1,7 @@
+[[!comment format=mdwn
+ username="joey"
+ subject="""comment 1"""
+ date="2017-09-24T19:58:47Z"
+ content="""
+Already fixed in git head.
+"""]]
diff --git a/doc/forum/git-annex_fails_on_Windows.mdwn b/doc/forum/git-annex_fails_on_Windows.mdwn
new file mode 100644
index 0000000000..6c10003eb0
--- /dev/null
+++ b/doc/forum/git-annex_fails_on_Windows.mdwn
@@ -0,0 +1,56 @@
+Not sure if this is a bug or user error. I'm trying to use git-annex on Windows 2008 R2, but it isn't working for me. Works fine on Linux. I have a local git repo and then added some binary files to annex. Then I used git annex sync --content to push ennex'd files up to git server. Then I can clone that repo and sync again to pull all the files down. On windows I just get error messages when I try to clone the repo and use annex to init or sync.
+
+git version: 1.8.0.msysgit.0
+git-annex version:
+git-annex version: 5.20150710-g8fd7052
+build flags: Assistant Webapp Webapp-secure Pairing Testsuite S3 WebDAV DNS Feeds Quvi TDFA TorrentP
+arser
+key/value backends: SHA256E SHA1E SHA512E SHA224E SHA384E SKEIN256E SKEIN512E MD5E SHA256 SHA1 SHA51
+2 SHA224 SHA384 SKEIN256 SKEIN512 MD5 WORM URL
+remote types: git gcrypt S3 bup directory rsync web bittorrent webdav tahoe glacier ddar hook extern
+al
+local repository version: unknown
+supported repository version: 5
+upgrade supported from repository versions: 2 3 4
+
+If I try to init the cloned repo:
+
+$ git-annex init
+init  Unknown option: --literal-pathspecs
+usage: git [--version] [--exec-path[=]] [--html-path] [--man-path] [--info-path]
+           [-p|--paginate|--no-pager] [--no-replace-objects] [--bare]
+           [--git-dir=] [--work-tree=] [--namespace=]
+           [-c name=value] [--help]
+            []
+Unknown option: --literal-pathspecs
+usage: git [--version] [--exec-path[=]] [--html-path] [--man-path] [--info-path]
+           [-p|--paginate|--no-pager] [--no-replace-objects] [--bare]
+           [--git-dir=] [--work-tree=] [--namespace=]
+           [-c name=value] [--help]
+            []
+Unknown option: --literal-pathspecs
+usage: git [--version] [--exec-path[=]] [--html-path] [--man-path] [--info-path]
+           [-p|--paginate|--no-pager] [--no-replace-objects] [--bare]
+           [--git-dir=] [--work-tree=] [--namespace=]
+           [-c name=value] [--help]
+            []
+Unknown option: --literal-pathspecs
+usage: git [--version] [--exec-path[=]] [--html-path] [--man-path] [--info-path]
+           [-p|--paginate|--no-pager] [--no-replace-objects] [--bare]
+           [--git-dir=] [--work-tree=] [--namespace=]
+           [-c name=value] [--help]
+            []
+git-annex.exe: git [Param "config",Param "user.name",Param "user"] failed
+
+If I try to sync:
+
+$ git annex sync --content
+Unknown option: --literal-pathspecs
+usage: git [--version] [--exec-path[=]] [--html-path] [--man-path] [--info-path]
+           [-p|--paginate|--no-pager] [--no-replace-objects] [--bare]
+           [--git-dir=] [--work-tree=] [--namespace=]
+           [-c name=value] [--help]
+            []
+git-annex: First run: git-annex init
+
+What am I missing? I love the idea of annex and really need it for an urgent project I'm working on, but unfortunately I need to use it with Windows...in addition to Linux.
diff --git a/doc/forum/git-annex_fails_on_Windows/comment_1_451a2cdfa6f26e2bd319ebf4ce09da68._comment b/doc/forum/git-annex_fails_on_Windows/comment_1_451a2cdfa6f26e2bd319ebf4ce09da68._comment
new file mode 100644
index 0000000000..f429669017
--- /dev/null
+++ b/doc/forum/git-annex_fails_on_Windows/comment_1_451a2cdfa6f26e2bd319ebf4ce09da68._comment
@@ -0,0 +1,15 @@
+[[!comment format=mdwn
+ username="joey"
+ subject="""comment 1"""
+ date="2015-07-20T16:21:32Z"
+ content="""
+This is a mismatch between the version of git that git-annex was compiled
+to use, and the version of git that you have installed.
+
+You seem to have a git version older than 1.8.1.6 installed, since
+"--literal-pathspecs" was first added in that version. According
+to the [Windows installation instructions](http://git-annex.branchable.com/install/Windows),
+you need to install msysgit 1.9 to use git-annex.
+
+So, upgrade..
+"""]]
diff --git a/doc/forum/git-annex_fails_on_Windows/comment_2_57b4fd347f74b3536ab0f0a5d7709af8._comment b/doc/forum/git-annex_fails_on_Windows/comment_2_57b4fd347f74b3536ab0f0a5d7709af8._comment
new file mode 100644
index 0000000000..c10e85cd63
--- /dev/null
+++ b/doc/forum/git-annex_fails_on_Windows/comment_2_57b4fd347f74b3536ab0f0a5d7709af8._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="fusionx86@2cab672ef75a8502e153ceb90177dde38985dba9"
+ nickname="fusionx86"
+ subject="comment 2"
+ date="2015-07-20T16:34:30Z"
+ content="""
+Thanks. I have another box with the latest version 1.9 which I thought I had tested this on. But I went back and tried again to find it does indeed work with a newer version. My fault for thinking I had tried this with 1.9. Thanks again...
+"""]]
diff --git a/doc/forum/git-annex_for_multiple_repositories___40__ssh_server__41__.mdwn b/doc/forum/git-annex_for_multiple_repositories___40__ssh_server__41__.mdwn
new file mode 100644
index 0000000000..9168bc5ef9
--- /dev/null
+++ b/doc/forum/git-annex_for_multiple_repositories___40__ssh_server__41__.mdwn
@@ -0,0 +1,73 @@
+For git-annex version 6 with 'thin'. [I want 1) not store date twice 2) to be able to have self-sufficient repository at different machines (i.e. not special remotes, which don't contain the git tree)].
+
+Say I've two machines A and B and the server S. 
+
+I setup a local repository on A:
+
+    git init
+    git-annex init $HOSTNAME --version=6
+    git config annex.thin true
+    git add .
+
+on S:
+
+    mkdir test
+    cd test
+    git init
+    git-annex init $HOSTNAME --version=6
+    git config annex.thin true
+
+on A:
+
+    git remote add S:~/test.git
+    git-annex sync --content
+
+on B:
+
+    git clone S:~/test.git
+    cd test
+    git-annex init $HOSTNAME --version=6
+    git config annex.thin true
+    git-annex sync
+
+(the reply)
+
+    ...
+    merge: refs/remotes/origin/master - not something we can merge
+
+    failed
+    push origin 
+    Counting objects: 6, done.
+    Delta compression using up to 4 threads.
+    Compressing objects: 100% (5/5), done.
+    Writing objects: 100% (6/6), 714 bytes | 0 bytes/s, done.
+    Total 6 (delta 1), reused 1 (delta 0)
+    remote: git-annex: unknown command post-receive
+    ...
+
+and the file contains only '/annex/objects/SHA256E-s0--e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855'.
+
+Another time with another repo which I tried to sync from S at B, I got
+
+    $ git-annex sync --content
+    Initial commit
+
+    Untracked files: 
+(all my files untracked)
+
+    merge: refs/remotes/origin/master - not something we can merge
+
+    error: Untracked working tree file 'applications/algorithms/file.pdf' would be overwritten by merge.
+    fatal: read-tree failed
+    failed
+    git-annex: sync: 1 failed
+
+On walkthrough it's written how to sync with USB. How should one initialize remote repositories and use them e.g. as central repositories (though they should work probably not depending on 'central' or not they are)? 
+
+I've read a lot of manuals on git-annex and the forum; the questions here remain unanswered: 
+
+
+
+
+
+I hope this is a basic and popular usage of git-annex and that one can write a howto for that or at least answer this question.
diff --git a/doc/forum/git-annex_for_multiple_repositories___40__ssh_server__41__/comment_1_cb691410d2a9ef4393af5362000a768c._comment b/doc/forum/git-annex_for_multiple_repositories___40__ssh_server__41__/comment_1_cb691410d2a9ef4393af5362000a768c._comment
new file mode 100644
index 0000000000..dfe735e614
--- /dev/null
+++ b/doc/forum/git-annex_for_multiple_repositories___40__ssh_server__41__/comment_1_cb691410d2a9ef4393af5362000a768c._comment
@@ -0,0 +1,12 @@
+[[!comment format=mdwn
+ username="ynikitenko"
+ avatar="http://cdn.libravatar.org/avatar/168d629704097ddc596f75ca32a687a3"
+ subject="Works"
+ date="2017-11-02T19:21:43Z"
+ content="""
+Surprisingly, these very same repositories work today. I don't know why this happened. Seems that the commands I wrote above work.
+
+I've also found a useful post with various git-annex usage cases (basically the one descibed in this post).
+
+[http://ewen.mcneill.gen.nz/blog/entry/2014-11-15-git-annex-to-manage-media-backups/](http://ewen.mcneill.gen.nz/blog/entry/2014-11-15-git-annex-to-manage-media-backups/)
+"""]]
diff --git a/doc/forum/git-annex_help_import_outdated__63__.mdwn b/doc/forum/git-annex_help_import_outdated__63__.mdwn
new file mode 100644
index 0000000000..6367fe1489
--- /dev/null
+++ b/doc/forum/git-annex_help_import_outdated__63__.mdwn
@@ -0,0 +1,13 @@
+Hi,
+
+I just saw that my installed version of git-annex is too old for:
+
+    git-annex import --reinject-duplicates [...]
+
+Thus, I downloaded the precompiled binary: git-annex-standalone-amd64.tar.gz	2017-05-10 16:19 	54M
+
+First of all, it works fine with this version. However, the "--reinject-duplicates" option is not shown, when calling:
+
+    ./git-annex help import  ## in ~/git-annex.linux/
+
+So, is the help page just outdated, or does git-annex help gets confused because I still have an older version of git-annex installed in the system?
diff --git a/doc/forum/git-annex_help_import_outdated__63__/comment_1_ada30e2fead4b5b8f5e53388f941eb07._comment b/doc/forum/git-annex_help_import_outdated__63__/comment_1_ada30e2fead4b5b8f5e53388f941eb07._comment
new file mode 100644
index 0000000000..b054e1520a
--- /dev/null
+++ b/doc/forum/git-annex_help_import_outdated__63__/comment_1_ada30e2fead4b5b8f5e53388f941eb07._comment
@@ -0,0 +1,13 @@
+[[!comment format=mdwn
+ username="mario"
+ avatar="http://cdn.libravatar.org/avatar/4c63b0935789d29210d0bd8cad8d7ac7"
+ subject="comment 1"
+ date="2017-05-19T12:00:01Z"
+ content="""
+Okay, I did the same on a machine when git-annex was not installed before. Here everything works fine.
+
+So does git-annex help [...] open the locally installed manpages?
+
+I think it should show the help message that belongs to the running binary..
+
+"""]]
diff --git a/doc/forum/git-annex_help_import_outdated__63__/comment_2_ca898237f42af1205acab509e4d67d38._comment b/doc/forum/git-annex_help_import_outdated__63__/comment_2_ca898237f42af1205acab509e4d67d38._comment
new file mode 100644
index 0000000000..f5a9c8ae0a
--- /dev/null
+++ b/doc/forum/git-annex_help_import_outdated__63__/comment_2_ca898237f42af1205acab509e4d67d38._comment
@@ -0,0 +1,15 @@
+[[!comment format=mdwn
+ username="joey"
+ subject="""comment 2"""
+ date="2017-05-24T17:03:11Z"
+ content="""
+The man page is not outdated, see
+
+
+It seem that `git help`'s handling of MANPATH makes it not look
+first at the man pages from the standalone tarball in this case,
+and instead it displays the system man page.
+
+`git annex importfeed --help` will display the shorter documentation that's
+built into the binary.
+"""]]
diff --git a/doc/forum/git-annex_in_sane_language_for_mere_humans_of_us__63__.mdwn b/doc/forum/git-annex_in_sane_language_for_mere_humans_of_us__63__.mdwn
new file mode 100644
index 0000000000..e091460dcb
--- /dev/null
+++ b/doc/forum/git-annex_in_sane_language_for_mere_humans_of_us__63__.mdwn
@@ -0,0 +1 @@
+So, you provide ARM build. But you probably don't know that my NAS box runs OABI. No, you don't know, you can't know, and you shouldn't know. The only thing worth knowing is that writing great software in obscure and esoteric languages drastically limits its usage, impact, and collaboration around it. So, any idea of writing git-annex implementation in a sane, interpreted, "just works" language, e.g. Python? Thanks.
diff --git a/doc/forum/git-annex_in_sane_language_for_mere_humans_of_us__63__/comment_1_29eda7ec1519f339d5b3601559fe0bb0._comment b/doc/forum/git-annex_in_sane_language_for_mere_humans_of_us__63__/comment_1_29eda7ec1519f339d5b3601559fe0bb0._comment
new file mode 100644
index 0000000000..1cf6c84d6f
--- /dev/null
+++ b/doc/forum/git-annex_in_sane_language_for_mere_humans_of_us__63__/comment_1_29eda7ec1519f339d5b3601559fe0bb0._comment
@@ -0,0 +1,14 @@
+[[!comment format=mdwn
+ username="http://svario.it/gioele"
+ nickname="gioele"
+ subject="comment 1"
+ date="2014-08-24T10:41:58Z"
+ content="""
+> git-annex implementation in a sane, interpreted, \"just works\" language, e.g. Python? Thanks.
+
+Good luck finding anything that works on OARM. Python itself does not support OABI:
+
+> This issue remains as \"won't fix\". ARM is supported; just OABI is not, and never will be. If anybody needs that, they will have to maintain their own fork of Python.
+
+(from )
+"""]]
diff --git a/doc/forum/git-annex_in_sane_language_for_mere_humans_of_us__63__/comment_2_a2b2183ee86377cdfef7c3acbe9552fb._comment b/doc/forum/git-annex_in_sane_language_for_mere_humans_of_us__63__/comment_2_a2b2183ee86377cdfef7c3acbe9552fb._comment
new file mode 100644
index 0000000000..32318b404a
--- /dev/null
+++ b/doc/forum/git-annex_in_sane_language_for_mere_humans_of_us__63__/comment_2_a2b2183ee86377cdfef7c3acbe9552fb._comment
@@ -0,0 +1,11 @@
+[[!comment format=mdwn
+ username="https://www.google.com/accounts/o8/id?id=AItOawld54zdyk6b0W4jXnssSO_j2Nn3W1uVsUE"
+ nickname="Paul"
+ subject="comment 2"
+ date="2014-08-24T11:04:21Z"
+ content="""
+Python on this system \"just works\". That's because Python is a project with a real community, so if one pundit said \"not supported\", dozen of people shrugged and typed \"make\", then packed up result for thousands to use.
+
+But don't get carried away by OABI, that was just one random example how deployment of git-annex is problematic. There're bigger issues like community involvement, being able to investigate and resolve issues, submit patches, bring new working ideas, make git-annex development and lifecycle sustainable, in the end - as vividly cared by the author.
+
+"""]]
diff --git a/doc/forum/git-annex_in_sane_language_for_mere_humans_of_us__63__/comment_3_5605d42a68b3140cb660eb710ce5031e._comment b/doc/forum/git-annex_in_sane_language_for_mere_humans_of_us__63__/comment_3_5605d42a68b3140cb660eb710ce5031e._comment
new file mode 100644
index 0000000000..8e613dd211
--- /dev/null
+++ b/doc/forum/git-annex_in_sane_language_for_mere_humans_of_us__63__/comment_3_5605d42a68b3140cb660eb710ce5031e._comment
@@ -0,0 +1,12 @@
+[[!comment format=mdwn
+ username="https://www.google.com/accounts/o8/id?id=AItOawld54zdyk6b0W4jXnssSO_j2Nn3W1uVsUE"
+ nickname="Paul"
+ subject="comment 3"
+ date="2014-08-24T18:29:23Z"
+ content="""
+Sure, that's the plan. But first I'm doing my homework to understand how it got to that and how community copes with that. Maybe I don't get something and every open-source project should have a notice like: \"Installation from scratch. This is not recommended.\" (http://git-annex.branchable.com/install/). Interested in building software you run? Interested to help? Get lost, you won't get it. Am I surprised? Nope, I'm doing my homework and know where that Haskell thing came from. A piece of Microsoft was largely involved with it, so no surprise of such attitudes.
+
+Surely I'm not the only one who got jaundiced eye on git-annex: https://github.com/tv42/big : \"big is not like git-annex, because: it's not written in Haskell, so it might even work across distribution upgrades and platforms\". Certainly, stories of cvsup and unison, which are now where they should be - rest in peace, didn't help. So, once again, I'm interested to know how other people deal with this lack of proper compilation instructions, ability to get simple and easy tweaks, etc. - short of not using it, which seems to be a popular choice, despite all the git-annex coolness (I for one have been having its deployment in my queue fro half a year, instead of spending exactly a weekend to do tweaks I need and contribute them back).
+
+
+"""]]
diff --git a/doc/forum/git-annex_in_sane_language_for_mere_humans_of_us__63__/comment_4_f56508164c71b2080150bc354e5de4b7._comment b/doc/forum/git-annex_in_sane_language_for_mere_humans_of_us__63__/comment_4_f56508164c71b2080150bc354e5de4b7._comment
new file mode 100644
index 0000000000..8afa4c6fbc
--- /dev/null
+++ b/doc/forum/git-annex_in_sane_language_for_mere_humans_of_us__63__/comment_4_f56508164c71b2080150bc354e5de4b7._comment
@@ -0,0 +1,9 @@
+[[!comment format=mdwn
+ username="https://www.google.com/accounts/o8/id?id=AItOawld54zdyk6b0W4jXnssSO_j2Nn3W1uVsUE"
+ nickname="Paul"
+ subject="comment 4"
+ date="2014-08-24T18:31:25Z"
+ content="""
+Previous comment was written in response to a comment suggesting me to rewrite it in Python myself - which was then removed for some reason.
+
+"""]]
diff --git a/doc/forum/git-annex_in_sane_language_for_mere_humans_of_us__63__/comment_5_c8cdb0faa342fe1f9407ad4c97e6bc3c._comment b/doc/forum/git-annex_in_sane_language_for_mere_humans_of_us__63__/comment_5_c8cdb0faa342fe1f9407ad4c97e6bc3c._comment
new file mode 100644
index 0000000000..e58191a7f2
--- /dev/null
+++ b/doc/forum/git-annex_in_sane_language_for_mere_humans_of_us__63__/comment_5_c8cdb0faa342fe1f9407ad4c97e6bc3c._comment
@@ -0,0 +1,10 @@
+[[!comment format=mdwn
+ username="Ganwell"
+ ip="178.174.3.166"
+ subject="How I solved it..."
+ date="2014-08-26T23:03:09Z"
+ content="""
+I decided it was a bit harsh, so I removed comment. Here is how I solved problem:
+
+I have a server without much storage which runs the git-annex process, the data is stored on the NAS mounted via iSCSI. I never even thought of trying to compile git-annex on a NAS. I did things like that many years ago and it used to much time, whether the language was, common or not, didn't change much. Missing floating point units on the NAS killed performance of the programms I wanted to run anyways.
+"""]]
diff --git a/doc/forum/git-annex_in_sane_language_for_mere_humans_of_us__63__/comment_6_25ce5eddeb1b65aacd5d86e09c3719b8._comment b/doc/forum/git-annex_in_sane_language_for_mere_humans_of_us__63__/comment_6_25ce5eddeb1b65aacd5d86e09c3719b8._comment
new file mode 100644
index 0000000000..f9fa78e380
--- /dev/null
+++ b/doc/forum/git-annex_in_sane_language_for_mere_humans_of_us__63__/comment_6_25ce5eddeb1b65aacd5d86e09c3719b8._comment
@@ -0,0 +1,13 @@
+[[!comment format=mdwn
+ username="http://joeyh.name/"
+ ip="209.250.56.22"
+ subject="comment 6"
+ date="2014-09-16T19:19:00Z"
+ content="""
+There is too much negativity on this page, starting perhaps with its title.
+
+The \"this is not recommended\" is not because we don't want people to build git-annex from source. It's to try to dissuade users who just want to get git-annex installed, and who would be scared off by a transient build failure, from building from source, when there are many builds provided by Linux distributions, etc that would be better choices for them to get started. Hopefully the new page and new wording gets this across better: \"Experienced users should not find it too hard to build and install it from source.\"
+
+If someone wants to reimplement git-annex, or parts of it, I think that the existing documentation (eg [[internals]]) should get them most of the way there.
+
+"""]]
diff --git a/doc/forum/git-annex_move_does_not_appear_to_respect_numcopies.mdwn b/doc/forum/git-annex_move_does_not_appear_to_respect_numcopies.mdwn
new file mode 100644
index 0000000000..039ed005bb
--- /dev/null
+++ b/doc/forum/git-annex_move_does_not_appear_to_respect_numcopies.mdwn
@@ -0,0 +1,36 @@
+I have a git-annex repository checked out onto two separate drives (connected to the same machine). git-annex is aware of other copies of the repository on other drives that are not currently accessible. The local repository version is 5, and I'm running git-annex version 6.20170818. I have configured numcopies, and it displays correctly on both machines:
+
+```
+$ git annex numcopies
+2
+```
+
+However, I can't figure out how to use `git annex move` to match the behavior I'd expect. If I run:
+
+```
+$ git annex move my_file.txt --to otherdrive
+```
+
+it will copy `my_file.txt` to otherdrive and drop it from my primary drive, even though it's only aware of two copies. `git annex drop`, however, behaves as I'd expect.
+
+    $ git annex drop my_file.txt
+    drop my_file.txt (unsafe)
+        Could only verify the existence of 1 out of 2 necessary copies
+
+        Rather than dropping this file, try using: git annex move
+
+        (Use --force to override this check, or adjust numcopies.)
+
+It appears that `move` is equivalent to `git annex copy && git annex drop --force`, rather than `git annex copy && git annex drop`. That's surprising to me, but the message here implies that it's the intended behavior. In any case, I can't figure out which arguments to pass to `git-annex move` to simulate the behavior of `git annex copy && git annex drop` - that is, I want to move a file to a different drive (to free up space), but I do *not* want to drop it if I cannot ensure the correct number of copies exist. 
+
+
+Passing `--numcopies 2` doesn't do what I want either:
+
+```
+$ git annex move --numcopies 2 --to otherdrive my_file.txt 
+```
+
+will still copy the file to otherdrive and drop my local copy, leaving me with only one copy.
+
+
+Is there a mode for `git annex move` which behaves the desired way: moving a file to a different drive, while still ensuring that I don't end up with fewer than N verifiable copies (where N is the value already configured via `git-annex-numcopies`)?
diff --git a/doc/forum/git-annex_move_does_not_appear_to_respect_numcopies/comment_1_5dba0ff04c86c7b2697a0750fd1e4b5d._comment b/doc/forum/git-annex_move_does_not_appear_to_respect_numcopies/comment_1_5dba0ff04c86c7b2697a0750fd1e4b5d._comment
new file mode 100644
index 0000000000..52b85d16ac
--- /dev/null
+++ b/doc/forum/git-annex_move_does_not_appear_to_respect_numcopies/comment_1_5dba0ff04c86c7b2697a0750fd1e4b5d._comment
@@ -0,0 +1,15 @@
+[[!comment format=mdwn
+ username="CandyAngel"
+ avatar="http://cdn.libravatar.org/avatar/15c0aade8bec5bf004f939dd73cf9ed8"
+ subject="comment 1"
+ date="2018-04-09T14:56:52Z"
+ content="""
+Well.. it actually says this in the source code for the move command:
+
+    - Note that unlike drop, this does not honor numcopies.
+    - A file's content can be moved even if there are insufficient copies to
+    - allow it to be dropped.
+
+So the behaviour seems to be known/intended. You told it to move the file, so it did.
+Perhaps it should output a warning if numcopies is not satisfied after the move?
+"""]]
diff --git a/doc/forum/git-annex_move_does_not_appear_to_respect_numcopies/comment_2_114cc3c9236ba930117cc14a0dddc3dd._comment b/doc/forum/git-annex_move_does_not_appear_to_respect_numcopies/comment_2_114cc3c9236ba930117cc14a0dddc3dd._comment
new file mode 100644
index 0000000000..dda1d00f07
--- /dev/null
+++ b/doc/forum/git-annex_move_does_not_appear_to_respect_numcopies/comment_2_114cc3c9236ba930117cc14a0dddc3dd._comment
@@ -0,0 +1,18 @@
+[[!comment format=mdwn
+ username="joey"
+ subject="""comment 2"""
+ date="2018-04-09T17:47:38Z"
+ content="""
+You're exactly right; the move command bypasses the normal numcopies check,
+in a way that is otherwise (AFAICR) only ever done when using --force.
+
+There's also no way to get exactly the behavior of a numcopies honoring
+move. As you note copy followed by drop has that behavior, but all the
+drops then happen at the end, which is clumsy. There's clearly a good case
+for an option to make move honor numcopies if it does not by default.
+I've added that as `git annex move --safe`.
+
+I feel that the divergence from everything else in honoring numcopies by
+default is important enough to treat as a bug, so I opened
+[[bugs/move_violates_numcopies]].
+"""]]
diff --git a/doc/forum/git-annex_on_OSX.mdwn b/doc/forum/git-annex_on_OSX.mdwn
new file mode 100644
index 0000000000..a00548366a
--- /dev/null
+++ b/doc/forum/git-annex_on_OSX.mdwn
@@ -0,0 +1 @@
+See [[install/OSX]].
diff --git a/doc/forum/git-annex_on_Samba_share.mdwn b/doc/forum/git-annex_on_Samba_share.mdwn
new file mode 100644
index 0000000000..0235820c06
--- /dev/null
+++ b/doc/forum/git-annex_on_Samba_share.mdwn
@@ -0,0 +1,9 @@
+I want to put my photos into a git-annex repository, syncing it with my girlfriends computer and our NAS.
+If possible, I'd like to be able to add files to the NAS and have them synced, so I guess a special remote doesn't work here.
+
+Unfortunately, git-annex doesn't run on my NAS directly (yet), so I thought of mounting the NAS with CIFS, and creating an annex there that syncs with the local ones.
+While this seems to work fine with one computer, I wonder what will happen if I mount the Samba share on my and my girlfriend's computer at the same time.
+
+In theory, the NAS supports Samba Unix extensions which includes POSIX locking, but a weird bug that prevents you from removing a named pipe from Samba, making git-annex init fail (sent a bug report to Synology). When I disable Unix extensions it works. It then detects a crippled file system though.
+
+Any thoughts?
diff --git a/doc/forum/git-annex_on_Samba_share/comment_1_3e9cfdf2c088e48c967ad08f79966742._comment b/doc/forum/git-annex_on_Samba_share/comment_1_3e9cfdf2c088e48c967ad08f79966742._comment
new file mode 100644
index 0000000000..dead483161
--- /dev/null
+++ b/doc/forum/git-annex_on_Samba_share/comment_1_3e9cfdf2c088e48c967ad08f79966742._comment
@@ -0,0 +1,12 @@
+[[!comment format=mdwn
+ username="https://www.google.com/accounts/o8/id?id=AItOawmqz6wCn-Q1vzrsHGvEJHOt_T5ZESilxhc"
+ nickname="Sören"
+ subject="Update"
+ date="2013-06-26T17:26:05Z"
+ content="""
+After reading trough this again, I'd like to rephrase my question.
+
+Is it safe if two independent assistants access the same repository at the same time (assuming locking works)?
+
+But, much more important, a big thanks for creating git-annex! I'm using it successfully to keep my files in sync at home and at work and I think it is a really amazing tool. So thanks for the hard work!
+"""]]
diff --git a/doc/forum/git-annex_on_Samba_share/comment_2_9d3df393b7b727653598453d94dd33db._comment b/doc/forum/git-annex_on_Samba_share/comment_2_9d3df393b7b727653598453d94dd33db._comment
new file mode 100644
index 0000000000..7b17cb2a2d
--- /dev/null
+++ b/doc/forum/git-annex_on_Samba_share/comment_2_9d3df393b7b727653598453d94dd33db._comment
@@ -0,0 +1,12 @@
+[[!comment format=mdwn
+ username="http://joeyh.name/"
+ ip="4.152.108.193"
+ subject="comment 2"
+ date="2013-06-26T17:40:21Z"
+ content="""
+If you're asking if you can run two git-annex assistant programs inside the same repository at the same time, the answer is no. Assuming locking works (grin), the second program will detect the first is running and refuse to start.
+
+If your question is, can two or more git-annex assistant or other programs, running in separate repositories, access files in the same remote safely at the same time, the answer is yes! You can, for that matter, run multiple git-annex commands at the same time in the same repository, and they're also entirely safe, thanks to some locking it does.
+
+You should post your story to [[design/assistant/blog/day_288__success_stories]]!
+"""]]
diff --git a/doc/forum/git-annex_on_Ubuntu_13.04_and_13.10_not_working.mdwn b/doc/forum/git-annex_on_Ubuntu_13.04_and_13.10_not_working.mdwn
new file mode 100644
index 0000000000..1a00326d79
--- /dev/null
+++ b/doc/forum/git-annex_on_Ubuntu_13.04_and_13.10_not_working.mdwn
@@ -0,0 +1,12 @@
+I am trying to get git-annex working on Ubuntu, specifically 13.04 and now 13.10 (I thought upgrading to a new distro. would fix the issue).
+
+I have tried:
+
+  * installing via apt-get install git-annex
+  * installing via cabal (full-build)
+
+Both times git-annex install successfully with no errors. I then start the webapp and create a repository (which git-annex creates successfully), I then add the folders and files I want it to sync. Git-annex finds the files and says it has began syncing them, but it never moves past the first batch of files it 'says' it had started syncing. I have waited 5+ hours at one point and nothing has changed. Also, the webapp is incredibly slow, it takes 15+ seconds to perform ANY action (such as saving options or viewing the logs). The issue definitely isn't my computer as I am using an high-end SSD, core i7, DDR3 RAM, etc... 
+
+Also the logs produce no errors of any kind and actually show git-annex adding files, most of the log entries say add 'filedirectory/filename.txt' as an example.
+
+I have gotten this to successfully work on Fedora, but Ubuntu (13.04/13.10) is having serious problems.
diff --git a/doc/forum/git-annex_on_Ubuntu_13.04_and_13.10_not_working/comment_1_209956f3812450a43986d4ca5e647da6._comment b/doc/forum/git-annex_on_Ubuntu_13.04_and_13.10_not_working/comment_1_209956f3812450a43986d4ca5e647da6._comment
new file mode 100644
index 0000000000..2c4cbd2917
--- /dev/null
+++ b/doc/forum/git-annex_on_Ubuntu_13.04_and_13.10_not_working/comment_1_209956f3812450a43986d4ca5e647da6._comment
@@ -0,0 +1,10 @@
+[[!comment format=mdwn
+ username="http://joeyh.name/"
+ ip="64.134.31.139"
+ subject="comment 1"
+ date="2013-10-22T17:57:27Z"
+ content="""
+I don't really understand the problem you're describing. Is the problem with it uploading the files it's added to another remote? Does it sit there saying \"syncing with $remote\" forever? Or what?
+
+As to the webapp speed problem, it seems likely to me this is not related to CPU speed or load. It sounds rather like a DNS timeout problem, or an ipv6 to ipv4 fallback problem. Although since the URL to the webapp is http://127.0.0.1, it can't really be either of those..
+"""]]
diff --git a/doc/forum/git-annex_on_Ubuntu_13.04_and_13.10_not_working/comment_3_12a58b8efe545e09b64760c87849839b._comment b/doc/forum/git-annex_on_Ubuntu_13.04_and_13.10_not_working/comment_3_12a58b8efe545e09b64760c87849839b._comment
new file mode 100644
index 0000000000..6e2d780ec8
--- /dev/null
+++ b/doc/forum/git-annex_on_Ubuntu_13.04_and_13.10_not_working/comment_3_12a58b8efe545e09b64760c87849839b._comment
@@ -0,0 +1,12 @@
+[[!comment format=mdwn
+ username="https://www.google.com/accounts/o8/id?id=AItOawlEeLtIxQCU0qe4q5lbrZp8AcUSUUcRrkg"
+ nickname="Zach"
+ subject="comment 3"
+ date="2013-10-23T02:01:09Z"
+ content="""
+The problem is not with a remote annex. The issue I am having is git-annex never finishes syncing.
+
+Maybe a screenshot will help. http://i.imgur.com/9HSr8bl.png
+
+It never gets passed the Startup scan.
+"""]]
diff --git a/doc/forum/git-annex_on_Ubuntu_13.04_and_13.10_not_working/comment_3_e0f7328603256f25c3be3706ecc9c76c._comment b/doc/forum/git-annex_on_Ubuntu_13.04_and_13.10_not_working/comment_3_e0f7328603256f25c3be3706ecc9c76c._comment
new file mode 100644
index 0000000000..413a807ad7
--- /dev/null
+++ b/doc/forum/git-annex_on_Ubuntu_13.04_and_13.10_not_working/comment_3_e0f7328603256f25c3be3706ecc9c76c._comment
@@ -0,0 +1,12 @@
+[[!comment format=mdwn
+ username="http://joeyh.name/"
+ ip="64.134.31.139"
+ subject="comment 3"
+ date="2013-10-23T15:27:40Z"
+ content="""
+Your screenshot shows git-annex is in the process of adding 6244 files. Depending on the size of those files, it could take some time to read their contents in order to calculate a checksum and add them to the repository. This would not normally take hours, unless your disk is very slow or the files are extremely big.
+
+You might get some useful information if you enable debugging (Configuration -> Preferences -> Enable debug logging) and look at the log.
+
+(This is not \"syncing\" btw. You have not set up any other repository for it to sync data with.)
+"""]]
diff --git a/doc/forum/git-annex_on_Ubuntu_13.04_and_13.10_not_working/comment_4_6bb8e4522241556fb82784d9b834cbfe._comment b/doc/forum/git-annex_on_Ubuntu_13.04_and_13.10_not_working/comment_4_6bb8e4522241556fb82784d9b834cbfe._comment
new file mode 100644
index 0000000000..314d01e891
--- /dev/null
+++ b/doc/forum/git-annex_on_Ubuntu_13.04_and_13.10_not_working/comment_4_6bb8e4522241556fb82784d9b834cbfe._comment
@@ -0,0 +1,10 @@
+[[!comment format=mdwn
+ username="https://www.google.com/accounts/o8/id?id=AItOawlEeLtIxQCU0qe4q5lbrZp8AcUSUUcRrkg"
+ nickname="Zach"
+ subject="comment 4"
+ date="2013-10-25T20:40:26Z"
+ content="""
+Everything works, it was just taking a while on one the folders; let it run overnight, and it was done.
+
+Thanks for the help!
+"""]]
diff --git a/doc/forum/git-annex_on_Ubuntu_13.04_and_13.10_not_working/comment_5_89a5296b461d400b51006074a13a4560._comment b/doc/forum/git-annex_on_Ubuntu_13.04_and_13.10_not_working/comment_5_89a5296b461d400b51006074a13a4560._comment
new file mode 100644
index 0000000000..e6508fe54b
--- /dev/null
+++ b/doc/forum/git-annex_on_Ubuntu_13.04_and_13.10_not_working/comment_5_89a5296b461d400b51006074a13a4560._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="http://joeyh.name/"
+ ip="209.250.56.47"
+ subject="comment 5"
+ date="2013-10-26T15:53:28Z"
+ content="""
+Can you confirm that the files in the directory it was adding were so large that it would normally take hours to checksum them all?
+"""]]
diff --git a/doc/forum/git-annex_on_Ubuntu_13.04_and_13.10_not_working/comment_6_62daef4b4392c951b914a01b3effac11._comment b/doc/forum/git-annex_on_Ubuntu_13.04_and_13.10_not_working/comment_6_62daef4b4392c951b914a01b3effac11._comment
new file mode 100644
index 0000000000..4adcc568f5
--- /dev/null
+++ b/doc/forum/git-annex_on_Ubuntu_13.04_and_13.10_not_working/comment_6_62daef4b4392c951b914a01b3effac11._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="https://www.google.com/accounts/o8/id?id=AItOawlEeLtIxQCU0qe4q5lbrZp8AcUSUUcRrkg"
+ nickname="Zach"
+ subject="comment 6"
+ date="2013-10-26T21:05:07Z"
+ content="""
+The folder that took a while is actually only 291.5 MB total, containing 25,977 items in total. I know that is the folder that was having some issues because I committed each folder separately.
+"""]]
diff --git a/doc/forum/git-annex_on_archlinuxarm__44___armv6.mdwn b/doc/forum/git-annex_on_archlinuxarm__44___armv6.mdwn
new file mode 100644
index 0000000000..edfd0ad4eb
--- /dev/null
+++ b/doc/forum/git-annex_on_archlinuxarm__44___armv6.mdwn
@@ -0,0 +1,20 @@
+Hi!
+
+I can't get git-annex to run on my archlinuxARM sytem. It fails with
+
+    $ git annex test
+    error: git-annex died of signal 11
+
+ Its a RaspberryPi (model B I think) so its an ARM11 (armv6h if I'm not mistaken). I use archlinuxarm because I am familiar with the distribution and you can have a very lean system. There is a very crude package for git-annex arm and armv7 that is build like this: https://github.com/archlinuxarm/PKGBUILDs/blob/master/alarm/git-annex/PKGBUILD . Crude, because it just takes the debian package, changes the version of libraries where apropriate and packs it as archlinux package. Two problems: 1) It's not been tested on armv6. 2) The 4.20130417 version does not exist anymore on the debian mirrors.
+
+1. The debian package is not marked as armv5 or armv7 specific and is runs on the RaspberryPi with Raspbian (but in an pretty old version if I remember correctly). So I would have imagined it runs on armv6h in general.\\
+2. I just did the same magic as in the linked PKGBUILD with a more current version to build the package but it still crashes. Can I get the old version anywhere to check if it works with that? 
+
+
+Cheers,
+  Hannes
+
+Background for whoever's interested :)
+I tried to set up a RaspberryPi to manage my files, contacts and calendars. That shouts out loud for owncloud because of its convenience (caldav, carddav, webdav and a nice interface bundled together). However, compared to git-annex owncloud is unbearably slow (on an RaspberryPi) and lacks git-annex's easy way to keep your files sorted even if you don't have them on your local disk.
+
+For these reasons I tried to install a hybrid git-annex/owncloud system. There, basically, the files are served by owncloud but at the same time also managed (and served) by git-annex (+ssh). As far as I can see that should not have any noticeable side effects in my use case (mostly single user). I have some external hard drives connected to the RaspberryPi of which I will just symlink the appropriate folders to the owncloud 'files' folder. This way I can also just take the drives with me and still have the same (UUID) git annex repository on them. Any comments?
diff --git a/doc/forum/git-annex_on_archlinuxarm__44___armv6/comment_1_88fa644df8614c2db0d092b3eb1d3156._comment b/doc/forum/git-annex_on_archlinuxarm__44___armv6/comment_1_88fa644df8614c2db0d092b3eb1d3156._comment
new file mode 100644
index 0000000000..516ec0017d
--- /dev/null
+++ b/doc/forum/git-annex_on_archlinuxarm__44___armv6/comment_1_88fa644df8614c2db0d092b3eb1d3156._comment
@@ -0,0 +1,10 @@
+[[!comment format=mdwn
+ username="http://joeyh.name/"
+ ip="4.154.4.51"
+ subject="comment 1"
+ date="2013-09-12T16:56:50Z"
+ content="""
+You seem to be trying to take a binary built for one distribution with one CPU architecture and use it on another distribution on another CPU architecture. It's not very susprising that this does not work.
+
+Compiling git-annex from source for your particular architecture and distribution should not be difficult.
+"""]]
diff --git a/doc/forum/git-annex_on_archlinuxarm__44___armv6/comment_2_b25ca7520ff7e339ec887a379d5100ee._comment b/doc/forum/git-annex_on_archlinuxarm__44___armv6/comment_2_b25ca7520ff7e339ec887a379d5100ee._comment
new file mode 100644
index 0000000000..c0a13cfa46
--- /dev/null
+++ b/doc/forum/git-annex_on_archlinuxarm__44___armv6/comment_2_b25ca7520ff7e339ec887a379d5100ee._comment
@@ -0,0 +1,10 @@
+[[!comment format=mdwn
+ username="https://www.google.com/accounts/o8/id?id=AItOawmNvX8PQVP5sLzQ78sKpB6VeH3Fu8HvZ5g"
+ nickname="Jeff"
+ subject="comment 2"
+ date="2014-01-26T04:29:09Z"
+ content="""
+I've actually been having a pretty tough time with it. Could you point me in the right direction? There's not any Haskell infrastructure on Arch ARM yet so I'd either be compiling the whole Haskell Platform or cross-compiling on x86_64 right?
+Thanks
+Jeff
+"""]]
diff --git a/doc/forum/git-annex_on_archlinuxarm__44___armv6/comment_3_eda0e90c1285396b1ab20ecc04ea6e29._comment b/doc/forum/git-annex_on_archlinuxarm__44___armv6/comment_3_eda0e90c1285396b1ab20ecc04ea6e29._comment
new file mode 100644
index 0000000000..5c8f96ac32
--- /dev/null
+++ b/doc/forum/git-annex_on_archlinuxarm__44___armv6/comment_3_eda0e90c1285396b1ab20ecc04ea6e29._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="http://joeyh.name/"
+ ip="209.250.56.46"
+ subject="comment 3"
+ date="2014-01-26T17:59:52Z"
+ content="""
+Well, yes, the first step will be getting a ghc for the architecture, and then it's just a matter of following [[install/cabal]]
+"""]]
diff --git a/doc/forum/git-annex_on_openSUSE.mdwn b/doc/forum/git-annex_on_openSUSE.mdwn
new file mode 100644
index 0000000000..08c3cd22fc
--- /dev/null
+++ b/doc/forum/git-annex_on_openSUSE.mdwn
@@ -0,0 +1,20 @@
+I run into some problems installing git-annex on openSuSE:
+
+1) cabal install ends up with the failure:
+
+-->>> ExitFailure 1 network-protocol-xmpp-0.4.6 depends on gnuidn-0.2.1 which failed to install.
+
+2) with the tarball the files are not synchronized, only linked
+
+Here the link to the discussion in a suseforum:
+
+https://forums.opensuse.org/showthread.php/506098-How-to-install-git-annex-on-13-1-2
+
+It would be great to see git-annex in the opensuse repos,
+maybe via the open build service, https://build.opensuse.org
+
+Are there other ways to install git-annex on opensuse successfully?
+
+Thanks in advance
+
+saneP
diff --git a/doc/forum/git-annex_on_openSUSE/comment_1_97313a8a7fc7f558b5956989442a0f0b._comment b/doc/forum/git-annex_on_openSUSE/comment_1_97313a8a7fc7f558b5956989442a0f0b._comment
new file mode 100644
index 0000000000..9a000b817a
--- /dev/null
+++ b/doc/forum/git-annex_on_openSUSE/comment_1_97313a8a7fc7f558b5956989442a0f0b._comment
@@ -0,0 +1,13 @@
+[[!comment format=mdwn
+ username="joey"
+ subject="""comment 1"""
+ date="2015-03-27T20:53:23Z"
+ content="""
+To install gnuidn, you need to install the libidn C library.
+
+That is only needed for XMPP support. So you can avoid it with:
+cabal configure -f-XMPP
+
+There is already an openSUSE build, afaik.
+
+"""]]
diff --git a/doc/forum/git-annex_on_openSUSE/comment_2_bbe6925ca465d13b050639c18abcd890._comment b/doc/forum/git-annex_on_openSUSE/comment_2_bbe6925ca465d13b050639c18abcd890._comment
new file mode 100644
index 0000000000..fe6fa73f20
--- /dev/null
+++ b/doc/forum/git-annex_on_openSUSE/comment_2_bbe6925ca465d13b050639c18abcd890._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="saneP"
+ subject="comment 2"
+ date="2015-03-28T14:05:17Z"
+ content="""
+No, there is no adequate package. Maybe it has been removed..
+
+"""]]
diff --git a/doc/forum/git-annex_on_osx_10.9.1_just_crashes__47__closes__47__doesn__39__t_run_on_launch.mdwn b/doc/forum/git-annex_on_osx_10.9.1_just_crashes__47__closes__47__doesn__39__t_run_on_launch.mdwn
new file mode 100644
index 0000000000..c5d4c40bcf
--- /dev/null
+++ b/doc/forum/git-annex_on_osx_10.9.1_just_crashes__47__closes__47__doesn__39__t_run_on_launch.mdwn
@@ -0,0 +1,7 @@
+on a Mac OSX 10.9.1 client I would like to try out git-annex.  
+
+I download the latest Mavericks build, extract from the .dmg and it just does not run.  I open the program, one "bounce" on the dock and it is gone, nothing happens.  
+
+I sadly do not have another OSX client to try it on.  But nothing shows in the Console.app or anything.
+
+Am I missing something?
diff --git a/doc/forum/git-annex_on_osx_10.9.1_just_crashes__47__closes__47__doesn__39__t_run_on_launch/comment_1_a47174f8438bfaa42fb8067bca77bf4c._comment b/doc/forum/git-annex_on_osx_10.9.1_just_crashes__47__closes__47__doesn__39__t_run_on_launch/comment_1_a47174f8438bfaa42fb8067bca77bf4c._comment
new file mode 100644
index 0000000000..3c9361cbdc
--- /dev/null
+++ b/doc/forum/git-annex_on_osx_10.9.1_just_crashes__47__closes__47__doesn__39__t_run_on_launch/comment_1_a47174f8438bfaa42fb8067bca77bf4c._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="http://joeyh.name/"
+ ip="209.250.56.146"
+ subject="comment 1"
+ date="2014-03-05T22:30:00Z"
+ content="""
+The thing to do is to try to start git-annex from the command line in a terminal. This should get an error message that will help find out what's going wrong.
+"""]]
diff --git a/doc/forum/git-annex_on_osx_10.9.1_just_crashes__47__closes__47__doesn__39__t_run_on_launch/comment_2_567bb460cec7cd2135386acf4e7dceb4._comment b/doc/forum/git-annex_on_osx_10.9.1_just_crashes__47__closes__47__doesn__39__t_run_on_launch/comment_2_567bb460cec7cd2135386acf4e7dceb4._comment
new file mode 100644
index 0000000000..21141e45e7
--- /dev/null
+++ b/doc/forum/git-annex_on_osx_10.9.1_just_crashes__47__closes__47__doesn__39__t_run_on_launch/comment_2_567bb460cec7cd2135386acf4e7dceb4._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="http://joeyh.name/"
+ ip="209.250.56.146"
+ subject="comment 2"
+ date="2014-03-07T15:17:50Z"
+ content="""
+Probably this was because the OSX build was missing the webapp. Fixed now.
+"""]]
diff --git a/doc/forum/git-annex_pre-commit_eats_all_my_4GB_of_ram.mdwn b/doc/forum/git-annex_pre-commit_eats_all_my_4GB_of_ram.mdwn
new file mode 100644
index 0000000000..82ca06b85f
--- /dev/null
+++ b/doc/forum/git-annex_pre-commit_eats_all_my_4GB_of_ram.mdwn
@@ -0,0 +1,24 @@
+Hi all,
+
+git-annex basically renders my repository unmanageble.  What is the
+best and save (!) way to recover?
+
+Here is my situation:
+
+I have a fairly large repository with ~8000 managed files taking about
+65GB of disk space.
+
+git-annex worked well there.  But some programs choke on the
+symlinks.  So, I converted the repository to direct mode.  The
+transition worked well.
+
+Now git status reports a type change for the ~8000 files.
+
+But as soon as I run
+
+    git commit -m "typechange" even-only-one-of-the-files
+
+the process `git-annex pre-commit .` eats 3.5GB of ram, where I
+usually kill it, as I only have 4GB of ram....
+
+-- Andreas
diff --git a/doc/forum/git-annex_pre-commit_eats_all_my_4GB_of_ram/comment_1_ff16c7932b60b85c744bafc48bb040e4._comment b/doc/forum/git-annex_pre-commit_eats_all_my_4GB_of_ram/comment_1_ff16c7932b60b85c744bafc48bb040e4._comment
new file mode 100644
index 0000000000..23e5e79e88
--- /dev/null
+++ b/doc/forum/git-annex_pre-commit_eats_all_my_4GB_of_ram/comment_1_ff16c7932b60b85c744bafc48bb040e4._comment
@@ -0,0 +1,13 @@
+[[!comment format=mdwn
+ username="https://me.yahoo.com/a/FHnTlSBo1eCGJRwueeKeB6.RCaPbGMPr5jxx8A--#ce0d8"
+ nickname="Hamza"
+ subject="comment 1"
+ date="2013-09-18T07:19:49Z"
+ content="""
+AFAIK in direct mode you are not supposed to commit you just run
+
+    git annex sync
+
+and it will commit if files are changed. you only add new files annexed files are handled by sync
+
+"""]]
diff --git a/doc/forum/git-annex_pre-commit_eats_all_my_4GB_of_ram/comment_2_5599cddf579d18f70cab6e48d04ae99d._comment b/doc/forum/git-annex_pre-commit_eats_all_my_4GB_of_ram/comment_2_5599cddf579d18f70cab6e48d04ae99d._comment
new file mode 100644
index 0000000000..337c1627fb
--- /dev/null
+++ b/doc/forum/git-annex_pre-commit_eats_all_my_4GB_of_ram/comment_2_5599cddf579d18f70cab6e48d04ae99d._comment
@@ -0,0 +1,12 @@
+[[!comment format=mdwn
+ username="ringprince"
+ ip="134.76.140.110"
+ subject="comment 2"
+ date="2013-09-18T12:27:38Z"
+ content="""
+Hi Hamza,
+
+thanks for that comment.  I thought `git annex sync` is just a wrapper around `git commit -a` (among others).
+
+Using `git annex sync` does not help, as it just means that now `git annex sync` eats all my memory until swapping starts.
+"""]]
diff --git a/doc/forum/git-annex_pre-commit_eats_all_my_4GB_of_ram/comment_3_412941e9385f63153b23695641e71deb._comment b/doc/forum/git-annex_pre-commit_eats_all_my_4GB_of_ram/comment_3_412941e9385f63153b23695641e71deb._comment
new file mode 100644
index 0000000000..6349500caa
--- /dev/null
+++ b/doc/forum/git-annex_pre-commit_eats_all_my_4GB_of_ram/comment_3_412941e9385f63153b23695641e71deb._comment
@@ -0,0 +1,15 @@
+[[!comment format=mdwn
+ username="http://joeyh.name/"
+ ip="4.153.14.105"
+ subject="comment 3"
+ date="2013-09-19T18:49:05Z"
+ content="""
+git-annex sync is a wrapper around git commit. But not -a! git commit -a will stage every one of your large files directly into the git repository, wasting much memory and worse, disk space. It is ok to use `git commit` or `git commit --staged` in direct mode after eg `git annex add`. But not `git commit -a` or `git commit even-only-one-of-the-files`. It's best to just use `git annex sync` rather than `git commit`, as it avoids finger memory causing you to run the wrong type of commit command. Please see [[direct_mode]] for the details.
+
+I was able to make pre-commit take a lot of memory by committing a 1 gb file directly to git. git-annex was buffering the whole file content in memory
+due to not thinking to check first if it was a symlink. I have fixed that bug.
+
+So I think you must have run that command you showed, and you now have a lot of data stored in your git repository that you had meant git-annex to handle. You might need to use git-filter-branch to remove it..
+
+This kind of thing is why I need to write the [[todo/direct_mode_guard]].
+"""]]
diff --git a/doc/forum/git-annex_pre-commit_eats_all_my_4GB_of_ram/comment_4_112ad140d9006c530db2121bec24de30._comment b/doc/forum/git-annex_pre-commit_eats_all_my_4GB_of_ram/comment_4_112ad140d9006c530db2121bec24de30._comment
new file mode 100644
index 0000000000..e5ebfa3281
--- /dev/null
+++ b/doc/forum/git-annex_pre-commit_eats_all_my_4GB_of_ram/comment_4_112ad140d9006c530db2121bec24de30._comment
@@ -0,0 +1,10 @@
+[[!comment format=mdwn
+ username="ringprince"
+ ip="134.76.140.110"
+ subject="comment 4"
+ date="2013-09-19T20:06:39Z"
+ content="""
+Let me ask a simple question:  All files are regular files (no symlinks).  If I can live with loosing the history, is it save to just remove the .git directory and start over?  Or do I risk anything?
+
+PS: .git is about 7GB
+"""]]
diff --git a/doc/forum/git-annex_pre-commit_eats_all_my_4GB_of_ram/comment_5_9178023b95683a649355f291165a1467._comment b/doc/forum/git-annex_pre-commit_eats_all_my_4GB_of_ram/comment_5_9178023b95683a649355f291165a1467._comment
new file mode 100644
index 0000000000..7f5047ddd6
--- /dev/null
+++ b/doc/forum/git-annex_pre-commit_eats_all_my_4GB_of_ram/comment_5_9178023b95683a649355f291165a1467._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="http://joeyh.name/"
+ ip="4.153.14.105"
+ subject="comment 5"
+ date="2013-09-19T20:57:18Z"
+ content="""
+You can safely delete the git repository in the situation you describe. Better yet: Tar it up, and check the tarball into git-annex with the other files when you start over. :) Then if I'm wrong, you still have that data.
+"""]]
diff --git a/doc/forum/git-annex_pre-commit_eats_all_my_4GB_of_ram/comment_6_9251203421c1c3c3aed7828c4b97ecb8._comment b/doc/forum/git-annex_pre-commit_eats_all_my_4GB_of_ram/comment_6_9251203421c1c3c3aed7828c4b97ecb8._comment
new file mode 100644
index 0000000000..ebb378e959
--- /dev/null
+++ b/doc/forum/git-annex_pre-commit_eats_all_my_4GB_of_ram/comment_6_9251203421c1c3c3aed7828c4b97ecb8._comment
@@ -0,0 +1,17 @@
+[[!comment format=mdwn
+ username="ringprince"
+ ip="134.76.140.110"
+ subject="comment 6"
+ date="2013-09-19T22:27:38Z"
+ content="""
+Thanks a bunch!  That is what I did now.  Seems good.
+
+May I additionally suggest a change in the man page of git annex?  There I read
+
+    sync [remote ...]
+              ...
+              The sync process involves first committing all local changes (git commit -a), ...
+              ...
+
+which made me think, I could do `git commit -a` manually as well.
+"""]]
diff --git a/doc/forum/git-annex_sync_content_available_from_which_version__63__.mdwn b/doc/forum/git-annex_sync_content_available_from_which_version__63__.mdwn
new file mode 100644
index 0000000000..511a0ba0ca
--- /dev/null
+++ b/doc/forum/git-annex_sync_content_available_from_which_version__63__.mdwn
@@ -0,0 +1,12 @@
+Hi,
+
+I'm on a centos 6.4 64bits & installed git-annex
+version : git-annex-3.20120522-2.1.el6.x86_64
+
+From my understanding of the help page, I should be able to sync content with :
+
+git annex sync --content
+
+but it is not recognized. The help tells me I may be able to use "get" but I'm unclear how...
+
+Thanks for any help
diff --git a/doc/forum/git-annex_sync_content_available_from_which_version__63__/comment_1_104e1b7e7643844f221f85dcbe9c9c4b._comment b/doc/forum/git-annex_sync_content_available_from_which_version__63__/comment_1_104e1b7e7643844f221f85dcbe9c9c4b._comment
new file mode 100644
index 0000000000..3b7967e32d
--- /dev/null
+++ b/doc/forum/git-annex_sync_content_available_from_which_version__63__/comment_1_104e1b7e7643844f221f85dcbe9c9c4b._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="http://joeyh.name/"
+ ip="108.236.230.124"
+ subject="comment 1"
+ date="2014-06-05T17:32:33Z"
+ content="""
+This feature was added in 5.20140210, which can be easily seen by reading the changelog.
+"""]]
diff --git a/doc/forum/git-annex_sync_fails_in_repo1_but_not_in_repo2.mdwn b/doc/forum/git-annex_sync_fails_in_repo1_but_not_in_repo2.mdwn
new file mode 100644
index 0000000000..222befd167
--- /dev/null
+++ b/doc/forum/git-annex_sync_fails_in_repo1_but_not_in_repo2.mdwn
@@ -0,0 +1,45 @@
+Hello,
+
+I am using two repos in direct mode that are connected via ssh.
+Syncing fails in one direction but works in the other, I don't understand how to fix that.
+
+## repo2 -> repo1 works:
+
+    server$ git-annex sync
+    commit  ok
+    pull repo1 
+    From client:~/media/img/db
+     + be62780...2d74673 master     -> client/master  (forced update)
+     + be62780...2d74673 synced/master -> client/synced/master  (forced update)
+    ok
+    push repo1 
+    Total 0 (delta 0), reused 0 (delta 0)
+    To client:~/media/img/db
+       2d74673..be62780  annex/direct/master -> synced/master
+    ok
+    server$ git-annex status
+    ? .t/repo
+    ? .t/tmprepo0
+    server$
+
+## repo1 -> repo2 fails:
+
+    client$ git-annex sync
+    commit  ok
+    pull repo2 
+    
+    fatal: You have not concluded your merge (MERGE_HEAD exists).
+    Please, commit your changes before you merge.
+    
+    fatal: You have not concluded your merge (MERGE_HEAD exists).
+    Please, commit your changes before you merge.
+    failed
+    git-annex: sync: 1 failed
+    client$ git-annex status
+    client$
+
+Hopefully somebody knows how to fix that.
+
+Regards,
+
+Felix
diff --git a/doc/forum/git-annex_sync_fails_in_repo1_but_not_in_repo2/comment_1_5cca2fbce3db1c691f43e43f9343f9bd._comment b/doc/forum/git-annex_sync_fails_in_repo1_but_not_in_repo2/comment_1_5cca2fbce3db1c691f43e43f9343f9bd._comment
new file mode 100644
index 0000000000..8f2bd5712a
--- /dev/null
+++ b/doc/forum/git-annex_sync_fails_in_repo1_but_not_in_repo2/comment_1_5cca2fbce3db1c691f43e43f9343f9bd._comment
@@ -0,0 +1,10 @@
+[[!comment format=mdwn
+ username="joey"
+ subject="""comment 1"""
+ date="2018-01-15T18:19:02Z"
+ content="""
+It sounds like deleting `.git/MERGE_HEAD` might fix the problem in the repo
+with the problem.
+
+I don't know how you could have gotten the repository into that state..
+"""]]
diff --git a/doc/forum/git-annex_sync_fails_in_repo1_but_not_in_repo2/comment_2_1dfeb722edd54c68b4220d00c0a175e5._comment b/doc/forum/git-annex_sync_fails_in_repo1_but_not_in_repo2/comment_2_1dfeb722edd54c68b4220d00c0a175e5._comment
new file mode 100644
index 0000000000..b40f223947
--- /dev/null
+++ b/doc/forum/git-annex_sync_fails_in_repo1_but_not_in_repo2/comment_2_1dfeb722edd54c68b4220d00c0a175e5._comment
@@ -0,0 +1,35 @@
+[[!comment format=mdwn
+ username="felix.hagemann@b76e9ea0928cf33dacffc37ec3dbecf33171a8a5"
+ nickname="felix.hagemann"
+ avatar="http://cdn.libravatar.org/avatar/1f7e89860de517a494f35ebfb385288e"
+ subject="comment 2"
+ date="2018-01-16T19:26:03Z"
+ content="""
+Deleting `.git/MERGE_HEAD` sort of fixed the problem:
+
+        client$ git-annex sync
+        commit  ok
+        pull repo2 
+
+        Already up to date!
+        Automatic merge went well; stopped before committing as requested
+
+        error: duplicate parent be6278073504cfd19400c100b1871ba4324f55db ignored
+        ok
+        push repo2 
+        Counting objects: 1, done.
+        Writing objects: 100% (1/1), 209 bytes | 209.00 KiB/s, done.
+        Total 1 (delta 0), reused 0 (delta 0)
+        To repo2:/media/srv/img
+           be62780735..020ce83c1f  annex/direct/master -> synced/master
+        ok
+        client$ git-annex sync
+        commit  ok
+        pull repo2 
+        ok
+        client$
+
+Thanks!
+
+I have no idea how the repo got in that state. Syncing started to fail some time early December and for a few months I've been only using `git-annex assistant` and `git-annex webapp`.
+"""]]
diff --git a/doc/forum/git-annex_sync_here.mdwn b/doc/forum/git-annex_sync_here.mdwn
new file mode 100644
index 0000000000..9aa91ed2f4
--- /dev/null
+++ b/doc/forum/git-annex_sync_here.mdwn
@@ -0,0 +1,19 @@
+Hello,
+
+git-annex does a lot of convenient stuff, also locally. With git-annex sync [remote-name] the synchronization can be restricted to a single remote.
+
+However, I think being able to restrict sync to no remote at all, only doing the local stuff, would be helpful.
+
+The natural syntax for me would be git-annex sync here. But it says:
+
+    $ git-annex sync here
+    commit  ok
+    git-annex: there is no available git remote named "here"
+
+I think a workaround would be to do:
+
+    $ git-annex sync --no-push --no-pull
+
+Is that correct?
+
+Still, I think "git-annex sync here" would be a convenient shortcut.
diff --git a/doc/forum/git-annex_sync_here/comment_1_3ebef27614bd72b38404c1940007e68c._comment b/doc/forum/git-annex_sync_here/comment_1_3ebef27614bd72b38404c1940007e68c._comment
new file mode 100644
index 0000000000..7003ce5a4f
--- /dev/null
+++ b/doc/forum/git-annex_sync_here/comment_1_3ebef27614bd72b38404c1940007e68c._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="joey"
+ subject="""comment 1"""
+ date="2017-05-24T17:06:29Z"
+ content="""
+Well, that does the same thing as `git commit -a`, so I don't see
+any benefit complicating the command with additional syntax.
+"""]]
diff --git a/doc/forum/git-annex_sync_here/comment_2_1b0543443759202ec6a85a26088ddb43._comment b/doc/forum/git-annex_sync_here/comment_2_1b0543443759202ec6a85a26088ddb43._comment
new file mode 100644
index 0000000000..ba3ff18cea
--- /dev/null
+++ b/doc/forum/git-annex_sync_here/comment_2_1b0543443759202ec6a85a26088ddb43._comment
@@ -0,0 +1,9 @@
+[[!comment format=mdwn
+ username="mario"
+ avatar="http://cdn.libravatar.org/avatar/4c63b0935789d29210d0bd8cad8d7ac7"
+ subject="comment 2"
+ date="2017-05-25T12:47:18Z"
+ content="""
+git-annex sync does more than that. Especially merging the synced branches and, thus, showing files that were pushed into a repo from a remote. Also, git commit -a cannot be done in direct mode.
+But, I think I can just create an alias for git-annex sync --no-push --no-pull to save me some typing.
+"""]]
diff --git a/doc/forum/git-annex_teams___47___groups.mdwn b/doc/forum/git-annex_teams___47___groups.mdwn
new file mode 100644
index 0000000000..b499f3a6e4
--- /dev/null
+++ b/doc/forum/git-annex_teams___47___groups.mdwn
@@ -0,0 +1,5 @@
+Hi,
+
+Does git-annex (preferably assistant) have any management of users / groups or teams? Our use case is we have many users which should only have access to certain folders (repos). Is this a planned feature or any manual way to work around this for now?
+
+Thanks!
diff --git a/doc/forum/git-annex_teams___47___groups/comment_1_0450673ab74f184a47ac7bab568d26dc._comment b/doc/forum/git-annex_teams___47___groups/comment_1_0450673ab74f184a47ac7bab568d26dc._comment
new file mode 100644
index 0000000000..4b77da8f55
--- /dev/null
+++ b/doc/forum/git-annex_teams___47___groups/comment_1_0450673ab74f184a47ac7bab568d26dc._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="http://joeyh.name/"
+ ip="4.152.108.193"
+ subject="comment 1"
+ date="2013-06-25T17:10:17Z"
+ content="""
+Anyone with access to a git repository can access all the files stored in it. git-annex does not try to change this, so you need to find a way to have one repository for each group.
+"""]]
diff --git a/doc/forum/git-annex_unused_not_dropping_deleted_files.mdwn b/doc/forum/git-annex_unused_not_dropping_deleted_files.mdwn
new file mode 100644
index 0000000000..1669461bc5
--- /dev/null
+++ b/doc/forum/git-annex_unused_not_dropping_deleted_files.mdwn
@@ -0,0 +1,34 @@
+I've been organizing my music collection deleted/replaces some files but git-annex unused does not show any unused files even though they are deleted and all repos are in sync.
+
+
+    git log --stat --all -S'SHA256E-s8034842--5c3475d7fef6f0c3545721f34e7cbfb6727a00708bdde192f0f1d53af251b444'
+    commit a0fecdc02f7564f8bce9726f6b934fefc11de58b
+    Date:   Thu Sep 12 17:04:53 2013 +0300
+
+        Deleted
+
+     .../Dido - Sitting On the Roof of the World.mp3    |    1 -
+     1 file changed, 1 deletion(-)
+
+    commit 7f216228fc0e6298f0290ee1d8646bc9b16eca10
+    Date:   Mon Aug 5 03:49:55 2013 +0200
+
+        Initial Import
+
+     .../Dido - Sitting On the Roof of the World.mp3    |    1 +
+     1 file changed, 1 insertion(+)`
+
+even though the file is deleted its still present in annex objects,
+
+    find .git -name '*5c3475d7fef6f0c3545721f34e7cbfb6727a00708bdde192f0f1d53af251b444*' -exec ls -al '{}' \;
+    total 7856
+    dr-xr-xr-x 2 user user    4096 Sep  9 01:24 .
+    drwxrwxr-x 3 user user    4096 Sep  9 01:24 ..
+    -r--r--r-- 1 user user 8034842 Aug  5 04:52 SHA256E-s8034842--5c3475d7fef6f0c3545721f34e7cbfb6727a00708bdde192f0f1d53af251b444.mp3
+    -r--r--r-- 1 user user 8034842 Aug  5 04:52 .git/annex/objects/Jz/74/SHA256E-s8034842--5c3475d7fef6f0c3545721f34e7cbfb6727a00708bdde192f0f1d53af251b444.mp3/SHA256E-s8034842--5c3475d7fef6f0c3545721f34e7cbfb6727a00708bdde192f0f1d53af251b444.mp3
+
+I got around 200 files that should be deleted but not showing up in unused. I though maybe one of the dead repos is causing the problem so i did a,
+
+    git annex forget --drop-dead
+
+and synced all repos. Still I can not get them to drop.
diff --git a/doc/forum/git-annex_unused_not_dropping_deleted_files/comment_1_2152cfb09675e46e7492e198dd3ea094._comment b/doc/forum/git-annex_unused_not_dropping_deleted_files/comment_1_2152cfb09675e46e7492e198dd3ea094._comment
new file mode 100644
index 0000000000..72ed0bbe17
--- /dev/null
+++ b/doc/forum/git-annex_unused_not_dropping_deleted_files/comment_1_2152cfb09675e46e7492e198dd3ea094._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="http://joeyh.name/"
+ ip="4.154.2.134"
+ subject="comment 1"
+ date="2013-09-13T15:34:03Z"
+ content="""
+The most likely explanation is that you have a branch in git that still contains the deleted file, which will prevent unused from removing it. Try `git branch -a` to see all branches (a tag could also do it).
+"""]]
diff --git a/doc/forum/git-annex_unused_not_dropping_deleted_files/comment_2_97e666dbac9de2a5e688921cba8a42e9._comment b/doc/forum/git-annex_unused_not_dropping_deleted_files/comment_2_97e666dbac9de2a5e688921cba8a42e9._comment
new file mode 100644
index 0000000000..134106482c
--- /dev/null
+++ b/doc/forum/git-annex_unused_not_dropping_deleted_files/comment_2_97e666dbac9de2a5e688921cba8a42e9._comment
@@ -0,0 +1,20 @@
+[[!comment format=mdwn
+ username="https://me.yahoo.com/a/FHnTlSBo1eCGJRwueeKeB6.RCaPbGMPr5jxx8A--#ce0d8"
+ nickname="Hamza"
+ subject="comment 2"
+ date="2013-09-13T15:59:29Z"
+ content="""
+I did not create tags or branches, running returns,
+
+    git branch -a  
+
+      git-annex
+    * master
+      synced/git-annex
+      synced/master
+      remotes/origin/HEAD -> origin/master
+      remotes/origin/git-annex
+      remotes/origin/master
+      remotes/origin/synced/master
+
+"""]]
diff --git a/doc/forum/git-annex_unused_not_dropping_deleted_files/comment_3_d7b0e9515bface28f3650b8aa20ec2f4._comment b/doc/forum/git-annex_unused_not_dropping_deleted_files/comment_3_d7b0e9515bface28f3650b8aa20ec2f4._comment
new file mode 100644
index 0000000000..666172b289
--- /dev/null
+++ b/doc/forum/git-annex_unused_not_dropping_deleted_files/comment_3_d7b0e9515bface28f3650b8aa20ec2f4._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="http://joeyh.name/"
+ ip="4.154.2.134"
+ subject="comment 3"
+ date="2013-09-13T16:17:39Z"
+ content="""
+Depending on your git configuration, `git annex sync` might not be updating certian of these branches, such as remotes/origin/HEAD. You can check the branches out and see which still contains your deleted files.
+"""]]
diff --git a/doc/forum/git-annex_unused_not_dropping_deleted_files/comment_4_5816f6cab42e27e724e735368f693b09._comment b/doc/forum/git-annex_unused_not_dropping_deleted_files/comment_4_5816f6cab42e27e724e735368f693b09._comment
new file mode 100644
index 0000000000..e1a503d232
--- /dev/null
+++ b/doc/forum/git-annex_unused_not_dropping_deleted_files/comment_4_5816f6cab42e27e724e735368f693b09._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="https://me.yahoo.com/a/FHnTlSBo1eCGJRwueeKeB6.RCaPbGMPr5jxx8A--#ce0d8"
+ nickname="Hamza"
+ subject="comment 4"
+ date="2013-09-13T17:57:38Z"
+ content="""
+Which one of those branches I can safely delete? this is a central repo all other clients drop their stuff here it does not have any remotes (I did clones it from a client then removed origin section from config.)
+"""]]
diff --git a/doc/forum/git-annex_unused_not_dropping_deleted_files/comment_5_8e97f39225515f0bf8b168dfd6a0efab._comment b/doc/forum/git-annex_unused_not_dropping_deleted_files/comment_5_8e97f39225515f0bf8b168dfd6a0efab._comment
new file mode 100644
index 0000000000..1da1f80db7
--- /dev/null
+++ b/doc/forum/git-annex_unused_not_dropping_deleted_files/comment_5_8e97f39225515f0bf8b168dfd6a0efab._comment
@@ -0,0 +1,10 @@
+[[!comment format=mdwn
+ username="http://joeyh.name/"
+ ip="4.154.2.134"
+ subject="comment 5"
+ date="2013-09-13T18:23:02Z"
+ content="""
+So this is a bare repository?
+
+It seems to have had an origin at one point; see the remotes/origin/* branches. If it no longer has that remote, then of course those branches will not be getting updated, and so will contain the old files and that's why unused won't find them. In that case, you could safely remove the remotes/origin/* branches.
+"""]]
diff --git a/doc/forum/git-annex_unused_not_dropping_deleted_files/comment_6_bef37f8ec9c337387b79ffd6d56fe425._comment b/doc/forum/git-annex_unused_not_dropping_deleted_files/comment_6_bef37f8ec9c337387b79ffd6d56fe425._comment
new file mode 100644
index 0000000000..fe1c94eca3
--- /dev/null
+++ b/doc/forum/git-annex_unused_not_dropping_deleted_files/comment_6_bef37f8ec9c337387b79ffd6d56fe425._comment
@@ -0,0 +1,10 @@
+[[!comment format=mdwn
+ username="https://me.yahoo.com/a/FHnTlSBo1eCGJRwueeKeB6.RCaPbGMPr5jxx8A--#ce0d8"
+ nickname="Hamza"
+ subject="comment 6"
+ date="2013-09-13T18:41:30Z"
+ content="""
+It is a non bare repo, I am using the following setup,
+
+http://git-annex.branchable.com/forum/Annex_dropping_files/
+"""]]
diff --git a/doc/forum/git-annex_unused_not_dropping_deleted_files/comment_7_bff3bcf9f1b7c458afa98cdb18883153._comment b/doc/forum/git-annex_unused_not_dropping_deleted_files/comment_7_bff3bcf9f1b7c458afa98cdb18883153._comment
new file mode 100644
index 0000000000..f75c321ba0
--- /dev/null
+++ b/doc/forum/git-annex_unused_not_dropping_deleted_files/comment_7_bff3bcf9f1b7c458afa98cdb18883153._comment
@@ -0,0 +1,28 @@
+[[!comment format=mdwn
+ username="joey"
+ subject="""comment 7"""
+ date="2016-01-29T18:20:06Z"
+ content="""
+As someone else ran into a similar case of this, let me explain a little
+bit more..
+
+`git-annex unused` finds files that are not used by the currently checked
+out branch, or any other branch or tag. This includes remote branches.
+
+This is an important safeguard, because there might be a clone of the
+repository that intentionally still has that file checked out, and it
+wouldn't be good for `git-annex unused` to treat such a file as unused.
+
+But if there's an old remote, that's perhaps not getting updated any longer
+and has a branch that points to a file, `git annex unused` won't find that file
+as unused.
+
+So, one solution is to use `git remote rm` to delete the old remote. Or,
+otherwise get git to delete the old remote branch that is keeping the file
+around.
+
+The other solution is to run something like `git annex unused
+--used-refspec=+master` What this does is only considers the local master
+branch as used, and not any other branch or tags. I'd recommend only using
+this with caution.
+"""]]
diff --git a/doc/forum/git-annex_unused_not_dropping_deleted_files/comment_8_7259b8e9616009472ad3a10426f3d8ea._comment b/doc/forum/git-annex_unused_not_dropping_deleted_files/comment_8_7259b8e9616009472ad3a10426f3d8ea._comment
new file mode 100644
index 0000000000..0e81791f14
--- /dev/null
+++ b/doc/forum/git-annex_unused_not_dropping_deleted_files/comment_8_7259b8e9616009472ad3a10426f3d8ea._comment
@@ -0,0 +1,17 @@
+[[!comment format=mdwn
+ username="viric@582d0845fdeae54b262502f49509b4577a5adbf8"
+ nickname="viric"
+ subject="Use --used-refspec"
+ date="2016-01-29T21:37:35Z"
+ content="""
+\"joeyh\" just helped me on irc, because I had this very same problem. I have git-annex repositories in A, and in external hard disks B, C, D, ...
+
+If I remove a file in A, I mean to 1) free space in A (drop) and 2) this to propagate later to the drives.
+
+No matter what \"git annex sync\" you do from A, the hard disks in your armchair will keep references to the file in their checkout, so A will refuse to drop the file contents until you run \"git annex sync\" from *each* of your external hard disks.
+
+What you need (and what I also needed) was to rule the drop/unused from A alone. You have to use \"git annex unused --used-refspec=+master\" from A, and that
+will only keep into account the contents referenced by your current A master, ignoring the checkouts of B, C, D, ...
+
+The \"used-refspec\" can even be set up in the git config.
+"""]]
diff --git a/doc/forum/git-annex_vicfg_preferred_content_settings_are_not_being_saved.mdwn b/doc/forum/git-annex_vicfg_preferred_content_settings_are_not_being_saved.mdwn
new file mode 100644
index 0000000000..a4386add79
--- /dev/null
+++ b/doc/forum/git-annex_vicfg_preferred_content_settings_are_not_being_saved.mdwn
@@ -0,0 +1 @@
+The settings don't seem to be saving. If I edit it, then immediately open the file again, the settings are what they were before. This happens no matter what repo I do it from.
diff --git a/doc/forum/git-annex_vicfg_preferred_content_settings_are_not_being_saved/comment_1_81111f59caea9f70cb9d597381e42c96._comment b/doc/forum/git-annex_vicfg_preferred_content_settings_are_not_being_saved/comment_1_81111f59caea9f70cb9d597381e42c96._comment
new file mode 100644
index 0000000000..27a719853e
--- /dev/null
+++ b/doc/forum/git-annex_vicfg_preferred_content_settings_are_not_being_saved/comment_1_81111f59caea9f70cb9d597381e42c96._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="http://joeyh.name/"
+ ip="209.250.56.54"
+ subject="comment 1"
+ date="2014-10-12T17:26:35Z"
+ content="""
+Please paste the line that you've configured, and then show how it looks when you open vicfg again.
+"""]]
diff --git a/doc/forum/git-annex_vicfg_preferred_content_settings_are_not_being_saved/comment_2_8ca9156d21d9f3db0d83d6aa9b69caa0._comment b/doc/forum/git-annex_vicfg_preferred_content_settings_are_not_being_saved/comment_2_8ca9156d21d9f3db0d83d6aa9b69caa0._comment
new file mode 100644
index 0000000000..467d45740f
--- /dev/null
+++ b/doc/forum/git-annex_vicfg_preferred_content_settings_are_not_being_saved/comment_2_8ca9156d21d9f3db0d83d6aa9b69caa0._comment
@@ -0,0 +1,17 @@
+[[!comment format=mdwn
+ username="ghen1"
+ ip="66.41.70.34"
+ subject="comment 2"
+ date="2014-10-12T21:03:01Z"
+ content="""
+The custom settings I had made before (the rest are still commented out):
+trust a8372263-6eba-47e2-9604-3e2c9bbb6d42 = trusted
+wanted 3e2bcb1e-39ad-4863-a9ce-a18a262644c1 = present or (not include=/Books/*)
+wanted a8372263-6eba-47e2-9604-3e2c9bbb6d42 = present or include=/Books/*
+
+I am trying to comment them out again.
+
+Based on your answer, I guessed that these settings don't just reset to default when they are commented out, so I tried changing the above trusted setting to semitrusted, and sure enough it worked.
+
+So it seems once these settings are made they can't be commented out again, and any invalid settings are ignored?
+"""]]
diff --git a/doc/forum/git-annex_vicfg_preferred_content_settings_are_not_being_saved/comment_3_9da6ca0250ab0dcfc9a012df75e2e711._comment b/doc/forum/git-annex_vicfg_preferred_content_settings_are_not_being_saved/comment_3_9da6ca0250ab0dcfc9a012df75e2e711._comment
new file mode 100644
index 0000000000..375ee1254b
--- /dev/null
+++ b/doc/forum/git-annex_vicfg_preferred_content_settings_are_not_being_saved/comment_3_9da6ca0250ab0dcfc9a012df75e2e711._comment
@@ -0,0 +1,12 @@
+[[!comment format=mdwn
+ username="http://joeyh.name/"
+ ip="209.250.56.54"
+ subject="comment 3"
+ date="2014-10-12T23:26:27Z"
+ content="""
+vicfg won't accept invalid input, if you make a type in the syntax it'll land you back in the editor with the problem line marked.
+
+But right, commenting out a line does not reset it to the default. Basically, once one of these settings is changed, there is no longer a default to go back to. You have to leave the line uncommented and change the value to what you want it to be. Ie, \"= semitrusted\" for the trust levels, and \"= \" for the wanted expressions.
+
+I think that vicfg should avoid this [[todo/vicfg_comment_gotcha]] and have filed that as a todo item.
+"""]]
diff --git a/doc/forum/git-annex_vicfg_preferred_content_settings_are_not_being_saved/comment_4_55c52c45f3aaddfb63a1f53efe2ee582._comment b/doc/forum/git-annex_vicfg_preferred_content_settings_are_not_being_saved/comment_4_55c52c45f3aaddfb63a1f53efe2ee582._comment
new file mode 100644
index 0000000000..0e73c48695
--- /dev/null
+++ b/doc/forum/git-annex_vicfg_preferred_content_settings_are_not_being_saved/comment_4_55c52c45f3aaddfb63a1f53efe2ee582._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="joey"
+ subject="""comment 4"""
+ date="2014-10-20T15:40:38Z"
+ content="""
+Right. In the meantime, I've fixed the behavior of vicfg when deleting
+lines, so it resets them to the default automatically.
+"""]]
diff --git a/doc/forum/git-annex_webapp_launch_behavior_is_confusing.mdwn b/doc/forum/git-annex_webapp_launch_behavior_is_confusing.mdwn
new file mode 100644
index 0000000000..d061913e82
--- /dev/null
+++ b/doc/forum/git-annex_webapp_launch_behavior_is_confusing.mdwn
@@ -0,0 +1,8 @@
+After first installing git-annex if I run `git-annex webapp` on Ubuntu 16.04.4 LTS (annex version 6.20180528-g5300386c2) then the process does not go to the background, but instead stays a foreground process. I can add files to a repo that assistant is watching and it will see them. But, if I then close the web browser and then kill the foreground process that launched the webapp, then all annex processes go away, and git-annex assistant is no longer running anywhere. IE if I then add a file to an assistant repo it won't be seen.
+
+If I reboot, then a bunch of annex processes are automatically started as daemons. Now my repos are watched again. If I launch git-annex webapp, this time the process immediately forks to the background which is different behavior.
+
+The behavior I would expect is that calling `git-annex webapp` should launch, the webapp dameon, load the page in a browser and call something like `git-annex assistant --autostart` as well.
+
+
+—[Andrew](http://git-annex.branchable.com/users/andrew/)
diff --git a/doc/forum/git-assistant_clarification.mdwn b/doc/forum/git-assistant_clarification.mdwn
new file mode 100644
index 0000000000..472f96c6a9
--- /dev/null
+++ b/doc/forum/git-assistant_clarification.mdwn
@@ -0,0 +1,11 @@
+I've noticed that if I'm using git-assistant, it wants to pull down all my files from other repos onto my laptop, even after I've dropped them.  (My laptop is set up as "client," my usb drive and an ssh server as "backup".)
+
+I want to use git annex to save space on my laptop, but of course when I'm running the assistant, it pulls everything down there, even things I've manually dropped.
+
+Is my "I want to save space, with a partial archive on my laptop" use case simply out of scope for the assistant?  So I should just be using the command line for my needs?  That's fine if that's the case.
+
+Or maybe something like this is what I should be doing?  http://git-annex.branchable.com/assistant/archival_walkthrough/  ?  so instead of manually "git annex drop"-ping files in place, I should set up a directory called "archive" on my machine, from which files will magically disappear and get backed up elsewhere?
+
+If it's the case that a directory named "archive" in your checkout has the magical property of having the assistant drop and archive its contents, that's awesome, maybe just what I need, but if that behavior is spelled out in so many words anywhere I managed to miss it.
+
+Apologies for all these questions, just enjoying the software immensely and wanting to get to know it.
diff --git a/doc/forum/git-assistant_clarification/comment_1_8f553e59da12f798b854a457b96b5778._comment b/doc/forum/git-assistant_clarification/comment_1_8f553e59da12f798b854a457b96b5778._comment
new file mode 100644
index 0000000000..03f9dc2735
--- /dev/null
+++ b/doc/forum/git-assistant_clarification/comment_1_8f553e59da12f798b854a457b96b5778._comment
@@ -0,0 +1,14 @@
+[[!comment format=mdwn
+ username="https://www.google.com/accounts/o8/id?id=AItOawn7gQ1zZDdWhXy9H51W2krZYShNmKL3qfM"
+ nickname="Karsten"
+ subject="comment 1"
+ date="2012-11-30T08:26:24Z"
+ content="""
+I'm second this request. Also, I'm posting this to enable the rss comment subscription button for this post.
+
+Maybe these design documents are interesting:
+
+
+
+
+"""]]
diff --git a/doc/forum/git-assistant_clarification/comment_2_06cf62b599edea6ad8396776f0081494._comment b/doc/forum/git-assistant_clarification/comment_2_06cf62b599edea6ad8396776f0081494._comment
new file mode 100644
index 0000000000..90137e722f
--- /dev/null
+++ b/doc/forum/git-assistant_clarification/comment_2_06cf62b599edea6ad8396776f0081494._comment
@@ -0,0 +1,10 @@
+[[!comment format=mdwn
+ username="http://joeyh.name/"
+ ip="4.154.6.49"
+ subject="comment 2"
+ date="2012-11-30T16:31:21Z"
+ content="""
+The archive directory is a new feature, but yes, it does work. Just use the webapp to edit the configuration of a remote, put it into the \"small archive\" group, and the contents of archive directories will be sent to it.
+
+This is a particular application of [[preferred_content]] settings, which give you a large amount of control over which data end up where when using the assistant.
+"""]]
diff --git a/doc/forum/git-assistant_clarification/comment_3_36f0bd6e7a824e6ef40a309850bb087b._comment b/doc/forum/git-assistant_clarification/comment_3_36f0bd6e7a824e6ef40a309850bb087b._comment
new file mode 100644
index 0000000000..1cf51c296c
--- /dev/null
+++ b/doc/forum/git-assistant_clarification/comment_3_36f0bd6e7a824e6ef40a309850bb087b._comment
@@ -0,0 +1,15 @@
+[[!comment format=mdwn
+ username="http://edheil.wordpress.com/"
+ ip="173.162.44.162"
+ subject="comment 3"
+ date="2012-11-30T18:49:37Z"
+ content="""
+Very cool!
+
+So not only is archive/ a magic subdirectory due to the \"preferred content settings\" for \"client,\" but *every* subdirectory named \"archive\" similarly drops content when possible.  That's awesome to know.
+
+Also if I really wanted to manually manage all my content and still have git-annex assistant running, I could simply use \"present\" for my preferred content settings on my laptop, and it would respect my manual gets and drops.
+
+Thank you!
+
+"""]]
diff --git a/doc/forum/git-like_git-annex_diff.mdwn b/doc/forum/git-like_git-annex_diff.mdwn
new file mode 100644
index 0000000000..59205d6242
--- /dev/null
+++ b/doc/forum/git-like_git-annex_diff.mdwn
@@ -0,0 +1,11 @@
+It took me some trial and error to get `git-annex diffdriver` to work close to the way I wanted (basically, the same way `git diff` works in a normal git repository with `color.ui` set to `always` with `git config`, so I thought I'd put my notes here for others:
+
+* Install `colordiff` (I couldn't figure out what `git`'s actual default diff program was or if it was possible to use it outside of `git diff`, so I went with this.
+* Create a wrapper script to pass to `git-annex diffdriver`. `LANG=C` was to avoid a warning produced by `perl` when it runs `colordiff`. `exit 0` is needed because `colordiff`/`diff` has a non-zero exit status when the files it is passed are not identical and `git` expects the external diff program to exit with status 0:
+
+        #!/usr/bin/env bash                                                             
+        LANG=C colordiff -c "$1" "$2"                                                   
+        exit 0
+
+* Put the wrapper script somewhere on `PATH`, make it executable, and name it something like `git-annex-diff-wrapper`.
+* Configure the repo to use the wrapper: `git config diff.external "git-annex diffdriver -- git-annex-diff-wrapper --"`
diff --git a/doc/forum/git-remote-gcrypt.mdwn b/doc/forum/git-remote-gcrypt.mdwn
new file mode 100644
index 0000000000..eaf5889e7c
--- /dev/null
+++ b/doc/forum/git-remote-gcrypt.mdwn
@@ -0,0 +1 @@
+Back in January Joey [mentioned](http://git-annex.branchable.com/design/assistant/blog/day_179__brief_updates/) the [git-remote-gcrypt](https://github.com/blake2-ppc/git-remote-gcrypt) and possibly adding support for it in the Assistant. I think this would be a great addition. Now that the first big Android push is complete, is there a schedule for this feature?
diff --git a/doc/forum/git-remote-gcrypt/comment_1_175c8c35d9bbb470fcc17697eb8cc6b8._comment b/doc/forum/git-remote-gcrypt/comment_1_175c8c35d9bbb470fcc17697eb8cc6b8._comment
new file mode 100644
index 0000000000..ca38da4cd9
--- /dev/null
+++ b/doc/forum/git-remote-gcrypt/comment_1_175c8c35d9bbb470fcc17697eb8cc6b8._comment
@@ -0,0 +1,12 @@
+[[!comment format=mdwn
+ username="http://joeyh.name/"
+ nickname="joey"
+ subject="comment 1"
+ date="2013-03-07T18:42:37Z"
+ content="""
+I have not put it on the schedule (and don't think I'll get to it this month), but I have added it to the [[roadmap|design/assistant]].
+
+The main use cases it adds are encrypting git remotes on ssh servers, and on USB drives. Both are useful for sure..
+
+It also opens the possibility of using github for encrypted git remotes, but with xmpp git push that is not a pressing use case. With significantly more work, encrypted git remotes could be stored on S3 etc, but again xmpp git push covers the same use cases.
+"""]]
diff --git a/doc/forum/git-remote-gcrypt/comment_2_fdcaf507e14c995636dd93a41e488df3._comment b/doc/forum/git-remote-gcrypt/comment_2_fdcaf507e14c995636dd93a41e488df3._comment
new file mode 100644
index 0000000000..a20fc7e761
--- /dev/null
+++ b/doc/forum/git-remote-gcrypt/comment_2_fdcaf507e14c995636dd93a41e488df3._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="spwhitton"
+ ip="82.36.235.9"
+ subject="comment 2"
+ date="2013-03-12T09:33:17Z"
+ content="""
+The use case this is really good for, that XMPP push can't help with, is for when two paired machines are almost never both turned on at once.
+"""]]
diff --git a/doc/forum/git-remote-gcrypt/comment_3_f4e830f961dbe1c60ddd277b9d888133._comment b/doc/forum/git-remote-gcrypt/comment_3_f4e830f961dbe1c60ddd277b9d888133._comment
new file mode 100644
index 0000000000..03ac7b0f56
--- /dev/null
+++ b/doc/forum/git-remote-gcrypt/comment_3_f4e830f961dbe1c60ddd277b9d888133._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="annexuser"
+ ip="50.125.45.235"
+ subject="comment 3"
+ date="2013-05-25T22:58:16Z"
+ content="""
+I'm still hoping to see this. The feature received some attention on the [April poll](http://git-annex.branchable.com/design/assistant/polls/goals_for_April/).
+"""]]
diff --git a/doc/forum/git-repair_doesn__39__t_build_with_GHC_8.0.2.mdwn b/doc/forum/git-repair_doesn__39__t_build_with_GHC_8.0.2.mdwn
new file mode 100644
index 0000000000..bd6ae58fd7
--- /dev/null
+++ b/doc/forum/git-repair_doesn__39__t_build_with_GHC_8.0.2.mdwn
@@ -0,0 +1,10 @@
+This should be related to directory 1.3.
+
+    Common.hs:3:16: error:
+        Conflicting exports for ‘getFileSize’:
+           ‘module X’ exports ‘X.getFileSize’
+             imported from ‘Utility.FileSize’ at Common.hs:32:1-28
+             (and originally defined at Utility/FileSize.hs:26:1-11)
+           ‘module X’ exports ‘X.getFileSize’
+             imported from ‘Utility.Directory’ at Common.hs:33:1-29
+             (and originally defined in ‘System.Directory’)
diff --git a/doc/forum/git-repair_doesn__39__t_build_with_GHC_8.0.2/comment_1_0a6ea834f6515b888e5c40fef08f4e2d._comment b/doc/forum/git-repair_doesn__39__t_build_with_GHC_8.0.2/comment_1_0a6ea834f6515b888e5c40fef08f4e2d._comment
new file mode 100644
index 0000000000..873e47324b
--- /dev/null
+++ b/doc/forum/git-repair_doesn__39__t_build_with_GHC_8.0.2/comment_1_0a6ea834f6515b888e5c40fef08f4e2d._comment
@@ -0,0 +1,9 @@
+[[!comment format=mdwn
+ username="joey"
+ subject="""comment 1"""
+ date="2017-06-26T16:08:20Z"
+ content="""
+AFAIK, this was fixed already in git-repair's master branch.
+
+I've released a new git-repair with the fixes.
+"""]]
diff --git a/doc/forum/git-repair_doesn__39__t_build_with_GHC_8.0.2/comment_2_bb490f71aed0cfe119beac66cecae99c._comment b/doc/forum/git-repair_doesn__39__t_build_with_GHC_8.0.2/comment_2_bb490f71aed0cfe119beac66cecae99c._comment
new file mode 100644
index 0000000000..b9cb6a9642
--- /dev/null
+++ b/doc/forum/git-repair_doesn__39__t_build_with_GHC_8.0.2/comment_2_bb490f71aed0cfe119beac66cecae99c._comment
@@ -0,0 +1,9 @@
+[[!comment format=mdwn
+ username="https://launchpad.net/~felixonmars"
+ nickname="felixonmars"
+ avatar="http://cdn.libravatar.org/avatar/17284a3bb2e4ad9d3be8fab31f49865be9c1dc22143c728de731fe800a335d38"
+ subject="comment 2"
+ date="2017-06-27T15:57:40Z"
+ content="""
+Works nice here. Thanks a lot!
+"""]]
diff --git a/doc/forum/git-status_typechange_in_direct_mode.mdwn b/doc/forum/git-status_typechange_in_direct_mode.mdwn
new file mode 100644
index 0000000000..6438fb890d
--- /dev/null
+++ b/doc/forum/git-status_typechange_in_direct_mode.mdwn
@@ -0,0 +1,48 @@
+Hi all,
+
+how can I get rid of all those 'typechange' messages in `git status`
+when in direct mode?
+
+Here is an example session:
+
+    > git init
+    Initialized empty Git repository in /some/path/.git/
+    > git config user.name dtrn
+    > git config user.email drn@drn.com
+    > git annex init
+    init  ok
+    (Recording state in git...)
+    > git annex direct
+    commit
+    # On branch master
+    #
+    # Initial commit
+    #
+    nothing to commit (create/copy files and use "git add" to track)
+    ok
+    direct  ok
+    > dd if=/dev/zero of=testfile.bin count=1000
+    1000+0 records in
+    1000+0 records out
+    512000 bytes (512 kB) copied, 0.00317424 s, 161 MB/s
+    > git annex add testfile.bin
+    add testfile.bin (checksum...) ok
+    (Recording state in git...)
+    > git commit -m "annexed testfile.bin"
+    ok
+    [master (root-commit) 281e740] annexed testfile.bin
+     1 file changed, 1 insertion(+)
+     create mode 120000 testfile.bin
+    >  git status
+    # On branch master
+    # Changes not staged for commit:
+    #   (use "git add ..." to update what will be committed)
+    #   (use "git checkout -- ..." to discard changes in working directory)
+    #
+    #       typechange: testfile.bin
+    #
+    no changes added to commit (use "git add" and/or "git commit -a")
+
+
+Regards,
+Andreas
diff --git a/doc/forum/git-status_typechange_in_direct_mode/comment_1_12c8b67aadbfa2b073e12997a55d49a7._comment b/doc/forum/git-status_typechange_in_direct_mode/comment_1_12c8b67aadbfa2b073e12997a55d49a7._comment
new file mode 100644
index 0000000000..31e476087d
--- /dev/null
+++ b/doc/forum/git-status_typechange_in_direct_mode/comment_1_12c8b67aadbfa2b073e12997a55d49a7._comment
@@ -0,0 +1,12 @@
+[[!comment format=mdwn
+ username="http://joeyh.name/"
+ ip="2001:4830:1600:187::2"
+ subject="comment 1"
+ date="2013-09-20T15:24:25Z"
+ content="""
+These messages are normal in direct mode. 
+
+They happen because git status does not know about git-annex's direct mode. So it sees a file that has a symlink checked into git, but a normal file in place in the working tree. Thus, its type has changed. Short of hacking or wrapping git, or switching to indirect mode ;), there's not much that can be done about this.
+
+You might want to read [[direct_mode]] for more about this, and some of the problems it can cause when running certian git commands.
+"""]]
diff --git a/doc/forum/git-status_typechange_in_direct_mode/comment_2_005d1b17f3c2ae192aa30c6e5163989e._comment b/doc/forum/git-status_typechange_in_direct_mode/comment_2_005d1b17f3c2ae192aa30c6e5163989e._comment
new file mode 100644
index 0000000000..c598b212fe
--- /dev/null
+++ b/doc/forum/git-status_typechange_in_direct_mode/comment_2_005d1b17f3c2ae192aa30c6e5163989e._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="ringprince"
+ ip="134.76.140.110"
+ subject="comment 2"
+ date="2013-09-20T20:16:47Z"
+ content="""
+Thanks for the info and the quick reply.
+"""]]
diff --git a/doc/forum/git-subtree_support__63__.mdwn b/doc/forum/git-subtree_support__63__.mdwn
new file mode 100644
index 0000000000..a95277a235
--- /dev/null
+++ b/doc/forum/git-subtree_support__63__.mdwn
@@ -0,0 +1,9 @@
+Hi,
+
+I am a happy user of [git-subtree](http://github.com/apenwarr/git-subtree), and I wonder whether it integrates nicely with git-annex?
+
+My use-case looks like this: I have two annex repositories -- one at home and one at work. The annex at work is a strict subset of the one at home, i.e. all files that I have at work ought to be part of the annex at home, but not the other way round. Now, I realize that I could have one annex and selectively copy files to the checked out copy at work, but I don't want to do that because I don't want to have (broken) symlinks for all kinds of stuff visible on my machine at work that is not supposed to be there (such as MP3 files, etc.). Instead, I would like to use git-subtree to import the work annex into a sub-directory of the one at home, so that both annex are logically separate, but still the one at home always contains everything that the one at work contains.
+
+Is that possible?
+
+And if not, is there maybe another way to accomplish this kind of thing?
diff --git a/doc/forum/git-subtree_support__63__/comment_1_4f333cb71ed1ff259bbfd86704806aa6._comment b/doc/forum/git-subtree_support__63__/comment_1_4f333cb71ed1ff259bbfd86704806aa6._comment
new file mode 100644
index 0000000000..e4ded4c577
--- /dev/null
+++ b/doc/forum/git-subtree_support__63__/comment_1_4f333cb71ed1ff259bbfd86704806aa6._comment
@@ -0,0 +1,10 @@
+[[!comment format=mdwn
+ username="http://joey.kitenet.net/"
+ nickname="joey"
+ subject="comment 1"
+ date="2012-01-03T17:00:53Z"
+ content="""
+I have no experience using git-subtree, but as long as the home repository has the work one as a git remote, it will automatically merge work's git-annex branch with its own git-annex branch, and so will know what files are present at work, and will be able to get them.
+
+Probably you won't want to make work have home as a remote, so work's git-annex will not know which files home has, nor will it be able to copy files to home (but home will be able to copy files to work).
+"""]]
diff --git a/doc/forum/git-subtree_support__63__/comment_2_73d2a015b1ac79ec99e071a8b1e29034._comment b/doc/forum/git-subtree_support__63__/comment_2_73d2a015b1ac79ec99e071a8b1e29034._comment
new file mode 100644
index 0000000000..9bac6e997b
--- /dev/null
+++ b/doc/forum/git-subtree_support__63__/comment_2_73d2a015b1ac79ec99e071a8b1e29034._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="http://peter-simons.myopenid.com/"
+ ip="77.188.44.113"
+ subject="comment 2"
+ date="2012-01-03T18:11:37Z"
+ content="""
+The point of git-subtree is that I can import another repository into sub-directory, i.e. I can have a directory called \"work\" that contains all files from the annex I have at work. If I make the other annex a remote and merge its contents, then all contents is going to be merged at the top-level, which is somewhat undesirable in my particular case.
+"""]]
diff --git a/doc/forum/git-subtree_support__63__/comment_3_c533400e22c306c033fcd56e64761b0b._comment b/doc/forum/git-subtree_support__63__/comment_3_c533400e22c306c033fcd56e64761b0b._comment
new file mode 100644
index 0000000000..83edaf743d
--- /dev/null
+++ b/doc/forum/git-subtree_support__63__/comment_3_c533400e22c306c033fcd56e64761b0b._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="http://joey.kitenet.net/"
+ nickname="joey"
+ subject="comment 3"
+ date="2012-01-03T18:42:08Z"
+ content="""
+You can make it a remote without merging its contents. Git will not merge its contents by default unless it's named \"origin\". git-annex will be prefectly happy with that.
+"""]]
diff --git a/doc/forum/git-subtree_support__63__/comment_4_75b0e072e668aa46ff0a8d62a6620306._comment b/doc/forum/git-subtree_support__63__/comment_4_75b0e072e668aa46ff0a8d62a6620306._comment
new file mode 100644
index 0000000000..642f952e4d
--- /dev/null
+++ b/doc/forum/git-subtree_support__63__/comment_4_75b0e072e668aa46ff0a8d62a6620306._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="http://peter-simons.myopenid.com/"
+ ip="77.188.44.113"
+ subject="comment 4"
+ date="2012-01-03T19:17:36Z"
+ content="""
+Okay, I see, but is `git annex get --auto .` going to import all those files from the work remote into my home if the `master` branch of that remote isn't merged?
+"""]]
diff --git a/doc/forum/git-subtree_support__63__/comment_5_f5ec9649d9f1dc122e715de5533bc674._comment b/doc/forum/git-subtree_support__63__/comment_5_f5ec9649d9f1dc122e715de5533bc674._comment
new file mode 100644
index 0000000000..5b2f3dad55
--- /dev/null
+++ b/doc/forum/git-subtree_support__63__/comment_5_f5ec9649d9f1dc122e715de5533bc674._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="http://joey.kitenet.net/"
+ nickname="joey"
+ subject="comment 5"
+ date="2012-01-03T19:31:45Z"
+ content="""
+Yes; git-annex uses the git-annex branch independently of the branch you have checked out. You may find [[internals]] interesting reading, but the short answer is it will work.
+"""]]
diff --git a/doc/forum/git-subtree_support__63__/comment_6_85df530f7b6d76b74ac8017c6034f95e._comment b/doc/forum/git-subtree_support__63__/comment_6_85df530f7b6d76b74ac8017c6034f95e._comment
new file mode 100644
index 0000000000..77af85258b
--- /dev/null
+++ b/doc/forum/git-subtree_support__63__/comment_6_85df530f7b6d76b74ac8017c6034f95e._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="http://peter-simons.myopenid.com/"
+ ip="77.188.44.113"
+ subject="comment 6"
+ date="2012-01-03T19:47:11Z"
+ content="""
+Very cool! Thank you for the explanation.
+"""]]
diff --git a/doc/forum/git__58_____39__annex__39___is_not_a_git_command_error_after_installing.mdwn b/doc/forum/git__58_____39__annex__39___is_not_a_git_command_error_after_installing.mdwn
new file mode 100644
index 0000000000..ac6560e491
--- /dev/null
+++ b/doc/forum/git__58_____39__annex__39___is_not_a_git_command_error_after_installing.mdwn
@@ -0,0 +1,38 @@
+Installed git and git annex precisely as directed on my Windows 7 system, but 'git annex test' fails with above mentioned error - as if it's not been installed.
+I tried uninstalling and reinstalling
+
+These are the contents of my git directory (from git bash prompt):
+
+-rw-r--r--    1 iainso   Administ      708 Dec 17 21:23 Git Bash.vbs
+
+-rw-r--r--    1 iainso   Administ    46106 Dec 17 21:23 ReleaseNotes.rtf
+
+drwxr-xr-x  147 iainso   Administ    40960 Jan 29 14:44 bin
+
+drwxr-xr-x   51 iainso   Administ    12288 Jan 29 16:10 cmd
+
+drwxr-xr-x    3 iainso   Administ        0 Jan 29 14:44 doc
+
+drwxr-xr-x   13 iainso   Administ     4096 Jan 29 14:44 etc
+
+-rw-r--r--    1 iainso   Administ      103 Jan 28 18:38 git-annex-autostart.vbs
+
+-rw-r--r--    1 iainso   Administ   232368 Jan 28 18:38 git-annex-licenses.txt
+
+-rwxr-xr-x    1 iainso   Administ    49799 Jan 29 16:10 git-annex-uninstall.exe
+
+-rw-r--r--    1 iainso   Administ       88 Jan 28 18:38 git-annex-webapp.vbs
+
+drwxr-xr-x    4 iainso   Administ        0 Jan 29 14:44 git-cheetah
+
+drwxr-xr-x   11 iainso   Administ     4096 Jan 29 14:44 lib
+
+drwxr-xr-x    3 iainso   Administ        0 Jan 29 14:43 libexec
+
+drwxr-xr-x    9 iainso   Administ        0 Jan 29 14:44 share
+
+drwxr-xr-x    5 iainso   Administ        0 Jan 29 14:44 ssl
+
+-rw-r--r--    1 iainso   Administ   635621 Jan 29 14:44 unins000.dat
+
+-rwxr-xr-x    1 iainso   Administ  1244929 Jan 29 14:42 unins000.exe
diff --git a/doc/forum/git__58_____39__annex__39___is_not_a_git_command_error_after_installing/comment_1_8a2e3be32800b7b8a6fa78bbdcba7608._comment b/doc/forum/git__58_____39__annex__39___is_not_a_git_command_error_after_installing/comment_1_8a2e3be32800b7b8a6fa78bbdcba7608._comment
new file mode 100644
index 0000000000..b856d79fd3
--- /dev/null
+++ b/doc/forum/git__58_____39__annex__39___is_not_a_git_command_error_after_installing/comment_1_8a2e3be32800b7b8a6fa78bbdcba7608._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="https://www.google.com/accounts/o8/id?id=AItOawk53Hn5PfMJ9Y0rmLdAeRGmgq7XSGWcoNA"
+ nickname="Iain"
+ subject="comment 1"
+ date="2015-01-29T16:28:59Z"
+ content="""
+Solved it, you need to add git/cmd to your windows path variable
+"""]]
diff --git a/doc/forum/git__58_____39__annex__39___is_not_a_git_command_error_after_installing/comment_2_451b3c89d1b98212d5c21efa4e535424._comment b/doc/forum/git__58_____39__annex__39___is_not_a_git_command_error_after_installing/comment_2_451b3c89d1b98212d5c21efa4e535424._comment
new file mode 100644
index 0000000000..9d09e096d6
--- /dev/null
+++ b/doc/forum/git__58_____39__annex__39___is_not_a_git_command_error_after_installing/comment_2_451b3c89d1b98212d5c21efa4e535424._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="joey"
+ subject="""comment 2"""
+ date="2015-02-04T17:51:26Z"
+ content="""
+Msysgit will add itself to the PATH if you check that box at installation
+time. If not, you probably can't run "git" either..
+"""]]
diff --git a/doc/forum/git_annex_add_crash_and_subsequent_recovery.mdwn b/doc/forum/git_annex_add_crash_and_subsequent_recovery.mdwn
new file mode 100644
index 0000000000..3f3b943a0b
--- /dev/null
+++ b/doc/forum/git_annex_add_crash_and_subsequent_recovery.mdwn
@@ -0,0 +1,25 @@
+Perhaps stupidly I added some very large bare git repos into a git-annex.
+
+This took a very long time, used lot's of memory, and then crashed.  I didn't catch the error (which is annoying) - sorry about that.  IIRC it is the same error if one Ctrl-c's the addition.
+
+I ran `git annex add .` a second time and eventually killed it (I perhaps should have waited - I now think it was working).
+
+A `git annex unannex` fixed up some files but somehow I managed to end up with tonnes of files all sym-linked into the git annex object directory but not somehow recognised as annexed files.  I'm assuming that they somehow didn't make it into git annex's meta-data layer (or equivalent).
+
+Commands such as `git annex {fsck,whereis,unannex} weirdfile` immediately returned without error.
+
+I've now spent a lot of manual time copying the files back. Doing the following, not the cleverest but I was a little panicky about my data...
+
+    find . -type l -exec mv \{} \{}.link \; #Move link names out of the way
+    find . -type l -exec cp \{} \{}.cp \; #Copy follows links so we can copy target back to link location
+    find . -type f -name "*.link.cp" | xargs -n 1 rename 's/\.link\.cp//' #Change to original name
+    find . -type l -exec rm \{} \; #Ditch the links
+    git annex unused
+    git annex dropunused `seq 9228`
+
+9228 files were found to be unused, this gives an idea of the scale of the number of "lost" files for want of a better term.
+
+A pretty poor bug report as these things go.  Anyone any idea what might have happened (it didn't seem space or memory related)?  Or how I might have fixed it a little more cleverly?
+
+For reference I am using stable Debian, git annex version 3.20111011.
+
diff --git a/doc/forum/git_annex_add_crash_and_subsequent_recovery/comment_1_062d0153a379c1ba1df8585b90220d3d._comment b/doc/forum/git_annex_add_crash_and_subsequent_recovery/comment_1_062d0153a379c1ba1df8585b90220d3d._comment
new file mode 100644
index 0000000000..e879441ff8
--- /dev/null
+++ b/doc/forum/git_annex_add_crash_and_subsequent_recovery/comment_1_062d0153a379c1ba1df8585b90220d3d._comment
@@ -0,0 +1,18 @@
+[[!comment format=mdwn
+ username="https://www.google.com/accounts/o8/id?id=AItOawnXybLxkPMYpP3yw4b_I6IdC3cKTD-xEdU"
+ nickname="Matt"
+ subject="comment 1"
+ date="2011-12-06T12:50:27Z"
+ content="""
+Ah HA!  Looks like I found the cause of this.
+
+    [matt@rss01:~/files/matt_ford]0> git annex add mhs
+    add mhs/Accessing_Web_Manager_V10.pdf ok
+    ....
+    add mhs/MAHSC Costing Request Form Dual
+    Organisations - FINAL v20 Oct 2010.xls git-annex: unknown response from git cat-file refs/heads/git-annex:8d5/ed4/WORM-s568832-m1323164214--MAHSC Costing Request Form Dual missing
+
+Spot the file name with a newline character in it!  This causes the error message above.  It seems that the files proceeding this badly named file are sym-linked but not registered.
+
+Perhaps a bug?
+"""]]
diff --git a/doc/forum/git_annex_add_crash_and_subsequent_recovery/comment_2_6fc6be43c488c468a4811cd0a1360225._comment b/doc/forum/git_annex_add_crash_and_subsequent_recovery/comment_2_6fc6be43c488c468a4811cd0a1360225._comment
new file mode 100644
index 0000000000..38f2434f49
--- /dev/null
+++ b/doc/forum/git_annex_add_crash_and_subsequent_recovery/comment_2_6fc6be43c488c468a4811cd0a1360225._comment
@@ -0,0 +1,19 @@
+[[!comment format=mdwn
+ username="http://joey.kitenet.net/"
+ nickname="joey"
+ subject="comment 2"
+ date="2011-12-06T17:08:37Z"
+ content="""
+The bug with newlines is now fixed.
+
+Thought I'd mention how to clean up from interrupting `git annex add`.
+When you do that, it doesn't get a chance to `git add` the files it's
+added (this is normally done at the end, or sometimes at points in the middle when you're adding a *lot* of files).
+Which is also why fsck, whereis, and unannex wouldn't operate on them, since they only deal with files in git.
+
+So the first step is to manually use `git add` on any symlinks.
+
+Then, `git commit` as usual.
+
+At that point, `git annex unannex` would get you back to your starting state.
+"""]]
diff --git a/doc/forum/git_annex_add_crash_and_subsequent_recovery/comment_3_45efaaf27d9b580c4c75cbcdc4f65b64._comment b/doc/forum/git_annex_add_crash_and_subsequent_recovery/comment_3_45efaaf27d9b580c4c75cbcdc4f65b64._comment
new file mode 100644
index 0000000000..b58f81c5b7
--- /dev/null
+++ b/doc/forum/git_annex_add_crash_and_subsequent_recovery/comment_3_45efaaf27d9b580c4c75cbcdc4f65b64._comment
@@ -0,0 +1,10 @@
+[[!comment format=mdwn
+ username="https://www.google.com/accounts/o8/id?id=AItOawnXybLxkPMYpP3yw4b_I6IdC3cKTD-xEdU"
+ nickname="Matt"
+ subject="comment 3"
+ date="2011-12-07T07:39:15Z"
+ content="""
+Ah - very good to know that recovery is easier than the method I used.  
+
+I wonder if it could be made a feature to automatically and safely recover/resume from an interrupted `git add`?
+"""]]
diff --git a/doc/forum/git_annex_add_crash_and_subsequent_recovery/comment_4_c560eae40867512b0af2cbef161fc8ac._comment b/doc/forum/git_annex_add_crash_and_subsequent_recovery/comment_4_c560eae40867512b0af2cbef161fc8ac._comment
new file mode 100644
index 0000000000..8fca16cada
--- /dev/null
+++ b/doc/forum/git_annex_add_crash_and_subsequent_recovery/comment_4_c560eae40867512b0af2cbef161fc8ac._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="http://joey.kitenet.net/"
+ nickname="joey"
+ subject="comment 4"
+ date="2011-12-07T20:54:51Z"
+ content="""
+Good idea! I've made `git annex add` recover when ran a second time.
+"""]]
diff --git a/doc/forum/git_annex_add_freezes_on_direct_repo.mdwn b/doc/forum/git_annex_add_freezes_on_direct_repo.mdwn
new file mode 100644
index 0000000000..8c997724af
--- /dev/null
+++ b/doc/forum/git_annex_add_freezes_on_direct_repo.mdwn
@@ -0,0 +1,21 @@
+I've found that running git annex add on a directory in a direct repository freezes. Example output:
+
+    > git annex add Signs\ \(2002\ Film\) --debug
+    
+    [2015-02-22 10:10:04 GMT] read: git ["--git-dir=/Volumes/plato/Films/.git","--work-tree=/Volumes/plato/Films","-c","core.bare=false","ls-files","--others","--exclude-standard","-z","--","Signs (2002 Film)"]
+    [2015-02-22 10:10:04 GMT] chat: git ["--git-dir=/Volumes/plato/Films/.git","--work-tree=/Volumes/plato/Films","-c","core.bare=false","cat-file","--batch"]
+    add Signs (2002 Film)/VIDEO_TS/VIDEO_TS.BUP [2015-02-22 10:10:04 GMT] chat: git ["--git-dir=/Volumes/plato/Films/.git","--work-tree=/Volumes/plato/Films","-c","core.bare=false","check-attr","-z","--stdin","annex.backend","annex.numcopies","--"]
+    [2015-02-22 10:10:05 GMT] chat: git ["--git-dir=/Volumes/plato/Films/.git","--work-tree=/Volumes/plato/Films","-c","core.bare=false","hash-object","-t","blob","-w","--stdin","--no-filters"]
+    [2015-02-22 10:10:05 GMT] chat: git ["--git-dir=/Volumes/plato/Films/.git","--work-tree=/Volumes/plato/Films","-c","core.bare=false","cat-file","--batch"]
+    ok
+    add Signs (2002 Film)/VIDEO_TS/VIDEO_TS.IFO [2015-02-22 10:10:05 GMT] chat: git ["--git-dir=/Volumes/plato/Films/.git","--work-tree=/Volumes/plato/Films","-c","core.bare=false","hash-object","-t","blob","-w","--stdin","--no-filters"]
+    ok
+
+Lots of files, ending in:
+
+    add Signs (2002 Film)/VIDEO_TS/VTS_13_1.VOB [2015-02-22 10:56:49 GMT] chat: git ["--git-dir=/Volumes/plato/Films/.git","--work-tree=/Volumes/plato/Films","-c","core.bare=false","hash-object","-t","blob","-w","--stdin","--no-filters"]
+    ok
+    [2015-02-22 10:56:49 GMT] read: git ["--git-dir=/Volumes/plato/Films/.git","--work-tree=/Volumes/plato/Films","-c","core.bare=false","ls-files","--modified","-z","--","Signs (2002 Film)"]
+
+
+It then hung for just under two hours before I hit ctrl+C. The files are on a remote SMB server mounted via OS X (hence direct mode) and git annex is being run on an OS X machine. git version 2.3.0, git-annex version 5.20150205. Both installed using homebrew. Any thoughts?
diff --git a/doc/forum/git_annex_add_freezes_on_direct_repo/comment_1_290df6fdd3c9042308638f1e8b66b805._comment b/doc/forum/git_annex_add_freezes_on_direct_repo/comment_1_290df6fdd3c9042308638f1e8b66b805._comment
new file mode 100644
index 0000000000..2d1792e577
--- /dev/null
+++ b/doc/forum/git_annex_add_freezes_on_direct_repo/comment_1_290df6fdd3c9042308638f1e8b66b805._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="https://www.google.com/accounts/o8/id?id=AItOawlEXDOqKvKFW9zl1DY5U9ofpXpeDRf6eN4"
+ nickname="Freddie"
+ subject="comment 1"
+ date="2015-02-24T13:00:50Z"
+ content="""
+I should mention that it does not hang when run on files.
+"""]]
diff --git a/doc/forum/git_annex_add_freezes_on_direct_repo/comment_2_3aa5d0ef823592baab79382851385979._comment b/doc/forum/git_annex_add_freezes_on_direct_repo/comment_2_3aa5d0ef823592baab79382851385979._comment
new file mode 100644
index 0000000000..a99f7cfdfd
--- /dev/null
+++ b/doc/forum/git_annex_add_freezes_on_direct_repo/comment_2_3aa5d0ef823592baab79382851385979._comment
@@ -0,0 +1,11 @@
+[[!comment format=mdwn
+ username="joey"
+ subject="""comment 2"""
+ date="2015-02-25T18:17:45Z"
+ content="""
+So you're using git-annex in a directory that is not on the local computer,
+but that uses a network filesystem?
+
+That seems likely to be the cause of your problem. For example, such
+filesystem often have trouble with POSIX file locks that git-annex uses.
+"""]]
diff --git a/doc/forum/git_annex_add_freezes_on_direct_repo/comment_3_cba1ba0c8b0f2e0f4186e7695d9df569._comment b/doc/forum/git_annex_add_freezes_on_direct_repo/comment_3_cba1ba0c8b0f2e0f4186e7695d9df569._comment
new file mode 100644
index 0000000000..d283ea37e6
--- /dev/null
+++ b/doc/forum/git_annex_add_freezes_on_direct_repo/comment_3_cba1ba0c8b0f2e0f4186e7695d9df569._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="https://www.google.com/accounts/o8/id?id=AItOawlEXDOqKvKFW9zl1DY5U9ofpXpeDRf6eN4"
+ nickname="Freddie"
+ subject="comment 3"
+ date="2015-02-25T18:28:48Z"
+ content="""
+Yeah, that's right. I though direct mode would avoid those problems, but evidently not - thanks anyway! :)
+"""]]
diff --git a/doc/forum/git_annex_alternative.mdwn b/doc/forum/git_annex_alternative.mdwn
new file mode 100644
index 0000000000..d7c41e66d9
--- /dev/null
+++ b/doc/forum/git_annex_alternative.mdwn
@@ -0,0 +1,10 @@
+I really like git-annex and am thankfull to its creator!
+
+I especially like the high level of abstraction it provides when handling backups or syncing.
+
+But git and my ext4 filesystem and HDD just hate the amount of files/folder/symlinks it throws at them and I am asking, what the alternatives are. I am especially looking for the numcopies and "--auto" functionallity. 
+
+
+Most alternatives I checked out (eps. distributed redundant filesystems) come up with other burdens or restrictions.
+
+Do you have any experience with different systems?
diff --git a/doc/forum/git_annex_assistant_-_Changing_repository_information.mdwn b/doc/forum/git_annex_assistant_-_Changing_repository_information.mdwn
new file mode 100644
index 0000000000..c4ef39a8f9
--- /dev/null
+++ b/doc/forum/git_annex_assistant_-_Changing_repository_information.mdwn
@@ -0,0 +1 @@
+Here's one thing I don't fully understand yet. If I add a remote repository, like an archive repository on Box—or if I want to change a transfer repository to an archive repository—do I need to add it or change it separately on each of my computers? Or just one? 
diff --git a/doc/forum/git_annex_assistant_-_Changing_repository_information/comment_1_cde71a410200a7478180748fdcde0352._comment b/doc/forum/git_annex_assistant_-_Changing_repository_information/comment_1_cde71a410200a7478180748fdcde0352._comment
new file mode 100644
index 0000000000..fa0c615173
--- /dev/null
+++ b/doc/forum/git_annex_assistant_-_Changing_repository_information/comment_1_cde71a410200a7478180748fdcde0352._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="http://joeyh.name/"
+ ip="209.250.56.191"
+ subject="comment 1"
+ date="2014-04-17T20:59:43Z"
+ content="""
+This information is stored on the git-annex branch, and so it is synced between repositories with the rest of git-annex's data. You only need to change it in one place.. But it's also fine to change it in multiple places if necessary.
+"""]]
diff --git a/doc/forum/git_annex_assistant__44___share_with_other_devices.mdwn b/doc/forum/git_annex_assistant__44___share_with_other_devices.mdwn
new file mode 100644
index 0000000000..faebb14a1b
--- /dev/null
+++ b/doc/forum/git_annex_assistant__44___share_with_other_devices.mdwn
@@ -0,0 +1,3 @@
+I am trying to share files between my PC at home at that at work using the walkthrough here: http://git-annex.branchable.com/assistant/remote_sharing_walkthrough/. However, I don't have the option on my machine to "Share with other devices". Any ideas why this would be missing? I am using Ubuntu 13.04 if that helps. 
+
+Update: it now works after a software update, I guess I just had an older version of git-annex. Now, I have version 4.20130723. 
diff --git a/doc/forum/git_annex_assistant_repository_history__63__.mdwn b/doc/forum/git_annex_assistant_repository_history__63__.mdwn
new file mode 100644
index 0000000000..d357b78876
--- /dev/null
+++ b/doc/forum/git_annex_assistant_repository_history__63__.mdwn
@@ -0,0 +1,7 @@
+Hi,
+
+Even after googling for a while, I'm still having a hard time finding an answer for this:
+
+How can I access the revision history information for an annex-assistant repository?  For example, to recover a file on one computer that another computer deleted.  Or, to view how another computer modified a file.
+
+Thanks!
diff --git a/doc/forum/git_annex_assistant_repository_history__63__/comment_1_6f87f31e18cefedcc6bce95dbeb85a0b._comment b/doc/forum/git_annex_assistant_repository_history__63__/comment_1_6f87f31e18cefedcc6bce95dbeb85a0b._comment
new file mode 100644
index 0000000000..7425ef1eba
--- /dev/null
+++ b/doc/forum/git_annex_assistant_repository_history__63__/comment_1_6f87f31e18cefedcc6bce95dbeb85a0b._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="joey"
+ subject="""comment 1"""
+ date="2015-01-15T18:59:01Z"
+ content="""
+The repository is just a git repository, so you can use whatever git tools
+you like to examine its history. For example, gitk.
+"""]]
diff --git a/doc/forum/git_annex_copy_--fast_--to_blah_much_slower_than_--from_blah.mdwn b/doc/forum/git_annex_copy_--fast_--to_blah_much_slower_than_--from_blah.mdwn
new file mode 100644
index 0000000000..37b86c4623
--- /dev/null
+++ b/doc/forum/git_annex_copy_--fast_--to_blah_much_slower_than_--from_blah.mdwn
@@ -0,0 +1,15 @@
+I keep a repo synced between machines over ssh. Assuming all the files are in sync, so no actual file transfer needs to takes place, when I do
+
+```
+git annex copy --fast --quiet --to blah
+```
+
+is quite slow, about 10 seconds, using 100% CPU on one core, just to decide nothing needs to be done. On the other hand, doing
+
+```
+  git annex copy --fast --quiet --from blah
+```
+
+takes about 1 second.
+
+I'm confused, as it seems to me that, since I'm using --fast, both transactions should use only locally available data, and both should need about the same amount of computing. Am I missing something? Can this be fixed?
diff --git a/doc/forum/git_annex_copy_--fast_--to_blah_much_slower_than_--from_blah/comment_1_5b6e0b749b01a97a6b52a2c1cca6e35a._comment b/doc/forum/git_annex_copy_--fast_--to_blah_much_slower_than_--from_blah/comment_1_5b6e0b749b01a97a6b52a2c1cca6e35a._comment
new file mode 100644
index 0000000000..90d4292789
--- /dev/null
+++ b/doc/forum/git_annex_copy_--fast_--to_blah_much_slower_than_--from_blah/comment_1_5b6e0b749b01a97a6b52a2c1cca6e35a._comment
@@ -0,0 +1,12 @@
+[[!comment format=mdwn
+ username="http://joeyh.name/"
+ ip="4.154.4.90"
+ subject="comment 1"
+ date="2013-07-17T19:11:38Z"
+ content="""
+How many files are in the directory tree you're copying?
+
+`copy --fast --to` does indeed avoid the check to see if the remote already has the file before copying it.
+
+However, it still needs to look in the location log to see which files are already present on the remote. Whereas `copy --from` can do a single stat of the file on disk to see if it's present in the local repo. Location log lookups are about as fast as I can make them, but they still require requesting info from out of the git repository. If you have a lot of files this otherwise minor difference in speed can stack up..
+"""]]
diff --git a/doc/forum/git_annex_copy_--fast_--to_blah_much_slower_than_--from_blah/comment_2_8f2567f4c4f6db2078211a87689757d3._comment b/doc/forum/git_annex_copy_--fast_--to_blah_much_slower_than_--from_blah/comment_2_8f2567f4c4f6db2078211a87689757d3._comment
new file mode 100644
index 0000000000..c0bfe9df28
--- /dev/null
+++ b/doc/forum/git_annex_copy_--fast_--to_blah_much_slower_than_--from_blah/comment_2_8f2567f4c4f6db2078211a87689757d3._comment
@@ -0,0 +1,17 @@
+[[!comment format=mdwn
+ username="https://www.google.com/accounts/o8/id?id=AItOawkkyBDsfOB7JZvPZ4a8F3rwv0wk6Nb9n48"
+ nickname="Abdó"
+ subject="comment 2"
+ date="2013-07-17T20:45:17Z"
+ content="""
+That example I gave: 10 sec vs 1 sec is on a repository of pictures with about 6200 files on a SSD.
+
+Oh, I think I understand the source of the asymmetry, now! So, `git annex copy --to` queries the location log file by file? I've tested a `git grep` on the git-annex branch as follows
+
+```
+git grep -e  git-annex
+```
+
+and seems to be quite fast, less than a second on my test repo. Could git annex make use of this to speed up bulk queries to the location log?
+
+"""]]
diff --git a/doc/forum/git_annex_copy_--fast_--to_blah_much_slower_than_--from_blah/comment_3_ab98121076b88f351fc8cd9197e6bf64._comment b/doc/forum/git_annex_copy_--fast_--to_blah_much_slower_than_--from_blah/comment_3_ab98121076b88f351fc8cd9197e6bf64._comment
new file mode 100644
index 0000000000..d5f399b309
--- /dev/null
+++ b/doc/forum/git_annex_copy_--fast_--to_blah_much_slower_than_--from_blah/comment_3_ab98121076b88f351fc8cd9197e6bf64._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="http://joeyh.name/"
+ ip="4.154.0.140"
+ subject="comment 3"
+ date="2013-07-18T17:39:37Z"
+ content="""
+Using git grep like that would only be an optimisation when it's acting on every file in the repository. If it's only copying a subdirectory, or a single file, git grep is likely to come out much slower.
+"""]]
diff --git a/doc/forum/git_annex_copy_--fast_--to_blah_much_slower_than_--from_blah/comment_4_cb13328add1b7a812efd817ad3dd1a4f._comment b/doc/forum/git_annex_copy_--fast_--to_blah_much_slower_than_--from_blah/comment_4_cb13328add1b7a812efd817ad3dd1a4f._comment
new file mode 100644
index 0000000000..0c70777a46
--- /dev/null
+++ b/doc/forum/git_annex_copy_--fast_--to_blah_much_slower_than_--from_blah/comment_4_cb13328add1b7a812efd817ad3dd1a4f._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="https://www.google.com/accounts/o8/id?id=AItOawkkyBDsfOB7JZvPZ4a8F3rwv0wk6Nb9n48"
+ nickname="Abdó"
+ subject="comment 4"
+ date="2013-07-18T22:58:24Z"
+ content="""
+I see. Thanks!
+"""]]
diff --git a/doc/forum/git_annex_copy_more_informative_about_why_some_files_are_not_copied.mdwn b/doc/forum/git_annex_copy_more_informative_about_why_some_files_are_not_copied.mdwn
new file mode 100644
index 0000000000..3bb2fe1768
--- /dev/null
+++ b/doc/forum/git_annex_copy_more_informative_about_why_some_files_are_not_copied.mdwn
@@ -0,0 +1,59 @@
+I just spend an hour to understand why the following command did not do anything:
+
+    $ git annex copy --to someremote SomeDirectory
+
+With SomeDirectory containing a lot of file that are not in the someremote repository.
+I just realized that the annex from which I run the command did not contain those files either.
+
+I then did
+
+    $ git annex get SomeDirectory
+
+So that now the git annex copy command correctly copies the files to the someremote remote.
+
+I hope I won't fall into that trap again, but I think it is very annoying not to know why nothing happened.
+
+Wouldn't it be great if the git annex copy command would indicate why some file are not copied, like:
+
+    $ git annex copy --to someremote SomeDirectory
+    Cannot copy SomeFile1 since it is not present in here, (hint: run git annex get first)
+    Cannot copy SomeFile2 since it is not present in here, (hint: run git annex get first)
+    Cannot copy SomeFile3 since it is not present in here, (hint: run git annex get first)
+    ...
+
+May be this kind of information could be provided if the verbose flag is set.
+
+Another awesome (totally subjectively speaking) way of doing it could be to get the file before copying it
+
+    $ git annex copy --to someremote SomeDirectory
+    Cannot copy SomeFile1 since it is not present in here, attempting getting it first
+    get SomeFile1 (from someotherremote...)
+    SHA256E-s109353088--71734pq1p4qo6qs1p156r48s2290q7p61p1658029p103591nrs1rr708s064p59
+         12,615,808 100%  161.27kB/s    0:01:16 (xfr#1, to-chk=0/1)
+    ok
+    copy SomeFile1 (checking someremote...) (to someremote...)
+    SHA256E-s109353088--71734pq1p4qo6qs1p156r48s2290q7p61p1658029p103591nrs1rr708s064p59
+         12,615,808 100%  161.27kB/s    0:01:16 (xfr#1, to-chk=0/1)
+    ok
+    Cannot copy SomeFile2 since it is not present in here, attempting getting it first
+    get SomeFile2 (from someotherremote...)
+    SHA256E-s109353088--71734pq1p4qo6qs1p156r48s2290q7p61p1658029p103591nrs1rr708s064p59
+         12,615,808 100%  161.27kB/s    0:01:16 (xfr#1, to-chk=0/1)
+    ok
+    copy SomeFile2 (checking someremote...) (to someremote...)
+    SHA256E-s109353088--71734pq1p4qo6qs1p156r48s2290q7p61p1658029p103591nrs1rr708s064p59
+         12,615,808 100%  161.27kB/s    0:01:16 (xfr#1, to-chk=0/1)
+    ok
+    Cannot copy SomeFile3 since it is not present in here, attempting getting it first
+    get SomeFile3 (from someotherremote...)
+    SHA256E-s109353088--71734pq1p4qo6qs1p156r48s2290q7p61p1658029p103591nrs1rr708s064p59
+         12,615,808 100%  161.27kB/s    0:01:16 (xfr#1, to-chk=0/1)
+    ok
+    copy SomeFile3 (checking someremote...) (to someremote...)
+    SHA256E-s109353088--71734pq1p4qo6qs1p156r48s2290q7p61p1658029p103591nrs1rr708s064p59
+         12,615,808 100%  161.27kB/s    0:01:16 (xfr#1, to-chk=0/1)
+    ok
+
+We could also specify from which remote to get the file with
+
+    $ git annex copy --to someremote --from someotherremote SomeDirectory
diff --git a/doc/forum/git_annex_copy_more_informative_about_why_some_files_are_not_copied/comment_1_75445fc0e01ee99bae1c1f5a60e314bc._comment b/doc/forum/git_annex_copy_more_informative_about_why_some_files_are_not_copied/comment_1_75445fc0e01ee99bae1c1f5a60e314bc._comment
new file mode 100644
index 0000000000..15373be70d
--- /dev/null
+++ b/doc/forum/git_annex_copy_more_informative_about_why_some_files_are_not_copied/comment_1_75445fc0e01ee99bae1c1f5a60e314bc._comment
@@ -0,0 +1,12 @@
+[[!comment format=mdwn
+ username="http://joeyh.name/"
+ ip="206.74.132.139"
+ subject="comment 1"
+ date="2014-02-02T21:13:21Z"
+ content="""
+I think that this behavior of git-annex is quite useful once you get used to it. It can sometimes trip up new users, but new users would not know about some --explain switch that made it say why it skipped each file. So, I consider this a documentation issue, and I've added a section to the walkthrough to help users learn about it:
+
+
+
+As far as copy --from --to, it has been suggested before; I think there is even a todo about it somewhere, but such remote-to-remote transfers are expensive and I would hope it would not normally be used.
+"""]]
diff --git a/doc/forum/git_annex_drop_not_freeing_space_on_filesystem.mdwn b/doc/forum/git_annex_drop_not_freeing_space_on_filesystem.mdwn
new file mode 100644
index 0000000000..93f64f78c0
--- /dev/null
+++ b/doc/forum/git_annex_drop_not_freeing_space_on_filesystem.mdwn
@@ -0,0 +1,25 @@
+I am using git-annex to store the output of a build process that produces large binary files (~1GB). These files are built on a number of workers which then commit the files to a central server (git annex add && copy to origin)
+
+I am trying to delete some of the older builds but have been unable to see any disk space be freed on the server (Disk usage is identical before and after the delete).
+
+    df -h 
+    mcchicken-srv://repo 284G 114G 157G  43% /srv/repo
+
+To delete the files I have performed the following:
+
+* git clone 
+* git annex drop <1gb_file>
+* git rm <1gb_file>
+* git commit
+* git annex sync
+* [log into server]
+* git annex sync
+* git annex unused
+* git annex dropunused
+
+
+I have also tried other variations of the above technique that I have found online, but to no avail.
+
+In all cases the files appear to not be in the repo when a clone is performed, However the disk usage on the server never decreases.
+
+How would I delete the older builds in a manner that will free up disk space?
diff --git a/doc/forum/git_annex_drop_not_freeing_space_on_filesystem/comment_1_ce1b32c2765edb695e5d9dc8f0ac86e3._comment b/doc/forum/git_annex_drop_not_freeing_space_on_filesystem/comment_1_ce1b32c2765edb695e5d9dc8f0ac86e3._comment
new file mode 100644
index 0000000000..eb0903def2
--- /dev/null
+++ b/doc/forum/git_annex_drop_not_freeing_space_on_filesystem/comment_1_ce1b32c2765edb695e5d9dc8f0ac86e3._comment
@@ -0,0 +1,16 @@
+[[!comment format=mdwn
+ username="joey"
+ subject="""comment 1"""
+ date="2015-02-09T17:09:43Z"
+ content="""
+`git annex drop` removes the file's content from the local repository.
+It does not affect other repositories, such as in your case the server.
+You could use `git annex drop --from origin` to remove the file from
+the origin repository.
+
+If `git annex unused` is not listing the deleted file as unused, there
+must be some git tag or branch that still refers to it.
+
+Also, `git annex dropunused` won't remove the last copy of a file unless you pass
+`--force`
+"""]]
diff --git a/doc/forum/git_annex_drop_not_freeing_space_on_filesystem/comment_3_248cb012c87605479fb17fa4aadc1185._comment b/doc/forum/git_annex_drop_not_freeing_space_on_filesystem/comment_3_248cb012c87605479fb17fa4aadc1185._comment
new file mode 100644
index 0000000000..3f35c5016b
--- /dev/null
+++ b/doc/forum/git_annex_drop_not_freeing_space_on_filesystem/comment_3_248cb012c87605479fb17fa4aadc1185._comment
@@ -0,0 +1,16 @@
+[[!comment format=mdwn
+ username="joey"
+ subject="""comment 3"""
+ date="2015-11-30T19:42:47Z"
+ content="""
+git annex drop can only drop files that the local repository knows about.
+Perhaps you forgot to update your local repo (eg with `git annex sync`) so
+it didn't know about the files in the remote and so couldn't drop them.
+
+Of course, git annex drop also will refuse to drop a file from a repository
+if that not preserve the required number of copies of the file. It will
+tell you if this is the case.
+
+You need to post more details. I can assure you that git annex drop drops
+content from a repository.
+"""]]
diff --git a/doc/forum/git_annex_drop_not_freeing_space_on_filesystem/comment_3_de4aa12c84a6025497f8c743a397ef63._comment b/doc/forum/git_annex_drop_not_freeing_space_on_filesystem/comment_3_de4aa12c84a6025497f8c743a397ef63._comment
new file mode 100644
index 0000000000..554ef36bdb
--- /dev/null
+++ b/doc/forum/git_annex_drop_not_freeing_space_on_filesystem/comment_3_de4aa12c84a6025497f8c743a397ef63._comment
@@ -0,0 +1,14 @@
+[[!comment format=mdwn
+ username="jhannwong@c9c7a67b5632a4bbc0c959cfeb3d340e02f28565"
+ nickname="jhannwong"
+ subject="Nope, git-annex drop is still not reducing size of .git/annex"
+ date="2015-11-20T13:32:11Z"
+ content="""
+A remote has 200MB of file contents in .git/annex.
+
+A local repo issues a ``git annex drop --from=myremote``.
+
+The remote .git/annex still has 200MB of file contents.
+
+Sometimes it works. I'm guessing it has to do with the order of commands issued.
+"""]]
diff --git a/doc/forum/git_annex_drop_not_freeing_space_on_filesystem/comment_4_368b2c07dc7d98534a77c6153e8486ae._comment b/doc/forum/git_annex_drop_not_freeing_space_on_filesystem/comment_4_368b2c07dc7d98534a77c6153e8486ae._comment
new file mode 100644
index 0000000000..47e7498c40
--- /dev/null
+++ b/doc/forum/git_annex_drop_not_freeing_space_on_filesystem/comment_4_368b2c07dc7d98534a77c6153e8486ae._comment
@@ -0,0 +1,18 @@
+[[!comment format=mdwn
+ username="joey"
+ subject="""comment 4"""
+ date="2015-11-30T20:00:20Z"
+ content="""
+I notice you also filed the bug report
+[[bugs/Unable_to_take_transfer_lock]], where you said something about having
+copied the repository over SMB using `cp` and some kind of OSX copy and
+paste. 
+
+So, if you did that, it seems likely that you have 2 git repositories with
+the same annex.uuid setting. In fact this is confirmed by [another bug
+report](http://git-annex.branchable.com/bugs/remote_repo_marked_as___34__here__34__/) you filed.
+
+Since git-annex expects each repository to have a unique annex.uuid, this is
+going to confuse it badly. Don't do that. Use `git clone` to make a clone of
+a git repository.
+"""]]
diff --git a/doc/forum/git_annex_failing_to_get_non-English_filenames.__Rsync_problem__63__.mdwn b/doc/forum/git_annex_failing_to_get_non-English_filenames.__Rsync_problem__63__.mdwn
new file mode 100644
index 0000000000..c9dcf2eda8
--- /dev/null
+++ b/doc/forum/git_annex_failing_to_get_non-English_filenames.__Rsync_problem__63__.mdwn
@@ -0,0 +1,22 @@
+Whenever I try to get a file with a non-English character, git annex gives me trouble that I don't have with English filenames from the same repo.  I'm hesitant to file a bug report here since the problem seems to stem from rsync not being able to handle the characters.  Here's the output (everything enclosed in pointy brackets are redacted):
+
+    get // (from origin...) 
+    git-annex: /home//Music/.git/annex/transfer/upload/53e11324-041f-11e2-93ca-27250d76416f/SHA256-s7941079--c5a31f15302d57563d8cd35e43ba34669b5485a99b617a83c89fba02fb2ca981: commitBuffer: invalid argument (invalid character)
+    protocol version mismatch -- is your shell clean?
+    (see the rsync man page for an explanation)
+    rsync error: protocol incompatibility (code 2) at compat.c(174) [Receiver=3.0.9]
+
+    git-annex-shell: sendkey: 1 failed
+      rsync failed -- run git annex again to resume file transfer
+
+      Unable to access these remotes: origin
+
+      Try making some of these repositories available:
+            604f093f-a980-4740-a9df-d21b580a2ba4 -- origin ()
+    failed
+
+
+
+It spits out similar errors for all files with Japanese/German/French/special characters, and ends with this on the last line:
+
+    git-annex: get: 138 failed
diff --git a/doc/forum/git_annex_failing_to_get_non-English_filenames.__Rsync_problem__63__/comment_1_292ee7c8b37cbd13f03eb67d0359b99e._comment b/doc/forum/git_annex_failing_to_get_non-English_filenames.__Rsync_problem__63__/comment_1_292ee7c8b37cbd13f03eb67d0359b99e._comment
new file mode 100644
index 0000000000..3542dbd220
--- /dev/null
+++ b/doc/forum/git_annex_failing_to_get_non-English_filenames.__Rsync_problem__63__/comment_1_292ee7c8b37cbd13f03eb67d0359b99e._comment
@@ -0,0 +1,10 @@
+[[!comment format=mdwn
+ username="sauluskahn"
+ ip="67.171.218.222"
+ subject="comment 1"
+ date="2012-09-21T21:40:30Z"
+ content="""
+Also, git annex is 3.20120825 on both machines, and rsync is 3.0.9
+
+Both have no problem handling the characters on their own, but when I try to rsync them I get trouble.
+"""]]
diff --git a/doc/forum/git_annex_failing_to_get_non-English_filenames.__Rsync_problem__63__/comment_2_f6341119fcfde5d8160c8f603b1a6fea._comment b/doc/forum/git_annex_failing_to_get_non-English_filenames.__Rsync_problem__63__/comment_2_f6341119fcfde5d8160c8f603b1a6fea._comment
new file mode 100644
index 0000000000..080dba0e97
--- /dev/null
+++ b/doc/forum/git_annex_failing_to_get_non-English_filenames.__Rsync_problem__63__/comment_2_f6341119fcfde5d8160c8f603b1a6fea._comment
@@ -0,0 +1,10 @@
+[[!comment format=mdwn
+ username="http://joeyh.name/"
+ ip="4.153.14.141"
+ subject="comment 2"
+ date="2012-09-21T23:23:04Z"
+ content="""
+git-annex 3.20120825 is not an actual release, just some version number used in git head.
+
+This bug, TTBOMK, was introduced in version 3.20120721, and fixed by git commit 0b12db64d834979d49ed378235b0c19b34e4a4d6 last Sunday.
+"""]]
diff --git a/doc/forum/git_annex_failing_to_get_non-English_filenames.__Rsync_problem__63__/comment_3_8ad3a1d1fe5995d61e5e137280bc76c3._comment b/doc/forum/git_annex_failing_to_get_non-English_filenames.__Rsync_problem__63__/comment_3_8ad3a1d1fe5995d61e5e137280bc76c3._comment
new file mode 100644
index 0000000000..b61eff5065
--- /dev/null
+++ b/doc/forum/git_annex_failing_to_get_non-English_filenames.__Rsync_problem__63__/comment_3_8ad3a1d1fe5995d61e5e137280bc76c3._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="http://joeyh.name/"
+ ip="4.153.14.141"
+ subject="comment 3"
+ date="2012-09-21T23:23:52Z"
+ content="""
+I take that back about 3.20120825, it was so a real release! :) Anyway, bug is fixed in git head, and there will be a release along soon with it.
+"""]]
diff --git a/doc/forum/git_annex_failing_to_get_non-English_filenames.__Rsync_problem__63__/comment_4_86b61b0484f3f4ecff657e46333b3d4f._comment b/doc/forum/git_annex_failing_to_get_non-English_filenames.__Rsync_problem__63__/comment_4_86b61b0484f3f4ecff657e46333b3d4f._comment
new file mode 100644
index 0000000000..678c13b7af
--- /dev/null
+++ b/doc/forum/git_annex_failing_to_get_non-English_filenames.__Rsync_problem__63__/comment_4_86b61b0484f3f4ecff657e46333b3d4f._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="sauluskahn"
+ ip="67.171.218.222"
+ subject="comment 4"
+ date="2012-09-22T03:01:02Z"
+ content="""
+Sounds good, I'll get the next version through cabal and let you know if it's fixed.  Thanks for your time!
+"""]]
diff --git a/doc/forum/git_annex_failing_to_get_non-English_filenames.__Rsync_problem__63__/comment_5_5ffac00d08d26acaba8c3513b24c4d65._comment b/doc/forum/git_annex_failing_to_get_non-English_filenames.__Rsync_problem__63__/comment_5_5ffac00d08d26acaba8c3513b24c4d65._comment
new file mode 100644
index 0000000000..c328519d77
--- /dev/null
+++ b/doc/forum/git_annex_failing_to_get_non-English_filenames.__Rsync_problem__63__/comment_5_5ffac00d08d26acaba8c3513b24c4d65._comment
@@ -0,0 +1,10 @@
+[[!comment format=mdwn
+ username="sauluskahn"
+ ip="67.171.218.222"
+ subject="comment 5"
+ date="2012-09-22T03:02:15Z"
+ content="""
+>I'll let you know if it's *not* fixed.
+
+Typo.
+"""]]
diff --git a/doc/forum/git_annex_file_content_replaced_with_symlink_content.mdwn b/doc/forum/git_annex_file_content_replaced_with_symlink_content.mdwn
new file mode 100644
index 0000000000..654a1af9e2
--- /dev/null
+++ b/doc/forum/git_annex_file_content_replaced_with_symlink_content.mdwn
@@ -0,0 +1,32 @@
+I have a git-annex repository on a USB drive (full of pictures), however, when I go to the actual drive:
+
+    $ cd /run/media/hinmanm/MINIDRIVE1/pics
+    $ ls -lh australia-2014-03
+
+I get:
+
+    -rw-r--r--. 1 hinmanm hinmanm 1.1K Aug 22  2015 IMG_20140321_181130.jpg
+    -rw-r--r--. 1 hinmanm hinmanm 1.1K Aug 22  2015 IMG_20140321_181137.jpg
+    -rw-r--r--. 1 hinmanm hinmanm 1.1K Aug 22  2015 IMG_20140321_181221.jpg
+    -rw-r--r--. 1 hinmanm hinmanm 1.1K Aug 22  2015 IMG_20140321_181228.jpg
+    -rw-r--r--. 1 hinmanm hinmanm 1.1K Aug 22  2015 IMG_20140321_181233.jpg
+    ... hundreds more .jpg files that are all 1.1K ...
+
+It looks like the files think they are symlinks, instead of pictures:
+
+```
+$ cat australia-2014-03/IMG_20140321_181233.jpg
+XSym
+0201
+2ac7522707415dfe67ad5afa1ab3e516
+../.git/annex/objects/5F/Xw/SHA256E-s2499217--ad9525adf9a82e4ac83c89fd2bddfd2b2aef690ccbbddf38d91abc85f72509f6.jpg/SHA256E-s2499217--ad9525adf9a82e4ac83c89fd2bddfd2b2aef690ccbbddf38d91abc85f72509f6.jpg
+```
+
+Running `git-annex fsck` doesn't do anything, and trying to do `git-annex indirect` gives me:
+
+```
+$ git-annex indirect                                  
+git-annex: This repository seems to be on a crippled filesystem, you must use direct mode.
+```
+
+It looks like there is still a bunch of data in `.git/annex`, as its 6.7gb. Is there a way I can get the files back and get git-annex working again, right now none of my photos are available :(
diff --git a/doc/forum/git_annex_file_content_replaced_with_symlink_content/comment_1_f6f4be7161d4c35c7de1d9751ddbd3c3._comment b/doc/forum/git_annex_file_content_replaced_with_symlink_content/comment_1_f6f4be7161d4c35c7de1d9751ddbd3c3._comment
new file mode 100644
index 0000000000..c55ee2d954
--- /dev/null
+++ b/doc/forum/git_annex_file_content_replaced_with_symlink_content/comment_1_f6f4be7161d4c35c7de1d9751ddbd3c3._comment
@@ -0,0 +1,14 @@
+[[!comment format=mdwn
+ username="jimparis"
+ subject="comment 1"
+ date="2016-08-16T14:41:34Z"
+ content="""
+It seems like that is how [OSX stores symlinks on FAT filesystems](https://opensource.apple.com/source/msdosfs/msdosfs-185.2/msdosfs.kextproj/msdosfs.kmodproj/fat.h?txt) and how [Samba stores symlinks on Windows](https://wiki.samba.org/index.php/UNIX_Extensions#Storing_symlinks_on_Windows_servers).  Did you initially make the drive on one of those systems?  Using the same system again would probably make them appear like normal usable symlinks.
+
+At the very least, since you said `.git/annex` is full of data, I think you should be able to find the photo at the location listed in the last line of those files:
+
+```
+../.git/annex/objects/5F/Xw/SHA256E-s2499217--ad9525adf9a82e4ac83c89fd2bddfd2b2aef690ccbbddf38d91abc85f72509f6.jpg/SHA256E-s2499217--ad9525adf9a82e4ac83c89fd2bddfd2b2aef690ccbbddf38d91abc85f72509f6.jpg
+```
+
+"""]]
diff --git a/doc/forum/git_annex_file_content_replaced_with_symlink_content/comment_2_da848024c40eb65e6214bd95b73e078e._comment b/doc/forum/git_annex_file_content_replaced_with_symlink_content/comment_2_da848024c40eb65e6214bd95b73e078e._comment
new file mode 100644
index 0000000000..2e619f856d
--- /dev/null
+++ b/doc/forum/git_annex_file_content_replaced_with_symlink_content/comment_2_da848024c40eb65e6214bd95b73e078e._comment
@@ -0,0 +1,11 @@
+[[!comment format=mdwn
+ username="thnetos"
+ subject="RE: comment 1"
+ date="2016-08-16T20:39:38Z"
+ content="""
+Thanks for the info, that was very helpful. This is indeed a FAT partition I created on OSX.
+
+Unfortunately, I no longer use OSX, so I don't have a machine that can read this data. I'll write a script to grab it from the contents of the symlink file itself to hopefully retrieve everything. In the future, I'll partition my external harddrives with ext4 :)
+
+Thanks again!
+"""]]
diff --git a/doc/forum/git_annex_file_content_replaced_with_symlink_content/comment_3_cc703ef5015527425572002295568923._comment b/doc/forum/git_annex_file_content_replaced_with_symlink_content/comment_3_cc703ef5015527425572002295568923._comment
new file mode 100644
index 0000000000..43f5107ce0
--- /dev/null
+++ b/doc/forum/git_annex_file_content_replaced_with_symlink_content/comment_3_cc703ef5015527425572002295568923._comment
@@ -0,0 +1,13 @@
+[[!comment format=mdwn
+ username="joey"
+ subject="""comment 3"""
+ date="2016-09-21T20:44:26Z"
+ content="""
+Gold star for @jimparis!
+
+Another way to get out of this situation would be to edit the files,
+and remove the initial lines so it contains a single line that is
+the link target of the symlink (the bit with .git/annex/objects in it).
+
+Then, `git annex fsck` would be able to fix up the files.
+"""]]
diff --git a/doc/forum/git_annex_get_--want-get_another__95__repo.mdwn b/doc/forum/git_annex_get_--want-get_another__95__repo.mdwn
new file mode 100644
index 0000000000..4ed0878766
--- /dev/null
+++ b/doc/forum/git_annex_get_--want-get_another__95__repo.mdwn
@@ -0,0 +1,68 @@
+Hi,
+
+Git-annex is really awesome. It has made my life really easier when having to
+move files around.
+
+Yet, I have been struggling with a use case that I cannot get working with git
+annex.
+
+In short, my request is: could it be possible to have --want-get and --want-drop
+accept a repository as argument to match the preferred content of that
+repository instead of here?
+
+Now, let me explain why I need this:a
+
+All my files are stored into a NAS accessible via a local network.
+
+I have an annex in my desktop computer. Using preferred content (via "git annex
+wanted") and "git annex get|drop --auto", I am able to almost automatically
+handle what files are put into my computer. What I do is to "git annex wanted"
+to indicate what I want to be here and launch a home made script that basically
+does "git annex get --auto" and "git annex drop --auto".
+
+Let's say I have a android phone to which I connect via ssh over adb. It
+contains a git repository but few files are in it. It has no wifi and so no
+access to the network, meaning no access to the NAS.
+
+The links between annexes then looks like:
+
+    NAS <-> Computer <-> Phone
+
+When I want to put a file into my phone, I generally launch "git annex get file"
+from my computer (then I get the file from the NAS) and "git annex copy --to
+phone file".
+
+I want to be able to automatise this a bit by playing with preferred content
+(like I do with my computer). This means that I want to launch "git annex
+wanted" to edit the preferred content of the phone annex and then "git annex get
+--auto" and launch "git annex copy --auto --to phone". This way, when I am not
+in front of my computer, I can still from my phone run "git annex wanted here
+'preferred content'" and hope for my synchronisation scripts (run in my
+computer) to put the good files into my phone.
+
+Obviously those commands won't work since the git annex get --auto command will
+only get what my computer wants, not what my phone wants.
+
+The intuitive (IMHO) way to do would be to launch:
+
+    git annex get --want-get phone
+	git annex copy --auto --to phone
+	git annex drop --auto
+
+With "--want-get repository" meaning, "Matches files that the preferred content
+settings for the repository make it want to get.".
+
+For the time being, I succeed in doing this with
+
+	OLD_WANTED=$(git annex wanted here)
+	git annex wanted here $(git annex wanted phone)
+	git annex copy --auto --to phone
+	git annex wanted ${OLD_WANTED}
+	git annex drop --auto
+
+This is complicated and adds two extra commits in the git-annex branch (one for
+each setting of git annex wanted) each time I call the script.
+
+What do you think?
+
+Thanks for reading.
diff --git a/doc/forum/git_annex_get_--want-get_another__95__repo/comment_1_0be0b3981ddd0743ff26cf6d396e521d._comment b/doc/forum/git_annex_get_--want-get_another__95__repo/comment_1_0be0b3981ddd0743ff26cf6d396e521d._comment
new file mode 100644
index 0000000000..7a58e35d62
--- /dev/null
+++ b/doc/forum/git_annex_get_--want-get_another__95__repo/comment_1_0be0b3981ddd0743ff26cf6d396e521d._comment
@@ -0,0 +1,16 @@
+[[!comment format=mdwn
+ username="http://joeyh.name/"
+ ip="209.250.56.154"
+ subject="comment 1"
+ date="2014-03-12T17:15:23Z"
+ content="""
+Well, I suppose this is doable, but the way this scenario is generally handled is to make a transfer repository (which your desktop is serving as here) have a preferred content expression that makes it want files that the client repositories (phones) want, until the files have reached the clients:
+
+In your case you could have:
+
+    (not inallgroup=phones and ($phone_preferred_content)) or ($desktop_preferred_content)
+
+Where `$desktop_preferred_content` is whatever files the desktop actually wants on its own, and `$phone_preferred_content` is a copy of the preferred content setting for the phone.
+
+To make this work, you also need to put your phone in the phones group.
+"""]]
diff --git a/doc/forum/git_annex_get_--want-get_another__95__repo/comment_2_b1ead1085a87818625579bf1ef151b5d._comment b/doc/forum/git_annex_get_--want-get_another__95__repo/comment_2_b1ead1085a87818625579bf1ef151b5d._comment
new file mode 100644
index 0000000000..1f8ce270e8
--- /dev/null
+++ b/doc/forum/git_annex_get_--want-get_another__95__repo/comment_2_b1ead1085a87818625579bf1ef151b5d._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="http://joeyh.name/"
+ ip="209.250.56.154"
+ subject="comment 2"
+ date="2014-03-12T17:16:59Z"
+ content="""
+BTW expressing this in the preferred content as above also has the benefit that you can use the new `git annex sync --content` on the desktop and it will automatically get files, copy the right ones to the phone, and drop any then-unwanted files.
+"""]]
diff --git a/doc/forum/git_annex_get_--want-get_another__95__repo/comment_3_cf2018852c84b0bf1ac061def6f0ac5d._comment b/doc/forum/git_annex_get_--want-get_another__95__repo/comment_3_cf2018852c84b0bf1ac061def6f0ac5d._comment
new file mode 100644
index 0000000000..c6bd20e1ff
--- /dev/null
+++ b/doc/forum/git_annex_get_--want-get_another__95__repo/comment_3_cf2018852c84b0bf1ac061def6f0ac5d._comment
@@ -0,0 +1,27 @@
+[[!comment format=mdwn
+ username="https://www.google.com/accounts/o8/id?id=AItOawm3vKzS4eOWYpKMoYXqMIjNsIg_nYF-loU"
+ nickname="Konubinix"
+ subject="That does it but not totally"
+ date="2014-03-13T11:36:02Z"
+ content="""
+I agree with the fact that it answers my initial question.
+
+Nevertheless, I guess it is really annoying to have to change the preferred content in two locations each time I change want to modify what is on my phone.
+Indeed, I quite often change what should be really present on my phone, depending on several factors (my mood, the time I will have in transports and the phase of the moon).
+
+The reason why I use \"git annex wanted\" is that it is straightforward: I just launch : \"git annex wanted phone include=some/file\" and that's all.
+
+With the solution you propose, I would have to each time additionally launch \"git annex wanted here '(not inallgroup=phones and (include=some/file)) or ($desktop_preferred_content)'\"
+
+Where I probably have to previously launch \"git annex wanted here\" to remember what is the preferred content of my computer (and put it in $desktop_preferred_content).
+
+Another option would be to run \"git annex vicfg\" and edit both fields manually, but IMHO this appears also to be too complicated relatively to the use case.
+
+About your second comment, I really enjoy the idea if \"git annex sync --content\", but it is really long in big repositories. I guess it is because I cannot restrict the command to a directory like I do with get, drop, move and copy.
+
+Besides, the use case wants to get the files whenever they are and copy them only to the phone. With git annex sync --content, the files are also put in the other repositories.
+
+For the time being, I prefer falling back to explicit commands that are much faster when I know a directory to sync \"git annex get --auto directory && git annex copy --auto --to phone directory\".
+
+For example, I just tried \"git annex sync --content phone\" and I killed it after 5 minutes and nothing was copied yet. With the set of two commands above, the synchronization of directory (get --to phone directory + drop --from here directory) took about 3 minutes.
+"""]]
diff --git a/doc/forum/git_annex_get_--want-get_another__95__repo/comment_4_22562e8f1f2f91b9f9a5939ec9006cb5._comment b/doc/forum/git_annex_get_--want-get_another__95__repo/comment_4_22562e8f1f2f91b9f9a5939ec9006cb5._comment
new file mode 100644
index 0000000000..25624c2c04
--- /dev/null
+++ b/doc/forum/git_annex_get_--want-get_another__95__repo/comment_4_22562e8f1f2f91b9f9a5939ec9006cb5._comment
@@ -0,0 +1,38 @@
+[[!comment format=mdwn
+ username="https://www.google.com/accounts/o8/id?id=AItOawm3vKzS4eOWYpKMoYXqMIjNsIg_nYF-loU"
+ nickname="Konubinix"
+ subject="Precision of the use case"
+ date="2014-03-13T11:47:59Z"
+ content="""
+I realized that the directory restriction described earlier is not clear. Let me explain it.
+
+Say I have a big repository of files with the following structure.
+
+A/...
+B/...
+C/...
+D/D1/...
+D/D2/...
+D/D3/...
+
+Imagine that each of A, B, C and D contains a lot of files.
+
+Now imagine that I have often put in the preferred content of my phone files in one of D1, D2 or D3. (for instance, include=D/D1/*)
+
+I implicitly know that I can restrict the command to D, and I can rely on preferred content to know what file from D I have to put into my phone.
+
+Then, I can run
+      $ git annex get --auto D
+      $ git annex copy --auto --to phone D
+      $ git annex drop --auto D
+
+This in my use case takes 3 minutes.
+
+When I run
+     $ git annex sync --content phone
+
+git annex goes recursively through A, B and C and that takes a long time (much more than 3 minutes). This time is really wasted since I know I only want to sync files from D.
+
+I cannot test the behavior of \"git annex sync --content\", but if I restrict the sync to phone, will it take the files from the NAS to put them on the phone?
+If I don't precise phone in the command line, it will try to sync with other repositories not in sync that I don't want to be in sync for the time being.
+"""]]
diff --git a/doc/forum/git_annex_get_creates_a_new_uuid.mdwn b/doc/forum/git_annex_get_creates_a_new_uuid.mdwn
new file mode 100644
index 0000000000..72dcd97373
--- /dev/null
+++ b/doc/forum/git_annex_get_creates_a_new_uuid.mdwn
@@ -0,0 +1,6 @@
+From git-annex(1), init should precede any operation of git-annex as a safety feature.
+
+However git cloning a repo and running `git annex get` created a new uuid for that repo, which got
+propagated via sync.
+
+Is this intended behaviour?
diff --git a/doc/forum/git_annex_get_creates_a_new_uuid/comment_1_004c87183968c326058bd3159a5baa0b._comment b/doc/forum/git_annex_get_creates_a_new_uuid/comment_1_004c87183968c326058bd3159a5baa0b._comment
new file mode 100644
index 0000000000..d2f3c4f734
--- /dev/null
+++ b/doc/forum/git_annex_get_creates_a_new_uuid/comment_1_004c87183968c326058bd3159a5baa0b._comment
@@ -0,0 +1,14 @@
+[[!comment format=mdwn
+ username="http://joeyh.name/"
+ ip="4.154.0.21"
+ subject="comment 1"
+ date="2013-07-30T20:24:06Z"
+ content="""
+From the man page:
+
+> Until a repository (**or one of its remotes**) has been initialized
+> git-annex will refuse to operate on it, to avoid accidentally
+> using it in a repository that was not intended to have an annex.
+
+So, yes, it assumes that if a repository has a git-annex branch already, git-annex is being used, and no explicit init is necessary. You can still run `git annex init` after the fact if you like.
+"""]]
diff --git a/doc/forum/git_annex_info__58___local_annex_keys__63__.mdwn b/doc/forum/git_annex_info__58___local_annex_keys__63__.mdwn
new file mode 100644
index 0000000000..e1a79f038a
--- /dev/null
+++ b/doc/forum/git_annex_info__58___local_annex_keys__63__.mdwn
@@ -0,0 +1,27 @@
+When I run `git annex info`, it stops to ask me for "local annex keys":
+
+    $ git annex info
+    repository mode: indirect
+    trusted repositories: 0
+    semitrusted repositories: 3
+            00000000-0000-0000-0000-000000000001 -- web
+            1xxxxxx7-6xx1-4xx1-8xx8-5xxxxxxxxxx1 -- here (usbdisk)
+            bxxxxxx6-bxxe-4xx2-bxxe-fxxxxxxxxxxf -- origin (local)
+    untrusted repositories: 0
+    transfers in progress: none
+    available local disk space: 36.25 gigabytes (+1 megabyte reserved)
+    local annex keys: 
+
+What are these "local annex keys"? 
+
+*Edit*: Ah! it wasn't asking me anything, just echoing my keystrokes. It was counting keys stored in the annex, and taking a while. Sorry for the noise!
+
+    [...]
+    local annex keys: 19586
+    local annex size: 33.96 gigabytes
+    annexed files in working tree: 19814
+    size of annexed files in working tree: 35.12 gigabytes
+    bloom filter size: 16 mebibytes (3.9% full)
+    backend usage: 
+            SHA256E: 39400
+
diff --git a/doc/forum/git_annex_init_timeout.mdwn b/doc/forum/git_annex_init_timeout.mdwn
new file mode 100644
index 0000000000..5bd64f943a
--- /dev/null
+++ b/doc/forum/git_annex_init_timeout.mdwn
@@ -0,0 +1,43 @@
+I'm trying git-annex for the first time and having some issues.
+
+I have a WD MyBook 2TB Cloud Drive and using OSX Sierra.
+
+I want to create an Annex on one the NAS smb shares and then a Amazon Cloud Drive replica for offsite backup.
+
+When I run `git annex init` from the mounted share in OSX it fails with the below:
+
+```
+git-annex init
+init
+  Detected a filesystem without POSIX fcntl lock support.
+
+  Enabling annex.pidlock.
+
+  Detected a filesystem without fifo support.
+
+  Disabling ssh connection caching.
+
+  Detected a crippled filesystem.
+
+  Disabling core.symlinks.
+
+  Enabling direct mode.
+A300 second timeout exceeded while waiting for pid lock file .git/annex/pidlock
+```
+
+Below are the results of `git-annex version`
+
+```
+git-annex version: 6.20170101
+build flags: Assistant Webapp Pairing Testsuite S3(multipartupload)(storageclasses) WebDAV FsEvents ConcurrentOutput TorrentParser MagicMime Feeds Quvi
+key/value backends: SHA256E SHA256 SHA512E SHA512 SHA224E SHA224 SHA384E SHA384 SHA3_256E SHA3_256 SHA3_512E SHA3_512 SHA3_224E SHA3_224 SHA3_384E SHA3_384 SKEIN256E SKEIN256 SKEIN512E SKEIN512 SHA1E SHA1 MD5E MD5 WORM URL
+remote types: git gcrypt p2p S3 bup directory rsync web bittorrent webdav tahoe glacier ddar hook external
+local repository version: 5
+supported repository versions: 3 5 6
+upgrade supported from repository versions: 0 1 2 3 4 5
+operating system: darwin x86_64
+```
+
+
+
+Is it possible to have an annex on a SMB share?
diff --git a/doc/forum/git_annex_init_timeout/comment_1_e456f5367af5a5a3a31fc982952079e3._comment b/doc/forum/git_annex_init_timeout/comment_1_e456f5367af5a5a3a31fc982952079e3._comment
new file mode 100644
index 0000000000..06aed071a9
--- /dev/null
+++ b/doc/forum/git_annex_init_timeout/comment_1_e456f5367af5a5a3a31fc982952079e3._comment
@@ -0,0 +1,22 @@
+[[!comment format=mdwn
+ username="joey"
+ subject="""comment 1"""
+ date="2017-02-10T18:06:28Z"
+ content="""
+git-annex needs flock locking to be supported by the filesystem it's used
+on. There's a workaround using pid locks, but that is apparently not
+working on OSX in your case. I've only tested it on Linux.
+
+I tried reproducing this on OSX by enabling annex.pidlock before using
+git-annex. But it worked ok.
+
+So, I suspect there may be a problem with the network filesystem's handling
+of the pidlock file. One likely problem is if it doesn't support hard
+links, since the pidlock file is hard linked into place.
+
+You can check if it supports hard links by running this on the drive where
+you had the problem:
+
+	touch foo
+	ln foo bar
+"""]]
diff --git a/doc/forum/git_annex_init_timeout/comment_2_bf15b686b07c5e6e95796affa497a541._comment b/doc/forum/git_annex_init_timeout/comment_2_bf15b686b07c5e6e95796affa497a541._comment
new file mode 100644
index 0000000000..c7f39ae530
--- /dev/null
+++ b/doc/forum/git_annex_init_timeout/comment_2_bf15b686b07c5e6e95796affa497a541._comment
@@ -0,0 +1,13 @@
+[[!comment format=mdwn
+ username="joey"
+ subject="""comment 2"""
+ date="2017-02-10T18:52:59Z"
+ content="""
+I was able to reproduce the problem with a FAT filesystem
+mounted on Linux, if I manually enabled annex.pidlock
+before git-annex init. Hard linking does not work on FAT, so that
+matches my earlier guess.
+
+I've committed a fix for this, so try again with a recent autobuild of
+git-annex and it will probably fix your problem.
+"""]]
diff --git a/doc/forum/git_annex_init_timeout/comment_3_1e733fad01e6b420c7fd9f7832e9b3f7._comment b/doc/forum/git_annex_init_timeout/comment_3_1e733fad01e6b420c7fd9f7832e9b3f7._comment
new file mode 100644
index 0000000000..a047306c0e
--- /dev/null
+++ b/doc/forum/git_annex_init_timeout/comment_3_1e733fad01e6b420c7fd9f7832e9b3f7._comment
@@ -0,0 +1,10 @@
+[[!comment format=mdwn
+ username="ewen"
+ avatar="http://cdn.libravatar.org/avatar/605b2981cb52b4af268455dee7a4f64e"
+ subject="git annex get ... transfer lock issues"
+ date="2017-04-23T07:36:08Z"
+ content="""
+The fix a couple of months ago appears to allow `git annex init` to complete on a SMB file share, which is great.  But FTR there are currently [issues with file transfers](https://git-annex.branchable.com/bugs/SMB__58___git_annex_clone_works__44___get_fails_on_transfer_lock/) (link is to bug report about failing to get transfer lock), so use with a NAS is still \"work in progress\".
+
+Ewen
+"""]]
diff --git a/doc/forum/git_annex_ls___47___metadata_in_git_annex_whereis.mdwn b/doc/forum/git_annex_ls___47___metadata_in_git_annex_whereis.mdwn
new file mode 100644
index 0000000000..f1aa5cc06d
--- /dev/null
+++ b/doc/forum/git_annex_ls___47___metadata_in_git_annex_whereis.mdwn
@@ -0,0 +1 @@
+I just started experimenting with git annex, and I found that I would like to have a way to figure out metadata (well, size. Maybe modification date) of a non-local file. I first checked if there is "git annex ls" (which could list known files in an ls-like way) and found "git annex whereis" as somewhat a replacement, but it does not give metadata information.
diff --git a/doc/forum/git_annex_ls___47___metadata_in_git_annex_whereis/comment_1_7fba10b85f4d9289c7782eccef46949e._comment b/doc/forum/git_annex_ls___47___metadata_in_git_annex_whereis/comment_1_7fba10b85f4d9289c7782eccef46949e._comment
new file mode 100644
index 0000000000..379b9f976b
--- /dev/null
+++ b/doc/forum/git_annex_ls___47___metadata_in_git_annex_whereis/comment_1_7fba10b85f4d9289c7782eccef46949e._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="http://joey.kitenet.net/"
+ nickname="joey"
+ subject="comment 1"
+ date="2011-11-14T22:46:35Z"
+ content="""
+When  I want that, I ls -l the file and look at the symlink to the key. Ie, in SHA1-s10481423--efc7eec0d711212842cd6bb8f957e1628146d6ed the size is 10481423 bytes. 
+"""]]
diff --git a/doc/forum/git_annex_ls___47___metadata_in_git_annex_whereis/comment_2_7dcec124ea7d0291ed40d80e2ffd5c7e._comment b/doc/forum/git_annex_ls___47___metadata_in_git_annex_whereis/comment_2_7dcec124ea7d0291ed40d80e2ffd5c7e._comment
new file mode 100644
index 0000000000..3dd14bf010
--- /dev/null
+++ b/doc/forum/git_annex_ls___47___metadata_in_git_annex_whereis/comment_2_7dcec124ea7d0291ed40d80e2ffd5c7e._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="http://joey.kitenet.net/"
+ nickname="joey"
+ subject="comment 2"
+ date="2011-11-14T22:48:03Z"
+ content="""
+It might make sense to put this functionality in git annex find. Perhaps a format string with a %s for example. 
+"""]]
diff --git a/doc/forum/git_annex_ls___47___metadata_in_git_annex_whereis/comment_3_24c54ed70220974b98700bf717d1e770._comment b/doc/forum/git_annex_ls___47___metadata_in_git_annex_whereis/comment_3_24c54ed70220974b98700bf717d1e770._comment
new file mode 100644
index 0000000000..67ff59d324
--- /dev/null
+++ b/doc/forum/git_annex_ls___47___metadata_in_git_annex_whereis/comment_3_24c54ed70220974b98700bf717d1e770._comment
@@ -0,0 +1,29 @@
+[[!comment format=mdwn
+ username="sudoman"
+ ip="216.15.125.93"
+ subject="ls symlink workaround; idea for a solution"
+ date="2014-09-29T18:58:23Z"
+ content="""
+as a workaround, you could make a bash alias for `ls -l` -> `ls -lL`. the problem with this is that links to other links are fully dereferenced.
+
+what looks like this in a non-git-annex directory with `ls -lh`:
+
+    total 3.8M
+    -rw-r--r-- 1 sudoman sudoman 3.8M Sep 29 13:56 42x3551_02.pdf
+    lrwxrwxrwx 1 sudoman sudoman   14 Sep 29 14:00 tmp -> 42x3551_02.pdf
+
+looks like this in an indirect git annex repo with `ls -lhL`:
+
+    total 7.5M
+    -r--r--r-- 1 sudoman sudoman 3.8M Sep 29 13:56 42x3551_02.pdf
+    -r--r--r-- 1 sudoman sudoman 3.8M Sep 29 13:56 tmp
+
+
+the ls alias is a bit hackish, but for some purposes it's an improvement.
+
+rsync may work as desired when using a command like `rsync -l --safe-links` (haven't tried it. users might want to experiment by adding `--exclude` to that command.)
+
+
+a potential solution for ls (and cp) could be the inclusion of a patched version under `git annex util ls`. writing shim programs using `LD_PRELOAD` instead of patching may drastically reduce the amount of code needing future security updates.
+
+"""]]
diff --git a/doc/forum/git_annex_on_osx_only_creating_symlinks__63____63__.mdwn b/doc/forum/git_annex_on_osx_only_creating_symlinks__63____63__.mdwn
new file mode 100644
index 0000000000..592e26b936
--- /dev/null
+++ b/doc/forum/git_annex_on_osx_only_creating_symlinks__63____63__.mdwn
@@ -0,0 +1,18 @@
+I'm testing out git-annex between a few computers one being a mac (osx 10.9) and a laptop (linux mint 16).  With a vps running git annex as a transfer annex.
+
+Anyway sync is *almost* working...
+
+When I add files on my mac's annex they upload, and go to my laptop as they should.
+
+When I add files on my laptop's annex they upload, and the mac "downloads" them but only creates broken symlinks to the files.
+
+
+I looked in the log and nothing out of the ordinary is happening... is this a bug with the osx version or what?
+
+
+**more info**  
+
+* the symlink on the osx annex is symlinking to a file in .git/annex/objects  
+* the osx annex changes the aliases to real files every time I restart the git annex daemon  
+
+Thanks!
diff --git a/doc/forum/git_annex_on_osx_only_creating_symlinks__63____63__/comment_1_6889c4452e636474b4e70798b404fed2._comment b/doc/forum/git_annex_on_osx_only_creating_symlinks__63____63__/comment_1_6889c4452e636474b4e70798b404fed2._comment
new file mode 100644
index 0000000000..c97f26e860
--- /dev/null
+++ b/doc/forum/git_annex_on_osx_only_creating_symlinks__63____63__/comment_1_6889c4452e636474b4e70798b404fed2._comment
@@ -0,0 +1,10 @@
+[[!comment format=mdwn
+ username="http://joeyh.name/"
+ ip="209.250.56.87"
+ subject="comment 1"
+ date="2013-12-12T17:11:59Z"
+ content="""
+It sounds like the contents of the files are being successfully transferred.
+
+It may be that you're using an old and buggy version of git-annex, in which the direct mode code lost track of things and so failed to update the tree correctly when receiving the content of a file. If so, you could fix it by upgrading git-annex and running `git annex fsck`.
+"""]]
diff --git a/doc/forum/git_annex_on_osx_only_creating_symlinks__63____63__/comment_2_978fc11c463a457382fddd668cd1d0dd._comment b/doc/forum/git_annex_on_osx_only_creating_symlinks__63____63__/comment_2_978fc11c463a457382fddd668cd1d0dd._comment
new file mode 100644
index 0000000000..b3a848ffe3
--- /dev/null
+++ b/doc/forum/git_annex_on_osx_only_creating_symlinks__63____63__/comment_2_978fc11c463a457382fddd668cd1d0dd._comment
@@ -0,0 +1,20 @@
+[[!comment format=mdwn
+ username="https://www.google.com/accounts/o8/id?id=AItOawkipQLNyt8RHREHpg2k5wdYeRSCCvSNSBg"
+ nickname="Tim"
+ subject="I had this same problem too"
+ date="2014-01-04T01:00:10Z"
+ content="""
+I was using the 2013-12-13 build and now I've upgrade to the 2014-01-03 build.  git-annex has symlinked some important workspace.xml files to nowhere.  One looks like this:
+
+workspace.xml -> ../../../.git/annex/objects/M2/8W/SHA256E-s68106--90e9cc4f617c9034db1bf462d058b82c59ade0be58de1d3a3e2f8c02606631c2.xml/SHA256E-s68106--90e9cc4f617c9034db1bf462d058b82c59ade0be58de1d3a3e2f8c02606631c2.xml
+
+The referenced file doesn't exist.  Actually, that entire M2 directory doesn't exist.
+
+This has cost me a lot of time trying to fix this.  I tried git-annex fsck and I get this message:
+
+ ** No known copies exist of XXXXXXX/workspace.xml
+
+\"XXXXXXX\" is a path I needed to obscure
+
+I can see it in the git-annex repo but it is a reference to a symlink that doesn't exist.  Is there any way to get this file back?  git-annex whereis tells me it is nowhere.  This appears to be real data loss.
+"""]]
diff --git a/doc/forum/git_annex_on_osx_only_creating_symlinks__63____63__/comment_3_4420bd3afaecd7536b02fc08cee82dbe._comment b/doc/forum/git_annex_on_osx_only_creating_symlinks__63____63__/comment_3_4420bd3afaecd7536b02fc08cee82dbe._comment
new file mode 100644
index 0000000000..3f4ac940ee
--- /dev/null
+++ b/doc/forum/git_annex_on_osx_only_creating_symlinks__63____63__/comment_3_4420bd3afaecd7536b02fc08cee82dbe._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="https://www.google.com/accounts/o8/id?id=AItOawkipQLNyt8RHREHpg2k5wdYeRSCCvSNSBg"
+ nickname="Tim"
+ subject="Sorry for the bad formatting on that last post"
+ date="2014-01-04T17:16:51Z"
+ content="""
+I have some additional information that may be helpful.  I had git-annex syncing my Mac with a Debian system.  It appears that at some point in time the Debian system's NFS connection disconnected and then may have reconnected with sshfs.  I don't know if this happened when the syncing issue occurred or not and I can't validate that since the machine was rebooted but it is possible that that caused git-annex on the Mac to choke while syncing and create the phantom symlink.  I am experiencing NFS issues at the moment so it may have contributed to this hiccup.
+"""]]
diff --git a/doc/forum/git_annex_on_osx_only_creating_symlinks__63____63__/comment_4_99286f17a87049c303f2aa34c0a90286._comment b/doc/forum/git_annex_on_osx_only_creating_symlinks__63____63__/comment_4_99286f17a87049c303f2aa34c0a90286._comment
new file mode 100644
index 0000000000..d21d871458
--- /dev/null
+++ b/doc/forum/git_annex_on_osx_only_creating_symlinks__63____63__/comment_4_99286f17a87049c303f2aa34c0a90286._comment
@@ -0,0 +1,13 @@
+[[!comment format=mdwn
+ username="http://joeyh.name/"
+ ip="209.250.56.35"
+ subject="comment 4"
+ date="2014-01-06T15:48:36Z"
+ content="""
+@Tim, you do not seem to be having the problem that the OP described.
+
+It is completely normal for git-annex to represent a file whose content is not present in the repository as a broken symlink.
+Typically, git-annex preserves at least one copy of the file in one of your repositories. But if you `git annex add $file; git annex drop --force $file`, you will be left in exactly the situation you describe.
+
+You can use `git annex log $file` to see a log of which repositories contained a copy of the file in the past, and see what times the file was removed from each repository. This might give clues to what operation caused the last copy of the file to be lose.
+"""]]
diff --git a/doc/forum/git_annex_on_osx_only_creating_symlinks__63____63__/comment_5_39bad7441dcea4da4b389700301233de._comment b/doc/forum/git_annex_on_osx_only_creating_symlinks__63____63__/comment_5_39bad7441dcea4da4b389700301233de._comment
new file mode 100644
index 0000000000..f7975fc0ee
--- /dev/null
+++ b/doc/forum/git_annex_on_osx_only_creating_symlinks__63____63__/comment_5_39bad7441dcea4da4b389700301233de._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="https://www.google.com/accounts/o8/id?id=AItOawkipQLNyt8RHREHpg2k5wdYeRSCCvSNSBg"
+ nickname="Tim"
+ subject="I'll start another thread"
+ date="2014-01-06T18:17:44Z"
+ content="""
+Let me start another thread then and not pollute this one.  Thanks!
+"""]]
diff --git a/doc/forum/git_annex_repairs_since_thursday.mdwn b/doc/forum/git_annex_repairs_since_thursday.mdwn
new file mode 100644
index 0000000000..c1763b1f90
--- /dev/null
+++ b/doc/forum/git_annex_repairs_since_thursday.mdwn
@@ -0,0 +1,5 @@
+I have not tried to get git annex to work in a bit. My troubles of recovering this annex repo have been documented [here](http://git-annex.branchable.com/forum/git_annex_wants_to_repair_every_time_it__39__s_running/), [here](http://git-annex.branchable.com/forum/how_to_disaster_recovery/) and [here](http://git-annex.branchable.com/forum/__34__git_annex_assistant_--stop__34___doesn__39__t_work_in_OSX_10.11.4/). On Thurs day I thought I give it another try. Since Thursday it tries to repair two out of three repos: my local repo on my laptop and my repo on an external harddrive I have. The b2 special remote repo appears to be fine, or not checked, I'm not sure.
+
+What can I do, to get it working again?
+
+P.S.: In my mind this setup should work like this: I want to move contents from my laptop to the external hdd and to b2 so that there are two copies at all times. I want to use git annex so that I don't need to carry all that stuff around with me anymore, because space on my ssd is sparse.
diff --git a/doc/forum/git_annex_repairs_since_thursday/comment_1_38b4563db47ce9bdc52eab5506de20f8._comment b/doc/forum/git_annex_repairs_since_thursday/comment_1_38b4563db47ce9bdc52eab5506de20f8._comment
new file mode 100644
index 0000000000..89fa07cc73
--- /dev/null
+++ b/doc/forum/git_annex_repairs_since_thursday/comment_1_38b4563db47ce9bdc52eab5506de20f8._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="openmedi"
+ avatar="http://cdn.libravatar.org/avatar/563ffaff3b492c579bd8f094472e4506"
+ subject="comment 1"
+ date="2017-02-19T15:27:08Z"
+ content="""
+It's now Sunday and I stopped all three running git-annex processes on my system, after git used about 80% of my cpu, the daemon.log showing nothing and git annex info -F not responding.
+"""]]
diff --git a/doc/forum/git_annex_repairs_since_thursday/comment_2_b072ad2864e1caa7459d17d3426c8eee._comment b/doc/forum/git_annex_repairs_since_thursday/comment_2_b072ad2864e1caa7459d17d3426c8eee._comment
new file mode 100644
index 0000000000..726227c43d
--- /dev/null
+++ b/doc/forum/git_annex_repairs_since_thursday/comment_2_b072ad2864e1caa7459d17d3426c8eee._comment
@@ -0,0 +1,12 @@
+[[!comment format=mdwn
+ username="openmedi"
+ avatar="http://cdn.libravatar.org/avatar/563ffaff3b492c579bd8f094472e4506"
+ subject="comment 2"
+ date="2017-02-19T22:03:26Z"
+ content="""
+I am now trying to just start again. I have initiated a completely new repo to which I'm now adding files from the old repo manually or rather I have already done so, set up two remotes (flash drive and external hdd) and have now a repo which hold at least those files which were unproblematic to move over.
+
+I try now to recover any lost files and at the same time try to get rid of the repo on my laptop. Therefore I initiated on my external drive for disk space reasons a cloned copy of my local repo to which I'm now `get`ing  everything via `git annex get -A`. After this is done, I'll move the missing files, if there are any, over to the new repo by hand and will drop all files on my laptop's repo (to make sure that the new external hdd repo includes everything) and then delete any and all repos and special remotes.
+
+A fresh start seems to be the best way to go, since the old repo just couldn't be repaired by the assistant or me for that matter - or so it seemed.
+"""]]
diff --git a/doc/forum/git_annex_sync__58___only_git-annex.mdwn b/doc/forum/git_annex_sync__58___only_git-annex.mdwn
new file mode 100644
index 0000000000..84ae140cdd
--- /dev/null
+++ b/doc/forum/git_annex_sync__58___only_git-annex.mdwn
@@ -0,0 +1,3 @@
+Hi,
+
+i have an already existing git repository with a branch (*master*) and i added git annex to it (*git annex init*). Now i want to synchronise the file tracking information with annex through *git annex sync*, but keep the master branch unsynchronised (i want push/pull it manually as there are not only annexed files but also code). What is the best approach for my setup?
diff --git a/doc/forum/git_annex_sync__58___only_git-annex/comment_1_2be68ed36a1e6bfc896d5aea9463d3c7._comment b/doc/forum/git_annex_sync__58___only_git-annex/comment_1_2be68ed36a1e6bfc896d5aea9463d3c7._comment
new file mode 100644
index 0000000000..bf90184aa9
--- /dev/null
+++ b/doc/forum/git_annex_sync__58___only_git-annex/comment_1_2be68ed36a1e6bfc896d5aea9463d3c7._comment
@@ -0,0 +1,14 @@
+[[!comment format=mdwn
+ username="joey"
+ subject="""comment 1"""
+ date="2014-10-20T15:19:13Z"
+ content="""
+Sounds like you should just use normal `git push`/`git pull` commands.
+Works fine with git-annex. 
+
+Just be sure to include the `git-annex` branch in your pushes.
+Eg, `git push origin master git-annex`
+
+You'll probably want to run `git annex merge` after pulling, to merge the
+local and remote git-annex branches.
+"""]]
diff --git a/doc/forum/git_annex_sync__58___only_git-annex/comment_2_50e137e4d278dfd0103a41aff0cfa3a9._comment b/doc/forum/git_annex_sync__58___only_git-annex/comment_2_50e137e4d278dfd0103a41aff0cfa3a9._comment
new file mode 100644
index 0000000000..10f3792b35
--- /dev/null
+++ b/doc/forum/git_annex_sync__58___only_git-annex/comment_2_50e137e4d278dfd0103a41aff0cfa3a9._comment
@@ -0,0 +1,10 @@
+[[!comment format=mdwn
+ username="https://www.google.com/accounts/o8/id?id=AItOawlog_5wIICaMcrKTexlFNA6IO6UTp323aE"
+ nickname="Torkaly"
+ subject="comment 2"
+ date="2014-10-22T08:56:43Z"
+ content="""
+Thank you for your response.
+
+So annex looks like it's not really designed to work with an existing git repository, but only standalone?!
+"""]]
diff --git a/doc/forum/git_annex_sync__58___only_git-annex/comment_3_7753f8276478e0e05c10dba2b84bbc49._comment b/doc/forum/git_annex_sync__58___only_git-annex/comment_3_7753f8276478e0e05c10dba2b84bbc49._comment
new file mode 100644
index 0000000000..49a61c0b76
--- /dev/null
+++ b/doc/forum/git_annex_sync__58___only_git-annex/comment_3_7753f8276478e0e05c10dba2b84bbc49._comment
@@ -0,0 +1,12 @@
+[[!comment format=mdwn
+ username="http://joeyh.name/"
+ ip="209.250.56.96"
+ subject="comment 3"
+ date="2014-10-22T16:18:16Z"
+ content="""
+I struggle to see how you could draw that conclusion from what I said.
+
+git-annex will work fine in an existing git repository. You can mix regular git commands like `git add`, `git push`, `git pull`, `git merge` with git-annex commands like `git annex add`, `git annex copy --to origin`, `git annex get`, `git annex merge`, in the same repository.
+
+The `git annex sync` command effcetively runs `git commit; git pull; git annex merge; git push; git annex copy --to origin; git annex get`. If you don't want to run all those commands at once, you don't want to run `git annex sync`. That will not prevent you from using git-annex in any way.
+"""]]
diff --git a/doc/forum/git_annex_sync__58___only_git-annex/comment_5_52010f21a15d76d68986aa1fba29aaf1._comment b/doc/forum/git_annex_sync__58___only_git-annex/comment_5_52010f21a15d76d68986aa1fba29aaf1._comment
new file mode 100644
index 0000000000..cee2a9bd22
--- /dev/null
+++ b/doc/forum/git_annex_sync__58___only_git-annex/comment_5_52010f21a15d76d68986aa1fba29aaf1._comment
@@ -0,0 +1,10 @@
+[[!comment format=mdwn
+ username="https://www.google.com/accounts/o8/id?id=AItOawlog_5wIICaMcrKTexlFNA6IO6UTp323aE"
+ nickname="Torkaly"
+ subject="comment 5"
+ date="2014-10-29T18:51:29Z"
+ content="""
+Hi,
+
+thank you for your response. I just want to control my branches (master, dev and so on ...) by myself, without sync/master or sync/dev and without merging it automatically. But the git-annex branch should be populated between the repositories \"magically\" (some kind of \"git annex syncannex\"). As annex can't deliver such a basic functionality i assumed, that it was not designed to work with existing \"real\" git repositories.
+"""]]
diff --git a/doc/forum/git_annex_sync__58___only_git-annex/comment_6_df94154fbbc4edbf7ff658f61bde48b5._comment b/doc/forum/git_annex_sync__58___only_git-annex/comment_6_df94154fbbc4edbf7ff658f61bde48b5._comment
new file mode 100644
index 0000000000..8772255eb4
--- /dev/null
+++ b/doc/forum/git_annex_sync__58___only_git-annex/comment_6_df94154fbbc4edbf7ff658f61bde48b5._comment
@@ -0,0 +1,11 @@
+[[!comment format=mdwn
+ username="joey"
+ subject="""comment 1"""
+ date="2014-10-31T20:25:40Z"
+ content="""
+I have explained clearly in comment #1 above how to do what you want to do,
+using git-annex, so it is a pity if still think that "annex
+can't deliver such a basic functionality". 
+
+It can. I have explained how. I don't know how to explain any better.
+"""]]
diff --git a/doc/forum/git_annex_sync_dies___40__sometimes__41__.mdwn b/doc/forum/git_annex_sync_dies___40__sometimes__41__.mdwn
new file mode 100644
index 0000000000..84f9d11257
--- /dev/null
+++ b/doc/forum/git_annex_sync_dies___40__sometimes__41__.mdwn
@@ -0,0 +1,22 @@
+I've set up git annex on my laptop and on a remote server using the gitolite v3 git-annex branch.
+When I 'git annex sync' from my laptop (mac OSX) to the server, and 'git annex copy . --to server' all works fine.
+
+Later, I tried to clone the repository on another machine (linux 2.6.32). again all is well. I 'git annex init' in the new clone and then try to 'git annex sync' -- now I run into problems.
+
+Specifically, the first request for the configlist (when I do 'git annex --debug sync' all is fine until the configlist request) somehow breaks and my ssh session shows:
+
+muxserver_listen bind(): Input/output error
+
+This seems to confuse the client, who now believes that git annex is not installed on the server (it is).
+
+If I issue the same command as git annex via ssh, all seems to work ok with the caveats below:
+
+I have a feeling it's related to the -o 'ControlPersist=yes' argument that git-annex appears to be giving ssh.  If I include this option when I run via ssh, ssh dies:
+
+>ssh git@server -o "ControlMaster=auto" -o "ControlPersist=yes" "git-annex-shell 'configlist' '/~/PRJ'"
+command-line: line 0: Bad configuration option: ControlPersist
+
+
+(note that it appears to be ok to leave "ControlMaster=auto" in).
+
+any thoughts?
diff --git a/doc/forum/git_annex_sync_dies___40__sometimes__41__/comment_1_48bbac0545bf13bbf04da723e418d037._comment b/doc/forum/git_annex_sync_dies___40__sometimes__41__/comment_1_48bbac0545bf13bbf04da723e418d037._comment
new file mode 100644
index 0000000000..9445421a42
--- /dev/null
+++ b/doc/forum/git_annex_sync_dies___40__sometimes__41__/comment_1_48bbac0545bf13bbf04da723e418d037._comment
@@ -0,0 +1,14 @@
+[[!comment format=mdwn
+ username="http://joeyh.name/"
+ ip="64.134.31.139"
+ subject="comment 1"
+ date="2013-10-15T18:16:23Z"
+ content="""
+Since this seems to be a problem with ssh connection caching, you can disable that:
+
+git config annex.sshcaching false
+
+Seems like the problem would be in the version of ssh you have installed on your linux client. Or possibly some interaction between its version and the server's version. 
+
+Since linux 2.6.32 is quite old, I'll bet you have an old ssh too.. what version?
+"""]]
diff --git a/doc/forum/git_annex_test_on_windows.mdwn b/doc/forum/git_annex_test_on_windows.mdwn
new file mode 100644
index 0000000000..4e2f8dea65
--- /dev/null
+++ b/doc/forum/git_annex_test_on_windows.mdwn
@@ -0,0 +1,5 @@
+is it worth sharing back git annex test failures occourring on some windows setup ?
+i'm having 3 failures, what kind of report can i produce shall it be of any use ?
+is it enough a test log coming from git annex test > test.log 2>&1 ?
+thanks again
+
diff --git a/doc/forum/git_annex_test_on_windows/comment_1_258ac5cfa2f5d24e737d94dc48f06899._comment b/doc/forum/git_annex_test_on_windows/comment_1_258ac5cfa2f5d24e737d94dc48f06899._comment
new file mode 100644
index 0000000000..d7968c16ed
--- /dev/null
+++ b/doc/forum/git_annex_test_on_windows/comment_1_258ac5cfa2f5d24e737d94dc48f06899._comment
@@ -0,0 +1,10 @@
+[[!comment format=mdwn
+ username="http://joeyh.name/"
+ ip="209.250.56.246"
+ subject="comment 1"
+ date="2013-11-15T20:53:17Z"
+ content="""
+Yes, it's fine to file a bug about test failures. Be sure you try with a current autobuild.
+
+The redirection you show would work on linux, I don't know about windows. But capturing a transcript of the test output is certainly what I need.
+"""]]
diff --git a/doc/forum/git_annex_uninit_creates_broken_symlinks.mdwn b/doc/forum/git_annex_uninit_creates_broken_symlinks.mdwn
new file mode 100644
index 0000000000..5ba13653c6
--- /dev/null
+++ b/doc/forum/git_annex_uninit_creates_broken_symlinks.mdwn
@@ -0,0 +1,7 @@
+Hi, I'm attempting to uninit my giant git annex repo for my home folder. I've upgraded this to the latest repo version that uses smudge files instead of direct mode.
+
+However, when I try to uninit it, I end up with many broken symlinks, similar to that described [here](https://git-annex.branchable.com/forum/Broken_symlinks_remain_after_drop/).
+
+Any suggestions on how I can best restore these files to my home directory and migrate to using annex for an annex folder for stuff I need to transfer?
+
+Thanks!
diff --git a/doc/forum/git_annex_uninit_creates_broken_symlinks/comment_1_b7fb7fac51f59b23aafbc8dcce0aef07._comment b/doc/forum/git_annex_uninit_creates_broken_symlinks/comment_1_b7fb7fac51f59b23aafbc8dcce0aef07._comment
new file mode 100644
index 0000000000..b6de55622f
--- /dev/null
+++ b/doc/forum/git_annex_uninit_creates_broken_symlinks/comment_1_b7fb7fac51f59b23aafbc8dcce0aef07._comment
@@ -0,0 +1,11 @@
+[[!comment format=mdwn
+ username="joey"
+ subject="""comment 1"""
+ date="2017-06-06T16:59:00Z"
+ content="""
+If you uninit a repository that does not have the content of some files
+locally present, you'll be left with broken symlinks for those files.
+
+The way to avoid this is to run `git annex get` to get the contents of
+all files downloaded into the repository before you uninit.
+"""]]
diff --git a/doc/forum/git_annex_vicfg___40__preferred_content__41___examples.mdwn b/doc/forum/git_annex_vicfg___40__preferred_content__41___examples.mdwn
new file mode 100644
index 0000000000..54676514ca
--- /dev/null
+++ b/doc/forum/git_annex_vicfg___40__preferred_content__41___examples.mdwn
@@ -0,0 +1,37 @@
+My problem: I want to free up some disk space by doing `git annex drop FOO`, but then when I do `git annex add BAR; git annex sync --content`, FOO reappears.
+
+Looking into the man pages, I need to set up `git annex vicfg` to make `sync --content` ignore (exclude) FOO.
+
+I am overwhelmed by the configuration file and would just like to see a minimal complete example configuration file that does what I want. In the long run I probably want to use options like "groupwanted" and such, but am not sure what a "group" is exactly.
+
+Grepping for 'exclude', my config file has this already in it:
+
+```
+# Standard preferred contents
+# (Used by wanted or groupwanted expressions containing "standard")
+# (For reference only; built-in and cannot be changed!)
+# standard client = (include=* and ((exclude=*/archive/* and exclude=archive/*) or (not (copies=archive:1 or copies=smallarchive:1)))) or approxlackingcopies=1
+```
+
+but I am not sure how to go about it. For one, the comment hints that I need to use a secondary `wanted` or `groupwanted` expression to reference the `standard` keyword to actually use it, but I am just confused by all of these interlocking components.
+
+Also, I see stuff like this:
+
+```
+# Repository preferred contents
+# (Set to "standard" to use a repository's group's preferred contents)
+# (for web)
+#wanted 00000000-0000-0000-0000-000000000001 = 
+# (for bittorrent)
+#wanted 00000000-0000-0000-0000-000000000002 = 
+# (for l@k0:/dev/shm/annex)
+#wanted 56e6081b-4282-4f07-b53c-9042240cd75e = 
+# (for k0)
+#wanted a71d805d-df77-42e7-97b0-ba7686c05083 = 
+# (for l@m0:/mnt/a/annex.git [origin])
+#wanted c5c012cc-2080-481f-81bc-7f449f33490b =                                                                                                                                     
+```
+
+and am confused by these UUIDs. Where do these UUIDs come from? Am I supposed to use UUIDs?
+
+If there is a link or another doc I missed, please point me in the right direction.
diff --git a/doc/forum/git_annex_vicfg___40__preferred_content__41___examples/comment_1_1e59c9476b206613285e4f39b2c9efa8._comment b/doc/forum/git_annex_vicfg___40__preferred_content__41___examples/comment_1_1e59c9476b206613285e4f39b2c9efa8._comment
new file mode 100644
index 0000000000..1c408e7ab0
--- /dev/null
+++ b/doc/forum/git_annex_vicfg___40__preferred_content__41___examples/comment_1_1e59c9476b206613285e4f39b2c9efa8._comment
@@ -0,0 +1,14 @@
+[[!comment format=mdwn
+ username="joey"
+ subject="""comment 1"""
+ date="2016-07-30T16:30:58Z"
+ content="""
+If you're finding vicfg confusing, you don't have to use it; the same
+configuration can be done at the command line using `git annex wanted`.
+See [[preferred_content]] for documentation.
+
+If you just want to have full control over what files are stored in the
+local repository, the easiest way to do that is to run `git annex get` and
+`git annex drop` manually when you want or don't want a file's content,
+and don't pass --content to `git annex sync`.
+"""]]
diff --git a/doc/forum/git_annex_vicfg___40__preferred_content__41___examples/comment_2_9d0cfb1d221e3ac6ed15db9cefab60fa._comment b/doc/forum/git_annex_vicfg___40__preferred_content__41___examples/comment_2_9d0cfb1d221e3ac6ed15db9cefab60fa._comment
new file mode 100644
index 0000000000..a2d05304ad
--- /dev/null
+++ b/doc/forum/git_annex_vicfg___40__preferred_content__41___examples/comment_2_9d0cfb1d221e3ac6ed15db9cefab60fa._comment
@@ -0,0 +1,7 @@
+[[!comment format=mdwn
+ username="listx"
+ subject="comment 2"
+ date="2016-08-01T20:06:46Z"
+ content="""
+OK; but are the UUIDs documented anywhere? I'd prefer not to dive into the sources if possible. Thanks.
+"""]]
diff --git a/doc/forum/git_annex_vicfg___40__preferred_content__41___examples/comment_3_78bc3e56502e425f4f77bc35ed2199b8._comment b/doc/forum/git_annex_vicfg___40__preferred_content__41___examples/comment_3_78bc3e56502e425f4f77bc35ed2199b8._comment
new file mode 100644
index 0000000000..c1345862bb
--- /dev/null
+++ b/doc/forum/git_annex_vicfg___40__preferred_content__41___examples/comment_3_78bc3e56502e425f4f77bc35ed2199b8._comment
@@ -0,0 +1,41 @@
+[[!comment format=mdwn
+ username="listx"
+ subject="comment 3"
+ date="2016-08-04T02:07:55Z"
+ content="""
+I went ahead and used `git annex wanted . \"exclude=FOO\"` and it did what I wanted. For posterity, here are some other observations:
+
+- The file that `git annex vicfg` shows was updated to include the invocation of `git annex wanted`, like this:
+
+```
+...(truncated for brevity)...
+# Repository preferred contents                                                                                                                                                    
+# (Set to \"standard\" to use a repository's group's preferred contents)
+# (for k0)
+wanted a71d805d-df77-42e7-97b0-ba7686c05083 = exclude=FOO
+# (for web)
+...(truncated for brevity)...
+```
+
+()- I noticed that the UUID above matches the UUID found in my `.git/config` file, which looks like this:
+
+```
+[core]
+	repositoryformatversion = 0
+	filemode = true
+	bare = false
+	logallrefupdates = true
+[remote \"origin\"]
+	url = m0:/mnt/a/annex.git
+	fetch = +refs/heads/*:refs/remotes/origin/*
+	annex-uuid = c5c012cc-2080-481f-81bc-7f449f33490b
+[branch \"master\"]
+	remote = origin
+	merge = refs/heads/master
+[annex]
+	uuid = a71d805d-df77-42e7-97b0-ba7686c05083
+	version = 5
+```
+
+.
+"""]]
diff --git a/doc/forum/git_annex_wants_to_repair_every_time_it__39__s_running.mdwn b/doc/forum/git_annex_wants_to_repair_every_time_it__39__s_running.mdwn
new file mode 100644
index 0000000000..db4b368a23
--- /dev/null
+++ b/doc/forum/git_annex_wants_to_repair_every_time_it__39__s_running.mdwn
@@ -0,0 +1,5 @@
+As I tried to explain [here](http://git-annex.branchable.com/forum/__34__git_annex_assistant_--stop__34___doesn__39__t_work_in_OSX_10.11.4/): Every time I try to start annex, I have the problem that it wants to repair my repo. I'm not sure what's causing this and how to proceed. I would appreciate any pointers on how to triage the problem. There is a lot more info in the other thread.
+
+OS X: 10.11.4
+git: 2.8.3 (installed via brew)
+git annex: 6.20160527  (installed via brew)
diff --git a/doc/forum/git_annex_wants_to_repair_every_time_it__39__s_running/comment_1_c182b03bae6e691eb68465fedf5c77d5._comment b/doc/forum/git_annex_wants_to_repair_every_time_it__39__s_running/comment_1_c182b03bae6e691eb68465fedf5c77d5._comment
new file mode 100644
index 0000000000..585beea319
--- /dev/null
+++ b/doc/forum/git_annex_wants_to_repair_every_time_it__39__s_running/comment_1_c182b03bae6e691eb68465fedf5c77d5._comment
@@ -0,0 +1,9 @@
+[[!comment format=mdwn
+ username="openmedi"
+ subject="comment 1"
+ date="2016-06-18T21:48:42Z"
+ content="""
+So there are no ideas on how to proceed from anyone? Was the way I have written this message in any way too complicated or was it for some reason formulated in a manner, that made it look too demanding of others peoples time? I was not intentionally doing so, but I would appreciate any kind of commend on how to better phrase the question or on how to improve it in a way that this issue can be solved. Thanks in advance!
+
+P.S.: Since people probably don't stumble onto this comment, I'll give it another week or so before I would open up a new post containing the same issue.
+"""]]
diff --git a/doc/forum/git_annex_wants_to_repair_every_time_it__39__s_running/comment_2_6b3427b2d3e1c5f4c0582be1020c89f8._comment b/doc/forum/git_annex_wants_to_repair_every_time_it__39__s_running/comment_2_6b3427b2d3e1c5f4c0582be1020c89f8._comment
new file mode 100644
index 0000000000..951365aec2
--- /dev/null
+++ b/doc/forum/git_annex_wants_to_repair_every_time_it__39__s_running/comment_2_6b3427b2d3e1c5f4c0582be1020c89f8._comment
@@ -0,0 +1,12 @@
+[[!comment format=mdwn
+ username="joey"
+ subject="""comment 2"""
+ date="2016-09-05T20:10:18Z"
+ content="""
+This seems to be the same person with the same problem discussed in 
+[[how_to_disaster_recovery]].
+
+I'm glad that we seem to have gotten to the bottom of it over there.
+
+Posting the same question over and over just wastes my time, please don't.
+"""]]
diff --git a/doc/forum/git_annex_whereis_--json_output_with_two_variables_with_same_name.mdwn b/doc/forum/git_annex_whereis_--json_output_with_two_variables_with_same_name.mdwn
new file mode 100644
index 0000000000..0cc7ae530a
--- /dev/null
+++ b/doc/forum/git_annex_whereis_--json_output_with_two_variables_with_same_name.mdwn
@@ -0,0 +1,21 @@
+Hi Joey,
+
+I'm trying to extract data from a git annex whereis --json, but discovered that in json, the output has two values with same name.
+For instance: 
+
+{
+ "command":"whereis",
+ "file":"filename.webm",
+ "note":"3 copies",
+ "whereis":[
+  {"uuid":"1b7d69fe-22e2-11e4-bc47-279f5115dfde","description":"chasqui","here":false},
+  {"uuid":"e5cc9824-450b-4340-b30f-c2d92c6a52f7","description":"coco","here":false},
+  {"uuid":"f84a1327-febb-4199-a106-9c3fd2288826","description":"[dpadua]","here":false}],
+ "note":"\t1b7d69fe-22e2-11e4-bc47-279f5115dfde -- chasqui\n \te5cc9824-450b-4340-b30f-c2d92c6a52f7 -- coco\n \tf84a1327-febb-4199-a106-9c3fd2288826 -- [dpadua]\n",
+ "untrusted":[],
+ "success":true
+}
+
+When I read the json from another program, it gets only the second value (and I wanted the first, the number of copies). I'm using git-annex version 5.20140831+b1
+
+
diff --git a/doc/forum/git_annex_whereis_--json_output_with_two_variables_with_same_name/comment_1_3bfde59729b904aa1ef815427dd35ae6._comment b/doc/forum/git_annex_whereis_--json_output_with_two_variables_with_same_name/comment_1_3bfde59729b904aa1ef815427dd35ae6._comment
new file mode 100644
index 0000000000..a027b5276c
--- /dev/null
+++ b/doc/forum/git_annex_whereis_--json_output_with_two_variables_with_same_name/comment_1_3bfde59729b904aa1ef815427dd35ae6._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="http://joeyh.name/"
+ ip="209.250.56.54"
+ subject="comment 1"
+ date="2014-10-10T17:09:32Z"
+ content="""
+The \"3 copies\" string is not meant to be machine-parsable anyway. Why don't you just look at the whereis object in the json, and count the number of items in its list, which will give you the number of copies.
+"""]]
diff --git a/doc/forum/git_annex_whereis_--json_output_with_two_variables_with_same_name/comment_2_d08a955a11953cc783f09bfba180dbd6._comment b/doc/forum/git_annex_whereis_--json_output_with_two_variables_with_same_name/comment_2_d08a955a11953cc783f09bfba180dbd6._comment
new file mode 100644
index 0000000000..02bd6de1ff
--- /dev/null
+++ b/doc/forum/git_annex_whereis_--json_output_with_two_variables_with_same_name/comment_2_d08a955a11953cc783f09bfba180dbd6._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="https://www.google.com/accounts/o8/id?id=AItOawkR2dZjxqujBXcitvsP_PeWG8A-LfLU_tg"
+ nickname="Fernão"
+ subject="comment 2"
+ date="2014-10-10T18:18:04Z"
+ content="""
+nice, it worked =)
+"""]]
diff --git a/doc/forum/git_annex_windows_and_rsync.mdwn b/doc/forum/git_annex_windows_and_rsync.mdwn
new file mode 100644
index 0000000000..c45cd8d2d7
--- /dev/null
+++ b/doc/forum/git_annex_windows_and_rsync.mdwn
@@ -0,0 +1,26 @@
+Issue with getting files from a Linux ssh/rsync repo.
+
+I have a centralized repo on a small linux VM that I synchronize my workstations with, most are linux machines but one is a windows one.  I installed msysgit and git-annex for windows, and then run the following:
+
+
+    git clone ssh://user@IP.ADDRESS/home/user/annex annex  
+    cd annex  
+    git annex init 'windows'  
+    git annex copy --from origin  
+
+
+So basically I am just trying to get a copy of the central repo onto this windows machine for starters and get:
+
+
+    rsync: connection unexpectedly closed (0 bytes received so far) [sender]
+    rsync error: error in rsync protocol data stream (code 12) at io.c(226) [sender=3.1.1] 
+    rsync: connection unexpectedly closed (0 bytes received so far) [Receiver]  
+    rsync error: error in rsync protoco  rsync failed -- run git annex again to resume file transfer  
+    l dafailed
+    ta stream (code 12) atcopy  
+
+
+And I get this message for every file.
+
+
+I do have cygwin with rsync and ssh installed on this machine previously so I tried on a separate machine thinking there may be compatibility issues with no avail either.  I am not sure if this is an existing issue/work in progress with Windows/git-annex or if it is something I am just experiencing.  
diff --git a/doc/forum/git_annex_windows_and_rsync/comment_1_33249bf910446fcf98ffb2e7e35017bf._comment b/doc/forum/git_annex_windows_and_rsync/comment_1_33249bf910446fcf98ffb2e7e35017bf._comment
new file mode 100644
index 0000000000..270f499e9c
--- /dev/null
+++ b/doc/forum/git_annex_windows_and_rsync/comment_1_33249bf910446fcf98ffb2e7e35017bf._comment
@@ -0,0 +1,7 @@
+[[!comment format=mdwn
+ username="joey"
+ subject="""comment 1"""
+ date="2015-05-07T18:14:43Z"
+ content="""
+[[bugs/rsync_on_windows_broken_by_upgrade]]
+"""]]
diff --git a/doc/forum/git_annex_with_local_apache_webdav_server.mdwn b/doc/forum/git_annex_with_local_apache_webdav_server.mdwn
new file mode 100644
index 0000000000..db489fdfa8
--- /dev/null
+++ b/doc/forum/git_annex_with_local_apache_webdav_server.mdwn
@@ -0,0 +1,23 @@
+Hi, 
+
+trying to make git annex work locally with an apache webdav server.
+
+I have the webdav server working without issue on computers. When we try to init the repository there we get the following error:
+
+WEBDAV_USERNAME=user WEBDAV_PASSWORD=xxxxxx git annex initremote webdavtest type=webdav url=http://webdavserver/webdavsgare/annextest4 encryption=none 
+initremote webdavtest (testing WebDAV server...)
+
+git-annex: WebDAV failed to delete file: "Locked": user error
+failed
+git-annex: initremote: 1 failed
+
+
+Does anyone have any thoughts? I can post config of webdav if it helps, though the error I receive in the error_log of apache is as follows:
+
+This resource is locked and an "If:" header was not supplied to allow access to the resource.  [423, #0]
+
+but I can manage the files through command line, web interface and mounted drive with no issue.
+
+thank you in advance.
+
+Damien
diff --git a/doc/forum/git_annex_with_local_apache_webdav_server/comment_1_a3b89f90f9ac70e0a9b0711ede1cb810._comment b/doc/forum/git_annex_with_local_apache_webdav_server/comment_1_a3b89f90f9ac70e0a9b0711ede1cb810._comment
new file mode 100644
index 0000000000..7963f64f03
--- /dev/null
+++ b/doc/forum/git_annex_with_local_apache_webdav_server/comment_1_a3b89f90f9ac70e0a9b0711ede1cb810._comment
@@ -0,0 +1,12 @@
+[[!comment format=mdwn
+ username="http://joeyh.name/"
+ ip="209.250.56.146"
+ subject="comment 1"
+ date="2014-02-26T19:40:18Z"
+ content="""
+The webdav library that git-annex uses does some webdav file locking. When deleting a file, it first locks it. It sounds like your webdav server does not like to delete a file that has been locked. Whether this is in spec I don't know.
+
+I do know that got rid of all the webdav locking stuff in a revamp of git-annex's webdav support 2 days ago! git-annex does not need locking the way it uses webdav.
+
+However, the daily builds are still built with an old version of the DAV library, which still does the locking ... and will probably remain like that for some time. To get the new version without locking, you need to build git-annex from its git master on a system with http://hackage.haskell.org/package/DAV version 0.6.1. Building from source with cabal is one way to do it. I'll be releasing a build for Debian unstable in the next day or two.
+"""]]
diff --git a/doc/forum/git_annex_with_local_apache_webdav_server/comment_2_d8e9237cf6e7f7558f836ba1352f5517._comment b/doc/forum/git_annex_with_local_apache_webdav_server/comment_2_d8e9237cf6e7f7558f836ba1352f5517._comment
new file mode 100644
index 0000000000..2b60a9967e
--- /dev/null
+++ b/doc/forum/git_annex_with_local_apache_webdav_server/comment_2_d8e9237cf6e7f7558f836ba1352f5517._comment
@@ -0,0 +1,13 @@
+[[!comment format=mdwn
+ username="https://www.google.com/accounts/o8/id?id=AItOawm7hS2LGu4sLUxLMdBA16PAMcVO7CDJmjw"
+ nickname="Damien"
+ subject="comment 2"
+ date="2014-03-10T12:16:43Z"
+ content="""
+I'm curious if anyone has set this up using a local apache webdav server?
+
+I'm stuck and could use some insight.
+
+thanks
+Damien
+"""]]
diff --git a/doc/forum/git_pull_remote_git-annex.mdwn b/doc/forum/git_pull_remote_git-annex.mdwn
new file mode 100644
index 0000000000..349610693b
--- /dev/null
+++ b/doc/forum/git_pull_remote_git-annex.mdwn
@@ -0,0 +1,11 @@
+I thought I'd followed the walk through when initially setting up my repos.
+
+However I find that I have to do the following to sync my annex's.
+
+    git pull remote master
+    git checkout git-annex
+    git pull remote git-annex
+    git checkout master
+    git annex get .
+
+Has something gone wrong? I see no mention of syncing git-annex repos in the walk-through...
diff --git a/doc/forum/git_pull_remote_git-annex/comment_1_9c245db3518d8b889ecdf5115ad9e053._comment b/doc/forum/git_pull_remote_git-annex/comment_1_9c245db3518d8b889ecdf5115ad9e053._comment
new file mode 100644
index 0000000000..989ab9bcd8
--- /dev/null
+++ b/doc/forum/git_pull_remote_git-annex/comment_1_9c245db3518d8b889ecdf5115ad9e053._comment
@@ -0,0 +1,36 @@
+[[!comment format=mdwn
+ username="http://joey.kitenet.net/"
+ nickname="joey"
+ subject="comment 1"
+ date="2011-12-06T16:43:29Z"
+ content="""
+You're taking a very long and strange way to a place that you can reach as follows:
+
+
    +git pull remote
+git annex get .
+
    
+
+Which is just as shown in [[walkthrough/getting_file_content]]. 
+
+In particular, \"git pull remote\" first fetches all branches from the remote, including the git-annex branch.
+When you say \"git pull remote master\", you're preventing it from fetching the git-annex branch. 
+If for some reason you want the slightly longer way around, it is:
+
+
    +git pull remote master
+git fetch remote git-annex
+git annex get .
+
    
+
+Or, eqivilantly but with less network connections:
+
+
    +git fetch remote
+git merge remote/master
+git annex get .
+
    
+
+BTW, notice that this is all bog-standard git branch pulling stuff, not specific to git-annex in the least.
+Consult your extensive and friendly git documentation for details. :)
+"""]]
diff --git a/doc/forum/git_pull_remote_git-annex/comment_2_0f7f4a311b0ec1d89613e80847e69b42._comment b/doc/forum/git_pull_remote_git-annex/comment_2_0f7f4a311b0ec1d89613e80847e69b42._comment
new file mode 100644
index 0000000000..198f95cee8
--- /dev/null
+++ b/doc/forum/git_pull_remote_git-annex/comment_2_0f7f4a311b0ec1d89613e80847e69b42._comment
@@ -0,0 +1,14 @@
+[[!comment format=mdwn
+ username="https://www.google.com/accounts/o8/id?id=AItOawnXybLxkPMYpP3yw4b_I6IdC3cKTD-xEdU"
+ nickname="Matt"
+ subject="comment 2"
+ date="2011-12-06T23:23:29Z"
+ content="""
+Doh! Total brain melt on my part. Thanks for the additional info.  Not taking my time and reading things properly - kept assuming that the full remote pull failed due to the warning:
+
+    You asked to pull from the remote 'rss', but did not specify
+    a branch. Because this is not the default configured remote
+    for your current branch, you must specify a branch on the command line.
+
+Rookie mistake indeed.
+"""]]
diff --git a/doc/forum/git_pull_remote_git-annex/comment_3_1aa89725b5196e40a16edeeb5ccfa371._comment b/doc/forum/git_pull_remote_git-annex/comment_3_1aa89725b5196e40a16edeeb5ccfa371._comment
new file mode 100644
index 0000000000..0ead32dad2
--- /dev/null
+++ b/doc/forum/git_pull_remote_git-annex/comment_3_1aa89725b5196e40a16edeeb5ccfa371._comment
@@ -0,0 +1,14 @@
+[[!comment format=mdwn
+ username="https://www.google.com/accounts/o8/id?id=AItOawnXybLxkPMYpP3yw4b_I6IdC3cKTD-xEdU"
+ nickname="Matt"
+ subject="comment 3"
+ date="2011-12-21T16:06:25Z"
+ content="""
+hmmmm - I'm still not sure I get this.
+
+If I'm using a whole bunch of distributed annexs with no central repo, then I can not do a `git pull remote` without either specifying the branch to use or changing default tracked remote via `git branch --set-upstream`.  The former like you note doesn't pull the git-annex branch down the latter only works one-at-a-time.
+
+The docs read to me as though I ought to be able to do a `git pull remote ; git annex get .` using anyone of my distributed annexs.
+
+Am I doing something wrong?  Or is the above correct?
+"""]]
diff --git a/doc/forum/git_pull_remote_git-annex/comment_4_646f2077edcabc000a7d9cb75a93cf55._comment b/doc/forum/git_pull_remote_git-annex/comment_4_646f2077edcabc000a7d9cb75a93cf55._comment
new file mode 100644
index 0000000000..6ba1796939
--- /dev/null
+++ b/doc/forum/git_pull_remote_git-annex/comment_4_646f2077edcabc000a7d9cb75a93cf55._comment
@@ -0,0 +1,37 @@
+[[!comment format=mdwn
+ username="http://adamspiers.myopenid.com/"
+ nickname="Adam"
+ subject="I think Matt is right."
+ date="2011-12-23T14:04:44Z"
+ content="""
+I got bitten by this too.  It seems that the user is expected to fetch
+remote git-annex branches themselves, but this is not documented
+anywhere.
+
+The man page says of \"git annex merge\":
+
+    Automatically merges any changes from remotes into the git-annex
+    branch.
+
+I am not a git newbie, but even so I had incorrectly assumed that git
+annex merge would take care of pulling the git-annex branch from the
+remote prior to merging, thereby ensuring all versions of the
+git-annex branch would be merged, and that the location tracking data
+would be synced across all peer repositories.
+
+My master branches do not track any specific upstream branch, because
+I am operating in a decentralized fashion.  Therefore the error
+message caused by `git pull $remote` succeeded in encouraging me to
+instead use `git pull $remote master`, and this excludes the git-annex
+branch from the fetch.  Even worse, a git newbie might realise this
+and be tempted to do `git pull $remote git-annex`.
+
+Therefore I think it needs to be explicitly documented that
+
+    git fetch $remote
+    git merge $remote/master
+
+is required when the local branch doesn't track an upstream branch.
+Or maybe a `--fetch` option could be added to `git annex merge` to
+perform the fetch from all remotes before running the merge(s).
+"""]]
diff --git a/doc/forum/git_pull_remote_git-annex/comment_5_4f2a05ef6551806dd0ec65372f183ca4._comment b/doc/forum/git_pull_remote_git-annex/comment_5_4f2a05ef6551806dd0ec65372f183ca4._comment
new file mode 100644
index 0000000000..c01f241202
--- /dev/null
+++ b/doc/forum/git_pull_remote_git-annex/comment_5_4f2a05ef6551806dd0ec65372f183ca4._comment
@@ -0,0 +1,10 @@
+[[!comment format=mdwn
+ username="http://joey.kitenet.net/"
+ nickname="joey"
+ subject="comment 5"
+ date="2011-12-23T16:50:26Z"
+ content="""
+My goal for `git-annex merge` is that users should not need to know about it, so it should not be doing expensive pulls.
+
+I hope that `git annex sync` will grow some useful features to support fully distributed git usage, as being discussed in [[pure_git-annex_only_workflow]]. I still use centralized git to avoid these problems myself.
+"""]]
diff --git a/doc/forum/git_pull_remote_git-annex/comment_6_3925d1aa56bce9380f712e238d63080f._comment b/doc/forum/git_pull_remote_git-annex/comment_6_3925d1aa56bce9380f712e238d63080f._comment
new file mode 100644
index 0000000000..f4b5ebec20
--- /dev/null
+++ b/doc/forum/git_pull_remote_git-annex/comment_6_3925d1aa56bce9380f712e238d63080f._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="http://adamspiers.myopenid.com/"
+ nickname="Adam"
+ subject="comment 6"
+ date="2011-12-23T17:14:03Z"
+ content="""
+Extending `git annex sync` would be nice, although auto-commit does not suit every use case, so it would be better not to couple one to the other.
+"""]]
diff --git a/doc/forum/git_pull_remote_git-annex/comment_7_24c45ee981b18bc78325c768242e635d._comment b/doc/forum/git_pull_remote_git-annex/comment_7_24c45ee981b18bc78325c768242e635d._comment
new file mode 100644
index 0000000000..dad2c0af21
--- /dev/null
+++ b/doc/forum/git_pull_remote_git-annex/comment_7_24c45ee981b18bc78325c768242e635d._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="http://adamspiers.myopenid.com/"
+ nickname="Adam"
+ subject="comment 7"
+ date="2011-12-23T17:24:58Z"
+ content="""
+P.S. I see you already [fixed the docs](http://source.git-annex.branchable.com/?p=source.git;a=commitdiff;h=a0227e81f9c82afc12ac1bd1cecd63cc0894d751) - thanks! :)
+"""]]
diff --git a/doc/forum/git_pull_remote_git-annex/comment_8_7e76ee9b6520cbffaf484c9299a63ad3._comment b/doc/forum/git_pull_remote_git-annex/comment_8_7e76ee9b6520cbffaf484c9299a63ad3._comment
new file mode 100644
index 0000000000..5943d93127
--- /dev/null
+++ b/doc/forum/git_pull_remote_git-annex/comment_8_7e76ee9b6520cbffaf484c9299a63ad3._comment
@@ -0,0 +1,12 @@
+[[!comment format=mdwn
+ username="http://joey.kitenet.net/"
+ nickname="joey"
+ subject="git tweak-fetch"
+ date="2011-12-26T18:50:35Z"
+ content="""
+The git tweak-fetch hook that I have been developing, and hope will be accepted into git soon, provides some abilities that could be used to make \"git pull remote\" always merge remote/master. Normall, git can only be configured to do that merge automatically for one remote (ie, origin). But the tweak-fetch hook can flag arbitrary branches as needing merge. 
+
+So, it could always flag tracking branches of the currently checked out branch for merge. This would be enabled by some setting, probably, since it's not necessarily the case that everyone wants to auto-merge when they pull like this. (Which is why git doesn't do it by default after all.)
+
+(The tweak-fetch hook will also entirely eliminate the need to run git annex merge manually, since it can always take care of merging the git-annex branch.)
+"""]]
diff --git a/doc/forum/git_status_typechange_in_v5_direct_false.mdwn b/doc/forum/git_status_typechange_in_v5_direct_false.mdwn
new file mode 100644
index 0000000000..f326850d3c
--- /dev/null
+++ b/doc/forum/git_status_typechange_in_v5_direct_false.mdwn
@@ -0,0 +1,33 @@
+Should I see a typechange from `git status` when using direct mode = false?
+
+I'm using digiKam to manage photos in a git-annex repository. digiKam will only write XMP sidecar files for real files, not symlinks, so I need to unlock every file in order to sync metadata to XMP files.
+
+Is it possible to suppress typechange in some way?
+
+Thanks.
+
+    On branch master
+    Changes not staged for commit:
+    (use "git add ..." to update what will be committed)
+    (use "git checkout -- ..." to discard changes in working directory)
+
+        typechange: photographers/jboxman/2004/06/16/20040616-004528.jpg
+        .......
+        typechange: photographers/jboxman/2004/06/17/20040617-010127.jpg
+
+    [annex]
+    uuid = 56e0d203-7898-xxxx-xxxx-1b04cec6597c
+    version = 5
+    largefiles = largerthan=20kb and not (include=*.xmp or include=*.thor or include=bin/*)
+    direct = false
+
+    git-annex version: 6.20171124
+    build flags: Assistant Webapp Pairing Testsuite S3(multipartupload)(storageclasses) WebDAV FsEvents TorrentParser MagicMime Feeds Quvi
+    dependency versions: aws-0.17.1 bloomfilter-2.0.1.0 cryptonite-0.24 DAV-1.3.1 feed-1.0.0.0 ghc-8.2.2 http-client-0.5.7.1 persistent-sqlite-2.6.3.1torrent-10000.1.1 uuid-1.3.13 yesod-1.4.5
+    key/value backends: SHA256E SHA256 SHA512E SHA512 SHA224E SHA224 SHA384E SHA384 SHA3_256E SHA3_256 SHA3_512E SHA3_512 SHA3_224E SHA3_224 SHA3_384ESHA3_384 SKEIN256E SKEIN256 SKEIN512E SKEIN512 SHA1E SHA1 MD5E MD5 WORM URL
+    remote types: git gcrypt p2p S3 bup directory rsync web bittorrent webdav tahoe glacier ddar hook external
+    local repository version: 5
+    supported repository versions: 3 5 6
+    upgrade supported from repository versions: 0 1 2 3 4 5
+    operating system: darwin x86_64
+
diff --git a/doc/forum/git_status_typechange_in_v5_direct_false/comment_1_eb4e2b1fd7615e2b6936b5df01d68d1f._comment b/doc/forum/git_status_typechange_in_v5_direct_false/comment_1_eb4e2b1fd7615e2b6936b5df01d68d1f._comment
new file mode 100644
index 0000000000..2e71853af1
--- /dev/null
+++ b/doc/forum/git_status_typechange_in_v5_direct_false/comment_1_eb4e2b1fd7615e2b6936b5df01d68d1f._comment
@@ -0,0 +1,16 @@
+[[!comment format=mdwn
+ username="andrew"
+ avatar="http://cdn.libravatar.org/avatar/acc0ece1eedf07dd9631e7d7d343c435"
+ subject="comment 1"
+ date="2018-05-31T17:39:10Z"
+ content="""
+This all seems very OK and expected to me.
+
+When you do `git-annex unlock photographers/jboxman/2004/06/16/20040616-004528.jpg` you are changing out the symlink for the actual file, so git has recognized the thing with this name is different. If you do `git-annex add photographers/jboxman/2004/06/16/20040616-004528.jpg` git-annex will recognize that you have not modified any content and will replace the file with the symlink, the file will no longer appear in git status and the file won't have any commits associated with it.
+
+So if you unlock everything, launch digiKam and generate your sidecars, then we see both jpgs and xmps in `git status`. If you then run `git annex add .` all your jpgs will turn back into symlinks and all your xmp files will appear in `git status` per your `largefiles` rule staged for committing.
+
+I haven't tested with your exact setup so I would recommend testing the workflow on a small folder to start. 
+
+
+"""]]
diff --git a/doc/forum/git_status_typechange_in_v5_direct_false/comment_2_19df04820aa80549cfabac0141c372c0._comment b/doc/forum/git_status_typechange_in_v5_direct_false/comment_2_19df04820aa80549cfabac0141c372c0._comment
new file mode 100644
index 0000000000..7b5d7d53b7
--- /dev/null
+++ b/doc/forum/git_status_typechange_in_v5_direct_false/comment_2_19df04820aa80549cfabac0141c372c0._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="jasonb885"
+ avatar="http://cdn.libravatar.org/avatar/c7330f4da122c671b935fc1d58bb02b1"
+ subject="oh"
+ date="2018-07-02T01:08:37Z"
+ content="""
+To usefully use files committed to git annex with digiKam, I have to have every single file unlocked. So in this case, what git status is telling me isn't useful. I can grep-v this out, but I wasn't expecting to see it since it seems like normal usage for git annex. It seems git doesn't see it that way and thinks something special has happened. I was hoping I was doing something wrong and if I used some other approach, git would be silent about the typechange. Thanks.
+"""]]
diff --git a/doc/forum/git_status_typechange_in_v5_direct_false/comment_3_20e04806a59e05a453431cb90086ccac._comment b/doc/forum/git_status_typechange_in_v5_direct_false/comment_3_20e04806a59e05a453431cb90086ccac._comment
new file mode 100644
index 0000000000..010081671c
--- /dev/null
+++ b/doc/forum/git_status_typechange_in_v5_direct_false/comment_3_20e04806a59e05a453431cb90086ccac._comment
@@ -0,0 +1,10 @@
+[[!comment format=mdwn
+ username="andrew"
+ avatar="http://cdn.libravatar.org/avatar/acc0ece1eedf07dd9631e7d7d343c435"
+ subject="comment 3"
+ date="2018-07-10T18:45:37Z"
+ content="""
+I don't believe the `git` team has provided any way for `git annex` to hook into the `status` command to provide different information.
+
+You can always do `git annex status` if you want to know what `git annex` thinks about the current situation.
+"""]]
diff --git a/doc/forum/git_tag_missing_for_3.20111011.mdwn b/doc/forum/git_tag_missing_for_3.20111011.mdwn
new file mode 100644
index 0000000000..781d0c91a0
--- /dev/null
+++ b/doc/forum/git_tag_missing_for_3.20111011.mdwn
@@ -0,0 +1 @@
+Well, the subject pretty much says it all :)
diff --git a/doc/forum/git_tag_missing_for_3.20111011/comment_1_7a53bf273f3078ab3351369ef2b5f2a6._comment b/doc/forum/git_tag_missing_for_3.20111011/comment_1_7a53bf273f3078ab3351369ef2b5f2a6._comment
new file mode 100644
index 0000000000..87cda998bf
--- /dev/null
+++ b/doc/forum/git_tag_missing_for_3.20111011/comment_1_7a53bf273f3078ab3351369ef2b5f2a6._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="http://joey.kitenet.net/"
+ nickname="joey"
+ subject="fixed that"
+ date="2011-10-13T15:36:59Z"
+ content="""
+:)
+"""]]
diff --git a/doc/forum/git_unannex_speed.mdwn b/doc/forum/git_unannex_speed.mdwn
new file mode 100644
index 0000000000..a5e6ad5719
--- /dev/null
+++ b/doc/forum/git_unannex_speed.mdwn
@@ -0,0 +1 @@
+It was fast to git annex a bunch of files. But git unannex seems a lot slower. Is there a faster way to get files out of git annex? Or to replace the symlinks with real files and then I could just remove the .git directory? I shouldn't have put so many in as a test but they are there now.
diff --git a/doc/forum/git_unannex_speed/comment_1_10cf326248f4e89e1f75bf97d7574763._comment b/doc/forum/git_unannex_speed/comment_1_10cf326248f4e89e1f75bf97d7574763._comment
new file mode 100644
index 0000000000..39fc4793cc
--- /dev/null
+++ b/doc/forum/git_unannex_speed/comment_1_10cf326248f4e89e1f75bf97d7574763._comment
@@ -0,0 +1,7 @@
+[[!comment format=mdwn
+ username="http://joeyh.name/"
+ subject="comment 1"
+ date="2012-08-02T16:58:41Z"
+ content="""
+It currently commits once per file, which is slow. `Command/Unannex.hs` explains why this is necessary. If I think of a way to avoid that, I will.
+"""]]
diff --git a/doc/forum/glacier_-_range_retrievals_and_daily_free_retrieval_allowance.mdwn b/doc/forum/glacier_-_range_retrievals_and_daily_free_retrieval_allowance.mdwn
new file mode 100644
index 0000000000..d9db436c61
--- /dev/null
+++ b/doc/forum/glacier_-_range_retrievals_and_daily_free_retrieval_allowance.mdwn
@@ -0,0 +1,6 @@
+I propose to add some functionality to git-annex, to automatically "throttle" the data retrieval from amazon glacier to stay within the daily free retrieval allowance. If someone would need to get his/her files faster, there should be an option to disable this throttling (or even better, specify the retrieval rate).
+
+As far as I understand glacier, this could be implemented using range retrievals. In short range retrievals enable you, to only retrieve a part of an archive in glacier. This can be used to only retrieve / request so much data, that you stay within the free retrieval allowance. ( please see [Q: Why would I retrieve only a range of an archive?](http://aws.amazon.com/glacier/faqs/#Why_would_I_retrieve_only_a_range_of_an_archive) ).
+
+
+This would be somewhat similar to the [smart retrieval feature in cloudberry](http://blog.cloudberrylab.com/2012/12/introducing-smart-restore-for-amazon.html) .
diff --git a/doc/forum/handling_MP3_metadata_changes.mdwn b/doc/forum/handling_MP3_metadata_changes.mdwn
new file mode 100644
index 0000000000..9b6caa27aa
--- /dev/null
+++ b/doc/forum/handling_MP3_metadata_changes.mdwn
@@ -0,0 +1,12 @@
+Hello,
+
+I'm still looking for a way to version control the metadata (title, artist, album name, ...) of my MP3s, I wonder if git annex could help for this problem ?
+The method I use now (without git annex) is to export the MP3 metadata to an textual format with one line per tag. 
+It's this textual file I handles with git. 
+The problem is to handle the mapping between the orignal file and the export file with file renaming or moving.
+I consider to use the checksum of the audio content (without metadata, this checksum never changes) to handle this problem.
+Maybe git annex has a different approach (better) to solve this problem ?
+How git annex would be use to solve the orignal problem ?
+
+Regards,
+Emmanuel Berry
diff --git a/doc/forum/handling_MP3_metadata_changes/comment_1_aa4955fd64ea5aa836f1a591e185c4a2._comment b/doc/forum/handling_MP3_metadata_changes/comment_1_aa4955fd64ea5aa836f1a591e185c4a2._comment
new file mode 100644
index 0000000000..483c1fb78d
--- /dev/null
+++ b/doc/forum/handling_MP3_metadata_changes/comment_1_aa4955fd64ea5aa836f1a591e185c4a2._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="http://joeyh.name/"
+ ip="209.250.56.154"
+ subject="comment 1"
+ date="2014-03-18T19:29:53Z"
+ content="""
+Check out [[metadata]]
+"""]]
diff --git a/doc/forum/hashing_objects_directories.mdwn b/doc/forum/hashing_objects_directories.mdwn
new file mode 100644
index 0000000000..5b7708fb58
--- /dev/null
+++ b/doc/forum/hashing_objects_directories.mdwn
@@ -0,0 +1,27 @@
+I'm wondering how easy the addition of hashing to the directories of the objects would be.
+
+Currently a tree directory structure becomes a flat two level tree under the .git/annex/objects directory ([[internals]]).  This, through the 555 mode on the directory prevents the accidental destruction of content, which is _good_.  However file and directory numbers soon add up in there and as such any file-systems with sub directory limitations will quickly realize the limit (certainly quicker than maybe expected).
+
+Suggestion is therefore to change from 
+
+ `.git/annex/objects/SHA1:123456789abcdef0123456789abcdef012345678/SHA1:123456789abcdef0123456789abcdef012345678`
+
+to 
+
+ `.git/annex/objects/SHA1:1/2/3456789abcdef0123456789abcdef012345678/SHA1:123456789abcdef0123456789abcdef012345678`
+
+or anything in between to a paranoid
+
+ `.git/annex/objects/SHA1:123/456/789/abc/def/012/345/678/9ab/cde/f01/234/5678/SHA1:123456789abcdef0123456789abcdef012345678`
+
+Also the use of a colon specifically breaks FAT32 ([[bugs/fat_support]]), must it be a colon or could an extra directory be used? i.e. `.git/annex/objects/SHA1/*/...`
+
+`git annex init` could also create all but the last level directory on initialization. I'm thinking `SHA1/1/1, SHA1/1/2, ..., SHA256/f/f, ..., URL/f/f, ..., WORM/f/f`
+
+> This is done now with a 2-level hash. It also hashes .git-annex/ log
+> files which were the worse problem really. Scales to hundreds of millions
+> of files with each dir having 1024 or fewer contents. Example:
+>
+> `me -> .git/annex/objects/71/9t/WORM-s3-m1300247299--me/WORM-s3-m1300247299--me`
+>
+> --[[Joey]]
diff --git a/doc/forum/hashing_objects_directories/comment_1_c55c56076be4f54251b0b7f79f28a607._comment b/doc/forum/hashing_objects_directories/comment_1_c55c56076be4f54251b0b7f79f28a607._comment
new file mode 100644
index 0000000000..3a19310b63
--- /dev/null
+++ b/doc/forum/hashing_objects_directories/comment_1_c55c56076be4f54251b0b7f79f28a607._comment
@@ -0,0 +1,12 @@
+[[!comment format=mdwn
+ username="http://joey.kitenet.net/"
+ nickname="joey"
+ subject="comment 1"
+ date="2011-03-14T16:12:49Z"
+ content="""
+My experience is that modern filesystems are not going to have many issues with tens to hundreds of thousands of items in the directory. However, if a transition does happen for FAT support I will consider adding hashing. Although getting a good balanced hash in general without, say, checksumming the filename and taking part of the checksum, is difficult. 
+
+I prefer to keep all the metadata in the filename, as this eases recovery if the files end up in lost+found. So while \"SHA/\" is a nice workaround for the FAT colon problem, I'll be doing something else. (What I'm not sure yet.)
+
+There is no point in creating unused hash directories on initialization. If anything, with a bad filesystem that just guarantees worst performance from the beginning..
+"""]]
diff --git a/doc/forum/hashing_objects_directories/comment_2_504c96959c779176f991f4125ea22009._comment b/doc/forum/hashing_objects_directories/comment_2_504c96959c779176f991f4125ea22009._comment
new file mode 100644
index 0000000000..64f1e16b50
--- /dev/null
+++ b/doc/forum/hashing_objects_directories/comment_2_504c96959c779176f991f4125ea22009._comment
@@ -0,0 +1,14 @@
+[[!comment format=mdwn
+ username="https://www.google.com/accounts/o8/id?id=AItOawl9sYlePmv1xK-VvjBdN-5doOa_Xw-jH4U"
+ nickname="Richard"
+ subject="comment 2"
+ date="2011-03-15T13:52:16Z"
+ content="""
+Can't you just use an underscore instead of a colon?
+
+Would it be feasible to split directories dynamically? I.e. start with SHA1_123456789abcdef0123456789abcdef012345678/SHA1_123456789abcdef0123456789abcdef012345678 and, at a certain cut-off point, switch to shorter directory names? This could even be done per subdirectory and based purely on a locally-configured number. Different annexes on different file systems or with different file subsets might even have different thresholds. This would ensure scale while not forcing you to segment from the start. Also, while segmenting with longer directory names means a flatter tree, segments longer than four characters might not make too much sense. Segmenting too often could lead to some directories becoming too populated, bringing us back to the dynamic segmentation.
+
+All of the above would make merging annexes by hand a _lot_ harder, but I don't know if this is a valid use case. And if all else fails, one could merge everything with the unsegemented directory names and start again from there.
+
+-- RichiH
+"""]]
diff --git a/doc/forum/hashing_objects_directories/comment_3_9134bde0a13aac0b6a4e5ebabd7f22e8._comment b/doc/forum/hashing_objects_directories/comment_3_9134bde0a13aac0b6a4e5ebabd7f22e8._comment
new file mode 100644
index 0000000000..51deb2f959
--- /dev/null
+++ b/doc/forum/hashing_objects_directories/comment_3_9134bde0a13aac0b6a4e5ebabd7f22e8._comment
@@ -0,0 +1,12 @@
+[[!comment format=mdwn
+ username="http://joey.kitenet.net/"
+ nickname="joey"
+ subject="comment 3"
+ date="2011-03-16T03:13:39Z"
+ content="""
+It is unfortunatly not possible to do system-dependant hashing, so long as git-annex stores symlinks to the content in git.
+
+It might be possible to start without hashing, and add hashing for new files after a cutoff point. It would add complexity.
+
+I'm currently looking at a 2 character hash directory segment, based on an md5sum of the key, which splits it into 1024 buckets. git uses just 256 buckets for its object directory, but then its objects tend to get packed away. I sorta hope that one level is enough, but guess I could go to 2 levels (objects/ab/cd/key), which would provide 1048576 buckets, probably plenty, as if you are storing more than a million files, you are probably using a modern enough system to have a filesystem that doesn't need hashing.
+"""]]
diff --git a/doc/forum/hashing_objects_directories/comment_4_0de9170e429cbfea66f5afa8980d45ac._comment b/doc/forum/hashing_objects_directories/comment_4_0de9170e429cbfea66f5afa8980d45ac._comment
new file mode 100644
index 0000000000..b29eea1b2b
--- /dev/null
+++ b/doc/forum/hashing_objects_directories/comment_4_0de9170e429cbfea66f5afa8980d45ac._comment
@@ -0,0 +1,12 @@
+[[!comment format=mdwn
+ username="http://joey.kitenet.net/"
+ nickname="joey"
+ subject="comment 4"
+ date="2011-03-16T04:06:19Z"
+ content="""
+The .git-annex/ directory is what really needs hashing.
+
+Consider that when git looks for changes in there, it has to scan every file in the directory. With hashing, it should be able to more quickly identify just the subdirectories that contained changed files, by the directory mtimes.
+
+And the real kicker is that when committing there, git has to create a tree object containing every single file, even if only 1 file changed. That will be a lot of extra work; with hashed subdirs it will instead create just 2 or 3 small tree objects leading down to the changed file. (Probably these trees both pack down to similar size pack files, not sure.)
+"""]]
diff --git a/doc/forum/hashing_objects_directories/comment_5_ef6cfd49d24c180c2d0a062e5bd3a0be._comment b/doc/forum/hashing_objects_directories/comment_5_ef6cfd49d24c180c2d0a062e5bd3a0be._comment
new file mode 100644
index 0000000000..c558ee65ee
--- /dev/null
+++ b/doc/forum/hashing_objects_directories/comment_5_ef6cfd49d24c180c2d0a062e5bd3a0be._comment
@@ -0,0 +1,12 @@
+[[!comment format=mdwn
+ username="https://www.google.com/accounts/o8/id?id=AItOawl9sYlePmv1xK-VvjBdN-5doOa_Xw-jH4U"
+ nickname="Richard"
+ subject="comment 5"
+ date="2011-03-16T15:47:17Z"
+ content="""
+If you can't segment the names retroactively, it's better to start with segmenting, imo.
+
+As subdirectories are cheap, going with ab/cd/rest or even ab/cd/ef/rest by default wouldn't hurt.
+
+Your point about git not needing to create as many tree objects is a kicker indeed. If I were you, I would default to segmentation.
+"""]]
diff --git a/doc/forum/help__44___a_bunch_of_files_disappeared.mdwn b/doc/forum/help__44___a_bunch_of_files_disappeared.mdwn
new file mode 100644
index 0000000000..b3479e9030
--- /dev/null
+++ b/doc/forum/help__44___a_bunch_of_files_disappeared.mdwn
@@ -0,0 +1,15 @@
+I have a large repo full of movies (dvd and blue-ray rips). I host it on my ubuntu linux 16.04 xenial system which sits next to my tv.
+
+I use git annex sync to copy stuff to my mac laptop running high sierra and I just updated to the brew version of git-annex there, 6.20171109
+
+I also have a copy on my really old laptop running mac os x lion, that one has a really really old version running.
+
+I hadn't done a git annex sync in a long time. so I updated via brew on my high sierra laptop to 6.20171109 and ran git annex sync on the laptop, and then on my server.
+
+when I ran git annex sync on my server, i saw it modify the 6 or so movies which I had copied over to the old lion laptop long ago. those six movies are now pointed to hashes which don't exist, and when I look at the history, the previous hashes don't exist either.
+
+then I ran a git annex fsck on the server, and it listed tons of movies which is says have no known copies. at some point, all of them were on the server. I suspect, but don't know how to confirm, that all the movies are still sitting in the annex on the server, but many many of the links are wrong.
+
+how can I fix the links?
+
+thanks.
diff --git a/doc/forum/help__44___a_bunch_of_files_disappeared/comment_1_56cffd43ad7056a34c23b139f95ea58e._comment b/doc/forum/help__44___a_bunch_of_files_disappeared/comment_1_56cffd43ad7056a34c23b139f95ea58e._comment
new file mode 100644
index 0000000000..53fea0012c
--- /dev/null
+++ b/doc/forum/help__44___a_bunch_of_files_disappeared/comment_1_56cffd43ad7056a34c23b139f95ea58e._comment
@@ -0,0 +1,15 @@
+[[!comment format=mdwn
+ username="astrophoenix"
+ avatar="http://cdn.libravatar.org/avatar/cb0b7982b6877c58c3860d8ad5fb3148"
+ subject="possible cause (my error)"
+ date="2017-11-12T21:01:28Z"
+ content="""
+oh, it might have been something dumb i did:
+
+at some point I wanted to delete a few files, so trying to be cute I did this:
+
+m=OneMovie.m4v git annex drop --force $m; git rm $m; git ci -m \"rm $m\"
+
+I noted that didn't work, but didn't realize it might've taken out most/all the files in the folder
+
+"""]]
diff --git a/doc/forum/help__44___a_bunch_of_files_disappeared/comment_2_7675ff78cf1b5f58618c5c57da8591b5._comment b/doc/forum/help__44___a_bunch_of_files_disappeared/comment_2_7675ff78cf1b5f58618c5c57da8591b5._comment
new file mode 100644
index 0000000000..a99aa7bb90
--- /dev/null
+++ b/doc/forum/help__44___a_bunch_of_files_disappeared/comment_2_7675ff78cf1b5f58618c5c57da8591b5._comment
@@ -0,0 +1,23 @@
+[[!comment format=mdwn
+ username="joey"
+ subject="""comment 2"""
+ date="2017-11-14T17:16:54Z"
+ content="""
+There are two possible things that could have happened.
+
+1. Maybe something somehow wrong got committed to the git repository
+   for the missing files. 
+   
+   If so, you can *always* use git to check out
+   the older version before that commit, and will then be able to see
+   the files; git-annex will know where the content of them is, etc.
+   You can also use `git revert` to revert such a bad commit.
+
+2. Maybe you accidentially dropped the content of the files with --force,
+   and if so, you may have lost the content.
+
+   If you run `git annex log` on one of the files, and look at the lines
+   starting with "-", you can see when the file content was removed from
+   repositories. But there's no way to get it back unless you have another
+   copy somewhere.
+"""]]
diff --git a/doc/forum/help__44___a_bunch_of_files_disappeared/comment_3_4b700ccba2b40fdc1d172a3dcbc441e5._comment b/doc/forum/help__44___a_bunch_of_files_disappeared/comment_3_4b700ccba2b40fdc1d172a3dcbc441e5._comment
new file mode 100644
index 0000000000..c5b402ec94
--- /dev/null
+++ b/doc/forum/help__44___a_bunch_of_files_disappeared/comment_3_4b700ccba2b40fdc1d172a3dcbc441e5._comment
@@ -0,0 +1,10 @@
+[[!comment format=mdwn
+ username="astrophoenix"
+ avatar="http://cdn.libravatar.org/avatar/cb0b7982b6877c58c3860d8ad5fb3148"
+ subject="comment 3"
+ date="2017-11-14T19:45:41Z"
+ content="""
+I'm nearly 100% certain I did a drop --force by mistake. I'm pulling the files from backups. 
+
+thank you!
+"""]]
diff --git a/doc/forum/help_running_git-annex_on_top_of_existing_repo.mdwn b/doc/forum/help_running_git-annex_on_top_of_existing_repo.mdwn
new file mode 100644
index 0000000000..365ad71db7
--- /dev/null
+++ b/doc/forum/help_running_git-annex_on_top_of_existing_repo.mdwn
@@ -0,0 +1,12 @@
+## Question
+Can git-annex run passively with an existing repo?
+
+## Scenario
+We have an existing web application with large binary assets spread throughout modules in the repo. The project is in constant development with weekly deploys to production and there are many developers working on the project.
+
+## Goal
+We need to maintain the directory structure for these assets without actually committing large binaries to the main remote (hosted on GitHub). I need a solution that has a low barrier of entry when on-boarding new developers. I was thinking maybe a script the is executed on every commit that would filter binaries files based on extensions and commit them to the git-annex remote instead. We are flexible on the type of remote storage type (SSH, S3, etc)
+
+## Notes
+I have gone through the last couple months of forum posts and done a bit of Googling but have come up empty. If anyone can point me in the right direction that would be great. 
+Thanks!
diff --git a/doc/forum/help_running_git-annex_on_top_of_existing_repo/comment_1_4cb38d71c943657c5ba0896cd70d2e64._comment b/doc/forum/help_running_git-annex_on_top_of_existing_repo/comment_1_4cb38d71c943657c5ba0896cd70d2e64._comment
new file mode 100644
index 0000000000..b98ad63232
--- /dev/null
+++ b/doc/forum/help_running_git-annex_on_top_of_existing_repo/comment_1_4cb38d71c943657c5ba0896cd70d2e64._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="https://www.google.com/accounts/o8/id?id=AItOawn7gQ1zZDdWhXy9H51W2krZYShNmKL3qfM"
+ nickname="Karsten"
+ subject="comment 1"
+ date="2013-06-01T09:21:54Z"
+ content="""
+Mixing files controlled by git directly and files controlled by git annex is possible, simply 'git annex add' the large files and 'git add' the rest. You will not be able to use direct mode and you should not run the git annex assisstant, as it will add all files to the annex. 
+"""]]
diff --git a/doc/forum/help_running_git-annex_on_top_of_existing_repo/comment_2_b5e94c10ebbed9125c7e2332f75709ca._comment b/doc/forum/help_running_git-annex_on_top_of_existing_repo/comment_2_b5e94c10ebbed9125c7e2332f75709ca._comment
new file mode 100644
index 0000000000..ba321df6bc
--- /dev/null
+++ b/doc/forum/help_running_git-annex_on_top_of_existing_repo/comment_2_b5e94c10ebbed9125c7e2332f75709ca._comment
@@ -0,0 +1,13 @@
+[[!comment format=mdwn
+ username="https://www.google.com/accounts/o8/id?id=AItOawl9FMhhhM2sJ68Zjx_RmWd8cTdpI-mrkbE"
+ nickname="Hans"
+ subject="what about with git-svn?"
+ date="2013-06-04T08:00:15Z"
+ content="""
+I'm trying to wrap my head around a similar situation.
+I've tested this by git cloning my repo; the symlinks are copied, and end up broken because the annex directory under .git doesn't exist in the new repo.
+
+So to be specific: can I conclude that when I use git to copy my repo, as long as I don't explicitly use git-annex in the process, I end up with a 'bare' git repo and I don't have to worry about my annexed files coming along?
+
+Also: anyone know of anything that would be different about using git-svn? (i.e. git svn dcommit to push my changes to an svn repo)?
+"""]]
diff --git a/doc/forum/help_running_git-annex_on_top_of_existing_repo/comment_3_2b3b93bbc60fbc24d436231954d6822a._comment b/doc/forum/help_running_git-annex_on_top_of_existing_repo/comment_3_2b3b93bbc60fbc24d436231954d6822a._comment
new file mode 100644
index 0000000000..8d2fd0dff1
--- /dev/null
+++ b/doc/forum/help_running_git-annex_on_top_of_existing_repo/comment_3_2b3b93bbc60fbc24d436231954d6822a._comment
@@ -0,0 +1,10 @@
+[[!comment format=mdwn
+ username="http://edheil.wordpress.com/"
+ ip="173.162.44.162"
+ subject="comment 3"
+ date="2013-06-04T12:29:04Z"
+ content="""
+As long as you're not using Direct Mode, then a git-annex repo is just an absolutely ordinary git repo, which happens to contain some symlinks that point to .git/annex/objects/*
+
+
+"""]]
diff --git a/doc/forum/help_running_git-annex_on_top_of_existing_repo/comment_4_2dfda33ffa39b92b16c8bd9005e1cefe._comment b/doc/forum/help_running_git-annex_on_top_of_existing_repo/comment_4_2dfda33ffa39b92b16c8bd9005e1cefe._comment
new file mode 100644
index 0000000000..e9f22adfd4
--- /dev/null
+++ b/doc/forum/help_running_git-annex_on_top_of_existing_repo/comment_4_2dfda33ffa39b92b16c8bd9005e1cefe._comment
@@ -0,0 +1,21 @@
+[[!comment format=mdwn
+ username="https://www.google.com/accounts/o8/id?id=AItOawl9J51AO9t75xN5k0sJgg8taUo4y0a4hpQ"
+ nickname="Daniel"
+ subject="comment 4"
+ date="2013-06-07T23:02:29Z"
+ content="""
+I'm trying to find a way to prevent developers from adding certain file types to the main repo. Is there something like a pre-add hook that could be used?
+
+I found this in another forum but I'm not sure how it was intended to be implemented.
+
+```
+git check-attr addtoannex \"$FILE\" | grep -q \": set$\"  
+if [ $? -eq 0 ]; then  
+    git annex add \"$FILE\"  
+else  
+    git add \"$FILE\"  
+fi  
+```
+
+[http://git-annex.branchable.com/forum/Let_watch_selectively_annex_files/](http://git-annex.branchable.com/forum/Let_watch_selectively_annex_files/)
+"""]]
diff --git a/doc/forum/help_running_git-annex_on_top_of_existing_repo/comment_5_96b1eb1e8e9f315c646f4686870f9b52._comment b/doc/forum/help_running_git-annex_on_top_of_existing_repo/comment_5_96b1eb1e8e9f315c646f4686870f9b52._comment
new file mode 100644
index 0000000000..6435fc2528
--- /dev/null
+++ b/doc/forum/help_running_git-annex_on_top_of_existing_repo/comment_5_96b1eb1e8e9f315c646f4686870f9b52._comment
@@ -0,0 +1,10 @@
+[[!comment format=mdwn
+ username="http://joeyh.name/"
+ nickname="joey"
+ subject="comment 5"
+ date="2013-06-11T14:30:08Z"
+ content="""
+I think the most useful thing for people in this thread to know about is [[tips/replacing_Sparkleshare_or_dvcs-autosync_with_the_assistant]].
+
+This doesn't solve it at the command line, where you still need to choose between git add and git annex add, but you can use git annex watch to automatically commit small files to git, and large files to the git annex.
+"""]]
diff --git a/doc/forum/help_running_git-annex_on_top_of_existing_repo/comment_6_e85c3fa1d17f1d6ec625b9c4f9b698c3._comment b/doc/forum/help_running_git-annex_on_top_of_existing_repo/comment_6_e85c3fa1d17f1d6ec625b9c4f9b698c3._comment
new file mode 100644
index 0000000000..6220a44166
--- /dev/null
+++ b/doc/forum/help_running_git-annex_on_top_of_existing_repo/comment_6_e85c3fa1d17f1d6ec625b9c4f9b698c3._comment
@@ -0,0 +1,47 @@
+[[!comment format=mdwn
+ username="https://www.google.com/accounts/o8/id?id=AItOawl9J51AO9t75xN5k0sJgg8taUo4y0a4hpQ"
+ nickname="Daniel"
+ subject="comment 6"
+ date="2013-06-11T22:28:52Z"
+ content="""
+This is what I ended up doing.  
+[https://gist.github.com/ifnull/5761255](https://gist.github.com/ifnull/5761255)
+
+Basically you just add the extensions of the files you want to exclude to .gitignore_large_binaries and run \"git a .\" instead of \"git add .\"
+
+    #######################
+    # Setup
+    #######################
+    mkdir annex-test
+    cd annex-test
+    git init
+    git annex init master
+
+    #######################
+    # Fab setup task
+    #######################
+    git config --local core.excludesfile ./.gitignore_large_binaries
+    git config --local alias.a '! sh ./git-add.sh'
+
+    #######################
+    # git a (git-add.sh)
+    #######################
+
+    # Generate annex include arg from .gitignore_large_binaries
+    include_str=\"--include='.lazy'\";
+
+    while read line
+    do
+        if [[ \"$line\" != *\"#\"* ]] && [[ \"$line\" != \"\" ]]; then 
+            include_str=\"$include_str --or --include=${line}\";
+        fi
+    done < \"./.gitignore_large_binaries\"
+
+    # git annex add
+    git config --local core.excludesfile ./.gitignore;
+    git annex add $1 $include_str;
+
+    # git add
+    git config --local core.excludesfile ./.gitignore_large_binaries;
+    git add $1
+"""]]
diff --git a/doc/forum/help_running_git-annex_on_top_of_existing_repo/comment_7_15d918ededb5b8375b0ca13d0b3523ff._comment b/doc/forum/help_running_git-annex_on_top_of_existing_repo/comment_7_15d918ededb5b8375b0ca13d0b3523ff._comment
new file mode 100644
index 0000000000..1ca6b80ad5
--- /dev/null
+++ b/doc/forum/help_running_git-annex_on_top_of_existing_repo/comment_7_15d918ededb5b8375b0ca13d0b3523ff._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="https://www.google.com/accounts/o8/id?id=AItOawlog_5wIICaMcrKTexlFNA6IO6UTp323aE"
+ nickname="Torkaly"
+ subject="comment 7"
+ date="2014-10-17T11:00:06Z"
+ content="""
+Is there a way just to sync *git-annex* branch with the *git annex sync* command? As we have an already existing git branch and want to push/pull *master* manually.
+"""]]
diff --git a/doc/forum/help_running_git-annex_on_top_of_existing_repo/comment_8_dcc3f2c6d55006776610e8d770b61d12._comment b/doc/forum/help_running_git-annex_on_top_of_existing_repo/comment_8_dcc3f2c6d55006776610e8d770b61d12._comment
new file mode 100644
index 0000000000..35983e802f
--- /dev/null
+++ b/doc/forum/help_running_git-annex_on_top_of_existing_repo/comment_8_dcc3f2c6d55006776610e8d770b61d12._comment
@@ -0,0 +1,9 @@
+[[!comment format=mdwn
+ username="joey"
+ subject="""comment 8"""
+ date="2014-10-20T15:25:13Z"
+ content="""
+@Torkaly just `git pull` as usual, and then run `git annex merge`
+to auto-merge the git-anne branches. Then `git push origin git-annex`
+to push the git-annex branch.
+"""]]
diff --git a/doc/forum/how_do_automated_upgrades_work__63__.mdwn b/doc/forum/how_do_automated_upgrades_work__63__.mdwn
new file mode 100644
index 0000000000..1f404d1449
--- /dev/null
+++ b/doc/forum/how_do_automated_upgrades_work__63__.mdwn
@@ -0,0 +1,11 @@
+When i start the assistant, it's nicely telling me:
+
+
    +[2015-05-27 20:15:20 UTC] Upgrader: An upgrade of git-annex is available.  (version 5.20150522)
+
    
+
+That's really cool, but it's not actually upgrading. I looked around the website to understand how that worked and i found [[git-annex-upgrade]] and [[upgrades]] but those pages were not really useful, as they talk more about repository upgrades, and not the automated upgrade system. I was expecting [[upgrades]] to talk a bit about automated upgrades, or maybe the [[install]] page... 
+
+i am running `5.20150508-g883d57f`, with a standalone image installed by root in `/opt`. Should that directory be writable by the user running git-annex to solve this?
+
+Thanks! --[[anarcat]]
diff --git a/doc/forum/how_do_automated_upgrades_work__63__/comment_1_cfd00ac3a3eaa3c2a1d430f6544c32f0._comment b/doc/forum/how_do_automated_upgrades_work__63__/comment_1_cfd00ac3a3eaa3c2a1d430f6544c32f0._comment
new file mode 100644
index 0000000000..7269a2a198
--- /dev/null
+++ b/doc/forum/how_do_automated_upgrades_work__63__/comment_1_cfd00ac3a3eaa3c2a1d430f6544c32f0._comment
@@ -0,0 +1,15 @@
+[[!comment format=mdwn
+ username="joey"
+ subject="""comment 1"""
+ date="2015-06-02T18:57:43Z"
+ content="""
+The assistant checks for available upgrades a few times a day.
+By default, it only pops up a message in the webapp with an upgrade button.
+
+annex.autoupgrade can configure this, so the assistant can be made
+to upgrade without confirmation if you want.
+
+The upgrade process needs to move the git-annex.linux directory out of the
+way and replace it with a new one from the tarball, so it will only
+work if the user running the assistant can write to it.
+"""]]
diff --git a/doc/forum/how_do_automated_upgrades_work__63__/comment_2_5ef98e46c514e9b794ee719e4cc8fda4._comment b/doc/forum/how_do_automated_upgrades_work__63__/comment_2_5ef98e46c514e9b794ee719e4cc8fda4._comment
new file mode 100644
index 0000000000..76401a89ea
--- /dev/null
+++ b/doc/forum/how_do_automated_upgrades_work__63__/comment_2_5ef98e46c514e9b794ee719e4cc8fda4._comment
@@ -0,0 +1,7 @@
+[[!comment format=mdwn
+ username="https://id.koumbit.net/anarcat"
+ subject="understood"
+ date="2015-06-03T14:25:39Z"
+ content="""
+alright, i have added this to the [[upgrades]] page, i hope that's okay.
+"""]]
diff --git a/doc/forum/how_do_i_manually_sync_my_external_drive__63__.mdwn b/doc/forum/how_do_i_manually_sync_my_external_drive__63__.mdwn
new file mode 100644
index 0000000000..a5685761d6
--- /dev/null
+++ b/doc/forum/how_do_i_manually_sync_my_external_drive__63__.mdwn
@@ -0,0 +1,9 @@
+I am working on rehauling my backup scripts here, and I was originally doing an rsync of everything to an external drive, but now I think I can be smarted and skip my annexes in that rsync, and use git-annex superpowers to do the sync instead.
+
+I have added this to my backup script:
+
+    ( cd /srv/video && git annex copy --to backup . )
+
+And it works, so yaaay. :) However, I feel it could be faster. This seems to check each file one at a time, but doesn't git-annex keep a state of the remote internally, which would allow it to copy over only the missing files, without checking if each file is present individually?
+
+I mean, it's still pretty fast considering the dataset, but I wonder if there isn't some fast/easier way. Keep in mind I am hesitant of using the assistant for this because I am [[confused_about_external_drives]], the [[bugs/webapp takes 100% of the cpu]] and the [[bugs/assistant eats all CPU]]. I would prefer to script this anyways. --[[anarcat]]
diff --git a/doc/forum/how_do_i_manually_sync_my_external_drive__63__/comment_1_4fd8722cafd55b0503c802289206645a._comment b/doc/forum/how_do_i_manually_sync_my_external_drive__63__/comment_1_4fd8722cafd55b0503c802289206645a._comment
new file mode 100644
index 0000000000..290221541f
--- /dev/null
+++ b/doc/forum/how_do_i_manually_sync_my_external_drive__63__/comment_1_4fd8722cafd55b0503c802289206645a._comment
@@ -0,0 +1,10 @@
+[[!comment format=mdwn
+ username="http://joeyh.name/"
+ ip="209.250.56.227"
+ subject="comment 1"
+ date="2014-01-01T20:43:05Z"
+ content="""
+Pulling location tracking data out of git is unlikely to be faster than statting a single file on the drive, unless the drive is dead slow.
+
+If you really want to force it to use the location tracking information, use `--not --in backup`
+"""]]
diff --git a/doc/forum/how_long_are_git_annex_commands_supposed_to_take__63__.mdwn b/doc/forum/how_long_are_git_annex_commands_supposed_to_take__63__.mdwn
new file mode 100644
index 0000000000..67ba99b986
--- /dev/null
+++ b/doc/forum/how_long_are_git_annex_commands_supposed_to_take__63__.mdwn
@@ -0,0 +1,9 @@
+I have a git annex repository that contains only photos and videos. I am using an NTFS partition on Linux (because dual-boot), and recently did a `git annex upgrade` to version 6. (But the question below is general, and not about version 6). The size of my repo must be around 80G.
+
+When I run commands like status and sync, nothing happens for a really long time. I finished a `git annex fsck` today morning after the version upgrade, and so wanted to see what the repo now looks like:
+
+    git annex status --debug
+
+This is stuck at internally calling `git status -uall -z` with some other options. The process is stuck here for almost an hour, and I finally gave up and cancelled it. Like I said, this is not about the recent upgrade that I finished. I have previously seen the process succeed after one or two hours.
+
+Is it normal for `status` to take this long? Or is there something wrong with my repo? For example, maybe a large chunk of my files are checked into git without being annexed? My repo has a long history of making mistakes with git annex, so this is actually possible.
diff --git a/doc/forum/how_long_are_git_annex_commands_supposed_to_take__63__/comment_1_26d959bb3cfdc690b1eab41c9b0b5dc4._comment b/doc/forum/how_long_are_git_annex_commands_supposed_to_take__63__/comment_1_26d959bb3cfdc690b1eab41c9b0b5dc4._comment
new file mode 100644
index 0000000000..89cc134bde
--- /dev/null
+++ b/doc/forum/how_long_are_git_annex_commands_supposed_to_take__63__/comment_1_26d959bb3cfdc690b1eab41c9b0b5dc4._comment
@@ -0,0 +1,31 @@
+[[!comment format=mdwn
+ username="joey"
+ subject="""comment 1"""
+ date="2017-01-30T17:35:27Z"
+ content="""
+Generally git-annex takes longer the more files in the repository it needs
+to deal with. If a repository gets a great many files (typically 
+hundreds of thousands to millions), various inneficiencies in git annex
+git-annex will slow things down enough that it gets annoying.
+Splitting the files into different branches 
+(or separate repositories) is a common way to deal with that.
+
+Also, running on a spinning disk tends to be a lot slower than a SSD.
+
+Just for comparison, `git annex status` in a repository with 75000 files
+takes 0.5 seconds on my laptop's SSD. `git status` takes 0.2 seconds.
+
+In your particular case, the NTFS partition and/or v6
+mode seems likely to be the reason for slowdowns. Both git and git-annex
+record the inode numbers used for files in the repository. Those numbers
+are supposed to be stable, but mounting a filesystem on windows and then
+linux will make the inode numbers change. (Even remounting a FAT
+partition on linux will change the inodes, although that doesn't seem
+to happen for NTFS in a quick test).
+
+When the inodes have changed, much slower code paths get activated, since
+git and git-annex have to then assume the contents of the files may have
+changed since the last time they saw them. In a v6 repository where this
+has happened, `git status` is quite likely running `git-annex smudge`
+once per file in the working tree, which is quite slow.
+"""]]
diff --git a/doc/forum/how_to_commit_removed_files_as_repo-droped_entries.mdwn b/doc/forum/how_to_commit_removed_files_as_repo-droped_entries.mdwn
new file mode 100644
index 0000000000..30250b12e8
--- /dev/null
+++ b/doc/forum/how_to_commit_removed_files_as_repo-droped_entries.mdwn
@@ -0,0 +1,9 @@
+I have following usecase:
+
+I want to use a central repos for xbmc/kodi where I can play and delete files from kodi somethimes also with a random file manager. But I normaly if a file gets deleted and synced with git annex sync it deletes the file "head" in every repository.
+
+I would like if its only a git annex drop of the content, if thats was the last copy ok then its ok for me that its gone. But if its not the last copy it shhould just delete it from this repo but not delete the heads of the other repositories.
+
+I know that the actual file is still in the other repositories but the entry is gone, I would love if it would be more like a git annex drop instead of a git rm.
+
+Can I do that with setting this repos readonly or is my usecase not supported/doable with git annex?
diff --git a/doc/forum/how_to_commit_removed_files_as_repo-droped_entries/comment_1_a9a4baebfb647898a76ee3d5ab6febfd._comment b/doc/forum/how_to_commit_removed_files_as_repo-droped_entries/comment_1_a9a4baebfb647898a76ee3d5ab6febfd._comment
new file mode 100644
index 0000000000..0312ac8db8
--- /dev/null
+++ b/doc/forum/how_to_commit_removed_files_as_repo-droped_entries/comment_1_a9a4baebfb647898a76ee3d5ab6febfd._comment
@@ -0,0 +1,13 @@
+[[!comment format=mdwn
+ username="joey"
+ subject="""comment 1"""
+ date="2015-02-09T17:15:19Z"
+ content="""
+One way to do this is to switch to a different branch. Then, deletion
+of a file won't propigate to other repositories, that don't have that
+same branch checked out.
+
+Or, see 
+-- file managers can be taught to run `git annex drop` rather than deleting
+files. Perhaps xbmc/kodi could be too..
+"""]]
diff --git a/doc/forum/how_to_commit_removed_files_as_repo-droped_entries/comment_2_68392de15bae234de1332b8bc80cc2ff._comment b/doc/forum/how_to_commit_removed_files_as_repo-droped_entries/comment_2_68392de15bae234de1332b8bc80cc2ff._comment
new file mode 100644
index 0000000000..829402e583
--- /dev/null
+++ b/doc/forum/how_to_commit_removed_files_as_repo-droped_entries/comment_2_68392de15bae234de1332b8bc80cc2ff._comment
@@ -0,0 +1,26 @@
+[[!comment format=mdwn
+ username="https://www.google.com/accounts/o8/id?id=AItOawn4bbuawnh-nSo9pAh8irYAcV4MQCcfdHo"
+ nickname="Stefan"
+ subject="a workaround"
+ date="2015-02-10T05:00:01Z"
+ content="""
+Hacked together a small line that would convert my deletion to a annex drop, would be needed to be used in a cronjob or something like that:
+
+git st -s | grep \"^ D .*\" |  tee >(git checkout -- `grep -oE \"\S*$\"`) >(git annex drop `grep -oE \"\S*$\"`) 
+
+I guess it works only in indirect mode, maybe a git annex fsck before it would be good? (adding --force to the drop command would be good too I think)
+
+So either I rethink my usecase, maybe its ok that its a deletion or use something like that. I am not shure yet.
+I find it somethimes a bit hard to track unused files down and delete em (over several remotes).
+
+So I like the direct mode, and some programs still have problems using that files, as example du dont give you real size except you use -D of files, and even nautilus supports basicly dropping files, you cant drop on other repository with it.
+
+maybe I should also look how to easily remote git annex from some trees then I would be more willing to just play with that files around and go back to normal more painless, or I start with a smaller directory.
+
+the real fun I guess happens if you automate things like cloning of some paths to another remote as soon as its available I guess. I also dont like the interfaces to annex to much, 
+mv /mnt/share/file_x /path
+ is just simpler than 
+git annex move file_x --from=kodi --to=local-repos
+
+but thats maybe just practise :)
+"""]]
diff --git a/doc/forum/how_to_decrypt_file_from_encrypted_special_remote__63__.mdwn b/doc/forum/how_to_decrypt_file_from_encrypted_special_remote__63__.mdwn
new file mode 100644
index 0000000000..d16131eeab
--- /dev/null
+++ b/doc/forum/how_to_decrypt_file_from_encrypted_special_remote__63__.mdwn
@@ -0,0 +1,14 @@
+I have an encrypted S3 remote with which I recently ran: 
+
+$> git annex move . --from cloud
+
+which moved all my files to the local repo. When I looked at my S3 bucket, I saw some files left. I then ran:
+
+$> git annex unused --from cloud
+
+to get all the unused data. I dropped each unused data from cloud so that now I think git annex believes it has nothing on the cloud remote.
+
+However there is still one file left on the bucket. Is this normal? (if not I'll create a bug report)
+More importantly, how can I decrypt this file so I can run git log -S'KEY' to figure out what it is (I want to make sure I haven't lost any data).
+
+To be clear, when I currently run 'git annex move . --from cloud' it moves nothing and when I run 'git annex unused --from cloud' it reports no unused data.
diff --git a/doc/forum/how_to_decrypt_file_from_encrypted_special_remote__63__/comment_1_d4dc451892e7a6e230bf32adb7f3f9fa._comment b/doc/forum/how_to_decrypt_file_from_encrypted_special_remote__63__/comment_1_d4dc451892e7a6e230bf32adb7f3f9fa._comment
new file mode 100644
index 0000000000..c1eccfb0ff
--- /dev/null
+++ b/doc/forum/how_to_decrypt_file_from_encrypted_special_remote__63__/comment_1_d4dc451892e7a6e230bf32adb7f3f9fa._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="https://www.google.com/accounts/o8/id?id=AItOawkiAsTXFXZbLE8iyy6yDtvz4MPFbzsk3c0"
+ nickname="Tony"
+ subject="comment 1"
+ date="2013-07-27T22:14:11Z"
+ content="""
+one more thing, I forgot to mention I use gpg keys for encryption on the remote and not a shared key.
+"""]]
diff --git a/doc/forum/how_to_decrypt_file_from_encrypted_special_remote__63__/comment_2_79340bf3c0691073a9808c5ac2da0a3d._comment b/doc/forum/how_to_decrypt_file_from_encrypted_special_remote__63__/comment_2_79340bf3c0691073a9808c5ac2da0a3d._comment
new file mode 100644
index 0000000000..0cea29414e
--- /dev/null
+++ b/doc/forum/how_to_decrypt_file_from_encrypted_special_remote__63__/comment_2_79340bf3c0691073a9808c5ac2da0a3d._comment
@@ -0,0 +1,14 @@
+[[!comment format=mdwn
+ username="http://joeyh.name/"
+ ip="4.152.246.110"
+ subject="comment 2"
+ date="2013-07-27T22:30:50Z"
+ content="""
+I suppose this could happen if you have a tag or a branch or another repository that still refers to the file that's stored in S3. It would then not be found by `unused`.
+
+git-annex 4.20130709 has a nice new --all switch you can use: `git annex move --all --from cloud`
+
+If that doesn't move the file, nothing in the entire history of the git repository refers to it. The question then would be how did that file get there.
+
+It's by [[design|design/encryption]] not possible to get from the name of an encrypted file back to the key, unless you already know all the possible keys that the file could be. It's possible to decrypt the content if you have the gpg key and the git repository, but this space is a bit too small to explain every step in doing so, and I have some nice code that explains how in detail. ;)
+"""]]
diff --git a/doc/forum/how_to_decrypt_file_from_encrypted_special_remote__63__/comment_3_6302fb6e5bb7cbddf2cfe74d98d32897._comment b/doc/forum/how_to_decrypt_file_from_encrypted_special_remote__63__/comment_3_6302fb6e5bb7cbddf2cfe74d98d32897._comment
new file mode 100644
index 0000000000..5c8d6356c8
--- /dev/null
+++ b/doc/forum/how_to_decrypt_file_from_encrypted_special_remote__63__/comment_3_6302fb6e5bb7cbddf2cfe74d98d32897._comment
@@ -0,0 +1,12 @@
+[[!comment format=mdwn
+ username="https://www.google.com/accounts/o8/id?id=AItOawkiAsTXFXZbLE8iyy6yDtvz4MPFbzsk3c0"
+ nickname="Tony"
+ subject="comment 3"
+ date="2013-07-27T22:47:14Z"
+ content="""
+thanks, running
+
+'git annex move --all --from cloud'
+
+grabbed the one remaining file from the bucket.
+"""]]
diff --git a/doc/forum/how_to_decrypt_file_from_encrypted_special_remote__63__/comment_4_e3d95bc09c9fb21e8e9bbacc642aa60f._comment b/doc/forum/how_to_decrypt_file_from_encrypted_special_remote__63__/comment_4_e3d95bc09c9fb21e8e9bbacc642aa60f._comment
new file mode 100644
index 0000000000..493fa3ba17
--- /dev/null
+++ b/doc/forum/how_to_decrypt_file_from_encrypted_special_remote__63__/comment_4_e3d95bc09c9fb21e8e9bbacc642aa60f._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="http://joeyh.name/"
+ ip="4.152.246.110"
+ subject="comment 4"
+ date="2013-07-27T22:52:00Z"
+ content="""
+Great! And it should have also printed out the key it downloaded, if you're curious about looking where that file is used in your git repository.
+"""]]
diff --git a/doc/forum/how_to_decrypt_file_from_encrypted_special_remote__63__/comment_5_f2f0a1c2fb0c6323707b11e2b06aa2db._comment b/doc/forum/how_to_decrypt_file_from_encrypted_special_remote__63__/comment_5_f2f0a1c2fb0c6323707b11e2b06aa2db._comment
new file mode 100644
index 0000000000..042704e529
--- /dev/null
+++ b/doc/forum/how_to_decrypt_file_from_encrypted_special_remote__63__/comment_5_f2f0a1c2fb0c6323707b11e2b06aa2db._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="https://www.google.com/accounts/o8/id?id=AItOawkiAsTXFXZbLE8iyy6yDtvz4MPFbzsk3c0"
+ nickname="Tony"
+ subject="comment 5"
+ date="2013-07-27T23:02:19Z"
+ content="""
+yeah it did print out the name of the file so I was able to verify that it was nothing I was worried about losing. thanks again :D
+"""]]
diff --git a/doc/forum/how_to_decrypt_file_from_encrypted_special_remote__63__/comment_6_66fe80e634a8f13cce18fe68974ec67a._comment b/doc/forum/how_to_decrypt_file_from_encrypted_special_remote__63__/comment_6_66fe80e634a8f13cce18fe68974ec67a._comment
new file mode 100644
index 0000000000..8e79b4dc28
--- /dev/null
+++ b/doc/forum/how_to_decrypt_file_from_encrypted_special_remote__63__/comment_6_66fe80e634a8f13cce18fe68974ec67a._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="http://joeyh.name/"
+ ip="4.152.246.110"
+ subject="comment 6"
+ date="2013-07-27T23:06:38Z"
+ content="""
+I'm curious, why didn't `git annex unused --from cloud` show the file? Was it a tag/branch/etc still referring to it?
+"""]]
diff --git a/doc/forum/how_to_disaster_recovery.mdwn b/doc/forum/how_to_disaster_recovery.mdwn
new file mode 100644
index 0000000000..b4bbe123d9
--- /dev/null
+++ b/doc/forum/how_to_disaster_recovery.mdwn
@@ -0,0 +1,6 @@
+OS X: 10.11.6
+Annex: git-annex-6.20160619 (via brew)
+
+I'm [still strugling](http://git-annex.branchable.com/forum/git_annex_wants_to_repair_every_time_it__39__s_running/) with my annex repo over here. My problem is that for some reason git annex wants to repair my repo all the time when I'm starting the assistant and this (seemingly) never finishes, so I'm caught in an infinite loop in which no files get uploaded to my various remotes and nothing is downloaded either leaving me with all the data on my hard drive and nothing else. I would like to move on. How can I recover from this state? The daemon.log file doesn't show any information about what is happening during the repair and what might cause the failure. Is there another log file I could look at?
+
+Any help is appreciated at that point. Where should I look for more information? What should I do to change the behaviour? I want to solve this once and for all.
diff --git a/doc/forum/how_to_disaster_recovery/comment_10_be3cf3736166855012e10c2d251043e9._comment b/doc/forum/how_to_disaster_recovery/comment_10_be3cf3736166855012e10c2d251043e9._comment
new file mode 100644
index 0000000000..0326e2aaa1
--- /dev/null
+++ b/doc/forum/how_to_disaster_recovery/comment_10_be3cf3736166855012e10c2d251043e9._comment
@@ -0,0 +1,13 @@
+[[!comment format=mdwn
+ username="joey"
+ subject="""comment 10"""
+ date="2016-10-26T18:14:30Z"
+ content="""
+The first try at making git-annex ignore the fsck lines about duplicate
+entries didn't quite work; the second try landed 8 days ago and it's not
+been in a release yet so that's probably why you continue to see the
+problem.
+
+I don't see how deleting a special remote could lead to this. But we know
+that `git annex adjust --unlock` did, for another user.
+"""]]
diff --git a/doc/forum/how_to_disaster_recovery/comment_11_7fa36912ae21a66d3174fec35356d7ba._comment b/doc/forum/how_to_disaster_recovery/comment_11_7fa36912ae21a66d3174fec35356d7ba._comment
new file mode 100644
index 0000000000..ce3fd739a7
--- /dev/null
+++ b/doc/forum/how_to_disaster_recovery/comment_11_7fa36912ae21a66d3174fec35356d7ba._comment
@@ -0,0 +1,7 @@
+[[!comment format=mdwn
+ username="openmedi"
+ subject="comment 11"
+ date="2016-11-10T12:53:34Z"
+ content="""
+Two days ago I updated my annex through brew and the constant repairing seems to have stopped. There is one problem where the assistant shows up without any design, maybe an error with the homebrew formula? I had the too many open files error, but this seems to have disappeared. I reactivated synching for my home repo and am now letting the assistant run for a few days to see how it will work.
+"""]]
diff --git a/doc/forum/how_to_disaster_recovery/comment_12_f2e570dc60a6f16e8f696d94e253775f._comment b/doc/forum/how_to_disaster_recovery/comment_12_f2e570dc60a6f16e8f696d94e253775f._comment
new file mode 100644
index 0000000000..7c7da0ef47
--- /dev/null
+++ b/doc/forum/how_to_disaster_recovery/comment_12_f2e570dc60a6f16e8f696d94e253775f._comment
@@ -0,0 +1,7 @@
+[[!comment format=mdwn
+ username="openmedi"
+ subject="comment 12"
+ date="2016-11-18T12:03:44Z"
+ content="""
+A recent update to annex via homebrew now reslolves the issue with the weird looking webapp.
+"""]]
diff --git a/doc/forum/how_to_disaster_recovery/comment_1_c8db90fd7af13b9605e152ea884b5241._comment b/doc/forum/how_to_disaster_recovery/comment_1_c8db90fd7af13b9605e152ea884b5241._comment
new file mode 100644
index 0000000000..4b14667d89
--- /dev/null
+++ b/doc/forum/how_to_disaster_recovery/comment_1_c8db90fd7af13b9605e152ea884b5241._comment
@@ -0,0 +1,15 @@
+[[!comment format=mdwn
+ username="jd.schroeder@0c8632a8f2bffdd4b0de05a0a3660f32acdfeeca"
+ nickname="jd.schroeder"
+ subject="comment 1"
+ date="2016-08-17T21:18:40Z"
+ content="""
+I don't use the assistant myself, so I may not be much help.
+
+Have you tried opening the terminal and running 
+
+    cd (your repository path)
+    git annex fsck
+
+it may yield more information about what's wrong.
+"""]]
diff --git a/doc/forum/how_to_disaster_recovery/comment_2_31cce673114eb9733e0f1c732c8f08e6._comment b/doc/forum/how_to_disaster_recovery/comment_2_31cce673114eb9733e0f1c732c8f08e6._comment
new file mode 100644
index 0000000000..cef32b77e9
--- /dev/null
+++ b/doc/forum/how_to_disaster_recovery/comment_2_31cce673114eb9733e0f1c732c8f08e6._comment
@@ -0,0 +1,12 @@
+[[!comment format=mdwn
+ username="jd.schroeder@0c8632a8f2bffdd4b0de05a0a3660f32acdfeeca"
+ nickname="jd.schroeder"
+ subject="comment 2"
+ date="2016-08-17T21:23:34Z"
+ content="""
+You may want to also try
+
+    git fsck
+
+as well.
+"""]]
diff --git a/doc/forum/how_to_disaster_recovery/comment_3_52ad88121efa92d2c64bb2816884d66b._comment b/doc/forum/how_to_disaster_recovery/comment_3_52ad88121efa92d2c64bb2816884d66b._comment
new file mode 100644
index 0000000000..c069190359
--- /dev/null
+++ b/doc/forum/how_to_disaster_recovery/comment_3_52ad88121efa92d2c64bb2816884d66b._comment
@@ -0,0 +1,67 @@
+[[!comment format=mdwn
+ username="openmedi"
+ subject="comment 3"
+ date="2016-08-20T14:59:17Z"
+ content="""
+Thanks you so much for the suggestions! I tried running `git annex fsck` which yielded no errors and also ran `git fsck` which gave the following output:
+
+```
+error in tree 317eed096daa6d5efde4621ea57b698e17a44827: duplicateEntries: contains duplicate file entries
+error in tree 5d03ab540eb58fef3cba9466d0f404a2f487acbe: duplicateEntries: contains duplicate file entries
+error in tree 8001d121ff90053597e56ee6b9e4dd7d36e5ff25: duplicateEntries: contains duplicate file entries
+error in tree 9eea5dcfc9d0ab387165b91e11ecf6433d501e88: duplicateEntries: contains duplicate file entries
+error in tree c7c1bbde3d9479cbb44a5adbe763e9a90577fcbd: duplicateEntries: contains duplicate file entries
+error in tree f7544950f810acacb43647719ab77427dced34da: duplicateEntries: contains duplicate file entries
+Checking object directories: 100% (256/256), done.
+Checking connectivity: 403770, done.
+dangling blob 0a37708d72fe5cfeb0421372874557c45be3bed5
+dangling blob 7c6e000baabaa9a3b451bc4a058c8a1743598a4b
+dangling tree 8001d121ff90053597e56ee6b9e4dd7d36e5ff25
+dangling blob 7c76f1e9b6c2bc2189d11db526d4d615bfbda98a
+dangling blob f69212b0c58b43e5d935cfc39fdf6b1751cb5286
+dangling blob 3ddcb33a4a10db7a7c5f7baba730515758b47552
+dangling blob 9feae3d7a5786071826d0ec92289178c2fbbf915
+dangling blob 683f54561577372de1083e231e25eede7a978a2a
+dangling blob ffd8254bc8d78f4dfa180b7b3846bd9af7ad4e0a
+dangling blob 20e2b593b9d93cb3b9f39fd9e22dcbe3427cc369
+dangling blob 500da653dfcb37895e64be4959155e03d352fcbb
+dangling blob d61e26a6e41c547ad135ef6b58f64bd00ad0b31f
+dangling blob 38a9462b54afe85ee24e01b0c4d0fbd75eb5a240
+dangling blob 39b6962714399c604a65b350a21795e00e44b4f5
+dangling blob 71f3b6ca386913456a7de6a31b56729d6f8c36c9
+dangling blob dde4a7fea11398020fed287c4d31780909580e5d
+dangling blob 943b98bbee781a7c68befd37700ab8503a17503c
+dangling blob dc99f8b37db7bdc361c6d7973ab863674b2c59b3
+dangling blob 83b7388589b510388e4c08e2963ec10f21355e63
+dangling blob 63c1e85eedc424c297e5f36f6bc705982471fdce
+dangling blob 11d9d8a2c45f4d8b0479513e2f65352cfd0863d0
+dangling blob 9ce519f47e5a7dd8c923fcb55693a22d82e21779
+dangling blob c1ea698236b21eede67d01a8dd52c877962d1845
+dangling blob 24f6096646c9df777f2a81f40bbdef456f0e754d
+dangling blob a4fb09b357878b1cf31cd2151eb9d0fab2f0b452
+dangling blob ff0dda6cd8a6f882fdf81717519b1abb2c347a3e
+dangling blob a0500a7a450c487a3deeba9915c8f0a88f91b538
+dangling blob 31542a921878a024ec2dbe5815f58da6b6b33e2c
+dangling blob 6b6c6a8eb15fbab7f878396d0f084faa22c73382
+dangling tree 5d03ab540eb58fef3cba9466d0f404a2f487acbe
+dangling blob 87319bda4de4424f60e384fe5c3c14c2ce399d10
+dangling blob d8360b3ffc67ccb1478a797370cbb2112589e13a
+dangling blob 90e18b4beb5e5aedefdab98b57ccf636fb77e046
+dangling blob 06e24bc228ae12504a2bcbe9d3d24f263ba0a209
+dangling blob d2eb8b104d2c002ded4ec76e440dee88f5174a39
+dangling blob e7addc9b07f4938d5871eb67278f64c1d15b2cb4
+dangling blob ff02ddf430604f4a60e3bc7b70804ff1f539dedb
+dangling blob 8c04bdb8cef6265ccde6b7491a57f3869c7d6b23
+dangling blob 8d83dd50a078efda7b60de1900736fbd77a2c5a6
+dangling tree 9eea5dcfc9d0ab387165b91e11ecf6433d501e88
+dangling blob 2011ce2b12529ec512b5c6b75bb6acdf3cbc5e82
+dangling blob 8d6d6e3ea481ddc9eb697c7e4c61479b212157c6
+dangling blob 68f62e8d6902b0de28c6f28899be73c6864d58e2
+dangling blob 97f77e273d92ebc908cfcc3b81dbbc28d10ae090
+dangling blob be24efa8af600f0ac331872e0769b18aa36a17c8
+dangling blob cd257f99eef789d2ce8f7f78d2e80c99311bf8b5
+dangling blob 4e973f9d9b9900d01f2d1f1815609e80b1fcf4a5
+```
+
+I'm now running git annex repair through the command line and hoping for the best. I'm sure this will run for a while. The output so far is just `Running git fsck ...` but I can see it's doing something since the terminal application in osx shows what commands it's currently running in the header of the window itself. I'm hoping this repair will finnish until tomorrow morning because I have a flight coming up at 8. Let's see.
+"""]]
diff --git a/doc/forum/how_to_disaster_recovery/comment_4_444579d0dac2ca14e6b1cbadcca5fcf3._comment b/doc/forum/how_to_disaster_recovery/comment_4_444579d0dac2ca14e6b1cbadcca5fcf3._comment
new file mode 100644
index 0000000000..7ded573bf6
--- /dev/null
+++ b/doc/forum/how_to_disaster_recovery/comment_4_444579d0dac2ca14e6b1cbadcca5fcf3._comment
@@ -0,0 +1,24 @@
+[[!comment format=mdwn
+ username="jd.schroeder@0c8632a8f2bffdd4b0de05a0a3660f32acdfeeca"
+ nickname="jd.schroeder"
+ subject="comment 4"
+ date="2016-08-20T16:17:07Z"
+ content="""
+Unfortunately, I'm not familiar with the errors that git fsck is giving you, so I should defer to someone who knows more. I don't want to tell you something that makes the problem worse :)
+
+If repairing doesn't work, one other thing you could try is cloning the repository that's giving you trouble, and using git annex to move the data into the new clone. This won't modify your original (defective) repository.
+
+    git clone /path/to/repository /path/to/newClone
+    cd /path/to/newClone
+    git annex get --all
+
+If you have files in your original repository that the assistant hasn't committed yet, they won't be in the new clone. So you should look around in /path/to/newClone to see if everything is there. You can run
+
+    cd /path/to/repository
+    git status
+
+to see any files that have not yet been committed in the defective repository, and copy them over manually.
+
+Good luck! Hope you get it working.
+
+"""]]
diff --git a/doc/forum/how_to_disaster_recovery/comment_5_9b9403a80c5b0a1a441dfb94feece37d._comment b/doc/forum/how_to_disaster_recovery/comment_5_9b9403a80c5b0a1a441dfb94feece37d._comment
new file mode 100644
index 0000000000..52f1d1e454
--- /dev/null
+++ b/doc/forum/how_to_disaster_recovery/comment_5_9b9403a80c5b0a1a441dfb94feece37d._comment
@@ -0,0 +1,20 @@
+[[!comment format=mdwn
+ username="joey"
+ subject="""comment 5"""
+ date="2016-09-05T20:02:39Z"
+ content="""
+Very interesting git fsck output!
+
+It seems like your git repository has somehow gotten a tree object in it
+which contains two files with the same name.
+
+While git's data structures allow this, it's pretty nonsensical, and it's
+not something that would normally ever happen. It would be interesting to
+take a look and see what commits use those tree objects, and where those
+strange trees came from.
+
+git-annex in turn is seeing that tree object's sha in the fsck output and
+assumes fsck means that the data for that object is broken or missing.
+So, I suppose what I'll do is make it skip over lines containing
+`duplicateEntries`, and then it won't try to repair this repository.
+"""]]
diff --git a/doc/forum/how_to_disaster_recovery/comment_6_e8ec7b1fbbb81d61d06a7def61b06354._comment b/doc/forum/how_to_disaster_recovery/comment_6_e8ec7b1fbbb81d61d06a7def61b06354._comment
new file mode 100644
index 0000000000..425558b254
--- /dev/null
+++ b/doc/forum/how_to_disaster_recovery/comment_6_e8ec7b1fbbb81d61d06a7def61b06354._comment
@@ -0,0 +1,16 @@
+[[!comment format=mdwn
+ username="joey"
+ subject="""comment 6"""
+ date="2016-10-18T15:27:05Z"
+ content="""
+Another user reported the same problem. Their repo indeed had a tree object
+that contained two items with the same name. In this case, two trees,
+with different contents.
+
+	040000 tree cd854aba649cee8855ec72579f4c98100471f7cf    Dokument
+	040000 tree 1ae499997746c8e976cf02edc2276532978dafc5    Dokument
+
+This user had been using adjusted branches, which would be my lead suspect.
+
+Original reporter, did you use git-annex's adjusted branches feature.
+"""]]
diff --git a/doc/forum/how_to_disaster_recovery/comment_7_7e08eb18e965f47782a39d0426b75b93._comment b/doc/forum/how_to_disaster_recovery/comment_7_7e08eb18e965f47782a39d0426b75b93._comment
new file mode 100644
index 0000000000..7053bb282b
--- /dev/null
+++ b/doc/forum/how_to_disaster_recovery/comment_7_7e08eb18e965f47782a39d0426b75b93._comment
@@ -0,0 +1,9 @@
+[[!comment format=mdwn
+ username="openmedi"
+ subject="comment 7"
+ date="2016-10-18T17:37:38Z"
+ content="""
+Not that I know of. I tried a whole bunch of things, like deleting remotes, killing processes trying to switch between repo versions, modes, playing around in config files etc. etc. Not all of them were smart ideas, I admit. At some point it worked and if I remember correctly I tried to delete a branch and then the problems started. But adjusted branches seems to me like a too advanced feature to use.
+
+Btw.: Skipping over these entries doesn't help for me. It still tries to repair the repo (which takes maybe 4 hours, including having to initiate a temporary repo).
+"""]]
diff --git a/doc/forum/how_to_disaster_recovery/comment_8_78720819bea58f30146451dd953f10a5._comment b/doc/forum/how_to_disaster_recovery/comment_8_78720819bea58f30146451dd953f10a5._comment
new file mode 100644
index 0000000000..1eb3694668
--- /dev/null
+++ b/doc/forum/how_to_disaster_recovery/comment_8_78720819bea58f30146451dd953f10a5._comment
@@ -0,0 +1,7 @@
+[[!comment format=mdwn
+ username="openmedi"
+ subject="comment 8"
+ date="2016-10-18T17:38:53Z"
+ content="""
+The \"at some point\" portion of my previous message refers to a state before the problems started, not after or in between. Sorry for being unclear.
+"""]]
diff --git a/doc/forum/how_to_disaster_recovery/comment_9_b16c74c0add6d15b75163859a752d207._comment b/doc/forum/how_to_disaster_recovery/comment_9_b16c74c0add6d15b75163859a752d207._comment
new file mode 100644
index 0000000000..b6ee528a21
--- /dev/null
+++ b/doc/forum/how_to_disaster_recovery/comment_9_b16c74c0add6d15b75163859a752d207._comment
@@ -0,0 +1,9 @@
+[[!comment format=mdwn
+ username="openmedi"
+ subject="comment 9"
+ date="2016-10-25T18:02:56Z"
+ content="""
+> […] if I remember correctly I tried to delete a branch and then the problems started […]
+
+This is not correct. I tried to delete a special remote.
+"""]]
diff --git a/doc/forum/how_to_edit_the_git-annex_branch__63__.mdwn b/doc/forum/how_to_edit_the_git-annex_branch__63__.mdwn
new file mode 100644
index 0000000000..a37391001a
--- /dev/null
+++ b/doc/forum/how_to_edit_the_git-annex_branch__63__.mdwn
@@ -0,0 +1,9 @@
+as a followup to [[forum/remote-specific_meta-data/]], now i'm thinking of injecting data in the `git-annex` branch. i'm still a little hesitant because [[forum/optimising_lookupkey/]] mentions that i need to "delete or update `.git/annex/index`" when doing so. yet I also saw in the source code there is a `.git/annex/index.lock` file that is being used to avoid concurrent access to that file - is that something that should be watched out for? how do I manage that lockfile? is it mandatory to stage files into the index at first?
+
+maybe i'm just not familiar enough with how git operates internally myself. i've reviewed this section of the progit book again: http://git-scm.com/book/en/v2/Git-Internals-Git-Objects - and i feel i have again some handle on how to do those things, but it seems like a major hurdle to go through just to change a single setting in one git-annex metadata file... wouldn't `enableremote` be sufficient here? I figured to append custom fields to a remote, i could do:
+
+    git annex enableremote myremoteuuid "$(git cat-file -p git-annex:remote.log | grep myremoteuuid) extratag=true"
+
+Would that work at all? Should i operate directly on the git-annex branch myself instead?
+
+Thanks! --[[anarcat]]
diff --git a/doc/forum/how_to_edit_the_git-annex_branch__63__/comment_1_fa1936401d1abe1a28dc88968528e186._comment b/doc/forum/how_to_edit_the_git-annex_branch__63__/comment_1_fa1936401d1abe1a28dc88968528e186._comment
new file mode 100644
index 0000000000..6200e3792e
--- /dev/null
+++ b/doc/forum/how_to_edit_the_git-annex_branch__63__/comment_1_fa1936401d1abe1a28dc88968528e186._comment
@@ -0,0 +1,7 @@
+[[!comment format=mdwn
+ username="joey"
+ subject="""comment 1"""
+ date="2015-03-31T18:26:06Z"
+ content="""
+Yes, you can pass any (unused) fields you like to git-annex enableremote.
+"""]]
diff --git a/doc/forum/how_to_force_read-only_in_enableremote__63__.mdwn b/doc/forum/how_to_force_read-only_in_enableremote__63__.mdwn
new file mode 100644
index 0000000000..ff90fbbcdc
--- /dev/null
+++ b/doc/forum/how_to_force_read-only_in_enableremote__63__.mdwn
@@ -0,0 +1,3 @@
+I see that `enableremote` changes the source remote to log the presence of a new annex (the annex where `enableremote` was called). I have tested with a gcrypt remote and I see that a new commit appears in the git-annex branch of the source remote after `enableremote` was called. Since I have cloned the annex just for testing and will delete it afterwards, I would like to leave the original remote untouched.
+
+Is there a way to enable the remote read-only?
diff --git a/doc/forum/how_to_force_read-only_in_enableremote__63__/comment_1_a1206c8517f467d9a584ba89ef0e101a._comment b/doc/forum/how_to_force_read-only_in_enableremote__63__/comment_1_a1206c8517f467d9a584ba89ef0e101a._comment
new file mode 100644
index 0000000000..6dd90c28ba
--- /dev/null
+++ b/doc/forum/how_to_force_read-only_in_enableremote__63__/comment_1_a1206c8517f467d9a584ba89ef0e101a._comment
@@ -0,0 +1,23 @@
+[[!comment format=mdwn
+ username="joey"
+ subject="""comment 1"""
+ date="2016-11-10T18:13:28Z"
+ content="""
+git-annex auto-init the local repository whenever it sees it's not
+enabled and there is a git-annex branch (or origin/git-annex etc).
+This is what adds the uuid of the local repository to remote.log.
+So, it's not specific to `enableremote`; many git-annex commands
+will do the same thing. And git-annex needs to allocate a uuid for the
+local repo in order to work.
+
+One way to deal with it is to use `git annex reinit` so it uses
+some already existing uuid. Although then getting files etc will
+update the records using that uuid. Might be ok if you have one
+uuid, perhaps one you've marked dead, that you use for this purpose
+and don't care what's recorded for that uuid.
+
+Or, it might be better to let git-annex auto-init a new uuid, 
+and just configure this test repo to not push changes out to
+elsewhere. For example, `git config remote.origin.annex-readonly true`
+will make `git annex sync` avoid pushing back to the origin remote.
+"""]]
diff --git a/doc/forum/how_to_prevent_accidentally_running___96__git_add__96____63__.mdwn b/doc/forum/how_to_prevent_accidentally_running___96__git_add__96____63__.mdwn
new file mode 100644
index 0000000000..e7de592100
--- /dev/null
+++ b/doc/forum/how_to_prevent_accidentally_running___96__git_add__96____63__.mdwn
@@ -0,0 +1,5 @@
+I fear that while using git annex I will at some point accidentally `git add` some small files and not notice it until the only way to fix the problem is to rewrite history. What would be the best way to prevent myself from ever `git add`-ing a file into my annex repository instead of `git annex add`-ing it?
+
+And secondly, how can I best search in my git annex repository whether I already did this mistake in the past or not? Currently I'm using this which returns everything that's not a symlink or a git submodule:
+
+    git ls-files -s | awk ' $1 != 120000 && $1 != 160000 { print $4 }'
diff --git a/doc/forum/how_to_prevent_accidentally_running___96__git_add__96____63__/comment_1_440dcd19ea2512f968858b780c2a2913._comment b/doc/forum/how_to_prevent_accidentally_running___96__git_add__96____63__/comment_1_440dcd19ea2512f968858b780c2a2913._comment
new file mode 100644
index 0000000000..166d549fd8
--- /dev/null
+++ b/doc/forum/how_to_prevent_accidentally_running___96__git_add__96____63__/comment_1_440dcd19ea2512f968858b780c2a2913._comment
@@ -0,0 +1,22 @@
+[[!comment format=mdwn
+ username="Xyem"
+ ip="178.79.137.64"
+ subject="comment 1"
+ date="2014-06-25T10:35:07Z"
+ content="""
+I'm not sure if you can prevent 'git add' but you can at least prevent it getting commited with the pre-commit hook. Mine is this:
+
+    #!/bin/sh
+    
+    FILES=$(find -name \".git\" -prune -o -type f -not -name \".gitignore\" -print);
+    FILE_COUNT=$(echo -n \"$FILES\" | wc -l)
+    
+    if [ $FILE_COUNT -gt 0 ]; then
+            echo \"$FILE_COUNT non-symlink files found.\"
+            echo \"$FILES\"
+            exit 1
+    fi
+    
+    # automatically configured by git-annex
+    git annex pre-commit .
+"""]]
diff --git a/doc/forum/how_to_prevent_accidentally_running___96__git_add__96____63__/comment_2_e9b70386774996a3d0446faaa3219120._comment b/doc/forum/how_to_prevent_accidentally_running___96__git_add__96____63__/comment_2_e9b70386774996a3d0446faaa3219120._comment
new file mode 100644
index 0000000000..bc8acf020d
--- /dev/null
+++ b/doc/forum/how_to_prevent_accidentally_running___96__git_add__96____63__/comment_2_e9b70386774996a3d0446faaa3219120._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="josch"
+ ip="2001:638:709:5:2ad2:44ff:fe4b:56aa"
+ subject="comment 2"
+ date="2014-06-27T04:50:03Z"
+ content="""
+Xyem, your method does not work well because it finds all regular files in the whole annex directory and does not only check for those that are to be committed.
+"""]]
diff --git a/doc/forum/how_to_prevent_accidentally_running___96__git_add__96____63__/comment_3_3dbd76accad2df2fff14b55452c828ef._comment b/doc/forum/how_to_prevent_accidentally_running___96__git_add__96____63__/comment_3_3dbd76accad2df2fff14b55452c828ef._comment
new file mode 100644
index 0000000000..d8f681a1bf
--- /dev/null
+++ b/doc/forum/how_to_prevent_accidentally_running___96__git_add__96____63__/comment_3_3dbd76accad2df2fff14b55452c828ef._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="Xyem"
+ ip="178.79.137.64"
+ subject="comment 3"
+ date="2014-06-27T12:02:08Z"
+ content="""
+That is intentional. In my use case, there should never be any regular files in this particular annex at all. If there are, it means I have missed something! :)
+"""]]
diff --git a/doc/forum/how_to_prevent_accidentally_running___96__git_add__96____63__/comment_4_bfb4758fdb2a4afafd9c4d45de5c5c6b._comment b/doc/forum/how_to_prevent_accidentally_running___96__git_add__96____63__/comment_4_bfb4758fdb2a4afafd9c4d45de5c5c6b._comment
new file mode 100644
index 0000000000..de2a279a72
--- /dev/null
+++ b/doc/forum/how_to_prevent_accidentally_running___96__git_add__96____63__/comment_4_bfb4758fdb2a4afafd9c4d45de5c5c6b._comment
@@ -0,0 +1,28 @@
+[[!comment format=mdwn
+ username="https://www.google.com/accounts/o8/id?id=AItOawnWvnTWY6LrcPB4BzYEBn5mRTpNhg5EtEg"
+ nickname="Bence"
+ subject="comment 4"
+ date="2015-01-15T16:15:21Z"
+ content="""
+Based on the above, I'm using this script in the `pre-commit` hook.
+
+It gets the staged files and checks if they are links or not. If a file being committed is not a link (maybe it was added with `git add filename` ???), the script aborts the commit.
+
+    #!/bin/sh
+    
+    # Don't allow files to be added to the normal repo.
+    stagedfiles=$(git diff --cached --name-only);
+    echo \"$stagedfiles\" | while IFS= read -r file; do
+    	if [ ! -L \"$file\" ]; then
+    		echo \"[Error] The file \\"$file\\" should not be added to the repo.\";
+    		echo \"#Remove from the index and add with git-annex:\";
+    		echo \"\$ git rm --cached \\"$file\\" && git annex add \\"$file\\"\";
+    		exit 1;
+    	#else
+    	#	echo \"OK : $file\";
+    	fi;
+    done;
+    
+    # automatically configured by git-annex
+    git annex pre-commit .
+"""]]
diff --git a/doc/forum/how_to_remove_files_and_symlinks_but_keep_historical_file_contents.mdwn b/doc/forum/how_to_remove_files_and_symlinks_but_keep_historical_file_contents.mdwn
new file mode 100644
index 0000000000..4f24117f68
--- /dev/null
+++ b/doc/forum/how_to_remove_files_and_symlinks_but_keep_historical_file_contents.mdwn
@@ -0,0 +1,7 @@
+It would be nice to have a way to drop files without leaving broken symlinks around, at least while in direct mode.
+
+Here is my user case. I have a collection of music CDs ripped in FLAC formats. At the same time I convert all these files to MP3 files to that I can use them in various other devices and to save space.
+
+The problem is that if I `git annex drop` the FLAC files once they are converted and copied, they leave broken symlinks around; this result in various annoying error messages in almost all the music playback I tried. At the same time, if I `rm` or `git rm` these symlinks, the content of these files will be removed also from the remotes as they are signalled as no longer wanted.
+
+Couldn't git-annex keep a separate index of files that have been removed but are meant to be kept?
diff --git a/doc/forum/how_to_remove_files_and_symlinks_but_keep_historical_file_contents/comment_1_cba76311e146dabb8ffc789bf4c8b714._comment b/doc/forum/how_to_remove_files_and_symlinks_but_keep_historical_file_contents/comment_1_cba76311e146dabb8ffc789bf4c8b714._comment
new file mode 100644
index 0000000000..c490c826a8
--- /dev/null
+++ b/doc/forum/how_to_remove_files_and_symlinks_but_keep_historical_file_contents/comment_1_cba76311e146dabb8ffc789bf4c8b714._comment
@@ -0,0 +1,14 @@
+[[!comment format=mdwn
+ username="http://svario.it/gioele"
+ ip="2001:1418:100:262::2"
+ subject="Maybe using branches?"
+ date="2013-05-16T10:22:48Z"
+ content="""
+A suggestion from #git-annex:
+
+> <mhameed> i think others are already doing this, using branches, but not sure if we have a writeup/tutorial of it
+>
+> <mhameed> gioele:  instead of working on master branch, switch to laptop branch, git rm the files that are not wanted and commit
+>
+> <gioele> mhameed: I'm lazy, I'd like git-annex to do that for me :)
+"""]]
diff --git a/doc/forum/how_to_remove_files_and_symlinks_but_keep_historical_file_contents/comment_2_8d99c50fc1347367ccc0714e8d1af385._comment b/doc/forum/how_to_remove_files_and_symlinks_but_keep_historical_file_contents/comment_2_8d99c50fc1347367ccc0714e8d1af385._comment
new file mode 100644
index 0000000000..f0b20f1a4a
--- /dev/null
+++ b/doc/forum/how_to_remove_files_and_symlinks_but_keep_historical_file_contents/comment_2_8d99c50fc1347367ccc0714e8d1af385._comment
@@ -0,0 +1,40 @@
+[[!comment format=mdwn
+ username="http://joeyh.name/"
+ nickname="joey"
+ subject="comment 2"
+ date="2013-05-19T20:43:43Z"
+ content="""
+The original poster seems to have a misunderstanding of what git-annex does with the content of files. Deleting a file does not remove the content of the file. You can always use git to check out a previous version of a file, and the content from the annex will still be there (or you can `git annex get` it to get it from whereever git-annex stored the content).
+
+The only exception to this rule is is you, manually, chose to run `git annex unused` and then `git annex dropunused`. That can delete the contents of files, when no branch or tag refers to them. As long as some branch refers to the content of the files, even if it's not the currently checked out branch, the file content is retained.
+
+So a branch *is* the \"index of files that have been removed but are wanted to be kept\".
+
+For example, you could do:
+
+[[!format sh \"\"\"
+git checkout -b keepflacs
+
+cdrom-rip-command
+git annex add *.flac
+git commit -m \"ripped a cd\"
+flac-convert-command
+git annex add *.mp3
+git commit -m \"converted to mp3\"
+
+git checkout master
+git merge keepflacs
+git rm *.flac
+git commit -m \"removed flac files from this branch. They are still available in the keepflacs branch\"
+\"\"\"]]
+
+As long as you always switch to the keepflacs branch to add flac files, and never merge the master branch into keepflacs,
+but only merge keepflacs into master, keepflacs will have every flac file you have ripped. And so git-annex will retain those files
+even when you `git annex unused; git-annex dropunused`.
+
+Or, just make a promise to yourself that you'll never run `git-annex unused`, similarly to how you'd probably never run `rm -rf .git/objects/$rand`, and you don't need the branches; like git, git-annex will retain all data that has ever been checked into it.
+
+(The branches are still a good idea. For one thing, they let you run `git annex fsck` and other repository maintenance commands with the keepflacs branch checked out.)
+
+I am going to move this thread to the [[forum]], because it's not really a TODO item, but is something others may want to read.
+"""]]
diff --git a/doc/forum/how_to_remove_files_and_symlinks_but_keep_historical_file_contents/comment_3_a7a9c55c2ad448179dff5d5b69976c7d._comment b/doc/forum/how_to_remove_files_and_symlinks_but_keep_historical_file_contents/comment_3_a7a9c55c2ad448179dff5d5b69976c7d._comment
new file mode 100644
index 0000000000..39ed8306c0
--- /dev/null
+++ b/doc/forum/how_to_remove_files_and_symlinks_but_keep_historical_file_contents/comment_3_a7a9c55c2ad448179dff5d5b69976c7d._comment
@@ -0,0 +1,10 @@
+[[!comment format=mdwn
+ username="http://joeyh.name/"
+ nickname="joey"
+ subject="comment 3"
+ date="2013-05-20T17:46:08Z"
+ content="""
+Erm, I missed that you want to use direct mode. All this fun with git won't really work in direct mode, and indeed direct mode is not able to guarantee that old versions of modified files are retained.
+
+Direct mode is nice for some applications involving syncing and less than ideal devices, but not for this.
+"""]]
diff --git a/doc/forum/how_to_set_up_syncing_for_multiple_computer_and_a_centralized_backup_unit.mdwn b/doc/forum/how_to_set_up_syncing_for_multiple_computer_and_a_centralized_backup_unit.mdwn
new file mode 100644
index 0000000000..960f563ced
--- /dev/null
+++ b/doc/forum/how_to_set_up_syncing_for_multiple_computer_and_a_centralized_backup_unit.mdwn
@@ -0,0 +1,9 @@
+Been using Unison for a while to sync a folder at three different computers (can be on or off simultanously). However, I've for different reasons been looking for a different solution. Here comes git annex.e
+
+I started out with one computer, generated a new repo locally and at the server (a synology nas) with the assistant. Then i put all my files into the local directory and the files uploaded to the server. All good. I put the local computer in client mode and the server in backup mode. Then i configured a second computer with an empty repo folder, was asked to join repos, did that. Also put this computer in client mode and the server in backup. After a while all the files was located in the second computers repo directory. Yes, I was confident this was the way to go. 
+
+Howewer, doing the same with my laptop was not successful at all. After similar setup as above it only downloaded about 5% of the data. Bummer. While I tried to study the logs I thought I should check the second computers repo. That was modified and a lot of files where gone. Wow...of course I have backup, but I was hoping for a proper and consistent operation.
+
+So what causes this? Should I not put the server in backup mode? Anyone have an idea?
+
+Other thoughts?
diff --git a/doc/forum/how_to_set_up_syncing_for_multiple_computer_and_a_centralized_backup_unit/comment_1_a0551431a57ccab2463f2a6d43553337._comment b/doc/forum/how_to_set_up_syncing_for_multiple_computer_and_a_centralized_backup_unit/comment_1_a0551431a57ccab2463f2a6d43553337._comment
new file mode 100644
index 0000000000..99fa06512a
--- /dev/null
+++ b/doc/forum/how_to_set_up_syncing_for_multiple_computer_and_a_centralized_backup_unit/comment_1_a0551431a57ccab2463f2a6d43553337._comment
@@ -0,0 +1,15 @@
+[[!comment format=mdwn
+ username="https://www.google.com/accounts/o8/id?id=AItOawncVeolylM8VoRbWhIYDlfGhIP69-aNXm4"
+ nickname="Espen"
+ subject="comment 1"
+ date="2014-10-10T07:14:36Z"
+ content="""
+Okey, came in to the office today to check the first computer I set up. It lacks all the files except the ones located on the third computer. So all sites are synced (only a lot of files are dropped). Seems something happened while the third computer was set up that caused just a few files to sync from the backup server and then git somehow thought that was it and then synced computer one, two and the server according to files present on computer three (obviously way to few files). This is slightly worrisome and I wonder what caused this. I'll inspect the log files for the transfer on computer three when I get home.
+
+Can anyone of you confirm that having this setup should not cause problems:
+
+computer 1 - client mode
+computer 2 - client mode
+computer 3 - client mode
+backup unit/nas - backup mode
+"""]]
diff --git a/doc/forum/how_to_set_up_syncing_for_multiple_computer_and_a_centralized_backup_unit/comment_2_e96e8cf6e08e3a21bfcefbc202e78fe2._comment b/doc/forum/how_to_set_up_syncing_for_multiple_computer_and_a_centralized_backup_unit/comment_2_e96e8cf6e08e3a21bfcefbc202e78fe2._comment
new file mode 100644
index 0000000000..fa0f491202
--- /dev/null
+++ b/doc/forum/how_to_set_up_syncing_for_multiple_computer_and_a_centralized_backup_unit/comment_2_e96e8cf6e08e3a21bfcefbc202e78fe2._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="https://www.google.com/accounts/o8/id?id=AItOawlM_DRhi_5pJrTA0HbApHR25iAgy-NBXTY"
+ nickname="Tor Arne"
+ subject="comment 2"
+ date="2014-10-14T22:13:44Z"
+ content="""
+Did you figure this one out?
+"""]]
diff --git a/doc/forum/how_to_set_up_syncing_for_multiple_computer_and_a_centralized_backup_unit/comment_3_2ad4c1a4bfe00c22444ab878c84a8830._comment b/doc/forum/how_to_set_up_syncing_for_multiple_computer_and_a_centralized_backup_unit/comment_3_2ad4c1a4bfe00c22444ab878c84a8830._comment
new file mode 100644
index 0000000000..8c42c43fd5
--- /dev/null
+++ b/doc/forum/how_to_set_up_syncing_for_multiple_computer_and_a_centralized_backup_unit/comment_3_2ad4c1a4bfe00c22444ab878c84a8830._comment
@@ -0,0 +1,47 @@
+[[!comment format=mdwn
+ username="https://www.google.com/accounts/o8/id?id=AItOawncVeolylM8VoRbWhIYDlfGhIP69-aNXm4"
+ nickname="Espen"
+ subject="comment 3"
+ date="2014-10-15T17:32:10Z"
+ content="""
+Finally had some time today to lock at the logs. However, I started from scratch. Created a local repo with the assistant, transferred some real files into the local repo dir (not .git of course). Waiting until everything was added etc. Then I created the remote server repo and it immediately started to transfer files from the local repo to the remote. Then I was slightly surprised when I got back home and saw this in the logfile:
+
+----- cut -----
+
+somefile
+
+         32,768   1%    0.00kB/s    0:00:00  To ssh://someserverrepodir
+ * [new branch]      git-annex -> synced/git-annex
+ * [new branch]      annex/direct/master -> synced/master
+error: Ref refs/heads/synced/git-annex is at 112bb35566a0ee9434fb74524cdced45792bf8ed but expected 0000000000000000000000000000000000000000
+error: Ref refs/heads/synced/master is at d593f3c3a5090009789154bd60c3390d9a1b90d6 but expected 0000000000000000000000000000000000000000
+remote: error: failed to lock refs/heads/synced/git-annex        
+remote: error: failed to lock refs/heads/synced/master        
+To ssh://someserverrepodir
+ ! [remote rejected] git-annex -> synced/git-annex (failed to lock)
+ ! [remote rejected] annex/direct/master -> synced/master (failed to lock)
+error: failed to push some refs to 'ssh://someserverrepodir'
+
+      1,966,080  96%    1.73MB/s    0:00:00  
+      2,034,254 100%    1.79MB/s    0:00:01 (xfr#1, to-chk=0/1)
+fatal: 'someserverrepodescription' does not appear to be a git repository
+fatal: Could not read from remote repository.
+
+Please make sure you have the correct access rights
+and the repository exists.
+fatal: 'someserverrepodescription' does not appear to be a git repository
+fatal: Could not read from remote repository.
+
+Please make sure you have the correct access rights
+and the repository exists.
+fatal: 'someserverrepodescription' does not appear to be a git repository
+fatal: Could not read from remote repository.
+
+Please make sure you have the correct access rights
+and the repository exists.
+[2014-10-13 21:58:05 CEST] Transferrer: Uploaded somefile
+
+----- cut -----
+
+This was a total surprise to me. If this was the stuff that made the previous setup fail, I don not know, but I will try to track down this error first. I would be quite surprised if it indeed was a permission issue. Also visible is an upload session for one of the files...that seems to go on fine and is probably not related to the error at all. This was the only error printed once during the entire transfer/sync process. Thought the first thing I would check was the size and number of files in the local and remote directory, but I'm not really sure how to do this with all the git stuff around on the remote. Typically \"git ls-files | wc -l\" or similar gets me going on the local repo. How would I do this on the remote (in backup mode if that counts)?
+"""]]
diff --git a/doc/forum/how_to_set_up_syncing_for_multiple_computer_and_a_centralized_backup_unit/comment_4_44639388349a9ea5eabda9ebf79817b3._comment b/doc/forum/how_to_set_up_syncing_for_multiple_computer_and_a_centralized_backup_unit/comment_4_44639388349a9ea5eabda9ebf79817b3._comment
new file mode 100644
index 0000000000..c97438d8ab
--- /dev/null
+++ b/doc/forum/how_to_set_up_syncing_for_multiple_computer_and_a_centralized_backup_unit/comment_4_44639388349a9ea5eabda9ebf79817b3._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="https://www.google.com/accounts/o8/id?id=AItOawncVeolylM8VoRbWhIYDlfGhIP69-aNXm4"
+ nickname="Espen"
+ subject="comment 4"
+ date="2014-10-15T17:34:52Z"
+ content="""
+Sorry, forgot to format the text and the typo. Can I edit my own comments?
+"""]]
diff --git a/doc/forum/how_to_set_up_syncing_for_multiple_computer_and_a_centralized_backup_unit/comment_5_339123ab87b69b11d6e999ad6eaf6df5._comment b/doc/forum/how_to_set_up_syncing_for_multiple_computer_and_a_centralized_backup_unit/comment_5_339123ab87b69b11d6e999ad6eaf6df5._comment
new file mode 100644
index 0000000000..58bed56e57
--- /dev/null
+++ b/doc/forum/how_to_set_up_syncing_for_multiple_computer_and_a_centralized_backup_unit/comment_5_339123ab87b69b11d6e999ad6eaf6df5._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="https://www.google.com/accounts/o8/id?id=AItOawncVeolylM8VoRbWhIYDlfGhIP69-aNXm4"
+ nickname="Espen"
+ subject="comment 5"
+ date="2014-10-15T17:44:55Z"
+ content="""
+Did a quick \"du -h\" on both directories. Local is 27 and remote is 23.7 GB, so some data is obviously missing on the remote. Also, how do I increase either the permitted size of one log file, or the number of log files stored? Seems the logs where rotated over the whole allowed log batch and size before the job was done during the night, so difficult to tell if there were more errors.
+"""]]
diff --git a/doc/forum/howto_to_link_to_existing_direct_mode_git-annexes.mdwn b/doc/forum/howto_to_link_to_existing_direct_mode_git-annexes.mdwn
new file mode 100644
index 0000000000..cab2ec0c6b
--- /dev/null
+++ b/doc/forum/howto_to_link_to_existing_direct_mode_git-annexes.mdwn
@@ -0,0 +1,5 @@
+I have created two git-annexes (one on my laptop and one on my work pc) The are using nearly the same files with some files existing only on the pc but not on the laptop as they are too large for the laptop. How do I get the two into sync. I found the manual in the tips section but that will create normal git-annexes with all files linked only. However, I would like to have my exiting file structure unchanged (the way the assistant does it by default)
+
+any ideas?
+
+Gregor
diff --git a/doc/forum/howto_to_link_to_existing_direct_mode_git-annexes/comment_1_7bd0edaf2352293678f0942aaa885d13._comment b/doc/forum/howto_to_link_to_existing_direct_mode_git-annexes/comment_1_7bd0edaf2352293678f0942aaa885d13._comment
new file mode 100644
index 0000000000..4b3b095c33
--- /dev/null
+++ b/doc/forum/howto_to_link_to_existing_direct_mode_git-annexes/comment_1_7bd0edaf2352293678f0942aaa885d13._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="http://joeyh.name/"
+ ip="71.80.94.56"
+ subject="comment 1"
+ date="2014-02-07T19:11:00Z"
+ content="""
+The webapp will help you do this, go to Add Another Repository -> Share With Your Other Devices. 
+"""]]
diff --git a/doc/forum/howto_update_feed.mdwn b/doc/forum/howto_update_feed.mdwn
new file mode 100644
index 0000000000..5323e92eb3
--- /dev/null
+++ b/doc/forum/howto_update_feed.mdwn
@@ -0,0 +1,14 @@
+I am using the importfeed [1] functionality.
+
+How am I supposed to update the feed/feeds?
+
+I understand that running
+
+    cd annex; git annex importfeed http://url/to/podcast
+
+a second time will 'do the right thing'.  But that is cumbersome as I have to know the url again.  Is there sth like git annex updatefeeds?
+
+
+
+
+[1] http://joeyh.name/blog/entry/git-annex_as_a_podcatcher/
diff --git a/doc/forum/howto_update_feed/comment_1_bec356619f370a618f19a187d09d2e35._comment b/doc/forum/howto_update_feed/comment_1_bec356619f370a618f19a187d09d2e35._comment
new file mode 100644
index 0000000000..fa87e3494d
--- /dev/null
+++ b/doc/forum/howto_update_feed/comment_1_bec356619f370a618f19a187d09d2e35._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="http://joeyh.name/"
+ ip="4.154.0.63"
+ subject="comment 1"
+ date="2013-08-26T18:11:07Z"
+ content="""
+I have purposefully kept a list of feeds out of git-annex. I handle this by having a `feeds` file, which I check into git. Then a cron job runs: `xargs git-annex importfeed < feeds`
+"""]]
diff --git a/doc/forum/howto_update_feed/comment_2_84dfb80ba3db8d41164eb97022becae3._comment b/doc/forum/howto_update_feed/comment_2_84dfb80ba3db8d41164eb97022becae3._comment
new file mode 100644
index 0000000000..4f28d8f4ef
--- /dev/null
+++ b/doc/forum/howto_update_feed/comment_2_84dfb80ba3db8d41164eb97022becae3._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="ringprince"
+ ip="134.76.140.110"
+ subject="comment 2"
+ date="2013-08-26T19:04:21Z"
+ content="""
+Thanks for that.  I'll do the same ;-)
+"""]]
diff --git a/doc/forum/howto_update_feed/comment_3_20166db298c10074e062aecad59ffd71._comment b/doc/forum/howto_update_feed/comment_3_20166db298c10074e062aecad59ffd71._comment
new file mode 100644
index 0000000000..73f48ac9b0
--- /dev/null
+++ b/doc/forum/howto_update_feed/comment_3_20166db298c10074e062aecad59ffd71._comment
@@ -0,0 +1,20 @@
+[[!comment format=mdwn
+ username="http://a-or-b.myopenid.com/"
+ ip="203.45.2.230"
+ subject="comment 3"
+ date="2013-08-28T01:32:38Z"
+ content="""
+One thing I do slightly differently is put comments in my ```feeds``` file so I know what feed is what...
+
+    # NPR - Wait Wait... Don't Tell Me!
+    http://www.npr.org/rss/podcast.php?id=35
+    # NPR - Car Talk
+    http://www.npr.org/rss/podcast.php?id=510208
+
+
+Then I use 
+
+    grep -v '^#' feeds| xargs git-annex importfeed --relaxed
+
+I reckon that makes maintenance easier.
+"""]]
diff --git a/doc/forum/howto_update_feed/comment_4_f040e31b763fc9a7aa092442b4d6b8e8._comment b/doc/forum/howto_update_feed/comment_4_f040e31b763fc9a7aa092442b4d6b8e8._comment
new file mode 100644
index 0000000000..f7a35d1b65
--- /dev/null
+++ b/doc/forum/howto_update_feed/comment_4_f040e31b763fc9a7aa092442b4d6b8e8._comment
@@ -0,0 +1,20 @@
+[[!comment format=mdwn
+ username="arand"
+ ip="130.243.226.21"
+ subject="comment 4"
+ date="2013-09-01T18:06:51Z"
+ content="""
+Yet another solution, keeping it all in one script
+
+    #!/bin/sh
+    
+    while IFS= read line
+    do
+            test -n \"${line%%#*}\" && echo git annex importfeed --relaxed \"$line\"
+    done <: Data.ByteString.hGetLine: timeout (Operation timed out)"
+
+    failed                  
+
+    copy newdir/file2.tgz (checking my-s3-remote...) (to my-s3-remote...) 
+
+    15%        2.3MB/s 3h40m
+
+      ErrorMisc ": Data.ByteString.hGetLine: resource vanished (Connection reset by peer)"
+
+    failed                  
+
+    copy newdir/file3.tgz (checking my-s3-remote...) (checking my-s3-remote...) (checking my-s3-remote...) (checking my-s3-remote...) (checking my-s3-remote...) (checking my-s3-remote...) (checking my-s3-remote...) ok
+    
+One common cause of this is if my Internet connection is intermittent. But even when my connection seems steady, it can happen. I'm willing to chalk that up to network problems elsewhere though.
+
+If I keep just hitting "up enter" to re-execute the command each time it fails, eventually everything gets up there.
+
+But this can actually take weeks, because often uploading these big files, I'll let it go overnight, and then wake up every morning and find out with dismay that it has failed again.
+
+My questions:
+
+- Is there a way to make it automatically retry? I am sure that upon any of these errors, an immediate automatic reply would amost assuredly work.
+
+- If not, is there at least a way to make it pick up where it left off? Even though I'm using chunks, it seems to start the file over again.
+
+Thanks.
diff --git a/doc/forum/is_there_a_way_to_automatically_retry_when_special_remotes_fail__63__/comment_1_82820ce711b87b092a21e31fd381ffce._comment b/doc/forum/is_there_a_way_to_automatically_retry_when_special_remotes_fail__63__/comment_1_82820ce711b87b092a21e31fd381ffce._comment
new file mode 100644
index 0000000000..2ab0a8ebb8
--- /dev/null
+++ b/doc/forum/is_there_a_way_to_automatically_retry_when_special_remotes_fail__63__/comment_1_82820ce711b87b092a21e31fd381ffce._comment
@@ -0,0 +1,19 @@
+[[!comment format=mdwn
+ username="joey"
+ subject="""comment 1"""
+ date="2015-01-05T20:23:40Z"
+ content="""
+The git-annex assistant will automatically retry uploads/downloads, if you
+want to use it.
+
+Or, you can use a simple loop until git-annex succeeds:
+
+	while ! git-annex annex copy newdir/* --to my-s3-remote ; do echo retry ; done
+
+As to picking up where it left off, make sure you have a recent
+release of git-annex, and then you can enable chunking for a S3
+remote. git-annex can then resume an upload or download starting
+at the next chunk that needs to be transferred.
+
+See [[chunking]].
+"""]]
diff --git a/doc/forum/is_there_a_way_to_automatically_retry_when_special_remotes_fail__63__/comment_2_160e80583b647eae53aa120ca734e34a._comment b/doc/forum/is_there_a_way_to_automatically_retry_when_special_remotes_fail__63__/comment_2_160e80583b647eae53aa120ca734e34a._comment
new file mode 100644
index 0000000000..cb2188efdd
--- /dev/null
+++ b/doc/forum/is_there_a_way_to_automatically_retry_when_special_remotes_fail__63__/comment_2_160e80583b647eae53aa120ca734e34a._comment
@@ -0,0 +1,9 @@
+[[!comment format=mdwn
+ username="http://digiuser.livejournal.com/"
+ subject="Is there a way to make this upload all files that have not been uploaded yet?"
+ date="2015-01-08T02:37:51Z"
+ content="""
+Let's say I want my S3 repo to have all files in it that are in my current repo. Is there a variation of this while loop that will keep uploading until it's done?
+
+I recognize I could use annex assistant but let's say I just want to do it this way.
+"""]]
diff --git a/doc/forum/is_there_a_way_to_automatically_retry_when_special_remotes_fail__63__/comment_3_6d81a41edf052e310e1f80cc6add3c09._comment b/doc/forum/is_there_a_way_to_automatically_retry_when_special_remotes_fail__63__/comment_3_6d81a41edf052e310e1f80cc6add3c09._comment
new file mode 100644
index 0000000000..05e0cd3e3d
--- /dev/null
+++ b/doc/forum/is_there_a_way_to_automatically_retry_when_special_remotes_fail__63__/comment_3_6d81a41edf052e310e1f80cc6add3c09._comment
@@ -0,0 +1,7 @@
+[[!comment format=mdwn
+ username="http://joeyh.name/"
+ subject="comment 3"
+ date="2015-01-08T18:13:47Z"
+ content="""
+That's what the while loop above does; runs the git-annex command, and retries it until it succeeds.
+"""]]
diff --git a/doc/forum/is_there_a_way_to_only_commit_some_files_in_direct-mode___63__.mdwn b/doc/forum/is_there_a_way_to_only_commit_some_files_in_direct-mode___63__.mdwn
new file mode 100644
index 0000000000..9c76e1d425
--- /dev/null
+++ b/doc/forum/is_there_a_way_to_only_commit_some_files_in_direct-mode___63__.mdwn
@@ -0,0 +1,5 @@
+I would like to be able to use git-annex to not only sync binary files for a project but also to be able to use it more like git, that is adding manually files to index and only commiting that.
+I think it is still possible in indirect-mode to manage files like I want, but as I am stuck in windows, I would like to be able to do that in direct-mode...
+Due to the safety measures taken to avoid doing bad things in direct-mode, it doesn't seem possible to commit (and push) files without syncing the whole repository.
+
+Is it currently possible to do what I would like to do, or will it be possible in the future ?
diff --git a/doc/forum/is_there_a_way_to_only_commit_some_files_in_direct-mode___63__/comment_1_804e43111ee97dff15e49e50b6c29d91._comment b/doc/forum/is_there_a_way_to_only_commit_some_files_in_direct-mode___63__/comment_1_804e43111ee97dff15e49e50b6c29d91._comment
new file mode 100644
index 0000000000..9a75526146
--- /dev/null
+++ b/doc/forum/is_there_a_way_to_only_commit_some_files_in_direct-mode___63__/comment_1_804e43111ee97dff15e49e50b6c29d91._comment
@@ -0,0 +1,14 @@
+[[!comment format=mdwn
+ username="http://joeyh.name/"
+ ip="209.250.56.87"
+ subject="comment 1"
+ date="2013-12-12T17:01:50Z"
+ content="""
+You can `git annex add` some files and leave others not added, and `git annex sync` will commit only the added ones. 
+
+What is missing is a way to have changed some files that were added, and avoid `git annex sync` committing them. 
+
+There is no technical reason this cannot be supported in direct mode, there is just no user interface built to do it. 
+
+One way is to temporarily bypass the direct mode guard, by using the appropriate git command line option to set core.bare=false temporarily while committing files. Note that it's then up to you to avoid staging the entire content of the file into the index (so no `git commit -a` etc).
+"""]]
diff --git a/doc/forum/is_there_a_way_to_only_commit_some_files_in_direct-mode___63__/comment_2_299ad6cd0225daa166d36af3726a9ef2._comment b/doc/forum/is_there_a_way_to_only_commit_some_files_in_direct-mode___63__/comment_2_299ad6cd0225daa166d36af3726a9ef2._comment
new file mode 100644
index 0000000000..0fb4a00977
--- /dev/null
+++ b/doc/forum/is_there_a_way_to_only_commit_some_files_in_direct-mode___63__/comment_2_299ad6cd0225daa166d36af3726a9ef2._comment
@@ -0,0 +1,14 @@
+[[!comment format=mdwn
+ username="Renaud"
+ ip="182.171.224.143"
+ subject="comment 2"
+ date="2013-12-13T06:16:12Z"
+ content="""
+Thanks for the reply!
+
+I tried to commit manually but I got confused by the (recently introduced?) annex/direct/master branch and I am not sure which branch I need to commit/merge/pull/push my changes to.
+I also got blocked when pulling from origin fail due to file changes needed to be overwritten.
+I see git-annex cleverly merges those in .git/annex/merge folder so I could try doing the same but I am not too confortable doing a large part of git-annex's job...
+
+Is there a simpler way to sync with a remote repository?
+"""]]
diff --git a/doc/forum/known_and_local_annex_keys.mdwn b/doc/forum/known_and_local_annex_keys.mdwn
new file mode 100644
index 0000000000..7dcef8693e
--- /dev/null
+++ b/doc/forum/known_and_local_annex_keys.mdwn
@@ -0,0 +1,14 @@
+I have a direct repository with the assistant running (v4.20131002). The repo is in the backup group and should be grabbing every known annex file. Yet `git annex status` says:
+
+    local annex keys: 1386
+    local annex size: 94.53 gigabytes
+    known annex keys: 1387
+    known annex size: 94.53 gigabytes
+
+As you can seem there is one more known annex key than there are local ones. Neither `git annex get` nor `git annex sync` changes the numbers. According to `tree` the repo dir contains exactly 1387 files, which means that the missing file must exist on disk. 
+
+1) How do I find out which is the file in question? How do I get it synchronized? 
+
+Yesterday there were 1376 known annex keys. Today there are 1387. For some reason the number of keys went up by 11, and I would like to know how and why. The log suggests that a couple of files have been added from inside the repo directory, but I am positive that these files already existed in the annex directory yesterday, and that I ran `git annex sync` manually without the number of local and/or known keys going up.
+
+2) Why weren't the files synced before? Is there a way to find out whether they were or were not part of the annex yesterday - maybe list all files sorted by the date they were added? 
diff --git a/doc/forum/known_and_local_annex_keys/comment_1_3cb4828dc7116e4cf49e119f055ae9a3._comment b/doc/forum/known_and_local_annex_keys/comment_1_3cb4828dc7116e4cf49e119f055ae9a3._comment
new file mode 100644
index 0000000000..24a5ef03e4
--- /dev/null
+++ b/doc/forum/known_and_local_annex_keys/comment_1_3cb4828dc7116e4cf49e119f055ae9a3._comment
@@ -0,0 +1,20 @@
+[[!comment format=mdwn
+ username="http://joeyh.name/"
+ ip="209.250.56.47"
+ subject="comment 1"
+ date="2013-10-28T16:56:20Z"
+ content="""
+I think you're stressing way too much about a number.
+
+It's quite possible for the known and local key counts to diverge while no data is missing. For example:
+
+
    +joey@darkstar:~/tmp/test>touch foo
+joey@darkstar:~/tmp/test>touch bar
+joey@darkstar:~/tmp/test>git annex add foo bar
+
+local annex keys: 1
+known annex keys: 2
+
    
+
+"""]]
diff --git a/doc/forum/known_and_local_annex_keys/comment_2_68f20c881eafe986694bde10571cf1c0._comment b/doc/forum/known_and_local_annex_keys/comment_2_68f20c881eafe986694bde10571cf1c0._comment
new file mode 100644
index 0000000000..944ec9cafc
--- /dev/null
+++ b/doc/forum/known_and_local_annex_keys/comment_2_68f20c881eafe986694bde10571cf1c0._comment
@@ -0,0 +1,10 @@
+[[!comment format=mdwn
+ username="RaspberryPie"
+ ip="77.247.181.165"
+ subject="comment 2"
+ date="2013-10-28T18:31:21Z"
+ content="""
+I don't think I'm stressing too much. Rather, I'm inquiring about why git-annex is giving me wrong (or at least unclear) output that makes me worry about losing data. I plan to trust git-annex with a lot of important files, and in order to do that I want to make sure it's trustworthy. And that's where the inaccurate key numbers get in the way.
+
+So why do the local and known numbers diverge anyway? Would you consider adjusting the status output to be more accurate, or at least less confusing? I just want to be able to check that every single file in the repo is being backed up.
+"""]]
diff --git a/doc/forum/known_and_local_annex_keys/comment_3_e195a7091a06ce0427bb28aca9a17d04._comment b/doc/forum/known_and_local_annex_keys/comment_3_e195a7091a06ce0427bb28aca9a17d04._comment
new file mode 100644
index 0000000000..fd9228b4f0
--- /dev/null
+++ b/doc/forum/known_and_local_annex_keys/comment_3_e195a7091a06ce0427bb28aca9a17d04._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="http://joeyh.name/"
+ ip="209.250.56.47"
+ subject="comment 3"
+ date="2013-10-28T19:06:00Z"
+ content="""
+I have adjusted the labels to be more clear.
+"""]]
diff --git a/doc/forum/known_and_local_annex_keys/comment_4_d81f0bf7465832cb676fd89f5d53ec18._comment b/doc/forum/known_and_local_annex_keys/comment_4_d81f0bf7465832cb676fd89f5d53ec18._comment
new file mode 100644
index 0000000000..fccc576d30
--- /dev/null
+++ b/doc/forum/known_and_local_annex_keys/comment_4_d81f0bf7465832cb676fd89f5d53ec18._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="RaspberryPie"
+ ip="77.247.181.165"
+ subject="comment 4"
+ date="2013-10-28T19:24:12Z"
+ content="""
+Thanks, I'll check it out. And if you ever got around to introduce a means that can answer the above-mentioned integrity questions, that'd be really awesome.
+"""]]
diff --git a/doc/forum/library__44___backup_and_music_player.mdwn b/doc/forum/library__44___backup_and_music_player.mdwn
new file mode 100644
index 0000000000..024899885a
--- /dev/null
+++ b/doc/forum/library__44___backup_and_music_player.mdwn
@@ -0,0 +1,11 @@
+Hello, I'm not sure on how to proceed with a fairly simple workflow. I keep my music library in a debian box with plenty of space. I run the plex media server on it so it should be a direct repository to avoid strange player behavior.
+I also perform a monthly (more or less) backup on an external hard drive. This should be simple too, just add a remote and sync.
+My problem now is: I have also a portable music player that I want to keep in sync with the music library. Not the whole music library but only a small subset. This needs to be a direct repository too (the best would be also without the .git/ folder but I don't believe it's possible). How to create a small subset of files to keep in sync with a direct repository? I've tried with branches but they are not simple to use in direct mode.
+
+So, to recap: a (direct) main repository, a standard backup repository and another (direct) repository with only a subset of files.
+
+Anyone with a similar workflow wants to share some tips?
+
+Thanks,
+
+F
diff --git a/doc/forum/library__44___backup_and_music_player/comment_1_a6e38a1678c433e5a5d96afae4ae4332._comment b/doc/forum/library__44___backup_and_music_player/comment_1_a6e38a1678c433e5a5d96afae4ae4332._comment
new file mode 100644
index 0000000000..edf4c4e6a7
--- /dev/null
+++ b/doc/forum/library__44___backup_and_music_player/comment_1_a6e38a1678c433e5a5d96afae4ae4332._comment
@@ -0,0 +1,26 @@
+[[!comment format=mdwn
+ username="joey"
+ subject="""comment 1"""
+ date="2017-04-05T15:52:30Z"
+ content="""
+Good question! I don't have a perfect solution for this, 
+because what you really probably want is to be able to `git annex copy`
+files to the music player and have only the present files show up in
+its repository.
+
+One approach is to use a [[directory_special_remote|special_remotes/directory]]
+on the music player. This way, only present files will show up there,
+but they'll have git-annex's internal filenames. If the music player
+doesn't care about filenames, this can work ok.
+
+Another approach is to use a v6 git-annex repository
+on the media player, and use [[git-annex-adjust]] to generate
+a version of a branch where all files that are present are unlocked
+(and thus regular files with the expected filenames). Files
+that are not present will still show up in the working tree though,
+either as symlinks or as the small text files git uses when a filesystem
+does not support symlinks. So this is much like direct mode, but it has
+the benefit that you can use all the normal git branching features
+to create a trimmed down branch for the media player with just the files
+you want.
+"""]]
diff --git a/doc/forum/linux_standalone_tarballs.mdwn b/doc/forum/linux_standalone_tarballs.mdwn
new file mode 100644
index 0000000000..1c6380a7d8
--- /dev/null
+++ b/doc/forum/linux_standalone_tarballs.mdwn
@@ -0,0 +1 @@
+Just saw that there is now a linux standalone tarball, I've a SL5 (RHEL5) based machine which I can churn out git-annex binaries if there is interest.
diff --git a/doc/forum/linux_standalone_tarballs/comment_1_5c3ceb845a45e50784f7098bfbf94df1._comment b/doc/forum/linux_standalone_tarballs/comment_1_5c3ceb845a45e50784f7098bfbf94df1._comment
new file mode 100644
index 0000000000..64a37f366f
--- /dev/null
+++ b/doc/forum/linux_standalone_tarballs/comment_1_5c3ceb845a45e50784f7098bfbf94df1._comment
@@ -0,0 +1,13 @@
+[[!comment format=mdwn
+ username="http://joeyh.name/"
+ ip="4.154.0.149"
+ subject="comment 1"
+ date="2012-09-29T16:03:36Z"
+ content="""
+I'll be curious to see if the standalone tarball works on such a system.. it might need a newer libc
+than RHEL5 has.
+
+Conversely, that'd make it a good system to build the tarball on, so it's built to work with a nice old libc. :) 
+
+I have a machine where I could put chroots to do both 64 bit and 32 bit builds, but I don't know if I'll automate it there. Too lazy..
+"""]]
diff --git a/doc/forum/local_pairing_with_2_mac.mdwn b/doc/forum/local_pairing_with_2_mac.mdwn
new file mode 100644
index 0000000000..814ecc123f
--- /dev/null
+++ b/doc/forum/local_pairing_with_2_mac.mdwn
@@ -0,0 +1,27 @@
+Hey !
+I am trying to find ways to use git annex as a tool to share big binary files for our development projects (such as PSD files).
+Our team has artists that will not use command line tools so I have been testing git-annex assistant with a lot of hope.
+However all the tests I am doing are just failing. °.° '
+
+in this thread I would like to focus on what would seem to me as an easy first approach : local pairing.
+
+So my team and I are on mac. I am just trying to sync 2 computers on the same network. But the process is stuck on the "local pairing in process" step, just after I enter the secret phrase.
+
+I have enabled sshd (preferences -> remote login) on both computers, checked the firewall to authorize git-annex, so I am probably missing something here. 
+the logs on my computer does not say anything usefull ([2013-12-04 22:04:34 CET] main: Pairing in progress) but on the other computer I have this :
+
+git-annex: ssh-keygen failed
+[2013-12-04 22:04:33 CET] PairListener: utku@MacBook-Air-de-utku.local:~/Documents/git-test is sending a pair request.
+dyld: lazy symbol binding failed: Symbol not found: ___strlcpy_chk
+ Referenced from: /Applications/git-annex.app/Contents/MacOS/bundle/ssh-keygen
+ Expected in: /usr/lib/libSystem.B.dylib
+
+dyld: Symbol not found: ___strlcpy_chk
+ Referenced from: /Applications/git-annex.app/Contents/MacOS/bundle/ssh-keygen
+ Expected in: /usr/lib/libSystem.B.dylib
+
+git-annex: ssh-keygen failed
+
+Don't know if that helps. I don't know which info I need to provide.
+
+cheers
diff --git a/doc/forum/local_pairing_with_2_mac/comment_10_508585e72c81d197a9a1e193c25a702a._comment b/doc/forum/local_pairing_with_2_mac/comment_10_508585e72c81d197a9a1e193c25a702a._comment
new file mode 100644
index 0000000000..1d69f41fda
--- /dev/null
+++ b/doc/forum/local_pairing_with_2_mac/comment_10_508585e72c81d197a9a1e193c25a702a._comment
@@ -0,0 +1,15 @@
+[[!comment format=mdwn
+ username="https://www.google.com/accounts/o8/id?id=AItOawnfC02DTOZygFDDvvg0maRciYYMhEIFgkw"
+ nickname="Utku"
+ subject="comment 10"
+ date="2013-12-15T09:51:28Z"
+ content="""
+unfortunately, it is not working on the computer where ssh was not working
+I click an got annex and nothing seems to open
+I try to go on the system logs but I see nothing really obvious. The only log that may be related would be : 
+15/12/13 10:46:23,230 Dock[166]: find_shared_window: WID 21425
+15/12/13 10:46:23,230 Dock[166]: CGSGetWindowTags: Invalid window 0x53b1
+
+if you tell me where to look at I can give you something more useful. 
+
+"""]]
diff --git a/doc/forum/local_pairing_with_2_mac/comment_1_6ca4fed183340a2902d1d4d91284b772._comment b/doc/forum/local_pairing_with_2_mac/comment_1_6ca4fed183340a2902d1d4d91284b772._comment
new file mode 100644
index 0000000000..25c97805b0
--- /dev/null
+++ b/doc/forum/local_pairing_with_2_mac/comment_1_6ca4fed183340a2902d1d4d91284b772._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="http://joeyh.name/"
+ ip="209.250.56.87"
+ subject="comment 1"
+ date="2013-12-12T17:47:30Z"
+ content="""
+What version of OSX is the failing computer running? What version of git-annex? How did you install git-annex on it?
+"""]]
diff --git a/doc/forum/local_pairing_with_2_mac/comment_2_7c2b3d0a8a69c7056508f8ec73ebefcd._comment b/doc/forum/local_pairing_with_2_mac/comment_2_7c2b3d0a8a69c7056508f8ec73ebefcd._comment
new file mode 100644
index 0000000000..03b06e45ae
--- /dev/null
+++ b/doc/forum/local_pairing_with_2_mac/comment_2_7c2b3d0a8a69c7056508f8ec73ebefcd._comment
@@ -0,0 +1,13 @@
+[[!comment format=mdwn
+ username="https://www.google.com/accounts/o8/id?id=AItOawnfC02DTOZygFDDvvg0maRciYYMhEIFgkw"
+ nickname="Utku"
+ subject="comment 2"
+ date="2013-12-12T20:08:50Z"
+ content="""
+hello
+both computers are on mac os maverick
+latest version of git annex, and we used the git annex assistant dmg found on the install section...
+I am on a personnal home router so I don't think there is any port that might be blocked by a firewall for local connections...
+
+ 
+"""]]
diff --git a/doc/forum/local_pairing_with_2_mac/comment_3_07fa468aac1288e770487973052bccea._comment b/doc/forum/local_pairing_with_2_mac/comment_3_07fa468aac1288e770487973052bccea._comment
new file mode 100644
index 0000000000..bab5009f46
--- /dev/null
+++ b/doc/forum/local_pairing_with_2_mac/comment_3_07fa468aac1288e770487973052bccea._comment
@@ -0,0 +1,12 @@
+[[!comment format=mdwn
+ username="http://joeyh.name/"
+ ip="209.250.56.87"
+ subject="comment 3"
+ date="2013-12-12T20:37:55Z"
+ content="""
+It's not a firewall or network problem, what you pasted earlier shows that the ssh-keygen command bundled in the git annex dmg is trying to use a symbol that is not present in your libSystem library.
+
+I am puzzled why ssh-keygen apparently works on one of your macs and not on the other, if they indeed both have the same build of git-annex installed.
+
+Please paste output of `otool -L /Applications/git-annex.app/Contents/MacOS/bundle/ssh-keygen` on the mac with the problem.
+"""]]
diff --git a/doc/forum/local_pairing_with_2_mac/comment_4_dc0494213d2b57b5b9a489b096a5b8d0._comment b/doc/forum/local_pairing_with_2_mac/comment_4_dc0494213d2b57b5b9a489b096a5b8d0._comment
new file mode 100644
index 0000000000..f3f25504aa
--- /dev/null
+++ b/doc/forum/local_pairing_with_2_mac/comment_4_dc0494213d2b57b5b9a489b096a5b8d0._comment
@@ -0,0 +1,31 @@
+[[!comment format=mdwn
+ username="https://www.google.com/accounts/o8/id?id=AItOawnfC02DTOZygFDDvvg0maRciYYMhEIFgkw"
+ nickname="Utku"
+ subject="comment 4"
+ date="2013-12-12T20:58:05Z"
+ content="""
+so as I don't really know which mac has a problem, here is what I get on both computers (looks they are quite the same): 
+
+/Applications/git-annex.app/Contents/MacOS/bundle/ssh-keygen:
+	/System/Library/Frameworks/CoreFoundation.framework/Versions/A/CoreFoundation (compatibility version 150.0.0, current version 855.0.0)
+	/System/Library/Frameworks/Kerberos.framework/Versions/A/Kerberos (compatibility version 5.0.0, current version 6.0.0)
+	/System/Library/Frameworks/OpenDirectory.framework/Versions/A/OpenDirectory (compatibility version 1.0.0, current version 1.0.0)
+	/System/Library/Frameworks/Security.framework/Versions/A/Security (compatibility version 1.0.0, current version 55456.0.0)
+	@executable_path/O (compatibility version 1.0.0, current version 1.0.0)
+	@executable_path/P (compatibility version 3.0.0, current version 3.0.0)
+	@executable_path/B (compatibility version 1.0.0, current version 1.2.5)
+	@executable_path/L (compatibility version 1.0.0, current version 1.0.0)
+	/usr/lib/libSystem.B.dylib (compatibility version 1.0.0, current version 1197.1.1)
+
+
+/Applications/git-annex.app/Contents/MacOS/bundle/ssh-keygen:
+       /System/Library/Frameworks/CoreFoundation.framework/Versions/A/CoreFoundation (compatibility version 150.0.0, current version 855.0.0)
+       /System/Library/Frameworks/Kerberos.framework/Versions/A/Kerberos (compatibility version 5.0.0, current version 6.0.0)
+       /System/Library/Frameworks/OpenDirectory.framework/Versions/A/OpenDirectory (compatibility version 1.0.0, current version 1.0.0)
+       /System/Library/Frameworks/Security.framework/Versions/A/Security (compatibility version 1.0.0, current version 55456.0.0)
+       @executable_path/O (compatibility version 1.0.0, current version 1.0.0)
+       @executable_path/P (compatibility version 3.0.0, current version 3.0.0)
+       @executable_path/B (compatibility version 1.0.0, current version 1.2.5)
+       @executable_path/L (compatibility version 1.0.0, current version 1.0.0)
+       /usr/lib/libSystem.B.dylib (compatibility version 1.0.0, current version 1197.1.1)
+"""]]
diff --git a/doc/forum/local_pairing_with_2_mac/comment_5_bcac18e137c00d4279774dec51963289._comment b/doc/forum/local_pairing_with_2_mac/comment_5_bcac18e137c00d4279774dec51963289._comment
new file mode 100644
index 0000000000..0639068661
--- /dev/null
+++ b/doc/forum/local_pairing_with_2_mac/comment_5_bcac18e137c00d4279774dec51963289._comment
@@ -0,0 +1,10 @@
+[[!comment format=mdwn
+ username="http://joeyh.name/"
+ ip="209.250.56.87"
+ subject="comment 5"
+ date="2013-12-12T21:40:16Z"
+ content="""
+Well, I was wrong, we don't know only one of the 2 has the problem. The other one will probably not have run ssh-keygen yet.
+
+Can you run `/Applications/git-annex.app/Contents/MacOS/bundle/ssh-keygen --help` on both of them and see if it manages to print its help, or fails with an error?
+"""]]
diff --git a/doc/forum/local_pairing_with_2_mac/comment_6_76b62eafda1ecbf88abe288cbe778e17._comment b/doc/forum/local_pairing_with_2_mac/comment_6_76b62eafda1ecbf88abe288cbe778e17._comment
new file mode 100644
index 0000000000..e9cd8cc327
--- /dev/null
+++ b/doc/forum/local_pairing_with_2_mac/comment_6_76b62eafda1ecbf88abe288cbe778e17._comment
@@ -0,0 +1,17 @@
+[[!comment format=mdwn
+ username="https://www.google.com/accounts/o8/id?id=AItOawnfC02DTOZygFDDvvg0maRciYYMhEIFgkw"
+ nickname="Utku"
+ subject="comment 6"
+ date="2013-12-12T21:50:15Z"
+ content="""
+ok so it is ok on my computer, but here is what I got on the other one : 
+dyld: lazy symbol binding failed: Symbol not found: ___strlcpy_chk
+ Referenced from: /Applications/git-annex.app/Contents/MacOS/bundle/ssh-keygen
+ Expected in: /usr/lib/libSystem.B.dylib
+
+dyld: Symbol not found: ___strlcpy_chk
+ Referenced from: /Applications/git-annex.app/Contents/MacOS/bundle/ssh-keygen
+ Expected in: /usr/lib/libSystem.B.dylib
+
+Trace/BPT trap: 5
+"""]]
diff --git a/doc/forum/local_pairing_with_2_mac/comment_7_13fe788f2e9a823ad2e4844f114675a7._comment b/doc/forum/local_pairing_with_2_mac/comment_7_13fe788f2e9a823ad2e4844f114675a7._comment
new file mode 100644
index 0000000000..883cb4e143
--- /dev/null
+++ b/doc/forum/local_pairing_with_2_mac/comment_7_13fe788f2e9a823ad2e4844f114675a7._comment
@@ -0,0 +1,10 @@
+[[!comment format=mdwn
+ username="http://joeyh.name/"
+ ip="209.250.56.87"
+ subject="comment 7"
+ date="2013-12-12T21:53:56Z"
+ content="""
+So it does only fail on one machine and not the other. I wonder what is different?
+
+Can you also try running `/Volumes/git-annex/git-annex.app/Contents/MacOS/bundle/cp` on both? (I think it uses the same symbol.)
+"""]]
diff --git a/doc/forum/local_pairing_with_2_mac/comment_8_f81b454e9cfc14dcb33148798be55de3._comment b/doc/forum/local_pairing_with_2_mac/comment_8_f81b454e9cfc14dcb33148798be55de3._comment
new file mode 100644
index 0000000000..b4af1ce699
--- /dev/null
+++ b/doc/forum/local_pairing_with_2_mac/comment_8_f81b454e9cfc14dcb33148798be55de3._comment
@@ -0,0 +1,16 @@
+[[!comment format=mdwn
+ username="https://www.google.com/accounts/o8/id?id=AItOawnfC02DTOZygFDDvvg0maRciYYMhEIFgkw"
+ nickname="Utku"
+ subject="comment 8"
+ date="2013-12-12T22:03:54Z"
+ content="""
+you mean /Applications/git-annex.app/Contents/MacOS/bundle/cp ?
+
+on both computers it prints : 
+usage: cp [-R [-H | -L | -P]] [-fi | -n] [-apvX] source_file target_file
+       cp [-R [-H | -L | -P]] [-fi | -n] [-apvX] source_file ... target_directory
+
+
+/Volumes/git-annex/git-annex.app/Contents/MacOS/bundle/cp doesn't work
+
+"""]]
diff --git a/doc/forum/local_pairing_with_2_mac/comment_9_f7a1ce9627ebfc854dfde2f6c924db80._comment b/doc/forum/local_pairing_with_2_mac/comment_9_f7a1ce9627ebfc854dfde2f6c924db80._comment
new file mode 100644
index 0000000000..deaf9b68c4
--- /dev/null
+++ b/doc/forum/local_pairing_with_2_mac/comment_9_f7a1ce9627ebfc854dfde2f6c924db80._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="http://joeyh.name/"
+ ip="209.250.56.87"
+ subject="comment 9"
+ date="2013-12-13T18:52:40Z"
+ content="""
+As a workaround, I have removed ssh-keygen and ssh from the OSX DMG. AIUI, OSX ships ssh by default anyway.
+"""]]
diff --git a/doc/forum/local_subtree_and_broken_symlinks.mdwn b/doc/forum/local_subtree_and_broken_symlinks.mdwn
new file mode 100644
index 0000000000..de6dd6e491
--- /dev/null
+++ b/doc/forum/local_subtree_and_broken_symlinks.mdwn
@@ -0,0 +1,21 @@
+Here's a simple example on a repository with three branches, where we'll be adding images-annex as a subtree into master.
+
+    $ git branch
+        git-annex
+        images-annex
+      * master
+    $ git subtree add --squash --prefix=images/ images-annex
+      Added dir 'images'
+    $ ls
+      FILE_A        FILE_B        images/
+
+...checkout images-annex, make changes, commit...
+
+    $ git checkout master
+    $ git subtree pull --squash --prefix=images/ . images-annex
+      From .
+      * branch images-annex -> FETCH_HEAD
+      Merge made by the 'recursive' strategy.
+      ...(files created/modified/etc)
+
+I have tried a few different methods for merging the subtree in and so far have not been able to keep git-annex links up to date. Running `git-annex fix .` does what it's supposed to but then git sees everything as modified. Is this entirely the expected behavior because of the --prefix? I have not used subtrees much before but the model appears to be very helpful for what I'm trying to do.
diff --git a/doc/forum/local_subtree_and_broken_symlinks/comment_1_779cc4e49cb4da8aea7f5743e6257f21._comment b/doc/forum/local_subtree_and_broken_symlinks/comment_1_779cc4e49cb4da8aea7f5743e6257f21._comment
new file mode 100644
index 0000000000..e6ccb814f0
--- /dev/null
+++ b/doc/forum/local_subtree_and_broken_symlinks/comment_1_779cc4e49cb4da8aea7f5743e6257f21._comment
@@ -0,0 +1,10 @@
+[[!comment format=mdwn
+ username="http://joeyh.name/"
+ ip="24.159.78.125"
+ subject="comment 1"
+ date="2014-07-30T14:55:16Z"
+ content="""
+Known bug: [[bugs/Git_annexed_files_symlink_are_wrong_when_submodule_is_not_in_the_same_path]]
+
+I don't think there's much likelyhood of a fix though. Using direct mode probably works around the problem. Or you can use something like myrepos instead of git subtrees.
+"""]]
diff --git a/doc/forum/location_tracking_cleanup.mdwn b/doc/forum/location_tracking_cleanup.mdwn
new file mode 100644
index 0000000000..7e2e230af7
--- /dev/null
+++ b/doc/forum/location_tracking_cleanup.mdwn
@@ -0,0 +1,24 @@
+I recently started experimenting with git annex, adding files that I've had
+floating across several computers to repositories. During the testing I had
+a few occasions where I wrecked a repository somehow, and decided to wipe it
+and start anew (at this point there was no important files in them so I thought
+this is the easiest way). Well, as it turns out this interacts badly with location
+tracking, since now `git annex whereis` shows files residing in all those destroyed
+repositories, all having same names as some existing repositories. This makes it hard
+to follow whether a repo actually has a file, or was the file only seen in some dead
+repo with the same name.
+
+I planned on cleaning this up by looking up the UUIDs of the now stable, existing
+repos and untrusting all the dead copies (they should effectively disappear from
+git annex´s output then, right?), but I didn't find an easy way to look up the UUID
+of the current repository (maybe this could be included in `git annex status`?)
+I also noticed that untrust cannot remove the trust based on the UUID -- if I try
+it I simply get "there is no git remote named "11908472-...", so I guess untrust
+works with git remote names, which I find a bit confusing, since trust.log logs the
+trust levels based on the UUID. I could just write into trust.log manually, but I'm
+unsure how the changes would get propagated.
+
+What should I do? As a related wishlist item I would ask for some additional mechanisms
+for purging known-dead repositories from the location tracking database. And the ability
+to look up the UUID of the current repo, and to use the UUID to specify repositories when
+applicable (untrust and describe maybe).
diff --git a/doc/forum/location_tracking_cleanup/comment_1_7d6319e8c94dfe998af9cfcbf170efb2._comment b/doc/forum/location_tracking_cleanup/comment_1_7d6319e8c94dfe998af9cfcbf170efb2._comment
new file mode 100644
index 0000000000..8915ea3518
--- /dev/null
+++ b/doc/forum/location_tracking_cleanup/comment_1_7d6319e8c94dfe998af9cfcbf170efb2._comment
@@ -0,0 +1,10 @@
+[[!comment format=mdwn
+ username="http://joey.kitenet.net/"
+ nickname="joey"
+ subject="comment 1"
+ date="2011-09-30T06:55:34Z"
+ content="""
+Specifying the UUID was supposed to work, I think I broke it a while ago. Fixed now in git.
+
+I'm not sure why you need to look up the UUID of the current repository. You can always refer to the current repository as \".\". Anyway, the UUID of the current repository is in `.git/config`, or use `git config annex.uuid`.
+"""]]
diff --git a/doc/forum/location_tracking_cleanup/comment_2_e7395cb6e01f42da72adf71ea3ebcde4._comment b/doc/forum/location_tracking_cleanup/comment_2_e7395cb6e01f42da72adf71ea3ebcde4._comment
new file mode 100644
index 0000000000..f612e53a4d
--- /dev/null
+++ b/doc/forum/location_tracking_cleanup/comment_2_e7395cb6e01f42da72adf71ea3ebcde4._comment
@@ -0,0 +1,15 @@
+[[!comment format=mdwn
+ username="https://www.google.com/accounts/o8/id?id=AItOawncBlzaDI248OZGjKQMXrLVQIx4XrZrzFo"
+ nickname="Perttu"
+ subject="comment 2"
+ date="2011-09-30T11:55:35Z"
+ content="""
+Thanks for the quick reply :)
+
+I wanted to look up the UUID of the current repo so that I can find out which repo is alive from the collection of repos with the same name.
+I could have looked for it in .git/config though, since it's pretty obvious. I just looked into the git-annex branch and didn't find it there.
+Thanks for the tip about using \".\". By the way, could there be some kind of warning about using non-unique names for repos? That would make this
+scenario less likely. Or maybe that is a bad idea given the decentralized nature of git.
+
+By the way, do the trust settings propagate to other repos? If I mark some UUID as untrusted on one computer does it become globally untrusted?
+"""]]
diff --git a/doc/forum/location_tracking_cleanup/comment_3_c15428cec90e969284a5e690fb4b2fde._comment b/doc/forum/location_tracking_cleanup/comment_3_c15428cec90e969284a5e690fb4b2fde._comment
new file mode 100644
index 0000000000..c676b9b615
--- /dev/null
+++ b/doc/forum/location_tracking_cleanup/comment_3_c15428cec90e969284a5e690fb4b2fde._comment
@@ -0,0 +1,10 @@
+[[!comment format=mdwn
+ username="http://joey.kitenet.net/"
+ nickname="joey"
+ subject="comment 3"
+ date="2011-09-30T16:47:27Z"
+ content="""
+`git annex status` now includes a list of all known repositories.
+
+Yes, trust setting propigate on git push/pull like any other git-annex information.
+"""]]
diff --git a/doc/forum/long_paths_on_windows_10_-_workaround.mdwn b/doc/forum/long_paths_on_windows_10_-_workaround.mdwn
new file mode 100644
index 0000000000..b51bcac5ce
--- /dev/null
+++ b/doc/forum/long_paths_on_windows_10_-_workaround.mdwn
@@ -0,0 +1,9 @@
+It seems git-annex has some serious problems with long paths on windows systems. I would like to suggest a possible solution and a simple workaround.
+
+## Possible solution:
+Although by default winapi functions work only with paths up to MAX_PATH, their unicode versions can work with much longer paths, provided that the paths are prefixed with "\\\\?\\". Functions defined in System.Win32.File module (e.g. moveFileEx) call the unicode versions. Solving the length limitation problem might not be as simple as prefixing all arguments with "\\\\?\\", as it does not work with relative paths. I also do not know the git-annex internals so I have no idea how easy would it be to implement, but maybe it could be an easier way than trying to limit the paths' lengths to fit in the archaic MAX_PATH limit. The MSDN page linked below has some more information.
+
+## Workaround:
+It turns out that Windows 10 supports long paths without the "\\\\?\\" prefix, but this setting is by default turned off. According to [this MSDN page](https://msdn.microsoft.com/en-us/library/windows/desktop/aa365247(v=vs.85\).aspx#maxpath) one can enable it by setting HKLM\SYSTEM\CurrentControlSet\Control\FileSystem\LongPathsEnabled to 1 or by using the group policy editor (Computer Configuration > Administrative Templates > System > Filesystem > Enable NTFS long paths). With default settings I was unable to "git annex get" some files in nested directories (it failed with a MoveFileEx error when moving a file from annex\tmp to annex\objects), but after I enabled the long paths option it worked!
+
+This is more of a workaround than a real solution because it requires modifying a global setting which can affect other applications. I also do not know if it solves all path problems with git-annex, but it worked for me so far. I hope this workaround will help others having similar problems.
diff --git a/doc/forum/lost_in_walkthrough.mdwn b/doc/forum/lost_in_walkthrough.mdwn
new file mode 100644
index 0000000000..89f4af0261
--- /dev/null
+++ b/doc/forum/lost_in_walkthrough.mdwn
@@ -0,0 +1,78 @@
+I'm trying to follow the steps of the "walkthrough" but I'm experiencing the following issue: when
+I sync one repository and do "git annex get ." I don't get the files from the other repository.
+Here is the transcript of the steps I followed - I've put them in a script (ga.sh) so I can replay
+them and show them on the shell while executing.
+Basically I have two repositories, "/tmp/a/annex" and "/tmp/b/annex", the second cloned from
+the first. All the other steps are the same as in the walkthrough.
+-----------------------------------
+> bash -x ga.sh 
++ cd /tmp
++ mkdir a
++ mkdir b
++ cd a
++ mkdir annex
++ cd annex
++ git init
+Initialized empty Git repository in /tmp/a/annex/.git/
++ git annex init a
+init a ok
+(Recording state in git...)
++ cd /tmp/b
++ git clone /tmp/a/annex
+Cloning into 'annex'...
+done.
+warning: remote HEAD refers to nonexistent ref, unable to checkout.
+
++ cd annex
++ git annex init b
+init b ok
+(Recording state in git...)
++ git remote add a /tmp/a/annex
++ cd /tmp/a/annex
++ git remote add b /tmp/b/annex
++ dd if=/dev/urandom of=first bs=1024 count=1
+1+0 records in
+1+0 records out
+1024 bytes (1.0 kB) copied, 9.9167e-05 s, 10.3 MB/s
++ dd if=/dev/urandom of=second bs=1024 count=1
+1+0 records in
+1+0 records out
+1024 bytes (1.0 kB) copied, 0.000241635 s, 4.2 MB/s
++ git annex add .
+add first (checksum...) ok
+add second (checksum...) ok
+(Recording state in git...)
++ git commit -am added
+[master (root-commit) 5078564] added
+ 2 files changed, 2 insertions(+)
+ create mode 120000 first
+ create mode 120000 second
++ git mv first e
++ mkdir x
++ git mv second x
++ git commit -m moved
+fix x/second ok
+(Recording state in git...)
+[master 422492d] moved
+ 3 files changed, 1 insertion(+), 1 deletion(-)
+ rename first => e (100%)
+ delete mode 120000 second
+ create mode 120000 x/second
++ cd /tmp/b/annex
++ git annex sync a
+(merging origin/git-annex into git-annex...)
+(Recording state in git...)
+commit  
+ok
+git-annex: no branch is checked out
++ git annex get .
+-------------------
+
+The last "git annex get ." does not retrieve the files in /tmp/a/annex ... why?
+I guess the issue starts when cloning /tmp/a/annex where no commit was done.
+
+Emanuele
+
+PS: I'm using git v1.7.9.5 (ubuntu 12.04) and the latest git-annex static binary
+downloaded a few minutes ago from the git-annex website.
+
diff --git a/doc/forum/lost_in_walkthrough/comment_1_51b703b961ca762f0359e1c169f1ee75._comment b/doc/forum/lost_in_walkthrough/comment_1_51b703b961ca762f0359e1c169f1ee75._comment
new file mode 100644
index 0000000000..a5c574b4a6
--- /dev/null
+++ b/doc/forum/lost_in_walkthrough/comment_1_51b703b961ca762f0359e1c169f1ee75._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="http://joeyh.name/"
+ ip="209.250.56.47"
+ subject="comment 1"
+ date="2013-11-02T19:32:50Z"
+ content="""
+I have fixed `git annex sync` to work even when run in a repository with no master branch. 
+"""]]
diff --git a/doc/forum/lost_in_walkthrough/comment_2_a9de0401a103bdd4401ba2d5983dc54a._comment b/doc/forum/lost_in_walkthrough/comment_2_a9de0401a103bdd4401ba2d5983dc54a._comment
new file mode 100644
index 0000000000..4a8110eb4b
--- /dev/null
+++ b/doc/forum/lost_in_walkthrough/comment_2_a9de0401a103bdd4401ba2d5983dc54a._comment
@@ -0,0 +1,11 @@
+[[!comment format=mdwn
+ username="https://www.google.com/accounts/o8/id?id=AItOawn2MI-Ci-bS9W5NITngsBJiL8LqWYcJ458"
+ nickname="Emanuele"
+ subject="comment 2"
+ date="2013-11-02T23:36:54Z"
+ content="""
+Great thanks! I'm glad my feedback helped improve git-annex!
+
+af6cef
+
+"""]]
diff --git a/doc/forum/lsof_resource_use_problems.mdwn b/doc/forum/lsof_resource_use_problems.mdwn
new file mode 100644
index 0000000000..f15b81fe92
--- /dev/null
+++ b/doc/forum/lsof_resource_use_problems.mdwn
@@ -0,0 +1,42 @@
+When the assistant runs lsof on my file system, the lsof process consumes a horrendous amount of memory (>11GB). This forces a large amount of swapping, and brings the system to its knees until the process exits. The same thing occurs when I run lsof manually, but this is currently making the assistant unusable for me. Is this normal when running lsof on a large number of files, or is something wrong with my particular setup?
+
+An example of resource usage from top, and some system details:
+
+---
+      PID USERNAME    THR PRI NICE   SIZE    RES STATE   C   TIME   WCPU COMMAND
+    33735 username       1  23    0 28208M 11507M pfault  0   0:07  58.50% lsof
+---
+    [username@hostname /mnt/media]$ uname -a
+    FreeBSD hostname 9.2-RELEASE-p10 FreeBSD 9.2-RELEASE-p10 #0 r262572+4fb5adc: Wed Aug  6 17:07:16 PDT 2014     root@build3.ixsystems.com:/fusion/jkh/921/freenas/os-base/amd64/fusion/jkh/921/freenas/FreeBSD/src/sys/FREENAS.amd64  amd64
+---
+    [username@hostname /mnt/media]$ lsof -h
+    lsof 4.88
+---
+    [username@hostname /mnt/media]$ git annex info
+    repository mode: direct
+    trusted repositories: 0
+    semitrusted repositories: 1
+            d03b21fc-666d-457d-b953-0ca0ac7393d8 -- [hostname_media_indirect]
+    untrusted repositories: 2
+            00000000-0000-0000-0000-000000000001 -- web
+            31497a4d-290e-409a-9fd2-20c7340c245b -- hostname_mnt/media [here]
+    transfers in progress: none
+    available local disk space: 780.1 gigabytes (+10 gigabytes reserved)
+    local annex keys: 41576
+    local annex size: 943.95 gigabytes (+ 49 unknown size)
+    annexed files in working tree: 41887
+    size of annexed files in working tree: 945.14 gigabytes (+ 50 unknown size)
+    bloom filter size: 16 mebibytes (8.3% full)
+    backend usage:
+            SHA512E: 81518
+            WORM: 1846
+            URL: 99
+---
+    [username@hostname /mnt/media]$ git annex version
+    git-annex version: 5.20140817
+    build flags: Assistant Webapp Webapp-secure Pairing S3 WebDAV Kqueue XMPP DNS Feeds Quvi TDFA CryptoHash
+    key/value backends: SHA256E SHA1E SHA512E SHA224E SHA384E SKEIN256E SKEIN512E SHA256 SHA1 SHA512 SHA224 SHA384 SKEIN256 SKEIN512 WORM URL
+    remote types: git gcrypt S3 bup directory rsync web webdav tahoe glacier ddar hook external
+    local repository version: 5
+    supported repository version: 5
+    upgrade supported from repository versions: 0 1 2 4
diff --git a/doc/forum/lsof_resource_use_problems/comment_1_a5e5d410545fa7f93f08936ec6aeee42._comment b/doc/forum/lsof_resource_use_problems/comment_1_a5e5d410545fa7f93f08936ec6aeee42._comment
new file mode 100644
index 0000000000..55312113e3
--- /dev/null
+++ b/doc/forum/lsof_resource_use_problems/comment_1_a5e5d410545fa7f93f08936ec6aeee42._comment
@@ -0,0 +1,10 @@
+[[!comment format=mdwn
+ username="http://joeyh.name/"
+ ip="209.250.56.111"
+ subject="comment 1"
+ date="2014-10-13T21:21:18Z"
+ content="""
+That seems very wrong.. On my linux system here, I can `lsof /` and it uses 4 kb total, and runs in 0.09 seconds, to report on 3500 open files.
+
+But, I don't know about freebsd. lsof may be more expensive there for some reason, like needing to look in kernel memory rather than in /proc perhaps? But expensive to the tune of gigabytes of space used, that must be lsof misbehaving.
+"""]]
diff --git a/doc/forum/lsof_resource_use_problems/comment_2_7b0d4109c04c47e65420105f0de3b7a2._comment b/doc/forum/lsof_resource_use_problems/comment_2_7b0d4109c04c47e65420105f0de3b7a2._comment
new file mode 100644
index 0000000000..a74832bba6
--- /dev/null
+++ b/doc/forum/lsof_resource_use_problems/comment_2_7b0d4109c04c47e65420105f0de3b7a2._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="https://www.google.com/accounts/o8/id?id=AItOawmuT_R0AecWcfCFTN055N8rD_WwQkgo1PE"
+ nickname="Tomas"
+ subject="macosx 10.9.5"
+ date="2015-02-19T20:18:05Z"
+ content="""
+I have the same issue.
+"""]]
diff --git a/doc/forum/luks_encrypted_disk_support.mdwn b/doc/forum/luks_encrypted_disk_support.mdwn
new file mode 100644
index 0000000000..e2f9efc066
--- /dev/null
+++ b/doc/forum/luks_encrypted_disk_support.mdwn
@@ -0,0 +1,3 @@
+Is there a way to easily access, say, a FDE (Full Disk Encrypted) external drive using git-annex?
+
+My use case is that I do not wish to encrypt each file individually - and find it more reliable to use a real filesystem encryption on this external hard drive. --[[anarcat]]
diff --git a/doc/forum/luks_encrypted_disk_support/comment_1_0ea476c778888f34196d9031f72b1844._comment b/doc/forum/luks_encrypted_disk_support/comment_1_0ea476c778888f34196d9031f72b1844._comment
new file mode 100644
index 0000000000..9fda6b324f
--- /dev/null
+++ b/doc/forum/luks_encrypted_disk_support/comment_1_0ea476c778888f34196d9031f72b1844._comment
@@ -0,0 +1,10 @@
+[[!comment format=mdwn
+ username="http://joeyh.name/"
+ ip="209.250.56.227"
+ subject="comment 1"
+ date="2013-12-29T17:53:39Z"
+ content="""
+Mount the drive and put a git repository or directory special remote on it.
+
+(This question seems to easy.. am I missing something?)
+"""]]
diff --git a/doc/forum/luks_encrypted_disk_support/comment_2_0af149bfe1868dde0a132c5d835c50df._comment b/doc/forum/luks_encrypted_disk_support/comment_2_0af149bfe1868dde0a132c5d835c50df._comment
new file mode 100644
index 0000000000..9170502f27
--- /dev/null
+++ b/doc/forum/luks_encrypted_disk_support/comment_2_0af149bfe1868dde0a132c5d835c50df._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="https://id.koumbit.net/anarcat"
+ ip="2001:1928:1:9::1"
+ subject="comment 2"
+ date="2013-12-30T17:44:33Z"
+ content="""
+I guess this is related to [[confused about external drives]] - I would have liked the assistant to propose using the full drive for storing stuff, as opposed to encrypting data with GPG, including prompting me for the crypto password.
+"""]]
diff --git a/doc/forum/make___34__git_annex_fsck__34___only_display__44___not_move.mdwn b/doc/forum/make___34__git_annex_fsck__34___only_display__44___not_move.mdwn
new file mode 100644
index 0000000000..a8e78b7ccc
--- /dev/null
+++ b/doc/forum/make___34__git_annex_fsck__34___only_display__44___not_move.mdwn
@@ -0,0 +1,3 @@
+Is there a way to make fsck only display corrupted files instead of move them to the internal /bad folder? 
+
+Especially since the -q option (to only print corrupted files) doesn't even print the filename, only that *a* file has been moved...it seems a bit aggressive to compulsorily move a file to some obfuscated name AND not indicate its original location anywhere...
diff --git a/doc/forum/make___34__git_annex_fsck__34___only_display__44___not_move/comment_1_018cb2fd618a448088d9e91e4872333d._comment b/doc/forum/make___34__git_annex_fsck__34___only_display__44___not_move/comment_1_018cb2fd618a448088d9e91e4872333d._comment
new file mode 100644
index 0000000000..7220c49463
--- /dev/null
+++ b/doc/forum/make___34__git_annex_fsck__34___only_display__44___not_move/comment_1_018cb2fd618a448088d9e91e4872333d._comment
@@ -0,0 +1,11 @@
+[[!comment format=mdwn
+ username="joey"
+ subject="""comment 1"""
+ date="2017-03-08T18:58:19Z"
+ content="""
+I've made fsck -q include the name of the file in that warning message.
+
+Note that, given any file in `.git/annex/bad`, you can find the files
+in the git repository that linked to it, by running 
+`git log --stat -S$filename`
+"""]]
diff --git a/doc/forum/make___34__git_annex_fsck__34___only_display__44___not_move/comment_2_fdfd1d1ded3fb98a83f2593605250a87._comment b/doc/forum/make___34__git_annex_fsck__34___only_display__44___not_move/comment_2_fdfd1d1ded3fb98a83f2593605250a87._comment
new file mode 100644
index 0000000000..d920d0607d
--- /dev/null
+++ b/doc/forum/make___34__git_annex_fsck__34___only_display__44___not_move/comment_2_fdfd1d1ded3fb98a83f2593605250a87._comment
@@ -0,0 +1,22 @@
+[[!comment format=mdwn
+ username="archimedes"
+ avatar="http://cdn.libravatar.org/avatar/5b2ed40aabedcb1b8c2c0ce69f55a1e1"
+ subject="comment 2"
+ date="2017-03-08T22:34:59Z"
+ content="""
+Do you mean \"made\"==\"updated\" or made==\"already exists\"?
+
+Maybe I have an old version (installed via brew)...
+
+I get: 
+
+~/s/a/annex8 git:master ❯❯❯ git annex fsck -q
+  Bad file size (1 B smaller); moved to .git/annex/bad/
+git-annex: fsck: 1 failed
+
+unless there are no other copies, in which case the original filename it is printed on a new line with the \"No known copies\" warning. 
+
+My use-case is for images - a small amount of corruption to some old images might not be visually perceptible, and hence tolerable, especially if I'm short-sighted enough to only have 1 copy. 
+I mean its possible for the user to navigate to /bad, visually inspect that they're ok with a corrupted image, parse the git log --stat message and restore it. 
+But it seems beneficial to have something like a rsync-like \"--dryrun\" option for fsck.
+"""]]
diff --git a/doc/forum/make___34__git_annex_fsck__34___only_display__44___not_move/comment_3_369daddf95ab43f2c3d6b81c35810c19._comment b/doc/forum/make___34__git_annex_fsck__34___only_display__44___not_move/comment_3_369daddf95ab43f2c3d6b81c35810c19._comment
new file mode 100644
index 0000000000..6ee28f5d93
--- /dev/null
+++ b/doc/forum/make___34__git_annex_fsck__34___only_display__44___not_move/comment_3_369daddf95ab43f2c3d6b81c35810c19._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="archimedes"
+ avatar="http://cdn.libravatar.org/avatar/5b2ed40aabedcb1b8c2c0ce69f55a1e1"
+ subject="comment 3"
+ date="2017-03-08T22:36:14Z"
+ content="""
+Apologies for editing, forgot it was markdown.
+"""]]
diff --git a/doc/forum/making_good_use_of_my_shiny_new_rsync.net_account.mdwn b/doc/forum/making_good_use_of_my_shiny_new_rsync.net_account.mdwn
new file mode 100644
index 0000000000..aef5a58a78
--- /dev/null
+++ b/doc/forum/making_good_use_of_my_shiny_new_rsync.net_account.mdwn
@@ -0,0 +1,20 @@
+So I got a shiny new rsync.net account, but it's not (yet) big enough for my bloated, kitchen-sink annex.  How do I make the best use of it?   Well, on the whole small files are more valuable to me than large.  Large files are usually replaceable, like ripped DVDs.  Small files are more likely to be family photos and such.  So I shut down the assistant, and I've been doing a series of these:
+
+    git annex copy --to=rsyncnet --smallerthan 10M --not --in=rsyncnet # I called my remote "rsyncnet"
+    
+    git annex copy --to=rsyncnet --smallerthan 20M --not --in=rsyncnet
+    
+    git annex copy --to=rsyncnet --smallerthan 40M --not --in=rsyncnet
+
+until the repo fills up.  With a few of these thrown in to check on my progress and what's ahead.
+
+
+    git annex status . --in=rsyncnet  # how much have I already done?
+    git annex status . --smallerthan 40M --not --in=rsyncnet # how much will this next command copy?
+
+
+It'd be possible to write a script to do all this automagically but at least this time it's easier to do by hand.
+
+Just sharing in case anybody's in a similar situation.
+
+(You could let the assistant do this for you if you configured a preferred content expression designating small files for the rsyncnet remote, of course.)
diff --git a/doc/forum/making_good_use_of_my_shiny_new_rsync.net_account/comment_1_0ebe509b768d46081db2100f5b712ef7._comment b/doc/forum/making_good_use_of_my_shiny_new_rsync.net_account/comment_1_0ebe509b768d46081db2100f5b712ef7._comment
new file mode 100644
index 0000000000..617c87d972
--- /dev/null
+++ b/doc/forum/making_good_use_of_my_shiny_new_rsync.net_account/comment_1_0ebe509b768d46081db2100f5b712ef7._comment
@@ -0,0 +1,10 @@
+[[!comment format=mdwn
+ username="https://www.google.com/accounts/o8/id?id=AItOawln3ckqKx0x_xDZMYwa9Q1bn4I06oWjkog"
+ nickname="Michael"
+ subject="comment 1"
+ date="2013-07-26T16:16:04Z"
+ content="""
+How about
+
+    git annex find . --format='${bytesize} ${file}\n' |sort -n|sed 's/[0-9]* //'|while read file; do git annex copy \"$file\" --to=your_remote; done
+"""]]
diff --git a/doc/forum/making_good_use_of_my_shiny_new_rsync.net_account/comment_2_ef63d893531d93d2eb09f48f8baff4dd._comment b/doc/forum/making_good_use_of_my_shiny_new_rsync.net_account/comment_2_ef63d893531d93d2eb09f48f8baff4dd._comment
new file mode 100644
index 0000000000..229a2dda99
--- /dev/null
+++ b/doc/forum/making_good_use_of_my_shiny_new_rsync.net_account/comment_2_ef63d893531d93d2eb09f48f8baff4dd._comment
@@ -0,0 +1,13 @@
+[[!comment format=mdwn
+ username="http://joeyh.name/"
+ ip="2001:4978:f:21a::2"
+ subject="comment 2"
+ date="2013-07-26T16:44:35Z"
+ content="""
+Note that you don't need to shut down the assistant; you can run multiple git-annex commands simulantaneously safely.
+You will probably want to set that repository to be in the manual group in the assistant though.
+
+Another way to do it is to make rsync.net a small archive, and move or copy your important files into `archive` directories.
+
+You can also set up multiple remotes using the same rsync.net account. So you could have a transfer remote that's used to ship files between clients, as well as your archive remote.
+"""]]
diff --git a/doc/forum/man_pages_in_the_prebuilt_linux_tarball.mdwn b/doc/forum/man_pages_in_the_prebuilt_linux_tarball.mdwn
new file mode 100644
index 0000000000..045dccb822
--- /dev/null
+++ b/doc/forum/man_pages_in_the_prebuilt_linux_tarball.mdwn
@@ -0,0 +1 @@
+Is there an easy way for a noob to get the man pages, after installing the prebuilt tarball?
diff --git a/doc/forum/man_pages_in_the_prebuilt_linux_tarball/comment_1_a7bc2e84e6d7c0e2de5900685207af78._comment b/doc/forum/man_pages_in_the_prebuilt_linux_tarball/comment_1_a7bc2e84e6d7c0e2de5900685207af78._comment
new file mode 100644
index 0000000000..01f0109e31
--- /dev/null
+++ b/doc/forum/man_pages_in_the_prebuilt_linux_tarball/comment_1_a7bc2e84e6d7c0e2de5900685207af78._comment
@@ -0,0 +1,9 @@
+[[!comment format=mdwn
+ username="http://joeyh.name/"
+ ip="4.154.6.49"
+ subject="comment 1"
+ date="2012-11-27T20:27:37Z"
+ content="""
+Yes, go to http://git-annex.branchable.com/git-annex/ and http://git-annex.branchable.com/git-annex-shell/
+
+"""]]
diff --git a/doc/forum/manage_repository_connection_between_external_HDD_and_server.mdwn b/doc/forum/manage_repository_connection_between_external_HDD_and_server.mdwn
new file mode 100644
index 0000000000..d5d49e9e88
--- /dev/null
+++ b/doc/forum/manage_repository_connection_between_external_HDD_and_server.mdwn
@@ -0,0 +1,32 @@
+I have one server, two computers, and one external HDD.
+
+The git-annex repo on my server (`nhost`) has the following remotes:
+
+```
+qhost	qhost:NIdata (fetch)
+qhost	qhost:NIdata (push)
+zhost	zhost:NIdata (fetch)
+zhost	zhost:NIdata (push)
+```
+
+The repo on `zhost` has the following remotes:
+
+```
+data0	/run/media/mee/data0/NIdata/ (fetch)
+data0	/run/media/mee/data0/NIdata/ (push)
+nhost	nhost:NIdata (fetch)
+nhost	nhost:NIdata (push)
+```
+
+The repo on my HDD (`data0`) has thefollowing remotes:
+
+```
+nhost		nhost:NIdata (fetch)
+nhost		nhost:NIdata (push)
+qhost		/qhost/home/mee/NIdata/ (fetch)
+qhost		/qhost/home/mee/NIdata/ (push)
+zhost		/zhost/home/mee/NIdata/ (fetch)
+zhost		/zhost/home/mee/NIdata/ (push)
+```
+
+Any ideas how I can set up a remote from `nhost` to `data0`, without making the identity of data0 dependent on which machine I am using on each particular session?
diff --git a/doc/forum/manage_repository_connection_between_external_HDD_and_server/comment_1_ba8ed548bf228a7d1c82b68283f99a5b._comment b/doc/forum/manage_repository_connection_between_external_HDD_and_server/comment_1_ba8ed548bf228a7d1c82b68283f99a5b._comment
new file mode 100644
index 0000000000..bbb509f640
--- /dev/null
+++ b/doc/forum/manage_repository_connection_between_external_HDD_and_server/comment_1_ba8ed548bf228a7d1c82b68283f99a5b._comment
@@ -0,0 +1,25 @@
+[[!comment format=mdwn
+ username="joey"
+ subject="""comment 1"""
+ date="2016-09-21T20:06:28Z"
+ content="""
+I'm not entirely sure I understand the question, but think you're
+probably trying to make nhost be able to access data0 independant
+of whether the external HDD is plugged into nhost or zhost
+(or perhaps qhost, whatever that is?)
+
+The way to do that is simply to set up one remote for each place
+the HDD can be plugged into. If the HDD can be plugged in locally,
+make a remote pointing at the local mount point. If the HDD can be
+plugged into zhost, make a remote pointing at
+zhost:/run/media/mee/data0/NIdata/ and so on.
+
+git-annex will figure out that these different remotes are pointing
+at the same repository. When running `git annex get` or `git annex sync
+--content`, it will check if the HDD is mounted locally,
+and will use it there if so; if not it will fall back to trying
+to access it via the remotes that use zhost etc.
+
+So in summary, git-annex is designed to just work in this kind of scenario.
+Set up lots of remotes and let it figure out which ones to use.
+"""]]
diff --git a/doc/forum/managing_git-annex_with_java.mdwn b/doc/forum/managing_git-annex_with_java.mdwn
new file mode 100644
index 0000000000..3356f714ba
--- /dev/null
+++ b/doc/forum/managing_git-annex_with_java.mdwn
@@ -0,0 +1,11 @@
+Hi,
+
+Our need is to create a JAVA application under Windows that manages programmatically the synchronisation of a local directory structure from a remote directory on a Linux machine:
+- locally the user may choose the content of the remote directory to synchronise
+- then the application is able to synchronize ths content on demand or on a regular basis
+
+Can git-annex + git a soluton that answer those requirements ?
+Does git-annex expose an API callable from JAVA ?
+
+Thank you in advance,
+Regards
diff --git a/doc/forum/managing_git-annex_with_java/comment_1_fd12f0d0727b5bdddef1d7ef6236fe57._comment b/doc/forum/managing_git-annex_with_java/comment_1_fd12f0d0727b5bdddef1d7ef6236fe57._comment
new file mode 100644
index 0000000000..91d758920d
--- /dev/null
+++ b/doc/forum/managing_git-annex_with_java/comment_1_fd12f0d0727b5bdddef1d7ef6236fe57._comment
@@ -0,0 +1,11 @@
+[[!comment format=mdwn
+ username="joey"
+ subject="""comment 1"""
+ date="2016-09-05T16:28:01Z"
+ content="""
+git-annex exposes an extensive traditional command line interface,
+very much like the interface exposed by git.
+
+I presume that java can, perhaps with some difficulty, use such an
+interface.
+"""]]
diff --git a/doc/forum/managing_multiple_repositories.mdwn b/doc/forum/managing_multiple_repositories.mdwn
new file mode 100644
index 0000000000..7450b42710
--- /dev/null
+++ b/doc/forum/managing_multiple_repositories.mdwn
@@ -0,0 +1,3 @@
+I tried about 2 weeks ago using the assistant to create on my netbook and my home laptop 3 repositores, for music, pictures and documents.  Since I was also going to be away from home, I set up jabber pairing and box.com for a transfer repository, and then watched everything crash as I didn't know yet about jabber only really working for 1 repository at a time.
+
+So my new idea is this: one repository at ~/annex, and changing documents, music and pictures to be symlinks to actual documents, music and pictures folders inside ~/annex, allowing me to have just 1 annex to manage.  Is this the best way to put everything together if I want to just jabber pairing (and just 1 jabber account)?
diff --git a/doc/forum/manual_update_of_.git__47__annex__47__objects.mdwn b/doc/forum/manual_update_of_.git__47__annex__47__objects.mdwn
new file mode 100644
index 0000000000..9de5779c12
--- /dev/null
+++ b/doc/forum/manual_update_of_.git__47__annex__47__objects.mdwn
@@ -0,0 +1,8 @@
+Let's suppose that I've manually modified files in .git/annex/objects,
+for example I ran an rsync or some other file synchronization software
+to copy files. As a result, some objects have disappeared, others have
+appeared. After that `git annex whereis .' displays stale information,
+it doesn't take the manual modifications to accound. `git annex fsck'
+seems to fix this, but it runs the rehashing of all new files, so it's
+slow. Is there a fast alternative, which notices all the object file
+changes, trusts them, and just updates .git/annex/index quickly?
diff --git a/doc/forum/manual_update_of_.git__47__annex__47__objects/comment_1_ea6ec91150c8962e2711631f2422bf3a._comment b/doc/forum/manual_update_of_.git__47__annex__47__objects/comment_1_ea6ec91150c8962e2711631f2422bf3a._comment
new file mode 100644
index 0000000000..f721e21a59
--- /dev/null
+++ b/doc/forum/manual_update_of_.git__47__annex__47__objects/comment_1_ea6ec91150c8962e2711631f2422bf3a._comment
@@ -0,0 +1,10 @@
+[[!comment format=mdwn
+ username="http://joeyh.name/"
+ ip="4.154.4.51"
+ subject="comment 1"
+ date="2013-09-12T17:11:41Z"
+ content="""
+I do not recommend manually messing with the contents of .git/annex/objects. It is a good way to lose data.
+
+If you look at the documentation for fsck in git-annex's man page, you will find the answer to your question.
+"""]]
diff --git a/doc/forum/manual_update_of_.git__47__annex__47__objects/comment_2_a7bbf304b26650a786e358bdc01e3069._comment b/doc/forum/manual_update_of_.git__47__annex__47__objects/comment_2_a7bbf304b26650a786e358bdc01e3069._comment
new file mode 100644
index 0000000000..5468ed3d06
--- /dev/null
+++ b/doc/forum/manual_update_of_.git__47__annex__47__objects/comment_2_a7bbf304b26650a786e358bdc01e3069._comment
@@ -0,0 +1,33 @@
+[[!comment format=mdwn
+ username="rasmus"
+ ip="109.201.154.177"
+ subject="But how to clean this folder?"
+ date="2014-03-20T21:55:26Z"
+ content="""
+Joey,
+
+I have sometimes experienced that there is way too much content in the `.git/annex/objects` folder.  For instance, my terminal emulator recently crashed during a sync which seemed to create a lot of folders.  
+
+In the annex repo where this happened I now got way too many folders (annex sometimes complain about no. of inodes or something like that) and I don't know how to clean it correctly.
+
+Here's a \"screenshot\":
+
+    [doc.annex] $ ls -a
+    .  ..  documents  .git  .gitignore
+    [doc.annex] $ find . -type d | wc -l
+    38568
+    [doc.annex] $ git annex fsck  > /dev/null 2>&1
+    [doc.annex] $ find . -type d | wc -l
+    38568
+    [doc.annex] $ git annex repair  > /dev/null 2>&1
+    [doc.annex] $ find . -type d | wc -l
+    38568
+    [doc.annex] $ find documents -type d | wc -l
+    1513
+    [doc.annex] $ find .git/annex/objects -type d | wc -l
+    36712
+
+This is a `direct`-mode repo.  With `.git/objectcts` I can use `git gc`.  How can I tell annex to tidy up?  I have tried `fsck`, `repair` and `forget`.
+
+Any hints on how to deal with this?  Other than manually clean up by deleting `.git/annex/objects` and rerunning `fsck`?
+"""]]
diff --git a/doc/forum/manual_update_of_.git__47__annex__47__objects/comment_3_a855096b683c4c4f84e72c797e065d59._comment b/doc/forum/manual_update_of_.git__47__annex__47__objects/comment_3_a855096b683c4c4f84e72c797e065d59._comment
new file mode 100644
index 0000000000..2271e04ffe
--- /dev/null
+++ b/doc/forum/manual_update_of_.git__47__annex__47__objects/comment_3_a855096b683c4c4f84e72c797e065d59._comment
@@ -0,0 +1,12 @@
+[[!comment format=mdwn
+ username="http://joeyh.name/"
+ ip="209.250.56.41"
+ subject="comment 3"
+ date="2014-03-26T19:09:07Z"
+ content="""
+1. This followup question seems to have nothing to do with the original question. (The aswer to the original question was `git annex fsck --fast`.)
+2. I find it hard to believe that a crashing terminal emulator will create a lot of folders.
+3. You show 36k subdirectories in .git/annex/objects. If I make a *tiny* 100 mb ext2 filesystem, it has 26k inodes. A 200 mb filesystem has 50k. So we're talking about a number of directories that is only large when using hardware from 1996.
+
+I'm not convinced yet that anything is wrong, or that there's anything you can do to improve matters. In any case, see [[todo/wishlist:_pack_metadata_in_direct_mode]].
+"""]]
diff --git a/doc/forum/many_remotes.mdwn b/doc/forum/many_remotes.mdwn
new file mode 100644
index 0000000000..4e463cb3c3
--- /dev/null
+++ b/doc/forum/many_remotes.mdwn
@@ -0,0 +1,24 @@
+Thanks Joey for the great work.
+
+I'm using git annex for my tv-shows and movies Folder.
+
+I have 3 USB HDD (ext.150Gb,ext.200Gb, ext.2Tb) and a USB Stick (ATV), which are traveling between 3 Devices
+
+2 Notebooks (Lappi,Kiste) and a Nas.
+
+First of all, I'm made a mistake and mixed the remote Locations from the tvshows folder with the movies folder and did a git annex sync, ( this happened about two weeks ago) 
+I think i can't undo this, only unannex will help.
+
+I've now watched the annex status output and noticed that there are many Remotes, some of them have a timestamp as name.
+
+see log file at http://pastebin.com/79bRVkK6
+
+Running git annex on ubuntu 12.04
+
+    christian@Lappi:~/Serien$ git annex version
+    git-annex version: 4.20130417
+    local repository version: 3
+    default repository version: 3
+    supported repository versions: 3 4
+    upgrade supported from repository versions: 0 1 2
+    build flags: Assistant Webapp Pairing Testsuite S3 WebDAV Inotify DBus XMPP DNS
diff --git a/doc/forum/mesh_configurations.mdwn b/doc/forum/mesh_configurations.mdwn
new file mode 100644
index 0000000000..c3904f4280
--- /dev/null
+++ b/doc/forum/mesh_configurations.mdwn
@@ -0,0 +1,181 @@
+I have a setup where a *source* repository `a` is connected to a *source* repository `b` (through SSH) which is then connected to  *backup* repository `c` (on amazon S3). I was expecting a file added on `a` to be moved to `c` *through* `b`, but that doesn't seem to be happening...
+
+I tried to reproduce with this basic setup:
+
+
    +[1009]anarcat@angela:g-a$ git init a
+Dépôt Git vide initialisé dans /home/anarcat/test/g-a/a/.git/
+[1010]anarcat@angela:g-a$ git init b
+Dépôt Git vide initialisé dans /home/anarcat/test/g-a/b/.git/
+[1011]anarcat@angela:g-a$ git init c
+Dépôt Git vide initialisé dans /home/anarcat/test/g-a/c/.git/
+[1012]anarcat@angela:g-a$ cd a/
+[1013]anarcat@angela:a$ git annex init
+init  ok
+(Recording state in git...)
+[1014]anarcat@angela:a$ git annex group . source
+group . ok
+(Recording state in git...)
+[1015]anarcat@angela:a$ git annex wanted . groupwanted
+wanted . ok
+(Recording state in git...)
+[1036]anarcat@angela:a$ git remote add origin ../b
+[1016]anarcat@angela:a$ cd ../b
+[1025]anarcat@angela:b$ git annex init
+init  ok
+(Recording state in git...)
+[1026]anarcat@angela:b$ git annex group . source
+group . ok
+(Recording state in git...)
+[1027]anarcat@angela:b$ git annex wanted . groupwanted
+wanted . ok
+(Recording state in git...)
+[1038]anarcat@angela:b$ git remote add origin ../c
+[1019]anarcat@angela:b$ cd ../c
+[1021]anarcat@angela:c$ git annex init
+init  ok
+(Recording state in git...)
+[1022]anarcat@angela:c$ git annex group . backup
+group . ok
+(Recording state in git...)
+[1023]anarcat@angela:c$ git annex wanted . groupwanted
+wanted . ok
+(Recording state in git...)
+anarcat@angela:c$ cd ../a
+[1041]anarcat@angela:a$ git annex sync
+commit  ok
+pull origin
+warning: no common commits
+remote: Décompte des objets: 11, fait.
+remote: Compression des objets: 100% (9/9), fait.
+remote: Total 11 (delta 1), reused 0 (delta 0)
+Dépaquetage des objets: 100% (11/11), fait.
+Depuis ../b
+ * [nouvelle branche] git-annex  -> origin/git-annex
+
+merge: refs/remotes/origin/master - not something we can merge
+
+merge: refs/remotes/origin/synced/master - not something we can merge
+failed
+(merging origin/git-annex into git-annex...)
+(Recording state in git...)
+(Recording state in git...)
+git-annex: sync: 1 failed
+[1042]anarcat@angela:a1$ cd ../b
+[1043]anarcat@angela:b$ git annex sync
+commit  ok
+pull origin
+warning: no common commits
+remote: Décompte des objets: 11, fait.
+remote: Compression des objets: 100% (9/9), fait.
+remote: Total 11 (delta 1), reused 0 (delta 0)
+Dépaquetage des objets: 100% (11/11), fait.
+Depuis ../c
+ * [nouvelle branche] git-annex  -> origin/git-annex
+
+merge: refs/remotes/origin/master - not something we can merge
+
+merge: refs/remotes/origin/synced/master - not something we can merge
+failed
+(merging origin/git-annex into git-annex...)
+(Recording state in git...)
+(Recording state in git...)
+git-annex: sync: 1 failed
+[1063]anarcat@angela:b$ touch bar
+[1064]anarcat@angela:b$ ls
+bar
+[1065]anarcat@angela:b$ ls -al
+total 16K
+drwxr-xr-x 3 anarcat anarcat 4096 aoû 18 14:41 .
+drwxr-xr-x 5 anarcat anarcat 4096 aoû 18 14:33 ..
+lrwxrwxrwx 1 anarcat anarcat  178 aoû 18 14:41 bar -> .git/annex/objects/pX/ZJ/SHA256E-s0--e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855/SHA256E-s0--e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855
+drwxr-xr-x 9 anarcat anarcat 4096 aoû 18 14:41 .git
+[1066]anarcat@angela:b$ git annex sync
+commit  ok
+pull origin
+ok
+push origin
+Décompte des objets: 26, fait.
+Delta compression using up to 2 threads.
+Compression des objets: 100% (22/22), fait.
+Écriture des objets: 100% (26/26), 2.47 KiB | 0 bytes/s, fait.
+Total 26 (delta 5), reused 0 (delta 0)
+To ../c
+ * [new branch]      git-annex -> synced/git-annex
+ * [new branch]      master -> synced/master
+ok
+[1067]anarcat@angela:b$ cd ../a
+[1068]anarcat@angela:a$ git annex sync
+commit  ok
+pull origin
+remote: Décompte des objets: 8, fait.
+remote: Compression des objets: 100% (6/6), fait.
+remote: Total 8 (delta 1), reused 0 (delta 0)
+Dépaquetage des objets: 100% (8/8), fait.
+Depuis ../b
+   5d3090f..9e345e6  git-annex  -> origin/git-annex
+ * [nouvelle branche] master     -> origin/master
+ * [nouvelle branche] synced/master -> origin/synced/master
+
+Merge made by the 'recursive' strategy.
+ bar | 1 +
+ 1 file changed, 1 insertion(+)
+ create mode 120000 bar
+
+Already up-to-date.
+ok
+(merging origin/git-annex into git-annex...)
+(Recording state in git...)
+(Recording state in git...)
+push origin
+Décompte des objets: 41, fait.
+Delta compression using up to 2 threads.
+Compression des objets: 100% (36/36), fait.
+Écriture des objets: 100% (41/41), 3.50 KiB | 0 bytes/s, fait.
+Total 41 (delta 20), reused 0 (delta 0)
+To ../b
+   6019ab8..368ca15  master -> synced/master
+ * [new branch]      git-annex -> synced/git-annex
+ok
+[1069]anarcat@angela:a$ touch quu^C
+[1069]anarcat@angela:a130$ echo foo > quux
+[1070]anarcat@angela:a$ cd ../b
+[1071]anarcat@angela:b$ ls
+bar  foo
+[1072]anarcat@angela:b$ cd ..
+[1073]anarcat@angela:g-a$ cd a
+[1074]anarcat@angela:a$ git annex list
+here
+|origin
+||web
+|||
+XX_ bar
+XX_ foo
+X__ quux
+[1075]anarcat@angela:a$ git annex list --help
+git-annex: unrecognized option `--help'
+
+Usage: git-annex list [PATH ...] [option ...]
+    --allrepos  show all repositories, not only remotes
+
+To see additional options common to all commands, run: git annex help options
+
+
+[1076]anarcat@angela:a1$ git annex list --allrepos
+here-
+|origin
+||web
+|||anarcat@angela:~/test/g-a/c
+||||
+XX__ bar
+XX__ foo
+X___ quux
+
    
+
+why don't the files get copied over to the backup repo by the assistant?
+
+i somewhat understand that files don't get sent from `a` to `b`, but why doesn't the assistant copy the files from `b` to `c`?
+
+i have tried using `required` instead of `wanted` and it doesn't work much better.
+
+tested with `5.20150610+gitg608172f-1~ndall+1` (prod) and `5.20141125` (the above test). --[[anarcat]]
diff --git a/doc/forum/mesh_configurations/comment_10_dc99216ee2fc0bd3d43234915a0305ab._comment b/doc/forum/mesh_configurations/comment_10_dc99216ee2fc0bd3d43234915a0305ab._comment
new file mode 100644
index 0000000000..64e98f70cc
--- /dev/null
+++ b/doc/forum/mesh_configurations/comment_10_dc99216ee2fc0bd3d43234915a0305ab._comment
@@ -0,0 +1,19 @@
+[[!comment format=mdwn
+ username="joey"
+ subject="""comment 10"""
+ date="2015-09-21T18:44:20Z"
+ content="""
+I'm not comfortable making core.sharedrepository settings affect creds
+files. You don't normally want to give out S3 creds to other users in a
+unix group. And in the "everybody" case, it certianly seems entirely wrong
+to make the creds files world-readable. Willing to live with a little
+inconsistency here in order to not blow user's bank accounts off. It would
+be good to document it somewhere.
+
+Your paste seems to show A as being in the "sourcethis" group, not the
+"source" group. I don't know what that means.
+
+The assistant should notice config changes to the git-annex branch within
+60 seconds of them being received. Syncs happen when changes are detected,
+or every 30 minutes.
+"""]]
diff --git a/doc/forum/mesh_configurations/comment_1_0712fa3d50ca6092bd7d98c945b5cc31._comment b/doc/forum/mesh_configurations/comment_1_0712fa3d50ca6092bd7d98c945b5cc31._comment
new file mode 100644
index 0000000000..a97ade40dc
--- /dev/null
+++ b/doc/forum/mesh_configurations/comment_1_0712fa3d50ca6092bd7d98c945b5cc31._comment
@@ -0,0 +1,23 @@
+[[!comment format=mdwn
+ username="joey"
+ subject="""comment 1"""
+ date="2015-08-19T16:40:16Z"
+ content="""
+It's easy to see why a file is not copied from source repo A to source repo
+B: The preferred content expression for a source repo is "not (copies=1)",
+so a source repo will not want to get any files from any other repo.
+
+You probably want to make the central repo use transfer; that's basically
+what it's for. Note that a transfer repo hangs onto file content until it
+reaches all client repos. So you might want to change the preferred content
+expression to refer to backup repos. Something as simple as this might
+work:
+
+	not inallgroup=backup
+
+I don't see any reason why a file wouldn't move from B to backup repo C,
+but I don't see anything in your transcript that shows that not happening
+either. Your transcript doesn't actually show running any git-annex commands
+that move file contents at all; no git annex copy/move, and the sync is not
+run with --content...
+"""]]
diff --git a/doc/forum/mesh_configurations/comment_2_0d9ba4017343475a96975145c9db29c5._comment b/doc/forum/mesh_configurations/comment_2_0d9ba4017343475a96975145c9db29c5._comment
new file mode 100644
index 0000000000..9296cf7195
--- /dev/null
+++ b/doc/forum/mesh_configurations/comment_2_0d9ba4017343475a96975145c9db29c5._comment
@@ -0,0 +1,15 @@
+[[!comment format=mdwn
+ username="anarcat"
+ subject="clarifications"
+ date="2015-08-19T16:57:49Z"
+ content="""
+the assistant is runnin on repos a and b. i was expecting it to move the files automatically.
+
+about repo groups, if i understand correctly, it should be:
+
+* `a`: *source*
+* `b`: *transfer* (or *not inallgroup=backup*?)
+* `c`: *backup*
+
+is that correct?
+"""]]
diff --git a/doc/forum/mesh_configurations/comment_3_d228158cf00cd04dc9ef5602acebdaac._comment b/doc/forum/mesh_configurations/comment_3_d228158cf00cd04dc9ef5602acebdaac._comment
new file mode 100644
index 0000000000..47774576b6
--- /dev/null
+++ b/doc/forum/mesh_configurations/comment_3_d228158cf00cd04dc9ef5602acebdaac._comment
@@ -0,0 +1,14 @@
+[[!comment format=mdwn
+ username="joey"
+ subject="""comment 3"""
+ date="2015-08-19T17:04:30Z"
+ content="""
+Your transcript doesn't show the assistant running. It's not clear that, in
+your live deployment, any files ever got to repo B for the assistant there
+to deal with.
+
+Repos such as B need a preferred content expression like the one I gave. It
+doesn't matter a whole lot what group is used for them; overriding the
+groupwanted for the transfer group to use that expression would be one way
+to do it.
+"""]]
diff --git a/doc/forum/mesh_configurations/comment_4_d5c8e99cadf976434460d6d0f410136e._comment b/doc/forum/mesh_configurations/comment_4_d5c8e99cadf976434460d6d0f410136e._comment
new file mode 100644
index 0000000000..39446c78d1
--- /dev/null
+++ b/doc/forum/mesh_configurations/comment_4_d5c8e99cadf976434460d6d0f410136e._comment
@@ -0,0 +1,282 @@
+[[!comment format=mdwn
+ username="anarcat"
+ subject="comment 4"
+ date="2015-08-19T17:17:50Z"
+ content="""
+darn, you're right... then i screwed up my copy-paste. :( the assistant *was* running on a and b, that i am sure of.
+
+here's a more complete transcript (hopefully):
+
+
    +[997]anarcat@desktop008:test$ mkdir g-a
+[998]anarcat@desktop008:test$ cd g-a/
+[999]anarcat@desktop008:g-a$ git init a
+Initialized empty Git repository in /home/anarcat/test/g-a/a/.git/
+[1000]anarcat@desktop008:g-a$ git init b
+Initialized empty Git repository in /home/anarcat/test/g-a/b/.git/
+[1001]anarcat@desktop008:g-a$ git init c
+Initialized empty Git repository in /home/anarcat/test/g-a/c/.git/
+[1002]anarcat@desktop008:g-a$ cd a
+[1003]anarcat@desktop008:a$ git annex init
+init  ok
+(recording state in git...)
+[1004]anarcat@desktop008:a$ git remote add origin ../b
+[1005]anarcat@desktop008:a$ cd ../b
+[1006]anarcat@desktop008:b$ git annex init
+init  ok
+(recording state in git...)
+[1007]anarcat@desktop008:b$ git remote add origin ../c
+[1008]anarcat@desktop008:b$ cd ../c
+[1009]anarcat@desktop008:c$ git annex init
+init  ok
+(recording state in git...)
+[1010]anarcat@desktop008:c$ cd ../a/
+[1011]anarcat@desktop008:a$ git annex ^C
+[1011]anarcat@desktop008:a130$ touch test
+[1012]anarcat@desktop008:a$ git annex add tset
+git-annex: tset not found
+git-annex: add: 1 failed
+[1013]anarcat@desktop008:a1$ git annex add test
+add test ok
+(recording state in git...)
+[1014]anarcat@desktop008:a$ git annex sync
+commit  ok
+pull origin
+warning: no common commits
+remote: Counting objects: 5, done.
+remote: Compressing objects: 100% (3/3), done.
+remote: Total 5 (delta 0), reused 0 (delta 0)
+Unpacking objects: 100% (5/5), done.
+From ../b
+ * [new branch]      git-annex  -> origin/git-annex
+ok
+(merging origin/git-annex into git-annex...)
+(recording state in git...)
+push origin
+Counting objects: 16, done.
+Delta compression using up to 2 threads.
+Compressing objects: 100% (12/12), done.
+Writing objects: 100% (16/16), 1.56 KiB | 0 bytes/s, done.
+Total 16 (delta 1), reused 0 (delta 0)
+To ../b
+ * [new branch]      git-annex -> synced/git-annex
+ * [new branch]      master -> synced/master
+ok
+[1015]anarcat@desktop008:a$ git annex assistant
+[1016]anarcat@desktop008:a$ cd ../b
+[1017]anarcat@desktop008:b$ git annex assistant
+[1018]anarcat@desktop008:b$ ls
+[1019]anarcat@desktop008:b$ git anne^C
+[1019]anarcat@desktop008:b130$ ls
+[1019]anarcat@desktop008:b$ git annex sync
+commit  ok
+pull origin
+
+merge: refs/remotes/origin/master - not something we can merge
+
+merge: refs/remotes/origin/synced/master - not something we can merge
+failed
+git-annex: sync: 1 failed
+[1020]anarcat@desktop008:b1$ git remote -v^C
+[1020]anarcat@desktop008:b130$ git re^C
+[1020]anarcat@desktop008:b130$ ls
+[1021]anarcat@desktop008:b$ cd ../
+[1022]anarcat@desktop008:g-a$ git annexccd ^C
+[1022]anarcat@desktop008:g-a130$ cd c
+[1023]anarcat@desktop008:c$ git annex sync
+commit  ok
+[1024]anarcat@desktop008:c$ ls
+[1025]anarcat@desktop008:c$ cd ../b
+[1026]anarcat@desktop008:b$ git co master
+error: pathspec 'master' did not match any file(s) known to git.
+[1027]anarcat@desktop008:b1$ git branch -a
+  git-annex
+  synced/git-annex
+  synced/master
+  remotes/origin/git-annex
+[1028]anarcat@desktop008:b$ git co -b master synced/master
+Already on 'master'
+[1029]anarcat@desktop008:b$ ls
+test
+[1030]anarcat@desktop008:b$ git annex sync
+commit  ok
+pull origin
+ok
+[1031]anarcat@desktop008:b$ cd ..c
+bash: cd: ..c: No such file or directory
+[1032]anarcat@desktop008:b1$ cd ../c
+[1033]anarcat@desktop008:c$ ls
+[1034]anarcat@desktop008:c$ git annex sync
+(recording state in git...)
+commit  ok
+[1035]anarcat@desktop008:c$ ls
+[1036]anarcat@desktop008:c$ cd ../a
+[1037]anarcat@desktop008:a$ echo test > foo
+[1038]anarcat@desktop008:a$ ls -al
+total 20K
+drwxr-xr-x  3 anarcat 4294967294 4096 aoû 19 13:12 .
+drwxr-xr-x  5 anarcat 4294967294 4096 aoû 19 13:10 ..
+lrwxrwxrwx  1 anarcat 4294967294  178 aoû 19 13:12 foo -> .git/annex/objects/w8/pv/SHA256E-s5--f2ca1bb6c7e907d06dafe4687e579fce76b37e4e93b7605022da52e6ccc26fd2/SHA256E-s5--f2ca1bb6c7e907d06dafe4687e579fce76b37e4e93b7605022da52e6ccc26fd2
+drwxr-xr-x 10 anarcat 4294967294 4096 aoû 19 13:12 .git
+lrwxrwxrwx  1 anarcat 4294967294  178 aoû 19 13:11 test -> .git/annex/objects/pX/ZJ/SHA256E-s0--e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855/SHA256E-s0--e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855
+[1039]anarcat@desktop008:a$ git status
+On branch master
+nothing to commit, working directory clean
+[1040]anarcat@desktop008:a$ git annex list
+here
+|origin
+||web
+|||bittorrent
+||||
+XX__ foo
+XX__ test
+[1041]anarcat@desktop008:a$ git annex list --allrepos
+here
+|origin
+||web
+|||bittorrent
+||||
+XX__ foo
+XX__ test
+[1042]anarcat@desktop008:a$ cd ../b
+[1043]anarcat@desktop008:b$ git annex wanted transfer
+git-annex: there is no available git remote named \"transfer\"
+[1044]anarcat@desktop008:b1$ git annex wanted . transfer
+wanted . git-annex: Parse error: Parse failure: near \"transfer\"
+[1045]anarcat@desktop008:b1$ git annex wanted . groupwanted
+wanted . ok
+(recording state in git...)
+[1046]anarcat@desktop008:b$ git annex group . transfer
+group . c ok
+(recording state in git...)
+[1047]anarcat@desktop008:b$ cd ../a
+[1048]anarcat@desktop008:a$ git annex group . source
+group . ok
+(recording state in git...)
+[1049]anarcat@desktop008:a$ git annex wanted . groupwanted
+wanted . ok
+(recording state in git...)
+[1050]anarcat@desktop008:a$ cd ../c
+[1051]anarcat@desktop008:c$ git annex wanted . groupwanted
+wanted . ok
+(recording state in git...)
+[1052]anarcat@desktop008:c$ git annex group . backup
+group . (merging synced/git-annex into git-annex...)
+(recording state in git...)
+ok
+(recording state in git...)
+[1053]anarcat@desktop008:c$ cd ../
+[1054]anarcat@desktop008:g-a$ cd b
+[1055]anarcat@desktop008:b$ cd ../a
+[1056]anarcat@desktop008:a$ git annex list --allrepos
+here
+|origin
+||web
+|||bittorrent
+||||
+XX__ foo
+XX__ test
+[1057]anarcat@desktop008:a$ git annex^C
+[1057]anarcat@desktop008:a130$ cd ../
+[1058]anarcat@desktop008:g-a$ git ^C
+[1058]anarcat@desktop008:g-a130$ cd b
+[1059]anarcat@desktop008:b$ git remote -v
+origin  ../c (fetch)
+origin  ../c (push)
+[1060]anarcat@desktop008:b$ git annex sync
+commit  ok
+pull origin
+remote: Counting objects: 23, done.
+remote: Compressing objects: 100% (19/19), done.
+remote: Total 23 (delta 8), reused 0 (delta 0)
+Unpacking objects: 100% (23/23), done.
+From ../c
+   ac66bb1..43cfe35  git-annex  -> origin/git-annex
+ok
+(merging origin/git-annex into git-annex...)
+[1061]anarcat@desktop008:b$ git annex list --allrepos
+here
+|origin
+||web
+|||bittorrent
+||||anarcat@desktop008:~/test/g-a/a
+|||||
+XX__X foo
+XX__X test
+[1062]anarcat@desktop008:b$ cd ../a/
+[1063]anarcat@desktop008:a$ echo bar > bar
+[1064]anarcat@desktop008:a$ git status
+On branch master
+nothing to commit, working directory clean
+[1065]anarcat@desktop008:a$ cd ^C
+[1065]anarcat@desktop008:a130$ git annex list --allrepos
+here
+|origin
+||web
+|||bittorrent
+||||
+XX__ bar
+XX__ foo
+XX__ test
+[1066]anarcat@desktop008:a$ cd ../b
+[1067]anarcat@desktop008:b$ git annex list --allrepos
+here
+|origin
+||web
+|||bittorrent
+||||anarcat@desktop008:~/test/g-a/a
+|||||
+XX__X bar
+XX__X foo
+XX__X test
+
    
+
+now it seems that setting repo `b` in the transfer group helped, but the files didn't get purged from `a` (or `b`, for that matter).
+
+setting the central `b` wanted expression seems to help in dropping the file from `a`, but not from `b`:
+
+
    +[1070]anarcat@desktop008:b$ cd ../a
+[1071]anarcat@desktop008:a$ echo quuex > quuex
+[1072]anarcat@desktop008:a$ git status
+On branch master
+nothing to commit, working directory clean
+[1073]anarcat@desktop008:a$ cd -
+/home/anarcat/test/g-a/b
+[1074]anarcat@desktop008:b$ git annex list --allrepos
+here
+|origin
+||web
+|||bittorrent
+||||anarcat@desktop008:~/test/g-a/a
+|||||
+XX__X bar
+XX__X foo
+X___X quuex
+XX__X test
+[1075]anarcat@desktop008:b$ cd ../c
+[1076]anarcat@desktop008:c$ git annex group
+git-annex: Specify a repository and a group.
+[1077]anarcat@desktop008:c1$ git annex group .
+(recording state in git...)
+backup
+[1078]anarcat@desktop008:c$ cd -
+/home/anarcat/test/g-a/b
+[1079]anarcat@desktop008:b$ git annex list --allrepos
+here
+|origin
+||web
+|||bittorrent
+||||anarcat@desktop008:~/test/g-a/a
+|||||
+XX__X bar
+XX__X foo
+X___X quuex
+XX__X test
+[1080]anarcat@desktop008:b$ git annex numcopies
+global numcopies is not set
+(default is 1)
+
    
+
+i think i'm almost getting this now. :)
+"""]]
diff --git a/doc/forum/mesh_configurations/comment_5_a181fde7b7f6d0559331e1afa395ba0a._comment b/doc/forum/mesh_configurations/comment_5_a181fde7b7f6d0559331e1afa395ba0a._comment
new file mode 100644
index 0000000000..4cc4121055
--- /dev/null
+++ b/doc/forum/mesh_configurations/comment_5_a181fde7b7f6d0559331e1afa395ba0a._comment
@@ -0,0 +1,15 @@
+[[!comment format=mdwn
+ username="joey"
+ subject="""comment 5"""
+ date="2015-08-19T17:54:36Z"
+ content="""
+You have B configured as a regular transfer repo, so it only wants to drop
+files once they have reached some client repos. Settings its preferred
+content to "not inallgroup=backup" should fix that.
+
+Also, the assistant can notice some configuration changes that are made
+while it's running, but maybe not all of them, and maybe it won't rescan
+files to transfer right away after such a change is made. I'd use `git
+annex sync --content` to test such changes, and save the assistant for once
+I have a working setup.
+"""]]
diff --git a/doc/forum/mesh_configurations/comment_6_7cb33d2416289f744828d8d9d24e9ef6._comment b/doc/forum/mesh_configurations/comment_6_7cb33d2416289f744828d8d9d24e9ef6._comment
new file mode 100644
index 0000000000..cd7e6777da
--- /dev/null
+++ b/doc/forum/mesh_configurations/comment_6_7cb33d2416289f744828d8d9d24e9ef6._comment
@@ -0,0 +1,117 @@
+[[!comment format=mdwn
+ username="anarcat"
+ subject="comment 6"
+ date="2015-08-19T18:13:22Z"
+ content="""
+right, i thought as well that the assistant wouldn't pickup some stuff...
+
+but `sync --content` doesn't behave as expected either:
+
+
    +[1081]anarcat@desktop008:b$ git annex list --allrepos
+here
+|origin
+||web
+|||bittorrent
+||||anarcat@desktop008:~/test/g-a/a
+|||||
+XX__X bar
+XX__X foo
+X___X quuex
+XX__X test
+[1082]anarcat@desktop008:b$ git annex sync --content
+commit  ok
+pull origin
+remote: Counting objects: 6, done.
+remote: Compressing objects: 100% (5/5), done.
+remote: Total 6 (delta 3), reused 0 (delta 0)
+Unpacking objects: 100% (6/6), done.
+From ../c
+   29d3b7b..6afbb52  git-annex  -> origin/git-annex
+ok
+(merging origin/git-annex into git-annex...)
+(recording state in git...)
+(recovering from race...)
+drop bar ok
+drop foo ok
+drop test ok
+pull origin
+ok
+(recording state in git...)
+push origin
+Counting objects: 18, done.
+Delta compression using up to 2 threads.
+Compressing objects: 100% (14/14), done.
+Writing objects: 100% (18/18), 1.47 KiB | 0 bytes/s, done.
+Total 18 (delta 11), reused 0 (delta 0)
+To ../c
+   873a124..8390704  git-annex -> synced/git-annex
+ok
+[1083]anarcat@desktop008:b$ git annex list --allrepos
+here
+|origin
+||web
+|||bittorrent
+||||anarcat@desktop008:~/test/g-a/a
+|||||
+_X__X bar
+_X__X foo
+X___X quuex
+_X__X test
+[1084]anarcat@desktop008:b$ git annex list --allrepos
+here
+|origin
+||web
+|||bittorrent
+||||anarcat@desktop008:~/test/g-a/a
+|||||
+_X__X bar
+_X__X foo
+X___X quuex
+_X__X test
+[1084]anarcat@desktop008:b$ git annex wanted .
+not inallgroup=backup
+[1085]anarcat@desktop008:b$ cd ../a
+[1086]anarcat@desktop008:a$ git annex list --allrepos
+here
+|origin
+||web
+|||bittorrent
+||||
+XX__ bar
+XX__ foo
+XX__ quuex
+XX__ test
+[1087]anarcat@desktop008:a$ git annex sync --content
+commit  ok
+pull origin
+remote: Counting objects: 114, done.
+remote: Compressing objects: 100% (94/94), done.
+remote: Total 114 (delta 54), reused 0 (delta 0)
+Receiving objects: 100% (114/114), 9.20 KiB | 0 bytes/s, done.
+Resolving deltas: 100% (54/54), completed with 6 local objects.
+From ../b
+   de5f95e..158c3cc  git-annex  -> origin/git-annex
+ok
+(merging origin/git-annex into git-annex...)
+pull origin
+ok
+[1088]anarcat@desktop008:a$ git annex list --allrepos
+here
+|origin
+||web
+|||bittorrent
+||||anarcat@desktop008:~/test/g-a/c
+|||||
+X___X bar
+X___X foo
+XX___ quuex
+X___X test
+[1089]anarcat@desktop008:a$ git annex wanted .
+groupwanted
+[1090]anarcat@desktop008:a$ git annex group .
+source
+
    
+
+files are still not removed from `a` and a few of them were dropped from `b`, but not all of them. but worse, one file still isn't sent to the backup server `c` at all.
+"""]]
diff --git a/doc/forum/mesh_configurations/comment_7_6c9d499f64067ee9d3721dc763f3c425._comment b/doc/forum/mesh_configurations/comment_7_6c9d499f64067ee9d3721dc763f3c425._comment
new file mode 100644
index 0000000000..1767d5389a
--- /dev/null
+++ b/doc/forum/mesh_configurations/comment_7_6c9d499f64067ee9d3721dc763f3c425._comment
@@ -0,0 +1,123 @@
+[[!comment format=mdwn
+ username="joey"
+ subject="""comment 7"""
+ date="2015-08-19T18:51:42Z"
+ content="""
+I'm afraid I don't have time to continue to read and try to debug
+transcripts of this being set up incorrectly in various ways. 
+
+So, here's a transcript of the configuration I described, which seems to be
+working as I'd expect it to work:
+
+	joey@darkstar:~/tmp>mkdir bench
+	joey@darkstar:~/tmp>cd bench
+	joey@darkstar:~/tmp/bench>git init A
+	Initialized empty Git repository in /home/joey/tmp/bench/A/.git/
+	joey@darkstar:~/tmp/bench>cd A
+	joey@darkstar:~/tmp/bench/A>git annex init
+	init  ok
+	(recording state in git...)
+	joey@darkstar:~/tmp/bench/A>git annex wanted . standard
+	wanted . ok
+	(recording state in git...)
+	joey@darkstar:~/tmp/bench/A>git annex group . source
+	group . ok
+	(recording state in git...)
+	joey@darkstar:~/tmp/bench/A>date > somefile
+	joey@darkstar:~/tmp/bench/A>git annex add
+	add somefile ok
+	(recording state in git...)
+	joey@darkstar:~/tmp/bench/A>git commit -m added
+	[master (root-commit) 4a322e1] added
+	 1 file changed, 1 insertion(+)
+	 create mode 120000 somefile
+	joey@darkstar:~/tmp/bench/A>cd ..
+	joey@darkstar:~/tmp/bench>git clone A B
+	Cloning into 'B'...
+	done.
+	joey@darkstar:~/tmp/bench>cd B
+	joey@darkstar:~/tmp/bench/B>git annex wanted . "not inallgroup=backup"
+	(merging origin/git-annex into git-annex...)
+	(recording state in git...)
+	wanted . ok
+	(recording state in git...)
+	joey@darkstar:~/tmp/bench/B>cd ..
+	joey@darkstar:~/tmp/bench>git clone B C
+	Cloning into 'C'...
+	done.
+	joey@darkstar:~/tmp/bench>cd C
+	joey@darkstar:~/tmp/bench/C>git annex group . backup
+	(merging origin/git-annex into git-annex...)
+	(recording state in git...)
+	group . ok
+	(recording state in git...)
+	joey@darkstar:~/tmp/bench/C>git annex wanted . standard
+	wanted . ok
+	(recording state in git...)
+	joey@darkstar:~/tmp/bench/C>cd ..
+	joey@darkstar:~/tmp/bench>cd B
+	joey@darkstar:~/tmp/bench/B>git remote add A ../A
+	joey@darkstar:~/tmp/bench/B>git remote add C ../C
+
+Now observe sync moving the file from A thru B to C:
+
+	joey@darkstar:~/tmp/bench/B>git annex sync --content
+	commit  ok
+	pull origin 
+	ok
+	pull C 
+	remote: Counting objects: 10, done.
+	remote: Compressing objects: 100% (9/9), done.
+	remote: Total 10 (delta 3), reused 0 (delta 0)
+	Unpacking objects: 100% (10/10), done.
+	From ../C
+	 * [new branch]      git-annex  -> C/git-annex
+	 * [new branch]      master     -> C/master
+	ok
+	pull A 
+	From ../A
+	 * [new branch]      git-annex  -> A/git-annex
+	 * [new branch]      master     -> A/master
+	ok
+	(merging C/git-annex into git-annex...)
+	get somefile (from origin...) ok
+	copy somefile copy somefile (to C...) ok
+	drop somefile ok
+	drop origin somefile ok
+	pull origin 
+	ok
+	pull C 
+	ok
+	pull A 
+	ok
+	(recording state in git...)
+	push origin 
+	Counting objects: 21, done.
+	Delta compression using up to 4 threads.
+	Compressing objects: 100% (19/19), done.
+	Writing objects: 100% (21/21), 2.19 KiB | 0 bytes/s, done.
+	Total 21 (delta 7), reused 0 (delta 0)
+	To /home/joey/tmp/bench/A
+	 * [new branch]      git-annex -> synced/git-annex
+	 * [new branch]      master -> synced/master
+	ok
+	push C 
+	Counting objects: 5, done.
+	Delta compression using up to 4 threads.
+	Compressing objects: 100% (4/4), done.
+	Writing objects: 100% (5/5), 474 bytes | 0 bytes/s, done.
+	Total 5 (delta 2), reused 0 (delta 0)
+	To ../C
+	 * [new branch]      git-annex -> synced/git-annex
+	 * [new branch]      master -> synced/master
+	ok
+	push A 
+	Everything up-to-date
+	ok
+	joey@darkstar:~/tmp/bench/B>git annex whereis
+	whereis somefile (1 copy) 
+	  	65092dc3-ea1e-4267-89b7-5fcb8df2c6ae -- joey@darkstar:~/tmp/bench/C [C]
+	ok
+
+Er, the 'A' remote in 'B' was unnecessary since A is origin. But otherwise, I think that's what you asked for.. HTH.
+"""]]
diff --git a/doc/forum/mesh_configurations/comment_8_a2b5da1ea55a222dd30f0e982d5ee807._comment b/doc/forum/mesh_configurations/comment_8_a2b5da1ea55a222dd30f0e982d5ee807._comment
new file mode 100644
index 0000000000..b77b1b0f80
--- /dev/null
+++ b/doc/forum/mesh_configurations/comment_8_a2b5da1ea55a222dd30f0e982d5ee807._comment
@@ -0,0 +1,197 @@
+[[!comment format=mdwn
+ username="anarcat"
+ subject="works with sync --content, but the assistant is slow to pickup tracking info"
+ date="2015-08-19T20:29:46Z"
+ content="""
+well, this is not exactly the topology i have here.
+
+you setup a topology like this:
+
+    A <- B <-> C
+
+`X <- Y` means `X is a remote of Y`.
+
+My topology is:
+
+    A -> B -> C
+
+So B can't directly manage objects from A. It can *receive* objects from it, but that's it.
+
+So here's a clearer transcript of the session, using the same semantics you have been using for the different repos, but with the remotes setup differently, as I describe above:
+
+
    +[1113]anarcat@desktop008:bench$ git init B
+Initialized empty Git repository in /home/anarcat/test/bench/B/.git/
+[1114]anarcat@desktop008:bench$ cd B/
+[1115]anarcat@desktop008:B$ git annex init
+init  ok
+(recording state in git...)
+[1116]anarcat@desktop008:B$ date > somefile
+[1117]anarcat@desktop008:B$ git annex add
+add somefile ok
+(recording state in git...)
+[1118]anarcat@desktop008:B$ git commit -madded
+[master (root-commit) d2f6177] added
+ 1 file changed, 1 insertion(+)
+ create mode 120000 somefile
+[1119]anarcat@desktop008:B$ git annex wanted . \"not inallgroup=backup\"
+wanted . ok
+(recording state in git...)
+[1138]anarcat@desktop008:B$ git remote add C ../C # actually did that later
+[1120]anarcat@desktop008:B$ cd ../
+[1121]anarcat@desktop008:bench$ git clone -o B B A
+Cloning into 'A'...
+done.
+[1122]anarcat@desktop008:bench$ git clone B C
+Cloning into 'C'...
+done.
+[1123]anarcat@desktop008:bench$ cd A
+[1124]anarcat@desktop008:A$ git annex wanted . standard
+(merging B/git-annex into git-annex...)
+(recording state in git...)
+wanted . ok
+(recording state in git...)
+[1125]anarcat@desktop008:A$ git annex group . source
+group . ok
+(recording state in git...)
+[1126]anarcat@desktop008:A$ cd ../C
+[1127]anarcat@desktop008:C$ git annex wanted . standard
+(merging origin/git-annex into git-annex...)
+(recording state in git...)
+wanted . ok
+(recording state in git...)
+[1128]anarcat@desktop008:C$ git annex group . backup
+group . ok
+(recording state in git...)
+[1142]anarcat@desktop008:C$ git remote rm origin # because this is S3 in production, and can't do anything on its own
+
    
+
+And of course now, it actually works fine, with `sync --content`:
+
+
    +[1144]anarcat@desktop008:A$ git annex sync --content
+commit  ok
+pull B
+remote: Counting objects: 20, done.
+remote: Compressing objects: 100% (18/18), done.
+remote: Total 20 (delta 8), reused 0 (delta 0)
+Unpacking objects: 100% (20/20), done.
+From /home/anarcat/test/bench/B
+   d2f6177..b625a42  master     -> B/master
+   0432685..2061cf9  git-annex  -> B/git-annex
+ok
+(merging B/git-annex into git-annex...)
+copy otherfile copy otherfile (to B...) ok
+drop otherfile ok
+pull B
+ok
+(recording state in git...)
+push B
+Counting objects: 8, done.
+Delta compression using up to 2 threads.
+Compressing objects: 100% (6/6), done.
+Writing objects: 100% (8/8), 720 bytes | 0 bytes/s, done.
+Total 8 (delta 3), reused 0 (delta 0)
+To /home/anarcat/test/bench/B
+   0432685..e0fccbd  git-annex -> synced/git-annex
+ok
+[1145]anarcat@desktop008:A$ cd ../B
+[1146]anarcat@desktop008:B$ git annex sync --content
+commit  ok
+pull C
+remote: Counting objects: 5, done.
+remote: Compressing objects: 100% (4/4), done.
+remote: Total 5 (delta 2), reused 0 (delta 0)
+Unpacking objects: 100% (5/5), done.
+From ../C
+   2061cf9..1d0a3d5  git-annex  -> C/git-annex
+ok
+(merging C/git-annex into git-annex...)
+(recording state in git...)
+copy otherfile copy otherfile (to C...) ok
+drop otherfile ok
+pull C
+ok
+(recording state in git...)
+push C
+Counting objects: 20, done.
+Delta compression using up to 2 threads.
+Compressing objects: 100% (16/16), done.
+Writing objects: 100% (20/20), 1.55 KiB | 0 bytes/s, done.
+Total 20 (delta 10), reused 0 (delta 0)
+To ../C
+   2061cf9..86a892f  git-annex -> synced/git-annex
+ok
+[1147]anarcat@desktop008:B$ git annex list
+here
+|C
+||web
+|||bittorrent
+||||
+_X__ otherfile
+_X__ somefile
+
    
+
+Now, that's all great for `sync --content` - but what about the assistant?
+
+
    +[1150]anarcat@desktop008:B$ git annex assistant
+[1151]anarcat@desktop008:B$ cd ../A
+[1152]anarcat@desktop008:A$ git annex assistant
+[1154]anarcat@desktop008:A$ date > foo
+[1157]anarcat@desktop008:A$ git annex list --allrepos
+here
+|B
+||web
+|||bittorrent
+||||anarcat@desktop008:~/test/bench/C
+|||||
+_X___ foo
+____X otherfile
+____X somefile
+[1158]anarcat@desktop008:A$ sleep 600; git annex list --allrepos
+here
+|B
+||web
+|||bittorrent
+||||anarcat@desktop008:~/test/bench/C
+|||||
+_X___ foo
+____X otherfile
+____X somefile
+
    
+
+so from `A`'s perspective, it looks like the file didn't migrate properly. *but* it actually did!
+
+
    +[1159]anarcat@desktop008:A$ cd ../B
+[1160]anarcat@desktop008:B$ git annex list --allrepos
+here
+|C
+||web
+|||bittorrent
+||||anarcat@desktop008:~/test/bench/A
+|||||
+_X___ foo
+_X___ otherfile
+_X___ somefile
+[1161]anarcat@desktop008:B$ cd -
+/home/anarcat/test/bench/A
+[1162]anarcat@desktop008:A$ git annex list --allrepos
+here
+|B
+||web
+|||bittorrent
+||||anarcat@desktop008:~/test/bench/C
+|||||
+_X___ foo
+____X otherfile
+____X somefile
+
    
+
+how long does it take for the assistant to start syncs like this? are those timers user-accessible somehow? this problem sure looks like [[bugs/sync-git-annex_branch_not_syncing_in_the_assistant]] - but maybe i'm confused there as well.
+
+anyways, it does seem like content actually does gets synced around properly by the assistant. i'll try to deploy the new preferred content expression in production and report back here.
+
+and sorry for the noisy pastes and hand-holding, but i was thoroughly confused by this one. i thought i had a good grasp on preferred content and all that, but it seems i was wrong... :(
+"""]]
diff --git a/doc/forum/mesh_configurations/comment_9_ac3e1faaefaed222f725345ab4b5f01a._comment b/doc/forum/mesh_configurations/comment_9_ac3e1faaefaed222f725345ab4b5f01a._comment
new file mode 100644
index 0000000000..33b6ab0a03
--- /dev/null
+++ b/doc/forum/mesh_configurations/comment_9_ac3e1faaefaed222f725345ab4b5f01a._comment
@@ -0,0 +1,89 @@
+[[!comment format=mdwn
+ username="anarcat"
+ subject="a bit more info and solutions"
+ date="2015-08-20T14:09:54Z"
+ content="""
+so short version here: thanks for your help and i figured it, there were problems with the S3 credentials perms not respecting `sharedRepository`, multiple group support confusion, bare/non-bare, groupwanted vs standard confusion and so on... now the files are migrating properly in production. hurray! i believe documentation could be improved, and i have questions about timeouts, below.
+
+so one thing that was definitely broken in production was that, on the central server, the S3 credentials were accessible only to the user that ran the \"enable s3\" command (that is `anarcat`). it was *not* accessible to the user actually running the assistant (the `git-annex` sandbox account), which made it simply impossible for the assistant to upload files to s3. so maybe one problem here is that the `.git/annex/creds` file do not respect the `core.sharedRepository = group` setting i have there...
+
+another problem in production, of course, was the *transfer* preferred group setting. once changed to `not inallgroup=backup`, things were better... but it was still keeping too many files. the problem then was that the repo was originally set to the *source* group (PEBKAC here again, sorry) and then it was *added* to the *standard* group, with the `git annex group . standard` command. i didn't expect that: i expected the group command to *change* the group, not to add to it. the documentation ([[git-annex-group]]) is clear enough, however, so this is yet another PEBKAC.
+
+Another possible problem is that the central server (`b`) is not a bare git repository. I am not sure why it was setup that way... maybe i was worried about bare repo suport and past experiences, or concerns about being able to actually interact with the files directly on the central server. i had to do `git co -b master synced/master` to eventually see the files locally, and this helped a little in diagnosing all of this.
+
+A problem remained after that though: files are *still* not removed from `A` in my tests in production. i don't understand why: `A` is setup as a source repository:
+
+
    +antoine@ip-10-88-150-10:/mnt/media$ git annex group .
+sourcethis 
+antoine@ip-10-88-150-10:/mnt/media$ git annex wanted .
+groupwanted
+
    
+
+yet some files have more than one copy:
+
+
    +antoine@ip-10-88-150-10:/mnt/media$ git annex list test
+here
+|origin
+||s3
+|||web (untrusted)
+||||bittorrent
+|||||
+X_X__ test/motd
+__X__ test/test1
+__X__ test/test2
+__X__ test/test3
+
    
+
+Indeed, it doesn't sem to want to drop that local file:
+
+
    +antoine@ip-10-88-150-10:/mnt/media$ git annex list --want-drop test
+here
+|origin
+||s3
+|||web (untrusted)
+||||bittorrent
+|||||
+__X__ test/test1
+__X__ test/test2
+__X__ test/test3
+
    
+
+It turns out that I had documentation wrong again about that as well: i was using *groupwanted* as a `wanted` expression, but the groups were standard groups, so git-annex was just failing to use the proper content expression. setting the `wanted` expression back to standard (yes, again as you showed, sorry about that) fixed the problem:
+
+
    +antoine@ip-10-88-150-10:/mnt/media$ sudo -u www-data -H git annex wanted . standard
+wanted . ok
+(recording state in git...)
+antoine@ip-10-88-150-10:/mnt/media$ git annex list --want-drop test
+here
+|origin
+||s3
+|||web (untrusted)
+||||bittorrent
+|||||
+X_X__ test/motd
+__X__ test/test1
+__X__ test/test2
+__X__ test/test3
+antoine@ip-10-88-150-10:/mnt/media$ sudo -u www-data -H git annex drop --auto test
+drop test/motd ok
+(recording state in git...)
+
    
+
+hurray!
+
+again, i apologise for all the hand holding here... as a software developper myself, i understand how frustrating it can be to try to make users come out of their cave of ignorance and see the light of day...
+
+but i do feel there could be more work done to clarify how all of this works. i will certainly try to give a few kicks in the [[preferred content]] section and maybe this forum post will be helpful for future endeavors... or maybe just write up a new tips page about such hairy setups?
+
+the questions that remain here for me are:
+
+* how long does the assistant wait before refreshing the wanted content expressions
+* how long it waits before triggering syncs
+* are those timeouts configurable
+
+thanks so much for helping us through this, it's really appreciated!
+"""]]
diff --git a/doc/forum/messed_up_annex_by_using_git_checkout.mdwn b/doc/forum/messed_up_annex_by_using_git_checkout.mdwn
new file mode 100644
index 0000000000..f9b2eb63c4
--- /dev/null
+++ b/doc/forum/messed_up_annex_by_using_git_checkout.mdwn
@@ -0,0 +1,40 @@
+Hello. Linux experienced user here, but with no development or git experience ever. This directly leads me to my biggest trouble with git-annex because I constantly suffer from misunderstanding each and everything. That being said, my problem is the following: I had the terrible idea to have my .thunderbird directory synchronized over several machines, leading the thunderbird profile to total corruption because after a few weeks I finally managed to have multiple machines accessing it. As I started trying to recover, things got worse and this is my last seek for help before starting over by creating the repositories from scratch.
+
+What I did:
+
+1. I did some research on how to recover an old state of the repo, which should not have been a problem because there is a "full backup" repo. I came across this [1] page and the pain started with me looking for my wanted commit to roll-back in "git log" and then did some tries in the way something like, "git checkout -b old-state4 012345678".
+
+2. Of course this recovered the whole repo and not only thunderbird, so I used a file synchronizer to put everything else back into place after the action.
+
+3. Unfortunately, the "branches" seem to have been messed up and the repos are no longer in sync.
+
+This is what the machine says that I have used to create the mess:
+
+    [2016-03-08 19:15:26.915116] Pusher: Syncing with host123
+    (recording state in git...)
+    To ssh://user@10.0.0.1/mnt/foo/bar
+       d35c699..fed0636  git-annex -> synced/git-annex
+       a44bfb2..818b7b5  annex/direct/old_state4 -> synced/old_state4
+
+This is what another machine says:
+
+    [2016-03-08 21:17:48.649949] Pusher: Syncing with host123
+    (recording state in git...)
+    To ssh://user@10.0.0.1/mnt/foo/bar
+    
+       423f50f..4c8fad8  annex/direct/master -> q/annex/direct/master
+       2a67458..fed0636  git-annex  -> host123/git-annex
+       6a1076b..4c8fad8  master     -> host123/master
+       7f55414..818b7b5  old_state4 -> host123/old_state4
+
+Long story, short...
+
+I apologize for being a total git noob while at the same time performing git magic leading into a total desaster.
+However, I hope someone can give me a hint what to do to have the "old_state4" solved?
+
+Also, I really would like to get used to the git internals but since git is quite powerful, I always get overwhelmed because the tutorials out there are either developer-focused (which I'm not in the correct target group) or they simply cover each and everything (which is of no use because I don't want to administrate a GitHub repo but a private git-annex for my files and documents). 
+
+Thank you so much!
+
+
+[1] http://stackoverflow.com/questions/4114095/revert-git-repo-to-a-previous-commit
diff --git a/doc/forum/messed_up_annex_by_using_git_checkout/comment_1_04f33895f6451690372595c576be2811._comment b/doc/forum/messed_up_annex_by_using_git_checkout/comment_1_04f33895f6451690372595c576be2811._comment
new file mode 100644
index 0000000000..cd480bc153
--- /dev/null
+++ b/doc/forum/messed_up_annex_by_using_git_checkout/comment_1_04f33895f6451690372595c576be2811._comment
@@ -0,0 +1,11 @@
+[[!comment format=mdwn
+ username="joey"
+ subject="""comment 1"""
+ date="2016-03-14T17:27:17Z"
+ content="""
+You don't seem to really say what the problem is, but I guess it has
+something to do with this `old_state4` branch which you have made.
+
+Maybe check what `git branch` shows to see if that branch is checked out,
+and if so, `git checkout master` to get back to the master branch.
+"""]]
diff --git a/doc/forum/metadata_limits.mdwn b/doc/forum/metadata_limits.mdwn
new file mode 100644
index 0000000000..523db70d85
--- /dev/null
+++ b/doc/forum/metadata_limits.mdwn
@@ -0,0 +1,10 @@
+Hello,
+
+I was wondering what `The metadata values can contain absolutely anything you like -- but you're recommended to keep it simple and reasonably short.` listed on the [[metadata]] page really means?
+
+
+Here is my use case (which is part of a larger solution)...
+
+I've been storing my photos in git annex for a while (years perhaps), I (or actually the wife) tried to find one and couldn't, so I'm thinking I need to add metadata (Tags and captions, etc) to each picture. I was looking for ways to do so. I was actually considering using gthumb or shotwell or similar to create the tags and somehow pull that out and associate it with the file (either with a plugin or program) and then sort out how to distribute the metadata with the pics. The git annex metadata seems like a reasonable solution for keeping metadata with the pictures. The question is in this case, how long can the text in say a caption tag before git-annex has problems? I could probably do something like caption- to handle really long captions, but then captions are short by definition, so perhaps I should say description instead. And for this particular use case, I am aware this doesn't solve the problem of actually creating the data, just trying to sort out if get annex metadata to the right solution to tie the metadata to a pic, err file.
+
+
diff --git a/doc/forum/metadata_limits/comment_1_a20bbe49f2f23d23131221ec79c8b973._comment b/doc/forum/metadata_limits/comment_1_a20bbe49f2f23d23131221ec79c8b973._comment
new file mode 100644
index 0000000000..d296a1cb99
--- /dev/null
+++ b/doc/forum/metadata_limits/comment_1_a20bbe49f2f23d23131221ec79c8b973._comment
@@ -0,0 +1,28 @@
+[[!comment format=mdwn
+ username="joey"
+ subject="""comment 1"""
+ date="2014-12-02T18:39:34Z"
+ content="""
+There's no particular size limit for metadata. But things may not scale
+well past some size.
+
+Having a big metadata value could make git-annex be a little slow when
+dealing with metadata, since it has to parse the metadata log file.
+So if you stored say, 1 mb in metadata, that's one mb of disk IO and data
+processing every time git-annex looks at the metadata for that file.
+
+All the metadata for a file will also be buffered in memory when git-annex
+is looking at that file's metadata. So `git-annex view` would use 1 mb or
+so more memory in the above example. (But git-annex only looks at one
+file's metadata at a time, so its memory use won't grow if you have a lot
+of files with metadata.)
+
+Also of course, the metadata is stored in git, and so it will make your git
+repository bigger.
+
+I would not expect any reasonable description of a photo to be large enough
+for its size to be a problem.
+
+I'd even feel ok with putting the full text of a license in as metadata
+(GPL is 30kb, and git would deduplicate redundant license metadata).
+"""]]
diff --git a/doc/forum/metadata_limits/comment_2_4e8995ccd5fc2e5ee5d89a0780e5df9d._comment b/doc/forum/metadata_limits/comment_2_4e8995ccd5fc2e5ee5d89a0780e5df9d._comment
new file mode 100644
index 0000000000..097558f861
--- /dev/null
+++ b/doc/forum/metadata_limits/comment_2_4e8995ccd5fc2e5ee5d89a0780e5df9d._comment
@@ -0,0 +1,10 @@
+[[!comment format=mdwn
+ username="http://jamestechnotes.com/"
+ nickname="James"
+ subject="comment 2"
+ date="2014-12-02T22:37:33Z"
+ content="""
+Thanks Joey.
+
+Now to sort out how to create the data ;) -- [[james]]
+"""]]
diff --git a/doc/forum/migrate_existing_git_repository_to_git-annex.mdwn b/doc/forum/migrate_existing_git_repository_to_git-annex.mdwn
new file mode 100644
index 0000000000..f673de765b
--- /dev/null
+++ b/doc/forum/migrate_existing_git_repository_to_git-annex.mdwn
@@ -0,0 +1,66 @@
+I have a large git repository with binary files scattered over different branches. I want to switch to git-annex mainly for performance reasons, but I don't want to loose my history.
+
+I tried to rewrite the (cloned) repository with git-filter-branch but failed miserably for several reasons:
+
+* --tree-filter performs its operations in a temporary directory (.git-rewrite/t/) so the symlinks point to the wrong destination (../../.git/annex/).
+* annex log files are stored in .git-annex/ instead of .git-rewrite/t/.git-annex/ so the filter operation misses them
+
+Any suggestions how to proceed?
+
+EDIT 3/2/2010
+I finally got it working for my purposes. Hardest part was preserving the branches while injecting the new `git annex setup` base commit.
+
+#### Clone repository
+    git clone original migrate
+    cd migrate
+    git checkout mybranch
+    git checkout master
+    git remote rm origin
+
+#### Inject `git annex setup` base commit and repair branches
+    git symbolic-ref HEAD refs/heads/newroot
+    git rm --cached *
+    git clean -f -d
+    git annex init master
+    echo \*.rpm annex.backend=SHA1 >> .gitattributes
+    git commit -m "store rpms in git annex" .gitattributes
+    git cherry-pick $(git rev-list --reverse master | head -1)
+    git rebase --onto newroot newroot master
+    git rebase --onto master mybranch~1 mybranch
+    git branch -d newroot
+
+#### Migrate repository
+    mkdir .temp
+    cp .git-annex/* .temp/
+    MYWORKDIR=$(pwd) git filter-branch \
+     --tag-name-filter cat \
+     --tree-filter '
+        mkdir -p .git-annex;
+        cp ${MYWORKDIR}/.temp/* .git-annex/;
+        for rpm in $(git ls-files | grep "\.rpm$"); do
+            echo;
+            git annex add $rpm;
+            annexdest=$(readlink $rpm);
+            if [ -e .git-annex/$(basename $annexdest).log ]; then
+                echo "FOUND $(basename $annexdest).log";
+            else
+                echo "COPY $(basename $annexdest).log";
+                cp ${MYWORKDIR}/.git-annex/$(basename $annexdest).log .git-annex/;
+                cp ${MYWORKDIR}/.git-annex/$(basename $annexdest).log ${MYWORKDIR}/.temp/;
+            fi;
+            ln -sf ${annexdest#../../} $rpm;
+        done;
+        git reset HEAD .git-rewrite;
+        :
+        ' -- $(git branch | cut -c 3-)
+    rm -rf .temp
+    git reset --hard
+
+
+TODO:
+
+* Find a way to repair branches automatically (detect branch points and run appropriate `git rebase` commands)
+
+I'll be happy to try any suggestions to improve this migration script.
+
+P.S. Is there a way to edit comments?
diff --git a/doc/forum/migrate_existing_git_repository_to_git-annex/comment_1_4181bf34c71e2e8845e6e5fb55d53381._comment b/doc/forum/migrate_existing_git_repository_to_git-annex/comment_1_4181bf34c71e2e8845e6e5fb55d53381._comment
new file mode 100644
index 0000000000..e88794d621
--- /dev/null
+++ b/doc/forum/migrate_existing_git_repository_to_git-annex/comment_1_4181bf34c71e2e8845e6e5fb55d53381._comment
@@ -0,0 +1,10 @@
+[[!comment format=mdwn
+ username="http://joey.kitenet.net/"
+ nickname="joey"
+ subject="comment 1"
+ date="2011-02-25T05:16:48Z"
+ content="""
+I don't know how to approach this yet, but I support the idea -- it would be great if there was a tool that could punch files out of git history and put them in the annex. (Of course with typical git history rewriting caveats.)
+
+Sounds like it might be enough to add a switch to git-annex that overrides where it considers the top of the git repository to be? 
+"""]]
diff --git a/doc/forum/migrate_existing_git_repository_to_git-annex/comment_2_5f08da5e21c0b3b5a8d1e4408c0d6405._comment b/doc/forum/migrate_existing_git_repository_to_git-annex/comment_2_5f08da5e21c0b3b5a8d1e4408c0d6405._comment
new file mode 100644
index 0000000000..71a40ad8cb
--- /dev/null
+++ b/doc/forum/migrate_existing_git_repository_to_git-annex/comment_2_5f08da5e21c0b3b5a8d1e4408c0d6405._comment
@@ -0,0 +1,60 @@
+[[!comment format=mdwn
+ username="tyger"
+ ip="80.66.20.180"
+ subject="comment 2"
+ date="2011-03-01T14:07:50Z"
+ content="""
+My current workflow looks like this (I'm still experimenting):
+
+### Create backup clone for migration
+
+    git clone original migrate
+    cd migrate
+    for branch in $(git branch -a | grep remotes/origin | grep -v HEAD); do git checkout --track $branch; done
+
+### Inject git annex initialization at repository base
+
+    git symbolic-ref HEAD refs/heads/newroot
+    git rm --cached *.rpm
+    git clean -f -d
+    git annex init master
+    git cherry-pick $(git rev-list --reverse master | head -1)
+    git rebase --onto newroot newroot master
+    git rebase master mybranch # how to automate this for all branches?
+    git branch -d newroot
+
+### Start migration with tree filter
+
+    echo \*.rpm annex.backend=SHA1 > .git/info/attributes
+    MYWORKDIR=$(pwd) git filter-branch --tree-filter ' \
+        if [ ! -d .git-annex ]; then \
+            mkdir .git-annex; \
+            cp ${MYWORKDIR}/.git-annex/uuid.log .git-annex/; \
+            cp ${MYWORKDIR}/.gitattributes .; \
+        fi
+        for rpm in $(git ls-files | grep \"\.rpm$\"); do \
+            echo; \
+            git annex add $rpm; \
+            annexdest=$(readlink $rpm); \
+            if [ -e .git-annex/$(basename $annexdest).log ]; then \
+                echo \"FOUND $(basename $annexdest).log\"; \
+            else \
+                echo \"COPY $(basename $annexdest).log\"; \
+                cp ${MYWORKDIR}/.git-annex/$(basename $annexdest).log .git-annex/; \
+            fi; \
+            ln -sf ${annexdest#../../} $rpm; \
+        done; \
+        git reset HEAD .git-rewrite; \
+        : \
+        ' -- $(git branch | cut -c 3-)
+    rm -rf .temp
+    git reset --hard
+
+
+There are still some drawbacks:
+
+* git history shows that git annex log files are modified with each checkin
+* branches have to be rebased manually before starting migration
+
+
+"""]]
diff --git a/doc/forum/migrate_existing_git_repository_to_git-annex/comment_3_f483038c006cf7dcccf1014fa771744f._comment b/doc/forum/migrate_existing_git_repository_to_git-annex/comment_3_f483038c006cf7dcccf1014fa771744f._comment
new file mode 100644
index 0000000000..90bf23b6cf
--- /dev/null
+++ b/doc/forum/migrate_existing_git_repository_to_git-annex/comment_3_f483038c006cf7dcccf1014fa771744f._comment
@@ -0,0 +1,12 @@
+[[!comment format=mdwn
+ username="tyger"
+ ip="80.66.20.180"
+ subject="comment 3"
+ date="2011-03-02T08:15:37Z"
+ content="""
+> Sounds like it might be enough to add a switch to git-annex that overrides where it considers the top of the git repository to be?
+
+It should sufficient to honor GIT_DIR/GIT_WORK_TREE/GIT_INDEX_FILE environment variables. git filter-branch sets GIT_WORK_TREE to ., but this can be mitigated by starting the filter script with 'GIT_WORK_TREE=$(pwd $GIT_WORK_TREE)'. E.g. GIT_DIR=/home/tyger/repo/.git, GIT_WORK_TREE=/home/tyger/repo/.git-rewrite/t, then git annex should be able to compute the correct relative path or maybe use absolute pathes in symlinks.
+
+Another problem I observed is that git annex add automatically commits the symlink; this behaviour doesn't work well with filter-tree. git annex commits the wrong path (.git-rewrite/t/LINK instead of LINK). Also filter-tree doesn't expect that the filter script commmits anything; new files in the temporary work tree will be committed by filter-tree on each iteration of the filter script (missing files will be removed).
+"""]]
diff --git a/doc/forum/migrate_existing_git_repository_to_git-annex/comment_4_057f0079fbee3451ccda08026bab21d4._comment b/doc/forum/migrate_existing_git_repository_to_git-annex/comment_4_057f0079fbee3451ccda08026bab21d4._comment
new file mode 100644
index 0000000000..e8495560ae
--- /dev/null
+++ b/doc/forum/migrate_existing_git_repository_to_git-annex/comment_4_057f0079fbee3451ccda08026bab21d4._comment
@@ -0,0 +1,20 @@
+[[!comment format=sh
+ username="https://www.google.com/accounts/o8/id?id=AItOawll4Kgp7nMuOKdB0FfbcYZ3KRq7HCS0Slc"
+ nickname="Laura"
+ subject="Rebase all branches"
+ date="2014-01-16T17:47:45Z"
+ content="""
+
+For the portion: git rebase master mybranch # how to automate this for all branches?
+
+Try this:
+
+branch_to_ignore='git-annex|master|newroot'
+for branch in $(git for-each-ref --sort=-committerdate refs/heads --format='%(refname:short)' | egrep -v $branch_to_ignore )
+    do git rebase --onto master \"$branch~\" \"$branch\"
+    echo \"Rebasing branch $branch onto master....\" 
+done
+
+Feel free to add/correct as necessary
+
+"""]]
diff --git a/doc/forum/migration_to_git-annex_and_rsync.mdwn b/doc/forum/migration_to_git-annex_and_rsync.mdwn
new file mode 100644
index 0000000000..d99dab8728
--- /dev/null
+++ b/doc/forum/migration_to_git-annex_and_rsync.mdwn
@@ -0,0 +1,33 @@
+When migrating large file repositories to git-annex that are backuped in a way that uses an rsync-style mechanism (e.g. [dirvish](http://www.dirvish.org/)) and thus keeps incremental backups small by using hardlinks, space can be saved by manually reflecting the migration on the backup. So, instead of making a last pre-git-annex backup, migrating, and duplicating all backupped data with the next backup, I used the attached migrate.py file below, and it saved me roughly a day of backuping.
+
+A note on terminology: "migrating" here means migrating from not using git-annex at all to using it, not to the ``git annex migrate`` command, for which a similar but different solution may be created.
+
+**WARNING**: This is a quickly hacked-together script. It worked for me, but is untested apart from that. It's just a dozen lines of code, so have a look at it and make sure you understand what it does, and what migrate.sh looks like. Take special care as this tampers with your backups, and if something goes wrong, well...
+
+First, have an up-to-date backup; then, git annex init / add etc as described in the [[walkthrough]]. In the directory in which you use git-annex, run:
+
+    $ python migrate.py > migrate.sh
+
+Then copy the resulting migrate.sh to the equivalent location inside your backups and run it there. It will move all files that are now symlinked on the master to their new positions according to the symlinks (inside .git/annex/objects), but not create the symlinks (you will do a backup later anyway).
+
+After that, do a backup as usual. As rsync sees the moved files at their new locations, it will accept them and not duplicate the data.
+
+**migrate.py**:
+
+    #!/usr/bin/env python
+    
+    import os
+    from pipes import quote
+    
+    print "#!/bin/sh"
+    print "set -e"
+    print ""
+    
+    for (dirpath, dirnames, filenames) in os.walk("."):
+    	for f in filenames:
+    		fn = os.path.join(dirpath, f)
+    		if os.path.islink(fn):
+    			link = os.path.normpath(os.path.join(dirpath, os.readlink(fn)))
+    			assert link.startswith(".git/annex/objects/")
+    			print "mkdir -p %s"%quote(os.path.dirname(link))
+    			print "mv %s %s"%(quote(fn), quote(link))
diff --git a/doc/forum/misctmp_filling_up.mdwn b/doc/forum/misctmp_filling_up.mdwn
new file mode 100644
index 0000000000..a633760c97
--- /dev/null
+++ b/doc/forum/misctmp_filling_up.mdwn
@@ -0,0 +1,11 @@
+My directory .git/annex/misctmp is quite filled up with files like P39923, P33083, and also with 7 characters P310000 up to P331998. The beginning of these filenames may come from the annexed files, which all start with 'P3'.
+
+There is a total of 12198 of them (4.0TB... yes this repo is quite big ;)).
+
+Each of those has the content of individual annexed files (about 300-400MB targzipped files). Sometimes they are hard liked to each other, up to 8 copies.
+
+Since I have copied+dropped the whole repo content to other repos, there should be nothing left locally. I have verified several of those files (the annexed files which are identical to the misctmp/* files), are they indeed are located elsewhere, and not "here".
+
+So I was wondering if it is safe to remove them, and why are they not listed by git annex unused?
+
+Thanks for your help!!
diff --git a/doc/forum/misctmp_filling_up/comment_1_2739dec72fe0950dd070c8fab9fbd751._comment b/doc/forum/misctmp_filling_up/comment_1_2739dec72fe0950dd070c8fab9fbd751._comment
new file mode 100644
index 0000000000..9d1713745c
--- /dev/null
+++ b/doc/forum/misctmp_filling_up/comment_1_2739dec72fe0950dd070c8fab9fbd751._comment
@@ -0,0 +1,10 @@
+[[!comment format=mdwn
+ username="http://joeyh.name/"
+ ip="209.250.56.114"
+ subject="comment 1"
+ date="2014-04-27T00:16:49Z"
+ content="""
+You can delete them.
+
+AFAIK this should only happen if the assistant is interrupted while it's adding files. I plan to make the assistant clean up old tmp files on startup.
+"""]]
diff --git a/doc/forum/misctmp_filling_up/comment_2_440081b5e2b9b5b19e8cd5db3649a976._comment b/doc/forum/misctmp_filling_up/comment_2_440081b5e2b9b5b19e8cd5db3649a976._comment
new file mode 100644
index 0000000000..5048f601e8
--- /dev/null
+++ b/doc/forum/misctmp_filling_up/comment_2_440081b5e2b9b5b19e8cd5db3649a976._comment
@@ -0,0 +1,10 @@
+[[!comment format=mdwn
+ username="https://www.google.com/accounts/o8/id?id=AItOawmdbVIGiDH8KarAGAy8y2FHJD_F990JzXI"
+ nickname="François"
+ subject="comment 2"
+ date="2014-04-27T05:21:48Z"
+ content="""
+Indeed I had to interrupt the add a few times.
+
+Thanks!
+"""]]
diff --git a/doc/forum/mistakenly_checked___42__files__42___into_an_annex.__bummer.mdwn b/doc/forum/mistakenly_checked___42__files__42___into_an_annex.__bummer.mdwn
new file mode 100644
index 0000000000..1e2e44cfab
--- /dev/null
+++ b/doc/forum/mistakenly_checked___42__files__42___into_an_annex.__bummer.mdwn
@@ -0,0 +1,3 @@
+A couple times working in an annex manually, I accidentally did "git add" instead of "git annex add" and ended up with files checked into my repo instead of symlinks.  So now there is some file content in git, rather than in the annex.  Is there any way to root that out, or is it there forever?  (My limited knowledge of git says: it's there forever, unless maybe you go into the branches where it lives, do an interactive rebase to the time before it was added, remove that commit, replay history, and then garbage collect, but I have no idea if that would suddenly break git annex, and it sounds painful anyway.)
+
+If say I were a neat freak and wanted to just start over with a clean annex, I imagine the thing to do would be to just start the hell over -- and if I wanted to do that, the way to go would be to get into a full repository, do a "git annex uninit" and then throw away the .git directory, then do "git init" and "git annex init" and then "git annex add ." ?
diff --git a/doc/forum/mistakenly_checked___42__files__42___into_an_annex.__bummer/comment_1_752db25abb647804a1cc12c5b247378a._comment b/doc/forum/mistakenly_checked___42__files__42___into_an_annex.__bummer/comment_1_752db25abb647804a1cc12c5b247378a._comment
new file mode 100644
index 0000000000..486d434803
--- /dev/null
+++ b/doc/forum/mistakenly_checked___42__files__42___into_an_annex.__bummer/comment_1_752db25abb647804a1cc12c5b247378a._comment
@@ -0,0 +1,10 @@
+[[!comment format=mdwn
+ username="http://joeyh.name/"
+ ip="4.154.6.49"
+ subject="comment 1"
+ date="2012-12-02T05:41:56Z"
+ content="""
+You can rebase or use git-filter-branch to rewrite history. As long as you don't touch the git-annex branch, git-annex won't care, at all. Have at it!
+
+Your uninit plan would also work.
+"""]]
diff --git a/doc/forum/mistakenly_checked___42__files__42___into_an_annex.__bummer/comment_2_db6f4959c35732f72e7a90bd9f4c665c._comment b/doc/forum/mistakenly_checked___42__files__42___into_an_annex.__bummer/comment_2_db6f4959c35732f72e7a90bd9f4c665c._comment
new file mode 100644
index 0000000000..9c7f35a771
--- /dev/null
+++ b/doc/forum/mistakenly_checked___42__files__42___into_an_annex.__bummer/comment_2_db6f4959c35732f72e7a90bd9f4c665c._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="http://edheil.wordpress.com/"
+ ip="173.162.44.162"
+ subject="comment 2"
+ date="2012-12-03T15:46:48Z"
+ content="""
+Thanks!  I ended up going with the uninit plan.
+"""]]
diff --git a/doc/forum/monitoring_disk_usage_by_different_repos_on_same_drive.mdwn b/doc/forum/monitoring_disk_usage_by_different_repos_on_same_drive.mdwn
new file mode 100644
index 0000000000..698413ce34
--- /dev/null
+++ b/doc/forum/monitoring_disk_usage_by_different_repos_on_same_drive.mdwn
@@ -0,0 +1,7 @@
+I've setup a server using gitolite3 to host repositories for our users. All these repos share the same disk (2.5 Tb ;-). They're mainly used as backup of all contents.
+
+I'd like to monitor disk usage (possibly with statistics related to git-annex specificities) for these repos.
+
+Would anyone suggest a solution (web based maybe), for easily monitoring the growth of the repos storage ?
+
+Thanks in advance.
diff --git a/doc/forum/monitoring_disk_usage_by_different_repos_on_same_drive/comment_1_04718e5f87a7e0521a3165bc9fc951a8._comment b/doc/forum/monitoring_disk_usage_by_different_repos_on_same_drive/comment_1_04718e5f87a7e0521a3165bc9fc951a8._comment
new file mode 100644
index 0000000000..17ad194039
--- /dev/null
+++ b/doc/forum/monitoring_disk_usage_by_different_repos_on_same_drive/comment_1_04718e5f87a7e0521a3165bc9fc951a8._comment
@@ -0,0 +1,12 @@
+[[!comment format=mdwn
+ username="joey"
+ subject="""comment 1"""
+ date="2015-09-29T15:34:08Z"
+ content="""
+If you go to one of the repos and run `git annex info --json --bytes`
+you'll get a json object that has keys including "local annex size".
+You can extract that value, and feed it into a graphing engine, such
+as graphite, or whatever to generate graphs of the repo sizes over time.
+
+(Of course, you could do the same thing with `du`)
+"""]]
diff --git a/doc/forum/more_intelligent_copy_.mdwn b/doc/forum/more_intelligent_copy_.mdwn
new file mode 100644
index 0000000000..1c9889a74e
--- /dev/null
+++ b/doc/forum/more_intelligent_copy_.mdwn
@@ -0,0 +1,15 @@
+Hi,
+
+I noticed, that 
+
+git annex copy --to REMOTE FILES
+
+and 
+
+git annex copy --to REMOTE --not --in REMOTE FILES
+
+behave differently. The first does not check, whether file contents are already in the remote the latter does that. I realize that this mimics "normal" (UNIX) copy behaviour but I was not entirely certain this was desired.
+Depending on the type of the remote and its configuration (encryption) the latter is considerably faster.
+
+Just my two cents.
+
diff --git a/doc/forum/more_intelligent_copy_/comment_1_526f6a007f44f389ef7c904024752541._comment b/doc/forum/more_intelligent_copy_/comment_1_526f6a007f44f389ef7c904024752541._comment
new file mode 100644
index 0000000000..9b5866c5c5
--- /dev/null
+++ b/doc/forum/more_intelligent_copy_/comment_1_526f6a007f44f389ef7c904024752541._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="annexuser"
+ avatar="http://cdn.libravatar.org/avatar/6ae692503ee113b1ab7b329b40084d5c"
+ subject="comment 1"
+ date="2016-12-13T02:00:08Z"
+ content="""
+git annex copy --to REMOTE FILES --fast
+"""]]
diff --git a/doc/forum/more_intelligent_copy_/comment_2_7b3f5d2e9de4b13de821177db2f57bcd._comment b/doc/forum/more_intelligent_copy_/comment_2_7b3f5d2e9de4b13de821177db2f57bcd._comment
new file mode 100644
index 0000000000..474a068b65
--- /dev/null
+++ b/doc/forum/more_intelligent_copy_/comment_2_7b3f5d2e9de4b13de821177db2f57bcd._comment
@@ -0,0 +1,15 @@
+[[!comment format=mdwn
+ username="joey"
+ subject="""comment 2"""
+ date="2016-12-13T16:51:03Z"
+ content="""
+`git annex copy --fast` has the same behavior as `--not --in REMOTE`
+
+The reason this is not the default behavior is that git-annex's location
+tracking information can sometimes be out of date, and then those
+two will not copy some files despite their content not being any longer
+in the remote. This won't lead to data loss, but could result
+in unexpected behavior, and so the slower, more understandable behavior
+is used by default. (Although I sometimes go back and forth on switching
+it.)
+"""]]
diff --git a/doc/forum/multiple_git_repositories_inside_git_annex_assistant_repository.mdwn b/doc/forum/multiple_git_repositories_inside_git_annex_assistant_repository.mdwn
new file mode 100644
index 0000000000..b9500fa10e
--- /dev/null
+++ b/doc/forum/multiple_git_repositories_inside_git_annex_assistant_repository.mdwn
@@ -0,0 +1,19 @@
+I have a Desktop computer at home and another at work, and work from home every other day.  I'd like to use git-annex assistant to keep my git checkouts in sync across both of them.  I may be a bit unusual in that I don't always commit before going home.
+
+I also would like to have anything in the git stash synchronized between the two, the .git/config file, etc.
+
+So to be clear, I have a file structure like the following:
+
+    ./annex/.git
+    ./annex/project/README.md
+    ./annex/project/.git
+    ./annex/project2/README.md
+    ./annex/project2/.git
+
+In my testing it seems that the "project/.git" folders aren't synchronized between my two desktops, even though the rest of the files are.  A workaround is to rename "project/.git" to something else, like "project/.gitfoo", and then use --git-dir when issuing my git commands.
+
+Is this something that can be worked around?  I apologize if this is covered elsewhere, as I wasn't thinking of the right terms to search for.
+
+I've looked through the code and can't find anything obvious.  I imagine this is because git hard-codes all ".git" subdirectories as something it should ignore.
+
+Is there a better workaround?  I realize this is probably a niche use case.
diff --git a/doc/forum/multiple_git_repositories_inside_git_annex_assistant_repository/comment_1_419b27cb1c71bce021ef9f2e471aa92e._comment b/doc/forum/multiple_git_repositories_inside_git_annex_assistant_repository/comment_1_419b27cb1c71bce021ef9f2e471aa92e._comment
new file mode 100644
index 0000000000..8d698b1ba6
--- /dev/null
+++ b/doc/forum/multiple_git_repositories_inside_git_annex_assistant_repository/comment_1_419b27cb1c71bce021ef9f2e471aa92e._comment
@@ -0,0 +1,10 @@
+[[!comment format=mdwn
+ username="jewel"
+ ip="23.30.55.105"
+ subject="comment 1"
+ date="2014-10-12T05:22:03Z"
+ content="""
+Now I've found the relevant bug: `http://git-annex.branchable.com/bugs/Can__39__t_add_a_git_repo_to_git_annex:___34__Invalid_path_repo__47__.git__47__X__34___for_many_X`.
+
+Note the symlink workaround near the end.  A quick test shows that it's working great.  I'll try it out for a few months and report back.
+"""]]
diff --git a/doc/forum/multiple_git_repositories_inside_git_annex_assistant_repository/comment_2_dae4c7a42080dd89150159b2946839b1._comment b/doc/forum/multiple_git_repositories_inside_git_annex_assistant_repository/comment_2_dae4c7a42080dd89150159b2946839b1._comment
new file mode 100644
index 0000000000..2f574c8fe5
--- /dev/null
+++ b/doc/forum/multiple_git_repositories_inside_git_annex_assistant_repository/comment_2_dae4c7a42080dd89150159b2946839b1._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="http://joeyh.name/"
+ ip="209.250.56.54"
+ subject="comment 2"
+ date="2014-10-12T17:30:23Z"
+ content="""
+This is fundamentally not a good idea. All  it takes is one conflict inside the .git directories, and your git repositories will be trashed/corrupted. The page you link to has comments explaining why.
+"""]]
diff --git a/doc/forum/multiple_git_repositories_inside_git_annex_assistant_repository/comment_3_9d9fa65559ba4bb0e4676289b5a65684._comment b/doc/forum/multiple_git_repositories_inside_git_annex_assistant_repository/comment_3_9d9fa65559ba4bb0e4676289b5a65684._comment
new file mode 100644
index 0000000000..483ff007e8
--- /dev/null
+++ b/doc/forum/multiple_git_repositories_inside_git_annex_assistant_repository/comment_3_9d9fa65559ba4bb0e4676289b5a65684._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="jewel"
+ ip="23.30.55.105"
+ subject="comment 3"
+ date="2014-10-12T19:04:00Z"
+ content="""
+Thanks for the warning.  I'll keep hourly incremental backups (using obnam) on each computer just in case something terrible happens.
+"""]]
diff --git a/doc/forum/multiple_git_repositories_inside_git_annex_assistant_repository/comment_4_4e89b3590cc33b2565cd173ef7c85013._comment b/doc/forum/multiple_git_repositories_inside_git_annex_assistant_repository/comment_4_4e89b3590cc33b2565cd173ef7c85013._comment
new file mode 100644
index 0000000000..b32abd3ecd
--- /dev/null
+++ b/doc/forum/multiple_git_repositories_inside_git_annex_assistant_repository/comment_4_4e89b3590cc33b2565cd173ef7c85013._comment
@@ -0,0 +1,11 @@
+[[!comment format=mdwn
+ username="https://www.google.com/accounts/o8/id?id=AItOawlM_DRhi_5pJrTA0HbApHR25iAgy-NBXTY"
+ nickname="Tor Arne"
+ subject="comment 4"
+ date="2014-10-14T22:09:37Z"
+ content="""
+If I use git-annex to keep my home and work computer in sync, the conflict would only happen if I somehow manage to make a change in the git repo on one machine that's not propagated to the other straight away (eg, network down), and I then modify the git repo of the other as well?
+
+Could git-annex have a different merge strategy in this case, not try to merge anything under .git and give a warning?
+
+"""]]
diff --git a/doc/forum/multiple_repositories_single_backup.mdwn b/doc/forum/multiple_repositories_single_backup.mdwn
new file mode 100644
index 0000000000..10f8c0e435
--- /dev/null
+++ b/doc/forum/multiple_repositories_single_backup.mdwn
@@ -0,0 +1,6 @@
+hi
+
+is it possible to have multiple repositories that share single backup directory ?
+
+for example.
+i have mp3, docs on my laptop as separate repositories. i would like to use single backup directory that is on my usb drive.
diff --git a/doc/forum/multiple_repositories_single_backup/comment_1_4a479fa78e0b366fcff1a27bc37081de._comment b/doc/forum/multiple_repositories_single_backup/comment_1_4a479fa78e0b366fcff1a27bc37081de._comment
new file mode 100644
index 0000000000..52da6a2724
--- /dev/null
+++ b/doc/forum/multiple_repositories_single_backup/comment_1_4a479fa78e0b366fcff1a27bc37081de._comment
@@ -0,0 +1,12 @@
+[[!comment format=mdwn
+ username="http://joeyh.name/"
+ ip="209.250.56.227"
+ subject="comment 1"
+ date="2013-12-29T20:29:26Z"
+ content="""
+If you set up an identically configured special remote in the same location for each of your separate mp3 and docs repositories, then yes, you can store files for both of them in it. In the unlikely case where you have the same file in both mp3 and docs, only one copy will be stored in the remote. (Unless it's using encryption, which will tend to defeat that.)
+
+It should work ok, but unless you're trying to get deduplication like that, I don't see any reason to do it rather than having a separate special remote for each of the separate git repositories.
+
+(Also, I'm talking about special remotes here, not git remotes. If you add the same git remote to 2 separate repositories, then `git annex sync` will connect them all together.)
+"""]]
diff --git a/doc/forum/multiple_repositories_single_backup/comment_2_bbe19eec0969385a0d4682bf9e9de21a._comment b/doc/forum/multiple_repositories_single_backup/comment_2_bbe19eec0969385a0d4682bf9e9de21a._comment
new file mode 100644
index 0000000000..bac577ce99
--- /dev/null
+++ b/doc/forum/multiple_repositories_single_backup/comment_2_bbe19eec0969385a0d4682bf9e9de21a._comment
@@ -0,0 +1,11 @@
+[[!comment format=mdwn
+ username="https://www.google.com/accounts/o8/id?id=AItOawkNE-H4vEcbcGndxq5daT8qUb7yIf7r1OE"
+ nickname="Łukasz"
+ subject="comment 2"
+ date="2013-12-29T21:34:19Z"
+ content="""
+thx for reply. i think i will stick with separate directories for every repositiry
+
+mp3 on laptop -> mp3-backup directory on usb drive
+docs on laptop -> docs-backup on usb drive
+"""]]
diff --git a/doc/forum/multiple_repositories_single_backup/comment_3_1b18ea058e3eb34852055cffe51de176._comment b/doc/forum/multiple_repositories_single_backup/comment_3_1b18ea058e3eb34852055cffe51de176._comment
new file mode 100644
index 0000000000..4accf7f07b
--- /dev/null
+++ b/doc/forum/multiple_repositories_single_backup/comment_3_1b18ea058e3eb34852055cffe51de176._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="http://cstork.org/"
+ nickname="Chris Stork"
+ subject="Using different git branches"
+ date="2014-01-14T09:12:19Z"
+ content="""
+This might not be practical (especially for a non-technical user) but wouldn't it be possible to share a (bare) git-annex repo for backup if the other non-bare repos use different branches of the same git repo? (These non-bare repos could also share their .git directory.) Hmm, I don't see this discussed much...
+"""]]
diff --git a/doc/forum/multiple_routes_to_same_repository.mdwn b/doc/forum/multiple_routes_to_same_repository.mdwn
new file mode 100644
index 0000000000..a94acac2f8
--- /dev/null
+++ b/doc/forum/multiple_routes_to_same_repository.mdwn
@@ -0,0 +1,2 @@
+I am trying to configure the local repo to be connected with remote computer. It can either find this computer on local network using local host name, or go through public IP host where I have ssh tunnel. First confusing thing is, that when setting up remote server, it successfuly establishes connection, but then asks me if the folder should be git or rsync - I believe this step should be skipped if the git repo is already there. Also choosing repository group should be detected from the repo itself.
+But the real problem is that I cannot add the public IP repo at all. Is this a known limitation?
diff --git a/doc/forum/multiple_routes_to_same_repository/comment_1_26c1734d41d5374f18fc688d862d6b8e._comment b/doc/forum/multiple_routes_to_same_repository/comment_1_26c1734d41d5374f18fc688d862d6b8e._comment
new file mode 100644
index 0000000000..94eadcd557
--- /dev/null
+++ b/doc/forum/multiple_routes_to_same_repository/comment_1_26c1734d41d5374f18fc688d862d6b8e._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="https://www.google.com/accounts/o8/id?id=AItOawmZ9EwisYI1DDev8HXY6zFjPrLKt60-3QI"
+ nickname="Yo"
+ subject="comment 1"
+ date="2013-06-24T12:23:15Z"
+ content="""
+I believe I have a similar set up as you're trying to achieve. What I did was just add a repository using the public domain name only (DDNS). My home router seems to be smart enough to realize the IP the domain name resolves to is its own, so I get gigabit speeds when at home, and I can still access it when I'm not at home.
+"""]]
diff --git a/doc/forum/multiple_routes_to_same_repository/comment_2_d119ab485fb2d5512c15372efdb2327d._comment b/doc/forum/multiple_routes_to_same_repository/comment_2_d119ab485fb2d5512c15372efdb2327d._comment
new file mode 100644
index 0000000000..48cae0211b
--- /dev/null
+++ b/doc/forum/multiple_routes_to_same_repository/comment_2_d119ab485fb2d5512c15372efdb2327d._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="http://joeyh.name/"
+ ip="4.152.108.193"
+ subject="comment 2"
+ date="2013-06-25T20:05:46Z"
+ content="""
+I don't see anything that prevents you from adding two repositories that use different names for the same server.
+"""]]
diff --git a/doc/forum/multiple_sym_links___40__for_tagging_photos__41____63__.mdwn b/doc/forum/multiple_sym_links___40__for_tagging_photos__41____63__.mdwn
new file mode 100644
index 0000000000..c5c6f3a94b
--- /dev/null
+++ b/doc/forum/multiple_sym_links___40__for_tagging_photos__41____63__.mdwn
@@ -0,0 +1,11 @@
+I saw a cool page talking about "tagging" photos by using symlinks: http://www.trueelena.org/computers/articles/photo_management_with_git-annex_and_bash.html
+
+so say I have a photo in git annex, called DSC_3285.JPG, which of course is really a symlink to ../.git/annex/objects/Zk/kj/WORM-s5296770-m1338516288--DSC_3285.JPG/WORM-s5296770-m1338516288--DSC_3285.JPG.
+
+I want to make an additional link to that photo in a directory called tags/.
+
+should I link to the symlink (DSC_3285.JPG), or to the annexed file? (../.git/annex/objects/Zk/kj/WORM-s5296770-m1338516288--DSC_3285.JPG/WORM-s5296770-m1338516288--DSC_3285.JPG)
+
+I might occasionally rename DSC_3285.JPG or edit the photo itself. will the git annex commit hooks update both links, or should I prepare a script to update links in tags/ after I change DSC_3285 or the annexed data? 
+
+thank you.
diff --git a/doc/forum/multiple_sym_links___40__for_tagging_photos__41____63__/comment_1_96beb9ea895c80285748adb940b4f57d._comment b/doc/forum/multiple_sym_links___40__for_tagging_photos__41____63__/comment_1_96beb9ea895c80285748adb940b4f57d._comment
new file mode 100644
index 0000000000..b4d2433aa3
--- /dev/null
+++ b/doc/forum/multiple_sym_links___40__for_tagging_photos__41____63__/comment_1_96beb9ea895c80285748adb940b4f57d._comment
@@ -0,0 +1,9 @@
+[[!comment format=mdwn
+ username="http://joeyh.name/"
+ ip="4.152.108.236"
+ subject="comment 1"
+ date="2012-09-05T19:56:06Z"
+ content="""
+I'd recommend linking to the annexed file. Which is what `cp -a` does BTW. Then it'll just be a peice of content that is used by more than one filename, which git-annex handles well. However, if you then edited
+one file, git-annex would not update the other link to point to the new content.
+"""]]
diff --git a/doc/forum/multiple_sym_links___40__for_tagging_photos__41____63__/comment_2_985065c1feed9300631dac7a2701f669._comment b/doc/forum/multiple_sym_links___40__for_tagging_photos__41____63__/comment_2_985065c1feed9300631dac7a2701f669._comment
new file mode 100644
index 0000000000..a53fe06b5e
--- /dev/null
+++ b/doc/forum/multiple_sym_links___40__for_tagging_photos__41____63__/comment_2_985065c1feed9300631dac7a2701f669._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="https://www.google.com/accounts/o8/id?id=AItOawnh0pv5PyOJH8dHB1Ly2rHJM4nvVd17CZ0"
+ nickname="Kevin"
+ subject="comment 2"
+ date="2012-09-05T19:57:36Z"
+ content="""
+Thank you very much! (for your answer, and for git annex)
+"""]]
diff --git a/doc/forum/multiple_urls_for_the_same_UUID.mdwn b/doc/forum/multiple_urls_for_the_same_UUID.mdwn
new file mode 100644
index 0000000000..2f8106b597
--- /dev/null
+++ b/doc/forum/multiple_urls_for_the_same_UUID.mdwn
@@ -0,0 +1,26 @@
+I've been doing a sort of experiment but I'm not sure if it's working or, really, how to even tell.
+
+I have two macbooks that are both configured as clients as well as a USB HDD, an rsync endpoint on a home NAS, and a glacier endpoint.
+
+For the purposes of this example, lets call the macbooks "chrissy" and "brodie". Chrissy's was initially configured with a remote for brodie with the url as
+
+    ssh://Brodie.88195848.members.btmm.icloud.com./Users/akraut/Desktop/annex
+
+This allows me to leverage the "Back To My Mac" free IPv6 roaming I get from Apple.  Now, occasionally, that dns resolution fails. Since I'm frequently on the same network, I can also use the mDNS address of brodie.local. which is much more reliable.
+
+So my brilliant/terrible idea was to put this in my git config:
+
+    [remote "brodie"]
+    	url = ssh://Brodie.88195848.members.btmm.icloud.com./Users/akraut/Desktop/annex
+    	fetch = +refs/heads/*:refs/remotes/brodie/*
+    	annex-uuid = BF4BCA6D-9252-4B5B-BE12-36DD755FAF4B
+    	annex-cost-command = /Users/akraut/Desktop/annex/tools/annex-cost6.sh Brodie.88195848.members.btmm.icloud.com.
+    [remote "brodie-local"]
+    	url = ssh://brodie.local./Users/akraut/Desktop/annex
+    	fetch = +refs/heads/*:refs/remotes/brodie/*
+    	annex-uuid = BF4BCA6D-9252-4B5B-BE12-36DD755FAF4B
+    	annex-cost-command = /Users/akraut/Desktop/annex/tools/annex-cost.sh brodie.local.
+
+Is there any reason why I shouldn't do this? Is annex smart enough to know that it can reach the same remote through both urls? Will the cost calculations be considered and the "local" url chosen if it's cost is less than the other?
+
+(I posted the annex-cost.sh stuff at [[forum/Calculating Annex Cost by Ping Times]].)
diff --git a/doc/forum/multiple_urls_for_the_same_UUID/comment_1_de7410d8824a864c4d106c9f1afaec3f._comment b/doc/forum/multiple_urls_for_the_same_UUID/comment_1_de7410d8824a864c4d106c9f1afaec3f._comment
new file mode 100644
index 0000000000..c06cf01c35
--- /dev/null
+++ b/doc/forum/multiple_urls_for_the_same_UUID/comment_1_de7410d8824a864c4d106c9f1afaec3f._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="http://joeyh.name/"
+ ip="4.152.246.6"
+ subject="comment 1"
+ date="2012-12-30T04:19:48Z"
+ content="""
+Yes, this is absolutely supported. If it's trying to `get` a file and it fails to access the first remote, it'll even automatically fall back to using the second. I do similar things with my own remote, for example I have a foo that uses ssh with a foo-local that uses NFS.
+"""]]
diff --git a/doc/forum/multiple_urls_for_the_same_UUID/comment_2_309a86cf7e08448be64357a30d8b56ae._comment b/doc/forum/multiple_urls_for_the_same_UUID/comment_2_309a86cf7e08448be64357a30d8b56ae._comment
new file mode 100644
index 0000000000..9d0dc9f66b
--- /dev/null
+++ b/doc/forum/multiple_urls_for_the_same_UUID/comment_2_309a86cf7e08448be64357a30d8b56ae._comment
@@ -0,0 +1,13 @@
+[[!comment format=mdwn
+ username="https://www.google.com/accounts/o8/id?id=AItOawln3ckqKx0x_xDZMYwa9Q1bn4I06oWjkog"
+ nickname="Michael"
+ subject="multiple ways to access a special remote?"
+ date="2013-07-10T05:52:28Z"
+ content="""
+The use case is like this: an external USB drive could be accessed both locally (as a special directory remote) or it could be mounted remotely (then it would need to be an rsync special remote).
+Is there a way to handle this?
+
+Failing that, can one switch the same remote from being a directory to an rsync without having to --move all of the content?
+
+Finally, what's the best way to check directory remote's directory= parameter?
+"""]]
diff --git a/doc/forum/multiple_urls_for_the_same_UUID/comment_3_fa97a45fc1392935fd5e0714db999bc2._comment b/doc/forum/multiple_urls_for_the_same_UUID/comment_3_fa97a45fc1392935fd5e0714db999bc2._comment
new file mode 100644
index 0000000000..c6f5dfde91
--- /dev/null
+++ b/doc/forum/multiple_urls_for_the_same_UUID/comment_3_fa97a45fc1392935fd5e0714db999bc2._comment
@@ -0,0 +1,14 @@
+[[!comment format=mdwn
+ username="http://joeyh.name/"
+ ip="4.154.7.235"
+ subject="comment 3"
+ date="2013-07-10T16:38:04Z"
+ content="""
+@Michael, like I said above, this use case is completely supported. That's why git-annex uses UUIDs to uniquely identify repositories, no matter where or how many urls are used for them. 
+
+Just set up the remotes you need, and if they end up pointing to the same repository by different routes, git-annex will automatically notice.
+
+---
+
+The directory= parameter used when initializing a directory remote is only used to set up the remote in the .git/config file. It is not stored anywhere else, since the directory could be mounted at different locations on different computers, eg when a drive is moved between computers.
+"""]]
diff --git a/doc/forum/multiple_urls_for_the_same_UUID/comment_4_139178b1ba45b62eec0c89a660c0c81e._comment b/doc/forum/multiple_urls_for_the_same_UUID/comment_4_139178b1ba45b62eec0c89a660c0c81e._comment
new file mode 100644
index 0000000000..945aa3b6bf
--- /dev/null
+++ b/doc/forum/multiple_urls_for_the_same_UUID/comment_4_139178b1ba45b62eec0c89a660c0c81e._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="https://www.google.com/accounts/o8/id?id=AItOawln3ckqKx0x_xDZMYwa9Q1bn4I06oWjkog"
+ nickname="Michael"
+ subject="comment 4"
+ date="2013-07-10T21:26:06Z"
+ content="""
+Very cool, thanks Joey.
+"""]]
diff --git a/doc/forum/multiple_urls_for_the_same_UUID/comment_5_7237986a34228282c6b764309afc1d57._comment b/doc/forum/multiple_urls_for_the_same_UUID/comment_5_7237986a34228282c6b764309afc1d57._comment
new file mode 100644
index 0000000000..3014e6c042
--- /dev/null
+++ b/doc/forum/multiple_urls_for_the_same_UUID/comment_5_7237986a34228282c6b764309afc1d57._comment
@@ -0,0 +1,33 @@
+[[!comment format=mdwn
+ username="https://www.google.com/accounts/o8/id?id=AItOawkMTPqZZWoz396ABpx6nh3osxKQCFaSW6M"
+ nickname="Mark"
+ subject="annex sync when inside an itinerant repository"
+ date="2015-02-12T12:54:21Z"
+ content="""
+I'm having trouble with this where the different remotes have the same URL but different UUIDs. My situation is a repository on a USB drive that can be plugged into one of two machines and used to transport large files between them. On each machine there is a local repository in a consistent location, so I can rely on paths to things in the repos being consistent across machines. Each repo obviously has a different UUID. The USB repo has remotes for local filesystem access and remotes for over-the-network access as a convenience - something like this:
+
+    [remote \"host1\"]
+        url = /m/stuff
+        fetch = +refs/heads/*:refs/remotes/host1/*
+        annex-uuid = ce6175ba-4a0d-49e6-88b1-615dac7a37c1
+    [remote \"host1-net\"]
+        url = ssh://host1.network/m/stuff
+        fetch = +refs/heads/*:refs/remotes/host1/*
+        annex-uuid = ce6175ba-4a0d-49e6-88b1-615dac7a37c1
+    [remote \"host2\"]
+        url = /m/stuff
+        fetch = +refs/heads/*:refs/remotes/host2/*
+        annex-uuid = f7e3fbe8-f7f5-4231-a885-a72a46680d0b
+    [remote \"host2-net\"]
+        url = ssh://host2.network/m/stuff
+        fetch = +refs/heads/*:refs/remotes/host2/*
+        annex-uuid = f7e3fbe8-f7f5-4231-a885-a72a46680d0b
+
+The over-the-network path is useful for keeping everything in sync, but it doesn't have enough bandwidth to sensibly sync the content as well.
+
+If I run 'git annex sync' in this repository while it's attached to host1 I'd hope it would sync with host1 and host2-net, as those are the URLs through which the two repositories can be reached. What actually happens is that it syncs with all of the repositories and updates the annex-uuid of remote 'host2' to be the UUID of the host1 repository. It also obviously gets a bit confused because it updates the remote branches for host2 from host1.
+
+Is there some way to configure it so that sync works with all repositories based on unique uuid values, rather than all remotes?
+
+
+"""]]
diff --git a/doc/forum/multiple_urls_for_the_same_UUID/comment_6_392819ba657569a1b997b58aa921a0ad._comment b/doc/forum/multiple_urls_for_the_same_UUID/comment_6_392819ba657569a1b997b58aa921a0ad._comment
new file mode 100644
index 0000000000..783b9a41a7
--- /dev/null
+++ b/doc/forum/multiple_urls_for_the_same_UUID/comment_6_392819ba657569a1b997b58aa921a0ad._comment
@@ -0,0 +1,25 @@
+[[!comment format=mdwn
+ username="https://www.google.com/accounts/o8/id?id=AItOawnm19dBCRphmtjXfopm_NpvnRwz-qIJ2Tw"
+ nickname="Remi"
+ subject="comment 6"
+ date="2015-02-12T14:47:38Z"
+ content="""
+I had the same problem, and I solved it using a host specific directory with symlink:
+
+On host \"host1\", I've a directory named \"/home/me/host1/\" that contain a symlink \"mygitrepos\" to \"/home/me/mygitrepos/\".
+On host \"host2\", I've a directory named \"/home/me/host2/\" that contain a symlink \"mygitrepos\" to \"/home/me/mygitrepos/\".
+
+On the usb drive, the remote are set as:
+
+    [remote \\"host1\\"]
+        url = /home/me/host1/mygitrepos
+        fetch = +refs/heads/*:refs/remotes/host1/*
+        annex-uuid = ce6175ba-4a0d-49e6-88b1-615dac7a37c1
+    [remote \\"host2\\"]
+        url = /home/me/host2/mygitrepos
+        fetch = +refs/heads/*:refs/remotes/host2/*
+        annex-uuid = f7e3fbe8-f7f5-4231-a885-a72a46680d0b
+
+(I didn't set the net remote, but it should work). With this added indirection I protect myself against git (and git-annex) confusion.
+
+"""]]
diff --git a/doc/forum/multiple_urls_for_the_same_UUID/comment_7_276cb5d94cbd10e9fc7d1cf4ac607273._comment b/doc/forum/multiple_urls_for_the_same_UUID/comment_7_276cb5d94cbd10e9fc7d1cf4ac607273._comment
new file mode 100644
index 0000000000..a80bffe10c
--- /dev/null
+++ b/doc/forum/multiple_urls_for_the_same_UUID/comment_7_276cb5d94cbd10e9fc7d1cf4ac607273._comment
@@ -0,0 +1,27 @@
+[[!comment format=mdwn
+ username="joey"
+ subject="""comment 7"""
+ date="2015-02-12T20:25:12Z"
+ content="""
+You have two configured git remotes, host1 and host2, with the same
+"url = /m/stuff". Only one of these remotes can be accessed at a time,
+depending on where the drive is docked.
+
+So, why not just combine those two remote configs
+into a single remote. Call it "host".
+
+git-annex will automatically notice when the uuid of the repository
+pointed to by "host" changes, and it will update the .git/config
+appropriately.
+
+That BTW, happens to be just how I use git-annex with my own USB drive,
+and it works great.
+
+---
+
+git annex sync will still try to sync with "host1-net" and "host2-net", as well
+as which ever one of the two "host" points to. There's a small redundancy
+there, but since it will sync with "host" first, as it knows local file
+access is less expensive, the redundant sync will not involve much work.
+
+"""]]
diff --git a/doc/forum/multiple_urls_for_the_same_UUID/comment_8_c44144c677d54aaea6e900d0d7e000a3._comment b/doc/forum/multiple_urls_for_the_same_UUID/comment_8_c44144c677d54aaea6e900d0d7e000a3._comment
new file mode 100644
index 0000000000..f333c2bc0e
--- /dev/null
+++ b/doc/forum/multiple_urls_for_the_same_UUID/comment_8_c44144c677d54aaea6e900d0d7e000a3._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="https://www.google.com/accounts/o8/id?id=AItOawkMTPqZZWoz396ABpx6nh3osxKQCFaSW6M"
+ nickname="Mark"
+ subject="comment 8"
+ date="2015-02-13T13:55:21Z"
+ content="""
+Ah, ok. So it doesn't cause any issues that the host/* remote branches will also keep getting swapped from one repository to another? The operation of `git annex sync` is sufficiently (and happily) opaque to me, so I was concerned that this might break some of its basic assumptions.
+"""]]
diff --git a/doc/forum/multiple_urls_for_the_same_UUID/comment_9_38f3007635b0a7b7d30bad0af8a2d0a9._comment b/doc/forum/multiple_urls_for_the_same_UUID/comment_9_38f3007635b0a7b7d30bad0af8a2d0a9._comment
new file mode 100644
index 0000000000..5a89f6e16b
--- /dev/null
+++ b/doc/forum/multiple_urls_for_the_same_UUID/comment_9_38f3007635b0a7b7d30bad0af8a2d0a9._comment
@@ -0,0 +1,13 @@
+[[!comment format=mdwn
+ username="joey"
+ subject="""comment 9"""
+ date="2015-02-17T21:43:16Z"
+ content="""
+It's entirely expected and normal for git-annex to update the UUID
+of a remote with `url = somepath` when it notices that the repo at 
+`somepath` has changed.
+
+This is what you want to happen. If git-annex didn't notice and react to
+the UUID change, its location tracking information (for UUID A) would be
+inconsistent with the actual status of the repo (using UUID B).
+"""]]
diff --git a/doc/forum/n00b_question._Can_I_use_git-annex_to_manage_a_tree_that_contains_git_repos__63__.mdwn b/doc/forum/n00b_question._Can_I_use_git-annex_to_manage_a_tree_that_contains_git_repos__63__.mdwn
new file mode 100644
index 0000000000..46257f6461
--- /dev/null
+++ b/doc/forum/n00b_question._Can_I_use_git-annex_to_manage_a_tree_that_contains_git_repos__63__.mdwn
@@ -0,0 +1,7 @@
+Just started looking at git-annex and it's very interesting to me as I manage a large tree of different types of projects, spread across two computers and some hard-drives (some backups, some offline resources and archives)
+
+But one question that's bothering me. A lot of my projects are already git repos. Is it possible to use git-annex with this tree or will the gits fight unless I explicitly make them submodules? 
+
+cheers
+
+Phil
diff --git a/doc/forum/n00b_question._Can_I_use_git-annex_to_manage_a_tree_that_contains_git_repos__63__/comment_1_ac45ad341f026289ce56183b9ff463af._comment b/doc/forum/n00b_question._Can_I_use_git-annex_to_manage_a_tree_that_contains_git_repos__63__/comment_1_ac45ad341f026289ce56183b9ff463af._comment
new file mode 100644
index 0000000000..8e409bb5e0
--- /dev/null
+++ b/doc/forum/n00b_question._Can_I_use_git-annex_to_manage_a_tree_that_contains_git_repos__63__/comment_1_ac45ad341f026289ce56183b9ff463af._comment
@@ -0,0 +1,10 @@
+[[!comment format=mdwn
+ username="http://id.clacke.se/"
+ nickname="clacke"
+ subject="git-annex is git"
+ date="2016-04-22T07:44:06Z"
+ content="""
+git-annex is first git, then some additional very neat functionality. Being git, it refuses to handle git repositories as that would create all kinds of potential confusion and inefficiency.
+
+Submodules would be a way to go, if you like how they work. Or you could just ignore the git repos, as you probably have them replicated somewhere else.
+"""]]
diff --git a/doc/forum/n00b_question._Can_I_use_git-annex_to_manage_a_tree_that_contains_git_repos__63__/comment_2_53e0ffe901a1a1c22361bfd7cf71affd._comment b/doc/forum/n00b_question._Can_I_use_git-annex_to_manage_a_tree_that_contains_git_repos__63__/comment_2_53e0ffe901a1a1c22361bfd7cf71affd._comment
new file mode 100644
index 0000000000..73a2e8ce06
--- /dev/null
+++ b/doc/forum/n00b_question._Can_I_use_git-annex_to_manage_a_tree_that_contains_git_repos__63__/comment_2_53e0ffe901a1a1c22361bfd7cf71affd._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="interstar@551341a1dbca18391e90a3f719eae492868ceb7a"
+ nickname="interstar"
+ subject="comment 2"
+ date="2016-04-22T12:38:16Z"
+ content="""
+OK. Thanks.
+"""]]
diff --git a/doc/forum/n00b_question._Can_I_use_git-annex_to_manage_a_tree_that_contains_git_repos__63__/comment_2_f7416c26daca543ad08c11e187e1589e._comment b/doc/forum/n00b_question._Can_I_use_git-annex_to_manage_a_tree_that_contains_git_repos__63__/comment_2_f7416c26daca543ad08c11e187e1589e._comment
new file mode 100644
index 0000000000..9ad9c73417
--- /dev/null
+++ b/doc/forum/n00b_question._Can_I_use_git-annex_to_manage_a_tree_that_contains_git_repos__63__/comment_2_f7416c26daca543ad08c11e187e1589e._comment
@@ -0,0 +1,37 @@
+[[!comment format=mdwn
+ username="joey"
+ subject="""furthermore.."""
+ date="2016-04-22T18:52:09Z"
+ content="""
+You might also use  as a somewhat
+more flexible alternative to submodules.
+
+It's worth thinking about what would happen if you were able to check a git
+repository into a git (annex) repository. A git repository contains files
+like `.git/index` that are git internals, and binary files. Now what
+happens if you have two checkouts of that nested git-in-git repository, and
+git writes two different versions of the `.git/index` file? You'd get a
+merge conflict that you have no way of resolving, involving two versions of
+an internal use binary file. This is a lot worse than a merge conflict involving
+some regular binary file like a jpeg, because at least with jpegs you can
+look at the two versions of the file and pick the better one.
+
+While git prevents checking in `.git` directories, you could technically work
+around it, if you really wanted to, by eg using `GIT_DIR` to rename
+the `.git` directory to something else. But it's just setting yourself up for
+unresolvable merge conflicts and pain.
+
+It's likewise not good to check in other version control system
+directories, like `.svn`, `.bzr`, or `.hg` into git repositories or
+vice-versa.
+
+Sometimes people complain that the git-annex assistant should support
+syncing nested git repositories, because after all other directory syncing
+tools like dropbox support that. But, a little known fact about dropbox is
+that it too can end up with a conflicted merge type situation, and when
+this happens it will do *something* to auto-resolve it. That something
+almost certianly does not include leaving the git repository what was
+stored in dropbox in an ideal state. So, while people put git repos into
+dropbox and get away with it, they are just being lucky to not run into the
+edge cases where doing that blows up.
+"""]]
diff --git a/doc/forum/n00b_question._Can_I_use_git-annex_to_manage_a_tree_that_contains_git_repos__63__/comment_4_9c6386710387f7334ea44c1de7cc8729._comment b/doc/forum/n00b_question._Can_I_use_git-annex_to_manage_a_tree_that_contains_git_repos__63__/comment_4_9c6386710387f7334ea44c1de7cc8729._comment
new file mode 100644
index 0000000000..a411d8de88
--- /dev/null
+++ b/doc/forum/n00b_question._Can_I_use_git-annex_to_manage_a_tree_that_contains_git_repos__63__/comment_4_9c6386710387f7334ea44c1de7cc8729._comment
@@ -0,0 +1,13 @@
+[[!comment format=mdwn
+ username="xloem"
+ subject="did this today"
+ date="2016-04-25T20:40:02Z"
+ content="""
+it's interesting to find this discussion, I just finished implementing this on my system.
+
+I've been storing bare repos in annex in order to back them up with the same system.  I have really huge files in some of my git repos and I wanted to get those files into the annex system but still keep a record of their changes (the git history).
+
+Today I removed the core.bare = true setting on the repos and instead set core.worktree = projectdir, and ran git checkout in projectdir.  I have the index file in .gitignore, so there won't be that weird unresolvable conflict.  Now all my bloated git history is stored in the annex, and I can still work with it in the annexed checkout.
+
+I was thinking I'd do this again with the root of the repository when the annex grows too large, to back up the old history in a connected way.
+"""]]
diff --git a/doc/forum/new_linux_arm_tarball_build.mdwn b/doc/forum/new_linux_arm_tarball_build.mdwn
new file mode 100644
index 0000000000..e3b9c3de5c
--- /dev/null
+++ b/doc/forum/new_linux_arm_tarball_build.mdwn
@@ -0,0 +1,12 @@
+I've added an arm build to the autobuilds in [[install/Linux_standalone]].
+
+I'm curious to see how this works out. I tried to make it as self-contained
+as possible. It should work even on systems that do not use glibc, as long
+as the kernel is new enough for the glibc included in it, and supports the
+arm EABI.
+
+If it seems sufficiently useful, I might try to add the webapp to the
+build, which would be somewhat complicated, but doable (since I'm building
+using qemu, it can run a build on amd64 first to get the TH splices).
+
+--[[Joey]]
diff --git a/doc/forum/new_linux_arm_tarball_build/comment_10_5f9735ec62478c99b8c814055206cff0._comment b/doc/forum/new_linux_arm_tarball_build/comment_10_5f9735ec62478c99b8c814055206cff0._comment
new file mode 100644
index 0000000000..eb3995eb15
--- /dev/null
+++ b/doc/forum/new_linux_arm_tarball_build/comment_10_5f9735ec62478c99b8c814055206cff0._comment
@@ -0,0 +1,12 @@
+[[!comment format=mdwn
+ username="https://www.google.com/accounts/o8/id?id=AItOawmH9ARM62C6zcEpzh2muCs4wq-GkLRntgQ"
+ nickname="Randy"
+ subject="raspbian"
+ date="2014-04-16T10:59:10Z"
+ content="""
+This works fairly well for me on Raspian. However I am getting the same error as Justin. 
+
+    ERROR: ld.so: object '/usr/lib/arm-linux-gnueabihf/libcofi_rpi.so' from /etc/ld.so.preload cannot be preloaded: ignored.
+
+I'm ignoring the errors for now, but it's a lot of noise that actually makes it slightly difficult to see the important output.
+"""]]
diff --git a/doc/forum/new_linux_arm_tarball_build/comment_11_859c44046b00fe885f6878cfe0e46360._comment b/doc/forum/new_linux_arm_tarball_build/comment_11_859c44046b00fe885f6878cfe0e46360._comment
new file mode 100644
index 0000000000..7be8a67316
--- /dev/null
+++ b/doc/forum/new_linux_arm_tarball_build/comment_11_859c44046b00fe885f6878cfe0e46360._comment
@@ -0,0 +1,10 @@
+[[!comment format=mdwn
+ username="http://joeyh.name/"
+ ip="209.250.56.191"
+ subject="comment 11"
+ date="2014-04-17T19:39:41Z"
+ content="""
+I can't see any good way to prevent ld-linux.so from preloading things listed in `/etc/ld.so.preload`. I don't know why raspbian wants to preload that -- probably for optimisation purposes?
+
+I could modify the ld-linux.so shipped in the git-annex tarball, but that way lies pointless complication..
+"""]]
diff --git a/doc/forum/new_linux_arm_tarball_build/comment_12_35ade68d62e95036344ad33db3279c21._comment b/doc/forum/new_linux_arm_tarball_build/comment_12_35ade68d62e95036344ad33db3279c21._comment
new file mode 100644
index 0000000000..ba6468a3f6
--- /dev/null
+++ b/doc/forum/new_linux_arm_tarball_build/comment_12_35ade68d62e95036344ad33db3279c21._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="https://www.google.com/accounts/o8/id?id=AItOawmH9ARM62C6zcEpzh2muCs4wq-GkLRntgQ"
+ nickname="Randy"
+ subject="comment 12"
+ date="2014-04-18T13:38:13Z"
+ content="""
+There is no need to modify anything here. The errors were mostly just confusing at first and made me wonder if it was working properly. It is indeed working perfectly! Thanks!!!
+"""]]
diff --git a/doc/forum/new_linux_arm_tarball_build/comment_13_36f48c30894b9b225b812ba5e5b2f504._comment b/doc/forum/new_linux_arm_tarball_build/comment_13_36f48c30894b9b225b812ba5e5b2f504._comment
new file mode 100644
index 0000000000..312082ae1c
--- /dev/null
+++ b/doc/forum/new_linux_arm_tarball_build/comment_13_36f48c30894b9b225b812ba5e5b2f504._comment
@@ -0,0 +1,17 @@
+[[!comment format=mdwn
+ username="https://www.google.com/accounts/o8/id?id=AItOawnClfG_kAo0drU5dVZiTRXo9WnqjW4I5dA"
+ nickname="Julian"
+ subject="comment 13"
+ date="2014-12-06T17:04:04Z"
+ content="""
+Unfortunately, it isn't working for me on DS214 (MARVELL Armada XP MV78230) and DSM 5.1-5004 Update 2.
+git is installed, git-annex tar extracted and runshell worked. Within this shell git-annex seems to work (e.g. git-annex init works)
+
+However, git-annex is complaining that git-annex is not installed on remote when I try to add the NAS as a remote.
+
+I guess it is related to the fact that git-annex-shell is not working on the NAS. When I execute git-annex-shell it gives:
+/volume1/homes/julian/bin/git-annex.linux/shimmed/git-annex-shell/git-annex-shell: error while loading shared libraries: /volume1/homes/julian/bin/git-annex.linux/shimmed/git-annex-shell/git-annex-shell: file too short
+Actually /volume1/homes/julian/bin/git-annex.linux/shimmed/git-annex-shell/git-annex-shell has 0 Bytes.
+
+I can't execute git-annex-shell inside the bin folder due to permission issues. I tried to change the file permissions but it didn't help.
+"""]]
diff --git a/doc/forum/new_linux_arm_tarball_build/comment_14_67021b6d239690c9d18e8630aa2254ff._comment b/doc/forum/new_linux_arm_tarball_build/comment_14_67021b6d239690c9d18e8630aa2254ff._comment
new file mode 100644
index 0000000000..fd9591d9d2
--- /dev/null
+++ b/doc/forum/new_linux_arm_tarball_build/comment_14_67021b6d239690c9d18e8630aa2254ff._comment
@@ -0,0 +1,9 @@
+[[!comment format=mdwn
+ username="joey"
+ subject="""comment 14"""
+ date="2014-12-06T20:25:06Z"
+ content="""
+@Julian, shimmed/git-annex-shell/git-annex-shell is supposed to be a hard
+link to shimmed/git-annex/git-annex. Maybe there's a problem with hard
+links on your NAS? A symlink would also work I think.
+"""]]
diff --git a/doc/forum/new_linux_arm_tarball_build/comment_15_646a19555f982fb7ab302289400adc3d._comment b/doc/forum/new_linux_arm_tarball_build/comment_15_646a19555f982fb7ab302289400adc3d._comment
new file mode 100644
index 0000000000..fee8cc91d0
--- /dev/null
+++ b/doc/forum/new_linux_arm_tarball_build/comment_15_646a19555f982fb7ab302289400adc3d._comment
@@ -0,0 +1,16 @@
+[[!comment format=mdwn
+ username="db"
+ subject="How do I start it?"
+ date="2016-01-23T22:42:52Z"
+ content="""
+This happens when I launch git-annex-webapp
+
+	[\u@\h \W]$ ./git-annex-webapp
+	Launching web browser on file:///tmp/webapp1804289383846930886.html
+	git-annex: xdg-open: createProcess: runInteractiveProcess: exec: does not exist (No such file or directory)
+
+
+And the webapp1804289383846930886.html file seem to be deleted immediately.
+
+This is on a headless NAS and obviously I don't have a browser or anything on it. I just want to run Annex as a server/service and admin it from a browser on another machine on my LAN.
+"""]]
diff --git a/doc/forum/new_linux_arm_tarball_build/comment_16_442015225a14f60064aa3ba3cfbdf2ae._comment b/doc/forum/new_linux_arm_tarball_build/comment_16_442015225a14f60064aa3ba3cfbdf2ae._comment
new file mode 100644
index 0000000000..9e592df13f
--- /dev/null
+++ b/doc/forum/new_linux_arm_tarball_build/comment_16_442015225a14f60064aa3ba3cfbdf2ae._comment
@@ -0,0 +1,9 @@
+[[!comment format=mdwn
+ username="joey"
+ subject="""comment 16"""
+ date="2016-01-24T18:16:24Z"
+ content="""
+You can use the --listen parameter to make the webapp listen on an address
+and then connect to it from a web browser on your LAN. See
+
+"""]]
diff --git a/doc/forum/new_linux_arm_tarball_build/comment_17_653448e38c3bb71a97023cb5d6b21af6._comment b/doc/forum/new_linux_arm_tarball_build/comment_17_653448e38c3bb71a97023cb5d6b21af6._comment
new file mode 100644
index 0000000000..9915cb9c8f
--- /dev/null
+++ b/doc/forum/new_linux_arm_tarball_build/comment_17_653448e38c3bb71a97023cb5d6b21af6._comment
@@ -0,0 +1,16 @@
+[[!comment format=mdwn
+ username="db"
+ subject="comment 17"
+ date="2016-01-24T19:30:41Z"
+ content="""
+Doesn't seem to work:
+
+	$ sudo ./git-annex-webapp --listen=192.168.1.3                                                                                           
+	git-annex: /tmp/webapp1804289383846930886.html16816927771714636915.tmp: hClose: resource exhausted (No space left on device)
+	WebAp$ 
+	$ touch /tmp/test
+	$
+
+I don't know why I have two different /tmp catalogs. How do I modify which temp dir git-annex uses?
+
+"""]]
diff --git a/doc/forum/new_linux_arm_tarball_build/comment_18_0a76ce3895ba9598bd79625268d95ef4._comment b/doc/forum/new_linux_arm_tarball_build/comment_18_0a76ce3895ba9598bd79625268d95ef4._comment
new file mode 100644
index 0000000000..b5e5881007
--- /dev/null
+++ b/doc/forum/new_linux_arm_tarball_build/comment_18_0a76ce3895ba9598bd79625268d95ef4._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="joey"
+ subject="""comment 18"""
+ date="2016-01-29T17:27:42Z"
+ content="""
+@db, git-annex supports the standard TMPDIR environment variable.
+But, this is getting quite far afield from the topic of this thread.
+"""]]
diff --git a/doc/forum/new_linux_arm_tarball_build/comment_1_7211ddc626bae97d4140c723c3cf028f._comment b/doc/forum/new_linux_arm_tarball_build/comment_1_7211ddc626bae97d4140c723c3cf028f._comment
new file mode 100644
index 0000000000..2e91a7a831
--- /dev/null
+++ b/doc/forum/new_linux_arm_tarball_build/comment_1_7211ddc626bae97d4140c723c3cf028f._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="http://joeyh.name/"
+ ip="209.250.56.87"
+ subject="works on raspbian"
+ date="2013-12-16T23:26:51Z"
+ content="""
+This build has been verified to work on raspbian.
+"""]]
diff --git a/doc/forum/new_linux_arm_tarball_build/comment_2_fcbe3f7fa9d012b21c7a771553fa9142._comment b/doc/forum/new_linux_arm_tarball_build/comment_2_fcbe3f7fa9d012b21c7a771553fa9142._comment
new file mode 100644
index 0000000000..9499141fbd
--- /dev/null
+++ b/doc/forum/new_linux_arm_tarball_build/comment_2_fcbe3f7fa9d012b21c7a771553fa9142._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="http://joeyh.name/"
+ ip="209.250.56.87"
+ subject="works on Synology NAS"
+ date="2013-12-17T02:52:22Z"
+ content="""
+Using latest build (fixed some issues).
+"""]]
diff --git a/doc/forum/new_linux_arm_tarball_build/comment_3_2702cdbae4179a7a103d2a7098a8ed5e._comment b/doc/forum/new_linux_arm_tarball_build/comment_3_2702cdbae4179a7a103d2a7098a8ed5e._comment
new file mode 100644
index 0000000000..ec63cccc92
--- /dev/null
+++ b/doc/forum/new_linux_arm_tarball_build/comment_3_2702cdbae4179a7a103d2a7098a8ed5e._comment
@@ -0,0 +1,34 @@
+[[!comment format=mdwn
+ username="http://julien.lefrique.name/"
+ nickname="jlefrique"
+ subject="QNAP"
+ date="2013-12-17T07:57:37Z"
+ content="""
+Hi Joey,
+
+Thank you for taking the time to setup an ARM build. I am trying to run the last standalone build on a QNAP TS-219PII. I get the following error. Do you have any ideas?
+
+    $ uname -a
+    Linux willow 2.6.33.2 #1 Fri Mar 1 04:41:48 CST 2013 armv5tel unknown
+    $ ./runshell
+    $ cd ../annex
+    $ git annex version
+    git-annex version: 5.20131216-g07252d6
+    build flags: Assistant Pairing S3 Inotify DBus XMPP Feeds Quvi TDFA CryptoHash
+    key/value backends: SHA256E SHA1E SHA512E SHA224E SHA384E SKEIN256E SKEIN512E SHA256 SHA1 SHA512 SHA224 SHA384 SKEIN256 SKEIN512 WORM URL
+    remote types: git gcrypt S3 bup directory rsync web glacier hook
+    local repository version: 3
+    default repository version: 3
+    supported repository versions: 3 5
+    upgrade supported from repository versions: 0 1 2 4
+    $ git annex info        
+    repository mode: indirect
+    trusted repositories: fatal: cannot get RLIMIT_NOFILE: Bad address
+    fatal: cannot get RLIMIT_NOFILE: Bad address
+    fatal: cannot get RLIMIT_NOFILE: Bad address
+    fatal: cannot get RLIMIT_NOFILE: Bad address
+    fatal: cannot get RLIMIT_NOFILE: Bad address
+    fatal: cannot get RLIMIT_NOFILE: Bad address
+
+I am still using the version 3.8.3 of the QNAP distribution. I will try to update to the last version (4.0.5) to see if it helps.
+"""]]
diff --git a/doc/forum/new_linux_arm_tarball_build/comment_4_e1d802fbcc9d699ece5267e80990255a._comment b/doc/forum/new_linux_arm_tarball_build/comment_4_e1d802fbcc9d699ece5267e80990255a._comment
new file mode 100644
index 0000000000..e95d86b729
--- /dev/null
+++ b/doc/forum/new_linux_arm_tarball_build/comment_4_e1d802fbcc9d699ece5267e80990255a._comment
@@ -0,0 +1,10 @@
+[[!comment format=mdwn
+ username="http://joeyh.name/"
+ ip="209.250.56.87"
+ subject="RLIMIT_NOFILE"
+ date="2013-12-17T16:05:12Z"
+ content="""
+This is clearly failing when it runs git. Looking at the git source code, it does have a `RLIMIT_NOFILE` ifdef, so I could disable this, but would have to maintain my own build of git.
+
+What kernel version is this? Upgrading is likely to fix it.
+"""]]
diff --git a/doc/forum/new_linux_arm_tarball_build/comment_5_257b91ecbf5a6040a8e4c9a360c775ba._comment b/doc/forum/new_linux_arm_tarball_build/comment_5_257b91ecbf5a6040a8e4c9a360c775ba._comment
new file mode 100644
index 0000000000..94ea05a495
--- /dev/null
+++ b/doc/forum/new_linux_arm_tarball_build/comment_5_257b91ecbf5a6040a8e4c9a360c775ba._comment
@@ -0,0 +1,9 @@
+[[!comment format=mdwn
+ username="http://julien.lefrique.name/"
+ nickname="jlefrique"
+ subject="QNAP kernel"
+ date="2013-12-17T16:27:21Z"
+ content="""
+It's kernel 2.6.33.2 (see uname in my previous message).
+A new version with kernel 3.4.x is available, I will try to update when the data on my NAS will be (more) safely backup-ed.
+"""]]
diff --git a/doc/forum/new_linux_arm_tarball_build/comment_6_bd8cca86a63be7e330111618c1959a74._comment b/doc/forum/new_linux_arm_tarball_build/comment_6_bd8cca86a63be7e330111618c1959a74._comment
new file mode 100644
index 0000000000..a57d48356b
--- /dev/null
+++ b/doc/forum/new_linux_arm_tarball_build/comment_6_bd8cca86a63be7e330111618c1959a74._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="http://joeyh.name/"
+ ip="209.250.56.87"
+ subject="Re: RLIMIT_NOFILE"
+ date="2013-12-19T02:10:45Z"
+ content="""
+I've passed this problem on to the git developers, and they have quickly developed a patch that will solve it, so I'll just wait for the next git release.
+"""]]
diff --git a/doc/forum/new_linux_arm_tarball_build/comment_7_6814bdeca94328fe6c3f407795ff923a._comment b/doc/forum/new_linux_arm_tarball_build/comment_7_6814bdeca94328fe6c3f407795ff923a._comment
new file mode 100644
index 0000000000..a4b597bdf3
--- /dev/null
+++ b/doc/forum/new_linux_arm_tarball_build/comment_7_6814bdeca94328fe6c3f407795ff923a._comment
@@ -0,0 +1,10 @@
+[[!comment format=mdwn
+ username="http://julien.lefrique.name/"
+ nickname="jlefrique"
+ subject="Thanks"
+ date="2013-12-19T07:20:31Z"
+ content="""
+That's perfect!
+On my side, I upgraded to firmware 4.0.2 but the kernel is still the same.
+The kernel 3.4.x is available in the beta of QTS 4.1.0, I will wait the official release to upgrade.
+"""]]
diff --git a/doc/forum/new_linux_arm_tarball_build/comment_8_6db99d998ca990494c8f2826ff1ca273._comment b/doc/forum/new_linux_arm_tarball_build/comment_8_6db99d998ca990494c8f2826ff1ca273._comment
new file mode 100644
index 0000000000..9d58cb11ca
--- /dev/null
+++ b/doc/forum/new_linux_arm_tarball_build/comment_8_6db99d998ca990494c8f2826ff1ca273._comment
@@ -0,0 +1,11 @@
+[[!comment format=mdwn
+ username="https://renaud.casenave.fr/"
+ ip="126.10.66.235"
+ subject="comment 8"
+ date="2014-02-17T13:15:24Z"
+ content="""
+Hi,
+
+You might be happy to know git-annex works well on sailfishOS, even the webapp.
+It does segfault when trying to setup a xmpp account, though.
+"""]]
diff --git a/doc/forum/new_linux_arm_tarball_build/comment_9_2802b24ccb24f1615c9d61904f916d05._comment b/doc/forum/new_linux_arm_tarball_build/comment_9_2802b24ccb24f1615c9d61904f916d05._comment
new file mode 100644
index 0000000000..fbb0e2f219
--- /dev/null
+++ b/doc/forum/new_linux_arm_tarball_build/comment_9_2802b24ccb24f1615c9d61904f916d05._comment
@@ -0,0 +1,12 @@
+[[!comment format=mdwn
+ username="https://www.google.com/accounts/o8/id?id=AItOawmUJBh1lYmvfCCiGr3yrdx-QhuLCSRnU5c"
+ nickname="Justin"
+ subject="comment 9"
+ date="2014-03-01T19:13:16Z"
+ content="""
+This seems to work great for me on Raspbian, except I get many errors of the form
+
+    ERROR: ld.so: object '/usr/lib/arm-linux-gnueabihf/libcofi_rpi.so' from /etc/ld.so.preload cannot be preloaded: ignored.
+
+I've tried, but I haven't been able to figure out why ld doesn't like this particular lib.  I can keep ignoring the error, but help would certainly be appreciated!
+"""]]
diff --git a/doc/forum/new_microfeatures.mdwn b/doc/forum/new_microfeatures.mdwn
new file mode 100644
index 0000000000..bfe44272a7
--- /dev/null
+++ b/doc/forum/new_microfeatures.mdwn
@@ -0,0 +1,59 @@
+I'm soliciting ideas for new small features that let git-annex do things that currently have to be done manually or whatever.
+
+Here are a few I've been considering:
+
+---
+
+* --numcopies would be a useful command line switch.
+  > Update: Added. Also allows for things like `git annex drop --numcopies=2` when in a repo that normally needs 3 copies, if you need
+  > to urgently free up space.
+* A way to make `drop` and other commands temporarily trust a given remote, or possibly all remotes. 
+
+Combined, this would allow `git annex drop --numcopies=2 --trust=repoa --trust=repob` to remove files that have been replicated out to the other 2 repositories, which could be offline. (Slightly unsafe, but in this case the files are podcasts so not really.)
+
+> Update: done --[[Joey]] 
+
+---
+
+[[wishlist:_git-annex_replicate]] suggests some way for git-annex to have the smarts to copy content around on its own to ensure numcopies is satisfied. I'd be satisfied with a `git annex copy --to foo --if-needed-by-numcopies`
+
+  > Contrary to the "basic" solution, I would love to have a git annex distribute which is smart enough to simply distribute all data according to certain rules. My ideal, personal use case during the next holidays where I will have two external disks, several SD cards with 32 GB each and a local disk with 20 GB (yes....) would be:
+
+    cd ~/photos.annex # this repository does not have any objects!
+    git annex inject --bare /path/to/SD/card  # this adds softlinks, but does **not** add anything to the index. it would calculate checksums (if enabled) and have to add a temporary location list, though
+    git annex distribute # this checks the config. it would see that my two external disks have a low cost whereas the two remotes have a higher cost.
+     # check numcopies. it's 3
+     # copy to external disk one (cost x)
+     # copy to external disk two (cost x)
+     # copy to remote one (cost x * 2)
+     # remove file from temporary tracking list
+    git annex fsck # everything ok. yay!
+
+Come to think of it, the inject --bare thing is probably not a microfeature. Should I add a new wishlist item for that? -- RichiH
+
+> I've thought about such things before; does not seem really micro and I'm unsure how well it would work, but it would be worth a [[todo]]. --[[Joey]]
+
+>> Update: Done as --auto. --[[Joey]] 
+
+---
+
+Along similar lines, it might be nice to have a mode where git-annex tries to fill up a disk up to the `annex.diskreserve` with files, preferring files that have relatively few copies. Then as storage prices continue to fall, new large drives could just be plopped in and git-annex used to fill it up in a way that improves the overall redundancy without needing to manually pick and choose.
+
+> Update: git annex get --auto basically does this; you can tune
+> --numcopies on the fly to make it get more files than needed by the
+> current numcopies setting. --[[Joey]] 
+
+---
+
+If a remote could send on received files to another remote, I could use my own local bandwith efficiently while still having my git-annex repos replicate data. -- RichiH
+
+---
+
+Really micro:
+
+    % grep annex-push .git/config
+        annex-push = !git pull && git annex add . && git annex copy . --to origin --fast --quiet && git commit -a -m "$HOST $(date +%F--%H-%M-%S-%Z)" && git push
+    %
+
+-- RichiH
+--[[Joey]]
diff --git a/doc/forum/new_microfeatures/comment_1_058bd517c6fffaf3446b1f5d5be63623._comment b/doc/forum/new_microfeatures/comment_1_058bd517c6fffaf3446b1f5d5be63623._comment
new file mode 100644
index 0000000000..84fdd325dc
--- /dev/null
+++ b/doc/forum/new_microfeatures/comment_1_058bd517c6fffaf3446b1f5d5be63623._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="https://www.google.com/accounts/o8/id?id=AItOawkSq2FDpK2n66QRUxtqqdbyDuwgbQmUWus"
+ nickname="Jimmy"
+ subject="comment 1"
+ date="2011-06-01T17:36:50Z"
+ content="""
+I've been longing for an automated way of removing references to a remote assuming I know the exact uuid that I want to remove. i.e. I have lost a portable HDD due to a destructive process, I now want to delete all references to copies of data that was on that disk. Unless this feature exists, I would love to see it implemented.
+"""]]
diff --git a/doc/forum/new_microfeatures/comment_2_41ad904c68e89c85e1fc49c9e9106969._comment b/doc/forum/new_microfeatures/comment_2_41ad904c68e89c85e1fc49c9e9106969._comment
new file mode 100644
index 0000000000..4451e20baf
--- /dev/null
+++ b/doc/forum/new_microfeatures/comment_2_41ad904c68e89c85e1fc49c9e9106969._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="http://joey.kitenet.net/"
+ nickname="joey"
+ subject="comment 2"
+ date="2011-06-01T20:24:33Z"
+ content="""
+@jimmy [[walkthrough/what_to_do_when_you_lose_a_repository]].. I have not seen a convincing argument that removing the location tracking data entirely serves any purpose
+"""]]
diff --git a/doc/forum/new_microfeatures/comment_3_a1a9347b5bc517f2a89a8b292c3f8517._comment b/doc/forum/new_microfeatures/comment_3_a1a9347b5bc517f2a89a8b292c3f8517._comment
new file mode 100644
index 0000000000..4bb3aa684f
--- /dev/null
+++ b/doc/forum/new_microfeatures/comment_3_a1a9347b5bc517f2a89a8b292c3f8517._comment
@@ -0,0 +1,15 @@
+[[!comment format=mdwn
+ username="https://www.google.com/accounts/o8/id?id=AItOawnpdM9F8VbtQ_H5PaPMpGSxPe_d5L1eJ6w"
+ nickname="Rafaël"
+ subject="git annex unlock --readonly"
+ date="2011-06-02T11:34:42Z"
+ content="""
+This was already asked [here](http://bugs.debian.org/cgi-bin/bugreport.cgi?bug=606577), but I have a use case where I need to unlock with the files being hardlinked instead of copied (my fs does not support CoW), even though 'git annex lock' is now much faster ;-) . The idea is that 1) I want the external world see my repo \"as if\" it wasn't annexed (because of its own limitation to deal with soft links), and 2) I know what I do, and am sure that files won't be written to but only read.
+
+My case is: the repo contains a snapshot A1 of a certain remote directory. Later I want to rsync this dir into a new snapshot A2. Of course, I want to transfer only new or changed files, with the --copy-dest=A1 (or --compare-dest) rsync's options. Unfortunately, rsync won't recognize soft-links from git-annex, and will re-transfer everything.
+
+
+Maybe I'm overusing git-annex ;-) but still, I find it is a legitimate use case, and even though there are workarounds (I don't even remember what I had to do), it would be much more straightforward to have 'git annex unlock --readonly' (or '--readonly-unsafe'?), ... or have rsync take soft-links into account, but I did not see the author ask for microfeatures ideas :) (it was discussed, and only some convoluted workarounds were proposed). Thanks.
+
+
+"""]]
diff --git a/doc/forum/new_microfeatures/comment_4_5a6786dc52382fff5cc42fdb05770196._comment b/doc/forum/new_microfeatures/comment_4_5a6786dc52382fff5cc42fdb05770196._comment
new file mode 100644
index 0000000000..cc98109e6b
--- /dev/null
+++ b/doc/forum/new_microfeatures/comment_4_5a6786dc52382fff5cc42fdb05770196._comment
@@ -0,0 +1,18 @@
+[[!comment format=mdwn
+ username="https://www.google.com/accounts/o8/id?id=AItOawnpdM9F8VbtQ_H5PaPMpGSxPe_d5L1eJ6w"
+ nickname="Rafaël"
+ subject="git annex unused"
+ date="2011-06-02T11:55:58Z"
+ content="""
+Before dropping unsused items, sometimes I want to check the content of the files manually.
+But currently, from e.g. a sha1 key, I don't know how to find the corresponding file, except with
+'find .git/annex/objects -type f -name 'SHA1-s1678--70....', wich is too slow (I'm in the case where \"git log --stat -S'KEY'\"
+won't work, either because it is too slow or it was never commited). By the way,
+is it documented somewhere how to determine the 2 (nested) sub-directories in which a given
+(by name) object is located?
+
+So I would like 'git-annex unused' be able to give me the list of *paths* to the unused items.
+Also, I would really appreciate a command like 'git annex unused --log NUMBER [NUMBER2...]' which would do for me the suggested command
+\"git log --stat -S'KEY'\", where NUMBER is from the 'git annex unused' output.
+Thanks.
+"""]]
diff --git a/doc/forum/new_microfeatures/comment_5_3c627d275586ff499d928a8f8136babf._comment b/doc/forum/new_microfeatures/comment_5_3c627d275586ff499d928a8f8136babf._comment
new file mode 100644
index 0000000000..f7361f5d1c
--- /dev/null
+++ b/doc/forum/new_microfeatures/comment_5_3c627d275586ff499d928a8f8136babf._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="https://www.google.com/accounts/o8/id?id=AItOawnpdM9F8VbtQ_H5PaPMpGSxPe_d5L1eJ6w"
+ nickname="Rafaël"
+ subject="git annex unused"
+ date="2011-06-02T19:51:49Z"
+ content="""
+ps: concerning the command 'find .git/annex/objects -type f -name 'SHA1-s1678--70....' from my previous comment, it is \"significantly\" faster to search for the containing directory which have the same name: 'find .git/annex/objects -maxdepth 2 -mindepth 2 -type d -name 'SHA1-s1678--70....'. I am just curious: what is the need to have each file object in its own directory, itself nested under two more sub-directories?
+"""]]
diff --git a/doc/forum/new_microfeatures/comment_6_31ea08c008500560c0b96c6601bc6362._comment b/doc/forum/new_microfeatures/comment_6_31ea08c008500560c0b96c6601bc6362._comment
new file mode 100644
index 0000000000..868e9677c8
--- /dev/null
+++ b/doc/forum/new_microfeatures/comment_6_31ea08c008500560c0b96c6601bc6362._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="https://www.google.com/accounts/o8/id?id=AItOawnpdM9F8VbtQ_H5PaPMpGSxPe_d5L1eJ6w"
+ nickname="Rafaël"
+ subject="git annex fetch"
+ date="2011-07-03T14:39:41Z"
+ content="""
+I'm not sure it is worth adding a command for such a small feature, but I would certainly use it: having something like \"git annex fetch remote\" do \"git fetch remote && git annex copy --from=remote\", and \"git annex push remote\" do \"git push remote && git annex copy --to=remote\". And maybe the same for a pull operation?
+"""]]
diff --git a/doc/forum/new_microfeatures/comment_7_94045b9078b1fff877933b012d1b49e2._comment b/doc/forum/new_microfeatures/comment_7_94045b9078b1fff877933b012d1b49e2._comment
new file mode 100644
index 0000000000..e39e162322
--- /dev/null
+++ b/doc/forum/new_microfeatures/comment_7_94045b9078b1fff877933b012d1b49e2._comment
@@ -0,0 +1,10 @@
+[[!comment format=mdwn
+ username="https://www.google.com/accounts/o8/id?id=AItOawnpdM9F8VbtQ_H5PaPMpGSxPe_d5L1eJ6w"
+ nickname="Rafaël"
+ subject="git annex fetch"
+ date="2011-07-03T17:57:00Z"
+ content="""
+My last comment is a bit confused. The \"git fetch\" command allows to get all the information from a remote, and it is then possible to merge while being offline (without access to the remote). I would like a \"git annex fetch remote\" command to be able to get all annexed files from remote, so that if I later merge with remote, all annexed files are already here. And \"git annex fetch\" could (optionally) call \"git fetch\" before getting the files.
+
+It seems also that in my last post, I should have written \"git annex get --from=remote\" instead of \"git annex copy --from=remote\", because \"annex copy --from\" copies all files, even if the local repo already have them (is this the case? if yes, when is it useful?)
+"""]]
diff --git a/doc/forum/nntp__47__usenet_special_remote.mdwn b/doc/forum/nntp__47__usenet_special_remote.mdwn
new file mode 100644
index 0000000000..86efa48cff
--- /dev/null
+++ b/doc/forum/nntp__47__usenet_special_remote.mdwn
@@ -0,0 +1,18 @@
+Over the last few weeks i've been working on a seperate python nntp project. But decided that it would be fun to make it usable with git-annex.
+
+The code is almost complete. One major feature left(make it usable without a mysql/sqlite database, and without fetching headers). And of course a lot of bugfixes.
+
+I'm wondering if there is any interest in this special remote hook.
+
+The thought is, buy unlimited usenet account, and you can have a git-annex repository without any upper size limit. You want 100GB, well that'll be 10$ a month, want 100000GB, well that'll be 10$ a month.
+
+Of course there are plenty of caveats(retention on server, it is doubtfull the data will stay up for more than 4 years.). Because the code is in python3 i had to use a lot of external tools (uudeview, par2, ydecode/yydecode) and parse their output. Which is a horrid horrid solution. Since the code is nearing "beta" stage i might see if i can improve on that(possibly with ctypes), if there is any interest.
+
+Currently working on freebsd, ubuntu, debian, mac os x(10.7, not tested in 10.8)
+
+The last feature to be done (and this is UUUUUGLY) is to make it get nzb files from a source like this: http://nzbindex.nl/search/?q=GPGHMACSHA (yes, that is my git-annex repo) instead of from a mysql database.
+
+So, this is a call to see if anyone is interested, and if anyone proficient in *NIX want to test it out.
+
+Sincerely
+Tobias
diff --git a/doc/forum/nntp__47__usenet_special_remote/comment_1_171a0b95b1f95cfd82073e88bdefaab9._comment b/doc/forum/nntp__47__usenet_special_remote/comment_1_171a0b95b1f95cfd82073e88bdefaab9._comment
new file mode 100644
index 0000000000..ff14f6f2ec
--- /dev/null
+++ b/doc/forum/nntp__47__usenet_special_remote/comment_1_171a0b95b1f95cfd82073e88bdefaab9._comment
@@ -0,0 +1,10 @@
+[[!comment format=mdwn
+ username="http://joeyh.name/"
+ nickname="joey"
+ subject="comment 1"
+ date="2013-05-19T20:49:06Z"
+ content="""
+I think it's a neat idea. I can't see using it for my own data due to retention, but if I had a use case where I wanted to spread data around and didn't care if it vanished after X months, I can see using it.
+
+I think you should publish the details, if getting it to a state others can use is not too much work,
+"""]]
diff --git a/doc/forum/nntp__47__usenet_special_remote/comment_2_48736ed17c98ffcfb13ec00b901b2dd6._comment b/doc/forum/nntp__47__usenet_special_remote/comment_2_48736ed17c98ffcfb13ec00b901b2dd6._comment
new file mode 100644
index 0000000000..90b264f7b6
--- /dev/null
+++ b/doc/forum/nntp__47__usenet_special_remote/comment_2_48736ed17c98ffcfb13ec00b901b2dd6._comment
@@ -0,0 +1,13 @@
+[[!comment format=mdwn
+ username="https://www.google.com/accounts/o8/id?id=AItOawmkBwMWvNKZZCge_YqobCSILPMeK6xbFw8"
+ nickname="develop"
+ subject="comment 2"
+ date="2013-09-11T07:46:07Z"
+ content="""
+Only played a tiny bit with this since the initial post.
+
+I've decided that sql backend should be dropped entirely. Mostly the SQL backend was there because the code does something else entirely, and this was just a test.
+
+But since there has been no interest it will probably take ages before i actually get this finished :)
+
+"""]]
diff --git a/doc/forum/non-bare_repo_on_cloud_remote.mdwn b/doc/forum/non-bare_repo_on_cloud_remote.mdwn
new file mode 100644
index 0000000000..3949e18dac
--- /dev/null
+++ b/doc/forum/non-bare_repo_on_cloud_remote.mdwn
@@ -0,0 +1,6 @@
+Hi,
+
+I wondered if it's possible to have a non-bare repo on a cloud remote. I want to have some sort of Dropbox-like workflow were I simply put stuff into a directory on my drive and it gets synced via SSH to a central VPS so I can also always access it via HTTP, for example.
+
+Regards,
+Lukas
diff --git a/doc/forum/non-bare_repo_on_cloud_remote/comment_1_da0c023af7c78f1ef1cfe1143a900a9f._comment b/doc/forum/non-bare_repo_on_cloud_remote/comment_1_da0c023af7c78f1ef1cfe1143a900a9f._comment
new file mode 100644
index 0000000000..27e908cacd
--- /dev/null
+++ b/doc/forum/non-bare_repo_on_cloud_remote/comment_1_da0c023af7c78f1ef1cfe1143a900a9f._comment
@@ -0,0 +1,12 @@
+[[!comment format=mdwn
+ username="http://joeyh.name/"
+ ip="4.154.1.10"
+ subject="comment 1"
+ date="2013-07-22T22:04:53Z"
+ content="""
+Sure you can.. That's how  works, which is where people download git-annex builds from!
+
+[[tips/making_a_remote_repo_update_when_changes_are_pushed_to_it]]
+
+[[tips/setup_a_public_repository_on_a_web_site]]
+"""]]
diff --git a/doc/forum/non-bare_repo_on_cloud_remote/comment_2_71baea93f6caaf7b81a9ac00bee91e5e._comment b/doc/forum/non-bare_repo_on_cloud_remote/comment_2_71baea93f6caaf7b81a9ac00bee91e5e._comment
new file mode 100644
index 0000000000..145c57120c
--- /dev/null
+++ b/doc/forum/non-bare_repo_on_cloud_remote/comment_2_71baea93f6caaf7b81a9ac00bee91e5e._comment
@@ -0,0 +1,67 @@
+[[!comment format=mdwn
+ username="https://www.google.com/accounts/o8/id?id=AItOawniayrgSdVLUc3c6bf93VbO-_HT4hzxmyo"
+ nickname="Tobias"
+ subject="comment 2"
+ date="2013-08-08T19:05:07Z"
+ content="""
+Seems that I don't get it... Here is what I did:
+
+First, I create a new repo with the assistant on `user@laptop ~/temp/annex-test`, then:
+
+    user@laptop ~/temp/annex-test % echo test1 > test1
+    user@laptop ~/temp/annex-test % echo test2 > test2
+    
+    user@server ~/tmp % mkdir annex-test
+    user@server ~/tmp % cd annex-test
+    user@server ~/tmp/annex-test % git init
+    Initialized empty Git repository in /home/tobru/tmp/annex-test/.git/
+    user@server ~/tmp/annex-test % git annex init
+    init  ok
+    (Recording state in git...)
+    user@server ~/tmp/annex-test % mv .git/hooks/post-update.sample .git/hooks/post-update       
+    user@server ~/tmp/annex-test % cat .git/hooks/post-update
+    #!/bin/sh
+    #
+    # An example hook script to prepare a packed repository for use over
+    # dumb transports.
+    #
+    # To enable this hook, rename this file to \"post-update\".
+    
+    exec git update-server-info
+    user@server ~/tmp/annex-test % chmod +x .git/hooks/post-update
+    user@server ~/tmp/annex-test % echo '#!/bin/sh\ngit annex merge' > .git/hooks/post-receive
+    user@server ~/tmp/annex-test % chmod +x .git/hooks/post-receive
+    
+    user@laptop ~/temp/annex-test % git remote add server ssh://user@server/home/tobru/tmp/annex-test
+    user@laptop ~/temp/annex-test % git annex assistant --stop
+    user@laptop ~/temp/annex-test % git annex sync
+    commit  
+    ok
+    pull server 
+    warning: no common commits
+    remote: Counting objects: 5, done.
+    remote: Compressing objects: 100% (3/3), done.
+    remote: Total 5 (delta 1), reused 0 (delta 0)
+    Unpacking objects: 100% (5/5), done.
+    From ssh://server.tobru.local/home/tobru/tmp/annex-test
+     * [new branch]      git-annex  -> server/git-annex
+    ok
+    (merging server/git-annex into git-annex...)
+    (Recording state in git...)
+    push server 
+    Counting objects: 27, done.
+    Delta compression using up to 4 threads.
+    Compressing objects: 100% (20/20), done.
+    Writing objects: 100% (25/25), 2.36 KiB, done.
+    Total 25 (delta 2), reused 0 (delta 0)
+    remote: merge git-annex (merging synced/git-annex into git-annex...)
+    remote: ok
+    To ssh://tobru@server.tobru.local/home/tobru/tmp/annex-test
+     * [new branch]      git-annex -> synced/git-annex
+     * [new branch]      master -> synced/master
+    ok
+    user@laptop ~/temp/annex-test % git annex assistant
+
+Now I expect the files `test1` and `test2` to be at `user@server ~/tmp/annex-test`. But they are not displayed.
+What am I doing wrong? The main thing I wan't to achieve: Push the files from `laptop` to `server` without adding `laptop` as remote on the server.
+"""]]
diff --git a/doc/forum/non_fast_forward_error_with_git_annex_sync.mdwn b/doc/forum/non_fast_forward_error_with_git_annex_sync.mdwn
new file mode 100644
index 0000000000..19dad625bc
--- /dev/null
+++ b/doc/forum/non_fast_forward_error_with_git_annex_sync.mdwn
@@ -0,0 +1,34 @@
+I'm trying to sync from a direct repo to another direct repo but getting a non-fast-forward rejection:
+
+    push v-crateris 
+    Counting objects: 509, done.
+    Delta compression using up to 4 threads.
+    Compressing objects: 100% (455/455), done.
+    Writing objects: 100% (509/509), 33.21 KiB | 4.74 MiB/s, done.
+    Total 509 (delta 350), reused 0 (delta 0)
+    remote: Resolving deltas: 100% (350/350), completed with 174 local objects.
+    To v-crateris:/annex/Music
+       c60621302..803920707  git-annex -> synced/git-annex
+     ! [rejected]            annex/direct/master -> synced/master (non-fast-forward)
+    error: failed to push some refs to 'v-crateris:/annex/Music'
+    hint: Updates were rejected because a pushed branch tip is behind its remote
+    hint: counterpart. Check out this branch and integrate the remote changes
+    hint: (e.g. 'git pull ...') before pushing again.
+    hint: See the 'Note about fast-forwards' in 'git push --help' for details.
+    To v-crateris:/annex/Music
+     ! [rejected]            master -> master (non-fast-forward)
+    error: failed to push some refs to 'v-crateris:/annex/Music'
+    hint: Updates were rejected because a pushed branch tip is behind its remote
+    hint: counterpart. Check out this branch and integrate the remote changes
+    hint: (e.g. 'git pull ...') before pushing again.
+    hint: See the 'Note about fast-forwards' in 'git push --help' for details.
+
+      Pushing to v-crateris failed.
+
+      (non-fast-forward problems can be solved by setting receive.denyNonFastforwards to false in the remote's git config)
+    failed
+
+I tried setting `receive.denyNonFastforwards` to `false` as the message said, but it made no difference.
+Both repos have `annex/direct/master` checked out.
+
+What should I do to resolve this?
diff --git a/doc/forum/non_fast_forward_error_with_git_annex_sync/comment_1_48d66e5314de5b6a26f78622a4e93ba6._comment b/doc/forum/non_fast_forward_error_with_git_annex_sync/comment_1_48d66e5314de5b6a26f78622a4e93ba6._comment
new file mode 100644
index 0000000000..acde90fd24
--- /dev/null
+++ b/doc/forum/non_fast_forward_error_with_git_annex_sync/comment_1_48d66e5314de5b6a26f78622a4e93ba6._comment
@@ -0,0 +1,9 @@
+[[!comment format=mdwn
+ username="matyasbot@fd008517d046c382e18306c0b3db48eb58d45dee"
+ nickname="matyasbot"
+ avatar="http://cdn.libravatar.org/avatar/c1b6b47dcdf2962858e6aad44a3cec66"
+ subject="pull didn't do anything either"
+ date="2017-10-11T14:23:29Z"
+ content="""
+Also, despite the pull claiming to have succeded, there was a new directory on the remote that did not get transferred to the local repo during the sync.
+"""]]
diff --git a/doc/forum/non_fast_forward_error_with_git_annex_sync/comment_2_f7101a8e84bbbfb2fbea6c65ca62fdb9._comment b/doc/forum/non_fast_forward_error_with_git_annex_sync/comment_2_f7101a8e84bbbfb2fbea6c65ca62fdb9._comment
new file mode 100644
index 0000000000..a6ad752db3
--- /dev/null
+++ b/doc/forum/non_fast_forward_error_with_git_annex_sync/comment_2_f7101a8e84bbbfb2fbea6c65ca62fdb9._comment
@@ -0,0 +1,9 @@
+[[!comment format=mdwn
+ username="joey"
+ subject="""comment 2"""
+ date="2017-10-11T15:14:18Z"
+ content="""
+You need to set the receive.denyNonFastforwards configuration in
+the remote git repository, not on the local git repository. It kind of
+sounds like you may have misunderstood where to set it.
+"""]]
diff --git a/doc/forum/non_fast_forward_error_with_git_annex_sync/comment_3_bbb3a5572fa2427d9bf859ae1ba68764._comment b/doc/forum/non_fast_forward_error_with_git_annex_sync/comment_3_bbb3a5572fa2427d9bf859ae1ba68764._comment
new file mode 100644
index 0000000000..53e19eef4a
--- /dev/null
+++ b/doc/forum/non_fast_forward_error_with_git_annex_sync/comment_3_bbb3a5572fa2427d9bf859ae1ba68764._comment
@@ -0,0 +1,9 @@
+[[!comment format=mdwn
+ username="matyasbot@fd008517d046c382e18306c0b3db48eb58d45dee"
+ nickname="matyasbot"
+ avatar="http://cdn.libravatar.org/avatar/c1b6b47dcdf2962858e6aad44a3cec66"
+ subject="comment 3"
+ date="2017-10-11T15:17:55Z"
+ content="""
+No, that is exactly where I set it.
+"""]]
diff --git a/doc/forum/noob_question._VPS_web_assistant.mdwn b/doc/forum/noob_question._VPS_web_assistant.mdwn
new file mode 100644
index 0000000000..410cb74d55
--- /dev/null
+++ b/doc/forum/noob_question._VPS_web_assistant.mdwn
@@ -0,0 +1,7 @@
+Hi
+
+So I've installed git-annex on my Debian VPS.
+
+How do I find out the URL to access the web UI / assistant?
+
+Thanks!
diff --git a/doc/forum/noob_question._VPS_web_assistant/comment_1_49f349192d64048753bedcf10ee22355._comment b/doc/forum/noob_question._VPS_web_assistant/comment_1_49f349192d64048753bedcf10ee22355._comment
new file mode 100644
index 0000000000..f08d3e8c5b
--- /dev/null
+++ b/doc/forum/noob_question._VPS_web_assistant/comment_1_49f349192d64048753bedcf10ee22355._comment
@@ -0,0 +1,11 @@
+[[!comment format=mdwn
+ username="joey"
+ subject="""comment 1"""
+ date="2015-03-16T17:26:33Z"
+ content="""
+The url inculudes a secret token, so a new url is generated each time
+the assistant is started.
+
+The file `.git/annex/url` in your git-annex repository contains the current
+url.
+"""]]
diff --git a/doc/forum/not_finding_git-annex-shell_on_remote.mdwn b/doc/forum/not_finding_git-annex-shell_on_remote.mdwn
new file mode 100644
index 0000000000..0d743bf78c
--- /dev/null
+++ b/doc/forum/not_finding_git-annex-shell_on_remote.mdwn
@@ -0,0 +1,21 @@
+I have set up an annex on a remote machine and I am connecting via ssh.  But, since it is a managed machine, I installed the git-annex binary in my own ~/bin.  Well, when I try 
+$git annex sync
+
+I get:
+  $git annex sync
+(merging origin/git-annex into git-annex...)
+(Recording state in git...)
+bash: git-annex-shell: command not found
+
+  Remote origin does not have git-annex installed; setting annex-ignore
+commit  ok
+pull origin 
+
+merge: refs/remotes/origin/master - not something we can merge
+
+merge: refs/remotes/origin/synced/master - not something we can merge
+failed
+git-annex: sync: 1 failed
+
+
+The git remote -v looks correct.  So, how do I tell git annex on my local machine where to use $HOME/bin in PATH on the remote machine when syncing  with remotes?
diff --git a/doc/forum/not_finding_git-annex-shell_on_remote/comment_1_84881cad02c251a2515cec50fc22bf16._comment b/doc/forum/not_finding_git-annex-shell_on_remote/comment_1_84881cad02c251a2515cec50fc22bf16._comment
new file mode 100644
index 0000000000..8441f412af
--- /dev/null
+++ b/doc/forum/not_finding_git-annex-shell_on_remote/comment_1_84881cad02c251a2515cec50fc22bf16._comment
@@ -0,0 +1,16 @@
+[[!comment format=mdwn
+ username="https://www.google.com/accounts/o8/id?id=AItOawk3HGoDpnOPob5jOjvIootmkve1-nCpRiI"
+ nickname="Kalle"
+ subject="Declare path on first line of bashrc?"
+ date="2014-02-15T18:36:35Z"
+ content="""
+If i don't misremember some systems including Debian require you to set the path on the very first line of .bashrc for it to work. Can't remember why just now.
+
+in other words paste the following into the very first line of `$HOME/.bashrc`
+
+`PATH=$PATH:$HOME/bin:$HOME/bin/git-annex.linux`
+
+Modify the line above if you haven't installed to `~/bin/git-annex.linux`
+
+
+"""]]
diff --git a/doc/forum/not_finding_git-annex-shell_on_remote/comment_2_f32412f8d3b84cd5cb3c4d5d6bb60f32._comment b/doc/forum/not_finding_git-annex-shell_on_remote/comment_2_f32412f8d3b84cd5cb3c4d5d6bb60f32._comment
new file mode 100644
index 0000000000..7acee2035a
--- /dev/null
+++ b/doc/forum/not_finding_git-annex-shell_on_remote/comment_2_f32412f8d3b84cd5cb3c4d5d6bb60f32._comment
@@ -0,0 +1,36 @@
+[[!comment format=mdwn
+ username="https://www.google.com/accounts/o8/id?id=AItOawn3rK4VDzxyhmrIc18z7F5OuXvEbUsgUac"
+ nickname="Srinath"
+ subject="Progress, but still issues"
+ date="2014-02-17T04:09:23Z"
+ content="""
+\"I now realize that the git annex system still requires the standard \"add\" and \"commit\" process. But I'm still getting:
+
+$git annex sync
+commit  ok
+pull origin 
+remote: Counting objects: 37, done.
+remote: Compressing objects: 100% (35/35), done.
+remote: Total 36 (delta 0), reused 0 (delta 0)
+Unpacking objects: 100% (36/36), done.
+From ssh://stampede.tacc.utexas.edu/work/02463/srinathv/cesm1_3_beta07/scripts
+ * [new branch]      master     -> origin/master
+
+
+merge: refs/remotes/origin/synced/master - not something we can merge
+failed
+push origin 
+Counting objects: 11, done.
+Delta compression using up to 4 threads.
+Compressing objects: 100% (6/6), done.
+Writing objects: 100% (8/8), 736 bytes | 0 bytes/s, done.
+Total 8 (delta 3), reused 1 (delta 0)
+To ssh://srinathv@stampede.tacc.utexas.edu/work/02463/srinathv/cesm1_3_beta07/scripts
+ * [new branch]      git-annex -> synced/git-annex
+ * [new branch]      master -> synced/master
+ok
+git-annex: sync: 1 failed
+
+
+So the fails appear, and the suggestion of \"export PATH\" placement did not help, though appreciated.
+"""]]
diff --git a/doc/forum/not_finding_git-annex-shell_on_remote/comment_3_dfbf7f41dd4d17f2ce8b67daa9dcd11d._comment b/doc/forum/not_finding_git-annex-shell_on_remote/comment_3_dfbf7f41dd4d17f2ce8b67daa9dcd11d._comment
new file mode 100644
index 0000000000..53419452c2
--- /dev/null
+++ b/doc/forum/not_finding_git-annex-shell_on_remote/comment_3_dfbf7f41dd4d17f2ce8b67daa9dcd11d._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="https://www.google.com/accounts/o8/id?id=AItOawn3rK4VDzxyhmrIc18z7F5OuXvEbUsgUac"
+ nickname="Srinath"
+ subject="no more issue"
+ date="2014-02-17T04:25:31Z"
+ content="""
+After doing 1 more sync, the error messages are now gone.  I wonder if the refs directory needed to be synched correctly.
+"""]]
diff --git a/doc/forum/not_finding_git-annex-shell_on_remote/comment_4_71ae60efcacdba5e11548923b2c85b95._comment b/doc/forum/not_finding_git-annex-shell_on_remote/comment_4_71ae60efcacdba5e11548923b2c85b95._comment
new file mode 100644
index 0000000000..12bf8fa75e
--- /dev/null
+++ b/doc/forum/not_finding_git-annex-shell_on_remote/comment_4_71ae60efcacdba5e11548923b2c85b95._comment
@@ -0,0 +1,10 @@
+[[!comment format=mdwn
+ username="http://joeyh.name/"
+ ip="209.250.56.172"
+ subject="comment 4"
+ date="2014-02-20T19:41:05Z"
+ content="""
+@Srinath I wonder if the machine you are running git-annex sync on is a Windows machine? I have seen that \" refs/remotes/origin/synced/master - not something we can merge\" intermittently when testing on Windows, but not other times.
+
+(It's not related to the original PATH configuration problem on your server.)
+"""]]
diff --git a/doc/forum/not_getting_file_contents.mdwn b/doc/forum/not_getting_file_contents.mdwn
new file mode 100644
index 0000000000..23150216dc
--- /dev/null
+++ b/doc/forum/not_getting_file_contents.mdwn
@@ -0,0 +1 @@
+I have 2 computers successfully exchanging files, but the contexts are incomplete.  Used text files, documents, mp3s as tests. Both computers report in the webapp that they are synched and file names are there.  But file sizes are wrong by 75% less and contents are unusable.  Is there something in the config?  Both machines on LinuxMint 13. 
diff --git a/doc/forum/not_getting_file_contents/comment_1_4a0f7f4de9c9bc4d13db033cb75d20af._comment b/doc/forum/not_getting_file_contents/comment_1_4a0f7f4de9c9bc4d13db033cb75d20af._comment
new file mode 100644
index 0000000000..e03ec333a6
--- /dev/null
+++ b/doc/forum/not_getting_file_contents/comment_1_4a0f7f4de9c9bc4d13db033cb75d20af._comment
@@ -0,0 +1,10 @@
+[[!comment format=mdwn
+ username="http://joeyh.name/"
+ ip="4.154.7.235"
+ subject="comment 1"
+ date="2013-07-11T16:06:59Z"
+ content="""
+That certainly shouldn't be happening! 
+
+Are these computers syncing via local pairing, or are you using a transfer remote, and if so, which one?
+"""]]
diff --git a/doc/forum/not_getting_file_contents/comment_2_dc7403e1b551552f9fd00da6a1453570._comment b/doc/forum/not_getting_file_contents/comment_2_dc7403e1b551552f9fd00da6a1453570._comment
new file mode 100644
index 0000000000..499176b01e
--- /dev/null
+++ b/doc/forum/not_getting_file_contents/comment_2_dc7403e1b551552f9fd00da6a1453570._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="trilobite"
+ ip="71.17.134.14"
+ subject="comment 2"
+ date="2013-07-11T23:23:41Z"
+ content="""
+I did this with local synch in home lan.  Filenames do travel both directions.  Just not contents.  They are identifed as symlinks rather than files.
+"""]]
diff --git a/doc/forum/notify-start_and_notify-finish_for_the_assistant.mdwn b/doc/forum/notify-start_and_notify-finish_for_the_assistant.mdwn
new file mode 100644
index 0000000000..e9d1dccd7d
--- /dev/null
+++ b/doc/forum/notify-start_and_notify-finish_for_the_assistant.mdwn
@@ -0,0 +1,5 @@
+Is there a way to use notify-start and notify-finish, not when using git annex
+on the command line but also when using the assistant ?
+
+I would like to get notifications when I move files to annexed folders for when
+the transfer to the server is finished and such things.
diff --git a/doc/forum/notify-start_and_notify-finish_for_the_assistant/comment_1_d3da4ce5cfea6c0a3277e83e0181102d._comment b/doc/forum/notify-start_and_notify-finish_for_the_assistant/comment_1_d3da4ce5cfea6c0a3277e83e0181102d._comment
new file mode 100644
index 0000000000..80cc5a35cf
--- /dev/null
+++ b/doc/forum/notify-start_and_notify-finish_for_the_assistant/comment_1_d3da4ce5cfea6c0a3277e83e0181102d._comment
@@ -0,0 +1,11 @@
+[[!comment format=mdwn
+ username="joey"
+ subject="""comment 1"""
+ date="2015-03-26T15:22:32Z"
+ content="""
+You can pass the --notify-start and --notify-finish to the assistant
+when starting it up if you want to.
+
+It would perhaps be worth adding a config option so it could remember
+persistently to notify.
+"""]]
diff --git a/doc/forum/notify-start_and_notify-finish_for_the_assistant/comment_2_01dfd865c469a5bc465f73347a65ce44._comment b/doc/forum/notify-start_and_notify-finish_for_the_assistant/comment_2_01dfd865c469a5bc465f73347a65ce44._comment
new file mode 100644
index 0000000000..b0522755dc
--- /dev/null
+++ b/doc/forum/notify-start_and_notify-finish_for_the_assistant/comment_2_01dfd865c469a5bc465f73347a65ce44._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="https://renaud.casenave.fr/"
+ subject="comment 2"
+ date="2015-03-27T18:25:17Z"
+ content="""
+Well, I thought about that but it doesn't seem to work.
+with `git annex get --notify-start --notify-finish` I properly get the notifications but with `git annex assistant --notify-start --notify-finish` I get nothing when a file is uploaded or downloaded
+"""]]
diff --git a/doc/forum/offsite_repo.mdwn b/doc/forum/offsite_repo.mdwn
new file mode 100644
index 0000000000..0b950a95eb
--- /dev/null
+++ b/doc/forum/offsite_repo.mdwn
@@ -0,0 +1,8 @@
+Hi there,
+
+Let's say I have a Pictures repo in my laptop. I want to have a local copy in an external backup drive and an offsite copy in google-drive. I set numcopies to 3. Is it possible to require that files in a repository have a copy in an offsite repo e.g. a google-drive special remote? So that when I run e.g. git annex fsck, I will be reminded that some local files are not yet backed up in an offsite repo. Probably via .git/config, I can declare a remote repo as an "offsite" repo. Something like: 
+
+    remote.repo_name.offsite = true
+
+
+Eric
diff --git a/doc/forum/offsite_repo/comment_1_1f3ab9a39baa16e3e307b23fd1aabeb8._comment b/doc/forum/offsite_repo/comment_1_1f3ab9a39baa16e3e307b23fd1aabeb8._comment
new file mode 100644
index 0000000000..79f1810b3d
--- /dev/null
+++ b/doc/forum/offsite_repo/comment_1_1f3ab9a39baa16e3e307b23fd1aabeb8._comment
@@ -0,0 +1,10 @@
+[[!comment format=mdwn
+ username="joey"
+ subject="""comment 1"""
+ date="2018-02-08T17:32:00Z"
+ content="""
+What you're looking for is probably the [[required_content]] settings.
+
+However, `git annex fsck` did not perform any checks that required content
+was present. Now it does. Enjoy!
+"""]]
diff --git a/doc/forum/offsite_repo/comment_2_d4ecd0dc29597e3e0988e06ee9c4d245._comment b/doc/forum/offsite_repo/comment_2_d4ecd0dc29597e3e0988e06ee9c4d245._comment
new file mode 100644
index 0000000000..bcbb5ae366
--- /dev/null
+++ b/doc/forum/offsite_repo/comment_2_d4ecd0dc29597e3e0988e06ee9c4d245._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="ericm"
+ avatar="http://cdn.libravatar.org/avatar/67ca64c5a99fb142fc2e3916333881ca"
+ subject="comment 2"
+ date="2018-02-11T08:59:07Z"
+ content="""
+Ok I think that will do. I'll give it a try when I get some time. Thanks!
+"""]]
diff --git a/doc/forum/offsite_repo/comment_3_4b233762116c52bd2a754f49152201b7._comment b/doc/forum/offsite_repo/comment_3_4b233762116c52bd2a754f49152201b7._comment
new file mode 100644
index 0000000000..a0ae5865aa
--- /dev/null
+++ b/doc/forum/offsite_repo/comment_3_4b233762116c52bd2a754f49152201b7._comment
@@ -0,0 +1,29 @@
+[[!comment format=mdwn
+ username="ericm"
+ avatar="http://cdn.libravatar.org/avatar/67ca64c5a99fb142fc2e3916333881ca"
+ subject="comment 3"
+ date="2018-02-18T13:46:00Z"
+ content="""
+Hi,
+
+I first tried messing around with required content settings in a test repo in direct mode. This is what I did to initialize a test repo:
+
+    git clone existing-repo
+    git annex init
+    git annex required . \"include=*.jpg\"
+
+Then I run fsck:
+
+    git annex fsck 
+
+And I get this output:
+
+    fsck viber/0.02.01.0e411306e0b3fc90.jpg (fixing link) ok
+    fsck viber/0.02.01.20943bf9c18e42.jpg (fixing link) ok
+    fsck viber/0.02.01.5126da68987493.jpg (fixing link) ok
+    ...
+
+I expected the fsck command to complain that the required jpg files are not present in the current repo and fail. However, that is not what I'm getting.
+
+Is this the expected behavior for a direct-mode repository? If not, is there something I missed?
+"""]]
diff --git a/doc/forum/offsite_repo/comment_4_2e3dfd605df004330396040737a7c5ee._comment b/doc/forum/offsite_repo/comment_4_2e3dfd605df004330396040737a7c5ee._comment
new file mode 100644
index 0000000000..15f8a3effc
--- /dev/null
+++ b/doc/forum/offsite_repo/comment_4_2e3dfd605df004330396040737a7c5ee._comment
@@ -0,0 +1,10 @@
+[[!comment format=mdwn
+ username="joey"
+ subject="""comment 4"""
+ date="2018-02-22T16:51:34Z"
+ content="""
+It seems likely that you have not upgraded git-annex to a version where
+fsck checks the required content yet, especially since that change has not
+yet made it into a release of git-annex. It will be in the next release, or
+you can download a daily build to get it now.
+"""]]
diff --git a/doc/forum/offsite_repo/comment_5_59a7718fde634c96dd781ed5fca1f61a._comment b/doc/forum/offsite_repo/comment_5_59a7718fde634c96dd781ed5fca1f61a._comment
new file mode 100644
index 0000000000..ee5afb9fe7
--- /dev/null
+++ b/doc/forum/offsite_repo/comment_5_59a7718fde634c96dd781ed5fca1f61a._comment
@@ -0,0 +1,9 @@
+[[!comment format=mdwn
+ username="ericm"
+ avatar="http://cdn.libravatar.org/avatar/67ca64c5a99fb142fc2e3916333881ca"
+ subject="comment 5"
+ date="2018-02-24T03:29:21Z"
+ content="""
+Ok I'll just wait for the next version. Btw I'm using the latest version: 6.20180112.
+Thanks!
+"""]]
diff --git a/doc/forum/one_annex_versus_many_annexes__63__.mdwn b/doc/forum/one_annex_versus_many_annexes__63__.mdwn
new file mode 100644
index 0000000000..cafe2aa9f8
--- /dev/null
+++ b/doc/forum/one_annex_versus_many_annexes__63__.mdwn
@@ -0,0 +1,10 @@
+I'm curious how other people are using git-annex:
+
+ * A single annex containing everything they want to keep track of (albeit with some files only available in some remotes).
+ * Several annexes for different purposes.
+
+I'm mostly asking because I don't want to get too far into using git-annex without thinking clearly about which is the most suitable choice for me.  At present I have things split into two annexes, one for things I really wouldn't want to lose (and some of which are somewhat private), and another for comparatively throwaway items.  However, given the possibility of setting different annex.numcopies on different file globs or directories, it seems like it might be kind of silly to make my life more complicated by having more than one annex.
+
+Primarily, I'm curious if there are some implications of the decision that I haven't considered.  It seems like an obvious point that one would want different annexes is if they're used in distinctly different security-level environments (e.g. I'm happy to copy my FLAC (music) files over to my work computer which lives at the office, but I wouldn't want to copy personal financial documents - or even the filenames - there.)
+
+-Mike
diff --git a/doc/forum/one_or_many_annexes__63__.mdwn b/doc/forum/one_or_many_annexes__63__.mdwn
new file mode 100644
index 0000000000..57b311c18e
--- /dev/null
+++ b/doc/forum/one_or_many_annexes__63__.mdwn
@@ -0,0 +1,7 @@
+Just a question about other peoples' annex usage.  I've got one uber-annex (in ~/annex of course) where I keep ALL THE THINGS, movies, music, pictures, whatever.
+
+It's very big, of course.  And I worry about doing anything unusual with it (like trying out direct mode) because, hey, that's all my stuff.
+
+Like, just now, I tried flipping it into direct mode, and a handful of files stayed symlinks, which weirded me out.  (Maybe because I had the assistant running while I switched it, and that caused chaos?)  Then I flipped it back to indirect, and a bunch of files were left un-tracked, as real files rather than links!  I kind of freaked out, and did a big "git annex add" to add them all back. I haven't left a bug report because it's not something I want to try again experimentally to reproduce, I just want my repo back, you know?
+
+Anyways, this kind of thing might not happen if I kept multiple small repos.  I was wondering if anybody else worked that way, and what advantages/disadvantages you've found compared to having one super-repo.
diff --git a/doc/forum/one_or_many_annexes__63__/comment_1_656d96011801d67a45b0b3bb3d70fa63._comment b/doc/forum/one_or_many_annexes__63__/comment_1_656d96011801d67a45b0b3bb3d70fa63._comment
new file mode 100644
index 0000000000..240f9868fd
--- /dev/null
+++ b/doc/forum/one_or_many_annexes__63__/comment_1_656d96011801d67a45b0b3bb3d70fa63._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="https://www.google.com/accounts/o8/id?id=AItOawn7gQ1zZDdWhXy9H51W2krZYShNmKL3qfM"
+ nickname="Karsten"
+ subject="comment 1"
+ date="2013-01-21T21:25:12Z"
+ content="""
+That's an interesting question. Personally, I have two annexes. One for my \"digital library\": movies, music, ebooks, downloads, everything I want to keep but was not created by me. I don't want to loose them because getting them back would be expensive and cumbersome. The other is my personal archive for Photos and Videos I've taken, which are valuable as they contain many memories which would be impossible to recreate if lost. With the first one, I'm experimenting quite a lot, with the second one I'm very conservative. Hope that helps.
+"""]]
diff --git a/doc/forum/optimising_lookupkey.mdwn b/doc/forum/optimising_lookupkey.mdwn
new file mode 100644
index 0000000000..5a7da73839
--- /dev/null
+++ b/doc/forum/optimising_lookupkey.mdwn
@@ -0,0 +1,13 @@
+to work around [[forum/original_filename_on_s3/]], i need to get the key from a file, and i'm not within the git-annex process. i know there's `git annex lookupkey $FILE`, but that incurs significant overhead because the whole git annex runtime needs to fire up. in my tests, this takes around 25ms on average.
+
+could i optimise this by simply doing a `readlink` call on the git checkout? it sure looks like `readlink | basename` is all I really need, and that can probably be done below 10ms (4ms in my tests). how reliable are those links anyways, and is that what lookupkey does?
+
+similarly, i wonder if it's safe to bypass git-annex and talk straight with git to extract location tracking? i can jump from 90ms to below 10ms for such requests if I turn `git annex find ` into the convoluted:
+
+
    +git annex lookupkey $file
+printf $key | md5sum
+git cat-file -p refs/heads/git-annex:$hash/${key}.log
+
    
+
+thanks. --[[anarcat]]
diff --git a/doc/forum/optimising_lookupkey/comment_1_e06db4754805c1e0ee298ecc676427d2._comment b/doc/forum/optimising_lookupkey/comment_1_e06db4754805c1e0ee298ecc676427d2._comment
new file mode 100644
index 0000000000..2b26d5e58b
--- /dev/null
+++ b/doc/forum/optimising_lookupkey/comment_1_e06db4754805c1e0ee298ecc676427d2._comment
@@ -0,0 +1,18 @@
+[[!comment format=mdwn
+ username="joey"
+ subject="""comment 1"""
+ date="2015-02-17T21:46:01Z"
+ content="""
+Yes, that's the same, except lookupkey only operates on files that are
+checked into git.
+
+(Also, lookupkey will work in a direct mode repo, while such a repo
+may not have a symlink to examine.)
+
+25ms doesn't seem bad for a "whole runtime" to fire up. :) I think most of
+the overhead probably involves reading the git config and running
+git-ls-files.
+
+Note that lookupkey can be passed a whole set of files, so you could avoid
+the startup overhead that way too.
+"""]]
diff --git a/doc/forum/optimising_lookupkey/comment_2_7dbfa3da6ae72a1f0669396664dd0c1a._comment b/doc/forum/optimising_lookupkey/comment_2_7dbfa3da6ae72a1f0669396664dd0c1a._comment
new file mode 100644
index 0000000000..8b3c973d48
--- /dev/null
+++ b/doc/forum/optimising_lookupkey/comment_2_7dbfa3da6ae72a1f0669396664dd0c1a._comment
@@ -0,0 +1,11 @@
+[[!comment format=mdwn
+ username="joey"
+ subject="""comment 2"""
+ date="2015-02-17T21:50:11Z"
+ content="""
+And yes, it's fine to bypass git-annex when querying git. 
+
+Or even when manipulating the git-annex branch, so long as you either
+delete or update .git/annex/index. git-annex is not intended to be magical,
+see [[internals]].
+"""]]
diff --git a/doc/forum/optimising_lookupkey/comment_3_2c895e4494cddab70ba761c6c48e7cf8._comment b/doc/forum/optimising_lookupkey/comment_3_2c895e4494cddab70ba761c6c48e7cf8._comment
new file mode 100644
index 0000000000..5ee6599586
--- /dev/null
+++ b/doc/forum/optimising_lookupkey/comment_3_2c895e4494cddab70ba761c6c48e7cf8._comment
@@ -0,0 +1,11 @@
+[[!comment format=mdwn
+ username="anarcat"
+ subject="thanks"
+ date="2015-02-18T21:53:52Z"
+ content="""
+great, thanks for the feedback!
+
+i agree that 25ms is quite fast to fire up a 52MB binary. :) i am just saying that if this is going to end up as part of building a webpage, i need something faster, or cache the results somewhere.
+
+duly noted for the other points, thanks again. i see a great intimate relationship building between `git cat-file` and me. ;)
+"""]]
diff --git a/doc/forum/original_filename_on_s3.mdwn b/doc/forum/original_filename_on_s3.mdwn
new file mode 100644
index 0000000000..7ecaab4e15
--- /dev/null
+++ b/doc/forum/original_filename_on_s3.mdwn
@@ -0,0 +1,5 @@
+I understand that [[special_remotes/S3/]] is seen as a "backend" storage mechanism, but since S3 files are available directly on the web, it would be nice to have the real filenames up there.
+
+Is there a way to set that up? I know about [[tips/publishing_your_files_to_the_public/]], but it assumes you have a local git repo with all the data in the first place, something which may not be available...
+
+my use case is that we have ~1TB of files already stored in S3 under specific filenames, and those filenames are how the files are accessed on the main website. changing all those filenames would be a significant burden... i'm not even sure this can be done cheaply on S3 in the first place. --[[anarcat]]
diff --git a/doc/forum/original_filename_on_s3/comment_1_870f3f01f95b5878d483ffedd2ae2698._comment b/doc/forum/original_filename_on_s3/comment_1_870f3f01f95b5878d483ffedd2ae2698._comment
new file mode 100644
index 0000000000..165b7f8f65
--- /dev/null
+++ b/doc/forum/original_filename_on_s3/comment_1_870f3f01f95b5878d483ffedd2ae2698._comment
@@ -0,0 +1,7 @@
+[[!comment format=mdwn
+ username="https://id.koumbit.net/anarcat"
+ subject="dupe?"
+ date="2015-03-07T02:06:06Z"
+ content="""
+see also [[todo/Facilitate_public_pretty_S3_URLs]]
+"""]]
diff --git a/doc/forum/original_filename_on_s3/comment_2_2a0d5d77219ae2410090eb1811ee1fec._comment b/doc/forum/original_filename_on_s3/comment_2_2a0d5d77219ae2410090eb1811ee1fec._comment
new file mode 100644
index 0000000000..bba4a63555
--- /dev/null
+++ b/doc/forum/original_filename_on_s3/comment_2_2a0d5d77219ae2410090eb1811ee1fec._comment
@@ -0,0 +1,13 @@
+[[!comment format=mdwn
+ username="joey"
+ subject="""comment 2"""
+ date="2016-10-31T20:10:22Z"
+ content="""
+What real filenames? A git-annex special remote is a key/value store, where
+any key can correspond to zero or any number of filenames in the working
+tree. 
+
+What you're doing with S3 is using it to publish some files over the web.
+Fine. That's not a S3 special remote, so don't use it that way, just `git
+annex addurl` the files from S3.
+"""]]
diff --git a/doc/forum/original_filename_on_s3/comment_3_0aea0eef336dd648013a9cf7789fc445._comment b/doc/forum/original_filename_on_s3/comment_3_0aea0eef336dd648013a9cf7789fc445._comment
new file mode 100644
index 0000000000..1d75687304
--- /dev/null
+++ b/doc/forum/original_filename_on_s3/comment_3_0aea0eef336dd648013a9cf7789fc445._comment
@@ -0,0 +1,10 @@
+[[!comment format=mdwn
+ username="joey"
+ subject="""comment 3"""
+ date="2017-09-07T20:13:23Z"
+ content="""
+The new `git-annex export` feature allows you to export a tree
+to a special remote, with the origiginal filenames being visible there.
+
+S3 should support it soon.
+"""]]
diff --git a/doc/forum/overmounting_repository_at_home.mdwn b/doc/forum/overmounting_repository_at_home.mdwn
new file mode 100644
index 0000000000..2bac1994f0
--- /dev/null
+++ b/doc/forum/overmounting_repository_at_home.mdwn
@@ -0,0 +1,12 @@
+Consider the following scenario:
+
+- A Server which holds the full repository
+- A Laptop with a small disk, cloned the repository
+
+Now when i am online i'd just like to mount the repository from the server on over to the client, shadowing the local repository. But when offline and the server is not
+mounted the local repository takes place.
+
+The Question is now what would be a viable concept to get this right. Especially will the assistant become confused when it runs on the server side and locally on the laptop
+while the mount is in effect. Would it be sensible not to mount the whole repository tree but only parts like `.git/annex` or `.git/annex/objects`?
+
+Not tried this yet, but I am wondering whats the most viable approach would be.
diff --git a/doc/forum/overmounting_repository_at_home/comment_1_399ac5014c489698e1e45deec4db7311._comment b/doc/forum/overmounting_repository_at_home/comment_1_399ac5014c489698e1e45deec4db7311._comment
new file mode 100644
index 0000000000..90ce2ae403
--- /dev/null
+++ b/doc/forum/overmounting_repository_at_home/comment_1_399ac5014c489698e1e45deec4db7311._comment
@@ -0,0 +1,10 @@
+[[!comment format=mdwn
+ username="https://openid.stackexchange.com/user/e65e6d0e-58ba-41de-84cc-1f2ba54cf574"
+ nickname="Mica Semrick"
+ subject="sounds like a bad idea... but..."
+ date="2014-06-24T04:18:51Z"
+ content="""
+From the few details you give, this sounds like a bad idea. git-annex tells different repos apart by using a UUID. You can see this UUID in `.git/config`.
+
+But if you are really insistent on doing this, you could always system link the folder to the spot you want it, then write a script that mounts the server in an arbitrary location and redoes the system link. I'd try this on a copy of the data or with some data that doesn't matter before going for it.
+"""]]
diff --git a/doc/forum/overmounting_repository_at_home/comment_2_d006d89ba204568cdee0731b6251ec1a._comment b/doc/forum/overmounting_repository_at_home/comment_2_d006d89ba204568cdee0731b6251ec1a._comment
new file mode 100644
index 0000000000..2413948b7a
--- /dev/null
+++ b/doc/forum/overmounting_repository_at_home/comment_2_d006d89ba204568cdee0731b6251ec1a._comment
@@ -0,0 +1,11 @@
+[[!comment format=mdwn
+ username="cehteh"
+ ip="217.8.62.137"
+ subject="comment 2"
+ date="2014-06-24T14:03:01Z"
+ content="""
+I know that this is a bad idea right now when done in a naive approach, I am wondering if git-annex could be made aware of this case and how it could be implemented.
+A immature idea for example would be only to overmount the .git/annex/objects folder and let the git-annex check if the objects lay on a different device, if so, refuse to do operation on the object store.
+
+I started this thread to figure out if there is some interest in such functionality and if so, polishing it out to see how it could be done. 
+"""]]
diff --git a/doc/forum/overmounting_repository_at_home/comment_3_3734b50c37cbec675813cbeca7bf4ce9._comment b/doc/forum/overmounting_repository_at_home/comment_3_3734b50c37cbec675813cbeca7bf4ce9._comment
new file mode 100644
index 0000000000..029e9eba09
--- /dev/null
+++ b/doc/forum/overmounting_repository_at_home/comment_3_3734b50c37cbec675813cbeca7bf4ce9._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="http://ypid.wordpress.com/"
+ ip="213.153.84.215"
+ subject="Already on todo list"
+ date="2014-06-24T16:58:05Z"
+ content="""
+As is appears to me, the feature you are suggesting is already on \"the todo list\" as [[union_mounting|todo/union_mounting]]. I would also really appreciate this feature …
+"""]]
diff --git a/doc/forum/partial_annex_repository.mdwn b/doc/forum/partial_annex_repository.mdwn
new file mode 100644
index 0000000000..d83eac1a77
--- /dev/null
+++ b/doc/forum/partial_annex_repository.mdwn
@@ -0,0 +1,14 @@
+After I created a git repository, I added annex to it.
+
+Now, if I add a new file with 'git add', it goes into the annex.   However previously existing files are not in the annex.
+
+What are the implications of this?   Should I care?
+
+FWIW, most of the files are small, so I see no reason for the annex for them.
+
+How do I add a new file and not put it in the annex?
+
+How to I switch a file from one to the other?
+
+Thanks.
+
diff --git a/doc/forum/partial_annex_repository/comment_1_5e594914724a04e21087e1e4e57c2d30._comment b/doc/forum/partial_annex_repository/comment_1_5e594914724a04e21087e1e4e57c2d30._comment
new file mode 100644
index 0000000000..a92c9e6a98
--- /dev/null
+++ b/doc/forum/partial_annex_repository/comment_1_5e594914724a04e21087e1e4e57c2d30._comment
@@ -0,0 +1,14 @@
+[[!comment format=mdwn
+ username="git-starter"
+ subject="clarification"
+ date="2016-09-13T20:27:22Z"
+ content="""
+My previous posting looks like I'm not aware of largefiles, which would mostly answer my question.    My problem was that, after reading some doc that might not have been fully updated for v6, I got confused about the difference between an unlocked file in the annex, and a file not in the annex.    
+
+In any case, since I added some small files before using largefiles, those small files were put in the annex.   How do I change them to be in the git repository not in the annex?
+
+Thanks.
+
+
+
+"""]]
diff --git a/doc/forum/partial_annex_repository/comment_2_e6a9584bacc3ec98c4757c82a195b414._comment b/doc/forum/partial_annex_repository/comment_2_e6a9584bacc3ec98c4757c82a195b414._comment
new file mode 100644
index 0000000000..a7c0d1ec02
--- /dev/null
+++ b/doc/forum/partial_annex_repository/comment_2_e6a9584bacc3ec98c4757c82a195b414._comment
@@ -0,0 +1,15 @@
+[[!comment format=mdwn
+ username="joey"
+ subject="""comment 2"""
+ date="2016-09-21T16:54:24Z"
+ content="""
+Right, it's a feature of v6 mode that `git add` adds the file to the annex,
+unless annex.largefiles is configured to make it be added directly to the
+git repository.
+
+To switch your small files that are currently annexed to be added
+directly to git, the easiest way is:
+
+	git annex unannex $file
+	git -c 'annex.largefiles=exclude=*' add $file
+"""]]
diff --git a/doc/forum/partial_synchronisation._android_phone.mdwn b/doc/forum/partial_synchronisation._android_phone.mdwn
new file mode 100644
index 0000000000..7abfb24cd7
--- /dev/null
+++ b/doc/forum/partial_synchronisation._android_phone.mdwn
@@ -0,0 +1,7 @@
+hi
+
+i have a repository that is 30 gb large. i would like to sync some content onto my android phone. take mp3's for example. i would like to see whole content on android but my phone does not have that much flash space. i would like to manualy select what folders, files will be copied onto phone.
+
+this use case may not even refer strictly to android devices but even pc's. for exaple i have small ssd drive on laptop. whole contentis kept on some hudge  raid array. i will see whole directory structure but whenever i want a file i will just shedule it for download.
+
+is it possible ? or maybe i just can not find the answer on project page ?
diff --git a/doc/forum/partial_synchronisation._android_phone/comment_1_2a48569277945a9c334bdfc51f8fd01f._comment b/doc/forum/partial_synchronisation._android_phone/comment_1_2a48569277945a9c334bdfc51f8fd01f._comment
new file mode 100644
index 0000000000..b9c28a1229
--- /dev/null
+++ b/doc/forum/partial_synchronisation._android_phone/comment_1_2a48569277945a9c334bdfc51f8fd01f._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="http://joeyh.name/"
+ ip="209.250.56.227"
+ subject="comment 1"
+ date="2013-12-29T21:57:05Z"
+ content="""
+You can either put a repository in manual mode, which requires you run `git annex get` etc to get just the files you want, or you can find or write a [[preferred_content]] expression that matches only the files you want in a repository, and then the assistant will make it contain only those.
+"""]]
diff --git a/doc/forum/partial_synchronisation._android_phone/comment_2_550ae91c8a1fe060368c4682d37514b6._comment b/doc/forum/partial_synchronisation._android_phone/comment_2_550ae91c8a1fe060368c4682d37514b6._comment
new file mode 100644
index 0000000000..e549bf7cc9
--- /dev/null
+++ b/doc/forum/partial_synchronisation._android_phone/comment_2_550ae91c8a1fe060368c4682d37514b6._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="https://www.google.com/accounts/o8/id?id=AItOawkNE-H4vEcbcGndxq5daT8qUb7yIf7r1OE"
+ nickname="Łukasz"
+ subject="comment 2"
+ date="2013-12-30T22:13:15Z"
+ content="""
+thx for reply
+"""]]
diff --git a/doc/forum/performance_and_multiple_replication_problems.mdwn b/doc/forum/performance_and_multiple_replication_problems.mdwn
new file mode 100644
index 0000000000..7f2f90fddf
--- /dev/null
+++ b/doc/forum/performance_and_multiple_replication_problems.mdwn
@@ -0,0 +1,17 @@
+Hi,
+
+I just was setting um my git-annex repository and started to sync my whole stuff in it.
+
+Background: I have choosen git-annex to sync my whole stuff (pictures, mp3s, documents, etc) between my pc, notebook and a home-server
+
+My Problems:
+1) When I'm starting the git-annex deamon, the "Performing startup scan" message occurs for hours
+2) git-annex synchronizes folders from the server which already on my pc, and that every time I restart the deamon on client
+
+My Questions:
+For 1) is git-annex when running one repository suitable to manage > 100gb and > 50000 files?
+For 2) do I have to wait until every tasks are completed (everything is committed) to get rid of multiple downloads of the same folders/files
+3) what is the best schema to sync between >2 devices should I use a Mesh or Star Schema (where my server is in the middle)
+
+Thank You in advance!
+Regards J
diff --git a/doc/forum/performance_and_multiple_replication_problems/comment_1_a2cdf1a4840f099f6bc941fd8de966c7._comment b/doc/forum/performance_and_multiple_replication_problems/comment_1_a2cdf1a4840f099f6bc941fd8de966c7._comment
new file mode 100644
index 0000000000..aab0ec17fd
--- /dev/null
+++ b/doc/forum/performance_and_multiple_replication_problems/comment_1_a2cdf1a4840f099f6bc941fd8de966c7._comment
@@ -0,0 +1,16 @@
+[[!comment format=mdwn
+ username="http://joeyh.name/"
+ nickname="joey"
+ subject="comment 1"
+ date="2013-06-11T15:06:58Z"
+ content="""
+It sounds like you are using git-annex to sync a *lot* of files. The startup scan is a basic check that there are no new, deleted, or modified files since the last time it ran. This requires a little work, like statting the files, but won't take long for reasonable numbers of files. 
+
+The size of files doesn't affect the length of the scan. I have a repository with on the order of 50k files, and the startup scan takes only a few minutes. One thing that could matter is that my repository is in indirect mode. Direct mode is less efficient. You could try to switch your repository to indirect mode: `git annex indirect` (you can always switch back: `git annex direct`)
+
+It would be possible to disable that scan, but at the expense of not being able to sync changes made while git-annex is not running. There's also a trick you can use: Start the assistant running in a subdirectory and it will only scan that subdirectory (it will only notice new files in that subdirectory too..)
+
+I don't quite understand what you mean with problem #2. If files were repeatedly being uploaded or downloaded, that have already been sent, that would be a bug. Please file a bug report with full debug logs if that is the case.
+
+Which topology is best? I think the best way is to start with the one you like, and if it doesn't work well, add more links between repositories. A star topology will certainly work ok. A mesh can work ok but can be hard to maintain.
+"""]]
diff --git a/doc/forum/performance_and_multiple_replication_problems/comment_2_e65b360706c66ede6e0e841b2ebbbfbc._comment b/doc/forum/performance_and_multiple_replication_problems/comment_2_e65b360706c66ede6e0e841b2ebbbfbc._comment
new file mode 100644
index 0000000000..3ce2447bc7
--- /dev/null
+++ b/doc/forum/performance_and_multiple_replication_problems/comment_2_e65b360706c66ede6e0e841b2ebbbfbc._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="https://me.yahoo.com/a/bBy7WkgQicYHIiiyj.Vm0TcMbxi2quzbPFef#6f9f7"
+ nickname="Frederik Vanrenterghem"
+ subject="comment 2"
+ date="2013-06-12T00:47:21Z"
+ content="""
+I have witnessed that second problem as well, to the point where I've stopped autostarting (and hence using) git annex for the moment. I'll try to get some debugging data.
+"""]]
diff --git a/doc/forum/performance_and_multiple_replication_problems/comment_3_ad7cb4c510e2ab26959ea7cb40a43fef._comment b/doc/forum/performance_and_multiple_replication_problems/comment_3_ad7cb4c510e2ab26959ea7cb40a43fef._comment
new file mode 100644
index 0000000000..6e4e1b1c63
--- /dev/null
+++ b/doc/forum/performance_and_multiple_replication_problems/comment_3_ad7cb4c510e2ab26959ea7cb40a43fef._comment
@@ -0,0 +1,14 @@
+[[!comment format=mdwn
+ username="https://www.google.com/accounts/o8/id?id=AItOawnNqLKszWk9EoD4CDCqNXJRIklKFBCN1Ao"
+ nickname="maurizio"
+ subject="the startup check is not a small issue"
+ date="2014-02-25T11:37:15Z"
+ content="""
+I would like to add that this startup check has probably been a blocker for my use case for a long long time. I tried to use git-annex to synchronize a huge number of files, most of them never changing. My plan was to have a few tens of GB of data which more or less never change in an archive directory and then add from time to time new data (by batches of a few hundreds of files, each of them not necessarily very large) to the annex. Once this new data has been processed or otherwise become less immediately useful, it would be shifted to the archive. It would have been very useful to have such a setup, because the amount of data is too large to be replicated everywhere, especially on a laptop. After finding this post I finally understand that the seemingly never ending \"performing startup scan\" that I observed are probably not due to the assistant somehow hanging, contrary to what I thought. It seems it is just normal operation. The problem is that this normal operation makes it unusable for the use case I was considering, since it does not make much sense to have git-annex scanning about 10^6 files or links on every boot of a laptop. On my workstation this \"startup scan\" has now been running for close to one hour now and is not finished yet, this is not thinkable on laptop boot. 
+
+Maybe an analysis of how well git-annex operation scales with number of files should be part of the documentation, since \"large files\" is not the only issue when trying to sync different computers. One finds references to \"very large number of files\" about annex.queuesize, but \"very large\" has no clear meaning. One also finds a reference to \"1 million files\" being a bit of a git limitation on comments of a bug report . 
+
+Orders of magnitude of the number of files that git-annex is supposed to be able to handle would be very useful. 
+
+
+"""]]
diff --git a/doc/forum/performance_and_multiple_replication_problems/comment_4_23a6dc7ea569944ca55bd21851dd770d._comment b/doc/forum/performance_and_multiple_replication_problems/comment_4_23a6dc7ea569944ca55bd21851dd770d._comment
new file mode 100644
index 0000000000..77fe564fc8
--- /dev/null
+++ b/doc/forum/performance_and_multiple_replication_problems/comment_4_23a6dc7ea569944ca55bd21851dd770d._comment
@@ -0,0 +1,14 @@
+[[!comment format=mdwn
+ username="http://joeyh.name/"
+ ip="209.250.56.146"
+ subject="comment 4"
+ date="2014-03-05T22:06:00Z"
+ content="""
+@maurizio, that's a good motivating example.
+
+So, I have made `git config annex.startupscan false` disable the startup scan (except for a minor tree walk that needs to be done to get inotify working).
+
+Of course, if you set that, the assistant won't notice any changes that are made when it's not running. It would work well to set it if the assistant is started at boot and left running until shutdown.
+
+My goal for git-annex is for it to scale to how ever many files git scales to. So I try to always make git be the limiting factor. I feel that git scales fairly well into the 100k file range.
+"""]]
diff --git a/doc/forum/performance_and_multiple_replication_problems/comment_5_8df6cc8b72e0e78c7380f7d471124498._comment b/doc/forum/performance_and_multiple_replication_problems/comment_5_8df6cc8b72e0e78c7380f7d471124498._comment
new file mode 100644
index 0000000000..f881942f90
--- /dev/null
+++ b/doc/forum/performance_and_multiple_replication_problems/comment_5_8df6cc8b72e0e78c7380f7d471124498._comment
@@ -0,0 +1,12 @@
+[[!comment format=mdwn
+ username="zardoz"
+ ip="134.147.14.84"
+ subject="comment 5"
+ date="2014-07-08T12:26:58Z"
+ content="""
+Something to keep in mind for some point in the future: btrfs supports an efficient method by which one can inquire exactly which files have changed since a specified point in time, where the point in time is measured by a unit called «generation».
+
+A program can request the current «generation» index, and later ask btrfs which files have changed after that index, without needing to walk the file-system. This is currently implemented in the user-space program «btrfs filesystem find-new». While it’s called «find-new», this not only find newly created files, but also changed files.
+
+Currently, it seems one needs to be super-user to use «find-new», because it lists changed files for complete subvolumes, but since generations are stored per-file, in the future there will likely be a user-space method for regular users.
+"""]]
diff --git a/doc/forum/performance_for_FTP_site_mirroring.mdwn b/doc/forum/performance_for_FTP_site_mirroring.mdwn
new file mode 100644
index 0000000000..0bee18b790
--- /dev/null
+++ b/doc/forum/performance_for_FTP_site_mirroring.mdwn
@@ -0,0 +1,23 @@
+Hello --
+
+I want to use git (and git-annex) to take point-in-time snapshots of an FTP site.  Think "web.archive.org" but for an FTP site.  I'm using LFTP to mirror the site into a directory, which I then check into git.  This way I can go back in time to see what the state of the site was in the past.
+
+The site itself is currently about 10K files and about 30GB.  The files themselves are mostly zip files, as well as some xml files.  I expect files to not change much, and when they do I expect their sizes and modification times to change.  I'm using v6 mode with unlocked files.
+
+Here are my questions:
+
+1. I found that plain "git status" and "git diff" (not using git-annex) is quite slow.  i assume this is because git is computing checksums of all the files?
+
+2. On the assumption that the problem is that git is computing checksums, it seems like the appropriate way to get more performance is to tell git-annex to ignore checksum, i.e. use the WORM backend.  Is this correct?  I found that git status with the WORM backend set is fast.
+
+3. What is the proper way to set a backend globally?  'git annex init' has an option "--backend" but it doesn't seem to have any effect.  The correct way to set this globally is "git config annex.backends WORM", yes?
+
+4. Since I'm using another program to mirror the site, it appears I cannot use "locked" mode, as the mirroring program (lftp) will see that git-annex has replaced everything with symlinks and re-download the files.  Correct?  Therefore I'm using plain "git add" instead of "git annex add."
+
+5. Another reason why I appear to be forced to use "unlocked" mode is that, as part of the mirroring, the directory permissions are set to match the site, which are not writable.  git-annex appears to be unable to move the files that are inside of directories without write permissions.  Note that I am the owner of the local files/directories, and lftp happily adds and modifies files insides of these unwritable directories just fine, presumably by temporarily changing the permissions.  Is this correct?  Should I submit a feature request here?
+
+5. Although I am using WORM and unlocked mode, I found the initial "git add" and "git commit" of the 10K / 30GB of files to be pretty slow.  It takes on the order of 30 minutes for the add and an hour for the commit.  I didn't see a ton of either CPU or I/O activity.  A subsequent update of about 600 files is taking a long time to add (running 'git add -A'.)  I assume commit time will also be long.  Is this to be expected?  I would have hoped that the WORM backend prevents git from needing to actually read the files for a checksum.
+
+6. I understand that "thin = false" will lead to data duplication.  I assume this will make the initial commits slower.  Are there other performance implications of changing the thin setting?
+
+Thank you for creating a great tool.
diff --git a/doc/forum/performance_for_FTP_site_mirroring/comment_1_7648fced001ce1c34726e913e06273e9._comment b/doc/forum/performance_for_FTP_site_mirroring/comment_1_7648fced001ce1c34726e913e06273e9._comment
new file mode 100644
index 0000000000..9e886e44ec
--- /dev/null
+++ b/doc/forum/performance_for_FTP_site_mirroring/comment_1_7648fced001ce1c34726e913e06273e9._comment
@@ -0,0 +1,32 @@
+[[!comment format=mdwn
+ username="joey"
+ subject="""comment 1"""
+ date="2016-05-12T18:50:10Z"
+ content="""
+Are you using v6 mode? I'd have two entirely different sets of anwsers to
+all of these questions depending on whether you're using v6 mode or not.
+
+Since you mentioned annex.thin, I'm going to guess v6 mode...
+
+1. `git status` will be slow in v6 mode if files have been dropped or
+   git's index has otherwise gotten out of sync. This is the main reason v6
+   mode is still considered experiemental. It's being worked on.
+2. WORM backend won't help with 1. The only thing WORM is going to speed
+   up is avoiding checksumming when adding files. At the cost that two
+   versions of a file with the same size and different content can't be
+   differentiated.
+3. Yes, git config annex.backends WORM would set the WORM backend by default.
+   I don't think that WORM is a good fit for your use case though.
+4. `git add` is much slower than `git-annex add`, because the former has to run
+   git-annex once per file added. Instead, run: `git config annex.addunlocked true`
+   and then `git annex add` will add the files unlocked.
+5. We could think about making `git annex add` fiddle with directory perms
+   to allow replacing a file with the annex symlink. But, what happens
+   if it loses power before it can fix the perms back to original mode?
+   Etc. Perhaps there's a way to make lftp not remove write perms to start
+   with. But I think you're going to need to use unlocked files anyway,
+   otherwise lftp mirror is probably going to see the annexed symlink as
+   different than the remote file, and replace it.
+6. See #4.
+7. No, annex.thin does not make anything slower AFAIK.
+"""]]
diff --git a/doc/forum/performance_for_FTP_site_mirroring/comment_2_fa2b7400c85356d4de6da8459a5c3b50._comment b/doc/forum/performance_for_FTP_site_mirroring/comment_2_fa2b7400c85356d4de6da8459a5c3b50._comment
new file mode 100644
index 0000000000..e5d44d0f6d
--- /dev/null
+++ b/doc/forum/performance_for_FTP_site_mirroring/comment_2_fa2b7400c85356d4de6da8459a5c3b50._comment
@@ -0,0 +1,7 @@
+[[!comment format=mdwn
+ username="midfield"
+ subject="v6 mode"
+ date="2016-05-12T20:19:15Z"
+ content="""
+i am using v6 mode, edited original question to reflect this.
+"""]]
diff --git a/doc/forum/performance_for_FTP_site_mirroring/comment_3_cb4d7b283f20b43283036f20ebc648ec._comment b/doc/forum/performance_for_FTP_site_mirroring/comment_3_cb4d7b283f20b43283036f20ebc648ec._comment
new file mode 100644
index 0000000000..36b1595dd0
--- /dev/null
+++ b/doc/forum/performance_for_FTP_site_mirroring/comment_3_cb4d7b283f20b43283036f20ebc648ec._comment
@@ -0,0 +1,7 @@
+[[!comment format=mdwn
+ username="midfield"
+ subject="timings"
+ date="2016-05-15T19:58:37Z"
+ content="""
+for what it's worth, even with your suggestions i was unable to make git commit work quickly (it takes over an hour for the initial commit.  it only takes 4 minutes to copy the files themselves.)  thank you for the help though.
+"""]]
diff --git a/doc/forum/performance_improvement__58___git_on_ssd__44___annex_on_spindle_disk.mdwn b/doc/forum/performance_improvement__58___git_on_ssd__44___annex_on_spindle_disk.mdwn
new file mode 100644
index 0000000000..f70c127025
--- /dev/null
+++ b/doc/forum/performance_improvement__58___git_on_ssd__44___annex_on_spindle_disk.mdwn
@@ -0,0 +1,12 @@
+This works with bind-mount, I might try with softlinks as well.
+
+Going through git's data on push/pull can take ages on a spindle disk even
+if the repo is rather small in size. This is especially true if you are
+used to ssd speeds, but ssd storage is expensive. Storing the annex objects
+on a cheap spindle disk and everything else on a ssd makes things a _lot_
+faster.
+
+> Update: git-annex supports `.git/annex/` being moved to a different disk
+> than the rest of the repisitory, but does *not* support individual 
+> subdirectories, like `.git/annex/objects/` being on a different disk
+> than the main `.git/annex/` directory. --[[Joey]]
diff --git a/doc/forum/performance_improvement__58___git_on_ssd__44___annex_on_spindle_disk/comment_1_b3f22f9be02bc4f2d5a121db3d753ff5._comment b/doc/forum/performance_improvement__58___git_on_ssd__44___annex_on_spindle_disk/comment_1_b3f22f9be02bc4f2d5a121db3d753ff5._comment
new file mode 100644
index 0000000000..124993bcf1
--- /dev/null
+++ b/doc/forum/performance_improvement__58___git_on_ssd__44___annex_on_spindle_disk/comment_1_b3f22f9be02bc4f2d5a121db3d753ff5._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="http://joey.kitenet.net/"
+ nickname="joey"
+ subject="comment 1"
+ date="2011-04-02T17:48:29Z"
+ content="""
+Either option should work fine, but git gc --aggressive will probably avoid most of git's seeking.
+"""]]
diff --git a/doc/forum/performance_improvement__58___git_on_ssd__44___annex_on_spindle_disk/comment_2_f94abce32ef818176b42a3cc860691ae._comment b/doc/forum/performance_improvement__58___git_on_ssd__44___annex_on_spindle_disk/comment_2_f94abce32ef818176b42a3cc860691ae._comment
new file mode 100644
index 0000000000..eddc8c6315
--- /dev/null
+++ b/doc/forum/performance_improvement__58___git_on_ssd__44___annex_on_spindle_disk/comment_2_f94abce32ef818176b42a3cc860691ae._comment
@@ -0,0 +1,20 @@
+[[!comment format=mdwn
+ username="https://www.google.com/accounts/o8/id?id=AItOawl9sYlePmv1xK-VvjBdN-5doOa_Xw-jH4U"
+ nickname="Richard"
+ subject="comment 2"
+ date="2011-04-02T21:34:24Z"
+ content="""
+I'll give it a try as soon as I get rid of this:
+
+    % git annex fsck
+fatal: index file smaller than expected
+fatal: index file smaller than expected
+    % git status
+fatal: index file smaller than expected
+    %    
+
+And no, I am not sure where that is coming from all of a sudden... (it might have to do with a hard lockup of the whole system due to a faulty hdd I tested, but I didn't do anything to it for ages before that lock-up. So meh. Also, this is prolly off topic in here)
+
+
+Richard
+"""]]
diff --git a/doc/forum/performance_improvement__58___git_on_ssd__44___annex_on_spindle_disk/comment_3_0c8e77fe248e00bd990d568623e5a5c9._comment b/doc/forum/performance_improvement__58___git_on_ssd__44___annex_on_spindle_disk/comment_3_0c8e77fe248e00bd990d568623e5a5c9._comment
new file mode 100644
index 0000000000..fc29236c6d
--- /dev/null
+++ b/doc/forum/performance_improvement__58___git_on_ssd__44___annex_on_spindle_disk/comment_3_0c8e77fe248e00bd990d568623e5a5c9._comment
@@ -0,0 +1,10 @@
+[[!comment format=mdwn
+ username="http://joey.kitenet.net/"
+ nickname="joey"
+ subject="comment 3"
+ date="2011-04-03T01:48:57Z"
+ content="""
+For future reference, git can recover from a corrupted index file with `rm .git/index; git reset --mixed`.
+
+Of course, you lose any staged changes that were in the old index file, and may need to re-stage some files.
+"""]]
diff --git a/doc/forum/performance_improvement__58___git_on_ssd__44___annex_on_spindle_disk/comment_4_4b7e8f9521d61900d9ad418e74808ffb._comment b/doc/forum/performance_improvement__58___git_on_ssd__44___annex_on_spindle_disk/comment_4_4b7e8f9521d61900d9ad418e74808ffb._comment
new file mode 100644
index 0000000000..ec0f88d13c
--- /dev/null
+++ b/doc/forum/performance_improvement__58___git_on_ssd__44___annex_on_spindle_disk/comment_4_4b7e8f9521d61900d9ad418e74808ffb._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="https://www.google.com/accounts/o8/id?id=AItOawl9sYlePmv1xK-VvjBdN-5doOa_Xw-jH4U"
+ nickname="Richard"
+ subject="comment 4"
+ date="2011-04-03T09:03:22Z"
+ content="""
+Thanks a lot. I tried various howtos around the net, but none of them worked; yours did. (I tried it in one of the copies of the broken repo which I keep around for obvious reasons).
+"""]]
diff --git a/doc/forum/performance_improvement__58___git_on_ssd__44___annex_on_spindle_disk/comment_5_7abbbe7db3988a2d239d11b0a4c597e7._comment b/doc/forum/performance_improvement__58___git_on_ssd__44___annex_on_spindle_disk/comment_5_7abbbe7db3988a2d239d11b0a4c597e7._comment
new file mode 100644
index 0000000000..651fc81448
--- /dev/null
+++ b/doc/forum/performance_improvement__58___git_on_ssd__44___annex_on_spindle_disk/comment_5_7abbbe7db3988a2d239d11b0a4c597e7._comment
@@ -0,0 +1,10 @@
+[[!comment format=mdwn
+ username="stoile"
+ ip="2a01:198:242:0:219:66ff:fef2:c021"
+ subject="Error with .git/annex as a symlink"
+ date="2013-11-16T12:20:58Z"
+ content="""
+I had my git annex directory on a ssd and .git/annex on a hdd (using a symlink). I got errors when adding files because a move failed. I forgot to copy the error message. When I had .git/annex/objects on a hdd, everything worked as expected, there were no problems.
+
+So what exactly will go wrong if I only have this one subdirectory on another filesystem?
+"""]]
diff --git a/doc/forum/performance_improvement__58___git_on_ssd__44___annex_on_spindle_disk/comment_6_46bd45fdc25d9c583f4ebe3a9730ab9f._comment b/doc/forum/performance_improvement__58___git_on_ssd__44___annex_on_spindle_disk/comment_6_46bd45fdc25d9c583f4ebe3a9730ab9f._comment
new file mode 100644
index 0000000000..0c0e3e4957
--- /dev/null
+++ b/doc/forum/performance_improvement__58___git_on_ssd__44___annex_on_spindle_disk/comment_6_46bd45fdc25d9c583f4ebe3a9730ab9f._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="stoile"
+ ip="2a01:198:242:0:219:66ff:fef2:c021"
+ subject="Bug report answering my question"
+ date="2013-11-17T10:28:40Z"
+ content="""
+My question was answered in .
+"""]]
diff --git a/doc/forum/pgp_issue_in_log_file_after_upgrade_to_5.20140517.mdwn b/doc/forum/pgp_issue_in_log_file_after_upgrade_to_5.20140517.mdwn
new file mode 100644
index 0000000000..0d226d5df8
--- /dev/null
+++ b/doc/forum/pgp_issue_in_log_file_after_upgrade_to_5.20140517.mdwn
@@ -0,0 +1,13 @@
+    [2014-05-18 22:38:21 SAST] main: starting assistant version 5.20140517-g0aed6d9
+    Already up-to-date.
+    
+    (scanning...) [2014-05-18 22:38:22 SAST] Watcher: Performing startup scan
+    gpg: WARNING: unsafe permissions on homedir `/var/folders/m6/zkd11n111m38ff37zbtgq0lr0000gp/T/git-annex-gpg.tmp.0'
+    gpg: keyring `/Applications/git-annex.app/Contents/MacOS/trustedkeys.gpg' created
+    gpg: Signature made Sun May 18 04:16:11 2014 SAST using DSA key ID XXXXXXXX
+    gpg: Can't check signature: public key not found
+    (started...) 
+
+And still facing:
+
+    git-annex-remote-skydrive is not installed in PATH (/Applications/git-annex.app/Contents/MacOS/bundle:/Applications/git-annex.app/Contents/MacOS/bundle:/usr/bin:/bin:/usr/sbin:/sbin)
diff --git a/doc/forum/pgp_issue_in_log_file_after_upgrade_to_5.20140517/comment_1_807d7da99f732f2fa5f9d3cb1ba9f1a1._comment b/doc/forum/pgp_issue_in_log_file_after_upgrade_to_5.20140517/comment_1_807d7da99f732f2fa5f9d3cb1ba9f1a1._comment
new file mode 100644
index 0000000000..0cd6a21d98
--- /dev/null
+++ b/doc/forum/pgp_issue_in_log_file_after_upgrade_to_5.20140517/comment_1_807d7da99f732f2fa5f9d3cb1ba9f1a1._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="https://www.google.com/accounts/o8/id?id=AItOawkAUMhKOSkh9JaBA6xst3XxQIIsDEq5Zd4"
+ nickname="Ovidiu"
+ subject="comment 1"
+ date="2014-06-21T13:07:51Z"
+ content="""
+What exactly are safe permissions?
+"""]]
diff --git a/doc/forum/pgp_issue_in_log_file_after_upgrade_to_5.20140517/comment_2_92a7509fc42ab2347d57f080081d14b5._comment b/doc/forum/pgp_issue_in_log_file_after_upgrade_to_5.20140517/comment_2_92a7509fc42ab2347d57f080081d14b5._comment
new file mode 100644
index 0000000000..4767ae75bb
--- /dev/null
+++ b/doc/forum/pgp_issue_in_log_file_after_upgrade_to_5.20140517/comment_2_92a7509fc42ab2347d57f080081d14b5._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="http://joeyh.name/"
+ ip="209.250.56.55"
+ subject="comment 2"
+ date="2014-07-03T19:49:03Z"
+ content="""
+That error message seems to occur when git-annex is trying to upgrade itself to a newer version, but for some reason the temporary directory it uses for gpg allows some other user to write to it. Perhaps because of a wacky umask, I don't know. AFAICS it's only a warning and the upgrade still works, but I have made git-annex force a sane umask when creating this temp directory.
+"""]]
diff --git a/doc/forum/possible_gpg_issue.mdwn b/doc/forum/possible_gpg_issue.mdwn
new file mode 100644
index 0000000000..5a3d9390b7
--- /dev/null
+++ b/doc/forum/possible_gpg_issue.mdwn
@@ -0,0 +1,28 @@
+Hi,
+
+I'm having an issue and not sure if it's to do with git-annex or something external. I'm on Ubuntu 14.04, and have been using git-annex for several months without this issue, but it suddenly started (possibly) after several packages were updated via apt-get upgrade.
+
+We have two remotes configured, one is a local (LAN) smb share, the other is on Amazon S3. We're using shared encryption on the S3 remote, and no encryption on the smb remote.
+
+The problem that started happening recently is when copying to the smb remote. There is no problem copying from the remote, and no problem reading or writing to the drive outside of git-annex. However, copying to the remote fails after it seems to copy most or all of the file and then hang on a gpg step. Again, there is no encryption on this remote. The other S3 remote with shared encryption has no issues. Other devices on the LAN, all OS X, have no issues writing to the remote.
+
+I don't have enough info to necessarily claim this is a bug in git-annex, but I'm not sure what to poke at next to try to figure it out. Any help or advice would be greatly appreciated.
+
+Below is the debug output from a failed git annex copy command.
+
+    cw@ubuntu$ git annex copy annexedfile --to smbremote --debug
+    [2014-11-07 15:35:13 PST] read: git ["--git-dir=/repobase/.git","--work-tree=/repobase","show-ref","git-annex"]
+    [2014-11-07 15:35:13 PST] read: git ["--git-dir=/repobase/.git","--work-tree=/repobase","show-ref","--hash","refs/heads/git-annex"]
+    [2014-11-07 15:35:13 PST] read: git ["--git-dir=/repobase/.git","--work-tree=/repobase","log","refs/heads/git-annex..aa8813d486939544701359dc28fa7b0916917961","--oneline","-n1"]
+    [2014-11-07 15:35:13 PST] read: git ["--git-dir=/repobase/.git","--work-tree=/repobase","log","refs/heads/git-annex..097d5b482d6856ce22814a0c2c5eee43e3e030e4","--oneline","-n1"]
+    [2014-11-07 15:35:13 PST] chat: git ["--git-dir=/repobase/.git","--work-tree=/repobase","cat-file","--batch"]
+    [2014-11-07 15:35:13 PST] read: git ["--git-dir=/repobase/.git","--work-tree=/repobase","ls-files","--cached","-z","--","annexedfile"]
+    copy annexedfile (gpg) (to smbremote...) 
+    [2014-11-07 15:35:13 PST] chat: gpg ["--quiet","--trust-model","always","--batch","--passphrase-fd","11","--symmetric","--force-mdc","--no-textmode"]
+    95%           0.0 B/s 0s/mnt/annex/tmp/GPGHMACSHA1--a097a9b653d1facbe7d37d0e8f9f580261d9adef/GPGHMACSHA1--a097a9b653d1facbe7d37d0e8f9f580261d9adef: hClose: does not exist (Host is down)
+    failed                  
+    git-annex: copy: 1 failed
+    cw@ubuntu$
+
+Thanks,
+cw
diff --git a/doc/forum/possible_gpg_issue/comment_1_c5a33ff375cddd001e6cb7be8d0ce940._comment b/doc/forum/possible_gpg_issue/comment_1_c5a33ff375cddd001e6cb7be8d0ce940._comment
new file mode 100644
index 0000000000..9c43e38a70
--- /dev/null
+++ b/doc/forum/possible_gpg_issue/comment_1_c5a33ff375cddd001e6cb7be8d0ce940._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="https://openid.stackexchange.com/user/e65e6d0e-58ba-41de-84cc-1f2ba54cf574"
+ nickname="Mica Semrick"
+ subject="is your samba share mounted?"
+ date="2014-11-10T05:08:39Z"
+ content="""
+You should make sure your samba share is mounted and you can read/write to it.
+"""]]
diff --git a/doc/forum/possible_gpg_issue/comment_2_e5ea7fc9be496f71516a238522193744._comment b/doc/forum/possible_gpg_issue/comment_2_e5ea7fc9be496f71516a238522193744._comment
new file mode 100644
index 0000000000..ffc69036da
--- /dev/null
+++ b/doc/forum/possible_gpg_issue/comment_2_e5ea7fc9be496f71516a238522193744._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="https://www.google.com/accounts/o8/id?id=AItOawknHkJJRE0K7_G4sG3YgOkAaihnw0eg9Ao"
+ nickname="Chad"
+ subject="comment 2"
+ date="2014-11-10T18:06:11Z"
+ content="""
+Yes, the drive is mounted rw for all users with no username or password required to connect. There is no problem copying from the drive, and no problem reading or writing to it outside of git-annex.
+"""]]
diff --git a/doc/forum/possible_gpg_issue/comment_3_d505b6e5050d2afa7c8543e21918193d._comment b/doc/forum/possible_gpg_issue/comment_3_d505b6e5050d2afa7c8543e21918193d._comment
new file mode 100644
index 0000000000..ade7846db2
--- /dev/null
+++ b/doc/forum/possible_gpg_issue/comment_3_d505b6e5050d2afa7c8543e21918193d._comment
@@ -0,0 +1,31 @@
+[[!comment format=mdwn
+ username="joey"
+ subject="""comment 3"""
+ date="2014-12-01T21:36:40Z"
+ content="""
+The relevant error message seems to be this:
+
+	hClose: does not exist (Host is down)
+
+Which doesn't point at gpg being the problem to me. git-annex is trying to
+close a file after writing it, and that's failing with EHOSTDOWN. Not a usual
+error code, but then you're using a network filesystem, which has many unusual
+failure modes.
+
+The next step would probably be to strace git-annex to see what syscall
+is failing. Looks like it might be close(2), but then again hClose might be
+doing something else first.
+
+It's possible that this has something to do with the SMB share not supporting
+some POSIX filesystem feature that git-annex uses. Lack of support for fcntl
+locking is a problem with NFS, dunno about SMB.
+
+What would probably work better would be to set up a
+[[directory special remote|special_remotes/directory]] on the SMB share.
+That requires a lot less from the filesystem than a full-fledged git remote
+does.
+
+(Two parts of what you said don't make sense to me BTS. I see no evidence of it hanging in the transcript, so am unclear why
+you said it was hanging, as opposed to gracefully failing. And, you said the SMB remote was not configured to use encryption, but it's clearly
+encrypted in the transcript.)
+"""]]
diff --git a/doc/forum/possible_gpg_issue/comment_4_e52099d461d0df99040f664a3af23caf._comment b/doc/forum/possible_gpg_issue/comment_4_e52099d461d0df99040f664a3af23caf._comment
new file mode 100644
index 0000000000..a2e7bd6315
--- /dev/null
+++ b/doc/forum/possible_gpg_issue/comment_4_e52099d461d0df99040f664a3af23caf._comment
@@ -0,0 +1,17 @@
+[[!comment format=mdwn
+ username="https://www.google.com/accounts/o8/id?id=AItOawknHkJJRE0K7_G4sG3YgOkAaihnw0eg9Ao"
+ nickname="Chad"
+ subject="comment 4"
+ date="2014-12-02T22:51:24Z"
+ content="""
+Thanks so much. I'll try digging some more into the syscalls and see what I can figure out. We do have it set up as a directory remote.
+
+To clarify, when I said it was \"hanging\" maybe that wasn't a good word choice. What I meant was that it very quickly gets to 95% or 100% progress, then freezes and does nothing for a while, before ultimately reporting the error and exiting.
+
+Also I take back what I said about it not using encryption. It is. I was confused by the fact that there are unencrypted files on the disk, but it turns out they are all from a long time ago, perhaps before we started using encryption. Everything more recent is encrypted.
+
+Thanks!
+-c
+
+
+"""]]
diff --git a/doc/forum/possible_gpg_issue/comment_5_a0c8afc2d8583d30073b62396b254a6e._comment b/doc/forum/possible_gpg_issue/comment_5_a0c8afc2d8583d30073b62396b254a6e._comment
new file mode 100644
index 0000000000..7011e7fd5c
--- /dev/null
+++ b/doc/forum/possible_gpg_issue/comment_5_a0c8afc2d8583d30073b62396b254a6e._comment
@@ -0,0 +1,17 @@
+[[!comment format=mdwn
+ username="https://www.google.com/accounts/o8/id?id=AItOawknHkJJRE0K7_G4sG3YgOkAaihnw0eg9Ao"
+ nickname="Chad"
+ subject="comment 5"
+ date="2014-12-03T01:05:17Z"
+ content="""
+Yeah, it's close returning the error. As far as I can tell the calls involving that file are:
+
+    open(path, O_WRONLY|O_CREAT|O_NOCTTY|O_NONBLOCK, 0666) = fh
+    fstat(fh, {st_mode=S_IFREG|0755, st_size=sz, ...}) = 0
+    ftruncate(fh, 0) = 0
+    ioctl(fh, SNDCTL_TMR_TIMEBASE or SNDRV_TIMER_IOCTL_NEXT_DEVICE or TCGETS, ptr) = -1 ENOTTY (Inappropriate ioctl for device)
+    write(fh, data, len) x 60
+    close(fh) = -1 EHOSTDOWN (Host is down)
+
+All the calls to write appear to succeed, followed by the call to close that fails. Maybe the previously failed ioctl command has something to do with it? I guess as was mentioned previously, there may be an operation the device doesn't support, but the weird thing is this used to work fine.
+"""]]
diff --git a/doc/forum/possible_gpg_issue/comment_6_2a0120d6b586594cfd9baabbd6605373._comment b/doc/forum/possible_gpg_issue/comment_6_2a0120d6b586594cfd9baabbd6605373._comment
new file mode 100644
index 0000000000..84c9c5b4fe
--- /dev/null
+++ b/doc/forum/possible_gpg_issue/comment_6_2a0120d6b586594cfd9baabbd6605373._comment
@@ -0,0 +1,15 @@
+[[!comment format=mdwn
+ username="joey"
+ subject="""comment 6"""
+ date="2015-02-09T18:41:23Z"
+ content="""
+Sorry about the delay getting back to this.. It's great you were able
+to provide the strace.
+
+I don't think that the ioctl is at fault. That seems to be something
+that's done by the IO layer when opening a handle. It does not involve
+locking.
+
+Looks to me like the writes get buffered and it fails to flush to the SMB
+server on close.
+"""]]
diff --git a/doc/forum/possible_gpg_issue/comment_7_f095eadcd9f6947f64e6830acea8228e._comment b/doc/forum/possible_gpg_issue/comment_7_f095eadcd9f6947f64e6830acea8228e._comment
new file mode 100644
index 0000000000..c262c9b524
--- /dev/null
+++ b/doc/forum/possible_gpg_issue/comment_7_f095eadcd9f6947f64e6830acea8228e._comment
@@ -0,0 +1,11 @@
+[[!comment format=mdwn
+ username="ka7"
+ subject="comment 7"
+ date="2015-05-05T16:58:42Z"
+ content="""
+is on the \"SMB share\" running something special ? ..like virus-scanner, quota, backup-in-progress
+
+and.. smb like SAMBA or Windows ?
+
+in theory you can do lots of funny stuff to get a smb share: sharing a samba which is a webdav mounted via nfs on a clamFS. (*scary*)
+"""]]
diff --git a/doc/forum/post-copy__47__sync_hook.mdwn b/doc/forum/post-copy__47__sync_hook.mdwn
new file mode 100644
index 0000000000..05fbc0b291
--- /dev/null
+++ b/doc/forum/post-copy__47__sync_hook.mdwn
@@ -0,0 +1,14 @@
+Hi,
+
+I have the following setup:
+- normal git repository with website code.
+- git annex repository to hold large set of binary data (pdfs, flashmovies, etc) that belongs to the site.
+
+I use git annex so I (and other developers) don't need to copy 1.4Gb+ of binary data for every working copy. (Data that is mostly left untouched.) Using git annex copy --to=origin I can simply only add new additions to this media/binary repository, without first pulling all the data. So far so good.
+
+When commits are pushed to a certain branch on the normal git repository, a post-receive hook exports (GIT_WORK_TREE=/data/site/ git checkout $branch -f) the updated repository to an apache documentroot. Thereby updating the staging server of the website.
+
+My question is, how can I do the same thing for my git annex repository? Since post-receive fires on receiving the annex hashes, and not the actual files. Those are rsynced, and I cannot find a way to trigger an action after all files are copied by git annex via rsync.
+
+Any tips?
+
diff --git a/doc/forum/post-copy__47__sync_hook/comment_1_c8322d4b9bbf5eac80b48c312a42fbcf._comment b/doc/forum/post-copy__47__sync_hook/comment_1_c8322d4b9bbf5eac80b48c312a42fbcf._comment
new file mode 100644
index 0000000000..04a6f4668e
--- /dev/null
+++ b/doc/forum/post-copy__47__sync_hook/comment_1_c8322d4b9bbf5eac80b48c312a42fbcf._comment
@@ -0,0 +1,11 @@
+[[!comment format=mdwn
+ username="http://joey.kitenet.net/"
+ nickname="joey"
+ subject="comment 1"
+ date="2012-03-14T16:23:25Z"
+ content="""
+I've made git-annex-shell run the git `hooks/annex-content` after content is received or dropped. 
+
+Note that the clients need to be running at least git-annex version 3.20120227 , which runs git-annex-shell commit, which runs the hook.
+
+"""]]
diff --git a/doc/forum/pre-commit_hook_to_use_git_annex_for_only_large_files.mdwn b/doc/forum/pre-commit_hook_to_use_git_annex_for_only_large_files.mdwn
new file mode 100644
index 0000000000..2a8135d992
--- /dev/null
+++ b/doc/forum/pre-commit_hook_to_use_git_annex_for_only_large_files.mdwn
@@ -0,0 +1,55 @@
+I've wanted to use git-annex for the longest time, but I really only wanted to use it if the files were over a certain size, otherwise, I just want to use regular git.
+
+After writing this pre-commit hook, I wanted to share and get some feedback.
+
+This would be saved as `.git/hooks/pre-commit`
+
+    #!/bin/sh
+    let MAX=1*1024*1024  # 1048576 == 1 MB
+    if [ ! -d '.git/annex/' ]; then
+      /usr/local/bin/git annex init >/dev/null 2>&1
+    fi
+    if git rev-parse --verify HEAD >/dev/null 2>&1; then
+      against=HEAD
+    else
+      # Initial commit: diff against an empty tree object
+      against=$(/usr/local/bin/git hash-object -t tree /dev/null)
+    fi
+    /usr/local/bin/git diff-index --cached $against | \
+      /usr/bin/tr '\t' ' ' | \
+      /usr/bin/cut -d ' ' -f4,6- | \
+      while read line; do
+        sha1=$(/usr/bin/cut -d ' ' -f1 <<< "$line")
+        if [ "$sha1" == "0000000000000000000000000000000000000000" ]; then
+          continue
+        fi
+        size=$(/usr/local/bin/git cat-file -s "$sha1")
+        if [ $size -ge $MAX ]; then
+          file=$(/usr/bin/cut -d ' ' -f2- <<< "$line")
+          /usr/local/bin/git update-index --force-remove "$file"
+          /usr/local/bin/git annex add "$file"
+          /usr/bin/killall -TERM Finder
+        fi
+    done
+    /usr/local/bin/git annex pre-commit .
+
+I also wrote an `Unlock Git Annex File.workflow` service for OS X:
+
+    set gitAnnex to "/Applications/git-annex.app/Contents/MacOS/git-annex"
+
+    tell application "Finder"
+            repeat with theItem in (get selection)
+                    if file type of theItem is "slnk" then
+                            set theFolder to quoted form of POSIX path of (container of theItem as alias)
+                            set filePath to do shell script "/usr/bin/basename " & quoted form of POSIX path of (theItem as text)
+                            set theCommand to "cd " & theFolder & "; " & gitAnnex & " unlock '" & filePath & "'"
+                            do shell script theCommand
+                    end if
+            end repeat
+    end tell
+
+Use Automator to create a new service that receives selected "files or folders" in "Finder.app". Then drag the "Run AppleScript" action to the workflow panel. The script above should be copied into the code area replacing all the default content.
+
+Am I all alone in wanting these types of scripts?
+
+- Peter
diff --git a/doc/forum/pre-commit_hook_to_use_git_annex_for_only_large_files/comment_1_01db183b1f1d081066d88332e2b6166a._comment b/doc/forum/pre-commit_hook_to_use_git_annex_for_only_large_files/comment_1_01db183b1f1d081066d88332e2b6166a._comment
new file mode 100644
index 0000000000..98618cd518
--- /dev/null
+++ b/doc/forum/pre-commit_hook_to_use_git_annex_for_only_large_files/comment_1_01db183b1f1d081066d88332e2b6166a._comment
@@ -0,0 +1,15 @@
+[[!comment format=mdwn
+ username="joey"
+ subject="""comment 1"""
+ date="2015-05-05T18:46:42Z"
+ content="""
+The problem with this pre-commit hook is that by the time you run `git add
+largfile`, it has copied it into the git repository. Your hook will prevent
+it getting into a commit, so the repository will eventually garbage collect
+the copy away, but this can take some time or manual work to do.
+
+Recent versions of `git-annex add` will look at the annex.largefiles
+configuration and if the file does not match, add it to git directly.
+So that's an alternate workflow, where you `git annex add` files and let
+git-annex decide whether to put them in the annex or the git repository.
+"""]]
diff --git a/doc/forum/pre-commit_hook_to_use_git_annex_for_only_large_files/comment_2_2b76809869e0289f78f137afbdcf36c8._comment b/doc/forum/pre-commit_hook_to_use_git_annex_for_only_large_files/comment_2_2b76809869e0289f78f137afbdcf36c8._comment
new file mode 100644
index 0000000000..7ed0c5768e
--- /dev/null
+++ b/doc/forum/pre-commit_hook_to_use_git_annex_for_only_large_files/comment_2_2b76809869e0289f78f137afbdcf36c8._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="joey"
+ subject="""comment 2"""
+ date="2015-05-05T18:56:33Z"
+ content="""
+For the Finder script, there's a page collecting such stuff,
+[[tips/file_manager_integration]]
+"""]]
diff --git a/doc/forum/preferred_content.mdwn b/doc/forum/preferred_content.mdwn
new file mode 100644
index 0000000000..b4c215328d
--- /dev/null
+++ b/doc/forum/preferred_content.mdwn
@@ -0,0 +1,11 @@
+Hello all,
+
+I'm trying to use "preferred content" with git-annex but I suspect that my version doesn't provide it.
+
+I'm using Debian "wheezy" that packs git-annex 3.20120629.
+
+I appreciate any help you can give.
+
+Thanks.
+
+Zaka.
diff --git a/doc/forum/preferred_content/comment_1_9c9e5f2ee5ae4d8459358ad16f879ef1._comment b/doc/forum/preferred_content/comment_1_9c9e5f2ee5ae4d8459358ad16f879ef1._comment
new file mode 100644
index 0000000000..24de5840b3
--- /dev/null
+++ b/doc/forum/preferred_content/comment_1_9c9e5f2ee5ae4d8459358ad16f879ef1._comment
@@ -0,0 +1,10 @@
+[[!comment format=mdwn
+ username="http://joeyh.name/"
+ ip="209.250.56.47"
+ subject="comment 1"
+ date="2013-11-07T18:38:22Z"
+ content="""
+The first version of git-annex to support preferred content settings was 3.20121009.
+
+There is a quite up-to-date backport available for Debian stable.
+"""]]
diff --git a/doc/forum/preferred_content_settings_for_multiple_symlinks.mdwn b/doc/forum/preferred_content_settings_for_multiple_symlinks.mdwn
new file mode 100644
index 0000000000..3c67a93a5f
--- /dev/null
+++ b/doc/forum/preferred_content_settings_for_multiple_symlinks.mdwn
@@ -0,0 +1,7 @@
+I have my music library in `music/` and some really old files I recently added in `reallyold/`.  There are some MP3s in the really old files and some are the same as my library, so of course git annex is only keeping one copy.  Now, I have an rsync remote, `ma`, which prefers content from `music/` but doesn't want anything from `reallyold/`.  So while right now it is trying to drop stuff, I suspect at some point that it will try to re-add them in virtue of being in `music/`, as I've got a loop.
+
+I want to eliminate this by using the present keyword to disable dropping for stuff in `reallyold/` and `music/`.  Here is my attempt, which doesn't work--I am hoping someone can spot what's wrong.
+
+    (present and include=music/*) or (present and include=reallyold/*) or (exclude=reallyold/* and exclude=video/* and exclude= ...)
+
+Note that music is included by virtue of not being excluded so it should satisfy the third disjunct.  Thanks.
diff --git a/doc/forum/preferred_content_settings_for_multiple_symlinks/comment_1_70da012d96ab576151fe3e081ef905d1._comment b/doc/forum/preferred_content_settings_for_multiple_symlinks/comment_1_70da012d96ab576151fe3e081ef905d1._comment
new file mode 100644
index 0000000000..b5fbd345eb
--- /dev/null
+++ b/doc/forum/preferred_content_settings_for_multiple_symlinks/comment_1_70da012d96ab576151fe3e081ef905d1._comment
@@ -0,0 +1,10 @@
+[[!comment format=mdwn
+ username="http://joeyh.name/"
+ ip="4.152.108.210"
+ subject="Ever figure this out?"
+ date="2013-02-24T22:00:40Z"
+ content="""
+I find it's useful, when stuck on this kind of thing, to run `git annex find` at the command line passing it parts of the expression and making sure it works, and gradually building up to the complete expression.
+
+You have to do some conversion due to differences between command line and preferred content expression format. For example `git annex find -\( --in here --and --include \"music/*\" -\)`
+"""]]
diff --git a/doc/forum/preferred_content_settings_for_multiple_symlinks/comment_2_ccea74d8b5a4de1f3cd1f6da6694ae0e._comment b/doc/forum/preferred_content_settings_for_multiple_symlinks/comment_2_ccea74d8b5a4de1f3cd1f6da6694ae0e._comment
new file mode 100644
index 0000000000..09b9d8564f
--- /dev/null
+++ b/doc/forum/preferred_content_settings_for_multiple_symlinks/comment_2_ccea74d8b5a4de1f3cd1f6da6694ae0e._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="spwhitton"
+ ip="82.36.235.9"
+ subject="comment 2"
+ date="2013-03-12T09:34:08Z"
+ content="""
+Thanks for the tip--I will definitely make another attempt at some point.
+"""]]
diff --git a/doc/forum/preferred_content_settings_for_multiple_symlinks/comment_3_fab70c642d5aaf26de05270860281030._comment b/doc/forum/preferred_content_settings_for_multiple_symlinks/comment_3_fab70c642d5aaf26de05270860281030._comment
new file mode 100644
index 0000000000..12c49a89c7
--- /dev/null
+++ b/doc/forum/preferred_content_settings_for_multiple_symlinks/comment_3_fab70c642d5aaf26de05270860281030._comment
@@ -0,0 +1,10 @@
+[[!comment format=mdwn
+ username="spwhitton"
+ ip="82.36.235.9"
+ subject="Impossible?"
+ date="2013-06-24T17:48:02Z"
+ content="""
+I have now investigated this further and believe that what I am trying to do is impossible.  Suppose we have a file `song.mp3` in both `music/` and `old/` and the special remote's preferred content expression is set-up to prefer `music/` and not to prefer `old/`.  So initially, git-annex won't try to upload the content when it scans `old/song.mp3`, but it will when it scans `music/song.mp3`.  Fine.  Now consider a later scan.  On scanning the filename `music/song.mp3` git-annex will suppose the file was dropped from the remote, and then see it would be preferred again so won't drop it.  But then when it gets to scanning the filename `old/song.mp3` it will see that it wouldn't be preferred and will drop the content.  So we get the loop.
+
+In order to stop the unwanted drop here, the preferred content expression must prefer \"files in old/* that is also in music/*\" but since preferred content expressions concern filenames without reference to their contents, this is impossible to express.  The expression `include=music/* and include=old/*` will never match anything.
+"""]]
diff --git a/doc/forum/preferred_content_settings_for_multiple_symlinks/comment_4_3cbd06de53b6a13e2741124a8e7b5b5b._comment b/doc/forum/preferred_content_settings_for_multiple_symlinks/comment_4_3cbd06de53b6a13e2741124a8e7b5b5b._comment
new file mode 100644
index 0000000000..1868e9736a
--- /dev/null
+++ b/doc/forum/preferred_content_settings_for_multiple_symlinks/comment_4_3cbd06de53b6a13e2741124a8e7b5b5b._comment
@@ -0,0 +1,10 @@
+[[!comment format=mdwn
+ username="http://joeyh.name/"
+ ip="4.152.108.193"
+ subject="comment 4"
+ date="2013-06-25T17:22:53Z"
+ content="""
+spwhitton, your analysis is correct -- except this problem was finessed in git-annex version 4.20130621. Now, when using direct mode, it knows that `old/song.mp3` is also `music/song.mp3`, and so it only drops it if the preferred content expression allows dropping both files.
+
+Still doesn't work in indirect mode, and I think not likely to in the forseeable future. See [[bugs/Handling_of_files_inside_and_outside_archive_directory_at_the_same_time]]
+"""]]
diff --git a/doc/forum/preferred_content_settings_for_multiple_symlinks/comment_5_963558ab261d8a6315402d371e8348f9._comment b/doc/forum/preferred_content_settings_for_multiple_symlinks/comment_5_963558ab261d8a6315402d371e8348f9._comment
new file mode 100644
index 0000000000..f7ae3216a4
--- /dev/null
+++ b/doc/forum/preferred_content_settings_for_multiple_symlinks/comment_5_963558ab261d8a6315402d371e8348f9._comment
@@ -0,0 +1,10 @@
+[[!comment format=mdwn
+ username="spwhitton"
+ ip="82.36.235.9"
+ subject="comment 5"
+ date="2013-06-25T19:47:19Z"
+ content="""
+I assume you mean that if I use direct mode locally, it will get things right.  I didn't think a special remote could ever be in direct mode.
+
+Thank you for the info.  I don't want to lose the safeguards of indirect mode just for this, so I'll stick with my inelegant manual preferred content strings for now.
+"""]]
diff --git a/doc/forum/problem_with_non-ascii_filenames.mdwn b/doc/forum/problem_with_non-ascii_filenames.mdwn
new file mode 100644
index 0000000000..64885c1f28
--- /dev/null
+++ b/doc/forum/problem_with_non-ascii_filenames.mdwn
@@ -0,0 +1,5 @@
+git-annex add . fails with:
+
+ git-annex: unknown response from git cat-file ("HEAD:./MYDIRECTORYNAME/Icon missing","HEAD:./MYDIRECTORYNAME/Icon\r")
+
+There's a file in the directory called Icon^M or Icon? (there's some kind of control character or unicode or other weirdness at the end)
diff --git a/doc/forum/public-web-frontend.mdwn b/doc/forum/public-web-frontend.mdwn
new file mode 100644
index 0000000000..d3fdb9b94a
--- /dev/null
+++ b/doc/forum/public-web-frontend.mdwn
@@ -0,0 +1,16 @@
+Hi,
+
+Use case: I would like to have a "Public" top-level directory in my annex, which gets files in there published over HTTP on a particular server.
+
+How I see doing this:
+
+1. Put my annex to an http server with exported-over-http `/Public/` directory.
+2. Configure a `post-update` hook with the following:
+
+        $ git annex fix
+
+3. Push files on `/Public/` to that annex.
+
+Does it make sense? If yes, are there any gotchas I should beware of?
+
+Thanks.
diff --git a/doc/forum/public-web-frontend/comment_1_c73bd2dfe020c25eaad1c0707dd2db01._comment b/doc/forum/public-web-frontend/comment_1_c73bd2dfe020c25eaad1c0707dd2db01._comment
new file mode 100644
index 0000000000..b9f6c98676
--- /dev/null
+++ b/doc/forum/public-web-frontend/comment_1_c73bd2dfe020c25eaad1c0707dd2db01._comment
@@ -0,0 +1,9 @@
+[[!comment format=mdwn
+ username="http://joeyh.name/"
+ subject="should work.."
+ date="2012-08-05T17:18:47Z"
+ content="""
+I see no need for `git annex fix` here.
+
+Web server default configurations may not allow following symlinks outside the web server document root. On apache, it can be enabled with `Options FollowSymLinks`
+"""]]
diff --git a/doc/forum/public-web-frontend/comment_2_0026d7be6b17e50d86b3b54985882f80._comment b/doc/forum/public-web-frontend/comment_2_0026d7be6b17e50d86b3b54985882f80._comment
new file mode 100644
index 0000000000..bfb60ede1e
--- /dev/null
+++ b/doc/forum/public-web-frontend/comment_2_0026d7be6b17e50d86b3b54985882f80._comment
@@ -0,0 +1,14 @@
+[[!comment format=mdwn
+ username="https://www.google.com/accounts/o8/id?id=AItOawmdLmVxigDvxiGhnQ34xhyAaxl0CRRe9gs"
+ nickname="Bryan"
+ subject="Alternate version - smarter integration with other remotes/web special remote?"
+ date="2013-03-03T14:19:12Z"
+ content="""
+I love the idea of pushing files to \"visible\" using annex, but I'm wondering how difficult to would be to make this even fancier.
+
+For instance, if there are files that were originally imported from the web special remote (and have http:// prefixes, since that's not guaranteed), and they're not actually in the pushed annex, generate a redirect for all of them and write a special remote-local .htaccess file.
+
+Or, similarly, allow each remote to have some metadata which specifies what, if any, the URL prefix for a directory in annex is. Then, whenever you update a web remote, the auto-generated redirects list could contain not only files not present but with web-remote locations, but also other remotes which have the file under a web-visible path prefix. Bonus for allowing URL prefixes for S3-style remotes that are not encrypted.
+
+My use case for this sort of fancy is that I have a large server with gobs of space but on a personal (slower) internet connection, and a much smaller hosted instance with much faster bandwidth. With this smarts, I could push and pull large/expected high traffic files to the higher bandwidth/smaller machine ahead of, say, announcing a new feature or sharing a link to a video via e-mail.. or even reactively if the lower-bandwidth server is getting overwhelmed. 
+"""]]
diff --git a/doc/forum/public__44___read_only_annex_without_location_tracking.mdwn b/doc/forum/public__44___read_only_annex_without_location_tracking.mdwn
new file mode 100644
index 0000000000..88f8b21c2c
--- /dev/null
+++ b/doc/forum/public__44___read_only_annex_without_location_tracking.mdwn
@@ -0,0 +1,7 @@
+I would like to use a public, read only annex to publish photos. I have a server, running gitolite, with git-annex setup, which I use successfully to sync content between my own devices.
+
+But, with a public annex, I would like the view of the repository available from the server to be that only the server has the content, and not to have, or give out any of the location tracking information about any other annexes?
+
+A more concrete example would be, how do I get a photo in to git annex locally, and then push this to the server for public access, without publishing information about my local repository?
+
+From the public annexes I have looked at, this does not appear to be done. So I am unsure if this is even possible, however it seems a desirable thing to do?
diff --git a/doc/forum/public__44___read_only_annex_without_location_tracking/comment_1_47262f048a87fd6b781090f880a9bf99._comment b/doc/forum/public__44___read_only_annex_without_location_tracking/comment_1_47262f048a87fd6b781090f880a9bf99._comment
new file mode 100644
index 0000000000..99ed6f25a7
--- /dev/null
+++ b/doc/forum/public__44___read_only_annex_without_location_tracking/comment_1_47262f048a87fd6b781090f880a9bf99._comment
@@ -0,0 +1,12 @@
+[[!comment format=mdwn
+ username="http://joeyh.name/"
+ ip="209.250.56.55"
+ subject="comment 1"
+ date="2014-07-03T19:35:21Z"
+ content="""
+There's not a way to do it yet.
+
+The fundamental problem is that there is a variety of information stored on the git-annex branch, including location tracking information that you don't want (locations on non-public repositories), location tracking information you do want (locations on public repositories), urls, metadata, repository configurations and descriptions (some for public and some for non-public repositories). It seems fairly hard to draw a line. And once a line is drawn, there would be two diverged git-annex branches on the public and private repos, and the private repo would need to synthesize an updated version of the public branch every time it synced.
+
+Instead, the last time this came up, I added the remote..annex-readonly setting. This allows for at least having a private repository (or a whole network of repositories that all communicate together but remain private) that pulls changes from a public repo, but avoids making changes to it. Certainly not a complete solution, since changes have to be contributed to the public repo in some out-of-band way, like perhaps using git-format-patch and git-am.
+"""]]
diff --git a/doc/forum/public__44___read_only_annex_without_location_tracking/comment_2_ec3ff6487c9e5c89c7e508d72518bd50._comment b/doc/forum/public__44___read_only_annex_without_location_tracking/comment_2_ec3ff6487c9e5c89c7e508d72518bd50._comment
new file mode 100644
index 0000000000..ad74e68660
--- /dev/null
+++ b/doc/forum/public__44___read_only_annex_without_location_tracking/comment_2_ec3ff6487c9e5c89c7e508d72518bd50._comment
@@ -0,0 +1,10 @@
+[[!comment format=mdwn
+ username="cbaines"
+ ip="82.19.50.118"
+ subject="comment 2"
+ date="2014-07-03T20:13:20Z"
+ content="""
+Thanks for your response.
+
+The method you describe sounds like it might just do. I'll have a try, and see if it works out :)
+"""]]
diff --git a/doc/forum/pulling_from_encrypted_remote.mdwn b/doc/forum/pulling_from_encrypted_remote.mdwn
new file mode 100644
index 0000000000..e9486b597d
--- /dev/null
+++ b/doc/forum/pulling_from_encrypted_remote.mdwn
@@ -0,0 +1,12 @@
+Is there a way to pull from an encrypted remote?
+
+Use case:
+
+1. Have annex in an encrypted public rsync remote
+2. Have USB stick with PGP keys (but not the annex repository)
+3. Get to a new computer
+4. Set up a new annex using the PGP keys I have.
+
+1-3 work fine :) However, 4'th is the issue:
+
+How would I do `git pull ` for an encrypted remote? Is it possible?
diff --git a/doc/forum/pulling_from_encrypted_remote/comment_1_e9d6a9a6e01d01edb41a11b0da11d74d._comment b/doc/forum/pulling_from_encrypted_remote/comment_1_e9d6a9a6e01d01edb41a11b0da11d74d._comment
new file mode 100644
index 0000000000..2da698f4a9
--- /dev/null
+++ b/doc/forum/pulling_from_encrypted_remote/comment_1_e9d6a9a6e01d01edb41a11b0da11d74d._comment
@@ -0,0 +1,10 @@
+[[!comment format=mdwn
+ username="https://www.google.com/accounts/o8/id?id=AItOawmBUR4O9mofxVbpb8JV9mEbVfIYv670uJo"
+ nickname="Justin"
+ subject="comment 1"
+ date="2012-07-10T18:14:31Z"
+ content="""
+You just need to `git clone` the existing repository and make sure the `git remote`s are setup.
+
+the 'rsync remote' is not actually the annex, it's just a collection of encrypted files with obfuscated names.  You need a copy of the actual repository to restore the files.
+"""]]
diff --git a/doc/forum/pulling_from_encrypted_remote/comment_2_8d0db2ff65ce935c6e68044a3e0721a8._comment b/doc/forum/pulling_from_encrypted_remote/comment_2_8d0db2ff65ce935c6e68044a3e0721a8._comment
new file mode 100644
index 0000000000..8fd73cd92c
--- /dev/null
+++ b/doc/forum/pulling_from_encrypted_remote/comment_2_8d0db2ff65ce935c6e68044a3e0721a8._comment
@@ -0,0 +1,16 @@
+[[!comment format=mdwn
+ username="https://www.google.com/accounts/o8/id?id=AItOawmLB39PC89rfGaA8SwrsnB6tbumezj-aC0"
+ nickname="Tobias"
+ subject="Thats a fair solution"
+ date="2012-07-22T13:51:25Z"
+ content="""
+Until you don't have(access to) an existing repository to clone from.
+
+I really hope you revisit this when you come to the encryption part of the assistant.
+
+Btw, I also run FreeBSD if you need a tester on that at some point.
+
+Sincerely
+Tobias Ussing
+
+"""]]
diff --git a/doc/forum/pure_git-annex_only_workflow.mdwn b/doc/forum/pure_git-annex_only_workflow.mdwn
new file mode 100644
index 0000000000..36648a9058
--- /dev/null
+++ b/doc/forum/pure_git-annex_only_workflow.mdwn
@@ -0,0 +1,46 @@
+I’m using git annex to manage my movie collection on various devices – my laptop, a NSLU tucked away somewhere with lots of space, some external hard drives. For this use case, I do not need the full power of git as a version control system, so having to run "git commit" and coming up with commit messages is annoying. Also, this makes sense for a version control system, but not for my media collection:
+
+	$ git annex add Hot\ Fuzz\ -\ English.mkv 
+	add Hot Fuzz - English.mkv (checksum...) ok
+	(Recording state in git...)
+	$ git commit -m 'another movie added'
+	[master 851dc8a] another movie added
+	 1 files changed, 1 insertions(+), 0 deletions(-)
+	 create mode 120000 00 Noch nicht gesehen/Hot Fuzz - English.mkv
+	$ git push jeff
+	Counting objects: 38, done.
+	Delta compression using up to 2 threads.
+	Compressing objects: 100% (20/20), done.
+	Writing objects: 100% (26/26), 2.00 KiB, done.
+	Total 26 (delta 11), reused 0 (delta 0)
+	remote: error: refusing to update checked out branch: refs/heads/master
+	remote: error: By default, updating the current branch in a non-bare repository
+	remote: error: is denied, because it will make the index and work tree inconsistent
+	remote: error: with what you pushed, and will require 'git reset --hard' to match
+	remote: error: the work tree to HEAD.
+	remote: error: 
+	remote: error: You can set 'receive.denyCurrentBranch' configuration variable to
+	remote: error: 'ignore' or 'warn' in the remote repository to allow pushing into
+	remote: error: its current branch; however, this is not recommended unless you
+	remote: error: arranged to update its work tree to match what you pushed in some
+	remote: error: other way.
+	remote: error: 
+	remote: error: To squelch this message and still keep the default behaviour, set
+	remote: error: 'receive.denyCurrentBranch' configuration variable to 'refuse'.
+	To jeff:/mnt/media/Movies
+	 ! [rejected]        git-annex -> git-annex (non-fast-forward)
+	 ! [remote rejected] master -> master (branch is currently checked out)
+	error: failed to push some refs to 'jeff:/mnt/media/Movies'
+	To prevent you from losing history, non-fast-forward updates were rejected
+	Merge the remote changes (e.g. 'git pull') before pushing again.  See the
+	'Note about fast-forwards' section of 'git push --help' for details.
+
+It seems that to successfully make the new files known to the other side, I have to log into jeff and pull _from_ my current machine.	
+
+What I would like to have is that
+
+* git annex add does not require a commit afterwards.
+* Changes to the files are automatically picked up with the next git-annex call (similar to how etckeeper works).
+* Commands "git annex push" and "git annex pull" that will sync the metadata (i.e. the list of files) in both directions without further manual intervention, at least not until the two repositories have diverged in a way that is not possible to merge sensible.
+
+Summay: git-annex is great. git is not always. Please make it possible to use git annex without having to use git.
diff --git a/doc/forum/pure_git-annex_only_workflow/comment_10_683768c9826b0bf0f267e8734b9eb872._comment b/doc/forum/pure_git-annex_only_workflow/comment_10_683768c9826b0bf0f267e8734b9eb872._comment
new file mode 100644
index 0000000000..dc499cbc90
--- /dev/null
+++ b/doc/forum/pure_git-annex_only_workflow/comment_10_683768c9826b0bf0f267e8734b9eb872._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="http://www.joachim-breitner.de/"
+ nickname="nomeata"
+ subject="Finally some code"
+ date="2011-12-29T19:58:31Z"
+ content="""
+The repository at http://git.nomeata.de/?p=git-annex.git;a=summary contains changes to Commands/Sync.hs (and to the manpage) that implements this behavior. The functionality should be fine; the progress output is not very nice yet, but I’m not sure if I really understood the various Command types. It also should be more easily discoverable how to activate the behavior (by running \"git branch synced/master\") by providing a helpful message, at least unless git annex init creates the branch by default.
+"""]]
diff --git a/doc/forum/pure_git-annex_only_workflow/comment_11_6b541ed834ef45606f3b98779a25a148._comment b/doc/forum/pure_git-annex_only_workflow/comment_11_6b541ed834ef45606f3b98779a25a148._comment
new file mode 100644
index 0000000000..30932e301f
--- /dev/null
+++ b/doc/forum/pure_git-annex_only_workflow/comment_11_6b541ed834ef45606f3b98779a25a148._comment
@@ -0,0 +1,30 @@
+[[!comment format=mdwn
+ username="http://joey.kitenet.net/"
+ nickname="joey"
+ subject="comment 11"
+ date="2011-12-30T21:49:06Z"
+ content="""
+OMG, my first sizable haskell patch!
+
+So trying this out..
+
+In each repo I want to sync, I first `git branch synced/master`
+
+Then in each repo, I found I had to pull from each of its remotes, to get the tracking branches that `defaultSyncRemotes` looks for to know those remotes are syncable. This was the surprising thing for me, I had expected sync to somehow work out which remotes were syncable without my explicit pull. And it was not very obvious that sync was not doing its thing before I did that, since it still does a lot of \"stuff\".
+
+Once set up properly, `git annex sync` fetches from each remote, merges, and then pushes to each remote that has a synced branch. Changes propigate around even when some links are one-directional. Cool!
+
+So it works fine, but I think more needs to be done to make setting up syncing easier. Ideally, all a user would need to do is run \"git annex sync\" and it syncs from all remotes, without needing to manually set up the synced/master branch.
+
+While this would lose the ability to control which remotes are synced, I think that being able to `git annex sync origin` and only sync from/to origin is sufficient, for the centralized use case.
+
+---
+
+Code review:
+
+Why did you make `branch` strict?
+
+There is a bit of a bug in your use of Command.Merge.start. The git-annex branch merge code only runs once per git-annex run, and often this comes before sync fetches from the remotes, leading to a push conflict. I've fixed this in my \"sync\" branch, along with a few other minor things.
+
+`mergeRemote` merges from `refs/remotes/foo/synced/master`. But that will only be up-to-date if `git annex sync` has recently been run there. Is there any reason it couldn't merge from `refs/remotes/foo/master`?
+"""]]
diff --git a/doc/forum/pure_git-annex_only_workflow/comment_12_ca8ca35d6cd4a9f94568536736c12adc._comment b/doc/forum/pure_git-annex_only_workflow/comment_12_ca8ca35d6cd4a9f94568536736c12adc._comment
new file mode 100644
index 0000000000..5b0259b97a
--- /dev/null
+++ b/doc/forum/pure_git-annex_only_workflow/comment_12_ca8ca35d6cd4a9f94568536736c12adc._comment
@@ -0,0 +1,10 @@
+[[!comment format=mdwn
+ username="http://joey.kitenet.net/"
+ nickname="joey"
+ subject="comment 12"
+ date="2011-12-30T23:45:57Z"
+ content="""
+I have made a new `autosync` branch, where all that the user needs to do is run `git annex sync` and it automatically sets up the synced/master branch. I find this very easy to use, what do you think?
+
+Note that `autosync` is also pretty smart about not running commands like \"git merge\" and \"git push\" when they would not do anything. So you may find `git annex sync` not showing all the steps you'd expect. The only step a sync always performs now is pulling from the remotes.
+"""]]
diff --git a/doc/forum/pure_git-annex_only_workflow/comment_13_00c82d320c7b4bb51078beba17e14dc8._comment b/doc/forum/pure_git-annex_only_workflow/comment_13_00c82d320c7b4bb51078beba17e14dc8._comment
new file mode 100644
index 0000000000..84515ec6c5
--- /dev/null
+++ b/doc/forum/pure_git-annex_only_workflow/comment_13_00c82d320c7b4bb51078beba17e14dc8._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="http://joey.kitenet.net/"
+ nickname="joey"
+ subject="comment 13"
+ date="2011-12-31T18:34:31Z"
+ content="""
+I have merged my autosync branch, the improved sync command will be in this year's last git-annex release!
+"""]]
diff --git a/doc/forum/pure_git-annex_only_workflow/comment_14_b63568b327215ef8f646a39d760fdfc0._comment b/doc/forum/pure_git-annex_only_workflow/comment_14_b63568b327215ef8f646a39d760fdfc0._comment
new file mode 100644
index 0000000000..194415dfb6
--- /dev/null
+++ b/doc/forum/pure_git-annex_only_workflow/comment_14_b63568b327215ef8f646a39d760fdfc0._comment
@@ -0,0 +1,32 @@
+[[!comment format=mdwn
+ username="http://www.joachim-breitner.de/"
+ nickname="nomeata"
+ subject="comment 14"
+ date="2012-01-02T14:02:04Z"
+ content="""
+Sorry for not replying earlier, but my non-mailinglist-communications-workflows are suboptimal :-)
+
+> Then in each repo, I found I had to pull from each of its remotes, to get the tracking branches that defaultSyncRemotes looks for to know those remotes are syncable.  This was the surprising thing for me, I had expected sync to somehow work out which remotes were syncable without my explicit pull. And it was not very obvious that sync was not doing its thing before I did that, since it still does a lot of \"stuff\".
+
+Right. But \"git fetch\" ought to be enough.
+
+Personally, I’d just pull and push everywhere, but you pointed out that it ought to be manageable. The existence of the synced/master branch is the flag that indicates this, so you need to propagate this once. Note that if the branch were already created by \"git annex init\", then this would not be a problem.
+
+It is not required to use \"git fetch\" once, you can also call \"git annex sync \" once with the remote explicitly mentioned; this would involve a fetch.
+
+> While this would lose the ability to control which remotes are synced, I think that being able to git annex sync origin and only sync from/to origin is sufficient, for the centralized use case.
+
+I’d leave this decision to you. But I see that you took the decision already, as your code now creates the synced/master branch when it does not exist (e290f4a8).
+
+> Why did you make branch strict?
+
+Because it did not work otherwise :-). It uses pipeRead, which is lazy, and for some reason git and/or your utility functions did not like that the output of the command was not consumed before the next git command was called. I did not investigate further. For better code, I’d suggest to add a function like pipeRead that completely reads the git output before returning, thus avoiding any issues with lazyIO.
+
+> mergeRemote merges from refs/remotes/foo/synced/master. But that will only be up-to-date if git annex sync has recently been run there. Is there any reason it couldn't merge from refs/remotes/foo/master?
+
+Hmm, good question. It is probably save to merge from both, and push only to synced/master. But which one first? synced/master can be ahead if the repo was synced to from somewhere else, master can be ahead if there are local changes. Maybe git merge should be called on all remote heads simultaniously, thus generating only one commit for the merge. I don’t know how well that works in practice.
+
+Thanks for including my code,
+Joachim
+
+"""]]
diff --git a/doc/forum/pure_git-annex_only_workflow/comment_15_cb7c856d8141b2de3cc95874753f1ee5._comment b/doc/forum/pure_git-annex_only_workflow/comment_15_cb7c856d8141b2de3cc95874753f1ee5._comment
new file mode 100644
index 0000000000..a526d5db8a
--- /dev/null
+++ b/doc/forum/pure_git-annex_only_workflow/comment_15_cb7c856d8141b2de3cc95874753f1ee5._comment
@@ -0,0 +1,12 @@
+[[!comment format=mdwn
+ username="http://joey.kitenet.net/"
+ nickname="joey"
+ subject="comment 15"
+ date="2012-01-02T16:01:49Z"
+ content="""
+With a lazy branch, I get \"git-annex: no branch is checked out\". Weird.. my best guess is that it's because this is running at the seek stage, which is unusual, and the value is not used until a later stage and so perhaps the git command gets reaped by some cleanup code before its output is read.
+
+(pipeRead is lazy because often it's used to read large quantities of data from git that are processed progressively.)
+
+I did make it merge both branches, separately. It would be possible to do one single merge, but it's probably harder for the user to recover if there are conflicts in an octopus merge. The order of the merges does not seem to me to matter much, barring conflicts it will work either way. Dealing with conflicts during sync is probably a weakness of all this; after the first conflict the rest of the sync will continue failing.
+"""]]
diff --git a/doc/forum/pure_git-annex_only_workflow/comment_1_a32f7efd18d174845099a4ed59e6feae._comment b/doc/forum/pure_git-annex_only_workflow/comment_1_a32f7efd18d174845099a4ed59e6feae._comment
new file mode 100644
index 0000000000..def1794a3e
--- /dev/null
+++ b/doc/forum/pure_git-annex_only_workflow/comment_1_a32f7efd18d174845099a4ed59e6feae._comment
@@ -0,0 +1,32 @@
+[[!comment format=mdwn
+ username="http://joey.kitenet.net/"
+ nickname="joey"
+ subject="comment 1"
+ date="2011-12-09T22:56:11Z"
+ content="""
+First, you need a bare git repository that you can push to, and pull from. This simplifies most git workflow.
+
+Secondly, I use [mr](http://kitenet.net/~joey/code/mr/), with this in `.mrconfig`:
+
+
    +[DEFAULT]
+lib =
+        annexupdate() {
+                git commit -a -m update || true
+                git pull \"$@\"
+                git annex merge
+                git push || true
+        }
+
+[lib/sound]
+update = annexupdate
+[lib/big]
+update = annexupdate
+
    
+
+Which makes \"mr update\" in repositories where I rarely care about git details take care of syncing my changes.
+
+I also make \"mr update\" do a \"git annex get\" of some files in some repositories that I want to always populate. git-annex and mr go well together. :)
+
+Perhaps my annexupdate above should be available as \"git annex sync\"?
+"""]]
diff --git a/doc/forum/pure_git-annex_only_workflow/comment_2_66dc9b65523a9912411db03c039ba848._comment b/doc/forum/pure_git-annex_only_workflow/comment_2_66dc9b65523a9912411db03c039ba848._comment
new file mode 100644
index 0000000000..473a0287d0
--- /dev/null
+++ b/doc/forum/pure_git-annex_only_workflow/comment_2_66dc9b65523a9912411db03c039ba848._comment
@@ -0,0 +1,15 @@
+[[!comment format=mdwn
+ username="http://www.joachim-breitner.de/"
+ nickname="nomeata"
+ subject="comment 2"
+ date="2011-12-10T16:28:29Z"
+ content="""
+Thanks for the tips so far. I guess a bare-only repo helps, but as well is something that I don’t _need_ (for my use case), any only have to do because git works like this.
+
+Also, if I have a mobile device that I want to push to, then I’d have to have two repositories on the device, as I might not be able to reach my main bare repository when traveling, but I cannot push to the „real“ repo on the mobile device from my computer. I guess I am spoiled by darcs, which will happily push to a checked out 
+remote repository, updating the checkout if possible without conflict.
+
+If I introduce a central bare repository to push to and from; I’d still have to have the other non-bare repos as remotes, so that git-annex will know about them and their files, right?
+
+I’d appreciate a \"git annex sync\" that does what you described (commit all, pull, merge, push). Especially if it comes in a \"git annex sync --all\" variant that syncs all reachable repositories.
+"""]]
diff --git a/doc/forum/pure_git-annex_only_workflow/comment_3_9b7d89da52f7ebb7801f9ec8545c3aba._comment b/doc/forum/pure_git-annex_only_workflow/comment_3_9b7d89da52f7ebb7801f9ec8545c3aba._comment
new file mode 100644
index 0000000000..9b6e6d7c4d
--- /dev/null
+++ b/doc/forum/pure_git-annex_only_workflow/comment_3_9b7d89da52f7ebb7801f9ec8545c3aba._comment
@@ -0,0 +1,12 @@
+[[!comment format=mdwn
+ username="http://joey.kitenet.net/"
+ nickname="joey"
+ subject="comment 3"
+ date="2011-12-10T19:43:04Z"
+ content="""
+Git can actually push into a non-bare repository, so long as the branch you change there is not a checked out one. Pushing into `remotes/$foo/master` and `remotes/$foo/git-annex` would work, however determining the value that the repository expects for `$foo` is something git cannot do on its own. And of course you'd still have to `git merge remotes/$foo/master` to get the changes. 
+
+Yes, you still keep the non-bare repos as remotes when adding a bare repository, so git-annex knows how to get to them.
+
+I've made `git annex sync` run the simple script above. Perhaps it can later be improved to sync all repositories.
+"""]]
diff --git a/doc/forum/pure_git-annex_only_workflow/comment_4_dc8a3f75533906ad3756fcc47f7e96bb._comment b/doc/forum/pure_git-annex_only_workflow/comment_4_dc8a3f75533906ad3756fcc47f7e96bb._comment
new file mode 100644
index 0000000000..1ac9e798a8
--- /dev/null
+++ b/doc/forum/pure_git-annex_only_workflow/comment_4_dc8a3f75533906ad3756fcc47f7e96bb._comment
@@ -0,0 +1,20 @@
+[[!comment format=mdwn
+ username="http://www.joachim-breitner.de/"
+ nickname="nomeata"
+ subject="comment 4"
+ date="2011-12-13T18:16:08Z"
+ content="""
+I thought about this some more, and I think I have a pretty decent solution that avoids a central bare repository. Instead of pushing to master (which git does not like) or trying to guess the remote branch name on the other side, there is a well-known branch name, say git-annex-master. Then a sync command would do something like this (untested):
+
+    git commit -a -m 'git annex sync' # ideally with a description derived from the diff
+    git merge git-annex-master
+    git pull someremote git-annex-master # for all reachable remotes. Or better to use fetch and then merge everything in one command?
+    git branch -f git-annex-master # (or checkout git-annex-master, merge master, checkout master, but since we merged before this should have the same effect
+    git annex merge
+    git push someremote git-annex-master # for all reachable remotes
+
+The nice things are: One can push to any remote repository, and thus avoid the issue of pushing to a portable device; the merging happens on the master branch, so if it fails to merge automatically, regular git foo can resolve it, and all changes eventually reach every repository.
+
+What do you think?
+    
+"""]]
diff --git a/doc/forum/pure_git-annex_only_workflow/comment_5_afe5035a6b35ed2c7e193fb69cc182e2._comment b/doc/forum/pure_git-annex_only_workflow/comment_5_afe5035a6b35ed2c7e193fb69cc182e2._comment
new file mode 100644
index 0000000000..0847daae9d
--- /dev/null
+++ b/doc/forum/pure_git-annex_only_workflow/comment_5_afe5035a6b35ed2c7e193fb69cc182e2._comment
@@ -0,0 +1,24 @@
+[[!comment format=mdwn
+ username="http://www.joachim-breitner.de/"
+ nickname="nomeata"
+ subject="comment 5"
+ date="2011-12-13T18:47:18Z"
+ content="""
+After some experimentation, this seems to work better:
+
+    	git commit -a -m 'git annex sync'
+	git merge git-annex-master
+	for remote in $(git remote)
+	do
+		git fetch $remote
+		git merge $remote git-annex-master
+	done
+	git branch -f git-annex-master
+	git annex merge
+	for remote in $(git remote)
+	do
+		git push $remote git-annex git-annex-master
+	done
+
+Maybe this approach can be enhance to skip stuff gracefully if there is no git-annex-master branch and then be added to what \"git annex sync\" does, this way those who want to use the feature can do so by running \"git branch git-annex-master\" once. Or, if you like this and want to make it default, just make git-annex-init create the git-annex-master branch :-)
+"""]]
diff --git a/doc/forum/pure_git-annex_only_workflow/comment_6_3660d45c5656f68924acbd23790024ee._comment b/doc/forum/pure_git-annex_only_workflow/comment_6_3660d45c5656f68924acbd23790024ee._comment
new file mode 100644
index 0000000000..fc66fbb8e1
--- /dev/null
+++ b/doc/forum/pure_git-annex_only_workflow/comment_6_3660d45c5656f68924acbd23790024ee._comment
@@ -0,0 +1,12 @@
+[[!comment format=mdwn
+ username="http://joey.kitenet.net/"
+ nickname="joey"
+ subject="comment 6"
+ date="2011-12-13T20:53:23Z"
+ content="""
+It would be clearer to call \"git-annex-master\" \"synced/master\" (or really \"synced/$current_branch\"). That does highlight that this method of syncing is not particularly specific to git-annex.
+
+I think this would be annoying to those who do use a central bare repository, because of the unnecessary pushing and pulling to other repos, which could be expensive to do, especially if you have a lot of interconnected repos. So having a way to enable/disable it seems best.
+
+Maybe you should work up a patch to Command/Sync.hs, since I know you know haskell :)
+"""]]
diff --git a/doc/forum/pure_git-annex_only_workflow/comment_7_33db51096f568c65b22b4be0b5538c0d._comment b/doc/forum/pure_git-annex_only_workflow/comment_7_33db51096f568c65b22b4be0b5538c0d._comment
new file mode 100644
index 0000000000..753a2af169
--- /dev/null
+++ b/doc/forum/pure_git-annex_only_workflow/comment_7_33db51096f568c65b22b4be0b5538c0d._comment
@@ -0,0 +1,15 @@
+[[!comment format=mdwn
+ username="http://www.joachim-breitner.de/"
+ nickname="nomeata"
+ subject="comment 7"
+ date="2011-12-18T12:08:51Z"
+ content="""
+I agree on the naming suggestions, and that it does not suit everybody. Maybe I’ll think some more about it. The point is: I’m trying to make live easy for those who do not want to manually create some complicated setup, so if it needs configuration, it is already off that track. But turning the current behavior into something people have to configure is also not well received by the users.
+
+Given that \"git annex sync\" is a new command, maybe it is fine to have this as a default behavior, and offer an easy way out. The easy way out could be one of two flags that can be set for a repo (or a remote):
+
+* \"central\", which makes git annex sync only push and pull to and that repo (unless a different remote is given on the command line)
+* \"unsynced\", which makes git annex sync skip the repo.
+
+Maybe central is enough.
+"""]]
diff --git a/doc/forum/pure_git-annex_only_workflow/comment_8_6e5b42fdb7801daadc0b3046cbc3d51e._comment b/doc/forum/pure_git-annex_only_workflow/comment_8_6e5b42fdb7801daadc0b3046cbc3d51e._comment
new file mode 100644
index 0000000000..67953860ae
--- /dev/null
+++ b/doc/forum/pure_git-annex_only_workflow/comment_8_6e5b42fdb7801daadc0b3046cbc3d51e._comment
@@ -0,0 +1,12 @@
+[[!comment format=mdwn
+ username="http://joey.kitenet.net/"
+ nickname="joey"
+ subject="comment 8"
+ date="2011-12-19T18:29:01Z"
+ content="""
+I don't mind changing the behavior of git-annex sync, certainly..
+
+Looking thru git's documentation, I found some existing configuration that could be reused following your idea.
+There is a remote.name.skipDefaultUpdate and a remote.name.skipFetchAll. Though both have to do with fetches, not pushes.
+Another approach might be to use git's remote group stuff.
+"""]]
diff --git a/doc/forum/pure_git-annex_only_workflow/comment_9_ace319652f9c7546883b5152ddc82591._comment b/doc/forum/pure_git-annex_only_workflow/comment_9_ace319652f9c7546883b5152ddc82591._comment
new file mode 100644
index 0000000000..de656d6629
--- /dev/null
+++ b/doc/forum/pure_git-annex_only_workflow/comment_9_ace319652f9c7546883b5152ddc82591._comment
@@ -0,0 +1,14 @@
+[[!comment format=mdwn
+ username="http://www.joachim-breitner.de/"
+ nickname="nomeata"
+ subject="comment 9"
+ date="2011-12-19T22:56:26Z"
+ content="""
+Another option that would please the naive user without hindering the more advanced user: \"git annex init\", by default, creates a synced/master branch. \"git annex sync\" will pull from every /sync/master branch it finds, and also push to any /sync/master branch it finds, but will not create any. So by default (at least for new users), this provides simple one-step syncing.
+
+Advanced users can disable this per-repo by just deleting the synced/master branch. Presumably the logic will be: Every repo that should not be pushed to, because it has access to some central repo, should not have a synced/master branch. Every other repo, including the (or one of the few) central repos, will have the branch.
+
+This is not the most expressive solution, as it does not allow configuring syncing between arbitrary pairs of repos, but it feels like a good compromise between that and simplicity and transparency.
+
+I think it's about time that I provide less talk and more code. I’ll see when I find the time :-)
+"""]]
diff --git a/doc/forum/purge_files_with_no_copies.mdwn b/doc/forum/purge_files_with_no_copies.mdwn
new file mode 100644
index 0000000000..5761d9290e
--- /dev/null
+++ b/doc/forum/purge_files_with_no_copies.mdwn
@@ -0,0 +1,3 @@
+Hi guys, as a result of some improper handling I am left with a couple of files with no copies scattered all around my repos. Now, while my data loss is unfortunate, it is not tragic. What I would like to do now is to purge all of these files without having to look for them one by one. Is there any easy way to do this? 
+
+Cheers
diff --git a/doc/forum/purge_files_with_no_copies/comment_1_12b578689eb8d5d38c06261ec65e2109._comment b/doc/forum/purge_files_with_no_copies/comment_1_12b578689eb8d5d38c06261ec65e2109._comment
new file mode 100644
index 0000000000..0ea81ecfe7
--- /dev/null
+++ b/doc/forum/purge_files_with_no_copies/comment_1_12b578689eb8d5d38c06261ec65e2109._comment
@@ -0,0 +1,12 @@
+[[!comment format=mdwn
+ username="http://joeyh.name/"
+ ip="4.153.14.105"
+ subject="comment 1"
+ date="2013-09-19T20:59:32Z"
+ content="""
+This will output all files in your repository that git-annex thinks have no copies left:
+
+`git annex find --not --copies=1`
+
+Piping it to git rm is left as an excercise for the reader..
+"""]]
diff --git a/doc/forum/pushing_to_android.mdwn b/doc/forum/pushing_to_android.mdwn
new file mode 100644
index 0000000000..030ca5c793
--- /dev/null
+++ b/doc/forum/pushing_to_android.mdwn
@@ -0,0 +1 @@
+Dear git-annex community, I've just installed git-annex on my CyanogenMod phone. What I'd like to do is to synchronize my music folder from my desktop to my phone. I.e. I'm using [sshelper](http://arachnoid.com/android/SSHelper/index.html) to start OpenSSH server on the phone. The question is, how can I make git and git-annex available to SSHelper's shell so that I can git push to the phone?
diff --git a/doc/forum/pushing_to_android/comment_1_1a7f6d21a3eef88d20d6f2bbad4c39d5._comment b/doc/forum/pushing_to_android/comment_1_1a7f6d21a3eef88d20d6f2bbad4c39d5._comment
new file mode 100644
index 0000000000..aa60bbee58
--- /dev/null
+++ b/doc/forum/pushing_to_android/comment_1_1a7f6d21a3eef88d20d6f2bbad4c39d5._comment
@@ -0,0 +1,19 @@
+[[!comment format=mdwn
+ username="joey"
+ subject="""comment 1"""
+ date="2016-01-20T19:17:47Z"
+ content="""
+Gosh, I don't know. Android's permission model is not really designed to
+make it very usable as a general unix system like this. Unfortunately.
+
+You might be able to get away with just copying the files from the
+git-annex app into the sheper app.
+
+I think that a better option though, would be to run git-annex on the phone
+and let it notice when a change is made to the repository, and pull.
+The best way to set that up is to use the web app to add a repository on a
+ssh server. As long as the ssh server is running git-annex 5.20140405
+or newer, the git-annex on the phone will be able to instantly notice when
+changes are pushed into the ssh server, and will pull them down to the
+phone.
+"""]]
diff --git a/doc/forum/question_about_assistant_and___47__archive__47__.mdwn b/doc/forum/question_about_assistant_and___47__archive__47__.mdwn
new file mode 100644
index 0000000000..af8cee4c9f
--- /dev/null
+++ b/doc/forum/question_about_assistant_and___47__archive__47__.mdwn
@@ -0,0 +1,22 @@
+I have a git-annex repository located at ~/annex which has been set up using git-annex assistant.
+
+This repository is configured as "client".
+
+My other repository is a huge USB drive configured as "full archive".
+
+Now everything seems to work fine except there is one thing I don't understand:
+
+    alip@client:~/annex> git-annex whereis ./archive/kus.png
+    whereis archive/kus.png (1 copy) 
+        e79a4cf6-4c48-4833-93de-98ba6eb625d6 -- deniz
+    ok
+
+Fine, there is only one copy according to git-annex but the file is still present
+in **this** client repository:
+
+    alip@client:> du -hs ~/annex/archive
+    20G	/home/alip/annex/archive/
+
+How do I free this space? Am I supposed to call git-annex drop manually?
+
+git-annex version: 3.20130124
diff --git a/doc/forum/question_about_assistant_and___47__archive__47__/comment_1_97890e26072af9277144651e3fdcada0._comment b/doc/forum/question_about_assistant_and___47__archive__47__/comment_1_97890e26072af9277144651e3fdcada0._comment
new file mode 100644
index 0000000000..2736016c54
--- /dev/null
+++ b/doc/forum/question_about_assistant_and___47__archive__47__/comment_1_97890e26072af9277144651e3fdcada0._comment
@@ -0,0 +1,10 @@
+[[!comment format=mdwn
+ username="https://www.google.com/accounts/o8/id?id=AItOawmBUR4O9mofxVbpb8JV9mEbVfIYv670uJo"
+ nickname="Justin"
+ subject="comment 1"
+ date="2013-01-30T18:44:52Z"
+ content="""
+Give this a try:
+
+http://git-annex.branchable.com/walkthrough/unused_data/
+"""]]
diff --git a/doc/forum/question_about_assistant_and___47__archive__47__/comment_2_542bf265e35a976ac76767762d67d617._comment b/doc/forum/question_about_assistant_and___47__archive__47__/comment_2_542bf265e35a976ac76767762d67d617._comment
new file mode 100644
index 0000000000..00cc063747
--- /dev/null
+++ b/doc/forum/question_about_assistant_and___47__archive__47__/comment_2_542bf265e35a976ac76767762d67d617._comment
@@ -0,0 +1,104 @@
+[[!comment format=mdwn
+ username="https://www.google.com/accounts/o8/id?id=AItOawkZktNHFhxC1kYA9KKdKpYJO4clq9WDsjE"
+ nickname="Jason"
+ subject="comment 2"
+ date="2013-01-30T19:14:48Z"
+ content="""
+I'm seeing the same thing and I think it's a bug.  Everything I'm reading says that when you move a file from anywhere into an archive directory, it's supposed to upload it to an archive and drop it locally.
+
+If I copy the file from outside my annex into the archive directory it will usually do exactly that and I'll end up with a symlink, but not always.  Sometimes I'll end up with a file.
+
+If I move the file from within my annex into the archive, everything gets confused.  Here is an example:
+
+    [jchu@april annex(master)]$ mkdir testdir
+    [jchu@april annex(master)]$ cd testdir/
+    [jchu@april testdir(master)]$ ls
+    [jchu@april testdir(master)]$ cat > test.txt << EOF
+    > I have a file.  It is very nice.
+    > EOF
+    [jchu@april testdir(master)]$ ls -l
+    total 4
+    -rw-r--r-- 1 jchu users 33 Jan 30 10:53 test.txt
+    [jchu@april testdir(master)]$ mkdir archive
+    [jchu@april testdir(master)]$ mv test.txt archive/
+    [jchu@april testdir(master)]$ ls -l
+    total 0
+    drwxr-xr-x 2 jchu users 21 Jan 30 10:54 archive
+    [jchu@april testdir(master)]$ ls -l archive/
+    total 4
+    -rw-r--r-- 1 jchu users 33 Jan 30 10:53 test.txt
+    [jchu@april testdir(master)]$ # Shouldn't this be a symlink?
+    [jchu@april testdir(master)]$ git show
+    commit 2fb2cba3fae98e0e0b8b7891b1af0cc655754896
+    Author: Jason Chu 
+    Date:   Wed Jan 30 10:54:11 2013 -0800
+
+    diff --git a/testdir/archive/test.txt b/testdir/archive/test.txt
+    new file mode 120000
+    index 0000000..5bd7982
+    --- /dev/null
+    +++ b/testdir/archive/test.txt
+    @@ -0,0 +1 @@
+    +../../.git/annex/objects/x5/Q1/SHA256E-s33--1859a57b635b0702dd3813dd7637f11842730b0464c35a4ad361dd0d37fc3dab.txt/SHA256E-s33--1859a57b635b0702dd3813dd7637f11842730b0464c35a4ad3
+    \ No newline at end of file
+    [jchu@april testdir(master)]$ git show HEAD^
+    commit 410f58f782348599d42496db2c9b21cfb2e59124
+    Author: Jason Chu 
+    Date:   Wed Jan 30 10:54:10 2013 -0800
+
+    diff --git a/testdir/test.txt b/testdir/test.txt
+    deleted file mode 120000
+    index 82343e3..0000000
+    --- a/testdir/test.txt
+    +++ /dev/null
+    @@ -1 +0,0 @@
+    -../.git/annex/objects/x5/Q1/SHA256E-s33--1859a57b635b0702dd3813dd7637f11842730b0464c35a4ad361dd0d37fc3dab.txt/SHA256E-s33--1859a57b635b0702dd3813dd7637f11842730b0464c35a4ad361d
+    \ No newline at end of file
+
+From what I can see here, the file was archived properly and the delete in git was issued correctly.  The daemon.log shows me that the file was uploaded correctly when it was initially created and then added to the archive directory, but I don't see a direct delete in the log file (even though the git log shows me it happened).
+
+Now on to the really weird part.  If I restart the daemon (or there are other ways to trigger it, but this seems to be the easiest for me), I see that log lines that say testdir/archive/test.txt was dropped (good) but then testdir/test.txt is re-downloaded and appears back in that directory.  I wanted the file deleted from testdir, why is it back?!?  Since it gets redownloaded and readded, my new logs look like this:
+
+    [jchu@april testdir(master)]$ git show
+    commit 850464dd2f8ae5cd407a214543afab668d4b34eb
+    Author: Jason Chu 
+    Date:   Wed Jan 30 11:00:03 2013 -0800
+
+    diff --git a/testdir/archive/test.txt b/testdir/archive/test.txt
+    deleted file mode 120000
+    index 5bd7982..0000000
+    --- a/testdir/archive/test.txt
+    +++ /dev/null
+    @@ -1 +0,0 @@
+    -../../.git/annex/objects/x5/Q1/SHA256E-s33--1859a57b635b0702dd3813dd7637f11842730b0464c35a4ad361dd0d37fc3dab.txt/SHA256E-s33--1859a57b635b0702dd3813dd7637f11842730b0464c35a4ad3
+    \ No newline at end of file
+    [jchu@april testdir(master)]$ git show HEAD^
+    commit 8cf9019e66e5a2cb0478e965fda352c742972591
+    Author: Jason Chu 
+    Date:   Wed Jan 30 11:00:00 2013 -0800
+
+    diff --git a/testdir/test.txt b/testdir/test.txt
+    new file mode 120000
+    index 0000000..82343e3
+    --- /dev/null
+    +++ b/testdir/test.txt
+    @@ -0,0 +1 @@
+    +../.git/annex/objects/x5/Q1/SHA256E-s33--1859a57b635b0702dd3813dd7637f11842730b0464c35a4ad361dd0d37fc3dab.txt/SHA256E-s33--1859a57b635b0702dd3813dd7637f11842730b0464c35a4ad361d
+    \ No newline at end of file
+
+And the filesystem looks like this:
+
+    [jchu@april testdir(master)]$ ls -lR
+    .:
+    total 4
+    drwxr-xr-x 2 jchu users 21 Jan 30 11:00 archive
+    -rw-r--r-- 1 jchu users 33 Jan 30 11:00 test.txt
+
+    ./archive:
+    total 4
+    -rw-r--r-- 1 jchu users 33 Jan 30 11:00 test.txt
+
+I have 3 remotes set up: a usb drive (client, it was created like that directly by annex when I created a Removable drive repo, it's a bare repo), s3 (archive), and nas (a remote server repo, that connects over ssh and uses rsync).  I thought maybe that the usb drive still had a record of testdir/test.txt existing, but the logs in refs/heads/synced/master match.
+
+It's as if the git annex thinks that the original content is still within a non-archive directory, so the data can't be dropped, but everywhere I look the references to that content are gone.  The only way I've found to fix this is to convert to indirect mode and delete the symlinks from there (which seems to work better).  It seems to be an interaction between direct mode, the archive, and maybe my filesystem (xfs).
+"""]]
diff --git a/doc/forum/question_about_assistant_and___47__archive__47__/comment_3_bafe99159df2adcd5fecc0d67bbf05a5._comment b/doc/forum/question_about_assistant_and___47__archive__47__/comment_3_bafe99159df2adcd5fecc0d67bbf05a5._comment
new file mode 100644
index 0000000000..b1c9e1774e
--- /dev/null
+++ b/doc/forum/question_about_assistant_and___47__archive__47__/comment_3_bafe99159df2adcd5fecc0d67bbf05a5._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="http://joeyh.name/"
+ ip="4.154.3.125"
+ subject="comment 3"
+ date="2013-02-05T17:46:56Z"
+ content="""
+@alip I've just fixed a bug which made the assistant get location tracking info wrong for files it added in direct mode. That probably explains what you're seeing. You can run `git annex fsck` to fix the location log info, and if you restart the assistant it'll probably then realize the files are locally present and drop them. Of course, the USB drive has to be connected for that to work; git-annex needs to verify the files have been moved to it before it can drop them.
+"""]]
diff --git a/doc/forum/question_about_assistant_and___47__archive__47__/comment_4_e77fa2992d9302a49a05f514c81612ca._comment b/doc/forum/question_about_assistant_and___47__archive__47__/comment_4_e77fa2992d9302a49a05f514c81612ca._comment
new file mode 100644
index 0000000000..3f78b4813f
--- /dev/null
+++ b/doc/forum/question_about_assistant_and___47__archive__47__/comment_4_e77fa2992d9302a49a05f514c81612ca._comment
@@ -0,0 +1,10 @@
+[[!comment format=mdwn
+ username="http://joeyh.name/"
+ ip="4.154.3.125"
+ subject="comment 4"
+ date="2013-02-05T19:05:49Z"
+ content="""
+@Jason That is a bug.  Please don't report what are obviously bugs in the forum! 
+
+I've posted a bug report about it here: [[bugs/direct_mode_renames]]
+"""]]
diff --git a/doc/forum/race_condition_with_drop_-J__63__.mdwn b/doc/forum/race_condition_with_drop_-J__63__.mdwn
new file mode 100644
index 0000000000..77ece255e0
--- /dev/null
+++ b/doc/forum/race_condition_with_drop_-J__63__.mdwn
@@ -0,0 +1,7 @@
+When I run a command like: "git-annex drop . -J 20"
+
+I get: "git-annex: content is locked"
+
+Running with -J 5 works.
+
+EDIT: Even as low as -J 2 runs into problems. The issue is simply more likely to trigger with more threads.
diff --git a/doc/forum/race_condition_with_drop_-J__63__/comment_1_0607c11410d99a6933e02c28b54c1f65._comment b/doc/forum/race_condition_with_drop_-J__63__/comment_1_0607c11410d99a6933e02c28b54c1f65._comment
new file mode 100644
index 0000000000..e6d2357142
--- /dev/null
+++ b/doc/forum/race_condition_with_drop_-J__63__/comment_1_0607c11410d99a6933e02c28b54c1f65._comment
@@ -0,0 +1,12 @@
+[[!comment format=mdwn
+ username="joey"
+ subject="""comment 1"""
+ date="2016-05-04T17:13:55Z"
+ content="""
+This can happen if the working tree contains two files that both point to
+the same content. Since git-annex is dropping multiple files in parallel,
+it might try to drop both files at the same time. One of the drops will
+succeed, and the other fail as shown.
+
+It should be nothing to worry about since the content does get dropped.
+"""]]
diff --git a/doc/forum/rebuild_location_log_from_encrypted_remote.mdwn b/doc/forum/rebuild_location_log_from_encrypted_remote.mdwn
new file mode 100644
index 0000000000..bf3f0e830c
--- /dev/null
+++ b/doc/forum/rebuild_location_log_from_encrypted_remote.mdwn
@@ -0,0 +1,35 @@
+Hi,
+
+I apologize, but I can't retrace my steps, so my problem is not replicable. The current situation is: 1 main repo and 1 directory/encrypted special remote. From the main repo, I see 0 available copy of the files I need. But when I do a fsck I do see the file. The problem is that the location log does not seem to get fixed after fsck, as I cannot use git-annex get to obtain the file, even after the fsck command. 
+
+Do you have any suggestions for next diagnostic/repair steps?
+
+Thanks a lot for your time and answer!
+
+Vincent
+
+macbook% git annex fsck foo.zip --from needle
+fsck foo.zip (gpg) 
+You need a passphrase to unlock the secret key for
+user: “me ”
+2048-bit RSA key, ID ???????, created 2013-01-11 (main key ID ????????)
+
+(checking needle...) 
+GPGHMACSHA1--d7036fd17214c06c83d53479dcec9554a7dc2109
+    13815014 100%   14.16MB/s    0:00:00 (xfer#1, to-check=0/1)
+
+sent 13816832 bytes  received 42 bytes  9211249.33 bytes/sec
+total size is 13815014  speedup is 1.00
+(fixing location log) (checksum...) 
+  ** No known copies exist of foo.zip
+failed
+(Recording state in git...)
+git-annex: fsck: 1 failed
+
+
+macbook% git annex get foo.zip
+get foo.zip (not available) 
+  No other repository is known to contain the file.
+failed
+git-annex: get: 1 failed
+macbook% 
diff --git a/doc/forum/rebuild_location_log_from_encrypted_remote/comment_1_f84f955fed7b96ae6208b6ff2ec650cd._comment b/doc/forum/rebuild_location_log_from_encrypted_remote/comment_1_f84f955fed7b96ae6208b6ff2ec650cd._comment
new file mode 100644
index 0000000000..af3a411593
--- /dev/null
+++ b/doc/forum/rebuild_location_log_from_encrypted_remote/comment_1_f84f955fed7b96ae6208b6ff2ec650cd._comment
@@ -0,0 +1,12 @@
+[[!comment format=mdwn
+ username="http://joeyh.name/"
+ ip="209.250.56.64"
+ subject="comment 1"
+ date="2013-12-01T19:59:16Z"
+ content="""
+I think you should try to run `git annex copy --from needle`, which will be able to get the files if needle has them, even if the location log is busted. 
+
+(Sometimes people complain that `git annex copy` goes and checks a remote, which can be unncessarily slow, rather than just trusting the location log  ... but this is why it does it!)
+
+From what I can tell, the fsck downloads some file from the remote, but the download is not entirely successful. One possibility is that it downloads a file, but the file fails to be decrypted, either because you don't have the right encyption key, or because the encrypted file has actually gotten corrupted on the remote. It might be that the same problem will affect `git annex copy --from needle`, but if so it should be more clear what's happening.
+"""]]
diff --git a/doc/forum/rebuild_location_log_from_encrypted_remote/comment_2_c0b2ce3bc7cd55a0c77ddc31493068c2._comment b/doc/forum/rebuild_location_log_from_encrypted_remote/comment_2_c0b2ce3bc7cd55a0c77ddc31493068c2._comment
new file mode 100644
index 0000000000..10b1c038ef
--- /dev/null
+++ b/doc/forum/rebuild_location_log_from_encrypted_remote/comment_2_c0b2ce3bc7cd55a0c77ddc31493068c2._comment
@@ -0,0 +1,10 @@
+[[!comment format=mdwn
+ username="https://www.google.com/accounts/o8/id?id=AItOawmLD-g4d-EGXD5KJh1kVKnRX0a9dTnEigg"
+ nickname="Vincent"
+ subject="comment 2"
+ date="2013-12-02T01:28:18Z"
+ content="""
+Thanks a lot for the quick response! Unfortunately, copy is just silent and does not copy the files over.
+
+I don't want to waste more of your time on this. I just wish I had kept better track of the steps I took to get myself into this mess...
+"""]]
diff --git a/doc/forum/rebuild_location_log_from_encrypted_remote/comment_3_06a73ca3dc73399ff000b642cca72de7._comment b/doc/forum/rebuild_location_log_from_encrypted_remote/comment_3_06a73ca3dc73399ff000b642cca72de7._comment
new file mode 100644
index 0000000000..9cd4424270
--- /dev/null
+++ b/doc/forum/rebuild_location_log_from_encrypted_remote/comment_3_06a73ca3dc73399ff000b642cca72de7._comment
@@ -0,0 +1,10 @@
+[[!comment format=mdwn
+ username="http://joeyh.name/"
+ ip="209.250.56.64"
+ subject="comment 3"
+ date="2013-12-02T19:53:11Z"
+ content="""
+Hmm, it seems I was wrong about copy --from not checking the location log. It does.. Probably to avoid large copies trying to copy a lot of files that are not present in a remote.
+
+I have now made `git annex copy --force --from remote` do what I said it would do, and that should work in your situation, I think.
+"""]]
diff --git a/doc/forum/receiving_indirect_renames_on_direct_repo___63__.mdwn b/doc/forum/receiving_indirect_renames_on_direct_repo___63__.mdwn
new file mode 100644
index 0000000000..51a3aa7b64
--- /dev/null
+++ b/doc/forum/receiving_indirect_renames_on_direct_repo___63__.mdwn
@@ -0,0 +1,254 @@
+I've been playing with a mixed setup, and I frequentely end up with conflicts which can be ascribed to mixing direct windows repos with indirect linux one(s), and making renames on the indirect ones.
+Possibly someone can address what I miss of the git-annex/git interaction.  
+
+#### versions involved ####
+
+linux: git-annex version: 4.20131024
+win: git-annex version: 4.20131024-gca7b71e
+
+### Steps to reproduce behaviour ###
+
+###### 1. On linux, i setup bare origin "casa" and client repo "local":
+    
+    [michele@home ~]$ git init --bare casa
+    Initialized empty Git repository in /home/sambahome/michele/casa/
+    [michele@home ~]$ cd casa
+    [michele@home casa]$ git annex init casa
+    init casa ok
+    (Recording state in git...)
+    [michele@home ~]$ cd ..; git clone casa local
+    Cloning into 'local'...
+    done.
+    warning: remote HEAD refers to nonexistent ref, unable to checkout.
+    [michele@home ~]$ cd local; git annex init local
+    init local ok
+    (Recording state in git...)
+    [michele@home local]$ echo lintest > lintest
+    [michele@home local]$ git annex add lintest
+    add lintest (checksum...) ok
+    (Recording state in git...)
+    [michele@home local]$ git annex sync
+    (merging origin/git-annex into git-annex...)
+    (Recording state in git...)
+    commit  
+    ok
+    pull origin 
+    ok
+    push origin 
+    Counting objects: 18, done.
+    Delta compression using up to 4 threads.
+    Compressing objects: 100% (12/12), done.
+    Writing objects: 100% (16/16), 1.48 KiB | 0 bytes/s, done.
+    Total 16 (delta 1), reused 0 (delta 0)
+    To /home/sambahome/michele/casa
+     * [new branch]      git-annex -> synced/git-annex
+     * [new branch]      master -> synced/master
+    ok
+    ```
+    
+###### 2. On windows I clone origin, and I sync empty
+
+    ```cmd
+    M:\>git clone ssh://michele@home/home/michele/casa win
+    Cloning into 'win'...
+    remote: Counting objects: 20, done.
+    remote: Compressing objects: 100% (15/15), done.
+    remote: Total 20 (delta 3), reused 0 (delta 0)
+    Receiving objects: 100% (20/20), done.
+    Resolving deltas: 100% (3/3), done.
+    M:\>cd win
+    M:\win>git annex status
+      Detected a crippled filesystem.
+      Enabling direct mode.
+      Detected a filesystem without fifo support.
+      Disabling ssh connection caching.
+    repository mode: direct
+    trusted repositories: (merging origin/git-annex origin/synced/git-annex into git-annex...)
+    (Recording state in git...)
+    0
+    semitrusted repositories: 4
+            00000000-0000-0000-0000-000000000001 -- web
+            598ecfac-087d-49a3-b48d-beafd0d71805 -- origin (casa)
+            b2699c17-d0bc-40a0-b447-a64ad109b2a2 -- here (ALICUDI:M:\win)
+            bd4166eb-296b-4f0f-a3be-6c25e4c7cbb0 -- local
+    untrusted repositories: 0
+    transfers in progress: none
+    available local disk space: unknown
+    local annex keys: 0
+    local annex size: 0 bytes
+    known annex keys: 1
+    known annex size: 8 bytes
+    bloom filter size: 16 mebibytes (0% full)
+    backend usage:
+            SHA256E: 1
+    ```
+    
+###### 3. I copy content from local to win
+
+    ```bash
+    [michele@home local]$ git annex copy --to origin lintest
+    copy lintest (to origin...) ok
+    (Recording state in git...)
+    [michele@home local]$ git annex sync
+    ...runs ok...
+    ```
+
+###### 4. and
+
+    ```cmd
+    M:\win>git annex sync
+    ...works
+    M:\win>git annex get .
+    ...works
+    M:\win>cat lintest
+    lintest
+    ```
+
+so far so good.
+###### 5. Now the renaming part (performed on linux indirect repo)
+
+    ```bash
+    [michele@home local]$ git mv lintest renamed
+    [michele@home local]$ git annex list
+    here
+    |origin
+    ||web
+    |||
+    XX_ renamed
+    [michele@home local]$ git annex sync
+    ...works
+    ```
+
+###### 6.  now, by issuing sync on windows I start getting a "push issue":
+
+    ```
+    M:\win>git annex sync
+    commit
+    ok
+    pull origin
+    remote: Counting objects: 3, done.
+    remote: Total 2 (delta 0), reused 0 (delta 0)
+    Unpacking objects: 100% (2/2), done.
+    From ssh://home/home/michele/casa
+       c3b7a63..a0854bf  master     -> origin/master
+       c3b7a63..a0854bf  synced/master -> origin/synced/master
+    ok
+    push origin
+    Counting objects: 9, done.
+    Delta compression using up to 2 threads.
+    Compressing objects: 100% (4/4), done.
+    Writing objects: 100% (5/5), 484 bytes, done.
+    Total 5 (delta 1), reused 0 (delta 0)
+    To ssh://michele@home/home/michele/casa
+       6c18669..8cc74a0  git-annex -> synced/git-annex
+     ! [rejected]        master -> synced/master (non-fast-forward)
+    error: failed to push some refs to 'ssh://michele@home/home/michele/casa'
+    hint: Updates were rejected because a pushed branch tip is behind its remote
+    hint: counterpart. Check out this branch and merge the remote changes
+    hint: (e.g. 'git pull') before pushing again.
+    hint: See the 'Note about fast-forwards' in 'git push --help' for details.
+    failed
+    git-annex: sync: 1 failed
+    
+    M:\win>
+    ```
+
+at this stage I tried to issue a git annex merge, git annex sync, leading to same result.
+
+somewhere in the forum I read i could try issuing a git pull origin master (this could be the problem).
+and the result is as such:
+
+    ```
+    M:\win>git pull master
+    fatal: 'master' does not appear to be a git repository
+    fatal: The remote end hung up unexpectedly
+    
+    M:\win>git pull origin master
+    From ssh://home/home/michele/casa
+     * branch            master     -> FETCH_HEAD
+    Updating c3b7a63..a0854bf
+    error: Your local changes to the following files would be overwritten by merge:
+            lintest
+    Please, commit your changes or stash them before you can merge.
+    Aborting
+    
+    M:\win>git status
+    # On branch master
+    # Your branch is behind 'origin/master' by 1 commit, and can be fast-forwarded.
+    #
+    # Changes not staged for commit:
+    #   (use "git add ..." to update what will be committed)
+    #   (use "git checkout -- ..." to discard changes in working directory)
+    #
+    #       modified:   lintest
+    #
+    no changes added to commit (use "git add" and/or "git commit -a")
+    
+    M:\win>cat lintest
+    lintest
+    ```
+    
+well, ok it appears modified for some weirdness (crlf?), we can live with it.
+
+    ```
+    M:\win>git checkout -->> this replaces files contents with simlink (due to git pull above?)
+    ```
+
+at this stage content is lost, and annex has no knowledge about it.
+
+    ```
+    M:\win>git annex fsck
+    fsck lintest ok
+    
+    M:\win>cat lintest
+    .git/annex/objects/9Z/82/SHA256E-s8--2b721dbe9afe6031cce3004e909dd62e0b4b2f3944438b6a000dffc7ad657715/SHA256E-s8--2b721dbe9afe6031cce3004e90
+    M:\win>git annex list
+    here
+    |origin
+    ||web
+    |||
+    XX_ lintest
+    ```
+
+still I cannot sync, but now i can pull origin master:
+
+    ```cmd
+    M:\win>git pull origin master                          From ssh://home/home/michele/casa                       * branch            master     -> FETCH_HEAD          Updating c3b7a63..a0854bf                              Fast-forward                                            lintest => renamed | 0                                 1 file changed, 0 insertions(+), 0 deletions(-)        rename lintest => renamed (100%)                      
+    ```
+
+this doesnt restore content (annex thinks its already there:
+
+    ```
+    M:\win>cat renamed
+    .git/annex/objects/9Z/82/SHA256E-s8--2b721dbe9afe6031cce3004e909dd62e0b4b2f3944438b6a000dffc7ad657715/SHA256E-s8--2b721dbe9afe6031cce3004e909dd62e0b4b2f3944438b6a000dffc7ad657715
+    ```
+
+I think my mistake is the use of the ```git checkout``` in direct mode. But why is the file detected as modified in the first place ?
+
+note: as long as i didn't drop on origin, i still can recover contents by 'forcing' a content refresh:
+
+    ```
+    M:\win>git annex get renamed --from origin
+    get renamed (from origin...)
+    SHA256E-s8--2b721dbe9afe6031cce3004e909dd62e0b4b2f3944438b6a000dffc7ad657715
+               8 100%    7.81kB/s    0:00:00 (xfer#1, to-check=0/1)
+    
+    sent 30 bytes  received 153 bytes  8.13 bytes/sec
+    total size is 8  speedup is 0.04
+    ok
+    (Recording state in git...)
+    
+    M:\win>cat renamed
+    .git/annex/objects/9Z/82/SHA256E-s8--2b721dbe9afe6031cce3004e909dd62e0b4b2f3944438b6a000dffc7ad657715/SHA256E-s8--2b721dbe9afe6031cce3004e909d
+    M:\win>git annex fsck
+    fsck renamed (fixing direct mode) (checksum...) ok
+    
+    M:\win>cat renamed
+    lintest
+```
+
+
+
+
+
+
diff --git a/doc/forum/receiving_indirect_renames_on_direct_repo___63__/comment_1_f4b0a14373c75cb752597c832e296bcc._comment b/doc/forum/receiving_indirect_renames_on_direct_repo___63__/comment_1_f4b0a14373c75cb752597c832e296bcc._comment
new file mode 100644
index 0000000000..7b38c6af4d
--- /dev/null
+++ b/doc/forum/receiving_indirect_renames_on_direct_repo___63__/comment_1_f4b0a14373c75cb752597c832e296bcc._comment
@@ -0,0 +1,17 @@
+[[!comment format=mdwn
+ username="http://joeyh.name/"
+ ip="209.250.56.47"
+ subject="comment 1"
+ date="2013-10-26T19:35:46Z"
+ content="""
+You should **never, ever, ever** run `git pull`, or `git checkout` inside a direct repository. You need to read [[direct_mode]]. If you do not fully understand it, you should avoid running any git commands that are not `git annex foo` in a direct mode repository. If you forget and do run such git commands, you can generally recover by running `git annex fsck`, although it's possible that the git command you run overwrites your only copy of a file, and so you'd lose it.
+
+
    +To ssh://michele@home/home/michele/casa
+   6c18669..8cc74a0  git-annex -> synced/git-annex
+ ! [rejected]        master -> synced/master (non-fast-forward)
+error: failed to push some refs to 'ssh://michele@home/home/michele/casa'
+
    
+
+I'll bet that if you look at the `git config` of this repository it failed to push to, you'll find that it has `receive.denyNonFastforwards` set to true. If you unset that, the push should work.
+"""]]
diff --git a/doc/forum/receiving_indirect_renames_on_direct_repo___63__/comment_2_8c86dfc99f0b9040402c9d746decda53._comment b/doc/forum/receiving_indirect_renames_on_direct_repo___63__/comment_2_8c86dfc99f0b9040402c9d746decda53._comment
new file mode 100644
index 0000000000..8b4ff46a40
--- /dev/null
+++ b/doc/forum/receiving_indirect_renames_on_direct_repo___63__/comment_2_8c86dfc99f0b9040402c9d746decda53._comment
@@ -0,0 +1,41 @@
+[[!comment format=mdwn
+ username="https://www.google.com/accounts/o8/id?id=AItOawkJafmCf-sg9_OM0pynFYM3AO4WCgJiaMI"
+ nickname="Michele"
+ subject="still cannot push when remote has renames"
+ date="2013-10-27T23:06:03Z"
+ content="""
+now, I went again through docs, and i realized how stupid was issuing a git pull on a direct repo. thanks for your patience.
+
+but, i double checked the configuration, I assume \"receive.denyNonFastForwards\" is false by default, but anyway I set it up explicitely so that now my git config (on the linux indirect repo - with respect to my previous example, I got rid of the \"extra\" bare repo in the middle) shows:
+
+    $ git config --list
+    user.email=m@g.com
+    user.name=michele
+    core.repositoryformatversion=0
+    core.filemode=true
+    core.bare=false
+    core.logallrefupdates=true
+    annex.uuid=d084e0fd-95a7-4c98-a206-fbf2c85b779d
+    annex.version=3
+    receive.denynonfastforwards=false
+
+still I am receiving the push refusal:
+
+    M:\win>git annex sync
+    commit
+    ok
+    pull origin
+    ok
+    push origin
+    To ssh://michele@home/home/michele/casa
+       ! [rejected]        master -> synced/master (non-fast-forward)
+    error: failed to push some refs to 'ssh://michele@home/home/michele/casa'
+    hint: Updates were rejected because a pushed branch tip is behind its remote
+    hint: counterpart. Check out this branch and merge the remote changes
+    hint: (e.g. 'git pull') before pushing again.
+    hint: See the 'Note about fast-forwards' in 'git push --help' for details.
+    failed
+    git-annex: sync: 1 failed
+
+Same happens with a bare repository in the middle. BTW: the windows \"client\" repository is behind NAT, so that the linux indirect doesn't actively sync against it: could that be source of the problem ?
+"""]]
diff --git a/doc/forum/receiving_indirect_renames_on_direct_repo___63__/comment_3_0246fff6c7c75f6be45bd257ec3872a5._comment b/doc/forum/receiving_indirect_renames_on_direct_repo___63__/comment_3_0246fff6c7c75f6be45bd257ec3872a5._comment
new file mode 100644
index 0000000000..0ffb2a0972
--- /dev/null
+++ b/doc/forum/receiving_indirect_renames_on_direct_repo___63__/comment_3_0246fff6c7c75f6be45bd257ec3872a5._comment
@@ -0,0 +1,75 @@
+[[!comment format=mdwn
+ username="https://www.google.com/accounts/o8/id?id=AItOawkJafmCf-sg9_OM0pynFYM3AO4WCgJiaMI"
+ nickname="Michele"
+ subject="possible explanation"
+ date="2013-10-29T11:56:21Z"
+ content="""
+now, i tried to understand what happens. Instead of issuing the *git annex sync*, I relied on *git pull origin, git merge origin/master*, (I red [[http://git-annex.branchable.com/forum/Help_Windows_walkthrough/]] and I assume that pull origin / merge origin/master would work similarly to the \"download\" part of sync, *except for losing all my direct content*) just to understand what was going on, with a clarifying result:
+
+while git annex sync fails on push, git pull origin fails on pull:
+
+    M:\win>git pull origin
+    Updating 5408d6f..c566a69
+    error: Your local changes to the following files would be overwritten by merge:
+            myfile
+    Please, commit your changes or stash them before you can merge.
+    Aborting 
+
+note that the file has not been modified locally (just got it through git annex get).
+issuing a git diff, reveals:
+
+    M:\win>git diff myfile
+    diff --git a/myfile b/myfile
+    index beaf3e8..dc5b4ff 120000
+    --- a/myfile
+    +++ b/myfile
+    @@ -1 +1 @@
+    -.git/annex/objects/z5/v7/SHA256E-s8--6090923ed0931dcc6699f32fb66fa4ba32c10924088b12c66fb4ce35a91ba98c/SHA256E-s8\ No newline at end of file
+    +linux.1
+    (END)
+
+ok, i follow suggestion, and I perform a git stash. that still wouldn't suffice for git annex sync:
+
+    M:\win>git annex sync
+    commit
+    ok
+    pull origin
+    ok
+    push origin
+    To ssh://michele@home/home/michele/homebase
+     ! [rejected]        master -> synced/master (non-fast-forward)
+    error: failed to push some refs to 'ssh://michele@home/home/michele/homebase'
+    hint: Updates were rejected because a pushed branch tip is behind its remote
+    hint: counterpart. Check out this branch and merge the remote changes
+    hint: (e.g. 'git pull') before pushing again.
+    hint: See the 'Note about fast-forwards' in 'git push --help' for details.
+    failed
+    git-annex: sync: 1 failed
+
+now, i can perform instead a *git pull origin*, since I am confident my content is stashed.
+
+    M:\win>git pull origin
+    Updating 5408d6f..c566a69
+    Fast-forward
+     myfile => myfile.renamed | 0
+     1 file changed, 0 insertions(+), 0 deletions(-)
+     rename myfile => myfile.renamed (100%)
+
+merge is not doing anything more: at this stage content has gone (file is a direct-mode symlink nad it cannot be fixed by fsck). 
+But i can recover it from stash (and I must do it unless I want to get the annex to think i still have content).
+
+    git stash apply 
+
+voilà: the content is there! and the repos seems in good order. 
+this only adds up that this is possibily a bug in the fact that git reports direct content as modified when indeed it hasn't been modified: but this affects git annex sync only when merging renaming files.
+git annex sync now works perfectly .
+
+to sum it up, I have two questions: 
+
+1) does using stash to circumvent the problem expose me to any risk ?
+2) would the behaviour on receiving renames in the abovementioned situation worth to be signaled as a bug ? 
+
+
+
+
+"""]]
diff --git a/doc/forum/recover_deleted_files___63__.mdwn b/doc/forum/recover_deleted_files___63__.mdwn
new file mode 100644
index 0000000000..7bec369685
--- /dev/null
+++ b/doc/forum/recover_deleted_files___63__.mdwn
@@ -0,0 +1,66 @@
+hi, 
+
+i think of use git-annex as the backbone of a archival systems. at first point no distributed storage, just 1 node.
+but now i run into the topic below ( deleted the "named" symlink of the "object" -- how to recover ?)
+
+maybe someone can enlighten me...
+
+thanks,
+.ka
+
+// about the version. ( debian-squeeze, bpo )
+
+$ git-annex version
+git-annex version: 3.20120629~bpo60+2
+local repository version: 3
+default repository version: 3
+supported repository versions: 3
+upgrade supported from repository versions: 0 1 2
+
+// building up a testcase.
+
+$ git init
+Initialized empty Git repository in ...test2/.git/
+
+$ git annex init
+init  ok
+(Recording state in git...)
+
+$ echo "aaa" > 1.txt
+
+$ echo "bbb" > 2.txt
+
+$ git-annex add .
+add 1.txt (checksum...) ok
+add 2.txt (checksum...) ok
+(Recording state in git...)
+
+$ git commit -a -m "added 2 files"
+fatal: No HEAD commit to compare with (yet)
+fatal: No HEAD commit to compare with (yet)
+[master (root-commit) fc2a5d7] added 2 files
+ Committer: userhere user 
+Your name and email address were configured automatically based
+on your username and hostname. Please check that they are accurate.
+...
+ 2 files changed, 2 insertions(+), 0 deletions(-)
+ create mode 120000 1.txt
+ create mode 120000 2.txt
+
+// ok, so far standard. i have now 2 files - lets delete one.
+
+$ rm 2.txt 
+$ ls -l
+lrwxrwxrwx 1 xp xp 176 24. Okt 22:55 1.txt -> .git/annex/objects/Z6/7q/SHA256-s4--17e682f060b5f8e47ea04c5c4855908b0a5ad612022260fe50e11ecb0cc0ab76/SHA256-s4--17e682f060b5f8e47ea04c5c4855908b0a5ad612022260fe50e11ecb0cc0ab76
+
+// eek, delete of 2.txt was a bad idea (it was just the symlink) -- try to recover...
+
+$ git-annex fix
+$ git-annex fsck
+fsck 1.txt (checksum...) ok
+$ ls
+1.txt
+
+// still not here.. how to recover the link to 2.txt ???
+// i still see the content of the file in the object folder
+// if I want to use git-annex as the backend of a archival system, this is important.
diff --git a/doc/forum/recover_deleted_files___63__/comment_1_d7abb7c45c6ec2723a04f153ed215453._comment b/doc/forum/recover_deleted_files___63__/comment_1_d7abb7c45c6ec2723a04f153ed215453._comment
new file mode 100644
index 0000000000..646e7ad540
--- /dev/null
+++ b/doc/forum/recover_deleted_files___63__/comment_1_d7abb7c45c6ec2723a04f153ed215453._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="http://ciffer.net/~svend/"
+ ip="2001:1938:81:1ff::2"
+ subject="git checkout"
+ date="2012-10-24T22:50:05Z"
+ content="""
+You can use `git checkout -- file` to restore the link. `git status` will display information for restoring the file.
+"""]]
diff --git a/doc/forum/recover_deleted_files___63__/comment_2_8ea2acaa30d3ee7e9f75310f4ec859b2._comment b/doc/forum/recover_deleted_files___63__/comment_2_8ea2acaa30d3ee7e9f75310f4ec859b2._comment
new file mode 100644
index 0000000000..c99461ae0b
--- /dev/null
+++ b/doc/forum/recover_deleted_files___63__/comment_2_8ea2acaa30d3ee7e9f75310f4ec859b2._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="http://joeyh.name/"
+ ip="4.152.108.138"
+ subject="it's a git repository..."
+ date="2012-10-25T03:29:20Z"
+ content="""
+So any git stuff can be used. If you deleted a file, committed it, and want to undo that, you can `git log --stat` to find the commit, and `git revert` it. If you deleted a file, haven't committed yet, and want one more look at it, you can `git stash` to get it back, and `git stash apply` to re-stash the change. Or yeah, just `git checkout` to get back a deleted file you have not committed yet.
+"""]]
diff --git a/doc/forum/recover_deleted_files___63__/comment_3_376de81c70799bf409be189a48234815._comment b/doc/forum/recover_deleted_files___63__/comment_3_376de81c70799bf409be189a48234815._comment
new file mode 100644
index 0000000000..89edb17eb6
--- /dev/null
+++ b/doc/forum/recover_deleted_files___63__/comment_3_376de81c70799bf409be189a48234815._comment
@@ -0,0 +1,12 @@
+[[!comment format=mdwn
+ username="ka7"
+ ip="2001:7b8:155d:0:222:64ff:fe16:dc52"
+ subject="ok, that worked."
+ date="2012-10-25T20:15:26Z"
+ content="""
+i think of a kind of \"WORM-library\", so basically just add, not allow to remove content. (at least not for the user thru the mounted device)
+- so a script to add/commit -- but as stag-1 check for delete files and get them back. some git magic needed, but should be doable.
+- or thru \"samba\" parameters set to add but not delete/overwrite files.  (read yes, write yes, delete no) -- to be proved thats possible, but not your job :) ( annex-ing via cron every /5 or via inotify)
+so yea, will play for a while and maybe come back with new. thanks to everybody. 
+<3 git-annex <3
+"""]]
diff --git a/doc/forum/recover_deleted_files___63__/comment_4_2e73ac530d65a01768a57058b7220a29._comment b/doc/forum/recover_deleted_files___63__/comment_4_2e73ac530d65a01768a57058b7220a29._comment
new file mode 100644
index 0000000000..9e9c1b4bf2
--- /dev/null
+++ b/doc/forum/recover_deleted_files___63__/comment_4_2e73ac530d65a01768a57058b7220a29._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="https://www.google.com/accounts/o8/id?id=AItOawnXlmnJKsPulm2S_hwwRm3Ky27Zyf-wKMw"
+ nickname="Chad"
+ subject="After the fact..."
+ date="2013-12-22T03:51:36Z"
+ content="""
+I've run into a situation where I've inadvertently deleted a file from my Andriod-driven annex repo rather than dropping the file. (A set of very large *.flac files, to be exact.) These are now gone from the repo on my laptop and the repo on my workstation. When looking at the `git --log stat' call, I see the addition of these files, but I do not see their deletion. When running `git annex unused', I found references to the *.flac files, their SHA key, and indeed found the commit they were associated with. I have not dropped these unused files. How to I recover them?
+"""]]
diff --git a/doc/forum/recovering_from_repo_corruption.mdwn b/doc/forum/recovering_from_repo_corruption.mdwn
new file mode 100644
index 0000000000..42df61b789
--- /dev/null
+++ b/doc/forum/recovering_from_repo_corruption.mdwn
@@ -0,0 +1,11 @@
+In my current annex config, I have 4 computers with "traditional" git annexes as well as an external drive that is a git annex, an rsync'd backup annex, and a glacier archive.  Today, one of the computers got a corrupted git repo. It was complaining that a pack file was invalid. In my attempts to fix it, a commit was logged that deleted every file in the annex. I didn't find this out until I did 'git annex sync' and watched git delete everything, then send all those commits to my other 3 systems and the external drive. *facepalm*
+
+Fortunately, I had one of those other systems in direct mode and I copied everything from the annex as a backup. Now, when I try to re-add files to the annex, I'm running into some errors. These appear to be "collisions" within the annex part of the .git folder:
+
+    %  › git annex add House.netspd 
+    add House.netspd (checksum...) 
+    git-annex: /Users/akraut/Desktop/annex/.git/annex/objects/31/Gw/SHA256E-s167433--41e68ea0adb5a4086a0b7b39d0556b9b86523ffb6b498d58f12f96460da315e9/SHA256E-s167433--41e68ea0adb5a4086a0b7b39d0556b9b86523ffb6b498d58f12f96460da315e9.map.tmp62699: openFile: permission denied (Permission denied)
+    failed
+    git-annex: add: 1 failed
+
+Any ideas on what's going on here? Perhaps how to get things added back in or recovered? It seems all the actual file contents are here, but annex doesn't seem to know they're there anymore.
diff --git a/doc/forum/recovering_from_repo_corruption/comment_1_01fc85037e24fc70e5c5329898cf6781._comment b/doc/forum/recovering_from_repo_corruption/comment_1_01fc85037e24fc70e5c5329898cf6781._comment
new file mode 100644
index 0000000000..ab65b42cbc
--- /dev/null
+++ b/doc/forum/recovering_from_repo_corruption/comment_1_01fc85037e24fc70e5c5329898cf6781._comment
@@ -0,0 +1,15 @@
+[[!comment format=mdwn
+ username="http://edheil.wordpress.com/"
+ ip="99.54.57.201"
+ subject="comment 1"
+ date="2013-01-26T07:11:35Z"
+ content="""
+speaking strictly from the git point of view, you can create a new commit that reverses an old commit completely.
+
+    git revert 
+
+creates a new commit which does the exact opposite of the commit you specify.  Theoretically it bring everything back to you quick and clean!
+
+
+
+"""]]
diff --git a/doc/forum/recovering_from_repo_corruption/comment_2_3bd1c0bf25a0e892e711a60f53cd5298._comment b/doc/forum/recovering_from_repo_corruption/comment_2_3bd1c0bf25a0e892e711a60f53cd5298._comment
new file mode 100644
index 0000000000..c8df9c0a06
--- /dev/null
+++ b/doc/forum/recovering_from_repo_corruption/comment_2_3bd1c0bf25a0e892e711a60f53cd5298._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="https://www.google.com/accounts/o8/id?id=AItOawmRFKwny4rArBaz-36xTcsJYqKIgdDaw5Q"
+ nickname="Andrew"
+ subject="comment 2"
+ date="2013-01-26T08:13:34Z"
+ content="""
+Oh! Good call. That worked, though I did have to sync and revert alternately a few times to get things resolved.  I still haven't figured out what exactly happened, though. In two separate instances, the commit message was \"git-annex automatic sync\"
+"""]]
diff --git a/doc/forum/recovering_from_repo_corruption/comment_3_679dde8ca0081fc6854d6d2e8a42abdb._comment b/doc/forum/recovering_from_repo_corruption/comment_3_679dde8ca0081fc6854d6d2e8a42abdb._comment
new file mode 100644
index 0000000000..9b724b1431
--- /dev/null
+++ b/doc/forum/recovering_from_repo_corruption/comment_3_679dde8ca0081fc6854d6d2e8a42abdb._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="http://joeyh.name/"
+ ip="2001:4978:f:21a::2"
+ subject="comment 3"
+ date="2013-01-26T09:11:00Z"
+ content="""
+I've fixed the bug that led to the permissions denied error.
+"""]]
diff --git a/doc/forum/recovery_from_failed_merge.mdwn b/doc/forum/recovery_from_failed_merge.mdwn
new file mode 100644
index 0000000000..802c66e511
--- /dev/null
+++ b/doc/forum/recovery_from_failed_merge.mdwn
@@ -0,0 +1,7 @@
+Starting the assistant version 5.20140613 on my  repository (~60GB), it performed a merge
+with an offline-repository, deleting a large part of the files.
+Since the repo is in direct mode, I cannot do a git revert. The other repo is not available anymore
+
+Any way of getting them back?
+Also, du shows me that the .git/annex/objects folder has approximately the size of my repo before the incident.
+
diff --git a/doc/forum/recovery_from_failed_merge/comment_1_84e5b55d473d16bc9bdba5d88dc29bc3._comment b/doc/forum/recovery_from_failed_merge/comment_1_84e5b55d473d16bc9bdba5d88dc29bc3._comment
new file mode 100644
index 0000000000..3f61d8cb9e
--- /dev/null
+++ b/doc/forum/recovery_from_failed_merge/comment_1_84e5b55d473d16bc9bdba5d88dc29bc3._comment
@@ -0,0 +1,10 @@
+[[!comment format=mdwn
+ username="http://joeyh.name/"
+ ip="108.236.230.124"
+ subject="comment 1"
+ date="2014-06-16T17:58:22Z"
+ content="""
+All your files are still there..
+
+You can either use `git annex indirect` to switch to indirect mode and then `git revert`, or you could git clone the repo to someplace else, do the revert there, and then run `git annex sync` in first the clone and then in the direct mode repo to sync the reversion back.
+"""]]
diff --git a/doc/forum/reflog_of_pruned_commits.mdwn b/doc/forum/reflog_of_pruned_commits.mdwn
new file mode 100644
index 0000000000..ecfa8558e1
--- /dev/null
+++ b/doc/forum/reflog_of_pruned_commits.mdwn
@@ -0,0 +1,27 @@
+Hello,
+
+I ran into some issues with one repo (repo X). I didn't konw what to do, so I did several git annex sync on the other repos, and git annex repair on repo X.
+
+Now, git annex sync on repo X gives me:
+
+```
+
+pull repo W/Y/Z/etc.
+Compression automatique du dépôt en tâche de fond pour optimiser les performances.
+Voir "git help gc" pour toute information sur le nettoyage manuel.
+error: Le dernier lancement de gc a rapporté l'erreur suivante. Veuillez corriger
+la cause et supprimer .git/gc.log.
+Le nettoyage automatique n'aura pas lieu jusqu'à ce que le fichier soit supprimé.
+
+warning: reflog of 'refs/heads/git-annex' references pruned commits
+error: Could not read 7bb7bcb765b1fa81c92ac6746fe124da0f22b8b6
+fatal: Failed to traverse parents of commit 6bf9f29be6bcca7b1c0d93d0132c9e402ce433cd
+error: failed to run repack
+
+ok
+
+
+```
+
+What should I do?
+
diff --git a/doc/forum/reflog_of_pruned_commits/comment_1_edb31e8722edc58be7e2593556dc3ddf._comment b/doc/forum/reflog_of_pruned_commits/comment_1_edb31e8722edc58be7e2593556dc3ddf._comment
new file mode 100644
index 0000000000..812d06b204
--- /dev/null
+++ b/doc/forum/reflog_of_pruned_commits/comment_1_edb31e8722edc58be7e2593556dc3ddf._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="hugo"
+ avatar="http://cdn.libravatar.org/avatar/950651406c30b9987cdd26060aef034b"
+ subject="Comment"
+ date="2017-08-21T06:21:33Z"
+ content="""
+(just a comment to check \"email replies to me\")
+"""]]
diff --git a/doc/forum/reflog_of_pruned_commits/comment_2_cd0f057a730389509bd12dcaf21e3150._comment b/doc/forum/reflog_of_pruned_commits/comment_2_cd0f057a730389509bd12dcaf21e3150._comment
new file mode 100644
index 0000000000..e43c880e50
--- /dev/null
+++ b/doc/forum/reflog_of_pruned_commits/comment_2_cd0f057a730389509bd12dcaf21e3150._comment
@@ -0,0 +1,9 @@
+[[!comment format=mdwn
+ username="joey"
+ subject="""comment 2"""
+ date="2017-08-28T17:03:19Z"
+ content="""
+This does not seem to be a problem. If the warning message is bothering
+you, you could remove the part of the reflog that references the pruned
+commits, or remove the whole reflog.
+"""]]
diff --git a/doc/forum/refs__47__heads__47__synced__47__git-annex_receives_from_more_than_one_src.mdwn b/doc/forum/refs__47__heads__47__synced__47__git-annex_receives_from_more_than_one_src.mdwn
new file mode 100644
index 0000000000..ff9d1b6c7c
--- /dev/null
+++ b/doc/forum/refs__47__heads__47__synced__47__git-annex_receives_from_more_than_one_src.mdwn
@@ -0,0 +1,12 @@
+Total newbie here, so I apologize in advice for asking dumb questions. I did my best to look through the branchable docs, Google, etc to no avail.     
+I'm trying to use git-annex to manage my music collection (on Windows). I added and pushed 669 songs to GitLab from my desktop.
+At this point, my music is on both the first folder and GitLab (worth noting that I still don't know where to find the binaries on GitLab...).      
+      
+Then just as a test, tried to clone the repo and run `git annex sync --content`.      
+      
+I get the following error:       
+`error: dst ref refs/heads/synced/git-annex receives from more than one src.`      
+`error: failed to push some refs to 'git@gitlab.com:repolocation/Songs.git'`      
+      
+I figure it's because the files are both on GitLab and the other folder, but I'm not sure how to proceed.      
+Would appreciate any help, thanks!
diff --git a/doc/forum/refs__47__heads__47__synced__47__git-annex_receives_from_more_than_one_src/comment_1_c805561fa714ff3930742bb330b340c9._comment b/doc/forum/refs__47__heads__47__synced__47__git-annex_receives_from_more_than_one_src/comment_1_c805561fa714ff3930742bb330b340c9._comment
new file mode 100644
index 0000000000..3f4be57a64
--- /dev/null
+++ b/doc/forum/refs__47__heads__47__synced__47__git-annex_receives_from_more_than_one_src/comment_1_c805561fa714ff3930742bb330b340c9._comment
@@ -0,0 +1,20 @@
+[[!comment format=mdwn
+ username="joey"
+ subject="""comment 1"""
+ date="2015-09-09T18:47:17Z"
+ content="""
+This error message is a `git push` error message.
+I've never seen this error before.. 
+
+
+Shows one way to get into this situation, where a repository somehow has
+a tag with the same name as the branch that is being pushed.
+
+>
+Similarly shows it being a conflict between a tag and a branch;
+this time the tag exists in the local repository.
+
+I guess you've run into some problem like this. If there's an easy way to
+reproduce the problem using just git-annex commands, and not manaul tag
+creation etc, please file a bug report with details.
+"""]]
diff --git a/doc/forum/reinstalled_os__44___cloned_annex__44___does_not_recognize_remote.mdwn b/doc/forum/reinstalled_os__44___cloned_annex__44___does_not_recognize_remote.mdwn
new file mode 100644
index 0000000000..2c437fe359
--- /dev/null
+++ b/doc/forum/reinstalled_os__44___cloned_annex__44___does_not_recognize_remote.mdwn
@@ -0,0 +1,8 @@
+Hi,
+
+I switched distribution and cloned my annex from the backup afterwards. getting the files from my backup worked fine, but i have a remote on an external disk and git-annex keeps saying it's unavailable despite it being there. I have the same username and hostname as on my old system so and the remote repo still has the uuid 3d661c5e-c84e-4902-b52e-decbafadfa4f. The only thing that changed is the path, it's know mounted unter /var/run/media/phaer/95... but a symlinked to the path below did not help. Any suggestions? 
+
+    3d661c5e-c84e-4902-b52e-decbafadfa4f -- phaer@kassiopeia:/media/phaer/9502bc14-b83e-471a-95a8-80ff9072a4ab/annex
+
+
+and thank you for this amazing piece of software.
diff --git a/doc/forum/reinstalled_os__44___cloned_annex__44___does_not_recognize_remote/comment_1_c1962d757dd22f49e774afa13a9862ca._comment b/doc/forum/reinstalled_os__44___cloned_annex__44___does_not_recognize_remote/comment_1_c1962d757dd22f49e774afa13a9862ca._comment
new file mode 100644
index 0000000000..1223b24145
--- /dev/null
+++ b/doc/forum/reinstalled_os__44___cloned_annex__44___does_not_recognize_remote/comment_1_c1962d757dd22f49e774afa13a9862ca._comment
@@ -0,0 +1,10 @@
+[[!comment format=mdwn
+ username="http://joeyh.name/"
+ ip="64.134.31.139"
+ subject="comment 1"
+ date="2013-10-21T22:24:35Z"
+ content="""
+It might help if you pasted the actual error message that git-annex shows when you try to use this remote. Along with the part of .git/config that configures this remote.
+
+In general, git-annex does not are if you've reinstalled your OS, moved things around, etc. As long as the git remote's path points to a repository, git-annex will see it and be able to use it.
+"""]]
diff --git a/doc/forum/reinstalled_os__44___cloned_annex__44___does_not_recognize_remote/comment_2_1f0f4a1dc89643cee81ff7199b55e747._comment b/doc/forum/reinstalled_os__44___cloned_annex__44___does_not_recognize_remote/comment_2_1f0f4a1dc89643cee81ff7199b55e747._comment
new file mode 100644
index 0000000000..f38c0c7520
--- /dev/null
+++ b/doc/forum/reinstalled_os__44___cloned_annex__44___does_not_recognize_remote/comment_2_1f0f4a1dc89643cee81ff7199b55e747._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="https://www.google.com/accounts/o8/id?id=AItOawmBhQgaA5QrFwT67-Bo0qPIx0HD9roDrso"
+ nickname="Paul"
+ subject="comment 2"
+ date="2013-11-20T11:20:13Z"
+ content="""
+The problem was just a missing git remote. Sorry & thanks
+"""]]
diff --git a/doc/forum/released_finder_integration_for_macOS_called_git-annex-turtle.mdwn b/doc/forum/released_finder_integration_for_macOS_called_git-annex-turtle.mdwn
new file mode 100644
index 0000000000..6a97d38534
--- /dev/null
+++ b/doc/forum/released_finder_integration_for_macOS_called_git-annex-turtle.mdwn
@@ -0,0 +1,12 @@
+I just released a new app (in beta) for macOS called [git-annex-turtle](https://github.com/andrewringler/git-annex-turtle). It provides badges in finder, context menus and very limited status updates via a menubar icon. **Requires** macOS 10.12 or later.
+
+Badges indicate whether files are present or not, how many copies they have and if they are lacking copies. Context menus (currently very limited) allow for add/get/drop/lock and unlock. A menubar icon gives (currently very limited) feedback on current operations.
+
+The is definitely a preview beta release so there are bound to be bugs. Please report any feature requests or bugs with github issues, by emailing me directly at public@andrewringler.com or posting comments to this thread.
+
+Thanks!
+
+
+Andrew
+
+**[Download git-annex-turtle](https://github.com/andrewringler/git-annex-turtle)**
diff --git a/doc/forum/released_finder_integration_for_macOS_called_git-annex-turtle/comment_1_58f236c940ca20bfbae53324a530da0d._comment b/doc/forum/released_finder_integration_for_macOS_called_git-annex-turtle/comment_1_58f236c940ca20bfbae53324a530da0d._comment
new file mode 100644
index 0000000000..e9c7f4107b
--- /dev/null
+++ b/doc/forum/released_finder_integration_for_macOS_called_git-annex-turtle/comment_1_58f236c940ca20bfbae53324a530da0d._comment
@@ -0,0 +1,22 @@
+[[!comment format=mdwn
+ username="joey"
+ subject="""comment 1"""
+ date="2018-03-09T18:10:11Z"
+ content="""
+Thanks very much for building this. It looks great.
+
+I especially appreciated the notes in DESIGN.md about how it uses git-annex
+interfaces and what is fast/too slow. Please do file todo requests if there
+are interface improvements that git-annex can offer to query for the data
+you need.
+
+I suppose it does not make sense to integrate the code into git-annex,
+as is done for the much simpler Nautilus and Konqueror file
+manager integration. git-annex-turtle is something like 33M large with all
+the assets, and has a significant quantity of source code that I would 
+only get in the way of maintaining.
+
+And it looks like it's easy enough to install it from the zip file.
+But I do wonder if it would make sense for the git-annex.app to bundle
+git-annex-turtle, so users don't need that extra step. What do you think?
+"""]]
diff --git a/doc/forum/released_finder_integration_for_macOS_called_git-annex-turtle/comment_2_54ba7672150ee32df4d2d92390a960c1._comment b/doc/forum/released_finder_integration_for_macOS_called_git-annex-turtle/comment_2_54ba7672150ee32df4d2d92390a960c1._comment
new file mode 100644
index 0000000000..8c09eb78c5
--- /dev/null
+++ b/doc/forum/released_finder_integration_for_macOS_called_git-annex-turtle/comment_2_54ba7672150ee32df4d2d92390a960c1._comment
@@ -0,0 +1,21 @@
+[[!comment format=mdwn
+ username="andrew"
+ avatar="http://cdn.libravatar.org/avatar/acc0ece1eedf07dd9631e7d7d343c435"
+ subject="comment 2"
+ date="2018-03-10T18:33:22Z"
+ content="""
+Thanks for the positive feedback!
+
+>Please do file todo requests if there are interface improvements that git-annex can offer to query for the data you need.
+
+Will do. I think I have most of the queries I need right now. I really wanted to show nice, meaningful icons for folders; I ended up showing some measure of the number of copies of all files contained within folders. This is something I decided to cache in a database since, to my knowledge, neither git nor git-annex tracks folders directly. I don't think it would make sense to cache folder level queries in git-annex anytime soon?
+
+
+>But I do wonder if it would make sense for the git-annex.app to bundle git-annex-turtle, so users don't need that extra step.
+
+Great! Yes, that sounds like an excellent idea! Yes, as you mentioned git-annex-turtle.app is ~30mb with all the icon assets. I believe this is because the icons are uncompressed PDFs exported from Adobe Illustrator, oops. I'll see if I can compress them and get the .app size down. Currently, zipping git-annex-turtle.app brings down the size to ~8mb.
+
+Thanks,
+
+—Andrew
+"""]]
diff --git a/doc/forum/released_finder_integration_for_macOS_called_git-annex-turtle/comment_3_107d0a3c9de6880d0ee164a651d2c20e._comment b/doc/forum/released_finder_integration_for_macOS_called_git-annex-turtle/comment_3_107d0a3c9de6880d0ee164a651d2c20e._comment
new file mode 100644
index 0000000000..127fb8d8db
--- /dev/null
+++ b/doc/forum/released_finder_integration_for_macOS_called_git-annex-turtle/comment_3_107d0a3c9de6880d0ee164a651d2c20e._comment
@@ -0,0 +1,10 @@
+[[!comment format=mdwn
+ username="anarcat"
+ avatar="http://cdn.libravatar.org/avatar/4ad594c1e13211c1ad9edb81ce5110b7"
+ subject="beautiful"
+ date="2018-03-12T13:52:53Z"
+ content="""
+wow, that is absolutely gorgeous. i wish we had the same in linux! it would make git-annex so much simpler to use... but i don't know if contrib plugins can add graphic stuff like that in file manager displays in nautilus/etc?
+
+seems like we're clearly lagging behind when I look at the finder... can't believe we still don't have a proper file manager in linux yet... --[[anarcat]]
+"""]]
diff --git a/doc/forum/released_finder_integration_for_macOS_called_git-annex-turtle/comment_4_ddb6d671ddd28ba79efce74e41eab7e1._comment b/doc/forum/released_finder_integration_for_macOS_called_git-annex-turtle/comment_4_ddb6d671ddd28ba79efce74e41eab7e1._comment
new file mode 100644
index 0000000000..d1c2e3dd6f
--- /dev/null
+++ b/doc/forum/released_finder_integration_for_macOS_called_git-annex-turtle/comment_4_ddb6d671ddd28ba79efce74e41eab7e1._comment
@@ -0,0 +1,10 @@
+[[!comment format=mdwn
+ username="andrew"
+ avatar="http://cdn.libravatar.org/avatar/acc0ece1eedf07dd9631e7d7d343c435"
+ subject="comment 4"
+ date="2018-03-12T19:24:56Z"
+ content="""
+Thanks [anarcat](http://git-annex.branchable.com/users/anarcat/)!
+
+[RabbitVCS](http://rabbitvcs.org/) provides Nautilus 3 icons for git, that might be a reasonable starting point for enhanced git-annex icons on Linux. Also, [nautilus-git](https://github.com/bil-elmoussaoui/nautilus-git) is minimal and quite pretty. I don't know what flavor of file manager they were using when they took their screenshots.
+"""]]
diff --git a/doc/forum/released_finder_integration_for_macOS_called_git-annex-turtle/comment_5_d496549c4f8bf3500e8ac5c2f2ae0d6e._comment b/doc/forum/released_finder_integration_for_macOS_called_git-annex-turtle/comment_5_d496549c4f8bf3500e8ac5c2f2ae0d6e._comment
new file mode 100644
index 0000000000..ed82fa19bd
--- /dev/null
+++ b/doc/forum/released_finder_integration_for_macOS_called_git-annex-turtle/comment_5_d496549c4f8bf3500e8ac5c2f2ae0d6e._comment
@@ -0,0 +1,12 @@
+[[!comment format=mdwn
+ username="joey"
+ subject="""comment 5"""
+ date="2018-04-04T18:17:02Z"
+ content="""
+I guess what would be helpful is if you can take the current git-annex.app
+and copy its files to a directory and then add in the
+git-annex-turtle.app's files in a way that combines them both.
+I don't currently have an OSX desktop to play with that, but if you can
+show me the changes that need to be made to the file tree in git-annex.app
+to merge turtle into it, I can update the build scripts.
+"""]]
diff --git a/doc/forum/released_finder_integration_for_macOS_called_git-annex-turtle/comment_6_8b3555fe33da55c01b199efd492951b0._comment b/doc/forum/released_finder_integration_for_macOS_called_git-annex-turtle/comment_6_8b3555fe33da55c01b199efd492951b0._comment
new file mode 100644
index 0000000000..f88587d359
--- /dev/null
+++ b/doc/forum/released_finder_integration_for_macOS_called_git-annex-turtle/comment_6_8b3555fe33da55c01b199efd492951b0._comment
@@ -0,0 +1,37 @@
+[[!comment format=mdwn
+ username="andrew"
+ avatar="http://cdn.libravatar.org/avatar/acc0ece1eedf07dd9631e7d7d343c435"
+ subject="integrating git-annex-turtle into git-annex"
+ date="2018-04-20T18:24:21Z"
+ content="""
+Ooops…, I missed this comment.
+
+I am not exactly sure what the best approach will be, but I have some thoughts. Yes, I can make an example of a folder structure I think could work well. I'll try to explain all the positives and negatives of various approaches below.
+
+The standard for application installation on the mac is to deliver a dmg that when double-clicked opens a small Finder window that contains the application (.app bundle), a link to /Applications and [a background image with an arrow](https://asmaloney.com/2013/07/howto/packaging-a-mac-os-x-application-using-a-dmg/) hinting that users should drag the .app file onto to the /Applications link (which causes Finder to initiate a copy).
+
+Currently, after installing git-annex from the dmg, when you double-click the git-annex.app folder git-annex-webapp is launched.
+
+1) One approach, to bundling git-annex-turtle would be to place git-annex-turtle inside the git-annex.app folder. That would mean, however, users need to ctrl-click then click `Show Package Contents` then browse the file structure until they find git-annex-turtle, then double-click it to launch it. That would be a bit confusing, and not very Mac-y. This could be slightly mitigated by adding an installer for git-annex that asks users if they want to auto-launch git-annex-turtle on login-in, but then we still have the issue of them not being able to re-launch it if they ever quit it. Also, installers are not very popular on the Mac.
+
+2) Another approach would be for the dmg file to contain a folder that contains two items git-annex.app and git-annex-turtle.app. Users would then drag the entire folder to their /Applications folder. Then git-annex-webapp would be launchable (from Finder) by double-clicking on /Applications/git-annex/git-annex.app, and git-annex-turtle would be launch-able by double-clicking on /Applications/git-annex/git-annex-turtle.app. The biggest drawback with this approach is that it would change all of the absolute paths to the binaries inside git-annex.app.
+
+For example, I currently have the following line in my bash_profile:
+
+    export PATH=/Applications/git-annex.app/Contents/MacOS:$PATH
+
+This would need to change to
+
+    export PATH=/Applications/git-annex/git-annex.app/Contents/MacOS:$PATH
+
+I think we probably don't want to make all users have to update all their paths to git-annex binaries.
+
+This approach, of having a folder with multiple apps is quite rare. Users, really expect their /Applications folder to only contain .app files with nice icons that they can double-click to launch things. I could only find a handful of examples of apps that are doing this (all cross-platform apps): Blender, Supercollider, Unity, Adobe (sometimes), Autodesk.
+
+3) Another approach would be to just include both the existing git-annex.app and git-annex-turtle.app (as they are) in the git-annex.dmg file. Then I could create a nice background image that suggests users should drag both git-annex.app and git-annex-turtle.app to their /Applications folder. Then users would have /Applications/git-annex.app and /Applications/git-annex-turtle.app installed. The main drawback here is that users need to know to drag both files to their /Applications folder, and they are slightly more likely to not always upgrade them, and there is the potential for them to just drag git-annex-turtle to /Applications and not git-annex, and wonder why one doesn't work. Or uninstall git-annex at some point, and then again, wonder why git-annex-turtle no longer works.
+
+Currently I am leaning towards this last option. But, I think 2) could work as well, although it would be unfortunate to break people's paths.
+
+
+Andrew
+"""]]
diff --git a/doc/forum/reliability__47__completeness_of_XMPP_updates.mdwn b/doc/forum/reliability__47__completeness_of_XMPP_updates.mdwn
new file mode 100644
index 0000000000..fc558aad60
--- /dev/null
+++ b/doc/forum/reliability__47__completeness_of_XMPP_updates.mdwn
@@ -0,0 +1,7 @@
+This falls into the category of "noob questions" I think.
+
+The one piece of the git-annex assistant puzzle I've never messed with is XMPP pairing.  I'm wondering how well a pair of repos can keep in sync with each other if their only connection is via XMPP.  Will things go badly if changes are made to one while the other is offline?  Do messages get queued up to deliver when they're both online?  (Or do they get queued on the server side so they can be delivered even if one of them is online, makes changes, then goes offline, and the other one comes online later?)
+
+If some xmpp messages don't go through for whatever reason, will the remotes be able to "catch up" with each other later on and make up for lost time?
+
+Just hoping for a general sense of the limitations of XMPP pairing.   TIA.
diff --git a/doc/forum/reliability__47__completeness_of_XMPP_updates/comment_1_e0f7aa48d54fc0564f41c3a569c723b7._comment b/doc/forum/reliability__47__completeness_of_XMPP_updates/comment_1_e0f7aa48d54fc0564f41c3a569c723b7._comment
new file mode 100644
index 0000000000..d72cad5b97
--- /dev/null
+++ b/doc/forum/reliability__47__completeness_of_XMPP_updates/comment_1_e0f7aa48d54fc0564f41c3a569c723b7._comment
@@ -0,0 +1,12 @@
+[[!comment format=mdwn
+ username="http://joeyh.name/"
+ ip="4.154.4.90"
+ subject="comment 1"
+ date="2013-07-16T19:27:07Z"
+ content="""
+Both clients have to be online at the same time for XMPP push to work. Once they're able to see each other, they'll sync up, even if they've diverged since the last sync.
+
+If you tend to only have one client or the other online, you should set up a git repo on a ssh server. Then clients will drop off their changes there, and the other one will check it when it comes online. (You don't really need to use XMPP at all in this case.)
+
+The most robust and fast combo is to use XMPP pairing, and also have a git repo on a ssh server. This way, when both clients are online, they'll use XMPP to instantly propagate changes, and when not the server is there to fall back to.
+"""]]
diff --git a/doc/forum/reliability__47__completeness_of_XMPP_updates/comment_2_4e74039a673c16c0163f2cfb406dc4c3._comment b/doc/forum/reliability__47__completeness_of_XMPP_updates/comment_2_4e74039a673c16c0163f2cfb406dc4c3._comment
new file mode 100644
index 0000000000..b4e329441b
--- /dev/null
+++ b/doc/forum/reliability__47__completeness_of_XMPP_updates/comment_2_4e74039a673c16c0163f2cfb406dc4c3._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="http://edheil.wordpress.com/"
+ ip="173.162.44.162"
+ subject="comment 2"
+ date="2013-07-16T23:36:22Z"
+ content="""
+Thanks, that was exactly the kind of explanation I needed.
+"""]]
diff --git a/doc/forum/reliability__47__completeness_of_XMPP_updates/comment_3_41ade4fe72804b2f06cd4dbf405c1746._comment b/doc/forum/reliability__47__completeness_of_XMPP_updates/comment_3_41ade4fe72804b2f06cd4dbf405c1746._comment
new file mode 100644
index 0000000000..a212dbd092
--- /dev/null
+++ b/doc/forum/reliability__47__completeness_of_XMPP_updates/comment_3_41ade4fe72804b2f06cd4dbf405c1746._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="Flo"
+ ip="141.76.46.31"
+ subject="Git repositery?"
+ date="2013-07-19T08:56:04Z"
+ content="""
+I'm not sure to know what you mean with \"git repository on a SSH server\". I tried to use the assistant to add this git synchronization fall-back if XMPP is not available. However, it synchronizes all files too with this new repository. I tried to make it manual, unwanted, etc. But it stills send all my files on the transfer repository on rsync.net. I don't want that...
+"""]]
diff --git a/doc/forum/relying_on_git_for_numcopies.mdwn b/doc/forum/relying_on_git_for_numcopies.mdwn
new file mode 100644
index 0000000000..37b46cf4e7
--- /dev/null
+++ b/doc/forum/relying_on_git_for_numcopies.mdwn
@@ -0,0 +1,47 @@
+**<out-of-date-warning>**The main problems this is supposed to solve are addressed in a different way with [[todo/hidden files]] and the `--fast` option introduced in [[batch check on remote when using copy]], so while this is not technically obsolete, the main reasons for it are gone. --[[chrysn]]**</out-of-date-warning>**
+
+This is a rough sketch of a modification of git-annex to rely more on git commit semantics. It might be flawed due to my lack of understanding of git-annex internals. --[[chrysn]]
+
+Summary
+=========
+
+Currently, [[location tracking]] is only used for informational purposes unless a repository is [[trust]]ed, in which case there is no checking at all. It is proposed to use the location tracking information as a commitment to keep track of a file until another repository takes over responsibility.
+
+git's semantics for atomic commits are proposed to be used, which makes sure that before files are actually deleted, another repository has accepted the deletion.
+
+Modified git-annex-drop behavior
+==========================
+
+The most important (if not only) git-annex command that is affected by this is `git annex drop`. Currently, for dropping a large number of files, every file is checked with another (or multiple, if so configured) host if it's safe to delete.
+
+The new behavior would be to
+
+* decrement the location tracking counter for all files to be dropped,
+* commit that change,
+* try to push it to at least as many repositories that the numcopies constraints are met,
+* revert if that fails,
+* otherwise really drop the files from the backend.
+
+Unlike explicit checking, this never looks at the remote backend if the file is really present -- otoh, git-annex already relies on the files in the backend to not be touched by anyone but git-annex itself, and git-annex would only drop them if they were derefed and committed, in which case git would not accept the push. (git by itself would accept a merged push, but even if the reverting step failed due to a power outage or similar, git-annex would, before really deleting files from the backend, check again if the numcopies restraint is still met, and revert its own delete commit as the files are still present anyway.)
+
+Implications for trust
+==============
+
+The proposed change also changes the semantics of trust. Trust can now be controlled in a finer-grained way between untrusted and semi-trusted, as best illustrated by a use case:
+
+> Alice takes her netbook with her on a trip through Spain, and will fill most of its disk up with pictures she takes. As she expects to meet some old friends during the first days, she wants to take older pictures with her, which are safely backed up at home, so they can be deleted on demand.
+>
+> She tells her netbook's repository to dereference the old images (but not other parts of the repository she has not copied anywhere yet) and pushes to the server before leaving. When she adds pictures from her camera to the repository, git-annex can now free up space as needed.
+
+Dereferencing could be implemented as `git annex drop --no-rm` (or `move --no-rm`), freeing space is similar to `dropunused`.
+
+A trusted repository with the new semantics would mean that the repository would not accept dropping anything, just as before.
+
+Advantages / Disadvantages
+=====================
+
+The advantage of this proposal is that the round trips required for dropping something could be greatly reduced.
+
+There should also be simplifications in the `git annex drop` command as it doesn't need to take care of locking any more (git should already do that between checking if HEAD is a parent of the pushed commit and replacing HEAD).
+
+Besides being a major change in git-annex (with the requirement to track hosts' git-annex versions for migration, as the new trust system is incompatible with the old one), no disadvantages of that stragegy are known to the author (hoping for discussion below).
diff --git a/doc/forum/relying_on_git_for_numcopies/comment_1_8ad3cccd7f66f6423341d71241ba89fc._comment b/doc/forum/relying_on_git_for_numcopies/comment_1_8ad3cccd7f66f6423341d71241ba89fc._comment
new file mode 100644
index 0000000000..83a908da8c
--- /dev/null
+++ b/doc/forum/relying_on_git_for_numcopies/comment_1_8ad3cccd7f66f6423341d71241ba89fc._comment
@@ -0,0 +1,36 @@
+[[!comment format=mdwn
+ username="http://joey.kitenet.net/"
+ nickname="joey"
+ subject="comment 1"
+ date="2011-02-22T18:44:28Z"
+ content="""
+I see the following problems with this scheme:
+
+- Disallows removal of files when disconnected. It's currently safe to force that, as long as
+  git-annex tells you enough other repos are belived to have the file. Just as long as you
+  only force on one machine (say your laptop). With your scheme, if you drop a file while 
+  disconnected, any other host could see that the counter is still at N, because your
+  laptop had the file last time it was online, and can decide to drop the file, and lose the last 
+version.
+
+- pushing a changed counter commit to other repos is tricky, because they're not bare, and 
+  the network topology to get the commit pulled into the other repo could vary.
+
+- Merging counter files issues.  If the counter file doesn't automerge, two repos dropping the same file will conflict. But, if it does automerge, it breaks the counter conflict detection.
+
+- Needing to revert commits is going to be annoying. An actual git revert
+  could probably not reliably be done. It's need to construct a revert
+  and commit it as a new commit. And then try to push that to remotes, and
+  what if *that* push conflicts?
+
+- I do like the pre-removal dropping somewhat as an alternative to
+  trust checking. I think that can be done with current git-annex though,
+  just remove the files from the location log, but keep them in-annex.
+  Dropping a file only looks at repos that the location log says have a
+  file; so other repos can have retained a copy of a file secretly like
+  this, and can safely remove it at any time. I'd need to look into this a bit more to be 100% sure it's safe, but have started [[todo/hidden_files]]. 
+
+- I don't see any reduced round trips. It still has to contact N other
+  repos on drop. Now, rather than checking that they have a file, it needs
+  to push a change to them.
+"""]]
diff --git a/doc/forum/relying_on_git_for_numcopies/comment_2_be6acbc26008a9cb54e7b8f498f2c2a2._comment b/doc/forum/relying_on_git_for_numcopies/comment_2_be6acbc26008a9cb54e7b8f498f2c2a2._comment
new file mode 100644
index 0000000000..d9ce8b50e0
--- /dev/null
+++ b/doc/forum/relying_on_git_for_numcopies/comment_2_be6acbc26008a9cb54e7b8f498f2c2a2._comment
@@ -0,0 +1,18 @@
+[[!comment format=mdwn
+ username="http://christian.amsuess.com/chrysn"
+ nickname="chrysn"
+ subject="comment 2"
+ date="2011-02-23T16:43:59Z"
+ content="""
+i'll comment on each of the points separately, well aware that even a single little leftover issue can show that my plan is faulty:
+
+* force removal: well, yes -- but the file that is currently force-removed on the laptop could just as well be the last of its kind itself. i see the problem, but am not sure if it's fatal (after all, if we rely on out-of-band knowledge when forcing something, we could just as well ask a little more)
+* non-bare repos: pushing is tricky with non-bare repos now just as well; a post-commit hook could auto-accept counter changes. (but pushing causes problems with counters anyway, doesn't it?)
+* merging: i'd have them auto-merge. git-annex will have to check the validity of the current state anyway, and a situation in which a counter-decrementing commit is not a fast-forward one would be reverted in the next step (or upon discovery, in case the next step never took place).
+* reverting: my wording was bad as \"revert\" is already taken in git-lingo. the correct term for what i was thinking of is \"reset\". (as the commit could not be pushed, it would be rolled back completely).
+    * we might have to resort to reverting, though, if the commit has already been pused to a first server of many.
+* [[todo/hidden files]]: yes, this solves pre-removal dropping :-)
+* round trips: it's not the number of servers, it's the number of files (up to 30k in my case). it seems to me that an individual request was made for every single file i wanted to drop (that would be N*M roundtrips for N affected servers and M files, and N roundtrips with git managed numcopies)
+
+all together, it seems to be a bit more complicated than i imagined, although not completely impossible. a combination of [[todo/hidden files]] and maybe a simpler reduction of the number of requests might though achieve the important goals as well.
+"""]]
diff --git a/doc/forum/relying_on_git_for_numcopies/comment_3_43d8e1513eb9947f8a503f094c03f307._comment b/doc/forum/relying_on_git_for_numcopies/comment_3_43d8e1513eb9947f8a503f094c03f307._comment
new file mode 100644
index 0000000000..27076a877f
--- /dev/null
+++ b/doc/forum/relying_on_git_for_numcopies/comment_3_43d8e1513eb9947f8a503f094c03f307._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="http://christian.amsuess.com/chrysn"
+ nickname="chrysn"
+ subject="relation to [[todo/branching]]"
+ date="2011-02-23T21:48:14Z"
+ content="""
+the non-bare repository issue would go away if this was combined with the \"alternate\" approach to [[todo/branching]]. (with the \"fleshed out proposal\" of branching, this would not work at all for lack of shared commits.)
+"""]]
diff --git a/doc/forum/remembering_state.mdwn b/doc/forum/remembering_state.mdwn
new file mode 100644
index 0000000000..3edc193bce
--- /dev/null
+++ b/doc/forum/remembering_state.mdwn
@@ -0,0 +1,5 @@
+so i am wondering how better to remember the state of a repository.
+
+the use case is this: i have this part of my music collection which i consider to be my "favorites". i would like to have only those on an external hard drive, but also on my portable music player. so i cloned my main music collection to the external hard drive and did `git annex get` on the files i wanted there. now i want to replicate that to the music player - how would i go around doing that? if i clone that repo again, it will all have empty symlinks and i would have to reproduce the same process all over again, which i can script, but it seems a little quirky...
+
+ideas? --[[anarcat]]
diff --git a/doc/forum/remembering_state/comment_1_4a6ac5f50dfa5a17a0f0ccd0c2e7a466._comment b/doc/forum/remembering_state/comment_1_4a6ac5f50dfa5a17a0f0ccd0c2e7a466._comment
new file mode 100644
index 0000000000..bc9a072237
--- /dev/null
+++ b/doc/forum/remembering_state/comment_1_4a6ac5f50dfa5a17a0f0ccd0c2e7a466._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="https://www.google.com/accounts/o8/id?id=AItOawlQeMMI82sfgZ9lim_9GqLkc4fsd0Z3Qt8"
+ nickname="Ronald"
+ subject="comment 1"
+ date="2014-05-14T09:08:22Z"
+ content="""
+I'm still learning git-annex myself, but I think you can make a new branch 'favorites'. Sort out your favorite music and make a checkout on your external HDD and music player. In the walkthrough you can see how that works. 
+"""]]
diff --git a/doc/forum/remembering_state/comment_2_1b02d3713a2986bc027d166589a11c3f._comment b/doc/forum/remembering_state/comment_2_1b02d3713a2986bc027d166589a11c3f._comment
new file mode 100644
index 0000000000..3c3f393a62
--- /dev/null
+++ b/doc/forum/remembering_state/comment_2_1b02d3713a2986bc027d166589a11c3f._comment
@@ -0,0 +1,9 @@
+[[!comment format=mdwn
+ username="https://www.google.com/accounts/o8/id?id=AItOawkYmMFDdf3GJ9Oba6NCVkzGc4JyB9WavMs"
+ nickname="Xinruo"
+ subject="comment 2"
+ date="2014-05-15T02:07:06Z"
+ content="""
+I think a better solution is to use http://git-annex.branchable.com/tips/metadata_driven_views/
+You can tag the files as favorites, then use get --metadata tag=favorites to get only favorites files.
+"""]]
diff --git a/doc/forum/remembering_state/comment_3_b48775ea1e90b061b084f61a4a9baca5._comment b/doc/forum/remembering_state/comment_3_b48775ea1e90b061b084f61a4a9baca5._comment
new file mode 100644
index 0000000000..2b349527ce
--- /dev/null
+++ b/doc/forum/remembering_state/comment_3_b48775ea1e90b061b084f61a4a9baca5._comment
@@ -0,0 +1,19 @@
+[[!comment format=mdwn
+ username="http://joeyh.name/"
+ ip="108.236.230.124"
+ subject="comment 3"
+ date="2014-05-15T20:01:14Z"
+ content="""
+While you can certianly use views for this, I tend to do the same with my sound files without view, but just using location tracking info. I have a set of music I'm currently listening to, and want to have more or less the same files on multiple computers as I have on my laptop. With maybe a bit more on some computers with more space. So, I do:
+
+    git annex sync
+    git annex get --in darkstar
+
+Where \"darkstar\" is the name of the repository on my laptop. 
+
+Another nice trick that also takes advantage of the location tracking logs is this to get back some files you dropped temporarily to free up space:
+
+    git annex get --in='here@{yesterday}'
+
+
+"""]]
diff --git a/doc/forum/remembering_state/comment_4_cbebcc6ed4bdae6815c0576475e96f6a._comment b/doc/forum/remembering_state/comment_4_cbebcc6ed4bdae6815c0576475e96f6a._comment
new file mode 100644
index 0000000000..c4bf94390b
--- /dev/null
+++ b/doc/forum/remembering_state/comment_4_cbebcc6ed4bdae6815c0576475e96f6a._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="https://id.koumbit.net/anarcat"
+ ip="70.83.139.100"
+ subject="comment 4"
+ date="2014-05-21T18:02:51Z"
+ content="""
+i like @yesterday, that's a neat trick that would have basically resolved my problem here. thanks!
+"""]]
diff --git a/doc/forum/remote-specific_meta-data.mdwn b/doc/forum/remote-specific_meta-data.mdwn
new file mode 100644
index 0000000000..04b3919556
--- /dev/null
+++ b/doc/forum/remote-specific_meta-data.mdwn
@@ -0,0 +1,5 @@
+I know we can associate arbitrary [[metadata]] to files. I'd like to be able to sync *remote* specific metadata (say `ip = 10.0.0.1`) fields around. I was originally thinking of shoving this in the description but I am not sure that (a) it gets synced around and (b) it will scale well enough.
+
+Then I thought we could use the key/value storage, but then the keyname would change with metadata changes so maybe that won't work...? Another option I thought of was to just throw stuff in a separate directory in the git-annex branch.
+
+Thoughts? --[[anarcat]]
diff --git a/doc/forum/remote-specific_meta-data/comment_1_179ec4787d93ca4710c5bd448754d404._comment b/doc/forum/remote-specific_meta-data/comment_1_179ec4787d93ca4710c5bd448754d404._comment
new file mode 100644
index 0000000000..5eafa54051
--- /dev/null
+++ b/doc/forum/remote-specific_meta-data/comment_1_179ec4787d93ca4710c5bd448754d404._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="joey"
+ subject="""comment 1"""
+ date="2015-03-26T16:10:19Z"
+ content="""
+Well, git-annex uses remote.log for such data, and it ignores any
+extraneous fields, so..
+"""]]
diff --git a/doc/forum/remote-specific_meta-data/comment_2_51347629414ac42a6709d00d38ebef9c._comment b/doc/forum/remote-specific_meta-data/comment_2_51347629414ac42a6709d00d38ebef9c._comment
new file mode 100644
index 0000000000..e49e2c7e6e
--- /dev/null
+++ b/doc/forum/remote-specific_meta-data/comment_2_51347629414ac42a6709d00d38ebef9c._comment
@@ -0,0 +1,9 @@
+[[!comment format=mdwn
+ username="https://id.koumbit.net/anarcat"
+ subject="remote.log or uuid.log?"
+ date="2015-03-31T04:29:49Z"
+ content="""
+`remote.log` is present only for \"special remotes\" - could i also use it for... well, \"non-special remotes\"? :) in other words, right now my `remote.log` is empty or even missing in most of the git-annex repos I used, because i don't use the special remotes so much. could i still store information in that file even though it's not being used by git-annex?
+
+or could i just use `uuid.log` instead, which is present for all types of remotes?
+"""]]
diff --git a/doc/forum/remote_server_client_repositories_are_bare__33____63__.mdwn b/doc/forum/remote_server_client_repositories_are_bare__33____63__.mdwn
new file mode 100644
index 0000000000..21c7568366
--- /dev/null
+++ b/doc/forum/remote_server_client_repositories_are_bare__33____63__.mdwn
@@ -0,0 +1,17 @@
+I've gone through most of the screen casts and lots of documentation,
+but my first attempt produced an unexpected result.
+
+I wanted to use git-annex is a Dropbox-simplistic setting - syncing
+a directory between my notebook (OSX) and my desktop (OpenSUSE 12.3).
+Running the assistant on OSX and adding a "Remote server" repository,
+changing the repository group to client produced a bare repository
+on the desktop.
+
+Am I missing something obvious?
+
+I've tried a few different version of git-annex, most recently
+
+   git-annex version: 4.20130727-g9399845
+   build flags: Assistant Webapp Pairing Testsuite S3 WebDAV Inotify DBus XMPP
+
+but it remains the same.
diff --git a/doc/forum/remote_server_client_repositories_are_bare__33____63__/comment_1_234241460f6c75a8376b303b8dd4e882._comment b/doc/forum/remote_server_client_repositories_are_bare__33____63__/comment_1_234241460f6c75a8376b303b8dd4e882._comment
new file mode 100644
index 0000000000..41e599c6d4
--- /dev/null
+++ b/doc/forum/remote_server_client_repositories_are_bare__33____63__/comment_1_234241460f6c75a8376b303b8dd4e882._comment
@@ -0,0 +1,11 @@
+[[!comment format=mdwn
+ username="https://www.google.com/accounts/o8/id?id=AItOawmg8DSRA425Aik2clIuge8Jj-s7VJltsBI"
+ nickname="Tommy"
+ subject="comment 1"
+ date="2013-07-28T06:05:48Z"
+ content="""
+I found a partial answer on http://git-annex.branchable.com/forum/Add_a___34__local__34___remote :
+\"The webapp does not set up remote non-bare repositories because unless the user is comfortable with the cli, nothing is going to keep them up-to-date.\"
+
+So, clearly \"Remote server\" isn't what I want, but my \"desktop\" is headless and using jabber seems a silly complication when I have a perfectly fine ssh connection.  This isn't as trivial as I had expected.
+"""]]
diff --git a/doc/forum/remote_server_client_repositories_are_bare__33____63__/comment_2_42dfc382d07af2a4f29c76016084f87c._comment b/doc/forum/remote_server_client_repositories_are_bare__33____63__/comment_2_42dfc382d07af2a4f29c76016084f87c._comment
new file mode 100644
index 0000000000..84a2f36d66
--- /dev/null
+++ b/doc/forum/remote_server_client_repositories_are_bare__33____63__/comment_2_42dfc382d07af2a4f29c76016084f87c._comment
@@ -0,0 +1,12 @@
+[[!comment format=mdwn
+ username="http://joeyh.name/"
+ ip="4.154.0.21"
+ subject="comment 2"
+ date="2013-07-30T18:56:00Z"
+ content="""
+Make a non-bare git repository on your server. Run `git annex init` in it. Run `git annex assistant` in it.
+
+Now on your notebook, you can tell the assistant to use a remote ssh server, and point it at that directory. It will see the repository already exists, and use it, rather than making a new bare repository. The assistant daemon you've run on the server will notice when changes are pushed to the directory, and update it. 
+
+You can further set up XMPP, so that the server can tell the notebook if you make any modifications to the server. You can do this by running `git annex webapp --listen=IPADDRESS` on the server, and opening the url it prints out. But this is optional, only needed if you're going to be editing files on the server.
+"""]]
diff --git a/doc/forum/remote_server_client_repositories_are_bare__33____63__/comment_3_32bf10cf837db16566dcc99d0b9aaf67._comment b/doc/forum/remote_server_client_repositories_are_bare__33____63__/comment_3_32bf10cf837db16566dcc99d0b9aaf67._comment
new file mode 100644
index 0000000000..bf302f7c89
--- /dev/null
+++ b/doc/forum/remote_server_client_repositories_are_bare__33____63__/comment_3_32bf10cf837db16566dcc99d0b9aaf67._comment
@@ -0,0 +1,10 @@
+[[!comment format=mdwn
+ username="https://www.google.com/accounts/o8/id?id=AItOawkftzaCvV7EDKVDfJhsQZ3E1Vn-0db516w"
+ nickname="Edward"
+ subject="One snag"
+ date="2014-07-28T19:37:04Z"
+ content="""
+I setup a non-bare repo on a server by following the above steps (git init, git annex init, then add it as a Remote Server from elsewhere and combine repos). It worked, but I hit a snag and needed to add another step.
+
+After git init, you're not sitting on any branch yet, and that seems to have prevented the assistant from doing anything to synchronize the working tree on the server. After I did \"git checkout synced/master\", it started working.
+"""]]
diff --git a/doc/forum/remote_server_client_repositories_are_bare__33____63__/comment_4_cd04cfaf97f200d5e581b83bb8d018b2._comment b/doc/forum/remote_server_client_repositories_are_bare__33____63__/comment_4_cd04cfaf97f200d5e581b83bb8d018b2._comment
new file mode 100644
index 0000000000..5213a7f950
--- /dev/null
+++ b/doc/forum/remote_server_client_repositories_are_bare__33____63__/comment_4_cd04cfaf97f200d5e581b83bb8d018b2._comment
@@ -0,0 +1,26 @@
+[[!comment format=mdwn
+ username="https://www.google.com/accounts/o8/id?id=AItOawkRW96vF6lsjg57muQ4nPnQqJJUAKGKGzw"
+ nickname="Catalin"
+ subject="Caveat with 'git checkout synced/master'"
+ date="2014-12-26T07:43:18Z"
+ content="""
+There's at least one caveat with the 'git checkout synced/master' workaround. When the local assistant next tries to sync with the remote, it will try to push, and the remote will refuse the push into the currently checked out branch:
+
+[2014-12-26 02:40:10 EST] main: Syncing with nyc.nanobit.org_annex 
+remote: error: refusing to update checked out branch: refs/heads/synced/master        
+remote: error: By default, updating the current branch in a non-bare repository        
+remote: error: is denied, because it will make the index and work tree inconsistent        
+remote: error: with what you pushed, and will require 'git reset --hard' to match        
+remote: error: the work tree to HEAD.        
+remote: error:         
+remote: error: You can set 'receive.denyCurrentBranch' configuration variable to        
+remote: error: 'ignore' or 'warn' in the remote repository to allow pushing into        
+remote: error: its current branch; however, this is not recommended unless you        
+remote: error: arranged to update its work tree to match what you pushed in some        
+remote: error: other way.        
+remote: error:         
+remote: error: To squelch this message and still keep the default behaviour, set        
+remote: error: 'receive.denyCurrentBranch' configuration variable to 'refuse'.        
+To ssh://catalinp@git-annex-nyc.nanobit.org-catalinp_22_annex/~/annex/
+ ! [remote rejected] annex/direct/master -> synced/master (branch is currently checked out)
+"""]]
diff --git a/doc/forum/removing_information_from_special_remote_file.mdwn b/doc/forum/removing_information_from_special_remote_file.mdwn
new file mode 100644
index 0000000000..de740ee330
--- /dev/null
+++ b/doc/forum/removing_information_from_special_remote_file.mdwn
@@ -0,0 +1,8 @@
+I'm working on a project using git-annex and Tahoe as a special remote. I want to encrypt that Tahoe capability for various file(s) (the capability is present in the git-annex branch), so that it is impossible for a malicious actor to access a file. However, due to the union-merge that happens on that branch, no matter what I try, the plain text capability keeps coming back.
+
+How can I either remove an entry from a file in the git-annex branch, or maybe remove all history for a file so that it has nothing to merge against, or otherwise remove the plain-text capability?
+
+This is very similar/same as https://git-annex.branchable.com/forum/removing_remote.log_information_completely/ but hopefully I can get a updated perspective, or any further guidance. Any help at all would be greatly appreciated.
+
+If there's a simple way to modify the git-annex source code to accommodate this, I would be perfectly happy with that solution. Unfortunately I am not well versed in Haskell, but if there was a way to say, for example, ignore *.rmt files when doing the union-merge, that would probably work. I'd be happy with any solution though.
+
diff --git a/doc/forum/removing_information_from_special_remote_file/comment_1_a3c41731dbbcd53715c205e29ab99e4e._comment b/doc/forum/removing_information_from_special_remote_file/comment_1_a3c41731dbbcd53715c205e29ab99e4e._comment
new file mode 100644
index 0000000000..e117af21db
--- /dev/null
+++ b/doc/forum/removing_information_from_special_remote_file/comment_1_a3c41731dbbcd53715c205e29ab99e4e._comment
@@ -0,0 +1,17 @@
+[[!comment format=mdwn
+ username="joey"
+ subject="""comment 1"""
+ date="2017-06-06T18:40:54Z"
+ content="""
+Even if you remove the information from the remote log, it would still be
+there in the git history of the file.
+
+The thing to do, then, is to not store those capabilities in git at all.
+
+If you have a use case for using tahoe without storing the capability in
+git, we could talk about implementing that. 
+
+I don't see how it would be useful to store content in tahoe if you can't
+get it back out, so seems that some form of encryption would need to be
+involved. Perhaps it could encrypt it using your gpg key or something?
+"""]]
diff --git a/doc/forum/removing_remote.log_information_completely.mdwn b/doc/forum/removing_remote.log_information_completely.mdwn
new file mode 100644
index 0000000000..0a0e91cd40
--- /dev/null
+++ b/doc/forum/removing_remote.log_information_completely.mdwn
@@ -0,0 +1,9 @@
+in [[forum/remote-specific_meta-data/]], we have learned how to insert our own remote-specific metadata, in remote.log. now, we need a way to remove that data. for some reason, injecting commits in the `git-annex` branch doesn't quite work, because other assistants will overwrite that merge thanks to the [[git-union-merge]] driver.
+
+so far, i have found that it *can* be possible to work around this problem by repeatedly doing commits on the git-annex branch and running `git-annex sync` by hand after. it stumbles and flips around for a while, but eventually does it. it does create nice sparkles in gitk: 
+
+![a tangled mess in gitk](http://i.imgur.com/PD4ne50.png)]
+
+What is the proper way of removing entries from `remote.log`? How about propagating changes to the `synced/git-annex` branch? I can generate commits on `git-annex` using the git-annex index, and on the `git-annex` branch. But how should those changes be propagated to other branches?
+
+Thanks! --[[anarcat]]
diff --git a/doc/forum/removing_remote.log_information_completely/comment_1_033f5b8f2b306c756b3f12e0a9bc6637._comment b/doc/forum/removing_remote.log_information_completely/comment_1_033f5b8f2b306c756b3f12e0a9bc6637._comment
new file mode 100644
index 0000000000..762f1ef4c8
--- /dev/null
+++ b/doc/forum/removing_remote.log_information_completely/comment_1_033f5b8f2b306c756b3f12e0a9bc6637._comment
@@ -0,0 +1,14 @@
+[[!comment format=mdwn
+ username="joey"
+ subject="""comment 1"""
+ date="2015-09-09T19:57:26Z"
+ content="""
+There's no guaranteed way to remove lines from log files on the git-annex
+branch. A union merge can always result in a removed line being added back later.
+
+The thing to do is to add a new line, that has a newer timestamp and
+omits the information you want to remove with new information.
+
+When reading the remote.log, git-annex will look for the line for a remote
+that has the newest timestamp, and use it in preference to older lines.
+"""]]
diff --git a/doc/forum/renaming_directories_not_supported__63__.mdwn b/doc/forum/renaming_directories_not_supported__63__.mdwn
new file mode 100644
index 0000000000..42df24051e
--- /dev/null
+++ b/doc/forum/renaming_directories_not_supported__63__.mdwn
@@ -0,0 +1,45 @@
+I have an annex repository with several of my CDs. They were ripped using MusicBrainz data, causing some directory names to be in Russian. This is something I wanted to change.
+
+I have cloned this repository on my laptop, where I wanted to implement the change.
+
+When I do a git mv Пётр_Ильич_Чайковский P_I_Tchaikovsky on my laptop, I can commit the change. git annex sync also doesn't object
+
+However, the original directory remains on the origin repository, and the renamed directory is not present.
+
+Am I doing something wrong?
+
+
+
+    [master e112a02] Renamed Tchaikovsky directory.
+     9 files changed, 0 insertions(+), 0 deletions(-)
+     rename "\320\237\321\221\321\202\321\200_\320\230\320\273\321\214\320\270\321\207_\320\247\320\260\320\271\320\272\320\276\320\262\321\201\320\272\320\270\320\271/Dvo\305\231\303\241k_ Cellokonzert _ Tschaikowsky_ Rokoko-Variationen/04 Variations on a Rococo Theme, Op. 33_ Moderato quasi Andante.flac" => "P_I_Tchaikovsky/Dvo\305\231\303\241k_ Cellokonzert _ Tschaikowsky_ Rokoko-Variationen/04 Variations on a Rococo Theme, Op. 33_ Moderato quasi Andante.flac" (100%)
+     rename "\320\237\321\221\321\202\321\200_\320\230\320\273\321\214\320\270\321\207_\320\247\320\260\320\271\320\272\320\276\320\262\321\201\320\272\320\270\320\271/Dvo\305\231\303\241k_ Cellokonzert _ Tschaikowsky_ Rokoko-Variationen/05 Variations on a Rococo Theme, Op. 33_ Tema. Moderato semplice.flac" => "P_I_Tchaikovsky/Dvo\305\231\303\241k_ Cellokonzert _ Tschaikowsky_ Rokoko-Variationen/05 Variations on a Rococo Theme, Op. 33_ Tema. Moderato semplice.flac" (100%)
+     rename "\320\237\321\221\321\202\321\200_\320\230\320\273\321\214\320\270\321\207_\320\247\320\260\320\271\320\272\320\276\320\262\321\201\320\272\320\270\320\271/Dvo\305\231\303\241k_ Cellokonzert _ Tschaikowsky_ Rokoko-Variationen/06 Variations on a Rococo Theme, Op. 33_ Variazione I. Tempo del Tema.flac" => "P_I_Tchaikovsky/Dvo\305\231\303\241k_ Cellokonzert _ Tschaikowsky_ Rokoko-Variationen/06 Variations on a Rococo Theme, Op. 33_ Variazione I. Tempo del Tema.flac" (100%)
+     rename "\320\237\321\221\321\202\321\200_\320\230\320\273\321\214\320\270\321\207_\320\247\320\260\320\271\320\272\320\276\320\262\321\201\320\272\320\270\320\271/Dvo\305\231\303\241k_ Cellokonzert _ Tschaikowsky_ Rokoko-Variationen/07 Variations on a Rococo Theme, Op. 33_ Variazione II. Tempo del Tema.flac" => "P_I_Tchaikovsky/Dvo\305\231\303\241k_ Cellokonzert _ Tschaikowsky_ Rokoko-Variationen/07 Variations on a Rococo Theme, Op. 33_ Variazione II. Tempo del Tema.flac" (100%)
+     rename "\320\237\321\221\321\202\321\200_\320\230\320\273\321\214\320\270\321\207_\320\247\320\260\320\271\320\272\320\276\320\262\321\201\320\272\320\270\320\271/Dvo\305\231\303\241k_ Cellokonzert _ Tschaikowsky_ Rokoko-Variationen/08 Variations on a Rococo Theme, Op. 33_ Variazione III. Andante sostenuto.flac" => "P_I_Tchaikovsky/Dvo\305\231\303\241k_ Cellokonzert _ Tschaikowsky_ Rokoko-Variationen/08 Variations on a Rococo Theme, Op. 33_ Variazione III. Andante sostenuto.flac" (100%)
+     rename "\320\237\321\221\321\202\321\200_\320\230\320\273\321\214\320\270\321\207_\320\247\320\260\320\271\320\272\320\276\320\262\321\201\320\272\320\270\320\271/Dvo\305\231\303\241k_ Cellokonzert _ Tschaikowsky_ Rokoko-Variationen/09 Variations on a Rococo Theme, Op. 33_ Variazione IV. Andante grazioso.flac" => "P_I_Tchaikovsky/Dvo\305\231\303\241k_ Cellokonzert _ Tschaikowsky_ Rokoko-Variationen/09 Variations on a Rococo Theme, Op. 33_ Variazione IV. Andante grazioso.flac" (100%)
+     rename "\320\237\321\221\321\202\321\200_\320\230\320\273\321\214\320\270\321\207_\320\247\320\260\320\271\320\272\320\276\320\262\321\201\320\272\320\270\320\271/Dvo\305\231\303\241k_ Cellokonzert _ Tschaikowsky_ Rokoko-Variationen/10 Variations on a Rococo Theme, Op.,33_ Variazione V. Allegro moderato.flac" => "P_I_Tchaikovsky/Dvo\305\231\303\241k_ Cellokonzert _ Tschaikowsky_ Rokoko-Variationen/10 Variations on a Rococo Theme, Op.,33_ Variazione V. Allegro moderato.flac" (100%)
+     rename "\320\237\321\221\321\202\321\200_\320\230\320\273\321\214\320\270\321\207_\320\247\320\260\320\271\320\272\320\276\320\262\321\201\320\272\320\270\320\271/Dvo\305\231\303\241k_ Cellokonzert _ Tschaikowsky_ Rokoko-Variationen/11 Variations on a Rococo Theme, Op. 33_ Variazione VI. Andante.flac" => "P_I_Tchaikovsky/Dvo\305\231\303\241k_ Cellokonzert _ Tschaikowsky_ Rokoko-Variationen/11 Variations on a Rococo Theme, Op. 33_ Variazione VI. Andante.flac" (100%)
+     rename "\320\237\321\221\321\202\321\200_\320\230\320\273\321\214\320\270\321\207_\320\247\320\260\320\271\320\272\320\276\320\262\321\201\320\272\320\270\320\271/Dvo\305\231\303\241k_ Cellokonzert _ Tschaikowsky_ Rokoko-Variationen/12 Variations on a Rococo Theme, Op. 33_ Variazione VII e Coda. Allegro vivo.flac" => "P_I_Tchaikovsky/Dvo\305\231\303\241k_ Cellokonzert _ Tschaikowsky_ Rokoko-Variationen/12 Variations on a Rococo Theme, Op. 33_ Variazione VII e Coda. Allegro vivo.flac" (100%)
+    frederik@freo:~/Music$ git annex sync
+    commit  ok
+    pull A20 
+    remote: Counting objects: 83, done.
+    remote: Compressing objects: 100% (74/74), done.
+    remote: Total 83 (delta 2), reused 0 (delta 0)
+    Unpacking objects: 100% (83/83), done.
+    From ssh://A20:/mnt/usb1/Music
+       5c0bae2..72eb235  git-annex  -> A20/git-annex
+    ok
+    (merging A20/git-annex into git-annex...)
+    (Recording state in git...)
+    push A20 
+    Counting objects: 284, done.
+    Delta compression using up to 4 threads.
+    Compressing objects: 100% (257/257), done.
+    Writing objects: 100% (284/284), 21.06 KiB | 0 bytes/s, done.
+    Total 284 (delta 153), reused 0 (delta 0)
+    To ssh://A20:/mnt/usb1/Music
+       0a391ea..ad7251c  git-annex -> synced/git-annex
+       4af0061..e112a02  master -> synced/master
+    ok
diff --git a/doc/forum/renaming_directories_not_supported__63__/comment_1_94bb80e70fe619c6f1913b4a233e47ec._comment b/doc/forum/renaming_directories_not_supported__63__/comment_1_94bb80e70fe619c6f1913b4a233e47ec._comment
new file mode 100644
index 0000000000..8d51caa25c
--- /dev/null
+++ b/doc/forum/renaming_directories_not_supported__63__/comment_1_94bb80e70fe619c6f1913b4a233e47ec._comment
@@ -0,0 +1,15 @@
+[[!comment format=mdwn
+ username="CandyAngel"
+ subject="comment 1"
+ date="2015-06-05T07:21:44Z"
+ content="""
+Looks like you missed a step from your output (difficult to be sure because you don't show a full transcript showing a rename and it not being present in the other repo).
+
+Have you updated the remote repo where the changes aren't present? 'git annex sync' or 'git annex merge'?
+
+If you made the changes in repoA and ran sync there, it pushes the changes to repoB, but it can't update repoB's working directory remotely (restriction from git). You have to update it locally to see them.
+
+First few paragraphs of [sync](http://git-annex.branchable.com/sync/) mention this.
+
+So if you run sync or merge in the remote repository, you should see your changes.
+"""]]
diff --git a/doc/forum/renaming_directories_not_supported__63__/comment_2_4bd6386dbd5cb907a9c85fb240265ee5._comment b/doc/forum/renaming_directories_not_supported__63__/comment_2_4bd6386dbd5cb907a9c85fb240265ee5._comment
new file mode 100644
index 0000000000..4896dd0f29
--- /dev/null
+++ b/doc/forum/renaming_directories_not_supported__63__/comment_2_4bd6386dbd5cb907a9c85fb240265ee5._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="frederik@ffbea6a549cb3f460d110386c0f634c1ddc6a68a"
+ nickname="frederik"
+ subject="comment 2"
+ date="2015-06-05T07:41:55Z"
+ content="""
+Thanks, I had missed that part.
+"""]]
diff --git a/doc/forum/repair_stuck_on_ls-tree_command.mdwn b/doc/forum/repair_stuck_on_ls-tree_command.mdwn
new file mode 100644
index 0000000000..3ff9f56c57
--- /dev/null
+++ b/doc/forum/repair_stuck_on_ls-tree_command.mdwn
@@ -0,0 +1,54 @@
+Hi,
+
+I have run the git annex repair command on one of my repositories and it has been running for 38h hours now. I have monitored the process and its sons with htop and it seems that git annex repair is stuck.
+
+It repeatedly launches the following command: 
+
+    git --git-dir=/home/vincent/photos2/.git --work-tree=/home/vincent/photos2 ls-tree --full-tree -z -r -- 5fe5193e079631c0ceac0688ae0a6c1636491b61
+
+I have tried to execute it by hand and it produces a lot of output. I have redirected it into some file in order to count the content produced by this command.
+
+    vincent@berlioz:~/photos2/.git$ git --git-dir=/home/vincent/photos2/.git --work-tree=/home/vincent/photos2 ls-tree --full-tree -z -r -- 5fe5193e079631c0ceac0688ae0a6c1636491b61 > /tmp/ls-tree
+    vincent@berlioz:/tmp$ wc ls-tree 
+           0   232525 11611220 ls-tree
+
+Could this cause some troubles to git-annex ? Maybe my repository is just too big and it demands some time to deal with the whole thing?
+
+Before doing the git annex repair command I have tried to delete a distant annex (hosted on a raspberry pi, running an arm raspbian) from this one, using the git annex webapp website. Since it was kind of stuck I asked webapp to shutdown. I ran git annex repair for some time. It looked like stuck. I stopped it, then tried git fsck. It ran well. I tried git annex fsck, it took all my RAM and swap. The process was eating 14 Giga bytes of memory. I tried git gc and the same memory problem occured. I ran git annex repair again and it has been running for 38 hours now. I am lost. Please help.
+
+    vincent@berlioz:~$ LANG=C apt-cache policy git-annex
+    git-annex:
+      Installed: 5.20141125
+      Candidate: 5.20141125
+      Package pin: 5.20141125
+      Version table:
+     *** 5.20141125 950
+            400 http://ftp.fr.debian.org/debian/ testing/main amd64 Packages
+             50 http://ftp.fr.debian.org/debian/ unstable/main amd64 Packages
+            100 /var/lib/dpkg/status
+         3.20120629 950
+            900 http://ftp.fr.debian.org/debian/ wheezy/main amd64 Packages
+
+    vincent@berlioz:~/photos2$ du -sh .
+    152G	.
+
+Info gathered from another client annex:
+
+    vincent@tigrou:/mnt/mars/images/photos2$ git annex info
+    repository mode: indirect
+    trusted repositories: 0
+    semitrusted repositories: 11
+        8<------------skipped repositories list------------->8
+    untrusted repositories: 1
+    	a357bb9e-7a9d-4112-b74c-13707e2c7f85 -- vincent@rhett:~/photos [gateway]
+    transfers in progress: none
+    available local disk space: 945.91 gigabytes (+1 megabyte reserved)
+    bad keys size: 368 bytes (clean up with git-annex unused)
+    local annex keys: 76070
+    local annex size: 383.91 gigabytes
+    annexed files in working tree: 59117
+    size of annexed files in working tree: 319.73 gigabytes
+    bloom filter size: 16 mebibytes (15.2% full)
+    backend usage: 
+            SHA256E: 101902
+            SHA256: 33285
diff --git a/doc/forum/repair_stuck_on_ls-tree_command/comment_10_791c50f8a2284b704e34cacf15637341._comment b/doc/forum/repair_stuck_on_ls-tree_command/comment_10_791c50f8a2284b704e34cacf15637341._comment
new file mode 100644
index 0000000000..3058cbaa08
--- /dev/null
+++ b/doc/forum/repair_stuck_on_ls-tree_command/comment_10_791c50f8a2284b704e34cacf15637341._comment
@@ -0,0 +1,14 @@
+[[!comment format=mdwn
+ username="vho"
+ subject="comment 10"
+ date="2014-12-18T10:27:07Z"
+ content="""
+This one should work:
+https://www.dropbox.com/s/t7b406wm6m3vm6c/photos2.tar.xz.gpg?dl=0
+
+    $ md5sum photos2.tar.xz.gpg
+    df2b9bde3d1ad23d9f3c4247d2f5b21a  photos2.tar.xz.gpg
+    $ sha1sum photos2.tar.xz.gpg
+    cac6f8c07a1d95fb48e3d9aa0f4699a2c77c00d598  photos2.tar.xz.gpg
+
+"""]]
diff --git a/doc/forum/repair_stuck_on_ls-tree_command/comment_11_1d951126a9633b206dffbc77bfc65f6a._comment b/doc/forum/repair_stuck_on_ls-tree_command/comment_11_1d951126a9633b206dffbc77bfc65f6a._comment
new file mode 100644
index 0000000000..a569fcbdff
--- /dev/null
+++ b/doc/forum/repair_stuck_on_ls-tree_command/comment_11_1d951126a9633b206dffbc77bfc65f6a._comment
@@ -0,0 +1,12 @@
+[[!comment format=mdwn
+ username="joey"
+ subject="""comment 11"""
+ date="2015-02-09T21:13:28Z"
+ content="""
+Finally got back to this. I downloaded the file.
+
+You may be able to fix your repository by running `git annex forget`
+
+I guess this is the same problem described in
+[[bugs/git-annex_branch_shows_commit_with_looong_commitlog]]
+"""]]
diff --git a/doc/forum/repair_stuck_on_ls-tree_command/comment_12_1a844376915c6a40ad03c2b9b599367b._comment b/doc/forum/repair_stuck_on_ls-tree_command/comment_12_1a844376915c6a40ad03c2b9b599367b._comment
new file mode 100644
index 0000000000..397214c6dc
--- /dev/null
+++ b/doc/forum/repair_stuck_on_ls-tree_command/comment_12_1a844376915c6a40ad03c2b9b599367b._comment
@@ -0,0 +1,10 @@
+[[!comment format=mdwn
+ username="vho"
+ subject="comment 12"
+ date="2015-11-30T15:23:22Z"
+ content="""
+Sorry I didn't see your answer before!
+
+Thanks! Fortunately I kept a copy of the defective repository. I am going to test this =)
+
+"""]]
diff --git a/doc/forum/repair_stuck_on_ls-tree_command/comment_1_9555c925516ce3be83b4dd17d587100f._comment b/doc/forum/repair_stuck_on_ls-tree_command/comment_1_9555c925516ce3be83b4dd17d587100f._comment
new file mode 100644
index 0000000000..f2c7a67d28
--- /dev/null
+++ b/doc/forum/repair_stuck_on_ls-tree_command/comment_1_9555c925516ce3be83b4dd17d587100f._comment
@@ -0,0 +1,37 @@
+[[!comment format=mdwn
+ username="vho"
+ subject="Update"
+ date="2014-12-01T15:47:23Z"
+ content="""
+It seems I could sort things of.
+
+I have run git annex sync from another annex. It took 30 minutes but could synchronize with the slow annex. Then I launched git annex sync on the slow annex (the one which seemed to had troubles with the repair command). It took 278 minutes, used all my ram and swap, but managed to end successfully.
+
+I launched git annex sync again today, another 279 minutes:
+
+    vincent@berlioz:~/photos2$ time git annex sync
+    commit  ok
+    pull tigrou
+    Error reading response length from authentication socket.
+    Compression automatique du dépôt en tâche de fond pour optimiser les performances.
+    Voir \"git help gc\" pour toute information sur le nettoyage manuel.
+    ok
+    pull gateway
+    Compression automatique du dépôt en tâche de fond pour optimiser les performances.
+    Voir \"git help gc\" pour toute information sur le nettoyage manuel.
+    ok
+    push gateway
+    fatal: The remote end hung up unexpectedly
+     
+      Pushing to gateway failed.
+      
+      (non-fast-forward problems can be solved by setting receive.denyNonFastforwards to false in the remote's git config)
+    failed
+    git-annex: sync: 1 failed
+    
+    real    279m27.760s
+    user    7m15.159s
+    sys     3m4.340s
+
+I will try to do some git gc.
+"""]]
diff --git a/doc/forum/repair_stuck_on_ls-tree_command/comment_2_17327b90d09f8dd0bf7f359c16cf69dd._comment b/doc/forum/repair_stuck_on_ls-tree_command/comment_2_17327b90d09f8dd0bf7f359c16cf69dd._comment
new file mode 100644
index 0000000000..fa046da218
--- /dev/null
+++ b/doc/forum/repair_stuck_on_ls-tree_command/comment_2_17327b90d09f8dd0bf7f359c16cf69dd._comment
@@ -0,0 +1,21 @@
+[[!comment format=mdwn
+ username="joey"
+ subject="""comment 2"""
+ date="2014-12-02T17:24:30Z"
+ content="""
+According to the ls-tree output you posted, the git tree object
+5fe5193e079631c0ceac0688ae0a6c1636491b61 is somehow 11 mb
+in size, which seems quite enormous, since a git tree object
+is just a list of files in a directory in the repo.
+
+Does your repository actually contain a directory that big? 
+(This seems unlikely, since git annex info says there are only 59117 annxed
+files.)
+
+What is in the output of `git-ls-tree 5fe5193e079631c0ceac0688ae0a6c1636491b61`
+that is so enormous?
+
+I think you need to figure this out to get anywhere, because it's quite
+strange. Or, if you're willing to put the git repository someplace
+(minus .git/annex/), I could take a look at it.
+"""]]
diff --git a/doc/forum/repair_stuck_on_ls-tree_command/comment_2_2ae0c755f5a0bf5c93afe5e081fcb915._comment b/doc/forum/repair_stuck_on_ls-tree_command/comment_2_2ae0c755f5a0bf5c93afe5e081fcb915._comment
new file mode 100644
index 0000000000..da2f90797a
--- /dev/null
+++ b/doc/forum/repair_stuck_on_ls-tree_command/comment_2_2ae0c755f5a0bf5c93afe5e081fcb915._comment
@@ -0,0 +1,19 @@
+[[!comment format=mdwn
+ username="vho"
+ subject="Still stuck"
+ date="2014-12-02T09:59:20Z"
+ content="""
+vincent@berlioz:~/photos2$ git config --global pack.packSizeLimit 100m
+vincent@berlioz:~/photos2$ git config --global pack.windowMemory 100m
+vincent@berlioz:~/photos2$ git config --global pack.threads 1
+vincent@berlioz:~/photos2$ time git gc
+error: pack-objects died of signal 9
+error: failed to run repack
+
+real    21m29.451s
+user    1m31.414s
+sys     0m9.141s
+
+:(
+
+"""]]
diff --git a/doc/forum/repair_stuck_on_ls-tree_command/comment_4_875f9e36a94264a92eb097e9f4bda444._comment b/doc/forum/repair_stuck_on_ls-tree_command/comment_4_875f9e36a94264a92eb097e9f4bda444._comment
new file mode 100644
index 0000000000..77f7a8535a
--- /dev/null
+++ b/doc/forum/repair_stuck_on_ls-tree_command/comment_4_875f9e36a94264a92eb097e9f4bda444._comment
@@ -0,0 +1,20 @@
+[[!comment format=mdwn
+ username="vho"
+ subject="comment 4"
+ date="2014-12-02T21:11:39Z"
+ content="""
+Thanks for your time and help.
+
+http://hobeika.fr/~vincent/git/ls-tree
+
+    vincent@berlioz:~$ wc -l /tmp/ls-tree 
+    4103 /tmp/ls-tree
+
+    vincent@berlioz:~/photos2$ du -sh --exclude=.git/annex
+    863M	.
+
+I'll have to find some place to store that.
+
+Yet I don't think my repository holds such a big directory since there are no problems on other annexes.
+
+"""]]
diff --git a/doc/forum/repair_stuck_on_ls-tree_command/comment_5_d5c29dc4ea49542f7053bf2e83e0f07f._comment b/doc/forum/repair_stuck_on_ls-tree_command/comment_5_d5c29dc4ea49542f7053bf2e83e0f07f._comment
new file mode 100644
index 0000000000..5af49a7e44
--- /dev/null
+++ b/doc/forum/repair_stuck_on_ls-tree_command/comment_5_d5c29dc4ea49542f7053bf2e83e0f07f._comment
@@ -0,0 +1,17 @@
+[[!comment format=mdwn
+ username="https://www.google.com/accounts/o8/id?id=AItOawnPgn611P6ym5yyL0BS8rUzO0_ZKRldMt0"
+ nickname="Samuel"
+ subject="Not so big tree"
+ date="2014-12-03T07:36:22Z"
+ content="""
+The output of the ls-tree command suggests this tree is the git-annex branch at some point in time.
+
+I tried on my photos annex to ls-tree the git-annex branch and got.
+
+    $ git ls-tree -r --full-tree git-annex | wc
+    156405  625620 23685538
+
+My git-annex tree is about 22Mib in size, that means twice bigger than yours (no
+dumb joke intended) and I don't suffer such problem.
+
+"""]]
diff --git a/doc/forum/repair_stuck_on_ls-tree_command/comment_6_061770159851c0f06a962937dff035b9._comment b/doc/forum/repair_stuck_on_ls-tree_command/comment_6_061770159851c0f06a962937dff035b9._comment
new file mode 100644
index 0000000000..5f73e65cf0
--- /dev/null
+++ b/doc/forum/repair_stuck_on_ls-tree_command/comment_6_061770159851c0f06a962937dff035b9._comment
@@ -0,0 +1,30 @@
+[[!comment format=mdwn
+ username="vho"
+ subject="Trying to make git gc pass with a huge swap"
+ date="2014-12-11T16:52:16Z"
+ content="""
+I have created a 100 GB swap file and ran git gc again.
+It ate my 8 GB of RAM + 38 GB of swap. It ended successfully:
+
+    vincent@berlioz:~/photos2$ time git gc --aggressive
+    Décompte des objets: 2263783, fait.
+    Compression des objets: 100% (2253014/2253014), fait.
+    Écriture des objets: 100% (2263783/2263783), fait.
+    Total 2263783 (delta 1567807), reused 284965 (delta 0)
+    Suppression des objets dupliqués: 100% (256/256), fait.
+    Vérification de la connectivité: 2263783, fait.
+    
+    real	110m18.565s
+    user	37m38.225s
+    sys	3m23.457s
+
+Yet the following git annex sync didn't do better than previous ones and took 285 minutes to complete.
+
+I am going to check my hard disk drive to see if there aren't any third party problem.
+
+    vincent@berlioz:~/photos2$ du -sh --exclude=.git/annex
+    510M	.
+
+Yet it shrank by 300 MB.
+
+"""]]
diff --git a/doc/forum/repair_stuck_on_ls-tree_command/comment_7_5d9e5fd148d5f9e918ad818e07009d69._comment b/doc/forum/repair_stuck_on_ls-tree_command/comment_7_5d9e5fd148d5f9e918ad818e07009d69._comment
new file mode 100644
index 0000000000..b27b503773
--- /dev/null
+++ b/doc/forum/repair_stuck_on_ls-tree_command/comment_7_5d9e5fd148d5f9e918ad818e07009d69._comment
@@ -0,0 +1,30 @@
+[[!comment format=mdwn
+ username="vho"
+ subject="update"
+ date="2014-12-18T00:43:22Z"
+ content="""
+Hi,
+
+I have uploaded the repository to:
+http://dl.free.fr/bczxhyOhy
+
+You can decrypt it with your gpg key. It weights 200 Mo.
+
+Moreover when I try to run git annex whereis on files present only on the defective annex from another annex I get the following error:
+
+    $ git annex whereis IMG_4701.JPG
+    whereis IMG_4701.JPG (0 copies) failed
+    git-annex: whereis: 1 failed
+
+I guess the defective annex could not completely synchronize with the others.
+
+I also tried to look at the git tree with gitk --all.
+Well there seems to be a lot of commit with message update (recovery from race)
+I am forced to kill gitk because it starts lagging a lot.
+
+Hopefully you will be able to provide me some hints on how to resolve this issue.
+
+Best regards
+-- 
+vho
+"""]]
diff --git a/doc/forum/repair_stuck_on_ls-tree_command/comment_8_726c9a887b7df1833d7aef3bdce50517._comment b/doc/forum/repair_stuck_on_ls-tree_command/comment_8_726c9a887b7df1833d7aef3bdce50517._comment
new file mode 100644
index 0000000000..f79b16f198
--- /dev/null
+++ b/doc/forum/repair_stuck_on_ls-tree_command/comment_8_726c9a887b7df1833d7aef3bdce50517._comment
@@ -0,0 +1,7 @@
+[[!comment format=mdwn
+ username="vho"
+ subject="(recovery from race)"
+ date="2014-12-18T00:46:09Z"
+ content="""
+git show 7dc2be23ddb9bda2edc0f01d4bbec2fdb5216763 seems quite abnormal.
+"""]]
diff --git a/doc/forum/repair_stuck_on_ls-tree_command/comment_9_163cdd18380a13aaa13d68d516af1e30._comment b/doc/forum/repair_stuck_on_ls-tree_command/comment_9_163cdd18380a13aaa13d68d516af1e30._comment
new file mode 100644
index 0000000000..f0d02075b3
--- /dev/null
+++ b/doc/forum/repair_stuck_on_ls-tree_command/comment_9_163cdd18380a13aaa13d68d516af1e30._comment
@@ -0,0 +1,7 @@
+[[!comment format=mdwn
+ username="vho"
+ subject="failed upload"
+ date="2014-12-18T09:18:14Z"
+ content="""
+It seems the upload failed at some point. I am going to share it somewhere else
+"""]]
diff --git a/doc/forum/repo_corruption_in_usb_external_drive.mdwn b/doc/forum/repo_corruption_in_usb_external_drive.mdwn
new file mode 100644
index 0000000000..989126837e
--- /dev/null
+++ b/doc/forum/repo_corruption_in_usb_external_drive.mdwn
@@ -0,0 +1,41 @@
+I'm sorry to ask this again, but I'm not being able to find my previous message regarding this issue. I'm finding very hard to find older posts in forums. Is there a way to search for all my posted messages?
+Anyway, after some automount problems (changed mount point name), my usb repo (direct) ended in a weird state.
+See for instance the screenshot: 
+
+In addition making a git-annex info gives the following:
+
+ * repository mode: direct
+ * trusted repositories: error: refs/heads/master does not point to a valid object!
+ * error: refs/heads/synced/git-annex does not point to a valid object!
+ * error: refs/heads/synced/master does not point to a valid object!
+ * error: refs/remotes/sapo/annex/direct/master does not point to a valid object!
+ * error: refs/remotes/sapo/git-annex does not point to a valid object!
+ * error: refs/remotes/sapo/master does not point to a valid object!
+ * error: refs/synced/1912d5a7-3929-47f0-8e25-a071d7079cc4/git-annex does not point to a valid object!
+ * error: refs/synced/1912d5a7-3929-47f0-8e25-a071d7079cc4/master does not point to a valid object!
+ * error: refs/heads/master does not point to a valid object!
+ * error: refs/heads/synced/git-annex does not point to a valid object!
+ * error: refs/heads/synced/master does not point to a valid object!
+ * error: refs/remotes/sapo/annex/direct/master does not point to a valid object!
+ * error: refs/remotes/sapo/git-annex does not point to a valid object!
+ * error: refs/remotes/sapo/master does not point to a valid object!
+ * error: refs/synced/1912d5a7-3929-47f0-8e25-a071d7079cc4/git-annex does not point to a valid object!
+ * error: refs/synced/1912d5a7-3929-47f0-8e25-a071d7079cc4/master does not point to a valid object!
+ * fatal: index file smaller than expected
+ * git-annex: failed to read sha from git write-tree
+
+
+What should I do? delete the usb repo and start again. Thank god this is just a trial.
+
+What happens if a file gets corrupted. Lets say it is corrupted in repo A, and fine in repo B. Will the good copy be overridden?
+
+git-repair takes a long time and seems to be stuck there, or maybe it just takes a long time (10Gb repo). Is this normal?
+    > git-annex repair
+    Running git fsck ...
+    Initialized empty Git repository in /tmp/tmprepo.1/.git/
+    Trying to recover missing objects from remote sapo.
+    Unpacking all pack files.
+    Unpacking objects: 100% (348848/348848), done.
+
+
+Thanks in advance.
diff --git a/doc/forum/repo_corruption_in_usb_external_drive/comment_1_d9122bb0cc3551d92877c299a25b9c9e._comment b/doc/forum/repo_corruption_in_usb_external_drive/comment_1_d9122bb0cc3551d92877c299a25b9c9e._comment
new file mode 100644
index 0000000000..ff325983d3
--- /dev/null
+++ b/doc/forum/repo_corruption_in_usb_external_drive/comment_1_d9122bb0cc3551d92877c299a25b9c9e._comment
@@ -0,0 +1,10 @@
+[[!comment format=mdwn
+ username="http://joeyh.name/"
+ ip="209.250.56.2"
+ subject="comment 1"
+ date="2014-07-15T18:50:29Z"
+ content="""
+It's normal for git-repair to take a long time.
+
+git-annex uses checksums to detect if a file gets corrupted.
+"""]]
diff --git a/doc/forum/repos_syncs_but_fails_to_show_files_in_one_direction.mdwn b/doc/forum/repos_syncs_but_fails_to_show_files_in_one_direction.mdwn
new file mode 100644
index 0000000000..0b66c848ee
--- /dev/null
+++ b/doc/forum/repos_syncs_but_fails_to_show_files_in_one_direction.mdwn
@@ -0,0 +1,7 @@
+I have a repo on two hosts. One host is my laptop which is configured as "manual" repo due to limited disk space and the other host is my NAS as "full backup" repo. My laptop connects via SSH to the NAS.
+
+Strange thing is, syncing looks good. New files created on either side lead to a transfer. However, it doesn't show changes on the NAS that have been triggered on the notebook. For example, folders are missing and I can't see changes in files on the NAS which have been modified and synced from the laptop.
+
+I suspect a problem being stuck with some old branch. Unfortunately, I have little to know knowledge to git commands. :-(
+
+Any idea on how to troubleshoot the issue?
diff --git a/doc/forum/repos_syncs_but_fails_to_show_files_in_one_direction/comment_1_412c319a1610b447fcf372584c2bb15c._comment b/doc/forum/repos_syncs_but_fails_to_show_files_in_one_direction/comment_1_412c319a1610b447fcf372584c2bb15c._comment
new file mode 100644
index 0000000000..ebac42c49a
--- /dev/null
+++ b/doc/forum/repos_syncs_but_fails_to_show_files_in_one_direction/comment_1_412c319a1610b447fcf372584c2bb15c._comment
@@ -0,0 +1,49 @@
+[[!comment format=mdwn
+ username="torpidus"
+ avatar="http://cdn.libravatar.org/avatar/997fb77747b008a26383426ae6561368"
+ subject="comment 1"
+ date="2018-09-12T19:05:40Z"
+ content="""
+possible found a first trace. the committer on the NAS side crashes. log file tells:
+
+    [2018-09-12 20:54:19.423764064] Committer: Adding ___git-an..ed_on_NAS
+    add ___git-annex_file_created_on_NAS ok
+    [2018-09-12 20:54:19.524746614] Committer: committing 1 changes
+    [2018-09-12 20:54:19.524993706] Committer: Committing changes to git
+    (recording state in git...)
+    [2018-09-12 20:54:19.590762812] feed: git [\"--git-dir=.git\",\"--work-tree=.\",\"--literal-pathspecs\",\"-c\",\"core.bare=false\",\"update-index\",\"-z\",\"--index-info\"]
+    [2018-09-12 20:54:19.857623683] process done ExitSuccess
+    [2018-09-12 20:54:19.85896977] read: git [\"--git-dir=.git\",\"--work-tree=.\",\"--literal-pathspecs\",\"-c\",\"core.bare=false\",\"symbolic-ref\",\"-q\",\"HEAD\"]
+    [2018-09-12 20:54:19.880377072] process done ExitSuccess
+    [2018-09-12 20:54:19.880645044] read: git [\"--git-dir=.git\",\"--work-tree=.\",\"--literal-pathspecs\",\"-c\",\"core.bare=false\",\"show-ref\",\"--hash\",\"refs/heads/master\"]
+    [2018-09-12 20:54:20.836749623] process done ExitSuccess
+    [2018-09-12 20:54:20.837005176] read: git [\"--git-dir=.git\",\"--work-tree=.\",\"--literal-pathspecs\",\"-c\",\"core.bare=false\",\"write-tree\"]
+    [2018-09-12 20:54:21.296523156] process done ExitSuccess
+    [2018-09-12 20:54:21.304290499] read: git [\"--git-dir=.git\",\"--work-tree=.\",\"--literal-pathspecs\",\"-c\",\"core.bare=false\",\"rev-parse\",\"1469f15a8f89aae0e9b582e141e4e31f0c0ea31f:\"]
+    [2018-09-12 20:54:21.615407387] process done ExitSuccess
+    [2018-09-12 20:54:21.61567249] chat: git [\"--git-dir=.git\",\"--work-tree=.\",\"--literal-pathspecs\",\"-c\",\"core.bare=false\",\"commit-tree\",\"0ec03203b806a3125bb6677c363897b346473aab\",\"--no-gpg-sign\",\"-p\",\"1469f15a8f89aae0e9b582e141e4e31f0c0ea31f\"]
+    [2018-09-12 20:54:21.952881183] process done ExitSuccess
+    [2018-09-12 20:54:21.953184061] call: git [\"--git-dir=.git\",\"--work-tree=.\",\"--literal-pathspecs\",\"-c\",\"core.bare=false\",\"update-ref\",\"refs/heads/master\",\"b15389e29d1853771a4993954a297d5dea99b21d\"]
+    [2018-09-12 20:54:22.263498786] process done ExitSuccess
+    [2018-09-12 20:54:22.26375159] read: git [\"--git-dir=.git\",\"--work-tree=.\",\"--literal-pathspecs\",\"-c\",\"core.bare=false\",\"symbolic-ref\",\"-q\",\"HEAD\"]
+    [2018-09-12 20:54:22.297865786] process done ExitSuccess
+    [2018-09-12 20:54:22.306865822] read: git [\"--git-dir=.git\",\"--work-tree=.\",\"--literal-pathspecs\",\"-c\",\"core.bare=false\",\"show-ref\",\"refs/heads/master\"]
+    [2018-09-12 20:54:22.808817784] process done ExitSuccess
+    [2018-09-12 20:54:22.809135542] call: git [\"--git-dir=.git\",\"--work-tree=.\",\"--literal-pathspecs\",\"-c\",\"core.bare=false\",\"branch\",\"-f\",\"synced/master\",\"refs/heads/master\"]
+    [2018-09-12 20:54:23.32085476] process done ExitSuccess
+    [2018-09-12 20:54:23.321141703] call: git [\"--git-dir=.git\",\"--work-tree=.\",\"--literal-pathspecs\",\"-c\",\"core.bare=false\",\"branch\",\"-f\",\"master\",\"refs/heads/master\"]
+    fatal: Cannot force update the current branch.
+    [2018-09-12 20:54:23.339070508] process done ExitFailure 128
+    Committer crashed: failed to update refs/heads/master
+    [2018-09-12 20:54:23.343355069] read: git [\"--git-dir=.git\",\"--work-tree=.\",\"--literal-pathspecs\",\"-c\",\"core.bare=false\",\"symbolic-ref\",\"-q\",\"HEAD\"]
+    [2018-09-12 20:54:23.343600554] Committer: warning Committer crashed: failed to update refs/heads/master
+    [2018-09-12 20:54:23.36031901] process done ExitSuccess
+    [2018-09-12 20:54:23.360576051] read: git [\"--git-dir=.git\",\"--work-tree=.\",\"--literal-pathspecs\",\"-c\",\"core.bare=false\",\"show-ref\",\"refs/heads/master\"]
+    [2018-09-12 20:54:23.68816538] process done ExitSuccess
+    [2018-09-12 20:54:23.690946926] read: git [\"--git-dir=.git\",\"--work-tree=.\",\"--literal-pathspecs\",\"-c\",\"core.bare=false\",\"symbolic-ref\",\"-q\",\"HEAD\"]
+    [2018-09-12 20:54:23.710158388] process done ExitSuccess
+    [2018-09-12 20:54:23.71041453] read: git [\"--git-dir=.git\",\"--work-tree=.\",\"--literal-pathspecs\",\"-c\",\"core.bare=false\",\"show-ref\",\"refs/heads/master\"]
+    [2018-09-12 20:54:23.993586343] process done ExitSuccess
+    [2018-09-12 20:54:23.994052834] read: git [\"--git-dir=.git\",\"--work-tree=.\",\"--literal-pathspecs\",\"-c\",\"core.bare=false\",\"log\",\"refs/heads/master..refs/heads/synced/master\",\"--pretty=%H\",\"-n1\"]
+    [2018-09-12 20:54:24.276305404] process done ExitSuccess
+"""]]
diff --git a/doc/forum/repos_syncs_but_fails_to_show_files_in_one_direction/comment_2_c10d614039b82c5be289e8e34bc70f86._comment b/doc/forum/repos_syncs_but_fails_to_show_files_in_one_direction/comment_2_c10d614039b82c5be289e8e34bc70f86._comment
new file mode 100644
index 0000000000..3dc63e78db
--- /dev/null
+++ b/doc/forum/repos_syncs_but_fails_to_show_files_in_one_direction/comment_2_c10d614039b82c5be289e8e34bc70f86._comment
@@ -0,0 +1,17 @@
+[[!comment format=mdwn
+ username="joey"
+ subject="""comment 2"""
+ date="2018-09-12T19:35:54Z"
+ content="""
+	[2018-09-12 20:54:23.321141703] call: git [\"--git-dir=.git\",\"--work-tree=.\",\"--literal-pathspecs\",\"-c\",\"core.bare=false\",\"branch\",\"-f\",\"master\",\"refs/heads/master\"]
+	fatal: Cannot force update the current branch.
+
+Good spotting. This problem was reported by some people 
+[back in 2013](forum/Can_Not_Sync_to_Git_Repo),
+but not since then. They were all using direct mode, and I see you are
+too.
+
+But, as far as I can see, current git-annex *never* runs "git branch -f".
+
+So, I think you may have an old version of git-annex there..
+"""]]
diff --git a/doc/forum/repos_syncs_but_fails_to_show_files_in_one_direction/comment_3_4fec14b132bea863b58aa07e03fc31cc._comment b/doc/forum/repos_syncs_but_fails_to_show_files_in_one_direction/comment_3_4fec14b132bea863b58aa07e03fc31cc._comment
new file mode 100644
index 0000000000..3f2d36e1e9
--- /dev/null
+++ b/doc/forum/repos_syncs_but_fails_to_show_files_in_one_direction/comment_3_4fec14b132bea863b58aa07e03fc31cc._comment
@@ -0,0 +1,11 @@
+[[!comment format=mdwn
+ username="torpidus"
+ avatar="http://cdn.libravatar.org/avatar/997fb77747b008a26383426ae6561368"
+ subject="comment 3"
+ date="2018-09-16T18:45:15Z"
+ content="""
+Thanks Joey,
+after your comment, I decided to migrate the repo's contents to a new one in v6 format. However, I run into several other issues. I wished there was a live support that I can pay with coffee or beer... :-) Instead, I try to figure around the issues by myself, hopefully not spamming too much here into this forum. :-(
+
+BTW, thank you so much for the effort you have put into git-annex!
+"""]]
diff --git a/doc/forum/reserving_space_with_directory_special_remotes.mdwn b/doc/forum/reserving_space_with_directory_special_remotes.mdwn
new file mode 100644
index 0000000000..fea762c397
--- /dev/null
+++ b/doc/forum/reserving_space_with_directory_special_remotes.mdwn
@@ -0,0 +1,2 @@
+Can diskreserve be done with directory special remotes?
+Where should such per directory remote setting be put?
diff --git a/doc/forum/reserving_space_with_directory_special_remotes/comment_1_cd17b624704d93b51931023f69573323._comment b/doc/forum/reserving_space_with_directory_special_remotes/comment_1_cd17b624704d93b51931023f69573323._comment
new file mode 100644
index 0000000000..83ef649ea3
--- /dev/null
+++ b/doc/forum/reserving_space_with_directory_special_remotes/comment_1_cd17b624704d93b51931023f69573323._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="http://joeyh.name/"
+ ip="4.154.7.235"
+ subject="comment 1"
+ date="2013-07-10T23:50:53Z"
+ content="""
+Not currently supported. I've added a todo: [[todo/free_space_checking_for_local_special_remotes]]
+"""]]
diff --git a/doc/forum/reserving_space_with_directory_special_remotes/comment_2_877ca1be23d1484a8a30cdaeb6630053._comment b/doc/forum/reserving_space_with_directory_special_remotes/comment_2_877ca1be23d1484a8a30cdaeb6630053._comment
new file mode 100644
index 0000000000..c34052ae3c
--- /dev/null
+++ b/doc/forum/reserving_space_with_directory_special_remotes/comment_2_877ca1be23d1484a8a30cdaeb6630053._comment
@@ -0,0 +1,15 @@
+[[!comment format=mdwn
+ username="https://www.google.com/accounts/o8/id?id=AItOawln3ckqKx0x_xDZMYwa9Q1bn4I06oWjkog"
+ nickname="Michael"
+ subject="comment 2"
+ date="2013-07-10T23:56:08Z"
+ content="""
+I was looking at git-annex git and saw this and thought it might be supported.
+
+commit 5cc76098ca7b702772ccf37a47f03da088148003
+Author: Joey Hess 
+Date:   Fri Apr 20 16:24:44 2012 -0400
+
+    Directory special remotes now check annex.diskreserve.
+
+"""]]
diff --git a/doc/forum/reserving_space_with_directory_special_remotes/comment_3_65910eeaf3c6fcfd03f22c2957293232._comment b/doc/forum/reserving_space_with_directory_special_remotes/comment_3_65910eeaf3c6fcfd03f22c2957293232._comment
new file mode 100644
index 0000000000..245849c5f8
--- /dev/null
+++ b/doc/forum/reserving_space_with_directory_special_remotes/comment_3_65910eeaf3c6fcfd03f22c2957293232._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="http://joeyh.name/"
+ ip="4.154.7.235"
+ subject="comment 3"
+ date="2013-07-11T16:17:36Z"
+ content="""
+Shows what I know! It is supported. Repurposed the bug report to allow configuring a different diskreserve for the remote than the main annex.diskreserve..
+"""]]
diff --git a/doc/forum/retrieving_previous_versions.mdwn b/doc/forum/retrieving_previous_versions.mdwn
new file mode 100644
index 0000000000..7626b7935b
--- /dev/null
+++ b/doc/forum/retrieving_previous_versions.mdwn
@@ -0,0 +1,7 @@
+Hi,
+
+This might be a stupid question, but I did not find any information about it.
+Can I retrieve previous versions of a file?
+Let's say, I wanna do a "git annex get file", but considering a specific commit id. Is it possible? Are all the versions of the files kept inside .git/annex/objects?
+
+Thanks!
diff --git a/doc/forum/retrieving_previous_versions/comment_1_a4e83f688d4ec9177e7bf520f12ed26d._comment b/doc/forum/retrieving_previous_versions/comment_1_a4e83f688d4ec9177e7bf520f12ed26d._comment
new file mode 100644
index 0000000000..ab2ecee317
--- /dev/null
+++ b/doc/forum/retrieving_previous_versions/comment_1_a4e83f688d4ec9177e7bf520f12ed26d._comment
@@ -0,0 +1,11 @@
+[[!comment format=mdwn
+ username="http://joey.kitenet.net/"
+ nickname="joey"
+ subject="comment 1"
+ date="2012-04-24T21:14:15Z"
+ content="""
+To get to a specific version of a file, you need to have a tag or a branch that includes that version of the file. Check out the branch and `git annex get $file`. 
+
+(Of course, even without a tag or branch, old file versions are retained, unless dropped with `unused`/`dropunused`.
+So you could even `git checkout $COMMITID`.)
+"""]]
diff --git a/doc/forum/retrieving_previous_versions/comment_2_e9251f66154e49a77be661c4b4918e18._comment b/doc/forum/retrieving_previous_versions/comment_2_e9251f66154e49a77be661c4b4918e18._comment
new file mode 100644
index 0000000000..04d7b8e6f5
--- /dev/null
+++ b/doc/forum/retrieving_previous_versions/comment_2_e9251f66154e49a77be661c4b4918e18._comment
@@ -0,0 +1,16 @@
+[[!comment format=mdwn
+ username="https://www.google.com/accounts/o8/id?id=AItOawlbKgUeSjtEVC2PoXZsOEcQqTDwI52fsb8"
+ nickname="John"
+ subject="what is stored?"
+ date="2014-11-25T20:57:29Z"
+ content="""
+I have an image 'logo.jpg' that is 100kb.  I am using indirect mode.
+I add the logo to the annex; my understanding is that the file contents is stored in .git/annex/....
+
+I do more work in Photoshop to improve logo.jpg.  Now, I have a new version of 'logo.jpg' that is 150kb.
+I add this new version of logo.jpg to the annex.
+
+Does the .git/annex area now contain two instances of the logo.jpg contents, one for the 100kb and one for the 150kb, with metadata pointing to each?
+
+
+"""]]
diff --git a/doc/forum/retrieving_previous_versions/comment_3_5b0ca6fbc000727bf78f5fc4ba34b91a._comment b/doc/forum/retrieving_previous_versions/comment_3_5b0ca6fbc000727bf78f5fc4ba34b91a._comment
new file mode 100644
index 0000000000..e8497dbbb4
--- /dev/null
+++ b/doc/forum/retrieving_previous_versions/comment_3_5b0ca6fbc000727bf78f5fc4ba34b91a._comment
@@ -0,0 +1,9 @@
+[[!comment format=mdwn
+ username="joey"
+ subject="""comment 3"""
+ date="2014-12-01T22:30:16Z"
+ content="""
+Yes, both versions of the file are stored in `.git/annex/objects`,
+unless you move them to some other repository, or unless
+you use `git annex dropunused` to delete the old version.
+"""]]
diff --git a/doc/forum/root_assistant__63__.mdwn b/doc/forum/root_assistant__63__.mdwn
new file mode 100644
index 0000000000..54b8729440
--- /dev/null
+++ b/doc/forum/root_assistant__63__.mdwn
@@ -0,0 +1,3 @@
+How safe (or not) is it to run the assistant as root?
+
+If not safe, what would be a good way to sync directories like /usr/local ?
diff --git a/doc/forum/root_assistant__63__/comment_1_fccab2e36d393f420d0fa23958e6a9d2._comment b/doc/forum/root_assistant__63__/comment_1_fccab2e36d393f420d0fa23958e6a9d2._comment
new file mode 100644
index 0000000000..2fd418d564
--- /dev/null
+++ b/doc/forum/root_assistant__63__/comment_1_fccab2e36d393f420d0fa23958e6a9d2._comment
@@ -0,0 +1,15 @@
+[[!comment format=mdwn
+ username="joey"
+ subject="""comment 1"""
+ date="2015-02-17T21:41:27Z"
+ content="""
+I would not recommend running the assistant as root. Any security issue
+would escalate the root access; any bug could result in some root level
+damage to system.
+
+Of course, I don't know of any such security issues or bugs. If I did, I'd
+be fixing them.
+
+On my system, /usr/local is managed by group staff. It seems much safer to
+make the assistant be run by some non-root user who is in the staff group.
+"""]]
diff --git a/doc/forum/rsync.net__58___Too_many_authentication_failures_for___42____42____42____42____42__.mdwn b/doc/forum/rsync.net__58___Too_many_authentication_failures_for___42____42____42____42____42__.mdwn
new file mode 100644
index 0000000000..2a22d6ee78
--- /dev/null
+++ b/doc/forum/rsync.net__58___Too_many_authentication_failures_for___42____42____42____42____42__.mdwn
@@ -0,0 +1,13 @@
+When trying to setup an rsync.net repo I always get the following error:
+
+Permission denied, please try again.
+Permission denied, please try again.
+Received disconnect from 114.xxx.xxx.xxx: 2: Too many authentication failures for 2*****
+
+I can ssh into the account without any problems and couldn't find anything, which would have helped me any further.
+Any ideas? Is the problem sitting in front of the computer? Is it a bug?
+
+Thanks.
+David
+
+This is happening on Mavericks (10.9)
diff --git a/doc/forum/rsync.net__58___Too_many_authentication_failures_for___42____42____42____42____42__/comment_1_7754e2cfb72b034effe8642c1b3e593e._comment b/doc/forum/rsync.net__58___Too_many_authentication_failures_for___42____42____42____42____42__/comment_1_7754e2cfb72b034effe8642c1b3e593e._comment
new file mode 100644
index 0000000000..d29f6482c3
--- /dev/null
+++ b/doc/forum/rsync.net__58___Too_many_authentication_failures_for___42____42____42____42____42__/comment_1_7754e2cfb72b034effe8642c1b3e593e._comment
@@ -0,0 +1,11 @@
+[[!comment format=mdwn
+ username="https://www.google.com/accounts/o8/id?id=AItOawn_3tllXjSmtgm__aGr9Z4gVNFgJgGyJ30"
+ nickname="David Alan"
+ subject="comment 1"
+ date="2014-04-02T15:58:57Z"
+ content="""
+This problem also exists, when connecting to other ssh remote servers.
+The error msg would be:
+
+ Failed to ssh to the server. Transcript: Permission denied, please try again. Received disconnect from 80.xxx.xxx.xxx: 2: Too many authentication failures for ssh-xxxxxx-git-annex-assist 
+"""]]
diff --git a/doc/forum/rsync.net__58___Too_many_authentication_failures_for___42____42____42____42____42__/comment_2_04e1da4352ef9f9be90253ea726e5f24._comment b/doc/forum/rsync.net__58___Too_many_authentication_failures_for___42____42____42____42____42__/comment_2_04e1da4352ef9f9be90253ea726e5f24._comment
new file mode 100644
index 0000000000..3181c4fe01
--- /dev/null
+++ b/doc/forum/rsync.net__58___Too_many_authentication_failures_for___42____42____42____42____42__/comment_2_04e1da4352ef9f9be90253ea726e5f24._comment
@@ -0,0 +1,12 @@
+[[!comment format=mdwn
+ username="http://joeyh.name/"
+ ip="209.250.56.244"
+ subject="comment 2"
+ date="2014-04-02T19:41:29Z"
+ content="""
+What version of git-annex?
+
+Do you have ssh-askpass installed?
+
+You might try running git webapp from a terminal, in case ssh is unable to prompt for the password in a window using ssh-askpass.
+"""]]
diff --git a/doc/forum/rsync.net__58___Too_many_authentication_failures_for___42____42____42____42____42__/comment_3_84aceb9a9d3e5bd14c044861f47e3b9d._comment b/doc/forum/rsync.net__58___Too_many_authentication_failures_for___42____42____42____42____42__/comment_3_84aceb9a9d3e5bd14c044861f47e3b9d._comment
new file mode 100644
index 0000000000..ef70eea48a
--- /dev/null
+++ b/doc/forum/rsync.net__58___Too_many_authentication_failures_for___42____42____42____42____42__/comment_3_84aceb9a9d3e5bd14c044861f47e3b9d._comment
@@ -0,0 +1,10 @@
+[[!comment format=mdwn
+ username="https://www.google.com/accounts/o8/id?id=AItOawn_3tllXjSmtgm__aGr9Z4gVNFgJgGyJ30"
+ nickname="David Alan"
+ subject="comment 3"
+ date="2014-04-03T00:31:14Z"
+ content="""
+I'm running Version: 5.20140318-gdcf93d0. Do I have to install a ssh-askpass manually? I only found one thread referring to ssh-askpass, but didn't really know what to do with that information. So, no I don't have it installed as far as I know... _I_ never did anyway...
+
+Do I need it? Did I miss something in the docs?
+"""]]
diff --git a/doc/forum/rsync.net__58___Too_many_authentication_failures_for___42____42____42____42____42__/comment_4_2cd17d01edeb230197865e6ea0884de0._comment b/doc/forum/rsync.net__58___Too_many_authentication_failures_for___42____42____42____42____42__/comment_4_2cd17d01edeb230197865e6ea0884de0._comment
new file mode 100644
index 0000000000..bfb4c2d74a
--- /dev/null
+++ b/doc/forum/rsync.net__58___Too_many_authentication_failures_for___42____42____42____42____42__/comment_4_2cd17d01edeb230197865e6ea0884de0._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="http://joeyh.name/"
+ ip="216.145.95.162"
+ subject="comment 4"
+ date="2014-05-19T16:46:55Z"
+ content="""
+I suggest you upgrade to yesterday's release. It moves ssh password prompting during repository setup into a field in the webapp, so should avoid this class of problems.
+"""]]
diff --git a/doc/forum/rsync_asking_for_an_unknown_password.mdwn b/doc/forum/rsync_asking_for_an_unknown_password.mdwn
new file mode 100644
index 0000000000..1126c31eab
--- /dev/null
+++ b/doc/forum/rsync_asking_for_an_unknown_password.mdwn
@@ -0,0 +1,60 @@
+Hello,
+
+I am on OS X, using version 5.20140613.  I installed using brew.
+
+I have a really simple example where I'm trying to use local rsync to test transfer between two repos.  It keeps asking me for a password over and over.  I do not know what password it is.
+
+I'm literally doing just this:
+
+    $ cd annex-tests
+    $ mkdir target
+    $ mkdir source
+    $ cat > source/text
+    hi this is a test
+    $ cd source
+    $ git init
+    Initialized empty Git repository in /Users/me/Documents/annex-tests/source/.git/
+    $ git annex init
+    init  ok
+    (Recording state in git...)
+    $ git annex initremote rsync-login type=rsync rsyncurl=me@localhost:/Users/me/Documents/annex-tests/target encryption=none
+    initremote rsync-login ok
+    (Recording state in git...)
+    $ git annex describe rsync-login "test rsync url with a login"
+    describe rsync-login ok
+    (Recording state in git...)
+    $ git annex add text 
+    add text ok
+    (Recording state in git...)
+    $ git annex sync rsync-login --content
+    commit  ok
+    copy text copy text (checking rsync-login...) Password:
+    Password:
+    Password:
+    Password:
+    Password:
+
+I try a blank password, I try my login password, I tried 'none' (the encryption type), I try everything..  no dice
+
+So right off the bat let me explain a few things.
+
+1 -  Why not use rsa keys?  Because I was having the exact same problem and I wanted to make it even easier and use login/password
+
+2 -  Why not use the webapp since a few forum posts say you can set passwords in there?  because my annex doesn't seem to recognize the command.
+
+    $ git annex webapp
+    git-annex: unknown command webapp
+
+3 -  I verified my rsync command/url appears ok by doing the following.  The password for that command is as expected, it is my username password:
+
+    $ cat > test2
+    hi test
+    $ rsync test2 me@localhost:/Users/me/Documents/annex-tests/target
+    Password:
+    $ cat /Users/me/Documents/annex-tests/target/test2
+    hi test
+
+
+Any help will be appreciated!
+
+
diff --git a/doc/forum/rsync_asking_for_an_unknown_password/comment_1_f23d1c04a27625089eaef5b4bb7f8456._comment b/doc/forum/rsync_asking_for_an_unknown_password/comment_1_f23d1c04a27625089eaef5b4bb7f8456._comment
new file mode 100644
index 0000000000..b761838721
--- /dev/null
+++ b/doc/forum/rsync_asking_for_an_unknown_password/comment_1_f23d1c04a27625089eaef5b4bb7f8456._comment
@@ -0,0 +1,12 @@
+[[!comment format=mdwn
+ username="http://joeyh.name/"
+ ip="209.250.56.2"
+ subject="comment 1"
+ date="2014-07-11T18:41:32Z"
+ content="""
+Re brew not being built with support for the webapp, I suggest you file a bug on brew about this. IIRC they were, or were planning to include the webapp in the build. They may have lost track of that.
+
+Now, what does rsyncurl=me@localhost:/Users/me/Documents/annex-tests/target say? It says to use rsync with the \"url\" me@localhost:/Users/me/Documents/annex-tests/target. What does rsync do with that url? It tries to ssh into localhost, as user \"me\". There's no magic here. I don't know why your login password isn't working, but you can verify by passing --debug to git-annex that it just runs rsync with the url you've given it.
+
+BTW, are you sure you want to use a rsync special remote at all, rather than a regular git remote using ssh?
+"""]]
diff --git a/doc/forum/rsync_asking_for_an_unknown_password/comment_2_ce4b399fdb2f04e30bd8a951994f1c80._comment b/doc/forum/rsync_asking_for_an_unknown_password/comment_2_ce4b399fdb2f04e30bd8a951994f1c80._comment
new file mode 100644
index 0000000000..494aaa2359
--- /dev/null
+++ b/doc/forum/rsync_asking_for_an_unknown_password/comment_2_ce4b399fdb2f04e30bd8a951994f1c80._comment
@@ -0,0 +1,10 @@
+[[!comment format=mdwn
+ username="https://www.google.com/accounts/o8/id?id=AItOawk3JJvaarUKd7xShCMze_kpHVcuCzFrWTI"
+ nickname="Lucas"
+ subject="comment 2"
+ date="2014-07-15T16:18:01Z"
+ content="""
+Well I got the password issue straightened out (was me, don't ask).
+
+Both the rsync and git via ssh are working well, thanks!
+"""]]
diff --git a/doc/forum/rsync_over_ssh__63__.mdwn b/doc/forum/rsync_over_ssh__63__.mdwn
new file mode 100644
index 0000000000..9c0c9add63
--- /dev/null
+++ b/doc/forum/rsync_over_ssh__63__.mdwn
@@ -0,0 +1,2 @@
+[Walkthrough](http://git-annex.branchable.com/walkthrough/using_ssh_remotes/) says that when using ssh remotes rsync is used for transfering files. Is rsync used via ssh or unsecure?
+-- Michael K.
diff --git a/doc/forum/rsync_over_ssh__63__/comment_1_ee21f32e90303e20339e0a568321bbbe._comment b/doc/forum/rsync_over_ssh__63__/comment_1_ee21f32e90303e20339e0a568321bbbe._comment
new file mode 100644
index 0000000000..2b9fc9552d
--- /dev/null
+++ b/doc/forum/rsync_over_ssh__63__/comment_1_ee21f32e90303e20339e0a568321bbbe._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="http://joey.kitenet.net/"
+ nickname="joey"
+ subject="comment 1"
+ date="2011-03-06T15:59:37Z"
+ content="""
+Everything is done over ssh unless both repos are on the same system (or unless you NFS mount a repo)
+"""]]
diff --git a/doc/forum/rsync_over_ssh__63__/comment_2_aa690da6ecfb2b30fc5080ad76dc77b1._comment b/doc/forum/rsync_over_ssh__63__/comment_2_aa690da6ecfb2b30fc5080ad76dc77b1._comment
new file mode 100644
index 0000000000..49003937b6
--- /dev/null
+++ b/doc/forum/rsync_over_ssh__63__/comment_2_aa690da6ecfb2b30fc5080ad76dc77b1._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="http://m-f-k.myopenid.com/"
+ ip="92.194.43.135"
+ subject="comment 2"
+ date="2011-03-06T16:33:19Z"
+ content="""
+Great! This was the only thing about git-annex which could have kept me from using it. --Michael
+"""]]
diff --git a/doc/forum/rsync_protocol_mismatch_due_to_encfs_folder.mdwn b/doc/forum/rsync_protocol_mismatch_due_to_encfs_folder.mdwn
new file mode 100644
index 0000000000..c3667a2b69
--- /dev/null
+++ b/doc/forum/rsync_protocol_mismatch_due_to_encfs_folder.mdwn
@@ -0,0 +1,24 @@
+One of my repositories worked just fine for months, until I have decided to put an encfs folder within it. Encfs is encrypting folders and files, resulting in weird file and folder names. However, rsync just seems to fail.
+
+
+This has already been a topic here a few times, for example [1] and [2].
+
+Unfortunately, I have not been lucky enough to recover the situation by the hints given.
+
+
+What I did to investigate is that I have re-run one of the shell commands stuck client-side and appended --debug to gain a deeper look. This is the output:
+
+    $ /opt/git-annex/2016-01-14/exe/ssh --library-path /opt/git-annex/2016-01-14//etc/ld.so.conf.d:/opt/git-annex/2016-01-14//usr/lib/x86_64-linux-gnu/gconv:/opt/git-annex/2016-01-14//usr/lib/x86_64-linux-gnu/audit:/opt/git-annex/2016-01-14//usr/lib:/opt/git-annex/2016-01-14//usr/lib/x86_64-linux-gnu:/opt/git-annex/2016-01-14//lib64:/opt/git-annex/2016-01-14//lib/x86_64-linux-gnu: /opt/git-annex/2016-01-14/shimmed/ssh/ssh -S .git/annex/ssh/user@10.0.0.1 -o ControlMaster=auto -o ControlPersist=yes -T user@10.0.0.1 git-annex-shell 'recvkey' '/mnt/path/to/share/' 'SHA256E-s262670--e004c6e3f87ad2b5d74d206de85de5449c9f263d83e6ef1c87ffb03be8b7e725' --debug --uuid e34522e4-1dab-459c-b295-1fe41e7692e4 '--' 'remoteuuid=07820aec-e2a3-4c60-bd57-b39b7213e056' 'unlocked=1' 'direct=1' 'associatedfile=.encfs_folder/CMG23xCwkemocE-Gq9WHpO-n/1vqfTkJChNm5CgjBsZ--,9r,ZOcSBTxE-Ogz5ytA8R23st52LWf4d9MjRref56VhbEB/4ezakoEn8dqOT8s,K4X,ueSs' '--' dummy rsync --server -pe.Lsfx --log-format=X --inplace . .
+    [2016-03-16 22:52:18.922914] transfer start
+    [2016-03-16 22:52:18.949417] call: rsync ["--server","-t","--inplace","-e.Lsf",".","../../mnt/path/to/share/.git/annex/tmp/SHA256E-s262670--e004c6e3f87ad2b5d74d206de85de5449c9f263d83e6ef1c87ffb03be8b7e725"]
+    protocol version mismatch -- is your shell clean?
+    (see the rsync man page for an explanation)
+    rsync error: protocol incompatibility (code 2) at compat.c(176) [Receiver=3.1.1]
+    [2016-03-16 22:59:05.890987] process done ExitFailure 2
+    [2016-03-16 22:59:05.899506] transfer done
+
+What can I do other than to just move the encfs outside my repository, which is of course the least favorable option for me to do?
+
+[1] 
+
+[2] 
diff --git a/doc/forum/rsync_regression_on_Windows.mdwn b/doc/forum/rsync_regression_on_Windows.mdwn
new file mode 100644
index 0000000000..d0bc0f14d1
--- /dev/null
+++ b/doc/forum/rsync_regression_on_Windows.mdwn
@@ -0,0 +1,16 @@
+I'm on Windows 7 and msysgit. The rsync that comes with the git-annex installer downloaded on January 19 2015 works fine (rsync version 3.0.9, protocol version 30; git annex version 5.20150113-gcf247cf). The rsync that comes with the current git-annex installer errors out (rsync version 3.1.1  protocol version 31; git annex v ersion 5.20150420-gb0ebb23). I don't think it's a version/protocol mismatch, as I get the error against a server with the same version and protocol.
+
+[[!format sh """
+$ rsync -rvvp anton@myhost:wtmp/ wtmp
+opening connection using: ssh -l anton myhost rsync --server --sender -vv
+pre.iLsfx . wtmp/  (10 args)
+rsync: connection unexpectedly closed (0 bytes received so far) [Receiver]
+rsync error: error in rsync protocol data stream (code 12) at io.c(226) [Receiver=3.1.1]
+
+$ ssh -l anton myhost rsync --version
+rsync  version 3.1.1  protocol version 31
+...
+
+"""]]
+
+Bug or user error?
diff --git a/doc/forum/rsync_regression_on_Windows/comment_1_0e713f8949a3e5a949489fc89c0b9078._comment b/doc/forum/rsync_regression_on_Windows/comment_1_0e713f8949a3e5a949489fc89c0b9078._comment
new file mode 100644
index 0000000000..5642560fd1
--- /dev/null
+++ b/doc/forum/rsync_regression_on_Windows/comment_1_0e713f8949a3e5a949489fc89c0b9078._comment
@@ -0,0 +1,7 @@
+[[!comment format=mdwn
+ username="joey"
+ subject="""comment 1"""
+ date="2015-05-07T18:16:07Z"
+ content="""
+[[bugs/rsync_on_windows_broken_by_upgrade]]
+"""]]
diff --git a/doc/forum/rsync_remote_is_not_available_from_a_cloned_repo/comment_1_2e340c5a6473f165dc06cc35db38e5c0._comment b/doc/forum/rsync_remote_is_not_available_from_a_cloned_repo/comment_1_2e340c5a6473f165dc06cc35db38e5c0._comment
new file mode 100644
index 0000000000..2b6ff45df9
--- /dev/null
+++ b/doc/forum/rsync_remote_is_not_available_from_a_cloned_repo/comment_1_2e340c5a6473f165dc06cc35db38e5c0._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="hannes"
+ ip="130.226.142.243"
+ subject="original repo git annex version"
+ date="2012-07-08T11:55:42Z"
+ content="""
+is 3.20120629
+"""]]
diff --git a/doc/forum/rsyncing_.git__47__annex__47__objects.mdwn b/doc/forum/rsyncing_.git__47__annex__47__objects.mdwn
new file mode 100644
index 0000000000..876b5e04a7
--- /dev/null
+++ b/doc/forum/rsyncing_.git__47__annex__47__objects.mdwn
@@ -0,0 +1,24 @@
+In short: is it safe to rsync .git/annex/objects/ to other clones?
+
+I'm in the process of migrating a lot of my files to a new backend.  Most them
+still use the old non-file-size-tracking SHA1 backend from when WORM was the
+default.  I have multiple clones that each use about 460 GB of their 500 GB
+drives.
+
+I've noticed that git-annex creates hardlinks to migrated content in
+.git/annex/objects, so I'm able to migrate all content despite only having 40
+GB of free space.  Excellent.
+
+Now I'm planning to `rsync -a --hard-links .git/annex/objects/
+${CLONE}/.git/annex/objects/` to recreate the hardlinks and save space on the
+clones as well.  If I `fsck` afterwards, this should be fine, right?
+
+I've tried this with a test repository and it works but I'd like to be extra
+sure that I'm not missing something crucial.
+
+The alternatives that I'm aware of are:
+
+ * making space first: `git annex dropunused "1-20000"` and `git annex get .`
+ * running `git annex migrate` in each clone.
+
+I expect rsync to be safer and faster than these alternatives.
diff --git a/doc/forum/rsyncing_.git__47__annex__47__objects/comment_1_25eb9f7be5730337b9efd96dce9efd2e._comment b/doc/forum/rsyncing_.git__47__annex__47__objects/comment_1_25eb9f7be5730337b9efd96dce9efd2e._comment
new file mode 100644
index 0000000000..8d9e22a7e9
--- /dev/null
+++ b/doc/forum/rsyncing_.git__47__annex__47__objects/comment_1_25eb9f7be5730337b9efd96dce9efd2e._comment
@@ -0,0 +1,10 @@
+[[!comment format=mdwn
+ username="http://joeyh.name/"
+ ip="209.250.56.246"
+ subject="comment 1"
+ date="2013-11-16T17:36:48Z"
+ content="""
+As long as you fsck afterwards, this is fine.
+
+You might want to `git annex fsck --all` in case you have objects for old versions of files.
+"""]]
diff --git a/doc/forum/rsyncing_.git__47__annex__47__objects/comment_2_d7ceae666c8a1951021d3c6e6ac39a11._comment b/doc/forum/rsyncing_.git__47__annex__47__objects/comment_2_d7ceae666c8a1951021d3c6e6ac39a11._comment
new file mode 100644
index 0000000000..f9c449f5eb
--- /dev/null
+++ b/doc/forum/rsyncing_.git__47__annex__47__objects/comment_2_d7ceae666c8a1951021d3c6e6ac39a11._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="gernot"
+ ip="89.0.237.214"
+ subject="comment 2"
+ date="2013-11-18T15:09:59Z"
+ content="""
+Thanks for the confirmation, Joey!  I also wouldn't have thought of `--all`.
+"""]]
diff --git a/doc/forum/s3_bandwidth_limitations_and_next_release.mdwn b/doc/forum/s3_bandwidth_limitations_and_next_release.mdwn
new file mode 100644
index 0000000000..c1e67a8c0b
--- /dev/null
+++ b/doc/forum/s3_bandwidth_limitations_and_next_release.mdwn
@@ -0,0 +1,7 @@
+Is there a way to set bandwidth limits for [[special_remotes/s3]]?
+
+From what i can see in the [[todo/credentials-less_access_to_s3]] patch, the `downloadUrl` function is used, does that mean that the `annex.web-download-command` is used? If that's the case, it's great because it means we can use the `--bwlimit` parameter in `wget` to limit transfers.
+
+But what about uploads? Are there limits there as well?
+
+I'll also abuse this forum to see if/when it will be possible to have a shiny new release to ship that amazing new feature? There seems to be sufficient stuff piled up in the unreleased changelog to warrant a release, no? :) --[[anarcat]]
diff --git a/doc/forum/s3_bandwidth_limitations_and_next_release/comment_1_6fba752ada4ca86c96a026a4d09ef16b._comment b/doc/forum/s3_bandwidth_limitations_and_next_release/comment_1_6fba752ada4ca86c96a026a4d09ef16b._comment
new file mode 100644
index 0000000000..60e24bcbc7
--- /dev/null
+++ b/doc/forum/s3_bandwidth_limitations_and_next_release/comment_1_6fba752ada4ca86c96a026a4d09ef16b._comment
@@ -0,0 +1,11 @@
+[[!comment format=mdwn
+ username="joey"
+ subject="""comment 1"""
+ date="2015-06-14T16:53:22Z"
+ content="""
+Yeah, I did use wget for the read-only S3 downloads, so you can configure
+with `annex.web-download-command`. No rate limiting available for
+authenticated S3 access yet.
+
+Release tuesday or wednesday.
+"""]]
diff --git a/doc/forum/s3_server_side_encryption.mdwn b/doc/forum/s3_server_side_encryption.mdwn
new file mode 100644
index 0000000000..f7b64921ba
--- /dev/null
+++ b/doc/forum/s3_server_side_encryption.mdwn
@@ -0,0 +1,9 @@
+AWS S3 offers a feature to enable server-side encryption of files.  
+If I understand correctly, this is enabled by sending a specific HTTP header with the request to upload the file in question.   
+So, this header needs to be set every time we want to upload a new file.  
+
+Is this feature already supported / being considered for future versions?
+
+If not, am I correct in assuming it would have to be implemented in https://github.com/joeyh/git-annex/blob/master/Remote/S3.hs ?
+
+Thank you
diff --git a/doc/forum/s3_server_side_encryption/comment_1_68345d01b016abf96c226d2bfa17c641._comment b/doc/forum/s3_server_side_encryption/comment_1_68345d01b016abf96c226d2bfa17c641._comment
new file mode 100644
index 0000000000..1dbf60047d
--- /dev/null
+++ b/doc/forum/s3_server_side_encryption/comment_1_68345d01b016abf96c226d2bfa17c641._comment
@@ -0,0 +1,16 @@
+[[!comment format=mdwn
+ username="joey"
+ subject="""comment 1"""
+ date="2015-01-05T20:33:13Z"
+ content="""
+I have not looked into this particular S3 feature, since I see little point
+in using it. git-annex can encrypt files client-side before sending to S3,
+which is much better.
+
+However, you can probably configure git-annex to send the header.
+See the `x-amx-meta-*` option documented in [[special_remotes/S3]].
+If the header was named encryptplz and needed to be set to
+"canhazsecurityburger", you would enable it with something like:
+
+	git annex enableremote mys3remote x-amz-meta-encryptplz=canhazsecurityburger
+"""]]
diff --git a/doc/forum/s3_server_side_encryption/comment_2_b2ccef6dc00d58e103ac0fda48ee94d3._comment b/doc/forum/s3_server_side_encryption/comment_2_b2ccef6dc00d58e103ac0fda48ee94d3._comment
new file mode 100644
index 0000000000..f3d8c94bcd
--- /dev/null
+++ b/doc/forum/s3_server_side_encryption/comment_2_b2ccef6dc00d58e103ac0fda48ee94d3._comment
@@ -0,0 +1,10 @@
+[[!comment format=mdwn
+ username="https://www.google.com/accounts/o8/id?id=AItOawne3YCXovu_ERUYNpH46AVo0FlgyQwt3HI"
+ nickname="Pierre"
+ subject="comment 2"
+ date="2015-01-05T20:47:31Z"
+ content="""
+Thanks for your reponse. I'll try specifying the header as you are suggesting.
+
+FYI, one reason for not handling encryption locally is to not have to manage encryption keys. If multiple people need to be able to decrypt the data, I would have to manage all their keys.
+"""]]
diff --git a/doc/forum/s3_server_side_encryption/comment_3_3eb57b98e4136b8550ea5d19393fe967._comment b/doc/forum/s3_server_side_encryption/comment_3_3eb57b98e4136b8550ea5d19393fe967._comment
new file mode 100644
index 0000000000..cee29d8770
--- /dev/null
+++ b/doc/forum/s3_server_side_encryption/comment_3_3eb57b98e4136b8550ea5d19393fe967._comment
@@ -0,0 +1,12 @@
+[[!comment format=mdwn
+ username="joey"
+ subject="""comment 3"""
+ date="2015-01-05T21:09:47Z"
+ content="""
+Using git-annex's encryption=shared will have the same
+key management ease as letting Amazon encrypt. 
+
+In both cases you're putting the shared encryption key into the
+configuration of a git-annex S3 remote. Anyone who can access the git
+repository can access the encryption key.
+"""]]
diff --git a/doc/forum/s3_server_side_encryption/comment_4_71d6c2356af8974cb848c3574cf3eb6d._comment b/doc/forum/s3_server_side_encryption/comment_4_71d6c2356af8974cb848c3574cf3eb6d._comment
new file mode 100644
index 0000000000..7cfa0195b7
--- /dev/null
+++ b/doc/forum/s3_server_side_encryption/comment_4_71d6c2356af8974cb848c3574cf3eb6d._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="https://www.google.com/accounts/o8/id?id=AItOawne3YCXovu_ERUYNpH46AVo0FlgyQwt3HI"
+ nickname="Pierre"
+ subject="comment 4"
+ date="2015-01-05T21:19:37Z"
+ content="""
+Ok, thank you!
+"""]]
diff --git a/doc/forum/s3_vs_ssh_Performance_Problems.mdwn b/doc/forum/s3_vs_ssh_Performance_Problems.mdwn
new file mode 100644
index 0000000000..5176e196fc
--- /dev/null
+++ b/doc/forum/s3_vs_ssh_Performance_Problems.mdwn
@@ -0,0 +1,8 @@
+I backup/sync my music using annex. I used to have one repo with 3 clones, one full copy on my vps, one full copy and one partial copy on my laptops. I decided to move all data to s3. Declared my vps repo dead, purged history (I do not care about history for this particular repo) pushed the git repo to a different computer (bare repo no data) and data to s3 (gpg encrypted). I've just finished uploading all files (around 200gb) couple files failed during the initial upload so I did a `git annex copy --quiet --to mys3 --fast` this command used to take 15 20 secs on my laptop when sending data to vps using ssh but now it took 2 hours to complete (pushing mem usage to 90%).
+
+I have one other repo (1.5gb in size 42k files using indirect mode, data gpg encrypted on s3) using the same setup except this repos content has always been on s3  i had the same behavior on this repo too. adding a file and running a copy to push content to s3 took couple hours even if I add a single 1kb file. I used to blame my hard drive, damn thing is slow but now I think this is related to s3. is there any workaround for this?
+
+Both machines are using,
+
+    git-annex version: 4.20130913-gd20a4f2
+    build flags: Assistant Webapp Pairing Testsuite S3 WebDAV Inotify DBus XMPP Feeds Quvi
diff --git a/doc/forum/s3_vs_ssh_Performance_Problems/comment_1_65f064f09d7850abecab97007b0d30f0._comment b/doc/forum/s3_vs_ssh_Performance_Problems/comment_1_65f064f09d7850abecab97007b0d30f0._comment
new file mode 100644
index 0000000000..aa3e06cc0c
--- /dev/null
+++ b/doc/forum/s3_vs_ssh_Performance_Problems/comment_1_65f064f09d7850abecab97007b0d30f0._comment
@@ -0,0 +1,12 @@
+[[!comment format=mdwn
+ username="https://www.google.com/accounts/o8/id?id=AItOawmBUR4O9mofxVbpb8JV9mEbVfIYv670uJo"
+ nickname="Justin"
+ subject="comment 1"
+ date="2013-11-21T04:09:15Z"
+ content="""
+Is
+
+    git annex copy --not --in mys3 --to mys3 .
+
+any faster?
+"""]]
diff --git a/doc/forum/s3_vs_ssh_Performance_Problems/comment_2_baaf2384d9196077268e9ca9bbe3b871._comment b/doc/forum/s3_vs_ssh_Performance_Problems/comment_2_baaf2384d9196077268e9ca9bbe3b871._comment
new file mode 100644
index 0000000000..f54e49b2b5
--- /dev/null
+++ b/doc/forum/s3_vs_ssh_Performance_Problems/comment_2_baaf2384d9196077268e9ca9bbe3b871._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="https://me.yahoo.com/a/FHnTlSBo1eCGJRwueeKeB6.RCaPbGMPr5jxx8A--#ce0d8"
+ nickname="Hamza"
+ subject="comment 2"
+ date="2013-11-21T17:47:57Z"
+ content="""
+No difference. Both take roughly the same time but I did time a couple of runs with both commands they usually take 3 to 5 minutes to complete. Maybe git did something behind the scenes like gc? But still slower than it used to be. My other repo (one with 42k files) still takes a 2 hours.
+"""]]
diff --git a/doc/forum/s3_vs_ssh_Performance_Problems/comment_3_dc44be42070c073d150c476406e9b425._comment b/doc/forum/s3_vs_ssh_Performance_Problems/comment_3_dc44be42070c073d150c476406e9b425._comment
new file mode 100644
index 0000000000..cbd460818f
--- /dev/null
+++ b/doc/forum/s3_vs_ssh_Performance_Problems/comment_3_dc44be42070c073d150c476406e9b425._comment
@@ -0,0 +1,10 @@
+[[!comment format=mdwn
+ username="https://www.google.com/accounts/o8/id?id=AItOawmBUR4O9mofxVbpb8JV9mEbVfIYv670uJo"
+ nickname="Justin"
+ subject="comment 3"
+ date="2013-11-22T20:21:11Z"
+ content="""
+Well it's not related to s3... that copy command won't even do any network traffic if there is nothing to copy.  I have a similarly configured annex with 4500 files and that command takes 10 seconds to run.
+
+I do remember there being a recent fix that reduced the algorithmic complexity of an operation, but I forget which.
+"""]]
diff --git a/doc/forum/s3_vs_ssh_Performance_Problems/comment_4_f9c3ef3b1b44bfb29125acb6ec621f38._comment b/doc/forum/s3_vs_ssh_Performance_Problems/comment_4_f9c3ef3b1b44bfb29125acb6ec621f38._comment
new file mode 100644
index 0000000000..2e4cda6342
--- /dev/null
+++ b/doc/forum/s3_vs_ssh_Performance_Problems/comment_4_f9c3ef3b1b44bfb29125acb6ec621f38._comment
@@ -0,0 +1,10 @@
+[[!comment format=mdwn
+ username="http://joeyh.name/"
+ ip="209.250.56.87"
+ subject="comment 4"
+ date="2013-12-12T20:14:01Z"
+ content="""
+You mentioned something about high memory usage when copying. How much memory are we talking about?
+
+Have you run `git annex forget` in this repository before? It kind of sounds like you have, and it might be possible that it's repeatedly trying to forget old history for some reason.
+"""]]
diff --git a/doc/forum/safely_dropping_git-annex_history.mdwn b/doc/forum/safely_dropping_git-annex_history.mdwn
new file mode 100644
index 0000000000..a6d8e66779
--- /dev/null
+++ b/doc/forum/safely_dropping_git-annex_history.mdwn
@@ -0,0 +1,20 @@
+the git-annex branch of a repository i've had running since 2010 has grown to unmanagable dimensions (5gb in a fresh clone of the git-annex branch, while the master branch has merely 40mb, part of which is due to checked-in files), resulting in git-annex-merges to take in the order of magnitude of 15 minutes. getting an initial clone of the git-annex branch (not the data) takes hours alone in the "remote: Counting objects" phase (admittedly, the origin server is limited in ram, so it spends its time swapping the git process back and forth).
+
+is there a recommended way for how to reset the git-annex branch in a coordinated way? of course, this would have to happen on all copies of the repo at the same time.
+
+the workflow i currently imagine is
+
+* rename all copies of the repository (the_repo → the_repo-old, the_repo.git → the_repo-old.git)
+* clone the old origin repository to a new origin with --single-branch. (this would be *the* oportunity to ``git filter-branch --prune-empty --index-filter 'git rm --cached --ignore-unmatch .git-annex -r' master`` as well, to get rid of commits of pre-whatever versions)
+* ``git annex init`` on the master repository
+* clone it to all the other copies and ``git annex init`` there
+* set all the configuration options (untrusted repos etc) again
+* either
+  * ``git annex reinject`` the files that are already present on the respective machines, or
+  * move the .git/annex/objects files over from the original locations, and use ``git annex fsck`` to make git-annex discover which files it already has, if that works. (i have numcopies=2, thus i'd dare to move instead of copy even when trying this out the first time. complete copies, even of partially checked out clones, will exceed the capacities of most clients)
+
+my questions in that endeavor are:
+
+* is there already a standard workflow for this?
+* if not, will the above do the trick?
+* can anything be done to avoid such problems in future?
diff --git a/doc/forum/safely_dropping_git-annex_history/comment_10_a4b93a3fbc98d9b86e942f95e0039862._comment b/doc/forum/safely_dropping_git-annex_history/comment_10_a4b93a3fbc98d9b86e942f95e0039862._comment
new file mode 100644
index 0000000000..b24a058f6c
--- /dev/null
+++ b/doc/forum/safely_dropping_git-annex_history/comment_10_a4b93a3fbc98d9b86e942f95e0039862._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="http://joeyh.name/"
+ ip="4.154.0.140"
+ subject="comment 10"
+ date="2013-07-20T20:03:20Z"
+ content="""
+Since you seem to have found a way that works, I'm only answering for completeness: To resolve a conflicted merge on the git-annex branch, you can just add all lines present in either side of the merge, in any order. This won't necessarily be the most minimal resolution, but it is guaranteed to always be a valid one. There is a git-union-merge program in the git-annex source (not built by default) that can do that when merging any set of branches.
+"""]]
diff --git a/doc/forum/safely_dropping_git-annex_history/comment_11_383882fafd32f25ed22b5eb2fb3691b9._comment b/doc/forum/safely_dropping_git-annex_history/comment_11_383882fafd32f25ed22b5eb2fb3691b9._comment
new file mode 100644
index 0000000000..fbfe641564
--- /dev/null
+++ b/doc/forum/safely_dropping_git-annex_history/comment_11_383882fafd32f25ed22b5eb2fb3691b9._comment
@@ -0,0 +1,18 @@
+[[!comment format=mdwn
+ username="https://www.google.com/accounts/o8/id?id=AItOawln3ckqKx0x_xDZMYwa9Q1bn4I06oWjkog"
+ nickname="Michael"
+ subject="propagating squashed history to other remotes"
+ date="2013-08-12T01:44:59Z"
+ content="""
+The easiest method seems to be to force-push git-annex and master to other remotes, e.g.
+
+    git push -f myremote git-annex:refs/heads/git-annex
+
+Before doing this, make sure location logs etc had a chance to propagate across all remotes.
+
+It's a good idea to remove synched/ branches before doing git-annex sync on the repos with rewritten history, too:
+    git branch -D synced/master
+    git branch -D synced/annex
+
+
+"""]]
diff --git a/doc/forum/safely_dropping_git-annex_history/comment_12_47794a2abf29bf4ea2763ff89d872ab4._comment b/doc/forum/safely_dropping_git-annex_history/comment_12_47794a2abf29bf4ea2763ff89d872ab4._comment
new file mode 100644
index 0000000000..5c09316041
--- /dev/null
+++ b/doc/forum/safely_dropping_git-annex_history/comment_12_47794a2abf29bf4ea2763ff89d872ab4._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="http://joeyh.name/"
+ ip="2001:4830:1600:187::2"
+ subject="comment 12"
+ date="2013-09-04T06:38:00Z"
+ content="""
+`git annex forget` automates this now, without needing to force-push or have a flag day. Needs a version of git-annex supporting it installed on *all* the computers you use the repo on. Repos notice they need to forget when git annex is run in them, and do, automatically.
+"""]]
diff --git a/doc/forum/safely_dropping_git-annex_history/comment_13_b7b920862fc82eaa5d7deae34b4aebc4._comment b/doc/forum/safely_dropping_git-annex_history/comment_13_b7b920862fc82eaa5d7deae34b4aebc4._comment
new file mode 100644
index 0000000000..7a8f62260f
--- /dev/null
+++ b/doc/forum/safely_dropping_git-annex_history/comment_13_b7b920862fc82eaa5d7deae34b4aebc4._comment
@@ -0,0 +1,12 @@
+[[!comment format=mdwn
+ username="Rizwan"
+ avatar="http://cdn.libravatar.org/avatar/15c58acba4fc6f75abd783d5836e0254"
+ subject="Are these methods still working?"
+ date="2018-06-12T14:58:03Z"
+ content="""
+I am new to git annex. I like the concept of partial checkout, given that I have multiple storages. 
+
+My repo is small right now, 2000+ files collectively amounting to 24 MB in size. However, as soon as I add and unlock the repo, the size of the Annex folder becomes 50+ MB. Is this normal? I try to sync with google drive and the size goes to 90+ Mb post syncing. 
+
+I feel like I must be doing something wrong. Does the size of gorw upto 4 times the original data. I tried the methods listed in above comments, none worked. Seeing that last comment is 4+ years old, I wanted to know if these methods are still viable?
+"""]]
diff --git a/doc/forum/safely_dropping_git-annex_history/comment_14_979bbccf31dbee1707d403957bde3493._comment b/doc/forum/safely_dropping_git-annex_history/comment_14_979bbccf31dbee1707d403957bde3493._comment
new file mode 100644
index 0000000000..8c0d512d23
--- /dev/null
+++ b/doc/forum/safely_dropping_git-annex_history/comment_14_979bbccf31dbee1707d403957bde3493._comment
@@ -0,0 +1,16 @@
+[[!comment format=mdwn
+ username="joey"
+ subject="""comment 14"""
+ date="2018-06-12T16:11:27Z"
+ content="""
+@Rizwan yes, the things documented here still work. But, they drop old
+history about the past locations of files, they do not drop *all* information
+about the current locations of files. git-annex is keeping track of which
+files you have stored in the local repository and on the remote, and that
+information necessisarily takes up space.
+
+You're using git-annex with small files -- around 12
+kilobytes average file size. It's designed for mostly much larger files,
+so you will see a larger percentage of overhead in using it than you would
+if your files were bigger.
+"""]]
diff --git a/doc/forum/safely_dropping_git-annex_history/comment_1_4fd76d10a93fe01588fce7a621f9254d._comment b/doc/forum/safely_dropping_git-annex_history/comment_1_4fd76d10a93fe01588fce7a621f9254d._comment
new file mode 100644
index 0000000000..e46f7291a2
--- /dev/null
+++ b/doc/forum/safely_dropping_git-annex_history/comment_1_4fd76d10a93fe01588fce7a621f9254d._comment
@@ -0,0 +1,12 @@
+[[!comment format=mdwn
+ username="http://joeyh.name/"
+ ip="4.152.108.194"
+ subject="comment 1"
+ date="2012-10-31T16:03:55Z"
+ content="""
+Yes, you can use fsck like that. I outlined a similar approach [here](http://git-annex.branchable.com/forum/Delete_unused_files__47__metadata/#comment-e2bd822214df96606a60e8aee2c8637a), and I think you don't even need to make new git repositories, just delete the old branch and git gc it -- but I've not heard of anyone doing this yet.
+
+So, since 2010, your repo must have gone through at least one and probably two repository format changes, which bloated the git branch. Hopefully we'll have no more of those. My largest repo that also went through that is under 150 mb however. 
+
+There was a recent bug fix where `git annex copy` unnecessarily updated location log even when the file was already copied. That kind of thing can bloat the repository, especially if you had that in a cron job... You might find `git annex log` useful to look through history of files and see if there have been a lot of location changes logged for whatever reason.
+"""]]
diff --git a/doc/forum/safely_dropping_git-annex_history/comment_2_10ecf3220ffcbbe94ba09da225458f18._comment b/doc/forum/safely_dropping_git-annex_history/comment_2_10ecf3220ffcbbe94ba09da225458f18._comment
new file mode 100644
index 0000000000..21befd41a0
--- /dev/null
+++ b/doc/forum/safely_dropping_git-annex_history/comment_2_10ecf3220ffcbbe94ba09da225458f18._comment
@@ -0,0 +1,12 @@
+[[!comment format=mdwn
+ username="http://christian.amsuess.com/chrysn"
+ nickname="chrysn"
+ subject="worked well"
+ date="2012-11-04T12:23:56Z"
+ content="""
+the procedure i outlined originally worked well for me; the method chosen for reinjection was moving over the .git/annex/objects directory and doing a ``git annex fsck``.
+
+special care had to be taken of the special remote (rsync+gpg) -- i guess that's why they are called special ;-) . as described in the forum post you linked, i had to copy over remote.log and the uuid.log line from the old git-annex branch -- otherwise, a ``git annex initremote`` would have generated a new hmac, effectively resetting the remote repo.
+
+the formerly 5gb git-annex branch (admittedly not ``git gc``'d recently, but that just wasn't feasible any more) shrunk down to around 25mb of current location information. i'll keep an eye on how it's growing to see if the problem is inherent or if it was just old bugs causing trouble.
+"""]]
diff --git a/doc/forum/safely_dropping_git-annex_history/comment_3_e3beb8acb075faaeef6c052aecbf0a41._comment b/doc/forum/safely_dropping_git-annex_history/comment_3_e3beb8acb075faaeef6c052aecbf0a41._comment
new file mode 100644
index 0000000000..fbcbdbc4fa
--- /dev/null
+++ b/doc/forum/safely_dropping_git-annex_history/comment_3_e3beb8acb075faaeef6c052aecbf0a41._comment
@@ -0,0 +1,50 @@
+[[!comment format=mdwn
+ username="http://vjt.myopenid.com/"
+ nickname="vjt"
+ subject="drop "content removed from annex" history"
+ date="2013-06-18T02:12:01Z"
+ content="""
+Joey,
+
+dropping the git-annex branch and subsequent fsck worked. Moreover, as I turned my repository in containing over 700k objects due to a silly cycle of `git annex add` / `git annex unannex`, bloating *both* `git-annex` and `master` history, to clean up I successfully performed a squashed rebase of master onto itself.
+
+Here's what I did, in detail:
+
+    $ git checkout git-annex
+    $ cp *.log ..
+    $ git checkout master
+    $ git br -D git-annex
+    $ git br -D synced/git-annex
+    $ git checkout 
+    $ git checkout -b git-annex
+    $ cp ../*.log .
+    $ 
+    $ git add *.log
+    $ git commit --amend -m 'Init'
+
+With this, I got rid of the many `update` commits. Now, the fun part:
+
+    $ git checkout master
+    $ git rebase -i 
+    
+    $ :wq
+
+Rebase went fine, and I was left with a clean master. I brought also `synced/master` up to date:
+
+    $ git checkout synced/master
+    $ git reset --hard master
+
+Now I re-created all the location links with fsck:
+
+    $ git annex fsck
+
+And eventually, got rid of the redundant history:
+
+    $ git reflog expire --expire=now --expire-unreachable=now --all
+    $ git gc --prune=now
+    $ git repack
+    $ git prune
+
+yay, 500k objects less ^_^'.
+
+"""]]
diff --git a/doc/forum/safely_dropping_git-annex_history/comment_4_61a5fe2e7e47c60a8b237ea69404a37f._comment b/doc/forum/safely_dropping_git-annex_history/comment_4_61a5fe2e7e47c60a8b237ea69404a37f._comment
new file mode 100644
index 0000000000..d257e6f622
--- /dev/null
+++ b/doc/forum/safely_dropping_git-annex_history/comment_4_61a5fe2e7e47c60a8b237ea69404a37f._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="https://www.google.com/accounts/o8/id?id=AItOawln3ckqKx0x_xDZMYwa9Q1bn4I06oWjkog"
+ nickname="Michael"
+ subject="would git annex fsck also recreate location tracking info for special remotes?"
+ date="2013-07-17T17:02:50Z"
+ content="""
+Suppose I want to drop history like discussed in this entry. How well does this deal with special (e.g. directory on an offline disk) remotes?
+"""]]
diff --git a/doc/forum/safely_dropping_git-annex_history/comment_5_426d02e2f2a2ae4ec7eae02dfe4519b3._comment b/doc/forum/safely_dropping_git-annex_history/comment_5_426d02e2f2a2ae4ec7eae02dfe4519b3._comment
new file mode 100644
index 0000000000..aafa8fa1bc
--- /dev/null
+++ b/doc/forum/safely_dropping_git-annex_history/comment_5_426d02e2f2a2ae4ec7eae02dfe4519b3._comment
@@ -0,0 +1,9 @@
+[[!comment format=mdwn
+ username="https://www.google.com/accounts/o8/id?id=AItOawln3ckqKx0x_xDZMYwa9Q1bn4I06oWjkog"
+ nickname="Michael"
+ subject="comment 5"
+ date="2013-07-17T17:06:39Z"
+ content="""
+Also, would simply squashing git-annex branch history (without fsck etc) work?
+This seems easier.
+"""]]
diff --git a/doc/forum/safely_dropping_git-annex_history/comment_6_410a7296c2cee16d3d5bb618a5a41c1d._comment b/doc/forum/safely_dropping_git-annex_history/comment_6_410a7296c2cee16d3d5bb618a5a41c1d._comment
new file mode 100644
index 0000000000..05ac273616
--- /dev/null
+++ b/doc/forum/safely_dropping_git-annex_history/comment_6_410a7296c2cee16d3d5bb618a5a41c1d._comment
@@ -0,0 +1,10 @@
+[[!comment format=mdwn
+ username="http://joeyh.name/"
+ ip="4.154.4.90"
+ subject="comment 6"
+ date="2013-07-17T18:57:12Z"
+ content="""
+You can use `git annex fsck --from remote` to verify that every file location tracking thinks is on the remote still is. It's innefficient though -- it has to download the whole file to check the special remote still has the right content! That transfer can be avoided by adding --fast.
+
+This is documented in the man page. :)
+"""]]
diff --git a/doc/forum/safely_dropping_git-annex_history/comment_7_42cf492fc98a9eba8176387749ef12e0._comment b/doc/forum/safely_dropping_git-annex_history/comment_7_42cf492fc98a9eba8176387749ef12e0._comment
new file mode 100644
index 0000000000..5c10b988cd
--- /dev/null
+++ b/doc/forum/safely_dropping_git-annex_history/comment_7_42cf492fc98a9eba8176387749ef12e0._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="http://joeyh.name/"
+ ip="4.154.4.90"
+ subject="comment 7"
+ date="2013-07-17T18:59:03Z"
+ content="""
+I don't see any reason why squashing git-annex branch history would not work. If you squash it to the same sha in each clone, things would be very happy, but even if you squash it to different shas, the union merge should result in those  different versions of the same data automatically merging together.
+"""]]
diff --git a/doc/forum/safely_dropping_git-annex_history/comment_8_c0327ada073d8b69535f71b4dc6aa57e._comment b/doc/forum/safely_dropping_git-annex_history/comment_8_c0327ada073d8b69535f71b4dc6aa57e._comment
new file mode 100644
index 0000000000..120e4daefa
--- /dev/null
+++ b/doc/forum/safely_dropping_git-annex_history/comment_8_c0327ada073d8b69535f71b4dc6aa57e._comment
@@ -0,0 +1,21 @@
+[[!comment format=mdwn
+ username="https://www.google.com/accounts/o8/id?id=AItOawln3ckqKx0x_xDZMYwa9Q1bn4I06oWjkog"
+ nickname="Michael"
+ subject="git annex merge driver?"
+ date="2013-07-19T17:04:53Z"
+ content="""
+
+I've tried rebasing git-annex branch, and I hit a bunch of conflicts (both in uuid.log and for individual content file logs) of the form:
+
+
    
+<<<<<<< HEAD
+1369615760.859476s 1 016d9095-0cbc-4734-a498-4e0421e257d7
+=======
+1369615760.845334s 1 016d9095-0cbc-4734-a498-4e0421e257d7
+>>>>>>> 52e60e8... update
+1369615359.195672s 1 38c359dc-a7d9-498d-a818-2e9beae995b8
+
    
+
+As I understand, git-annex has a special timestamp-based merge driver to deal with these. Is there a way to use that with git rebase?
+
+"""]]
diff --git a/doc/forum/safely_dropping_git-annex_history/comment_9_f83d6090aea2b7d5d54c876df940cbad._comment b/doc/forum/safely_dropping_git-annex_history/comment_9_f83d6090aea2b7d5d54c876df940cbad._comment
new file mode 100644
index 0000000000..cbfd678508
--- /dev/null
+++ b/doc/forum/safely_dropping_git-annex_history/comment_9_f83d6090aea2b7d5d54c876df940cbad._comment
@@ -0,0 +1,40 @@
+[[!comment format=mdwn
+ username="https://www.google.com/accounts/o8/id?id=AItOawln3ckqKx0x_xDZMYwa9Q1bn4I06oWjkog"
+ nickname="Michael"
+ subject="git checkout --orphan"
+ date="2013-07-19T17:49:37Z"
+ content="""
+Instead of rebase, --orphan seems to be the right answer for pruning history: create a new git-annex orphan branch and git add and commit the files. So:
+
    
+
+git status 
+
+# verify there are no uncommitted or untracked files
+
+# master branch
+git branch -m old-master
+git checkout --orphan master
+git add .
+git commit -m 'first commit'
+
+# git annex branch
+git branch -m git-annex old-git-annex
+git checkout git-annex
+git checkout --orphan git-annex
+git add .
+git commit -m 'first commit'
+git checkout master
+
+# at this point, you may want to double-check that everything is still OK
+
+# finally, remove branches and clean up the objects:
+git branch -D old-master old-git-annex
+git reflog expire --expire=now --all
+git prune
+git gc
+
+
    
+
+The repo remains functional and .git is smaller.
+
+"""]]
diff --git a/doc/forum/scalability_with_lots_of_files.mdwn b/doc/forum/scalability_with_lots_of_files.mdwn
new file mode 100644
index 0000000000..3bbd877cf4
--- /dev/null
+++ b/doc/forum/scalability_with_lots_of_files.mdwn
@@ -0,0 +1,43 @@
+What is git-annex's [[scalability]] with large (10k+) number of files and a few (~10) repositories?
+
+I have had difficult times maintaining a music archive of around 20k files, spread around 17 repositories.
+
+`ncdu` tells me, of the actual files in the direct repository:
+
+
    +$ ncdu --exclude .git
+ Total disk usage: 109,3GiB  Apparent size: 109,3GiB  Items: 23771
+
    
+
+Now looking at the git-annex metadata:
+
+
    +$ time git clone -b git-annex /srv/mp3
+Cloning into 'mp3'...
+done.
+Checking out files: 100% (31207/31207), done.
+0.69user 1.72system 0:04.65elapsed 51%CPU (0avgtext+0avgdata 47732maxresident)k
+40inputs+489552outputs (1major+2906minor)pagefaults 0swaps
+$ git branch
+  annex/direct/master
+* git-annex
+  master
+$ wc -l uuid.log
+7 uuid.log
+$ find -type f | wc
+  31429   62214 3013920
+$ du -sh .
+361M    .
+$ du -sch *  | tail -1
+243M    total
+
    
+
+So basically, it looks like the git-annex location tracking takes up around 243M, 361M if we include git's history of it (I assume). This means around 8KiB of storage per file, and 4KiB/file for history (git is doing a pretty good job here). (8KiB kind of makes sense here: one file for the tracking log (4KiB) and another directory to hold it (another 4KiB)...)
+
+Is that about right? Are there ways to compress that somehow? Could I at least drop the *history* of that from git without too much harm - that would already save 120MiB...
+
+That repository is around 18 months old.
+
+(It's interesting to notice the limitation of the "one file per record" storage format here: since git-annex has so many little files, and all of those take at least $blocksize (it seems like it's 4KB here), it takes up space pretty quickly. Another good point for git here: packing files together saves a *lot* of space! Could files be packed *before* being stored in the git-annex branch? or is that totally stupid. :)
+
+Thanks! --[[anarcat]]
diff --git a/doc/forum/scalability_with_lots_of_files/comment_1_b9bb0a71d81b8cbc2a751bb7bab84812._comment b/doc/forum/scalability_with_lots_of_files/comment_1_b9bb0a71d81b8cbc2a751bb7bab84812._comment
new file mode 100644
index 0000000000..0a58c1c71d
--- /dev/null
+++ b/doc/forum/scalability_with_lots_of_files/comment_1_b9bb0a71d81b8cbc2a751bb7bab84812._comment
@@ -0,0 +1,18 @@
+[[!comment format=mdwn
+ username="joey"
+ subject="""comment 1"""
+ date="2015-02-04T19:21:07Z"
+ content="""
+Git automatically packs loose object files after a while, which includes
+git-annex branch files. See `git-gc --help`.
+
+It also delta compresses files, and git-annex's log files will delta
+compress and compress pretty well, so your du's of a checked out git-annex
+branch don't mean much.
+
+As far as I can tell, your git repository is 118mb in size, including
+the git-annex branch and all history.
+
+You can use `git annex forget` to drop historical location tracking
+info for files.
+"""]]
diff --git a/doc/forum/schedule_repository___91__expression__93__.mdwn b/doc/forum/schedule_repository___91__expression__93__.mdwn
new file mode 100644
index 0000000000..0904dbb03a
--- /dev/null
+++ b/doc/forum/schedule_repository___91__expression__93__.mdwn
@@ -0,0 +1 @@
+what sort of options can we use in the expression field? from the [git annex bible](http://git-annex.branchable.com/git-annex/) it suggests for incremental fscks, but I'm wondering if it can run shell scripts or `git annex importurl` lines too.
diff --git a/doc/forum/schedule_repository___91__expression__93__/comment_1_b123b657a92897017973927e3e47673b._comment b/doc/forum/schedule_repository___91__expression__93__/comment_1_b123b657a92897017973927e3e47673b._comment
new file mode 100644
index 0000000000..a0c4e82221
--- /dev/null
+++ b/doc/forum/schedule_repository___91__expression__93__/comment_1_b123b657a92897017973927e3e47673b._comment
@@ -0,0 +1,12 @@
+[[!comment format=mdwn
+ username="http://joeyh.name/"
+ ip="209.250.56.22"
+ subject="comment 1"
+ date="2014-09-16T18:13:18Z"
+ content="""
+Only fscking can be scheduled. From the man page:
+
+       These  actions  are available: \"fsck self\", \"fsck UUID\" 
+
+It won't be expanded to allow arbitrary commands, because that would let anyone who you share an annex with schedule arbitrary commands to run on your computer..
+"""]]
diff --git a/doc/forum/script_to_generate___34__.hidden__34___files_for_files_not_in_local_remote.mdwn b/doc/forum/script_to_generate___34__.hidden__34___files_for_files_not_in_local_remote.mdwn
new file mode 100644
index 0000000000..efb3d1caf7
--- /dev/null
+++ b/doc/forum/script_to_generate___34__.hidden__34___files_for_files_not_in_local_remote.mdwn
@@ -0,0 +1,8 @@
+Hello,
+
+I would like to know if someone made a script to generate ".hidden" files listing all files in a git annex repository which are not available in the local remote (and also listing all directories where all files within are not available in the local remote).
+
+I'm not a prgrammer myself, but I will try to make this with a script greping in "git annex whereis" all files which are not available "[here]". But if somebody has a better idea (git annex view?) or wants to try :-)
+
+Best,  
+Hugo
diff --git a/doc/forum/script_to_generate___34__.hidden__34___files_for_files_not_in_local_remote/comment_1_76453756be78f06ef3a3d7d1bf011b9f._comment b/doc/forum/script_to_generate___34__.hidden__34___files_for_files_not_in_local_remote/comment_1_76453756be78f06ef3a3d7d1bf011b9f._comment
new file mode 100644
index 0000000000..a565d56a14
--- /dev/null
+++ b/doc/forum/script_to_generate___34__.hidden__34___files_for_files_not_in_local_remote/comment_1_76453756be78f06ef3a3d7d1bf011b9f._comment
@@ -0,0 +1,11 @@
+[[!comment format=mdwn
+ username="joey"
+ subject="""comment 1"""
+ date="2017-08-28T17:15:30Z"
+ content="""
+I don't know what those .hidden files are, but you will probably
+find it easier to use `git annex find` with some options to make
+it list the files you want. Eg:
+
+	git annex find --not --in here > .hidden
+"""]]
diff --git a/doc/forum/security_risk_presented_by_remote.log__63__.mdwn b/doc/forum/security_risk_presented_by_remote.log__63__.mdwn
new file mode 100644
index 0000000000..c320c7e30c
--- /dev/null
+++ b/doc/forum/security_risk_presented_by_remote.log__63__.mdwn
@@ -0,0 +1,5 @@
+I am curious if remote.log in the repository (or anything in the repository for that matter) gives someone access to the data stored in the special remotes.
+
+If I'm using an encrypted s3 special remote, and someone gets access to the entire repository, will they be able to decrypt things? Or is the gpg stuff stored completely separately and therefore completely unaccessable, even if someone got access to the entire contents of the git repository?
+
+What is the cipher and the cipherkeys stored in git show git-annex:remote.log?
diff --git a/doc/forum/security_risk_presented_by_remote.log__63__/comment_1_9715488033f2a8939a32ed12259377ba._comment b/doc/forum/security_risk_presented_by_remote.log__63__/comment_1_9715488033f2a8939a32ed12259377ba._comment
new file mode 100644
index 0000000000..2b2805adde
--- /dev/null
+++ b/doc/forum/security_risk_presented_by_remote.log__63__/comment_1_9715488033f2a8939a32ed12259377ba._comment
@@ -0,0 +1,9 @@
+[[!comment format=mdwn
+ username="anarcat"
+ subject="comment 1"
+ date="2015-06-24T00:30:32Z"
+ content="""
+it depends how you configured encryption. this is pretty well described in the [[encryption]] page, but basically, unless you used \"shared\" encryption, only the holders of the GPG private key material will be able to decrypt the contents. Objects are encrypted with GPG using a symmetric key, and that key is then encrypted with OpenPGP public keys.
+
+This is all pretty well described in [[encryption]]. --[[anarcat]]
+"""]]
diff --git a/doc/forum/security_risk_presented_by_remote.log__63__/comment_2_40383346237de0bae4760dc85fefc348._comment b/doc/forum/security_risk_presented_by_remote.log__63__/comment_2_40383346237de0bae4760dc85fefc348._comment
new file mode 100644
index 0000000000..7dc3bd825b
--- /dev/null
+++ b/doc/forum/security_risk_presented_by_remote.log__63__/comment_2_40383346237de0bae4760dc85fefc348._comment
@@ -0,0 +1,15 @@
+[[!comment format=mdwn
+ username="digiuser"
+ subject="comment 2"
+ date="2015-06-24T00:51:44Z"
+ content="""
+I read the encryption page but I just want to know if I'm understanding it correctly.
+
+Let's say I initiated my remote with this command:
+
+    git annex initremote myremote type=S3 chunk=256MiB keyid=XXXXXXXX bucket=mybucket
+
+And then, I handed out my remote.log file to people publicly. Does that expose *any* security hole at all? Or is 100% of the information in remote.log secured using gpg?
+
+I would believe that people couldn't decrypt my file contents, but could they get into my bucket or my S3 account?
+"""]]
diff --git a/doc/forum/security_risk_presented_by_remote.log__63__/comment_3_b1ddf06b34f750e9bbd88e6d5348e765._comment b/doc/forum/security_risk_presented_by_remote.log__63__/comment_3_b1ddf06b34f750e9bbd88e6d5348e765._comment
new file mode 100644
index 0000000000..2bc041b15b
--- /dev/null
+++ b/doc/forum/security_risk_presented_by_remote.log__63__/comment_3_b1ddf06b34f750e9bbd88e6d5348e765._comment
@@ -0,0 +1,29 @@
+[[!comment format=mdwn
+ username="anarcat"
+ subject="comment 3"
+ date="2015-06-24T01:17:23Z"
+ content="""
+> I read the encryption page but I just want to know if I'm understanding it correctly.
+
+I think you are understanding this correctly... let's see..
+
+> Let's say I initiated my remote with this command:
+>
+>     git annex initremote myremote type=S3 chunk=256MiB keyid=XXXXXXXX bucket=mybucket
+>
+> And then, I handed out my remote.log file to people publicly. Does that expose any security hole at all?
+
+It won't expose your S3 credentials, if that's what your are asking. Those are stored in `.git/annex/creds/` and not in the git-annex branch. You can see the content of `remote.log` yourself with:
+
+    git cat-file -p git-annex:remote.log
+
+... if that helps you at all..
+
+>  Or is 100% of the information in remote.log secured using gpg?
+
+Well, it *would* expose the bucket name and the GPG key id (\"XXXXXXXX\") that you set there. The remote.log, itself, is *not* encrypted with gpg, from what I understand. Or to put it another way, the `remote.log` is not actually sent to the S3 remote there, and if you put the git repo publicly, then its content will be publicly readable. To protect against that, you would need a [[special_remotes/gcrypt]] remote.
+
+> I would believe that people couldn't decrypt my file contents, but could they get into my bucket or my S3 account?
+
+Not unless they have the S3 credentials, no. Furthermore, if the bucket is not publicly readable (see [[tips/public_Amazon_S3_remote/]] for that), they won't be able to get the file contents either. And *even* if it is public, they would get the *encrypted* content which they couldn't decrypt without the private key associated with the keyid you supplied.
+"""]]
diff --git a/doc/forum/security_risk_presented_by_remote.log__63__/comment_4_428bbced6406a35eb093a27e35831f38._comment b/doc/forum/security_risk_presented_by_remote.log__63__/comment_4_428bbced6406a35eb093a27e35831f38._comment
new file mode 100644
index 0000000000..0b653de604
--- /dev/null
+++ b/doc/forum/security_risk_presented_by_remote.log__63__/comment_4_428bbced6406a35eb093a27e35831f38._comment
@@ -0,0 +1,7 @@
+[[!comment format=mdwn
+ username="digiuser"
+ subject="comment 4"
+ date="2015-06-24T01:30:26Z"
+ content="""
+What is `s3creds` in `remote.log` then? Something base64 encoded that is named suspiciously like creds for logging into s3?
+"""]]
diff --git a/doc/forum/security_risk_presented_by_remote.log__63__/comment_5_df1eab397c8e08698064391438bbff1b._comment b/doc/forum/security_risk_presented_by_remote.log__63__/comment_5_df1eab397c8e08698064391438bbff1b._comment
new file mode 100644
index 0000000000..4076cd295b
--- /dev/null
+++ b/doc/forum/security_risk_presented_by_remote.log__63__/comment_5_df1eab397c8e08698064391438bbff1b._comment
@@ -0,0 +1,9 @@
+[[!comment format=mdwn
+ username="anarcat"
+ subject="comment 5"
+ date="2015-06-24T01:34:11Z"
+ content="""
+interesting! i don't see that here, maybe that could be a problem there. my s3 remote doesn't use encryption, so maybe i don't have this problem...
+
+i don't know!
+"""]]
diff --git a/doc/forum/security_risk_presented_by_remote.log__63__/comment_6_6a3911dc346d506d4350b5aec7619462._comment b/doc/forum/security_risk_presented_by_remote.log__63__/comment_6_6a3911dc346d506d4350b5aec7619462._comment
new file mode 100644
index 0000000000..0b331bfcc2
--- /dev/null
+++ b/doc/forum/security_risk_presented_by_remote.log__63__/comment_6_6a3911dc346d506d4350b5aec7619462._comment
@@ -0,0 +1,21 @@
+[[!comment format=mdwn
+ username="joey"
+ subject="""comment 6"""
+ date="2015-07-02T20:23:48Z"
+ content="""
+There are two cases there s3creds= can be in the remote.log.
+
+If you enabled gpg encryption, it stores the S3 creds there, encrypted with
+the gpg key you told it to use. So you can share the repo to users who don't
+have the gpg key, and they cannot access the S3 login credentials.
+
+If you didn't use gpg encryption, and you manually set `embedcreds=yes`
+then the s3creds= will contain the un-encrypted creds. 
+And like the docs for embedcreds says, you then need to be careful who
+you give the git repo to, since they can then access those S3 credentials.
+This is not a default configuration.
+
+(There was also the [[upgrades/insecure_embedded_creds]] bug in 2014.
+But, git-annex will detect repos with that problem and warns very verbosely
+about it.)
+"""]]
diff --git a/doc/forum/seems_to_build_fine_on_haskell_platform_2011.mdwn b/doc/forum/seems_to_build_fine_on_haskell_platform_2011.mdwn
new file mode 100644
index 0000000000..60014a7f53
--- /dev/null
+++ b/doc/forum/seems_to_build_fine_on_haskell_platform_2011.mdwn
@@ -0,0 +1 @@
+This is just a comment on git-annex building on haskell platform 2011.2.0.0 on archlinux. It just works.
diff --git a/doc/forum/share_.git__47__annex__47__objects_across_multiple_repositories_on_one_machine.mdwn b/doc/forum/share_.git__47__annex__47__objects_across_multiple_repositories_on_one_machine.mdwn
new file mode 100644
index 0000000000..faf8f4d237
--- /dev/null
+++ b/doc/forum/share_.git__47__annex__47__objects_across_multiple_repositories_on_one_machine.mdwn
@@ -0,0 +1,28 @@
+Hello,
+
+git-annex looks very interesting and I would like it to version large binary artifacts for testing in our source code repository.
+
+My question:
+
+I want to have/can have multiple clones of the same repository on the same machine.
+However, as the binary files can be huge, I would like to store the files only exactly ONCE per machine and not again in the .git/annex/objects folder of each similar cloned repository.
+
+To achieve that, I first created in
+
+    /tmp/repo-clone1/.git/annex/objects
+
+and then symlinked
+
+    ln -s /tmp/repo-clone1/.git/annex/objects /tmp/repo-clone2/.git/annex/objects
+
+such that
+
+    /tmp/repo-clone1
+    /tmp/repo-clone2
+
+share the same big files and the big files are only once on the machine.
+
+Is this a good idea? Is there a better way to achieve this? Looks a bit hacky. Would be nicer if you can specify a dedicated "objects" folder from the start?!
+
+Thanks and Regards,
+J
diff --git a/doc/forum/share_.git__47__annex__47__objects_across_multiple_repositories_on_one_machine/comment_1_1606849eb13b6069394455297395dc30._comment b/doc/forum/share_.git__47__annex__47__objects_across_multiple_repositories_on_one_machine/comment_1_1606849eb13b6069394455297395dc30._comment
new file mode 100644
index 0000000000..bd7ece79d9
--- /dev/null
+++ b/doc/forum/share_.git__47__annex__47__objects_across_multiple_repositories_on_one_machine/comment_1_1606849eb13b6069394455297395dc30._comment
@@ -0,0 +1,15 @@
+[[!comment format=mdwn
+ username="https://id.koumbit.net/anarcat"
+ subject="use hardlinks"
+ date="2015-05-29T01:40:21Z"
+ content="""
+that is a pretty bad idea! :) git-annex will believe it has two copy of the files and could allow you to drop the last copy, and loose data.
+
+instead, you should clone the repo with `--shared`, like this:
+
+    git clone --shared /tmp/repo-clone1 /tmp/repo-clone2
+
+according to the [[git-annex]] manpage, this will set the `annex.hardlink` setting and mark the repo as \"untrusted\". files will be hardlinked between the two repositories, using only the space once.
+
+see also [[todo/wishlist:_use_hardlinks_for_local_clones/]]. --[[anarcat]]
+"""]]
diff --git a/doc/forum/share_.git__47__annex__47__objects_across_multiple_repositories_on_one_machine/comment_2_f60c43ec9d493db03e2a5a5595501e55._comment b/doc/forum/share_.git__47__annex__47__objects_across_multiple_repositories_on_one_machine/comment_2_f60c43ec9d493db03e2a5a5595501e55._comment
new file mode 100644
index 0000000000..2d8d6e7d92
--- /dev/null
+++ b/doc/forum/share_.git__47__annex__47__objects_across_multiple_repositories_on_one_machine/comment_2_f60c43ec9d493db03e2a5a5595501e55._comment
@@ -0,0 +1,11 @@
+[[!comment format=mdwn
+ username="davicastro"
+ avatar="http://cdn.libravatar.org/avatar/4e708663cf4d5b9e8cfea74caf4307fc"
+ subject="any drawbacks in using --shared (hardlinks) clones on same Linux machine?"
+ date="2018-01-07T00:37:10Z"
+ content="""
+Hi, I'm new to git and we might be adopting it on next project solely because of git annex!
+As our files are quite big, having users cloning the repository with --shared is very very nice!
+We use Linux, so no issue with hardlinks. I did some testing and so far so good. No duplicated space is very very nice!
+But I'm wondering: is there any disadvantage on this approach?
+"""]]
diff --git a/doc/forum/share_.git__47__annex__47__objects_across_multiple_repositories_on_one_machine/comment_3_b041da6694d9d9d21a67f7bbdda0f62e._comment b/doc/forum/share_.git__47__annex__47__objects_across_multiple_repositories_on_one_machine/comment_3_b041da6694d9d9d21a67f7bbdda0f62e._comment
new file mode 100644
index 0000000000..0a48c1ca63
--- /dev/null
+++ b/doc/forum/share_.git__47__annex__47__objects_across_multiple_repositories_on_one_machine/comment_3_b041da6694d9d9d21a67f7bbdda0f62e._comment
@@ -0,0 +1,20 @@
+[[!comment format=mdwn
+ username="joey"
+ subject="""comment 3"""
+ date="2018-01-10T17:24:04Z"
+ content="""
+@davicastro using --shared makes git-annex not trust the shared clone, 
+which is necessary to avoid situations that could result in data loss.
+The downside though, is that the lack of trust can change git-annex
+behavior in some situations.
+
+For example, normally you can run `git annex get myfile` and then `git
+annex drop myfile --from someremote` will remove it from the remote, since
+you now have a local copy. But, with the shared clone being untrusted, the
+drop will fail if it would be the only remaining copy of the file. In
+this situation, you would need to first `git annex copy myfile --to origin`
+or something like that before dropping.
+
+Of course, that copy would run fast and cheap since it only has to make a
+hardlink!
+"""]]
diff --git a/doc/forum/shared_cipher_tries_to_use_gpg.mdwn b/doc/forum/shared_cipher_tries_to_use_gpg.mdwn
new file mode 100644
index 0000000000..ccaa0c59d6
--- /dev/null
+++ b/doc/forum/shared_cipher_tries_to_use_gpg.mdwn
@@ -0,0 +1,10 @@
+I tried 
+
+    git annex initremote encsharedtest type=directory encryption=shared directory=/home/lee/gitannexplay
+
+and got errors:
+
+    initremote encsharedtest gpg: error reading key: public key not found
+
+Looks like it thinks "shared" should be the name of a key rather than an instruction to use the shared cipher.
+Am I doing something wrong?
diff --git a/doc/forum/shared_cipher_tries_to_use_gpg/comment_1_760961eaaa7d5c254dd71c5792437c9e._comment b/doc/forum/shared_cipher_tries_to_use_gpg/comment_1_760961eaaa7d5c254dd71c5792437c9e._comment
new file mode 100644
index 0000000000..e97118b8a7
--- /dev/null
+++ b/doc/forum/shared_cipher_tries_to_use_gpg/comment_1_760961eaaa7d5c254dd71c5792437c9e._comment
@@ -0,0 +1,12 @@
+[[!comment format=mdwn
+ username="http://joeyh.name/"
+ ip="4.154.0.23"
+ subject="comment 1"
+ date="2012-10-23T19:51:03Z"
+ content="""
+That's the right syntax, and it works when I try it, even if I move my `~/.gnupg` directory away.
+
+You sure you have a new enough git-annex for the shared cipher feature? It appeared in version 3.20120430
+
+You could try passing the `--debug` option to see what it's trying to do with gpg when it fails.
+"""]]
diff --git a/doc/forum/shared_cipher_tries_to_use_gpg/comment_2_f3260aea3a5bb9b95a9bdf1d0dfce090._comment b/doc/forum/shared_cipher_tries_to_use_gpg/comment_2_f3260aea3a5bb9b95a9bdf1d0dfce090._comment
new file mode 100644
index 0000000000..7c36e7cf4d
--- /dev/null
+++ b/doc/forum/shared_cipher_tries_to_use_gpg/comment_2_f3260aea3a5bb9b95a9bdf1d0dfce090._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="https://www.google.com/accounts/o8/id?id=AItOawlFUH5H4OUj9vMJIoXQs8bheiptgANQ6fU"
+ nickname="lee"
+ subject="A little too old"
+ date="2012-10-23T20:47:40Z"
+ content="""
+That's the problem, then. I have 3.20120406. For my purposes a gpg key with no passphrase works, though. Thanks.
+"""]]
diff --git a/doc/forum/sneakernet_with_a___34__directory__34___special_remote_on_fat.mdwn b/doc/forum/sneakernet_with_a___34__directory__34___special_remote_on_fat.mdwn
new file mode 100644
index 0000000000..ed0ffb5f34
--- /dev/null
+++ b/doc/forum/sneakernet_with_a___34__directory__34___special_remote_on_fat.mdwn
@@ -0,0 +1,26 @@
+Hello,
+
+I'm starting to use git annex as a replacement for unison to sync/backup my family pictures.
+It looks very promising, thank you very much for all this work!
+
+My big plan (for now) is to use git annex
+
+* on my Linux home server
+* on my mac OS laptop
+* on a (big) USB drive, that I could plug either on the server (at home) or the laptop (during holidays)
+
+Since I want the USB drive to be usable from both Linux and mac OS, I picked FAT as a file system.
+
+I first tried to set up a bare repository on the USB drive, but did hit the [[bugs/VFAT_crazy_limit_on_max_filenames_in_directory]].
+So I used a "directory" special remote, and copied files from the server. It works fine.
+
+I then created the SSH remote on the laptop and synced some content. It works quite well too.
+
+I would now like to plug the USB drive on the laptop and tell it
+"hey, this is the same remote as the one you've seen from the server, just mounted somewhere else"
+(the path is `/Volumes/usbdrive/annex` on the laptop and `/media/usbdrive/annex` on the server).
+
+The [[special_remotes/directory]] doc suggests it is possible, but I don't know how.
+Shall I copy the relevant bit from the server `.git/config`, keeping the UUID and tweaking the mount point?
+
+Thank you!
diff --git a/doc/forum/sneakernet_with_a___34__directory__34___special_remote_on_fat/comment_1_a6f0ae755b9e24fd0de5868b57c6a497._comment b/doc/forum/sneakernet_with_a___34__directory__34___special_remote_on_fat/comment_1_a6f0ae755b9e24fd0de5868b57c6a497._comment
new file mode 100644
index 0000000000..73c60f3b39
--- /dev/null
+++ b/doc/forum/sneakernet_with_a___34__directory__34___special_remote_on_fat/comment_1_a6f0ae755b9e24fd0de5868b57c6a497._comment
@@ -0,0 +1,13 @@
+[[!comment format=mdwn
+ username="https://launchpad.net/~barthelemy"
+ nickname="barthelemy"
+ avatar="http://cdn.libravatar.org/avatar/e99cb15f6029de3225721b3ebdd0233905eb69698e9b229a8c4cc510a4135438"
+ subject="I found it out!"
+ date="2017-05-05T12:41:59Z"
+ content="""
+On the laptop, I just have to run
+
+    git annex enableremote usbdrive directory=/Volumes/usbdrive/annex
+
+But I have not found a way to disable it (for use when the drive is unplugged) yet. It may not matter.
+"""]]
diff --git a/doc/forum/sneakernet_with_a___34__directory__34___special_remote_on_fat/comment_2_9fe4a596aea6bd1a898b4c235126dce2._comment b/doc/forum/sneakernet_with_a___34__directory__34___special_remote_on_fat/comment_2_9fe4a596aea6bd1a898b4c235126dce2._comment
new file mode 100644
index 0000000000..1ef9386c49
--- /dev/null
+++ b/doc/forum/sneakernet_with_a___34__directory__34___special_remote_on_fat/comment_2_9fe4a596aea6bd1a898b4c235126dce2._comment
@@ -0,0 +1,14 @@
+[[!comment format=mdwn
+ username="joey"
+ subject="""comment 2"""
+ date="2017-05-09T17:48:24Z"
+ content="""
+The directory= parameter to enableremote only affects the local
+configuration of that remote, it's not stored anywhere. Which is why you
+have to pass that parameter each time you're enabling that same
+directory remote in different repositories.
+
+So, it's fine to have directory= set to different values in different
+repositories, or even to re-run enableremote with a new directory= setting
+as needed.
+"""]]
diff --git a/doc/forum/sneakernet_with_a___34__directory__34___special_remote_on_fat/comment_3_02eae41ba9ad2f2fb15cbd20069bf1da._comment b/doc/forum/sneakernet_with_a___34__directory__34___special_remote_on_fat/comment_3_02eae41ba9ad2f2fb15cbd20069bf1da._comment
new file mode 100644
index 0000000000..f9803584ef
--- /dev/null
+++ b/doc/forum/sneakernet_with_a___34__directory__34___special_remote_on_fat/comment_3_02eae41ba9ad2f2fb15cbd20069bf1da._comment
@@ -0,0 +1,11 @@
+[[!comment format=mdwn
+ username="https://launchpad.net/~barthelemy"
+ nickname="barthelemy"
+ avatar="http://cdn.libravatar.org/avatar/e99cb15f6029de3225721b3ebdd0233905eb69698e9b229a8c4cc510a4135438"
+ subject="comment 3"
+ date="2017-05-09T23:38:27Z"
+ content="""
+Hi Joel,
+thank you for the precision (and for git annex, and for all the rest!)
+Cheers
+"""]]
diff --git a/doc/forum/some_symlinks_left_in_direct_mode.mdwn b/doc/forum/some_symlinks_left_in_direct_mode.mdwn
new file mode 100644
index 0000000000..1f4e447ee0
--- /dev/null
+++ b/doc/forum/some_symlinks_left_in_direct_mode.mdwn
@@ -0,0 +1,5 @@
+Dear all,
+   I have a repository in direct mode which still has some symlinks pointing to .git/annex/objects. Is there a way to turn them back into regular files. I don't recall why this happen, I guess I started on indirect mode and after that I converted it to direct mode.
+Thanks in advance.
+Best regards,
+              Juan
diff --git a/doc/forum/some_symlinks_left_in_direct_mode/comment_1_fd8d8107861453f0aa6b2c390e369307._comment b/doc/forum/some_symlinks_left_in_direct_mode/comment_1_fd8d8107861453f0aa6b2c390e369307._comment
new file mode 100644
index 0000000000..29e8b7490d
--- /dev/null
+++ b/doc/forum/some_symlinks_left_in_direct_mode/comment_1_fd8d8107861453f0aa6b2c390e369307._comment
@@ -0,0 +1,17 @@
+[[!comment format=mdwn
+ username="joey"
+ subject="""comment 1"""
+ date="2014-12-02T18:53:32Z"
+ content="""
+Well, it's normal to have broken symlinks if the content of a file
+is not present. `git annex get` will get the file content
+and replace the symlink with it.
+
+If the symlink is not broken, you can run `git annex fsck` on the file
+and it will notice and fix this inconsistency. 
+
+Some old versions of git-annex had bugs in direct mode that could result
+in files being left as symlinks like that. I have not seen any bugs
+lately in that area. If you have a way to get a repository into that
+state, file a bug report with a test case.
+"""]]
diff --git a/doc/forum/some_symlinks_left_in_direct_mode/comment_2_63f61caf28e834d48e12467bf09b2e3d._comment b/doc/forum/some_symlinks_left_in_direct_mode/comment_2_63f61caf28e834d48e12467bf09b2e3d._comment
new file mode 100644
index 0000000000..47d8a5bec8
--- /dev/null
+++ b/doc/forum/some_symlinks_left_in_direct_mode/comment_2_63f61caf28e834d48e12467bf09b2e3d._comment
@@ -0,0 +1,9 @@
+[[!comment format=mdwn
+ username="https://www.google.com/accounts/o8/id?id=AItOawmTNrhkVQ26GBLaLD5-zNuEiR8syTj4mI8"
+ nickname="Juan"
+ subject="going back to indirect mode fixed it"
+ date="2014-12-03T18:35:58Z"
+ content="""
+I went back to indirect mode and stayed there. Every file is now accesible. I don't know what happened.
+Thanks.
+"""]]
diff --git a/doc/forum/something_really_good_happened_with_3.20130124.mdwn b/doc/forum/something_really_good_happened_with_3.20130124.mdwn
new file mode 100644
index 0000000000..73b2838222
--- /dev/null
+++ b/doc/forum/something_really_good_happened_with_3.20130124.mdwn
@@ -0,0 +1,5 @@
+The web app used to be unresponsive without manual refreshes.  Now it's all snappy and responsive and quickly reflects what the assistant is doing.
+
+It used to be a complete crapshoot whether the assistant actually shut down when you told it to (either on the command line or through the webapp).  I used to have to manually kill leftover assistants and child processes routinely.  Now... you want it gone, and it's gone instantly, every time.
+
+Something really good happened with this last version, thank you joey!
diff --git a/doc/forum/something_really_good_happened_with_3.20130124/comment_1_1712bddd2f483a353f6313aa626445f1._comment b/doc/forum/something_really_good_happened_with_3.20130124/comment_1_1712bddd2f483a353f6313aa626445f1._comment
new file mode 100644
index 0000000000..e0a4907c68
--- /dev/null
+++ b/doc/forum/something_really_good_happened_with_3.20130124/comment_1_1712bddd2f483a353f6313aa626445f1._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="http://joeyh.name/"
+ ip="2001:4978:f:21a::2"
+ subject="comment 1"
+ date="2013-01-25T22:35:00Z"
+ content="""
+We fixed some broken javascript that only worked in all the browsers I tried, but not yours. It's always been this cool, just glad you can also experience it now!
+"""]]
diff --git a/doc/forum/sparse_git_checkouts_with_annex.mdwn b/doc/forum/sparse_git_checkouts_with_annex.mdwn
new file mode 100644
index 0000000000..97d2f445d3
--- /dev/null
+++ b/doc/forum/sparse_git_checkouts_with_annex.mdwn
@@ -0,0 +1,31 @@
+I checked in my music collection into git annex (about 25000 files) and i'm really impressed by the performance of git annex (after i've done an git-repack). Now i'm also moving my movies into the same git-annex, but i have the following layout of my disk drives:
+
+* small raid-1 for important stuff (music, documents), which is also backupped (aka: raid)
+* big bulk data store (aka: media)
+
+In the git-annex the following layout of files is used:
+
+* documents/ <- on raid
+* music/ <- on raid
+* videos/ <- on media
+
+Now i didn't simply clone the raid-annex to media, but did an sparse-checkout (possible since version 1.7.0)
+
+* raid: .git-annex/, documents/ and music
+* media: .git-annex/, videos/
+
+As you can see i have to checkout the .git-annex directory with the file-logs twice which slows down git operations. Everything else works fine until now. git-annex doesn't have any problem, that only a part of the symlinks are present, which is really great. Is there a possibility to sparse checkout the .git-annex directory also? Perhaps splitting the log files in .git-annex/ into N subfolders, corresponding to the toplevel subfolders, like this?
+
+Before:
+
+     $ ls .git-annex
+     00 01 02....
+
+After:
+
+     $ ls .git-annex
+     documents/ music/ videos/
+     $ ls .git-annex/documents
+     00 01 02....
+
+This would make it possible to checkout only the part of the log files which i'm interested in.
diff --git a/doc/forum/sparse_git_checkouts_with_annex/comment_1_c7dc199c5740a0e7ba606dfb5e3e579a._comment b/doc/forum/sparse_git_checkouts_with_annex/comment_1_c7dc199c5740a0e7ba606dfb5e3e579a._comment
new file mode 100644
index 0000000000..7adf4fc4d6
--- /dev/null
+++ b/doc/forum/sparse_git_checkouts_with_annex/comment_1_c7dc199c5740a0e7ba606dfb5e3e579a._comment
@@ -0,0 +1,12 @@
+[[!comment format=mdwn
+ username="http://joey.kitenet.net/"
+ nickname="joey"
+ subject="comment 1"
+ date="2011-04-07T16:32:04Z"
+ content="""
+That's awesome, I had not heard of git sparse checkouts before. 
+
+It does not make sense to tie the log files to the directory of the corresponding files, as then the logs would have to move when the files are moved, which would be a PITA and likely make merging log file changes very complex. Also, of course, multiple files in different locations can point at the same content, which has the same log file. And, to cap it off, git-annex can need to access the log file for a given key without having the slightest idea what file in the repository might point to it, and it would be very expensive to scan the whole repository to find out what that file is in order to lookup the filename of the log file.
+
+The most likely change in git-annex that will make this better is in [[this_todo_item|todo/branching]] -- but it's unknown how to do it yet.
+"""]]
diff --git a/doc/forum/sparse_git_checkouts_with_annex/comment_2_e357db3ccc4079f07a291843975535eb._comment b/doc/forum/sparse_git_checkouts_with_annex/comment_2_e357db3ccc4079f07a291843975535eb._comment
new file mode 100644
index 0000000000..d8088a2d82
--- /dev/null
+++ b/doc/forum/sparse_git_checkouts_with_annex/comment_2_e357db3ccc4079f07a291843975535eb._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="http://joey.kitenet.net/"
+ nickname="joey"
+ subject="comment 2"
+ date="2011-04-07T16:33:30Z"
+ content="""
+BTW, git-annex unused *will* have a problem that not all the symlinks are present. It will suggest dropping content belonging to the excluded symlinks.
+"""]]
diff --git a/doc/forum/sparse_git_checkouts_with_annex/comment_3_fcfafca994194d57dccf5319c7c9e646._comment b/doc/forum/sparse_git_checkouts_with_annex/comment_3_fcfafca994194d57dccf5319c7c9e646._comment
new file mode 100644
index 0000000000..1b849ef891
--- /dev/null
+++ b/doc/forum/sparse_git_checkouts_with_annex/comment_3_fcfafca994194d57dccf5319c7c9e646._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="https://www.google.com/accounts/o8/id?id=AItOawkptNW1PzrVjYlJWP_9e499uH0mjnBV6GQ"
+ nickname="Christian"
+ subject="comment 3"
+ date="2011-04-08T07:31:03Z"
+ content="""
+So perhaps checking if git-status (or similar) complains about missing files is a possible solution for this?
+"""]]
diff --git a/doc/forum/sparse_git_checkouts_with_annex/comment_4_04dc14880f31eee2b6d767d4d4258c5a._comment b/doc/forum/sparse_git_checkouts_with_annex/comment_4_04dc14880f31eee2b6d767d4d4258c5a._comment
new file mode 100644
index 0000000000..9280fc51da
--- /dev/null
+++ b/doc/forum/sparse_git_checkouts_with_annex/comment_4_04dc14880f31eee2b6d767d4d4258c5a._comment
@@ -0,0 +1,20 @@
+[[!comment format=mdwn
+ username="https://www.google.com/accounts/o8/id?id=AItOawkptNW1PzrVjYlJWP_9e499uH0mjnBV6GQ"
+ nickname="Christian"
+ subject="comment 4"
+ date="2011-04-08T07:54:37Z"
+ content="""
+And something else i've done is, that i symlinked the video/ directory from the media annex to the normal raid annex
+
+    ln -s ~/media/annex/video ~/annex
+
+And it's working out great.
+
+    ~annex $ git annex whereis video/series/episode1.avi
+    whereis video/series/episode1.avi(1 copy)
+        	f210b45a-60d3-11e0-b593-3318d96f2520  -- Trantor - Media
+    ok
+
+I really like this, perhaps it is a good idea to store all log files in every repo, but maybe there is a possibilitiy to to pack multiple log files into one single file, where not only the time, the present bit and the annex-repository is stored, but also the file key. I don't know if this format would also be merged correctly by the union merge driver.
+
+"""]]
diff --git a/doc/forum/special_remote_for_IMAP.mdwn b/doc/forum/special_remote_for_IMAP.mdwn
new file mode 100644
index 0000000000..2aa9565515
--- /dev/null
+++ b/doc/forum/special_remote_for_IMAP.mdwn
@@ -0,0 +1,44 @@
+I have implemented a special remote that stores files as email messages on an imap server. You need   to install three utilities that the hooks invoke to deal with the email: mutt, imaputils, and         munpack. I use mutt to send the email with the file as a mime attachment; imaputils talks to the imap server to check for and retrieve the message containing the desired file; and munpack extracts and    decodes the attachment to get our file back.
+
+Several programs could be used in place of mutt, but the latter has a convenient command-line option  for attaching files; mutt is of course available in the repositories of most linux distributions.
+
+imaputils is a perl program available at http://sourceforge.net/projects/imaputils/
+It has several perl library dependencies that you might need to download using the cpan tool.
+What imaputils does for you is provide a command line tool for interacting with the mail server. You  can search for mail with a particular subject (for example), delete mail, retrieve messages, and in   general do anything that you can do with a mail client such as mutt, but from the command line rather than a curses interface. This allows you to in turn write scripts that talk to imap servers.
+
+munpack is part of the mpack package. This is included in the Ubuntu and Debian repositories, and can probably be easily obtained for most linuxes. munpack extracts and decodes mime attachments from the  command line.
+
+I define the special remote with
+
+    git annex initremote hogneygmail type=hook encryption=gitannex hooktype=hogneygmail
+
+The pgp key "gitannex" is a key established just for this purpose, that has no passphrase. This       allows me to use encryption transparently. You could also use encryption=shared if your version of    git-annex is recent enough. I also did
+
+    git annex untrust hogneygmail
+
+Here are the hooks:
+
+        hogneygmail-store-hook = mutt -n -s $ANNEX_KEY -a $ANNEX_FILE -- {email address} < /dev/null
+        hogneygmail-checkpresent-hook = "(imaputils.pl --conf {imap config file} --subject $ANNEX_KEY --count | grep -q \"1 messages\" -) && echo $ANNEX_KEY"
+        hogneygmail-retrieve-hook =  "imaputils.pl --conf {imap config file} --subject $ANNEX_KEY --display | munpack -fq && mv $ANNEX_KEY $ANNEX_FILE; rm $ANNEX_KEY.desc"
+        hogneygmail-remove-hook = "imaputils.pl --conf {imap config file} --subject $ANNEX_KEY --delete"
+
+The bits inside of the curly brackets, for example {email address}, are what you need to specialize   for your particular case (removing the brackets as well). The {imap config file} is a file that       contains some configuration for imaputils. In my case it contains
+
+    ssl
+    pass {password}
+    host {mail host}
+    user {mail username}
+    box  {name of mailbox to check}
+
+The first line tells imaputils to use ssl, and the third line is the address of the mail host. I      include my username and password so I won't need to type these in repeatedly. Of course this means    that you need to protect this file carefully.
+
+
+
+The operation of the hooks is pretty straightforward. The store-hook attaches the file to an          otherwise empty email message with a subject equal to the name of the key, and mails it off. Note     that if you use encryption then the keys generated by git-annex here will not be the same ones you    see on your local disc. The checkpresent-hook asks the imap server how many emails have the subject   equal to the key we are looking for; only if the reply contains "1 messages" are we sure the right    one is there. The retrieve-hook uses the "--display" option to imaputils to stream the message, pipes it to munpack to silently extract the attachment (which is our (encrypted) file), and moves the       result into the file contents. It then cleans up by deleting the .desc file that I can't prevent      munpack from leaving on the disk, even when the message is blank. The remove-hook passes the "--      delete" option to imaputils to supposedly delete the target message from the mail server.
+
+I've performed very limited testing of this, and my knowledge of git and, especially, git-annex is    very primitive, so I'm sure this could be vastly improved. In my testing with gmail this seems to     work fine, except that messages do not get deleted from the server - I don't know why.
+
+I've developed this as an experiment and proof of concept, and have no knowledge of whether actually  using this is in accord with the terms of service of gmail or any other mail service you might be     using, nor whether it is safe or a good idea.
+
+-- Lee
diff --git a/doc/forum/special_remote_for_IMAP/comment_1_7c7d4b57a1b6508fff1a6b0508c861f8._comment b/doc/forum/special_remote_for_IMAP/comment_1_7c7d4b57a1b6508fff1a6b0508c861f8._comment
new file mode 100644
index 0000000000..8d81dcfd03
--- /dev/null
+++ b/doc/forum/special_remote_for_IMAP/comment_1_7c7d4b57a1b6508fff1a6b0508c861f8._comment
@@ -0,0 +1,10 @@
+[[!comment format=mdwn
+ username="http://joeyh.name/"
+ ip="2001:4978:f:21a::2"
+ subject="Neat idea"
+ date="2012-10-26T18:35:21Z"
+ content="""
+I imagine that most mail servers have a maximum attachment size; that seems the most likely problem with using this.
+
+The directory special remote has support for splitting up keys into chunks to work around file size limits. It would be good, I suppose, to add that to the hook special remote too.
+"""]]
diff --git a/doc/forum/special_remote_for_IMAP/comment_2_9c46fe8a857aa7a5ce797288144386bd._comment b/doc/forum/special_remote_for_IMAP/comment_2_9c46fe8a857aa7a5ce797288144386bd._comment
new file mode 100644
index 0000000000..50358ea53a
--- /dev/null
+++ b/doc/forum/special_remote_for_IMAP/comment_2_9c46fe8a857aa7a5ce797288144386bd._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="https://www.google.com/accounts/o8/id?id=AItOawlFUH5H4OUj9vMJIoXQs8bheiptgANQ6fU"
+ nickname="lee"
+ subject="chunking"
+ date="2012-10-26T22:05:16Z"
+ content="""
+That's a good point. But the splitting could easily be added to the hooks, and wouldn't need to be built in to git-annex.
+"""]]
diff --git a/doc/forum/special_remote_for_IMAP/comment_3_27e3b644df6942ce4c103236d0d5cb1b._comment b/doc/forum/special_remote_for_IMAP/comment_3_27e3b644df6942ce4c103236d0d5cb1b._comment
new file mode 100644
index 0000000000..b5d9137e64
--- /dev/null
+++ b/doc/forum/special_remote_for_IMAP/comment_3_27e3b644df6942ce4c103236d0d5cb1b._comment
@@ -0,0 +1,23 @@
+[[!comment format=mdwn
+ username="https://www.google.com/accounts/o8/id?id=AItOawmWg4VvDTer9f49Y3z-R0AH16P4d1ygotA"
+ nickname="Tobias"
+ subject="comment 3"
+ date="2013-05-22T09:53:47Z"
+ content="""
+I had to change the checkpresent hook to:
+
+	hogneygmail-checkpresent-hook = (imaputils.pl --subject $ANNEX_KEY --count | grep -q \\"1 messages\\" -) && echo $ANNEX_KEY || exit 0
+
+Also, i have 5 files more than 30MB in size, that refuse to transfer.
+
+ postdrop: warning: uid=1001: File too large
+ sendmail: fatal: tou(1001): message file too big
+ Error sending message, child exited 75 (Deferred.).
+ Could not send the message.
+
+  store hook exited nonzero!
+ failed
+
+This really needs some split operations to support large files.
+
+"""]]
diff --git a/doc/forum/special_remote_for_IMAP/comment_4_0f8e01c453afb02aebf44b3fb2c9a7c1._comment b/doc/forum/special_remote_for_IMAP/comment_4_0f8e01c453afb02aebf44b3fb2c9a7c1._comment
new file mode 100644
index 0000000000..6e39fd7307
--- /dev/null
+++ b/doc/forum/special_remote_for_IMAP/comment_4_0f8e01c453afb02aebf44b3fb2c9a7c1._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="https://www.google.com/accounts/o8/id?id=AItOawmH7o6q2l99M-PQolOfbR3_i5B_jtTIcAE"
+ nickname="Giovanni"
+ subject="comment 4"
+ date="2014-09-12T18:57:07Z"
+ content="""
+Does this somehow scan the IMAP server for attachments already there?
+"""]]
diff --git a/doc/forum/special_remote_for_iPods.mdwn b/doc/forum/special_remote_for_iPods.mdwn
new file mode 100644
index 0000000000..eddda5936f
--- /dev/null
+++ b/doc/forum/special_remote_for_iPods.mdwn
@@ -0,0 +1,5 @@
+I know versions of this question have been asked before, but I'm looking for a different answer.
+
+I would like a braindead special remote that can be used with devices such as portable music players. No symbolic links, no hashing, no rewriting of the filenames. The remote can be untrusted, file identity can be checked with just the filename and maybe the size. The "directory" special remote with the WORM backend seems to come closest, but does too much. 
+
+Should I just try to roll it using hooks?
diff --git a/doc/forum/special_remote_for_iPods/comment_1_37cc3dc740341cc663074fd3bfb85947._comment b/doc/forum/special_remote_for_iPods/comment_1_37cc3dc740341cc663074fd3bfb85947._comment
new file mode 100644
index 0000000000..35ac105c15
--- /dev/null
+++ b/doc/forum/special_remote_for_iPods/comment_1_37cc3dc740341cc663074fd3bfb85947._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="http://joeyh.name/"
+ ip="4.154.0.149"
+ subject="comment 1"
+ date="2012-10-05T00:39:43Z"
+ content="""
+I've been thinking in similar directions on this page: [[design/assistant/desymlink]]
+"""]]
diff --git a/doc/forum/special_remote_which_applies_a_lossless_operation_in__47__out.mdwn b/doc/forum/special_remote_which_applies_a_lossless_operation_in__47__out.mdwn
new file mode 100644
index 0000000000..37a60bc341
--- /dev/null
+++ b/doc/forum/special_remote_which_applies_a_lossless_operation_in__47__out.mdwn
@@ -0,0 +1,9 @@
+Hi,
+
+I'd like to modify the directory special remote, to compress/deflate files before the store/retrieve operations. Purpose is to gain size, with a lossless compression, different depending on the file type (LZMA as a default, dropbox-lepton for jpegs, etc...).
+
+Before I go on, I wonder if the file content modification will interfere with git annex internals.
+It seems it won't, from what I see in the special remote basic exemple given in https://git-annex.branchable.com/special_remotes/external/
+Can someone confirm ?
+
+Thks
diff --git a/doc/forum/special_remote_which_applies_a_lossless_operation_in__47__out/comment_1_195f52aaaa36ce20955f2eb40af3c79a._comment b/doc/forum/special_remote_which_applies_a_lossless_operation_in__47__out/comment_1_195f52aaaa36ce20955f2eb40af3c79a._comment
new file mode 100644
index 0000000000..ea497a570e
--- /dev/null
+++ b/doc/forum/special_remote_which_applies_a_lossless_operation_in__47__out/comment_1_195f52aaaa36ce20955f2eb40af3c79a._comment
@@ -0,0 +1,11 @@
+[[!comment format=mdwn
+ username="joey"
+ subject="""comment 1"""
+ date="2017-04-07T20:34:38Z"
+ content="""
+Of course that will work fine, as long as the operation is reversable.
+
+Plenty of existing special remotes
+compress/encrypt/slice/dice/mangle/obfuscate/whatever the content of files
+stored in them, and undo it all when the contents are retreived.
+"""]]
diff --git a/doc/forum/speed_up_assistant_startup_on_large_repositories.mdwn b/doc/forum/speed_up_assistant_startup_on_large_repositories.mdwn
new file mode 100644
index 0000000000..e43476c7b9
--- /dev/null
+++ b/doc/forum/speed_up_assistant_startup_on_large_repositories.mdwn
@@ -0,0 +1 @@
+Starting the assistant on a repository with a huge amount of files (f.e. `annexed files in working tree: 22087`) takes a very long time. Is there a way to speed it up? F.e. just checking the file stamps and comparing them with the last assistant run time? Checking if the files are modified could be done with the regular repository check. Thoughts?
diff --git a/doc/forum/speed_up_assistant_startup_on_large_repositories/comment_1_5ba637a0f6d01ba24fe25e6265134e0a._comment b/doc/forum/speed_up_assistant_startup_on_large_repositories/comment_1_5ba637a0f6d01ba24fe25e6265134e0a._comment
new file mode 100644
index 0000000000..0abe4dec8a
--- /dev/null
+++ b/doc/forum/speed_up_assistant_startup_on_large_repositories/comment_1_5ba637a0f6d01ba24fe25e6265134e0a._comment
@@ -0,0 +1,10 @@
+[[!comment format=mdwn
+ username="http://joeyh.name/"
+ ip="209.250.56.47"
+ subject="comment 1"
+ date="2013-11-05T16:43:26Z"
+ content="""
+Checking the time stamps is what it does.
+
+How long is a very long time?
+"""]]
diff --git a/doc/forum/speed_up_assistant_startup_on_large_repositories/comment_2_d65746697977f8971a4b59f5b413f926._comment b/doc/forum/speed_up_assistant_startup_on_large_repositories/comment_2_d65746697977f8971a4b59f5b413f926._comment
new file mode 100644
index 0000000000..e6915d2328
--- /dev/null
+++ b/doc/forum/speed_up_assistant_startup_on_large_repositories/comment_2_d65746697977f8971a4b59f5b413f926._comment
@@ -0,0 +1,13 @@
+[[!comment format=mdwn
+ username="https://www.google.com/accounts/o8/id?id=AItOawne_amN4fko4p5cRY_9EYwaYuJKNn7LRio"
+ nickname="Tobias"
+ subject="measuring how-to"
+ date="2013-11-08T16:55:58Z"
+ content="""
+How can I measure how much time the startup scan takes? I tried to find it out using the debug log, but couldn't find any usable notice when the startup scan was finished. Maybe this two lines?
+
+    [...]
+    [2013-11-08 16:00:45 CET] Watcher: Performing startup scan
+    [...]
+    [2013-11-08 16:15:30 CET] Watcher: watching 
+"""]]
diff --git a/doc/forum/speed_up_assistant_startup_on_large_repositories/comment_3_be6c4fe5a0c745688438b716973791cc._comment b/doc/forum/speed_up_assistant_startup_on_large_repositories/comment_3_be6c4fe5a0c745688438b716973791cc._comment
new file mode 100644
index 0000000000..d3fe6ef4c8
--- /dev/null
+++ b/doc/forum/speed_up_assistant_startup_on_large_repositories/comment_3_be6c4fe5a0c745688438b716973791cc._comment
@@ -0,0 +1,12 @@
+[[!comment format=mdwn
+ username="http://joeyh.name/"
+ ip="108.236.230.124"
+ subject="comment 3"
+ date="2013-11-08T17:04:23Z"
+ content="""
+That's right.
+
+15 minutes is certainly a very long time.
+
+Is this on a slow spinning disk? USB disk?
+"""]]
diff --git a/doc/forum/speed_up_assistant_startup_on_large_repositories/comment_4_a07472338a08c068a9b88b2176fc2bee._comment b/doc/forum/speed_up_assistant_startup_on_large_repositories/comment_4_a07472338a08c068a9b88b2176fc2bee._comment
new file mode 100644
index 0000000000..41e69ab1e4
--- /dev/null
+++ b/doc/forum/speed_up_assistant_startup_on_large_repositories/comment_4_a07472338a08c068a9b88b2176fc2bee._comment
@@ -0,0 +1,9 @@
+[[!comment format=mdwn
+ username="https://www.google.com/accounts/o8/id?id=AItOawniayrgSdVLUc3c6bf93VbO-_HT4hzxmyo"
+ nickname="Tobias"
+ subject="comment 4"
+ date="2013-11-17T17:28:06Z"
+ content="""
+The mentioned repository is stored on spinning disk.
+Maybe it would help to store the .git directory on an SSD, but I'm not sure how to do it.
+"""]]
diff --git a/doc/forum/ssh__95__exchange__95__identification__58___read__58___Connection_reset_by_peer.mdwn b/doc/forum/ssh__95__exchange__95__identification__58___read__58___Connection_reset_by_peer.mdwn
new file mode 100644
index 0000000000..f40b083241
--- /dev/null
+++ b/doc/forum/ssh__95__exchange__95__identification__58___read__58___Connection_reset_by_peer.mdwn
@@ -0,0 +1,28 @@
+I setup a local git annex repo and tried to pair it with an empty repo on my notebook. All done via the webapp.
+
+But pairing failed somehow, because I get this error while transfer seems to go on forever.
+
+    ssh_exchange_identification: read: Connection reset by peer
+    ssh_exchange_identification: read: Connection reset by peer
+    fatal: Could not read from remote repository.
+
+    Please make sure you have the correct access rights
+    and the repository exists.
+
+The webapp shows the remote repo as "unfinished repository".
+
+I can access the other computer via ssh and an other git-annex repo pairing between the two computers work fine.
+
+UPDATE: In the meantime I setup another paired repo on these two computers. Everything works fine. 
+
+I added the repos from scratch again and the same errors occurs again. The repositories which do not pair are the Documents folder on my desktop and on my macbook. Is there something special with macs documents folder?
+
+    [2013-12-31 10:53:23 CET] PushRetrier: Syncing with macbook.local_Documents 
+    fatal: 'macbook.local_Documents' does not appear to be a git repository
+    fatal: Could not read from remote repository.
+
+    Please make sure you have the correct access rights
+    and the repository exists.
+
+
+juh
diff --git a/doc/forum/ssh__95__exchange__95__identification__58___read__58___Connection_reset_by_peer/comment_1_87b9540e37abb16c0ec7605f5ab204a5._comment b/doc/forum/ssh__95__exchange__95__identification__58___read__58___Connection_reset_by_peer/comment_1_87b9540e37abb16c0ec7605f5ab204a5._comment
new file mode 100644
index 0000000000..b44c0c3b53
--- /dev/null
+++ b/doc/forum/ssh__95__exchange__95__identification__58___read__58___Connection_reset_by_peer/comment_1_87b9540e37abb16c0ec7605f5ab204a5._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="Uic1ohv6"
+ ip="62.253.227.100"
+ subject="Are you using encfs?"
+ date="2014-01-09T16:30:50Z"
+ content="""
+I've seen similar error messages when the local annex was on an encfs-mounted decrypted directory. Seemingly git-annex tries to create SSH sockets in .git, and this goes wrong at least in some such situations.
+"""]]
diff --git a/doc/forum/ssh__95__exchange__95__identification__58___read__58___Connection_reset_by_peer/comment_2_bd3383c142bf93d9cd496cb668d7782c._comment b/doc/forum/ssh__95__exchange__95__identification__58___read__58___Connection_reset_by_peer/comment_2_bd3383c142bf93d9cd496cb668d7782c._comment
new file mode 100644
index 0000000000..719b66eb76
--- /dev/null
+++ b/doc/forum/ssh__95__exchange__95__identification__58___read__58___Connection_reset_by_peer/comment_2_bd3383c142bf93d9cd496cb668d7782c._comment
@@ -0,0 +1,12 @@
+[[!comment format=mdwn
+ username="http://joeyh.name/"
+ ip="209.250.56.43"
+ subject="comment 2"
+ date="2014-01-14T21:16:14Z"
+ content="""
+I tried using git-annex on encfs to see if I could reproduce this problem, and I could not get exactly this error, but I certianly did see it cause problems with the ssh connection caching socket.
+
+It would be a good idea to `git config annex.sshcaching false` in repositories stored on encfs. 
+
+Actually, given other reports of problems caused by encfs, it would be a good idea to not use git-annex on encfs. There are better ways to encrypt your files.
+"""]]
diff --git a/doc/forum/ssh__95__exchange__95__identification__58___read__58___Connection_reset_by_peer/comment_3_9b1911ae6468d09dae74ab1a60d2757b._comment b/doc/forum/ssh__95__exchange__95__identification__58___read__58___Connection_reset_by_peer/comment_3_9b1911ae6468d09dae74ab1a60d2757b._comment
new file mode 100644
index 0000000000..59f0f9f96f
--- /dev/null
+++ b/doc/forum/ssh__95__exchange__95__identification__58___read__58___Connection_reset_by_peer/comment_3_9b1911ae6468d09dae74ab1a60d2757b._comment
@@ -0,0 +1,12 @@
+[[!comment format=mdwn
+ username="https://www.google.com/accounts/o8/id?id=AItOawnPgn611P6ym5yyL0BS8rUzO0_ZKRldMt0"
+ nickname="Samuel"
+ subject="What are those ?"
+ date="2014-01-31T07:37:49Z"
+ content="""
+Hi,
+
+Could you please indicate what other means of file encryption systems you have in mind?
+
+The others I know are at a partition level (cryptsetup, LUKS) and are less pratical to setup than a simple encfs.
+"""]]
diff --git a/doc/forum/ssh_key_setup_woes_in_Android.mdwn b/doc/forum/ssh_key_setup_woes_in_Android.mdwn
new file mode 100644
index 0000000000..87e2dd25ff
--- /dev/null
+++ b/doc/forum/ssh_key_setup_woes_in_Android.mdwn
@@ -0,0 +1,15 @@
+I'm trying to set up a ssh remote on Android using the latest 5.20140421. For some
+reason the assistant fails to set up passwordless login on the remote server. It
+adds the required line in `.ssh/authorized_keys` on the server side, but it still
+keeps asking for the password for every connection. Nothing suspicious appears in
+the assistant's log. Also, if I set up a ssh remote on a different directory on the
+same server, a new key is generated and added to `authorized_keys`, but the passwordless
+login still doesn't work.
+
+I didn't file a bug since this would make for a very lousy bug report. How could I
+look more into what's causing this? Where should the generated keys reside on the
+Android filesystem?
+
+I think it would be useful for the assistant to check that the generated ssh keys are
+working properly, and inform the user and/or try to set them up again if there is a
+problem, instead of silently falling back to asking for the login password on the console.
diff --git a/doc/forum/ssh_key_setup_woes_in_Android/comment_1_f16fbff27a449409699f3dbcf9590622._comment b/doc/forum/ssh_key_setup_woes_in_Android/comment_1_f16fbff27a449409699f3dbcf9590622._comment
new file mode 100644
index 0000000000..1b2d9a1859
--- /dev/null
+++ b/doc/forum/ssh_key_setup_woes_in_Android/comment_1_f16fbff27a449409699f3dbcf9590622._comment
@@ -0,0 +1,10 @@
+[[!comment format=mdwn
+ username="http://joeyh.name/"
+ ip="216.145.95.162"
+ subject="comment 1"
+ date="2014-05-19T15:50:34Z"
+ content="""
+There have been recent improvements in the assistant's handling of ssh keys. This includes no more prompting for ssh keys in the console, ever. Instead, the ssh remote setup would presumably fail if the ssh key didn't work for some reason.
+
+To debug your problem, you need to use the shell. I would first try running \"git annex get\" or \"git annex copy\" or \"git annex drop\" on a file, and verify that it prompts for the ssh password. Then, take a look at /sdcard/git-annex.home/.ssh/config, and see if you can ssh to the special hostname set up there, from android, and if it asks for a password. If so, ssh -v might be interesting, as it should show it presenting the ssh key to the server.
+"""]]
diff --git a/doc/forum/ssh_password.mdwn b/doc/forum/ssh_password.mdwn
new file mode 100644
index 0000000000..855e5992f6
--- /dev/null
+++ b/doc/forum/ssh_password.mdwn
@@ -0,0 +1,3 @@
+Currently I have my netbook running git annex and a raspberry pi as a remote storage set up as a git repository.  I currently have it set up so that from the terminal i can just do 'ssh pi@mypi -p XXX -C' and it has the password saved.  Currently I'm having a problem that any time I add a file to my annex folder on my netbook I have to reenter my password for ssh-askpass for it to upload the file.  Once that file is uploaded, it looks to me like it closes the connection and then reopens it and asks for my password again.
+
+I'm looking in ~/.ssh/ and I'm sure that something there needs to be edited or copied, I'm not really sure what though.  Any suggestions?
diff --git a/doc/forum/ssh_password/comment_1_a3e5a41e1d4da683d577976b134b11ee._comment b/doc/forum/ssh_password/comment_1_a3e5a41e1d4da683d577976b134b11ee._comment
new file mode 100644
index 0000000000..7ef9383309
--- /dev/null
+++ b/doc/forum/ssh_password/comment_1_a3e5a41e1d4da683d577976b134b11ee._comment
@@ -0,0 +1,10 @@
+[[!comment format=mdwn
+ username="http://a-or-b.myopenid.com/"
+ ip="203.45.2.230"
+ subject="comment 1"
+ date="2013-05-30T00:08:52Z"
+ content="""
+It sounds like you don't have the ssh-agent set up properly in your environment.
+
+Try starting the git annex agent from the command line and see if that helps.  If it works then it is a problem with environment variables.
+"""]]
diff --git a/doc/forum/ssh_password/comment_2_fa261676a99d49d4b237b0d43048d76d._comment b/doc/forum/ssh_password/comment_2_fa261676a99d49d4b237b0d43048d76d._comment
new file mode 100644
index 0000000000..b7a47c9fe7
--- /dev/null
+++ b/doc/forum/ssh_password/comment_2_fa261676a99d49d4b237b0d43048d76d._comment
@@ -0,0 +1,10 @@
+[[!comment format=mdwn
+ username="http://joeyh.name/"
+ nickname="joey"
+ subject="comment 2"
+ date="2013-05-30T15:49:27Z"
+ content="""
+If you set this repository up using the git-annex webapp, and choosing \"remote ssh server\", it will generate a new ssh key, stored in `~/.ssh/git-annex/`, and it will configure `~/.ssh/config` to use this key for a special made-up hostname which is used in the url for the git remote. On the server, it will set up `~/.ssh/authorized_keys` to allow this ssh key to connect without a password and run git-annex-shell.
+
+I don't know if you used the webapp to set this up. If not, you can try to replicate this setup on your own. If you did and it's not working, you now know where all the peices it sets up are, to figure out what went wrong.
+"""]]
diff --git a/doc/forum/ssh_remote_-_no___34__use_a_git_repository__34___option_during_setup__63__.mdwn b/doc/forum/ssh_remote_-_no___34__use_a_git_repository__34___option_during_setup__63__.mdwn
new file mode 100644
index 0000000000..06b58f9738
--- /dev/null
+++ b/doc/forum/ssh_remote_-_no___34__use_a_git_repository__34___option_during_setup__63__.mdwn
@@ -0,0 +1,3 @@
+Hi,
+
+I own a computer that I use as a home server. I'm just getting started with git-annex, and I was trying to add an ssh remote. It connected fine, but only gave me the "encrypted rsync" option. I'd like to be able to access these files on the server as well, but the "use a git repository" option didn't show up. How do I enable this?
diff --git a/doc/forum/ssh_remote_-_no___34__use_a_git_repository__34___option_during_setup__63__/comment_1_7244794579a191a677190c60758f32e7._comment b/doc/forum/ssh_remote_-_no___34__use_a_git_repository__34___option_during_setup__63__/comment_1_7244794579a191a677190c60758f32e7._comment
new file mode 100644
index 0000000000..14e51e79a1
--- /dev/null
+++ b/doc/forum/ssh_remote_-_no___34__use_a_git_repository__34___option_during_setup__63__/comment_1_7244794579a191a677190c60758f32e7._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="https://www.google.com/accounts/o8/id?id=AItOawmR-xG9O2HWoybxYZPVbYVYxwitfcfDgtE"
+ nickname="Charles"
+ subject="comment 1"
+ date="2013-09-09T00:29:14Z"
+ content="""
+Oh actually, I just clicked the available option and it gave me a \"connection refused\" due to my using a port other than 22. I edited the .ssh/config, but it decided to use 22 anyway?
+"""]]
diff --git a/doc/forum/ssh_remote_-_no___34__use_a_git_repository__34___option_during_setup__63__/comment_2_277cf12907bd7c5930eb4f137b115e29._comment b/doc/forum/ssh_remote_-_no___34__use_a_git_repository__34___option_during_setup__63__/comment_2_277cf12907bd7c5930eb4f137b115e29._comment
new file mode 100644
index 0000000000..55f50b3f55
--- /dev/null
+++ b/doc/forum/ssh_remote_-_no___34__use_a_git_repository__34___option_during_setup__63__/comment_2_277cf12907bd7c5930eb4f137b115e29._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="https://www.google.com/accounts/o8/id?id=AItOawmR-xG9O2HWoybxYZPVbYVYxwitfcfDgtE"
+ nickname="Charles"
+ subject="comment 2"
+ date="2013-09-09T14:50:42Z"
+ content="""
+Never mind, I simply didn't have git-annex installed on the other computer. I'm going to make a thread in bugs about assistant always assuming port 22, you can delete this thread.
+"""]]
diff --git a/doc/forum/ssh_vs_cifs__47__webdav.mdwn b/doc/forum/ssh_vs_cifs__47__webdav.mdwn
new file mode 100644
index 0000000000..62bb9d3d9a
--- /dev/null
+++ b/doc/forum/ssh_vs_cifs__47__webdav.mdwn
@@ -0,0 +1,7 @@
+Hello,
+
+I'm new to git-annex and I would like to know which is the best method to deploy it for my case. I would like to have a central repo in my Synology NAS but with files to be able to browse files (not a bare repo). This repo will not have remotes configured. The mobile clients will use the central repo to exchange files. I thinks that this setup is totally valid x-D. Now my question is more related to how to share the repo in the Synology NAS. I am considering to use cifs/webdav for LAN/WAN access to the repo or SSH for LAN/WAN access. Which is the recommended setup? Which is the most optimum setup in terms of bandwidth or cpu usage (the CPU of my synology NAS is ARM). Thanks in advance and for this great piece of software!
+
+Best regards,
+
+Christian
diff --git a/doc/forum/start_assistant_from_command_line.mdwn b/doc/forum/start_assistant_from_command_line.mdwn
new file mode 100644
index 0000000000..3f48dd6d34
--- /dev/null
+++ b/doc/forum/start_assistant_from_command_line.mdwn
@@ -0,0 +1,11 @@
+I wanted to try out this probably great piece of software you wrote, alas I do not use a Debian distro with managed software menus (i.e. I use OpenBox), so I cannot simply open the assistant through my programs-menu. In fact, to do that I'd need the command to start the assistant so I can add it manually myself. Sadly, the man page did not gave me any advices... :(
+
+Any chance to give me the terminal command to start the assistant?
+
+***
+edit: just to update the thread: everythings okay now. Thanks for this great program!
+
+the commands (as posted by Paul):
+
+    git annex assistant
+    git annex webapp
diff --git a/doc/forum/start_assistant_from_command_line/comment_1_f8dfce1fca9f1212ccaf84e431db71a9._comment b/doc/forum/start_assistant_from_command_line/comment_1_f8dfce1fca9f1212ccaf84e431db71a9._comment
new file mode 100644
index 0000000000..dc35d61bc0
--- /dev/null
+++ b/doc/forum/start_assistant_from_command_line/comment_1_f8dfce1fca9f1212ccaf84e431db71a9._comment
@@ -0,0 +1,12 @@
+[[!comment format=mdwn
+ username="https://www.google.com/accounts/o8/id?id=AItOawmBhQgaA5QrFwT67-Bo0qPIx0HD9roDrso"
+ nickname="Paul"
+ subject="comment 1"
+ date="2013-10-21T19:20:28Z"
+ content="""
+    git annex assitant
+
+starts the assitant itself, you propably want the gui which you start by
+
+   git annex webapp
+"""]]
diff --git a/doc/forum/start_assistant_from_command_line/comment_2_e769c5d09afbff85961363ddc5eb4019._comment b/doc/forum/start_assistant_from_command_line/comment_2_e769c5d09afbff85961363ddc5eb4019._comment
new file mode 100644
index 0000000000..b970990d46
--- /dev/null
+++ b/doc/forum/start_assistant_from_command_line/comment_2_e769c5d09afbff85961363ddc5eb4019._comment
@@ -0,0 +1,10 @@
+[[!comment format=mdwn
+ username="https://www.google.com/accounts/o8/id?id=AItOawkj7tMEJKcZpNXIFkHAAcNi5qJFSFyjn6o"
+ nickname="thissideup"
+ subject="comment 2"
+ date="2013-10-22T11:05:13Z"
+ content="""
+ah okay. That was what I was trying to do, but git-annex failed on my. Remembering that I was on Debian and checking the version (which was June 2012) - I supposed that I wasn't able to use the assistant yet. 
+
+So now I am trying to compile from source using cabal, however I sadly ran into a roadblock: http://git-annex.branchable.com/forum/cabal_install_fails_on_uuid/
+"""]]
diff --git a/doc/forum/store_annex_in_different_place_on_hard_drive.mdwn b/doc/forum/store_annex_in_different_place_on_hard_drive.mdwn
new file mode 100644
index 0000000000..8bc0e86536
--- /dev/null
+++ b/doc/forum/store_annex_in_different_place_on_hard_drive.mdwn
@@ -0,0 +1,2 @@
+I have the following setup on my work computer. The /home folder has limited space, but excellent backups, so I would like to keep my main repository there.
+The /scratch/ folder has no backup, but has faster read/write speed, and more storage space. Is there a way to keep the main repo in /home, but somehow store the large annexed files on /scratch. I am open to making a remote repository, but the standard "git annex get" workflow will simply copy large files into the /home directory, which is what i want to avoid altogther. Can I just make a symlink from "/home//repo/.git/annex" to "/scratch/repodata" somehow? Thanks!
diff --git a/doc/forum/store_annex_in_different_place_on_hard_drive/comment_1_a95add69028ad71f3c66b544c1a28db6._comment b/doc/forum/store_annex_in_different_place_on_hard_drive/comment_1_a95add69028ad71f3c66b544c1a28db6._comment
new file mode 100644
index 0000000000..af79351a36
--- /dev/null
+++ b/doc/forum/store_annex_in_different_place_on_hard_drive/comment_1_a95add69028ad71f3c66b544c1a28db6._comment
@@ -0,0 +1,16 @@
+[[!comment format=mdwn
+ username="joey"
+ subject="""comment 1"""
+ date="2015-10-06T20:17:07Z"
+ content="""
+You should avoid making two repositories that share the same .git/annex
+or .git/annex/objects directories. Doing that can get git-annex confused,
+and let it drop a file that is only located in one place .. because it sees
+it existing in 2 places due to the symlink!
+
+IIRC, git-annex supports making .git/annex a symlink to some directory
+elsewhere, that is not itself used by another git repository.
+
+Or, you can use standard `GIT_DIR` settings to make git 
+(and git-annex) look for the .git directory elsewhere.
+"""]]
diff --git a/doc/forum/switching_backends.mdwn b/doc/forum/switching_backends.mdwn
new file mode 100644
index 0000000000..dedf1ffcba
--- /dev/null
+++ b/doc/forum/switching_backends.mdwn
@@ -0,0 +1,12 @@
+so git annex migrate can switch a file from using one backend to the other.
+
+I've done that with a bunch of files.
+
+The old files should become "unused" right?  But they don't seem to be.  "git annex unused" still shows me only 2 unused files, and I've just migrated dozens of files from SHA256 to SHA256E.  
+
+Is it possible they're still "used" by other repos?  I have two other repos, one reached by SSH and one on a USB drive.  Neither one of them is "bare."  So maybe those files are still used by the "master" branches there, I thought... I went over and did "git annex sync" on each.  Still my newly migrated files are not showing up as "unused."  
+
+I'm worried that my repo is going to bloat with unused files with the SHA256 backend, which mysteriously do not show up as "unused" in git annex unused, if I migrate any more.
+
+any ideas what piece of the puzzle I could be missing here?
+
diff --git a/doc/forum/switching_backends/comment_1_ecf4109c1148dafde3519243ae3c5a03._comment b/doc/forum/switching_backends/comment_1_ecf4109c1148dafde3519243ae3c5a03._comment
new file mode 100644
index 0000000000..29bc9f9c3f
--- /dev/null
+++ b/doc/forum/switching_backends/comment_1_ecf4109c1148dafde3519243ae3c5a03._comment
@@ -0,0 +1,10 @@
+[[!comment format=mdwn
+ username="http://edheil.wordpress.com/"
+ ip="99.54.57.201"
+ subject="comment 1"
+ date="2013-01-28T03:20:38Z"
+ content="""
+Just noticed this -- http://git-annex.branchable.com/bugs/git_annex_migrate_leaves_old_backend_versions_around/
+
+so I don't need to worry about disk space bloat; they're all just hardlinks.  That's cool.  Still would be nice to know why they don't show up as \"unused.\"  But not as important.
+"""]]
diff --git a/doc/forum/switching_backends/comment_2_21f465a18f40b95dafd307fce0de659a._comment b/doc/forum/switching_backends/comment_2_21f465a18f40b95dafd307fce0de659a._comment
new file mode 100644
index 0000000000..82c9f26213
--- /dev/null
+++ b/doc/forum/switching_backends/comment_2_21f465a18f40b95dafd307fce0de659a._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="http://joeyh.name/"
+ ip="2001:4978:f:21a::2"
+ subject="comment 2"
+ date="2013-01-28T06:28:28Z"
+ content="""
+It won't show as unused as long as there are git refs that still refer to the file. That could be branches, or more likely you have not synced everything to the local repo yet.
+"""]]
diff --git a/doc/forum/switching_backends/comment_4_4c13d22c1695195e6b101bd20ef6bb42._comment b/doc/forum/switching_backends/comment_4_4c13d22c1695195e6b101bd20ef6bb42._comment
new file mode 100644
index 0000000000..7ffa519e0c
--- /dev/null
+++ b/doc/forum/switching_backends/comment_4_4c13d22c1695195e6b101bd20ef6bb42._comment
@@ -0,0 +1,35 @@
+[[!comment format=mdwn
+ username="http://edheil.wordpress.com/"
+ ip="173.162.44.162"
+ subject="comment 4"
+ date="2013-01-28T14:30:55Z"
+ content="""
+If by \"synced to the local repo\" you mean run a git annex sync, then yes, I have done that.
+
+Here's an example of what I'm seeing:
+
+    annex$ readlink subdir/subdir/file.txt 
+    ../../.git/annex/objects/zV/57/SHA256E-s3952--cb67d2cbadfea373ed8aa5ec434f246acccb33471a7e28a444138cc4fe8552c2.txt/SHA256E-s3952--cb67d2cbadfea373ed8aa5ec434f246acccb33471a7e28a444138cc4fe8552c2.txt
+    annex$ find .git/annex/objects/ -name SHA256-s3952--cb67d2cbadfea373ed8aa5ec434f246acccb33471a7e28a444138cc4fe8552c2
+    .git/annex/objects//1w/Q4/SHA256-s3952--cb67d2cbadfea373ed8aa5ec434f246acccb33471a7e28a444138cc4fe8552c2
+    .git/annex/objects//1w/Q4/SHA256-s3952--cb67d2cbadfea373ed8aa5ec434f246acccb33471a7e28a444138cc4fe8552c2/SHA256-s3952--cb67d2cbadfea373ed8aa5ec434f246acccb33471a7e28a444138cc4fe8552c2
+    annex$ git annex sync
+    commit  
+    # On branch master
+    nothing to commit, working directory clean
+    ok
+    pull homeworld 
+    ok
+    push homeworld 
+    Everything up-to-date
+    ok
+    annex$ git annex unused
+    unused . (checking for unused data...) (checking master...) (checking sync/master...) (checking homeworld/master...) (checking toshiba/master...) (checking toshiba/synced/master...) ok
+    annex$ find .git/annex/objects/ -name SHA256-s3952--cb67d2cbadfea373ed8aa5ec434f246acccb33471a7e28a444138cc4fe8552c2
+    .git/annex/objects//1w/Q4/SHA256-s3952--cb67d2cbadfea373ed8aa5ec434f246acccb33471a7e28a444138cc4fe8552c2
+    .git/annex/objects//1w/Q4/SHA256-s3952--cb67d2cbadfea373ed8aa5ec434f246acccb33471a7e28a444138cc4fe8552c2/SHA256-s3952--cb67d2cbadfea373ed8aa5ec434f246acccb33471a7e28a444138cc4fe8552c2
+    annex$ 
+
+
+As you can see, there are two remotes, \"toshiba\" and \"homeworld\".  Neither of them is bare, they have checked out \"master\" branches.  Could they be the reason that some of this data isn't \"unused\"?  (I ask because I notice it reports that it is checking homeworld/master and toshiba/master when it does the unused check)
+"""]]
diff --git a/doc/forum/switching_backends/comment_4_e1d4a48baac23fd3f67b20eba4eee8af._comment b/doc/forum/switching_backends/comment_4_e1d4a48baac23fd3f67b20eba4eee8af._comment
new file mode 100644
index 0000000000..d5f4e021ee
--- /dev/null
+++ b/doc/forum/switching_backends/comment_4_e1d4a48baac23fd3f67b20eba4eee8af._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="http://joeyh.name/"
+ ip="4.154.3.125"
+ subject="comment 4"
+ date="2013-02-05T19:16:48Z"
+ content="""
+If you have not yet run `git annex sync` on the remotes, then they are still referring to the old unmigrated keys, which is why `unused` will not remove them.
+"""]]
diff --git a/doc/forum/switching_to__47__from_direct_mode_while_assistant_is_running.mdwn b/doc/forum/switching_to__47__from_direct_mode_while_assistant_is_running.mdwn
new file mode 100644
index 0000000000..f92b5aca35
--- /dev/null
+++ b/doc/forum/switching_to__47__from_direct_mode_while_assistant_is_running.mdwn
@@ -0,0 +1,2 @@
+Is that really unsafe?  Because I did that and the switch to direct mode seemed to leave a lot of files still as links, and when I panicked and switched back to indirect mode, a whole lot of stuff seemed to have become unannexed and I reannexed it.
+
diff --git a/doc/forum/switching_to__47__from_direct_mode_while_assistant_is_running/comment_1_7832243a36613c48d0077b438dbf8b4a._comment b/doc/forum/switching_to__47__from_direct_mode_while_assistant_is_running/comment_1_7832243a36613c48d0077b438dbf8b4a._comment
new file mode 100644
index 0000000000..b27115bbb1
--- /dev/null
+++ b/doc/forum/switching_to__47__from_direct_mode_while_assistant_is_running/comment_1_7832243a36613c48d0077b438dbf8b4a._comment
@@ -0,0 +1,10 @@
+[[!comment format=mdwn
+ username="http://edheil.wordpress.com/"
+ ip="99.54.57.201"
+ subject="comment 1"
+ date="2013-01-25T03:39:16Z"
+ content="""
+This may have been an unrelated issue -- I just realized that I somehow ended up with a bunch of files in the annex under the directory \"misc\" and others under the directory \"Misc\" -- I'm not sure how they got that way, since I'm on OS X and those are supposed to be the same directory -- but they get synced over to a linux box where they're different directories.  This may be the source of some of the weirdness I saw; I'm not sure.  I tried to fix it all up on the linux box so everything was in its own directory.
+
+
+"""]]
diff --git a/doc/forum/symlinks_corrupt__58___link_in_textfile__44___100_bytes_of_size.mdwn b/doc/forum/symlinks_corrupt__58___link_in_textfile__44___100_bytes_of_size.mdwn
new file mode 100644
index 0000000000..d851075873
--- /dev/null
+++ b/doc/forum/symlinks_corrupt__58___link_in_textfile__44___100_bytes_of_size.mdwn
@@ -0,0 +1,7 @@
+I can't find out why, but under certain circumstances the symlinks don't work in my environment. On all hosts I have latest Ubuntu Linux and latest git-annex. V6 repos. Issue is, that instead of real symlinks, the filenames remain the original ones and the content is the relative link to the corresponding annex object.
+
+Strange behavior is, I can drop and get the files but it doesn't seem to affect the situation after all.
+
+Comparing the size of .git/*, it is very likely that the contents are present and so does "git annex whereis bla.jpg". Leading me to the conclusion that everything works except the symlinking. Is this a known issue? What steps can I perform to find out the root cause?
+
+Thank you.
diff --git a/doc/forum/sync_--content__44___fatal_is_outside_repository_errors.mdwn b/doc/forum/sync_--content__44___fatal_is_outside_repository_errors.mdwn
new file mode 100644
index 0000000000..0f31e49171
--- /dev/null
+++ b/doc/forum/sync_--content__44___fatal_is_outside_repository_errors.mdwn
@@ -0,0 +1,50 @@
+I am getting errors when I try to do `sync --content`. I have a git-annex repo on my laptop and I sync all content to an S3 compatible remote (Dreamhost DreamObjects). I ran the following command in my attempt to better understand the issue:
+
+    andrew@bumblebee ~/notes$ git-annex --verbose --debug sync --content cloud &> /tmp/sync_log_2016-11-7-7pm.log
+
+For reference the repos preferred content settings are:
+
+    andrew@bumblebee ~/notes$ git-annex wanted here
+    standard
+    andrew@bumblebee ~/notes$ git-annex group here
+    manual
+    andrew@bumblebee ~/notes$ git-annex wanted cloud
+    standard
+    andrew@bumblebee ~/notes$ git-annex group cloud
+    backup
+    andrew@bumblebee ~/notes$ 
+
+Looking at the log file I see errors like this, repeated dozens of times:
+
+    fatal: '../Users/andrew/notes/cv/submissions/BCA_Submission/small_images/macropavilion_2016_IMG_0297.jpg' is outside repository
+    [2016-11-07 19:34:37.542383] process done ExitFailure 128
+    [2016-11-07 19:34:37.542473] chat: git ["--git-dir=.git","--work-tree=.","--literal-pathspecs","check-attr","-z","--stdin","annex.backend","annex.numcopies","annex.largefiles","--"]
+    fatal: '../Users/andrew/notes/cv/submissions/BCA_Submission/small_images/macropavilion_2016_IMG_0297.jpg' is outside repository
+    [2016-11-07 19:34:37.551241] process done ExitFailure 128
+    [2016-11-07 19:34:37.551382] chat: git ["--git-dir=.git","--work-tree=.","--literal-pathspecs","check-attr","-z","--stdin","annex.backend","annex.numcopies","annex.largefiles","--"]
+
+The relative path `../Users` is curious. `/Users/andrew` is the absolute path of my home directory (I am on OS-X 10.11.6); `/Users/andrew/notes` is the absolute path of my git-annex repo.
+
+the above logs are followed by:
+
+    git-annex: git check-attr EOF: user error
+    failed
+
+    git-annex: fd:24: hFlush: resource vanished (Broken pipe)
+    failed
+
+hFlush lines appears hundreds of times, then change to:
+
+    git-annex: fd:24: commitBuffer: resource vanished (Broken pipe)
+    failed
+
+commitBuffer lines also apear hundreds of times.
+
+Then the logs move onto successful transfers, and similar errors repeat with different file names.
+
+I have tried `git annex fsck` on the local repo, and also tried `git annex fsck --fast --from=cloud` (although I didn't let it finish) and didn't see any errors. Thoughts?
+
+
+
+Thanks,
+Andrew
diff --git a/doc/forum/sync_--content__44___fatal_is_outside_repository_errors/comment_10_d4a649874ee9c9e6a4c0252700607713._comment b/doc/forum/sync_--content__44___fatal_is_outside_repository_errors/comment_10_d4a649874ee9c9e6a4c0252700607713._comment
new file mode 100644
index 0000000000..7e936266bc
--- /dev/null
+++ b/doc/forum/sync_--content__44___fatal_is_outside_repository_errors/comment_10_d4a649874ee9c9e6a4c0252700607713._comment
@@ -0,0 +1,12 @@
+[[!comment format=mdwn
+ username="joey"
+ subject="""comment 9"""
+ date="2018-07-17T19:16:23Z"
+ content="""
+Hmm, maybe.
+
+It's very difficult to say without enough of a transcript to have any idea
+at what point in the `git annex sync` it was failing. If it failed in the
+"commit" part, then yes, it could be git commit running the smudge/clean filter
+and the worktree path problem affecting it.
+"""]]
diff --git a/doc/forum/sync_--content__44___fatal_is_outside_repository_errors/comment_1_668dae37dd46047b46c9e1c23f7bcec4._comment b/doc/forum/sync_--content__44___fatal_is_outside_repository_errors/comment_1_668dae37dd46047b46c9e1c23f7bcec4._comment
new file mode 100644
index 0000000000..4a61d0eee6
--- /dev/null
+++ b/doc/forum/sync_--content__44___fatal_is_outside_repository_errors/comment_1_668dae37dd46047b46c9e1c23f7bcec4._comment
@@ -0,0 +1,11 @@
+[[!comment format=mdwn
+ username="public@881708f3cb16d1f967ca9f517a310dd211474013"
+ nickname="public"
+ avatar="http://cdn.libravatar.org/avatar/e60c5356a8001ac2a78b489d6ac3731c"
+ subject="not specific to cloud repo"
+ date="2016-11-09T21:24:59Z"
+ content="""
+I am getting the same errors when syncing to my USB drive. So it does not appear to have anything to do with syncing to the cloud.
+
+Andrew
+"""]]
diff --git a/doc/forum/sync_--content__44___fatal_is_outside_repository_errors/comment_2_09c62e4abf4ccc0d2e030ef5e1bcdf71._comment b/doc/forum/sync_--content__44___fatal_is_outside_repository_errors/comment_2_09c62e4abf4ccc0d2e030ef5e1bcdf71._comment
new file mode 100644
index 0000000000..fcca1b28e8
--- /dev/null
+++ b/doc/forum/sync_--content__44___fatal_is_outside_repository_errors/comment_2_09c62e4abf4ccc0d2e030ef5e1bcdf71._comment
@@ -0,0 +1,12 @@
+[[!comment format=mdwn
+ username="andrew"
+ avatar="http://cdn.libravatar.org/avatar/acc0ece1eedf07dd9631e7d7d343c435"
+ subject="how to investigate"
+ date="2016-11-16T15:37:01Z"
+ content="""
+Any thoughts? I am unsure how to investigate where this problem is. I assume these files are in my git repo or git-annex objects but I can't seem to find them using any search commands.
+
+Thanks,
+
+Andrew
+"""]]
diff --git a/doc/forum/sync_--content__44___fatal_is_outside_repository_errors/comment_2_8f694afa77f5a835c826d29d46d44615._comment b/doc/forum/sync_--content__44___fatal_is_outside_repository_errors/comment_2_8f694afa77f5a835c826d29d46d44615._comment
new file mode 100644
index 0000000000..8228b6eb04
--- /dev/null
+++ b/doc/forum/sync_--content__44___fatal_is_outside_repository_errors/comment_2_8f694afa77f5a835c826d29d46d44615._comment
@@ -0,0 +1,30 @@
+	[[!comment format=mdwn
+ username="joey"
+ subject="""comment 2"""
+ date="2016-11-16T18:52:07Z"
+ content="""
+It would be helpful if you said what version of git-annex you are using.
+
+And, is your git-annex repository using the new experimental v6 format? One
+user reported a similar error message with a v6 git-annex repository. See
+[[bugs/assistant_crashes_in_TransferScanner]]
+
+Or might your repository be using direct mode?
+
+So, please paste in `git annex version` and `git annex info` output.
+
+It kind of looks like it's having difficulty determining where the top of
+the git repository is, or constructing a relative path to the git
+repository.
+
+Are there any symlinks in the path to /Users/andrew/notes ? Eg, is /Users
+a symlink, or /Users/andrew a symlink, or //Users/andrew/notes itself
+symlinked to elsewhere?
+
+Does only `git annex sync --content` fail? What if you run, eg
+`git annex copy --auto --to cloud` and `git annex get --auto --from cloud`,
+does that fail similarly, or does it succeed?
+
+You say it's only failing for some files. Do the filenames that it's
+failing on contain any non-ascii characters?
+"""]]
diff --git a/doc/forum/sync_--content__44___fatal_is_outside_repository_errors/comment_4_a7f476aeacf88679f25badc78fad886a._comment b/doc/forum/sync_--content__44___fatal_is_outside_repository_errors/comment_4_a7f476aeacf88679f25badc78fad886a._comment
new file mode 100644
index 0000000000..eec45e333e
--- /dev/null
+++ b/doc/forum/sync_--content__44___fatal_is_outside_repository_errors/comment_4_a7f476aeacf88679f25badc78fad886a._comment
@@ -0,0 +1,57 @@
+[[!comment format=mdwn
+ username="andrew"
+ avatar="http://cdn.libravatar.org/avatar/acc0ece1eedf07dd9631e7d7d343c435"
+ subject="git annex copy --auto --to cloud works"
+ date="2016-11-17T17:49:27Z"
+ content="""
+Yes, only `git annex sync --content` seems to fail. I am using v6 with a mix of unlocked and locked files. I did not know about the --auto flags for copy/get.
+
+* `git annex copy --auto --to cloud` works fine
+* `git annex get --auto --from cloud` works fine
+
+
+*Are there any symlinks in the path to /Users/andrew/notes ? Eg, is /Users a symlink, or /Users/andrew a symlink, or //Users/andrew/notes itself symlinked to elsewhere?*
+
+**No**
+
+*You say it's only failing for some files. Do the filenames that it's failing on contain any non-ascii characters?*
+
+**They seem normal.**
+
+*So, please paste in git annex version and git annex info output.*
+
+    git-annex version: 6.20161110-gd48f4ca
+    build flags: Assistant Webapp Pairing Testsuite S3(multipartupload)(storageclasses) WebDAV FsEvents XMPP ConcurrentOutput TorrentParser MagicMime Feeds Quvi
+    key/value backends: SHA256E SHA256 SHA512E SHA512 SHA224E SHA224 SHA384E SHA384 SHA3_256E SHA3_256 SHA3_512E SHA3_512 SHA3_224E SHA3_224 SHA3_384E SHA3_384 SKEIN256E SKEIN256 SKEIN512E SKEIN512 SHA1E SHA1 MD5E MD5 WORM URL
+    remote types: git gcrypt S3 bup directory rsync web bittorrent webdav tahoe glacier ddar hook external
+    local repository version: 6
+    supported repository versions: 3 5 6
+    upgrade supported from repository versions: 0 1 2 3 4 5
+    operating system: darwin x86_64
+
+    repository mode: indirect
+    trusted repositories: 0
+    semitrusted repositories: 10
+    00000000-0000-0000-0000-000000000001 -- web
+    00000000-0000-0000-0000-000000000002 -- bittorrent
+    22de57a0-c9ca-4bfe-8349-3141b3a87c8f -- Dream Objects [cloud]
+    334791ca-c284-4a87-a233-fc29be00d31a -- [disc_May-2-2015_a]
+    4c57ac0e-b8fe-4b4b-98d3-fb0a1b6b9657 -- MacBook Air [here]
+    6a85150d-6ea2-4ba1-92ce-8f4ef575b8e0 -- prowl MacBook Mini
+    896c3d52-427a-41a1-867c-d18e6740d758 -- disc_May_4_2015_1
+    96391b13-3981-430f-ac3b-6210e3d4e759 -- [disc_May-2-2015_b]
+    b4a41e90-2398-4bba-aaf5-d8f8cd78a5bc -- 2TB USB Drive [usbdrive]
+    e42b223d-ec04-4ad8-bdf7-8429a45d844c -- disc_May-2-2015_a
+    untrusted repositories: 0
+    transfers in progress: none
+    available local disk space: 2.32 gigabytes (+1 megabyte reserved)
+    temporary object directory size: 29.47 megabytes (clean up with git-annex unused)
+    local annex keys: 4104
+    local annex size: 10.53 gigabytes
+    annexed files in working tree: 6417
+    size of annexed files in working tree: 80.75 gigabytes
+    bloom filter size: 32 mebibytes (0.8% full)
+    backend usage: 
+    SHA256E: 6417
+
+"""]]
diff --git a/doc/forum/sync_--content__44___fatal_is_outside_repository_errors/comment_5_e96503f38cba755b2e6bd89b1ffab6ff._comment b/doc/forum/sync_--content__44___fatal_is_outside_repository_errors/comment_5_e96503f38cba755b2e6bd89b1ffab6ff._comment
new file mode 100644
index 0000000000..17fa7b04b3
--- /dev/null
+++ b/doc/forum/sync_--content__44___fatal_is_outside_repository_errors/comment_5_e96503f38cba755b2e6bd89b1ffab6ff._comment
@@ -0,0 +1,28 @@
+[[!comment format=mdwn
+ username="andrew"
+ avatar="http://cdn.libravatar.org/avatar/acc0ece1eedf07dd9631e7d7d343c435"
+ subject="updates sync vs copy?"
+ date="2017-01-21T14:56:26Z"
+ content="""
+Any thoughts Joey? Reminder I have a v6 repo, that spits out errors `git-annex: fd:24: hFlush: resource vanished (Broken pipe)` when using sync --content.
+
+I am not sure why `git annex sync --content cloud` should spew errors, and `git annex copy --auto --to cloud` runs without issue. Anything I should do on my end to investigate this issue further?
+
+Thanks,
+
+Andrew
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+"""]]
diff --git a/doc/forum/sync_--content__44___fatal_is_outside_repository_errors/comment_6_7443fe5f7384431914c714c2b462cf5c._comment b/doc/forum/sync_--content__44___fatal_is_outside_repository_errors/comment_6_7443fe5f7384431914c714c2b462cf5c._comment
new file mode 100644
index 0000000000..c8b0e3356b
--- /dev/null
+++ b/doc/forum/sync_--content__44___fatal_is_outside_repository_errors/comment_6_7443fe5f7384431914c714c2b462cf5c._comment
@@ -0,0 +1,24 @@
+[[!comment format=mdwn
+ username="andrew"
+ avatar="http://cdn.libravatar.org/avatar/acc0ece1eedf07dd9631e7d7d343c435"
+ subject="RESOLVED / deleted all files then re-added as zip"
+ date="2017-01-23T02:28:23Z"
+ content="""
+OK. I ended up having to delete all the files from git, then re-add them zipped up.
+
+I re-read the post you mentioned, this helped me resolve my issue:
+[assistant crashes in TransferScanner](http://git-annex.branchable.com/bugs/assistant_crashes_in_TransferScanner/) (from johannes)
+
+Johannes dropped the file, then re-added, that fixed his issue. So I tried that first.
+
+I did the following:
+
+ * git-annex dropped all problem files. this didn't fix issue
+ * deleted all problem files (using OS-X Finder), committed the deletion with git (git add -A), issue then fixed
+    * re-added deleted files, issue came back
+    * re-added deleted files in different folder, issue came back
+    * re-added deleted files all with different filenames in different path, issue came back
+    * zipped up folder of problem files, then added to git-annex, no issues
+
+
+"""]]
diff --git a/doc/forum/sync_--content__44___fatal_is_outside_repository_errors/comment_7_63225cc9557886ae12b3450f68812815._comment b/doc/forum/sync_--content__44___fatal_is_outside_repository_errors/comment_7_63225cc9557886ae12b3450f68812815._comment
new file mode 100644
index 0000000000..fd1c563534
--- /dev/null
+++ b/doc/forum/sync_--content__44___fatal_is_outside_repository_errors/comment_7_63225cc9557886ae12b3450f68812815._comment
@@ -0,0 +1,7 @@
+[[!comment format=mdwn
+ username="joey"
+ subject="""comment 7"""
+ date="2017-02-20T16:54:19Z"
+ content="""
+Can you give me some examples of the names of the "problem files"?
+"""]]
diff --git a/doc/forum/sync_--content__44___fatal_is_outside_repository_errors/comment_8_baefe35266021184ee78440226fe2a8d._comment b/doc/forum/sync_--content__44___fatal_is_outside_repository_errors/comment_8_baefe35266021184ee78440226fe2a8d._comment
new file mode 100644
index 0000000000..df94899406
--- /dev/null
+++ b/doc/forum/sync_--content__44___fatal_is_outside_repository_errors/comment_8_baefe35266021184ee78440226fe2a8d._comment
@@ -0,0 +1,25 @@
+[[!comment format=mdwn
+ username="andrew"
+ avatar="http://cdn.libravatar.org/avatar/acc0ece1eedf07dd9631e7d7d343c435"
+ subject="the problem files"
+ date="2017-02-20T18:11:03Z"
+ content="""
+Joey,
+
+the file names seem quite ordinary:
+
+    cv/submissions/BCA_Submission/small_images/drawblocks_2015_IMG_1719.jpg
+    cv/submissions/BCA_Submission/small_images/drawblocks_2015_IMG_3848.jpg
+    cv/submissions/BCA_Submission/small_images/macropavilion_2016_IMG_0391.jpg
+    cv/submissions/BCA_Submission/small_images/sequencing_2016_DSC5048.jpg
+    cv/submissions/BCA_Submission/small_images/sequencing_2016_IMG_8231.jpg
+
+Possibly this is related to an issue I had with v6 and `annex.largefiles` about 12-months ago. I had done `git config annex.largefiles 'largerthan=100kb and not (include=*.c or include=*.h)'`. But I believe this resulted in git-annex thinking locked files with no content present should be added to git instead of annex? After doing a `git-annex add .` I now had a bunch of files whose content was lost. Or perhaps I did a sync with a v5 repo, or perhaps I did a sync with a repo with a different large files settings, I can't remember. Anyhow, I managed to lose links to the file content and `git-annex get` or fsck couldn't retrieve them. I never filed a bug report because I was never able to reproduce the issue on a clean repo. 
+
+Anyhow, perhaps spelunking into the git log would give some answers?
+
+
+
+Andrew
+
+"""]]
diff --git a/doc/forum/sync_--content__44___fatal_is_outside_repository_errors/comment_9_a9ed50cd4a1f436822ae97bfcd1ced48._comment b/doc/forum/sync_--content__44___fatal_is_outside_repository_errors/comment_9_a9ed50cd4a1f436822ae97bfcd1ced48._comment
new file mode 100644
index 0000000000..1df369d84e
--- /dev/null
+++ b/doc/forum/sync_--content__44___fatal_is_outside_repository_errors/comment_9_a9ed50cd4a1f436822ae97bfcd1ced48._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="andrew"
+ avatar="http://cdn.libravatar.org/avatar/acc0ece1eedf07dd9631e7d7d343c435"
+ subject="comment 9"
+ date="2018-07-17T18:58:58Z"
+ content="""
+Most likely related to and fixed by now fixed bug: 
+"""]]
diff --git a/doc/forum/sync_between_indirect_and_direct_mode.mdwn b/doc/forum/sync_between_indirect_and_direct_mode.mdwn
new file mode 100644
index 0000000000..66833cbe51
--- /dev/null
+++ b/doc/forum/sync_between_indirect_and_direct_mode.mdwn
@@ -0,0 +1,6 @@
+I have a music repository(direct mode) in my Nexus 5 which I want to sync with remote repository(indirect mode).
+
+When I run 'git annex sync --content', it did not sync the content but when I changed remote repository to direct mode, content got synced.
+
+Do I need to set some configuration ?
+Is it possible to sync content between direct and indirect mode repositories ?
diff --git a/doc/forum/sync_between_indirect_and_direct_mode/comment_1_7efc0d79196675582571c05fdd133b53._comment b/doc/forum/sync_between_indirect_and_direct_mode/comment_1_7efc0d79196675582571c05fdd133b53._comment
new file mode 100644
index 0000000000..7cbcd3c39b
--- /dev/null
+++ b/doc/forum/sync_between_indirect_and_direct_mode/comment_1_7efc0d79196675582571c05fdd133b53._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="http://joeyh.name/"
+ ip="108.236.230.124"
+ subject="comment 1"
+ date="2014-03-10T17:57:51Z"
+ content="""
+There is no difference between direct and indirect mode repositories when it comes to communication between repositories, so they can be used in any combination.
+"""]]
diff --git a/doc/forum/sync_between_indirect_and_direct_mode/comment_2_8ac84dbaf7a4d503497487cbdb1749d8._comment b/doc/forum/sync_between_indirect_and_direct_mode/comment_2_8ac84dbaf7a4d503497487cbdb1749d8._comment
new file mode 100644
index 0000000000..42b5126271
--- /dev/null
+++ b/doc/forum/sync_between_indirect_and_direct_mode/comment_2_8ac84dbaf7a4d503497487cbdb1749d8._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="https://www.google.com/accounts/o8/id?id=AItOawnDXvDvWTXmCqQ90ATTD4dV3Ii4YbnE8E0"
+ nickname="sys"
+ subject="git annex direct when not all content is in this repo"
+ date="2014-03-12T14:41:53Z"
+ content="""
+what does git annex direct do when all the content isn't available in the current repo (i am assuming it will leave symlinks for the missing content)
+"""]]
diff --git a/doc/forum/sync_between_indirect_and_direct_mode/comment_3_9acb237711669ec6046a8d07f9ed3b2c._comment b/doc/forum/sync_between_indirect_and_direct_mode/comment_3_9acb237711669ec6046a8d07f9ed3b2c._comment
new file mode 100644
index 0000000000..f743487b2b
--- /dev/null
+++ b/doc/forum/sync_between_indirect_and_direct_mode/comment_3_9acb237711669ec6046a8d07f9ed3b2c._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="http://joeyh.name/"
+ ip="209.250.56.154"
+ subject="comment 3"
+ date="2014-03-12T17:03:30Z"
+ content="""
+Direct mode does not require all content be present any more than indirect mode does, and missing content is represented the same in both modes, with broken symlinks.
+"""]]
diff --git a/doc/forum/sync_quits_after_first_non-successfull_remote.mdwn b/doc/forum/sync_quits_after_first_non-successfull_remote.mdwn
new file mode 100644
index 0000000000..8caee35c26
--- /dev/null
+++ b/doc/forum/sync_quits_after_first_non-successfull_remote.mdwn
@@ -0,0 +1,27 @@
+Hello,
+
+I have observed what I think is a new behavior:
+
+
+    % git annex add . && git annex sync --content
+    commit
+    Auf Branch master
+    nichts zu committen, Arbeitsverzeichnis unverändert
+    ok
+    pull xgm
+    ok
+    pull horus
+    ssh: Could not resolve hostname horus.local: Name or service not known
+    fatal: Konnte nicht vom Remote-Repository lesen.
+    
+    Bitte stellen Sie sicher, dass die korrekten Zugriffsberechtigungen bestehen
+    und das Repository existiert.
+    failed
+    copy Backgrounds/Flo Wallpaper/Cirrus_front_over_Austnesfjorden,_Austvågøya,_Lofoten,_Norway,_2015_April.jpg git-annex: unable to check horus
+
+git annex sync quits with exist code 1. What I rather expect is that it tries to sync --content the other repositories. If I call `git annex sync --content S3` it sync to S3 just fine.
+
+S3 is backup/client, local repo is manual/client.
+
+Thanks,
+Florian
diff --git a/doc/forum/sync_quits_after_first_non-successfull_remote/comment_1_dbca6ea763702a82ae7563722cdc7fab._comment b/doc/forum/sync_quits_after_first_non-successfull_remote/comment_1_dbca6ea763702a82ae7563722cdc7fab._comment
new file mode 100644
index 0000000000..e890b13b14
--- /dev/null
+++ b/doc/forum/sync_quits_after_first_non-successfull_remote/comment_1_dbca6ea763702a82ae7563722cdc7fab._comment
@@ -0,0 +1,11 @@
+[[!comment format=mdwn
+ username="joey"
+ subject="""comment 1"""
+ date="2018-07-03T17:00:26Z"
+ content="""
+Thanks for reporting, I've reproduced and fixed the bug.
+
+Please file bugs in the bug tracker in the future,
+that will prevent me missing them or responding as slowly as I did with
+this one.
+"""]]
diff --git a/doc/forum/sync_remote_repo_on_local_sync_upstream.mdwn b/doc/forum/sync_remote_repo_on_local_sync_upstream.mdwn
new file mode 100644
index 0000000000..c2b566cc5d
--- /dev/null
+++ b/doc/forum/sync_remote_repo_on_local_sync_upstream.mdwn
@@ -0,0 +1,21 @@
+I am looking for a short cut for my workflow where I sync a (somewhat) central repo and a few client repos.
+
+So, I sync upstream 
+
+    laptop > git annex sync --content
+
+and ssh to the central repo and run another sync
+
+    server > git annex sync
+
+to bring it up to date so that I can sync/pull it again from my desktop.
+
+Is there an easy way to script/do both steps in one for different protocols? E.g., update a ssh repo and a USB-drive repo when syncing on the local one?
+
+At the moment, I would try to check for all known remotes
+
+    > git remote -v
+
+and depending on the protocol ssh/cd into each for triggering a sync.
+
+
diff --git a/doc/forum/sync_remote_repo_on_local_sync_upstream/comment_1_db650b4b5764e33219cb28eba2987ea5._comment b/doc/forum/sync_remote_repo_on_local_sync_upstream/comment_1_db650b4b5764e33219cb28eba2987ea5._comment
new file mode 100644
index 0000000000..0e2bb28e37
--- /dev/null
+++ b/doc/forum/sync_remote_repo_on_local_sync_upstream/comment_1_db650b4b5764e33219cb28eba2987ea5._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="joey"
+ subject="""comment 1"""
+ date="2017-09-29T17:44:42Z"
+ content="""
+See
+
+"""]]
diff --git a/doc/forum/sync_stages_deletions_on_remote.mdwn b/doc/forum/sync_stages_deletions_on_remote.mdwn
new file mode 100644
index 0000000000..73a51d0b0a
--- /dev/null
+++ b/doc/forum/sync_stages_deletions_on_remote.mdwn
@@ -0,0 +1,72 @@
+I'm having an issue with 2 repos: one on my laptop, the other on my NAS. Both are in indirect mode, running Arch Linux, and have the latest Git version. Laptop has git-annex 5.20140411-gda795e0, NAS has 5.20140319-g9aa31b7 (from prebuilt tarballs).
+
+The issue is quite simple. When I `git-annex add` new files on my laptop, commit them, and then `git-annex sync` them, they show up as staged for deletion on my NAS.
+
+    laptop $ git annex add some-file
+    laptop $ git commit -m "Add some-file"
+    laptop $ git annex sync
+    commit  ok
+    pull ds413j 
+    ok
+    push ds413j 
+    Counting objects: 133, done.
+    Delta compression using up to 8 threads.
+    Compressing objects: 100% (78/78), done.
+    Writing objects: 100% (80/80), 10.64 KiB | 0 bytes/s, done.
+    Total 80 (delta 12), reused 0 (delta 0)
+    To ssh://**/**
+       1dcd188..8ef4249  git-annex -> synced/git-annex
+       c0f45a6..21711d6  master -> synced/master
+    ok
+    laptop $ ssh $NAS
+    nas $ git status
+    On branch master
+    Changes to be committed:
+      (use "git reset HEAD ..." to unstage)
+
+            deleted:    some-file
+
+    nas $
+
+If I run `git annex sync` on the NAS, it will create a new commit that deletes that file. So I have to play with `git reset`/`git checkout` by hand to make sure that the new file won't be deleted.
+
+I'm not sure when this started, but I think it was after I did some stupid mistake (`git checkout -B master synced/master`, kill a `git annex sync` with Ctrl+C, or something else that even resulted in my non-bare repo to have "bare=true" in .git/config...). And I haven't yet been able to fix this.
+
+Any idea what can have caused this, how to fix it, and how to prevent it from happening again in the future?
+
+.git/config on NAS:
+
+    [core]
+            repositoryformatversion = 0
+            filemode = true
+            logallrefupdates = true
+    [annex]
+            uuid = d54ae60a-1f59-403c-923f-32ea3bf2d00f
+            version = 5
+            diskreserve = 1 megabyte
+            autoupgrade = ask
+            debug = false
+
+.git/config on laptop:
+
+    [core]
+            repositoryformatversion = 0
+            filemode = true
+            bare = false
+            logallrefupdates = true
+    [branch "master"]
+    [annex]
+            uuid = f20cb506-945d-4c78-af1a-0aa884bb899b
+            version = 5
+            diskreserve = 20 gigabytes
+            autoupgrade = ask
+            debug = false
+            expireunused = 7d
+            genmetadata = true
+    [push]
+            default = matching
+    [remote "ds413j"]
+            url = ssh://**/**
+            fetch = +refs/heads/*:refs/remotes/ds413j/*
+            annex-uuid = d54ae60a-1f59-403c-923f-32ea3bf2d00f
+            annex-sync = true
diff --git a/doc/forum/sync_stages_deletions_on_remote/comment_1_2b639066095e450c2d9be3b2775d24b3._comment b/doc/forum/sync_stages_deletions_on_remote/comment_1_2b639066095e450c2d9be3b2775d24b3._comment
new file mode 100644
index 0000000000..24495fd024
--- /dev/null
+++ b/doc/forum/sync_stages_deletions_on_remote/comment_1_2b639066095e450c2d9be3b2775d24b3._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="http://schnouki.net/"
+ nickname="Schnouki"
+ subject="comment 1"
+ date="2014-04-17T12:16:41Z"
+ content="""
+Here's the output of `git annex sync --debug` (for a different commit): 
+"""]]
diff --git a/doc/forum/sync_stages_deletions_on_remote/comment_2_da5775526a2a476b6ead1cd1a735b8bd._comment b/doc/forum/sync_stages_deletions_on_remote/comment_2_da5775526a2a476b6ead1cd1a735b8bd._comment
new file mode 100644
index 0000000000..a3076faf7c
--- /dev/null
+++ b/doc/forum/sync_stages_deletions_on_remote/comment_2_da5775526a2a476b6ead1cd1a735b8bd._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="http://joeyh.name/"
+ ip="209.250.56.191"
+ subject="comment 2"
+ date="2014-04-17T19:28:59Z"
+ content="""
+`git annex sync` will push changes to your NAS, but this does not cause the work tree there to be updated. You must be running some command on the NAS that gets it work tree into the state you show. You need to tell us what that command is, since it seems to be where things are going wrong. Maybe you're running the git-annex assistant on the NAS, or maybe you run `git annex sync` on the NAS. 
+"""]]
diff --git a/doc/forum/sync_stages_deletions_on_remote/comment_3_9e07593228915936fadcf90373be9f4e._comment b/doc/forum/sync_stages_deletions_on_remote/comment_3_9e07593228915936fadcf90373be9f4e._comment
new file mode 100644
index 0000000000..fb9020252d
--- /dev/null
+++ b/doc/forum/sync_stages_deletions_on_remote/comment_3_9e07593228915936fadcf90373be9f4e._comment
@@ -0,0 +1,14 @@
+[[!comment format=mdwn
+ username="http://schnouki.net/"
+ nickname="Schnouki"
+ subject="comment 3"
+ date="2014-04-22T14:59:47Z"
+ content="""
+Not running the assistant (freshly rebooted NAS, checked with `ps ax | grep annex`), not running any other git command. Still happened, until a few minutes ago: I found the cause for this issue... and it's quite frustrating.
+
+Every now and then, I interrupt a running `git annex sync` with Ctrl+C. And sometimes this causes my NAS repository to end up with `bare = true` in its `.git/config`. When this happens, I just remove the offending line and call it a day. Now I just added a `bare = false` line to that file, and that solved the problem. It seems that somehow, if you don't explicitely tell git that your repo is not bare, it considers it as a bare repository -- and the final push from `git annex sync` actually updates the `master` branch without updating the work tree.
+
+This is probably not a bug in git-annex, but rather a weird behaviour in git -- and for sure something I didn't expect to happend. But anyway it works now :) Leaving this here in case it helps other people.
+
+Joey, thanks again for your time and your help.
+"""]]
diff --git a/doc/forum/sync_stages_deletions_on_remote/comment_4_e5a3dc34c6229ec40bc999c3cab28041._comment b/doc/forum/sync_stages_deletions_on_remote/comment_4_e5a3dc34c6229ec40bc999c3cab28041._comment
new file mode 100644
index 0000000000..710392daa4
--- /dev/null
+++ b/doc/forum/sync_stages_deletions_on_remote/comment_4_e5a3dc34c6229ec40bc999c3cab28041._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="http://joeyh.name/"
+ ip="209.250.56.114"
+ subject="comment 4"
+ date="2014-04-24T18:22:16Z"
+ content="""
+git sets bare = false in newly created non-bare repositories. Perhaps this is why? It seems likely that git has an inconsistent default if there's no configuration. Sounds worth filing a bug on git for. OTOH, messing with .git/config  in nonstandard ways seems like a good way to cause yourself random pain too.
+"""]]
diff --git a/doc/forum/sync_stages_deletions_on_remote/comment_5_f3350d336c6c66c3aacc7caade2ef12c._comment b/doc/forum/sync_stages_deletions_on_remote/comment_5_f3350d336c6c66c3aacc7caade2ef12c._comment
new file mode 100644
index 0000000000..55f05331a0
--- /dev/null
+++ b/doc/forum/sync_stages_deletions_on_remote/comment_5_f3350d336c6c66c3aacc7caade2ef12c._comment
@@ -0,0 +1,31 @@
+[[!comment format=mdwn
+ username="edward"
+ subject="Same problem"
+ date="2015-08-27T09:24:38Z"
+ content="""
+I think I'm having the same problem. See my comments on [[bugs/git annex sync deleted a bunch of files (not expected)]]
+
+I've run `git annex sync` or `git annex webapp` on the laptop annex, then `git annex sync` on the external drive. I'm pretty sure some of the syncs have been interrupted. Does it help to see the .git/config from the external drive?
+
+    [core]
+    	repositoryformatversion = 0
+    	filemode = true
+    	bare = false
+    	logallrefupdates = true
+    [remote \"origin\"]
+    	url = /home/edward/annex
+    	fetch = +refs/heads/*:refs/remotes/origin/*
+    	annex-uuid = 38d109c9-7f5f-47cd-b15a-7b2beac22c64
+    [branch \"master\"]
+    	remote = origin
+    	merge = refs/heads/master
+    [annex]
+    	uuid = 822dec0f-a0d3-42f6-b0dc-a47b6bf91944
+    	version = 5
+    [remote \"x230\"]
+    	url = /home/edward/annex
+    	fetch = +refs/heads/*:refs/remotes/x230/*
+    	annex-uuid = 38d109c9-7f5f-47cd-b15a-7b2beac22c64
+
+Observations about my config, I have `bare = false`, which is correct. Do you think it is a problem that I have two remotes, `\"origin\"` and `\"x230\"` pointing at the same location?
+"""]]
diff --git a/doc/forum/syncing_home_directories.mdwn b/doc/forum/syncing_home_directories.mdwn
new file mode 100644
index 0000000000..fcd3917325
--- /dev/null
+++ b/doc/forum/syncing_home_directories.mdwn
@@ -0,0 +1,7 @@
+Hi,
+
+I synchronize home directories on a couple of machines using unison.  Is there a recommended way to do the equivalent using git-annex? 
+
+Thanks,
+
+John
diff --git a/doc/forum/syncing_home_directories/comment_1_220a6e0ffe0ea610921a63c0a6e3beab._comment b/doc/forum/syncing_home_directories/comment_1_220a6e0ffe0ea610921a63c0a6e3beab._comment
new file mode 100644
index 0000000000..f4699e74a1
--- /dev/null
+++ b/doc/forum/syncing_home_directories/comment_1_220a6e0ffe0ea610921a63c0a6e3beab._comment
@@ -0,0 +1,16 @@
+[[!comment format=mdwn
+ username="http://joeyh.name/"
+ nickname="joey"
+ subject="comment 1"
+ date="2013-03-05T20:34:22Z"
+ content="""
+There are a few problems you are likely to run into if you try to store everything in git-annex:
+
+* git does not store certain things, including special file permissions, in a git repository
+* you can't check a .git directory into another git repository
+* git-annex locks files to prevent them being changed and this involves replacing them with symlinks (much less of a problem now that it has the direct mode)
+
+For these reasons, I currently encourage users to use one or more git-annex repositories to store and sync whatever large data files they need, but to confine this to subdirectories of the home directory. There are other tools like vcsh that are better suited to storing things like dotfiles in git, rather than using git-annex for that. And other tools like mr can make it easy to work with a set of repositories, so you can have your dotfiles stored in one or more repositories and your large files in others and update them all with one command. 
+
+See  for more on this..
+"""]]
diff --git a/doc/forum/syncing_music_to_my_android_device.mdwn b/doc/forum/syncing_music_to_my_android_device.mdwn
new file mode 100644
index 0000000000..ba4b0f80a3
--- /dev/null
+++ b/doc/forum/syncing_music_to_my_android_device.mdwn
@@ -0,0 +1,17 @@
+Hi!
+
+So I am not sure how to deal with this issue right now. I have described in [[bugs/direct_cripple_mode_crippled_my_other_non-crippled_repos/]] how it takes a long time to replicate my `mp3` repository to another, crippled, filesystem (namely, FAT32). I am not sure what is going on there, but is seems difficult to setup an external device with only a subset of my music *and* keeping a proper directory structure in place.
+
+When I `git clone` the git-annex repository onto the device, it fails because it (apparently?) transfers all the files and runs out of space. I have also tried to `git init; git annex init; git remote add [...] ; git annex sync` with similar failure modes. All those tests take a long time and I would prefer avoiding to have to reproduce those again, but I will if necessary. :) There is over 20 000 files in the git-annex repository, and about 115GB of data in there. `.git` is about 130MB on a fresh clone on a non-crippled filesystem.
+
+Basically, my workaround so far has been to use a bare repository on the device: it works fairly well! I can transfer files to it with `git annex copy --to` and the clone is actually much faster. 
+
+This was working well on my Nokia N900 device, as the music player was fairly smart and could figure out that similar album tags belonged together. It did have trouble finding all the files (as it does a inotify on all the directories it finds, which obviously runs out of ram on that bit git-annex), but after a few restarts it worked.
+
+On this new Android 5.1 device (Cyanogenmod 12.1, to be more accurate), things don't go so well. The music player finds all the files much faster, but unfortunately, each song is put in a separate album, because they are all in different directories (because this is a bare repository).
+
+This is an issue that a [[todo/dumb__44___unsafe__44___human-readable_backend/]] would solve, but since that approach doesn't seem feasible right now, I am wondering how I can manage to deploy that repository more reliably.
+
+This is also similar to [[forum/usability__58___creating_an_archive_on_a_new_external_drive/]].
+
+Thanks for any advice! -- [[anarcat]]
diff --git a/doc/forum/syncing_music_to_my_android_device/comment_1_81653a23424ac04675651fb7632c57ae._comment b/doc/forum/syncing_music_to_my_android_device/comment_1_81653a23424ac04675651fb7632c57ae._comment
new file mode 100644
index 0000000000..ad30d0c41a
--- /dev/null
+++ b/doc/forum/syncing_music_to_my_android_device/comment_1_81653a23424ac04675651fb7632c57ae._comment
@@ -0,0 +1,11 @@
+[[!comment format=mdwn
+ username="anarcat"
+ subject="special remote"
+ date="2016-06-23T13:03:48Z"
+ content="""
+I ended up writing a special remote for this, named [git-annex-remote-dumb](http://src.anarc.at/scripts.git/blob_plain/HEAD:/git-annex-remote-dumb) for lack of a better name. it mixes things up badly, is a abhorrent violation of protocol boundaries and is probably due to be taken to the back of the barn and shot, but it works for my case.
+
+this would be better implemented as [[todo/dumb, unsafe, human-readable_backend]], but I suspect this cannot be directly implemented either without significant changes of the git-annex design, something which is unlikely to happen since [[todo/hide missing files]] is likely to be implemented with [[design/adjusted branches]].
+
+more details about known problems, remaining tasks and limitations in the script header.
+"""]]
diff --git a/doc/forum/syncing_music_to_my_android_device/comment_2_e49c9e5da5c5d1db228b9b753bed53ff._comment b/doc/forum/syncing_music_to_my_android_device/comment_2_e49c9e5da5c5d1db228b9b753bed53ff._comment
new file mode 100644
index 0000000000..c14c9e9231
--- /dev/null
+++ b/doc/forum/syncing_music_to_my_android_device/comment_2_e49c9e5da5c5d1db228b9b753bed53ff._comment
@@ -0,0 +1,18 @@
+[[!comment format=mdwn
+ username="joey"
+ subject="""comment 2"""
+ date="2017-09-07T20:15:10Z"
+ content="""
+The new `git annex export` command can be used for this. It exports the
+current tree of files to a special remote. If your Android device can be
+mounted to the filesystem, you can initremote a directory special remote on
+it, and then export to it.
+
+It would also be possible to make a special remote that uses `adb` or some
+other method to manipulate the files on the Android device, rather than
+going via the directory special remote.
+
+The main downside is, if you modify files on the Android device, or add new
+files, there's no way to commit the changes from there back to
+your git repository.
+"""]]
diff --git a/doc/forum/syncing_non-git_trees_with_git-annex.mdwn b/doc/forum/syncing_non-git_trees_with_git-annex.mdwn
new file mode 100644
index 0000000000..9973782610
--- /dev/null
+++ b/doc/forum/syncing_non-git_trees_with_git-annex.mdwn
@@ -0,0 +1,46 @@
+I have a bunch of directory trees with large data files scattered over various computers and disk drives - they contain photos, videos, music, and so on.  In many cases I initially copied one of these trees from one machine to another just as a cheap and dirty backup, and then made small modifications to both trees in ways I no longer remember. For example, I returned from a trip with a bunch of new photos, and then might have rotated some of them 90 degrees on one machine, and edited or renamed them on another.
+
+What I want to do now is use git-annex as a way of initially synchronising the trees, and then fully managing them on an ongoing basis.  Note that the trees are *not* yet git repositories.  In order to be able to detect straight-forward file renames, I believe that [[the SHA1 backend|tips/using_the_SHA1_backend]] probably makes the most sense.
+
+I've been playing around and arrived at the following setup procedure.  For the sake of discussion, I assume that we have two trees `a` and `b` which live in the same directory referred to by `$td`, and that all large files end with the `.avi` suffix.
+
+     # Setup git in 'a'.
+     cd $td/a
+     git init
+
+     # Setup git-annex in 'a'.
+     echo '* annex.backend=SHA1' > .gitattributes
+     git add .gitattributes
+     git commit -m'use SHA1 backend'
+     git annex init
+
+     # Annex all large files.
+     find -name \*.avi | xargs git annex add
+     git add .
+     git commit -m'Initial import'
+
+     # Setup git in 'b'.
+     cd $td/b
+     git clone -n $td/a new
+     mv new/.git .
+     rmdir new
+     git reset # reset git index to b's wd - hangover from cloning from 'a'
+
+     # Setup git-annex in 'b'.
+     # This merges a's (origin's) git-annex branch into the local git-annex branch.
+     git annex init
+
+     # Annex all large files - because we're using SHA1 backend, some
+     # should hash to the same keys as in 'a'.
+     find -name \*.avi | xargs git annex add
+     git add .
+     git commit -m'Changes in b tree'
+
+     git remote add a $td/a
+
+     # Now pull changes in 'b' back to 'a'.
+     cd $td/a
+     git remote add b $td/b
+     git pull b master
+
+This seems to work, but have I missed anything?
diff --git a/doc/forum/syncing_non-git_trees_with_git-annex/comment_1_7f9593bdfd95e4a8814e6cc5c44619e6._comment b/doc/forum/syncing_non-git_trees_with_git-annex/comment_1_7f9593bdfd95e4a8814e6cc5c44619e6._comment
new file mode 100644
index 0000000000..bdec508792
--- /dev/null
+++ b/doc/forum/syncing_non-git_trees_with_git-annex/comment_1_7f9593bdfd95e4a8814e6cc5c44619e6._comment
@@ -0,0 +1,24 @@
+[[!comment format=mdwn
+ username="http://joey.kitenet.net/"
+ nickname="joey"
+ subject="comment 1"
+ date="2011-12-14T17:31:31Z"
+ content="""
+This is an entirely reasonable way to go about it.
+
+However, doing it this way causes files in B to always \"win\" -- If the same filename is in both repositories, with differing content, the version added in B will superscede the version from A. If A has a file that is not in B, a git commit -a in B will commit a deletion of that file.
+
+I might do it your way and look at the changes in B before (or even after) committing them to see if files from A were deleted or changed.
+
+Or, I might just instead keep B in a separate subdirectory in the repository, set up like so:
+
+
    +mv b old_b
+git clone a b
+cd b
+mv ../old_b .
+git annex add old_b --not --exclude '*.avi'
+
    
+
+Or, a third way would be to commit A to a branch like branchA and B to a separate branchB, and not merge the branches at all.
+"""]]
diff --git a/doc/forum/syncing_non-git_trees_with_git-annex/comment_2_49f15478781a0ad5e46e75319070335c._comment b/doc/forum/syncing_non-git_trees_with_git-annex/comment_2_49f15478781a0ad5e46e75319070335c._comment
new file mode 100644
index 0000000000..94b5c2ec11
--- /dev/null
+++ b/doc/forum/syncing_non-git_trees_with_git-annex/comment_2_49f15478781a0ad5e46e75319070335c._comment
@@ -0,0 +1,16 @@
+[[!comment format=mdwn
+ username="https://www.google.com/accounts/o8/id?id=AItOawmKPMUX0YHBjE93eBsEnacwZsddSDue3PY"
+ nickname="Oliver"
+ subject="comment 2"
+ date="2011-12-23T22:04:08Z"
+ content="""
+As joey points out the problem is B overwrites A, so that any files in A that aren't in B will be removed. But the suggestion to keep B in a separate subdirectory in the repository means I'll end up with duplicates of files in both A and B. What I want is to have the merged superset of all files from both A and B with only one copy of identical files. 
+
+The problem is that unique symlinks in A/master are deleted when B/master is merged in. To add back the deleted files after the merge you can do this:
+
+    git checkout master~1 deleted_file_name                                                              #checkout a single deleted file called deleted_file_name
+    git diff master~1 master --numstat --name-only --diff-filter=D                                       #get the names of all files deleted between master and master~1
+    git diff master~1 master --numstat --name-only --diff-filter=D | xargs git checkout master~1         #checkout all deleted files between master and master~1
+
+Once the first merge has been done after set up, you can continue to make changes to A and B and future merges won't require accounting for deleted files in this way.
+"""]]
diff --git a/doc/forum/syncing_non-git_trees_with_git-annex/comment_3_6d8f399f0549eddd1d1f5c9c9a10c654._comment b/doc/forum/syncing_non-git_trees_with_git-annex/comment_3_6d8f399f0549eddd1d1f5c9c9a10c654._comment
new file mode 100644
index 0000000000..1793e686fa
--- /dev/null
+++ b/doc/forum/syncing_non-git_trees_with_git-annex/comment_3_6d8f399f0549eddd1d1f5c9c9a10c654._comment
@@ -0,0 +1,13 @@
+[[!comment format=mdwn
+ username="http://adamspiers.myopenid.com/"
+ nickname="Adam"
+ subject="re-annexing previously annexed files"
+ date="2012-03-29T21:41:54Z"
+ content="""
+Here's another handy command-line which annexes all files in repo B which have already been annexed in repo A:
+
+    git status --porcelain | sed -n '/^ T /{s///;p}' | xargs git annex add
+
+The 'T' outputted by git status for these files indicates a type change: it's a symlink to the annex in repo A, but a normal file in repo B.
+
+"""]]
diff --git a/doc/forum/syncing_to_an_encrypted_remove_won__39__t_prompt_with_pinentry-gtk2.mdwn b/doc/forum/syncing_to_an_encrypted_remove_won__39__t_prompt_with_pinentry-gtk2.mdwn
new file mode 100644
index 0000000000..8e6f2805a2
--- /dev/null
+++ b/doc/forum/syncing_to_an_encrypted_remove_won__39__t_prompt_with_pinentry-gtk2.mdwn
@@ -0,0 +1,3 @@
+I'm trying to sync my unencrypted local repo to an encrypted remote located on an external drive. I'm using Debian 8.1. When I run `git-annex sync encrypted-remote` I get prompted in my shell for the password. I'm not sure why, as other places pinentry-gtk2 pops up. This is annoying because my gpg key isn't getting cached, so I have to enter my long password several times. 
+
+How can I make git-annex/git-remote-gcrypt use pinentry-gtk2?
diff --git a/doc/forum/syncing_to_an_encrypted_remove_won__39__t_prompt_with_pinentry-gtk2/comment_1_daf41d0e59d384dfe5c86a53600ea3f7._comment b/doc/forum/syncing_to_an_encrypted_remove_won__39__t_prompt_with_pinentry-gtk2/comment_1_daf41d0e59d384dfe5c86a53600ea3f7._comment
new file mode 100644
index 0000000000..0b521a3e62
--- /dev/null
+++ b/doc/forum/syncing_to_an_encrypted_remove_won__39__t_prompt_with_pinentry-gtk2/comment_1_daf41d0e59d384dfe5c86a53600ea3f7._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="https://openid.stackexchange.com/user/e65e6d0e-58ba-41de-84cc-1f2ba54cf574"
+ nickname="Mica"
+ subject="needed a gpg.conif"
+ date="2015-09-16T04:20:53Z"
+ content="""
+Not sure why Debian stable didn't give me a default gpg.config, but it didn't. Copying one from an old installation worked.
+"""]]
diff --git a/doc/forum/taskwarrior.mdwn b/doc/forum/taskwarrior.mdwn
new file mode 100644
index 0000000000..3e44cc610c
--- /dev/null
+++ b/doc/forum/taskwarrior.mdwn
@@ -0,0 +1,11 @@
+I try to sync my taskWarrior files .task/*.data with git-annex ... but there is two problem :
+
+- i need to chmod 755 my files because taskWarrior doesn't recognize them, or say "problem with permission" 
+
+- taskwarrior seems crazy with symbolic link used by git-annex, undo not work, task appear multiple times , etc.
+
+Is there any solution ?
+Any user experienced the same problem?
+
+Thanks
+Sr.
diff --git a/doc/forum/taskwarrior/comment_1_1c3a29e7d292cb602d9d349f8009b51e._comment b/doc/forum/taskwarrior/comment_1_1c3a29e7d292cb602d9d349f8009b51e._comment
new file mode 100644
index 0000000000..c8682b3787
--- /dev/null
+++ b/doc/forum/taskwarrior/comment_1_1c3a29e7d292cb602d9d349f8009b51e._comment
@@ -0,0 +1,10 @@
+[[!comment format=mdwn
+ username="https://www.google.com/accounts/o8/id?id=AItOawl9sYlePmv1xK-VvjBdN-5doOa_Xw-jH4U"
+ nickname="Richard"
+ subject="comment 1"
+ date="2013-08-06T12:44:11Z"
+ content="""
+* You could try metastore (not merge friendly), git-cache-meta (pretty minimal) or metamonger (not done yet) to sync your file permissions
+* Look into direct mode to avoid symlinks
+* Alternatively, check your taskwarrior files into git, not git-annex, to avoid symlinks
+"""]]
diff --git a/doc/forum/taskwarrior/comment_2_4b3d70501763f6d36c927ae37bbd33c2._comment b/doc/forum/taskwarrior/comment_2_4b3d70501763f6d36c927ae37bbd33c2._comment
new file mode 100644
index 0000000000..ec6bcb9529
--- /dev/null
+++ b/doc/forum/taskwarrior/comment_2_4b3d70501763f6d36c927ae37bbd33c2._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="https://me.yahoo.com/a/FHnTlSBo1eCGJRwueeKeB6.RCaPbGMPr5jxx8A--#ce0d8"
+ nickname="Hamza"
+ subject="comment 2"
+ date="2014-04-14T15:27:48Z"
+ content="""
+Using direct mode would replace symlinks with actual files.
+"""]]
diff --git a/doc/forum/telehash_syncing.mdwn b/doc/forum/telehash_syncing.mdwn
new file mode 100644
index 0000000000..3a5266506e
--- /dev/null
+++ b/doc/forum/telehash_syncing.mdwn
@@ -0,0 +1,10 @@
+Hi
+
+I have read some info about telehash. It looks verry promising. I was wondering though how syncing will work. For example. I have 2 computers. Normal PC and a laptop. Mostly only one is on at a time. 
+
+* Sync messages will be sent over telehash protocoll ?
+* What if I push some changes (they will be synced to a shared repository) and laptop is not online. How will git-annex know what to sync from a shared repository ?
+* Do you plan to send files/commits directly to online clients ? If 2 friends are online at the same time.
+* What will happen with data on a shared repository if all clients have synced content ? Will it be deleted since it is not longer needed ?
+
+I was thinking of a model where you sync directly (if possible), and just drop shared content to repo for offline users. Whan everyone have pulled content it may be removed from shared repo.
diff --git a/doc/forum/tell_us_how_you__39__re_using_git-annex.mdwn b/doc/forum/tell_us_how_you__39__re_using_git-annex.mdwn
new file mode 100644
index 0000000000..d289b9f50a
--- /dev/null
+++ b/doc/forum/tell_us_how_you__39__re_using_git-annex.mdwn
@@ -0,0 +1,6 @@
+Tell your git-annex stories here. Feel free to give as little or as much detail
+as appropriate about how you're using it. How's it working out for you?
+
+See [[testimonials]] for some other stories.
+
+[[!meta author=Joey]]
diff --git a/doc/forum/tell_us_how_you__39__re_using_git-annex/comment_1_4884803ddee7f642a3ac995a19967a6a._comment b/doc/forum/tell_us_how_you__39__re_using_git-annex/comment_1_4884803ddee7f642a3ac995a19967a6a._comment
new file mode 100644
index 0000000000..2351378bca
--- /dev/null
+++ b/doc/forum/tell_us_how_you__39__re_using_git-annex/comment_1_4884803ddee7f642a3ac995a19967a6a._comment
@@ -0,0 +1,17 @@
+[[!comment format=mdwn
+ username="http://mildred.fr/"
+ subject="just amazing"
+ date="2012-04-12T17:12:41Z"
+ content="""
+git-annex is just amazing. I just started using it and for once, I have hope to be able to organize my files a little better than now.
+
+Currently, I have a huge homedir. From time to time, I move file away in external hard drives, then forget about them. When I want to look at them back, I just can't because I have forgotten where they are. I have also a ton of files on those drives that I can't access because they are not indexed. With git-annex I have hope to put all of these files on a git repository. I will be able to see them everywhere, and find them when I need to.
+
+I might stop loosing files for once.
+
+I might avoid having multiple copies of the same things over and over again, without knowing so. and regain some more disk space.
+
+For the moment, I'm archiving my photographs. But there is one thing that might not go very well: directory hierarchies where everything is important (file owner, specific permissions, symlinks). I won't just be able to blindly annex all of these files. But for the moment I'll stick at archiving ocuments and it should be amazing.
+
+[Mildred](http://mildred.fr)
+"""]]
diff --git a/doc/forum/tell_us_how_you__39__re_using_git-annex/comment_2_61f5054918e7b36c191454365bc7f3b7._comment b/doc/forum/tell_us_how_you__39__re_using_git-annex/comment_2_61f5054918e7b36c191454365bc7f3b7._comment
new file mode 100644
index 0000000000..3bd981c5db
--- /dev/null
+++ b/doc/forum/tell_us_how_you__39__re_using_git-annex/comment_2_61f5054918e7b36c191454365bc7f3b7._comment
@@ -0,0 +1,10 @@
+[[!comment format=mdwn
+ username="http://cgray.myopenid.com/"
+ nickname="cgray"
+ subject="comment 2"
+ date="2012-04-14T01:18:53Z"
+ content="""
+Git-annex has really helped me with my media files.  I have a big NAS drive where I keep all my music, tv, and movies files, each in their own git annex.  I tend to keep the media that I want to watch or listen to on my laptop and then drop it when it is done.  This way I don't have too much on my laptop at any one time, but I have a nice selection for when I'm traveling and don't have access to my NAS.
+
+Additionally, I have a mp3 player that will format itself randomly every few months or so.  I keep my podcasts on it in a git annex and in a git annex on my laptop.  When I am done with a podcast, I can delete it from the mp3 player and then sync that information with my laptop.  With this method, I have a backup of what should be on my mp3 player, so I don't need to worry about losing it all when the mp3 player decides it's had enough.
+"""]]
diff --git a/doc/forum/tell_us_how_you__39__re_using_git-annex/comment_3_db07e8703be606c998c831e91d300d69._comment b/doc/forum/tell_us_how_you__39__re_using_git-annex/comment_3_db07e8703be606c998c831e91d300d69._comment
new file mode 100644
index 0000000000..1a80bb4808
--- /dev/null
+++ b/doc/forum/tell_us_how_you__39__re_using_git-annex/comment_3_db07e8703be606c998c831e91d300d69._comment
@@ -0,0 +1,10 @@
+[[!comment format=mdwn
+ username="http://bergey.dreamwidth.org/"
+ ip="66.80.90.109"
+ subject="git-annex, media, emacs"
+ date="2012-08-31T01:47:36Z"
+ content="""
+I've been using git-annex for a month or two to manage my music collection.  It's great to know I can drop files if I need space on the laptop, knowing exactly where I have other copies.  Now I'm writing an emacs mode to help me keep track: https://gitorious.org/emacs-contrib/annex-mode
+
+Locally available files are colored differently, and pressing g runs `git annex get` on the file at point.
+"""]]
diff --git a/doc/forum/tell_us_how_you__39__re_using_git-annex/comment_4_a58595969cdd42ed20210e9615b42e42._comment b/doc/forum/tell_us_how_you__39__re_using_git-annex/comment_4_a58595969cdd42ed20210e9615b42e42._comment
new file mode 100644
index 0000000000..d80423eac0
--- /dev/null
+++ b/doc/forum/tell_us_how_you__39__re_using_git-annex/comment_4_a58595969cdd42ed20210e9615b42e42._comment
@@ -0,0 +1,22 @@
+[[!comment format=mdwn
+ username="https://www.google.com/accounts/o8/id?id=AItOawnZEanlyzay_QlEAL0CWpyZcRTyN7vay8U"
+ nickname="Carlo"
+ subject="Best thing since sliced bread"
+ date="2012-11-21T13:08:57Z"
+ content="""
+It's amazing how perfect git annex is for my use case.
+
+I record music, which results in a bunch of big files that don't change, and a bunch of small files which change often.
+
+I also need these files to be preserved for the long haul, and be verified now and then. They also need to be backed up three or four times, because they're original files! Keeping track of all that was a headache before git annex, and I was also constantly awake at night because it was impossible to verify if everything's still okay.
+
+Now I just dump everything into my archive annex and I know where everything is, and that it's safe, period. I even set myself up a raspberry pi with a usb drive adapter and put some spindown code in a cron job. Now I have a hard drive at rest that I can access from anywhere at any time. The whole thing gets backed up by Amazon Glacier, but if Amazon starts getting pesky I can just add another backend and sync it up. I can even version my small files if I want to by putting them directly into git. This entire thing would have been such a huge undertaking but now it's easy. I dump everything into the annex and to the actual backups whenever I get round to it. It just has a natural flow to it.
+
+Et voila. Professional grade backups, at home, independant from any specific vendor. I absolutely never have to worry about my files going anywhere, and if I get super paranoid, I can just \"git annex fsck\" and mathematically prove to myself everything's still there.
+
+It's totally sad how many creative people will be losing their life's work in a couple of years because they rely on regular filesystems on their USB drives. Stuff like DropBox and SpiderOak is better than nothing but trusting single vendor to not screw up or mess with your life's work is just plain creepy.
+
+I just have a huge, warm fuzzy feeling now. If this sounds like I'm relentlessly hyping git annex, it's because I am! It deserves it!
+
+
+"""]]
diff --git a/doc/forum/temporary_AWS_credentials.mdwn b/doc/forum/temporary_AWS_credentials.mdwn
new file mode 100644
index 0000000000..c62287ac38
--- /dev/null
+++ b/doc/forum/temporary_AWS_credentials.mdwn
@@ -0,0 +1,2 @@
+Can you add support for temporary AWS credentials?  You just have to pass AWS_SESSION_TOKEN, in addition to the usual AWS_ACCESS_KEY_ID and AWS_SECRET_ACCESS_KEY, to the AWS APIs.
+This will enable working with time-limited and multifactor-authentication-protected credentials.
diff --git a/doc/forum/temporary_AWS_credentials/comment_1_655825b3aa981e97969e226c98034c4b._comment b/doc/forum/temporary_AWS_credentials/comment_1_655825b3aa981e97969e226c98034c4b._comment
new file mode 100644
index 0000000000..90788412a4
--- /dev/null
+++ b/doc/forum/temporary_AWS_credentials/comment_1_655825b3aa981e97969e226c98034c4b._comment
@@ -0,0 +1,7 @@
+[[!comment format=mdwn
+ username="joey"
+ subject="""comment 1"""
+ date="2018-09-05T19:36:52Z"
+ content="""
+Sure, done.
+"""]]
diff --git a/doc/forum/temporary_AWS_credentials/comment_2_d4b66fd587c3a0ce74436ec09b398f10._comment b/doc/forum/temporary_AWS_credentials/comment_2_d4b66fd587c3a0ce74436ec09b398f10._comment
new file mode 100644
index 0000000000..aafead83ae
--- /dev/null
+++ b/doc/forum/temporary_AWS_credentials/comment_2_d4b66fd587c3a0ce74436ec09b398f10._comment
@@ -0,0 +1,9 @@
+[[!comment format=mdwn
+ username="https://me.yahoo.com/a/iOGTltEpmOTQ.xZ99NFP5c7Zdcc-#6a7ba"
+ nickname="Ilya S"
+ avatar="http://cdn.libravatar.org/avatar/8a133555cc739a35b83b07d5724d28d9e2f7852c224e949eec6fd4fb7693331e"
+ subject="comment 2"
+ date="2018-09-06T04:17:35Z"
+ content="""
+Thanks a lot!
+"""]]
diff --git a/doc/forum/test_whether_a_file_is_already_annexed.mdwn b/doc/forum/test_whether_a_file_is_already_annexed.mdwn
new file mode 100644
index 0000000000..55ac739ae9
--- /dev/null
+++ b/doc/forum/test_whether_a_file_is_already_annexed.mdwn
@@ -0,0 +1 @@
+I am storing the pictures I took over the years with git-annex. Frequently I come across messy old directories with lots of pictures inside and I want to know which ones were already annexed - or which ones were not. Is there a quick way to test whether the content of a given file is already annexed? I mean computing the key (hash) of the given file and testing whether it is already present among the annex objects.
diff --git a/doc/forum/test_whether_a_file_is_already_annexed/comment_1_55bcb41abae2b70c046e3da1c4d8c761._comment b/doc/forum/test_whether_a_file_is_already_annexed/comment_1_55bcb41abae2b70c046e3da1c4d8c761._comment
new file mode 100644
index 0000000000..d175983e38
--- /dev/null
+++ b/doc/forum/test_whether_a_file_is_already_annexed/comment_1_55bcb41abae2b70c046e3da1c4d8c761._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="https://me.yahoo.com/a/EbvxpTI_xP9Aod7Mg4cwGhgjrCrdM5s-#7c0f4"
+ subject="whereis"
+ date="2015-09-03T01:07:32Z"
+ content="""
+Thought to say that you could use 'info' but  http://git-annex.branchable.com/bugs/git_annex_info_is_reporting_file_as_not_annexed_in_direct_mode/ .
+What about 'git annex whereis FILE' ?  it should be empty output for non-annexed file
+"""]]
diff --git a/doc/forum/test_whether_a_file_is_already_annexed/comment_2_e2531c7e77bbc75f24af7ce5f789826e._comment b/doc/forum/test_whether_a_file_is_already_annexed/comment_2_e2531c7e77bbc75f24af7ce5f789826e._comment
new file mode 100644
index 0000000000..2d0962fc3d
--- /dev/null
+++ b/doc/forum/test_whether_a_file_is_already_annexed/comment_2_e2531c7e77bbc75f24af7ce5f789826e._comment
@@ -0,0 +1,22 @@
+[[!comment format=mdwn
+ username="emanuele.olivetti@47d88ed185b03191e25329caa6fabc2efb3118b2"
+ nickname="emanuele.olivetti"
+ subject="not whereis"
+ date="2015-09-03T07:44:49Z"
+ content="""
+First of all thank you for the answer and the bug report. Unfortunately git-annex whereis seems not to be the answer to my problem, because it works only when queried on already annexed files, while I'd like to test the yet-not-annexed ones. Here's an example using whereis:
+
+    git-annex whereis SAM_7198.m4v     # SAM_7198.m4v is an already annexed files in the current directory, so within the repository: whereis works well
+    whereis SAM_7198.m4v (2 copies)
+        5dd7f891-579a-45c6-a5bf-3ca42e3a5f9a -- laptop [here]
+        ebf56bea-2970-42b1-bace-9d72ac9ed8d1 -- emanuele@desktop:/home/emanuele/annex [origin]
+    ok
+    
+    cp -L SAM_7198.m4v zzz  # creating a copy of a that file
+    git-annex whereis zzz  # output is empty, so the file was not found, despite having the same content of SAM_7198.m4v and being in the repo dir
+    
+    cp -L SAM_7198.m4v /tmp/  # copying the annexed file outside the repo
+    git-annex whereis /tmp/SAM_7198.m4v 
+    fatal: /tmp/SAM_7198.m4v: '/tmp/SAM_7198.m4v' is outside repository
+
+"""]]
diff --git a/doc/forum/test_whether_a_file_is_already_annexed/comment_3_21f51c99b1aac8b3292f60bd83493e94._comment b/doc/forum/test_whether_a_file_is_already_annexed/comment_3_21f51c99b1aac8b3292f60bd83493e94._comment
new file mode 100644
index 0000000000..769c1c2b7d
--- /dev/null
+++ b/doc/forum/test_whether_a_file_is_already_annexed/comment_3_21f51c99b1aac8b3292f60bd83493e94._comment
@@ -0,0 +1,15 @@
+[[!comment format=mdwn
+ username="CandyAngel"
+ subject="comment 3"
+ date="2015-09-03T08:05:52Z"
+ content="""
+Hm.. having some kind of exposure of the key generation code on the command line would actually be pretty useful. So you can do something like:
+
+    find -type f | xargs git annex genkey --format '${key} ${file}'
+
+Probably worth a TODO, really.
+
+Regarding your situation, one way of doing it would be to recursively copy the photo directory as hardlinks, `git annex import --clean-duplicates` the hardlinked copies, then diff the directories. This would give you a list of removals and those removals are already in the repo.
+
+Or if you just want to remove them, just run '*git annex import --clean-duplicates*' on the original photos directory. **NOTE: There was recently an issue with git-annex deleting files that it didn't have any known copies of, so a recent version is highly recommended if using --clean-duplicates.**
+"""]]
diff --git a/doc/forum/test_whether_a_file_is_already_annexed/comment_4_2c61629777f47ce3e697298d5736f997._comment b/doc/forum/test_whether_a_file_is_already_annexed/comment_4_2c61629777f47ce3e697298d5736f997._comment
new file mode 100644
index 0000000000..d5d2d1dea4
--- /dev/null
+++ b/doc/forum/test_whether_a_file_is_already_annexed/comment_4_2c61629777f47ce3e697298d5736f997._comment
@@ -0,0 +1,22 @@
+[[!comment format=mdwn
+ username="joey"
+ subject="""comment 4"""
+ date="2015-09-09T17:43:22Z"
+ content="""
+You can use either `git-annex import --deduplicate` 
+or `git annex import --skip-duplicates` to import files from a directory
+except for ones already in the repository. The former deletes the duplicate
+files, and the latter leaves them as-is.
+
+I don't think there's currently a good stand-alone way to check if a file is a
+duplicate of content already in the annex, before adding it.
+Would need a new command to be added. 
+
+Of course, if you simply `git annex add` everything, regular hash based
+deduplication will point the duplicate file to the same object used by
+the file when you added it earlier. So, you don't need to worry about
+adding duplicates wasting much space. They may make your repo more
+cluttered than you like, is all.
+
+See also this tip: [[tips/finding_duplicate_files]].
+"""]]
diff --git a/doc/forum/time_profiling_the_assistant.mdwn b/doc/forum/time_profiling_the_assistant.mdwn
new file mode 100644
index 0000000000..95a02975de
--- /dev/null
+++ b/doc/forum/time_profiling_the_assistant.mdwn
@@ -0,0 +1,19 @@
+From the repo (commit ecc548) I built git-annex with stack issuing this command:
+
+    stack install --executable-profiling --library-profiling --ghc-options="-rtsopts -auto-all"
+
+then I run the output executable with +RTS -p like
+
+    git-annex version +RTS -p
+
+and obtain the corresponding git-annex.prof with the time profile.
+
+What I am unable to do is to get a time profile of the assistant: if I kill it the git-annex.prof file is empty, same happens if I stop the assistant with
+
+    git-annex assistant --stop
+
+So... is there another way to time-profile a git-annex command? Am I missing something in this procedure?
+
+Any advice is welcome :)
+
+-- zarel
diff --git a/doc/forum/time_profiling_the_assistant/comment_1_0227d9c0db24c6abadfe091f92848ec6._comment b/doc/forum/time_profiling_the_assistant/comment_1_0227d9c0db24c6abadfe091f92848ec6._comment
new file mode 100644
index 0000000000..63971d291f
--- /dev/null
+++ b/doc/forum/time_profiling_the_assistant/comment_1_0227d9c0db24c6abadfe091f92848ec6._comment
@@ -0,0 +1,16 @@
+[[!comment format=mdwn
+ username="joey"
+ subject="""comment 1"""
+ date="2016-05-03T19:23:59Z"
+ content="""
+The assistant is stopped by sending it a TERM signal. Seems that the RTS
+does not save profiling on TERM.
+
+Try manually sending it an INT signal instead. The RTS does save profiling
+when the program is ctrl-c'd.
+
+I'd probably just run git annex assistant --foreground and then ctrl-c it.
+
+There's also a cabal flag to enable EKG, so you can watch pretty graphs in
+a web browser while it's running.
+"""]]
diff --git a/doc/forum/tips__58___special__95__remotes__47__hook_with_tahoe-lafs.mdwn b/doc/forum/tips__58___special__95__remotes__47__hook_with_tahoe-lafs.mdwn
new file mode 100644
index 0000000000..8981200d88
--- /dev/null
+++ b/doc/forum/tips__58___special__95__remotes__47__hook_with_tahoe-lafs.mdwn
@@ -0,0 +1,22 @@
+This is work in progress, since there is now a [[special_remotes/hook]] for users to plug in whatever they want as a remote, here's my recipe for using tahoe-lafs as a remote, this is a copy and paste the relavent section from my .git/config file
+
+        tahoe-store-hook = tahoe put $ANNEX_FILE tahoe:$ANNEX_KEY
+        tahoe-retrieve-hook = tahoe get tahoe:$ANNEX_KEY $ANNEX_FILE
+        tahoe-remove-hook = tahoe rm tahoe:$ANNEX_KEY
+        tahoe-checkpresent-hook = tahoe ls tahoe:$ANNEX_KEY 2>&1 || echo FAIL
+
+Where `tahoe:` is a tahoe-lafs alias, ideally you should create a new alias (DIR-CAP or whatever the terminolgy is) to store your files, I just used the default `tahoe:` alias for testing.
+
+The only quirk I've noticed is this...
+
+
    +$ git annex whereis .
+whereis frink.jar (2 copies) 
+  	084603a8-7243-11e0-b1f5-83102bcd7953  -- here (testtest)
+   	1d1bc312-7243-11e0-a9ce-5f10c0ce9b0a
+ok
+
    
+
+1d1bc312-7243-11e0-a9ce-5f10c0ce9b0a is my [[!google tahoe-lafs]] remote, but there is no label/description on it. The checkpresent-hook was a little confusing when I was setting it up, I'm currently unsure if I am doing the right thing or not with my hook. My get and put commands are a little verbose for now, i might redirect it to /dev/null once I am happier with the overall performance/behaviour my setup.
+
+Other than the quirks above, I am able to put and get files from my tahoe-lafs remote. The only thing that I have not figured out is how to "remove a file" on the remote to free up space on the remote.
diff --git a/doc/forum/tips__58___special__95__remotes__47__hook_with_tahoe-lafs/comment_1_76bb33ce45ce6a91b86454147463193b._comment b/doc/forum/tips__58___special__95__remotes__47__hook_with_tahoe-lafs/comment_1_76bb33ce45ce6a91b86454147463193b._comment
new file mode 100644
index 0000000000..388641f69e
--- /dev/null
+++ b/doc/forum/tips__58___special__95__remotes__47__hook_with_tahoe-lafs/comment_1_76bb33ce45ce6a91b86454147463193b._comment
@@ -0,0 +1,10 @@
+[[!comment format=mdwn
+ username="https://www.google.com/accounts/o8/id?id=AItOawmBUR4O9mofxVbpb8JV9mEbVfIYv670uJo"
+ nickname="Justin"
+ subject="whereis labels"
+ date="2011-04-29T13:08:35Z"
+ content="""
+You should be able to fix the missing label by editing .git-annex/uuid.log and adding
+
+    1d1bc312-7243-11e0-a9ce-5f10c0ce9b0a tahoe
+"""]]
diff --git a/doc/forum/tips__58___special__95__remotes__47__hook_with_tahoe-lafs/comment_2_4d9b9d47d01d606a475678f630797bf9._comment b/doc/forum/tips__58___special__95__remotes__47__hook_with_tahoe-lafs/comment_2_4d9b9d47d01d606a475678f630797bf9._comment
new file mode 100644
index 0000000000..e7c3d619dd
--- /dev/null
+++ b/doc/forum/tips__58___special__95__remotes__47__hook_with_tahoe-lafs/comment_2_4d9b9d47d01d606a475678f630797bf9._comment
@@ -0,0 +1,10 @@
+[[!comment format=mdwn
+ username="http://joey.kitenet.net/"
+ nickname="joey"
+ subject="comment 2"
+ date="2011-04-29T15:24:56Z"
+ content="""
+If `tahoe ls` outputs only the key, on its own line, and exits nonzero if it's not present, then I think you did the right thing.
+
+To remove a file, use `git annex move file --from tahoe` and then you can drop it locally.
+"""]]
diff --git a/doc/forum/tips__58___special__95__remotes__47__hook_with_tahoe-lafs/comment_3_8a812b11fcc2dc3b6fcf01cdbbb8459d._comment b/doc/forum/tips__58___special__95__remotes__47__hook_with_tahoe-lafs/comment_3_8a812b11fcc2dc3b6fcf01cdbbb8459d._comment
new file mode 100644
index 0000000000..16ad9e9886
--- /dev/null
+++ b/doc/forum/tips__58___special__95__remotes__47__hook_with_tahoe-lafs/comment_3_8a812b11fcc2dc3b6fcf01cdbbb8459d._comment
@@ -0,0 +1,12 @@
+[[!comment format=mdwn
+ username="https://www.google.com/accounts/o8/id?id=AItOawkSq2FDpK2n66QRUxtqqdbyDuwgbQmUWus"
+ nickname="Jimmy"
+ subject="comment 3"
+ date="2011-04-29T15:33:24Z"
+ content="""
+@justin, I discovered that \"git annex describe\" did what I wanted
+
+@joey, yep that is the behaviour of \"tahoe ls\", thanks for the tip on removing the file from the remote.
+
+It seems to be working okay for now, the only concern is that on the remote everything is dumped into the same directory, but I can live with that, since I want to track biggish blobs and not lots of small little files.
+"""]]
diff --git a/doc/forum/tips__58___special__95__remotes__47__hook_with_tahoe-lafs/comment_4_fc98c819bc5eb4d7c9e74d87fb4f6f3b._comment b/doc/forum/tips__58___special__95__remotes__47__hook_with_tahoe-lafs/comment_4_fc98c819bc5eb4d7c9e74d87fb4f6f3b._comment
new file mode 100644
index 0000000000..5d271c6f3c
--- /dev/null
+++ b/doc/forum/tips__58___special__95__remotes__47__hook_with_tahoe-lafs/comment_4_fc98c819bc5eb4d7c9e74d87fb4f6f3b._comment
@@ -0,0 +1,39 @@
+[[!comment format=mdwn
+ username="https://www.google.com/accounts/o8/id?id=AItOawkSq2FDpK2n66QRUxtqqdbyDuwgbQmUWus"
+ nickname="Jimmy"
+ subject="comment 4"
+ date="2011-04-29T16:17:11Z"
+ content="""
+I've just tried to use the ANNEX_HASH_ variables, example of my configuration
+
+
    +    git config annex.tahoe-store-hook 'tahoe mkdir $ANNEX_HASH_1 && tahoe put $ANNEX_FILE tahoe:$ANNEX_HASH_1/$ANNEX_KEY'
+    git config annex.tahoe-retrieve-hook 'tahoe get tahoe:$ANNEX_HASH_1/$ANNEX_KEY $ANNEX_FILE'
+    git config annex.tahoe-remove-hook 'tahoe rm tahoe:$ANNEX_HASH_1/$ANNEX_KEY'
+    git config annex.tahoe-checkpresent-hook 'tahoe ls tahoe:$ANNEX_HASH_1/$ANNEX_KEY 2>&1 || echo FAIL'
+    git annex initremote library type=hook hooktype=tahoe encryption=none
+    git annex describe 1d1bc312-7243-11e0-a9ce-5f10c0ce9b0a library
+
    
+
+It's seems to work quite well for me now, I did run across this when I tried to drop a file locally, leaving the file on my remote
+
+
    +jtang@x00:/tmp/annex3 $ git annex drop .
+drop frink.sh (checking library...) (unsafe) 
+  Could only verify the existence of 0 out of 1 necessary copies
+  Try making some of these repositories available:
+  	1d1bc312-7243-11e0-a9ce-5f10c0ce9b0a  -- library
+  (Use --force to override this check, or adjust annex.numcopies.)
+failed
+drop t/frink.jar (checking library...) (unsafe) 
+  Could only verify the existence of 0 out of 1 necessary copies
+  Try making some of these repositories available:
+  	1d1bc312-7243-11e0-a9ce-5f10c0ce9b0a  -- library
+  (Use --force to override this check, or adjust annex.numcopies.)
+failed
+git-annex: 2 failed
+1|jtang@x00:/tmp/annex3 $ 
+
    
+
+I do know that the files exist in my library as I have just inserted them, it seemed to work when I didnt have the hashing, it appears that the checkpresent doesn't seem to pass the ANNEX_HASH_* variables (from the limited debugging I did)
+"""]]
diff --git a/doc/forum/tips__58___special__95__remotes__47__hook_with_tahoe-lafs/comment_5_c459fb479fe7b13eaea2377cfc1923a6._comment b/doc/forum/tips__58___special__95__remotes__47__hook_with_tahoe-lafs/comment_5_c459fb479fe7b13eaea2377cfc1923a6._comment
new file mode 100644
index 0000000000..9127cdeeaa
--- /dev/null
+++ b/doc/forum/tips__58___special__95__remotes__47__hook_with_tahoe-lafs/comment_5_c459fb479fe7b13eaea2377cfc1923a6._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="http://joey.kitenet.net/"
+ nickname="joey"
+ subject="comment 5"
+ date="2011-04-29T18:01:04Z"
+ content="""
+I've corrected the missing `ANNEX_HASH_*` oversight. (It also affected removal, btw.)
+"""]]
diff --git a/doc/forum/tips__58___special__95__remotes__47__hook_with_tahoe-lafs/comment_6_2e9da5a919bbbc27b32de3b243867d4f._comment b/doc/forum/tips__58___special__95__remotes__47__hook_with_tahoe-lafs/comment_6_2e9da5a919bbbc27b32de3b243867d4f._comment
new file mode 100644
index 0000000000..80874db31d
--- /dev/null
+++ b/doc/forum/tips__58___special__95__remotes__47__hook_with_tahoe-lafs/comment_6_2e9da5a919bbbc27b32de3b243867d4f._comment
@@ -0,0 +1,23 @@
+[[!comment format=mdwn
+ username="https://www.google.com/accounts/o8/id?id=AItOawkSq2FDpK2n66QRUxtqqdbyDuwgbQmUWus"
+ nickname="Jimmy"
+ subject="comment 6"
+ date="2011-04-29T20:11:08Z"
+ content="""
+Cool, that seems to make things work as expected, here's an updated recipe
+
+
+
    +git config annex.tahoe-store-hook 'tahoe mkdir tahoe:$ANNEX_HASH_1/$ANNEX_HASH_2 && tahoe put $ANNEX_FILE tahoe:$ANNEX_HASH_1/$ANNEX_HASH_2/$ANNEX_KEY'
+git config annex.tahoe-retrieve-hook 'tahoe get tahoe:$ANNEX_HASH_1/$ANNEX_HASH_2/$ANNEX_KEY $ANNEX_FILE'
+git config annex.tahoe-remove-hook 'tahoe rm tahoe:$ANNEX_HASH_1/$ANNEX_HASH_2/$ANNEX_KEY'
+git config annex.tahoe-checkpresent-hook 'tahoe ls tahoe:$ANNEX_HASH_1/$ANNEX_HASH_2/$ANNEX_KEY 2>&1 || echo FAIL'
+git annex initremote library type=hook hooktype=tahoe encryption=none
+git annex describe 1d1bc312-7243-11e0-a9ce-5f10c0ce9b0a library
+
    
+
+
+I just needs some of the output redirected to /dev/null.
+
+(I updated this comment to fix a bug. --[[Joey]])
+"""]]
diff --git a/doc/forum/tips__58___special__95__remotes__47__hook_with_tahoe-lafs/comment_7_d636c868524b2055ee85832527437f90._comment b/doc/forum/tips__58___special__95__remotes__47__hook_with_tahoe-lafs/comment_7_d636c868524b2055ee85832527437f90._comment
new file mode 100644
index 0000000000..1d75fb9631
--- /dev/null
+++ b/doc/forum/tips__58___special__95__remotes__47__hook_with_tahoe-lafs/comment_7_d636c868524b2055ee85832527437f90._comment
@@ -0,0 +1,20 @@
+[[!comment format=mdwn
+ username="zooko"
+ ip="97.118.97.117"
+ subject="request for information, plus some ideas"
+ date="2011-05-14T05:07:17Z"
+ content="""
+Hey Jimmy: how's this working for you now? I would expect it to go slower and slower since Tahoe-LAFS has an O(N) algorithm for reading or updating directories.
+
+Of course, if it is still fast enough for your uses then that's okay. :-)
+
+(We're working on optimizations of this for future releases of Tahoe-LAFS.)
+
+I'd like to understand the desired behavior of store-hook and retrieve-hook better, in order to see if there is a more efficient way to use Tahoe-LAFS for this.
+
+Off to look for docs.
+
+Regards,
+
+Zooko
+"""]]
diff --git a/doc/forum/tips__58___special__95__remotes__47__hook_with_tahoe-lafs/comment_8_39dc449cc60a787c3bfbfaaac6f9be0c._comment b/doc/forum/tips__58___special__95__remotes__47__hook_with_tahoe-lafs/comment_8_39dc449cc60a787c3bfbfaaac6f9be0c._comment
new file mode 100644
index 0000000000..dc97128bd1
--- /dev/null
+++ b/doc/forum/tips__58___special__95__remotes__47__hook_with_tahoe-lafs/comment_8_39dc449cc60a787c3bfbfaaac6f9be0c._comment
@@ -0,0 +1,10 @@
+[[!comment format=mdwn
+ username="https://www.google.com/accounts/o8/id?id=AItOawkSq2FDpK2n66QRUxtqqdbyDuwgbQmUWus"
+ nickname="Jimmy"
+ subject="comment 8"
+ date="2011-05-14T10:02:26Z"
+ content="""
+@joey thanks for the update in the previous comment, I had forgotten about updating it.
+
+@zooko it's working okay for me right now, since I'm only putting fairly big blogs on stuff on to it and only things that I *really* care about. On the performance side, if it ran faster then it would be nicer :)
+"""]]
diff --git a/doc/forum/tips__58___special__95__remotes__47__hook_with_tahoe-lafs/comment_9_2592749c2f02b3e151896e31acba359b._comment b/doc/forum/tips__58___special__95__remotes__47__hook_with_tahoe-lafs/comment_9_2592749c2f02b3e151896e31acba359b._comment
new file mode 100644
index 0000000000..ba58643b97
--- /dev/null
+++ b/doc/forum/tips__58___special__95__remotes__47__hook_with_tahoe-lafs/comment_9_2592749c2f02b3e151896e31acba359b._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="https://www.google.com/accounts/o8/id?id=AItOawmsz4weoPXV2oEtv3zpo9dOxn_SEPz-7Iw"
+ nickname="Zooko"
+ subject="more notes about Tahoe-LAFS performance"
+ date="2014-04-21T07:18:06Z"
+ content="""
+In case anyone is reading this thread: https://github.com/zooko/tahoe-lafs/blob/3c13c138cf09e83d2f8001888e2a7de85564d406/docs/frontends/key-value-store.rst
+"""]]
diff --git a/doc/forum/treat_directory_with_multiple_files_as_a_single_item.mdwn b/doc/forum/treat_directory_with_multiple_files_as_a_single_item.mdwn
new file mode 100644
index 0000000000..4adbedb6cc
--- /dev/null
+++ b/doc/forum/treat_directory_with_multiple_files_as_a_single_item.mdwn
@@ -0,0 +1,7 @@
+In my analyses I often have multiple (>10k) generated  small files in a single directory.
+
+
+I would like to store this in git annex, in order to version them and probably even synchronize. The problem is that if a huge number of files is stored inside the repository, the repository itself becomes huge and slow. There are some ways to improve the performance ([[1|tips/Repositories_with_large_number_of_files]], [[2|forum/Handling_a_large_number_of_files]], [[3|forum/__34__git_annex_sync__34___synced_after_8_hours]]), but it doesn't solve the issue completely.
+
+
+I was wondering if it is possible to force git annex to treat a single directory with multiple files as a single item? Probably with abandoning the checksum verification.
diff --git a/doc/forum/treat_directory_with_multiple_files_as_a_single_item/comment_1_89d2819893ae954041d65a5758993be8._comment b/doc/forum/treat_directory_with_multiple_files_as_a_single_item/comment_1_89d2819893ae954041d65a5758993be8._comment
new file mode 100644
index 0000000000..5d13ae708f
--- /dev/null
+++ b/doc/forum/treat_directory_with_multiple_files_as_a_single_item/comment_1_89d2819893ae954041d65a5758993be8._comment
@@ -0,0 +1,11 @@
+[[!comment format=mdwn
+ username="leavingchicago@c04c893e78d1c4c76cb3e32b5c227cf42bbf7682"
+ nickname="leavingchicago"
+ avatar="http://cdn.libravatar.org/avatar/4ae498d3d6ee558d6b65caa658f72572"
+ subject="comment 1"
+ date="2017-05-19T01:02:57Z"
+ content="""
+Have you thought about (compress/tar)ing the files and then only include the single object in the annex?
+
+Would give you the benefits of deduping any data too.
+"""]]
diff --git a/doc/forum/treat_directory_with_multiple_files_as_a_single_item/comment_2_6889f4695bca77c18efe0000fc491b57._comment b/doc/forum/treat_directory_with_multiple_files_as_a_single_item/comment_2_6889f4695bca77c18efe0000fc491b57._comment
new file mode 100644
index 0000000000..7ca756be6f
--- /dev/null
+++ b/doc/forum/treat_directory_with_multiple_files_as_a_single_item/comment_2_6889f4695bca77c18efe0000fc491b57._comment
@@ -0,0 +1,11 @@
+[[!comment format=mdwn
+ username="dav02.git@4d920fe040aa5df86fcd9f4dd57d3fcf85ee5641"
+ nickname="dav02.git"
+ avatar="http://cdn.libravatar.org/avatar/c452f62e60af3c49990973d008ca1b76"
+ subject="(compress/tar)ing the files"
+ date="2017-05-19T12:14:14Z"
+ content="""
+I considered this, but unfortunately in this form I cannot work with files.
+
+There are hacks like archivemount, but they are extremely slow for large number of files and not available on many systems.
+"""]]
diff --git a/doc/forum/treat_directory_with_multiple_files_as_a_single_item/comment_3_61924cf10643a6d2d011899d95b4ae56._comment b/doc/forum/treat_directory_with_multiple_files_as_a_single_item/comment_3_61924cf10643a6d2d011899d95b4ae56._comment
new file mode 100644
index 0000000000..0025126698
--- /dev/null
+++ b/doc/forum/treat_directory_with_multiple_files_as_a_single_item/comment_3_61924cf10643a6d2d011899d95b4ae56._comment
@@ -0,0 +1,12 @@
+[[!comment format=mdwn
+ username="joey"
+ subject="""comment 3"""
+ date="2017-05-24T17:08:59Z"
+ content="""
+Well, there's no way I know of to make git treat a directory as one item in
+history. So even if git-annex didn't, it would not help much.
+
+You can check a tar archive into git-annex as a single file, and have hooks
+to handle unpacking it into the directory and repacking the directory back
+to it.
+"""]]
diff --git a/doc/forum/trouble_with_multiple_remotes_syncing_to_gcrypt_based_ssh_repo_.mdwn b/doc/forum/trouble_with_multiple_remotes_syncing_to_gcrypt_based_ssh_repo_.mdwn
new file mode 100644
index 0000000000..95cfe25389
--- /dev/null
+++ b/doc/forum/trouble_with_multiple_remotes_syncing_to_gcrypt_based_ssh_repo_.mdwn
@@ -0,0 +1,7 @@
+Hi, 
+
+Is anyone having difficulties with the assistant and gpg-agent?  Particularly with multiple repos syncing to a crypt backup?
+
+It seems to work for a while and then according to the error logs I can't decrypt the manifest...I'm wondering if there is some sort of locking involved that only lets one repo sync to a gcrypt repo at a time?  I can't think why the gpg-agent would stop providing the key.
+
+I also have a rather horrible time with gpg-agent in general:  I now start each repos git annex assistant individually in the shell (from which I can confirm the gpg-agent is working) rather than launch the web-app direct. 
diff --git a/doc/forum/trouble_with_multiple_remotes_syncing_to_gcrypt_based_ssh_repo_/comment_1_4ad9a6a7cf5678ac0bc6d46a54f64cd3._comment b/doc/forum/trouble_with_multiple_remotes_syncing_to_gcrypt_based_ssh_repo_/comment_1_4ad9a6a7cf5678ac0bc6d46a54f64cd3._comment
new file mode 100644
index 0000000000..3ec0167440
--- /dev/null
+++ b/doc/forum/trouble_with_multiple_remotes_syncing_to_gcrypt_based_ssh_repo_/comment_1_4ad9a6a7cf5678ac0bc6d46a54f64cd3._comment
@@ -0,0 +1,10 @@
+[[!comment format=mdwn
+ username="https://www.google.com/accounts/o8/id?id=AItOawnXybLxkPMYpP3yw4b_I6IdC3cKTD-xEdU"
+ nickname="Matt"
+ subject="comment 1"
+ date="2014-04-29T10:16:11Z"
+ content="""
+hmmm - a command line based git annex sync always works.  So either the assistant has a separate locking mechanism or the assistant is somehow losing the gpg-agent environment.  Does the assistant restart/respawn itself and lose access to the gpg-agent env?
+
+I'm somewhat clutching at straws here.
+"""]]
diff --git a/doc/forum/trouble_with_multiple_remotes_syncing_to_gcrypt_based_ssh_repo_/comment_2_82dc18ed14879936d04133f248879fb9._comment b/doc/forum/trouble_with_multiple_remotes_syncing_to_gcrypt_based_ssh_repo_/comment_2_82dc18ed14879936d04133f248879fb9._comment
new file mode 100644
index 0000000000..30bc72f549
--- /dev/null
+++ b/doc/forum/trouble_with_multiple_remotes_syncing_to_gcrypt_based_ssh_repo_/comment_2_82dc18ed14879936d04133f248879fb9._comment
@@ -0,0 +1,10 @@
+[[!comment format=mdwn
+ username="https://www.google.com/accounts/o8/id?id=AItOawnXybLxkPMYpP3yw4b_I6IdC3cKTD-xEdU"
+ nickname="Matt"
+ subject="comment 2"
+ date="2014-04-30T12:25:55Z"
+ content="""
+I see in the comments here a possible explanation...
+
+https://git-annex.branchable.com/bugs/GPG_passphrase_repeated_prompt/
+"""]]
diff --git a/doc/forum/trusted_repositories__58___fatal__58___not_a_git_repo.mdwn b/doc/forum/trusted_repositories__58___fatal__58___not_a_git_repo.mdwn
new file mode 100644
index 0000000000..f94bd3804e
--- /dev/null
+++ b/doc/forum/trusted_repositories__58___fatal__58___not_a_git_repo.mdwn
@@ -0,0 +1,32 @@
+When issuing a `git annex info`, I get:
+
+~~~
+$ git-annex info
+repository mode: indirect
+trusted repositories: fatal: Not a git repository: '/home/micas/Music/.git' fatal: Not a git repository: '/home/micas/Music/.git' 0
+semitrusted repositories: 3
+	00000000-0000-0000-0000-000000000001 -- web
+ 	85f8a5ea-6278-11e2-9978-ebb59e8f37a2 -- here (Music annex backup)
+ 	9aff38f2-6447-11e2-8c89-ef50e6c0ea6c -- backupone (Music annex backupone)
+untrusted repositories: 0
+transfers in progress: none
+available local disk space: 174.28 gigabytes (+1 megabyte reserved)
+local annex keys: 5348
+local annex size: 25.62 gigabytes
+annexed files in working tree: 5374
+size of annexed files in working tree: 25.68 gigabytes
+bloom filter size: 16 mebibytes (1.1% full)
+backend usage: 
+	SHA256: 10692
+	SHA256E: 30
+~~~
+
+The troubling part (I think) is `trusted repositories: fatal: Not a git repository: '/home/micas/Music/.git' fatal: Not a git repository: '/home/micas/Music/.git' 0`
+
+Is there a command I can use to show all the uuid of known remotes? I thought I had marked all remotes pointing to /home/micas/Music/.git as `dead`
+
+Is there another reason for the output? How can I get trusted repos back to 0?
+
+**EDIT:**
+
+I removed the folder (was not a git repo at the time of the error) `/home/micas/Music` and the error went away.
diff --git a/doc/forum/trusted_repositories__58___fatal__58___not_a_git_repo/comment_1_0a755a4a281c3bd130722093c8ddd080._comment b/doc/forum/trusted_repositories__58___fatal__58___not_a_git_repo/comment_1_0a755a4a281c3bd130722093c8ddd080._comment
new file mode 100644
index 0000000000..133a80e14b
--- /dev/null
+++ b/doc/forum/trusted_repositories__58___fatal__58___not_a_git_repo/comment_1_0a755a4a281c3bd130722093c8ddd080._comment
@@ -0,0 +1,10 @@
+[[!comment format=mdwn
+ username="http://joeyh.name/"
+ ip="209.250.56.227"
+ subject="comment 1"
+ date="2013-12-29T20:10:14Z"
+ content="""
+Yeah, git-annex will skip a repository if its directory doesn't exist, but if the directory exists, it expects it to be a git repository, and will run git commands that use it (to get its config specifically).
+
+The ugly message is doing just what it should; letting you know you have a problem. 
+"""]]
diff --git a/doc/forum/two-way_assistant_sync_with_ssh_special_remote.mdwn b/doc/forum/two-way_assistant_sync_with_ssh_special_remote.mdwn
new file mode 100644
index 0000000000..ca04e442ce
--- /dev/null
+++ b/doc/forum/two-way_assistant_sync_with_ssh_special_remote.mdwn
@@ -0,0 +1,32 @@
+I am attempting to set up automatic two-way synchronization between my laptop and a server via ssh by running assistant on both machines. I want to have both machines be non-bare and unlocked.
+
+On the rhel server:
+
+    $ mkdir ~/annex
+    $ cd ~/annex
+    $ git init
+    $ git annex init u --version=6
+    $ echo This is test file 1. >testfile1.txt
+    $ git annex add testfile1.txt
+    $ git annex sync
+    $ git remote add ml2 ssh://laptop/Users/username/annex
+    $ git annex adjust --unlock
+    $ git annex wanted . standard
+    $ git annex group . client
+
+On my mac laptop:
+
+    $ cd ~/
+    $ git clone ssh://server/home/username/annex
+    $ cd annex
+    $ git annex init ml2 --version=6
+    $ git annex sync
+    $ git annex adjust --unlock
+    $ git annex wanted . standard
+    $ git annex group . client
+
+Everything seems to work when I manually sync. But when I run
+
+    $ git annex assistant
+
+on both machines, I only get one-way automatic synchronization. Changes on the laptop are immediately propagated to the server. But changes on the server do not show up on the laptop until I manually sync. What am I doing wrong?
diff --git a/doc/forum/two-way_assistant_sync_with_ssh_special_remote/comment_1_d42def5dfc1cf814fdb07f7cf808bb12._comment b/doc/forum/two-way_assistant_sync_with_ssh_special_remote/comment_1_d42def5dfc1cf814fdb07f7cf808bb12._comment
new file mode 100644
index 0000000000..a9db035800
--- /dev/null
+++ b/doc/forum/two-way_assistant_sync_with_ssh_special_remote/comment_1_d42def5dfc1cf814fdb07f7cf808bb12._comment
@@ -0,0 +1,24 @@
+[[!comment format=mdwn
+ username="binx"
+ avatar="http://cdn.libravatar.org/avatar/1c2b6fe37ed500f4b72c105e42e81ba9"
+ subject="comment 1"
+ date="2016-12-16T13:57:41Z"
+ content="""
+I updated git-annex on the server. I now *sometimes* get automatic two-way syncing to happen with assistant. But it is not really consistent or reliable. One thing that seems to consistently fail is when I drag and drop a file into the annex folder on the mac. When I drag filename.txt into ~/annex, and then run git annex sync, I get the message:
+
+    $ git annex sync
+    commit  
+    On branch adjusted/master(unlocked)
+    Your branch is up-to-date with 'origin/adjusted/master(unlocked)'.
+    Untracked files:
+    	.DS_Store
+    	filename.txt
+    
+    nothing added to commit but untracked files present
+    ok
+    pull u 
+    ok
+    pull origin 
+    ok
+
+"""]]
diff --git a/doc/forum/two_lines_back_to_remote__63__.mdwn b/doc/forum/two_lines_back_to_remote__63__.mdwn
new file mode 100644
index 0000000000..3ac54134ef
--- /dev/null
+++ b/doc/forum/two_lines_back_to_remote__63__.mdwn
@@ -0,0 +1,10 @@
+`git annex map` generates a diagram roughly like this:
+
+    +------+    +-------+
+    |laptop| -> |usbdisk|
+    |      | <- |       |
+    |      | <- |       |
+    +------+    +-------+
+
+Why are there *two* lines back from the disk to the laptop?
+
diff --git a/doc/forum/two_lines_back_to_remote__63__/comment_1_064d786c5550e4470440717994e62051._comment b/doc/forum/two_lines_back_to_remote__63__/comment_1_064d786c5550e4470440717994e62051._comment
new file mode 100644
index 0000000000..bb551908b6
--- /dev/null
+++ b/doc/forum/two_lines_back_to_remote__63__/comment_1_064d786c5550e4470440717994e62051._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="https://openid.stackexchange.com/user/e65e6d0e-58ba-41de-84cc-1f2ba54cf574"
+ nickname="Mica"
+ subject="look at all your remotes"
+ date="2015-03-29T20:39:00Z"
+ content="""
+Have a look at all your remotes, you can reference the same repo with more than one remote name.
+"""]]
diff --git a/doc/forum/two_lines_back_to_remote__63__/comment_2_b1debe6f63e811c79ad97b9e38dd2935._comment b/doc/forum/two_lines_back_to_remote__63__/comment_2_b1debe6f63e811c79ad97b9e38dd2935._comment
new file mode 100644
index 0000000000..1bd4329393
--- /dev/null
+++ b/doc/forum/two_lines_back_to_remote__63__/comment_2_b1debe6f63e811c79ad97b9e38dd2935._comment
@@ -0,0 +1,7 @@
+[[!comment format=mdwn
+ username="joey"
+ subject="""comment 2"""
+ date="2015-04-06T16:59:07Z"
+ content="""
+
+"""]]
diff --git a/doc/forum/two_lines_back_to_remote__63__/comment_3_70ee40b99e06c4f64d3e2e4433c8b4ba._comment b/doc/forum/two_lines_back_to_remote__63__/comment_3_70ee40b99e06c4f64d3e2e4433c8b4ba._comment
new file mode 100644
index 0000000000..7f7389b8b8
--- /dev/null
+++ b/doc/forum/two_lines_back_to_remote__63__/comment_3_70ee40b99e06c4f64d3e2e4433c8b4ba._comment
@@ -0,0 +1,12 @@
+[[!comment format=mdwn
+ username="joey"
+ subject="""comment 3"""
+ date="2015-04-06T16:59:10Z"
+ content="""
+The map shows one line for each git remote. So if you have two git remotes
+configured that are both pointing at the laptop, that'll be the result.
+For example, you might have both "origin" and "laptop" remotes pointing to
+it, possibly via two different urls.
+
+There's nothing bad about this.
+"""]]
diff --git a/doc/forum/ui.mdwn b/doc/forum/ui.mdwn
new file mode 100644
index 0000000000..2ef8841a9e
--- /dev/null
+++ b/doc/forum/ui.mdwn
@@ -0,0 +1,11 @@
+I just briefly tried git-annex. It's an interesting concept.
+
+My main frustration so far: I find it difficult to visualise and use the annex. Shell script contortions can obviously show you anything, but I don't have the ready knowledge to compose them. It would be nice to have some more user-friendly commands or even a GUI.
+
+'annex whereis' is fine for one file, but unweildly for many (e.g. a directory whose contents are always together, never divided between repositories). Is it possible to treat a given directory and all its contents as a single object?
+
+It is not clear what files are immediately available. Sometimes you don't care about annexed files that aren't stored in the current repository. Might it be nice to temporarily remove or hide symlinks for files that are not here right now? Then you could treat the repository more like a normal file heirarchy. Or how about something like 'git annex ls' to show only currently available files?
+
+assistant and sharebox fs sound like great basic synch options, but it would be really cool to have a fuse fs somewhere between raw git-annex and assistant. Putting a file in the directory would trigger git-annex to add it and sync. rm  on that new file would not work by default because it's the only copy. rm --annex to get rid of the file from all repos. mv/cp   to transfer between repos. ls to show only the files that are here now, ls --annex to display complete information in a clear way (well-formatted, colorized). Something like that.
+
+This has just been a random brain dump from a new user, hopefully it made some sense.
diff --git a/doc/forum/ui/comment_1_f3e3446b05d6b573e29e6cad300fb635._comment b/doc/forum/ui/comment_1_f3e3446b05d6b573e29e6cad300fb635._comment
new file mode 100644
index 0000000000..84f30c5ea7
--- /dev/null
+++ b/doc/forum/ui/comment_1_f3e3446b05d6b573e29e6cad300fb635._comment
@@ -0,0 +1,10 @@
+[[!comment format=mdwn
+ username="http://joeyh.name/"
+ ip="4.154.0.140"
+ subject="comment 1"
+ date="2013-07-20T19:51:19Z"
+ content="""
+It's pretty easy to configure ls to show broken symlinks in a different color. Run dircolors, and alias ls='ls --color==auto'. Then you can see which files are not there at a glance.
+
+The Linux file manager's I've tried (nautilus and thunar) also shows which files are present; you get a X on the icon of annexed files that are not present.
+"""]]
diff --git a/doc/forum/ui/comment_2_b493ee97eb2378e72c12f3d137109580._comment b/doc/forum/ui/comment_2_b493ee97eb2378e72c12f3d137109580._comment
new file mode 100644
index 0000000000..ee79958832
--- /dev/null
+++ b/doc/forum/ui/comment_2_b493ee97eb2378e72c12f3d137109580._comment
@@ -0,0 +1,15 @@
+[[!comment format=mdwn
+ username="http://mylesenglish.myopenid.com/"
+ ip="217.39.94.195"
+ subject="comment 2"
+ date="2013-09-16T19:21:31Z"
+ content="""
+This shell function behaves more like ls and shows the files that are present:
+
+    ga-ls () {
+      CWD=`pwd`
+      cd ${1:-.} && git annex find | cut -d / -f 1 | uniq
+      cd ${CWD}
+    }
+
+"""]]
diff --git a/doc/forum/unable_to_clone_annex_repo_in_windows.mdwn b/doc/forum/unable_to_clone_annex_repo_in_windows.mdwn
new file mode 100644
index 0000000000..6e58c7af8a
--- /dev/null
+++ b/doc/forum/unable_to_clone_annex_repo_in_windows.mdwn
@@ -0,0 +1,29 @@
+Dear all,
+        I successfuly cloned my annex repo (hosted on gitlab.com) on my windows box (win 7, 64bits), and my data is on an exFat partition. 
+git annex version is 
+----------------
+git-annex version: 6.20160126-g2336107
+build flags: Assistant Webapp Pairing Testsuite S3(multipartupload) WebDAV ConcurrentOutput TorrentParser Feeds Quvi
+key/value backends: SHA256E SHA256 SHA512E SHA512 SHA224E SHA224 SHA384E SHA384 SHA3_256E SHA3_256 SHA3_512E SHA3_512 SHA3_224E SHA3_224 SHA3_384E SHA3_384 SKEIN256E SKEIN256 SKEIN512E SKEIN512 SHA1E SHA1 MD5E MD5 WORM URL
+remote types: git gcrypt S3 bup directory rsync web bittorrent webdav tahoe glacier ddar hook external
+-------------
+However, when I try to initialize annex on the newly cloned repo fails with the following error:
+
+$ git annex init "repo_win"
+init repo_win
+  Detected a filesystem without fifo support.
+
+  Disabling ssh connection caching.
+
+  Detected a crippled filesystem.
+
+  Enabling direct mode.
+
+git-annex: .git\annex\objects\c89\57a\SHA256E-s1339658--e7cfebb4ac9c81a6a060506c537b55aed9f13bd02d861d2b341fb8a79ad6e046.png\: openTempFile: does not exist (No such file or directory)
+failed
+git-annex: init: 1 failed
+
+any hints about this?
+Thanks in advance.
+Best regards,
+             Juan
diff --git a/doc/forum/unable_to_clone_annex_repo_in_windows/comment_1_588af0e0b655966c2d344f3f4b4d2e5f._comment b/doc/forum/unable_to_clone_annex_repo_in_windows/comment_1_588af0e0b655966c2d344f3f4b4d2e5f._comment
new file mode 100644
index 0000000000..b7ab2df61f
--- /dev/null
+++ b/doc/forum/unable_to_clone_annex_repo_in_windows/comment_1_588af0e0b655966c2d344f3f4b4d2e5f._comment
@@ -0,0 +1,18 @@
+[[!comment format=mdwn
+ username="joey"
+ subject="""comment 1"""
+ date="2016-05-03T16:55:42Z"
+ content="""
+That sounds like this bug:
+
+
+Windows has some path length issues that come up especially when using
+git-annex in a deeply nested directory tree. Try cloning the repository to
+somewhere like c:\\ and see if that works around the problem.
+
+Although, the path it's complaining about is relative and only 113
+characters long, which should not be long enough to trigger that problem.
+So, this could be some other problem. I tried cloning a git-annex repo
+and git annex init did not fail like this, so any information you can
+provide to help me reproduce the problem would be good.
+"""]]
diff --git a/doc/forum/unable_to_clone_annex_repo_in_windows/comment_2_e4fcc8329b2deb058b7368ce1c226a3e._comment b/doc/forum/unable_to_clone_annex_repo_in_windows/comment_2_e4fcc8329b2deb058b7368ce1c226a3e._comment
new file mode 100644
index 0000000000..828906df7e
--- /dev/null
+++ b/doc/forum/unable_to_clone_annex_repo_in_windows/comment_2_e4fcc8329b2deb058b7368ce1c226a3e._comment
@@ -0,0 +1,11 @@
+[[!comment format=mdwn
+ username="drunken_sapo"
+ subject="comment 2"
+ date="2016-05-13T18:12:24Z"
+ content="""
+Hey Joey,
+        Thanks for the answer, I'm trying to reproduce it on a test repo, but I'm having problems with my ssh-key. As soon as I fix it I'll be back with the results.
+Thanks for your time and effort.
+Best,
+         Juan
+"""]]
diff --git a/doc/forum/unable_to_clone_annex_repo_in_windows/comment_3_20aa8de2513fdb6b19a4b24f20983f0a._comment b/doc/forum/unable_to_clone_annex_repo_in_windows/comment_3_20aa8de2513fdb6b19a4b24f20983f0a._comment
new file mode 100644
index 0000000000..e9e462b561
--- /dev/null
+++ b/doc/forum/unable_to_clone_annex_repo_in_windows/comment_3_20aa8de2513fdb6b19a4b24f20983f0a._comment
@@ -0,0 +1,12 @@
+[[!comment format=mdwn
+ username="drunken_sapo"
+ subject="The problem was path length"
+ date="2016-05-14T13:11:35Z"
+ content="""
+After fixing my ssh key problem I was able to test it appropiately.
+The path to the .git directory of my repo is 41 characters long (/d/juan/develop/git/xxxxxx_xxxxx_xxxxxxxx), if we add the offending file name: .git\annex\objects\fca\de3\SHA256E-s1312136--a79e30c5121f7843f023abbd36b211a245385b952926760d55a07baf8da1b24d.png\
+which is 114 characters long, it adds up to 155 characters.
+Moving the repo to /d/xxxxxx_xxxxx_xxxxxxxx solved the problem.
+However, I need it to be on that path, because some other projects use this on a relative path.
+Is there any way to circumvent this issue?
+"""]]
diff --git a/doc/forum/unannex_--fast_+_uninit_leaves_files_in_.git__47__annex__47__objects__63__.mdwn b/doc/forum/unannex_--fast_+_uninit_leaves_files_in_.git__47__annex__47__objects__63__.mdwn
new file mode 100644
index 0000000000..c746e3d6d9
--- /dev/null
+++ b/doc/forum/unannex_--fast_+_uninit_leaves_files_in_.git__47__annex__47__objects__63__.mdwn
@@ -0,0 +1,47 @@
+I have been trying to uninit a very large directory and ran into problems.
+
+`git annex unannex` seems to be completely infeasible because of how long it takes, also it seemed to grow the directory enormously, probably because it copies large files and then waits to delete them?
+
+I tried unannexing with the following commands instead:
+
+    `git annex get --from=backup`
+    `git annex unannex --fast`
+    `git annex uninit`
+
+This does what it is supposed to it seems, it hard links all copies of files. However, the unint step gives the following message:
+
+    git-annex: Not fully uninitialized
+    Some annexed data is still left in /science/carlo/GR_Coverage_Manuscript_Revisions_140616/.git/annex/objects/
+    This may include deleted files, or old versions of modified files.
+
+    If you don't care about preserving the data, just delete the
+    directory.
+
+    Or, you can move it to another location, in case it turns out
+    something in there is important.
+
+    Or, you can run `git annex unused` followed by `git annex dropunused`
+    to remove data that is not used by any tag or branch, which might
+    take care of all the data.
+
+    Then run `git annex uninit` again to finish.
+
+However, running `git annex unused` returns nothing.
+
+When I run `du -l -h --max-depth=1` I get the following output:
+
+    646G ./01-Collate_New_Species_Data
+    6.3G ./02-Prep_Annotations
+    1.8T ./03-Map_Riboprofiling_Data
+    111G ./04-Generate_Preprocessed_Files
+    94G  ./05-Det_Codon_Specfic_Occupancy
+    3.6T ./.git
+    6.2T .
+
+The .git/annex/objects directory remains 3.5TB in size, while the root directory, minus the contents of .git is only 2.7TB.
+
+I want to delete the .git folder to free up space, but I am very nervous now, because it looks like there is extra data in git annex that isn't present in the main directory. I don't want that to be lost, that would be a complete disaster.
+
+I tried the exact same process on a test directory, and it seemed to work completely, but losing this data would be an absolute disaster, so I don't want to risk it.
+
+Any thoughts?
diff --git a/doc/forum/unannex_--fast_+_uninit_leaves_files_in_.git__47__annex__47__objects__63__/comment_1_4aaf93801119b36a01e452c7bb0fc7e9._comment b/doc/forum/unannex_--fast_+_uninit_leaves_files_in_.git__47__annex__47__objects__63__/comment_1_4aaf93801119b36a01e452c7bb0fc7e9._comment
new file mode 100644
index 0000000000..86cd83d3a7
--- /dev/null
+++ b/doc/forum/unannex_--fast_+_uninit_leaves_files_in_.git__47__annex__47__objects__63__/comment_1_4aaf93801119b36a01e452c7bb0fc7e9._comment
@@ -0,0 +1,18 @@
+[[!comment format=mdwn
+ username="joey"
+ subject="""comment 1"""
+ date="2014-10-20T15:33:02Z"
+ content="""
+Well, are there any files in your repository (outside .git) that
+are still symlinks to content in .git? If not, you know that
+every file in the repository's working tree has been unannexed
+ok.
+
+The remaining files in .git/annex/objects are not unused, so some branch or tag
+must refer to those files.
+
+You might try running `git log --stat -S'KEY'`
+where KEY is the basename name of one of the files in .git/annex/objects.
+This will find commits to the repo that refer to that object, so you'll
+know where it was used and what filename corresponded to it.
+"""]]
diff --git a/doc/forum/unannex_alternatives.mdwn b/doc/forum/unannex_alternatives.mdwn
new file mode 100644
index 0000000000..efd05838e6
--- /dev/null
+++ b/doc/forum/unannex_alternatives.mdwn
@@ -0,0 +1,9 @@
+what is the work flow to get a file that is in git-annex out of there and into git? (current situation: `git-annex add`ed a bunch of pictures, later found make files in there which i'd rather have in git for proper source code control)
+
+the most intuitive thing to do is `git unannex`, which at first seemed to do the right thing, but when committing there came the hook and everything was back to where it was before.
+
+i could disable the hook as a workaround, but that doesn't smell like a good work flow.
+
+the [[man page|git-annex]] does warn that `unannex` is only supposed to be used against unintentional `git annex add`s (probably meaning that it should be used before something is committed), but the alternatives it suggests (`git rm` and `git annex drop`) don't to what i want to do.
+
+am i missing something or is there really no work flow for this? --[[chrysn]]
diff --git a/doc/forum/unannex_alternatives/comment_1_dcd4cd41280b41512bbdffafaf307993._comment b/doc/forum/unannex_alternatives/comment_1_dcd4cd41280b41512bbdffafaf307993._comment
new file mode 100644
index 0000000000..7f278d2bc9
--- /dev/null
+++ b/doc/forum/unannex_alternatives/comment_1_dcd4cd41280b41512bbdffafaf307993._comment
@@ -0,0 +1,46 @@
+[[!comment format=mdwn
+ username="http://joey.kitenet.net/"
+ nickname="joey"
+ subject="comment 1"
+ date="2011-02-02T00:39:10Z"
+ content="""
+Git-annex's commit hook does not prevent unannex being used. The file you unannex will not be checked into git anymore and will be a regular file again, not a git-annex symlink. 
+
+For example, here's a transcript:
+
+
    +joey@gnu:~/tmp>mkdir demo
+joey@gnu:~/tmp>cd demo
+joey@gnu:~/tmp/demo>git init
+Initialized empty Git repository in /home/joey/tmp/demo/.git/
+joey@gnu:~/tmp/demo>git annex init demo
+init demo ok
+joey@gnu:~/tmp/demo>echo hi > file
+joey@gnu:~/tmp/demo>git annex add file 
+add file ok
+(Recording state in git...)
+joey@gnu:~/tmp/demo>git commit -m add
+[master 64cf267] add
+ 2 files changed, 2 insertions(+), 0 deletions(-)
+ create mode 100644 .git-annex/WORM:1296607093:3:file.log
+ create mode 120000 file
+joey@gnu:~/tmp/demo>git annex unannex file
+unannex file ok
+(Recording state in git...)
+joey@gnu:~/tmp/demo>ls -l file
+-rw-r--r-- 1 joey joey 3 Feb  1 20:38 file
+joey@gnu:~/tmp/demo>git commit
+[master 78a09cc] unannex
+ 2 files changed, 1 insertions(+), 2 deletions(-)
+ delete mode 120000 file
+joey@gnu:~/tmp/demo>ls -l file
+-rw-r--r-- 1 joey joey 3 Feb  1 20:38 file
+joey@gnu:~/tmp/demo>git status
+# On branch master
+# Untracked files:
+#   (use \"git add ...\" to include in what will be committed)
+#
+#	file
+nothing added to commit but untracked files present (use \"git add\" to track)
+
    
+"""]]
diff --git a/doc/forum/unannex_alternatives/comment_2_58a72a9fe0f58c7af0b4d7927a2dd21d._comment b/doc/forum/unannex_alternatives/comment_2_58a72a9fe0f58c7af0b4d7927a2dd21d._comment
new file mode 100644
index 0000000000..91ddadf8c6
--- /dev/null
+++ b/doc/forum/unannex_alternatives/comment_2_58a72a9fe0f58c7af0b4d7927a2dd21d._comment
@@ -0,0 +1,36 @@
+[[!comment format=mdwn
+ username="http://joey.kitenet.net/"
+ nickname="joey"
+ subject="comment 2"
+ date="2011-02-02T00:41:24Z"
+ content="""
+And following on to my transcript, you can then add the file to git in the regular git way, and it works fine:
+
+
    +joey@gnu:~/tmp/demo>git add file
+joey@gnu:~/tmp/demo>git commit
+[master 225ffc0] added as regular git file, not in annex
+ 1 files changed, 1 insertions(+), 0 deletions(-)
+ create mode 100644 file
+joey@gnu:~/tmp/demo>ls -l file
+-rw-r--r-- 1 joey joey 3 Feb  1 20:38 file
+joey@gnu:~/tmp/demo>git log file
+commit 225ffc048f5af7c0466b3b1fe549a6d5e9a9e9fe
+Author: Joey Hess 
+Date:   Tue Feb 1 20:43:13 2011 -0400
+
+    added as regular git file, not in annex
+
+commit 78a09cc791b875c3b859ca9401e5b6472bf19d08
+Author: Joey Hess 
+Date:   Tue Feb 1 20:38:30 2011 -0400
+
+    unannex
+
+commit 64cf267734adae05c020d9fd4d5a7ff7c64390db
+Author: Joey Hess 
+Date:   Tue Feb 1 20:38:18 2011 -0400
+
+    add
+
    
+"""]]
diff --git a/doc/forum/unannex_alternatives/comment_3_b1687fc8f9e7744327bbeb6f0635d1cd._comment b/doc/forum/unannex_alternatives/comment_3_b1687fc8f9e7744327bbeb6f0635d1cd._comment
new file mode 100644
index 0000000000..9f3223578b
--- /dev/null
+++ b/doc/forum/unannex_alternatives/comment_3_b1687fc8f9e7744327bbeb6f0635d1cd._comment
@@ -0,0 +1,16 @@
+[[!comment format=mdwn
+ username="http://joey.kitenet.net/"
+ nickname="joey"
+ subject="comment 3"
+ date="2011-02-02T00:46:00Z"
+ content="""
+Sorry for all the followups, but I see now that if you unannex, then add the file to git normally, and commit, the hook *does* misbehave.
+
+This seems to be a bug. git-annex's hook thinks that you have used git annex unlock (or \"git annex edit\") on the file and are now committing a changed version, and the right thing to do there is to add the new content to the annex and update the symlink accordingly. I'll track this bug over at [[bugs/unannex_vs_unlock_hook_confusion]].
+
+So, committing after unannex, and before checking the file into git in the
+usual way, is a workaround. But only if you do a "git commit" to commit
+staged changes.
+
+Anyway, this confusing point is fixed in git now!
+"""]]
diff --git a/doc/forum/unknown_response_from_git_cat-file.mdwn b/doc/forum/unknown_response_from_git_cat-file.mdwn
new file mode 100644
index 0000000000..e4c559815f
--- /dev/null
+++ b/doc/forum/unknown_response_from_git_cat-file.mdwn
@@ -0,0 +1,8 @@
+Hi,
+
+when running git annex add in my direct mode repository, since a few days ago I only get:
+
+$ git annex add
+git-annex: unknown response from git cat-file (":./Archiv/Someone missing",:./Archiv/Someone Like You Cover-fCvjvEGkTu4.flv)
+
+The :./Archiv/Someone missing part strikes me odd because it so much looks like broken shell meta-character escaping in git-annex, but I doubt that because it stopped working just suddenly.
diff --git a/doc/forum/unknown_response_from_git_cat-file/comment_1_f26ba569e715fe69b6de3093930362ee._comment b/doc/forum/unknown_response_from_git_cat-file/comment_1_f26ba569e715fe69b6de3093930362ee._comment
new file mode 100644
index 0000000000..0ddef17af4
--- /dev/null
+++ b/doc/forum/unknown_response_from_git_cat-file/comment_1_f26ba569e715fe69b6de3093930362ee._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="http://joeyh.name/"
+ ip="2001:4978:f:21a::2"
+ subject="comment 1"
+ date="2013-08-07T16:34:18Z"
+ content="""
+This is a bug in the 1.8.4 pre-release version of git. It will be fixed in the 1.8.4 release or another pre-release. git-annex version 4.20130802 has a workaround for this problem.
+"""]]
diff --git a/doc/forum/unknown_response_from_git_cat-file/comment_2_bba450e83609225c161b4fec1e006f8a._comment b/doc/forum/unknown_response_from_git_cat-file/comment_2_bba450e83609225c161b4fec1e006f8a._comment
new file mode 100644
index 0000000000..07d1d4de30
--- /dev/null
+++ b/doc/forum/unknown_response_from_git_cat-file/comment_2_bba450e83609225c161b4fec1e006f8a._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="https://www.google.com/accounts/o8/id?id=AItOawmNvX8PQVP5sLzQ78sKpB6VeH3Fu8HvZ5g"
+ nickname="Jeff"
+ subject="comment 2"
+ date="2015-04-01T01:08:53Z"
+ content="""
+I'm getting a similar error on git-annex version: 5.20141203, but I'm not in direct mode. It happens when I call 'git annex unused'.
+"""]]
diff --git a/doc/forum/unknown_response_from_git_cat-file/comment_3_8e0458e86764242e9e35940b4db302b7._comment b/doc/forum/unknown_response_from_git_cat-file/comment_3_8e0458e86764242e9e35940b4db302b7._comment
new file mode 100644
index 0000000000..e03348a423
--- /dev/null
+++ b/doc/forum/unknown_response_from_git_cat-file/comment_3_8e0458e86764242e9e35940b4db302b7._comment
@@ -0,0 +1,32 @@
+[[!comment format=mdwn
+ username="GregGrossmeier"
+ avatar="http://cdn.libravatar.org/avatar/66db9e436738991619a59e8ab8061e64"
+ subject="Recurring?"
+ date="2017-01-02T21:13:47Z"
+ content="""
+Still getting this myself. First I've ran into it (a few weeks ago, but I hadn't used these repos recently).
+
+	greg@x230 (git)-[master] ~/Photos % git-annex add .               
+	add 2016/10/16/2016-10-16 07.47.52.jpg git-annex: unknown response from git cat-file (\"HEAD:./2016/10/16/2016-10-16 07.47.52.jpg missing\",Ref \"HEAD:./2016/10/16/2016-10-16 07.47.52.jpg\")
+	greg@x230 (git)-[master] ~/Photos % apt-cache policy git-annex git
+	zsh: correct 'git' to '.git'? [N/y/a/e] n
+	git-annex:
+	  Installed: 6.20161012-1
+	  Candidate: 6.20161012-1
+	  Version table:
+	 *** 6.20161012-1 500
+	        500 http://httpredir.debian.org/debian testing/main amd64 Packages
+	        100 /var/lib/dpkg/status
+	     5.20141125 500
+	        500 http://httpredir.debian.org/debian stable/main amd64 Packages
+	git:
+	  Installed: 1:2.11.0-1
+	  Candidate: 1:2.11.0-1
+	  Version table:
+	 *** 1:2.11.0-1 500
+	        500 http://httpredir.debian.org/debian testing/main amd64 Packages
+	        100 /var/lib/dpkg/status
+	     1:2.1.4-2.1+deb8u2 500
+	        500 http://httpredir.debian.org/debian stable/main amd64 Packages
+	
+"""]]
diff --git a/doc/forum/unknown_response_from_git_cat-file/comment_4_275fcf1e6643cb247f8ed5afc4c69e56._comment b/doc/forum/unknown_response_from_git_cat-file/comment_4_275fcf1e6643cb247f8ed5afc4c69e56._comment
new file mode 100644
index 0000000000..ad6731ab44
--- /dev/null
+++ b/doc/forum/unknown_response_from_git_cat-file/comment_4_275fcf1e6643cb247f8ed5afc4c69e56._comment
@@ -0,0 +1,15 @@
+[[!comment format=mdwn
+ username="fossil"
+ avatar="http://cdn.libravatar.org/avatar/951f4f4e1dca2ebe880ddb392c2d3e73"
+ subject="comment 4"
+ date="2017-01-08T15:57:42Z"
+ content="""
+This is a bug, introduced in [commit 34530e59](https://github.com/joeyh/git-annex/commit/34530e59) (release 6.20161012) and fixed in [commit b530e432](https://github.com/joeyh/git-annex/commit/b530e432) (release 6.20161031).
+
+* [git-annex chokes on filenames including spaces](https://git-annex.branchable.com/bugs/git-annex_chokes_on_filenames_including_spaces/)
+* [Single space in file name causes git annex add to fail](https://git-annex.branchable.com/bugs/Single_space_in_file_name_causes_git_annex_add_to_fail/)
+
+It happens only when the filename contains a single space, so a workaround is to first add a filename without any spaces, and then rename it:
+
+    fn=\"foo bar.txt\"; cp \"$fn\" tmp && git annex add tmp && git mv -f tmp \"$fn\"
+"""]]
diff --git a/doc/forum/unlock__47__lock_always_gets_me.mdwn b/doc/forum/unlock__47__lock_always_gets_me.mdwn
new file mode 100644
index 0000000000..0891eed0e5
--- /dev/null
+++ b/doc/forum/unlock__47__lock_always_gets_me.mdwn
@@ -0,0 +1,11 @@
+Several times now I've done something like:
+
+    $ git annex unlock movie.avi
+    $ mv /tmp/fixed.avi movie.avi
+    $ git annex lock movie.avi
+
+Oops, I just lost my fixed.avi!  That really feels like the right sequence of operations to me, so I'm always surprised when I make that mistake.  I would like to see the current `lock` renamed to something like `undo-unlock`, or have the behavior changed to be the same as `add`, or maybe warn and require a `--force` when the file has been changed.
+
+If changing current behavior is undesirable, maybe `unlock` could just print a reminder that `git annex add` is the correct next step after making changes?
+
+Failing that, I suppose I could slowly start to learn from my mistakes.
diff --git a/doc/forum/unlock__47__lock_always_gets_me/comment_1_dee73a7ea3e1a5154601adb59782831f._comment b/doc/forum/unlock__47__lock_always_gets_me/comment_1_dee73a7ea3e1a5154601adb59782831f._comment
new file mode 100644
index 0000000000..c37561665f
--- /dev/null
+++ b/doc/forum/unlock__47__lock_always_gets_me/comment_1_dee73a7ea3e1a5154601adb59782831f._comment
@@ -0,0 +1,12 @@
+[[!comment format=mdwn
+ username="http://joey.kitenet.net/"
+ nickname="joey"
+ subject="comment 1"
+ date="2012-01-07T17:15:31Z"
+ content="""
+Well, lock could check for modifications and require --force to lose them. But the check could be expensive for large files.
+
+But `git annex lock` is just a convenient way to run `git checkout`. And running `git checkout` or `git reset --hard` will lose your uncommitted file the same way obviously. 
+
+Perhaps the best fix would be to get rid of `lock` entirely, and let the user use the underlying git commands same as they would to drop modifications to other files. It would then also make sense to remove `unlock`, leaving only `edit`.
+"""]]
diff --git a/doc/forum/unlock__47__lock_always_gets_me/comment_2_f89b4349dde840c355a3bc28908decdf._comment b/doc/forum/unlock__47__lock_always_gets_me/comment_2_f89b4349dde840c355a3bc28908decdf._comment
new file mode 100644
index 0000000000..f732c87da0
--- /dev/null
+++ b/doc/forum/unlock__47__lock_always_gets_me/comment_2_f89b4349dde840c355a3bc28908decdf._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="https://www.google.com/accounts/o8/id?id=AItOawnRai_qFYPVvEgC6i1nlM1bh-C__jbhqS0"
+ nickname="Matthew"
+ subject="comment 2"
+ date="2013-09-07T12:26:46Z"
+ content="""
+I too was totally flummoxed by this.
+"""]]
diff --git a/doc/forum/unlock__47__lock_always_gets_me/comment_3_acbab7b75726d34dccb5c9dab7b3e728._comment b/doc/forum/unlock__47__lock_always_gets_me/comment_3_acbab7b75726d34dccb5c9dab7b3e728._comment
new file mode 100644
index 0000000000..835222f190
--- /dev/null
+++ b/doc/forum/unlock__47__lock_always_gets_me/comment_3_acbab7b75726d34dccb5c9dab7b3e728._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="http://olivier.berger.myopenid.com/"
+ nickname="obergix"
+ subject="It seems we've been heard"
+ date="2013-12-07T15:13:10Z"
+ content="""
+https://github.com/joeyh/git-annex/commit/cde099fe80cd2bc6345496c9266c41baf46c1976 : lock: Require --force. 
+"""]]
diff --git a/doc/forum/unrelated_repositories_sync.mdwn b/doc/forum/unrelated_repositories_sync.mdwn
new file mode 100644
index 0000000000..030fb380d7
--- /dev/null
+++ b/doc/forum/unrelated_repositories_sync.mdwn
@@ -0,0 +1,15 @@
+I have no idea how to search for this here, so I'll just go the "lazy web" approach and just ask.
+
+Say I have two "conference" repos. One is the famous [conference procedings](https://github.com/RichiH/conference_proceedings) repo, and another one is a totally unrelated repo of local conferences that are not of world-wide significance. Let's call this second repo `presentations`.
+
+I would like to have my videos of both repos in a single repo.
+
+Can I add the `conference procedings` repo as a git remote to the `presentations` repo and have it do the right thing?
+
+In fact, I'm not even sure what the right thing would be here, I guess that's the first thing I would like to clear up. But I would like to do things like what the new [[metadata]] system does. For example, I would have only the "Debian" directory from `conference procedings` in my `presentations` repo.
+
+How would that work? Would I need to do some [subtree merging](http://git-scm.com/book/ch6-7.html) magic? or `git subtree`? or submodules? or should i just use myrepos and pretend I never brought up this idea?
+
+thanks! -- [[anarcat]]
+
+related: [[tips/migrating_two_seperate_disconnected_directories_to_git_annex/]] - but that creates a merged repo...
diff --git a/doc/forum/unrelated_repositories_sync/comment_1_c899b7b05a96d14e25c2efadff3b4e52._comment b/doc/forum/unrelated_repositories_sync/comment_1_c899b7b05a96d14e25c2efadff3b4e52._comment
new file mode 100644
index 0000000000..7df8a0eb93
--- /dev/null
+++ b/doc/forum/unrelated_repositories_sync/comment_1_c899b7b05a96d14e25c2efadff3b4e52._comment
@@ -0,0 +1,10 @@
+[[!comment format=mdwn
+ username="http://joeyh.name/"
+ ip="209.250.56.244"
+ subject="comment 1"
+ date="2014-04-02T19:51:06Z"
+ content="""
+It might help if you think about these two different repositories as branches. You have 2 branches with different files in them, and you want to produce a third branch with some mix of the two.
+
+I think git is perfectly capable of doing that. Where it gets hairy is dealing with merges when either of the 2 repositories change going forward. The same as if you've forked and modified source code, you will need to do *something* to resolve merges.
+"""]]
diff --git a/doc/forum/unsynced_folder.mdwn b/doc/forum/unsynced_folder.mdwn
new file mode 100644
index 0000000000..ceb04011e8
--- /dev/null
+++ b/doc/forum/unsynced_folder.mdwn
@@ -0,0 +1,3 @@
+If I have an archive with a lot of big files, is it possible to get one of them without using the command line, and without it popping up on other clients of that repository?
+Is it possible to have a folder that will not be synced with other clients but will still download the file when i copy it from archive?
+.gitignore probably wont work because that will not get the file from archive.
diff --git a/doc/forum/unsynced_folder/comment_1_7d7a262f067c7b02d76e82637223934c._comment b/doc/forum/unsynced_folder/comment_1_7d7a262f067c7b02d76e82637223934c._comment
new file mode 100644
index 0000000000..7f746df12c
--- /dev/null
+++ b/doc/forum/unsynced_folder/comment_1_7d7a262f067c7b02d76e82637223934c._comment
@@ -0,0 +1,22 @@
+[[!comment format=mdwn
+ username="http://joeyh.name/"
+ ip="209.250.56.87"
+ subject="comment 1"
+ date="2013-12-12T20:09:48Z"
+ content="""
+Sorry I took so long to answer. I think that it took me a few tries to understand your question.
+
+You want to have a special folder on a single clone of a repository, and be able to move files to that folder from the archive folder, and have it get the file content from the archive repository. But you don't want that file content to spread to any of the other repositories.
+
+This is possible to do by writing a custom [[preferred_content]] expression, and configuring the repository to use it. For simplicity, let's make the folder be called `archive/local`. This way, all the clients that do not use this preferred content expression will see that the file is still under the archive folder, and not want its content.
+
+Now we can take the preferred content expression used by client repositories normally:
+
+  ((exclude=*/archive/* and exclude=archive/*) or (not (copies=archive:1 or copies=smallarchive:1))) or (not copies=semitrusted+:1)
+
+And modify it:
+
+  ((exclude=*/archive/* and exclude=archive/* and (not */archive/local/*) and (not archive/local/*)) or (not (copies=archive:1 or copies=smallarchive:1))) or (not copies=semitrusted+:1)
+
+You can use `git annex vifg` to edit the preferred content expressions, and paste that in. Should work. Untested.
+"""]]
diff --git a/doc/forum/untitled.mdwn b/doc/forum/untitled.mdwn
new file mode 100644
index 0000000000..ce75143cf1
--- /dev/null
+++ b/doc/forum/untitled.mdwn
@@ -0,0 +1,5 @@
+Hi,
+
+How git-annex-get chooses remote when it's called without the '--from' option? Is it anyhow controllable to prefer specific remotes/set priorities?
+
+What about downloading multiple files? Does it download from multiple remotes at the same time to optimize for bandwidth?
diff --git a/doc/forum/untitled/comment_1_0176d1d50bd62b113ff690346ba4161b._comment b/doc/forum/untitled/comment_1_0176d1d50bd62b113ff690346ba4161b._comment
new file mode 100644
index 0000000000..5e030b962e
--- /dev/null
+++ b/doc/forum/untitled/comment_1_0176d1d50bd62b113ff690346ba4161b._comment
@@ -0,0 +1,15 @@
+[[!comment format=mdwn
+ username="joey"
+ subject="""comment 1"""
+ date="2016-09-05T16:34:35Z"
+ content="""
+Remotes have configurable costs, see `remote..annex-cost`
+documentation in [[git-annex]].
+
+`git annex get` and some other commands can be used with `-J3` to
+download multiple files at once. However, it only downloads
+each file from the cheapest remote that has a copy.
+
+It would perhaps be an interesting improvement to make it round-robin
+amoung remotes if there are multiple ones that have the same cost.
+"""]]
diff --git a/doc/forum/update_via_cabal_fails.mdwn b/doc/forum/update_via_cabal_fails.mdwn
new file mode 100644
index 0000000000..aae8461c6a
--- /dev/null
+++ b/doc/forum/update_via_cabal_fails.mdwn
@@ -0,0 +1,35 @@
+I tried to update git-annex via cabal. It fails due to lens not installing:
+
+    juh@sokrates:~$ cabal update
+    Downloading the latest package list from hackage.haskell.org
+    juh@sokrates:~$ cabal install git-annex
+    Resolving dependencies...
+    In order, the following will be installed:
+    lens-4.2 (new version)
+    DAV-0.6.2 (reinstall) changes: http-client-0.3.1.1 -> 0.3.3.1,
+    http-client-tls-0.2.1.1 added, http-types-0.8.4 -> 0.8.5, lens-4.1.2 ->     4.2,
+    network-2.4.1.2 added, optparse-applicative-0.7.0.2 added, xml-    conduit-1.2.0.1
+    -> 1.2.0.2, xml-hamlet-0.4.0.8 added
+    git-annex-5.20140707 (new package)
+    Warning: Note that reinstalls are always dangerous. Continuing anyway...
+    [1 of 1] Compiling Main             ( /tmp/lens-4.2-3107/lens-4.2/Setup.    lhs, /tmp/lens-4.2-3107/lens-4.2/dist/setup/Main.o )
+    Linking /tmp/lens-4.2-3107/lens-4.2/dist/setup/setup ...
+    Configuring lens-4.2...
+    Building lens-4.2...
+    Preprocessing library lens-4.2...
+    
+    src/Control/Lens/Internal/Zoom.hs:47:8:
+        Could not find module `Control.Monad.Trans.Except'
+        Perhaps you meant
+          Control.Monad.Trans.Cont (from transformers-0.3.0.0)
+          Control.Monad.Trans.Error (from transformers-0.3.0.0)
+          Control.Monad.Trans.List (from transformers-0.3.0.0)
+        Use -v to see a list of the files searched for.
+    Failed to install lens-4.2
+    cabal: Error: some packages failed to install:
+    DAV-0.6.2 depends on lens-4.2 which failed to install.
+    git-annex-5.20140707 depends on lens-4.2 which failed to install.
+    lens-4.2 failed during the building phase. The exception was:
+    ExitFailure 1
+
+I am not an experienced user of cabal. So any hints to solve the conflicts are appreciated.
diff --git a/doc/forum/update_via_cabal_fails/comment_1_e1235dc2acd3bac3dd51b7614dabbb88._comment b/doc/forum/update_via_cabal_fails/comment_1_e1235dc2acd3bac3dd51b7614dabbb88._comment
new file mode 100644
index 0000000000..71fd0e60f9
--- /dev/null
+++ b/doc/forum/update_via_cabal_fails/comment_1_e1235dc2acd3bac3dd51b7614dabbb88._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="http://joeyh.name/"
+ ip="209.250.56.55"
+ subject="comment 1"
+ date="2014-07-09T17:58:48Z"
+ content="""
+It seems like you must have transformers-compat 0.1.1, and lens's dependency is slightly wrong, accepting that extremly old version. You'll need to upgrade it.
+"""]]
diff --git a/doc/forum/updating_the___34__number_of_copies__34__.mdwn b/doc/forum/updating_the___34__number_of_copies__34__.mdwn
new file mode 100644
index 0000000000..c28d265307
--- /dev/null
+++ b/doc/forum/updating_the___34__number_of_copies__34__.mdwn
@@ -0,0 +1,14 @@
+Is it possible to let git-annex check and update the "number of copies" or "whereis" information without recalculating the checksome?
+
+The use case is this:
+
+I have a very large "repo1": 300.000 files 1.5 TB.
+
+I copy it using low level tools for performance (clone the partition, netcat over network, or whatever...)
+
+Then i do "git annex init 'repo2'". 
+All files exist in both repos and the simlinks are valid, but git-annex does not know about it. 
+The "git-annex wehereis" only shows "repo1". How can I tell him without rehashing 1.6TB?
+
+Thx
+
diff --git a/doc/forum/updating_the___34__number_of_copies__34__/comment_1_327bdb0d9c190c60c7147b3acf07af09._comment b/doc/forum/updating_the___34__number_of_copies__34__/comment_1_327bdb0d9c190c60c7147b3acf07af09._comment
new file mode 100644
index 0000000000..2fd4ffaee0
--- /dev/null
+++ b/doc/forum/updating_the___34__number_of_copies__34__/comment_1_327bdb0d9c190c60c7147b3acf07af09._comment
@@ -0,0 +1,10 @@
+[[!comment format=mdwn
+ username="http://joeyh.name/"
+ ip="4.154.6.49"
+ subject="comment 1"
+ date="2012-12-01T18:40:46Z"
+ content="""
+Well, first of all, after copying a repository like this, you need to edit its .git/config and delete the annex.uuid setting. Otherwise, you will have two repositories with the same UUID, which is not good.
+
+Once you've done that, run `git annex fsck` in the new repository and it will do what you want.
+"""]]
diff --git a/doc/forum/updating_the___34__number_of_copies__34__/comment_2_7e11c839637e0894332e413cde02cee9._comment b/doc/forum/updating_the___34__number_of_copies__34__/comment_2_7e11c839637e0894332e413cde02cee9._comment
new file mode 100644
index 0000000000..269fc22bc9
--- /dev/null
+++ b/doc/forum/updating_the___34__number_of_copies__34__/comment_2_7e11c839637e0894332e413cde02cee9._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="https://www.google.com/accounts/o8/id?id=AItOawlgyVag95OnpvSzQofjyX0WjW__MOMKsl0"
+ nickname="Sehr"
+ subject="comment 2"
+ date="2012-12-01T19:27:59Z"
+ content="""
+Ok, that did the trick, except that it recalculates all cehcksums, which is exactly wat I do not want, as it is unnecessary and takes at 40MB/s just too long. Any other way? I think about adding the whereis info by hand, which simply feels wrong!
+"""]]
diff --git a/doc/forum/updating_the___34__number_of_copies__34__/comment_3_8b7a70fb3bb41e4eda412302834730bb._comment b/doc/forum/updating_the___34__number_of_copies__34__/comment_3_8b7a70fb3bb41e4eda412302834730bb._comment
new file mode 100644
index 0000000000..c130950906
--- /dev/null
+++ b/doc/forum/updating_the___34__number_of_copies__34__/comment_3_8b7a70fb3bb41e4eda412302834730bb._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="http://joeyh.name/"
+ ip="4.154.6.49"
+ subject="comment 3"
+ date="2012-12-01T19:37:26Z"
+ content="""
+git annex fsck --fast will skip the checksumming.
+"""]]
diff --git a/doc/forum/usability__58___creating_an_archive_on_a_new_external_drive.mdwn b/doc/forum/usability__58___creating_an_archive_on_a_new_external_drive.mdwn
new file mode 100644
index 0000000000..a6bc0b7bd6
--- /dev/null
+++ b/doc/forum/usability__58___creating_an_archive_on_a_new_external_drive.mdwn
@@ -0,0 +1,13 @@
+story: i got a new drive from the store. plug it in. it's recognized by XFCE and mounted automatically. turns out it's NTFS, but Debian doesn't seem to mind.
+
+go in the assistant, add it as an "External drive", make it a "full backup" and let it sync.
+
+expected result: i see files
+
+actual result: i see a bare git repository.
+
+if i first `git init` the directory so it's not bare, at least i get a better handle on it: i can sync with the assistant, then `git merge synced/master` to see the files.
+
+really confusing to me, sorry if it's a dupe... 
+
+workaround: use the commandline: `git clone ` and `git annex get`. --[[anarcat]]
diff --git a/doc/forum/usability__58___creating_an_archive_on_a_new_external_drive/comment_1_27b5283c65c402f330263426e4ca6ac1._comment b/doc/forum/usability__58___creating_an_archive_on_a_new_external_drive/comment_1_27b5283c65c402f330263426e4ca6ac1._comment
new file mode 100644
index 0000000000..1b5508f978
--- /dev/null
+++ b/doc/forum/usability__58___creating_an_archive_on_a_new_external_drive/comment_1_27b5283c65c402f330263426e4ca6ac1._comment
@@ -0,0 +1,10 @@
+[[!comment format=mdwn
+ username="http://joeyh.name/"
+ ip="209.250.56.22"
+ subject="comment 1"
+ date="2014-09-16T18:07:54Z"
+ content="""
+I started to make this change, and then I realized this problem: If a non-bare repository is made on an external drive, then to the user this is another place they can edit their files. Which means they will expect their changes made there to be committed. Which is highly problematic, because the assistant cannot be left running on an external drive or it won't be able to be unmounted. Or, a periodic `git annex add; git annex sync` could be run on the external drive, but that is a more expensive process (especially when run on a slow drive) and would not meet the expectations of users of the assistant that their changes will promptly propagate.
+
+So, I feel that leaving bare repositories is actually the best choice.
+"""]]
diff --git a/doc/forum/usability__58___creating_an_archive_on_a_new_external_drive/comment_2_b3a6b5ff0aaddd78903fc7bc7fbd6ee2._comment b/doc/forum/usability__58___creating_an_archive_on_a_new_external_drive/comment_2_b3a6b5ff0aaddd78903fc7bc7fbd6ee2._comment
new file mode 100644
index 0000000000..e264c8f08c
--- /dev/null
+++ b/doc/forum/usability__58___creating_an_archive_on_a_new_external_drive/comment_2_b3a6b5ff0aaddd78903fc7bc7fbd6ee2._comment
@@ -0,0 +1,10 @@
+[[!comment format=mdwn
+ username="https://id.koumbit.net/anarcat"
+ ip="70.83.139.100"
+ subject="comment 2"
+ date="2014-09-16T20:35:23Z"
+ content="""
+i wouldn't expect those changes to be committed. it's an external drive, and unless i manually sync it, it should take into account my changes.
+
+i am more surprised by the bare repository than i would be surprised by my changes not propagating back, i think. --[[anarcat]]
+"""]]
diff --git a/doc/forum/usability__58___creating_an_archive_on_a_new_external_drive/comment_3_aab934c0e37771d7f834d2567a9e76a1._comment b/doc/forum/usability__58___creating_an_archive_on_a_new_external_drive/comment_3_aab934c0e37771d7f834d2567a9e76a1._comment
new file mode 100644
index 0000000000..50d97fa269
--- /dev/null
+++ b/doc/forum/usability__58___creating_an_archive_on_a_new_external_drive/comment_3_aab934c0e37771d7f834d2567a9e76a1._comment
@@ -0,0 +1,13 @@
+[[!comment format=mdwn
+ username="http://joeyh.name/"
+ subject="comment 3"
+ date="2014-11-25T21:26:44Z"
+ content="""
+Of course you wouldn't expect that, I've explained the problem to you and you're technically adept and know about issues with keeping open files on removable drives.
+
+However, I have to consider the affordances of what the assistant sets up, and the natural affordance of a drive containing a directory with your files in it is to be able edit those files. And when changes in every other clone of that directory get automatically synced, the expectation would be for the same to hold true on this directory on the removable drive.
+
+I can see perhaps having the assistant run `git annex merge` in a removable repo after pushing to it (but `git annex sync` should not do that; violates least surprise for a git pull+merge+push wrapper like that to affect the work trees of other repos). As long as the assistant defaults to making removable repos bare, it won't expose normal users to the problem, and only advanced users who know how to set up non-bare repos. 
+
+If this is only for advanced users though, it (both assistant and `git annex sync`) could just as well run a configurable command on a removable repo after pushing to it. Or, the advanced user could use their skillz to make the removable drive be formatted with a reasonable filesystem that allows executable files, and then set up a git post-receive hook.
+"""]]
diff --git a/doc/forum/usability__58___creating_an_archive_on_a_new_external_drive/comment_4_42beace277e009ddff449cb220775d44._comment b/doc/forum/usability__58___creating_an_archive_on_a_new_external_drive/comment_4_42beace277e009ddff449cb220775d44._comment
new file mode 100644
index 0000000000..816a81da9f
--- /dev/null
+++ b/doc/forum/usability__58___creating_an_archive_on_a_new_external_drive/comment_4_42beace277e009ddff449cb220775d44._comment
@@ -0,0 +1,13 @@
+[[!comment format=mdwn
+ username="joey"
+ subject="""comment 4"""
+ date="2017-02-17T19:38:45Z"
+ content="""
+It's now pretty easy to set up a remote on a USB drive so that `git annex
+sync` will update the work tree. See 
+
+
+However, the assistant does not do this by default when adding repos on USB
+drives, due to the likely confusion that setting up such a repo would
+cause, as described in my previous comment.
+"""]]
diff --git a/doc/forum/usability__58___what_are_those_arrow_things__63__.mdwn b/doc/forum/usability__58___what_are_those_arrow_things__63__.mdwn
new file mode 100644
index 0000000000..cde7d8d505
--- /dev/null
+++ b/doc/forum/usability__58___what_are_those_arrow_things__63__.mdwn
@@ -0,0 +1,29 @@
+# Introduction
+
+i want to relate a usability story that happens fairly regularly when I show git-annex to people. the story goes like this.
+
+# The story
+
+Antoine sat down at his computer saying, "i have this great movie collection I want to share with you, my friend, because the fair use provisions allow for that, and I use this great git-annex tool that allows me to sync my movie collection between different places". His friend Charlie, a Linux user only vaguely familiar with the internals of how his operating system or legal system actually works, reads this as "yay free movies" and wholeheartedly agrees to lend himself to the experiment.
+
+Antoine creates a user account for Charlie on his home computer, because he doesn't want to have to do everything himself. "That way you can choose which movies you want, because you probably don't want my complete movie collection!" Charlie emphatically responds: "right, I only have my laptop and this USB key here, so I don't think I can get it all".
+
+Charlie logs into Antoine's computer, named `marcos`. Antoine shows Charlie where the movies are located (`/srv/video`) through the file browser (Thunar, for the record). Charlie inserts his USB key into `marcos` and a new icon for the USB key shows up. Then Charlie finds a video he likes, copies and pastes it into the USB key. But instead of a familiar progress bar, Charlie is prompted with a dialog that says "Le système de fichiers ne gère pas les liens symboliques." (Antoine is french, so excuse him, this weird message says that the filesystem doesn't support symbolic links.) Puzzled, Charlie tries to copy the file to his home directory instead. This works better, but the file has a little arrow on it, which seems odd to Charlie. He then asks Antoine for advice.
+
+Antoine then has no solution but to convert the git-annex repository into direct mode, something which takes a significant amount of time and is actually [[designated as "untrusted"|direct_mode]] in the documentation. In fact, so much so that he actually did [[screw up his repository magnificently|bugs/direct_command_leaves_repository_inconsistent_if_interrupted]] because he freaked out when `git-annex direct` started and interrupted it because he tought it would take too long.
+
+# Technical analysis
+
+Now I understand it is not necessarily `git-annex`'s responsability if Thunar (or Nautilus, for that matter), doesn't know how to properly deal with symlinks (hint: just dereference the damn thing already). Maybe I should file a bug about this against thunar? I also understand that symlinks are useful to ensure the security of the data hosted in `git-annex`, and that I could have used direct mode in the first place. But I like to track changes in git to those files, and direct mode makes that really difficult.
+
+I didn't file this as a bug because I want to start the conversation, but maybe it should qualify as a usability bug. As things stand, this is one of the biggest hurdle in teaching people about git annex.
+
+(The other being "how do i actually use git annex to sync those files instead of just copying them by hand", but that's for another story!)
+
+-- [[anarcat]]
+
+# Followup
+
+Here is a bug report filed against Thunar, with a patch to fix this behavior: https://bugzilla.xfce.org/show_bug.cgi?id=11065
+
+Similar bugs would need to be filed against Nautilus, at the very least, but probably other file managers, which makes this task a little daunting, to say the least. -- [[anarcat]]
diff --git a/doc/forum/usability__58___what_are_those_arrow_things__63__/comment_1_bf34c169c725f9504e0f2114ba53be4b._comment b/doc/forum/usability__58___what_are_those_arrow_things__63__/comment_1_bf34c169c725f9504e0f2114ba53be4b._comment
new file mode 100644
index 0000000000..86099c0d53
--- /dev/null
+++ b/doc/forum/usability__58___what_are_those_arrow_things__63__/comment_1_bf34c169c725f9504e0f2114ba53be4b._comment
@@ -0,0 +1,10 @@
+[[!comment format=mdwn
+ username="http://joeyh.name/"
+ ip="209.250.56.7"
+ subject="comment 1"
+ date="2014-08-12T19:49:35Z"
+ content="""
+If I wanted to share files with someone, I'd set them up with a direct mode repository and link it to my (probably indirect mode) repository.
+
+The question then becomes, how can this person decide which files to get if they don't want to or cannot get everything. I think that [[tips/File_manager_integration]] is a pretty good answer, although it does involve adding extensions to file managers. At least it involves adding something, rather than convincing a suprisingly large number of people that their ideas about symlinks are wrong. There are other possible answers, like building a file selection UI into the webapp..
+"""]]
diff --git a/doc/forum/usability__58___what_are_those_arrow_things__63__/comment_2_364ce8b369fd0ba7ddaec3127840ff39._comment b/doc/forum/usability__58___what_are_those_arrow_things__63__/comment_2_364ce8b369fd0ba7ddaec3127840ff39._comment
new file mode 100644
index 0000000000..32ecdf0df8
--- /dev/null
+++ b/doc/forum/usability__58___what_are_those_arrow_things__63__/comment_2_364ce8b369fd0ba7ddaec3127840ff39._comment
@@ -0,0 +1,16 @@
+[[!comment format=mdwn
+ username="https://id.koumbit.net/anarcat"
+ ip="70.83.139.100"
+ subject="comment 2"
+ date="2014-08-12T19:56:58Z"
+ content="""
+the problem with this is that you end up having two copies of the same files (the direct and indirect repositories). also, the switch to direct mode exploded (because i screwed up, granted)....
+
+i have been thinking more and more than the webapp needs to have some sort of file manager as well, but that seems like a huge undertaking...
+
+a better file manager integration could certainly allow to improve this experience. for me the requirements would be:
+
+* \"clone this repo to\" - make a copy of this git annex repo to the specified target
+* \"annex-copy those files to\" - the above + a file-transfer-like dialog that would track the total file transfer (as opposed to \"begin/end\" of single files, see also [[todo/do_not_bug_me_about_intermediate_files/]])
+* probably some more stuff
+"""]]
diff --git a/doc/forum/use_case__58___unique_indentifier_of_objets___40__doi_like__41__.mdwn b/doc/forum/use_case__58___unique_indentifier_of_objets___40__doi_like__41__.mdwn
new file mode 100644
index 0000000000..76f03452dc
--- /dev/null
+++ b/doc/forum/use_case__58___unique_indentifier_of_objets___40__doi_like__41__.mdwn
@@ -0,0 +1,20 @@
+Dear all,
+           From some time now I am wondering about a way to index a set of files, lets say PDF documents. The idea is to have a unique identifier for each file and to cross-reference using this identifier. For instance, I use a project management (PM) software (web based) on a public server of my university. Then I have a set of tasks saying, review document X, or Y. And those documents are stored on an internal server of my lab.
+I see several options:
+
+1. Upload the required documents to the PM site and directly link
+2. Share online my internal server and use the URL of the docs in the PM
+3. Just use the unique identifier in the PM, and then look in git annex for that ID
+4. Use some sort of document management system (DMS)
+
+Options 1 and 2 are impractical for several reasons. Option 4 usually requires that your files are inside the DMS.
+So my questions are: 
+
+* Do you think this is doable with git-annex?
+* Is there an easy way to ask it: give me the document with this index?
+* I think the best answer for this question is: git annex find --include '*' --format='${key} ${file}' | grep 
+* And conversely, how do I find the key of a certain document?
+
+Thanks in advance.
+Best,
+       Juan
diff --git a/doc/forum/use_case__58___unique_indentifier_of_objets___40__doi_like__41__/comment_1_955f3aac12c1ddb41267c5a23ccb79e3._comment b/doc/forum/use_case__58___unique_indentifier_of_objets___40__doi_like__41__/comment_1_955f3aac12c1ddb41267c5a23ccb79e3._comment
new file mode 100644
index 0000000000..069d41c106
--- /dev/null
+++ b/doc/forum/use_case__58___unique_indentifier_of_objets___40__doi_like__41__/comment_1_955f3aac12c1ddb41267c5a23ccb79e3._comment
@@ -0,0 +1,12 @@
+[[!comment format=mdwn
+ username="http://joeyh.name/"
+ ip="209.250.56.55"
+ subject="comment 1"
+ date="2014-07-03T19:10:53Z"
+ content="""
+To make git-annex output the key of a file, run: `git annex lookupkey $file`
+
+I don't know if the git-annex key is appropriate for your use-case. If the files never get changed, then it's a nice stable identifier. If ongoing changes are made to a file, and you want to link to the most recent version, the key would not be useful.
+
+You might also look at git-annex's [[metadata]]; you could make up some metadata field and value and attach it to a file, and it would persist as the file was modified.
+"""]]
diff --git a/doc/forum/use_case__58___unique_indentifier_of_objets___40__doi_like__41__/comment_2_0aff36755f49afddd5482a602a1ccd2b._comment b/doc/forum/use_case__58___unique_indentifier_of_objets___40__doi_like__41__/comment_2_0aff36755f49afddd5482a602a1ccd2b._comment
new file mode 100644
index 0000000000..2ef754f163
--- /dev/null
+++ b/doc/forum/use_case__58___unique_indentifier_of_objets___40__doi_like__41__/comment_2_0aff36755f49afddd5482a602a1ccd2b._comment
@@ -0,0 +1,10 @@
+[[!comment format=mdwn
+ username="https://www.google.com/accounts/o8/id?id=AItOawmTNrhkVQ26GBLaLD5-zNuEiR8syTj4mI8"
+ nickname="Juan"
+ subject="comment 2"
+ date="2014-07-09T15:32:49Z"
+ content="""
+If I get this right, the key will change every time the file is changed.
+If that's the case, seems that it won't be useful for my case.
+Thanks for the time.
+"""]]
diff --git a/doc/forum/use_existing_ssh_keys__63__.mdwn b/doc/forum/use_existing_ssh_keys__63__.mdwn
new file mode 100644
index 0000000000..7f73c18bc2
--- /dev/null
+++ b/doc/forum/use_existing_ssh_keys__63__.mdwn
@@ -0,0 +1,5 @@
+Hi, a new user trying out git-annex 3.20130124 here. So far looks really promising, thanks.
+
+I setup a local annex using the assistant webapp. I then created a rsync cloud repo over ssh to my shell box in a nearby data center for it. Works ok. I already have ssh key setup inclding ssh-agent for quick logins to the server. It would be great if you could configure the assistant to use these existing keys instead of creating a new set of keys. Maybe keep the defaults as is, but provide the config for this hidden behind "trust me, I know what I'm doing" check box or something.
+
+Thanks again, I keep researching.
diff --git a/doc/forum/use_existing_ssh_keys__63__/comment_1_c420c53f022bbd1b28494bc44d076feb._comment b/doc/forum/use_existing_ssh_keys__63__/comment_1_c420c53f022bbd1b28494bc44d076feb._comment
new file mode 100644
index 0000000000..93cb92c557
--- /dev/null
+++ b/doc/forum/use_existing_ssh_keys__63__/comment_1_c420c53f022bbd1b28494bc44d076feb._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="http://joeyh.name/"
+ ip="4.154.3.125"
+ subject="comment 1"
+ date="2013-02-07T18:11:30Z"
+ content="""
+Hmm. The assistant already tries to cater to this case of having an existing key. When setting up a rsync or ssh remote, it first tries to ssh to the server without allowing a password to be prompted for. If this succeeds, it assumes you have a passwordless key and uses it. Otherwise, it sshes in once with a password prompt, and sets up its own key.
+"""]]
diff --git a/doc/forum/use_existing_ssh_keys__63__/comment_2_e4cae848e5701852073ced307832872b._comment b/doc/forum/use_existing_ssh_keys__63__/comment_2_e4cae848e5701852073ced307832872b._comment
new file mode 100644
index 0000000000..4aaaeacce8
--- /dev/null
+++ b/doc/forum/use_existing_ssh_keys__63__/comment_2_e4cae848e5701852073ced307832872b._comment
@@ -0,0 +1,12 @@
+[[!comment format=mdwn
+ username="https://www.google.com/accounts/o8/id?id=AItOawnwz07HJdp3nzLwCAl7OGKn_3m0tTO4Mxw"
+ nickname="Teemu"
+ subject="use existing ssh keys?"
+ date="2013-03-02T12:05:30Z"
+ content="""
+I also found the key setup confusing. I have many identity files for several server / service combinations, so the webapp key setup for an ssh+rsync cloud server always failed with too many identification attempts before I had the sense to move my keys temporarily away from ~/.ssh
+
+It would be really nice if the webapp would allow one either to pick an existing identity, or to create a new one with the aid of a couple of password logins.
+
+Also, having the created key to be restricted to only running rsync in the server would improve the chances of 'Share with a friend' use case being used. I don't think many people will want to give out full shell access to their servers in order to share some files.
+"""]]
diff --git a/doc/forum/use_existing_ssh_keys__63__/comment_3_a97c20b6df74c49e5f57c7caf962f1e2._comment b/doc/forum/use_existing_ssh_keys__63__/comment_3_a97c20b6df74c49e5f57c7caf962f1e2._comment
new file mode 100644
index 0000000000..e90ed07be4
--- /dev/null
+++ b/doc/forum/use_existing_ssh_keys__63__/comment_3_a97c20b6df74c49e5f57c7caf962f1e2._comment
@@ -0,0 +1,10 @@
+[[!comment format=mdwn
+ username="http://joeyh.name/"
+ nickname="joey"
+ subject="comment 3"
+ date="2013-03-03T16:57:22Z"
+ content="""
+When pairing with a local computer, both systems set up locked down ssh keys that only allow the pair to run git-annex on the repository being paired with. This does *not* involve full shell access.
+
+The \"remote server\" necessarily involves you already having shell access to the remote server. However, the ssh key that the assistant generates is again locked down to only being able to run git-annex on a single repository on the remote server.
+"""]]
diff --git a/doc/forum/use_existing_ssh_keys__63__/comment_4_8977bb8ee662c30dfcecae73cede9dfa._comment b/doc/forum/use_existing_ssh_keys__63__/comment_4_8977bb8ee662c30dfcecae73cede9dfa._comment
new file mode 100644
index 0000000000..696120da9c
--- /dev/null
+++ b/doc/forum/use_existing_ssh_keys__63__/comment_4_8977bb8ee662c30dfcecae73cede9dfa._comment
@@ -0,0 +1,12 @@
+[[!comment format=mdwn
+ username="80d8aa@c71d4a9510ad0353dbcf7df399c2e6bde0012474"
+ nickname="80d8aa"
+ avatar="http://cdn.libravatar.org/avatar/b3f9ee295805d0758d569ca97157fb65"
+ subject="Use existing ssh keys"
+ date="2017-09-30T17:07:39Z"
+ content="""
+Is there an argument against the original request of being able to use an existing *non-default* ssh key when setting up a remote ssh repo with the webapp?
+
+As suggested above, one could reasonably have different identities, not all of which should have access to the git-annex repo. AFAIU, the way things are one has to either use their default identity with a remote ssh repo OR temporarily enable password authentication on the server so that the webapp will generate and install its own key, then turn it off. That is a tall order and might not even be possible if the server is not under the user's control. Am I missing something?
+
+"""]]
diff --git a/doc/forum/using_git-annex_with_lightroom.mdwn b/doc/forum/using_git-annex_with_lightroom.mdwn
new file mode 100644
index 0000000000..605f2b9030
--- /dev/null
+++ b/doc/forum/using_git-annex_with_lightroom.mdwn
@@ -0,0 +1,6 @@
+I'm using git-annex to sync my photos across multiple computers, and it works beautifully. I would also like to sync Lightroom catalogues. The photo editing program creates a *.lrdata directory where it stores the edits in its own tree format. Merging two such directories obviously creates a mess. 
+
+Is there an elegant way to tell git-annex to treat the whole directory as a single file and overwrite the whole directory structure at once? I'm guessing the same problem occurs with mac os packages.
+
+Many thanks!
+Alex
diff --git a/doc/forum/using_git-annex_with_lightroom/comment_1_ec977efd277f0644767a4fc7064e4baf._comment b/doc/forum/using_git-annex_with_lightroom/comment_1_ec977efd277f0644767a4fc7064e4baf._comment
new file mode 100644
index 0000000000..e7e1fa9a60
--- /dev/null
+++ b/doc/forum/using_git-annex_with_lightroom/comment_1_ec977efd277f0644767a4fc7064e4baf._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="http://joeyh.name/"
+ ip="209.250.56.54"
+ subject="comment 1"
+ date="2014-10-02T15:23:18Z"
+ content="""
+There's not really a way to handle this sort of thing. I suggest that you put *.lrdata in `.gitignore`
+"""]]
diff --git a/doc/forum/using_git-annex_with_lightroom/comment_2_4dc4f1ea91c72418843a682fab1854d2._comment b/doc/forum/using_git-annex_with_lightroom/comment_2_4dc4f1ea91c72418843a682fab1854d2._comment
new file mode 100644
index 0000000000..d0626d4c72
--- /dev/null
+++ b/doc/forum/using_git-annex_with_lightroom/comment_2_4dc4f1ea91c72418843a682fab1854d2._comment
@@ -0,0 +1,28 @@
+[[!comment format=mdwn
+ username="Jan"
+ subject="comment 2"
+ date="2015-01-10T10:37:11Z"
+ content="""
+I had the same problem an got Lightroom to work nicely with the following configuration.
+
+Place catalog and photo data in the same directory tree, for example:
+
+
+    /Lightroom/2015/2015-01-10/IMG_4432.JPG
+    /Lightroom/2015/2015-01-10/IMG_4433.JPG
+    ...
+    /Lightroom/Catalog/Lightroom 5 Catalog.lrcat
+    /Lightroom/Catalog/Lightroom 5 Catalog Previews.lrdata
+    /Lightroom/Catalog/Backups
+
+
+Now the image data and Lightroom metadata (= catalog) can be archived at the same time. In order to prevent temporary files and previews to get annexed use the following `.gitignore`:
+
+
+    */*.lrdata
+    Temporary Folder*/
+    */Temporary Import Data*/
+    *.lock
+
+The downside is that previews will have to get regenerated on other machines which pull this data, but I don't use it to share my Lightroom config across computers but as an archival tool.
+"""]]
diff --git a/doc/forum/using_git_annex_to_merge_and_synchronize_2_directories___40__like_unison__41__.mdwn b/doc/forum/using_git_annex_to_merge_and_synchronize_2_directories___40__like_unison__41__.mdwn
new file mode 100644
index 0000000000..86e317da87
--- /dev/null
+++ b/doc/forum/using_git_annex_to_merge_and_synchronize_2_directories___40__like_unison__41__.mdwn
@@ -0,0 +1,7 @@
+I would like to use git-annex to synchronize 2 directories in the same manner as unison.
+
+I'm starting with 2 directories. There is an overlap of the same set of files in each directory, but each directory also has additional files as well.
+
+I create a git annex in each directory but when I do a git pull it merges and produces conflicts on those files that are the same.
+
+What is the correct workflow for this type of scenario?
diff --git a/doc/forum/using_git_annex_to_merge_and_synchronize_2_directories___40__like_unison__41__/comment_1_5c3ee8a8aaa6d0918c0cc9683ce177ae._comment b/doc/forum/using_git_annex_to_merge_and_synchronize_2_directories___40__like_unison__41__/comment_1_5c3ee8a8aaa6d0918c0cc9683ce177ae._comment
new file mode 100644
index 0000000000..4682ea64f7
--- /dev/null
+++ b/doc/forum/using_git_annex_to_merge_and_synchronize_2_directories___40__like_unison__41__/comment_1_5c3ee8a8aaa6d0918c0cc9683ce177ae._comment
@@ -0,0 +1,10 @@
+[[!comment format=mdwn
+ username="http://www.joachim-breitner.de/"
+ nickname="nomeata"
+ subject="comment 1"
+ date="2011-12-18T13:57:33Z"
+ content="""
+Are the files identical or different? I today did something like that with similar, but not identical directories containing media files, and git happily merged them. but there, same files had same content.
+
+Also, make sure you use the same backend. In my case, one of the machines runs Debian stable, so I use the WORM backend, not the SHA backend.
+"""]]
diff --git a/doc/forum/using_git_annex_to_merge_and_synchronize_2_directories___40__like_unison__41__/comment_2_648946353c6d90c57351cce4010f1301._comment b/doc/forum/using_git_annex_to_merge_and_synchronize_2_directories___40__like_unison__41__/comment_2_648946353c6d90c57351cce4010f1301._comment
new file mode 100644
index 0000000000..bdd4b25e43
--- /dev/null
+++ b/doc/forum/using_git_annex_to_merge_and_synchronize_2_directories___40__like_unison__41__/comment_2_648946353c6d90c57351cce4010f1301._comment
@@ -0,0 +1,10 @@
+[[!comment format=mdwn
+ username="http://joey.kitenet.net/"
+ nickname="joey"
+ subject="comment 2"
+ date="2011-12-19T18:24:59Z"
+ content="""
+I'd recommend using the SHA backend for this, the WORM backend would produce conflicts if the files' modification times changed. 
+
+[[syncing_non-git_trees_with_git-annex]] describes one way to do it.
+"""]]
diff --git a/doc/forum/uuid_mismatch.mdwn b/doc/forum/uuid_mismatch.mdwn
new file mode 100644
index 0000000000..2d1626e385
--- /dev/null
+++ b/doc/forum/uuid_mismatch.mdwn
@@ -0,0 +1,17 @@
+Hi,
+
+After reinstalling my laptop with the same GPG keys, I've got this error during re-syncing a remote crypted repo :
+
+```
+Internal Server Error
+; expected Just (UUID "e7fc2bc8-b630-5eeb-a0b3-b4806d7ab064") but remote gitrepo has UUID "e2f02805-eae7-5bb8-875e-c901ba9929b1" (":id:SCPxErEQVlqp+Hl6Su0f")
+```
+
+Then, I can see the structure of the hard links in the local directory but pointing to nothing ?
+
+FYI all sub directories in .git/annex/objects contain .map files only.
+
+Could you please help me ? 
+Those data are critical, thanks for your great app and your help!
+
+Guillaume
diff --git a/doc/forum/uuid_mismatch/comment_1_ed29dcf70b916bdd094015c1c9616550._comment b/doc/forum/uuid_mismatch/comment_1_ed29dcf70b916bdd094015c1c9616550._comment
new file mode 100644
index 0000000000..a97f5baa53
--- /dev/null
+++ b/doc/forum/uuid_mismatch/comment_1_ed29dcf70b916bdd094015c1c9616550._comment
@@ -0,0 +1,32 @@
+[[!comment format=mdwn
+ username="joey"
+ subject="""comment 1"""
+ date="2016-10-05T19:30:48Z"
+ content="""
+It would help if you described what command you ran, or buttons you clicked
+on to get this to happen. Otherwise, I'm left making a lot of guesses..
+
+It looks like you might be running `git-annex enableremote` of a gcrypt
+special remote that you set up earlier. 
+
+It appears that the gcrypt remote has a gcrypt-id which is not the same as
+the one that was used when that special remote was originally initialized.
+
+I don't know why this would happen, and without details about what you did,
+I can barely venture a guess. All I can say is that, gcrypt is supposed to
+pick a gcrypt-id the first time a gcrypt remote is used, and not change it
+afterwards.
+
+... It might be that you did something that caused gcrypt to pick a new
+gcrypt-id.
+
+... It might be that the special remote has been pointed at a different
+gcrypt repository than the one that it was initially set up to use.
+
+It might help to look at both the local repository's `.git/config`, and the
+gcrypt git repositories git config, and see which of
+them have the wrong gcrypt-id value of ":id:SCPxErEQVlqp+Hl6Su0f".
+If only the local .git/config has that value and the gcrypt git repository
+has some other gcrypt-id, then the other one is probably the right one, and
+it might work to replace the wrong one with the right one.
+"""]]
diff --git a/doc/forum/uuid_mismatch/comment_2_de4beb1eb76ae7084387e8efc28e55d3._comment b/doc/forum/uuid_mismatch/comment_2_de4beb1eb76ae7084387e8efc28e55d3._comment
new file mode 100644
index 0000000000..9e85d931aa
--- /dev/null
+++ b/doc/forum/uuid_mismatch/comment_2_de4beb1eb76ae7084387e8efc28e55d3._comment
@@ -0,0 +1,15 @@
+[[!comment format=mdwn
+ username="yomguy"
+ subject="comment 2"
+ date="2016-10-07T09:15:07Z"
+ content="""
+Dear Joey,
+
+Thanks you so much for your quick response.
+
+I did not use the command line for this but the web app. So I will retry the sync sequence in a shell after tweaking the gcrypt-id as you proposed and then give you some more feedback here.
+
+Cheers,
+Guillaume
+
+"""]]
diff --git a/doc/forum/uuid_mismatch/comment_3_a681a4847acbe890c4e486288b3c81d3._comment b/doc/forum/uuid_mismatch/comment_3_a681a4847acbe890c4e486288b3c81d3._comment
new file mode 100644
index 0000000000..d92f3fe17a
--- /dev/null
+++ b/doc/forum/uuid_mismatch/comment_3_a681a4847acbe890c4e486288b3c81d3._comment
@@ -0,0 +1,19 @@
+[[!comment format=mdwn
+ username="yomguy"
+ avatar="http://cdn.libravatar.org/avatar/03db077c04f8b753f3f504d9a2b06a29"
+ subject="comment 3"
+ date="2016-11-18T14:00:51Z"
+ content="""
+Hi joey,
+
+After modifying the gcrypt-id as you proposed, I have finally managed to clone the repo with  
+
+`git clone gcrypt::ssh://my.domain/home/admin/`
+
+But now I get only unresolved symbolic links for each files, that is .git/annex/objects directory only contains .map files.
+
+Would you have an idea about the reason/source of this behavior?
+
+Thank you so much,
+Guillaume
+"""]]
diff --git a/doc/forum/v5_to_v6_upgrade_strategy.mdwn b/doc/forum/v5_to_v6_upgrade_strategy.mdwn
new file mode 100644
index 0000000000..9a34b01e3a
--- /dev/null
+++ b/doc/forum/v5_to_v6_upgrade_strategy.mdwn
@@ -0,0 +1,21 @@
+I have recently jumped into git-annex, and have several TB stored in several remote annexes.
+I am able to sync to clients, and have some understanding of the operations needed to get files back etc.
+
+So far I have used v5 in Ubuntu and Debian 8. I have tried out assistant but will leave this until I better understand how git-annex works. CLI is best for this learning. 
+
+I am however concerned re the upgrade path from v5 to v6. I have not been able to find a comprehensive strategy. I have tried git-annex v6 and have upgraded a client repo to v6.
+
+All of my remote repos are v5, and to be honest I am very nervous about touching them.
+
+I do have a test bed where I try out all things before doing so on my 'production' repos. So, I can play, break, redo, in safety.
+
+Any direction as to formulating a game plan for upgrading my whole network of repos, clients, etc would be very much appreciated.
+
+And, if I have missed this, a pointer would be good for sure.
+
+
+Regards,
+
+Stan
+
+While it is obvious to all: this is amazing software. Genius.
diff --git a/doc/forum/v5_to_v6_upgrade_strategy/comment_1_1505166c806ab3d1a3148a245faa713a._comment b/doc/forum/v5_to_v6_upgrade_strategy/comment_1_1505166c806ab3d1a3148a245faa713a._comment
new file mode 100644
index 0000000000..68c9c52807
--- /dev/null
+++ b/doc/forum/v5_to_v6_upgrade_strategy/comment_1_1505166c806ab3d1a3148a245faa713a._comment
@@ -0,0 +1,15 @@
+[[!comment format=mdwn
+ username="anarcat"
+ subject="upgrades instruction"
+ date="2016-06-23T14:44:18Z"
+ content="""
+since you haven't explicitly mentionned it, i'll just mention a pointer to the [[upgrades]] page which has specific instructions regarding upgrading repositories to the v6 layout.
+
+as far as I know, you can just upgrade the git-annex *program* itself harmlessly. it will *not* upgrade your repositories without you deliberately running [[git-annex-upgrade]].
+
+then [[git-annex-upgrade]] will operate changes, but only on direct mode repositories, as far as i can tell. those repositories are switched to [[design/adjusted branches]], which have their own set of issues right name, mostly limitations with the webapp but have also seen a few bugfixes on crippled filesystems recently.
+
+i am personnally doing the \"wait and see\" game especially considering my specific use case for v6 ([[todo/hide_missing_files/]]) has not been completely implemented yet. but I am of course using 6.x binaries without problems.
+
+i am not the git-annex author, so take my comments with a grain of salt of course. :) --[[anarcat]]
+"""]]
diff --git a/doc/forum/v5_to_v6_upgrade_strategy/comment_2_ef2648a58037b867e2d745eb4b3a15a2._comment b/doc/forum/v5_to_v6_upgrade_strategy/comment_2_ef2648a58037b867e2d745eb4b3a15a2._comment
new file mode 100644
index 0000000000..e39d8f2688
--- /dev/null
+++ b/doc/forum/v5_to_v6_upgrade_strategy/comment_2_ef2648a58037b867e2d745eb4b3a15a2._comment
@@ -0,0 +1,17 @@
+[[!comment format=mdwn
+ username="Stan"
+ subject="comment 2"
+ date="2016-06-28T23:10:07Z"
+ content="""
+Many thanks for the reply. I did not add too much detail to my post as it was my first, and I was a bit shy.
+
+I did do all of the steps indicated re v6. The binary has been fine for me also.
+
+My question is a bit more complex as I am looking to apply v6 at some point to a set of distributed repos. Essentially it has the typical duplicated system topology: 2 remotes, several clients, each of which is linked to both remotes. Thus it will survive multiple failure scenarios.
+
+I would be looking at the strategy of the order of repo upgrade. Say, a client first, and run a set of tests. And so on. Remotes last perhaps.
+
+In any case I built a test bed as above and have been experimenting with a single client at a v6 repo.
+
+So far it has been smooth, but right now I seem to be stuck where thinning is not having any effect: I still have a particular big file (1.5GB) in both the annex and the working dir. Lock remove the big file from the working dir, and leaves a link, and unlock restores the big file in the working dir. Yet, the annex still contains the big file no matter what. I was under the expectation that one of the big file copies would go away.
+"""]]
diff --git a/doc/forum/v5_to_v6_upgrade_strategy/comment_3_46fb3f5b0a7fa13b6f16ccb9832dac7e._comment b/doc/forum/v5_to_v6_upgrade_strategy/comment_3_46fb3f5b0a7fa13b6f16ccb9832dac7e._comment
new file mode 100644
index 0000000000..189b50aefc
--- /dev/null
+++ b/doc/forum/v5_to_v6_upgrade_strategy/comment_3_46fb3f5b0a7fa13b6f16ccb9832dac7e._comment
@@ -0,0 +1,22 @@
+[[!comment format=mdwn
+ username="joey"
+ subject="""comment 3"""
+ date="2016-07-06T18:50:34Z"
+ content="""
+I'd encourage you to not upgrade any production repositories to v6 yet.
+It's still in beta state and has known problems. Currently all known
+problems are only innefficiencies, but there could also still be bugs.
+
+When you do upgrade, you can upgrade your distibuted set of repos
+peicemeal. v5 and v6 fully interoperate, until you start unlocking files in
+the v6 clones of the repo. Once those unlocks get committed, the v5 clones
+of the repo won't be able to access the unlocked file contents, and you'll
+need to upgrade the clones then.
+
+At some point in the future, I expect v6 mode to be done and then git-annex
+will automatically upgrade for you. So the easist approach is to just wait
+for that.
+
+(Not addressing your problems with annex.thin here as you filed a separate
+forum post and bug report about that.)
+"""]]
diff --git a/doc/forum/v6_thinning_seems_to_be_not_working_for_me.mdwn b/doc/forum/v6_thinning_seems_to_be_not_working_for_me.mdwn
new file mode 100644
index 0000000000..a87421dd32
--- /dev/null
+++ b/doc/forum/v6_thinning_seems_to_be_not_working_for_me.mdwn
@@ -0,0 +1,42 @@
+git-annex version: 6.20160524+gitg2b7b2c4-1~ndall+1
+
+repo config:
+
+[core]
+	repositoryformatversion = 0
+	filemode = true
+	bare = false
+	logallrefupdates = true
+[annex]
+	uuid = c5c203ec-9353-4213-8af5-6fcb8de36ca2
+	version = 6
+	thin = true
+[filter "annex"]
+	smudge = git-annex smudge %f
+	clean = git-annex smudge --clean %f
+
+
+
+copied a big file to the working directory
+
+git add
+git commit
+
+
+Locking the big file creates the symlink, with the file in the annex.
+
+Unlocking the big file creates a real file in the working dir, and the annex file stays there.
+
+The total dir is therefore twice the size of the big file.
+
+ls -li shows a unique inode for each file.
+
+
+It seems thinning is not doing what I expected:
+
+Which is: unlocking a file creates a hard link in the working directory, linked to the annex file, with
+the resulting dir size being the size of the one big file.
+
+Are my expectations correct?
+
+Is there something I am missing?
diff --git a/doc/forum/v6_thinning_seems_to_be_not_working_for_me/comment_1_751e603141513faaa43abc38f3329a3e._comment b/doc/forum/v6_thinning_seems_to_be_not_working_for_me/comment_1_751e603141513faaa43abc38f3329a3e._comment
new file mode 100644
index 0000000000..7587a0d1d4
--- /dev/null
+++ b/doc/forum/v6_thinning_seems_to_be_not_working_for_me/comment_1_751e603141513faaa43abc38f3329a3e._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="joey"
+ subject="""comment 1"""
+ date="2016-07-06T18:48:46Z"
+ content="""
+[[bugs/v6_appears_to_not_thin]] was also filed by you about this;
+so I'll followup there instead of here.
+"""]]
diff --git a/doc/forum/vanilla_git_repo_as_special_remote__63__.mdwn b/doc/forum/vanilla_git_repo_as_special_remote__63__.mdwn
new file mode 100644
index 0000000000..94fa548653
--- /dev/null
+++ b/doc/forum/vanilla_git_repo_as_special_remote__63__.mdwn
@@ -0,0 +1,27 @@
+Right now I have separate "normal" Git repositories and separate Git annex repositories and I would love to have Git annex track and sync everything for me. The problem I have is I'd like to use "real" Git content tracking for some data (ex: text files) where I'd like to get normal Git features with (ex: diff). I'd like to combine normal Git content tracking with Git annex location tracking and syncing if possible. Ideally the cost (ex: increased git repo size and git slowdown) of content tracking would not need to be propagated across the entire git annex network, just on repos that want it (just like git annex only copies content to clients who want it and symlink the rest).
+
+The largefiles config provides a mechanism to add content to git directly in git annex, but that cost would be applied across the entire network, not opt-in per client.
+
+Ideally I'd like this situation:
+
+1. Git annex tracking everything as symlinks. No content is checked into these git repos.
+2. A subset of git annex content (ex: subfolder) synced to a normal remote non-annex git repository (ex: GitHub). This Git repo has content tracked in git itself.
+
+And I could use the git annex repos to sync everything. Somehow the git annex repo would know that the #2 remote was a "special content git remote" and push any content updates as normal git content commits.
+
+Or an adjusted branch that had the content tracked and I could sync that content branch around to only the remotes where I wanted the content history stored in git (since adjusted branches don't seem to annex sync by default). But master would just track the symlinks of those files and be synced around to all annexes.
+
+Can adjusted branches do this somehow?
+
+Some references:
+
+* [[special_remotes/external]]
+* [[design/adjusted_branches]]
+* [[todo/hide_missing_files]]
+* [[tips/largefiles]]
+* [[submodules]]
+* [[forum/git-subtree_support__63__]]
+
+Thanks!
+
+-neocryptek
diff --git a/doc/forum/vanilla_git_repo_as_special_remote__63__/comment_1_67e186265ae21f2cd8451750152f2a6d._comment b/doc/forum/vanilla_git_repo_as_special_remote__63__/comment_1_67e186265ae21f2cd8451750152f2a6d._comment
new file mode 100644
index 0000000000..65397495a2
--- /dev/null
+++ b/doc/forum/vanilla_git_repo_as_special_remote__63__/comment_1_67e186265ae21f2cd8451750152f2a6d._comment
@@ -0,0 +1,13 @@
+[[!comment format=mdwn
+ username="joey"
+ subject="""comment 1"""
+ date="2016-11-16T18:30:12Z"
+ content="""
+You can use bup as a special remote, which will store the content in a git
+repository. But, not in a form that git diff can be used with.
+
+[[git-annex-diffdriver]] can be used to make `git diff` work on annexed
+files. For example:
+
+	export GIT_EXTERNAL_DIFF="git-annex diffdriver -- diff -u --"
+"""]]
diff --git a/doc/forum/vanilla_git_repo_as_special_remote__63__/comment_2_6314256da98966f4c7d02aa0d6bf94ff._comment b/doc/forum/vanilla_git_repo_as_special_remote__63__/comment_2_6314256da98966f4c7d02aa0d6bf94ff._comment
new file mode 100644
index 0000000000..f463554031
--- /dev/null
+++ b/doc/forum/vanilla_git_repo_as_special_remote__63__/comment_2_6314256da98966f4c7d02aa0d6bf94ff._comment
@@ -0,0 +1,17 @@
+[[!comment format=mdwn
+ username="neocryptek@659edac901ffbc8e541a974f8f18987eeafc63bd"
+ nickname="neocryptek"
+ avatar="http://cdn.libravatar.org/avatar/d9bfdefa9b503f1ac4844a686618374e"
+ subject="comment 2"
+ date="2016-11-21T22:26:52Z"
+ content="""
+Right, though bup also requires installation on the server. I'm looking for a way to store content into a vanilla git repo (as I don't have permission to install anything custom on the server).
+
+Since I want to store the content outside of git annex, it feels like a special remote. Though ideally it would have human readable files like:
+
+* 
+
+But since it's git and not just a normal (single version) filesystem, it could dedupe and save previous versions. Is there an easy way to hook git up safely to the external remote protocol:
+
+* [[special_remotes/external]]
+"""]]
diff --git a/doc/forum/version_3_upgrade.mdwn b/doc/forum/version_3_upgrade.mdwn
new file mode 100644
index 0000000000..7fdbcbc805
--- /dev/null
+++ b/doc/forum/version_3_upgrade.mdwn
@@ -0,0 +1,9 @@
+after upgrading to git-annex 3, i'm stuck with diverging git-annex branches -- i didn't manage to follow this line in the directions:
+
+> After this upgrade, you should make sure you include the git-annex branch when git pushing and pulling.
+
+could you explain how to do that in a littel more detail? git pull seems to only merge master, although i have these ``.git/config`` settings:
+
+    [branch "git-annex"]
+    	remote = origin
+    	merge = git-annex
diff --git a/doc/forum/version_3_upgrade/comment_1_05fc9c9cad26c520bebb98c852c71e35._comment b/doc/forum/version_3_upgrade/comment_1_05fc9c9cad26c520bebb98c852c71e35._comment
new file mode 100644
index 0000000000..18746225e9
--- /dev/null
+++ b/doc/forum/version_3_upgrade/comment_1_05fc9c9cad26c520bebb98c852c71e35._comment
@@ -0,0 +1,13 @@
+[[!comment format=mdwn
+ username="http://joey.kitenet.net/"
+ nickname="joey"
+ subject="comment 1"
+ date="2011-08-17T01:33:08Z"
+ content="""
+It's ok that `git pull` does not merge the git-annex branch. You can merge it with `git annex merge`, or it will be done
+automatically when you use other git-annex commands.
+
+If you use `git pull` and `git push` without any options, the defaults will make git pull and push the git-annex branch automatically.
+
+But if you're in the habit of doing `git push origin master`, that won't cause the git-annex branch to be pushed (use `git push origin git-annex` to manually push it then). Similarly, `git pull origin master` won't pull it. And also, the `remote.origin.fetch` setting in `.git/config` can be modified in ways that make `git pull` not automatically pull the git-annex branch. So those are the things to avoid after upgrade to v3, basically.
+"""]]
diff --git a/doc/forum/version_5_repo_with_version_6_repo.mdwn b/doc/forum/version_5_repo_with_version_6_repo.mdwn
new file mode 100644
index 0000000000..d9b4aba7a8
--- /dev/null
+++ b/doc/forum/version_5_repo_with_version_6_repo.mdwn
@@ -0,0 +1,3 @@
+I've found that when a file is unlocked in a v6 repo, syncing with a v5 repo causes the file in the v5 repo to be a symlink-replacement file (the file contents are, e.g., /annex/objects/SHA256E-s10--114811b0b8998cb9853a5379598021410feddf69bb2ee7b7145d052a7e9b5d45). I'm wondering if this is a bug, or just something that users should be aware of.
+Thanks for Git Annex---I really enjoy it!
+--Andy
diff --git a/doc/forum/version_5_repo_with_version_6_repo/comment_1_95037082d15758d7b2c23ad041d7f6f0._comment b/doc/forum/version_5_repo_with_version_6_repo/comment_1_95037082d15758d7b2c23ad041d7f6f0._comment
new file mode 100644
index 0000000000..0e7d17a20e
--- /dev/null
+++ b/doc/forum/version_5_repo_with_version_6_repo/comment_1_95037082d15758d7b2c23ad041d7f6f0._comment
@@ -0,0 +1,12 @@
+[[!comment format=mdwn
+ username="joey"
+ subject="""comment 1"""
+ date="2016-10-31T17:04:42Z"
+ content="""
+It's something that users need to be aware of if mixing use of v6 and v5
+repositories and unlocking files. 
+
+Of course, if it becomes a problem, you can simply lock the files in the v6
+repository and then the v5 repo will be able to access them via symlinks
+again.
+"""]]
diff --git a/doc/forum/view_from_numeric_values.mdwn b/doc/forum/view_from_numeric_values.mdwn
new file mode 100644
index 0000000000..e1fc48a0f2
--- /dev/null
+++ b/doc/forum/view_from_numeric_values.mdwn
@@ -0,0 +1,9 @@
+Hi Joey,
+
+it would be nice when views could take numeric comparisons as filters.
+
+    git annex metadata -s length=273.0 john_cage_4_33.mp3
+
+    git annex view length<=300
+
+... here is the catch, < and > don't work well in shell, this needs some other Syntax. I think the underlying machinery (using numeric comparisons instead globs) should be quite trivial. Any Ideas about a Syntax?
diff --git a/doc/forum/view_from_numeric_values/comment_1_f3c440f3f0104002a0020ba96ddcf87b._comment b/doc/forum/view_from_numeric_values/comment_1_f3c440f3f0104002a0020ba96ddcf87b._comment
new file mode 100644
index 0000000000..4df3034a2d
--- /dev/null
+++ b/doc/forum/view_from_numeric_values/comment_1_f3c440f3f0104002a0020ba96ddcf87b._comment
@@ -0,0 +1,11 @@
+[[!comment format=mdwn
+ username="cehteh"
+ ip="217.8.62.137"
+ subject="consistent syntax"
+ date="2014-03-25T15:25:43Z"
+ content="""
+Further thinking led to the idea to use the test(1) like syntax to filter matches.
+
+    git annex view length=.* -lt 300 -gt 30 -o -eq 273 bpm=.* -eq 0
+
+"""]]
diff --git a/doc/forum/view_from_numeric_values/comment_2_2414e1a8cfd154c339d8fc0e4a630ae9._comment b/doc/forum/view_from_numeric_values/comment_2_2414e1a8cfd154c339d8fc0e4a630ae9._comment
new file mode 100644
index 0000000000..ff4fcc551f
--- /dev/null
+++ b/doc/forum/view_from_numeric_values/comment_2_2414e1a8cfd154c339d8fc0e4a630ae9._comment
@@ -0,0 +1,12 @@
+[[!comment format=mdwn
+ username="http://joeyh.name/"
+ ip="209.250.56.41"
+ subject="comment 2"
+ date="2014-03-26T17:53:28Z"
+ content="""
+I'm a little bit worried about the potential to reinvent SQL, badly. ;)
+
+As shown in your example, once you have ranges, it's natural to also want disjunctions, and then probably parenthesized expressions, and suddenly things are very complicated.
+
+Also, it's important that views remain reversable, so that committing a moved file in a view can unambiguously calculate the new metadata for it. I think that quickly becomes hard when adding these complications.
+"""]]
diff --git a/doc/forum/view_from_numeric_values/comment_3_7879a11cc9767cdaac14f9993182dc25._comment b/doc/forum/view_from_numeric_values/comment_3_7879a11cc9767cdaac14f9993182dc25._comment
new file mode 100644
index 0000000000..da6dded172
--- /dev/null
+++ b/doc/forum/view_from_numeric_values/comment_3_7879a11cc9767cdaac14f9993182dc25._comment
@@ -0,0 +1,13 @@
+[[!comment format=mdwn
+ username="cehteh"
+ ip="217.8.62.137"
+ subject="comment 3"
+ date="2014-03-26T18:55:41Z"
+ content="""
+I'm a little bit worried about the potential to reinvent SQL, badly. ;)
+
+As shown in your example, once you have ranges, it's natural to also want disjunctions, and then probably parenthesized expressions, and suddenly things are very complicated.
+
+Also, it's important that views remain reversable, so that committing a moved file in a view can unambiguously calculate the new metadata for it. I think that quickly becomes hard when adding these complications.
+
+"""]]
diff --git a/doc/forum/view_from_numeric_values/comment_4_517c7659654a6fc608eb3332053df8a4._comment b/doc/forum/view_from_numeric_values/comment_4_517c7659654a6fc608eb3332053df8a4._comment
new file mode 100644
index 0000000000..90cb1c22a9
--- /dev/null
+++ b/doc/forum/view_from_numeric_values/comment_4_517c7659654a6fc608eb3332053df8a4._comment
@@ -0,0 +1,28 @@
+[[!comment format=mdwn
+ username="cehteh"
+ ip="217.8.62.137"
+ subject="comment 4"
+ date="2014-03-26T20:03:51Z"
+ content="""
+I agree with you that things must stay simple. All what should be done is having the same effects like normal globs but adding arithmetic comparisons to it (could you think about a globbing extension over numeric values?). Then the generated views will have the same properties/semnatic as the normal glob'ed views without other side effects (if you want to go that far, this would even hold true for disjunct, parenthesized and otherwise complex expression).
+
+Example (how it should work, except my bug report 'set metadata on wrong files')
+
+    git annex metadata a.txt -s foo=bar -s num=1
+    git annex metadata b.txt -s foo=baz -s num=2
+    git annex metadata c.txt -s foo=barf -s num=3
+
+    git annex view foo=bar* num=* -ne 2
+
+should give
+    ./bar/1/a.txt ./barf/3/c.txt
+
+am I right now than one could
+
+    mkdir -p ./baz/2
+    mv /bar/1/a.txt ./baz/2
+
+to change the metadata of a.txt, despite the foo=baz and num=2 fields where initially filtered out when creating the view?
+If this assumption is true then having arithmetic filters, no matter how complex they are won't change the existing semantics over what globs do.
+
+"""]]
diff --git a/doc/forum/view_including_files_with_no_tags.mdwn b/doc/forum/view_including_files_with_no_tags.mdwn
new file mode 100644
index 0000000000..7ed64fc7df
--- /dev/null
+++ b/doc/forum/view_including_files_with_no_tags.mdwn
@@ -0,0 +1,5 @@
+Hi
+
+Is it possible to create a view which also includes files with no tag?
+
+I use something like `git annex view 'rating=*'` to view files sorted by rating but this view does not include files which don‘t have a rating yet. What I was looking for is a way to show tagged files and untagged files in one view.
diff --git a/doc/forum/view_including_files_with_no_tags/comment_1_b0aafc023fbec33af268576c4c199af3._comment b/doc/forum/view_including_files_with_no_tags/comment_1_b0aafc023fbec33af268576c4c199af3._comment
new file mode 100644
index 0000000000..ec58308b75
--- /dev/null
+++ b/doc/forum/view_including_files_with_no_tags/comment_1_b0aafc023fbec33af268576c4c199af3._comment
@@ -0,0 +1,24 @@
+[[!comment format=mdwn
+ username="Xyem"
+ ip="87.194.19.134"
+ subject="comment 1"
+ date="2014-03-25T08:54:23Z"
+ content="""
+This is a TODO:
+
+http://git-annex.branchable.com/design/metadata/
+
+    unmatched files in filtered branches
+
+    TODO Files not matching the view should be able to be included in the filtered branch, in a special location, an \"other\" directory.
+
+In the meantime, I do this before switching to the view:
+
+    git annex metadata -s fieldIwant?=untagged
+
+This shows the files without any \"fieldIwant\" in the directory \"untagged\". Afterwards, I could (but don't need to in my workflow, it automatically gets removed), do this to remove it:
+
+    git annex metadata -s fieldIwant-=untagged
+
+What the \"?=\" does is add that metadata only if the \"fieldIwant\" hasn't been set at all.
+"""]]
diff --git a/doc/forum/view_including_files_with_no_tags/comment_2_5ae9d5308371bdb1f94342c9f9b01aff._comment b/doc/forum/view_including_files_with_no_tags/comment_2_5ae9d5308371bdb1f94342c9f9b01aff._comment
new file mode 100644
index 0000000000..3f16a163f8
--- /dev/null
+++ b/doc/forum/view_including_files_with_no_tags/comment_2_5ae9d5308371bdb1f94342c9f9b01aff._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="http://ypid.wordpress.com/"
+ ip="213.153.84.215"
+ subject="Perfect"
+ date="2014-03-25T20:11:26Z"
+ content="""
+This works great. Nice workaround, thanks very much.
+"""]]
diff --git a/doc/forum/views___40__branches__41___never_get_deleted.mdwn b/doc/forum/views___40__branches__41___never_get_deleted.mdwn
new file mode 100644
index 0000000000..bba7762dae
--- /dev/null
+++ b/doc/forum/views___40__branches__41___never_get_deleted.mdwn
@@ -0,0 +1,16 @@
+Hello everyone,
+I would like to know if this is normal behavior or if it's a problem with my repository:
+
+Whenever I set a view with 
+
+git annex view attr="\*"'
+
+a new branch representing the selected view gets created, as expected. The problem is that when I switch back to master ('git checkout master' or even 'git annex vpop') the view branch stays there, and all subsequent operations on the annex also consider the view branch, resulting a great slowdown if one has done many views (attr="this", attr="that", etc.). Is this normal? If so, why is it necessary for the branch to stay on? Does it speed up going back to the branch? Redoing git annex view attr="*" does not seem to take less time. 
+
+Am I doing it wrong? Should I be deleting used view branches on my own? How?
+
+thanks for your replies.
+
+**EDIT:** I just found out that even if I delete view branches with git branch -D "views/attr=_" (which I'm not sure I should be doing), the branches are still checked when doing "git annex unused". That is, "git annex unused" lists "checking..." a whole lot of past views/branches which are not even there anymore (not listed with "git branch"). I also suspect that this is preventing deleted (git-rm) files from being collected from "unused". Is this a problem with my repo? Any way to fix this?
+
+=== git-annex version: **5.20140529-gb71f9bf** ===
diff --git a/doc/forum/views___40__branches__41___never_get_deleted/comment_1_ff53fa0b5f0c4a6554a37e3309e26925._comment b/doc/forum/views___40__branches__41___never_get_deleted/comment_1_ff53fa0b5f0c4a6554a37e3309e26925._comment
new file mode 100644
index 0000000000..61fcbfeb47
--- /dev/null
+++ b/doc/forum/views___40__branches__41___never_get_deleted/comment_1_ff53fa0b5f0c4a6554a37e3309e26925._comment
@@ -0,0 +1,12 @@
+[[!comment format=mdwn
+ username="http://joeyh.name/"
+ ip="108.236.230.124"
+ subject="comment 1"
+ date="2014-06-04T18:04:30Z"
+ content="""
+You can delete the view branches if you like. I have considered making moving views clean up old branches, but I have also considered reusing existing view branches when popping back to an old view.
+
+git-annex unused can't look at branches that aren't there. Perhaps you have pushed the view branches to a remote repository, and it's checking those branches of the remote? See git branch -a.
+
+I think that it makes sense for unused to ignore (local) view branches, since these are by definition supposed to be views of an existing branch, so looking at the branch should be sufficient (and if the view is out of date and has files that have since been deleted from the branch, the user's intent is not to preserve those from unused reaping). So, made that change.
+"""]]
diff --git a/doc/forum/views___40__branches__41___never_get_deleted/comment_3_1d4a3f4e83b288262e291262a6636602._comment b/doc/forum/views___40__branches__41___never_get_deleted/comment_3_1d4a3f4e83b288262e291262a6636602._comment
new file mode 100644
index 0000000000..0a5d61e548
--- /dev/null
+++ b/doc/forum/views___40__branches__41___never_get_deleted/comment_3_1d4a3f4e83b288262e291262a6636602._comment
@@ -0,0 +1,21 @@
+[[!comment format=mdwn
+ username="https://www.google.com/accounts/o8/id?id=AItOawmYMS6tQ8q9k1fylGyA9Q6fdFzRBotME2Q"
+ nickname="tom"
+ subject="EDIT reply comment:"
+ date="2014-06-04T19:44:18Z"
+ content="""
+> *You can delete the view branches if you like. I have considered making moving views clean up old branches, but I have also considered reusing existing view branches when popping back to an old view.*
+
+Alright, I'd very much welcome anyone of the two possibilities as I'm making heavy use of views and would benefit from any of the two solutions.
+
+> *Perhaps you have pushed the view branches to a remote repository, and it's checking those branches of the remote? See git branch -a.*
+
+That's it! Solution for future reference: The remote branches referenced in local repo did not exist anymore in any repo (I had deleted them earlier from the remote) so \"git push origin --delete \\" wouldn't work. This other command worked very well in cleaning up the overwhelming missing remote views all in one go: \"git fetch --prune \\"
+
+> *I think that it makes sense for unused to ignore (local) view branches, since these are by definition supposed to be views of an existing branch, so looking at the branch should be sufficient. So, made that change.*
+
+Thanks a lot! It makes perfect sense, looking forward to it on the next release.
+
+Now, it seems that even after pruning the view branches on the remote, *unused* would still not collect deleted files. I'm wondering if this is because there are other offline repos (backups which are not practical to take out of their resting place) I have not yet synchronized. I can see their main branches with git branch -a: remotes/oldbackup/git-annex, remotes/oldbackup/master, remotes/oldbackup/synced/git-annex, remotes/oldbackup/synced/master. Does this mean that until all repos (even the offline ones I never sync and work mostly as a backup for old stuff) are synchronized, 'unused' won't collect the deleted files? If so, is there a workaround for this?
+
+"""]]
diff --git a/doc/forum/views___40__branches__41___never_get_deleted/comment_3_4e96e5325fd12e48f190fe551a6ac07e._comment b/doc/forum/views___40__branches__41___never_get_deleted/comment_3_4e96e5325fd12e48f190fe551a6ac07e._comment
new file mode 100644
index 0000000000..7ac31b1e9b
--- /dev/null
+++ b/doc/forum/views___40__branches__41___never_get_deleted/comment_3_4e96e5325fd12e48f190fe551a6ac07e._comment
@@ -0,0 +1,12 @@
+[[!comment format=mdwn
+ username="https://www.google.com/accounts/o8/id?id=AItOawmYMS6tQ8q9k1fylGyA9Q6fdFzRBotME2Q"
+ nickname="tom"
+ subject="added comment"
+ date="2014-06-04T22:41:40Z"
+ content="""
+Specifically I'm looking for a workaround that would make this behavior possible in this use case:
+
+The user decides some file in the local repository are not needed anymore, they are not needed to be in any repository at all and wants to reclaim the space as soon as possible (or at least make the files available to 'unused' for later removal). Removing the content the from local repository (git-rm ?) the space can be immediately reclaimed in the local hard drive, without the need to connect and sync unavailable/far remotes. As the other remotes and old backups become available and sync, they delete the unused content as well (or at least make it available to 'unused').
+
+Is there a way to achieve this? 
+"""]]
diff --git a/doc/forum/views___40__branches__41___never_get_deleted/comment_4_a4764b5bfb08ebe90430ea14fcb6e8e0._comment b/doc/forum/views___40__branches__41___never_get_deleted/comment_4_a4764b5bfb08ebe90430ea14fcb6e8e0._comment
new file mode 100644
index 0000000000..762e4db533
--- /dev/null
+++ b/doc/forum/views___40__branches__41___never_get_deleted/comment_4_a4764b5bfb08ebe90430ea14fcb6e8e0._comment
@@ -0,0 +1,10 @@
+[[!comment format=mdwn
+ username="http://joeyh.name/"
+ ip="108.236.230.124"
+ subject="comment 4"
+ date="2014-06-05T17:35:31Z"
+ content="""
+`git annex drop --force $file; git rm $file`
+
+Makes sense in some situations anyway..
+"""]]
diff --git a/doc/forum/views___40__branches__41___never_get_deleted/comment_5_02e08dffb01246010b390aeef8f32234._comment b/doc/forum/views___40__branches__41___never_get_deleted/comment_5_02e08dffb01246010b390aeef8f32234._comment
new file mode 100644
index 0000000000..a9b72087e0
--- /dev/null
+++ b/doc/forum/views___40__branches__41___never_get_deleted/comment_5_02e08dffb01246010b390aeef8f32234._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="https://www.google.com/accounts/o8/id?id=AItOawmYMS6tQ8q9k1fylGyA9Q6fdFzRBotME2Q"
+ nickname="tom"
+ subject="comment 5"
+ date="2014-06-06T09:57:32Z"
+ content="""
+ah, I get it. Thanks!
+"""]]
diff --git a/doc/forum/vlc_and_git-annex.mdwn b/doc/forum/vlc_and_git-annex.mdwn
new file mode 100644
index 0000000000..cb07f8183c
--- /dev/null
+++ b/doc/forum/vlc_and_git-annex.mdwn
@@ -0,0 +1,11 @@
+I used to save movies with the srt subtitle files next to them. 
+
+Usually vlc finds it because it's on the same directory than the movie file, however with git annex the link is located on another folder.
+So after adding movies to git, the subtitles doesn't load anymore.
+
+couldn't find a quick fix. I'm thinking a bash script, but wanted to discuss it here with all annex users.
+
+I know It's out of annex scope, but I think a movie archive is a great scenario for git-annex.
+most of my HD is filled up with movies from the camcorder, screencast, etc... 
+And we usually don't modify those files
+
diff --git a/doc/forum/vlc_and_git-annex/comment_1_9c9ab8ce463cf74418aa2f385955f165._comment b/doc/forum/vlc_and_git-annex/comment_1_9c9ab8ce463cf74418aa2f385955f165._comment
new file mode 100644
index 0000000000..700b3808de
--- /dev/null
+++ b/doc/forum/vlc_and_git-annex/comment_1_9c9ab8ce463cf74418aa2f385955f165._comment
@@ -0,0 +1,10 @@
+[[!comment format=mdwn
+ username="http://joey.kitenet.net/"
+ nickname="joey"
+ subject="comment 1"
+ date="2011-12-23T16:16:19Z"
+ content="""
+From what you say, it seems that vlc is following the symlink to the movie content, and then looking for subtitles next to the file the symlink points to. It would have to explicitly realpath the symlink to have this behavior, and this sounds like a misfeature.. perhaps you could point out to the vlc people the mistake in doing so?
+
+There's a simple use-case where this behavior is obviously wrong, without involving git-annex. Suppose I have a movie, and one version of subtitles for it, in directory `foo`. I want to modify the subtitles, so I make a new directory `bar`, symlink the large movie file from `foo` to save space, and copy over and edit the subtitles from `foo`. Now I run vlc in `bar` to test my new subtitles. If it ignores the locally present subtitles and goes off looking for the ones in `bar`, I say this is broken behavior.
+"""]]
diff --git a/doc/forum/vlc_and_git-annex/comment_2_037f94c1deeac873dbdb36cd4c927e45._comment b/doc/forum/vlc_and_git-annex/comment_2_037f94c1deeac873dbdb36cd4c927e45._comment
new file mode 100644
index 0000000000..3c69f5fe46
--- /dev/null
+++ b/doc/forum/vlc_and_git-annex/comment_2_037f94c1deeac873dbdb36cd4c927e45._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="http://joey.kitenet.net/"
+ nickname="joey"
+ subject="comment 2"
+ date="2011-12-23T18:43:05Z"
+ content="""
+Since subtitle files are typically pretty small, a workaround is to simply check them into git directly, and only use git-annex for the movies. (Or `git annex unannex` the ones you've already annexed.)
+"""]]
diff --git a/doc/forum/warning__58___remote_HEAD_refers_to_nonexistent_ref__44___unable_to_checkout.mdwn b/doc/forum/warning__58___remote_HEAD_refers_to_nonexistent_ref__44___unable_to_checkout.mdwn
new file mode 100644
index 0000000000..f57e0ee838
--- /dev/null
+++ b/doc/forum/warning__58___remote_HEAD_refers_to_nonexistent_ref__44___unable_to_checkout.mdwn
@@ -0,0 +1,69 @@
+I am struggling for so long time, everytime getting different errors :( :(
+
+I digged up my backup and started from scratch. Still not working.
+
+On the (linux) server (directory contains already files):
+
+     cd bin
+     git init
+     git annex init server
+     git annex direct
+     git annex add .
+
+Then, on the Windows client:
+
+     $ git clone ssh://me@server/srv/data/bin bin
+     Cloning into 'bin'...
+     warning: remote HEAD refers to nonexistent ref, unable to checkout.
+
+What's going on here?
+
+EDIT: Of course, I would not ask if everything would work as expected. But the client repository in windows is always empty, no matter what I do (issueing git annex init, git add remote, git annex sync, ...) does not seem to help me ...
+
+EDIT2: When I continue on the windows side (despite the warning):
+
+     cd bin
+     git annex init client
+     git annex sync
+     Permission denied (publickey,keyboard-interactive).
+     (merging origin/git-annex origin/synced/git-annex into git-annex...)
+     (Recording state in git...)
+     
+       Remote origin does not have git-annex installed; setting annex-ignore
+     commit  ok
+     pull origin Already up-to-date!
+     Merge made by the 'recursive' strategy.
+     
+     
+     ok
+     push origin To ssh://me@server/srv/data/bin
+        b79922f..4fe0505  git-annex -> synced/git-annex
+        2f6a601..02a2603  annex/direct/master -> synced/master
+     
+     ok
+
+
+But that's not true, git-annex is definitely installed on the server (it's just the Debian package and it resides in /usr/bin/git-annex). In any case, the client repository is still empty
+
+EDIT3: Struggle, struggle, ...
+
+Finally, for whatever reason, the client dir is not empty. What seems to be missing on the server was:
+
+     git annex sync
+
+but this is never written (also not in the walkthrough). Can anyone tell me if it's required and why?
+
+Furthermore, even that I "see" at least the file references now, I am not able to get them:
+
+     $ git annex get apps
+     get apps/gitignore (not available)
+       Try making some of these repositories available:
+             10d0ffd8-f499-4b55-83e6-ff58cd98edc5 -- server
+     
+       (Note that these git remotes have annex-ignore set: origin)
+     failed
+     git-annex: get: 1 failed
+
+That's an SSH remote, it *is* available! What is meant by that?
+
+
diff --git a/doc/forum/warning__58___remote_HEAD_refers_to_nonexistent_ref__44___unable_to_checkout/comment_1_c0d9758be80d1a349ffe82c80075bebd._comment b/doc/forum/warning__58___remote_HEAD_refers_to_nonexistent_ref__44___unable_to_checkout/comment_1_c0d9758be80d1a349ffe82c80075bebd._comment
new file mode 100644
index 0000000000..3601e2639c
--- /dev/null
+++ b/doc/forum/warning__58___remote_HEAD_refers_to_nonexistent_ref__44___unable_to_checkout/comment_1_c0d9758be80d1a349ffe82c80075bebd._comment
@@ -0,0 +1,12 @@
+[[!comment format=mdwn
+ username="divB"
+ ip="171.67.174.99"
+ subject="comment 1"
+ date="2014-05-10T21:20:36Z"
+ content="""
+I think, although not sure, that all of this is related of failing/inconsistent ssh calls.
+
+I filed a bug here: https://git-annex.branchable.com/bugs/git-annex_ignores_GIT__95__SSH
+
+The problem is that failure in ssh does not result in proper error messages in git-annex, resulting in inconsistent repositories.
+"""]]
diff --git a/doc/forum/warning__58___remote_HEAD_refers_to_nonexistent_ref__44___unable_to_checkout/comment_2_c28dae2eb0ab825ee6d43735e04a18a3._comment b/doc/forum/warning__58___remote_HEAD_refers_to_nonexistent_ref__44___unable_to_checkout/comment_2_c28dae2eb0ab825ee6d43735e04a18a3._comment
new file mode 100644
index 0000000000..97671a2c52
--- /dev/null
+++ b/doc/forum/warning__58___remote_HEAD_refers_to_nonexistent_ref__44___unable_to_checkout/comment_2_c28dae2eb0ab825ee6d43735e04a18a3._comment
@@ -0,0 +1,22 @@
+[[!comment format=mdwn
+ username="http://joeyh.name/"
+ ip="108.236.230.124"
+ subject="comment 2"
+ date="2014-05-16T16:49:09Z"
+ content="""
+After you add a file to a git repository, you have to git commit it. Using git-annex is no different. The walkthrough shows using `git commit`: ; you can use `git-annex sync` if you prefer.
+
+
    +Permission denied (publickey,keyboard-interactive).
+ (merging origin/git-annex origin/synced/git-annex into git-annex...)
+ (Recording state in git...)
+
+   Remote origin does not have git-annex installed; setting annex-ignore
+
    
+
+git-annex uses a heuristic to determine if git-annex-shell is not installed on a remote server: It tries to run it, and if that fails, it tries to run git fetch. If that succeeds it assumes this means it can log into the server, but git-annex-shell is not installed.
+
+Above, that heuristic seems to have failed; you seem to have given the wrong password or something so it didn't run git-annex-shell, but then it was able to log in for the git fetch. 
+
+You will need to run this command to fix it up: `git config remote.origin.annex-ignore false`
+"""]]
diff --git a/doc/forum/web-browser_for_the_repository.mdwn b/doc/forum/web-browser_for_the_repository.mdwn
new file mode 100644
index 0000000000..40a332b6e8
--- /dev/null
+++ b/doc/forum/web-browser_for_the_repository.mdwn
@@ -0,0 +1,10 @@
+Hi!
+
+I am a teacher and I would like to organize my school-files with git-annex in combination with a wiki. In the wiki I would like to outline lessons with included pictures (and links) of work-sheets etc. git-annex would help me to transfer and organize all the files. 
+
+Now I'm searching for a web-browser for a git-annex-repository, like the source-browser of trac (or gitweb). Actually I would really like to use trac (or dokuwiki) for my school-project. I tried trac with a bare-annex-repos, but it only shows the hash-links. In the web-assistent of annex I couldn't find such a feature.
+
+
+Is there any web-browser for a git-annex-repository?
+
+-- Tobias 
diff --git a/doc/forum/web-browser_for_the_repository/comment_1_1f8e7e54e9f377417cd672d66ae9bd9d._comment b/doc/forum/web-browser_for_the_repository/comment_1_1f8e7e54e9f377417cd672d66ae9bd9d._comment
new file mode 100644
index 0000000000..32296609a0
--- /dev/null
+++ b/doc/forum/web-browser_for_the_repository/comment_1_1f8e7e54e9f377417cd672d66ae9bd9d._comment
@@ -0,0 +1,9 @@
+[[!comment format=mdwn
+ username="Kalle"
+ subject="Different solution"
+ date="2016-03-21T21:02:57Z"
+ content="""
+Hi
+
+Are you able to install git-annex on the server? If so perhaps switching the server repo to **direct mode** will work. Then you should be able to use any directory listing method you like. My guess is that you need to make sure the file listing software is configured to ignore `.git` directories. I've never used TRAC so can't tell if this is possible or required.
+"""]]
diff --git a/doc/forum/web-browser_for_the_repository/comment_2_551d5bb2d24212de65720d796365177f._comment b/doc/forum/web-browser_for_the_repository/comment_2_551d5bb2d24212de65720d796365177f._comment
new file mode 100644
index 0000000000..dfee3959e5
--- /dev/null
+++ b/doc/forum/web-browser_for_the_repository/comment_2_551d5bb2d24212de65720d796365177f._comment
@@ -0,0 +1,9 @@
+[[!comment format=mdwn
+ username="Tobias"
+ subject="bare repository in direct mode"
+ date="2016-03-21T21:28:57Z"
+ content="""
+It is not possible to put a bare repository in direct mode. I tried it and it did not work. 
+
+Tobias
+"""]]
diff --git a/doc/forum/web-browser_for_the_repository/comment_3_d410df4b7160f17e3846674c3e0e9368._comment b/doc/forum/web-browser_for_the_repository/comment_3_d410df4b7160f17e3846674c3e0e9368._comment
new file mode 100644
index 0000000000..c84959a570
--- /dev/null
+++ b/doc/forum/web-browser_for_the_repository/comment_3_d410df4b7160f17e3846674c3e0e9368._comment
@@ -0,0 +1,9 @@
+[[!comment format=mdwn
+ username="Kalle"
+ subject="comment 3"
+ date="2016-03-23T21:22:15Z"
+ content="""
+A normal direct mode repo with git annex installed on the server should give you a synced directory structure of files that you can browse with most directory listing software/modules and download files.
+
+Are you looking to enable something beyond browsing and downloading the files?
+"""]]
diff --git a/doc/forum/web-browser_for_the_repository/comment_4_d06217403bf53a713b845bbc9ca8b962._comment b/doc/forum/web-browser_for_the_repository/comment_4_d06217403bf53a713b845bbc9ca8b962._comment
new file mode 100644
index 0000000000..e0ce7359bc
--- /dev/null
+++ b/doc/forum/web-browser_for_the_repository/comment_4_d06217403bf53a713b845bbc9ca8b962._comment
@@ -0,0 +1,7 @@
+[[!comment format=mdwn
+ username="Tobias"
+ subject="History"
+ date="2016-03-25T15:24:21Z"
+ content="""
+I'm not only interested in the current state of the files but also in a browser for the revisions. Therefore I need a \"real\" repository-browser. 
+"""]]
diff --git a/doc/forum/webapp__58___disabling_a_paired_repository.mdwn b/doc/forum/webapp__58___disabling_a_paired_repository.mdwn
new file mode 100644
index 0000000000..bc2c809c35
--- /dev/null
+++ b/doc/forum/webapp__58___disabling_a_paired_repository.mdwn
@@ -0,0 +1,12 @@
+I had two repositories, on different machines, created with the assistant, so they were both direct mode.
+
+After pairing each with the other, so that files were merrily flowing back and forth, I happened to do the following:
+ * on machine A, in the dashboard
+ * click on the 'actions' emnu for the repository on machine B
+ * select 'disable'
+
+Result: all references to the repo on machine B have disappeared from the assistant in machine A.
+
+On machine B, I still have the pairing with machine A, and selecting the 'sync now' action seems to be working.
+
+The thing I am curious about is the behaviour on machine A - why is it that I cannot find any reference to it any more in the assistant, so that I can re-enable it?
diff --git a/doc/forum/webapp__58___disabling_a_paired_repository/comment_1_9dd316fedf55afecbc77f3b99e66d837._comment b/doc/forum/webapp__58___disabling_a_paired_repository/comment_1_9dd316fedf55afecbc77f3b99e66d837._comment
new file mode 100644
index 0000000000..077832deb4
--- /dev/null
+++ b/doc/forum/webapp__58___disabling_a_paired_repository/comment_1_9dd316fedf55afecbc77f3b99e66d837._comment
@@ -0,0 +1,18 @@
+[[!comment format=mdwn
+ username="vincent.mcintyre@1318ebde5cb96fc17e59dfa86f399f5634b1facc"
+ nickname="vincent.mcintyre"
+ subject="addendum"
+ date="2015-08-25T10:45:56Z"
+ content="""
+Syncing seems to work only in one direction.
+
+If I  go to machine B and:
+ * change a file
+ * select 'sync now' from the 'actions' menu for machine A
+the webapp says it worked.
+
+But if I login to machine A and inspect the file I expect to have changed, it is 'not available'.
+
+If I run 'git annex get 'baz.txt' on machine A it complains and asks me to make available the reposiotry on machine B.
+
+"""]]
diff --git a/doc/forum/webapp__58___disabling_a_paired_repository/comment_2_28cf58629b9774426c084d80964151d7._comment b/doc/forum/webapp__58___disabling_a_paired_repository/comment_2_28cf58629b9774426c084d80964151d7._comment
new file mode 100644
index 0000000000..5d78439b67
--- /dev/null
+++ b/doc/forum/webapp__58___disabling_a_paired_repository/comment_2_28cf58629b9774426c084d80964151d7._comment
@@ -0,0 +1,19 @@
+[[!comment format=mdwn
+ username="joey"
+ subject="""comment 2"""
+ date="2015-09-21T16:55:47Z"
+ content="""
+Disabling a remote is a weak form of deleting it; the remote repository is
+left as-is, but the git remote is removed so git-annex won't sync with it,
+and the webapp won't show it.
+
+I don't know if it makes sense for the webapp to expose this possibility.
+It seems that it certianly doesn't makes sense to do it without getting
+confirmation, since to re-enable the repository, the user has to do some
+probably unknown to them process to set it back up.
+
+In your case, I think that re-doing the pairing process would get them back
+to how they were.
+
+On consideration, I am going to remove this feature from the webapp UI.
+"""]]
diff --git a/doc/forum/webapp__58___disabling_a_paired_repository/comment_3_799d66b116e8ddf42e624bb4a082337c._comment b/doc/forum/webapp__58___disabling_a_paired_repository/comment_3_799d66b116e8ddf42e624bb4a082337c._comment
new file mode 100644
index 0000000000..1e38b31851
--- /dev/null
+++ b/doc/forum/webapp__58___disabling_a_paired_repository/comment_3_799d66b116e8ddf42e624bb4a082337c._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="https://www.google.com/accounts/o8/id?id=AItOawnmF_9CAtfqdZkC4e-_dCX-rK5bqh4RWkw"
+ nickname="Carl"
+ subject="Thanks"
+ date="2015-09-24T13:30:26Z"
+ content="""
+I think this is a very good usability improvement. I hav more than once wanted to temporarily turn off syncing (for example for a server inly available when on my lan at home) and ended up disabling the remote altogether. One possibility less to shoot myself in the foot. Thanks!
+"""]]
diff --git a/doc/forum/webapp___47___assistant_without_watch.mdwn b/doc/forum/webapp___47___assistant_without_watch.mdwn
new file mode 100644
index 0000000000..9bb1fcf1ad
--- /dev/null
+++ b/doc/forum/webapp___47___assistant_without_watch.mdwn
@@ -0,0 +1,9 @@
+I did not recieve feedback on my comment [1], so I try to post my question again but more clearly.
+
+Is it possible to run the assistent/the webapp without the functionality of 'git annex watch'?
+
+I'd like to use the assistant and to have the automatic syncing but I do not want the local repository to be watched.  Instead I prefer to manually add/drop my files.
+
+I do not see the 'pause button' mentioned in my earlier question [1].
+
+[1] http://git-annex.branchable.com/forum/webapp_and_manual_mode/
diff --git a/doc/forum/webapp___47___assistant_without_watch/comment_1_1bcd99aa81f937ded683e19a69d33dd9._comment b/doc/forum/webapp___47___assistant_without_watch/comment_1_1bcd99aa81f937ded683e19a69d33dd9._comment
new file mode 100644
index 0000000000..c76128ae39
--- /dev/null
+++ b/doc/forum/webapp___47___assistant_without_watch/comment_1_1bcd99aa81f937ded683e19a69d33dd9._comment
@@ -0,0 +1,10 @@
+[[!comment format=mdwn
+ username="http://joeyh.name/"
+ ip="4.154.0.63"
+ subject="comment 1"
+ date="2013-08-26T18:14:17Z"
+ content="""
+The assistant is currently only able to transfer files that it has added itself. So if you disable syncing, you have to manually upload any files you add.
+
+I doubt that I will change this in the assistant, because supporting this use case would complicate it unnecessarily for a use case that is not what it's designed to do. It's more likely that `git annex sync` will get an option to also transfer file contents.
+"""]]
diff --git a/doc/forum/webapp___47___assistant_without_watch/comment_2_9f5b3f5bf7fedcd5baec519d97d3aa8c._comment b/doc/forum/webapp___47___assistant_without_watch/comment_2_9f5b3f5bf7fedcd5baec519d97d3aa8c._comment
new file mode 100644
index 0000000000..a441558768
--- /dev/null
+++ b/doc/forum/webapp___47___assistant_without_watch/comment_2_9f5b3f5bf7fedcd5baec519d97d3aa8c._comment
@@ -0,0 +1,16 @@
+[[!comment format=mdwn
+ username="ringprince"
+ ip="134.76.140.110"
+ subject="comment 2"
+ date="2013-08-26T18:54:40Z"
+ content="""
+Thanks for the clarification.
+
+Although
+
+* not what I hoped for ;-)
+* I don't see that it would get complicated
+* I do not consider my use case special
+
+Anyway, thanks for this great piece of software.
+"""]]
diff --git a/doc/forum/webapp_and_manual_mode.mdwn b/doc/forum/webapp_and_manual_mode.mdwn
new file mode 100644
index 0000000000..f7a903387b
--- /dev/null
+++ b/doc/forum/webapp_and_manual_mode.mdwn
@@ -0,0 +1,7 @@
+Somehow I seemingly have not understood the manual mode.
+
+If I have a file in my repository, that is not yet in the annex (or in git), and I start the webapp, that file is automatically added to git-annex (and git).
+
+Why is that?
+
+And how can I use the webapp without such interference?
diff --git a/doc/forum/webapp_and_manual_mode/comment_1_5b5df5ffeb6ee15779972f13fdc11729._comment b/doc/forum/webapp_and_manual_mode/comment_1_5b5df5ffeb6ee15779972f13fdc11729._comment
new file mode 100644
index 0000000000..677dd04148
--- /dev/null
+++ b/doc/forum/webapp_and_manual_mode/comment_1_5b5df5ffeb6ee15779972f13fdc11729._comment
@@ -0,0 +1,10 @@
+[[!comment format=mdwn
+ username="http://joeyh.name/"
+ ip="4.154.0.140"
+ subject="comment 1"
+ date="2013-07-21T17:17:58Z"
+ content="""
+Setting a repository to manual mode prevents the assistant from transferring new files into that repository. You have to manually run `git annex get`.
+
+To prevent the assistant from automatically adding files, you can click the pause button next to the current repository in the webapp.
+"""]]
diff --git a/doc/forum/webapp_and_manual_mode/comment_2_a1f06b50d1317c78a301b47eb05d2617._comment b/doc/forum/webapp_and_manual_mode/comment_2_a1f06b50d1317c78a301b47eb05d2617._comment
new file mode 100644
index 0000000000..99efad90a2
--- /dev/null
+++ b/doc/forum/webapp_and_manual_mode/comment_2_a1f06b50d1317c78a301b47eb05d2617._comment
@@ -0,0 +1,19 @@
+[[!comment format=mdwn
+ username="ringprince"
+ ip="134.76.2.1"
+ subject="comment 2"
+ date="2013-07-23T08:47:58Z"
+ content="""
+Not sure, what you mean.
+
+Disabling sync?
+But that also disables sync, right?  ;-)  So, if I add a file manually (via command line) that file is not synced automatically, correct?
+
+I do not see any other 'pause' button next to the repository.
+
+I am fine with git-annex doing the syncing automatically for me, but I want to be in control over the files git annex actually uses.
+
+If I just downloaded a big file, I might want to check it first before having git-annex move it to other repositories.
+
+
+"""]]
diff --git a/doc/forum/webapp_and_manual_mode/comment_3_f0739bf4304a91a5d4ec33ac2421c966._comment b/doc/forum/webapp_and_manual_mode/comment_3_f0739bf4304a91a5d4ec33ac2421c966._comment
new file mode 100644
index 0000000000..04369e7cd7
--- /dev/null
+++ b/doc/forum/webapp_and_manual_mode/comment_3_f0739bf4304a91a5d4ec33ac2421c966._comment
@@ -0,0 +1,9 @@
+[[!comment format=mdwn
+ username="https://www.google.com/accounts/o8/id?id=AItOawkAUMhKOSkh9JaBA6xst3XxQIIsDEq5Zd4"
+ nickname="Ovidiu"
+ subject="Also confused by the manual mode"
+ date="2014-05-18T06:39:54Z"
+ content="""
+@Joeyh: would you mind explaining the modes maybe on their own wiki page? 
+I've got most of them but the sense and functioning of the manual method escapes me :-/
+"""]]
diff --git a/doc/forum/webapp_does_not_start.mdwn b/doc/forum/webapp_does_not_start.mdwn
new file mode 100644
index 0000000000..ad42759707
--- /dev/null
+++ b/doc/forum/webapp_does_not_start.mdwn
@@ -0,0 +1,72 @@
+Sorry, I have again an issue :(
+
+What could be the problem that the webapp is not starting (in Windows)? git annex just exists without messages, connection to localhost failes of course
+
+    me@laptop /cygdrive/c/Data
+    $ git annex --verbose --debug webapp
+    
+    me@laptop /cygdrive/c/Data
+    $
+
+If I run it within an annexed directory I get at least one message - but it still immideately exits:
+
+    me@laptop /cygdrive/c/Data/annex
+    $ git annex --verbose --debug webapp
+    [2014-07-11 19:06:58 Pacific Daylight Time] chat: git-annex ["--verbose","--debug","webapp"]
+
+    me@laptop /cygdrive/c/Data/annex
+    $
+
+    git-annex version: 5.20140707-g8116d10
+    build flags: Assistant Webapp Webapp-secure Pairing Testsuite S3 WebDAV DNS Feeds Quvi TDFA CryptoHash
+    key/value backends: SHA256E SHA1E SHA512E SHA224E SHA384E SKEIN256E SKEIN512E SHA256 SHA1 SHA512 SHA224 SHA384 SKEIN256 SKEIN512 WORM URL
+    remote types: git gcrypt S3 bup directory rsync web webdav tahoe glacier ddar hook external
+    local repository version: 5
+    supported repository version: 5
+    upgrade supported from repository versions: 2 3 4
+
+
+EDIT:
+1.) Interestingly, also "git annex test" fails:
+
+    [...]
+    ok
+    push origin To C:/Data/.t\repo
+       bcc4611..91f5218  git-annex -> synced/git-annex
+       6f47922..92c578a  annex/direct/master -> synced/master
+    
+    ok
+    add ../dir2/foo ok
+    (Recording state in git...)
+    FAIL
+          Exception: .t/tmprepo9/.git/annex/journal: getDirectoryContents: permission denied (Access is denied.)
+
+    2 out of 83 tests failed
+      (This could be due to a bug in git-annex, or an incompatibility
+       with utilities, such as git, installed on this system.)
+
+
+
+2.) There's really something very weird going on. At the beginning it worked. Then it stopped (as per above). Sometimes, when I start it using "Git Bash" it works. But then not any more.
+3.) Rebooting the machine does not help
+
+EDIT once again:
+4.) I am afraid there is something big broken. Suddenly I could start the daemon again. The thing I did was to delete my complete annex.
+But if I shutdown the daemon now I get:
+
+    $ git annex webapp
+    Launching web browser on file://C:\cygwin\tmp\webapp85796.html
+    
+      Detected a filesystem without fifo support.
+    
+      Disabling ssh connection caching.
+    
+      Detected a crippled filesystem.
+    
+      Enabling direct mode.
+    
+    (Recording state in git...)
+    WebApp crashed: ExitFailu
+
+... and the whole loop starts again. webapp daemon does not start as long as I delete the annex. Interestingly this is independent from %USERPROFILE%\.config. Deleting this directory does not change anything. So somehow git-annex "knows" that there is somewhere an annex and fails if it is.
+
diff --git a/doc/forum/webapp_does_not_start/comment_1_dd27d30ce305562a1552f46c87b1cd27._comment b/doc/forum/webapp_does_not_start/comment_1_dd27d30ce305562a1552f46c87b1cd27._comment
new file mode 100644
index 0000000000..cd3a7d3bc7
--- /dev/null
+++ b/doc/forum/webapp_does_not_start/comment_1_dd27d30ce305562a1552f46c87b1cd27._comment
@@ -0,0 +1,12 @@
+[[!comment format=mdwn
+ username="http://joeyh.name/"
+ ip="209.250.56.2"
+ subject="comment 1"
+ date="2014-07-14T19:05:32Z"
+ content="""
+git-annex stores the locations of repositories in `.config/git-annex/autostart` in what passes for a home directory on Windows.
+
+Are you sure it doesn't manage to at least open a web browser tab, possibly one with an error in it? This would be easy to miss if you have a lot of tabs open. I seem to have partially reproduced this, at least that's the behavior I see. 
+
+The root of the problem seems to be that it thinks the daemon is running, but it's not. So it just opens the web browser and then stops. After I deleted .git/annex/daemon*, and .git/annex/url and .git/annex/daemon.log, `git annex webapp` worked again. 
+"""]]
diff --git a/doc/forum/webapp_does_not_start/comment_2_ef37f40288a1181ca619ae13b0f7a994._comment b/doc/forum/webapp_does_not_start/comment_2_ef37f40288a1181ca619ae13b0f7a994._comment
new file mode 100644
index 0000000000..551f518459
--- /dev/null
+++ b/doc/forum/webapp_does_not_start/comment_2_ef37f40288a1181ca619ae13b0f7a994._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="http://joeyh.name/"
+ ip="209.250.56.2"
+ subject="comment 2"
+ date="2014-07-14T20:06:03Z"
+ content="""
+I think that I have fixed this problem. The Windows autobuilder should produce a fixed build within an hour.
+"""]]
diff --git a/doc/forum/webapp_does_not_start/comment_3_6e625dba9f7fa36bf9c7e9d77fbadeff._comment b/doc/forum/webapp_does_not_start/comment_3_6e625dba9f7fa36bf9c7e9d77fbadeff._comment
new file mode 100644
index 0000000000..296e066a7a
--- /dev/null
+++ b/doc/forum/webapp_does_not_start/comment_3_6e625dba9f7fa36bf9c7e9d77fbadeff._comment
@@ -0,0 +1,9 @@
+[[!comment format=mdwn
+ username="divB"
+ ip="128.12.90.218"
+ subject="comment 3"
+ date="2014-07-15T05:43:35Z"
+ content="""
+Thank you, really really great. This part seems to work again.
+For some reason, the assistant takes the wrong (?) git.exe (as I described here: http://git-annex.branchable.com/forum/Restricting_SSH_+_supply_key/)
+"""]]
diff --git a/doc/forum/webapp_listen_port_with_autostart.mdwn b/doc/forum/webapp_listen_port_with_autostart.mdwn
new file mode 100644
index 0000000000..fc9d1241b8
--- /dev/null
+++ b/doc/forum/webapp_listen_port_with_autostart.mdwn
@@ -0,0 +1,3 @@
+To start the webapp on a X-less server it's possible to use the parameter `--listen` and then connect from a client with X to it.
+
+How can I start the webapp on all annex repos on this server like on a client with `assistant --autostart`? Would be nice to start the assistant on all configured repos which listens on a defined port, f.e. `git annex assistant --autostart --listen 0.0.0.0:8888`. It would be even nicer to have a parameter to chose a autostart configuration file, f.e. `git annex assistant --autostart --autostart-config /etc/git-annex/autostart --listen 0.0.0.0:8888`, so this could end up in a real server application, a init script would be easy to write...
diff --git a/doc/forum/webapp_listen_port_with_autostart/comment_1_65dbcf3d8f6c16568f5a326242eab9c5._comment b/doc/forum/webapp_listen_port_with_autostart/comment_1_65dbcf3d8f6c16568f5a326242eab9c5._comment
new file mode 100644
index 0000000000..0c430cf7c6
--- /dev/null
+++ b/doc/forum/webapp_listen_port_with_autostart/comment_1_65dbcf3d8f6c16568f5a326242eab9c5._comment
@@ -0,0 +1,20 @@
+[[!comment format=mdwn
+ username="http://joeyh.name/"
+ nickname="joey"
+ subject="comment 1"
+ date="2013-05-13T18:49:25Z"
+ content="""
+Well, it would be possible to add a configuration setting in the repository that tells the assistant what address:port to listen on.
+
+But maybe it would be better to not use `assistant --autostart` in your situation? That command is basically the same as this shell script:
+
+[[!format sh "
+#!/bin/sh
+for dir in $(cat $HOME/.config/git-annex/autostart); do
+    cd $dir
+    git annex webapp &
+done
+"]]
+
+So you can write similar shell scripts that start the webapp with whatever options you like.
+"""]]
diff --git a/doc/forum/webapp_listen_port_with_autostart/comment_2_39664f833dedc1a4fe083eec9bc4a7cd._comment b/doc/forum/webapp_listen_port_with_autostart/comment_2_39664f833dedc1a4fe083eec9bc4a7cd._comment
new file mode 100644
index 0000000000..0d48ba8255
--- /dev/null
+++ b/doc/forum/webapp_listen_port_with_autostart/comment_2_39664f833dedc1a4fe083eec9bc4a7cd._comment
@@ -0,0 +1,75 @@
+[[!comment format=mdwn
+ username="https://www.google.com/accounts/o8/id?id=AItOawmiqeXJtP04fzHOjXs17kHO33v7dWR2xwA"
+ nickname="Jaco"
+ subject="Init Script"
+ date="2013-11-23T08:28:32Z"
+ content="""
+Hi Joey,
+
+Could you help out with writing an init.d script to safely start and stop the webapp on a headless server? 
+I made an attempt below based on examples from the internet, but have no idea if it will work.
+
+	#!/bin/bash
+	# git-annex
+	# chkconfig: 345 20 80
+	# description: Git Annex WebApp startup and shutdown script.
+	# processname: git-annex
+	
+	
+	DAEMON=git-annex webapp
+	
+	NAME=git-annex
+	DESC=\"Git Annex WebApp init script\"
+	PIDFILE=/var/run/$NAME.pid
+	SCRIPTNAME=/etc/init.d/$NAME
+	
+	case \"$1\" in
+	start)
+		printf \"%-50s\" \"Starting $NAME...\"
+		for dir in $(cat $HOME/.config/git-annex/autostart); do
+			cd $dir
+			PID=`$DAEMON > /dev/null 2>&1 & echo $!`
+			#echo \"Saving PID\" $PID \" to \" $PIDFILE
+		        if [ -z $PID ]; then
+	        		printf \"%s\n\" \"Fail\"
+		        else
+	        		echo $PID > $PIDFILE
+				printf \"%s\n\" \"Ok\"
+			fi
+		done
+	;;
+	status)
+	        printf \"%-50s\" \"Checking $NAME...\"
+	        if [ -f $PIDFILE ]; then
+			for PID in $(cat $PIDFILE); do
+				if [ -z \"`ps axf | grep ${PID} | grep -v grep`\" ]; then
+					printf \"%s\n\" \"Process dead but pidfile exists\"
+				else
+					echo \"Running\"
+				fi
+			done
+	        else
+	            printf \"%s\n\" \"Service not running\"
+	        fi
+	;;
+	stop)
+	        printf \"%-50s\" \"Stopping $NAME\"
+		if [ -f $PIDFILE ]; then
+			for PID in $(cat $PIDFILE); do
+				kill -HUP $PID
+				printf \"%s\n\" \"Ok\"
+			done
+			rm -f $PIDFILE
+	        else
+	            printf \"%s\n\" \"pidfile not found\"
+	        fi
+	;;
+	restart)
+	  	$0 stop
+	  	$0 start
+	;;
+	*)
+	        echo \"Usage: $0 {status|start|stop|restart}\"
+	        exit 1
+	esac
+"""]]
diff --git a/doc/forum/weird_view_file_naming.mdwn b/doc/forum/weird_view_file_naming.mdwn
new file mode 100644
index 0000000000..1520c6ad3d
--- /dev/null
+++ b/doc/forum/weird_view_file_naming.mdwn
@@ -0,0 +1,13 @@
+Very quick one, if anyone has to share experience with views (i would find them useful to get partial checkout views, as long as I can find a special setup to preserve the tree/filenames hyerarchy): 
+
+by using (filtering) views I am getting file names changed to include the (orignal) dir name,
+for instance:
+
+2008/pippo.doc
+
+after a git annex view 2008/=*, comes out as pippo_%2008%.doc
+is this avoidable ?
+
+(i've been trying to follow metadata walkthrough etc. but i always get the dirname in the filename between percents...)
+
+btw: version is 5.20141105
diff --git a/doc/forum/weird_view_file_naming/comment_1_24a44f94194231bb8efe9b20ee311d4c._comment b/doc/forum/weird_view_file_naming/comment_1_24a44f94194231bb8efe9b20ee311d4c._comment
new file mode 100644
index 0000000000..5ce5ed6ea1
--- /dev/null
+++ b/doc/forum/weird_view_file_naming/comment_1_24a44f94194231bb8efe9b20ee311d4c._comment
@@ -0,0 +1,12 @@
+[[!comment format=mdwn
+ username="https://www.google.com/accounts/o8/id?id=AItOawkJafmCf-sg9_OM0pynFYM3AO4WCgJiaMI"
+ nickname="Michele"
+ subject="what I was trying to do is probabily done better with sparse checkouts"
+ date="2014-11-26T11:46:04Z"
+ content="""
+therefore instead of tagging the files _and_ placing them into subdirectories (which somehow seems confounding views reperesentation), i'll be using sparse checkout for the subdirectories I need.
+
+i'll be looking in sparse checkouts so that i have separate trees checked out from same annex ( see http://git-annex.branchable.com/forum/sparse_git_checkouts_with_annex/ )
+
+cheers
+"""]]
diff --git a/doc/forum/weird_view_file_naming/comment_2_2ff3002e45a7443f0502243b3690df9d._comment b/doc/forum/weird_view_file_naming/comment_2_2ff3002e45a7443f0502243b3690df9d._comment
new file mode 100644
index 0000000000..a78231438d
--- /dev/null
+++ b/doc/forum/weird_view_file_naming/comment_2_2ff3002e45a7443f0502243b3690df9d._comment
@@ -0,0 +1,11 @@
+[[!comment format=mdwn
+ username="joey"
+ subject="""comment 2"""
+ date="2014-12-01T22:00:54Z"
+ content="""
+This naming is necessary to deal with the problem that a view could
+contain two files with the name name. So the directory is included to 
+work around it.
+
+Menomic: `%` looks sort of like a `/`
+"""]]
diff --git a/doc/forum/what_are___95__exactly__95___limitations_of_git-annex_when_using_it_like_dropbox.mdwn b/doc/forum/what_are___95__exactly__95___limitations_of_git-annex_when_using_it_like_dropbox.mdwn
new file mode 100644
index 0000000000..276ba646e9
--- /dev/null
+++ b/doc/forum/what_are___95__exactly__95___limitations_of_git-annex_when_using_it_like_dropbox.mdwn
@@ -0,0 +1,6 @@
+I'm trying to migrate from dropbox to git-annex. But I'm learning that git-annex has "limitations", as in
+"but it can be used to sync files such a way, with certain limitations". So what exactly are these limitations? I've learned already the hard way that newline can't be a character in a filename, and apparently git directories can't be synced?
+
+These are two  big issues imo (especiall the second one) - right now I don't have any filenames with newlines, and no git directories, but the brilliant thing about dropbox is that I don't have to think about such things, everything gets synced.
+
+But in any case, if I'm to use git-annex as a dropbox replacement, I really need to know what precisely are its limitations, so that it doesn't become an issue in two months out of nowhere...
diff --git a/doc/forum/what_are___95__exactly__95___limitations_of_git-annex_when_using_it_like_dropbox/comment_1_b76e6d3c4d530fd6735460aa5ab03ad5._comment b/doc/forum/what_are___95__exactly__95___limitations_of_git-annex_when_using_it_like_dropbox/comment_1_b76e6d3c4d530fd6735460aa5ab03ad5._comment
new file mode 100644
index 0000000000..66e618d3e9
--- /dev/null
+++ b/doc/forum/what_are___95__exactly__95___limitations_of_git-annex_when_using_it_like_dropbox/comment_1_b76e6d3c4d530fd6735460aa5ab03ad5._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="Agenta"
+ subject="not easy to access history"
+ date="2015-10-02T13:16:30Z"
+ content="""
+Another limitation is that it is not easy to access files history and to recover previous versions of a file. I have \"solved\" that by taking a snapshot of my repo every night with bup:
+https://github.com/bup/bup
+"""]]
diff --git a/doc/forum/what_happens_to_deleted_files__63__.mdwn b/doc/forum/what_happens_to_deleted_files__63__.mdwn
new file mode 100644
index 0000000000..5890a06b3d
--- /dev/null
+++ b/doc/forum/what_happens_to_deleted_files__63__.mdwn
@@ -0,0 +1,24 @@
+I have two repositories: A(direct) and B(indirect). At some point, I performed the following steps:
+
+1. Add a file in A.
+2. Sync A.
+3. Sync B, but not get all files.
+4. In B, remove a file F that was newly added in A, using `git rm` and `git commit`.
+5. Now get all files from A.
+
+This results in all "current" files being copied from A to B, but not file F.
+
+If I checkout a branch in B for an older commit which still had file F, I can see the symlink created in the branch. But now if I try to get the file in B from A, annex says that the remote is not available: 
+
+    sameerds@gajanan:/mnt/mandos/test$ git annex get
+    get IMG_0570.JPG 
+      Unable to access these remotes: A
+    
+      Try making some of these repositories available:
+      	530e4f81-474d-4b0b-9f13-4beb7361302c -- A
+    failed
+    git-annex: get: 1 failed
+
+The remote for A exists in the listing for `git remote`, and `git annex sync` does not make this error message go away. If I checkout `master` again, then everything works fine.
+
+But of course, this a different problem from my original one. It would a way to bring all files from the remote, that are ever used by any commit reachable in the local git history, irrespective of the currently checkout out commit. Perhaps an option `--deep-copy` to the standard `git annex get`.
diff --git a/doc/forum/what_happens_to_deleted_files__63__/comment_1_a476174646ad3adfbbe0cafdd7d42d08._comment b/doc/forum/what_happens_to_deleted_files__63__/comment_1_a476174646ad3adfbbe0cafdd7d42d08._comment
new file mode 100644
index 0000000000..5959db5355
--- /dev/null
+++ b/doc/forum/what_happens_to_deleted_files__63__/comment_1_a476174646ad3adfbbe0cafdd7d42d08._comment
@@ -0,0 +1,10 @@
+[[!comment format=mdwn
+ username="http://sameer.sbuddhe.net/blog/"
+ nickname="sameerds"
+ subject="never mind ..."
+ date="2013-12-31T04:06:43Z"
+ content="""
+... I discovered the [[`--all` option|http://git-annex.branchable.com/forum/Moving_older_version__39__s_file_content_without_doing_checkout/]] which does just what I was looking for!
+
+
+"""]]
diff --git a/doc/forum/what_rules_is_git_annex_sync_--content_using_to_send_files_out.mdwn b/doc/forum/what_rules_is_git_annex_sync_--content_using_to_send_files_out.mdwn
new file mode 100644
index 0000000000..2f12784a99
--- /dev/null
+++ b/doc/forum/what_rules_is_git_annex_sync_--content_using_to_send_files_out.mdwn
@@ -0,0 +1,31 @@
+I have two regular git clones and 4 special remotes (cloud
+storage mostly).
+
+My `numcopies` is set to 2 which means I always need to have at
+least two copies.  One usually lives in one of my git clones
+which is a backup and so has everything.  The other is somewhere
+in the cloud.
+
+When I add a new file to my client clone and do `git annex sync
+--content` is this respecting the prefered content of each remote
+or is it only trying to satisfy the numcopy?
+
+These are my settings for archive and smallarchive
+
+    groupwanted smallarchive = ((include=*/archive/* or include=archive/*) and not (copies=archive:2 or copies=smallarchive:2 or (copies=archive:1 and copies=smallarchive:1))) or approxlackingcopies=1
+
+
+
+    groupwanted archive = (not (copies=archive:2 or copies=smallarchive:2 or (copies=archive:1 and copies=smallarchive:1)) or approxlackingcopies=1
+
+
+One of the remotes is set as archive and two as smallarchive yet `sync
+--content` only ever copies to one so as to satisfy `numcopies`.  Is
+this expected?  Shouldn't it always try to make two copies in archive
+or smallarchive? 
+
+Interestingly, it copies data to my `backup` also (which is an
+extarnal drive, so always present).  So it seems the prefered content
+is only respected for clones and not special remotes.  Is that true?
+
+Do I have to run `git annex copy --to  --auto` to satisfy the content preferences?
diff --git a/doc/forum/what_rules_is_git_annex_sync_--content_using_to_send_files_out/comment_1_86428fc8286266d01cc51af8ae4aa335._comment b/doc/forum/what_rules_is_git_annex_sync_--content_using_to_send_files_out/comment_1_86428fc8286266d01cc51af8ae4aa335._comment
new file mode 100644
index 0000000000..ce0a539924
--- /dev/null
+++ b/doc/forum/what_rules_is_git_annex_sync_--content_using_to_send_files_out/comment_1_86428fc8286266d01cc51af8ae4aa335._comment
@@ -0,0 +1,34 @@
+[[!comment format=mdwn
+ username="andrew"
+ avatar="http://cdn.libravatar.org/avatar/acc0ece1eedf07dd9631e7d7d343c435"
+ subject="numcopies"
+ date="2018-03-27T22:25:30Z"
+ content="""
+||| *When I add a new file to my client clone and do git annex sync --content is this respecting the … content of each remote or is it only trying to satisfy the numcopy?*
+
+Both.
+
+Running `git annex sync --content` will copy content to any remote where that content is wanted (IE look at the preferred content settings), and will drop files that are not wanted and don't violate numcopies totals (at the moment of the drop?).
+
+
+||| *One of the remotes is set as archive and two as smallarchive yet sync --content only ever copies to one so as to satisfy numcopies.*
+
+I don't believe that `git annex sync --content` is ever trying to “satisfy numcopies,” I would think of numcopies as more of a limit or restriction on when git-annex is allowed to drop content that is not wanted by a remote.
+
+
+
+
+||| *Shouldn't it always try to make two copies in archive or smallarchive?*
+
+Hmmmm. I would guess there is some issue with your archive or smallarchive expressions, or they aren't actually set (being used) or you have discovered an issue… You have overridden the standard groups, and can see your overrides with `git annex groupwanted archive` and `git annex groupwanted smallarchive`? And your remotes have `git annex group archive` and `git annex wanted groupwanted` set?
+
+So `(include=*/archive/* or include=archive/*) and` means you are only copying files in the archive directory, was that your intention?
+
+If so, the rest of your content expression seems like it should want 1 copy in an archive and 1 copy in a smallarchive, or 2 copies in 2 remotes marked archive or 2 copies in two remotes marked smallarchive.
+
+
+||| *Interestingly, it copies data to my backup*
+
+Right. If you are using a standard content group backup means “All content is wanted. Even content of old/deleted files.” This expression will want all content and never drop content.
+
+"""]]
diff --git a/doc/forum/what_rules_is_git_annex_sync_--content_using_to_send_files_out/comment_2_50c1a19a7278e8748c2e6e789f6b418a._comment b/doc/forum/what_rules_is_git_annex_sync_--content_using_to_send_files_out/comment_2_50c1a19a7278e8748c2e6e789f6b418a._comment
new file mode 100644
index 0000000000..efbf232b89
--- /dev/null
+++ b/doc/forum/what_rules_is_git_annex_sync_--content_using_to_send_files_out/comment_2_50c1a19a7278e8748c2e6e789f6b418a._comment
@@ -0,0 +1,16 @@
+[[!comment format=mdwn
+ username="MatusGoljer"
+ avatar="http://cdn.libravatar.org/avatar/8152eed1d594c570563ed46e7fd8356f"
+ subject="comment 2"
+ date="2018-03-28T23:16:51Z"
+ content="""
+Hi Andrew!
+
+Actually, you are entirely correct, I just misinterpreted the `smallarchive` formula.  What I was thinking it would do was it would upload the file in case there are not two copies in any combination of `archive` or `smallarchive` and as soon as there are two `archive` copies it would ideally drop the file if not in `*/archive/*` directory.
+
+But now I see that the formula does exactly what you say it would do, the whole long `and not` condition is additional to the first, which means a file will not get uploaded if not in those directories.
+
+I will play around with the expression, I'm sure it can be modified to do what I want.
+
+Thanks!  It was mostly me being silly, sometimes having other people re-state the obvious helps!
+"""]]
diff --git a/doc/forum/what_rules_is_git_annex_sync_--content_using_to_send_files_out/comment_3_368403625451415c66b9e62005d9ddea._comment b/doc/forum/what_rules_is_git_annex_sync_--content_using_to_send_files_out/comment_3_368403625451415c66b9e62005d9ddea._comment
new file mode 100644
index 0000000000..43822f3ca7
--- /dev/null
+++ b/doc/forum/what_rules_is_git_annex_sync_--content_using_to_send_files_out/comment_3_368403625451415c66b9e62005d9ddea._comment
@@ -0,0 +1,14 @@
+[[!comment format=mdwn
+ username="joey"
+ subject="""comment 3"""
+ date="2018-04-04T16:14:45Z"
+ content="""
+I just wanted to say thanks to andrew for a really excellent response here.
+I've answered plenty of such questions less well.
+I hope you answer many more questions about git-annex with such care and
+attention to detail.
+
+And, thanks to MatusGoljer for following up to confirm that you were
+misunderstanding -- and I agree that you should be able to adjust the
+preferred content expression to do what you want it to do.
+"""]]
diff --git a/doc/forum/what_rules_is_git_annex_sync_--content_using_to_send_files_out/comment_4_53ccb5c7c8f15f44fb2f113742fe77f3._comment b/doc/forum/what_rules_is_git_annex_sync_--content_using_to_send_files_out/comment_4_53ccb5c7c8f15f44fb2f113742fe77f3._comment
new file mode 100644
index 0000000000..de464c3ac5
--- /dev/null
+++ b/doc/forum/what_rules_is_git_annex_sync_--content_using_to_send_files_out/comment_4_53ccb5c7c8f15f44fb2f113742fe77f3._comment
@@ -0,0 +1,10 @@
+[[!comment format=mdwn
+ username="andrew"
+ avatar="http://cdn.libravatar.org/avatar/acc0ece1eedf07dd9631e7d7d343c435"
+ subject="comment 4"
+ date="2018-05-04T16:00:58Z"
+ content="""
+Thanks Joey! I am glad to hopefully take some work off your plate.
+
+—Andrew
+"""]]
diff --git a/doc/forum/when_repo_offline_list_whereis_slow.mdwn b/doc/forum/when_repo_offline_list_whereis_slow.mdwn
new file mode 100644
index 0000000000..0a8dfe66bc
--- /dev/null
+++ b/doc/forum/when_repo_offline_list_whereis_slow.mdwn
@@ -0,0 +1,2 @@
+All is well except for when a repo is offline, everything becomes slow, git annex list, whereis...
+Tried to google find in site but can't find it.
diff --git a/doc/forum/when_repo_offline_list_whereis_slow/comment_1_79052583012da5097ca7bdc52dcdfb7d._comment b/doc/forum/when_repo_offline_list_whereis_slow/comment_1_79052583012da5097ca7bdc52dcdfb7d._comment
new file mode 100644
index 0000000000..1d9a167940
--- /dev/null
+++ b/doc/forum/when_repo_offline_list_whereis_slow/comment_1_79052583012da5097ca7bdc52dcdfb7d._comment
@@ -0,0 +1,9 @@
+[[!comment format=mdwn
+ username="yonkers"
+ avatar="http://cdn.libravatar.org/avatar/c57f0d4698ad9cff11094798b530942c"
+ subject="I think I found the answer, will try laer"
+ date="2018-09-13T18:18:30Z"
+ content="""
+It is about making the repo an archive!?
+Will test later...
+"""]]
diff --git a/doc/forum/whereis__58___Why_not_here__63__.mdwn b/doc/forum/whereis__58___Why_not_here__63__.mdwn
new file mode 100644
index 0000000000..fb2a325d5e
--- /dev/null
+++ b/doc/forum/whereis__58___Why_not_here__63__.mdwn
@@ -0,0 +1,33 @@
+Hey,
+
+I have three repos, all are run by git-annex assistent. Files are synced just fine, but there is one oddity:
+
+```
+florian@horus ~/.synced_configuration (git)-[annex/direct/master] % git annex whereis .zshenv 
+whereis .zshenv (3 copies) 
+        69902190-d4c1-4abb-a000-f0712769f1cd -- Horus [here]
+        c7fc42ca-49f7-4688-b81d-c9cfd4a42564 -- Asaru
+        d5b0586f-32dd-4aae-a7a2-b23d9edd0c32 -- Marduk [marduk]
+ok
+florian@horus ~/.synced_configuration (git)-[annex/direct/master] % git annex whereis .emacs 
+whereis .emacs (2 copies) 
+        c7fc42ca-49f7-4688-b81d-c9cfd4a42564 -- Asaru
+        d5b0586f-32dd-4aae-a7a2-b23d9edd0c32 -- Marduk [marduk]
+ok
+```
+for `.zshenv` everything is fine, but why is `.emacs` not in Horus? Changes from and to Horus are propagated:
+
+```
+florian@horus ~/.synced_configuration (git)-[annex/direct/master] % git annex log .emacs
+- Wed, 12 Apr 2017 22:48:03 CEST .emacs | 69902190-d4c1-4abb-a000-f0712769f1cd -- Horus
++ Wed, 12 Apr 2017 22:02:55 CEST .emacs | 69902190-d4c1-4abb-a000-f0712769f1cd -- Horus
++ Wed, 12 Apr 2017 22:02:55 CEST .emacs | d5b0586f-32dd-4aae-a7a2-b23d9edd0c32 -- Marduk [marduk]
+- Wed, 12 Apr 2017 22:02:54 CEST .emacs | 69902190-d4c1-4abb-a000-f0712769f1cd -- Horus
+- Wed, 12 Apr 2017 22:02:53 CEST .emacs | d5b0586f-32dd-4aae-a7a2-b23d9edd0c32 -- Marduk [marduk]
++ Wed, 12 Apr 2017 22:00:40 CEST .emacs | d5b0586f-32dd-4aae-a7a2-b23d9edd0c32 -- Marduk [marduk]
++ Wed, 12 Apr 2017 22:00:36 CEST .emacs | 69902190-d4c1-4abb-a000-f0712769f1cd -- Horus
+```
+
+Anyone any idea what could be wrong there?
+
+Thanks!
diff --git a/doc/forum/whereis__58___Why_not_here__63__/comment_1_02a423918efcc760bb89dfb78586dc2a._comment b/doc/forum/whereis__58___Why_not_here__63__/comment_1_02a423918efcc760bb89dfb78586dc2a._comment
new file mode 100644
index 0000000000..4f032f4ce5
--- /dev/null
+++ b/doc/forum/whereis__58___Why_not_here__63__/comment_1_02a423918efcc760bb89dfb78586dc2a._comment
@@ -0,0 +1,12 @@
+[[!comment format=mdwn
+ username="Horus"
+ avatar="http://cdn.libravatar.org/avatar/8f0ee08b98ea5bba76c3fe112c08851c"
+ subject="comment 1"
+ date="2017-04-12T21:21:08Z"
+ content="""
+It's strange. I made a change to the file on Horus, it gets propagated to Marduk, but Marduk does not know about. whereis still shows the file is only at Horus. At Asaru the symlink is deleted (so the change info was propagated), but instead there is a symlink on invalid content (like in indirect mode)
+
+The connections are all over SSH: Horus -> Marduk <- Asaru
+
+Sorry, I'm utterly confused...
+"""]]
diff --git a/doc/forum/whereis__58___Why_not_here__63__/comment_2_e55ce8cbb6bbabc654ba743d755a6eee._comment b/doc/forum/whereis__58___Why_not_here__63__/comment_2_e55ce8cbb6bbabc654ba743d755a6eee._comment
new file mode 100644
index 0000000000..cffe96e28b
--- /dev/null
+++ b/doc/forum/whereis__58___Why_not_here__63__/comment_2_e55ce8cbb6bbabc654ba743d755a6eee._comment
@@ -0,0 +1,10 @@
+[[!comment format=mdwn
+ username="joey"
+ subject="""comment 2"""
+ date="2017-05-11T17:22:01Z"
+ content="""
+Your log shows that the .emacs file reached Horus, but was then dropped
+from it some 46 minutes later.
+
+Something had to drop it. Are you using the git-annex assistant or what?
+"""]]
diff --git a/doc/forum/whereis__58___Why_not_here__63__/comment_3_7de87ad1fd8a0f53562d3083af1410d6._comment b/doc/forum/whereis__58___Why_not_here__63__/comment_3_7de87ad1fd8a0f53562d3083af1410d6._comment
new file mode 100644
index 0000000000..9b53cd06df
--- /dev/null
+++ b/doc/forum/whereis__58___Why_not_here__63__/comment_3_7de87ad1fd8a0f53562d3083af1410d6._comment
@@ -0,0 +1,33 @@
+[[!comment format=mdwn
+ username="Horus"
+ avatar="http://cdn.libravatar.org/avatar/8f0ee08b98ea5bba76c3fe112c08851c"
+ subject="comment 3"
+ date="2017-05-29T18:10:55Z"
+ content="""
+Sorry, just saw your comment...
+
+Yes, I'm using the assistant, as I pointed out in my original posting. Just right now, I'm having the same situation:
+
+```
+florian@horus ~/.synced_configuration (git)-[annex/direct/master] % git annex whereis .emacs
+whereis .emacs (2 copies) 
+        c7fc42ca-49f7-4688-b81d-c9cfd4a42564 -- Asaru
+        d5b0586f-32dd-4aae-a7a2-b23d9edd0c32 -- Marduk [marduk]
+ok
+```
+
+```
+florian@horus ~/.synced_configuration (git)-[annex/direct/master] % git annex log .emacs    
++ Mon, 29 May 2017 20:04:11 CEST .emacs | c7fc42ca-49f7-4688-b81d-c9cfd4a42564 -- Asaru
+- Mon, 29 May 2017 20:04:10 CEST .emacs | 69902190-d4c1-4abb-a000-f0712769f1cd -- Horus
++ Mon, 29 May 2017 20:02:37 CEST .emacs | 69902190-d4c1-4abb-a000-f0712769f1cd -- Horus
++ Mon, 29 May 2017 20:02:37 CEST .emacs | d5b0586f-32dd-4aae-a7a2-b23d9edd0c32 -- Marduk [marduk]
+- Mon, 29 May 2017 20:02:36 CEST .emacs | 69902190-d4c1-4abb-a000-f0712769f1cd -- Horus
+- Mon, 29 May 2017 20:02:35 CEST .emacs | d5b0586f-32dd-4aae-a7a2-b23d9edd0c32 -- Marduk [marduk]
++ Mon, 29 May 2017 20:02:35 CEST .emacs | d5b0586f-32dd-4aae-a7a2-b23d9edd0c32 -- Marduk [marduk]
++ Mon, 29 May 2017 20:02:33 CEST .emacs | 69902190-d4c1-4abb-a000-f0712769f1cd -- Horus
+```
+
+Thanks!
+Florian
+"""]]
diff --git a/doc/forum/whereis_command_with_file_names_instead_of_hashes.mdwn b/doc/forum/whereis_command_with_file_names_instead_of_hashes.mdwn
new file mode 100644
index 0000000000..6b4b3bf016
--- /dev/null
+++ b/doc/forum/whereis_command_with_file_names_instead_of_hashes.mdwn
@@ -0,0 +1,7 @@
+Hi
+
+I have a laptop, server and a regular PC. Laptop and PC are using same "backup" repo on server. Thay all get notified via ssh. My goal is to have synchronization and backup in one remote repo. When I wan to list all files on all repos i do:
+
+    git-annex whereis -A (some files do fail. Don't know why)
+
+But that displays hashes. Not file names. How can to list file names on all repos instead of hashes ?
diff --git a/doc/forum/whereis_command_with_file_names_instead_of_hashes/comment_1_4eaca07152916adc18032fb404e4dd92._comment b/doc/forum/whereis_command_with_file_names_instead_of_hashes/comment_1_4eaca07152916adc18032fb404e4dd92._comment
new file mode 100644
index 0000000000..3ad0becfe7
--- /dev/null
+++ b/doc/forum/whereis_command_with_file_names_instead_of_hashes/comment_1_4eaca07152916adc18032fb404e4dd92._comment
@@ -0,0 +1,10 @@
+[[!comment format=mdwn
+ username="http://ypid.wordpress.com/"
+ ip="213.153.84.215"
+ subject="-A  --all      operate on all versions of all files"
+ date="2014-07-07T07:42:03Z"
+ content="""
+Hi
+
+`-A` operates on all the files even on files which are not accessible from your current working directory. If you omit the `-A` then filenames will be printed.
+"""]]
diff --git a/doc/forum/whereis_command_with_file_names_instead_of_hashes/comment_2_94b43ac23ff8332b35723422eede8997._comment b/doc/forum/whereis_command_with_file_names_instead_of_hashes/comment_2_94b43ac23ff8332b35723422eede8997._comment
new file mode 100644
index 0000000000..2ab6b8e0bf
--- /dev/null
+++ b/doc/forum/whereis_command_with_file_names_instead_of_hashes/comment_2_94b43ac23ff8332b35723422eede8997._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="https://www.google.com/accounts/o8/id?id=AItOawkNE-H4vEcbcGndxq5daT8qUb7yIf7r1OE"
+ nickname="Łukasz"
+ subject="comment 2"
+ date="2014-07-07T08:40:57Z"
+ content="""
+Yes they will be omitted. I just want to list files that are on my laptop (for example) and on remote repo.
+"""]]
diff --git a/doc/forum/whereis_command_with_file_names_instead_of_hashes/comment_3_45ffa6dd17667ecd6685f85f34046eff._comment b/doc/forum/whereis_command_with_file_names_instead_of_hashes/comment_3_45ffa6dd17667ecd6685f85f34046eff._comment
new file mode 100644
index 0000000000..b0d48ef990
--- /dev/null
+++ b/doc/forum/whereis_command_with_file_names_instead_of_hashes/comment_3_45ffa6dd17667ecd6685f85f34046eff._comment
@@ -0,0 +1,10 @@
+[[!comment format=mdwn
+ username="http://ypid.wordpress.com/"
+ ip="213.153.84.215"
+ subject="git annex whereis --in origin"
+ date="2014-07-07T09:45:48Z"
+ content="""
+So you only want to see whereis for files which are present on certain annexes? You can use `git annex whereis --in e64553d0-779a-4c79-939a-d5b7f3e56a57` or `git annex whereis --in origin` (modify the repo names).
+
+Have a look on [QUERY COMMANDS](/git-annex/).
+"""]]
diff --git a/doc/forum/whereis_command_with_file_names_instead_of_hashes/comment_4_d459fbcf0db59b821ae67f4949e48103._comment b/doc/forum/whereis_command_with_file_names_instead_of_hashes/comment_4_d459fbcf0db59b821ae67f4949e48103._comment
new file mode 100644
index 0000000000..24643f063d
--- /dev/null
+++ b/doc/forum/whereis_command_with_file_names_instead_of_hashes/comment_4_d459fbcf0db59b821ae67f4949e48103._comment
@@ -0,0 +1,14 @@
+[[!comment format=mdwn
+ username="https://www.google.com/accounts/o8/id?id=AItOawkNE-H4vEcbcGndxq5daT8qUb7yIf7r1OE"
+ nickname="Łukasz"
+ subject="comment 4"
+ date="2014-07-07T10:05:06Z"
+ content="""
+git-annex whereis --not --in here
+
+Does not display anything.
+
+git-annex whereis --not --in here -A
+
+Displays files on remote only. Every file has an additional message (0 copies) failed. This displays hash names. How to display file names ? I am 100% sure that my remote repo has some files that are not present on laptop or PC.
+"""]]
diff --git a/doc/forum/whereis_command_with_file_names_instead_of_hashes/comment_5_e52a8c9cb418fbc2e2cba71f37bd44ad._comment b/doc/forum/whereis_command_with_file_names_instead_of_hashes/comment_5_e52a8c9cb418fbc2e2cba71f37bd44ad._comment
new file mode 100644
index 0000000000..eff34edb6a
--- /dev/null
+++ b/doc/forum/whereis_command_with_file_names_instead_of_hashes/comment_5_e52a8c9cb418fbc2e2cba71f37bd44ad._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="https://www.google.com/accounts/o8/id?id=AItOawkNE-H4vEcbcGndxq5daT8qUb7yIf7r1OE"
+ nickname="Łukasz"
+ subject="comment 5"
+ date="2014-07-07T10:06:03Z"
+ content="""
+One more thing. I am using indirect mode with assistant. If that matters in some way.
+"""]]
diff --git a/doc/forum/why_doesn__39__t___96__git_annex_fix__96___fix_a_link__63__.mdwn b/doc/forum/why_doesn__39__t___96__git_annex_fix__96___fix_a_link__63__.mdwn
new file mode 100644
index 0000000000..305629ada7
--- /dev/null
+++ b/doc/forum/why_doesn__39__t___96__git_annex_fix__96___fix_a_link__63__.mdwn
@@ -0,0 +1,32 @@
+We have the correct symlink:
+
+    $ ls -lh Sita_Sings_the_Blues_480p_2150kbps.mp4 
+    lrwxrwxrwx 1 user user 204 Apr  8 20:22 Sita_Sings_the_Blues_480p_2150kbps.mp4 -> .git/annex/objects/6q/Wz/SHA256E-s1463013630--2a18a317a536d8e2d28e7916a45a007679b7192102a71067de488c9faa6aab45.mp4/SHA256E-s1463013630--2a18a317a536d8e2d28e7916a45a007679b7192102a71067de488c9faa6aab45.mp4
+
+The object is in place:
+
+    $ ls -lh .git/annex/objects/6q/Wz/SHA256E-s1463013630--2a18a317a536d8e2d28e7916a45a007679b7192102a71067de488c9faa6aab45.mp4/SHA256E-s1463013630--2a18a317a536d8e2d28e7916a45a007679b7192102a71067de488c9faa6aab45.mp4
+    -r-------- 1 user user 1,4G Dec 28  2010 .git/annex/objects/6q/Wz/SHA256E-s1463013630--2a18a317a536d8e2d28e7916a45a007679b7192102a71067de488c9faa6aab45.mp4/SHA256E-s1463013630--2a18a317a536d8e2d28e7916a45a007679b7192102a71067de488c9faa6aab45.mp4
+
+Now I move the symlink, which becomes broken:
+
+    $ mv Sita_Sings_the_Blues_480p_2150kbps.mp4 Videos/
+
+I try to fix the symlink:
+
+    $ git annex sync
+    $ git annex fix Videos/Sita_Sings_the_Blues_480p_2150kbps.mp4 
+
+But it stays broken:
+
+    $ ls -lh Videos/Sita_Sings_the_Blues_480p_2150kbps.mp4 
+    lrwxrwxrwx 1 user user 204 Apr  8 20:22 Videos/Sita_Sings_the_Blues_480p_2150kbps.mp4 -> .git/annex/objects/6q/Wz/SHA256E-s1463013630--2a18a317a536d8e2d28e7916a45a007679b7192102a71067de488c9faa6aab45.mp4/SHA256E-s1463013630--2a18a317a536d8e2d28e7916a45a007679b7192102a71067de488c9faa6aab45.mp4
+
+Ah, when I try to `git mv Sita_Sings_the_Blues_480p_2150kbps.mp4 Videos/`, 
+I am warned that the link is not under version control. How did the symlink get 
+made in the first place, if not by git-annex? It should be known, right?
+
+After I `git annex add`, I can `git mv` and then `git annex fix`.
+
+How did I get into the state where I have a symlink pointing at an imported
+object, which is not under version control?
diff --git a/doc/forum/why_doesn__39__t___96__git_annex_fix__96___fix_a_link__63__/comment_1_865245182d765fed7bbeb42a35dd605e._comment b/doc/forum/why_doesn__39__t___96__git_annex_fix__96___fix_a_link__63__/comment_1_865245182d765fed7bbeb42a35dd605e._comment
new file mode 100644
index 0000000000..df24ad53e9
--- /dev/null
+++ b/doc/forum/why_doesn__39__t___96__git_annex_fix__96___fix_a_link__63__/comment_1_865245182d765fed7bbeb42a35dd605e._comment
@@ -0,0 +1,9 @@
+[[!comment format=mdwn
+ username="CandyAngel"
+ subject="comment 1"
+ date="2015-04-08T14:29:59Z"
+ content="""
+'git annex fix' will only fix symlinks that git is tracking (either staged, or part of the repository) because git-annex uses git-ls-files (or something like that) to get the filenames to fix.
+
+'git annex add' adds the files to the annex and replaces them with symlinks, stages the symlinks but does not commit the symlinks.
+"""]]
diff --git a/doc/forum/why_doesn__39__t___96__git_annex_fix__96___fix_a_link__63__/comment_2_c1f5a285b7cfb92d218b9b0d31caeed6._comment b/doc/forum/why_doesn__39__t___96__git_annex_fix__96___fix_a_link__63__/comment_2_c1f5a285b7cfb92d218b9b0d31caeed6._comment
new file mode 100644
index 0000000000..f46aeac5b4
--- /dev/null
+++ b/doc/forum/why_doesn__39__t___96__git_annex_fix__96___fix_a_link__63__/comment_2_c1f5a285b7cfb92d218b9b0d31caeed6._comment
@@ -0,0 +1,7 @@
+[[!comment format=mdwn
+ username="CandyAngel"
+ subject="comment 2"
+ date="2015-04-08T14:32:56Z"
+ content="""
+It's also funny that this has cropped up [just after I asked for git-annex to fix untracked symlinks](https://git-annex.branchable.com/todo/__34__git-annex_fix__34___on_untracked__44___but_git-annexy_symlinks/) :P
+"""]]
diff --git a/doc/forum/why_doesn__39__t___96__git_annex_fix__96___fix_a_link__63__/comment_3_be2816c47091842b829cd626e18c5fda._comment b/doc/forum/why_doesn__39__t___96__git_annex_fix__96___fix_a_link__63__/comment_3_be2816c47091842b829cd626e18c5fda._comment
new file mode 100644
index 0000000000..731f4d2253
--- /dev/null
+++ b/doc/forum/why_doesn__39__t___96__git_annex_fix__96___fix_a_link__63__/comment_3_be2816c47091842b829cd626e18c5fda._comment
@@ -0,0 +1,9 @@
+[[!comment format=mdwn
+ username="https://www.google.com/accounts/o8/id?id=AItOawnVnsqEy82M-MuS2gLri-az83wSQ6lXSrc"
+ nickname="Jean"
+ subject="comment 3"
+ date="2015-04-09T05:37:03Z"
+ content="""
+Hi CandyAngel, thanks for the pointer, I 
+[commented](http://git-annex.branchable.com/todo/__34__git-annex_fix__34___on_untracked__44___but_git-annexy_symlinks/#comment-5251d819e83c66dbdd9d7bcee4a87f9f) there.
+"""]]
diff --git a/doc/forum/windows_port__63__.mdwn b/doc/forum/windows_port__63__.mdwn
new file mode 100644
index 0000000000..092b8f8e1c
--- /dev/null
+++ b/doc/forum/windows_port__63__.mdwn
@@ -0,0 +1,2 @@
+Any progress on Windows port? That would be very nice to have!
+Depending on the scale of it, I might be able to help.
diff --git a/doc/forum/windows_port__63__/comment_1_23fa9aa3b00940a1c1b3876c35eef019._comment b/doc/forum/windows_port__63__/comment_1_23fa9aa3b00940a1c1b3876c35eef019._comment
new file mode 100644
index 0000000000..95323ff997
--- /dev/null
+++ b/doc/forum/windows_port__63__/comment_1_23fa9aa3b00940a1c1b3876c35eef019._comment
@@ -0,0 +1,9 @@
+[[!comment format=mdwn
+ username="http://joey.kitenet.net/"
+ nickname="joey"
+ subject="comment 1"
+ date="2012-03-12T06:43:02Z"
+ content="""
+[[todo/windows_support]] has everything I know about making a windows port. This badly needs someone who understand Windows to dive into it. The question of how to create a symbolic link (or the relevant Windows equivilant) from haskell on Windows
+is a good starting point..
+"""]]
diff --git a/doc/forum/wishlist__58___get__47__drop_via_webapp_file_explorer.mdwn b/doc/forum/wishlist__58___get__47__drop_via_webapp_file_explorer.mdwn
new file mode 100644
index 0000000000..3be01480ea
--- /dev/null
+++ b/doc/forum/wishlist__58___get__47__drop_via_webapp_file_explorer.mdwn
@@ -0,0 +1 @@
+Know what'd be a sweet feature?  A file explorer in the webapp that lets you get and drop files (for a Manual local repository).  Especially when the webapp becomes available on Android.  I'd love to be able to select what is and isn't present on a small device by some means other than moving files around.
diff --git a/doc/forum/wishlist__58___get__47__drop_via_webapp_file_explorer/comment_1_c818a6d44dc13a56460b1865f70eb97c._comment b/doc/forum/wishlist__58___get__47__drop_via_webapp_file_explorer/comment_1_c818a6d44dc13a56460b1865f70eb97c._comment
new file mode 100644
index 0000000000..6cc0cf38bc
--- /dev/null
+++ b/doc/forum/wishlist__58___get__47__drop_via_webapp_file_explorer/comment_1_c818a6d44dc13a56460b1865f70eb97c._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="http://joeyh.name/"
+ nickname="joey"
+ subject="comment 1"
+ date="2013-04-11T14:55:43Z"
+ content="""
+There's a page called [[design/assistant/partial_content]] that discusses this idea and related things.
+"""]]
diff --git a/doc/forum/wishlist__58___make_copy_stop_on_exhausted_disk_space.mdwn b/doc/forum/wishlist__58___make_copy_stop_on_exhausted_disk_space.mdwn
new file mode 100644
index 0000000000..8896fee37a
--- /dev/null
+++ b/doc/forum/wishlist__58___make_copy_stop_on_exhausted_disk_space.mdwn
@@ -0,0 +1,4 @@
+I'm trying to distribute a large annex to a number of smaller archive drives.
+
+While copying to a directory special remote, the current behaviour is to continue trying copying files into a remote, even as diskspace there has been exhausted.
+It would make sense for git-annex copy to actually stop instead.
diff --git a/doc/forum/wishlist__58___make_copy_stop_on_exhausted_disk_space/comment_1_467e5e3db3e836030bc4b4f15846951f._comment b/doc/forum/wishlist__58___make_copy_stop_on_exhausted_disk_space/comment_1_467e5e3db3e836030bc4b4f15846951f._comment
new file mode 100644
index 0000000000..5c5f17ca9d
--- /dev/null
+++ b/doc/forum/wishlist__58___make_copy_stop_on_exhausted_disk_space/comment_1_467e5e3db3e836030bc4b4f15846951f._comment
@@ -0,0 +1,12 @@
+[[!comment format=mdwn
+ username="http://joeyh.name/"
+ ip="4.154.0.140"
+ subject="comment 1"
+ date="2013-07-18T19:49:43Z"
+ content="""
+Hmm, it might make sense to stop if there is no more space at all. However, just because the first file doesn't fit doesn't mean several following, smaller files can't be copied into the remaining space.
+
+Since it checks the free space before actually copying anything, it's not like there's much overhead in running through the list of files it was asked to copy and trying to copy them all, either.
+
+So, I'm unconvinced on this one.
+"""]]
diff --git a/doc/forum/wishlist__58___make_copy_stop_on_exhausted_disk_space/comment_2_e3ca3db9bea11d3e085ee9c3c56b33fe._comment b/doc/forum/wishlist__58___make_copy_stop_on_exhausted_disk_space/comment_2_e3ca3db9bea11d3e085ee9c3c56b33fe._comment
new file mode 100644
index 0000000000..ee535549f3
--- /dev/null
+++ b/doc/forum/wishlist__58___make_copy_stop_on_exhausted_disk_space/comment_2_e3ca3db9bea11d3e085ee9c3c56b33fe._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="https://www.google.com/accounts/o8/id?id=AItOawln3ckqKx0x_xDZMYwa9Q1bn4I06oWjkog"
+ nickname="Michael"
+ subject="comment 2"
+ date="2013-07-18T20:22:05Z"
+ content="""
+Actually with 4.20130709, I see the drive (ntfs-3g under Linux, gpg-encrypted special remote) being filled to 0 remaining space, even with reservespace of 300mb.  Guess it's another bug anyway.
+"""]]
diff --git a/doc/forum/wishlist__58___make_copy_stop_on_exhausted_disk_space/comment_3_0ef8c37350fc192d9b784fbab1d9f318._comment b/doc/forum/wishlist__58___make_copy_stop_on_exhausted_disk_space/comment_3_0ef8c37350fc192d9b784fbab1d9f318._comment
new file mode 100644
index 0000000000..a33cbcd27c
--- /dev/null
+++ b/doc/forum/wishlist__58___make_copy_stop_on_exhausted_disk_space/comment_3_0ef8c37350fc192d9b784fbab1d9f318._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="http://joeyh.name/"
+ ip="4.154.0.140"
+ subject="comment 3"
+ date="2013-07-20T20:33:37Z"
+ content="""
+Indeed, that was a bug with encrypted directory special remotes. Fixed it.
+"""]]
diff --git a/doc/forum/working_without_git-annex_commits.mdwn b/doc/forum/working_without_git-annex_commits.mdwn
new file mode 100644
index 0000000000..00a61f3f0f
--- /dev/null
+++ b/doc/forum/working_without_git-annex_commits.mdwn
@@ -0,0 +1,20 @@
+Is it possible to use git-annex without having [[location tracking]] commits in the style of "got a video I want to rewatch on the plane" or "freed up space" in the main tree?
+
+I consider these changes to be volatile, and irrelevant to the archive history. While they are unproblematic when it comes to merging, they make the commit tree rather complicated, especially with multiple users (as opposed to a single user managing his files on an external disk, a server and his laptop). Some users might even want to contribute to a shared repository without reporting on what they checked out.
+
+As a minimal solution, I configured a repository to ``.gitignore`` ``.git-annex/*:*.log``, but even when using modes that do not require that information (``git annex copy --from`` instead of ``git annex get``), that failes when git-annex tried to git-add ignored files.
+
+A more elaborate solution might be to keep location tracking information in a branch on its own (as suggested in [[todo/branching]]), keeping the main tree clean of such commits. A stealth user could then configure that branch to never be pushed. (Alternatively, if git-annex respects .gitignore and doesn't try to check in changes on ignored files, he could locally ``.gitignore`` ``.git-annex/*:*.log``.)
+
+> A stealth user can simply avoid pushing, and so keep their repository 
+> in a forked state, that can still pull changes from origin.
+> 
+> Beyond that, [[todo/branching]] is the best solution.
+> 
+> I don't think that gitignoring the log files is a good plan, because
+> if the files are left modified and uncommitted, git will not be able to 
+> merge other changes it pulls. The automerging of log files only works
+> if any local changes to them have been committed.
+> 
+> It would be possible to add a knob that
+> simply blocks all local modifications to the log files. --[[Joey]]
diff --git a/doc/forum/workspace.xml_file_disappeared__44___broken_symlink_showed_up.mdwn b/doc/forum/workspace.xml_file_disappeared__44___broken_symlink_showed_up.mdwn
new file mode 100644
index 0000000000..ee3185ddcc
--- /dev/null
+++ b/doc/forum/workspace.xml_file_disappeared__44___broken_symlink_showed_up.mdwn
@@ -0,0 +1,27 @@
+I am running git-annex on Mac OS 10.9 and syncing to a Linux device that was attached to a Synology NAS.  Both were running December versions of git-annex.
+
+While working on an application in IntelliJ my workspace.xml file disappeared.  It went from being a real file to being a symlink to a location that didn't exist.
+
+The symlink looks like this: workspace.xml -> ../../../.git/annex/objects/M2/8W/SHA256E-s68106--90e9cc4f617c9034db1bf462d058b82c59ade0be58de1d3a3e2f8c02606631c2.xml/SHA256E-s68106--90e9cc4f617c9034db1bf462d058b82c59ade0be58de1d3a3e2f8c02606631c2.xml
+
+I tried git-annex fsck and I get this message: ** No known copies exist of XXXXXXX/workspace.xml
+
+"XXXXXXX" is a path I needed to obscure I can see it in the git-annex repo but it is a reference to a symlink that doesn't exist
+
+It was mentioned in another thread ( http://git-annex.branchable.com/forum/git_annex_on_osx_only_creating_symlinks__63____63__/ ) that this could happen if I did a forced drop but I do not use git-annex on the command line.  I'm using the assistant and having it manage all of my files for me.
+
+I tried looking at the log for it and it shows this:
+
+
    +timmattison$ /Applications/git-annex.app/Contents/MacOS/git-annex log XXXXXXX/.idea/workspace.xml 
+- 2014-01-03 19:49:58 XXXXXXX/.idea/workspace.xml | d6747880-b355-4d41-b4e5-d1ad3afcb4a5 -- timmattison@MacBook-Pro.local:~/Desktop/annex
++ 2014-01-03 17:08:32 XXXXXXX/.idea/workspace.xml | d6747880-b355-4d41-b4e5-d1ad3afcb4a5 -- timmattison@MacBook-Pro.local:~/Desktop/annex
+(Recording state in git...)
+
    
+
+So I can see that the git-annex thinks the file was deleted but what really happened is that IntelliJ just modified it.  workspace.xml files are modified very often when working in IntelliJ.
+
+Is this a known bug that has been fixed in the new version (2013-12-30 or later)?
+
+Thanks,
+Tim
diff --git a/doc/forum/workspace.xml_file_disappeared__44___broken_symlink_showed_up/comment_1_00b084f9786de6516f46065c0cb00e79._comment b/doc/forum/workspace.xml_file_disappeared__44___broken_symlink_showed_up/comment_1_00b084f9786de6516f46065c0cb00e79._comment
new file mode 100644
index 0000000000..32e4972ca1
--- /dev/null
+++ b/doc/forum/workspace.xml_file_disappeared__44___broken_symlink_showed_up/comment_1_00b084f9786de6516f46065c0cb00e79._comment
@@ -0,0 +1,11 @@
+[[!comment format=mdwn
+ username="http://joeyh.name/"
+ ip="209.250.56.35"
+ subject="comment 1"
+ date="2014-01-06T18:38:51Z"
+ content="""
+If the file was modified, git-annex should notice and add the new version. So the mere act of modifying the file does not explain how you ended up with a symlink to the old version of the file. Although it does explain why git-annex log would show that the old version of the file had been deleted.
+
+What does `git log` say when run on the workspace.xml file?
+
+"""]]
diff --git a/doc/forum/workspace.xml_file_disappeared__44___broken_symlink_showed_up/comment_2_138499b36d28c5e267b4aad8792dc87e._comment b/doc/forum/workspace.xml_file_disappeared__44___broken_symlink_showed_up/comment_2_138499b36d28c5e267b4aad8792dc87e._comment
new file mode 100644
index 0000000000..1407a8ad36
--- /dev/null
+++ b/doc/forum/workspace.xml_file_disappeared__44___broken_symlink_showed_up/comment_2_138499b36d28c5e267b4aad8792dc87e._comment
@@ -0,0 +1,48 @@
+[[!comment format=mdwn
+ username="https://www.google.com/accounts/o8/id?id=AItOawkipQLNyt8RHREHpg2k5wdYeRSCCvSNSBg"
+ nickname="Tim"
+ subject="git log for workspace.xml"
+ date="2014-01-06T19:17:13Z"
+ content="""
+git log on the file gives a ton of output, is this what you need?
+
+commit 93157022d8ee57097db29fcc669e99e218b54657
+Author: Tim Mattison 
+Date:   Fri Jan 3 17:08:03 2014 -0500
+
+commit 04dab5520f33140a21882515b4036f71729a5f89
+Author: Tim Mattison 
+Date:   Fri Jan 3 17:06:54 2014 -0500
+
+commit b75be90bf91c9f9314e567d5a28e4497070de9af
+Author: Tim Mattison 
+Date:   Fri Jan 3 16:58:24 2014 -0500
+
+commit 4045b2d7a71b9ab692c20eeab73bf196a280940d
+Author: Tim Mattison 
+Date:   Fri Jan 3 16:58:05 2014 -0500
+
+commit 25de8a1fcd5cce53b8077443dccba1591255431f
+Author: Tim Mattison 
+Date:   Fri Jan 3 16:57:32 2014 -0500
+
+commit dd118ae88cdccb5a7d68a7ae14033304865cfc4e
+Author: Tim Mattison 
+Date:   Fri Jan 3 16:53:37 2014 -0500
+
+commit eaa5f85b7feed8ba17e3874763b4d7aa6e05b5c1
+Author: Tim Mattison 
+Date:   Fri Jan 3 16:53:23 2014 -0500
+
+commit 4d310d7d16234a1a6bcf0f14ebae232c57d904a7
+Author: Tim Mattison 
+Date:   Fri Jan 3 16:52:42 2014 -0500
+
+commit 2487ae7f99c7b81c1648fdc88d0b39a1cf6e4d03
+Author: Tim Mattison 
+Date:   Fri Jan 3 16:52:21 2014 -0500
+
+commit 9e0040311c43e6e1f04ff3b8fdd3ac5570c8f2d1
+Author: Tim Mattison 
+Date:   Fri Jan 3 16:51:54 2014 -0500
+"""]]
diff --git a/doc/forum/workspace.xml_file_disappeared__44___broken_symlink_showed_up/comment_3_6c59c494b563e56d061417eb2216bb19._comment b/doc/forum/workspace.xml_file_disappeared__44___broken_symlink_showed_up/comment_3_6c59c494b563e56d061417eb2216bb19._comment
new file mode 100644
index 0000000000..f31e7cf4da
--- /dev/null
+++ b/doc/forum/workspace.xml_file_disappeared__44___broken_symlink_showed_up/comment_3_6c59c494b563e56d061417eb2216bb19._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="https://www.google.com/accounts/o8/id?id=AItOawkipQLNyt8RHREHpg2k5wdYeRSCCvSNSBg"
+ nickname="Tim"
+ subject="Formatting!"
+ date="2014-01-06T19:22:36Z"
+ content="""
+I swear, some day I'll get the formatting correct.  Sorry!
+"""]]
diff --git a/doc/forum/workspace.xml_file_disappeared__44___broken_symlink_showed_up/comment_4_ccbba61cdd6fce3e5de82417bcc0cbfb._comment b/doc/forum/workspace.xml_file_disappeared__44___broken_symlink_showed_up/comment_4_ccbba61cdd6fce3e5de82417bcc0cbfb._comment
new file mode 100644
index 0000000000..3ca8a6295f
--- /dev/null
+++ b/doc/forum/workspace.xml_file_disappeared__44___broken_symlink_showed_up/comment_4_ccbba61cdd6fce3e5de82417bcc0cbfb._comment
@@ -0,0 +1,12 @@
+[[!comment format=mdwn
+ username="https://www.google.com/accounts/o8/id?id=AItOawkipQLNyt8RHREHpg2k5wdYeRSCCvSNSBg"
+ nickname="Tim"
+ subject="Just happened again"
+ date="2014-01-16T16:23:00Z"
+ content="""
+This happened again and now I have this:
+
+lrwxr-xr-x  1 timmattison  staff  197 Jan 16 11:18 distribute-client-applications.sh -> ../../../.git/annex/objects/JV/91/SHA256E-s921--51fc6a6e2924ff6e2673971b5786fe0f204a5984a700a2241cc77383f04f544e.sh/SHA256E-s921--51fc6a6e2924ff6e2673971b5786fe0f204a5984a700a2241cc77383f04f544e.sh
+
+This was on the machine that had the file, not a remote machine it was syncing to.  I was editing the script, I ran it, the bash script segfaulted (expected), and then git-annex essentially deleted the file.  Can you help me understand why this happens?  It's obvious when it is a file I'm working on but I'm afraid it could be happening to other files and I don't know about it.
+"""]]
diff --git a/doc/forum/workspace.xml_file_disappeared__44___broken_symlink_showed_up/comment_5_50526283b35997cece2f087507cdd4ee._comment b/doc/forum/workspace.xml_file_disappeared__44___broken_symlink_showed_up/comment_5_50526283b35997cece2f087507cdd4ee._comment
new file mode 100644
index 0000000000..ebf175cf22
--- /dev/null
+++ b/doc/forum/workspace.xml_file_disappeared__44___broken_symlink_showed_up/comment_5_50526283b35997cece2f087507cdd4ee._comment
@@ -0,0 +1,12 @@
+[[!comment format=mdwn
+ username="http://joeyh.name/"
+ ip="209.250.56.146"
+ subject="comment 5"
+ date="2014-03-06T18:22:31Z"
+ content="""
+Do you edit this file in multiple places? This could be an occurance of this bug: [[bugs/direct_mode_merge_can_overwrite_local,_non-annexed_files]] which is fixed in the latest release.
+
+If the program that writes the workspace.xml file did so by first deleting it, and then writing the new version, this could result in the assistant committing the deletion, which makes the new version a local, non-annexed file, and then if a pull is received that modified the file, I think the above bug could happen. 
+
+You could tell if this was the case by looking at the git log of the directory containing the file, and see if it has been repeatedly deleted and added back to the repository. The git log snippet you pasted unfortunately does not let me tell this information.
+"""]]
diff --git a/doc/forum/would_you_call_me_crazy_if_I_use_git_annex_in_direct_mode_to_track_my___47__home__63____63__.mdwn b/doc/forum/would_you_call_me_crazy_if_I_use_git_annex_in_direct_mode_to_track_my___47__home__63____63__.mdwn
new file mode 100644
index 0000000000..2332f1bd52
--- /dev/null
+++ b/doc/forum/would_you_call_me_crazy_if_I_use_git_annex_in_direct_mode_to_track_my___47__home__63____63__.mdwn
@@ -0,0 +1,7 @@
+Hello people :)
+
+I'm learning to use git annex and I'm doing some tests to know how it works from an user point of view... My big goal would be to use git annex to track my entire /home directory in direct mode, so I would be able to sync my main computer with my laptop in a really effective way. But of course I feel some fear because things could go wrong and of course I have lots of important files I can't lose —nothing really important since I'll backup that /home in an external hard drive, but it would make me cry if I have to spend some hours restoring my /home :S
+
+So, do you think that tracking all the /home would be a reasonable goal?
+
+Thanks a lot for your time :)
diff --git a/doc/forum/would_you_call_me_crazy_if_I_use_git_annex_in_direct_mode_to_track_my___47__home__63____63__/comment_1_b011442de2f67f3ad340031a0767e990._comment b/doc/forum/would_you_call_me_crazy_if_I_use_git_annex_in_direct_mode_to_track_my___47__home__63____63__/comment_1_b011442de2f67f3ad340031a0767e990._comment
new file mode 100644
index 0000000000..94d07524bc
--- /dev/null
+++ b/doc/forum/would_you_call_me_crazy_if_I_use_git_annex_in_direct_mode_to_track_my___47__home__63____63__/comment_1_b011442de2f67f3ad340031a0767e990._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="https://me.yahoo.com/a/FHnTlSBo1eCGJRwueeKeB6.RCaPbGMPr5jxx8A--#ce0d8"
+ nickname="Hamza"
+ subject="comment 1"
+ date="2013-12-09T06:07:00Z"
+ content="""
+I do track my home dir using annex, but instead of one giant annex I have multiple annex repos. It is much faster than one giant annex and when used in combination with mr you can threat all repos as one. 
+"""]]
diff --git a/doc/forum/would_you_call_me_crazy_if_I_use_git_annex_in_direct_mode_to_track_my___47__home__63____63__/comment_2_c69865c08c3eb49d64310fc76e80c65d._comment b/doc/forum/would_you_call_me_crazy_if_I_use_git_annex_in_direct_mode_to_track_my___47__home__63____63__/comment_2_c69865c08c3eb49d64310fc76e80c65d._comment
new file mode 100644
index 0000000000..e74d169459
--- /dev/null
+++ b/doc/forum/would_you_call_me_crazy_if_I_use_git_annex_in_direct_mode_to_track_my___47__home__63____63__/comment_2_c69865c08c3eb49d64310fc76e80c65d._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="https://www.google.com/accounts/o8/id?id=AItOawleVyKk2kQsB_HgEdS7w1s0BmgRGy1aay0"
+ nickname="Milan"
+ subject="comment 2"
+ date="2013-12-09T11:16:48Z"
+ content="""
+what is 'mr'?
+"""]]
diff --git a/doc/forum/would_you_call_me_crazy_if_I_use_git_annex_in_direct_mode_to_track_my___47__home__63____63__/comment_3_7651fb48fc71b2c7b4e7b6830a0f9865._comment b/doc/forum/would_you_call_me_crazy_if_I_use_git_annex_in_direct_mode_to_track_my___47__home__63____63__/comment_3_7651fb48fc71b2c7b4e7b6830a0f9865._comment
new file mode 100644
index 0000000000..669046584e
--- /dev/null
+++ b/doc/forum/would_you_call_me_crazy_if_I_use_git_annex_in_direct_mode_to_track_my___47__home__63____63__/comment_3_7651fb48fc71b2c7b4e7b6830a0f9865._comment
@@ -0,0 +1,10 @@
+[[!comment format=mdwn
+ username="https://me.yahoo.com/a/FHnTlSBo1eCGJRwueeKeB6.RCaPbGMPr5jxx8A--#ce0d8"
+ nickname="Hamza"
+ subject="comment 3"
+ date="2013-12-09T14:56:13Z"
+ content="""
+It [1] allows you to run commands on a set of repositories.
+
+[1] http://myrepos.branchable.com/
+"""]]
diff --git a/doc/forum/would_you_call_me_crazy_if_I_use_git_annex_in_direct_mode_to_track_my___47__home__63____63__/comment_4_7d88f1aa163185c801b7697846086c7f._comment b/doc/forum/would_you_call_me_crazy_if_I_use_git_annex_in_direct_mode_to_track_my___47__home__63____63__/comment_4_7d88f1aa163185c801b7697846086c7f._comment
new file mode 100644
index 0000000000..3e4c8aeb2e
--- /dev/null
+++ b/doc/forum/would_you_call_me_crazy_if_I_use_git_annex_in_direct_mode_to_track_my___47__home__63____63__/comment_4_7d88f1aa163185c801b7697846086c7f._comment
@@ -0,0 +1,12 @@
+[[!comment format=mdwn
+ username="https://www.google.com/accounts/o8/id?id=AItOawlqimC5gbxsILaBlnuVXYtfSMDIhiyvHfw"
+ nickname="Valentín"
+ subject="comment 4"
+ date="2013-12-10T19:34:06Z"
+ content="""
+Hi Hamza :)
+
+I like your way of achieving the same goal —and existing ``mr``, which I hadn't known until now, it looks like the best alternative.
+
+Thanks a lot for your time :)
+"""]]
diff --git a/doc/future_proofing.mdwn b/doc/future_proofing.mdwn
new file mode 100644
index 0000000000..a4d9ac24f8
--- /dev/null
+++ b/doc/future_proofing.mdwn
@@ -0,0 +1,53 @@
+Imagine putting a git-annex drive in a time capsule. In 20, or 50, or 100
+years, you'd like its contents to be as accessible as possible to whoever
+digs it up.
+
+This is a hard problem. git-annex cannot completely solve it, but it does
+its best to not contribute to the problem. Here are some aspects of the
+problem:
+
+* How are files accessed? Git-annex carefully adds minimal complexity
+  to access files in a repository. Nothing needs to be done to extract
+  files from the repository; they are there on disk in the usual way,
+  with just some symlinks pointing at the annexed file contents.
+  Neither git-annex nor git is needed to get at the file contents.
+  
+  (Also, git-annex provides an "uninit" command that moves everything out
+  of the annex, if you should ever want to stop using it.)
+
+* What file formats are used? Will they still be readable? To deal with
+  this, it's best to stick to plain text files, and the most common
+  image, sound, etc formats. Consider storing the same content in multiple
+  formats.
+
+* What filesystem is used on the drive? Will that filesystem still be
+  available? Whatever you choose to use, git-annex can put files on it.
+  Even if you choose (ugh) FAT.
+
+* What is the hardware interface of the drive? Will hardware still exist
+  to talk to it?
+
+* What if some of the data is damaged? git-annex facilitates storing a
+  configurable number of [[copies]] of the file contents. The metadata
+  about your files is stored in git, and so every clone of the repository
+  means another copy of that is stored. Also, git-annex uses filenames
+  for the data that encode everything needed to match it back to the
+  metadata. So if a filesystem is badly corrupted and all your annexed
+  files end up in `lost+found`, they can easily be lifted back out into
+  another clone of the repository. Even if the filenames are lost,
+  it's possible to [[tips/recover_data_from_lost+found]].
+
+* What about upgrades to the git-annex repository format?
+  git-annex supports [[upgrades]] from all previous repository versions,
+  and will always support upgrading from all of them to any new versions.
+  Note that the upgrade process needs to modify the content of the
+  repository -- if modifying the original archived repository is not
+  desirable you can always make a copy of the repository and upgrade the
+  copy.
+
+* What about encrypted special remotes? A 
+  [[fairly simple shell script using standard tools|Decrypting_files_in_special_remotes_without_git-annex]]
+  (gpg and openssl) can decrypt files stored on such
+  a remote, as long as you have access to the encryption keys (which
+  are stored in the git-annex branch of the repository, sometimes
+  encrypted with your gpg key).
diff --git a/doc/future_proofing/comment_1_2614eb2e9b7b23fa9bb4251c0d025909._comment b/doc/future_proofing/comment_1_2614eb2e9b7b23fa9bb4251c0d025909._comment
new file mode 100644
index 0000000000..dbe429fa9c
--- /dev/null
+++ b/doc/future_proofing/comment_1_2614eb2e9b7b23fa9bb4251c0d025909._comment
@@ -0,0 +1,16 @@
+[[!comment format=mdwn
+ username="https://launchpad.net/~electrichead"
+ nickname="electrichead"
+ subject="Regarding accessing files in a time capsule..."
+ date="2014-08-25T15:51:00Z"
+ content="""
+Imagine a rather contrived doomsday scenario: the file paths and/or basenames are important and, for some reason, the symlinks are not present (perhaps they got deleted, or aren't supported). `git` and `git-annex` no longer exist and let's assume knowledge of `git` internals is not useful here. All the *content* is there, stored under hashed file names under `.git/annex/objects`.
+
+I may be missing something obvious but I think options for restoring file paths include:
+
+  - direct mode bypasses this issue; all the files are right there. 
+  - the WORM backend perhaps carries enough information in the object file names to work with.
+  - file content/metadata may be sufficient to easily recreate a sensible directory structure in some cases, so no worries.
+
+These first two options may represent compromises in various use-cases and the last may not be applicable or, if it is, practical. The object-path mapping could trivially be backed up in plain text in lieu of these. Like I said, I may be overlooking something here that makes this unnecessary or even a non-concern (actually, I've convinced myself it's not a serious concern in most of the use-cases I've considered, but crossing i's and dotting t's).
+"""]]
diff --git a/doc/git-annex-add.mdwn b/doc/git-annex-add.mdwn
new file mode 100644
index 0000000000..05de37c929
--- /dev/null
+++ b/doc/git-annex-add.mdwn
@@ -0,0 +1,109 @@
+# NAME
+
+git-annex add - adds files to the git annex
+
+# SYNOPSIS
+
+git annex add `[path ...]`
+
+# DESCRIPTION
+
+Adds the specified files to the annex. If a directory is specified,
+acts on all files inside the directory and its subdirectories.
+If no path is specified, adds files from the current directory and below.
+
+Files that are already checked into git and are unmodified, or that
+git has been configured to ignore will be silently skipped.
+
+If annex.largefiles is configured, and does not match a file, 
+`git annex add` will behave the same as `git add` and add the
+non-large file directly to the git repository, instead of to the annex.
+
+Large files are added to the annex in locked form, which prevents further
+modification of their content unless unlocked by [[git-annex-unlock]](1).
+(This is not the case however when a repository is in a filesystem not
+supporting symlinks, or is in direct mode.)
+To add a file to the annex in unlocked form, `git add` can be used instead 
+(that only works when the repository has annex.version 6 or higher).
+
+This command can also be used to add symbolic links, both symlinks to
+annexed content, and other symlinks.
+
+# OPTIONS
+
+* `--include-dotfiles`
+
+  Dotfiles are skipped unless explicitly listed, or unless this option is
+  used.
+
+* `--force`
+
+  Add gitignored files.
+
+* `--backend`
+
+  Specifies which key-value backend to use.
+
+* file matching options
+
+  Many of the [[git-annex-matching-options]](1)
+  can be used to specify files to add.
+
+  For example: `--largerthan=1GB`
+
+* `--jobs=N` `-JN`
+
+  Adds multiple files in parallel. This may be faster.
+  For example: `-J4`  
+
+* `--update` `-u`
+
+  Like `git add --update`, this does not add new files, but any updates
+  to tracked files will be added to the index.
+
+* `--json`
+
+  Enable JSON output. This is intended to be parsed by programs that use
+  git-annex. Each line of output is a JSON object.
+
+* `--json-error-messages`
+
+  Messages that would normally be output to standard error are included in
+  the json instead.
+
+* `--batch`
+
+  Enables batch mode, in which a file to add is read in a line from stdin,
+  the file is added, and repeat.
+
+  Note that if a file is skipped (due to not existing, being gitignored,
+  already being in git, or doesn't meet the matching options), 
+  an empty line will be output instead of the normal output produced
+  when adding a file.
+
+* `-z`
+
+  Makes the `--batch` input be delimited by nulls instead of the usual
+  newlines.
+
+# SEE ALSO
+
+[[git-annex]](1)
+
+[[git-annex-unlock]](1)
+
+[[git-annex-lock]](1)
+
+[[git-annex-undo]](1)
+
+[[git-annex-import]](1)
+
+[[git-annex-unannex]](1)
+
+[[git-annex-reinject]](1)
+
+# AUTHOR
+
+Joey Hess 
+
+Warning: Automatically converted into a man page by mdwn2man. Edit with care.
diff --git a/doc/git-annex-add/comment_1_3286fb304f161df9775366db27cf9530._comment b/doc/git-annex-add/comment_1_3286fb304f161df9775366db27cf9530._comment
new file mode 100644
index 0000000000..f752bbbf9f
--- /dev/null
+++ b/doc/git-annex-add/comment_1_3286fb304f161df9775366db27cf9530._comment
@@ -0,0 +1,12 @@
+[[!comment format=mdwn
+ username="rrnewton@63c9faa1997c908b1dc04dfdca33c809660cd158"
+ nickname="rrnewton"
+ avatar="http://cdn.libravatar.org/avatar/638acc3e55c2bb09aa0dcca5b5c8acb6"
+ subject="Flag to force same behavior as annex.largefiles attribute?"
+ date="2018-05-21T05:29:06Z"
+ content="""
+When in [direct mode](https://git-annex.branchable.com/direct_mode), the \"add the non-large file directly to the git repository\" behavior described above is very useful, because the option of typing simply `git add foo`, does not exist as it does in [indirect mode](https://git-annex.branchable.com/git-annex-indirect/).
+
+However, I can't see any combination of flags that trigger this behavior.  I suppose it can be accomplished by temporarily setting [annex.largefiles](https://git-annex.branchable.com/tips/largefiles/) to a huge value before executing `git annex add` (i.e. creating a `.gitattributes` and then deleting it).  I think I'll try that as a work-around, but it would be great to have a flag that accomplishes this.
+
+"""]]
diff --git a/doc/git-annex-add/comment_2_9bce0639d6c0767e39465db00f8120f1._comment b/doc/git-annex-add/comment_2_9bce0639d6c0767e39465db00f8120f1._comment
new file mode 100644
index 0000000000..8efa9a63a8
--- /dev/null
+++ b/doc/git-annex-add/comment_2_9bce0639d6c0767e39465db00f8120f1._comment
@@ -0,0 +1,12 @@
+[[!comment format=mdwn
+ username="joey"
+ subject="""comment 2"""
+ date="2018-05-21T16:36:51Z"
+ content="""
+@rrnewton I know people do commonly accomplish this
+by something like `git -c annex.largefiles='exclude(*)' annex add` 
+
+A shorter way to write that would only be useful for direct mode,
+so I'm inclined not to add it, but open a todo item if you want to discuss
+that.
+"""]]
diff --git a/doc/git-annex-add/comment_3_352e8782fc9e4d02c069a527e8c88f40._comment b/doc/git-annex-add/comment_3_352e8782fc9e4d02c069a527e8c88f40._comment
new file mode 100644
index 0000000000..0ff2ba568f
--- /dev/null
+++ b/doc/git-annex-add/comment_3_352e8782fc9e4d02c069a527e8c88f40._comment
@@ -0,0 +1,14 @@
+[[!comment format=mdwn
+ username="rrnewton@63c9faa1997c908b1dc04dfdca33c809660cd158"
+ nickname="rrnewton"
+ avatar="http://cdn.libravatar.org/avatar/638acc3e55c2bb09aa0dcca5b5c8acb6"
+ subject="Sounds great!"
+ date="2018-05-21T18:09:35Z"
+ content="""
+That's fabulous.  A Bash alias around that command is really all I need when working in direct mode.  (And the archive's too damn big to switch back and forth between direct/indirect.)
+
+I was just too much a newb with git attributes to know it could be done that way.  For discoverability, maybe that command could be placed in an \"examples\" section in the primary documentation above?
+
+
+
+"""]]
diff --git a/doc/git-annex-addunused.mdwn b/doc/git-annex-addunused.mdwn
new file mode 100644
index 0000000000..3dbddd6416
--- /dev/null
+++ b/doc/git-annex-addunused.mdwn
@@ -0,0 +1,28 @@
+# NAME
+
+git-annex addunused - add back unused files
+
+# SYNOPSIS
+
+git annex addunused `[number|range ...]`
+
+# DESCRIPTION
+
+Adds back files for the content corresponding to the numbers or ranges,
+as listed by the last `git annex unused`. 
+
+The files will have names starting with "unused."
+
+# SEE ALSO
+
+[[git-annex]](1)
+
+[[git-annex-add]](1)
+
+[[git-annex-unused]](1)
+
+# AUTHOR
+
+Joey Hess 
+
+Warning: Automatically converted into a man page by mdwn2man. Edit with care.
diff --git a/doc/git-annex-addurl.mdwn b/doc/git-annex-addurl.mdwn
new file mode 100644
index 0000000000..62e05d468b
--- /dev/null
+++ b/doc/git-annex-addurl.mdwn
@@ -0,0 +1,134 @@
+# NAME
+
+git-annex addurl - add urls to annex
+
+# SYNOPSIS
+
+git annex addurl `[url ...]`
+
+# DESCRIPTION
+
+Downloads each url to its own file, which is added to the annex.
+
+When `youtube-dl` is installed, it can be used to check for a video
+embedded in  a web page at the url, and that is added to the annex instead.
+(However, this is disabled by default as it can be a security risk. 
+See the documentation of annex.security.allowed-http-addresses
+in [[git-annex]](1) for details.)
+
+Urls to torrent files (including magnet links) will cause the content of
+the torrent to be downloaded, using `aria2c`.
+
+Normally the filename is based on the full url, so will look like
+"www.example.com_dir_subdir_bigfile". In some cases, addurl is able to
+come up with a better filename based on other information. Options can also
+be used to get better filenames.
+
+# OPTIONS
+
+* `--fast`
+
+  Avoid immediately downloading the url. The url is still checked
+  (via HEAD) to verify that it exists, and to get its size if possible.
+
+* `--relaxed`
+
+  Don't immediately download the url, and avoid storing the size of the
+  url's content. This makes git-annex accept whatever content is there
+  at a future point.
+
+  This is the fastest option, but it still has to access the network
+  to check if the url contains embedded media. When adding large numbers
+  of urls, using `--relaxed --raw` is much faster.
+  
+* `--raw`
+
+  Prevent special handling of urls by youtube-dl, bittorrent, and other
+  special remotes. This will for example, make addurl
+  download the .torrent file and not the contents it points to.
+
+* `--file=name`
+
+  Use with a filename that does not yet exist to add a new file
+  with the specified name and the content downloaded from the url.
+
+  If the file already exists, addurl will record that it can be downloaded
+  from the specified url(s).
+
+* `--pathdepth=N`
+
+  Rather than basing the filename on the whole url, this causes a path to
+  be constructed, starting at the specified depth within the path of the
+  url.
+
+  For example, adding the url http://www.example.com/dir/subdir/bigfile
+  with `--pathdepth=1` will use "dir/subdir/bigfile",
+  while `--pathdepth=3` will use "bigfile". 
+
+  It can also be negative; `--pathdepth=-2` will use the last
+  two parts of the url.
+
+* `--prefix=foo` `--suffix=bar`
+
+  Use to adjust the filenames that are created by addurl. For example,
+  `--suffix=.mp3` can be used to add an extension to the file.
+
+* `--jobs=N` `-JN`
+
+  Enables parallel downloads when multiple urls are being added.
+  For example: `-J4`  
+
+* `--batch`
+
+  Enables batch mode, in which lines containing urls to add are read from
+  stdin.
+
+* `-z`
+
+  Makes the `--batch` input be delimited by nulls instead of the usual
+  newlines. 
+
+* `--with-files`
+
+  When batch mode is enabled, makes it parse lines of the form: "$url $file"
+
+  That adds the specified url to the specified file, downloading its
+  content if the file does not yet exist; the same as
+  `git annex addurl $url --file $file`
+
+* `--json`
+
+  Enable JSON output. This is intended to be parsed by programs that use
+  git-annex. Each line of output is a JSON object.
+
+* `--json-progress`
+
+  Include progress objects in JSON output.
+
+* `--json-error-messages`
+
+  Messages that would normally be output to standard error are included in
+  the json instead.
+
+# CAVEATS
+
+If annex.largefiles is configured, and does not match a file, `git annex
+addurl` will add the non-large file directly to the git repository,
+instead of to the annex. However, this is not done when --fast or --relaxed
+is used.
+
+# SEE ALSO
+
+[[git-annex]](1)
+
+[[git-annex-rmurl]](1)
+
+[[git-annex-registerurl]](1)
+
+[[git-annex-importfeed]](1)
+
+# AUTHOR
+
+Joey Hess 
+
+Warning: Automatically converted into a man page by mdwn2man. Edit with care.
diff --git a/doc/git-annex-addurl/comment_1_ce9c660be160a22c28aeb6de8b3b5818._comment b/doc/git-annex-addurl/comment_1_ce9c660be160a22c28aeb6de8b3b5818._comment
new file mode 100644
index 0000000000..cff3bca56c
--- /dev/null
+++ b/doc/git-annex-addurl/comment_1_ce9c660be160a22c28aeb6de8b3b5818._comment
@@ -0,0 +1,16 @@
+[[!comment format=mdwn
+ username="joseph.rawson.works@85a210ab8c0e37a0b2d6bb235738b20e23e8878f"
+ nickname="joseph.rawson.works"
+ avatar="http://cdn.libravatar.org/avatar/6b473d5484b68803e8c47eeff9197397"
+ subject="Using youtube-dl commandline options with git-annex-addurl"
+ date="2018-05-30T15:29:16Z"
+ content="""
+I have been trying to figure out how to use addurl to get this video.
+I have this in my mscourtstuff annex as a large binary, but I would really like to 
+use the web as a remote for this.
+
+Hughes v Hosemann 2010-CA-01949-SCT-43112001.mp4
+youtube-dl --referer 'http://judicial.mc.edu/case.php?id=24206'  http://player.vimeo.com/video/43112001
+
+
+"""]]
diff --git a/doc/git-annex-addurl/comment_2_9bc984fb80d77309b62a4e915e65a31a._comment b/doc/git-annex-addurl/comment_2_9bc984fb80d77309b62a4e915e65a31a._comment
new file mode 100644
index 0000000000..a12fc56ca1
--- /dev/null
+++ b/doc/git-annex-addurl/comment_2_9bc984fb80d77309b62a4e915e65a31a._comment
@@ -0,0 +1,14 @@
+[[!comment format=mdwn
+ username="joey"
+ subject="""comment 2"""
+ date="2018-05-30T16:30:26Z"
+ content="""
+There's not currently a way to do per-file youtube-dl options.
+The difficulty is that we don't know what youtube-dl options might be
+unsafe, and which such a feature could make eg `git annex get` use when run
+by a different user.
+
+I feel that this needs some support in youtube-dl to avoid git-annex
+needing to know about all its safe options. Especially since which options
+are available, or safe, could vary between versions of youtube-dl.
+"""]]
diff --git a/doc/git-annex-adjust.mdwn b/doc/git-annex-adjust.mdwn
new file mode 100644
index 0000000000..6f32f30298
--- /dev/null
+++ b/doc/git-annex-adjust.mdwn
@@ -0,0 +1,57 @@
+# NAME
+
+git-annex adjust - enter an adjusted branch
+
+# SYNOPSIS
+
+git annex adjust `--unlock|--fix`
+
+# DESCRIPTION
+
+Enters an adjusted form of the current branch. The annexed files will
+be treated differently. For example with --unlock all annexed files will
+be unlocked.
+
+The adjusted branch will have a name like "adjusted/master(unlocked)".
+Since it's a regular git branch, you can use `git checkout` to switch
+back to the original branch at any time.
+
+While in the adjusted branch, you can use git-annex and git commands as
+usual. Any commits that you make will initially only be made to the
+adjusted branch. 
+
+To propagate changes from the adjusted branch back to the original branch,
+and to other repositories, as well as to merge in changes from other
+repositories, use `git annex sync`.
+
+This command can only be used in a v6 git-annex repository.
+
+# OPTIONS
+
+* `--unlock`
+
+  Unlock all annexed files in the adjusted branch. This allows
+  annexed files to be modified.
+
+* `--fix`
+
+  Fix the symlinks to annexed files to point to the local git annex
+  object directory. This can be useful if a repository is checked out in an
+  unusual way that prevents the symlinks committed to git from pointing at
+  the annex objects.
+
+# SEE ALSO
+
+[[git-annex]](1)
+
+[[git-annex-unlock]](1)
+
+[[git-annex-upgrade]](1)
+
+[[git-annex-sync]](1)
+
+# AUTHOR
+
+Joey Hess 
+
+Warning: Automatically converted into a man page by mdwn2man. Edit with care.
diff --git a/doc/git-annex-adjust/comment_1_364d1d5546c0f448a4b27cfe69a59a07._comment b/doc/git-annex-adjust/comment_1_364d1d5546c0f448a4b27cfe69a59a07._comment
new file mode 100644
index 0000000000..1e2d7a602c
--- /dev/null
+++ b/doc/git-annex-adjust/comment_1_364d1d5546c0f448a4b27cfe69a59a07._comment
@@ -0,0 +1,14 @@
+[[!comment format=mdwn
+ username="ynikitenko"
+ avatar="http://cdn.libravatar.org/avatar/168d629704097ddc596f75ca32a687a3"
+ subject="difference with git-annex-fix?"
+ date="2017-11-02T19:57:48Z"
+ content="""
+From git-annex-fix manual:
+
+       Fixes up symlinks that have become broken to again point to annexed content.
+       This is useful to run manually when you have been moving the symlinks around, but is done automatically when committing a change with git too.
+       Also, adjusts unlocked files to be copies or hard links as configured by annex.thin.
+
+So what is the difference between that and git-annex adjust --fix?
+"""]]
diff --git a/doc/git-annex-adjust/comment_2_383af8e1e26e4119671437aace0a58f5._comment b/doc/git-annex-adjust/comment_2_383af8e1e26e4119671437aace0a58f5._comment
new file mode 100644
index 0000000000..a0a2700954
--- /dev/null
+++ b/doc/git-annex-adjust/comment_2_383af8e1e26e4119671437aace0a58f5._comment
@@ -0,0 +1,20 @@
+[[!comment format=mdwn
+ username="joey"
+ subject="""comment 2"""
+ date="2017-11-07T20:18:19Z"
+ content="""
+`git annex fix` fixes up the symlinks before they're committed.
+It's run by the pre-commit hook, so even when annexed files are
+manually moved around, the symlinks that get committed are always
+right.
+
+So then, if the symlinks committed are always right,
+how would `git annex adjust --fix` be useful? Well,
+there are ways to check out git repositories that make
+the .git directory not be in the usual place. For example,
+when using submodules, git puts that directory in a different place.
+And then the committed symlinks won't point to .git. So,
+`git annex adjust --fix` is useful as a way to adjust the symlinks
+locally, without committing any changes to them, in that kind of
+situation.
+"""]]
diff --git a/doc/git-annex-assistant.mdwn b/doc/git-annex-assistant.mdwn
new file mode 100644
index 0000000000..98cdab4d04
--- /dev/null
+++ b/doc/git-annex-assistant.mdwn
@@ -0,0 +1,58 @@
+# NAME
+
+git-annex assistant - automatically sync changes
+
+# SYNOPSIS
+
+git annex assistant
+
+# DESCRIPTION
+
+Watches for changes to files in the current directory and its subdirectories,
+and automatically syncs them to other remotes.
+
+# OPTIONS
+
+* `--autostart`
+
+  Automatically starts the assistant running in each repository listed
+  in the file `~/.config/git-annex/autostart`
+
+  This is typically started at boot, or when you log in.
+
+* `--startdelay=N`
+
+  Wait N seconds before running the startup scan. This process can
+  be expensive and you may not want to run it immediately upon login.
+
+  When --autostart is used, defaults to --startdelay=5.
+
+* `--foreground`
+
+  Avoid forking to the background.
+
+* `--stop`
+
+  Stop a running daemon in the current repository.
+
+* `--autostop`
+
+  The complement to --autostart; stops all running daemons in the
+  repositories listed in the autostart file.
+
+# SEE ALSO
+
+[[git-annex]](1)
+
+[[git-annex-watch]](1)
+
+[[git-annex-schedule]](1)
+
+For more details about the git-annex assistant, see
+
+
+# AUTHOR
+
+Joey Hess 
+
+Warning: Automatically converted into a man page by mdwn2man. Edit with care.
diff --git a/doc/git-annex-calckey.mdwn b/doc/git-annex-calckey.mdwn
new file mode 100644
index 0000000000..f9e90b30c4
--- /dev/null
+++ b/doc/git-annex-calckey.mdwn
@@ -0,0 +1,45 @@
+# NAME
+
+git-annex calckey - calculates the key that would be used to refer to a file
+
+# SYNOPSIS
+
+git annex calckey `[file ...]`
+
+# DESCRIPTION
+
+This plumbing-level command calculates the key that would be used
+to refer to a file. The file is not added to the annex by this command.
+The key is output to stdout.
+
+The backend used is the one from the annex.backend configuration
+setting, which can be overridden by the --backend option.
+For example, to force use of the SHA1 backend:
+
+	git annex calckey --backend=SHA1 file
+
+# OPTIONS
+
+* `--backend=name`
+
+  Specifies which key-value backend to use.
+
+* `--batch`
+
+  Enable batch mode, in which a line containing the filename is read from
+  stdin, the key is output to stdout (with a trailing newline), and repeat.
+
+* `-z`
+
+  Makes the `--batch` input be delimited by nulls instead of the usual
+  newlines.
+
+# SEE ALSO
+
+[[git-annex]](1)
+
+# AUTHOR
+
+Joey Hess 
+
+Warning: Automatically converted into a man page by mdwn2man. Edit with care.
diff --git a/doc/git-annex-checkpresentkey.mdwn b/doc/git-annex-checkpresentkey.mdwn
new file mode 100644
index 0000000000..5e85e569f5
--- /dev/null
+++ b/doc/git-annex-checkpresentkey.mdwn
@@ -0,0 +1,38 @@
+# NAME
+
+git-annex checkpresentkey - check if key is present in remote
+
+# SYNOPSIS
+
+git annex checkpresentkey `key` `[remote]`
+
+# DESCRIPTION
+
+This plumbing-level command verifies if the specified key's content
+is present in the specified remote.
+
+When no remote is specified, it verifies if the key's content is present
+somewhere, checking accessible remotes until it finds the content.
+
+Exits 0 if the content is verified present, or 1 if it is verified to not
+be present. If there is a problem, the special exit code 100 is used,
+and an error message is output to stderr.
+
+# OPTIONS
+
+* `--batch`
+
+  Enables batch mode. In this mode, the `key` is not specified at the
+  command line, but the `remote` may still be. Lines containing keys are
+  read from stdin, and a line is output with "1" if the key is verified to
+  be present, and "0" otherwise.
+
+# SEE ALSO
+
+[[git-annex]](1)
+
+# AUTHOR
+
+Joey Hess 
+
+Warning: Automatically converted into a man page by mdwn2man. Edit with care.
diff --git a/doc/git-annex-checkpresentkey/comment_1_756f276929a0b0ff73baea141d3ef07d._comment b/doc/git-annex-checkpresentkey/comment_1_756f276929a0b0ff73baea141d3ef07d._comment
new file mode 100644
index 0000000000..3dc75ab300
--- /dev/null
+++ b/doc/git-annex-checkpresentkey/comment_1_756f276929a0b0ff73baea141d3ef07d._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="Ilya_Shlyakhter"
+ avatar="http://cdn.libravatar.org/avatar/1647044369aa7747829c38b9dcc84df0"
+ subject="comment 1"
+ date="2018-09-19T17:40:16Z"
+ content="""
+If a key has a URL by which it can be downloaded from an external special remote, and the remote supports checkurl but not checkpresent, checkpresentkey says the key isn't in the remote; but git annex whereis --key says there is.  Maybe it's as intended, checkpresentkey being a plumbing command, but just wanted to note.
+"""]]
diff --git a/doc/git-annex-checkpresentkey/comment_2_e277962791b64e67b08ccff47dd22447._comment b/doc/git-annex-checkpresentkey/comment_2_e277962791b64e67b08ccff47dd22447._comment
new file mode 100644
index 0000000000..38bb1d9e5e
--- /dev/null
+++ b/doc/git-annex-checkpresentkey/comment_2_e277962791b64e67b08ccff47dd22447._comment
@@ -0,0 +1,14 @@
+[[!comment format=mdwn
+ username="joey"
+ subject="""comment 2"""
+ date="2018-09-24T15:28:59Z"
+ content="""
+It makes sense that checkpresentkey could report different information than
+whereis, because whereis only reports on logged location information,
+while checkpresentkey actively checks the remote.
+
+I'm not entirely clear on how an external special remote could not support
+checkpresent, I suppose you might mean it always fails or is just not
+implemented (so basically fails). That would be a problem with the remote's
+implementation. If it's something else, open a bug report.
+"""]]
diff --git a/doc/git-annex-config.mdwn b/doc/git-annex-config.mdwn
new file mode 100644
index 0000000000..ed602ec958
--- /dev/null
+++ b/doc/git-annex-config.mdwn
@@ -0,0 +1,91 @@
+# NAME
+
+git-annex config - configuration stored in git-annex branch
+
+# SYNOPSIS
+
+git annex config --set name value
+
+git annex config --get name
+
+git annex config --unset name
+
+# DESCRIPTION
+
+Set or get configuration settings stored in the git-annex branch.
+
+Unlike `git config` settings, these settings can be seen
+in all clones of the repository, once they have gotten their
+git-annex branches in sync.
+
+# SUPPORTED SETTINGS
+
+git-annex does not check the git-annex branch for all settings.
+Only a few make sense to be able to set such that all clones of a
+repository see the setting, and so git-annex only looks for these:
+
+These settings can be overridden on a per-repository basis using
+`git config`.
+
+* `annex.autocommit`
+
+  Set to false to prevent the git-annex assistant and git-annex sync
+  from automatically committing changes to files in the repository.
+
+* `annex.resolvemerge`
+
+  Set to false to prevent merge conflicts in the checked out branch
+  being automatically resolved by the git-annex assitant,
+  git-annex sync, git-annex merge, and the git-annex post-receive hook.
+
+* `annex.synccontent`
+
+  Set to true to make git-annex sync default to syncing content.
+
+* `annex.securehashesonly`
+
+  Set to true to indicate that the repository should only use
+  cryptographically secure hashes 
+  (SHA2, SHA3) and not insecure hashes (MD5, SHA1) for content.
+
+  When this is set, the contents of files using cryptographically
+  insecure hashes will not be allowed to be added to the repository.
+
+  Also, git-annex fsck` will complain about any files present in
+  the repository that use insecure hashes.
+
+  Note that this is only read from the git-annex branch by
+  `git annex init`, and is copied to the corresponding git config setting. 
+  So, changes to the value in the git-annex branch won't affect a
+  repository once it has been initialized.
+
+# EXAMPLE
+
+Suppose you want to prevent git annex sync from committing changes
+to files, so a manual git commit workflow is used in all clones of the
+repository. Then run:
+
+	git annex config --set annex.autocommit false
+
+If you want to override that in a partiticular clone, just use git config
+in the clone:
+
+	git config annex.autocommit true
+
+And to get back to the default behavior:
+
+	git annex config --unset annex.autocommit
+
+# SEE ALSO
+
+[[git-annex]](1)
+
+git-config(1)
+
+[[git-annex-vicfg]](1)
+
+# AUTHOR
+
+Joey Hess 
+
+Warning: Automatically converted into a man page by mdwn2man. Edit with care.
diff --git a/doc/git-annex-config/comment_1_26bd743439e87e71b1eea2c620e6fc12._comment b/doc/git-annex-config/comment_1_26bd743439e87e71b1eea2c620e6fc12._comment
new file mode 100644
index 0000000000..b344fdbc09
--- /dev/null
+++ b/doc/git-annex-config/comment_1_26bd743439e87e71b1eea2c620e6fc12._comment
@@ -0,0 +1,10 @@
+[[!comment format=mdwn
+ username="davi.castro@13c06889d1a1aef6321227888bd7d6b73d5692ea"
+ nickname="davi.castro"
+ avatar="http://cdn.libravatar.org/avatar/615c6641965d1fa6b58b96d52f8a96b3"
+ subject="Is git annex config still supported?"
+ date="2018-03-06T03:03:45Z"
+ content="""
+I`m using version: 5.20151208 and there is no such command as git annex config. 
+It is also not listed on git annex help
+"""]]
diff --git a/doc/git-annex-config/comment_2_0cf3e61f86b217b89dcace22bdfaaa44._comment b/doc/git-annex-config/comment_2_0cf3e61f86b217b89dcace22bdfaaa44._comment
new file mode 100644
index 0000000000..a55b4990e9
--- /dev/null
+++ b/doc/git-annex-config/comment_2_0cf3e61f86b217b89dcace22bdfaaa44._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="joey"
+ subject="""comment 2"""
+ date="2018-03-06T17:39:40Z"
+ content="""
+@davi, you are using an *ancient* version of git-annex, 
+from before this command was added. Upgrade.
+"""]]
diff --git a/doc/git-annex-contentlocation.mdwn b/doc/git-annex-contentlocation.mdwn
new file mode 100644
index 0000000000..a090e37541
--- /dev/null
+++ b/doc/git-annex-contentlocation.mdwn
@@ -0,0 +1,38 @@
+# NAME
+
+git-annex contentlocation - looks up content for a key
+
+# SYNOPSIS
+
+git annex contentlocation `[key ...]`
+
+# DESCRIPTION
+
+This plumbing-level command looks up filename used to store the content 
+of a key. The filename is output to stdout. If the key's content is not
+present in the local repository, nothing is output, and it exits nonzero.
+
+Note that in direct mode, the file will typically be in the git work
+tree, and while its content should correspond to the key, the file
+could become modified at any time after git-annex checks it.
+
+# OPTIONS
+
+* `--batch`
+
+  Enable batch mode, in which a line containing the key is read from
+  stdin, the filename to its content is output to stdout (with a trailing
+  newline), and repeat.
+
+  Note that if a key's content is not present, an empty line is output to
+  stdout instead.
+
+# SEE ALSO
+
+[[git-annex]](1)
+
+# AUTHOR
+
+Joey Hess 
+
+Warning: Automatically converted into a man page by mdwn2man. Edit with care.
diff --git a/doc/git-annex-copy.mdwn b/doc/git-annex-copy.mdwn
new file mode 100644
index 0000000000..92fe6bee9e
--- /dev/null
+++ b/doc/git-annex-copy.mdwn
@@ -0,0 +1,124 @@
+# NAME
+
+git-annex copy - copy content of files to/from another repository
+
+# SYNOPSIS
+
+git annex copy `[path ...] [--from=remote|--to=remote]`
+
+# DESCRIPTION
+
+Copies the content of files from or to another remote.
+
+# OPTIONS
+
+* `--from=remote`
+
+  Copy the content of files from the specified
+  remote to the local repository.
+  
+  Any files that are not available on the remote will be silently skipped.
+
+* `--to=remote`
+
+  Copy the content of files from the local repository
+  to the specified remote.
+
+* `--to=here`
+
+  Copy the content of files from all reachable remotes to the local
+  repository.
+
+* `--jobs=N` `-JN`
+
+  Enables parallel transfers with up to the specified number of jobs
+  running at once. For example: `-J10`
+
+* `--auto`
+
+  Rather than copying all files, only copy files that don't yet have
+  the desired number of copies, or that are preferred content of the
+  destination repository. See [[git-annex-preferred-content]](1)
+
+* `--fast`
+
+  When copying content to a remote, avoid a round trip to check if the remote
+  already has content. This can be faster, but might skip copying content
+  to the remote in some cases.
+
+* `--all` `-A`
+
+  Rather than specifying a filename or path to copy, this option can be
+  used to copy all available versions of all files.
+
+  This is the default behavior when running git-annex in a bare repository.
+
+* `--branch=ref`
+
+  Operate on files in the specified branch or treeish.
+
+* `--unused`
+
+  Operate on files found by last run of git-annex unused.
+
+* `--failed`
+
+  Operate on files that have recently failed to be transferred.
+
+* `--key=keyname`
+
+  Use this option to move a specified key.
+
+* file matching options
+
+  The [[git-annex-matching-options]](1)
+  can be used to specify files to copy.
+
+* `--batch`
+
+  Enables batch mode, in which lines containing names of files to copy
+  are read from stdin.
+
+  As each specified file is processed, the usual progress output is
+  displayed. If a file's content does not need to be copied, or it does not
+  match specified matching options, or it is not an annexed file,
+  a blank line is output in response instead.
+
+  Since the usual output while copying a file is verbose and not
+  machine-parseable, you may want to use --json in combination with
+  --batch.
+
+* `-z`
+
+  Makes the `--batch` input be delimited by nulls instead of the usual
+  newlines.
+
+* `--json`
+
+  Enable JSON output. This is intended to be parsed by programs that use
+  git-annex. Each line of output is a JSON object.
+
+* `--json-progress`
+
+  Include progress objects in JSON output.
+
+* `--json-error-messages`
+
+  Messages that would normally be output to standard error are included in
+  the json instead.
+
+# SEE ALSO
+
+[[git-annex]](1)
+
+[[git-annex-get]](1)
+
+[[git-annex-move]](1)
+
+[[git-annex-drop]](1)
+
+# AUTHOR
+
+Joey Hess 
+
+Warning: Automatically converted into a man page by mdwn2man. Edit with care.
diff --git a/doc/git-annex-copy/comment_1_9be279110a112335bc72ee8c8a347da2._comment b/doc/git-annex-copy/comment_1_9be279110a112335bc72ee8c8a347da2._comment
new file mode 100644
index 0000000000..026a2b1455
--- /dev/null
+++ b/doc/git-annex-copy/comment_1_9be279110a112335bc72ee8c8a347da2._comment
@@ -0,0 +1,12 @@
+[[!comment format=mdwn
+ username="tomekwi"
+ subject="--all seems to do nothing"
+ date="2015-07-03T05:14:37Z"
+ content="""
+Hi,
+
+I want to back up all my files including previous versions on an external drive. I’ve tried these two commands – both of them exited instantly without producing any output:
+
+    $ git-annex copy --all --to=external-drive
+    $ git-annex copy --unused --to=external-drive
+"""]]
diff --git a/doc/git-annex-copy/comment_2_599958b0a57d2f8f4383733d28ad9c8d._comment b/doc/git-annex-copy/comment_2_599958b0a57d2f8f4383733d28ad9c8d._comment
new file mode 100644
index 0000000000..687018bb01
--- /dev/null
+++ b/doc/git-annex-copy/comment_2_599958b0a57d2f8f4383733d28ad9c8d._comment
@@ -0,0 +1,7 @@
+[[!comment format=mdwn
+ username="tomekwi"
+ subject="comment 2"
+ date="2015-07-03T05:17:12Z"
+ content="""
+By the way I’m running 5.20140717 – the version distributed by Fedora.
+"""]]
diff --git a/doc/git-annex-copy/comment_3_a9408822e314c32bebf8230f9d40216b._comment b/doc/git-annex-copy/comment_3_a9408822e314c32bebf8230f9d40216b._comment
new file mode 100644
index 0000000000..b62a73221f
--- /dev/null
+++ b/doc/git-annex-copy/comment_3_a9408822e314c32bebf8230f9d40216b._comment
@@ -0,0 +1,12 @@
+[[!comment format=mdwn
+ username="joey"
+ subject="""comment 3"""
+ date="2015-07-06T17:39:58Z"
+ content="""
+Commenting on a man page is not the right way to report a problem. File a
+bug report.
+
+FWIW, copy --all works fine in my tests. Of course, there are certianly
+situations where it copies nothing. Eg, if there are no files with ontents
+in the repository for it to copy.
+"""]]
diff --git a/doc/git-annex-copy/comment_4_44dcb42a011cc203655bccca06de2e10._comment b/doc/git-annex-copy/comment_4_44dcb42a011cc203655bccca06de2e10._comment
new file mode 100644
index 0000000000..abd34e5978
--- /dev/null
+++ b/doc/git-annex-copy/comment_4_44dcb42a011cc203655bccca06de2e10._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="b@526b37d9a821b5f9ce4e48014422239233ded155"
+ nickname="b"
+ subject="Fetch files from two (or more) remotes simultaneously?"
+ date="2016-03-21T22:47:42Z"
+ content="""
+Let's assume that user has two slow remotes. Is it possible to fetch the data from both to at least partially speed-up the process?
+"""]]
diff --git a/doc/git-annex-copy/comment_5_fe7c2c6617b3a5ca153af2225fc66498._comment b/doc/git-annex-copy/comment_5_fe7c2c6617b3a5ca153af2225fc66498._comment
new file mode 100644
index 0000000000..c243d981a7
--- /dev/null
+++ b/doc/git-annex-copy/comment_5_fe7c2c6617b3a5ca153af2225fc66498._comment
@@ -0,0 +1,11 @@
+[[!comment format=mdwn
+ username="joey"
+ subject="""comment 5"""
+ date="2016-04-04T20:30:33Z"
+ content="""
+If you run `git annex copy --from slowremote1` in one terminal and at the
+same time run `git annex copy --from slowremote2` in another terminal,
+the two processes will cooperatively get the files, spreading the load 
+amoung the remotes. This works because git-annex avoids downloading a file
+if the same file is already being downloaded by a different process.
+"""]]
diff --git a/doc/git-annex-dead.mdwn b/doc/git-annex-dead.mdwn
new file mode 100644
index 0000000000..d7acaa2d59
--- /dev/null
+++ b/doc/git-annex-dead.mdwn
@@ -0,0 +1,45 @@
+# NAME
+
+git-annex dead - hide a lost repository or key
+
+# SYNOPSIS
+
+git annex dead `[repository ...] [--key key]`
+
+# DESCRIPTION
+
+This command exists to deal with situations where data has been lost,
+and you know it has, and you want to stop being reminded of that fact.
+
+When a repository is specified, indicates that the repository has
+been irretrievably lost, so it will not be listed in eg, `git annex info`.
+Repositories can be specified using their remote name, their
+description, or their UUID. (To undo, use `git-annex semitrust`.)
+
+When a key is specified, indicates that the content of that key has been
+irretrievably lost. This prevents commands like `git annex fsck --all`
+from complaining about it; `--all` will not operate on the key anymore.
+(To undo, add the key's content back to the repository, 
+by using eg, `git-annex reinject`.)
+
+# SEE ALSO
+
+[[git-annex]](1)
+
+[[git-annex-trust]](1)
+
+[[git-annex-semitrust]](1)
+
+[[git-annex-untrust]](1)
+
+[[git-annex-expire]](1)
+
+[[git-annex-fsck]](1)
+
+[[git-annex-reinject]](1)
+
+# AUTHOR
+
+Joey Hess 
+
+Warning: Automatically converted into a man page by mdwn2man. Edit with care.
diff --git a/doc/git-annex-describe.mdwn b/doc/git-annex-describe.mdwn
new file mode 100644
index 0000000000..cc5439ddda
--- /dev/null
+++ b/doc/git-annex-describe.mdwn
@@ -0,0 +1,31 @@
+# NAME
+
+git-annex describe - change description of a repository
+
+# SYNOPSIS
+
+git annex describe repository description
+
+# DESCRIPTION
+
+Changes the description of a repository.
+
+The repository to describe can be specified by git remote name or
+by uuid. To change the description of the current repository, use
+"here".
+
+Repository descriptions are displayed by git-annex in various places.
+They are most useful when git-annex knows about a repository, but there is
+no git remote corresponding to it.
+
+# SEE ALSO
+
+[[git-annex]](1)
+
+[[git-annex-init]](1)
+
+# AUTHOR
+
+Joey Hess 
+
+Warning: Automatically converted into a man page by mdwn2man. Edit with care.
diff --git a/doc/git-annex-diffdriver.mdwn b/doc/git-annex-diffdriver.mdwn
new file mode 100644
index 0000000000..3df61154e1
--- /dev/null
+++ b/doc/git-annex-diffdriver.mdwn
@@ -0,0 +1,37 @@
+# NAME
+
+git-annex diffdriver - external git diff driver shim
+
+# SYNOPSIS
+
+git annex diffdriver `-- cmd --opts --`
+
+# DESCRIPTION
+
+This is an external git diff driver shim. Normally, when using `git diff`
+with an external diff driver, the symlinks to annexed files are not set up
+right, so the external diff driver cannot read them in order to perform
+smart diffing of their contents. This command works around the problem,
+by passing the fixed up files to the real external diff driver.
+
+To use this, you will need to have installed some git external diff driver
+command. This is not the regular diff command; it takes a git-specific
+input. See git's documentation of `GIT_EXTERNAL_DIFF` and
+gitattributes(5)'s documentation of external diff drivers.
+
+Configure git to use "git-annex diffdriver -- cmd params --"
+as the external diff driver, where cmd is the external diff
+driver you want it to run, and params are any extra parameters to pass
+to it. Note the trailing "--", which is required.
+
+For example, set `GIT_EXTERNAL_DIFF=git-annex diffdriver -- j-c-diff --`
+
+# SEE ALSO
+
+[[git-annex]](1)
+
+# AUTHOR
+
+Joey Hess 
+
+Warning: Automatically converted into a man page by mdwn2man. Edit with care.
diff --git a/doc/git-annex-direct.mdwn b/doc/git-annex-direct.mdwn
new file mode 100644
index 0000000000..c3d7dfadc2
--- /dev/null
+++ b/doc/git-annex-direct.mdwn
@@ -0,0 +1,40 @@
+# NAME
+
+git-annex direct - switch repository to direct mode
+
+# SYNOPSIS
+
+git annex direct
+
+# DESCRIPTION
+
+Switches a repository to use direct mode, where rather than symlinks to
+files, the files are directly present in the repository.
+  
+As part of the switch to direct mode, any changed files will be committed.
+
+Note that git commands that operate on the work tree will refuse to
+run in direct mode repositories. Use `git annex proxy` to safely run such
+commands.
+
+Note that the direct mode/indirect mode distinction is removed in v6
+git-annex repositories. In such a repository, you can
+use [[git-annex-unlock]](1) to make a file's content be directly present.
+You can also use [[git-annex-adjust]](1) to enter a branch where all
+annexed files are unlocked, which is similar to the old direct mode.
+
+# SEE ALSO
+
+[[git-annex]](1)
+
+[[git-annex-indirect]](1)
+
+[[git-annex-unlock]](1)
+
+[[git-annex-adjust]](1)
+
+# AUTHOR
+
+Joey Hess 
+
+Warning: Automatically converted into a man page by mdwn2man. Edit with care.
diff --git a/doc/git-annex-direct/comment_1_5d6f9e9393e534e027ddcd555861196b._comment b/doc/git-annex-direct/comment_1_5d6f9e9393e534e027ddcd555861196b._comment
new file mode 100644
index 0000000000..4b94b68fe6
--- /dev/null
+++ b/doc/git-annex-direct/comment_1_5d6f9e9393e534e027ddcd555861196b._comment
@@ -0,0 +1,9 @@
+[[!comment format=mdwn
+ username="tomekwi"
+ subject="direct mode within a directory?"
+ date="2015-06-13T13:03:33Z"
+ content="""
+Hi, is it possible to enable direct mode only on a single directory?
+
+I can’t use symlinks on a part of my repo because of interoperability with http://tagspaces.org . And I don’t want to have all of my repo in direct mode.
+"""]]
diff --git a/doc/git-annex-direct/comment_2_ef8ac748470549776c1ee299b062add2._comment b/doc/git-annex-direct/comment_2_ef8ac748470549776c1ee299b062add2._comment
new file mode 100644
index 0000000000..2329a5ee83
--- /dev/null
+++ b/doc/git-annex-direct/comment_2_ef8ac748470549776c1ee299b062add2._comment
@@ -0,0 +1,7 @@
+[[!comment format=mdwn
+ username="tomekwi"
+ subject="…"
+ date="2015-06-13T13:07:49Z"
+ content="""
+Using submodules to split the repo doesn’t work cleanly for me, by the way.
+"""]]
diff --git a/doc/git-annex-drop.mdwn b/doc/git-annex-drop.mdwn
new file mode 100644
index 0000000000..35e890f75c
--- /dev/null
+++ b/doc/git-annex-drop.mdwn
@@ -0,0 +1,119 @@
+# NAME
+
+git-annex drop - remove content of files from repository
+
+# SYNOPSIS
+
+git annex drop `[path ...]`
+
+# DESCRIPTION
+
+Drops the content of annexed files from this repository, when
+possible.
+
+git-annex will refuse to drop content if it cannot verify it is
+safe to do so.
+
+# OPTIONS
+
+* `--from=remote`
+
+  Rather than dropping the content of files in the local repository,
+  this option can specify a remote from which the files'
+  contents should be removed.
+
+* `--auto`
+
+  Rather than trying to drop all specified files, drop only files that
+  are not preferred content of the repository.
+  See [[git-annex-preferred-content]](1)
+
+* `--force`
+
+  Use this option with care! It bypasses safety checks, and forces
+  git-annex to delete the content of the specified files, even from
+  the last repository that is storing their content. Data loss can
+  result from using this option.
+
+* `--all` `-A`
+
+  Rather than specifying a filename or path to drop, this option can be
+  used to drop all available versions of all files.
+
+  This is the default behavior when running git-annex drop in a bare repository.
+
+  Note that this bypasses checking the .gitattributes annex.numcopies
+  setting.
+
+* `--branch=ref`
+
+  Drop files in the specified branch or treeish.
+
+  Note that this bypasses checking the .gitattributes annex.numcopies
+  setting.
+
+* `--unused`
+
+  Drop files found by last run of git-annex unused.
+
+  Note that this bypasses checking the .gitattributes annex.numcopies
+  setting.
+
+* `--key=keyname`
+
+  Use this option to drop a specified key.
+
+  Note that this bypasses checking the .gitattributes annex.numcopies
+  setting.
+
+* file matching options
+
+  The [[git-annex-matching-options]](1)
+  can be used to specify files to drop.
+
+* `--jobs=N` `-JN`
+
+  Runs multiple drop jobs in parallel. This is particularly useful
+  when git-annex has to contact remotes to check if it can drop files.
+  For example: `-J4`  
+
+* `--batch`
+
+  Enables batch mode, in which lines containing names of files to drop
+  are read from stdin.
+
+  As each specified file is processed, the usual output is
+  displayed. If a file's content is not present, or it does not
+  match specified matching options, or it is not an annexed file,
+  a blank line is output in response instead.
+
+* `-z`
+
+  Makes the `--batch` input be delimited by nulls instead of the usual
+  newlines.
+
+* `--json`
+
+  Enable JSON output. This is intended to be parsed by programs that use
+  git-annex. Each line of output is a JSON object.
+
+* `--json-error-messages`
+
+  Messages that would normally be output to standard error are included in
+  the json instead.
+
+# SEE ALSO
+
+[[git-annex]](1)
+
+[[git-annex-get]](1)
+
+[[git-annex-move]](1)
+
+[[git-annex-copy]](1)
+
+# AUTHOR
+
+Joey Hess 
+
+Warning: Automatically converted into a man page by mdwn2man. Edit with care.
diff --git a/doc/git-annex-drop/comment_1_ecb863aea64eadbaeb7759c5cb0bf5a5._comment b/doc/git-annex-drop/comment_1_ecb863aea64eadbaeb7759c5cb0bf5a5._comment
new file mode 100644
index 0000000000..1ca83ec18f
--- /dev/null
+++ b/doc/git-annex-drop/comment_1_ecb863aea64eadbaeb7759c5cb0bf5a5._comment
@@ -0,0 +1,12 @@
+[[!comment format=mdwn
+ username="zpeters@669a56d871ec94de11e1bf63f4a21e80ea90b2fa"
+ nickname="zpeters"
+ subject="Choose which remote to lock when dropping"
+ date="2016-05-03T23:37:09Z"
+ content="""
+I might have a unique situation but i have a git annex repo that has multiple remotes for the same actual location.  I have a server at home and when i'm \"out in the world\" i use it's external IP and when i'm at home i use it's internal IP.  It appears that when i do an git annex drop on my laptop to compares and locks whichever remote is alphabetically first.  
+
+What i'm wondering is is there a way to tell git annex which remote i want to compare against?  For example when i'm remote i want to do something like \"git annex drop . --compare remoteServer\"
+
+Thank you!
+"""]]
diff --git a/doc/git-annex-drop/comment_2_c9017041b01d803b346b1b14b8d1e066._comment b/doc/git-annex-drop/comment_2_c9017041b01d803b346b1b14b8d1e066._comment
new file mode 100644
index 0000000000..63047d7447
--- /dev/null
+++ b/doc/git-annex-drop/comment_2_c9017041b01d803b346b1b14b8d1e066._comment
@@ -0,0 +1,14 @@
+[[!comment format=mdwn
+ username="joey"
+ subject="""comment 2"""
+ date="2016-05-04T17:23:41Z"
+ content="""
+@zpeters, the order that remotes are accessed are ordered by cost.
+
+You can configure the costs of remotes by setting remote.$name.annex-cost
+to eg 100 to make it be checked first or 300 to make it be checked later.
+
+There's also a remote.$name.annex-cost-command that runs a command to get
+the cost. You could perhaps use that to detect which network you're on and
+alter the costs appropriately.
+"""]]
diff --git a/doc/git-annex-drop/comment_3_16095aad685d142672911e799450634a._comment b/doc/git-annex-drop/comment_3_16095aad685d142672911e799450634a._comment
new file mode 100644
index 0000000000..13ac15c7e0
--- /dev/null
+++ b/doc/git-annex-drop/comment_3_16095aad685d142672911e799450634a._comment
@@ -0,0 +1,14 @@
+[[!comment format=mdwn
+ username="zpeters"
+ avatar="http://cdn.libravatar.org/avatar/a8695228ff82f18c9a43c0763194c1cd"
+ subject="RE: choosing remotes and annex-cost-command"
+ date="2017-02-28T01:10:05Z"
+ content="""
+Finally got around to playing with this today figured i'd post an example for anyone looking fr this in the future.  Please forgive the TCL code if that isn't your thing.
+
+1. Adding the \"cost command\" to your remotes - Example here 
+2. The actual script being referenced is really basic.  It's basically taking the name of the remote as an argument and depending on where i am it will give that remote a \"high\" or \"low\" cost.  Code here 
+3. The cost script needs to know if I'm \"home\" or not.  Here is an example of \"whereami\" 
+
+
+"""]]
diff --git a/doc/git-annex-drop/comment_4_156873ca9820b316a2099b6646562891._comment b/doc/git-annex-drop/comment_4_156873ca9820b316a2099b6646562891._comment
new file mode 100644
index 0000000000..4bf20dd61b
--- /dev/null
+++ b/doc/git-annex-drop/comment_4_156873ca9820b316a2099b6646562891._comment
@@ -0,0 +1,9 @@
+[[!comment format=mdwn
+ username="https://launchpad.net/~liori"
+ nickname="liori"
+ avatar="http://cdn.libravatar.org/avatar/e1d0fdc746b3d21bb147160d40815e37b257b9119774d21784939b2d3ba95a91"
+ subject="The meaning of "safe""
+ date="2018-06-28T23:27:05Z"
+ content="""
+What does \"safe\" mean? Does `drop` just check if there are enough copies in other repositories in its local cache, does it contact remote repositories to check its own cache, or maybe even checks if the physical file in the remote repository has the right checksum?
+"""]]
diff --git a/doc/git-annex-drop/comment_5_655f0bcaafdf03eb637af715e7642e27._comment b/doc/git-annex-drop/comment_5_655f0bcaafdf03eb637af715e7642e27._comment
new file mode 100644
index 0000000000..55986f5ee6
--- /dev/null
+++ b/doc/git-annex-drop/comment_5_655f0bcaafdf03eb637af715e7642e27._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="joey"
+ subject="""comment 5"""
+ date="2018-07-02T16:08:27Z"
+ content="""
+@liori, see [[copies]] for how git-annex assures safety when dropping
+content.
+"""]]
diff --git a/doc/git-annex-drop/comment_6_ed46bc0407457fc5db0d5241ba69d9ed._comment b/doc/git-annex-drop/comment_6_ed46bc0407457fc5db0d5241ba69d9ed._comment
new file mode 100644
index 0000000000..db896522e8
--- /dev/null
+++ b/doc/git-annex-drop/comment_6_ed46bc0407457fc5db0d5241ba69d9ed._comment
@@ -0,0 +1,10 @@
+[[!comment format=mdwn
+ username="https://me.yahoo.com/a/iOGTltEpmOTQ.xZ99NFP5c7Zdcc-#6a7ba"
+ nickname="Ilya S"
+ avatar="http://cdn.libravatar.org/avatar/8a133555cc739a35b83b07d5724d28d9e2f7852c224e949eec6fd4fb7693331e"
+ subject="comment 6"
+ date="2018-09-07T18:10:05Z"
+ content="""
+Is there a way to drop everything _except_ files under a given path in the repository?   If not, it would be good to add this option.
+It should also not drop a file if it is outside the path but is pointed to by a relative symlink under the path.
+"""]]
diff --git a/doc/git-annex-drop/comment_7_f9b255923bdb20c92b878d86998b7ae2._comment b/doc/git-annex-drop/comment_7_f9b255923bdb20c92b878d86998b7ae2._comment
new file mode 100644
index 0000000000..3050fcbc4f
--- /dev/null
+++ b/doc/git-annex-drop/comment_7_f9b255923bdb20c92b878d86998b7ae2._comment
@@ -0,0 +1,11 @@
+[[!comment format=mdwn
+ username="kyle"
+ avatar="http://cdn.libravatar.org/avatar/7d6e85cde1422ad60607c87fa87c63f3"
+ subject="re: drop everything but subdirectory"
+ date="2018-09-07T20:13:03Z"
+ content="""
+> Is there a way to drop everything except files under a given path in the repository?
+
+You can use something like `git annex drop --exclude='subdir/*'`.  See `man git-annex-matching-options`.
+
+"""]]
diff --git a/doc/git-annex-dropkey.mdwn b/doc/git-annex-dropkey.mdwn
new file mode 100644
index 0000000000..6816407371
--- /dev/null
+++ b/doc/git-annex-dropkey.mdwn
@@ -0,0 +1,47 @@
+# NAME
+
+git-annex dropkey - drops annexed content for specified keys
+
+# SYNOPSIS
+
+git annex dropkey `[key ...]`
+
+# DESCRIPTION
+
+This plumbing-level command drops the annexed data for the specified
+keys from this repository.
+
+This can be used to drop content for arbitrary keys, which do not need
+to have a file in the git repository pointing at them.
+
+Warning: This command does not check that enough other copies of the content
+exist; using it can easily result in data loss.
+
+# OPTIONS
+
+* `--batch`
+
+  Enables batch mode, in which lines containing keys to drop are read from
+  stdin.
+
+* `--json`
+
+  Enable JSON output. This is intended to be parsed by programs that use
+  git-annex. Each line of output is a JSON object.
+
+* `--json-error-messages`
+
+  Messages that would normally be output to standard error are included in
+  the json instead.
+
+# SEE ALSO
+
+[[git-annex]](1)
+
+[[git-annex-setkey]](1)
+
+# AUTHOR
+
+Joey Hess 
+
+Warning: Automatically converted into a man page by mdwn2man. Edit with care.
diff --git a/doc/git-annex-dropunused.mdwn b/doc/git-annex-dropunused.mdwn
new file mode 100644
index 0000000000..baadd0cc3c
--- /dev/null
+++ b/doc/git-annex-dropunused.mdwn
@@ -0,0 +1,41 @@
+# NAME
+
+git-annex dropunused - drop unused file content
+
+# SYNOPSIS
+
+git annex dropunused `[number|range ...]`
+
+# DESCRIPTION
+
+Drops the data corresponding to the numbers, as listed by the last
+git annex unused`
+
+You can also specify ranges of numbers, such as "1-1000".
+Or, specify "all" to drop all unused data.
+
+# OPTIONS
+
+* `--from=remote`
+
+  Rather than dropping the unused files from the local repository,
+  drop them from the remote repository.
+
+* `--force`
+
+  Use this option with care! It bypasses safety checks, and forces
+  git-annex to delete the content of the specified files, even from
+  the last repository that is storing their content. Data loss can
+  result from using this option.
+
+# SEE ALSO
+
+[[git-annex]](1)
+
+[[git-annex-unused]](1)
+
+# AUTHOR
+
+Joey Hess 
+
+Warning: Automatically converted into a man page by mdwn2man. Edit with care.
diff --git a/doc/git-annex-edit.mdwn b/doc/git-annex-edit.mdwn
new file mode 100644
index 0000000000..ef8502cfa6
--- /dev/null
+++ b/doc/git-annex-edit.mdwn
@@ -0,0 +1,18 @@
+# NAME
+
+git-annex unlock - unlock files for modification
+
+# SYNOPSIS
+
+git annex edit `[path ...]`
+
+# DESCRIPTION
+
+This is an alias for the `unlock` command; see [[git-annex-unlock]](1)
+for details.
+
+# AUTHOR
+
+Joey Hess 
+
+Warning: Automatically converted into a man page by mdwn2man. Edit with care.
diff --git a/doc/git-annex-enable-tor.mdwn b/doc/git-annex-enable-tor.mdwn
new file mode 100644
index 0000000000..f06966400e
--- /dev/null
+++ b/doc/git-annex-enable-tor.mdwn
@@ -0,0 +1,36 @@
+# NAME
+
+git-annex enable-tor - enable tor hidden service
+
+# SYNOPSIS
+
+git annex enable-tor
+
+sudo git annex enable-tor $(id -u)
+
+# DESCRIPTION
+
+This command enables a tor hidden service for git-annex.
+
+It modifies `/etc/tor/torrc` to register the hidden service. If run as a
+normal user, it will try to use sudo/su/etc to get root access to modify
+that file. If you run it as root, pass it your non-root user id number,
+as output by `id -u`
+
+After this command is run, `git annex remotedaemon` can be run to serve the
+tor hidden service, and then `git-annex p2p --gen-address` can be run to
+give other users access to your repository via the tor hidden service.
+
+# SEE ALSO
+
+[[git-annex]](1)
+
+[[git-annex-p2p-auth]](1)
+
+[[git-annex-remotedaemon]](1)
+
+# AUTHOR
+
+Joey Hess 
+
+Warning: Automatically converted into a man page by mdwn2man. Edit with care.
diff --git a/doc/git-annex-enableremote.mdwn b/doc/git-annex-enableremote.mdwn
new file mode 100644
index 0000000000..d424f6056c
--- /dev/null
+++ b/doc/git-annex-enableremote.mdwn
@@ -0,0 +1,75 @@
+# NAME
+
+git-annex enableremote - enables git-annex to use a remote
+
+# SYNOPSIS
+
+git annex enableremote `name|uuid|desc [param=value ...]`
+
+# DESCRIPTION
+
+Enables use of an existing remote in the current repository.
+
+This is often used to enable use of a special (non-git) remote, by
+a different repository than the one in which it was
+originally created with the initremote command. 
+
+It can also be used to explicitly enable a git remote,
+so that git-annex can store the contents of files there. First
+run `git remote add`, and then `git annex enableremote` with the name of
+the remote.
+
+When enabling a special remote, specify the same name used when originally
+creating that remote with `git annex initremote`. Run 
+`git annex enableremote` without any name to get a list of
+special remote names. Or you can specify the uuid or description of the
+special remote.
+  
+Some special remotes may need parameters to be specified every time they are
+enabled. For example, the directory special remote requires a directory=
+parameter every time.
+
+This command can also be used to modify the configuration of an existing
+special remote, by specifying new values for parameters that are
+usually set when using initremote. (However, some settings such as
+the as the encryption scheme cannot be changed once a special remote
+has been created.)
+
+The GPG keys that an encrypted special remote is encrypted with can be
+changed using the keyid+= and keyid-= parameters. These respectively
+add and remove keys from the list. However, note that removing a key
+does NOT necessarily prevent the key's owner from accessing data
+in the encrypted special remote
+(which is by design impossible, short of deleting the remote).
+  
+One use-case of keyid-= is to replace a revoked key with
+a new key:
+  
+	git annex enableremote mys3 keyid-=revokedkey keyid+=newkey
+  
+Also, note that for encrypted special remotes using plain public-key
+encryption (encryption=pubkey), adding or removing a key has NO effect
+on files that have already been copied to the remote. Hence using
+keyid+= and keyid-= with such remotes should be used with care, and
+make little sense except in cases like the revoked key example above.
+
+If you get tired of manually enabling a special remote in each new clone,
+you can pass "autoenable=true". Then when [[git-annex-init]](1) is run in
+a new clone, it will will attempt to enable the special remote. Of course,
+this works best when the special remote does not need anything special
+to be done to get it enabled.
+
+(This command also can be used to enable a remote that git-annex has been
+prevented from using by the `remote..annex-ignore` setting.)
+
+# SEE ALSO
+
+[[git-annex]](1)
+
+[[git-annex-initremote]](1)
+
+# AUTHOR
+
+Joey Hess 
+
+Warning: Automatically converted into a man page by mdwn2man. Edit with care.
diff --git a/doc/git-annex-enableremote/comment_2_43876327581d6bcfb69c318c02d6389a._comment b/doc/git-annex-enableremote/comment_2_43876327581d6bcfb69c318c02d6389a._comment
new file mode 100644
index 0000000000..e3b51b6a9c
--- /dev/null
+++ b/doc/git-annex-enableremote/comment_2_43876327581d6bcfb69c318c02d6389a._comment
@@ -0,0 +1,21 @@
+[[!comment format=mdwn
+ username="joey"
+ subject="""comment 2"""
+ date="2017-02-14T18:16:52Z"
+ content="""
+@Sundar, good question.
+
+`git annex enableremote` will always refuse to enable the remote if there's
+a missing parameter, and prompt for the parameter. Finding the right value
+is up to you. Most of the time, no additional parameters are needed, or
+the parameters are fairly self-explanatory, eg login passwords for remote
+services.
+
+The difficulty with directory special remotes is that my /foo may not be
+the same as your /foo, so it can't reuse the directory= that was provided
+to initremote, and it's up to you to enter the right directory path.
+
+I think this needs to come down to documentation in the repository. The
+description of the remote (set by `git annex describe` 
+is a reasonable place to put that, unless you have somewhere better.
+"""]]
diff --git a/doc/git-annex-enableremote/comment_2_78223f5c6ad120d5e0b98583543425db._comment b/doc/git-annex-enableremote/comment_2_78223f5c6ad120d5e0b98583543425db._comment
new file mode 100644
index 0000000000..a42e2c6796
--- /dev/null
+++ b/doc/git-annex-enableremote/comment_2_78223f5c6ad120d5e0b98583543425db._comment
@@ -0,0 +1,37 @@
+[[!comment format=mdwn
+ username="https://me.yahoo.com/a/80VlVB0Bx9TaNOXIj3OCQ8eimAtIOhqjUQ--#1e80e"
+ nickname="Sundar Raman"
+ avatar="http://cdn.libravatar.org/avatar/a5db56ac8777aee80bbe2b1e1d697c3961973049a22def38b4d15b7507e59c01"
+ subject="Retrieving initremote parameters for new user"
+ date="2017-02-14T00:04:50Z"
+ content="""
+Is there a way to determine the parameters that an enableremote command must use, if one does not know it? The use case is as follows:
+* Dev 1 performs an `initremote annexed-media directory=/path/to/media ...`
+* Dev 1 syncs content
+* Dev 2 comes along (or Dev 1 comes along months later with a different machine) and clones the repo, but needs to know the directory=/path... in order to 'enableremote'. Is there any way to glean this information from the source repo itself?
+
+The steps would be:
+
+```
+dev1$ git clone git@gitserver:myproject.git && cd myproject
+dev1$ mkdir images && touch images/foo1.png
+dev1$ git annex initremote annexation.dir directory=/mnt/media/myproject.annex/ encrypted=false
+dev1$ git commit && git push && git annex sync --content
+```
+
+
    
+ENDBODY]]
+
+Use this template to insert a note into a page. The note will be styled to
+float to the right of other text on the page. This template has one
+parameter:
+
      
+
    • `text` - the text to display in the note
+
    
diff --git a/doc/templates/walkthrough.tmpl b/doc/templates/walkthrough.tmpl
new file mode 100644
index 0000000000..a500a5a865
--- /dev/null
+++ b/doc/templates/walkthrough.tmpl
@@ -0,0 +1,2 @@
+
    
+
diff --git a/doc/testimonials.mdwn b/doc/testimonials.mdwn
new file mode 100644
index 0000000000..65fe8adce7
--- /dev/null
+++ b/doc/testimonials.mdwn
@@ -0,0 +1,34 @@
+
    
+
+
+
+
+
+
+
+
+
+
+
    
+
+
    
+What excites me about GIT ANNEX is how it fundamentally tracks the
+backup and availability of any data you own, and allows you to share
+data with a large or small audience, ensuring that the data survives.
+
    
+-- Jason Scott 
+
+Seen on IRC:
+
    +oh my god, git-annex is amazing
+this is the revolution in fucking with gigantic piles of files that I've been waiting for
+
    
+
+And then my own story: I have a ton of drives. I have a lot of servers. I
+live in a cabin on **dialup** and often have 1 hour on broadband in a week
+to get everything I need. Without git-annex, managing all this would not be
+possible. It works perfectly for me, not a surprise since I wrote it, but
+still, it's a different level of "perfect" than anything I could put
+together before. --[[Joey]]
+
+See also: [[design/assistant/blog/day_288__success_stories]]
diff --git a/doc/testimonials/comment_1_2bf439f7a3bc3d6fab91849017946182._comment b/doc/testimonials/comment_1_2bf439f7a3bc3d6fab91849017946182._comment
new file mode 100644
index 0000000000..2c5facd1f9
--- /dev/null
+++ b/doc/testimonials/comment_1_2bf439f7a3bc3d6fab91849017946182._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="gjost"
+ ip="108.205.152.179"
+ subject="distributed digital repository"
+ date="2013-11-14T00:39:49Z"
+ content="""
+I am in the process of building a distributed digital repository using Git and git-annex as a foundation. Our project is to collect and preserve archival-quality audio and video from multiple bandwidth-challenged remote sites. We're using git-annex to help us sneakernet the metadata and binaries around and to maintain multiple copies in geographically separate locations. Git very neatly addresses the problem of tracking changes, while git-annex makes it possible to track the flow of information through a network of distributed repositories. 
+"""]]
diff --git a/doc/thanks.mdwn b/doc/thanks.mdwn
new file mode 100644
index 0000000000..8efa692b2e
--- /dev/null
+++ b/doc/thanks.mdwn
@@ -0,0 +1,388 @@
+The development of git-annex was made possible by the generous
+donations of many people. I want to say "Thank You!" to each of
+you individually, but until I meet all 1500+ of you, this page will have to
+do. You have my most sincere thanks. --[[Joey]]
+
+You can support my git-annex development
+[on Patreon](https://www.patreon.com/joeyh) or
+[on Liberapay](https://liberapay.com/joeyh/) or
+[other ways](https://joeyh.name/thanks/)
+
+(If I got your name wrong, or you don't want it publically posted here,
+email .)
+
+## 2014-2018
+
+NSF logo
+
+git-annex development is partially supported by the
+[NSF](https://www.nsf.gov/awardsearch/showAward?AWD_ID=1429999) as a part of the
+[DataLad project](http://datalad.org/).
+
+Thanks also to these folks for their support: 
+[[!inline raw=yes pages="thanks/list"]] and anonymous supporters.
+
+## 2013-2014
+
+Continued git-annex development was [crowd funded](https://campaign.joeyh.name/)
+by these excellent people. *Hug*
+
+Eric Hanchrow, Michael Barabanov, Douglas Butts, Cody Woodard, Anna Hess,
+Maggie Hess, Francois Marier, Michael Linksvayer, Alexander Brem, Yaroslav
+Halchenko, Otavio Real Salvador, Giovanni Moretti, Mark Hepburn, Jim
+Cheetham, Ali Gündüz, Chia Shee Liang, Vincent Arel-Bundock, Mark R Jones,
+Willard Korfhage, Faidon Liambotis, Daniel Callahan, Sheila Miguez,
+Christopher Webber, Daniel Patterson, Charity Hanif, Blair Mckenzie, Romain
+Lenglet, Joseph Liu, Robert Beaty, Remy van Elst, Juho Snellman, Jason
+Davidson, Dane Larsen, Kuno Woudt, Navishkar Rao, Ryan Sorensen, Jonathan
+Castello, Nimrod Mesika, Amitai Schlair, Zellyn Hunter, Jan Pieper, Boyd
+Smith, Brett Eisenberg, Péter Károly Juhász, Allan Franta, Aaron De Vries,
+Donald Plummer, Joona Lehtomäki, Evan Anderson, Pedro Côrte-Real, Bradley
+Unterrheiner, Wietse van Buitenen, James Shubin, Daniel de Kok, Michael
+Zehrer, Peter deHaan, David Schmitt, Ian Downes, Gioele Barabucci, Gergely
+Nagy, Diego Ongaro, Jonas Wiklund, Strakhov Artem, £ukasz Szczęsny, Peter
+Olson, Thomas Berker, Greg Young, narf, Michael Jakl, Andrea Rota, Tobias
+Nix, Andreas Olsson, Tobias Ussing, Pere Quintana Segui, Pierre Gambarotto,
+Yuriy Zaytsev, Thomas Djärv, Thomas Hochstein, Simon Lundmark, Damien
+Diederen, Arthur Lutz, Gert Van Gool, Jed Brown, Sören Brunk, Thorsten
+Bormer, Vincent Sanders, Thomas Lundstrøm, Robert Connolly, Marc Seeger,
+Per Andreas Buer, Thomas Andersen, Hendrik Müller Hofstede, Dominik
+Wagenknecht, Robert Walker, Nicholas Weigand, Maximilian Weigand, Thomas
+Jost, John, Angel Ramboi, Thomas Bartels, Fernando Jimenez, Tobias
+Mersmann, András Veres-Szentkirályi, Richard Hartmann, John Hedges,
+Nedialko Andreev, Roderik Koenders, Sławomir Gwizdowski, Aapo Laitinen,
+Jonathan Harrington, Dirk Kraft, Fabian Bartschke, Philip Frank, Patrick
+Steegstra, Roberto Zedda, Leon Major, Torbjørn Thorsen, Lukas Anzinger,
+Pollan Ruben, Daniel Hofer, Johannes Krampf, Jörn Gersdorf, Kristoffer Egil
+Bonarjee, David Iwanowitsch, Daniel Egger, Titus Stahl, Robin Sheat, Thomas
+Nicolaisen, Karsten Heymann, Askings & Medina Business Services, Pieter
+Mulder, Geoffrey Huntley, Wouter Mooij, Kari Oikarinen, sven dowideit,
+Ethan Aubin, Nathan Rickerby, Tobias Brunner, Stian Haukaas Eikeland, Jan
+Dittberner, Jeffrey Goeke-Smith, Joseph Cullen, James Orr, Markus Plangg,
+Glendon Solsberry, Georg Bauer, Andreas Leha, Ian Kozhurin, Boris Hupkens
+van der Elst, Daniel Brockman, Scott Robinson, Olaf Lorenzen, Øyvind
+Andersen Holm, Logan Owen, Christopher Douglass, Nicolas Pouillard, Florian
+Schlegel, Hans-Christian Jehg, Thomas Mechtler, Joshua McKinney, Tom
+Atkins, Paul Scott-Wilson, Rafal Czlonka, Benjamin Williams, Ulises Vitulli, Mesar
+Hameed, Vishakh Vishakh, Daniel Washburn, Maximilian Haack, Jan Gondol,
+Carlo Matteo Capocasa, Jevgenijs Zolotarskis, John Cooper, Matthew Gregg,
+Peter Jarosak, Edward Heil, Kyle Kurtenbach, David Kaminski, Joonathan
+Mägi, Piotr Ożarowski, Ernest Braganza, Adam Sjøgren, Simon Michael,
+Protonet, Valentin Churavy, Trenton Cronholm, Ellis Whitehead, Maarten
+Everts, Paul Tötterman, Julien Lefrique, Robert F Donnelly,
+LeastAuthority.com, Floris Bruynooghe, Kevin Krouse, Bernhard Schmidt,
+Jason Woofenden, Jacek Nykis, Benjamin Carrillo, Teemu Hukkanen, William
+Heinbockel, Kristofer Bergstrom, Richard Gibson, Julian Andres Klode,
+Jeffrey Guenthner, Jochen Kupperschmidt, Andrew Eskridge, Eric Wallmander,
+Aaron Haviland, Evan Deaubl, Felix Crux, Matthew Marshall, Hamish Coleman,
+xymostech, Matthew Mercer, Ben Challenor, András Csáki, Stelian Iancu,
+Dmitri Popov, Anthony DiSanti, Perttu Luukko, Daniel Glassey, Aurélien
+Pinceaux, Svenne Krap, Sven Hartge, Frank Blendinger, Lisa Feilen, Kyle
+Kujala-Korpela, Walter Somerville, Filippo Giunchedi, Tyler Willingham,
+Gernot Schulz, Daniel Gervais, Denver Gingerich, Nicolas Joseph, tommy,
+bak, Grejdi G, Gabriel de Perthuis, Mayank Jain, Radu Raduta, Gregory
+Grossmeier, Errol Hess, Janin Kolenc, Sean Whitton, Philippe Gauthier,
+Ananda Widyadharma, Jochen Bartl, Fredrik Hammar, THM Schoemaker, Marek
+Jawurek, Johan Herland, Gian-Maria Daffre, Justine Lam, Ori Livneh, Arnaud
+Berthomier, Chad Horohoe, Lois DeFiore, Lieven Baes, Patrick Wheeler, James
+Kim, Carlos Trijueque Albarran, Ritesh Nadhani, chesty, Andre Pereira,
+Eskild Hustvedt, David Wagner, Maximiliano Curia, András Széll, Allan
+Holman, Thomas Langewouters, Anonymous, Peter Daengeli, Josh Taylor, Abhishek Dasgupta, Maarten Aertsen, Mark Sheppard,
+Daengeli, Josh Taylor, Abhishek Dasgupta, Maarten Aertsen, Mark Sheppard,
+Markus Engström, Samuel Tardieu, Geog Wechslberger, Abdó Roig, Dmitry
+Markushevich, Sergio Rubio, Jim Paris, Vivek Gani, Brock Spratlen, Nathan
+Howell, Alan Chan, Alexandre Dupas, Daniel Atlas, Patrick Hallen, Matthew
+Forrester, Binyamin Cherniavsky, Florent Fourcot, Nathan Yergler, Hannibal
+Skorepa, Cedric Staub, Melissa Binde, Marc Bobillier, Zoran Zaric, Ralph
+Mayer, Jay Pozo, Walter Somerville, Remy Honig, Bernhard Urban, Nurullah
+Akkaya, Ari Pollak, Kalle Svensson, Stephan Schulz, Joshua Honeycutt,
+Justin Keyes, Filias Heidt, Manish Sharma, Benoît Zugmeyer, Andrew Kraut,
+Markus Breitenberger, Dominik Deobald, Aren Olson, Tobias Ammann, Peter
+Lloyd, Pierre Chambart, Michael Alan Dorman, Kyle Meyer, Kristoffer
+Gronlund, Jay Dugger, Jeff McNeill, Oskari Timperi, Mathias Linden,
+Ole-Morten Duesund, Tom Lowenthal, Anders Lannerback, Tony Cantor,
+Stephanie Eikenberry, Vanja Cvelbar, Fabrice Rossi, Franz Pletz, Karl
+Wiberg, Sam Kleinman, Vincent Demeester, Tristan Helmich, Zero Art Radio,
+Bruno Bigras, Ævar Arnfjörð Bjarmason, Stanley Yamane, Christopher Browne,
+David Whittington, Fredrik Gustafsson, Peter Hogg, Tom Francart, Wouter
+Verhelst, Christian Savard, wundersolutions, Andreas Fuchs, Eric Kidd,
+Georg Lehner, Berin Martini, Stewart Wright, Bence parhuzamos, Stefan
+Schmitt, Antoine Boegli, jscit, Christopher Kernahan, A Marshall, Jürgen
+Peters, Aaron Whitehouse, Jouni K Seppanen, Michael Albertson, Andreas
+Laas, Thomas Herok, Aurelien Gazagne, Bryan W Stitt, anonymous, Chris
+Kastorff, Henning Hasemann, Leonardo Taglialegne, Frank Thomas, Frédéric
+Schütz, Riad Wahby, Maxwell Swadling, Bram Senders, Péter Párkányi, Tom
+Dimiduk, Christian Kellermann, Kilian Evang, Christian Studer, Tim
+Humphrey, Jürgen, Richard Collins, Renaud Casenave-Péré, Patrick Lynn,
+Gregor Gramlich, Hugo Mougard, Arun Srinivasan, Paul Tagliamonte, Don
+Armstrong, Adam DeWitt, Timothy Hobbs, Jacob Briggs, Mark Eichin, Matthew
+Raso-Barnett, Damien Raude-Morvan, Mark Lopez, Mattias Olsson, Marco A L
+Barbosa, Thomas de Ruiter, Anders Claesson, Philipp Meier, Henrik Ahlgren,
+Michael Hofmann, Audric SCHILTKNECHT, Wojtek Burakiewicz, Christian
+Dietrich, Jacob Kirkwood, Fabian Jansen, gregor herrmann, Martin Werner,
+Adam Baxter, Sören Köpping, Pierce Lopez, David Lehn, Michal Politowski,
+John Lawrence, Nicholas Golder, Adam Spiers, Pär Mattsson, Matt Lee,
+Timo Heister, Asbjørn Sloth Tønnesen, James Valleroy, Felix Gruber, Björn
+Pettersson, Nicola Chiapolini, Christer Stenbrenden, Hannes Maier-Flaig,
+Paul Walker, Cary Kempston, Andrew Cant, Albert Drengg,
+Martin Pollow, Karl-Georg Sommer, Francois Cocquemas, Daniel Bross, Martin
+Florian, Grigorios Prasinos, Olivier Berger, Andreas Herten, Kyle MacLea,
+Sozykin Mikhail, André Klärner, Pascal VITOUX, Gunilla Johansson, Dara
+Adib, Nicolas Bonifas, Glen Ogilvie, TasLUG, Shaun Westmacott, Georgi
+Valkov, Nicolas Dietrich, Auroch, Rasmus Pank Roulund, Michael Niewiera,
+Benjamin Koch, Joe Bogner, Christoph Varga, Florian Eitel, Tobias Braun,
+Jochen Schulz, Diggory Hardy, Stephane Barland, Sebastian Bober, Arash
+Rouhani-Kalleh, Geoffrey Irving, Hans Tzou, Mark Booth, Christoph Neckel,
+Aleksandar Iliev, Baldur Kristinsson, Adam Colton, KAWACHI TAKASHI, Michal
+Sojka, Alexander Brandstedt, Jarno Elonen, Rob Sharp, John Pybus, Antonino
+Mazzurco, Damon Haley, Jason Locklin, ULRICH KORTENKAMP, Karthik
+Poobalasubramanian, Alexandre Garel, David Clark, Jeff Johnson,
+Mica Semrick, Paul Staab, Rémi Vanicat, Martin Holtschneider, Jan Ivar
+Beddari, Peter Simons, Thomas Koch, Justin Geibel, Guillaume DELVIT, Shanti
+Bouchez, Oliver Brandt, François Deppierraz, Chad Walstrom, Tim Mattison,
+Jakub Antoni Tyszko, Casa do Boneco, Florian Tham,
+and 30 anonymous bitcoin users
+
+With an especial thanks to the WikiMedia foundation,
+and Rede Mocambos.
+
+## 2012-2013
+
+
+The git-annex assistant was [crowd funded on Kickstarter](http://www.kickstarter.com/projects/joeyh/git-annex-assistant-like-dropbox-but-with-your-own/).
+
+### Major Backers
+
+These people are just inspiring in their enthusiasm and generosity to this
+project.
+
+* Jason Scott
+* strager
+
+### Beta Testers
+
+Whole weeks of my time were made possible thanks to each of these
+people, and their testing is invaluable to the development of
+the git-annex assistant.
+
+* Jimmy Tang
+* David Pollak
+* Pater
+* Francois Marier
+* Paul Sherwood
+* Fred Epma
+* Robert Ristroph
+* Josh Triplett
+* David Haslem
+* AJ Ashton
+* Svenne Krap
+* Drew Hess
+* Peter van Westen
+
+### Prioritizers
+
+These forward-thinking people contributed generously just to help
+set my priorities in which parts of the git-annex assistant were most
+important to develop.
+
+Paul C. Bryan, Paul Tötterman, Don Marti, Dean Thompson, Djoume, David Johnston
+Asokan Pichai, Anders Østhus, Dominik Wagenknecht, Charlie Fox, Yazz D. Atlas,
+fenchel, Erik Penninga, Richard Hartmann, Graham, Stan Yamane, Ben Skelton,
+Ian McEwen, asc, Paul Tagliamonte, Sherif Abouseda, Igor Támara, Anne Wind,
+Mesar Hameed, Brandur K. Holm Petersen, Takahiro Inoue, Kai Hendry,
+Stephen Youndt, Lee Roberson, Ben Strawbridge, Andrew Greenberg, Alfred Adams
+Andrew, Aaron De Vries, Monti Knazze, Jorge Canseco, Hamish, Mark Eichin,
+Sherif Abouseda, Ben Strawbridge, chee rabbits, Pedro Côrte-Real
+
+And special thanks to Kevin McKenzie, who also gave me a login to a Mac OSX
+machine, which has proven invaluable, Jimmy Tang who has helped
+with Mac OSX autobuilding and packaging, and Yury V. Zaytsev who
+provides the Windows autobuilder.
+
+### Other Backers
+
+Most of the success of the Kickstarter is thanks to these folks. Some of
+them spent significant amounts of money in the guise of getting some
+swag. For others, being listed here, and being crucial to making the 
+git-annex assistant happen was reward enough. Large or small, these
+contributions were, literally, my bread and butter this year.
+
+Amitai Schlair, mvime, Romain Lenglet, James Petts, Jouni Uuksulainen,
+Wichert Akkerman, Robert Bellus, Kasper Souren, rob, Michiel Buddingh',
+Kevin, Rob Avina, Alon Levy, Vikash, Michael Alan Dorman, Harley Pig,
+Andreas Olsson, Pietpiet, Christine Spang, Liz Young, Oleg Kosorukov,
+Allard Hoeve, Valentin Haenel, Joost Baaij, Nathan Yergler, Nathan Howell,
+Frédéric Schütz, Matti Eskelinen, Neil McGovern, Lane Lillquist, db48x,
+Stuart Prescott, Mark Matienzo, KarlTheGood, leonm, Drew Slininger, 
+Andreas Fuchs, Conrad Parker, Johannes Engelke, Battlegarden, Justin Kelly,
+Robin Wagner, Thad Ward, crenquis, Trudy Goold, Mike Cochrane, Adam Venturella,
+Russell Foo, furankupan, Giorgio Occhioni, andy, mind, Mike Linksvayer,
+Stefan Strahl, Jelmer Vernooij, Markus Fix, David Hicks, Justin Azoff,
+Iain Nicol, Bob Ippolito, Thomas Lundstrøm, Jason Mandel, federico2,
+Edd Cochran, Jose Ortega, Emmett Agnew, Rudy Garcia, Kodi, Nick Rusnov,
+Michael Rubin, Tom de Grunt, Richard Murray, Peter, Suzanne Pierce, Jared
+Marcotte, folk, Eamon, Jeff Richards, Leo Sutedja, dann frazier, Mikkel
+kristiansen, Matt Thomas, Kilian Evang, Gergely Risko, Kristian Rumberg,
+Peter Kropf, Mark Hepburn, greymont, D. Joe Anderson, Jeremy Zunker, ctebo,
+Manuel Roumain, Jason Walsh, np, Shawn, Johan Tibell, Branden Tyree, Dinyar
+Rabady, Andrew Mason, damond armstead, Ethan Aubin, TomTom Tommie, Jimmy
+Kaplowitz, Steven Zakulec, mike smith, Jacob Kirkwood, Mark Hymers, Nathan
+Collins, Asbjørn Sloth Tønnesen, Misty De Meo, James Shubin,
+Jim Paris, Adam Sjøgren, miniBill, Taneli, Kumar Appaiah, Greg Grossmeier,
+Sten Turpin, Otavio Salvador, Teemu Hukkanen, Brian Stengaard, bob walker,
+bibeneus, andrelo, Yaroslav Halchenko, hesfalling, Tommy L, jlargentaye,
+Serafeim Zanikolas, Don Armstrong, Chris Cormack, shayne.oneill, Radu
+Raduta, Josh S, Robin Sheat, Henrik Mygind, kodx, Christian, Geoff
+Crompton, Brian May, Olivier Berger, Filippo Gadotti, Daniel Curto-Millet,
+Eskild Hustvedt, Douglas Soares de Andrade, Tom L, Michael Nacos, Michaël
+P., William Roe, Joshua Honeycutt, Brian Kelly, Nathan Rasch, jorge, Martin
+Galese, alex cox, Avery Brooks, David Whittington, Dan Martinez, Forrest
+Sutton, Jouni K. Seppänen, Arnold Cano, Robert Beaty, Daniel, Kevin Savetz,
+Randy, Ernie Braganza, Aaron Haviland, Brian Brunswick, asmw, sean, Michael
+Williams, Alexander, Dougal Campbell, Robert Bacchas, Michael Lewis, Collin
+Price, Wes Frazier, Matt Wall, Brandon Barclay, Derek van Vliet, Martin
+DeMello, kitt hodsden, Stephen Kitt, Leif Bergman, Simon Lilburn, Michael
+Prokop, Christiaan Conover, Nick Coombe, Tim Dysinger, Brandon Robinson,
+Philip Newborough, keith, Mike Fullerton, Kyle, Phil Windley, Tyler Head,
+George V. Reilly, Matthew, Ali Gündüz, Vasyl Diakonov, Paolo Capriotti,
+allanfranta, Martin Haeberli, msingle, Vincent Sanders, Steven King, Dmitry
+Gribanov, Brandon High, Ben Hughes, Mike Dank, JohnE, Diggory Hardy,
+Michael Hanke, valhalla, Samuli Tuomola, Jeff Rau, Benjamin Lebsanft, John
+Drago, James, Aidan Cooper, rondie, Paul Kohler, Matthew Knights, Aaron
+Junod, Patrick R McDonald, Christopher Browne, Daniel Engel, John SJ
+Anderson, Peter Sarossy, Mike Prasad, Christoph Ender, Jan Dittberner,
+Zohar, Alexander Jelinek, stefan, Danny O'Brien, Matthew Thode, Nicole
+Aptekar, maurice gaston, Chris Adams, Mike Klemencic, Reedy, Subito, Tobias
+Gruetzmacher, Ole-Morten Duesund, André Koot, mp, gdop, Cole Scott, Blaker,
+Matt Sottile, W. Craig Trader, Louis-Philippe Dubrule, Brian Boucheron,
+Duncan Smith, Brenton Buchbach, Kyle Stevenson, Eliot Lash, Egon Elbre,
+Praveen, williamji, Thomas Schreiber, Neil Ford, Ryan Pratt, Joshua Brand,
+Peter Cox, Markus Engstrom, John Sutherland, Dean Bailey, Ed Summers,
+Hillel Arnold, David Fiander, Kurt Yoder, Trevor Muñoz, keri, Ivan
+Sergeyenko, Shad Bolling, Tal Kelrich, Steve Presley, gerald ocker, Essex
+Taylor, Josh Thomson, Trevor Bramble, Lance Higgins, Frank Motta, Dirk
+Kraft, soundray, Joe Haskins, nmjk, Apurva Desai, Colin Dean, docwhat,
+Joseph Costello, Jst, flamsmark, Alex Lang, Bill Traynor, Anthony David,
+Marc-André Lureau, AlNapp, Giovanni Moretti, John Lawrence, João Paulo
+Pizani Flor, Jim Ray, Gregory Thrush, Alistair McGann, Andrew Wied,
+Koutarou Furukawa, Xiscu Ignacio, Aaron Sachs, Matt, Quirijn, Chet
+Williams, Chris Gray, Bruce Segal, Tom Conder, Louis Tovar, Alex Duryee,
+booltox, d8uv, Decklin Foster, Rafael Cervantes, Micah R Ledbetter, Kevin
+Sjöberg, Johan Strombom, Zachary Cohen, Jason Lewis, Yves Bilgeri, Ville
+Aine, Mark Hurley, Marco Bonetti, Maximilian Haack, Hynek Schlawack,
+Michael Leinartas, Andreas Liebschner, Duotrig, Nat Fairbanks, David
+Deutsch, Colin Hayhurst, calca, Kyle Goodrick, Marc Bobillier, Robert
+Snook, James Kim, Olivier Serres, Jon Redfern, Itai Tavor, Michael
+Fladischer, Rob, Jan Schmid, Thomas H., Anders Söderbäck, Abhishek
+Dasgupta, Jeff Goeke-Smith, Tommy Thorn, bonuswavepilot, Philipp Edelmann,
+Nick, Alejandro Navarro Fulleda, Yann Lallemant, andrew brennan, 
+Dave Allen Barker Jr, Fabian, Lukas Anzinger, Carl Witty, Andy Taylor,
+Andre Klärner, Andrew Chilton, Adam Gibbins, Alexander Wauck, Shane O'Dea,
+Paul Waite, Iain McLaren, Maggie Ellen Robin Hess, Willard Korfhage,
+Nicolas, Eric Nikolaisen, Magnus Enger, Philipp Kern, Andrew Alderwick,
+Raphael Wimmer, Benjamin Schötz, Ana Guerrero, Pete, Pieter van der Eems,
+Aaron Suggs, Fred Benenson, Cedric Howe, Lance Ivy, Tieg Zaharia, Kevin
+Cooney, Jon Williams, Anton Kudris, Roman Komarov, Brad Hilton, Rick Dakan,
+Adam Whitcomb, Paul Casagrande, Evgueni Baldin, Robert Sanders, Kagan
+Kayal, Dean Gardiner, micah altman, Cameron Banga, Ross Mcnairn, Oscar
+Vilaplana, Robin Graham, Dan Gervais, Jon Åslund, Ragan Webber, Noble Hays,
+stephen brown, Sean True, Maciek Swiech, faser, eikenberry, Kai Laborenz,
+Sergey Fedoseev, Chris Fournier, Svend Sorensen, Greg K, wojtek, Johan
+Ribenfors, Anton, Benjamin, Oleg Tsarev, PsychoHazard, John Cochrane,
+Kasper Lauritzen, Patrick Naish, Rob, Keith Nasman, zenmaster, David Royer,
+Max Woolf, Dan Gabber, martin rhoads, Martin Schmidinger, Paul
+Scott-Wilson, Tom Gromak, Andy Webster, Dale Webb, Jim Watson, Stephen
+Hansen, Mircea, Dan Goodenberger, Matthias Fink, Andy Gott, Daniel, Jai
+Nelson, Shrayas Rajagopal, Vladimir Rutsky, Alexander, Thorben Westerhuys,
+hiruki, Tao Neuendorffer Flaherty, Elline, Marco Hegenberg, robert, Balda,
+Brennen Bearnes, Richard Parkins, David Gwilliam, Mark Johnson, Jeff Eaton,
+Reddawn90, Heather Pusey, Chris Heinz, Colin, Phatsaphong Thadabusapa,
+valunthar, Michael Martinez, redlukas, Yury V. Zaytsev, Blake, Tobias
+"betabrain" A., Leon, sopyer, Steve Burnett, bessarabov, sarble, krsch.com,
+Jack Self, Jeff Welch, Sam Pettiford, Jimmy Stridh, Diego Barberá, David
+Steele, Oscar Ciudad, John Braman, Jacob, Nick Jenkins, Ben Sullivan, Brian
+Cleary, James Brosnahan, Darryn Ten, Alex Brem, Jonathan Hitchcock, Jan
+Schmidle, Wolfrzx99, Steve Pomeroy, Matthew Sitton, Finkregh, Derek Reeve,
+GDR!, Cory Chapman, Marc Olivier Chouinard, Andreas Ryland, Justin, Andreas
+Persenius, Games That Teach, Walter Somerville, Bill Haecker, Brandon
+Phenix, Justin Shultz, Colin Scroggins, Tim Goddard, Ben Margolin, Michael
+Martinez, David Hobbs, Andre Le, Jason Roberts, Bob Lopez, Gert Van Gool,
+Robert Carnel, Anders Lundqvist, Aniruddha Sathaye, Marco Gillies, Basti
+von Bejga, Esko Arajärvi, Dominik Deobald, Pavel Yudaev, Fionn Behrens,
+Davide Favargiotti, Perttu Luukko, Silvan Jegen, Marcelo Bittencourt,
+Leonard Peris, smercer, Alexandre Dupas, Solomon Matthews, Peter Hogg,
+Richard E. Varian, Ian Oswald, James W. Sarvey, Ed Grether, Frederic
+Savard, Sebastian Nerz, Hans-Chr. Jehg, Matija Nalis, Josh DiMauro, Jason
+Harris, Adam Byrtek, Tellef, Magnus, Bart Schuurmans, Giel van Schijndel,
+Ryan, kiodos, Richard 'maddog' Collins, PawZubr, Jason Gassel, Alex
+Boisvert, Richard Thompson, maddi bolton, csights, Aaron Bryson, Jason Chu,
+Maxime Côté, Kineteka Systems, Joe Cabezas, Mike Czepiel, Rami Nasra,
+Christian Simonsen, Wouter Beugelsdijk, Adam Gibson, Gal Buki, James
+Marble, Alan Chambers, Bernd Wiehle, Simon Korzun, Daniel Glassey, Eero af
+Heurlin, Mikael, Timo Engelhardt, Wesley Faulkner, Jay Wo, Mike Belle,
+David Fowlkes Jr., Karl-Heinz Strass, Ed Mullins, Sam Flint,
+Hendrica, Mark Emer Anderson, Joshua Cole, Jan Gondol, Henrik Lindhe,
+Albert Delgado, Patrick, Alexa Avellar, Chris, sebsn1349, Maxim Kachur,
+Andrew Marshall, Navjot Narula, Alwin Mathew, Christian Mangelsdorf, Avi
+Shevin, Kevin S., Guillermo Sanchez Estrada, Alex Krieger, Luca Meier, Will
+Jessop, Nick Ruest, Lani Aung, Ulf Benjaminsson, Rudi Engelbrecht, Miles
+Matton, Cpt_Threepwood, Adam Kuyper, reacocard, David Kilsheimer, Peter
+Olson, Bill Fischofer, Prashant Shah, Simon Bonnick, Alexander Grudtsov,
+Antoine Boegli, Richard Warren, Sebastian Rust, AlmostHuman, Timmy
+Crawford, PC, Marek Belski, pontus, Douglas S Butts, Eric Wallmander, Joe
+Pelar, VIjay, Trahloc, Vernon Vantucci, Matthew baya, Viktor Štujber,
+Stephen Akiki, Daniil Zamoldinov, Atley, Chris Thomson, Jacob Briggs, Falko
+Richter, Andy Schmitz, Sergi Sagas, Peder Refsnes, Jonatan, Ben, Bill
+Niblock, Agustin Acuna, Jeff Curl, Tim Humphrey, bib, James Zarbock,
+Lachlan Devantier, Michal Berg, Jeff Lucas, Sid Irish, Franklyn, Jared
+Dickson, Olli Jarva, Adam Gibson, Lukas Loesche, Jukka Määttä, Alexander
+Lin, Dao Tran, Kirk, briankb, Ryan Villasenor, Daniel Wong, barista, Tomas
+Jungwirth, Jesper Hansen, Nivin Singh, Alessandro Tieghi, Billy Roessler,
+Peter Fetterer, Pallav Laskar, jcherney, Tyler Wang, Steve, Gigahost, Beat
+Wolf, Hannibal Skorepa, aktiveradio, Mark Nunnikhoven, Bret Comnes, Alan
+Ruttenberg, Anthony DiSanti, Adam Warkiewicz, Brian Bowman, Jonathan, Mark
+Filley, Tobias Mohr, Christian St. Cyr, j. faceless user, Karl Miller,
+Thomas Taimre, Vikram, Jason Mountcastle, Jason, Paul Elliott, Alexander,
+Stephen Farmer, rayslava, Peter Leurs, Sky Kruse, JP Reeves, John J Schatz,
+Martin Sandmair, Will Thompson, John Hergenroeder, Thomas, Christophe
+Ponsart, Wolfdog, Eagertolearn, LukasM, Federico Hernandez, Vincent Bernat,
+Christian Schmidt, Cameron Colby Thomson, Josh Duff, James Brown, Theron
+Trowbridge, Falke, Don Meares, tauu, Greg Cornford, Max Fenton, Kenneth
+Reitz, Bruce Bensetler, Mark Booth, Herb Mann, Sindre Sorhus, Chris
+Knadler, Daniel Digerås, Derek, Sin Valentine, Ben Gamari, david
+lampenscherf, fardles, Richard Burdeniuk, Tobias Kienzler, Dawid Humbla,
+Bruno Barbaroxa, D Malt, krivar, James Valleroy, Peter, Tim Geddings,
+Matthias Holzinger, Hanen, Petr Vacek, Raymond, Griff Maloney, Andreas
+Helveg Rudolph, Nelson Blaha, Colonel Fubar, Skyjacker Captain Gavin
+Phoenix, shaun, Michael, Kari Salminen, Rodrigo Miranda, Alan Chan, Justin
+Eugene Evans, Isaac, Ben Staffin, Matthew Loar, Magos, Roderik, Eugenio
+Piasini, Nico B, Scott Walter, Lior Amsalem, Thongrop Rodsavas, Alberto de
+Paola, Shawn Poulen, John Swiderski, lluks, Waelen, Mark Slosarek, Jim
+Cristol, mikesol, Bilal Quadri, LuP, Allan Nicolson, Kevin Washington,
+Isaac Wedin, Paul Anguiano, ldacruz, Jason Manheim, Sawyer, Jason
+Woofenden, Joe Danziger, Declan Morahan, KaptainUfolog, Vladron, bart, Jeff
+McNeill, Christian Schlotter, Ben McQuillan, Anthony, Julian, Martin O,
+altruism, Eric Solheim, MarkS, ndrwc, Matthew, David Lehn, Matthew
+Cisneros, Mike Skoglund, Kristy Carey, fmotta, Tom Lowenthal, Branden
+Tyree, Aaron Whitehouse
+
+### Also thanks to
+
+* The Kickstarter team, who have unleashed much good on the world.
+* The Haskell developers, who toiled for 20 years in obscurity
+  before most of us noticed them, and on whose giant shoulders I now stand,
+  in awe of the view.
+* The Git developers, for obvious reasons.
+* All of git-annex's early adopters, who turned it from a personal
+  toy project into something much more, and showed me the interest was there.
+* Rsync.net, for providing me a free account so I can make sure git-annex
+  works well with it.
+* LeastAuthority.com, for providing me a free Tahoe-LAFS grid account,
+  so I can test git-annex with that.
+* Yury V. Zaytsev for running the Windows autobuilder.
+* Kevin McKenzie for providing a OSX account for testing.
+* Anna and Mark, for the loan of the video camera; as well as the rest of
+  my family, for your support. Even when I couldn't explain what I was
+  working on.
+* And Mom, for stamping and stuffing so many thank you envelopes, and all the
+  rhubarb pies.
diff --git a/doc/thanks/list b/doc/thanks/list
new file mode 100644
index 0000000000..0339544ffb
--- /dev/null
+++ b/doc/thanks/list
@@ -0,0 +1,105 @@
+martin f. krafft, 
+John Byrnes, 
+Aurélien Couderc, 
+Jeffrey Chiu, 
+Amitai Schlair, 
+Andreas, 
+Anthony DeRobertis, 
+Baldur Kristinsson, 
+Boyd Stephen Smith Jr., 
+Brock Spratlen, 
+Christian Diller, 
+Denis Dzyubenko, 
+Eskild Hustvedt, 
+Evgeni Kunev, 
+FBC, 
+Fernando Jimenez, 
+Greg Young, 
+Henrik RIomar, 
+Ignacio, 
+Jake Vosloo, 
+James Valleroy, 
+Jason Woofenden, 
+Jeff Goeke-Smith, 
+Jim, 
+Jochen Bartl, 
+Johannes Schlatow, 
+John Carr, 
+Josh Taylor, 
+Josh Triplett, 
+Kuno Woudt, 
+Matthias Urlichs, 
+Mattias J, 
+Nathan Howell, 
+Nick Daly, 
+Nicolas Schodet, 
+Ole-Morten Duesund, 
+Remy van Elst, 
+RémiV, 
+Thom May, 
+Thomas Ferris Nicolaisen, 
+Thomas Hochstein, 
+Tyler Cipriani, 
+encryptio, 
+Øyvind A. Holm, 
+Bruno BEAUFILS, 
+Rémi Vanicat, 
+Trenton Cronholm, 
+Francois Marier, 
+Peter Hogg, 
+Amitai Schleier, 
+Brennen Bearnes, 
+Tim Howes, 
+Willard Korfhage, 
+Shane-o, 
+Michal Sojka, 
+BonusWavePilot, 
+William Hay, 
+Jack Hill, 
+Alexander Thompson, 
+Jeff Moe, 
+Michal Politowski, 
+Michael Tolly, 
+Daniel Takamori, 
+Chris Lamb, 
+Anton Grensjö, 
+Stan Yamane, 
+John Peloquin, 
+Thomas Schwinge, 
+Lars Wallenborn, 
+tj, 
+Carlos Pita, 
+Lee Hinman, 
+Lukas Platz, 
+Sergey Karpukhin, 
+Silvio Ankermann, 
+Paul Tötterman, 
+Erik Bjäreholt, 
+André Pereira, 
+Kanak Kshetri, 
+Josh, 
+tdittr, 
+Peter, 
+Nicolas Pouillard, 
+Johannes Grødem, 
+Calvin Beck, 
+mo, 
+Ewen McNeill, 
+Markus Hauru, 
+Christopher Goes, 
+Michael Magin, 
+Andreas Karlsson, 
+andrea rota, 
+Riku Voipio, 
+Ethan Aubin, 
+Pedro Luz, 
+John Pellman, 
+Duncan Holm, 
+Eric Drechsel, 
+Eric Prestemon, 
+Diederik de Haas, 
+Nick Piper, 
+Ryan Newton, 
+Brett Eisenberg, 
+paul walmsley, 
+John Lee, 
diff --git a/doc/tips.mdwn b/doc/tips.mdwn
new file mode 100644
index 0000000000..eda84c8672
--- /dev/null
+++ b/doc/tips.mdwn
@@ -0,0 +1,4 @@
+This page is a place to document tips and techniques for using git-annex.
+
+[[!inline pages="tips/* and !tips/*/*" archive="yes"
+rootpage="tips" postformtext="Add a new tip about:" show=0]]
diff --git a/doc/tips/Accessing_files_in_bare_remotes_without_git-annex.mdwn b/doc/tips/Accessing_files_in_bare_remotes_without_git-annex.mdwn
new file mode 100644
index 0000000000..c9af79cdfd
--- /dev/null
+++ b/doc/tips/Accessing_files_in_bare_remotes_without_git-annex.mdwn
@@ -0,0 +1,8 @@
+
+git-annex is amazing for my large files backup necessities, but I am a bit scared for the long-term possibility of accessing data, without git-annex.
+
+For this reason I have prepared a small Python tool that accesses the content of bare git-annex repositories. The tool retrieves the locations of a file, and the path to the file in the current annex. This works with the last v6 version of the git-annex format.
+
+This is motivated by the fact that in the master branch the files are stores as links to .git/annex/objects/XX/YY/KEY/KEY (as it is correct for non-bare repos), while in the bare the files are stored in annex/objecst/ZZ/WW/KEY/KEY.
+
+The source code is available here: 
diff --git a/doc/tips/Accessing_files_in_bare_remotes_without_git-annex/comment_1_9243af1c87fdaf33e336ec34e99c0f74._comment b/doc/tips/Accessing_files_in_bare_remotes_without_git-annex/comment_1_9243af1c87fdaf33e336ec34e99c0f74._comment
new file mode 100644
index 0000000000..4843f702a1
--- /dev/null
+++ b/doc/tips/Accessing_files_in_bare_remotes_without_git-annex/comment_1_9243af1c87fdaf33e336ec34e99c0f74._comment
@@ -0,0 +1,9 @@
+[[!comment format=mdwn
+ username="emanuele.ruffaldi@56b9c9a5c1f873994b869248e9b53aa20f437d20"
+ nickname="emanuele.ruffaldi"
+ subject="Updated"
+ date="2016-01-12T22:47:49Z"
+ content="""
+- support for non-bare repositories
+- option for creating a tar file containing all the master checked out with the symbolic links to the content files. This is mountable with the archivemount tool
+"""]]
diff --git a/doc/tips/Announcing_recastex_-___40__re__41__podcast__from_your_annex.mdwn b/doc/tips/Announcing_recastex_-___40__re__41__podcast__from_your_annex.mdwn
new file mode 100644
index 0000000000..9e6d7654f5
--- /dev/null
+++ b/doc/tips/Announcing_recastex_-___40__re__41__podcast__from_your_annex.mdwn
@@ -0,0 +1,15 @@
+Hi all,
+
+I've written a simple tool in Python to re-podcast from an annex:  [recastex](https://github.com/stewart123579/recastex)
+
+Starting with the [downloading podcasts](https://git-annex.branchable.com/tips/downloading_podcasts/) page, I've got a number of podcasts on my laptop, but they were not really synced to my podcast app on my phone.  Not a problem any longer.
+
+The app uses the metadata associated with *locally available* files to generate feeds for each of your "subscribed" podcasts - and collects anything else you have (like individual files) into a catch-all feed.
+
+It's designed with git-annex + limited network + privacy in mind separating the public internet queries from the things that can be done over git-annex.
+
+*(As the author of [git-annex-metadata-gui](https://git-annex.branchable.com/tips/a_gui_for_metadata_operations/) said...)*  I hope these can be useful to someone other than myself.
+
+
+
+
diff --git a/doc/tips/Building_git-annex_on_Debian_OR___37____164____35____34____164____37____38____34____35___Haskell__33__.mdwn b/doc/tips/Building_git-annex_on_Debian_OR___37____164____35____34____164____37____38____34____35___Haskell__33__.mdwn
new file mode 100644
index 0000000000..c32eb966f7
--- /dev/null
+++ b/doc/tips/Building_git-annex_on_Debian_OR___37____164____35____34____164____37____38____34____35___Haskell__33__.mdwn
@@ -0,0 +1,111 @@
+I've been wrestling with git-annex to try to make it build on Debian, or more specifically, wrestling with Haskell dependencies.
+
+After a fair amount of futzing around, and pestering a bunch of people in the process (thanks for the help! :) ) I finally managed to make it build.
+
+I figured I would post the steps here, since it's not completely trivial, and I expect that a few others might be interested in building newer versions as well.
+
+There appears to currently be two methods:
+
+* Debian packages on Wheezy plus Sid
+  * Starting out on Wheezy, and then picking the rest from Sid (it seems at least libghc-safesemaphore-dev from Sid is critical for newer git-annex)
+  * WebDAV suport will not be available with this method
+* Cabal packages
+
+
+#Debian packages on Wheezy plus Sid
+
+##Start off with a clean wheezy chroot
+
+    sudo debootstrap wheezy debian-wheezy
+    sudo chroot debian-wheezy
+
+##Install some build tools
+
+    apt-get update
+    apt-get install devscripts git
+
+##Get git-annex (either by cloning or simply moving the source into the chroot)
+
+    mkdir /src
+    cd /src
+    git clone git://git-annex.branchable.com/source.git git-annex
+    cd git-annex
+
+##Remove WebDAV dependency which can't be satisfied anywhere
+
+    sed '/libghc-dav-dev/d' -i debian/control
+
+##Create dummy build-depends package and install all available Wheezy dependencies using it
+
+    mk-build-deps
+    dpkg -i git-annex-build-deps*.deb
+    apt-get install -f
+
+(this will remove the build-depends package)
+
+##Add Sid sources and install all available Sid dependencies
+
+    echo "deb http://http.debian.net/debian sid main" >>/etc/apt/sources.list
+    apt-get update
+    dpkg -i git-annex-build-deps*.deb
+    apt-get install -f
+
+(the build-depends package should now be fully installed)
+
+##Disable the 'make test' that fails due to missing hothasktags
+
+    echo >>debian/rules
+    echo "override_dh_auto_test:" >>debian/rules
+
+##Build!
+
+    debuild -us -uc -Igit
+
+
+#Cabal packages
+
+##Start off with a clean Sid(/Wheezy) chroot
+
+    sudo debootstrap sid debian-sid
+    sudo chroot debian-sid
+
+##Install a smaller set of tools and build-depends from Debian (cabal needs these to compile the Haskell stuff)
+
+    apt-get update
+    apt-get install ghc cabal-install devscripts libz-dev pkg-config c2hs libgsasl7-dev libxml2-dev libgnutls-dev c2hs git debhelper ikiwiki perlmagick uuid rsync openssh-client fakeroot
+
+##Get git-annex (either by cloning or simply moving the source into the chroot)
+
+    mkdir /src
+    cd /src
+    git clone git://git-annex.branchable.com/source.git git-annex
+    cd git-annex
+
+##Install the Haskell build-dependencies from cabal
+
+    cabal update
+    cabal install --only-dependencies
+
+##Optional step which doesn't work (might in the future)
+If we want to run the 'make test' after build we need hothasktags, which is only available via cabal
+
+    apt-get install happy
+    cabal install hothasktags
+    export PATH=$PATH:~/.cabal/bin
+
+But this currently fails silently inside make test->fast->tags, and if you dig a bit (manually edit the makefile to be more verbose) you see
+
+    hothasktags: ./Command/AddUnused.hs: hGetContents: invalid argument (invalid byte sequence)
+
+##Disable the 'make test' that fails
+
+    echo >>debian/rules
+    echo "override_dh_auto_test:" >>debian/rules
+
+##Remove all Debian package haskell depends (taken care of by cabal instead)
+
+    sed '/\tlibghc/d' -i debian/control
+
+## Build!
+
+    debuild -us -uc -Igit
diff --git a/doc/tips/Building_git-annex_on_Debian_OR___37____164____35____34____164____37____38____34____35___Haskell__33__/comment_1_835a3608df3e9d044cabe822d0f3e7e4._comment b/doc/tips/Building_git-annex_on_Debian_OR___37____164____35____34____164____37____38____34____35___Haskell__33__/comment_1_835a3608df3e9d044cabe822d0f3e7e4._comment
new file mode 100644
index 0000000000..55cf0b97b9
--- /dev/null
+++ b/doc/tips/Building_git-annex_on_Debian_OR___37____164____35____34____164____37____38____34____35___Haskell__33__/comment_1_835a3608df3e9d044cabe822d0f3e7e4._comment
@@ -0,0 +1,27 @@
+[[!comment format=mdwn
+ username="https://www.google.com/accounts/o8/id?id=AItOawkCw26IdxXXPBoLcZsQFslM67OJSJynb1w"
+ nickname="Alexander"
+ subject="can't install git-annex on OS X Mountain Lion without disabling WebDAV support"
+ date="2013-04-29T17:57:03Z"
+ content="""
+possibly related to this Debian issue:
+
+trying to install git-annex with cabal on OS X 10.8.3, the build fails with
+
+
+    Loading package DAV-0.4 ... linking ... ghc: 
+    lookupSymbol failed in relocateSection (relocate external)
+    ~/.cabal/lib/DAV-0.4/ghc-7.4.2/HSDAV-0.4.o: unknown symbol `_DAVzm0zi4_PathszuDAV_version1_closure'
+    ghc: unable to load package `DAV-0.4'
+    Failed to install git-annex-4.20130417
+    cabal: Error: some packages failed to install:
+    git-annex-4.20130417 failed during the building phase. The exception was:
+    ExitFailure 1
+
+
+This was after following all of the instructions for the Homebrew install at [http://git-annex.branchable.com/install/OSX/](http://git-annex.branchable.com/install/OSX/)
+I was able to work around this issue by installing with the WebDAV flag disabled (ie, added the option --flags=\"-WebDAV\" to last command in the OS X install instructions):
+
+    cabal install git-annex --bindir=$HOME/bin --flags=\"-WebDAV\"
+
+"""]]
diff --git a/doc/tips/Building_git-annex_on_Debian_OR___37____164____35____34____164____37____38____34____35___Haskell__33__/comment_2_080b30cba72a718e73ea715e259e1cfb._comment b/doc/tips/Building_git-annex_on_Debian_OR___37____164____35____34____164____37____38____34____35___Haskell__33__/comment_2_080b30cba72a718e73ea715e259e1cfb._comment
new file mode 100644
index 0000000000..7e8e295fdc
--- /dev/null
+++ b/doc/tips/Building_git-annex_on_Debian_OR___37____164____35____34____164____37____38____34____35___Haskell__33__/comment_2_080b30cba72a718e73ea715e259e1cfb._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="http://joeyh.name/"
+ nickname="joey"
+ subject="comment 2"
+ date="2013-04-30T21:51:50Z"
+ content="""
+@Alexander that DAV-0.4 problem is a bug in DAV, not git-annex. I've informed its author and it should be fixed soon, in a new version of DAV.
+"""]]
diff --git a/doc/tips/Bup_repositories_in_git-annex.mdwn b/doc/tips/Bup_repositories_in_git-annex.mdwn
new file mode 100644
index 0000000000..542dc2f587
--- /dev/null
+++ b/doc/tips/Bup_repositories_in_git-annex.mdwn
@@ -0,0 +1,51 @@
+I'd like to share my setup for keeping [bup](https://github.com/bup/bup/) repositories in git-annex.¹
+I'm not sure if this is a *good* tip, so comments are welcome.
+
+The purpose of this setup is to (kind of) bring encryption to bup,
+and make it easy to keep bup backups in untrusted storage by making use of the encryption modes and backends provided by git-annex.
+This approach can be used to make encrypted *backups of bup repositories*;
+it can not replace encrypted filesystems such as EncFS or S3QL
+which wouldn't necessarily require local bup repositories but also can't be combined with storage like Amazon Glacier.
+
+To add a bup repository to git-annex, initialize a regular indirect git-annex repository,
+and make the bup repository a subdirectory of it.²
+Then `git annex add $BUP_REPO/objects/packs`, i.e. the location of the large data files (.pack & .par2).
+The rest of the bup repository should be tracked by Git (`git add $BUP_REPO`).³
+This way the repository stays fully functional.
+
+After a bup-save the following steps will synchronize all remotes:⁴
+
+    git annex add $BUP_REPO/objects/pack
+    git add $BUP_REPO
+    git commit -m "Backup on $(date)"
+	git annex sync --content
+
+In my current setup, the git-annex repositories are located on a local file server.
+Various clients use bup to create backups on the server.
+This server also makes backups of other servers.
+Afterwards, it uploads the annexed data to Glacier
+(via an [encrypted S3 special remote](/special_remotes/S3/)),
+and pushes the small Git repositories to an S3QL filesystem and another off-site server.
+Using these repositories (and my GPG key) the bup repositories could be recovered.
+
+It may be important to note that in order to be able to *access* a bup repository,
+*all* files have to be available locally.
+Bup will not function if any pack files are missing (maybe this can be improved?).
+
+-----
+
+¹) Not to be confused with git-annex's [bup special remote](/special_remotes/bup/).
+
+²) You can't initialize git-annex repositories directly inside bup repositories
+because git-annex will (rightfully) identify them as bare git repositories and set itself up accordingly.
+
+³) I've come up with these .gitignore rules to exclude potentially large files not needed for recovery:
+
+    /bup_repo/bupindex*
+    /bup_repo/objects/pack/bup.bloom
+    /bup_repo/objects/pack/midx*midx
+    /bup_repo/objects/tmp*.pack
+    /bup_repo/index-cache/
+
+⁴) `git annex sync` might not be the safest command to use because it would merge changes from the remotes.
+However, assuming normal bup usage, external changes to the bup repository are not to be expected.
diff --git a/doc/tips/Bup_repositories_in_git-annex/comment_1_738959a699755704baab6df778ecd120._comment b/doc/tips/Bup_repositories_in_git-annex/comment_1_738959a699755704baab6df778ecd120._comment
new file mode 100644
index 0000000000..5b9566c3ae
--- /dev/null
+++ b/doc/tips/Bup_repositories_in_git-annex/comment_1_738959a699755704baab6df778ecd120._comment
@@ -0,0 +1,7 @@
+[[!comment format=mdwn
+ username="sfowijowa"
+ subject="bup.bloom must be unlocked"
+ date="2015-07-17T17:45:24Z"
+ content="""
+Hi, Thanks for the help :) I didn't exlude any files. Therefore \"$BUP_REPO/objects/pack/bup.bloom\" gets added by git-annex. This file has to be unlocked beforehand for bup to be running correctly. Best regards
+"""]]
diff --git a/doc/tips/Crude_Windows_Sync.mdwn b/doc/tips/Crude_Windows_Sync.mdwn
new file mode 100644
index 0000000000..88138a0b1d
--- /dev/null
+++ b/doc/tips/Crude_Windows_Sync.mdwn
@@ -0,0 +1,35 @@
+Here's a workaround to start syncing folders on Windows right now. It's a bit command line heavy, so you might need to set this up for your users. But I would much rather do this than use some other syncing solution and then have to migrate.
+
+(1) Create a remote server git annex repository with the assistant on Linux or Mac.
+
+(2) [Install git](http://git-scm.com/) on the Windows machine.
+
+(3) [Install git-annex for Windows](http://git-annex.branchable.com/install/Windows/) on the Windows machine. Don't forget to run the installer as administrator.
+
+(4) Run _Git Bash_ from the system menu, and run these commands to clone your repository.
+
+    ssh-keygen
+    cat .ssh/id_rsa.pub | ssh username@my-server.com "cat >> ~/.ssh/authorized_keys"
+    git clone username@my-server.com:/path/to/annex
+    cd annex
+    git annex init
+
+(5) Create a script that will trigger a full sync
+
+    echo '
+    #!/bin/bash
+    git annex sync
+    git annex get *
+    git annex add .
+    git annex sync
+    git annex copy * --to origin  
+    ' > sync.sh
+    chmod +x sync.sh
+    ./sync.sh
+
+(6) Copy the "Git Bash" shortcut from your windows menu to your desktop, and change the link target to:
+
+    C:\Program Files\Git\bin\sh.exe" --login -i "annex/sync.sh"
+
+Now ask your users to run this shortcut before and after they change files. You can also put it into the "autostart" folder to sync at boot.
+
diff --git a/doc/tips/Decentralized_repository_behind_a_Firewall.mdwn b/doc/tips/Decentralized_repository_behind_a_Firewall.mdwn
new file mode 100644
index 0000000000..9e347c73f7
--- /dev/null
+++ b/doc/tips/Decentralized_repository_behind_a_Firewall.mdwn
@@ -0,0 +1,59 @@
+If you're anything like me¹, you have a copy of your annex on a computer running at home², set up so you can access it from anywhere like this:
+
+    ssh myhome.no-ip.org
+
+This is totally great! Except, there is no way for your home computer to pull your changes, because there is no *on-the-go.no-ip.org*. You can get clunky and use a *bare git repository and git push*, but there is a better way.
+
+First, install *openssh-server* on your *on-the-go* computer
+
+    sudo apt-get install openssh-server # Adjust to your flavor of unix
+
+Then, log into your *home* computer, with *port forwarding*:
+
+    ssh me@myhome.no-ip.org -R 2201:localhost:22
+
+Your *home* computer can now ssh into your *on-the-go* computer, as long as you keep the above shell running.
+
+You can now add your *on-the-go* computer as a remote on your *home* computer. Use the port forwarding shell you just connected with the command above, if you like. 
+
+    ssh-keygen -t rsa
+    ssh-copy-id "me@localhost -p 2201"
+    cd ~/annex
+    git remote add on-the-go ssh://me@localhost:2201/home/myuser/annex
+
+Now you can run normal annex operations, as long as the port forwarding shell is running³.
+
+    git annex sync
+    git annex get on-the-go some/big/file
+    git annex info
+
+You can add more computers by repeating with a different port, e.g. 2202 or 2203 (or any other).
+
+If you're security paranoid (like me), read on. If you're not, that's it! Thanks for reading!
+
+---
+Paranoid Area
+
+Note you're granting passwordless access to your on-the-go computer to your home computer. I believe that's all right, as long as:
+
+* Your home computer is really in your home, and not at a friend's house or some datacenter
+* Your home computer can be accessed only by ssh, and not HTTP or Samba or NTP or (shoot me now!) FTP
+* Only you (and perhaps trustworthy family) have access to your home computer
+* You have reasonably strong passwords or key-only logins on both your home and on-the-go computers.
+* You regularly install security updates on both computers (sudo apt-get update && sudo apt-get upgrade)
+
+In any case, the setup is much, much, much more secure than Dropbox. With Dropbox, you have exactly the same setup, but:
+
+* Your data is stored in some datacenter. It's supposed to be encrypted. It might not be.
+* Lot's of people have routine access to your files, and plausible reason to. Bored employees might regularly be doing some 'maintenance work' involving your pictures.
+* The dropbox software can do anything it likes on your computer, and it's closed source so you don't know if it does. A disgruntled employee could put a trojan into it.
+* Dropbox might have a backdoor for employee access to any file on your computer. This might be done with the best of intentions, but a mal-intentioned or careless employee might still erase things or send sensitive files from your computer by email.
+* A truly huge amount of eyes connected to incredibly smart brains have looked at openssh and found it secure. Everybody trusts openssh. With dropbox, there is, well, dropbox. Whoever that is.
+
+-----
+
+¹ Me=Carlo, not Joey. I'm pretty sure doing what I wrote here is a good idea, but in case it turns out to be catastrophically dumb, it's my fault, not his.
+
+² My always-on computer at home is a raspberry pi with a 32GB USB stick. Best self-hosted dropbox you could imagine.
+
+³ You can just forward the port, but not open a shell, by adding the -N command. This could be useful for connecting on startup, e.g. in /etc/rc.local. I prefer to open the shell to forward the ports, maybe use it, and close it to stop it.
diff --git a/doc/tips/Decentralized_repository_behind_a_Firewall/comment_1_78b9035234a690ca5a7c9f3cc78fa092._comment b/doc/tips/Decentralized_repository_behind_a_Firewall/comment_1_78b9035234a690ca5a7c9f3cc78fa092._comment
new file mode 100644
index 0000000000..71a1db9c88
--- /dev/null
+++ b/doc/tips/Decentralized_repository_behind_a_Firewall/comment_1_78b9035234a690ca5a7c9f3cc78fa092._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="http://joeyh.name/"
+ ip="4.154.6.49"
+ subject="comment 1"
+ date="2012-11-30T16:25:58Z"
+ content="""
+If you don't trust your home computer with shell access, you can lock it down in `.ssh/authorized_keys` to only be able to run git-annex-shell. See [[forum/Restricting_git-annex-shell_to_a_specific_repository]]
+"""]]
diff --git a/doc/tips/Decrypting_files_in_special_remotes_without_git-annex.mdwn b/doc/tips/Decrypting_files_in_special_remotes_without_git-annex.mdwn
new file mode 100644
index 0000000000..3d2fd35a11
--- /dev/null
+++ b/doc/tips/Decrypting_files_in_special_remotes_without_git-annex.mdwn
@@ -0,0 +1,123 @@
+One of the selling points of `git-annex` is that it uses standard tools like `git` and `gpg` to deal with files, so that years from now it should be possible to explore and get useful data out of an old annex repository (this helps with [[future_proofing]]). If for whatever reason you need to decrypt files on [[special_remotes]] that use [[encryption]] without using `git-annex`, this can be done fairly easily using `gpg` (and `openssl` to compute the HMAC keys used to create the file names used on the special remote so you can look up the right file to decrypt). Here is an example script demonstrating how to compute the special remote file names and how to decrypt the special remote files.
+
+    #!/usr/bin/env bash
+    
+    usage() {
+    	echo "Usage: ga_decrypt.sh -r REMOTE [-k SYMLINK] [-d FILE]"
+    	echo ""
+    	echo "    Either lookups up key on REMOTE for annex file linked with SYMLINK"
+    	echo "    or decrypts FILE encrypted for REMOTE."
+    	echo ""
+    	echo "    -r: REMOTE is special remote to use"
+    	echo "    -k: SYMLINK is symlink in annex to print encrypted special remote key for"
+    	echo "    -d: FILE is path to special remote file to decrypt to STDOUT"
+    	echo ""
+    	echo "NOTES: "
+    	echo "    * Run in an indirect git annex repo."
+    	echo "    * Must specify -k or -d."
+    	echo "    * -k prints the key including the leading directory names used for a "
+    	echo "       directory remote (even if REMOTE is not a directory remote)"
+    	echo "    * -d works on a locally accessible file. It does not fetch a remote file"
+    	echo "    * Must have gpg and openssl"
+    }
+    
+    decrypt_cipher() {
+    	cipher="$1"
+    	echo "$(echo -n "$cipher" | base64 -d | gpg --decrypt --quiet)"
+    }
+    
+    lookup_key() {
+    	encryption="$1"
+    	cipher="$2"
+    	symlink="$3"
+    
+    	if [ "$encryption" == "hybrid" ] || [ "$encryption" == "pubkey" ]; then
+    		cipher="$(decrypt_cipher "$cipher")"
+    	fi
+    
+    	# Pull out MAC cipher from beginning of cipher
+    	if [ "$encryption" = "hybrid" ] ; then
+    		cipher="$(echo -n "$cipher" | head  -c 256 )"
+    	elif [ "$encryption" = "shared" ] ; then
+    		cipher="$(echo -n "$cipher" | base64 -d | tr -d '\n' | head  -c 256 )"
+    	elif [ "$encryption" = "pubkey" ] ; then
+    		# pubkey cipher includes a trailing newline which was stripped in
+    		# decrypt_cipher process substitution step above
+    		IFS= read -rd '' cipher < <( printf "$cipher\n" )
+    	fi
+    
+    	annex_key="$(basename "$(readlink "$symlink")")"
+    	hash="$(echo -n "$annex_key" | openssl dgst -sha1 -hmac "$cipher" | sed 's/(stdin)= //')"
+    	key="GPGHMACSHA1--$hash"
+    	checksum="$(echo -n $key | md5sum)"
+    	echo "${checksum:0:3}/${checksum:3:3}/$key"
+    }
+    
+    decrypt_file() {
+    	encryption="$1"
+    	cipher="$2"
+    	file_path="$3"
+    
+    	if [ "$encryption" = "pubkey" ] ; then
+    		gpg --quiet --decrypt "${file_path}"
+    	else
+    		if [ "$encryption" = "hybrid" ] ; then
+    			cipher="$(decrypt_cipher "$cipher" | tail -c +257)"
+    		elif [ "$encryption" = "shared" ] ; then
+    			cipher="$(echo -n "$cipher" | base64 -d | tr -d '\n' | tail  -c +257 )"
+    		fi
+    		gpg --quiet --batch --passphrase "$cipher" --output - "${file_path}"
+    	fi
+    }
+    
+    main() {
+    	OPTIND=1
+    
+    	mode=""
+    	remote=""
+    
+    	while getopts "r:k:d:" opt; do
+    		case "$opt" in
+    			r)  remote="$OPTARG"
+    				;;
+    			k)  if [ -z "$mode" ] ; then
+    					mode="lookup key"
+    				else
+    					usage
+    					exit 2
+    				fi
+    				symlink="$OPTARG"
+    				;;
+    			d)  if [ -z "$mode" ] ; then
+    					mode="decrypt file"
+    				else
+    					usage
+    					exit 2
+    				fi
+    				file_path="$OPTARG"
+    				;;
+    		esac
+    	done
+    
+    	if [ -z "$mode" ] || [ -z "$remote" ] ; then
+    		usage
+    		exit 2
+    	fi
+    
+        shift $((OPTIND-1))
+    
+    	# Pull out config for desired remote name
+    	remote_config="$(git show git-annex:remote.log | grep 'name='"$remote ")"
+    
+    	# Get encryption type and cipher from config
+    	encryption="$(echo "$remote_config" | grep -oP 'encryption\=.*? ' | tr -d ' \n' | sed 's/encryption=//')"
+    	cipher="$(echo "$remote_config" | grep -oP 'cipher\=.*? ' | tr -d ' \n' | sed 's/cipher=//')"
+    
+    	if [ "$mode" = "lookup key" ] ; then
+    		lookup_key "$encryption" "$cipher" "$symlink"
+    	elif [ "$mode" = "decrypt file" ] ; then
+    		decrypt_file "$encryption" "$cipher" "${file_path}"
+    	fi
+    }
+
+    main "$@"
diff --git a/doc/tips/Decrypting_files_in_special_remotes_without_git-annex/comment_1_9288805fae0710b19da6c82f9dc32f2a._comment b/doc/tips/Decrypting_files_in_special_remotes_without_git-annex/comment_1_9288805fae0710b19da6c82f9dc32f2a._comment
new file mode 100644
index 0000000000..d61c152740
--- /dev/null
+++ b/doc/tips/Decrypting_files_in_special_remotes_without_git-annex/comment_1_9288805fae0710b19da6c82f9dc32f2a._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="yibe"
+ avatar="http://cdn.libravatar.org/avatar/649c9e417b6c509a17f7b48caf065c91"
+ subject="comment 1"
+ date="2017-06-04T19:26:47Z"
+ content="""
+I have modified the script to add support for chunks and auto-detection of the MAC algorithm used: 
+"""]]
diff --git a/doc/tips/Delay_Assistant_Startup_on_Login.mdwn b/doc/tips/Delay_Assistant_Startup_on_Login.mdwn
new file mode 100644
index 0000000000..74652308ae
--- /dev/null
+++ b/doc/tips/Delay_Assistant_Startup_on_Login.mdwn
@@ -0,0 +1,13 @@
+# Problem
+I noticed that after installing git-annex assistant, my start up times greatly increased because the assistant does a startup scan while everything else is loading.
+# Solution (for people using Gnome)
+The solution I came up with is to delay the assistant's startup, as well as setting its IO priority as idle. To do this in Gnome 3, run:
+
+    gnome-session-properties
+Find the "Git Annex Assistant" entry in the Startup Programs tab, then click edit. Change this:
+
+    /usr/local/bin/git-annex assistant --autostart (your location of git-annex may be different)
+to this:
+
+    bash -c "sleep 30; ionice -c3 /usr/local/bin/git-annex assistant --autostart" (replace /usr/local/bin to wherever git-annex is installed)
+The "sleep 30" command delays the startup of the assistant by 30 seconds, and "ionice -c3" sets git-annex's IO priority to "idle," the lowest level.
diff --git a/doc/tips/Delay_Assistant_Startup_on_Login/comment_1_c63917150527efab4b1106183b3aa7ef._comment b/doc/tips/Delay_Assistant_Startup_on_Login/comment_1_c63917150527efab4b1106183b3aa7ef._comment
new file mode 100644
index 0000000000..fe8cb80ba3
--- /dev/null
+++ b/doc/tips/Delay_Assistant_Startup_on_Login/comment_1_c63917150527efab4b1106183b3aa7ef._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="https://launchpad.net/~alphapapa"
+ nickname="alphapapa"
+ subject="ionice not supported by deadline scheduler"
+ date="2013-06-28T17:43:47Z"
+ content="""
+Linux's deadline I/O scheduler does not support ionice.  It is now the default on some distros, including Ubuntu.  CFQ does support ionice.
+"""]]
diff --git a/doc/tips/Faster_bash_autocompletion_with_big_annex_repos.mdwn b/doc/tips/Faster_bash_autocompletion_with_big_annex_repos.mdwn
new file mode 100644
index 0000000000..61d49192ca
--- /dev/null
+++ b/doc/tips/Faster_bash_autocompletion_with_big_annex_repos.mdwn
@@ -0,0 +1,50 @@
+I'm currently using git annex to manage my entire file collection
+(including tons of music and books) and I noticed how slow
+autocompletion has become for files in the index (say for git add).
+The main offender is a while-read-case-echo bash loop in
+`__git_index_files` that can be readily substituted with a much faster
+sed invocation. Here is my benchmark:
+
+```
+__git_index_files ()
+{
+    local dir="$(__gitdir)" root="${2-.}" file;
+    if [ -d "$dir" ]; then
+        __git_ls_files_helper "$root" "$1" | while read -r file; do
+            case "$file" in
+                ?*/*)
+                    echo "${file%%/*}"
+                ;;
+                *)
+                    echo "$file"
+                ;;
+            esac;
+        done | sort | uniq;
+    fi
+}
+
+time __git_index_files > /dev/null
+
+real    0m0.830s
+user    0m0.597s
+sys    0m0.310s
+
+__git_index_files ()
+{
+    local dir="$(__gitdir)" root="${2-.}" file;
+    if [ -d "$dir" ]; then
+        __git_ls_files_helper "$root" "$1" | \
+            sed -r 's@/.*@@' | uniq | sort | uniq
+    fi
+}
+
+
+time __git_index_files > /dev/null
+
+real    0m0.075s
+user    0m0.083s
+sys    0m0.010s
+
+```
+
+10 times faster! So you might redefine `__git_index_files` as above in your .bashrc after sourcing the git autocomplete script. 
diff --git a/doc/tips/Faster_bash_autocompletion_with_big_annex_repos/comment_1_a41fb1037186319bbb44f7a67df6120f._comment b/doc/tips/Faster_bash_autocompletion_with_big_annex_repos/comment_1_a41fb1037186319bbb44f7a67df6120f._comment
new file mode 100644
index 0000000000..3a44fef020
--- /dev/null
+++ b/doc/tips/Faster_bash_autocompletion_with_big_annex_repos/comment_1_a41fb1037186319bbb44f7a67df6120f._comment
@@ -0,0 +1,7 @@
+[[!comment format=mdwn
+ username="joey"
+ subject="""comment 1"""
+ date="2017-05-11T17:29:47Z"
+ content="""
+Could you send this improvement to the git mailing list?
+"""]]
diff --git a/doc/tips/Git_annex_and_Calibre.mdwn b/doc/tips/Git_annex_and_Calibre.mdwn
new file mode 100644
index 0000000000..95d010611a
--- /dev/null
+++ b/doc/tips/Git_annex_and_Calibre.mdwn
@@ -0,0 +1,120 @@
+The problem
+===========
+
+[Calibre](http://calibre-ebook.com/) is a ebook manager that is
+available in [debian](http://packages.debian.org/sid/calibre). I use
+it to maintain my library, but also to dowload every day an epub
+version of a French newspaper and then put it on my kobo.
+
+Configuring git annex for this
+==============================
+
+I wanted to use git-annex, so
+
+    $ git init
+    $ git annex init "some useful name"
+
+But I don't want every thing in annex, because Calibre use some text
+file to save some metadata, so I used:
+
+    $ git config annex.largefiles "include=* exclude=*.opf exclude=*.json"
+
+then lets add everything
+
+    $ git annex add *
+    $ git add *
+    $ git commit -m "first commit"
+
+Calibre need read and write access on the its database, so let unlock it:
+
+    $ git annex unlock metadata.db
+
+On my other computer I only need to do
+
+    $ git clone $user@$host:Calibre\ library
+    $ cd Calibre\ library
+    $ git annex init "another useful name"
+    $ git annex get .
+    $ git annex unlock metadata.db
+
+The problem is that every time you will `git annex sync`, git annex
+will lock again the metadata.db, so lets unlock it automatically. I
+use git hooks, in `.git/hooks/post-commit` I have
+
+    #!/bin/bash
+
+    git annex edit metadata.db
+
+don't forget to make this file executable
+
+    $ chmod a+x .git/hooks/post-commit
+
+Day to day operation
+====================
+
+    $ git annex add .
+
+Will put new file into the annex
+
+    $ git add .
+
+Will take care of the files that should no go into annex
+
+    $ git annex sync
+
+Will make the repositories exchange informations about all this, and
+make remote change local
+
+    $ git annex get .
+
+Will make remote book locally available
+
+Merge conflict
+--------------
+You should not run calibre on the two computer simultaneously, or
+without syncing before it. If you do, you will have a conflict that
+git-annex will automatically *solve* by rename both of the file.
+
+You can then either:
+
+ - Choose one. If no books have been changed or added on one of the
+   computer, to use the other `metadata.db` will not make you loose
+   any information
+ - rebuild it. `calibredb restore_database` won't do it, but will tell
+   you how to do it.
+
+Checking the library
+--------------------
+You can use `calibredb check_library` to check you library is
+correct. If you use git for it, it will always tell you that it is not
+correct: there is this author ".git" it doesn't know about. Just don't
+care about it.
+
+Maybe this can be solved by using `vcsh` but apparently
+`vcsh`+`git annex` it not well tested yet.
+
+Automatic stuff
+---------------
+I use `mr` to automatically run all this, but some config could be
+done (I believe) to have `git annex copy --auto` do what it should.
+
+There are also the git annex assistant for this kind of automatic
+synchronizations of contents, but I don't know if my automatic
+unlocking of one file will break this.
+
+It might be interesting to find someway to unlock and lock the library
+only when running calibre, a simple script to launch calibre will do
+that. Note that each time you will lock and unlock, you will have a
+new commit in git.
+
+Another solution
+===================
+You could also use direct mode in place of the auto unlock feature
+
+    git annex direct
+
+Then remove the `post-commit` git hook (or do not add it). Its a
+simpler solution, but remember that interaction between git annex direct
+repositories and plain git are complex and sometimes downright dangerous. See [[direct mode]] for details.
+
+In particular, do *not* called `git add *` in the above steps, as that will commit all books into git.
diff --git a/doc/tips/Git_annex_and_Calibre/comment_1_b0ef346eaab9ff616aa1ba6b5f4530bc._comment b/doc/tips/Git_annex_and_Calibre/comment_1_b0ef346eaab9ff616aa1ba6b5f4530bc._comment
new file mode 100644
index 0000000000..7636b77022
--- /dev/null
+++ b/doc/tips/Git_annex_and_Calibre/comment_1_b0ef346eaab9ff616aa1ba6b5f4530bc._comment
@@ -0,0 +1,7 @@
+[[!comment format=mdwn
+ username="https://id.koumbit.net/anarcat"
+ subject="spurious changes in metadata.db"
+ date="2015-02-15T05:03:20Z"
+ content="""
+note that metadata.db seems to change even though no change was performed on the library. i filed a [bug report upstream](https://bugs.launchpad.net/calibre/+bug/1422058) to try and figure out what is going on here. -- [[anarcat]]
+"""]]
diff --git a/doc/tips/Git_annex_and_Calibre/comment_2_9e8122ea81bbd0a86bd6c5173db801f8._comment b/doc/tips/Git_annex_and_Calibre/comment_2_9e8122ea81bbd0a86bd6c5173db801f8._comment
new file mode 100644
index 0000000000..de485fb73b
--- /dev/null
+++ b/doc/tips/Git_annex_and_Calibre/comment_2_9e8122ea81bbd0a86bd6c5173db801f8._comment
@@ -0,0 +1,7 @@
+[[!comment format=mdwn
+ username="https://id.koumbit.net/anarcat"
+ subject="undo to the rescue"
+ date="2015-02-15T05:52:58Z"
+ content="""
+note: to avoid having too many such changes, i end up using [[todo/direct_mode_undo]] quite often.
+"""]]
diff --git a/doc/tips/Git_annex_and_Calibre/comment_3_ee1372977919229d17d216cfc3fb0f61._comment b/doc/tips/Git_annex_and_Calibre/comment_3_ee1372977919229d17d216cfc3fb0f61._comment
new file mode 100644
index 0000000000..998280d29b
--- /dev/null
+++ b/doc/tips/Git_annex_and_Calibre/comment_3_ee1372977919229d17d216cfc3fb0f61._comment
@@ -0,0 +1,11 @@
+[[!comment format=mdwn
+ username="Dilyin"
+ subject="comment 3"
+ date="2016-06-02T18:10:33Z"
+ content="""
+Actually you do want to exclude \"cover.jpg\" from being annexed because it will allow you to view the covers of all your books without getting them.
+Keeping metadata.opf out of annex is not that important because you cannot safely edit the metadata without getting the books first.
+
+It would be great to teach Calibre to see broken symlinks as normal zero length files and work with them accordingly.
+And we do need a Calibre plugin with buttons \"annex get\" and \"annex unlock\".
+"""]]
diff --git a/doc/tips/How_to_retroactively_annex_a_file_already_in_a_git_repo.mdwn b/doc/tips/How_to_retroactively_annex_a_file_already_in_a_git_repo.mdwn
new file mode 100644
index 0000000000..97f5828d39
--- /dev/null
+++ b/doc/tips/How_to_retroactively_annex_a_file_already_in_a_git_repo.mdwn
@@ -0,0 +1,19 @@
+I worked out how to retroactively annex a large file that had been checked into a git repo some time ago.  I thought this might be useful for others, so I am posting it here.
+
+Suppose you have a git repo where somebody had checked in a large file you would like to have annexed, but there are a bunch of commits after it and you don't want to loose history, but you also don't want everybody to have to retrieve the large file when they clone the repo.  This will re-write history as if the file had been annexed when it was originally added.
+
+This command works for me, it relies on the current behavior of git which is to use a directory named .git-rewrite/t/ at the top of the git tree for the extracted tree.  This will not be fast and it will rewrite history, so be sure that everybody who has a copy of your repo is OK with accepting the new history.  If the behavior of git changes, you can specify the directory to use with the -d option.  Currently, the t/ directory is created inside the directory you specify, so "-d ./.git-rewrite/" should be roughly equivalent to the default.
+
+Enough with the explanation, on to the command:
+
    +git filter-branch --tree-filter 'for FILE in file1 file2 file3;do if [ -f "$FILE" ] && [ ! -L "$FILE" ];then git rm --cached "$FILE";git annex add "$FILE";ln -sf `readlink "$FILE"|sed -e "s:^../../::"` "$FILE";fi;done' --tag-name-filter cat -- --all
+
    
+
+replace file1 file2 file3... with whatever paths you want retroactively annexed.  If you wanted bigfile1.bin in the top dir and subdir1/bigfile2.bin to be retroactively annexed try:
+
    +git filter-branch --tree-filter 'for FILE in bigfile1.bin subdir1/bigfile2.bin;do if [ -f "$FILE" ] && [ ! -L "$FILE" ];then git rm --cached "$FILE";git annex add "$FILE";ln -sf `readlink "$FILE"|sed -e "s:^../../::"` "$FILE";fi;done' --tag-name-filter cat -- --all
+
    
+
+**If your repo has tags** then you should take a look at the git-filter-branch man page about the --tag-name-filter option and decide what you want to do.  By default this will re-write the tags "nearly properly".
+
+You'll probably also want to look at the git-filter-branch man page's section titled "CHECKLIST FOR SHRINKING A REPOSITORY" if you want to free up the space in the existing repo that you just changed history on.
diff --git a/doc/tips/How_to_retroactively_annex_a_file_already_in_a_git_repo/comment_1_7eaf73fb3355bd706ab18a43790b3c10._comment b/doc/tips/How_to_retroactively_annex_a_file_already_in_a_git_repo/comment_1_7eaf73fb3355bd706ab18a43790b3c10._comment
new file mode 100644
index 0000000000..d4e34e8cda
--- /dev/null
+++ b/doc/tips/How_to_retroactively_annex_a_file_already_in_a_git_repo/comment_1_7eaf73fb3355bd706ab18a43790b3c10._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="http://edheil.wordpress.com/"
+ ip="173.162.44.162"
+ subject="comment 1"
+ date="2012-12-16T00:11:38Z"
+ content="""
+Man, I wish you'd written this a couple weeks ago. :)  I was never able to figure that incantation out and ended up unannexing and re-annexing the whole thing to get rid of the file I inadvertently checked into git instead of the annex.
+"""]]
diff --git a/doc/tips/How_to_retroactively_annex_a_file_already_in_a_git_repo/comment_2_dac1a171204f30d7c906e878eb6bd461._comment b/doc/tips/How_to_retroactively_annex_a_file_already_in_a_git_repo/comment_2_dac1a171204f30d7c906e878eb6bd461._comment
new file mode 100644
index 0000000000..a3ea62385e
--- /dev/null
+++ b/doc/tips/How_to_retroactively_annex_a_file_already_in_a_git_repo/comment_2_dac1a171204f30d7c906e878eb6bd461._comment
@@ -0,0 +1,45 @@
+[[!comment format=mdwn
+ username="https://launchpad.net/~arand"
+ nickname="arand"
+ subject="comment 2"
+ date="2013-03-13T12:05:49Z"
+ content="""
+Based on the hints given here I've worked on a filter to both annex and add urls via filter-branch:
+
+[https://gitorious.org/arand-scripts/arand-scripts/blobs/master/annex-filter](https://gitorious.org/arand-scripts/arand-scripts/blobs/master/annex-filter)
+
+The script above is very specific but I think there are a few ideas that can be used in general, the general structure is
+
+    #!/bin/bash
+    
+    # links that already exist
+    links=$(mktemp)
+    find . -type l >\"$links\"
+    
+    # remove from staging area first to not block and then annex
+    git rm --cached --ignore-unmatch -r bin*
+    git annex add -c annex.alwayscommit=false bin*
+
+    # compare links before and after annexing, remove links that existed before
+    newlinks=$(mktemp -u)
+    mkfifo \"$newlinks\"
+    comm -13 <(sort \"$links\") <(find . -type l | sort) > \"$newlinks\" &
+    
+    # rewrite links
+    while IFS= read -r file
+    do
+        # link is created below .git-rewrite/t/ during filter-branch, strip two parents for correct target
+        ln -sf \"$(readlink \"$file\" | sed -e 's%^\.\./\.\./%%')\" \"$file\"
+    done < \"$newlinks\"
+
+    git annex merge
+
+which would be run using
+
+    git filter-branch --tree-filter path/annex-filter --tag-filter cat -- --all
+
+or similar.
+
+* I'm using `find` to make sure the only rewritten symlinks are for the newly annexed files, this way it is possible to annex an unknown set of filenames
+* If doing several git annex commands using `-c annex.alwayscommit=false` and doing a `git annex merge` at the end instead might be faster.
+"""]]
diff --git a/doc/tips/How_to_retroactively_annex_a_file_already_in_a_git_repo/comment_3_b62ec0b848d2487d68d7032682622193._comment b/doc/tips/How_to_retroactively_annex_a_file_already_in_a_git_repo/comment_3_b62ec0b848d2487d68d7032682622193._comment
new file mode 100644
index 0000000000..9b8aa58f8e
--- /dev/null
+++ b/doc/tips/How_to_retroactively_annex_a_file_already_in_a_git_repo/comment_3_b62ec0b848d2487d68d7032682622193._comment
@@ -0,0 +1,36 @@
+[[!comment format=mdwn
+ username="arand"
+ ip="130.238.245.202"
+ subject="comment 3"
+ date="2013-03-18T14:39:52Z"
+ content="""
+One thing I noticed is that git-annex needs to checksum each file even if they were previously annexed (rather obviously since there is no general way to tell if the file is the same as the old one without checksumming), but in the specific case that we are replacing files that are already in git, we do actually have the sha1 checksum for each file in question, which could be used.
+
+So, trying to work with this, I wrote a filter script that starts out annexing stuff in the first commit, and continously writes out sha1<->filename<->git-annex-object triplets to a global file, when it then starts with the next commit, it compares the sha1s of the index with those of the global file, and any matches are manually symlinked directly to the corresponding git-annex-object without checksumming.
+
+I've done a few tests and this seems to be considerably faster than letting git-annex checksum everything.
+
+This is from a git-svn import of the (free software) Red Eclipse game project, there are approximately 3500 files (images, maps, models, etc.) being annexed in each commit (and around 5300 commits, hence why I really, really care about speed):
+
+10 commits: ~7min
+
+100 commits: ~38min
+
+For comparison, the old and new method (the difference should increase with the amount of commits):
+
+old, 20 commits ~32min
+
+new, 20 commits: ~11min
+
+The script itself is a bit of a monstrosity in bash(/grep/sed/awk/git), and the files that are annexed are hardcoded (removed in forming $oldindexfiles), but should be fairly easy to adapt:
+
+[https://gitorious.org/arand-scripts/arand-scripts/blobs/master/annex-ffilter](https://gitorious.org/arand-scripts/arand-scripts/blobs/master/annex-ffilter)
+
+The usage would be something like:
+
+    rm /tmp/annex-ffilter.log; git filter-branch --tree-filter 'ANNEX_FFILTER_LOG=/tmp/annex-ffilter.log ~/utv/scripts/annex-ffilter' --tag-name-filter cat -- branchname
+
+I suggest you use it with at least two orders of magnitude more caution than normal filter-branch.
+
+Hope it might be useful for someone else wrestling with filter-branch and git-annex :)
+"""]]
diff --git a/doc/tips/How_to_retroactively_annex_a_file_already_in_a_git_repo/comment_4_2423904e41a86cd1c6bc155d7b733642._comment b/doc/tips/How_to_retroactively_annex_a_file_already_in_a_git_repo/comment_4_2423904e41a86cd1c6bc155d7b733642._comment
new file mode 100644
index 0000000000..ab1d4e0064
--- /dev/null
+++ b/doc/tips/How_to_retroactively_annex_a_file_already_in_a_git_repo/comment_4_2423904e41a86cd1c6bc155d7b733642._comment
@@ -0,0 +1,9 @@
+[[!comment format=mdwn
+ username="https://www.google.com/accounts/o8/id?id=AItOawknOATcOkmzX4jKuET5Z2RsaFUNnLKnQsU"
+ nickname="Stephen"
+ subject="comment 4"
+ date="2013-06-22T07:43:09Z"
+ content="""
+Thanks for the tip :) One question though: how do I push this new history out throughout my other Annexes?
+All I managed to make it do was revert the rewrite so the raw file appeared again...
+"""]]
diff --git a/doc/tips/How_to_retroactively_annex_a_file_already_in_a_git_repo/comment_5_3062c0794ecd7c6237efae66f4d9b62f._comment b/doc/tips/How_to_retroactively_annex_a_file_already_in_a_git_repo/comment_5_3062c0794ecd7c6237efae66f4d9b62f._comment
new file mode 100644
index 0000000000..e8ef98d999
--- /dev/null
+++ b/doc/tips/How_to_retroactively_annex_a_file_already_in_a_git_repo/comment_5_3062c0794ecd7c6237efae66f4d9b62f._comment
@@ -0,0 +1,13 @@
+[[!comment format=mdwn
+ username="https://www.google.com/accounts/o8/id?id=AItOawlcfH7xkyz1kyG_neK4GcFFfFWuIY7l_6A"
+ nickname="Primiano"
+ subject="large scale rewrite tips"
+ date="2015-01-06T22:55:20Z"
+ content="""
+I recently had the need of re-kind-of-annexing an unusually large repo (one of the largest?).
+With some tricks and the right code I managed to get it down to 170000 commits in 19 minutes and extracing ~8GB of blobs. 
+Attaching the link here as I feel it might be helpful for very large projects (where git-filter-branch can become prohibitively slow) 
+
+[https://www.primianotucci.com/blog/large-scale-git-history-rewrites](https://www.primianotucci.com/blog/large-scale-git-history-rewrites)
+
+"""]]
diff --git a/doc/tips/How_to_retroactively_annex_a_file_already_in_a_git_repo/comment_6_1eceb980814f95b28caac9d4d9894f01._comment b/doc/tips/How_to_retroactively_annex_a_file_already_in_a_git_repo/comment_6_1eceb980814f95b28caac9d4d9894f01._comment
new file mode 100644
index 0000000000..17592b2870
--- /dev/null
+++ b/doc/tips/How_to_retroactively_annex_a_file_already_in_a_git_repo/comment_6_1eceb980814f95b28caac9d4d9894f01._comment
@@ -0,0 +1,17 @@
+[[!comment format=mdwn
+ username="Dilyin"
+ subject="Retroactively annex"
+ date="2016-06-01T18:36:40Z"
+ content="""
+Hmm, guyz? Are you serious with these scripts?
+
+1. git rm -r --cached large_files
+# files are indexed as both removed and untracked and are still in place
+2. commit the changes
+# files are seen as untracked
+3. git annex add large_files
+# files are replaced with symlinks and are in the index
+4. commit changes again
+
+Make sure that you don't have annex.largefiles settings that would prevent annexing the files.
+"""]]
diff --git a/doc/tips/How_to_retroactively_annex_a_file_already_in_a_git_repo/comment_7_603db6818d33663b70b917c04fd8485b._comment b/doc/tips/How_to_retroactively_annex_a_file_already_in_a_git_repo/comment_7_603db6818d33663b70b917c04fd8485b._comment
new file mode 100644
index 0000000000..5527c2b430
--- /dev/null
+++ b/doc/tips/How_to_retroactively_annex_a_file_already_in_a_git_repo/comment_7_603db6818d33663b70b917c04fd8485b._comment
@@ -0,0 +1,30 @@
+[[!comment format=mdwn
+ username="https://launchpad.net/~stephane-gourichon-lpad"
+ nickname="stephane-gourichon-lpad"
+ avatar="http://cdn.libravatar.org/avatar/02d4a0af59175f9123720b4481d55a769ba954e20f6dd9b2792217d9fa0c6089"
+ subject=""Hmm, guyz? Are you serious with these scripts?" Well, what's the matter?"
+ date="2016-11-15T10:58:32Z"
+ content="""
+## Wow, scary
+
+Dilyin's comment is scary.  It suggests bad things can happen, but is not very clear.
+
+Bloated history is one thing.  
+Obviously broken repo is bad but can be (slowly) recovered from remotes.  
+Subtly crippled history that you don't notice can be a major problem (especially once you have propagated it to all your remotes to \"recover from bloat\").
+
+## More common than it seems
+
+There's a case probably more common than people actually report: mistakenly doing `git add` instead of `git annex add` and realizing it only after a number of commits. Doing `git annex add` at that time will have the file duplicated (regular git and annex). 
+
+Extra wish: when doing `git annex add` of a file that is already present in git history, `git-annex` could notice and tell.
+
+## Simple solution?
+
+Can anyone elaborate on the scripts provided here, are they safe?  What can happen if improperly used or in corner cases?
+
+* \"files are replaced with symlinks and are in the index\" -> so what ?
+* \"Make sure that you don't have annex.largefiles settings that would prevent annexing the files.\" -> What would happen? Also `.gitattributes`.
+
+Thank you.
+"""]]
diff --git a/doc/tips/How_to_retroactively_annex_a_file_already_in_a_git_repo/comment_8_834410421ccede5194bd8fbaccea8d1a._comment b/doc/tips/How_to_retroactively_annex_a_file_already_in_a_git_repo/comment_8_834410421ccede5194bd8fbaccea8d1a._comment
new file mode 100644
index 0000000000..2c36962aa5
--- /dev/null
+++ b/doc/tips/How_to_retroactively_annex_a_file_already_in_a_git_repo/comment_8_834410421ccede5194bd8fbaccea8d1a._comment
@@ -0,0 +1,82 @@
+[[!comment format=mdwn
+ username="StephaneGourichon"
+ avatar="http://cdn.libravatar.org/avatar/8cea01af2c7a8bf529d0a3d918ed4abf"
+ subject="Walkthrough of a prudent retroactive annex."
+ date="2016-11-24T11:27:59Z"
+ content="""
+Been using the one-liner.  Despite the warning, I'm not dead yet.
+
+There's much more to do than the one-liner.
+
+This post offers instructions.
+
+# First simple try: slow
+
+Was slow (estimated >600s for 189 commits).
+
+# In tmpfs: about 6 times faster
+
+I have cloned repository into /run/user/1000/rewrite-git, which is a tmpfs mount point. (Machine has plenty of RAM.)
+
+There I also did `git annex init`, git-annex found its state branches.
+
+On second try I also did 
+
+    git checkout -t remotes/origin/synced/master
+
+So that filter-branch would clean that, too.
+
+There, `filter-branch` operation finished in 90s first try, 149s second try.
+
+`.git/objects` wasn't smaller.
+
+# Practicing reduction on clone
+
+This produced no visible benefit:
+
+time git gc --aggressive
+time git repack -a -d 
+
+Even cloning and retrying on clone. Oh, but I should have done `git clone file:///path` as said on git-filter-branch man page's section titled \"CHECKLIST FOR SHRINKING A REPOSITORY\" 
+
+This (as seen on https://rtyley.github.io/bfg-repo-cleaner/ ) was efficient:
+
+    git reflog expire --expire=now --all && git gc --prune=now --aggressive
+
+`.git/objects` shrunk from 148M to 58M
+
+All this was on a clone of the repo in tmpfs.
+
+# Propagating cleaned up branches to origin
+
+This confirmed that filter-branch did not change last tree:
+
+    git diff remotes/origin/master..master
+    git diff remotes/origin/synced/master synced/master
+
+This, expectedly, was refused:
+
+    git push origin master
+    git push origin synced/master
+
+On origin, I checked out the hash of current master, then on tmpfs clone
+
+    git push -f origin master
+    git push -f origin synced/master
+
+Looks good.
+
+I'm not doing the aggressive shrink now, because of the \"two orders of magnitude more caution than normal filter-branch\" recommended by arand.
+
+# Now what? Check if precious not broken
+
+I'm planning to do the same operation on the other repos, then :
+
+* if everything seems right, 
+* if `git annex sync` works between all those fellows
+* etc, 
+* then I would perform the reflog expire, gc prune on some then all of them, etc.
+
+Joey, does this seem okay?  Any comment?
+
+"""]]
diff --git a/doc/tips/Internet_Archive_via_S3.mdwn b/doc/tips/Internet_Archive_via_S3.mdwn
new file mode 100644
index 0000000000..ba3c75891c
--- /dev/null
+++ b/doc/tips/Internet_Archive_via_S3.mdwn
@@ -0,0 +1,75 @@
+[The Internet Archive](http://www.archive.org/) allows members to upload
+collections using an Amazon S3 
+[compatible API](http://www.archive.org/help/abouts3.txt), and this can
+be used with git-annex's [[special_remotes/S3]] support. 
+
+So, you can locally archive things with git-annex, define remotes that
+correspond to "items" at the Internet Archive, and use git-annex to upload
+your files to there. Of course, your use of the Internet Archive must
+comply with their [terms of service](http://www.archive.org/about/terms.php).
+
+A nice added feature is that whenever git-annex sends a file to the
+Internet Archive, it records its url, the same as if you'd run `git annex
+addurl`. So any users who can clone your repository can download the files
+from archive.org, without needing any login or password info. 
+The url to the content in the Internet Archive is also displayed by
+`git annex whereis`. This makes the Internet Archive a nice way to
+publish the large files associated with a public git repository.
+
+## webapp setup
+
+Just go to "Add Another Repository", pick "Internet Archive",
+and you're on your way.
+
+## basic setup
+
+Sign up for an account, and get your access keys here:
+
+	
+	# export AWS_ACCESS_KEY_ID=blahblah
+	# export AWS_SECRET_ACCESS_KEY=xxxxxxx
+
+Specify `host=s3.us.archive.org` when doing `initremote` to set up
+a remote at the Archive. This will enable a special Internet Archive mode:
+Encryption is not allowed; you are required to specify a bucket name
+rather than having git-annex pick a random one; and you can optionally
+specify `x-archive-meta*` headers to add metadata as explained in their
+[documentation](http://www.archive.org/help/abouts3.txt).
+
+	# git annex initremote archive-panama type=S3 \
+		host=s3.us.archive.org bucket=panama-canal-lock-blueprints \
+		x-archive-meta-mediatype=texts x-archive-meta-language=eng \
+		x-archive-meta-title="original Panama Canal lock design blueprints"
+	initremote archive-panama (Internet Archive mode) ok
+	# git annex describe archive-panama "a man, a plan, a canal: panama"
+	describe archive-panama ok
+
+Then you can annex files and copy them to the remote as usual:
+
+	# git annex add photo1.jpeg --backend=SHA256E
+	add photo1.jpeg (checksum...) ok
+	# git annex copy photo1.jpeg --fast --to archive-panama
+	copy (to archive-panama...) ok
+
+## update lag
+
+It may take a while for archive.org to make files publically visible after
+they've been uploaded.
+
+While files can be removed from the Internet Archive, 
+[derived versions](https://archive.org/help/derivatives.php)
+of some files may continued to be stored there for a while
+after the originals were removed.
+
+## exporting trees
+
+By default, files stored in the Internet Archive will show up there named
+by their git-annex key, not the original filename. If the filenames
+are important, you can run `git annex initremote` with an additional
+parameter "exporttree=yes", and then use [[git-annex-export]] to publish
+a tree of files to the Internet Archive.
+
+Note that the Internet Archive may not support certian characters
+in filenames ([see FAQ](http://archive.org/about/faqs.php#1099)).
+If exporting a filename fails due to such limitations, you would need
+to rename it in your git annex repository in order to export it.
diff --git a/doc/tips/Internet_Archive_via_S3/comment_1_d53a3848c20dce61867283fc03c2adaa._comment b/doc/tips/Internet_Archive_via_S3/comment_1_d53a3848c20dce61867283fc03c2adaa._comment
new file mode 100644
index 0000000000..7c2ce48fcf
--- /dev/null
+++ b/doc/tips/Internet_Archive_via_S3/comment_1_d53a3848c20dce61867283fc03c2adaa._comment
@@ -0,0 +1,34 @@
+[[!comment format=mdwn
+ username="https://id.koumbit.net/anarcat"
+ ip="72.0.72.144"
+ subject="how to use with simply addurl?"
+ date="2013-10-09T22:27:27Z"
+ content="""
+It doesn't seem like git annex addurl by itself supports the archive.org urls...
+
+[[!format txt \"\"\"
+anarcat@marcos:presentations$ git annex addurl --file=re_publica_2012___Eben_Moglen___Freedom_of_Thought_Requires_Free_Media.webm http://archive.org/download/Republica2012-EbenMoglen-FreedomOfThoughtRequiresFreeMedia/re_publica_2012___Eben_Moglen___Freedom_of_Thought_Requires_Free_Media.webm
+addurl re_publica_2012___Eben_Moglen___Freedom_of_Thought_Requires_Free_Media.webm
+  failed to verify url exists: http://archive.org/download/Republica2012-EbenMoglen-FreedomOfThoughtRequiresFreeMedia/re_publica_2012___Eben_Moglen___Freedom_of_Thought_Requires_Free_Media.webm
+failed
+git-annex: addurl: 1 failed
+\"\"\"]]
+
+I also tried the \"details\" url () - but that just downloads the webpage, not the video either...
+
+Even the ultimate video URL doesn't work:
+
+[[!format txt \"\"\"
+anarcat@marcos:presentations$ git annex addurl --debug --file=re_publica_2012___Eben_Moglen___Freedom_of_Thought_Requires_Free_Media.webm http://ia601009.us.archive.org/9/items/Republica2012-EbenMoglen-FreedomOfThoughtRequiresFreeMedia/re_publica_2012___Eben_Moglen___Freedom_of_Thought_Requires_Free_Media.webm
+[2013-10-09 18:26:30 EDT] call: quvi [\"-v\",\"mute\",\"--support\",\"http://ia601009.us.archive.org/9/items/Republica2012-EbenMoglen-FreedomOfThoughtRequiresFreeMedia/re_publica_2012___Eben_Moglen___Freedom_of_Thought_Requires_Free_Media.webm\"]
+addurl re_publica_2012___Eben_Moglen___Freedom_of_Thought_Requires_Free_Media.webm [2013-10-09 18:26:30 EDT] read: curl [\"-s\",\"--head\",\"-L\",\"http://ia601009.us.archive.org/9/items/Republica2012-EbenMoglen-FreedomOfThoughtRequiresFreeMedia/re_publica_2012___Eben_Moglen___Freedom_of_Thought_Requires_Free_Media.webm\",\"-w\",\"%{http_code}\"]
+
+  failed to verify url exists: http://ia601009.us.archive.org/9/items/Republica2012-EbenMoglen-FreedomOfThoughtRequiresFreeMedia/re_publica_2012___Eben_Moglen___Freedom_of_Thought_Requires_Free_Media.webm
+failed
+git-annex: addurl: 1 failed
+\"\"\"]]
+
+... even though that URL actually gives out a proper 200 OK response code.
+
+Any ideas? --[[anarcat]]
+"""]]
diff --git a/doc/tips/Internet_Archive_via_S3/comment_2_91c1472da27b00e5d682d22bc1ef04e0._comment b/doc/tips/Internet_Archive_via_S3/comment_2_91c1472da27b00e5d682d22bc1ef04e0._comment
new file mode 100644
index 0000000000..9b4ac58b23
--- /dev/null
+++ b/doc/tips/Internet_Archive_via_S3/comment_2_91c1472da27b00e5d682d22bc1ef04e0._comment
@@ -0,0 +1,10 @@
+[[!comment format=mdwn
+ username="http://joeyh.name/"
+ ip="4.154.4.22"
+ subject="comment 2"
+ date="2013-10-11T17:08:27Z"
+ content="""
+This was a misleading error message. The url you are trying to add to the file does not match the size recorded for the file already in the annex. (Or possibly the file's key has no recorded size). If you really want to add the url to the file despite it being a different encoding, you can use --relaxed, although fsck may not like the result if you ever end up downloading that url..
+
+(Please file bug reports for problems in the future, rather than posting comments on only vaguely related pages which as we can see here can turn out to be entirely offtopic.)
+"""]]
diff --git a/doc/tips/Internet_Archive_via_S3/comment_3_e23cf781c532f80d47d52265f2b2c87e._comment b/doc/tips/Internet_Archive_via_S3/comment_3_e23cf781c532f80d47d52265f2b2c87e._comment
new file mode 100644
index 0000000000..3745d544ce
--- /dev/null
+++ b/doc/tips/Internet_Archive_via_S3/comment_3_e23cf781c532f80d47d52265f2b2c87e._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="https://id.koumbit.net/anarcat"
+ ip="72.0.72.144"
+ subject="still a bug, filed separately!"
+ date="2013-10-11T18:49:06Z"
+ content="""
+Aaah, of course, sorry for the noise here. It turns out that this is *not* because the filesize (or even the checksum, for that matter) are different, so there's clearly a bug there, and i filed it in [[bugs/addurl_fails_on_the_internet_archive]]. Thanks!
+"""]]
diff --git a/doc/tips/Repositories_with_large_number_of_files.mdwn b/doc/tips/Repositories_with_large_number_of_files.mdwn
new file mode 100644
index 0000000000..347f6f94aa
--- /dev/null
+++ b/doc/tips/Repositories_with_large_number_of_files.mdwn
@@ -0,0 +1,55 @@
+Just as git does not scale well with large files, it can also become painful to work with when you have a large *number* of files. Below are things I have found to minimise the pain.
+
+[[!toc]]
+
+# Using version 4 index files
+
+During operations which affect the index, git writes an entirely new index out to index.lck and then replaces .git/index with it. With a large number of files, this index file can be quite large and take several seconds to write every time you manipulate the index!
+
+This can be mitigated by changing it to version 4 which uses path compression to reduce the filesize:
+
+    git update-index --index-version 4
+
+*NOTE: The git documentation warns that this version may not be supported by other git implementations like JGit and libgit2.*
+
+Personally, I saw a reduction from 516MB to 206MB (*40% of original size*) and got a much more responsive git!
+
+It may also be worth doing the same to git-annex's index:
+
+    GIT_INDEX_FILE=.git/annex/index git update-index --index-version 4
+
+Though I didn't gain as much here with 89MB to 86MB (96% of original size).
+
+# Packing
+
+As I have gc disabled:
+
+    git config gc.auto 0
+
+so I control when it is run, I ended up with a lot of loose objects which also cause slowness in git. Using
+
+    git count-objects
+
+to tell me how many loose objects I have, when I reach a threshold (~25000), I pack those loose objects and clean things up:
+
+    git repack -d
+    git gc
+    git prune
+
+# File count per directory
+
+If it takes a long time to list the files in a directory, naturally, git(-annex) will be affected by this bottleneck.
+
+You can avoid this by keeping the number of files in a directory to between 5000 and 20000 (depends on the filesystem and its settings).
+
+[fpart](https://sourceforge.net/projects/fpart/) can be a very useful tool to achieve this.
+
+This sort of usage was discussed in [[forum/Handling_a_large_number_of_files]] and [[forum/__34__git_annex_sync__34___synced_after_8_hours]]. -- [[CandyAngel]]
+
+# Forget tracking information
+
+In addition to keeping track of where files are, git-annex keeps a *log* that keeps track of where files *were*. This can take up space as well and slow down certain operations.
+
+You can use the [[git-annex-forget]] command to drop historical location tracking info for files.
+
+Note: this was discussed in [[forum/scalability_with_lots_of_files]]. -- [[anarcat]]
diff --git a/doc/tips/Repositories_with_large_number_of_files/comment_1_fac49a6d3f6a18c9925c34521221307d._comment b/doc/tips/Repositories_with_large_number_of_files/comment_1_fac49a6d3f6a18c9925c34521221307d._comment
new file mode 100644
index 0000000000..42550a0645
--- /dev/null
+++ b/doc/tips/Repositories_with_large_number_of_files/comment_1_fac49a6d3f6a18c9925c34521221307d._comment
@@ -0,0 +1,7 @@
+[[!comment format=mdwn
+ username="ghen1"
+ subject="comment 1"
+ date="2015-07-08T04:08:28Z"
+ content="""
+This is some excellent information. Thank you.
+"""]]
diff --git a/doc/tips/Repositories_with_large_number_of_files/comment_2_f8417d9ebcfdcba34dfbbf76070ad95c._comment b/doc/tips/Repositories_with_large_number_of_files/comment_2_f8417d9ebcfdcba34dfbbf76070ad95c._comment
new file mode 100644
index 0000000000..86bb069b5b
--- /dev/null
+++ b/doc/tips/Repositories_with_large_number_of_files/comment_2_f8417d9ebcfdcba34dfbbf76070ad95c._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="joey"
+ subject="""comment 2"""
+ date="2015-09-22T16:51:51Z"
+ content="""
+As writing the index file becomes the bottleneck, turning on split index
+mode might be a help as well. See git-update-index's man page.
+"""]]
diff --git a/doc/tips/Repositories_with_large_number_of_files/comment_3_992f2a85ce0cdcef2f97ff978560fdb8._comment b/doc/tips/Repositories_with_large_number_of_files/comment_3_992f2a85ce0cdcef2f97ff978560fdb8._comment
new file mode 100644
index 0000000000..514c829fef
--- /dev/null
+++ b/doc/tips/Repositories_with_large_number_of_files/comment_3_992f2a85ce0cdcef2f97ff978560fdb8._comment
@@ -0,0 +1,19 @@
+[[!comment format=mdwn
+ username="umeboshi"
+ subject="comment 3"
+ date="2016-01-04T21:26:05Z"
+ content="""
+I have been playing with tracking a large number of url's for about one month now.  Having already been disappointed by how git performs when there are a very large amount of files in the annex, I tested making multiple annexes.  I did find that splitting the url's into multiple annexes increased performance, but at the cost of extra housekeeping, duplicated url's, and more work needed to keep track of the url's.  Part of the duplication and tracking problem was mitigated by using a dumb remote, such as rsync or directory, where a very large amount of objects can be stored.  The dumb remotes perform very well, however each annex needed to be synced regularly with the dumb remote.
+
+I found the dumb remote to be great for multiple annexes.  I have noticed that a person can create a new annex and extract a tarball of symlinks into the repo, the ```git commit``` the links.  Subsequently, executing ```git-annex fsck --from dummy``` would setup the tracking info, which was pretty useful.
+
+However, I found that by the time I got to over fifty annexes, the overall performance far worse than just storing the url's and file paths in a postgresql database.  In fact, the url's are already being stored and accessed from such a database, but I had the desire to access the url's from multiple machines, which is a bit more difficult with a centralized database.
+
+After reading the tips and pages discussing splitting the files into multiple directories, and changing the index version, I decided to try a single annex to hold the url's.  Over the new year's weekend, I decided to write a script that generates rss files to use with importfeed to add url's to this annex.  I have noticed that when using ```git commit``` the load average of the host was in the mid twenties and persisted for hours until I had to kill the processes to be able to use the machine again (**I would really like to know if there is a git config setting that would keep the load down, so the machine can be used during a commit**).  I gave up on ```git-annex sync``` this morning, since it was taking longer than I was able to sit in the coffee shop and wait for it (~3 hrs).
+
+I came back to the office, and started ```git gc``` which has been running for ~1hr.
+
+When making the larger annex, I decided to use the hexidecimal value of uuid5(url) for each filename, and use the two high nybbles and the two low nybbles for a two state directory structure, with respect for the advice from [CandyAngel](http://git-annex.branchable.com/forum/Handling_a_large_number_of_files/#comment-48ac38361b131b18125f8c43eb6ad577).  When my url's are organized in this manner, I still need access to the database to perform the appropriate ```git-annex get``` which impairs the use of this strategy, but I'm still testing and playing around.  I suspended adding url's to this annex until I get at least one sync performed.
+
+The url annex itself is not very big, and I am guessing the average file size to be close to 500K.  The large number of url's seems to be a problem I have yet to solve.  I wanted to experiment with this to further the idea of the public git-annex repositories that seem to be a useful idea, even though the utility of this idea is very limited at the moment.
+"""]]
diff --git a/doc/tips/Repositories_with_large_number_of_files/comment_4_8ff3aa032fb778ff69276984152578b0._comment b/doc/tips/Repositories_with_large_number_of_files/comment_4_8ff3aa032fb778ff69276984152578b0._comment
new file mode 100644
index 0000000000..78b79ef081
--- /dev/null
+++ b/doc/tips/Repositories_with_large_number_of_files/comment_4_8ff3aa032fb778ff69276984152578b0._comment
@@ -0,0 +1,11 @@
+[[!comment format=mdwn
+ username="CandyAngel"
+ subject="comment 4"
+ date="2016-01-05T11:41:13Z"
+ content="""
+@umeboshi: Odd that you report your machine freezes during commits.. I find the exact opposite.. waiting for a long time with no load at all.
+
+My current setup for my sorting annex (where I import all the files off old HDDs) is to have the HDD plugged into my home server (Atom 330 @ 1.6Ghz) and import the files on a cloned (empty) annex. Doing so for 1.1M files (latest HDD) is a long wait, because 80% of the time is waiting for something to happen (but there being no load on the machine). Once that is done, the HDD is transferred to my desktop, where the annex is \"joined\" to the others and files are sorted in a dedicated VM[1], where commit times are reasonable.
+
+[1] Fully virtualising my desktop is possibly the best thing I've ever done, in terms of setup. Locking up any VM affects none of the others (which is handy, as I discovered an issue that causes X to almost hardlock whenever libvo is used..).
+"""]]
diff --git a/doc/tips/Repositories_with_large_number_of_files/comment_5_8540bf807eaf1b4e927eafc03deef304._comment b/doc/tips/Repositories_with_large_number_of_files/comment_5_8540bf807eaf1b4e927eafc03deef304._comment
new file mode 100644
index 0000000000..2d1e238662
--- /dev/null
+++ b/doc/tips/Repositories_with_large_number_of_files/comment_5_8540bf807eaf1b4e927eafc03deef304._comment
@@ -0,0 +1,21 @@
+[[!comment format=mdwn
+ username="CandyAngel"
+ subject="comment 5"
+ date="2016-02-02T10:08:06Z"
+ content="""
+Splitting the index (git update-index --split-index) doesn't work for me at all. While it may initially reduce the size of .git/index, making a commit inflates it back to its original size anyway.
+
+I thought it might be some interaction with v4 index and its compression mechanics but it does the same if I set it to v3 index. For (manufactured) example:
+
+    $ git update-index --split-index
+    $ du -sh .git/*index*
+    4.0K    .git/index
+    76M     .git/sharedindex.70e661456302b51a7ec59bf5b32d630e74b34c7c
+
+    ... add 8000 files ...
+
+    $ git commit -m \"add files\"
+    $ du -sh .git/*index*
+    80M     .git/index
+    76M     .git/sharedindex.70e661456302b51a7ec59bf5b32d630e74b34c7c
+"""]]
diff --git a/doc/tips/Repositories_with_large_number_of_files/comment_6_9169a33c06cf8aea231cdd8f51ce17b6._comment b/doc/tips/Repositories_with_large_number_of_files/comment_6_9169a33c06cf8aea231cdd8f51ce17b6._comment
new file mode 100644
index 0000000000..1fb1de999a
--- /dev/null
+++ b/doc/tips/Repositories_with_large_number_of_files/comment_6_9169a33c06cf8aea231cdd8f51ce17b6._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="anarcat"
+ avatar="http://cdn.libravatar.org/avatar/4ad594c1e13211c1ad9edb81ce5110b7"
+ subject="merge with scalability?"
+ date="2017-04-24T14:10:29Z"
+ content="""
+shouldn't this tip be merged into the [[scalability]] page directly?
+"""]]
diff --git a/doc/tips/Shamir_secret_sharing_and_git-annex.mdwn b/doc/tips/Shamir_secret_sharing_and_git-annex.mdwn
new file mode 100644
index 0000000000..df19f68b8e
--- /dev/null
+++ b/doc/tips/Shamir_secret_sharing_and_git-annex.mdwn
@@ -0,0 +1,21 @@
+Combining git-annex with [Shamir secret sharing](http://en.wikipedia.org/wiki/Shamir%27s_Secret_Sharing)
+is an useful way to securely back up highly sensitive files,
+such as a gpg key or bitcoin wallet.
+
+Shamir secret sharing creates N shares of a file, of which any M can be
+used to reconstitute the original file. Anyone who has less than M shares
+cannot tell anything about the original file, other than its size.
+
+Where git-annex comes in is as a way to manage these shares. They can be
+added to the annex, and then git-annex used to move one share to each clone
+of the repository. Since git-annex keeps track of where each file is
+stored, this can aid later finding the shares again when they're needed, as
+well as making ongoing management of the shares easier.
+
+Note that this conveniece comes at a price: Any attacker who gets a copy
+of the git repository can use it to figure out where the shares are
+located. While this is not a crippling flaw, and can be worked around, it
+needs to be considered when implementing this technique.
+
+Here is an example of this method being used for a ~/.gnupg directory:
+
diff --git a/doc/tips/Synology_NAS_and_git_annex.mdwn b/doc/tips/Synology_NAS_and_git_annex.mdwn
new file mode 100644
index 0000000000..8a6d282c92
--- /dev/null
+++ b/doc/tips/Synology_NAS_and_git_annex.mdwn
@@ -0,0 +1,59 @@
+# How to use git-annex on a Synology NAS
+
+This is known to work with DSM 4.3-3810 Update 1 and git-annex standalone version 5.20131224-g6ca5271.
+
+## Installation Steps
+
+(1) In the DSM Package Center, install Git, which is available from Synology (no third-party repository needed).
+
+(2) Download the latest [[standalone|install/Linux_standalone]] git-annex build for Linux on armel.
+
+(3) Extract it somewhere sensible (eg: a bin/ directory your users home directory)
+
+(4) Go into the git-annex.linux directory and ./runshell. You can now run git-annex as you normally would.
+
+## How to sync with the Synology NAS
+
+### On the Synology
+(1) Setup port forwarding and associated dynamic dns, if applicable. Many good guides online for this.
+
+(2) Setup ssh key based authentication with the Synology for each computer you want to sync with it. You want a specific key that is used only by git-annex, for each computer. Again, many good guides online.
+
+(3) In the Synology .ssh/authorized_keys file for your account, add (substituting your username)
+[[!format sh """
+command="/home/$yourusername/.ssh/git-annex-shell"
+"""]]
+to the beginning of the line. Eg, it would look like this:
+[[!format sh """
+command="/home/greg/.ssh/git-annex-shell" ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABAQDT1yE96E/JQNPt0ziiNYJRvndCvLK4uG5h/SNYoAIBF1uH6L7VYAt3HWVqSyi3BcV70WDZ/yWgtNzbrcir46JpvEHMcvYaXLbANwoDGNjG/gsz7kP/8VUxZ6hG3P3ICuwnqVum5+rYXm6oj3xzWPfTRhhRoDZLOQdevSNpdGNaa/lSg8Vuq2suHwjQlQb8AIUuCZmS5cm6XwoUq/jJtN4LTuTPqMjzA6NkdhWM2Kigi9jPQBFborkYBPMphmZwBZiVnhsH1XpaOff+mP03D2gF/huC+b1vbWQstjuehUbY59rvJ4ijb3810Uq2ep7dwLagmILtX5GbL+GS64pAn9sIP annex-othercomputer
+"""]]
+
+(4) the git-annex-shell script in your .ssh should be created for you aftering your initial ./runshell
+
+(5) Double check that the script points to the correct directory of where your extracted git-annex.linux lives.
+
+### On the other computers - the manual way
+(1) See step 2 above about creating the specific git-annex ssh keys.
+
+(2) In your .ssh/config, create an alias for your Synology that includes specifying the right sshkey. For example, mine looks like:
+[[!format sh """
+Host synologyhost
+    HostName mydynamicdomain.no-ip.org
+    IdentityFile /home/greg/.ssh/annex_rsa
+"""]]
+
+(3) Now when you clone the git repo from the Synology, or add it as a remote, do the following:
+[[!format sh """
+git clone greg@synologyhost:/absolute/path/to/annexname annexname
+"""]]
+or
+
+[[!format sh """
+git remote add synology greg@synologyhost:/absolute/path/to/annexname
+"""]]
+
+(4) Run git-annex sync
+
+### On the other computers - Using the assistant
+
+(1) Use the webapp to add the remote. I'm not sure if there are any gotchas here as I have not done it this way yet.
diff --git a/doc/tips/Synology_NAS_and_git_annex/comment_10_ee03a12ac5ba760b038ac5b93eea4be3._comment b/doc/tips/Synology_NAS_and_git_annex/comment_10_ee03a12ac5ba760b038ac5b93eea4be3._comment
new file mode 100644
index 0000000000..562326bd62
--- /dev/null
+++ b/doc/tips/Synology_NAS_and_git_annex/comment_10_ee03a12ac5ba760b038ac5b93eea4be3._comment
@@ -0,0 +1,10 @@
+[[!comment format=mdwn
+ username="joey"
+ subject="""comment 10"""
+ date="2016-04-13T18:04:42Z"
+ content="""
+Right, it would need a standalone package. It's quite easy to build such a
+package on any Debian system, just run "make linuxstandalone" in
+git-annex's source tree. I don't have a PowerPC system handy to do such builds
+on myself.
+"""]]
diff --git a/doc/tips/Synology_NAS_and_git_annex/comment_11_ab2487da061a23e14198875a00ae801e._comment b/doc/tips/Synology_NAS_and_git_annex/comment_11_ab2487da061a23e14198875a00ae801e._comment
new file mode 100644
index 0000000000..6a4e9128c6
--- /dev/null
+++ b/doc/tips/Synology_NAS_and_git_annex/comment_11_ab2487da061a23e14198875a00ae801e._comment
@@ -0,0 +1,10 @@
+[[!comment format=mdwn
+ username="ewen"
+ avatar="http://cdn.libravatar.org/avatar/605b2981cb52b4af268455dee7a4f64e"
+ subject="git-annex on Synology DS216+ (x86-64)"
+ date="2017-05-28T01:19:16Z"
+ content="""
+I used a similar approach, with a bit of refinement to not require a custom ssh key/user, to [get `git-annex` running on a Synology DS216+ NAS](http://ewen.mcneill.gen.nz/blog/entry/2017-05-28-git-annex-on-synology-ds216+/), which is based around a Celeron chip (and thus needs the `x86-64` build).  Once all the `PATH` related issues are taken care of (which some symlinks into `/usr/bin`) it appears to work like any other Linux/Unix-based `git-annex` install.  (Definitely much more successful than [trying to use `git-annex` via a SMB share](http://git-annex.branchable.com/bugs/SMB__58___git_annex_clone_works__44___get_fails_on_transfer_lock/) at this point.)
+
+Ewen
+"""]]
diff --git a/doc/tips/Synology_NAS_and_git_annex/comment_1_ef7e19f1fd2005eb7cc74509ffb92766._comment b/doc/tips/Synology_NAS_and_git_annex/comment_1_ef7e19f1fd2005eb7cc74509ffb92766._comment
new file mode 100644
index 0000000000..e7a1db9373
--- /dev/null
+++ b/doc/tips/Synology_NAS_and_git_annex/comment_1_ef7e19f1fd2005eb7cc74509ffb92766._comment
@@ -0,0 +1,9 @@
+[[!comment format=mdwn
+ username="https://www.google.com/accounts/o8/id?id=AItOawnrP-0DGtHDJbWSXeiyk0swNkK1aejoN3c"
+ nickname="sebastien"
+ subject="new dev site for syno dsm 5.0"
+ date="2014-03-25T10:20:46Z"
+ content="""
+There is a new website with documentation to create App for new synology dsm 5.0 here : http://www.synology.com/en-us/support/developer
+A good way to distribute git annex to lot of people ? :)
+"""]]
diff --git a/doc/tips/Synology_NAS_and_git_annex/comment_2_5e723ccf026fe970ad31207f9f036b69._comment b/doc/tips/Synology_NAS_and_git_annex/comment_2_5e723ccf026fe970ad31207f9f036b69._comment
new file mode 100644
index 0000000000..60122bb137
--- /dev/null
+++ b/doc/tips/Synology_NAS_and_git_annex/comment_2_5e723ccf026fe970ad31207f9f036b69._comment
@@ -0,0 +1,30 @@
+[[!comment format=mdwn
+ username="https://www.google.com/accounts/o8/id?id=AItOawllyDAwjTPuM6G2d3eKE481V0qGXRKtF8s"
+ nickname="Pieter"
+ subject="Syncing metadata only with Synology NAS and git annex"
+ date="2014-10-06T12:03:35Z"
+ content="""
+I am not able to get full syncing working. It says \"syncing enabled (metadata only)\".
+When I click on Actions -> Edit it says:
+
+Just a git repository. 
+
+This repository is not currently set up as a git annex; only git metadata is synced with this repository.
+
+If this repository's ssh server has git-annex installed, you can upgrade this repository to a full git annex, which will store the contents of your files, not only their metadata.
+
+When I try to upgrade the repository it does not work. The log says:
+
+sh: git-annex-shell: not found
+
+rsync: connection unexpectedly closed (0 bytes received so far) [sender]
+
+rsync error: remote command not found (code 127) at io.c(226) [sender=3.1.1]
+
+I'm using Version: 5.20140717 on the Linux Ubuntu 14.10.
+
+Ssh'ing from Ubuntu to gituser@synology works fine and shows the git-annex-shell options
+
+Any ideas?
+
+"""]]
diff --git a/doc/tips/Synology_NAS_and_git_annex/comment_3_8beb2b4b79c7787a92689aaad3bfc452._comment b/doc/tips/Synology_NAS_and_git_annex/comment_3_8beb2b4b79c7787a92689aaad3bfc452._comment
new file mode 100644
index 0000000000..681b001d02
--- /dev/null
+++ b/doc/tips/Synology_NAS_and_git_annex/comment_3_8beb2b4b79c7787a92689aaad3bfc452._comment
@@ -0,0 +1,10 @@
+[[!comment format=mdwn
+ username="http://joeyh.name/"
+ ip="209.250.56.54"
+ subject="comment 3"
+ date="2014-10-06T15:51:38Z"
+ content="""
+Pieter, I suspect you didn't follow the part of the instructions where it says to run \"./runshell\" on the NAS. If you didn't do that, there will be no ~/.ssh/git-annex-shell script set up.
+
+You have put git-annex-shell in your PATH somehow, but you probably did it by editing a bash dotfile. Those files are used for interactive login shells, but not when git-annex is sshing noninteractively into the NAS to run git-annex-shell.
+"""]]
diff --git a/doc/tips/Synology_NAS_and_git_annex/comment_4_b04a1245378d3580432e85dff8eefdec._comment b/doc/tips/Synology_NAS_and_git_annex/comment_4_b04a1245378d3580432e85dff8eefdec._comment
new file mode 100644
index 0000000000..b0eb9bdfa6
--- /dev/null
+++ b/doc/tips/Synology_NAS_and_git_annex/comment_4_b04a1245378d3580432e85dff8eefdec._comment
@@ -0,0 +1,26 @@
+[[!comment format=mdwn
+ username="https://www.google.com/accounts/o8/id?id=AItOawnjDzZkA6SzZHgT5_Jy3Rw3s7m_W3-oNLY"
+ nickname="Antoni"
+ subject="Process seems to hang on Synology"
+ date="2015-01-22T20:18:12Z"
+ content="""
+I think I've completed the setup correctly, as I
+
+* am able to execute git-annex-shell on the server
+* locally running ``git-annex sync`` triggers the git-annex process on the server
+
+However, once I provide the password to the key on my local machine, nothing seemingly happens - there is no output after the password prompt.
+
+The repository I'm trying to sync with the remote on the server is ~200 mb and is in my lan.
+
+When monitoring the remote server, I see the following in process being triggered:
+
+````
+31433 31431 amk      D     656m270.7  7.1 /volume1/homes/amk/bin/git-annex.linux//lib/ld-linux.so.3 --library-path /volume1/homes/amk/bin/git-annex.linux//etc/ld.so.conf.d:/volume1/homes/amk/bin/git-annex.linux//usr/lib/arm-linux-gnueabi/audit:/volume1/homes/amk/bin/git-annex.linux//usr/lib/arm-linux-gnueabi/gconv:/volume1/homes/amk/bin/git-annex.linux//usr/lib:/volume1/homes/amk/bin/git-annex.linux//usr/lib/arm-linux-gnueabi:/volume1/homes/amk/bin/git-annex.linux//lib:/volume1/homes/amk/bin/git-annex.linux//lib/arm-linux-gnueabi: /volume1/homes/amk/bin/git-annex.linux/shimmed/git-annex-shell/git-annex-shell -c git-annex-shell 'configlist' '/volume1/homes/amk/archive/annex'
+````
+
+The memory usage goes up graudally, reaching well over 200mb, hogging the cpu significantly. I have never used git-annex before, so I'm not sure what else I should be looking for. This is what the server runs on:
+``armv7l GNU/Linux synology_armada370_214se``
+
+Is this an issue caused by the host architecture mismatch? 
+"""]]
diff --git a/doc/tips/Synology_NAS_and_git_annex/comment_5_7e4ec2b22fb15e653d3b2274493e460c._comment b/doc/tips/Synology_NAS_and_git_annex/comment_5_7e4ec2b22fb15e653d3b2274493e460c._comment
new file mode 100644
index 0000000000..af36ba5131
--- /dev/null
+++ b/doc/tips/Synology_NAS_and_git_annex/comment_5_7e4ec2b22fb15e653d3b2274493e460c._comment
@@ -0,0 +1,11 @@
+[[!comment format=mdwn
+ username="joey"
+ subject="""comment 5"""
+ date="2015-02-04T19:16:36Z"
+ content="""
+@Antoni git-annex-shell configlist does not do anything that would
+normally make it use much memory at all.
+
+You might find strace or similar tools useful to figure out what's going
+on.
+"""]]
diff --git a/doc/tips/Synology_NAS_and_git_annex/comment_6_954fc1b423208928fafdeb8cdac47c50._comment b/doc/tips/Synology_NAS_and_git_annex/comment_6_954fc1b423208928fafdeb8cdac47c50._comment
new file mode 100644
index 0000000000..b9d037bd1d
--- /dev/null
+++ b/doc/tips/Synology_NAS_and_git_annex/comment_6_954fc1b423208928fafdeb8cdac47c50._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="joey"
+ subject="""comment 6"""
+ date="2015-02-11T17:05:15Z"
+ content="""
+There's a bug report for the memory use hang now: 
+[[bugs/git-annex_on_NAS_eats_all_memory]]
+"""]]
diff --git a/doc/tips/Synology_NAS_and_git_annex/comment_7_b862cb6ad0f912cc2ffede21df10c2fe._comment b/doc/tips/Synology_NAS_and_git_annex/comment_7_b862cb6ad0f912cc2ffede21df10c2fe._comment
new file mode 100644
index 0000000000..15b66d40a6
--- /dev/null
+++ b/doc/tips/Synology_NAS_and_git_annex/comment_7_b862cb6ad0f912cc2ffede21df10c2fe._comment
@@ -0,0 +1,7 @@
+[[!comment format=mdwn
+ username="Jan"
+ subject="git-annex on a PowerPC based synology?"
+ date="2016-04-12T15:41:24Z"
+ content="""
+I use a DS210 which is based on the PowerPC e500 CPU. Has anyone successfully compiled git-annex  for that architecture?
+"""]]
diff --git a/doc/tips/Synology_NAS_and_git_annex/comment_8_38775efe367033a9a96cf98a49f8cdde._comment b/doc/tips/Synology_NAS_and_git_annex/comment_8_38775efe367033a9a96cf98a49f8cdde._comment
new file mode 100644
index 0000000000..45b27df189
--- /dev/null
+++ b/doc/tips/Synology_NAS_and_git_annex/comment_8_38775efe367033a9a96cf98a49f8cdde._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="joey"
+ subject="""comment 8"""
+ date="2016-04-12T19:28:15Z"
+ content="""
+@Jan, there is a powerpc build of git-annex in Debian. Not sure if it
+targets the e500 CPU.
+"""]]
diff --git a/doc/tips/Synology_NAS_and_git_annex/comment_9_52e1684fc40d1ab487a06cf13530871b._comment b/doc/tips/Synology_NAS_and_git_annex/comment_9_52e1684fc40d1ab487a06cf13530871b._comment
new file mode 100644
index 0000000000..452f68fe4d
--- /dev/null
+++ b/doc/tips/Synology_NAS_and_git_annex/comment_9_52e1684fc40d1ab487a06cf13530871b._comment
@@ -0,0 +1,7 @@
+[[!comment format=mdwn
+ username="Jan"
+ subject="comment 9"
+ date="2016-04-13T09:05:23Z"
+ content="""
+I doubt that the Debian package would be usable as-is on a Synology NAS? Perhaps it would require a standalone package, similar to armel but for PowerPC?
+"""]]
diff --git a/doc/tips/Systemd_unit.mdwn b/doc/tips/Systemd_unit.mdwn
new file mode 100644
index 0000000000..f56e20dfee
--- /dev/null
+++ b/doc/tips/Systemd_unit.mdwn
@@ -0,0 +1,103 @@
+---
+---
+
+/!\ __THIS PAGE IS A DRAFT__ /!\
+
+---
+---
+
+## Introduction
+
+Systemd is a suite of tools for system and user daemon management.
+
+It can be used as an alternative to XDG autostart files to start the git-annex daemon and optionally the webapp, either at startup (system service) or when an user logs in (user service).
+
+---
+
+##Setup
+
+### User service
+
+Sample unit file (`/etc/systemd/user/git-annex.service`):
+
+```
+[Unit]
+Description=git-annex assistant daemon
+
+[Service]
+ExecStart=/usr/bin/git-annex assistant --autostart --foreground
+Restart=on-failure
+
+[Install]
+WantedBy=default.target
+```
+
+Commands for enabling and starting the service as the current user:
+
+```
+systemctl --user enable git-annex.service
+systemctl --user start git-annex.service
+```
+
+Usually services in `default.target` start during login. (Note however that they also _delay_ the login process.) However, if you enable "linger" via `loginctl`, then these services start on boot instead.
+
+### System service
+
+If for some reason you cannot use `systemd --user`, the other option is to have system-wide services:
+
+Sample unit file (`/etc/systemd/system/git-annex@.service`):
+
+```
+[Unit]
+Description=git-annex assistant daemon
+After=network.target
+
+[Service]
+User=%i
+ExecStart=/usr/bin/git-annex assistant --autostart --foreground
+Restart=on-failure
+
+[Install]
+WantedBy=multi-user.target
+```
+
+Commands for enabling and starting the service as user `u`:
+
+```
+systemctl enable git-annex@u.service
+systemctl start git-annex@u.service
+
+```
+
+---
+
+## Considerations
+
+### Webapp
+
+The webapp may be started instead of the assistant, by launching the associated command in the unit file. This will run an associated assistant automatically. However, it may also attempt to open a potentially unwanted browser window.
+
+```
+ExecStart=git-annex webapp
+```
+
+_TODO: try this._
+
+### Encrypted home directory
+
+Users may store their keyring and repositories in their encrypted home directory mounted at login. This may break a system service running at boot.
+
+_TODO: try this._
+
+### Common daemon
+
+One daemon may be used to sync repositories for multiple users. For this, it might be helpful to make use of ACLs to access other user directories.
+
+---
+
+## References
+
+- [systemd.unit man page](https://www.freedesktop.org/software/systemd/man/systemd.unit.html)
+- [systemd on ArchWiki](https://wiki.archlinux.org/index.php/Systemd)
+- [TL;DR Syncthing with systemd documentation](https://docs.syncthing.net/users/autostart.html#using-systemd)
+- [Syncthing systemd unit files](https://github.com/syncthing/syncthing/tree/master/etc/linux-systemd)
diff --git a/doc/tips/Systemd_unit/comment_1_06399a293f08401032bef1b94f05547c._comment b/doc/tips/Systemd_unit/comment_1_06399a293f08401032bef1b94f05547c._comment
new file mode 100644
index 0000000000..7af170ef1c
--- /dev/null
+++ b/doc/tips/Systemd_unit/comment_1_06399a293f08401032bef1b94f05547c._comment
@@ -0,0 +1,25 @@
+[[!comment format=mdwn
+ username="oberix@c7a19cddb1663df0c612a979b9d13b0d67f1f69a"
+ nickname="oberix"
+ avatar="http://cdn.libravatar.org/avatar/e8b871f3d0bf96df9a3fc8cdca7abe09"
+ subject="autostart and foreground together doesn't seem to work"
+ date="2017-03-30T10:43:18Z"
+ content="""
+With systemd using `--autostart --foreground` either ignore foreground or quit immediatelly.
+
+I managed to have the process stay alive with `RemainAfterExit=on`:
+
+    [Service]
+    User=%i
+    ExecStart=/usr/bin/git-annex assistant --autostart --foreground
+    ExecStop=/usr/bin/git-annex assistant --autostop
+    RemainAfterExit=on
+    Restart=on-failure
+    RestartSec=5
+
+but git-annex processes does not maintain the `--foreground` option which is causing a lot of zombies in the long period (not totally clear why).
+
+My current solution is to have a service for each annex repository and avoid `--autosart` but this is annoying because it require to pass the path as `%I` and wrap git-annex in bash script to get the repo owner as the user.
+
+
+"""]]
diff --git a/doc/tips/Systemd_unit/comment_2_389a2b36c3115eb3342429b0b68ddef2._comment b/doc/tips/Systemd_unit/comment_2_389a2b36c3115eb3342429b0b68ddef2._comment
new file mode 100644
index 0000000000..ed61b19ebf
--- /dev/null
+++ b/doc/tips/Systemd_unit/comment_2_389a2b36c3115eb3342429b0b68ddef2._comment
@@ -0,0 +1,9 @@
+[[!comment format=mdwn
+ username="joey"
+ subject="""comment 2"""
+ date="2017-04-05T15:43:21Z"
+ content="""
+@oberix, the --autostart --foreground combination is only supported
+properly since git-annex version 6.20170214. Before that, the --forground
+was ignored when using --autostart.
+"""]]
diff --git a/doc/tips/The_perfect_preferred_content_settings_for_my_android_phone.mdwn b/doc/tips/The_perfect_preferred_content_settings_for_my_android_phone.mdwn
new file mode 100644
index 0000000000..ccae0fced2
--- /dev/null
+++ b/doc/tips/The_perfect_preferred_content_settings_for_my_android_phone.mdwn
@@ -0,0 +1,36 @@
+I have an annex that syncs my personal files on all my computers. It works great. Phones are different.
+
+For one, everything's a bit slower to sync, there's battery considerations, and I just don't need every last old file on my phone. Then there's some files I explicitly don't want on my phone in case it gets lost, like family pictures, passport scans, or private keys.
+
+But I still want photos, videos and voice recordings I make on my phone to be synced to my server. A transfer repo would work, but I want to keep them. Then there's my PDF book collection; that would certainly be nice to always have around in case I have half on hour on a bus. And my music collection ought to be around as well.
+
+So I came up with this solution, and I'm very happy with it.
+
+    include=Music/* or include=Books/* or present
+
+This will sync my music and book collections to my phone whenever I add something new on my computers, and it will sync and keep anything I add to the annex on my phone. Best of all worlds! Impressed how flexible preferred content is. More full-sync folders can be added like this: 
+
+    include=Music/* or include=Books/* or include = Notes/* or present
+
+To add them, I first had to figure out the uuid of my phone repo. So I added a new tab on android, and did
+
+    cd /sdcard/annex
+    git config annex.uuid
+
+Then I went to one of my computers, and did
+
+    git annex vicfg
+
+And changed the line
+
+    content [phone-uuid] = standard
+
+to
+
+    content [phone-uuid] = include=Music/* or include=Books/* or Notes/* or present
+
+and commented out
+
+    #group [phone-uuid] = client
+
+And waited for it to sync.
diff --git a/doc/tips/The_perfect_preferred_content_settings_for_my_android_phone/comment_1_393d1636bb313530be383a075bd3440a._comment b/doc/tips/The_perfect_preferred_content_settings_for_my_android_phone/comment_1_393d1636bb313530be383a075bd3440a._comment
new file mode 100644
index 0000000000..8938684f89
--- /dev/null
+++ b/doc/tips/The_perfect_preferred_content_settings_for_my_android_phone/comment_1_393d1636bb313530be383a075bd3440a._comment
@@ -0,0 +1,14 @@
+[[!comment format=mdwn
+ username="http://joeyh.name/"
+ ip="209.250.56.246"
+ subject="comment 1"
+ date="2013-11-16T17:29:03Z"
+ content="""
+That's great, that's how I hoped people would be able to use preferred content settings.
+
+I'd suggest adding support for archive directories to this. So if you create a file on the phone and are done with it, you can move it to an archive directory, and it will then be dropped from the phone once it reaches an archive repository. 
+
+This should accomplish that. (Untested)
+
+`((exclude=*/archive/* and exclude=archive/*) or (not (copies=archive:1 or copies=smallarchive:1))) and (include=Music/* or include=Books/* or present)`
+"""]]
diff --git a/doc/tips/The_perfect_preferred_content_settings_for_my_android_phone/comment_2_51a013213118660bdc06ff4d6c8110ba._comment b/doc/tips/The_perfect_preferred_content_settings_for_my_android_phone/comment_2_51a013213118660bdc06ff4d6c8110ba._comment
new file mode 100644
index 0000000000..1b84eee813
--- /dev/null
+++ b/doc/tips/The_perfect_preferred_content_settings_for_my_android_phone/comment_2_51a013213118660bdc06ff4d6c8110ba._comment
@@ -0,0 +1,12 @@
+[[!comment format=mdwn
+ username="https://www.google.com/accounts/o8/id?id=AItOawnXlmnJKsPulm2S_hwwRm3Ky27Zyf-wKMw"
+ nickname="Chad"
+ subject="Pulling *.flac files as well... too much data"
+ date="2013-12-31T19:26:36Z"
+ content="""
+I've tried setting up my own phone in a similar manner. Unfortunately for me, I store my `*.flac` files in the same location as my `*.mp3` files. For obvious reasons, I do not want to copy these up to my phone. This, however, is not working for me.
+
+    content PHONEUID = ((exclude=*/archive/* and exclude=archive/*) or (not (copies=archive:1 or copies=smallarchive:1))) and (include=keepass/* or include=Books/* or present) and (include=Music/* and exclude=Music/*.flac and exclude=Music/*/*.flac)
+
+I still get `*.flac` files in the repo.
+"""]]
diff --git a/doc/tips/Using_Git-annex_as_a_web_browsing_assistant.mdwn b/doc/tips/Using_Git-annex_as_a_web_browsing_assistant.mdwn
new file mode 100644
index 0000000000..e8239abd40
--- /dev/null
+++ b/doc/tips/Using_Git-annex_as_a_web_browsing_assistant.mdwn
@@ -0,0 +1,40 @@
+[[todo/wishlist: an "assistant" for web-browsing -- tracking the sources of the downloads]] suggests using git-annex as a tool to store downloads tied
+to their URLs. This also enables people to have their files stored offline,
+while being able to git annex drop them at any time and redownload them
+with git annex get. Additionally, a clone of the repo can be used to
+download whatever files are desired from online.
+
+This tip explains how to implement a similar system to the one described in
+the linked wishlist with existing software and features of git-annex.
+
+The first step is to install the Firefox plugin
+[FlashGot](http://flashgot.net/). We will use it to provide the Firefox
+shortcuts to add things to our annex.
+
+Once we have installed all that, we need a script that has an interface
+which FlashGot can treat as a downloader, but which calls git-annex to do
+the actual downloading. Such a script is available from
+. Download it and store it
+somewhere it can live, or cut and paste:
+
+[[!format sh """
+#!/bin/bash
+# $1=folder to cd to (must be a git annex repo)
+# $2=URL to download
+
+cd "$1"
+git-annex addurl "$2"
+"""]]
+
+Finally, we need to configure FlashGot to use the script as a downloader.
+Go to Tools > Add-ons in Firefox. Click "Preferences" on FlashGot. Click
+the Add button next to the list of download managers. Enter a name for the
+git-annex downloader. Choose the script that was downloaded from the
+"Locate executable file" dialog that appears. Now set the command line
+arguments template to be "[FOLDER] [URL]" (you can find more substitution
+expressions in the Placeholders dropdown above the Command line arguments
+template field). You're done!
+
+Go ahead and test it by trying to download a file using FlashGot. It should
+offer as one of its available download managers the new manager you created
+just above. Select it and have fun!
diff --git a/doc/tips/ZSH_completion.mdwn b/doc/tips/ZSH_completion.mdwn
new file mode 100644
index 0000000000..f3ececd466
--- /dev/null
+++ b/doc/tips/ZSH_completion.mdwn
@@ -0,0 +1,13 @@
+ZSH users, here's some good news: after 2 years of silence, the completion function for git-annex has been updated. It now supports *all* git-annex commands (as of 5.20140517) and has many improvements for completing arguments, remotes, groups, and backends.
+
+To install it:
+
+1. make sure your have Python 3 installed (as `python3` somewhere in your `$PATH`; tested with 3.4, should work with 3.2+)
+2. get it from [GitHub](https://github.com/Schnouki/git-annex-zsh-completion)
+3. copy `_git-annex` to somewhere in your `$fpath` (I use `$HOME/.config/zsh/completion`)
+4. run `autoload -U path/to/_git-annex`
+5. type `git annex `
+
+This is very far from being perfect, but it's (IMHO) better than nothing. If you have any issue or suggestion, please [tell me](https://github.com/Schnouki/git-annex-zsh-completion/issues)!
+
+Many thanks to Frank Terbeck and Valentin Haenel, the original authors of this completion function ([source](https://github.com/esc/git-annex-zsh-completion)).
diff --git a/doc/tips/ZSH_completion/comment_1_a810be47633dc83d737a9ef6e8870431._comment b/doc/tips/ZSH_completion/comment_1_a810be47633dc83d737a9ef6e8870431._comment
new file mode 100644
index 0000000000..c3c12ab318
--- /dev/null
+++ b/doc/tips/ZSH_completion/comment_1_a810be47633dc83d737a9ef6e8870431._comment
@@ -0,0 +1,12 @@
+[[!comment format=mdwn
+ username="joey"
+ subject="""comment 1"""
+ date="2017-06-09T15:34:59Z"
+ content="""
+When git-annex is built with optparse-applicative version 0.14.0.0,
+it can generate its own zsh completion file. The Makefile installs that,
+and it seems to work well.
+
+So, I think that the hand-built zsh completions won't need to be maintained
+much longer!
+"""]]
diff --git a/doc/tips/ZSH_completion/comment_4_0e6f7035350ec0c9c61eeafb02ab7af7._comment b/doc/tips/ZSH_completion/comment_4_0e6f7035350ec0c9c61eeafb02ab7af7._comment
new file mode 100644
index 0000000000..bc35bf98a0
--- /dev/null
+++ b/doc/tips/ZSH_completion/comment_4_0e6f7035350ec0c9c61eeafb02ab7af7._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="Lykos153"
+ avatar="http://cdn.libravatar.org/avatar/085df7b04d3408ba23c19f9c49be9ea2"
+ subject="comment 4"
+ date="2017-11-09T17:35:14Z"
+ content="""
+Well I guess that depends on habit/workflow. I often let tab completion show me the directory content before beginning to type. Thanks for the links, I'll look into it (though I have no experience with Haskell yet).
+"""]]
diff --git a/doc/tips/ZSH_completion/comment_4_2bdaf6a00723b975808a52863e417321._comment b/doc/tips/ZSH_completion/comment_4_2bdaf6a00723b975808a52863e417321._comment
new file mode 100644
index 0000000000..3c037688a2
--- /dev/null
+++ b/doc/tips/ZSH_completion/comment_4_2bdaf6a00723b975808a52863e417321._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="Lykos153"
+ avatar="http://cdn.libravatar.org/avatar/085df7b04d3408ba23c19f9c49be9ea2"
+ subject="comment 4"
+ date="2017-11-09T02:15:37Z"
+ content="""
+Doesn't work so well for me. The hand-build one performs way better in terms of clearness and efficiency. It groups the suggestions instead of sorting them by name and it prints a description for each command. But most importently, it does only suggest switches when `-` was typed and completes file names otherwise. The auto generated one (at least the one that is shipped with the Arch Linux package) puts the file names last, which makes it unusable.
+"""]]
diff --git a/doc/tips/ZSH_completion/comment_5_182c13292016baf46d8f53a306cb4b05._comment b/doc/tips/ZSH_completion/comment_5_182c13292016baf46d8f53a306cb4b05._comment
new file mode 100644
index 0000000000..9fd7b628fb
--- /dev/null
+++ b/doc/tips/ZSH_completion/comment_5_182c13292016baf46d8f53a306cb4b05._comment
@@ -0,0 +1,15 @@
+[[!comment format=mdwn
+ username="joey"
+ subject="""comment 2"""
+ date="2017-11-09T16:11:10Z"
+ content="""
+Well, the code that generates the zsh completions could
+always be improved.
+
+
+I don't really understand why completing options before filenames makes
+it unusable. It means that "git-annex tabtabtab" cycles through options
+first, but if you type part of the filename tab will complete the rest.
+It does seem that someone else agrees with you though, as they filed
+this issue: 
+"""]]
diff --git a/doc/tips/a_gui_for_metadata_operations.mdwn b/doc/tips/a_gui_for_metadata_operations.mdwn
new file mode 100644
index 0000000000..1e11800682
--- /dev/null
+++ b/doc/tips/a_gui_for_metadata_operations.mdwn
@@ -0,0 +1,13 @@
+Hey everyone.
+
+I wrote a GUI for git-annex metadata in Python: [git-annex-metadata-gui](https://github.com/alpernebbi/git-annex-metadata-gui). 
+It shows the files that are in the current branch (only those in the annex) in the respective folder hierarchy.
+The keys that are in the repository, but not in the current branch are also shown in another tab.
+You can view, edit or remove fields for individual files with support for multiple values for fields.
+There is a file preview for image and text files as well.
+I uploaded some screenshots in the repository to show it in action.
+
+While making it, I decided to move the git-annex calls into its own Python package, 
+which became [git-annex-adapter](https://github.com/alpernebbi/git-annex-adapter).
+
+I hope these can be useful to someone other than myself as well.
diff --git a/doc/tips/a_gui_for_metadata_operations/comment_1_1ce311d8328ea370a6a3494adea0f5db._comment b/doc/tips/a_gui_for_metadata_operations/comment_1_1ce311d8328ea370a6a3494adea0f5db._comment
new file mode 100644
index 0000000000..2a55de0beb
--- /dev/null
+++ b/doc/tips/a_gui_for_metadata_operations/comment_1_1ce311d8328ea370a6a3494adea0f5db._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="joey"
+ subject="""comment 1"""
+ date="2016-12-07T19:58:11Z"
+ content="""
+Thank you for this, I've always wanted such a GUI, and it's been a common
+user request!
+"""]]
diff --git a/doc/tips/antipatterns.mdwn b/doc/tips/antipatterns.mdwn
new file mode 100644
index 0000000000..0beed5b005
--- /dev/null
+++ b/doc/tips/antipatterns.mdwn
@@ -0,0 +1,150 @@
+This page tries to regroup a set of Really Bad Ideas people had with
+git-annex in the past that can lead to catastrophic data loss, abusive
+disk usage, improper swearing and other unfortunate experiences.
+
+This could also be called the "git annex worst practices", but is
+different than [[what git annex is not|not]] in that it covers normal
+use cases of git-annex, just implemented in the wrong way. Hopefully,
+git-annex should make it as hard as possible to do those things, but
+sometimes, you just can't help it, people figure out the worst
+possible ways of doing things.
+
+[[!toc]]
+
+---
+
+# **Symlinking the `.git/annex` directory**
+
+Symlinking the `.git/annex` directory, in the hope of saving
+disk space, is a horrible idea. The general antipattern is:
+
+    git clone repoA repoB
+    mv repoB/.git/annex repoB/.git/annex.bak
+    ln -s repoA/.git/annex repoB/.git/annex
+
+This is bad because git-annex will believe it has two copies of the
+files and then would let you drop the single copy, therefore leading
+to data loss.
+
+Proper pattern
+--------------
+
+The proper way of doing this is through git-annex's hardlink support,
+by cloning the repository with the `--shared` option:
+
+    git clone --shared repoA repoB
+
+This will setup repoB as an "untrusted" repository and use hardlinks
+to copy files between the two repos, using space only once. This
+works, of course, only on filesystems that support hardlinks, but
+that's usually the case for filesystems that support symlinks.
+
+Alternatively, `git worktree` can be used to add another worktree to a git
+repository. This way, multiple worktrees can share the same git-annex
+object store.
+
+Real world cases
+----------------
+
+ * [[forum/share_.git__47__annex__47__objects_across_multiple_repositories_on_one_machine/]]
+ * at least one IRC discussion
+
+Fixes
+-----
+
+Probably no way to fix this in git-annex - if users want to shoot
+themselves in the foot by messing with the backend, there's not much
+we can do to change that in this case.
+
+---
+
+# **Reinit repo with an existing uuid without `fsck`**
+
+To quote the [[git-annex-reinit]] manpage:
+
+> Normally, initializing a repository generates a new, unique
+> identifier (UUID) for that repository. Occasionally it may be useful
+> to reuse a UUID -- for example, if a repository got deleted, and
+> you're setting it back up.
+
+[[git-annex-reinit]] can be used to reuse UUIDs for deleted
+repositories. But what happens if you reuse the UUID of an *existing*
+repository, or a repository that hasn't been properly emptied before
+being declared dead? This can lead to git-annex getting confused
+because, in that case, git-annex may think some files are still
+present in the revived repository (while they may not actually be).
+
+This should never result in data loss, because git-annex does not
+trust its records about the contents of a repository, and checks
+that it really contains files before dropping them from other
+repositories. (The one exception to this rule is trusted repositories,
+whose contents are never checked. See the next two sections for more
+about problems with trusted repositories.)
+
+Proper pattern
+--------------
+
+The proper way of using reinit is to make sure you run
+[[git-annex-fsck]] (optionally with `--fast` to save time) on the
+revived repo right after running reinit. This will ensure that at
+least the location log will be updated, and git-annex will notice if
+files are missing.
+
+Real world cases
+----------------
+
+ * [[bugs/remotes_disappeared]]
+
+Fixes
+-----
+
+An improvement to git-annex here would be to allow
+[[reinit to work without arguments|todo/reinit_should_work_without_arguments]]
+to at least not encourage UUID reuse.
+
+# **Deleting data from trusted repositories**
+
+When you use [[git-annex-trust]] on a repository, you disable
+some very important sanity checks that make sure that git-annex
+never loses the content of files. So trusting a repository
+is a good way to shoot yourself in the foot and lose data. Like the
+man page says, "Use with care."
+
+When you have made git-annex trust a repository, you can lose data
+by dropping files from that repository. For example, suppose file `foo` is
+present in the trusted repository, and also in a second repository.
+
+Now suppose you run `git annex drop foo` in both repositories.
+Normally, git-annex will not let both copies of the file be removed,
+but if the trusted repository is able to verify that the second
+repository has a copy, it will delete its copy. Then the drop in the second
+repository will *trust* the trusted repository still has its copy,
+and so the last copy of the file gets deleted.
+
+Proper pattern
+--------------
+
+Either avoid using trusted repositories, or avoid dropping content
+from them, or make sure you `git annex sync` just right, so
+other reposities know that data has been removed from a trusted repository.
+
+# **Deleting trusted repositories**
+
+Another way trusted repositories are unsafe is that even after they're
+deleted, git-annex will trust that they contained the files they
+used to contain.
+
+Proper pattern
+--------------
+
+Always use [[git-annex-dead]] to tell git-annex when a repository has
+been deleted, especially if it was trusted.
+
+Other cases
+===========
+
+Feel free to add your lessons in catastrophe here! It's educational
+and fun, and will improve git-annex for everyone.
+
+PS: should this be a toplevel page instead of being drowned in the
+[[tips]] section? Where should it be linked to?  -- [[anarcat]]
diff --git a/doc/tips/assume-unstaged.mdwn b/doc/tips/assume-unstaged.mdwn
new file mode 100644
index 0000000000..63f5f820a5
--- /dev/null
+++ b/doc/tips/assume-unstaged.mdwn
@@ -0,0 +1,31 @@
+[[!meta title="using assume-unstages to speed up git with large trees of annexed files"]]
+
+Git update-index's assume-unstaged feature can be used to speed
+up `git status` and stuff by not statting the whole tree looking for changed
+files.
+
+This feature works quite well with git-annex. Especially because git
+annex's files are immutable, so aren't going to change out from under it,
+this is a nice fit. If you have a very large tree and `git status` is
+annoyingly slow, you can turn it on:
+
+	git config core.ignoreStat true
+
+When `git mv` and `git rm` are used, those changes *do* get noticed, even
+on assume-unchanged files. When new files are added, eg by `git annex add`,
+they are also noticed.
+
+There are two gotchas. Both occur because `git add` does not stage
+assume-unchanged files.
+
+1. When an annexed file is moved to a different directory, it updates
+   the symlink, and runs `git add` on it. So the file will move,
+   but the changed symlink will not be noticed by git and it will commit a
+   dangling symlink.
+2. When using `git annex migrate`, it changes the symlink and `git adds`
+   it. Again this won't be committed.
+
+These can be worked around by running `git update-index --really-refresh`
+after performing such operations. I hope that `git add` will be changed
+to stage changes to assume-unchanged files, which would remove this
+only complication. --[[Joey]] 
diff --git a/doc/tips/assume-unstaged/comment_1_44abd811ef79a85e557418e17a3927be._comment b/doc/tips/assume-unstaged/comment_1_44abd811ef79a85e557418e17a3927be._comment
new file mode 100644
index 0000000000..d253feb5b8
--- /dev/null
+++ b/doc/tips/assume-unstaged/comment_1_44abd811ef79a85e557418e17a3927be._comment
@@ -0,0 +1,10 @@
+[[!comment format=mdwn
+ username="https://me.yahoo.com/a/2djv2EYwk43rfJIAQXjYt_vfuOU-#a11a6"
+ nickname="Olivier R"
+ subject="It doesn't work 100%"
+ date="2012-05-03T21:42:54Z"
+ content="""
+When you remove tracked files... it doesn't show the new status. it's like if the file was ignored.
+
+
+"""]]
diff --git a/doc/tips/assume-unstaged/comment_2_5b589f37cfc03bf7be33a51826cc4dba._comment b/doc/tips/assume-unstaged/comment_2_5b589f37cfc03bf7be33a51826cc4dba._comment
new file mode 100644
index 0000000000..474d7b3995
--- /dev/null
+++ b/doc/tips/assume-unstaged/comment_2_5b589f37cfc03bf7be33a51826cc4dba._comment
@@ -0,0 +1,13 @@
+[[!comment format=mdwn
+ username="https://www.google.com/accounts/o8/id?id=AItOawnxlx1UrzVhdy6_gFjzmF42x6QXxBUxg00"
+ nickname="Jakukyo"
+ subject="comment 2"
+ date="2013-09-05T12:14:42Z"
+ content="""
+> There are two gotchas...
+
+So just always run `git annex add` after editing a file
+and `git update-index --really-refresh` after migrating
+backend?
+
+"""]]
diff --git a/doc/tips/automatically_adding_metadata.mdwn b/doc/tips/automatically_adding_metadata.mdwn
new file mode 100644
index 0000000000..b3118a75ed
--- /dev/null
+++ b/doc/tips/automatically_adding_metadata.mdwn
@@ -0,0 +1,49 @@
+git-annex's [[metadata]] works best when files have a lot of useful
+metadata attached to them.
+
+To make git-annex automatically set the year and month when adding files,
+run: `git config annex.genmetadata true`
+
+## git commit hook
+
+A git commit hook can be set up to extract lots of metadata from files
+like photos, mp3s, etc. Whenever annexed files are committed, their
+metadata will be extracted and stored.
+
+Download [[pre-commit-annex]] and install it in your git-annex repository
+as `.git/hooks/pre-commit-annex`  
+Remember to make the script executable! `chmod +x .git/hooks/pre-commit-annex`
+
+### using extract
+
+The git commit hook can use extract to get metadata.
+
+Install it from   
+`apt-get install extract`
+
+Configure which metadata fields to ask extract for: `git config metadata.extract "artist album title camera_make video_dimensions"`
+
+To get a list of all possible fields, run: `extract -L | sed 's/ /_/g'`
+
+### using exiftool
+
+The git commit hook can also use exiftool to get metadata.
+
+Install it from   
+`apt-get install libimage-exiftool-perl`
+
+Configure which metadata fields to ask exiftool for: `git config metadata.exiftool "Model ImageSize FocusRange GPSAltitude GPSCoordinates"`
+
+To get a list of all possible fields, run: `exiftool -list`
+
+### using both extract and exiftool
+
+If you want some metadata that extract knows about, and other metadata
+that exiftool knows about, just install them both, and set both
+`metadata.extract` and `metadata.exiftool`.
+
+### overwriting existing metadata
+
+By default, if a git-annex already has a metadata field for a file,
+its value will not be overwritten with metadata taken from files.
+To allow overwriting, run: `git config metadata.overwrite true`
diff --git a/doc/tips/automatically_adding_metadata/comment_1_ffc308cc6aedabbc55820db4f401e0fb._comment b/doc/tips/automatically_adding_metadata/comment_1_ffc308cc6aedabbc55820db4f401e0fb._comment
new file mode 100644
index 0000000000..59f0e958a9
--- /dev/null
+++ b/doc/tips/automatically_adding_metadata/comment_1_ffc308cc6aedabbc55820db4f401e0fb._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="https://www.google.com/accounts/o8/id?id=AItOawkSq2FDpK2n66QRUxtqqdbyDuwgbQmUWus"
+ nickname="Jimmy"
+ subject="comment 1"
+ date="2014-03-03T07:44:29Z"
+ content="""
+http://projects.iq.harvard.edu/fits might be an even better choice than libextractor. We use it in work and its not too bad, but it can be slow to startup due to the JVM.
+"""]]
diff --git a/doc/tips/automatically_adding_metadata/comment_2_bd64a53914107bc000c887b4d4bdf6af._comment b/doc/tips/automatically_adding_metadata/comment_2_bd64a53914107bc000c887b4d4bdf6af._comment
new file mode 100644
index 0000000000..13b3865e10
--- /dev/null
+++ b/doc/tips/automatically_adding_metadata/comment_2_bd64a53914107bc000c887b4d4bdf6af._comment
@@ -0,0 +1,12 @@
+[[!comment format=mdwn
+ username="https://id.koumbit.net/anarcat"
+ ip="2001:1928:1:9::1"
+ subject="comment 2"
+ date="2014-04-01T04:18:10Z"
+ content="""
+is there a way for this to be done globally, without having to install and configure the hook for each repository? it seems like a fairly useful feature that could be factored in git-annex itself (as opposed to be shipped as a shell script)...
+
+also, is there a way to retroactively parse the tags from existing files (as opposed to only new files added to the repo).
+
+thanks
+"""]]
diff --git a/doc/tips/automatically_adding_metadata/comment_3_02e5314f827d17d482343e8f22c42fd9._comment b/doc/tips/automatically_adding_metadata/comment_3_02e5314f827d17d482343e8f22c42fd9._comment
new file mode 100644
index 0000000000..644ece527d
--- /dev/null
+++ b/doc/tips/automatically_adding_metadata/comment_3_02e5314f827d17d482343e8f22c42fd9._comment
@@ -0,0 +1,10 @@
+[[!comment format=mdwn
+ username="http://joeyh.name/"
+ ip="209.250.56.191"
+ subject="comment 3"
+ date="2014-04-17T20:15:07Z"
+ content="""
+@anarcat, I have modified [[pre-commit-annex]] so if it's passed already annexed files, it'll extract their metadata. 
+
+So this can be used to add metadata to files added before you installed the hook, or if you've configured more fields to be extracted.
+"""]]
diff --git a/doc/tips/automatically_adding_metadata/comment_4_cd3c8e2f45db93576d1b82cfbfbe601b._comment b/doc/tips/automatically_adding_metadata/comment_4_cd3c8e2f45db93576d1b82cfbfbe601b._comment
new file mode 100644
index 0000000000..f6b2773eb2
--- /dev/null
+++ b/doc/tips/automatically_adding_metadata/comment_4_cd3c8e2f45db93576d1b82cfbfbe601b._comment
@@ -0,0 +1,12 @@
+[[!comment format=mdwn
+ username="https://www.google.com/accounts/o8/id?id=AItOawnW_CrhP9p50n9UUhTg_a9glyKWSvnrjRQ"
+ nickname="Michele"
+ subject="direct mode pre-commit hooks [on windows]"
+ date="2015-01-20T12:43:24Z"
+ content="""
+seemingly pre-commit hooks are not being called on windows, it could have to do with git annex sync bypassing them when doing commits ?
+
+on the other side genmetadata works. although that is not enough for me since I'd want to preserve complete last modification date/time and I was in the process of modifying the supplied pre-commit script to call for \"stat %Y\" (which btw is working fine on windows, while the last binaries for extract are failing there).
+
+am I correct in assuming that direct mode [on windows at least] bypasses hooks [namely pre-commit as well as pre-commit-annex] ?
+"""]]
diff --git a/doc/tips/automatically_adding_metadata/comment_5_888f0a77405d616a0d51a6176b44f605._comment b/doc/tips/automatically_adding_metadata/comment_5_888f0a77405d616a0d51a6176b44f605._comment
new file mode 100644
index 0000000000..29f5cecb46
--- /dev/null
+++ b/doc/tips/automatically_adding_metadata/comment_5_888f0a77405d616a0d51a6176b44f605._comment
@@ -0,0 +1,11 @@
+[[!comment format=mdwn
+ username="joey"
+ subject="""comment 5"""
+ date="2015-01-20T16:52:28Z"
+ content="""
+@Michele git annex sync in a direct mode repository does bypass the
+pre-commit hook. However, it will try to run the pre-commit-annex hook.
+
+Most likely, the hook script does not appear executable on Windows, so 
+git-annex cannot run it.
+"""]]
diff --git a/doc/tips/automatically_adding_metadata/comment_6_34f0c55d09ddee3de642f6b25a9f6269._comment b/doc/tips/automatically_adding_metadata/comment_6_34f0c55d09ddee3de642f6b25a9f6269._comment
new file mode 100644
index 0000000000..ab5bc7b76b
--- /dev/null
+++ b/doc/tips/automatically_adding_metadata/comment_6_34f0c55d09ddee3de642f6b25a9f6269._comment
@@ -0,0 +1,9 @@
+[[!comment format=mdwn
+ username="joey"
+ subject="""comment 6"""
+ date="2015-01-20T17:19:34Z"
+ content="""
+@Michele after testing, git-annex on Windows seems to not see a file that
+does have the executable bit set as executable. I have opened a bug report
+[[bugs/windows_isExecutable_fail]], and worked around the problem now.
+"""]]
diff --git a/doc/tips/automatically_adding_metadata/comment_7_94877b21bf80374c2874b971f26f0e55._comment b/doc/tips/automatically_adding_metadata/comment_7_94877b21bf80374c2874b971f26f0e55._comment
new file mode 100644
index 0000000000..74bd602243
--- /dev/null
+++ b/doc/tips/automatically_adding_metadata/comment_7_94877b21bf80374c2874b971f26f0e55._comment
@@ -0,0 +1,16 @@
+[[!comment format=mdwn
+ username="https://www.google.com/accounts/o8/id?id=AItOawnW_CrhP9p50n9UUhTg_a9glyKWSvnrjRQ"
+ nickname="Michele"
+ subject="pre-commit is OK on windows now - auto adding last mod datetime"
+ date="2015-01-30T11:48:24Z"
+ content="""
+@Joey just tested a nightly build and now pre-commit-annex is called, and with my modifications it autoadds last modified times for content.
+Trivially it's just the matter of adding:
+
+	field=\"datemod\"
+	value=$(stat -c %Y $f)
+	addmeta \"$f\" \"$field\" \"$value\"
+
+to the body of the process() function to the supplied pre-commit-annex script.
+thanks
+"""]]
diff --git a/doc/tips/automatically_adding_metadata/comment_8_3880fb7f13e74d33d9c4e86772cc6b0e._comment b/doc/tips/automatically_adding_metadata/comment_8_3880fb7f13e74d33d9c4e86772cc6b0e._comment
new file mode 100644
index 0000000000..813c3f0065
--- /dev/null
+++ b/doc/tips/automatically_adding_metadata/comment_8_3880fb7f13e74d33d9c4e86772cc6b0e._comment
@@ -0,0 +1,10 @@
+[[!comment format=mdwn
+ username="woffs"
+ avatar="http://cdn.libravatar.org/avatar/198790d74209efe4896fd4cfc37ec2a6"
+ subject="hook and quoting"
+ date="2017-12-09T11:18:10Z"
+ content="""
+Hi,
+
+just detected that the provided pre-commit-annex script is broken for filenames containing a '
+"""]]
diff --git a/doc/tips/automatically_adding_metadata/comment_9_db9c2fa8545188520d4bdda5ba545624._comment b/doc/tips/automatically_adding_metadata/comment_9_db9c2fa8545188520d4bdda5ba545624._comment
new file mode 100644
index 0000000000..d833316253
--- /dev/null
+++ b/doc/tips/automatically_adding_metadata/comment_9_db9c2fa8545188520d4bdda5ba545624._comment
@@ -0,0 +1,12 @@
+[[!comment format=mdwn
+ username="joey"
+ subject="""comment 9"""
+ date="2017-12-11T18:23:58Z"
+ content="""
+@woffs hmm, the hook script seems to quote every use of the filename,
+which should avoid such problems. I tested it, using both extract
+and exiftool, and a file named "foo'bar.jpg", and it worked fine.
+
+If you have a case where it does not work, suggest you file a bug
+report with enough information to reproduce the problem.
+"""]]
diff --git a/doc/tips/automatically_adding_metadata/pre-commit-annex b/doc/tips/automatically_adding_metadata/pre-commit-annex
new file mode 100755
index 0000000000..982fcca7de
--- /dev/null
+++ b/doc/tips/automatically_adding_metadata/pre-commit-annex
@@ -0,0 +1,118 @@
+#!/bin/sh
+#
+# Copyright (C) 2014 Joey Hess 
+# Copyright (C) 2016 Klaus Ethgen 
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program.  If not, see .
+#
+# This script can be used to add git-annex metadata to files when they're
+# committed. It is typically installed as .git/hooks/pre-commit-annex
+#
+# You can also run this script by hand, passing it the names of files
+# already checked into git-annex, and it will extract/refresh the git-annex
+# metadata from the files.
+
+tool="$(git config metadata.tool || :)"
+if [ -z "$tool" ]; then
+	tool=extract
+fi
+case "$tool" in
+	exiftool)
+		tool_exec="exiftool -unknown -zip -veryShort -ignoreMinorErrors -use MWG -dateFormat '%Y-%m-%dT%H:%M:%S'"
+	;;
+	*)
+		tool_exec="$tool"
+	;;
+esac
+
+extract_fields="$(git config metadata.extract || :)"
+if [ -n "$extract_fields" ]; then
+	tools=extract
+	extract_want="^($(echo "$extract_fields" | sed -e 's/ /|/g' -e 's/_/ /g'))"
+fi
+exiftool_fields="$(git config metadata.exiftool || :)"
+if [ -n "$exiftool_fields" ]; then
+	tools="exiftool $tools"
+	exiftool_want="^($(echo "$exiftool_fields" | sed -e 's/ /|/g' -e 's/_/ /g'))"
+fi
+if [ -z "$tools" ]; then
+	exit 0
+fi
+
+case "$(git config --bool metadata.overwrite || :)" in
+	true)
+		equal="="
+	;;
+	*)
+		equal="?="
+	;;
+esac
+
+if git rev-parse --verify HEAD >/dev/null 2>&1; then
+	against="HEAD"
+else
+	# Initial commit: diff against an empty tree object
+	against="4b825dc642cb6eb9a060e54bf8d69288fbee4904"
+fi
+
+addmeta() {
+	file="$1"
+	field="$2"
+	value="$3"
+	afield="$(echo "$field" | tr ' ' '_')"
+	git -c annex.alwayscommit=false annex metadata \
+		--set "$afield$equal$value" --quiet -- "$file"
+}
+
+process() {
+	if [ -e "$f" ]; then
+		echo "adding metadata for $f"
+		for tool in $tools; do
+			case "$tool" in
+				exiftool)
+					tool_exec="exiftool -unknown -zip -veryShort -ignoreMinorErrors -use MWG -dateFormat '%Y-%m-%dT%H:%M:%S'"
+				;;
+				*)
+					tool_exec="$tool"
+				;;
+			esac
+			LC_ALL=C $tool_exec "./$f" | eval egrep --text -i \""\$${tool}_want"\" | while read line; do
+				case "$tool" in
+					extract)
+						field="${line%% - *}"
+						value="${line#* - }"
+					;;
+					exiftool)
+						field="${line%%: *}"
+						value="${line#*: }"
+					;;
+				esac
+				
+				if [ -n "$value" ]; then
+					addmeta "$f" "$field" "$value"
+				fi
+			done
+		done
+	fi
+}
+
+if [ -n "$*" ]; then
+	for f in "$@"; do
+		process "$f"
+	done
+else
+	git diff-index -z --name-only --cached $against | sed 's/\x00/\n/g' | while read f; do
+		process "$f"
+	done
+fi
diff --git a/doc/tips/automatically_getting_files_on_checkout.mdwn b/doc/tips/automatically_getting_files_on_checkout.mdwn
new file mode 100644
index 0000000000..bbb3b302eb
--- /dev/null
+++ b/doc/tips/automatically_getting_files_on_checkout.mdwn
@@ -0,0 +1,15 @@
+Normally git-annex does not retrieve file contents when checking out a
+tree. In some use cases, it makes sense to always have the contents of
+files available after a `git checkout` or `git update`. This can be
+accomplished by installing the following as `.git/hooks/post-checkout`
+
+	#!/bin/sh
+	# Uses git-annex to get all files in the specified directories
+	# (relative to the top of the repository) on checkout.
+	dirs=.
+	top="$(git rev-parse --show-toplevel)"
+	for dir in "$dirs"; do git annex get $top/$dir"; done
+
+By default, all files in the whole repository will be made available. The
+`dirs` setting can be configured if you only want to get files in certian
+directories. 
diff --git a/doc/tips/beware_of_SSD_wear_when_doing_fsck_on_large_special_remotes.mdwn b/doc/tips/beware_of_SSD_wear_when_doing_fsck_on_large_special_remotes.mdwn
new file mode 100644
index 0000000000..0983c7d31b
--- /dev/null
+++ b/doc/tips/beware_of_SSD_wear_when_doing_fsck_on_large_special_remotes.mdwn
@@ -0,0 +1,2 @@
+When git annex does fsck on (for example) a GPG-encrypted special directory remote, it first transfers the whole file into .git/annex/tmp directory.
+If your annex is on an SSD, it's a good idea to make .git/annex/tmp a symlink to say /var/tmp so SSD isn't worn down.  This actually may be a better default.
diff --git a/doc/tips/beware_of_SSD_wear_when_doing_fsck_on_large_special_remotes/comment_1_e7c5c46112a2406b873d08bbf53c40d8._comment b/doc/tips/beware_of_SSD_wear_when_doing_fsck_on_large_special_remotes/comment_1_e7c5c46112a2406b873d08bbf53c40d8._comment
new file mode 100644
index 0000000000..9c7bc2ed17
--- /dev/null
+++ b/doc/tips/beware_of_SSD_wear_when_doing_fsck_on_large_special_remotes/comment_1_e7c5c46112a2406b873d08bbf53c40d8._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="https://www.google.com/accounts/o8/id?id=AItOawln3ckqKx0x_xDZMYwa9Q1bn4I06oWjkog"
+ nickname="Michael"
+ subject="comment 1"
+ date="2013-07-31T15:15:41Z"
+ content="""
+Of course, this only works when /var/tmp isn't on SSD itself.  Perhaps tmpfs (e.g. a /tmp on many distros) is good -- after checking that there's enough space to transfer a particular file.
+"""]]
diff --git a/doc/tips/beware_of_SSD_wear_when_doing_fsck_on_large_special_remotes/comment_2_daf45ce29fed986fa9aa8b173760d0b7._comment b/doc/tips/beware_of_SSD_wear_when_doing_fsck_on_large_special_remotes/comment_2_daf45ce29fed986fa9aa8b173760d0b7._comment
new file mode 100644
index 0000000000..929019705b
--- /dev/null
+++ b/doc/tips/beware_of_SSD_wear_when_doing_fsck_on_large_special_remotes/comment_2_daf45ce29fed986fa9aa8b173760d0b7._comment
@@ -0,0 +1,14 @@
+[[!comment format=mdwn
+ username="https://www.google.com/accounts/o8/id?id=AItOawln3ckqKx0x_xDZMYwa9Q1bn4I06oWjkog"
+ nickname="Michael"
+ subject="there's a problem"
+ date="2013-08-04T17:15:05Z"
+ content="""
+If .git/annex/tmp is a symlink to another fs, then adding doesn't work:
+
+    add file1.jpg (checksum...) 
+    git-annex: /path/to/.git/annex/tmp/tmp30148: rename: unsupported operation (Invalid cross-device link)
+
+It looks like it would be good to have two types of tmp directories here, one for adding, another one for verifying (and that one could be redirected off SSD).
+
+"""]]
diff --git a/doc/tips/beware_of_SSD_wear_when_doing_fsck_on_large_special_remotes/comment_3_72d222020af4a9c6c753eb1ee7e1f1cf._comment b/doc/tips/beware_of_SSD_wear_when_doing_fsck_on_large_special_remotes/comment_3_72d222020af4a9c6c753eb1ee7e1f1cf._comment
new file mode 100644
index 0000000000..2624a4fd34
--- /dev/null
+++ b/doc/tips/beware_of_SSD_wear_when_doing_fsck_on_large_special_remotes/comment_3_72d222020af4a9c6c753eb1ee7e1f1cf._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="guilhem"
+ ip="46.239.117.180"
+ subject="comment 3"
+ date="2013-08-19T01:05:40Z"
+ content="""
+A nice feature would be to perform the `fsck` on the (encrypted) remote itself, as it would avoid to clutter either the network or the tmpdir. However, that requires some changes in git-annex's backend. Indeed it would no longer be enough to store a single digest per (plain) file: a new digest needs to be stored for each encrypted copy. It is not necessarily a big deal, but the backend would need to be reorganized carefully.
+"""]]
diff --git a/doc/tips/centralised_repository__58___starting_from_nothing.mdwn b/doc/tips/centralised_repository__58___starting_from_nothing.mdwn
new file mode 100644
index 0000000000..7018697a0e
--- /dev/null
+++ b/doc/tips/centralised_repository__58___starting_from_nothing.mdwn
@@ -0,0 +1,71 @@
+If you are starting from nothing (no existing `git` or `git-annex` repository) and want to use a server as a centralised repository, try the following steps.
+
+On the server where you'll hold the "master" repository:
+
+	server$ cd /one/git
+	server$ mkdir m
+	server$ cd m
+	server$ git init --bare
+	Initialized empty Git repository in /one/git/m/
+	server$ git annex init origin
+	init origin ok
+	server$ 
+
+Clone that to the laptop:
+
+	laptop$ cd /other
+	laptop$ git clone ssh://server//one/git/m
+	Cloning into 'm'...
+	remote: Counting objects: 5, done.        
+	remote: Compressing objects: 100% (3/3), done.        
+	remote: Total 5 (delta 0), reused 0 (delta 0)        
+	Receiving objects: 100% (5/5), done.
+	warning: remote HEAD refers to nonexistent ref, unable to checkout.
+
+	laptop$ cd m
+	laptop$ git annex init laptop
+	init laptop ok
+        laptop$ 
+
+Add some content:
+
+	laptop$ git annex addurl http://kitenet.net/~joey/screencasts/git-annex_coding_in_haskell.ogg
+	addurl kitenet.net_~joey_screencasts_git-annex_coding_in_haskell.ogg (downloading http://kitenet.net/~joey/screencasts/git-annex_coding_in_haskell.ogg ...) --2011-12-15 08:13:10--  http://kitenet.net/~joey/screencasts/git-annex_coding_in_haskell.ogg
+	Resolving kitenet.net (kitenet.net)... 2001:41c8:125:49::10, 80.68.85.49
+	Connecting to kitenet.net (kitenet.net)|2001:41c8:125:49::10|:80... connected.
+	HTTP request sent, awaiting response... 200 OK
+	Length: 39362757 (38M) [audio/ogg]
+	Saving to: `/other/m/.git/annex/tmp/URL--http&c%%kitenet.net%~joey%screencasts%git-annex_coding_in_haskell.ogg'
+
+	100%[======================================>] 39,362,757  2.31M/s   in 17s     
+
+	2011-12-15 08:13:27 (2.21 MB/s) - `/other/m/.git/annex/tmp/URL--http&c%%kitenet.net%~joey%screencasts%git-annex_coding_in_haskell.ogg' saved [39362757/39362757]
+
+	(checksum...) ok
+	(Recording state in git...)
+
+Don't forget to commit it:
+
+	laptop$ git commit -m 'See Joey play.'
+	[master (root-commit) 106e923] See Joey play.
+	 1 files changed, 1 insertions(+), 0 deletions(-)
+	  create mode 120000 kitenet.net_~joey_screencasts_git-annex_coding_in_haskell.ogg
+	laptop$ 
+
+All fine, now push it back to the centralised master:
+
+	laptop$ git push origin master
+	Counting objects: 20, done.
+	Delta compression using up to 4 threads.
+	Compressing objects: 100% (11/11), done.
+	Writing objects: 100% (18/18), 1.50 KiB, done.
+	Total 18 (delta 1), reused 1 (delta 0)
+	To ssh://server//one/git/m
+	   3ba1386..ad3bc9e  git-annex -> git-annex
+	laptop$ 
+
+You'll probably want to use `git annex copy --to origin` to copy the
+annexed file contents to the server. See the [[walkthrough]] for details.
+
+You can add more "client" repositories by following the `laptop`
+sequence of operations.
diff --git a/doc/tips/centralised_repository__58___starting_from_nothing/comment_1_b0d22822017646775869ce1292e676f4._comment b/doc/tips/centralised_repository__58___starting_from_nothing/comment_1_b0d22822017646775869ce1292e676f4._comment
new file mode 100644
index 0000000000..22857af3e8
--- /dev/null
+++ b/doc/tips/centralised_repository__58___starting_from_nothing/comment_1_b0d22822017646775869ce1292e676f4._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="http://joey.kitenet.net/"
+ nickname="joey"
+ subject="comment 1"
+ date="2011-12-23T19:19:53Z"
+ content="""
+See also: [[centralized_git_repository_tutorial]]
+"""]]
diff --git a/doc/tips/centralized_git_repository_tutorial.mdwn b/doc/tips/centralized_git_repository_tutorial.mdwn
new file mode 100644
index 0000000000..4019c116e2
--- /dev/null
+++ b/doc/tips/centralized_git_repository_tutorial.mdwn
@@ -0,0 +1,18 @@
+The [[walkthrough]] builds up a decentralized git repository setup, but
+git-annex can also be used with a centralized git repository. 
+
+We have separate tutorials depending on where the centralized git
+repository is hosted.
+
+* [[centralized_git_repository_tutorial/On_GitHub]] --
+  However, GitHub does not currently let git-annex
+  store the contents of large files there. So, things get a little more
+  complicated when using it.
+
+* [[centralized_git_repository_tutorial/On_GitLab]] -- 
+  This service is similar to GitHub, but supports
+  git-annex.
+
+* [[centralized_git_repository_tutorial/On_your_own_server]] --
+  use any unix system with ssh and git and git-annex installed.
+  A VPS, a home server, etc.
diff --git a/doc/tips/centralized_git_repository_tutorial/comment_1_9072ebc0c61446d7b151fcfab616fea9._comment b/doc/tips/centralized_git_repository_tutorial/comment_1_9072ebc0c61446d7b151fcfab616fea9._comment
new file mode 100644
index 0000000000..c509d75796
--- /dev/null
+++ b/doc/tips/centralized_git_repository_tutorial/comment_1_9072ebc0c61446d7b151fcfab616fea9._comment
@@ -0,0 +1,33 @@
+[[!comment format=mdwn
+ username="https://www.google.com/accounts/o8/id?id=AItOawkC0W3ZQERUaTkHoks6k68Tsp1tz510nGo"
+ nickname="Georg"
+ subject="sync, push, pull with/to/from centralized bare repository"
+ date="2013-10-07T06:45:19Z"
+ content="""
+Hi Joey,
+
+thanks for tutorial with the centralized repo. I am currently trying to set up a central bare repo for two clients (they cannot communicate directly with each other). I am not sure if I am pushing/pulling the right way.
+
+On the server I did:
+
+    git init --bare
+    git annex init origin
+
+On Cĺient Alice (I want to give Bob a chance get call \"git annex get\" from \"origin\"):
+
+    git clone ssh://tktest@192.168.56.104/~/annex .
+    git annex init Alice
+    git annex merge
+    git annex add .
+    git commit -a -m \"Added tutorial\"
+    git push origin master git-annex
+    git annex copy . --to origin
+
+On Client Bob I have called \"clone, init, merge, add, push, copy\" also.
+
+Now the tricky part - do I have to call \"git annex sync\" at Alice's side to get the updates from Bob over origin?
+I ran into troubles if I called \"copy --to origin\" before \"git push origin master git-annex\". How can I resolve a non-fast-forware on the git-annex branch?
+Some notes about how to sync over a central bare repo would be nice here =)
+
+Thanks a lot, Georg
+"""]]
diff --git a/doc/tips/centralized_git_repository_tutorial/comment_2_528e92b21f0551fde4adb956654953ae._comment b/doc/tips/centralized_git_repository_tutorial/comment_2_528e92b21f0551fde4adb956654953ae._comment
new file mode 100644
index 0000000000..21b286fff2
--- /dev/null
+++ b/doc/tips/centralized_git_repository_tutorial/comment_2_528e92b21f0551fde4adb956654953ae._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="http://joeyh.name/"
+ ip="4.153.253.80"
+ subject="How can I resolve a non-fast-forware on the git-annex branch?"
+ date="2013-10-07T17:08:32Z"
+ content="""
+By either running `git annex sync`, or if you want to pull and push yourself, by running `git annex merge` before pushing.
+"""]]
diff --git a/doc/tips/centralized_git_repository_tutorial/comment_3_8894229043935e70089caf67f0cc4ddb._comment b/doc/tips/centralized_git_repository_tutorial/comment_3_8894229043935e70089caf67f0cc4ddb._comment
new file mode 100644
index 0000000000..b0324d47a1
--- /dev/null
+++ b/doc/tips/centralized_git_repository_tutorial/comment_3_8894229043935e70089caf67f0cc4ddb._comment
@@ -0,0 +1,10 @@
+[[!comment format=mdwn
+ username="https://www.google.com/accounts/o8/id?id=AItOawn8d6WBPNtmyeANT-gjlN19QnCTQ0psRPs"
+ nickname="James"
+ subject="S3 special remote?"
+ date="2015-01-22T21:52:13Z"
+ content="""
+Is it possible to use special remotes (like S3) with this centralized method?
+
+Public web content works fine, but I've had no luck with S3.
+"""]]
diff --git a/doc/tips/centralized_git_repository_tutorial/comment_4_b496622537d2491854b02884acf1672c._comment b/doc/tips/centralized_git_repository_tutorial/comment_4_b496622537d2491854b02884acf1672c._comment
new file mode 100644
index 0000000000..9ef4ddd1d2
--- /dev/null
+++ b/doc/tips/centralized_git_repository_tutorial/comment_4_b496622537d2491854b02884acf1672c._comment
@@ -0,0 +1,11 @@
+[[!comment format=mdwn
+ username="joey"
+ subject="""comment 4"""
+ date="2015-02-04T19:29:01Z"
+ content="""
+You can add any kind of special remotes to such a repository.
+
+Each clone from the central repository will need to have `git annex
+enableremote` used to enable any special remote(s) you need to use
+from that clone.
+"""]]
diff --git a/doc/tips/centralized_git_repository_tutorial/comment_5_c6e3468c95bc875e17724ee4a3a283a3._comment b/doc/tips/centralized_git_repository_tutorial/comment_5_c6e3468c95bc875e17724ee4a3a283a3._comment
new file mode 100644
index 0000000000..d9a62cccbc
--- /dev/null
+++ b/doc/tips/centralized_git_repository_tutorial/comment_5_c6e3468c95bc875e17724ee4a3a283a3._comment
@@ -0,0 +1,11 @@
+[[!comment format=mdwn
+ username="spalax@b201acef21dca7798b874036bbbaa9e0079a0b7e"
+ nickname="spalax"
+ avatar="http://cdn.libravatar.org/avatar/3f1353e4135221fc25bfecd1b812bcc8"
+ subject="git-annex support dropped"
+ date="2017-06-27T09:26:11Z"
+ content="""
+For information, gitlab is going to drop support for git-annex (see issue [#1648](https://gitlab.com/gitlab-org/gitlab-ee/issues/1648)).
+
+-- Louis
+"""]]
diff --git a/doc/tips/centralized_git_repository_tutorial/on_GitHub.mdwn b/doc/tips/centralized_git_repository_tutorial/on_GitHub.mdwn
new file mode 100644
index 0000000000..4522319b3f
--- /dev/null
+++ b/doc/tips/centralized_git_repository_tutorial/on_GitHub.mdwn
@@ -0,0 +1,129 @@
+This tutorial shows how to set up a centralized repository hosted on
+GitHub.
+
+GitHub does not currently let git-annex store the contents of large files
+there. This doesn't prevent using git-annex with GitHub, it just means you
+have to set up some other centralized location for the large files.
+
+## set up the repository, and make a checkout
+
+I've created a repository for technical talk videos, which you can
+[fork on Github](https://github.com/joeyh/techtalks).
+Or make your own repository on GitHub now.
+
+On your laptop, [[install]] git-annex, and clone the repository:
+
+	# git clone git@github.com:joeyh/techtalks.git
+	# cd techtalks
+
+Tell git-annex to use the repository, and describe where this clone is
+located:
+
+	# git annex init 'my laptop'
+	init my laptop ok
+
+## add files to the repository
+
+Add some files, obtained however.
+
+	# git annex add *.mp4
+	add Haskell_Amuse_Bouche-b9OVqxmI.mp4 (checksum) ok
+	(Recording state in git...)
+	# git commit -m "added a video. I have not watched it yet but it sounds interesting"
+
+This file is available on the web; so git-annex can download it:
+
+	# git annex addurl http://kitenet.net/~joey/screencasts/git-annex_coding_in_haskell.ogg
+	addurl kitenet.net_~joey_screencasts_git-annex_coding_in_haskell.ogg
+	(downloading http://kitenet.net/~joey/screencasts/git-annex_coding_in_haskell.ogg ...)
+	(checksum...) ok
+	(Recording state in git...)
+	# git commit -a -m 'added a screencast I made'
+
+Feel free to rename the files, etc, using normal git commands:
+
+	# git mv Haskell_Amuse_Bouche-b9OVqxmI.mp4 Haskell_Amuse_Bouche.mp4
+	# git mv kitenet.net_~joey_screencasts_git-annex_coding_in_haskell.ogg git-annex_coding_in_haskell.ogg
+	# git commit -m 'better filenames'
+
+Now push your changes back to the central repository on GitHub. As well as 
+pushing the master branch, remember to push the git-annex branch, which is
+used to track the file contents. You can do this push manually as shown
+below, or you can just run `git annex sync` to do the same thing.
+
+	# git push origin master git-annex
+	To git@github.com:joeyh/techtalks.git
+	 * [new branch]      master -> master
+	 * [new branch]      git-annex -> git-annex
+
+That push went fast, because it didn't upload large videos to GitHub.
+To check this, you can ask git-annex where the contents of the videos are:
+
+	# git annex whereis
+	whereis Haskell_Amuse_Bouche.mp4 (1 copy) 
+	  	767e8558-0955-11e1-be83-cbbeaab7fff8 -- here
+	ok
+	whereis git-annex_coding_in_haskell.ogg (2 copies) 
+	  	00000000-0000-0000-0000-000000000001 -- web
+	   	767e8558-0955-11e1-be83-cbbeaab7fff8 -- here
+	ok
+
+## make more checkouts
+
+So far you have a central repository, and a checkout on a laptop.
+You, or anyone you allow to can clone the central repository, and
+use git-annex with it.
+
+But, since GitHub doesn't currently support storing large files there
+with git-annex, other checkouts of your repository won't be able to
+access the files you added to the repository on your laptop.
+
+	# git clone git@github.com:myrepo/techtalks.git
+	# git annex get Haskell_Amuse_Bouche-b9OVqxmI.mp4
+	get Haskell_Amuse_Bouche-b9OVqxmI.mp4
+
+	  Try making some of these repositories available:
+		767e8558-0955-11e1-be83-cbbeaab7fff8 -- my laptop
+	failed
+
+## add a special remote
+
+So, to complete your setup, you need to set up a repository where git-annex
+can store the contents of large files. This is often done by setting up
+a [[special_remote|special_remotes]]. One free option is explained in
+[[using_box.com_as_a_special_remote]]. Another useful approach is
+explained in [[public_Amazon_S3_remote]].
+
+Once you have the special remote set up on your laptop, you can
+send files to it:
+
+	# git annex copy --to myspecialremote Haskell_Amuse_Bouche-b9OVqxmI.mp4
+	copy Haskell_Amuse_Bouche-b9OVqxmI.mp4 (to myspecialremote...)
+	100% 255.11kB/s
+	ok
+
+You can also `git annex move` files to it, to free up space on your laptop.
+And then you can `git annex get` files back to your laptop later on, as
+desired.
+
+After you use git-annex to move files around, remember to sync,
+which will broadcast its updated location information.
+
+	# git annex sync
+
+After setting up the special remote and storing some files on it,
+you can download them on other clones. You'll first need to enable the same
+special remote on the clones.
+
+	# git annex sync
+	# git annex enableremote myspecialremote
+	# git annex get git-annex_coding_in_haskell.ogg
+	100% 255.11kB/s
+	ok
+
+## take it farther
+
+You can add remotes for each direct connection between machines you find you
+need -- so make the laptop have the desktop as a remote, and the desktop
+have the laptop as a remote, and then on either machine git-annex can
+access files stored on the other.
diff --git a/doc/tips/centralized_git_repository_tutorial/on_GitLab.mdwn b/doc/tips/centralized_git_repository_tutorial/on_GitLab.mdwn
new file mode 100644
index 0000000000..2f4a0765ca
--- /dev/null
+++ b/doc/tips/centralized_git_repository_tutorial/on_GitLab.mdwn
@@ -0,0 +1,76 @@
+This tutorial shows how to set up a centralized repository hosted on
+GitLab. 
+
+Since GitLab has [added support for git-annex on their servers](https://about.gitlab.com/2015/02/17/gitlab-annex-solves-the-problem-of-versioning-large-binaries-with-git/),
+you can store your large files on GitLab, quite easily.
+
+Note that as I'm writing this, GitLab is providing this service for free,
+and will store up to 10 gb per project.
+
+## create the repository
+
+Go to  and sign up for an account, and create the
+repository there. Take note of the SSH clone url for the repository, which
+will be something like `git@gitlab.com:yourlogin/annex.git`. 
+
+We want to clone this locally, on your laptop. (If the clone fails, you
+need to generate a ssh key and add it to GitLab.)
+
+	# git clone git@gitlab.com:yourlogin/annex.git
+	# cd annex
+
+Tell git-annex to use the repository, and describe where this clone is
+located:
+
+	# git annex init 'my laptop'
+	init my laptop ok
+
+Add some files, obtained however.
+
+	# git annex add *.mp4
+	add Haskell_Amuse_Bouche-b9OVqxmI.mp4 (checksum) ok
+	(Recording state in git...)
+	# git commit -m "added a video. I have not watched it yet but it sounds interesting"
+
+Feel free to rename the files, etc, using normal git commands:
+
+	# git mv Haskell_Amuse_Bouche-b9OVqxmI.mp4 Haskell_Amuse_Bouche.mp4
+	# git commit -m 'better filenames'
+
+## push to GitLab
+
+Now make a first push to the GitLab repository.
+As well as pushing the master branch, remember to push the git-annex
+branch, which is used to track the file contents.
+
+	# git push origin master git-annex
+	To git@gitlab.com:yourlogin/annex.git
+	 * [new branch]      master -> master
+	 * [new branch]      git-annex -> git-annex
+
+That push went fast, because it didn't upload the large file contents yet.
+
+So, to finish up, tell git-annex to sync all the data in the repository
+to GitLab:
+
+	# git annex sync --content
+	...
+
+## make more checkouts
+
+So far you have a central repository on GitLab, and a checkout on a laptop.
+Let's make another checkout elsewhere. Clone the central repository as before.
+(If the clone fails, you need to generate a ssh key and add it to GitLab.)
+
+	elsewhere# git clone git@gitlab.com:yourlogin/annex.git
+	elsewhere# cd annex
+
+Notice that your clone does not have the contents of any of the files yet.
+If you run `ls`, you'll see broken symlinks. It's easy to download them from
+GitLab either by running `git annex sync --content`, or by asking
+git-annex to download individual files:
+
+	# git annex get Haskell_Amuse_Bouche.mp4
+	get Haskell_Amuse_Bouche.mp4 (from origin...)
+	12877824   2%  255.11kB/s    00:00
+	ok
diff --git a/doc/tips/centralized_git_repository_tutorial/on_GitLab/comment_1_9d0694204984ec96379a62f0f70ba696._comment b/doc/tips/centralized_git_repository_tutorial/on_GitLab/comment_1_9d0694204984ec96379a62f0f70ba696._comment
new file mode 100644
index 0000000000..3a17029ee7
--- /dev/null
+++ b/doc/tips/centralized_git_repository_tutorial/on_GitLab/comment_1_9d0694204984ec96379a62f0f70ba696._comment
@@ -0,0 +1,15 @@
+[[!comment format=mdwn
+ username="ovidiu@66ace8a8d99ce938b0538ffa0f26d30db02a9626"
+ nickname="ovidiu"
+ subject="Need advice on using gitlab with the assistant"
+ date="2015-10-03T21:03:45Z"
+ content="""
+There is a GUI and I followed it but when upgrading it to a full git-annex repository I get:
+
+remote: GitLab: You are not allowed to force push code to a protected branch on this project.        
+To git@git-annex-gitlab.com-git_22_pacuraru.2Fgitannex.2Egit:pacuraru/gitannex.git
+ ! [remote rejected] refs/gcrypt/gitception+ -> master (pre-receive hook declined)
+error: failed to push some refs to 'git@git-annex-gitlab.com-git_22_pacuraru.2Fgitannex.2Egit:pacuraru/gitannex.git'
+error: failed to push some refs to 'gcrypt::git@git-annex-gitlab.com-git_22_pacuraru.2Fgitannex.2Egit:pacuraru/gitannex.git'
+[2015-10-03 23:01:30.409376] main: Syncing with gitlab.com_pacuraru_gitannex.git 
+"""]]
diff --git a/doc/tips/centralized_git_repository_tutorial/on_GitLab/comment_2_1c33d3f62ec7119c116ad02b75c91f8a._comment b/doc/tips/centralized_git_repository_tutorial/on_GitLab/comment_2_1c33d3f62ec7119c116ad02b75c91f8a._comment
new file mode 100644
index 0000000000..cdbc035e4e
--- /dev/null
+++ b/doc/tips/centralized_git_repository_tutorial/on_GitLab/comment_2_1c33d3f62ec7119c116ad02b75c91f8a._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="ovidiu@66ace8a8d99ce938b0538ffa0f26d30db02a9626"
+ nickname="ovidiu"
+ subject="comment 2"
+ date="2015-10-03T21:27:46Z"
+ content="""
+I managed to overcome that hurdle but now gitlab shows as \"just a git repository\" and I don't seem to be able to do the conversion \"Upgrade Repository\" - nothing in the logs, any ideas?
+"""]]
diff --git a/doc/tips/centralized_git_repository_tutorial/on_GitLab/comment_3_7d3ea328a42c3a1df01f87e5233eca62._comment b/doc/tips/centralized_git_repository_tutorial/on_GitLab/comment_3_7d3ea328a42c3a1df01f87e5233eca62._comment
new file mode 100644
index 0000000000..5143342473
--- /dev/null
+++ b/doc/tips/centralized_git_repository_tutorial/on_GitLab/comment_3_7d3ea328a42c3a1df01f87e5233eca62._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="ovidiu@66ace8a8d99ce938b0538ffa0f26d30db02a9626"
+ nickname="ovidiu"
+ subject="comment 3"
+ date="2015-10-03T21:29:29Z"
+ content="""
+oh, and btw. there's no button in the assistant to delete this useless gitlab repo, I can only edit it and try and upgrade it which doesn't work :-(
+"""]]
diff --git a/doc/tips/centralized_git_repository_tutorial/on_GitLab/comment_4_3c40b855e9914d8987b2f89d9abc29ae._comment b/doc/tips/centralized_git_repository_tutorial/on_GitLab/comment_4_3c40b855e9914d8987b2f89d9abc29ae._comment
new file mode 100644
index 0000000000..cc2eab05eb
--- /dev/null
+++ b/doc/tips/centralized_git_repository_tutorial/on_GitLab/comment_4_3c40b855e9914d8987b2f89d9abc29ae._comment
@@ -0,0 +1,14 @@
+[[!comment format=mdwn
+ username="joey"
+ subject="""comment 4"""
+ date="2015-11-18T19:14:32Z"
+ content="""
+@ovidiu, this tutorial is about setting up GitLab as a remote using
+git-annex at the command line. The steps given here should work.
+
+You're trying to use GitLab from the git-annex assistant. The first
+version that supported that was version 5.20150731. I guess you might have
+been trying to use an older version. Or encountered some other problem
+which is offtopic to this page. If you have a problem, file a bug report
+with full details about the version of git-annex you used and what you did.
+"""]]
diff --git a/doc/tips/centralized_git_repository_tutorial/on_your_own_server.mdwn b/doc/tips/centralized_git_repository_tutorial/on_your_own_server.mdwn
new file mode 100644
index 0000000000..6d5d1cd02c
--- /dev/null
+++ b/doc/tips/centralized_git_repository_tutorial/on_your_own_server.mdwn
@@ -0,0 +1,94 @@
+This tutorial shows how to set up a centralized git repository
+hosted on your own git server, which can be any unix system with
+ssh and git and git-annex installed. A VPS, a home server, etc.
+
+This sets up a very simple git server. More complex setups are possible.
+See for example [[using_gitolite_with_git-annex]].
+
+## set up the server
+
+On the server, you'll want to [[install]] git, and git-annex, if you haven't
+already. If possible, install it using your distribution's package manager:
+
+	server# sudo apt-get install git git-annex
+
+Note that git-annex-shell needs to be located somewhere in the PATH, so
+that a client can successfully run "ssh yourserver git-annex-shell".
+Installing git-annex using a package manager will take care of this for
+you. But if you're not root or otherwise can't install git-annex that way,
+you may need to do more work; see [[get_git-annex-shell_into_PATH]].
+
+Decide where to put the repository on the server, and create a bare git repo
+there. In your home directory is a simple choice:
+
+	server# cd
+	server# git init annex.git --bare --shared
+
+That's the server setup done!
+
+## make a checkout
+
+Now on your laptop, clone the git repository from the server:
+
+	laptop# git clone ssh://example.com/~/annex.git
+	Cloning into 'annex'...
+	warning: You appear to have cloned an empty repository.
+	Checking connectivity... done.
+
+	
+Tell git-annex to use the repository, and describe where this clone is
+located:
+
+
+	laptop# cd annex
+	laptop# git annex init 'my laptop'
+	init my laptop ok
+
+## add files to the repository
+
+Add some files, obtained however.
+
+	# git annex add *.mp4
+	add Haskell_Amuse_Bouche-b9OVqxmI.mp4 (checksum) ok
+	(Recording state in git...)
+	# git commit -m "added a video. I have not watched it yet but it sounds interesting"
+
+Feel free to rename the files, etc, using normal git commands:
+
+	# git mv Haskell_Amuse_Bouche-b9OVqxmI.mp4 Haskell_Amuse_Bouche.mp4
+	# git commit -m 'better filenames'
+
+Now push your changes back to the central repository on your server. As
+well as pushing the master branch, remember to push the git-annex branch,
+which is used to track the file contents.
+
+	# git push origin master git-annex
+	To git@github.com:joeyh/techtalks.git
+	 * [new branch]      master -> master
+	 * [new branch]      git-annex -> git-annex
+
+That push went fast, because it didn't upload large videos to the server.
+
+So, to finish up, tell git-annex to sync all the data in the repository
+to your server:
+
+	# git annex sync --content
+	...
+
+## make more checkouts
+
+So far you have a central repository on your server, and a checkout on a laptop.
+Let's make another checkout elsewhere. Clone the central repository as before.
+
+	elsewhere# git clone ssh://example.com/~/annex.git
+	elsewhere# cd annex
+
+Notice that your clone does not have the contents of any of the files yet.
+If you run `ls`, you'll see broken symlinks. It's easy to download them from
+your server either by running `git annex sync --content`, or by asking
+git-annex to download individual files:
+
+	# git annex get Haskell_Amuse_Bouche.mp4
+	get Haskell_Amuse_Bouche.mp4 (from origin...)
+	12877824   2%  255.11kB/s    00:00
+	ok
diff --git a/doc/tips/centralized_git_repository_tutorial/on_your_own_server/comment_1_ae5af47faf95f008f1b07dbed5181286._comment b/doc/tips/centralized_git_repository_tutorial/on_your_own_server/comment_1_ae5af47faf95f008f1b07dbed5181286._comment
new file mode 100644
index 0000000000..2534105664
--- /dev/null
+++ b/doc/tips/centralized_git_repository_tutorial/on_your_own_server/comment_1_ae5af47faf95f008f1b07dbed5181286._comment
@@ -0,0 +1,28 @@
+[[!comment format=mdwn
+ username="tom_clune"
+ subject="git annex using the "wrong" ssh socket"
+ date="2016-02-24T19:23:13Z"
+ content="""
+To avoid frequent typing of pin + RSA passcode + password,  we typically establish an ssh control master just once.   This works fine with regular git commands, but the git-annex command apparently try to create a different socket.  Even that would be ok, except that apparently it is a new socket each time we enter a command.
+
+With sufficient \"-vvvv\" we see things like:  
+
+    ...
+    debug1: Executing proxy command: exec ssh -l fred proxy.xxx.yyy direct host
+    ...
+
+(Note I have eliminated references to the actual machine names and userid's.)
+
+If the command had instead been:
+
+     exec ssh -l fred proxy.xxx.yyy direct /home/fred/.ssh/master_host:22
+
+everything would have worked fine.  In fact, we are now using:
+
+     git config remote.origin.annex-ssh-options '-S /home/fred/.ssh/master_host:22'
+
+and this eliminates the issue.    But it would be nice if git annex could somehow automatically use
+the pre-existing connection.     Is there a better way to achieve this?
+
+
+"""]]
diff --git a/doc/tips/centralized_git_repository_tutorial/on_your_own_server/comment_2_b6a70e698eb4c46e753a620716c5956c._comment b/doc/tips/centralized_git_repository_tutorial/on_your_own_server/comment_2_b6a70e698eb4c46e753a620716c5956c._comment
new file mode 100644
index 0000000000..54c8a6b3e0
--- /dev/null
+++ b/doc/tips/centralized_git_repository_tutorial/on_your_own_server/comment_2_b6a70e698eb4c46e753a620716c5956c._comment
@@ -0,0 +1,14 @@
+[[!comment format=mdwn
+ username="joey"
+ subject="""comment 2"""
+ date="2016-04-14T19:40:32Z"
+ content="""
+git-annex sets up its own ssh connection caching because this makes it a
+lot faster.
+
+To disable this feature, you can set annex.sshcaching=false, or set
+remote.origin.annex-ssh-options as you have.
+
+git-annex has no way to know if you have another ssh socket to use, so it
+seems fine for you to need to configure it if you want it to use one.
+"""]]
diff --git a/doc/tips/centralized_git_repository_tutorial/on_your_own_server/comment_3_371f22ec58cf953c2b8abe3af71d3f91._comment b/doc/tips/centralized_git_repository_tutorial/on_your_own_server/comment_3_371f22ec58cf953c2b8abe3af71d3f91._comment
new file mode 100644
index 0000000000..c39263fa51
--- /dev/null
+++ b/doc/tips/centralized_git_repository_tutorial/on_your_own_server/comment_3_371f22ec58cf953c2b8abe3af71d3f91._comment
@@ -0,0 +1,10 @@
+[[!comment format=mdwn
+ username="ynikitenko"
+ avatar="http://cdn.libravatar.org/avatar/168d629704097ddc596f75ca32a687a3"
+ subject="Is it essential to clone from the server?"
+ date="2017-11-03T11:28:01Z"
+ content="""
+    Now on your laptop, clone the git repository from the server:
+
+Is it possible to init git-annex repo on your local machine, add git-remote there and push local data to the server? Won't there be any problems with this 'non-official' approach?
+"""]]
diff --git a/doc/tips/centralized_git_repository_tutorial/on_your_own_server/comment_4_6610f58b031d587f07f05b31754e4d97._comment b/doc/tips/centralized_git_repository_tutorial/on_your_own_server/comment_4_6610f58b031d587f07f05b31754e4d97._comment
new file mode 100644
index 0000000000..6fa1891374
--- /dev/null
+++ b/doc/tips/centralized_git_repository_tutorial/on_your_own_server/comment_4_6610f58b031d587f07f05b31754e4d97._comment
@@ -0,0 +1,17 @@
+[[!comment format=mdwn
+ username="ynikitenko"
+ avatar="http://cdn.libravatar.org/avatar/168d629704097ddc596f75ca32a687a3"
+ subject="tutorial up to date?"
+ date="2017-11-03T20:23:12Z"
+ content="""
+When I follow the tutorial and run
+
+    git push origin master git-annex
+
+it replies
+
+    error: src refspec master does not match any.
+    error: failed to push some refs to 'vds:~/test.git'
+
+However one can use `git-annex sync` instead and that works fine.
+"""]]
diff --git a/doc/tips/deleting_unwanted_files.mdwn b/doc/tips/deleting_unwanted_files.mdwn
new file mode 100644
index 0000000000..8c09b42aaa
--- /dev/null
+++ b/doc/tips/deleting_unwanted_files.mdwn
@@ -0,0 +1,40 @@
+It's quite hard to delete a file from a git repository once it's checked in and pushed to origin. This is normally ok, since git repositories contain mostly small files, and a good thing since losing hard work stinks. 
+
+With git-annex this changes some: Very large files can be managed with git-annex, and it's not uncommon to be done with such a file and want to delete it. So, git-annex provides a number of ways to handle this, while still trying to avoid accidental foot shooting that would lose the last copy of an important file.
+
+## the garbage collecting method
+
+In this method, you just remove annexed files whenever you want, and commit the changes. This is probably the most natural way to go.
+
+In an indirect mode repo, you can do this the same way you would in a regular git repository. For example, `git rm foo; git commit -m "removed foo"`. This leaves the contents of the files still in the annex, not really deleted yet.
+
+If you have a direct mode repo, you can't run `git rm` in it. Instead, you can just delete files using `rm` or your file manager, and then run `git annex sync` to commit the deletion. That will delete the file's content from your disk. Even if it's the only copy of the file!
+
+Either way, deleting files can leave some garbage lying around in either the local repository, or other repositories that contained a copy of the content of the file you deleted. Eventually you'll want to free up some disk space used by one of these repositories, and then it's time to take out the garbage.
+
+To collect the garbage, you can run `git annex unused` inside the repository which you want to slim down. That will list files stored in the annex that are not used by any git branches or tags. Followed by `git annex dropunused 1-10` to delete a range of the unused files from the annex.
+
+In recent versions of git-annex, `git annex dropunused` checks that enough other copies of a file's content exist in other repositories before deleting it, so this won't ever delete the last copy of some file. This is a good default, because these unused files are still referred to by some commits in the git history, and you might want to retain the full history of every version of a file.
+
+But, let's say you don't care about that, you only want to keep files that are in use by branches and tags. Then you can use `git annex dropunused --force` with a range of files, which will delete them even if it's the last copy.
+
+Finally, sometimes you want to remove unused files from a special remote. To accomplish this, pass `--from remotename` to the unused and dropunused commands, and they will act on
+files stored in that remote, rather than on the local repository.
+
+## let the assistant take care of it
+
+If you're using the git-annex assistant, you don't normally need to worry about this. Just delete files however you normally would. The assistant will try to migrate unused file contents away from your local repository and store them in whatever backup repositories you've set up.
+
+## delete all the copies method
+
+You have a file. You want that file to immediately vanish from the face of the earth to the best of your abilities.
+
+Note that, since git-annex deduplicates files by default, any files with
+the same content will be removed by these commands.
+
+1. `git annex drop --force file`
+2. `git annex whereis file`
+3. `git annex drop --force file --from $repo` repeat for each repository listed by the whereis command
+4. `rm file; git annex sync`
+
+Of course, if you have offline backup repositories that contain this file, you'll have to bring them online before you can drop it from them, etc.
diff --git a/doc/tips/disabling_a_special_remote.mdwn b/doc/tips/disabling_a_special_remote.mdwn
new file mode 100644
index 0000000000..3b80ef25ce
--- /dev/null
+++ b/doc/tips/disabling_a_special_remote.mdwn
@@ -0,0 +1,35 @@
+In our quest to find dumb replacements for [[todo/wishlist: 'get' queue and schedule.]] (and a more complete [[git-annex-schedule]]), we have setup a cronjob that would start and stop the assistant during certain time windows, to ensure that it would not download during prime bandwidth time.
+
+But that isn't exactly what we're looking for: we would like the assistant to continue doing its usual thing of adding and removing files, and even syncing the git branches. Just not get/move files around.
+
+One way I thought of doing this was to disable a remote locally. [[git-annex-dead]] of course comes to mind, but that applies to all repositories, so it's not an option. If the remote was in git, i could just `git remote rm origin` and `git remote add origin` and be done with it, but this is the *web* remote, so it doesn't even show up in `git remote -v`.
+
+But this doesn't work with [[special remotes]]. Another solution is
+simply to use the `remote.name.annex-ignore` configuration documented
+in the main [[git-annex]] manpage. For example, to disable the web
+remote, you would use:
+
+    git config remote.web.annex-ignore false
+
+The result would be:
+
+	joey@darkstar:~/tmp/a>git annex addurl --fast http://localhost
+	addurl localhost ok
+	(recording state in git...)
+	joey@darkstar:~/tmp/a>git config remote.web.annex-ignore true
+	joey@darkstar:~/tmp/a>git annex get localhost
+	get localhost (not available) 
+	  Try making some of these repositories available:
+	  	00000000-0000-0000-0000-000000000001 -- web
+	
+	  (Note that these git remotes have annex-ignore set: web)
+	failed
+	git-annex: get: 1 failed
+	joey@darkstar:~/tmp/a>git config remote.web.annex-ignore false
+	joey@darkstar:~/tmp/a>git annex get localhost
+	get localhost (from web...) 
+	/home/joey/tmp/a/.g 100%[=====================>]      10  --.-KB/s   in 0s     
+	ok
+
+The assistant (probably?) needs to be restarted for those changes to
+take effect.  --[[anarcat]] and [[joeyh]].
diff --git a/doc/tips/downloading_podcasts.mdwn b/doc/tips/downloading_podcasts.mdwn
new file mode 100644
index 0000000000..2c523606f0
--- /dev/null
+++ b/doc/tips/downloading_podcasts.mdwn
@@ -0,0 +1,98 @@
+You can use git-annex as a podcatcher, to download podcast contents.
+No additional software is required, but your git-annex must be built
+with the Feeds feature (run `git annex version` to check).
+
+All you need to do is put something like this in a cron job:
+
+`cd somerepo && git annex importfeed http://url/to/podcast http://other/podcast/url`
+
+This downloads the urls, and parses them as RSS, Atom, or RDF feeds.
+All enclosures are downloaded and added to the repository, the same as if you
+had manually run `git annex addurl` on each of them.
+
+git-annex will avoid downloading a file from a feed if its url has already
+been stored in the repository before. So once a file is downloaded,
+you can move it around, delete it, `git annex drop` its content, etc,
+and it will not be downloaded again by repeated runs of
+`git annex importfeed`. Just how a podcatcher should behave.  (git-annex versions 
+since 2015 also tracks the podcast `guid` values, as metadata, to help avoid 
+duplication if the media file url changes; use `git annex metadata ...` to inspect.)
+
+## templates
+
+To control the filenames used for items downloaded from a feed,
+there's a --template option. The default is
+`--template='${feedtitle}/${itemtitle}${extension}'`
+
+Other available template variables:  
+feedauthor, itemauthor, itemsummary, itemdescription, itemrights, itemid,
+itempubdate, author, title.
+
+## catching up
+
+To catch up on a feed without downloading its contents,
+use `git annex importfeed --relaxed`, and delete the symlinks it creates.
+Next time you run `git annex addurl` it will only fetch any new items.
+
+## fast mode
+
+To add a feed without downloading its contents right now,
+use `git annex importfeed --fast`. Then you can use `git annex get` as
+usual to download the content of an item.
+
+## storing the podcast list in git
+
+You can check the list of podcast urls into git right next to the
+files it downloads. Just make a file named feeds and add one podcast url
+per line.
+
+Then you can run git-annex on all the feeds:
+
+`xargs git-annex importfeed < feeds`
+
+## recreating lost episodes
+
+If for some reason git-annex refuses to download files you are certain are in the podcast, it is quite possible it is because they have already been downloaded. In any case, you can use `--force` to redownload them:
+
+`git-annex importfeed --force http://example.com/feed`
+
+## distributed podcatching
+
+A nice benefit of using git-annex as a podcatcher is that you can
+run `git annex importfeed` on the same url in different clones
+of a repository, and `git annex sync` will sync it all up.
+
+## centralized podcatching
+
+You can also have a designated machine which always fetches all podcstas
+to local disk and stores them. That way, you can archive podcasts with
+time-delayed deletion of upstream content. You can also work around slow
+downloads upstream by podcatching to a server with ample bandwidth or work
+around a slow local Internet connection by podcatching to your home server
+and transferring to your laptop on demand.
+
+## youtube channels
+
+You can also use `git annex importfeed` on youtube channels.
+It will use [youtube-dl](https://rg3.github.io/youtube-dl/) to automatically
+download the videos.
+
+To download a youtube channel, you need to find the feed associated with that
+channel, and pass it to `git annex importfeed`. There does not seem to be
+an easy link anywhere to get the feed, but you can construct its url
+manually. For a channel url like
+"https://www.youtube.com/channel/$foo", the
+feed is "https://www.youtube.com/feeds/videos.xml?channel_id=$foo"
+
+Use of youtube-dl is disabled by default as it can be a security risk. 
+See the documentation of annex.security.allowed-http-addresses
+in [[git-annex]] for details.)
+
+## metadata
+
+As well as storing the urls for items imported from a feed, git-annex can
+store additional [[metadata]], like the author, and itemdescription.
+This can then be looked up later, used in [[metadata_driven_views]], etc.
+
+To make all available metadata from the feed be stored:
+`git config annex.genmetadata true`
diff --git a/doc/tips/downloading_podcasts/comment_10_4d4f6c22070b58918ee8d34c5e7290ad._comment b/doc/tips/downloading_podcasts/comment_10_4d4f6c22070b58918ee8d34c5e7290ad._comment
new file mode 100644
index 0000000000..3bf5afe681
--- /dev/null
+++ b/doc/tips/downloading_podcasts/comment_10_4d4f6c22070b58918ee8d34c5e7290ad._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="http://joeyh.name/"
+ ip="2001:4978:f:21a::2"
+ subject="comment 10"
+ date="2013-08-05T16:47:30Z"
+ content="""
+`cabal install feed` should get the necessary library installed so that git-annex will build with feeds support.
+"""]]
diff --git a/doc/tips/downloading_podcasts/comment_11_d8d77048c7e2524968c188e1ad517873._comment b/doc/tips/downloading_podcasts/comment_11_d8d77048c7e2524968c188e1ad517873._comment
new file mode 100644
index 0000000000..fd34599265
--- /dev/null
+++ b/doc/tips/downloading_podcasts/comment_11_d8d77048c7e2524968c188e1ad517873._comment
@@ -0,0 +1,24 @@
+[[!comment format=mdwn
+ username="http://a-or-b.myopenid.com/"
+ ip="220.244.41.108"
+ subject="comment 11"
+ date="2013-08-06T04:20:16Z"
+ content="""
+    $ cabal install feed
+    Resolving dependencies...
+    All the requested packages are already installed:
+    feed-0.3.9.1
+    Use --reinstall if you want to reinstall anyway.
+
+Then I reinstalled `git-annex` but it still doesn't find the feeds flag.
+
+    $ git annex version
+    git-annex version: 4.20130802
+    build flags: Assistant Webapp Pairing Testsuite S3 WebDAV FsEvents XMPP DNS
+
+Do I need to do something like:
+
+    cabal install git-annex --bindir=$HOME/bin -f\"-assistant -webapp -webdav -pairing -xmpp -dns -feed\"
+
+...but what are the default flags to include in addition to `-feed`
+"""]]
diff --git a/doc/tips/downloading_podcasts/comment_12_0859317471b43c88744dd3df95c879f7._comment b/doc/tips/downloading_podcasts/comment_12_0859317471b43c88744dd3df95c879f7._comment
new file mode 100644
index 0000000000..e75a44a8cd
--- /dev/null
+++ b/doc/tips/downloading_podcasts/comment_12_0859317471b43c88744dd3df95c879f7._comment
@@ -0,0 +1,10 @@
+[[!comment format=mdwn
+ username="http://joeyh.name/"
+ ip="2001:4978:f:21a::2"
+ subject="comment 12"
+ date="2013-08-06T04:24:10Z"
+ content="""
+-f-Feed will disable the feature. -fFeed will try to force it on.
+
+You can probably work out what's going wrong using cabal install -v3
+"""]]
diff --git a/doc/tips/downloading_podcasts/comment_13_e8c3c97282d17e2a1d47fb9d5e2b2f7b._comment b/doc/tips/downloading_podcasts/comment_13_e8c3c97282d17e2a1d47fb9d5e2b2f7b._comment
new file mode 100644
index 0000000000..8d12428185
--- /dev/null
+++ b/doc/tips/downloading_podcasts/comment_13_e8c3c97282d17e2a1d47fb9d5e2b2f7b._comment
@@ -0,0 +1,18 @@
+[[!comment format=mdwn
+ username="http://a-or-b.myopenid.com/"
+ ip="220.244.41.108"
+ subject="comment 13"
+ date="2013-08-06T05:42:45Z"
+ content="""
+So I ran `cabal install -v3` and looked at the output, 
+
+    Flags chosen: feed=True, tdfa=True, testsuite=True, android=False,
+    production=True, dns=True, xmpp=True, pairing=True, webapp=True,
+    assistant=True, dbus=True, inotify=True, webdav=True, s3=True
+
+This looks like feed should be on.
+
+There doesn't appear to be any errors in the compile either.
+
+Is it as simple as a bug where this flag just doesn't show in the `git annex version` command?
+"""]]
diff --git a/doc/tips/downloading_podcasts/comment_14_05a3694052de36848fbbad6eeeada895._comment b/doc/tips/downloading_podcasts/comment_14_05a3694052de36848fbbad6eeeada895._comment
new file mode 100644
index 0000000000..4bc831f7f1
--- /dev/null
+++ b/doc/tips/downloading_podcasts/comment_14_05a3694052de36848fbbad6eeeada895._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="http://joeyh.name/"
+ ip="2001:4978:f:21a::2"
+ subject="comment 14"
+ date="2013-08-07T16:03:12Z"
+ content="""
+Yes, it did turn out to be as simple as my having forgotten that I have to manually add features to the version list.
+"""]]
diff --git a/doc/tips/downloading_podcasts/comment_15_21028bed8858c2dae1ac9c2d014fd2a1._comment b/doc/tips/downloading_podcasts/comment_15_21028bed8858c2dae1ac9c2d014fd2a1._comment
new file mode 100644
index 0000000000..0f998d0669
--- /dev/null
+++ b/doc/tips/downloading_podcasts/comment_15_21028bed8858c2dae1ac9c2d014fd2a1._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="http://23.gs/"
+ ip="46.165.197.5"
+ subject="No file extension?"
+ date="2013-08-12T13:21:50Z"
+ content="""
+It seems git-annex is a bit overzealous when sanitizing the file extension, currently I get: \"Nerdkunde/Let_s_go_to_the_D_M_C_A_m4a\" from http://www.nerdkunde.de/episodes.m4a.rss with the default template and only \"Nerdkunde/Let_s_go_to_the_D_M_C_A._m4a\" if I add the \".\" in the template myself...
+"""]]
diff --git a/doc/tips/downloading_podcasts/comment_16_4869fb5c9f896acc477c44de06c36ca7._comment b/doc/tips/downloading_podcasts/comment_16_4869fb5c9f896acc477c44de06c36ca7._comment
new file mode 100644
index 0000000000..4419d02a81
--- /dev/null
+++ b/doc/tips/downloading_podcasts/comment_16_4869fb5c9f896acc477c44de06c36ca7._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="arand"
+ ip="130.243.226.21"
+ subject="comment 16"
+ date="2013-08-12T13:32:46Z"
+ content="""
+The filename extension is a known issue and already fixed in the development version, see 
+"""]]
diff --git a/doc/tips/downloading_podcasts/comment_17_2e278ff200c1c15efd27c46a3e0aed40._comment b/doc/tips/downloading_podcasts/comment_17_2e278ff200c1c15efd27c46a3e0aed40._comment
new file mode 100644
index 0000000000..bc49e5dd07
--- /dev/null
+++ b/doc/tips/downloading_podcasts/comment_17_2e278ff200c1c15efd27c46a3e0aed40._comment
@@ -0,0 +1,9 @@
+[[!comment format=mdwn
+ username="https://www.google.com/accounts/o8/id?id=AItOawlpKmTa1OPwy5Jk24pOoD8Vlo2jahzTPnw"
+ nickname="Stephen"
+ subject="rss authentication"
+ date="2013-08-13T13:32:52Z"
+ content="""
+If a podcast requires authentication, is there a way to pass credentials through? I tried `http://user:pass@site.com/rss.xml` but it didn't work.
+
+"""]]
diff --git a/doc/tips/downloading_podcasts/comment_18_382f2b970738d9b1af577955c3083e90._comment b/doc/tips/downloading_podcasts/comment_18_382f2b970738d9b1af577955c3083e90._comment
new file mode 100644
index 0000000000..9e32443159
--- /dev/null
+++ b/doc/tips/downloading_podcasts/comment_18_382f2b970738d9b1af577955c3083e90._comment
@@ -0,0 +1,15 @@
+[[!comment format=mdwn
+ username="http://www.joachim-breitner.de/"
+ nickname="nomeata"
+ subject="--fast and --relaxed"
+ date="2013-08-16T07:27:59Z"
+ content="""
+Hi,
+
+the explanations to --fast and --relaxed on this page could be extended a bit. I looked it up in the man page, but it is not yet clear to me when I would use one or the other with feeds. Also, does “Next time you run git annex addurl it will only fetch any new items.” really only apply to --relaxed, and not --fast?
+
+Furthermore, it would be good if there were a template variable `itemnum` that I can use to ensure that `ls` prints the casts in the right order, even when the titles of the items are not helpful.
+
+Greetings,
+Joachim
+"""]]
diff --git a/doc/tips/downloading_podcasts/comment_19_f76fc6835e5787b0156380bf09fd81ca._comment b/doc/tips/downloading_podcasts/comment_19_f76fc6835e5787b0156380bf09fd81ca._comment
new file mode 100644
index 0000000000..41313a87d1
--- /dev/null
+++ b/doc/tips/downloading_podcasts/comment_19_f76fc6835e5787b0156380bf09fd81ca._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="http://joeyh.name/"
+ ip="4.154.0.63"
+ subject="comment 19"
+ date="2013-08-22T15:25:02Z"
+ content="""
+I would expect user:pass@site.com to work if the site is using http basic auth. `importfeed` just runs `wget` (or `curl`) to do all downloads, and wget's documentation says that works. It also says you can use ~/.netrc to store the password for a site.
+"""]]
diff --git a/doc/tips/downloading_podcasts/comment_1_f04bc32a34baeeffcd691e9f7cce0230._comment b/doc/tips/downloading_podcasts/comment_1_f04bc32a34baeeffcd691e9f7cce0230._comment
new file mode 100644
index 0000000000..014fe3f50e
--- /dev/null
+++ b/doc/tips/downloading_podcasts/comment_1_f04bc32a34baeeffcd691e9f7cce0230._comment
@@ -0,0 +1,13 @@
+[[!comment format=mdwn
+ username="ckeen"
+ ip="79.249.110.228"
+ subject="Filename too long"
+ date="2013-07-30T14:39:44Z"
+ content="""
+It seems that some of my feeds get stored into keys that generate a too long filename:
+
+    podcasts/.git/annex/tmp/b1f_325_URL-s143660317--http&c%%feedproxy.google.com%~r%mixotic%~5%urTIRWQK2OQ%Mixotic__258__-__Michael__Miller__-__Galactic__Technolgies.mp3.log.web:
+    openBinaryFile: invalid argument (File name too long)
+
+Is there a way to work around this?
+"""]]
diff --git a/doc/tips/downloading_podcasts/comment_20_65ebf3a3bbf0a2aebd2b69640b757e16._comment b/doc/tips/downloading_podcasts/comment_20_65ebf3a3bbf0a2aebd2b69640b757e16._comment
new file mode 100644
index 0000000000..6060be6557
--- /dev/null
+++ b/doc/tips/downloading_podcasts/comment_20_65ebf3a3bbf0a2aebd2b69640b757e16._comment
@@ -0,0 +1,10 @@
+[[!comment format=mdwn
+ username="http://joeyh.name/"
+ ip="4.154.0.63"
+ subject="comment 20"
+ date="2013-08-22T15:29:11Z"
+ content="""
+The git-annex man page has a bit more to say about --relaxed and --fast. Their behavior when used with `importfeed` is the same as with `addurl`.
+
+If the podcast feed provides an `itemid`, you can use that in the filename template. I don't know how common that is. Due to the way `importfeed` works, it cannot keep track of eg, an incrementing item number itself.
+"""]]
diff --git a/doc/tips/downloading_podcasts/comment_21_98a1dacc8d264ff31801e6c5c5f2612d._comment b/doc/tips/downloading_podcasts/comment_21_98a1dacc8d264ff31801e6c5c5f2612d._comment
new file mode 100644
index 0000000000..eebc00ad96
--- /dev/null
+++ b/doc/tips/downloading_podcasts/comment_21_98a1dacc8d264ff31801e6c5c5f2612d._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="Sazius"
+ ip="62.78.213.233"
+ subject="comment 21"
+ date="2014-07-01T20:52:06Z"
+ content="""
+For some podcast feeds I typically wish to view the description of the show before I decide to download it or not. Is there some way to perform that use case using git annex? I know `itemdescription` is something I can include in the template for the filename, but the descriptions can be really long... doesn't seem very elegant to have that in the file name. Could the description for example be included as metadata of the item?
+"""]]
diff --git a/doc/tips/downloading_podcasts/comment_22_00cc7a2fb936d7ea3d5d3764a1637663._comment b/doc/tips/downloading_podcasts/comment_22_00cc7a2fb936d7ea3d5d3764a1637663._comment
new file mode 100644
index 0000000000..2fba1b57dd
--- /dev/null
+++ b/doc/tips/downloading_podcasts/comment_22_00cc7a2fb936d7ea3d5d3764a1637663._comment
@@ -0,0 +1,10 @@
+[[!comment format=mdwn
+ username="http://joeyh.name/"
+ ip="209.250.56.55"
+ subject="metadata"
+ date="2014-07-03T18:25:32Z"
+ content="""
+Good idea, Sazius!
+
+I've made importfeed store the metadata, as long as annex.genmetadata is set in .git/config.
+"""]]
diff --git a/doc/tips/downloading_podcasts/comment_23_62603cda8e581a2eb2cc799dffe8a740._comment b/doc/tips/downloading_podcasts/comment_23_62603cda8e581a2eb2cc799dffe8a740._comment
new file mode 100644
index 0000000000..13909c27cd
--- /dev/null
+++ b/doc/tips/downloading_podcasts/comment_23_62603cda8e581a2eb2cc799dffe8a740._comment
@@ -0,0 +1,15 @@
+[[!comment format=mdwn
+ username="ewen"
+ subject="itempubdate"
+ date="2015-01-03T22:01:37Z"
+ content="""
+Using a `--template='${feedtitle}/${itempubdate}-${itemtitle}${extension}'` with a libsyn RSS feed (eg, [Poly Weekly](http://polyweekly.libsyn.com/rss)), I found that `itempubdate` was expanding to \"none\", even though there is a date with each entry in the RSS, eg,
+
+    Fri, 26 Dec 2014 15:25:38 +0000
+
+Maybe the date string cannot be parsed?  But it does look like a fairly typical datestamp to me.  If the cause is the mixed-case in the tag, could `pubDate` be supported in addition to `pubdate`?  (AFAICT [`pubDate` is the standardised mix of lower/upper case](http://validator.w3.org/feed/docs/rss2.html), but maybe not the most common, in which case supporting both `pubDate` and `pubdate` might help?)  As seen with `git-annex version: 5.20141024~bpo70+1`, installed from Debian Backports; AFAICT it's still the latest release to make it to backports.
+
+For now I'm just omitting \"itempubdate\" from my template.
+
+Ewen
+"""]]
diff --git a/doc/tips/downloading_podcasts/comment_24_e75af243654d15bc7b917fcd888bcf2f._comment b/doc/tips/downloading_podcasts/comment_24_e75af243654d15bc7b917fcd888bcf2f._comment
new file mode 100644
index 0000000000..25deb97b41
--- /dev/null
+++ b/doc/tips/downloading_podcasts/comment_24_e75af243654d15bc7b917fcd888bcf2f._comment
@@ -0,0 +1,13 @@
+[[!comment format=mdwn
+ username="joey"
+ subject="""pubDate"""
+ date="2015-01-05T22:55:06Z"
+ content="""
+@ewen, I tested that feed and it is able to get the pubDate from it and
+parses it ok.
+
+Most likely, your version of git-annex is not built with a new enough
+version of the haskell feed library. Version 0.3.9 or newer is needed to be
+able to extract pubdates. For example, Debian stable doesn't have a new
+enough version.
+"""]]
diff --git a/doc/tips/downloading_podcasts/comment_25_2ee88c3375eca23fe34cab65df1e7aeb._comment b/doc/tips/downloading_podcasts/comment_25_2ee88c3375eca23fe34cab65df1e7aeb._comment
new file mode 100644
index 0000000000..4f39988a9c
--- /dev/null
+++ b/doc/tips/downloading_podcasts/comment_25_2ee88c3375eca23fe34cab65df1e7aeb._comment
@@ -0,0 +1,17 @@
+[[!comment format=mdwn
+ username="ewen"
+ avatar="http://cdn.libravatar.org/avatar/605b2981cb52b4af268455dee7a4f64e"
+ subject="Track GUIDs to avoid duplicate downloads"
+ date="2017-03-21T08:59:59Z"
+ content="""
+While tracking podcast media URLs usually works to avoid duplicate downloads, when it fails it usually fails spectacularly. In particular if a podcast feed decides to update all the URLs (for old and new podcasts) to use a different URL scheme, then suddenly that looks like a huge volume of new URLs, and all of them get downloaded again -- even if the content has actually already been retrieved from a different URL (ie, older URL scheme). For instance the `acast.com` service has changed their URL scheme a couple of times in the last 1-2 years, rewriting all the historical URLs, so I have three copies of many of the episodes on podcasts on their service :-( (Many downloaded; some skipped once I caught the bulk download and stopped it/reran with `--fast` or `--relaxed` to make placeholders instead. `acast.com` seem to have managed to cause even more confusion by rewriting many of the older `mp3` files with new `id3` tags, thus changing the file size/hashes -- it definitely made cleaning up more complicated.)
+
+Some (all?) podcast feeds also have a `guid` field, which specifies what should be a unique per-episode and unchanging, that other podcatchers use to track \"seen this\" content.  In theory that `guid` value should be stable even across media URL changes -- at least if it isn't, then a podcaster changing the `guid` *and* media URL will almost certainly induce re-downloads in most podcatchers, and thus hopefully realise early on (eg, during testing) rather than in production.
+
+Can `git-annex` be extended to track the `guid` values as well as the filenames, so `git annex importfeed` can avoid downloading episodes where it has already processed that `guid`, and instead just add the newly listed url as an alternate web URL for that specific episode (which has been my manual work around).  Perhaps the episode `guid` could be stored as additional metadata, along with some sort of feed unique ID (link?), and then an index built/consulted when `importfeed` runs (although that \"feed unique ID\" would probably also have to be updatable by the user, to cope with \"the feed URL has now changed from `http://` to `https://` which also seems to be happening a bunch at present.)
+
+Ewen
+
+PS: Apologies for duplicate partial comment; I think my browser decided some key combination meant \"do default form action\", which is post -- and I wasn't finished writing.  I couldn't see a way to edit the comment, hence deleting/readding.
+
+"""]]
diff --git a/doc/tips/downloading_podcasts/comment_26_a69b4c033d85406675bb70e6996590ce._comment b/doc/tips/downloading_podcasts/comment_26_a69b4c033d85406675bb70e6996590ce._comment
new file mode 100644
index 0000000000..c884f0569b
--- /dev/null
+++ b/doc/tips/downloading_podcasts/comment_26_a69b4c033d85406675bb70e6996590ce._comment
@@ -0,0 +1,18 @@
+[[!comment format=mdwn
+ username="joey"
+ subject="""comment 26"""
+ date="2017-03-21T17:28:27Z"
+ content="""
+@ewen importfeed already tracks guids, since 2015. Relevant commit is
+[[!commit f95a8c867223b2e17d036d0d3377bf0fc9d3adff]]
+
+You may well have an
+older version of git-annex that didn't do that. But there are probably also
+feeds that lack a useful guid, or that even make a change that changes the
+guid of an existing item.
+
+With `git annex metadata`, you can see the `itemid` which is where the guid
+is stored.
+
+PS, please post in [[todo]] when you have a request..
+"""]]
diff --git a/doc/tips/downloading_podcasts/comment_27_e343aeda7c16c834599fb3caab2a51a2._comment b/doc/tips/downloading_podcasts/comment_27_e343aeda7c16c834599fb3caab2a51a2._comment
new file mode 100644
index 0000000000..ec2aef02aa
--- /dev/null
+++ b/doc/tips/downloading_podcasts/comment_27_e343aeda7c16c834599fb3caab2a51a2._comment
@@ -0,0 +1,14 @@
+[[!comment format=mdwn
+ username="ewen"
+ avatar="http://cdn.libravatar.org/avatar/605b2981cb52b4af268455dee7a4f64e"
+ subject="Tracking GUIDs"
+ date="2017-03-21T21:46:27Z"
+ content="""
+@joey - thanks, that's prompt feature request fulfilment :-)
+
+Looking more closely at the duplicates, it turns out that not *everything* got duplicated, just the \"older\" episodes.  It turns out the newer episodes do have `guid` values saved (as `itemid` in the metadata) and the older episodes do not.  I think this is most likely because I *was* running a fairly old git-annex until about October 2016, on a fairly old OS install, but then upgraded to a more recent one (now about 6 months old) which does track them.  My assumption (without checking every file) is the episodes downloaded before October 2016 are ones that got duplicated.
+
+I've edited the main page and added a note that GUIDs are tracked in versions since 2015, since I didn't obviously find that listed anywhere before.
+
+Ewen
+"""]]
diff --git a/doc/tips/downloading_podcasts/comment_2_a9a98cad7358d16792853a2ee413fe6c._comment b/doc/tips/downloading_podcasts/comment_2_a9a98cad7358d16792853a2ee413fe6c._comment
new file mode 100644
index 0000000000..f8ba1155cb
--- /dev/null
+++ b/doc/tips/downloading_podcasts/comment_2_a9a98cad7358d16792853a2ee413fe6c._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="http://joeyh.name/"
+ ip="4.154.0.21"
+ subject="comment 2"
+ date="2013-07-30T17:16:07Z"
+ content="""
+@ckeen You seem to be using a filesystem that does not support filenames 150 characters long. This is unusual -- even windows and android can support a filename up to 255 characters in length. `git-annex addurl` already deals with this sort of problem by limiting the filename to 255 characters. If you'd like to file a bug report with details about your system, I can try to make git-annex support its limitations, I suppose.
+"""]]
diff --git a/doc/tips/downloading_podcasts/comment_3_5a8068a5cb0fd864581157a3aa5d1113._comment b/doc/tips/downloading_podcasts/comment_3_5a8068a5cb0fd864581157a3aa5d1113._comment
new file mode 100644
index 0000000000..7e5633865a
--- /dev/null
+++ b/doc/tips/downloading_podcasts/comment_3_5a8068a5cb0fd864581157a3aa5d1113._comment
@@ -0,0 +1,10 @@
+[[!comment format=mdwn
+ username="http://www.joachim-breitner.de/"
+ nickname="nomeata"
+ subject="Great stuff!"
+ date="2013-07-30T21:21:57Z"
+ content="""
+Looking forward to seeing it in Debian unstable; where it will definitely replace my hpodder setup.
+
+I guess there is no easy way to re-use the files already downloaded with hpodder? At first I thought that `git annex importfeed --relaxed` followed by adding the files to the git annex would work, but `importfeed` stores URLs, not content-based hashes, so it wouldn’t match up.
+"""]]
diff --git a/doc/tips/downloading_podcasts/comment_4_e7072a9da30b4c4b4c526013144238d4._comment b/doc/tips/downloading_podcasts/comment_4_e7072a9da30b4c4b4c526013144238d4._comment
new file mode 100644
index 0000000000..1693c4bdcf
--- /dev/null
+++ b/doc/tips/downloading_podcasts/comment_4_e7072a9da30b4c4b4c526013144238d4._comment
@@ -0,0 +1,10 @@
+[[!comment format=mdwn
+ username="http://joeyh.name/"
+ ip="4.154.0.21"
+ subject="comment 4"
+ date="2013-07-30T21:29:50Z"
+ content="""
+@nomeata, well, you can, but it has to download the files again.
+
+When run without --fast, `importfeed` does use content based hashes, so if you run it in a temporary directory, it will download the content redundantly, hash it and see it's the same, and add the url to that hash. You can then delete the temporary directory, and the files hpodder had downloaded will have the url attached to them now. I don't know if this really buys you anything over deleting the hpodder files and starting over though.
+"""]]
diff --git a/doc/tips/downloading_podcasts/comment_5_79b3f8d678ac9f67df4c0cd649657283._comment b/doc/tips/downloading_podcasts/comment_5_79b3f8d678ac9f67df4c0cd649657283._comment
new file mode 100644
index 0000000000..f5df9910f3
--- /dev/null
+++ b/doc/tips/downloading_podcasts/comment_5_79b3f8d678ac9f67df4c0cd649657283._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="ckeen"
+ ip="79.249.110.228"
+ subject="Force a reload of a feed?"
+ date="2013-07-31T10:35:50Z"
+ content="""
+Currently I have my podcasts imported with --fast. For some reason there are podcast episodes missing. This has been done propably during my period of toying with the feature. If I retry on a clean annex I see all episodes. My suspicion is that git-annex has been interrupted during downloading a feed but now somehow thinks it's already there. How can I debug this situation and/or force git annex to retry all the links in a feed?
+"""]]
diff --git a/doc/tips/downloading_podcasts/comment_6_35106fee5458bdd5c21868fbc49d3616._comment b/doc/tips/downloading_podcasts/comment_6_35106fee5458bdd5c21868fbc49d3616._comment
new file mode 100644
index 0000000000..caeca01511
--- /dev/null
+++ b/doc/tips/downloading_podcasts/comment_6_35106fee5458bdd5c21868fbc49d3616._comment
@@ -0,0 +1,10 @@
+[[!comment format=mdwn
+ username="http://joeyh.name/"
+ ip="4.154.0.21"
+ subject="use the force"
+ date="2013-07-31T16:20:39Z"
+ content="""
+The only way it can skip downloading a file is if its url has already been seen before. Perhaps you deleted them?
+
+I've made `importfeed --force` re-download files it's seen before.
+"""]]
diff --git a/doc/tips/downloading_podcasts/comment_7_ceb16498b7aadbf04a27acd5d6561d46._comment b/doc/tips/downloading_podcasts/comment_7_ceb16498b7aadbf04a27acd5d6561d46._comment
new file mode 100644
index 0000000000..ac2c89a366
--- /dev/null
+++ b/doc/tips/downloading_podcasts/comment_7_ceb16498b7aadbf04a27acd5d6561d46._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="ckeen"
+ ip="78.108.63.46"
+ subject="--force reload all URLs"
+ date="2013-08-01T09:47:34Z"
+ content="""
+Is it intentionally saving URLs with a prefixed 2_? I have sorted out all missing URLs and renamed it, so no harm done, but it has been a bit of a hassle to get there.
+"""]]
diff --git a/doc/tips/downloading_podcasts/comment_8_147397603f0b3fdb42ca387d1da7c5ef._comment b/doc/tips/downloading_podcasts/comment_8_147397603f0b3fdb42ca387d1da7c5ef._comment
new file mode 100644
index 0000000000..0995d80752
--- /dev/null
+++ b/doc/tips/downloading_podcasts/comment_8_147397603f0b3fdb42ca387d1da7c5ef._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="http://joeyh.name/"
+ ip="4.152.108.145"
+ subject="comment 8"
+ date="2013-08-01T16:05:10Z"
+ content="""
+I've now made importfeed --force a bit smarter about reusing existing files.
+"""]]
diff --git a/doc/tips/downloading_podcasts/comment_9_6a26a6cc7683d38fae0f23c5a52d1e23._comment b/doc/tips/downloading_podcasts/comment_9_6a26a6cc7683d38fae0f23c5a52d1e23._comment
new file mode 100644
index 0000000000..3045c98949
--- /dev/null
+++ b/doc/tips/downloading_podcasts/comment_9_6a26a6cc7683d38fae0f23c5a52d1e23._comment
@@ -0,0 +1,24 @@
+[[!comment format=mdwn
+ username="http://a-or-b.myopenid.com/"
+ ip="220.244.41.108"
+ subject="How do I switch on the 'feeds' feature?"
+ date="2013-08-05T04:52:41Z"
+ content="""
+Joey - your initial post said:
+
+    git-annex must be built with the Feeds feature (run git annex version to check).
+
+...but how do I actually switch on the feeds feature?
+
+I install git-annex from cabal, so I do
+
+    cabal update
+    cabal install git-annex
+
+which I did this morning and now `git annex version` gives me:
+
+    git-annex version: 4.20130802
+    build flags: Assistant Webapp Pairing Testsuite S3 WebDAV FsEvents XMPP DNS
+
+So it is the latest version, but without Feeds.  :-(
+"""]]
diff --git a/doc/tips/dropboxannex.mdwn b/doc/tips/dropboxannex.mdwn
new file mode 100644
index 0000000000..0f82f659d7
--- /dev/null
+++ b/doc/tips/dropboxannex.mdwn
@@ -0,0 +1,29 @@
+dropboxannex 0.2.0
+=========
+
+Hook program for gitannex to use dropbox as backend
+
+# Requirements:
+
+    python2
+    python-pkg-resources
+
+Credit for the Dropbox api interface goes to Dropbox.
+
+# Install
+Clone the git repository in your home folder.
+
+    git clone git://github.com/TobiasTheViking/dropboxannex.git 
+
+This should make a ~/dropboxannex folder
+
+# Setup
+Make the file executable, and link it into PATH
+
+    cd ~/dropboxannex; chmod +x git-annex-remote-dropbox; sudo ln -sf `pwd`/git-annex-remote-dropbox /usr/local/bin/git-annex-remote-dropbox
+
+# Commands for gitannex:
+
+    git annex initremote dropbox type=external externaltype=dropbox encryption=shared folder=gitannex
+    git annex describe dropbox "the dropbox library"
+
diff --git a/doc/tips/dumb_metadata_extraction_from_xbmc.mdwn b/doc/tips/dumb_metadata_extraction_from_xbmc.mdwn
new file mode 100644
index 0000000000..e3611ec30b
--- /dev/null
+++ b/doc/tips/dumb_metadata_extraction_from_xbmc.mdwn
@@ -0,0 +1,29 @@
+I wanted to get the list of movies I haven't seen yet in XBMC, and i'm lazy. So I'll use [[metadata]] to be able to extract those movies only, for the road for example.
+
+First I fiddled around with shell scripts to extract the list of those films, which in XBMC-speak means that have a `NULL playCount`. Since there are two ways that XMBC can represent those files (in a `stack://` if there is multiple files for the movie or not), there are two scripts. For "stacked" movies:
+
+    echo 'SELECT files.strFileName FROM movie JOIN files ON files.idFile=movie.idFile JOIN path ON path.idPath=files.idPath WHERE playCount IS NULL AND files.strFileName LIKE "stack://%";' | sqlite3 /home/video/.xbmc/userdata/Database/MyVideos75.db | sed "s#stack://##;s/, /\n/g" | sed "s#/home/media/video/##"
+
+And the rest:
+
+    echo 'SELECT path.strPath || files.strFileName FROM movie JOIN files ON files.idFile=movie.idFile JOIN path ON path.idPath=files.idPath WHERE playCount IS NULL AND files.strFileName NOT LIKE "stack://%";' | sqlite3 /home/video/.xbmc/userdata/Database/MyVideos75.db | sed "s#/home/media/video/##"
+
+Also notice how I remove the absolute prefix for the annex so that i can refer to files as a relative path.
+
+So this quick and dirty hack could have been used to mark files as "new". Unfortunately, this won't unmark them when the playcount increases. So instead I think this should be a field, and we need to extract the playcount. Play around with shell scripting enough to get sick, get back into bad perl habits and you'll end up with this nasty script: [[git-annex-xbmc-playcount.pl]].
+
+After the script is ran, you can sort the files by play count with:
+
+    git annex view "playCount=*"
+
+Or just show the files that haven't been played yet:
+
+    git annex view playCount=0
+
+Use `git checkout master` to reset the view. Note that the above will flatten the tree hierarchy, which you may not want. Try this in that case:
+
+    git annex view playCount=0 films/=*
+
+For more information, see [[tips/metadata_driven_views/]].
+
+-- [[anarcat]]
diff --git a/doc/tips/dumb_metadata_extraction_from_xbmc/git-annex-xbmc-playcount.pl b/doc/tips/dumb_metadata_extraction_from_xbmc/git-annex-xbmc-playcount.pl
new file mode 100644
index 0000000000..56f902cb03
--- /dev/null
+++ b/doc/tips/dumb_metadata_extraction_from_xbmc/git-annex-xbmc-playcount.pl
@@ -0,0 +1,227 @@
+#! /usr/bin/perl -w
+
+use Getopt::Long;
+use Pod::Usage;
+
+my $help = 0;
+my $usage = 0;
+my $dryrun = 0;
+my $verbose = 0;
+my $path = '';
+my $annex = '';
+my $home = $ENV{'HOME'};
+
+sub main() {
+    checkargs();
+    if (!$path) {
+        $path = $home . '/.xbmc/userdata/Database';
+    }
+    print("# checking XBMC directory '$path'\n") if ($verbose);
+    $dbpath = finddb($path);
+    if (!$dbpath) {
+        pod2usage("$0: can't find a XBMC database in '$path'.");
+    }
+    print("# using database '$dbpath'\n") if ($verbose);
+    checkdb();
+}
+
+# list videos database, find the latest one
+# modified version of
+# http://stackoverflow.com/questions/4651092/getting-the-list-of-files-sorted-by-modification-date-in-perl
+sub finddb($) {
+    my $path = shift(@_);
+    opendir my($dirh), $path or die "can't opendir $path: $!";
+    my @flist = sort {  -M $a <=> -M $b } # Sort by modification time
+        map  { "$path/$_" } # We need full paths for sorting
+        grep { /^MyVideos.*\.db$/ }
+        readdir $dirh;
+    closedir $dirh;
+    if ($#flist > 0) {
+        return $flist[0];
+    }
+    else {
+        return 0;
+    }
+}
+
+sub checkargs() {
+    pod2usage(1) if $help;
+    pod2usage(-exitval => 0, -verbose => 2) if $usage;
+
+    GetOptions('h|?' => \$help,
+               'help|usage' => \$usage,
+               # we want to operate on relative links, so set this to
+               # the common annex to the git annex repo
+               'annex=s' => \$annex,
+               'path=s' => \$path,
+               'home=s' => \$home,
+               'dryrun|n' => \$dryrun,
+               'verbose|v' => \$verbose,
+        )
+        or die("Error parsing commandline\n");
+}
+
+sub checkdb() {
+    my @lines = `echo 'SELECT playCount, path.strPath, files.strFileName FROM movie JOIN files ON files.idFile=movie.idFile JOIN path ON path.idPath=files.idPath;' | sqlite3 $dbpath`;
+    print "# finding files...\n" if $verbose;
+    for (@lines) {
+        my ($count, $dir, $file) = split /\|/;
+        chomp $file;
+        # empty or non-numeric count is zero
+        if ($count !~ /[0-9]/) {
+            $count = 0;
+        }
+        print "# $dir/$file\n" if $verbose;
+        if ($file =~ s#stack://##) {
+            for (split /,/, $file) {
+                s/$annex//;
+                s/^ //;
+                s/ $//;
+                my @cmd = (qw(git annex metadata --set), "playCount=$count", $_);
+                if ($dryrun) {
+                    print join(' ', @cmd) . "\n";
+                }
+                else {
+                    system(@cmd);
+                }
+            }
+        }
+        else {
+            $dir =~ s/$annex//;
+            my @cmd = (qw(git annex metadata --set), "playCount=$count", "$dir$file");
+            if ($dryrun) {
+                print join(' ', @cmd) . "\n";
+            }
+            else {
+                system(@cmd);
+            }
+        }
+    }
+}
+
+main();
+
+__END__
+=encoding utf8
+
+=head1 NAME
+
+git-annex-xbmc-playcount - register XBMC playcounts as git-annex metadata
+
+=head1 SYNOPSIS
+
+git-annex-xbmc-playcount [--path .xbmc/userdata/Database]
+
+ Options:
+  -h         short usage
+  --help     complete help
+  --dryrun, -n do nothing and show the commands that would be ran
+  --annex    path to the git-annex repo
+  --home     the home directory where the .xbmc directory is located
+  --path     the location of the Database directory of XBMC, overrides --home
+  --verbose  show interaction details with the database
+
+=head1 DESCRIPTION
+
+This program will look into the XBMC database for the "playcount"
+field to register that number as metadata in the git-annex repository.
+
+=head1 OPTIONS
+
+=over 8
+
+=item B<--dryrun>
+
+Do nothing but show all the steps that would be ran. The output can be
+piped through a POSIX shell after inspection. B<-n> is an alias of
+this command. Example:
+
+    git-annex-xbmc-playcount -n | tee runme
+    # inspect the output
+    sh < runme
+
+=item B<--annex>
+
+This option allows the user to specify the root of the git-annex
+repository, which is then stripped off the paths found in the XBMC
+database.
+
+=item B<--home>
+
+Home of the user running XBMC. If not specified, defaults to the $HOME
+environment variables. The script will look into
+B<$home/.xbmc/userdata/Database> for a file matching
+B<^MyVideos.*\.db$> and will fail if none is found.
+
+=item B<--path>
+
+Manually specify the path to B<.xbmc/userdata/Database>. This
+overrides B<--home>.
+
+Note that this doesn't point directly to the database itself, because
+there are usually many database files and we want to automatically
+find the latest. This may be a stupid limitation.
+
+=item B<--verbose>
+
+Show more information about path discovery. Doesn't obstruct
+B<--dryrun> output because lines are prefixed with C<#>.
+
+=back
+
+=head1 EXAMPLES
+
+You have a git annex in B and XBMC is ran as the
+B    
+
+There's support for downloading videos from sites like YouTube, Vimeo,
+and many more. This relies on [youtube-dl](https://rg3.github.io/youtube-dl/)
+to download the videos.
+
+When you have youtube-dl installed, you can just 
+`git annex addurl http://youtube.com/foo` and it will detect that
+it is a video and download the video content for offline viewing.
+
+(However, this is disabled by default as it can be a security risk. 
+See the documentation of annex.security.allowed-http-addresses
+in [[git-annex]] for details.)
+
+Later, in another clone of the repository, you can run `git annex get` on
+the file and it will also be downloaded with youtube-dl. This works
+even if the video host has transcoded or otherwise changed the video
+in the meantime; the assumption is that these video files are equivalent.
+
+There is an `annex.youtube-dl-options` configuration setting that can be used
+to pass parameters to quvi. For example, you could set `git config
+annex.youtube-dl-options "--format worst"` to configure it to download low
+quality videos from YouTube. Note that the youtube-dl configuration files
+are not read when git-annex runs youtube-dl, to avoid config settings that
+break its integration.
+
+To download a youtube channel, you need to find the RSS feed associated with
+that channel, and pass it to `git annex importfeed`. There does not seem to
+be an easy link anywhere to get the RSS feed, but you can construct its url
+manually. For a channel like
+"https://www.youtube.com/channel/$foo", the
+feed is "https://www.youtube.com/feeds/videos.xml?channel_id=$foo"
+
+## bittorrent
+
+The [[bittorrent_special_remote|special_remotes/bittorrent]] lets git-annex
+also download the content of torrent files, and magnet links to torrents.
+
+You can simply pass the url to a torrent to `git annex addurl`
+the same as any other url.
+
+You have to have [aria2](http://aria2.sourceforge.net/) 
+and bittornado (or the original bittorrent) installed for this
+to work.
+
+## podcasts
+
+This is done using `git annex importfeed`. See [[downloading podcasts]].
diff --git a/doc/tips/using_the_web_as_a_special_remote/comment_11_9889828caa47aad88267d0ec35f2240d._comment b/doc/tips/using_the_web_as_a_special_remote/comment_11_9889828caa47aad88267d0ec35f2240d._comment
new file mode 100644
index 0000000000..ccd27bf3c4
--- /dev/null
+++ b/doc/tips/using_the_web_as_a_special_remote/comment_11_9889828caa47aad88267d0ec35f2240d._comment
@@ -0,0 +1,7 @@
+[[!comment format=mdwn
+ username="thnetos"
+ subject="How to get the URL of an added file?"
+ date="2014-12-18T18:56:57Z"
+ content="""
+Once a file has been added with either `addurl` or `importfeed`, how can I get the URL of the file or feed from git-annex?
+"""]]
diff --git a/doc/tips/using_the_web_as_a_special_remote/comment_12_1704dcd7fc94432af5aa18459495c9ab._comment b/doc/tips/using_the_web_as_a_special_remote/comment_12_1704dcd7fc94432af5aa18459495c9ab._comment
new file mode 100644
index 0000000000..522f8c63f1
--- /dev/null
+++ b/doc/tips/using_the_web_as_a_special_remote/comment_12_1704dcd7fc94432af5aa18459495c9ab._comment
@@ -0,0 +1,7 @@
+[[!comment format=mdwn
+ username="https://me.yahoo.com/a/EbvxpTI_xP9Aod7Mg4cwGhgjrCrdM5s-#7c0f4"
+ subject="how to get access to urls needing authentication without private info in the url"
+ date="2015-09-22T14:05:45Z"
+ content="""
+wget by default consults ~/.netrc where you could specify your account information (man netrc for more detail)
+"""]]
diff --git a/doc/tips/using_the_web_as_a_special_remote/comment_12_2ce018f181e039b5dd52e2b712f63eea._comment b/doc/tips/using_the_web_as_a_special_remote/comment_12_2ce018f181e039b5dd52e2b712f63eea._comment
new file mode 100644
index 0000000000..7e3d8c9c72
--- /dev/null
+++ b/doc/tips/using_the_web_as_a_special_remote/comment_12_2ce018f181e039b5dd52e2b712f63eea._comment
@@ -0,0 +1,7 @@
+[[!comment format=mdwn
+ username="joey"
+ subject="""comment 12"""
+ date="2014-12-18T19:59:54Z"
+ content="""
+You can see the url(s) of a file when you run `git annex whereis $file`
+"""]]
diff --git a/doc/tips/using_the_web_as_a_special_remote/comment_13_39122e2587f84633fda262bfc9d076eb._comment b/doc/tips/using_the_web_as_a_special_remote/comment_13_39122e2587f84633fda262bfc9d076eb._comment
new file mode 100644
index 0000000000..d7448cd71a
--- /dev/null
+++ b/doc/tips/using_the_web_as_a_special_remote/comment_13_39122e2587f84633fda262bfc9d076eb._comment
@@ -0,0 +1,11 @@
+[[!comment format=mdwn
+ username="hobbes@b2cacef69071743c3a831e60511062f7e014e52f"
+ nickname="hobbes"
+ avatar="http://cdn.libravatar.org/avatar/44b70169c4d862b3619812c360aa8f1e"
+ subject="Youtube-dl options per file?"
+ date="2018-03-08T18:47:57Z"
+ content="""
+If I have a big repo of YouTube stuff I might have some videos that I want to download with different options. Maybe I want higher quality for some videos, and don't care for others. It seems like youtube-dl-options can only be specified for an entire annex, though. I'd like to be able to do it *per file*.
+
+The main motivation for this is YouTube videos where I only want the audio. Is there a good way to do this? Best I can think of is having a separate annex for audio and video.
+"""]]
diff --git a/doc/tips/using_the_web_as_a_special_remote/comment_14_3a923511cf20d816b468044c03eb171f._comment b/doc/tips/using_the_web_as_a_special_remote/comment_14_3a923511cf20d816b468044c03eb171f._comment
new file mode 100644
index 0000000000..7fd9e7f697
--- /dev/null
+++ b/doc/tips/using_the_web_as_a_special_remote/comment_14_3a923511cf20d816b468044c03eb171f._comment
@@ -0,0 +1,13 @@
+[[!comment format=mdwn
+ username="joey"
+ subject="""comment 14"""
+ date="2018-03-21T19:55:52Z"
+ content="""
+@hobbes, there's not currently a way to do per-file youtube-dl options.
+The difficulty is that we don't know what youtube-dl options might be
+unsafe, and which such a feature could make eg `git annex get` use when run
+by a different user.
+
+I feel that this needs some support in youtube-dl to avoid git-annex
+needing to know about all its safe options.
+"""]]
diff --git a/doc/tips/using_the_web_as_a_special_remote/comment_1_321a41d611c6fe45e047af9c96c5176c._comment b/doc/tips/using_the_web_as_a_special_remote/comment_1_321a41d611c6fe45e047af9c96c5176c._comment
new file mode 100644
index 0000000000..ee1a271eac
--- /dev/null
+++ b/doc/tips/using_the_web_as_a_special_remote/comment_1_321a41d611c6fe45e047af9c96c5176c._comment
@@ -0,0 +1,26 @@
+[[!comment format=mdwn
+ username="https://www.google.com/accounts/o8/id?id=AItOawlc1og3PqIGudOMkFNrCCNg66vB7s-jLpc"
+ nickname="Paul"
+ subject="can addurl use hashing once the file is downloaded?"
+ date="2012-09-20T21:01:30Z"
+ content="""
+There are resources that I want to add to my annex that are currently available
+via a URL, but it seems like if I add these using `git-annex addurl`, they get
+symlinked to file in the annex/objects directory that starts with `URL-...`,
+instead of the more typical `SHA256-...`, and this does not change even after
+the files are downloaded.
+
+My concern is that I really want to ensure that these files don't change, which
+is the appeal of content-addressable symlinking of normal files (as opposed to
+URL addressable ones).
+
+Would there be a way to automate the injection of hash-based symlinking for
+files that are added via addurl? Sometimes I add a bunch of files via ``addurl
+--fast``, and after I've download them via ``get``, it would be nice to have
+those files have the same level of data integrity as when I download them using
+something outside of git-annex, add them to the annex, and do an ``addurl
+--file`` afterward.
+
+Thanks for all of your hard work!
+
+"""]]
diff --git a/doc/tips/using_the_web_as_a_special_remote/comment_2_dfe9c8c49aadff80d2020288584e0390._comment b/doc/tips/using_the_web_as_a_special_remote/comment_2_dfe9c8c49aadff80d2020288584e0390._comment
new file mode 100644
index 0000000000..b015cdcece
--- /dev/null
+++ b/doc/tips/using_the_web_as_a_special_remote/comment_2_dfe9c8c49aadff80d2020288584e0390._comment
@@ -0,0 +1,10 @@
+[[!comment format=mdwn
+ username="http://joeyh.name/"
+ subject="comment 2"
+ date="2012-09-20T21:55:57Z"
+ content="""
+`addurl` only uses the URL- keys if you run it with --fast. Otherwise it downloads the content and hashes it the same as `add` does.
+
+If you use `--fast`, you can go back and `git annex migrate` the file once it's been downloaded, to convert 
+it to the SHA backend.
+"""]]
diff --git a/doc/tips/using_the_web_as_a_special_remote/comment_3_ed8dd3bbd9b9ae7f2309b72b94f61eb1._comment b/doc/tips/using_the_web_as_a_special_remote/comment_3_ed8dd3bbd9b9ae7f2309b72b94f61eb1._comment
new file mode 100644
index 0000000000..0601f30058
--- /dev/null
+++ b/doc/tips/using_the_web_as_a_special_remote/comment_3_ed8dd3bbd9b9ae7f2309b72b94f61eb1._comment
@@ -0,0 +1,18 @@
+[[!comment format=mdwn
+ username="https://www.google.com/accounts/o8/id?id=AItOawnx8kHW66N3BqmkVpgtXDlYMvr8TJ5VvfY"
+ nickname="Yaroslav"
+ subject="how to drop one of the urls?"
+ date="2013-04-12T14:53:29Z"
+ content="""
+is there a way to remove one of the urls? e.g. if I have
+
+    $> git annex whereis fail2ban_logo.png
+    whereis fail2ban_logo.png (1 copy) 
+      	00000000-0000-0000-0000-000000000001 -- web
+
+      web: http://www.fail2ban.org/fail2ban_logo.png
+      web: http://www.onerussian.com/tmp/statsmodes.png
+    ok
+
+and would like to remove the fail2ban.org one... ?
+"""]]
diff --git a/doc/tips/using_the_web_as_a_special_remote/comment_4_c1133a524989a940f1b5db588707157a._comment b/doc/tips/using_the_web_as_a_special_remote/comment_4_c1133a524989a940f1b5db588707157a._comment
new file mode 100644
index 0000000000..bd55a78727
--- /dev/null
+++ b/doc/tips/using_the_web_as_a_special_remote/comment_4_c1133a524989a940f1b5db588707157a._comment
@@ -0,0 +1,10 @@
+[[!comment format=mdwn
+ username="http://joeyh.name/"
+ nickname="joey"
+ subject="comment 4"
+ date="2013-04-22T21:28:03Z"
+ content="""
+You can use `git annex rmurl $file $url`, which I just added to git-annex.
+
+(Also, `git annex drop $file --from web` will remove all the urls..)
+"""]]
diff --git a/doc/tips/using_the_web_as_a_special_remote/comment_5_5ee9717e74ca2afed98e81fc0ea98a95._comment b/doc/tips/using_the_web_as_a_special_remote/comment_5_5ee9717e74ca2afed98e81fc0ea98a95._comment
new file mode 100644
index 0000000000..b1060147f2
--- /dev/null
+++ b/doc/tips/using_the_web_as_a_special_remote/comment_5_5ee9717e74ca2afed98e81fc0ea98a95._comment
@@ -0,0 +1,18 @@
+[[!comment format=mdwn
+ username="Xyem"
+ ip="87.194.19.134"
+ subject="comment 5"
+ date="2014-04-04T15:25:39Z"
+ content="""
+Adding videos from youtube ends up with it using the URL backend, even without fast.
+
+    $ git init quvitest
+    $ cd quvitest/
+    $ git annex init
+    $ git annex addurl https://www.youtube.com/watch?v=mghhLqu31cQ
+    (... file is downloaded ...)
+    $ find .git/annex/objects/ -type f
+    .git/annex/objects/1J/Wp/URL--quvi&chttps&c%%www.youtube.com%watch,63v,61mghhLqu31cQ/URL--quvi&chttps&c%%www.youtube.com%watch,63v,61mghhLqu31cQ
+
+Is migrating manually required or should I log a bug?
+"""]]
diff --git a/doc/tips/using_the_web_as_a_special_remote/comment_6_dceb15bd656e69eefa3ca975d9d642de._comment b/doc/tips/using_the_web_as_a_special_remote/comment_6_dceb15bd656e69eefa3ca975d9d642de._comment
new file mode 100644
index 0000000000..0c39eb1831
--- /dev/null
+++ b/doc/tips/using_the_web_as_a_special_remote/comment_6_dceb15bd656e69eefa3ca975d9d642de._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="http://joeyh.name/"
+ ip="209.250.56.244"
+ subject="comment 6"
+ date="2014-04-07T20:07:45Z"
+ content="""
+Using the URL backend for youtube is intentional. Youtube may serve up different encodings for the same video over time, and this way git-annex treats them all as equvilant. If you want to \"freeze\" the repository to the current one, use `git annex migrate`, and be prepared for `git annex get --from web` to not work long term.
+"""]]
diff --git a/doc/tips/using_the_web_as_a_special_remote/comment_7_0bde977c62a53c90cb20491936bc399d._comment b/doc/tips/using_the_web_as_a_special_remote/comment_7_0bde977c62a53c90cb20491936bc399d._comment
new file mode 100644
index 0000000000..df5b0ef41e
--- /dev/null
+++ b/doc/tips/using_the_web_as_a_special_remote/comment_7_0bde977c62a53c90cb20491936bc399d._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="https://www.google.com/accounts/o8/id?id=AItOawnkXKIAfgPsuV-EJpXm3jCc3q9IZwzvCcw"
+ nickname="Garrett"
+ subject="Default pathdepth"
+ date="2014-05-08T21:36:00Z"
+ content="""
+Is there away to change the default pathdepth so I do not need to add --pathdepth=-1 everything I run addurl?
+"""]]
diff --git a/doc/tips/using_the_web_as_a_special_remote/comment_8_3f32d536f51d5e9908953caf5736b0a0._comment b/doc/tips/using_the_web_as_a_special_remote/comment_8_3f32d536f51d5e9908953caf5736b0a0._comment
new file mode 100644
index 0000000000..c3403500a3
--- /dev/null
+++ b/doc/tips/using_the_web_as_a_special_remote/comment_8_3f32d536f51d5e9908953caf5736b0a0._comment
@@ -0,0 +1,16 @@
+[[!comment format=mdwn
+ username="https://www.google.com/accounts/o8/id?id=AItOawnvr2UPmp7ABeH0yI8KGAHCqFhl91Ju4Tc"
+ nickname="Calvin"
+ subject="HTTP Authentication?"
+ date="2014-09-29T21:37:44Z"
+ content="""
+Hi!
+
+I have a somewhat interesting use case. My course notes require HTTP authentication. This is possible with wget, but is there any way to make git annex do it?
+
+[wget authentication stuff!](http://stackoverflow.com/questions/4272770/wget-with-authentication)
+
+It would be nice to have the user and pass encrypted with GPG too. This might be a strange use case, but I can see other people wanting to do something like this in the future.
+
+Thanks!
+"""]]
diff --git a/doc/tips/using_the_web_as_a_special_remote/comment_9_b420b1f320d620a9909cce5086c549bf._comment b/doc/tips/using_the_web_as_a_special_remote/comment_9_b420b1f320d620a9909cce5086c549bf._comment
new file mode 100644
index 0000000000..d6b194d69a
--- /dev/null
+++ b/doc/tips/using_the_web_as_a_special_remote/comment_9_b420b1f320d620a9909cce5086c549bf._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="http://joeyh.name/"
+ ip="209.250.56.55"
+ subject="comment 9"
+ date="2014-09-30T18:09:04Z"
+ content="""
+For urls using http basic auth, you can use the standard url form, http://username:password@example.org/url/ , which should work with `git annex addurl`. The url, including the password, will be stored in the git-annex branch though. If you want to protect the password from being exposed to anyone who gets a clone of the repository, just download manually, and then `git annex add` the file.
+"""]]
diff --git a/doc/tips/visualizing_repositories_with_gource.mdwn b/doc/tips/visualizing_repositories_with_gource.mdwn
new file mode 100644
index 0000000000..25a69c1b7a
--- /dev/null
+++ b/doc/tips/visualizing_repositories_with_gource.mdwn
@@ -0,0 +1,22 @@
+[Gource](http://code.google.com/p/gource/) is an amazing animated
+visualisation of a git repository.
+
+Normally, gource shows files being added, removed, and changed in
+the repository, and the user(s) making the changes. Of course it can be
+used in this way in a repository using git-annex too; just run `gource`.
+
+The other way to use gource with git-annex is to visualise the movement of
+annexed file contents between repositories. In this view, the "users" are
+repositories, and they move around the file contents that are being added
+or removed from them with git-annex.
+
+[[!img screenshot.jpg]]
+
+To use gource this way, first go into the directory you want to visualize,
+and use `git annex log` to make an input file for `gource`:
+
+	git annex log --gource | tee gource.log
+	sort gource.log | gource --log-format custom -
+
+The `git annex log` can take a while, to speed it up you can use something
+like `--after "4 months ago"` to limit how far back it goes.
diff --git a/doc/tips/visualizing_repositories_with_gource/comment_1_01c5cd21375990c612b8f291904ddb3e._comment b/doc/tips/visualizing_repositories_with_gource/comment_1_01c5cd21375990c612b8f291904ddb3e._comment
new file mode 100644
index 0000000000..38bde579da
--- /dev/null
+++ b/doc/tips/visualizing_repositories_with_gource/comment_1_01c5cd21375990c612b8f291904ddb3e._comment
@@ -0,0 +1,10 @@
+[[!comment format=mdwn
+ username="https://www.google.com/accounts/o8/id?id=AItOawm9IQStaE1el95_9s77CgmJhxZwCwUeN9A"
+ nickname="Sam"
+ subject="You can also use it for LIVE display now"
+ date="2013-12-19T14:29:18Z"
+ content="""
+hi, check out http://puffingdev.com/live-gource-visually-display-your-svngit-activity-live/
+
+You can use it to display LIVE updates of a git repo, nice to have on a screen in your company's common area for example.
+"""]]
diff --git a/doc/tips/visualizing_repositories_with_gource/screenshot.jpg b/doc/tips/visualizing_repositories_with_gource/screenshot.jpg
new file mode 100644
index 0000000000..9d3096b99b
Binary files /dev/null and b/doc/tips/visualizing_repositories_with_gource/screenshot.jpg differ
diff --git a/doc/tips/what_to_do_when_a_repository_is_corrupted.mdwn b/doc/tips/what_to_do_when_a_repository_is_corrupted.mdwn
new file mode 100644
index 0000000000..78f4977a27
--- /dev/null
+++ b/doc/tips/what_to_do_when_a_repository_is_corrupted.mdwn
@@ -0,0 +1,27 @@
+A git-annex repository on a removable USB drive is great, until the cable
+falls out at the wrong time and git's repository gets trashed. The way
+git checksums everything and the poor quality of USB media makes this
+perhaps more likely than you would expect. If this happens to you,
+here's a way to recover that makes the most of whatever data is left
+on the drive.
+
+* First, run `git fsck`. If it does not report any problems, your data
+  is fine, and you don't need to proceed further.
+* So `git fsck` says the git repository is corrupted. But probably the data
+  git-annex stored is fine. Your first step is to clone another copy
+  of the git repository from somewhere else. Let's call this clone
+  "$good", and the corrupted repository "$bad".
+* Preserve your git configuration changes, and the `annex.uuid` setting:
+  `mv $bad/.git/config $good/.git/config`
+* Move annexed data into the new repository: `mkdir $good/.git/annex; mv
+  $bad/.git/annex/objects $good/.git/annex/objects`
+* Reinitalize git-annex: `cd $good; git annex init`
+* Check for any problems with the annexed data: `cd $good; git annex fsck`
+* Now you can remove the corrupted repository, the new one is ready to use.
+
+Alternatively, recent versions of git-annex have a `git annex repair`
+command that uses  to repair a
+repository in-place. The git-annex assistant will detect most corruptions
+and offer to run the repair for you automatically.
+
+--[[Joey]]
diff --git a/doc/tips/what_to_do_when_a_repository_is_corrupted/comment_1_c3543190eae2af594f3e050057e80db6._comment b/doc/tips/what_to_do_when_a_repository_is_corrupted/comment_1_c3543190eae2af594f3e050057e80db6._comment
new file mode 100644
index 0000000000..3954bef0d9
--- /dev/null
+++ b/doc/tips/what_to_do_when_a_repository_is_corrupted/comment_1_c3543190eae2af594f3e050057e80db6._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="https://www.google.com/accounts/o8/id?id=AItOawnq-RfkVpFN15SWvQ2lpSGAi0XpNQuLxKM"
+ nickname="Yuval"
+ subject="Automation"
+ date="2014-04-27T11:34:44Z"
+ content="""
+Could this step be automated?
+"""]]
diff --git a/doc/tips/what_to_do_when_a_repository_is_corrupted/comment_2_025178c2b11affe2d42a87544b897dc8._comment b/doc/tips/what_to_do_when_a_repository_is_corrupted/comment_2_025178c2b11affe2d42a87544b897dc8._comment
new file mode 100644
index 0000000000..beb177d001
--- /dev/null
+++ b/doc/tips/what_to_do_when_a_repository_is_corrupted/comment_2_025178c2b11affe2d42a87544b897dc8._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="https://www.google.com/accounts/o8/id?id=AItOawmZilYULa6CDEGfuagoDlesyakBgnf-dF8"
+ nickname="Maarten"
+ subject="comment 2"
+ date="2014-05-23T23:50:58Z"
+ content="""
+What if the drive is destroyed?  How would I re-initialize a new drive to act as a substitute for the remote that was just lost?
+"""]]
diff --git a/doc/tips/what_to_do_when_a_repository_is_corrupted/comment_3_9a6bae9c0326ecc7610f5415db20f49e._comment b/doc/tips/what_to_do_when_a_repository_is_corrupted/comment_3_9a6bae9c0326ecc7610f5415db20f49e._comment
new file mode 100644
index 0000000000..870d7b93fe
--- /dev/null
+++ b/doc/tips/what_to_do_when_a_repository_is_corrupted/comment_3_9a6bae9c0326ecc7610f5415db20f49e._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="http://joeyh.name/"
+ ip="209.250.56.36"
+ subject="comment 3"
+ date="2014-05-24T18:29:03Z"
+ content="""
+@Maarten git clone from one of the other clones of the repository to the new drive. Also, recent versions of git-annex have a `git annex reinit` command that can be used to easily initialize a repository with the same settings as a repository that was lost. 
+"""]]
diff --git a/doc/tips/what_to_do_when_you_lose_a_repository.mdwn b/doc/tips/what_to_do_when_you_lose_a_repository.mdwn
new file mode 100644
index 0000000000..363eeea4e0
--- /dev/null
+++ b/doc/tips/what_to_do_when_you_lose_a_repository.mdwn
@@ -0,0 +1,19 @@
+So you lost a thumb drive containing a git-annex repository. Or a hard
+drive died or some other misfortune has befallen your data.
+
+Unless you configured backups, git-annex can't get your data back. But it
+can help you deal with the loss.
+
+Go somewhere that knows about the lost repository, and mark it as
+dead:
+
+	git annex dead usbdrive
+
+This retains the [[location_tracking]] information for the repository,
+but avoids trying to access it, or list it as a location where files
+are present.
+
+If you later found the drive, you could let git-annex know it's found
+like so:
+
+	git annex semitrust usbdrive
diff --git a/doc/tips/what_to_do_when_you_lose_a_repository/comment_1_cf19b8dc304dc37c26717174c4a98aa4._comment b/doc/tips/what_to_do_when_you_lose_a_repository/comment_1_cf19b8dc304dc37c26717174c4a98aa4._comment
new file mode 100644
index 0000000000..a7fce26ef8
--- /dev/null
+++ b/doc/tips/what_to_do_when_you_lose_a_repository/comment_1_cf19b8dc304dc37c26717174c4a98aa4._comment
@@ -0,0 +1,11 @@
+[[!comment format=mdwn
+ username="http://dlaxalde.myopenid.com/"
+ nickname="dl"
+ subject="comment 1"
+ date="2012-05-31T14:36:33Z"
+ content="""
+Is there a way to have git-annex completely ignore a repository? I see that
+the `dead` command adds the uuid of the repository to `trust.log` but does
+not change `uuid.log`. Is it enough to remove the corresponding line in
+`uuid.log` and `trust.log`?
+"""]]
diff --git a/doc/tips/what_to_do_when_you_lose_a_repository/comment_3_fa9ca81668f5faebf2f61b10f82c97d2._comment b/doc/tips/what_to_do_when_you_lose_a_repository/comment_3_fa9ca81668f5faebf2f61b10f82c97d2._comment
new file mode 100644
index 0000000000..a8d044c287
--- /dev/null
+++ b/doc/tips/what_to_do_when_you_lose_a_repository/comment_3_fa9ca81668f5faebf2f61b10f82c97d2._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="http://joeyh.name/"
+ ip="4.153.8.243"
+ subject="comment 3"
+ date="2012-05-31T17:01:37Z"
+ content="""
+`dead` is the best we can do. The automatic merging used on the git-annex branch tends to re-add lines that are deleted in one repo when merging with another that still has them.
+"""]]
diff --git a/doc/tips/what_to_do_when_you_lose_a_repository/comment_3_fdcfca8707e310ca7bb94d359adf8607._comment b/doc/tips/what_to_do_when_you_lose_a_repository/comment_3_fdcfca8707e310ca7bb94d359adf8607._comment
new file mode 100644
index 0000000000..37058399c2
--- /dev/null
+++ b/doc/tips/what_to_do_when_you_lose_a_repository/comment_3_fdcfca8707e310ca7bb94d359adf8607._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="https://openid.stackexchange.com/user/e65e6d0e-58ba-41de-84cc-1f2ba54cf574"
+ nickname="Mica Semrick"
+ subject="Update?"
+ date="2013-12-16T00:31:27Z"
+ content="""
+Is it reasonable to update this page to include `git-annex forget`?
+"""]]
diff --git a/doc/tips/what_to_do_when_you_lose_a_repository/comment_4_679eb9be0bfb9d48a2b96383c4816f62._comment b/doc/tips/what_to_do_when_you_lose_a_repository/comment_4_679eb9be0bfb9d48a2b96383c4816f62._comment
new file mode 100644
index 0000000000..e0aa916a65
--- /dev/null
+++ b/doc/tips/what_to_do_when_you_lose_a_repository/comment_4_679eb9be0bfb9d48a2b96383c4816f62._comment
@@ -0,0 +1,10 @@
+[[!comment format=mdwn
+ username="http://joeyh.name/"
+ ip="209.250.56.87"
+ subject="comment 4"
+ date="2013-12-16T17:04:44Z"
+ content="""
+I wouldn't recommend running `git annex forget --drop-dead` right after losing a drive. You might find the drive again, or get the dead drive to work again. Even if you know you'll never find it, you may want to query git-annex to find files that were only located on that drive, and deal with them, etc.
+
+But if you're worried about that information in the git-annex branch cluttering up the git repository, you keep in mind that `git annex forget` will let you remove it at some point in the future.
+"""]]
diff --git a/doc/tips/what_to_do_when_you_lose_a_repository/comment_5_4fb04b70d88ec93ff9ed4f884747d5d4._comment b/doc/tips/what_to_do_when_you_lose_a_repository/comment_5_4fb04b70d88ec93ff9ed4f884747d5d4._comment
new file mode 100644
index 0000000000..49323b29e5
--- /dev/null
+++ b/doc/tips/what_to_do_when_you_lose_a_repository/comment_5_4fb04b70d88ec93ff9ed4f884747d5d4._comment
@@ -0,0 +1,12 @@
+[[!comment format=mdwn
+ username="https://openid.stackexchange.com/user/e65e6d0e-58ba-41de-84cc-1f2ba54cf574"
+ nickname="Mica Semrick"
+ subject="forget command"
+ date="2013-12-17T05:52:52Z"
+ content="""
+Unfortunately, I plugged in the drive and it was alive just long enough to clone the repo and pull down about half of the data. Then it just started clicking. I did my best to wipe the drive before I returned it, but it didn't even make it that far.
+
+Now I've got a replacement drive. Since my system auto-mounts based on the partition label; I had been using the label to identify external drives. So I want to describe the drive the same way to my system & git-annex. 
+
+In this case, I think I'd like to completely get rid of any notion the repos have of the old drive. 
+"""]]
diff --git a/doc/tips/what_to_do_when_you_lose_a_repository/comment_6_05db504cbff2ec2a6346bf43e57a3c25._comment b/doc/tips/what_to_do_when_you_lose_a_repository/comment_6_05db504cbff2ec2a6346bf43e57a3c25._comment
new file mode 100644
index 0000000000..5a124cc1b0
--- /dev/null
+++ b/doc/tips/what_to_do_when_you_lose_a_repository/comment_6_05db504cbff2ec2a6346bf43e57a3c25._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="http://joeyh.name/"
+ ip="209.250.56.87"
+ subject="comment 6"
+ date="2013-12-17T16:11:52Z"
+ content="""
+You could use `git annex forget --drop-dead` in that situation, but what I would do is first `git annex describe lostremote \"my old drive that I lost\"` and then set up the replacement drive with whatever description you had before.
+"""]]
diff --git a/doc/tips/what_to_do_when_you_lose_a_repository/comment_7_4e6baa41bfee6edf2b17d4ade2909c7b._comment b/doc/tips/what_to_do_when_you_lose_a_repository/comment_7_4e6baa41bfee6edf2b17d4ade2909c7b._comment
new file mode 100644
index 0000000000..3b4b79da36
--- /dev/null
+++ b/doc/tips/what_to_do_when_you_lose_a_repository/comment_7_4e6baa41bfee6edf2b17d4ade2909c7b._comment
@@ -0,0 +1,16 @@
+[[!comment format=mdwn
+ username="http://alan.petitepomme.net/"
+ nickname="Alan Schmitt"
+ subject="still need to remove the remote after a "dead""
+ date="2014-03-07T18:03:47Z"
+ content="""
+Hello,
+
+I decommissioned a machine and installed git annex on another machine, with the same IP as the initial one. To avoid errors, I declared the first repository as dead and created a new one (with a new remote). However, I also had to remove the git remote to the dead machine as \"git annex sync\" kept telling me synchronizing to this dead machine failed (the ssh succeeded, using the name automatically generated by git annex, but the repository was not at the same path on the new machine, so it could not find it).
+
+Should a dead repository not tried to be synchronized at all during git annex sync?
+
+Thanks,
+
+Alan
+"""]]
diff --git a/doc/tips/what_to_do_when_you_lose_a_repository/comment_8_596bcfa25a4fa6e81bb6a0940c55a2f2._comment b/doc/tips/what_to_do_when_you_lose_a_repository/comment_8_596bcfa25a4fa6e81bb6a0940c55a2f2._comment
new file mode 100644
index 0000000000..addeccbf5b
--- /dev/null
+++ b/doc/tips/what_to_do_when_you_lose_a_repository/comment_8_596bcfa25a4fa6e81bb6a0940c55a2f2._comment
@@ -0,0 +1,7 @@
+[[!comment format=mdwn
+ username="fiatjaf"
+ subject="comment 8"
+ date="2016-04-23T15:01:23Z"
+ content="""
+I've issued `git annex dead` for a full git-annex repo (not a special remote) I had in a VPS, but my local machine keeps trying to sync with it on `git annex sync`, asking me to confirm the ssh public key and so on. Is that correct?
+"""]]
diff --git a/doc/tips/what_to_do_when_you_lose_a_repository/comment_9_525e58e5941220b23b0ed73fa7611db1._comment b/doc/tips/what_to_do_when_you_lose_a_repository/comment_9_525e58e5941220b23b0ed73fa7611db1._comment
new file mode 100644
index 0000000000..a917935689
--- /dev/null
+++ b/doc/tips/what_to_do_when_you_lose_a_repository/comment_9_525e58e5941220b23b0ed73fa7611db1._comment
@@ -0,0 +1,14 @@
+[[!comment format=mdwn
+ username="joey"
+ subject="""comment 9"""
+ date="2016-05-03T19:15:58Z"
+ content="""
+Both `git annex sync` and `git pull` will still try to access the git
+remote for the repository. Thing to do to prevent this is `git remote
+remove` it.
+
+I think it makes sense for `git annex sync` to still try to push/pull
+from a dead remote, since git pull/push will still try to access it
+too. Marking the remote dead does prevent other git-annex commands
+from trying to use it for data storage.
+"""]]
diff --git a/doc/tips/yet_another_simple_disk_usage_like_utility.mdwn b/doc/tips/yet_another_simple_disk_usage_like_utility.mdwn
new file mode 100644
index 0000000000..961776e193
--- /dev/null
+++ b/doc/tips/yet_another_simple_disk_usage_like_utility.mdwn
@@ -0,0 +1,9 @@
+Here's the annex-du script that I use:
+
+#!/bin/sh
+git annex find "$@" --include '*' --format='${bytesize}\n' |awk '{ sum += $1; nfiles++; } END { printf "%d files, %.3f MB\n", nfiles, sum/1000000 } '
+
+This one can be slow on a large number of files, but it has an advantage of being able to use all of the filtering available in git annex find.
+For example, to figure out how much is stored in remote X, do
+
+annex-du --in=X
diff --git a/doc/tips/yet_another_simple_disk_usage_like_utility/comment_1_41b212bde8bc88d2a5dea93bd0dc75f1._comment b/doc/tips/yet_another_simple_disk_usage_like_utility/comment_1_41b212bde8bc88d2a5dea93bd0dc75f1._comment
new file mode 100644
index 0000000000..4c3e3c22b3
--- /dev/null
+++ b/doc/tips/yet_another_simple_disk_usage_like_utility/comment_1_41b212bde8bc88d2a5dea93bd0dc75f1._comment
@@ -0,0 +1,9 @@
+[[!comment format=mdwn
+ username="https://www.google.com/accounts/o8/id?id=AItOawln3ckqKx0x_xDZMYwa9Q1bn4I06oWjkog"
+ nickname="Michael"
+ subject="comment 1"
+ date="2013-07-12T19:36:28Z"
+ content="""
+Ah, I just found that git annex info can do the same :)
+Disregard this.
+"""]]
diff --git a/doc/tips/yet_another_simple_disk_usage_like_utility/comment_2_73698913837bfd5a58cf15721211e43e._comment b/doc/tips/yet_another_simple_disk_usage_like_utility/comment_2_73698913837bfd5a58cf15721211e43e._comment
new file mode 100644
index 0000000000..fe4b3d0d21
--- /dev/null
+++ b/doc/tips/yet_another_simple_disk_usage_like_utility/comment_2_73698913837bfd5a58cf15721211e43e._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="https://me.yahoo.com/a/2grhJvAC049fJnvALDXek.6MRZMTlg--#eec89"
+ nickname="John"
+ subject="comment 2"
+ date="2013-08-30T06:09:29Z"
+ content="""
+You may want to try my `sizes` tool on Hackage.  Just pass `-A` and it will be aware of the annex and report sizes as if no files were annexed.  The only downside is that it reports file usage for replicated content multiple times, as if you'd copied the data out of the annex rather than hardlinked all duplicate copies (although, this may be exactly the behavior some people want).
+"""]]
diff --git a/doc/todo.mdwn b/doc/todo.mdwn
new file mode 100644
index 0000000000..e2661da384
--- /dev/null
+++ b/doc/todo.mdwn
@@ -0,0 +1,5 @@
+This is git-annex's todo list. Link items to [[todo/done]] when done. A more complete [[design/roadmap/]] is also available.
+
+[[!inline pages="./todo/* and !./todo/done and !link(done) 
+and !*/Discussion" actions=yes postform=yes postformtext="Add a new todo titled:"
+show=0 feedlimit=10 archive=yes]]
diff --git a/doc/todo/--batch_for_add.mdwn b/doc/todo/--batch_for_add.mdwn
new file mode 100644
index 0000000000..c0450c11f8
--- /dev/null
+++ b/doc/todo/--batch_for_add.mdwn
@@ -0,0 +1,7 @@
+should be extremely helpful when adding many files one at a time ;)
+
+[[!meta author=yoh]]
+
+> Implemented; made it not recurse into directories and output a blank line
+> if it doesn't add the file, so there's aways 1 line of output for each
+> input. [[done]] --[[Joey]]
diff --git a/doc/todo/--batch_for_add/comment_1_b1362c86e162a49717d7e3b0816025ba._comment b/doc/todo/--batch_for_add/comment_1_b1362c86e162a49717d7e3b0816025ba._comment
new file mode 100644
index 0000000000..2e70d8ab25
--- /dev/null
+++ b/doc/todo/--batch_for_add/comment_1_b1362c86e162a49717d7e3b0816025ba._comment
@@ -0,0 +1,12 @@
+[[!comment format=mdwn
+ username="joey"
+ subject="""comment 1"""
+ date="2016-01-19T15:32:48Z"
+ content="""
+Has similar problems to batching find; if a whole directory is added any
+number of lines can be output.
+
+So, either needs a similar thing to indicate end of output for current
+item, or perhaps it would make sense for the batched add to not recurse
+into directories, so it would always output exactly 1 line per input.
+"""]]
diff --git a/doc/todo/--batch_for_copy.mdwn b/doc/todo/--batch_for_copy.mdwn
new file mode 100644
index 0000000000..0db9656771
--- /dev/null
+++ b/doc/todo/--batch_for_copy.mdwn
@@ -0,0 +1,11 @@
+apparently not in 6.20170525+gitge1cf095ae-1~ndall+1  yet
+
+[[!format sh """
+$> git annex copy --help | grep -q batch || echo nothing
+nothing
+
+"""]]
+
+[!meta author=yoh]
+
+> [[done]] for copy and move --[[Joey]]
diff --git a/doc/todo/--batch_for_find.mdwn b/doc/todo/--batch_for_find.mdwn
new file mode 100644
index 0000000000..825ca560fa
--- /dev/null
+++ b/doc/todo/--batch_for_find.mdwn
@@ -0,0 +1,5 @@
+I am using `annex find filename` after running 'annex add` to figure out if file was added to annex or to git.
+
+[[!meta author=yoh]]
+
+> [[done]] --[[Joey]]
diff --git a/doc/todo/--batch_for_find/comment_1_d164ad661617d29cb1ceb3d6fc3bb37f._comment b/doc/todo/--batch_for_find/comment_1_d164ad661617d29cb1ceb3d6fc3bb37f._comment
new file mode 100644
index 0000000000..d9a04c220e
--- /dev/null
+++ b/doc/todo/--batch_for_find/comment_1_d164ad661617d29cb1ceb3d6fc3bb37f._comment
@@ -0,0 +1,12 @@
+[[!comment format=mdwn
+ username="joey"
+ subject="""comment 1"""
+ date="2016-01-19T15:30:01Z"
+ content="""
+Couldn't you use the existing info --batch for the same purpose?
+
+batching find has the problem that it can output any number of lines
+(0 for a non-annexed file, 1 for an annexed file, N for a directory). So,
+the consumer has no way to tell when it reaches the end of output. So some
+additional thing would be needed to indicate end of output.
+"""]]
diff --git a/doc/todo/--batch_for_find/comment_2_9873396186f4d24088fd746f42373678._comment b/doc/todo/--batch_for_find/comment_2_9873396186f4d24088fd746f42373678._comment
new file mode 100644
index 0000000000..416e588e1b
--- /dev/null
+++ b/doc/todo/--batch_for_find/comment_2_9873396186f4d24088fd746f42373678._comment
@@ -0,0 +1,27 @@
+[[!comment format=mdwn
+ username="https://me.yahoo.com/a/EbvxpTI_xP9Aod7Mg4cwGhgjrCrdM5s-#7c0f4"
+ subject="comment 2"
+ date="2016-01-19T16:39:49Z"
+ content="""
+info -- nope, since it returns the same success and size = 0 bytes for a 0-length file which is under annex regardless either its precious load is present locally or not
+
+[[!format sh \"\"\"
+
+$> git annex find empty
+$> git annex info --json empty
+{\"command\":\"info empty\",\"file\":\"empty\",\"size\":\"0 bytes\",\"key\":\"SHA256E-s0--e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855\",\"success\":true}
+
+$> git annex get empty
+get empty (from web...) 
+/tmp/123/.git/annex/tmp/SHA256E-s0--e3b0c44298fc1c149af     [ <=>                                                                                                                            ]       0  --.-KB/s    in 0s      
+(checksum...) ok
+(recording state in git...)
+
+$> git annex info --json empty
+{\"command\":\"info empty\",\"file\":\"empty\",\"size\":\"0 bytes\",\"key\":\"SHA256E-s0--e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855\",\"success\":true}
+
+
+\"\"\"]]
+
+I could use whereis (--batch is TODO for it as well) and analyze output either any remote claims it to be \"here\" though.
+"""]]
diff --git a/doc/todo/--batch_for_find/comment_3_598a0500a3a4573c50a559d1931f1918._comment b/doc/todo/--batch_for_find/comment_3_598a0500a3a4573c50a559d1931f1918._comment
new file mode 100644
index 0000000000..e2e75dd881
--- /dev/null
+++ b/doc/todo/--batch_for_find/comment_3_598a0500a3a4573c50a559d1931f1918._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="joey"
+ subject="""comment 3"""
+ date="2016-01-19T17:13:05Z"
+ content="""
+You were talking about determining if a file is in git-annex or git, and
+git-annex info can do that.
+"""]]
diff --git a/doc/todo/--batch_for_find/comment_4_830cf616968e25cee8e7d35f9709f79b._comment b/doc/todo/--batch_for_find/comment_4_830cf616968e25cee8e7d35f9709f79b._comment
new file mode 100644
index 0000000000..35f2873c03
--- /dev/null
+++ b/doc/todo/--batch_for_find/comment_4_830cf616968e25cee8e7d35f9709f79b._comment
@@ -0,0 +1,7 @@
+[[!comment format=mdwn
+ username="https://me.yahoo.com/a/EbvxpTI_xP9Aod7Mg4cwGhgjrCrdM5s-#7c0f4"
+ subject="comment 4"
+ date="2016-01-19T19:24:11Z"
+ content="""
+you caught me! ;) indeed in this case we could use 'info' for that purpose.  But if we decide if content is present under annex (which is what actually our function does, we just used it for determining \"if under annex\" after addurl) -- then find --batch would be needed, or whois --batch could be used.  So we are both correct! ;)
+"""]]
diff --git a/doc/todo/--batch_for_info.mdwn b/doc/todo/--batch_for_info.mdwn
new file mode 100644
index 0000000000..8f7fba456c
--- /dev/null
+++ b/doc/todo/--batch_for_info.mdwn
@@ -0,0 +1,5 @@
+I guess as other commands which take separate files/keys as its argument(s), having --batch for info command would be of benefit
+
+[[!meta author=yoh]]
+
+> [[done]] --[[Joey]]
diff --git a/doc/todo/--get_option_for_diffdriver.mdwn b/doc/todo/--get_option_for_diffdriver.mdwn
new file mode 100644
index 0000000000..d6ad9a2b98
--- /dev/null
+++ b/doc/todo/--get_option_for_diffdriver.mdwn
@@ -0,0 +1 @@
+since there is no generic 'fuse' mode, I would like to request to have `--get` (or `--auto-get`) option for diffdriver.  I am trying to compare files across two branches on a repo I just cloned.  I cannot download all the files  and downloading differing keys across branches for the same file is a bit painful.  So I felt that it would be super nice if git annex could auto get those files from somewhere (well -- original clone)
diff --git a/doc/todo/--get_option_for_diffdriver/comment_1_6b2501b2dc0318242e444e8d4b8b314f._comment b/doc/todo/--get_option_for_diffdriver/comment_1_6b2501b2dc0318242e444e8d4b8b314f._comment
new file mode 100644
index 0000000000..afaa14e986
--- /dev/null
+++ b/doc/todo/--get_option_for_diffdriver/comment_1_6b2501b2dc0318242e444e8d4b8b314f._comment
@@ -0,0 +1,17 @@
+[[!comment format=mdwn
+ username="joey"
+ subject="""comment 1"""
+ date="2018-02-08T17:28:21Z"
+ content="""
+This is a good idea. I wonder what to do about the objects it downloads for
+a diff; should they be left in the annex for later use/dropunused, or
+immediately deleted after the diff completes. 
+
+My inclination is to keep
+them around, for one thing when I'm diffing stuff I often run diff more
+than once, perhaps to widen the diff or because I want to take a second
+look at it, and re-downloading a bunch of big files would be painful then.
+
+By the way, what diffdriver program are you using? I've had a hard time
+finding any real-life examples of git diffdrivers.
+"""]]
diff --git a/doc/todo/Add_a_way_to_mark_exporttree_remotes_dead.mdwn b/doc/todo/Add_a_way_to_mark_exporttree_remotes_dead.mdwn
new file mode 100644
index 0000000000..c534f59ba7
--- /dev/null
+++ b/doc/todo/Add_a_way_to_mark_exporttree_remotes_dead.mdwn
@@ -0,0 +1,10 @@
+There is currently no way to get rid of an exporttree remote, because the trust level `untrusted` is enforced.
+
+
+    $ git annex dead exp
+    dead exp
+      This remote's trust level is overridden to untrusted.
+    ok
+    (recording state in git...)
+
+> Fun bug! I've fixed it so that will work. [[done]] --[[Joey]]
diff --git a/doc/todo/Add_confirmation_dialog_to_the_restart_option.mdwn b/doc/todo/Add_confirmation_dialog_to_the_restart_option.mdwn
new file mode 100644
index 0000000000..1ff909b8cf
--- /dev/null
+++ b/doc/todo/Add_confirmation_dialog_to_the_restart_option.mdwn
@@ -0,0 +1,4 @@
+I have four git-annex repositories and I often use the option "Switch repository" in the webapp, but this option is just above the "Restart daemon" option and I have clicked it quite a few time unintentionally.
+Could it be possible to add a confirmation dialog just like in the "Shutdown daemon" option?
+
+> spacer added; [[done]]  --[[Joey]]
diff --git a/doc/todo/Add_confirmation_dialog_to_the_restart_option/comment_1_00da59dc2d9b86a51eb462c481ab665d._comment b/doc/todo/Add_confirmation_dialog_to_the_restart_option/comment_1_00da59dc2d9b86a51eb462c481ab665d._comment
new file mode 100644
index 0000000000..0732b7dde0
--- /dev/null
+++ b/doc/todo/Add_confirmation_dialog_to_the_restart_option/comment_1_00da59dc2d9b86a51eb462c481ab665d._comment
@@ -0,0 +1,13 @@
+[[!comment format=mdwn
+ username="joey"
+ subject="""comment 1"""
+ date="2016-02-15T16:54:54Z"
+ content="""
+Restarting the daemon should just take a second and the web browser should
+redirect to the new instance. I guess it does move away from the current
+page to the dashboard, and in a way that the back button can't undo, but I
+see no other harm in accidentally restarting the daemon.
+
+So, I think I'll just add a spacer to separate the repository part of that
+menu from the daemon part.
+"""]]
diff --git a/doc/todo/Allow_for_TRANSFER-SUCCESS_to_report_also_a_URL_where_key_could_now_be_obtained_from.mdwn b/doc/todo/Allow_for_TRANSFER-SUCCESS_to_report_also_a_URL_where_key_could_now_be_obtained_from.mdwn
new file mode 100644
index 0000000000..3c53d114c6
--- /dev/null
+++ b/doc/todo/Allow_for_TRANSFER-SUCCESS_to_report_also_a_URL_where_key_could_now_be_obtained_from.mdwn
@@ -0,0 +1,13 @@
+ATM, special remotes store content (keys) in a keystore which has no reflection of original files hierarchy.
+
+In many cases though it is plausible and reasonable to replicate original files hierarchy, e.g. when uploading path/file1.ext, have it stored on a remote also under such name with additional part of the path to allow for multiple versions, e.g. path/file1-Key[:5].ext (collision is possible but unlikely) or path/file1.ext/Key . This way we could replicate initial files hierarchy on a special remote, making it useful/usable without annex.  I guess (didn't check yet) that File in TRANSFER STORE request points to original file location (not resolved key location), so we can have information about original sample location within special remote.  Since it would be virtually impossible (or expensive to locate) to retrieve content solely by a Key, we could use URLs mechanism to associate given uploaded Key with a new custom URL (e.g. custom-schema:path-file1.ext/Key) so later this special remote could provide the content by claiming that url.  Sure thing custom remote could just use 'addurl' call independently within a call to "TRANSFER" to it, but I wondered if may be protocol could be adjusted to support
+
+TRANSFER-SUCCESS STORE Key URL
+
+response when upon STORE success special remote provides a url under which content should be registered available from.
+
+[[!meta author=yoh]]
+
+> I think this was trying to implement something like what `git annex
+> export`, and since that's implemented now, we shouldn't need to worry
+> about this. [[done]] --[[Joey]]
diff --git a/doc/todo/Allow_for_TRANSFER-SUCCESS_to_report_also_a_URL_where_key_could_now_be_obtained_from/comment_1_9f4245bd016f9283464a25698563786c._comment b/doc/todo/Allow_for_TRANSFER-SUCCESS_to_report_also_a_URL_where_key_could_now_be_obtained_from/comment_1_9f4245bd016f9283464a25698563786c._comment
new file mode 100644
index 0000000000..1976eda523
--- /dev/null
+++ b/doc/todo/Allow_for_TRANSFER-SUCCESS_to_report_also_a_URL_where_key_could_now_be_obtained_from/comment_1_9f4245bd016f9283464a25698563786c._comment
@@ -0,0 +1,18 @@
+[[!comment format=mdwn
+ username="joey"
+ subject="""comment 1"""
+ date="2016-10-26T17:29:46Z"
+ content="""
+Well no, the filename passed to "TRANSFER STORE" is wherever the content
+of the file is, in most circumstances it will not be a file in the working
+tree.
+
+(And even if the filename is a worktree file in some case, the special
+remote needs to support storing multiple versions of a file. So trying to
+use the name used in the working tree on the special remote seems very
+problimatic.)
+
+In any case, the external special remote protocol already has
+`SETURLPRESENT` which can be used if a TRANSFER STORE makes a key be
+available at an url.
+"""]]
diff --git a/doc/todo/Allow_for_TRANSFER-SUCCESS_to_report_also_a_URL_where_key_could_now_be_obtained_from/comment_2_34d1eab194b7913dd453131d9187287a._comment b/doc/todo/Allow_for_TRANSFER-SUCCESS_to_report_also_a_URL_where_key_could_now_be_obtained_from/comment_2_34d1eab194b7913dd453131d9187287a._comment
new file mode 100644
index 0000000000..c99ddd8ca0
--- /dev/null
+++ b/doc/todo/Allow_for_TRANSFER-SUCCESS_to_report_also_a_URL_where_key_could_now_be_obtained_from/comment_2_34d1eab194b7913dd453131d9187287a._comment
@@ -0,0 +1,24 @@
+[[!comment format=mdwn
+ username="https://me.yahoo.com/a/EbvxpTI_xP9Aod7Mg4cwGhgjrCrdM5s-#7c0f4"
+ avatar="http://cdn.libravatar.org/avatar/8122123b4c2bc77187e32d7e025f7d445d7a08de1ba532237876a31159ac01da"
+ subject="how to get ahold of the filename?"
+ date="2016-10-27T10:10:24Z"
+ content="""
+SETURLPRESENT -- sounds like could be used indeed, but
+
+- *Note that this does not make git-annex think that the url is present on the web special remote.* -- so does it mean that this git/annex repo will not be able to download from that URL (which could be just a simple public http:// url) without having this special remote available?
+- *The readonly=true parameter makes git-annex download content from the urls recorded earlier by SETURLPRESENT.*  -- so otherwise URL would not be used?  a bit confused
+
+**\"remote needs to support storing multiple versions of a file\"**
+
+SURE (e.g. as I have described before)
+
+**\"in most circumstances it will not be a file in the working tree\"**
+
+could then there be a reasonable (scalable, via git annex interface) way to discover the (original) path(s) within repository which was given to \"git add PATH\"?
+
+Once again -- the idea is to make some special remotes useful on their own without relying on having an annex and original git/annex repository to associate those with specific files.
+
+Related:
+I see that there is now also SETURIPRESENT.  Is there difference how annex handles URIs in comparison to URLs? in an example which I saw with ipfs: URI.  it feels that those are still URLs as prescribing \"how\" content should be retrieved (via ipfs). We have used similarly addurl command to register our urls for content from archives (e.g. dl+archive:MD5E-s2416581890--662e0713d0ce42bcdbadb8251b893b8a.tgz#path=ds001/sub-01/anat/sub-01_T1w.nii.gz)
+"""]]
diff --git a/doc/todo/Allow_for_TRANSFER-SUCCESS_to_report_also_a_URL_where_key_could_now_be_obtained_from/comment_3_f667aa26dadadede318d31176292102d._comment b/doc/todo/Allow_for_TRANSFER-SUCCESS_to_report_also_a_URL_where_key_could_now_be_obtained_from/comment_3_f667aa26dadadede318d31176292102d._comment
new file mode 100644
index 0000000000..ba7c8ded88
--- /dev/null
+++ b/doc/todo/Allow_for_TRANSFER-SUCCESS_to_report_also_a_URL_where_key_could_now_be_obtained_from/comment_3_f667aa26dadadede318d31176292102d._comment
@@ -0,0 +1,31 @@
+[[!comment format=mdwn
+ username="joey"
+ subject="""SETURLPRESENT"""
+ date="2016-10-31T18:51:07Z"
+ content="""
+git-annex does not keep track of which urls belong to which remote.
+Urls are, after all, Universal; it shoudn't matter which remote
+set an url.
+
+So, if `SETURLPRESENT` was used, and if git-annex thinks that the web
+special remote is recorded as having the content, it will try to download
+from that url, as well as any other urls that might be set.
+
+But, `SETURLPRESENT` does not make it think that the web special remote
+has the content. So, if the special remote that git-annex does think has to
+content is not enabled, `git annex get` won't try the web special remote.
+
+So, what you can do is run `git annex setpresentkey $key 00000000-0000-0000-0000-000000000001`
+to make it think  the web special remote has the url after `SETURLPRESENT`.
+Then it'll be the same as if `addurl` had been used; it will download from
+the web.
+
+(There's also a way to enable a external special remote in readonly mode.
+In this mode, the special remote program does not have to be in PATH, and
+when git-annex wants to get content from the remote it will download
+content from any urls.)
+
+(The difference with `SETURIPRESENT` is that it's assumed the URI cannot
+be downloaded via HTTP/FTP. So, while `git annex whereis` displays
+URIs, git-annex won't try to download them itself.)
+"""]]
diff --git a/doc/todo/Allow_for_TRANSFER-SUCCESS_to_report_also_a_URL_where_key_could_now_be_obtained_from/comment_4_16d30e86132d6a3cefaa8e2fecbda7ca._comment b/doc/todo/Allow_for_TRANSFER-SUCCESS_to_report_also_a_URL_where_key_could_now_be_obtained_from/comment_4_16d30e86132d6a3cefaa8e2fecbda7ca._comment
new file mode 100644
index 0000000000..ccef300fc4
--- /dev/null
+++ b/doc/todo/Allow_for_TRANSFER-SUCCESS_to_report_also_a_URL_where_key_could_now_be_obtained_from/comment_4_16d30e86132d6a3cefaa8e2fecbda7ca._comment
@@ -0,0 +1,21 @@
+[[!comment format=mdwn
+ username="joey"
+ subject="""associated files"""
+ date="2016-10-31T19:12:56Z"
+ content="""
+git-annex keeps track of the AssociatedFile, which (when available) is 
+the worktree file corresponding to the Key that's being operated on.
+
+This information is not exposed in the external special remote interface.
+I'm worried that, if it were, people would try to do stuff that
+just can't work, like
+
+
+Worktree files can be renamed or deleted or copied at any time and can have
+multiple versions, and any special remote that used this information to try
+to create something that resembles the worktree would have massive
+problems.
+
+I am having a hard time thinking of any use that an external special remote
+could make of the information that would not be a mistake.
+"""]]
diff --git a/doc/todo/Allow_globally_limiting_filename_length.mdwn b/doc/todo/Allow_globally_limiting_filename_length.mdwn
new file mode 100644
index 0000000000..e187f69a8c
--- /dev/null
+++ b/doc/todo/Allow_globally_limiting_filename_length.mdwn
@@ -0,0 +1,9 @@
+I have some Git Annex repos that I keep copies of on both NTFS on Linux and on ecryptfs (which Ubuntu uses for home directory encryption) on Linux. Now, ecryptfs allows each path component of a filename to be only up to 140-ish characters, because it has to encrypt that filename, add some encryption info to it, and store it inside another filename on a backing ext4 filesystem (which limits path components to 255 characters).
+
+Several times now I've added a bunch of stuff to my annex on the NTFS checkout, where path components are allowed to be longer than 140 characters, synced it over to my other annex checkout on ecryptfs, and then had Git Annex fail during the sync, trying to create these empty symlinks with path components too long for the filesystem it is on. When in this state, I don't really know how to fix it. I can't just "git mv" the offending file to a valid name, both because "git mv" needs the source file to be on disk in the first place and because the failed "git checkout" leaves my repo thinking it has thousands of untracked files (because some stuff did get created, but git refused to officially move to the commit it was trying to check out, because the checkout failed).
+
+I am looking for a solution for this inside Git Annex. The simplest thing, I think, would be to set a max path component length for the whole set of repos, so I could get an error when I go to "git annex add" on the NTFS checkout that the filenames being added are too long for some of the repos that will eventually want to check them out. Is it possible to do this with a pre-commit hook somehow?
+
+The next simplest thing would be for Git Annex to look at the filesystem it is running on and do something smarter than exploding and leaving my repo in a weird out-of-sync state if some of the filenames it wants to create can't be created. Maybe it should fail the sync earlier, in Git Annex itself rather than in git checkout. Maybe it should just leave those files out of the checkout, or force/allow me to rename them right then.
+
+The most complex thing would be to somehow make it work anyway and check out the symlinks under different, valid names. Perhaps it could just truncate those path components in the symlink view? There's already support for different metadata views; this would be sort of like that. You get a special view of the repo subject to the constraints of your filesystem.
diff --git a/doc/todo/Allow_globally_limiting_filename_length/comment_1_e38f78a1263b15258d2aa4472e7bc31b._comment b/doc/todo/Allow_globally_limiting_filename_length/comment_1_e38f78a1263b15258d2aa4472e7bc31b._comment
new file mode 100644
index 0000000000..0d718bf674
--- /dev/null
+++ b/doc/todo/Allow_globally_limiting_filename_length/comment_1_e38f78a1263b15258d2aa4472e7bc31b._comment
@@ -0,0 +1,23 @@
+[[!comment format=mdwn
+ username="joey"
+ subject="""comment 1"""
+ date="2016-09-21T17:11:06Z"
+ content="""
+Standard encfs warning: It's buggy and insecure. Don't use it.
+
+You can find many other problems caused by encfs on this site, and 
+ has described security problems with
+encfs for years.
+
+It would not help for `git-annex add` to check some kind of filename limit,
+because it would not prevent you doing this:
+
+	git annex add smallenough
+	git mv smallenough oh-oops-my-name-is-too-long-for-encfs
+	git commit -m haha
+
+A git pre-commit hook can of course be written that blocks such commits.
+
+I am not inclined to complicate git-annex just to handle encfs given how
+broken encfs is.
+"""]]
diff --git a/doc/todo/Allow_globally_limiting_filename_length/comment_2_cdb91ea855d9bcdeccf19ccb6d55ed10._comment b/doc/todo/Allow_globally_limiting_filename_length/comment_2_cdb91ea855d9bcdeccf19ccb6d55ed10._comment
new file mode 100644
index 0000000000..dc36053d08
--- /dev/null
+++ b/doc/todo/Allow_globally_limiting_filename_length/comment_2_cdb91ea855d9bcdeccf19ccb6d55ed10._comment
@@ -0,0 +1,14 @@
+[[!comment format=mdwn
+ username="interfect@b151490178830f44348aa57b77ad58c7d18e8fe7"
+ nickname="interfect"
+ subject="Pre Commit Hook"
+ date="2016-09-21T19:20:04Z"
+ content="""
+I'm basically stuck with whatever home directory encryption Canonical deigns to give me in their setup wizard, given my time and attention budget. I've looked a bit at the security problems with it and they mostly seem to be that it's a bit leaky due to not hiding structures and sizes. Hiding contents is better than not hiding contents, so that's what I've got.
+
+Anyway, a pre-commit hook, or maybe an update hook, would be a great solution. I'd like one to be on the wiki somewhere as a useful tip for actually using git annex effectively across a bunch of non-ideal environments. It would be great if a \"git annex init\" could set it up for me, too.
+
+Any ideas for writing a pre-commit script that works on Linux, Mac, Windows, Android, and whatever weird embedded NAS things people might want to use it on? If I went with an update script over a pre-commit, that would make platform support less of a problem, but then you'd get Git Annex into weird situations when syncing.
+
+How would Git Annex react if I made a commit on one system, but then my central syncing repo's update script rejected the commit for breaking the rules on file names? If I have a commit that isn't allowed to be pushed to a particular remote, how would I use git annex to get it out of the history of any repos it might have already gotten to?
+"""]]
diff --git a/doc/todo/Allow_globally_limiting_filename_length/comment_3_748d5b01e502370ed98937d0077d1ec5._comment b/doc/todo/Allow_globally_limiting_filename_length/comment_3_748d5b01e502370ed98937d0077d1ec5._comment
new file mode 100644
index 0000000000..61e98dfc44
--- /dev/null
+++ b/doc/todo/Allow_globally_limiting_filename_length/comment_3_748d5b01e502370ed98937d0077d1ec5._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="interfect@b151490178830f44348aa57b77ad58c7d18e8fe7"
+ nickname="interfect"
+ subject="comment 3"
+ date="2016-09-21T19:32:06Z"
+ content="""
+Also, I think Ubuntu is \"ecryptfs\" and not \"encfs\" anyway.
+"""]]
diff --git a/doc/todo/Allow_globally_limiting_filename_length/comment_4_918d87f7a961bfeb834cc28f7175ccd8._comment b/doc/todo/Allow_globally_limiting_filename_length/comment_4_918d87f7a961bfeb834cc28f7175ccd8._comment
new file mode 100644
index 0000000000..ec18558f60
--- /dev/null
+++ b/doc/todo/Allow_globally_limiting_filename_length/comment_4_918d87f7a961bfeb834cc28f7175ccd8._comment
@@ -0,0 +1,23 @@
+[[!comment format=mdwn
+ username="joey"
+ subject="""comment 4"""
+ date="2016-09-21T20:16:44Z"
+ content="""
+If an update hook rejects a push, then `git annex sync` will just note that
+it was unable to push. It will sync in only one direction until the problem
+that prevents pushing gets resolved.
+
+It might try pushing to a different branch name than usual to get around
+some other problems that cause pushes to fail so be sure to have the update
+hook check pushes to all branches (except for the git-annex branch)).
+
+I don't know why you'd want to filter such a commit out of the git history.
+You could just fix it by renaming the problem file and make a commit on top
+of the problem commit. Just make the update hook only look at the diff
+between the old version of the branch and the new version, so it won't be
+tripped up by intermediate commits that violate its rules.
+
+(I know that Ubuntu uses encfs or something like that by default, but
+surely they have not removed the Debian installer's support for full
+disk encryption?)
+"""]]
diff --git a/doc/todo/Allow_globally_limiting_filename_length/comment_5_2b49a293b044d0c2fcfe1701c76424c4._comment b/doc/todo/Allow_globally_limiting_filename_length/comment_5_2b49a293b044d0c2fcfe1701c76424c4._comment
new file mode 100644
index 0000000000..8c4c6f09ab
--- /dev/null
+++ b/doc/todo/Allow_globally_limiting_filename_length/comment_5_2b49a293b044d0c2fcfe1701c76424c4._comment
@@ -0,0 +1,10 @@
+[[!comment format=mdwn
+ username="interfect@b151490178830f44348aa57b77ad58c7d18e8fe7"
+ nickname="interfect"
+ subject="comment 5"
+ date="2016-09-21T22:49:55Z"
+ content="""
+OK, I'll try something like that.
+
+(Full disk encryption is still there; I think on one system I just have ecryptfs, because I want to be able to get in over ssh sometimes, and on one I have *both* FDE and ecryptfs on, because I enjoy performance penalties.)
+"""]]
diff --git a/doc/todo/Alternative_mode_control_for_import.mdwn b/doc/todo/Alternative_mode_control_for_import.mdwn
new file mode 100644
index 0000000000..24bbd5ace5
--- /dev/null
+++ b/doc/todo/Alternative_mode_control_for_import.mdwn
@@ -0,0 +1,17 @@
+This suggestion has come from being surprised at the behaviour of "import --skip-duplicates" which copies files instead of moving them and leaves the source directory untouched (description implies it will just leave duplicates alone).
+
+Apologies for the brevity, I've already typed this out once..
+
+"import" has several behaviours which can be controlled through some options, but they don't cover all wanted behaviours. This suggestion is for an alternative interface to control these behaviours, totally stolen from rsync :P
+
+    # create symlinks (s), inject content (i) and delete from source (d)
+    # duplicate (D) and new (N) files
+    git annex import --mode=Dsid,Nsid $src # (default behaviour)
+    git annex import --mode=Dsi,Nsi $src   # --duplicate
+    git annex import --mode=Dd,Nsid $src   # --deduplicate
+    git annex import --mode=Nsi $src       # --skip-duplicates
+    git annex import --mode=Dd $src        # --clean-duplicates
+    git annex import --mode=Did,Nsid $src  # (import new, reinject duplicate.. really want this!)
+    git annex import --mode=Ns $src        # (just creates symlinks for new)
+    git annex import --mode=Nsd $src       # (invalid mode due to data loss)
+    git annex import --mode=Nid $src       # (invalid or require --force)
diff --git a/doc/todo/Alternative_mode_control_for_import/comment_1_77dc4b03e243d46c227f43cb3f573ec0._comment b/doc/todo/Alternative_mode_control_for_import/comment_1_77dc4b03e243d46c227f43cb3f573ec0._comment
new file mode 100644
index 0000000000..7b94f92815
--- /dev/null
+++ b/doc/todo/Alternative_mode_control_for_import/comment_1_77dc4b03e243d46c227f43cb3f573ec0._comment
@@ -0,0 +1,11 @@
+[[!comment format=mdwn
+ username="CandyAngel"
+ avatar="http://cdn.libravatar.org/avatar/15c0aade8bec5bf004f939dd73cf9ed8"
+ subject="comment 1"
+ date="2017-01-16T10:30:55Z"
+ content="""
+This [[TODO|todo/import_--reinject/]] (and \"reinject --known\") would then be:
+
+    git annex import --mode=Did
+
+"""]]
diff --git a/doc/todo/Alternative_mode_control_for_import/comment_2_21540a75c78c3bb7745f76b8e92a2ec5._comment b/doc/todo/Alternative_mode_control_for_import/comment_2_21540a75c78c3bb7745f76b8e92a2ec5._comment
new file mode 100644
index 0000000000..278616359e
--- /dev/null
+++ b/doc/todo/Alternative_mode_control_for_import/comment_2_21540a75c78c3bb7745f76b8e92a2ec5._comment
@@ -0,0 +1,33 @@
+[[!comment format=mdwn
+ username="joey"
+ subject="""comment 2"""
+ date="2017-02-07T20:24:29Z"
+ content="""
+Bearing in mind that I would have to *support* all of the resulting
+combinatorial explosion, and that several combinations don't make sense,
+or are unsafe, or seem useless, I think I'd rather keep it limited to
+well-selected points from the space.
+
+I've fixed the description of --skip-duplicates to match its behavior.
+I don't know if there's a good motivation for it not deleting the files it
+does import. I'd almost rather have thought that was a bug in the
+implementation, but the implementation explicitly copies rather than moves
+files for --skip-duplicates, so that does seem to have been done
+intentionally. In any case, `--clean-duplicates` can be run after it to
+delete dups, I suppose.
+
+An implementation of --mode=Did,Nsid seemed worth adding at first, perhaps
+as --reinject-duplicates. But thinking about it some more,
+that would be the same as:
+
+	git annex reinject --known /path/*
+	git annex import /path/*
+
+The first command moves all known files into the annex, which leaves
+only non-duplicate files for the second command to import. 
+
+The only time I can think of that this might not be suitable is if `/path` is
+getting new files added to it while the commands run... But in that case
+you can `mkdir /path/toimport; mv /path/* /path/toimport` and then
+run the 2 commands on `/path/toimport/*`
+"""]]
diff --git a/doc/todo/Alternative_mode_control_for_import/comment_3_cd4b0f6752457a70f2ab908d23913d76._comment b/doc/todo/Alternative_mode_control_for_import/comment_3_cd4b0f6752457a70f2ab908d23913d76._comment
new file mode 100644
index 0000000000..09ce568ee3
--- /dev/null
+++ b/doc/todo/Alternative_mode_control_for_import/comment_3_cd4b0f6752457a70f2ab908d23913d76._comment
@@ -0,0 +1,16 @@
+[[!comment format=mdwn
+ username="CandyAngel"
+ avatar="http://cdn.libravatar.org/avatar/15c0aade8bec5bf004f939dd73cf9ed8"
+ subject="comment 3"
+ date="2017-02-07T22:51:15Z"
+ content="""
+    An implementation of --mode=Did,Nsid seemed worth adding at first, perhaps as --reinject-duplicates. But thinking about it some more, that would be the same as
+
+        git annex reinject --known /path/*
+        git annex import /path/*
+
+
+--mode=Did,Nsid would be quite a bit faster because it wouldn't hash the files twice, which is an advantage this suggestion has over any multiple command alternative.
+
+If you want to keep it to certain points in space rather than deal with all combinations, you could whitelist which ones are acceptable and people can request more to be whitelisted as they discover use cases for those modes. The current commands would alias to the modes (which would also make their behaviour obvious if this alias is mentioned in the documentation).
+"""]]
diff --git a/doc/todo/Alternative_mode_control_for_import/comment_4_767dfbaf72de52bd5fbe4c37add5bd91._comment b/doc/todo/Alternative_mode_control_for_import/comment_4_767dfbaf72de52bd5fbe4c37add5bd91._comment
new file mode 100644
index 0000000000..70c49cf794
--- /dev/null
+++ b/doc/todo/Alternative_mode_control_for_import/comment_4_767dfbaf72de52bd5fbe4c37add5bd91._comment
@@ -0,0 +1,15 @@
+[[!comment format=mdwn
+ username="joey"
+ subject="""comment 4"""
+ date="2017-02-09T19:33:46Z"
+ content="""
+Actually, import --deduplicate, --skip-duplicates, --clean-duplicates 
+are implemeted naively and do hash files twice. So it's
+the same efficiency..
+
+But, I just finished a more complicated implementation that avoids
+the second hashing.
+
+That does make the combined action worth adding, I suppose. Done so as
+--reinject-duplicates.
+"""]]
diff --git a/doc/todo/Alternative_mode_control_for_import/comment_5_ded83da89e70cadfa4076de9ddc36b55._comment b/doc/todo/Alternative_mode_control_for_import/comment_5_ded83da89e70cadfa4076de9ddc36b55._comment
new file mode 100644
index 0000000000..8e5471c310
--- /dev/null
+++ b/doc/todo/Alternative_mode_control_for_import/comment_5_ded83da89e70cadfa4076de9ddc36b55._comment
@@ -0,0 +1,27 @@
+[[!comment format=mdwn
+ username="joey"
+ subject="""comment 5"""
+ date="2017-02-09T19:45:26Z"
+ content="""
+I feel that the problem with this idea is that the suggested
+actions "create symlinks (s), inject content (i) and delete from source (d)"
+are only an approximation of how import is implemented. If they perfectly
+matched the implementation, then import could treat them as a DSL and
+simply evaluate the expression to do its work. But it's not that simple.
+For one thing, --deduplicate and --clean-duplicates don't simply "delete from source" the
+duplicates; they first check that numcopies can be satisfied. The default
+import behavior doesn't "sid", in fact it moves from source to the work tree
+(thus implicitly deleting from source first), then injects, and then creates
+the symlink. Everything has dependencies and interrelationships, and the best
+way I've found to express that so far is as the Haskell code in
+Command/Import.hs.
+
+Even exposing that interface and using the current implementation for
+particular canned expressions seems risky; exposing imperfect abstractions
+can shoot you in the foot later when something under the abstraction needs
+to change.
+
+So I'd rather improve the documentation for git-annex import if it is
+unclear. Not opposed to finding a way to work in these "Dsid,Nsid"
+summaries to the the documentation.
+"""]]
diff --git a/doc/todo/Amazon_Cloud_Drive.mdwn b/doc/todo/Amazon_Cloud_Drive.mdwn
new file mode 100644
index 0000000000..7ec91b4d31
--- /dev/null
+++ b/doc/todo/Amazon_Cloud_Drive.mdwn
@@ -0,0 +1,23 @@
+I've released a special remote which uses rclone to enable Amazon Cloud Drive:  -- [[DanielDent]]
+
+---
+
+Is there a special remote implementation for Amazon Cloud Drive?
+
+It's just became unlimited for a fair price: $60/year ( https://www.amazon.com/clouddrive/pricing ).
+
+http://techcrunch.com/2015/03/26/amazon-goes-after-dropbox-google-microsoft-with-unlimited-cloud-drive-storage/
+
+-- bence aka [[parhuzamos]] 
+
+> Not yet, but I just need to investigate haskell api bindings for it, I
+> suppose. --[[Joey]]
+
+> > Don't know if there's such a thing... there seems to be SDKs for android and IOS, but nothing more. It seems like it's a REST API: https://developer.amazon.com/public/apis/experience/cloud-drive/... --[[anarcat]]
+
+>> requested it be added to amazonka  --[[Joey]]
+
+>>> In the meantime,  supports Amazon Cloud drive.
+>>> I may revisit adding support for it directly to git-annex if amazonka
+>>> does get support, but for now, that seems good enough, so [[done]]
+>>> --[[Joey]]
diff --git a/doc/todo/Amazon_Cloud_Drive/comment_1_a2fea3eecf0971bf1b9d3c71377d3be9._comment b/doc/todo/Amazon_Cloud_Drive/comment_1_a2fea3eecf0971bf1b9d3c71377d3be9._comment
new file mode 100644
index 0000000000..c7db6882c4
--- /dev/null
+++ b/doc/todo/Amazon_Cloud_Drive/comment_1_a2fea3eecf0971bf1b9d3c71377d3be9._comment
@@ -0,0 +1,14 @@
+[[!comment format=mdwn
+ username="interfect@b151490178830f44348aa57b77ad58c7d18e8fe7"
+ nickname="interfect"
+ subject="Windows support?"
+ date="2016-09-01T03:31:30Z"
+ content="""
+The git-annex-remote-rclone solution doesn't seem to work in Windows.
+
+I have the script on my PATH, and I'm doing everything in Git Bash. I can run the script from the shell just fine, but Git Annex itself says it can't execute it.
+
+git-annex: Cannot run git-annex-remote-rclone -- Make sure it's in your PATH and is executable.
+
+Presumably it doesn't know how to talk to bash.exe to get it to run the script.
+"""]]
diff --git a/doc/todo/Amazon_Cloud_Drive/comment_2_9adfd0dc49540a90d8d8a941c9c56fbe._comment b/doc/todo/Amazon_Cloud_Drive/comment_2_9adfd0dc49540a90d8d8a941c9c56fbe._comment
new file mode 100644
index 0000000000..2d50ddaba7
--- /dev/null
+++ b/doc/todo/Amazon_Cloud_Drive/comment_2_9adfd0dc49540a90d8d8a941c9c56fbe._comment
@@ -0,0 +1,11 @@
+[[!comment format=mdwn
+ username="joey"
+ subject="""comment 2"""
+ date="2016-09-05T18:26:03Z"
+ content="""
+@interfect probably what was discussed here
+
+
+I've made the necessary change to git-annex this morning so it should be
+able to run external special remote scripts on Windows. Have not tested it.
+"""]]
diff --git a/doc/todo/Amazon_Cloud_Drive/comment_3_6a4833f0c52d3726cd95d0818e0a89bf._comment b/doc/todo/Amazon_Cloud_Drive/comment_3_6a4833f0c52d3726cd95d0818e0a89bf._comment
new file mode 100644
index 0000000000..c1fd609194
--- /dev/null
+++ b/doc/todo/Amazon_Cloud_Drive/comment_3_6a4833f0c52d3726cd95d0818e0a89bf._comment
@@ -0,0 +1,11 @@
+[[!comment format=mdwn
+ username="jason.dixon.email@aa0e536a2ec2877d6f666108dbbc6e39bbe87ac0"
+ nickname="jason.dixon.email"
+ avatar="http://cdn.libravatar.org/avatar/fbe9050fc83bbd536d307d87ea14d4bc"
+ subject="Same issue with rclone on windows."
+ date="2017-03-08T08:55:48Z"
+ content="""
+Using linux it works perfectly. Really great. But on windows no such luck. Trying many things to get it to function and no success.
+
+Given the thinness of the bash wrapper that binds rclone to annex, and rclones inherent cross platform nature. It would be awesome to have this remote as a first class citizen. Similar to amazon glacier requiring glacier-cli. Simply requiring rclone to open up so many backends is a very light dependency. :)
+"""]]
diff --git a/doc/todo/Amazon_Cloud_Drive/comment_4_7e66da169c3decb5771226cd3d4a905b._comment b/doc/todo/Amazon_Cloud_Drive/comment_4_7e66da169c3decb5771226cd3d4a905b._comment
new file mode 100644
index 0000000000..f66db898a2
--- /dev/null
+++ b/doc/todo/Amazon_Cloud_Drive/comment_4_7e66da169c3decb5771226cd3d4a905b._comment
@@ -0,0 +1,9 @@
+[[!comment format=mdwn
+ username="jason.dixon.email@aa0e536a2ec2877d6f666108dbbc6e39bbe87ac0"
+ nickname="jason.dixon.email"
+ avatar="http://cdn.libravatar.org/avatar/fbe9050fc83bbd536d307d87ea14d4bc"
+ subject="Oh and to be slightly helpful"
+ date="2017-03-08T08:57:08Z"
+ content="""
+This is with version 6.20170302. So as of this writing, the current version I believe.
+"""]]
diff --git a/doc/todo/Amazon_Cloud_Drive/comment_5_441e2cf615735bc3fc403702edc8a809._comment b/doc/todo/Amazon_Cloud_Drive/comment_5_441e2cf615735bc3fc403702edc8a809._comment
new file mode 100644
index 0000000000..6df8c84411
--- /dev/null
+++ b/doc/todo/Amazon_Cloud_Drive/comment_5_441e2cf615735bc3fc403702edc8a809._comment
@@ -0,0 +1,10 @@
+[[!comment format=mdwn
+ username="anarcat"
+ avatar="http://cdn.libravatar.org/avatar/4ad594c1e13211c1ad9edb81ce5110b7"
+ subject="added as a special remote"
+ date="2018-09-06T00:58:52Z"
+ content="""
+it was not obvious to me at all that rclone was supported by git-annex! that's awesome!
+
+to make it clearer it exists, i've added it it to the list of [[special remotes]] as [[special_remotes/rclone]].
+"""]]
diff --git a/doc/todo/Android__58___add_a___34__Share_via__34___shortcut___34__Add_to_Annex__34__.mdwn b/doc/todo/Android__58___add_a___34__Share_via__34___shortcut___34__Add_to_Annex__34__.mdwn
new file mode 100644
index 0000000000..d04eee18d9
--- /dev/null
+++ b/doc/todo/Android__58___add_a___34__Share_via__34___shortcut___34__Add_to_Annex__34__.mdwn
@@ -0,0 +1 @@
+Having it named starting with "Add to ..." would improve its convenience visibility (would even position before "Add to Dropbox" ;) )
diff --git a/doc/todo/Android__58___add_a___34__Share_via__34___shortcut___34__Add_to_Annex__34__/comment_1_286279b96fc7940c9e7eeb0eb57cc279._comment b/doc/todo/Android__58___add_a___34__Share_via__34___shortcut___34__Add_to_Annex__34__/comment_1_286279b96fc7940c9e7eeb0eb57cc279._comment
new file mode 100644
index 0000000000..803f657f0f
--- /dev/null
+++ b/doc/todo/Android__58___add_a___34__Share_via__34___shortcut___34__Add_to_Annex__34__/comment_1_286279b96fc7940c9e7eeb0eb57cc279._comment
@@ -0,0 +1,13 @@
+[[!comment format=mdwn
+ username="joey"
+ subject="""comment 1"""
+ date="2018-05-08T18:59:08Z"
+ content="""
+We have a way to run the regular git-annex build for linux on Android in termux:
+
+
+So you can use the x86 autobuild on android now. I don't currently have a
+mips autobuild, but you can build git-annex on a Debian mips system
+with "make linuxstandalone" and the resulting tarball should work with the
+above instructions on a mips android system, presumably.
+"""]]
diff --git a/doc/todo/Bidirectional_metadata.mdwn b/doc/todo/Bidirectional_metadata.mdwn
new file mode 100644
index 0000000000..cdbdf614c5
--- /dev/null
+++ b/doc/todo/Bidirectional_metadata.mdwn
@@ -0,0 +1,21 @@
+In one of my annexes I will g-a-add a (e.g.) zip file, extract and g-a-add its contents and then set metadata on "each side"..
+
+    git annex metadata -s extracted_from+=$key_for_original.zip files_from_zip/
+    git annex metadata -s extracted_to+=$key_for_file1 original.zip
+    git annex metadata -s extracted_to+=$key_for_file2 original.zip
+    git annex metadata -s extracted_to+=$key_for_file3 original.zip
+    git annex metadata -s extracted_to+=$key_for_file4 original.zip
+
+This means that if the content for a file in files_from_zip/ is lost, you can easily find that it came from original.zip (which may not be lost) *or any other zip in that annex that also contains that file*.
+
+I was wondering if this would be worth building in?
+
+    git annex metadata --fromto original.zip files_from_zip/
+
+There are other situations this is useful (and I use), for example, when I convert an annexed file to another format, I record the "source" and "target" keys against each other. I suppose a very general name would be:
+
+    git annex metadata --parentchild original.zip files_from_zip/
+    git annex metadata --parentchild original.wav compressed.flac
+    git annex metadata --parentchild original.svg compressed.png
+
+and this would set 'parent' and 'child' metadata respectively.
diff --git a/doc/todo/Bidirectional_metadata/comment_1_9974d17e407fab14b12399016d3d66c2._comment b/doc/todo/Bidirectional_metadata/comment_1_9974d17e407fab14b12399016d3d66c2._comment
new file mode 100644
index 0000000000..011482b803
--- /dev/null
+++ b/doc/todo/Bidirectional_metadata/comment_1_9974d17e407fab14b12399016d3d66c2._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="CandyAngel"
+ avatar="http://cdn.libravatar.org/avatar/15c0aade8bec5bf004f939dd73cf9ed8"
+ subject="comment 1"
+ date="2017-05-09T14:25:50Z"
+ content="""
+It could also be possible to extend 'whereis' (maybe with a '--check-metadata' option?) to let the user know that *some* version (source or derivative) may be available.
+"""]]
diff --git a/doc/todo/Bidirectional_metadata/comment_2_c791aca6da2bb1be70991bc1a76982a4._comment b/doc/todo/Bidirectional_metadata/comment_2_c791aca6da2bb1be70991bc1a76982a4._comment
new file mode 100644
index 0000000000..449f415b78
--- /dev/null
+++ b/doc/todo/Bidirectional_metadata/comment_2_c791aca6da2bb1be70991bc1a76982a4._comment
@@ -0,0 +1,20 @@
+[[!comment format=mdwn
+ username="joey"
+ subject="""comment 2"""
+ date="2017-05-09T17:27:35Z"
+ content="""
+I think that the core primitive here may not be the bidirectional
+relationship, but instead setting a metadata field to a key.
+
+If the metadata is known to be a key rather than some other type
+of value, then key-specific things can be done with it.
+
+So perhaps the real core thing is to be able to define what the type of a
+metadata field is. That could be done using $field-type to define the type
+of $field, but it seems it would be better to define it somewhat
+globally so that fields with the same name can't have different types.
+
+(The datalad project does something similar for tarballs, using a custom
+special remote that retrieves the tarball as an annexed object and then
+explodes it to get the requested member object.)
+"""]]
diff --git a/doc/todo/Bittorrent-like_features.mdwn b/doc/todo/Bittorrent-like_features.mdwn
new file mode 100644
index 0000000000..2ffe7fecc7
--- /dev/null
+++ b/doc/todo/Bittorrent-like_features.mdwn
@@ -0,0 +1,66 @@
+There are two different possible ways git-annex could use bittorrent:
+
+Let's describe those one by one.
+
+[[!toc]]
+
+Downloading files from multiple git-annex sources simultaneously
+================================================================
+
+Having your remotes (optionally!) act like a swarm would be an awesome feature to have because you bring in a lot of new features that optimize storage, bandwidth, and overall traffic usage. This would be made a lot easier if parts of it were implemented in small steps that added a nifty feature. The best part is, each of these could be implemented by themselves, and they're all features that would be really useful.
+
+ 1. Concurrent downloads of a file from remotes.
+
+    This would make sense to have, it saves upload traffic on your remotes, and you also get faster DL speeds on the receiving end.
+
+ 2. Implementing part of the super-seeding capabilities.
+
+    You upload pieces of a file to different remotes from your laptop, and on your desktop you can download all those pieces and put them together again to get a complete file. If you really wanted to get fancy, you could build in redundancy (ala RAID) so if a remote or two gets lost, you don't lose the entire file. This would be a very efficient use of storage if you have a bunch of free cloud storage accounts (~1GB each) and some big files you want to back up.
+
+ 3. Setting it up so that those remotes could talk to one another and share those pieces.
+
+    This is where it gets more like bittorrent. Useful because you upload 1 copy and in a few hours, have say, 5 complete copies on 5 different remotes. You could add or remove remotes from a swarm locally, and push those changes to those remotes, which then adapt themselves to suit the new rules and share those with other remotes in the swarm (rules should be GPG-signed as a safety precaution). Also, if/when deltas get implemented, you could push that delta to the swarm and have all the remotes adopt it. This is cooler than regular bittorrent because the shared file can be updated. As a safety precaution, the delta could be GPG signed so a corrupt file doesn't contaminate the entire swarm. Each remote could have bandwidth/storage limits set in a dotfile.
+
+This is a high-level idea of how it might work, and it's also a HUGE set of features to add, but if implemented, you'd be saving a ton of resources, adding new use cases, and making git-annex more flexible.
+
+Obviously, Step 3 would only work on remotes that you have control of processes on, but if given login credentials to cloud storage remotes (potentially dangerous!) they could read/write to something like dropbox or rsync.
+
+Another thing, this would be completely trackerless. You just use remote groups (or create swarm definitions) and share those with your remotes. **It's completely decentralized!**
+
+This was originally posted [[as a forum post|forum/Wishlist:_Bittorrent-like_transfers]] by [[users/GLITTAH]].
+
+Update: there are multiple projects trying to solve this problem space outside of git-annex, which git-annex should reuse.
+
+ * [[design/assistant/telehash/]] support is still not complete, as the upstream spec and implementation (particularly Haskell bits), need to mature
+ * [[special_remotes/ipfs/]] is now a special remote that does respond to some of the requirements (but [duplicates files around](https://github.com/ipfs/go-ipfs/issues/875) even more and is [kind of](https://github.com/ipfs/go-ipfs/issues/872) [slow](https://github.com/ipfs/go-ipfs/issues/898))
+ * [dat](https://datproject.org/) is a bit like IPFS, except it doesn't duplicate files around so it might be a better match to sync stuff around
+ * [Maidsafe](http://maidsafe.net/) is another option, which provides storage and uses crypto-currency incentives
+ * [Storj](http://storj.io/) is similar
+ * [camlistore](https://camlistore.org/) is yet another option
+ * [syncthing](https://syncthing.net/) looks like a btsync replacement, and could also be interesting, see the [[todo/syncthing_special_remote]] discussion
+ * [gittorrent](http://blog.printf.net/articles/2015/05/29/announcing-gittorrent-a-decentralized-github/) allows for decentralised sharing of the git objects, which could replace pairing between repositories, except it [doesn't support push yet](https://github.com/cjb/GitTorrent/issues/3)
+ * [gitocalypse](https://github.com/SeekingFor/gitocalypse) is similar to gittorrent, except it uses Freenet and HG (Mercurial?!) instead of the bittorrent DHT
+ * [tox](https://tox.chat/) - DHT-enabled chat, file transfer and messaging platform, [no stable release](https://github.com/irungentoo/toxcore/issues/1353), [security concerns](http://lists.alioth.debian.org/pipermail/pkg-privacy-maintainers/Week-of-Mon-20150928/000046.html)
+ * [ricochet](https://ricochet.im/) - similar, but relies on Tor, unclear if it supports file transfers, packaged in Debian
+ * [pond](https://pond.imperialviolet.org/) - similar to ricochet
+ * [scout](https://github.com/bittorrent/scout) - DHT-based messaging and peer discovery, used for the Bittorrent-based [bleep messenger](http://www.bleep.pm), could be used to sync git history? [introduction article](http://blog.bittorrent.com/2016/09/09/scout-securely-locate-peers-without-central-servers/)
+ * [pwnat](http://samy.pl/pwnat/) can similarly be used to create tunnels to bypass NATs
+
+joeyh worked on [[design/assistant/telehash/]] for a while but then switched to [[wait and see|devblog/day_219__catching_up_and_looking_back/]], when it became clear telehash wasn't quite mature enough yet. he eventually implemented a solution for part 3 of this problem (connecting to peers without setting up a server) with the [[tor special remote|special_remotes/tor/]], see [[tips/peer_to_peer_network_with_tor/]] for instructions. This doesn't directly address the other two requirements in the sense that each remote need a custom setup to join the p2p exchange, which is still one-to-one and not many-to-many as typical p2p applications are. -- [[anarcat]] 2018-08-24
+
+Using an external client (addurl torrent support)
+=================================================
+
+The alternative to this would be to add `addurl` support for bittorrent files. The same way we can now add Youtube videos to a git-annex repository thanks to [[quvi]], we could also simply do:
+
+    git annex addtorrent debian-live-7.0.0-amd64-standard.iso.torrent
+
+or even better:
+
+    git annex addurl http://cdimage.debian.org/debian-cd/current-live/amd64/bt-hybrid/debian-live-7.0.0-amd64-standard.iso.torrent
+
+This way, a torrent would just become another source for a specific file. When we `get` the file, it fires up `$YOUR_FAVORITE_TORRENT_CLIENT` to download the file.
+
+That way we avoid the implementation complexity of shoving a complete bittorrent client within the assistant. The `get` operation would block until the torrent is downloaded, i guess... --[[anarcat]]
+
+> This is now [[implemented|special_remotes/bittorrent/]]. Including magnet link support, and multi-file torrent support. Leaving todo item open for the blue-sky stuff at top. --[[Joey]]
diff --git a/doc/todo/Bittorrent-like_features/comment_1_f4c110ef35ebf4fd89f06edf2c4f0c48._comment b/doc/todo/Bittorrent-like_features/comment_1_f4c110ef35ebf4fd89f06edf2c4f0c48._comment
new file mode 100644
index 0000000000..eba291af97
--- /dev/null
+++ b/doc/todo/Bittorrent-like_features/comment_1_f4c110ef35ebf4fd89f06edf2c4f0c48._comment
@@ -0,0 +1,13 @@
+[[!comment format=mdwn
+ username="Gastlag"
+ ip="109.190.97.30"
+ subject="Gittorrent"
+ date="2013-08-28T21:49:56Z"
+ content="""
+May this could interest you : few years ago somes tried to mix Git and Bittorrent.
+
+http://www.advogato.org/article/994.html
+http://utsl.gen.nz/gittorrent/rfc.html
+http://code.google.com/p/gittorrent/
+https://git.wiki.kernel.org/index.php/SoC2010Application#Did_your_organization_participate_in_past_GSoCs.3F_If_so.2C_please_summarize_your_involvement_and_the_successes_and_challenges_of_your_participation
+"""]]
diff --git a/doc/todo/Bittorrent-like_features/comment_2_83148bd5c5c5e6c2eff3ad6e1d4fb82c._comment b/doc/todo/Bittorrent-like_features/comment_2_83148bd5c5c5e6c2eff3ad6e1d4fb82c._comment
new file mode 100644
index 0000000000..ed6e92fd8f
--- /dev/null
+++ b/doc/todo/Bittorrent-like_features/comment_2_83148bd5c5c5e6c2eff3ad6e1d4fb82c._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="https://www.google.com/accounts/o8/id?id=AItOawmTIL7ubr5opWM69Q5VtCxuxC2H0SSnzic"
+ nickname="Petr"
+ subject="Regarding the approach to connect all nodes in one network and stream from them"
+ date="2013-12-29T07:32:05Z"
+ content="""
+Have a look at http://socialvpn.wordpress.com/. Behind that project is couple of interesting tools worth to investigate. 
+"""]]
diff --git a/doc/todo/Bittorrent-like_features/comment_3_84f149b30de1562593623aa23dc0396c._comment b/doc/todo/Bittorrent-like_features/comment_3_84f149b30de1562593623aa23dc0396c._comment
new file mode 100644
index 0000000000..a405bed09f
--- /dev/null
+++ b/doc/todo/Bittorrent-like_features/comment_3_84f149b30de1562593623aa23dc0396c._comment
@@ -0,0 +1,10 @@
+[[!comment format=mdwn
+ username="https://www.google.com/accounts/o8/id?id=AItOawl9sYlePmv1xK-VvjBdN-5doOa_Xw-jH4U"
+ nickname="Richard"
+ subject="comment 3"
+ date="2014-01-02T01:31:05Z"
+ content="""
+> git annex addurl http://cdimage.debian.org/debian-cd/current-live/amd64/bt-hybrid/debian-live-7.0.0-amd64-standard.iso.torrent
+
+This is bad idea; it means you will be reliant on torrent files and trackers when magnet links are the way to go.
+"""]]
diff --git a/doc/todo/Bittorrent-like_features/comment_4_7c54c83e582c0d4848aaf3d70e312707._comment b/doc/todo/Bittorrent-like_features/comment_4_7c54c83e582c0d4848aaf3d70e312707._comment
new file mode 100644
index 0000000000..6787a24607
--- /dev/null
+++ b/doc/todo/Bittorrent-like_features/comment_4_7c54c83e582c0d4848aaf3d70e312707._comment
@@ -0,0 +1,10 @@
+[[!comment format=mdwn
+ username="https://www.google.com/accounts/o8/id?id=AItOawl9sYlePmv1xK-VvjBdN-5doOa_Xw-jH4U"
+ nickname="Richard"
+ subject="comment 4"
+ date="2014-01-02T01:32:37Z"
+ content="""
+There's a bittorrent library for Haskell, btw: http://hackage.haskell.org/package/bittorrent
+
+Native support in form of a special remote would arguably be best...
+"""]]
diff --git a/doc/todo/Bittorrent-like_features/comment_5_194dd0e8404ea72af9fb6ff34b994998._comment b/doc/todo/Bittorrent-like_features/comment_5_194dd0e8404ea72af9fb6ff34b994998._comment
new file mode 100644
index 0000000000..620c82e973
--- /dev/null
+++ b/doc/todo/Bittorrent-like_features/comment_5_194dd0e8404ea72af9fb6ff34b994998._comment
@@ -0,0 +1,20 @@
+[[!comment format=mdwn
+ username="https://id.koumbit.net/anarcat"
+ ip="2001:1928:1:9::1"
+ subject="comment 5"
+ date="2014-04-01T04:43:16Z"
+ content="""
+re #3, sure, magnet link support would be awesome as well but i'd prefer to start with something i could digest more easily.
+
+looking at the source, it seems to me that the [quvi implementation](http://source.git-annex.branchable.com/?p=source.git;a=commitdiff;h=46b6d75) could serve as an example as to how this would work. more particularly, there's this concept of a [downloader](http://source.git-annex.branchable.com/?p=source.git;a=commitdiff;h=46b6d75#patch5) that can be used to tap into `addurl` directly. there's a check to see if the downloader is supported, for example.
+
+so we would need:
+
+1. see if the URL / magnet link can be turned into a .torrent somehow
+2. figure out what the filename(s!) will be
+3. start the torrent and wait for its completion, ideally with some progress bar
+
+i asked around to see if transmission-remote could do this, because it would be nice if we could use an existing daemon (instead of having to rebootstrap the whole DHT at every download). so far, I can't see how it could be done cleanly - maybe we would need to use the simpler \"bittorrent\" commandline client, or maybe tap into libtorrent...
+
+in any case, one of the key problems here is that addurl assumes that the URL maps to a single file, not a directory full of file, which is the way bittorrent works. I am not sure how to fix that assumption.
+"""]]
diff --git a/doc/todo/Bittorrent-like_features/comment_6_489505da4143fb1c2bf21e7af695cdef._comment b/doc/todo/Bittorrent-like_features/comment_6_489505da4143fb1c2bf21e7af695cdef._comment
new file mode 100644
index 0000000000..678bb109b9
--- /dev/null
+++ b/doc/todo/Bittorrent-like_features/comment_6_489505da4143fb1c2bf21e7af695cdef._comment
@@ -0,0 +1,9 @@
+[[!comment format=mdwn
+ username="https://www.google.com/accounts/o8/id?id=AItOawmHp5oVW8Z9v_vHs5FRtlYj80TYMQWVTS0"
+ nickname="dhead"
+ subject="comment 6"
+ date="2014-06-10T02:18:43Z"
+ content="""
+What about Syncthing protocol ? 
+https://github.com/calmh/syncthing/blob/master/protocol/PROTOCOL.md
+"""]]
diff --git a/doc/todo/Can__39__t_access_files_from___39__Removable_drive__39___repo_even_if_set_as_client.mdwn b/doc/todo/Can__39__t_access_files_from___39__Removable_drive__39___repo_even_if_set_as_client.mdwn
new file mode 100644
index 0000000000..941ac9e5ac
--- /dev/null
+++ b/doc/todo/Can__39__t_access_files_from___39__Removable_drive__39___repo_even_if_set_as_client.mdwn
@@ -0,0 +1,22 @@
+Note: This may not be a bug maybe the software doesn't work the way I think.
+
+What steps will reproduce the problem?
+
+Using the webapp, I created a normal repo on my desktop. I then added a 'Removable drive' repo on my usb stick.
+
+I want all my files to be synced and accessibles on both repos so I set the 'Removable drive' repo to 'client'.
+
+
+What is the expected output? What do you see instead?
+
+When I look in the 'annex' folder on my usb stick. I see a git repo (annex, branches, hooks) instead of seeing the files in the same way the 'annex' repo on my desktop is.
+
+
+What version of git-annex are you using? On what operating system?
+
+I'm using 9e57edff287ac53fc4b1cefef7271e9ed17f2285 (Fri Feb 22 15:19:25 2013 +0000).
+
+Ubuntu 12.10 x86_64
+
+[[!tag /design/assistant]]
+[[!meta title="assistant should set up non-bare repos on removable drives, and update them when syncing with them"]]
diff --git a/doc/todo/Can__39__t_access_files_from___39__Removable_drive__39___repo_even_if_set_as_client/comment_10_e33dddf4d1f104107c86800a0a9a89b2._comment b/doc/todo/Can__39__t_access_files_from___39__Removable_drive__39___repo_even_if_set_as_client/comment_10_e33dddf4d1f104107c86800a0a9a89b2._comment
new file mode 100644
index 0000000000..0e2c11656e
--- /dev/null
+++ b/doc/todo/Can__39__t_access_files_from___39__Removable_drive__39___repo_even_if_set_as_client/comment_10_e33dddf4d1f104107c86800a0a9a89b2._comment
@@ -0,0 +1,7 @@
+[[!comment format=mdwn
+ username="http://joeyh.name/"
+ subject="comment 10"
+ date="2014-11-25T21:28:56Z"
+ content="""
+[[forum/usability:_creating_an_archive_on_a_new_external_drive]] has discussion of why the assistant cannot do this by default.
+"""]]
diff --git a/doc/todo/Can__39__t_access_files_from___39__Removable_drive__39___repo_even_if_set_as_client/comment_1_25eb2d7d0a9cdd1c55df0cec68472723._comment b/doc/todo/Can__39__t_access_files_from___39__Removable_drive__39___repo_even_if_set_as_client/comment_1_25eb2d7d0a9cdd1c55df0cec68472723._comment
new file mode 100644
index 0000000000..282e78298b
--- /dev/null
+++ b/doc/todo/Can__39__t_access_files_from___39__Removable_drive__39___repo_even_if_set_as_client/comment_1_25eb2d7d0a9cdd1c55df0cec68472723._comment
@@ -0,0 +1,10 @@
+[[!comment format=mdwn
+ username="http://joeyh.name/"
+ ip="4.153.253.75"
+ subject="comment 1"
+ date="2013-02-22T21:46:59Z"
+ content="""
+You have a bare git repository on your USB stick. The assistant uses bare repositories on USB sticks currently, although recent changes to support FAT make it possible to use non-bare repositories.
+
+For this to really work the way you want it to, the assistant would need to start updating local non-bare repositories whenever it pushed changes to them. Currently the assistant only updates the repository it's running in, so a non-bare repository on USB would lag behind and show an old version of the directory until manually updated with `git annex sync`.
+"""]]
diff --git a/doc/todo/Can__39__t_access_files_from___39__Removable_drive__39___repo_even_if_set_as_client/comment_2_9e9b96e5113a50533251e946c2560d81._comment b/doc/todo/Can__39__t_access_files_from___39__Removable_drive__39___repo_even_if_set_as_client/comment_2_9e9b96e5113a50533251e946c2560d81._comment
new file mode 100644
index 0000000000..da1fcbc0f2
--- /dev/null
+++ b/doc/todo/Can__39__t_access_files_from___39__Removable_drive__39___repo_even_if_set_as_client/comment_2_9e9b96e5113a50533251e946c2560d81._comment
@@ -0,0 +1,17 @@
+[[!comment format=mdwn
+ username="Xyem"
+ ip="87.194.19.134"
+ subject="comment 2"
+ date="2013-02-27T00:02:01Z"
+ content="""
+Struggling to get the assistant to behave properly, including a similar situation to the above.
+
+I start the assistant with 'git annex webapp' and create an annex from the webapp.
+Adding an SSH remote (even if just a different directory on localhost) with \"Remote server\", creates the remote directory with what looks like what the contents of .git should be (annex, branches, hooks, objects etc.), regardless of the group chosen.
+
+Judging from your above comment, this means it is creating a bare repository. Why would it be doing this for an SSH remote, where git-annex-shell is available?
+
+System: Arch Linux
+git-annex version: 3.20130216
+Installed with: yaourt -S git-annex-bin
+"""]]
diff --git a/doc/todo/Can__39__t_access_files_from___39__Removable_drive__39___repo_even_if_set_as_client/comment_3_6b091198ddd6ed709b076df1296aeb77._comment b/doc/todo/Can__39__t_access_files_from___39__Removable_drive__39___repo_even_if_set_as_client/comment_3_6b091198ddd6ed709b076df1296aeb77._comment
new file mode 100644
index 0000000000..48d1b26040
--- /dev/null
+++ b/doc/todo/Can__39__t_access_files_from___39__Removable_drive__39___repo_even_if_set_as_client/comment_3_6b091198ddd6ed709b076df1296aeb77._comment
@@ -0,0 +1,11 @@
+[[!comment format=mdwn
+ username="http://edheil.wordpress.com/"
+ ip="173.162.44.162"
+ subject="comment 3"
+ date="2013-02-27T01:12:29Z"
+ content="""
+Xyem, if you use git to clone a repo, instead of letting the assistant do it for you, you can get a non-bare repo.  See the walkthrough for some examples.  You can keep its working tree up to date with \"git annex sync\" run in that directory.
+
+Only helpful if you don't mind using git annex at the command line, I know.
+
+"""]]
diff --git a/doc/todo/Can__39__t_access_files_from___39__Removable_drive__39___repo_even_if_set_as_client/comment_4_118b588685b535cca4c02eb6ef297c67._comment b/doc/todo/Can__39__t_access_files_from___39__Removable_drive__39___repo_even_if_set_as_client/comment_4_118b588685b535cca4c02eb6ef297c67._comment
new file mode 100644
index 0000000000..8ad0de3a5c
--- /dev/null
+++ b/doc/todo/Can__39__t_access_files_from___39__Removable_drive__39___repo_even_if_set_as_client/comment_4_118b588685b535cca4c02eb6ef297c67._comment
@@ -0,0 +1,21 @@
+[[!comment format=mdwn
+ username="Xyem"
+ ip="87.194.19.134"
+ subject="comment 4"
+ date="2013-02-27T12:16:45Z"
+ content="""
+@edheil:
+Yes, I have found that creating the repositories outside of the assistant (or directly on each machine through the assistant) and adding the remotes manually doesn't have this issue.. but as I am not setting this up for myself, so dropping to the command line is an issue (she's not averse to it, but also isn't a power user).
+
+It seems that the assistant/webapp is replacing dropbox functionality, while ignoring the functionality of git-annex. Perhaps I am misunderstanding the goal of it..?
+
+My intended use case in the situation where this issue comes up is:
+
+Repository on desktop has a copy of all files in indirect mode.
+Repository on netbook only has a copy of files added locally or fetched manually.
+Dropping a file in the netbook annex should cause it to be uploaded to the desktop repository (when it is available). Files should be fetchable/droppable (and unlockable) using the webapp from the desktop/netbook repository respectively.
+
+But it looks like this is (currently) unachievable as the webapp provides no method of fetching/dropping files, the assistant does not create the remote repositories correctly and requires dropping to the command line if worked around. Eep!
+
+Hope this doesn't come off as sour complaining. I was introduced to git-annex by a friend about a week ago (she only learned about it about 2-3 weeks ago and *loves* it) and it is a very nice and flexible tool which fits my own use case perfectly, meeting and exceeding every expectation.. until it came to the assistant and webapp (which makes me wonder why Joey is working on an Android port..).
+"""]]
diff --git a/doc/todo/Can__39__t_access_files_from___39__Removable_drive__39___repo_even_if_set_as_client/comment_5_5cead277493e1c020e16be6f9245fe33._comment b/doc/todo/Can__39__t_access_files_from___39__Removable_drive__39___repo_even_if_set_as_client/comment_5_5cead277493e1c020e16be6f9245fe33._comment
new file mode 100644
index 0000000000..26b472dc75
--- /dev/null
+++ b/doc/todo/Can__39__t_access_files_from___39__Removable_drive__39___repo_even_if_set_as_client/comment_5_5cead277493e1c020e16be6f9245fe33._comment
@@ -0,0 +1,12 @@
+[[!comment format=mdwn
+ username="http://edheil.wordpress.com/"
+ ip="173.162.44.162"
+ subject="comment 5"
+ date="2013-02-27T17:05:07Z"
+ content="""
+There is a way to fetch/drop files if you don't want to use the command line, though it's not nearly as flexible as using a command line.  If you've got a machine set as \"client\" then dragging files into a subdirectory named \"archive\" drops them, and dragging them out of that subdirectory fetches them.
+
+Honestly, moving files in and out of directories, and preferred content settings, have both been subject to the occasional regression, so there have been times this hasn't worked right, but in theory, that's how you do it.
+
+Unfortunately if you've got more than one \"client\" machine, this means adding/dropping files on one will affect them all, as file movements propagate.
+"""]]
diff --git a/doc/todo/Can__39__t_access_files_from___39__Removable_drive__39___repo_even_if_set_as_client/comment_6_0f135f97c2808dce094628dc6608e617._comment b/doc/todo/Can__39__t_access_files_from___39__Removable_drive__39___repo_even_if_set_as_client/comment_6_0f135f97c2808dce094628dc6608e617._comment
new file mode 100644
index 0000000000..3fadb6219f
--- /dev/null
+++ b/doc/todo/Can__39__t_access_files_from___39__Removable_drive__39___repo_even_if_set_as_client/comment_6_0f135f97c2808dce094628dc6608e617._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="http://edheil.wordpress.com/"
+ ip="173.162.44.162"
+ subject="comment 6"
+ date="2013-02-27T17:06:45Z"
+ content="""
+also... I've never used the feature where you have two local machines running git-annex and you tell one to share a repo with the other through the web interface, but maybe it would be helpful for this scenario?  Apologies if you've already tried that.
+"""]]
diff --git a/doc/todo/Can__39__t_access_files_from___39__Removable_drive__39___repo_even_if_set_as_client/comment_7_1d6f47f9e6cf935f19d68af6d5aa92fa._comment b/doc/todo/Can__39__t_access_files_from___39__Removable_drive__39___repo_even_if_set_as_client/comment_7_1d6f47f9e6cf935f19d68af6d5aa92fa._comment
new file mode 100644
index 0000000000..03d9950bca
--- /dev/null
+++ b/doc/todo/Can__39__t_access_files_from___39__Removable_drive__39___repo_even_if_set_as_client/comment_7_1d6f47f9e6cf935f19d68af6d5aa92fa._comment
@@ -0,0 +1,10 @@
+[[!comment format=mdwn
+ username="http://joeyh.name/"
+ nickname="joey"
+ subject="comment 7"
+ date="2013-02-27T17:43:45Z"
+ content="""
+The way this is supposed to (and does) work is that you use the webapp to create a repository on each computer, and then you pair them together using either local pairing or xmpp pairing.
+
+Today's release of git-annex also adds UI in the webapp to create additional local repositories that are connected to the current repository. However these are not really intended to be put on removable drives since the assistant needs to be running on them to keep them up-to-date.
+"""]]
diff --git a/doc/todo/Can__39__t_access_files_from___39__Removable_drive__39___repo_even_if_set_as_client/comment_8_c5758fdb32348b9cd804ff17d27864e1._comment b/doc/todo/Can__39__t_access_files_from___39__Removable_drive__39___repo_even_if_set_as_client/comment_8_c5758fdb32348b9cd804ff17d27864e1._comment
new file mode 100644
index 0000000000..1ef87e6a65
--- /dev/null
+++ b/doc/todo/Can__39__t_access_files_from___39__Removable_drive__39___repo_even_if_set_as_client/comment_8_c5758fdb32348b9cd804ff17d27864e1._comment
@@ -0,0 +1,16 @@
+[[!comment format=mdwn
+ username="Xyem"
+ ip="87.194.19.134"
+ subject="comment 8"
+ date="2013-02-27T18:36:16Z"
+ content="""
+@Joey:
+As my use case is 1 person using multiple machines, I didn't think local/XMPP pairing was appropriate.
+
+Local pairing because the netbook leaves the network the desktop machine is on, which is when transferring the files is wanted (otherwise, they would just use the desktop...).
+XMPP pairing description implies that it is for 2 different people (thus 2 different addresses). Can it be used where both annexes are using the same address? Or does the address also allow you to include the location (i.e. name@host.com/netbook)?
+
+It definitely sounds like I am misunderstanding how this is supposed to work so I apologise for that. Please keep up the excellent work. git-annex is one of those tools I wish I had learned about a long time ago!
+
+@edheil: Interesting workaround, thanks.
+"""]]
diff --git a/doc/todo/Can__39__t_access_files_from___39__Removable_drive__39___repo_even_if_set_as_client/comment_9_3f0bfc5a79aa59ac76a6968aacda6655._comment b/doc/todo/Can__39__t_access_files_from___39__Removable_drive__39___repo_even_if_set_as_client/comment_9_3f0bfc5a79aa59ac76a6968aacda6655._comment
new file mode 100644
index 0000000000..d370bb28d2
--- /dev/null
+++ b/doc/todo/Can__39__t_access_files_from___39__Removable_drive__39___repo_even_if_set_as_client/comment_9_3f0bfc5a79aa59ac76a6968aacda6655._comment
@@ -0,0 +1,10 @@
+[[!comment format=mdwn
+ username="https://id.koumbit.net/anarcat"
+ ip="72.0.72.144"
+ subject="confusing..."
+ date="2014-05-14T05:04:01Z"
+ content="""
+this is still really confusing. having to create the repo by hand is a really confusing step - i have setup an external drive to have my files with me as i move around, having to go through hoops to find them in a bare repository seems contrary to the spirit of \"future proofing\" inherent to git annex.
+
+the solution i found was to `git init` the repo on the removable drive before adding it through the webapp. i hope this can eventually be fixed, as it led to confusion among many other users (e.g. [[forum/Accessing_files_in_bare_repository/]], [[bugs/Adding a repository as a \"remote server\" creates a bare repository next to the existing one/]],  [[forum/remote server client repositories are bare!?/]] and so on). --[[anarcat]]
+"""]]
diff --git a/doc/todo/Configuring_metadata_view_filenames.mdwn b/doc/todo/Configuring_metadata_view_filenames.mdwn
new file mode 100644
index 0000000000..220c470014
--- /dev/null
+++ b/doc/todo/Configuring_metadata_view_filenames.mdwn
@@ -0,0 +1 @@
+Currently, in a git annex view, filenames take the form basename%path%.ext. I understand that this is a carefully drafted mapping to allow changes to be merged back into metadata. However, maybe it would be possible to make the separator ('%') and the order (e.g. path%basename.ext instead) configurable?
diff --git a/doc/todo/Configuring_metadata_view_filenames/comment_1_08f8f27e5a8dbd80a91ffd9fd6f64e6c._comment b/doc/todo/Configuring_metadata_view_filenames/comment_1_08f8f27e5a8dbd80a91ffd9fd6f64e6c._comment
new file mode 100644
index 0000000000..b498df6316
--- /dev/null
+++ b/doc/todo/Configuring_metadata_view_filenames/comment_1_08f8f27e5a8dbd80a91ffd9fd6f64e6c._comment
@@ -0,0 +1,23 @@
+[[!comment format=mdwn
+ username="joey"
+ subject="""comment 1"""
+ date="2015-02-04T19:38:58Z"
+ content="""
+git-annex really doesn't care what filenames are used with in a view.
+It only needs to ensure that each file gets a unique filename. Which
+is why the directory is included in the filename, to avoid conflicts
+if 2 files with the same name appear in different directories.
+
+It would probably be better to make it avoid needing to include the
+directory in the filename unless there is such a conflict, rather than
+adding complexity configuring that.
+
+However, since views are currently built by streaming the contents of the
+branch to git update-index, git-annex can't just eg, examine the working
+tree to see if a conflicting file exists. It seems it would need to keep
+a map of the files it's added to the view branch so far, and check against
+the map. But that would make memory use scale with the number of files in
+the view, which I'd prefer to avoid..
+
+I'm going to move this from bugs to todo.
+"""]]
diff --git a/doc/todo/Configuring_metadata_view_filenames/comment_2_34efe1424a9b02dca706565517900bd6._comment b/doc/todo/Configuring_metadata_view_filenames/comment_2_34efe1424a9b02dca706565517900bd6._comment
new file mode 100644
index 0000000000..99429806a2
--- /dev/null
+++ b/doc/todo/Configuring_metadata_view_filenames/comment_2_34efe1424a9b02dca706565517900bd6._comment
@@ -0,0 +1,9 @@
+[[!comment format=mdwn
+ username="Tafnzart"
+ avatar="http://cdn.libravatar.org/avatar/45f62eccc7caece0d1f963f6662ccfde"
+ subject="Maybe provide an option to force without name change"
+ date="2017-05-07T14:05:03Z"
+ content="""
+This name change shouldn't be necessary if on view that has directory structure from master:
+git annex view todo=* \"/=*\"
+"""]]
diff --git a/doc/todo/Configuring_metadata_view_filenames/comment_3_dd7aa7560412d4589d2fc28eb978a71e._comment b/doc/todo/Configuring_metadata_view_filenames/comment_3_dd7aa7560412d4589d2fc28eb978a71e._comment
new file mode 100644
index 0000000000..f8377f8fec
--- /dev/null
+++ b/doc/todo/Configuring_metadata_view_filenames/comment_3_dd7aa7560412d4589d2fc28eb978a71e._comment
@@ -0,0 +1,10 @@
+[[!comment format=mdwn
+ username="CandyAngel"
+ avatar="http://cdn.libravatar.org/avatar/15c0aade8bec5bf004f939dd73cf9ed8"
+ subject="comment 3"
+ date="2017-05-09T10:05:14Z"
+ content="""
+>  But that would make memory use scale with the number of files in the view, which I'd prefer to avoid..
+
+This sounds like another use case for bloom filters :)
+"""]]
diff --git a/doc/todo/Configuring_metadata_view_filenames/comment_4_1873c7cdd142dffc210ec0172bf29997._comment b/doc/todo/Configuring_metadata_view_filenames/comment_4_1873c7cdd142dffc210ec0172bf29997._comment
new file mode 100644
index 0000000000..310cd68bc6
--- /dev/null
+++ b/doc/todo/Configuring_metadata_view_filenames/comment_4_1873c7cdd142dffc210ec0172bf29997._comment
@@ -0,0 +1,14 @@
+[[!comment format=mdwn
+ username="joey"
+ subject="""comment 4"""
+ date="2017-05-09T17:41:33Z"
+ content="""
+True, it could use a bloom filter.
+
+I had not thought of `/=*` (or forgot about it). Views could, as a special
+case, use the original paths in that case. That's getting very close to
+adjusted branch territory, and I want to rewrite the view branch generation
+code to use adjusted branches eventually (so changes made in the view
+branch can be propigated back out to the source branch and so view branches
+can be updated when the source branch changes).
+"""]]
diff --git a/doc/todo/Display_fingerprint_to_WebApps_GPG___34__create_encrypted_new_repo__34__.mdwn b/doc/todo/Display_fingerprint_to_WebApps_GPG___34__create_encrypted_new_repo__34__.mdwn
new file mode 100644
index 0000000000..146b0622b3
--- /dev/null
+++ b/doc/todo/Display_fingerprint_to_WebApps_GPG___34__create_encrypted_new_repo__34__.mdwn
@@ -0,0 +1,14 @@
+ I'm using git-annex 5.20150205-gbf9058a and just used the WebApp to create a new remote SSH repo, and thought I'd try the encrypted option.
+
+It give me three GPG keys to choose from (all valid keys) but only displayed the email addresses which were all identical so I couldn't tell which was which.
+
+I then clicked the first key selection button, hoping it would display more info but it seemed to start doing things immediately. It requested the GPG passphrase which I cancelled but it was still doing things, and worse it wasn't clear what state the repo was in (encrypted or not), so I deleted it and started again (it's fine now).
+
+The passphrase dialog box does display the key fingerprint, but it's then too late to alter the key selection.
+
+Request: Could the WebApp always display the fingerprint after the email address?
+
+Some clarity on what happens when you cancel would be nice too.
+
+Thanks
+Giovanni
diff --git a/doc/todo/Display_the_version_of_a_library_corresponding_to_a_build_flag.mdwn b/doc/todo/Display_the_version_of_a_library_corresponding_to_a_build_flag.mdwn
new file mode 100644
index 0000000000..42dfce2a9a
--- /dev/null
+++ b/doc/todo/Display_the_version_of_a_library_corresponding_to_a_build_flag.mdwn
@@ -0,0 +1,23 @@
+It would be great when diagnosing issues to know the version of a particular library that git-annex is compiled with.
+
+Because there are so many dependencies though, perhaps only the libraries corresponding to a build flag should be displayed, so instead of
+
+    ~ λ git annex version
+    git-annex version: 6.20170321-g4642912
+    build flags: Assistant Webapp Pairing Testsuite S3(multipartupload)(storageclasses) WebDAV Inotify ConcurrentOutput TorrentParser Feeds Quvi
+    key/value backends: SHA256E SHA256 SHA512E SHA512 SHA224E SHA224 SHA384E SHA384 SHA3_256E SHA3_256 SHA3_512E SHA3_512 SHA3_224E SHA3_224 SHA3_384E SHA3_384 SKEIN256E SKEIN256 SKEIN512E SKEIN512 SHA1E SHA1 MD5E MD5 WORM URL
+    remote types: git gcrypt p2p S3 bup directory rsync web bittorrent webdav tahoe glacier ddar hook external
+
+It would show:
+
+    ~ λ git annex version
+    git-annex version: 6.20170321-g4642912
+    build flags: ...etc... TorrentParser-1.2.1 Feeds-2.3.1 Quvi-1.0.0
+    key/value backends: SHA256E SHA256 SHA512E SHA512 SHA224E SHA224 SHA384E SHA384 SHA3_256E SHA3_256 SHA3_512E SHA3_512 SHA3_224E SHA3_224 SHA3_384E SHA3_384 SKEIN256E SKEIN256 SKEIN512E SKEIN512 SHA1E SHA1 MD5E MD5 WORM URL
+    remote types: git gcrypt p2p S3 bup directory rsync web bittorrent webdav tahoe glacier ddar hook external
+
+> Well, I think better not to complicate the build flags list, which the
+> user may want to refer to, with this. Also, there should be a way to
+> indicate versions for libraries that don't have a build flag, when the
+> version is a common question. So, let's add it as a separate line of
+> data. [[done]] --[[Joey]] 
diff --git a/doc/todo/Error_when_using_6.20180913_with_6.20170101-1+deb9u2.mdwn b/doc/todo/Error_when_using_6.20180913_with_6.20170101-1+deb9u2.mdwn
new file mode 100644
index 0000000000..36dccb8454
--- /dev/null
+++ b/doc/todo/Error_when_using_6.20180913_with_6.20170101-1+deb9u2.mdwn
@@ -0,0 +1,10 @@
+[A Debian user reports](https://bugs.debian.org/909023) that 6.20180913 cannot be used with a server running the version of git-annex in Debian stretch.  The error is:
+
+    fd:23: hClose: resource vanished (Broken pipe)
+
+Presumably this sudden lack of backwards incompatibility is unintentional.
+
+--spwhitton
+
+> Indeed it was, and I'll be making a release in the next couple of days
+> because of the breakage. [[done]] --[[Joey]]
diff --git a/doc/todo/Exporting_with_exporttree_should_sync_files_deleted_from_the_remote.mdwn b/doc/todo/Exporting_with_exporttree_should_sync_files_deleted_from_the_remote.mdwn
new file mode 100644
index 0000000000..98ddef9f56
--- /dev/null
+++ b/doc/todo/Exporting_with_exporttree_should_sync_files_deleted_from_the_remote.mdwn
@@ -0,0 +1,3 @@
+If git-tracked files are removed from the remote, they don't get synced over after a "git annex fsck" and "git annex export".
+
+Is there some way that they could make it to the remote? I'm imagining an rsync-like behavior to copy over files that have different time stamps or file sizes. Would such a feature be welcome in git annex?
diff --git a/doc/todo/Facilitate_public_pretty_S3_URLs.mdwn b/doc/todo/Facilitate_public_pretty_S3_URLs.mdwn
new file mode 100644
index 0000000000..69d09e3fa2
--- /dev/null
+++ b/doc/todo/Facilitate_public_pretty_S3_URLs.mdwn
@@ -0,0 +1,23 @@
+I archive all my photos/video to a bucket CNAMED to http://s.natalian.org/ with a simple YYYY-MM-DD prefix.
+
+E.g. 
+
+I'm not doing a great job of backing up the S3 bucket to another S3 compatible host, since `s3cmd sync`/`aws sync` is so slow, but that's beside the point. Ideally it could be tracked by **git-annex**!
+
+Adding all the objects into git-annex, IIUC currently would require me:
+
+* to download the ~80GB and then add them to git-annex
+* there is no way to keep my current S3 URLs with the [[special_remotes/S3]] since `git-annex` has it's own special way of storing to a bucket, e.g. https://s3-ap-southeast-1.amazonaws.com/s3-10418340-834d-41c2-b38f-7ee84bf6a23a/SHA256E-s1034208123--235e4f288d094c2e1870bc3d9d353abf34542c04c1d26905e882718a7ccf74cf.mp4 - I'd rather not have HTTP redirects
+* AFAICT there is no way currently with git-annex to mark the [[special_remotes/S3]] as public, which is needed for public URLs to work
+* AFAICT there is no current automated method the mapping via `git-annex addurl` with the public URLs of the each file in the bucket
+
+The ideal solution in my mind is for git-annex to track the contents of S3 as they are now, preserving the URLs and tracking the checksums in a separate index file.
+
+Thank you!
+
+> I don't think this is something git-annex can usefully do.
+> Instead, see
+> . --[[Joey]]
+
+> [[done]]; the new `git-annex export` feature allows you to export a tree
+> to a special remote. --[[Joey]]
diff --git a/doc/todo/Facilitate_public_pretty_S3_URLs/comment_1_c803bfef1b881c6f64ed44d49dfb4547._comment b/doc/todo/Facilitate_public_pretty_S3_URLs/comment_1_c803bfef1b881c6f64ed44d49dfb4547._comment
new file mode 100644
index 0000000000..3eb005b945
--- /dev/null
+++ b/doc/todo/Facilitate_public_pretty_S3_URLs/comment_1_c803bfef1b881c6f64ed44d49dfb4547._comment
@@ -0,0 +1,7 @@
+[[!comment format=mdwn
+ username="https://id.koumbit.net/anarcat"
+ subject="similar to a forum question"
+ date="2015-03-07T02:05:26Z"
+ content="""
+this is similar to a forum question i asked: [[forum/original_filename_on_s3/]]. --[[anarcat]]
+"""]]
diff --git a/doc/todo/Faster___171__git_annex_status__187___when___171__git_annex_watch__187___is_running.mdwn b/doc/todo/Faster___171__git_annex_status__187___when___171__git_annex_watch__187___is_running.mdwn
new file mode 100644
index 0000000000..7f7a482b9c
--- /dev/null
+++ b/doc/todo/Faster___171__git_annex_status__187___when___171__git_annex_watch__187___is_running.mdwn
@@ -0,0 +1,5 @@
+I’m just experimenting with git-annex on a repo of ~150k files in about the same number of directories (WORM-backend). Calling, e.g., «git annex status» will take several minutes while stat()-ing all the files (for changes, I presume).
+
+This might already be on you todo list, but I was wondering whether it is possible to increase the performance of «annex status» (or related commands) when «annex watch» is running in the background. In that case, «status» could rely on cached data built-up at some point during initialization, plus based on the data that was accumulated via inotify. (Hopefully, all this won’t even be needed anymore on btrfs at some point in the future.)
+
+(I’m not very knowledgable in these things, so just out of curiosity: I noticed that, even though the «status» invocation takes ages, no HDD activity occurs and all the metadata is probably already in the Linux caches from a run I conducted immediately beforehand. Why do you figure that is? Is context switching so hugely expensive?)
diff --git a/doc/todo/Faster___171__git_annex_status__187___when___171__git_annex_watch__187___is_running/comment_1_e5f2630591ffa7758ca4250a061a8589._comment b/doc/todo/Faster___171__git_annex_status__187___when___171__git_annex_watch__187___is_running/comment_1_e5f2630591ffa7758ca4250a061a8589._comment
new file mode 100644
index 0000000000..446562f858
--- /dev/null
+++ b/doc/todo/Faster___171__git_annex_status__187___when___171__git_annex_watch__187___is_running/comment_1_e5f2630591ffa7758ca4250a061a8589._comment
@@ -0,0 +1,12 @@
+[[!comment format=mdwn
+ username="http://joeyh.name/"
+ ip="209.250.56.2"
+ subject="comment 1"
+ date="2014-07-10T19:37:38Z"
+ content="""
+Neither `git annex watch` nor the `assistant` persistently store any data about the files in the repository in memory. They cannot speed up `git annex status`.
+
+I'm not sure what's the point of running `git annex status` while the daemon is running, since the daemon will normally immediately notice changes and commit them. The status output should thus generally be empty.
+
+FWIW, `git annex status` takes 0.3s in my largest repo (55k files), on an SSD. That's in indirect mode, and the time is almost completely spent in running `git ls-files --modified`, which is probably about as fast as it can be. In a direct mode repository, it will be rather slower, since it has to query git for the key that was last committed for each file, in order to look up that key's info and see if the file has been modified. 
+"""]]
diff --git a/doc/todo/Faster___171__git_annex_status__187___when___171__git_annex_watch__187___is_running/comment_2_f8a5cc905d5b06bdbb1a778ab866a28c._comment b/doc/todo/Faster___171__git_annex_status__187___when___171__git_annex_watch__187___is_running/comment_2_f8a5cc905d5b06bdbb1a778ab866a28c._comment
new file mode 100644
index 0000000000..315569ca15
--- /dev/null
+++ b/doc/todo/Faster___171__git_annex_status__187___when___171__git_annex_watch__187___is_running/comment_2_f8a5cc905d5b06bdbb1a778ab866a28c._comment
@@ -0,0 +1,44 @@
+[[!comment format=mdwn
+ username="zardoz"
+ ip="134.147.14.84"
+ subject="comment 2"
+ date="2014-07-11T11:46:08Z"
+ content="""
+Yes, you’re probably right that the benefit of this is slim when the
+watching daemon auto-adds new files. So the «status» output would
+never change and show the status that upheld before starting the
+daemon.
+
+The reason I brought this up that I recall reading a comment of yours
+somewhere on the site, to the effect that the assistant can sometimes
+speed up certain operations, because it can make certain valid
+assumptions on the state of the repo due to the fact that the
+assistant has been monitoring it. I don’t recall what those operations
+were, though. That’s why it occurred to me whether there might be a
+daemon that just monitors via inotify, and neither adds nor syncs, and
+only provides information to certain commands to speed things up under
+some circumstances.
+
+In general, is it accurate to say that git-annex mostly takes the
+«space» option when making a space/time-trade-offs? I noticed that the
+memory consumption is really slim most of the time, and wondered
+whether there might be options of speeding operations up by relying on
+more memory instead (perhaps also doing persistent caching). On the
+other hand, in some regards you are probably committed to the
+time/memory trade-offs taken by vanilla git, so maybe there’s not much
+room for improvement, git-annex wise…
+
+Maybe direct-mode repos on the order of 100k’s of files are just not
+practical. I’m using indirect mode for my really big repos now, and
+it’s now responsive enough to use (except for «annex unused», which is
+inherently expensive, as you once explained). At least commiting won’t
+take tens of minutes that way. I’ll just have to make the software
+play nicely with the symlinks.
+
+BTW, the file-system seems to have a huge impact on this. My large
+direct mode annex is practically unusable on ext (tens of minutes per
+commit), but still usable on btrfs (a few minutes). I’m migrating one
+disk to btrfs at home and will do some controlled benchmarks then. The
+added bonus is that directories don’t always take up a full block.
+
+"""]]
diff --git a/doc/todo/Faster___171__git_annex_status__187___when___171__git_annex_watch__187___is_running/comment_3_626c629654508d0d948ece849d43ed86._comment b/doc/todo/Faster___171__git_annex_status__187___when___171__git_annex_watch__187___is_running/comment_3_626c629654508d0d948ece849d43ed86._comment
new file mode 100644
index 0000000000..58cf33c105
--- /dev/null
+++ b/doc/todo/Faster___171__git_annex_status__187___when___171__git_annex_watch__187___is_running/comment_3_626c629654508d0d948ece849d43ed86._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="zardoz"
+ ip="134.147.14.84"
+ subject="comment 3"
+ date="2014-07-11T11:54:26Z"
+ content="""
+Ah, interesting, some caching ideas are already discussed at [https://git-annex.branchable.com/design/caching_database/].
+"""]]
diff --git a/doc/todo/Finding_content_in_other_repositories_should_not_need_a_git_remote.mdwn b/doc/todo/Finding_content_in_other_repositories_should_not_need_a_git_remote.mdwn
new file mode 100644
index 0000000000..3025ae7351
--- /dev/null
+++ b/doc/todo/Finding_content_in_other_repositories_should_not_need_a_git_remote.mdwn
@@ -0,0 +1,14 @@
+git annex unused supports an --from option to select a specific remote to find unused data in. When you use that on a repository which is not linked to from the current repository by a git remote, git annex fails:
+
+    $ git annex unused --from somerepository
+    git-annex: there is no available git remote named somerepository
+    $ git annex unused --from repowithremote
+    unused repowithremote (checking for unused data...) 
+    [...]
+
+As git annex at least for newly-deleted files knows when those are stored in the given repository, git annex should at least find those.
+
+> [[done]], made it suppose the uuid or description.
+> 
+> Of course, `dropunused --from` still needs the name of a remote, so this
+> is only useful for querying.. --[[Joey]]
diff --git a/doc/todo/INFO_message_for_custom_special_remotes.mdwn b/doc/todo/INFO_message_for_custom_special_remotes.mdwn
new file mode 100644
index 0000000000..b2ed382580
--- /dev/null
+++ b/doc/todo/INFO_message_for_custom_special_remotes.mdwn
@@ -0,0 +1,5 @@
+I wondered if it would be sensible to ask to extend [externals special remote protocol](https://git-annex.branchable.com/design/external_special_remote_protocol/) with ability for custom remotes to pass back some INFO level message (not only DEBUG or ERROR).  The reason is:  in datalad-archives special remote we usually need to `git annex get` first the key containing the archive, which might be sizeable.  Since there is ATM no way to communicate back to git-annex, so it could communicate back to the datalad which runs it, it results in no output/message to the user that possibly a heavy download is happening in the background.  So, we would need to establish our own communication from datalad-archives special remote all the way to top level datalad process to report that, or I wondered if may be we could report back to git-annex, and it in turn report back to the original process (running e.g. `annex get --json --json-progress`) so it could spit out that message wrapped into a json record within the stream, so we could process and output that to the user.
+
+[[!meta author=yoh]]
+
+> [[done]] --[[Joey]]
diff --git a/doc/todo/INFO_message_for_custom_special_remotes/comment_1_ea99a5099f78767859c05aeb5217a12d._comment b/doc/todo/INFO_message_for_custom_special_remotes/comment_1_ea99a5099f78767859c05aeb5217a12d._comment
new file mode 100644
index 0000000000..c2e9f318e9
--- /dev/null
+++ b/doc/todo/INFO_message_for_custom_special_remotes/comment_1_ea99a5099f78767859c05aeb5217a12d._comment
@@ -0,0 +1,11 @@
+[[!comment format=mdwn
+ username="joey"
+ subject="""comment 1"""
+ date="2018-02-06T16:58:04Z"
+ content="""
+I've added it. However, note that previous versions of git-annex will
+not react well to an unknown message being sent, so to use it safely you
+will need to detect a new enough version of git-annex. (I've had a todo item
+on the protocol for a while to have a way to detect what messages git-annex
+understands.)
+"""]]
diff --git a/doc/todo/INFO_message_for_custom_special_remotes/comment_2_69535d7d9b7266eaac6f9bb70d2660dc._comment b/doc/todo/INFO_message_for_custom_special_remotes/comment_2_69535d7d9b7266eaac6f9bb70d2660dc._comment
new file mode 100644
index 0000000000..5714e78369
--- /dev/null
+++ b/doc/todo/INFO_message_for_custom_special_remotes/comment_2_69535d7d9b7266eaac6f9bb70d2660dc._comment
@@ -0,0 +1,10 @@
+[[!comment format=mdwn
+ username="yarikoptic"
+ avatar="http://cdn.libravatar.org/avatar/f11e9c84cb18d26a1748c33b48c924b4"
+ subject="comment 2"
+ date="2018-02-06T20:15:35Z"
+ content="""
+Great!  I will give it a shot.  No problem checking version of git-annex available so I will make it conditional on git-annex version.
+
+As for \"todo on protocol...\" -- FWIW I left some comments [close to original discussion](http://git-annex.branchable.com/design/exporting_trees_to_special_remotes/#comment-b3876cc7b791c6b8cb12949bdbbf8268)
+"""]]
diff --git a/doc/todo/Invert_remote_selection.mdwn b/doc/todo/Invert_remote_selection.mdwn
new file mode 100644
index 0000000000..a091569f27
--- /dev/null
+++ b/doc/todo/Invert_remote_selection.mdwn
@@ -0,0 +1,30 @@
+Say I have:
+
+    $> git remote
+    Alpha
+    Beta
+    Gamma
+    Delta
+
+It is easy to sync with all repos via:
+
+    $> git annex sync
+
+Or specific repos via:
+
+    $> git annex sync Alpha Beta
+
+However, it is currently awkward to exclude specific repos. Is it possible to 'invert' or 'negate' specific remotes, so that the following are equivalent?
+
+    $> git annex sync \! Gamma
+    $> git annex sync Alpha Beta Delta
+
+This problem comes up surprisingly often due to:
+
+  1. An external host being temporarily down (which causes sync to hang for a while),
+  2. Repos being available, but the machine is under heavy IO load or memory pressure,
+  3. Repos on external drives that are swapped out and mounted to a specific location (e.g., /mnt/),
+  4. Wanting to roll out repo-wide changes in stages, or keeping 1-2 repos untouched for whatever reason, or
+  5. Some repos being too large for a machine (e.g., repacking fails due to low memory), but which can still act like a dumb file-store.
+
+The problem gets worse when you have a lot of remotes or a lot of repos to manage (I have both). My impression is that this feature would require a syntax addition for git-annex-sync only. I like '!' because it behaves the same in GNU find and sh.
diff --git a/doc/todo/Invert_remote_selection/comment_1_5aa12a96f54a40adbfe18267a6bde87b._comment b/doc/todo/Invert_remote_selection/comment_1_5aa12a96f54a40adbfe18267a6bde87b._comment
new file mode 100644
index 0000000000..3742da867f
--- /dev/null
+++ b/doc/todo/Invert_remote_selection/comment_1_5aa12a96f54a40adbfe18267a6bde87b._comment
@@ -0,0 +1,9 @@
+[[!comment format=mdwn
+ username="joey"
+ subject="""comment 1"""
+ date="2017-08-28T17:45:11Z"
+ content="""
+I think you could just use this:
+	
+	git annex sync -c remote.Gamma.annex-sync=false
+"""]]
diff --git a/doc/todo/Invert_remote_selection/comment_2_9ad4c9b2217f739e67198d16d14d32e7._comment b/doc/todo/Invert_remote_selection/comment_2_9ad4c9b2217f739e67198d16d14d32e7._comment
new file mode 100644
index 0000000000..5acf39f956
--- /dev/null
+++ b/doc/todo/Invert_remote_selection/comment_2_9ad4c9b2217f739e67198d16d14d32e7._comment
@@ -0,0 +1,12 @@
+[[!comment format=mdwn
+ username="supernaught"
+ avatar="http://cdn.libravatar.org/avatar/55f92a50f2617099e2dc7509130ce158"
+ subject="comment 2"
+ date="2017-08-28T22:01:23Z"
+ content="""
+It's not very ergonomic to type out so much each for each sync, but I suppose it technically accomplishes the idea.
+
+Still -- wouldn't making '\!x' alias to '-c remote.x.annex-sync=false' have minimal impact and provide a bit more symmetry with the matching-options?
+
+I'm not familiar with Haskell, but could probably fumble my way through this one. Would you accept a patch?
+"""]]
diff --git a/doc/todo/Invert_remote_selection/comment_3_2aec5f535c84c37d59802e4759a1c242._comment b/doc/todo/Invert_remote_selection/comment_3_2aec5f535c84c37d59802e4759a1c242._comment
new file mode 100644
index 0000000000..549beaaa78
--- /dev/null
+++ b/doc/todo/Invert_remote_selection/comment_3_2aec5f535c84c37d59802e4759a1c242._comment
@@ -0,0 +1,7 @@
+[[!comment format=mdwn
+ username="joey"
+ subject="""comment 3"""
+ date="2017-09-29T16:26:54Z"
+ content="""
+Sure, if you want to develop a patch..
+"""]]
diff --git a/doc/todo/LIst_of_Available_Remotes_in_Webapp.mdwn b/doc/todo/LIst_of_Available_Remotes_in_Webapp.mdwn
new file mode 100644
index 0000000000..89274bb8dd
--- /dev/null
+++ b/doc/todo/LIst_of_Available_Remotes_in_Webapp.mdwn
@@ -0,0 +1 @@
+When using git-annex in a distributed fashion (lots of repos everywhere) It is easy to lose track of which remotes has a particular repo and enable it. Currently I have to run `git annex info` and see which remotes are available then add them through the webapp. Would it be possible to make webapp show all repos not just the ones it is syncing give an option to enable it.
diff --git a/doc/todo/LIst_of_Available_Remotes_in_Webapp/comment_1_23fe2f3cd44c4357a385452dcd5eedef._comment b/doc/todo/LIst_of_Available_Remotes_in_Webapp/comment_1_23fe2f3cd44c4357a385452dcd5eedef._comment
new file mode 100644
index 0000000000..9e22dafca3
--- /dev/null
+++ b/doc/todo/LIst_of_Available_Remotes_in_Webapp/comment_1_23fe2f3cd44c4357a385452dcd5eedef._comment
@@ -0,0 +1,10 @@
+[[!comment format=mdwn
+ username="http://joeyh.name/"
+ ip="209.250.56.191"
+ subject="comment 1"
+ date="2014-04-17T20:54:18Z"
+ content="""
+I chose to make the webapp only show available remotes when it knows how to enable them. So it will show S3 special remotes, Box.com special remotes, etc. This avoids cluttering up the display of a shared repository with a list of all of your friend's removable drives, for example.
+
+I would like to make the webapp smarter about handling repositories on remote ssh servers. As long as the server name is in the global DNS, the webapp could easily walk the user through setting up such a remote. The missing piece is that nothing is logged in remotes.log for these remotes, and so the assistant doesn't know the server name.
+"""]]
diff --git a/doc/todo/Long_Running_Filter_Process.mdwn b/doc/todo/Long_Running_Filter_Process.mdwn
new file mode 100644
index 0000000000..329abaf453
--- /dev/null
+++ b/doc/todo/Long_Running_Filter_Process.mdwn
@@ -0,0 +1,22 @@
+Hello,
+
+while reading the release notes of git 2.11 I noticed a cool new feature has been merged:
+
+> If the filter command (a string value) is defined via
+> `filter..process` then Git can process all blobs with a
+> single filter invocation for the entire life of a single Git
+> command.
+
+see the [git documentation][1].
+
+This has been developed in the context of git-lfs (see [PR 1382] [2]).
+
+If I understand correctly how it works this could speed up v6 repos. Looking at the history/website 
+of git-annex there doesn't seem to be yet any work on this so I though it was worth calling the 
+attention on the feature.
+
+Thanks a lot for all the work on git-annex, it's a really amazing project! The more I study it the more cool features I discover :)
+
+
+[1]: https://github.com/git/git/blob/v2.11.0/Documentation/gitattributes.txt#L384
+[2]: https://github.com/git-lfs/git-lfs/pull/1382
diff --git a/doc/todo/Long_Running_Filter_Process/comment_1_f155ffc7dbd074964dd53165274ec8a0._comment b/doc/todo/Long_Running_Filter_Process/comment_1_f155ffc7dbd074964dd53165274ec8a0._comment
new file mode 100644
index 0000000000..34d05d7711
--- /dev/null
+++ b/doc/todo/Long_Running_Filter_Process/comment_1_f155ffc7dbd074964dd53165274ec8a0._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="joey"
+ subject="""comment 1"""
+ date="2016-12-13T15:57:05Z"
+ content="""
+Yes, this will make [[smudge]] faster when eg checking out a lot of working
+tree changes. I will need to add support for it.
+"""]]
diff --git a/doc/todo/Long_Running_Filter_Process/comment_3_24d89d0e8eb2da6e43d107caa71e042b._comment b/doc/todo/Long_Running_Filter_Process/comment_3_24d89d0e8eb2da6e43d107caa71e042b._comment
new file mode 100644
index 0000000000..f37fdf21f3
--- /dev/null
+++ b/doc/todo/Long_Running_Filter_Process/comment_3_24d89d0e8eb2da6e43d107caa71e042b._comment
@@ -0,0 +1,16 @@
+[[!comment format=mdwn
+ username="joey"
+ subject="""comment 3"""
+ date="2018-08-13T20:24:02Z"
+ content="""
+The "filterdriver" branch implements support for these.
+
+However, it's actually slower than the old interface, because the new
+interface requires git-annex read the whole file content from git when
+adding a file, and the old interface let it not read any content.
+
+Since the new interface does have capabilities, a new capability could
+prevent schepping the content over the pipe, and let the filter driver
+refer to the worktree file instead, and respond with the path of a file.
+This would be similar to my old patch set for the old interface.
+"""]]
diff --git a/doc/todo/Lower-case_extension_for_SHA256E_and_similar.mdwn b/doc/todo/Lower-case_extension_for_SHA256E_and_similar.mdwn
new file mode 100644
index 0000000000..fccd2510c0
--- /dev/null
+++ b/doc/todo/Lower-case_extension_for_SHA256E_and_similar.mdwn
@@ -0,0 +1,19 @@
+Please consider making the `*E` backends lower-case the file extensions. This improves deduplication especially with image files, where cameras often use `*.JPG`, while other software such as the Dropbox Android app rename the images to `*.jpg`.
+We talked about that very shortly a few months ago:
+
+(Log starts 2015-08-06 21:11:40 CET)
+
+     tribut  is there a backend that lowercases the extension? or do the *E backends do that anyway?
+     joeyh   it does not
+     tribut  but would it make sense? or am i missing something?
+     joeyh   I don't know.. the extension is only there for stupid programs that follow symlinks and
+             check extensions. If such a program cares about .GIF vss .gif, you might have a problem
+     joeyh   I think that you can git-annex migrate from hashE to hash, then migrate back, and it'll
+             update to the new file extension.
+     tribut  i was thinking about content-identical images with .JPG or .jpg extension
+     tribut  and because even the most retarded of programs wont care, i thought the backend could
+             lowercase the extension
+     joeyh   ah, sure, using the E backend reduces the ability to de-duplicate
+     joeyh   I'd not want to add a e backend set just for this. There's no requirement that the
+             extension extraction code be stable, so it could be considered changing it to lower-case
+     joeyh   otoh, I have no idea if some programs are dumb enough to care about .git vs .GIF
diff --git a/doc/todo/Lower-case_extension_for_SHA256E_and_similar/comment_1_71a32962c081e42be0bfc468f74230e8._comment b/doc/todo/Lower-case_extension_for_SHA256E_and_similar/comment_1_71a32962c081e42be0bfc468f74230e8._comment
new file mode 100644
index 0000000000..29cd817c63
--- /dev/null
+++ b/doc/todo/Lower-case_extension_for_SHA256E_and_similar/comment_1_71a32962c081e42be0bfc468f74230e8._comment
@@ -0,0 +1,13 @@
+[[!comment format=mdwn
+ username="CandyAngel"
+ subject="comment 1"
+ date="2016-01-18T13:16:59Z"
+ content="""
+Joey implemented the '--force' option in *git annex migrate* at my request to deal with this issue.
+
+* Rename symlinks to lowercase extension
+* git annex migrate --force (causes the keys to be recreated using the new, lowercase extension)
+* git annex unused (for removing the now unused upper/mixed case files in the store).
+
+It's not the smoothest method, but it works well enough :)
+"""]]
diff --git a/doc/todo/Lower-case_extension_for_SHA256E_and_similar/comment_2_db432973421d3974e027cef13caa3033._comment b/doc/todo/Lower-case_extension_for_SHA256E_and_similar/comment_2_db432973421d3974e027cef13caa3033._comment
new file mode 100644
index 0000000000..2fc61b5149
--- /dev/null
+++ b/doc/todo/Lower-case_extension_for_SHA256E_and_similar/comment_2_db432973421d3974e027cef13caa3033._comment
@@ -0,0 +1,9 @@
+[[!comment format=mdwn
+ username="tribut"
+ subject="comment 2"
+ date="2016-01-18T16:22:54Z"
+ content="""
+Thanks for the hint, CandyAngel.
+
+This is useful, however it has to be done periodically. I would still prefer if those keys were never created in the first place.
+"""]]
diff --git a/doc/todo/Make_default_annex_version_match___40__clone_or_sync__41___parent.mdwn b/doc/todo/Make_default_annex_version_match___40__clone_or_sync__41___parent.mdwn
new file mode 100644
index 0000000000..723efdea14
--- /dev/null
+++ b/doc/todo/Make_default_annex_version_match___40__clone_or_sync__41___parent.mdwn
@@ -0,0 +1,10 @@
+by default, git-annex will set repo version to v5  
+--even if I clone a v6 repo  
+--or sync from a v6 repo  
+----(if I sync and there is a git remote in the network which hasn't been git annex initialized yet, it become a v5 repo in its config file)  
+
+so I must always upgrade after creating a new repo, or else specify v6 in the creation command.
+
+Given the potential negative effects of mixed version networks, it is probably best to follow the version of the repo cloned, unless the version is specified in the command.
+
+> [[closing|done]] based on my comment --[[Joey]]
diff --git a/doc/todo/Make_default_annex_version_match___40__clone_or_sync__41___parent/comment_1_ce82ccf68108d47977e32ba005098415._comment b/doc/todo/Make_default_annex_version_match___40__clone_or_sync__41___parent/comment_1_ce82ccf68108d47977e32ba005098415._comment
new file mode 100644
index 0000000000..007a1161cf
--- /dev/null
+++ b/doc/todo/Make_default_annex_version_match___40__clone_or_sync__41___parent/comment_1_ce82ccf68108d47977e32ba005098415._comment
@@ -0,0 +1,17 @@
+[[!comment format=mdwn
+ username="joey"
+ subject="""comment 1"""
+ date="2017-06-06T17:45:40Z"
+ content="""
+I don't feel that the benefits of doing this would be worth the added
+complexity. v6 repositories are still an experimental feature, and just
+because one user is using them does not mean another user necessarily
+will want to. One user might be using a v6 repository with `git annex
+adjust --unlock`, and if so it will remain compatible with v5 clones.
+Only if unlocked files are being committed to a v6 repository is there
+a backwards compatability problem.
+
+And, when there's a central git repository, one clone might be using v6
+mode but clonding from the central repo would still produce a v5 repo,
+so making this change wouldn't help at all with a common case.
+"""]]
diff --git a/doc/todo/Metadata_changes_are_not_reflected_in_a_view.mdwn b/doc/todo/Metadata_changes_are_not_reflected_in_a_view.mdwn
new file mode 100644
index 0000000000..cf8e1063cc
--- /dev/null
+++ b/doc/todo/Metadata_changes_are_not_reflected_in_a_view.mdwn
@@ -0,0 +1,31 @@
+### Please describe the problem.
+Changing metadata while being in an active view will not update the view.
+
+### What steps will reproduce the problem?
+(inside a repository)
+
+1. Create a file
+
+        $ uuidgen >file
+
+2. switch into a view
+
+       $ git annex view !blah
+       $ ls
+       file
+
+3. changed the metadata the view is based upon
+
+       $ git annex metadata -t blah file
+       $ ls
+       file
+
+   It would be nice/expected that the view gets updated when the metadata changes, hiding 'file' now
+  
+
+### What version of git-annex are you using? On what operating system?
+
+git-annex version: 5.20141024~bpo70+1
+on debian wheezy with backports
+
+### Please provide any additional information below.
diff --git a/doc/todo/Metadata_changes_are_not_reflected_in_a_view/comment_1_5c3a0d8ec7b1da69b0d81aa4acc0c75b._comment b/doc/todo/Metadata_changes_are_not_reflected_in_a_view/comment_1_5c3a0d8ec7b1da69b0d81aa4acc0c75b._comment
new file mode 100644
index 0000000000..9b079791b1
--- /dev/null
+++ b/doc/todo/Metadata_changes_are_not_reflected_in_a_view/comment_1_5c3a0d8ec7b1da69b0d81aa4acc0c75b._comment
@@ -0,0 +1,18 @@
+[[!comment format=mdwn
+ username="joey"
+ subject="""comment 1"""
+ date="2015-07-20T17:45:13Z"
+ content="""
+This might be worth adding. But, one of the ways views can be used
+is to use file operations to adjust the metadata that is being viewed.
+
+So, when you delete a file from a view, and git commit, git-annex
+will remove the metadata that had made the file be in the view.
+
+And, if you copy or move a file in a view to a different subdirectory,
+and add and git commit the change, the metadata will be updated to
+update the metadata to reflect the files new location.
+
+As such, I see this as a wishlist todo item at best, and will move it from
+the bugs list to the todo list.
+"""]]
diff --git a/doc/todo/Metadata_on_regular_git_objects___40__blob__44___trees__41____63__.mdwn b/doc/todo/Metadata_on_regular_git_objects___40__blob__44___trees__41____63__.mdwn
new file mode 100644
index 0000000000..b3793501a7
--- /dev/null
+++ b/doc/todo/Metadata_on_regular_git_objects___40__blob__44___trees__41____63__.mdwn
@@ -0,0 +1 @@
+Is it technically possible for the metadata interface to also support regular git objects, so that committed files or directories were also subject to tagging? For most metadata-related intents and purposes it seems I’d like to treat annexed and regular files alike; so I’d like to tag the LaTeX/ReST/etc. source for a presentation the same way I treat the output PDF, only that the former are likely not annexed. In the design docs it says metadata is attached to annex keys. Could it equally be attached to git blobs and trees, or would that break horribly somewhere?
diff --git a/doc/todo/Metadata_on_regular_git_objects___40__blob__44___trees__41____63__/comment_1_30c513b68101442dbcdec0eb109203fe._comment b/doc/todo/Metadata_on_regular_git_objects___40__blob__44___trees__41____63__/comment_1_30c513b68101442dbcdec0eb109203fe._comment
new file mode 100644
index 0000000000..9990e18678
--- /dev/null
+++ b/doc/todo/Metadata_on_regular_git_objects___40__blob__44___trees__41____63__/comment_1_30c513b68101442dbcdec0eb109203fe._comment
@@ -0,0 +1,35 @@
+[[!comment format=mdwn
+ username="joey"
+ subject="""comment 1"""
+ date="2015-06-02T17:08:32Z"
+ content="""
+metadata can be attached to sha1 git-annex keys, which can correspond
+to the sha1's of git objects.
+
+	joey@darkstar:~/tmp/a>git annex metadata -t wow --key=SHA1--b4432e7473849adb8d408696c81355bc708a1bd0
+	metadata SHA1--b4432e7473849adb8d408696c81355bc708a1bd0 
+	  lastchanged=2015-06-02@17-10-37
+	  tag=wow
+	  tag-lastchanged=2015-06-02@17-10-37
+	ok
+	joey@darkstar:~/tmp/a>git annex metadata --key SHA1--$(git show-ref master -s)
+	metadata SHA1--b4432e7473849adb8d408696c81355bc708a1bd0 
+	  lastchanged=2015-06-02@17-10-37
+	  tag=wow
+	  tag-lastchanged=2015-06-02@17-10-37
+	ok
+
+Might be worth improving this interface. A `--sha1=` would be a minor
+improvement.
+
+It would certianly be possible for git-annex metadata to be taught that
+if the file is not annexed, and is in git, to look up the sha1 of its
+blob object, and attach the metadata to it.
+
+Of course, modifying a file would not carry over the metadata. But the 
+same problem exists when modifying annexed files that have metadata attached.
+
+This would also need changes to view generation, I guess that when building
+a view it would need to consider regular git-tracked files, look up 
+their metadata, and add matching ones to the view tree.
+"""]]
diff --git a/doc/todo/Move_ssh_config_to___126____47__ssh__47__git-annex__47__config.mdwn b/doc/todo/Move_ssh_config_to___126____47__ssh__47__git-annex__47__config.mdwn
new file mode 100644
index 0000000000..419b6795e4
--- /dev/null
+++ b/doc/todo/Move_ssh_config_to___126____47__ssh__47__git-annex__47__config.mdwn
@@ -0,0 +1,9 @@
+### Please describe the problem.
+Instead of storing config for each remote in ~/.ssh/config, which mixes the user own config with that of git-annex-assistant, which is irritating if (like me) you store your ssh config in a vcs. Since the option -F allows the choice of the config file, it should be possible to move the config into ~/.ssh/git-annex/config. The only issue I see is according to the ssh man page on my system states that the system-wide config is ignored if a config file is specified on the command line.
+
+### What version of git-annex are you using? On what operating system?
+I'm using git-annex 4.20130601 on a Debian Testing/Unstable/Experimental mix.
+
+[[!tag design/assistant]]
+
+[[wontfix]]
diff --git a/doc/todo/Move_ssh_config_to___126____47__ssh__47__git-annex__47__config/comment_1_284c806e83a32af81b02aea7c7bc285a._comment b/doc/todo/Move_ssh_config_to___126____47__ssh__47__git-annex__47__config/comment_1_284c806e83a32af81b02aea7c7bc285a._comment
new file mode 100644
index 0000000000..5997664e0d
--- /dev/null
+++ b/doc/todo/Move_ssh_config_to___126____47__ssh__47__git-annex__47__config/comment_1_284c806e83a32af81b02aea7c7bc285a._comment
@@ -0,0 +1,10 @@
+[[!comment format=mdwn
+ username="http://joeyh.name/"
+ nickname="joey"
+ subject="comment 1"
+ date="2013-06-11T14:44:47Z"
+ content="""
+The only interface git provides to do this is `GIT_SSH`, which would have to be set to a wrapper script that runs ssh with the desirned options.
+
+And if that were used, `git pull` by itself would not work on the repositories set up by the assistant. I don't consider that very nice.
+"""]]
diff --git a/doc/todo/Move_ssh_config_to___126____47__ssh__47__git-annex__47__config/comment_2_1f55ad6b39906458779b2d604b003ffe._comment b/doc/todo/Move_ssh_config_to___126____47__ssh__47__git-annex__47__config/comment_2_1f55ad6b39906458779b2d604b003ffe._comment
new file mode 100644
index 0000000000..3cf75df08d
--- /dev/null
+++ b/doc/todo/Move_ssh_config_to___126____47__ssh__47__git-annex__47__config/comment_2_1f55ad6b39906458779b2d604b003ffe._comment
@@ -0,0 +1,10 @@
+[[!comment format=mdwn
+ username="http://joeyh.name/"
+ nickname="joey"
+ subject="comment 2"
+ date="2013-06-11T14:48:01Z"
+ content="""
+Also, if you're going to set up something like local pairing, why would you *not* want to commit that config to git along with your other ssh configs? Config files in $HOME are quite frequently edited by helper programs to configure changes, and I personally commit those changes all the time.
+
+Perhaps your real problem is that you have one `.ssh/config` that is shared between multiple hosts, and the git-annex settings are specific to a single host. Have you considered using [vcsh](https://github.com/RichiH/vcsh)?
+"""]]
diff --git a/doc/todo/Move_ssh_config_to___126____47__ssh__47__git-annex__47__config/comment_3_b00dce2374aac6968317d05d23bcfaf7._comment b/doc/todo/Move_ssh_config_to___126____47__ssh__47__git-annex__47__config/comment_3_b00dce2374aac6968317d05d23bcfaf7._comment
new file mode 100644
index 0000000000..3442fb2b2b
--- /dev/null
+++ b/doc/todo/Move_ssh_config_to___126____47__ssh__47__git-annex__47__config/comment_3_b00dce2374aac6968317d05d23bcfaf7._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="https://www.google.com/accounts/o8/id?id=AItOawk3Wgg0XiqYFwM_Pw1RxZwlpNFi65g17sM"
+ nickname="James"
+ subject="comment 3"
+ date="2013-06-12T01:12:24Z"
+ content="""
+Ah, ok, I presumed there was an option in git to set a per-repository ssh command. I've looked at vcsh, but I'm not that confident with git remotes, so I don't use it (I use hg). If a per-repository ssh command added to git, would you consider adding this?
+"""]]
diff --git a/doc/todo/Move_ssh_config_to___126____47__ssh__47__git-annex__47__config/comment_4_743d0b077110c5cac1e2f47187b75333._comment b/doc/todo/Move_ssh_config_to___126____47__ssh__47__git-annex__47__config/comment_4_743d0b077110c5cac1e2f47187b75333._comment
new file mode 100644
index 0000000000..5a22c98f72
--- /dev/null
+++ b/doc/todo/Move_ssh_config_to___126____47__ssh__47__git-annex__47__config/comment_4_743d0b077110c5cac1e2f47187b75333._comment
@@ -0,0 +1,10 @@
+[[!comment format=mdwn
+ username="http://joeyh.name/"
+ nickname="joey"
+ subject="comment 4"
+ date="2013-06-12T19:23:50Z"
+ content="""
+If it were sane, I'd probably use it.
+
+In the meantime, I'm moving this to [[todo]].
+"""]]
diff --git a/doc/todo/Natively_support_s3__58____47____47___urls___40__for_addurl__44___get__44___etc__41__.mdwn b/doc/todo/Natively_support_s3__58____47____47___urls___40__for_addurl__44___get__44___etc__41__.mdwn
new file mode 100644
index 0000000000..1967904bb8
--- /dev/null
+++ b/doc/todo/Natively_support_s3__58____47____47___urls___40__for_addurl__44___get__44___etc__41__.mdwn
@@ -0,0 +1,25 @@
+Given that git-annex has interactions with AWS S3 built-in, similar to my whining about ssh:// urls, I wondered if may be s3:// urls could be supported directly by git-annex.
+Unfortunately not the case, and messages are a tiny bit misleading (see below) that initially annex just says that configuration disallows access to S3 but when tried to allow, seems to offload that to libcurl which doesn't support it.
+
+The reason I am asking, is that lots of data is on S3 and for now we either relied on our datalad special remote to provide access to S3:// so we could authenticate, but for public buckets it would be overkill to demand datalad.  Although we could replace them with http urls, I thought it might be better if annex could just download s3:// directly.
+
+[[!format sh """
+$> git annex addurl s3://images.cocodataset.org/annotations/image_info_test2014.zip         
+addurl s3://images.cocodataset.org/annotations/image_info_test2014.zip Configuration does not allow accessing s3://images.cocodataset.org/annotations/image_info_test2014.zip
+
+Configuration does not allow accessing s3://images.cocodataset.org/annotations/image_info_test2014.zip
+failed
+git-annex: addurl: 1 failed
+
+$> git -c annex.security.allowed-url-schemes="http https s3" -c annex.security.allowed-http-addresses=all annex addurl s3://images.cocodataset.org/annotations/image_info_test2014.zip
+addurl s3://images.cocodataset.org/annotations/image_info_test2014.zip 
+curl: (1) Protocol "s3" not supported or disabled in libcurl
+failed
+git-annex: addurl: 1 failed
+
+$> git annex version
+git-annex version: 6.20180913+git33-g2cd5a723f-1~ndall+1
+
+"""]]
+
+[[!meta author=yoh]]
diff --git a/doc/todo/Not_working_on_Android-x86.mdwn b/doc/todo/Not_working_on_Android-x86.mdwn
new file mode 100644
index 0000000000..56f2ce962d
--- /dev/null
+++ b/doc/todo/Not_working_on_Android-x86.mdwn
@@ -0,0 +1,19 @@
+[[!meta title="Android is only autobuilt for arm, not x86 or mips"]]
+
+### Please describe the problem.
+
+git-annex doesn't start on [Android-x86](http://www.android-x86.org) in VirtualBox (version 4.1.18-dfsg-2+deb7u1).
+
+On Android 4.2.2 (android-x86-4.2-20130228.iso) it starts the terminal which prints nothing but `[Terminal session finished]`.
+On Android 4.3 (android-x86-4.3-20130725.iso) it starts the terminal and prints:
+
+    In mgmain JNI_OnLoad
+    
+    [Terminal session finished]
+
+The browser/webapp is never started.
+
+### What version of git-annex are you using? On what operating system?
+
+Version 1.0.52 for Android. I made sure to install the correct APK files for each version of Android.
+
diff --git a/doc/todo/Not_working_on_Android-x86/comment_1_5eec4d7530c9df68f1bd1b1ca7021ef5._comment b/doc/todo/Not_working_on_Android-x86/comment_1_5eec4d7530c9df68f1bd1b1ca7021ef5._comment
new file mode 100644
index 0000000000..4bcca0ab4f
--- /dev/null
+++ b/doc/todo/Not_working_on_Android-x86/comment_1_5eec4d7530c9df68f1bd1b1ca7021ef5._comment
@@ -0,0 +1,14 @@
+[[!comment format=mdwn
+ username="http://joeyh.name/"
+ ip="209.250.56.64"
+ subject="comment 1"
+ date="2013-11-22T16:37:36Z"
+ content="""
+git-annex is a native binary, and is currently only being built for arm android.
+
+Is Android on x86 a thing used on real-world hardware? I have only seen it in the context of a developer's test environment.
+
+The instructions for building git-annex from source for Android should work on x86. The ghc-android build would need to be tweaked to build a cross compiler targeting that architecture. This can be done by editing settings near the top of ghc-android's `build.sh`.
+
+I am going to move this from bugs/ to todo/ after posting this comment, because it is certiantly not a bug, but a wishlist item at best.
+"""]]
diff --git a/doc/todo/Not_working_on_Android-x86/comment_2_e5c4c99cb0675ad69bf8d7796be23c8e._comment b/doc/todo/Not_working_on_Android-x86/comment_2_e5c4c99cb0675ad69bf8d7796be23c8e._comment
new file mode 100644
index 0000000000..3f568f5555
--- /dev/null
+++ b/doc/todo/Not_working_on_Android-x86/comment_2_e5c4c99cb0675ad69bf8d7796be23c8e._comment
@@ -0,0 +1,12 @@
+[[!comment format=mdwn
+ username="https://www.google.com/accounts/o8/id?id=AItOawlYSw35UMlOf9rFlP5EUTBz7is5X09qVXI"
+ nickname="Johannes"
+ subject="Support on Motorola razr i"
+ date="2013-11-28T16:10:38Z"
+ content="""
+Hi,
+I'm using the Motorola razr i. So there is a usecase, where a x86 CPU is used in a mobile phone. If it's easy as you described, can you build a Version with x86 support?
+
+Thanks
+Jack
+"""]]
diff --git a/doc/todo/Not_working_on_Android-x86/comment_3_6b609af60bf1c477139e40eba5cb0c4e._comment b/doc/todo/Not_working_on_Android-x86/comment_3_6b609af60bf1c477139e40eba5cb0c4e._comment
new file mode 100644
index 0000000000..e4ac8f820d
--- /dev/null
+++ b/doc/todo/Not_working_on_Android-x86/comment_3_6b609af60bf1c477139e40eba5cb0c4e._comment
@@ -0,0 +1,10 @@
+[[!comment format=mdwn
+ username="http://joeyh.name/"
+ ip="209.250.56.191"
+ subject="comment 3"
+ date="2014-04-20T21:01:50Z"
+ content="""
+Samsung Galaxy Tab 3 (GT-P5210)  also needs this.
+
+Each new git-annex builder takes me days to set up, and is an ongoing drain to keep running. I would much rather walk someone else through setting one up, unless this becomes a very common architecture for android.
+"""]]
diff --git a/doc/todo/PATCH__58___drop_url_parameters_from_extension.hs b/doc/todo/PATCH__58___drop_url_parameters_from_extension.hs
new file mode 100644
index 0000000000..580ed96bab
--- /dev/null
+++ b/doc/todo/PATCH__58___drop_url_parameters_from_extension.hs
@@ -0,0 +1,56 @@
+commit e233211d1d3bdc844d44f73be37c926393a57571
+Author: James MacMahon 
+Date:   Sat Oct 8 01:23:22 2016 -0400
+
+    Drop URL parameters from file extension
+    
+    During RSS feed importing, some enclosures have URL parameters in them:
+    
+        http://www.podtrac.com/pts/redirect.mp3/traffic.libsyn.com/classictales/CT_491_The_Blood_is_the_Life.mp3?dest-id=60626
+    
+    With --template='${feedtitle}/${itempubdate}-${itemtitle}${extension}',
+    this is sanitized to
+    
+        "The_Classic_Tales_Podcast/2016_10_07-Ep._491__The_Blood_Is_The_Life__by_F._Marion_Crawford.mp3_dest_id_60626"
+    
+    The culprit here is `takeExtension` in Command/ImportFeed.hs:
+    
+        rundownload url (takeExtension url) $ \f -> do
+    
+    `takeExtension` is a Posix function that returns the file extension,
+    which means all characters after the last '.':
+    
+        > takeExtension "http://www.podtrac.com/pts/redirect.mp3/traffic.libsyn.com/classictales/CT_491_The_Blood_is_the_Life.mp3?dest-id=60626"
+        ".mp3?dest-id=60626"
+    
+    This commit implements the function dropUrlParameters to take care of
+    this:
+    
+        > dropUrlParameters $ takeExtension "http://www.podtrac.com/pts/redirect.mp3/traffic.libsyn.com/classictales/CT_491_The_Blood_is_the_Life.mp3?dest-id=60626"
+        ".mp3"
+
+diff --git a/Command/ImportFeed.hs b/Command/ImportFeed.hs
+index 498d504..210aca0 100644
+--- a/Command/ImportFeed.hs
++++ b/Command/ImportFeed.hs
+@@ -158,10 +158,16 @@ downloadFeed url
+                                , return Nothing
+                                )
+ 
++dropUrlParameters :: String -> String
++dropUrlParameters (x:xs) = case x of
++       '?'       -> []
++       otherwise -> [x] ++ dropUrlParameters xs
++dropUrlParameters (x) = x
++
+ performDownload :: ImportFeedOptions -> Cache -> ToDownload -> Annex Bool
+ performDownload opts cache todownload = case location todownload of
+        Enclosure url -> checkknown url $
+-               rundownload url (takeExtension url) $ \f -> do
++               rundownload url (dropUrlParameters $ takeExtension url) $ \f -> do
+                        r <- Remote.claimingUrl url
+                        if Remote.uuid r == webUUID || rawOption opts
+                                then do
+
+> Hmm, didn't cleanly apply for some reason. And, `takeWhile (/= '?')` is a 
+> simpler way to do that. Thank you for the bug report and patch; [[done]] --[[Joey]]
diff --git a/doc/todo/Package_for_Lacie_NasOS__63__.mdwn b/doc/todo/Package_for_Lacie_NasOS__63__.mdwn
new file mode 100644
index 0000000000..f5353e258e
--- /dev/null
+++ b/doc/todo/Package_for_Lacie_NasOS__63__.mdwn
@@ -0,0 +1 @@
+Would it be possible to create a package for Lacie's NacOS?
diff --git a/doc/todo/Package_for_Lacie_NasOS__63__/comment_1_1b1b23b95b1fa29a8d71848a5cec9008._comment b/doc/todo/Package_for_Lacie_NasOS__63__/comment_1_1b1b23b95b1fa29a8d71848a5cec9008._comment
new file mode 100644
index 0000000000..4fffad28c5
--- /dev/null
+++ b/doc/todo/Package_for_Lacie_NasOS__63__/comment_1_1b1b23b95b1fa29a8d71848a5cec9008._comment
@@ -0,0 +1,12 @@
+[[!comment format=mdwn
+ username="joey"
+ subject="""comment 1"""
+ date="2016-01-20T18:56:36Z"
+ content="""
+If that is a Linux-based NAS, there's a reasonably good chance that one
+of the [Linux standalone builds](https://git-annex.branchable.com/install/Linux_standalone/)
+of git-annex will run on it.
+
+Actually, from what I can see, it's based on Debian, so you might be able
+to just apt-get install git-annex
+"""]]
diff --git a/doc/todo/Package_for_Lacie_NasOS__63__/comment_2_81b181781da615c56aa30eab131cec53._comment b/doc/todo/Package_for_Lacie_NasOS__63__/comment_2_81b181781da615c56aa30eab131cec53._comment
new file mode 100644
index 0000000000..7795a7ca4d
--- /dev/null
+++ b/doc/todo/Package_for_Lacie_NasOS__63__/comment_2_81b181781da615c56aa30eab131cec53._comment
@@ -0,0 +1,14 @@
+[[!comment format=mdwn
+ username="db"
+ subject="comment 2"
+ date="2016-01-22T19:00:57Z"
+ content="""
+Yes it is Linux:
+
+$ uname -a
+Linux nas 3.10.59-svn15282 #1 SMP Fri Jul 31 13:36:10 UTC 2015 x86_64 GNU/Linux
+
+But it does not have apt. And probably not a compiler either.
+
+Is it complicated to install a compiled version manually? Or is it \"self-containing\"? Any instructions for cross compiling on a Mac?
+"""]]
diff --git a/doc/todo/Package_for_Lacie_NasOS__63__/comment_3_fd7eeee6a86101867f0ab17f883f0c6a._comment b/doc/todo/Package_for_Lacie_NasOS__63__/comment_3_fd7eeee6a86101867f0ab17f883f0c6a._comment
new file mode 100644
index 0000000000..b69146873c
--- /dev/null
+++ b/doc/todo/Package_for_Lacie_NasOS__63__/comment_3_fd7eeee6a86101867f0ab17f883f0c6a._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="joey"
+ subject="""comment 3"""
+ date="2016-01-22T19:39:17Z"
+ content="""
+Get a build from
+
+"""]]
diff --git a/doc/todo/Pause_all_transfers_in_all_annexes_watched_by_the_assistant.mdwn b/doc/todo/Pause_all_transfers_in_all_annexes_watched_by_the_assistant.mdwn
new file mode 100644
index 0000000000..2ce33f726b
--- /dev/null
+++ b/doc/todo/Pause_all_transfers_in_all_annexes_watched_by_the_assistant.mdwn
@@ -0,0 +1,11 @@
+## Use case:
+
+You have a few annexes that the assistant is watching for you. You're somewhere with poor wifi speed. You also just added a bunch of big files to a few annexes. Now all of a sudden your connection suffers and you want an easy way to pause all transfers until you're on a faster connection without losing the automatic 'add' and such of the assistant (iow: without having to shutdown the daemon).
+
+## Proposal:
+
+A "Pause all transfers" button in the webapp that pauses all transfers from all annexes the assistant is watching.
+
+It should toggle to "Resume all transfers" when pushed so you can also easily start the transfers again when you get somewhere else.
+
+This may or may not make more sense if the webapp showed all watched repos in a single view (instead of the separate pages/views as it is now).
diff --git a/doc/todo/S3_anonymous_exporttree_support.mdwn b/doc/todo/S3_anonymous_exporttree_support.mdwn
new file mode 100644
index 0000000000..681a5943ea
--- /dev/null
+++ b/doc/todo/S3_anonymous_exporttree_support.mdwn
@@ -0,0 +1,4 @@
+S3 remotes using exporttree=yes cannot currently be accessed w/o creds.
+This is possible to support with some work. --[[Joey]]
+
+> [[done]] --[[Joey]]
diff --git a/doc/todo/S3_fsck_support.mdwn b/doc/todo/S3_fsck_support.mdwn
new file mode 100644
index 0000000000..328cd0d7db
--- /dev/null
+++ b/doc/todo/S3_fsck_support.mdwn
@@ -0,0 +1,55 @@
+I have (i think?) noticed that the s3 remote doesn't really do an fsck:
+
+http://source.git-annex.branchable.com/?p=source.git;a=blob;f=Remote/S3.hs;hb=HEAD#l86
+
+Besides, unless S3 does something magic and amazingly fast, the checksum is just too slow for it to be really operational:
+
+
    +$ time git annex fsck -f s3 video/original/quartet_for_deafblind_h264kbs18000_24.mov
+fsck video/original/quartet_for_deafblind_h264kbs18000_24.mov (checking s3...) ok
+(recording state in git...)
+
+real    0m1.188s
+user    0m0.444s
+sys     0m0.324s
+$ time git annex fsck video/original/quartet_for_deafblind_h264kbs18000_24.mov
+fsck video/original/quartet_for_deafblind_h264kbs18000_24.mov (checksum...)
+ok
+(recording state in git...)
+
+real    3m14.478s
+user    1m55.679s
+sys     0m8.325s
+
    
+
+1s is barely the time for git-annex to do an HTTP request to amazon, and what is returned doesn't seem to have a checksum of any kind:
+
+
    +fsck video/original/quartet_for_deafblind_h264kbs18000_24.mov (checking s3...) [2015-06-16 00:31:46 UTC] String to sign: "HEAD\n\n\nTue, 16 Jun 2015 00:31:46 GMT\n/isuma-files/SHA256E-s11855411701--ba268f1c401321db08d4cb149d73a51a10f02968687cb41f06051943b4720465.mov"
+[2015-06-16 00:31:46 UTC] Host: "isuma-files.s3.amazonaws.com"
+[2015-06-16 00:31:46 UTC] Response header 'x-amz-request-id': '9BF7B64EB5A619F3'
+[2015-06-16 00:31:46 UTC] Response header 'x-amz-id-2': '84ZO7IZ0dqJeEghADjt7hTGKGqGAWwbwwaCFVft3ama+oDOVJrvpiFjqn8EY3Z0R'
+[2015-06-16 00:31:46 UTC] Response header 'Content-Type': 'application/xml'
+[2015-06-16 00:31:46 UTC] Response header 'Transfer-Encoding': 'chunked'
+[2015-06-16 00:31:46 UTC] Response header 'Date': 'Tue, 16 Jun 2015 00:32:10 GMT'
+[2015-06-16 00:31:46 UTC] Response header 'Server': 'AmazonS3'
+[2015-06-16 00:31:46 UTC] Response metadata: S3: request ID=, x-amz-id-2=
+ok
+
    
+
+did i miss something? are there fsck checks for s3 remotes?
+
+if not, i think it would be useful to leverage the "md5summing" functionality that the S3 API provides. there are two relevant stackoverflow responses here:
+
+http://stackoverflow.com/questions/1775816/how-to-get-the-md5sum-of-a-file-on-amazons-s3
+http://stackoverflow.com/questions/8618218/amazon-s3-checksum
+
+... to paraphrase: when a file is `PUT` on S3, one can provide a `Content-MD5` header that S3 will check against the uploaded file content for corruption, when doing the upload. then there is some talk about how the `ETag` header *may* hold the MD5, but that seems inconclusive. There's a specific API call for getting the MD5 sum:
+
+https://docs.aws.amazon.com/AWSAndroidSDK/latest/javadoc/com/amazonaws/services/s3/model/ObjectMetadata.html#getContentMD5()
+
+the android client also happens to check with that API on downloads:
+
+https://github.com/aws/aws-sdk-android/blob/4de3a3146d66d9ab5684eb5e71d5a2cef9f4dec9/aws-android-sdk-s3/src/main/java/com/amazonaws/services/s3/AmazonS3Client.java#L1302
+
+now of course MD5 is a pile of dung nowadays, but having that checksum beats not having any checksum at all. *and* it is at no cost on the client side... --[[anarcat]]
diff --git a/doc/todo/S3_fsck_support/comment_1_9317598723ea38ae9432ad50e0de666d._comment b/doc/todo/S3_fsck_support/comment_1_9317598723ea38ae9432ad50e0de666d._comment
new file mode 100644
index 0000000000..6a66335d74
--- /dev/null
+++ b/doc/todo/S3_fsck_support/comment_1_9317598723ea38ae9432ad50e0de666d._comment
@@ -0,0 +1,26 @@
+[[!comment format=mdwn
+ username="joey"
+ subject="""comment 1"""
+ date="2015-06-16T17:50:56Z"
+ content="""
+You're incorrect; `git-annex fsck --from remote` works to fsck *any*
+remote. For remotes like S3, it has to download the content to check it
+locally, which is why `remoteFsck` is not provided.
+
+Since you passed -f (--fast) to fsck, it avoids checksuming the content,
+so avoids downloading it, and only verifies that S3 still says it has
+the content. As documented on the git-annex fsck man page.
+
+AFAICS, the Content-MD5 is only used by S3 to check that the data uploaded
+to S3 didn't get corrupted over the wire. I assume that S3 implements its
+own checksums to detect when data already stored on it gets corrupted, so
+it seems redundant and complicating for git-annex to query it for md5sums.
+It would work just as well for git-annex to verify a key after downloading
+it, using the key's own hash, per [[todo/ checksum verification on transfer]].
+
+It **might** be worth filling in the `poContentMD5` field with the md5 of
+the file when uploading it to S3. Of course, this requires hashing the file
+locally. And when storing an encrypted object on S3, it would require
+buffering the whole encrypted object to disk first, in order to hash it
+(but that's currently done anyway).
+"""]]
diff --git a/doc/todo/S3_fsck_support/comment_2_7a1ce64d362b8f75adf22709771a7787._comment b/doc/todo/S3_fsck_support/comment_2_7a1ce64d362b8f75adf22709771a7787._comment
new file mode 100644
index 0000000000..a27ed8e567
--- /dev/null
+++ b/doc/todo/S3_fsck_support/comment_2_7a1ce64d362b8f75adf22709771a7787._comment
@@ -0,0 +1,11 @@
+[[!comment format=mdwn
+ username="anarcat"
+ subject="comment 2"
+ date="2015-06-16T20:10:50Z"
+ content="""
+understood: i thought `-f` was `--from`... hence my confusion.
+
+as for `remoteFsck`, i guess what i am saying is exactly that: there *does* seem to be a way to do a remote checksum of the file *without* downloading it. it seems to be a critical advantage over having to download the whole repository to check it... maybe `--fast` could use that technique and `non--fast` would download?
+
+as for the on-wire MD5 stuff, that does seem to be overkill...
+"""]]
diff --git a/doc/todo/S3_fsck_support/comment_3_2b96f61188b2ada729a284e95fc89dab._comment b/doc/todo/S3_fsck_support/comment_3_2b96f61188b2ada729a284e95fc89dab._comment
new file mode 100644
index 0000000000..fea0477543
--- /dev/null
+++ b/doc/todo/S3_fsck_support/comment_3_2b96f61188b2ada729a284e95fc89dab._comment
@@ -0,0 +1,21 @@
+[[!comment format=mdwn
+ username="joey"
+ subject="""comment 3"""
+ date="2015-07-02T16:43:03Z"
+ content="""
+Not knowing how S3's backend is implemented, I have to assume that, in
+order to get the advertised number of 9's of reliability, it involves some
+replication of data, as well as some method to detect if a bit has flipped
+or a drive has died, and recover.
+
+This is requesting that git-annex ask S3 for a md5sum, and compare it
+against a md5sum that it, presumably, keeps track of locally. If the two
+are different, git-annex could tell that S3 has lost data. But, git-annex is
+in a much worse position to tell if S3 has lost data then S3 itself is.
+It seems very unlikely that this extra checking would ever detect a problem
+that S3 didn't itself detect and fix (or in the 0.00001% case, 
+fail to fix and delete the lost file?)
+
+Bit flips during transfer seem more likely than that. `poContentMD5` could
+help guard against those.
+"""]]
diff --git a/doc/todo/S3_multipart_interruption_cleanup.mdwn b/doc/todo/S3_multipart_interruption_cleanup.mdwn
new file mode 100644
index 0000000000..adb5fd2cb0
--- /dev/null
+++ b/doc/todo/S3_multipart_interruption_cleanup.mdwn
@@ -0,0 +1,14 @@
+When a multipart S3 upload is being made, and gets interrupted,
+the parts remain in the bucket, and S3 may charge for them.
+
+I am not sure what happens if the same object gets uploaded again. Is S3
+nice enough to remove the old parts? I need to find out..
+
+If not, this needs to be dealt with somehow. One way would be to configure an
+expiry of the uploaded parts, but this is tricky as a huge upload could
+take arbitrarily long. Another way would be to record the uploadid and the
+etags of the parts, and then resume where it left off the next time the
+object is sent to S3. (Or at least cancel the old upload; resume isn't
+practical when uploading an encrypted object.) 
+
+It could store that info in either the local FS or the git-annex branch.
diff --git a/doc/todo/Set_total_storage_limit_for_special_remotes.mdwn b/doc/todo/Set_total_storage_limit_for_special_remotes.mdwn
new file mode 100644
index 0000000000..2b66e98284
--- /dev/null
+++ b/doc/todo/Set_total_storage_limit_for_special_remotes.mdwn
@@ -0,0 +1 @@
+I'd like to be able to set a fixed limit on how much storage can be uploaded to a special remote. A use case for this may be that I want to spend no more than Y dollars, on a storage service that charges $X per gigabyte. I would thus set a limit where I a upload would be interrupted with a warning about the limit, and to continue I would need to use a --force option.
diff --git a/doc/todo/Set_total_storage_limit_for_special_remotes/comment_1_8945025ad54e28f48474f8931746a775._comment b/doc/todo/Set_total_storage_limit_for_special_remotes/comment_1_8945025ad54e28f48474f8931746a775._comment
new file mode 100644
index 0000000000..0df5669d66
--- /dev/null
+++ b/doc/todo/Set_total_storage_limit_for_special_remotes/comment_1_8945025ad54e28f48474f8931746a775._comment
@@ -0,0 +1,9 @@
+[[!comment format=mdwn
+ username="joey"
+ subject="""comment 1"""
+ date="2015-07-02T20:01:42Z"
+ content="""
+This totally makes sense to want, but it's rather difficult to implement since
+multiple git-annex repos can all be uploading content to the same special
+remote, while disconnected from each other.
+"""]]
diff --git a/doc/todo/Set_total_storage_limit_for_special_remotes/comment_2_86aa368db46dfedb065a18edfa7c9ad4._comment b/doc/todo/Set_total_storage_limit_for_special_remotes/comment_2_86aa368db46dfedb065a18edfa7c9ad4._comment
new file mode 100644
index 0000000000..818b7f3862
--- /dev/null
+++ b/doc/todo/Set_total_storage_limit_for_special_remotes/comment_2_86aa368db46dfedb065a18edfa7c9ad4._comment
@@ -0,0 +1,11 @@
+[[!comment format=mdwn
+ username="mawillcockson"
+ subject="comment 2"
+ date="2015-07-18T19:13:03Z"
+ content="""
+Perhaps server-side [git hooks](https://git-scm.com/book/en/v2/Customizing-Git-Git-Hooks)?
+
+Maybe a git hook script that uses [inotify](https://en.wikipedia.org/wiki/Inotify) for inode size notifications.
+
+I would have no idea how to implement this, though.
+"""]]
diff --git a/doc/todo/Show_repo_type_in_repo_list.mdwn b/doc/todo/Show_repo_type_in_repo_list.mdwn
new file mode 100644
index 0000000000..40fbe6537e
--- /dev/null
+++ b/doc/todo/Show_repo_type_in_repo_list.mdwn
@@ -0,0 +1 @@
+It would be helpful to show each repo's type in the list.
diff --git a/doc/todo/Show_repo_type_in_repo_list/comment_1_ac6eb1072ef902a094b79dd8e0917c4d._comment b/doc/todo/Show_repo_type_in_repo_list/comment_1_ac6eb1072ef902a094b79dd8e0917c4d._comment
new file mode 100644
index 0000000000..10c0c17df4
--- /dev/null
+++ b/doc/todo/Show_repo_type_in_repo_list/comment_1_ac6eb1072ef902a094b79dd8e0917c4d._comment
@@ -0,0 +1,10 @@
+[[!comment format=mdwn
+ username="http://joeyh.name/"
+ ip="209.250.56.47"
+ subject="comment 1"
+ date="2013-11-02T23:45:39Z"
+ content="""
+Currently if you go to the edit page for the repository, it shows some information about it, including its type and often its location, at the bottom of the page.
+
+I tend to feel that putting anything else in the repo list would result in it being too cluttered.
+"""]]
diff --git a/doc/todo/Show_repo_type_in_repo_list/comment_2_6979c487f707a724a048d20e2e5744e6._comment b/doc/todo/Show_repo_type_in_repo_list/comment_2_6979c487f707a724a048d20e2e5744e6._comment
new file mode 100644
index 0000000000..7d4a363242
--- /dev/null
+++ b/doc/todo/Show_repo_type_in_repo_list/comment_2_6979c487f707a724a048d20e2e5744e6._comment
@@ -0,0 +1,14 @@
+[[!comment format=mdwn
+ username="https://www.google.com/accounts/o8/id?id=AItOawmNu4V5fvpLlBhaCUfXXOB0MI5NXwh8SkU"
+ nickname="Adam"
+ subject="comment 2"
+ date="2013-11-02T23:59:16Z"
+ content="""
+I understand.  Well, here's what I'm looking at right now.  :)
+
+[[http://alphapapa.net/outbox/view.png]]
+
+I just think it would be very useful to say \"Client\" or \"Transfer\" or \"Full Backup\" next to each one, and there's often plenty of room, depending on the size of the window.  Especially when one's trying to wrap one's head around git-annex, being reminded what each repo is for would help a lot.  Otherwise you basically have to put it in the name or description yourself...so why not just go ahead and show it?  :)
+
+Another option might be to have an icon for each type.
+"""]]
diff --git a/doc/todo/Show_repo_type_in_repo_list/comment_3_529254a6cc20de7259d60a3cbc5ccaf7._comment b/doc/todo/Show_repo_type_in_repo_list/comment_3_529254a6cc20de7259d60a3cbc5ccaf7._comment
new file mode 100644
index 0000000000..3f8b826950
--- /dev/null
+++ b/doc/todo/Show_repo_type_in_repo_list/comment_3_529254a6cc20de7259d60a3cbc5ccaf7._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="http://joeyh.name/"
+ ip="209.250.56.47"
+ subject="comment 3"
+ date="2013-11-03T00:19:34Z"
+ content="""
+I'd be happy to put in some icons if someone finds some good (and suitably licensed) ones that cover the different types of repositories.
+"""]]
diff --git a/doc/todo/Slow_transfer_for_a_lot_of_small_files.mdwn b/doc/todo/Slow_transfer_for_a_lot_of_small_files.mdwn
new file mode 100644
index 0000000000..00cdad0fe1
--- /dev/null
+++ b/doc/todo/Slow_transfer_for_a_lot_of_small_files.mdwn
@@ -0,0 +1,20 @@
+What steps will reproduce the problem?
+Sync a lot of small files.
+
+What is the expected output? What do you see instead?
+The expected output is hopefully a fast transfer.
+
+But currently it seems like git-annex is only using one thread to transfer(per host or total?)
+
+An option to select number of transfer threads to use(possibly per host) would be very nice.
+
+> Opening a lot of connections to a single host is probably not desirable.
+> 
+> I do want to do something to allow slow hosts to not hold up transfers to
+> other hosts, which might involve running multiple queued transfers at
+> once. The webapp already allows the user to force a given transfer to
+> happen immediately. --[[Joey]] 
+
+And maybe also an option to limit how long a queue the browser should show, it can become quite resource intensive with a long queue.
+
+> The queue is limited to 20 items for this reason. --[[Joey]]
diff --git a/doc/todo/Slow_transfer_for_a_lot_of_small_files/comment_1_fa76bc744e30aaeed4a3b3e2fe3dd75e._comment b/doc/todo/Slow_transfer_for_a_lot_of_small_files/comment_1_fa76bc744e30aaeed4a3b3e2fe3dd75e._comment
new file mode 100644
index 0000000000..c877f93596
--- /dev/null
+++ b/doc/todo/Slow_transfer_for_a_lot_of_small_files/comment_1_fa76bc744e30aaeed4a3b3e2fe3dd75e._comment
@@ -0,0 +1,12 @@
+[[!comment format=mdwn
+ username="https://www.google.com/accounts/o8/id?id=AItOawnKT33H9qVVGJOybP00Zq2NZmNAyB65mic"
+ nickname="Lucas"
+ subject="comment 1"
+ date="2014-11-12T07:58:07Z"
+ content="""
+Opening multiple connections to a host can be preferable sometimes and it's unlikely to be an issue at all for the larger remotes like Google, Microsoft or S3.
+
+For example, the OneDrive provider spends a lot of time sitting around waiting for initialisation between uploads. Using, say 5 threads instead of 1 would allow it to continue doing things while it waits.
+
+Multiple connections can also vastly improve upload speeds for users with congested home internet connections.
+"""]]
diff --git a/doc/todo/Slow_transfer_for_a_lot_of_small_files/comment_2_80d1080bf6e82bd8aaccde9d7c1669c7._comment b/doc/todo/Slow_transfer_for_a_lot_of_small_files/comment_2_80d1080bf6e82bd8aaccde9d7c1669c7._comment
new file mode 100644
index 0000000000..d531914ab3
--- /dev/null
+++ b/doc/todo/Slow_transfer_for_a_lot_of_small_files/comment_2_80d1080bf6e82bd8aaccde9d7c1669c7._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="https://launchpad.net/~krastanov-stefan"
+ nickname="krastanov-stefan"
+ subject="Status of this issue"
+ date="2014-12-27T15:18:42Z"
+ content="""
+I was unable to find a way to tell git-annex that certain remotes should receive multiple transfers in parallel. Is this implemented yet or on the roadmap? If neither would modifying the webapp to bear this logic without touching git-annex itself be a solution (asking mainly because it can be done with a greasemonkey script)? 
+"""]]
diff --git a/doc/todo/Slow_transfer_for_a_lot_of_small_files/comment_3_dac38d9e48041b59dd1a53180e456ec9._comment b/doc/todo/Slow_transfer_for_a_lot_of_small_files/comment_3_dac38d9e48041b59dd1a53180e456ec9._comment
new file mode 100644
index 0000000000..1b10f9e32c
--- /dev/null
+++ b/doc/todo/Slow_transfer_for_a_lot_of_small_files/comment_3_dac38d9e48041b59dd1a53180e456ec9._comment
@@ -0,0 +1,12 @@
+[[!comment format=mdwn
+ username="lhunath@3b4ff15f4600f3276d1776a490b734fca0f5c245"
+ nickname="lhunath"
+ subject="Simultaneous transfers"
+ date="2018-02-02T17:37:27Z"
+ content="""
+I highly recommend ensuring that:
+1. Each remote can configure a number of maximum simultaneous transfers, where each type of remote comes with a sensible default number.
+2. Transfers to multiple individual remotes happen in parallel regardless of their simultaneous transfers setting.
+
+Judging from the fact that simultaneous transfers happen just fine when you hit the > icon in the webapp, I would assume that most of the underbelly for this is already present.
+"""]]
diff --git a/doc/todo/Specify_maximum_usable_space_per_remote.mdwn b/doc/todo/Specify_maximum_usable_space_per_remote.mdwn
new file mode 100644
index 0000000000..14f9d548b2
--- /dev/null
+++ b/doc/todo/Specify_maximum_usable_space_per_remote.mdwn
@@ -0,0 +1,9 @@
+I would like to be able to limit the amount of space git-annex will use on a remote.
+
+Obviously this would not limit the total amount of space used on whatever filesystem the remote is on, but would only apply to space used within the remote for the particular annex in question.
+
+Many of the cloud storage providers that git-annex supports through special remotes have free tiers. For instance, Google offers 15GB of free storage. If I have an annex with > 15 GB of storage I don't just want to add a Google Drive special remote and start attempting to copy everything over. But it would be great to be able to take advantage of that storage by adding a special remote to the annex and telling git-annex to only use a maximum of 13 GB (to leave myself a 2 GB buffer, in case things get added to the Google Drive through some mechanism outside of the annex). I could then add the remote to a preferred content group like `backup` and do a `git-annex copy --auto gdrive`, which would copy everything to Google Drive, unless transferring the next file would cause the remote to use over 13 GB.
+
+Currently I can see the size of a remote using `git annex info gdrive`, so git-annex appears to have the needed information.
+
+This is sort of like `annex.diskreserve`, but more useful for special remotes where setting an amount of space to keep free is not relevant.
diff --git a/doc/todo/Specify_maximum_usable_space_per_remote/comment_1_fdc0c518b6cbc5dc7e5b64748c2d7b01._comment b/doc/todo/Specify_maximum_usable_space_per_remote/comment_1_fdc0c518b6cbc5dc7e5b64748c2d7b01._comment
new file mode 100644
index 0000000000..1365e932c7
--- /dev/null
+++ b/doc/todo/Specify_maximum_usable_space_per_remote/comment_1_fdc0c518b6cbc5dc7e5b64748c2d7b01._comment
@@ -0,0 +1,17 @@
+[[!comment format=mdwn
+ username="joey"
+ subject="""comment 1"""
+ date="2016-09-05T16:40:56Z"
+ content="""
+This poses difficulties when several different git-annex repositories
+are uploading to the same special remote. They would have to somehow maintain a
+count of the amount of space used on the remote, but the repositories
+could be completely disconnected, other than being able to all access the
+special remote. So, the count would have to be stored on the remote, and
+updated atomically. But, not all remotes support atomic operations well
+enough to support such a distributed counter.
+
+If the storage service has an API call to get the space used, then the
+special remote can use that. I think that such a thing unfortunately can
+only be implemented that way, on a per-special-remote implementation basis.
+"""]]
diff --git a/doc/todo/Specify_maximum_usable_space_per_remote/comment_2_b81af6dac4042c0d4dab58c91a46b09d._comment b/doc/todo/Specify_maximum_usable_space_per_remote/comment_2_b81af6dac4042c0d4dab58c91a46b09d._comment
new file mode 100644
index 0000000000..8fe3597c79
--- /dev/null
+++ b/doc/todo/Specify_maximum_usable_space_per_remote/comment_2_b81af6dac4042c0d4dab58c91a46b09d._comment
@@ -0,0 +1,15 @@
+[[!comment format=mdwn
+ username="pigmonkey"
+ subject="comment 2"
+ date="2016-09-06T23:10:47Z"
+ content="""
+Is it not possible to implement per-annex?
+
+For instance, take the walkthrough example where you create an annex at `~/annex` and then clone it at `/media/usb/annex`. Take that setup and add a special remote -- say, a Google Drive remote called `annex-gdrive`. Now both `~/annex` and `/media/usb/annex` know which files are stored on the `annex-gdrive` remote. So we go into `~/annex` and set the maximum usable space for `annex-gdrive` and upload a couple files. Then we can go into `/media/usb/annex`, sync, and it will be able to determine the space still available on `annex-gdrive`, not through any special API but just by looking at the size of the files it knows are stored on that special remote.
+
+In the case that we create a brand new annex `~/foo` and add a Google Drive special remote to it `foo-gdrive` that happens to use the same Google account as the `annex-gdrive` remote from the previous annex, I would not expect that new annex to know about the storage used for `annex-gdrive`. That would need some sort of special API from the provider.
+
+I just want the annex `~/annex` to know about the storage used at `annex-gdrive` and the annex `~/foo` to know about the storage used at `foo-gdrive`.
+
+Hope that makes sense!
+"""]]
diff --git a/doc/todo/Specify_maximum_usable_space_per_remote/comment_3_afc7f278c02d7bf9f92a90e51cb5a2a9._comment b/doc/todo/Specify_maximum_usable_space_per_remote/comment_3_afc7f278c02d7bf9f92a90e51cb5a2a9._comment
new file mode 100644
index 0000000000..02ad7ca406
--- /dev/null
+++ b/doc/todo/Specify_maximum_usable_space_per_remote/comment_3_afc7f278c02d7bf9f92a90e51cb5a2a9._comment
@@ -0,0 +1,19 @@
+[[!comment format=mdwn
+ username="joey"
+ subject="""comment 3"""
+ date="2016-09-07T15:08:37Z"
+ content="""
+git-annex repositories can all be clones of one another, but not currently
+connected. They can stay disconnected for an arbitrary amount of time,
+up to forever.
+
+For example, /media/usb/annex can be packed up, sent half way around the
+world and used with a different computer, which has no way to communicate
+with its parent ~/annex on the original computer.
+
+So, git-annex has to deal with split-brain and other distributed
+inconsistencies. And it does. However, a keeping count of how much space is
+used on a remote can't be consistent in these kinds of situations, unless,
+as I said, the count is kept in sync using the remote in question, and not
+information in the git-annex repositories.
+"""]]
diff --git a/doc/todo/Specify_maximum_usable_space_per_remote/comment_4_a760351ed7a6d3d232af7c22b2063871._comment b/doc/todo/Specify_maximum_usable_space_per_remote/comment_4_a760351ed7a6d3d232af7c22b2063871._comment
new file mode 100644
index 0000000000..f4af1c867f
--- /dev/null
+++ b/doc/todo/Specify_maximum_usable_space_per_remote/comment_4_a760351ed7a6d3d232af7c22b2063871._comment
@@ -0,0 +1,7 @@
+[[!comment format=mdwn
+ username="pigmonkey"
+ subject="comment 4"
+ date="2016-09-07T20:38:35Z"
+ content="""
+That makes sense. Thanks.
+"""]]
diff --git a/doc/todo/Specify_maximum_usable_space_per_remote/comment_5_c5069ffed5e8c4e1d001aedb971aca75._comment b/doc/todo/Specify_maximum_usable_space_per_remote/comment_5_c5069ffed5e8c4e1d001aedb971aca75._comment
new file mode 100644
index 0000000000..f9c24659e7
--- /dev/null
+++ b/doc/todo/Specify_maximum_usable_space_per_remote/comment_5_c5069ffed5e8c4e1d001aedb971aca75._comment
@@ -0,0 +1,13 @@
+[[!comment format=mdwn
+ username="jason.dixon.email@aa0e536a2ec2877d6f666108dbbc6e39bbe87ac0"
+ nickname="jason.dixon.email"
+ avatar="http://cdn.libravatar.org/avatar/fbe9050fc83bbd536d307d87ea14d4bc"
+ subject="Still seems useful"
+ date="2017-03-07T10:19:41Z"
+ content="""
+I dunno. I get your point. However I can also see this being a really useful feature. I can imagine it being implemented simply with a warning / note mentioning disconnected repos might overflow the limit (and perhaps a buffer should be left ie: limit = 15gb, allocate = 13gb). It would be up to the user to figure out what is right for them.
+
+I'm sure plenty of people either using the remote alone, in a small team, or with linked up repos (github/gitlab/whatever) would appreciate this option. I know I came to this page looking to see if it was possible already. :)
+
+Thanks too for the hard work! Amazing software!
+"""]]
diff --git a/doc/todo/Speed_up___39__import_--clean-duplicates__39__.mdwn b/doc/todo/Speed_up___39__import_--clean-duplicates__39__.mdwn
new file mode 100644
index 0000000000..34c21ab01c
--- /dev/null
+++ b/doc/todo/Speed_up___39__import_--clean-duplicates__39__.mdwn
@@ -0,0 +1,7 @@
+I'm currently in the process of gutting old (some broken) git-annex's and cleaning out download directories from before I started using git-annex.
+
+To do this, I am running `git annex import --clean--duplicates $PATH` on the directories I want to clear out but sometimes, this takes a unnecessarily long time.
+
+For example, git-annex will calculate the digest for a huge file (30GB+) in $TARGET, even though there are no files in the annex of that size.
+
+It's a common shortcut to check for duplicate sizes first to eliminate definite non-matches really quickly. Can this be added to git-annex's `import` in some way or is this a no-go due to the constant memory constraint?
diff --git a/doc/todo/Speed_up___39__import_--clean-duplicates__39__/comment_1_9268c639d3d21cce4ca7b60d08e9cb65._comment b/doc/todo/Speed_up___39__import_--clean-duplicates__39__/comment_1_9268c639d3d21cce4ca7b60d08e9cb65._comment
new file mode 100644
index 0000000000..8584d5ae8c
--- /dev/null
+++ b/doc/todo/Speed_up___39__import_--clean-duplicates__39__/comment_1_9268c639d3d21cce4ca7b60d08e9cb65._comment
@@ -0,0 +1,10 @@
+[[!comment format=mdwn
+ username="http://joeyh.name/"
+ ip="24.159.78.125"
+ subject="interesting idea"
+ date="2014-07-30T15:03:46Z"
+ content="""
+This could be done in constant space using a bloom filter of known file sizes. Files with wrong sizes would sometimes match, but no problem, it would then just do the work it does now.
+
+However, to build such a filter, git-annex would need to do a scan of all keys it knows about. This would take approximately as long to run as `git annex unused` does. It might make sense to only build the filter if it runs into a fairly large file. Alternatively, a bloom filter of file sizes could be cached and updated on the fly as things change (but this gets pretty complex).
+"""]]
diff --git a/doc/todo/Speed_up___39__import_--clean-duplicates__39__/comment_2_9c6688901ef20badd834419202627d5c._comment b/doc/todo/Speed_up___39__import_--clean-duplicates__39__/comment_2_9c6688901ef20badd834419202627d5c._comment
new file mode 100644
index 0000000000..e372b405e3
--- /dev/null
+++ b/doc/todo/Speed_up___39__import_--clean-duplicates__39__/comment_2_9c6688901ef20badd834419202627d5c._comment
@@ -0,0 +1,21 @@
+[[!comment format=mdwn
+ username="Xyem"
+ ip="81.111.193.130"
+ subject="comment 2"
+ date="2014-08-01T09:05:45Z"
+ content="""
+Could be tested out with an additional flag `--with-size-bloom` on import?
+
+It would then build a bloom (and use a cached one with --fast) and do the usual import.
+
+So I could do this:
+
+    # Bloom is created and the import is done using it
+    git annex import --clean-duplicates --with-size-bloom $TARGET
+
+    # Previously created bloom is used
+    git annex import --clean-duplicates --with-size-bloom --fast $TARGET2
+    git annex import --clean-duplicates --with-size-bloom --fast $TARGET3
+
+I can implement this behaviour in Perl with Bloom::Filter and let you know how it performs if that would be useful to you..?
+"""]]
diff --git a/doc/todo/Store_git_pack_files_on_special_remotes.mdwn b/doc/todo/Store_git_pack_files_on_special_remotes.mdwn
new file mode 100644
index 0000000000..e4a86c7524
--- /dev/null
+++ b/doc/todo/Store_git_pack_files_on_special_remotes.mdwn
@@ -0,0 +1 @@
+It would be nice to be able to upload and download git history with special remotes.  This could be a move towards full special remote syncing.
diff --git a/doc/todo/Support_for_include_matching_option_in_findref.mdwn b/doc/todo/Support_for_include_matching_option_in_findref.mdwn
new file mode 100644
index 0000000000..36bef2afc2
--- /dev/null
+++ b/doc/todo/Support_for_include_matching_option_in_findref.mdwn
@@ -0,0 +1,5 @@
+The documentation for `findref` says it accepts the same options as `find` but the matching option `--include` isn't supported here.
+
+This leads to the confusing behavior where `findref` is sensitive to the presence of content, but you can't tell it not to be in the same manner as `find`.
+
+Could the difference be removed by adding support for `--include`?
diff --git a/doc/todo/Test_cases_for_exporttree_special_remotes.mdwn b/doc/todo/Test_cases_for_exporttree_special_remotes.mdwn
new file mode 100644
index 0000000000..3d32b2c710
--- /dev/null
+++ b/doc/todo/Test_cases_for_exporttree_special_remotes.mdwn
@@ -0,0 +1,3 @@
+As far as I can tell, `git annex testremote` doesn't test exporting yet
+
+> [[fixed|done]] --[[Joey]]
diff --git a/doc/todo/Test_cases_for_exporttree_special_remotes/comment_1_4db9a4b9cb94ba7ee460ee81fe52bf86._comment b/doc/todo/Test_cases_for_exporttree_special_remotes/comment_1_4db9a4b9cb94ba7ee460ee81fe52bf86._comment
new file mode 100644
index 0000000000..18caa43f80
--- /dev/null
+++ b/doc/todo/Test_cases_for_exporttree_special_remotes/comment_1_4db9a4b9cb94ba7ee460ee81fe52bf86._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="joey"
+ subject="""comment 1"""
+ date="2017-11-07T21:07:00Z"
+ content="""
+That's right, exporttree is not tested yet.
+Thanks for the reminder.
+"""]]
diff --git a/doc/todo/Test_cases_for_exporttree_special_remotes/comment_2_0e280ec5691dbb0eef68f6e6c1424d08._comment b/doc/todo/Test_cases_for_exporttree_special_remotes/comment_2_0e280ec5691dbb0eef68f6e6c1424d08._comment
new file mode 100644
index 0000000000..5cbb24e598
--- /dev/null
+++ b/doc/todo/Test_cases_for_exporttree_special_remotes/comment_2_0e280ec5691dbb0eef68f6e6c1424d08._comment
@@ -0,0 +1,7 @@
+[[!comment format=mdwn
+ username="joey"
+ subject="""comment 2"""
+ date="2017-11-08T17:38:02Z"
+ content="""
+Added some fairly comprehensive tests.
+"""]]
diff --git a/doc/todo/Use_MediaScannerConnection_on_Android.mdwn b/doc/todo/Use_MediaScannerConnection_on_Android.mdwn
new file mode 100644
index 0000000000..afce9308d3
--- /dev/null
+++ b/doc/todo/Use_MediaScannerConnection_on_Android.mdwn
@@ -0,0 +1,7 @@
+Currently if photos or videos are copied into the Camera/DCIM directory on an Android device, or deleted the Gallery doesn't notice the changes.
+
+It is necessary to call MediaScannerConnection - http://developer.android.com/reference/android/media/MediaScannerConnection.html - to notify the system of the change.
+
+More info, and some sample Java code: http://stackoverflow.com/questions/13270789/how-to-run-media-scanner-in-android
+
+It'd be awesome if the assistant did this on files it has changed.  Possibly just under Camera/DCIM, but perhaps it should be configurable. MediaScannerConnection is also used to notify and index new music files.
diff --git a/doc/todo/Use_a_remote_as_a_sharing_site_for_files_with_obfuscated_URLs.mdwn b/doc/todo/Use_a_remote_as_a_sharing_site_for_files_with_obfuscated_URLs.mdwn
new file mode 100644
index 0000000000..a42a81d02b
--- /dev/null
+++ b/doc/todo/Use_a_remote_as_a_sharing_site_for_files_with_obfuscated_URLs.mdwn
@@ -0,0 +1,7 @@
+There are times when it is handy to be able to upload a file to a web host somewhere and share a link for that file to a select few people.
+
+It seems to be that the assistant could handle this scenario. It could generate a directory with a random name on the remote, and transfer the file there (using the existing filename) and the appropriate URL could be displayed in the assistant webapp to allow the user to copy the URL to send it to the appropriate people.
+
+Note: Joey and I had a quick chat about this use case at LCA2013.
+
+[[!tag design/assistant]]
diff --git a/doc/todo/Use_a_remote_as_a_sharing_site_for_files_with_obfuscated_URLs/comment_1_1a1f34f4f389267d67e79409c0ca8b1d._comment b/doc/todo/Use_a_remote_as_a_sharing_site_for_files_with_obfuscated_URLs/comment_1_1a1f34f4f389267d67e79409c0ca8b1d._comment
new file mode 100644
index 0000000000..35f7351915
--- /dev/null
+++ b/doc/todo/Use_a_remote_as_a_sharing_site_for_files_with_obfuscated_URLs/comment_1_1a1f34f4f389267d67e79409c0ca8b1d._comment
@@ -0,0 +1,9 @@
+[[!comment format=mdwn
+ username="http://edheil.wordpress.com/"
+ ip="99.54.57.201"
+ subject="comment 1"
+ date="2013-02-09T22:38:56Z"
+ content="""
+This would be an extremely cool feature to rip off from Dropbox. :)
+
+"""]]
diff --git a/doc/todo/Use_a_remote_as_a_sharing_site_for_files_with_obfuscated_URLs/comment_2_735afa6f87a93cdf333c17da32010620._comment b/doc/todo/Use_a_remote_as_a_sharing_site_for_files_with_obfuscated_URLs/comment_2_735afa6f87a93cdf333c17da32010620._comment
new file mode 100644
index 0000000000..b2e6c0dace
--- /dev/null
+++ b/doc/todo/Use_a_remote_as_a_sharing_site_for_files_with_obfuscated_URLs/comment_2_735afa6f87a93cdf333c17da32010620._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="https://id.koumbit.net/anarcat"
+ ip="2001:1928:1:9::1"
+ subject="comment 2"
+ date="2014-04-07T04:35:39Z"
+ content="""
+i wonder if the assistant or the new daemon couldn't do this job as well... -- [[anarcat]]
+"""]]
diff --git a/doc/todo/When_autostarting_the_assistant_on_boot_delay_the_execution_2min.mdwn b/doc/todo/When_autostarting_the_assistant_on_boot_delay_the_execution_2min.mdwn
new file mode 100644
index 0000000000..08a24bcb1b
--- /dev/null
+++ b/doc/todo/When_autostarting_the_assistant_on_boot_delay_the_execution_2min.mdwn
@@ -0,0 +1,34 @@
+I think the assistant should have a delay before starting so it doesn't impact on the system boot performance.
+
+I know this can be configured per user (I have it set to 5min), but I think 2min is a sane default.
+
+I don't really know how to submit patches (I can't attach it), so I simply copy it here. It's trivial enough.
+
+[[!format txt """
+From bc6b90ce333659bd2511eedd4ab9067241f73780 Mon Sep 17 00:00:00 2001
+From: =?UTF-8?q?Jon=20Ander=20Pe=C3=B1alba?= 
+Date: Wed, 28 Oct 2015 12:00:51 +0100
+Subject: [PATCH] When autostarting the assistant on boot delay the execution
+ 2min
+
+---
+ Assistant/Install/AutoStart.hs | 2 +-
+ 1 file changed, 1 insertion(+), 1 deletion(-)
+
+diff --git a/Assistant/Install/AutoStart.hs b/Assistant/Install/AutoStart.hs
+index 5745030..6a09c3a 100644
+--- a/Assistant/Install/AutoStart.hs
++++ b/Assistant/Install/AutoStart.hs
+@@ -35,6 +35,6 @@ fdoAutostart command = genDesktopEntry
+ 	"Git Annex Assistant"
+ 	"Autostart"
+ 	False
+-	(command ++ " assistant --autostart")
++	(command ++ " assistant --autostart --startdelay=120")
+ 	Nothing
+ 	[]
+-- 
+2.6.2
+
+
+"""]]
diff --git a/doc/todo/When_autostarting_the_assistant_on_boot_delay_the_execution_2min/comment_1_49990eaf4a3d6ee89f668dfc159f5eae._comment b/doc/todo/When_autostarting_the_assistant_on_boot_delay_the_execution_2min/comment_1_49990eaf4a3d6ee89f668dfc159f5eae._comment
new file mode 100644
index 0000000000..129eeac977
--- /dev/null
+++ b/doc/todo/When_autostarting_the_assistant_on_boot_delay_the_execution_2min/comment_1_49990eaf4a3d6ee89f668dfc159f5eae._comment
@@ -0,0 +1,17 @@
+[[!comment format=mdwn
+ username="joey"
+ subject="""comment 1"""
+ date="2015-11-02T15:14:20Z"
+ content="""
+There's actually an undocumented default startdelay of 5 seconds when using
+autostart. I've now documented it.
+
+Maybe 5 seconds is too little. OTOH, 2 minutes may be a bit too much;
+if the user is booting up their computer to access a new file in their
+annex they'd expect it to start syncing ASAP.
+
+I think my justification for the 5 seconds default was that about how long
+it seems to take a typical desktop system to get loaded (after the login
+screen) and for the user to open a web browser, on reasonably modern
+hardware. Even windows in an emulator doesn't take much longer than that.
+"""]]
diff --git a/doc/todo/Wishlist__58___Parity_files_on_all_files.mdwn b/doc/todo/Wishlist__58___Parity_files_on_all_files.mdwn
new file mode 100644
index 0000000000..01ebf8ce52
--- /dev/null
+++ b/doc/todo/Wishlist__58___Parity_files_on_all_files.mdwn
@@ -0,0 +1,65 @@
+To make sure we can archive our data safely, we need to:
+
+- Store revisions
+- Allow files to be tracked while moved to archival spaces
+- Be platform-agnostic
+- Sync
+- Protect against bit-rot
+
+
+1 and 3 are handled by git itself; everything is a straight forward graph-structure comprised of plain text pointers *(accepting that some filesystems do not easily expose file metadata, but that's on them as we can simply chose to use a different system if that's important)
+
+2 and 4 seem to be handled by git-annex
+
+**But 5 is missing.**
+
+
+Thankfully, we already have a technology that can fill in elegantly here: parity files.
+
+
+### 2 potential user stories:
+
+#### Put everything together
+
+- This user wants everything together and in the filesystem in case one of the tools she relies on disappears.
+- Might have a structure like this:
+   - Project
+     - documents
+       - contract.pdf
+       - contract.pdf.vol000+01.par2
+       - contract.pdf.vol001+02.par2
+       - contract.pdf.vol003+04.par2
+       - Client brochure.zip
+       - Client brochure.zip.vol000+01.par2
+       - Client brochure.zip.vol001+02.par2
+       - Client brochure.zip.vol003+04.par2
+
+- Or like this:
+   - Project
+     - documents
+       - contract.pdf
+       - Client brochure.zip
+     - documents.vol000+01.par2
+     - documents.vol001+02.par2
+     - documents.vol003+04.par2
+
+
+
+#### Keep everything clean
+
+- This user doesn't want to clutter folders with extra files. He would rather only have the data files themselves in case they need to be zipped and sent to clients. If he had setup 1, he would delete *.par before zipping, leading to potential data loss.
+- Might have a structure like this:
+   - Project
+     - documents
+       - contract.pdf
+       - Client brochure.zip
+   - [git-annex]
+       - contract.pdf.vol000+01.par2
+       - contract.pdf.vol001+02.par2
+       - contract.pdf.vol003+04.par2
+       - Client brochure.zip.vol000+01.par2
+       - Client brochure.zip.vol001+02.par2
+       - Client brochure.zip.vol003+04.par2
+
+
+This would also enhance the data-checking capabilities of git-annex, as data loss could be fixed and new parity files generated from the recovered files transparently, self-healing the archive.
diff --git a/doc/todo/Wishlist__58___Parity_files_on_all_files/comment_1_3900fa7c3ed1455fe07d097c3dc0c9f2._comment b/doc/todo/Wishlist__58___Parity_files_on_all_files/comment_1_3900fa7c3ed1455fe07d097c3dc0c9f2._comment
new file mode 100644
index 0000000000..86b1111d96
--- /dev/null
+++ b/doc/todo/Wishlist__58___Parity_files_on_all_files/comment_1_3900fa7c3ed1455fe07d097c3dc0c9f2._comment
@@ -0,0 +1,21 @@
+[[!comment format=mdwn
+ username="joey"
+ subject="""comment 1"""
+ date="2017-01-31T16:55:27Z"
+ content="""
+git repositories don't contain parity files for their data. Instead, git
+relies on multiple copies of the repository to keep things safe. Not as
+efficient as parity files, but a lot easier, and protects against many more
+disasters than do parity files. git-annex takes the same approach.
+Lots Of Copies Keeps Stuff Safe.
+
+Even if git-annex started generating parity files for its objects,
+the git repository would still not have them, so bit flips could still
+corrupt your git-annex repository.
+
+Nothing stops you from writing git hooks that maintain parity files
+alongside all the files in a git repository. If you do that, you'll get
+parity files for the git-annex files too. But I don't see this being needed
+in git-annex itself and AFAICS there are plenty of hooks in git and
+git-annex to allow doing that.
+"""]]
diff --git a/doc/todo/Wishlist__58___Parity_files_on_all_files/comment_2_60dc7823a1814af18861b459a9b3cd0e._comment b/doc/todo/Wishlist__58___Parity_files_on_all_files/comment_2_60dc7823a1814af18861b459a9b3cd0e._comment
new file mode 100644
index 0000000000..637e4a5b3f
--- /dev/null
+++ b/doc/todo/Wishlist__58___Parity_files_on_all_files/comment_2_60dc7823a1814af18861b459a9b3cd0e._comment
@@ -0,0 +1,16 @@
+[[!comment format=mdwn
+ username="joshfindit"
+ avatar="http://cdn.libravatar.org/avatar/180f1a763647bfc099e97ac88b8f7b37"
+ subject="comment 2"
+ date="2017-02-04T16:24:32Z"
+ content="""
+Unless I'm not understanding correctly, Git and git-annex have different expected use-cases.
+
+With Git, it assumed that you will have a repository for each contained project, and keep the number of files small for easy replication across devices (which leads to multiple copies)
+With git-annex, it's possible (and seems more convenient for setup/clients) to have a monolithic repository where parts of it are replicated to devices.
+Yes, it's best practice to have multiple complete copies, but as the repository grows to 4TB, 12TB, or more, it's much less likely especially for a 'I guess I'm not a casual user any more' user who has simply been purchasing hard drives and maybe a NAS when needed.
+
+
+Certainly, I'm just being selfish: git-annex looks like an excellent answer to my ongoing question of 'how could I do files better', and this suggestion is squarely aimed at my personal wishlist. If checking integrity and self-healing large-file repositories doesn't interest anyone else then I'll look in to rolling my own solution to share.
+In the meantime; which git-annex hooks do you suggest I look at? (Bonus points if you can share the high-level logic that would make this work)
+"""]]
diff --git a/doc/todo/Wishlist__58___Parity_files_on_all_files/comment_3_d69eba0afc48c49bdcd7f1f6a7716bbb._comment b/doc/todo/Wishlist__58___Parity_files_on_all_files/comment_3_d69eba0afc48c49bdcd7f1f6a7716bbb._comment
new file mode 100644
index 0000000000..06d9ed17a2
--- /dev/null
+++ b/doc/todo/Wishlist__58___Parity_files_on_all_files/comment_3_d69eba0afc48c49bdcd7f1f6a7716bbb._comment
@@ -0,0 +1,10 @@
+[[!comment format=mdwn
+ username="CandyAngel"
+ avatar="http://cdn.libravatar.org/avatar/15c0aade8bec5bf004f939dd73cf9ed8"
+ subject="comment 3"
+ date="2017-02-05T02:20:31Z"
+ content="""
+I've wanted parity files in git-annex for a really long time, but never asked about it because I expected the response it got :P
+
+So there are other people interested in this!
+"""]]
diff --git a/doc/todo/Wishlist__58___Parity_files_on_all_files/comment_4_01c382342237e4713ab32a3e95497926._comment b/doc/todo/Wishlist__58___Parity_files_on_all_files/comment_4_01c382342237e4713ab32a3e95497926._comment
new file mode 100644
index 0000000000..0a341158dc
--- /dev/null
+++ b/doc/todo/Wishlist__58___Parity_files_on_all_files/comment_4_01c382342237e4713ab32a3e95497926._comment
@@ -0,0 +1,15 @@
+[[!comment format=mdwn
+ username="joey"
+ subject="""comment 4"""
+ date="2017-02-07T17:02:03Z"
+ content="""
+The post-update-annex hook is run whenever the git-annex branch changes,
+so any change to the annexed files stored in the repository will be
+followed by this hook running.
+
+But then how would that hook figure out which content files have been added,
+in order to add parity files for them? Well, someone else had a similar
+type of redundancy information they wanted to add, see
+[[Log_function_to_enumerate_all_recent_git-annex_changes]] for a thread
+about it.
+"""]]
diff --git a/doc/todo/Wishlist__58___additional_environment_variables_for_hooks.mdwn b/doc/todo/Wishlist__58___additional_environment_variables_for_hooks.mdwn
new file mode 100644
index 0000000000..6e852b9f25
--- /dev/null
+++ b/doc/todo/Wishlist__58___additional_environment_variables_for_hooks.mdwn
@@ -0,0 +1,14 @@
+It would be nice if a couple of additional environment variables to be set for hook uses.
+
+In particular:
+
+    GIT_ANNEX_DIRECT=`git config annex.direct`
+
+and
+
+    GIT_TOP_LEVEL=`git rev-parse --show-toplevel`
+
+
+I've made some changes to flickrannex to allow the sub-directories above the uploaded image to be added as tags.  This change has been merged into trunk: [[https://github.com/TobiasTheViking/flickrannex]]
+
+What I needed was both the environment variables mentioned above.  One is set as part of the annex-hook and the other I guestimate from the file path.  If it was set in git-annex it would be much cleaner (and accurate).  So...I think this info would be useful for other hook.
diff --git a/doc/todo/Wishlist__58___additional_environment_variables_for_hooks/comment_1_d82cbbb478a81a651fbe6cb8b71c1192._comment b/doc/todo/Wishlist__58___additional_environment_variables_for_hooks/comment_1_d82cbbb478a81a651fbe6cb8b71c1192._comment
new file mode 100644
index 0000000000..4d0409d58e
--- /dev/null
+++ b/doc/todo/Wishlist__58___additional_environment_variables_for_hooks/comment_1_d82cbbb478a81a651fbe6cb8b71c1192._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="http://joeyh.name/"
+ ip="209.250.56.154"
+ subject="comment 1"
+ date="2014-03-18T19:36:08Z"
+ content="""
+Is this still relevant?
+"""]]
diff --git a/doc/todo/Wishlist__58___disable_auto-repair_for_the_assistant.mdwn b/doc/todo/Wishlist__58___disable_auto-repair_for_the_assistant.mdwn
new file mode 100644
index 0000000000..972247c840
--- /dev/null
+++ b/doc/todo/Wishlist__58___disable_auto-repair_for_the_assistant.mdwn
@@ -0,0 +1,3 @@
+Auto-repair takes up a lot of computing power, greatly slowing down the machine. It also still happens very often here, several times a month on different hardware, leading it to become an annoyance (and it would be easier and faster to perform manual repairs if that should be required).
+
+Therefore I'd like a setting that makes the assistant not attempt to repair on its own. Either not do anything at all, or display a message about it instead.
diff --git a/doc/todo/Wishlist__58___disable_auto-repair_for_the_assistant/comment_1_3274820a0d1f10c505f15cd29a37b95a._comment b/doc/todo/Wishlist__58___disable_auto-repair_for_the_assistant/comment_1_3274820a0d1f10c505f15cd29a37b95a._comment
new file mode 100644
index 0000000000..d4839cb1d5
--- /dev/null
+++ b/doc/todo/Wishlist__58___disable_auto-repair_for_the_assistant/comment_1_3274820a0d1f10c505f15cd29a37b95a._comment
@@ -0,0 +1,12 @@
+[[!comment format=mdwn
+ username="http://joeyh.name/"
+ ip="209.250.56.3"
+ subject="comment 1"
+ date="2014-05-21T18:29:16Z"
+ content="""
+What auto-repair are you referring to?
+
+If you schedule a fsck of the repository in the webapp, then when that finds a problem with the git repository, it will be repaired. So if you don't want this, remove any scheduled fsck jobs.
+
+The assistant also detects a few common problems at startup that prevent it from working, such as a corrupt index file, and automatically repairs those. These repairs only happen at startup. If the index file is corrupt, it has to delete it and re-add every file to the repository, which is expensive, but otherwise it would be completely non-functional.
+"""]]
diff --git a/doc/todo/Wishlist__58___disable_auto-repair_for_the_assistant/comment_2_2cf5aef3f1d340c4ed6249ef94c1b607._comment b/doc/todo/Wishlist__58___disable_auto-repair_for_the_assistant/comment_2_2cf5aef3f1d340c4ed6249ef94c1b607._comment
new file mode 100644
index 0000000000..015dd8cd74
--- /dev/null
+++ b/doc/todo/Wishlist__58___disable_auto-repair_for_the_assistant/comment_2_2cf5aef3f1d340c4ed6249ef94c1b607._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="http://joeyh.name/"
+ ip="209.250.56.3"
+ subject="comment 2"
+ date="2014-05-21T18:35:42Z"
+ content="""
+Note that this is asking for a workaround for the bug [[bugs/Auto-repair_greatly_slows_down_the_machine]]. I would rather get to the bottom of that bug, but need more information to do so.
+"""]]
diff --git a/doc/todo/Wishlist__58___disable_auto-repair_for_the_assistant/comment_3_6c53d82e62b2d269a941ba967d05adf5._comment b/doc/todo/Wishlist__58___disable_auto-repair_for_the_assistant/comment_3_6c53d82e62b2d269a941ba967d05adf5._comment
new file mode 100644
index 0000000000..07b1b2557a
--- /dev/null
+++ b/doc/todo/Wishlist__58___disable_auto-repair_for_the_assistant/comment_3_6c53d82e62b2d269a941ba967d05adf5._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="EskildHustvedt"
+ ip="80.202.103.55"
+ subject="comment 3"
+ date="2014-05-21T19:18:19Z"
+ content="""
+I'm referring to the large auto-repairs of the actual git repository, as mentioned in the bug report you are referring to. I was under the impression that the fscks also did the equivalent of 'git annex fsck' (which is greatly desired, even without the 'git fsck' checks). I realize it is a sort of workaround, but it is because the 'annex fsck' bit is desired, while the 'git annex repair' bit makes my computer unusable, the last time for over 24 hours, while it was running (at which point I needed to kill it and perform a manual repair).
+"""]]
diff --git a/doc/todo/Wishlist__58___sanitychecker_fix_wrong_UUID__47__duplicate_remote.mdwn b/doc/todo/Wishlist__58___sanitychecker_fix_wrong_UUID__47__duplicate_remote.mdwn
new file mode 100644
index 0000000000..ff7773d3e6
--- /dev/null
+++ b/doc/todo/Wishlist__58___sanitychecker_fix_wrong_UUID__47__duplicate_remote.mdwn
@@ -0,0 +1,7 @@
+In certain situations different client annexes might get the same remote repository added, but before being synced.
+
+Once the two clients sync they will both have two remotes with the same name. But only one UUID will have any content(Assuming only one client pushed).
+
+It would be nice to have some (automatic?) way to resolve this conflict.
+
+Not sure if anything sane can be done if both clients have pushed?
diff --git a/doc/todo/Workflow_guide.mdwn b/doc/todo/Workflow_guide.mdwn
new file mode 100644
index 0000000000..5cc971b13d
--- /dev/null
+++ b/doc/todo/Workflow_guide.mdwn
@@ -0,0 +1,15 @@
+I try to use git annex, but I frequently don't know if I'm doing things correctly, and my files are getting messed up.
+
+An expert guide to the full workflow would go a long way toward user-friendliness for me. The walkthrough currently has guides to a number discrete items in the workflow, but it doesn't give me a clear sense of the process.
+
+I'm always confused about when I'm supposed to be using pure git commands and when they should be git annex commands, when to commit, add, and sync --content, and when each of these is redundant.
+
+If possible, most helpful would be a guide to how you imagine the workflow from the beginning and including each step of the process, in the order you'd do it.
+
+I want to start keeping track of some files I have in a directory
+I want to copy them to a second computer.
+From a third place, I want to get them from the second computer.
+I change the files on one computer, and I want to make sure the changes get synced to the others.
+What are the commands you'd run at each step?
+
+Many thanks.
diff --git a/doc/todo/Workflow_guide/comment_1_f4242a3bdeffb90336f562f9ec182e5d._comment b/doc/todo/Workflow_guide/comment_1_f4242a3bdeffb90336f562f9ec182e5d._comment
new file mode 100644
index 0000000000..40b109ce0a
--- /dev/null
+++ b/doc/todo/Workflow_guide/comment_1_f4242a3bdeffb90336f562f9ec182e5d._comment
@@ -0,0 +1,9 @@
+[[!comment format=mdwn
+ username="joey"
+ subject="""comment 1"""
+ date="2016-09-21T16:52:10Z"
+ content="""
+I think this is a good idea for an extension to the walkthrough. I
+am probably the worst person to write it. How about, you write it, and I'll
+fact check and edit it.
+"""]]
diff --git a/doc/todo/Workflow_guide/comment_2_ee6da4cc639bca0b17156d51ebfc05c2._comment b/doc/todo/Workflow_guide/comment_2_ee6da4cc639bca0b17156d51ebfc05c2._comment
new file mode 100644
index 0000000000..a396da0784
--- /dev/null
+++ b/doc/todo/Workflow_guide/comment_2_ee6da4cc639bca0b17156d51ebfc05c2._comment
@@ -0,0 +1,29 @@
+[[!comment format=mdwn
+ username="xloem"
+ subject="Workflow Summary"
+ date="2016-10-09T13:39:38Z"
+ content="""
+I'll try to address some of this.
+
+Git-annex supports a wide variety of workflows, a spectrum that ranges from completely automatic behavior where git-annex handles everything, through manual behavior where git-annex does only what you say, when you tell it to, down to internal behavior, where you have complete control and understand how everything is stored and exactly what changes are happening.
+
+I will proceed to summarize all of these.  Your question sounds like a high-level question, so I will begin at the automatic end, hoping that this is most useful, and drill down to the low level approaches.  Note, however, that this the opposite order of how git-annex was apparently developed.  A list of workflows that started from manual, commandline usage would be much more intuitive, but you'd have to be willing to read the man page and wiki pages to get started, and that's pretty much what's already out there anyway.
+
+Note that for each of these levels of interaction, all the levels _following_ will also work as well.  So you can actually manually move annexed files around while the webapp is running, etc.
+
+(0) `git annex webapp`.  This command launches a local web server which serves a graphical user interface and automatically manages git annex.  It will attempt to guide you through the whole process and do everything for you.  I think the intent is that no other commands are needed.  This should be run on every machine that may produce file changes.
+
+(1) `git annex assistant` without running the webapp.  You could call this the command-line version of the webapp, giving you more control over creating and connecting your repositories, and configuring how files are moved between them.  The assistant, when running, will automatically watch for file changes and synchronize them to other repositories, but you must manually create the repositories and configure the rules for syncing.  To create a repository, use `git init` and then `git annex init`, and then `git remote add` it to any other repositories.  If you want more than one annex, you can add their paths to ~/.config/git-annex/autostart if you would like them to automatically begin syncing when `git annex assistant --autostart` is run, perhaps on boot or login.  You can configure rules for where files are copied using the repository setup commands such as preferred content expressions, `git annex numcopies`, and `git config annex.largefiles`; most of the settings are accessible in one place with `git annex vicfg`.
+
+(2) `git annex watch` without running the assistant.  This command is like the assistant but has no automatic network behavior, giving you complete control over when repositories are pushed and pulled, and when files are moved between systems.  The repository is watched, and any file changes are added to git-annex.  In order to synchronize between repositories, you must run `git annex sync --content` in the repository with the changes, which will merge the git history and logs with your remotes, and automatically transfer files to match your preferred and required content expressions.
+
+(3) No background processes, allowing you to decide when and what files are annexed.  In order to tell git-annex to manage files, you must `git annex add` the files.
+
+(4) Plain `git annex sync` without `--content`, giving you fine-grained control of where copies of your files are stored.  This tells git-annex to merge git histories, but it does not automatically transfer your large files between systems.  To transfer files and directories, you can use `git annex get`, `git annex drop`, `git annex move`, and `git annex copy`.  Git-annex will not violate a required content expression or your numcopies setting unless you pass --force, so your files are still safe.  This is the workflow I mostly use, and I find it the most stable.  I'm trying to migrate up to `--content`, but I have too many large files that haven't reached their numcopies yet for that to be effective.
+
+(5) Manual management of git history without running the syncronizer, allowing you to control precisely what is committed, what commit message is used, and how your history is merged between repositories.  You must have an understanding of git, and run `git commit` after `git annex add` to store the change.  You must manage the git history yourself, using `git pull` and `git push`, to synchronize repositories.  You may freely use git normally side-by-side with git-annex.
+
+(6) Manual management of git annex keys, giving you control of what and where git annex stores your files under the hood, and how they are associated with your working tree, rather than using the `git annex add` and `git annex get` commands which reference files automatically.  Git-annex has a variety of plumbing commands listed in the man page that let you directly store and retrieve data in an annex associated with your git repository, where every datafile is enumerated by a unique hashkey.
+
+There are a lot of possible workflows with git annex.  It provides an array of tools that allow you to both completely automate things and have very fine control of them.  The more you can learn, the more power you will have.
+"""]]
diff --git a/doc/todo/Workflow_guide/comment_3_8cf0f9913dc307cd18001656ba390159._comment b/doc/todo/Workflow_guide/comment_3_8cf0f9913dc307cd18001656ba390159._comment
new file mode 100644
index 0000000000..ff5fa0327a
--- /dev/null
+++ b/doc/todo/Workflow_guide/comment_3_8cf0f9913dc307cd18001656ba390159._comment
@@ -0,0 +1,12 @@
+[[!comment format=mdwn
+ username="joey"
+ subject="""comment 3"""
+ date="2016-10-17T19:42:08Z"
+ content="""
+@xloem nice start!
+
+It would be helpful to get this into a set of wiki pages documenting the
+workflows in detail. Since this site is a wiki, you can add pages as you like.
+Start by clicking the question mark to create [[/workflow]], and see
+[[ikiwiki/wikilink]] for how to add sub-pages like [[/workflow/assistant]].
+"""]]
diff --git a/doc/todo/Workflow_guide/comment_4_b6f5ce361529356a77b0e6141a62c06d._comment b/doc/todo/Workflow_guide/comment_4_b6f5ce361529356a77b0e6141a62c06d._comment
new file mode 100644
index 0000000000..6127e3e8d7
--- /dev/null
+++ b/doc/todo/Workflow_guide/comment_4_b6f5ce361529356a77b0e6141a62c06d._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="marekj"
+ avatar="http://cdn.libravatar.org/avatar/65a60e8f5183feeeef8cef815bf73e61"
+ subject="I took the liberty to do it"
+ date="2016-12-14T07:49:26Z"
+ content="""
+I simply copied @xloem's into a new [[workflow]] page. I have been looking for such a guide myself for quite some time.
+"""]]
diff --git a/doc/todo/Workflow_guide/comment_5_6ec6fb45021ba82ed6a4bb9a6f3cfceb._comment b/doc/todo/Workflow_guide/comment_5_6ec6fb45021ba82ed6a4bb9a6f3cfceb._comment
new file mode 100644
index 0000000000..fe30f61064
--- /dev/null
+++ b/doc/todo/Workflow_guide/comment_5_6ec6fb45021ba82ed6a4bb9a6f3cfceb._comment
@@ -0,0 +1,19 @@
+[[!comment format=mdwn
+ username="joey"
+ subject="""comment 5"""
+ date="2016-12-20T19:04:12Z"
+ content="""
+Good start on the workflow page!
+
+I've added some links to it to make it discoverable.
+
+Not sure if the workflow page quite gets to what was originally requested:
+
+> I want to start keeping track of some files I have in a directory
+> I want to copy them to a second computer.
+> From a third place, I want to get them from the second computer.
+> I change the files on one computer, and I want to make sure the changes get synced to the others.
+> What are the commands you'd run at each step?
+
+Leaving this todo open for now..
+"""]]
diff --git a/doc/todo/Workflow_guide/comment_6_640e5c6cdea8a6fae63c3fab6970f1f2._comment b/doc/todo/Workflow_guide/comment_6_640e5c6cdea8a6fae63c3fab6970f1f2._comment
new file mode 100644
index 0000000000..9eae8e9119
--- /dev/null
+++ b/doc/todo/Workflow_guide/comment_6_640e5c6cdea8a6fae63c3fab6970f1f2._comment
@@ -0,0 +1,10 @@
+[[!comment format=mdwn
+ username="joey"
+ subject="""comment 6"""
+ date="2016-12-21T18:19:07Z"
+ content="""
+In a way the use cases on the front page of the website are trying to
+accomplish the same thing requested here. I think that section could be
+moved more in the direction of listing some ways to use git-annex and
+linking to walkthroughs for the different use cases.
+"""]]
diff --git a/doc/todo/Workflow_guide/comment_7_805e877b23adcf562feeb7d927fca08d._comment b/doc/todo/Workflow_guide/comment_7_805e877b23adcf562feeb7d927fca08d._comment
new file mode 100644
index 0000000000..01ad920cec
--- /dev/null
+++ b/doc/todo/Workflow_guide/comment_7_805e877b23adcf562feeb7d927fca08d._comment
@@ -0,0 +1,15 @@
+[[!comment format=mdwn
+ username="https://anarc.at/openid/"
+ nickname="anarcat"
+ avatar="http://cdn.libravatar.org/avatar/b36dcf65657dd36128161355d8920a99503def9461c1bb212410980fe6f07125"
+ subject="answers"
+ date="2017-01-17T16:56:39Z"
+ content="""
+i actually answered this in [[forum/Full_workflow_guide]] before realizing this post was a duplicate.
+
+i am not sure where that should be put - it's not a \"full workflow guide\", it's actually a \"specific workflow guide\", namely \"i have 3 computers and need to sync files from A to B to C\"... 
+
+i wonder if this should be marked as resolved or if we want to add a specific workflow for this or what?
+
+this reminds me of my broader [[todo/build_a_user_guide]] task... -- [[anarcat]]
+"""]]
diff --git a/doc/todo/__39__info_filename__39___to_provide_information_either_content_is_locally_present.mdwn b/doc/todo/__39__info_filename__39___to_provide_information_either_content_is_locally_present.mdwn
new file mode 100644
index 0000000000..5e05988168
--- /dev/null
+++ b/doc/todo/__39__info_filename__39___to_provide_information_either_content_is_locally_present.mdwn
@@ -0,0 +1,6 @@
+ATM in DataLad we rely on 'git annex find' to determine either files have content locally.  Even though it could be used in a batch mode, I wondered if we could may be just use 'annex info' to obtain information either a file (or a key) has content locally?  Another benefit would be is that within single command output we could determine also if a file under annex or not (instead of first doing e.g. 'info' to figure out if under annex and then 'find' again to figure out if content is present locally)
+
+
+[[!meta author=yoh]]
+
+> sure, [[done]] --[[Joey]]
diff --git a/doc/todo/__91__FR__93___No_consistency_check_while_on_battery-only.mdwn b/doc/todo/__91__FR__93___No_consistency_check_while_on_battery-only.mdwn
new file mode 100644
index 0000000000..815ce6a1b7
--- /dev/null
+++ b/doc/todo/__91__FR__93___No_consistency_check_while_on_battery-only.mdwn
@@ -0,0 +1,14 @@
+I hope this has not been mentioned before.  I only found in mentioned in a comment of a devblog post.
+
+### Please describe the problem.
+With `git annex assistant` running, consistency checks is sometimes triggered while the computer is battery-powered (as opposed to plugged into AC).
+
+### What steps will reproduce the problem?
+Configure `git annex assistant` to make consistency check daily (or whatever the default is), unplug from power and wait.
+
+### What version of git-annex are you using? On what operating system?
+`5.20141231` on Archlinux 64 bit.  (From the changelog I guess it's also present in the newest version).
+
+### Please provide any additional information below.
+
+It would be nice if one could toggle whether consistency check may run while not plugged into AC.
diff --git a/doc/todo/__91__FR__93___No_consistency_check_while_on_battery-only/comment_1_b557db02c3719152d392fa454c9c5ce5._comment b/doc/todo/__91__FR__93___No_consistency_check_while_on_battery-only/comment_1_b557db02c3719152d392fa454c9c5ce5._comment
new file mode 100644
index 0000000000..99de27120f
--- /dev/null
+++ b/doc/todo/__91__FR__93___No_consistency_check_while_on_battery-only/comment_1_b557db02c3719152d392fa454c9c5ce5._comment
@@ -0,0 +1,15 @@
+[[!comment format=mdwn
+ username="joey"
+ subject="""comment 1"""
+ date="2015-02-04T19:48:10Z"
+ content="""
+I agree this would be nice. However, how to detect if it's on battery?
+
+Debian has a `on_ac_power` command which is rather complicated; 88 lines of
+code that deal with APM, PMU (powerpc), and two different kernel interfaces
+for ACPI (new sysfs and old /proc/acpi).
+
+Then there's OSX, Windows, Android..
+
+I'm going to move this from bugs to todo.
+"""]]
diff --git a/doc/todo/__91__FR__93___No_consistency_check_while_on_battery-only/comment_2_374567f37635613a5d671c877d601367._comment b/doc/todo/__91__FR__93___No_consistency_check_while_on_battery-only/comment_2_374567f37635613a5d671c877d601367._comment
new file mode 100644
index 0000000000..bf6179b72d
--- /dev/null
+++ b/doc/todo/__91__FR__93___No_consistency_check_while_on_battery-only/comment_2_374567f37635613a5d671c877d601367._comment
@@ -0,0 +1,7 @@
+[[!comment format=mdwn
+ username="rasmus"
+ subject="comment 2"
+ date="2015-02-10T18:33:20Z"
+ content="""
+Yeah, probably it's more complicated than just checking `/sys/class/power_supply/BAT0/`...  There's something called [`pmset`](https://developer.apple.com/library/mac/documentation/Darwin/Reference/ManPages/man1/pmset.1.html) for macs, but I don't have a mac so I don't know how relevant it is.  For Windows there seems also to be some sort of [api](https://msdn.microsoft.com/en-us/library/windows/desktop/aa372659%28v=vs.85%29.aspx), but again I don't know anything about windows...
+"""]]
diff --git a/doc/todo/accellerate_ssh_remotes_with_git-annex-shell_mass_protocol.mdwn b/doc/todo/accellerate_ssh_remotes_with_git-annex-shell_mass_protocol.mdwn
new file mode 100644
index 0000000000..25dea6eef5
--- /dev/null
+++ b/doc/todo/accellerate_ssh_remotes_with_git-annex-shell_mass_protocol.mdwn
@@ -0,0 +1,57 @@
+As shown by benchmarks in
+*[[here|todo/speed_up_transfers_over_ssh+rsync_--_directly_reuse_the_same_connection__63__]]*,
+there is some overhead for each file transfer to a rsync special remote, to
+set up the connection. Idea is to extend git-annex-shell with a command or
+commands that don't use rsync for transferring objects, and that can handle
+transferring or otherwise operating on multiple objects inside a single tcp
+session.
+
+This might only be used when it doesn't need to resume transfer of a file;
+it could fall back to rsync for resuming.
+
+Of course, when talking with a git-annex-shell that does not support this
+new command, git-annex would still need to fall back to the old commands
+using rsync. And should remember for the session that the remote doesn't
+support the new command.
+
+It could use sftp, but that seems kind of difficult; it would need to lock
+down sftp-server to only write annexed objects to the right repository.
+And, using sftp would mean that git-annex would need to figure out the
+filenames to use for annexed objects in the remote repository, rather than
+letting git-annex-shell on the remote work that out.
+
+So, it seems better to not use sftp, and instead roll our own simple
+file transfer protocol.
+
+So, "git-annex-shell -c p2pstdio" would speak a protocol over stdin/stdout
+that essentially contains the commands inannex, lockcontent, dropkey,
+recvkey, and sendkey.
+
+P2P.Protocol already contains such a similar protocol, used over tor.
+That protocol even supports resuming interrupted transfers.
+It has stuff including auth that this wouldn't need, but it would be
+good to unify with it as much as possible.
+
+----
+
+Benchmarks
+
+Dropping 200 files from a remote over a satellite internet connection,
+speed increased from 364s to 183s.
+
+Dropping 1000 files from a remote over ssh to localhost, with -J4, 
+speed increased from 20s to 6s. Without -J4, speed increased from 41s to
+10s.
+
+(By comparison, dropping 1000 files from a remote on the same filesystem
+took 12s, so remote access over localhost seems faster now! Possibly
+there's a little bit more concurrency when git-annex and git-annex-shell
+are both running?)
+
+Transferring a 30 mb file over ssh to localhost, speed increased from
+3.288s to 3.031s. Due to rsync's several levels of checksumming?
+
+Dropping 1000 files from local, with each being locked on a ssh localhost
+remote, speed increased from 30s to 7s.
+
+[[done]]
diff --git a/doc/todo/accellerate_ssh_remotes_with_git-annex-shell_mass_protocol/comment_1_504f0e728edad9b35cd88e8a657fd5bb._comment b/doc/todo/accellerate_ssh_remotes_with_git-annex-shell_mass_protocol/comment_1_504f0e728edad9b35cd88e8a657fd5bb._comment
new file mode 100644
index 0000000000..2de537d423
--- /dev/null
+++ b/doc/todo/accellerate_ssh_remotes_with_git-annex-shell_mass_protocol/comment_1_504f0e728edad9b35cd88e8a657fd5bb._comment
@@ -0,0 +1,21 @@
+[[!comment format=mdwn
+ username="yarikoptic"
+ avatar="http://cdn.libravatar.org/avatar/f11e9c84cb18d26a1748c33b48c924b4"
+ subject="comment 1"
+ date="2018-03-06T20:33:44Z"
+ content="""
+only marginally related since may be the abstraction level is at different level? but thought to add thoughts on \"bundling\" also calls to get stuff from web remote:
+
+although both curl and wget do allow for multiple URLs to be specified within a single invocation (which manual also recommends for download of multiple files since avoids reestablishing the session) I could not find any way to provide those URLs with target filenames as it would be needed for git-annex to store into corresponding target key files. The only way would be to parse wget output (which IIRC git annex already does for progress reporting), get the target filename and move into a proper location
+
+[[!format sh \"\"\"
+$> wget -iwget-links -nv --show-progress --clobber
+highres001.nii.gz?ve 100%[===================>]   5.40M  2.30MB/s    in 2.3s    
+2018-03-06 15:31:49 URL:http://openneuro.s3.amazonaws.com/ds000001/ds000001_R1.1.0/uncompressed/sub001/anatomy/highres001.nii.gz?versionId=8TJ17W9WInNkQPdiQ9vS7wo8ZJ9llF80 [5663237/5663237] -> \"highres001.nii.gz?versionId=8TJ17W9WInNkQPdiQ9vS7wo8ZJ9llF80.2\" [1]
+inplane001.nii.gz?ve 100%[===================>] 653.88K  1.10MB/s    in 0.6s    
+2018-03-06 15:31:50 URL:http://openneuro.s3.amazonaws.com/ds000001/ds000001_R1.1.0/uncompressed/sub001/anatomy/inplane001.nii.gz?versionId=ystKDnaPkdzSwzdRPZH0PtMMknZJCQV4 [669578/669578] -> \"inplane001.nii.gz?versionId=ystKDnaPkdzSwzdRPZH0PtMMknZJCQV4.1\" [1]
+FINISHED --2018-03-06 15:31:50--
+Total wall clock time: 3.2s
+Downloaded: 2 files, 6.0M in 2.9s (2.06 MB/s)
+\"\"\"]]
+"""]]
diff --git a/doc/todo/accellerate_ssh_remotes_with_git-annex-shell_mass_protocol/comment_2_660c9ece3dbfb2eb20b42b71826a4fbc._comment b/doc/todo/accellerate_ssh_remotes_with_git-annex-shell_mass_protocol/comment_2_660c9ece3dbfb2eb20b42b71826a4fbc._comment
new file mode 100644
index 0000000000..7adf5f3a96
--- /dev/null
+++ b/doc/todo/accellerate_ssh_remotes_with_git-annex-shell_mass_protocol/comment_2_660c9ece3dbfb2eb20b42b71826a4fbc._comment
@@ -0,0 +1,13 @@
+[[!comment format=mdwn
+ username="joey"
+ subject="""comment 2"""
+ date="2018-03-09T17:50:24Z"
+ content="""
+That's the same problem as bundling multiple files with rsync, just a
+different remote with a different program.
+
+If wget/curl invocation speed is actually a bottleneck, it might help to
+have git-annex use native haskell http-conduit for http downloads. That
+would allow for http connection pipelining. Please open a separate todo
+though, this one is only about git-annex-shell speed.
+"""]]
diff --git a/doc/todo/adb_special_remote.mdwn b/doc/todo/adb_special_remote.mdwn
new file mode 100644
index 0000000000..f89eeac6dd
--- /dev/null
+++ b/doc/todo/adb_special_remote.mdwn
@@ -0,0 +1,19 @@
+Make a special remote using adb to send file to an android device.
+
+While there is an android port, a special remote will suffice for many use
+cases, and may work better overall.
+
+It should support exporttree=yes, since most use cases involve exporting a
+tree of files and consuming them on the android device.
+
+Use adb push, adb pull, and use adb shell for checkpresent and remove.
+
+Ought to implement [[import tree]] too, so that changes made
+to files on the android device can be imported back into the git
+repository.
+
+And, [[export preferred content]] would be a useful feature for
+excluding some files from a tree exported to android.
+
+> Status: Remote implemented, but not yet [[import tree]] and
+> [[export preferred content]]. --[[Joey]]
diff --git a/doc/todo/add_a_--branch_to_applicable_git-annex_commands.mdwn b/doc/todo/add_a_--branch_to_applicable_git-annex_commands.mdwn
new file mode 100644
index 0000000000..b2678bee81
--- /dev/null
+++ b/doc/todo/add_a_--branch_to_applicable_git-annex_commands.mdwn
@@ -0,0 +1,2 @@
+My original use case was for using git-annex find from scripts, where I didn't want to depend on the branch 
+checked out at the time, but rather write something like "git annex find --branch=master $searchterms"
diff --git a/doc/todo/add_a_--branch_to_applicable_git-annex_commands/comment_1_3e0a1d1c41f317514dfc496f2274ad1c._comment b/doc/todo/add_a_--branch_to_applicable_git-annex_commands/comment_1_3e0a1d1c41f317514dfc496f2274ad1c._comment
new file mode 100644
index 0000000000..6d5320d41a
--- /dev/null
+++ b/doc/todo/add_a_--branch_to_applicable_git-annex_commands/comment_1_3e0a1d1c41f317514dfc496f2274ad1c._comment
@@ -0,0 +1,10 @@
+[[!comment format=mdwn
+ username="http://joeyh.name/"
+ ip="209.250.56.154"
+ subject="comment 1"
+ date="2014-03-17T19:48:57Z"
+ content="""
+The difficulty with adding a --branch is that if it causes git-annex to operate on a list of (file, key) from the branch, then commands that actually modify the working tree would modify it, instead of the branch. So the options seem to be only generating a list of keys, and so only letting commands that operate on keys work (which rules out the `git annex find` example), or carefully arranging for commands that actually affect the work tree to not be usable with this option.
+
+I'm not sure how many commands are affected. The ones I can immediately think of are sync, lock, unlock. (Commands like get obviously affect the work tree in direct mode, but it's fine to have getting a file from a branch also update files in the work tree, if they pointed at the same key.)
+"""]]
diff --git a/doc/todo/add_ancient_armel_build.mdwn b/doc/todo/add_ancient_armel_build.mdwn
new file mode 100644
index 0000000000..52ce1a73da
--- /dev/null
+++ b/doc/todo/add_ancient_armel_build.mdwn
@@ -0,0 +1,10 @@
+Add an armel build like the i386ancient build.
+
+The current arm autobuilder doesn't have enough free space for the chroot
+this would need. I need to upgrade its microsd card first, adding
+approximately 5 gb. (A 16 gb card would suffice.)
+
+Now, it would be possible to switch it to only do an ancient build,
+instead of the current modern build. The downside of that is the ancient
+build environment uses debian stable, so it has old versions of git,
+libraries etc, that go into the build. --[[Joey]]
diff --git a/doc/todo/add_ancient_armel_build/comment_1_704f1a854e5ef62edbca60ab209d2123._comment b/doc/todo/add_ancient_armel_build/comment_1_704f1a854e5ef62edbca60ab209d2123._comment
new file mode 100644
index 0000000000..28842b5c08
--- /dev/null
+++ b/doc/todo/add_ancient_armel_build/comment_1_704f1a854e5ef62edbca60ab209d2123._comment
@@ -0,0 +1,14 @@
+[[!comment format=mdwn
+ username="rustikus@9db90d0c115a1825e2f1e5f15257ec1298a6c7b6"
+ nickname="rustikus"
+ avatar="http://cdn.libravatar.org/avatar/6fe372a7a46198829723c408af75c8cc"
+ subject="Any update or help needed?"
+ date="2017-01-12T13:40:38Z"
+ content="""
+Hi,
+
+is there any update? I do have an older version of Synology DiskStation (DS211+) which does have this CPU. Anything I can do to help?
+Disclaimer: I am quite familiar with Linux but I am no expert in programming with haskel.
+
+Thanks!
+"""]]
diff --git a/doc/todo/add_ancient_armel_build/comment_2_0b02f96699e9ec9447e336b74b255801._comment b/doc/todo/add_ancient_armel_build/comment_2_0b02f96699e9ec9447e336b74b255801._comment
new file mode 100644
index 0000000000..5964c7a486
--- /dev/null
+++ b/doc/todo/add_ancient_armel_build/comment_2_0b02f96699e9ec9447e336b74b255801._comment
@@ -0,0 +1,9 @@
+[[!comment format=mdwn
+ username="http://schnouki.net/"
+ nickname="Schnouki"
+ avatar="http://cdn.libravatar.org/avatar/5f6406e9db28564121169f0051645b8c30a12a20ca7bc40287ac9bf2cd3ad283"
+ subject="comment 2"
+ date="2017-02-02T22:05:23Z"
+ content="""
++1. Can't upgrade the kernel on my NAS (Synology DS413j, running Linux 2.6.32...), so I'm stuck with git-annex 5.20150731...
+"""]]
diff --git a/doc/todo/add_ancient_armel_build/comment_3_eaeb9aec42fde6b8a25199b1e1e6a78c._comment b/doc/todo/add_ancient_armel_build/comment_3_eaeb9aec42fde6b8a25199b1e1e6a78c._comment
new file mode 100644
index 0000000000..b645d7b6c1
--- /dev/null
+++ b/doc/todo/add_ancient_armel_build/comment_3_eaeb9aec42fde6b8a25199b1e1e6a78c._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="joey"
+ subject="""comment 3"""
+ date="2017-02-20T16:22:58Z"
+ content="""
+The build is set up; it's not run to completion yet, but fingers crossed it
+will eventually.
+"""]]
diff --git a/doc/todo/add_option_to_whereis_to_avoid_network_interactions.mdwn b/doc/todo/add_option_to_whereis_to_avoid_network_interactions.mdwn
new file mode 100644
index 0000000000..c322660dfc
--- /dev/null
+++ b/doc/todo/add_option_to_whereis_to_avoid_network_interactions.mdwn
@@ -0,0 +1,21 @@
+I thought that whereis command would report only based on the knowledge annex has locally in git-annex branch, but apparently it is trying to query for information even in --fast mode:
+
+[[!format sh """
+$> git annex whereis --fast bold.nii.gz
+yoh@dat....
+Permission denied (publickey,password).
+fatal: Could not read from remote repository.
+
+Please make sure you have the correct access rights
+and the repository exists.
+whereis bold.nii.gz 
+(2 copies) 
+  	899f0347-0888-48ef-91b6-bac213ca8cef -- [datalad-archives]
+   	c8bd3d05-33d4-4b59-9d53-ca7efbdcdd13 -- yoh@smaug:/mnt/btrfs/datasets/datalad/crawl/openfmri/ds000001 [here]
+
+  datalad-archives: dl+archive:MD5E-s2527262329--bd3ea399057c529b37b09dcecec1ca60.0raw.tgz/ds001_R1.1.0/sub001/BOLD/task001_run001/bold.nii.gz#size=47241449
+
+"""]]
+
+[[!meta author=yoh]]
+Was [[done]] by [[Joey]] as of 6.20160524+gitg2b7b2c4-1~ndall+1
diff --git a/doc/todo/add_sftp_special_remote.mdwn b/doc/todo/add_sftp_special_remote.mdwn
new file mode 100644
index 0000000000..c4dd1e2944
--- /dev/null
+++ b/doc/todo/add_sftp_special_remote.mdwn
@@ -0,0 +1,13 @@
+A sftp special remote would be nice because gpg operations could be
+pipelined to the network transfer, not requiring the creation of a full
+file to disk with gpg before the network transmission, as it happens with
+the rsync special remote.
+
+There should be some libraries that can handle the sftp connections and
+transfers. I read that even curl has support for that.
+
+> Another reason to build this is that sftp has a `SFTP_FXP_STAT`
+> that can get disk free space information. "echo df | sftp user@host"
+> exposes this, when available. Some sftp servers can be locked down
+> so that the user can't run git-annex on them, so that could be the only
+> way to get diskreserve working for such a remote. --[[Joey]]
diff --git a/doc/todo/addurl___8211__force-torrent_option.mdwn b/doc/todo/addurl___8211__force-torrent_option.mdwn
new file mode 100644
index 0000000000..acbb953c0c
--- /dev/null
+++ b/doc/todo/addurl___8211__force-torrent_option.mdwn
@@ -0,0 +1 @@
+There are sites that don't provide direct links to `.torrent` files. Currently there is no way to download contents of such torrents with `git annex`, it simply uses web remote instead of bittorrent.  Something like `--force-torrent` option could help here.
diff --git a/doc/todo/addurl___8211__force-torrent_option/comment_1_15be1914c8d05cd1ad8220bcfea9d0bf._comment b/doc/todo/addurl___8211__force-torrent_option/comment_1_15be1914c8d05cd1ad8220bcfea9d0bf._comment
new file mode 100644
index 0000000000..456fb1ab4a
--- /dev/null
+++ b/doc/todo/addurl___8211__force-torrent_option/comment_1_15be1914c8d05cd1ad8220bcfea9d0bf._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="joey"
+ subject="""comment 1"""
+ date="2015-04-14T19:11:28Z"
+ content="""
+I'd prefer torrent:url; this is consistent with quvi:url for forcing quvi
+be used.
+"""]]
diff --git a/doc/todo/addurl___8211__force-torrent_option/comment_2_52c04c388b807993cecacc7f98b73cd3._comment b/doc/todo/addurl___8211__force-torrent_option/comment_2_52c04c388b807993cecacc7f98b73cd3._comment
new file mode 100644
index 0000000000..f6d52be655
--- /dev/null
+++ b/doc/todo/addurl___8211__force-torrent_option/comment_2_52c04c388b807993cecacc7f98b73cd3._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="https://www.google.com/accounts/o8/id?id=AItOawkutSE8_3fFAETmO_E598zja4gKwYXbb8E"
+ nickname="Сергей"
+ subject="comment 2"
+ date="2015-04-14T19:57:52Z"
+ content="""
+Sure, that's even better.
+"""]]
diff --git a/doc/todo/addurl_should_fail_when_youtube-dl_disabled.mdwn b/doc/todo/addurl_should_fail_when_youtube-dl_disabled.mdwn
new file mode 100644
index 0000000000..1d6cf80778
--- /dev/null
+++ b/doc/todo/addurl_should_fail_when_youtube-dl_disabled.mdwn
@@ -0,0 +1,7 @@
+When using addurl on an url that youtube-dl supports, if it's installed but
+not enabled to be used due to the recent security fix, git-annex will
+download the web page and add it, which is unlikely to be desired behavior.
+Instead, it should check if youtube-dl supports the page, and then error
+out at the download stage, with a message that points at how to enable it.
+
+> [[done]] --[[Joey]]
diff --git a/doc/todo/allow_disk_space_quota_independent_of_free_disk_space.mdwn b/doc/todo/allow_disk_space_quota_independent_of_free_disk_space.mdwn
new file mode 100644
index 0000000000..a972ee77c0
--- /dev/null
+++ b/doc/todo/allow_disk_space_quota_independent_of_free_disk_space.mdwn
@@ -0,0 +1,7 @@
+Feature Request. Instead of (or perhaps in addition to) setting
+`annex.diskreserve`, I'd like to be able to tell git-annex "use up to 2TB
+of disk space".
+
+> This would need changes for git-annex to keep a running total of the
+> space it's using, which becomes a little hairy in light of concurrency,
+> possibly manual changes to the object store, etc. --[[Joey]]
diff --git a/doc/todo/amazon_prime_photos.mdwn b/doc/todo/amazon_prime_photos.mdwn
new file mode 100644
index 0000000000..3956939fd2
--- /dev/null
+++ b/doc/todo/amazon_prime_photos.mdwn
@@ -0,0 +1,3 @@
+ just  thought to check if you anyhow considered it  Joey: https://www.amazon.ca/clouddrive/primephotos ,  which could be a  great  feature for  those with prime,  even if only for photos.  didn't check about any API
+
+ cheers
diff --git a/doc/todo/annex_add___40__-u__124__--update__41___mode.mdwn b/doc/todo/annex_add___40__-u__124__--update__41___mode.mdwn
new file mode 100644
index 0000000000..ab0eac108a
--- /dev/null
+++ b/doc/todo/annex_add___40__-u__124__--update__41___mode.mdwn
@@ -0,0 +1,5 @@
+to supplement 'git add -u' behavior -- to add only updated (tracked only, no untracked) files to be committed.  ATM all files would be added, including untracked.
+
+[[!meta author=yoh]]
+
+> [[done]]
diff --git a/doc/todo/annex_add___40__-u__124__--update__41___mode/comment_1_bde2b1e2c45e110d56ce98b43dd77743._comment b/doc/todo/annex_add___40__-u__124__--update__41___mode/comment_1_bde2b1e2c45e110d56ce98b43dd77743._comment
new file mode 100644
index 0000000000..e6a632902e
--- /dev/null
+++ b/doc/todo/annex_add___40__-u__124__--update__41___mode/comment_1_bde2b1e2c45e110d56ce98b43dd77743._comment
@@ -0,0 +1,7 @@
+[[!comment format=mdwn
+ username="joey"
+ subject="""comment 1"""
+ date="2017-04-07T19:41:02Z"
+ content="""
+Good idea, adding.
+"""]]
diff --git a/doc/todo/annex_merge_--remotes.mdwn b/doc/todo/annex_merge_--remotes.mdwn
new file mode 100644
index 0000000000..d1fef7a66a
--- /dev/null
+++ b/doc/todo/annex_merge_--remotes.mdwn
@@ -0,0 +1,3 @@
+ATM 'annex merge' does not accept any parameter to specify which remotes to consider -- it merges all.  In some cases it might be desirable to merge only information from some remotes (e.g. I keep some annex remote "private" or smth like that)
+
+[[!meta author=yoh]]
diff --git a/doc/todo/annex_merge_--remotes/comment_1_d4d1c3dae7cff4119b7c111ac6e6f947._comment b/doc/todo/annex_merge_--remotes/comment_1_d4d1c3dae7cff4119b7c111ac6e6f947._comment
new file mode 100644
index 0000000000..a6d05f56a4
--- /dev/null
+++ b/doc/todo/annex_merge_--remotes/comment_1_d4d1c3dae7cff4119b7c111ac6e6f947._comment
@@ -0,0 +1,29 @@
+[[!comment format=mdwn
+ username="joey"
+ subject="""comment 1"""
+ date="2016-08-08T15:32:19Z"
+ content="""
+There's been discussion of keeping private forks of git-annex
+repositories before.
+
+IIRC, the remote.name.annex-sync and remote.name.annex-readonly
+settings can accomplish that. 
+
+For example, if the private repository A has a remote B, it can set
+annex-readonly, and this will prevent A from pushing any data to B. A
+can still pull from B. If A is on a locked down machine that B cannot
+itsef access, this guarantees that the changes in A remain private.
+I think this is the best way to accomplish this kind of scenario.
+
+If B has A as a remote, then B could set annex-sync to false, which
+would prevent it from pulling from A, and so B would never merge
+in git-annex branches from A, at least unless A pushed them to B.
+Of course, in this scenario, a manual `git pull A` on B bypasses
+the protection.
+
+It might make sense to make git-annex merge honor annex-ignore,
+and skip merging branches that belong to a remote, even if they
+were somehow pulled down. Unfortunately, git's remote branch
+name mapping can be quite complicated; IIRC it's not as simple
+as skipping branches remotes/B/*
+"""]]
diff --git a/doc/todo/annex_merge_--remotes/comment_2_be8596676823e916a56d774e0a161945._comment b/doc/todo/annex_merge_--remotes/comment_2_be8596676823e916a56d774e0a161945._comment
new file mode 100644
index 0000000000..38bd4e45b6
--- /dev/null
+++ b/doc/todo/annex_merge_--remotes/comment_2_be8596676823e916a56d774e0a161945._comment
@@ -0,0 +1,68 @@
+[[!comment format=mdwn
+ username="joey"
+ subject="""comment 2"""
+ date="2016-10-13T17:27:36Z"
+ content="""
+For a `git annex merge --remotes` to be useful, there would need to be a
+config to disable the automatic merging of git-annex branches, which all
+git-annex commands do when they notice it needs to be done.
+So, this needs to be a git config and not a switch, so it can also
+control the automatic git-annex branch merging.
+
+Using remote.name.annex-ignore as the config does not make sense on second
+look, because that gets set automatically when the remote is on eg
+github.
+
+Using remote.name.annex-sync=false as the config makes some sense, although
+as noted above, that prevents `git annex sync` from fetching from the
+remote already, so unless `git pull` is run manually, the existing config
+should suffice.
+
+To implement that, git-annex would have to parse the remote.name.fetch
+config in order to tell what name a remote's git-annex branch is fetched
+to. I am reluctant to do this for several reasons:
+
+* The syntax of remote.name.fetch is only documented by example. It's
+  not clear what's supposed to be done if eg, the `*` appears twice
+  in a branch name or different numbers of times on the left and right hand
+  sides.
+
+* Two remotes can have remote.name.fetch set such that the same remote
+  tracking branch is locally used for fetches from both remotes. So
+  git-annex would not know if such a branch should be synced or not.
+
+* remote.name.fetch can be overridden when using git fetch or git pull at
+  the command line, so again git-annex can't know for sure what remote a
+  given tracking branch came from.
+
+Four approaches that could work:
+
+1. Add a config that is a list of remote tracking branches, and
+   make `git annex merge` and the automatic git-annex branch merging merge
+   only those tracking branches. For example, 
+   `annex.allowmerge=refs/remotes/origin/git-annex refs/remotes/origin/master`
+
+   Doable, but it seems this would be an annoying list to maintain,
+   especially when new branches are made.
+
+2. Embed some information in a branch that can be looked at to determine
+   that git-annex should not auto-merge this branch. Note that this would
+   need to be done for both the git-annex branch and the regular branch.
+   The latter seems particularly hard to do.
+
+3. Configure remote.name.fetch so that the remote git-annex branch is
+   either not fetched, or fetched to a tracking branch that does not end
+   in `/git-annex`. I think this is possible to do, but due to the lack of
+   documentation for that config, it would take some experimentation to
+   find how to do it. This would prevent the automatic merging of that
+   branch by git-annex. 
+   
+   And if you make the remote master branch be
+   fetched to eg refs/remotes/name/master/nomerge then `git annex merge`
+   won't merge that into master.
+
+4. Prevent adding a remote to a repository if that remote contains private
+   information that you don't want to get merged into the local repository.
+   This still seems like the best solution to me; if the information is
+   private it should not be possible to fetch it from the remote.
+"""]]
diff --git a/doc/todo/annex_merge_--remotes/comment_3_35614da544e315529b236a36e1b28e2d._comment b/doc/todo/annex_merge_--remotes/comment_3_35614da544e315529b236a36e1b28e2d._comment
new file mode 100644
index 0000000000..be9e733035
--- /dev/null
+++ b/doc/todo/annex_merge_--remotes/comment_3_35614da544e315529b236a36e1b28e2d._comment
@@ -0,0 +1,17 @@
+[[!comment format=mdwn
+ username="joey"
+ subject="""comment 3"""
+ date="2018-08-06T15:47:56Z"
+ content="""
+This came up again in 
+and there it was sufficient to configure remote.name.fetch so that no
+branches were fetched from the cache remote.
+
+Approach #3 can be implemented using:
+
+	fetch = refs/heads/master:refs/remotes/private/nomerge/master
+
+This prevents git-fetch from fetching the git-annex branch, and
+it makes the remote master banch fetch into a name that
+git-annex won't automatically merge into master.
+"""]]
diff --git a/doc/todo/append-only_mode.mdwn b/doc/todo/append-only_mode.mdwn
new file mode 100644
index 0000000000..c3b6d621a4
--- /dev/null
+++ b/doc/todo/append-only_mode.mdwn
@@ -0,0 +1,49 @@
+The [[git-annex-shell]] wrapper allows the configuration of a readonly
+repository (through the `GIT_ANNEX_READONLY` environment and friends)
+but that is useful only when we want users to access the data and not
+add to it.
+
+It would be nice to have a *write-only* or "append-only" mode. My use
+case is a backup server that would receive git-annex objects and
+changes, but would forbid the client from deleting content on the
+server. This is to protect contents from being destroyed (or encrypted
+as is a common pattern with ransomware) by a compromised client.
+
+There has been some discussions and work done to protect *branches* in
+such a way, in
+[[todo/git-hook_to_sanity-check_git-annex_branch_pushes]], and that
+could help, but even with git hooks, a malicious client could still
+drop content.
+
+It seems to me this would require modifications to the
+`git-annex-shell` wrapper to forbid certain operations like `dropkey`,
+`lockcontent`, or `p2pstdio` although I'm unfamiliar with the last two
+so I am not certain they could be harmful. Maybe `p2pstdio` itself
+could be somewhat fixed to allow only append commands.
+
+Is it fair to assume that `recvkey` is safe in this context, ie. that
+it wouldn't overwrite an existing bit of content without first doing a
+checksum?
+
+Thanks! -- [[anarcat]]
+
+> Good idea.. Implemented. 
+> 
+> I'm not entirely happy with the name, but could not think of
+> a better one.
+> 
+> Yes, `recvkey` will never overwrite content already in the annex,
+> and unless you turn off annex.verify, hashes will also be checked
+> before letting anything into the annex.
+> 
+> Of course, if non-hashed keys are used, and an object has not
+> reached the repository yet from a trusted source, an attacker
+> could slip in something malicious without being noticed.
+> Setting annex.securehashesonly would be a good idea to prevent this.
+> 
+> p2pstdio implements the same security policies as the rest of
+> git-annex-shell.
+> 
+> --[[Joey]]
+
+> Update: I didn't close this before, but it seems [[done]] --[[Joey]]
diff --git a/doc/todo/append-only_mode/comment_1_eaa7ae80c3758bccd23c4e0a8d1eefc4._comment b/doc/todo/append-only_mode/comment_1_eaa7ae80c3758bccd23c4e0a8d1eefc4._comment
new file mode 100644
index 0000000000..fb904da842
--- /dev/null
+++ b/doc/todo/append-only_mode/comment_1_eaa7ae80c3758bccd23c4e0a8d1eefc4._comment
@@ -0,0 +1,13 @@
+[[!comment format=mdwn
+ username="https://christian.amsuess.com/chrysn"
+ nickname="chrysn"
+ avatar="http://christian.amsuess.com/avatar/c6c0d57d63ac88f3541522c4b21198c3c7169a665a2f2d733b4f78670322ffdc"
+ subject="append-only and gitolite"
+ date="2018-05-28T11:47:13Z"
+ content="""
+Thanks, that's a nice feature.
+
+How could this be integrated with gitolite? Should the `if ( can_write($repo) )` branch be amended by a check for the \"+\" flag ([short documentation](http://gitolite.com/gitolite/conf-2/) says that \"RW\" is roughly \"create or fast-forward, no deletes\" while \"RW+\" includes deletes and non-fast-forwards)?
+
+(This might change the user experience on repositories run with \"RW\" permissions currently, but IMO patching a possibly unwanted access that can be easily granted if so desired.)
+"""]]
diff --git a/doc/todo/append-only_mode/comment_2_fd1f869f97e0658ec3986fe05f28345e._comment b/doc/todo/append-only_mode/comment_2_fd1f869f97e0658ec3986fe05f28345e._comment
new file mode 100644
index 0000000000..6bf43ec0a9
--- /dev/null
+++ b/doc/todo/append-only_mode/comment_2_fd1f869f97e0658ec3986fe05f28345e._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="anarcat"
+ avatar="http://cdn.libravatar.org/avatar/4ad594c1e13211c1ad9edb81ce5110b7"
+ subject="some docs"
+ date="2018-07-06T01:44:08Z"
+ content="""
+I have tried to document this in the [[git-annex-shell]] manpage, see if it works for you... thanks so much for implementing this key feature so quickly!
+"""]]
diff --git a/doc/todo/append-only_mode/comment_3_790ce76fe02ca35b9da7309ee2384c01._comment b/doc/todo/append-only_mode/comment_3_790ce76fe02ca35b9da7309ee2384c01._comment
new file mode 100644
index 0000000000..d4eac48cf2
--- /dev/null
+++ b/doc/todo/append-only_mode/comment_3_790ce76fe02ca35b9da7309ee2384c01._comment
@@ -0,0 +1,15 @@
+[[!comment format=mdwn
+ username="joey"
+ subject="""comment 3"""
+ date="2018-07-06T16:34:02Z"
+ content="""
+I think it would go better in a tips page than on the man page.
+Especially since no complete solution is available and it's well beyond the
+scope of a manpage for git-annex-shell to document such a solution.
+
+It seems to me that allowing untrusted people to push to the git-annex
+branch is asking for various mischeif. One that comes to mind is they can
+add a repository, configure it to be trusted, and make git-annex think
+it has a copy of all content. Combined with preferred content settings this
+could make git-annex drop the only actual copy of content.
+"""]]
diff --git a/doc/todo/append-only_mode/comment_4_cb00ed89aa7931fdadbf44355bdc78b2._comment b/doc/todo/append-only_mode/comment_4_cb00ed89aa7931fdadbf44355bdc78b2._comment
new file mode 100644
index 0000000000..f961d85e9f
--- /dev/null
+++ b/doc/todo/append-only_mode/comment_4_cb00ed89aa7931fdadbf44355bdc78b2._comment
@@ -0,0 +1,12 @@
+[[!comment format=mdwn
+ username="anarcat"
+ avatar="http://cdn.libravatar.org/avatar/4ad594c1e13211c1ad9edb81ce5110b7"
+ subject="comment 4"
+ date="2018-07-06T16:48:29Z"
+ content="""
+understood, i'll add a tips page as well.. i started some personal notes here:
+
+https://anarc.at/services/backup/#append-only-git-repositories
+
+regarding pushes to the git-annex branch... in my threat model, it is not a real problem. they can drop all the contents locally and destroy everything there anyways. if an attacker adds an extra repo, sure, a \"smart\" git-annex command might mistakenly drop contents in the wrong location, but the attacker could do that anyways, without adding that branch. what is actually precious are the blobs stored on the remote server: if those can't be removed, we are safe, if I understand things correctly.
+"""]]
diff --git a/doc/todo/assistant_cannot_set_up_remote_repo_via_an_ssh_alias_or_an_ip_address.mdwn b/doc/todo/assistant_cannot_set_up_remote_repo_via_an_ssh_alias_or_an_ip_address.mdwn
new file mode 100644
index 0000000000..f5088ccee3
--- /dev/null
+++ b/doc/todo/assistant_cannot_set_up_remote_repo_via_an_ssh_alias_or_an_ip_address.mdwn
@@ -0,0 +1,50 @@
+[[!meta title="webapp does not accept ssh alias for ssh remote"]]
+
+What steps will reproduce the problem?
+
+Using the assistant, create an SSH remote.  Try to use an alias as the name
+of the remote (e.g. I have a server which I have aliased to "homeworld" in
+my .ssh/config.  When I'm at home, that is an alias for 192.168.1.253.
+When I'm not at home, I edit .ssh/config so that "homeworld" becomes an
+alias for a hostname at no-ip.com.)  Despite the fact that "homeworld" is a
+viable ssh target because of the alias, the assistant doesn't recognize it
+as a valid host to ssh to.
+
+I had trouble with an ip address the first time I tried it but just tried
+it again and it worked fine, so please disregard that part of the title of
+this bug report.
+
+
+What is the expected output? What do you see instead?
+
+expected output = move to the "create a repository -- rsync or regular" page.
+observed output = "cannot resolve host name"
+
+
+What version of git-annex are you using? On what operating system?
+
+ Version: 3.20130102 OS X Lion
+
+
+Please provide any additional information below.
+
+I realize this is kind of a power user whine.  Using an ssh alias which
+does not correspond to an actual resolvable hostname (and cannot, because
+it's supposed to be a layer of indirection over the hostname) is not an
+everyday problem for an average user.
+
+> The assistant tries to resolve the hostname explicitly
+> to catch user's typos, and also expands it to a FQDN, to make
+> it more likely to be able to reach the host when roaming to other
+> networks.
+> 
+> Also, the assistant sets up it *own* .ssh/config hostname alias,
+> in order to make it use the special ssh key that it generates for the host.
+> So that is not compatable with using a ssh host alias you've set up.
+> Even if it knew about your alias, it would set up a new hostname alias, and
+> whatever machinery you have to update the alias would not work.
+>  
+> You can, of course, add git remotes using any ssh alias you like, by
+> hand, and restart the assistant and it will use them. --[[Joey]]
+
+[[!tag /design/assistant]]
diff --git a/doc/todo/assistant_cannot_set_up_remote_repo_via_an_ssh_alias_or_an_ip_address/comment_1_6b71a390fd16f593216793aec590d9a8._comment b/doc/todo/assistant_cannot_set_up_remote_repo_via_an_ssh_alias_or_an_ip_address/comment_1_6b71a390fd16f593216793aec590d9a8._comment
new file mode 100644
index 0000000000..6f4d00b2b7
--- /dev/null
+++ b/doc/todo/assistant_cannot_set_up_remote_repo_via_an_ssh_alias_or_an_ip_address/comment_1_6b71a390fd16f593216793aec590d9a8._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="https://www.google.com/accounts/o8/id?id=AItOawmBUR4O9mofxVbpb8JV9mEbVfIYv670uJo"
+ nickname="Justin"
+ subject="comment 1"
+ date="2014-05-22T20:05:54Z"
+ content="""
+A temporary workaround is to add an entry to /etc/hosts until the remote is added.
+"""]]
diff --git a/doc/todo/assistant_parallel_file_transfers.mdwn b/doc/todo/assistant_parallel_file_transfers.mdwn
new file mode 100644
index 0000000000..aafddf0383
--- /dev/null
+++ b/doc/todo/assistant_parallel_file_transfers.mdwn
@@ -0,0 +1,15 @@
+Hi and thank you for an incredible piece of software and great work!
+
+I've noticed that when I add new files to a repository and I have my USB drive connected, the assistant alternate it's transfers of files. And only transfers one queued file at the time.
+
+file1 -->> Internet offsite computer
+file1 -->> USB drive
+file2 -->> Internet offsite computer
+file2 -->> USB drive
+
+
+I would prefer a logic where the assistant transfer files in parallel to my different repositories. I know that it might not be a good thing doing that with network accessed repositories, but when I have "low cost", locally attached USB drives it would be great if the transfers could be done in parallel.
+
+
+Is there a configuration option for this already?
+
diff --git a/doc/todo/be_able_to_refer_to_remotes_by_uuid.mdwn b/doc/todo/be_able_to_refer_to_remotes_by_uuid.mdwn
new file mode 100644
index 0000000000..df6bac4360
--- /dev/null
+++ b/doc/todo/be_able_to_refer_to_remotes_by_uuid.mdwn
@@ -0,0 +1,6 @@
+in light of https://github.com/datalad/datalad/issues/1259 (conflicting remote names)  I wondered if both initremote and enableremote (and probably others) should acquire ability to set/refer to remotes by their UUIDs instead of relying purely on name which is not guaranteed to be unique.
+
+[[!meta name=yoh]]
+
+> This seems [[done]], at least initremote and enableremote support uuid=.
+> --[[Joey]]
diff --git a/doc/todo/be_able_to_refer_to_remotes_by_uuid/comment_1_54585f8fa43b315637af60eaf9d6ad50._comment b/doc/todo/be_able_to_refer_to_remotes_by_uuid/comment_1_54585f8fa43b315637af60eaf9d6ad50._comment
new file mode 100644
index 0000000000..926c6f6fc6
--- /dev/null
+++ b/doc/todo/be_able_to_refer_to_remotes_by_uuid/comment_1_54585f8fa43b315637af60eaf9d6ad50._comment
@@ -0,0 +1,11 @@
+[[!comment format=mdwn
+ username="joey"
+ subject="""comment 1"""
+ date="2017-02-07T16:34:57Z"
+ content="""
+enableremote can already enable a remote by uuid instead of by name.
+
+It might be useful to let a uuid be provided to initremote. There's also
+some foot shooting potential (reuse the wrong uuid and then the special
+remote has been set up, so how would you change it).
+"""]]
diff --git a/doc/todo/be_able_to_refer_to_remotes_by_uuid/comment_2_8ace9ec8cb5264fd424b3bc550d96ba5._comment b/doc/todo/be_able_to_refer_to_remotes_by_uuid/comment_2_8ace9ec8cb5264fd424b3bc550d96ba5._comment
new file mode 100644
index 0000000000..fc22f0c904
--- /dev/null
+++ b/doc/todo/be_able_to_refer_to_remotes_by_uuid/comment_2_8ace9ec8cb5264fd424b3bc550d96ba5._comment
@@ -0,0 +1,25 @@
+[[!comment format=mdwn
+ username="joey"
+ subject="""comment 2"""
+ date="2017-02-07T18:59:41Z"
+ content="""
+I've made `initremote` be able to be provided with a uuid=whatever
+parameter to use whatever UUID you like.
+
+Valid use cases include setting up two special remotes that access the same
+data store through two different interfaces. For example, a rsync special
+remote that is also accessible via a NFS mount as a directory special remote.
+
+It can also be used when two unrelated repositories want to use the same
+data store. Of course, dropping data from the data store then becomes a
+problem, since one of the repositories will know it was dropped, and the
+other one won't. Can get into situations where one of the repositories
+was relying on its remote as the only place a file was stored, and so loses
+the only copy it knows about when the other repository moves the content
+from the remote.
+
+For datalad-archives, I think dropping content from that special remote is
+not supported. Which nearly avoids such problems. If so, it should be fine
+to reuse some UUID for all the datalad-archives special remotes in
+different unrelated datalad repositories.
+"""]]
diff --git a/doc/todo/be_able_to_specify_custom_commit_message_for_git-annex_branch_commit.mdwn b/doc/todo/be_able_to_specify_custom_commit_message_for_git-annex_branch_commit.mdwn
new file mode 100644
index 0000000000..3335899c82
--- /dev/null
+++ b/doc/todo/be_able_to_specify_custom_commit_message_for_git-annex_branch_commit.mdwn
@@ -0,0 +1,5 @@
+ATM many commit messages in git-annex branch are just an "update".  But I wondered if it could be relatively simple to implement to be able to provide a custom commit message, e.g. by defining some environment variable (e.g. GIT_ANNEX_UPDATE_MESSAGE) during whatever process is initiating the actual commit?  sure thing there could be cases whenever multiple processes "contributed" to the changes which are finally "flushed" by committing, but imho it could provide a good way to annotate changes within git-annex branch at least to some degree.  E.g. I could use "update: added URLs to the files for datalad-archives remote" ;)
+
+[[!meta author=yoh]]
+
+> [[done]] as annex.commitmessage config. --[[Joey]]
diff --git a/doc/todo/be_able_to_specify_custom_commit_message_for_git-annex_branch_commit/comment_1_c8836a164c8c1efedd5467233237bfd0._comment b/doc/todo/be_able_to_specify_custom_commit_message_for_git-annex_branch_commit/comment_1_c8836a164c8c1efedd5467233237bfd0._comment
new file mode 100644
index 0000000000..6d4f9b0267
--- /dev/null
+++ b/doc/todo/be_able_to_specify_custom_commit_message_for_git-annex_branch_commit/comment_1_c8836a164c8c1efedd5467233237bfd0._comment
@@ -0,0 +1,24 @@
+[[!comment format=mdwn
+ username="joey"
+ subject="""comment 1"""
+ date="2017-02-07T16:39:59Z"
+ content="""
+Such an env var could easily be supported.
+
+It's desirable to mostly keep the mantainace of the git-annex branch
+out of the user's view, because it's a complication that should not
+need to worry about.
+
+Keeping it in an env var does avoid complicating the discoverable
+interface with some parameter to specify the commit message. Another
+way would be to make the commit message configurable in git config.
+Then it could be overridden with -c. Seems like the user of this 
+feature is likely going to also use annex.alwayscommit=false sometimes
+in order to construct git-annex branch commits that bundle up several
+changes.
+
+Anyway, I'm curious what kind of use cases make using custom commit
+messages for the git-annex branch makes sense. I doubt that typical git
+workflows with cherry-picking patches, patch review, etc, work with that
+branch.
+"""]]
diff --git a/doc/todo/be_able_to_specify_custom_commit_message_for_git-annex_branch_commit/comment_2_41a25215d9a1f635573d5e5f5c30fd62._comment b/doc/todo/be_able_to_specify_custom_commit_message_for_git-annex_branch_commit/comment_2_41a25215d9a1f635573d5e5f5c30fd62._comment
new file mode 100644
index 0000000000..4044f7d1b3
--- /dev/null
+++ b/doc/todo/be_able_to_specify_custom_commit_message_for_git-annex_branch_commit/comment_2_41a25215d9a1f635573d5e5f5c30fd62._comment
@@ -0,0 +1,25 @@
+[[!comment format=mdwn
+ username="yarikoptic"
+ avatar="http://cdn.libravatar.org/avatar/f11e9c84cb18d26a1748c33b48c924b4"
+ subject="comment 2"
+ date="2018-07-31T14:27:17Z"
+ content="""
+My usecase is to be able to associate changes in main branches and in git-annex branch, so instead of non-informative \"update\" commit message it could carry e.g.
+
+    [DATALAD] Updated git/annex from a remote location
+    
+    URLs processed: 7
+     downloaded: 7
+     size: 17.1 GB
+    Files processed: 7
+     skipped: 7
+     overwritten: 7
+
+or I might even include the branch in which changes were done, e.g. short ones become
+
+    [DATALAD, incoming] Updated git/annex from a remote location
+    [DATALAD, incoming-processed] Added files from extracted archives
+    ...
+    
+Then looking at `git log git-annex` would allow me to quickly see what actions lead to \"updates\" in git-annex branch
+"""]]
diff --git a/doc/todo/be_able_to_specify_custom_commit_message_for_git-annex_branch_commit/comment_3_5241eace21e873678a0fbb353f4ece69._comment b/doc/todo/be_able_to_specify_custom_commit_message_for_git-annex_branch_commit/comment_3_5241eace21e873678a0fbb353f4ece69._comment
new file mode 100644
index 0000000000..68432661f1
--- /dev/null
+++ b/doc/todo/be_able_to_specify_custom_commit_message_for_git-annex_branch_commit/comment_3_5241eace21e873678a0fbb353f4ece69._comment
@@ -0,0 +1,16 @@
+[[!comment format=mdwn
+ username="joey"
+ subject="""comment 3"""
+ date="2018-08-02T17:38:23Z"
+ content="""
+There are a few other messages than "update" that show up in some rare
+occasions including (message ++ " (recovery from race #"..) and
+"new branch for transition".
+
+And, there's "merging remote/git-annex into git-annex"
+which is common enough.
+
+I kind of have the feeling that the rare ones are rare enough
+that we might want to always use them, and only override 
+merging and update.
+"""]]
diff --git a/doc/todo/be_able_to_specify_custom_commit_message_for_git-annex_branch_commit/comment_4_caf913b53a54ac010dba253fca1ef76e._comment b/doc/todo/be_able_to_specify_custom_commit_message_for_git-annex_branch_commit/comment_4_caf913b53a54ac010dba253fca1ef76e._comment
new file mode 100644
index 0000000000..fa3e71d147
--- /dev/null
+++ b/doc/todo/be_able_to_specify_custom_commit_message_for_git-annex_branch_commit/comment_4_caf913b53a54ac010dba253fca1ef76e._comment
@@ -0,0 +1,9 @@
+[[!comment format=mdwn
+ username="joey"
+ subject="""comment 4"""
+ date="2018-08-02T17:46:51Z"
+ content="""
+OTOH, maybe merging is never something one would want to add a custom
+message for -- it's an entirely a function of the inputs -- and so only
+"update" should get replaced.
+"""]]
diff --git a/doc/todo/better_exceptions_to_annex.security.allow-unverified-downloads.mdwn b/doc/todo/better_exceptions_to_annex.security.allow-unverified-downloads.mdwn
new file mode 100644
index 0000000000..b0021033be
--- /dev/null
+++ b/doc/todo/better_exceptions_to_annex.security.allow-unverified-downloads.mdwn
@@ -0,0 +1,7 @@
+"Downloading unverified content from (non-encrypted) external special remotes is prevented, because they could follow http redirects to web servers on localhost or on a private network, or in some cases to a file:/// url" -- it's be good if an exception to this could be configured for a given type of external special remote, and/or for specific special remotes.
+Sometimes I _know_ that a given external special remote doesn't do redirects, or that a given special remote repository won't have bad URLs.  Remembering to do
+git -c annex.security.allow-unverified-downloads=ACKTHPPT annex get myfile
+every time is another thing to think about, when the whole point of git-annex is to not have to think about where things are :)  While configuring 
+annex.security.allow-unverified-downloads=ACKTHPPT permanently opens security holes.
+
+> [[done]] --[[Joey]]
diff --git a/doc/todo/better_exceptions_to_annex.security.allow-unverified-downloads/comment_1_048cdb0a814ec560373d8c5c01663fab._comment b/doc/todo/better_exceptions_to_annex.security.allow-unverified-downloads/comment_1_048cdb0a814ec560373d8c5c01663fab._comment
new file mode 100644
index 0000000000..22469ad9d9
--- /dev/null
+++ b/doc/todo/better_exceptions_to_annex.security.allow-unverified-downloads/comment_1_048cdb0a814ec560373d8c5c01663fab._comment
@@ -0,0 +1,15 @@
+[[!comment format=mdwn
+ username="joey"
+ subject="""comment 1"""
+ date="2018-09-25T18:51:11Z"
+ content="""
+Added a per-remote configuration of that.
+
+I thought about adding something to the external special remote protocol
+to let them indicate what they are not vulnerable to CVE-2018-10857.
+But it only affects WORM and URL keys, which I'm reluctant to complicate the
+protocol for. (It would be better perhaps to just remove those types of
+keys.) And it's actually rather difficult for external special
+remote authors to guarantee that is the case, since libraries they use may
+change over time.
+"""]]
diff --git a/doc/todo/better_exceptions_to_annex.security.allow-unverified-downloads/comment_2_485e090092e6ccddbe79f59903d493ec._comment b/doc/todo/better_exceptions_to_annex.security.allow-unverified-downloads/comment_2_485e090092e6ccddbe79f59903d493ec._comment
new file mode 100644
index 0000000000..8f8c26516e
--- /dev/null
+++ b/doc/todo/better_exceptions_to_annex.security.allow-unverified-downloads/comment_2_485e090092e6ccddbe79f59903d493ec._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="Ilya_Shlyakhter"
+ avatar="http://cdn.libravatar.org/avatar/1647044369aa7747829c38b9dcc84df0"
+ subject="comment 2"
+ date="2018-09-26T00:14:53Z"
+ content="""
+per-remote config solves this, thanks.  \"It would be better perhaps to just remove those types of keys\" -- please don't, I've had various uses for both.
+"""]]
diff --git a/doc/todo/better_exceptions_to_annex.security.allow-unverified-downloads/comment_3_f625216ca793d73cb42f70bf8b10a9cc._comment b/doc/todo/better_exceptions_to_annex.security.allow-unverified-downloads/comment_3_f625216ca793d73cb42f70bf8b10a9cc._comment
new file mode 100644
index 0000000000..24dc81f7dc
--- /dev/null
+++ b/doc/todo/better_exceptions_to_annex.security.allow-unverified-downloads/comment_3_f625216ca793d73cb42f70bf8b10a9cc._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="Ilya_Shlyakhter"
+ avatar="http://cdn.libravatar.org/avatar/1647044369aa7747829c38b9dcc84df0"
+ subject="comment 3"
+ date="2018-09-26T00:51:29Z"
+ content="""
+\"I thought about adding something to the external special remote protocol to let them indicate what they are not vulnerable to CVE-2018-10857\" -- you could instead add a special remote message FETCH_URL_FOR_ME that asks git-annex to fetch the URL, and git-annex can then fetch it with its internal fetcher that does the needed security checks.
+"""]]
diff --git a/doc/todo/bittorrent__58___support_offline_operation_and_verification.mdwn b/doc/todo/bittorrent__58___support_offline_operation_and_verification.mdwn
new file mode 100644
index 0000000000..bfee0e7d9a
--- /dev/null
+++ b/doc/todo/bittorrent__58___support_offline_operation_and_verification.mdwn
@@ -0,0 +1,77 @@
+adding torrents whose data is already present on disk or in the annex seems not
+to be supported right now.
+
+let's assume for example that the files were obtained by means outside of
+git-annex' scope, and that all files in here were `git annex add`ed and
+committed:
+
+    .
+    ├── chaos-math_multi-language_1080p_mkv.ea15601881aa1be1.torrent
+    └── chaos-math_multi-language_1080p_mkv
+        ├── 01. Motion and determinism - Panta Rhei [1080p].mkv
+        ├── [...]
+        ├── 10. Générique Chaos - French [1080p].mkv
+        └── subtitles
+            ├── Chaos1_ar.srt
+            ├── [...]
+            └── README.txt
+
+starting the `addurl` from a local file is supported as described in the
+comment to [[special_remotes/bittorrent]], but then goes ahead to download all
+the data in a different location. neither can be used the `--file` option
+(because it's a multi-file torrent) nor the `--pathdepth` option (because it
+would result in the very filename already used for the .torrent file), so first
+i'd have to rename the torrent directory, then start the addurl:
+
+    $ mv chaos-math_multi-language_1080p_mkv _tmp_annex_test_chaos_math_multi_language_1080p_mkv.ea15601881aa1be1.torrent
+    $ git annex addurl file:///tmp/annex-test/chaos-math_multi-language_1080p_mkv.ea15601881aa1be1.torrent
+
+even then, most of the files in the torrent are downloaded again, because the
+file names are meddled with by git-annex (`01. Motion and determinism - Panta Rhei [1080p].mkv`
+becomes `01._Motion_and_determinism___Panta_Rhei__1080p_.mkv`). and if the
+files come pre-renamed, their content does *not* get checked against the
+torrent info file (with all files pre-renamed and some subjected to random
+bit-flipping, the whole torrent was still accepted and entered as the wrong
+hash file's web remote).
+
+suggestions
+===========
+
+to make git-annex suitable for archiving torrents, i'd like to suggest the
+following additions:
+
+* add an option to switch off the file name meddling. (both file systems
+  and tools nowadays accept whitespace in file names, and all but
+  you-know-which do support colons and other characters except '\0' and '/').
+  this would have the additional benefit of making a http/ftp view of the
+  repository suitable as a [BEP19](http://bittorrent.org/beps/bep_0019.html)
+  web seed.
+
+* support `--file` to mean the base directory of a multi-file download, or just
+  pick the directory file name from inside the torrent info's directory name
+  (without that, the BEP19 use case would require explicit renaming).
+
+* before adding the `timestamp 1 $URL#n` line to the .log.web records of a
+  locally present file, check against the hashes from the torrent. (this may
+  require adjacent files from the torrent to be present, but it should be
+  complete anyway at check-in time).
+
+related issues / suggestions
+============================
+
+* especially on systems where file name meddling can not be turned off,
+  guessing suitable present files from their length could be worth considering,
+  as under those conditions, it might be impossible to find the already
+  downloaded file from its name (as the original downloading torrent client
+  might have used a different filename escaping scheme). with those guesses, it
+  would be up to aria2c to determine whether that file fits or not, and
+  download without complaining on demand.
+
+* to later allow seeding from the downloaed torrent, it'd be nice to have a
+  less convoluted way of always having the .torrent file present than
+  constructing a `file:///` url of it. having `addurl` (and thus also `get`
+  when dereferencing the .log.web entry) accept local file names would be
+  practical; a `--additionally-raw` option on `addurl` (or equivalent setting)
+  that stores the torrent file in git or git-annex would be nice.
+
+--[[chrysn]]
diff --git a/doc/todo/bittorrent__58___support_offline_operation_and_verification/comment_1_ab16dfb6c3fe2a70ddfb0cee287c6127._comment b/doc/todo/bittorrent__58___support_offline_operation_and_verification/comment_1_ab16dfb6c3fe2a70ddfb0cee287c6127._comment
new file mode 100644
index 0000000000..47eddfaeac
--- /dev/null
+++ b/doc/todo/bittorrent__58___support_offline_operation_and_verification/comment_1_ab16dfb6c3fe2a70ddfb0cee287c6127._comment
@@ -0,0 +1,35 @@
+[[!comment format=mdwn
+ username="joey"
+ subject="""comment 1"""
+ date="2016-10-17T20:05:51Z"
+ content="""
+The filename **sanitization** is needed for security reasons. A
+bittorrent file could contain `../` and similar evil which should not be
+allowed to be written to disk as-is. Or control characters which could
+cause an exploit via terminal key remapping. Or filenames starting with
+dashes to make an unguarded `rm *` end up expanding to `rm -rf something`.
+
+I'd not be surprised if whatever bittorrent program you used to download
+that does some filename sanitization too. Opinions on safe sanitization
+will vary, so it's not practical to expect git-annex and multiple
+bittorrent programs to behave identically.
+
+----
+
+It would be possible to make `addurl --file` usable with a multi-file
+torrent. Something like:
+
+	git annex addurl http://example.com/foo.torrent --file bar.mkv
+	That url contains multiple files; pick the one that corresponds to
+	local file "bar.mkv", and pass it in the --subfile option
+		bar.sub
+		bar.mkv
+		README.txt
+	Eg: git annex addurl 'http://example.com/foo.torrent' --file 'bar.mkv' --subfile 'bar.mkv'
+
+Of course this could be skipped if the torrent only contains one file
+with the same size and name as the `--file` file.
+
+I don't know if such an interface would be too annoying to be worth
+using in your use case or not?
+"""]]
diff --git a/doc/todo/bittorrent__58___support_offline_operation_and_verification/comment_2_262f9d68a865300d894c4077e0e8a70c._comment b/doc/todo/bittorrent__58___support_offline_operation_and_verification/comment_2_262f9d68a865300d894c4077e0e8a70c._comment
new file mode 100644
index 0000000000..781af4ef6c
--- /dev/null
+++ b/doc/todo/bittorrent__58___support_offline_operation_and_verification/comment_2_262f9d68a865300d894c4077e0e8a70c._comment
@@ -0,0 +1,11 @@
+[[!comment format=mdwn
+ username="http://christian.amsuess.com/chrysn"
+ nickname="chrysn"
+ avatar="http://christian.amsuess.com/avatar/bf293be5f036c0b6a1ac26519c8b6b07bfe749b971329c5014e8d84b53579735"
+ subject="comment 2"
+ date="2016-10-24T13:00:56Z"
+ content="""
+the `--file`/`--subfile` syntax would be workable; i'd probably employ a dedicated script to either know the sanitization mapping of the downloading client or to do heuristics based on file sizes. (side note on sanitization mappings: at least transmission seems to map every sane name through the identity map).
+
+i'm worried though that adding the pre-downloaded files file-by-file will then make it harder for git-annex to do a verified check-in; if, for example, we have a two-file torrent, and add the first one, git-annex can't veriyf that file because the second half of its last chunk is missing. unless git-annex started a download of the last chunk, it couldn't be sure that the #1 .log.web line is valid, and would need to keep that in some unverified staging area a la \"./file1 could be ./two-part.torrent#1, but i couldn't check that yet\", and only commit that information when file2 is added.
+"""]]
diff --git a/doc/todo/borg_special_remote.mdwn b/doc/todo/borg_special_remote.mdwn
new file mode 100644
index 0000000000..0d4e417289
--- /dev/null
+++ b/doc/todo/borg_special_remote.mdwn
@@ -0,0 +1,15 @@
+borg backup is pretty cool, and could be a great special remote backend.
+In particular it does delta compression and stuff.
+
+There seem to be two ways it could work. Probably there are borg commands
+that allow storing a given blob in it, and retrieving a given blob. And
+that could be used for a traditional special remote.
+
+But also, if a whole git-annex repository has been backed up with borg,
+then git-annex could look inside such a backup, and see if
+.git/annex/object/ contains an object. It could then mark it as
+present in the borg special remote. This way you'd use borg to take
+backups, and git-annex would then be aware of what was backed up in borg,
+and could do things like count that as a copy.
+
+--[[Joey]]
diff --git a/doc/todo/borg_special_remote/comment_1_f8543e05bea3a88769d9c54fdfce3723._comment b/doc/todo/borg_special_remote/comment_1_f8543e05bea3a88769d9c54fdfce3723._comment
new file mode 100644
index 0000000000..0a8caf6d9d
--- /dev/null
+++ b/doc/todo/borg_special_remote/comment_1_f8543e05bea3a88769d9c54fdfce3723._comment
@@ -0,0 +1,20 @@
+[[!comment format=mdwn
+ username="RonnyPfannschmidt"
+ avatar="http://cdn.libravatar.org/avatar/c5379a3fe2188b7571858c49f9db63c6"
+ subject="the remote im working on"
+ date="2018-06-04T07:51:57Z"
+ content="""
+Hi Joey,
+
+i am currently working on a remote to use borg as a tree import source and a content souce
+
+the work is started in https://github.com/RonnyPfannschmidt/git-annex-borg
+
+note that borg does **not** do delta storage - it does content informed dynamic chunk sizes (which helps deduplication)
+
+freestanding borg will not be a good remote for putting things out,
+so i will be pulling things out mostly (but i hope to hit a point where its viable to generate a borg archive from the tree of expected contents thats viable for putting things in)
+
+-- Ronny
+
+"""]]
diff --git a/doc/todo/build_a_user_guide.mdwn b/doc/todo/build_a_user_guide.mdwn
new file mode 100644
index 0000000000..1fcf9bf939
--- /dev/null
+++ b/doc/todo/build_a_user_guide.mdwn
@@ -0,0 +1,45 @@
+there's a lot of good documentation on this wiki, but it's hard to find sometimes. it's also unclear if we should look in the [[git-annex]] manpage or elsewhere in the wiki or where. this is a typical problem with the use of wikis for documentation: it's there, but hard to find. it doesn't mean a wiki shouldn't be used but, as with any user manual, special care needs to be taken about structure, organisation and making sure the manual is exhaustive.
+
+a good example of this problem is [[todo/document_standard_groups_more_extensively_in_the_UI]]. --[[anarcat]]
+
+update: a beginning of this may be the the [[workflow]] page but it lacks a lot of details... 
+
+So we have those entry points so far:
+
+  * [[git-annex]] - the manpage
+  * [[walkthrough]] - "A walkthrough of some of the basic features of git-annex, using the command line", described as "only one possible workflow for using git-annex"
+  * [[assistant]] - a whole subtree of pages describing the assistant, includes a [[assistant/quickstart]] - introduction to the assistant with a series of screenshots, described in [[walkthrough]] as "If you don't want to use the command line, see quickstart instead.", linked from the [[assistant]] page
+  * [[workflow]] - a summary of the different workflows that git-annex can use
+  * [[special remotes]] - a good list of "supported backends", which may be a better wording
+  * inversely, [[not]] is what is *not* supported, obviously
+  * [[install]] - how to install git-annex, of course
+  * [[tips]] - a mish-mash list of "how to do X in git-annex", 68 pages at the time of writing
+  * there's the "details" section on the frontpage which covers lots of the [[internals]], [[design]] and so on
+  * there are also what i consider to be "leaf" pages like [[how it works]] or [[sync]] there
+
+So it seems the fundamentals of such a user guide are there. It's just a matter of grouping this in a meaningful way.
+
+I am thinking the following structure may be a good basis:
+
+ * Introduction
+   * [[how it works]]?
+   * ...?
+ * [[Install|install]]
+ * Walkthrough
+   * [[comandline|walkthrough]]
+   * [[graphical|assistant/quickstart]]?
+ * How do I...
+   * [[Supported backends|special remotes]]
+   * [[Unsupported features|not]]
+   * [[split repositories|tips/splitting_a_repository]]
+   * [[merge repositories|tips/migrating_two_seperate_disconnected_directories_to_git_annex]]
+   * deal with lots of files: [[tips/Repositories_with_large_number_of_files]] or merge into [[scalability/]]?
+   * decide which files go where? something about [[preferred_content]] and [[preferred_content/standard_groups/]]?
+   * sort and regroup the best [[tips]] pages
+ * Troubleshooting / FAQ?
+   * [[Common mistakes|tips/antipatterns]]
+   * sort out the best of [[forums]] and [[tips]] that commonly occur
+ * [[Design]]
+   * [[internals]]
+   * [[encryption]]
+   * ... etc - all the developer stuff users shouldn't usually have to know unless they care about performance or need to reimplement something
diff --git a/doc/todo/cache_key_info.mdwn b/doc/todo/cache_key_info.mdwn
new file mode 100644
index 0000000000..d4352ccf7f
--- /dev/null
+++ b/doc/todo/cache_key_info.mdwn
@@ -0,0 +1,37 @@
+Most of git-annex is designed to be fast no matter how many other files are
+in the annex. Things like add/get/drop/move/fsck have good locality;
+they will only operate on as many files as you need them to. 
+
+(git commit can get a little slow with a great deal of files,
+but that's out of scope -- and recent git-annex versions use queuing
+to save git add from piling up too much in the index.)
+
+But currently two git-annex commands are quite slow when annexes become large
+in quantity of files. These are unused and status.
+(Both have --fast versions that don't do as much).
+> (Update: status has become acceptably fast; most of its slowdown was due to using a bad data structure; scanning the tree is not particularly slow and it no longer looks at the git-annex branch.)
+
+unused is slow because it needs two pieces of information that are not
+quick to look up, and require examining the whole repo, very seekily:
+
+1. The keys present in the annex. Found by looking thru .git/annex/objects
+2. The keys referenced by files in git. Found by finding every file
+   in git, and looking at its symlink.
+
+Of these, the first is less expensive (typically, an annex does not have every
+key in it). It could be optimized fairly simply, by adding a database
+of keys present in the annex that is optimised to list them all. The
+database would be updated by the few functions that move content in and
+out.
+
+The second is harder to optimise, because the user can delete, revert,
+copy, add, etc files in git at will, and git-annex does not have a good way
+to watch that and maintain a database of what keys are being referenced.
+
+It could use a post-commit hook and examine files changed by commits, etc.
+But then staged files would be left out. It might be sufficient to 
+make --fast trust the database... except unused will suggest *deleting*
+data if nothing references it. Or maybe it could be required to have a
+clean tree with nothing staged before running git-annex unused.
+
+Anyway, this is a semi-longterm item for me. --[[Joey]] 
diff --git a/doc/todo/cache_key_info/comment_1_578df1b3b2cbfdc4aa1805378f35dc48._comment b/doc/todo/cache_key_info/comment_1_578df1b3b2cbfdc4aa1805378f35dc48._comment
new file mode 100644
index 0000000000..086e7f3e84
--- /dev/null
+++ b/doc/todo/cache_key_info/comment_1_578df1b3b2cbfdc4aa1805378f35dc48._comment
@@ -0,0 +1,11 @@
+[[!comment format=mdwn
+ username="https://www.google.com/accounts/o8/id?id=AItOawl9sYlePmv1xK-VvjBdN-5doOa_Xw-jH4U"
+ nickname="Richard"
+ subject="comment 1"
+ date="2011-05-17T07:27:02Z"
+ content="""
+Sounds like a good idea.
+
+* git annex fsck (or similar) should check/rebuild the caches
+* I would simply require a clean tree with a verbose error. 80/20 rule and defaulting to save actions.
+"""]]
diff --git a/doc/todo/calckey_cannot_be_used_outside_of_git_repo.mdwn b/doc/todo/calckey_cannot_be_used_outside_of_git_repo.mdwn
new file mode 100644
index 0000000000..06645a196b
--- /dev/null
+++ b/doc/todo/calckey_cannot_be_used_outside_of_git_repo.mdwn
@@ -0,0 +1,11 @@
+Not sure if this is a bug or a feature request, but calckey can only be used while inside a git repository (though the specified file can be anywhere).
+
+Can this be changed so it can be run from anywhere?
+
+    ~$ git annex version
+    git-annex version: 6.20160613-g1e4e6f4
+
+    ~$ git annex calckey /some/file/some/wh.ere
+    git-annex: Not in a git repository.
+    CallStack (from HasCallStack):
+      error, called at ./Git/CurrentRepo.hs:55:37 in main:Git.CurrentRepo
diff --git a/doc/todo/calckey_cannot_be_used_outside_of_git_repo/comment_1_c54b53c5f81132bedf75eb6b439a202d._comment b/doc/todo/calckey_cannot_be_used_outside_of_git_repo/comment_1_c54b53c5f81132bedf75eb6b439a202d._comment
new file mode 100644
index 0000000000..6a9f823028
--- /dev/null
+++ b/doc/todo/calckey_cannot_be_used_outside_of_git_repo/comment_1_c54b53c5f81132bedf75eb6b439a202d._comment
@@ -0,0 +1,21 @@
+[[!comment format=mdwn
+ username="joey"
+ subject="""comment 1"""
+ date="2016-09-05T16:17:22Z"
+ content="""
+Well, calckey's behavior is influenced by the annex.backends setting,
+so it might behave differently when run in a different git repo, or outside
+any git repo.
+
+In fact, `git annex calckey` currently can be run in a git repo that is not
+a git-annex repo; it does not check if the repo is initialized for
+git-annex (which many commands do). The failure about "Not in a git
+repository." comes when it tries to run code in the Annex monad, which 
+requires a git repository.
+
+To support being run without being in a git repo at all would
+really need an entirely separate code path to be implemented, not using the
+Annex monad, and so necessarily not looking at git configs. It's doable,
+but it would be a lot of work; I'd need some convincing motivation to add
+a lot of code to accomplish that.
+"""]]
diff --git a/doc/todo/checkpresentkey_without_explicit_remote.mdwn b/doc/todo/checkpresentkey_without_explicit_remote.mdwn
new file mode 100644
index 0000000000..7ad667f28a
--- /dev/null
+++ b/doc/todo/checkpresentkey_without_explicit_remote.mdwn
@@ -0,0 +1,5 @@
+While being asked to check if file is available from "[datalad-archives]" remote I need to check if the archive's key available.  Ideally I wish I could ask through the ongoing interaction protocol, but if not, I could use smth like 'git annex checkpresentkey' but that one demands specification also of a remote which to check.  In my case I just want to know if that key is available from any remote, so I could confirm that the file is still present in our archives remote, i.e. that it could be retrieved later on
+
+[[!meta author=yoh]]
+
+> [[done]] --[[Joey]]
diff --git a/doc/todo/checksum_verification_on_transfer.mdwn b/doc/todo/checksum_verification_on_transfer.mdwn
new file mode 100644
index 0000000000..c9d505aecc
--- /dev/null
+++ b/doc/todo/checksum_verification_on_transfer.mdwn
@@ -0,0 +1,7 @@
+Since most file transfers, particularly to/from encrypted special remotes involve git-annex streaming through the contents of the file anyway, it should be possible to add a verification of the checksum nearly for free. The main thing needed is probably a faster haskell checksum library than Data.Digest.Pure.Sha, which is probably slow enough to be annoying.
+
+I have not verified if an upload could be aborted before sending the data to the remote if a checksum failure is detected. It may be dependent on the individual special remote implementations. Some probably stream the encrypted data directly out the wire, while others need to set up a temp file to run a command on. It would certainly be possible to at least make the upload abort and fail if a bad checksum was detected.
+
+Doing the same for downloads is less useful, because the data is there locally to be fscked. The real advantage would be doing the check for uploads, to ensure that hard-to-detect corrupted files don't reach special remotes.
+
+--[[Joey]]
diff --git a/doc/todo/checksum_verification_on_transfer/comment_1_30f77e631608b9751f9032f97d58cc30._comment b/doc/todo/checksum_verification_on_transfer/comment_1_30f77e631608b9751f9032f97d58cc30._comment
new file mode 100644
index 0000000000..5de1251da3
--- /dev/null
+++ b/doc/todo/checksum_verification_on_transfer/comment_1_30f77e631608b9751f9032f97d58cc30._comment
@@ -0,0 +1,17 @@
+[[!comment format=mdwn
+ username="https://www.google.com/accounts/o8/id?id=AItOawl9sYlePmv1xK-VvjBdN-5doOa_Xw-jH4U"
+ nickname="Richard"
+ subject="comment 1"
+ date="2013-09-09T11:50:05Z"
+ content="""
+Doing this during downloads would still be nice.
+
+While the files are easier to fsck, users will need to actually do this. If it happenend automatically, it would increase safety and reduce disk i/o.
+
+Of course, this will not detect degradation during/after writing.
+
+If you don't make it the default, please at least make it optional for us bordering on OCD when it comes to data storage.
+
+
+Richard
+"""]]
diff --git a/doc/todo/checksum_verification_on_transfer/comment_2_1267ff79ddc84dad146bdb11a7bdf8b2._comment b/doc/todo/checksum_verification_on_transfer/comment_2_1267ff79ddc84dad146bdb11a7bdf8b2._comment
new file mode 100644
index 0000000000..39f0a9e2c6
--- /dev/null
+++ b/doc/todo/checksum_verification_on_transfer/comment_2_1267ff79ddc84dad146bdb11a7bdf8b2._comment
@@ -0,0 +1,35 @@
+[[!comment format=mdwn
+ username="joey"
+ subject="""comment 2"""
+ date="2015-10-01T15:45:18Z"
+ content="""
+My original reasoning makes sense for uploads, I think. 
+
+The checksum library used is a lot faster now, but it would still be best
+to do the checksum as part of the same file read used to transfer the file,
+when possible.
+
+There is a good reason to want to verify checksums when downloading objects
+too: Git does that, and so if git-annex does too, the same reasoning about
+security can be done about git-annex repositories as can be done about git
+repositories. In other words, not verifying checksums when downloading objects
+violates least surprise.
+
+A concrete example: If the user is uploading objects to gitlab, they should
+be able to git pull, and verify their signed commit, and git annex get, and
+not need to worry about whether gitlab (or a MITM) could do something evil
+to the downloaded objects.
+
+Similarly, a S3 special remote does not include the git repo, so users
+should be able to assume that, given their locally trusted git repo, git
+annex get will only ever get verified objects from the S3 remote.
+
+Question: What about local repositories, eg on a removable drive?
+Git does do checksum verification between local repositories, unless
+cloned with --shared. Probably follows git-annex should too.
+
+My current thinking is that this verification should be done by default.
+Security features that are not enabled by default are not very useful.
+It should, however, be able to be turned off, either globally, or on a
+per-remote basis.
+"""]]
diff --git a/doc/todo/checksum_verification_on_transfer/comment_3_2fa9445619032a378264de8b59958c60._comment b/doc/todo/checksum_verification_on_transfer/comment_3_2fa9445619032a378264de8b59958c60._comment
new file mode 100644
index 0000000000..b18e7dcb5d
--- /dev/null
+++ b/doc/todo/checksum_verification_on_transfer/comment_3_2fa9445619032a378264de8b59958c60._comment
@@ -0,0 +1,17 @@
+[[!comment format=mdwn
+ username="joey"
+ subject="""status update"""
+ date="2015-10-01T19:17:38Z"
+ content="""
+Checksum verification is now done for all downloads, unless disabled via
+annex.verify=false.
+
+When an object is uploaded to a regular git remote, checksum verification
+also also done. (For a local directory, git-annex runs a download from the
+perspective of the remote, so we get it for free, and when git-annex-shell
+recvkey is used, it checksums the data it receives and compares it with the
+key.)
+
+For uploads to special remotes, no checksum verification is done yet.
+Leaving this todo item open because of that gap in the coverage.
+"""]]
diff --git a/doc/todo/cleaner_hack_for_man_pages.mdwn b/doc/todo/cleaner_hack_for_man_pages.mdwn
new file mode 100644
index 0000000000..63a4a2d15a
--- /dev/null
+++ b/doc/todo/cleaner_hack_for_man_pages.mdwn
@@ -0,0 +1,119 @@
+In recent history, we have realized that the small Perl script that
+generates the man pages from Markdown is fairly limited.
+
+Two approaches have been considered:
+
+* go-md2man
+* pandoc
+
+Here is how pandoc does it:
+
+    $ pandoc -f markdown -t man doc/git-annex-shell.mdwn | pastebinit
+    http://paste.debian.net/424341/
+
+Both initially fail at setting a proper `.TH` line on top, but
+otherwise seem to work more or less correctly. --[[anarcat]]
+
+Okay, update: the above commandline was incorrect for some reason. The
+proper incantation is:
+
+    pandoc -s -t man doc/git-annex-shell.mdwn -o git-annex-shell.1
+
+For example:
+
+    $ pandoc -s -t man doc/git-annex-shell.mdwn | man -l - | pastebinit
+    http://paste.debian.net/430630/
+
+So by default, there is no title or section header, which is, if you
+ask me a little stupid: pandoc could guess a little better and parse
+the `.SH NAME` section.
+
+The workaround for this is to add Pandoc metadata either to the file,
+for example:
+
+[[!format diff """
+diff --git a/doc/git-annex-shell.mdwn b/doc/git-annex-shell.mdwn
+index 9b3d126..13f64ae 100644
+--- a/doc/git-annex-shell.mdwn
++++ b/doc/git-annex-shell.mdwn
+@@ -1,3 +1,6 @@
++% git-annex-shell(1) Git-annex manual | Version 5
++% Joey Hess
++
+ # NAME
+ 
+ git-annex-shell - Restricted login shell for git-annex only SSH access
+"""]]
+
+But Ikiwiki is likely to barf on such comments, so it's probably
+preferable to pass those parameters at build time:
+
+    $ pandoc -s -V title="git-annex-shell" -V section=1 -t man doc/git-annex-shell.mdwn  | man -l - | pastebinit
+    http://paste.debian.net/430632/
+
+Looks better already! But we can improve on that even more!
+
+    $ pandoc -s -V title="git-annex-shell" -V section=1 \
+        -V header="Git Annex manual" -V footer="Version 5.xxx" \
+        -t man doc/git-annex-shell.mdwn  | man -l - | pastebinit
+    http://paste.debian.net/430633/
+
+Much better. And the version can probably be passed in from the build
+system (or that footer can just be dropped).
+
+So a more complete patch would involve fixing the build system to use
+(and depend on!) pandoc then remove the pesky warnings at the bottom
+of all Markdown files.
+
+More investigation would probably be necessary to check the resulting
+man pages for syntax errors. For example, the above rendering, in the
+`SEE ALSO` section, has `[git-annex] (1)` instead of
+`git-annex(1)`, since Pandoc doesn't know about ikiwiki links. Maybe
+some pre-processing would be necessary there? :/ It sure is useful to
+have those links working in the web version!
+
+I hope that helps regardless.
+
+Update: regarding preprocessing, it seems there are basically two
+options for preprocessing with pandoc:
+
+* [scripting](http://pandoc.org/scripting.html)
+* [preprocessors](https://github.com/jgm/pandoc/wiki/Pandoc-Extras#preprocessors)
+  like gpp
+* external filters
+
+Preprocessors don't support arbitrary patterns like Ikiwiki links, so
+they're out already. Scripting looks a little too complicated for us,
+but could be a more stable alternative in the long term (if not for
+Ikiwiki itself ;).
+
+So I ended up using a simple sed(1) filter for now. It would be quite
+interesting to implement a better filter directly in Pandoc, if only
+for the benefit of Ikiwiki (which could then all be reimplemented in
+Haskell, No Big Deal™). But for now, I am afraid that will have to do.
+
+The resulting manpages are quite different, [here's the diff][]. It
+seems mostly better escapes and markup, in general. We get nicer
+bullet lists, cleaner command names (`git-annex add` instead of
+`git-annex-add`).
+
+[here's the diff]: http://fpaste.org/353524/
+
+I have pushed a [patch][] that implements a Pandoc build to my
+[personnal git-annex repo][]. It also adds pandoc as a build-dep to
+the debian package - and I may be missing some parts of the build
+system, hopefully you will find the missing bits and fix them, if
+any.
+
+Pandoc is awesome, and I think a great fit for this! --[[anarcat]]
+
+Oh, and I forgot: another thing that is not handled through this
+(yet?) is the "Version" footer I was using in the above examples. I
+couldn't figure out how to pull that value out of the Makefile, but
+maybe I missed something obvious there. I also didn't notice the
+`git-union-merge.1` manpage, but it seems to be broken anyways, as it
+is not in the `man/` directory and there is a comment there saying it
+is "not built normally" so I ignored it.
+
+ [personnal git-annex repo]: http://src.anarc.at/git-annex.git/
+ [patch]: http://src.anarc.at/git-annex.git/commitdiff/d8a8f42f5a7ec0458718d65b72f1f814862b125b
diff --git a/doc/todo/cleaner_hack_for_man_pages/comment_1_d51be0df42cba6f3eba2b5157dc9d427._comment b/doc/todo/cleaner_hack_for_man_pages/comment_1_d51be0df42cba6f3eba2b5157dc9d427._comment
new file mode 100644
index 0000000000..b2243d75db
--- /dev/null
+++ b/doc/todo/cleaner_hack_for_man_pages/comment_1_d51be0df42cba6f3eba2b5157dc9d427._comment
@@ -0,0 +1,10 @@
+[[!comment format=mdwn
+ username="joey"
+ subject="""comment 1"""
+ date="2016-04-11T15:35:43Z"
+ content="""
+Comparing the outputs, the pandoc man page uses `\f[C]` for
+`backticked literals`. At least in my terminal, that does not render
+specially at all, unlike the `\fB` that mdwn2man uses to render them in
+bold. This makes some man pages significantly harder to follow.
+"""]]
diff --git a/doc/todo/cloning_direct_mode_repo_over_http.mdwn b/doc/todo/cloning_direct_mode_repo_over_http.mdwn
new file mode 100644
index 0000000000..8f471f6e47
--- /dev/null
+++ b/doc/todo/cloning_direct_mode_repo_over_http.mdwn
@@ -0,0 +1,36 @@
+Indirect mode repos can be cloned over http, and just work. But, direct
+mode repos don't currently work; while git can clone them ok, git-annex get
+doesn't know where to get the file contents from.
+
+To support this, git-annex would have to check if the remote is in direct
+mode, and when it is, it would need to download the direct mode mapping
+file, to find out which file has the content of a key. Then, after
+downloading the a file, it would need to make sure to checksum it, since
+nothing prevents a direct mode file from being modified at the same time
+it's downloaded.
+
+All seems doable. However.. [[design/caching_database]] wants to switch the
+direct mode mapping files from simple flat text files to a sqlite database.
+Which would complicate this a lot. Can sqlite databases be accessed over
+http, or would the whole, possibly large database need to be downloaded?
+If so, what to do when the database changes? Re-downloading a possibly
+large db is not good.
+
+---
+
+Alternatively, the direct mode mapping files of the remote could be
+bypassed. Instead, look at what the remote HEAD branch is, and look at that
+branch locally. Create local direct mode mappings for the remote HEAD
+branch, and use them when downloading.
+
+This approach would mean that, if the remote's HEAD changes and we haven't
+noticed, we might download the wrong file (that has eg, been moved).
+checksumming would detect this, but it does make it more fragile.
+
+Also, creating the direct mode mappings for a remote HEAD would currently
+be pretty slow. Probably implementing the caching database for direct mode
+mappings would lead to faster code. So, this feature seems best blocked on
+the direct mode database either way!
+
+--[[Joey]]
+[[!meta tag=deprecateddirectmode]]
diff --git a/doc/todo/cloning_direct_mode_repo_over_http/comment_1_36701696af41f8b71bb2fd4829f05b7c._comment b/doc/todo/cloning_direct_mode_repo_over_http/comment_1_36701696af41f8b71bb2fd4829f05b7c._comment
new file mode 100644
index 0000000000..a1abe3f2ba
--- /dev/null
+++ b/doc/todo/cloning_direct_mode_repo_over_http/comment_1_36701696af41f8b71bb2fd4829f05b7c._comment
@@ -0,0 +1,13 @@
+[[!comment format=mdwn
+ username="joey"
+ subject="""comment 1"""
+ date="2016-02-19T19:55:20Z"
+ content="""
+v6 unlocked files don't have this problem; a repo containing them can be
+accessed over http.
+
+So, I think that's a better way to go than complicating the http support
+with direct mode stuff. Especially since direct mode is going away.
+
+Won't close this yet, but will when I get around to removing direct mode..
+"""]]
diff --git a/doc/todo/config_setting_to_force_--progress_reporting_for_some_git_calls.mdwn b/doc/todo/config_setting_to_force_--progress_reporting_for_some_git_calls.mdwn
new file mode 100644
index 0000000000..711ca1828b
--- /dev/null
+++ b/doc/todo/config_setting_to_force_--progress_reporting_for_some_git_calls.mdwn
@@ -0,0 +1,7 @@
+Ref: all heated discussions around https://github.com/datalad/datalad/issues/2844
+
+The main point is -- `git annex init --version=6`, while running  on a crippled system might need some time to accomplish the mission of unlocking a big number of files, which leads to `Checking out` progress update spit out by `git`.  BUT if someone (us) swallows stderr, `git` does not produce any progress output unless that command (checkout, pull, fetch, etc...) provides explicit `--progress` flag.  According to IRC there is no config setting to trigger that behavior in `git`.  Could may be `git annex` get a config setting alike `force-git-progress` which would add `--progress` to any relevant `git` call whenever git is ran without stderr being swallowed by annex?
+
+Thanks in advance for considering
+
+[[!meta author=yoh]]
diff --git a/doc/todo/configuration_option_for_default___34__mode__34___on_crippled_file_systems.mdwn b/doc/todo/configuration_option_for_default___34__mode__34___on_crippled_file_systems.mdwn
new file mode 100644
index 0000000000..1e66af6f6c
--- /dev/null
+++ b/doc/todo/configuration_option_for_default___34__mode__34___on_crippled_file_systems.mdwn
@@ -0,0 +1,3 @@
+ATM `git-annex init` switches repository to direct mode if detects a crippled filesystem.  Since version 6 is now generally more usable than direct mode in many scenarios, it would be nice if it was possible to instruct to choose v6 mode instead of direct whenever a crippled system is encountered.
+
+[[!meta author=yoh]]
diff --git a/doc/todo/configuration_option_for_default___34__mode__34___on_crippled_file_systems/comment_1_e3cd8d86869d8a63337f7eff0b91ddbc._comment b/doc/todo/configuration_option_for_default___34__mode__34___on_crippled_file_systems/comment_1_e3cd8d86869d8a63337f7eff0b91ddbc._comment
new file mode 100644
index 0000000000..dbe286a286
--- /dev/null
+++ b/doc/todo/configuration_option_for_default___34__mode__34___on_crippled_file_systems/comment_1_e3cd8d86869d8a63337f7eff0b91ddbc._comment
@@ -0,0 +1,7 @@
+[[!comment format=mdwn
+ username="joey"
+ subject="""comment 1"""
+ date="2018-10-01T15:57:40Z"
+ content="""
+Wouldn't git annex init --version=6 work? If not that is a bug.
+"""]]
diff --git a/doc/todo/configuration_option_for_default___34__mode__34___on_crippled_file_systems/comment_2_3580025d828b27072530eb8ccda9bdf4._comment b/doc/todo/configuration_option_for_default___34__mode__34___on_crippled_file_systems/comment_2_3580025d828b27072530eb8ccda9bdf4._comment
new file mode 100644
index 0000000000..319002b34e
--- /dev/null
+++ b/doc/todo/configuration_option_for_default___34__mode__34___on_crippled_file_systems/comment_2_3580025d828b27072530eb8ccda9bdf4._comment
@@ -0,0 +1,9 @@
+[[!comment format=mdwn
+ username="yarikoptic"
+ avatar="http://cdn.libravatar.org/avatar/f11e9c84cb18d26a1748c33b48c924b4"
+ subject="comment 2"
+ date="2018-10-01T16:29:19Z"
+ content="""
+I think that is correct.
+But isn't `annex init` is also indirectly invoked by any annex command, e.g. if I just do `git clone URL ; cd DIR; git annex get FILEs`?
+"""]]
diff --git a/doc/todo/could_standalone___39__fixed__39___git-annex_binaries_be_prelinked__63__.mdwn b/doc/todo/could_standalone___39__fixed__39___git-annex_binaries_be_prelinked__63__.mdwn
new file mode 100644
index 0000000000..3b1b929eab
--- /dev/null
+++ b/doc/todo/could_standalone___39__fixed__39___git-annex_binaries_be_prelinked__63__.mdwn
@@ -0,0 +1,7 @@
+Since in datalad we are invoking git and git-annex quite frequently, and on debian systems atm relying on git-annex-standalone pkg, I wondered, if there is a possibility to get all 'shimmed' binaries prelinked against shipped core libs to avoid a current bunch of unsucesfull searches for libraries.... I thought it might provide a notable benefit.
+
+just an idea
+
+[[!meta author=yoh]]
+
+> [[fixed|done]], but without prelinking. --[[Joey]]
diff --git a/doc/todo/ctrl_c_handling.mdwn b/doc/todo/ctrl_c_handling.mdwn
new file mode 100644
index 0000000000..7101d578f5
--- /dev/null
+++ b/doc/todo/ctrl_c_handling.mdwn
@@ -0,0 +1,5 @@
+Sometimes I start off a large file transfer to a new remote (a la "git-annex copy . --to glacier").
+
+I believe all of the special remotes transfer the files one at a time, which is good, and provides a sensible place to interrupt a copy/move operation.
+
+Wish: When I press ctrl+c in the terminal, git-annex will catch that and finish it's current transfer and then exit cleanly (ie: no odd backtraces in the special remote code). For the case where the file currently being transfered also needs to be killed (ie: it's a big .iso) then subsequent ctrl+c's can do that.
diff --git a/doc/todo/ctrl_c_handling/comment_1_3addbe33817db5de836c014287b14c07._comment b/doc/todo/ctrl_c_handling/comment_1_3addbe33817db5de836c014287b14c07._comment
new file mode 100644
index 0000000000..16139c78da
--- /dev/null
+++ b/doc/todo/ctrl_c_handling/comment_1_3addbe33817db5de836c014287b14c07._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="http://joeyh.name/"
+ ip="209.250.56.172"
+ subject="comment 1"
+ date="2014-02-21T21:36:14Z"
+ content="""
+This really depends on the remote, some can resume where they were interrupted, such as rsync, and some cannot, such as glacier (and, er, encrypted rsync).
+"""]]
diff --git a/doc/todo/ctrl_c_handling/comment_2_cc2776dc4805421180edcdf96a89fcaa._comment b/doc/todo/ctrl_c_handling/comment_2_cc2776dc4805421180edcdf96a89fcaa._comment
new file mode 100644
index 0000000000..827b99afa1
--- /dev/null
+++ b/doc/todo/ctrl_c_handling/comment_2_cc2776dc4805421180edcdf96a89fcaa._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="http://grossmeier.net/"
+ nickname="greg"
+ subject="very remote specific"
+ date="2014-02-21T22:11:16Z"
+ content="""
+Yeah, this is very remote specific and probably means adding the functionality there as well (eg: in the glacier.py code, not only in git-annex haskell). Maybe I should file bugs there accordingly :)
+"""]]
diff --git a/doc/todo/ctrl_c_handling/comment_3_8d7d357368987f5d5d59b4d8d99a0e06._comment b/doc/todo/ctrl_c_handling/comment_3_8d7d357368987f5d5d59b4d8d99a0e06._comment
new file mode 100644
index 0000000000..ed7e4d3b62
--- /dev/null
+++ b/doc/todo/ctrl_c_handling/comment_3_8d7d357368987f5d5d59b4d8d99a0e06._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="http://joeyh.name/"
+ ip="209.250.56.172"
+ subject="comment 3"
+ date="2014-02-21T22:34:14Z"
+ content="""
+Hmm, I forget if it's possible for git-annex to mask SIGINT when it runs glacier or rsync, so that the child process does not receive it, but the parent git-annex does.
+"""]]
diff --git a/doc/todo/custom_f-droid_repo.mdwn b/doc/todo/custom_f-droid_repo.mdwn
new file mode 100644
index 0000000000..7d32d45d57
--- /dev/null
+++ b/doc/todo/custom_f-droid_repo.mdwn
@@ -0,0 +1,3 @@
+It would be great to have a custom f-droid repo "alla guardianproject.info" (before getting git-annex into the main f-droid repo).
+
+See  ().
diff --git a/doc/todo/custom_f-droid_repo/comment_1_d2bdc001584d4b5f925390910ec1ef73._comment b/doc/todo/custom_f-droid_repo/comment_1_d2bdc001584d4b5f925390910ec1ef73._comment
new file mode 100644
index 0000000000..f1c682fbc6
--- /dev/null
+++ b/doc/todo/custom_f-droid_repo/comment_1_d2bdc001584d4b5f925390910ec1ef73._comment
@@ -0,0 +1,10 @@
+[[!comment format=mdwn
+ username="http://joeyh.name/"
+ ip="209.250.56.146"
+ subject="comment 1"
+ date="2014-03-05T20:45:32Z"
+ content="""
+The f-droid developers are willing to add git-annex in principle, their build environment just needs to have its build dependencies added to it. I've completely automated (except for occasional breakage due to changes on hackage) to setup of the necessary cross build environment. I have not had time to look into how to integrate that into the f-droid build system, and it would be great if someone with interest could do so.
+
+I suspect that setting up a custom f-droid repo would be just as much work, plus more work on an ongoing basis.
+"""]]
diff --git a/doc/todo/custom_f-droid_repo/comment_2_20eebe13b76d5279a3d09b346b65ff6e._comment b/doc/todo/custom_f-droid_repo/comment_2_20eebe13b76d5279a3d09b346b65ff6e._comment
new file mode 100644
index 0000000000..34aa063f4c
--- /dev/null
+++ b/doc/todo/custom_f-droid_repo/comment_2_20eebe13b76d5279a3d09b346b65ff6e._comment
@@ -0,0 +1,9 @@
+[[!comment format=mdwn
+ username="https://www.google.com/accounts/o8/id?id=AItOawnR6E5iUghMWdUGlbA9CCs8DKaoigMjJXw"
+ nickname="Efraim"
+ subject="comment 2"
+ date="2014-03-06T10:06:02Z"
+ content="""
+I've noticed that fdroidserver is available in the main debian repos, and not having really looked at it too much yet, if it shares similar code to wannabuild then it should be possible to continue everything as you're already doing it, just in addition rsync or git push the code base to a local fdroidserver instance and also copy the binary over once it's set up.
+(if I understood wannabuild correctly, if you upload new source and a prebuilt binary it will mark that architecture as built.)
+"""]]
diff --git a/doc/todo/custom_f-droid_repo/comment_3_5a79abb8b1dd12426e111e733fa6493b._comment b/doc/todo/custom_f-droid_repo/comment_3_5a79abb8b1dd12426e111e733fa6493b._comment
new file mode 100644
index 0000000000..dda348874d
--- /dev/null
+++ b/doc/todo/custom_f-droid_repo/comment_3_5a79abb8b1dd12426e111e733fa6493b._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="https://www.google.com/accounts/o8/id?id=AItOawmLB39PC89rfGaA8SwrsnB6tbumezj-aC0"
+ nickname="Tobias"
+ subject="comment 3"
+ date="2014-03-06T22:49:22Z"
+ content="""
+F-droid compiles all the binaries themselves internally.
+"""]]
diff --git a/doc/todo/custom_f-droid_repo/comment_4_55f05624f0e939f7b8d0c505285e5690._comment b/doc/todo/custom_f-droid_repo/comment_4_55f05624f0e939f7b8d0c505285e5690._comment
new file mode 100644
index 0000000000..a122aae927
--- /dev/null
+++ b/doc/todo/custom_f-droid_repo/comment_4_55f05624f0e939f7b8d0c505285e5690._comment
@@ -0,0 +1,11 @@
+[[!comment format=mdwn
+ username="http://lj.rossia.org/users/imz/"
+ ip="79.165.57.121"
+ subject="this topic in their Submission Queue"
+ date="2014-05-03T14:59:11Z"
+ content="""
+I opened [a request to add git-annex](https://f-droid.org/forums/topic/git-annex/) in the F-Droid's submission queue about a year ago.
+
+Just posting here the link, so that all the discussion (not much) can be found by those interested.
+
+"""]]
diff --git a/doc/todo/custom_f-droid_repo/comment_6_de4229f04daf48a153e2f44f57a05a3b._comment b/doc/todo/custom_f-droid_repo/comment_6_de4229f04daf48a153e2f44f57a05a3b._comment
new file mode 100644
index 0000000000..033e35377a
--- /dev/null
+++ b/doc/todo/custom_f-droid_repo/comment_6_de4229f04daf48a153e2f44f57a05a3b._comment
@@ -0,0 +1,11 @@
+[[!comment format=mdwn
+ username="http://lj.rossia.org/users/imz/"
+ ip="79.165.57.121"
+ subject=" clarification: a custom F-Droid server just to post your updated apps"
+ date="2014-05-03T15:57:55Z"
+ content="""
+If I've understood Efraim correctly, he means that someone (perhaps, Joey) can run a custom F-Droid server with Joey's builds of the app **copied to the repo as is (without rebuilding there)**, so that those using an F-Droid client would get the updates through this standard path (for them) instead of manually looking for updates here.
+
+Like from a custom APT repository run not by Debian (in this case -- F-Droid team), but by an independent developer who is providing package builds not avaialble in Debian by building the packages himself, signing them, and putting them into his own APT repo (indexing them).
+
+"""]]
diff --git a/doc/todo/deferred_update_mode.mdwn b/doc/todo/deferred_update_mode.mdwn
new file mode 100644
index 0000000000..d7eec4ad7e
--- /dev/null
+++ b/doc/todo/deferred_update_mode.mdwn
@@ -0,0 +1,35 @@
+`git annex sync` and the assistant do a merge of new revs, and then
+download the content of files. However, this means that broken links can
+show up, when a file has changed, or a new file was added. In some
+workflows, the user would prefer not to ever see such broken links
+(or at least never for files that are in the repo's preferred content).
+
+So, how about a new mode, that defers updating the work tree until
+the content of everything wanted is available?
+
+This could be a annex.merge=downloadfirst setting; it would make sync/assistant
+look at the diff between HEAD and the new rev, and try to get all annexed files
+added in that diff, before merging it. 
+
+Of course, it could take a long time to get to see a new work tree.
+Might have to download a lot of content.
+
+What to do if it fails to download a file's content? Could either
+abort, leaving the current work tree as-is, or could go ahead and merge,
+letting broken links show up in this case. I kind of prefer the abort
+option. But, if the content never reaches any remote, or has gone missing
+entirely, that would make sync never succeed. That could be surprising
+behavior.
+
+Probably best to let the user pick either behavior, so
+annex.merge=trydownloadfirst and annex.merge=reqdownloadfirst
+
+Let `git annex merge` be used to force a merge, even when content is not
+available.
+
+## alternatively
+
+What about just making `git-annex sync --content` try to get the content of
+all files before updating the work tree? (The assistant would need changes
+too; it would need to queue all the downloads and trigger a work tree
+update once all the downloads have been tried.)
diff --git a/doc/todo/ditch_yesod.mdwn b/doc/todo/ditch_yesod.mdwn
new file mode 100644
index 0000000000..0c85ab77d2
--- /dev/null
+++ b/doc/todo/ditch_yesod.mdwn
@@ -0,0 +1,36 @@
+I'd like to move away from using yesod for the web app, for a number of
+reasons:
+
+* It's by far the largest use of TH in git-annex, and TH is of course very
+  painful for the android port, other builds like debian mips that don't
+  currently support TH, etc. (Update: mips does support it now, it seems!)
+* I think it's responsible for at least 50% of the executable size, and I
+  suspect a lot of that is unncessary bloat for parts of yesod that
+  git-annex doesn't really use.
+* Hamlet constantly annoys me by rejecting any file that contains tabs.
+  **Rage**
+* Hamlet contains code that's not really haskell, but looks a lot like it.
+  This is a continual frustration when dealing with the hamlet files.
+* I find that Hamlet does not lend itself to being refactored, everything
+  is essentially an IO action with side effects of generating html, so
+  can't really bring proper FP tools to bear.
+
+At the moment, servant seems like the nicest place to end up. Just as type
+safe as yesod afaik, and very lightweight and simple and rather awesome.
+
+As for the html generation, lucid seems like a good way to do it. Just as
+fast as hamlet, and pure haskell code.
+
+Game plan:
+
+1. Upload a yesod-lucid to hackage. (done)
+2. Get servant (done), lucid (done), maybe yesod-lucid, packaged in Debian
+3. Start converting individual yesod Handler Html to Handler LucidHtml.
+   This will ditch the hamlet. This is by far the most work, but it can be
+   done incrementally w/o breaking the build.
+4. Need to find a way to deal with Hamlet widgets that add CS and JS to
+   the page. Hamlet basically serves as a Writer monad to collect the HTML,
+   CS, and JS that widgets add to a page, so a replacement for that would
+   need to be developed.
+5. Once all the hamlet is gone, remove the rest of the yesod stuff and
+   re-implement the routing etc with servant.
diff --git a/doc/todo/ditch_yesod/comment_1_976dcb99ebe30c02c9b223f19a67c875._comment b/doc/todo/ditch_yesod/comment_1_976dcb99ebe30c02c9b223f19a67c875._comment
new file mode 100644
index 0000000000..d09065e016
--- /dev/null
+++ b/doc/todo/ditch_yesod/comment_1_976dcb99ebe30c02c9b223f19a67c875._comment
@@ -0,0 +1,7 @@
+[[!comment format=mdwn
+ username="umeboshi"
+ subject="Link to TH"
+ date="2015-12-29T20:36:41Z"
+ content="""
+I had to google for [TH](https://wiki.haskell.org/Template_Haskell).  I'm posting this link for those who, like me, are less familiar with Haskell.
+"""]]
diff --git a/doc/todo/do_not_block_on_crypto_key_generation.mdwn b/doc/todo/do_not_block_on_crypto_key_generation.mdwn
new file mode 100644
index 0000000000..90e4e9e58a
--- /dev/null
+++ b/doc/todo/do_not_block_on_crypto_key_generation.mdwn
@@ -0,0 +1,22 @@
+While trying to reproduce a [[problem with the gcrypt remote|bugs/gcrypt_repository_not_found]], I found that the `encryption step` can hand on this gpg command:
+
+    gpg --batch --no-tty --use-agent --quiet --trust-model always --gen-random --armor 2 512
+
+I think that "depletes the entropy pool", as witnessed by the state of the kernel entropy pool in `/proc/sys/kernel/random/entropy_avail`. As it turns out, `--gen-random 2` reads bytes from `/dev/random` and, indeed, by default that pool is 4096 bits large so the above command completely drains that pool to zero because it creates a 4096 bit (512 bytes * 8) key. It can take several minutes to "refill" the pool, which means batch-creating keys like this can take forever and also have an impact on other components of the system which rely on the kernel random number generator.
+
+Using `/dev/random` in that way is a somewhat controversial practice. Indeed, [some people recommend using urandom for all purposes](https://www.2uo.de/myths-about-urandom/), including secret key material generation. 
+
+From what I understand `urandom` and `random` basically use the same entropy source on Linux: the only difference is the latter blocks when it "thinks" it does not have enough entropy. But this (running out of entropy, not "thinking" that we do) basically never happens on Linux systems unless we are on the very first boot after an install. Because it's unlikely that git-annex will run on such an environment, I would discourage the use of `--gen-random 2` in git-annex.
+
+I strangely could not find out *where* exactly gpg is called in that way. All i could find was Util.Gpg.genRandom but that seems to hardcode `--gen-random 1`, not `2`, so I don't exactly know what's going on here. But I'm pretty sure it's git-annex calling it: 
+
+    anarcat  12745  0.3  0.2 1074098148 36304 pts/6 Sl+ 13:34   0:00              \_ /usr/bin/git-annex initremote encrypted type=gcrypt gitrepo=/home/anarcat/tmp/b keyid=8DC901CE64146C048AD50FBB792152527B75921E
+    anarcat  12753  0.0  0.0  16028  3652 pts/6    S+   13:34   0:00                  \_ git --git-dir=.git --work-tree=. --literal-pathspecs cat-file --batch
+    anarcat  12754  0.0  0.0  16028  3308 pts/6    S+   13:34   0:00                  \_ git --git-dir=.git --work-tree=. --literal-pathspecs cat-file --batch-check=%(objectname) %(objecttype) %(objectsize)
+    anarcat  12756  0.0  0.0  33476  3784 pts/6    SL+  13:34   0:00                  \_ gpg --batch --no-tty --use-agent --quiet --trust-model always --gen-random --armor 2 512
+
+I was hoping this was something git-remote-gcrypt would be doing, but it's not: this is git-annex calling. Maybe some off-by-one conversion error somewhere?
+
+Thank you for your time... -- [[anarcat]]
+
+> [[closing|done]] --[[Joey]]
diff --git a/doc/todo/do_not_block_on_crypto_key_generation/comment_1_339ffe281eb75b70dd7ea8bc5ae55125._comment b/doc/todo/do_not_block_on_crypto_key_generation/comment_1_339ffe281eb75b70dd7ea8bc5ae55125._comment
new file mode 100644
index 0000000000..b92ea47447
--- /dev/null
+++ b/doc/todo/do_not_block_on_crypto_key_generation/comment_1_339ffe281eb75b70dd7ea8bc5ae55125._comment
@@ -0,0 +1,15 @@
+[[!comment format=mdwn
+ username="joey"
+ subject="""comment 1"""
+ date="2018-07-03T16:23:05Z"
+ content="""
+This is implemented in Utility.Gpg.genRandom. There is no off-by-one,
+git-annex intentially makes the same default choices that gnupg does
+about random quality.
+
+--fast makes it use /dev/urandom for people who lean on that side of the
+entropy controversy. initremote's man page documents this.
+
+("Some people recommend" is often not a good basis for security defaults.
+Some people recommend using RDRAND and trusting Intel...)
+"""]]
diff --git a/doc/todo/do_not_block_on_crypto_key_generation/comment_2_34b1e7f04ca17438e2a3e6209993a3a9._comment b/doc/todo/do_not_block_on_crypto_key_generation/comment_2_34b1e7f04ca17438e2a3e6209993a3a9._comment
new file mode 100644
index 0000000000..edb4ca05f2
--- /dev/null
+++ b/doc/todo/do_not_block_on_crypto_key_generation/comment_2_34b1e7f04ca17438e2a3e6209993a3a9._comment
@@ -0,0 +1,32 @@
+[[!comment format=mdwn
+ username="anarcat"
+ avatar="http://cdn.libravatar.org/avatar/4ad594c1e13211c1ad9edb81ce5110b7"
+ subject="some further considerations"
+ date="2018-07-04T02:17:49Z"
+ content="""
+> This is implemented in Utility.Gpg.genRandom. There is no off-by-one, git-annex intentially makes the same default choices that gnupg does about random quality.
+
+Then you'll be happy to know there is a proposal to change those defaults in gnupg as well, for similar reasons:
+
+https://dev.gnupg.org/T3894
+
+Cargo-culting those defaults does not seem like a reasonable way either. ;)
+
+> --fast makes it use /dev/urandom for people who lean on that side of the entropy controversy. initremote's man page documents this.
+
+I did not notice that, thanks. I would still argue it is better to use `/dev/urandom` for cryptoraphic purposes.
+
+> (\"Some people recommend\" is often not a good basis for security defaults. Some people recommend using RDRAND and trusting Intel...)
+
+That's a straw man. I am not recommending RDRAND. I'm recommending `/dev/urandom`. There are drawbacks in using `/dev/random`. To quote from the above URL:
+
+> djb has argued that an attacker capable of controlling the inputs to your seeding algorithm might gain an advantage in a continuous-seeding scenario that they wouldn't get in a seed-once-and-done approach.
+
+There are, as far as I am aware, no drawback to using `/dev/urandom`. I know there is some sort of controversy about this: I'm actually taking a stand here and asking people to justify why they are using `/dev/random`.
+
+I understand you might decide to not justify this directly in git-annex, however, and defer to projects like GnuPG, but I figured you might like to know the issue is not so clear cut there either... Werner Koch, the main author of GnuPG, says this of using `/dev/urandom`:
+
+> I use it for a long time now and it is really helpful. Do this only on Linux with the getrandom system call or if you are sure that Libgcrypt is not used in the early boot stage.
+
+So I think it might be worth reconsidering this here as well.
+"""]]
diff --git a/doc/todo/do_not_block_on_crypto_key_generation/comment_3_6fad0d287d5aaa76ec5a8177f7bec231._comment b/doc/todo/do_not_block_on_crypto_key_generation/comment_3_6fad0d287d5aaa76ec5a8177f7bec231._comment
new file mode 100644
index 0000000000..1df6756597
--- /dev/null
+++ b/doc/todo/do_not_block_on_crypto_key_generation/comment_3_6fad0d287d5aaa76ec5a8177f7bec231._comment
@@ -0,0 +1,13 @@
+[[!comment format=mdwn
+ username="joey"
+ subject="""comment 3"""
+ date="2018-07-04T16:17:41Z"
+ content="""
+git-annex outsources all cryptography (aside from hashes) to gnupg; it is
+not cargo-culting to include cryptographic key generation in that set.
+
+If gnupg changes to use getrandom by default then git-annex will follow suite.
+
+(Also, use of "cargo-culting" "straw man" etc tends to not lead to good
+quality discussions. EOT for me.)
+"""]]
diff --git a/doc/todo/do_not_block_on_crypto_key_generation/comment_4_33d80e721c6a922574ac98dd3075c4d8._comment b/doc/todo/do_not_block_on_crypto_key_generation/comment_4_33d80e721c6a922574ac98dd3075c4d8._comment
new file mode 100644
index 0000000000..dbb9c4cc7b
--- /dev/null
+++ b/doc/todo/do_not_block_on_crypto_key_generation/comment_4_33d80e721c6a922574ac98dd3075c4d8._comment
@@ -0,0 +1,14 @@
+[[!comment format=mdwn
+ username="anarcat"
+ avatar="http://cdn.libravatar.org/avatar/4ad594c1e13211c1ad9edb81ce5110b7"
+ subject="apologies"
+ date="2018-07-05T15:56:27Z"
+ content="""
+I'm sorry, I misunderstood this comment:
+
+> git-annex intentially makes the same default choices that gnupg does about random quality
+
+I thought you were saying you replicated gnupg's policy decisions but now I understand you delegate those to GnuPG which seems like a reasonable choice! And hopefully this will all be fixed when GnuPG is fixed.
+
+Sorry for the trouble... :/
+"""]]
diff --git a/doc/todo/do_not_bug_me_about_intermediate_files.mdwn b/doc/todo/do_not_bug_me_about_intermediate_files.mdwn
new file mode 100644
index 0000000000..99a8f5b194
--- /dev/null
+++ b/doc/todo/do_not_bug_me_about_intermediate_files.mdwn
@@ -0,0 +1,7 @@
+[[!meta title="--notify-finish operates on a per-file, not per-process basis"]]
+
+so this is another UX pickyness, but it seems important to me.
+
+i like the new [[desktop notifications|tips/file_manager_integration]], but they are little too verbose. when i choose "git annex get" on the folder, if there's a lot of files, it will flood me with all the files being transfered in a mostly incomprehensible list of files being transfered.
+
+what i would expect is more: "starting transfer of folder X", "transfer of folder X finished!", only two message per item i chose. this is especially a problem with DVD backups, which have a bunch of small files (screenshots, .nfos and so on) and large video files - so it seems the thing has finished transfering, while it's only partly done. --[[anarcat]]
diff --git a/doc/todo/document_standard_groups_more_extensively_in_the_UI.mdwn b/doc/todo/document_standard_groups_more_extensively_in_the_UI.mdwn
new file mode 100644
index 0000000000..1e9afdef90
--- /dev/null
+++ b/doc/todo/document_standard_groups_more_extensively_in_the_UI.mdwn
@@ -0,0 +1,14 @@
+i have been using git-annex for a while now, yet I still can't quite wrap my head around [[preferred_content/standard groups]], especially how they are documented in about/repogroups in the assistant web interface. i have repeatedly synced files where they shouldn't have been synced (usually by setting the repo as "client" or "transfer") and also destroyed files I wanted to keep by setting it to "unwanted" (actually, that was by pressing the "delete" button on the repo, which i didn't expect to drop the files on the remote...)
+
+i have been able to understand a lot of what's going on by trial and error and by decrypting the [[preferred_content]] expressions on the wiki.
+
+it seems to me the [[preferred_content/standard groups]] wiki page and the `about/repogroups` URL in the assistant should be merged:
+
+ 1. the assistant should be more explicit: maybe it should have examples of what will happen in some cases to give an idea. maybe "stories" like "a transfer repo is for when you have two client repos that can't talk to each other, so you use a transfer repo, e.g. a portable hard drive, to transfer files between them". having the actual, current [[preferred_content]] expressions from the [[preferred_content/standard groups]] groups page would also help, maybe in a smaller font to not scare people of
+ 2. the [[preferred_content/standard groups]] wiki page should be expanded to include narratives like the ones that are in the `about/repogroups` page of the assistant. that way people looking at the software from the outside can understand the mechanics better
+
+ideally, that documentation would be the one and the same so that a change on one side would reflect on the other.
+
+in fact, having an inline manual in the assistant would be a must: we want this thing to work offline, so it should be able to access this wiki, or whatever of it is shipped with git-annex.
+
+that way we wouldn't have this kind of inconsistencies... more generally, maybe we could even [[build a user guide]]! -- [[anarcat]]
diff --git a/doc/todo/does_not_preserve_timestamps.mdwn b/doc/todo/does_not_preserve_timestamps.mdwn
new file mode 100644
index 0000000000..0d8f2371d4
--- /dev/null
+++ b/doc/todo/does_not_preserve_timestamps.mdwn
@@ -0,0 +1,16 @@
+### Please describe the problem.
+I see that files are synced between my computers with git-annex but the timestamps do not match. The one that receives files always puts the current time of file creation on the file.
+
+### What steps will reproduce the problem?
+Install git-annex on two computers. Connect with XMPP. Then add cloud storage with shared encryption for transferring files. Since you want also backup, choose "full backup" as the type of cloud storage.
+
+
+### What version of git-annex are you using? On what operating system?
+Downloaded binary package dated 13/09/2014 amd64 Ubuntu 14.04.
+
+
+### Please provide any additional information below.
+
+Files are in sync. For example, I move a file from a directory to my synced annex directory. It contains timestamp of 01/01/2010 for example. Once the file gets transferred to the remote computer, it gets current time, for example 20/09/2014 rather than keeping 01/01/2010. 
+
+All computers are linux based, ext4 filesystems. File transfers are done through shared encryption rsync remote.
diff --git a/doc/todo/does_not_preserve_timestamps/comment_10_8cce043ff86fefc3dcd97cf5c4428786._comment b/doc/todo/does_not_preserve_timestamps/comment_10_8cce043ff86fefc3dcd97cf5c4428786._comment
new file mode 100644
index 0000000000..7080a5877e
--- /dev/null
+++ b/doc/todo/does_not_preserve_timestamps/comment_10_8cce043ff86fefc3dcd97cf5c4428786._comment
@@ -0,0 +1,14 @@
+[[!comment format=mdwn
+ username="vrs+annex@ea5fa24dbb279be61a8e50adb638bf8366300717"
+ nickname="vrs+annex"
+ avatar="http://cdn.libravatar.org/avatar/74219abcec6eece8e2c9d4351c2c912c"
+ subject="related bugs"
+ date="2018-03-17T19:06:46Z"
+ content="""
+I've created a number of timestamp-related bugs that will hopefully result in better timestamp support:
+
+* [annex.genmetadata should default to true](/bugs/annex.genmetadata_should_default_to_true/)
+* [file modification time should be stored in exactly one metadata field](/bugs/file_modification_time_should_be_stored_in_exactly_one_metadata_field/)
+* [if annex.genmetadata is true, modification metadata is imported from older file versions after unlock+add](/bugs/if_annex.genmetadata_is_true__44___modification_metadata_is_imported_from_older_file_versions_after_unlock+add/)
+* [proposal for timestamp semantics](/bugs/proposal_for_timestamp_semantics/)
+"""]]
diff --git a/doc/todo/does_not_preserve_timestamps/comment_1_caf5e5cb17f4d05fff8c2fab661cd93f._comment b/doc/todo/does_not_preserve_timestamps/comment_1_caf5e5cb17f4d05fff8c2fab661cd93f._comment
new file mode 100644
index 0000000000..48ec44d2b1
--- /dev/null
+++ b/doc/todo/does_not_preserve_timestamps/comment_1_caf5e5cb17f4d05fff8c2fab661cd93f._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="http://joeyh.name/"
+ ip="209.250.56.144"
+ subject="comment 1"
+ date="2014-09-23T20:27:25Z"
+ content="""
+The closest git comes to storing a timestamp is the date of the last commit of a file for mtime, and first commit for ctime. However, those are pretty expensive to look up for a given file. And git doesn't try to preserve timestamps in checkouts at all, which argues that git-annex, at least at the command line, should not either.
+"""]]
diff --git a/doc/todo/does_not_preserve_timestamps/comment_2_c337fca1474b5b78f61ad6f421138ae4._comment b/doc/todo/does_not_preserve_timestamps/comment_2_c337fca1474b5b78f61ad6f421138ae4._comment
new file mode 100644
index 0000000000..4b5a750cf2
--- /dev/null
+++ b/doc/todo/does_not_preserve_timestamps/comment_2_c337fca1474b5b78f61ad6f421138ae4._comment
@@ -0,0 +1,12 @@
+[[!comment format=mdwn
+ username="https://www.google.com/accounts/o8/id?id=AItOawmK0703vNSIQsP1mGf-4MAPnsBZiSc6yVo"
+ nickname="Emre"
+ subject="comment 2"
+ date="2014-09-23T20:58:10Z"
+ content="""
+Thanks Joey for the comment. 
+
+But when syncing two repos, timestamps are critical at least for my use case. I can't lose this info. Even if it's expensive. 
+
+Appreciate if you can consider to add it for direct mode repos, ie when a file is synced to another repo and created there, it shall carry at least the mtime of the file in source repo. Owncloud sync does it, btsync does it, although I know git-annex is different than those. 
+"""]]
diff --git a/doc/todo/does_not_preserve_timestamps/comment_3_9a3eeddc46e5a420575f00cb47caf703._comment b/doc/todo/does_not_preserve_timestamps/comment_3_9a3eeddc46e5a420575f00cb47caf703._comment
new file mode 100644
index 0000000000..ba34823d45
--- /dev/null
+++ b/doc/todo/does_not_preserve_timestamps/comment_3_9a3eeddc46e5a420575f00cb47caf703._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="https://www.google.com/accounts/o8/id?id=AItOawmK0703vNSIQsP1mGf-4MAPnsBZiSc6yVo"
+ nickname="Emre"
+ subject="comment 3"
+ date="2014-09-23T21:15:29Z"
+ content="""
+Btw, git storing the last commit time as Mtime is not enough, it shall store the original timestamp of the file, not the date of commit. Hope I could explain and hope this is something doable.
+"""]]
diff --git a/doc/todo/does_not_preserve_timestamps/comment_4_99b064259fc2e3c6eb83c3da3b2d3bac._comment b/doc/todo/does_not_preserve_timestamps/comment_4_99b064259fc2e3c6eb83c3da3b2d3bac._comment
new file mode 100644
index 0000000000..08de756190
--- /dev/null
+++ b/doc/todo/does_not_preserve_timestamps/comment_4_99b064259fc2e3c6eb83c3da3b2d3bac._comment
@@ -0,0 +1,10 @@
+[[!comment format=mdwn
+ username="http://svario.it/gioele"
+ nickname="gioele"
+ subject="comment 4"
+ date="2014-09-24T07:15:09Z"
+ content="""
+You can try to store the timestamps just before commit and restore them on checkout.
+
+Have a look at [metastore](https://github.com/przemoc/metastore): it is a ready-made solution for plain git. Maybe you can adapt it to git-annex.
+"""]]
diff --git a/doc/todo/does_not_preserve_timestamps/comment_5_c95c8b9bd617830604500213c962fc7a._comment b/doc/todo/does_not_preserve_timestamps/comment_5_c95c8b9bd617830604500213c962fc7a._comment
new file mode 100644
index 0000000000..7632548cdf
--- /dev/null
+++ b/doc/todo/does_not_preserve_timestamps/comment_5_c95c8b9bd617830604500213c962fc7a._comment
@@ -0,0 +1,9 @@
+[[!comment format=mdwn
+ username="https://www.google.com/accounts/o8/id?id=AItOawlM_DRhi_5pJrTA0HbApHR25iAgy-NBXTY"
+ nickname="Tor Arne"
+ subject="comment 5"
+ date="2014-10-01T22:02:43Z"
+ content="""
+I was just about to start using git-annex for my data when I saw this bug report, which unfortunately is a blocker. Persisting basic metadata of files that are synced seems like a core feature of a file sync/transfer tool, so I'm really hoping this can be solved somehow :/
+
+"""]]
diff --git a/doc/todo/does_not_preserve_timestamps/comment_6_b99e00d0bc4258c4cb28b544b19ea3b8._comment b/doc/todo/does_not_preserve_timestamps/comment_6_b99e00d0bc4258c4cb28b544b19ea3b8._comment
new file mode 100644
index 0000000000..67fe255950
--- /dev/null
+++ b/doc/todo/does_not_preserve_timestamps/comment_6_b99e00d0bc4258c4cb28b544b19ea3b8._comment
@@ -0,0 +1,12 @@
+[[!comment format=mdwn
+ username="https://www.google.com/accounts/o8/id?id=AItOawlM_DRhi_5pJrTA0HbApHR25iAgy-NBXTY"
+ nickname="Tor Arne"
+ subject="comment 6"
+ date="2014-10-01T22:39:01Z"
+ content="""
+Isn't this what the metadata feature does though? http://git-annex.branchable.com/design/metadata/
+
+With annex.genmetadata true set, it should store year and mont (but not day/time? if so why not?
+
+Is the missing piece of the puzzle to apply the metadata again on checkout?
+"""]]
diff --git a/doc/todo/does_not_preserve_timestamps/comment_7_eefa6afce824f1069bb5216b392405b7._comment b/doc/todo/does_not_preserve_timestamps/comment_7_eefa6afce824f1069bb5216b392405b7._comment
new file mode 100644
index 0000000000..19c508c508
--- /dev/null
+++ b/doc/todo/does_not_preserve_timestamps/comment_7_eefa6afce824f1069bb5216b392405b7._comment
@@ -0,0 +1,7 @@
+[[!comment format=mdwn
+ username="fastguy"
+ subject="Any updates?"
+ date="2015-09-25T19:35:18Z"
+ content="""
+Hi, would there be any updates on this issue or do we still not preserve timestamps when syncing between locations?
+"""]]
diff --git a/doc/todo/does_not_preserve_timestamps/comment_8_f2a32e39356f7b772ab9cf3136436700._comment b/doc/todo/does_not_preserve_timestamps/comment_8_f2a32e39356f7b772ab9cf3136436700._comment
new file mode 100644
index 0000000000..6085171f25
--- /dev/null
+++ b/doc/todo/does_not_preserve_timestamps/comment_8_f2a32e39356f7b772ab9cf3136436700._comment
@@ -0,0 +1,10 @@
+[[!comment format=mdwn
+ username="graboluk@f6de53961ab0f884e203f602f65eb5cdc0fb7513"
+ nickname="graboluk"
+ subject="timestamps are wrong as of 5.20150731"
+ date="2015-09-26T18:31:46Z"
+ content="""
+I confirm this issue is still present in the debian testing version 5.20150731. This really makes git-annex quite unusable for me, as part of my setup is that my server downloads various podcasts (think news videos from euronews, german tagesschau, some youtube podcasts), which then get distributed to my other computers. This way I don't have to use youtube etc., which makes a huge difference in terms of speed (my main home  computer is from 2006 and it plays all hte videos just fine, but a website likie youtube.com is too much for it). At a speed of single shortcut I see a list of newest podcasts and I can choose one without bothering to open the webbrowser.
+
+The point is that I really like to have all those podcasts to be sorted by their modified date, for rather obvious reasons. (with dropbox it works fine.) 
+"""]]
diff --git a/doc/todo/does_not_preserve_timestamps/comment_9_62736ee2f299d62f685800353fc36ccf._comment b/doc/todo/does_not_preserve_timestamps/comment_9_62736ee2f299d62f685800353fc36ccf._comment
new file mode 100644
index 0000000000..ed9c0e41ea
--- /dev/null
+++ b/doc/todo/does_not_preserve_timestamps/comment_9_62736ee2f299d62f685800353fc36ccf._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="kalle@bdf75651b439b088e51f28f10f5a46ffcd2a704d"
+ nickname="kalle"
+ subject="importfeed template"
+ date="2015-09-28T19:52:16Z"
+ content="""
+You can use the `--template` flag and prefix the filename with the date. See [[/tips/downloading_podcasts]]. If you need very accurate modification times that won't work but should work in this case?
+"""]]
diff --git a/doc/todo/done.mdwn b/doc/todo/done.mdwn
new file mode 100644
index 0000000000..2ab037cfbd
--- /dev/null
+++ b/doc/todo/done.mdwn
@@ -0,0 +1,4 @@
+recently fixed [[todo]] items.
+
+[[!inline pages="./* and link(./done) and !*/Discussion" sort=mtime show=0
+archive=yes feedlimit=10]]
diff --git a/doc/todo/drop_--batch.mdwn b/doc/todo/drop_--batch.mdwn
new file mode 100644
index 0000000000..7757987525
--- /dev/null
+++ b/doc/todo/drop_--batch.mdwn
@@ -0,0 +1,6 @@
+There is a dropkey --batch, so I guess I could workaround but probably would be nice for consistency to have --batch mode for drop itself as well
+
+[[!meta author=yoh]]
+
+> [[done]]; went ahead and added drop --batch to be symmetric with get
+> --batch. --[[Joey]]
diff --git a/doc/todo/dumb__44___unsafe__44___human-readable_backend.mdwn b/doc/todo/dumb__44___unsafe__44___human-readable_backend.mdwn
new file mode 100644
index 0000000000..6bb1b5bf64
--- /dev/null
+++ b/doc/todo/dumb__44___unsafe__44___human-readable_backend.mdwn
@@ -0,0 +1,15 @@
+This was already discussed twice previously: [[forum/original_filename_on_s3/]] and [[todo/Facilitate_public_pretty_S3_URLs/]]. Yet I am still constantly having problems with this when trying to use git-annex to do some particular stuff.
+
+My latest example is a friend that wanted to sync files to a remote Webdav server. While I know he can use [[special_remotes/webdav/]] support, the resulting filenames on the webdav server will be basically garbled beyond recognition for any user that will use the Webdav server without git-annex.
+
+I understand the rationale behind the existing [[backends]], but wouldn't it be possible to implement a new backend that would just use the filename as a key?
+
+I know this would have several downsides:
+
+* lack of deduplication
+* much more exposed to corruption (no checksum to check against recorded? or can this be put somewhere else?)
+
+The main advantage, for me, is much better interoperability: any remote becomes usable by other non-git-annex clients... It would also be great as it would allow me to store only a *part* of my git-annex files on a remote without having a forest of empty files (on broken filesystems) or symlinks (on real filesystems) for files that are missing, something that is a massive source of confusion for users I work with. It could, for example, allow me to create thumb drives that would solve the [[hide missing files]] problem. -- [[anarcat]]
+
+> [[done]]; the new `git-annex export` feature allows you to export a tree
+> to a special remote. --[[Joey]]
diff --git a/doc/todo/dumb__44___unsafe__44___human-readable_backend/comment_1_d8db68bd9ebc4447617cfcb5c0c164a0._comment b/doc/todo/dumb__44___unsafe__44___human-readable_backend/comment_1_d8db68bd9ebc4447617cfcb5c0c164a0._comment
new file mode 100644
index 0000000000..b3614e8d15
--- /dev/null
+++ b/doc/todo/dumb__44___unsafe__44___human-readable_backend/comment_1_d8db68bd9ebc4447617cfcb5c0c164a0._comment
@@ -0,0 +1,11 @@
+[[!comment format=mdwn
+ username="joey"
+ subject="""comment 1"""
+ date="2016-02-09T16:37:11Z"
+ content="""
+Use what filename? A key can have any number of associated filenames in one
+or several git branches. The filenames can change at any time.
+
+AFAICS, this is a "pony" request. It seems like a good idea, but doesn't
+really work when you look at the details.
+"""]]
diff --git a/doc/todo/dumb__44___unsafe__44___human-readable_backend/comment_2_7620c429812d5ea1a649cbfda1cb09bb._comment b/doc/todo/dumb__44___unsafe__44___human-readable_backend/comment_2_7620c429812d5ea1a649cbfda1cb09bb._comment
new file mode 100644
index 0000000000..fce8a7879b
--- /dev/null
+++ b/doc/todo/dumb__44___unsafe__44___human-readable_backend/comment_2_7620c429812d5ea1a649cbfda1cb09bb._comment
@@ -0,0 +1,23 @@
+[[!comment format=mdwn
+ username="konubinix"
+ subject="fuser and unison?"
+ date="2016-02-10T09:12:52Z"
+ content="""
+Dear anarcat, I totally understand your use case since I often get into situations like that. I think that, by design (content addressable storage etc.), git-annex is not at all suited to fulfill this use case.
+
+I managed to use a combination of a fuse mount and unison to reach that use case.
+
+I create a fuse view of the git annex working copy with pyfs (http://pyfilesystem.org/) via a command line like : fsmount -f -a file:///my_git_annex_working_copy my_share_directory
+
+Generally, I share the fuse mount over samba, ftp and http.
+
+In order to synchronize with a distant directory (the use case you mentioned), I use another fuse mount and unison to sync both.
+
+Generally, I share the content on a remote samba server, using: 'smbnetfs ~/Network' and then 'unison my_share_directory ~/Network/PathToTheRemote'
+
+In the case of a s3 drive, I don't know the details since I never went there, but you could try: 'fsmount -f -a s3:///my_s3_remote my_s3_directory', then 'unison my_share_directory my_s3_directory'
+
+You may think that all those mounts are difficult to manage. As a matter of fact, I use supervisord (http://supervisord.org/) that helps making sure that everything is running OK. I have been using that setup for a few years now, without trouble.
+
+I hope that I could help.
+"""]]
diff --git a/doc/todo/dumb__44___unsafe__44___human-readable_backend/comment_3_d46183e53b1e6293681efadec9b46c6b._comment b/doc/todo/dumb__44___unsafe__44___human-readable_backend/comment_3_d46183e53b1e6293681efadec9b46c6b._comment
new file mode 100644
index 0000000000..b21e9e21db
--- /dev/null
+++ b/doc/todo/dumb__44___unsafe__44___human-readable_backend/comment_3_d46183e53b1e6293681efadec9b46c6b._comment
@@ -0,0 +1,18 @@
+[[!comment format=mdwn
+ username="anarcat"
+ subject="""comment 3"""
+ date="2016-04-09T04:39:11Z"
+ content="""
+What about having a direct more in an external work tree?
+
+I could have my .git directory in some local directory (say
+`~/.git-annex/remote-mp3.git`) and the actual files on the device (say
+`/media/anarcat/mounted/mp3`).
+
+I know that there are many possible names for a single key, but isn't
+that a problem for direct mode as well?
+
+And I know it's a poney... I keep on asking for poneys here, but
+poneys are great, and actually quite useful and even friendly animals
+if you can use and take care of them correctly. :)
+"""]]
diff --git a/doc/todo/dumb__44___unsafe__44___human-readable_backend/comment_4_bbee2fa4b4fed1c5117cc4ddb0395278._comment b/doc/todo/dumb__44___unsafe__44___human-readable_backend/comment_4_bbee2fa4b4fed1c5117cc4ddb0395278._comment
new file mode 100644
index 0000000000..845eb7f169
--- /dev/null
+++ b/doc/todo/dumb__44___unsafe__44___human-readable_backend/comment_4_bbee2fa4b4fed1c5117cc4ddb0395278._comment
@@ -0,0 +1,13 @@
+[[!comment format=mdwn
+ username="anarcat"
+ subject="dumb external remote implemented"
+ date="2016-06-23T00:34:59Z"
+ content="""
+so I went ahead and scratched my itch of implementing this dumb special remote. i think it would be better implemented as a backend, but i am not sure how to implement that without going deep into haskell, and it seems like a mad idea anyways, because it would need to modify the hashing structure as well (to keep the directory structure).
+
+the remote is a simple shell script inspired by the example script, but that does some nasty (and slow) `git log -S` stuff to find a matching file path to a given key. ideally, this would be streamlined into a git-annex command that could then be optimized, because as it is now, each call is a separate history search.
+
+the script is here: 
+
+i haven't added it to the [[special_remotes]] list yet, because maybe it warrants a separate [[tips]] page. i also do not want to introduce what can be seen as a \"bad idea\" from git-annex's integrity principles to the list of special remotes before agreement here.
+"""]]
diff --git a/doc/todo/dumb__44___unsafe__44___human-readable_backend/comment_5_941b2364433ae42bc67a75f93564331d._comment b/doc/todo/dumb__44___unsafe__44___human-readable_backend/comment_5_941b2364433ae42bc67a75f93564331d._comment
new file mode 100644
index 0000000000..ac2b16ae1e
--- /dev/null
+++ b/doc/todo/dumb__44___unsafe__44___human-readable_backend/comment_5_941b2364433ae42bc67a75f93564331d._comment
@@ -0,0 +1,26 @@
+[[!comment format=mdwn
+ username="anarcat"
+ subject="related issues and response to problems raised"
+ date="2016-06-23T14:44:06Z"
+ content="""
+there are two related issues that were closed as wontfix here, which i missed in my original search as well:
+
+* [[todo/clear file names in special remotes]] ([archive](http://web.archive.org/web/20160413162353/http://git-annex.branchable.com/todo/clear_file_names_in_special_remotes/))
+* [[todo/New special remote suggeston - clean directory]] ([archive](http://web.archive.org/web/20160413171909/http://git-annex.branchable.com/todo/New_special_remote_suggeston_-_clean_directory/))
+
+those issues were actually removed, but are still on the internet archive for future reference.
+
+to summarize those issues, the rationale there is that those remotes are potentially destructuve (lack of locking and checks) and have workarounds (`rsync -a` was suggested). it is also clearly stated that this is contrary to the git-annex design and is a \"pony\" feature.
+
+i would counter that this is an often requested feature that seems to be a major usability issue for a lot of people. there are other unsafe remotes out there, like the `bittorrent` special remote, which is explicitely documented as such, and where we recommend users `untrust` the remote when it is setup. yet, those remotes have their uses and the rich diversitry of such remotes makes git-annex one of the most interesting projects out there.
+
+furthermore, `rsync -a` is not the same as git-annex's excellent tracking features. in my use case ([[forum/syncing_music_to_my_android_device]]), there is no git annex repository that has the same set of files which I want present on the android device, so I cannot use `rsync` without incurring a large storage space cost.
+
+as for this being contrary to git-annex design, you obviously know more about this than me, but from the outside it doesn't seem *that* counter-intuitive. it seems that we have go through a lot of hoops to try to make stuff like [[todo/Facilitate_public_pretty_S3_URLs]] work at all. having a different backend for specially crafted remotes would be a huge gain in usability and impact on data security could be limited with the usual trust/untrust mechanisms.
+
+i would be curious to hear more about how such a backend would be contrary to git-annex's design. i am assuming here that my main repo could still have a SHA256E backend and *some* remotes could have *different* backends. Obviously, maybe that's a flawed assumption and I obviously see how such a dumb backend could break way more assumptions git-annex makes about its data, if it *has* to be used on all the remotes. Yet, there *are* backends right now like `URL` and `WORM` that could be considered \"unsafe\", yet do not provide as great usability gains as this dumb backend could provide.
+
+i understand you have been requested this feature often and would understand if this other request would just be closed again. but considering how often it comes up, from different users, i think it should at least be considered as something that should be more explicitely documented (in [[not]], [[backends]] and [[special_remotes]] maybe?) to keep further requests from coming in. keeping an issue like this opened would also help in avoiding duplicate requests.
+
+thank you for your time and efforts on git-annex, i cannot state enough how helpful it is to me. the fact that I could write a special remote to accomplish the above filthy todo is a testament to how flexible and powerful git-annex is. :)
+"""]]
diff --git a/doc/todo/dumb__44___unsafe__44___human-readable_backend/comment_6_63b0da8fe5cba74411e16070d4dcf4bb._comment b/doc/todo/dumb__44___unsafe__44___human-readable_backend/comment_6_63b0da8fe5cba74411e16070d4dcf4bb._comment
new file mode 100644
index 0000000000..859dfb461e
--- /dev/null
+++ b/doc/todo/dumb__44___unsafe__44___human-readable_backend/comment_6_63b0da8fe5cba74411e16070d4dcf4bb._comment
@@ -0,0 +1,45 @@
+[[!comment format=mdwn
+ username="joey"
+ subject="""comment 6"""
+ date="2016-10-31T19:50:06Z"
+ content="""
+So, I tried your git-annex-remote-dumb. After fixing its calckey to work with the current version of git-annex ...
+
+	joey@darkstar:~/tmp/repo>git annex initremote dumb type=external externaltype=dumb  directory=/tmp/dumb encryption=none
+	initremote dumb ok
+	joey@darkstar:~/tmp/repo>echo hello > hello
+	joey@darkstar:~/tmp/repo>git annex add
+	add hello ok
+	joey@darkstar:~/tmp/repo>git commit -m add
+	joey@darkstar:~/tmp/repo>git annex move hello --to dumb
+	move hello (to dumb...) 
+	ok
+	joey@darkstar:~/tmp/repo>cat /tmp/dumb/hello
+	hello
+	joey@darkstar:~/tmp/repo>git mv hello goodbye
+	joey@darkstar:~/tmp/repo>git commit -a -m rename
+	joey@darkstar:~/tmp/repo>echo oh no > hello
+	joey@darkstar:~/tmp/repo>git annex add hello
+	add hello ok
+	joey@darkstar:~/tmp/repo>git commit -a -m foo
+	joey@darkstar:~/tmp/repo>git annex copy hello --to dumb
+	copy hello (to dumb...) 
+	ok                      
+	joey@darkstar:~/tmp/repo>cat /tmp/dumb/hello
+	oh no
+	joey@darkstar:~/tmp/repo>git annex whereis goodbye
+	whereis goodbye (1 copy) 
+	   	3ef932e6-0989-46bd-a4c6-86562ed713f5 -- [dumb]
+	ok
+	joey@darkstar:~/tmp/repo>git annex fsck --from dumb goodbye
+	fsck goodbye (fixing location log) 
+	  ** Based on the location log, goodbye
+	  ** was expected to be present, but its content is missing.
+	failed
+	(recording state in git...)
+
+So yeah, that's data loss, and it's just the first thing I thought of.
+
+Aside from data loss bugs, /tmp/dumb just can't be kept fully up-to-date
+with changes to the working tree. So what's the point of it?
+"""]]
diff --git a/doc/todo/dumb__44___unsafe__44___human-readable_backend/comment_7_10086c35bbd54e502303a8763cc06afd._comment b/doc/todo/dumb__44___unsafe__44___human-readable_backend/comment_7_10086c35bbd54e502303a8763cc06afd._comment
new file mode 100644
index 0000000000..39bac1887d
--- /dev/null
+++ b/doc/todo/dumb__44___unsafe__44___human-readable_backend/comment_7_10086c35bbd54e502303a8763cc06afd._comment
@@ -0,0 +1,28 @@
+[[!comment format=mdwn
+ username="https://anarc.at/openid/"
+ nickname="anarcat"
+ avatar="http://cdn.libravatar.org/avatar/b36dcf65657dd36128161355d8920a99503def9461c1bb212410980fe6f07125"
+ subject="re dataloss and point"
+ date="2017-01-17T01:34:43Z"
+ content="""
+I think you wouldn't have had data loss if you untrusted the repository, am I correct? The remote is explicitly marked as \"unsafe\" (and ideally, it would automatically declare that to git-annex, but maybe there's no way to setup the trust level of remotes automatically like that) so operations that *move* files to it will be denied unless there's another copy somewhere...
+
+As for \"what's the point of it\", I thought I made that pretty clear... It enables use cases like:
+
+* [[forum/syncing_music_to_my_android_device/]]
+* [[forum/original_filename_on_s3/]]
+* [[todo/Facilitate_public_pretty_S3_URLs]]
+* etc
+
+> Aside from data loss bugs, /tmp/dumb just can't be kept fully up-to-date with changes to the working tree. So what's the point of it?
+
+I'm not sure what you mean there, by \"can't be kept fully up up-to-date\", could you clarify?
+
+Do you mean it won't follow renames and such? In my use case, I don't mind: i can just trash the remote and recreate it, if the working tree changes - and it doesn't change often enough for this to be a problem.
+
+> After fixing its calckey to work with the current version of git-annex...
+
+would you mind sharing that fix? :)
+
+thanks for the feedback, and sorry for the delay - for some reason i never got notifications for the reply... :/
+"""]]
diff --git a/doc/todo/easy_way_to_reproduce_normal_download_command.mdwn b/doc/todo/easy_way_to_reproduce_normal_download_command.mdwn
new file mode 100644
index 0000000000..5e913a544d
--- /dev/null
+++ b/doc/todo/easy_way_to_reproduce_normal_download_command.mdwn
@@ -0,0 +1,31 @@
+so there's a `annex.web-download-command` settings, great! i have used
+it to do bandwidth limitations with:
+
+    wget --limit-rate=$download -O %file %url
+
+...and it works well.. however, i notice the output is different, and
+that's because once we override that setting, git-annex doesn't add
+other magic settings it uses. so by turning that thing on and off and
+inspecting the process list, i was able to figure out that the command
+is actually:
+
+    wget -q --show-progress --clobber -c --limit-rate=$download -O %file %url
+
+that gets me closer to the command generated by git-annex. however, i
+am not sure the `--clobber` and `-c` flags should always be
+present. furthermore, the `--user-agent
+git-annex/5.20150610+gitg608172f-1~ndall+1` flag is missing.
+
+could there be an (optional) %useragent interpolation to fix that?
+
+what about the other settings, is it okay to hardcode those?
+
+maybe this would be easier if there would be an options override just
+like rsync, but separate ones for curl and wget... --[[anarcat]]
+
+> git-annex now only uses curl, and defaults to a built-in http downloader.
+> The annex.web-download-command is no longer supported. annex.web-options
+> can be used to pass options to curl.
+> 
+> So, I don't think this todo is relevant anymore, closing [[done]].
+> --[[Joey]]
diff --git a/doc/todo/easy_way_to_reproduce_normal_download_command/comment_1_26ddfa03fcf51250b963cb9511a38404._comment b/doc/todo/easy_way_to_reproduce_normal_download_command/comment_1_26ddfa03fcf51250b963cb9511a38404._comment
new file mode 100644
index 0000000000..2c050ffa34
--- /dev/null
+++ b/doc/todo/easy_way_to_reproduce_normal_download_command/comment_1_26ddfa03fcf51250b963cb9511a38404._comment
@@ -0,0 +1,15 @@
+[[!comment format=mdwn
+ username="joey"
+ subject="""comment 1"""
+ date="2015-07-02T20:36:22Z"
+ content="""
+Separate options could be added, but I'm not sure they're needed.
+git-annex only ever runs one of curl or wget on a given system.
+
+Which one varies depending on what is installed (ie, it uses wget when
+available and otherwise uses curl), but as long as that's
+stable, you can just use annex.web-options to pass additional options to
+whichever one it runs.
+
+FWIW, --limit-rate works for both curl and wget, more or less the same.
+"""]]
diff --git a/doc/todo/enable_a_discussion_forum_or_support_system.mdwn b/doc/todo/enable_a_discussion_forum_or_support_system.mdwn
new file mode 100644
index 0000000000..ba057ea384
--- /dev/null
+++ b/doc/todo/enable_a_discussion_forum_or_support_system.mdwn
@@ -0,0 +1,134 @@
+I has been [[discussed|devblog/day_268_stressed_out]]
+[[twice|devblog/day_285__tuning_git-annex_unused_refs]] that the
+current [[forum]] support system is not ideal for providing
+support. It constitutes the [largest part of this wiki][] yet we
+sometimes get duplicated questions and it seems the forum may be a
+barrier to entry for people.
+
+ [largest part of this wiki]: http://git-annex.branchable.com/devblog/day_268_stressed_out/#comment-e0814585df0047e6d4e11515aebe1dec
+
+A few alternatives have been proposed, from a mailing list to using a
+StackExchange site. This post is to discuss the possible
+alternatives. The requirement is that the system may be available
+offline and on low-bandwidth connections yet enable conversations
+better than a simple web forum or mailing list; which may end up being
+the only solution considering the first requirements, but let's give
+it a try. :)
+
+[[!toc]]
+
+Stack exchange
+==============
+
+i looked up how this works in stackexchange, and it turns out they
+provide [regular dumps][] of the data hosted. unfortunately, it's a
+gigantic zip file and not really designed for low-bandwidth
+use. There's also a [data explorer][] that was promising, until i
+realized that the query engine takes up 154KB, three times the space
+of the regular search engine (~54KB, which I guess is way too much for
+dialup).
+
+ [regular dumps]: http://blog.stackoverflow.com/category/cc-wiki-dump/
+ [data explorer]: http://data.stackexchange.com/
+
+Also: there is a [mobile version][] of all the stackexchange sites,
+which take up around 3KB, but still only works online. There's an app
+for mobile devices as well, but it [doesn't support offline access
+either][] - so another dead-end.
+
+ [mobile version]: http://meta.stackexchange.com/questions/104458/switch-to-mobile-site-on-standard-browser
+ [doesn't support offline access either]: http://meta.stackexchange.com/questions/220036/offline-features-in-stack-exchange-app
+
+Stackexchange runs a [garbled mess][] of IIS, MSSQL, ASP, Redis and
+Elasticsearch so there is probably little hope in integrating into
+something like git. I have asked for the [proverbial pony in this
+stackexchange question][] (yes, it's all very meta), describing the
+use case, but i am not holding my breath for a change.
+
+ [garbled mess]: http://en.wikipedia.org/wiki/Stack_Exchange#Technologies_used
+ [proverbial pony in this stackexchange question]: http://meta.stackexchange.com/questions/256573/best-way-to-use-stackexchange-sites-offline-or-on-dialup
+ 
+Askbot
+======
+
+Besides, SE sites are based on proprietary software, and i am not sure
+we'd want to advocate that here. There are free software alternatives,
+of which i made an [evaluation about 2 years ago][] after which we
+tried Askbot at [Koumbit][]. it never took off: staff didn't seem
+interested in it so much and we never made people aware of it too
+much. plus, it didn't integrate with existing authentication
+mechanisms (which are a little bit of a mess for us)...
+
+But it does seem like an interesting alternative, and has [primitive
+email integration][] where you can ask questions by email, but not
+answer (!). I haven't looked at such support in other software, but i
+suspect it is similarly not a priority as they struggle to monetize
+their free software...
+
+Most of those free software alternatives of Stackexchange are written
+in Python/Django or Ruby/Rails (iirc) with SQL backends, which would
+probably make git integration... also "challenging".
+
+ [evaluation about 2 years ago]: (https://wiki.koumbit.net/FaqService#A.2BAMk-valuation_logicielle)
+ [Koumbit]: http://koumbit.org/
+ [primitive email integration]: http://askbot.org/doc/sending-email-to-askbot.html
+
+Discourse
+=========
+
+From that point of view, maybe [discourse][] could be an interesting
+alternative. It has built-in email support, as it is designed as an
+alternative to mailing lists. With a simple setting, you can
+collaborate directly by email. What is interesting, compared to the
+mailing list model, is that posts can be collaboratively edited to
+(for example) arrive at a collective wisdom by refining an answer (or
+a question). There are also interesting community moderation and
+reputation systems.
+
+There seems to be some [offline tool][] for Discourse, but it's
+unclear to me how it works, as it seems to be a separate (nodejs!)
+application that connects with a discourse plugin. There's a
+[discussion about offline suport in discourse][] in the meta site...
+
+Discourse uses Ruby/Rails and PostgreSQL, and somewhat is only
+[officially supported in Docker][]...
+
+The discourse people do [warn against switching existing
+communities][] to Discourse because of the resulting friction, so it
+is also in that spirit that I open discussion about this here, that
+is, in the hope the change can be acknowledged, supported and
+discussed by everyone here, in a spirit of consensus building. :)
+
+ [offline tool]: https://github.com/etewiah/offcourse
+ [discussion about offline suport in discourse]: https://meta.discourse.org/t/offcourse-a-proof-of-concept-offline-reader-for-discourse/22356
+ [discourse]: http://discourse.org/
+ [warn against switching existing communities]: http://www.discourse.org/faq/#switch
+ [officially supported in Docker]: https://github.com/discourse/discourse/blob/master/docs/INSTALL.md
+
+Mailing lists
+=============
+
+Of course, the forum could simply be moved to a mailing list. The
+would have the advantage of reducing the barrier of entry to the site,
+reduce the load on the wiki, at the cost of adding the additionnal
+"how do i subscribe to a mailing list" barrier of entry, which, oddly
+enough, exists for a lot of mailing lists...
+
+Like it or not, web forums do enable a lot of less technical users to
+participate because they only need their web browser, and crossing
+that boundary can be hard for some people. Mailing lists also do not
+necessarily favor collaboration as much as the other systems, because
+they are articulated more towards discussion, as opposed to establishing
+common knowledge.
+
+A good option may be Mailman 3 which, with the new web interface, allows
+forum-like functionalities. Mailman 2 also used to have a usenet interface,
+not sure what became of that in Mailman 3.
+
+Web forums
+==========
+
+There's a plethora of "php-bb-like" software out there, i am just
+ignoring them for now, as I am not very familiar with them and they
+have never been too attractive for me, but people are of course free
+to edit this page to add suggestions! :)
diff --git a/doc/todo/enable_a_discussion_forum_or_support_system/comment_1_578c0ab40d53dd56bb3f2d98491c5e85._comment b/doc/todo/enable_a_discussion_forum_or_support_system/comment_1_578c0ab40d53dd56bb3f2d98491c5e85._comment
new file mode 100644
index 0000000000..4b57009bc2
--- /dev/null
+++ b/doc/todo/enable_a_discussion_forum_or_support_system/comment_1_578c0ab40d53dd56bb3f2d98491c5e85._comment
@@ -0,0 +1,13 @@
+[[!comment format=mdwn
+ username="joey"
+ subject="""comment 1"""
+ date="2015-05-19T21:32:58Z"
+ content="""
+Of these, only Discourse seems interesting to me. (Or email list.)
+As long as it supports responding by email, it would work for me, and
+collaboratively editing is a good thing. Of course, we try to do that on
+the tips pages already..
+
+Note that as of today, users can log into this website using
+just their email address.
+"""]]
diff --git a/doc/todo/enable_a_discussion_forum_or_support_system/comment_2_b1129b4cdfe39e740c4f7cf0d6ff2b91._comment b/doc/todo/enable_a_discussion_forum_or_support_system/comment_2_b1129b4cdfe39e740c4f7cf0d6ff2b91._comment
new file mode 100644
index 0000000000..c0252718bb
--- /dev/null
+++ b/doc/todo/enable_a_discussion_forum_or_support_system/comment_2_b1129b4cdfe39e740c4f7cf0d6ff2b91._comment
@@ -0,0 +1,12 @@
+[[!comment format=mdwn
+ username="joey"
+ subject="""comment 2"""
+ date="2015-05-22T17:26:08Z"
+ content="""
+I think that mailman 3 could be a good solution for lots of different
+users.
+
+It is emphatically not the kind of thing I enjoy maintaining though! :)
+
+Example: 
+"""]]
diff --git a/doc/todo/enable_a_discussion_forum_or_support_system/comment_3_52598af3d7b050981e32ad1569f69db9._comment b/doc/todo/enable_a_discussion_forum_or_support_system/comment_3_52598af3d7b050981e32ad1569f69db9._comment
new file mode 100644
index 0000000000..263753ba7d
--- /dev/null
+++ b/doc/todo/enable_a_discussion_forum_or_support_system/comment_3_52598af3d7b050981e32ad1569f69db9._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="joey"
+ subject="""comment 3"""
+ date="2015-06-02T17:22:49Z"
+ content="""
+FWIW, now that logging in by email address is supported, I've seen traffic
+to this website tick back up. 
+"""]]
diff --git a/doc/todo/export.mdwn b/doc/todo/export.mdwn
new file mode 100644
index 0000000000..7c3bf0ee64
--- /dev/null
+++ b/doc/todo/export.mdwn
@@ -0,0 +1,17 @@
+`git annex export` corresponding to import. This might be useful for eg,
+datalad. There are some requests to make eg a S3 bucket mirror the
+filenames in the git annex repository with incremental updates, 
+which seem out of scope (and there are many tools to do stuff like that
+search "deploy files to S3 bucket"), 
+but something simpler like `git annex export` could be worth doing.
+
+`git annex export --to remote files` would copy the files to the remote,
+using the names in the working tree. For remotes like S3, it could add the
+url of the exported file, so that another clone of the repo could use the
+exported data.
+
+Would this be able to reuse the existing `storeKey` interface, or would
+there need to be a new interface in supported remotes?
+
+> [[done]]! --[[Joey]]
+
diff --git a/doc/todo/export/comment_1_cb87f7518da252b950d70c60352e848e._comment b/doc/todo/export/comment_1_cb87f7518da252b950d70c60352e848e._comment
new file mode 100644
index 0000000000..c07acc5caa
--- /dev/null
+++ b/doc/todo/export/comment_1_cb87f7518da252b950d70c60352e848e._comment
@@ -0,0 +1,28 @@
+[[!comment format=mdwn
+ username="anarcat"
+ avatar="http://cdn.libravatar.org/avatar/4ad594c1e13211c1ad9edb81ce5110b7"
+ subject="sounds like the dumb backend, except not dumb"
+ date="2017-04-08T20:21:41Z"
+ content="""
+This sounds a lot like what i was trying to do in
+[[todo/dumb, unsafe, human-readable_backend]], except done properly. :)
+
+I was wondering about that asymmetry recentrly, and it would seem like
+a good idea to fix this. the `--to remote` flag could especially be
+useful for directory, rsync or even webdav remotes. i am not sure how
+this could be implemented, but i would certainly use this.
+
+having addurl work would be an awesome bonus, especially with webdav,
+where the mapping can be easily guessed (like S3). could be trickier
+with rsync/directory because then the user would need to tell
+git-annex what the base url is somewhere, but would fix a ton of use
+cases i had for this, like [[forum/original filename on s3]],
+[[forum/Facilitate public pretty S3 URLs]], [[todo/hide_missing_files]]
+and, my favorite, [[forum/syncing music to my android device]].
+it would also automate, extend and simplify the
+[[tips/publishing_your_files_to_the_public/]] use case.
+
+so thanks for this effort, i think it's a great idea and i'm excited
+to test this! -- [[anarcat]]
+
+"""]]
diff --git a/doc/todo/export/comment_2_b13f12dd00347114f99fd11d85d236a0._comment b/doc/todo/export/comment_2_b13f12dd00347114f99fd11d85d236a0._comment
new file mode 100644
index 0000000000..4e394dd9e9
--- /dev/null
+++ b/doc/todo/export/comment_2_b13f12dd00347114f99fd11d85d236a0._comment
@@ -0,0 +1,16 @@
+[[!comment format=mdwn
+ username="joey"
+ subject="""comment 2"""
+ date="2017-05-11T17:25:48Z"
+ content="""
+Note that making `git annex export` need a remote to export to would be
+somewhat asymmetric compared to `git annex import` which can operate on any
+directory full of files.
+
+Still, if you want to export to a directory full of files, setting up a
+directory special remote before exporting is not too big a pain, probably.
+
+I suppose the question is whether this asymetricity and complication of
+perhaps needing changes to the remote interface for `export` is a price
+worth paying to be able to have export upload to lots of sorts of remotes.
+"""]]
diff --git a/doc/todo/export/comment_3_38e0b7bac8f2cfad492704ab6ab81e2b._comment b/doc/todo/export/comment_3_38e0b7bac8f2cfad492704ab6ab81e2b._comment
new file mode 100644
index 0000000000..307ff40a76
--- /dev/null
+++ b/doc/todo/export/comment_3_38e0b7bac8f2cfad492704ab6ab81e2b._comment
@@ -0,0 +1,42 @@
+[[!comment format=mdwn
+ username="joey"
+ subject="""comment 3"""
+ date="2017-05-24T18:59:30Z"
+ content="""
+`storeKey` could not be used for this, so it would need a new remote method
+to store a file on the remote under a specified name. Call it `storeFile`.
+
+What should `storeFile` with an encrypted special remote do? Encrypting
+the data does not seem very useful, especially for hybrid and shared
+where the secret key is embedded in the git repo. Not encrypting the data
+is surely a least surprise violation that would be a security hole.
+So probably best to not support exporting to encrypted special remotes.
+
+`git annex export --to specialremote` cannot handle incremental updates,
+resuming uploads etc, because special remotes can be so limited they only
+support putting the whole content of a file. Even resuming interrupted
+uploads is problimatic, because we don't know for sure what key was
+(partially or completely) exported before. The best that seems doable
+is to make `storeFile` avoid resending the file if the remote has the file
+on it already, and move the file into place atomically once it's all sent. 
+
+Then `git annex export --to remote` could be run repeatedly to export files
+until everything gets exported.
+
+But, when a file has been modified, that would prevent the modified version
+being exported. Unless state is stored somewhere to say that the given file
+on a remote is the content of a given key.
+That state could take the form of a .git-annex/filename.exported stored
+on the remote, which contains the name of the key. When exporting a new
+key over an existing file, that would be overwritten. (Really needs to be
+done atomically..)
+
+What about deleted files? Should export somehow notice that a file
+has been deleted locally, and remove it from the remote? How?
+
+----
+
+Alternatively, leave the incremental updating, deletion etc to
+third-party tools, and have `git annex export` simply export the current
+files to a directory.
+"""]]
diff --git a/doc/todo/export/comment_4_da8b5b8c9fc371fe138590744b2adce3._comment b/doc/todo/export/comment_4_da8b5b8c9fc371fe138590744b2adce3._comment
new file mode 100644
index 0000000000..1d694e0e25
--- /dev/null
+++ b/doc/todo/export/comment_4_da8b5b8c9fc371fe138590744b2adce3._comment
@@ -0,0 +1,51 @@
+[[!comment format=mdwn
+ username="joey"
+ subject="""comment 4"""
+ date="2017-07-10T17:30:23Z"
+ content="""
+Yoh suggested storing a treeish associated with the export to a special
+remote. Pointed out that the diff from that treeish to the next one could
+be used to update the export.
+
+(That does seem very close to [[todo/dumb, unsafe, human-readable_backend]].)
+
+Would need to somehow deal with partial uploads. There are two ways
+an upload can be partial:
+
+* Some of the files in the treeish have been uploaded; others have not.
+* A file has been partially uploaded.
+
+These two cases need to be disentangled somehow in order to handle
+them. It could use the location log, so once a key gets uploaded
+to the special remote, its content is marked present. However, using the
+location log does not seem sufficient to handle all cases (eg two files
+swapping names between two treeishes, where one of the files has been
+uploaded only partially to the special remote).
+
+It seems promosing to keep track of two separate treeishes:
+
+1. The treeish that is the current goal to have exported to the special
+   remote.
+2. The treeish for the current state of the special remote. Note that
+   after even an interrupted transfer, this treeish needs to be updated to
+   contain the current state of the special remote, which would mean
+   constructing a new treeish. (Perhaps a per-remote index file could be
+   used.)
+
+Having these two treeishes allows:
+
+* Detecting renames of exported files, which some special remotes can do
+  more efficiently.
+* Determining the key that a given file on the special remote is
+  storing the content of.
+* Resuming an interrupted export, without re-uploading all the files.
+* Detecting a partially uploaded file, because the current state treeish
+  for the remote should not contain that file.
+* Knowing what key was in the process of being sent to a partially uploaded
+  file, and so resuming that upload. Look at the goal treeish and see what
+  key it has for the file; as long as the goal treeish is the same goal
+  that was used for the interrupted export, that's the key. (This needs a
+  way to track if the goal has changed.)
+* Optionally, making `git annex sync` and the assistant upload as needed
+  to satisfy goal treeishes for special remotes.
+"""]]
diff --git a/doc/todo/export/comment_5_e5ac435b73818d9002f5ada84062e933._comment b/doc/todo/export/comment_5_e5ac435b73818d9002f5ada84062e933._comment
new file mode 100644
index 0000000000..037754881e
--- /dev/null
+++ b/doc/todo/export/comment_5_e5ac435b73818d9002f5ada84062e933._comment
@@ -0,0 +1,21 @@
+[[!comment format=mdwn
+ username="joey"
+ subject="""comment 5"""
+ date="2017-07-10T18:15:02Z"
+ content="""
+Using trees like that would not work well in a distributed setting
+where two repositories could be storing content on the same special remote.
+
+The per-remote trees could be stored, by eg grafting them into the
+git-annex branches's tree under the uuid of the special remote.
+
+But, there could then be merge conflicts, when different trees have been
+exported to the same special remote concurrently. And there's no way to
+resolve such a merge: If repo A uploaded F containing K and B uploaded F
+containing L, we don't know which file the special remote ended up with.
+If that happened it would have to delete and re-export from scratch.
+
+I think it's fine for exporting to only be able to be done from one
+repository. But, if a user tries to do the above, it needs to fail in some
+reasonable way, not leave a mess.
+"""]]
diff --git a/doc/todo/export/comment_6_88529583c38bc0c13dbe9a097e97b744._comment b/doc/todo/export/comment_6_88529583c38bc0c13dbe9a097e97b744._comment
new file mode 100644
index 0000000000..376e5ad900
--- /dev/null
+++ b/doc/todo/export/comment_6_88529583c38bc0c13dbe9a097e97b744._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="joey"
+ subject="""comment 6"""
+ date="2017-07-11T15:32:07Z"
+ content="""
+I've started a more detailed/coherent design document at
+[[design/exporting_trees_to_special_remotes]].
+"""]]
diff --git a/doc/todo/export/comment_7_8f1c20d78e3d8e00757a8819045fdcc8._comment b/doc/todo/export/comment_7_8f1c20d78e3d8e00757a8819045fdcc8._comment
new file mode 100644
index 0000000000..3b32fa7b0c
--- /dev/null
+++ b/doc/todo/export/comment_7_8f1c20d78e3d8e00757a8819045fdcc8._comment
@@ -0,0 +1,11 @@
+[[!comment format=mdwn
+ username="roger.herikstad@ca3b99b0263344ccfd8ec134a12261be25ef3504"
+ nickname="roger.herikstad"
+ avatar="http://cdn.libravatar.org/avatar/ad4ab4eded6bcd5d60bacc446419a4fd"
+ subject="Not working with rsync?"
+ date="2018-02-28T02:41:38Z"
+ content="""
+I was hoping to use export as a way of exporting my repository to a server where I'd like non-git users to be able to directly access the files. 
+The server is a Synology box, and I would like users to be able to see and download files through File Station. I have successfully set up git-annex on this box, but when I tried navigating to the path of the repository in File Station, it seems that File Station was not able to follow the symbolic links to download file contents. I then thought of using export in combination with an rsync special remote, but git annex returns an error saying that exporttree=yes is not support for this kind of remote. Is this something that could be fixed in future versions, or is there something about rsync special remote that prevents export from working in principle?
+
+"""]]
diff --git a/doc/todo/export/comment_8_3b93389a1f50aad0f759d361f06200e9._comment b/doc/todo/export/comment_8_3b93389a1f50aad0f759d361f06200e9._comment
new file mode 100644
index 0000000000..c68f7161ab
--- /dev/null
+++ b/doc/todo/export/comment_8_3b93389a1f50aad0f759d361f06200e9._comment
@@ -0,0 +1,9 @@
+[[!comment format=mdwn
+ username="joey"
+ subject="""comment 8"""
+ date="2018-02-28T15:54:06Z"
+ content="""
+Remotes need to have a nontrivial amount of code added to them in order to
+support export. That had not been done for rsync yet. I've implemented it
+now.
+"""]]
diff --git a/doc/todo/export_paired_rename_innefficenctcy.mdwn b/doc/todo/export_paired_rename_innefficenctcy.mdwn
new file mode 100644
index 0000000000..9284b74d9a
--- /dev/null
+++ b/doc/todo/export_paired_rename_innefficenctcy.mdwn
@@ -0,0 +1,10 @@
+`git annex export` can efficiently handle renames, including renames that swap
+content between files. However, when there are two pairs of duplicate files,
+and the filenames are swapped around, the current rename handling renames both
+dups to a single  temp file, and so the other file in the pair gets re-uploaded
+unncessarily. This could be improved.
+
+Perhaps: Find pairs of renames that swap content between two files.
+Run each pair in turn. Then run the current rename code. Although this
+still probably misses cases, where eg, content cycles amoung 3 files, and
+the same content amoung 3 other files. Is there a general algorythm?
diff --git a/doc/todo/export_preferred_content.mdwn b/doc/todo/export_preferred_content.mdwn
new file mode 100644
index 0000000000..4919986697
--- /dev/null
+++ b/doc/todo/export_preferred_content.mdwn
@@ -0,0 +1,14 @@
+`git annex export` normally exports all files in the specified tree,
+which is generally what the user wants.
+But, in some situations, the user may want to export a subset of files,
+in a way that can be well expressed by a preferred content expression.
+
+For example, they may want to export .mp3 files but not the .wav
+files used to produce those.
+
+Or, export podcasts, but not ones in a "old" directory that have already
+been listened to.
+
+It seems doable to make `git annex export` honor whatever
+preferred content settings have been configured for the remote.
+(And `git annex sync --content` too.)
diff --git a/doc/todo/feature_request__58___pubkey-only_encryption_mode.mdwn b/doc/todo/feature_request__58___pubkey-only_encryption_mode.mdwn
new file mode 100644
index 0000000000..5a3c108856
--- /dev/null
+++ b/doc/todo/feature_request__58___pubkey-only_encryption_mode.mdwn
@@ -0,0 +1,16 @@
+### Feature request
+
+It is not possible to put encrypted content in place on remotes with just a
+public GPG key. You always need the private key, even for encryption. I
+guess this is because how the cipher HMAC is used for replacing file names
+with their hashes. However, if that requirement (having secret file names)
+was dropped, I assume a pubkey-only mode could be implemented?
+
+My specific use case is backup archiving. I have my backups packed in
+archive files and want to use git-annex to copy the archives to offsite
+remotes (S3). In that case, I don't care much about hiding file names, but
+would appreciate the increased security of not having the secret key on the
+backup server. It would only be needed if I wanted to verify or restore
+backups.
+
+> Added "encryption=sharedpubkey" [[done]] --[[Joey]]
diff --git a/doc/todo/feature_request__58___pubkey-only_encryption_mode/comment_1_684d36c06429306be68fd60019564db3._comment b/doc/todo/feature_request__58___pubkey-only_encryption_mode/comment_1_684d36c06429306be68fd60019564db3._comment
new file mode 100644
index 0000000000..0e2f5e3ba6
--- /dev/null
+++ b/doc/todo/feature_request__58___pubkey-only_encryption_mode/comment_1_684d36c06429306be68fd60019564db3._comment
@@ -0,0 +1,23 @@
+[[!comment format=mdwn
+ username="joey"
+ subject="""comment 1"""
+ date="2015-03-31T19:37:20Z"
+ content="""
+When you use encryption=pubkey, the symmetric key that is used for
+HMAC encryption of filenames is encrypted using your gpg private key.
+The contents of files are also encrypted using your gpg private key
+(not using the symmetric key; that mode is encryption=hybrid).
+
+So, with encryption=pubkey, all that can be done with that symmetric key is
+to HMAC encrypt filenames and try to find results that match the HMACed
+filenames used on the remote. So, if you don't care about filenames
+leaking, you could publish that symmetric key with no bad effects. Its
+security is not important to you based on what you've said.
+
+But again, that symmetric key is encrypted with your gpg private key.
+The only way to decrypt it would be to break your gpg key somehow. In which
+case you have big problems. But not ones caused by the existence of the
+symmetric key.
+
+So, I see no benefit to the suggested mode.
+"""]]
diff --git a/doc/todo/feature_request__58___pubkey-only_encryption_mode/comment_2_13995d4f1142a393ff977859b86497b4._comment b/doc/todo/feature_request__58___pubkey-only_encryption_mode/comment_2_13995d4f1142a393ff977859b86497b4._comment
new file mode 100644
index 0000000000..f75bb58b6c
--- /dev/null
+++ b/doc/todo/feature_request__58___pubkey-only_encryption_mode/comment_2_13995d4f1142a393ff977859b86497b4._comment
@@ -0,0 +1,24 @@
+[[!comment format=mdwn
+ username="https://www.google.com/accounts/o8/id?id=AItOawl6rte43qSRK1o2zn7Ww4Z8pgBmJm8gDrc"
+ nickname="Rickard"
+ subject="comment 2"
+ date="2015-04-04T07:34:58Z"
+ content="""
+> The contents of files are also encrypted using your gpg private key
+
+I assume you meant to say gpg *public* key here?
+
+You're correct in that I can publish the symmetric HMAC key unencrypted with no bad effects for me. I've searched the documents but haven't found a way to tell git-annex to use a specific, unencrypted, symmetric key for HMAC, though. Is there a way?
+
+> So, I see no benefit to the suggested mode.
+
+I don't understand the reasoning that made you come to this conclusion.
+
+Let me restate my use case:
+
+With only the public part of a gpg key id available to a user, I would like that user to be able to add files to a git-annex repository. The user should then be able to copy the files encrypted to remotes that support encryption (S3 etc). The user should not be able to fetch or verify files from the encrypted remotes (since she lacks the private gpg key). The remote would be write-only for the user, basically. However, a friend of the user, possessing the private key (and having access to the remote), should be able to use the remote just like a normal git-annex remote.
+
+This is the normal way of using gpg for asymmetric encryption of files. I would find it useful to be able to use git-annex in a similar way. As far as I can understand, only the encrypted HMAC key is stopping me from using git-annex in this way. 
+
+However, there might be other things in git-annex' design that would make it difficult or even impossible to implement this functionality. It could also be the case that there's no benefit to adding this functionality to git-annex because there is some other (simpler) way to achieve the same thing. Both these cases are perfectly acceptable, but I would then be interested in knowing a bit more details.
+"""]]
diff --git a/doc/todo/feature_request__58___pubkey-only_encryption_mode/comment_3_a37d5d27d073020ccf7eadb8e47d378a._comment b/doc/todo/feature_request__58___pubkey-only_encryption_mode/comment_3_a37d5d27d073020ccf7eadb8e47d378a._comment
new file mode 100644
index 0000000000..1278aaab38
--- /dev/null
+++ b/doc/todo/feature_request__58___pubkey-only_encryption_mode/comment_3_a37d5d27d073020ccf7eadb8e47d378a._comment
@@ -0,0 +1,24 @@
+[[!comment format=mdwn
+ username="joey"
+ subject="""comment 3"""
+ date="2015-04-06T17:04:34Z"
+ content="""
+I somehow completely misread what you wanted! Thanks, it makes sense now.
+
+I anticipate there would be one problem with this mode; `git annex fsck --from remote` 
+would fail because it would be unable to decrypt the encrypted content
+when run on the client that is only able to encrypt to the public key, but
+lacks the necessary private key to decrypt.
+
+(So would `git annex move --to remote; git annex get --from remote`, but
+presumably that failure is the point of the mode..)
+
+It would be fairly easy to add this, I think. There is already support
+for configuring the MAC algorithm to use to encrypt filenames. Your mode seems
+to just need a "clear" mode that doesn't encrypt filenames at all.
+
+It does add complication to crypto paths, and potential for user
+foot-shooting though.
+
+I'm going to move this feature request from bugs/ to todo/ 
+"""]]
diff --git a/doc/todo/feature_request__58___pubkey-only_encryption_mode/comment_4_2ccd5e75f175f09b08cee2290720fdea._comment b/doc/todo/feature_request__58___pubkey-only_encryption_mode/comment_4_2ccd5e75f175f09b08cee2290720fdea._comment
new file mode 100644
index 0000000000..558b037962
--- /dev/null
+++ b/doc/todo/feature_request__58___pubkey-only_encryption_mode/comment_4_2ccd5e75f175f09b08cee2290720fdea._comment
@@ -0,0 +1,21 @@
+[[!comment format=mdwn
+ username="joey"
+ subject="""comment 4"""
+ date="2016-05-10T17:59:03Z"
+ content="""
+Thinking about this some more, I think it makes sense that your friend who
+is doing the uploading is doing it from a clone of your repository.
+
+So, they could have access to the HMAC key, and could use it to encrypt
+filenames, rather than using the un-encrypted keys. filenames seems better,
+because there's no point in exposing the un-encrypted filenames to S3.
+
+So, the encryption setup on such a repository would be the un-encrypted
+HMAC key, and an indication of what gpg public key to encrypt file contents
+to.
+
+(Of course, you might choose to expose a sanitized form of your real
+repository for cloning, that's more or less empty. And could even expose
+it to the whole world if you want to let anyone use it for sending files
+to you. In this case the un-encrypted HMAC key would be a pretty open secret.)
+"""]]
diff --git a/doc/todo/free_space_checking_for_local_special_remotes.mdwn b/doc/todo/free_space_checking_for_local_special_remotes.mdwn
new file mode 100644
index 0000000000..0fd2eac595
--- /dev/null
+++ b/doc/todo/free_space_checking_for_local_special_remotes.mdwn
@@ -0,0 +1,4 @@
+Should be possible to configure an annex.diskreserve setting for local
+special remotes, such as a directory special remote or possibly a bup
+special remote. (Although bup's deltas will make storing some versions of
+files take less space than git-annex would have to assume it would take.)
diff --git a/doc/todo/free_space_checking_for_local_special_remotes/comment_1_47c254cec58cbbb3ea84c93ef8282f01._comment b/doc/todo/free_space_checking_for_local_special_remotes/comment_1_47c254cec58cbbb3ea84c93ef8282f01._comment
new file mode 100644
index 0000000000..00c4c56254
--- /dev/null
+++ b/doc/todo/free_space_checking_for_local_special_remotes/comment_1_47c254cec58cbbb3ea84c93ef8282f01._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="http://joeyh.name/"
+ ip="4.154.7.235"
+ subject="comment 1"
+ date="2013-07-11T16:19:26Z"
+ content="""
+Directory special remotes do already honor annex.diskreserve, it turns out. Let's repurpose this bug to be about adding a per-remote configuration, in case the main annex.diskreserve is not appropriate.
+"""]]
diff --git a/doc/todo/get_--batch.mdwn b/doc/todo/get_--batch.mdwn
new file mode 100644
index 0000000000..a23b36de0d
--- /dev/null
+++ b/doc/todo/get_--batch.mdwn
@@ -0,0 +1,7 @@
+It seems that it would be tremendously useful, see e.g. our [datalad install](https://github.com/datalad/datalad/issues/553)
+
+[[!meta author=yoh]]
+
+> [[done]] although the output while getting a file is not
+> machine-parseable. So, I made --json also work for get, but enabling
+> json output disables any progress display. --[[Joey]]
diff --git a/doc/todo/git-annex-repair_claims_success_then_failure.mdwn b/doc/todo/git-annex-repair_claims_success_then_failure.mdwn
new file mode 100644
index 0000000000..8e2c4c29cc
--- /dev/null
+++ b/doc/todo/git-annex-repair_claims_success_then_failure.mdwn
@@ -0,0 +1,50 @@
+Hello.
+
+What does this log mean?  It seems to tell" "success", then "openDirStream" fails, then "1 failed".  What failed?
+
+Context is in [todo/git annex repair: performance can be abysmal, huge improvements possible](https://git-annex.branchable.com/ikiwiki.cgi?do=goto&page=todo%2Fgit_annex_repair__58___performance_can_be_abysmal__44___huge_improvements_possible)
+
+    fatal: bad object refs/heads/git-annex
+    fatal: bad object refs/heads/git-annex
+    fatal: bad object refs/heads/git-annex
+    error: Could not read somehashA
+    fatal: Failed to traverse parents of commit somehashB
+    error: Could not read somehashA
+    fatal: Failed to traverse parents of commit somehashB
+    error: Could not read somehashA
+    fatal: Failed to traverse parents of commit somehashB
+    error: Could not read somehashA
+    fatal: Failed to traverse parents of commit somehashB
+    Deleted these local branches, which could not be recovered due to missing objects:
+    	refs/heads/master
+    	refs/heads/git-annex
+    You currently have refs/heads/master checked out. You may have staged changes in the index that can be committed to recover the lost state of this branch!
+    Successfully recovered repository!
+    Please carefully check that the changes mentioned above are ok..
+
+    git-annex: .git/annex/journal/: openDirStream: does not exist (No such file or directory)
+    failed
+    git-annex: repair: 1 failed
+
+The fact is: this repo is a plain git clone of a git annex repository.
+
+There is no `.git/annex` directory there before `git-annex-repair` is run.
+
+After it ran, there is a `.git/annex` directory with that content:
+
+    total 24
+    drwxrwxr-x 3 4096 Jul 22 15:41 .
+    drwxrwxr-x 9 4096 Jul 23 07:24 ..
+    -rw-rw-r-- 1   65 Jul 20 11:59 index
+    -rw-rw-r-- 1   41 Jul 20 11:59 index.lck
+    -rw-rw-r-- 1    0 Jul 22 15:41 journal.lck
+    -rw-rw-r-- 1  211 Jul 20 11:59 mergedrefs
+    drwxrwxr-x 2 4096 Jul 22 15:41 misctmp
+
+Perhaps git-annex-repair gets confused when recovering a repository that is a plain git clone of a git annex repository?
+
+I did that because annexed objects are 1.7TB big here, so I wanted a local copy of pure git part only to perform repair of the repo, and propagate things somehow the objects at a later stage.
+
+I'll keep the repo lying around for a few days, maybe weeks, if some experiment or further feedback is needed.
+
+Thank you for your attention.
diff --git a/doc/todo/git-annex_ignores_GIT__95__SSH.mdwn b/doc/todo/git-annex_ignores_GIT__95__SSH.mdwn
new file mode 100644
index 0000000000..9af05496a0
--- /dev/null
+++ b/doc/todo/git-annex_ignores_GIT__95__SSH.mdwn
@@ -0,0 +1,39 @@
+### Please describe the problem.
+
+git uses environment variable GIT_SSH to determine SSH client.
+
+I set it to plink.exe because I extensively use pageant infrastructure and do NOT want to have 2 systems lying around.
+
+Unfortunately git-annex seems to ignore that.
+
+Even worse, it results in unpredicted behavior because the git part works (e.g. clone) whereas annex/rsync does not resulting in half-ok repositories without meaningful error messages.
+
+It only becomes evident when ssh.exe in the git repository is deleted.
+
+### What steps will reproduce the problem?
+
+Set %GIT_SSH% and remove ssh.exe
+
+You will get
+
+     git-annex: ssh: createProcess: does not exist (No such file or directory)
+     failed
+     git-annex: drop: 1 failed
+
+### What version of git-annex are you using? On what operating system?
+
+Windows 8,
+
+     $ git annex version
+     git-annex version: 5.20140411-gda795e0
+     build flags: Assistant Webapp Webapp-secure Pairing Testsuite S3 WebDAV DNS Feeds Quvi TDFA CryptoHash
+     key/value backends: SHA256E SHA1E SHA512E SHA224E SHA384E SKEIN256E SKEIN512E SHA256 SHA1 SHA512 SHA224 SHA384 SKEIN256 SKEIN512 WORM URL
+     remote types: git gcrypt S3 bup directory rsync web webdav tahoe glacier hook external
+     local repository version: 5
+     supported repository version: 5
+     upgrade supported from repository versions: 2 3 4
+
+
+### Please provide any additional information below.
+
+
diff --git a/doc/todo/git-annex_ignores_GIT__95__SSH/comment_1_958dd21d7e981232f03b4516521ac226._comment b/doc/todo/git-annex_ignores_GIT__95__SSH/comment_1_958dd21d7e981232f03b4516521ac226._comment
new file mode 100644
index 0000000000..c13c3d0896
--- /dev/null
+++ b/doc/todo/git-annex_ignores_GIT__95__SSH/comment_1_958dd21d7e981232f03b4516521ac226._comment
@@ -0,0 +1,10 @@
+[[!comment format=mdwn
+ username="http://joeyh.name/"
+ ip="108.236.230.124"
+ subject="comment 1"
+ date="2014-05-16T16:42:41Z"
+ content="""
+This is good spotting of a git configuration that git-annex does not support.
+
+However, I doubt that even if I made it use `GIT_SSH`, it would be useful. git-annex uses several features that are probably unique to openssh, including connection caching. While you could disable annex.sshcaching and perhaps get a different ssh to work, it would be much slower.
+"""]]
diff --git a/doc/todo/git-annex_ignores_GIT__95__SSH/comment_2_319a7e8122e7bc25d9399ba463a16158._comment b/doc/todo/git-annex_ignores_GIT__95__SSH/comment_2_319a7e8122e7bc25d9399ba463a16158._comment
new file mode 100644
index 0000000000..1294196ba9
--- /dev/null
+++ b/doc/todo/git-annex_ignores_GIT__95__SSH/comment_2_319a7e8122e7bc25d9399ba463a16158._comment
@@ -0,0 +1,22 @@
+[[!comment format=mdwn
+ username="divB"
+ ip="171.67.172.81"
+ subject="comment 2"
+ date="2014-05-17T23:58:19Z"
+ content="""
+Hi Joey,
+
+Thanks for your answer. In my opinion, this would be an important requirement for various reasons:
+
+1.) It is very confusing and results in unpredictable errors. I spent days in finding out what caused all the weird stuff that happened. Even if it is not supported, an error message or at least warning should be issued.
+
+2.) At least in Windows, plink.exe is the quasi-standard SSH client. All SW I am aware of supports at least plink.exe as alternative to openssh (SVN, git, unison, ...). Even within cygwin I often use plink for X11 forwarding etc. If features like SSH caching do not work with that it's totally fine.
+
+3.) Even for a unix environment, it is critical to be able to use a wrapper (or at least to configure SSH parameters). In my opinion, this should and must work consistently (git, git-annex and rsync). For example what if I have a dedicated public key for a repository and to not want to use %HOME%\.ssh\id_rsa ?
+For unison, I use a wrapper my_ssh.cmd which wraps specialized parameters (in particular SSH key, port) with plink.exe to ssh.exe's interface. Similarly, I might be interested in disabling agent functionality and use GSSAPI etc. etc.
+
+A little bit OT now: I already wondered if and how inefficient git-annex is in this regard. For example, if I sync content, it seems that ssh opens a new connection for each file! (at least each file results in a signing request in my agent). This happens even if I use ssh.exe. Is there anything wrong?
+
+Thanks
+
+"""]]
diff --git a/doc/todo/git-annex_ignores_GIT__95__SSH/comment_3_cc1936f18721a912bb77903be6c4a45f._comment b/doc/todo/git-annex_ignores_GIT__95__SSH/comment_3_cc1936f18721a912bb77903be6c4a45f._comment
new file mode 100644
index 0000000000..ff50c0599c
--- /dev/null
+++ b/doc/todo/git-annex_ignores_GIT__95__SSH/comment_3_cc1936f18721a912bb77903be6c4a45f._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="https://www.google.com/accounts/o8/id?id=AItOawm5WyknJirJJridJjiPNgrlYxGG9xrZBvA"
+ nickname="Daniel"
+ subject="comment 3"
+ date="2014-06-06T10:49:27Z"
+ content="""
+Agreed. I'd rather lose connection caching than have to maintain both putty keys and openssh keys on my windows machines.
+"""]]
diff --git a/doc/todo/git-hook_to_sanity-check_git-annex_branch_pushes.mdwn b/doc/todo/git-hook_to_sanity-check_git-annex_branch_pushes.mdwn
new file mode 100644
index 0000000000..bee234e107
--- /dev/null
+++ b/doc/todo/git-hook_to_sanity-check_git-annex_branch_pushes.mdwn
@@ -0,0 +1,57 @@
+IA.BAK and another project both need a way to let untrusted clients push
+git-annex branch changes to a central server. It's desired to only
+let a client make non-malicious pushes; a malicious client could screw
+up a lot of info in the branch.
+
+I propose adding a git-annex command that can be used in a git pre-receive
+hook to do this. --[[Joey]]
+
+There are two levels of checking it seems such a command could do:
+
+1. Only allow certain files to be changed. For example, maybe clients are only
+   expected to change location tracking files, and the activity.log
+   file, but not others like trust.log.
+
+2. Only allow modifications of data about a specific UUID. The UUID
+   would be provided to the command (and could be determined based on a
+   per-client ssh key or etc).
+
+  The changes to the branch would be checked, so this needs centralized
+  knowledge about the format of each file on the branch. I think this
+  mostly exists already in Logs.hs.
+
+Of these the second seems more likely to be useful, but the first would
+be by far the easier to add. So, do both?
+
+This might be too limiting for some situations:
+
+* If someone has 2 clients, that are talking with one-another,
+  then a push would include changes involving the UUIDs of both clients.
+  The command could be given multiple UUIDs to allow, to allow
+  for these kinds of setups.
+
+* A client might add a special remote somewhere, but this would need
+  changes to remote.log, which the first level of checking would not allow.
+  And, it would add another UUID, which the second level of checking would
+  need to be configured to allow.
+
+Python implementation
+---------------------
+
+I started doing an implementation of this in Python here. The code is
+available in [pre-receive-gitannex-check.py](https://gitlab.com/anarcat/puppet-git-annex/raw/a7333fd26af19eb5ee662c261a498c868b8b67e3/files/pre-receive-gitannex-check.py) (permalink, see also
+the [latest version](https://gitlab.com/anarcat/puppet-git-annex/blob/master/files/pre-receive-gitannex-check.py)).
+
+I went through what seems to be a rather convoluted design with libgit
+because I wanted to have some proper unit tests and generating git
+commands by hand in a shell script is rather painful. Also, it
+currently adopts a "blocking" approach, ie. it blocks known problems,
+but maybe it should be based on an "allow" approach, that is: only
+allow certain things to go through. 
+
+So far it only forbids removals and changes to trust.log.  A bunch of
+stuff is still missing like parameters (to allow changing the list of
+protected files) and checking the log tracking info. Feedback
+welcome. --[[anarcat]]
+
+Similar: [[todo/append-only_mode]]
diff --git a/doc/todo/git_annex_info___60__remote__62___does_not_list_all_the_parameters_for_the_remote.mdwn b/doc/todo/git_annex_info___60__remote__62___does_not_list_all_the_parameters_for_the_remote.mdwn
new file mode 100644
index 0000000000..647b0054f3
--- /dev/null
+++ b/doc/todo/git_annex_info___60__remote__62___does_not_list_all_the_parameters_for_the_remote.mdwn
@@ -0,0 +1,33 @@
+[[!format sh """
+$> git annex info  gdrive2 --verbose    
+remote: gdrive2   
+description: [gdrive2]
+uuid: d7e13bf3-0c0e-44c9-a626-c7af6a628df7
+trust: semitrusted
+cost: 200.0
+type: external
+externaltype: rclone
+encryption: none
+chunking: 52.43 megabyteschunks
+remote annex keys: 3
+remote annex size: 112.51 megabytes
+(dev) 2 29865.....................................:Fri 21 Jul 2017 12:35:35 PM EDT:.
+hopa:/tmp/testds1
+$> git co git-annex
+Switched to branch 'git-annex'
+W: git-annex repositories not (yet) supported in the prompt                                                     
+(dev) 2 29866.....................................:Fri 21 Jul 2017 12:35:40 PM EDT:.
+hopa:/tmp/testds1
+$> cat remote.log 
+ace2983e-5e2b-4c6a-8251-5344392d563c chunk=50MiB encryption=none externaltype=rclone mac=HMACSHA512 name=gdrive1 prefix=git-annex/testds1 rclone_layout=lower target=google-drive1 type=external timestamp=1500494328.845312147s
+d7e13bf3-0c0e-44c9-a626-c7af6a628df7 chunk=50MiB encryption=none externaltype=rclone mac=HMACSHA512 name=gdrive2 prefix=git-annex/raiders2 rclone_layout=lower target=google-drive1 type=external timestamp=1500654564.923893997s
+
+"""]]
+
+needed to see what is the prefix -- which is stored in remote.log -- but not printed by 'git annex info' neither in --verbose nor --json mode
+
+[[!meta author=yoh]]
+
+> This got fixed already it turns out, GETINFO.
+> Of course this and other special remotes will need changes to use it,
+> but that's outside the scope of git-annex, so [[done]]. --[[Joey]]
diff --git a/doc/todo/git_annex_info___60__remote__62___does_not_list_all_the_parameters_for_the_remote/comment_1_875fdf1aacff3cc2c3732450f052d711._comment b/doc/todo/git_annex_info___60__remote__62___does_not_list_all_the_parameters_for_the_remote/comment_1_875fdf1aacff3cc2c3732450f052d711._comment
new file mode 100644
index 0000000000..a9ca1a645c
--- /dev/null
+++ b/doc/todo/git_annex_info___60__remote__62___does_not_list_all_the_parameters_for_the_remote/comment_1_875fdf1aacff3cc2c3732450f052d711._comment
@@ -0,0 +1,17 @@
+[[!comment format=mdwn
+ username="joey"
+ subject="""comment 1"""
+ date="2017-08-15T17:20:10Z"
+ content="""
+This is an external special remote, and the meaning of prefix= and other
+parameters used when initializing an external special remote is up to that
+special remote. 
+
+For all git-annex knows, that may be some kind of binary
+value, or access token that you don't want scrawled on the screen. It makes
+sense for `git-annex info` not to display this stuff since it does not know
+what it is.
+
+It might make sense to extend the external special remote protocol
+with a command to get additional info to display.
+"""]]
diff --git a/doc/todo/git_annex_info_to_include_information_about_repo_version__63__.mdwn b/doc/todo/git_annex_info_to_include_information_about_repo_version__63__.mdwn
new file mode 100644
index 0000000000..d983adcd06
--- /dev/null
+++ b/doc/todo/git_annex_info_to_include_information_about_repo_version__63__.mdwn
@@ -0,0 +1,5 @@
+I thought that 'git annex info' should be the best location to report current version (6 or before ATM?) of the repository, which seems to be stored only within .git/config.  so I see how it somewhat possibly doesn't fit 'git annex info' which reports primarily from the state of git-annex branch, but since even "available local disk space" is in there, version better be reported there as well imho
+
+[[!meta author=yoh]]
+
+> Closing because it seems I don't need to do anything. [[done]] --[[Joey]]
diff --git a/doc/todo/git_annex_info_to_include_information_about_repo_version__63__/comment_1_783c53fd99747bc65bd832f918c50a59._comment b/doc/todo/git_annex_info_to_include_information_about_repo_version__63__/comment_1_783c53fd99747bc65bd832f918c50a59._comment
new file mode 100644
index 0000000000..ae1514dad9
--- /dev/null
+++ b/doc/todo/git_annex_info_to_include_information_about_repo_version__63__/comment_1_783c53fd99747bc65bd832f918c50a59._comment
@@ -0,0 +1,10 @@
+[[!comment format=mdwn
+ username="joey"
+ subject="""comment 1"""
+ date="2016-07-06T17:11:42Z"
+ content="""
+Currently `git annex version` shows the repo version. I think this makes
+sense since it also shows repo versions that can be upgraded to, etc.
+
+It seems redundant to report it in two places.
+"""]]
diff --git a/doc/todo/git_annex_info_to_include_information_about_repo_version__63__/comment_2_60d3b088de83c86243b19c3d0d050080._comment b/doc/todo/git_annex_info_to_include_information_about_repo_version__63__/comment_2_60d3b088de83c86243b19c3d0d050080._comment
new file mode 100644
index 0000000000..4cf5c5f487
--- /dev/null
+++ b/doc/todo/git_annex_info_to_include_information_about_repo_version__63__/comment_2_60d3b088de83c86243b19c3d0d050080._comment
@@ -0,0 +1,7 @@
+[[!comment format=mdwn
+ username="https://me.yahoo.com/a/EbvxpTI_xP9Aod7Mg4cwGhgjrCrdM5s-#7c0f4"
+ subject="comment 2"
+ date="2016-07-06T19:27:24Z"
+ content="""
+yikes, I never thought that 'git annex version', which works outside of repos as well, shows any information about the repository itself.  IMHO any such information should be within 'git annex info', not 'git annex version'  (does 'git version' show any repo specific version information? ;) )  Moreover there is no --json for 'annex version' ;)
+"""]]
diff --git a/doc/todo/git_annex_info_to_include_information_about_repo_version__63__/comment_3_d6e84551319439ba50bc10edbd25335c._comment b/doc/todo/git_annex_info_to_include_information_about_repo_version__63__/comment_3_d6e84551319439ba50bc10edbd25335c._comment
new file mode 100644
index 0000000000..209f9bfc7c
--- /dev/null
+++ b/doc/todo/git_annex_info_to_include_information_about_repo_version__63__/comment_3_d6e84551319439ba50bc10edbd25335c._comment
@@ -0,0 +1,11 @@
+[[!comment format=mdwn
+ username="joey"
+ subject="""comment 3"""
+ date="2016-07-11T16:52:33Z"
+ content="""
+git doesn't provide any command that outputs the repo version AFAIK, so
+it's not a useful precedent.
+
+Good point about --json. But, you can get the annex repo version 
+in machine parsable format using `git config annex.version`
+"""]]
diff --git a/doc/todo/git_annex_info_to_include_information_about_repo_version__63__/comment_4_5b3e45c746e6cfa7f511790bbd9f6c9f._comment b/doc/todo/git_annex_info_to_include_information_about_repo_version__63__/comment_4_5b3e45c746e6cfa7f511790bbd9f6c9f._comment
new file mode 100644
index 0000000000..3d5860fd0c
--- /dev/null
+++ b/doc/todo/git_annex_info_to_include_information_about_repo_version__63__/comment_4_5b3e45c746e6cfa7f511790bbd9f6c9f._comment
@@ -0,0 +1,10 @@
+[[!comment format=mdwn
+ username="https://me.yahoo.com/a/EbvxpTI_xP9Aod7Mg4cwGhgjrCrdM5s-#7c0f4"
+ subject="comment 4"
+ date="2016-07-11T17:25:28Z"
+ content="""
+> git doesn't provide any command that outputs the repo version AFAIK, so it's not a useful precedent.
+
+well -- exactly that -- `git version` DOES NOT output repos (its index or whatnot) version HERE. It only provides information about git's version
+Either there is some git command which outputs repo (index, ...) version is somewhat independent.  But ok -- I don't care much enough, can take version from git config I guess indeed, even though 'annex info' seems to be the most logical location for that information to be provided from as well IMHO ;-)
+"""]]
diff --git a/doc/todo/git_annex_open.mdwn b/doc/todo/git_annex_open.mdwn
new file mode 100644
index 0000000000..8edbfbd373
--- /dev/null
+++ b/doc/todo/git_annex_open.mdwn
@@ -0,0 +1,11 @@
+I had an idea the other night that there could be a `git annex open` command. What this would do would be the following:
+
+* if the file is already present locally: just `xdg-open` it
+* if it is not present and cannot be streamed: `git annex get "$@" && xdg-open "$@"`
+* if it can be streamed: `git annex get "$@"` and `xdg-open` when enough content has been streamed that we are confident it will play completely (unless network conditions change)
+
+This would need some [[metadata]] support partly to guess if the file can be streamed, but also to find the content. It would also assume some more intelligence in `git annex get` where git annex would have progress information (maybe through [[chunking]]?).
+
+How does that idea sound? --[[anarcat]]
+
+I think this idea can be considered obsoleted by the [[git-annex-inprogress]] command. --[[anarcat]] [[done]]
diff --git a/doc/todo/git_annex_open/comment_1_67d90a1cb104d98e816354d96e1b0306._comment b/doc/todo/git_annex_open/comment_1_67d90a1cb104d98e816354d96e1b0306._comment
new file mode 100644
index 0000000000..e1b442d174
--- /dev/null
+++ b/doc/todo/git_annex_open/comment_1_67d90a1cb104d98e816354d96e1b0306._comment
@@ -0,0 +1,21 @@
+[[!comment format=mdwn
+ username="joey"
+ subject="""comment 1"""
+ date="2014-11-10T17:11:52Z"
+ content="""
+Hmm, well, there have been requests for access to files as they're being
+get-ed before. This does have the advantage of providing that,
+but I don't know if I like trying in xdg-open, which might not always
+be the way a user wants to open a file.
+
+git-annex does have progress info available while a file is being
+transferred. (If nothing else, it tends to know the key size, and can
+see the file size.) However I don't know how it could tell if "enough"
+of the file was available to stream it.
+
+I guess what I'd be most comfortable with is putting in plumbing.
+Like making `examinekey` be able to report the temporary file
+that is used when a key is being downloaded. It might also make sense to
+have a bit of plumbing that waits for a file being downloaded to get
+to X% complete, or something.
+"""]]
diff --git a/doc/todo/git_annex_repair__58___performance_can_be_abysmal__44___huge_improvements_possible.mdwn b/doc/todo/git_annex_repair__58___performance_can_be_abysmal__44___huge_improvements_possible.mdwn
new file mode 100644
index 0000000000..407e099719
--- /dev/null
+++ b/doc/todo/git_annex_repair__58___performance_can_be_abysmal__44___huge_improvements_possible.mdwn
@@ -0,0 +1,28 @@
+# Situation
+
+Since yesterday evening (18 hours ago), `git annex repair --verbose` is repairing a repository from a remote.  This is on a fast machine (i7 4 physical cores, 8 logical CPUs @2.6Ghz sitting idle, 16GB RAM mostly unused, hard drive with measured 111MB/s sustained capacity). `.git` folder to repair grew to 8Gb while remote was only about 640MB.
+
+# What `git annex repair` does
+
+Currently, `git annex repair` appears to :
+
+* make a complete local clone from the remotes it finds, 
+* expand all packs into individual objects files,
+* then pour (with rsync) all those objects into the repository
+* and I guess it ends with a git fsck/gc/whatever to glue things back.
+
+The expected result (a complete repaired repo) is great but it didn't work without help and the performance is disappointing.
+
+# Suggested room for improvements
+
+I would be willing to contribute some patches and although I have a respectable experience in programming, including some functional traits, I'm not savvy enough in Haskell at the time.
+
+(1) a complete clone in this case means *between one and two hours* of download and easily gets interrupted losing all eforts (just like a plain git clone). Actually I tried several times and it never completed.  I worked around this by doing a `git clone` on the server and `rsync`ing that to a local storage and adding that locally as a git remote.
+
+(2) Even when a local "git remote" is available, `git annex repair` first tried the network one instead. Perhaps it would be better if it sorted git remotes and first try the ones that appear to be available locally (no URL or file:// URL scheme)? Workaround: manually break the transfer to that `git repair` switches to the next remote.
+
+(3) does `git annex repair` really have to explode the repository into individual objects? In my case it took about one hour to create 1454978 (one million four hundred thousands) object files for a total of 6.8GB. (I should have put `TEMP` and `TMPDIR` to point to a SSD-based storage as a workaround or even dare a tmpfs.) `git-annex` then runs a `rsync` that has been going on thrashing the disk (I can hear and feel it) for 7 hours and a half, with an expected total time of 8 hours 20 minutes. That's a very inefficient way to copy 6.8GB (incidentally, rsync does it in alphabetical order of hash as shown by `strace` and confirmed by man page and [here](https://bugs.debian.org/cgi-bin/bugreport.cgi?bug=640492) and [there](https://bugs.debian.org/cgi-bin/bugreport.cgi?bug=160982)). There must be a more efficient way, right? 
+
+A a sidenote, I don't know how a repo containing about 300k files jumped to 1400k git objects within the last 2 months.
+
+Any feedback welcome, thanks.
diff --git a/doc/todo/git_annex_repair__58___performance_can_be_abysmal__44___huge_improvements_possible/comment_1_4828098be3e7cd6109bd1934bf029f1f._comment b/doc/todo/git_annex_repair__58___performance_can_be_abysmal__44___huge_improvements_possible/comment_1_4828098be3e7cd6109bd1934bf029f1f._comment
new file mode 100644
index 0000000000..0765bcedc9
--- /dev/null
+++ b/doc/todo/git_annex_repair__58___performance_can_be_abysmal__44___huge_improvements_possible/comment_1_4828098be3e7cd6109bd1934bf029f1f._comment
@@ -0,0 +1,90 @@
+[[!comment format=mdwn
+ username="https://launchpad.net/~stephane-gourichon-lpad"
+ nickname="stephane-gourichon-lpad"
+ avatar="http://cdn.libravatar.org/avatar/02d4a0af59175f9123720b4481d55a769ba954e20f6dd9b2792217d9fa0c6089"
+ subject="Experiment to run git-annex-repair as fast as possible."
+ date="2017-07-28T04:27:06Z"
+ content="""
+# Introduction
+
+There has been some progress somehow. I managed to have `git-annex-repair` somehow succeed while complaining (see details in [git-annex-repair claims success then failure](https://git-annex.branchable.com/todo/git-annex-repair_claims_success_then_failure/)).
+
+I've had a look at:
+
+* [repair stuck on ls-tree command](http://git-annex.branchable.com/forum/repair_stuck_on_ls-tree_command/)
+* [git-annex branch shows commit with looong commitlog](http://git-annex.branchable.com/bugs/git-annex_branch_shows_commit_with_looong_commitlog/)
+* [git-annex wants to repair because of duplicateEntries in git fsck](https://git-annex.branchable.com/bugs/git-annex_wants_to_repair_because_of_duplicateEntries_in_git_fsck/)
+
+Perhaps I'll use `git-annex-forget` after I've rebuilt the repo.
+
+# Approach: how to speed up git-annex-repair with warm filesystem cache
+
+With a rather \"violent\" approach I could have it run in 78 minutes instead of thrashing for tens of hours.  The approach is:
+
+* copy `.git` folder (without `.git/annex/objects`, yet copied `.git/objects` takes about 9GB) to a SSD,
+* have a shell command line repeatedly make a `tar` of that to keep all that data warm in the filesystem cache (it repeatedly made a 4.3GB `tar` to `/dev/null` nearly always under one minute, even down to 7 seconds quite a few times)
+* use a tmpfs as temp directory `mkdir /dev/shm/tmp/ ; export TMPDIR=/dev/shm/tmp/ ; export TEMP=/dev/shm/tmp/`
+* then in that context `git-annex-repair --force`
+
+This approach is intended to ensure the fastest processing time for git-annex-repair by providing it fully warm filesystem cache.  Since no L1 L2 or L3 cache is gigabyte-sized, that means all this runs at RAM speed.
+
+# Performance result
+
+That machine capable of processing (make a tar of) the whole repo in 7 seconds ran the `git-annex-repair` process for 78 minutes (never more than one core busy at a time) and completed.  78 minutes is enough to make between 70 and 600 `tar`s of the full content that git-annex is supposed to repair.  IIRC the CPU was not active all the time.
+
+In other words, currently, repairing a repository looks as costly somehow as reading it many tens or hundreds of time.  Intuition says it probably doesn't have to be, even considering costs like computations and launching external processes.
+
+# Quick analysis
+
+I have seen with `strace` that `git-annex-repair` runs a number of `git show` commands.  Actually, most of the processing time appears to be in those `git-show` commands.
+
+# Hunch about what happens
+
+`git-show` (at least sometimes) first walks a lot of (all ?) `.git/object` content.
+
+`git-show` (at least sometimes) spit tens of megabytes of data, including **full-text patches of changed files**.
+
+Adding `--stat` to the git show command makes it much much quicker.
+
+# A few figures
+
+I took one example of git-show command and ran it separately.
+
+With `--stat`, cold cache: 41 seconds to produce 10MB.
+
+    $ git --literal-pathspecs show somehash --stat | pv | wc
+
+    10,2MiO 0:00:41 [ 251KiB/s]
+     134215  536851 10736844
+
+With `--stat`, warm cache: 2.5 seconds to produce 10MB.
+
+    $ time git --literal-pathspecs show somehash --stat | pv | wc
+
+    10,2MiO 0:00:02 [4,08MiB/s]
+     134215  536851 10736844
+
+    real	0m2.514s
+    user	0m1.492s
+    sys	0m1.044s
+
+Without `--stat`, 502 seconds to produce 57MB.
+
+    $ time git --literal-pathspecs show somehash | pv | wc
+
+     2243  time git --literal-pathspecs show somehash | pv | wc
+
+    57,4MiO 0:08:22 [ 116KiB/s]
+     939462 2818382 60230220
+
+    real	8m22.846s
+    user	0m57.716s
+    sys	7m25.912s
+
+# Summary
+
+* Does `git-annex-repair` need to ask git to reconstruct text-diff-style information from compressed data? Is that the source of the bad performance?
+* What does `git-annex-repair` need in `git-show` output?
+* Can `git-annex-repair` be made faster (much faster?) by tuning the way it calls git?
+
+"""]]
diff --git a/doc/todo/hidden_files.mdwn b/doc/todo/hidden_files.mdwn
new file mode 100644
index 0000000000..191e9c3286
--- /dev/null
+++ b/doc/todo/hidden_files.mdwn
@@ -0,0 +1,30 @@
+Add a `git annex hide $file` that behaves like drop, checking counter info
+and updating location log to say the current repo no longer has a file --
+but does not actually remove the content.
+
+Then `git annex unused` can be used to clean it up later. And in the
+meantime, it's still locally accessible. This can be useful if you're
+planning to need to free up space later, but want to hold onto the content
+for a while. Possibly you'll be disconnected later, so it's easier to push
+out that intent now.
+
+--
+
+TODO:
+
+* Make 100% sure this is safe. Drop, etc should never check content files
+  are present on other repos if the location log doesn't say the repo
+  has the content.
+
+* What will `git annex get` do if it's asked to get a file that has been
+  hidden?
+
+> Unless I am missing something: Make sure the data is correct (for SHA1 or other tracking) and restore locally. If that's not the case, delete and restore from remote. -- RichiH
+
+----
+
+Is 'unused' a good name? 'clean' and 'autoclean' would make more sense, imo. 'clean' deletes everything, whereas an optional 'autoclean' could try to be smart based on disk usage and/or SHA1, etc. -- RichiH
+
+> Nah, `git annex unused/dropunused` already exist.  --[[Joey]] 
+
+>> OK, in that case forget what I said. No idea about your internal policy, but feel free to delete this part of the page, then. -- RichiH
diff --git a/doc/todo/hide_missing_files.mdwn b/doc/todo/hide_missing_files.mdwn
new file mode 100644
index 0000000000..c11845a8f9
--- /dev/null
+++ b/doc/todo/hide_missing_files.mdwn
@@ -0,0 +1,27 @@
+I seem to recall a time when [[direct mode]] would hide files that were not present in the local repository. While this created a bunch of headaches for people that wanted to get the missing files, it was great for users that didn't care and only wanted to see what was actually there. (In fact, it was suggested as a solution in [[forum/usability:_what_are_those_arrow_things__63__/]], a year and a half ago.)
+
+Now this behavior has changed, and with v6 coming up, all those distinctions have basically gone away anyways...
+
+So what's someone to do if he wants to deal with the usability issue mentioned above? To restate: the problem is users I share files with often see a *lot* of files, with only a fraction of them being actually present. A lot of those files are obviously large, and so they are having a frustrating experience with git-annex because they see all those promises of "files being there" but they have a hard time actually finding which files *really* are there. So they click one one broken link after the other and generally give up before they figure out how to pop a terminal open, use `find -L -type l` or `find -type l` (i can never remember which!) - something we can hardly expect from the average GUI user...
+
+Maybe this could be accomplished through a [[dumb, unsafe, human-readable backend]]?
+
+Thanks for any advice! --[[anarcat]]
+
+> This has never been done by direct mode AFAICR.
+> 
+> But, [[design/adjusted_branches]] can do it. The basic functionality is
+> implemented already; what's missing is that, once in an adjusted branch 
+> with missing files hidden, when a file's content arrives the adjusted
+> branch needs to be updated to expose the file.
+> 
+> This will need hooks into
+> the adjusted branch code when file contents are get/dropped, and the
+> naive way to do it could make things quite slow, if the branch needs to
+> be re-adjusted after each get/drop of each file. --[[Joey]]
+
+> > I ended up creating a new special remote for this, documented in
+> > [[forum/syncing_music_to_my_android_device]]. It would be much 
+> > cleaner, of course, if adjusted branches could do this. --[[anarcat]]
+
+Other discussions: [[forum/How_to_hide_broken_symlinks]], where the idea of using views to hide missing files is introduced and [[forum/How_do_I_hide_files_not_present_in_the_local_annex__63__]], where it is said that placeholders shouldn't be used in direct mode... --[[anarcat]]
diff --git a/doc/todo/hide_missing_files/comment_1_9851558a528119279a901e97de285d39._comment b/doc/todo/hide_missing_files/comment_1_9851558a528119279a901e97de285d39._comment
new file mode 100644
index 0000000000..555fbefdeb
--- /dev/null
+++ b/doc/todo/hide_missing_files/comment_1_9851558a528119279a901e97de285d39._comment
@@ -0,0 +1,13 @@
+[[!comment format=mdwn
+ username="jason.dixon.email@aa0e536a2ec2877d6f666108dbbc6e39bbe87ac0"
+ nickname="jason.dixon.email"
+ avatar="http://cdn.libravatar.org/avatar/fbe9050fc83bbd536d307d87ea14d4bc"
+ subject="I hope the option to show missing files remains."
+ date="2017-03-07T06:30:37Z"
+ content="""
+I can certainly understand the issue when people are looking to git annex like a syncthing / dropbox / bittorrent sync alternative and expecting to only see files they have. But, at least for me personally, the feature of listing files I don't have, but *can get* is HUGE. I love being able to see what I have at a glance. Being able to rename and shuffle around files that are not present on my system. This to me is one of the biggest advantages of annex over the competition.
+
+This is both in context of personal file bookkeeping (ie photo library / music etc etc) and in projects (many maya files that are not currently being referenced, but at a glance can be seen).
+
+I suppose, a compromise for usability could be to add custom icons to missing files (yeah I know... cross platform!) such as other sync programs do for incomplete-syncing files, or gui's such as tortoiseGit/SVN do for tracked files.
+"""]]
diff --git a/doc/todo/immediate_stream-to-sync.mdwn b/doc/todo/immediate_stream-to-sync.mdwn
new file mode 100644
index 0000000000..919651df50
--- /dev/null
+++ b/doc/todo/immediate_stream-to-sync.mdwn
@@ -0,0 +1,9 @@
+git-annex is really great for secure distributed archival, but it's missing a piece of the reliability portion: when data is generated and saved a system may crash before finishing, and some data may be lost, before the add is initiated.
+
+If there were a way to add a file at the same time as writing it -- and then at the same time stream that live data to a remote -- it would increase the utility of git-add for such uses as storing live recording and logfiles.
+
+I imagine e.g.
+
+$ cat /proc/kmsg | git annex addstreamsync livelogs/kernel-log-$(date)
+
+and kmsg would be streamed to enough remotes to meet numcopies, and hashed live .  If possible, if the system then crashed and even the entire source repo was lost, the remote would have some incomplete file and be able to complete the add properly at next sync.  Hashing the data at the same time as generating it would also speed adding any new file.
diff --git a/doc/todo/immediate_stream-to-sync/comment_1_457523845c5e63bae90ea5a84d57bf96._comment b/doc/todo/immediate_stream-to-sync/comment_1_457523845c5e63bae90ea5a84d57bf96._comment
new file mode 100644
index 0000000000..b208ded011
--- /dev/null
+++ b/doc/todo/immediate_stream-to-sync/comment_1_457523845c5e63bae90ea5a84d57bf96._comment
@@ -0,0 +1,7 @@
+[[!comment format=mdwn
+ username="xloem"
+ subject="comment 1"
+ date="2016-05-06T11:19:42Z"
+ content="""
+This could be even more powerfully implemented with a FUSE filesystem.  Then locking/unlocking concerns would go away too.
+"""]]
diff --git a/doc/todo/import_--reinject.mdwn b/doc/todo/import_--reinject.mdwn
new file mode 100644
index 0000000000..b9f15cb472
--- /dev/null
+++ b/doc/todo/import_--reinject.mdwn
@@ -0,0 +1,8 @@
+There's `git annex reinject  ` for re-adding one file's contents to the local annex. But what if I have a whole bunch of files, and want git-annex itself to decide whether & where it needs to reinject them? (And if the file doesn't need to be reinjected, it would remain in its original place.)
+
+None of the `git annex import` modes work properly in this case. By default, importing adds another, unnecessary copy of the imported file (which I have to `rm` after importing). The `--clean-duplicates` mode seems close, but it insists on verifying the content in other repositories rather than just reinjecting it locally. (Let's assume that the main reason I'm trying to reinject is that I cannot access other repos.)
+
+So I'm hoping for something like `git annex import --reinject ...`. Or are there other existing ways to achieve the same? I couldn't find any.
+
+> implemented `git annex reinject --known` [[done]] (and also `git annex
+> import --reinject-known` now) --[[Joey]] 
diff --git a/doc/todo/import_--reinject/comment_1_659f0d18de4f5ee94adaf85ca106e196._comment b/doc/todo/import_--reinject/comment_1_659f0d18de4f5ee94adaf85ca106e196._comment
new file mode 100644
index 0000000000..2aeb49cfa7
--- /dev/null
+++ b/doc/todo/import_--reinject/comment_1_659f0d18de4f5ee94adaf85ca106e196._comment
@@ -0,0 +1,9 @@
+[[!comment format=mdwn
+ username="CandyAngel"
+ subject="comment 1"
+ date="2016-02-23T12:24:48Z"
+ content="""
+git-annex verifies the content in other repositories when you use --clean-duplicates because if it did not, it could delete the only copy of a file you had, because it was deleting files it knew about, but didn't have.
+
+As for what you are attempting, [maybe something like this](https://git-annex.branchable.com/tips/recover_data_from_lost+found/)?
+"""]]
diff --git a/doc/todo/import_--reinject/comment_2_04074324f866420ebf0d39ddfae85ff7._comment b/doc/todo/import_--reinject/comment_2_04074324f866420ebf0d39ddfae85ff7._comment
new file mode 100644
index 0000000000..eed8434b57
--- /dev/null
+++ b/doc/todo/import_--reinject/comment_2_04074324f866420ebf0d39ddfae85ff7._comment
@@ -0,0 +1,22 @@
+[[!comment format=mdwn
+ username="grawity@2ea26be48562f66fcb9b66307da72b1e2e37453f"
+ nickname="grawity"
+ subject="comment 2"
+ date="2016-03-01T07:10:55Z"
+ content="""
+Thanks, but you missed my point entirely... I wasn't asking for a mode that would delete data without checking. I was asking for the complete opposite – a mode that would _inject an extra copy_ of the data without checking.
+
+Yeah, I guess I could `annex add` the files, then un-annex them, and _then_ `annex import --clean-duplicates`, but that's a somewhat long-winded approach, needing twice the space and twice the time.
+
+(...speaking of losing data, it seems that `git annex reinject` is perfectly happy to delete files if I accidentally give it the wrong target. I.e. after failing content verification, it still throws away the source.)
+
+---
+
+It doesn't have to be part of git-annex; I could _script_ this feature myself, though there aren't nearly enough plumbing commands either. (For example, a command to hash a file and give its key (like `git hash-object`), or a command to find all paths for a key.)
+
+Having an equivalent of `git hash-object -w` (inject an arbitrary object) would make it even easier, but I couldn't find anything like that either.
+
+---
+
+Anyway, let's cancel this todo, I'll find other ways.
+"""]]
diff --git a/doc/todo/import_--reinject/comment_3_25d650c160db9114f13c192d9fee0748._comment b/doc/todo/import_--reinject/comment_3_25d650c160db9114f13c192d9fee0748._comment
new file mode 100644
index 0000000000..79bf038d9e
--- /dev/null
+++ b/doc/todo/import_--reinject/comment_3_25d650c160db9114f13c192d9fee0748._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="joey"
+ subject="""comment 3"""
+ date="2016-04-20T17:20:10Z"
+ content="""
+Good point about reinject deleting files that don't verify. I've fixed that
+so it leaves them alone.
+"""]]
diff --git a/doc/todo/import_--reinject/comment_4_2ff1a97aea8c0a565a23d5f007e4490c._comment b/doc/todo/import_--reinject/comment_4_2ff1a97aea8c0a565a23d5f007e4490c._comment
new file mode 100644
index 0000000000..8c6004a00a
--- /dev/null
+++ b/doc/todo/import_--reinject/comment_4_2ff1a97aea8c0a565a23d5f007e4490c._comment
@@ -0,0 +1,31 @@
+[[!comment format=mdwn
+ username="joey"
+ subject="""comment 4"""
+ date="2016-04-20T17:22:06Z"
+ content="""
+I think this would fit better as an option to `git annex reinject` than
+it would to `git annex import`. The latter has too many options anyway.
+
+It would not be hard to add something like `git annex reinject
+--all-known-files`, which would check if a source file hashes to a known
+key and ingest its content into the annex if so, otherwise leaving the
+source file along.
+
+That would reinject files that had been added to the repo long ago, and
+then were deleted. I don't know if that would be considered suprising
+behavior, but it's hard to only ingest files that have a current link in
+the repo (because a. git-annex keeps no such index (mostly) and b. branches and c.
+tags). Even if it was surprising behavior to reinject old deleted files,
+I suppose `git annex unused` could be used to drop such old unused file
+contents after reinjecting them.
+
+There's also the problem of different backends; it seems such a thing would
+need to hash a file 5 different ways to make sure no hash of it is known.
+
+As to adding plumbing, I'm always open to ideas for more useful plumbing.
+
+* You can use `git annex find` to get eg, a list of keys of files in the
+  currently checked out branch. 
+* I've added a `git annex calckey` that can calculate the key that would be
+  used for a file. (eg, hashing it)
+"""]]
diff --git a/doc/todo/import_--reinject/comment_5_ab63877b47869c78a3f17465cd985bb1._comment b/doc/todo/import_--reinject/comment_5_ab63877b47869c78a3f17465cd985bb1._comment
new file mode 100644
index 0000000000..d80e6c14dd
--- /dev/null
+++ b/doc/todo/import_--reinject/comment_5_ab63877b47869c78a3f17465cd985bb1._comment
@@ -0,0 +1,12 @@
+[[!comment format=mdwn
+ username="grawity@2ea26be48562f66fcb9b66307da72b1e2e37453f"
+ nickname="grawity"
+ subject="comment 5"
+ date="2016-04-20T19:21:33Z"
+ content="""
+\"There's also the problem of different backends; it seems such a thing would need to hash a file 5 different ways to make sure no hash of it is known.\"
+
+Yeah, I guess you're right – and there might even be different 'hashes' in the same backend, e.g. SHA256E considers `Foo.ISO` different from `Foo.iso`...
+
+Actually I ended up doing this only twice, so manual `annex add everything` + duplicate cleaner wasn't really that bad in the end. And `annex calckey` and `annex find` with ${key} will be useful for the other scripts I have; thanks.
+"""]]
diff --git a/doc/todo/import_--reinject/comment_6_2254aee4729f8710076e08992c3ed746._comment b/doc/todo/import_--reinject/comment_6_2254aee4729f8710076e08992c3ed746._comment
new file mode 100644
index 0000000000..95d0f588ae
--- /dev/null
+++ b/doc/todo/import_--reinject/comment_6_2254aee4729f8710076e08992c3ed746._comment
@@ -0,0 +1,13 @@
+[[!comment format=mdwn
+ username="CandyAngel"
+ subject="comment 6"
+ date="2016-04-21T11:41:09Z"
+ content="""
+    There's also the problem of different backends; it seems such a thing would need to hash a file 5 different ways to make sure no hash of it is known.
+
+As it leaves non-matched files alone, the user could specify the backend to minimise this requirement.
+
+Perhaps a new command \"ingest\", with an option for the backend and require --force to try all known backends?
+
+This would be very useful for my file sorting project (when I restart it).
+"""]]
diff --git a/doc/todo/import_--reinject/comment_7_76eba3fba57f8db0e7fd3a5832650145._comment b/doc/todo/import_--reinject/comment_7_76eba3fba57f8db0e7fd3a5832650145._comment
new file mode 100644
index 0000000000..4ef9660110
--- /dev/null
+++ b/doc/todo/import_--reinject/comment_7_76eba3fba57f8db0e7fd3a5832650145._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="CandyAngel"
+ avatar="http://cdn.libravatar.org/avatar/15c0aade8bec5bf004f939dd73cf9ed8"
+ subject="comment 7"
+ date="2017-01-11T22:29:55Z"
+ content="""
+Can I avoid git-annex making a copy of the file when using *reinject --known*? I have files larger than the available free space..
+"""]]
diff --git a/doc/todo/import_--reinject/comment_8_cd6ef504e8779088ac50cfaf2c8d96e4._comment b/doc/todo/import_--reinject/comment_8_cd6ef504e8779088ac50cfaf2c8d96e4._comment
new file mode 100644
index 0000000000..84518a8697
--- /dev/null
+++ b/doc/todo/import_--reinject/comment_8_cd6ef504e8779088ac50cfaf2c8d96e4._comment
@@ -0,0 +1,14 @@
+[[!comment format=mdwn
+ username="joey"
+ subject="""comment 8"""
+ date="2017-02-07T21:09:58Z"
+ content="""
+@CandyAngel, reinject --known moves the file into the annex. The only time I can
+see that it would make a copy is when importing from some other drive
+than the drive the repository is on, and it would then delete the source file
+after copying.
+
+It does currently always check that enough disk space exists to copy
+the file, even though it's probably going to move it. Perhaps that's why
+you thought it copied it?
+"""]]
diff --git a/doc/todo/import_tree.mdwn b/doc/todo/import_tree.mdwn
new file mode 100644
index 0000000000..dfa816b9c3
--- /dev/null
+++ b/doc/todo/import_tree.mdwn
@@ -0,0 +1,200 @@
+When `git annex export treeish` is used to export to a remote, and the
+remote allows files to somehow be edited on it, then there ought to be a
+way to import the changes back from the remote into the git repository.
+
+The command could be `git annex import treeish` or something like that.
+
+It would ask the special remote to list changed/new files, and deleted
+files. Download the changed/new files and inject into the annex. 
+Generate a new treeish, with parent the treeish that was exported,
+that has the modifications in it.
+
+Updating the working copy is then done by merging the import treeish.
+This way, conflicts will be detected and handled as normal by git.
+
+----
+
+The remote interface could have a new method, to list the changed/new and
+deleted files. It will be up to remotes to implement that if they can
+support importing.
+
+One way for a remote to do it, assuming it has mtimes, is to export
+files to the remote with their mtime set to the date of the treeish
+being exported (when the treeish is a commit, which has dates, and not
+a raw tree). Then the remote can simply enumerate all files,
+with their mtimes, and look for files that have mtimes
+newer than the last exported treeish's date.
+
+> But: If files on the remote are being changed at around the time
+> of the export, they could have older mtimes than the exported treeish's
+> date, and so be missed.
+> 
+> Also, a rename that swaps two files would be missed if mtimes
+> are only compared to the treeish's date.
+
+A perhaps better way is for the remote to keep track of the mtime,
+size, etc of all exported files, and use that state to find changes.
+Where to store that data?
+
+The data could be stored in a file/files on the remote, or perhaps
+the remote has a way to store some arbitrary metadata about a file
+that could be used.
+
+It could be stored in git-annex branch per-remote state. However,
+that state is per-key, not per-file. The export database could be
+used to convert a ExportLocation to a Key, which could be used
+to access the per-remote state. Querying the database for each file
+in the export could be a bottleneck without the right interface.
+
+If only one repository will ever access the remote, it could be stored
+in eg a local database. But access from only one repository is a 
+hard invariant to guarantee.
+
+Would local storage pose a problem when multiple repositories import from
+the same remote? In that case, perhaps different trees would be imported,
+and merged into master. So the two repositories then have differing
+masters, which can be reconciled as usual. It would mean extra downloads
+of content from the remote, since each import would download its own copy.
+Perhaps this is acceptable?
+
+This feels like it's reimplementing the git index, on a per-remote basis.
+So perhaps this is not the right interface.
+
+----
+
+Alternate interface: The remote is responsible for collecting a list of
+files currently in it, along with some content identifier. That data is
+sent to git-annex. git-annex keep track of which content identifier(s) map
+to which keys, and uses the information to determine when a file on the
+remote has changed or is new.
+
+This way, each special remote doesn't have to reimplement the equivilant of
+the git index, or comparing lists of files, it only needs a way to list
+files, and a good content identifier.
+
+This also simplifies implementation in git-annex, because it does not
+even need to look for changed/new/deleted files compared with the
+old tree. Instead, it can simply build git tree objects as the file list
+comes in, looking up the key corresponding to each content identifier
+(or downloading the content from the remote and adding it to the annex
+when there's no corresponding key yet). It might be possible to avoid
+git-annex buffering much tree data in memory.
+
+----
+
+A good content identifier needs to:
+
+* Be stable, so when a file has not changed, the content identifier
+  remains the same.
+* Change when a file is modified.
+* Be reasonably unique, but not necessarily fully unique.  
+  For example, if the mtime of a file is used as the content identifier, then
+  a rename that swaps two files would be noticed, except for in the
+  unusual case where they have the same mtime. If a new file
+  is added with the same mtime as some other file in the tree though,
+  git-annex will see that the filename is new, and so can still import it,
+  even though it's seen that content identifier before. Of course, that might
+  result in unncessary downloads (eg of a renamed file), so a more unique
+  content identifer would be better.
+
+A (size, mtime, inode) tuple is as good a content identifier as git uses in
+its index. That or a hash of the content would be ideal. 
+
+Do remotes need to tell git-annex about the properties of content
+identifiers they use, or does git-annex assume a minimum bar, and pay the
+price with some unncessary transfers of renamed files etc?
+
+git-annex will need a way to get the content identifiers of files
+that it stores on the remote when exporting a tree to it, so it can later
+know if those files have changed.
+
+----
+
+## race conditions TODO
+
+There's a race here, since a file could be modified on the remote while
+it's being exported, and if the remote then uses the mtime of the modified
+file in the content identifier, the modification would never be noticed by
+imports. 
+
+To fix this race, we need an atomic move operation on the remote. Upload
+the file to a temp file, then get its content identifier, and then move it
+from the temp file to its final location.
+
+There's also a race where a file gets changed on the remote after an
+import tree, and an export then overwrites it with something else.
+
+One solution would be to only allow one of importtree or exporttree
+to a given remote. This reduces the use cases a lot though, and perhaps
+so far that the import tree feature is not worth building. The adb
+special remote needs both.
+
+Really fixing this race needs locking or an atomic operation. Locking seems
+unlikely to be a portable enough solution.
+
+An atomic move could at least narrow the race significantly, eg:
+
+1. upload new version of $file to $tmp1
+2. atomic move current $file to $tmp2
+3. Get content identifier of $tmp2, check if it's what was expected to
+   be. If not, $file was modified after the last import tree, and that
+   conflict has to be resolved. Otherwise, delete $tmp2
+4. atomic move $tmp1 to $file
+
+A remaining race is that, if the file is open for write at the same
+time it's renamed, the write might happen after the content identifer
+is checked, and then whatever is written to it will be lost. 
+
+But: Git worktree update has the same race condition. Verified with
+this perl oneliner, run in a worktree, followed by a git pull. The lines
+that it appended to the file got lost:
+
+	perl -e 'open (OUT, ">>foo") || die "$!"; while (<>) { print OUT $_ }'
+
+Since this is acceptable in git, I suppose we can accept it here too..
+
+Another remaining race is if the file gets recreated after it's moved out
+of the way. If the atomic move refuses to overwrite existing files, that race
+would be detected by it failing. renameat(2) with `RENAME_NOREPLACE` can do that, 
+but probably many special remote interfaces don't provide a way to do that.
+
+----
+
+Since exporttree remotes don't have content identifier information yet, it
+needs to be collected the first time import tree is used. (Or import
+everything, but that is probably too expensive). Any modifications made to
+exported files before the first import tree would not be noticed. Seems
+acceptible as long as this only affects exporttree remotes created before
+this feature was added.
+
+What if repo A is being used to import tree from R for a while, and the
+user gets used to editing files on R and importing them. Then they stop
+using A and switch to clone B. It would not have the content identifier
+information that A did (unless it's stored in git-annex branch rather than
+locally). It seems that in this case, B needs to re-download everything,
+since anything could have changed since the last time A imported.
+That seems too expensive!
+
+Would storing content identifiers in the git-annex branch be too expensive?
+
+----
+
+If multiple repos can access the remote at the same time, then there's a
+potential problem when one is exporting a new tree, and the other one is
+importing from the remote.
+
+> This can be reduced to the same problem as exports of two
+> different trees to the same remote, which is already handled with the
+> export log.
+> 
+> Once a tree has been imported from the remote, it's
+> in the same state as exporting that same tree to the remote, so
+> update the export log to say that the remote has that treeish exported
+> to it. A conflict between two export log entries will be handled as
+> usual, with the user being prompted to re-export the tree they want
+> to be on the remote. (May need to reword that prompt.)
+> --[[Joey]]
+
+----
+
+See also, [[adb_special_remote]]
diff --git a/doc/todo/importfeed__58___allow___36____123__itemdate__125___with_--template.mdwn b/doc/todo/importfeed__58___allow___36____123__itemdate__125___with_--template.mdwn
new file mode 100644
index 0000000000..46d9de34f4
--- /dev/null
+++ b/doc/todo/importfeed__58___allow___36____123__itemdate__125___with_--template.mdwn
@@ -0,0 +1,5 @@
+It would be great to be able to use the pubDate of the entries with the --template option of importfeed.
+
+Text.Feed.Query has a getItemPublishDate (and a getFeedPubDate, if we want some kind of ${feeddate}).
+
+The best would be to allow a reformating of the date(s) with (for example) %Y-%m-%D
diff --git a/doc/todo/importfeed__58___allow___36____123__itemdate__125___with_--template/comment_1_62752c760fc12eca0c34d67d58753d00._comment b/doc/todo/importfeed__58___allow___36____123__itemdate__125___with_--template/comment_1_62752c760fc12eca0c34d67d58753d00._comment
new file mode 100644
index 0000000000..cc3d85faf0
--- /dev/null
+++ b/doc/todo/importfeed__58___allow___36____123__itemdate__125___with_--template/comment_1_62752c760fc12eca0c34d67d58753d00._comment
@@ -0,0 +1,10 @@
+[[!comment format=mdwn
+ username="gueux"
+ ip="2a01:240:fe6d:0:7986:3659:a8bd:64f1"
+ subject="syntax"
+ date="2013-09-12T14:05:16Z"
+ content="""
+use \"itemdate\" and \"feeddate\" as names?
+
+use ${itemdate=%Y-%m-%D} syntax option?
+"""]]
diff --git a/doc/todo/importfeed__58___allow___36____123__itemdate__125___with_--template/comment_2_21672360060f48bc2eacfa535ff4c94d._comment b/doc/todo/importfeed__58___allow___36____123__itemdate__125___with_--template/comment_2_21672360060f48bc2eacfa535ff4c94d._comment
new file mode 100644
index 0000000000..c8770ec6ed
--- /dev/null
+++ b/doc/todo/importfeed__58___allow___36____123__itemdate__125___with_--template/comment_2_21672360060f48bc2eacfa535ff4c94d._comment
@@ -0,0 +1,11 @@
+[[!comment format=mdwn
+ username="http://joeyh.name/"
+ ip="4.154.2.134"
+ subject="comment 2"
+ date="2013-09-13T19:53:52Z"
+ content="""
+getItemPublishDate returns a String, which can contain any of several date formats. Deferred until the feed library has something more sane.
+Upstream bug: 
+
+As for how to format the date in the feed, I would be ok with having itemdate (YYYYMMDD), itemyear (YYYY), itemmonth (MM) and itemday (DD). Full date formatting seems like overkill here.
+"""]]
diff --git a/doc/todo/improve_memory_usage_of_--all.mdwn b/doc/todo/improve_memory_usage_of_--all.mdwn
new file mode 100644
index 0000000000..785d8aa2dc
--- /dev/null
+++ b/doc/todo/improve_memory_usage_of_--all.mdwn
@@ -0,0 +1,14 @@
+Using --all, or running in a bare repo, as well as
+`git annex unused` and `git annex info` all end up buffering the list of
+all keys that have uncommitted journalled changes in memory.
+This is due to Annex.Branch.files's call to getJournalledFilesStale which
+reads all the files in the directory into a buffer.
+
+Note that the list of keys in the branch *does* stream in, so this
+is only really a problem when using annex.alwayscommit=false to build
+up big git-annex branch commits via the journal.
+
+An attempt at making it stream via unsafeInterleaveIO failed miserably
+and that is not the right approach. This would be a good place to use
+ResourceT, but it might need some changes to the Annex monad to allow
+combining the two. --[[Joey]]
diff --git a/doc/todo/include_msg_with_possible_reason_why_command___40__e.g._add__41___failed_into_--json_output.mdwn b/doc/todo/include_msg_with_possible_reason_why_command___40__e.g._add__41___failed_into_--json_output.mdwn
new file mode 100644
index 0000000000..c634297d13
--- /dev/null
+++ b/doc/todo/include_msg_with_possible_reason_why_command___40__e.g._add__41___failed_into_--json_output.mdwn
@@ -0,0 +1,20 @@
+ATM errors are output into stderr while json record lacks any hints on what went wrong
+
+[[!format sh """
+$> git annex add --json longfilenametogetlotsoferrorsandevenlonger4 
+  longfilenametogetlotsoferrorsandevenlonger4: setFileMode: permission denied (Operation not permitted)
+{"command":"add","success":false,"file":"longfilenametogetlotsoferrorsandevenlonger4"}
+"""]]
+
+would be nice(r) to have
+
+[[!format sh """
+$> git annex add --json longfilenametogetlotsoferrorsandevenlonger4 
+{"command":"add","success":false,"file":"longfilenametogetlotsoferrorsandevenlonger4", "msg": "setFileMode: permission denied (Operation not permitted)"}
+"""]]
+or may be even an explicit "errormsg" or alike instead of just a generic "msg"
+
+
+[[!meta author=yoh]]
+
+> [[done]] --[[Joey]]
diff --git a/doc/todo/include_msg_with_possible_reason_why_command___40__e.g._add__41___failed_into_--json_output/comment_1_f02e43b68c219b2d7d65ea868fac125f._comment b/doc/todo/include_msg_with_possible_reason_why_command___40__e.g._add__41___failed_into_--json_output/comment_1_f02e43b68c219b2d7d65ea868fac125f._comment
new file mode 100644
index 0000000000..eeaafa9ed1
--- /dev/null
+++ b/doc/todo/include_msg_with_possible_reason_why_command___40__e.g._add__41___failed_into_--json_output/comment_1_f02e43b68c219b2d7d65ea868fac125f._comment
@@ -0,0 +1,23 @@
+[[!comment format=mdwn
+ username="joey"
+ subject="""comment 1"""
+ date="2018-02-08T16:59:16Z"
+ content="""
+Is the benefit of doing this being able to correlate the error message with
+the filename whose processing caused the error?
+
+For that benefit to be realized, the error message would have to be included
+in the same json object with the file being processed. If a separate json
+object were emitted containing only the error message it would not be clear
+what file it was related to (especially when git-annex is running
+concurrent jobs), and so that does not seem any better than
+output to stderr.
+
+Looks like implementation would just involve making outputError 
+check if there is a jsonBuffer, and add the error message to it,
+probably using an array since there could be multiple warning messages.
+
+I kind of feel like they should still go to stderr in addition to any json
+output, because any existing json consumers may rely on the current stderr
+behavior to let the user know what went wrong.
+"""]]
diff --git a/doc/todo/include_msg_with_possible_reason_why_command___40__e.g._add__41___failed_into_--json_output/comment_2_3a7043652c692796b1a438a2c4c26828._comment b/doc/todo/include_msg_with_possible_reason_why_command___40__e.g._add__41___failed_into_--json_output/comment_2_3a7043652c692796b1a438a2c4c26828._comment
new file mode 100644
index 0000000000..0dbd715bf7
--- /dev/null
+++ b/doc/todo/include_msg_with_possible_reason_why_command___40__e.g._add__41___failed_into_--json_output/comment_2_3a7043652c692796b1a438a2c4c26828._comment
@@ -0,0 +1,9 @@
+[[!comment format=mdwn
+ username="yarikoptic"
+ avatar="http://cdn.libravatar.org/avatar/f11e9c84cb18d26a1748c33b48c924b4"
+ subject="comment 2"
+ date="2018-02-08T18:06:54Z"
+ content="""
+yes -- I could have been clearer that association to the file (if present, may be error is generic and still would be worth to communicate via a simple json message, alike recently added INFO passed from custom special remotes) would be valuable!
+As for stderr output -- would lead to duplication (we just changed to output at least a \"tip\" of stderr output which might be heavy).  If it is a matter of not breaking tools which would not support this new behavior, having a config or --option-hate-to-suggest-to-addone, would be  valuable so we could disable \"double-casting\" the error messages
+"""]]
diff --git a/doc/todo/include_msg_with_possible_reason_why_command___40__e.g._add__41___failed_into_--json_output/comment_3_2d41d67dcb6e700b1ecda3106e444a5e._comment b/doc/todo/include_msg_with_possible_reason_why_command___40__e.g._add__41___failed_into_--json_output/comment_3_2d41d67dcb6e700b1ecda3106e444a5e._comment
new file mode 100644
index 0000000000..d3872b23e7
--- /dev/null
+++ b/doc/todo/include_msg_with_possible_reason_why_command___40__e.g._add__41___failed_into_--json_output/comment_3_2d41d67dcb6e700b1ecda3106e444a5e._comment
@@ -0,0 +1,15 @@
+[[!comment format=mdwn
+ username="joey"
+ subject="""comment 3"""
+ date="2018-02-16T16:50:27Z"
+ content="""
+A problem with this idea is that error messages are relatively uncommon,
+and if they're hidden away in an extra field of an existing json message,
+it would be easy for the consumer to forget to handle that uncommon case.
+
+Note that git-annex currently doesn't document the actual json structure it
+uses, which is more or less ok because any given git-annex subcommand will
+always output the same json structure (except some note fields that
+only sometimes appear, whose data is targeted at humans).
+It's self-documenting. Using json for errors breaks that pattern.
+"""]]
diff --git a/doc/todo/include_msg_with_possible_reason_why_command___40__e.g._add__41___failed_into_--json_output/comment_4_c2e5041a07787ae84f85b898b1aae6b7._comment b/doc/todo/include_msg_with_possible_reason_why_command___40__e.g._add__41___failed_into_--json_output/comment_4_c2e5041a07787ae84f85b898b1aae6b7._comment
new file mode 100644
index 0000000000..8d0d2faa5b
--- /dev/null
+++ b/doc/todo/include_msg_with_possible_reason_why_command___40__e.g._add__41___failed_into_--json_output/comment_4_c2e5041a07787ae84f85b898b1aae6b7._comment
@@ -0,0 +1,20 @@
+[[!comment format=mdwn
+ username="joey"
+ subject="""comment 4"""
+ date="2018-02-16T17:29:56Z"
+ content="""
+Currently when there's an exception in processing a file, the error is thrown
+and then caught and output to stderr, and this prevents the accumulated
+json buffer for the file from being output, since the processing never
+finishes. So, you get some error message on stderr, and no indication
+in the json what file it belongs to.
+
+So, perhaps an easier fix would be to emit the json buffer in this case
+after the error message. Then stderr output, whether error or warning,
+would always precede the json for the same file.
+
+The consumer would need to use select() over stdin+stderr to observe
+the order they were interleaved. May be too much to expect consumers
+to get that right. Oh, and still would not help untangle the stream
+when run with -J.
+"""]]
diff --git a/doc/todo/include_msg_with_possible_reason_why_command___40__e.g._add__41___failed_into_--json_output/comment_5_429e42b9c1989dade01300f08fc3c13c._comment b/doc/todo/include_msg_with_possible_reason_why_command___40__e.g._add__41___failed_into_--json_output/comment_5_429e42b9c1989dade01300f08fc3c13c._comment
new file mode 100644
index 0000000000..a1d9edc00a
--- /dev/null
+++ b/doc/todo/include_msg_with_possible_reason_why_command___40__e.g._add__41___failed_into_--json_output/comment_5_429e42b9c1989dade01300f08fc3c13c._comment
@@ -0,0 +1,12 @@
+[[!comment format=mdwn
+ username="joey"
+ subject="""comment 5"""
+ date="2018-02-16T17:59:43Z"
+ content="""
+I think I'm tending toward adding a new --json-error-messages option, and
+making it add "error-messages:[]" to the json when there are no errors,
+thus keeping the json self-documenting.
+
+The json buffer will need to be flushed in the exception handler,
+as noted in my previous comment.
+"""]]
diff --git a/doc/todo/include_msg_with_possible_reason_why_command___40__e.g._add__41___failed_into_--json_output/comment_6_3d5a33476a3fa773fda1e5f7a2d8c453._comment b/doc/todo/include_msg_with_possible_reason_why_command___40__e.g._add__41___failed_into_--json_output/comment_6_3d5a33476a3fa773fda1e5f7a2d8c453._comment
new file mode 100644
index 0000000000..f7d965f3b7
--- /dev/null
+++ b/doc/todo/include_msg_with_possible_reason_why_command___40__e.g._add__41___failed_into_--json_output/comment_6_3d5a33476a3fa773fda1e5f7a2d8c453._comment
@@ -0,0 +1,10 @@
+[[!comment format=mdwn
+ username="yarikoptic"
+ avatar="http://cdn.libravatar.org/avatar/f11e9c84cb18d26a1748c33b48c924b4"
+ subject="comment 6"
+ date="2018-02-16T18:42:28Z"
+ content="""
+FWIW `--json-error-messages` would be great to have. 
+
+`The consumer would need to use select() over stdin+stderr to observe the order they were interleaved.` would not work for us atm without major refactoring (if possible at all without significant pains)
+"""]]
diff --git a/doc/todo/include_msg_with_possible_reason_why_command___40__e.g._add__41___failed_into_--json_output/comment_7_246e35f32f77af3b2924577b1bf45001._comment b/doc/todo/include_msg_with_possible_reason_why_command___40__e.g._add__41___failed_into_--json_output/comment_7_246e35f32f77af3b2924577b1bf45001._comment
new file mode 100644
index 0000000000..7bbe0e00cc
--- /dev/null
+++ b/doc/todo/include_msg_with_possible_reason_why_command___40__e.g._add__41___failed_into_--json_output/comment_7_246e35f32f77af3b2924577b1bf45001._comment
@@ -0,0 +1,14 @@
+[[!comment format=mdwn
+ username="joey"
+ subject="""comment 7"""
+ date="2018-02-19T19:49:11Z"
+ content="""
+Basic --json-error-messages implemented after 4 hours work.
+
+Still needing to be done:
+
+* A few commands like `git annex info` have a custom json outputter,
+  and may not output the error-messages field by default, or may not make
+  sense to support the option.
+* Flush json buffer after fatal error.
+"""]]
diff --git a/doc/todo/include_msg_with_possible_reason_why_command___40__e.g._add__41___failed_into_--json_output/comment_8_2e0ec2bb51a2a06184b94710616f211c._comment b/doc/todo/include_msg_with_possible_reason_why_command___40__e.g._add__41___failed_into_--json_output/comment_8_2e0ec2bb51a2a06184b94710616f211c._comment
new file mode 100644
index 0000000000..b66f78fef1
--- /dev/null
+++ b/doc/todo/include_msg_with_possible_reason_why_command___40__e.g._add__41___failed_into_--json_output/comment_8_2e0ec2bb51a2a06184b94710616f211c._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="joey"
+ subject="""comment 8"""
+ date="2018-02-20T17:00:12Z"
+ content="""
+Testing, both of the cases in my previous message were actually already
+working as desired. I think it's done!
+"""]]
diff --git a/doc/todo/integrate_support_for_spideroak_as_archive__47__backup.mdwn b/doc/todo/integrate_support_for_spideroak_as_archive__47__backup.mdwn
new file mode 100644
index 0000000000..c71df466ca
--- /dev/null
+++ b/doc/todo/integrate_support_for_spideroak_as_archive__47__backup.mdwn
@@ -0,0 +1,9 @@
+SpiderOak is a great backup service and many of us have unlimited accounts with them since World Backup Day.  That makes SpiderOak a very interesting candidate for use as an archive or backup node.  I can think of only two ways to go about this:
+
+1. Designate one of your computers as an archive/backup and use spideroak independantly to sync that archive.  This is very unattractive, since it makes the spideroak backup completely unknown to git-annex.
+
+2. Integrate the SpiderOak CLI tool somehow as a remote.  I don't know to what extent this would be possible, but if it were, that'd be awesome.  And a lot of work, presumably.
+
+Bonus option:
+
+3. Can the SpiderOak API be useful? https://spideroak.com/faq/questions/37/how_do_i_use_the_spideroak_web_api/
diff --git a/doc/todo/integrate_support_for_spideroak_as_archive__47__backup/comment_1_a47ea814f6d7727bbd0eeca6d1fd3219._comment b/doc/todo/integrate_support_for_spideroak_as_archive__47__backup/comment_1_a47ea814f6d7727bbd0eeca6d1fd3219._comment
new file mode 100644
index 0000000000..7790a7f2d4
--- /dev/null
+++ b/doc/todo/integrate_support_for_spideroak_as_archive__47__backup/comment_1_a47ea814f6d7727bbd0eeca6d1fd3219._comment
@@ -0,0 +1,10 @@
+[[!comment format=mdwn
+ username="http://joeyh.name/"
+ ip="216.145.95.162"
+ subject="comment 1"
+ date="2014-05-19T15:05:49Z"
+ content="""
+If spideroak has a CLI tool that can get/put/delete individual files, it should be quite easy to use [[the_external_special_remote|special_remotes/external]] to support it. The demo shell script could be used as a starting place.
+
+I built that so that others can easily write special remotes, and so unless spideroak's CLI is free software, I don't anticipate working on this myself.
+"""]]
diff --git a/doc/todo/integrate_support_for_spideroak_as_archive__47__backup/comment_2_6b14f729446c2e52f07ce6c9ad2ab627._comment b/doc/todo/integrate_support_for_spideroak_as_archive__47__backup/comment_2_6b14f729446c2e52f07ce6c9ad2ab627._comment
new file mode 100644
index 0000000000..51efa49b4d
--- /dev/null
+++ b/doc/todo/integrate_support_for_spideroak_as_archive__47__backup/comment_2_6b14f729446c2e52f07ce6c9ad2ab627._comment
@@ -0,0 +1,15 @@
+[[!comment format=mdwn
+ username="https://id.koumbit.net/anarcat"
+ subject="not free software"
+ date="2015-06-04T14:37:33Z"
+ content="""
+According to [wikipedia](https://en.wikipedia.org/wiki/Spideroak):
+
+> *Some components of SpiderOak are open-source, and as early as 2009 the company announced their intent for the client to be fully open-source in the future.[4] As of 2015, SpiderOak's source code is not available.[5]*
+
+So that seems to be a dead end, since 2010 - total vaporware and a bunch of broken links. There is supposedly an API, but this is also a bunch of broken links. There's a copy of a very basic API [on archive.org](https://web.archive.org/web/20141019212745/https://spideroak.com/faq/questions/37/how_do_i_use_the_spideroak_web_api/) but it doesn't look like it answers the requirement here.
+
+There is a [Github page](https://github.com/SpiderOak/) with [some HTML5 client](https://github.com/SpiderOak/SpiderOakMobileClient) but it's unclear the status of this project...
+
+One has to wonder how or why Snowden gave his benediction to this project... --[[anarcat]]
+"""]]
diff --git a/doc/todo/interface_to_the___34__progress__34___of_annex_operations.mdwn b/doc/todo/interface_to_the___34__progress__34___of_annex_operations.mdwn
new file mode 100644
index 0000000000..68125c53ba
--- /dev/null
+++ b/doc/todo/interface_to_the___34__progress__34___of_annex_operations.mdwn
@@ -0,0 +1,19 @@
+It would be really nice if external tools working with annex could obtain updates on the progress of operations so they could report using their own UI back to the user.  Especially relevant for --batch --json modes of operations (e.g. for get or addurl) whenever no progress is reported by annex itself at all.  
+
+Related:
+[datalad #478](https://github.com/datalad/datalad/issues/478)
+
+
+[[!meta author=yoh]]
+
+> --status-fd is one way, or the progress could be included as
+> part of the --batch protocol. In either case, it might make sense to
+> reuse part of the external special remote protocol. (Which would let you
+> relay the progress messages when datalad is doing a nested retrieval, I
+> suppose.) --[[Joey]]
+
+>> [[done]]; --json-progress implemented. I limited the frequency of json
+>> progress items to 10 per second max, and it's typically only 1 per
+>> second or less, so didn't implement
+>> --json-progress=N to tune it. Also added --json and --json-progress to
+>> copy, move, mirror commands. --[[Joey]]
diff --git a/doc/todo/keep_annexed_files_for_a_while.mdwn b/doc/todo/keep_annexed_files_for_a_while.mdwn
new file mode 100644
index 0000000000..cf85b11f3b
--- /dev/null
+++ b/doc/todo/keep_annexed_files_for_a_while.mdwn
@@ -0,0 +1,8 @@
+I don't want files that I dropped to immediately disappear from my local or all of my remotes repos on the next sync. Especially in situations where changes to the git-annex repo get automatically and immediately replicated to remote repos, I want a configurable "grace" period before files in .git/annex/objects get really deleted.
+
+This has similarities to the "trash" on a desktop. It might also be nice to
+
+* configure a maximum amount of space of the "trash"
+* have a way to see the contents of the trash to easily recover deleted files
+
+Maybe it would make sense to just move dropped files to the desktops trash? "git annex trash" as an alternative to drop?
diff --git a/doc/todo/learn_about_remotes_that_are_currently_unavailable.mdwn b/doc/todo/learn_about_remotes_that_are_currently_unavailable.mdwn
new file mode 100644
index 0000000000..d37aedee66
--- /dev/null
+++ b/doc/todo/learn_about_remotes_that_are_currently_unavailable.mdwn
@@ -0,0 +1,28 @@
+If a remote is only available on some networks, a command like `git annex drop`
+or `git annex get` may try to access the remote each time a file is
+processed, and suffer a long timeout each time. It seems git-annex could
+remember that a previous access of a remote failed, and automatically
+de-prioritize that remote, eg adjust its cost to below the next remote on
+the list. So it would learn about the current situation the process finds
+itself in.
+
+Seems this would be easy for checkPresent, since it throws an exception
+if the remote cannot be accessed.
+
+Other methods like storeKey and retrieveKeyFile don't differentiate between
+the remote not being accessible, and the action failing. It could be a lot
+of work and complication to add that distinction, including needing to
+change the external special remote protocol.
+
+Implementing it for at least checkPresent would be a good start. That would
+cover `git annex drop` and `git annex copy --to` and probably a few other
+commands.
+
+This learning could be unwanted behavior if git-annex is running while the
+computer is migrating between networks. Then it might de-prioritize a
+remote before it travels to the network where it can use that remote. This
+would mostly affect the assistant since it's run for long periods of time.
+It could undo the de-prioritization when it sees that the network has
+changed.
+
+--[[Joey]]
diff --git a/doc/todo/limit_to_low_cost_remotes.mdwn b/doc/todo/limit_to_low_cost_remotes.mdwn
new file mode 100644
index 0000000000..d92d674b68
--- /dev/null
+++ b/doc/todo/limit_to_low_cost_remotes.mdwn
@@ -0,0 +1,16 @@
+Add --maximum-cost=N which prevents trying to access any remotes with a
+larger cost. May as well add --minimum-cost too for completeness.
+
+My use case: Want to git annex get --auto and pull from any of 3 usb
+drives, but not from the network. --[[Joey]]
+
+> Hmm, [[todo/to_and_from_multiple_remotes]] might be another way to do
+> that. Put the 3 drives in a git remote group, or list the remotes on the
+> fly.
+> 
+> There could still be benefit in avoiding high cost remotes. But, the cost
+> numbers are only intended to create a local ordering, so making them part of a
+> user interface is kind of weird. While 50 might be a high cost in one
+> repository, in another repository it could be a fairly low cost. The user
+> would need to examine all the costs to pick the cost they want; using
+> remote names seems better UI. --[[Joey]]
diff --git a/doc/todo/lockdown_hooks.mdwn b/doc/todo/lockdown_hooks.mdwn
new file mode 100644
index 0000000000..0ec8b46241
--- /dev/null
+++ b/doc/todo/lockdown_hooks.mdwn
@@ -0,0 +1,50 @@
+Add git hooks that are used to [[internals/lockdown]] annexed objects.
+--[[Joey]]
+
+Use cases include:
+
+* Setting immutable bit on systems where git-annex is given the ability to
+  do so, to fully guard against accidental deletion in all circumstances.
+
+* For systems that ignore the write bit, but have some other way to prevent
+  write to a file (eg, ACLs or something).
+
+  Note that in such a case, `git-annex init`'s probe of the write bit
+  handling fails; as long as the hook is configured globally, it should
+  run the hook instead, and if it works, can avoid direct mode.
+
+Design:
+
+Configs: annex.lockdown-command, annex.unlockdown-command
+In these, "%path" is replaced with the file/directory to act on.
+
+Locking down a directory only needs to do the equivilant of removing its
+write bit, does not need to lockdown the files within it.
+
+It would be up to the command to decide how to handle the 
+core.sharedRepository configuration.
+
+These could be set in the global gitconfig file. The IncludeIf directive
+can be used to make them be used only for repositories located within a given
+mount point.
+
+git-annex test disables use of global gitconfig settings. There would need
+to be a way to let it use these.
+
+Perfomance:
+
+Hook would be called twice per store/drop of an annexed object, 
+once for the file and once for the parent directory.
+
+On windows, called four times per lock of an annexed object, to first thaw
+it and then freeze it. This could be reduced to 2, I think. 
+On posix, the file is locked without being thawed, 
+as only read access is needed.
+
+Probably running a shell script is not too much overhead in many cases,
+if it was too slow, there could be a variant that is run once and 
+fed the names of files to operate on via stdin.
+
+> These hooks may be too specific to this purpose, while a more generalized
+> hook could also support things like [[storing_xattrs|support_for_storing_xattrs]]
+> --[[Joey]]
diff --git a/doc/todo/lockdown_hooks/comment_1_649dbc3c951bbabc550e12e87e45b319._comment b/doc/todo/lockdown_hooks/comment_1_649dbc3c951bbabc550e12e87e45b319._comment
new file mode 100644
index 0000000000..8c35613c43
--- /dev/null
+++ b/doc/todo/lockdown_hooks/comment_1_649dbc3c951bbabc550e12e87e45b319._comment
@@ -0,0 +1,10 @@
+[[!comment format=mdwn
+ username="yarikoptic"
+ avatar="http://cdn.libravatar.org/avatar/f11e9c84cb18d26a1748c33b48c924b4"
+ subject="any chance to avoid necessity to config the hook(s)?"
+ date="2018-02-01T18:54:18Z"
+ content="""
+Thank you Joey for looking into this issue!  I am though a bit worried that necessity to configure using some kind of a hook would be ... suboptimal for a number of reasons.  Before laying out my argumentation, let me first ask:  why alternative \"lockdown\" mechanisms could not be sensed/configured per each repository during `init` and implemented within git-annex?
+
+As you have noted ```git-annex init's probe of the write bit handling fails...``` so git-annex already checks for a possible way to establish the \"lockdown\" for a given repository location. It just tries one possible mechanism ATM. But it could as well try multiple ways to achieve it, starting from current \"POSIX\", and then trying \"ACL\" if appropriate (i.e. tools found). Then if non-POSIX handling is needed, would simple add yet another configuration to .git/config of that repository, and consult to it to switch to corresponding lockdown implementation **within git-annex**.  This would be much more user friendly, and it would allow 3rd party tools using git-annex (such as datalad) to not worry about necessity to configure some additional hooks for a particular location, etc.
+"""]]
diff --git a/doc/todo/lockdown_hooks/comment_2_575c33970014662c664d71e573e718e7._comment b/doc/todo/lockdown_hooks/comment_2_575c33970014662c664d71e573e718e7._comment
new file mode 100644
index 0000000000..f261801824
--- /dev/null
+++ b/doc/todo/lockdown_hooks/comment_2_575c33970014662c664d71e573e718e7._comment
@@ -0,0 +1,28 @@
+[[!comment format=mdwn
+ username="joey"
+ subject="""comment 2"""
+ date="2018-02-05T17:04:36Z"
+ content="""
+Seems likely that there are a couple of different ways to use
+ACLs to remove write access. In the simple case, any existing ACL can be
+overwritten. In other cases, some other existing ACLs will need to be
+preserved and only a single part changed. In some cases, the ACL for a user
+should be changed, in others the ACL for a group.
+
+And there are several different varieties of ACLs (POSIX, NFS, Windows). 
+And there's the immutable bit, which might be wanted in some specific
+circumstances but certianly not by most people.
+
+So it makes sense to me to not embed specific knowledge of this into git-annex.
+
+This feels to me like something that the system administrator is going to
+want to set up. It would mostly be limited to repositories inside a given
+mount point that needs the unusual lockdown method due to using NFS or
+whatever. The global gitconfig can be set up to switch on the config only
+for those repositories, and the system administrator can set up hooks
+for the particular use case.
+
+I don't see why something like datalad would need to worry about this
+detail, any more than they worry about the PATH to system programs or other
+such things that the administrator sets up.
+"""]]
diff --git a/doc/todo/machine_readable_information_about_reason_of_failure_in_enableremote.mdwn b/doc/todo/machine_readable_information_about_reason_of_failure_in_enableremote.mdwn
new file mode 100644
index 0000000000..67d26d1d74
--- /dev/null
+++ b/doc/todo/machine_readable_information_about_reason_of_failure_in_enableremote.mdwn
@@ -0,0 +1,5 @@
+As initially brought up in https://github.com/datalad/datalad/pull/2515#issuecomment-391022075 it would be nice if there was a clear/robust way to programmatically obtain information why `enableremote` failed
+
+May be even more generally -- if there was a robust way to obtain errors for other commands as well, e.g. such as `fatal` messages such as `this operation must be run in a work tree`
+
+[[!meta author=yoh]]
diff --git a/doc/todo/machine_readable_information_about_reason_of_failure_in_enableremote/comment_1_dbe339cb1860d27ebc7116ab4d6e50fb._comment b/doc/todo/machine_readable_information_about_reason_of_failure_in_enableremote/comment_1_dbe339cb1860d27ebc7116ab4d6e50fb._comment
new file mode 100644
index 0000000000..9e15508505
--- /dev/null
+++ b/doc/todo/machine_readable_information_about_reason_of_failure_in_enableremote/comment_1_dbe339cb1860d27ebc7116ab4d6e50fb._comment
@@ -0,0 +1,37 @@
+[[!comment format=mdwn
+ username="joey"
+ subject="""initial thoughts"""
+ date="2018-05-22T19:27:24Z"
+ content="""
+* Could be --json-errors, with one json object per line per error.
+
+* When git-annex is processing multiple files, there could be multiple
+  error messages output by a single command. If --json-errors goes to
+  stdout then it becomes easy to tell what file an error is associated
+  with, since it would be adjacent to the --json object for the file.
+
+  (If --json-errors goes to stderr it would probably need to include
+  the file or other thing being processed, otherwise a consumer would
+  need to read stdout and stderr in a select loop.)
+
+* Whether it goes to stdout or stderr, other non-json output could
+  be mixed in, which complicates parsing.
+  
+  A good way to keep parsing
+  simple would be to send the json error objects to stdout, and only
+  support them for commands that support --json. (Which might be an
+  annoying limitation since not all commands do, and adding --json to some
+  commands, eg initremote, might be pretty hard.)  
+
+  Or there could be a
+  prefix that is documented for programs to look for the tell if a line
+  of output is a json error object. Eg, `{"git-annex-json-error":`
+
+* There ought to be a unique id in each particular json error,
+  so consumers don't need to expect a particular error text.
+
+* For some errors it would be useful to have more structure
+  than just an error message. For example, HTTP status codes could
+  be broken out; `git annex drop` might want to break out the number
+  of copies it was unable to lock.
+"""]]
diff --git a/doc/todo/machine_readable_information_about_reason_of_failure_in_enableremote/comment_2_06e98b0e0704dce8ced214600d0d6347._comment b/doc/todo/machine_readable_information_about_reason_of_failure_in_enableremote/comment_2_06e98b0e0704dce8ced214600d0d6347._comment
new file mode 100644
index 0000000000..0c6f45591c
--- /dev/null
+++ b/doc/todo/machine_readable_information_about_reason_of_failure_in_enableremote/comment_2_06e98b0e0704dce8ced214600d0d6347._comment
@@ -0,0 +1,9 @@
+[[!comment format=mdwn
+ username="joey"
+ subject="""comment 2"""
+ date="2018-05-22T19:56:15Z"
+ content="""
+datalad is looking for StatusCodeException and FailedConnectionException
+from http-client when enableremote fails. Which special remote is that,
+S3 or WebDAV or something?
+"""]]
diff --git a/doc/todo/make_addurl_respect_annex.largefiles_option.mdwn b/doc/todo/make_addurl_respect_annex.largefiles_option.mdwn
new file mode 100644
index 0000000000..b80349a760
--- /dev/null
+++ b/doc/todo/make_addurl_respect_annex.largefiles_option.mdwn
@@ -0,0 +1,6 @@
+ATM git annex addurl ignores  annex.largefiles option so to automate annexification or direct add to git for a list of files I need manually to download each one of them into a FILE and then "git annex add -c annex.largefiles='exclude=*.txt'  FILE".  But it would have been convenient if I could "addurl" some files directly from urls directly into git, as per largefiles settings.
+
+N.B. I do understand that use-case might be somewhat vague, let me know if I should expand reasoning
+[[!meta author=yoh]]
+
+> [[done]] --[[Joey]]
diff --git a/doc/todo/make_annex_info_more_efficient.mdwn b/doc/todo/make_annex_info_more_efficient.mdwn
new file mode 100644
index 0000000000..49a711965f
--- /dev/null
+++ b/doc/todo/make_annex_info_more_efficient.mdwn
@@ -0,0 +1,3 @@
+ATM it takes about a minute for 'git annex info' on a sizeable but not huge repository with only ~450 files under annex but a few thousand of files  (~7000) in the tree.  I am not quite sure why it takes that long since it seems to care only about annexed files.  Also it might be of benefit to parallelize some traversal operations to take advantage of multiple cpu/cores
+
+[[!meta author=yoh]]
diff --git a/doc/todo/make_annex_info_more_efficient/comment_1_afd5c806f7285442401b027f82a8c629._comment b/doc/todo/make_annex_info_more_efficient/comment_1_afd5c806f7285442401b027f82a8c629._comment
new file mode 100644
index 0000000000..77090b680a
--- /dev/null
+++ b/doc/todo/make_annex_info_more_efficient/comment_1_afd5c806f7285442401b027f82a8c629._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="https://me.yahoo.com/a/EbvxpTI_xP9Aod7Mg4cwGhgjrCrdM5s-#7c0f4"
+ subject="comment 1"
+ date="2016-02-15T15:28:40Z"
+ content="""
+actually it does indeed relate with all the files, since \"annexed files in working tree\"... if interested -- try on this repo https://surfer.nmr.mgh.harvard.edu/pub/dist/freesurfer/repo/freesurfer.git . On a clean new clone it took only 30 seconds ;)
+ 
+"""]]
diff --git a/doc/todo/make_annex_info_more_efficient/comment_2_0770ef5c4c261949a565723073480dcb._comment b/doc/todo/make_annex_info_more_efficient/comment_2_0770ef5c4c261949a565723073480dcb._comment
new file mode 100644
index 0000000000..16f0bea930
--- /dev/null
+++ b/doc/todo/make_annex_info_more_efficient/comment_2_0770ef5c4c261949a565723073480dcb._comment
@@ -0,0 +1,10 @@
+[[!comment format=mdwn
+ username="joey"
+ subject="""comment 2"""
+ date="2016-02-15T16:51:20Z"
+ content="""
+There's --fast if you don't need the expensive to obtain data.
+
+AFAIK, the working tree traversal that makes up most of the overhead of
+info is mostly disk-bound.
+"""]]
diff --git a/doc/todo/make_annex_info_more_efficient/comment_3_c022caab61061b1f77b78485089a9052._comment b/doc/todo/make_annex_info_more_efficient/comment_3_c022caab61061b1f77b78485089a9052._comment
new file mode 100644
index 0000000000..6bea4c6a6a
--- /dev/null
+++ b/doc/todo/make_annex_info_more_efficient/comment_3_c022caab61061b1f77b78485089a9052._comment
@@ -0,0 +1,18 @@
+[[!comment format=mdwn
+ username="https://me.yahoo.com/a/EbvxpTI_xP9Aod7Mg4cwGhgjrCrdM5s-#7c0f4"
+ subject="comment 3"
+ date="2016-02-15T17:02:59Z"
+ content="""
+yeap, saw fast but it is kinda of limited use. doesn't even go through git-annex branch to report amount of storage occupied.  I didn't check but \"working tree\" is probably a current checked out branch, and that is why requires traversal and thus slow.  What is often also desired to know what is the total number/size of annexed files I have locally in any given annex.  So something along of 'du -scm .git/annex/objects' (but via info in git-annex branch instead).  May be that could be included in the --fast portion?
+
+as for disk-bound:  in my case /tmp is in ram, there is no disk activities, git-annex is 100% (1 core) busy, FWIW:
+
+[[!format sh \"\"\"
+	Command being timed: \"git annex info\"
+	User time (seconds): 27.69
+	System time (seconds): 0.68
+	Percent of CPU this job got: 99%
+	Elapsed (wall clock) time (h:mm:ss or m:ss): 0:28.42
+\"\"\"]]
+
+"""]]
diff --git a/doc/todo/make_annex_info_more_efficient/comment_4_923dd7c22920b389488ca2625225164c._comment b/doc/todo/make_annex_info_more_efficient/comment_4_923dd7c22920b389488ca2625225164c._comment
new file mode 100644
index 0000000000..54ecf5aa8b
--- /dev/null
+++ b/doc/todo/make_annex_info_more_efficient/comment_4_923dd7c22920b389488ca2625225164c._comment
@@ -0,0 +1,7 @@
+[[!comment format=mdwn
+ username="https://me.yahoo.com/a/EbvxpTI_xP9Aod7Mg4cwGhgjrCrdM5s-#7c0f4"
+ subject="comment 4"
+ date="2016-02-15T17:05:08Z"
+ content="""
+FWIW the acctual git ls-tree command annex invokes takes 0.01 sec ;)
+"""]]
diff --git a/doc/todo/make_annex_info_more_efficient/comment_5_c6c8850aefe3ab81f1f113daa734695b._comment b/doc/todo/make_annex_info_more_efficient/comment_5_c6c8850aefe3ab81f1f113daa734695b._comment
new file mode 100644
index 0000000000..9509195364
--- /dev/null
+++ b/doc/todo/make_annex_info_more_efficient/comment_5_c6c8850aefe3ab81f1f113daa734695b._comment
@@ -0,0 +1,17 @@
+[[!comment format=mdwn
+ username="joey"
+ subject="""comment 5"""
+ date="2016-02-19T15:58:17Z"
+ content="""
+When run on a local repository, git-annex info does not look at the
+git-annex branch. That would be slower than traversing the directories.
+(Asking for info about a remote does look at the git-annex branch.)
+
+git ls-tree does not have to look at files on disk, so is not comparable.
+The work `git annex info` does is roughly the same as a du of
+.git/annex/objects and a readlink of each symlink in the work tree.
+
+If you just want the .git/annex/objects size, perhaps it would make sense
+to have a way to get only one stat from git-annex status. Or perhaps 
+it would be as good to `du .git/annex/objects`?
+"""]]
diff --git a/doc/todo/make_annex_info_more_efficient/comment_6_804dbb72757b09e3abad3e249f704da0._comment b/doc/todo/make_annex_info_more_efficient/comment_6_804dbb72757b09e3abad3e249f704da0._comment
new file mode 100644
index 0000000000..f4bedf504c
--- /dev/null
+++ b/doc/todo/make_annex_info_more_efficient/comment_6_804dbb72757b09e3abad3e249f704da0._comment
@@ -0,0 +1,11 @@
+[[!comment format=mdwn
+ username="joey"
+ subject="""comment 6"""
+ date="2016-02-19T18:49:38Z"
+ content="""
+Also, around the time you filed this, there was a change which turned out
+to cause all files in the work tree to be read in full, and buffered in
+memory for some time. This may have to do with some of the slowdown you
+saw, especially since you have a lot of non-annexed files in the tree.
+I fixed that problem in commit b0081598c7c580d6760374c42765e94e4750e793.
+"""]]
diff --git a/doc/todo/make_annex_info_more_efficient/comment_7_44efc2bfcdde576aaca002595476a2a2._comment b/doc/todo/make_annex_info_more_efficient/comment_7_44efc2bfcdde576aaca002595476a2a2._comment
new file mode 100644
index 0000000000..174ea100a5
--- /dev/null
+++ b/doc/todo/make_annex_info_more_efficient/comment_7_44efc2bfcdde576aaca002595476a2a2._comment
@@ -0,0 +1,10 @@
+[[!comment format=mdwn
+ username="https://me.yahoo.com/a/EbvxpTI_xP9Aod7Mg4cwGhgjrCrdM5s-#7c0f4"
+ subject="cache?"
+ date="2016-04-01T19:56:43Z"
+ content="""
+I was about to whine in a separate TODO but then remembered that the issue is not new...
+I wondered -- since sizes report depends on what is present or not locally, and that all directly relates to the state of git-annex branch, could may be annex cache collected information associated with a given annex / current branch treeishes?  Then subsequent invocations would be fast.
+
+In my case I would want to list information on multiple annexes e.g. in current directory.  If each one takes 3-4 seconds, for 30 of them -- minutes.  With caching, at least subsequent runs should be much faster (in case of no changes, which would be frequent case I think)
+"""]]
diff --git a/doc/todo/make_annex_info_more_efficient/comment_8_90a22c51b19707ee0cecbe652c6ffa98._comment b/doc/todo/make_annex_info_more_efficient/comment_8_90a22c51b19707ee0cecbe652c6ffa98._comment
new file mode 100644
index 0000000000..7122ce91f9
--- /dev/null
+++ b/doc/todo/make_annex_info_more_efficient/comment_8_90a22c51b19707ee0cecbe652c6ffa98._comment
@@ -0,0 +1,15 @@
+[[!comment format=mdwn
+ username="joey"
+ subject="""comment 8"""
+ date="2017-10-30T18:56:14Z"
+ content="""
+It's not clear to me what needs to be done here.
+
+I tried cloning the freesurfer repository, and in it with a cold
+disk cache, `git annex info` took 0.6 seconds (warm cache, 0.1 seconds).
+
+It seems like you wanted to get the "local annex size" value, perhaps
+without the overhead of the "size of annexed files in working tree"
+value? In an indirect mode repository, the former value is obtained
+the same as `du .git/annex/objects`, and should be similarly fast.
+"""]]
diff --git a/doc/todo/make_annex_info_more_efficient/comment_9_3bec13ce0c9f0932715d73106d576e86._comment b/doc/todo/make_annex_info_more_efficient/comment_9_3bec13ce0c9f0932715d73106d576e86._comment
new file mode 100644
index 0000000000..c3d0638976
--- /dev/null
+++ b/doc/todo/make_annex_info_more_efficient/comment_9_3bec13ce0c9f0932715d73106d576e86._comment
@@ -0,0 +1,10 @@
+[[!comment format=mdwn
+ username="yarikoptic"
+ avatar="http://cdn.libravatar.org/avatar/f11e9c84cb18d26a1748c33b48c924b4"
+ subject="comment 9"
+ date="2017-12-05T18:51:49Z"
+ content="""
+rereading my previous comment (it has been awhile), I was suggesting a super-dooper-feature.  I am not sure if it is still needed/desired since upon a quick try again, info seems to be indeed relatively speedy when ran on \"hot\" (e.g. 2nd time in a row; not sure how long the effect lasts ;)).
+The idea was -- to cache \"status\" information locally so subsequent invocations (if git-annex branch didn't change) could just pick it up from the cache if nothing has changed in terms of the git-annex branch and current branch position. E.g. if I am at a commit ABC and at git-annex branch state XYZ, why not to have .git/annex/caches/info/XYZ-ABC.json which would pretty much have output of `git annex info --json` which it could reuse, without ANY traversal of git-annex branch or local files tree, on a subsequent call if still in XYZ state and that commit.  Whenever git-annex branch progresses away from XYZ, all previous ones in the cache could be let RiP.
+
+"""]]
diff --git a/doc/todo/make_copy_--fast__faster.mdwn b/doc/todo/make_copy_--fast__faster.mdwn
new file mode 100644
index 0000000000..4731806eae
--- /dev/null
+++ b/doc/todo/make_copy_--fast__faster.mdwn
@@ -0,0 +1,7 @@
+I was trying to copy files which failed to copy (3 out of 6,000) to remote host after copy -J4.  Succeeded.  But with subsequent runs, apparently even with copy --fast it takes annex 10 sec for annex to realize there is nothing to copy. git ls-files  which annex calls returns list immediately, so it is really some parsing/access to data under git-annex branch which takes awhile.  I think we had similar discussion before but couldn't find.  So I wondered to whine again to see if some optimization is possible to make --fast copies faster, especially whenever there is nothing to copy.
+
+[[!meta author=yoh]]
+
+> closing as yoh is happy. [[done]]. Note that I copied benechmarking
+> related comments to the [[/benchmarking]] page for future reference.
+> --[[Joey]]
diff --git a/doc/todo/make_copy_--fast__faster/comment_10_1af4ac0d37c876912678522895c1656b._comment b/doc/todo/make_copy_--fast__faster/comment_10_1af4ac0d37c876912678522895c1656b._comment
new file mode 100644
index 0000000000..868b103646
--- /dev/null
+++ b/doc/todo/make_copy_--fast__faster/comment_10_1af4ac0d37c876912678522895c1656b._comment
@@ -0,0 +1,61 @@
+[[!comment format=mdwn
+ username="joey"
+ subject="""comment 10"""
+ date="2016-09-29T18:33:33Z"
+ content="""
+* Optimised key2file and file2key. 18% scanning time speedup.
+* Optimised adjustGitEnv. 50% git-annex branch query speedup
+* Optimised parsePOSIXTime. 10% git-annex branch query speedup
+* Tried making catObjectDetails.receive use ByteString for parsing, 
+  but that did not seem to speed it up significantly.
+  So it parsing is already fairly optimal, it's just that a
+  lot of data passes through it when querying the git-annex
+  branch.
+
+After all that, profiling `git-annex find`:
+
+	        Thu Sep 29 16:51 2016 Time and Allocation Profiling Report  (Final)
+	
+	           git-annex.1 +RTS -p -RTS find
+	
+	        total time  =        1.73 secs   (1730 ticks @ 1000 us, 1 processor)
+	        total alloc = 1,812,406,632 bytes  (excludes profiling overheads)
+	
+	COST CENTRE            MODULE                  %time %alloc
+	
+	md5                    Data.Hash.MD5            28.0   37.9
+	catchIO                Utility.Exception        10.2   12.5
+	inAnnex'.checkindirect Annex.Content             9.9    3.7
+	catches                Control.Monad.Catch       8.7    5.7
+	readish                Utility.PartialPrelude    5.7    3.0
+	isAnnexLink            Annex.Link                5.0    8.4
+	keyFile                Annex.Locations           4.2    5.8
+	spanList               Data.List.Utils           4.0    6.3
+	startswith             Data.List.Utils           2.0    1.3
+
+And `git-annex find --not --in web`:
+
+	        Thu Sep 29 16:35 2016 Time and Allocation Profiling Report  (Final)
+	
+	           git-annex +RTS -p -RTS find --not --in web
+	
+	        total time  =        5.24 secs   (5238 ticks @ 1000 us, 1 processor)
+	        total alloc = 3,293,314,472 bytes  (excludes profiling overheads)
+	
+	COST CENTRE               MODULE                      %time %alloc
+	
+	catObjectDetails.receive  Git.CatFile                  12.9    5.5
+	md5                       Data.Hash.MD5                10.6   20.8
+	readish                   Utility.PartialPrelude        7.3    8.2
+	catchIO                   Utility.Exception             6.7    7.3
+	spanList                  Data.List.Utils               4.1    7.4
+	readFileStrictAnyEncoding Utility.Misc                  3.5    1.3
+	catches                   Control.Monad.Catch           3.3    3.2
+
+So, quite a large speedup overall!
+
+This leaves md5 still unoptimised at 10-28% of CPU use. I looked at switching
+it to cryptohash's implementation, but it would require quite a lot of
+bit-banging math to pull the used values out of the ByteString containing
+the md5sum.
+"""]]
diff --git a/doc/todo/make_copy_--fast__faster/comment_11_1ca8d9765e6e3a18ae09df74bc390a00._comment b/doc/todo/make_copy_--fast__faster/comment_11_1ca8d9765e6e3a18ae09df74bc390a00._comment
new file mode 100644
index 0000000000..619351d4ca
--- /dev/null
+++ b/doc/todo/make_copy_--fast__faster/comment_11_1ca8d9765e6e3a18ae09df74bc390a00._comment
@@ -0,0 +1,49 @@
+[[!comment format=mdwn
+ username="joey"
+ subject="""comment 11"""
+ date="2017-05-15T21:56:52Z"
+ content="""
+Switched from MissingH to cryptonite for md5. It did move md5 out of the top CPU spot but
+the overall runtime didn't change much. Memory allocations did go down by a
+good amount.
+
+Updated profiles:
+
+	           git-annex +RTS -p -RTS find
+	
+	        total time  =        1.63 secs   (1629 ticks @ 1000 us, 1 processor)
+	        total alloc = 1,496,336,496 bytes  (excludes profiling overheads)
+	
+	COST CENTRE              MODULE                     SRC                                             %time %alloc
+	
+	catchIO                  Utility.Exception          Utility/Exception.hs:79:1-17                     14.1   15.1
+	inAnnex'.checkindirect   Annex.Content              Annex/Content.hs:(108,9)-(119,39)                10.6    4.8
+	catches                  Control.Monad.Catch        src/Control/Monad/Catch.hs:(432,1)-(436,76)       8.6    6.9
+	spanList                 Data.List.Utils            src/Data/List/Utils.hs:(150,1)-(155,36)           6.7   11.1
+	isAnnexLink              Annex.Link                 Annex/Link.hs:35:1-85                             5.0   10.2
+	keyFile                  Annex.Locations            Annex/Locations.hs:(456,1)-(462,19)               5.0    7.0
+	readish                  Utility.PartialPrelude     Utility/PartialPrelude.hs:(48,1)-(50,20)          3.8    2.0
+	startswith               Data.List.Utils            src/Data/List/Utils.hs:103:1-23                   3.6    2.3
+	splitc                   Utility.Misc               Utility/Misc.hs:(52,1)-(54,25)                    3.4    6.5
+	s2w8                     Data.Bits.Utils            src/Data/Bits/Utils.hs:65:1-15                    2.6    6.4
+	keyPath                  Annex.Locations            Annex/Locations.hs:(492,1)-(494,23)               2.5    4.4
+	fileKey.unesc            Annex.Locations            Annex/Locations.hs:(469,9)-(474,39)               2.0    3.5
+	copyAndFreeze            Data.ByteArray.Methods     Data/ByteArray/Methods.hs:(224,1)-(227,21)        1.8    0.5
+
+	           git-annex +RTS -p -RTS find --not --in web
+	
+	        total time  =        5.33 secs   (5327 ticks @ 1000 us, 1 processor)
+	        total alloc = 2,908,489,000 bytes  (excludes profiling overheads)
+	
+	COST CENTRE          MODULE                     SRC                                             %time %alloc
+	
+	catObjectDetails.\   Git.CatFile                Git/CatFile.hs:(80,72)-(88,97)                    7.8    2.8
+	catchIO              Utility.Exception          Utility/Exception.hs:79:1-17                      7.6    8.3
+	spanList             Data.List.Utils            src/Data/List/Utils.hs:(150,1)-(155,36)           5.8    9.1
+	readish              Utility.PartialPrelude     Utility/PartialPrelude.hs:(48,1)-(50,20)          4.5    4.0
+	parseResp            Git.CatFile                Git/CatFile.hs:(113,1)-(124,28)                   4.4    2.9
+	readFileStrict       Utility.Misc               Utility/Misc.hs:33:1-59                           3.7    1.6
+	catches              Control.Monad.Catch        src/Control/Monad/Catch.hs:(432,1)-(436,76)       3.1    3.6
+	encodeW8             Utility.FileSystemEncoding Utility/FileSystemEncoding.hs:(131,1)-(133,70)    3.1    2.3
+	
+"""]]
diff --git a/doc/todo/make_copy_--fast__faster/comment_12_14856a2886cf73d1bee57ef9fad01661._comment b/doc/todo/make_copy_--fast__faster/comment_12_14856a2886cf73d1bee57ef9fad01661._comment
new file mode 100644
index 0000000000..4c5b6f7d9b
--- /dev/null
+++ b/doc/todo/make_copy_--fast__faster/comment_12_14856a2886cf73d1bee57ef9fad01661._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="joey"
+ subject="""comment 12"""
+ date="2017-10-30T18:48:21Z"
+ content="""
+There's of course always possibility of more speed improvements, but I'm
+wondering if this has already been addressed sufficient to close it?
+"""]]
diff --git a/doc/todo/make_copy_--fast__faster/comment_13_0f8e2127cea96c4f9609fa7599b1eec9._comment b/doc/todo/make_copy_--fast__faster/comment_13_0f8e2127cea96c4f9609fa7599b1eec9._comment
new file mode 100644
index 0000000000..e52d1bc2f5
--- /dev/null
+++ b/doc/todo/make_copy_--fast__faster/comment_13_0f8e2127cea96c4f9609fa7599b1eec9._comment
@@ -0,0 +1,10 @@
+[[!comment format=mdwn
+ username="yarikoptic"
+ avatar="http://cdn.libravatar.org/avatar/f11e9c84cb18d26a1748c33b48c924b4"
+ subject="thanks"
+ date="2017-10-30T20:04:48Z"
+ content="""
+I do think that things became smoother/faster since then.  I guess we could consider this one closed for now, and I will keep in mind that --from mode is faster.
+
+Cheers,
+"""]]
diff --git a/doc/todo/make_copy_--fast__faster/comment_1_24a9ca652007a18f18b368232cf549da._comment b/doc/todo/make_copy_--fast__faster/comment_1_24a9ca652007a18f18b368232cf549da._comment
new file mode 100644
index 0000000000..b53e04fd9f
--- /dev/null
+++ b/doc/todo/make_copy_--fast__faster/comment_1_24a9ca652007a18f18b368232cf549da._comment
@@ -0,0 +1,7 @@
+[[!comment format=mdwn
+ username="joey"
+ subject="""comment 1"""
+ date="2016-05-21T13:53:15Z"
+ content="""
+--to or --from? The latter is faster due to locality..
+"""]]
diff --git a/doc/todo/make_copy_--fast__faster/comment_2_0c67f467d730a0966b43171de0382c42._comment b/doc/todo/make_copy_--fast__faster/comment_2_0c67f467d730a0966b43171de0382c42._comment
new file mode 100644
index 0000000000..6c37e77e53
--- /dev/null
+++ b/doc/todo/make_copy_--fast__faster/comment_2_0c67f467d730a0966b43171de0382c42._comment
@@ -0,0 +1,49 @@
+[[!comment format=mdwn
+ username="https://me.yahoo.com/a/EbvxpTI_xP9Aod7Mg4cwGhgjrCrdM5s-#7c0f4"
+ subject="comment 2"
+ date="2016-05-25T01:09:56Z"
+ content="""
+\"to remote host \"  so it was \"--to\". annex is already aware of having those files in that remote (see below).
+
+[[!format sh \"\"\"
+$> git annex copy --to=datalad-public --fast .        
+git annex copy --to=datalad-public --fast .  7.33s user 0.91s system 55% cpu 14.772 total
+
+$> git annex info
+repository mode: indirect
+trusted repositories: 0
+semitrusted repositories: 5
+	00000000-0000-0000-0000-000000000001 -- web
+ 	00000000-0000-0000-0000-000000000002 -- bittorrent
+ 	123c73e5-a8dc-4cff-8ffc-679c7ea67f94 -- yoh@smaug:/mnt/datasets/datalad/crawl/neurovault [here]
+ 	48c1556f-6241-45de-9497-338d437fcb62 -- yoh@falkor:/srv/datasets.datalad.org/www/neurovault/snapshots [datalad-public]
+ 	af2785da-2538-4346-a6f6-f2f30fc3f025 -- [datalad-archives]
+untrusted repositories: 0
+transfers in progress: none
+available local disk space: 31.42 terabytes (+1 megabyte reserved)
+local annex keys: 6615
+local annex size: 12.77 gigabytes
+annexed files in working tree: 6628
+size of annexed files in working tree: 6.31 gigabytes
+bloom filter size: 32 mebibytes (1.3% full)
+backend usage: 
+	SHA256E: 6628
+
+$> git annex whereis | head -30               
+whereis 1003/13873.nii.gz (3 copies) 
+  	123c73e5-a8dc-4cff-8ffc-679c7ea67f94 -- yoh@smaug:/mnt/datasets/datalad/crawl/neurovault [here]
+   	48c1556f-6241-45de-9497-338d437fcb62 -- yoh@falkor:/srv/datasets.datalad.org/www/neurovault/snapshots [datalad-public]
+   	af2785da-2538-4346-a6f6-f2f30fc3f025 -- [datalad-archives]
+
+  datalad-archives: dl+archive:SHA256E-s6460020224--710cc05117e2290e2f793271d11e26452cdc111121e09a937dbf5a34b3cc0107.tar/neurovault_snapshot/1003/13873.nii.gz#size=23262
+ok
+whereis 1003/13874.nii.gz (3 copies) 
+  	123c73e5-a8dc-4cff-8ffc-679c7ea67f94 -- yoh@smaug:/mnt/datasets/datalad/crawl/neurovault [here]
+   	48c1556f-6241-45de-9497-338d437fcb62 -- yoh@falkor:/srv/datasets.datalad.org/www/neurovault/snapshots [datalad-public]
+   	af2785da-2538-4346-a6f6-f2f30fc3f025 -- [datalad-archives]
+...
+> git annex copy --to=datalad-public .       
+copy 1003/13873.nii.gz (checking datalad-public...) yoh@datasets.datalad.org's password: 
+
+\"\"\"]]
+"""]]
diff --git a/doc/todo/make_copy_--fast__faster/comment_3_5cd9e6b5d6d015120b5852bd212314aa._comment b/doc/todo/make_copy_--fast__faster/comment_3_5cd9e6b5d6d015120b5852bd212314aa._comment
new file mode 100644
index 0000000000..d4aa80ecb3
--- /dev/null
+++ b/doc/todo/make_copy_--fast__faster/comment_3_5cd9e6b5d6d015120b5852bd212314aa._comment
@@ -0,0 +1,22 @@
+[[!comment format=mdwn
+ username="joey"
+ subject="""comment 3"""
+ date="2016-05-31T16:02:01Z"
+ content="""
+copy --to has to query the git-annex branch to see if the file is on the remote.
+So it has worse locality than copy --from, which can simply stat the local
+file to see if it's present.
+
+Whatever inneficiencies git-annex has here are well swamped by the overhead
+of git querying the branch. 
+
+When the remote has most of the files already, `git annex copy --to remote` is
+similar to `git annex find --not --in remote`.
+
+Here I've ran that under /usr/bin/time, and it looks like git-annex
+ran for 89 seconds out of the 260 second runtime. So at least 65% of the total
+runtime is spent by git querying the branch.
+
+	89.26user 6.92system 4:20.80elapsed 36%CPU (0avgtext+0avgdata 75584maxresident)k
+	516432inputs+0outputs (0major+31156minor)pagefaults 0swaps
+"""]]
diff --git a/doc/todo/make_copy_--fast__faster/comment_4_3ac10a07c74e5debafc9ae574d26c955._comment b/doc/todo/make_copy_--fast__faster/comment_4_3ac10a07c74e5debafc9ae574d26c955._comment
new file mode 100644
index 0000000000..bb9258b45c
--- /dev/null
+++ b/doc/todo/make_copy_--fast__faster/comment_4_3ac10a07c74e5debafc9ae574d26c955._comment
@@ -0,0 +1,11 @@
+[[!comment format=mdwn
+ username="joey"
+ subject="""comment 4"""
+ date="2016-08-03T16:02:46Z"
+ content="""
+--failed can now be used to retry only failed transfers. So that will be a
+lot faster in that specific case. 
+
+Leaving this bug open for the general wishlist that copy --fast be somehow
+a lot faster than it is at finding things that need to be copied.
+"""]]
diff --git a/doc/todo/make_copy_--fast__faster/comment_5_eb7008151a59e35c7850df3a86cf3587._comment b/doc/todo/make_copy_--fast__faster/comment_5_eb7008151a59e35c7850df3a86cf3587._comment
new file mode 100644
index 0000000000..8fbd3beba8
--- /dev/null
+++ b/doc/todo/make_copy_--fast__faster/comment_5_eb7008151a59e35c7850df3a86cf3587._comment
@@ -0,0 +1,11 @@
+[[!comment format=mdwn
+ username="https://me.yahoo.com/a/EbvxpTI_xP9Aod7Mg4cwGhgjrCrdM5s-#7c0f4"
+ subject="also CPU (on git and git-annex processes) doesn't go to 100%"
+ date="2016-09-08T16:32:08Z"
+ content="""
+seems to wobble around 50% for each one of git and git-annex processes... probably would be an overkill but may be it is easy in haskell (so throwing idea around) if communication was done in async fashion (git-annex wouldn't wait for git to respond but would process its own queue of already returned from git results, while submitting new ones as soon as previous comes out from the --batch). That might make both processes busy to a 100%.
+
+another idea -- could may be 'annex find' get a -J flag thus starting multiple git ls-files querying processes?
+
+or both ideas are too overengineered/not tractable? ;)
+"""]]
diff --git a/doc/todo/make_copy_--fast__faster/comment_6_3a08de49e9661f9df5bab272e170461a._comment b/doc/todo/make_copy_--fast__faster/comment_6_3a08de49e9661f9df5bab272e170461a._comment
new file mode 100644
index 0000000000..07cf33a546
--- /dev/null
+++ b/doc/todo/make_copy_--fast__faster/comment_6_3a08de49e9661f9df5bab272e170461a._comment
@@ -0,0 +1,28 @@
+[[!comment format=mdwn
+ username="https://me.yahoo.com/a/EbvxpTI_xP9Aod7Mg4cwGhgjrCrdM5s-#7c0f4"
+ subject="comment 6"
+ date="2016-09-09T12:47:30Z"
+ content="""
+ha -- a wild idea:  instead of   git ls-files git-annex | git cat-file   you could be much  better off  with using \"git archive\" to dump the content of all the files under git-annex branch!
+
+[[!format sh \"\"\"
+$> GIT_TRACE_PACKET=true GIT_TRACE_PERFORMANCE=true git annex find --not --in here >/dev/null                                      
+08:46:11.246625 trace.c:420             performance: 0.000291504 s: git command: '/usr/lib/git-annex.linux/shimmed/git/git' 'config' '--null' '--list'
+08:46:11.267559 trace.c:420             performance: 0.000466198 s: git command: '/usr/lib/git-annex.linux/shimmed/git/git' '--git-dir=.git' '--work-tree=.' '--literal-pathspecs' 'show-ref' 'git-annex'
+08:46:11.271522 trace.c:420             performance: 0.000434572 s: git command: '/usr/lib/git-annex.linux/shimmed/git/git' '--git-dir=.git' '--work-tree=.' '--literal-pathspecs' 'show-ref' '--hash' 'refs/heads/git-annex'
+08:46:22.647051 trace.c:420             performance: 11.387079176 s: git command: '/usr/lib/git-annex.linux/shimmed/git/git' '--git-dir=.git' '--work-tree=.' '--literal-pathspecs' 'ls-files' '--cached' '-z' '--'
+08:46:23.616005 trace.c:420             performance: 12.339791892 s: git command: '/usr/lib/git-annex.linux/shimmed/git/git' '--git-dir=.git' '--work-tree=.' '--literal-pathspecs' 'cat-file' '--batch'
+08:46:23.616052 trace.c:420             performance: 12.391364205 s: git command: 'git' 'annex' 'find' '--not' '--in' 'here'
+
+
+$> git ls-tree -r --name-only git-annex | sed -e \"s/^/git-annex:/g\" | time git --git-dir=.git cat-file --buffer --batch >| /tmp/111
+git --git-dir=.git cat-file --buffer --batch >| /tmp/111  7.80s user 0.40s system 99% cpu 8.214 total
+                
+
+$> time git archive git-annex > /dev/null                                                                                          
+git archive git-annex > /dev/null  0.20s user 0.00s system 97% cpu 0.212 total
+
+\"\"\"]]
+
+x40 times faster (if we disregard time to parse/split tar, but it should not be way too much I think)
+"""]]
diff --git a/doc/todo/make_copy_--fast__faster/comment_7_3f52b6e19035d3c891356c6d98035987._comment b/doc/todo/make_copy_--fast__faster/comment_7_3f52b6e19035d3c891356c6d98035987._comment
new file mode 100644
index 0000000000..c47132591d
--- /dev/null
+++ b/doc/todo/make_copy_--fast__faster/comment_7_3f52b6e19035d3c891356c6d98035987._comment
@@ -0,0 +1,58 @@
+[[!comment format=mdwn
+ username="joey"
+ subject="""comment 7"""
+ date="2016-09-14T15:28:23Z"
+ content="""
+First, note that git-annex 6.20160619 sped up the git-annex 
+command startup time significantly. Please be sure to use a current
+version in benchmarks, and state the version.
+
+`git archive` (and `git cat-file --batch --batch-all-objects`) are just
+reading packs and loose objects in disk order and dumping out the contents.
+`git cat-file --batch` has to look up objects in the pack index files, seek
+in the pack, etc. It's not a fair comparison.
+
+Note that `git annex find`, when used without options like --in or --copies, 
+does not need to read anything from `git cat-file` at all. The
+`GIT_TRACE_PERFORMANCE` you show is misleading; it's just showing how long
+the git command is left running, idle.
+
+`git annex find`'s overhead should be purely traversing the filesystem tree
+and checking what symlinks point to files. You can write programs that do
+the same thing without using git at all (or only `git ls-files`), and
+compare them to git-annex's time; that would be a fairer comparison.
+Ideally, `git annex find` would be entirely system call bound and would use
+very little CPU itself.
+
+By contrast, `git annex copy` makes significant use of `git cat-file --batch`,
+since it needs to look up location log information to see if the
+--to/--from remote has the files.
+
+`git annex copy -J` already parallelizes the parts of the code that look at
+the location log. Including spinning up a separate `git cat-file --batch`
+processes for each thread, so they won't contend on such queries. So I
+would expect that to make it faster, even leaving aside the speed benefits
+of doing the actual copies in parallel.
+
+My feeling is that the best way to speed these up is going to be in one 
+of these classes:
+
+* It's possible that `git cat-file --batch` is somehow slower than it needs
+  to be. Perhaps it's not doing good caching between queries or has
+  inneficient seralization/bad stdio buffering. It might just be the case 
+  that using something  like libgit2 instead would be faster.
+  (Due to libgit2's poor interface stability, it would have to be an
+  optional build flag.)
+
+* Many small optimisations to the code. The use of Strings throughout
+  git-annex could well be a source of systematic small innefficiences,
+  and using ByteString might eliminate those. (But this would be a huge job.)
+  (The `git cat-file --batch` communication is already done using
+  bytestrings.)
+
+* A completely lateral move. For example, if git-annex kept its own
+  database recording which files are present, then `git annex find`
+  could do a simple database query and not need to chase all the symlinks.
+  But such a database needs to somehow be kept in sync or reconciled
+  with the git index, it's not an easy thing.
+"""]]
diff --git a/doc/todo/make_copy_--fast__faster/comment_8_c1f99493f5e5c362d5c39f048280b11b._comment b/doc/todo/make_copy_--fast__faster/comment_8_c1f99493f5e5c362d5c39f048280b11b._comment
new file mode 100644
index 0000000000..e0f4987a02
--- /dev/null
+++ b/doc/todo/make_copy_--fast__faster/comment_8_c1f99493f5e5c362d5c39f048280b11b._comment
@@ -0,0 +1,45 @@
+[[!comment format=mdwn
+ username="joey"
+ subject="""profiling"""
+ date="2016-09-26T19:20:36Z"
+ content="""
+Built git-annex with profiling, using `stack build --profile`
+
+(For reproduciblity, running git-annex in a clone of the git-annex repo
+https://github.com/RichiH/conference_proceedings with rev
+2797a49023fc24aff6fcaec55421572e1eddcfa2 checked out. It has 9496 annexed
+objects.)
+
+Profiling `git-annex find +RTS -p`:
+
+	        total time  =        3.53 secs   (3530 ticks @ 1000 us, 1 processor)
+	        total alloc = 3,772,700,720 bytes  (excludes profiling overheads)
+	
+	COST CENTRE            MODULE                  %time %alloc
+	
+	spanList               Data.List.Utils          32.6   37.7
+	startswith             Data.List.Utils          14.3    8.1
+	md5                    Data.Hash.MD5            12.4   18.2
+	join                   Data.List.Utils           6.9   13.7
+	catchIO                Utility.Exception         5.9    6.0
+	catches                Control.Monad.Catch       5.0    2.8
+	inAnnex'.checkindirect Annex.Content             4.6    1.8
+	readish                Utility.PartialPrelude    3.0    1.4
+	isAnnexLink            Annex.Link                2.6    4.0
+	split                  Data.List.Utils           1.5    0.8
+	keyPath                Annex.Locations           1.2    1.7
+
+
+This is interesting!
+
+Fully 40% of CPU time and allocations are in list (really String) processing,
+and the details of the profiling report show that `spanList` and `startsWith`
+and `join` are all coming from calls to `replace` in `keyFile` and `fileKey`.
+Both functions nest several calls to replace, so perhaps that could be unwound
+into a single pass and/or a ByteString used to do it more efficiently.
+
+12% of run time is spent calculating the md5 hashes for the hash
+directories for .git/annex/objects. Data.Hash.MD5 is from missingh, and
+it is probably a quite unoptimised version. Switching to the version
+if cryptonite would probably speed it up a lot.
+"""]]
diff --git a/doc/todo/make_copy_--fast__faster/comment_9_f4d802a28b79905da0cb24af6cb65b0a._comment b/doc/todo/make_copy_--fast__faster/comment_9_f4d802a28b79905da0cb24af6cb65b0a._comment
new file mode 100644
index 0000000000..9692ad2d73
--- /dev/null
+++ b/doc/todo/make_copy_--fast__faster/comment_9_f4d802a28b79905da0cb24af6cb65b0a._comment
@@ -0,0 +1,42 @@
+[[!comment format=mdwn
+ username="joey"
+ subject="""more profiling"""
+ date="2016-09-26T19:59:43Z"
+ content="""
+Instead of profiling `git annex copy --to remote`, I profiled `git annex
+find --not --in web`, which needs to do the same kind of location log lookup.
+
+	        total time  =       12.41 secs   (12413 ticks @ 1000 us, 1 processor)
+	        total alloc = 8,645,057,104 bytes  (excludes profiling overheads)
+	
+	COST CENTRE               MODULE                      %time %alloc
+	
+	adjustGitEnv              Git.Env                      21.4   37.0
+	catchIO                   Utility.Exception            13.2    2.8
+	spanList                  Data.List.Utils              12.6   17.9
+	parsePOSIXTime            Logs.TimeStamp                6.1    5.0
+	catObjectDetails.receive  Git.CatFile                   5.9    2.1
+	startswith                Data.List.Utils               5.7    3.8
+	md5                       Data.Hash.MD5                 5.1    7.9
+	join                      Data.List.Utils               2.4    6.0
+	readFileStrictAnyEncoding Utility.Misc                  2.2    0.5
+
+The adjustGitEnv overhead is a surprise! It seems it is getting called once
+per file, and allocating a new copy of the environment each time. Call stack:
+withIndex calls withIndexFile calls addGitEnv calls adjustGitEnv.
+Looks like simply making gitEnv be cached at startup would avoid most of
+the adjustGitEnv slowdown.
+
+(The catchIO overhead is a false reading; the detailed profile shows
+that all its time and allocations are inherited. getAnnexLinkTarget
+is running catchIO in the expensive case, so readSymbolicLink is
+the actual expensive bit.)
+
+The parsePOSIXTime comes from reading location logs. It's implemented
+using a generic Data.Time.Format.parseTime, which uses a format string
+"%s%Qs". A custom parser that splits into seconds and picoseconds
+and simply reads both numbers might be more efficient.
+
+catObjectDetails.receive is implemented using mostly String and could
+probably be sped up by being converted to use ByteString.
+"""]]
diff --git a/doc/todo/make_glacier-cli_executable_path_configurable.mdwn b/doc/todo/make_glacier-cli_executable_path_configurable.mdwn
new file mode 100644
index 0000000000..4c4028dc04
--- /dev/null
+++ b/doc/todo/make_glacier-cli_executable_path_configurable.mdwn
@@ -0,0 +1,8 @@
+[glacier-cli](https://github.com/basak/glacier-cli) calls its own command `glacier` rather than `glacier-cli` or something else.  This conflicts with [boto](https://github.com/boto/boto/)'s own `glacier` executable, as noted here:
+
+* 
+* 
+
+Whilst the `glacier-cli` project should resolve this conflict, it would be good if git-annex could be made to use a configurable path for this executable, rather than just assuming that it has been installed as `glacier`.  After all, its installation procedure is simply telling the user to run `ln -s`, so there's no reason why the user couldn't make the target of this command `~/bin/glacier-cli` rather than `~/bin/glacier` - it's really irrelevant what the source file inside the git repo is called.
+
+Of course, [`checkSaneGlacierCommand`](https://github.com/joeyh/git-annex/blob/master/Remote/Glacier.hs#L307) is still very much worth having, for safety.
diff --git a/doc/todo/make_glacier-cli_executable_path_configurable/comment_1_08ab00266ad06fed9123d6a2ea0b5e6a._comment b/doc/todo/make_glacier-cli_executable_path_configurable/comment_1_08ab00266ad06fed9123d6a2ea0b5e6a._comment
new file mode 100644
index 0000000000..77c1bdd436
--- /dev/null
+++ b/doc/todo/make_glacier-cli_executable_path_configurable/comment_1_08ab00266ad06fed9123d6a2ea0b5e6a._comment
@@ -0,0 +1,7 @@
+[[!comment format=mdwn
+ username="basak"
+ subject="comment 1"
+ date="2015-04-24T15:48:48Z"
+ content="""
+Well, it's supposed to be a command line command, and I don't type `cd-cli` and `ls-cli`. So while `glacier-cli` might be fine as a project name and is fine for a name for integration, I don't think it makes sense to call it that in `/usr/bin/`, which is why I didn't. I'd prefer to have seen that boto integrate an improved `glacier` command, or for packaging to provide this one as an alternative (like `mawk` vs. `gawk` as `/usr/bin/awk`). But upstream boto considers themselves deprecated, so that's not going to happen. One of these days I'll package glacier-cli up for Debian, at which point I'll see if the boto maintainer is interested in doing something, since I don't actually believe anybody uses boto's glacier command (since it's mostly useless).
+"""]]
diff --git a/doc/todo/make_glacier-cli_executable_path_configurable/comment_2_d89e073643af0d80833b2d7c9752d23d._comment b/doc/todo/make_glacier-cli_executable_path_configurable/comment_2_d89e073643af0d80833b2d7c9752d23d._comment
new file mode 100644
index 0000000000..8032ed33b4
--- /dev/null
+++ b/doc/todo/make_glacier-cli_executable_path_configurable/comment_2_d89e073643af0d80833b2d7c9752d23d._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="https://adamspiers.wordpress.com/"
+ nickname="adamspiers"
+ subject="Good point"
+ date="2015-04-24T15:55:29Z"
+ content="""
+glacier-cli would be a rather silly name to put in `/usr/bin`.  How about `glcr`, as suggested [here](https://github.com/basak/glacier-cli/issues/30#issuecomment-95972840)?
+"""]]
diff --git a/doc/todo/make_glacier-cli_executable_path_configurable/comment_3_451c6788535e27482377cd60128c1cd6._comment b/doc/todo/make_glacier-cli_executable_path_configurable/comment_3_451c6788535e27482377cd60128c1cd6._comment
new file mode 100644
index 0000000000..da9033f79e
--- /dev/null
+++ b/doc/todo/make_glacier-cli_executable_path_configurable/comment_3_451c6788535e27482377cd60128c1cd6._comment
@@ -0,0 +1,10 @@
+[[!comment format=mdwn
+ username="joey"
+ subject="""comment 3"""
+ date="2015-04-24T17:23:10Z"
+ content="""
+I don't want to complicate git-annex more with configurable names for
+programs, and glacier is not at all special in this regard, any program
+could be installed under any namee. We pick non-conflicting names to
+avoid integration nightmares. Pick a name and I'll use it.
+"""]]
diff --git a/doc/todo/make_status_show_staged_files/comment_1_3874f5da5a300b7d443e1ad6373b4099._comment b/doc/todo/make_status_show_staged_files/comment_1_3874f5da5a300b7d443e1ad6373b4099._comment
new file mode 100644
index 0000000000..7e7cc139b1
--- /dev/null
+++ b/doc/todo/make_status_show_staged_files/comment_1_3874f5da5a300b7d443e1ad6373b4099._comment
@@ -0,0 +1,11 @@
+[[!comment format=mdwn
+ username="joey"
+ subject="""comment 1"""
+ date="2014-11-03T17:03:34Z"
+ content="""
+It's documentation says it shows files that have been deleted/modified/are
+not checked into git. Not staged files.
+
+So, this is not a bug report, it's a request to make git annex status list
+more files.
+"""]]
diff --git a/doc/todo/metadata_--batch.mdwn b/doc/todo/metadata_--batch.mdwn
new file mode 100644
index 0000000000..b65dc3ef58
--- /dev/null
+++ b/doc/todo/metadata_--batch.mdwn
@@ -0,0 +1,3 @@
+[[!meta author=yoh]]
+
+> [[done]] (using json input) --[[Joey]]
diff --git a/doc/todo/metadata_batch_command_should_allow_changes_by_key.mdwn b/doc/todo/metadata_batch_command_should_allow_changes_by_key.mdwn
new file mode 100644
index 0000000000..d249017288
--- /dev/null
+++ b/doc/todo/metadata_batch_command_should_allow_changes_by_key.mdwn
@@ -0,0 +1,16 @@
+while writing a script that links related files via meta-data i found that meta-data lookup by key works in batch mode, but meta-data storage by key doesn't
+
+my scripts output was lines like:
+
+    {"key": "SHA256E-s13238976--7f9f459b99e36e9d50e6da349258525c9f63a33bd5f5bf5a3284cc0e4bda5fd8.ARW", "fields": {"linked.image": "SHA256Es3983492--0fef9eba38c21629e2ae06bd7f64dae104d8576e3c4d31b2caf7337a1ed8c3a9.JPG"}}
+    {"key": "SHA256E-s3983492--0fef9eba38c21629e2ae06bd7f64dae104d8576e3c4d31b2caf7337a1ed8c3a9.JPG", "fields": {"linked.raw": "SHA256E-s13238976--7f9f459b99e36e9d50e6da349258525c9f63a33bd5f5bf5a3284cc0e4bda5fd8.ARW"}}
+
+and the full pipe was
+
+     git annex info */*.* --json --fast| python3 ./check_has_all_raws.py |git annex metadata --batch --json
+
+to my surprise all i got was the retrial of the existing meta-data instead of the addition of the linked fields
+
+IHO git annex should allow to store metadata in batch mode by key
+
+[[!meta title="metadata --batch parses json strictly, loosen?"]]
diff --git a/doc/todo/metadata_batch_command_should_allow_changes_by_key/comment_1_753ae21e1c43130159681c4e4e0b5cf3._comment b/doc/todo/metadata_batch_command_should_allow_changes_by_key/comment_1_753ae21e1c43130159681c4e4e0b5cf3._comment
new file mode 100644
index 0000000000..f58ea5f1bf
--- /dev/null
+++ b/doc/todo/metadata_batch_command_should_allow_changes_by_key/comment_1_753ae21e1c43130159681c4e4e0b5cf3._comment
@@ -0,0 +1,15 @@
+[[!comment format=mdwn
+ username="RonnyPfannschmidt"
+ avatar="http://cdn.libravatar.org/avatar/c5379a3fe2188b7571858c49f9db63c6"
+ subject="comment 1"
+ date="2018-07-29T21:41:56Z"
+ content="""
+i iterated a bit more on the script,
+i experienced the same issue when i ran with just field and added a few more filds to the output and changed from a string to a list
+
+the working version outputs lines like
+
+    {\"command\": \"set\", \"file\": \"DCIM/23980708/DSC04071.ARW\", \"fields\": {\"linked.image\": [\"SHA256E-s3983492--0fef9eba38c21629e2ae06bd7f64dae104d8576e3c4d31b2caf7337a1ed8c3a9.JPG\"]}}
+    {\"command\": \"set\", \"file\": \"DCIM/23980708/DSC04071.JPG\", \"fields\": {\"linked.raw\": [\"SHA256E-s13238976--7f9f459b99e36e9d50e6da349258525c9f63a33bd5f5bf5a3284cc0e4bda5fd8.ARW\"]}}
+
+"""]]
diff --git a/doc/todo/metadata_batch_command_should_allow_changes_by_key/comment_2_fd80a5425bef1aface37c9836b387fdd._comment b/doc/todo/metadata_batch_command_should_allow_changes_by_key/comment_2_fd80a5425bef1aface37c9836b387fdd._comment
new file mode 100644
index 0000000000..51d11d1c42
--- /dev/null
+++ b/doc/todo/metadata_batch_command_should_allow_changes_by_key/comment_2_fd80a5425bef1aface37c9836b387fdd._comment
@@ -0,0 +1,19 @@
+[[!comment format=mdwn
+ username="joey"
+ subject="""comment 2"""
+ date="2018-10-01T16:09:13Z"
+ content="""
+It does.
+
+	git annex metadata --batch --json
+	{"key": "SHA256E-s30--56c5f90f308696d997525622df4103a31d50ef70d22ceb457d5f87a8b72283cc", "fields":{"author":["bar"]}}
+	{"command":"metadata","note":"author=bar\nauthor-lastchanged=2018-10-01@16-08-50\nlastchanged=2018-10-01@16-08-50\n","success":true,"key":"SHA256E-s30--56c5f90f308696d997525622df4103a31d50ef70d22ceb457d5f87a8b72283cc","file":null,"fields":{"author":["bar"],"lastchanged":["2018-10-01@16-08-50"],"author-lastchanged":["2018-10-01@16-08-50"]}}
+
+Your input is malformed, the value of a field needs to be inside `[]` like `["bar"]` above.
+The same format as it's output.
+Since that part of your json did not parse it thinks you want to query the metadata, not set
+it.
+
+Does your json library actually convert a list containing a
+single value into that value not in a list and vice-versa?
+"""]]
diff --git a/doc/todo/more_efficient_memory_usage_with_git-annex_unused.mdwn b/doc/todo/more_efficient_memory_usage_with_git-annex_unused.mdwn
new file mode 100644
index 0000000000..908ddc2084
--- /dev/null
+++ b/doc/todo/more_efficient_memory_usage_with_git-annex_unused.mdwn
@@ -0,0 +1,6 @@
+While running *git-annex unused* on an annex with tens of thousands of items, *git-annex*'s memory usage ballooned to over 3 gigs and my PC froze. I cannot run *git-annex unused* on this annex because of this issue.
+
+If it's possible, more efficient memory management would prevent this from happening.
+
+> [[done]] -- assuming the memory leak I saw was the same one you saw...
+> --[[Joey]]
diff --git a/doc/todo/more_efficient_memory_usage_with_git-annex_unused/comment_1_e7811b548054d3d6851facb8d3bf8153._comment b/doc/todo/more_efficient_memory_usage_with_git-annex_unused/comment_1_e7811b548054d3d6851facb8d3bf8153._comment
new file mode 100644
index 0000000000..5247bb61cd
--- /dev/null
+++ b/doc/todo/more_efficient_memory_usage_with_git-annex_unused/comment_1_e7811b548054d3d6851facb8d3bf8153._comment
@@ -0,0 +1,25 @@
+[[!comment format=mdwn
+ username="joey"
+ subject="""comment 1"""
+ date="2017-01-31T15:36:58Z"
+ content="""
+You need to provide at least a version number, and ideally enough
+information to reproduce the bug when filing bug reports.
+
+Anyhow, I ran git-annex (HEAD) unused in my big repo, and its memory use
+got up to over 1 gb which is much more than I would expect (should be
+a couple hundred mb max).
+
+The memory growth happens in the stage when it's
+constructing the bloom filter for the keys in the diff between the
+index and other branches. In my big repo, those diffs are quite large;
+eg I have a branch with 70k files and another with 0 files.
+
+I replaced insertMB with noop, so the bloom filters are not really
+populated, and it still uses as much memory. So
+the memory is not being leaked by the bloom filters themselves, but
+is instead being leaked when processing the branch diffs,
+or something like that.
+
+Need to profile to find what's leaking.
+"""]]
diff --git a/doc/todo/more_efficient_memory_usage_with_git-annex_unused/comment_2_1eff4497bf0a3e87dc47a1226c5d4af8._comment b/doc/todo/more_efficient_memory_usage_with_git-annex_unused/comment_2_1eff4497bf0a3e87dc47a1226c5d4af8._comment
new file mode 100644
index 0000000000..df285478cb
--- /dev/null
+++ b/doc/todo/more_efficient_memory_usage_with_git-annex_unused/comment_2_1eff4497bf0a3e87dc47a1226c5d4af8._comment
@@ -0,0 +1,19 @@
+[[!comment format=mdwn
+ username="joey"
+ subject="""comment 2"""
+ date="2017-01-31T20:24:04Z"
+ content="""
+The heap profile has multiple spikes (so not an accumulating memory leak).
+The diff parsing code is indeed what's using so much memory. Looks like
+data is failing to stream through that code and instead the whole
+diff output gets buffered.
+
+Aha.. Git.DiffTree.parseDiffRaw used to return a list, but changed
+in [[!commit 8d124beba8]]
+to a Maybe list in order to avoid being a partial function. But 
+that change destroyed laziness, since the whole input has to be parsed
+in order to determine if Nothing should be returned.
+
+However, fixing that only eliminated part of the spike. There's something
+else keeping data from streaming.
+"""]]
diff --git a/doc/todo/more_efficient_memory_usage_with_git-annex_unused/comment_3_3c712e871ea3cd12916497f2d8152004._comment b/doc/todo/more_efficient_memory_usage_with_git-annex_unused/comment_3_3c712e871ea3cd12916497f2d8152004._comment
new file mode 100644
index 0000000000..2a5130a950
--- /dev/null
+++ b/doc/todo/more_efficient_memory_usage_with_git-annex_unused/comment_3_3c712e871ea3cd12916497f2d8152004._comment
@@ -0,0 +1,10 @@
+[[!comment format=mdwn
+ username="joey"
+ subject="""comment 3"""
+ date="2017-01-31T23:31:03Z"
+ content="""
+Fixed the rest of the streaming problem.
+
+(Also found/fixed an unrelated memory blow up in git annex unused that
+only happened when a large file got checked right into git.)
+"""]]
diff --git a/doc/todo/more_of_diagnostic_information_in_case_of_failures_into_returned_json.mdwn b/doc/todo/more_of_diagnostic_information_in_case_of_failures_into_returned_json.mdwn
new file mode 100644
index 0000000000..3e9c30970e
--- /dev/null
+++ b/doc/todo/more_of_diagnostic_information_in_case_of_failures_into_returned_json.mdwn
@@ -0,0 +1,22 @@
+ATM I am experiencing sporadic failures of the batched  git annex addurl call -- seems to report failure (success: False) once in a while, but succeeds on a retry:
+
+[[!format sh """
+(Pdb) p url
+'http://openneuro.s3.amazonaws.com/ds000001/ds000001_R1.1.0/uncompressed/sub016/BOLD/task001_run003/QA/QA_report.pdf?versionId=null'
+
+(Pdb) p out_json
+{u'note': u'from datalad', u'command': u'addurl', u'file': u'ds000001_R1.1.0/uncompressed/sub016/BOLD/task001_run003/QA/QA_report.pdf', u'success': False}
+
+(Pdb) up
+> /home/yoh/proj/datalad/datalad/datalad/support/gitrepo.py(210)newfunc()
+-> return func(self, file_new, *args, **kwargs)
+
+(Pdb) func(self, file_new, *args, **kwargs)
+{u'note': u'from datalad', u'file': u'ds000001_R1.1.0/uncompressed/sub016/BOLD/task001_run003/QA/QA_report.pdf', u'command': u'addurl', u'key': u'MD5E-s1191419--cb4efab8104b5117f64b58ee6d6a79ba.pdf', u'success': True}
+"""]]
+
+besides me blindly trying to re-run it e.g. 3 times and only then declare total failure, I wondered if json output could provide more information (if any known) about the failure... e.g. if a custom remote crashed/errorred (I guess the case here due to "from datalad") -- what was stderr/exit code for that process if crashed/ERROR msg... if wget -- what was stderr there
+
+[[!meta name=yoh]]
+
+> Switched to curl with -sS in --json mode. [[done]] I suppose. --[[Joey]]
diff --git a/doc/todo/more_of_diagnostic_information_in_case_of_failures_into_returned_json/comment_1_0eeb859b57d4dc8a3c9c9c3c4f70bb45._comment b/doc/todo/more_of_diagnostic_information_in_case_of_failures_into_returned_json/comment_1_0eeb859b57d4dc8a3c9c9c3c4f70bb45._comment
new file mode 100644
index 0000000000..80dfe504b3
--- /dev/null
+++ b/doc/todo/more_of_diagnostic_information_in_case_of_failures_into_returned_json/comment_1_0eeb859b57d4dc8a3c9c9c3c4f70bb45._comment
@@ -0,0 +1,16 @@
+[[!comment format=mdwn
+ username="joey"
+ subject="""comment 1"""
+ date="2017-02-20T18:50:18Z"
+ content="""
+Probably wget is just failing to download the url sometimes.
+Eg, `git annex addurl http://localhost/dne` fails with the same not useful
+output.
+
+wget is run with -q, which is the only way to turn off all its informational
+messages, but unfortunately that also turns off display of HTTP error messages.
+
+Using -nv instead of -q would display HTTP errors,
+but also 1 extra line of output once the download is complete.
+I suppose that's worth the trade-off.
+"""]]
diff --git a/doc/todo/more_of_diagnostic_information_in_case_of_failures_into_returned_json/comment_2_82b851629c695084cbf62e2b636bcc91._comment b/doc/todo/more_of_diagnostic_information_in_case_of_failures_into_returned_json/comment_2_82b851629c695084cbf62e2b636bcc91._comment
new file mode 100644
index 0000000000..2455f6f660
--- /dev/null
+++ b/doc/todo/more_of_diagnostic_information_in_case_of_failures_into_returned_json/comment_2_82b851629c695084cbf62e2b636bcc91._comment
@@ -0,0 +1,21 @@
+[[!comment format=mdwn
+ username="joey"
+ subject="""comment 2"""
+ date="2017-02-20T19:15:18Z"
+ content="""
+It seems there is no way to get only errors on
+stderr with wget; the choice is between no output, and a mixture of errors
+and informational messages on stderr. In --json or --quiet mode, only
+errors should be output to stderr.
+
+In general, the --json output does include a "note" with any
+available message about why an operation failed. 
+
+It would not be hard to use a HTTP library and propagate the HTTP errors
+into the json "note", but it might be hard to get resumption of partial
+downloads to work as well with a HTTP library as it works with wget/curl.
+
+What we could do is use curl in preference to wget in json mode; 
+curl -s -S avoids all progress etc output and displays the
+http errors to stderr.
+"""]]
diff --git a/doc/todo/multicast___34__broadcasting__34___of_content_on_local_net.mdwn b/doc/todo/multicast___34__broadcasting__34___of_content_on_local_net.mdwn
new file mode 100644
index 0000000000..bd34770724
--- /dev/null
+++ b/doc/todo/multicast___34__broadcasting__34___of_content_on_local_net.mdwn
@@ -0,0 +1,10 @@
+Hi Joey,
+
+Although I haven't remembered that "hash thing" to perform the job, we looked around for other ways to accomplish the mission:  quickly/simultaneously distribute annexed files to multiple hosts on the same network.  So, there are such tools as uftp to efficiently multicast files to multiple recipients.  Initial setup takes a bit but I wondered how feasible it could be to e.g. establish some "custom annex remote" (if to avoid coming up with a new command eg. "annex serve") so e.g. I could e.g. `git annex copy --to=udp-multicast ` on the host A which has all the keys, and then run `git annex get --from=udp-multicast` on the clients (B) which want to get it all.  To make it even more efficient, A could may be provide/announce on the local net a service to receive requests for desired keys to be transmitted.  But even if it just multicasts all the content of the current tree, while those clients suck them all in, it could be super useful.
+
+What do you think?
+
+[[!meta name=yoh]]
+
+> [[done]]! I've only tested it with sender and receiver on the same
+> laptop, but it seems to work. --[[Joey]] 
diff --git a/doc/todo/multicast___34__broadcasting__34___of_content_on_local_net/comment_1_5353dc46ccecc6077f2643d281d58f99._comment b/doc/todo/multicast___34__broadcasting__34___of_content_on_local_net/comment_1_5353dc46ccecc6077f2643d281d58f99._comment
new file mode 100644
index 0000000000..a38457d533
--- /dev/null
+++ b/doc/todo/multicast___34__broadcasting__34___of_content_on_local_net/comment_1_5353dc46ccecc6077f2643d281d58f99._comment
@@ -0,0 +1,92 @@
+[[!comment format=mdwn
+ username="joey"
+ subject="""comment 1"""
+ date="2017-03-30T16:51:24Z"
+ content="""
+Using a remote for this seems problimatic, because the remote is not
+pointing at a single well-defined data store, but instead whatever peers
+happen to exist on the network.
+
+For one, `copy --to=udp-multicast` would not try to send files that it thinks
+are already located in that remote. I suppose this could be dealt with by
+making the transfers always seem to fail, so it never thinks that the
+multicast remote has any files.
+
+But then, `copy --from=udp-multicast` would not try to receive files,
+unless it thinks they're in that remote. And we just established it should
+not think any files are in that remote. So that's a problem.
+
+Also, the copy from/to would need to be operating on the same file at the
+same time, which seems problimatic. If a receiving git-annex is a little
+slower than the sender, or is operating on a slightly different set of
+files, it would then miss a file being broadcast by the sender.
+
+These issues seem to point to this needing to use some other, more
+special-purpose commands for muticast.
+
+----
+
+It probably needs encryption, both for privacy and to ensure that files
+are being received from the sender you intended, and not someone else
+who might be broadcasting the contents of a different repository.
+
+Here's how to set up encryption and authentication with uftp,
+so that both client and server actually encrypt and check that they're
+talking with a known entity. It took a while to figure out.
+
+Client:
+
+	uftp_keymgt -g rsa:512 ~/client_key
+	# Parse the fingerprint value from its output; does not
+	# seem to be a better way, except perhaps using openssl to examine
+	# the key file. This is CLIENT_FINGERPRINT
+
+	# Pick a UID for the client. This is an 8-diget hex number,
+	# which needs to be unique across clients. Default is based on IP
+	# adddres, but for git-annex it could be derived from the git-annex
+	# UUID.	This is CLIENT_UID.
+
+Server:
+
+	uftp_keymgt -g rsa:512 ~/servant_key
+	# Parse the SERVER_FINGERPRINT from its output.
+	
+	# Pick a SERVER_UID for the server.
+
+Client:
+
+	# create a file "authlist" that contains "$SERVER_UID|$SERVER_FINGERPRINT"
+
+	uftpd -E -d -D/tmp/xx -k ~/client_key -U $CLIENT_UID -S '@authlist'
+
+Server:
+
+	# create file "authlist" that contains "$CLIENT_UID|$CLIENT_FINGERPRINT"
+	# lines for each allowed client
+
+	uftp -c -Y aes256-cbc -h sha1 -k ~/server_key -U $SERVER_UID -H '@authlist' file_to_send
+
+----
+
+Notice that the client and server UID and key generation steps above 
+are the same. So, a command like `git annex multicast --gen-address`
+could be run on both the server and clients, and could store
+the addresses in the git-annex branch. 
+The uftp authlist file would be derived from all known such addresses.
+
+(Unlike `p2p --gen-address`, where the address allows connecting with
+and authentication with a remote over TOR, these multicast addresses
+are safe to make public.)
+
+The process of setting up a multicast classroom would then be:
+
+1. Teacher runs `git annex multicast --gen-address; git annex sync`
+2. Students each run `git annex multicast --gen-address; git annex sync`
+3. Teacher runs `git annex sync` once all the students have generated addresses.  
+   (Now the students all have received the teacher's address, and the teacher
+   has received all the student's addresses.)
+4. Students each run `git annex multicast-receive`, which listens for 
+   files going by and stores them.
+5. Once the students are listening (*ahem*), teacher runs
+   `git annex multicast-send [file]` to distribute files.
+"""]]
diff --git a/doc/todo/need_to_remove_remoteGitConfig_for_checkuuid_support.mdwn b/doc/todo/need_to_remove_remoteGitConfig_for_checkuuid_support.mdwn
new file mode 100644
index 0000000000..215c27cb81
--- /dev/null
+++ b/doc/todo/need_to_remove_remoteGitConfig_for_checkuuid_support.mdwn
@@ -0,0 +1,12 @@
+annex-checkuuid=false prevents the git config of a remote from being read.
+So, the remoteGitConfig will be an empty config when that's set.
+
+I've mostly removed uses of remoteGitConfig, but there are two in
+Remote.Git, which are needed for annexDifferences.
+So, `annex.tune.*` config the remote won't be honored when
+annex-checkuuid=false.
+
+The best thing would be to remove remoteGitConfig, to avoid such problems
+in the future. --[[Joey]]
+
+> [[done]] --[[Joey]]
diff --git a/doc/todo/network_test_suite.mdwn b/doc/todo/network_test_suite.mdwn
new file mode 100644
index 0000000000..337dbaf11f
--- /dev/null
+++ b/doc/todo/network_test_suite.mdwn
@@ -0,0 +1,28 @@
+`git annex test` is not allowed to make network connections. (It does do some
+mocking to eg, test ssh special remotes without actually using ssh.)
+
+`git annex testremote` is allowed to use the network, but only when the
+user has set up a remote to test. Some parts of git-annex involve network
+connections, but not as part of accessing a remote, or are specific to a
+particular special remote. There outght to be a test suite for that stuff
+as well.
+
+A motivating example is `git annex p2p --pair`, which was 
+[[broken for years due to lack of testing|apparent_regression_in_git_annex_p2p_--pair_usage_of_magic_wormhole_invocaction]]. 
+Fully testing that would involve setting up tor hidden services which is far
+too extreme for a test suite (and needs root), but it should at least be
+possible to test the interface to magic-wormhole, allowing it to make a
+connection to the wormhole server.
+
+More examples: 
+
+* git-annex multicast (would need two hosts on a lan ideally but
+  could at least try to multicast back to the same host)
+* git-annex addurl and git-annex importfeed with http urls.
+  And testing the various security configs that affect them.
+
+This could be done as a new command or a flag to `git annex test`.
+In any case, the new test suite would need to be run somewhere;
+running it on at least some of the autobuilders might be a good way.
+
+--[[Joey]] 
diff --git a/doc/todo/openwrt_package.mdwn b/doc/todo/openwrt_package.mdwn
new file mode 100644
index 0000000000..70a4ae03fb
--- /dev/null
+++ b/doc/todo/openwrt_package.mdwn
@@ -0,0 +1,6 @@
+hi
+
+recently i have installed openwrt on my mikrotik routerboard. i am verry suprised how well it works. it lacks git-annex package. openwrt has git and i can install it.
+
+how can i build one on a mips arch ?
+is it possible to build multiple architecture standalone binaries ?
diff --git a/doc/todo/openwrt_package/comment_1_100d76109e04bc43979775d71b4152ac._comment b/doc/todo/openwrt_package/comment_1_100d76109e04bc43979775d71b4152ac._comment
new file mode 100644
index 0000000000..78029694a5
--- /dev/null
+++ b/doc/todo/openwrt_package/comment_1_100d76109e04bc43979775d71b4152ac._comment
@@ -0,0 +1,12 @@
+[[!comment format=mdwn
+ username="http://joeyh.name/"
+ ip="206.74.132.139"
+ subject="comment 1"
+ date="2014-02-06T17:26:58Z"
+ content="""
+I would be quite happy if someone took care of adding git-annex to openwrt.
+
+I don't have time to personally handle packaging for different linux distributions myself. What I could do is add mips builds of git-annex to the existing standalone linux builds. These would need to be built the same way the arm builds are done, using a Debian chroot and qemu to run tools from it. This is rather a lot of work for me to set up, and I don't know if I'd have to do it for both little and big endian mips.
+
+Also, it seems that Debian does not currently have a working haskell toolchain for mips. Which may well mean that ghc is not in a working state on mips at all.
+"""]]
diff --git a/doc/todo/openwrt_package/comment_2_2cb7dd4c0cc4413a4588b13cf7700de2._comment b/doc/todo/openwrt_package/comment_2_2cb7dd4c0cc4413a4588b13cf7700de2._comment
new file mode 100644
index 0000000000..d99fa13f1d
--- /dev/null
+++ b/doc/todo/openwrt_package/comment_2_2cb7dd4c0cc4413a4588b13cf7700de2._comment
@@ -0,0 +1,9 @@
+[[!comment format=mdwn
+ username="https://www.google.com/accounts/o8/id?id=AItOawkNE-H4vEcbcGndxq5daT8qUb7yIf7r1OE"
+ nickname="Łukasz"
+ subject="comment 2"
+ date="2014-02-11T21:05:00Z"
+ content="""
+if debian does not have working toolchain for mips then there is no way to port it into mips machines.
+am i correct ?
+"""]]
diff --git a/doc/todo/openwrt_package/comment_3_5ba8a325a683ff543d81a366c873070d._comment b/doc/todo/openwrt_package/comment_3_5ba8a325a683ff543d81a366c873070d._comment
new file mode 100644
index 0000000000..9c9157d18f
--- /dev/null
+++ b/doc/todo/openwrt_package/comment_3_5ba8a325a683ff543d81a366c873070d._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="http://joeyh.name/"
+ ip="209.250.56.172"
+ subject="comment 3"
+ date="2014-02-11T21:39:34Z"
+ content="""
+Actually, debian stable does still have ghc building for mips/mipsel. It's just newer versions that have failed to build and nobody has fixed it yet.
+"""]]
diff --git a/doc/todo/openwrt_package/comment_4_132e67b34f9c616217e037e4ecac70a4._comment b/doc/todo/openwrt_package/comment_4_132e67b34f9c616217e037e4ecac70a4._comment
new file mode 100644
index 0000000000..6e5411dc36
--- /dev/null
+++ b/doc/todo/openwrt_package/comment_4_132e67b34f9c616217e037e4ecac70a4._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="joey"
+ subject="""comment 4"""
+ date="2015-01-13T20:39:53Z"
+ content="""
+I checked, and the current status is that mips and mipsel in Debian have
+up-to-date builds of ghc and git-annex.
+"""]]
diff --git a/doc/todo/option_to_add_user-specified_string_to_key.mdwn b/doc/todo/option_to_add_user-specified_string_to_key.mdwn
new file mode 100644
index 0000000000..140a0a0f8f
--- /dev/null
+++ b/doc/todo/option_to_add_user-specified_string_to_key.mdwn
@@ -0,0 +1,13 @@
+Add a config (and gitattributes) option annex.userkeystring , such that git-annex-add (and calckey) will include this string in the key.  If the string is 'UUID' then a uuid (or shorter random string) will be included instead.
+
+From [[todo/support_longer_file_extensions]]:
+
+"The way git-annex picks extensions doesn't need to be stable across all versions of git-annex, because it's only done when initially adding a file, and then the key, with whatever extension, is added as-is and git-annex does not care about the extension thereafter."
+
+Then, for an MD5E key, the userkeystring can be included before the extension: MD5E-s0--d41d8cd98f00b204e9800998ecf8427e.USERKEYSTRING.txt .
+Cleaner would be to add a field to the key, as in MD5E-s0-uUSERKEYSTRING--d41d8cd98f00b204e9800998ecf8427e.txt , if that wouldn't break compatibility.
+
+
+This enables attaching metadata not to file contents, but to the file itself; or partitioning keys (and therefore key metadata) into namespaces.   The downside is some loss of 
+deduplication.  This loss may be acceptable.  The loss can be mitigated for local repo and non-special remotes: after storing an object with e.g. MD5 d41d8cd98f00b204e9800998ecf8427e under .git/annex/objects, check if there is a symlink .git/annex/contenthash/d41d8cd98f00b204e9800998ecf8427e ; if not, make this a symlink to the object just stored; if yes,
+erase the object just stored, and hardlink the symlink's target instead.
diff --git a/doc/todo/output_of_wanted___40__and_possibly_group_etc__41___should_not_be_polluted_with___34__informational__34___messages.mdwn b/doc/todo/output_of_wanted___40__and_possibly_group_etc__41___should_not_be_polluted_with___34__informational__34___messages.mdwn
new file mode 100644
index 0000000000..b2849f3f10
--- /dev/null
+++ b/doc/todo/output_of_wanted___40__and_possibly_group_etc__41___should_not_be_polluted_with___34__informational__34___messages.mdwn
@@ -0,0 +1,13 @@
+[[!format sh """
+$> git  clone http://datasets.datalad.org/labs/haxby/raiders/.git ; cd raiders; git annex wanted origin 2>/dev/null     
+Cloning into 'raiders'...
+(merging origin/git-annex into git-annex...)
+(recording state in git...)
+not metadata=distribution-restrictions=*
+"""]]
+
+so it is necessary to avoid considering all the merging and recording messages, complicating using wanted in the scripts etc
+
+[[!meta author=yoh]]
+
+> [[fixed|done]] --[[Joey]]
diff --git a/doc/todo/output_of_wanted___40__and_possibly_group_etc__41___should_not_be_polluted_with___34__informational__34___messages/comment_1_bfbf428da46eaea52cc54878f119512a._comment b/doc/todo/output_of_wanted___40__and_possibly_group_etc__41___should_not_be_polluted_with___34__informational__34___messages/comment_1_bfbf428da46eaea52cc54878f119512a._comment
new file mode 100644
index 0000000000..60a7036552
--- /dev/null
+++ b/doc/todo/output_of_wanted___40__and_possibly_group_etc__41___should_not_be_polluted_with___34__informational__34___messages/comment_1_bfbf428da46eaea52cc54878f119512a._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="CandyAngel"
+ avatar="http://cdn.libravatar.org/avatar/15c0aade8bec5bf004f939dd73cf9ed8"
+ subject="comment 1"
+ date="2017-02-16T14:10:56Z"
+ content="""
+In your example, it is polluted by these messages because git-annex is initialising the repository as a git-annex. There should be a 'git annex init' after the 'cd', before you run git-annex commands (e.g. wanted).
+"""]]
diff --git a/doc/todo/output_of_wanted___40__and_possibly_group_etc__41___should_not_be_polluted_with___34__informational__34___messages/comment_2_c43e838b750448fe4d1519e72c43242f._comment b/doc/todo/output_of_wanted___40__and_possibly_group_etc__41___should_not_be_polluted_with___34__informational__34___messages/comment_2_c43e838b750448fe4d1519e72c43242f._comment
new file mode 100644
index 0000000000..dbd8edbbe4
--- /dev/null
+++ b/doc/todo/output_of_wanted___40__and_possibly_group_etc__41___should_not_be_polluted_with___34__informational__34___messages/comment_2_c43e838b750448fe4d1519e72c43242f._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="yarikoptic"
+ avatar="http://cdn.libravatar.org/avatar/f11e9c84cb18d26a1748c33b48c924b4"
+ subject="comment 2"
+ date="2017-02-16T15:54:18Z"
+ content="""
+Thank you CandyAngel for the feedback.  I do know how to avoid it, but my point is that git-annex here uses stdout for both \"log messages\" and the output of the log command. IIRC some other commands, and git uses stderr for log/informational messages in cases where stdout is used for communicating output
+"""]]
diff --git a/doc/todo/output_of_wanted___40__and_possibly_group_etc__41___should_not_be_polluted_with___34__informational__34___messages/comment_3_e006b8efab713d4965316c9846ec437c._comment b/doc/todo/output_of_wanted___40__and_possibly_group_etc__41___should_not_be_polluted_with___34__informational__34___messages/comment_3_e006b8efab713d4965316c9846ec437c._comment
new file mode 100644
index 0000000000..7174ab82a9
--- /dev/null
+++ b/doc/todo/output_of_wanted___40__and_possibly_group_etc__41___should_not_be_polluted_with___34__informational__34___messages/comment_3_e006b8efab713d4965316c9846ec437c._comment
@@ -0,0 +1,18 @@
+[[!comment format=mdwn
+ username="joey"
+ subject="""comment 3"""
+ date="2017-02-16T16:53:30Z"
+ content="""
+This can happen at other times than repository auto-init too.
+
+Generally plumbing commands just turn off all such messages very early,
+but in this case, the command has one mode where it's supposed to get data,
+which is plumbing-like, and another mode where it sets data, which is
+supposed to display normal messages about what it's doing. So it didn't
+turn messages off until after parsing the command line, which is too late.
+
+Affected commands: config group groupwanted numcopies schedule wanted required
+
+Fairly sure that's a complete set, at least it's all the commands with
+both a get and a set mode.
+"""]]
diff --git a/doc/todo/p2p_protocol_flag_days.mdwn b/doc/todo/p2p_protocol_flag_days.mdwn
new file mode 100644
index 0000000000..6dc90727e5
--- /dev/null
+++ b/doc/todo/p2p_protocol_flag_days.mdwn
@@ -0,0 +1,22 @@
+Some things to do with the [[design/P2P_protocol]]
+are works in progress, needing a future flag day to complete.
+
+## VERSION over tor
+
+Old versions of git-annex, before 6.20180312, which speak the P2P protocol
+over tor, don't support VERSION, and attempting to negotiate a version
+will cause the server to hang up the connection. To deal with this
+historical bug, the version is not currently negotiated when using the
+protocol over tor. At some point in the future, when all peers can be
+assumed to be upgraded, this should be changed.
+
+## git-annex-shell fallbacks
+
+When using git-annex-shell p2pio, git-annex assumes that if it exits 1,
+it does not support that, and falls back to the old sendkey/rerecvkey,
+etc.
+
+At some point in the future, once all git-annex and git-annex-shell
+can be assumed to be upgraded to 6.20180312, this fallback can be removed.
+It will allows removing a lot of code from git-annex-shell and a lot of
+fallback code from Remote.Git.
diff --git a/doc/todo/parallel_get.mdwn b/doc/todo/parallel_get.mdwn
new file mode 100644
index 0000000000..fb3f8d0987
--- /dev/null
+++ b/doc/todo/parallel_get.mdwn
@@ -0,0 +1,85 @@
+Wish: `git annex get [files] -jN` should run up to N downloads of files
+concurrently.
+
+This can already be done by just starting up N separate git-annex
+processes all trying to get the same files. They'll coordinate themselves
+to avoid downloading the same file twice.
+
+But, the output of concurrent git annex get's in a single teminal is a
+mess.
+
+It would be nice to have something similar to docker's output when fetching
+layers of an image. Something like:
+
+	get foo1 ok
+	get foo2 ok
+	get foo3 -> 5% 100 KiB/s
+	get foo4 -> 3% 90 KiB/s
+	get foo5 -> 20% 1 MiB/s
+
+Where the bottom N lines are progress displays for the downloads that are
+currently in progress. When a download finishes, it can scroll up the
+screen with "ok".
+
+	get foo1 ok
+	get foo2 ok
+	get foo5 ok
+	get foo3 -> 5% 100 KiB/s
+	get foo4 -> 3% 90 KiB/s
+	get foo6 -> 0% 110 Kib/S
+
+This display could perhaps be generalized for other concurrent actions.
+For example, drop:
+
+	drop foo1 ok
+	drop foo2 failed
+	  Not enough copies ...
+	drop foo3 -> (checking r1...)
+	drop foo4 -> (checking r2...)
+
+But, do get first.
+
+Pain points: 
+
+1. Currently, git-annex lets tools like rsync and wget display their own
+   progress. This makes sense for the single-file at a time get, because
+   rsync can display better output than just a percentage. (This is especially
+   the case with aria2c for torrents, which displays seeder/leecher info in
+   addition to percentage.)
+
+   But in multi-get mode, the progress display would be simplified. git-annex
+   can already get percent done information, either as reported by individiual
+   backends, or by falling back to polling the file as it's downloaded.
+
+2. The mechanics of updating the screen for a multi-line progress output
+   require some terminal handling code. Using eg, curses, in a mode that
+   doesn't take over the whole screen display, but just moves the cursor
+   up to the line for the progress that needs updating and redraws that
+   line. Doing this portably is probably going to be a pain, especially
+   I have no idea if it can be done on Windows.
+
+   An alternative would be a display more like apt uses for concurrent
+   downloads, all on one line:
+
+	get foo1 ok
+	get foo2 ok
+	get [foo3 -> 5% 100 KiB/s] [foo4 -> 3% 90 KiB/s] [foo5 -> 20% 1 MiB/s]
+
+   The problem with that is it has to avoid scrolling off the right
+   side, so it probably has to truncate the line. Since filenames
+   are often longer than "fooN", it probably has to elipsise the filename.
+   This approach is just not as flexible or nice in general.
+
+See also: [[parallel_possibilities]]
+
+> I am looking at using the ascii-progress library for this.
+> It has nice support for multiple progress bars, and is portable.
+> I have filed 7 issues on it, around 4 of which need to get fixed before
+> it's suitable for git-annex to use.. --[[Joey]]
+
+>> `git annex get -JN` works now, but lacks any progress display.
+>> Waiting on some updates to ascii-progress. --[[Joey]]
+
+>>> Wrote concurrent-output; [[done]] --[[Joey]]
+
+[[!meta author=yoh]]
diff --git a/doc/todo/parallel_possibilities.mdwn b/doc/todo/parallel_possibilities.mdwn
new file mode 100644
index 0000000000..9c0e69e294
--- /dev/null
+++ b/doc/todo/parallel_possibilities.mdwn
@@ -0,0 +1,13 @@
+One of my reasons for using haskell was that it provides the possibility of
+some parallell processing. Although since git-annex hits the filesystem
+heavily and mostly runs other git commands, maybe not a whole lot.
+
+Anyway, each git-annex command is broken down into a series of independant 
+actions, which has some potential for parallelism.
+
+Each action has 3 distinct phases, basically "check", "perform", and
+"cleanup". The perform actions are probably parellizable; the cleanup may be
+(but not if it has to run git commands to stage state; it can queue
+commands though); the check should be easily parallelizable, although they
+may access the disk or run minor git query commands, so would probably not
+want to run too many of them at once.
diff --git a/doc/todo/parallel_possibilities/comment_1_d8e34fc2bc4e5cf761574608f970d496._comment b/doc/todo/parallel_possibilities/comment_1_d8e34fc2bc4e5cf761574608f970d496._comment
new file mode 100644
index 0000000000..4aceb3abd3
--- /dev/null
+++ b/doc/todo/parallel_possibilities/comment_1_d8e34fc2bc4e5cf761574608f970d496._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="https://www.google.com/accounts/o8/id?id=AItOawkptNW1PzrVjYlJWP_9e499uH0mjnBV6GQ"
+ nickname="Christian"
+ subject="comment 1"
+ date="2011-04-08T12:41:43Z"
+ content="""
+I also think, that fetching keys via rsync can be done by one rsync process, when the keys are fetched from one host. This would avoid establishing a new TCP connection for every file.
+"""]]
diff --git a/doc/todo/parallel_possibilities/comment_2_adb76f06a7997abe4559d3169a3181c3._comment b/doc/todo/parallel_possibilities/comment_2_adb76f06a7997abe4559d3169a3181c3._comment
new file mode 100644
index 0000000000..6ecce52c42
--- /dev/null
+++ b/doc/todo/parallel_possibilities/comment_2_adb76f06a7997abe4559d3169a3181c3._comment
@@ -0,0 +1,12 @@
+[[!comment format=mdwn
+ username="http://ertai.myopenid.com/"
+ nickname="npouillard"
+ subject="comment 2"
+ date="2011-05-20T20:14:15Z"
+ content="""
+I agree with Christian.
+
+One should first make a better use of connections to remotes before exploring parallel possibilities. One should pipeline the requests and answers.
+
+Of course this could be implemented using parallel&concurrency features of Haskell to do this.
+"""]]
diff --git a/doc/todo/parallel_possibilities/comment_3_145fb974f45da99b7d4b117a3699cccf._comment b/doc/todo/parallel_possibilities/comment_3_145fb974f45da99b7d4b117a3699cccf._comment
new file mode 100644
index 0000000000..0d646a7a80
--- /dev/null
+++ b/doc/todo/parallel_possibilities/comment_3_145fb974f45da99b7d4b117a3699cccf._comment
@@ -0,0 +1,12 @@
+[[!comment format=mdwn
+ username="http://joeyh.name/"
+ ip="4.154.4.90"
+ subject="comment 3"
+ date="2013-07-17T19:59:50Z"
+ content="""
+Note that git-annex now uses locks to communicate among multiple processes, so it's now possible to eg run two `git annex get` processes, and one will skip over the file the other is downloading and go on to the next file, and so on. 
+
+This is an especially nice speedup when downloading encrypted data, since the decryption of one file will tend to happen while the other process is downloading the next file (assuming files of approximately the same size, and that decryption takes approxiately as long as downloading).
+
+The only thing preventing this being done by threads in one process, enabled by a -jN option, is that the output would be a jumbled mess.
+"""]]
diff --git a/doc/todo/parallel_possibilities/comment_4_229af3089d01eef2bb5a9a9c0610a73c._comment b/doc/todo/parallel_possibilities/comment_4_229af3089d01eef2bb5a9a9c0610a73c._comment
new file mode 100644
index 0000000000..46e399ffef
--- /dev/null
+++ b/doc/todo/parallel_possibilities/comment_4_229af3089d01eef2bb5a9a9c0610a73c._comment
@@ -0,0 +1,23 @@
+[[!comment format=mdwn
+ username="joey"
+ subject="""comment 4"""
+ date="2015-11-04T21:00:02Z"
+ content="""
+Now, many git-annex commands support -Jn, the output is not a jumbled mess
+thanks to the concurrent-output library.
+
+At this point all I see that needs doing is:
+
+* Maybe support -Jn in more commands. Just needs changing a few lines of
+  code and testing each.
+
+* It would be nice to be able to run cleanup actions in the "background",
+  after a command has otherwise succeeded, even when -Jn is not used.
+  In particular, when getting files, their checksum is verified after
+  download. That would nicely parellize with the next file being
+  downloaded.
+
+  This could be implemented also using concurrent-output, but it would then
+  have to drive the display even when -J is not used. I'm not yet sure
+  enough about it to use it by default.
+"""]]
diff --git a/doc/todo/per-remote_metadata_needs_to_be_cleaned_in_dropdead.mdwn b/doc/todo/per-remote_metadata_needs_to_be_cleaned_in_dropdead.mdwn
new file mode 100644
index 0000000000..394f2bafd7
--- /dev/null
+++ b/doc/todo/per-remote_metadata_needs_to_be_cleaned_in_dropdead.mdwn
@@ -0,0 +1,4 @@
+The newly added per-remote metadata log files need to be scrubbed clean of
+dead remotes during a transition. --[[Joey]]
+
+> [[done]] --[[Joey]]
diff --git a/doc/todo/prevent_directly_printing_to_stderr_error_from_an_attempt_to_download_remote___47__config_file.mdwn b/doc/todo/prevent_directly_printing_to_stderr_error_from_an_attempt_to_download_remote___47__config_file.mdwn
new file mode 100644
index 0000000000..8075b28851
--- /dev/null
+++ b/doc/todo/prevent_directly_printing_to_stderr_error_from_an_attempt_to_download_remote___47__config_file.mdwn
@@ -0,0 +1,39 @@
+ATM git-annex init on a new clone would try to sense the remote regarding git-annex support and would try to fetch its /config.  If that fails it would announce that remote ignored for annex.
+
+[[!format sh """
+...
+Switched to branch 'adjusted/master(unlocked)'
+download failed: Not Found
+
+  Remote origin not usable by git-annex; setting annex-ignore
+
+"""]]
+and that download failed, thanks to --debug output comes from
+[[!format sh """
+[2018-09-28 12:52:12.541757374] Request {
+  host                 = "github.com"
+  port                 = 443
+  secure               = True
+  requestHeaders       = [("Range","bytes=0-"),("Accept-Encoding","identity")]
+  path                 = "/psychoinformatics-de/studyforrest-data-structural/config"
+  queryString          = ""
+  method               = "GET"
+  proxy                = Nothing
+  rawBody              = False
+  redirectCount        = 10
+  responseTimeout      = ResponseTimeoutDefault
+  requestVersion       = HTTP/1.1
+}
+
+download failed: Not Found
+
+  Remote origin not usable by git-annex; setting annex-ignore
+
+"""]]
+
+IMHO that "download failed: " message output should be provided only in `--debug` mode.  Otherwise, if exposed to naive user it is an uninformative alarm, which (if he/she knows) is clarified by the follow up message ("Remote origin not usable ...")
+
+
+[[!meta author=yoh]]
+
+ref: [https://github.com/datalad/datalad/pull/2881#issue-218977359](https://github.com/datalad/datalad/pull/2881#issue-218977359)
diff --git a/doc/todo/prevent_unwanted_init.mdwn b/doc/todo/prevent_unwanted_init.mdwn
new file mode 100644
index 0000000000..b452a4735d
--- /dev/null
+++ b/doc/todo/prevent_unwanted_init.mdwn
@@ -0,0 +1,10 @@
+My $HOME is a git repo, but I don't want to use git-annex in that repo,
+only in sub-repos. If I'm in ~/tmp/foo/ and forgot to git init, git annex
+init initializes my $HOME repo, and I have to go clean it up which is
+annoying. --[[Joey]]
+
+This could be a git configuration setting, or it could be something
+checked into the repo. Either might make sense depending on the scope
+in which one wants to prevent the accidental init.
+
+> [[done]] using .noannex file. --[[Joey]]
diff --git a/doc/todo/prevent_unwanted_init/comment_1_d3f2f6b7b56a57c70614114f88baaec6._comment b/doc/todo/prevent_unwanted_init/comment_1_d3f2f6b7b56a57c70614114f88baaec6._comment
new file mode 100644
index 0000000000..95b0d98a18
--- /dev/null
+++ b/doc/todo/prevent_unwanted_init/comment_1_d3f2f6b7b56a57c70614114f88baaec6._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="yarikoptic"
+ avatar="http://cdn.libravatar.org/avatar/f11e9c84cb18d26a1748c33b48c924b4"
+ subject="comment 1"
+ date="2017-12-11T19:41:35Z"
+ content="""
++1 for this feature.  For our usecase I would vote for \"something checked into the repo\", e.g. a file `.noannex`, optionally containing a message to be displayed whenever `annex init` detects its presence and refuses to proceed forward.  Message could then provide instructions on what repository authors advise to do (e.g. establish submodules with git-annex'ed repositories).
+"""]]
diff --git a/doc/todo/prevent_unwanted_init/comment_2_019ab8f24611f668ab57cd771564d6de._comment b/doc/todo/prevent_unwanted_init/comment_2_019ab8f24611f668ab57cd771564d6de._comment
new file mode 100644
index 0000000000..bc97f1400e
--- /dev/null
+++ b/doc/todo/prevent_unwanted_init/comment_2_019ab8f24611f668ab57cd771564d6de._comment
@@ -0,0 +1,10 @@
+[[!comment format=mdwn
+ username="joey"
+ subject="""comment 2"""
+ date="2017-12-13T16:41:26Z"
+ content="""
+Displaying the file content is a useful idea.
+(There is some possible security exposure with eg escape codes,
+but I think generally we accept those risks when using git repos
+and running "ls" or "cat" etc.)
+"""]]
diff --git a/doc/todo/prevent_unwanted_init/comment_3_08bdae4e0c07b7833089b8b40472c3b0._comment b/doc/todo/prevent_unwanted_init/comment_3_08bdae4e0c07b7833089b8b40472c3b0._comment
new file mode 100644
index 0000000000..7e105341f5
--- /dev/null
+++ b/doc/todo/prevent_unwanted_init/comment_3_08bdae4e0c07b7833089b8b40472c3b0._comment
@@ -0,0 +1,11 @@
+[[!comment format=mdwn
+ username="joey"
+ subject="""comment 3"""
+ date="2017-12-13T16:52:22Z"
+ content="""
+Of course, `git annex uninit` should make it easy to undo an accidental
+init if you know you don't want to use git-annex in the repo.
+
+So, this idea must be more about repos whose users don't know there's a
+policy reason not to use git-annex in them.
+"""]]
diff --git a/doc/todo/prevent_unwanted_init/comment_4_e73327b773db8ba8720af44ea46ffa2f._comment b/doc/todo/prevent_unwanted_init/comment_4_e73327b773db8ba8720af44ea46ffa2f._comment
new file mode 100644
index 0000000000..919f827b47
--- /dev/null
+++ b/doc/todo/prevent_unwanted_init/comment_4_e73327b773db8ba8720af44ea46ffa2f._comment
@@ -0,0 +1,12 @@
+[[!comment format=mdwn
+ username="joey"
+ subject="""comment 4"""
+ date="2017-12-13T17:44:18Z"
+ content="""
+If git had a general-purpose config file checked into the repo, I'd use
+that. The closest there is is the .gitattributes file, but that's very
+unsuited for this purpose. It is per-file, so a hack like "*" would be
+needed to refer to the whole repo. Worse, it doesn't allow setting values
+containing spaces, which prevents putting a message to the user in there
+in any reasonable format.
+"""]]
diff --git a/doc/todo/prevent_unwanted_init/comment_5_99af015fca81035af490919632fdcb5f._comment b/doc/todo/prevent_unwanted_init/comment_5_99af015fca81035af490919632fdcb5f._comment
new file mode 100644
index 0000000000..0d88351cb4
--- /dev/null
+++ b/doc/todo/prevent_unwanted_init/comment_5_99af015fca81035af490919632fdcb5f._comment
@@ -0,0 +1,22 @@
+[[!comment format=mdwn
+ username="yarikoptic"
+ avatar="http://cdn.libravatar.org/avatar/f11e9c84cb18d26a1748c33b48c924b4"
+ subject="comment 5"
+ date="2017-12-13T18:12:05Z"
+ content="""
+oh, indeed -- forgot about .gitattributes and indeed sad that spaces seems to be nohow allowed.  But could be as obscure as requiring _ instead of a space to encode the message:
+
+[[!format sh \"\"\"
+$> builtin cd /tmp/; rm -rf /tmp/testds3; datalad create --no-annex /tmp/testds3; cd /tmp/testds3; echo 123 > 123; echo '* annex.noannex=Because_I_said_so/and\"meant:it' > .gitattributes 123; datalad add -m atrr .gitattributes 123; git check-attr --all 123  
+[INFO   ] Creating a new git repo at /tmp/testds3 
+create(ok): /tmp/testds3 (dataset)
+add(ok): /tmp/testds3/.gitattributes (file)
+add(ok): /tmp/testds3/123 (file)
+save(ok): /tmp/testds3 (dataset)
+action summary:
+  add (ok: 2)
+  save (ok: 1)
+123: annex.noannex: Because_I_said_so/and\"meant:it
+123: 123: set
+\"\"\"]]
+"""]]
diff --git a/doc/todo/provide___39__file__39___in_--json-progress_record_for_addurl.mdwn b/doc/todo/provide___39__file__39___in_--json-progress_record_for_addurl.mdwn
new file mode 100644
index 0000000000..12b4cd91ec
--- /dev/null
+++ b/doc/todo/provide___39__file__39___in_--json-progress_record_for_addurl.mdwn
@@ -0,0 +1,19 @@
+Would it be sensibly easy to provide "file" field in progress json records for addurl?  I guess in any usecase (provided or deduced from url filename) it should be known at that stage.
+ATM it is just "null" and I guess (didn't try ATM) it would be impossible to associate particular progress reports with corresponding files in the `--batch -J` mode
+
+[[!format sh """
+$> git annex addurl --file bigone --json --json-progress https://s3.amazonaws.com/fcp-indi/data/Projects/ABIDE_Initiative/Outputs/freesurfer/5.1/UCLA_1_0051257/mri/T1.mgz                                              
+{"byte-progress":259645,"action":{"command":"addurl","file":null},"total-size":2459677,"percent-progress":"10.56%"}
+{"byte-progress":1304125,"action":{"command":"addurl","file":null},"total-size":2459677,"percent-progress":"53.02%"}
+{"command":"addurl","note":"to bigone","success":true,"key":"MD5E-s2459677--ad5bf54490212c7e9d88f15e16c4b0c1","file":"bigone"}
+"""]]
+
+Thanks in advance
+
+[[!meta author=yoh]]
+
+> In general addurl doesn't know the filename until after it's downloaded
+> the url (due to running youtube-dl on html urls), but when --file
+> or --batch --with-files is used, it does know the filename early.
+> So, made the json-progress include the filename when it's known.
+> [[done]] --[[Joey]]
diff --git a/doc/todo/read-only_removable_drives.mdwn b/doc/todo/read-only_removable_drives.mdwn
new file mode 100644
index 0000000000..9dfc3569c0
--- /dev/null
+++ b/doc/todo/read-only_removable_drives.mdwn
@@ -0,0 +1,17 @@
+Here's yet another weird use case. I have a ext3 external drive for storing media that i brought to the office. Naturally, all files on the drive are owned by my desktop user (uid = 1000). In the office, my userid is different (say 1001), so git-annex doesn't see the removable drive at all (which is a little confusing). But even if i try to add the repo on the drive as an external repo, it says it can't write to it (which is true).
+
+I would expect it to at least be able to leech the files off of it.
+
+Otherwise, I would welcome advice on how to fix this problem without doing a `sudo chown -R` every time i plug this drive somewhere ... --[[anarcat]]
+
+> Workaround: `sudo setfacl -R -m u:anarcat:rwx /media/foo/annex`
+
+Note: this seems like there was at least one dupe opened about this in [[bugs/annex_get_fails_from_read-only_filesystem]].
+
+I concede that this may refer to many different issues, so here's a short inventory of issues with readonly repositories:
+
+* trying to add an external readonly drive through the webapp: not detected: see [[todo/show_readonly_removable_drives_in_the_webapp]]
+* trying to add an external readonly drive through the commandline: fails to sync? - couldn't reproduce locally, i will need to go back to that machine for more tests :(
+* trying to add a ssh readonly remote through the webapp: fails to sync and considers the remote "git-only" (which also fails) - couldn't reproduce locally either - maybe this is related to the upgrade option in [[bugs/annex_get_fails_from_read-only_filesystem/]] 
+* trying to add a local readonly remote through the webapp: fails to add, see [[bugs/cannot_add_local_readonly_repo_through_the_webapp]]
+* failing to sync with a readonly remote of a different version: still an issue, see [[bugs/annex_get_fails_from_read-only_filesystem/]] - at least content should be syncable even if the upgrade fails (think of failure conditions such as broken hard drives that are put in readonly mode or ddrescue'd disk images)
diff --git a/doc/todo/read-only_removable_drives/comment_1_979455e3694ae2403134ed6fa2add2fa._comment b/doc/todo/read-only_removable_drives/comment_1_979455e3694ae2403134ed6fa2add2fa._comment
new file mode 100644
index 0000000000..ac98d68450
--- /dev/null
+++ b/doc/todo/read-only_removable_drives/comment_1_979455e3694ae2403134ed6fa2add2fa._comment
@@ -0,0 +1,21 @@
+[[!comment format=mdwn
+ username="http://joeyh.name/"
+ ip="108.236.230.124"
+ subject="comment 1"
+ date="2014-05-16T18:10:28Z"
+ content="""
+The assistant ignores drives that it cannot write to. This is a very good heuristic to avoid listing every special device in linux, OSX, etc as possible removable drives. I don't think it makes sense for the assistant to try to handle this use case.
+
+Is there some problem using git-annex at the command line with a remote that is read-only? I don't see any problem in my tests.
+
+
    +joey@darkstar:~/tmp/r>git annex drop
+drop foo ok
+(Recording state in git...)
+joey@darkstar:~/tmp/r>ls -ld ../readonly
+drwxr-xr-x 3 root root 4096 May 16 14:08 ../readonly/
+joey@darkstar:~/tmp/r>git annex get --from readonly
+get foo (from readonly...) ok
+(Recording state in git...)
+
    
+"""]]
diff --git a/doc/todo/read-only_removable_drives/comment_2_08fced29b86b21f63bb0868747227e08._comment b/doc/todo/read-only_removable_drives/comment_2_08fced29b86b21f63bb0868747227e08._comment
new file mode 100644
index 0000000000..e89e4a5467
--- /dev/null
+++ b/doc/todo/read-only_removable_drives/comment_2_08fced29b86b21f63bb0868747227e08._comment
@@ -0,0 +1,12 @@
+[[!comment format=mdwn
+ username="https://id.koumbit.net/anarcat"
+ ip="72.0.72.144"
+ subject="this also affects ssh remotes"
+ date="2014-10-05T15:26:52Z"
+ content="""
+so i tried another experiment today: i tried to allow access to a remote user to my /srv/foo annex. the annex is writable by me, but not by the user, yet i was expecting the user to be able to sync with it. not push, mind you, but at least pull: in a \"git-only\" scenario, that would be perfectly possible. yet the assistant freaks out because it can't run git-annex on the repo because of a write failure on some pack files, and downgrades the repository to a \"git-only\" repo, which is also inaccurate: it can't sync the metadata either...
+
+i would have expected this repository to be marked as \"readonly\" and the user be capable of fetching new changes automatically from the \"central repo\".
+
+maybe that's the essence of the todo here... --[[anarcat]]
+"""]]
diff --git a/doc/todo/read-only_removable_drives/comment_3_2675e211c7bd248b7f7c1bbc6fd46679._comment b/doc/todo/read-only_removable_drives/comment_3_2675e211c7bd248b7f7c1bbc6fd46679._comment
new file mode 100644
index 0000000000..3ce119e67c
--- /dev/null
+++ b/doc/todo/read-only_removable_drives/comment_3_2675e211c7bd248b7f7c1bbc6fd46679._comment
@@ -0,0 +1,10 @@
+[[!comment format=mdwn
+ username="http://joeyh.name/"
+ ip="209.250.56.54"
+ subject="comment 3"
+ date="2014-10-06T16:03:04Z"
+ content="""
+Note that I still don't have a test case that reproduces this. I have tried and failed a second time to reproduce the original reported problem.
+
+Also, one issue per bug report tends to result in happier developers.
+"""]]
diff --git a/doc/todo/read-only_removable_drives/comment_4_9e9bc6dd5fa8c4cf7f2511b771bd1bc7._comment b/doc/todo/read-only_removable_drives/comment_4_9e9bc6dd5fa8c4cf7f2511b771bd1bc7._comment
new file mode 100644
index 0000000000..2f79d2f4a7
--- /dev/null
+++ b/doc/todo/read-only_removable_drives/comment_4_9e9bc6dd5fa8c4cf7f2511b771bd1bc7._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="http://joeyh.name/"
+ ip="209.250.56.54"
+ subject="comment 4"
+ date="2014-10-06T16:06:52Z"
+ content="""
+Of course, git-annex sync fails when it tries to git push to the removable repo. I don't see that causing any problems in practice.
+"""]]
diff --git a/doc/todo/read-only_removable_drives/comment_5_a693c5744bfc6c33f5605aa9d9c0bfe0._comment b/doc/todo/read-only_removable_drives/comment_5_a693c5744bfc6c33f5605aa9d9c0bfe0._comment
new file mode 100644
index 0000000000..ed24a1054d
--- /dev/null
+++ b/doc/todo/read-only_removable_drives/comment_5_a693c5744bfc6c33f5605aa9d9c0bfe0._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="http://joeyh.name/"
+ ip="209.250.56.54"
+ subject="comment 5"
+ date="2014-10-06T16:10:02Z"
+ content="""
+I guess that the actual problem encountered is entirely limited to the webapp, where if the user chooses \"Add another repository\", then enters the path to an existing, read-only repository, and clicks Make Repository, the webapp replies \"Cannot write a repository there.\"
+"""]]
diff --git a/doc/todo/read-only_removable_drives/comment_6_737e3d315f29a4fc61597ce4f9ec6206._comment b/doc/todo/read-only_removable_drives/comment_6_737e3d315f29a4fc61597ce4f9ec6206._comment
new file mode 100644
index 0000000000..de662d9a5e
--- /dev/null
+++ b/doc/todo/read-only_removable_drives/comment_6_737e3d315f29a4fc61597ce4f9ec6206._comment
@@ -0,0 +1,20 @@
+[[!comment format=mdwn
+ username="anarcat"
+ ip="70.83.139.100"
+ subject="maybe this should be split up, here are the issues i know about"
+ date="2014-10-06T16:14:36Z"
+ content="""
+well this specific issue is more about how git-annex cannot get files off readonly medium, whether it is webapp or not.
+
+maybe it would be wiser to split this up in separate bug reports, because I had at least three different situations:
+
+* trying to add an external readonly drive through the webapp: not detected
+* trying to add an external readonly drive through the commandline: fails to sync? - to be confirmed
+* trying to add a ssh readonly remote through the webapp: fails to sync and considers the remote \"git-only\" (which also fails)
+
+I would need to try the third case through the commandline.
+
+And then there's the upgrade scenario in [[bugs/annex_get_fails_from_read-only_filesystem/]].
+
+Did i miss anything?
+"""]]
diff --git a/doc/todo/read-only_removable_drives/comment_7_16c8652d38ae57db4ed1860a4733a18b._comment b/doc/todo/read-only_removable_drives/comment_7_16c8652d38ae57db4ed1860a4733a18b._comment
new file mode 100644
index 0000000000..121d6768d3
--- /dev/null
+++ b/doc/todo/read-only_removable_drives/comment_7_16c8652d38ae57db4ed1860a4733a18b._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="https://id.koumbit.net/anarcat"
+ ip="72.0.72.144"
+ subject="comment 7"
+ date="2014-10-20T20:04:09Z"
+ content="""
+alright, i documented those issues more distinctively in the summary. hopefully that will clear things up a little. i still need to do some work to reproduce two of those issues, maybe a lot of this is related to the upgrade problem mentionned in [[bugs/annex_get_fails_from_read-only_filesystem/]].
+"""]]
diff --git a/doc/todo/read-only_removable_drives/comment_8_a802e0617c9ef72eb8d3842de99e44ae._comment b/doc/todo/read-only_removable_drives/comment_8_a802e0617c9ef72eb8d3842de99e44ae._comment
new file mode 100644
index 0000000000..f198658894
--- /dev/null
+++ b/doc/todo/read-only_removable_drives/comment_8_a802e0617c9ef72eb8d3842de99e44ae._comment
@@ -0,0 +1,11 @@
+[[!comment format=mdwn
+ username="joey"
+ subject="""comment 8"""
+ date="2016-02-12T18:08:18Z"
+ content="""
+I've made some changes today, which let files be downloaded from readonly
+repositories (both on local drives and remote, as long as git-annex-shell
+is updated to a version with the changes).
+
+The issues with the webapp probably remain.
+"""]]
diff --git a/doc/todo/redundancy_stats_in_status.mdwn b/doc/todo/redundancy_stats_in_status.mdwn
new file mode 100644
index 0000000000..56095fd333
--- /dev/null
+++ b/doc/todo/redundancy_stats_in_status.mdwn
@@ -0,0 +1,23 @@
+Currently, `git annex status` only shows the size of 1 copy of each file.
+If numcopies is being used for redundancy, much more disk can actually be
+in use than status shows.
+
+One idea:
+
+	known annex size: 2 terabytes (plus 4 terabytes of redundant copies)
+
+But, to get that number, it would have to walk every location log, 
+counting how many copies currently exist of each file. That would make
+status a lot slower than it is.
+
+One option is to just put it at the end of the status:
+
+	redundancy: 300% (4 terabytes of copies)
+
+And ctrl-c if it's taking too long.
+
+Hmm, fsck looks at that same info. Maybe it could cache the redundancy
+level it discovers? Since fsck can be run incrementally, it would be tricky
+to get an overall number. And the number would tend to be stale, but
+then again it might also be nice if status shows how long ago the last fsck
+was.
diff --git a/doc/todo/redundancy_stats_in_status/comment_1_9f1c10f8cea4fa60a99cbcc8306dd5de._comment b/doc/todo/redundancy_stats_in_status/comment_1_9f1c10f8cea4fa60a99cbcc8306dd5de._comment
new file mode 100644
index 0000000000..801c1da039
--- /dev/null
+++ b/doc/todo/redundancy_stats_in_status/comment_1_9f1c10f8cea4fa60a99cbcc8306dd5de._comment
@@ -0,0 +1,10 @@
+[[!comment format=mdwn
+ username="https://www.google.com/accounts/o8/id?id=AItOawl9sYlePmv1xK-VvjBdN-5doOa_Xw-jH4U"
+ nickname="Richard"
+ subject="comment 1"
+ date="2012-10-30T08:09:13Z"
+ content="""
+I like the idea of using fsck as a pre-run for status.
+
+Basically, it's the same as `updatedb` and `locate`; locate will warn the user if it considers its cache to be too old, as well.
+"""]]
diff --git a/doc/todo/redundancy_stats_in_status/comment_2_686ced0684d10511caf07953c64cd5b6._comment b/doc/todo/redundancy_stats_in_status/comment_2_686ced0684d10511caf07953c64cd5b6._comment
new file mode 100644
index 0000000000..a4711b2a3a
--- /dev/null
+++ b/doc/todo/redundancy_stats_in_status/comment_2_686ced0684d10511caf07953c64cd5b6._comment
@@ -0,0 +1,10 @@
+[[!comment format=mdwn
+ username="http://joeyh.name/"
+ ip="4.152.108.220"
+ subject="comment 2"
+ date="2013-09-25T18:03:55Z"
+ content="""
+`git annex status .` or otherwise running it with a directory has recently started walking all the location logs for all files in the directory in order to display variance from configured numcopies. It would be easy to add a redundancy counter to that.
+
+It would slow down the global status when not passed a directory to add redundancy info there. Maybe local is enough?
+"""]]
diff --git a/doc/todo/reinit_should_work_without_arguments.mdwn b/doc/todo/reinit_should_work_without_arguments.mdwn
new file mode 100644
index 0000000000..22d3fbbd0a
--- /dev/null
+++ b/doc/todo/reinit_should_work_without_arguments.mdwn
@@ -0,0 +1,65 @@
+Sometimes it may happen that you have multiple git-annex repositories
+with the same UUID. This is usually because you did something special,
+like copying a repository with `cp -a` or `dd` instead of cloning it,
+or you just replaced a drive with a fresh one. In my case, the latter
+happened: I ran out of space on my media drive and replaced it with a
+larger drive. Since it had multiple git-annex repositories (and data
+*not* managed by git-annex), I simply used `rsync` to copy the drive
+over, which created duplicate git-annex repositories with the same
+UUIDs.
+
+It may still be useful to reuse those repositories as distinct
+entities, and it is therefore important to assign new UUIDs to those
+cloned repositories so that content tracking works properly and you do
+not lose data.
+
+In this case, you actually do *not* want to specify an existing UUID
+when you run reinit: you want git-annex to generate a new one for
+you. So, in that context, I've always wondered why
+[[git-annex-reinit]] absolutely requires an argument.  I understand
+there may be *other* use cases where you may want to `reinit` a
+repository to an existing UUID, but that seems like a much *less*
+common use case, and one that may bring more trouble than is worth.
+
+So I believe there should be an easy way to assign a fresh new UUID to
+a repository. `reinit` should allow doing that when no arguments are
+given: it should show the old and new UUID and maybe a warning message
+indicating that tracking information may be wrong now if the old UUID
+is not in use anymore.
+
+As shown below, I also wonder if `reinit` should recommend the user
+perform a `fsck` to make sure the location logs reflect the change...
+
+Workaround
+----------
+
+It is obviously possible to assign a new UUID with the current
+command, by generating one by hand.
+
+Git-annex doesn't have a user-visible way of generating a new UUID, so
+we'll have to improvise something. It uses the
+[Data.UUID](https://hackage.haskell.org/package/uuid-1.3.13/docs/Data-UUID.html)
+Haskell module, in V4 mode, which is the [standard, random
+way](https://en.wikipedia.org/wiki/Universally_unique_identifier#Version_4_.28random.29)
+of generating UUIDs. I believe that the `uuidgen` command, when ran in
+`--random` mode, will produce similarly unique UUIDs that are good
+enough for our purpose. So I have used this to reassign new UUIDs to
+cloned copies of repositories:
+
+    git annex reinit $(uuidgen)
+
+The next step is to fix the location log so that the UUID change is
+reflected in the tracking information:
+
+    git annex fsck --fast
+
+You may also want to set a new description for the new clone:
+
+    git annex describe here "bare 2TB Seagate barracuda HD"
+
+Then, optionally, you will want to propagate that change to other
+repositories:
+
+    git annex sync
+
+Thanks for any feedback or comments... -- [[anarcat]]
diff --git a/doc/todo/reinit_should_work_without_arguments/comment_1_46ba213e20eee4a1a970c545eef82285._comment b/doc/todo/reinit_should_work_without_arguments/comment_1_46ba213e20eee4a1a970c545eef82285._comment
new file mode 100644
index 0000000000..1a332074a0
--- /dev/null
+++ b/doc/todo/reinit_should_work_without_arguments/comment_1_46ba213e20eee4a1a970c545eef82285._comment
@@ -0,0 +1,13 @@
+[[!comment format=mdwn
+ username="joey"
+ subject="""comment 1"""
+ date="2017-01-30T17:24:27Z"
+ content="""
+I don't know that reinit is the best place to put this.
+It's a different use case, and a different operation than the current
+reinit; making a single command do two opposite things is often a bad idea.
+
+One way to accomplish what you want is to just delete the old
+annex.uuid from git config, and then `git annex init` will
+allocate a new uuid.
+"""]]
diff --git a/doc/todo/reinit_should_work_without_arguments/comment_2_5efe582a5aacc8ae1ec0c4ff6f8bed08._comment b/doc/todo/reinit_should_work_without_arguments/comment_2_5efe582a5aacc8ae1ec0c4ff6f8bed08._comment
new file mode 100644
index 0000000000..bd081e6865
--- /dev/null
+++ b/doc/todo/reinit_should_work_without_arguments/comment_2_5efe582a5aacc8ae1ec0c4ff6f8bed08._comment
@@ -0,0 +1,13 @@
+[[!comment format=mdwn
+ username="https://anarc.at/openid/"
+ nickname="anarcat"
+ avatar="http://cdn.libravatar.org/avatar/b36dcf65657dd36128161355d8920a99503def9461c1bb212410980fe6f07125"
+ subject="usability..."
+ date="2017-01-30T18:06:22Z"
+ content="""
+This is all about usability. `reinit` is where I looked for this feature, and I didn't find it, so I figured it could be expanded to cover that use case.
+
+If `init` can do this, why not have a `overwrite` option or something that will actually remove the uuid key from the git config first?
+
+I don't think we should assume users know that they can edit `.git/config` safely. I certainly didn't think of doing that and sometimes shot myself in the foot playing in there... ;)
+"""]]
diff --git a/doc/todo/renameremote.mdwn b/doc/todo/renameremote.mdwn
new file mode 100644
index 0000000000..3a92bf5070
--- /dev/null
+++ b/doc/todo/renameremote.mdwn
@@ -0,0 +1,24 @@
+Sometimes a name has been used for a special remote, and you want to change
+the name. A common reason is that the special remote has become dead, and
+you want to reuse the name for a new special remote. 
+
+Initremote prevents reusing a name when the old one exists, even if the old
+one is dead. And that makes sense in general, because a dead remote can
+come back sometimes, and that would leave the repo with two special remotes
+with the same name, and so enableremote would need to be run with a uuid
+instead of a name to specify which one to enable, which is not a desirable
+state of affairs.
+
+So, add `git annex renameremote oldname newname`. This could also do a `git
+remote rename`, or equivilant. (`git remote rename` gets confused by special
+remotes not having a fetch url and fails; this can be worked around by
+manually renaming the stanza in git config.)
+
+Implementing that would need a way to remove the old name from remote.log.
+We can't remove lines from union merged files, but what we could do is
+add a new line like:
+
+	- name=oldname timestamp=
+
+And in parsing remote.log, if the UUID is "-", don't include the
+remote with that name in the the resulting map.
diff --git a/doc/todo/required_content.mdwn b/doc/todo/required_content.mdwn
new file mode 100644
index 0000000000..6afeee5c9a
--- /dev/null
+++ b/doc/todo/required_content.mdwn
@@ -0,0 +1,23 @@
+We have preferred content, which is advisory, and numcopies, which is
+enforced (except by `git annex move`). What is missing is an expression
+like preferred content, which is enforced. So, required content.
+
+For example, I might want a repository that is required to contain
+`*.jpeg`. This would make get --auto get it (it's implicitly part of the
+preferred content), and would make drop refuse to drop it.
+
+> I've implemented the basic required content. Currently only configurable
+> via `vicfg`, because I don't think a lot of people are going to want to
+> use it.
+> 
+> Note that I did not yet add the active verification discussed below.
+> So if required content is set to `not inallgroup=backup`, or
+> `not copies=10`, trying to drop a file will not go off and prove
+> that there are 10 copies or that the file is in every repository in 
+> the backup group. It will assume that the location log is accurate
+> and go by that.
+> 
+> I think this is enough to cover Richard's case, at least.
+> In his example, A B and C are in group anchor and have required
+> content set to `include=*`, and D E F have it set to
+> `not inallgroup=anchor`. --[[Joey]]
diff --git a/doc/todo/required_content/comment_1_42620a3c958666be2a0d5f5b8eadf7b4._comment b/doc/todo/required_content/comment_1_42620a3c958666be2a0d5f5b8eadf7b4._comment
new file mode 100644
index 0000000000..7d490a9d75
--- /dev/null
+++ b/doc/todo/required_content/comment_1_42620a3c958666be2a0d5f5b8eadf7b4._comment
@@ -0,0 +1,23 @@
+[[!comment format=mdwn
+ username="https://www.google.com/accounts/o8/id?id=AItOawl9sYlePmv1xK-VvjBdN-5doOa_Xw-jH4U"
+ nickname="Richard"
+ subject="comment 1"
+ date="2014-03-10T17:15:54Z"
+ content="""
+To give a specific example of what I want to do:
+
+I have a bunch of repos
+
+* A, Machine1 -- anchor repository and sometimes used to add new content
+* B, Machine2 -- anchor repository, offsite
+* C, External1 -- anchor repository, stored off-site
+* D, Laptop -- transient repo to carry data around and to add new content
+* E, USB1 -- transient, travel backups
+* F, USB2 -- transient, travel backups
+
+A-C should get _all_ data. A-F trust A-C to always retain all data, online checks for availability are not needed because A-C run frequent fsck. Ideally, `git annex drop` should be no-op in A-C.
+
+D-F should retain data as long as it's not been saved in _all_ of A-C. As soon as A-C have copies, `git annex drop` in D-F should drop that content; before that, they _must not_ drop said content.
+
+Richard
+"""]]
diff --git a/doc/todo/required_content/comment_2_132ec6378db63af6281569cf5748b9d3._comment b/doc/todo/required_content/comment_2_132ec6378db63af6281569cf5748b9d3._comment
new file mode 100644
index 0000000000..dece48c01b
--- /dev/null
+++ b/doc/todo/required_content/comment_2_132ec6378db63af6281569cf5748b9d3._comment
@@ -0,0 +1,14 @@
+[[!comment format=mdwn
+ username="http://joeyh.name/"
+ ip="209.250.56.154"
+ subject="partial implementation plan"
+ date="2014-03-19T16:37:05Z"
+ content="""
+* When there is a required content expression, OR it with the preferred content expression. This will make the assistant, --auto etc want to get required content.
+* When checking if something can be dropped, check the required content expression. Only Command.Drop does this so nicely centralized in one place.
+* When checking required content expression for drop, must do active verification of terminals that relate to other copies, for the same reason the numcopies check when dropping does active validation. This includes `copies=`, `lackingcopies=`, `approxlackingcopies=`, `inallgroup=`.
+
+The last is where the complication comes in. Seems to need a cache of places the key was just now verified to be present, which can be used to avoid unnecessary redundant active verification (for example \"not (copies=2 and inallgroup=bar)\" would otherwise redundantly check some remotes). The numcopies checking code should use the same cache too.
+
+* Finally, if the required content cannot be satisfied, it would be nice to know which terminal failed in order to show the user a useful error message. The current Matcher does not provide a way to do that though. Or could just say, \"unable to satisfy required content: $expression\"
+"""]]
diff --git a/doc/todo/required_content/comment_3_b16a8e8b45ceee887c8c0167b7859654._comment b/doc/todo/required_content/comment_3_b16a8e8b45ceee887c8c0167b7859654._comment
new file mode 100644
index 0000000000..f4047baf5f
--- /dev/null
+++ b/doc/todo/required_content/comment_3_b16a8e8b45ceee887c8c0167b7859654._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="https://www.google.com/accounts/o8/id?id=AItOawl9sYlePmv1xK-VvjBdN-5doOa_Xw-jH4U"
+ nickname="Richard"
+ subject="comment 3"
+ date="2014-05-18T06:49:35Z"
+ content="""
+    22:34:10 < joeyh> required content is implemented, though active verification is not
+"""]]
diff --git a/doc/todo/return___34__key__34___entry_in_--json_output_for_addurl___40__and_future_add__41___--batch.mdwn b/doc/todo/return___34__key__34___entry_in_--json_output_for_addurl___40__and_future_add__41___--batch.mdwn
new file mode 100644
index 0000000000..504af11b81
--- /dev/null
+++ b/doc/todo/return___34__key__34___entry_in_--json_output_for_addurl___40__and_future_add__41___--batch.mdwn
@@ -0,0 +1,5 @@
+You have noted that somewhere (may be in email), that it might help us to pipeline things if 'add' was returning the key if file was added to the annex.  I guess the same could apply to 'addurl' so decided to mark this separate todo
+
+[[!meta author=yoh]]
+
+> already done earlier today [[done]] --[[Joey]]
diff --git a/doc/todo/sha1_collision_embedding_in_git-annex_keys.mdwn b/doc/todo/sha1_collision_embedding_in_git-annex_keys.mdwn
new file mode 100644
index 0000000000..37da39a8d2
--- /dev/null
+++ b/doc/todo/sha1_collision_embedding_in_git-annex_keys.mdwn
@@ -0,0 +1,145 @@
+Some git-annex backends allow embedding enough data in the names of keys
+that it could be used for a SHA1 collision attack. So, a signed git commit
+could point to a tree with such a key in it, and the blob for the key could
+have two versions with the same SHA1.
+
+> All issues below are [[done]] --[[Joey]]
+
+Users who want to use git-annex with signed commits to mitigate git's own
+SHA1 insecurities would like at least a way to disable the insecure
+git-annex backends:
+
+* WORM can contain fairly arbitrary data in a key name
+* URL too (also, of course, URLs download arbitrary data from the web,
+  so a signed git commit pointing at URL keys doesn't have any security
+  even w/o SHA1 collisions)
+* SHA1 and MD5 backends are insecure because there can be colliding
+  versions of the data they point to.
+
+There could be a config setting, which would prevent git-annex from using
+keys with such insecure backends. A user who checks git commit signatures
+could enable the config setting when they initially clone their repository.
+This should prevent any file contents using insecure backends from being
+downloaded into the repository. (Even git-annex-shell recvkey would
+refuse data using such a key, since it would fail parsing the key.)
+The user would thus know that any file contents in their repository match
+the files in signed git commits.
+
+Enabling the config setting in a repository that already contains
+file contents would be a mistake, because it might contain insecure keys.
+And since git-annex would skip over such files, `git annex fsck` cannot
+warn about such a mistake.
+
+Perhaps, then, the config setting should be turned on by `git annex init`?
+Or, we can document this gotcha.
+
+> I've done some groundwork for this, but making git-annex not accept
+> insecure keys into the repo at all requires changing file2key,
+> which is a pure function that's used in eg, instances for serialization.
+> 
+> So, how to make it vary depending on git config? Can't. Alternative
+> would be to add lots of checks everywhere a key is read from disk
+> or network, which feels like it would be a hard security boundary to
+> manage.
+> 
+> It doesn't really matter if content under an insecure key is in the
+> repo, as long as there's not a signed commit referencing such a key.
+> So, we could say, this is up to the user constucting a signed commit, to not
+> put such keys in the commit.
+> 
+> Or, we could use the pre-commit hook, and when 
+> the config setting disallows insecure keys, make it reject commits
+> that contain them. But, if a past commit added a file using an insecure
+> key, and the current commit does not touch it, should it be rejected?
+> Rejecting it would then require a somewhat expensive look at the tree
+> being committed.
+> 
+> The user might be merging a branch from someone else; there seems no
+> git hook that can sanity check a fast-forward merge.
+> 
+> Perhaps leave it up to the person making signed commits to get it 
+> right, and make git annex fsck warn about such keys? That seems
+> reasonable. --[[Joey]]
+
+> > Rather than preventing SHA1/URL/WORM Keys, could put checks in
+> > Annex.Content.moveAnnex to prevent SHA1/URL/WORM objects reaching the
+> > repository. That would make moveAnnex a security boundary, which is is
+> > not currently. Would need to audid to check if anything else populates
+> > .git/annex/objects.
+> >
+> > Annex.Transfer.runTransfer could also check for disallowed objects,
+> > not as a security boundary, but to prevent accidental expensive
+> > transfers that would fail at the moveAnnex stage.
+
+> > As to how to enable this, it may make sense to use git-annex-config
+> > but only read the value from the git-annex branch when initializing the
+> > repository, and cache it in git-config.
+> > 
+> > This way, a repository can be created and configured not to allow
+> > SHA1/URL/WORM, and all clones will inherit this configuration. 
+> >
+> > Users can also set it in git-config on a per repository basis.
+> > 
+> > If the git-annex-config setting is changed, existing clone's won't
+> > change their behavior, although new ones will.  That's a mixed
+> > blessing; it makes it harder to switch an existing repo to disallowing
+> > SHA1/URL/WORM, but an accidental/malicious re-enabling won't affect
+> > clones made while it was disabled. 
+> > > This is done now.
+> > 
+> > Could a repository be configured to either always disallow
+> > SHA1/URL/WORM, or always allow them, and then not let that be changed?
+> > Maybe -- Look through all the history of the git-annex branch from the
+> > earliest commit forward. The first value stored in 
+> > git-annex/disableinsecurehashes (eg 0 or 1) is the value to use;
+> > any later changes are ignored. 
+> > That would be a little slow, but only needs to be done at init time.
+> > It might be possible to fool this though. Create a new empty branch,
+> > with an old date, make a commit enabling insecure hashes, and
+> > merge it into git-annex branch HEAD. It now looks as if insecure hashes
+> > were disabled earliest.
+
+> > > Well, annex.securehashesonly is implemented now. It currently needs to be
+> > > set in each clone that cares about it. --[[Joey]]
+
+----
+
+A few other potential problems:
+
+* A symlink target like .git/annex/objects/XX/YY/SHA256--foo
+  might be able to be manipulated to add collision data in the path.
+  For example, .git/annex/objects/collisiondata/../XX/YY/SHA256--foo
+
+  I think this is not a valid attack, because at least on linux,
+  such a symlink won't be followed, unless the
+  .git/annex/objects/collisiondata directory exists.
+
+* `*E` backends could embed sha1 collision data in a long filename
+  extension in a key.
+
+  The recent SHA1 common-prefix attack could be used to exploit this;
+  the result would be two keys that have the same SHA1.
+
+  This can be fixed by limiting the length
+  of an extension allowed in such a key to the longest such extension
+  git-annex has ever supported (probably < 20 bytes or so), which would
+  be less than the size of the data needed for current SHA1 collision
+  attacks. Now done; git-annex refuses to use keys with super
+  long extensions.
+
+* It might be possible to embed colliding data in a specially constructed
+  key name with an extra field in it, eg "SHA256-cXXXXXXXXXXXXXXX-...".
+  Need to review the code and see if such extra fields are allowed.  
+
+  Update: All fields are numeric, but could contain arbitrary data
+  after the number. Could have been used in a common-prefix attack.
+  This has been fixed; git-annex refuses to parse
+  such fields, so it won't work with files that try to exploit this.
+
+* A symlink target like .git/annex/objects/XX/YY/SHA256--foo
+  might be able to be manipulated to add collision data in the path.
+  For example, .git/annex/objects/collisiondata/../XX/YY/SHA256--foo
+
+  I think this is not a valid attack, because at least on linux,
+  such a symlink won't be followed, unless the
+  .git/annex/objects/collisiondata directory exists.
diff --git a/doc/todo/sharedRepository_mode_not_supported_by_git-annex.mdwn b/doc/todo/sharedRepository_mode_not_supported_by_git-annex.mdwn
new file mode 100644
index 0000000000..85005dbc1b
--- /dev/null
+++ b/doc/todo/sharedRepository_mode_not_supported_by_git-annex.mdwn
@@ -0,0 +1,7 @@
+git's core.SharedRepository is supported by git-annex, but only
+with the group/all/world/everybody settings. core.SharedRepository=0644
+etc is not supported.
+
+There's no insormountable reason why not, Joey just hates umask mode math
+stuff and nobody has sent a patch. Note that Annex.Content.freezeContent
+should remove the write bit from files, no matter what.
diff --git a/doc/todo/shorten_git-annex_manpage.mdwn b/doc/todo/shorten_git-annex_manpage.mdwn
new file mode 100644
index 0000000000..97d04eca80
--- /dev/null
+++ b/doc/todo/shorten_git-annex_manpage.mdwn
@@ -0,0 +1,38 @@
+the [[git-annex]] manpage is still pretty long. i wonder if work shouldn't be done to shorten it further.
+
+what i'm thinking of is that this entry, for example, takes up three lines where it could take two, reducing the size by 30% since most commands have only one line explaining it:
+
+
    + * undo `[filename|directory] ...`
+
+   Undo last change to a file or directory.
+
+   See [[git-annex-undo]](1) for details.
+
    
+
+the `git(1)` manpage resolves this by putting the reference in the first line:
+
+
    +       git-init(1)
+           Create an empty Git repository or reinitialize an existing one.
+
    
+
+So i would think there are two options here:
+
+1. do like git, but preserving the usage:
+
+
    + * [[git-annex-undo]](1) `[filename|directory]`
+
+   Undo last change to a file or directory.
+
    
+
+2. Put the reference on the same line:
+
+
    + * undo `[filename|directory] ...`
+
+   Undo last change to a file or directory. See [[git-annex-undo]](1) for details.
+
    
+
+Opinions? --[[anarcat]]
diff --git a/doc/todo/shorten_git-annex_manpage/comment_1_4b82eb4af9b923b23c6ecd8d355c0651._comment b/doc/todo/shorten_git-annex_manpage/comment_1_4b82eb4af9b923b23c6ecd8d355c0651._comment
new file mode 100644
index 0000000000..a0a1b6d266
--- /dev/null
+++ b/doc/todo/shorten_git-annex_manpage/comment_1_4b82eb4af9b923b23c6ecd8d355c0651._comment
@@ -0,0 +1,24 @@
+[[!comment format=mdwn
+ username="joey"
+ subject="""comment 1"""
+ date="2015-05-30T15:36:45Z"
+ content="""
+I don't like the "git-init" approach; git-init used to be in PATH,
+but no longer is, so the git man page is really a bit broken now.
+And of course, git-annex-undo is not actually a filename.
+
+The second approach doesn't shorten it much and makes it harder
+to skip over the boilerplate in the links when reading since it's moved
+into the same paragraph as the description and will appear at differing
+positions depending on the description length.
+
+I'd rather make the command name be a hyperlink. However,
+it's difficult to reconcile this with the limitations of man
+pages.
+
+Maybe aim for something like this in the man page display,
+although I don't know if it can be accomplished:
+
+	undo [filename|directory] ...                            git-annex-undo(1)
+	  Undo last change to a file or directory.
+"""]]
diff --git a/doc/todo/shorten_git-annex_manpage/comment_2_859444d7e2826ac4448b0ba0f716328f._comment b/doc/todo/shorten_git-annex_manpage/comment_2_859444d7e2826ac4448b0ba0f716328f._comment
new file mode 100644
index 0000000000..a0ec1c77cf
--- /dev/null
+++ b/doc/todo/shorten_git-annex_manpage/comment_2_859444d7e2826ac4448b0ba0f716328f._comment
@@ -0,0 +1,17 @@
+[[!comment format=mdwn
+ username="https://id.koumbit.net/anarcat"
+ subject="no easy fix, i guess then"
+ date="2015-05-31T16:51:36Z"
+ content="""
+right, that would be great. unfortunately, we use the `.IP` macro in the rendered groff output, which only has a leader and nothing on the right. i don't believe there's a macro that would answer the requirement, and i looked at `man(7)`, `mdoc(7)`, `man-pages(7)`, `groff_man(7)`, `groff_mdoc(7)`... which probably covers most of the man material i am aware of.
+
+there *is* the `\h'N'` macro that allows moving `N` spaces to the right, but that wouldn't align the references, so we'd be better off with something like:
+
+
    + * undo `[filename|directory]` # git-annex-undo(1)
+
+   Undo last change to a file or directory.
+
    
+
+... or something like that...
+"""]]
diff --git a/doc/todo/shorten_git-annex_manpage/comment_3_3543a2edda668c3f0a90c46aeee94812._comment b/doc/todo/shorten_git-annex_manpage/comment_3_3543a2edda668c3f0a90c46aeee94812._comment
new file mode 100644
index 0000000000..7952d4d53b
--- /dev/null
+++ b/doc/todo/shorten_git-annex_manpage/comment_3_3543a2edda668c3f0a90c46aeee94812._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="joey"
+ subject="""comment 3"""
+ date="2015-06-02T17:41:12Z"
+ content="""
+I'm not sure if it's clear enough that is a man-link and not some
+complexity added to the command line options.
+"""]]
diff --git a/doc/todo/shorter_keys_through_better_encoding.mdwn b/doc/todo/shorter_keys_through_better_encoding.mdwn
new file mode 100644
index 0000000000..c76598b73d
--- /dev/null
+++ b/doc/todo/shorter_keys_through_better_encoding.mdwn
@@ -0,0 +1,4 @@
+The link targets of annexed files are currently very long.   This creates problems e.g. when browsing directories in Emacs (I mostly work through a text terminal).  Ideally, the key would not be repeated twice in the link, but I understand this is hard to do compatibly.  Maybe, the following simpler alternative could be implemented?   Key checksums are currently represented in base16 using only the characters 0-9a-f .  The same information could be represented with shorter strings using base64url or other encoding, where a larger range of chars is used.  So for each backend you'd add a corresponding one that does the same thing, but encodes the checksum part of the key with shorter strings.
+
+
+Or, if you're tired of backend requests, maybe implement a scheme for external backends, like the one for external special remotes?  For external backend EXTNNN the user would put  a script git-annex-external-backend-NNN in the path; the script would support commands like calckey, examinekey .   Then I could also implement e.g. canonicalizing backends that strip away variable but semantically irrelevant information before computing the checksum.
diff --git a/doc/todo/shorter_keys_through_better_encoding/comment_1_4bd588a3cdca5cb94132549b7d78e6af._comment b/doc/todo/shorter_keys_through_better_encoding/comment_1_4bd588a3cdca5cb94132549b7d78e6af._comment
new file mode 100644
index 0000000000..94f752cab1
--- /dev/null
+++ b/doc/todo/shorter_keys_through_better_encoding/comment_1_4bd588a3cdca5cb94132549b7d78e6af._comment
@@ -0,0 +1,10 @@
+[[!comment format=mdwn
+ username="CandyAngel"
+ avatar="http://cdn.libravatar.org/avatar/15c0aade8bec5bf004f939dd73cf9ed8"
+ subject="comment 1"
+ date="2018-09-27T07:39:57Z"
+ content="""
+The key is in the path twice as a security measure (the write bit is removed from the directory, to prevent `rm -rf`ing all the files away by mistake).
+
+This is known to cause slowness while traversing the objects directory, which is why there is repository [[tuning]]. Perhaps you want to set some of these?
+"""]]
diff --git a/doc/todo/shorter_keys_through_better_encoding/comment_2_454cb3b2b1102ce4da18f245bc583e7c._comment b/doc/todo/shorter_keys_through_better_encoding/comment_2_454cb3b2b1102ce4da18f245bc583e7c._comment
new file mode 100644
index 0000000000..4687afe8a3
--- /dev/null
+++ b/doc/todo/shorter_keys_through_better_encoding/comment_2_454cb3b2b1102ce4da18f245bc583e7c._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="Ilya_Shlyakhter"
+ avatar="http://cdn.libravatar.org/avatar/1647044369aa7747829c38b9dcc84df0"
+ subject="comment 2"
+ date="2018-09-27T11:09:04Z"
+ content="""
+\"The key is in the path twice as a security measure\" -- would key/f.txt be less secure than key/key.txt?  I thought the security comes from having both a dir and a file, not from them both having the key in their name?
+"""]]
diff --git a/doc/todo/show_me_where_unused_file_was__44___i_can_wait.mdwn b/doc/todo/show_me_where_unused_file_was__44___i_can_wait.mdwn
new file mode 100644
index 0000000000..c080d2b7c7
--- /dev/null
+++ b/doc/todo/show_me_where_unused_file_was__44___i_can_wait.mdwn
@@ -0,0 +1,7 @@
+i know that `git annex unused` would be slower if, instead of just showing the hash, it would also show the pathname where the file was. it does tell me that i can use `git log --stat -SKEY` to find that out myself, but then i would need to make some silly shell script to loop over multiple files. i'm hoping that git-annex has more efficient and clever ways of doing that, and even if it's slower, i'd be ready to wait if there was an extra flag to show me where it was...
+
+i have used this oneliner so far, but it's ugly and painful, especially since `git annex unused` doesn't have a very parseable output format...
+
+    git annex unused 2>&1 | grep '^  *[0-9][0-9]*' | sed 's/^ *[0-9][0-9]* *//' | xargs -I'{}' git log --oneline --stat -S'{}' -1
+
+any way to do this more easily? --[[anarcat]]
diff --git a/doc/todo/show_me_where_unused_file_was__44___i_can_wait/comment_1_732f726f72f2540372d434974fda60f2._comment b/doc/todo/show_me_where_unused_file_was__44___i_can_wait/comment_1_732f726f72f2540372d434974fda60f2._comment
new file mode 100644
index 0000000000..522696b18e
--- /dev/null
+++ b/doc/todo/show_me_where_unused_file_was__44___i_can_wait/comment_1_732f726f72f2540372d434974fda60f2._comment
@@ -0,0 +1,11 @@
+[[!comment format=mdwn
+ username="joey"
+ subject="""comment 1"""
+ date="2015-10-26T19:29:12Z"
+ content="""
+git-annex has no better way to find files that linked to that key than the
+given log -S.
+
+However, your script can be improved by reading the list of unused keys
+from `.git/annex/unused`
+"""]]
diff --git a/doc/todo/show_readonly_removable_drives_in_the_webapp.mdwn b/doc/todo/show_readonly_removable_drives_in_the_webapp.mdwn
new file mode 100644
index 0000000000..e0e570fe03
--- /dev/null
+++ b/doc/todo/show_readonly_removable_drives_in_the_webapp.mdwn
@@ -0,0 +1,15 @@
+Coming from [[todo/read-only_removable_drives/]], this is use case 1: use inserts an `ext` formatted filesystem that he built at home (so files are owned by uid `1000`)  in the office computer (where he is uid `1001`). Now, this is a limitation of UNIX-style removable drives, admittedly, but I would expect to be able to sync "down" from the hard drives to copy the contents locally.
+
+So in short, expected behavior:
+
+1. insert the drive
+2. drive is shown in the webapp menu
+3. add the drive as a remote for the local repo
+4. sync the content from the drive to the local repo
+
+Actual behavior:
+
+1. insert the drive
+2. drive is not shown in the webapp menu
+
+--[[anarcat]]
diff --git a/doc/todo/show_readonly_removable_drives_in_the_webapp/comment_1_c41140289f9b062e96cfd5d9d5382155._comment b/doc/todo/show_readonly_removable_drives_in_the_webapp/comment_1_c41140289f9b062e96cfd5d9d5382155._comment
new file mode 100644
index 0000000000..faa49bf149
--- /dev/null
+++ b/doc/todo/show_readonly_removable_drives_in_the_webapp/comment_1_c41140289f9b062e96cfd5d9d5382155._comment
@@ -0,0 +1,15 @@
+[[!comment format=mdwn
+ username="joey"
+ subject="""comment 1"""
+ date="2014-10-21T16:33:25Z"
+ content="""
+The webapp only shows drives the user can write to, because
+in general, there are a vast number of mounted things in modern
+OS's that the user would be very puzzled to see listed as removable
+drives. Such as tmpfs and cgroup mount points, and efi boot partitions.
+Complicating the webapp with knowledge to filter such things out would be a
+constantly losing battle.
+
+I guess it would be ok to add a "not listed here?" link in the webapp
+that allowed chosing from a full list or entering the path by hand.
+"""]]
diff --git a/doc/todo/show_readonly_removable_drives_in_the_webapp/comment_2_b9236ea884a2934dd0c8e511a80f4fda._comment b/doc/todo/show_readonly_removable_drives_in_the_webapp/comment_2_b9236ea884a2934dd0c8e511a80f4fda._comment
new file mode 100644
index 0000000000..9aa4c179e6
--- /dev/null
+++ b/doc/todo/show_readonly_removable_drives_in_the_webapp/comment_2_b9236ea884a2934dd0c8e511a80f4fda._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="anarcat"
+ ip="70.83.139.100"
+ subject="comment 2"
+ date="2014-10-30T15:15:57Z"
+ content="""
+yep, that would be fine.
+"""]]
diff --git a/doc/todo/simpler_setup_for_remote_worktree_update_on_push.mdwn b/doc/todo/simpler_setup_for_remote_worktree_update_on_push.mdwn
new file mode 100644
index 0000000000..d3ce8cad05
--- /dev/null
+++ b/doc/todo/simpler_setup_for_remote_worktree_update_on_push.mdwn
@@ -0,0 +1,75 @@
+A git post-update hook can run git-annex merge and so make all pushes
+into a repository update its working tree.
+
+But, there are some complications to writing that hook. See
+
+IIRC different versions of git may behave differently.
+
+And, such a hook can't be used on a filesystem that doesn't support
+executables. (Except on Windows which has a workaround to allow
+non-executable hooks.)
+
+Could there be a single command that sets it up? Something like 
+`git annex merge --run-automatically`
+
+That could install the hook, and also set an annex.automerge config.
+
+The config could be checked by git-annex sync (and assistant) when pushing
+to a local remote, and they could perform a merge on the remote. This way,
+it would work for repos on removable drives that don't support execute
+bits. (Ssh remotes on such filesystems would not be handled, but that's a
+rare configuration; the hook would handle ssh remotes on non-crippled
+filesystems, and on Windows.)
+
+---
+
+Alternatively, receive.denyCurrentBranch can be set to updateInstead.
+With this configuration, `git annex sync` automatically updates the
+work-tree of the remote already. 
+
+This wouldn't work for direct mode repositories, which are often used
+on removable drives, since git thinks they're bare repos.
+
+Nor will it work for adjusted branches, since the adjusted branch is not
+updated by the push.
+
+---
+
+Could the updateInstead configuration be made to work for direct mode and
+adjusted branches? Could install a post-update hook, that runs a git-annex
+command that checks for updateInstead, and emulates its behavior, handling
+direct mode and adjusted branches.
+
+To support direct mode repos on removable drives w/o execute bits,
+could make sync check local remotes and run the equivilant action as the
+hook would run.
+
+To fully emulate updateInstead, the post-update hook 
+should abort if the tree is unclean or if there are merge conflicts.
+But, in a direct mode repo, the only way the user will likely resolve such
+a situation is git-annex sync/merge, so the hook could just run git-annex
+merge instead of trying to fully emulate regular updateInstead behavior.
+Similarly, in an adjusted branch, the push will update master, and git
+annex sync/merge is what the user will likely do. Although they could
+choose to reset changes to the tree.
+
+---
+
+Potential least surprise violation:
+
+If a repository is updating when git annex pushes changes to it, 
+the user might also expect that the same git annex sync
+would pull changes from that repository. Even though nothing has been
+run on the repository to commit changes made there.
+
+Particularly when the assistant is being used, this seems an easy confusion 
+to have. In one clone the user sees every file change getting committed
+and synced around, so why would that not happen in the other clone, on the
+removable drive?
+
+Keeping this a command-line setup, and not something the assistant does,
+will avoid that confusion.
+
+--[[Joey]]
+
+> all above [[done]] --[[Joey]]
diff --git a/doc/todo/smudge.mdwn b/doc/todo/smudge.mdwn
new file mode 100644
index 0000000000..eae1e30122
--- /dev/null
+++ b/doc/todo/smudge.mdwn
@@ -0,0 +1,437 @@
+git-annex should use smudge/clean filters. v6 mode
+
+### problems keeping v6 experimental
+
+* Checking out a different branch causes git to smudge all changed files,
+  and write their content. This does not honor annex.thin. A warning
+  message is printed in this case.  
+
+  This is particularly wasteful when checking out an adjusted unlocked
+  branch, which causes 2x the space to be used.  
+  
+  "git annex proxy" could be used to handle this.
+  Make it run the git command with smudge filters disabled and then
+  scan through the changed files in the work tree, and update pointer files
+  to be hard links to their content.
+  
+  git-annex adjust and git-annex sync could both use that internally
+  when checking out the adjusted branch, and merging a branch into HEAD.
+  
+  Or: Make the smudge filter not provide the actual file content, but the
+  pointer, at least when used in a checkout. And install a post-checkout
+  hook that populates the worktree files that were checked out.
+  Of course, it will also need to update the index..
+
+  (My enhanced smudge/clean patch set also fixed this problem, in a much
+  nicer way...)
+
+* 
+
+  I have this mostly analized and need to implement the planned fix.
+
+## other warts
+
+* There are several v6 bugs that are edge cases and
+  need more info or analysis. None of these seem like blockers
+  to keep v6 experimental or to replacing direct mode with v6.
+
+  - 
+  - 
+  - a
+  - 
+
+* When git runs the smudge filter, it buffers all its output in ram before
+  writing it to a file. So, checking out a branch with a large v6 unlocked files
+  can cause git to use a lot of memory.
+
+  This needs to be fixed in git, but my proposed interface in
+   would
+  avoid the problem for git checkout, since it would use the new interface
+  and not the smudge filter.
+
+  Last verified with git 2.18 in 2018. 
+
+  Note that the long-running filter process interface has the same problem.
+
+  The annex.thin idea above could work around this problem.
+
+### long term todos
+
+* Potentially: Use git's new `filter..process` interface, which will
+  let only 1 git-annex process be started by git when processing
+  multiple files, and so should be faster.
+
+  See [[todo/Long_Running_Filter_Process]] .. it's not currently actually a
+  win but might be a good way to improve git to work better with v6.
+
+* Eventually (but not yet), make v6 the default for new repositories.
+  Note that the assistant forces repos into direct mode; that will need to
+  be changed then, and it should enable annex.thin instead.
+
+* Later still, remove support for direct mode, and enable automatic
+  v5 to v6 upgrades.
+
+### historical notes
+
+2013: Currently, this does not look likely to work. In particular,
+the clean filter needs to consume all stdin from git, which consists of the
+entire content of the file. It cannot optimise by directly accessing
+the file in the repository, because git may be cleaning a different
+version of the file during a merge. 
+
+So every `git status` would need to read the entire content of all
+available files, and checksum them, which is too expensive.
+
+> Update from GitTogether: Peff thinks a new interface could be added to
+> git to handle this sort of case in an efficient way.. just needs someone
+> to do the work. --[[Joey]] 
+
+>> Update 2015: git status only calls the clean filter for files
+>> that the index says are modified, so this is no longer a problem.
+>> --[[Joey]]
+
+### background
+
+The clean filter is run when files are staged for commit. So a user could copy
+any file into the annex, git add it, and git-annex's clean filter causes
+the file's key to be staged, while its value is added to the annex.
+
+The smudge filter is run when files are checked out. Since git annex
+repos have partial content, this would not git annex get the file content.
+Instead, if the content is not currently available, it would need to do
+something like return empty file content. (Sadly, it cannot create a
+symlink, as git still wants to write the file afterwards.)
+
+So the nice current behavior of unavailable files being clearly missing due
+to dangling symlinks, would be lost when using smudge/clean filters.
+(Contact git developers to get an interface to do this?)
+
+Instead, we get the nice behavior of not having to remeber to `git annex
+add` files, and just being able to use `git add` or `git commit -a`,
+and have it use git-annex when .gitattributes says to. Also, annexed
+files can be directly modified without having to `git annex unlock`.
+
+### configuration
+
+In .gitattributes, the user would put something like "* filter=git-annex".
+This way they could control which files are annexed vs added normally.
+
+It would also be good to allow using this without having to specify
+the files in .gitattributes. Just use "* filter=git-annex" there, and then
+let git-annex decide which files to annex and which to pass through the
+smudge and clean filters as-is. The smudge filter can just read a little of
+its input to see if it's a pointer to an annexed file. The clean filter
+could apply annex.largefiles to decide whether to annex a file's content or
+not.
+
+For files not configured this way in .gitattributes, git-annex could
+continue to use its symlink method -- this would preserve backwards
+compatability, and even allow mixing the two methods in a repo as desired.
+(But not switching an existing repo between indirect and direct modes;
+the user decides which mode to use when adding files to the repo.)
+
+### clean
+
+The trick is doing it efficiently. Since git a2b665d, v1.7.4.1,
+something like this works to provide a filename to the clean script:
+
+	git config --global filter.huge.clean huge-clean %f
+
+This could avoid it needing to read all the current file content from stdin
+when doing eg, a git status or git commit. Instead it is passed the
+filename that git is operating on, in the working directory.
+(Update: No, doesn't work; git may be cleaning a different file content
+than is currently on disk, and git requires all stdin be consumed too.)
+
+So, WORM could just look at that file and easily tell if it is one
+it already knows (same mtime and size). If so, it can short-circuit and
+do nothing, file content is already cached.
+
+SHA1 has a harder job. Would not want to re-sha1 the file every time,
+probably. So it'd need a local cache of file stat info, mapped to known
+objects.
+
+But: Even with %f, git actually passes the full file content to the clean
+filter, and if it fails to consume it all, it will crash (may only happen
+if the file is larger than some chunk size; tried with 500 mb file and 
+saw a SIGPIPE.) This means unnecessary works needs to be done, 
+and it slows down *everything*, from `git status` to `git commit`.
+**showstopper** I have sent a patch to the git mailing list to address
+this.  (Update: apparently
+can't be fixed.)
+
+> Update: I tried this again (2015) and it seems that git status and git
+> add avoid re-sending the file content to the clean filter, as long as the
+> file stat has not changed. I'm not sure when git started doing that,
+> but it seems to avoid this problem.
+> --[[Joey]]
+
+### smudge
+
+The smudge script can also be provided a filename with %f, but it
+cannot directly write to the file or git gets unhappy.
+
+> Still the case in 2015. Means an unnecesary read and pipe of the file
+> even if the content is already locally available on disk. --[[Joey]]
+
+### partial checkouts
+
+.. Are very important, otherwise a repo can't scale past the size of the
+smallest client's disk!
+
+It would be nice if the smudge filter could hard link a work
+tree file to the annex object.
+
+But currently, the smudge filter can't modify the work tree file on its own
+-- git always modifies the file after getting the output of the smudge
+filter, and will stumble over any modifications that the smudge filter
+makes. And, it's important that the smudge filter never fail as that will
+leave the repo in a bad state.
+
+Seems the best that can be done is for the smudge filter to copy from the
+annex object when the object is present. When it's not present, the smudge
+filter should provide a pointer to its content.
+
+The clean filter should detect when it's operating on that pointer file.
+
+I've a demo implementation of this technique in the scripts below.
+
+### deduplication
+
+.. Is nice; needing 2 copies of every annexed file is annoying.
+
+Unfortunately, when using smudge/clean, `git merge` does not preserve a
+smudged file in the work tree when renaming it. It instead deletes the old
+file and asks the smudge filter to smudge the new filename.
+
+So, copies need to be maintained in .git/annex/objects, though it's ok
+to use hard links to the work tree files. (Although somewhat unsafe
+since modification of the file will lose the old version. annex.thin
+setting can enable this.)
+
+Even if hard links are used, smudge needs to output the content of an
+annexed file, which will result in duplication when merging in renames of
+files.
+
+### design
+
+Goal: Get rid of current direct mode, using smudge/clean filters instead to
+cover the same use cases, more flexibly and robustly.
+
+Use case 1:
+
+A user wants to be able to edit files, and git-add, git commit,
+without needing to worry about using git-annex to unlock files, add files,
+etc.
+
+Use case 2:
+
+Using git-annex on a crippled filesystem that does not support symlinks.
+
+Data:
+
+* An annex pointer file has as its first line the git-annex key
+  that it's standing in for (prefixed with "annex/objects/", similar to
+  an annex symlink target). Subsequent lines of the file might
+  be a message saying that the file's content is not currently available.
+  An annex pointer file is checked into the git repository the same way
+  that an annex symlink is checked in.
+* A file map is  maintained by git-annex, to keep track of the keys
+  that are used by files in the working tree.
+
+Configuration: 
+
+* .gitattributes tells git which files to use git-annex's smudge/clean
+  filters with. Typically, all files except for dotfiles:
+
+	* filter=annex
+	.* !filter
+
+* annex.largefiles tells git-annex which files should in fact be put in 
+  the annex. Other files are passed through the smudge/clean as-is and
+  have their contents stored in git.
+
+* annex.direct is repurposed to configure how git-annex adds files.
+  When set to false, it adds symlinks and when true it adds pointer files.
+
+git-annex clean:
+
+* Run by `git add` (and diff and status, etc), and passed the
+  filename, as well as fed the file content on stdin.
+
+  Look at configuration to decide if this file's content belongs in the
+  annex. If not, output the file content to stdout.
+
+  Generate annex key from filename and content from stdin.
+
+  Hard link (annex.thin) or copy .git/annex/objects to the file,
+  if it doesn't already exist.
+
+  This is done to prevent losing the only copy of a file when eg
+  doing a git checkout of a different branch, or merging a commit that
+  renames or deletes a file. But, with annex.thin no attempt is made to 
+  protect the object from being modified. If a user wants to
+  protect object contents from modification, they should use
+  `git annex add`, not `git add`, or they can `git annex lock` after adding,
+  or not enable annex.thin.
+
+  Update file map.
+
+  Output the pointer file content to stdout.
+
+git-annex smudge:
+
+* Run by eg `git checkout`
+  and passed the filename, as well as fed the pointer file content on stdin.
+
+  Update file map.
+
+  When an object is present in the annex, outputs its content to stdout.
+  Otherwise, outputs the file pointer content.
+
+git annex direct/indirect:
+
+  Previously these commands switched in and out of direct mode.
+  Now they become no-ops.
+
+git annex lock/unlock:
+
+  Makes sense for these to change to switch files between using
+  git-annex symlinks and pointers. So, this provides both a way to
+  transition repositories to using pointers, and a cleaner unlock/lock
+  for repos using symlinks.
+
+  unlock will stage a pointer file, and will link the content of the object
+  from .git/annex/objects to the work tree file.
+  
+  lock will replace the current work tree file with the symlink, and stage it,
+  and lock down the permissions of the annex object.
+
+#### file map
+
+The file map needs to map from `Key -> [File]`. `File -> Key`
+seems useful to have, but in practice is not worthwhile.
+
+Drop and get operations need to know what files in the work tree use a
+given key in order to update the work tree. And, we don't want to
+overwrite a work tree file if it's been modified when dropping or getting.
+
+git-annex commands that look at annex symlinks to get keys to act on will
+need fall back to either consulting the file map, or looking at the staged
+file to see if it's a pointer to a key. So a `File -> Key` map is a possible
+optimisation.
+
+Question: If the smudge/clean filters update the file map incrementally
+based on the pointer files they generate/see, will the result
+always be consistent with the content of the working tree?
+
+This depends on when git calls the smudge/clean filters and on what.
+In particular:
+
+* Does the clean filter always get called when adding a relevant 
+  file to git? Yes.
+* Is the clean filter called at any other time? Yes, for example
+  git diff will clean relevant modified files to generate the diff.
+  So, the clean filter may see file versions that have not yet been staged
+  in git.
+* Is the clean filter ever passed content not in the work tree?
+  I don't think so, but not 100% sure.
+* Is the smudge filter always called when git updates a relevant file
+  in the work tree? Yes.
+* Is the smudge filter called at any other time? Seems unlikely but then
+  there could be situations with a detached work tree or such.
+* Does git call any useful hooks when removing a file from the work tree,
+  or converting it to not be annexed, or for `git mv` of an annexed file?
+  No!
+
+From this analysis, any file map generated by the smudge/clean filters
+is necessary potentially innaccurate. It may list deleted files.
+It may or may not reflect current unstaged changes from the work tree.
+
+
+Follows that any use of the file map needs to verify the info from it,
+and throw out bad cached info (updating the map to match reality). 
+
+When downloading a key, check if the files listed in the file map are
+still pointer files in the work tree, and only replace them with the
+content if so. 
+
+When dropping a key, check if the files listed for it in the file map are
+unmodified in the work tree, and are staged as pointers to the key,
+and only reset them to the pointers if so. Note that this means that
+a modified work tree file that has not yet been staged, but that
+corresponds to a key, won't be reset when the key is dropped.
+This is probably not a big deal; the user will either add the
+file, which will add the key back, or reset the file.
+
+Does the `File -> Key` map have any benefits given this innaccuracy?
+Answer seems to be no; any answer that map gives may be innaccurate and
+needs to be verified by looking at actual repo content, so might as well
+just look at the repo content in the first place..
+
+#### Upgrading
+
+annex.version changes to 6
+
+git config for filter.annex.smudge and filter.annex.clean is set up.
+
+.gitattributes is updated with a stock configuration,
+unless it already mentions "filter=annex".
+
+Upgrading a direct mode repo needs to switch it out of bare mode, and
+needs to run `git annex unlock` on all files (or reach the same result).
+So will need to stage changes to all annexed files.
+
+When a repo has some clones indirect and some direct, the upgraded repo
+will have all files unlocked, necessarily in all clones. This happens
+automatically, because when the direct repos are upgraded that causes the
+files to be unlocked, while the indirect upgrades don't touch the files.
+
+----
+
+### test files
+
+huge-smudge:
+
+
    +#!/bin/sh
+read f
+file="$1"
+echo "smudging $f" >&2
+if [ -e ~/$f ]; then
+	cat ~/$f # possibly expensive copy here
+else
+	echo "$f not available"
+fi
+
    
+
+huge-clean:
+
+
    +#!/bin/sh
+file="$1"
+cat >/tmp/file
+# in real life, this should be done more efficiently, not trying to read
+# the whole file content!
+if grep -q 'not available' /tmp/file; then
+	awk '{print $1}' /tmp/file # provide what we would if the content were avail!
+	exit 0
+fi
+echo "cleaning $file" >&2
+# XXX store file content here
+echo $file
+
    
+
+.gitattributes:
+
+
    +*.huge filter=huge
+
    
+
+in .git/config:
+
+
    +[filter "huge"]
+        clean = huge-clean %f
+        smudge = huge-smudge %f
+
    diff --git a/doc/todo/smudge/comment_10_19dd6be938d71c7311aecb3081849792._comment b/doc/todo/smudge/comment_10_19dd6be938d71c7311aecb3081849792._comment
new file mode 100644
index 0000000000..634b34e83f
--- /dev/null
+++ b/doc/todo/smudge/comment_10_19dd6be938d71c7311aecb3081849792._comment
@@ -0,0 +1,7 @@
+[[!comment format=mdwn
+ username="joey"
+ subject="""comment 10"""
+ date="2016-04-09T17:22:11Z"
+ content="""
+Hardlinks now used on windows when annex.thin is enabled.
+"""]]
diff --git a/doc/todo/smudge/comment_11_ace695aa9d6c5d888b91a2eebb19f97b._comment b/doc/todo/smudge/comment_11_ace695aa9d6c5d888b91a2eebb19f97b._comment
new file mode 100644
index 0000000000..e67ccdf2d9
--- /dev/null
+++ b/doc/todo/smudge/comment_11_ace695aa9d6c5d888b91a2eebb19f97b._comment
@@ -0,0 +1,14 @@
+[[!comment format=mdwn
+ username="davicastro"
+ avatar="http://cdn.libravatar.org/avatar/4e708663cf4d5b9e8cfea74caf4307fc"
+ subject="What are the drawbacks of repo v6?"
+ date="2018-03-12T19:00:36Z"
+ content="""
+Hi. I read the post http://git-annex.branchable.com/tips/unlocked_files/, and got into this post through the comment to check the limitations of repo v6.
+I read through this post, but I still don't grasp the drawbacks of repo v6.
+I like the idea of mixed mode repositories with locked and unlocked co-existing. And I need repo v6 for that right?
+I also like the annex.largefiles to control what is annex what is normal git files.
+With these two combined feature I can configure my project and let users without too much burden on controlling what is annex, what is not, and what are locked or unlocked.
+I want to use repo v6, but what are the drawbacks on adopting it? Is it in terms of git could be bugged? Or the smudge filters could be bugged?
+Sorry for not being able to grasp this from the current post. I appreciate any help. Thanks.
+"""]]
diff --git a/doc/todo/smudge/comment_12_a4712d510432931062406c4e256f04b1._comment b/doc/todo/smudge/comment_12_a4712d510432931062406c4e256f04b1._comment
new file mode 100644
index 0000000000..cfa1be8bca
--- /dev/null
+++ b/doc/todo/smudge/comment_12_a4712d510432931062406c4e256f04b1._comment
@@ -0,0 +1,11 @@
+[[!comment format=mdwn
+ username="joey"
+ subject="""re: Git 2.5 allows smudge filters to not read all of stdin"""
+ date="2018-08-09T22:11:00Z"
+ content="""
+@torarnv thanks for pointing that out.. I finally got around to verifying
+that, and was able to speed up the smudge filter. Also this avoids the
+problem that git for some reason buffers the whole file content in memory
+when it sends it to the smudge filter, which is a pretty bad memory leak in git
+that no longer affects this.
+"""]]
diff --git a/doc/todo/smudge/comment_1_4ea616bcdbc9e9a6fae9f2e2795c31c9._comment b/doc/todo/smudge/comment_1_4ea616bcdbc9e9a6fae9f2e2795c31c9._comment
new file mode 100644
index 0000000000..a4eb3cf235
--- /dev/null
+++ b/doc/todo/smudge/comment_1_4ea616bcdbc9e9a6fae9f2e2795c31c9._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="http://christian.amsuess.com/chrysn"
+ nickname="chrysn"
+ subject="git-add instead of git-annex-add"
+ date="2011-02-26T21:43:21Z"
+ content="""
+would, with these modifications in place, there still be a way to *really* git-add a file? (my main repository contains both normal git and git-annex files.)
+"""]]
diff --git a/doc/todo/smudge/comment_2_e04b32caa0d2b4c577cdaf382a3ff7f6._comment b/doc/todo/smudge/comment_2_e04b32caa0d2b4c577cdaf382a3ff7f6._comment
new file mode 100644
index 0000000000..3a223e1c7b
--- /dev/null
+++ b/doc/todo/smudge/comment_2_e04b32caa0d2b4c577cdaf382a3ff7f6._comment
@@ -0,0 +1,12 @@
+[[!comment format=mdwn
+ username="http://dieter-be.myopenid.com/"
+ nickname="dieter"
+ subject="symlinks"
+ date="2011-04-03T20:30:21Z"
+ content="""
+> (Sadly, it cannot create a symlink, as git still wants to write the file afterwards.
+> So the nice current behavior of unavailable files being clearly missing due to dangling symlinks, would be lost when using smudge/clean filters. (Contact git developers to get an interface to do this?)
+
+Have you checked what the smudge filter sees when the input is a symlink? Because git supports tracking symlinks, so it should also support pushing symlinks through a smudge filter, right?
+Either way: yes, contact the git devs, one can only ask and hope.  And if you can demonstrate the awesomeness of git-annex they might get more 1interested :)
+"""]]
diff --git a/doc/todo/smudge/comment_3_4e7c25fe24a1e71f58a8354fa64f41c2._comment b/doc/todo/smudge/comment_3_4e7c25fe24a1e71f58a8354fa64f41c2._comment
new file mode 100644
index 0000000000..cd64b70019
--- /dev/null
+++ b/doc/todo/smudge/comment_3_4e7c25fe24a1e71f58a8354fa64f41c2._comment
@@ -0,0 +1,10 @@
+[[!comment format=mdwn
+ username="https://www.google.com/accounts/o8/id?id=AItOawn1QhtPvsRBV7pfaDW_ZTPFv_ZIxSzQ8Rg"
+ nickname="Paul Léo"
+ subject="comment 3"
+ date="2013-11-13T20:41:52Z"
+ content="""
+> SHA1 has a harder job. Would not want to re-sha1 the file every time, probably. So it'd need a local cache of file stat info, mapped to known objects.
+
+I think that is not true? If gits wants the file to be cleaned, it thinks that the file was changed. So you would have to SHA1 it anyway if you don't want rely on WORM (which git already does in the first step anyway).
+"""]]
diff --git a/doc/todo/smudge/comment_4_5ff4ad15865a93dc8c066220561936b2._comment b/doc/todo/smudge/comment_4_5ff4ad15865a93dc8c066220561936b2._comment
new file mode 100644
index 0000000000..392df53899
--- /dev/null
+++ b/doc/todo/smudge/comment_4_5ff4ad15865a93dc8c066220561936b2._comment
@@ -0,0 +1,9 @@
+[[!comment format=mdwn
+ username="tomekwi"
+ subject="How about the other way round?"
+ date="2015-04-25T09:13:31Z"
+ content="""
+If I understand this right, this feature should allow us to say to *git*: “Hey, from now on whenever I `git add` a `*.png` file, add it to *git-annex* instead!”
+
+How about saying to *git-annex*: “Hey, whenever I `git-annex add` a file which is *not* `*.png`, add it to *git* instead! Or at least leave it unadded so that I can decide later.” Is it possible now? If not, would it be reasonable to add such a feature?
+"""]]
diff --git a/doc/todo/smudge/comment_5_80bbc3710f9a18644571c6dd60baf4e5._comment b/doc/todo/smudge/comment_5_80bbc3710f9a18644571c6dd60baf4e5._comment
new file mode 100644
index 0000000000..04904dc7c1
--- /dev/null
+++ b/doc/todo/smudge/comment_5_80bbc3710f9a18644571c6dd60baf4e5._comment
@@ -0,0 +1,7 @@
+[[!comment format=mdwn
+ username="joey"
+ subject="""comment 5"""
+ date="2015-04-29T15:30:50Z"
+ content="""
+That feature already exists (annex.largefiles).
+"""]]
diff --git a/doc/todo/smudge/comment_6_86f536515575a6c2ed3a89c80b2c232f._comment b/doc/todo/smudge/comment_6_86f536515575a6c2ed3a89c80b2c232f._comment
new file mode 100644
index 0000000000..6a15157bc8
--- /dev/null
+++ b/doc/todo/smudge/comment_6_86f536515575a6c2ed3a89c80b2c232f._comment
@@ -0,0 +1,7 @@
+[[!comment format=mdwn
+ username="tomekwi"
+ subject="comment 6"
+ date="2015-04-29T20:38:51Z"
+ content="""
+Thanks! Looks great :)
+"""]]
diff --git a/doc/todo/smudge/comment_7_e428e4a1207d426a53e067fb5211510e._comment b/doc/todo/smudge/comment_7_e428e4a1207d426a53e067fb5211510e._comment
new file mode 100644
index 0000000000..f1fca6a9ef
--- /dev/null
+++ b/doc/todo/smudge/comment_7_e428e4a1207d426a53e067fb5211510e._comment
@@ -0,0 +1,22 @@
+[[!comment format=mdwn
+ username="torarnv@6179ecd599a0e00709a67306f015e46307a66eb6"
+ nickname="torarnv"
+ subject="Git 2.5 allows smudge filters to not read all of stdin"
+ date="2015-07-29T10:35:07Z"
+ content="""
+It seems git 2.5 allows smudge filters to not read all of stdin:
+
+https://github.com/git/git/blob/master/Documentation/RelNotes/2.5.0.txt
+
+\"
+ * Filter scripts were run with SIGPIPE disabled on the Git side,
+   expecting that they may not read what Git feeds them to filter.
+   We however treated a filter that does not read its input fully
+   before exiting as an error.  We no longer do and ignore EPIPE
+   when writing to feed the filter scripts.
+
+   This changes semantics, but arguably in a good way.  If a filter
+   can produce its output without fully consuming its input using
+   whatever magic, we now let it do so, instead of diagnosing it
+   as a programming error.\"
+"""]]
diff --git a/doc/todo/smudge/comment_8_ab39263b7145094a9eb4a86bb5410421._comment b/doc/todo/smudge/comment_8_ab39263b7145094a9eb4a86bb5410421._comment
new file mode 100644
index 0000000000..7f8882a504
--- /dev/null
+++ b/doc/todo/smudge/comment_8_ab39263b7145094a9eb4a86bb5410421._comment
@@ -0,0 +1,9 @@
+[[!comment format=mdwn
+ username="ExGen"
+ subject="Harlinks"
+ date="2016-04-02T23:32:24Z"
+ content="""
+Why not use **hardlinks** on Windows? This could fix \"only direct mode\" issue too. 
+Symlinks on windows sure want admin privilege but hardlinks won't. At least not with this tool: 
+
+"""]]
diff --git a/doc/todo/smudge/comment_9_e194d6371384f2db560f026d6ef728ff._comment b/doc/todo/smudge/comment_9_e194d6371384f2db560f026d6ef728ff._comment
new file mode 100644
index 0000000000..104fc6ffe6
--- /dev/null
+++ b/doc/todo/smudge/comment_9_e194d6371384f2db560f026d6ef728ff._comment
@@ -0,0 +1,7 @@
+[[!comment format=mdwn
+ username="ExGen"
+ subject="Harlinks"
+ date="2016-04-03T11:21:02Z"
+ content="""
+Also I tried now and confirmed both git's ln and cygwin's ln work on windows with hardlink, and it's supported since XP.
+"""]]
diff --git a/doc/todo/smudge/git-patches b/doc/todo/smudge/git-patches
new file mode 100644
index 0000000000..3736ba2e3e
--- /dev/null
+++ b/doc/todo/smudge/git-patches
@@ -0,0 +1,1643 @@
+From mairix@mairix Mon Jan  1 12:34:56 1970
+X-source-folder: /home/joey/mail/.git/annex/objects/VX/gq/SHA256E-s5263957--977af1c7373ecbca2f2df44e782afb143e71a38b0af965f02cd1178ab5b82b4e.gz/SHA256E-s5263957--977af1c7373ecbca2f2df44e782afb143e71a38b0af965f02cd1178ab5b82b4e.gz
+Return-Path: 
+X-Original-To: joeyh@joeyh.name
+Delivered-To: joey@kitenet.net
+X-Question: 42
+Authentication-Results: kitenet.net;
+	dkim=pass (1024-bit key; unprotected) header.d=joeyh.name header.i=@joeyh.name header.b=ier2ZdRe;
+	dkim-atps=neutral
+DKIM-Signature: v=1; a=rsa-sha256; c=simple/simple; d=joeyh.name; s=mail;
+	t=1468277197; bh=kOlU+hQ5LyTEz/eutsD10qEeMefFCuV9/88MgBjyH3k=;
+	h=From:To:Cc:Subject:Date:From;
+	b=ier2ZdReiz3+8cyLyfBhIuFH0zribYdza324RRso423MVVwX5Z5dXjkqHvfVpRxVA
+	 SLJCxSfGmOuiiQikb4CcFpmTi8rt+gcLqcgvQGgQMug+ee8CysZVHW76EpSILa8eIN
+	 1qIOb5q8Q8fHMaRifEWQvbQiw4XTzTdDJZdyPMIE=
+From: Joey Hess 
+To: git@vger.kernel.org
+Cc: Joey Hess 
+Subject: [PATCH v5 0/8] extend smudge/clean filters with direct file access (for pu)
+Date: Mon, 11 Jul 2016 18:45:04 -0400
+Message-Id: <1468277112-9909-1-git-send-email-joeyh@joeyh.name>
+X-Mailer: git-send-email 2.8.1
+X-Spam-Status: No, score=-95.2 required=5.0 tests=DKIM_SIGNED,DKIM_VALID,
+	DKIM_VALID_AU,HEADER_FROM_DIFFERENT_DOMAINS,RCVD_IN_DNSWL_BLOCKED,RCVD_IN_PBL,
+	RCVD_IN_SORBS_DUL,RDNS_DYNAMIC,SPF_SOFTFAIL,URIBL_BLOCKED,USER_IN_WHITELIST
+	autolearn=no autolearn_force=no version=3.4.1
+X-Spam-Checker-Version: SpamAssassin 3.4.1 (2015-04-28) on kite.kitenet.net
+Content-Length: 1441
+Lines: 40
+
+Back from vacation with a reroll of jh/clean-smudge-annex.
+
+Deals with conflicting changes from cc/apply-am in pu.
+
+Since tb/convert-peek-in-index is not currently in pu, this reroll isn't
+based on it, and will conflict if that topic gets added back into pu.
+Not sure what the status of tb/convert-peek-in-index is at this point?
+
+Improvements from Junio's review:
+
+	fix build with DEVELOPER=1
+	style fixes
+	use test_cmp in test cases
+	improve robustness of a test case
+	clean up some confusing code
+	small performance tweak
+
+Joey Hess (8):
+  clarify %f documentation
+  add smudgeToFile and cleanFromFile filter configs
+  use cleanFromFile in git add
+  use smudgeToFile in git checkout etc
+  warn on unusable smudgeToFile/cleanFromFile config
+  better recovery from failure of smudgeToFile filter
+  use smudgeToFile filter in git am
+  use smudgeToFile filter in recursive merge
+
+ Documentation/config.txt        |  18 ++++-
+ Documentation/gitattributes.txt |  42 ++++++++++++
+ apply.c                         |  16 +++++
+ convert.c                       | 148 ++++++++++++++++++++++++++++++++++++----
+ convert.h                       |  10 +++
+ entry.c                         |  59 ++++++++++++----
+ merge-recursive.c               |  53 +++++++++++---
+ sha1_file.c                     |  42 ++++++++++--
+ t/t0021-conversion.sh           | 117 +++++++++++++++++++++++++++++++
+ 9 files changed, 459 insertions(+), 46 deletions(-)
+
+-- 
+2.8.1
+
+From mairix@mairix Mon Jan  1 12:34:56 1970
+X-source-folder: /home/joey/mail/.git/annex/objects/0k/FJ/SHA256E-s162778--6d2a0fc6afd1077eccdca2e92f314c00ef70a06273242429035e0ce52cce2e13.gz/SHA256E-s162778--6d2a0fc6afd1077eccdca2e92f314c00ef70a06273242429035e0ce52cce2e13.gz
+Return-Path: 
+X-Original-To: joey@kitenet.net
+Delivered-To: joey@kitenet.net
+Received: from vger.kernel.org (vger.kernel.org [209.132.180.67])
+	by kitenet.net (Postfix) with ESMTP id B1F5B1C677
+	for ; Mon, 11 Jul 2016 18:46:53 -0400 (EDT)
+Authentication-Results: kitenet.net;
+	dkim=fail reason="signature verification failed" (1024-bit key; unprotected) header.d=joeyh.name header.i=@joeyh.name header.b=hbTM55ad;
+	dkim-atps=neutral
+Received: (majordomo@vger.kernel.org) by vger.kernel.org via listexpand
+	id S1752420AbcGKWqv (ORCPT );
+	Mon, 11 Jul 2016 18:46:51 -0400
+Received: from kitenet.net ([66.228.36.95]:60934 "EHLO kitenet.net"
+	rhost-flags-OK-OK-OK-OK) by vger.kernel.org with ESMTP
+	id S1751854AbcGKWqu (ORCPT );
+	Mon, 11 Jul 2016 18:46:50 -0400
+X-Question: 42
+DKIM-Signature:	v=1; a=rsa-sha256; c=simple/simple; d=joeyh.name; s=mail;
+	t=1468277197; bh=Q37/p1l59xZISgA1LWyXTUqwJEg7T96CGx2mCEyU9UM=;
+	h=From:To:Cc:Subject:Date:In-Reply-To:References:From;
+	b=hbTM55adTxAE7hwBgj9q7PkR66nfsMxhWSpQn/R9iSUOaJ11fW10rEUxkLzcXKK5x
+	 ktVQ6sdgyCC5CbgYfBn6sMJ+EH6sTTJlLS0czzSkl453Izvk85rBtBGYsVSCzgbPai
+	 1rM45vbD/pViJQ2IUyiT/WxLexQFFn0DnVLiWuWA=
+From:	Joey Hess 
+To:	git@vger.kernel.org
+Cc:	Joey Hess 
+Subject: [PATCH v5 1/8] clarify %f documentation
+Date:	Mon, 11 Jul 2016 18:45:05 -0400
+Message-Id: <1468277112-9909-2-git-send-email-joeyh@joeyh.name>
+X-Mailer: git-send-email 2.8.1
+In-Reply-To: <1468277112-9909-1-git-send-email-joeyh@joeyh.name>
+References: <1468277112-9909-1-git-send-email-joeyh@joeyh.name>
+X-Spam-Status: No, score=-8.2 required=10.0 tests=BAYES_00,DKIM_SIGNED,
+	HEADER_FROM_DIFFERENT_DOMAINS,RCVD_IN_DNSWL_HI,RP_MATCHES_RCVD,T_DKIM_INVALID,
+	URIBL_BLOCKED autolearn=ham autolearn_force=no version=3.4.1
+X-Spam-Checker-Version: SpamAssassin 3.4.1 (2015-04-28) on kite.kitenet.net
+Sender:	git-owner@vger.kernel.org
+Precedence: bulk
+List-ID: 
+X-Mailing-List:	git@vger.kernel.org
+Content-Length: 1148
+Lines: 31
+
+It's natural to expect %f to be an actual file on disk; help avoid that
+mistake.
+
+Signed-off-by: Joey Hess 
+---
+ Documentation/gitattributes.txt | 5 +++++
+ 1 file changed, 5 insertions(+)
+
+diff --git a/Documentation/gitattributes.txt b/Documentation/gitattributes.txt
+index f2afdb6..197ece8 100644
+--- a/Documentation/gitattributes.txt
++++ b/Documentation/gitattributes.txt
+@@ -379,6 +379,11 @@ substitution.  For example:
+ 	smudge = git-p4-filter --smudge %f
+ ------------------------
+ 
++Note that "%f" is the name of the path that is being worked on. Depending
++on the version that is being filtered, the corresponding file on disk may
++not exist, or may have different contents. So, smudge and clean commands
++should not try to access the file on disk, but only act as filters on the
++content provided to them on standard input.
+ 
+ Interaction between checkin/checkout attributes
+ ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+-- 
+2.8.1
+
+--
+To unsubscribe from this list: send the line "unsubscribe git" in
+the body of a message to majordomo@vger.kernel.org
+More majordomo info at  http://vger.kernel.org/majordomo-info.html
+
+From mairix@mairix Mon Jan  1 12:34:56 1970
+X-source-folder: /home/joey/mail/.git/annex/objects/0k/FJ/SHA256E-s162778--6d2a0fc6afd1077eccdca2e92f314c00ef70a06273242429035e0ce52cce2e13.gz/SHA256E-s162778--6d2a0fc6afd1077eccdca2e92f314c00ef70a06273242429035e0ce52cce2e13.gz
+Return-Path: 
+X-Original-To: joey@kitenet.net
+Delivered-To: joey@kitenet.net
+Received: from vger.kernel.org (vger.kernel.org [209.132.180.67])
+	by kitenet.net (Postfix) with ESMTP id 04F251C677
+	for ; Mon, 11 Jul 2016 18:47:07 -0400 (EDT)
+Authentication-Results: kitenet.net;
+	dkim=fail reason="signature verification failed" (1024-bit key; unprotected) header.d=joeyh.name header.i=@joeyh.name header.b=NOnwhtEt;
+	dkim-atps=neutral
+Received: (majordomo@vger.kernel.org) by vger.kernel.org via listexpand
+	id S1752532AbcGKWrD (ORCPT );
+	Mon, 11 Jul 2016 18:47:03 -0400
+Received: from kitenet.net ([66.228.36.95]:60970 "EHLO kitenet.net"
+	rhost-flags-OK-OK-OK-OK) by vger.kernel.org with ESMTP
+	id S1752439AbcGKWrA (ORCPT );
+	Mon, 11 Jul 2016 18:47:00 -0400
+X-Question: 42
+DKIM-Signature:	v=1; a=rsa-sha256; c=simple/simple; d=joeyh.name; s=mail;
+	t=1468277198; bh=NRbzz9lX+l0UUh9pAD1CQEaHhmhOg2qk05TuAMiBsPo=;
+	h=From:To:Cc:Subject:Date:In-Reply-To:References:From;
+	b=NOnwhtEtJu+wBwiPrABDPIiic0tTHORjOkouxfQbD+rgbTjsY2mKzfmOsW0JdL5c9
+	 UyIeOOTmDA8jFAvt/o/9Yj7P39nbgg7fjX/tS8xd5oYSRV15Vg0ePEZ98nSHu+KjOR
+	 q06IOF8YyqtTAz4S3lSXXlNFfpqrFw8bQT4Xhq6Y=
+From:	Joey Hess 
+To:	git@vger.kernel.org
+Cc:	Joey Hess 
+Subject: [PATCH v5 2/8] add smudgeToFile and cleanFromFile filter configs
+Date:	Mon, 11 Jul 2016 18:45:06 -0400
+Message-Id: <1468277112-9909-3-git-send-email-joeyh@joeyh.name>
+X-Mailer: git-send-email 2.8.1
+In-Reply-To: <1468277112-9909-1-git-send-email-joeyh@joeyh.name>
+References: <1468277112-9909-1-git-send-email-joeyh@joeyh.name>
+X-Spam-Status: No, score=-8.2 required=10.0 tests=BAYES_00,DKIM_SIGNED,
+	HEADER_FROM_DIFFERENT_DOMAINS,RCVD_IN_DNSWL_HI,RP_MATCHES_RCVD,T_DKIM_INVALID,
+	URIBL_BLOCKED autolearn=ham autolearn_force=no version=3.4.1
+X-Spam-Checker-Version: SpamAssassin 3.4.1 (2015-04-28) on kite.kitenet.net
+Sender:	git-owner@vger.kernel.org
+Precedence: bulk
+List-ID: 
+X-Mailing-List:	git@vger.kernel.org
+Content-Length: 13634
+Lines: 374
+
+This adds new smudgeToFile and cleanFromFile filter commands,
+which are similar to smudge and clean but allow direct access to files on
+disk.
+
+This interface can be much more efficient when operating on large files,
+because the whole file content does not need to be streamed through the
+filter. It even allows for things like cleanFromFile commands that avoid
+reading the whole content of the file, and for smudgeToFile commands that
+populate a work tree file using an efficient Copy On Write operation.
+
+The new filter commands will not be used for all filtering. They are
+efficient to use when git add is adding a file, or when the work tree is
+being updated, but not a good fit when git is internally filtering blob
+objects in memory for eg, a diff.
+
+So, a user who wants to use smudgeToFile should also provide a smudge
+command to be used in cases where smudgeToFile is not used. And ditto
+with cleanFromFile and clean. To avoid foot-shooting configurations, the
+new commands are not used unless the old commands are also configured.
+
+That also ensures that a filter driver configuration that includes these
+new commands will work, although less efficiently, when used with an older
+version of git that does not support them.
+
+Signed-off-by: Joey Hess 
+---
+ Documentation/config.txt        |  18 ++++++-
+ Documentation/gitattributes.txt |  37 ++++++++++++++
+ convert.c                       | 111 +++++++++++++++++++++++++++++++++++-----
+ convert.h                       |  10 ++++
+ 4 files changed, 160 insertions(+), 16 deletions(-)
+
+diff --git a/Documentation/config.txt b/Documentation/config.txt
+index 19493aa..a55bed8 100644
+--- a/Documentation/config.txt
++++ b/Documentation/config.txt
+@@ -1325,15 +1325,29 @@ format.useAutoBase::
+ 	format-patch by default.
+ 
+ filter..clean::
+-	The command which is used to convert the content of a worktree
++	The command which is used as a filter to convert the content of a worktree
+ 	file to a blob upon checkin.  See linkgit:gitattributes[5] for
+ 	details.
+ 
+ filter..smudge::
+-	The command which is used to convert the content of a blob
++	The command which is used as a filter to convert the content of a blob
+ 	object to a worktree file upon checkout.  See
+ 	linkgit:gitattributes[5] for details.
+ 
++filter..cleanFromFile::
++	Similar to filter..clean but the specified command
++	directly accesses a worktree file on disk, rather than
++	receiving the file content from standard input.
++	Only used when filter..clean is also configured.
++	See linkgit:gitattributes[5] for details.
++
++filter..smudgeToFile::
++	Similar to filter..smudge but the specified command
++	writes the content of a blob directly to a worktree file,
++	rather than to standard output.
++	Only used when filter..smudge is also configured.
++	See linkgit:gitattributes[5] for details.
++
+ fsck.::
+ 	Allows overriding the message type (error, warn or ignore) of a
+ 	specific message ID such as `missingEmail`.
+diff --git a/Documentation/gitattributes.txt b/Documentation/gitattributes.txt
+index 197ece8..a58aafc 100644
+--- a/Documentation/gitattributes.txt
++++ b/Documentation/gitattributes.txt
+@@ -385,6 +385,43 @@ not exist, or may have different contents. So, smudge and clean commands
+ should not try to access the file on disk, but only act as filters on the
+ content provided to them on standard input.
+ 
++There are two extra commands "cleanFromFile" and "smudgeToFile", which
++can optionally be set in a filter driver. These are similar to the "clean"
++and "smudge" commands, but avoid needing to pipe the contents of files
++through the filters, and instead read/write files in the filesystem.
++This can be more efficient when using filters with large files that are not
++directly stored in the repository.
++
++Both "cleanFromFile" and "smudgeToFile" are provided a path as an
++added parameter after the configured command line.
++
++The "cleanFromFile" command is provided the path to the file that
++it should clean. Like the "clean" command, it should output the cleaned
++version to standard output.
++
++The "smudgeToFile" command is provided a path to the file that it
++should write to. (This file will already exist, as an empty file that can
++be written to or replaced.) Like the "smudge" command, "smudgeToFile"
++is fed the blob object from its standard input.
++
++Some git operations that need to apply filters cannot use "cleanFromFile"
++and "smudgeToFile", since the files are not present to disk. So, to avoid
++inconsistent behavior, "cleanFromFile" will only be used if "clean" is
++also configured, and "smudgeToFile" will only be used if "smudge" is also
++configured.
++
++An example large file storage filter driver using cleanFromFile and
++smudgeToFile follows:
++
++------------------------
++[filter "bigfiles"]
++	clean = store-bigfile --from-stdin
++	cleanFromFile = store-bigfile --from-file
++	smudge = retrieve-bigfile --to-stdout
++	smudgeToFile = retrieve-bigfile --to-file
++	required
++------------------------
++
+ Interaction between checkin/checkout attributes
+ ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+ 
+diff --git a/convert.c b/convert.c
+index 214c99f..eb7774f 100644
+--- a/convert.c
++++ b/convert.c
+@@ -358,7 +358,8 @@ struct filter_params {
+ 	unsigned long size;
+ 	int fd;
+ 	const char *cmd;
+-	const char *path;
++	const char *path; /* Path within the git repository */
++	const char *fspath; /* Path to file on disk */
+ };
+ 
+ static int filter_buffer_or_fd(int in, int out, void *data)
+@@ -387,6 +388,15 @@ static int filter_buffer_or_fd(int in, int out, void *data)
+ 	strbuf_expand(&cmd, params->cmd, strbuf_expand_dict_cb, &dict);
+ 	strbuf_release(&path);
+ 
++	/* append fspath to the command if it's set, separated with a space */
++	if (params->fspath) {
++		struct strbuf fspath = STRBUF_INIT;
++		sq_quote_buf(&fspath, params->fspath);
++		strbuf_addstr(&cmd, " ");
++		strbuf_addbuf(&cmd, &fspath);
++		strbuf_release(&fspath);
++	}
++
+ 	argv[0] = cmd.buf;
+ 
+ 	child_process.argv = argv;
+@@ -425,7 +435,8 @@ static int filter_buffer_or_fd(int in, int out, void *data)
+ 	return (write_err || status);
+ }
+ 
+-static int apply_filter(const char *path, const char *src, size_t len, int fd,
++static int apply_filter(const char *path, const char *fspath,
++			const char *src, size_t len, int fd,
+                         struct strbuf *dst, const char *cmd)
+ {
+ 	/*
+@@ -454,6 +465,7 @@ static int apply_filter(const char *path, const char *src, size_t len, int fd,
+ 	params.fd = fd;
+ 	params.cmd = cmd;
+ 	params.path = path;
++	params.fspath = fspath;
+ 
+ 	fflush(NULL);
+ 	if (start_async(&async))
+@@ -484,6 +496,8 @@ static struct convert_driver {
+ 	struct convert_driver *next;
+ 	const char *smudge;
+ 	const char *clean;
++	const char *smudge_to_file;
++	const char *clean_from_file;
+ 	int required;
+ } *user_convert, **user_convert_tail;
+ 
+@@ -510,8 +524,9 @@ static int read_convert_config(const char *var, const char *value, void *cb)
+ 	}
+ 
+ 	/*
+-	 * filter..smudge and filter..clean specifies
+-	 * the command line:
++	 * filter..smudge, filter..clean,
++	 * filter..smudgeToFile, filter..cleanFromFile
++	 * specifies the command line:
+ 	 *
+ 	 *	command-line
+ 	 *
+@@ -524,6 +539,12 @@ static int read_convert_config(const char *var, const char *value, void *cb)
+ 	if (!strcmp("clean", key))
+ 		return git_config_string(&drv->clean, var, value);
+ 
++	if (!strcmp("smudgetofile", key))
++		return git_config_string(&drv->smudge_to_file, var, value);
++
++	if (!strcmp("cleanfromfile", key))
++		return git_config_string(&drv->clean_from_file, var, value);
++
+ 	if (!strcmp("required", key)) {
+ 		drv->required = git_config_bool(var, value);
+ 		return 0;
+@@ -821,7 +842,37 @@ int would_convert_to_git_filter_fd(const char *path)
+ 	if (!ca.drv->required)
+ 		return 0;
+ 
+-	return apply_filter(path, NULL, 0, -1, NULL, ca.drv->clean);
++	return apply_filter(path, NULL, NULL, 0, -1, NULL, ca.drv->clean);
++}
++
++int can_clean_from_file(const char *path)
++{
++	struct conv_attrs ca;
++
++	convert_attrs(&ca, path);
++	if (!ca.drv)
++		return 0;
++
++	/*
++	 * Only use the cleanFromFile filter when the clean filter is also
++	 * configured.
++	 */
++	return (ca.drv->clean_from_file && ca.drv->clean);
++}
++
++int can_smudge_to_file(const char *path)
++{
++	struct conv_attrs ca;
++
++	convert_attrs(&ca, path);
++	if (!ca.drv)
++		return 0;
++
++	/*
++	 * Only use the smudgeToFile filter when the smudge filter is also
++	 * configured.
++	 */
++	return (ca.drv->smudge_to_file && ca.drv->smudge);
+ }
+ 
+ const char *get_convert_attr_ascii(const char *path)
+@@ -864,7 +915,7 @@ int convert_to_git(const char *path, const char *src, size_t len,
+ 		required = ca.drv->required;
+ 	}
+ 
+-	ret |= apply_filter(path, src, len, -1, dst, filter);
++	ret |= apply_filter(path, NULL, src, len, -1, dst, filter);
+ 	if (!ret && required)
+ 		die("%s: clean filter '%s' failed", path, ca.drv->name);
+ 
+@@ -889,14 +940,34 @@ void convert_to_git_filter_fd(const char *path, int fd, struct strbuf *dst,
+ 	assert(ca.drv);
+ 	assert(ca.drv->clean);
+ 
+-	if (!apply_filter(path, NULL, 0, fd, dst, ca.drv->clean))
++	if (!apply_filter(path, NULL, NULL, 0, fd, dst, ca.drv->clean))
+ 		die("%s: clean filter '%s' failed", path, ca.drv->name);
+ 
+ 	crlf_to_git(path, dst->buf, dst->len, dst, ca.crlf_action, checksafe);
+ 	ident_to_git(path, dst->buf, dst->len, dst, ca.ident);
+ }
+ 
+-static int convert_to_working_tree_internal(const char *path, const char *src,
++void convert_to_git_filter_from_file(const char *path, struct strbuf *dst,
++				   enum safe_crlf checksafe)
++{
++	struct conv_attrs ca;
++	convert_attrs(&ca, path);
++
++	assert(ca.drv);
++	assert(ca.drv->clean);
++	assert(ca.drv->clean_from_file);
++
++	if (!apply_filter(path, path, "", 0, -1, dst, ca.drv->clean_from_file))
++		die("%s: cleanFromFile filter '%s' failed", path, ca.drv->name);
++
++	crlf_to_git(path, dst->buf, dst->len, dst, ca.crlf_action,
++		checksafe);
++	ident_to_git(path, dst->buf, dst->len, dst, ca.ident);
++}
++
++static int convert_to_working_tree_internal(const char *path,
++					    const char *destpath,
++					    const char *src,
+ 					    size_t len, struct strbuf *dst,
+ 					    int normalizing)
+ {
+@@ -907,7 +978,10 @@ static int convert_to_working_tree_internal(const char *path, const char *src,
+ 
+ 	convert_attrs(&ca, path);
+ 	if (ca.drv) {
+-		filter = ca.drv->smudge;
++		if (destpath)
++			filter = ca.drv->smudge_to_file;
++		else
++			filter = ca.drv->smudge;
+ 		required = ca.drv->required;
+ 	}
+ 
+@@ -918,7 +992,7 @@ static int convert_to_working_tree_internal(const char *path, const char *src,
+ 	}
+ 	/*
+ 	 * CRLF conversion can be skipped if normalizing, unless there
+-	 * is a smudge filter.  The filter might expect CRLFs.
++	 * is a filter.  The filter might expect CRLFs.
+ 	 */
+ 	if (filter || !normalizing) {
+ 		ret |= crlf_to_worktree(path, src, len, dst, ca.crlf_action);
+@@ -928,21 +1002,30 @@ static int convert_to_working_tree_internal(const char *path, const char *src,
+ 		}
+ 	}
+ 
+-	ret_filter = apply_filter(path, src, len, -1, dst, filter);
++	ret_filter = apply_filter(path, destpath, src, len, -1, dst, filter);
+ 	if (!ret_filter && required)
+-		die("%s: smudge filter %s failed", path, ca.drv->name);
++		die("%s: %s filter %s failed", path, destpath ? "smudgeToFile" : "smudge", ca.drv->name);
+ 
+ 	return ret | ret_filter;
+ }
+ 
+ int convert_to_working_tree(const char *path, const char *src, size_t len, struct strbuf *dst)
+ {
+-	return convert_to_working_tree_internal(path, src, len, dst, 0);
++	return convert_to_working_tree_internal(path, NULL, src, len, dst, 0);
++}
++
++int convert_to_working_tree_filter_to_file(const char *path, const char *destpath, const char *src, size_t len)
++{
++	struct strbuf output = STRBUF_INIT;
++	int ret = convert_to_working_tree_internal(path, destpath, src, len, &output, 0);
++	/* The smudgeToFile filter stdout is not used. */
++	strbuf_release(&output);
++	return ret;
+ }
+ 
+ int renormalize_buffer(const char *path, const char *src, size_t len, struct strbuf *dst)
+ {
+-	int ret = convert_to_working_tree_internal(path, src, len, dst, 1);
++	int ret = convert_to_working_tree_internal(path, NULL, src, len, dst, 1);
+ 	if (ret) {
+ 		src = dst->buf;
+ 		len = dst->len;
+diff --git a/convert.h b/convert.h
+index 82871a1..6f46d10 100644
+--- a/convert.h
++++ b/convert.h
+@@ -42,6 +42,10 @@ extern int convert_to_git(const char *path, const char *src, size_t len,
+ 			  struct strbuf *dst, enum safe_crlf checksafe);
+ extern int convert_to_working_tree(const char *path, const char *src,
+ 				   size_t len, struct strbuf *dst);
++extern int convert_to_working_tree_filter_to_file(const char *path,
++						  const char *destpath,
++						  const char *src,
++						  size_t len);
+ extern int renormalize_buffer(const char *path, const char *src, size_t len,
+ 			      struct strbuf *dst);
+ static inline int would_convert_to_git(const char *path)
+@@ -53,6 +57,12 @@ extern void convert_to_git_filter_fd(const char *path, int fd,
+ 				     struct strbuf *dst,
+ 				     enum safe_crlf checksafe);
+ extern int would_convert_to_git_filter_fd(const char *path);
++/* Precondition: can_clean_from_file(path) == true */
++extern void convert_to_git_filter_from_file(const char *path,
++					    struct strbuf *dst,
++					    enum safe_crlf checksafe);
++extern int can_clean_from_file(const char *path);
++extern int can_smudge_to_file(const char *path);
+ 
+ /*****************************************************************
+  *
+-- 
+2.8.1
+
+--
+To unsubscribe from this list: send the line "unsubscribe git" in
+the body of a message to majordomo@vger.kernel.org
+More majordomo info at  http://vger.kernel.org/majordomo-info.html
+
+From mairix@mairix Mon Jan  1 12:34:56 1970
+X-source-folder: /home/joey/mail/.git/annex/objects/0k/FJ/SHA256E-s162778--6d2a0fc6afd1077eccdca2e92f314c00ef70a06273242429035e0ce52cce2e13.gz/SHA256E-s162778--6d2a0fc6afd1077eccdca2e92f314c00ef70a06273242429035e0ce52cce2e13.gz
+Return-Path: 
+X-Original-To: joey@kitenet.net
+Delivered-To: joey@kitenet.net
+Received: from vger.kernel.org (vger.kernel.org [209.132.180.67])
+	by kitenet.net (Postfix) with ESMTP id 92F4B1C677
+	for ; Mon, 11 Jul 2016 18:47:02 -0400 (EDT)
+Authentication-Results: kitenet.net;
+	dkim=fail reason="signature verification failed" (1024-bit key; unprotected) header.d=joeyh.name header.i=@joeyh.name header.b=O21fZnmu;
+	dkim-atps=neutral
+Received: (majordomo@vger.kernel.org) by vger.kernel.org via listexpand
+	id S1752507AbcGKWq7 (ORCPT );
+	Mon, 11 Jul 2016 18:46:59 -0400
+Received: from kitenet.net ([66.228.36.95]:60948 "EHLO kitenet.net"
+	rhost-flags-OK-OK-OK-OK) by vger.kernel.org with ESMTP
+	id S1752439AbcGKWq6 (ORCPT );
+	Mon, 11 Jul 2016 18:46:58 -0400
+X-Question: 42
+DKIM-Signature:	v=1; a=rsa-sha256; c=simple/simple; d=joeyh.name; s=mail;
+	t=1468277197; bh=ID7Cvl2BMx/TZGvnWWIJ3y9tfZmZmZ0hcgqOsxLVm9Q=;
+	h=From:To:Cc:Subject:Date:In-Reply-To:References:From;
+	b=O21fZnmuEK0afkVNZdA2KL+bhF+skap1V8rdhqtR0vzDPg7cri/BlGPK8eY5G/xi6
+	 7f7vYgrq3QCN/0Q7cXsHSm0k8sgA2DLnriNR0Ga/gtk6OjqDr2JNHEMDoEt6MMyVto
+	 4wAJLa+dzSUA8Y+8bwZ760DVTucDOLRoEINUvJ0Y=
+From:	Joey Hess 
+To:	git@vger.kernel.org
+Cc:	Joey Hess 
+Subject: [PATCH v5 3/8] use cleanFromFile in git add
+Date:	Mon, 11 Jul 2016 18:45:07 -0400
+Message-Id: <1468277112-9909-4-git-send-email-joeyh@joeyh.name>
+X-Mailer: git-send-email 2.8.1
+In-Reply-To: <1468277112-9909-1-git-send-email-joeyh@joeyh.name>
+References: <1468277112-9909-1-git-send-email-joeyh@joeyh.name>
+X-Spam-Status: No, score=-8.2 required=10.0 tests=BAYES_00,DKIM_SIGNED,
+	HEADER_FROM_DIFFERENT_DOMAINS,RCVD_IN_DNSWL_HI,RP_MATCHES_RCVD,T_DKIM_INVALID,
+	URIBL_BLOCKED autolearn=ham autolearn_force=no version=3.4.1
+X-Spam-Checker-Version: SpamAssassin 3.4.1 (2015-04-28) on kite.kitenet.net
+Sender:	git-owner@vger.kernel.org
+Precedence: bulk
+List-ID: 
+X-Mailing-List:	git@vger.kernel.org
+Content-Length: 3641
+Lines: 127
+
+Includes test cases.
+
+Signed-off-by: Joey Hess 
+---
+ sha1_file.c           | 42 ++++++++++++++++++++++++++++++++++++------
+ t/t0021-conversion.sh | 36 ++++++++++++++++++++++++++++++++++++
+ 2 files changed, 72 insertions(+), 6 deletions(-)
+
+diff --git a/sha1_file.c b/sha1_file.c
+index 2fc22b0..549a20f 100644
+--- a/sha1_file.c
++++ b/sha1_file.c
+@@ -3335,6 +3335,29 @@ static int index_stream_convert_blob(unsigned char *sha1, int fd,
+ 	return ret;
+ }
+ 
++static int index_from_file_convert_blob(unsigned char *sha1,
++				      const char *path, unsigned flags)
++{
++	int ret;
++	const int write_object = flags & HASH_WRITE_OBJECT;
++	struct strbuf sbuf = STRBUF_INIT;
++
++	assert(path);
++	assert(can_clean_from_file(path));
++
++	convert_to_git_filter_from_file(path, &sbuf,
++				 write_object ? safe_crlf : SAFE_CRLF_FALSE);
++
++	if (write_object)
++		ret = write_sha1_file(sbuf.buf, sbuf.len, typename(OBJ_BLOB),
++				      sha1);
++	else
++		ret = hash_sha1_file(sbuf.buf, sbuf.len, typename(OBJ_BLOB),
++				     sha1);
++	strbuf_release(&sbuf);
++	return ret;
++}
++
+ static int index_pipe(unsigned char *sha1, int fd, enum object_type type,
+ 		      const char *path, unsigned flags)
+ {
+@@ -3427,12 +3450,19 @@ int index_path(unsigned char *sha1, const char *path, struct stat *st, unsigned
+ 
+ 	switch (st->st_mode & S_IFMT) {
+ 	case S_IFREG:
+-		fd = open(path, O_RDONLY);
+-		if (fd < 0)
+-			return error_errno("open(\"%s\")", path);
+-		if (index_fd(sha1, fd, st, OBJ_BLOB, path, flags) < 0)
+-			return error("%s: failed to insert into database",
+-				     path);
++		if (can_clean_from_file(path)) {
++			if (index_from_file_convert_blob(sha1, path, flags) < 0)
++				return error("%s: failed to insert into database",
++					     path);
++		}
++		else {
++			fd = open(path, O_RDONLY);
++			if (fd < 0)
++				return error_errno("open(\"%s\")", path);
++			if (index_fd(sha1, fd, st, OBJ_BLOB, path, flags) < 0)
++				return error("%s: failed to insert into database",
++					     path);
++		}
+ 		break;
+ 	case S_IFLNK:
+ 		if (strbuf_readlink(&sb, path, st->st_size))
+diff --git a/t/t0021-conversion.sh b/t/t0021-conversion.sh
+index 7bac2bc..bd84b80 100755
+--- a/t/t0021-conversion.sh
++++ b/t/t0021-conversion.sh
+@@ -12,6 +12,14 @@ tr \
+ EOF
+ chmod +x rot13.sh
+ 
++cat <rot13-from-file.sh
++#!$SHELL_PATH
++fsfile="\$1"
++touch rot13-from-file.ran
++cat "\$fsfile" | ./rot13.sh
++EOF
++chmod +x rot13-from-file.sh
++
+ test_expect_success setup '
+ 	git config filter.rot13.smudge ./rot13.sh &&
+ 	git config filter.rot13.clean ./rot13.sh &&
+@@ -268,4 +276,32 @@ test_expect_success 'disable filter with empty override' '
+ 	test_must_be_empty err
+ '
+ 
++test_expect_success 'cleanFromFile filter is used when adding a file' '
++	test_config filter.rot13.cleanFromFile ./rot13-from-file.sh &&
++
++	echo "*.t filter=rot13" >.gitattributes &&
++
++	cat test >fstest.t &&
++	git add fstest.t &&
++	test -e rot13-from-file.ran &&
++	rm -f rot13-from-file.ran &&
++
++	rm -f fstest.t &&
++	git checkout -- fstest.t &&
++	test_cmp test fstest.t
++'
++
++test_expect_success 'cleanFromFile filter is not used when clean filter is not configured' '
++	test_config filter.noclean.smudge ./rot13.sh &&
++	test_config filter.noclean.cleanFromFile ./rot13-from-file.sh &&
++
++	echo "*.no filter=noclean" >.gitattributes &&
++
++	cat test >test.no &&
++	git add test.no &&
++	test ! -e rot13-from-file.ran &&
++	git cat-file blob :test.no >actual &&
++	test_cmp test actual
++'
++
+ test_done
+-- 
+2.8.1
+
+--
+To unsubscribe from this list: send the line "unsubscribe git" in
+the body of a message to majordomo@vger.kernel.org
+More majordomo info at  http://vger.kernel.org/majordomo-info.html
+
+From mairix@mairix Mon Jan  1 12:34:56 1970
+X-source-folder: /home/joey/mail/.git/annex/objects/0k/FJ/SHA256E-s162778--6d2a0fc6afd1077eccdca2e92f314c00ef70a06273242429035e0ce52cce2e13.gz/SHA256E-s162778--6d2a0fc6afd1077eccdca2e92f314c00ef70a06273242429035e0ce52cce2e13.gz
+Return-Path: 
+X-Original-To: joey@kitenet.net
+Delivered-To: joey@kitenet.net
+Received: from vger.kernel.org (vger.kernel.org [209.132.180.67])
+	by kitenet.net (Postfix) with ESMTP id 363761C677
+	for ; Mon, 11 Jul 2016 18:47:18 -0400 (EDT)
+Authentication-Results: kitenet.net;
+	dkim=fail reason="signature verification failed" (1024-bit key; unprotected) header.d=joeyh.name header.i=@joeyh.name header.b=P+WpFgpS;
+	dkim-atps=neutral
+Received: (majordomo@vger.kernel.org) by vger.kernel.org via listexpand
+	id S1752612AbcGKWrO (ORCPT );
+	Mon, 11 Jul 2016 18:47:14 -0400
+Received: from kitenet.net ([66.228.36.95]:32768 "EHLO kitenet.net"
+	rhost-flags-OK-OK-OK-OK) by vger.kernel.org with ESMTP
+	id S1752439AbcGKWrN (ORCPT );
+	Mon, 11 Jul 2016 18:47:13 -0400
+X-Question: 42
+DKIM-Signature:	v=1; a=rsa-sha256; c=simple/simple; d=joeyh.name; s=mail;
+	t=1468277198; bh=tAWpo3wY2ayI3N2P5PIoaKI4XKz6M0EPI7WSh1kSoZc=;
+	h=From:To:Cc:Subject:Date:In-Reply-To:References:From;
+	b=P+WpFgpSST+oHI+yvWOFavctP60LP4dRf0CJA1N75Wk/pQIHue3j4sxVPNsjTAuj6
+	 IzUxRBE2Ilz3E0UF3r6iL2lCVMtITzMvvTqYjr9xEXi8mGgInDOWD1bi7bdmt2NY1z
+	 ZlAIY9kjZhF+1jmpxhc1YP+tkocCMOsmT/DbiGRI=
+From:	Joey Hess 
+To:	git@vger.kernel.org
+Cc:	Joey Hess 
+Subject: [PATCH v5 4/8] use smudgeToFile in git checkout etc
+Date:	Mon, 11 Jul 2016 18:45:08 -0400
+Message-Id: <1468277112-9909-5-git-send-email-joeyh@joeyh.name>
+X-Mailer: git-send-email 2.8.1
+In-Reply-To: <1468277112-9909-1-git-send-email-joeyh@joeyh.name>
+References: <1468277112-9909-1-git-send-email-joeyh@joeyh.name>
+X-Spam-Status: No, score=-8.2 required=10.0 tests=BAYES_00,DKIM_SIGNED,
+	HEADER_FROM_DIFFERENT_DOMAINS,RCVD_IN_DNSWL_HI,RP_MATCHES_RCVD,T_DKIM_INVALID,
+	URIBL_BLOCKED autolearn=ham autolearn_force=no version=3.4.1
+X-Spam-Checker-Version: SpamAssassin 3.4.1 (2015-04-28) on kite.kitenet.net
+Sender:	git-owner@vger.kernel.org
+Precedence: bulk
+List-ID: 
+X-Mailing-List:	git@vger.kernel.org
+Content-Length: 4367
+Lines: 147
+
+This makes git checkout, git reset, etc use smudgeToFile.
+
+Includes test cases.
+
+(There's a call to convert_to_working_tree in merge-recursive.c
+that could also be made to use smudgeToFile as well.)
+
+Signed-off-by: Joey Hess 
+---
+ entry.c               | 40 ++++++++++++++++++++++++++++++++--------
+ t/t0021-conversion.sh | 34 ++++++++++++++++++++++++++++++++--
+ 2 files changed, 64 insertions(+), 10 deletions(-)
+
+diff --git a/entry.c b/entry.c
+index 519e042..81d12a1 100644
+--- a/entry.c
++++ b/entry.c
+@@ -146,6 +146,7 @@ static int write_entry(struct cache_entry *ce,
+ 	unsigned long size;
+ 	size_t wrote, newsize = 0;
+ 	struct stat st;
++	int regular_file, smudge_to_file;
+ 
+ 	if (ce_mode_s_ifmt == S_IFREG) {
+ 		struct stream_filter *filter = get_stream_filter(ce->name, ce->sha1);
+@@ -175,8 +176,13 @@ static int write_entry(struct cache_entry *ce,
+ 
+ 		/*
+ 		 * Convert from git internal format to working tree format
++		 * unless the smudgeToFile filter can write to the
++		 * file directly.
+ 		 */
+-		if (ce_mode_s_ifmt == S_IFREG &&
++		regular_file = ce_mode_s_ifmt == S_IFREG;
++		smudge_to_file = regular_file
++			&& can_smudge_to_file(ce->name);
++		if (regular_file && !smudge_to_file &&
+ 		    convert_to_working_tree(ce->name, new, size, &buf)) {
+ 			free(new);
+ 			new = strbuf_detach(&buf, &newsize);
+@@ -189,13 +195,31 @@ static int write_entry(struct cache_entry *ce,
+ 			return error_errno("unable to create file %s", path);
+ 		}
+ 
+-		wrote = write_in_full(fd, new, size);
+-		if (!to_tempfile)
+-			fstat_done = fstat_output(fd, state, &st);
+-		close(fd);
+-		free(new);
+-		if (wrote != size)
+-			return error("unable to write file %s", path);
++		if (!smudge_to_file) {
++			wrote = write_in_full(fd, new, size);
++			if (!to_tempfile)
++				fstat_done = fstat_output(fd, state, &st);
++			close(fd);
++			free(new);
++			if (wrote != size)
++				return error("unable to write file %s", path);
++		}
++		else {
++			close(fd);
++			convert_to_working_tree_filter_to_file(ce->name, path, new, size);
++			free(new);
++			/*
++			 * The smudgeToFile filter may have replaced the
++			 * file; open it to make sure that the file
++			 * exists.
++			 */
++			fd = open(path, O_RDONLY);
++			if (fd < 0)
++				return error_errno("unable to create file %s", path);
++			if (!to_tempfile)
++				fstat_done = fstat_output(fd, state, &st);
++			close(fd);
++		}
+ 		break;
+ 	case S_IFGITLINK:
+ 		if (to_tempfile)
+diff --git a/t/t0021-conversion.sh b/t/t0021-conversion.sh
+index bd84b80..ea18b17 100755
+--- a/t/t0021-conversion.sh
++++ b/t/t0021-conversion.sh
+@@ -14,12 +14,20 @@ chmod +x rot13.sh
+ 
+ cat <rot13-from-file.sh
+ #!$SHELL_PATH
+-fsfile="\$1"
++srcfile="\$1"
+ touch rot13-from-file.ran
+-cat "\$fsfile" | ./rot13.sh
++cat "\$srcfile" | ./rot13.sh
+ EOF
+ chmod +x rot13-from-file.sh
+ 
++cat <rot13-to-file.sh
++#!$SHELL_PATH
++destfile="\$1"
++touch rot13-to-file.ran
++./rot13.sh >"\$destfile"
++EOF
++chmod +x rot13-to-file.sh
++
+ test_expect_success setup '
+ 	git config filter.rot13.smudge ./rot13.sh &&
+ 	git config filter.rot13.clean ./rot13.sh &&
+@@ -291,6 +299,17 @@ test_expect_success 'cleanFromFile filter is used when adding a file' '
+ 	test_cmp test fstest.t
+ '
+ 
++test_expect_success 'smudgeToFile filter is used when checking out a file' '
++	test_config filter.rot13.smudgeToFile ./rot13-to-file.sh &&
++
++	rm -f fstest.t &&
++	git checkout -- fstest.t &&
++	test_cmp test fstest.t &&
++
++	test -e rot13-to-file.ran &&
++	rm -f rot13-to-file.ran
++'
++
+ test_expect_success 'cleanFromFile filter is not used when clean filter is not configured' '
+ 	test_config filter.noclean.smudge ./rot13.sh &&
+ 	test_config filter.noclean.cleanFromFile ./rot13-from-file.sh &&
+@@ -304,4 +323,15 @@ test_expect_success 'cleanFromFile filter is not used when clean filter is not c
+ 	test_cmp test actual
+ '
+ 
++test_expect_success 'smudgeToFile filter is not used when smudge filter is not configured' '
++	test_config filter.nosmudge.clean ./rot13.sh &&
++	test_config filter.nosmudge.smudgeToFile ./rot13-to-file.sh &&
++
++	echo "*.no filter=nosmudge" >.gitattributes &&
++
++	rm -f fstest.t &&
++	git checkout -- fstest.t &&
++	test ! -e rot13-to-file.ran
++'
++
+ test_done
+-- 
+2.8.1
+
+--
+To unsubscribe from this list: send the line "unsubscribe git" in
+the body of a message to majordomo@vger.kernel.org
+More majordomo info at  http://vger.kernel.org/majordomo-info.html
+
+From mairix@mairix Mon Jan  1 12:34:56 1970
+X-source-folder: /home/joey/mail/.git/annex/objects/0k/FJ/SHA256E-s162778--6d2a0fc6afd1077eccdca2e92f314c00ef70a06273242429035e0ce52cce2e13.gz/SHA256E-s162778--6d2a0fc6afd1077eccdca2e92f314c00ef70a06273242429035e0ce52cce2e13.gz
+Return-Path: 
+X-Original-To: joey@kitenet.net
+Delivered-To: joey@kitenet.net
+Received: from vger.kernel.org (vger.kernel.org [209.132.180.67])
+	by kitenet.net (Postfix) with ESMTP id 714E61C67C
+	for ; Mon, 11 Jul 2016 18:47:12 -0400 (EDT)
+Authentication-Results: kitenet.net;
+	dkim=fail reason="signature verification failed" (1024-bit key; unprotected) header.d=joeyh.name header.i=@joeyh.name header.b=Yy9dgbAc;
+	dkim-atps=neutral
+Received: (majordomo@vger.kernel.org) by vger.kernel.org via listexpand
+	id S1752555AbcGKWrI (ORCPT );
+	Mon, 11 Jul 2016 18:47:08 -0400
+Received: from kitenet.net ([66.228.36.95]:60984 "EHLO kitenet.net"
+	rhost-flags-OK-OK-OK-OK) by vger.kernel.org with ESMTP
+	id S1752439AbcGKWrI (ORCPT );
+	Mon, 11 Jul 2016 18:47:08 -0400
+X-Question: 42
+DKIM-Signature:	v=1; a=rsa-sha256; c=simple/simple; d=joeyh.name; s=mail;
+	t=1468277198; bh=ZPsSRNaHriT6igc+PUYizSKSXZl5YDOyDcX0bJMWFqk=;
+	h=From:To:Cc:Subject:Date:In-Reply-To:References:From;
+	b=Yy9dgbAcC69LAmxyt4U9d2lqehM4dKl+NuDIHbQ//uTjX1mmOMkj/Pe4n7vrShEUP
+	 D4h/CYU5C4oRTJMhV1MF501fgcvPvOujxUMOviTB+g1EH5YfP989HvZT+7xafE3M4m
+	 8+IYWJC1JpDN45UcBwYBROl/ch/87+shPUlms+ds=
+From:	Joey Hess 
+To:	git@vger.kernel.org
+Cc:	Joey Hess 
+Subject: [PATCH v5 5/8] warn on unusable smudgeToFile/cleanFromFile config
+Date:	Mon, 11 Jul 2016 18:45:09 -0400
+Message-Id: <1468277112-9909-6-git-send-email-joeyh@joeyh.name>
+X-Mailer: git-send-email 2.8.1
+In-Reply-To: <1468277112-9909-1-git-send-email-joeyh@joeyh.name>
+References: <1468277112-9909-1-git-send-email-joeyh@joeyh.name>
+X-Spam-Status: No, score=-8.2 required=10.0 tests=BAYES_00,DKIM_SIGNED,
+	HEADER_FROM_DIFFERENT_DOMAINS,RCVD_IN_DNSWL_HI,RP_MATCHES_RCVD,T_DKIM_INVALID,
+	URIBL_BLOCKED autolearn=ham autolearn_force=no version=3.4.1
+X-Spam-Checker-Version: SpamAssassin 3.4.1 (2015-04-28) on kite.kitenet.net
+Sender:	git-owner@vger.kernel.org
+Precedence: bulk
+List-ID: 
+X-Mailing-List:	git@vger.kernel.org
+Content-Length: 2309
+Lines: 84
+
+Let the user know when they have a smudgeToFile/cleanFromFile config
+that cannot be used because the corresponding smudge/clean config
+is missing.
+
+The warning is only displayed a maximum of once per git invocation,
+and only when doing an operation that would use the filter.
+
+Signed-off-by: Joey Hess 
+---
+ convert.c | 36 ++++++++++++++++++++++++++----------
+ 1 file changed, 26 insertions(+), 10 deletions(-)
+
+diff --git a/convert.c b/convert.c
+index eb7774f..e1b0b44 100644
+--- a/convert.c
++++ b/convert.c
+@@ -845,34 +845,50 @@ int would_convert_to_git_filter_fd(const char *path)
+ 	return apply_filter(path, NULL, NULL, 0, -1, NULL, ca.drv->clean);
+ }
+ 
++static int can_filter_file(const char *filefilter, const char *filefiltername,
++			   const char *stdiofilter, const char *stdiofiltername,
++			   const struct conv_attrs *ca,
++			   int *warncount)
++{
++	if (!filefilter)
++		return 0;
++
++	if (stdiofilter)
++		return 1;
++
++	if (*warncount == 0)
++		warning("Not running your configured filter.%s.%s command, because filter.%s.%s is not configured",
++			ca->drv->name, filefiltername,
++			ca->drv->name, stdiofiltername);
++		*warncount=*warncount+1;
++
++	return 0;
++}
++
+ int can_clean_from_file(const char *path)
+ {
+ 	struct conv_attrs ca;
++	static int warncount = 0;
+ 
+ 	convert_attrs(&ca, path);
+ 	if (!ca.drv)
+ 		return 0;
+ 
+-	/*
+-	 * Only use the cleanFromFile filter when the clean filter is also
+-	 * configured.
+-	 */
+-	return (ca.drv->clean_from_file && ca.drv->clean);
++	return can_filter_file(ca.drv->clean_from_file, "cleanFromFile",
++			       ca.drv->clean, "clean", &ca, &warncount);
+ }
+ 
+ int can_smudge_to_file(const char *path)
+ {
+ 	struct conv_attrs ca;
++	static int warncount = 0;
+ 
+ 	convert_attrs(&ca, path);
+ 	if (!ca.drv)
+ 		return 0;
+ 
+-	/*
+-	 * Only use the smudgeToFile filter when the smudge filter is also
+-	 * configured.
+-	 */
+-	return (ca.drv->smudge_to_file && ca.drv->smudge);
++	return can_filter_file(ca.drv->smudge_to_file, "smudgeToFile",
++			       ca.drv->smudge, "smudge", &ca, &warncount);
+ }
+ 
+ const char *get_convert_attr_ascii(const char *path)
+-- 
+2.8.1
+
+--
+To unsubscribe from this list: send the line "unsubscribe git" in
+the body of a message to majordomo@vger.kernel.org
+More majordomo info at  http://vger.kernel.org/majordomo-info.html
+
+From mairix@mairix Mon Jan  1 12:34:56 1970
+X-source-folder: /home/joey/mail/.git/annex/objects/0k/FJ/SHA256E-s162778--6d2a0fc6afd1077eccdca2e92f314c00ef70a06273242429035e0ce52cce2e13.gz/SHA256E-s162778--6d2a0fc6afd1077eccdca2e92f314c00ef70a06273242429035e0ce52cce2e13.gz
+Return-Path: 
+X-Original-To: joey@kitenet.net
+Delivered-To: joey@kitenet.net
+Received: from vger.kernel.org (vger.kernel.org [209.132.180.67])
+	by kitenet.net (Postfix) with ESMTP id 839871C677
+	for ; Mon, 11 Jul 2016 18:47:27 -0400 (EDT)
+Authentication-Results: kitenet.net;
+	dkim=fail reason="signature verification failed" (1024-bit key; unprotected) header.d=joeyh.name header.i=@joeyh.name header.b=e1tj3LJk;
+	dkim-atps=neutral
+Received: (majordomo@vger.kernel.org) by vger.kernel.org via listexpand
+	id S1752603AbcGKWrW (ORCPT );
+	Mon, 11 Jul 2016 18:47:22 -0400
+Received: from kitenet.net ([66.228.36.95]:32788 "EHLO kitenet.net"
+	rhost-flags-OK-OK-OK-OK) by vger.kernel.org with ESMTP
+	id S1752616AbcGKWrT (ORCPT );
+	Mon, 11 Jul 2016 18:47:19 -0400
+X-Question: 42
+DKIM-Signature:	v=1; a=rsa-sha256; c=simple/simple; d=joeyh.name; s=mail;
+	t=1468277198; bh=4aNAzyRt9bz6y9MPC2VC5pj9+5O3Gb5fUsDopvg0K64=;
+	h=From:To:Cc:Subject:Date:In-Reply-To:References:From;
+	b=e1tj3LJkIi1ro6kUH77lZhTIqJt1N6KgyzFzcDDEdrFFz4x49MywJnSCKy9xtgY4h
+	 gc/cr1oZKUTrgmSXwMT8HD5yRltK8eWXnXOYn816pWnnRSkFlcwB+VWR4hhVcYahcV
+	 KM71p6oYWBbs2BULmxjndGZX1lRSx34zt8DePh2Y=
+From:	Joey Hess 
+To:	git@vger.kernel.org
+Cc:	Joey Hess 
+Subject: [PATCH v5 6/8] better recovery from failure of smudgeToFile filter
+Date:	Mon, 11 Jul 2016 18:45:10 -0400
+Message-Id: <1468277112-9909-7-git-send-email-joeyh@joeyh.name>
+X-Mailer: git-send-email 2.8.1
+In-Reply-To: <1468277112-9909-1-git-send-email-joeyh@joeyh.name>
+References: <1468277112-9909-1-git-send-email-joeyh@joeyh.name>
+X-Spam-Status: No, score=-8.2 required=10.0 tests=BAYES_00,DKIM_SIGNED,
+	HEADER_FROM_DIFFERENT_DOMAINS,RCVD_IN_DNSWL_HI,RP_MATCHES_RCVD,T_DKIM_INVALID,
+	URIBL_BLOCKED autolearn=ham autolearn_force=no version=3.4.1
+X-Spam-Checker-Version: SpamAssassin 3.4.1 (2015-04-28) on kite.kitenet.net
+Sender:	git-owner@vger.kernel.org
+Precedence: bulk
+List-ID: 
+X-Mailing-List:	git@vger.kernel.org
+Content-Length: 4558
+Lines: 151
+
+If the smudgeToFile filter fails, it can leave the worktree file with the
+wrong content, or even deleted. Recover from this by falling back to
+running the smudge filter.
+
+Signed-off-by: Joey Hess 
+---
+ entry.c               | 66 ++++++++++++++++++++++++++++++++++-----------------
+ t/t0021-conversion.sh | 24 +++++++++++++++++++
+ 2 files changed, 68 insertions(+), 22 deletions(-)
+
+diff --git a/entry.c b/entry.c
+index 81d12a1..7811e31 100644
+--- a/entry.c
++++ b/entry.c
+@@ -182,12 +182,6 @@ static int write_entry(struct cache_entry *ce,
+ 		regular_file = ce_mode_s_ifmt == S_IFREG;
+ 		smudge_to_file = regular_file
+ 			&& can_smudge_to_file(ce->name);
+-		if (regular_file && !smudge_to_file &&
+-		    convert_to_working_tree(ce->name, new, size, &buf)) {
+-			free(new);
+-			new = strbuf_detach(&buf, &newsize);
+-			size = newsize;
+-		}
+ 
+ 		fd = open_output_fd(path, ce, to_tempfile);
+ 		if (fd < 0) {
+@@ -195,7 +189,51 @@ static int write_entry(struct cache_entry *ce,
+ 			return error_errno("unable to create file %s", path);
+ 		}
+ 
++		if (smudge_to_file) {
++			close(fd);
++			if (convert_to_working_tree_filter_to_file(ce->name, path, new, size)) {
++				free(new);
++				/*
++				 * The smudgeToFile filter may have replaced
++				 * or deleted the file; reopen it to make
++				 * sure that the file exists.
++				 */
++				fd = open(path, O_RDONLY);
++				if (fd < 0)
++					return error_errno("unable to create file %s", path);
++				if (!to_tempfile)
++					fstat_done = fstat_output(fd, state, &st);
++				close(fd);
++			}
++			else {
++				/*
++				 * The failing smudgeToFile filter may have
++				 * deleted or replaced the file; delete
++				 * the file and re-open for recovery write.
++				 */
++				unlink(path);
++				fd = open_output_fd(path, ce, to_tempfile);
++				if (fd < 0) {
++					free(new);
++					return error_errno("unable to create file %s", path);
++				}
++				/* Fall through to normal write below. */
++				smudge_to_file = 0;
++			}
++		}
++
++		/*
++		 * Not an else of above if (smudge_to_file) because the
++		 * smudgeToFile filter may fail and in that case this is
++		 * run to recover.
++		 */
+ 		if (!smudge_to_file) {
++			if (regular_file &&
++			    convert_to_working_tree(ce->name, new, size, &buf)) {
++				free(new);
++				new = strbuf_detach(&buf, &newsize);
++				size = newsize;
++			}
+ 			wrote = write_in_full(fd, new, size);
+ 			if (!to_tempfile)
+ 				fstat_done = fstat_output(fd, state, &st);
+@@ -204,22 +242,6 @@ static int write_entry(struct cache_entry *ce,
+ 			if (wrote != size)
+ 				return error("unable to write file %s", path);
+ 		}
+-		else {
+-			close(fd);
+-			convert_to_working_tree_filter_to_file(ce->name, path, new, size);
+-			free(new);
+-			/*
+-			 * The smudgeToFile filter may have replaced the
+-			 * file; open it to make sure that the file
+-			 * exists.
+-			 */
+-			fd = open(path, O_RDONLY);
+-			if (fd < 0)
+-				return error_errno("unable to create file %s", path);
+-			if (!to_tempfile)
+-				fstat_done = fstat_output(fd, state, &st);
+-			close(fd);
+-		}
+ 		break;
+ 	case S_IFGITLINK:
+ 		if (to_tempfile)
+diff --git a/t/t0021-conversion.sh b/t/t0021-conversion.sh
+index ea18b17..0efad9b 100755
+--- a/t/t0021-conversion.sh
++++ b/t/t0021-conversion.sh
+@@ -28,6 +28,14 @@ touch rot13-to-file.ran
+ EOF
+ chmod +x rot13-to-file.sh
+ 
++cat <delete-file-and-fail.sh
++#!$SHELL_PATH
++destfile="\$1"
++rm -f "\$destfile"
++exit 1
++EOF
++chmod +x delete-file-and-fail.sh
++
+ test_expect_success setup '
+ 	git config filter.rot13.smudge ./rot13.sh &&
+ 	git config filter.rot13.clean ./rot13.sh &&
+@@ -310,6 +318,22 @@ test_expect_success 'smudgeToFile filter is used when checking out a file' '
+ 	rm -f rot13-to-file.ran
+ '
+ 
++test_expect_success 'recovery from failure of smudgeToFile filter, using smudge filter' '
++	test_config filter.rot13.smudgeToFile false &&
++
++	rm -f fstest.t &&
++	git checkout -- fstest.t &&
++	test_cmp test fstest.t
++'
++
++test_expect_success 'recovery from failure of smudgeToFile filter that deletes the worktree file' '
++	test_config filter.rot13.smudgeToFile ./delete-file-and-fail.sh &&
++
++	rm -f fstest.t &&
++	git checkout -- fstest.t &&
++	test_cmp test fstest.t
++'
++
+ test_expect_success 'cleanFromFile filter is not used when clean filter is not configured' '
+ 	test_config filter.noclean.smudge ./rot13.sh &&
+ 	test_config filter.noclean.cleanFromFile ./rot13-from-file.sh &&
+-- 
+2.8.1
+
+--
+To unsubscribe from this list: send the line "unsubscribe git" in
+the body of a message to majordomo@vger.kernel.org
+More majordomo info at  http://vger.kernel.org/majordomo-info.html
+
+From mairix@mairix Mon Jan  1 12:34:56 1970
+X-source-folder: /home/joey/mail/.git/annex/objects/0k/FJ/SHA256E-s162778--6d2a0fc6afd1077eccdca2e92f314c00ef70a06273242429035e0ce52cce2e13.gz/SHA256E-s162778--6d2a0fc6afd1077eccdca2e92f314c00ef70a06273242429035e0ce52cce2e13.gz
+Return-Path: 
+X-Original-To: joey@kitenet.net
+Delivered-To: joey@kitenet.net
+Received: from vger.kernel.org (vger.kernel.org [209.132.180.67])
+	by kitenet.net (Postfix) with ESMTP id 3B2531C677
+	for ; Mon, 11 Jul 2016 18:47:23 -0400 (EDT)
+Authentication-Results: kitenet.net;
+	dkim=fail reason="signature verification failed" (1024-bit key; unprotected) header.d=joeyh.name header.i=@joeyh.name header.b=P7RXYLHP;
+	dkim-atps=neutral
+Received: (majordomo@vger.kernel.org) by vger.kernel.org via listexpand
+	id S1752629AbcGKWrT (ORCPT );
+	Mon, 11 Jul 2016 18:47:19 -0400
+Received: from kitenet.net ([66.228.36.95]:32782 "EHLO kitenet.net"
+	rhost-flags-OK-OK-OK-OK) by vger.kernel.org with ESMTP
+	id S1752603AbcGKWrS (ORCPT );
+	Mon, 11 Jul 2016 18:47:18 -0400
+X-Question: 42
+DKIM-Signature:	v=1; a=rsa-sha256; c=simple/simple; d=joeyh.name; s=mail;
+	t=1468277198; bh=R6DA3FWAltlIvvD5mJEp7ZgM7JFZFfGDS/9auV9Z+0Q=;
+	h=From:To:Cc:Subject:Date:In-Reply-To:References:From;
+	b=P7RXYLHPbs0CXSl4ChFu0IJTC7WGua7f0VyocRm/r3ZZK/2HscKr3lMHDy0fxfD3I
+	 grgLgEiaffIZ5+vMX0FWQIYDLISJD4NlDMYYt3NoONm10nDHJp/vSm1/YdyoaT8+3p
+	 RnZbAzgY6m43Lhx7BxNApQ8/UPzUL5hWD45Jl5YU=
+From:	Joey Hess 
+To:	git@vger.kernel.org
+Cc:	Joey Hess 
+Subject: [PATCH v5 7/8] use smudgeToFile filter in git am
+Date:	Mon, 11 Jul 2016 18:45:11 -0400
+Message-Id: <1468277112-9909-8-git-send-email-joeyh@joeyh.name>
+X-Mailer: git-send-email 2.8.1
+In-Reply-To: <1468277112-9909-1-git-send-email-joeyh@joeyh.name>
+References: <1468277112-9909-1-git-send-email-joeyh@joeyh.name>
+X-Spam-Status: No, score=-8.2 required=10.0 tests=BAYES_00,DKIM_SIGNED,
+	HEADER_FROM_DIFFERENT_DOMAINS,RCVD_IN_DNSWL_HI,RP_MATCHES_RCVD,T_DKIM_INVALID,
+	URIBL_BLOCKED autolearn=ham autolearn_force=no version=3.4.1
+X-Spam-Checker-Version: SpamAssassin 3.4.1 (2015-04-28) on kite.kitenet.net
+Sender:	git-owner@vger.kernel.org
+Precedence: bulk
+List-ID: 
+X-Mailing-List:	git@vger.kernel.org
+Content-Length: 4808
+Lines: 156
+
+git am updates the work tree and so should use the smudgeToFile filter.
+
+This includes some refactoring into convert_to_working_tree_filter_to_file
+to make it check the file after running the smudgeToFile command, and clean
+up from a failing command.
+
+Signed-off-by: Joey Hess 
+---
+ apply.c               | 16 ++++++++++++++++
+ convert.c             | 25 +++++++++++++++++++++++--
+ entry.c               | 21 ++++-----------------
+ t/t0021-conversion.sh | 13 +++++++++++++
+ 4 files changed, 56 insertions(+), 19 deletions(-)
+
+diff --git a/apply.c b/apply.c
+index 4a6b2db..7db8344 100644
+--- a/apply.c
++++ b/apply.c
+@@ -4322,6 +4322,22 @@ static int try_create_file(const char *path, unsigned int mode, const char *buf,
+ 	if (fd < 0)
+ 		return 1;
+ 
++	if (can_smudge_to_file(path)) {
++		close(fd);
++		fd = convert_to_working_tree_filter_to_file(path, path, buf, size);
++		if (fd < 0) {
++			/* smudgeToFile filter failed; continue
++			 * with regular file creation instead. */
++			fd = open(path, O_CREAT | O_EXCL | O_WRONLY, (mode & 0100) ? 0777 : 0666);
++			if (fd < 0)
++				return -1;
++		}
++		else {
++			close(fd);
++			return 0;
++		}
++	}
++
+ 	if (convert_to_working_tree(path, buf, size, &nbuf)) {
+ 		size = nbuf.len;
+ 		buf  = nbuf.buf;
+diff --git a/convert.c b/convert.c
+index e1b0b44..3746ad5 100644
+--- a/convert.c
++++ b/convert.c
+@@ -1030,13 +1030,34 @@ int convert_to_working_tree(const char *path, const char *src, size_t len, struc
+ 	return convert_to_working_tree_internal(path, NULL, src, len, dst, 0);
+ }
+ 
++/*
++ * Returns fd open to read the worktree file on success.
++ * On failure, the worktree file will not exist.
++ */
+ int convert_to_working_tree_filter_to_file(const char *path, const char *destpath, const char *src, size_t len)
+ {
+ 	struct strbuf output = STRBUF_INIT;
+-	int ret = convert_to_working_tree_internal(path, destpath, src, len, &output, 0);
++	int ok = convert_to_working_tree_internal(path, destpath, src, len, &output, 0);
+ 	/* The smudgeToFile filter stdout is not used. */
+ 	strbuf_release(&output);
+-	return ret;
++	if (ok) {
++		/*
++		 * Open the file to make sure that it's present
++		 * (and readable) after the command populated it.
++		 */
++		int fd = open(path, O_RDONLY);
++		if (fd < 0)
++			unlink(path);
++		return fd;
++	}
++	else {
++		/*
++		 * The command could have created the file before failing,
++		 * so delete it.
++		 */
++		unlink(path);
++		return -1;
++	}
+ }
+ 
+ int renormalize_buffer(const char *path, const char *src, size_t len, struct strbuf *dst)
+diff --git a/entry.c b/entry.c
+index 7811e31..40662eb 100644
+--- a/entry.c
++++ b/entry.c
+@@ -191,34 +191,21 @@ static int write_entry(struct cache_entry *ce,
+ 
+ 		if (smudge_to_file) {
+ 			close(fd);
+-			if (convert_to_working_tree_filter_to_file(ce->name, path, new, size)) {
++			fd = convert_to_working_tree_filter_to_file(ce->name, path, new, size);
++			if (fd >= 0) {
+ 				free(new);
+-				/*
+-				 * The smudgeToFile filter may have replaced
+-				 * or deleted the file; reopen it to make
+-				 * sure that the file exists.
+-				 */
+-				fd = open(path, O_RDONLY);
+-				if (fd < 0)
+-					return error_errno("unable to create file %s", path);
+ 				if (!to_tempfile)
+ 					fstat_done = fstat_output(fd, state, &st);
+ 				close(fd);
+ 			}
+ 			else {
+-				/*
+-				 * The failing smudgeToFile filter may have
+-				 * deleted or replaced the file; delete
+-				 * the file and re-open for recovery write.
+-				 */
+-				unlink(path);
++				/* Fall through to normal write below. */
++				smudge_to_file = 0;
+ 				fd = open_output_fd(path, ce, to_tempfile);
+ 				if (fd < 0) {
+ 					free(new);
+ 					return error_errno("unable to create file %s", path);
+ 				}
+-				/* Fall through to normal write below. */
+-				smudge_to_file = 0;
+ 			}
+ 		}
+ 
+diff --git a/t/t0021-conversion.sh b/t/t0021-conversion.sh
+index 0efad9b..42b28aa 100755
+--- a/t/t0021-conversion.sh
++++ b/t/t0021-conversion.sh
+@@ -334,6 +334,19 @@ test_expect_success 'recovery from failure of smudgeToFile filter that deletes t
+ 	test_cmp test fstest.t
+ '
+ 
++test_expect_success 'smudgeToFile filter is used by git am' '
++	test_config filter.rot13.smudgeToFile ./rot13-to-file.sh &&
++
++	git commit fstest.t -m "added fstest.t" &&
++	git format-patch HEAD^ --stdout >fstest.patch &&
++	git reset --hard HEAD^ &&
++	git am fstest.patch &&
++
++	test -e rot13-to-file.ran &&
++	rm -f rot13-to-file.ran &&
++	test_cmp test fstest.t
++'
++
+ test_expect_success 'cleanFromFile filter is not used when clean filter is not configured' '
+ 	test_config filter.noclean.smudge ./rot13.sh &&
+ 	test_config filter.noclean.cleanFromFile ./rot13-from-file.sh &&
+-- 
+2.8.1
+
+--
+To unsubscribe from this list: send the line "unsubscribe git" in
+the body of a message to majordomo@vger.kernel.org
+More majordomo info at  http://vger.kernel.org/majordomo-info.html
+
+From mairix@mairix Mon Jan  1 12:34:56 1970
+X-source-folder: /home/joey/mail/.git/annex/objects/0k/FJ/SHA256E-s162778--6d2a0fc6afd1077eccdca2e92f314c00ef70a06273242429035e0ce52cce2e13.gz/SHA256E-s162778--6d2a0fc6afd1077eccdca2e92f314c00ef70a06273242429035e0ce52cce2e13.gz
+Return-Path: 
+X-Original-To: joey@kitenet.net
+Delivered-To: joey@kitenet.net
+Received: from vger.kernel.org (vger.kernel.org [209.132.180.67])
+	by kitenet.net (Postfix) with ESMTP id C58671C677
+	for ; Mon, 11 Jul 2016 18:47:18 -0400 (EDT)
+Authentication-Results: kitenet.net;
+	dkim=fail reason="signature verification failed" (1024-bit key; unprotected) header.d=joeyh.name header.i=@joeyh.name header.b=VKChWnWm;
+	dkim-atps=neutral
+Received: (majordomo@vger.kernel.org) by vger.kernel.org via listexpand
+	id S1752615AbcGKWrR (ORCPT );
+	Mon, 11 Jul 2016 18:47:17 -0400
+Received: from kitenet.net ([66.228.36.95]:32778 "EHLO kitenet.net"
+	rhost-flags-OK-OK-OK-OK) by vger.kernel.org with ESMTP
+	id S1752603AbcGKWrO (ORCPT );
+	Mon, 11 Jul 2016 18:47:14 -0400
+X-Question: 42
+DKIM-Signature:	v=1; a=rsa-sha256; c=simple/simple; d=joeyh.name; s=mail;
+	t=1468277198; bh=e7pd1+fJ7kJE3zbr6Uuf5rQkwlF5ZqMSMu31as4SNuw=;
+	h=From:To:Cc:Subject:Date:In-Reply-To:References:From;
+	b=VKChWnWmsu86vvKjaYknMgW76bozccToHJNzUBKXMXKt57vEc+PiDgxiMNKvA57iJ
+	 /S26T8zVRqNUsPdzXsLMoCVA+T2EPLwSv/dLjBIwrTT+yveUiaH2Q4U9jzqDkU/GYy
+	 8iEtbg7ly/pnrWgjApNsnbMY5OogcAINTDod/flY=
+From:	Joey Hess 
+To:	git@vger.kernel.org
+Cc:	Joey Hess 
+Subject: [PATCH v5 8/8] use smudgeToFile filter in recursive merge
+Date:	Mon, 11 Jul 2016 18:45:12 -0400
+Message-Id: <1468277112-9909-9-git-send-email-joeyh@joeyh.name>
+X-Mailer: git-send-email 2.8.1
+In-Reply-To: <1468277112-9909-1-git-send-email-joeyh@joeyh.name>
+References: <1468277112-9909-1-git-send-email-joeyh@joeyh.name>
+X-Spam-Status: No, score=-8.2 required=10.0 tests=BAYES_00,DKIM_SIGNED,
+	HEADER_FROM_DIFFERENT_DOMAINS,RCVD_IN_DNSWL_HI,RP_MATCHES_RCVD,T_DKIM_INVALID,
+	URIBL_BLOCKED autolearn=ham autolearn_force=no version=3.4.1
+X-Spam-Checker-Version: SpamAssassin 3.4.1 (2015-04-28) on kite.kitenet.net
+Sender:	git-owner@vger.kernel.org
+Precedence: bulk
+List-ID: 
+X-Mailing-List:	git@vger.kernel.org
+Content-Length: 3928
+Lines: 134
+
+Recursive merge updates the work tree and so should use the smudgeToFile
+filter.
+
+At this point, smudgeToFile is run by everything that updates work
+tree files.
+
+Signed-off-by: Joey Hess 
+---
+ merge-recursive.c     | 53 ++++++++++++++++++++++++++++++++++++++++-----------
+ t/t0021-conversion.sh | 16 +++++++++++++++-
+ 2 files changed, 57 insertions(+), 12 deletions(-)
+
+diff --git a/merge-recursive.c b/merge-recursive.c
+index a4a1195..5fe3f50 100644
+--- a/merge-recursive.c
++++ b/merge-recursive.c
+@@ -758,6 +758,7 @@ static void update_file_flags(struct merge_options *o,
+ 		enum object_type type;
+ 		void *buf;
+ 		unsigned long size;
++		int isreg;
+ 
+ 		if (S_ISGITLINK(mode)) {
+ 			/*
+@@ -774,22 +775,16 @@ static void update_file_flags(struct merge_options *o,
+ 			die(_("cannot read object %s '%s'"), oid_to_hex(oid), path);
+ 		if (type != OBJ_BLOB)
+ 			die(_("blob expected for %s '%s'"), oid_to_hex(oid), path);
+-		if (S_ISREG(mode)) {
+-			struct strbuf strbuf = STRBUF_INIT;
+-			if (convert_to_working_tree(path, buf, size, &strbuf)) {
+-				free(buf);
+-				size = strbuf.len;
+-				buf = strbuf_detach(&strbuf, NULL);
+-			}
+-		}
+ 
+ 		if (make_room_for_path(o, path) < 0) {
+ 			update_wd = 0;
+ 			free(buf);
+ 			goto update_index;
+ 		}
+-		if (S_ISREG(mode) || (!has_symlinks && S_ISLNK(mode))) {
++		isreg = S_ISREG(mode);
++		if (isreg || (!has_symlinks && S_ISLNK(mode))) {
+ 			int fd;
++			int smudge_to_file;
+ 			if (mode & 0100)
+ 				mode = 0777;
+ 			else
+@@ -797,8 +792,44 @@ static void update_file_flags(struct merge_options *o,
+ 			fd = open(path, O_WRONLY | O_TRUNC | O_CREAT, mode);
+ 			if (fd < 0)
+ 				die_errno(_("failed to open '%s'"), path);
+-			write_in_full(fd, buf, size);
+-			close(fd);
++
++			smudge_to_file = can_smudge_to_file(path);
++			if (smudge_to_file) {
++				close(fd);
++				fd = convert_to_working_tree_filter_to_file(path, path, buf, size);
++				if (fd < 0) {
++					/*
++					 * smudgeToFile filter failed;
++					 * continue with regular file
++					 * creation.
++					 */
++					smudge_to_file = 0;
++					fd = open(path, O_WRONLY | O_TRUNC | O_CREAT, mode);
++					if (fd < 0)
++						die_errno(_("failed to open '%s'"), path);
++				}
++				else {
++					close(fd);
++				}
++			}
++
++			/*
++			 * Not an else of above if (smudge_to_file) because
++			 * the smudgeToFile filter may fail and in that case
++			 * this is run to recover.
++			 */
++			if (!smudge_to_file) {
++				if (isreg) {
++					struct strbuf strbuf = STRBUF_INIT;
++					if (convert_to_working_tree(path, buf, size, &strbuf)) {
++						free(buf);
++						size = strbuf.len;
++						buf = strbuf_detach(&strbuf, NULL);
++					}
++				}
++				write_in_full(fd, buf, size);
++				close(fd);
++			}
+ 		} else if (S_ISLNK(mode)) {
+ 			char *lnk = xmemdupz(buf, size);
+ 			safe_create_leading_directories_const(path);
+diff --git a/t/t0021-conversion.sh b/t/t0021-conversion.sh
+index 42b28aa..64b2b8f 100755
+--- a/t/t0021-conversion.sh
++++ b/t/t0021-conversion.sh
+@@ -334,10 +334,24 @@ test_expect_success 'recovery from failure of smudgeToFile filter that deletes t
+ 	test_cmp test fstest.t
+ '
+ 
++test_expect_success 'smudgeToFile filter is used in merge' '
++	test_config filter.rot13.smudgeToFile ./rot13-to-file.sh &&
++
++	git commit -m "added fstest.t" fstest.t &&
++	git checkout -b old &&
++	git reset --hard HEAD^ &&
++	git merge master &&
++	git checkout master &&
++
++	test -e rot13-to-file.ran &&
++	rm -f rot13-to-file.ran &&
++
++	test_cmp test fstest.t
++'
++
+ test_expect_success 'smudgeToFile filter is used by git am' '
+ 	test_config filter.rot13.smudgeToFile ./rot13-to-file.sh &&
+ 
+-	git commit fstest.t -m "added fstest.t" &&
+ 	git format-patch HEAD^ --stdout >fstest.patch &&
+ 	git reset --hard HEAD^ &&
+ 	git am fstest.patch &&
+-- 
+2.8.1
+
+--
+To unsubscribe from this list: send the line "unsubscribe git" in
+the body of a message to majordomo@vger.kernel.org
+More majordomo info at  http://vger.kernel.org/majordomo-info.html
+
diff --git a/doc/todo/speed_up_transfers_over_ssh+rsync_--_directly_reuse_the_same_connection__63__.mdwn b/doc/todo/speed_up_transfers_over_ssh+rsync_--_directly_reuse_the_same_connection__63__.mdwn
new file mode 100644
index 0000000000..fdd660706d
--- /dev/null
+++ b/doc/todo/speed_up_transfers_over_ssh+rsync_--_directly_reuse_the_same_connection__63__.mdwn
@@ -0,0 +1,37 @@
+ATM, even with ControlPersist=yes, on a fast interconnection between hosts (so it a millisecond or so to transfer a single file I have there), majority of time is spent I guess on running a new ssh (although with instructions for persistent connection etc) or is there some intentional sleep somewhere (or just output flushed at 1 sec intervals so ts has those consistent subsec offset), which ends up spending up to a second to transfer a single file.  I have ~25,000 files so would need about 5 hours, although with direct rsync -e 'ssh' -L  it takes total of 2 seconds for those 100MB .  Here is the protocol (with ts with subsecond timing):
+
+    Aug 04 10:46:02.1438699562 get stimulus/task002/generate/part_1-182x121/000251.jpeg (from origin...)
+    Aug 04 10:46:02.1438699562 SHA256E-s1650--768d78ad49fc413d178a5cd9407b56bb442f40aaa629b7f608844330c2c4bbf9.jpeg
+    Aug 04 10:46:02.1438699562 ^M              0   0%    0.00kB/s    0:00:00  ^M          1,650 100%    1.57MB/s    0:00:00 (xfr#1, to-chk=0/1)
+    Aug 04 10:46:02.1438699562 ok
+    Aug 04 10:46:02.1438699562 get stimulus/task002/generate/part_1-182x121/000252.jpeg (from origin...)
+    Aug 04 10:46:02.1438699562 SHA256E-s1662--d156f08ecfcc248aeb01239f5e605f3ac9ed72fa77c54e593a54b1f6a8b3f0f4.jpeg
+    Aug 04 10:46:02.1438699562 ^M              0   0%    0.00kB/s    0:00:00  ^M          1,662 100%    1.59MB/s    0:00:00 (xfr#1, to-chk=0/1)
+    Aug 04 10:46:02.1438699562 ok
+    Aug 04 10:46:02.1438699562 get stimulus/task002/generate/part_1-182x121/000253.jpeg (from origin...)
+    Aug 04 10:46:02.1438699562 SHA256E-s1673--47562fe25853fe2972678cbaa1ef8e03bad068095d9f8575ba96f8df0a18cff0.jpeg
+    Aug 04 10:46:02.1438699562 ^M              0   0%    0.00kB/s    0:00:00  ^M          1,673 100%    1.60MB/s    0:00:00 (xfr#1, to-chk=0/1)
+    Aug 04 10:46:03.1438699563 ok
+    Aug 04 10:46:03.1438699563 get stimulus/task002/generate/part_1-182x121/000254.jpeg (from origin...)
+    Aug 04 10:46:03.1438699563 SHA256E-s1675--a3dae03d805040af4a7341479b782342431ee5377713c061e02daa075d188037.jpeg
+    Aug 04 10:46:03.1438699563 ^M              0   0%    0.00kB/s    0:00:00  ^M          1,675 100%    1.60MB/s    0:00:00 (xfr#1, to-chk=0/1)
+    Aug 04 10:46:03.1438699563 ok
+    Aug 04 10:46:03.1438699563 get stimulus/task002/generate/part_1-182x121/000255.jpeg (from origin...)
+    Aug 04 10:46:03.1438699563 SHA256E-s1674--a6743261238a87f9f3574295665896be2a8f373dd5b400fbf1552fb8d3b8fc66.jpeg
+    Aug 04 10:46:03.1438699563 ^M              0   0%    0.00kB/s    0:00:00  ^M          1,674 100%    1.60MB/s    0:00:00 (xfr#1, to-chk=0/1)
+    Aug 04 10:46:04.1438699564 ok
+    Aug 04 10:46:04.1438699564 get stimulus/task002/generate/part_1-182x121/000256.jpeg (from origin...)
+    Aug 04 10:46:04.1438699564 SHA256E-s1659--e1bd4829f53b226ad62ebccfbbb1a132d977af930545ade4161b5f9acf2e80b1.jpeg
+    Aug 04 10:46:04.1438699564 ^M              0   0%    0.00kB/s    0:00:00  ^M          1,659 100%    1.58MB/s    0:00:00 (xfr#1, to-chk=0/1)
+    Aug 04 10:46:04.1438699564 ok
+    Aug 04 10:46:04.1438699564 get stimulus/task002/generate/part_1-182x121/000257.jpeg (from origin...)
+    Aug 04 10:46:04.1438699564 SHA256E-s1663--ae0e673c60dede66c773441ee198fa8979c6de4a169e468fbb10dc22860afb27.jpeg
+    Aug 04 10:46:04.1438699564 ^M              0   0%    0.00kB/s    0:00:00  ^M          1,663 100%    1.59MB/s    0:00:00 (xfr#1, to-chk=0/1)
+    Aug 04 10:46:05.1438699565 ok  
+
+
+
+both hosts do not show any high CPU load
+
+> [[closed|done]]; wrung out all the perf gains we can without
+> [[accellerate_ssh_remotes_with_git-annex-shell_mass_protocol]] --[[Joey]]
diff --git a/doc/todo/speed_up_transfers_over_ssh+rsync_--_directly_reuse_the_same_connection__63__/comment_10_c2d87b07f8197003b9489a8124680b59._comment b/doc/todo/speed_up_transfers_over_ssh+rsync_--_directly_reuse_the_same_connection__63__/comment_10_c2d87b07f8197003b9489a8124680b59._comment
new file mode 100644
index 0000000000..381bed86fb
--- /dev/null
+++ b/doc/todo/speed_up_transfers_over_ssh+rsync_--_directly_reuse_the_same_connection__63__/comment_10_c2d87b07f8197003b9489a8124680b59._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="joey"
+ subject="""comment 10"""
+ date="2018-03-06T18:47:34Z"
+ content="""
+See [[accellerate_ssh_remotes_with_git-annex-shell_mass_protocol]].
+I'm going to close this todo in favor of that one.
+"""]]
diff --git a/doc/todo/speed_up_transfers_over_ssh+rsync_--_directly_reuse_the_same_connection__63__/comment_1_3c92115f8162ba81149949d4c810bf43._comment b/doc/todo/speed_up_transfers_over_ssh+rsync_--_directly_reuse_the_same_connection__63__/comment_1_3c92115f8162ba81149949d4c810bf43._comment
new file mode 100644
index 0000000000..10d34d3039
--- /dev/null
+++ b/doc/todo/speed_up_transfers_over_ssh+rsync_--_directly_reuse_the_same_connection__63__/comment_1_3c92115f8162ba81149949d4c810bf43._comment
@@ -0,0 +1,7 @@
+[[!comment format=mdwn
+ username="https://me.yahoo.com/a/EbvxpTI_xP9Aod7Mg4cwGhgjrCrdM5s-#7c0f4"
+ subject="comment 1"
+ date="2015-08-04T15:00:02Z"
+ content="""
+doh -- and apparently I had some aged annex -- git-annex version: 5.20141125, but even with a freshier   5.20150706+gitgefc3bcd-1~ndall+1  situation is the same
+"""]]
diff --git a/doc/todo/speed_up_transfers_over_ssh+rsync_--_directly_reuse_the_same_connection__63__/comment_2_7605d67785288e5945999ca6b677c579._comment b/doc/todo/speed_up_transfers_over_ssh+rsync_--_directly_reuse_the_same_connection__63__/comment_2_7605d67785288e5945999ca6b677c579._comment
new file mode 100644
index 0000000000..26d2c4ec59
--- /dev/null
+++ b/doc/todo/speed_up_transfers_over_ssh+rsync_--_directly_reuse_the_same_connection__63__/comment_2_7605d67785288e5945999ca6b677c579._comment
@@ -0,0 +1,25 @@
+[[!comment format=mdwn
+ username="https://me.yahoo.com/a/EbvxpTI_xP9Aod7Mg4cwGhgjrCrdM5s-#7c0f4"
+ subject="comment 2"
+ date="2015-08-04T15:28:29Z"
+ content="""
+since I am not sure what is the actual overhead here, can't provide any good advice, but may be it is worth looking at least into bundling multiple transfers within the same rsync call?  rsync man page says
+
+       The syntax for requesting multiple files from a remote host is done by specifying
+       additional remote-host args in the same style as the  first,  or
+       with the hostname omitted.  For instance, all these work:
+
+              rsync -av host:file1 :file2 host:file{3,4} /dest/
+
+so it should be quite possible to batch a hundred or two transfers into the same rsync call I guess.  Probably on other systems limit is different but on linux the cmdline size is quite hefty:
+
+    $> xargs --show-limits
+    Your environment variables take up 3441 bytes
+    POSIX upper limit on argument length (this system): 2091663
+    POSIX smallest allowable upper limit on argument length (all systems): 4096
+    Maximum length of command we could actually use: 2088222
+    Size of command buffer we are actually using: 131072
+
+not sure if there are inherent limits within ssh etc
+
+"""]]
diff --git a/doc/todo/speed_up_transfers_over_ssh+rsync_--_directly_reuse_the_same_connection__63__/comment_3_15c4b941a4af8354fee3209c64e89e33._comment b/doc/todo/speed_up_transfers_over_ssh+rsync_--_directly_reuse_the_same_connection__63__/comment_3_15c4b941a4af8354fee3209c64e89e33._comment
new file mode 100644
index 0000000000..d8a85006cd
--- /dev/null
+++ b/doc/todo/speed_up_transfers_over_ssh+rsync_--_directly_reuse_the_same_connection__63__/comment_3_15c4b941a4af8354fee3209c64e89e33._comment
@@ -0,0 +1,56 @@
+[[!comment format=mdwn
+ username="joey"
+ subject="""comment 3"""
+ date="2015-08-04T16:07:23Z"
+ content="""
+I don't know how you used ts to get that output, but it does not appear to
+be accurate. Here's `git annex get | ts "%b %d %H:%M:%.s"`:
+
+	Aug 04 12:10:1438704612.238164 get bob (from origin...) 
+	Aug 04 12:10:1438704612.238336 SHA256E-s30--7c3722f359d8cfbc594e5dec6d7f096bf4e88adbfb786548a607708d32ef49bb
+	             30 100%   29.30kB/s    0:00:00 (xfr#1, to-chk=0/1)
+	Aug 04 12:10:1438704612.597912 ok
+	Aug 04 12:10:1438704612.642574 get bob2 (from origin...) 
+	Aug 04 12:10:1438704612.642653 SHA256E-s30--22fdcad0fba3537a7a0bc2b824f805fdd2d2c032b6450352cb8d099d03c8d796
+	             30 100%   29.30kB/s    0:00:00 (xfr#1, to-chk=0/1)
+	Aug 04 12:10:1438704613.001088 ok
+	Aug 04 12:10:1438704613.043018 get bob3 (from origin...) 
+	Aug 04 12:10:1438704613.043105 SHA256E-s30--9f0e99611fe904e173fb1c81d57ba9f01a111afdf222a7763844c7b893af86ff
+	             30 100%   29.30kB/s    0:00:00 (xfr#1, to-chk=0/1)
+	Aug 04 12:10:1438704613.393817 ok
+	Aug 04 12:10:1438704613.421080 get bob5 (from origin...) 
+	Aug 04 12:10:1438704613.421198 SHA256E-s30--7b851aa7791136f783271c109c287614bf2c0e9014d0fab50b1bf32f4ad4678e
+	             30 100%   29.30kB/s    0:00:00 (xfr#1, to-chk=0/1)
+	Aug 04 12:10:1438704613.769410 ok
+	Aug 04 12:10:1438704613.800044 get bob6 (from origin...) 
+	Aug 04 12:10:1438704613.800133 SHA256E-s33--a165803131075f75132f632a6f295b12910f84a5d1776b60d1d998b96a6f20d5
+	             33 100%   32.23kB/s    0:00:00 (xfr#1, to-chk=0/1)
+	Aug 04 12:10:1438704614.148597 ok
+	Aug 04 12:10:1438704614.200786 get bob7 (from origin...) 
+	Aug 04 12:10:1438704614.200881 SHA256E-s30--8b5b0b239b465a407ca98c8dc82a0081ee5ced4f7402854dc9afac75b65b5b51
+	             30 100%   29.30kB/s    0:00:00 (xfr#1, to-chk=0/1)
+	Aug 04 12:10:1438704614.560315 ok
+	Aug 04 12:10:1438704614.602174 get bob8 (from origin...) 
+	Aug 04 12:10:1438704614.602255 SHA256E-s30--70657536d54051bd020f984f866b017d5919b7705bf08ffa2786c0dd14f90280
+	             30 100%   29.30kB/s    0:00:00 (xfr#1, to-chk=0/1)
+	Aug 04 12:10:1438704614.961264 ok
+
+Note that this shows up to 3 files sent per second. The output you pasted shows not 1 file/s, but 2 or almost 3.
+
+There are no 1s (or other) sleeps. This is quite likely simply the overhead of
+git-annex needing to query git to work out what remote each file is located on,
+coupled with the overhead of needing to start a new git-annex-shell and rsync
+processes.
+
+Using -J4 or so will speed this up quite a lot.
+
+Without -J4:
+
+	0.22user 0.09system 0:04.84elapsed 6%CPU (0avgtext+0avgdata 33644maxresident)k
+	0inputs+832outputs (0major+16191minor)pagefaults 0swaps
+
+With -J4:
+
+	0.34user 0.06system 0:01.35elapsed 30%CPU (0avgtext+0avgdata 35996maxresident)k
+	0inputs+752outputs (0major+17451minor)pagefaults 0swaps
+"""]]
diff --git a/doc/todo/speed_up_transfers_over_ssh+rsync_--_directly_reuse_the_same_connection__63__/comment_4_2669ce8914d75eda225607d4d0b45ab0._comment b/doc/todo/speed_up_transfers_over_ssh+rsync_--_directly_reuse_the_same_connection__63__/comment_4_2669ce8914d75eda225607d4d0b45ab0._comment
new file mode 100644
index 0000000000..e6c16db6d6
--- /dev/null
+++ b/doc/todo/speed_up_transfers_over_ssh+rsync_--_directly_reuse_the_same_connection__63__/comment_4_2669ce8914d75eda225607d4d0b45ab0._comment
@@ -0,0 +1,15 @@
+[[!comment format=mdwn
+ username="https://me.yahoo.com/a/EbvxpTI_xP9Aod7Mg4cwGhgjrCrdM5s-#7c0f4"
+ subject="my bad"
+ date="2015-08-05T02:33:46Z"
+ content="""
+indeed my use of ts timestamp was somewhat incorrect ;), I have used ts \"%b %d %H:%M:%S.%s\"
+
+\" This is quite likely simply the overhead of git-annex needing to query git to work out what remote each file is located on\" -- unlikely since CPU utilization is close to none.
+
+\"coupled with the overhead of needing to start a new git-annex-shell and rsync processes\" -- that is the likely major part of the overhead here 
+
+-J is indeed of notable help BUT overall disproportionate to mitigate the overhead at large.  I seem can successfully raise it to -J 4. With -J 6 I already start getting \"E: channel 22: open failed: administratively prohibited: open failed\" from time to time (not sure if it is benign or results in that particular transfer not succeeding). Not quite sure on the exact reason for it, i.e. why server side refuses to open a new channel -- I guess because of the same issue of too quickly following requests for new connections (i.e. \"the overhead\").
+  
+Why transfer requests could not be queued up and batched for execution within the same ssh invocation?  Keys are unique, so could be received within the same tmp/ in a batch transfer and then sorted into their corresponding hash subdirs.   Bundled with -J to possibly even split among multiple source remotes from which different content could be available from it could lead to greatly improved transfer rates.  Or would it be a major undertaking?
+"""]]
diff --git a/doc/todo/speed_up_transfers_over_ssh+rsync_--_directly_reuse_the_same_connection__63__/comment_5_fca151591d0c6c75013711cb5de81d47._comment b/doc/todo/speed_up_transfers_over_ssh+rsync_--_directly_reuse_the_same_connection__63__/comment_5_fca151591d0c6c75013711cb5de81d47._comment
new file mode 100644
index 0000000000..a7130ea749
--- /dev/null
+++ b/doc/todo/speed_up_transfers_over_ssh+rsync_--_directly_reuse_the_same_connection__63__/comment_5_fca151591d0c6c75013711cb5de81d47._comment
@@ -0,0 +1,18 @@
+[[!comment format=mdwn
+ username="joey"
+ subject="""comment 5"""
+ date="2015-08-11T17:27:23Z"
+ content="""
+AFAIK, it's not possible to run multiple separate rsync transfers over a single
+connection; rsync closes the connection after a transfer is complete (and
+the rsync protocol is nowhere documented, and has only one implementation
+so there's no way to avoid this behavior). 
+
+So, rsync would need to be told a whole list of files to transfer in one go,
+which poses many difficulties, including for git-annex's progress display,
+and for needing to queue up a set of files when eg `git annex get` is picking
+which ones to get.
+
+I'd want to see a lot more measurements of where bottlenecks are in
+the current system, before seriously considering such a thing.
+"""]]
diff --git a/doc/todo/speed_up_transfers_over_ssh+rsync_--_directly_reuse_the_same_connection__63__/comment_6_8f9a3dd1c021b862f97934b6b25f6aa9._comment b/doc/todo/speed_up_transfers_over_ssh+rsync_--_directly_reuse_the_same_connection__63__/comment_6_8f9a3dd1c021b862f97934b6b25f6aa9._comment
new file mode 100644
index 0000000000..009a21af07
--- /dev/null
+++ b/doc/todo/speed_up_transfers_over_ssh+rsync_--_directly_reuse_the_same_connection__63__/comment_6_8f9a3dd1c021b862f97934b6b25f6aa9._comment
@@ -0,0 +1,79 @@
+[[!comment format=mdwn
+ username="joey"
+ subject="""measure twice, cut once"""
+ date="2015-08-13T17:22:17Z"
+ content="""
+Debug output has been enhanced with fractional seconds and also shows when
+a command exits so the time spent in a command can be determined.
+
+	[2015-08-13 13:54:29.578099] process done ExitSuccess
+	ok
+	get ook8 (from origin...) 
+	[2015-08-13 13:54:29.588827] read: rsync ["--progress","--inplace","--perms","-e","'ssh' '-S' '.git/annex/ssh/localhost' '-o' 'ControlMaster=auto' '-o' 'ControlPersist=yes' '-T' 'localhost' 'git-annex-shell ''sendkey'' ''/home/joey/tmp/r'' ''SHA256E-s30--fc394d2854169d3a85b7ffda59f30b797e915fd98368c30d11588df7a20ee128'' --uuid ba2a51f9-f356-40a7-9600-2ac4193c7d58 ''--'' ''remoteuuid=72078ee3-1150-45f0-b00e-53e971921982'' ''direct='' ''associatedfile=ook8'' ''--'''","--","dummy:",".git/annex/tmp/SHA256E-s30--fc394d2854169d3a85b7ffda59f30b797e915fd98368c30d11588df7a20ee128"]
+	SHA256E-s30--fc394d2854169d3a85b7ffda59f30b797e915fd98368c30d11588df7a20ee128
+	             30 100%   29.30kB/s    0:00:00 (xfr#1, to-chk=0/1)
+	[2015-08-13 13:54:29.635771] transferinfo starting up
+	[2015-08-13 13:54:29.636097] feed: ssh ["-S",".git/annex/ssh/localhost","-o","ControlMaster=auto","-o","ControlPersist=yes","-T","localhost","git-annex-shell 'transferinfo' '/home/joey/tmp/r' 'SHA256E-s30--fc394d2854169d3a85b7ffda59f30b797e915fd98368c30d11588df7a20ee128' --uuid ba2a51f9-f356-40a7-9600-2ac4193c7d58 '--' 'remoteuuid=72078ee3-1150-45f0-b00e-53e971921982' 'associatedfile=ook8' '--'"]
+	[2015-08-13 13:54:29.656803] process done ExitSuccess
+	[2015-08-13 13:54:29.657487] transferinfo shutting down
+	[2015-08-13 13:54:29.987744] process done ExitSuccess
+	ok
+
+Here we can see:
+
+1. 0.409645s -- total time spent on this one file (12 files took 4.124s overall)
+2. 0.010728s -- is spent in between finishing one file and
+   starting the transfer of the next. This includes updating location
+   tracking, figuring out what the next file that needs transfer is 
+   (in this case the very next one in the work tree), and other bookkeeping.
+3. 0.067976s -- The rsync transfer of one (small) file, including ssh and
+   git-annex-shell overhead. It would be good to get this part broken down
+   in more detail.
+4. 0.330941 -- This is pure overhead involved in shutting down the
+   transferinfo process. Yikes!
+
+So, that transferinfo is the unexpected meat of the time. Since all
+that does it open another ssh connection back to the remote and tell git-annex-shell
+the progress of the download, for git-annex status or the assistant to display,
+that's really not a justifyable overhead.
+
+Here it is without the transferinfo being done:
+
+	[2015-08-13 13:49:26.860914] process done ExitSuccess
+	ok
+	get ook8 (from origin...) 
+	[2015-08-13 13:49:26.865482] read: rsync ["--progress","--inplace","--perms","-e","'ssh' '-S' '.git/annex/ssh/localhost' '-o' 'ControlMaster=auto' '-o' 'ControlPersist=yes' '-T' 'localhost' 'git-annex-shell ''sendkey'' ''/home/joey/tmp/r'' ''SHA256E-s30--fc394d2854169d3a85b7ffda59f30b797e915fd98368c30d11588df7a20ee128'' --uuid ba2a51f9-f356-40a7-9600-2ac4193c7d58 ''--'' ''remoteuuid=72078ee3-1150-45f0-b00e-53e971921982'' ''direct='' ''associatedfile=ook8'' ''--'''","--","dummy:",".git/annex/tmp/SHA256E-s30--fc394d2854169d3a85b7ffda59f30b797e915fd98368c30d11588df7a20ee128"]
+	SHA256E-s30--fc394d2854169d3a85b7ffda59f30b797e915fd98368c30d11588df7a20ee128
+	             30 100%   29.30kB/s    0:00:00 (xfr#1, to-chk=0/1)
+	[2015-08-13 13:49:26.926932] process done ExitSuccess
+	ok
+
+1. 0.066018s -- massive improvement! (12 files took 0.885s overall)
+2. 0.004568s -- Weird that this is half what it was before. I have made enough
+   measurements that I'm pretty sure it is, consistently lower though. Maybe
+   this time reduced because avoiding the transferinfo means less threads and/or
+   less garbage collection time.
+3. 0.061450s -- Much as before, so the concurrent ssh connection made
+   for transferinfo doesn't slow down the rsync appreciably.
+
+So, I made it spin off a new thread to do the transferinfo cleanup.
+This one word change to the code performs almost as well as eliminating
+transferinfo entirely did!
+
+	[2015-08-13 14:11:10.275867] process done ExitSuccess
+	[2015-08-13 14:11:10.276359] transferinfo shutting down
+	ok
+	get ook8 (from origin...) 
+	[2015-08-13 14:11:10.282027] read: rsync ["--progress","--inplace","--perms","-e","'ssh' '-S' '.git/annex/ssh/localhost' '-o' 'ControlMaster=auto' '-o' 'ControlPersist=yes' '-T' 'localhost' 'git-annex-shell ''sendkey'' ''/home/joey/tmp/r'' ''SHA256E-s30--fc394d2854169d3a85b7ffda59f30b797e915fd98368c30d11588df7a20ee128'' --uuid ba2a51f9-f356-40a7-9600-2ac4193c7d58 ''--'' ''remoteuuid=72078ee3-1150-45f0-b00e-53e971921982'' ''direct='' ''associatedfile=ook8'' ''--'''","--","dummy:",".git/annex/tmp/SHA256E-s30--fc394d2854169d3a85b7ffda59f30b797e915fd98368c30d11588df7a20ee128"]
+	SHA256E-s30--fc394d2854169d3a85b7ffda59f30b797e915fd98368c30d11588df7a20ee128
+	             30 100%   29.30kB/s    0:00:00 (xfr#1, to-chk=0/1)
+	[2015-08-13 14:11:10.324395] transferinfo starting up
+	[2015-08-13 14:11:10.324624] feed: ssh ["-S",".git/annex/ssh/localhost","-o","ControlMaster=auto","-o","ControlPersist=yes","-T","localhost","git-annex-shell 'transferinfo' '/home/joey/tmp/r' 'SHA256E-s30--fc394d2854169d3a85b7ffda59f30b797e915fd98368c30d11588df7a20ee128' --uuid ba2a51f9-f356-40a7-9600-2ac4193c7d58 '--' 'remoteuuid=72078ee3-1150-45f0-b00e-53e971921982' 'associatedfile=ook8' '--'"]
+	[2015-08-13 14:11:10.34482] process done ExitSuccess
+
+1. 0.068953s -- nearly as good as eliminating transferinfo (12 files took 0.925s overall)
+2. 0.006160 -- little bit higher, perhaps this is thread or GC overhead
+3. 0.062793 -- much as before (possibly slowed a tiny bit by the extra ssh traffic)
+
+Now, with -J4, all 12 files transfer in 0.661s.
+"""]]
diff --git a/doc/todo/speed_up_transfers_over_ssh+rsync_--_directly_reuse_the_same_connection__63__/comment_7_3161719c0e2a72cdc44d7c7263c44ed9._comment b/doc/todo/speed_up_transfers_over_ssh+rsync_--_directly_reuse_the_same_connection__63__/comment_7_3161719c0e2a72cdc44d7c7263c44ed9._comment
new file mode 100644
index 0000000000..09082f3368
--- /dev/null
+++ b/doc/todo/speed_up_transfers_over_ssh+rsync_--_directly_reuse_the_same_connection__63__/comment_7_3161719c0e2a72cdc44d7c7263c44ed9._comment
@@ -0,0 +1,35 @@
+[[!comment format=mdwn
+ username="joey"
+ subject="""comment 7"""
+ date="2015-08-13T18:27:35Z"
+ content="""
+I've made --debug be passed along to git-annex-shell, so here's a more
+detailed view:
+
+	[2015-08-13 15:29:05.183231] read: rsync ["--progress","--inplace","--perms","-e","'ssh' '-S' '.git/annex/ssh/localhost' '-o' 'ControlMaster=auto' '-o' 'ControlPersist=yes' '-T' 'localhost' 'git-annex-shell ''sendkey'' ''/home/joey/tmp/r'' ''--debug'' ''SHA256E-s30--6b1bd2aa19d658de59c3a5a1eb239cefb87bdfa0f9b64d1a670d88c211972c0d'' --uuid ba2a51f9-f356-40a7-9600-2ac4193c7d58 ''--'' ''remoteuuid=72078ee3-1150-45f0-b00e-53e971921982'' ''direct='' ''associatedfile=ook2'' ''--'''","--","dummy:",".git/annex/tmp/SHA256E-s30--6b1bd2aa19d658de59c3a5a1eb239cefb87bdfa0f9b64d1a670d88c211972c0d"]
+	[2015-08-13 15:29:05.219572] transfer start
+	[2015-08-13 15:29:05.221625] call: rsync ["--server","-t","--inplace","-e.Lsf",".","--sender","tmp/r/.git/annex/objects/wX/k9/SHA256E-s30--6b1bd2aa19d658de59c3a5a1eb239cefb87bdfa0f9b64d1a670d88c211972c0d/SHA256E-s30--6b1bd2aa19d658de59c3a5a1eb239cefb87bdfa0f9b64d1a670d88c211972c0d"]
+	[2015-08-13 15:29:05.226709] process done ExitSuccess # rsync server
+	[2015-08-13 15:29:05.22709] transfer done
+	[2015-08-13 15:29:05.247464] process done ExitSuccess # rsync client
+
+1. 0.036341s -- starting local rsync, ssh connection (using mux), and git-annex-shell startup to the point it parses parameters and can start
+   printing debug messages
+2. 0.002053s -- time git-annex-shell spends starting transfer (locking file, etc)
+3. 0.005084s -- actual file transfer time (including rsync server startup)
+4. 0.000381s -- git-annex-shell shutdown time
+5. 0.020374s -- time from when git-annex-shell exits until the local rsync exits.  
+   This must consist of rsync writing the file, the ssh connection shutdown, and rsync shutdown. 
+   I don't know in what porportions, except that writing the file is only a very small part of it.
+
+So, over 25000 files, I'd estimate the newly optimised git-annex to take
+around 28 minutes here (unoptimized would have taken 171 minutes).
+The rough time breakdown is:
+
+* 15 minutes is needed to start the rsync clients, ssh connections, git-annex-shell processes
+* 9 minutes more overhead seems to be involved in shutting down those 
+  connections
+* 1 minute is used by git-annex-shell to do locking, etc
+* 2 minutes is used by git-annex to find files to transfer, update location logs, etc
+* 1 minute is used by rsync server to start running and send the files (assuming very small files and fast network)
+"""]]
diff --git a/doc/todo/speed_up_transfers_over_ssh+rsync_--_directly_reuse_the_same_connection__63__/comment_8_0209467fbfb83b4b5aace9767179b7ab._comment b/doc/todo/speed_up_transfers_over_ssh+rsync_--_directly_reuse_the_same_connection__63__/comment_8_0209467fbfb83b4b5aace9767179b7ab._comment
new file mode 100644
index 0000000000..a3175b2199
--- /dev/null
+++ b/doc/todo/speed_up_transfers_over_ssh+rsync_--_directly_reuse_the_same_connection__63__/comment_8_0209467fbfb83b4b5aace9767179b7ab._comment
@@ -0,0 +1,13 @@
+[[!comment format=mdwn
+ username="anarcat"
+ subject="great!"
+ date="2015-08-17T04:07:28Z"
+ content="""
+it's great to see such improvements! i was wondering if git-annex was slower than an equivalent rsync transfer... 
+
+how does git-annex compare with rsync now?
+
+how should we decide how many -J we should use? are there good heuristics on that?
+
+should this todo be marked as done?
+"""]]
diff --git a/doc/todo/speed_up_transfers_over_ssh+rsync_--_directly_reuse_the_same_connection__63__/comment_9_88dbe6602f273b7600776228ec099955._comment b/doc/todo/speed_up_transfers_over_ssh+rsync_--_directly_reuse_the_same_connection__63__/comment_9_88dbe6602f273b7600776228ec099955._comment
new file mode 100644
index 0000000000..ae4e77279b
--- /dev/null
+++ b/doc/todo/speed_up_transfers_over_ssh+rsync_--_directly_reuse_the_same_connection__63__/comment_9_88dbe6602f273b7600776228ec099955._comment
@@ -0,0 +1,14 @@
+[[!comment format=mdwn
+ username="joey"
+ subject="""comment 9"""
+ date="2018-03-06T17:29:16Z"
+ content="""
+I think that any further improvements here would need to be a fundamental 
+change, rather than shaving away fractions of a second from the above
+benchmarks. Most of the remaining overhead seems to be outside of
+git-annex's own code.
+
+I think it could use sftp, rather than rsync, when git-annex does not
+need to resume transfer of a file. git-annex could keep sftp running, and
+ask it for each file in turn, and avoid all the connection overheads.
+"""]]
diff --git a/doc/todo/ssh_special_remote.mdwn b/doc/todo/ssh_special_remote.mdwn
new file mode 100644
index 0000000000..be38f42090
--- /dev/null
+++ b/doc/todo/ssh_special_remote.mdwn
@@ -0,0 +1,44 @@
+ssh:// remotes are not special remotes. Perhaps it would be useful to have
+a special remote that wraps a ssh:// remote? This would allow setting up a
+ssh:// remote that can be enabled using the webapp's normal UI for enabling
+special remotes.
+
+Enabling such a special remote would just make a regular git remote, so
+there would be no need to implement the methods to get/put data. (Although
+it might need to provide stubs to appease the compiler.)
+
+> Above is done. The command line interface in initremote and enableremote
+> is not too easy or perhaps useful, but it works great in the webapp.
+> --[[Joey]]
+
+It could optionally embed the ssh private key into the git-annex branch as
+a credential, for when you want anyone who has access to the git repo to be
+able to use the (locked-down) git-annex-shell on that server.
+
+> Leaving this todo open for this ssh private key embedcreds part.
+> I think it makes sense to do, but it it probably not too easy.
+> ([[webapp_ssh_setup_should_work_with_locked_down_git-annex-shell_account]]
+> needs to be fixed first). --[[Joey]]
+
+[[!meta title="remember ssh remote including optionally ssh key"]]
+
+----
+
+I am on the fence about whether this would be useful, and would appreciate
+use cases.
+
+One use case I was thinking about was a LAN with a central server, with a
+shared account with a git-annex repository on it. But then I realized this
+wouldn't really help set up git-annex in that situation, most of the time,
+because new clients would have the central server added as their first
+remote. 
+
+(It would help if one client paired with another new client, but
+that is unncessarily round-about most of the time.) 
+
+It might help in a more complex situation, where the LAN is not the whole
+network an a client might come onto the LAN already knowing about the
+central server there. --[[Joey]]
+
+A very compelling use case is switching from XMPP to a ssh server,
+and wanting to make it easy for users. --[[Joey]]
diff --git a/doc/todo/stop_using_curl_and_wget.mdwn b/doc/todo/stop_using_curl_and_wget.mdwn
new file mode 100644
index 0000000000..6de3c527b4
--- /dev/null
+++ b/doc/todo/stop_using_curl_and_wget.mdwn
@@ -0,0 +1,52 @@
+Currently git-annex uses wget and curl for downloading urls.
+Which is used depends on the situation, since both have their limitations
+and quirks.
+
+This often confuses users, who expect annex.web-options to only apply
+to whichever program git-annex was running, and put in an option that
+breaks the other program. Or, configure a netrc file, which wget uses by
+default, but curl does not.
+
+Also, using these external programs avoids keeping a http connection open
+and pipelining requests, so it makes mass url downloads a lot slower than
+if git-annex used http-conduit to do url downloads itself. [[users/yoh]]
+has requested http pipelining.
+
+(git-annex was creating a new http manager each time it hit an url,
+except for in the S3 remote which reused a single manager. That's now been
+improved, so all http-conduit use in git-annex reuses a http manager, and
+so will do http pipelining.)
+
+For file: ftp: and more unusual urls, http-conduit can't support them.
+git-annex does support those urls, and people rely on that, so it would
+still need to use wget or curl for those.
+
+wget is also not shipped with git-annex on Windows or OSX, only curl is,
+and it would be good to only use one of the programs, not both, when
+handing those unusual urls.
+
+See also, [[support_.netrc_for_fsck_--from_web]]. That some users rely on
+git-annex using wget and a netrc file is kind of problimatic if switching
+to http-conduit which does not support it. Maybe require users to set
+`annex.web-download-command` if they want to make it use something that
+supports netrc?
+
+--[[Joey]]
+
+> Implemented Utility.Url.downloadC that is the (nontrivial)
+> download a file with resume support using http-conduit.
+> It falls back to curl to handle urls that http-conduit does not support.
+> Now we only have to decide what to do about the above edge cases..
+
+> > Let's drop use of wget entirely, as it was only using it because I
+> > preferred wget's progress bar to curl's. The user can still force wget
+> > with annex.web-download-command.
+> >
+> > That leaves users who have a .netrc file or want to use
+> > annex.web-options. Since curl requires --netrc in order to use the
+> > .netrc file, require users who want to use the .netrc to
+> > set "annex.web-options = --netrc". When "annex.web-options" is
+> > set, always use curl (unless overridden by annex.web-download-command).
+> > Otherwise, use conduit.
+
+[[done]] --[[Joey]]
diff --git a/doc/todo/stop_using_curl_and_wget/comment_1_c1ac05dc1969b64f08f717667cda0240._comment b/doc/todo/stop_using_curl_and_wget/comment_1_c1ac05dc1969b64f08f717667cda0240._comment
new file mode 100644
index 0000000000..a341358cd7
--- /dev/null
+++ b/doc/todo/stop_using_curl_and_wget/comment_1_c1ac05dc1969b64f08f717667cda0240._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="CandyAngel"
+ avatar="http://cdn.libravatar.org/avatar/15c0aade8bec5bf004f939dd73cf9ed8"
+ subject="comment 1"
+ date="2018-05-17T20:15:13Z"
+ content="""
+Just as a cross-reference, this may have broken downloads in some cases, where [addurl results in a different file to wget](https://git-annex.branchable.com/bugs/addurl_results_in_different_file_to_wget), seemingly because it is being decompressedon-the-fly by the http library.
+"""]]
diff --git a/doc/todo/stream_feature__63__.mdwn b/doc/todo/stream_feature__63__.mdwn
new file mode 100644
index 0000000000..860edfc811
--- /dev/null
+++ b/doc/todo/stream_feature__63__.mdwn
@@ -0,0 +1,23 @@
+I am just asking myself, is it stupid to think that streaming in git annex would be a good idea and wouldnt it be totaly easy to implement?
+
+Ok just tried to link to files over ssh, it creates a link but you cant open with it its content ^^
+
+But at least the files you have access over some filesystem as example samba/sshfs or just a other directory or usb-drive you could stream instead of "get"
+
+you could add another mode like direct and indirect, like symbolic-links or something like that?
+
+Sadly linux is to stupid to allow direct ssh links ( thats maybe one of the biggest features hurd has over linux  ) but you could mount with sshfs readonly (checking first if sshfs is installed) to mount the files there and then map the links there.
+
+ok I am not so shure how hard it would be and how much bug potentials it creates, but it would be great I guess.
+
+git annex is a bit like a telephone book, where you get a list of where the targets are. So using it to call the persons so that they drive to you to talk with you is nice. But I think the better feature would be if you just talk with the guy over the telephone directly bevore he comes to you (streaming)
+
+I mean you did one great thing, you did make cloudy thing peer to peer, like git is targeted too but for smaller files, yes there are may use cases without this feature, but I would be really glad if it could do that too, if I give annex 5 locations on other pcs usb-sticks etc, I find it stupid to additionaly do setup all this sources again a second time for streaming, and then I have maybe even 2 different file names because you map stuff in git.
+
+So sorry its late here, I am a bit tired so I maybe dont know what I am talking right now, my english isnt the best, too, but I think it would be a great feature.
+
+I mean on your setup, with slow internet, you maybe always make a get command, but here, if I link to youtube, I have no problem to stream it, or even on internal network between my pcs I have gb-lan, I start directly movies streaming, I would only use get, in rare cases where I need them on a train, the normal thing is to stream stuff.
+
+So I have to go sleep now 
+
+bye
diff --git a/doc/todo/stream_feature__63__/comment_1_99f1f1872f86d4571c03a99fff1a7212._comment b/doc/todo/stream_feature__63__/comment_1_99f1f1872f86d4571c03a99fff1a7212._comment
new file mode 100644
index 0000000000..82b43335fd
--- /dev/null
+++ b/doc/todo/stream_feature__63__/comment_1_99f1f1872f86d4571c03a99fff1a7212._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="https://www.google.com/accounts/o8/id?id=AItOawn4bbuawnh-nSo9pAh8irYAcV4MQCcfdHo"
+ nickname="Stefan"
+ subject="did I wrote this suggestion"
+ date="2015-02-11T17:25:52Z"
+ content="""
+thought about that feature just now and found this feature request here, just wonder if I wrote it, sounds a bit like me. If not I add a +1 to it :)
+"""]]
diff --git a/doc/todo/support_.netrc_for_fsck_--from_web.mdwn b/doc/todo/support_.netrc_for_fsck_--from_web.mdwn
new file mode 100644
index 0000000000..fca0bd4e34
--- /dev/null
+++ b/doc/todo/support_.netrc_for_fsck_--from_web.mdwn
@@ -0,0 +1,18 @@
+On my computer, `git annex get filename --from web` uses wget which does support
+.netrc, but for some reason `git annex fsck filename --from web` does not.
+Instead, I get a message that the content is missing and git annex is "fixing
+location log" (slightly unrelated question, but is there any way to stop it from
+doing this? Just because a URL isn't accessible now, doesn't mean it's content
+is gone permanently).
+
+I do not want to encode my credentials into the URLs (eg.
+username:password@example.com) because my password changes frequently and I would
+have to update all of the URLs.
+
+> git-annex 6.20180406 and onwards use http-conduit for everything
+> by default. To use the .netrc file, run:
+
+	git config annex.web-options --netrc
+
+> That will make git-annex use curl for all web accesses, and configures
+> curl to use your netrc file. [[done]] --[[Joey]] 
diff --git a/doc/todo/support_.netrc_for_fsck_--from_web/comment_1_27fc7996be9cbff994fb19b0e213f28b._comment b/doc/todo/support_.netrc_for_fsck_--from_web/comment_1_27fc7996be9cbff994fb19b0e213f28b._comment
new file mode 100644
index 0000000000..7dfb25da97
--- /dev/null
+++ b/doc/todo/support_.netrc_for_fsck_--from_web/comment_1_27fc7996be9cbff994fb19b0e213f28b._comment
@@ -0,0 +1,29 @@
+[[!comment format=mdwn
+ username="joey"
+ subject="""comment 1"""
+ date="2018-01-15T17:12:43Z"
+ content="""
+This is complicated because git-annex uses wget, curl, and http-conduit at
+different times. wget always uses .netrc, curl only uses it with a --netrc
+option (which git-annex does not currently provide although the user can),
+and http-conduit does not support .netrc.
+
+The choices of which git-annex uses when are constrained by the limitations
+of all three, and were chosen to make it work as well as possible. Also,
+curl and wget are not available in all installations of git-annex.
+
+As far as I can see, `git annex fsck --from web` makes the same wget/curl
+choice as `git annex get --from web` will make, typically wget, but
+curl if wget is not available or if run with --quiet.
+
+However, `git annex fsck --fast --from web` uses http-conduit by default.
+It could be changed to default to using curl (wget is not an option there).
+
+But, I'm kind of inclined toward moving away from using wget/curl at all,
+and toward http-conduit for all http urls. Thus avoiding the
+inconsistencies and various annoying behaviors of wget/curl. So, making a
+change that requires .netrc be supported going forward needs to be
+considered in that light. One possibility would be to use
+ to make netrc work with
+http-conduit.
+"""]]
diff --git a/doc/todo/support_.netrc_for_fsck_--from_web/comment_2_01b513a530e8887567aca0e187ad0f7e._comment b/doc/todo/support_.netrc_for_fsck_--from_web/comment_2_01b513a530e8887567aca0e187ad0f7e._comment
new file mode 100644
index 0000000000..6542c5958f
--- /dev/null
+++ b/doc/todo/support_.netrc_for_fsck_--from_web/comment_2_01b513a530e8887567aca0e187ad0f7e._comment
@@ -0,0 +1,9 @@
+[[!comment format=mdwn
+ username="mbekkema97@66b135681014f005a3a14c4011d148fcb6655f81"
+ nickname="mbekkema97"
+ avatar="http://cdn.libravatar.org/avatar/a5037b8a5914bd9f803af7b7e881d632"
+ subject="comment 2"
+ date="2018-01-19T03:46:29Z"
+ content="""
+That's strange, I was observing the process list and didn't see any instances of wget or curl. Could it be that `fsck` uses http-conduit to ping the URL before proceeding with the full check?
+"""]]
diff --git a/doc/todo/support___126____47__.netrc_for_http_access_authentication_specs_as_was_done_when_wget_was_used.mdwn b/doc/todo/support___126____47__.netrc_for_http_access_authentication_specs_as_was_done_when_wget_was_used.mdwn
new file mode 100644
index 0000000000..9f1834a8ff
--- /dev/null
+++ b/doc/todo/support___126____47__.netrc_for_http_access_authentication_specs_as_was_done_when_wget_was_used.mdwn
@@ -0,0 +1,28 @@
+Recent git-annex started to handle http connections internally (instead of relying on `wget` and `curl`) which is great in general.  But I wondered if it is relatively easy to retain support for authentication via "standard" `~/.netrc` which `wget` supports nicely.  But ATM (e.g. with 6.20180416) it wouldn't work
+
+Example:
+[[!format sh """
+$> git annex addurl --debug http://onerussian.com/tmp/secret/index.html --file testDemo.mnc           
+[2018-04-27 21:10:52.699047352] read: git ["--git-dir=.git","--work-tree=.","--literal-pathspecs","show-ref","git-annex"]
+[2018-04-27 21:10:52.707407583] process done ExitSuccess
+[2018-04-27 21:10:52.707650197] read: git ["--git-dir=.git","--work-tree=.","--literal-pathspecs","show-ref","--hash","refs/heads/git-annex"]
+[2018-04-27 21:10:52.711677639] process done ExitSuccess
+[2018-04-27 21:10:52.711863237] read: git ["--git-dir=.git","--work-tree=.","--literal-pathspecs","log","refs/heads/git-annex..1aef0c7917a50ea5a23702f04c6e7778ea5c8843","--pretty=%H","-n1"]
+[2018-04-27 21:10:52.719171987] process done ExitSuccess
+[2018-04-27 21:10:52.719903748] chat: git ["--git-dir=.git","--work-tree=.","--literal-pathspecs","cat-file","--batch"]
+[2018-04-27 21:10:52.721168737] chat: git ["--git-dir=.git","--work-tree=.","--literal-pathspecs","cat-file","--batch-check=%(objectname) %(objecttype) %(objectsize)"]
+addurl http://onerussian.com/tmp/secret/index.html 
+
+failed
+[2018-04-27 21:10:53.010687336] process done ExitSuccess
+[2018-04-27 21:10:53.011302754] process done ExitSuccess
+"""]]
+
+whenever works with e.g. 6.20180316 if corresponding settings are setup in `~/.netrc`
+
+Or may be there is some other alternative support for such cases now?
+
+[[!meta author=yoh]]
+
+> closing as there is a documented way to make a netrc work. [[done]]
+> --[[Joey]]
diff --git a/doc/todo/support___126____47__.netrc_for_http_access_authentication_specs_as_was_done_when_wget_was_used/comment_1_0f2f5b58020879252d716930f2396b81._comment b/doc/todo/support___126____47__.netrc_for_http_access_authentication_specs_as_was_done_when_wget_was_used/comment_1_0f2f5b58020879252d716930f2396b81._comment
new file mode 100644
index 0000000000..661453a4b3
--- /dev/null
+++ b/doc/todo/support___126____47__.netrc_for_http_access_authentication_specs_as_was_done_when_wget_was_used/comment_1_0f2f5b58020879252d716930f2396b81._comment
@@ -0,0 +1,14 @@
+[[!comment format=mdwn
+ username="joey"
+ subject="""comment 1"""
+ date="2018-04-30T20:09:20Z"
+ content="""
+.netrc was considered in [[stop_using_curl_and_wget]]. The changelog for
+the release recommends that users who need a netrc do "git config
+annex.web-options --netrc"
+
+I don't anticipate adding support for netrc files to git-annex.
+Adding a dependency on http://hackage.haskell.org/package/netrc for such a
+relatively little-used feature which can be supported as above does not
+seem worth it.
+"""]]
diff --git a/doc/todo/support_disabling_verification_of_transfer_over_p2p_protocol.mdwn b/doc/todo/support_disabling_verification_of_transfer_over_p2p_protocol.mdwn
new file mode 100644
index 0000000000..c00e4663a1
--- /dev/null
+++ b/doc/todo/support_disabling_verification_of_transfer_over_p2p_protocol.mdwn
@@ -0,0 +1,61 @@
+git-annex-shell p2pstdio currently always verifies content it receives.
+git-annex-shell recvkey has a speed optimisation, when it's told the file
+being sent is locked, it can avoid an expensive verification, when
+annex.verify=false. (Similar for transfers in the other direction.)
+
+The P2P protocol does not have a way to communicate when that happens.
+File content can be modified while it's sent, and if annex.verify=false
+is allowed to take effect, bad data will get into the repository.
+
+It would be nice to support annex.verify=false when it's safe but not
+when the file got modified, but if it added an extra round trip
+to the P2P protocol, that could lose some of the speed gains.
+
+The way that git-annex-shell recvkey handles this is the client
+communicates to it if it's sending an unlocked file, which forces
+verification. Otherwise, verification can be skipped.
+
+Seems the best we could do with the P2P protocol, barring adding
+rsync-style rolling hashing to it, is to detect when a file got modified
+as it was being sent, and inform the peer that the data it got is invalid.
+It can then force verification.
+
+> [[done]] --[[Joey]]
+
+----
+
+A related problem is resumes. What if a file starts to be transferred,
+gets changed while it's transferred so some bad bytes are sent, then the
+transfer is interrupted, and later is resumed from a different remote
+that has the correct content. How can it tell that the bad data was sent
+in this case?
+
+In the case where an upload is started from one repository and later
+resumed by another, rsync wipes out any differences, so if the first
+repository was unlocked, and the second is locked, it's safe for recvkey to
+treat it locked and skip verification.
+
+This is not really unique to the P2P protocol -- special remotes
+can be written to support resuming. The web special remote does; there may
+be external special remotes that do too. While the content of a key on
+a special remote is not allowed to change, a download could start from
+an unlocked git repo, and then be resumed from such a special remote.
+When verification is disabled, this can result in bad content getting into
+the repository.
+
+So, let's solve this broadly. Whenever a download is resumed, force
+AlwaysVerify, unless the remote returns Verified. This can be done in
+Annex.Content.getViaTmp, so it will affect all downloads involving the tmp
+key for a file.
+
+> [[done]] --[[Joey]]
+
+This would change handling of resumes of downloads using rsync too.
+But those are always safe to skip verification of, although they don't
+quite do a full verification of the key's hash. To still allow disabling of
+verification of those, could add a third state in between UnVerified and
+Verified, that means it's sure it's gotten exactly the same bytes as are on
+the remote.
+
+> Decided this added too much complexity for such an edge case, so
+> skipped dealing with it. --[[Joey]]
diff --git a/doc/todo/support_disabling_verification_of_transfer_over_p2p_protocol/comment_1_70840f9fc06f2f111f89d5b6cf6ad7e0._comment b/doc/todo/support_disabling_verification_of_transfer_over_p2p_protocol/comment_1_70840f9fc06f2f111f89d5b6cf6ad7e0._comment
new file mode 100644
index 0000000000..a3346fa6a3
--- /dev/null
+++ b/doc/todo/support_disabling_verification_of_transfer_over_p2p_protocol/comment_1_70840f9fc06f2f111f89d5b6cf6ad7e0._comment
@@ -0,0 +1,11 @@
+[[!comment format=mdwn
+ username="https://openid.stackexchange.com/user/3ee5cf54-f022-4a71-8666-3c2b5ee231dd"
+ nickname="Anthony DeRobertis"
+ avatar="http://cdn.libravatar.org/avatar/1007bfece547e9f2d86fafa142cd240a62456c37f104a9d96ba9db5bf18e1934"
+ subject="How expensive is verification anyway?"
+ date="2018-03-13T17:58:19Z"
+ content="""
+A quick check with `openssl speed -evp sha256` (and `-evp sha512`) shows SHA-256 is ~210MB/sec and SHA-512 ~275MB/sec on my not that recent i7 930. I'm sure its more of an issue on embedded or phones... but there of course the network is often slower. If the hash is faster than the transfer, seems like it shouldn't really cause that much of a slowdown (unless I guess its competing with ssh for CPU time, but that seems unlikely on multi-core machines).
+
+I wonder if a easier way to speed it up wouldn't be to support BLAKE2 as a backend. Wouldn't help with existing hash keys, of course (without a migrate).
+"""]]
diff --git a/doc/todo/support_disabling_verification_of_transfer_over_p2p_protocol/comment_2_22831e802e6e4b6a95e1901699c11d09._comment b/doc/todo/support_disabling_verification_of_transfer_over_p2p_protocol/comment_2_22831e802e6e4b6a95e1901699c11d09._comment
new file mode 100644
index 0000000000..bef2c7c7e9
--- /dev/null
+++ b/doc/todo/support_disabling_verification_of_transfer_over_p2p_protocol/comment_2_22831e802e6e4b6a95e1901699c11d09._comment
@@ -0,0 +1,12 @@
+[[!comment format=mdwn
+ username="joey"
+ subject="""comment 2"""
+ date="2018-03-13T19:10:38Z"
+ content="""
+The hash speed is not really significant; re-reading the content of the
+file to hash it is the expensive part. At some point git-annex may gain the
+ability to do this on the fly as it's being downloaded and then the
+verification overhead would be negligible.
+
+Thanks for mentioning BLAKE2.. I've added support for it!
+"""]]
diff --git a/doc/todo/support_for_storing_xattrs.mdwn b/doc/todo/support_for_storing_xattrs.mdwn
new file mode 100644
index 0000000000..6edf0a9d26
--- /dev/null
+++ b/doc/todo/support_for_storing_xattrs.mdwn
@@ -0,0 +1,15 @@
+It would be nice to have support for storing extended attributes in git-annex.
+
+There are some useful [proposed metadata attributes](https://www.freedesktop.org/wiki/CommonExtendedAttributes/#proposedmetadataattributes):
+
+* user.xdg.comment: A comment specified by the user. This comment could be displayed by file managers
+* user.xdg.origin.url: Set on a file downloaded from a url. Its value should equal the url it was downloaded from.
+* user.xdg.origin.email.subject: Set on an email attachment when saved to disk. It should get its value from the originating message's "Subject" header
+* user.xdg.origin.email.from: Set on an email attachment when saved to disk. It should get its value from the originating messsage's "From" header. For example '"John Doe" ', or 'jdoe@example.com'
+* user.xdg.origin.email.message-id: Set on an email attachment when saved to disk. It should get its value from the originating message's "Message-Id" header. 
+
+At least `curl --xattr` saves `xdg.origin.url`.
+
+Perhaps `git-annex-metadata` could be leveraged to automatically store and restore xattrs? Might even work that addition of xattrs would always have to be done through a git-annex command, but restoration would be done automatically if git-annex noticed there are xattrs stored in metadata, and the file system is mounted with `user_xattr`.
+
+The `user` namespace is used for user xattrs and thus for "proposed metadata attributes" above. These attributes are valid git-annex metadata fields as-is.
diff --git a/doc/todo/support_for_storing_xattrs/comment_1_55c839d1b8af5150a5df29f0314e4b79._comment b/doc/todo/support_for_storing_xattrs/comment_1_55c839d1b8af5150a5df29f0314e4b79._comment
new file mode 100644
index 0000000000..5708a10062
--- /dev/null
+++ b/doc/todo/support_for_storing_xattrs/comment_1_55c839d1b8af5150a5df29f0314e4b79._comment
@@ -0,0 +1,12 @@
+[[!comment format=mdwn
+ username="unqueued"
+ avatar="http://cdn.libravatar.org/avatar/3bcbe0c9e9825637ad7efa70f458640d"
+ subject="comment 1"
+ date="2018-05-28T14:55:34Z"
+ content="""
+I wrote a quick and dirty script to store xattrs in a git repo, because I wanted to keep my OSX labels and other metadata like resource forks in my annex. But it only works on OSX at the moment.
+
+https://github.com/unqueued/git-xattr
+
+I think maybe metadata storage would be best implemented as a hook.
+"""]]
diff --git a/doc/todo/support_for_storing_xattrs/comment_2_c10f87259aef110510014ba16d0792ad._comment b/doc/todo/support_for_storing_xattrs/comment_2_c10f87259aef110510014ba16d0792ad._comment
new file mode 100644
index 0000000000..f35944910f
--- /dev/null
+++ b/doc/todo/support_for_storing_xattrs/comment_2_c10f87259aef110510014ba16d0792ad._comment
@@ -0,0 +1,26 @@
+[[!comment format=mdwn
+ username="joey"
+ subject="""comment 2"""
+ date="2018-05-29T16:51:48Z"
+ content="""
+I also would rather see this as a hook than built into git-annex. It's
+something that git-annex's metadata can be leveraged for, perhaps,
+but different use cases call for different amounts of data.
+
+I suppose the thing to provide a hook into is whenever git-annex adds an
+object content to .git/annex/objects. That will allow the hook
+to store the current xattrs after `git annex add`, and to restore them
+after eg `git annex get`.
+
+But, suppose you run: `git annex add file; git annex move file --to remote`
+Then the remote won't have the updated git-annex branch yet when it stores the
+object content, and so the hook run on it won't be able to do anything.
+
+When new git-annex branch version adds/changes xattr values,
+the local repository would need to be updated to reflect them.
+So, there would also need to be a hook that's run when git-annex
+metadata has changed.
+
+See also: [[lockdown_hooks]] which have different use cases, but seem to
+call for the same kind of hooks.
+"""]]
diff --git a/doc/todo/support_longer_file_extensions.mdwn b/doc/todo/support_longer_file_extensions.mdwn
new file mode 100644
index 0000000000..338cf295d7
--- /dev/null
+++ b/doc/todo/support_longer_file_extensions.mdwn
@@ -0,0 +1,7 @@
+Current *E key-value backends support file extensions of length <=4.  Files with longer extensions (such as .fasta files common in bioinformatics) get linked to extension-less files, potentially causing hard-to-predict problems.  Simple fix is to add backends like MD5E5 which keeps extensions of length <=5 .   Better fix would be to keep the entire filename:
+file myfile.fasta becomes the symlink .git/annex/objects/xx/xx/key/myfile.fasta .  If there's anotherfile.fasta with the same key but different filename, it becomes a symlink to
+.git/annex/objects/xx/xx/key/anotherfile.fasta which is a hardlink to myfile.fasta . An added plus is that the symlinks checked into git typically becomes shorter. Or, for better backwards compatibility, the symlinks checked into git don't change, but
+.git/annex/objects/xx/xx/key/key becomes a symlink to .git/annex/objects/xx/xx/key/myfile.fasta .   However, if there is anotherfile.fasta with the same key, its symlink will still end up terminating at myfile.fasta rather than anotherfile.fasta .
+It's useful to preserve full filenames, because it's not uncommon to e.g. encode parameter information in filenames (myresult.threshold100.dat); and it's not uncommon to call something like python's os.path.realpath to unwind symlink chains before processing a file.
+
+> [[done]] --[[Joey]]
diff --git a/doc/todo/support_longer_file_extensions/comment_1_c17fdd288b4912455f48d95b0bdc2390._comment b/doc/todo/support_longer_file_extensions/comment_1_c17fdd288b4912455f48d95b0bdc2390._comment
new file mode 100644
index 0000000000..d298c06d43
--- /dev/null
+++ b/doc/todo/support_longer_file_extensions/comment_1_c17fdd288b4912455f48d95b0bdc2390._comment
@@ -0,0 +1,20 @@
+[[!comment format=mdwn
+ username="joey"
+ subject="""comment 1"""
+ date="2018-09-24T15:45:07Z"
+ content="""
+What you're suggesting with the full filenames is basically on the same
+level of complexity as direct mode or v6's unlocked files, which both avoid
+symlinks and so avoid problems with the rare program that looks at the
+extension of a symlink target, instead of the extension of the symlink.
+
+Doing something to allow longer extensions is maybe worth talking more
+about though. I am not very keen on adding a parameter to the backend name.
+
+The way git-annex picks extensions doesn't need to be stable across all
+versions of git-annex, because it's only done when initially adding a file,
+and then the key, with whatever extension, is added as-is and git-annex
+does not care about the extension thereafter. So, a
+configuration setting to pick the extension length, or something like that,
+would be possible to add.
+"""]]
diff --git a/doc/todo/support_longer_file_extensions/comment_2_a01fbe5b4f72989532051f5e1bb55104._comment b/doc/todo/support_longer_file_extensions/comment_2_a01fbe5b4f72989532051f5e1bb55104._comment
new file mode 100644
index 0000000000..541dd24a35
--- /dev/null
+++ b/doc/todo/support_longer_file_extensions/comment_2_a01fbe5b4f72989532051f5e1bb55104._comment
@@ -0,0 +1,7 @@
+[[!comment format=mdwn
+ username="joey"
+ subject="""comment 2"""
+ date="2018-09-24T15:50:47Z"
+ content="""
+Added annex.maxextensionlength configuration.
+"""]]
diff --git a/doc/todo/support_longer_file_extensions/comment_3_4408f1bd38527b2fd83a31a904aaca95._comment b/doc/todo/support_longer_file_extensions/comment_3_4408f1bd38527b2fd83a31a904aaca95._comment
new file mode 100644
index 0000000000..a4e49c8b02
--- /dev/null
+++ b/doc/todo/support_longer_file_extensions/comment_3_4408f1bd38527b2fd83a31a904aaca95._comment
@@ -0,0 +1,9 @@
+[[!comment format=mdwn
+ username="Ilya_Shlyakhter"
+ avatar="http://cdn.libravatar.org/avatar/1647044369aa7747829c38b9dcc84df0"
+ subject="comment 3"
+ date="2018-09-24T18:50:46Z"
+ content="""
+thanks a lot!
+
+"""]]
diff --git a/doc/todo/support_public_versioned_S3_access.mdwn b/doc/todo/support_public_versioned_S3_access.mdwn
new file mode 100644
index 0000000000..f9e9cd6738
--- /dev/null
+++ b/doc/todo/support_public_versioned_S3_access.mdwn
@@ -0,0 +1,9 @@
+When a S3 remote uses exporttree=yes versioning=yes public=yes,
+it's possible to use anonymous http to download anything from it. git-annex
+does not yet support that, nor does whereis show the urls.
+
+Should not be super hard to add, but it involves converting `getpublicurl`
+into an Annex action and distinguishing between different uses of it,
+some of which work with this and some don't. --[[Joey]]
+
+> [[done]] --[[Joey]]
diff --git a/doc/todo/support_ssh__58____47____47___or_sftp__58____47____47___urls_via___34__built-in__34___ssh_support.mdwn b/doc/todo/support_ssh__58____47____47___or_sftp__58____47____47___urls_via___34__built-in__34___ssh_support.mdwn
new file mode 100644
index 0000000000..ccb891df86
--- /dev/null
+++ b/doc/todo/support_ssh__58____47____47___or_sftp__58____47____47___urls_via___34__built-in__34___ssh_support.mdwn
@@ -0,0 +1,38 @@
+ATM git-annex passes sftp:// urls to be handled with curl.
+But that one does not (re)use established ssh control sockets and/or credentials
+
+E.g.
+
+[[!format sh """
+$> git annex addurl --debug sftp://localhost/tmp/123  
+[2018-01-12 09:53:35.195801836] read: git ["--git-dir=.git","--work-tree=.","--literal-pathspecs","show-ref","git-annex"]
+[2018-01-12 09:53:35.199709648] process done ExitSuccess
+[2018-01-12 09:53:35.199781091] read: git ["--git-dir=.git","--work-tree=.","--literal-pathspecs","show-ref","--hash","refs/heads/git-annex"]
+[2018-01-12 09:53:35.203370157] process done ExitSuccess
+[2018-01-12 09:53:35.204373706] chat: git ["--git-dir=.git","--work-tree=.","--literal-pathspecs","cat-file","--batch"]
+[2018-01-12 09:53:35.204819259] chat: git ["--git-dir=.git","--work-tree=.","--literal-pathspecs","cat-file","--batch-check=%(objectname) %(objecttype) %(objectsize)"]
+addurl sftp://localhost/tmp/123 [2018-01-12 09:53:35.208782745] read: curl ["-s","--head","-L","sftp://localhost/tmp/123","-w","%{http_code}","--user-agent","git-annex/6.20171211+gitgba511f4de-1~ndall+1"]
+[2018-01-12 09:53:35.290168768] process done ExitFailure 51
+
+[2018-01-12 09:53:35.290835362] call: wget ["-nv","--show-progress","--clobber","-c","-O","/tmp/bu/.git/annex/tmp/URL--sftp&c%%localhost%tmp%123","sftp://localhost/tmp/123","--user-agent","git-annex/6.20171211+gitgba511f4de-1~ndall+1"]
+sftp://localhost/tmp/123: Unsupported scheme ‘sftp’.
+[2018-01-12 09:53:35.299065531] process done ExitFailure 1
+failed
+[2018-01-12 09:53:35.299781408] process done ExitSuccess
+[2018-01-12 09:53:35.299993431] process done ExitSuccess
+git-annex: addurl: 1 failed
+
+
+"""]]
+
+So far we haven't managed to make curl behave sanely with sftp urls without us passing explicitly the login name and then a password and then it still failed:
+
+[[!format sh """
+$> curl sftp://localhost/tmp/123 -u yoh            
+Enter host password for user 'yoh':
+curl: (51) SSL peer certificate or SSH remote key was not OK
+"""]]
+
+I wondered if it would be feasible for git annex natively support ssh (scp) and/or sftp urls?  All the machinery is there since it deals with copying files over ssh already.
+
+[[!meta author=yoh]]
diff --git a/doc/todo/support_ssh__58____47____47___or_sftp__58____47____47___urls_via___34__built-in__34___ssh_support/comment_1_7932fd28b14898a0b4846856a27405b8._comment b/doc/todo/support_ssh__58____47____47___or_sftp__58____47____47___urls_via___34__built-in__34___ssh_support/comment_1_7932fd28b14898a0b4846856a27405b8._comment
new file mode 100644
index 0000000000..13887bb87a
--- /dev/null
+++ b/doc/todo/support_ssh__58____47____47___or_sftp__58____47____47___urls_via___34__built-in__34___ssh_support/comment_1_7932fd28b14898a0b4846856a27405b8._comment
@@ -0,0 +1,9 @@
+[[!comment format=mdwn
+ username="joey"
+ subject="""comment 1"""
+ date="2018-01-15T17:28:16Z"
+ content="""
+I don't think that all the machinery *is* there, because
+for urls there needs to be a way to check the file size,
+and it's not clear how to do that with a sftp or ssh url.
+"""]]
diff --git a/doc/todo/support_ssh__58____47____47___or_sftp__58____47____47___urls_via___34__built-in__34___ssh_support/comment_2_048a8f7b52a0baf4c0abbae37de00257._comment b/doc/todo/support_ssh__58____47____47___or_sftp__58____47____47___urls_via___34__built-in__34___ssh_support/comment_2_048a8f7b52a0baf4c0abbae37de00257._comment
new file mode 100644
index 0000000000..9749c61503
--- /dev/null
+++ b/doc/todo/support_ssh__58____47____47___or_sftp__58____47____47___urls_via___34__built-in__34___ssh_support/comment_2_048a8f7b52a0baf4c0abbae37de00257._comment
@@ -0,0 +1,10 @@
+[[!comment format=mdwn
+ username="yarikoptic"
+ avatar="http://cdn.libravatar.org/avatar/f11e9c84cb18d26a1748c33b48c924b4"
+ subject="size"
+ date="2018-07-31T14:19:15Z"
+ content="""
+> to check the file size, and it's not clear how to do that with a sftp or ssh url
+
+what do you mean?  why something as simple as \"ssh host stat filepath\" wouldn't work?  or it seems e.g. libssh2 provides file size information via sftp: https://curl.haxx.se/mail/lib-2011-01/0094.html
+"""]]
diff --git a/doc/todo/support_ssh__58____47____47___or_sftp__58____47____47___urls_via___34__built-in__34___ssh_support/comment_3_04be2f010aeb792e070f1ff93435fabc._comment b/doc/todo/support_ssh__58____47____47___or_sftp__58____47____47___urls_via___34__built-in__34___ssh_support/comment_3_04be2f010aeb792e070f1ff93435fabc._comment
new file mode 100644
index 0000000000..372aa680f7
--- /dev/null
+++ b/doc/todo/support_ssh__58____47____47___or_sftp__58____47____47___urls_via___34__built-in__34___ssh_support/comment_3_04be2f010aeb792e070f1ff93435fabc._comment
@@ -0,0 +1,14 @@
+[[!comment format=mdwn
+ username="joey"
+ subject="""comment 3"""
+ date="2018-08-02T17:21:13Z"
+ content="""
+Well, depends on coreutils and needs a stat(1) parser. Also, will sftp
+servers necessarily let arbitrary commands be run over ssh? Driving sftp
+interactive and parsing its ls -l would add more complexity to getting file
+size.
+
+The url security fixes also mean these uris can't be used without relaxing
+the security policy.. Makes me wonder how much special-casing makes sense
+for such an edge feature.
+"""]]
diff --git a/doc/todo/switch_from_quvi_to_youtube-dl.mdwn b/doc/todo/switch_from_quvi_to_youtube-dl.mdwn
new file mode 100644
index 0000000000..3fa7b0ccb1
--- /dev/null
+++ b/doc/todo/switch_from_quvi_to_youtube-dl.mdwn
@@ -0,0 +1,71 @@
+quvi does not seem maintained (last upstream release in 2013)
+and it supports many fewer videos than youtube-dl does.
+
+The difficulty with using youtube-dl is it, by design, does not
+provide a way to probe if it supports an url, other than running it
+and seeing if it finds a video at the url. This would make `git annex
+addurl` significantly slower if it ran youtube-dl to probe every url.
+
+It is possible to use youtube-dl to download arbitrary non-video files;
+it stores the file to disk just as wget or curl. But, that's well outside
+its intended use case, and so it does not feel like a good idea to make
+git-annex depend on using youtube-dl to download generic urls.
+(Also, youtube-dl has bugs with downloading non-video 
+urls, see for example http://bugs.debian.org/874321)
+
+So, switching to youtube-dl would probably need a new switch, like `git
+annex addurl --rip` that enables using it.
+
+(Importfeed only treats links in the feed as video urls, not enclosures,
+so this problem does not affect it and it would not need a new switch.)
+
+That would need changes to users' workflows. git-annex could keep
+supporting quvi for some time, and warn when it uses quvi, to
+help with the transition.
+
+> Alternatively, git-annex addurl could download the url first, and then
+> check the file to see if it looks like html. If so, run youtube-dl (which
+> unfortunately has to download it again) and see if it manages to rip
+> media from it. This way, addurl of non-html files does not have extra
+> overhead, and the redundant download is fairly small compared to ripping
+> the media. Only the unusual case where addurl is being used on html that
+> does not contain media becomes more expensive.
+> 
+> However, for --relaxed, running youtube-dl --get-filename would be
+> significantly more expensive since it hits the network. It seems that
+> --relaxed would need to change to not rip videos; users who want that
+> could use --fast.
+> 
+> --fast already hits the network, but
+> if it uses youtube-dl --get-filename, it would fall afoul of
+> bugs like , although those can be worked
+> around (/dev/null stderr in cast youtube-dl crashes)
+
+Another gotcha is playlists. youtube-dl downloads playlists automatically.
+But, git-annex needs to record an url that downloads a single file so that
+`git annex get` works right. So, playlists will need to be disabled when
+git-annex runs youtube-dl. But, `--no-playlist` does not always disable
+playlists. Best option seems to be `--no-playlist --playlist-items 0` which works for
+non-playlists, and downloads only 1 item from playlists (hopefully a fairly
+stable item, but who knows..).
+
+(`git annex importfeed` handles youtube playlist downloads, but needs the
+user to find the url to the rss feed for the playlist. Youtube still has
+these, although it makes them hard to find.)
+
+Another gotcha is that youtube-dl's -o option does not fully determine the
+filename it downloads to. Sometims it will tack on an additional extension
+(seen with youtube videos where it added a ".mkv").
+And --get-filename does not report the actual filename when that happens.
+This seems to be due to format merging by ffmpeg; with -f best, it does
+not merge and so does not do that.
+
+
+To do disk free space checking will need a different technique than
+git-annex normally uses, because youtube-dl does not provide an easy way to
+query for size. Could use --dump-json, but that would require downloading
+the web page yet again, so too expensive.. and, the json seems to have
+"filesize: null" for youtube videos. What does work is the --max-filesize
+option, which makes youtube-dl abort if it's too big.
+
+> [[done]] --[[Joey]]
diff --git a/doc/todo/symlinks_to_symlinks_to_the_annex.mdwn b/doc/todo/symlinks_to_symlinks_to_the_annex.mdwn
new file mode 100644
index 0000000000..06f3b3da2e
--- /dev/null
+++ b/doc/todo/symlinks_to_symlinks_to_the_annex.mdwn
@@ -0,0 +1,3 @@
+If file A is annexed and dropped, and B is a relative symlink to A, then git annex get B should result in A being fetched, but currently doesn't.
+This would especially help if B is deep within some dir 'mydir', and you do git annex get mydir: annexed files under mydir get fetched,
+but not annexed files elsewhere in the repository to which symlinks under mydir point.   So such symlinks under mydir will continue to remain broken.
diff --git a/doc/todo/symlinks_to_symlinks_to_the_annex/comment_1_54bae401b8de13c9973ef5e6d2cf7e88._comment b/doc/todo/symlinks_to_symlinks_to_the_annex/comment_1_54bae401b8de13c9973ef5e6d2cf7e88._comment
new file mode 100644
index 0000000000..b015f5f316
--- /dev/null
+++ b/doc/todo/symlinks_to_symlinks_to_the_annex/comment_1_54bae401b8de13c9973ef5e6d2cf7e88._comment
@@ -0,0 +1,9 @@
+[[!comment format=mdwn
+ username="https://me.yahoo.com/a/iOGTltEpmOTQ.xZ99NFP5c7Zdcc-#6a7ba"
+ nickname="Ilya S"
+ avatar="http://cdn.libravatar.org/avatar/8a133555cc739a35b83b07d5724d28d9e2f7852c224e949eec6fd4fb7693331e"
+ subject="comment 1"
+ date="2018-09-07T18:04:51Z"
+ content="""
+Also relative symlinks can point to other subdirs in the repository, in addition to pointing to files.  Basically, it would be good to add a command-line flag so that when git-annex-get or other command operates on a path in the repository, it would also operate on paths pointed to by relative symlinks under the given path.
+"""]]
diff --git a/doc/todo/symlinks_to_symlinks_to_the_annex/comment_2_f5c08ecd3b0b4099186d78c85b7f1e6f._comment b/doc/todo/symlinks_to_symlinks_to_the_annex/comment_2_f5c08ecd3b0b4099186d78c85b7f1e6f._comment
new file mode 100644
index 0000000000..fe80bd7821
--- /dev/null
+++ b/doc/todo/symlinks_to_symlinks_to_the_annex/comment_2_f5c08ecd3b0b4099186d78c85b7f1e6f._comment
@@ -0,0 +1,24 @@
+[[!comment format=mdwn
+ username="joey"
+ subject="""comment 2"""
+ date="2018-09-11T17:04:29Z"
+ content="""
+Supporting this would require that git-annex stat every
+file that non-git-annex symlinks point to, which seems
+like it could have a performance impact. 
+
+Also, git-annex is often processing information from git in a pipe,
+which can include the link target. In such cases it can very efficiently
+see if the link target is an annex object, but to support symlinks to
+symlinks it would have to do an additonal round trip through git, which
+would be much more innefficient than statting a symlink.
+
+And then there's the question of symlinks which point outside the git
+repository, or to another git-annex repository, or symlink loops. 
+Now we have potentially security sensitive filename parsing.
+
+It seems like a really big can of worms to open, I am not eager to do this.
+You'd have to have some *extemely* compelling use cases. So far, my
+response to the use cases provided is:
+"Doctor, Doctor… It Hurts When I Do This!"
+"""]]
diff --git a/doc/todo/symlinks_to_symlinks_to_the_annex/comment_3_409de39da33b7ecfe5b8c7b2e866225e._comment b/doc/todo/symlinks_to_symlinks_to_the_annex/comment_3_409de39da33b7ecfe5b8c7b2e866225e._comment
new file mode 100644
index 0000000000..fbab813592
--- /dev/null
+++ b/doc/todo/symlinks_to_symlinks_to_the_annex/comment_3_409de39da33b7ecfe5b8c7b2e866225e._comment
@@ -0,0 +1,28 @@
+[[!comment format=mdwn
+ username="Ilya_Shlyakhter"
+ avatar="http://cdn.libravatar.org/avatar/1647044369aa7747829c38b9dcc84df0"
+ subject="comment 3"
+ date="2018-09-18T18:41:01Z"
+ content="""
+The main reason for wanting git-annex to follow symlinks, is that the semantics of its commands (get/add/copy) operating on directories would be much more intuitive.
+I want to know that after 'git annex get subdir' all annexed files accessible under subdir/ are available; that after 'git annex add subdir && git annex move subdir --to my-remote' all
+large files accessible under subdir/ are at my-remote; etc.   I want to be able to treat a subdirectory as a self-contained unit.  But this isn't possible if relative symlinks stored
+undir subdir/ but pointing outside subdir/ might still be broken after 'git annex get subdir', etc.
+
+E.g. I have
+
+proj1/
+      big_file.dat: symlink to ../.git/annex/objects/....
+proj2/
+      big_file.dat: symlink to ../proj1/big_file.dat
+then if I do 'git annex get proj1', I can safely work on proj1 knowing all its files have been fetched; but if I do 'git annex get proj2', I can't safely work on it because
+the symlink proj2/big_file.dat is still broken.   I can of course make proje2/big_file.dat a direct link to the annex, but that loses the relationship between proj1 and proj2.
+It's pretty common to want to create a \"variant\" of a project by making most files in proj2/ symlinks to corresponding files in proj1/ , except for a few files that differ.
+
+With 'git annex add', it'd help a lot to know that 'git annex add /my/dir' will definitely store _everything_ under /my/dir.   In fact, if a symlink under /my/dir points outside the
+repository, git-annex could still store the target file in the annex and check in a symlink to that (or perhaps warn the user about the situation).   Then I can 'git annex add' my local
+setup, even if it points to some absolute paths, and check it out on another machine, with all links working.
+
+\"Supporting this would require that git-annex stat every file that non-git-annex symlinks point to\" -- only ones that point outside the subdir being worked on.  If the target of
+such a symlink is a directory, you'd need to process that directory too.  But if this all is off by default, and turned on by a flag, then the normal operation won't be affected.
+"""]]
diff --git a/doc/todo/symlinks_to_symlinks_to_the_annex/comment_4_818c85d1a199d40daca7371fefb18bc3._comment b/doc/todo/symlinks_to_symlinks_to_the_annex/comment_4_818c85d1a199d40daca7371fefb18bc3._comment
new file mode 100644
index 0000000000..997ad4f78b
--- /dev/null
+++ b/doc/todo/symlinks_to_symlinks_to_the_annex/comment_4_818c85d1a199d40daca7371fefb18bc3._comment
@@ -0,0 +1,9 @@
+[[!comment format=mdwn
+ username="Ilya_Shlyakhter"
+ avatar="http://cdn.libravatar.org/avatar/1647044369aa7747829c38b9dcc84df0"
+ subject="comment 4"
+ date="2018-09-18T18:48:06Z"
+ content="""
+p.s. you're right that symlinks to outside the repository would be a security risk.   They should not be added to the annex without explicit options. But it's important to at least warn about them, so e.g. the user knows that 'git annex add' did not add everything, or that 'git annex get subdir/' did not get everything under subdir/ .  It  _would_ be useful to handle relative symlinks into submodules though.
+Symlink loops would of course be errors.  
+"""]]
diff --git a/doc/todo/symlinks_to_symlinks_to_the_annex/comment_5_94b84cd27c36d14dc16837484d232d72._comment b/doc/todo/symlinks_to_symlinks_to_the_annex/comment_5_94b84cd27c36d14dc16837484d232d72._comment
new file mode 100644
index 0000000000..680d252d44
--- /dev/null
+++ b/doc/todo/symlinks_to_symlinks_to_the_annex/comment_5_94b84cd27c36d14dc16837484d232d72._comment
@@ -0,0 +1,9 @@
+[[!comment format=mdwn
+ username="https://christian.amsuess.com/chrysn"
+ nickname="chrysn"
+ avatar="http://christian.amsuess.com/avatar/c6c0d57d63ac88f3541522c4b21198c3c7169a665a2f2d733b4f78670322ffdc"
+ subject="symlinks into git-annex"
+ date="2018-09-18T19:15:11Z"
+ content="""
+I often work with scenarios where I want to `git annex get` files where I'm not actually in a git annex repository but in a symlink farm that fans out (often via directory-level indirections) into git annices (is there a canonical plural?). There, what I use is a short script that `readlink`s the to-be-fetched files until it contains a `.git/annex/objects` part and then fetches that in its original annex. If there were a way to do that with a standard `git annex get`, possibly (if after some deliberation there remain security concerns) behind a --follow-symlinks switch, I'd appreciate that.
+"""]]
diff --git a/doc/todo/sync_--branches__to_sync_only_specified_branches___40__e.g._git-annex__41__.mdwn b/doc/todo/sync_--branches__to_sync_only_specified_branches___40__e.g._git-annex__41__.mdwn
new file mode 100644
index 0000000000..7a401da262
--- /dev/null
+++ b/doc/todo/sync_--branches__to_sync_only_specified_branches___40__e.g._git-annex__41__.mdwn
@@ -0,0 +1,3 @@
+As we briefly discussed via email, it would be nice if sync could sync only some branches (e.g. git-annex) not all at once.
+
+[[!meta author=yoh]]
diff --git a/doc/todo/sync_--branches__to_sync_only_specified_branches___40__e.g._git-annex__41__/comment_1_741720c1b65fff2f8ecb87abbcf16223._comment b/doc/todo/sync_--branches__to_sync_only_specified_branches___40__e.g._git-annex__41__/comment_1_741720c1b65fff2f8ecb87abbcf16223._comment
new file mode 100644
index 0000000000..a45ae84c38
--- /dev/null
+++ b/doc/todo/sync_--branches__to_sync_only_specified_branches___40__e.g._git-annex__41__/comment_1_741720c1b65fff2f8ecb87abbcf16223._comment
@@ -0,0 +1,10 @@
+[[!comment format=mdwn
+ username="joey"
+ subject="""comment 1"""
+ date="2016-08-08T15:40:55Z"
+ content="""
+As I suggested in email, this could be something like `git annex sync --branch B`
+and --branch could be repeated to add other branches, with the default
+being to sync them all. `git annex sync --branch git-annex` would need to
+be special cased I think.
+"""]]
diff --git a/doc/todo/sync_--branches__to_sync_only_specified_branches___40__e.g._git-annex__41__/comment_2_73bd5d343286f61c9ac753ed3b00c149._comment b/doc/todo/sync_--branches__to_sync_only_specified_branches___40__e.g._git-annex__41__/comment_2_73bd5d343286f61c9ac753ed3b00c149._comment
new file mode 100644
index 0000000000..ae5c7ec7e6
--- /dev/null
+++ b/doc/todo/sync_--branches__to_sync_only_specified_branches___40__e.g._git-annex__41__/comment_2_73bd5d343286f61c9ac753ed3b00c149._comment
@@ -0,0 +1,10 @@
+[[!comment format=mdwn
+ username="joey"
+ subject="""comment 2"""
+ date="2018-08-06T15:55:56Z"
+ content="""
+This is quite old, is it still wanted?
+
+git's remote.name.fetch config can make it only fetch a particular branch,
+so that's one way to do this without adding an option.
+"""]]
diff --git a/doc/todo/sync_content_of_a_single_directory_or_file.mdwn b/doc/todo/sync_content_of_a_single_directory_or_file.mdwn
new file mode 100644
index 0000000000..90eb3b4326
--- /dev/null
+++ b/doc/todo/sync_content_of_a_single_directory_or_file.mdwn
@@ -0,0 +1,14 @@
+`git annex sync --content` operates on the whole work tree, not only the
+current directory. This is different than other git-annex commands, and
+makes sense in a way since git pull works like that too. But, sometimes
+I only want the content of a single directory, or perhaps file. 
+
+This could be implemented as `git annex sync --content thedir`, except
+that would conflict with the name of the remote to sync with that it
+currently takes. Perhaps `git annex sync --dir==thedir`, which
+automatically enables content syncing?
+
+--[[Joey]]
+
+> Going with --content-of, so it's clear it enables content syncing.
+> With a -C short option. [[done]] --[[Joey]]
diff --git a/doc/todo/sync_content_of_a_single_directory_or_file/comment_1_2db914588ea635aeaf7031c5ca418b2f._comment b/doc/todo/sync_content_of_a_single_directory_or_file/comment_1_2db914588ea635aeaf7031c5ca418b2f._comment
new file mode 100644
index 0000000000..a881ebb911
--- /dev/null
+++ b/doc/todo/sync_content_of_a_single_directory_or_file/comment_1_2db914588ea635aeaf7031c5ca418b2f._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="konubinix"
+ avatar="http://cdn.libravatar.org/avatar/72f2376231d98f52c59abd26745174fc"
+ subject="Nice job!"
+ date="2017-03-21T15:35:27Z"
+ content="""
+I've been waiting for this for 5 years. I can't wait to use this :-).
+"""]]
diff --git a/doc/todo/sync_my_local_git-annex_from_a_dump_remote.mdwn b/doc/todo/sync_my_local_git-annex_from_a_dump_remote.mdwn
new file mode 100644
index 0000000000..524782bc72
--- /dev/null
+++ b/doc/todo/sync_my_local_git-annex_from_a_dump_remote.mdwn
@@ -0,0 +1,6 @@
+As discussed on debconf, I have the following use case:
+
+* I have a dump remote, a folder on my webserver where files are uploaded through the web app. I don't have git on the webserver, just a plain folder.
+* I have git-annex repo on a development server. The development server polls the webserver (ssh/ftp) once in an hour and synchronizes the state of the local git-annex repo with the state found on the webserver and commits that.
+* This is not meant to be backup facility. I just want to be able to have a state on my development machine that is very likely to the state on the webserver.
+
diff --git a/doc/todo/sync_my_local_git-annex_from_a_dump_remote/comment_1_81d63854f89f00855cda5ace0fc8262a._comment b/doc/todo/sync_my_local_git-annex_from_a_dump_remote/comment_1_81d63854f89f00855cda5ace0fc8262a._comment
new file mode 100644
index 0000000000..d9abb3a3cb
--- /dev/null
+++ b/doc/todo/sync_my_local_git-annex_from_a_dump_remote/comment_1_81d63854f89f00855cda5ace0fc8262a._comment
@@ -0,0 +1,14 @@
+[[!comment format=mdwn
+ username="http://joeyh.name/"
+ ip="2001:4978:f:21a::2"
+ subject="comment 1"
+ date="2013-08-13T21:44:17Z"
+ content="""
+We had a conversation about this IRL. At the time, I thought I understood what you wanted. Reading the above, I am not so sure.
+
+What I thought you wanted was something like `git annex mirror --from remote`, which would, for each object known to git-annex that the location log said was present on the remote, make sure that the local repo had the object too, and for each object that the location log said was not present on the remote, drop it from the local repo (if numcopies etc allowed).
+
+`git annex mirror --to remote` could also be used as the complement of the above.
+
+If that's not the sort of thing you meant, let me know.
+"""]]
diff --git a/doc/todo/sync_my_local_git-annex_from_a_dump_remote/comment_2_66822b72b1450e79e8edd0c6c21d5aa6._comment b/doc/todo/sync_my_local_git-annex_from_a_dump_remote/comment_2_66822b72b1450e79e8edd0c6c21d5aa6._comment
new file mode 100644
index 0000000000..3d459371f5
--- /dev/null
+++ b/doc/todo/sync_my_local_git-annex_from_a_dump_remote/comment_2_66822b72b1450e79e8edd0c6c21d5aa6._comment
@@ -0,0 +1,14 @@
+[[!comment format=mdwn
+ username="http://thkoch2001.myopenid.com/"
+ nickname="thkoch"
+ subject="pseudocode"
+ date="2013-08-14T04:58:22Z"
+ content="""
+lets say my local annex is in direct mode, then the following might already do what I want:
+
+cd $LOCAL_ANNEX
+rsync --recursive --delete $REMOTE .
+git annex add && git commit
+
+It would however be nice if I could do the same with an annex in indirect mode or even a bare annex.
+"""]]
diff --git a/doc/todo/sync_my_local_git-annex_from_a_dump_remote/comment_3_b9f73375e2c732e798495f8ee6970c7c._comment b/doc/todo/sync_my_local_git-annex_from_a_dump_remote/comment_3_b9f73375e2c732e798495f8ee6970c7c._comment
new file mode 100644
index 0000000000..df4be033bb
--- /dev/null
+++ b/doc/todo/sync_my_local_git-annex_from_a_dump_remote/comment_3_b9f73375e2c732e798495f8ee6970c7c._comment
@@ -0,0 +1,14 @@
+[[!comment format=mdwn
+ username="http://joeyh.name/"
+ ip="4.154.0.63"
+ subject="comment 3"
+ date="2013-08-24T16:35:33Z"
+ content="""
+Seems to me that this could easily be dealt with by installing git-annex on the webserver, making the directory there a git repository, and using either a cron job or `git annex watch` to commit files as they were changed there.
+
+Then you can make a direct mode, indirect mode, or even a bare clone on your local machine and use git-annex to get the files.
+
+Maybe you have good reasons for not wanting to go that route. And rsync on a direct mode repository should work, provided to tell it to not delete `.git`. :P I don't see any way to make rsync work in an indirect mode repository. As for trying to make git-annex handle this import over rsync itself in a way that would work in an indirect mode repository, let alone a bare repository -- I don't see a good way to do it and it seems quite special case and likely to get quite complicated to implement.
+
+In the meantime, I did implement `git annex mirror`, which I think is a much more interesting and generally useful tool to have. And could even be used in my recommended solution above.
+"""]]
diff --git a/doc/todo/sync_to_non_tracking_export_confusing.mdwn b/doc/todo/sync_to_non_tracking_export_confusing.mdwn
new file mode 100644
index 0000000000..2f67fed5d8
--- /dev/null
+++ b/doc/todo/sync_to_non_tracking_export_confusing.mdwn
@@ -0,0 +1,21 @@
+git annex sync --content to a exporttree remote that does not have a
+tracking branch configured can be confusing. It will sometimes try to
+upload files to the remote, but not files that were just committed.
+
+Part of the problem is that the tracking branch is stored in local git
+config, but the exported tree is stored in git-annex branch. So 
+`git annex sync --content` can be behaving as desired, updating a tracking
+branch, but then in a new clone of the same repository, it will behave
+differently.
+
+On the other hand, tracking branches are a locally configured concept in
+git, and that's why it seems to make sense to have them be locally
+configured in git-annex too.
+
+Maybe git-annex sync could just display a message when the remote doesn't
+have a tracking branch, to help the user understand why it's not syncing
+their recent changes to it.
+
+--[[Joey]] 
+
+> [[done]] --[[Joey]]
diff --git a/doc/todo/syncthing_special_remote.mdwn b/doc/todo/syncthing_special_remote.mdwn
new file mode 100644
index 0000000000..36c284e2f2
--- /dev/null
+++ b/doc/todo/syncthing_special_remote.mdwn
@@ -0,0 +1,110 @@
+Among all possible [[todo/Bittorrent-like_features]] implementations,
+i think [Syncthing][] is one of the most interesting ones.
+
+First off, it is already [packaged for Debian][] with an [ITP
+underway][]. Second, it seems to use a fairly simple protocol, the
+[Block Exchange Protocol][]. It doesn't try to do everything under the
+sun and keeps things simple: NAT transversal, reuse TLS primitives and
+TCP, etc. It also seems to scale pretty well, if we are to believe the
+[usage statistics][].
+
+ [Syncthing]: https://syncthing.net/
+ [packaged for Debian]: http://apt.syncthing.net/
+ [ITP underway]: https://bugs.debian.org/cgi-bin/bugreport.cgi?bug=749887
+ [Block Exchange Protocol]: https://github.com/syncthing/specs/blob/master/BEPv1.md
+ [usage statistics]: https://data.syncthing.net/
+
+It does require the syncthing daemon to be running in order to
+transfer files so it could have similar problems than the
+[[special_remotes/ipfs]] remote which is that files get locally copied
+between the git-annex repository and the special remote.
+
+Furthermore, one of the main problems with this remote is that [public
+shares are not supported][], that is, in order to share with another
+remote, both remotes need to explicitely add each other, in syncthing!
+That makes pairing a little more difficult that it needs to.
+
+ [public shares are not supported]: https://forum.syncthing.net/t/implementing-public-shares/1186
+
+[[!toc levels=2]]
+
+Possible implementations
+========================
+
+I can think of a few different ways of implementing such a remote:
+
+ 1. share the `.git/annex/objects` directory through syncthing
+ 2. copy objects to the `~/Sync` directory (or elsewhere)
+ 3. interoperate with syncthing through the API
+ 4. reimplement the [Block Exchange Protocol][] natively
+
+Sharing the objects
+-------------------
+
+This is the easiest, but maybe the most dangerous: start syncthing and
+expose the `.git/annex/objects` directory to other peers.
+
+This of course has the downside that syncthing could technically start
+destroying objects without git-annex's knowledge, which is really
+bad. Hopefully, the readonly permissions on files could keep that from
+happening, but it still seems pretty unsafe.
+
+There is a way to mark a folder as "master" which makes it ignore
+changes from other nodes, but then that breaks the peer to peer nature
+of the protocol, which is hardly what we want. Marking the repo as
+untrusted would also be an important requirement here.
+
+Copying objects
+---------------
+
+Copying objects is the safest and easiest way to implement this. Add a
+new key? You just copy it to the sync directory. Remove a key? Just
+remove the file, and syncthing picks up the change.
+
+The main problem with this approach is of course the duplication of
+data, doubling the disk usage of all objects stored in the syncthing
+remote locally.
+
+There's also the problem that we do not reflect the fact that the
+git-annex objects are (potentially) in multiple syncthing remotes, and
+thus changing the number of copies. Even worse, once a file is dropped
+on one syncthing remote, it gets dropped everywhere. The solution for
+this of course is simply treat syncthing as a single copy of the
+objects. Note that this also applies to the shared objects method
+above.
+
+This can be easily implemented with the [[special_remotes/directory]]
+special remote:
+
+    git annex initremote syncthing type=directory directory=$HOME/Sync/ encryption=none
+    git annex describe syncthing "default syncthing directory"
+    git annex untrust syncthing
+
+Note that the last step isn't necessary if the syncthing folder is
+marked as "master".
+
+Communicate with the API
+------------------------
+
+Another way would be to talk directly to the [REST API][] (there's
+also a separate [event API][] for GUIs). Currently, this doesn't seem
+to hold much promise because the APIs are mostly read-only and don't
+allow adding objects at all, for example.
+
+ [REST API]: http://docs.syncthing.net/dev/rest.html
+ [event API]: http://docs.syncthing.net/dev/events.html
+
+Reimplement the protocol
+------------------------
+
+This would involve writing a syncthing client using the
+[Block Exchange Protocol][] specification. This would allow more
+complete control over the distribution of objects and so on,
+respecting git-annex's wanted/required content policies while at the
+same time sharing the data with other syncthing endpoints. It would
+also allow for tracking the number of copies of the objects and so on.
+
+Of course, this is a major undertaking and probably the hardest
+approach, but also the one potentially giving the most benefits.
+
+-- [[anarcat]]
diff --git a/doc/todo/tahoe_lfs_for_reals.mdwn b/doc/todo/tahoe_lfs_for_reals.mdwn
new file mode 100644
index 0000000000..2caeef11d3
--- /dev/null
+++ b/doc/todo/tahoe_lfs_for_reals.mdwn
@@ -0,0 +1,23 @@
+[[forum/tips:_special__95__remotes__47__hook_with_tahoe-lafs]] is a good
+start, but Zooko points out that using Tahoe's directory translation layer
+incurs O(N^2) overhead as the number of objects grows. Also, making
+hash subdirectories in Tahoe is expensive. Instead it would be good to use
+it as a key/value store directly. The catch is that doing so involves
+sending the content to Tahoe, and getting back a key identifier.
+
+This would be fairly easy to do as a [[backend|backends]], which can assign its
+own key names (although typically done before data is stored in it),
+but a tahoe-lafs special remote would be more flexible.
+
+To support a special remote, a mapping is needed from git-annex keys to
+Tahoe keys, stored in the git-annex branch.
+
+> This is now done, however, there are 3 known
+> problems: 
+> 
+> * tahoe start run unncessarily 
+> * web.port can conflict 
+> * Nothing renews leases, which is a problem on grids that expire.
+>   
+
+> --[[Joey]] 
diff --git a/doc/todo/tahoe_lfs_for_reals/comment_1_0a4793ce6a867638f6e510e71dd4bb44._comment b/doc/todo/tahoe_lfs_for_reals/comment_1_0a4793ce6a867638f6e510e71dd4bb44._comment
new file mode 100644
index 0000000000..16ef882a42
--- /dev/null
+++ b/doc/todo/tahoe_lfs_for_reals/comment_1_0a4793ce6a867638f6e510e71dd4bb44._comment
@@ -0,0 +1,10 @@
+[[!comment format=mdwn
+ username="zooko"
+ ip="97.118.97.117"
+ subject="performance"
+ date="2011-05-17T19:20:39Z"
+ content="""
+Hm... O(N^2)? I think it just takes O(N). To read an entry out of a directory you have to download the entire directory (and store it in RAM and parse it). The constants are basically \"too big to be good but not big enough to be prohibitive\", I think. jctang has reported that his special remote hook performs well enough to use, but it would be nice if it were faster.
+
+The Tahoe-LAFS folks are working on speeding up mutable files, by the way, after which we would be able to speed up directories.
+"""]]
diff --git a/doc/todo/tahoe_lfs_for_reals/comment_2_80b9e848edfdc7be21baab7d0cef0e3a._comment b/doc/todo/tahoe_lfs_for_reals/comment_2_80b9e848edfdc7be21baab7d0cef0e3a._comment
new file mode 100644
index 0000000000..6dba86c47c
--- /dev/null
+++ b/doc/todo/tahoe_lfs_for_reals/comment_2_80b9e848edfdc7be21baab7d0cef0e3a._comment
@@ -0,0 +1,13 @@
+[[!comment format=mdwn
+ username="http://joey.kitenet.net/"
+ nickname="joey"
+ subject="comment 2"
+ date="2011-05-17T19:57:33Z"
+ content="""
+Whoops! You'd only told me O(N) twice before..
+
+So this is not too high priority. I think I would like to get the per-remote storage sorted out anyway, since probably it will be the thing needed to convert the URL backend into a special remote, which would then allow ripping out the otherwise unused pluggable backend infrastructure.
+
+Update: Per-remote storage is now sorted out, so this could be implemented
+if it actually made sense to do so.
+"""]]
diff --git a/doc/todo/termux_package.mdwn b/doc/todo/termux_package.mdwn
new file mode 100644
index 0000000000..4b2eeb1ee1
--- /dev/null
+++ b/doc/todo/termux_package.mdwn
@@ -0,0 +1,27 @@
+Termux is an android terminal with apt. It should be possible to build
+git-annex this way, and this would be a nice alternative (or perhaps
+replacement) for the git-annex.apk.
+
+Looks like termux uses ubuntu to build, but cross-compiles for android,
+using bionic. So, ghc-android would still need to be used to build
+git-annex.
+
+Packages: 
+
+May be easier to build an appropriate .deb from the android git-annex
+binary, and add it to termux's sources.list?
+
+> Update: There's already an open issue in termux for this!
+> 
+> --[[Joey]] 
+
+This would also help with [[tor]] support on android, since termux has a
+tor package. And would avoid needing to bundle other 
+often out of date stuff with git-annex.apk.
+
+--[[Joey]]
+
+> Retargeting this todo to be about making the git-annex linux standalone
+> build work well when unpacked in termux. --[[Joey]]
+> 
+> [[done]]; it seems to work pretty well now. --[[Joey]]
diff --git a/doc/todo/termux_package/comment_1_713568120dda33b83d02408ed9321fb8._comment b/doc/todo/termux_package/comment_1_713568120dda33b83d02408ed9321fb8._comment
new file mode 100644
index 0000000000..809e651ad9
--- /dev/null
+++ b/doc/todo/termux_package/comment_1_713568120dda33b83d02408ed9321fb8._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="sunny256"
+ avatar="http://cdn.libravatar.org/avatar/8a221001f74d0e8f4dadee3c7d1996e4"
+ subject="Oh yes"
+ date="2017-01-16T07:56:59Z"
+ content="""
+That would be great. I'm not very fond of Android because it's only a crippled Linux, but Termux has made my life with Android much better, almost like in the old days on my Nokia N900, R.I.P. :'( I often sync files between the mobile and laptop, and being able to do that in Termux from the command line would be totally awesome.
+"""]]
diff --git a/doc/todo/termux_package/comment_2_b5b80f02cf0bdebd6e5e2af681e62ca7._comment b/doc/todo/termux_package/comment_2_b5b80f02cf0bdebd6e5e2af681e62ca7._comment
new file mode 100644
index 0000000000..3f8974a861
--- /dev/null
+++ b/doc/todo/termux_package/comment_2_b5b80f02cf0bdebd6e5e2af681e62ca7._comment
@@ -0,0 +1,14 @@
+[[!comment format=mdwn
+ username="joey"
+ subject="""comment 2"""
+ date="2018-04-24T23:03:09Z"
+ content="""
+The Linux standalone arm build of git-annex WORKS in termux!
+
+Just untar somewhere other than /sdcard. Your termux user's home directory
+works. 
+
+`cd git-annex.linux && runshell`
+
+Just works. 100%. OMG
+"""]]
diff --git a/doc/todo/termux_package/comment_3_a60b6e4511c1df23d76c6e6e8028eeaa._comment b/doc/todo/termux_package/comment_3_a60b6e4511c1df23d76c6e6e8028eeaa._comment
new file mode 100644
index 0000000000..622862d73f
--- /dev/null
+++ b/doc/todo/termux_package/comment_3_a60b6e4511c1df23d76c6e6e8028eeaa._comment
@@ -0,0 +1,39 @@
+[[!comment format=mdwn
+ username="joey"
+ subject="""comment 3"""
+ date="2018-04-24T23:35:33Z"
+ content="""
+A few problems with using git-annex this way..
+
+git init fails in sdcard because it tries to chmod something. It works in
+the termux home directory. Note that only the git bundled with git-annex
+has this problem; the termux one works (probably it's patched like I did in
+the android git-annex port). Hacking the standalone build to not run the
+bundled git should work. This could be done only when uname -o = Android.
+(Update: Fixed)
+
+git annex init fails with a getUserEntryForID exception. 
+[[!commit 526243d6f5db6e16c32ed7f835da590b877f78b8]] probably fixed this,
+but not tested yet. (Update: Fixed)
+
+The webapp is able to open an url (after I upgraded termux) via xdg-open
+(alias for termux-open), but when in /sdcard/t2 it opened an url that
+chrome was not able to access, apparently a permissions problem. The
+webapp.html does not add any security on android, since it's not a
+multiuser unix system, and so it should open the url to the webapp
+directly. (Done)
+
+Accessing the sdcard may need termux-setup-storage to be run once, 
+depending on the version of android. That sets up $HOME/storage.
+The webapp ought to default to making a repository somewhere in there.
+(Done)
+
+The assistant won't start on boot, but could be made to using
+https://wiki.termux.com/wiki/Termux:Boot (Integration in place now)
+
+Apparently termux-exec sets a `LD_PRELOAD` that is not compatible, so
+the wrapper script would need to unset it. (Done now)
+
+Webapp mountwatcher crashes "getMounts: does not exist" (caught exception
+now))
+"""]]
diff --git a/doc/todo/to_and_from_multiple_remotes.mdwn b/doc/todo/to_and_from_multiple_remotes.mdwn
new file mode 100644
index 0000000000..1c8d1370fe
--- /dev/null
+++ b/doc/todo/to_and_from_multiple_remotes.mdwn
@@ -0,0 +1,67 @@
+git annex sync supports git remote groups, that should be expanded to all
+commands supporting --to and --from. And since there will be a [Remote]
+list, could also support --to foo --to bar etc.
+
+--[[Joey]]
+
+Started working on this in multifromto branch.
+
+Problem: Command.Move uses onlyActionOn, so only allows one remote to
+process a key at a time. Seems that onlyActionOn needs to be expanded to
+include the remote.
+
+Problem: The concurrency is not right yet. For example,
+`git-annex copy --to foo --to bar -J2` will start actions copying a
+file to both remotes. Assume the copy to bar finishes first.
+Then it will start an action copying the next file to foo, despite a
+copy to foo already running. 
+
+To fix this, it would help to look at Annex.activeremotes when starting the
+next action. If foo is busy, start the action on bar instead. It would
+be possible to build up a queue of actions that have not yet been started on
+a remote. That risks using a lot of memory if one remote is very slow.
+The queue would need to be capped at some amount, and when full,
+delay until the laggard remotes catches up.
+
+Bonus: git annex sync --content -J2 already works, but it has the same
+problem described above and ought to be able to be fixed the same way.
+
+Also worth noting that with --auto and -J, git-annex may make more transfers
+than preferred content settings demand, because it will start several
+transfers to different remotes at once. If only one copy is needed
+amoung all the remotes, it won't notice and a copy will be sent to all
+remotes. I think this is something the user can understand though?
+
+----
+
+Looking at commands that support --to and --from and what each should do,
+there is a lot of diversity.
+
+git annex move --to of course removes the local copy. So if moving to
+multiple remotes it would need to delay that removal until it's sent to all
+of them. And it really only ought to try to remove the local copy once
+at the end, not once per remote moved to.
+
+git annex move --from ought to spread the load amoung remotes with -Jn,
+and once a file is downloaded, it needs to try to remove it from all the
+remotes.
+
+git annex mirror --from mirrors one remote; mirroring from 
+multiple remotes does not really make any sense. mirror --to multiple
+could be done.
+
+git annex unused --from seems unlikely to make sense with multiple remotes,
+since it would result in a list of keys distributed amoung them, and what
+would be done with that? Perhaps a git annex drop --from multiple remotes,
+but that would be innefficient. A shell script looping over remotes makes
+more sense if the user wants to drop unused from multiple remotes.
+
+git annex get/fsck/copy/export/transferkey/drop/dropunused all make sense to
+support multiple remotes. But with -Jn the operations that get files behave
+differently than the operations that drop files. The gets want to balance
+load amoung the remotes, while the drops and uploads need to run each
+action over each remote.
+
+Seems two runners are needed with different concurrency behavior, one that
+balances the load amoung remotes, and one that runs the same action against
+multiple remotes concurrently.
diff --git a/doc/todo/tor.mdwn b/doc/todo/tor.mdwn
new file mode 100644
index 0000000000..3e4bc7384f
--- /dev/null
+++ b/doc/todo/tor.mdwn
@@ -0,0 +1,24 @@
+git-annex sync over tor
+
+Mostly working!
+
+Current todo list:
+
+* When a transfer can't be done because another transfer of the same 
+  object is already in progress, the message about this is output by the
+  remotedaemon --debug, but not forwarded to the peer, which shows
+  "Connection reset by peer"
+* Think about locking some more. What happens if the connection to the peer
+  is dropped while we think we're locking content there from being dropped?
+
+Eventually:
+
+* Windows and Android support.
+* Limiting authtokens to read-only access.
+* Revoking authtokens. (This and read-only need a name associated with an
+  authtoken, so the user can adjust its configuration after creating it.)
+* friend-of-a-friend peer discovery to build more interconnected networks
+  of nodes
+* Discovery of nodes on same LAN, and direct connection to them.
+* Make `git annex map` show a peer's remotes?  
+  Would it be surprising if peers can learn this information?
diff --git a/doc/todo/tracking_changes_to_metadata.mdwn b/doc/todo/tracking_changes_to_metadata.mdwn
new file mode 100644
index 0000000000..ed961c0d1a
--- /dev/null
+++ b/doc/todo/tracking_changes_to_metadata.mdwn
@@ -0,0 +1,12 @@
+I use git-annex to store my music collection, which includes many albums I've ripped but don't like, as well as many albums I do like to listen to regularly but can't put on all my devices because of storage space limits. Previously I've used a `Now Playing` directory to track this, with copies of all the symlinks, but when I edit files (to correct file tags or whatever), things can get out of sync, so now I'm trying to use metadata instead. I was thinking to add a `NowPlaying=machinename` scheme and then on each machine I can `git annex get --metadata NowPlaying=machinename`.
+
+While migrating to this scheme, I've discovered a few places where a file wasn't in `Now Playing` when I expected it to be. A good example is that some songs exist both on an artist's album as well as a compilation album, and I've `drop`ped the track from the compilation album because I already had it on the artist's album.
+
+This has made me think that it would be nice to track changes to metadata, the same way we do with git commits on the files themselves. Right now this is almost possible using the `git-annex` branch. However:
+
+1. Every change to metadata just has the commit message "update". It would be nice if I could pass a commit message using something like `git annex metadata -s 'NowPlaying-=machinename' -m "This already exists on Artist - Album"`.
+2. Looking at the metadata history for a given file requires dereferencing the git-annex symlink and knowing some git-annex internals. It would be nice if there was a `git annex metadata log ` command.
+
+From reading the documentation about [[tips/metadata_driven_views]], it seems almost like #1 would be possible using views and doing a `git commit` myself, but that still just generates an "update" message on the `git-annex` branch. Even if it did work, using `git annex view 'NowPlaying=*'` excludes all files that aren't tagged with any machine, which makes adding new files harder.
+
+What do you think? Is this abuse of the `git-annex` branch? Would this interfere with [[design/caching_database]]?
diff --git a/doc/todo/tracking_changes_to_metadata/comment_1_825c15ba36324aad58faf643057b256a._comment b/doc/todo/tracking_changes_to_metadata/comment_1_825c15ba36324aad58faf643057b256a._comment
new file mode 100644
index 0000000000..2942dbb413
--- /dev/null
+++ b/doc/todo/tracking_changes_to_metadata/comment_1_825c15ba36324aad58faf643057b256a._comment
@@ -0,0 +1,28 @@
+[[!comment format=mdwn
+ username="joey"
+ subject="""comment 1"""
+ date="2017-06-26T16:58:17Z"
+ content="""
+I don't see the benefit to having custom commit messages for metadata
+changes. The changes are there in machine-readable format, so why
+involve a human?
+
+I agree that it would be useful to have a way to look at the metadata
+history, much as `git annex log` looks at the location history.
+
+Indeed, a lot of `git annex log` could be reused; `getAllLog` and
+`getKeyLog` are the hard part and would be reusable for metadata logs.
+The result might be something like this, when run on a file "foo":
+
+	+ Thu, 22 Jun 2017 17:07:43 EST foo | author=foo
+	- Thu, 22 Jun 2017 17:07:43 EST foo | author=bar
+	+ Thu, 11 Jun 2017 11:11:11 EST foo | author=bar
+
+Note that git-annex log is necessarily slow when run on a lot of files,
+because it has to run a git command per file to get the log. `git-annex log
+--all` shows a fast stream of changes from newest first, but displays the
+git-annex key that was changed, not a filename. A version of `git annex
+log` for metadata would have these same limitations.
+
+Would this help with your use case?
+"""]]
diff --git a/doc/todo/tracking_changes_to_metadata/comment_2_03a631beaa20a479f5016def7585d363._comment b/doc/todo/tracking_changes_to_metadata/comment_2_03a631beaa20a479f5016def7585d363._comment
new file mode 100644
index 0000000000..96120ce6c5
--- /dev/null
+++ b/doc/todo/tracking_changes_to_metadata/comment_2_03a631beaa20a479f5016def7585d363._comment
@@ -0,0 +1,12 @@
+[[!comment format=mdwn
+ username="glasserc"
+ avatar="http://cdn.libravatar.org/avatar/8e865c04033751520601e9c15e58ddc4"
+ subject="comment 2"
+ date="2017-07-02T02:56:19Z"
+ content="""
+I agree that a human-readable commit message is probably the wrong place to put it. I didn't know about `git annex log` but that sounds helpful. I wouldn't need it to be fast -- it would be for rare interactive use.
+
+I'd like to not just be able to see what changed with the metadata but store a description of why as well, which is why I thought of commit messages. Would your design support that?
+
+Thanks for your consideration.
+"""]]
diff --git a/doc/todo/tracking_changes_to_metadata/comment_3_a05652472c37c463eca22f11cf6596a4._comment b/doc/todo/tracking_changes_to_metadata/comment_3_a05652472c37c463eca22f11cf6596a4._comment
new file mode 100644
index 0000000000..d95a6553f2
--- /dev/null
+++ b/doc/todo/tracking_changes_to_metadata/comment_3_a05652472c37c463eca22f11cf6596a4._comment
@@ -0,0 +1,14 @@
+[[!comment format=mdwn
+ username="joey"
+ subject="""comment 3"""
+ date="2018-09-06T17:17:38Z"
+ content="""
+While git-annex does now let annex.commitmessage be set to specify
+a commit message, it seems to me that you could include your reason for the
+metadata value as yet more metadata. Ie:
+
+	annex metadata -s 'NowPlaying-=machinename' -s 'message=This already exists on Artist - Album'
+
+And a metadata log display would then naturally include those messages next
+to the change they describe.
+"""]]
diff --git a/doc/todo/transfer_between_git-annexes.mdwn b/doc/todo/transfer_between_git-annexes.mdwn
new file mode 100644
index 0000000000..ccb6908c23
--- /dev/null
+++ b/doc/todo/transfer_between_git-annexes.mdwn
@@ -0,0 +1,20 @@
+What do you think of the ability to transfer a file between unrelated annexes? With "migrate" already taken, I would suggest "catapult" (or "teleport")!
+
+    git annex catapult dir1/ $HOME/otherannex/somedir/
+    git annex catapult dir2/thisfile.jpg $HOME/otherannex/somedir/
+
+git-annex would then:
+
+* Get list of present files
+* Copy the file to temporary space in $HOME/otherannex/.git/annex
+* fsck file
+* Move file to $HOME/otherannex/.git/annnex/objects
+* Create symlinks/directories in $HOME/otherannex/somedir/
+* Stage symlinks
+* Drop content and rm symlink
+
+with the usual modifiers (e.g. --fast would skip the fsck, --force to skip non-present files?).
+
+Reason I ask: I have a huge annex from importing the contents of a bunch of random harddrives and will eventually sort the contents into various other annexes I can put files into (personal, general family, specific people). Having git-annex guiding and checking the transfers from the sorting annex to the individual ones would be really nice.
+
+Not having this isn't a showstopper (I can use rsync) so no worries if you don't think it is worth it or think it is but put it on the backburner :) Would just be a nice-to-have.
diff --git a/doc/todo/transfer_between_git-annexes/comment_1_e90f55ea43ba9e2931d72eaf929de18c._comment b/doc/todo/transfer_between_git-annexes/comment_1_e90f55ea43ba9e2931d72eaf929de18c._comment
new file mode 100644
index 0000000000..79bad063b2
--- /dev/null
+++ b/doc/todo/transfer_between_git-annexes/comment_1_e90f55ea43ba9e2931d72eaf929de18c._comment
@@ -0,0 +1,11 @@
+[[!comment format=mdwn
+ username="joey"
+ subject="""comment 1"""
+ date="2015-06-02T17:46:24Z"
+ content="""
+This feels like it's too particular to your use case, and not general
+enough to put in git-annex.
+
+But, I think it should be possible to accomplish each of those steps using
+git-annex commands, so you could write a shell script to do it.
+"""]]
diff --git a/doc/todo/transfer_between_git-annexes/comment_2_2a494355a114a3df7ff0b35aa12ed10d._comment b/doc/todo/transfer_between_git-annexes/comment_2_2a494355a114a3df7ff0b35aa12ed10d._comment
new file mode 100644
index 0000000000..e81266a7d4
--- /dev/null
+++ b/doc/todo/transfer_between_git-annexes/comment_2_2a494355a114a3df7ff0b35aa12ed10d._comment
@@ -0,0 +1,13 @@
+[[!comment format=mdwn
+ username="CandyAngel"
+ subject="comment 2"
+ date="2015-07-15T10:20:55Z"
+ content="""
+Been thinking about this as I am getting close to needing it, but would like some advice.
+
+My current plan is to copy the symlink to the target annex, add it to the index (fix it?), copy the source file from $source/.git/annex/objects to $target/.git/annex/tmp, then use 'reinject $target/.git/annex/tmp/$keyed_file $path_to_symlink'.
+
+As far as I can tell, this is safest way (uses mostly git-annex) to transfer a file between annexes. However, when transferring a directory of files, this will end up with 1 commit per file on the git-annex branch, which may be a problem.
+
+Is there any easy way to make this \"atomic\", so that git-annex will only get a commit if everything went okay and if not, revert any changes to $target? Am I looking at 'git stash', recording the master/git-annex references before the move and resetting to them in case of an error or rebasing(fixup) git-annex on success?
+"""]]
diff --git a/doc/todo/transfer_between_git-annexes/comment_3_adc2bfd00ffa6d85e68daabf7ad12aaa._comment b/doc/todo/transfer_between_git-annexes/comment_3_adc2bfd00ffa6d85e68daabf7ad12aaa._comment
new file mode 100644
index 0000000000..3e90418eb7
--- /dev/null
+++ b/doc/todo/transfer_between_git-annexes/comment_3_adc2bfd00ffa6d85e68daabf7ad12aaa._comment
@@ -0,0 +1,12 @@
+[[!comment format=mdwn
+ username="joey"
+ subject="""comment 3"""
+ date="2015-07-15T15:50:35Z"
+ content="""
+You can avoid the git-annex branch commits by passing -c
+annex.alwayscommit=false to git-annex commands. At the end, run `git annex
+merge` to commit all the changes in one go.
+
+The git-annex branch changes are temporarily stored in .git/anenx/journal/
+files if you wanted to delete them for rollback.
+"""]]
diff --git a/doc/todo/transitive_transfers.mdwn b/doc/todo/transitive_transfers.mdwn
new file mode 100644
index 0000000000..9710402f04
--- /dev/null
+++ b/doc/todo/transitive_transfers.mdwn
@@ -0,0 +1,77 @@
+I have this situation:
+
+* `marcos`: home server, canonical repository with all my files
+  (`group=backup`)
+* `angela`: laptop, with a subset of the files (`group=manual`)
+* `VHS`: backup external USB storage, should have a redundant copy of
+  all files (`group=manual`)
+
+directly connecting the external USB drive to `marcos` is annoying, so
+I usually connect it to `angela` instead, which doesn't have all the
+files.
+
+This brings up the peculiar situation that I cannot actually backup
+all the files to `VHS` from `angela`, without first copying them
+locally.
+
+I have a few issues with that:
+
+ 1. it fails silently: if I try to copy to `VHS` and the file is not on
+    `angela`, it silently fails:
+   
+        [997]anarcat@angela:mp3$ git annex drop nothere
+        (recording state in git...)
+        [998]anarcat@angela:mp3$ git annex copy --to VHS nothere
+        [999]anarcat@angela:mp3$ git annex find --in VHS nothere
+        [1001]anarcat@angela:mp3$ git annex list nothere
+        here
+        |VHS
+        ||htcones
+        |||marcos
+        ||||web
+        |||||bittorrent
+        ||||||htconesdumb
+        |||||||
+        ___X___ nothere
+        [1002]anarcat@angela:mp3$
+        
+    this shouldn't silently fail to copy: it should warn me that it
+    can't find a file to copy, at least.
+
+ 1. it takes up more disk space: i need to download all the missing
+    files locally before I can transfer them to `VHS`. here's the way
+    I make sure files are transfered properly on `VHS`:
+   
+        git annex copy --to VHS --not --in VHS
+        git annex get --not --in VHS
+        git annex copy --to VHS --not --in VHS
+        git annex drop --not --in 'here@{yesterday}'
+        
+    the latter line is expecially problematic, because it is not
+    accurate...
+
+ 1. it's slower: i need to write files locally before I can transfer
+    them. ideally, those files would be streamed, or at least I would
+    need to buffer locally only one file at a time and not the whole
+    batch.
+
+Maybe I am missing something obvious here and there are other ways of
+doing this. I am running `6.20160902+gitgbc49d8a-1~ndall+1`.
+
+I know I could setup `angela` to be in the `transfer` group, but then
+files I don't want would end up stored on `angela`: files that are
+missing from other remotes, for example. Even worse, some files I *do*
+want could be candidates for removal on `angela` because they have
+been propagated everywhere, whereas I have a select set of files
+(hence `group=manual`) that are present in `angela` that I want to
+stay there.
+
+It seems to me at least #1 above should be fixed: `copy` shouldn't
+succeed when it can't comply with the requested preferred content
+expression.
+
+Somehow, I expected this to work, and maybe that's the core issue here:
+
+    git annex copy --from marcos --to VHS nothere
+
+Thanks for considering this! -- [[anarcat]]
diff --git a/doc/todo/transitive_transfers/comment_1_d244295696be5a8164db45f7fad1701a._comment b/doc/todo/transitive_transfers/comment_1_d244295696be5a8164db45f7fad1701a._comment
new file mode 100644
index 0000000000..48995dcb86
--- /dev/null
+++ b/doc/todo/transitive_transfers/comment_1_d244295696be5a8164db45f7fad1701a._comment
@@ -0,0 +1,47 @@
+[[!comment format=mdwn
+ username="joey"
+ subject="""comment 1"""
+ date="2016-09-21T17:53:12Z"
+ content="""
+(Let's not discuss the behavior of copy --to when the file is not locally
+present here; there is plenty of other discussion of that in eg
+)
+
+git-annex's special remote API does not allow remote-to-remote transfers
+without spooling it to a file on disk first. And it's not possible to do
+using rsync on either end, AFAICS. It would be possible in some other cases
+but this would need to be implemented for each type of remote as a new API
+call.
+
+Modern systems tend to have quite a large disk cache, so it's quite
+possible that going via a temp file on disk is not going to use a lot of
+disk IO to write and read it when the read and write occur fairly close
+together.
+
+The main benefit from streaming would probably be if it could run the
+download and the upload concurrently. But that would only be a benefit
+sometimes. With an asymmetric connection, saturating the uplink tends to
+swamp downloads. Also, if download is faster than upload, it would have to
+throttle downloads (which complicates the remote API much more), or buffer
+them to memory (which has its own complications).
+
+Streaming the download to the upload would at best speed things up by a
+factor of 2. It would probably work nearly as well to upload the previously
+downloaded file while downloading the next file.
+
+It would not be super hard to make `git annex copy --from --to` download a file,
+upload it, and then drop it, and parallelizing it with -J2 would
+keep both the --from and --to remotes bandwidth saturated pretty well.
+
+Alhough not perfectly, because two jobs could both be downloading while
+the uplink is left idle. To make it optimal, it would need to do the
+download and when done, push the upload+drop into another queue of actions
+that is processed concurrently with other downloads.
+
+And there is a complication with running that at the same time as eg `git
+annex get` of the same file. It would be surprising for get to succeed
+(because copy has already temporarily downloaded the file) and then have
+the file later get dropped. So, it seems that `copy --from --to` would need
+to stash the content away in a temp file somewhere instead of storing it
+in the annex proper.
+"""]]
diff --git a/doc/todo/transitive_transfers/comment_2_7255f5083283b0ae7e7fc41e127bd829._comment b/doc/todo/transitive_transfers/comment_2_7255f5083283b0ae7e7fc41e127bd829._comment
new file mode 100644
index 0000000000..54d9c8029d
--- /dev/null
+++ b/doc/todo/transitive_transfers/comment_2_7255f5083283b0ae7e7fc41e127bd829._comment
@@ -0,0 +1,26 @@
+[[!comment format=mdwn
+ username="JasonWoof"
+ subject="simpler use case"
+ date="2016-09-22T00:40:07Z"
+ content="""
+Here's my use case (much simpler)
+
+Three git repos:
+
+desktop: normal checkout, source of almost all annexd files, commits, etc.. The only place I run git annex commands. Not enough space to stored all annexed files
+
+main_external: bare git repo, stores all annext file contents, but no file tree. Usually connected. Purpose: primary backups
+
+old_external: like main_external, except connected only occasionally.
+
+
+I periodically copy from desktop to main_external. That's all well and good.
+
+The tricky part is when I plug in old_external and want to get everything on there. It's hard to get content onto old_external that is stored only on main_external. That's when I want to:
+
+    git annex copy --from=main_external --to=old_external --not --in old_external
+
+Note that this would _not_ copy obsolete data (ie only referenced from old git commits)  stored in old_external. I like that.
+
+To work around the lack of that feature, I try to keep coppies on desktop until I've had a chance to copy them to both external drives. It's good for numcopies, but I don't like trying to keep track of it, and I wish I could choose to let there be just one copy of things on main_external for replaceable data.
+"""]]
diff --git a/doc/todo/transitive_transfers/comment_3_e5c2ede203e7bdb5af8432df6c09268f._comment b/doc/todo/transitive_transfers/comment_3_e5c2ede203e7bdb5af8432df6c09268f._comment
new file mode 100644
index 0000000000..7023cd26a5
--- /dev/null
+++ b/doc/todo/transitive_transfers/comment_3_e5c2ede203e7bdb5af8432df6c09268f._comment
@@ -0,0 +1,81 @@
+[[!comment format=mdwn
+ username="https://anarc.at/openid/"
+ nickname="anarcat"
+ subject="thanks for considering this!"
+ date="2016-09-22T12:43:11Z"
+ content="""
+> (Let's not discuss the behavior of copy --to when the file is not
+> locally present here; there is plenty of other discussion of that in
+> eg http://bugs.debian.org/671179)
+
+Agreed, it's kind of secondary.
+
+> git-annex's special remote API does not allow remote-to-remote
+> transfers without spooling it to a file on disk first.
+
+yeah, i noticed that when writing my own special remote.
+
+> And it's not possible to do using rsync on either end, AFAICS.
+
+That is correct.
+
+> It would be possible in some other cases but this would need to be
+> implemented for each type of remote as a new API call.
+
+... and would fail for most, so there's little benefit there.
+
+how about a socket or FIFO of some sort? i know those break a lot of
+semantics (e.g. `[ -f /tmp/fifo ]` fails in bash) but they could be a
+solution...
+
+> Modern systems tend to have quite a large disk cache, so it's quite
+> possible that going via a temp file on disk is not going to use a
+> lot of disk IO to write and read it when the read and write occur
+> fairly close together.
+
+true. there are also in-memory files that could be used, although I
+don't think this would work across different process spaces.
+
+> The main benefit from streaming would probably be if it could run
+> the download and the upload concurrently.
+
+for me, the main benefit would be to deal with low disk space
+conditions, which is quite common on my machines: i often cram the
+disk almost to capacity with good stuff i want to listen to
+later... git-annex allows me to freely remove stuff when i need the
+space, but it often means i am close to 99% capacity on the media
+drives i use.
+
+>  But that would only be a benefit sometimes. With an asymmetric
+>  connection, saturating the uplink tends to swamp downloads. Also,
+>  if download is faster than upload, it would have to throttle
+>  downloads (which complicates the remote API much more), or buffer
+>  them to memory (which has its own complications).
+
+that is true.
+
+> Streaming the download to the upload would at best speed things up
+> by a factor of 2. It would probably work nearly as well to upload
+> the previously downloaded file while downloading the next file.
+
+presented like that, it's true that the benefits of streaming are not
+good enough to justify the complexity - the only problem is large
+files and low local disk space... but maybe we can delegate that
+solution to the user: \"free up at least enough space for one of those
+files you want to transfer\".
+
+[... -J magic stuff ...]
+
+> And there is a complication with running that at the same time as eg
+> git annex get of the same file. It would be surprising for get to
+> succeed (because copy has already temporarily downloaded the file)
+> and then have the file later get dropped. So, it seems that copy
+> --from --to would need to stash the content away in a temp file
+> somewhere instead of storing it in the annex proper.
+
+My thoughts exactly: actually copying the files to the local repo
+introduces all sorts of weird --numcopies nastiness and race
+conditions, it seems to me.
+
+thanks for considering this!
+"""]]
diff --git a/doc/todo/transitive_transfers/comment_4_cb92e81c1ea00ca72f9a5ee729d627f4._comment b/doc/todo/transitive_transfers/comment_4_cb92e81c1ea00ca72f9a5ee729d627f4._comment
new file mode 100644
index 0000000000..bf6887ec52
--- /dev/null
+++ b/doc/todo/transitive_transfers/comment_4_cb92e81c1ea00ca72f9a5ee729d627f4._comment
@@ -0,0 +1,11 @@
+[[!comment format=mdwn
+ username="erics"
+ subject="comment 4"
+ date="2016-09-25T00:33:56Z"
+ content="""
+> And there is a complication with running [`git annex copy --from --to`] at the same time as eg `git annex get` of the same file. It would be surprising for get to succeed (because copy has already temporarily downloaded the file) and then have the file later get dropped.
+
+A solution to this subproblem would transparently fall out of a facility for [logically dropping files](http://git-annex.branchable.com/todo/wishlist__58_____96__git_annex_drop_--relaxed__96__/#comment-9672d46a18a381971dd818cde6efbc33), which was briefly talked about a long time ago.  Just mark the file as *logically dropped*.  If the user `git annex get`s it while the copy-out is in progress, its status will change to \"present\", so *copy* will know not to physically delete it.
+
+(Of course there are race conditions involved, but I presume/hope that they're no worse than git-annex already has to deal with.)
+"""]]
diff --git a/doc/todo/transitive_transfers/comment_5_50670cb52b29dc9c0c6c981e6f6d198b._comment b/doc/todo/transitive_transfers/comment_5_50670cb52b29dc9c0c6c981e6f6d198b._comment
new file mode 100644
index 0000000000..5cc5cf25ef
--- /dev/null
+++ b/doc/todo/transitive_transfers/comment_5_50670cb52b29dc9c0c6c981e6f6d198b._comment
@@ -0,0 +1,15 @@
+[[!comment format=mdwn
+ username="joey"
+ subject="""comment 5"""
+ date="2016-10-05T19:59:13Z"
+ content="""
+@JasonWoof re your use case, you don't need transitive transfers.
+You can simply go to `old_external` and run:
+
+	git annex sync main_external
+	git annex get --branch main_external/master
+
+(Assuming that the `old_external` has a `main_external` remote.)
+
+This --branch option was added in a recent git-annex release.
+"""]]
diff --git a/doc/todo/unable_to_add_files_to_a_git-annex_repo_in_Windows.mdwn b/doc/todo/unable_to_add_files_to_a_git-annex_repo_in_Windows.mdwn
new file mode 100644
index 0000000000..bdcd43e8c8
--- /dev/null
+++ b/doc/todo/unable_to_add_files_to_a_git-annex_repo_in_Windows.mdwn
@@ -0,0 +1,15 @@
+I was trying to use the assistant, but when it didn't work I started troubleshooting. I wasn't able to get git-annex to work directly, so I'll start with it:
+
+    $ git annex add *jpg
+    add RnG.jpg
+    git-annex: .git\annex\objects\fc3\7ab\SHA256E-s63863--b26754195df2c07063401ab1dc8406a1f96c0a512724020b693e28bdbce9addf.jpg\: openTempFile: does not exist (No such file or directory)
+    failed
+    add Scan0005.jpg
+    git-annex: .git\annex\objects\3ae\9e3\SHA256E-s156803--617405c2e88c939a8cfa871a226c2fe04fe0b58b50667e4436250fcb7c59d09b.jpg\: openTempFile: does not exist (No such file or directory)
+    failed
+    (recording state in git...)
+    git-annex: add: 2 failed
+
+This is Windows 7. I've installed git 2.11 and git-annex 6.20161211-gc3ab3c6.
+
+> closing as dup of an existing bug [[done]] --[[Joey]]
diff --git a/doc/todo/unable_to_add_files_to_a_git-annex_repo_in_Windows/comment_1_2eeeab80aa00b8b47d57b59a49011820._comment b/doc/todo/unable_to_add_files_to_a_git-annex_repo_in_Windows/comment_1_2eeeab80aa00b8b47d57b59a49011820._comment
new file mode 100644
index 0000000000..83a91a6cba
--- /dev/null
+++ b/doc/todo/unable_to_add_files_to_a_git-annex_repo_in_Windows/comment_1_2eeeab80aa00b8b47d57b59a49011820._comment
@@ -0,0 +1,12 @@
+[[!comment format=mdwn
+ username="joey"
+ subject="""comment 1"""
+ date="2017-06-06T19:50:19Z"
+ content="""
+That sounds like this bug:
+
+
+Windows has some path length issues that come up especially when using
+git-annex in a deeply nested directory tree. Try cloning the repository to
+somewhere like c:\\ and see if that works around the problem.
+"""]]
diff --git a/doc/todo/union_mounting.mdwn b/doc/todo/union_mounting.mdwn
new file mode 100644
index 0000000000..c42a055021
--- /dev/null
+++ b/doc/todo/union_mounting.mdwn
@@ -0,0 +1,10 @@
+It should be possible to union mount annexes. So if multiple drives have
+content, an annex mounting them both would have available all the 
+content from all the drives.
+
+This could be done by just making .git/annex/KEY link to the actual content
+on the mounted annex.
+
+(Need to make sure the [[copy_tracking|copies]] code does not
+confused and think the symlink is a copy of the content.. Also need to make
+sure that code that writes to .git/annex does not follow symlinks.))
diff --git a/doc/todo/union_mounting/comment_1_cb08435812dd7766de26199c73f38e8b._comment b/doc/todo/union_mounting/comment_1_cb08435812dd7766de26199c73f38e8b._comment
new file mode 100644
index 0000000000..3fadf6fa3c
--- /dev/null
+++ b/doc/todo/union_mounting/comment_1_cb08435812dd7766de26199c73f38e8b._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="https://www.google.com/accounts/o8/id?id=AItOawln3ckqKx0x_xDZMYwa9Q1bn4I06oWjkog"
+ nickname="Michael"
+ subject="comment 1"
+ date="2013-03-01T01:26:36Z"
+ content="""
+This would indeed be very helpful when remotely mounting a photo/video collection over samba.
+"""]]
diff --git a/doc/todo/union_mounting/comment_2_240b1736f6bd4fbf87c372d3a46e661b._comment b/doc/todo/union_mounting/comment_2_240b1736f6bd4fbf87c372d3a46e661b._comment
new file mode 100644
index 0000000000..08901ee178
--- /dev/null
+++ b/doc/todo/union_mounting/comment_2_240b1736f6bd4fbf87c372d3a46e661b._comment
@@ -0,0 +1,9 @@
+[[!comment format=mdwn
+ username="http://edheil.wordpress.com/"
+ ip="173.162.44.162"
+ subject="comment 2"
+ date="2013-03-01T04:50:28Z"
+ content="""
++1 this would be sweet as hell
+
+"""]]
diff --git a/doc/todo/union_mounting/comment_3_cf0a0d4fbd929f24f7056115b2acb7de._comment b/doc/todo/union_mounting/comment_3_cf0a0d4fbd929f24f7056115b2acb7de._comment
new file mode 100644
index 0000000000..ae925adf05
--- /dev/null
+++ b/doc/todo/union_mounting/comment_3_cf0a0d4fbd929f24f7056115b2acb7de._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="http://ypid.wordpress.com/"
+ nickname="ypid"
+ subject="Please add this ;)"
+ date="2014-03-13T19:10:17Z"
+ content="""
++1 This would be so great. For me the only thing which is missing in this awesome project.
+"""]]
diff --git a/doc/todo/unlock_--json.mdwn b/doc/todo/unlock_--json.mdwn
new file mode 100644
index 0000000000..694effff81
--- /dev/null
+++ b/doc/todo/unlock_--json.mdwn
@@ -0,0 +1,9 @@
+Unlock is quite heavy in terms of I/O and thus waiting time, so we (in datalad) probably should provide also a progress bar indicator.
+
+1. how could I obtain a list of files which would be unlocked given a set of paths?
+
+2. could there be a --json mode of output (consistent with add --json etc)?
+
+[[!meta author=yoh]]
+
+> added --json option [[done]] --[[Joey]]
diff --git a/doc/todo/unlock_--json/comment_1_4cd41dca533de8147027eeb9a8ad89db._comment b/doc/todo/unlock_--json/comment_1_4cd41dca533de8147027eeb9a8ad89db._comment
new file mode 100644
index 0000000000..5a7136a23d
--- /dev/null
+++ b/doc/todo/unlock_--json/comment_1_4cd41dca533de8147027eeb9a8ad89db._comment
@@ -0,0 +1,16 @@
+[[!comment format=mdwn
+ username="joey"
+ subject="""comment 1"""
+ date="2016-10-31T18:04:04Z"
+ content="""
+`git annex unlock` only operates on files that are symlinks and link to
+.git/annex/objects.
+I don't think that `find` provides a way to check the link target
+of a symlink, so you'd have to write a program to do it. I suppose
+that `git annex find --locked` could be made to do that.
+
+`git annex unlock` often has to make a copy of the content of a file,
+which would be the majority of its IO. Only if you're using
+`annex.thin` with v6 might it not need to copy the file, and then a batch
+mode could speed it up. Are you?
+"""]]
diff --git a/doc/todo/unlock_--json/comment_2_f00fda45d300ce14a5ea6fd32a20defe._comment b/doc/todo/unlock_--json/comment_2_f00fda45d300ce14a5ea6fd32a20defe._comment
new file mode 100644
index 0000000000..294bbcfea9
--- /dev/null
+++ b/doc/todo/unlock_--json/comment_2_f00fda45d300ce14a5ea6fd32a20defe._comment
@@ -0,0 +1,12 @@
+[[!comment format=mdwn
+ username="joey"
+ subject="""comment 2"""
+ date="2017-10-30T18:40:53Z"
+ content="""
+Why was I talking about batch mode here before? --json of course makes
+sense to support for unlock (and lock). Adding that now.
+
+And, to find the files that `git annex unlock` would operate on, you
+can simply use `git annex find`, because that does not list already
+unlocked files, and only files with their content present can be unlocked.
+""]]
diff --git a/doc/todo/unlock_--read-only.mdwn b/doc/todo/unlock_--read-only.mdwn
new file mode 100644
index 0000000000..0c33cb173a
--- /dev/null
+++ b/doc/todo/unlock_--read-only.mdwn
@@ -0,0 +1,3 @@
+"annex unlock" in thin mode of v6 hard-links key into the file location and makes it RW.  This is obviously for the case where modifications to the file need to be done and danger is understood. In my case, I need unlock to avoid having symlinks in the files since some software doesn't digest them well (might copy without dereferencing or dereference and look for neighboring files in the directory), and I want to use unlock to pretty much provide "symlink-free" view of the tree BUT at least with some protection, which could be given if files are unlocked read-only, so no inplace modifications could happen without explicit change of the permissions.
+
+[[!meta author=yoh]]
diff --git a/doc/todo/unlock_--read-only/comment_1_19da9b33836df34b7d747039e6863586._comment b/doc/todo/unlock_--read-only/comment_1_19da9b33836df34b7d747039e6863586._comment
new file mode 100644
index 0000000000..6047827eac
--- /dev/null
+++ b/doc/todo/unlock_--read-only/comment_1_19da9b33836df34b7d747039e6863586._comment
@@ -0,0 +1,13 @@
+[[!comment format=mdwn
+ username="joey"
+ subject="""comment 1"""
+ date="2016-10-17T18:09:44Z"
+ content="""
+The protection offered by a read-only mode is pretty minimal; any program
+that writes a file atomically using rename will bypass it. So, as programs
+are implemented better, they'll bypass this "protection" more -- not much of
+a protection!
+
+Also, it doesn't make much sense to call this operation "unlock" if it's
+intended to not let programs modify the files.
+"""]]
diff --git a/doc/todo/unlock_--read-only/comment_2_1658f775ef9340e65711512049085dd7._comment b/doc/todo/unlock_--read-only/comment_2_1658f775ef9340e65711512049085dd7._comment
new file mode 100644
index 0000000000..7047c19075
--- /dev/null
+++ b/doc/todo/unlock_--read-only/comment_2_1658f775ef9340e65711512049085dd7._comment
@@ -0,0 +1,20 @@
+[[!comment format=mdwn
+ username="joey"
+ subject="""comment 2"""
+ date="2016-10-17T21:07:31Z"
+ content="""
+Actually, yoh is right: read-only would be sufficient protection here.
+Because, with annex.thin, the worktree file is a hard link to the annex
+object, and the annex object lives in a mode 400 directory. So, even if the
+file is deleted and a new version renamed into place, the annex object will
+still have captured the old version.
+
+Still don't like the self-contradition of "unlock read-only".
+
+Of course, you can do this yourself:
+
+	git annex unlock file
+	chmod 400 file
+
+So I wonder if there's any need for a git-annex command to do this.
+"""]]
diff --git a/doc/todo/unlock_--read-only/comment_3_305c1b867c38edf74cdff1638e5b1cfa._comment b/doc/todo/unlock_--read-only/comment_3_305c1b867c38edf74cdff1638e5b1cfa._comment
new file mode 100644
index 0000000000..f1628d4c89
--- /dev/null
+++ b/doc/todo/unlock_--read-only/comment_3_305c1b867c38edf74cdff1638e5b1cfa._comment
@@ -0,0 +1,11 @@
+[[!comment format=mdwn
+ username="https://me.yahoo.com/a/EbvxpTI_xP9Aod7Mg4cwGhgjrCrdM5s-#7c0f4"
+ avatar="http://cdn.libravatar.org/avatar/8122123b4c2bc77187e32d7e025f7d445d7a08de1ba532237876a31159ac01da"
+ subject="why option vs manual chmod"
+ date="2016-10-17T23:11:10Z"
+ content="""
+as far as I see it
+
+- much easier to specify the option instead of figuring out which files were unlocked and need to be chmodded (some might be under git, etc)
+- probably would be notably (avoiding double file-tree traversal) faster in case of unlocking large number of files
+"""]]
diff --git a/doc/todo/upload_large_chunks_without_buffering_in_memory.mdwn b/doc/todo/upload_large_chunks_without_buffering_in_memory.mdwn
new file mode 100644
index 0000000000..2c5c96c1f6
--- /dev/null
+++ b/doc/todo/upload_large_chunks_without_buffering_in_memory.mdwn
@@ -0,0 +1,26 @@
+Currently, when chunk=100MiB is used with a special remote, git-annex will
+use 100 mb ram when uploading a large file to it. The current chunk is
+buffered in ram.
+
+See comment in Remote.Helper.Chunked:
+
+	- More optimal versions of this can be written, that rely
+	- on L.toChunks to split the lazy bytestring into chunks (typically
+	- smaller than the ChunkSize), and eg, write those chunks to a Handle.
+	- But this is the best that can be done with the storer interface that
+	- writes a whole L.ByteString at a time.
+
+The buffering is probably fine for smaller chunks, in the < 10 mb or
+whatever range. Reading such a chunk from gpg and converting it into a http
+request will generally need to buffer it all in memory anyway. But, for
+eg external special remotes, the content is going to be written to a file
+anyway, so there's no point in buffering the content in memory first.
+
+So, need something like:
+
+	prefersFileContent :: Storer -> Bool
+
+And then when storeChunks is given such a Storer, it can then avoid buffering
+and write chunk data to a temp file to pass it to the Storer.
+
+--[[Joey]]
diff --git a/doc/todo/use_git-mktree_rather_than_index_file.mdwn b/doc/todo/use_git-mktree_rather_than_index_file.mdwn
new file mode 100644
index 0000000000..f02fafb08a
--- /dev/null
+++ b/doc/todo/use_git-mktree_rather_than_index_file.mdwn
@@ -0,0 +1,29 @@
+When git-annex is updating the git-annex branch, it currently
+uses a separate index file. This adds overhead and complexity to the code.
+Especially when there are many files, the index file gets large and writing
+it gets slow.
+
+It might be an improvement to use `git mktree --batch` to inject a
+tree object into git, without using the index file. `git hash-object`
+is already used to add the files to git. All that would be needed is to
+generate an updated tree containing the new file(s), and then update each
+parent tree up to the root tree. This new tree can then be committed using
+`git commit-tree`
+
+The only thing I can see that might make this slow at all is reading the old
+tree contents, in order to update it. This would need a `git ls-tree` for
+each tree; it does not have a batch mode, so 4 processes would need to be
+spawned when generating a tree that changes 1 file. For any repo that's not
+very small, that's probably still faster than rewriting the index file.
+
+Notes:
+
+* The union merge code currently uses the index. No particular reason
+  it needs to; that's just how the code is written, and it might be a large
+  rewrite to change it.
+* A new git-annex branch can be pushed into the repository at any time.
+  The current code uses the index to detect when this happens, and
+  union merges the new branch head into the index. Would need something
+  like a `GIT_ANNEX_HEAD` ref to do the same if the index is removed.
+
+Thanks to sm for indirectly suggesting this. --[[Joey]] 
diff --git a/doc/todo/using_file_metadata_for_preferred___40__wanted__41___content.mdwn b/doc/todo/using_file_metadata_for_preferred___40__wanted__41___content.mdwn
new file mode 100644
index 0000000000..d9226d21f8
--- /dev/null
+++ b/doc/todo/using_file_metadata_for_preferred___40__wanted__41___content.mdwn
@@ -0,0 +1,12 @@
+Having the option of choosing for every file if we want it in our repository or not would be a great feature. It is currently possible using the wanted expression, but it is not very flexible, or it becomes unmaintainable.
+
+I tried with two repositories a and b, with the following wanted expressions :
+
+* for a: `not metadata=unwanted=`
+* for b: `not metadata=unwanted=`
+
+I think those expressions should be included in standard wanted expressions.
+
+Also, to improbe the feature, it should be possible to set (or remove) metadata in directories, and those should automatically affect their content.
+
+And we could imagine a `git annex unwant` command that would add the unwanted metadata to a file, copy it to other repositories, and attempt to drop it.
diff --git a/doc/todo/utilising_the_mklink_command_on_windows_to_utilise_symlinks_and_therefore_indirect_mode_on_windows.mdwn b/doc/todo/utilising_the_mklink_command_on_windows_to_utilise_symlinks_and_therefore_indirect_mode_on_windows.mdwn
new file mode 100644
index 0000000000..263035fa0a
--- /dev/null
+++ b/doc/todo/utilising_the_mklink_command_on_windows_to_utilise_symlinks_and_therefore_indirect_mode_on_windows.mdwn
@@ -0,0 +1,7 @@
+It seems like it is possible to achieve this now on later versions of Windows (7 and above).
+
+The mklink tool creates a symlink that works on windows.
+
+There would be some work required so that a linux symlink and a windows symlink are considered the same, a user has recommend 'git update-index --assume-unchanged' for this.
+
+There is a good write up of someone using this approach on vanilla git here: http://stackoverflow.com/questions/5917249/git-symlinks-in-windows
diff --git a/doc/todo/utilising_the_mklink_command_on_windows_to_utilise_symlinks_and_therefore_indirect_mode_on_windows/comment_1_f974b0fc908277fcc35ee6c8073b65c8._comment b/doc/todo/utilising_the_mklink_command_on_windows_to_utilise_symlinks_and_therefore_indirect_mode_on_windows/comment_1_f974b0fc908277fcc35ee6c8073b65c8._comment
new file mode 100644
index 0000000000..7e44497e8f
--- /dev/null
+++ b/doc/todo/utilising_the_mklink_command_on_windows_to_utilise_symlinks_and_therefore_indirect_mode_on_windows/comment_1_f974b0fc908277fcc35ee6c8073b65c8._comment
@@ -0,0 +1,13 @@
+[[!comment format=mdwn
+ username="Karl"
+ subject="Symbolic links are not supported by Windows applications"
+ date="2015-02-06T22:27:27Z"
+ content="""
+Hi!
+
+I did research the practical usage of (real) NTFS-links (junktions + links) when I was programming http://tagstore.org since it would have been a clean solution for that purpose as well.
+
+However, I have to say that NTFS-links only work in theory. First of all, you have to be Administrator to create links by mklink. And secondly, most end-user applications can't cope with links.
+
+The latter one is the real fun-killing issue: many applications (even from Microsoft) are renaming a freshly opened file to a temporary file name. When the user is appending data, the temporary file gets updated. Only when the user is (manually) saving, a *new* file with the original file name is created. This results in *replacing* the original file with the new copy. Unfortunately, links are not handled properly. This way, many applications end up replacing the original linked file with an ordinary file when saving.
+"""]]
diff --git a/doc/todo/utilising_the_mklink_command_on_windows_to_utilise_symlinks_and_therefore_indirect_mode_on_windows/comment_2_575d96ffa2a8f2cb1ba3d0e5c4ba4e6f._comment b/doc/todo/utilising_the_mklink_command_on_windows_to_utilise_symlinks_and_therefore_indirect_mode_on_windows/comment_2_575d96ffa2a8f2cb1ba3d0e5c4ba4e6f._comment
new file mode 100644
index 0000000000..ad37ac8b05
--- /dev/null
+++ b/doc/todo/utilising_the_mklink_command_on_windows_to_utilise_symlinks_and_therefore_indirect_mode_on_windows/comment_2_575d96ffa2a8f2cb1ba3d0e5c4ba4e6f._comment
@@ -0,0 +1,11 @@
+[[!comment format=mdwn
+ username="jason.dixon.email@aa0e536a2ec2877d6f666108dbbc6e39bbe87ac0"
+ nickname="jason.dixon.email"
+ avatar="http://cdn.libravatar.org/avatar/fbe9050fc83bbd536d307d87ea14d4bc"
+ subject="Does this not work counter to locking / unlocking files?"
+ date="2017-03-03T13:40:06Z"
+ content="""
+Perhaps I understand this wrong. But if you're in a position to be viewing symlinks, then the file is in effect, locked right? So in the situation that a windows user is trying to edit a file that is currently symlinked, they shouldn't be allowed to save at all. They should have unlocked the file first, thus replacing the link with the actual file. A simple fix would be to have the symlink read-only just as the file tucked away in .git/annex/objects is.
+
+As I said, I'm probably looking at this in entirely the wrong way.
+"""]]
diff --git a/doc/todo/versioning_in_export_remotes.mdwn b/doc/todo/versioning_in_export_remotes.mdwn
new file mode 100644
index 0000000000..9714ba77e9
--- /dev/null
+++ b/doc/todo/versioning_in_export_remotes.mdwn
@@ -0,0 +1,83 @@
+Some remotes like S3 support versioning of data stored in them.
+When git-annex updates an export, it deletes the old
+content from eg the S3 bucket, but with versioning enabled, S3 retains the
+content and it can be accessed using a version ID (that S3 returns when
+storing the content). So it should be possible for git-annex to allow
+downloading old versions of files from such a remote.
+
+
+
+Basically, store the S3 version ID in git-annex branch and support
+downloading using it. 
+
+But this has the problem that dropping makes git-annex think it's not in S3
+any more, while what we want for export is for it to be removed from the
+current bucket, but still tracked as present in S3.
+
+The drop from S3 could fail, or "succeed" in a way that prevents the location
+tracking being updated to say it lacks the content. Failing is how bup deals
+with it. It seems confusing to have a drop appear to succeed but not really drop,
+especially since dropping again would seem to do something a second time.
+
+This does mean that git-annex drop/sync --content/assistant might try to do a
+lot of drops from the remote, and generate a lot of noise when they fail.
+Which is kind of ok for drop, since the user should be told that they can't
+delete the data. Could add a way to say "this remote does not support drop",
+and make at sync --content/assistant use that.
+
+Note that git-annex export does not rely on location tracking to determine
+which files still need to be sent to an export. It uses the export database
+to keep track of that. This is important, because the location tracking
+won't be updated, as discussed above.
+
+The haskell aws library does not seem to support enabling versioning when
+creating a bucket, so it would need to be done from the web console.
+
+## other considerations
+
+If the user enables versioning in git-annex but forgets to enable it
+in the bucket (or later suspends versioning in the bucket), it's no
+big problem; old files will not be retained and git-annex will notice
+this in the usual way (drop locking, fsck). So, it seems that initremote
+does not need to check if the versioning=yes setting matches the bucket
+configuration. For same reasons, it's ok to enable versioning for an
+existing remote.
+
+S3 does allow DELETE of a version of an object from a bucket. So it would
+be possible to support `git annex drop` of old versions of a file from an
+export remote. Dropping the current version though, would make the export
+database inconsistent; it would not know that a file in the exported tree
+was no longer present. I don't think that inconsitency can easily be
+resolved -- bear in ming that multiple repositories can have an export db,
+so it would need to look at location tracking for all objects in the export
+to find ones that some other repository dropped. And dropping of only
+keys that are not used in the current export doesn't help because another
+repository may have changed the exported tree and be relying on the dropped
+key being present in the export. Unless... Could export conflict resultion
+somehow detect that?
+
+## final plan
+
+Add an "appendOnly" field to Remote, indicating it retains all content stored
+in it. done
+
+Make `git annex export` check appendOnly when removing a file from an
+export, and not update the location log, since the remote still contains
+the content. done
+
+Make git-annex sync and the assistant skip trying to drop from appendOnly
+remotes since it's just going to fail. done
+
+Make exporttree=yes remotes that are appendOnly not be untrusted, and not force
+verification of content, since the usual concerns about losing data when an
+export is updated by someone else don't apply. done
+
+Let S3 remotes be configured with versioning=yes which enables appendOnly.
+done
+
+Make S3 store version IDs for exported files in the per-remote log when so
+configured. done
+
+Use version IDs when retrieving keys and for checkpresent. done
+
+> all [[done]]! (but see [[support_public_versioned_S3_access]]) --[[Joey]] 
diff --git a/doc/todo/webapp__58___show_times_of_events.mdwn b/doc/todo/webapp__58___show_times_of_events.mdwn
new file mode 100644
index 0000000000..b58e52e9d4
--- /dev/null
+++ b/doc/todo/webapp__58___show_times_of_events.mdwn
@@ -0,0 +1,5 @@
+In the webapp 'dashboard' page there is a column of 'message boxes' on the right hand side that appear as events occur.
+
+I would find it helpful to be able to hover over a message box and have a timestamp appear so that I know whether the event is a recent one that I might need to go look in the log for more detail about, or that I can click on the 'X' to dismiss the message.
+
+A further improvement that occurred to me is adding a link within the message box (link text something like "view log entry") that takes you to the corresponding part of the 'view log' page.
diff --git a/doc/todo/webapp_export_remote_configuration_interface.mdwn b/doc/todo/webapp_export_remote_configuration_interface.mdwn
new file mode 100644
index 0000000000..d6870a68e2
--- /dev/null
+++ b/doc/todo/webapp_export_remote_configuration_interface.mdwn
@@ -0,0 +1,11 @@
+Support configuring export remotes in the webapp
+
+Remember the little-used preferreddir= preferred content
+setting and the "public" repository group. The webapp uses
+those for IA, which could be replaced with setting up an export
+tracking branch.
+
+The UI for S3, WebDAV, directory special remote setup could also have a way
+to make it an export, and configure the directory to export. This would
+complicate the UI, so needs thought. --[[Joey]]
+
diff --git a/doc/todo/webapp_nudge_when_less_than_numcopies_clones.mdwn b/doc/todo/webapp_nudge_when_less_than_numcopies_clones.mdwn
new file mode 100644
index 0000000000..ee38d0fffb
--- /dev/null
+++ b/doc/todo/webapp_nudge_when_less_than_numcopies_clones.mdwn
@@ -0,0 +1,7 @@
+Currently, nothing stops a user from setting up ~/annex, adding some special remote, and never once ending up with a clone of their repository, so there is really no backup of the repository as a whole, despite the special remotes.
+
+Potentially adding to the confusion, they might have remotes in repository groups "full backup" or "backup", and so think everything is backed up.
+
+Webapp could count the number of known remote uuids that are not special remotes, and require there to be at least numcopies of them (excluding the current repo I suppose), and pop up a nudge with a button that presents the various available ways to make a non-special remote.
+
+Working out if a remote uuid is a special remote is probably the hard bit. A special remote will be listed in uuid.log, with a type other than gcrypt or git. Any other uuid, that is not dead, can count as 1 clone. This does not handle git remotes that are not using git-annex (eg github), so it could also look through the git remote list and count any that don't have an annex-uuid.
diff --git a/doc/todo/webapp_ssh_setup_should_work_with_locked_down_git-annex-shell_account.mdwn b/doc/todo/webapp_ssh_setup_should_work_with_locked_down_git-annex-shell_account.mdwn
new file mode 100644
index 0000000000..d91bd4be5a
--- /dev/null
+++ b/doc/todo/webapp_ssh_setup_should_work_with_locked_down_git-annex-shell_account.mdwn
@@ -0,0 +1,7 @@
+When the webapp is used to set up a ssh repository, it should detect if
+there is already a repository, and sshing in to the server forces running
+git-annex-shell (perhaps by it being set to the user's login shell). 
+
+In this case it should just use the already set up repository. Note that
+it would not be possible to add a new repository on the server using only
+git-annex-shell. --[[Joey]]
diff --git a/doc/todo/windows_support.mdwn b/doc/todo/windows_support.mdwn
new file mode 100644
index 0000000000..23fdaa5ca3
--- /dev/null
+++ b/doc/todo/windows_support.mdwn
@@ -0,0 +1,150 @@
+The git-annex Windows port is beta, but rapidly becoming polished and
+usable!
+
+
+## status
+
+* There can be problems when the git-annex repository is in a deep
+  or long path. Ie, `C:\loooooooooooooooooongdir\`.
+  [Details here](http://git-annex.branchable.com/bugs/__34__git-annex:_direct:_1_failed__34___on_Windows)
+
+  Workaround: Put your git-annex repo in `C:\annex` or some similar short
+  path if possible.
+
+  Workaround: Enable long paths in the registry. See 
+  
+
+  ghc 8.61 fixes this!
+  
+
+* Local pairing seems to fail, after acking on Linux box, it stalls.
+  (Also, of course, the Windows box is unlikely to have a ssh server,
+  so only pairing with a !Windows box will work.)
+
+* gcrypt is not ported to windows (and as a shell script, may need
+  to be rewritten)
+
+* Deleting a git repository from inside the webapp fails "RemoveDirectory
+  permision denied ... file is being used by another process"
+
+* Tor remotes are not supported yet. Should not be very hard to get it working.
+
+## potential encoding problems
+
+[[bugs/Unicode_file_names_ignored_on_Windows]] is fixed, but some potential
+problems remain, since the FileSystemEncoding that git-annex relies on
+seems unreliable/broken on Windows.
+
+* When git-annex displays a filename that it's acting on, there
+  can be mojibake on Windows. For example, "háčky.txt" displays
+  the accented characters as instead the pairs of bytes making
+  up the utf-8. Tried doing various things to the stdout handle
+  to avoid this, but only ended up with encoding crashes, or worse
+  mojibake than this.
+
+* `md5FilePath` still uses the filesystem encoding, and so may produce the
+  wrong value on Windows. This would impact keys that contain problem characters
+  (probably coming from the filename extension), and might cause
+  interoperability problems when git-annex generates the hash directories of a
+  remote, for example a rsync remote.
+
+* `encodeW8` is used in Git.UnionMerge, and while I fixed the other calls to
+  encodeW8, which all involved ByteStrings reading from git and so can just
+  treat it as utf-8 on Windows (via `decodeBS`), in the union merge case,
+  the ByteString has no defined encoding. It may have been written on Unix
+  and contain keys with invalid unicode in them. On windows, the union
+  merge code should probably check if it's valid utf-8, and if not,
+  abort the merge.
+
+* If interoperating with a git-annex repository from a unix system, it's
+  possible for a key to contain some invalid utf-8, which means its filename
+  cannot even be represented on Windows, so who knows what will happen in that
+  case -- probably it will fail in some way when adding the object file
+  to the Windows repo. 
+
+* If data from the git repo does not have a unicode encoding, it will be
+  mangled in various places on Windows, which can lead to undefined behavior.
+
+## minor problems
+
+* webapp lets user choose to encrypt repo, and generate gpg key,
+  before checking that gcrypt is not installed
+* Ssh connection caching does not work on Windows, so `git annex get`
+  has to connect twice to the remote system over ssh per file, which
+  is much slower than on systems supporting connection caching.
+* glacier-cli is not easily available (probably)
+
+## stuff needing testing
+
+* test that adding a repo on a removable drive works; that git is synced to
+  it and files can be transferred to it and back
+* Does stopping in progress transfers work in the webapp?
+
+## do we need this port anymore?
+
+See 
+
+If windows has transparent support for running linux executables, and those
+executables can access files in "." which are on the windows system, then
+you could just use this to run linux git-annex on windows. No port needed.
+
+That would be great!
+
+Seems like this would need Windows 10.
+
+> The latest builds of Windows 10 (build 15063) can run git-annex in the
+> Windows Subsystem for Linux. After following the instructions at
+> , run:
+> `sudo apt-get install git-annex`
+> 
+> git-annex in WSL passes its full test suite, and it avoids all
+> the problems discussed in sections above.
+> 
+> git-annex can access Windows files in eg `/mnt/c`, so a git-annex
+> repository can be stored there. However, if the git-annex repository uses
+> indirect mode, the symlinks used by git-annex won't be usable by Windows
+> programs. Use either direct mode, or v6 mode to avoid the symlink
+> problem.
+> 
+> Also, see this important caveat:
+> 
+> 
+> March 2017: WSL is currently rather annoying to enable. *If* it became easy enough
+> to enable, note that "bash -c git-annex" works from a windows command
+> prompt, and would probably work in a .bat file as well, so git-annex from
+> the WSL could be transparently used on the windows side.
+> 
+> The webapp does not currently work. It doesn't know how to open a web
+> browser from the linux side. There are also what look like some emulation
+> problems around the daemonization code. `git annex assistant
+> --foreground` does run, but while it notices when new files are added, it
+> does not notice when existing files get modified. Probably an inotify
+> emulation bug. --[[Joey]]
+> 
+> > Update May 2018: With the latest MS Edge VirtualBox image, 
+> > enabling WSL was much easier than before, just a matter of checking a
+> > single box and a reboot. No need to enable developer mode first.
+> > (Don't know if this will apply to commercial Windows 10 or only this
+> > VM image.)
+> > 
+> > Then bash was available in the menu, but it said no distro installed,
+> > and gave an url to a "Store" where Ubuntu, Debian, etc 
+> > could be installed.
+> > 
+> > git-annex test had a failure this time, something to do with disk
+> > IO error and sqlite.
+> > 
+> > The assistant can run as a daemon now. It still doesn't notice some changes. 
+> > Eg with "rm foo; echo new > foo", it got an inotify event for the removal,
+> > but missed the new creation.
+> >
+> > Running `git annex assistant` when the assistant is running
+> > complains of a stale pid file, with the wrong pid number, but it
+> > doesn't start a second one.
+> > 
+> > webapp still can't open a web browser, but there's a way to do
+> > it from within WSL that it should be possible for it to use:
+> > `powershell.exe Start http://google.com`
+> > ("powershell.exe" is in PATH)
+> > 
+> > --[[Joey]]
diff --git a/doc/todo/windows_support/comment_10_394127e34e07ab3dc0e7b94ee6898866._comment b/doc/todo/windows_support/comment_10_394127e34e07ab3dc0e7b94ee6898866._comment
new file mode 100644
index 0000000000..fb061962e5
--- /dev/null
+++ b/doc/todo/windows_support/comment_10_394127e34e07ab3dc0e7b94ee6898866._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="http://joeyh.name/"
+ ip="4.152.108.145"
+ subject="comment 10"
+ date="2013-08-04T18:23:00Z"
+ content="""
+Encryption is now working on Windows.
+"""]]
diff --git a/doc/todo/windows_support/comment_11_c91eb7da8ee05064a5bc4a6e2203314b._comment b/doc/todo/windows_support/comment_11_c91eb7da8ee05064a5bc4a6e2203314b._comment
new file mode 100644
index 0000000000..f4a2eaa7f3
--- /dev/null
+++ b/doc/todo/windows_support/comment_11_c91eb7da8ee05064a5bc4a6e2203314b._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="http://joeyh.name/"
+ ip="108.236.230.124"
+ subject="comment 11"
+ date="2014-06-16T23:48:03Z"
+ content="""
+service stuff moved to [[todo/windows_git-annex_service]]
+"""]]
diff --git a/doc/todo/windows_support/comment_12_0775dc22516cfa8c5e7deea954084947._comment b/doc/todo/windows_support/comment_12_0775dc22516cfa8c5e7deea954084947._comment
new file mode 100644
index 0000000000..0c3aa46d4f
--- /dev/null
+++ b/doc/todo/windows_support/comment_12_0775dc22516cfa8c5e7deea954084947._comment
@@ -0,0 +1,18 @@
+[[!comment format=mdwn
+ username="https://www.google.com/accounts/o8/id?id=AItOawkQOUUx4LVAk6EnstSLvdv7gZc0NsRlHXw"
+ nickname="Dave"
+ subject="windows port volunteer tester"
+ date="2014-09-08T15:39:03Z"
+ content="""
+Hello.
+
+I volunteer to test the Windows port.  I can only do so on the work computer.  :)
+
+I know how to produce a Minimal Reproducible Example.  Are there some areas that need attention?
+
+I've enabled the 'email replies to me' ikiwiki feature.  (Nice plugin.)
+
+Cheers,
+
+--Dave
+"""]]
diff --git a/doc/todo/windows_support/comment_13_4dfb65242280c2e28796f1ab8f022dea._comment b/doc/todo/windows_support/comment_13_4dfb65242280c2e28796f1ab8f022dea._comment
new file mode 100644
index 0000000000..65b24f25f4
--- /dev/null
+++ b/doc/todo/windows_support/comment_13_4dfb65242280c2e28796f1ab8f022dea._comment
@@ -0,0 +1,13 @@
+[[!comment format=mdwn
+ username="tomekwi"
+ subject="Linux-Windows repo"
+ date="2015-08-10T18:22:22Z"
+ content="""
+Hi, what’s the recommended way to manage a repo accessed from both a Linux and a Windows box? Has anyone had success with that?
+
+I’m dual-booting Fedora 22 and Windows 8.1. I do most of my work on Linux, but I need Windows programs to edit some of my files. I’ve had a go at two approaches:
+
+1) Have a “central” repo on an NTFS external drive which also holds the contents of files. This fails because Windows fails to check out the contents of most files. It isn’t random – specific files at specific points of the commit history fail to check out. Perhaps it has something to do with filenames? Characters in filenames?
+
+2) Keep a “direct” clone of my repo on a Windows NTFS partition. I do my work on Windows, and after rebooting to Linux I mount the partition and commit my changes directly in the repo. Here Windows seems to outsmart me. git-annex sometimes takes many files without content as modified – and commits them as empty.
+"""]]
diff --git a/doc/todo/windows_support/comment_14_a8519b722cb94f9363d4a8a8bd40e942._comment b/doc/todo/windows_support/comment_14_a8519b722cb94f9363d4a8a8bd40e942._comment
new file mode 100644
index 0000000000..587ea02277
--- /dev/null
+++ b/doc/todo/windows_support/comment_14_a8519b722cb94f9363d4a8a8bd40e942._comment
@@ -0,0 +1,9 @@
+[[!comment format=mdwn
+ username="tomekwi"
+ subject="Linux-Windows repo"
+ date="2016-01-26T20:46:58Z"
+ content="""
+Ping!
+
+I’m trying to organize my Linux + Windows workflow again. Has anyone had success managing an annex repo from Linux and Windows?
+"""]]
diff --git a/doc/todo/windows_support/comment_15_61d1281072de2bb418cf96fd8b563d45._comment b/doc/todo/windows_support/comment_15_61d1281072de2bb418cf96fd8b563d45._comment
new file mode 100644
index 0000000000..9eae2b7b12
--- /dev/null
+++ b/doc/todo/windows_support/comment_15_61d1281072de2bb418cf96fd8b563d45._comment
@@ -0,0 +1,10 @@
+[[!comment format=mdwn
+ username="ztl8702"
+ subject="Bash on Windows"
+ date="2016-09-30T19:14:34Z"
+ content="""
+Currently there is a bug that stops all Haskell programs from running on Bash on Windows:
+https://wpdev.uservoice.com/forums/266908-command-prompt-console-bash-on-ubuntu-on-windo/suggestions/13572504-add-support-for-timer-create
+
+https://github.com/Microsoft/BashOnWindows/issues/307
+"""]]
diff --git a/doc/todo/windows_support/comment_16_6bbcf900874e950b08cdf0410f7a75d2._comment b/doc/todo/windows_support/comment_16_6bbcf900874e950b08cdf0410f7a75d2._comment
new file mode 100644
index 0000000000..3e534d6d44
--- /dev/null
+++ b/doc/todo/windows_support/comment_16_6bbcf900874e950b08cdf0410f7a75d2._comment
@@ -0,0 +1,11 @@
+[[!comment format=mdwn
+ username="https://openid.stackexchange.com/user/26f3c692-0460-4cbc-8712-e9bfb5889fb2"
+ nickname="Tobias Kienzler"
+ avatar="http://cdn.libravatar.org/avatar/f66be680ced3f9dc4786e308ceade73902e185d407be873e98266fc745c8c075"
+ subject=""WSL adds inotify & filesystem change notification support""
+ date="2017-08-07T06:22:07Z"
+ content="""
+https://blogs.msdn.microsoft.com/commandline/2016/10/07/wsl-adds-inotify-filesystem-change-notification-support/
+
+So it should work now, right?
+"""]]
diff --git a/doc/todo/windows_support/comment_1_3cc26ad8101a22e95a8c60cf0c4dedcc._comment b/doc/todo/windows_support/comment_1_3cc26ad8101a22e95a8c60cf0c4dedcc._comment
new file mode 100644
index 0000000000..fd5b6f5cd3
--- /dev/null
+++ b/doc/todo/windows_support/comment_1_3cc26ad8101a22e95a8c60cf0c4dedcc._comment
@@ -0,0 +1,10 @@
+[[!comment format=mdwn
+ username="https://www.google.com/accounts/o8/id?id=AItOawkRITTYYsN0TFKN7G5sZ6BWGZOTQ88Pz4s"
+ nickname="Zoltán"
+ subject="cygwin"
+ date="2012-05-15T00:14:08Z"
+ content="""
+What about [Cygwin](http://cygwin.com/)? It emulates POSIX fairly well under Windows (including signals, forking, fs (also things like /dev/null, /proc), unix file permissions), has all standard gnu utilities. It also emulates symlinks, but they are unfortunately incompatible with NTFS symlinks introduced in Vista [due to some stupid restrictions on Windows](http://cygwin.com/ml/cygwin/2009-10/msg00756.html).
+
+If git-annex could be modified to not require symlinks to work, the it would be a pretty neat solution (and you get a real shell, not some command.com on drugs (aka cmd.exe))
+"""]]
diff --git a/doc/todo/windows_support/comment_2_8acae818ce468967499050bbe3c532ea._comment b/doc/todo/windows_support/comment_2_8acae818ce468967499050bbe3c532ea._comment
new file mode 100644
index 0000000000..e37a555756
--- /dev/null
+++ b/doc/todo/windows_support/comment_2_8acae818ce468967499050bbe3c532ea._comment
@@ -0,0 +1,12 @@
+[[!comment format=mdwn
+ username="https://www.google.com/accounts/o8/id?id=AItOawk5cj-itfFHq_yhJHdzk3QOPp-PNW_MjPU"
+ nickname="Michael"
+ subject="+1 Cygwin"
+ date="2012-05-23T19:30:21Z"
+ content="""
+Windows support is a must. In my experience, binary file means proprietary editor, which means Windows.
+
+Unfortunately, there's not much overlap between people who use graphical editors in Windows all day vs. people who are willing to tolerate Cygwin's setup.exe, compile a Haskell program, learn git and git-annex's 90-odd subcommands, and use a mintty terminal to manage their repository, especially now that there's a sexy GitHub app for Windows.
+
+That aside, I think Windows-based content producers are still *the* audience for git-annex. First Windows support, then a GUI, then the world.
+"""]]
diff --git a/doc/todo/windows_support/comment_3_bd0a12f4c9b884ab8a06082842381a01._comment b/doc/todo/windows_support/comment_3_bd0a12f4c9b884ab8a06082842381a01._comment
new file mode 100644
index 0000000000..0b48db7502
--- /dev/null
+++ b/doc/todo/windows_support/comment_3_bd0a12f4c9b884ab8a06082842381a01._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="http://xolus.net/openid/max"
+ nickname="B0FH"
+ subject="What about NTFS support ?"
+ date="2012-08-02T17:45:10Z"
+ content="""
+Has git-annex been tested with an NTFS-formatted disk under Linux ? NTFS is supposed to be case-sensitive and to allow symlinks, and these are supposed to work with ntfs3g.
+"""]]
diff --git a/doc/todo/windows_support/comment_4_ad06b98b2ddac866ffee334e41fee6a8._comment b/doc/todo/windows_support/comment_4_ad06b98b2ddac866ffee334e41fee6a8._comment
new file mode 100644
index 0000000000..66f9ca71fd
--- /dev/null
+++ b/doc/todo/windows_support/comment_4_ad06b98b2ddac866ffee334e41fee6a8._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="https://www.google.com/accounts/o8/id?id=AItOawlc1og3PqIGudOMkFNrCCNg66vB7s-jLpc"
+ nickname="Paul"
+ subject="Re: What about NTFS support?"
+ date="2012-08-16T19:30:38Z"
+ content="""
+I successfully use git-annex on an NTFS formatted external USB drive, so yes, it is possible and works well.
+"""]]
diff --git a/doc/todo/windows_support/comment_5_444fc7251f57db241b6e80abae41851c._comment b/doc/todo/windows_support/comment_5_444fc7251f57db241b6e80abae41851c._comment
new file mode 100644
index 0000000000..8f76ee2587
--- /dev/null
+++ b/doc/todo/windows_support/comment_5_444fc7251f57db241b6e80abae41851c._comment
@@ -0,0 +1,10 @@
+[[!comment format=mdwn
+ username="https://me.yahoo.com/a/dASECLNzvckz4VwqUGYsvthiplY.#d2c27"
+ nickname="A. D. Sicks"
+ subject="comment 5"
+ date="2012-09-09T23:48:21Z"
+ content="""
+Haskell has C++ binding, so it should be possible to port it to .net/Mono with a VB GUI for Windows users.  Windows has a primitive form of symlinks called shortcuts.  Perhaps shortcut support in Windows could replace the use of symlinks.  I've used shortcuts since XP to put my home Windows directory on another partition and never had a hitch...
+
+If anyone is interested in working on this, hit me up.  I would like to use this in my XP vbox to have access to files on my host OS...I have a student edition of Visual Studio 2005 to do an open source port...
+"""]]
diff --git a/doc/todo/windows_support/comment_6_34f1f60b570c389bb1e741b990064a7e._comment b/doc/todo/windows_support/comment_6_34f1f60b570c389bb1e741b990064a7e._comment
new file mode 100644
index 0000000000..bf9f86f419
--- /dev/null
+++ b/doc/todo/windows_support/comment_6_34f1f60b570c389bb1e741b990064a7e._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="https://www.google.com/accounts/o8/id?id=AItOawnlwEMhiNYv__mEUABW4scn83yMraC3hqE"
+ nickname="Sean"
+ subject="NTFS symlinks"
+ date="2013-01-11T21:44:21Z"
+ content="""
+It seems that NTFS (from Vista forward) has full POSIX support for symlinks. At least, Wikipedia [seems to think so.](http://en.wikipedia.org/wiki/NTFS_symbolic_link). What about doing like GitHub and using MinGW for compatibility? Cygwin absolutely blows in terms of installation size and compatability with the rest of Windows.
+"""]]
diff --git a/doc/todo/windows_support/comment_7_a5ca56c487257434650420acfa60e39f._comment b/doc/todo/windows_support/comment_7_a5ca56c487257434650420acfa60e39f._comment
new file mode 100644
index 0000000000..0ee09ac5a2
--- /dev/null
+++ b/doc/todo/windows_support/comment_7_a5ca56c487257434650420acfa60e39f._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="https://www.google.com/accounts/o8/id?id=AItOawlpSOjMH7Iaz56v6Pr9KCFSpbvMXvg-y9o"
+ nickname="Dominik"
+ subject="So close :-)"
+ date="2013-06-30T12:46:40Z"
+ content="""
+I was fighting my way forward until I read here that special remote with ssh+rsync and encryption doesn't work. Interestingly I got everything working so far, ssh login is keybased, gpg -k works and the remote setup also correctly cooperated with gpg... but it just didn't sync. Any ideas how complex it is to get this last missing piece moving?
+"""]]
diff --git a/doc/todo/windows_support/comment_8_61214de7d967740d42905f3823ce2f65._comment b/doc/todo/windows_support/comment_8_61214de7d967740d42905f3823ce2f65._comment
new file mode 100644
index 0000000000..fe193f7e07
--- /dev/null
+++ b/doc/todo/windows_support/comment_8_61214de7d967740d42905f3823ce2f65._comment
@@ -0,0 +1,12 @@
+[[!comment format=mdwn
+ username="http://joeyh.name/"
+ ip="4.154.4.193"
+ subject="comment 8"
+ date="2013-06-30T17:58:08Z"
+ content="""
+It should be easy to fix whatever's wrong the the rsync special remote. Just a matter of debugging that.
+
+Adding encryption support on Windows is stuck at a roadblock I don't know the way around. To drive gpg, git-annex uses the `--passphrase-fd` option, and sends the \"passphrase\" (really a big block of binary foo!) over a file descriptor of a pipe that it set up.
+
+Windows, AFAIK, doesn't have file descriptors, or at least there is no equivilant to them that I have access to in the Haskell POSIX compatability layer for Windows. I am reluctant to fall back to using `--passphrase-file` on Windows, since that would be a massive security hole (as would passing the passphrase as a parameter via `--passphrase=`).
+"""]]
diff --git a/doc/todo/windows_support/comment_9_259a0b1a6f4d8d1944173380adc5e7c8._comment b/doc/todo/windows_support/comment_9_259a0b1a6f4d8d1944173380adc5e7c8._comment
new file mode 100644
index 0000000000..9ae337886e
--- /dev/null
+++ b/doc/todo/windows_support/comment_9_259a0b1a6f4d8d1944173380adc5e7c8._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="https://www.google.com/accounts/o8/id?id=AItOawlpSOjMH7Iaz56v6Pr9KCFSpbvMXvg-y9o"
+ nickname="Dominik"
+ subject="comment 9"
+ date="2013-07-31T10:29:51Z"
+ content="""
+The tradeoff for me is a \"local\" security hole (where I can secure my own laptop) vs. a remotely exploitable thing... If it needs to go through a file, so be it -- it would however be good if that file would be overwritten with garbage before being deleted :-)
+"""]]
diff --git a/doc/todo/wishlist__58___--dry-run_option_for_all_commands.mdwn b/doc/todo/wishlist__58___--dry-run_option_for_all_commands.mdwn
new file mode 100644
index 0000000000..fbea1b6faf
--- /dev/null
+++ b/doc/todo/wishlist__58___--dry-run_option_for_all_commands.mdwn
@@ -0,0 +1,3 @@
+Could a --dry-run option be added to the git annex commands? Or, at least, to the most common ones like `git annex add`.
+
+Given that there is no undo command, it would be nice to have the ability to simulate what git annex will do.
diff --git a/doc/todo/wishlist__58___--dry-run_option_for_all_commands/comment_1_03bf493d5a7f957339f9aa388ba85ef8._comment b/doc/todo/wishlist__58___--dry-run_option_for_all_commands/comment_1_03bf493d5a7f957339f9aa388ba85ef8._comment
new file mode 100644
index 0000000000..7725747388
--- /dev/null
+++ b/doc/todo/wishlist__58___--dry-run_option_for_all_commands/comment_1_03bf493d5a7f957339f9aa388ba85ef8._comment
@@ -0,0 +1,20 @@
+[[!comment format=mdwn
+ username="joey"
+ subject="""comment 1"""
+ date="2014-11-03T16:44:45Z"
+ content="""
+This would add a lot of complexity; it's not like I could switch off
+running all external commands, since many external commands are run to
+query state to decide what to do. And then there's large chunks of code
+that actually do stuff and would have to all be guarded to not run.
+
+I don't see the benefit to justify this work. `git annex add` is entirely
+predictable; it's very similar to `git add`. Which itself lacks a dry-run
+option. And like `git add`, you can certianly undo the effects of `git
+annex add`.
+
+Matching options can make commands like `git annex find` list the same
+set of files that are acted on by commands like `git annex drop`
+(`git annex find --in here`)
+and `git annex get` (`git annex find --not --in here`).
+"""]]
diff --git a/doc/todo/wishlist__58___--maxdepth_option_for_git_annex_find.mdwn b/doc/todo/wishlist__58___--maxdepth_option_for_git_annex_find.mdwn
new file mode 100644
index 0000000000..c309f24911
--- /dev/null
+++ b/doc/todo/wishlist__58___--maxdepth_option_for_git_annex_find.mdwn
@@ -0,0 +1,22 @@
+`git annex find` currently makes for a great way to find which files are already local, and don't need to get `git annex get` gotten; obviously `ls` just shows me all the files in a given directory, disregarding git-annex (and without recursing to subdirectories). I think that adding a '--maxdepth' option to `git annex find` would make it much easier to use at directories high up in the directory structure, since currently `git annex find` recurses all subdirectories necessarily, when I really just want to see whether or not there are git-annex files present from a given directory.
+
+Obviously, since directories themselves are not git-annex objects, there is no way to say whether or not they are "present", but perhaps the most intuitive would be to say whether or not any git-annex files under a given directory are present.
+
+For example, if I have:  
+./  
++-- subdir0/  
+|  +-- file0 (present in local git-annex repo)  
+|  +-- file1 (present in local git-annex repo)  
++-- subdir1/  
+|  +-- file0 (not present in local git-annex repo)  
+|  +-- file1 (not present in local git-annex repo)  
++-- file2 (present in local git-annex repo)  
+
+and I type `git annex find --maxdepth 1 .`, the output might look something like:  
+subdir0/  
+file2  
+
+rather than:  
+subdir0/file0  
+subdir0/file1  
+file2  
diff --git a/doc/todo/wishlist__58___--maxdepth_option_for_git_annex_find/comment_1_c355878ac49bbb23a4cf82fe685da9ee._comment b/doc/todo/wishlist__58___--maxdepth_option_for_git_annex_find/comment_1_c355878ac49bbb23a4cf82fe685da9ee._comment
new file mode 100644
index 0000000000..73d4cba5f8
--- /dev/null
+++ b/doc/todo/wishlist__58___--maxdepth_option_for_git_annex_find/comment_1_c355878ac49bbb23a4cf82fe685da9ee._comment
@@ -0,0 +1,10 @@
+[[!comment format=mdwn
+ username="http://joeyh.name/"
+ ip="108.236.230.124"
+ subject="comment 1"
+ date="2014-07-21T17:28:53Z"
+ content="""
+`find --maxdepth` is a nice optimisation because it can short-circuit when it gets deep in the tree. However, `git annex find` is built on top of `git ls-files --cached`, which has no equivilant way to short-circuit. I am not sure if the format of the index makes it practical for it to get a --maxdepth option (it may need to traverse the whole index, or might be able to short-circuit).
+
+I don't see any point in adding a --matdepth to git-annex if it doesn't actually make it any faster, so getting such a thing into `git ls-files` would be the first step. So, suggest filing a feature request on git.
+"""]]
diff --git a/doc/todo/wishlist__58___--maxdepth_option_for_git_annex_find/comment_2_da30a066c4de465fe172ad01057e2380._comment b/doc/todo/wishlist__58___--maxdepth_option_for_git_annex_find/comment_2_da30a066c4de465fe172ad01057e2380._comment
new file mode 100644
index 0000000000..944f36088e
--- /dev/null
+++ b/doc/todo/wishlist__58___--maxdepth_option_for_git_annex_find/comment_2_da30a066c4de465fe172ad01057e2380._comment
@@ -0,0 +1,10 @@
+[[!comment format=mdwn
+ username="https://www.google.com/accounts/o8/id?id=AItOawlScsufvQF7s8TVTwPd-h_QiP5Hn_i-hrs"
+ nickname="Jason"
+ subject="comment 2"
+ date="2014-07-21T18:37:19Z"
+ content="""
+I see your point, `git ls-files` may still have to walk the whole tree, precluding a speed advantage.
+But I guess the point of what I was saying was more that a way summarize from a high level what is here and what is not would be nice.
+I certainly understand if this is not something you see as worthwhile, but if someone were inclined to write a patch (if ever I find the time) that would add a `--maxdepth` option that would merely summarize the results of `git annex find` would it be something you would be inclined to include in the main repo (providing, of course, that you find the behavior sensible)?
+"""]]
diff --git a/doc/todo/wishlist__58___--maxdepth_option_for_git_annex_find/comment_3_f3eadd6241f5cc2886515b2826dc5cf9._comment b/doc/todo/wishlist__58___--maxdepth_option_for_git_annex_find/comment_3_f3eadd6241f5cc2886515b2826dc5cf9._comment
new file mode 100644
index 0000000000..8da436e09e
--- /dev/null
+++ b/doc/todo/wishlist__58___--maxdepth_option_for_git_annex_find/comment_3_f3eadd6241f5cc2886515b2826dc5cf9._comment
@@ -0,0 +1,10 @@
+[[!comment format=mdwn
+ username="http://joeyh.name/"
+ ip="108.236.230.124"
+ subject="comment 3"
+ date="2014-07-21T18:40:00Z"
+ content="""
+I think that --maxdepth has a well-defined meaning and this summary option would need to be named something else.
+
+I don't object to the idea of implementing it. However, I don't know that it would be very easy to implement either.
+"""]]
diff --git a/doc/todo/wishlist__58___--maxdepth_option_for_git_annex_find/comment_4_c766c1465407324fc933db78be325b33._comment b/doc/todo/wishlist__58___--maxdepth_option_for_git_annex_find/comment_4_c766c1465407324fc933db78be325b33._comment
new file mode 100644
index 0000000000..6025101e2e
--- /dev/null
+++ b/doc/todo/wishlist__58___--maxdepth_option_for_git_annex_find/comment_4_c766c1465407324fc933db78be325b33._comment
@@ -0,0 +1,12 @@
+[[!comment format=mdwn
+ username="https://www.google.com/accounts/o8/id?id=AItOawlScsufvQF7s8TVTwPd-h_QiP5Hn_i-hrs"
+ nickname="Jason"
+ subject="good point"
+ date="2014-07-21T19:22:30Z"
+ content="""
+You make another good point `--maxdepth` is vague in this context...
+I guess if we were to decide to come up with a summary option, it will have be named something else, like `--summary-depth`, where the default would be to represent all files of whatever depth, and specifying the option would take the output that would otherwise get from `git annex find `, truncate the paths to a certain depth, and then make a set thereof (to remove the many dups), that way any directory that had any files that would have been output by `git annex find `, that would also be at or above a certain depth, would be listed.
+
+I think if I get a chance I'll try to implement something like this.
+
+"""]]
diff --git a/doc/todo/wishlist__58___Add_--byte-limit_option.mdwn b/doc/todo/wishlist__58___Add_--byte-limit_option.mdwn
new file mode 100644
index 0000000000..28642b50d7
--- /dev/null
+++ b/doc/todo/wishlist__58___Add_--byte-limit_option.mdwn
@@ -0,0 +1,14 @@
+This option is related to `--time-limit`, but stops after processing X 
+bytes. For example, I often need to free up a certain amount of space on 
+a disk and want `git-annex move` to stop after it has copied a specified 
+amount of data:
+
+    git annex move --to otherdisk --byte-limit 3G
+    git annex drop --auto --byte-limit 500m
+    git annex get --byte-limit 500m
+
+I've been using some `annex.diskreserve` trickery now and then to 
+accomplish this, but it's a bit cumbersome and also not a very precise 
+way to specify the amount I want to copy/move/get. The last example 
+would also be a useful command to limit the traffic when I'm connecting 
+via mobile – get as much as possible, but don't blow the mobile quota.
diff --git a/doc/todo/wishlist__58___Freeing_X_space_on_remote_Y.mdwn b/doc/todo/wishlist__58___Freeing_X_space_on_remote_Y.mdwn
new file mode 100644
index 0000000000..5fec39d98f
--- /dev/null
+++ b/doc/todo/wishlist__58___Freeing_X_space_on_remote_Y.mdwn
@@ -0,0 +1 @@
+As suggested during the first Gitify BoF during DebConf13: Adding a way to have on-demand dropping of content in a given remote would allow a user to quickly free up disk space on demand while still heeding numcopies etc.
diff --git a/doc/todo/wishlist__58___Option_to_specify_max_transfer_rate.mdwn b/doc/todo/wishlist__58___Option_to_specify_max_transfer_rate.mdwn
new file mode 100644
index 0000000000..3ecb421978
--- /dev/null
+++ b/doc/todo/wishlist__58___Option_to_specify_max_transfer_rate.mdwn
@@ -0,0 +1,3 @@
+A big part of my online use is done via a low-speed connection over my mobile phone, this is limited to 16KB/sec because I always use up my 500MB quota the very first day of the month. `;-/` So when I need to download big files, I first download them to my online server, then transfer the files to my laptop with git-annex. If I'm connected via GSM, this occupies all the bandwidth and everything else moves like a heavily sedated slug. So if I want to work via VNC or SSH, I have to terminate ongoing transfers with Ctrl-C and then hopefully remember to restart it when I work locally. I know git-annex is robust enough to handle this gracefully, but it would be really nice to have a continuous connection going on in the background, limited to a value I choose.
+
+rsync(1) has a `--bwlimit` (bandwidth limit) where you can specify max download/upload speed in kilobytes/sec. It would be great if a similar option was integrated into git-annex. Thanks in advance.
diff --git a/doc/todo/wishlist__58___Option_to_specify_max_transfer_rate/comment_1_4fd870e14b5b70c8a6ade41406294387._comment b/doc/todo/wishlist__58___Option_to_specify_max_transfer_rate/comment_1_4fd870e14b5b70c8a6ade41406294387._comment
new file mode 100644
index 0000000000..78ca76939a
--- /dev/null
+++ b/doc/todo/wishlist__58___Option_to_specify_max_transfer_rate/comment_1_4fd870e14b5b70c8a6ade41406294387._comment
@@ -0,0 +1,10 @@
+[[!comment format=mdwn
+ username="https://www.google.com/accounts/o8/id?id=AItOawmBUR4O9mofxVbpb8JV9mEbVfIYv670uJo"
+ nickname="Justin"
+ subject="trickle"
+ date="2013-01-22T22:44:52Z"
+ content="""
+not exactly integrated, but you can easily use trickle for this.
+
+    trickle -d 50 git annex ...
+"""]]
diff --git a/doc/todo/wishlist__58___Option_to_specify_max_transfer_rate/comment_2_dd854f297ad6a94b54be9f3edfd0f766._comment b/doc/todo/wishlist__58___Option_to_specify_max_transfer_rate/comment_2_dd854f297ad6a94b54be9f3edfd0f766._comment
new file mode 100644
index 0000000000..70f04c6168
--- /dev/null
+++ b/doc/todo/wishlist__58___Option_to_specify_max_transfer_rate/comment_2_dd854f297ad6a94b54be9f3edfd0f766._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="http://sunny256.sunbase.org/"
+ nickname="sunny256"
+ subject="Yay, trickle works"
+ date="2013-01-23T01:36:21Z"
+ content="""
+Justin, thanks a lot! trickle(1) works great. I didn't know about this program, but I'm not surprised that such a program is available. It never ceases to amaze me what's possible in a *NIX environment.
+"""]]
diff --git a/doc/todo/wishlist__58___Option_to_specify_max_transfer_rate/comment_3_a8b7e90a473d5937807cc7eb456efe33._comment b/doc/todo/wishlist__58___Option_to_specify_max_transfer_rate/comment_3_a8b7e90a473d5937807cc7eb456efe33._comment
new file mode 100644
index 0000000000..a5f8f6a1bf
--- /dev/null
+++ b/doc/todo/wishlist__58___Option_to_specify_max_transfer_rate/comment_3_a8b7e90a473d5937807cc7eb456efe33._comment
@@ -0,0 +1,10 @@
+[[!comment format=mdwn
+ username="http://joeyh.name/"
+ ip="2001:4978:f:21a::2"
+ subject="comment 3"
+ date="2013-01-24T01:55:10Z"
+ content="""
+In addition to trickle, the git-annex man page has examples of how to make rsync use --bwlimit
+
+Something like trickle is needed to limit rates for remotes not using rsync, however.
+"""]]
diff --git a/doc/todo/wishlist__58___Parity_files_for_encrypted_remotes.mdwn b/doc/todo/wishlist__58___Parity_files_for_encrypted_remotes.mdwn
new file mode 100644
index 0000000000..34f06ad633
--- /dev/null
+++ b/doc/todo/wishlist__58___Parity_files_for_encrypted_remotes.mdwn
@@ -0,0 +1,7 @@
+I have data that has accompanying parity files.  This is supposed to add some
+security to file integrity; however, it only works as long as the files are
+available unencrypted.  In case of encrypted special remotes the existing parity files
+won't be of any use if the encrypted versions of files get corrupted in the remote location.
+
+Would it be worthwhile for git-annex to generate its own
+parity files for the encrypted data in encrypted special remotes?
diff --git a/doc/todo/wishlist__58___Parity_files_for_encrypted_remotes/comment_1_5fbffd74f9d89857b807093cd7a5941e._comment b/doc/todo/wishlist__58___Parity_files_for_encrypted_remotes/comment_1_5fbffd74f9d89857b807093cd7a5941e._comment
new file mode 100644
index 0000000000..b440d64d2f
--- /dev/null
+++ b/doc/todo/wishlist__58___Parity_files_for_encrypted_remotes/comment_1_5fbffd74f9d89857b807093cd7a5941e._comment
@@ -0,0 +1,9 @@
+[[!comment format=mdwn
+ username="git-annex@31849d241f10c295b30a9707352ae5c7d743adb7"
+ nickname="git-annex"
+ avatar="http://cdn.libravatar.org/avatar/180f1a763647bfc099e97ac88b8f7b37"
+ subject="comment 1"
+ date="2017-03-02T16:16:05Z"
+ content="""
+If you're generating parity files for the content anyway, why not encrypt locally, generate parity files for the encrypted content, then check that in to git-annex?
+"""]]
diff --git a/doc/todo/wishlist__58___Restore_s3_files_moved_to_Glacier.mdwn b/doc/todo/wishlist__58___Restore_s3_files_moved_to_Glacier.mdwn
new file mode 100644
index 0000000000..85fc2785c4
--- /dev/null
+++ b/doc/todo/wishlist__58___Restore_s3_files_moved_to_Glacier.mdwn
@@ -0,0 +1,7 @@
+I would like to use the automated AWS lifecycle rules to move the git annex files store on S3 to Glacier after a bit of time. Git annex need must support this kind of S3 files explicitly in order for it to work.
+
+This is different from the adding a Glacier remote to git annex because of the reasons explained in .
+
+Basically, the files moved by AWS from S3 to Glacier are not available under the normal Glacier API. In fact, the moved S3 files are listed as available but under the `GLACIER` storage class and need a RESTORE request before they can be GET like other S3 files. Trying to GET an S3 file that has been moved to Glacier will not restore it from Glacier and will result in an 403 error.
+
+I suppose DELETE needs special care as well.
diff --git a/doc/todo/wishlist__58___Restore_s3_files_moved_to_Glacier/comment_1_eb934756cb2af7fa13ad3b5fad7f85b2._comment b/doc/todo/wishlist__58___Restore_s3_files_moved_to_Glacier/comment_1_eb934756cb2af7fa13ad3b5fad7f85b2._comment
new file mode 100644
index 0000000000..92554aa318
--- /dev/null
+++ b/doc/todo/wishlist__58___Restore_s3_files_moved_to_Glacier/comment_1_eb934756cb2af7fa13ad3b5fad7f85b2._comment
@@ -0,0 +1,9 @@
+[[!comment format=mdwn
+ username="https://www.google.com/accounts/o8/id?id=AItOawkAUMhKOSkh9JaBA6xst3XxQIIsDEq5Zd4"
+ nickname="Ovidiu"
+ subject="Keen to see this happen too!"
+ date="2014-05-08T21:44:30Z"
+ content="""
+Or alternatively support for S3 Lifecycle Rules. 
+I'm not exactly sure what git-annex assistant does if it backs up into a S3 bucket with i.e. a rule to archive everything 1 month later into Glacier. Can it restore/access?
+"""]]
diff --git a/doc/todo/wishlist__58_____39__get__39___queue_and_schedule.mdwn b/doc/todo/wishlist__58_____39__get__39___queue_and_schedule.mdwn
new file mode 100644
index 0000000000..8166479b8b
--- /dev/null
+++ b/doc/todo/wishlist__58_____39__get__39___queue_and_schedule.mdwn
@@ -0,0 +1,32 @@
+During the campaign adding a chunking feature to obscure filesize for encrypted files was added to the roadmap.  But there is still one potentially valuable set* of data that git-annex can help obscure: when you access your remotes.
+
+This data can be used to determine when a user is actively using a remote, but if a remote is always accessed at the same time that data becomes less useful.  Somebody could still monitor total traffic over a long period and figure out that a remote was more active in a given week or month, but scheduling reduces the resolution of your access times and their data.  Maybe this isn't the most important feature to add, but it would be nice to have, and could possibly be built on top of the existing git-annex scheduler.  It could work by queuing inter-remote transactions ('get', 'copy', 'sync', etc.), so that jobs start at the same time every day, or even the same time and day every week.
+
+Possible syntax examples:
+###Setting up the schedule:
+git annex queue schedule Monday:1730 (starts every monday at 5:30PM)
+
+git annex queue schedule 1400 (starts every day at 2PM)
+
+###Queuing git-annex commands:
+git annex queue prepend sync (pretends 'sync' to the very front of the queue)
+
+git annex queue append get file.ISO (appends to queue file.ISO for retrieval from a repo)
+
+###Viewing/editing queue:
+git annex queue view (view the current queue, jobs displayed with corresponding numbers)
+
+git annex queue rm 20 (removes job 20 from queue)
+
+
+\*The four I can think of are:
+
+* File contents (solved by crypto)
+
+* File size (solved on the remote by chunking, but total traffic usage can't be helped)
+
+* User IP/Remote IP (solved by VPN - outside scope of git-annex, unless someone writes a plugin)
+
+* Access times (obscured by a queue and scheduling)
+
+Note, there is a [[git-annex-schedule]] command now, but it only does `fsck`.
diff --git a/doc/todo/wishlist__58_____39__get__39___queue_and_schedule/comment_1_d6ab311beee2ce5f73ae325a4ae463d4._comment b/doc/todo/wishlist__58_____39__get__39___queue_and_schedule/comment_1_d6ab311beee2ce5f73ae325a4ae463d4._comment
new file mode 100644
index 0000000000..df273afb2c
--- /dev/null
+++ b/doc/todo/wishlist__58_____39__get__39___queue_and_schedule/comment_1_d6ab311beee2ce5f73ae325a4ae463d4._comment
@@ -0,0 +1,11 @@
+[[!comment format=mdwn
+ username="anarcat"
+ subject="how does the queue get built now?"
+ date="2015-04-28T21:32:36Z"
+ content="""
+how does the queue get created now? is it random or is there some deterministic thing that could be abuse to prioritise some content already (say alphabetical order or some similar hack)?
+
+the scheduling I think i can work around with clunky cron jobs to start/stop the assistant, but the queue I feel is a different beast. would you be available for hire to implement this feature in priority?
+
+thanks!
+"""]]
diff --git a/doc/todo/wishlist__58_____39__whereis__39___support_in_the_webapp.mdwn b/doc/todo/wishlist__58_____39__whereis__39___support_in_the_webapp.mdwn
new file mode 100644
index 0000000000..c074988b08
--- /dev/null
+++ b/doc/todo/wishlist__58_____39__whereis__39___support_in_the_webapp.mdwn
@@ -0,0 +1,4 @@
+I've looked for this feature in the webapp but can't find it...
+
+I mainly use the webapp and have been wondering 'how many copies of file X are there' and 'where are the copies of file Y'?
+This is available in the commandline interface but it would be nice to have this in the webapp too.
diff --git a/doc/todo/wishlist__58_____96__git_annex_drop_--relaxed__96__.mdwn b/doc/todo/wishlist__58_____96__git_annex_drop_--relaxed__96__.mdwn
new file mode 100644
index 0000000000..626f5a03f9
--- /dev/null
+++ b/doc/todo/wishlist__58_____96__git_annex_drop_--relaxed__96__.mdwn
@@ -0,0 +1,5 @@
+Also suggested during the first Gitify BoF during DebConf13:
+
+`git annex drop` deletes immediately. In some situations a mechanism to tell git-annex "I would like to hold onto this data if possible, but if you need the space, please delete it" could be nice.
+
+An obvious question would be how to do cleanups. With the assistant, that's easy. On CLI, at the very least `git annex fsck` should list, and optionally delete, that data.
diff --git a/doc/todo/wishlist__58_____96__git_annex_drop_--relaxed__96__/comment_1_c83a6cddd0ce222205a149cfa41ca395._comment b/doc/todo/wishlist__58_____96__git_annex_drop_--relaxed__96__/comment_1_c83a6cddd0ce222205a149cfa41ca395._comment
new file mode 100644
index 0000000000..32d0c01126
--- /dev/null
+++ b/doc/todo/wishlist__58_____96__git_annex_drop_--relaxed__96__/comment_1_c83a6cddd0ce222205a149cfa41ca395._comment
@@ -0,0 +1,10 @@
+[[!comment format=mdwn
+ username="http://cstork.org/"
+ nickname="Chris Stork"
+ subject="How should this interact with the trust model and location tracking?"
+ date="2013-10-04T11:13:11Z"
+ content="""
+This could become complicated.  AFAIU, right now git-annex keeps track of files as either present or absent.  With this feature it's tempting to introduce a third state 'potentially dropped' (or 'dropped in a relaxed fashion') but do you then treat them as if they were dropped depending in wether they are on a trusted or untrusted repo?  Or maybe a potentially dropped file in a trusted repo is treated as a file in a semitrusted repo?  This becomes convoluted.  You also need a command to undrop a file in case you decide that you really want to keep it and in order to do this you need a command to see which files are up for relaxed dropping....
+
+As an alternative approach maybe it makes sense to extend [[preferred content]] expressions to take file sizes and disk usage into account.
+"""]]
diff --git a/doc/todo/wishlist__58_____96__git_annex_drop_--relaxed__96__/comment_2_353fbc2bcc40cb8c9af42907a34c6e5a._comment b/doc/todo/wishlist__58_____96__git_annex_drop_--relaxed__96__/comment_2_353fbc2bcc40cb8c9af42907a34c6e5a._comment
new file mode 100644
index 0000000000..40f299fa02
--- /dev/null
+++ b/doc/todo/wishlist__58_____96__git_annex_drop_--relaxed__96__/comment_2_353fbc2bcc40cb8c9af42907a34c6e5a._comment
@@ -0,0 +1,11 @@
+[[!comment format=mdwn
+ username="http://joeyh.name/"
+ ip="4.152.108.243"
+ subject="comment 2"
+ date="2013-10-04T20:17:07Z"
+ content="""
+I don't think that a third state would be necessary. Actually dropping the file when it happens would need to do the same numcopies verification that `git annex drop` does now.
+
+I agree it might be simpler to first improve the power of preferred content expressions. Unfortunately one thing that cannot be put in them is anything that probes the current state of the system. This is because repo A on machine X needs to be able to calculate the preferred content of repo B on machine Y.
+But I could certainly add file size as a preferred content term, since that info is known throughout the network.
+"""]]
diff --git a/doc/todo/wishlist__58_____96__git_annex_drop_--relaxed__96__/comment_3_3e830035df580601f038ce3a7003c39d._comment b/doc/todo/wishlist__58_____96__git_annex_drop_--relaxed__96__/comment_3_3e830035df580601f038ce3a7003c39d._comment
new file mode 100644
index 0000000000..8b4b132cba
--- /dev/null
+++ b/doc/todo/wishlist__58_____96__git_annex_drop_--relaxed__96__/comment_3_3e830035df580601f038ce3a7003c39d._comment
@@ -0,0 +1,63 @@
+[[!comment format=mdwn
+ username="erics"
+ ip="76.10.136.8"
+ subject="comment 3"
+ date="2014-05-04T00:48:55Z"
+ content="""
+I think things would be simpler if a \"drop --relaxed\" file were to look to
+the outside world just like one that was dropped without \"--relaxed\".
+In particular, even if a file is dropped with \"--relaxed\":
+
+  - the file's work-tree symlink should be broken synchronously by the
+   \"drop --relaxed\" command (as opposed to only being broken later,
+   if and when the file physically goes away)
+
+  - other repos should no longer see the file as available from this repo
+
+Basically, the idea is to add a third state, but **not** a user-visible one.
+Rather, it should be a well hidden implementation detail, which doesn't
+affect the conceptual model (very much like git's own distinction between
+loose and packed objects).  Thus, *logically dropped* would be a
+better name than *potentially dropped*.
+
+Corollaries:
+
+  - A logically (but not physically) dropped file should *not* count towards
+    satisfying the numcopies limit, i.e. if some other repo has been asked
+    to drop the file too
+
+  - That in turn means that \"git annex drop --relaxed\" needs to satisfy a
+    numcopies check at the time the user runs it; it's not enough to only do
+    the check later, at physical-deletion time.  (At that point, there should probably be
+    a second numcopies check.  I don't know whether the model requires it,
+    but even if not, paranoia is good :-) )
+
+  - If the user wants to use the file again, they have to \"git annex get\" it
+    again, just like usual -- but if the file hasn't been physically deleted yet,
+    the \"get\" will be nearly instantaneous, since the data won't have to be copied
+
+One possible implementation would be to have \"drop --relaxed\" behave almost identically
+to a non-relaxed drop -- do all the same safety checks, bookkeeping, etc.  The only
+difference would be to have it rename the file at the end, rather
+than deleting it outright.  (Logically dropped files could stay in their same
+directory, but with a distinguishing filename, or they could be moved to a
+parallel tree, e.g. *.git/annex/dropped*.  I don't have an opinion on that choice;
+I've just picked one arbitrarily to keep talking about.)
+
+\"get\" would simply search .git/annex/dropped before going off to remote
+repos, and if the file is found there, would move (not copy) it back into
+.git/annex/objects.
+
+An alternative might be to set some kind of *logically dropped* flag, but
+that would probably be a much more intrusive change; a lot of places in the
+code would have to check the flag.  Doing it as a file rename would make for a much more
+localized change; most of git-annex would completely ignore .git/annex/dropped, and just go
+about its business as it has always done.
+
+(It might be tempting to think of (or even implement) .git/annex/dropped as a
+very low-cost remote, but that's not accurate; the semantics are different.)
+
+I'm just starting to experiment with git-annex, so I can only hope that what 
+I'm saying isn't completely silly...
+
+"""]]
diff --git a/doc/todo/wishlist__58_____96__git_annex_drop_--relaxed__96__/comment_4_e5516689bc128c061dcd66649dc69584._comment b/doc/todo/wishlist__58_____96__git_annex_drop_--relaxed__96__/comment_4_e5516689bc128c061dcd66649dc69584._comment
new file mode 100644
index 0000000000..3e2d6689c2
--- /dev/null
+++ b/doc/todo/wishlist__58_____96__git_annex_drop_--relaxed__96__/comment_4_e5516689bc128c061dcd66649dc69584._comment
@@ -0,0 +1,10 @@
+[[!comment format=mdwn
+ username="http://joeyh.name/"
+ ip="216.145.95.162"
+ subject="comment 4"
+ date="2014-05-19T16:56:15Z"
+ content="""
+erics, that all makes a lot of sense, except I don't know if there's actually a use case for a git-annex that behaves that way. It doesn't seem to solve the original use case.
+
+I'd be inclinded to instead use the new metadata support. A file could have a tag that indicates it's not strongly wanted, and if git-annex get doesn't have enough space it could seek out and drop such files.
+"""]]
diff --git a/doc/todo/wishlist__58_____96__git_annex_drop_--relaxed__96__/comment_5_be740e4b06d9130ae6afc5783da3c0e0._comment b/doc/todo/wishlist__58_____96__git_annex_drop_--relaxed__96__/comment_5_be740e4b06d9130ae6afc5783da3c0e0._comment
new file mode 100644
index 0000000000..32beef4271
--- /dev/null
+++ b/doc/todo/wishlist__58_____96__git_annex_drop_--relaxed__96__/comment_5_be740e4b06d9130ae6afc5783da3c0e0._comment
@@ -0,0 +1,14 @@
+[[!comment format=mdwn
+ username="erics"
+ ip="76.10.136.8"
+ subject="comment 5"
+ date="2014-06-03T17:49:10Z"
+ content="""
+> It doesn't seem to solve the original use case.
+
+It doesn't?  The OP requested:
+
+> I would like to hold onto this data if possible, but if you need the space, please delete it
+
+It looks to me as though my suggestion does just that -- or am I misunderstanding what they asked for?
+"""]]
diff --git a/doc/todo/wishlist__58_____96__git_annex_drop_--relaxed__96__/comment_6_79d115f95cec46bb51e7fba078524db1._comment b/doc/todo/wishlist__58_____96__git_annex_drop_--relaxed__96__/comment_6_79d115f95cec46bb51e7fba078524db1._comment
new file mode 100644
index 0000000000..6e376e3167
--- /dev/null
+++ b/doc/todo/wishlist__58_____96__git_annex_drop_--relaxed__96__/comment_6_79d115f95cec46bb51e7fba078524db1._comment
@@ -0,0 +1,17 @@
+[[!comment format=mdwn
+ username="erics"
+ ip="76.10.136.8"
+ subject="Metadata vs "drop --relaxed""
+ date="2014-06-03T18:20:50Z"
+ content="""
+[This isn't as much about my suggested implementation for \"drop --relaxed\" as about whether the feature is worth providing in the first place.  I'm not arguing strongly for it, actually; just continuing the discussion.]
+
+> I'd be inclinded to instead use the new metadata support.
+
+I see metadata as more for static attributes of a given file -- this thing is \"a picture\", \"related to project X\", \"from Mary\".  Thus, the combination of metadata plus preferred-content settings seems to me more suitable for static preferences (likely ones that implement some kind of policy, however informal); e.g. \"this repo wants pictures but not mp3s\", or \"Mary's stuff but not Alex's\".
+
+\"drop --relaxed\", on the other hand, would be good for more ad-hoc usage: \"disk space is getting tight; hmm, I'm not using *foo* today, so git-annex, please delete my local copy of *${myrepo}/foo* -- but only as much as you have to, because I'm going to want it again tomorrow\".
+
+One reason not to want to use metadata and preferred-content settings for such short-term, ad-hoc needs is that you then have to remember to go undo the changes later.  That's even worse if you had to add ad-hoc metadata, and now have to go delete it all again.  Undoing a \"drop --relaxed\", on the other hand, consists of a simple \"git annex get\".
+
+"""]]
diff --git a/doc/todo/wishlist__58_____96__git_annex_optimize__96__.mdwn b/doc/todo/wishlist__58_____96__git_annex_optimize__96__.mdwn
new file mode 100644
index 0000000000..8e6fced45f
--- /dev/null
+++ b/doc/todo/wishlist__58_____96__git_annex_optimize__96__.mdwn
@@ -0,0 +1,6 @@
+Given [this tip](https://git-annex.branchable.com/forum/__34__git_annex_sync__34___synced_after_8_hours/#comment-890ca1381d800ac833ccbb8c5db175ea), [this comment](https://git-annex.branchable.com/todo/wishlist:_rsync_efficiency/#comment-870ae805efd35343edefdbed792dac04) and possibly others, it would be nice if git-annex could look at any given repo and make suggestions for improvements. As a second step, it could look at remotes as well. And as a third, maybe even change repo settings and not just make suggestions.
+
+Having a few old repos with terabytes of data on various disks, I would just toss this stanza into my mr templates to eventually optimize all my repos to current best practices.
+
+
+-- Richard
diff --git a/doc/todo/wishlist__58___allow_custom_S3_url_in_webapp.mdwn b/doc/todo/wishlist__58___allow_custom_S3_url_in_webapp.mdwn
new file mode 100644
index 0000000000..2d291608b0
--- /dev/null
+++ b/doc/todo/wishlist__58___allow_custom_S3_url_in_webapp.mdwn
@@ -0,0 +1,3 @@
+It is now relatively easy to build your own S3-compatible storage system with software such as Ceph radosgw or Openstack Swift.
+
+So a way for users to specify their own S3 url would come pretty handy in the webapp's "add S3 remote" page.
diff --git a/doc/todo/wishlist__58___allow_custom_S3_url_in_webapp/comment_1_3d1ea5579a6ad0c0efde58dca11c10aa._comment b/doc/todo/wishlist__58___allow_custom_S3_url_in_webapp/comment_1_3d1ea5579a6ad0c0efde58dca11c10aa._comment
new file mode 100644
index 0000000000..b6125fd6eb
--- /dev/null
+++ b/doc/todo/wishlist__58___allow_custom_S3_url_in_webapp/comment_1_3d1ea5579a6ad0c0efde58dca11c10aa._comment
@@ -0,0 +1,10 @@
+[[!comment format=mdwn
+ username="http://joeyh.name/"
+ ip="209.250.56.227"
+ subject="comment 1"
+ date="2013-12-29T20:16:43Z"
+ content="""
+Such S3 clones are supported if you make the repository at the command line. See [[S3_special_remote_documentation|special_remotes/S3]].
+
+I think the question comes down to, is it worth adding UI in the webapp for this, or is it likely that someone who is using a S3 clone knows the endpoint url to use? Similarly, the webapp only offers box.com as a WebDAV remote, although there are many other DAV servers. What I'm trying to do with the UI is guide the user through finding a place to store their data, and covering every step as easily as possible.
+"""]]
diff --git a/doc/todo/wishlist__58___allow_custom_S3_url_in_webapp/comment_2_f96bb81fde4185368dc6ea5a5aed87da._comment b/doc/todo/wishlist__58___allow_custom_S3_url_in_webapp/comment_2_f96bb81fde4185368dc6ea5a5aed87da._comment
new file mode 100644
index 0000000000..bd3885a966
--- /dev/null
+++ b/doc/todo/wishlist__58___allow_custom_S3_url_in_webapp/comment_2_f96bb81fde4185368dc6ea5a5aed87da._comment
@@ -0,0 +1,10 @@
+[[!comment format=mdwn
+ username="https://www.google.com/accounts/o8/id?id=AItOawkgy5Qf4bivpTfYPw_yh92OcDlesrVTREg"
+ nickname="François"
+ subject="comment 2"
+ date="2014-01-12T20:31:44Z"
+ content="""
+I understand your concerns about keeping the UI straightforward.
+
+It makes sense however to allow users to add a custom backend without using the CLI. What about a new \"Custom backend (expert only)\" option which directly asks for a string with the arguments to 'git annex initremote'?
+"""]]
diff --git a/doc/todo/wishlist__58___an___34__assistant__34___for_web-browsing_--_tracking_the_sources_of_the_downloads.mdwn b/doc/todo/wishlist__58___an___34__assistant__34___for_web-browsing_--_tracking_the_sources_of_the_downloads.mdwn
new file mode 100644
index 0000000000..c910ace830
--- /dev/null
+++ b/doc/todo/wishlist__58___an___34__assistant__34___for_web-browsing_--_tracking_the_sources_of_the_downloads.mdwn
@@ -0,0 +1,28 @@
+A replacement for a web-browser's downloads menu that uses git-annex internally ([[`addurl`|tips/using the web as a special remote]] command for the download, and `drop` or `git rm` for the clean up) would be quite helpful:
+
+say, when working on a topic, writing a paper or similar things, I usually have a Git repo with my text, all relevant data and processing code, and possibly other background information. It's nice to store the literature you needed at the same place where you work. (So that it is easy to catch up with what I was doing and thinking over when I left this work aside for a while, perhaps after cloning the repo from another location.)
+
+When I find an interesting literature, I save the file to the directory with my work, and read it. Then I might return to it to re-read it. There might be references to this document from my work, so I'd like them to work as links (perhaps pointing at the local file, but also at the source URL for the case when my document is read by someone else not on my system).
+
+I need to keep track of the source URLs for the documents I have saved which I read and use.
+
+That's a task that fits well git-annex.
+
+Note that doing the dull work of copying and pasting the URL and the downloading it and then opening it for reading is a pain to do every time I'm interested in a document I have found on the web. (Of course, I would need to fill out the bibliogrphic information for this document if I want to refer to it, but that can be done later. Initially, I wish I just don't lose the source URL of a document at the moment when I get interested in it and start reading.)
+
+So, I could be assisted by a replacement of the "downloads" menu of, say, Firefox: whenever I want to open a file for viewing (like a PDF), it should ask me where to save it, and I'd choose the directory with my work, then it should register it with git-annex (so that the source URL is saved, and perhaps it should also write down the referring page's URL somewhere nearby automatically), download it, and open with a viewer for reading.
+
+Then I'll have the interesting literature there when I'm offline; the source URLs would be saved, so that they can be put into the references. Also, if I distribute this work with Git, at another location git-annex can be used to easily get all the literature again.
+
+(Hmmm... probably, the browser that this will be simplest for me to implement for is emacs-w3m; simply, some functions calling git-annex...)
+
+> This seems fairly doable to implement since the git-annex [[design/assistant]]
+> already has a webapp. So a javascript toolbar thing could be made that
+> submits the current url (or maybe links dragged into it?) to the webapp
+> for adding to the annex.
+> 
+> The only wrinkle is that the webapp runs under a new url each time
+> it starts, due to using a high port and embedding some auth token in the
+> url. --[[Joey]]
+
+[[!tag /design/assistant]]
diff --git a/doc/todo/wishlist__58___an___34__assistant__34___for_web-browsing_--_tracking_the_sources_of_the_downloads/comment_1_36ae3c75053d5ec278b5e6eb2084d57a._comment b/doc/todo/wishlist__58___an___34__assistant__34___for_web-browsing_--_tracking_the_sources_of_the_downloads/comment_1_36ae3c75053d5ec278b5e6eb2084d57a._comment
new file mode 100644
index 0000000000..4c20561c75
--- /dev/null
+++ b/doc/todo/wishlist__58___an___34__assistant__34___for_web-browsing_--_tracking_the_sources_of_the_downloads/comment_1_36ae3c75053d5ec278b5e6eb2084d57a._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="https://www.google.com/accounts/o8/id?id=AItOawmBUR4O9mofxVbpb8JV9mEbVfIYv670uJo"
+ nickname="Justin"
+ subject="comment 1"
+ date="2012-09-25T23:55:44Z"
+ content="""
+You know about `git annex addurl` right?  It doesn't help with the browser integration (though I bet there are existing download manager extensions you could re-use for this) but it takes care of the other use cases.
+"""]]
diff --git a/doc/todo/wishlist__58___an___34__assistant__34___for_web-browsing_--_tracking_the_sources_of_the_downloads/comment_3_be8eb800523db8cf7a2c68a28fbf5ea5._comment b/doc/todo/wishlist__58___an___34__assistant__34___for_web-browsing_--_tracking_the_sources_of_the_downloads/comment_3_be8eb800523db8cf7a2c68a28fbf5ea5._comment
new file mode 100644
index 0000000000..c958fd08f7
--- /dev/null
+++ b/doc/todo/wishlist__58___an___34__assistant__34___for_web-browsing_--_tracking_the_sources_of_the_downloads/comment_3_be8eb800523db8cf7a2c68a28fbf5ea5._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="http://m-f-k.myopenid.com/"
+ ip="93.207.162.28"
+ subject="+1"
+ date="2012-10-05T20:00:31Z"
+ content="""
+Copy+Paste+Open is to much for lazy people like me;)  I like the idea of a direct browser download manager integration.  This would make it so much easier to find find the original source of a downloaded file when you're to lazy to write it down somewhere in the first place …
+"""]]
diff --git a/doc/todo/wishlist__58___an___34__assistant__34___for_web-browsing_--_tracking_the_sources_of_the_downloads/comment_3_d9f725de41a8572c85e4c6d9b4bcc927._comment b/doc/todo/wishlist__58___an___34__assistant__34___for_web-browsing_--_tracking_the_sources_of_the_downloads/comment_3_d9f725de41a8572c85e4c6d9b4bcc927._comment
new file mode 100644
index 0000000000..30515a49b5
--- /dev/null
+++ b/doc/todo/wishlist__58___an___34__assistant__34___for_web-browsing_--_tracking_the_sources_of_the_downloads/comment_3_d9f725de41a8572c85e4c6d9b4bcc927._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="http://lj.rossia.org/users/imz/"
+ ip="79.165.56.162"
+ subject="the scheme to follow to use the addurl command is longer"
+ date="2012-09-26T00:07:11Z"
+ content="""
+Justin, yes, I know. To use [[addurl|tips/using the web as a special remote]], I should force myself not to click on a link to view it, but rather to copy it, paste into the command line with addurl (then it will be downloaded, and I can run a command to view it...). Not terrible, probably learnable.
+"""]]
diff --git a/doc/todo/wishlist__58___an___34__assistant__34___for_web-browsing_--_tracking_the_sources_of_the_downloads/comment_4_f52492e4cc6f965515800bd1c0e05c90._comment b/doc/todo/wishlist__58___an___34__assistant__34___for_web-browsing_--_tracking_the_sources_of_the_downloads/comment_4_f52492e4cc6f965515800bd1c0e05c90._comment
new file mode 100644
index 0000000000..7efd583c0f
--- /dev/null
+++ b/doc/todo/wishlist__58___an___34__assistant__34___for_web-browsing_--_tracking_the_sources_of_the_downloads/comment_4_f52492e4cc6f965515800bd1c0e05c90._comment
@@ -0,0 +1,10 @@
+[[!comment format=mdwn
+ username="andy"
+ ip="99.48.75.171"
+ subject="I've just written a script that does this (with a Firefox download manager)"
+ date="2013-04-09T02:51:26Z"
+ content="""
+So I've just written a two-line shell script that makes [FlashGot](http://flashgot.net/) able to do this. It's available on GitHub at .
+
+This could be even neater if Git-annex had the ability to set the download manager... Wishlist created: [[wishlist:_allow_configuration_of_downloader_for_addurl]]
+"""]]
diff --git a/doc/todo/wishlist__58___an___34__assistant__34___for_web-browsing_--_tracking_the_sources_of_the_downloads/comment_5_5b36656fc5fa124e763f06711d9da32b._comment b/doc/todo/wishlist__58___an___34__assistant__34___for_web-browsing_--_tracking_the_sources_of_the_downloads/comment_5_5b36656fc5fa124e763f06711d9da32b._comment
new file mode 100644
index 0000000000..494aab7a80
--- /dev/null
+++ b/doc/todo/wishlist__58___an___34__assistant__34___for_web-browsing_--_tracking_the_sources_of_the_downloads/comment_5_5b36656fc5fa124e763f06711d9da32b._comment
@@ -0,0 +1,10 @@
+[[!comment format=mdwn
+ username="http://joeyh.name/"
+ nickname="joey"
+ subject="comment 5"
+ date="2013-04-09T03:36:29Z"
+ content="""
+Wishlist implemented ;)
+
+A full example of doing this would make a nice addition to [[tips]] ...
+"""]]
diff --git a/doc/todo/wishlist__58___an___34__assistant__34___for_web-browsing_--_tracking_the_sources_of_the_downloads/comment_6_285215a4466806baf85b8606f680494a._comment b/doc/todo/wishlist__58___an___34__assistant__34___for_web-browsing_--_tracking_the_sources_of_the_downloads/comment_6_285215a4466806baf85b8606f680494a._comment
new file mode 100644
index 0000000000..d0545153f9
--- /dev/null
+++ b/doc/todo/wishlist__58___an___34__assistant__34___for_web-browsing_--_tracking_the_sources_of_the_downloads/comment_6_285215a4466806baf85b8606f680494a._comment
@@ -0,0 +1,12 @@
+[[!comment format=mdwn
+ username="http://joeyh.name/"
+ nickname="joey"
+ subject="comment 6"
+ date="2013-04-11T15:12:00Z"
+ content="""
+See [[tips/Using_Git-annex_as_a_web_browsing_assistant]].
+
+I am leaving this bug open for now, as some javascript that passes the url off to the webapp is still a nice idea to build.
+
+A solution to the webapp's url always changing could be to have a file:/// url that the webapp creates, and javascript submits to, that then passes the request off to the webapp, probably by using additional javadcript in the html file. Assuming javascript security allows this.
+"""]]
diff --git a/doc/todo/wishlist__58___an___34__assistant__34___for_web-browsing_--_tracking_the_sources_of_the_downloads/comment_7_15bf62e46db4b84ed3156f550f03de42._comment b/doc/todo/wishlist__58___an___34__assistant__34___for_web-browsing_--_tracking_the_sources_of_the_downloads/comment_7_15bf62e46db4b84ed3156f550f03de42._comment
new file mode 100644
index 0000000000..484db1e818
--- /dev/null
+++ b/doc/todo/wishlist__58___an___34__assistant__34___for_web-browsing_--_tracking_the_sources_of_the_downloads/comment_7_15bf62e46db4b84ed3156f550f03de42._comment
@@ -0,0 +1,12 @@
+[[!comment format=mdwn
+ username="http://joeyh.name/"
+ nickname="joey"
+ subject="comment 7"
+ date="2013-04-11T15:41:16Z"
+ content="""
+A less round-about method would be to have the webapp listen for links dropped into its page. You could then have the webapp open in a tab, and just drag urls there to add them to the annex.
+
+I'm not sure if it's possible to intercept drags into a tab. Default browser behavior is certainly to redirect the tab to the url dropped into it.
+
+If someone finds out how to do this, I will build it.
+"""]]
diff --git a/doc/todo/wishlist__58___annex.largefiles_configuration_in_webapp_and_sync.mdwn b/doc/todo/wishlist__58___annex.largefiles_configuration_in_webapp_and_sync.mdwn
new file mode 100644
index 0000000000..873988eea2
--- /dev/null
+++ b/doc/todo/wishlist__58___annex.largefiles_configuration_in_webapp_and_sync.mdwn
@@ -0,0 +1 @@
+The `annex.largefiles` feature is very nice to mix annexed files with normal git managed files. I'd like to be able to configure this setting on the webapp and that the configuration directive would be synchronized accross all remotes.
diff --git a/doc/todo/wishlist__58___annex.largefiles_configuration_in_webapp_and_sync/comment_1_db632de391ce9fce42af51a770ed3aeb._comment b/doc/todo/wishlist__58___annex.largefiles_configuration_in_webapp_and_sync/comment_1_db632de391ce9fce42af51a770ed3aeb._comment
new file mode 100644
index 0000000000..e402d26c3b
--- /dev/null
+++ b/doc/todo/wishlist__58___annex.largefiles_configuration_in_webapp_and_sync/comment_1_db632de391ce9fce42af51a770ed3aeb._comment
@@ -0,0 +1,10 @@
+[[!comment format=mdwn
+ username="http://joeyh.name/"
+ ip="209.250.56.47"
+ subject="comment 1"
+ date="2013-11-05T16:46:28Z"
+ content="""
+It might make sense to sync this across remotes and have it edited with `git annex vicfg`
+
+Putting it in the webapp would need some nice interface for constructing the underlying expression. Might be doable for at least simple filtering (ie, files larger than XX or with extensions .A, .B, .C). I tend to think of this as a setting for people comfortable with the command line though.
+"""]]
diff --git a/doc/todo/wishlist__58___annex.largefiles_configuration_in_webapp_and_sync/comment_2_4a0931d9884054d764fde315d4fe4851._comment b/doc/todo/wishlist__58___annex.largefiles_configuration_in_webapp_and_sync/comment_2_4a0931d9884054d764fde315d4fe4851._comment
new file mode 100644
index 0000000000..0e24014d21
--- /dev/null
+++ b/doc/todo/wishlist__58___annex.largefiles_configuration_in_webapp_and_sync/comment_2_4a0931d9884054d764fde315d4fe4851._comment
@@ -0,0 +1,14 @@
+[[!comment format=mdwn
+ username="https://www.google.com/accounts/o8/id?id=AItOawne_amN4fko4p5cRY_9EYwaYuJKNn7LRio"
+ nickname="Tobias"
+ subject="feedback"
+ date="2013-11-05T21:23:11Z"
+ content="""
+> It might make sense to sync this across remotes and have it edited with git annex vicfg
+
+That would be great!
+
+> Putting it in the webapp would need some nice interface for constructing the underlying expression. Might be doable for at least simple filtering (ie, files larger than XX or with extensions .A, .B, .C). I tend to think of this as a setting for people comfortable with the command line though.
+
+I could live with a simple filtering interface without too many fancy stuff. The fancy stuff could be done on the command line if needed...
+"""]]
diff --git a/doc/todo/wishlist__58___archive_from_remote_with_the_least_free_space.mdwn b/doc/todo/wishlist__58___archive_from_remote_with_the_least_free_space.mdwn
new file mode 100644
index 0000000000..acc8b363e2
--- /dev/null
+++ b/doc/todo/wishlist__58___archive_from_remote_with_the_least_free_space.mdwn
@@ -0,0 +1 @@
+An interesting feature, when an archived file cannot be removed from all clients because of the minimum number of copies required, would be to remove it from the repositories with the smallest amount of free space available.
diff --git a/doc/todo/wishlist__58___archive_from_remote_with_the_least_free_space/comment_1_6813fdc7ecc98765a5d35d34163a1712._comment b/doc/todo/wishlist__58___archive_from_remote_with_the_least_free_space/comment_1_6813fdc7ecc98765a5d35d34163a1712._comment
new file mode 100644
index 0000000000..89fe4c0690
--- /dev/null
+++ b/doc/todo/wishlist__58___archive_from_remote_with_the_least_free_space/comment_1_6813fdc7ecc98765a5d35d34163a1712._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="http://joeyh.name/"
+ ip="4.153.14.105"
+ subject="comment 1"
+ date="2013-09-19T21:01:30Z"
+ content="""
+That might be useful in some cases. git-annex can only tell how much free space is available on remotes that are mounted to the local filesystem. Did you use case involve removable drives?
+"""]]
diff --git a/doc/todo/wishlist__58___archive_from_remote_with_the_least_free_space/comment_2_21a249cedca1ceb80d10784004735524._comment b/doc/todo/wishlist__58___archive_from_remote_with_the_least_free_space/comment_2_21a249cedca1ceb80d10784004735524._comment
new file mode 100644
index 0000000000..d2b29c2396
--- /dev/null
+++ b/doc/todo/wishlist__58___archive_from_remote_with_the_least_free_space/comment_2_21a249cedca1ceb80d10784004735524._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="https://www.google.com/accounts/o8/id?id=AItOawmOsy6nbvPyXLd--qqjPMLnVIzxgZwtKlQ"
+ nickname="Nicolas"
+ subject="comment 2"
+ date="2013-09-20T19:03:13Z"
+ content="""
+I was not thinking of removable drives, but only of \"client\" repositories. Ideally, git-annex would query the remaining space of all connected client repositories to choose on which repositories to drop a copy.
+"""]]
diff --git a/doc/todo/wishlist__58___assistant_logging_improvements.mdwn b/doc/todo/wishlist__58___assistant_logging_improvements.mdwn
new file mode 100644
index 0000000000..bbc3588012
--- /dev/null
+++ b/doc/todo/wishlist__58___assistant_logging_improvements.mdwn
@@ -0,0 +1,3 @@
+1. Timestamps in daemon.log would be useful.
+
+2. An easy way to get annex/assistant to periodically mail daemon.log (or even better, only the new additions to daemon.log) to the local user would be great. This would seem to me like a good use of git-annex-schedule, which at the moment only does fsck, or it could be an optional part of log rotation. With this, when running annex on a remote server, one could easily get periodic confirmations that everything's working out as it should, that fscks are not failing, etc.
diff --git a/doc/todo/wishlist__58___assistant_logging_improvements/comment_1_683e2e3c94345cb6f35028cd130a4dcc._comment b/doc/todo/wishlist__58___assistant_logging_improvements/comment_1_683e2e3c94345cb6f35028cd130a4dcc._comment
new file mode 100644
index 0000000000..80075d33c3
--- /dev/null
+++ b/doc/todo/wishlist__58___assistant_logging_improvements/comment_1_683e2e3c94345cb6f35028cd130a4dcc._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="mhauru"
+ avatar="http://cdn.libravatar.org/avatar/2532433a0207ba772e6ca964e61899c0"
+ subject="comment 1"
+ date="2016-12-29T18:00:10Z"
+ content="""
+Just noticed that 1. has already been wished for here: https://git-annex.branchable.com/todo/wishlist__91__minor__93____58___add_time_stamps_to_annex_log_popups_in_webapp/
+"""]]
diff --git a/doc/todo/wishlist__58___derived_content_support.mdwn b/doc/todo/wishlist__58___derived_content_support.mdwn
new file mode 100644
index 0000000000..19556a13ec
--- /dev/null
+++ b/doc/todo/wishlist__58___derived_content_support.mdwn
@@ -0,0 +1,8 @@
+I handle some video and music files with git annex and it would be awesome if git annex supported tracking of relations between files.
+
+Example:
+I have music in flac on my htpc, but i think flac is way overkill to have on my Android device, so i transcode them to ogg/mp3 on my htpc and then sync them to my Android device.
+
+Transcoding is a good example for this type of feature, but there might be other uses too.
+
+The basic thing is to know a file is a "copy" of another file, but its a generated result, so the actual file is not important, but the source is.
diff --git a/doc/todo/wishlist__58___disable_automatic_commits.mdwn b/doc/todo/wishlist__58___disable_automatic_commits.mdwn
new file mode 100644
index 0000000000..03ed40211a
--- /dev/null
+++ b/doc/todo/wishlist__58___disable_automatic_commits.mdwn
@@ -0,0 +1,36 @@
+When using the [[/assistant]] on some of my repositories, I would like to
+retain manual control over the granularity and contents of the commit
+history.  Some motivating reasons:
+
+* manually specified commit messages makes the history easier to follow
+* make a series of minor changes to a file over a period of a few hours would result in a single commit rather than capturing intermediate incomplete edits
+
+* manual choice of which files to annex (based on predicted usage) could be useful, e.g. a repo might contain a 4MB PDF which you want available in *every* remote even without `git annex get`, and also some 2MB images which are only required in some remotes
+
+> This particular case is now catered to by the ["manual" repository group](/preferred_content/standard_groups/)
+> in [[preferred content]] settings. --[[Joey]]
+
+Obviously this needs to be configurable at least per repository, and
+ideally perhaps even per remote, since usage habits can vary from machine
+to machine (e.g. I could choose to commit manually from my desktop machine
+which has a nice comfy keyboard and large screen, but this would be too
+much pain to do from my tiny netbook).
+
+In fact, this is vaguely related to [[design/assistant/partial_content]],
+since the usefulness of the commit history depends on the context of the
+data being manipulated, which in turn depends on which subdirectories are
+being touched.  So any mechanism for disabling sync per directory could
+potentially be reused for disabling auto-commit per directory.
+
+According to Joey, it should be easy to arrange for the watcher thread not
+to run, but would need some more work for the assistant to notice manual
+commits in order to sync them; however the assistant already does some
+crazy inotify watching of git refs, in order to detect incoming pushes, so
+detecting manual commits wouldn't be a stretch.
+
+[[!tag design/assistant]]
+
+> You can do this now by pausing committing via the webapp,
+> or setting `annex.autocommit=false`.
+> 
+> The assistant probably doesn't push such commits yet.
diff --git a/doc/todo/wishlist__58___display_status_of_remotes_in_the_webapp.mdwn b/doc/todo/wishlist__58___display_status_of_remotes_in_the_webapp.mdwn
new file mode 100644
index 0000000000..7414669944
--- /dev/null
+++ b/doc/todo/wishlist__58___display_status_of_remotes_in_the_webapp.mdwn
@@ -0,0 +1 @@
+It would be nice to have an indication of the status of the remotes in the webapp, for example with a field showing "In Sync", "Syncing", or the date of the last successful synchronization for unreachable remotes.
diff --git a/doc/todo/wishlist__58___do_not_import_new_files.mdwn b/doc/todo/wishlist__58___do_not_import_new_files.mdwn
new file mode 100644
index 0000000000..6a0225f38b
--- /dev/null
+++ b/doc/todo/wishlist__58___do_not_import_new_files.mdwn
@@ -0,0 +1,10 @@
+[[!meta title="mass reinject of any known content from a directory"]]
+
+Right now `git annex import DIR/*` will import all the files in DIR, both those that are already known to git-annex and those that are new. Using the option `--skip-duplicates` one can import only new files that are in DIR but unknown to git-annex.
+
+It would be nice if there were an opposite `--only-duplicates` option that could be used to import only the files that are already known to git, ignoring the new files in DIR.
+
+PS: it would also be nice to have aliases like `--only-new-files` and `--skip-new-files` for `--skip-duplicates` and `--only-duplicates`.
+
+> I think that either `git annex reinject --known` or `git annex import
+> --reinject-known` can handle this use case. So, [[done]] --[[Joey]]
diff --git a/doc/todo/wishlist__58___do_not_import_new_files/comment_1_b41c214599d6601257a9d824cebbffcc._comment b/doc/todo/wishlist__58___do_not_import_new_files/comment_1_b41c214599d6601257a9d824cebbffcc._comment
new file mode 100644
index 0000000000..fbb5571cb8
--- /dev/null
+++ b/doc/todo/wishlist__58___do_not_import_new_files/comment_1_b41c214599d6601257a9d824cebbffcc._comment
@@ -0,0 +1,14 @@
+[[!comment format=mdwn
+ username="http://ypid.wordpress.com/"
+ ip="213.153.84.215"
+ subject="gitignore"
+ date="2014-07-12T17:59:42Z"
+ content="""
+Hi
+
+The gitignore file is probably what you are looking for which is also honored by git-annex. Some documentation:
+
+* [On git-scm](http://git-scm.com/docs/gitignore)
+* [On gitready](http://de.gitready.com/beginner/2009/01/19/ignoring-files.html)
+* [On github](https://help.github.com/articles/ignoring-files)
+"""]]
diff --git a/doc/todo/wishlist__58___do_not_import_new_files/comment_2_7b26171458baaf5c0057276d2d97e14c._comment b/doc/todo/wishlist__58___do_not_import_new_files/comment_2_7b26171458baaf5c0057276d2d97e14c._comment
new file mode 100644
index 0000000000..bedf9a54cd
--- /dev/null
+++ b/doc/todo/wishlist__58___do_not_import_new_files/comment_2_7b26171458baaf5c0057276d2d97e14c._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="http://joeyh.name/"
+ ip="209.250.56.2"
+ subject="comment 2"
+ date="2014-07-14T18:30:38Z"
+ content="""
+You can use --clean-duplicates unless your goal is for some reason to add the duplicate files to your repository a second time.
+"""]]
diff --git a/doc/todo/wishlist__58___do_not_import_new_files/comment_3_6f80ce6cee4519d4f69193d5086e194a._comment b/doc/todo/wishlist__58___do_not_import_new_files/comment_3_6f80ce6cee4519d4f69193d5086e194a._comment
new file mode 100644
index 0000000000..e66cc5ea9c
--- /dev/null
+++ b/doc/todo/wishlist__58___do_not_import_new_files/comment_3_6f80ce6cee4519d4f69193d5086e194a._comment
@@ -0,0 +1,20 @@
+[[!comment format=mdwn
+ username="http://svario.it/gioele"
+ nickname="gioele"
+ subject="comment 3"
+ date="2014-07-15T06:54:40Z"
+ content="""
+>  You can use --clean-duplicates unless your goal is for some reason to add the duplicate files to your repository a second time.
+
+My use case is that I clone an existing remote on a PC where there are already some of the annexed files (+ others).
+
+My workflow would be:
+
+* clone git-annex server:~/Documents.git
+* git annex init \"other pc\"
+* git annex import --skip-new (or --only-duplicates) ~/Dump/*
+
+~/Dump contains many other files in addition to those found in the Documents repository.
+
+In this case --clean-duplicate would not be the correct solution.
+"""]]
diff --git a/doc/todo/wishlist__58___do_not_import_new_files/comment_4_22a7a03c30174e42e6d8e639e31e1d34._comment b/doc/todo/wishlist__58___do_not_import_new_files/comment_4_22a7a03c30174e42e6d8e639e31e1d34._comment
new file mode 100644
index 0000000000..f53fb6395c
--- /dev/null
+++ b/doc/todo/wishlist__58___do_not_import_new_files/comment_4_22a7a03c30174e42e6d8e639e31e1d34._comment
@@ -0,0 +1,12 @@
+[[!comment format=mdwn
+ username="http://joeyh.name/"
+ ip="209.250.56.2"
+ subject="comment 4"
+ date="2014-07-15T19:01:24Z"
+ content="""
+So the goal is to inject any known objects from the dump into the local annex to avoid needing to re-transfer them.
+
+It seems to me that in this case, you would not even want to create new symlinks in the git repository.
+
+`git annex reinject` might be a better place to put code to handle this than `git annex import`.
+"""]]
diff --git a/doc/todo/wishlist__58___do_not_import_new_files/comment_5_4294e92e2f4efb9dd10b280f5c9843f7._comment b/doc/todo/wishlist__58___do_not_import_new_files/comment_5_4294e92e2f4efb9dd10b280f5c9843f7._comment
new file mode 100644
index 0000000000..e312c083d2
--- /dev/null
+++ b/doc/todo/wishlist__58___do_not_import_new_files/comment_5_4294e92e2f4efb9dd10b280f5c9843f7._comment
@@ -0,0 +1,10 @@
+[[!comment format=mdwn
+ username="http://joeyh.name/"
+ ip="209.250.56.2"
+ subject="comment 5"
+ date="2014-07-15T19:13:05Z"
+ content="""
+A fundamental problem with this idea is that git-annex's keys can use any of many checksumming backends. So, which checksum should it try? Running every possible checksum on a file is going to re-read it repeatedly and be expensive.
+
+`git annex import` avoids this problem by using whatever the default backend is configured to be for the filename it's importing. This is good enough to make repeated runs of `git annex import` work ok, but when we get into trying to reinject whole directory trees like this, I don't think that's good enough.
+"""]]
diff --git a/doc/todo/wishlist__58___encrypted_git_remote_on_hosting_site_from_webapp.mdwn b/doc/todo/wishlist__58___encrypted_git_remote_on_hosting_site_from_webapp.mdwn
new file mode 100644
index 0000000000..b7b1bad0c9
--- /dev/null
+++ b/doc/todo/wishlist__58___encrypted_git_remote_on_hosting_site_from_webapp.mdwn
@@ -0,0 +1 @@
+It would be great to be able to do **private encrypted git remote on hosting site** and **multiuser encrypted git remote on hosting site** as explained in [[tips/fully encrypted git repositories with gcrypt]] through the webapp. I think it's a pretty common usecase that can be very useful for people not owning a proper server.
diff --git a/doc/todo/wishlist__58___generic_annex.cost-command.mdwn b/doc/todo/wishlist__58___generic_annex.cost-command.mdwn
new file mode 100644
index 0000000000..6adf1460e8
--- /dev/null
+++ b/doc/todo/wishlist__58___generic_annex.cost-command.mdwn
@@ -0,0 +1,17 @@
+### Current setup
+
+ATM git-annex has
+
+remote..annex-cost
+remote..annex-cost-command  # command is not provided cmdline options by annex
+
+to set the cost for a given remote.  That requires setting up one of those variables per each host, and possibly hardcoding options for the annex-cost-command providing e.g. the remote name.
+
+### Suggestion
+
+wouldn't it be more general and thus more flexible to have a repository-wide
+
+annex.cost-command
+
+which could take options %remote, %file and assessed accordingly per each file upon '--get' request to allow maximal flexibility: e.g. some files might better be fetched from remotes supporting transfer compression, some from the web, etc.  Also it might be worth providing %remote_kind ("special" vs "git") to disambiguate %remote's?
+
diff --git a/doc/todo/wishlist__58___global_progress_status.mdwn b/doc/todo/wishlist__58___global_progress_status.mdwn
new file mode 100644
index 0000000000..3c6611af4f
--- /dev/null
+++ b/doc/todo/wishlist__58___global_progress_status.mdwn
@@ -0,0 +1,3 @@
+similar to [[do_not_bug_me_about_intermediate_files]] - i feel that massive `git annex get` operations should have better progress information than the current individual `rsync --progress` bits. i wonder if this couldn't be accomplished with `rsync --info=PROGRESS2`, which gives overall rsync progress, combined with copying multiple files at once with rsync (which would have the side-effect of speeding up `git annex get` for large number of small files).
+
+once this is done, it could be sent back to the webapp UI to give the user a global sense of the overall sync progress (as opposed to per-file progress). --[[anarcat]]
diff --git a/doc/todo/wishlist__58___global_progress_status/comment_1_d915dce144071f768ee9bcdead07a0e5._comment b/doc/todo/wishlist__58___global_progress_status/comment_1_d915dce144071f768ee9bcdead07a0e5._comment
new file mode 100644
index 0000000000..8a733f90bc
--- /dev/null
+++ b/doc/todo/wishlist__58___global_progress_status/comment_1_d915dce144071f768ee9bcdead07a0e5._comment
@@ -0,0 +1,9 @@
+[[!comment format=mdwn
+ username="joey"
+ subject="""comment 1"""
+ date="2015-04-03T15:23:48Z"
+ content="""
+To display global progress, git-annex would have to make 2
+passes over all the files to be processed. 
+That is the main reason it does not try to do that.
+"""]]
diff --git a/doc/todo/wishlist__58___global_progress_status/comment_2_816f41f5436543fc09e7d7df82a8c8d5._comment b/doc/todo/wishlist__58___global_progress_status/comment_2_816f41f5436543fc09e7d7df82a8c8d5._comment
new file mode 100644
index 0000000000..a75aea4c23
--- /dev/null
+++ b/doc/todo/wishlist__58___global_progress_status/comment_2_816f41f5436543fc09e7d7df82a8c8d5._comment
@@ -0,0 +1,9 @@
+[[!comment format=mdwn
+ username="atrent"
+ subject="adding a simple counter?"
+ date="2016-02-03T07:48:10Z"
+ content="""
+could you add a simple file counter?
+
+i.e. for a single \"git annex copy/get/etc.\" operation initiating a file counter and incrementing it on every examined file (transferred or not), thus giving a very rough idea of the progress on the whole set (the user should know the amount of files in the annex)
+"""]]
diff --git a/doc/todo/wishlist__58___global_progress_status/comment_3_587e8befa0e52d8880362a32799df492._comment b/doc/todo/wishlist__58___global_progress_status/comment_3_587e8befa0e52d8880362a32799df492._comment
new file mode 100644
index 0000000000..37effc4f9a
--- /dev/null
+++ b/doc/todo/wishlist__58___global_progress_status/comment_3_587e8befa0e52d8880362a32799df492._comment
@@ -0,0 +1,14 @@
+[[!comment format=mdwn
+ username="joey"
+ subject="""comment 3"""
+ date="2018-03-12T21:38:17Z"
+ content="""
+git-annex does not display rsync progress any longer, but you do still 
+get the progress display on a per-file basis. This is at least a lot more
+compact than the rsync output.
+
+Any kind of global progress display would require a separate pass to
+identify all the files that git-annex will be operating on. That would make
+it slower in large repos, and people already complain about seek speed in
+large repos.
+"""]]
diff --git a/doc/todo/wishlist__58___history_of_operations.mdwn b/doc/todo/wishlist__58___history_of_operations.mdwn
new file mode 100644
index 0000000000..a79f500ff6
--- /dev/null
+++ b/doc/todo/wishlist__58___history_of_operations.mdwn
@@ -0,0 +1,8 @@
+Hi,
+
+I would love to have a page of "history" or "events" in the webapp. Similar to how Dropbox or Box show it.
+I've been using git-annex for my personal files for a few months now, and I feel like this is the only feature missing to start using it in a production multi-user environment.
+
+Thanks
+
+[[!tag design/assistant]]
diff --git a/doc/todo/wishlist__58___history_of_operations/comment_1_f9a77ce83c6f39b6272d5c577ffbb9f9._comment b/doc/todo/wishlist__58___history_of_operations/comment_1_f9a77ce83c6f39b6272d5c577ffbb9f9._comment
new file mode 100644
index 0000000000..bb872532d7
--- /dev/null
+++ b/doc/todo/wishlist__58___history_of_operations/comment_1_f9a77ce83c6f39b6272d5c577ffbb9f9._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="https://www.google.com/accounts/o8/id?id=AItOawlJEI45rGczFAnuM7gRSj4C6s9AS9yPZDc"
+ nickname="Kevin"
+ subject="I second this wishlist item"
+ date="2013-07-24T16:48:59Z"
+ content="""
+I too would like to see a recent history of events in the assistant webapp.  A simple listing of what changed when and by whom would be sufficient.
+"""]]
diff --git a/doc/todo/wishlist__58___make_partial_files_available_during_transfer.mdwn b/doc/todo/wishlist__58___make_partial_files_available_during_transfer.mdwn
new file mode 100644
index 0000000000..b021c90914
--- /dev/null
+++ b/doc/todo/wishlist__58___make_partial_files_available_during_transfer.mdwn
@@ -0,0 +1,18 @@
+Imagine this situation:
+You have a laptop and a NAS.
+On your laptop you want to consume a large media file located on the NAS.
+So you type:
+
+    git annex get --from nas mediafile
+
+But now you have to wait for the download to complete, unless either 
+
+* rsync is pointed directly to the file in the object storage ("--inplace")
+or
+* the symlink temporarily points to the partial file during a transfer
+
+which would allow you instantaneous consumption of your media.
+It might make sense to make this behavior configurable, because not everyone might agree with having partial content (that mismatches its key) around.
+
+
+So what do you say?
diff --git a/doc/todo/wishlist__58___make_partial_files_available_during_transfer/comment_2_8b1cfae6f2b61929a9c6f48ae63c921d._comment b/doc/todo/wishlist__58___make_partial_files_available_during_transfer/comment_2_8b1cfae6f2b61929a9c6f48ae63c921d._comment
new file mode 100644
index 0000000000..c4c2224319
--- /dev/null
+++ b/doc/todo/wishlist__58___make_partial_files_available_during_transfer/comment_2_8b1cfae6f2b61929a9c6f48ae63c921d._comment
@@ -0,0 +1,12 @@
+[[!comment format=mdwn
+ username="http://joeyh.name/"
+ ip="209.250.56.154"
+ subject="comment 2"
+ date="2014-03-18T20:08:13Z"
+ content="""
+There's now an easy way to do this:
+
+        git annex find --include=* --format='.git/annex/tmp/${hashdirmixed}${key}/${key}\n' 
+
+Pass it the file or files you're interested in to get their partially transferred contents.
+"""]]
diff --git a/doc/todo/wishlist__58___make_partial_files_available_during_transfer/comment_3_1304a721da6f5133fdfa1dac507f1ecb._comment b/doc/todo/wishlist__58___make_partial_files_available_during_transfer/comment_3_1304a721da6f5133fdfa1dac507f1ecb._comment
new file mode 100644
index 0000000000..2907dafc3a
--- /dev/null
+++ b/doc/todo/wishlist__58___make_partial_files_available_during_transfer/comment_3_1304a721da6f5133fdfa1dac507f1ecb._comment
@@ -0,0 +1,10 @@
+[[!comment format=mdwn
+ username="http://joeyh.name/"
+ ip="4.152.108.194"
+ subject="comment 3"
+ date="2012-11-04T19:58:28Z"
+ content="""
+I'm not at all comfortable with either idea. Temporarily repointing the symlink could lead to accidentally git committing a bad symlink. Or the user accidentally doing something with a partially transferred file. Running rsync in place would break lots of things that assume that, once the file is present, it can be assumed to be the full and correct file. (Obviously fsck doesn't assume that, but checks made by `git annex drop` do, for example.)
+
+However, you can access partially transferred files by key in `.git/annex/tmp`. It would be easy to write some hack that looks at the symlink to get the key, and then spits out the name of the partial file in `.git/annex/tmp.`.
+"""]]
diff --git a/doc/todo/wishlist__58___matching_options_for_branches.mdwn b/doc/todo/wishlist__58___matching_options_for_branches.mdwn
new file mode 100644
index 0000000000..03086e0f92
--- /dev/null
+++ b/doc/todo/wishlist__58___matching_options_for_branches.mdwn
@@ -0,0 +1 @@
+I have a repository layout where I have multiple unrelated branches in a single repository. Different branches have different policies regarding where their content should be replicated. It would be nice to be able to reflect this with git-annex's matching and preferred content options, but currently there seems to be no way to say "include/exclude files referred to in these branches".
diff --git a/doc/todo/wishlist__58___matching_options_for_branches/comment_1_c0d442b646dc40d870b10098a0ee5408._comment b/doc/todo/wishlist__58___matching_options_for_branches/comment_1_c0d442b646dc40d870b10098a0ee5408._comment
new file mode 100644
index 0000000000..d606d80e11
--- /dev/null
+++ b/doc/todo/wishlist__58___matching_options_for_branches/comment_1_c0d442b646dc40d870b10098a0ee5408._comment
@@ -0,0 +1,20 @@
+[[!comment format=mdwn
+ username="joey"
+ subject="""comment 1"""
+ date="2015-09-09T18:34:29Z"
+ content="""
+It's hard to add this kind of thing to the preferred content expressions.
+
+First, it would be quite slow to check if a given key
+was present in some file in a given branch. There's no index for that
+information.
+
+Second, even if that were implemented, there's no guarantee at all that
+different repositories have the same git branches. If repo Foo wants files in
+branch B, and repo Bar sees that, and has no branch B, or a different
+version of branch B, how is it supposed to know which files to send to
+repo Foo?
+
+So, I can see the use case, but I don't see how preferred content can
+support it.
+"""]]
diff --git a/doc/todo/wishlist__58___more_info_in_the_standard_commit_message_of___96__sync__96__.mdwn b/doc/todo/wishlist__58___more_info_in_the_standard_commit_message_of___96__sync__96__.mdwn
new file mode 100644
index 0000000000..4e5e3a171b
--- /dev/null
+++ b/doc/todo/wishlist__58___more_info_in_the_standard_commit_message_of___96__sync__96__.mdwn
@@ -0,0 +1,3 @@
+Could you include the REPO and UUID information in the "automatic sync" commit message?
+
+This would make troubleshooting easier. (not there was much trouble with git-annex!)
diff --git a/doc/todo/wishlist__58___more_info_in_the_standard_commit_message_of___96__sync__96__/comment_1_b9c241cf94a35aa6a45f4d44334694b0._comment b/doc/todo/wishlist__58___more_info_in_the_standard_commit_message_of___96__sync__96__/comment_1_b9c241cf94a35aa6a45f4d44334694b0._comment
new file mode 100644
index 0000000000..bede77efcd
--- /dev/null
+++ b/doc/todo/wishlist__58___more_info_in_the_standard_commit_message_of___96__sync__96__/comment_1_b9c241cf94a35aa6a45f4d44334694b0._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="http://olivier.mehani.name/"
+ nickname="olivier-mehani"
+ subject="comment 1"
+ date="2014-01-15T02:26:31Z"
+ content="""
+See also [[wishlist: more info in commit messages in general]] for a similar but more generic wish.
+"""]]
diff --git a/doc/todo/wishlist__58___move_pending_transfers_for_a_host_to_the_end_of_the_queue_when_one_fails.mdwn b/doc/todo/wishlist__58___move_pending_transfers_for_a_host_to_the_end_of_the_queue_when_one_fails.mdwn
new file mode 100644
index 0000000000..e50ebbde5e
--- /dev/null
+++ b/doc/todo/wishlist__58___move_pending_transfers_for_a_host_to_the_end_of_the_queue_when_one_fails.mdwn
@@ -0,0 +1,7 @@
+Right now the assistant can have a huge list of pending transfers for certain hosts if its data is a bit outdated, or a host hasn't been synced lately. When starting up it will then attempt each transfer to said host (which will in turn fail, but at times take time to time out), possibly before doing other stuff like attempting to download new files, or copy files to online hosts.
+
+I suggest that if a transfer fails for host X, and there are other pending transfers, say to host Y and from Z, then all other pending transfers to/from X gets pushed to the back of the queue, to avoid having to wait a long time for several transfers to time out before doing useful stuff.
+
+The prime example for me was this morning, when a laptop that was turned off had a huge amount of queued transfers to it, resulting in the assistant attempting a load of transfers to that host before it retrieved a new file that I had created on another machine yesterday.
+
+[[!tag design/assistant]]
diff --git a/doc/todo/wishlist__58___move_pending_transfers_for_a_host_to_the_end_of_the_queue_when_one_fails/comment_1_82ee9de610a0ac55cd1c27c211079e5b._comment b/doc/todo/wishlist__58___move_pending_transfers_for_a_host_to_the_end_of_the_queue_when_one_fails/comment_1_82ee9de610a0ac55cd1c27c211079e5b._comment
new file mode 100644
index 0000000000..3b6101ed58
--- /dev/null
+++ b/doc/todo/wishlist__58___move_pending_transfers_for_a_host_to_the_end_of_the_queue_when_one_fails/comment_1_82ee9de610a0ac55cd1c27c211079e5b._comment
@@ -0,0 +1,10 @@
+[[!comment format=mdwn
+ username="http://joeyh.name/"
+ ip="4.154.6.49"
+ subject="comment 1"
+ date="2012-12-01T19:22:03Z"
+ content="""
+There are several difficulties with reordering the queue that way. One is that the failure may be intermittent; another is that the queue is fed by a scanning process, so doesn't always have a well-defined end.
+
+Another way to deal with this problem, which I think I prefer, is to allow multiple actions from the queue to run at once. Then slow or unreachable remotes don't block it from using other remotes. 
+"""]]
diff --git a/doc/todo/wishlist__58___move_pending_transfers_for_a_host_to_the_end_of_the_queue_when_one_fails/comment_2_bea55156bd32cf9e6dd9b946ba1bb8c1._comment b/doc/todo/wishlist__58___move_pending_transfers_for_a_host_to_the_end_of_the_queue_when_one_fails/comment_2_bea55156bd32cf9e6dd9b946ba1bb8c1._comment
new file mode 100644
index 0000000000..62d46bccd0
--- /dev/null
+++ b/doc/todo/wishlist__58___move_pending_transfers_for_a_host_to_the_end_of_the_queue_when_one_fails/comment_2_bea55156bd32cf9e6dd9b946ba1bb8c1._comment
@@ -0,0 +1,10 @@
+[[!comment format=mdwn
+ username="EskildHustvedt"
+ ip="84.48.83.221"
+ subject="comment 2"
+ date="2012-12-01T19:31:18Z"
+ content="""
+I agree your method might be preferable, the end result is the same, and would have avoided the issues I had (and, of course, running multiple transfers at once has other benefits as well).
+
+An alternate way would be to push every transfer NOT from host X to the front of the queue (avoiding most of the \"no defined end\" issue and largely solving the problem), but if multiple actions at once is feasible then that'd still be much nicer.
+"""]]
diff --git a/doc/todo/wishlist__58___option_to_print_more_info_with___39__unused__39__.mdwn b/doc/todo/wishlist__58___option_to_print_more_info_with___39__unused__39__.mdwn
new file mode 100644
index 0000000000..7a9b81f72b
--- /dev/null
+++ b/doc/todo/wishlist__58___option_to_print_more_info_with___39__unused__39__.mdwn
@@ -0,0 +1,37 @@
+It would be nice if the 'unused' command could optionally display info about the actual files behind its cryptic keys.
+
+I created a (very rough) bash script that simply splices in some info from git log -S'KEY' --numstat into the unused list, like so:
+
+    arand@mas:~/annex(master)$ bash ~/utv/scripts/annex-vunused 
+    unused . (checking for unused data...) (checking master...) (checking synced/master...) (checking origin/HEAD...) (checking seagate/master...)
+    Some annexed data is no longer used by any files:
+    NUMBER  KEY
+    1       SHA256E-s1073741824--49bc20df15e412a64472421e13fe86ff1c5165e18b2afccf160d4dc19fe68a14.img
+                    8f479a4 Sat Feb 23 16:14:12 2013 +0100 remove bigfile
+                    0       1       dummy_bigfile.img
+                    2988d18 Sat Feb 23 16:13:48 2013 +0100 dummy file
+                    1       0       dummy_bigfile.img
+    (To see where data was previously used, try: git log --stat -S'KEY')To remove unwanted data: git-annex dropunused NUMBER
+    ok
+The script:
+
+    #!/bin/bash
+    
+    pipe="$(mktemp -u)"
+    mkfifo "$pipe"
+    
+    git annex unused  >"$pipe" || exit 1 &
+    
+    while read -r line
+    do
+    	key="$(echo "$line" | sed 's/^[^-]*-\([^-]*\)-.*/\1/')"
+    	echo -n "$line"
+    	test -n "$key" && \
+    		echo && \
+    		git log --format="%h %cd %s" --numstat -S"$key" | \
+    			sed '/^$/d;/git-annex automatic sync/,/^ /d;s/^/\t\t/'
+    
+    done < "$pipe"
+    rm "$pipe"
+
+It would be nice if something like this was available as an option, since it's good way to get a quick overview of what the content is, and if it's safe to drop it.
diff --git a/doc/todo/wishlist__58___optionally_print_key_in_output_of_checkpresentkey_batch_mode.mdwn b/doc/todo/wishlist__58___optionally_print_key_in_output_of_checkpresentkey_batch_mode.mdwn
new file mode 100644
index 0000000000..b1e63f1251
--- /dev/null
+++ b/doc/todo/wishlist__58___optionally_print_key_in_output_of_checkpresentkey_batch_mode.mdwn
@@ -0,0 +1,3 @@
+For batch mode of `checkpresentkey`, it could be useful to have the option of outputting the key in a second (space- or tab-delimited?) column in addition to the `0/1` status indicator for that key; this could be useful for reporting, error recovery, or further key-specific action based on output. Presently it seems the only way to accomplish this would be to independently track corresponding input and output line indices, or externally join input and output (e.g., via the *nix `paste` command). 
+
+Thank you!
diff --git a/doc/todo/wishlist__58___optionally_print_key_in_output_of_checkpresentkey_batch_mode/comment_1_f95f2c92a46c1bd00c627aa19b08c875._comment b/doc/todo/wishlist__58___optionally_print_key_in_output_of_checkpresentkey_batch_mode/comment_1_f95f2c92a46c1bd00c627aa19b08c875._comment
new file mode 100644
index 0000000000..db469b88e2
--- /dev/null
+++ b/doc/todo/wishlist__58___optionally_print_key_in_output_of_checkpresentkey_batch_mode/comment_1_f95f2c92a46c1bd00c627aa19b08c875._comment
@@ -0,0 +1,13 @@
+[[!comment format=mdwn
+ username="joey"
+ subject="""comment 1"""
+ date="2016-07-11T16:36:11Z"
+ content="""
+Hmm, my assumption is that use of checkpresentkey --batch is in some kind
+of loop, or function call, where you'd write a key to it, read a response
+back, and now you know whether that key is present.
+
+Does your use case involve something like shell scripting? I suppose if you
+have a big list of keys and want to narrow it to only keys that are present
+or missing, you could be trying to pipe it through checkpresentkey --batch.
+"""]]
diff --git a/doc/todo/wishlist__58___optionally_print_key_in_output_of_checkpresentkey_batch_mode/comment_2_bb756ddf06c52ca52d73af319f0265b0._comment b/doc/todo/wishlist__58___optionally_print_key_in_output_of_checkpresentkey_batch_mode/comment_2_bb756ddf06c52ca52d73af319f0265b0._comment
new file mode 100644
index 0000000000..b871a772d0
--- /dev/null
+++ b/doc/todo/wishlist__58___optionally_print_key_in_output_of_checkpresentkey_batch_mode/comment_2_bb756ddf06c52ca52d73af319f0265b0._comment
@@ -0,0 +1,14 @@
+[[!comment format=mdwn
+ username="magibney@908c3d4677b9e87e203538d4d5e8c296255749a0"
+ nickname="magibney"
+ subject="comment 2"
+ date="2016-07-11T18:45:10Z"
+ content="""
+Exactly: shell scripting, \"big list of keys and want to narrow it to only keys that are present or missing\". 
+
+My motivating use case is as a sanity check that all keys have been successfully transferred on a Ceph S3 instance internally (i.e., `s3cmd cp s3://sourcebucket/[annex-key] s3://destbucket/[annex-key]`, without being read back to the local machine and separately written back up to a different bucket on the same server and locally deleted).  Post-transfer, I want to check that all keys are present as expected, but in the event that a failed transfer is detected, I'd like to be able to report a specific key back to the operator/calling script. 
+
+I'm thinking that if one is running `checkpresentkey` in batch mode (with multiple keys at once), there are many useful things one might want to do with the output that would require associating output status indicators with input keys; and since the output of `checkpresentkey --batch` is so terse, it could optionally be extended without too much parsing/escaping/etc complication. 
+
+Thanks for considering this, and apologies that the initial request was a bit cryptic about the use case! 
+"""]]
diff --git a/doc/todo/wishlist__58___pack_metadata_in_direct_mode.mdwn b/doc/todo/wishlist__58___pack_metadata_in_direct_mode.mdwn
new file mode 100644
index 0000000000..1d52ff5121
--- /dev/null
+++ b/doc/todo/wishlist__58___pack_metadata_in_direct_mode.mdwn
@@ -0,0 +1,5 @@
+The metadata storage for direct mode (V3) is this. In directory .git/annex/objects, there is one .map for all annexed file, and one .cache for all files in the working tree. Both are small files, containing only 1 line or a few lines. I have a repo with lots of photos, and this created lots of small files. I believe this will cause many performance issues. 
+
+It would be great if these files are packed, maybe also in the git pack files format.
+
+[[!meta tag=deprecateddirectmode]]
diff --git a/doc/todo/wishlist__58___pack_metadata_in_direct_mode/comment_1_1a550d6977a255b789337c3d1602db04._comment b/doc/todo/wishlist__58___pack_metadata_in_direct_mode/comment_1_1a550d6977a255b789337c3d1602db04._comment
new file mode 100644
index 0000000000..543e1c048b
--- /dev/null
+++ b/doc/todo/wishlist__58___pack_metadata_in_direct_mode/comment_1_1a550d6977a255b789337c3d1602db04._comment
@@ -0,0 +1,10 @@
+[[!comment format=mdwn
+ username="http://joeyh.name/"
+ ip="209.250.56.35"
+ subject="comment 1"
+ date="2014-01-04T19:18:16Z"
+ content="""
+\"I believe this will cause many performance issues.\"
+
+Such as?
+"""]]
diff --git a/doc/todo/wishlist__58___pack_metadata_in_direct_mode/comment_2_3cc9c69d33c658058daea9cb5e4ab669._comment b/doc/todo/wishlist__58___pack_metadata_in_direct_mode/comment_2_3cc9c69d33c658058daea9cb5e4ab669._comment
new file mode 100644
index 0000000000..5028636c40
--- /dev/null
+++ b/doc/todo/wishlist__58___pack_metadata_in_direct_mode/comment_2_3cc9c69d33c658058daea9cb5e4ab669._comment
@@ -0,0 +1,10 @@
+[[!comment format=mdwn
+ username="namelessjon"
+ ip="193.132.159.169"
+ subject="inode starvation"
+ date="2014-01-09T15:30:16Z"
+ content="""
+This happened to me a few times when creating new annex folders.
+
+I use lvm to create virtual partitions and have/had several 'bulk media' logical volumes which used ext4 with -T largefile4 (i.e. one inode per 4mb) because they're storing files with 20mb+ sizes (RAW images, downloaded screencasts, FLAC soundfiles, etc). Between the extra directories git annex creates, the extra files, and the .git/object directory, I ran out of inodes on a few occasions from the profusion of small files.  In some cases, I worked around this by shunting data around, or adding incrementally and then 'git gc'ing a lot to at least have a small .git/objects dir.  A packed metadata would help to deal with this.
+"""]]
diff --git a/doc/todo/wishlist__58___per-repository_autocommit__61__false.mdwn b/doc/todo/wishlist__58___per-repository_autocommit__61__false.mdwn
new file mode 100644
index 0000000000..40cdb0096f
--- /dev/null
+++ b/doc/todo/wishlist__58___per-repository_autocommit__61__false.mdwn
@@ -0,0 +1,7 @@
+when using git-annex as a minority participant in a repository (eg. because in a hardware project, at some point in time, photos get added), users will start to need to `git annex sync` (because a plain `git pull` / `git push` will work but show errors, and a `pull --all` / `annex merge` is difficult for users to remember). to stay in line with usual git usage, the `sync` must be used with `config annex.autocommit false`, but that needs to be configured on each repository.
+
+forgetting to do that explicit configuration results, in one sync command, easily results in an unwanted implicit commit that's pushed across remotes.
+
+could there be a per-repository option (somewhere around .gitattributes, or maybe in the git-annex branch) that disables autocommits for the repository?
+
+> [[done]] --[[Joey]]
diff --git a/doc/todo/wishlist__58___per-repository_autocommit__61__false/comment_1_851483817d97de212932203c2e830293._comment b/doc/todo/wishlist__58___per-repository_autocommit__61__false/comment_1_851483817d97de212932203c2e830293._comment
new file mode 100644
index 0000000000..7e5d850385
--- /dev/null
+++ b/doc/todo/wishlist__58___per-repository_autocommit__61__false/comment_1_851483817d97de212932203c2e830293._comment
@@ -0,0 +1,38 @@
+[[!comment format=mdwn
+ username="joey"
+ subject="""comment 1"""
+ date="2017-01-30T18:13:42Z"
+ content="""
+This could be put in the git-annex branch similarly to the `git annex
+numcopies` configuration so all clones see it.
+
+As well as --no-commit, --content is the other option that I think 
+might make sense to have a clone-wide setting for. sync's --no-pull and
+--no-push seem much less likely to need such a setting.
+
+I've been leaning toward eventually turning sync --content on by default,
+and such a clone-wide configuration would be useful to let users get back
+the current behavior.
+
+Hmm, this could be generalized all the way to having a file in the
+git-annex branch that stores default settings for `annex.*` configs.
+But then git-annex would have to pull that file out of the git-annex branch
+every time it's run, which would slow down commands that get run a lot in
+succession. So that's a generalization too far.
+
+Still, looking through the configs, I can see some other things
+that it would similarly make sense to sometimes want to set clone-wide,
+including: annex.genmetadata, annex.used-refspec, annex.diskreserve, 
+annex.addsmallfiles.
+
+So, we have some configs that are each only used by a few commands,
+that make sense to be allowed to be set clone-wide. We can make the file
+in the git-annex branch be only read by commands that look at those
+configs, and can consider when implementing each whether it would slow
+down any command too much to have it need to read the git-annex branch
+file.
+
+I've added a `git annex config` command that can set such clone-wide
+configurations. I have not yet made git annex sync look at it for
+annex.autocommit.
+"""]]
diff --git a/doc/todo/wishlist__58___per-repository_autocommit__61__false/comment_2_95e157d36c8ec91790cfd65037600332._comment b/doc/todo/wishlist__58___per-repository_autocommit__61__false/comment_2_95e157d36c8ec91790cfd65037600332._comment
new file mode 100644
index 0000000000..9455252cec
--- /dev/null
+++ b/doc/todo/wishlist__58___per-repository_autocommit__61__false/comment_2_95e157d36c8ec91790cfd65037600332._comment
@@ -0,0 +1,10 @@
+[[!comment format=mdwn
+ username="joey"
+ subject="""comment 2"""
+ date="2017-02-03T17:35:37Z"
+ content="""
+git annex config --set annex.autocommit false
+
+That works now, and will set the default autocommit behavior in all clones
+of the repository.
+"""]]
diff --git a/doc/todo/wishlist__58___perform_fsck_remotely.mdwn b/doc/todo/wishlist__58___perform_fsck_remotely.mdwn
new file mode 100644
index 0000000000..f2187d9122
--- /dev/null
+++ b/doc/todo/wishlist__58___perform_fsck_remotely.mdwn
@@ -0,0 +1,39 @@
+Currently, when `fsck`'ing a remote, files are first downloaded to a temporary 
+file locally, decrypted if needed, and finally digested; the temporary file is
+then either thrown away, or quarantined, depending on the value of that digest.
+
+Whereas this approach works with any kind of remote, in the particular case 
+where the user is granted execution rights on the digest command, one could
+avoid cluttering the network and digest the file remotely. I propose the
+addition of a per-remote git option `annex-remote-fsck` to switch between the
+two behaviors.
+
+
+There is an issue with encrypted specialremotes, though. As hinted at 
+[[here|tips/beware_of_SSD_wear_when_doing_fsck_on_large_special_remotes/#comment-70055f166f7eeca976021d24a736b471]],
+since the digest of a ciphertext can't be deduced from that of a plaintext in 
+general one would needs, before sending an encrypted file to such a remote, to
+digest it and store that digest somewhere (together with the cipher's size and
+perhaps other meta-information).
+
+The usual directory structure (`.../.../{backend}-s{size}--{digest}.log`) seems
+perfectly suitable to store these informations. Lines there would look like
+`{timestamp}s {numcopy} {UUID} {remote digest}`. Of course, it implies that
+remote digest commands are trustworthy (are doing the right thing), and that
+the digest output are not tampered by others who have access to the git repo.
+But that's outside the current threat model, I guess.
+
+Actually, since git-annex always includes a MDC in the ciphertexts, we could do
+something clever and even avoid running a digest algorithm. According to the
+[[OpenPGP standard|https://tools.ietf.org/html/rfc4880#section-5.14]] the MDC
+is essentially a SHA-1 hash of the plaintext. I'm still investigating if it's
+even possible, but in theory it would be enough (with non-chained ciphers at
+least) to download a few bytes from the encrypted remote, decrypt those bytes
+to retrieve the hash, and compare that hash with the known value. Of course
+there is a downside here, namely that files tampered anywhere but on the MDC
+packets would not be detected by `fsck` (but gpg will warn when decrypting the
+file).
+
+
+My 2 cents :-) Is there something I missed? I suppose there was a reason to 
+perform `fsck` locally at the first place...
diff --git a/doc/todo/wishlist__58___perform_fsck_remotely/comment_1_db92311dcdb1ef0ab0413f83e191c70c._comment b/doc/todo/wishlist__58___perform_fsck_remotely/comment_1_db92311dcdb1ef0ab0413f83e191c70c._comment
new file mode 100644
index 0000000000..6bf6af23ae
--- /dev/null
+++ b/doc/todo/wishlist__58___perform_fsck_remotely/comment_1_db92311dcdb1ef0ab0413f83e191c70c._comment
@@ -0,0 +1,15 @@
+[[!comment format=mdwn
+ username="http://joeyh.name/"
+ ip="4.154.0.63"
+ subject="comment 1"
+ date="2013-08-22T15:18:35Z"
+ content="""
+The only reason fsck is done locally for remotes is ease of implementation and it being a generic operation that supports any kind of special remote.
+
+Seems that the the only types of remotes where a remote fsck is a possibility are some rsync remotes and git remotes.
+git remotes already have git-annex installed, so the fsck could be run locally on the remote system using it.
+
+I don't know if I see a benefit with the MDC check. Any non-malicious data corruption on the remote is likely to affect the body of the file and not the small portion that holds the MDC. So checking the MDC does not seem much better than the current existence check done by `git annex fsck --fast --from remote`.
+
+As for storing the remote digest on the git-annex branch, my initial reaction was just that it's potentially a lot of bloat. Thinking about it some more, when using non-shared encryption, there is currently no way, given just a clone of a git repository, to match up files in git with encrypted objects stored on a special remote. So storing the remote digest might be considered to weaken the security.
+"""]]
diff --git a/doc/todo/wishlist__58___perform_fsck_remotely/comment_2_2f0dbaf143d94290bfbebb6869eb7241._comment b/doc/todo/wishlist__58___perform_fsck_remotely/comment_2_2f0dbaf143d94290bfbebb6869eb7241._comment
new file mode 100644
index 0000000000..5418ff991e
--- /dev/null
+++ b/doc/todo/wishlist__58___perform_fsck_remotely/comment_2_2f0dbaf143d94290bfbebb6869eb7241._comment
@@ -0,0 +1,12 @@
+[[!comment format=mdwn
+ username="guilhem"
+ ip="129.16.20.209"
+ subject="comment 2"
+ date="2013-08-22T16:56:55Z"
+ content="""
+Oh yeah, the MDC paragraph was pretty much pointless indeed. Oops :-P
+
+I agree that this would potentially add some noise to the index, and weaken the 
+security, but depending on the threat model and people's preferences that's an
+option that's worth considering IMHO.
+"""]]
diff --git a/doc/todo/wishlist__58___perform_fsck_remotely/comment_3_5ec2e0e248dfd4ca46aef89cc5658d18._comment b/doc/todo/wishlist__58___perform_fsck_remotely/comment_3_5ec2e0e248dfd4ca46aef89cc5658d18._comment
new file mode 100644
index 0000000000..91d07b7659
--- /dev/null
+++ b/doc/todo/wishlist__58___perform_fsck_remotely/comment_3_5ec2e0e248dfd4ca46aef89cc5658d18._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="http://joeyh.name/"
+ ip="68.119.96.78"
+ subject="comment 3"
+ date="2013-12-21T16:27:16Z"
+ content="""
+There is now a remoteFsck action in the Remote data structure. So far it is only implemented for git remotes, and `git annex fsck` does not use it; only the assistant does.
+"""]]
diff --git a/doc/todo/wishlist__58___print_locations_for_files_in_rsync_remote.mdwn b/doc/todo/wishlist__58___print_locations_for_files_in_rsync_remote.mdwn
new file mode 100644
index 0000000000..3876f21977
--- /dev/null
+++ b/doc/todo/wishlist__58___print_locations_for_files_in_rsync_remote.mdwn
@@ -0,0 +1,6 @@
+Based on an irc conversation earlier today:
+
+19:50 < warp> joeyh: what is the best way to figure out the (remote) filename for a file stored in an rsync remote?
+
+20:43 < joeyh> warp: re your other question, probably the best thing would be to make the whereis command print out locations for each remote, as it always does for the web special remotes
+
diff --git a/doc/todo/wishlist__58___provide_a_config_option_for_using_new_hashing_scheme_in_non-bare_remotes.mdwn b/doc/todo/wishlist__58___provide_a_config_option_for_using_new_hashing_scheme_in_non-bare_remotes.mdwn
new file mode 100644
index 0000000000..b18eace9a2
--- /dev/null
+++ b/doc/todo/wishlist__58___provide_a_config_option_for_using_new_hashing_scheme_in_non-bare_remotes.mdwn
@@ -0,0 +1,5 @@
+I understand that for backwards compatibility the non-bare remotes use the old "mixed" case scheme. However, for new annexes, it'd make sense to be able to use the new one so the scheme matches in all remotes.
+
+> If this option existed then every clone of a repository would need to set
+> it, or files would be hashed into the wrong location and would appear not
+> visible. Sounds like a bug magnet to me; not attractive. --[[Joey]]
diff --git a/doc/todo/wishlist__58___recursive_directory_remote_setup__47__addurl.mdwn b/doc/todo/wishlist__58___recursive_directory_remote_setup__47__addurl.mdwn
new file mode 100644
index 0000000000..2bfb90b540
--- /dev/null
+++ b/doc/todo/wishlist__58___recursive_directory_remote_setup__47__addurl.mdwn
@@ -0,0 +1,7 @@
+I think it would be interesting to have a way to recursively import a local directory without actually moving files around. And to be able to checksum these files as well (without moving them into the annex).
+
+This would work somewhat similar to looping over a directory and adding file:// remotes for each file.
+
+A use case is importing optical media (read-only), whilst keeping that media as a remote, and being able to calculate checksums directly without moving any files around.
+
+For single files, it would also be interesting if addurl had a "--localchecksum" option that would only work for file:// urls, and make it checksum files directly from their source location?)
diff --git a/doc/todo/wishlist__58___recursive_directory_remote_setup__47__addurl/comment_1_b79976afc2242791523e63831f30af71._comment b/doc/todo/wishlist__58___recursive_directory_remote_setup__47__addurl/comment_1_b79976afc2242791523e63831f30af71._comment
new file mode 100644
index 0000000000..caefee9a82
--- /dev/null
+++ b/doc/todo/wishlist__58___recursive_directory_remote_setup__47__addurl/comment_1_b79976afc2242791523e63831f30af71._comment
@@ -0,0 +1,12 @@
+[[!comment format=mdwn
+ username="https://launchpad.net/~arand"
+ nickname="arand"
+ subject="comment 1"
+ date="2013-03-10T23:29:55Z"
+ content="""
+Recursively adding urls is already feasiable with some simple scripting:
+
+[https://gitorious.org/arand-scripts/arand-scripts/blobs/master/annex-importdir](https://gitorious.org/arand-scripts/arand-scripts/blobs/master/annex-importdir)
+
+but this is obviously missing the checksumming bit though.
+"""]]
diff --git a/doc/todo/wishlist__58___recursive_directory_remote_setup__47__addurl/comment_2_1741d2392006a9af9cfd1f3b847600b9._comment b/doc/todo/wishlist__58___recursive_directory_remote_setup__47__addurl/comment_2_1741d2392006a9af9cfd1f3b847600b9._comment
new file mode 100644
index 0000000000..64d5924795
--- /dev/null
+++ b/doc/todo/wishlist__58___recursive_directory_remote_setup__47__addurl/comment_2_1741d2392006a9af9cfd1f3b847600b9._comment
@@ -0,0 +1,9 @@
+[[!comment format=mdwn
+ username="http://joeyh.name/"
+ nickname="joey"
+ subject="comment 2"
+ date="2013-03-11T03:04:44Z"
+ content="""
+See also: [[forum/Managing a large number of files archived on many pieces of read-only medium (E.G. DVDs)]]
+particularly [[this comment|forum/Managing a large number of files archived on many pieces of read-only medium (E.G. DVDs)#comment-908dbe02f29e011f030bba4ab5ef73d1]]
+"""]]
diff --git a/doc/todo/wishlist__58___recursive_directory_remote_setup__47__addurl/comment_3_b6fb39030f98bbc9915712e3d35d1838._comment b/doc/todo/wishlist__58___recursive_directory_remote_setup__47__addurl/comment_3_b6fb39030f98bbc9915712e3d35d1838._comment
new file mode 100644
index 0000000000..fd5dc9e051
--- /dev/null
+++ b/doc/todo/wishlist__58___recursive_directory_remote_setup__47__addurl/comment_3_b6fb39030f98bbc9915712e3d35d1838._comment
@@ -0,0 +1,9 @@
+[[!comment format=mdwn
+ username="0xloem@0bd8a79a57e4f0dcade8fc81d162c37eae4d6730"
+ nickname="0xloem"
+ subject="Assistant support"
+ date="2016-04-16T11:52:14Z"
+ content="""
+This would be really augmented if the assistant and webapp could watch external folders and automatically add new files that appeared.  Then e.g. your system logs and family photos could be archived together without maintenance needed.
+I'm running into this on android, wherw I'd like to store files my phone produces alongside my other files.
+"""]]
diff --git a/doc/todo/wishlist__58___rsync_efficiency.mdwn b/doc/todo/wishlist__58___rsync_efficiency.mdwn
new file mode 100644
index 0000000000..fe1848f1b3
--- /dev/null
+++ b/doc/todo/wishlist__58___rsync_efficiency.mdwn
@@ -0,0 +1,8 @@
+If you look at the transfer rates during a copy job to remotes, you see it going down to zero for a short time between files.
+
+While that's understandable from rsync's PoV, it's not as efficient as git-annex could be.
+
+Would parallelization be an option? Are there alternate improvements?
+
+
+-- Richard
diff --git a/doc/todo/wishlist__58___rsync_efficiency/comment_1_6e7cceb9d23f0cad3d9f839dd2a04901._comment b/doc/todo/wishlist__58___rsync_efficiency/comment_1_6e7cceb9d23f0cad3d9f839dd2a04901._comment
new file mode 100644
index 0000000000..35f6b885b4
--- /dev/null
+++ b/doc/todo/wishlist__58___rsync_efficiency/comment_1_6e7cceb9d23f0cad3d9f839dd2a04901._comment
@@ -0,0 +1,18 @@
+[[!comment format=mdwn
+ username="joey"
+ subject="""comment 1"""
+ date="2015-04-18T19:21:44Z"
+ content="""
+The `concurrentprogress` branch can already parallelize transfers.
+It's waiting on some progress display improvements for merging.
+
+I imagine this would keep the pipe full more consistently.
+
+Although, ssh caching is perhaps even more important, since
+it avoids a tcp slow start having to be done for each file
+that's transferred. So make sure it's working.
+
+Even with those, I would not expect git-annex to be as efficient
+as pure rsync of a directory. git-annex's more general design mean
+that there are more moving parts..
+"""]]
diff --git a/doc/todo/wishlist__58___rsync_efficiency/comment_2_d87df8f1012f71248ed33f64f9ace17e._comment b/doc/todo/wishlist__58___rsync_efficiency/comment_2_d87df8f1012f71248ed33f64f9ace17e._comment
new file mode 100644
index 0000000000..3acd6b74b7
--- /dev/null
+++ b/doc/todo/wishlist__58___rsync_efficiency/comment_2_d87df8f1012f71248ed33f64f9ace17e._comment
@@ -0,0 +1,12 @@
+[[!comment format=mdwn
+ username="https://www.google.com/accounts/o8/id?id=AItOawl9sYlePmv1xK-VvjBdN-5doOa_Xw-jH4U"
+ nickname="Richard"
+ subject="comment 2"
+ date="2015-04-20T14:37:03Z"
+ content="""
+I read the blog after my time abroad and chuckled about the timing of my request, yah :)
+
+Could git-annex reasonably detach generating the list of files to transfer from the actual transfer? That way, there is never a delay while git-annex looks for the next file.
+
+Richard
+"""]]
diff --git a/doc/todo/wishlist__58___spec.remotes_for_other_peer_network_data_stores___40__gnunet__44___freenet__41__.mdwn b/doc/todo/wishlist__58___spec.remotes_for_other_peer_network_data_stores___40__gnunet__44___freenet__41__.mdwn
new file mode 100644
index 0000000000..545bd861d6
--- /dev/null
+++ b/doc/todo/wishlist__58___spec.remotes_for_other_peer_network_data_stores___40__gnunet__44___freenet__41__.mdwn
@@ -0,0 +1,26 @@
+Apart from Tahoe-LAFS (covered by [[todo/tahoe lfs for reals]] and [[forum/tips: special_remotes/hook with tahoe-lafs]]), [[special remotes]] (which I understand as real storage backends) for other other [peer network data stores](http://en.wikipedia.org/wiki/Distributed_data_store#Peer_network_node_data_stores_2) would be interesting.
+
+I mean gnunet, freenet, BitTorrent (also trackerless).
+
+Before dropping a file locally, the BitTorrent client should check that all parts are still available from the peers.
+
+Of course, there is no guarantee assumed that the content won't disappear from the peer network in future: they act more like a cache rather than an archive on whose lifespan you decide. (I'm only not sure about gnunet now: whether there is a rule of dropping unused content from it, like in freenet.)
+
+So, a copy in peer networks shouldn't be counted on by git-annex as much as a copy on a storage you control: probably, by efault, it shouldn't let you delete the local copy if there is a copy in a peer network unless you saved it somewhere else. 
+
+(Think of such a scenario: I could save some of my public large data on external disks/DVDs and keep them at home, and also put them onto peer networks with the same nterface of git-annex which I would be used to; I would also use the git-annex interface to check from time to time that the content is still present, i.e. "cached", on the peer networks. Whenever I'm away from home, and unexpectedly need to show this content to someone, or have a look at it for some reason, I could get it from the peer network "cache".)
+
+Also networks like namecoin (derived from bitcoin) can be used as a key-value store. Despite being a peer network, a system like namecoin actually could offer the publisher more control over the lifespan of the content: he should be able to offer "financial" reward for others processing his key-value data. (But I'm not sure namecoin is designed reasonably for this reward system to work actually; but there might be appearing other similar systems.)
+
+## A different view: extend the key-value backends with ways to look for the content in other content-addressable storage systems
+We might want to look for the registered files in other [content-addressable storage systems](http://en.wikipedia.org/wiki/Content-addressable_storage#Open-source_implementations) (and also to be able to put the files there for storage).
+
+For example:
+
+* [**GNUnet**](http://en.wikipedia.org/wiki/Gnunet) uses its own hash format to address the content. git-annex could extend its own [[backends]] with a one to work with GNUnet, and by default have a built-in [[special remote|special remotes]] that would interact with GNUnet when looking for a content or storing some content. No special setup of the special remote in each repo should be necessary, because GNUnet is "global", so we'd just use the user's already configured GNUnet client. Just turning the builtin GNUnet special remote on or off should be an option (in the repo configuration, and when calling the commands that would query it, like `whereis`).
+* **freenet** is similar.
+* Similarly, a backend  for the hashes used in **BitTorrent** and **magnet links** could be used. If we want a trackerless mode, then probably it's a similar case for a "global"/built-in special remote that needs no local setup in each repo. Using a selected tracker would mean setting up a special remote in our repo.
+* **Git**  itself can be viwed as place to look for the content. There could be a corresponding backend and a builtin special remote (needing no extra setup) to look for the content among the objects stored in the local Git repo. (What if we have a copy of a file that we've put under the control of git-annex in a previous Git commit? We could get it from the object store of Git.)
+* **Venti**, [[**Tahoe-LAFS**|todo/tahoe lfs for reals]] would need a backend for their hashes, and a specially setup special remote in each repo where we'd like to use them--because these are not "global" system, we must setup the path to the instance of the filesystem we'd like to use.
+* probably, there must be other interesting cases of this kind...
+* (I'm also thinking about using somethng like a **bibliographic information** as a key, but then it wouldn't guarantee identical files: the same paper can be stored in different formats, etc. Cf. [**URNs**](http://en.wikipedia.org/wiki/Uniform_resource_name#Examples), via . Also, an URN like bibliographic information can't be computed from the file, it will have to be entered manually or obtained from another directory of URNs.)
diff --git a/doc/todo/wishlist__58___spec.remotes_for_other_peer_network_data_stores___40__gnunet__44___freenet__41__/comment_1_e2c2047e7401cb95a82ffb686a732859._comment b/doc/todo/wishlist__58___spec.remotes_for_other_peer_network_data_stores___40__gnunet__44___freenet__41__/comment_1_e2c2047e7401cb95a82ffb686a732859._comment
new file mode 100644
index 0000000000..80a245d144
--- /dev/null
+++ b/doc/todo/wishlist__58___spec.remotes_for_other_peer_network_data_stores___40__gnunet__44___freenet__41__/comment_1_e2c2047e7401cb95a82ffb686a732859._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="http://joeyh.name/"
+ ip="4.153.14.141"
+ subject="comment 1"
+ date="2012-09-25T22:57:19Z"
+ content="""
+The best first step to adding such kinds of data stores to git-annex is probably to use the [[special_remotes/hook]] special remote to access them. 
+"""]]
diff --git a/doc/todo/wishlist__58___spec.remotes_for_other_peer_network_data_stores___40__gnunet__44___freenet__41__/comment_2_472b576afdb169b233edd01adcb2123d._comment b/doc/todo/wishlist__58___spec.remotes_for_other_peer_network_data_stores___40__gnunet__44___freenet__41__/comment_2_472b576afdb169b233edd01adcb2123d._comment
new file mode 100644
index 0000000000..c58f97c1cb
--- /dev/null
+++ b/doc/todo/wishlist__58___spec.remotes_for_other_peer_network_data_stores___40__gnunet__44___freenet__41__/comment_2_472b576afdb169b233edd01adcb2123d._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="http://lj.rossia.org/users/imz/"
+ ip="79.165.56.162"
+ subject="comment 2"
+ date="2012-09-25T23:29:49Z"
+ content="""
+I see. But then, as with Tahoe-LAFS, they also have their own formats for checksums, keys, which could be re-used in git-annex, and that needs special treatment.
+"""]]
diff --git a/doc/todo/wishlist__58___spec.remotes_for_other_peer_network_data_stores___40__gnunet__44___freenet__41__/comment_3_b4ff519ece76c6c3fb29b981320e2e1c._comment b/doc/todo/wishlist__58___spec.remotes_for_other_peer_network_data_stores___40__gnunet__44___freenet__41__/comment_3_b4ff519ece76c6c3fb29b981320e2e1c._comment
new file mode 100644
index 0000000000..dac3331740
--- /dev/null
+++ b/doc/todo/wishlist__58___spec.remotes_for_other_peer_network_data_stores___40__gnunet__44___freenet__41__/comment_3_b4ff519ece76c6c3fb29b981320e2e1c._comment
@@ -0,0 +1,10 @@
+[[!comment format=mdwn
+ username="http://joeyh.name/"
+ ip="209.250.56.154"
+ subject="comment 3"
+ date="2014-03-18T19:49:09Z"
+ content="""
+The new [[special_remotes/external]] special remote's protocol has GETSTATE and SETSTATE commands that can be used to store per-remote values in the git-annex branch.
+
+So, please go make these special remotes using it!
+"""]]
diff --git a/doc/todo/wishlist__58___spec.remotes_for_other_peer_network_data_stores___40__gnunet__44___freenet__41__/comment_4_469c952a131d2aac45615afb3a69d10c._comment b/doc/todo/wishlist__58___spec.remotes_for_other_peer_network_data_stores___40__gnunet__44___freenet__41__/comment_4_469c952a131d2aac45615afb3a69d10c._comment
new file mode 100644
index 0000000000..4fd2ddc210
--- /dev/null
+++ b/doc/todo/wishlist__58___spec.remotes_for_other_peer_network_data_stores___40__gnunet__44___freenet__41__/comment_4_469c952a131d2aac45615afb3a69d10c._comment
@@ -0,0 +1,12 @@
+[[!comment format=mdwn
+ username="0xloem@0bd8a79a57e4f0dcade8fc81d162c37eae4d6730"
+ nickname="0xloem"
+ subject="freenet special remote"
+ date="2016-04-09T10:37:22Z"
+ content="""
+I've gotten derailed, but in case somebody else is interested for now, I've started a freenet special remote at https://github.com/xloem/gitlakepy .  Also includes beginnings of a generic python class for special remotes in the same scriptfile.
+
+SETSTATE is helpful but doesn't allow for any form of hashing the file offline, to ensure the hash matches.  It means there could be data corruption uploading the file and there would be no way to check that the hash matched the local data.  It would be nice to provide new hashing backends as well, perhaps then somebody could make a multi-hash which stores different hashes side-by-side to resolve such paranoia.
+
+I guess for now the right way to do these things is to add the new capabilities straight to the internals of git-annex, but learning haskell is an adventure when time is a constraint.
+"""]]
diff --git a/doc/todo/wishlist__58___spec.remotes_for_other_peer_network_data_stores___40__gnunet__44___freenet__41__/comment_5_0c40bb64126957a25ec1b20dc856529c._comment b/doc/todo/wishlist__58___spec.remotes_for_other_peer_network_data_stores___40__gnunet__44___freenet__41__/comment_5_0c40bb64126957a25ec1b20dc856529c._comment
new file mode 100644
index 0000000000..f345043dfe
--- /dev/null
+++ b/doc/todo/wishlist__58___spec.remotes_for_other_peer_network_data_stores___40__gnunet__44___freenet__41__/comment_5_0c40bb64126957a25ec1b20dc856529c._comment
@@ -0,0 +1,9 @@
+[[!comment format=mdwn
+ username="xloem"
+ subject="Freenet Special Remote"
+ date="2016-05-19T03:02:18Z"
+ content="""
+The freenet external special remote at https://github.com/xloem/gitlakepy is working now.
+
+No bells and whistles, but you can install it and start git-annex copy --to=freenet and git-annex get --from=freenet .
+"""]]
diff --git a/doc/todo/wishlist__58___special_remote_Ubuntu_One.mdwn b/doc/todo/wishlist__58___special_remote_Ubuntu_One.mdwn
new file mode 100644
index 0000000000..b88a038eac
--- /dev/null
+++ b/doc/todo/wishlist__58___special_remote_Ubuntu_One.mdwn
@@ -0,0 +1 @@
+Special remote support for [Ubuntu One](http://one.ubuntu.com) would be nice. They're [using propietary but open protocol](https://wiki.ubuntu.com/UbuntuOne/TechnicalDetails#ubuntuone-storageprotocol) based on [Google Protocol Buffers](http://code.google.com/p/protobuf/). There's [protobuf for Haskell](http://code.google.com/p/protobuf-haskell/) so it should be possible to compile [the protocol file](http://bazaar.launchpad.net/~ubuntuone-control-tower/ubuntuone-storage-protocol/trunk/view/head:/ubuntuone/storageprotocol/protocol.proto) to Haskell code and then use that to implement the native Ubuntu special remote.
diff --git a/doc/todo/wishlist__58___special_remote_Ubuntu_One/comment_1_ab0c761030bc55e8fb75d1b344bb98b9._comment b/doc/todo/wishlist__58___special_remote_Ubuntu_One/comment_1_ab0c761030bc55e8fb75d1b344bb98b9._comment
new file mode 100644
index 0000000000..4fb9bc95ea
--- /dev/null
+++ b/doc/todo/wishlist__58___special_remote_Ubuntu_One/comment_1_ab0c761030bc55e8fb75d1b344bb98b9._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="http://joeyh.name/"
+ ip="209.250.56.154"
+ subject="comment 1"
+ date="2014-03-18T20:02:14Z"
+ content="""
+I suggest that if someone wants to build this, they use the new external special remote protocol to do it.
+"""]]
diff --git a/doc/todo/wishlist__58___special_remote_Ubuntu_One/comment_2_17e948acb1e29793cf172cd6def4160b._comment b/doc/todo/wishlist__58___special_remote_Ubuntu_One/comment_2_17e948acb1e29793cf172cd6def4160b._comment
new file mode 100644
index 0000000000..20230e3f98
--- /dev/null
+++ b/doc/todo/wishlist__58___special_remote_Ubuntu_One/comment_2_17e948acb1e29793cf172cd6def4160b._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="madduck"
+ ip="2001:a60:f0fb:0:224:d7ff:fe04:c82c"
+ subject="Ubuntu One to be discontinued"
+ date="2014-04-07T05:11:52Z"
+ content="""
+Thanksfully, Canonical have stopped this silliness, Ubuntu One will be discontinued, so this todo can be marked \"wontfix\" and archived.
+"""]]
diff --git a/doc/todo/wishlist__58___traffic_accounting_for_git-annex.mdwn b/doc/todo/wishlist__58___traffic_accounting_for_git-annex.mdwn
new file mode 100644
index 0000000000..4b661101d7
--- /dev/null
+++ b/doc/todo/wishlist__58___traffic_accounting_for_git-annex.mdwn
@@ -0,0 +1,3 @@
+As git annex keeps logs about file transfers anyway, it should be relatively easy to add traffic accounting to a repo. That would allow me to monitor how much traffic a given repo generates. As I might end up hosting git-annex repos for a few personal friends, I need/want a way to track the heavy hitters. -- RichiH
+
+PS: If you ever plan to host git-annex similar branchable, this would probably be of interest to you, as well :)
diff --git a/doc/todo/wishlist__58___unify_directory_scheme_for_the_store.mdwn b/doc/todo/wishlist__58___unify_directory_scheme_for_the_store.mdwn
new file mode 100644
index 0000000000..83ce531278
--- /dev/null
+++ b/doc/todo/wishlist__58___unify_directory_scheme_for_the_store.mdwn
@@ -0,0 +1,20 @@
+In regular repos, objects are stored in files of the form: .git/annex/objects/xY/z1/SHA1-.../SHA1-.... (scheme 1)
+
+On (some) special remotes, the corresponding file is stored at: .../abc/def/SHA1-... (scheme 2)
+
+I'm not sure why the same scheme as in .git/objects isn't used, but it would be useful that the two-directory prefix were the same for all objects stores.
+
+My use case is: I synchronize a git repo, say containing photos, to a server on which I can't install git-annex. I want the server to store all annexed files. For the photos to be viewed online, the annex store must use the scheme 1 (because the symlinks point to files with scheme 1). So I need to rsync .git/annex/objects manually from my desktop, because a git-annex rsync remote uses scheme 2. On the other hand, the repo on this server is not known by git-annex (like it would if I used a rsync remote).
+
+At least it would be valuable (to get around above problem) to have a plumbing command giving the 2-directory prefix from a given key, for example:
+
+$ git annex prefix-dir SHA1-s2--3f786850e387550fdab836ed7e6dc881
+
+7w/88
+
+f18/122
+
+
+Even if the 2 schemes were unified, this prefix-dir command would still be useful when hacking around git-annex (for now I need to maintain a dictionary structure).
+
+Thanks a lot.
diff --git a/doc/todo/wishlist__58___unify_directory_scheme_for_the_store/comment_1_44da58beaaab359ecaba7fb905ca4ae1._comment b/doc/todo/wishlist__58___unify_directory_scheme_for_the_store/comment_1_44da58beaaab359ecaba7fb905ca4ae1._comment
new file mode 100644
index 0000000000..86c33d4349
--- /dev/null
+++ b/doc/todo/wishlist__58___unify_directory_scheme_for_the_store/comment_1_44da58beaaab359ecaba7fb905ca4ae1._comment
@@ -0,0 +1,10 @@
+[[!comment format=mdwn
+ username="http://joeyh.name/"
+ ip="4.152.108.243"
+ subject="comment 1"
+ date="2013-10-04T20:50:33Z"
+ content="""
+Using the mixed case hash directory names is not desirable because people want to use git-annex on a variety of filesystems and operating systems that treat them in a variety of broken ways. However, migrating to the all lower case hash directory names would require changing every git-annex symlink in every git repository, and I do not want to inflict that on my users.
+
+It seems to me that the best solution to your problem is to install git-annex on your server, which should not be very hard. 
+"""]]
diff --git a/doc/todo/wishlist__58___unify_directory_scheme_for_the_store/comment_2_bc698c501ecdb56df57171f4f3bb831a._comment b/doc/todo/wishlist__58___unify_directory_scheme_for_the_store/comment_2_bc698c501ecdb56df57171f4f3bb831a._comment
new file mode 100644
index 0000000000..5c5269f8cb
--- /dev/null
+++ b/doc/todo/wishlist__58___unify_directory_scheme_for_the_store/comment_2_bc698c501ecdb56df57171f4f3bb831a._comment
@@ -0,0 +1,16 @@
+[[!comment format=mdwn
+ username="https://www.google.com/accounts/o8/id?id=AItOawnpdM9F8VbtQ_H5PaPMpGSxPe_d5L1eJ6w"
+ nickname="Rafaël"
+ subject="comment 2"
+ date="2013-10-05T10:45:16Z"
+ content="""
+Thank you for prompt answer. I didn't know there were tarballs,
+and indeed I managed to install them easily (although I had to
+manually install glibc version 2.9, only 2.7 was installed).
+
+I found that bare git repos also use lower case hash directory
+names... I still would be happy with an optional migration to all
+lower case, with a new key-value backend, to avoid this little
+complication which happens sometimes (say when converting a repo
+from non-bare to bare).
+"""]]
diff --git a/doc/todo/wishlist__58___unify_directory_scheme_for_the_store/comment_3_e555d0dbbaa05528806905c6a940724b._comment b/doc/todo/wishlist__58___unify_directory_scheme_for_the_store/comment_3_e555d0dbbaa05528806905c6a940724b._comment
new file mode 100644
index 0000000000..e6acad3b94
--- /dev/null
+++ b/doc/todo/wishlist__58___unify_directory_scheme_for_the_store/comment_3_e555d0dbbaa05528806905c6a940724b._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="https://www.google.com/accounts/o8/id?id=AItOawnpdM9F8VbtQ_H5PaPMpGSxPe_d5L1eJ6w"
+ nickname="Rafaël"
+ subject="comment 3"
+ date="2013-10-07T13:33:40Z"
+ content="""
+By the way, I just had the case above, i.e. convert a bare repo to non-bare. In order to keep the annex files, I cloned the bare one, and git-annex move'ed all annex content to the new repo, and to my surprise it was slow, as if all files where copied (or maybe they were only checksummed?), instead of being only renamed (old and new repos were on the same partition). So I restate that at least a command line tool giving the prefix dirs would be useful, to allow scripting for this kind of situation.
+"""]]
diff --git a/doc/todo/wishlist__58___use_cp_--reflink__61__auto_for_git-annex-__123__copy__44__get__125__.mdwn b/doc/todo/wishlist__58___use_cp_--reflink__61__auto_for_git-annex-__123__copy__44__get__125__.mdwn
new file mode 100644
index 0000000000..425948efd6
--- /dev/null
+++ b/doc/todo/wishlist__58___use_cp_--reflink__61__auto_for_git-annex-__123__copy__44__get__125__.mdwn
@@ -0,0 +1,5 @@
+I believe cloning a repository on the same filesystem already makes use of cp --reflink=auto, but it appears that git-annex-{get,copy} use rsync even when both repositories involved in the copy/get are on the same filesystem.
+
+Would it be possible for git-annex-get and git-annex-copy to use cp --reflink=auto when the source and destination repositories are on the same filesystem?
+
+[[!meta title="allow remotes to do their own, smarter diskreserve checking"]]
diff --git a/doc/todo/wishlist__58___use_cp_--reflink__61__auto_for_git-annex-__123__copy__44__get__125__/comment_1_bbbb9c4c71b8c56eaf134b794d2345c3._comment b/doc/todo/wishlist__58___use_cp_--reflink__61__auto_for_git-annex-__123__copy__44__get__125__/comment_1_bbbb9c4c71b8c56eaf134b794d2345c3._comment
new file mode 100644
index 0000000000..316e45a505
--- /dev/null
+++ b/doc/todo/wishlist__58___use_cp_--reflink__61__auto_for_git-annex-__123__copy__44__get__125__/comment_1_bbbb9c4c71b8c56eaf134b794d2345c3._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="jscinoz@dc383eb6c5526eeb832a38ef5e5e626cf282ee6c"
+ nickname="jscinoz"
+ subject="comment 1"
+ date="2016-07-18T06:17:57Z"
+ content="""
+Oh, my mistake. It appears git-annex already does this - I mistakenly had one of the repositories in a different btrfs subvolume.
+"""]]
diff --git a/doc/todo/wishlist__58___use_cp_--reflink__61__auto_for_git-annex-__123__copy__44__get__125__/comment_2_1eb09a234616cb0df7fed9016827dd0c._comment b/doc/todo/wishlist__58___use_cp_--reflink__61__auto_for_git-annex-__123__copy__44__get__125__/comment_2_1eb09a234616cb0df7fed9016827dd0c._comment
new file mode 100644
index 0000000000..1669938332
--- /dev/null
+++ b/doc/todo/wishlist__58___use_cp_--reflink__61__auto_for_git-annex-__123__copy__44__get__125__/comment_2_1eb09a234616cb0df7fed9016827dd0c._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="jscinoz@dc383eb6c5526eeb832a38ef5e5e626cf282ee6c"
+ nickname="jscinoz"
+ subject="comment 2"
+ date="2016-07-18T06:32:53Z"
+ content="""
+I suppose one enhancement could be to ignore annex.diskreserve when content is obtained by reflink copy, but I can imagine this would be difficult to achieve, since we don't know in advance whether or not --reflink=auto will actually result in a reflink copy. I imagine you could try reflink=always first, ignoring annex.diskreserve, then if it fails, fallback to reflink=auto where annex.diskreserve is checked, but perhaps this is too much filesystem-specific logic to be appropriate in git-annex.
+"""]]
diff --git a/doc/todo/wishlist__58___use_cp_--reflink__61__auto_for_git-annex-__123__copy__44__get__125__/comment_3_942c2decd6dda0730e2efe4ed6e6cd16._comment b/doc/todo/wishlist__58___use_cp_--reflink__61__auto_for_git-annex-__123__copy__44__get__125__/comment_3_942c2decd6dda0730e2efe4ed6e6cd16._comment
new file mode 100644
index 0000000000..92766fc275
--- /dev/null
+++ b/doc/todo/wishlist__58___use_cp_--reflink__61__auto_for_git-annex-__123__copy__44__get__125__/comment_3_942c2decd6dda0730e2efe4ed6e6cd16._comment
@@ -0,0 +1,22 @@
+[[!comment format=mdwn
+ username="joey"
+ subject="""comment 3"""
+ date="2016-07-19T14:57:14Z"
+ content="""
+Yes, reflink is used instead of rsync when it's able to determine it's the
+same filesystem.
+
+Not checking diskreserve for reflink (and also for hard link when
+annex.hardlink is set) would be nice. But, it's a layering problem
+since currently the diskreserve check is done separately from the transfer.
+
+The same layering problem also makes downloads from encrypted special
+remotes not check if there's space for both the encrypted and de-encrypted
+file content, in cases where both files are present on disk at the same
+time.
+
+So, there would be multiple benefits to improving the api somehow so more
+smart diskreserve checks can be done. Although I'd then worry that if
+remotes were responsible for doing diskreserve checks, they might be buggy
+and forget to check. 
+"""]]
diff --git a/doc/todo/wishlist__91__minor__93____58___add_time_stamps_to_annex_log_popups_in_webapp.mdwn b/doc/todo/wishlist__91__minor__93____58___add_time_stamps_to_annex_log_popups_in_webapp.mdwn
new file mode 100644
index 0000000000..e0b1b5f5fd
--- /dev/null
+++ b/doc/todo/wishlist__91__minor__93____58___add_time_stamps_to_annex_log_popups_in_webapp.mdwn
@@ -0,0 +1 @@
+otherwise now it is impossible to judge how old those are when opening the dashboard
diff --git a/doc/todo/wishlist__91__minor__93____58___add_time_stamps_to_annex_log_popups_in_webapp/comment_1_ec90432a7d46383071401b05243d621f._comment b/doc/todo/wishlist__91__minor__93____58___add_time_stamps_to_annex_log_popups_in_webapp/comment_1_ec90432a7d46383071401b05243d621f._comment
new file mode 100644
index 0000000000..97760e7391
--- /dev/null
+++ b/doc/todo/wishlist__91__minor__93____58___add_time_stamps_to_annex_log_popups_in_webapp/comment_1_ec90432a7d46383071401b05243d621f._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="http://olivier.mehani.name/"
+ nickname="olivier-mehani"
+ subject="comment 1"
+ date="2014-01-16T02:22:18Z"
+ content="""
++1
+"""]]
diff --git a/doc/todo/wishlist_degraded_files.mdwn b/doc/todo/wishlist_degraded_files.mdwn
new file mode 100644
index 0000000000..0b265c5eba
--- /dev/null
+++ b/doc/todo/wishlist_degraded_files.mdwn
@@ -0,0 +1,5 @@
+This is an idea to have a small placeholder file that is put into
+place when the file's actual content is not available in the local
+annex.
+
+Details being discussed here: 
diff --git a/doc/transferring_data.mdwn b/doc/transferring_data.mdwn
new file mode 100644
index 0000000000..2aab3b01f9
--- /dev/null
+++ b/doc/transferring_data.mdwn
@@ -0,0 +1,19 @@
+git-annex can transfer data to or from any of a repository's git remotes.
+Depending on where the remote is, the data transfer is done using rsync
+(over ssh or locally), or plain cp (with copy-on-write
+optimisations on supported filesystems), or using curl (for repositories
+on the web). Some [[special_remotes]] are also supported that are not
+traditional git remotes.
+
+If a data transfer is interrupted, git-annex retains the partial transfer
+to allow it to be automatically resumed later.
+
+It's equally easy to transfer a single file to or from a repository,
+or to launch a retrieval of a massive pile of files from whatever
+repositories they are scattered amongst.
+
+git-annex automatically uses whatever remotes are currently accessible,
+preferring ones that are less expensive to talk to.
+
+[[!img repomap.png caption="A real-world repository interconnection map
+(generated by git-annex map)"]]
diff --git a/doc/trust.mdwn b/doc/trust.mdwn
new file mode 100644
index 0000000000..f93c4a9c91
--- /dev/null
+++ b/doc/trust.mdwn
@@ -0,0 +1,62 @@
+Git-annex supports several levels of trust of a repository:
+
+* semitrusted (default)
+* untrusted
+* trusted
+* dead
+
+## semitrusted
+
+Normally, git-annex does not fully trust its stored [[location_tracking]]
+information. When removing content, it will directly check
+that other repositories have enough [[copies]].
+
+Generally that explicit checking is a good idea. Consider that the current
+[[location_tracking]] information for a remote may not yet have propagated
+out. Or, a remote may have suffered a catastrophic loss of data, or itself
+been lost.
+
+There is still some trust involved here. A semitrusted repository is
+depended on to retain a copy of the file content; possibly the only
+[[copy|copies]].
+
+(Being semitrusted is the default. The [[git-annex semitrust|git-annex-semitrust]] command
+restores a repository to this default, when it has been overridden.
+The `--semitrust` option can temporarily restore a repository to this
+default.)
+
+## untrusted
+
+An untrusted repository is not trusted to retain data at all. Git-annex
+will retain sufficient [[copies]] of data elsewhere.
+
+This is a good choice for eg, portable drives that could get lost. Or,
+if a disk is known to be dying, you can set it to untrusted and let
+`git annex fsck` warn about data that needs to be copied off it.
+
+To configure a repository as untrusted, use the [[git-annex untrust|git-annex-untrust]]
+command.
+
+## trusted
+
+Sometimes, you may have reasons to fully trust the location tracking
+information for a repository. For example, it may be an offline
+archival drive, from which you rarely or never remove content. Deciding
+when it makes sense to trust the tracking info is up to you.
+
+One way to handle this is just to use `--force` when a command cannot
+access a remote you trust. Or to use `--trust` to specify a repository to
+trust temporarily.
+
+To configure a repository as fully and permanently trusted,
+use the [[git-annex-trust]] command.
+
+## dead
+
+This is used to indicate that you have no trust that the repository
+exists at all. It's appropriate to use when a drive has been lost,
+or a directory irretrievably deleted. It will make git-annex avoid
+even showing the repository as a place where data might still reside.
+
+To configure a repository as dead and lost, use the [[git-annex-dead]]
+command.
diff --git a/doc/trust/comment_1_305e4e7c6b75db29212b758e8504d8c9._comment b/doc/trust/comment_1_305e4e7c6b75db29212b758e8504d8c9._comment
new file mode 100644
index 0000000000..11c6ce1335
--- /dev/null
+++ b/doc/trust/comment_1_305e4e7c6b75db29212b758e8504d8c9._comment
@@ -0,0 +1,10 @@
+[[!comment format=mdwn
+ username="https://www.google.com/accounts/o8/id?id=AItOawlJl0OCe6AJEnIFIcg-t5Rhk-lI_Y-tWUs"
+ nickname="Michael"
+ subject="default trust for hosts"
+ date="2015-05-01T21:41:05Z"
+ content="""
+Is it possible to set a default trust per host (e.g. in `~/.gitconfig`)?
+
+I have one server that does its own backups, and a client that I'd like to keep thin across multiple repositories.
+"""]]
diff --git a/doc/trust/comment_2_2262eaa830306d3dc75999bc0433b6a8._comment b/doc/trust/comment_2_2262eaa830306d3dc75999bc0433b6a8._comment
new file mode 100644
index 0000000000..b2136f43ea
--- /dev/null
+++ b/doc/trust/comment_2_2262eaa830306d3dc75999bc0433b6a8._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="joey"
+ subject="""comment 2"""
+ date="2015-05-05T18:07:02Z"
+ content="""
+You can use `remote..annex-trustlevel` as documented in the git-annex
+man page.
+"""]]
diff --git a/doc/tuning.mdwn b/doc/tuning.mdwn
new file mode 100644
index 0000000000..1860295465
--- /dev/null
+++ b/doc/tuning.mdwn
@@ -0,0 +1,47 @@
+git-annex now has experimental support for tuning a repository for
+different work loads. 
+
+For example, a repository with a very large number of files in it may work
+better if git-annex uses some nonstandard hash format, for either the
+`.git/annex/objects/` directory, or for the log files in the git-annex
+branch.
+
+A repository can currently only be tuned when it is first created; this is
+done by passing `-c name=value` parameters to `git annex init`.
+
+For example, this will make git-annex use only 1 level for hash directories
+in `.git/annex/objects`:
+
+	git -c annex.tune.objecthash1=true annex init
+
+It's very important to keep in mind that this makes a nonstandard format
+git-annex repository. In general, this cannot safely be used with
+git-annex older than version 5.20150128. Older version of git-annex will
+not understand and will get confused and perhaps do bad things.
+
+Also, it's not safe to merge two separate git repositories that have been
+tuned differently (or one tuned and the other one not). git-annex will
+prevent merging their git-annex branches together, but it cannot prevent
+`git merge remote/master` merging two branches, and the result will be ugly
+at best (`git annex fix` can fix up the mess somewhat).
+
+Again, tuned repositories are an experimental feature; use with caution!
+
+The following tuning parameters are available:
+
+* `annex.tune.objecthash1=true`  
+  Use just one level of hash directories in `.git/annex/objects/`,
+  instead of the default two levels.
+
+* `annex.tune.objecthashlower=true`  
+  Make the hash directories in `.git/annex/objects/` use
+  all lower-case, instead of the default mixed-case.
+
+* `annex.tune.branchhash1=true`  
+  Use just one level of hash directories in the git-annex branch,
+  instead of the default two levels.
+
+Note that git-annex will automatically propagate these settings to
+`.git/config` for tuned repositories. You should never directly change
+these settings in `.git/config`, and should never set them in global
+gitconfig.
diff --git a/doc/tuning/comment_1_f8af8e9b696d32d238ebd56a3b8058c4._comment b/doc/tuning/comment_1_f8af8e9b696d32d238ebd56a3b8058c4._comment
new file mode 100644
index 0000000000..92a26fb1de
--- /dev/null
+++ b/doc/tuning/comment_1_f8af8e9b696d32d238ebd56a3b8058c4._comment
@@ -0,0 +1,17 @@
+[[!comment format=mdwn
+ username="https://www.google.com/accounts/o8/id?id=AItOawnx8kHW66N3BqmkVpgtXDlYMvr8TJ5VvfY"
+ nickname="Yaroslav"
+ subject="annex.tune.objecthashlower=true is not just "lower" letters used but a different strategy altogether"
+ date="2015-02-01T02:58:26Z"
+ content="""
+it starts to use 2 levels (even if annex.tune.objecthash1=true) of hash directories having 3 characters in the filename at each level.  So it is not just \"taken existing hash directories (1 or 2 levels) and use their lower-case version.  It is a different way to create the hash directories:
+
+e.g.  one with objecthas1=true
+
+1 -> .git/annex/objects/qj/SHA256E-s6--ecdc5536f73bdae8816f0ea40726ef5e9b810d914493075903bb90623d97b1d8/SHA256E-s6--ecdc5536f73bdae8816f0ea40726ef5e9b810d914493075903bb90623d97b1d8
+
+and if I provide all three options at once:
+
+1 -> .git/annex/objects/ccf/a40/SHA256E-s6--ecdc5536f73bdae8816f0ea40726ef5e9b810d914493075903bb90623d97b1d8/SHA256E-s6--ecdc5536f73bdae8816f0ea40726ef5e9b810d914493075903bb90623d97b1d8
+
+"""]]
diff --git a/doc/tuning/comment_2_a0091dbb39b79dfe101d05f9a5db216f._comment b/doc/tuning/comment_2_a0091dbb39b79dfe101d05f9a5db216f._comment
new file mode 100644
index 0000000000..364d2c9da1
--- /dev/null
+++ b/doc/tuning/comment_2_a0091dbb39b79dfe101d05f9a5db216f._comment
@@ -0,0 +1,12 @@
+[[!comment format=mdwn
+ username="joey"
+ subject="""comment 2"""
+ date="2015-02-04T17:09:41Z"
+ content="""
+Right, it's not simply lower-casing but a different hash strategy
+as described in [[internals/hashing]].
+
+Combining annex.tune.objecthashlower and annex.tune.objecthash1 will
+result in one level of hash directories. If you get two levels then
+you probabaly typoed "objecthas1" ...
+"""]]
diff --git a/doc/tuning/comment_4_1c576f9a5e0a0b5d7d8edf9d40462874._comment b/doc/tuning/comment_4_1c576f9a5e0a0b5d7d8edf9d40462874._comment
new file mode 100644
index 0000000000..5d3d2f0a47
--- /dev/null
+++ b/doc/tuning/comment_4_1c576f9a5e0a0b5d7d8edf9d40462874._comment
@@ -0,0 +1,10 @@
+[[!comment format=mdwn
+ username="joey"
+ subject="""comment 4"""
+ date="2017-08-28T17:40:06Z"
+ content="""
+It should be possible to write a `git-filter-branch` that converts
+a repository from one tuning to aonther, but it would not be trivial, and
+noone has done it yet. You'd still have to run it in every clone of the
+repository. Tuned and non-tuned repositories can't interoperate.
+"""]]
diff --git a/doc/tuning/comment_4_5b782975263480a405c5e8dcfe058007._comment b/doc/tuning/comment_4_5b782975263480a405c5e8dcfe058007._comment
new file mode 100644
index 0000000000..d7ee5b4a87
--- /dev/null
+++ b/doc/tuning/comment_4_5b782975263480a405c5e8dcfe058007._comment
@@ -0,0 +1,17 @@
+[[!comment format=mdwn
+ username="https://launchpad.net/~stephane-gourichon-lpad"
+ nickname="stephane-gourichon-lpad"
+ avatar="http://cdn.libravatar.org/avatar/02d4a0af59175f9123720b4481d55a769ba954e20f6dd9b2792217d9fa0c6089"
+ subject="Syncing between untuned and tuned repo?"
+ date="2017-07-18T11:49:10Z"
+ content="""
+> Also, it's not safe to merge two separate git repositories that have been tuned differently (or one tuned and the other one not). git-annex will prevent merging their git-annex branches together, but it cannot prevent git merge remote/master merging two branches, and the result will be ugly at best (git annex fix can fix up the mess somewhat).
+
+My main use repo is 1.7TB large and holds 172.000+ annexed files.
+Variations in filename case has lead to a number of file duplications that are still not solved (I have base scripts that can be used to flatten filename case and fix references in other files, but it will probably mean handling some corner cases and there are more urgent matters for now).  
+
+For these reasons I'm highly interested in the lowercase option and I'm probably not the only one in a similar case.
+
+Does migrating to a tuned repository mean unannexing everything and reimporting into a newly created annex, replica by replica then sync again? That's a high price in some setup.  Or is there a way to somehow `git annex sync` between a newly created repo and an old, untuned one?
+
+"""]]
diff --git a/doc/upgrades.mdwn b/doc/upgrades.mdwn
new file mode 100644
index 0000000000..c7ade4a9c3
--- /dev/null
+++ b/doc/upgrades.mdwn
@@ -0,0 +1,169 @@
+[[!toc levels=3]]
+
+# Software upgrades
+
+Upgrading the code base of git-annex will be done differently depending on
+your [[install]] method. For most distribution-based packages, it is
+handled by the package management software.
+
+For the standalone distribution, the [[git-annex-webapp]](1) will ask the
+user for confirmation when it detects a new version. Once that is
+confirmed, or if `annex.autoupgrade` is enabled (see the [[git-annex]](1)
+manpage) the assistant will start the upgrade. The upgrade process is
+fairly simple: the assistant will move the `git-annex.linux` directory out
+of the way and replace it with the new version, then re-execute itself. It
+therefore needs write access to the parent directory of the
+`git-annex.linux` directory.
+
+Note that "upgrading" from a distribution-based package to the
+[[install/Linux_standalone/]] version may cause weird problems, as an
+unexpected version of git-annex (e.g. the old one from packages) may be
+ran.
+
+# Repository upgrades
+
+Occasionally improvements are made to how git-annex stores its data,
+that require an upgrade process to convert repositories made with an older
+version to be used by a newer version. It's annoying, it should happen
+rarely, but sometimes, it's worth it.
+
+There's a commitment that git-annex will always support upgrades from all
+past versions. After all, you may have offline drives from an earlier
+git-annex, and might want to use them with a newer git-annex.
+
+git-annex will notice if it is run in a repository that
+needs an upgrade, and refuse to do anything. To upgrade,
+use the "git annex upgrade" command.
+
+The upgrade process is guaranteed to be conflict-free. Unless you
+already have git conflicts in your repository or between repositories.
+Upgrading a repository with conflicts is not recommended; resolve the
+conflicts first before upgrading git-annex.
+
+The upgrade process needs to write to the repository. If the original
+repository cannot be written to (due to eg being on readonly media),
+the upgrade would need to be run in a copy of the repository.
+
+The upgrade events, so far:
+
+## v5 -> v6 (git-annex version 6.x)
+
+The upgrade from v5 to v6 is handled manually for now.
+Run `git-annex upgrade` to perform the upgrade.
+
+A v6 git-annex repository can have some files locked while other files are
+unlocked, and all git and git-annex commands can be used on both locked and
+unlocked files. (Although for locked files to be accessible, the filesystem
+must support symbolic links..
+
+Direct mode repositories are upgraded to instead use the new 
+[[adjusted branches feature|git-annex-adjust]], which transparently unlocks
+all locked files in the local repository.
+
+The behavior of some commands changes in an upgraded repository:
+
+* `git add` will add files to the annex, rather than adding them directly
+   to the git repository. To cause some files to be added directly
+   to git, you can configure `annex.largefiles`. For example:
+
+   	`git config annex.largefiles "largerthan=100kb and not (include=*.c or include=*.h)"`
+
+* `git annex unlock` and `git annex lock` change how the pointer to 
+  the annexed content is stored in git.
+
+There is also a new `annex.thin` setting, which makes unlocked files in v6
+repositories be hard linked to their content, instead of a copy. This saves
+disk space but means any modification of an unlocked file will lose the
+local (and possibly only) copy of the old version. This is automatically
+enabled when upgrading a direct mode repository, since direct mode made the
+same tradeoff.
+
+See [[tips/unlocked_files/]] for more details about locked files and thin
+mode.
+
+## v4 -> v5 (git-annex version 5.x)
+
+The upgrade from v4 to v5 is handled
+automatically, and only affects [[direct mode]] repositories.
+
+This upgrade involves changing direct mode repositories to operate with
+core.bare=true.
+
+## v3 -> v4 (git-annex version 4.x)
+
+v4 was only used for [[direct_mode]], to ensure that a version of git-annex
+that understands direct mode was used with a direct mode repository. 
+
+## v2 -> v3 (git-annex version 3.x)
+
+Involved moving the .git-annex/ directory into a separate git-annex branch.
+
+After this upgrade, you should make sure you include the git-annex branch
+when git pushing and pulling.
+
+### tips for this upgrade
+
+This upgrade is easier (and faster!) than the previous upgrades.
+You don't need to upgrade every repository at once; it's sufficient
+to upgrade each repository only when you next use it.
+	
+Example upgrade process:
+
+	cd localrepo
+	git pull
+	git annex upgrade
+	git commit -m "upgrade v2 to v3"
+	git gc
+
+## v1 -> v2 (git-annex version 0.20110316)
+
+Involved adding hashing to .git/annex/ and changing the names of all keys.
+Symlinks changed.
+
+Also, hashing was added to location log files in .git-annex/.
+And .gitattributes needed to have another line added to it.
+
+Previously, files added to the SHA [[backends]] did not have their file
+size tracked, while files added to the WORM backend did. Files added to
+the SHA backends after the conversion will have their file size tracked,
+and that information will be used by git-annex for disk free space checking.
+To ensure that information is available for all your annexed files, see
+[[upgrades/SHA_size]].
+
+### tips for this upgrade
+
+This upgrade can tend to take a while, if you have a lot of files.
+
+Each clone of a repository should be individually upgraded.
+Until a repository's remotes have been upgraded, git-annex
+will refuse to communicate with them.
+
+Start by upgrading one repository, and then you can commit
+the changes git-annex staged during upgrade, and push them out to other
+repositories. And then upgrade those other repositories. Doing it this
+way avoids git-annex doing some duplicate work during the upgrade.
+
+Example upgrade process:
+
+	cd localrepo
+	git pull
+	git annex upgrade
+	git commit -m "upgrade v1 to v2"
+	git push
+
+	ssh remote
+	cd remoterepo
+	git pull
+	git annex upgrade
+	...
+
+## v0 -> v1 (git-annex version 0.04)
+
+Involved a reorganisation of the layout of .git/annex/. Symlinks changed.
+
+Handled more or less transparently, although git-annex was just 2 weeks
+old at the time, and had few users other than Joey.
+
+Before doing this upgrade, set annex.version:
+
+	git config annex.version 0
diff --git a/doc/upgrades/SHA_size.mdwn b/doc/upgrades/SHA_size.mdwn
new file mode 100644
index 0000000000..97603ba913
--- /dev/null
+++ b/doc/upgrades/SHA_size.mdwn
@@ -0,0 +1,20 @@
+Before version 2 of the git-annex repository, files added to the SHA
+[[backends]] did not have their file size tracked, while files added to the
+WORM backend did. The file size information is used for disk free space
+checking.
+
+Files added to the SHA backends after the conversion will have their file
+size tracked automatically. This disk free space checking is an optional
+feature and since you're more likely to be using more recently added files,
+you're unlikely to see any bad effect if you do nothing.
+
+That said, if you have old files added to SHA backends that lack file size
+tracking info, here's how you can add that info. After [[upgrading|upgrades]]
+to repository version 2, in each repository run:
+
+	git annex migrate
+	git commit -m 'migrated keys for v2'
+
+The usual caveats about [[tips/migrating_data_to_a_new_backend]]
+apply; you will end up with unused keys that you can later clean up with
+`git annex unused`.
diff --git a/doc/upgrades/SHA_size/comment_1_20f9b7b75786075de666b2146dc13a60._comment b/doc/upgrades/SHA_size/comment_1_20f9b7b75786075de666b2146dc13a60._comment
new file mode 100644
index 0000000000..7b6be15321
--- /dev/null
+++ b/doc/upgrades/SHA_size/comment_1_20f9b7b75786075de666b2146dc13a60._comment
@@ -0,0 +1,12 @@
+[[!comment format=mdwn
+ username="https://www.google.com/accounts/o8/id?id=AItOawkjvjLHW9Omza7x1VEzIFQ8Z5honhRB90I"
+ nickname="Asheesh"
+ subject="The fact that the keys changed causes merge conflicts"
+ date="2012-06-25T00:28:59Z"
+ content="""
+FYI, I have run into a problem where if you 'git annex sync' between various 'git annex v3' repositories, if the different repositories are using different encodings of the SHA1 information (one including size, one not), then the 'git merge' will declare that they conflict.
+
+There's no indication that 'git annex migrate' is the right tool to run, except from perusing the 'git annex' man page. In my opinion this is a major user interface problem.
+
+-- Asheesh.
+"""]]
diff --git a/doc/upgrades/comment_1_85fb435417aea2763f9e6631680bd5fa._comment b/doc/upgrades/comment_1_85fb435417aea2763f9e6631680bd5fa._comment
new file mode 100644
index 0000000000..c58f63ed54
--- /dev/null
+++ b/doc/upgrades/comment_1_85fb435417aea2763f9e6631680bd5fa._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="rok"
+ avatar="http://cdn.libravatar.org/avatar/a80e31241cb35a3cf9bf8de34e05fc2d"
+ subject="v6 default behavior"
+ date="2017-03-16T09:24:50Z"
+ content="""
+I'm curious about the choice to change the behavior of critical git commands (such as add) in such a drastic way in V6, i.e. why does \"git add\" no longer add files to the git repo? The default of annex should be exclusive rather than inclusive, I shouldn't need to specify which files to *exclude* from annex, rather the other way around. This is an especially confusing default for code. What is the logic here? 
+"""]]
diff --git a/doc/upgrades/comment_1_85fb435417aea2763f9e6631680bd5fa/comment_1_22eece2c51cd36a54a67434b317ee6e3._comment b/doc/upgrades/comment_1_85fb435417aea2763f9e6631680bd5fa/comment_1_22eece2c51cd36a54a67434b317ee6e3._comment
new file mode 100644
index 0000000000..51d5965b56
--- /dev/null
+++ b/doc/upgrades/comment_1_85fb435417aea2763f9e6631680bd5fa/comment_1_22eece2c51cd36a54a67434b317ee6e3._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="joey"
+ subject="""comment 1"""
+ date="2017-03-21T17:43:33Z"
+ content="""
+@rok it's a consequence of using smudge/clean filters; git add passes
+the file through the filters.
+"""]]
diff --git a/doc/upgrades/gcrypt.mdwn b/doc/upgrades/gcrypt.mdwn
new file mode 100644
index 0000000000..65f80f86e0
--- /dev/null
+++ b/doc/upgrades/gcrypt.mdwn
@@ -0,0 +1,25 @@
+Unfortunately the initial gcrypt repository layout had to be changed
+after git-annex version 4.20130920. If you have an encrypted git repository
+created using version 4.20130920 or 4.20130909, you need to manually
+upgrade it.
+
+If you look at the contents of your gcrypt repository, you will
+see a bare git repository, with a few three-letter subdirectories,
+which are where git-annex stores its encrypted file contents:
+
+
    +27f/  branches/  description  hooks/  objects/
+HEAD  config     f37/         info/   refs/
+
    
+
+In the example above, the subdirectories are `27f` and `f37`.
+
+All you need to do to transition is move those subdirectories
+into an `annex/objects` directory.
+
+	mkdir annex ; mkdir annex/objects ; mv 27f f37 annex/objects
+
+Probably those are the only 3 letter things inside your git repository,
+so this will probably work:
+
+	mkdir annex ; mkdir annex/objects ; mv ??? annex/objects
diff --git a/doc/upgrades/gcrypt/comment_1_606c1527735996ae671f78948e4ad84b._comment b/doc/upgrades/gcrypt/comment_1_606c1527735996ae671f78948e4ad84b._comment
new file mode 100644
index 0000000000..8805f4a8d3
--- /dev/null
+++ b/doc/upgrades/gcrypt/comment_1_606c1527735996ae671f78948e4ad84b._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="https://www.google.com/accounts/o8/id?id=AItOawmj3kEGlCiy_Y-wb6AIBBnJ0B_SiPHV5Bo"
+ nickname="Thomas"
+ subject="small omission"
+ date="2013-09-25T04:59:03Z"
+ content="""
+In your second example, `mv ??? annex` has to be `mv ??? annex/objects`, I think.
+"""]]
diff --git a/doc/upgrades/insecure_embedded_creds.mdwn b/doc/upgrades/insecure_embedded_creds.mdwn
new file mode 100644
index 0000000000..a221fb51f4
--- /dev/null
+++ b/doc/upgrades/insecure_embedded_creds.mdwn
@@ -0,0 +1,42 @@
+git-annex had a bug in the S3 and Glacier remotes where if embedcreds=yes
+was set, and the remote used encryption=pubkey or encryption=hybrid,
+the embedded AWS credentials were stored in the git repository
+in (effectively) plaintext, not encrypted as they were supposed to be.
+
+That means that anyone who gets a copy of the git repository can extract the
+AWS credentials from it. Which would be bad..
+
+A remote with this problem cannot be enabled using `git annex
+enableremote`. Old versions of git-annex will fail with a gpg error;
+the current version will fail with a pointer to this web page.
+
+If your repository has this problem, chose from one of these approaches
+to deal with it:
+
+1. Change your AWS credentials, so the ones stored in the clear in git
+   won't be used. 
+   
+   After changing the credentials, make sure you have a
+   fixed version of git-annex, and you can then re-embed the new creds
+   into the repository, encrypted this time, by setting the
+   `AWS_SECRET_ACCESS_KEY` and `AWS_ACCESS_KEY_ID` environment variables,
+   and running `git annex enableremote $remotename embedcreds=yes`
+
+2. Fix the problem and then remove the history of the git-annex branch
+   of the repository.
+
+   Make sure you have a fixed version of git-annex, and force git-annex
+   to rewrite the embedded creds, with encryption this time, by setting
+   by setting the `AWS_SECRET_ACCESS_KEY` and `AWS_ACCESS_KEY_ID`
+   environment variables, and running `git annex enableremote $remotename embedcreds=yes`
+
+   Then, to get rid of old versions of the git-annex branch that still
+   contain the creds in cleartext, you can use `git annex forget`;
+   note that it will remove other historical data too.
+   
+   Keep in mind that this will not necessarily delete data from clones
+   you do not control.
+
+3. If you're sure that you're the only one who has access to the repository,
+   you could decide to leave it as-is. It's no more insecure than if you
+   had used encryption=shared in the first place when setting it up.
diff --git a/doc/use_case/Alice.mdwn b/doc/use_case/Alice.mdwn
new file mode 100644
index 0000000000..cdd3ea546d
--- /dev/null
+++ b/doc/use_case/Alice.mdwn
@@ -0,0 +1,24 @@
+### use case: The Nomad
+
+Alice is always on the move, often with her trusty netbook and a small
+handheld terabyte USB drive, or a smaller USB keydrive. She has a server
+out there on the net. She stores data, encrypted in the Cloud.
+
+All these things can have different files on them, but Alice no longer
+has to deal with the tedious process of keeping them manually in sync,
+or remembering where she put a file. git-annex manages all these data
+sources as if they were git remotes.  
+[[more about special remotes|special_remotes]]
+
+When she has 1 bar on her cell, Alice queues up interesting files on her
+server for later. At a coffee shop, she has git-annex download them to her
+USB drive. High in the sky or in a remote cabin, she catches up on
+podcasts, videos, and games, first letting git-annex copy them from
+her USB drive to the netbook (this saves battery power).  
+[[more about transferring data|transferring_data]]
+
+When she's done, she tells git-annex which to keep and which to remove.
+They're all removed from her netbook to save space, and Alice knows
+that next time she syncs up to the net, her changes will be synced back 
+to her server.  
+[[more about distributed version control|distributed_version_control]]
diff --git a/doc/use_case/Bob.mdwn b/doc/use_case/Bob.mdwn
new file mode 100644
index 0000000000..9477dbd3dc
--- /dev/null
+++ b/doc/use_case/Bob.mdwn
@@ -0,0 +1,25 @@
+### use case: The Archivist
+
+Bob has many drives to archive his data, most of them 
+[[kept offline|tips/offline_archive_drives]], in a safe place.
+
+With git-annex, Bob has a single directory tree that includes all
+his files, even if their content is being stored offline. He can
+reorganize his files using that tree, committing new versions to git,
+without worry about accidentally deleting anything. 
+
+When Bob needs access to some files, git-annex can tell him which drive(s)
+they're on, and easily make them available. Indeed, every drive knows what
+is on every other drive.  
+[[more about location tracking|location_tracking]]
+
+Bob thinks long-term, and so he appreciates that git-annex uses a simple
+repository format. He knows his files will be accessible in the future
+even if the world has forgotten about git-annex and git.  
+[[more about future-proofing|future_proofing]]
+
+Run in a cron job, git-annex adds new files to archival drives at night. It
+also helps Bob keep track of intentional and unintentional copies of
+files, and logs information he can use to decide when it's time to duplicate
+the content of old drives.  
+[[more about backup copies|copies]]
diff --git a/doc/users.mdwn b/doc/users.mdwn
new file mode 100644
index 0000000000..b9bab48ecf
--- /dev/null
+++ b/doc/users.mdwn
@@ -0,0 +1,9 @@
+Users of this wiki, feel free to create a subpage of this one and talk
+about yourself on it, within reason. You can link to it to sign your
+comments.
+
+List of users
+=============
+[[!inline pages="users/* and !users/*/* and !*/Discussion" 
+feeds=no archive=yes sort=title template=titlepage
+rootpage="users" postformtext="Add yourself as an git-annex user:"]]
diff --git a/doc/users/anarcat.mdwn b/doc/users/anarcat.mdwn
new file mode 100644
index 0000000000..7a911e78fe
--- /dev/null
+++ b/doc/users/anarcat.mdwn
@@ -0,0 +1,55 @@
+I use git-annex to manage huge files, mostly video and audio attached to other git repositories (such as presentations), but I also use git-annex to manage my music collection across multiple devices. I also use it to manage the `ISO` images I download, podcasts, and youtube videos.
+
+See .
+
+[[!toc]]
+
+My tips
+=======
+
+... or the ones I commented it, to be more precise.
+
+[[!inline pages="tips/* and and link(users/anarcat)" sort=mtime feeds=no actions=yes archive=yes show=0]]
+
+
+My todos
+========
+
+... same.
+
+[[!inline pages="todo/* and !todo/done and !link(todo/done) and
+link(users/anarcat)" sort=mtime feeds=no actions=yes archive=yes show=0]]
+
+Done
+----
+
+[[!inline pages="todo/* and !todo/done and link(todo/done) and
+link(users/anarcat)" feeds=no actions=yes archive=yes show=0]]
+
+My bugs
+=======
+
+... same.
+
+[[!inline pages="bugs/* and !bugs/done and !link(bugs/done) and
+link(users/anarcat)" sort=mtime feeds=no actions=yes archive=yes show=0  template=buglist]]
+
+Fixed
+-----
+
+[[!inline pages="bugs/* and !bugs/done and link(bugs/done) and
+link(users/anarcat)" feeds=no actions=yes archive=yes show=0  template=buglist]]
+
+Forum posts
+===========
+
+Forums where I posted.
+
+[[!inline pages="forum/* and link(users/anarcat)" sort=mtime feeds=no actions=yes archive=yes show=0]]
+
+Git annex dev news
+==================
+
+I find the recent changes a little too noisy, but i want more than just the [[news]], so i made up this little hybrid feed:
+
+[[!inline pages="(news/* and !news/*/* and !*/Discussion) or (devblog/* and !devblog/*/*)" feedonly="yes"]]
diff --git a/doc/users/andrew.mdwn b/doc/users/andrew.mdwn
new file mode 100644
index 0000000000..b2ce48d23f
--- /dev/null
+++ b/doc/users/andrew.mdwn
@@ -0,0 +1,4 @@
+Andrew Ringler public@andrewringler.com
+
    
+
+[git-annex-turtle](https://github.com/andrewringler/git-annex-turtle) —Apple Finder integration for git-annex on macOS, including custom badge icons, contextual menus and a Menubar icon
diff --git a/doc/users/ben.mdwn b/doc/users/ben.mdwn
new file mode 100644
index 0000000000..1e6295cbd8
--- /dev/null
+++ b/doc/users/ben.mdwn
@@ -0,0 +1 @@
+No idea yet
diff --git a/doc/users/candyangel.mdwn b/doc/users/candyangel.mdwn
new file mode 100644
index 0000000000..e40b7a693a
--- /dev/null
+++ b/doc/users/candyangel.mdwn
@@ -0,0 +1,66 @@
+I hang out in #git-annex on OFTC (come and chat/ask for help).
+
+[Getting mentioned by Joey](http://git-annex.branchable.com/devblog/day_277__thanks/) is definitely a (if not **the**) highlight of my involvement in the git-annex community. That really made my month!
+
+Things that I am (or plan to) work on related to git-annex (in some way):
+
+* FUSE filesystem for dynamic get/drop of content
+* FUSE filesystem for metadata change/views (like tagsistant)
+* Patchset for ikiwiki which allows it to be used with perlbrew libs
+* Support for modifying git-annex metadata in my file sorting app
+* Support for git-annex in my VR environment
+
+# sorting annex
+
+## diagram
+
+      +------+
+      |      | clone
+      | base ------------+---------------------------+
+      |      |           |                           |
+      +----|-+           |                           |
+         clone    +------|-----+              +------|-----+
+           |      |            |              |            |
+           |      | hd1-serial | < populate > | hd3-serial |
+           |      |            |              |            |
+           |      +------|-----+              +------|-----+
+           |             |                           |
+           |           clone                       clone
+           |             |                           |
+           |      +------|-----+              +------|-----+
+           |      |            |              |            |
+           |      | hd2-serial |   < get >    | hd4-serial |
+           |      |            |              |            |
+           |      +------|-----+              +------|-----+
+           |             |                           |
+    +------|------+      |                           |
+    |             |      |                           |
+    | collate.git -------+---------------------------+
+    |             |remote add >
+    +------|------+
+           |
+         clone
+           |
+      +----|----+
+      |         |
+      | sorting |
+      |         |
+      +---------+
+
+## explanation
+
+* *base* created and a mkannex.sh script is added to it which inits a git-annex repository with the desired settings
+* *base* is cloned to an empty drive as the "type" of files to be added (hd1, hd3)
+  * types are file, filelink, keyfile and keylink
+  * e.g. `git clone ${base_uri} keyfile`
+* mkannex.sh is run and repository is named ${drive_serial}_${type}
+* annex is populated with contents of $type, in directories created with `uuidgen -r` to prevent collision
+* populated repository (hd1, hd3) is cloned to another drive (hd2, hd4)
+* mkannex.sh is run and repository is named ${drive_serial}_${type}
+* contents are `git annex get`'d from the populated drives
+
+* *collate.git* cloned from *base* and mkannex.sh run, named "collate"
+* remotes added to collate.git for secondary drives (hd2, hd4)
+* *collate.git* syncs metadata, learning of all repositories and their contents
+* *sorting* is cloned from *collate.git* to do sorting activities
+  * read-only remotes added for the content drives as required
diff --git a/doc/users/chris.mdwn b/doc/users/chris.mdwn
new file mode 100644
index 0000000000..3320ddcaa5
--- /dev/null
+++ b/doc/users/chris.mdwn
@@ -0,0 +1 @@
+Visit my page Cybr or my private page ca84 if you like to contact me.
diff --git a/doc/users/chrysn.mdwn b/doc/users/chrysn.mdwn
new file mode 100644
index 0000000000..ba42615678
--- /dev/null
+++ b/doc/users/chrysn.mdwn
@@ -0,0 +1,11 @@
+* **name**: chrysn
+* **website**: 
+* **uses git-annex for** managing the family's photos (and possibly videos and music in the future).
+* **likes git-annex because** it adds a layer of commit semantics over a regular file system without keeping everything in duplicate locally.
+* **would like git-annex not to** be required any more at all when
+  * git itself learns to use cow filesystems to avoid abundant disk usage, and
+  * git gets better with shallow clones.
+
+    git-annex might then still be a simpler tool that watches over what can be safely dropped from a particular shallow clone
+
+    (the issues with shallow clones seem to relate primarily to shallow history; i haven't read anything about what would happen if all commits were checked out, but not all trees and blobs)
diff --git a/doc/users/clacke.mdwn b/doc/users/clacke.mdwn
new file mode 100644
index 0000000000..c48cd311fc
--- /dev/null
+++ b/doc/users/clacke.mdwn
@@ -0,0 +1,3 @@
+Claes Wallin (韋嘉誠)
+
+[[https://microca.st/clacke]]
diff --git a/doc/users/claes.wallin.mdwn b/doc/users/claes.wallin.mdwn
new file mode 100644
index 0000000000..9e79a5fc9e
--- /dev/null
+++ b/doc/users/claes.wallin.mdwn
@@ -0,0 +1 @@
+[[!meta redir=clacke]]
diff --git a/doc/users/datalad.mdwn b/doc/users/datalad.mdwn
new file mode 100644
index 0000000000..4d33426a2f
--- /dev/null
+++ b/doc/users/datalad.mdwn
@@ -0,0 +1,23 @@
+TODOs for DataLad
+=================
+
+[[!inline pages="todo/* and !todo/done and !link(todo/done) and
+(author(yoh) or author(mih) or author(ben) or author(yarikoptic))" sort=mtime feeds=no actions=yes archive=yes show=0]]
+
+Done
+----
+
+[[!inline pages="todo/* and !todo/done and link(todo/done) and
+(author(yoh) or author(mih) or author(ben) or author(yarikoptic))" feeds=no actions=yes archive=yes show=0]]
+
+My bugs
+=======
+
+[[!inline pages="bugs/* and !bugs/done and !link(bugs/done) and
+(author(yoh) or author(mih) or author(ben) or author(yarikoptic))" sort=mtime feeds=no actions=yes archive=yes show=0  template=buglist]]
+
+Fixed
+-----
+
+[[!inline pages="bugs/* and !bugs/done and link(bugs/done) and
+(author(yoh) or author(mih) or author(ben) or author(yarikoptic))" feeds=no actions=yes archive=yes show=0  template=buglist]]
diff --git a/doc/users/dave.mdwn b/doc/users/dave.mdwn
new file mode 100644
index 0000000000..0a18da5d4e
--- /dev/null
+++ b/doc/users/dave.mdwn
@@ -0,0 +1 @@
+IM DAVE
diff --git a/doc/users/fmarier.mdwn b/doc/users/fmarier.mdwn
new file mode 100644
index 0000000000..d04b6968df
--- /dev/null
+++ b/doc/users/fmarier.mdwn
@@ -0,0 +1,6 @@
+# François Marier
+
+Free Software and Debian Developer. Lead developer of [Libravatar](https://www.libravatar.org)
+
+* [Blog](http://feeding.cloud.geek.nz) and [homepage](http://fmarier.org)
+* [Identica](http://identi.ca/fmarier) / [Twitter](https://twitter.com/fmarier)
diff --git a/doc/users/frederik.mdwn b/doc/users/frederik.mdwn
new file mode 100644
index 0000000000..110e4fd5b5
--- /dev/null
+++ b/doc/users/frederik.mdwn
@@ -0,0 +1 @@
+Hi, I am Frederik Vanrenterghem. I use git annex to manage my documents (mainly to have a backup and easy access) and my collection of ripped audio CDs (mainly to be able to save space on my laptop).
diff --git a/doc/users/gebi.mdwn b/doc/users/gebi.mdwn
new file mode 100644
index 0000000000..121bedbdd7
--- /dev/null
+++ b/doc/users/gebi.mdwn
@@ -0,0 +1 @@
+Michael Gebetsroither 
diff --git a/doc/users/greg.mdwn b/doc/users/greg.mdwn
new file mode 100644
index 0000000000..95873c5750
--- /dev/null
+++ b/doc/users/greg.mdwn
@@ -0,0 +1,3 @@
+Greg Grossmeier
+
+
diff --git a/doc/users/james.mdwn b/doc/users/james.mdwn
new file mode 100644
index 0000000000..4dc6223870
--- /dev/null
+++ b/doc/users/james.mdwn
@@ -0,0 +1 @@
+hello, I am [[James Richardson|http://jamestechnotes.com/]].
diff --git a/doc/users/joey.mdwn b/doc/users/joey.mdwn
new file mode 100644
index 0000000000..2180dd64f9
--- /dev/null
+++ b/doc/users/joey.mdwn
@@ -0,0 +1,2 @@
+Joey Hess   
+
diff --git a/doc/users/mih.mdwn b/doc/users/mih.mdwn
new file mode 100644
index 0000000000..1a0327a400
--- /dev/null
+++ b/doc/users/mih.mdwn
@@ -0,0 +1,23 @@
+My todos
+========
+
+[[!inline pages="todo/* and !todo/done and !link(todo/done) and
+author(mih)" sort=mtime feeds=no actions=yes archive=yes show=0]]
+
+Done
+----
+
+[[!inline pages="todo/* and !todo/done and link(todo/done) and
+author(mih)" feeds=no actions=yes archive=yes show=0]]
+
+My bugs
+=======
+
+[[!inline pages="bugs/* and !bugs/done and !link(bugs/done) and
+author(mih)" sort=mtime feeds=no actions=yes archive=yes show=0  template=buglist]]
+
+Fixed
+-----
+
+[[!inline pages="bugs/* and !bugs/done and link(bugs/done) and
+author(mih)" feeds=no actions=yes archive=yes show=0  template=buglist]]
diff --git a/doc/users/parhuzamos.mdwn b/doc/users/parhuzamos.mdwn
new file mode 100644
index 0000000000..df53229403
--- /dev/null
+++ b/doc/users/parhuzamos.mdwn
@@ -0,0 +1,8 @@
+My life with git-annex
+
+Todos: 
+
+- publish git-annex scripts
+
+[[!inline pages="todo/* and !todo/done and !link(todo/done) and
+link(users/parhuzamos)" sort=mtime feeds=no actions=yes archive=yes show=0]]
diff --git a/doc/users/sameerds.mdwn b/doc/users/sameerds.mdwn
new file mode 100644
index 0000000000..6df5f181c4
--- /dev/null
+++ b/doc/users/sameerds.mdwn
@@ -0,0 +1,3 @@
+Extremely grateful user of git-annex.
+
+[[http://sameer.sbuddhe.net/]]
diff --git a/doc/users/tobiastheviking.mdwn b/doc/users/tobiastheviking.mdwn
new file mode 100644
index 0000000000..31398dae90
--- /dev/null
+++ b/doc/users/tobiastheviking.mdwn
@@ -0,0 +1,13 @@
+Tobias Ussing
+
+See:
+
+* [[tips/flickrannex]] - [[https://github.com/TobiasTheViking/flickrannex/]]
+* [[tips/imapannex]] - [[https://github.com/TobiasTheViking/imapannex]]
+* [[tips/dropboxannex]] - [[https://github.com/TobiasTheViking/dropboxannex]]
+* [[tips/skydriveannex]] - [[https://github.com/TobiasTheViking/skydriveannex]]
+* [[tips/googledriveannex]] - [[https://github.com/TobiasTheViking/googledriveannex]]
+* [[tips/owncloudannex]] - [[https://github.com/TobiasTheViking/owncloudannex]]
+* [[tips/megaannex]] - [[https://github.com/TobiasTheViking/megaannex]]
+* [[forum/nntp__47__usenet_special_remote/]]
+
diff --git a/doc/users/yoh.mdwn b/doc/users/yoh.mdwn
new file mode 100644
index 0000000000..e32e7be67f
--- /dev/null
+++ b/doc/users/yoh.mdwn
@@ -0,0 +1,23 @@
+My todos
+========
+
+[[!inline pages="todo/* and !todo/done and !link(todo/done) and
+author(yoh)" sort=mtime feeds=no actions=yes archive=yes show=0]]
+
+Done
+----
+
+[[!inline pages="todo/* and !todo/done and link(todo/done) and
+author(yoh)" feeds=no actions=yes archive=yes show=0]]
+
+My bugs
+=======
+
+[[!inline pages="bugs/* and !bugs/done and !link(bugs/done) and
+author(yoh)" sort=mtime feeds=no actions=yes archive=yes show=0  template=buglist]]
+
+Fixed
+-----
+
+[[!inline pages="bugs/* and !bugs/done and link(bugs/done) and
+author(yoh)" feeds=no actions=yes archive=yes show=0  template=buglist]]
diff --git a/doc/videos.mdwn b/doc/videos.mdwn
new file mode 100644
index 0000000000..4bfcdbc9fc
--- /dev/null
+++ b/doc/videos.mdwn
@@ -0,0 +1,8 @@
+Talks and screencasts about git-annex.
+
+These videos are also available in a public git-annex repository
+`git clone https://downloads.kitenet.net/.git/`
+
+[[!inline pages="./videos/* and !./videos/*/* and !*/Discussion" show="2"]]
+
+[[!inline pages="./videos/* and !./videos/*/* and !*/Discussion" show="0" archive=yes skip=2 feeds=no]]
diff --git a/doc/videos/FOSDEM2012.mdwn b/doc/videos/FOSDEM2012.mdwn
new file mode 100644
index 0000000000..30d1e37d51
--- /dev/null
+++ b/doc/videos/FOSDEM2012.mdwn
@@ -0,0 +1,7 @@
+
    
+A 15 minute introduction to git-annex,
+presented by Richard Hartmann at FOSDEM 2012.
+
+[[!meta date="1 Jan 2012"]]
+[[!meta title="git-annex presentation by Richard Hartmann at FOSDEM 2012"]]
diff --git a/doc/videos/LCA2013.mdwn b/doc/videos/LCA2013.mdwn
new file mode 100644
index 0000000000..0f29ce0523
--- /dev/null
+++ b/doc/videos/LCA2013.mdwn
@@ -0,0 +1,8 @@
+
    
+A 45 minute talk and demo of git-annex and the assistant), presented by Joey Hess at LCA 2013.
+
+[[!meta date="1 Feb 2013"]]
+[[!meta title="git-annex presentation by Joey Hess at Linux.Conf.Au 2013"]]
diff --git a/doc/videos/git-annex_assistant_archiving.mdwn b/doc/videos/git-annex_assistant_archiving.mdwn
new file mode 100644
index 0000000000..7e891c2f77
--- /dev/null
+++ b/doc/videos/git-annex_assistant_archiving.mdwn
@@ -0,0 +1,5 @@
+
    
+A 9 minute screencast
+covering archiving your files with the [[git-annex assistant|/assistant]].
diff --git a/doc/videos/git-annex_assistant_introduction.mdwn b/doc/videos/git-annex_assistant_introduction.mdwn
new file mode 100644
index 0000000000..93f9df1bad
--- /dev/null
+++ b/doc/videos/git-annex_assistant_introduction.mdwn
@@ -0,0 +1,5 @@
+
    
+A 8 minute screencast
+introducing the [[git-annex assistant|/assistant]].
diff --git a/doc/videos/git-annex_assistant_introduction/comment_1_f42ad4183c2c28319d3705a82fceb82f._comment b/doc/videos/git-annex_assistant_introduction/comment_1_f42ad4183c2c28319d3705a82fceb82f._comment
new file mode 100644
index 0000000000..c969bacbcb
--- /dev/null
+++ b/doc/videos/git-annex_assistant_introduction/comment_1_f42ad4183c2c28319d3705a82fceb82f._comment
@@ -0,0 +1,15 @@
+[[!comment format=mdwn
+ username="modules"
+ ip="85.16.227.39"
+ subject="Great screencast"
+ date="2013-03-16T14:16:37Z"
+ content="""
+I am starting to understand the concept :) Thank you.
+
+The assistant is working perfect with local repos and removable drives but i have problems to setup a remote repos. On a fresh debian server with base setup and rsync installed and ssh-keys for login. I got a green \"Scanned 93.xxx.xx.xxx_annex\" message on dashboard, but there is never a finished transfer. Its \"queued\" on all files and nothing seems to happens on clicking the \"play\" button behind files (x-ing files removes them from dashboard). Also i saw there is a second option in your screencast for remote servers (\"Use a git reposoitory on the server\") which does not show up on my side. Do i need to setup git-annex on server for second option? (with git-annex version 4.20130314).
+
+
+
+
+
+"""]]
diff --git a/doc/videos/git-annex_assistant_introduction/comment_2_b62f4eeeac1138570f7cb8c98d41c2cb._comment b/doc/videos/git-annex_assistant_introduction/comment_2_b62f4eeeac1138570f7cb8c98d41c2cb._comment
new file mode 100644
index 0000000000..8b6d2a1e95
--- /dev/null
+++ b/doc/videos/git-annex_assistant_introduction/comment_2_b62f4eeeac1138570f7cb8c98d41c2cb._comment
@@ -0,0 +1,12 @@
+[[!comment format=mdwn
+ username="http://joeyh.name/"
+ nickname="joey"
+ subject="comment 2"
+ date="2013-03-16T16:04:30Z"
+ content="""
+@modules: Yes, you're only given the option to use a git repository on the server if it has both git and git-annex installed. You can install any version of git-annex there. For example, Debian stable ships with one that will work.
+I plan to make that screen clearer when the git repository option is not available.
+
+I'm not sure what'd going on with your rsync transfers not running. I can say that if the \"play\" icon is visible, the transfer has been paused. While a transfer is running, the \"pause\" icon is visible instead, to let you pause it. 
+This may be as simple as you having misunderstood the icons and paused the currently running transfer, which prevents any transfers from running. If not, suggest you enable debug logs in the Preferences page, and consider filing a [[bug_report|bugs]].
+"""]]
diff --git a/doc/videos/git-annex_assistant_lan.mdwn b/doc/videos/git-annex_assistant_lan.mdwn
new file mode 100644
index 0000000000..aa9245fe56
--- /dev/null
+++ b/doc/videos/git-annex_assistant_lan.mdwn
@@ -0,0 +1,6 @@
+
    
+A 10 minute screencast
+showing how to get started using the [[git-annex assistant|/assistant]],
+including sharing files on a local network, and installation on a server.
diff --git a/doc/videos/git-annex_assistant_lan/comment_1_df8c8b6d9d63fbf5462b225edbb23c82._comment b/doc/videos/git-annex_assistant_lan/comment_1_df8c8b6d9d63fbf5462b225edbb23c82._comment
new file mode 100644
index 0000000000..8c67a5cfbc
--- /dev/null
+++ b/doc/videos/git-annex_assistant_lan/comment_1_df8c8b6d9d63fbf5462b225edbb23c82._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="http://alan.petitepomme.net/"
+ nickname="Alan Schmitt"
+ subject="How is the binary found on the server?"
+ date="2014-06-05T12:05:40Z"
+ content="""
+Great screencast! There is one thing that intrigues me: since you simply uncompress the binary in some folder, how does gitannex figures out where the binary lives there?
+"""]]
diff --git a/doc/videos/git-annex_assistant_lan/comment_2_d4e3122da9c9e27fbe872e09fcde762b._comment b/doc/videos/git-annex_assistant_lan/comment_2_d4e3122da9c9e27fbe872e09fcde762b._comment
new file mode 100644
index 0000000000..c62f1d0160
--- /dev/null
+++ b/doc/videos/git-annex_assistant_lan/comment_2_d4e3122da9c9e27fbe872e09fcde762b._comment
@@ -0,0 +1,10 @@
+[[!comment format=mdwn
+ username="http://joeyh.name/"
+ ip="108.236.230.124"
+ subject="magic"
+ date="2014-06-10T16:13:35Z"
+ content="""
+In the screencast, I run git-annex after unpacking it, and this lets it register where it's installed at.
+
+So it is important to run git-annex after you unpack it.
+"""]]
diff --git a/doc/videos/git-annex_assistant_lan/comment_3_d43ee0a335c2f010b437cf28437455c2._comment b/doc/videos/git-annex_assistant_lan/comment_3_d43ee0a335c2f010b437cf28437455c2._comment
new file mode 100644
index 0000000000..39fb1aefcb
--- /dev/null
+++ b/doc/videos/git-annex_assistant_lan/comment_3_d43ee0a335c2f010b437cf28437455c2._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="http://alan.petitepomme.net/"
+ nickname="Alan Schmitt"
+ subject="Where is it registered?"
+ date="2014-06-11T11:27:02Z"
+ content="""
+I'm curious: how do you register it? Does it only work on Linux? (I've had to add paths to my setup under OS X, so it would be great if there was such a simpler way.)
+"""]]
diff --git a/doc/videos/git-annex_assistant_lan/comment_4_c710e27db41311b157d8caaafc32dc7e._comment b/doc/videos/git-annex_assistant_lan/comment_4_c710e27db41311b157d8caaafc32dc7e._comment
new file mode 100644
index 0000000000..1035f6348c
--- /dev/null
+++ b/doc/videos/git-annex_assistant_lan/comment_4_c710e27db41311b157d8caaafc32dc7e._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="http://joeyh.name/"
+ ip="108.236.230.124"
+ subject="comment 4"
+ date="2014-06-11T19:56:49Z"
+ content="""
+Running git-annex will also register it on OSX. The registration just consists of making a ~/.ssh/git-annex-shell that runs the real git-annex-shell. The assistant detects when it needs to use that wrapper when setting up a repository.
+"""]]
diff --git a/doc/videos/git-annex_assistant_sync_demo.mdwn b/doc/videos/git-annex_assistant_sync_demo.mdwn
new file mode 100644
index 0000000000..2df2a3a91a
--- /dev/null
+++ b/doc/videos/git-annex_assistant_sync_demo.mdwn
@@ -0,0 +1,8 @@
+A screencast demoing the git-annex assistant syncing between Nicaragua
+and the United Kingdom for the first time.
+
+
+
+[video](http://joeyh.name/screencasts/git-annex-assistant.ogg)
+
+[[!meta date="Thu Jul 5 16:36:06 2012 -0600"]]
diff --git a/doc/videos/git-annex_views_demo.mdwn b/doc/videos/git-annex_views_demo.mdwn
new file mode 100644
index 0000000000..c860129fd0
--- /dev/null
+++ b/doc/videos/git-annex_views_demo.mdwn
@@ -0,0 +1,11 @@
+A quick screencast demoing an experimental new feature,
+[[tips/metadata_driven_views]].
+
+
+
+[video](https://downloads.kitenet.net/videos/git-annex/git-annex_views_demo.ogg)
+
+Credits: 
+
+* RichiH for 
+* Michi for keyboard cat cameo
diff --git a/doc/videos/git-annex_watch_demo.mdwn b/doc/videos/git-annex_watch_demo.mdwn
new file mode 100644
index 0000000000..3909f73b5b
--- /dev/null
+++ b/doc/videos/git-annex_watch_demo.mdwn
@@ -0,0 +1,7 @@
+A quick screencast demoing the `git annex watch` daemon.
+
+
+
+[video](http://joeyh.name/screencasts/git-annex-watch.ogg)
+
+[[!meta date="Mon Jun 11 16:02:14 2012 -0400"]]
diff --git a/doc/videos/git-annex_weppapp_demo.mdwn b/doc/videos/git-annex_weppapp_demo.mdwn
new file mode 100644
index 0000000000..b982d32fd4
--- /dev/null
+++ b/doc/videos/git-annex_weppapp_demo.mdwn
@@ -0,0 +1,8 @@
+A quick screencast demoing the early `git annex webapp` and
+automatic USB drive mount detection and syncing.
+
+
+
+[video](http://joeyh.name/screencasts/git-annex-webapp.ogg)
+
+[[!meta date="Sun Jul 29 14:41:41 2012 -0400"]]
diff --git a/doc/walkthrough.mdwn b/doc/walkthrough.mdwn
new file mode 100644
index 0000000000..4745ca1172
--- /dev/null
+++ b/doc/walkthrough.mdwn
@@ -0,0 +1,32 @@
+A walkthrough of some of the basic features of git-annex, using the command
+line. If you don't want to use the command line, see [[assistant/quickstart]]
+instead.
+
+What follows is only one possible [[workflow]] for using git-annex,
+but following along will teach you the basic concepts from the ground up.
+
+[[!toc]]
+
+[[!inline feeds=no trail=yes show=0 template=walkthrough pagenames="""
+	walkthrough/creating_a_repository
+	walkthrough/adding_files
+	walkthrough/adding_a_remote
+	walkthrough/renaming_files
+	walkthrough/getting_file_content
+	walkthrough/syncing
+	walkthrough/transferring_files__58___When_things_go_wrong
+	walkthrough/removing_files
+	walkthrough/removing_files__58___When_things_go_wrong
+	walkthrough/modifying_annexed_files
+	walkthrough/using_ssh_remotes
+	walkthrough/using_special_remotes
+	walkthrough/moving_file_content_between_repositories
+	walkthrough/quiet_please__58___When_git-annex_seems_to_skip_files
+	walkthrough/using_tags_and_branches
+	walkthrough/unused_data
+	walkthrough/fsck__58___verifying_your_data
+	walkthrough/fsck__58___when_things_go_wrong
+	walkthrough/backups
+	walkthrough/automatically_managing_content
+	walkthrough/more
+"""]]
diff --git a/doc/walkthrough/adding_a_remote.mdwn b/doc/walkthrough/adding_a_remote.mdwn
new file mode 100644
index 0000000000..c02852544d
--- /dev/null
+++ b/doc/walkthrough/adding_a_remote.mdwn
@@ -0,0 +1,22 @@
+Like any other git repository, git-annex repositories have remotes.
+Let's start by adding a USB drive as a remote.
+
+	# sudo mount /media/usb
+	# cd /media/usb
+	# git clone ~/annex
+	# cd annex
+	# git annex init "portable USB drive"
+	# git remote add laptop ~/annex
+	# cd ~/annex
+	# git remote add usbdrive /media/usb/annex
+
+This is all standard ad-hoc distributed git repository setup.
+
+The only git-annex specific part is telling it a description
+of the new repository created on the USB drive. This is optional, but
+giving the repository a description helps when git-annex talks about it
+later.
+
+Notice that both repos are set up as remotes of one another. This lets
+either get annexed files from the other. You'll want to do that even
+if you are using git in a more centralized fashion.
diff --git a/doc/walkthrough/adding_a_remote/comment_1_0a59355bd33a796aec97173607e6adc9._comment b/doc/walkthrough/adding_a_remote/comment_1_0a59355bd33a796aec97173607e6adc9._comment
new file mode 100644
index 0000000000..4b0b9c0fd2
--- /dev/null
+++ b/doc/walkthrough/adding_a_remote/comment_1_0a59355bd33a796aec97173607e6adc9._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="https://www.google.com/accounts/o8/id?id=AItOawl9sYlePmv1xK-VvjBdN-5doOa_Xw-jH4U"
+ nickname="Richard"
+ subject="comment 1"
+ date="2011-03-19T01:18:49Z"
+ content="""
+After doing the above with two required copy per file, `git annex fsck` complained that I had only one copy per file even though I had created my clone, already. Once I `git pull`ed from the second repo, not getting any changes for obvious reasons, `git annex fsck` was happy. So I am not sure how my addition was incorrect. -- RichiH
+"""]]
diff --git a/doc/walkthrough/adding_a_remote/comment_2_f8cd79ef1593a8181a7f1086a87713e8._comment b/doc/walkthrough/adding_a_remote/comment_2_f8cd79ef1593a8181a7f1086a87713e8._comment
new file mode 100644
index 0000000000..015417a4f7
--- /dev/null
+++ b/doc/walkthrough/adding_a_remote/comment_2_f8cd79ef1593a8181a7f1086a87713e8._comment
@@ -0,0 +1,9 @@
+[[!comment format=mdwn
+ username="http://joey.kitenet.net/"
+ nickname="joey"
+ subject="comment 2"
+ date="2011-03-19T15:35:38Z"
+ content="""
+Yes, you have to pull down location tracking information in order for fsck to be satisfied in that situation. But since this is a walkthrough, and neither fsck or numcopies settings are mentioned until later, it's ok for this pull to be described a few steps along in [[getting file content]].
+
+"""]]
diff --git a/doc/walkthrough/adding_a_remote/comment_3_60691af4400521b5a8c8d75efe3b44cb._comment b/doc/walkthrough/adding_a_remote/comment_3_60691af4400521b5a8c8d75efe3b44cb._comment
new file mode 100644
index 0000000000..9280f2dccf
--- /dev/null
+++ b/doc/walkthrough/adding_a_remote/comment_3_60691af4400521b5a8c8d75efe3b44cb._comment
@@ -0,0 +1,9 @@
+[[!comment format=mdwn
+ username="http://dieter-be.myopenid.com/"
+ nickname="dieter"
+ subject="comment 3"
+ date="2011-04-02T20:24:33Z"
+ content="""
+ * why the `git remote add laptop ~/annex` ? this remote already exists under the name origin.
+ * doesn't the last command need to be `git remote add usbdrive /media/usb/annex`? because the actual repo would be in /media/usb/annex, not /media/usb?
+"""]]
diff --git a/doc/walkthrough/adding_a_remote/comment_4_6f7cf5c330272c96b3abeb6612075c9d._comment b/doc/walkthrough/adding_a_remote/comment_4_6f7cf5c330272c96b3abeb6612075c9d._comment
new file mode 100644
index 0000000000..b4dcb6422a
--- /dev/null
+++ b/doc/walkthrough/adding_a_remote/comment_4_6f7cf5c330272c96b3abeb6612075c9d._comment
@@ -0,0 +1,10 @@
+[[!comment format=mdwn
+ username="http://joey.kitenet.net/"
+ nickname="joey"
+ subject="comment 4"
+ date="2011-04-03T02:32:17Z"
+ content="""
+Good spotting on the last line, fixed.
+
+The laptop remote is indeed redundant, but it leads to clearer views of what is going on later in the walkthrough (\"git pull laptop master\", \"(copying from laptop...)\"). And if the original clone is made from a central bare repo, this reinforces that you'll want to set up remotes for other repos on the computer.
+"""]]
diff --git a/doc/walkthrough/adding_a_remote/comment_5_bcbe480aa710ce693ee03d86884f1820._comment b/doc/walkthrough/adding_a_remote/comment_5_bcbe480aa710ce693ee03d86884f1820._comment
new file mode 100644
index 0000000000..bb8271a841
--- /dev/null
+++ b/doc/walkthrough/adding_a_remote/comment_5_bcbe480aa710ce693ee03d86884f1820._comment
@@ -0,0 +1,10 @@
+[[!comment format=mdwn
+ username="elmimmo"
+ avatar="http://cdn.libravatar.org/avatar/4f00dfc3ad590ef7492788b854ceba78"
+ subject="Remotes when using multiple repos"
+ date="2018-01-18T07:02:18Z"
+ content="""
+I see that you manually add the first repo as a remote of the external drive, and the external drive as a remote of the original.
+
+What about when you add a second external drive? Should you then be adding the first two repos as remotes of the new one, and then go to the first two ones and add the new drive as a remote of them too? Doesn't this permutation scale out of control when you add a new drive to, say, a git annex made of 10 disks, 20 disks, etc?
+"""]]
diff --git a/doc/walkthrough/adding_a_remote/comment_6_15ea28114e69dddac82fddaf009079ca._comment b/doc/walkthrough/adding_a_remote/comment_6_15ea28114e69dddac82fddaf009079ca._comment
new file mode 100644
index 0000000000..a57115c80c
--- /dev/null
+++ b/doc/walkthrough/adding_a_remote/comment_6_15ea28114e69dddac82fddaf009079ca._comment
@@ -0,0 +1,9 @@
+[[!comment format=mdwn
+ username="joey"
+ subject="""comment 6"""
+ date="2018-03-02T19:09:36Z"
+ content="""
+@elmimmo, you could add all those permutations, but there's no need to
+connect repositories that you don't need git-annex to transfer data
+between.
+"""]]
diff --git a/doc/walkthrough/adding_files.mdwn b/doc/walkthrough/adding_files.mdwn
new file mode 100644
index 0000000000..b014c3ee70
--- /dev/null
+++ b/doc/walkthrough/adding_files.mdwn
@@ -0,0 +1,12 @@
+	# cd ~/annex
+	# cp /tmp/big_file .
+	# cp /tmp/debian.iso .
+	# git annex add .
+	add big_file (checksum...) ok
+	add debian.iso (checksum...) ok
+	# git commit -a -m added
+
+When you add a file to the annex and commit it, only a symlink to
+the content is committed to git. The content itself is stored in
+git-annex's backend, `.git/annex/` (or in [[direct_mode]] the file
+is left as-is).
diff --git a/doc/walkthrough/automatically_managing_content.mdwn b/doc/walkthrough/automatically_managing_content.mdwn
new file mode 100644
index 0000000000..ec55c1cc8b
--- /dev/null
+++ b/doc/walkthrough/automatically_managing_content.mdwn
@@ -0,0 +1,45 @@
+Once you have multiple repositories, and have perhaps configured numcopies,
+any given file can have many more copies than is needed, or perhaps fewer
+than you would like. How to manage this?
+
+The whereis subcommand can be used to see how many copies of a file are known,
+but then you have to decide what to get or drop. In this example, there
+are perhaps not enough copies of the first file, and too many of the second
+file.
+
+	# cd /media/usbdrive
+	# git annex whereis
+	whereis my_cool_big_file (1 copy)
+		0c443de8-e644-11df-acbf-f7cd7ca6210d  -- laptop
+	whereis other_file (3 copies)
+		0c443de8-e644-11df-acbf-f7cd7ca6210d  -- laptop
+		62b39bbe-4149-11e0-af01-bb89245a1e61  -- usb drive [here]
+		7570b02e-15e9-11e0-adf0-9f3f94cb2eaa  -- backup drive
+
+What would be handy is some automated versions of get and drop, that only
+gets a file if there are not yet enough copies of it, or only drops a file
+if there are too many copies. Well, these exist, just use the --auto option.
+
+	# git annex get --auto --numcopies=2
+	get my_cool_big_file (from laptop...) ok
+	# git annex drop --auto --numcopies=2
+	drop other_file ok
+
+With two quick commands, git-annex was able to decide for you how to
+work toward having two copies of your files.
+
+	# git annex whereis
+	whereis my_cool_big_file (2 copies)
+		0c443de8-e644-11df-acbf-f7cd7ca6210d  -- laptop
+		62b39bbe-4149-11e0-af01-bb89245a1e61  -- usb drive [here]
+	whereis other_file (2 copies)
+		0c443de8-e644-11df-acbf-f7cd7ca6210d  -- laptop
+		7570b02e-15e9-11e0-adf0-9f3f94cb2eaa  -- backup drive
+
+The --auto option can also be used with the copy command,
+again this lets git-annex decide whether to actually copy content.
+
+The above shows how to use --auto to manage content based on the number
+of copies. It's also possible to configure, on a per-repository basis,
+which content is desired. Then --auto also takes that into account
+see [[preferred_content]] for details.
diff --git a/doc/walkthrough/backups.mdwn b/doc/walkthrough/backups.mdwn
new file mode 100644
index 0000000000..223e0b2eef
--- /dev/null
+++ b/doc/walkthrough/backups.mdwn
@@ -0,0 +1,27 @@
+git-annex can be configured to require that more than one copy of a file exists,
+as a simple backup for your data. This is controlled by the
+numcopies setting, which defaults to 1 copy. Let's
+change that to require 2 copies, and send a copy of every file
+to a USB drive.
+
+	# git annex numcopies 2
+	# git annex copy . --to usbdrive
+
+Now when we try to `git annex drop` a file, it will verify that it
+knows of 2 other repositories that have a copy before removing its
+content from the current repository.
+
+The numcopies setting used above is the global default.
+You can also vary the number of copies needed, depending on the file name.
+So, if you want 3 copies of all your flac files, but only 1 copy of oggs:
+
+	# echo "*.ogg annex.numcopies=1" >> .gitattributes
+	# echo "*.flac annex.numcopies=3" >> .gitattributes
+
+Or, you might want to make a directory for important stuff, and configure
+it so anything put in there is backed up more thoroughly:
+
+	# mkdir important_stuff
+	# echo "* annex.numcopies=3" > important_stuff/.gitattributes
+
+For more details about the numcopies setting, see [[copies]].
diff --git a/doc/walkthrough/backups/comment_1_d0244791d2abbf29553546a6a6568a0f._comment b/doc/walkthrough/backups/comment_1_d0244791d2abbf29553546a6a6568a0f._comment
new file mode 100644
index 0000000000..466b3d3694
--- /dev/null
+++ b/doc/walkthrough/backups/comment_1_d0244791d2abbf29553546a6a6568a0f._comment
@@ -0,0 +1,10 @@
+[[!comment format=mdwn
+ username="madduck"
+ ip="2001:a60:f0fb:0:224:d7ff:fe04:c82c"
+ subject="Warn while inconsistent"
+ date="2014-04-06T20:44:17Z"
+ content="""
+Sure, git-annex prevents me from dropping files unless there are numcopies around elsewhere, but shouldn't it also ensure that numcopies cannot be set unless that requirement is already met?
+
+Furthermore, shouldn't it ensure that when new files are added, they are automatically distributed to fulfill the requirement?
+"""]]
diff --git a/doc/walkthrough/creating_a_repository.mdwn b/doc/walkthrough/creating_a_repository.mdwn
new file mode 100644
index 0000000000..b5b3ab4078
--- /dev/null
+++ b/doc/walkthrough/creating_a_repository.mdwn
@@ -0,0 +1,6 @@
+This is very straightforward.
+
+	# mkdir ~/annex
+	# cd ~/annex
+	# git init
+	# git annex init
diff --git a/doc/walkthrough/fsck__58___verifying_your_data.mdwn b/doc/walkthrough/fsck__58___verifying_your_data.mdwn
new file mode 100644
index 0000000000..2400146108
--- /dev/null
+++ b/doc/walkthrough/fsck__58___verifying_your_data.mdwn
@@ -0,0 +1,36 @@
+You can use the fsck subcommand to check for problems in your data. What
+can be checked depends on the key-value [[backend|backends]] you've used
+for the data. For example, when you use the SHA1 backend, fsck will verify
+that the checksums of your files are good. Fsck also checks that the
+[[numcopies|copies]] setting is satisfied for all files.
+
+	# git annex fsck
+	fsck some_file (checksum...) ok
+	fsck my_cool_big_file (checksum...) ok
+	...
+
+You can also specify the files to check.  This is particularly useful if 
+you're using sha1 and don't want to spend a long time checksumming everything.
+
+	# git annex fsck my_cool_big_file
+	fsck my_cool_big_file (checksum...) ok
+
+If you have a large repo, you may want to check it in smaller steps. You may
+start and continue an aborted or time-limited check.
+
+	# git annex fsck -S  --time-limit=1m
+	fsck some_file (checksum...) ok
+	fsck my_cool_big_file (checksum...) ok
+	
+	  Time limit (1m) reached!
+
+	# git annex fsck -m 
+	fsck my_other_big_file (checksum...) ok
+	...
+
+Use `-S` or `--incremental` to start the incremental check. Use `-m` 
+or `--more` to continue the started check and continue where it left 
+off. Note that saving the progress of `fsck` is performed after every 
+1000 files or 5 minutes or when `--time-limit` occours. There may be 
+files that will be checked again when `git-annex` exists abnormally 
+eg. Ctrl+C and the check is restarted.
diff --git a/doc/walkthrough/fsck__58___when_things_go_wrong.mdwn b/doc/walkthrough/fsck__58___when_things_go_wrong.mdwn
new file mode 100644
index 0000000000..85d9f20fe0
--- /dev/null
+++ b/doc/walkthrough/fsck__58___when_things_go_wrong.mdwn
@@ -0,0 +1,13 @@
+Fsck never deletes possibly bad data; instead it will be moved to
+`.git/annex/bad/` for you to recover. Here is a sample of what fsck
+might say about a badly messed up annex:
+
+	# git annex fsck
+	fsck my_cool_big_file (checksum...)
+	git-annex: Bad file content; moved to .git/annex/bad/SHA1:7da006579dd64330eb2456001fd01948430572f2
+	git-annex: ** No known copies exist of my_cool_big_file
+	failed
+	fsck important_file
+	git-annex: Only 1 of 2 copies exist. Run git annex get somewhere else to back it up.
+	failed
+	git-annex: 2 failed
diff --git a/doc/walkthrough/getting_file_content.mdwn b/doc/walkthrough/getting_file_content.mdwn
new file mode 100644
index 0000000000..f92704ff39
--- /dev/null
+++ b/doc/walkthrough/getting_file_content.mdwn
@@ -0,0 +1,12 @@
+A repository does not always have all annexed file contents available.
+When you need the content of a file, you can use "git annex get" to
+make it available.
+
+We can use this to copy everything in the laptop's annex to the
+USB drive.
+
+	# cd /media/usb/annex
+	# git annex sync laptop
+	# git annex get .
+	get my_cool_big_file (from laptop...) ok
+	get iso/debian.iso (from laptop...) ok
diff --git a/doc/walkthrough/modifying_annexed_files.mdwn b/doc/walkthrough/modifying_annexed_files.mdwn
new file mode 100644
index 0000000000..a35978d822
--- /dev/null
+++ b/doc/walkthrough/modifying_annexed_files.mdwn
@@ -0,0 +1,33 @@
+Normally, the content of files in the annex is prevented from being modified.
+(Unless your repository is using [[direct_mode]].)
+
+That's a good thing, because it might be the only copy, you wouldn't
+want to lose it in a fumblefingered mistake.
+
+	# echo oops > my_cool_big_file
+	bash: my_cool_big_file: Permission denied
+
+In order to modify a file, it should first be unlocked.
+
+	# git annex unlock my_cool_big_file
+	unlock my_cool_big_file (copying...) ok
+
+That replaces the symlink that normally points at its content with a copy
+of the content. You can then modify the file like any regular file. Because
+it is a regular file.
+
+(If you decide you don't need to modify the file after all, or want to discard
+modifications, just use `git annex lock`.)
+
+When you `git commit` it will notice that you are committing an unlocked
+file, add its new content to the annex, and a pointer to that content is
+what gets committed to git.
+
+	# echo "now smaller, but even cooler" > my_cool_big_file
+	# git commit my_cool_big_file -m "changed an annexed file"
+	add my_cool_big_file ok
+	[master 64cda67] changed an annexed file
+	 1 files changed, 1 insertions(+), 1 deletions(-)
+
+For more details on working with unlocked files vs the regular locked
+files, see [[tips/unlocked_files]].
diff --git a/doc/walkthrough/modifying_annexed_files/comment_1_624b4a0b521b553d68ab6049f7dbaf8c._comment b/doc/walkthrough/modifying_annexed_files/comment_1_624b4a0b521b553d68ab6049f7dbaf8c._comment
new file mode 100644
index 0000000000..5fc26496e0
--- /dev/null
+++ b/doc/walkthrough/modifying_annexed_files/comment_1_624b4a0b521b553d68ab6049f7dbaf8c._comment
@@ -0,0 +1,14 @@
+[[!comment format=mdwn
+ username="http://lj.rossia.org/users/imz/"
+ ip="79.165.56.162"
+ subject="sorta why git-annex exists in the first place -- not only the slow index "
+ date="2012-09-25T22:04:01Z"
+ content="""
+> Git wants to first stage the entire contents of the file in its index. That can be slow for big files (sorta why git-annex exists in the first place)
+
+I think that git-annex's usefulness is not only because of the Git's index overhead: I like its idea because it will help track the copies in the \"[[special remotes]]\", which are not Git because 
+
+* they are either not under my control (e.g., web URLs), 
+* or because it's not convenient to hold a Git repo there (an external disk/DVD with files can be viewed easily by a human, but imposing a Git repo structure on it would either at least double the consume space: for the history of the commits and for the working dir, or make it \"unreadable\" for a human, if it is a bare repo);
+* or because it's nearly impossible to put a Git repo on a storage like peer networks without special tools.
+"""]]
diff --git a/doc/walkthrough/modifying_annexed_files/comment_2_b000622304535d32b69db17d51156b21._comment b/doc/walkthrough/modifying_annexed_files/comment_2_b000622304535d32b69db17d51156b21._comment
new file mode 100644
index 0000000000..6a2d2c2a60
--- /dev/null
+++ b/doc/walkthrough/modifying_annexed_files/comment_2_b000622304535d32b69db17d51156b21._comment
@@ -0,0 +1,10 @@
+[[!comment format=mdwn
+ username="http://yarikoptic.myopenid.com/"
+ nickname="site-myopenid"
+ subject="feature request: unlock --drop"
+ date="2013-05-30T20:26:52Z"
+ content="""
+ATM unlock copies original file for modifications, so that original copy remains under annex and possibly to-be-edited copy created.
+But if I am \"unlock\"ing file I might simply not be interested in a previous copy and want to maintain only a single (possibly edited) new copy.
+What if there was a mode where the actual file is simply moved into \"unlocked\" location for editing, thus effectively dropping the actual content from git annex.  That would allow to not inquire lengthy copying/wasting local space.  If then I would need a previous copy I would just \"get\" it again.
+"""]]
diff --git a/doc/walkthrough/more.mdwn b/doc/walkthrough/more.mdwn
new file mode 100644
index 0000000000..0a4a5b94e8
--- /dev/null
+++ b/doc/walkthrough/more.mdwn
@@ -0,0 +1,3 @@
+So ends the walkthrough. By now you should be able to use git-annex.
+
+Want more? See [[tips]] for lots more features and advice.
diff --git a/doc/walkthrough/moving_file_content_between_repositories.mdwn b/doc/walkthrough/moving_file_content_between_repositories.mdwn
new file mode 100644
index 0000000000..3ffcc11750
--- /dev/null
+++ b/doc/walkthrough/moving_file_content_between_repositories.mdwn
@@ -0,0 +1,13 @@
+Often you will want to move some file contents from a repository to some
+other one. For example, your laptop's disk is getting full; time to move
+some files to an external disk before moving another file from a file
+server to your laptop. Doing that by hand (by using `git annex get` and
+`git annex drop`) is possible, but a bit of a pain. `git annex move`
+makes it very easy.
+
+	# git annex move my_cool_big_file --to usbdrive
+	move my_cool_big_file (to usbdrive...) ok
+	# git annex move video/hackity_hack_and_kaxxt.mov --from fileserver
+	move video/hackity_hack_and_kaxxt.mov (from fileserver...)
+	SHA256-s86050597--6ae2688bc533437766a48aa19f2c06be14d1bab9c70b468af445d4f07b65f41e   100%   82MB 199.1KB/s   07:02
+	ok
diff --git a/doc/walkthrough/moving_file_content_between_repositories/comment_1_4c30ade91fc7113a95960aa3bd1d5427._comment b/doc/walkthrough/moving_file_content_between_repositories/comment_1_4c30ade91fc7113a95960aa3bd1d5427._comment
new file mode 100644
index 0000000000..b3dc8fe7a2
--- /dev/null
+++ b/doc/walkthrough/moving_file_content_between_repositories/comment_1_4c30ade91fc7113a95960aa3bd1d5427._comment
@@ -0,0 +1,19 @@
+[[!comment format=mdwn
+ username="https://www.google.com/accounts/o8/id?id=AItOawl9sYlePmv1xK-VvjBdN-5doOa_Xw-jH4U"
+ nickname="Richard"
+ subject="comment 1"
+ date="2011-03-22T23:41:51Z"
+ content="""
+I may be missing something obvious, but when I copy to a remote repository, the object files are created, but no softlinks are created. When I pull everything from the remote, it pulls only files the local repo knows about already.
+
+        A
+       / \
+      B   C
+
+Moving from B to A creates no symlinks in A but the object files are moved to A. Copying back from A to B restores the object files in B and keeps them in A.
+
+Copying from A to an empty C does not create any object files nor symlinks. Copying from C to A creates no symlinks in A but the object files are copied to A.
+
+-- RichiH
+
+"""]]
diff --git a/doc/walkthrough/moving_file_content_between_repositories/comment_2_7d90e1e150e7524ba31687108fcc38d6._comment b/doc/walkthrough/moving_file_content_between_repositories/comment_2_7d90e1e150e7524ba31687108fcc38d6._comment
new file mode 100644
index 0000000000..a6f8e9cf97
--- /dev/null
+++ b/doc/walkthrough/moving_file_content_between_repositories/comment_2_7d90e1e150e7524ba31687108fcc38d6._comment
@@ -0,0 +1,10 @@
+[[!comment format=mdwn
+ username="http://joey.kitenet.net/"
+ nickname="joey"
+ subject="comment 2"
+ date="2011-03-23T00:38:10Z"
+ content="""
+`git annex move` only moves content. All symlink management is handled by git, so you have to keep repositories in sync using git as you would any other repo. When you `git pull B` in A, it will get whatever symlinks were added to B.
+
+(It can be useful to use a central bare repo and avoid needing to git pull from one repo to another, then you can just always push commits to the central repo, and pull down all changes from other repos.)
+"""]]
diff --git a/doc/walkthrough/moving_file_content_between_repositories/comment_3_558d80384434207b9cfc033763863de3._comment b/doc/walkthrough/moving_file_content_between_repositories/comment_3_558d80384434207b9cfc033763863de3._comment
new file mode 100644
index 0000000000..9a128f1ed6
--- /dev/null
+++ b/doc/walkthrough/moving_file_content_between_repositories/comment_3_558d80384434207b9cfc033763863de3._comment
@@ -0,0 +1,12 @@
+[[!comment format=mdwn
+ username="https://www.google.com/accounts/o8/id?id=AItOawl9sYlePmv1xK-VvjBdN-5doOa_Xw-jH4U"
+ nickname="Richard"
+ subject="comment 3"
+ date="2011-03-23T02:07:49Z"
+ content="""
+Ah yes, I feel kinda stupid in hindsight.
+
+As the central server is most likely a common use case, would you object if I added that to the walkthrough? If you have any best practices on how to automate a push with every copy to a bare remote? AFAIK, git does not store information about bare/non-bare remotes, but this could easily be put into .git/config by git annex.
+
+-- RichiH
+"""]]
diff --git a/doc/walkthrough/moving_file_content_between_repositories/comment_4_a2f343eceed9e9fba1670f21e0fc6af4._comment b/doc/walkthrough/moving_file_content_between_repositories/comment_4_a2f343eceed9e9fba1670f21e0fc6af4._comment
new file mode 100644
index 0000000000..8b4d9a0538
--- /dev/null
+++ b/doc/walkthrough/moving_file_content_between_repositories/comment_4_a2f343eceed9e9fba1670f21e0fc6af4._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="http://joey.kitenet.net/"
+ nickname="joey"
+ subject="comment 4"
+ date="2011-03-23T15:28:00Z"
+ content="""
+I would not mind if the walkthrough documented the central git repo case. But I don't want to complicate it unduely (it's long enough), and it's important that the fully distributed case be shown to work, and I assume that people already have basic git knowledge, so documenting the details of set up of a bare git repo is sorta out of scope. (There are also a lot of way to do it, using github, or gitosis, or raw git, etc.)
+"""]]
diff --git a/doc/walkthrough/quiet_please__58___When_git-annex_seems_to_skip_files.mdwn b/doc/walkthrough/quiet_please__58___When_git-annex_seems_to_skip_files.mdwn
new file mode 100644
index 0000000000..188ca634ba
--- /dev/null
+++ b/doc/walkthrough/quiet_please__58___When_git-annex_seems_to_skip_files.mdwn
@@ -0,0 +1,27 @@
+One behavior of git-annex is sometimes confusing at first, but it turns out
+to be useful once you get to know it.
+
+	# git annex drop *
+	# 
+
+Why didn't git-annex seem to do anything despite being asked to drop all the
+files? Because it checked them all, and none of them are present.
+
+Most git-annex commands will behave this way when they're able to quickly
+check that nothing needs to be done about a file.
+
+Running a git-annex command without specifying any file name will
+make git-annex look for files in the current directory and its
+subdirectories. So, we can add all new files to the annex easily:
+
+	# echo hi > subdir/subsubdir/newfile
+	# git annex add
+	add subdir/subsubdir/newfile ok
+
+When doing this kind of thing, having nothing shown for files
+that it doesn't need to act on is useful because it prevents swamping
+you with output. You only see the files it finds it does need to act on.
+
+So remember: If git-annex seems to not do anything when you tell it to, it's
+not being lazy -- It's checked that nothing needs to be done to get to the
+state you asked for!
diff --git a/doc/walkthrough/removing_files.mdwn b/doc/walkthrough/removing_files.mdwn
new file mode 100644
index 0000000000..9b85d9c3ba
--- /dev/null
+++ b/doc/walkthrough/removing_files.mdwn
@@ -0,0 +1,17 @@
+When you're using git-annex you can `git rm` a file just like you usually
+would with git. Just like with git, this removes the file from your work
+tree, but it does not remove the file's content from the git repository.
+If you check the file back out, or revert the removal, you can get it back.
+
+Git-annex adds the ability to remove the content of a file from your local
+repository to save space. This is called "dropping" the file. 
+
+You can always drop files safely. Git-annex checks that some other
+repository still has the file before removing it.
+
+	# git annex drop iso/debian.iso
+	drop iso/Debian_5.0.iso ok
+
+Once dropped, the file will still appear in your work tree as a broken symlink.
+You can use `git annex get` to as usual to get this file back to your local
+repository.
diff --git a/doc/walkthrough/removing_files/comment_1_cb65e7c510b75be1c51f655b058667c6._comment b/doc/walkthrough/removing_files/comment_1_cb65e7c510b75be1c51f655b058667c6._comment
new file mode 100644
index 0000000000..1c8719cecd
--- /dev/null
+++ b/doc/walkthrough/removing_files/comment_1_cb65e7c510b75be1c51f655b058667c6._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="DavidEdmondson"
+ subject="Is it necessary to commit after the 'drop'?"
+ date="2011-09-05T15:43:25Z"
+ content="""
+In fact is it possible? Nothing changed as far as git is concerned.
+
+"""]]
diff --git a/doc/walkthrough/removing_files/comment_2_64709ea4558915edd5c8ca4486965b07._comment b/doc/walkthrough/removing_files/comment_2_64709ea4558915edd5c8ca4486965b07._comment
new file mode 100644
index 0000000000..f5fb8dc7f5
--- /dev/null
+++ b/doc/walkthrough/removing_files/comment_2_64709ea4558915edd5c8ca4486965b07._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="http://joey.kitenet.net/"
+ nickname="joey"
+ subject="comment 2"
+ date="2011-09-05T15:59:27Z"
+ content="""
+Good catch. It used to be necessary before there was a git-annex branch, but not now.
+"""]]
diff --git a/doc/walkthrough/removing_files/comment_3_91e995867a731325dfdf3331f69a238c._comment b/doc/walkthrough/removing_files/comment_3_91e995867a731325dfdf3331f69a238c._comment
new file mode 100644
index 0000000000..abea8c740c
--- /dev/null
+++ b/doc/walkthrough/removing_files/comment_3_91e995867a731325dfdf3331f69a238c._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="fbicknel@01ede624a1a56b3998b823e9b60da0ff81cccb16"
+ nickname="fbicknel"
+ subject="Complete removal"
+ date="2015-12-10T16:16:43Z"
+ content="""
+So, and I hope this isn't too Captain Obvious, if we drop the file at each repo, we essentially remove it from existence as far as this git-annex cluster is concerned?
+"""]]
diff --git a/doc/walkthrough/removing_files/comment_4_9b575f7f2b67b36599890dd5455dcf47._comment b/doc/walkthrough/removing_files/comment_4_9b575f7f2b67b36599890dd5455dcf47._comment
new file mode 100644
index 0000000000..45cd29937f
--- /dev/null
+++ b/doc/walkthrough/removing_files/comment_4_9b575f7f2b67b36599890dd5455dcf47._comment
@@ -0,0 +1,9 @@
+[[!comment format=mdwn
+ username="http://joeyh.name/"
+ subject="comment 4"
+ date="2015-12-10T18:58:46Z"
+ content="""
+Correct, dropping a file from everywhere will lose its content entirely.
+
+But, git-annex has a [[copies]] tracking feature that prevents such foot-shooting. If you ask it to drop the last copy, it will refuse, although there is a way to override this if you really want to.
+"""]]
diff --git a/doc/walkthrough/removing_files__58___When_things_go_wrong.mdwn b/doc/walkthrough/removing_files__58___When_things_go_wrong.mdwn
new file mode 100644
index 0000000000..ccd2d197f5
--- /dev/null
+++ b/doc/walkthrough/removing_files__58___When_things_go_wrong.mdwn
@@ -0,0 +1,24 @@
+Before dropping a file, git-annex wants to be able to look at other
+remotes, and verify that they still have a file. After all, it could
+have been dropped from them too. If the remotes are not mounted/available,
+you'll see something like this.
+
+	# git annex drop important_file other.iso
+	drop important_file (unsafe)
+	  Could only verify the existence of 0 out of 1 necessary copies
+	  Unable to access these remotes: usbdrive
+	  Try making some of these repositories available:
+	   	58d84e8a-d9ae-11df-a1aa-ab9aa8c00826  -- portable USB drive
+	   	ca20064c-dbb5-11df-b2fe-002170d25c55  -- backup SATA drive
+	  (Use --force to override this check, or adjust numcopies.)
+	failed
+	drop other.iso (unsafe)
+	  Could only verify the existence of 0 out of 1 necessary copies
+          No other repository is known to contain the file.
+	  (Use --force to override this check, or adjust numcopies.)
+	failed
+
+Here you might --force it to drop `important_file` if you [[trust]] your backup.
+But `other.iso` looks to have never been copied to anywhere else, so if
+it's something you want to hold onto, you'd need to transfer it to
+some other repository before dropping it.
diff --git a/doc/walkthrough/renaming_files.mdwn b/doc/walkthrough/renaming_files.mdwn
new file mode 100644
index 0000000000..8dec44d8b8
--- /dev/null
+++ b/doc/walkthrough/renaming_files.mdwn
@@ -0,0 +1,17 @@
+	# cd ~/annex
+	# git mv big_file my_cool_big_file
+	# mkdir iso
+	# git mv debian.iso iso/
+	# git commit -m moved
+
+You can use any normal git operations to move files around, or even
+make copies or delete them.
+
+Notice that, since annexed files are represented by symlinks,
+the symlink will break when the file is moved into a subdirectory.
+But, git-annex will fix this up for you when you commit --
+it has a pre-commit hook that watches for and corrects broken symlinks.
+
+(Note that if a repository is in direct mode, you can't run normal git
+commands in it. Instead, just move the files using non-git commands, and
+`git annex add` and `git annex sync`.)
diff --git a/doc/walkthrough/renaming_files/comment_1_c05f70079c265d23e882f92aaa4d28f8._comment b/doc/walkthrough/renaming_files/comment_1_c05f70079c265d23e882f92aaa4d28f8._comment
new file mode 100644
index 0000000000..9173399aef
--- /dev/null
+++ b/doc/walkthrough/renaming_files/comment_1_c05f70079c265d23e882f92aaa4d28f8._comment
@@ -0,0 +1,31 @@
+[[!comment format=mdwn
+ username="ptspts@d0db5f2b8c4e6befae8899f31c117d093913f43e"
+ nickname="ptspts"
+ subject="renaming oesn't work as expected when changing the extension"
+ date="2015-10-15T08:54:08Z"
+ content="""
+I tried to rename a `.gif` file to `.jpg`:
+
+    $ git annex version
+    git-annex version: 5.20140127
+    build flags: S3 DBus Feeds Quvi TDFA CryptoHash
+    key/value backends: SHA256E SHA1E SHA512E SHA224E SHA384E SKEIN256E SKEIN512E SHA256 SHA1 SHA512 SHA224 SHA384 SKEIN256 SKEIN512 WORM URL
+    remote types: git gcrypt S3 bup directory rsync web tahoe glacier hook external
+    local repository version: 5
+    supported repository version: 5
+    upgrade supported from repository versions: 0 1 2 4
+    $ find .git/annex/objects -type f
+    .git/annex/objects/JW/wq/SHA256E-s231374848--01ca5e0bb09d068f15ccbf9b064b72f573198bd7516ff8edc72e455a09bf9bd4.gif/SHA256E-s231374848--01ca5e0bb09d068f15ccbf9b064b72f573198bd7516ff8edc72e455a09bf9bd4.gif
+    $ git mv foo.gif foo.jpg
+    $ git commit -a -m renamed
+    ...
+    $ git annex fsck
+    ...
+    $ find .git/annex/objects -type f
+    .git/annex/objects/JW/wq/SHA256E-s231374848--01ca5e0bb09d068f15ccbf9b064b72f573198bd7516ff8edc72e455a09bf9bd4.gif/SHA256E-s231374848--01ca5e0bb09d068f15ccbf9b064b72f573198bd7516ff8edc72e455a09bf9bd4.gif
+    $ ls -l foo.jpg
+    lrwxrwxrwx 1 pts pts 202 Oct 15 10:36 foo.jpg -> .git/annex/objects/JW/wq/SHA256E-s231374848--01ca5e0bb09d068f15ccbf9b064b72f573198bd7516ff8edc72e455a09bf9bd4.gif/SHA256E-s231374848--01ca5e0bb09d068f15ccbf9b064b72f573198bd7516ff8edc72e455a09bf9bd4.gif
+
+I would expect that the last extension printed is .jpg, rather than .gif. How do I make git-annex fix that, preferably without calling `git annex fsck`, which is very slow, because it rehashes the file?
+
+"""]]
diff --git a/doc/walkthrough/renaming_files/comment_2_fe83c600e080131b3109fe616940a25e._comment b/doc/walkthrough/renaming_files/comment_2_fe83c600e080131b3109fe616940a25e._comment
new file mode 100644
index 0000000000..7afbd1e03a
--- /dev/null
+++ b/doc/walkthrough/renaming_files/comment_2_fe83c600e080131b3109fe616940a25e._comment
@@ -0,0 +1,9 @@
+[[!comment format=mdwn
+ username="CandyAngel"
+ subject="comment 2"
+ date="2015-10-15T10:59:59Z"
+ content="""
+*migrate* takes the *--force* option (introduced 5.20150327), allowing you to migrate files to the backend they are already using, but using the new extension. It was actually added for this use case!
+
+This may still incur a hash of the file though as, I believe, *migrate* will check the file at the old key is valid. You will also end up with 2 versions of the file in .git/annex/objects though (you can clean it up with *unused*).
+"""]]
diff --git a/doc/walkthrough/renaming_files/comment_3_6b579a4f2990a28b33a8170ebd39da5c._comment b/doc/walkthrough/renaming_files/comment_3_6b579a4f2990a28b33a8170ebd39da5c._comment
new file mode 100644
index 0000000000..a5edbcf2d8
--- /dev/null
+++ b/doc/walkthrough/renaming_files/comment_3_6b579a4f2990a28b33a8170ebd39da5c._comment
@@ -0,0 +1,15 @@
+[[!comment format=mdwn
+ username="joey"
+ subject="""comment 3"""
+ date="2015-10-15T18:46:50Z"
+ content="""
+This is basically the same as thinking that git should change the SHA1
+used to refer to a file in the repository just because it was renamed.
+That does not normally make sense.
+
+Including the original extension in the git-annex key is done to avoid
+confusing some poorly-behaved programs that expect to follow the symlink
+and find a file with an extension. It's best to ignore the extension part
+of the key, or if you like, you can change the [[backend|doc/backends]] from SHA256E to
+SHA256, and then new keys won't include that extension.
+"""]]
diff --git a/doc/walkthrough/renaming_files/comment_4_1cf4cb6be73223755cb611ec097d1498._comment b/doc/walkthrough/renaming_files/comment_4_1cf4cb6be73223755cb611ec097d1498._comment
new file mode 100644
index 0000000000..beeccc6acf
--- /dev/null
+++ b/doc/walkthrough/renaming_files/comment_4_1cf4cb6be73223755cb611ec097d1498._comment
@@ -0,0 +1,12 @@
+[[!comment format=mdwn
+ username="dusty@dba9614abc02e013134f95c2c59181a6c12a8468"
+ nickname="dusty"
+ subject="moving files with git mv can yield broken symlink"
+ date="2016-08-23T21:48:59Z"
+ content="""
+Hey Joey,
+
+Just wondering how we rename/move files around into different directories within a repository without breaking the symbolic links. If we move to a different directory and the depth is different than where the files were originally in the repo then the symbolic link is broken.
+
+Thanks
+"""]]
diff --git a/doc/walkthrough/renaming_files/comment_5_bc3e37868b276c52a51586641cdb92ae._comment b/doc/walkthrough/renaming_files/comment_5_bc3e37868b276c52a51586641cdb92ae._comment
new file mode 100644
index 0000000000..5ca3bad944
--- /dev/null
+++ b/doc/walkthrough/renaming_files/comment_5_bc3e37868b276c52a51586641cdb92ae._comment
@@ -0,0 +1,13 @@
+[[!comment format=mdwn
+ username="dusty@dba9614abc02e013134f95c2c59181a6c12a8468"
+ nickname="dusty"
+ subject="Nevermind - broken symlinks will be fixed on commit"
+ date="2016-08-23T21:53:05Z"
+ content="""
+
+Didn't catch this from the docs above: 
+
+\"Notice that, since annexed files are represented by symlinks, the symlink will break when the file is moved into a subdirectory. But, git-annex will fix this up for you when you commit -- it has a pre-commit hook that watches for and corrects broken symlinks.\"
+
+
+"""]]
diff --git a/doc/walkthrough/renaming_files/comment_6_2ae2cc3dba5db4d4354a5b49c490272e._comment b/doc/walkthrough/renaming_files/comment_6_2ae2cc3dba5db4d4354a5b49c490272e._comment
new file mode 100644
index 0000000000..d6db355683
--- /dev/null
+++ b/doc/walkthrough/renaming_files/comment_6_2ae2cc3dba5db4d4354a5b49c490272e._comment
@@ -0,0 +1,32 @@
+[[!comment format=mdwn
+ username="https://launchpad.net/~stephane-gourichon-lpad"
+ nickname="stephane-gourichon-lpad"
+ avatar="http://cdn.libravatar.org/avatar/02d4a0af59175f9123720b4481d55a769ba954e20f6dd9b2792217d9fa0c6089"
+ subject="Bitten again by "E" backend, wish "e" backend."
+ date="2016-10-30T19:19:21Z"
+ content="""
+Changing file extension is arguably not a very common use case.  But changing extension case is more common, especially when copying files from FAT filesystems through different ways (Card reader, plugging device directly), and causes troubles.
+
+# Concrete case
+
+A set of files were imported with extension NEF and JPG, by mistake (forgot to rename them first), using default (or is it?) backend SHA256E.
+
+When using `git annex reinject --known` to cleanup another copy, files were not detected as known.
+
+This will happen again and feels not very good.
+
+# Workaround
+
+Thanks to CandyAngel I know I can rename then use `git annex migrate --force` and `reinject` again.
+
+# Better solution
+
+A SHA256e backend (that squashes extensions to lowercase) would not have this problem.
+
+# Additional information
+
+Why not switch to plain SHA256 backend?  I'm a little paranoid (which makes me a good fit for using git-annex overall).  
+
+One of the reason I use git-annex is that its storage format is actually simple, and files can be accessed even on a computer that can read the plain filesystem, without git or git-annex required.  But in the lifetime of a filesystem, many bad things can happen and details of storage format may make a difference in case of major recovery (or even sanity check) scenario.  Hashes without extension feel somehow uncomfortable.  Lowercase extension backends feel much better.
+
+"""]]
diff --git a/doc/walkthrough/syncing.mdwn b/doc/walkthrough/syncing.mdwn
new file mode 100644
index 0000000000..57fe47db05
--- /dev/null
+++ b/doc/walkthrough/syncing.mdwn
@@ -0,0 +1,27 @@
+Notice that in the [[previous example|getting_file_content]], `git annex
+sync` was used. This lets git-annex know what has changed in the other
+repositories like the laptop, and so it knows about the files present there and can
+get them.
+
+Let's look at what the sync command does in more detail:
+
+	# cd /media/usb/annex
+	# git annex sync
+	commit
+	nothing to commit (working directory clean)
+	ok
+	pull laptop
+	ok
+	push laptop
+	ok
+
+After you run sync, the git repository will be updated with all changes
+made to its remotes, and any changes in the git repository will be pushed
+out to its remotes, where a sync will get them. This is especially useful
+when using git in a distributed fashion, without a [[central bare
+repository|tips/centralized_git_repository_tutorial]]. See [[sync]] for
+details.
+
+By default `git annex sync` only syncs the metadata about your
+files that is stored in git. It does not sync the contents of files, that
+are managed by git-annex. To do that, you can use `git annex sync --content`
diff --git a/doc/walkthrough/transferring_files__58___When_things_go_wrong.mdwn b/doc/walkthrough/transferring_files__58___When_things_go_wrong.mdwn
new file mode 100644
index 0000000000..d3db8c3287
--- /dev/null
+++ b/doc/walkthrough/transferring_files__58___When_things_go_wrong.mdwn
@@ -0,0 +1,17 @@
+After a while, you'll have several annexes, with different file contents.
+You don't have to try to keep all that straight; git-annex does 
+[[location_tracking]] for you. If you ask it to get a file and the drive
+or file server is not accessible, it will let you know what it needs to get
+it:
+
+	# git annex get video/hackity_hack_and_kaxxt.mov
+	get video/hackity_hack_and_kaxxt.mov (not available)
+	  Unable to access these remotes: usbdrive, server
+	  Try making some of these repositories available:
+	  	5863d8c0-d9a9-11df-adb2-af51e6559a49  -- my home file server
+	   	58d84e8a-d9ae-11df-a1aa-ab9aa8c00826  -- portable USB drive
+	   	ca20064c-dbb5-11df-b2fe-002170d25c55  -- backup SATA drive
+	failed
+	# sudo mount /media/usb
+	# git annex get video/hackity_hack_and_kaxxt.mov
+	get video/hackity_hack_and_kaxxt.mov (from usbdrive...) ok
diff --git a/doc/walkthrough/unused_data.mdwn b/doc/walkthrough/unused_data.mdwn
new file mode 100644
index 0000000000..1d655b89ae
--- /dev/null
+++ b/doc/walkthrough/unused_data.mdwn
@@ -0,0 +1,35 @@
+It's possible for data to accumulate in the annex that no files in any
+branch point to anymore. One way it can happen is if you `git rm` a file
+without first calling `git annex drop`. And, when you modify an annexed
+file, the old content of the file remains in the annex. Another way is when
+migrating between key-value [[backends]].
+
+This might be historical data you want to preserve, so git-annex defaults to
+preserving it. So from time to time, you may want to check for such data:
+
+	# git annex unused
+	unused . (checking for unused data...) 
+	  Some annexed data is no longer used by any files in the repository.
+	    NUMBER  KEY
+	    1       SHA256-s86050597--6ae2688bc533437766a48aa19f2c06be14d1bab9c70b468af445d4f07b65f41e
+	    2       SHA1-s14--f1358ec1873d57350e3dc62054dc232bc93c2bd1
+	  (To see where data was previously used, try: git log --stat -S'KEY')
+	  (To remove unwanted data: git-annex dropunused NUMBER)
+	ok
+
+After running `git annex unused`, you can follow the instructions to examine
+the history of files that used the data, and if you decide you don't need that
+data anymore, you can easily remove it from your local repository.
+
+	# git annex dropunused 1
+	dropunused 1 ok
+
+Hint: To drop a lot of unused data, use a command like this:
+
+	# git annex dropunused 1-1000
+
+Rather than removing the data, you can instead send it to other
+repositories:
+
+	# git annex copy --unused --to backup
+	# git annex move --unused --to archive
diff --git a/doc/walkthrough/unused_data/comment_1_684b7b652d3a8ec04f32129c5528f1ab._comment b/doc/walkthrough/unused_data/comment_1_684b7b652d3a8ec04f32129c5528f1ab._comment
new file mode 100644
index 0000000000..2be2a6463f
--- /dev/null
+++ b/doc/walkthrough/unused_data/comment_1_684b7b652d3a8ec04f32129c5528f1ab._comment
@@ -0,0 +1,22 @@
+[[!comment format=mdwn
+ username="bremner"
+ ip="156.34.89.108"
+ subject="finding data that isn't unused, but should be."
+ date="2012-10-17T20:32:11Z"
+ content="""
+Sometimes links to annexed data still exists on some branch, when it was supposed to be dropped. Here is how I found these; perhaps there is a simpler way.
+
+    % git annex find --format '${key}\n' | sort > /tmp/known-keys
+    % find .git/annex/objects -type f -exec basename {} \; | sort  > /tmp/local-keys
+    % comm -23 /tmp/local-keys /tmp/known-keys
+
+to look for what branch these are on, try 
+
+    % git log --stat --all -S$key
+
+for one of the keys output above. In my case it was the same remote branch keeping them all alive.
+
+
+*EDIT* sort key lists to make comm work properly
+
+"""]]
diff --git a/doc/walkthrough/unused_data/comment_2_e2ccd9a209c7a104004e998135d4b675._comment b/doc/walkthrough/unused_data/comment_2_e2ccd9a209c7a104004e998135d4b675._comment
new file mode 100644
index 0000000000..91075ce24b
--- /dev/null
+++ b/doc/walkthrough/unused_data/comment_2_e2ccd9a209c7a104004e998135d4b675._comment
@@ -0,0 +1,13 @@
+[[!comment format=mdwn
+ username="https://www.google.com/accounts/o8/id?id=AItOawn3p4i4lk_zMilvjnJ9sS6g2nerpgz0Fjc"
+ nickname="Matthias"
+ subject="Keep historical data, but delete data never referenced"
+ date="2014-10-26T12:06:25Z"
+ content="""
+Is there an easy solution for the following? There are two kinds of \"unused\" I would like to treat differently:
+
+1. Kind \"really unused\": Was added once to the annex, but symlink was never committed
+2. Kind \"only history\": A commit contains a symlink to the data, but no active branch
+
+I want to preserve \"only history\", and only drop \"really unused\". What is an elegant way to do this? Thanks for your suggestions.
+"""]]
diff --git a/doc/walkthrough/unused_data/comment_3_5f973eb7c265015fcd400b6b0bbdf235._comment b/doc/walkthrough/unused_data/comment_3_5f973eb7c265015fcd400b6b0bbdf235._comment
new file mode 100644
index 0000000000..dcca0b0959
--- /dev/null
+++ b/doc/walkthrough/unused_data/comment_3_5f973eb7c265015fcd400b6b0bbdf235._comment
@@ -0,0 +1,10 @@
+[[!comment format=mdwn
+ username="joey"
+ subject="""comment 3"""
+ date="2014-10-31T20:31:39Z"
+ content="""
+`git-annex unused` looks at what data is used by git branches and tags, but
+not by other commits. It's a reasonable request and I have made a todo for
+it: [[todo/find_unused_in_any_commit]] .. But I am unure if it can be
+implemented to run fast enough to be usable.
+"""]]
diff --git a/doc/walkthrough/unused_data/comment_4_2110ed6316b6a0df4ef3e1c8bc97ab99._comment b/doc/walkthrough/unused_data/comment_4_2110ed6316b6a0df4ef3e1c8bc97ab99._comment
new file mode 100644
index 0000000000..8c6f04e274
--- /dev/null
+++ b/doc/walkthrough/unused_data/comment_4_2110ed6316b6a0df4ef3e1c8bc97ab99._comment
@@ -0,0 +1,11 @@
+[[!comment format=mdwn
+ username="tom_clune"
+ subject="dropping files after changing branches/tags"
+ date="2016-02-24T22:19:00Z"
+ content="""
+I have a use case in which a colleague wishes to have a working copy of my data repository to use with the current version of my model.    When a new version of the model is available they would likewise update their git-annex clone of my data.    The colleague wants to drop any files that have been made obsolete by this change, but I do not see an efficient way to make this determination.   They could of course drop everything and then do git annex get .   but that could be very expensive if only a small subset of the files have actually changed.
+
+I'm probably just missing something basic, as this seems to be a reasonably frequent use case.
+
+
+"""]]
diff --git a/doc/walkthrough/unused_data/comment_5_46acb9fbbd5ef19f65115723888f3eb6._comment b/doc/walkthrough/unused_data/comment_5_46acb9fbbd5ef19f65115723888f3eb6._comment
new file mode 100644
index 0000000000..2e8af7ff55
--- /dev/null
+++ b/doc/walkthrough/unused_data/comment_5_46acb9fbbd5ef19f65115723888f3eb6._comment
@@ -0,0 +1,14 @@
+[[!comment format=mdwn
+ username="joey"
+ subject="""comment 5"""
+ date="2016-03-14T20:33:13Z"
+ content="""
+Tom, it should suffice for your colleague to pull the new
+version from you, and then run `git annex unused` to find unused files.
+
+Now, if you have other branches or tags pointing at older versions
+of the data, those files will still be considered to be used, even if 
+the current version doesn't use them. There's a new
+`--used-refspec` option for [[git-annex unused|git-annex-unused]] that 
+you can use to specify which branches to consider to be used.
+"""]]
diff --git a/doc/walkthrough/using_bup.mdwn b/doc/walkthrough/using_bup.mdwn
new file mode 100644
index 0000000000..3a6a8776aa
--- /dev/null
+++ b/doc/walkthrough/using_bup.mdwn
@@ -0,0 +1,37 @@
+Another [[special_remote|special_remotes]] that git-annex can use is
+a [[special_remotes/bup]] repository. Bup stores large file contents
+in a git repository of its own, with deduplication. Combined with
+git-annex, you can have git on both the frontend and the backend.
+
+Here's how to create a bup remote, and describe it.
+
+[[!template id=note text="""
+Instead of specifying a remote system, you could choose to make a bup
+remote that is only accessible on the current system, by passing
+"buprepo=/big/mybup".
+"""]]
+
+	# git annex initremote mybup type=bup encryption=none buprepo=example.com:/big/mybup
+	initremote bup (bup init)
+	Initialized empty Git repository in /big/mybup/
+	ok
+	# git annex describe mybup "my bup repository at example.com"
+	describe mybup ok
+
+Now the remote can be used like any other remote.
+
+	# git annex move my_cool_big_file --to mybup
+	move my_cool_big_file (to mybup...)
+	Receiving index from server: 1100/1100, done.
+	ok
+
+Note that, unlike other remotes, bup does not really support removing
+content from its git repositories. This is a feature. :)
+
+	# git annex move my_cool_big_file --from mybup
+	move my_cool_big_file (from mybup...)
+	  content cannot be removed from bup remote
+	failed
+	git-annex: 1 failed
+
+See [[special_remotes/bup]] for details.
diff --git a/doc/walkthrough/using_ddar.mdwn b/doc/walkthrough/using_ddar.mdwn
new file mode 100644
index 0000000000..50f9eedde3
--- /dev/null
+++ b/doc/walkthrough/using_ddar.mdwn
@@ -0,0 +1,32 @@
+Another [[special_remote|special_remotes]] that git-annex can use is
+a [[special_remotes/ddar]] repository. ddar stores large file contents
+in a directory structure of its own, with deduplication. For remote
+repositories, ddar requires that ssh is available on the remote, with ddar also
+installed remotely. When copying files to the remote, ddar only needs to send
+over the network the parts of the files that are not already present remotely.
+
+Unlike bup, ddar uses its own storage format, which allows for both creation
+and deletion of de-deduplicated files.
+
+Here's how to create a ddar remote, and describe it.
+
+[[!template id=note text="""
+Instead of specifying a remote system, you could choose to make a ddar
+remote that is only accessible on the current system, by passing
+"ddarrepo=/big/myddar".
+"""]]
+
+	# git annex initremote myddar type=ddar encryption=none ddarrepo=example.com:/big/myddar
+	initremote ddar (ddar init)
+	Initialized empty Git repository in /big/myddar/
+	ok
+	# git annex describe myddar "my ddar repository at example.com"
+	describe myddar ok
+
+Now the remote can be used like any other remote.
+
+	# git annex move my_cool_big_file --to myddar
+	move my_cool_big_file (to myddar...)
+	ok
+
+See [[special_remotes/ddar]] for details.
diff --git a/doc/walkthrough/using_special_remotes.mdwn b/doc/walkthrough/using_special_remotes.mdwn
new file mode 100644
index 0000000000..427baadd9f
--- /dev/null
+++ b/doc/walkthrough/using_special_remotes.mdwn
@@ -0,0 +1,53 @@
+We've seen above that git-annex can be used to store files in
+regular git remotes, accessed either via ssh, or on a removable drive. But
+git-annex can also store files in Amazon S3, Glacier, on a rsync server, in
+WebDAV, or even pull files down from the web and bittorrent.
+This and much more is made possible by [[special_remotes]].
+
+These are not normal git repositories; indeed the git repository is not
+stored on a special remote. But git-annex can store the contents of files
+in special remotes, and operate on them much as it would on any other
+remote. Bonus: Files stored on special remotes can easily be
+[[encrypted|encryption]]!
+
+All you need to get started using a special remote is to initialize it.
+This is done using the `git annex initremote` command, which needs to be
+passed different parameters depending on the type of special remote.
+
+Some special remotes also need things like passwords to be set in
+environment variables. Don't worry -- it will prompt if you leave anything off.
+So feel free to make any kind of special remote instead of the S3 remote
+used in this example.
+
+	# export AWS_ACCESS_KEY_ID="somethingotherthanthis"
+	# export AWS_SECRET_ACCESS_KEY="s3kr1t"
+	# git annex initremote mys3 type=S3 chunk=1MiB encryption=shared
+	initremote mys3 (shared encryption) (checking bucket) (creating bucket in US) ok
+
+Now you can store files on the newly initialized special remote.
+
+	# git annex copy my_cool_big_file --to mys3
+	copy my_cool_big_file (to mys3...) ok
+
+Once you've initialized a special remote in one repository, you can enable
+use of the same special remote in other clones of the repository.
+If the mys3 remote above was initialized on your laptop, you'll also want
+to enable it on your desktop.
+
+To do so, first get git-annex in sync (so it knows about
+the special remote that was added in the other repository), and then
+use `git annex enableremote`.
+
+	desktop# git annex sync
+	desktop# export AWS_ACCESS_KEY_ID="somethingotherthanthis"
+	desktop# export AWS_SECRET_ACCESS_KEY="s3kr1t"
+	desktop# git annex enableremote mys3
+	enableremote mys3 (checking bucket) ok
+
+And now you can download files from the special remote:
+
+	desktop# git annex get my_cool_big_file --from mys3
+	get my_cool_big_file (from mys3...) ok
+
+This has only scratched the surface of what can be done with
+[[special_remotes]].
diff --git a/doc/walkthrough/using_ssh_remotes.mdwn b/doc/walkthrough/using_ssh_remotes.mdwn
new file mode 100644
index 0000000000..4797a58c34
--- /dev/null
+++ b/doc/walkthrough/using_ssh_remotes.mdwn
@@ -0,0 +1,36 @@
+So far in this walkthrough, git-annex has been used with a remote
+repository on a USB drive. But it can also be used with a git remote
+that is truly remote, a host accessed by ssh.
+
+Say you have a desktop on the same network as your laptop and want
+to clone the laptop's annex to it:
+
+	desktop# git clone ssh://mylaptop/home/me/annex ~/annex
+	desktop# cd ~/annex
+	desktop# git annex init "my desktop"
+
+Now you can get files and they will be transferred (using `rsync` via `ssh`):
+
+	desktop# git annex get my_cool_big_file
+	get my_cool_big_file (getting UUID for origin...) (from origin...)
+	SHA256-s86050597--6ae2688bc533437766a48aa19f2c06be14d1bab9c70b468af445d4f07b65f41e  100% 2159     2.1KB/s   00:00
+	ok
+
+When you drop files, git-annex will ssh over to the remote and make
+sure the file's content is still there before removing it locally:
+
+	desktop# git annex drop my_cool_big_file
+	drop my_cool_big_file (checking origin..) ok
+
+Note that normally git-annex prefers to use non-ssh remotes, like
+a USB drive, before ssh remotes. They are assumed to be faster/cheaper to
+access, if available. There is a annex-cost setting you can configure in
+`.git/config` to adjust which repositories it prefers. See
+[[the_man_page|git-annex]] for details.
+
+Also, note that you need full shell access for this to work -- 
+git-annex needs to be able to ssh in and run commands. Or at least,
+your shell needs to be able to run the [[git-annex-shell]] command.
+
+For details on setting up ssh remotes, see the
+[[tips/centralized_git_repository_tutorial]].
diff --git a/doc/walkthrough/using_ssh_remotes/comment_10_98e97c4d7fbbcd449eddf683967a71d6._comment b/doc/walkthrough/using_ssh_remotes/comment_10_98e97c4d7fbbcd449eddf683967a71d6._comment
new file mode 100644
index 0000000000..6cd9bb8485
--- /dev/null
+++ b/doc/walkthrough/using_ssh_remotes/comment_10_98e97c4d7fbbcd449eddf683967a71d6._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="https://www.google.com/accounts/o8/id?id=AItOawla7u6eLKNYZ09Z7xwBffqLaXquMQC07fU"
+ nickname="Matthias"
+ subject="Hint for Debian/Ubuntu"
+ date="2012-11-07T13:10:15Z"
+ content="""
+In Debian/Ubuntu the default .bashrc returns immediately if the shell is non-interactive. Make sure to setup the PATH such that it is updated with the location of git-annex-shell before this check! This just cost me an hour of debugging as I didn't notice the return statement early on...
+"""]]
diff --git a/doc/walkthrough/using_ssh_remotes/comment_11_f2775a151ed50caba27057bd9c38bae2._comment b/doc/walkthrough/using_ssh_remotes/comment_11_f2775a151ed50caba27057bd9c38bae2._comment
new file mode 100644
index 0000000000..9768300dc6
--- /dev/null
+++ b/doc/walkthrough/using_ssh_remotes/comment_11_f2775a151ed50caba27057bd9c38bae2._comment
@@ -0,0 +1,13 @@
+[[!comment format=mdwn
+ username="https://www.google.com/accounts/o8/id?id=AItOawniXLhO9mLn-7EDfawdENZ2fQwlGy5w_oc"
+ nickname="Michał"
+ subject="unrecognized command"
+ date="2013-03-12T06:32:56Z"
+ content="""
+Thanks Matthias, I fought with this as well, this was the tip I needed to move on.  I'm using the Linux standalone, and I had 2 issues getting everything to work without getting git-annex-shell errors.
+
+1. The autoinstalled wrapper could not be found, I had to comment the \"Ubuntu exit\" line and add the $HOME/.ssh to path to get rid of \"command not found\"
+2. Had to modify the wrapper by replacing the \"$SSH_ORIGINAL_COMMAND\" by \"$@\" to get rid of \"fatal: unrecognized command ''\"
+
+
+"""]]
diff --git a/doc/walkthrough/using_ssh_remotes/comment_12_a8bc6110128431ca2a8624ddc75ea364._comment b/doc/walkthrough/using_ssh_remotes/comment_12_a8bc6110128431ca2a8624ddc75ea364._comment
new file mode 100644
index 0000000000..28c3aa92dd
--- /dev/null
+++ b/doc/walkthrough/using_ssh_remotes/comment_12_a8bc6110128431ca2a8624ddc75ea364._comment
@@ -0,0 +1,10 @@
+[[!comment format=mdwn
+ username="http://joeyh.name/"
+ nickname="joey"
+ subject="comment 12"
+ date="2013-03-12T11:15:11Z"
+ content="""
+@Michael, the standalone tarball is really meant to run the git-annex assistant. The first time \"git annex webapp\" is run, it will set up the ssh wrapper for you.
+
+I have updated the wrapper to work when ssh is not configured to force a key to run a command.
+"""]]
diff --git a/doc/walkthrough/using_ssh_remotes/comment_2_365db5820d96d5daa62c19fd76fcdf1e._comment b/doc/walkthrough/using_ssh_remotes/comment_2_365db5820d96d5daa62c19fd76fcdf1e._comment
new file mode 100644
index 0000000000..8973978ad8
--- /dev/null
+++ b/doc/walkthrough/using_ssh_remotes/comment_2_365db5820d96d5daa62c19fd76fcdf1e._comment
@@ -0,0 +1,13 @@
+[[!comment format=mdwn
+ username="http://joeyh.name/"
+ ip="4.153.81.112"
+ subject="comment 2"
+ date="2012-05-27T20:53:05Z"
+ content="""
+When `git annex get` does nothing, it's because it doesn't know a place to get the file from.
+
+This can happen if the `git-annex` branch has not propigated from the place where the file was added.
+For example, if on the laptop you had run `git pull ssh master`, that would only pull the master branch, not the git-annex branch. 
+
+An easy way to ensure the git-annex branch is kept in sync is to run `git annex sync`
+"""]]
diff --git a/doc/walkthrough/using_ssh_remotes/comment_2_451fd0c6a25ee61ef137e8e5be0c286b._comment b/doc/walkthrough/using_ssh_remotes/comment_2_451fd0c6a25ee61ef137e8e5be0c286b._comment
new file mode 100644
index 0000000000..2121401968
--- /dev/null
+++ b/doc/walkthrough/using_ssh_remotes/comment_2_451fd0c6a25ee61ef137e8e5be0c286b._comment
@@ -0,0 +1,16 @@
+[[!comment format=mdwn
+ username="https://www.google.com/accounts/o8/id?id=AItOawkaT0B6s9jQuMzQUYRVBgWqtO7BhT_ZSaE"
+ nickname="Fernando Seabra"
+ subject="cannot get files"
+ date="2012-05-27T20:33:09Z"
+ content="""
+Hi,
+
+I could successfully clone my ssh repo's annex to my laptop, following these instructions.
+I'm also able to sync the repositories (laptop and ssh) when I commit new files in the ssh repo.
+
+However, every time I try to get files from the ssh repo (using 'git annex get some_file'), nothing happens.
+Do you know what can be happening?
+
+Thanks!
+"""]]
diff --git a/doc/walkthrough/using_ssh_remotes/comment_3_b2f15a46620385da26d5fe8f11ebfc1a._comment b/doc/walkthrough/using_ssh_remotes/comment_3_b2f15a46620385da26d5fe8f11ebfc1a._comment
new file mode 100644
index 0000000000..75a133d840
--- /dev/null
+++ b/doc/walkthrough/using_ssh_remotes/comment_3_b2f15a46620385da26d5fe8f11ebfc1a._comment
@@ -0,0 +1,15 @@
+[[!comment format=mdwn
+ username="https://www.google.com/accounts/o8/id?id=AItOawkaT0B6s9jQuMzQUYRVBgWqtO7BhT_ZSaE"
+ nickname="Fernando Seabra"
+ subject="comment 3"
+ date="2012-05-27T21:13:50Z"
+ content="""
+Thanks for the quick replay!
+
+I already did 'git annex sync', but it didn't work. The steps were: 'git clone ssh...', then 'cd annex', then 'git annex init \"laptop\"'
+
+After that, I did a 'git annex sync', and tried to get the file, but nothing happens. That's why I found it weird.
+Any other thing that might have happened?
+
+Thanks again!
+"""]]
diff --git a/doc/walkthrough/using_ssh_remotes/comment_4_433ccc87fbb0a13e32d59d77f0b4e56c._comment b/doc/walkthrough/using_ssh_remotes/comment_4_433ccc87fbb0a13e32d59d77f0b4e56c._comment
new file mode 100644
index 0000000000..3df03abc2c
--- /dev/null
+++ b/doc/walkthrough/using_ssh_remotes/comment_4_433ccc87fbb0a13e32d59d77f0b4e56c._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="http://joeyh.name/"
+ ip="4.153.81.112"
+ subject="comment 4"
+ date="2012-05-27T21:33:11Z"
+ content="""
+Try running `git annex whereis` on the file and see where it says it is.
+"""]]
diff --git a/doc/walkthrough/using_ssh_remotes/comment_5_a9805c7965da0b88a1c9f7f207c450a1._comment b/doc/walkthrough/using_ssh_remotes/comment_5_a9805c7965da0b88a1c9f7f207c450a1._comment
new file mode 100644
index 0000000000..703b89ebfa
--- /dev/null
+++ b/doc/walkthrough/using_ssh_remotes/comment_5_a9805c7965da0b88a1c9f7f207c450a1._comment
@@ -0,0 +1,18 @@
+[[!comment format=mdwn
+ username="https://www.google.com/accounts/o8/id?id=AItOawkaT0B6s9jQuMzQUYRVBgWqtO7BhT_ZSaE"
+ nickname="Fernando Seabra"
+ subject="comment 5"
+ date="2012-05-27T21:42:56Z"
+ content="""
+Hi,
+
+I guess the problem is with git-annex-shell. I tried to do 'git annex get file --from name_ssh_repo', and I got the following:
+
+bash: git-annex-shell: command not found; failed; exit code 127
+
+The same thing happens if I try to do 'git annex whereis'
+
+git-annex-shell is indeed installed. How can I make my shell recognize this command?
+
+Thanks a lot!
+"""]]
diff --git a/doc/walkthrough/using_ssh_remotes/comment_6_9d5c12c056892b706cf100ea01866685._comment b/doc/walkthrough/using_ssh_remotes/comment_6_9d5c12c056892b706cf100ea01866685._comment
new file mode 100644
index 0000000000..4d5961ca90
--- /dev/null
+++ b/doc/walkthrough/using_ssh_remotes/comment_6_9d5c12c056892b706cf100ea01866685._comment
@@ -0,0 +1,12 @@
+[[!comment format=mdwn
+ username="http://joeyh.name/"
+ ip="4.153.81.112"
+ subject="comment 6"
+ date="2012-05-27T22:08:50Z"
+ content="""
+git-annex-shell needs to be installed in the `PATH` on any host that will hold annexed files. 
+
+If you installed with cabal, it might be `.cabal/bin/`. Whereever it was installed to is apparently not on the PATH that is set when you ssh into that host. 
+
+
+"""]]
diff --git a/doc/walkthrough/using_ssh_remotes/comment_7_725e7dbb2d0a74a035127cb01ee0442c._comment b/doc/walkthrough/using_ssh_remotes/comment_7_725e7dbb2d0a74a035127cb01ee0442c._comment
new file mode 100644
index 0000000000..700b170ad6
--- /dev/null
+++ b/doc/walkthrough/using_ssh_remotes/comment_7_725e7dbb2d0a74a035127cb01ee0442c._comment
@@ -0,0 +1,16 @@
+[[!comment format=mdwn
+ username="https://www.google.com/accounts/o8/id?id=AItOawkaT0B6s9jQuMzQUYRVBgWqtO7BhT_ZSaE"
+ nickname="Fernando Seabra"
+ subject="comment 7"
+ date="2012-05-27T23:35:17Z"
+ content="""
+Hi,
+
+It was already installed in PATH. In fact, I can call it from the command line, and it is recognized (e.g. calling 'git-annex-shell' gives me 'git-annex-shell: bad parameters'). However, every time I do a 'git annex whereis' or 'git annex get file --from repo', it gives me the following error:
+
+bash: git-annex-shell: command not found
+Command ssh [\"-S\",\"/Users/username/annex/.git/annex/ssh/username@example.edu\",\"-o\",\"ControlMaster=auto\",\"-o\",\"ControlPersist=yes\",\"username@example.edu\",\"git-annex-shell 'configlist' '/~/annex'\"] failed; exit code 127
+
+I tried to run this ssh command, but it gives me the same 'command not found' error. It seems that the problem is with the ssh repo?
+The ssh repo has a git-annex-shell working and installed in PATH.
+"""]]
diff --git a/doc/walkthrough/using_ssh_remotes/comment_8_8448e55026d2c2b50d8da41707686bea._comment b/doc/walkthrough/using_ssh_remotes/comment_8_8448e55026d2c2b50d8da41707686bea._comment
new file mode 100644
index 0000000000..148f016db7
--- /dev/null
+++ b/doc/walkthrough/using_ssh_remotes/comment_8_8448e55026d2c2b50d8da41707686bea._comment
@@ -0,0 +1,16 @@
+[[!comment format=mdwn
+ username="https://www.google.com/accounts/o8/id?id=AItOawmB-gCGEs--zfmvYU-__Hj2FbliUXgxMDs"
+ nickname="Jakub"
+ subject="Path problems"
+ date="2012-07-13T19:15:15Z"
+ content="""
+Hi,
+
+I have a same 'git-annex-shell command not found' problem as above. I've installed git annex via cabal into my ~/.haskell_bin directory. Then I've added this dir both to ~/.bashrc and ~/.zshrc. I can run git annex or 'git annex-shell' and everything is fine. My guess is that haskell is trying to spawn git-annex-shell with some current $PATH unaware shell like dash maybe?
+
+I've fixed this behavior by using a really ugly hack - I've symlinked ~/.haskell_bin/git-annex-shell to /usr/bin/git-annex-shell on all my machines and the problem is gone. Somehow haskell (or whatever is trying to call git-annex-shell) is unaware of path modifications from .bashrc/.zshrc
+
+Here is the path modification I've used:
+
+export PATH=~/.haskell_bin:$PATH
+"""]]
diff --git a/doc/walkthrough/using_ssh_remotes/comment_9_61833299a9878f23ac57598fa6da8839._comment b/doc/walkthrough/using_ssh_remotes/comment_9_61833299a9878f23ac57598fa6da8839._comment
new file mode 100644
index 0000000000..ddb96da369
--- /dev/null
+++ b/doc/walkthrough/using_ssh_remotes/comment_9_61833299a9878f23ac57598fa6da8839._comment
@@ -0,0 +1,23 @@
+[[!comment format=mdwn
+ username="https://www.google.com/accounts/o8/id?id=AItOawmB-gCGEs--zfmvYU-__Hj2FbliUXgxMDs"
+ nickname="Jakub"
+ subject="Fixed"
+ date="2012-07-13T19:27:46Z"
+ content="""
+Found the problem:
+
+One should never use ~ in such path:
+
+WRONG export PATH=~/somedir:$PATH
+
+Instead one should use $HOME:
+
+GOOD export PATH=$HOME/somedir:$PATH
+
+Can I surpress the message that shell failed with status 255 when a repo is unavailible? I've got two repos pointing to one machine - either via vpn or local lan and I keep getting erros if one is unavailible:
+
+ssh: connect to host 10.9.0.1 port 39882: No route to host
+Command ssh [\"-S\",\"/home/pielgrzym/annex/.git/annex/ssh/nas\",\"-o\",\"ControlMaster=auto\",\"-o\",\"ControlPersist=yes\",\"nas\",\"git-annex-shell 'configlist' '/~/annex'\"] failed; exit code 255
+
+
+"""]]
diff --git a/doc/walkthrough/using_tags_and_branches.mdwn b/doc/walkthrough/using_tags_and_branches.mdwn
new file mode 100644
index 0000000000..b147ea4661
--- /dev/null
+++ b/doc/walkthrough/using_tags_and_branches.mdwn
@@ -0,0 +1,14 @@
+Like git, git-annex hangs on to every old version of a file (by default),
+so you can make tags and branches, and can check them out later to look at
+the old files.
+
+	# git tag 1.0
+	# rm -f my_cool_big_file
+	# git commit -m deleted
+	# git checkout 1.0
+	# cat my_cool_big_file
+	yay! old version still here
+
+Of course, when you `git checkout` an old branch, some old versions of
+files may not be locally available, and may be stored in some other
+repository. You can use `git annex get` to get them as usual.
diff --git a/doc/workflow.mdwn b/doc/workflow.mdwn
new file mode 100644
index 0000000000..e5d440c6de
--- /dev/null
+++ b/doc/workflow.mdwn
@@ -0,0 +1,102 @@
+Git-annex supports a wide variety of workflows, a spectrum that ranges from
+completely automatic behavior where git-annex handles everything, through
+manual behavior where git-annex does only what you say when you tell it to,
+all the way down to internal behavior, where you have complete control and
+understand how everything is stored and exactly what changes are happening.
+
+I will proceed to summarize all of these. I will begin at the automatic
+end, hoping that this is most useful, and drill down to the low level
+approaches. Note, however, that this is the opposite order of how git-annex
+was developed. A list of workflows that started from manual,
+commandline usage would be much more intuitive, but you'd have to be
+willing to read the man page and wiki pages to get started, and that's
+pretty much what's already out there anyway.
+
+Note that for each of these levels of interaction, all the levels following
+will also work as well. So you can actually manually move annexed files
+around while the webapp is running, etc.
+
+# 1. [[git annex webapp|git-annex-webapp]]
+
+The [[`git annex webapp`|git-annex-webapp]] command launches a local web
+server which serves a graphical user interface and automatically manages
+git annex. It will attempt to guide you through the whole process and do
+everything for you. [[You do not even need to type the command.
+|assistant/quickstart]] This should be run on every machine that may
+produce file changes.  When you move files into or out of your repository
+folder, git-annex should record the changes and automatically propagate
+them to other connected machines.
+
+The webapp is part of the larger [[assistant]].
+
+# 2. [[git annex assistant|git-annex-assistant]] without the webapp
+
+You could call [[`git annex assistant`|git-annex-assistant]] the
+command-line version of the webapp, giving you more control over creating
+and connecting your repositories, and configuring how files are moved
+between them.
+
+The assistant, when running, will automatically watch for file changes and
+synchronize them to other repositories, but you must manually create the
+repositories and configure the rules for syncing. To create a repository,
+use `git init` and then [[`git annex init`|git-annex-init]], and then `git
+remote add` it to any other repositories. If you want more than one annex,
+you can add their paths to `~/.config/git-annex/autostart` if you would
+like them to automatically begin syncing when `git annex assistant
+--autostart` is run, perhaps on boot or login. You can configure rules for
+where files are copied using the repository setup commands such as [[git
+annex preferred-content|git-annex-preferred-content]] to configure
+[[content preferences|preferred content]] for what goes where, [[`git annex
+numcopies`|git-annex-numcopies]] for how many [[copies]] must be kept of
+each file, and [[`git config annex.largefiles`|tips/largefiles]] to define
+small files that should be stored straight in git; most of the settings are
+accessible in one place with [[`git annex vicfg`|git-annex-vicfg]].
+
+# 3. [[git annex watch|git-annex-watch]] without the assistant
+
+The [[`git annex watch`|git-annex-watch]] command is like the assistant but
+has no automatic network behavior, giving you complete control over when
+repositories are pushed and pulled, and when files are moved between
+systems. The local repository is watched, and any file changes are added to
+git-annex. In order to synchronize between repositories, you must run
+[[`git annex sync --content`|git-annex-sync]] in the repository with the
+changes, which will merge the git history and logs with your remotes, and
+automatically transfer files to match your preferred and required content
+expressions.
+
+# 4. No background processes
+
+This allows you to decide when and what files are annexed. In order to tell
+git-annex to manage files, you must [[`git annex add`|git-annex-add]] the
+files.
+
+# 5. Plain [[git annex sync|git-annex-sync]] without `--content`
+
+This gives you fine-grained control of where copies of your files are
+stored. [[`git annex sync`|git-annex-sync]] without `--content` tells
+git-annex to merge git histories, but it does not automatically transfer
+your large files between systems. To transfer files and directories, you
+can use [[`git annex get`|git-annex-get]], [[`git annex
+drop`|git-annex-drop]], [[`git annex move`|git-annex-move]], and [[`git
+annex copy`|git-annex-copy]]. Git-annex will not violate a required content
+expression or your numcopies setting unless you pass `--force`, so your
+files are still safe.
+
+# 6. Manual management of git history without the synchronizer
+
+This allows you to control precisely what is committed to git, what commit
+message is used, and how your history is merged between repositories. You
+must have an understanding of git, and run `git commit` after `git annex
+add` to store the change. You must manage the git history yourself, using
+`git pull` and `git push`, to synchronize repositories. You may freely use
+git normally side-by-side with git-annex.
+
+# 7. Manual management of git annex keys
+
+This gives you control of what and where git annex stores your files under
+the hood, and how they are associated with your working tree, rather than
+using the `git annex add` and `git annex get` commands which reference
+files automatically. Git-annex has a variety of plumbing commands listed in
+the [[man page|git-annex]] that let you directly store and retrieve data in
+an annex associated with your git repository, where every datafile is
+enumerated by a unique hashkey.
diff --git a/ghci b/ghci
new file mode 100755
index 0000000000..fccfc661ab
--- /dev/null
+++ b/ghci
@@ -0,0 +1,4 @@
+#!/bin/sh
+# ghci using objects built by cabal
+make dist/caballog
+$(grep 'ghc --make' dist/caballog | head -n 1 | perl -pe 's/--make/--interactive/; s/.\/[^\.\s]+.hs//; s/-package-id [^\s]+//g; s/-hide-all-packages//; s/-threaded//') $@ -fno-warn-tabs
diff --git a/git-annex.cabal b/git-annex.cabal
new file mode 100644
index 0000000000..c7443c1549
--- /dev/null
+++ b/git-annex.cabal
@@ -0,0 +1,1104 @@
+Name: git-annex
+Version: 6.20180926
+Cabal-Version: >= 1.8
+License: GPL-3
+Maintainer: Joey Hess 
+Author: Joey Hess
+Stability: Stable
+Copyright: 2010-2017 Joey Hess
+License-File: COPYRIGHT
+Homepage: http://git-annex.branchable.com/
+Build-type: Custom
+Category: Utility
+Synopsis: manage files with git, without checking their contents into git
+Description:
+ git-annex allows managing files with git, without checking the file
+ contents into git. While that may seem paradoxical, it is useful when
+ dealing with files larger than git can currently easily handle, whether due
+ to limitations in memory, time, or disk space.
+ .
+ It can store large files in many places, from local hard drives, to a
+ large number of cloud storage services, including S3, WebDAV,
+ and rsync, with a dozen cloud storage providers usable via plugins.
+ Files can be stored encrypted with gpg, so that the cloud storage
+ provider cannot see your data. git-annex keeps track of where each file
+ is stored, so it knows how many copies are available, and has many
+ facilities to ensure your data is preserved.
+ .
+ git-annex can also be used to keep a folder in sync between computers,
+ noticing when files are changed, and automatically committing them
+ to git and transferring them to other computers. The git-annex webapp
+ makes it easy to set up and use git-annex this way.
+-- The tarball uploaded to hackage does not include every non-haskell
+-- file in the git repo. The website is left out, so is build machinary for
+-- standalone apps, and packages. Include only files that are needed
+-- make cabal install git-annex work.
+Extra-Source-Files:
+  stack.yaml
+  README
+  CHANGELOG
+  NEWS
+  doc/license/GPL
+  doc/license/AGPL
+  doc/git-annex.mdwn
+  doc/git-annex-add.mdwn
+  doc/git-annex-addunused.mdwn
+  doc/git-annex-addurl.mdwn
+  doc/git-annex-adjust.mdwn
+  doc/git-annex-assistant.mdwn
+  doc/git-annex-calckey.mdwn
+  doc/git-annex-checkpresentkey.mdwn
+  doc/git-annex-contentlocation.mdwn
+  doc/git-annex-copy.mdwn
+  doc/git-annex-dead.mdwn
+  doc/git-annex-describe.mdwn
+  doc/git-annex-diffdriver.mdwn
+  doc/git-annex-direct.mdwn
+  doc/git-annex-drop.mdwn
+  doc/git-annex-dropkey.mdwn
+  doc/git-annex-dropunused.mdwn
+  doc/git-annex-edit.mdwn
+  doc/git-annex-enableremote.mdwn
+  doc/git-annex-enable-tor.mdwn
+  doc/git-annex-examinekey.mdwn
+  doc/git-annex-expire.mdwn
+  doc/git-annex-export.mdwn
+  doc/git-annex-find.mdwn
+  doc/git-annex-findref.mdwn
+  doc/git-annex-fix.mdwn
+  doc/git-annex-forget.mdwn
+  doc/git-annex-fromkey.mdwn
+  doc/git-annex-fsck.mdwn
+  doc/git-annex-fuzztest.mdwn
+  doc/git-annex-get.mdwn
+  doc/git-annex-group.mdwn
+  doc/git-annex-groupwanted.mdwn
+  doc/git-annex-import.mdwn
+  doc/git-annex-importfeed.mdwn
+  doc/git-annex-indirect.mdwn
+  doc/git-annex-info.mdwn
+  doc/git-annex-init.mdwn
+  doc/git-annex-initremote.mdwn
+  doc/git-annex-inprogress.mdwn
+  doc/git-annex-list.mdwn
+  doc/git-annex-lock.mdwn
+  doc/git-annex-log.mdwn
+  doc/git-annex-lookupkey.mdwn
+  doc/git-annex-map.mdwn
+  doc/git-annex-matchexpression.mdwn
+  doc/git-annex-matching-options.mdwn
+  doc/git-annex-merge.mdwn
+  doc/git-annex-metadata.mdwn
+  doc/git-annex-migrate.mdwn
+  doc/git-annex-mirror.mdwn
+  doc/git-annex-move.mdwn
+  doc/git-annex-multicast.mdwn
+  doc/git-annex-numcopies.mdwn
+  doc/git-annex-p2p.mdwn
+  doc/git-annex-pre-commit.mdwn
+  doc/git-annex-preferred-content.mdwn
+  doc/git-annex-proxy.mdwn
+  doc/git-annex-readpresentkey.mdwn
+  doc/git-annex-registerurl.mdwn
+  doc/git-annex-reinit.mdwn
+  doc/git-annex-reinject.mdwn
+  doc/git-annex-rekey.mdwn
+  doc/git-annex-remotedaemon.mdwn
+  doc/git-annex-repair.mdwn
+  doc/git-annex-required.mdwn
+  doc/git-annex-resolvemerge.mdwn
+  doc/git-annex-rmurl.mdwn
+  doc/git-annex-schedule.mdwn
+  doc/git-annex-semitrust.mdwn
+  doc/git-annex-setkey.mdwn
+  doc/git-annex-setpresentkey.mdwn
+  doc/git-annex-shell.mdwn
+  doc/git-annex-smudge.mdwn
+  doc/git-annex-status.mdwn
+  doc/git-annex-sync.mdwn
+  doc/git-annex-test.mdwn
+  doc/git-annex-testremote.mdwn
+  doc/git-annex-transferkey.mdwn
+  doc/git-annex-transferkeys.mdwn
+  doc/git-annex-trust.mdwn
+  doc/git-annex-unannex.mdwn
+  doc/git-annex-undo.mdwn
+  doc/git-annex-ungroup.mdwn
+  doc/git-annex-uninit.mdwn
+  doc/git-annex-unlock.mdwn
+  doc/git-annex-untrust.mdwn
+  doc/git-annex-unused.mdwn
+  doc/git-annex-upgrade.mdwn
+  doc/git-annex-vadd.mdwn
+  doc/git-annex-vcycle.mdwn
+  doc/git-annex-version.mdwn
+  doc/git-annex-vfilter.mdwn
+  doc/git-annex-vicfg.mdwn
+  doc/git-annex-view.mdwn
+  doc/git-annex-vpop.mdwn
+  doc/git-annex-wanted.mdwn
+  doc/git-annex-watch.mdwn
+  doc/git-annex-webapp.mdwn
+  doc/git-annex-whereis.mdwn
+  doc/git-remote-tor-annex.mdwn
+  doc/logo.svg
+  doc/logo_16x16.png
+  Build/mdwn2man
+  Assistant/WebApp/routes
+  static/activityicon.gif
+  static/css/bootstrap.css
+  static/css/bootstrap-theme.css
+  static/js/jquery.ui.core.js
+  static/js/longpolling.js
+  static/js/jquery.full.js
+  static/js/jquery.ui.sortable.js
+  static/js/jquery.ui.mouse.js
+  static/js/jquery.ui.widget.js
+  static/js/bootstrap.js
+  static/syncicon.gif
+  static/favicon.ico
+  static/fonts/glyphicons-halflings-regular.woff
+  static/fonts/glyphicons-halflings-regular.eot
+  static/fonts/glyphicons-halflings-regular.svg
+  static/fonts/glyphicons-halflings-regular.ttf
+  templates/sidebar/main.hamlet
+  templates/sidebar/alert.hamlet
+  templates/bootstrap.hamlet
+  templates/error.cassius
+  templates/README
+  templates/error.hamlet
+  templates/documentation/license.hamlet
+  templates/documentation/repogroup.hamlet
+  templates/documentation/about.hamlet
+  templates/dashboard/main.hamlet
+  templates/dashboard/transfers.cassius
+  templates/dashboard/transfers.hamlet
+  templates/dashboard/metarefresh.hamlet
+  templates/page.cassius
+  templates/page.hamlet
+  templates/control/repairrepository.hamlet
+  templates/control/repairrepository/done.hamlet
+  templates/control/notrunning.julius
+  templates/control/notrunning.hamlet
+  templates/control/repositoryswitcher.hamlet
+  templates/control/shutdown.hamlet
+  templates/control/log.hamlet
+  templates/page.julius
+  templates/repolist.julius
+  templates/configurators/adddrive/combine.hamlet
+  templates/configurators/adddrive/setupmodal.hamlet
+  templates/configurators/adddrive/encrypt.hamlet
+  templates/configurators/newrepository.hamlet
+  templates/configurators/needglaciercli.hamlet
+  templates/configurators/adds3.hamlet
+  templates/configurators/genkeymodal.hamlet
+  templates/configurators/main.hamlet
+  templates/configurators/needconnection.hamlet
+  templates/configurators/newrepository/form.hamlet
+  templates/configurators/newrepository/first.hamlet
+  templates/configurators/newrepository/combine.hamlet
+  templates/configurators/enablewebdav.hamlet
+  templates/configurators/pairing/local/inprogress.hamlet
+  templates/configurators/pairing/local/prompt.hamlet
+  templates/configurators/pairing/wormhole/prompt.hamlet
+  templates/configurators/pairing/wormhole/start.hamlet
+  templates/configurators/pairing/disabled.hamlet
+  templates/configurators/addglacier.hamlet
+  templates/configurators/fsck.cassius
+  templates/configurators/edit/nonannexremote.hamlet
+  templates/configurators/edit/webrepository.hamlet
+  templates/configurators/edit/repository.hamlet
+  templates/configurators/unused.hamlet
+  templates/configurators/addbox.com.hamlet
+  templates/configurators/ssh/testmodal.hamlet
+  templates/configurators/ssh/expiredpassword.hamlet
+  templates/configurators/ssh/error.hamlet
+  templates/configurators/ssh/combine.hamlet
+  templates/configurators/ssh/enable.hamlet
+  templates/configurators/ssh/add.hamlet
+  templates/configurators/ssh/setupmodal.hamlet
+  templates/configurators/ssh/confirm.hamlet
+  templates/configurators/upgrade/android.hamlet
+  templates/configurators/enableia.hamlet
+  templates/configurators/fsck.hamlet
+  templates/configurators/addrepository/archive.hamlet
+  templates/configurators/addrepository/cloud.hamlet
+  templates/configurators/addrepository/connection.hamlet
+  templates/configurators/addrepository/ssh.hamlet
+  templates/configurators/addrepository/misc.hamlet
+  templates/configurators/addrepository/wormholepairing.hamlet
+  templates/configurators/rsync.net/add.hamlet
+  templates/configurators/rsync.net/encrypt.hamlet
+  templates/configurators/gitlab.com/add.hamlet
+  templates/configurators/needgcrypt.hamlet
+  templates/configurators/needtor.hamlet
+  templates/configurators/needmagicwormhole.hamlet
+  templates/configurators/enabledirectory.hamlet
+  templates/configurators/fsck/status.hamlet
+  templates/configurators/fsck/form.hamlet
+  templates/configurators/fsck/preferencesform.hamlet
+  templates/configurators/fsck/formcontent.hamlet
+  templates/configurators/delete/finished.hamlet
+  templates/configurators/delete/start.hamlet
+  templates/configurators/delete/currentrepository.hamlet
+  templates/configurators/unused/form.hamlet
+  templates/configurators/adddrive.hamlet
+  templates/configurators/preferences.hamlet
+  templates/configurators/addia.hamlet
+  templates/configurators/enableaws.hamlet
+  templates/configurators/addrepository.hamlet
+  templates/actionbutton.hamlet
+  templates/repolist.hamlet
+  templates/controlmenu.hamlet
+  templates/notifications/longpolling.julius
+
+Flag S3
+  Description: Enable S3 support
+
+Flag WebDAV
+  Description: Enable WebDAV support
+
+Flag Assistant
+  Description: Enable git-annex assistant and watch command
+
+Flag Webapp
+  Description: Enable git-annex webapp
+
+Flag Pairing
+  Description: Enable pairing
+
+Flag Production
+  Description: Enable production build (slower build; faster binary)
+
+Flag Android
+  Description: Cross building for Android
+  Default: False
+
+Flag AndroidSplice
+  Description: Building to get TH splices for Android
+  Default: False
+
+Flag TorrentParser
+  Description: Use haskell torrent library to parse torrent files
+
+Flag MagicMime
+  Description: Use libmagic to determine file MIME types
+
+Flag ConcurrentOutput
+  Description: Use concurrent-output library
+
+Flag Benchmark
+  Description: Enable benchmarking
+  Default: False
+
+Flag Dbus
+  Description: Enable dbus support
+
+source-repository head
+  type: git
+  location: git://git-annex.branchable.com/
+
+custom-setup
+  Setup-Depends: base (>= 4.6), hslogger, split, unix-compat, process,
+    filepath, exceptions, bytestring, directory, IfElse, data-default,
+    utf8-string, transformers, Cabal
+
+Executable git-annex
+  Main-Is: git-annex.hs
+  Build-Depends:
+   base (>= 4.6 && < 5.0),
+   network (>= 2.6.3.0),
+   network-uri (>= 2.6),
+   optparse-applicative (>= 0.11.0), 
+   containers (>= 0.5.0.0),
+   exceptions (>= 0.6),
+   stm (>= 2.3),
+   mtl (>= 2),
+   uuid (>= 1.2.6),
+   process,
+   data-default,
+   case-insensitive,
+   random,
+   dlist,
+   unix-compat,
+   SafeSemaphore,
+   async,
+   directory (>= 1.2),
+   filepath,
+   IfElse,
+   hslogger,
+   monad-logger,
+   free,
+   utf8-string,
+   bytestring,
+   text,
+   sandi,
+   monad-control,
+   transformers,
+   bloomfilter,
+   edit-distance,
+   resourcet,
+   connection (>= 0.2.6),
+   http-client (>= 0.4.31),
+   http-client-tls,
+   http-types (>= 0.7),
+   http-conduit (>= 2.0),
+   conduit,
+   time,
+   old-locale,
+   esqueleto,
+   persistent-sqlite, 
+   persistent,
+   persistent-template,
+   aeson,
+   vector,
+   tagsoup,
+   unordered-containers,
+   feed (>= 0.3.9),
+   regex-tdfa,
+   socks,
+   byteable,
+   stm-chans,
+   securemem,
+   crypto-api,
+   cryptonite,
+   memory,
+   split,
+   attoparsec,
+   QuickCheck (>= 2.1),
+   tasty (>= 0.7),
+   tasty-hunit,
+   tasty-quickcheck,
+   tasty-rerun
+  CC-Options: -Wall
+  GHC-Options: -Wall -fno-warn-tabs
+  Extensions: PackageImports, LambdaCase
+  -- Some things don't work with the non-threaded RTS.
+  GHC-Options: -threaded
+  Other-Extensions: TemplateHaskell
+
+  -- Fully optimize for production.
+  if flag(Production)
+    -- Lower memory systems can run out of memory with -O2, so
+    -- optimise slightly less.
+    -- This needs -O1 before the -optlo, due to this bug:
+    -- https://ghc.haskell.org/trac/ghc/ticket/14821
+    -- But unfortunately, hackage currently refuses to accept -O1
+    if arch(arm)
+      GHC-Options: -optlo-O2
+    else
+      GHC-Options: -O2
+
+  -- Avoid linking with unused dynamic libaries.
+  -- (Only tested on Linux).
+  if os(Linux)
+    GHC-Options: -optl-Wl,--as-needed
+
+  if (os(windows))
+    Build-Depends:
+      Win32 (>= 2.6.1.0),
+      unix-compat (>= 0.5),
+      setenv,
+      process (>= 1.6.2.0)
+  else
+    Build-Depends: unix
+    if impl(ghc <= 7.6.3)
+      Other-Modules: Utility.Touch.Old
+
+  if flag(S3)
+    Build-Depends: aws (>= 0.9.2)
+    CPP-Options: -DWITH_S3
+    Other-Modules: Remote.S3
+
+  if flag(WebDAV)
+    Build-Depends: DAV (>= 1.0)
+    CPP-Options: -DWITH_WEBDAV
+    Other-Modules:
+      Remote.WebDAV
+      Remote.WebDAV.DavLocation
+  if flag(S3) || flag(WebDAV)
+    Other-Modules:
+      Remote.Helper.Http
+
+  if flag(Assistant) && ! os(solaris) && ! os(gnu)
+    Build-Depends: mountpoints
+    CPP-Options: -DWITH_ASSISTANT
+    Other-Modules:
+      Assistant
+      Assistant.Alert
+      Assistant.Alert.Utility
+      Assistant.BranchChange
+      Assistant.Changes
+      Assistant.Commits
+      Assistant.Common
+      Assistant.CredPairCache
+      Assistant.DaemonStatus
+      Assistant.DeleteRemote
+      Assistant.Drop
+      Assistant.Fsck
+      Assistant.Gpg
+      Assistant.Install
+      Assistant.Install.AutoStart
+      Assistant.Install.Menu
+      Assistant.MakeRemote
+      Assistant.Monad
+      Assistant.NamedThread
+      Assistant.Pairing
+      Assistant.Pairing.MakeRemote
+      Assistant.Pairing.Network
+      Assistant.Pushes
+      Assistant.RemoteControl
+      Assistant.Repair
+      Assistant.RepoProblem
+      Assistant.Restart
+      Assistant.ScanRemotes
+      Assistant.Ssh
+      Assistant.Sync
+      Assistant.Threads.Committer
+      Assistant.Threads.ConfigMonitor
+      Assistant.Threads.Cronner
+      Assistant.Threads.DaemonStatus
+      Assistant.Threads.Exporter
+      Assistant.Threads.Glacier
+      Assistant.Threads.Merger
+      Assistant.Threads.MountWatcher
+      Assistant.Threads.NetWatcher
+      Assistant.Threads.ProblemFixer
+      Assistant.Threads.Pusher
+      Assistant.Threads.RemoteControl
+      Assistant.Threads.SanityChecker
+      Assistant.Threads.TransferPoller
+      Assistant.Threads.TransferScanner
+      Assistant.Threads.TransferWatcher
+      Assistant.Threads.Transferrer
+      Assistant.Threads.UpgradeWatcher
+      Assistant.Threads.Upgrader
+      Assistant.Threads.Watcher
+      Assistant.TransferQueue
+      Assistant.TransferSlots
+      Assistant.TransferrerPool
+      Assistant.Types.Alert
+      Assistant.Types.BranchChange
+      Assistant.Types.Changes
+      Assistant.Types.Commits
+      Assistant.Types.CredPairCache
+      Assistant.Types.DaemonStatus
+      Assistant.Types.NamedThread
+      Assistant.Types.Pushes
+      Assistant.Types.RemoteControl
+      Assistant.Types.RepoProblem
+      Assistant.Types.ScanRemotes
+      Assistant.Types.ThreadName
+      Assistant.Types.ThreadedMonad
+      Assistant.Types.TransferQueue
+      Assistant.Types.TransferSlots
+      Assistant.Types.TransferrerPool
+      Assistant.Types.UrlRenderer
+      Assistant.Unused
+      Assistant.Upgrade
+      Command.Assistant
+      Command.Watch
+      Utility.Mounts
+      Utility.OSX
+
+    if os(linux) || flag(Android)
+      Build-Depends: hinotify
+      CPP-Options: -DWITH_INOTIFY
+      Other-Modules: Utility.DirWatcher.INotify
+    else
+      if os(darwin)
+        Build-Depends: hfsevents
+        CPP-Options: -DWITH_FSEVENTS
+        Other-Modules:
+          Utility.DirWatcher.FSEvents
+      else
+        if os(windows)
+          Build-Depends: Win32-notify
+          CPP-Options: -DWITH_WIN32NOTIFY
+          Other-Modules: Utility.DirWatcher.Win32Notify
+        else
+          if (! os(solaris) && ! os(gnu) && ! os(linux))
+            CPP-Options: -DWITH_KQUEUE
+            C-Sources: Utility/libkqueue.c
+            Includes: Utility/libkqueue.h
+            Other-Modules: Utility.DirWatcher.Kqueue
+
+  if flag(Dbus)
+    if (os(linux))
+      Build-Depends: dbus (>= 0.10.7), fdo-notify (>= 0.3)
+      CPP-Options: -DWITH_DBUS -DWITH_DESKTOP_NOTIFY -DWITH_DBUS_NOTIFICATIONS
+      Other-Modules: Utility.DBus
+
+  if flag(Android)
+    Build-Depends: data-endian
+    CPP-Options: -D__ANDROID__ -DANDROID_SPLICES -D__NO_TH__
+  else
+    Build-Depends: disk-free-space
+
+  if flag(AndroidSplice)
+    CPP-Options: -DANDROID_SPLICES
+
+  if flag(Webapp)
+    Build-Depends:
+     yesod (>= 1.2.6), 
+     yesod-static (>= 1.2.4),
+     yesod-form (>= 1.3.15),
+     yesod-core (>= 1.2.19),
+     path-pieces (>= 0.1.4),
+     warp (>= 3.0.0.5),
+     warp-tls (>= 1.4),
+     wai,
+     wai-extra,
+     blaze-builder,
+     clientsession,
+     template-haskell,
+     shakespeare (>= 2.0.0)
+    CPP-Options: -DWITH_WEBAPP
+    Other-Modules:
+      Command.WebApp
+      Assistant.Threads.WebApp
+      Assistant.Threads.PairListener
+      Assistant.WebApp
+      Assistant.WebApp.Common
+      Assistant.WebApp.Configurators
+      Assistant.WebApp.Configurators.AWS
+      Assistant.WebApp.Configurators.Delete
+      Assistant.WebApp.Configurators.Edit
+      Assistant.WebApp.Configurators.Fsck
+      Assistant.WebApp.Configurators.IA
+      Assistant.WebApp.Configurators.Local
+      Assistant.WebApp.Configurators.Pairing
+      Assistant.WebApp.Configurators.Preferences
+      Assistant.WebApp.Configurators.Ssh
+      Assistant.WebApp.Configurators.Unused
+      Assistant.WebApp.Configurators.Upgrade
+      Assistant.WebApp.Configurators.WebDAV
+      Assistant.WebApp.Control
+      Assistant.WebApp.DashBoard
+      Assistant.WebApp.Documentation
+      Assistant.WebApp.Form
+      Assistant.WebApp.Gpg
+      Assistant.WebApp.MakeRemote
+      Assistant.WebApp.Notifications
+      Assistant.WebApp.OtherRepos
+      Assistant.WebApp.Page
+      Assistant.WebApp.Pairing
+      Assistant.WebApp.Repair
+      Assistant.WebApp.RepoId
+      Assistant.WebApp.RepoList
+      Assistant.WebApp.SideBar
+      Assistant.WebApp.Types
+      Annex.MakeRepo
+      Utility.Yesod
+      Utility.WebApp
+
+  if flag(Pairing)
+    Build-Depends: network-multicast, network-info
+    CPP-Options: -DWITH_PAIRING
+
+  if flag(TorrentParser)
+    Build-Depends: torrent (>= 10000.0.0)
+    CPP-Options: -DWITH_TORRENTPARSER
+
+  if flag(MagicMime)
+    if (! os(windows))
+      Build-Depends: magic
+      CPP-Options: -DWITH_MAGICMIME
+
+  if flag(ConcurrentOutput)
+    Build-Depends: concurrent-output (>= 1.6)
+    CPP-Options: -DWITH_CONCURRENTOUTPUT
+
+  if flag(Benchmark)
+    Build-Depends: criterion, deepseq
+    CPP-Options: -DWITH_BENCHMARK
+    Other-Modules: Command.Benchmark
+
+  Other-Modules:
+    Annex
+    Annex.Action
+    Annex.AdjustedBranch
+    Annex.AutoMerge
+    Annex.BloomFilter
+    Annex.Branch
+    Annex.Branch.Transitions
+    Annex.BranchState
+    Annex.CatFile
+    Annex.ChangedRefs
+    Annex.CheckAttr
+    Annex.CheckIgnore
+    Annex.Common
+    Annex.Concurrent
+    Annex.Content
+    Annex.Content.Direct
+    Annex.Content.LowLevel
+    Annex.Content.PointerFile
+    Annex.Difference
+    Annex.DirHashes
+    Annex.Direct
+    Annex.Drop
+    Annex.Environment
+    Annex.Export
+    Annex.FileMatcher
+    Annex.Fixup
+    Annex.GitOverlay
+    Annex.HashObject
+    Annex.Hook
+    Annex.Ingest
+    Annex.Init
+    Annex.InodeSentinal
+    Annex.Journal
+    Annex.Link
+    Annex.Locations
+    Annex.LockFile
+    Annex.LockPool
+    Annex.LockPool.PosixOrPid
+    Annex.MetaData
+    Annex.MetaData.StandardFields
+    Annex.Multicast
+    Annex.Notification
+    Annex.NumCopies
+    Annex.Path
+    Annex.Perms
+    Annex.Queue
+    Annex.ReplaceFile
+    Annex.SpecialRemote
+    Annex.Ssh
+    Annex.TaggedPush
+    Annex.Transfer
+    Annex.UpdateInstead
+    Annex.UUID
+    Annex.Url
+    Annex.VectorClock
+    Annex.VariantFile
+    Annex.Version
+    Annex.View
+    Annex.View.ViewedFile
+    Annex.Wanted
+    Annex.WorkTree
+    Annex.YoutubeDl
+    Backend
+    Backend.Hash
+    Backend.URL
+    Backend.Utilities
+    Backend.WORM
+    Build.BundledPrograms
+    Build.Configure
+    Build.DesktopFile
+    Build.Mans
+    Build.TestConfig
+    Build.Version
+    BuildInfo
+    BuildFlags
+    CmdLine
+    CmdLine.Action
+    CmdLine.Batch
+    CmdLine.GitAnnex
+    CmdLine.GitAnnex.Options
+    CmdLine.GitAnnexShell
+    CmdLine.GitAnnexShell.Checks
+    CmdLine.GitAnnexShell.Fields
+    CmdLine.GlobalSetter
+    CmdLine.Option
+    CmdLine.GitRemoteTorAnnex
+    CmdLine.Seek
+    CmdLine.Usage
+    Command
+    Command.Add
+    Command.AddUnused
+    Command.AddUrl
+    Command.Adjust
+    Command.CalcKey
+    Command.CheckPresentKey
+    Command.Commit
+    Command.Config
+    Command.ConfigList
+    Command.ContentLocation
+    Command.Copy
+    Command.Dead
+    Command.Describe
+    Command.DiffDriver
+    Command.Direct
+    Command.Drop
+    Command.DropKey
+    Command.DropUnused
+    Command.EnableRemote
+    Command.EnableTor
+    Command.ExamineKey
+    Command.Expire
+    Command.Export
+    Command.Find
+    Command.FindRef
+    Command.Fix
+    Command.Forget
+    Command.FromKey
+    Command.Fsck
+    Command.FuzzTest
+    Command.GCryptSetup
+    Command.Get
+    Command.Group
+    Command.GroupWanted
+    Command.Help
+    Command.Import
+    Command.ImportFeed
+    Command.InAnnex
+    Command.Indirect
+    Command.Info
+    Command.Init
+    Command.InitRemote
+    Command.Inprogress
+    Command.List
+    Command.Lock
+    Command.LockContent
+    Command.Log
+    Command.LookupKey
+    Command.Map
+    Command.MatchExpression
+    Command.Merge
+    Command.MetaData
+    Command.Migrate
+    Command.Mirror
+    Command.Move
+    Command.Multicast
+    Command.NotifyChanges
+    Command.NumCopies
+    Command.P2P
+    Command.P2PStdIO
+    Command.PostReceive
+    Command.PreCommit
+    Command.Proxy
+    Command.ReKey
+    Command.ReadPresentKey
+    Command.RecvKey
+    Command.RegisterUrl
+    Command.Reinit
+    Command.Reinject
+    Command.RemoteDaemon
+    Command.Repair
+    Command.Required
+    Command.ResolveMerge
+    Command.RmUrl
+    Command.Schedule
+    Command.Semitrust
+    Command.SendKey
+    Command.SetKey
+    Command.SetPresentKey
+    Command.Smudge
+    Command.Status
+    Command.Sync
+    Command.Test
+    Command.TestRemote
+    Command.TransferInfo
+    Command.TransferKey
+    Command.TransferKeys
+    Command.Trust
+    Command.Unannex
+    Command.Undo
+    Command.Ungroup
+    Command.Uninit
+    Command.Unlock
+    Command.Untrust
+    Command.Unused
+    Command.Upgrade
+    Command.VAdd
+    Command.VCycle
+    Command.VFilter
+    Command.VPop
+    Command.Version
+    Command.Vicfg
+    Command.View
+    Command.Wanted
+    Command.Whereis
+    Common
+    Config
+    Config.Cost
+    Config.Files
+    Config.DynamicConfig
+    Config.GitConfig
+    Config.Smudge
+    Creds
+    Crypto
+    Database.Export
+    Database.Fsck
+    Database.Handle
+    Database.Init
+    Database.Keys
+    Database.Keys.Handle
+    Database.Keys.SQL
+    Database.Queue
+    Database.Types
+    Git
+    Git.AutoCorrect
+    Git.Branch
+    Git.BuildVersion
+    Git.CatFile
+    Git.CheckAttr
+    Git.CheckIgnore
+    Git.Command
+    Git.Command.Batch
+    Git.Config
+    Git.ConfigTypes
+    Git.Construct
+    Git.CurrentRepo
+    Git.DiffTree
+    Git.DiffTreeItem
+    Git.Env
+    Git.FileMode
+    Git.FilePath
+    Git.Filename
+    Git.Fsck
+    Git.GCrypt
+    Git.HashObject
+    Git.Hook
+    Git.Index
+    Git.LockFile
+    Git.LsFiles
+    Git.LsTree
+    Git.Merge
+    Git.Objects
+    Git.Queue
+    Git.Ref
+    Git.RefLog
+    Git.Remote
+    Git.Remote.Remove
+    Git.Repair
+    Git.Sha
+    Git.Ssh
+    Git.Status
+    Git.Tree
+    Git.Types
+    Git.UnionMerge
+    Git.UpdateIndex
+    Git.Url
+    Git.Version
+    Key
+    Limit
+    Limit.Wanted
+    Logs
+    Logs.Activity
+    Logs.Chunk
+    Logs.Chunk.Pure
+    Logs.Config
+    Logs.Difference
+    Logs.Difference.Pure
+    Logs.Export
+    Logs.File
+    Logs.FsckResults
+    Logs.Group
+    Logs.Line
+    Logs.Location
+    Logs.MapLog
+    Logs.MetaData
+    Logs.MetaData.Pure
+    Logs.Multicast
+    Logs.NumCopies
+    Logs.PreferredContent
+    Logs.PreferredContent.Raw
+    Logs.Presence
+    Logs.Presence.Pure
+    Logs.Remote
+    Logs.RemoteState
+    Logs.Schedule
+    Logs.SingleValue
+    Logs.SingleValue.Pure
+    Logs.TimeStamp
+    Logs.Transfer
+    Logs.Transitions
+    Logs.Trust
+    Logs.Trust.Basic
+    Logs.Trust.Pure
+    Logs.UUID
+    Logs.UUIDBased
+    Logs.Unused
+    Logs.View
+    Logs.Web
+    Messages
+    Messages.Concurrent
+    Messages.Internal
+    Messages.JSON
+    Messages.Progress
+    P2P.Address
+    P2P.Annex
+    P2P.Auth
+    P2P.IO
+    P2P.Protocol
+    Remote
+    Remote.Adb
+    Remote.BitTorrent
+    Remote.Bup
+    Remote.Ddar
+    Remote.Directory
+    Remote.Directory.LegacyChunked
+    Remote.External
+    Remote.External.Types
+    Remote.GCrypt
+    Remote.Git
+    Remote.Glacier
+    Remote.Helper.AWS
+    Remote.Helper.Chunked
+    Remote.Helper.Chunked.Legacy
+    Remote.Helper.Encryptable
+    Remote.Helper.Export
+    Remote.Helper.Git
+    Remote.Helper.Hooks
+    Remote.Helper.Messages
+    Remote.Helper.P2P
+    Remote.Helper.ReadOnly
+    Remote.Helper.Special
+    Remote.Helper.Ssh
+    Remote.Hook
+    Remote.List
+    Remote.P2P
+    Remote.Rsync
+    Remote.Rsync.RsyncUrl
+    Remote.Tahoe
+    Remote.Web
+    RemoteDaemon.Common
+    RemoteDaemon.Core
+    RemoteDaemon.Transport
+    RemoteDaemon.Transport.GCrypt
+    RemoteDaemon.Transport.Tor
+    RemoteDaemon.Transport.Ssh
+    RemoteDaemon.Transport.Ssh.Types
+    RemoteDaemon.Types
+    Test
+    Test.Framework
+    Types
+    Types.ActionItem
+    Types.Availability
+    Types.Backend
+    Types.BranchState
+    Types.CleanupActions
+    Types.Command
+    Types.Concurrency
+    Types.Creds
+    Types.Crypto
+    Types.DeferredParse
+    Types.DesktopNotify
+    Types.Difference
+    Types.Distribution
+    Types.Export
+    Types.FileMatcher
+    Types.GitConfig
+    Types.Group
+    Types.Key
+    Types.KeySource
+    Types.LockCache
+    Types.Messages
+    Types.MetaData
+    Types.NumCopies
+    Types.RefSpec
+    Types.Remote
+    Types.ScheduledActivity
+    Types.StandardGroups
+    Types.StoreRetrieve
+    Types.Test
+    Types.Transfer
+    Types.TrustLevel
+    Types.UUID
+    Types.UrlContents
+    Types.View
+    Upgrade
+    Upgrade.V0
+    Upgrade.V1
+    Upgrade.V2
+    Upgrade.V3
+    Upgrade.V4
+    Upgrade.V5
+    Utility.Aeson
+    Utility.Android
+    Utility.Applicative
+    Utility.AuthToken
+    Utility.Base64
+    Utility.Batch
+    Utility.Bloom
+    Utility.CoProcess
+    Utility.CopyFile
+    Utility.Daemon
+    Utility.Data
+    Utility.DataUnits
+    Utility.DirWatcher
+    Utility.DirWatcher.Types
+    Utility.Directory
+    Utility.Directory.Stream
+    Utility.DiskFree
+    Utility.Dot
+    Utility.DottedVersion
+    Utility.Env
+    Utility.Env.Basic
+    Utility.Env.Set
+    Utility.Exception
+    Utility.FileMode
+    Utility.FileSize
+    Utility.FileSystemEncoding
+    Utility.Format
+    Utility.FreeDesktop
+    Utility.Glob
+    Utility.Gpg
+    Utility.Hash
+    Utility.HtmlDetect
+    Utility.HttpManagerRestricted
+    Utility.HumanNumber
+    Utility.HumanTime
+    Utility.InodeCache
+    Utility.IPAddress
+    Utility.LinuxMkLibs
+    Utility.LockFile
+    Utility.LockFile.LockStatus
+    Utility.LockFile.PidLock
+    Utility.LockPool
+    Utility.LockPool.LockHandle
+    Utility.LockPool.PidLock
+    Utility.LockPool.STM
+    Utility.LogFile
+    Utility.Lsof
+    Utility.MagicWormhole
+    Utility.Matcher
+    Utility.Metered
+    Utility.Misc
+    Utility.Monad
+    Utility.Network
+    Utility.NotificationBroadcaster
+    Utility.OptParse
+    Utility.PID
+    Utility.Parallel
+    Utility.PartialPrelude
+    Utility.Path
+    Utility.Path.Max
+    Utility.Percentage
+    Utility.Process
+    Utility.Process.Shim
+    Utility.Process.Transcript
+    Utility.QuickCheck
+    Utility.Rsync
+    Utility.SafeCommand
+    Utility.Scheduled
+    Utility.Scheduled.QuickCheck
+    Utility.Shell
+    Utility.SimpleProtocol
+    Utility.Split
+    Utility.SshConfig
+    Utility.SshHost
+    Utility.Su
+    Utility.SystemDirectory
+    Utility.TList
+    Utility.Tense
+    Utility.ThreadLock
+    Utility.ThreadScheduler
+    Utility.Tmp
+    Utility.Tmp.Dir
+    Utility.Tor
+    Utility.Touch
+    Utility.Tuple
+    Utility.Url
+    Utility.UserInfo
+    Utility.Verifiable
+
+  if (os(windows))
+    Other-Modules:
+      Utility.LockFile.Windows
+      Utility.LockPool.Windows
+  else
+    Other-Modules:
+      Utility.LockFile.Posix
+      Utility.LockPool.Posix
diff --git a/git-annex.hs b/git-annex.hs
new file mode 100644
index 0000000000..e9e8e7bc38
--- /dev/null
+++ b/git-annex.hs
@@ -0,0 +1,54 @@
+{- git-annex main program dispatch
+ -
+ - Copyright 2010-2016 Joey Hess 
+ -
+ - Licensed under the GNU GPL version 3 or higher.
+ -}
+
+{-# LANGUAGE CPP #-}
+
+import System.Environment (getArgs, getProgName)
+import System.FilePath
+import Network.Socket (withSocketsDo)
+
+import qualified CmdLine.GitAnnex
+import qualified CmdLine.GitAnnexShell
+import qualified CmdLine.GitRemoteTorAnnex
+import qualified Test
+import Utility.FileSystemEncoding
+
+#ifdef mingw32_HOST_OS
+import Utility.UserInfo
+import Utility.Env.Set
+#endif
+
+main :: IO ()
+main = withSocketsDo $ do
+	useFileSystemEncoding
+	ps <- getArgs
+#ifdef mingw32_HOST_OS
+	winEnv
+#endif
+	run ps =<< getProgName
+  where
+	run ps n = case takeFileName n of
+		"git-annex-shell" -> CmdLine.GitAnnexShell.run ps
+		"git-remote-tor-annex" -> CmdLine.GitRemoteTorAnnex.run ps
+		_  -> CmdLine.GitAnnex.run Test.optParser Test.runner ps
+
+#ifdef mingw32_HOST_OS
+{- On Windows, if HOME is not set, probe it and set it.
+ - This is a workaround for some Cygwin commands needing HOME to be set.
+ -
+ - If TZ is set, unset it.
+ - TZ being set can interfere with workarounds for Windows timezone
+ - horribleness, and prevents getCurrentTimeZone from seeing the system
+ - time zone.
+ -}
+winEnv :: IO ()
+winEnv = do
+	home <- myHomeDir
+	setEnv "HOME" home False
+	setEnv "CYGWIN" "nodosfilewarning" True
+	unsetEnv "TZ"
+#endif
diff --git a/git-union-merge.hs b/git-union-merge.hs
new file mode 100644
index 0000000000..18c88b1a9e
--- /dev/null
+++ b/git-union-merge.hs
@@ -0,0 +1,50 @@
+{- git-union-merge program
+ -
+ - Copyright 2011 Joey Hess 
+ -
+ - Licensed under the GNU GPL version 3 or higher.
+ -}
+
+import System.Environment
+
+import Common
+import qualified Git.UnionMerge
+import qualified Git.Config
+import qualified Git.CurrentRepo
+import qualified Git.Branch
+import qualified Git.Index
+import qualified Git
+import Utility.FileSystemEncoding
+
+header :: String
+header = "Usage: git-union-merge ref ref newref"
+
+usage :: IO a
+usage = error $ "bad parameters\n\n" ++ header
+
+tmpIndex :: Git.Repo -> FilePath
+tmpIndex g = Git.localGitDir g  "index.git-union-merge"
+
+setup :: Git.Repo -> IO ()
+setup = cleanup -- idempotency
+
+cleanup :: Git.Repo -> IO ()
+cleanup g = nukeFile $ tmpIndex g
+
+parseArgs :: IO [String]
+parseArgs = do
+	args <- getArgs
+	if length args /= 3
+		then usage
+		else return args
+
+main :: IO ()
+main = do
+	useFileSystemEncoding
+	[aref, bref, newref] <- map Git.Ref <$> parseArgs
+	g <- Git.Config.read =<< Git.CurrentRepo.get
+	_ <- Git.Index.override (tmpIndex g) g
+	setup g
+	Git.UnionMerge.merge aref bref g
+	_ <- Git.Branch.commit Git.Branch.ManualCommit False "union merge" newref [aref, bref] g
+	cleanup g
diff --git a/stack-windows.yaml b/stack-windows.yaml
new file mode 100644
index 0000000000..c5bdbfa622
--- /dev/null
+++ b/stack-windows.yaml
@@ -0,0 +1,29 @@
+flags:
+  git-annex:
+    concurrentoutput: true
+    production: true
+    assistant: true
+    pairing: true
+    s3: true
+    webdav: true
+    torrentparser: true
+    webapp: true
+    magicmime: false
+    dbus: false
+    android: false
+    androidsplice: false
+packages:
+- '.'
+extra-deps:
+- aws-0.17.1
+- bloomfilter-2.0.1.0
+- torrent-10000.1.1
+- Win32-2.6.1.0
+- directory-1.3.1.5
+- process-1.6.2.0
+- http-client-0.5.7.1
+- unix-compat-0.5.0.1
+allow-newer: true
+explicit-setup-deps:
+  git-annex: true
+resolver: lts-9.10
diff --git a/stack.yaml b/stack.yaml
new file mode 100644
index 0000000000..d135280fd6
--- /dev/null
+++ b/stack.yaml
@@ -0,0 +1,23 @@
+flags:
+  git-annex:
+    concurrentoutput: true
+    production: true
+    assistant: true
+    pairing: true
+    s3: true
+    webdav: true
+    torrentparser: true
+    webapp: true
+    magicmime: false
+    dbus: false
+    android: false
+    androidsplice: false
+packages:
+- '.'
+extra-deps:
+- aws-0.17.1
+- bloomfilter-2.0.1.0
+- torrent-10000.1.1
+explicit-setup-deps:
+  git-annex: true
+resolver: lts-9.9
diff --git a/standalone/android/.gitignore b/standalone/android/.gitignore
new file mode 100644
index 0000000000..e8792ba9fd
--- /dev/null
+++ b/standalone/android/.gitignore
@@ -0,0 +1,2 @@
+build-utils
+start
diff --git a/standalone/android/Makefile b/standalone/android/Makefile
new file mode 100644
index 0000000000..6f49b495f6
--- /dev/null
+++ b/standalone/android/Makefile
@@ -0,0 +1,84 @@
+# Cross-compiles utilities needed for git-annex on Android,
+# and builds the Android app.
+
+build:
+	./buildapk 4
+	./buildapk 5
+
+# Targets below are used by buildapk, which sets
+# GIT_ANNEX_ANDROID_SOURCETREE
+
+source: $(GIT_ANNEX_ANDROID_SOURCETREE)
+
+$(GIT_ANNEX_ANDROID_SOURCETREE):
+	mkdir -p $(GIT_ANNEX_ANDROID_SOURCETREE)
+	git clone git://git.debian.org/git/d-i/busybox $(GIT_ANNEX_ANDROID_SOURCETREE)/busybox
+	git clone git://git.kernel.org/pub/scm/git/git.git $(GIT_ANNEX_ANDROID_SOURCETREE)/git
+	git clone git://git.samba.org/rsync.git $(GIT_ANNEX_ANDROID_SOURCETREE)/rsync
+	git clone git://git.gnupg.org/gnupg.git $(GIT_ANNEX_ANDROID_SOURCETREE)/gnupg
+	git clone git://git.openssl.org/openssl $(GIT_ANNEX_ANDROID_SOURCETREE)/openssl
+	git clone git://github.com/CyanogenMod/android_external_openssh.git $(GIT_ANNEX_ANDROID_SOURCETREE)/openssh
+	git clone git://github.com/jackpal/Android-Terminal-Emulator.git $(GIT_ANNEX_ANDROID_SOURCETREE)/term
+
+$(GIT_ANNEX_ANDROID_SOURCETREE)/openssl/build-stamp:
+	# This is a version which the openssh below can build with.
+	# Newer versions changed something to do with BIGNUM.
+	cd $(GIT_ANNEX_ANDROID_SOURCETREE)/openssl && git reset --hard 616f71e486d693991b594439c884ec624b32c2d4
+	cd $(GIT_ANNEX_ANDROID_SOURCETREE)/openssl && CC=$$(which cc) ./Configure android
+	cd $(GIT_ANNEX_ANDROID_SOURCETREE)/openssl && $(MAKE)
+	touch $@
+
+$(GIT_ANNEX_ANDROID_SOURCETREE)/openssh/build-stamp: openssh.patch openssh.config.h
+	# This is a known-good version that the patch works with.
+	# TODO: Upgrade
+	cd $(GIT_ANNEX_ANDROID_SOURCETREE)/openssh && git reset --hard 0a8617ed5af2f0248d0e9648e26b224e16ada742
+	cd $(GIT_ANNEX_ANDROID_SOURCETREE)/openssh && ./configure --host=arm-linux-androideabi --with-ssl-dir=../openssl --without-openssl-header-check
+	cat openssh.patch | (cd $(GIT_ANNEX_ANDROID_SOURCETREE)/openssh && patch -p1)
+	cp openssh.config.h $(GIT_ANNEX_ANDROID_SOURCETREE)/openssh/config.h
+	cd $(GIT_ANNEX_ANDROID_SOURCETREE)/openssh && sed -i -e 's/getrrsetbyname.o //' openbsd-compat/Makefile
+	cd $(GIT_ANNEX_ANDROID_SOURCETREE)/openssh && sed -i -e 's/auth-passwd.o //' Makefile
+	cd $(GIT_ANNEX_ANDROID_SOURCETREE)/openssh && $(MAKE) ssh ssh-keygen
+	touch $@
+
+$(GIT_ANNEX_ANDROID_SOURCETREE)/busybox/build-stamp: busybox_config
+	cp busybox_config $(GIT_ANNEX_ANDROID_SOURCETREE)/busybox/.config
+	cd $(GIT_ANNEX_ANDROID_SOURCETREE)/busybox && git reset --hard a758e3e1e04e7705f5d37b2f27be654cd0e7282c
+	cd $(GIT_ANNEX_ANDROID_SOURCETREE)/busybox && yes '' | $(MAKE) oldconfig
+	cd $(GIT_ANNEX_ANDROID_SOURCETREE)/busybox && $(MAKE)
+	touch $@
+	
+$(GIT_ANNEX_ANDROID_SOURCETREE)/git/build-stamp: git.patch
+	# This is a known-good version that the patch works with.
+	cat git.patch | (cd $(GIT_ANNEX_ANDROID_SOURCETREE)/git && git reset --hard d9c691a759d62cef53a6cc11864a2ef4b0829244 && git am)
+	cd $(GIT_ANNEX_ANDROID_SOURCETREE)/git && $(MAKE) install NO_OPENSSL=1 NO_GETTEXT=1 NO_GECOS_IN_PWENT=1 NO_GETPASS=1 NO_NSEC=1 NO_MKDTEMP=1 NO_PTHREADS=1 NO_PERL=1 NO_CURL=1 NO_EXPAT=1 NO_TCLTK=1 NO_ICONV=1 HAVE_CLOCK_GETTIME= HAVE_GETDELIM= prefix= DESTDIR=installed-tree
+	touch $@
+
+$(GIT_ANNEX_ANDROID_SOURCETREE)/rsync/build-stamp: rsync.patch
+	# This is a known-good version that the patch works with.
+	cat rsync.patch | (cd $(GIT_ANNEX_ANDROID_SOURCETREE)/rsync && git reset --hard eec26089b1c7bdbb260674480ffe6ece257bca63 && git am)
+	cp /usr/share/misc/config.sub /usr/share/misc/config.guess $(GIT_ANNEX_ANDROID_SOURCETREE)/rsync/
+	cd $(GIT_ANNEX_ANDROID_SOURCETREE)/rsync && ./configure --host=arm-linux-androideabi --disable-locale --disable-iconv-open --disable-iconv --disable-acl-support --disable-xattr-support
+	cd $(GIT_ANNEX_ANDROID_SOURCETREE)/rsync && $(MAKE)
+	touch $@
+
+$(GIT_ANNEX_ANDROID_SOURCETREE)/gnupg/build-stamp:
+	cd $(GIT_ANNEX_ANDROID_SOURCETREE)/gnupg && git checkout gnupg-1.4.15
+	cd $(GIT_ANNEX_ANDROID_SOURCETREE)/gnupg && ./autogen.sh
+	cd $(GIT_ANNEX_ANDROID_SOURCETREE)/gnupg && ./configure --host=arm-linux-androideabi --disable-gnupg-iconv --disable-card-support --disable-agent-support --disable-photo-viewers --disable-keyserver-helpers --disable-nls
+	cd $(GIT_ANNEX_ANDROID_SOURCETREE)/gnupg; $(MAKE) || true # expected failure in doc build
+	touch $@
+
+$(GIT_ANNEX_ANDROID_SOURCETREE)/term/build-stamp: term.patch icons
+	# This is a known-good version that the patch works with.
+	cd $(GIT_ANNEX_ANDROID_SOURCETREE)/term && git reset --hard 3d34b3c42295c215b62e70f3ee696dd664ba08ce
+	cat term.patch | (cd $(GIT_ANNEX_ANDROID_SOURCETREE)/term && patch -p1)
+	(cd icons && tar c .) | (cd $(GIT_ANNEX_ANDROID_SOURCETREE)/term/res && tar x)
+	# This renaming has a purpose. It makes the path to the app's
+	# /data directory shorter, which makes ssh connection caching
+	# sockets placed there have more space for their filenames.
+	# Also, it avoids overlap with the Android Terminal Emulator
+	# app, if it's also installed.
+	cd $(GIT_ANNEX_ANDROID_SOURCETREE)/term && find -name .git -prune -o -type f -print0 | xargs -0 perl -pi -e 's/jackpal/ga/g'
+	cd $(GIT_ANNEX_ANDROID_SOURCETREE)/term && perl -pi -e 's/Terminal Emulator/Git Annex/g' res/*/strings.xml
+	cd $(GIT_ANNEX_ANDROID_SOURCETREE)/term && echo y | tools/update.sh || true
+	touch $@
diff --git a/standalone/android/abiversion b/standalone/android/abiversion
new file mode 100644
index 0000000000..4cc4c14cdb
--- /dev/null
+++ b/standalone/android/abiversion
@@ -0,0 +1 @@
+android-14/arm-linux-androideabi-4.8
diff --git a/standalone/android/buildapk b/standalone/android/buildapk
new file mode 100755
index 0000000000..310949c089
--- /dev/null
+++ b/standalone/android/buildapk
@@ -0,0 +1,112 @@
+#!/bin/sh
+#
+# Cross-compiles utilities needed for git-annex on Android,
+# and builds the Android app.
+
+set -e
+
+androidversion=$1
+if [ -z "$androidversion" ]; then
+	echo "need android version (4 or 5) as parameter" >&2
+	exit 1
+fi
+
+VER="$(perl -e '$_=<>;print m/\((.*?)\)/'<../../CHANGELOG)"
+
+PATH=$(./toolchainpath "$androidversion")
+export PATH
+
+# Paths to the Android SDK and NDK.
+export ANDROID_SDK_ROOT="$HOME/.android/adt-bundle-linux-x86/sdk"
+export ANDROID_NDK_ROOT="$HOME/.android/android-ndk"
+
+GIT_ANNEX_ANDROID_SOURCETREE="$HOME/.android/git-annex-sourcetree"
+export GIT_ANNEX_ANDROID_SOURCETREE
+if [ ! -e "$GIT_ANNEX_ANDROID_SOURCETREE" ]; then
+	make source
+fi
+src="$GIT_ANNEX_ANDROID_SOURCETREE-$androidversion"
+if [ ! -e "$src" ] ; then
+	cp -a "$GIT_ANNEX_ANDROID_SOURCETREE" "$src"
+fi
+GIT_ANNEX_ANDROID_SOURCETREE="$src"
+export GIT_ANNEX_ANDROID_SOURCETREE
+
+gittree="$GIT_ANNEX_ANDROID_SOURCETREE/git/installed-tree"
+
+make "$GIT_ANNEX_ANDROID_SOURCETREE/openssl/build-stamp"
+make "$GIT_ANNEX_ANDROID_SOURCETREE/openssh/build-stamp"
+make "$GIT_ANNEX_ANDROID_SOURCETREE/busybox/build-stamp"
+make "$GIT_ANNEX_ANDROID_SOURCETREE/rsync/build-stamp"
+make "$GIT_ANNEX_ANDROID_SOURCETREE/gnupg/build-stamp"
+make "$GIT_ANNEX_ANDROID_SOURCETREE/git/build-stamp"
+make "$GIT_ANNEX_ANDROID_SOURCETREE/term/build-stamp"
+
+perl -i -pe 's/(android:versionName=)"[^"]+"/$1"'"$VER"'"/' \
+	"$GIT_ANNEX_ANDROID_SOURCETREE/term/AndroidManifest.xml"
+
+# Debug build because it does not need signing keys.
+(cd "$GIT_ANNEX_ANDROID_SOURCETREE/term" && tools/build-debug)
+
+# Install executables as pseudo-libraries so they will be
+# unpacked from the .apk.
+mkdir -p "$GIT_ANNEX_ANDROID_SOURCETREE/term/libs/armeabi"
+cp "$GIT_ANNEX_ANDROID_SOURCETREE/busybox/busybox" "$GIT_ANNEX_ANDROID_SOURCETREE/term/libs/armeabi/lib.busybox.so"
+cp "$GIT_ANNEX_ANDROID_SOURCETREE/openssh/ssh" "$GIT_ANNEX_ANDROID_SOURCETREE/term/libs/armeabi/lib.ssh.so"
+cp "$GIT_ANNEX_ANDROID_SOURCETREE/openssh/ssh-keygen" "$GIT_ANNEX_ANDROID_SOURCETREE/term/libs/armeabi/lib.ssh-keygen.so"
+cp "$GIT_ANNEX_ANDROID_SOURCETREE/rsync/rsync" "$GIT_ANNEX_ANDROID_SOURCETREE/term/libs/armeabi/lib.rsync.so"
+cp "$GIT_ANNEX_ANDROID_SOURCETREE/gnupg/g10/gpg" "$GIT_ANNEX_ANDROID_SOURCETREE/term/libs/armeabi/lib.gpg.so"
+cp "$GIT_ANNEX_ANDROID_SOURCETREE/git/git" "$GIT_ANNEX_ANDROID_SOURCETREE/term/libs/armeabi/lib.git.so"
+cp "$GIT_ANNEX_ANDROID_SOURCETREE/git/git-shell" "$GIT_ANNEX_ANDROID_SOURCETREE/term/libs/armeabi/lib.git-shell.so"
+cp "$GIT_ANNEX_ANDROID_SOURCETREE/git/git-upload-pack" "$GIT_ANNEX_ANDROID_SOURCETREE/term/libs/armeabi/lib.git-upload-pack.so"
+arm-linux-androideabi-strip --strip-unneeded --remove-section=.comment --remove-section=.note "$GIT_ANNEX_ANDROID_SOURCETREE"/term/libs/armeabi/*
+cp runshell "$GIT_ANNEX_ANDROID_SOURCETREE/term/libs/armeabi/lib.runshell.so"
+cc start.c -o "$GIT_ANNEX_ANDROID_SOURCETREE/term/libs/armeabi/lib.start.so"
+
+# remove git stuff we don't need to save space
+rm -rf $gittree/bin/git-cvsserver \
+	$gittree/libexec/git-core/git-daemon \
+	$gittree/libexec/git-core/git-show-index \
+	$gittree/libexec/git-core/mergetools \
+	$gittree/libexec/git-core/git-credential-* \
+	$gittree/libexec/git-core/git-cvsserver \
+	$gittree/libexec/git-core/git-cvsimport \
+	$gittree/libexec/git-core/git-fast-import \
+	$gittree/libexec/git-core/git-http-backend \
+	$gittree/libexec/git-core/git-imap-send \
+	$gittree/libexec/git-core/git-instaweb \
+	$gittree/libexec/git-core/git-p4 \
+	$gittree/libexec/git-core/git-remote-test* \
+	$gittree/libexec/git-core/git-submodule \
+	$gittree/libexec/git-core/git-svn \
+	$gittree/libexec/git-core/git-web--browse
+
+# Most of git is in one multicall binary, but a few important
+# commands are still shell scripts. Those are put into
+# a tarball, along with a list of all the links that should be
+# set up.
+(cd $gittree && mkdir -p links)
+(cd $gittree && find -samefile bin/git -not -wholename ./bin/git > links/git)
+(cd $gittree && find -samefile bin/git-shell -not -wholename ./bin/git-shell > links/git-shell)
+(cd $gittree && find -samefile bin/git-upload-pack -not -wholename ./bin/git-upload-pack > links/git-upload-pack)
+(cd $gittree && find -type f -not -samefile bin/git -not -samefile bin/git-shell -not -samefile bin/git-upload-pack | tar czf ../git.tar.gz -T -)
+(cp "$GIT_ANNEX_ANDROID_SOURCETREE/git/git.tar.gz" "$GIT_ANNEX_ANDROID_SOURCETREE/term/libs/armeabi/lib.git.tar.gz.so")
+
+git rev-parse HEAD > "$GIT_ANNEX_ANDROID_SOURCETREE/term/libs/armeabi/lib.version.so"
+cp ../trustedkeys.gpg "$GIT_ANNEX_ANDROID_SOURCETREE/term/libs/armeabi/lib.trustedkeys.so"
+
+genapk () {
+	mkdir -p ../../tmp/$1; \
+	cp ../../tmp/androidtree/dist/build/git-annex/$1/git-annex "$GIT_ANNEX_ANDROID_SOURCETREE/term/libs/armeabi/lib.git-annex.so"
+	arm-linux-androideabi-strip --strip-unneeded --remove-section=.comment --remove-section=.note "$GIT_ANNEX_ANDROID_SOURCETREE/term/libs/armeabi/lib.git-annex.so"
+	(cd "$GIT_ANNEX_ANDROID_SOURCETREE/term" && ant debug)
+	cp "$GIT_ANNEX_ANDROID_SOURCETREE/term/bin/Term-debug.apk" ../../tmp/$1/git-annex.apk
+}
+
+if [ "$androidversion" = 4 ]; then
+	for v in 4.0 4.3; do
+		genapk $v
+	done
+else
+	genapk 5.0
+fi
diff --git a/standalone/android/buildchroot b/standalone/android/buildchroot
new file mode 100755
index 0000000000..e64bdd03d7
--- /dev/null
+++ b/standalone/android/buildchroot
@@ -0,0 +1,27 @@
+#!/bin/sh
+set -e
+if [ "$(whoami)" != root ]; then
+	echo "Must run this as root!" >&2
+	exit 1
+fi
+
+debootstrap --arch=i386 jessie debian-stable-android
+cp $0-inchroot debian-stable-android/tmp
+cp $0-inchroot-asuser debian-stable-android/tmp
+cp $(dirname $0)/abiversion debian-stable-android/tmp
+
+# Don't use these vars in the chroot.
+unset TMP
+unset TEMP
+unset TMPDIR
+unset TEMPDIR
+
+chroot debian-stable-android "tmp/$(basename $0)-inchroot"
+
+echo
+echo
+echo "debian-stable-android is set up, with a user builder"
+echo "your next step is probably to check out git-annex in this chroot"
+echo "and run standalone/android/install-haskell-packages"
+echo
+echo
diff --git a/standalone/android/buildchroot-inchroot b/standalone/android/buildchroot-inchroot
new file mode 100755
index 0000000000..69653b7585
--- /dev/null
+++ b/standalone/android/buildchroot-inchroot
@@ -0,0 +1,27 @@
+#!/bin/sh
+# Runs inside the chroot set up by buildchroot
+set -e
+set -x
+if [ "$(whoami)" != root ]; then
+	echo "Must run this as root!" >&2
+	exit 1
+fi
+
+# java needs this mounted to work
+mount -t proc proc /proc || true
+
+echo "deb-src http://ftp.us.debian.org/debian jessie main" >> /etc/apt/sources.list
+apt-get update
+apt-get -y install build-essential ghc git libncurses5-dev cabal-install
+apt-get -y install happy alex
+apt-get -y install llvm-3.4
+apt-get -y install ca-certificates curl file m4 autoconf zlib1g-dev
+apt-get -y install libgnutls28-dev libxml2-dev libgsasl7-dev pkg-config c2hs
+apt-get -y install ant default-jdk rsync wget gnupg lsof
+apt-get -y install gettext unzip python
+apt-get -y install locales automake
+echo en_US.UTF-8 UTF-8 >> /etc/locale.gen
+locale-gen
+apt-get clean
+useradd builder --create-home || true
+su builder -c $0-asuser
diff --git a/standalone/android/buildchroot-inchroot-asuser b/standalone/android/buildchroot-inchroot-asuser
new file mode 100755
index 0000000000..a7bea231a8
--- /dev/null
+++ b/standalone/android/buildchroot-inchroot-asuser
@@ -0,0 +1,40 @@
+#!/bin/sh
+# Runs inside the chroot set up by buildchroot, as the user it creates
+set -e
+set -x
+
+# Put in /tmp by buildchroot, but when bootstrapping with propellor,
+# this is run inside a checked out git-annex tree.
+if [ -e /tmp/abiversion ]; then
+	ABIVERSION=$(cat /tmp/abiversion)
+else
+	ABIVERSION=$(cat standalone/android/abiversion)
+fi
+
+cd
+rm -rf .ghc .cabal .android
+mkdir -p .android
+cd .android
+git clone https://github.com/joeyh/ghc-android
+cd ghc-android
+git checkout jessie-ghc-snapshot
+./build
+
+# This saves 2 gb, and the same sources are in build-*/ghc
+rm -rf stage0
+
+# Set up android SDK where the git-annex android Makefile
+# expects to find it.
+cd ..
+ln -s ghc-android/android-ndk-* android-ndk
+wget http://dl.google.com/android/adt/adt-bundle-linux-x86-20130917.zip
+unzip adt*.zip
+rm adt*.zip
+mv adt-bundle-linux-x86-* adt-bundle-linux-x86
+rm -rf adt-bundle-linux-x86/eclipse
+
+# The git-annex android Makefile needs this cc symlink.
+ln -s arm-linux-androideabi-gcc $HOME/.ghc/$ABIVERSION/bin/cc
+
+git config --global user.email androidbuilder@example.com
+git config --global user.name androidbuilder
diff --git a/standalone/android/busybox_config b/standalone/android/busybox_config
new file mode 100644
index 0000000000..28ea880d98
--- /dev/null
+++ b/standalone/android/busybox_config
@@ -0,0 +1,997 @@
+# Run "make android2_defconfig", then "make".
+#
+# Tested with the standalone toolchain from ndk r6:
+# android-ndk-r6/build/tools/make-standalone-toolchain.sh --platform=android-8
+#
+CONFIG_HAVE_DOT_CONFIG=y
+
+#
+# Busybox Settings
+#
+
+#
+# General Configuration
+#
+# CONFIG_DESKTOP is not set
+# CONFIG_EXTRA_COMPAT is not set
+# CONFIG_INCLUDE_SUSv2 is not set
+# CONFIG_USE_PORTABLE_CODE is not set
+CONFIG_PLATFORM_LINUX=y
+CONFIG_FEATURE_BUFFERS_USE_MALLOC=y
+# CONFIG_FEATURE_BUFFERS_GO_ON_STACK is not set
+# CONFIG_FEATURE_BUFFERS_GO_IN_BSS is not set
+# CONFIG_SHOW_USAGE is not set
+# CONFIG_FEATURE_VERBOSE_USAGE is not set
+# CONFIG_FEATURE_COMPRESS_USAGE is not set
+CONFIG_FEATURE_INSTALLER=y
+# CONFIG_INSTALL_NO_USR is not set
+# CONFIG_LOCALE_SUPPORT is not set
+# CONFIG_UNICODE_SUPPORT is not set
+# CONFIG_UNICODE_USING_LOCALE is not set
+# CONFIG_FEATURE_CHECK_UNICODE_IN_ENV is not set
+CONFIG_SUBST_WCHAR=0
+CONFIG_LAST_SUPPORTED_WCHAR=0
+# CONFIG_UNICODE_COMBINING_WCHARS is not set
+# CONFIG_UNICODE_WIDE_WCHARS is not set
+# CONFIG_UNICODE_BIDI_SUPPORT is not set
+# CONFIG_UNICODE_NEUTRAL_TABLE is not set
+# CONFIG_UNICODE_PRESERVE_BROKEN is not set
+# CONFIG_LONG_OPTS is not set
+# CONFIG_FEATURE_DEVPTS is not set
+# CONFIG_FEATURE_CLEAN_UP is not set
+# CONFIG_FEATURE_UTMP is not set
+# CONFIG_FEATURE_WTMP is not set
+# CONFIG_FEATURE_PIDFILE is not set
+# CONFIG_FEATURE_SUID is not set
+# CONFIG_FEATURE_SUID_CONFIG is not set
+# CONFIG_FEATURE_SUID_CONFIG_QUIET is not set
+# CONFIG_SELINUX is not set
+# CONFIG_FEATURE_PREFER_APPLETS is not set
+CONFIG_BUSYBOX_EXEC_PATH="/proc/self/exe"
+CONFIG_FEATURE_SYSLOG=y
+# CONFIG_FEATURE_HAVE_RPC is not set
+
+#
+# Build Options
+#
+# CONFIG_STATIC is not set
+# CONFIG_PIE is not set
+# CONFIG_NOMMU is not set
+# CONFIG_BUILD_LIBBUSYBOX is not set
+# CONFIG_FEATURE_INDIVIDUAL is not set
+# CONFIG_FEATURE_SHARED_BUSYBOX is not set
+# CONFIG_LFS is not set
+CONFIG_CROSS_COMPILER_PREFIX="arm-linux-androideabi-"
+CONFIG_EXTRA_CFLAGS=""
+
+#
+# Debugging Options
+#
+# CONFIG_DEBUG is not set
+# CONFIG_DEBUG_PESSIMIZE is not set
+# CONFIG_WERROR is not set
+CONFIG_NO_DEBUG_LIB=y
+# CONFIG_DMALLOC is not set
+# CONFIG_EFENCE is not set
+
+#
+# Installation Options ("make install" behavior)
+#
+CONFIG_INSTALL_APPLET_SYMLINKS=y
+# CONFIG_INSTALL_APPLET_HARDLINKS is not set
+# CONFIG_INSTALL_APPLET_SCRIPT_WRAPPERS is not set
+# CONFIG_INSTALL_APPLET_DONT is not set
+# CONFIG_INSTALL_SH_APPLET_SYMLINK is not set
+# CONFIG_INSTALL_SH_APPLET_HARDLINK is not set
+# CONFIG_INSTALL_SH_APPLET_SCRIPT_WRAPPER is not set
+CONFIG_PREFIX="./_install"
+
+#
+# Busybox Library Tuning
+#
+# CONFIG_FEATURE_SYSTEMD is not set
+# CONFIG_FEATURE_RTMINMAX is not set
+CONFIG_PASSWORD_MINLEN=6
+CONFIG_MD5_SMALL=1
+# CONFIG_FEATURE_FAST_TOP is not set
+# CONFIG_FEATURE_ETC_NETWORKS is not set
+CONFIG_FEATURE_USE_TERMIOS=y
+# CONFIG_FEATURE_EDITING is not set
+CONFIG_FEATURE_EDITING_MAX_LEN=0
+# CONFIG_FEATURE_EDITING_VI is not set
+CONFIG_FEATURE_EDITING_HISTORY=0
+# CONFIG_FEATURE_EDITING_SAVEHISTORY is not set
+# CONFIG_FEATURE_TAB_COMPLETION is not set
+# CONFIG_FEATURE_USERNAME_COMPLETION is not set
+# CONFIG_FEATURE_EDITING_FANCY_PROMPT is not set
+# CONFIG_FEATURE_EDITING_ASK_TERMINAL is not set
+# CONFIG_FEATURE_NON_POSIX_CP is not set
+# CONFIG_FEATURE_VERBOSE_CP_MESSAGE is not set
+CONFIG_FEATURE_COPYBUF_KB=4
+# CONFIG_FEATURE_SKIP_ROOTFS is not set
+# CONFIG_MONOTONIC_SYSCALL is not set
+# CONFIG_IOCTL_HEX2STR_ERROR is not set
+# CONFIG_FEATURE_HWIB is not set
+
+#
+# Applets
+#
+
+#
+# Archival Utilities
+#
+CONFIG_FEATURE_SEAMLESS_XZ=y
+CONFIG_FEATURE_SEAMLESS_LZMA=y
+CONFIG_FEATURE_SEAMLESS_BZ2=y
+CONFIG_FEATURE_SEAMLESS_GZ=y
+CONFIG_FEATURE_SEAMLESS_Z=y
+CONFIG_AR=y
+CONFIG_FEATURE_AR_LONG_FILENAMES=y
+CONFIG_FEATURE_AR_CREATE=y
+CONFIG_BUNZIP2=y
+CONFIG_BZIP2=y
+CONFIG_CPIO=y
+CONFIG_FEATURE_CPIO_O=y
+CONFIG_FEATURE_CPIO_P=y
+CONFIG_DPKG=y
+CONFIG_DPKG_DEB=y
+# CONFIG_FEATURE_DPKG_DEB_EXTRACT_ONLY is not set
+CONFIG_GUNZIP=y
+CONFIG_GZIP=y
+# CONFIG_FEATURE_GZIP_LONG_OPTIONS is not set
+CONFIG_LZOP=y
+CONFIG_LZOP_COMPR_HIGH=y
+CONFIG_RPM2CPIO=y
+CONFIG_RPM=y
+CONFIG_TAR=y
+CONFIG_FEATURE_TAR_CREATE=y
+CONFIG_FEATURE_TAR_AUTODETECT=y
+CONFIG_FEATURE_TAR_FROM=y
+CONFIG_FEATURE_TAR_OLDGNU_COMPATIBILITY=y
+CONFIG_FEATURE_TAR_OLDSUN_COMPATIBILITY=y
+CONFIG_FEATURE_TAR_GNU_EXTENSIONS=y
+# CONFIG_FEATURE_TAR_LONG_OPTIONS is not set
+# CONFIG_FEATURE_TAR_TO_COMMAND is not set
+CONFIG_FEATURE_TAR_UNAME_GNAME=y
+CONFIG_FEATURE_TAR_NOPRESERVE_TIME=y
+# CONFIG_FEATURE_TAR_SELINUX is not set
+CONFIG_UNCOMPRESS=y
+CONFIG_UNLZMA=y
+CONFIG_FEATURE_LZMA_FAST=y
+CONFIG_LZMA=y
+CONFIG_UNXZ=y
+CONFIG_XZ=y
+CONFIG_UNZIP=y
+
+#
+# Coreutils
+#
+CONFIG_BASENAME=y
+CONFIG_CAT=y
+# CONFIG_DATE is not set
+# CONFIG_FEATURE_DATE_ISOFMT is not set
+# CONFIG_FEATURE_DATE_NANO is not set
+# CONFIG_FEATURE_DATE_COMPAT is not set
+# CONFIG_ID is not set
+# CONFIG_GROUPS is not set
+CONFIG_TEST=y
+CONFIG_FEATURE_TEST_64=y
+CONFIG_TOUCH=y
+CONFIG_TR=y
+CONFIG_FEATURE_TR_CLASSES=y
+CONFIG_FEATURE_TR_EQUIV=y
+CONFIG_BASE64=y
+CONFIG_CAL=y
+CONFIG_CATV=y
+CONFIG_CHGRP=y
+CONFIG_CHMOD=y
+CONFIG_CHOWN=y
+# CONFIG_FEATURE_CHOWN_LONG_OPTIONS is not set
+CONFIG_CHROOT=y
+CONFIG_CKSUM=y
+CONFIG_COMM=y
+CONFIG_CP=y
+# CONFIG_FEATURE_CP_LONG_OPTIONS is not set
+CONFIG_CUT=y
+CONFIG_DD=y
+CONFIG_FEATURE_DD_SIGNAL_HANDLING=y
+CONFIG_FEATURE_DD_THIRD_STATUS_LINE=y
+CONFIG_FEATURE_DD_IBS_OBS=y
+# CONFIG_DF is not set
+# CONFIG_FEATURE_DF_FANCY is not set
+CONFIG_DIRNAME=y
+CONFIG_DOS2UNIX=y
+CONFIG_UNIX2DOS=y
+CONFIG_DU=y
+CONFIG_FEATURE_DU_DEFAULT_BLOCKSIZE_1K=y
+CONFIG_ECHO=y
+CONFIG_FEATURE_FANCY_ECHO=y
+# CONFIG_ENV is not set
+# CONFIG_FEATURE_ENV_LONG_OPTIONS is not set
+CONFIG_EXPAND=y
+# CONFIG_FEATURE_EXPAND_LONG_OPTIONS is not set
+# CONFIG_EXPR is not set
+# CONFIG_EXPR_MATH_SUPPORT_64 is not set
+CONFIG_FALSE=y
+CONFIG_FOLD=y
+# CONFIG_FSYNC is not set
+CONFIG_HEAD=y
+CONFIG_FEATURE_FANCY_HEAD=y
+# CONFIG_HOSTID is not set
+CONFIG_INSTALL=y
+# CONFIG_FEATURE_INSTALL_LONG_OPTIONS is not set
+CONFIG_LN=y
+# CONFIG_LOGNAME is not set
+CONFIG_LS=y
+CONFIG_FEATURE_LS_FILETYPES=y
+CONFIG_FEATURE_LS_FOLLOWLINKS=y
+CONFIG_FEATURE_LS_RECURSIVE=y
+CONFIG_FEATURE_LS_SORTFILES=y
+CONFIG_FEATURE_LS_TIMESTAMPS=y
+CONFIG_FEATURE_LS_USERNAME=y
+# CONFIG_FEATURE_LS_COLOR is not set
+# CONFIG_FEATURE_LS_COLOR_IS_DEFAULT is not set
+CONFIG_MD5SUM=y
+CONFIG_MKDIR=y
+# CONFIG_FEATURE_MKDIR_LONG_OPTIONS is not set
+CONFIG_MKFIFO=y
+CONFIG_MKNOD=y
+CONFIG_MV=y
+# CONFIG_FEATURE_MV_LONG_OPTIONS is not set
+CONFIG_NICE=y
+CONFIG_NOHUP=y
+CONFIG_OD=y
+CONFIG_PRINTENV=y
+CONFIG_PRINTF=y
+CONFIG_PWD=y
+CONFIG_READLINK=y
+CONFIG_FEATURE_READLINK_FOLLOW=y
+CONFIG_REALPATH=y
+CONFIG_RM=y
+CONFIG_RMDIR=y
+# CONFIG_FEATURE_RMDIR_LONG_OPTIONS is not set
+CONFIG_SEQ=y
+CONFIG_SHA1SUM=y
+CONFIG_SHA256SUM=y
+CONFIG_SHA512SUM=y
+CONFIG_SLEEP=y
+CONFIG_FEATURE_FANCY_SLEEP=y
+CONFIG_FEATURE_FLOAT_SLEEP=y
+CONFIG_SORT=y
+CONFIG_FEATURE_SORT_BIG=y
+CONFIG_SPLIT=y
+CONFIG_FEATURE_SPLIT_FANCY=y
+# CONFIG_STAT is not set
+# CONFIG_FEATURE_STAT_FORMAT is not set
+CONFIG_STTY=y
+CONFIG_SUM=y
+CONFIG_SYNC=y
+CONFIG_TAC=y
+CONFIG_TAIL=y
+CONFIG_FEATURE_FANCY_TAIL=y
+CONFIG_TEE=y
+CONFIG_FEATURE_TEE_USE_BLOCK_IO=y
+CONFIG_TRUE=y
+# CONFIG_TTY is not set
+CONFIG_UNAME=y
+CONFIG_UNEXPAND=y
+# CONFIG_FEATURE_UNEXPAND_LONG_OPTIONS is not set
+CONFIG_UNIQ=y
+# CONFIG_USLEEP is not set
+CONFIG_UUDECODE=y
+CONFIG_UUENCODE=y
+CONFIG_WC=y
+CONFIG_FEATURE_WC_LARGE=y
+# CONFIG_WHO is not set
+CONFIG_WHOAMI=y
+CONFIG_YES=y
+
+#
+# Common options for cp and mv
+#
+CONFIG_FEATURE_PRESERVE_HARDLINKS=y
+
+#
+# Common options for ls, more and telnet
+#
+CONFIG_FEATURE_AUTOWIDTH=y
+
+#
+# Common options for df, du, ls
+#
+CONFIG_FEATURE_HUMAN_READABLE=y
+
+#
+# Common options for md5sum, sha1sum, sha256sum, sha512sum
+#
+CONFIG_FEATURE_MD5_SHA1_SUM_CHECK=y
+
+#
+# Console Utilities
+#
+CONFIG_CHVT=y
+CONFIG_FGCONSOLE=y
+CONFIG_CLEAR=y
+CONFIG_DEALLOCVT=y
+CONFIG_DUMPKMAP=y
+# CONFIG_KBD_MODE is not set
+# CONFIG_LOADFONT is not set
+CONFIG_LOADKMAP=y
+CONFIG_OPENVT=y
+CONFIG_RESET=y
+CONFIG_RESIZE=y
+CONFIG_FEATURE_RESIZE_PRINT=y
+CONFIG_SETCONSOLE=y
+# CONFIG_FEATURE_SETCONSOLE_LONG_OPTIONS is not set
+# CONFIG_SETFONT is not set
+# CONFIG_FEATURE_SETFONT_TEXTUAL_MAP is not set
+CONFIG_DEFAULT_SETFONT_DIR=""
+CONFIG_SETKEYCODES=y
+CONFIG_SETLOGCONS=y
+CONFIG_SHOWKEY=y
+# CONFIG_FEATURE_LOADFONT_PSF2 is not set
+# CONFIG_FEATURE_LOADFONT_RAW is not set
+
+#
+# Debian Utilities
+#
+CONFIG_MKTEMP=y
+CONFIG_PIPE_PROGRESS=y
+CONFIG_RUN_PARTS=y
+# CONFIG_FEATURE_RUN_PARTS_LONG_OPTIONS is not set
+CONFIG_FEATURE_RUN_PARTS_FANCY=y
+CONFIG_START_STOP_DAEMON=y
+CONFIG_FEATURE_START_STOP_DAEMON_FANCY=y
+# CONFIG_FEATURE_START_STOP_DAEMON_LONG_OPTIONS is not set
+CONFIG_WHICH=y
+
+#
+# Editors
+#
+CONFIG_PATCH=y
+CONFIG_VI=y
+CONFIG_FEATURE_VI_MAX_LEN=0
+# CONFIG_FEATURE_VI_8BIT is not set
+# CONFIG_FEATURE_VI_COLON is not set
+# CONFIG_FEATURE_VI_YANKMARK is not set
+CONFIG_FEATURE_VI_SEARCH=y
+# CONFIG_FEATURE_VI_REGEX_SEARCH is not set
+CONFIG_FEATURE_VI_USE_SIGNALS=y
+# CONFIG_FEATURE_VI_DOT_CMD is not set
+# CONFIG_FEATURE_VI_READONLY is not set
+# CONFIG_FEATURE_VI_SETOPTS is not set
+# CONFIG_FEATURE_VI_SET is not set
+CONFIG_FEATURE_VI_WIN_RESIZE=y
+# CONFIG_FEATURE_VI_ASK_TERMINAL is not set
+# CONFIG_FEATURE_VI_OPTIMIZE_CURSOR is not set
+# CONFIG_AWK is not set
+# CONFIG_FEATURE_AWK_LIBM is not set
+CONFIG_CMP=y
+CONFIG_DIFF=y
+# CONFIG_FEATURE_DIFF_LONG_OPTIONS is not set
+CONFIG_FEATURE_DIFF_DIR=y
+# CONFIG_ED is not set
+CONFIG_SED=y
+# CONFIG_FEATURE_ALLOW_EXEC is not set
+
+#
+# Finding Utilities
+#
+CONFIG_FIND=y
+CONFIG_FEATURE_FIND_PRINT0=y
+CONFIG_FEATURE_FIND_MTIME=y
+# CONFIG_FEATURE_FIND_MMIN is not set
+CONFIG_FEATURE_FIND_PERM=y
+CONFIG_FEATURE_FIND_TYPE=y
+# CONFIG_FEATURE_FIND_XDEV is not set
+# CONFIG_FEATURE_FIND_MAXDEPTH is not set
+# CONFIG_FEATURE_FIND_NEWER is not set
+# CONFIG_FEATURE_FIND_INUM is not set
+# CONFIG_FEATURE_FIND_EXEC is not set
+# CONFIG_FEATURE_FIND_USER is not set
+# CONFIG_FEATURE_FIND_GROUP is not set
+# CONFIG_FEATURE_FIND_NOT is not set
+# CONFIG_FEATURE_FIND_DEPTH is not set
+# CONFIG_FEATURE_FIND_PAREN is not set
+# CONFIG_FEATURE_FIND_SIZE is not set
+# CONFIG_FEATURE_FIND_PRUNE is not set
+# CONFIG_FEATURE_FIND_DELETE is not set
+# CONFIG_FEATURE_FIND_PATH is not set
+# CONFIG_FEATURE_FIND_REGEX is not set
+# CONFIG_FEATURE_FIND_CONTEXT is not set
+# CONFIG_FEATURE_FIND_LINKS is not set
+CONFIG_GREP=y
+# CONFIG_FEATURE_GREP_EGREP_ALIAS is not set
+# CONFIG_FEATURE_GREP_FGREP_ALIAS is not set
+# CONFIG_FEATURE_GREP_CONTEXT is not set
+CONFIG_XARGS=y
+CONFIG_FEATURE_XARGS_SUPPORT_CONFIRMATION=y
+CONFIG_FEATURE_XARGS_SUPPORT_QUOTES=y
+CONFIG_FEATURE_XARGS_SUPPORT_TERMOPT=y
+CONFIG_FEATURE_XARGS_SUPPORT_ZERO_TERM=y
+
+#
+# Init Utilities
+#
+# CONFIG_BOOTCHARTD is not set
+# CONFIG_FEATURE_BOOTCHARTD_BLOATED_HEADER is not set
+# CONFIG_FEATURE_BOOTCHARTD_CONFIG_FILE is not set
+# CONFIG_HALT is not set
+# CONFIG_FEATURE_CALL_TELINIT is not set
+# CONFIG_TELINIT_PATH=""
+# CONFIG_INIT is not set
+# CONFIG_FEATURE_USE_INITTAB is not set
+# CONFIG_FEATURE_KILL_REMOVED is not set
+# CONFIG_FEATURE_KILL_DELAY=0
+# CONFIG_FEATURE_INIT_SCTTY is not set
+# CONFIG_FEATURE_INIT_SYSLOG is not set
+# CONFIG_FEATURE_EXTRA_QUIET is not set
+# CONFIG_FEATURE_INIT_COREDUMPS is not set
+# CONFIG_FEATURE_INITRD is not set
+# CONFIG_INIT_TERMINAL_TYPE="linux"
+# CONFIG_MESG is not set
+# CONFIG_FEATURE_MESG_ENABLE_ONLY_GROUP=y
+
+#
+# Login/Password Management Utilities
+#
+# CONFIG_ADD_SHELL is not set
+# CONFIG_REMOVE_SHELL is not set
+# CONFIG_FEATURE_SHADOWPASSWDS is not set
+# CONFIG_USE_BB_PWD_GRP is not set
+# CONFIG_USE_BB_SHADOW is not set
+# CONFIG_USE_BB_CRYPT is not set
+# CONFIG_USE_BB_CRYPT_SHA is not set
+# CONFIG_ADDUSER is not set
+# CONFIG_FEATURE_ADDUSER_LONG_OPTIONS is not set
+# CONFIG_FEATURE_CHECK_NAMES is not set
+CONFIG_FIRST_SYSTEM_ID=0
+CONFIG_LAST_SYSTEM_ID=0
+# CONFIG_ADDGROUP is not set
+# CONFIG_FEATURE_ADDGROUP_LONG_OPTIONS is not set
+# CONFIG_FEATURE_ADDUSER_TO_GROUP is not set
+# CONFIG_DELUSER is not set
+# CONFIG_DELGROUP is not set
+# CONFIG_FEATURE_DEL_USER_FROM_GROUP is not set
+# CONFIG_GETTY is not set
+# CONFIG_LOGIN is not set
+# CONFIG_PAM is not set
+# CONFIG_LOGIN_SCRIPTS is not set
+# CONFIG_FEATURE_NOLOGIN is not set
+# CONFIG_FEATURE_SECURETTY is not set
+# CONFIG_PASSWD is not set
+# CONFIG_FEATURE_PASSWD_WEAK_CHECK is not set
+# CONFIG_CRYPTPW is not set
+# CONFIG_CHPASSWD is not set
+# CONFIG_SU is not set
+# CONFIG_FEATURE_SU_SYSLOG is not set
+# CONFIG_FEATURE_SU_CHECKS_SHELLS is not set
+# CONFIG_SULOGIN is not set
+# CONFIG_VLOCK is not set
+
+#
+# Linux Ext2 FS Progs
+#
+CONFIG_CHATTR=y
+# CONFIG_FSCK is not set
+CONFIG_LSATTR=y
+CONFIG_TUNE2FS=y
+
+#
+# Linux Module Utilities
+#
+# CONFIG_MODINFO is not set
+# CONFIG_MODPROBE_SMALL is not set
+# CONFIG_FEATURE_MODPROBE_SMALL_OPTIONS_ON_CMDLINE=y
+# CONFIG_FEATURE_MODPROBE_SMALL_CHECK_ALREADY_LOADED=y
+# CONFIG_INSMOD is not set
+# CONFIG_RMMOD is not set
+# CONFIG_LSMOD is not set
+# CONFIG_FEATURE_LSMOD_PRETTY_2_6_OUTPUT is not set
+# CONFIG_MODPROBE is not set
+# CONFIG_FEATURE_MODPROBE_BLACKLIST is not set
+# CONFIG_DEPMOD is not set
+
+#
+# Options common to multiple modutils
+#
+# CONFIG_FEATURE_2_4_MODULES is not set
+# CONFIG_FEATURE_INSMOD_TRY_MMAP is not set
+# CONFIG_FEATURE_INSMOD_VERSION_CHECKING is not set
+# CONFIG_FEATURE_INSMOD_KSYMOOPS_SYMBOLS is not set
+# CONFIG_FEATURE_INSMOD_LOADINKMEM is not set
+# CONFIG_FEATURE_INSMOD_LOAD_MAP is not set
+# CONFIG_FEATURE_INSMOD_LOAD_MAP_FULL is not set
+# CONFIG_FEATURE_CHECK_TAINTED_MODULE is not set
+# CONFIG_FEATURE_MODUTILS_ALIAS is not set
+# CONFIG_FEATURE_MODUTILS_SYMBOLS is not set
+# CONFIG_DEFAULT_MODULES_DIR="/lib/modules"
+# CONFIG_DEFAULT_DEPMOD_FILE="modules.dep"
+
+#
+# Linux System Utilities
+#
+CONFIG_BLOCKDEV=y
+CONFIG_REV=y
+# CONFIG_ACPID is not set
+# CONFIG_FEATURE_ACPID_COMPAT is not set
+CONFIG_BLKID=y
+# CONFIG_FEATURE_BLKID_TYPE is not set
+CONFIG_DMESG=y
+CONFIG_FEATURE_DMESG_PRETTY=y
+CONFIG_FBSET=y
+CONFIG_FEATURE_FBSET_FANCY=y
+CONFIG_FEATURE_FBSET_READMODE=y
+CONFIG_FDFLUSH=y
+CONFIG_FDFORMAT=y
+CONFIG_FDISK=y
+CONFIG_FDISK_SUPPORT_LARGE_DISKS=y
+CONFIG_FEATURE_FDISK_WRITABLE=y
+# CONFIG_FEATURE_AIX_LABEL is not set
+# CONFIG_FEATURE_SGI_LABEL is not set
+# CONFIG_FEATURE_SUN_LABEL is not set
+# CONFIG_FEATURE_OSF_LABEL is not set
+# CONFIG_FEATURE_GPT_LABEL is not set
+CONFIG_FEATURE_FDISK_ADVANCED=y
+CONFIG_FINDFS=y
+CONFIG_FLOCK=y
+CONFIG_FREERAMDISK=y
+# CONFIG_FSCK_MINIX is not set
+# CONFIG_MKFS_EXT2 is not set
+# CONFIG_MKFS_MINIX is not set
+# CONFIG_FEATURE_MINIX2 is not set
+# CONFIG_MKFS_REISER is not set
+# CONFIG_MKFS_VFAT is not set
+CONFIG_GETOPT=y
+CONFIG_FEATURE_GETOPT_LONG=y
+CONFIG_HEXDUMP=y
+CONFIG_FEATURE_HEXDUMP_REVERSE=y
+CONFIG_HD=y
+# CONFIG_HWCLOCK is not set
+# CONFIG_FEATURE_HWCLOCK_LONG_OPTIONS is not set
+# CONFIG_FEATURE_HWCLOCK_ADJTIME_FHS is not set
+# CONFIG_IPCRM is not set
+# CONFIG_IPCS is not set
+CONFIG_LOSETUP=y
+CONFIG_LSPCI=y
+CONFIG_LSUSB=y
+# CONFIG_MDEV is not set
+# CONFIG_FEATURE_MDEV_CONF is not set
+# CONFIG_FEATURE_MDEV_RENAME is not set
+# CONFIG_FEATURE_MDEV_RENAME_REGEXP is not set
+# CONFIG_FEATURE_MDEV_EXEC is not set
+# CONFIG_FEATURE_MDEV_LOAD_FIRMWARE is not set
+CONFIG_MKSWAP=y
+CONFIG_FEATURE_MKSWAP_UUID=y
+CONFIG_MORE=y
+# CONFIG_MOUNT is not set
+# CONFIG_FEATURE_MOUNT_FAKE is not set
+# CONFIG_FEATURE_MOUNT_VERBOSE is not set
+# CONFIG_FEATURE_MOUNT_HELPERS is not set
+# CONFIG_FEATURE_MOUNT_LABEL is not set
+# CONFIG_FEATURE_MOUNT_NFS is not set
+# CONFIG_FEATURE_MOUNT_CIFS is not set
+# CONFIG_FEATURE_MOUNT_FLAGS is not set
+# CONFIG_FEATURE_MOUNT_FSTAB is not set
+# CONFIG_PIVOT_ROOT is not set
+# CONFIG_RDATE is not set
+CONFIG_RDEV=y
+CONFIG_READPROFILE=y
+CONFIG_RTCWAKE=y
+CONFIG_SCRIPT=y
+CONFIG_SCRIPTREPLAY=y
+# CONFIG_SETARCH is not set
+# CONFIG_SWAPONOFF is not set
+# CONFIG_FEATURE_SWAPON_PRI is not set
+# CONFIG_SWITCH_ROOT is not set
+# CONFIG_UMOUNT is not set
+# CONFIG_FEATURE_UMOUNT_ALL is not set
+# CONFIG_FEATURE_MOUNT_LOOP is not set
+# CONFIG_FEATURE_MOUNT_LOOP_CREATE is not set
+# CONFIG_FEATURE_MTAB_SUPPORT is not set
+CONFIG_VOLUMEID=y
+
+#
+# Filesystem/Volume identification
+#
+CONFIG_FEATURE_VOLUMEID_EXT=y
+CONFIG_FEATURE_VOLUMEID_BTRFS=y
+CONFIG_FEATURE_VOLUMEID_REISERFS=y
+CONFIG_FEATURE_VOLUMEID_FAT=y
+CONFIG_FEATURE_VOLUMEID_HFS=y
+CONFIG_FEATURE_VOLUMEID_JFS=y
+CONFIG_FEATURE_VOLUMEID_XFS=y
+CONFIG_FEATURE_VOLUMEID_NTFS=y
+CONFIG_FEATURE_VOLUMEID_ISO9660=y
+CONFIG_FEATURE_VOLUMEID_UDF=y
+CONFIG_FEATURE_VOLUMEID_LUKS=y
+CONFIG_FEATURE_VOLUMEID_LINUXSWAP=y
+CONFIG_FEATURE_VOLUMEID_CRAMFS=y
+CONFIG_FEATURE_VOLUMEID_ROMFS=y
+CONFIG_FEATURE_VOLUMEID_SYSV=y
+CONFIG_FEATURE_VOLUMEID_OCFS2=y
+CONFIG_FEATURE_VOLUMEID_LINUXRAID=y
+
+#
+# Miscellaneous Utilities
+#
+# CONFIG_CONSPY is not set
+# CONFIG_NANDWRITE is not set
+CONFIG_NANDDUMP=y
+CONFIG_SETSERIAL=y
+# CONFIG_UBIATTACH is not set
+# CONFIG_UBIDETACH is not set
+# CONFIG_UBIMKVOL is not set
+# CONFIG_UBIRMVOL is not set
+# CONFIG_UBIRSVOL is not set
+# CONFIG_UBIUPDATEVOL is not set
+# CONFIG_ADJTIMEX is not set
+# CONFIG_BBCONFIG is not set
+# CONFIG_FEATURE_COMPRESS_BBCONFIG is not set
+CONFIG_BEEP=y
+CONFIG_FEATURE_BEEP_FREQ=4000
+CONFIG_FEATURE_BEEP_LENGTH_MS=30
+CONFIG_CHAT=y
+CONFIG_FEATURE_CHAT_NOFAIL=y
+# CONFIG_FEATURE_CHAT_TTY_HIFI is not set
+CONFIG_FEATURE_CHAT_IMPLICIT_CR=y
+CONFIG_FEATURE_CHAT_SWALLOW_OPTS=y
+CONFIG_FEATURE_CHAT_SEND_ESCAPES=y
+CONFIG_FEATURE_CHAT_VAR_ABORT_LEN=y
+CONFIG_FEATURE_CHAT_CLR_ABORT=y
+CONFIG_CHRT=y
+# CONFIG_CROND is not set
+# CONFIG_FEATURE_CROND_D is not set
+# CONFIG_FEATURE_CROND_CALL_SENDMAIL is not set
+CONFIG_FEATURE_CROND_DIR=""
+# CONFIG_CRONTAB is not set
+CONFIG_DC=y
+CONFIG_FEATURE_DC_LIBM=y
+# CONFIG_DEVFSD is not set
+# CONFIG_DEVFSD_MODLOAD is not set
+# CONFIG_DEVFSD_FG_NP is not set
+# CONFIG_DEVFSD_VERBOSE is not set
+# CONFIG_FEATURE_DEVFS is not set
+CONFIG_DEVMEM=y
+# CONFIG_EJECT is not set
+# CONFIG_FEATURE_EJECT_SCSI is not set
+CONFIG_FBSPLASH=y
+CONFIG_FLASHCP=y
+CONFIG_FLASH_LOCK=y
+CONFIG_FLASH_UNLOCK=y
+# CONFIG_FLASH_ERASEALL is not set
+# CONFIG_IONICE is not set
+CONFIG_INOTIFYD=y
+# CONFIG_LAST is not set
+# CONFIG_FEATURE_LAST_SMALL is not set
+# CONFIG_FEATURE_LAST_FANCY is not set
+# CONFIG_LESS is not set
+CONFIG_FEATURE_LESS_MAXLINES=0
+# CONFIG_FEATURE_LESS_BRACKETS is not set
+# CONFIG_FEATURE_LESS_FLAGS is not set
+# CONFIG_FEATURE_LESS_MARKS is not set
+# CONFIG_FEATURE_LESS_REGEXP is not set
+# CONFIG_FEATURE_LESS_WINCH is not set
+# CONFIG_FEATURE_LESS_DASHCMD is not set
+# CONFIG_FEATURE_LESS_LINENUMS is not set
+CONFIG_HDPARM=y
+CONFIG_FEATURE_HDPARM_GET_IDENTITY=y
+CONFIG_FEATURE_HDPARM_HDIO_SCAN_HWIF=y
+CONFIG_FEATURE_HDPARM_HDIO_UNREGISTER_HWIF=y
+CONFIG_FEATURE_HDPARM_HDIO_DRIVE_RESET=y
+CONFIG_FEATURE_HDPARM_HDIO_TRISTATE_HWIF=y
+CONFIG_FEATURE_HDPARM_HDIO_GETSET_DMA=y
+CONFIG_MAKEDEVS=y
+# CONFIG_FEATURE_MAKEDEVS_LEAF is not set
+CONFIG_FEATURE_MAKEDEVS_TABLE=y
+CONFIG_MAN=y
+# CONFIG_MICROCOM is not set
+# CONFIG_MOUNTPOINT is not set
+# CONFIG_MT is not set
+CONFIG_RAIDAUTORUN=y
+# CONFIG_READAHEAD is not set
+# CONFIG_RFKILL is not set
+# CONFIG_RUNLEVEL is not set
+CONFIG_RX=y
+CONFIG_SETSID=y
+CONFIG_STRINGS=y
+# CONFIG_TASKSET is not set
+# CONFIG_FEATURE_TASKSET_FANCY is not set
+CONFIG_TIME=y
+CONFIG_TIMEOUT=y
+CONFIG_TTYSIZE=y
+CONFIG_VOLNAME=y
+# CONFIG_WALL is not set
+# CONFIG_WATCHDOG is not set
+
+#
+# Networking Utilities
+#
+# CONFIG_NAMEIF is not set
+# CONFIG_FEATURE_NAMEIF_EXTENDED is not set
+CONFIG_NBDCLIENT=y
+CONFIG_NC=y
+CONFIG_NC_SERVER=y
+CONFIG_NC_EXTRA=y
+# CONFIG_NC_110_COMPAT is not set
+# CONFIG_PING is not set
+# CONFIG_PING6 is not set
+# CONFIG_FEATURE_FANCY_PING is not set
+CONFIG_WHOIS=y
+# CONFIG_FEATURE_IPV6 is not set
+# CONFIG_FEATURE_UNIX_LOCAL is not set
+# CONFIG_FEATURE_PREFER_IPV4_ADDRESS is not set
+# CONFIG_VERBOSE_RESOLUTION_ERRORS is not set
+CONFIG_ARP=y
+# CONFIG_ARPING is not set
+# CONFIG_BRCTL is not set
+# CONFIG_FEATURE_BRCTL_FANCY is not set
+# CONFIG_FEATURE_BRCTL_SHOW is not set
+CONFIG_DNSD=y
+# CONFIG_ETHER_WAKE is not set
+CONFIG_FAKEIDENTD=y
+CONFIG_FTPD=y
+CONFIG_FEATURE_FTP_WRITE=y
+CONFIG_FEATURE_FTPD_ACCEPT_BROKEN_LIST=y
+CONFIG_FTPGET=y
+CONFIG_FTPPUT=y
+# CONFIG_FEATURE_FTPGETPUT_LONG_OPTIONS is not set
+# CONFIG_HOSTNAME is not set
+CONFIG_HTTPD=y
+CONFIG_FEATURE_HTTPD_RANGES=y
+CONFIG_FEATURE_HTTPD_USE_SENDFILE=y
+CONFIG_FEATURE_HTTPD_SETUID=y
+CONFIG_FEATURE_HTTPD_BASIC_AUTH=y
+# CONFIG_FEATURE_HTTPD_AUTH_MD5 is not set
+CONFIG_FEATURE_HTTPD_CGI=y
+CONFIG_FEATURE_HTTPD_CONFIG_WITH_SCRIPT_INTERPR=y
+CONFIG_FEATURE_HTTPD_SET_REMOTE_PORT_TO_ENV=y
+CONFIG_FEATURE_HTTPD_ENCODE_URL_STR=y
+CONFIG_FEATURE_HTTPD_ERROR_PAGES=y
+CONFIG_FEATURE_HTTPD_PROXY=y
+CONFIG_FEATURE_HTTPD_GZIP=y
+CONFIG_IFCONFIG=y
+CONFIG_FEATURE_IFCONFIG_STATUS=y
+# CONFIG_FEATURE_IFCONFIG_SLIP is not set
+CONFIG_FEATURE_IFCONFIG_MEMSTART_IOADDR_IRQ=y
+CONFIG_FEATURE_IFCONFIG_HW=y
+CONFIG_FEATURE_IFCONFIG_BROADCAST_PLUS=y
+# CONFIG_IFENSLAVE is not set
+# CONFIG_IFPLUGD is not set
+CONFIG_IFUPDOWN=y
+CONFIG_IFUPDOWN_IFSTATE_PATH="/var/run/ifstate"
+CONFIG_FEATURE_IFUPDOWN_IP=y
+CONFIG_FEATURE_IFUPDOWN_IP_BUILTIN=y
+# CONFIG_FEATURE_IFUPDOWN_IFCONFIG_BUILTIN is not set
+CONFIG_FEATURE_IFUPDOWN_IPV4=y
+# CONFIG_FEATURE_IFUPDOWN_IPV6 is not set
+CONFIG_FEATURE_IFUPDOWN_MAPPING=y
+# CONFIG_FEATURE_IFUPDOWN_EXTERNAL_DHCP is not set
+# CONFIG_INETD is not set
+# CONFIG_FEATURE_INETD_SUPPORT_BUILTIN_ECHO is not set
+# CONFIG_FEATURE_INETD_SUPPORT_BUILTIN_DISCARD is not set
+# CONFIG_FEATURE_INETD_SUPPORT_BUILTIN_TIME is not set
+# CONFIG_FEATURE_INETD_SUPPORT_BUILTIN_DAYTIME is not set
+# CONFIG_FEATURE_INETD_SUPPORT_BUILTIN_CHARGEN is not set
+# CONFIG_FEATURE_INETD_RPC is not set
+CONFIG_IP=y
+CONFIG_FEATURE_IP_ADDRESS=y
+CONFIG_FEATURE_IP_LINK=y
+CONFIG_FEATURE_IP_ROUTE=y
+CONFIG_FEATURE_IP_TUNNEL=y
+CONFIG_FEATURE_IP_RULE=y
+CONFIG_FEATURE_IP_SHORT_FORMS=y
+# CONFIG_FEATURE_IP_RARE_PROTOCOLS is not set
+CONFIG_IPADDR=y
+CONFIG_IPLINK=y
+CONFIG_IPROUTE=y
+CONFIG_IPTUNNEL=y
+CONFIG_IPRULE=y
+CONFIG_IPCALC=y
+CONFIG_FEATURE_IPCALC_FANCY=y
+# CONFIG_FEATURE_IPCALC_LONG_OPTIONS is not set
+CONFIG_NETSTAT=y
+CONFIG_FEATURE_NETSTAT_WIDE=y
+CONFIG_FEATURE_NETSTAT_PRG=y
+# CONFIG_NSLOOKUP is not set
+# CONFIG_NTPD is not set
+# CONFIG_FEATURE_NTPD_SERVER is not set
+CONFIG_PSCAN=y
+CONFIG_ROUTE=y
+# CONFIG_SLATTACH is not set
+CONFIG_TCPSVD=y
+# CONFIG_TELNET is not set
+# CONFIG_FEATURE_TELNET_TTYPE is not set
+# CONFIG_FEATURE_TELNET_AUTOLOGIN is not set
+# CONFIG_TELNETD is not set
+# CONFIG_FEATURE_TELNETD_STANDALONE is not set
+# CONFIG_FEATURE_TELNETD_INETD_WAIT is not set
+# CONFIG_TFTP is not set
+# CONFIG_TFTPD is not set
+# CONFIG_FEATURE_TFTP_GET is not set
+# CONFIG_FEATURE_TFTP_PUT is not set
+# CONFIG_FEATURE_TFTP_BLOCKSIZE is not set
+# CONFIG_FEATURE_TFTP_PROGRESS_BAR is not set
+# CONFIG_TFTP_DEBUG is not set
+# CONFIG_TRACEROUTE is not set
+# CONFIG_TRACEROUTE6 is not set
+# CONFIG_FEATURE_TRACEROUTE_VERBOSE is not set
+# CONFIG_FEATURE_TRACEROUTE_SOURCE_ROUTE is not set
+# CONFIG_FEATURE_TRACEROUTE_USE_ICMP is not set
+CONFIG_TUNCTL=y
+CONFIG_FEATURE_TUNCTL_UG=y
+# CONFIG_UDHCPD is not set
+# CONFIG_DHCPRELAY is not set
+# CONFIG_DUMPLEASES is not set
+# CONFIG_FEATURE_UDHCPD_WRITE_LEASES_EARLY is not set
+# CONFIG_FEATURE_UDHCPD_BASE_IP_ON_MAC is not set
+CONFIG_DHCPD_LEASES_FILE=""
+CONFIG_UDHCPC=y
+CONFIG_FEATURE_UDHCPC_ARPING=y
+# CONFIG_FEATURE_UDHCP_PORT is not set
+CONFIG_UDHCP_DEBUG=9
+CONFIG_FEATURE_UDHCP_RFC3397=y
+CONFIG_FEATURE_UDHCP_8021Q=y
+CONFIG_UDHCPC_DEFAULT_SCRIPT="/usr/share/udhcpc/default.script"
+CONFIG_UDHCPC_SLACK_FOR_BUGGY_SERVERS=80
+CONFIG_IFUPDOWN_UDHCPC_CMD_OPTIONS="-R -n"
+# CONFIG_UDPSVD is not set
+# CONFIG_VCONFIG is not set
+CONFIG_WGET=y
+CONFIG_FEATURE_WGET_STATUSBAR=y
+CONFIG_FEATURE_WGET_AUTHENTICATION=y
+# CONFIG_FEATURE_WGET_LONG_OPTIONS is not set
+CONFIG_FEATURE_WGET_TIMEOUT=y
+# CONFIG_ZCIP is not set
+
+#
+# Print Utilities
+#
+CONFIG_LPD=y
+CONFIG_LPR=y
+CONFIG_LPQ=y
+
+#
+# Mail Utilities
+#
+CONFIG_MAKEMIME=y
+CONFIG_FEATURE_MIME_CHARSET="us-ascii"
+CONFIG_POPMAILDIR=y
+CONFIG_FEATURE_POPMAILDIR_DELIVERY=y
+CONFIG_REFORMIME=y
+CONFIG_FEATURE_REFORMIME_COMPAT=y
+CONFIG_SENDMAIL=y
+
+#
+# Process Utilities
+#
+CONFIG_IOSTAT=y
+CONFIG_MPSTAT=y
+CONFIG_NMETER=y
+CONFIG_PMAP=y
+# CONFIG_POWERTOP is not set
+CONFIG_PSTREE=y
+CONFIG_PWDX=y
+CONFIG_SMEMCAP=y
+# CONFIG_FREE is not set
+CONFIG_FUSER=y
+# CONFIG_KILL is not set
+# CONFIG_KILLALL is not set
+# CONFIG_KILLALL5 is not set
+# CONFIG_PGREP is not set
+CONFIG_PIDOF=y
+CONFIG_FEATURE_PIDOF_SINGLE=y
+CONFIG_FEATURE_PIDOF_OMIT=y
+# CONFIG_PKILL is not set
+# CONFIG_PS is not set
+# CONFIG_FEATURE_PS_WIDE is not set
+# CONFIG_FEATURE_PS_TIME is not set
+# CONFIG_FEATURE_PS_ADDITIONAL_COLUMNS is not set
+# CONFIG_FEATURE_PS_UNUSUAL_SYSTEMS is not set
+CONFIG_RENICE=y
+CONFIG_BB_SYSCTL=y
+CONFIG_TOP=y
+CONFIG_FEATURE_TOP_CPU_USAGE_PERCENTAGE=y
+CONFIG_FEATURE_TOP_CPU_GLOBAL_PERCENTS=y
+CONFIG_FEATURE_TOP_SMP_CPU=y
+CONFIG_FEATURE_TOP_DECIMALS=y
+CONFIG_FEATURE_TOP_SMP_PROCESS=y
+CONFIG_FEATURE_TOPMEM=y
+CONFIG_FEATURE_SHOW_THREADS=y
+# CONFIG_UPTIME is not set
+CONFIG_WATCH=y
+
+#
+# Runit Utilities
+#
+CONFIG_RUNSV=y
+CONFIG_RUNSVDIR=y
+# CONFIG_FEATURE_RUNSVDIR_LOG is not set
+CONFIG_SV=y
+CONFIG_SV_DEFAULT_SERVICE_DIR="/var/service"
+CONFIG_SVLOGD=y
+CONFIG_CHPST=y
+CONFIG_SETUIDGID=y
+CONFIG_ENVUIDGID=y
+CONFIG_ENVDIR=y
+CONFIG_SOFTLIMIT=y
+# CONFIG_CHCON is not set
+# CONFIG_FEATURE_CHCON_LONG_OPTIONS is not set
+# CONFIG_GETENFORCE is not set
+# CONFIG_GETSEBOOL is not set
+# CONFIG_LOAD_POLICY is not set
+# CONFIG_MATCHPATHCON is not set
+# CONFIG_RESTORECON is not set
+# CONFIG_RUNCON is not set
+# CONFIG_FEATURE_RUNCON_LONG_OPTIONS is not set
+# CONFIG_SELINUXENABLED is not set
+# CONFIG_SETENFORCE is not set
+# CONFIG_SETFILES is not set
+# CONFIG_FEATURE_SETFILES_CHECK_OPTION is not set
+# CONFIG_SETSEBOOL is not set
+# CONFIG_SESTATUS is not set
+
+#
+# Shells
+#
+CONFIG_ASH=y
+# CONFIG_ASH_BASH_COMPAT is not set
+# CONFIG_ASH_IDLE_TIMEOUT is not set
+CONFIG_ASH_JOB_CONTROL=y
+# CONFIG_ASH_ALIAS is not set
+CONFIG_ASH_GETOPTS=y
+# CONFIG_ASH_BUILTIN_ECHO is not set
+# CONFIG_ASH_BUILTIN_PRINTF is not set
+# CONFIG_ASH_BUILTIN_TEST is not set
+# CONFIG_ASH_CMDCMD is not set
+# CONFIG_ASH_MAIL is not set
+# CONFIG_ASH_OPTIMIZE_FOR_SIZE is not set
+# CONFIG_ASH_RANDOM_SUPPORT is not set
+# CONFIG_ASH_EXPAND_PRMT is not set
+CONFIG_CTTYHACK=y
+# CONFIG_HUSH is not set
+# CONFIG_HUSH_BASH_COMPAT is not set
+# CONFIG_HUSH_BRACE_EXPANSION is not set
+# CONFIG_HUSH_HELP is not set
+# CONFIG_HUSH_INTERACTIVE is not set
+# CONFIG_HUSH_SAVEHISTORY is not set
+# CONFIG_HUSH_JOB is not set
+# CONFIG_HUSH_TICK is not set
+# CONFIG_HUSH_IF is not set
+# CONFIG_HUSH_LOOPS is not set
+# CONFIG_HUSH_CASE is not set
+# CONFIG_HUSH_FUNCTIONS is not set
+# CONFIG_HUSH_LOCAL is not set
+# CONFIG_HUSH_RANDOM_SUPPORT is not set
+# CONFIG_HUSH_EXPORT_N is not set
+# CONFIG_HUSH_MODE_X is not set
+# CONFIG_MSH is not set
+CONFIG_FEATURE_SH_IS_ASH=y
+# CONFIG_FEATURE_SH_IS_HUSH is not set
+# CONFIG_FEATURE_SH_IS_NONE is not set
+# CONFIG_FEATURE_BASH_IS_ASH is not set
+# CONFIG_FEATURE_BASH_IS_HUSH is not set
+CONFIG_FEATURE_BASH_IS_NONE=y
+# CONFIG_SH_MATH_SUPPORT is not set
+# CONFIG_SH_MATH_SUPPORT_64 is not set
+# CONFIG_FEATURE_SH_EXTRA_QUIET is not set
+CONFIG_FEATURE_SH_STANDALONE=y
+# CONFIG_FEATURE_SH_NOFORK is not set
+# CONFIG_FEATURE_SH_HISTFILESIZE is not set
+
+#
+# System Logging Utilities
+#
+# CONFIG_SYSLOGD is not set
+# CONFIG_FEATURE_ROTATE_LOGFILE is not set
+# CONFIG_FEATURE_REMOTE_LOG is not set
+# CONFIG_FEATURE_SYSLOGD_DUP is not set
+# CONFIG_FEATURE_SYSLOGD_CFG is not set
+CONFIG_FEATURE_SYSLOGD_READ_BUFFER_SIZE=0
+# CONFIG_FEATURE_IPC_SYSLOG is not set
+CONFIG_FEATURE_IPC_SYSLOG_BUFFER_SIZE=0
+# CONFIG_LOGREAD is not set
+# CONFIG_FEATURE_LOGREAD_REDUCED_LOCKING is not set
+CONFIG_KLOGD=y
+CONFIG_FEATURE_KLOGD_KLOGCTL=y
+# CONFIG_LOGGER is not set
diff --git a/standalone/android/cabal.config b/standalone/android/cabal.config
new file mode 100644
index 0000000000..1313b1c2e4
--- /dev/null
+++ b/standalone/android/cabal.config
@@ -0,0 +1,209 @@
+constraints: unix installed,
+             blaze-html ==0.8.1.3,
+	     blaze-markup ==0.7.0.3,
+	     basement ==0.0.7,
+	     memory ==0.14.9,
+	     connection ==0.2.8,
+             aws ==0.13.0,
+	     lifted-base ==0.2.3.6,
+             Crypto ==4.2.5.1,
+             binary ==0.7.6.1,
+             DAV ==1.0.3,
+             HTTP ==4000.2.17,
+             HUnit ==1.2.5.2,
+             IfElse ==0.85,
+             MissingH ==1.2.1.0,
+             directory ==1.2.2.0,
+             MonadRandom ==0.1.13,
+             QuickCheck ==2.7.6,
+             SafeSemaphore ==0.10.1,
+             aeson ==0.7.0.6,
+             ansi-wl-pprint ==0.6.7.1,
+             appar ==0.1.4,
+             asn1-encoding ==0.9.5,
+             asn1-parse ==0.9.4,
+             asn1-types ==0.3.2,
+             async ==2.0.1.5,
+             attoparsec ==0.11.3.4,
+             attoparsec-conduit ==1.1.0,
+             authenticate ==1.3.2.10,
+             base-unicode-symbols ==0.2.2.4,
+             base16-bytestring ==0.1.1.6,
+             base64-bytestring ==1.0.0.1,
+             bifunctors ==4.1.1.1,
+             bloomfilter ==2.0.0.0,
+             byteable ==0.1.1,
+             byteorder ==1.0.4,
+             case-insensitive ==1.2.0.1,
+             cereal ==0.4.0.1,
+             cipher-aes ==0.2.8,
+             cipher-des ==0.0.6,
+             cipher-rc4 ==0.1.4,
+             clientsession ==0.9.0.3,
+             comonad ==4.2,
+             conduit ==1.1.6,
+             conduit-extra ==1.1.3,
+             contravariant ==0.6.1.1,
+             cookie ==0.4.1.2,
+             cprng-aes ==0.5.2,
+             crypto-api ==0.13.2,
+             crypto-cipher-types ==0.0.9,
+             crypto-numbers ==0.2.3,
+             crypto-pubkey ==0.2.4,
+             crypto-pubkey-types ==0.4.2.3,
+             crypto-random ==0.0.7,
+             cryptohash ==0.11.6,
+             cryptohash-conduit ==0.1.1,
+             css-text ==0.1.2.1,
+             data-default ==0.5.3,
+             data-default-class ==0.0.1,
+             data-default-instances-base ==0.0.1,
+             data-default-instances-containers ==0.0.1,
+             data-default-instances-dlist ==0.0.1,
+             data-default-instances-old-locale ==0.0.1,
+             sandi ==0.3.0.1,
+             dbus ==0.10.8,
+             distributive ==0.4.4,
+             dlist ==0.7.0.1,
+             dns ==1.3.0,
+             edit-distance ==0.2.1.2,
+             either ==4.3,
+             email-validate ==1.0.0,
+             entropy ==0.2.1,
+             errors ==1.4.7,
+             exceptions ==0.6.1,
+             failure ==0.2.0.3,
+             fast-logger ==2.1.5,
+             fdo-notify ==0.3.1,
+             feed ==0.3.9.2,
+             file-embed ==0.0.6,
+             fingertree ==0.1.0.0,
+             free ==4.9,
+             gnuidn ==0.2,
+             gnutls ==0.1.4,
+             gsasl ==0.3.5,
+             hS3 ==0.5.7,
+             hashable ==1.2.1.0,
+             hinotify ==0.3.5,
+             hjsmin ==0.1.4.7,
+             hslogger ==1.2.10,
+             http-client ==0.4.31.1,
+             http-client-tls ==0.2.4.1,
+             http-conduit ==2.1.9,
+             http-date ==0.0.6.1,
+             http-types ==0.9.1,
+             blaze-builder ==0.3.3.2,
+             hxt ==9.3.1.4,
+             hxt-charproperties ==9.1.1.1,
+             hxt-regex-xmlschema ==9.0.4,
+             hxt-unicode ==9.0.2.2,
+             idna ==0.2,
+             iproute ==1.3.1,
+             json ==0.5,
+             keys ==3.10.1,
+             language-javascript ==0.5.13,
+             lens ==4.4.0.2,
+             libxml-sax ==0.7.5,
+             mime-mail ==0.4.1.2,
+             mime-types ==0.1.0.4,
+             mmorph ==1.0.3,
+             monad-control ==0.3.2.2,
+             monad-logger ==0.3.6.1,
+             monad-loops ==0.4.2.1,
+             monads-tf ==0.1.0.2,
+             mtl ==2.1.2,
+             nats ==0.1.2,
+             network ==2.6.3.1
+             network-conduit ==1.1.0,
+             network-info ==0.2.0.5,
+             network-multicast ==0.0.10,
+             network-uri ==2.6.0.1,
+             optparse-applicative ==0.11.0.2,
+             parallel ==3.2.0.4,
+             path-pieces ==0.1.4,
+             pem ==0.2.2,
+             persistent ==1.3.3,
+             persistent-template ==1.3.2.2,
+             pointed ==4.0,
+             prelude-extras ==0.4,
+             profunctors ==4.0.4,
+             publicsuffixlist ==0.1,
+             punycode ==2.0,
+             random ==1.0.1.1,
+             ranges ==0.2.4,
+             reducers ==3.10.2.1,
+             reflection ==1.2.0.1,
+             regex-base ==0.93.2,
+             regex-compat ==0.95.1,
+             regex-posix ==0.95.2,
+             regex-tdfa ==1.2.0,
+             resource-pool ==0.2.1.1,
+             resourcet ==1.1.2.3,
+             safe ==0.3.8,
+             securemem ==0.1.3,
+             semigroupoids ==4.2,
+             semigroups ==0.15.3,
+             shakespeare ==2.0.5,
+             silently ==1.2.4.1,
+             simple-sendfile ==0.2.14,
+             skein ==1.0.9,
+             socks ==0.5.5,
+             split ==0.2.2,
+             stm ==2.4.2,
+             stm-chans ==3.0.0.2,
+             streaming-commons ==0.1.4.1,
+             stringprep ==0.1.5,
+             stringsearch ==0.3.6.5,
+             syb ==0.4.0,
+             system-fileio ==0.3.14,
+             system-filepath ==0.4.12,
+             tagged ==0.7.2,
+             tagsoup ==0.13.1,
+             tagstream-conduit ==0.5.5.1,
+             text ==1.1.1.0,
+             text-icu ==0.6.3.7,
+             tf-random ==0.5,
+             tls ==1.3.8,
+             transformers ==0.3.0.0,
+             transformers-base ==0.4.1,
+             transformers-compat ==0.3.3.3,
+             unbounded-delays ==0.1.0.8,
+             unix-compat ==0.4.1.3,
+             unix-time ==0.2.2,
+             unordered-containers ==0.2.5.0,
+             utf8-string ==0.3.7,
+             uuid ==1.3.3,
+             vault ==0.3.0.3,
+             vector ==0.10.0.1,
+             void ==0.6.1,
+             wai ==3.0.1.1,
+             wai-app-static ==3.0.0.1,
+             wai-extra ==3.0.1.2,
+             wai-logger ==2.1.1,
+             warp ==3.0.8,
+             warp-tls ==3.0.4.2,
+             word8 ==0.1.1,
+             x509 ==1.6.5,
+             x509-store ==1.6.3,
+             x509-system ==1.6.6,
+             x509-validation ==1.6.8,
+             xml ==1.3.13,
+             xml-conduit ==1.2.1,
+             xml-hamlet ==0.4.0.9,
+             xml-types ==0.3.4,
+             xss-sanitize ==0.3.5.2,
+             yaml ==0.8.9.3,
+             yesod ==1.2.6.1,
+             yesod-auth ==1.3.4.6,
+             yesod-core ==1.2.20.1,
+             yesod-default ==1.2.0,
+             yesod-form ==1.3.16,
+             yesod-persistent ==1.2.3.1,
+             yesod-routes ==1.2.0.7,
+             yesod-static ==1.2.4,
+             zlib ==0.5.4.1,
+             bytestring installed,
+             process ==1.2.3.0,
+             scientific ==0.3.3.1,
+             clock ==0.4.6.0,
+             cryptonite ==0.15
diff --git a/standalone/android/clean-haskell-packages b/standalone/android/clean-haskell-packages
new file mode 100755
index 0000000000..b8c6132d6b
--- /dev/null
+++ b/standalone/android/clean-haskell-packages
@@ -0,0 +1,6 @@
+#!/bin/sh
+# Removes all currently installed cross-compiled haskell packages
+# except those part of ghc.
+# Useful if the build failed.
+rm -f $(grep -l $HOME/.ghc/$(cat abiversion)/.cabal/lib/ $HOME/.ghc/android-14/arm-linux-androideabi-4.8/lib/*-ghc-*/package.conf.d/*.conf)
+$HOME/.ghc/$(cat abiversion)/arm-linux-androideabi/bin/ghc-pkg recache
diff --git a/standalone/android/dropbear.patch b/standalone/android/dropbear.patch
new file mode 100644
index 0000000000..84c7dfb6d6
--- /dev/null
+++ b/standalone/android/dropbear.patch
@@ -0,0 +1,55 @@
+From 014dadb02fd984828a6232534c47dba8e2f7818a Mon Sep 17 00:00:00 2001
+From: Joey Hess 
+Date: Wed, 13 Feb 2013 15:29:52 -0400
+Subject: [PATCH] android patch for dropbear
+
+* Disable HOME override
+* Use urandom to avoid blocking on every ssh connection.
+* Enable use of netbsd_getpass.c
+---
+ cli-auth.c |    1 +
+ cli-main.c |    2 --
+ options.h  |    2 +-
+ 3 files changed, 2 insertions(+), 3 deletions(-)
+
+diff --git a/cli-auth.c b/cli-auth.c
+index 4c17a21..91dfdf8 100644
+--- a/cli-auth.c
++++ b/cli-auth.c
+@@ -31,6 +31,7 @@
+ #include "ssh.h"
+ #include "packet.h"
+ #include "runopts.h"
++#include "netbsd_getpass.c"
+ 
+ void cli_authinitialise() {
+ 
+diff --git a/cli-main.c b/cli-main.c
+index 106006b..68cf023 100644
+--- a/cli-main.c
++++ b/cli-main.c
+@@ -47,8 +47,6 @@ int main(int argc, char ** argv) {
+ 	_dropbear_exit = cli_dropbear_exit;
+ 	_dropbear_log = cli_dropbear_log;
+ 
+-	putenv("HOME=/data/local");
+-
+ 	disallow_core();
+ 
+ 	cli_getopts(argc, argv);
+diff --git a/options.h b/options.h
+index 7625151..48e404d 100644
+--- a/options.h
++++ b/options.h
+@@ -159,7 +159,7 @@ etc) slower (perhaps by 50%). Recommended for most small systems. */
+  * however significantly reduce the security of your ssh connections
+  * if the PRNG state becomes guessable - make sure you know what you are
+  * doing if you change this. */
+-#define DROPBEAR_RANDOM_DEV "/dev/random"
++#define DROPBEAR_RANDOM_DEV "/dev/urandom"
+ 
+ /* prngd must be manually set up to produce output */
+ /*#define DROPBEAR_PRNGD_SOCKET "/var/run/dropbear-rng"*/
+-- 
+1.7.10.4
+
diff --git a/standalone/android/git.patch b/standalone/android/git.patch
new file mode 100644
index 0000000000..ec81aaaeb4
--- /dev/null
+++ b/standalone/android/git.patch
@@ -0,0 +1,47 @@
+From e0fffe80a8815e64dbc1d690c79bf006651c7642 Mon Sep 17 00:00:00 2001
+From: Joey Hess 
+Date: Wed, 13 Aug 2014 13:50:56 -0400
+Subject: [PATCH] avoid using of chmod on android when changing config
+
+This breaks on Android's /sdcard, which has a variety of FUSE
+implentations, all total shite.
+---
+ config.c | 4 ++++
+ 1 file changed, 4 insertions(+)
+
+diff --git a/config.c b/config.c
+index 9ba40bc..a350638 100644
+--- a/config.c
++++ b/config.c
+@@ -2124,12 +2124,14 @@ int git_config_set_multivar_in_file_gently(const char *config_filename,
+ 		close(in_fd);
+ 		in_fd = -1;
+ 
++		/* not on android
+ 		if (chmod(get_lock_file_path(lock), st.st_mode & 07777) < 0) {
+ 			error("chmod on %s failed: %s",
+ 			      get_lock_file_path(lock), strerror(errno));
+ 			ret = CONFIG_NO_WRITE;
+ 			goto out_free;
+ 		}
++		*/
+ 
+ 		if (store.seen == 0)
+ 			store.seen = 1;
+@@ -2329,11 +2331,13 @@ int git_config_rename_section_in_file(const char *config_filename,
+ 
+ 	fstat(fileno(config_file), &st);
+ 
++	/* not on android
+ 	if (chmod(get_lock_file_path(lock), st.st_mode & 07777) < 0) {
+ 		ret = error("chmod on %s failed: %s",
+ 			    get_lock_file_path(lock), strerror(errno));
+ 		goto out;
+ 	}
++	*/
+ 
+ 	while (fgets(buf, sizeof(buf), config_file)) {
+ 		int i;
+-- 
+2.1.4
+
diff --git a/standalone/android/haskell-patches/basement_fix-build-on-android.patch b/standalone/android/haskell-patches/basement_fix-build-on-android.patch
new file mode 100644
index 0000000000..948473bc2e
--- /dev/null
+++ b/standalone/android/haskell-patches/basement_fix-build-on-android.patch
@@ -0,0 +1,41 @@
+From cc0c373b69f93057cbdcb634a461e10ec019d87a Mon Sep 17 00:00:00 2001
+From: androidbuilder 
+Date: Wed, 20 Jun 2018 00:29:11 +0100
+Subject: [PATCH] fix build on android
+
+---
+ Basement/Terminal.hs | 2 --
+ basement.cabal       | 1 -
+ 2 files changed, 3 deletions(-)
+
+diff --git a/Basement/Terminal.hs b/Basement/Terminal.hs
+index 8136e52..cca9606 100644
+--- a/Basement/Terminal.hs
++++ b/Basement/Terminal.hs
+@@ -1,11 +1,9 @@
+ {-# LANGUAGE CPP #-}
+ module Basement.Terminal
+     ( initialize
+-    , getDimensions
+     ) where
+ 
+ import Basement.Compat.Base
+-import Basement.Terminal.Size (getDimensions)
+ #ifdef mingw32_HOST_OS
+ import System.IO (hSetEncoding, utf8, hPutStrLn, stderr, stdin, stdout)
+ import System.Win32.Console (setConsoleCP, setConsoleOutputCP, getConsoleCP, getConsoleOutputCP)
+diff --git a/basement.cabal b/basement.cabal
+index af50291..0824c94 100644
+--- a/basement.cabal
++++ b/basement.cabal
+@@ -135,7 +135,6 @@ library
+                      Basement.String.Encoding.ASCII7
+                      Basement.String.Encoding.ISO_8859_1
+ 
+-                     Basement.Terminal.Size
+ 
+ 
+   build-depends:       base >= 4.7 && < 5
+-- 
+2.1.4
+
diff --git a/standalone/android/haskell-patches/certificate_1.3.7-0001-support-Android-cert-store.patch b/standalone/android/haskell-patches/certificate_1.3.7-0001-support-Android-cert-store.patch
new file mode 100644
index 0000000000..5f772bfdfe
--- /dev/null
+++ b/standalone/android/haskell-patches/certificate_1.3.7-0001-support-Android-cert-store.patch
@@ -0,0 +1,37 @@
+From 3779c75175e895f94b21341ebd6361e9d6af54fd Mon Sep 17 00:00:00 2001
+From: Joey Hess 
+Date: Thu, 9 May 2013 12:36:23 -0400
+Subject: [PATCH] support Android cert store
+
+Android puts it in a different place and has only hashed files.
+See https://github.com/vincenthz/hs-certificate/issues/19
+---
+ System/Certificate/X509/Unix.hs | 5 +++--
+ 1 file changed, 3 insertions(+), 2 deletions(-)
+
+diff --git a/System/Certificate/X509/Unix.hs b/System/Certificate/X509/Unix.hs
+index 8463465..74e9503 100644
+--- a/System/Certificate/X509/Unix.hs
++++ b/System/Certificate/X509/Unix.hs
+@@ -35,7 +35,8 @@ import qualified Control.Exception as E
+ import Data.Char
+ 
+ defaultSystemPath :: FilePath
+-defaultSystemPath = "/etc/ssl/certs/"
++defaultSystemPath = "/system/etc/security/cacerts/"
++--defaultSystemPath = "/etc/ssl/certs/"
+ 
+ envPathOverride :: String
+ envPathOverride = "SYSTEM_CERTIFICATE_PATH"
+@@ -47,7 +48,7 @@ listDirectoryCerts path = (map (path ) . filter isCert <$> getDirectoryConten
+                         && isDigit (s !! 9)
+                         && (s !! 8) == '.'
+                         && all isHexDigit (take 8 s)
+-          isCert x = (not $ isPrefixOf "." x) && (not $ isHashedFile x)
++          isCert x = (not $ isPrefixOf "." x)
+ 
+ getSystemCertificateStore :: IO CertificateStore
+ getSystemCertificateStore = makeCertificateStore . concat <$> (getSystemPath >>= listDirectoryCerts >>= mapM readCertificates)
+-- 
+1.8.2.rc3
+
diff --git a/standalone/android/haskell-patches/clock_hack-for-android.patch b/standalone/android/haskell-patches/clock_hack-for-android.patch
new file mode 100644
index 0000000000..d2c39ff539
--- /dev/null
+++ b/standalone/android/haskell-patches/clock_hack-for-android.patch
@@ -0,0 +1,76 @@
+From 5be3bdfc5ec83eaa5defd42e99f73a685123bea0 Mon Sep 17 00:00:00 2001
+From: androidbuilder 
+Date: Fri, 22 May 2015 18:35:43 +0000
+Subject: [PATCH] hack for android
+
+---
+ System/Clock.hsc | 32 ++------------------------------
+ 1 file changed, 2 insertions(+), 30 deletions(-)
+
+diff --git a/System/Clock.hsc b/System/Clock.hsc
+index b6a4968..92bcf8b 100644
+--- a/System/Clock.hsc
++++ b/System/Clock.hsc
+@@ -35,8 +35,6 @@ import GHC.Generics (Generic)
+ #  endif
+ #endif
+ 
+-#let alignment t = "%lu", (unsigned long)offsetof(struct {char x__; t (y__); }, y__)
+-
+ -- | Clock types. A clock may be system-wide (that is, visible to all processes)
+ --   or per-process (measuring time that is meaningful only within a process).
+ --   All implementations shall support CLOCK_REALTIME.
+@@ -119,7 +117,7 @@ getTime ThreadCPUTime = allocaAndPeek hs_clock_win32_gettime_threadtime
+ #elif defined(__MACH__)
+ getTime clk = allocaAndPeek $ hs_clock_darwin_gettime $ clockToConst clk
+ #else
+-getTime clk = allocaAndPeek $ clock_gettime $ clockToConst clk
++getTime clk = error "getTime clk not implemented"
+ #endif
+ 
+ #if defined(_WIN32)
+@@ -130,7 +128,7 @@ getRes ThreadCPUTime = allocaAndPeek hs_clock_win32_getres_threadtime
+ #elif defined(__MACH__)
+ getRes clk = allocaAndPeek $ hs_clock_darwin_getres $ clockToConst clk
+ #else
+-getRes clk = allocaAndPeek $ clock_getres $ clockToConst clk
++getRes clk = error "getRes clk not implemented"
+ #endif
+ 
+ -- | TimeSpec structure
+@@ -139,32 +137,6 @@ data TimeSpec = TimeSpec
+   , nsec :: {-# UNPACK #-} !Int64 -- ^ nanoseconds
+   } deriving (Eq, Generic, Read, Show, Typeable)
+ 
+-#if defined(_WIN32)
+-instance Storable TimeSpec where
+-  sizeOf _ = sizeOf (undefined :: Int64) * 2
+-  alignment _ = alignment (undefined :: Int64)
+-  poke ptr ts = do
+-      pokeByteOff ptr 0 (sec ts)
+-      pokeByteOff ptr (sizeOf (undefined :: Int64)) (nsec ts)
+-  peek ptr = do
+-      TimeSpec
+-        <$> peekByteOff ptr 0
+-        <*> peekByteOff ptr (sizeOf (undefined :: Int64))
+-#else
+-instance Storable TimeSpec where
+-  sizeOf _ = #{size struct timespec}
+-  alignment _ = #{alignment struct timespec}
+-  poke ptr ts = do
+-      let xs :: #{type time_t} = fromIntegral $ sec ts
+-          xn :: #{type long} = fromIntegral $ nsec ts
+-      #{poke struct timespec, tv_sec} ptr (xs)
+-      #{poke struct timespec, tv_nsec} ptr (xn)
+-  peek ptr = do
+-      xs :: #{type time_t} <- #{peek struct timespec, tv_sec} ptr
+-      xn :: #{type long} <- #{peek struct timespec, tv_nsec} ptr
+-      return $ TimeSpec (fromIntegral xs) (fromIntegral xn)
+-#endif
+-
+ normalize :: TimeSpec -> TimeSpec
+ normalize (TimeSpec xs xn) =
+     let (q, r) = xn `divMod` (10^9)
+-- 
+2.1.4
+
diff --git a/standalone/android/haskell-patches/comonad_cross-build.patch b/standalone/android/haskell-patches/comonad_cross-build.patch
new file mode 100644
index 0000000000..ee8ae42681
--- /dev/null
+++ b/standalone/android/haskell-patches/comonad_cross-build.patch
@@ -0,0 +1,25 @@
+From 589c6a87ec62e35942c9a86ea8d91b443c80da99 Mon Sep 17 00:00:00 2001
+From: dummy 
+Date: Fri, 18 Oct 2013 23:07:02 +0000
+Subject: [PATCH] cross build
+
+---
+ comonad.cabal |    2 +-
+ 1 file changed, 1 insertion(+), 1 deletion(-)
+
+diff --git a/comonad.cabal b/comonad.cabal
+index 5d34b13..756ed92 100644
+--- a/comonad.cabal
++++ b/comonad.cabal
+@@ -13,7 +13,7 @@ copyright:     Copyright (C) 2008-2013 Edward A. Kmett,
+                Copyright (C) 2004-2008 Dave Menendez
+ synopsis:      Comonads
+ description:   Comonads
+-build-type:    Custom
++build-type:    Simple
+ extra-source-files:
+   .ghci
+   .gitignore
+-- 
+1.7.10.4
+
diff --git a/standalone/android/haskell-patches/crypto-numbers_build-fix.patch b/standalone/android/haskell-patches/crypto-numbers_build-fix.patch
new file mode 100644
index 0000000000..5c0693a312
--- /dev/null
+++ b/standalone/android/haskell-patches/crypto-numbers_build-fix.patch
@@ -0,0 +1,227 @@
+From 0cfdb30120976290068f4bcbebbf236b960afbb6 Mon Sep 17 00:00:00 2001
+From: dummy 
+Date: Thu, 26 Dec 2013 20:01:30 -0400
+Subject: [PATCH] hack to build
+
+---
+ Crypto/Number/Basic.hs         |   14 --------------
+ Crypto/Number/ModArithmetic.hs |   29 -----------------------------
+ Crypto/Number/Prime.hs         |   18 ------------------
+ crypto-numbers.cabal           |    2 +-
+ 4 files changed, 1 insertion(+), 62 deletions(-)
+
+diff --git a/Crypto/Number/Basic.hs b/Crypto/Number/Basic.hs
+index 65c14b3..eaee853 100644
+--- a/Crypto/Number/Basic.hs
++++ b/Crypto/Number/Basic.hs
+@@ -20,11 +20,7 @@ module Crypto.Number.Basic
+     , areEven
+     ) where
+ 
+-#if MIN_VERSION_integer_gmp(0,5,1)
+-import GHC.Integer.GMP.Internals
+-#else
+ import Data.Bits
+-#endif
+ 
+ -- | sqrti returns two integer (l,b) so that l <= sqrt i <= b
+ -- the implementation is quite naive, use an approximation for the first number
+@@ -63,25 +59,16 @@ sqrti i
+ -- gcde 'a' 'b' find (x,y,gcd(a,b)) where ax + by = d
+ --
+ gcde :: Integer -> Integer -> (Integer, Integer, Integer)
+-#if MIN_VERSION_integer_gmp(0,5,1)
+-gcde a b = (s, t, g)
+-  where (# g, s #) = gcdExtInteger a b
+-        t = (g - s * a) `div` b
+-#else
+ gcde a b = if d < 0 then (-x,-y,-d) else (x,y,d) where
+     (d, x, y)                     = f (a,1,0) (b,0,1)
+     f t              (0, _, _)    = t
+     f (a', sa, ta) t@(b', sb, tb) =
+         let (q, r) = a' `divMod` b' in
+         f t (r, sa - (q * sb), ta - (q * tb))
+-#endif
+ 
+ -- | get the extended GCD of two integer using the extended binary algorithm (HAC 14.61)
+ -- get (x,y,d) where d = gcd(a,b) and x,y satisfying ax + by = d
+ gcde_binary :: Integer -> Integer -> (Integer, Integer, Integer)
+-#if MIN_VERSION_integer_gmp(0,5,1)
+-gcde_binary = gcde
+-#else
+ gcde_binary a' b'
+     | b' == 0   = (1,0,a')
+     | a' >= b'  = compute a' b'
+@@ -105,7 +92,6 @@ gcde_binary a' b'
+              in if u2 >= v2
+                 then loop g x y (u2 - v2) v2 (a2 - c2) (b2 - d2) c2 d2
+                 else loop g x y u2 (v2 - u2) a2 b2 (c2 - a2) (d2 - b2)
+-#endif
+ 
+ -- | check if a list of integer are all even
+ areEven :: [Integer] -> Bool
+diff --git a/Crypto/Number/ModArithmetic.hs b/Crypto/Number/ModArithmetic.hs
+index 942c12f..f8cfc32 100644
+--- a/Crypto/Number/ModArithmetic.hs
++++ b/Crypto/Number/ModArithmetic.hs
+@@ -29,12 +29,8 @@ module Crypto.Number.ModArithmetic
+ import Control.Exception (throw, Exception)
+ import Data.Typeable
+ 
+-#if MIN_VERSION_integer_gmp(0,5,1)
+-import GHC.Integer.GMP.Internals
+-#else
+ import Crypto.Number.Basic (gcde_binary)
+ import Data.Bits
+-#endif
+ 
+ -- | Raised when two numbers are supposed to be coprimes but are not.
+ data CoprimesAssertionError = CoprimesAssertionError
+@@ -55,13 +51,7 @@ expSafe :: Integer -- ^ base
+         -> Integer -- ^ exponant
+         -> Integer -- ^ modulo
+         -> Integer -- ^ result
+-#if MIN_VERSION_integer_gmp(0,5,1)
+-expSafe b e m
+-    | odd m     = powModSecInteger b e m
+-    | otherwise = powModInteger b e m
+-#else
+ expSafe = exponentiation
+-#endif
+ 
+ -- | Compute the modular exponentiation of base^exponant using
+ -- the fastest algorithm without any consideration for
+@@ -74,11 +64,7 @@ expFast :: Integer -- ^ base
+         -> Integer -- ^ modulo
+         -> Integer -- ^ result
+ expFast =
+-#if MIN_VERSION_integer_gmp(0,5,1)
+-    powModInteger
+-#else
+     exponentiation
+-#endif
+ 
+ -- note on exponentiation: 0^0 is treated as 1 for mimicking the standard library;
+ -- the mathematic debate is still open on whether or not this is true, but pratically
+@@ -87,22 +73,15 @@ expFast =
+ -- | exponentiation_rtl_binary computes modular exponentiation as b^e mod m
+ -- using the right-to-left binary exponentiation algorithm (HAC 14.79)
+ exponentiation_rtl_binary :: Integer -> Integer -> Integer -> Integer
+-#if MIN_VERSION_integer_gmp(0,5,1)
+-exponentiation_rtl_binary = expSafe
+-#else
+ exponentiation_rtl_binary 0 0 m = 1 `mod` m
+ exponentiation_rtl_binary b e m = loop e b 1
+     where sq x          = (x * x) `mod` m
+           loop !0 _  !a = a `mod` m
+           loop !i !s !a = loop (i `shiftR` 1) (sq s) (if odd i then a * s else a)
+-#endif
+ 
+ -- | exponentiation computes modular exponentiation as b^e mod m
+ -- using repetitive squaring.
+ exponentiation :: Integer -> Integer -> Integer -> Integer
+-#if MIN_VERSION_integer_gmp(0,5,1)
+-exponentiation = expSafe
+-#else
+ exponentiation b e m
+     | b == 1    = b
+     | e == 0    = 1
+@@ -110,7 +89,6 @@ exponentiation b e m
+     | even e    = let p = (exponentiation b (e `div` 2) m) `mod` m
+                    in (p^(2::Integer)) `mod` m
+     | otherwise = (b * exponentiation b (e-1) m) `mod` m
+-#endif
+ 
+ --{-# DEPRECATED exponantiation_rtl_binary "typo in API name it's called exponentiation_rtl_binary #-}
+ exponantiation_rtl_binary :: Integer -> Integer -> Integer -> Integer
+@@ -122,17 +100,10 @@ exponantiation = exponentiation
+ 
+ -- | inverse computes the modular inverse as in g^(-1) mod m
+ inverse :: Integer -> Integer -> Maybe Integer
+-#if MIN_VERSION_integer_gmp(0,5,1)
+-inverse g m
+-    | r == 0    = Nothing
+-    | otherwise = Just r
+-  where r = recipModInteger g m
+-#else
+ inverse g m
+     | d > 1     = Nothing
+     | otherwise = Just (x `mod` m)
+   where (x,_,d) = gcde_binary g m
+-#endif
+ 
+ -- | Compute the modular inverse of 2 coprime numbers.
+ -- This is equivalent to inverse except that the result
+diff --git a/Crypto/Number/Prime.hs b/Crypto/Number/Prime.hs
+index 0cea9da..458c94d 100644
+--- a/Crypto/Number/Prime.hs
++++ b/Crypto/Number/Prime.hs
+@@ -3,9 +3,7 @@
+ #ifndef MIN_VERSION_integer_gmp
+ #define MIN_VERSION_integer_gmp(a,b,c) 0
+ #endif
+-#if MIN_VERSION_integer_gmp(0,5,1)
+ {-# LANGUAGE MagicHash #-}
+-#endif
+ -- |
+ -- Module      : Crypto.Number.Prime
+ -- License     : BSD-style
+@@ -30,12 +28,7 @@ import Crypto.Number.Generate
+ import Crypto.Number.Basic (sqrti, gcde_binary)
+ import Crypto.Number.ModArithmetic (exponantiation)
+ 
+-#if MIN_VERSION_integer_gmp(0,5,1)
+-import GHC.Integer.GMP.Internals
+-import GHC.Base
+-#else
+ import Data.Bits
+-#endif
+ 
+ -- | returns if the number is probably prime.
+ -- first a list of small primes are implicitely tested for divisibility,
+@@ -78,21 +71,11 @@ findPrimeFromWith rng prop !n
+ -- | find a prime from a starting point with no specific property.
+ findPrimeFrom :: CPRG g => g -> Integer -> (Integer, g)
+ findPrimeFrom rng n =
+-#if MIN_VERSION_integer_gmp(0,5,1)
+-    (nextPrimeInteger n, rng)
+-#else
+     findPrimeFromWith rng (\g _ -> (True, g)) n
+-#endif
+ 
+ -- | Miller Rabin algorithm return if the number is probably prime or composite.
+ -- the tries parameter is the number of recursion, that determines the accuracy of the test.
+ primalityTestMillerRabin :: CPRG g => g -> Int -> Integer -> (Bool, g)
+-#if MIN_VERSION_integer_gmp(0,5,1)
+-primalityTestMillerRabin rng (I# tries) !n =
+-    case testPrimeInteger n tries of
+-        0# -> (False, rng)
+-        _  -> (True, rng)
+-#else
+ primalityTestMillerRabin rng tries !n
+     | n <= 3     = error "Miller-Rabin requires tested value to be > 3"
+     | even n     = (False, rng)
+@@ -129,7 +112,6 @@ primalityTestMillerRabin rng tries !n
+                   | x2 == 1   = False
+                   | x2 /= nm1 = loop' ws ((x2*x2) `mod` n) (r+1)
+                   | otherwise = loop ws
+-#endif
+ 
+ {-
+     n < z -> witness to test
+diff --git a/crypto-numbers.cabal b/crypto-numbers.cabal
+index 9610e34..6669d78 100644
+--- a/crypto-numbers.cabal
++++ b/crypto-numbers.cabal
+@@ -15,7 +15,7 @@ Extra-Source-Files:  Tests/*.hs
+ 
+ Flag integer-gmp
+   Description: Are we using integer-gmp?
+-  Default: True
++  Default: False
+ 
+ Library
+   Build-Depends:     base >= 4 && < 5
+-- 
+1.7.10.4
+
diff --git a/standalone/android/haskell-patches/distributive_0.4.4_0001-fixes-for-cross-build.patch b/standalone/android/haskell-patches/distributive_0.4.4_0001-fixes-for-cross-build.patch
new file mode 100644
index 0000000000..4229d644d8
--- /dev/null
+++ b/standalone/android/haskell-patches/distributive_0.4.4_0001-fixes-for-cross-build.patch
@@ -0,0 +1,25 @@
+From 86eca0993e1716b4db14570836efbe838626892f Mon Sep 17 00:00:00 2001
+From: dummy 
+Date: Sun, 25 May 2014 09:11:15 +0200
+Subject: [PATCH] cross build
+
+---
+ distributive.cabal | 2 +-
+ 1 file changed, 1 insertion(+), 1 deletion(-)
+
+diff --git a/distributive.cabal b/distributive.cabal
+index 9be5133..3e72c65 100644
+--- a/distributive.cabal
++++ b/distributive.cabal
+@@ -12,7 +12,7 @@ bug-reports:   http://github.com/ekmett/distributive/issues
+ copyright:     Copyright (C) 2011-2014 Edward A. Kmett
+ synopsis:      Distributive functors -- Dual to Traversable
+ description:   Distributive functors -- Dual to Traversable
+-build-type:    Custom
++build-type:    Simple
+ extra-source-files:
+   .ghci
+   .travis.yml
+-- 
+2.0.0.rc2
+
diff --git a/standalone/android/haskell-patches/dns_use-android-net.dns1-command-instead-of-resolv.conf.patch b/standalone/android/haskell-patches/dns_use-android-net.dns1-command-instead-of-resolv.conf.patch
new file mode 100644
index 0000000000..962a642075
--- /dev/null
+++ b/standalone/android/haskell-patches/dns_use-android-net.dns1-command-instead-of-resolv.conf.patch
@@ -0,0 +1,63 @@
+From 087f1ae5e17f0e6d7c9f6b4092a5bb5bb6f5bf60 Mon Sep 17 00:00:00 2001
+From: dummy 
+Date: Thu, 16 Oct 2014 02:59:11 +0000
+Subject: [PATCH] port
+
+---
+ Network/DNS/Resolver.hs | 13 ++++++++-----
+ dns.cabal               |  1 +
+ 2 files changed, 9 insertions(+), 5 deletions(-)
+
+diff --git a/Network/DNS/Resolver.hs b/Network/DNS/Resolver.hs
+index 5721e03..c4400d1 100644
+--- a/Network/DNS/Resolver.hs
++++ b/Network/DNS/Resolver.hs
+@@ -19,7 +19,7 @@ module Network.DNS.Resolver (
+   ) where
+ 
+ import Control.Applicative ((<$>), (<*>), pure)
+-import Control.Exception (bracket)
++import Control.Exception (bracket, catch, IOException)
+ import qualified Data.ByteString.Char8 as BS
+ import Data.Char (isSpace)
+ import Data.List (isPrefixOf)
+@@ -32,6 +32,7 @@ import Network.Socket (AddrInfoFlag(..), AddrInfo(..), defaultHints, getAddrInfo
+ import Prelude hiding (lookup)
+ import System.Random (getStdRandom, randomR)
+ import System.Timeout (timeout)
++import System.Process
+ 
+ #if mingw32_HOST_OS == 1
+ import Network.Socket (send)
+@@ -130,10 +131,12 @@ makeResolvSeed conf = ResolvSeed <$> addr
+   where
+     addr = case resolvInfo conf of
+         RCHostName numhost -> makeAddrInfo numhost
+-        RCFilePath file -> toAddr <$> readFile file >>= makeAddrInfo
+-    toAddr cs = let l:_ = filter ("nameserver" `isPrefixOf`) $ lines cs
+-                in extract l
+-    extract = reverse . dropWhile isSpace . reverse . dropWhile isSpace . drop 11
++        RCFilePath file -> do
++		-- Android has no /etc/resolv.conf; use getprop command.
++		ls <- catch (lines <$> readProcess "getprop" ["net.dns1"] []) (const (return []) :: IOException -> IO [String])
++		makeAddrInfo $ case ls of
++			[] -> "8.8.8.8" -- google public dns as a fallback only
++			(l:_) -> l
+ 
+ makeAddrInfo :: HostName -> IO AddrInfo
+ makeAddrInfo addr = do
+diff --git a/dns.cabal b/dns.cabal
+index ceaf5f4..cd15e61 100644
+--- a/dns.cabal
++++ b/dns.cabal
+@@ -37,6 +37,7 @@ Library
+                       , network >= 2.3
+                       , random
+                       , resourcet
++                      , process
+   else
+     Build-Depends:      base >= 4 && < 5
+                       , attoparsec
+-- 
+2.1.1
+
diff --git a/standalone/android/haskell-patches/gsasl_0.3.5-0001-link-with-libgsasl.patch b/standalone/android/haskell-patches/gsasl_0.3.5-0001-link-with-libgsasl.patch
new file mode 100644
index 0000000000..42206a1cf7
--- /dev/null
+++ b/standalone/android/haskell-patches/gsasl_0.3.5-0001-link-with-libgsasl.patch
@@ -0,0 +1,25 @@
+From df0f41f92d003f7d59ef927737ffec3a9bd61827 Mon Sep 17 00:00:00 2001
+From: Joey Hess 
+Date: Tue, 7 May 2013 18:41:01 -0400
+Subject: [PATCH] avoid cabal hell
+
+---
+ gsasl.cabal |    2 +-
+ 1 file changed, 1 insertion(+), 1 deletion(-)
+
+diff --git a/gsasl.cabal b/gsasl.cabal
+index d991873..c5c2b19 100644
+--- a/gsasl.cabal
++++ b/gsasl.cabal
+@@ -31,7 +31,7 @@ library
+   build-depends:
+       base >= 4.0 && < 5.0
+     , transformers >= 0.2
+-    , bytestring >= 0.9
++    , bytestring >= 0.10.3.0
+ 
+   pkgconfig-depends: libgsasl >= 1.1
+ 
+-- 
+1.7.10.4
+
diff --git a/standalone/android/haskell-patches/iproute_1.2.11_0001-build-without-IPv6-stuff.patch b/standalone/android/haskell-patches/iproute_1.2.11_0001-build-without-IPv6-stuff.patch
new file mode 100644
index 0000000000..bb9caec770
--- /dev/null
+++ b/standalone/android/haskell-patches/iproute_1.2.11_0001-build-without-IPv6-stuff.patch
@@ -0,0 +1,47 @@
+From 7beec2e707d59f9573aa2dc7c57bd2a62f16b480 Mon Sep 17 00:00:00 2001
+From: Joey Hess 
+Date: Wed, 15 May 2013 19:06:03 -0400
+Subject: [PATCH] build without IPv6 stuff
+
+---
+ Data/IP.hs      |    2 +-
+ Data/IP/Addr.hs |    3 +++
+ 2 files changed, 4 insertions(+), 1 deletion(-)
+
+diff --git a/Data/IP.hs b/Data/IP.hs
+index cffef93..ea486c9 100644
+--- a/Data/IP.hs
++++ b/Data/IP.hs
+@@ -6,7 +6,7 @@ module Data.IP (
+   -- ** IP data
+     IP (..)
+   , IPv4, toIPv4, fromIPv4, fromHostAddress, toHostAddress
+-  , IPv6, toIPv6, fromIPv6, fromHostAddress6, toHostAddress6
++  , IPv6, toIPv6, fromIPv6 -- , fromHostAddress6, toHostAddress6
+   -- ** IP range data
+   , IPRange (..)
+   , AddrRange (addr, mask, mlen)
+diff --git a/Data/IP/Addr.hs b/Data/IP/Addr.hs
+index faaf0c7..5b556fb 100644
+--- a/Data/IP/Addr.hs
++++ b/Data/IP/Addr.hs
+@@ -312,6 +312,7 @@ toHostAddress (IP4 addr4)
+   | byteOrder == LittleEndian = fixByteOrder addr4
+   | otherwise                 = addr4
+ 
++{-
+ -- | The 'fromHostAddress6' function converts 'HostAddress6' to 'IPv6'.
+ fromHostAddress6 :: HostAddress6 -> IPv6
+ fromHostAddress6 = IP6
+@@ -320,6 +321,8 @@ fromHostAddress6 = IP6
+ toHostAddress6 :: IPv6 -> HostAddress6
+ toHostAddress6 (IP6 addr6) = addr6
+ 
++-}
++
+ fixByteOrder :: Word32 -> Word32
+ fixByteOrder s = d1 .|. d2 .|. d3 .|. d4
+   where
+-- 
+1.7.10.4
+
diff --git a/standalone/android/haskell-patches/language-javascript_fix-build-with-new-ghc.patch b/standalone/android/haskell-patches/language-javascript_fix-build-with-new-ghc.patch
new file mode 100644
index 0000000000..6182cba442
--- /dev/null
+++ b/standalone/android/haskell-patches/language-javascript_fix-build-with-new-ghc.patch
@@ -0,0 +1,25 @@
+From cb5252db1a0d515da69d9167a8b2facd839940b2 Mon Sep 17 00:00:00 2001
+From: androidbuilder 
+Date: Mon, 11 Nov 2013 02:29:06 +0000
+Subject: [PATCH] fix build with new ghc
+
+---
+ src/Language/JavaScript/Parser/Lexer.hs |    2 +-
+ 1 file changed, 1 insertion(+), 1 deletion(-)
+
+diff --git a/src/Language/JavaScript/Parser/Lexer.hs b/src/Language/JavaScript/Parser/Lexer.hs
+index 79fa9c5..fa96e29 100644
+--- a/src/Language/JavaScript/Parser/Lexer.hs
++++ b/src/Language/JavaScript/Parser/Lexer.hs
+@@ -712,7 +712,7 @@ alex_scan_tkn user orig_input len input s last_acc =
+ 		(offset) = (base +# ord_c)
+ 		(check)  = alexIndexInt16OffAddr alex_check offset
+ 		
+-		(new_s) = if (offset >=# 0#) && (check ==# ord_c)
++		(new_s) = if (tagToEnum# (offset >=# 0#)) && (tagToEnum# (check ==# ord_c))
+ 			  then alexIndexInt16OffAddr alex_table offset
+ 			  else alexIndexInt16OffAddr alex_deflt s
+ 	in
+-- 
+1.7.10.4
+
diff --git a/standalone/android/haskell-patches/network_android-port-fixes.patch b/standalone/android/haskell-patches/network_android-port-fixes.patch
new file mode 100644
index 0000000000..784eec13a9
--- /dev/null
+++ b/standalone/android/haskell-patches/network_android-port-fixes.patch
@@ -0,0 +1,341 @@
+From 834a0d3bfe56b969a65eff834604442cde8798f7 Mon Sep 17 00:00:00 2001
+From: dummy 
+Date: Wed, 20 Jun 2018 05:06:41 +0100
+Subject: [PATCH] android port fixes
+
+Build note: Ensure a hsc2hs in PATH is modified to pass -x to the real
+one, to enable cross-compiling.
+---
+ Network/BSD.hsc               | 84 -------------------------------------------
+ Network/Socket.hsc            | 16 ++++-----
+ Network/Socket/ByteString.hsc |  2 +-
+ Network/Socket/Internal.hsc   |  2 +-
+ Network/Socket/Types.hsc      | 14 +++-----
+ cbits/HsNet.c                 | 18 ++++++++++
+ configure                     |  1 +
+ include/HsNetworkConfig.h     |  4 +--
+ 8 files changed, 36 insertions(+), 105 deletions(-)
+
+diff --git a/Network/BSD.hsc b/Network/BSD.hsc
+index 67f2fcd..4c86af5 100644
+--- a/Network/BSD.hsc
++++ b/Network/BSD.hsc
+@@ -28,12 +28,8 @@ module Network.BSD
+     , hostAddress
+ 
+ #if defined(HAVE_GETHOSTENT) && !defined(mingw32_HOST_OS)
+-    , getHostEntries
+-
+     -- ** Low level functionality
+-    , setHostEntry
+     , getHostEntry
+-    , endHostEntry
+ #endif
+ 
+     -- * Service names
+@@ -61,14 +57,6 @@ module Network.BSD
+     , getProtocolNumber
+     , defaultProtocol
+ 
+-#if !defined(mingw32_HOST_OS)
+-    , getProtocolEntries
+-    -- ** Low level functionality
+-    , setProtocolEntry
+-    , getProtocolEntry
+-    , endProtocolEntry
+-#endif
+-
+     -- * Port numbers
+     , PortNumber
+ 
+@@ -80,11 +68,6 @@ module Network.BSD
+ #if !defined(mingw32_HOST_OS)
+     , getNetworkByName
+     , getNetworkByAddr
+-    , getNetworkEntries
+-    -- ** Low level functionality
+-    , setNetworkEntry
+-    , getNetworkEntry
+-    , endNetworkEntry
+ #endif
+ 
+ #if defined(HAVE_IF_NAMETOINDEX)
+@@ -298,30 +281,6 @@ getProtocolNumber proto = do
+  (ProtocolEntry _ _ num) <- getProtocolByName proto
+  return num
+ 
+-#if !defined(mingw32_HOST_OS)
+-getProtocolEntry :: IO ProtocolEntry    -- Next Protocol Entry from DB
+-getProtocolEntry = withLock $ do
+- ent <- throwNoSuchThingIfNull "getProtocolEntry" "no such protocol entry"
+-                $ c_getprotoent
+- peek ent
+-
+-foreign import ccall unsafe  "getprotoent" c_getprotoent :: IO (Ptr ProtocolEntry)
+-
+-setProtocolEntry :: Bool -> IO ()       -- Keep DB Open ?
+-setProtocolEntry flg = withLock $ c_setprotoent (fromBool flg)
+-
+-foreign import ccall unsafe "setprotoent" c_setprotoent :: CInt -> IO ()
+-
+-endProtocolEntry :: IO ()
+-endProtocolEntry = withLock $ c_endprotoent
+-
+-foreign import ccall unsafe "endprotoent" c_endprotoent :: IO ()
+-
+-getProtocolEntries :: Bool -> IO [ProtocolEntry]
+-getProtocolEntries stayOpen = withLock $ do
+-  setProtocolEntry stayOpen
+-  getEntries (getProtocolEntry) (endProtocolEntry)
+-#endif
+ 
+ -- ---------------------------------------------------------------------------
+ -- Host lookups
+@@ -405,21 +364,6 @@ getHostEntry = withLock $ do
+  >>= peek
+ 
+ foreign import ccall unsafe "gethostent" c_gethostent :: IO (Ptr HostEntry)
+-
+-setHostEntry :: Bool -> IO ()
+-setHostEntry flg = withLock $ c_sethostent (fromBool flg)
+-
+-foreign import ccall unsafe "sethostent" c_sethostent :: CInt -> IO ()
+-
+-endHostEntry :: IO ()
+-endHostEntry = withLock $ c_endhostent
+-
+-foreign import ccall unsafe "endhostent" c_endhostent :: IO ()
+-
+-getHostEntries :: Bool -> IO [HostEntry]
+-getHostEntries stayOpen = do
+-  setHostEntry stayOpen
+-  getEntries (getHostEntry) (endHostEntry)
+ #endif
+ 
+ -- ---------------------------------------------------------------------------
+@@ -482,34 +426,6 @@ getNetworkByAddr addr family = withLock $ do
+ 
+ foreign import ccall unsafe "getnetbyaddr"
+    c_getnetbyaddr  :: NetworkAddr -> CInt -> IO (Ptr NetworkEntry)
+-
+-getNetworkEntry :: IO NetworkEntry
+-getNetworkEntry = withLock $ do
+- throwNoSuchThingIfNull "getNetworkEntry" "no more network entries"
+-          $ c_getnetent
+- >>= peek
+-
+-foreign import ccall unsafe "getnetent" c_getnetent :: IO (Ptr NetworkEntry)
+-
+--- | Open the network name database. The parameter specifies
+--- whether a connection is maintained open between various
+--- networkEntry calls
+-setNetworkEntry :: Bool -> IO ()
+-setNetworkEntry flg = withLock $ c_setnetent (fromBool flg)
+-
+-foreign import ccall unsafe "setnetent" c_setnetent :: CInt -> IO ()
+-
+--- | Close the connection to the network name database.
+-endNetworkEntry :: IO ()
+-endNetworkEntry = withLock $ c_endnetent
+-
+-foreign import ccall unsafe "endnetent" c_endnetent :: IO ()
+-
+--- | Get the list of network entries.
+-getNetworkEntries :: Bool -> IO [NetworkEntry]
+-getNetworkEntries stayOpen = do
+-  setNetworkEntry stayOpen
+-  getEntries (getNetworkEntry) (endNetworkEntry)
+ #endif
+ 
+ -- ---------------------------------------------------------------------------
+diff --git a/Network/Socket.hsc b/Network/Socket.hsc
+index 8b2e6fe..b02b80d 100644
+--- a/Network/Socket.hsc
++++ b/Network/Socket.hsc
+@@ -59,7 +59,7 @@ module Network.Socket
+     , HostName
+     , ServiceName
+ 
+-#if defined(IPV6_SOCKET_SUPPORT)
++#if defined(IPV6_SOCKET_SUPPORT) || 1
+     , AddrInfo(..)
+ 
+     , AddrInfoFlag(..)
+@@ -143,7 +143,7 @@ module Network.Socket
+     -- * Special constants
+     , aNY_PORT
+     , iNADDR_ANY
+-#if defined(IPV6_SOCKET_SUPPORT)
++#if defined(IPV6_SOCKET_SUPPORTNO)
+     , iN6ADDR_ANY
+ #endif
+     , sOMAXCONN
+@@ -521,7 +521,7 @@ accept sock@(MkSocket s family stype protocol status) = do
+                 return new_sock
+ #else
+      with (fromIntegral sz) $ \ ptr_len -> do
+-# ifdef HAVE_ACCEPT4
++#if 0
+      new_sock <- throwSocketErrorIfMinus1RetryMayBlock "accept"
+                         (threadWaitRead (fromIntegral s))
+                         (c_accept4 s sockaddr ptr_len (#const SOCK_NONBLOCK))
+@@ -903,7 +903,7 @@ packSocketOption so =
+     Just MaxSegment    -> Just ((#const IPPROTO_TCP), (#const TCP_MAXSEG))
+ #endif
+ #ifdef TCP_NODELAY
+-    Just NoDelay       -> Just ((#const IPPROTO_TCP), (#const TCP_NODELAY))
++    Just NoDelay       -> Nothing -- Just ((#const IPPROTO_TCP), (#const TCP_NODELAY))
+ #endif
+ #ifdef TCP_USER_TIMEOUT
+     Just UserTimeout   -> Just ((#const IPPROTO_TCP), (#const TCP_USER_TIMEOUT))
+@@ -1036,9 +1036,9 @@ iNADDR_ANY :: HostAddress
+ iNADDR_ANY = htonl (#const INADDR_ANY)
+ 
+ -- | Converts the from host byte order to network byte order.
+-foreign import CALLCONV unsafe "htonl" htonl :: Word32 -> Word32
++foreign import CALLCONV unsafe "my_htonl" htonl :: Word32 -> Word32
+ -- | Converts the from network byte order to host byte order.
+-foreign import CALLCONV unsafe "ntohl" ntohl :: Word32 -> Word32
++foreign import CALLCONV unsafe "my_ntohl" ntohl :: Word32 -> Word32
+ 
+ #if defined(IPV6_SOCKET_SUPPORT)
+ -- | The IPv6 wild card address.
+@@ -1206,7 +1206,7 @@ unpackBits ((k,v):xs) r
+ -----------------------------------------------------------------------------
+ -- Address and service lookups
+ 
+-#if defined(IPV6_SOCKET_SUPPORT)
++#if defined(IPV6_SOCKET_SUPPORT) || 1
+ 
+ -- | Flags that control the querying behaviour of 'getAddrInfo'.
+ --   For more information, see 
+@@ -1568,7 +1568,7 @@ foreign import CALLCONV unsafe "bind"
+   c_bind :: CInt -> Ptr SockAddr -> CInt{-CSockLen???-} -> IO CInt
+ foreign import CALLCONV SAFE_ON_WIN "connect"
+   c_connect :: CInt -> Ptr SockAddr -> CInt{-CSockLen???-} -> IO CInt
+-#ifdef HAVE_ACCEPT4
++#if 0
+ foreign import CALLCONV unsafe "accept4"
+   c_accept4 :: CInt -> Ptr SockAddr -> Ptr CInt{-CSockLen???-} -> CInt -> IO CInt
+ #else
+diff --git a/Network/Socket/ByteString.hsc b/Network/Socket/ByteString.hsc
+index 93e29c9..a736932 100644
+--- a/Network/Socket/ByteString.hsc
++++ b/Network/Socket/ByteString.hsc
+@@ -177,7 +177,7 @@ sendMany sock@(MkSocket fd _ _ _ _) cs = do
+       liftM fromIntegral . withIOVec cs $ \(iovsPtr, iovsLen) ->
+           throwSocketErrorWaitWrite sock "writev" $
+               c_writev (fromIntegral fd) iovsPtr
+-              (fromIntegral (min iovsLen (#const IOV_MAX)))
++              (fromIntegral (min iovsLen (0x0026)))
+ #else
+ sendMany sock = sendAll sock . B.concat
+ #endif
+diff --git a/Network/Socket/Internal.hsc b/Network/Socket/Internal.hsc
+index c8bf4f6..2463bd7 100644
+--- a/Network/Socket/Internal.hsc
++++ b/Network/Socket/Internal.hsc
+@@ -24,7 +24,7 @@ module Network.Socket.Internal
+     (
+     -- * Socket addresses
+       HostAddress
+-#if defined(IPV6_SOCKET_SUPPORT)
++#if defined(IPV6_SOCKET_SUPPORTNO)
+     , HostAddress6
+     , FlowInfo
+     , ScopeID
+diff --git a/Network/Socket/Types.hsc b/Network/Socket/Types.hsc
+index b42c98d..e5bb9fe 100644
+--- a/Network/Socket/Types.hsc
++++ b/Network/Socket/Types.hsc
+@@ -758,10 +758,10 @@ intToPortNumber v = PortNum (htons (fromIntegral v))
+ portNumberToInt :: PortNumber -> Int
+ portNumberToInt (PortNum po) = fromIntegral (ntohs po)
+ 
+-foreign import CALLCONV unsafe "ntohs" ntohs :: Word16 -> Word16
+-foreign import CALLCONV unsafe "htons" htons :: Word16 -> Word16
+-foreign import CALLCONV unsafe "ntohl" ntohl :: Word32 -> Word32
+-foreign import CALLCONV unsafe "htonl" htonl :: Word32 -> Word32
++foreign import CALLCONV unsafe "my_ntohs" ntohs :: Word16 -> Word16
++foreign import CALLCONV unsafe "my_htons" htons :: Word16 -> Word16
++foreign import CALLCONV unsafe "my_ntohl" ntohl :: Word32 -> Word32
++foreign import CALLCONV unsafe "my_htonl" htonl :: Word32 -> Word32
+ 
+ instance Enum PortNumber where
+     toEnum   = intToPortNumber
+@@ -1071,13 +1071,9 @@ poke32 p i0 a = do
+ -- | Private newtype proxy for the Storable instance. To avoid orphan instances.
+ newtype In6Addr = In6Addr HostAddress6
+ 
+-#if __GLASGOW_HASKELL__ < 800
+-#let alignment t = "%lu", (unsigned long)offsetof(struct {char x__; t (y__); }, y__)
+-#endif
+-
+ instance Storable In6Addr where
+     sizeOf _    = #const sizeof(struct in6_addr)
+-    alignment _ = #alignment struct in6_addr
++    alignment _ = 64
+ 
+     peek p = do
+         a <- peek32 p 0
+diff --git a/cbits/HsNet.c b/cbits/HsNet.c
+index 86b55dc..6225c32 100644
+--- a/cbits/HsNet.c
++++ b/cbits/HsNet.c
+@@ -6,3 +6,21 @@
+ 
+ #define INLINE
+ #include "HsNet.h"
++
++#include 
++uint16_t my_htons(uint16_t v)
++{
++	htons(v);
++}
++uint32_t my_htonl(uint32_t v)
++{
++	htonl(v);
++}
++uint16_t my_ntohs(uint16_t v)
++{
++	ntohs(v);
++}
++uint32_t my_ntohl(uint32_t v)
++{
++	ntohl(v);
++}
+diff --git a/configure b/configure
+index 9e82879..24ef3ce 100755
+--- a/configure
++++ b/configure
+@@ -1,4 +1,5 @@
+ #! /bin/sh
++set -- --host=arm-linux-androideabi
+ # Guess values for system-dependent variables and create Makefiles.
+ # Generated by GNU Autoconf 2.69 for Haskell network package 2.3.0.14.
+ #
+diff --git a/include/HsNetworkConfig.h b/include/HsNetworkConfig.h
+index 383f6e2..62b8852 100644
+--- a/include/HsNetworkConfig.h
++++ b/include/HsNetworkConfig.h
+@@ -2,7 +2,7 @@
+ /* include/HsNetworkConfig.h.in.  Generated from configure.ac by autoheader.  */
+ 
+ /* Define to 1 if you have the `accept4' function. */
+-#define HAVE_ACCEPT4 1
++/* #undef HAVE_ACCEPT4 */
+ 
+ /* Define to 1 if you have the  header file. */
+ #define HAVE_ARPA_INET_H 1
+@@ -73,7 +73,7 @@
+ #define HAVE_LIMITS_H 1
+ 
+ /* Define to 1 if you have the  header file. */
+-#define HAVE_LINUX_CAN_H 1
++/* #undef HAVE_LINUX_CAN_H */
+ 
+ /* Define to 1 if you have a Linux sendfile(2) implementation. */
+ #define HAVE_LINUX_SENDFILE 1
+-- 
+2.1.4
+
diff --git a/standalone/android/haskell-patches/primitive_0.5.3.0_0001-disable-i386-opt-stuff-to-allow-cross-compilation.patch b/standalone/android/haskell-patches/primitive_0.5.3.0_0001-disable-i386-opt-stuff-to-allow-cross-compilation.patch
new file mode 100644
index 0000000000..efee692b51
--- /dev/null
+++ b/standalone/android/haskell-patches/primitive_0.5.3.0_0001-disable-i386-opt-stuff-to-allow-cross-compilation.patch
@@ -0,0 +1,25 @@
+From ff2d1519fb294a123636ac6bd80e50741922c856 Mon Sep 17 00:00:00 2001
+From: dummy 
+Date: Sun, 25 May 2014 09:41:13 +0200
+Subject: [PATCH] disable i386 opt stuff to allow cross-compilation
+
+---
+ primitive.cabal | 2 --
+ 1 file changed, 2 deletions(-)
+
+diff --git a/primitive.cabal b/primitive.cabal
+index 9651dfd..b655e8d 100644
+--- a/primitive.cabal
++++ b/primitive.cabal
+@@ -47,8 +47,6 @@ Library
+   cc-options: -O3 -fomit-frame-pointer -Wall
+   if !os(solaris)
+       cc-options: -ftree-vectorize
+-  if arch(i386) || arch(x86_64)
+-      cc-options: -msse2
+ 
+ source-repository head
+   type:     git
+-- 
+2.0.0.rc2
+
diff --git a/standalone/android/haskell-patches/socks_0.4.2_0001-remove-IPv6-stuff.patch b/standalone/android/haskell-patches/socks_0.4.2_0001-remove-IPv6-stuff.patch
new file mode 100644
index 0000000000..c9723f3f76
--- /dev/null
+++ b/standalone/android/haskell-patches/socks_0.4.2_0001-remove-IPv6-stuff.patch
@@ -0,0 +1,135 @@
+From e1a2f80f6bec25921ab645a0aaf1c6422a8917ab Mon Sep 17 00:00:00 2001
+From: dummy 
+Date: Mon, 11 Nov 2013 01:06:58 +0000
+Subject: [PATCH] fix
+
+---
+ Network/Socks5/Command.hs  |    8 +-------
+ Network/Socks5/Conf.hs     |    1 -
+ Network/Socks5/Lowlevel.hs |    1 -
+ Network/Socks5/Types.hs    |   18 +-----------------
+ Network/Socks5/Wire.hs     |    2 --
+ 5 files changed, 2 insertions(+), 28 deletions(-)
+
+diff --git a/Network/Socks5/Command.hs b/Network/Socks5/Command.hs
+index db95fbd..fdba5ec 100644
+--- a/Network/Socks5/Command.hs
++++ b/Network/Socks5/Command.hs
+@@ -13,7 +13,6 @@ module Network.Socks5.Command
+     , Connect(..)
+     , Command(..)
+     , connectIPV4
+-    , connectIPV6
+     , connectDomainName
+     -- * lowlevel interface
+     , rpc
+@@ -29,7 +28,7 @@ import qualified Data.ByteString as B
+ import qualified Data.ByteString.Char8 as BC
+ import Data.Serialize
+ 
+-import Network.Socket (Socket, PortNumber, HostAddress, HostAddress6)
++import Network.Socket (Socket, PortNumber, HostAddress)
+ import Network.Socket.ByteString
+ 
+ import Network.Socks5.Types
+@@ -65,11 +64,6 @@ connectIPV4 socket hostaddr port = onReply <$> rpc_ socket (Connect $ SocksAddre
+     where onReply (SocksAddrIPV4 h, p) = (h, p)
+           onReply _                    = error "ipv4 requested, got something different"
+ 
+-connectIPV6 :: Socket -> HostAddress6 -> PortNumber -> IO (HostAddress6, PortNumber)
+-connectIPV6 socket hostaddr6 port = onReply <$> rpc_ socket (Connect $ SocksAddress (SocksAddrIPV6 hostaddr6) port)
+-    where onReply (SocksAddrIPV6 h, p) = (h, p)
+-          onReply _                    = error "ipv6 requested, got something different"
+-
+ -- TODO: FQDN should only be ascii, maybe putting a "fqdn" data type
+ -- in front to make sure and make the BC.pack safe.
+ connectDomainName :: Socket -> String -> PortNumber -> IO (SocksHostAddress, PortNumber)
+diff --git a/Network/Socks5/Conf.hs b/Network/Socks5/Conf.hs
+index c29ff7b..007d382 100644
+--- a/Network/Socks5/Conf.hs
++++ b/Network/Socks5/Conf.hs
+@@ -47,5 +47,4 @@ defaultSocksConfFromSockAddr sockaddr = SocksConf server SocksVer5
+     where server       = SocksAddress haddr port
+           (haddr,port) = case sockaddr of
+                              SockAddrInet p h      -> (SocksAddrIPV4 h, p)
+-                             SockAddrInet6 p _ h _ -> (SocksAddrIPV6 h, p)
+                              _                     -> error "unsupported unix sockaddr type"
+diff --git a/Network/Socks5/Lowlevel.hs b/Network/Socks5/Lowlevel.hs
+index c10d9b9..2c3d59c 100644
+--- a/Network/Socks5/Lowlevel.hs
++++ b/Network/Socks5/Lowlevel.hs
+@@ -17,7 +17,6 @@ resolveToSockAddr :: SocksAddress -> IO SockAddr
+ resolveToSockAddr (SocksAddress sockHostAddr port) =
+     case sockHostAddr of
+         SocksAddrIPV4 ha       -> return $ SockAddrInet port ha
+-        SocksAddrIPV6 ha6      -> return $ SockAddrInet6 port 0 ha6 0
+         SocksAddrDomainName bs -> do he <- getHostByName (BC.unpack bs)
+                                      return $ SockAddrInet port (hostAddress he)
+ 
+diff --git a/Network/Socks5/Types.hs b/Network/Socks5/Types.hs
+index 7fbec25..17c7c83 100644
+--- a/Network/Socks5/Types.hs
++++ b/Network/Socks5/Types.hs
+@@ -19,7 +19,7 @@ module Network.Socks5.Types
+ import Data.ByteString (ByteString)
+ import Data.Word
+ import Data.Data
+-import Network.Socket (HostAddress, HostAddress6, PortNumber)
++import Network.Socket (HostAddress, PortNumber)
+ import Control.Exception
+ import qualified Data.ByteString.Char8 as BC
+ import Numeric (showHex)
+@@ -53,12 +53,10 @@ data SocksMethod =
+ data SocksHostAddress =
+       SocksAddrIPV4 !HostAddress
+     | SocksAddrDomainName !ByteString
+-    | SocksAddrIPV6 !HostAddress6
+     deriving (Eq,Ord)
+ 
+ instance Show SocksHostAddress where
+     show (SocksAddrIPV4 ha)       = "SocksAddrIPV4(" ++ showHostAddress ha ++ ")"
+-    show (SocksAddrIPV6 ha6)      = "SocksAddrIPV6(" ++ showHostAddress6 ha6 ++ ")"
+     show (SocksAddrDomainName dn) = "SocksAddrDomainName(" ++ BC.unpack dn ++ ")"
+ 
+ -- | Converts a HostAddress to a String in dot-decimal notation
+@@ -69,20 +67,6 @@ showHostAddress num = concat [show q1, ".", show q2, ".", show q3, ".", show q4]
+         (num''',q3) = num'' `quotRem` 256
+         (_,q4)      = num''' `quotRem` 256
+ 
+--- | Converts a IPv6 HostAddress6 to standard hex notation
+-showHostAddress6 :: HostAddress6 -> String
+-showHostAddress6 (a,b,c,d) =
+-    (concat . intersperse ":" . map (flip showHex ""))
+-        [p1,p2,p3,p4,p5,p6,p7,p8]
+-    where (a',p2) = a `quotRem` 65536
+-          (_,p1)  = a' `quotRem` 65536
+-          (b',p4) = b `quotRem` 65536
+-          (_,p3)  = b' `quotRem` 65536
+-          (c',p6) = c `quotRem` 65536
+-          (_,p5)  = c' `quotRem` 65536
+-          (d',p8) = d `quotRem` 65536
+-          (_,p7)  = d' `quotRem` 65536
+-
+ -- | Describe a Socket address on the SOCKS protocol
+ data SocksAddress = SocksAddress !SocksHostAddress !PortNumber
+     deriving (Show,Eq,Ord)
+diff --git a/Network/Socks5/Wire.hs b/Network/Socks5/Wire.hs
+index 10bd262..a30f32e 100644
+--- a/Network/Socks5/Wire.hs
++++ b/Network/Socks5/Wire.hs
+@@ -46,12 +46,10 @@ data SocksResponse = SocksResponse
+ 
+ getAddr 1 = SocksAddrIPV4 <$> getWord32host
+ getAddr 3 = SocksAddrDomainName <$> (getWord8 >>= getByteString . fromIntegral)
+-getAddr 4 = SocksAddrIPV6 <$> (liftM4 (,,,) getWord32host getWord32host getWord32host getWord32host)
+ getAddr n = error ("cannot get unknown socket address type: " ++ show n)
+ 
+ putAddr (SocksAddrIPV4 h)         = putWord8 1 >> putWord32host h
+ putAddr (SocksAddrDomainName b)   = putWord8 3 >> putWord8 (fromIntegral $ B.length b) >> putByteString b
+-putAddr (SocksAddrIPV6 (a,b,c,d)) = putWord8 4 >> mapM_ putWord32host [a,b,c,d]
+ 
+ getSocksRequest 5 = do
+     cmd <- toEnum . fromIntegral <$> getWord8
+-- 
+1.7.10.4
+
diff --git a/standalone/android/haskell-patches/stm-chans_cross-build.patch b/standalone/android/haskell-patches/stm-chans_cross-build.patch
new file mode 100644
index 0000000000..f0964d693e
--- /dev/null
+++ b/standalone/android/haskell-patches/stm-chans_cross-build.patch
@@ -0,0 +1,25 @@
+From c1b166ad1dbed80f7eed7b9c1b2dc5c668eeb8fc Mon Sep 17 00:00:00 2001
+From: androidbuilder 
+Date: Fri, 18 Oct 2013 23:28:56 +0000
+Subject: [PATCH] cross build
+
+---
+ stm-chans.cabal |    2 +-
+ 1 file changed, 1 insertion(+), 1 deletion(-)
+
+diff --git a/stm-chans.cabal b/stm-chans.cabal
+index 89d4780..2119a74 100644
+--- a/stm-chans.cabal
++++ b/stm-chans.cabal
+@@ -6,7 +6,7 @@
+ -- and source-repository:.
+ Cabal-Version:  >= 1.6
+ -- We need a custom build in order to define __HADDOCK__
+-Build-Type:     Custom
++Build-Type:     Simple
+ 
+ Name:           stm-chans
+ Version:        3.0.0
+-- 
+1.7.10.4
+
diff --git a/standalone/android/haskell-patches/system-filepath_cross-build.patch b/standalone/android/haskell-patches/system-filepath_cross-build.patch
new file mode 100644
index 0000000000..c9f9304a48
--- /dev/null
+++ b/standalone/android/haskell-patches/system-filepath_cross-build.patch
@@ -0,0 +1,25 @@
+From 0e728d5b049224394908d793c73902a8c981e636 Mon Sep 17 00:00:00 2001
+From: dummy 
+Date: Mon, 26 May 2014 01:04:40 +0000
+Subject: [PATCH] fix cross build
+
+---
+ system-filepath.cabal |    2 +-
+ 1 file changed, 1 insertion(+), 1 deletion(-)
+
+diff --git a/system-filepath.cabal b/system-filepath.cabal
+index d6aa726..f4e5e0f 100644
+--- a/system-filepath.cabal
++++ b/system-filepath.cabal
+@@ -6,7 +6,7 @@ license-file: license.txt
+ author: John Millikin 
+ maintainer: John Millikin 
+ copyright: John Millikin 2010-2012
+-build-type: Custom
++build-type: Simple
+ cabal-version: >= 1.8
+ category: System
+ stability: experimental
+-- 
+1.7.10.4
+
diff --git a/standalone/android/haskell-patches/unbounded-delays_crossbuild.patch b/standalone/android/haskell-patches/unbounded-delays_crossbuild.patch
new file mode 100644
index 0000000000..dd0a7fca90
--- /dev/null
+++ b/standalone/android/haskell-patches/unbounded-delays_crossbuild.patch
@@ -0,0 +1,25 @@
+From 0ad071f80ee72e7b8ca5b0b70dfae5bbf8677969 Mon Sep 17 00:00:00 2001
+From: Joey Hess 
+Date: Wed, 12 Mar 2014 12:18:17 -0400
+Subject: [PATCH] cross build
+
+---
+ unbounded-delays.cabal |    2 +-
+ 1 file changed, 1 insertion(+), 1 deletion(-)
+
+diff --git a/unbounded-delays.cabal b/unbounded-delays.cabal
+index 76d0a50..0f27569 100644
+--- a/unbounded-delays.cabal
++++ b/unbounded-delays.cabal
+@@ -1,7 +1,7 @@
+ name:          unbounded-delays
+ version:       0.1.0.6
+ cabal-version: >= 1.6
+-build-type:    Custom
++build-type:    Simple
+ author:        Bas van Dijk 
+                Roel van Dijk 
+ maintainer:    Bas van Dijk 
+-- 
+1.7.10.4
+
diff --git a/standalone/android/haskell-patches/unix-time_hack-for-Bionic.patch b/standalone/android/haskell-patches/unix-time_hack-for-Bionic.patch
new file mode 100644
index 0000000000..16c4f92a21
--- /dev/null
+++ b/standalone/android/haskell-patches/unix-time_hack-for-Bionic.patch
@@ -0,0 +1,56 @@
+From db9eb179885874af342bb2c3adef7185496ba1f1 Mon Sep 17 00:00:00 2001
+From: dummy 
+Date: Wed, 15 Oct 2014 16:37:32 +0000
+Subject: [PATCH] hack for bionic
+
+---
+ Data/UnixTime/Types.hsc | 12 ------------
+ cbits/conv.c            |  2 +-
+ 2 files changed, 1 insertion(+), 13 deletions(-)
+
+diff --git a/Data/UnixTime/Types.hsc b/Data/UnixTime/Types.hsc
+index d30f39b..ec7ca4c 100644
+--- a/Data/UnixTime/Types.hsc
++++ b/Data/UnixTime/Types.hsc
+@@ -9,8 +9,6 @@ import Foreign.Storable
+ 
+ #include 
+ 
+-#let alignment t = "%lu", (unsigned long)offsetof(struct {char x__; t (y__); }, y__)
+-
+ -- |
+ -- Data structure for Unix time.
+ data UnixTime = UnixTime {
+@@ -20,16 +18,6 @@ data UnixTime = UnixTime {
+   , utMicroSeconds :: {-# UNPACK #-} !Int32
+   } deriving (Eq,Ord,Show)
+ 
+-instance Storable UnixTime where
+-    sizeOf _    = (#size struct timeval)
+-    alignment _ = (#alignment struct timeval)
+-    peek ptr    = UnixTime
+-            <$> (#peek struct timeval, tv_sec)  ptr
+-            <*> (#peek struct timeval, tv_usec) ptr
+-    poke ptr ut = do
+-            (#poke struct timeval, tv_sec)  ptr (utSeconds ut)
+-            (#poke struct timeval, tv_usec) ptr (utMicroSeconds ut)
+-
+ -- |
+ -- Format of the strptime()/strftime() style.
+ type Format = ByteString
+diff --git a/cbits/conv.c b/cbits/conv.c
+index ec31fef..b7bc0f9 100644
+--- a/cbits/conv.c
++++ b/cbits/conv.c
+@@ -96,7 +96,7 @@ time_t c_parse_unix_time_gmt(char *fmt, char *src) {
+ #else
+     strptime(src, fmt, &dst);
+ #endif
+-    return timegm(&dst);
++    return NULL; /* timegm(&dst); (not in Bionic) */
+ }
+ 
+ size_t c_format_unix_time(char *fmt, time_t src, char* dst, int siz) {
+-- 
+2.1.1
+
diff --git a/standalone/android/haskell-patches/uuid_build-without-v1-uuid-which-needs-network-info.patch b/standalone/android/haskell-patches/uuid_build-without-v1-uuid-which-needs-network-info.patch
new file mode 100644
index 0000000000..12cb2a922c
--- /dev/null
+++ b/standalone/android/haskell-patches/uuid_build-without-v1-uuid-which-needs-network-info.patch
@@ -0,0 +1,79 @@
+From 87283f9b6f992a7f0e36c7b1bafc288bf2bf106a Mon Sep 17 00:00:00 2001
+From: androidbuilder 
+Date: Mon, 11 Nov 2013 02:46:27 +0000
+Subject: [PATCH] build without v1 uuid which needs network-ino
+
+---
+ Data/UUID/Util.hs |   11 -----------
+ Data/UUID/V1.hs   |    2 --
+ uuid.cabal        |    2 --
+ 3 files changed, 15 deletions(-)
+
+diff --git a/Data/UUID/Util.hs b/Data/UUID/Util.hs
+index 581391a..399e508 100644
+--- a/Data/UUID/Util.hs
++++ b/Data/UUID/Util.hs
+@@ -3,7 +3,6 @@ module Data.UUID.Util (
+     UnpackedUUID(..)
+   , unpack, pack
+   , version
+-  , extractMac
+   , extractTime
+   , setTime
+   ) where
+@@ -13,7 +12,6 @@ import Data.Word
+ import Data.Word.Util
+ import Data.Bits
+ import Data.UUID.Internal
+-import Network.Info
+ import Data.Int (Int64)
+ 
+ version :: UUID -> Int
+@@ -43,12 +41,3 @@ extractTime uuid =
+ timeAndVersionToTime :: Word16 -> Word16
+ timeAndVersionToTime tv = tv .&. 0x0FFF
+ 
+-extractMac :: UUID -> Maybe MAC
+-extractMac uuid = 
+-  if version uuid == 1
+-  then Just $ 
+-       MAC (node_0 unpacked) (node_1 unpacked) (node_2 unpacked) (node_3 unpacked) (node_4 unpacked) (node_5 unpacked)
+-  else Nothing
+-  where
+-    unpacked = unpack uuid
+-
+diff --git a/Data/UUID/V1.hs b/Data/UUID/V1.hs
+index 067e729..ca4c235 100644
+--- a/Data/UUID/V1.hs
++++ b/Data/UUID/V1.hs
+@@ -37,8 +37,6 @@ import System.IO.Unsafe
+ 
+ import qualified System.Random as R
+ 
+-import Network.Info
+-
+ import Data.UUID.Builder
+ import Data.UUID.Internal
+ 
+diff --git a/uuid.cabal b/uuid.cabal
+index 0a53059..57b1b86 100644
+--- a/uuid.cabal
++++ b/uuid.cabal
+@@ -32,14 +32,12 @@ Library
+                 cryptohash >= 0.7 && < 0.12,
+                 deepseq == 1.3.*,
+                 hashable (>= 1.1.1.0 && < 1.2.0) || (>= 1.2.1 && < 1.3),
+-                network-info == 0.2.*,
+                 random >= 1.0.1 && < 1.1,
+                 time >= 1.1 && < 1.5
+ 
+  Exposed-Modules:
+    Data.UUID
+    Data.UUID.Util
+-   Data.UUID.V1
+    Data.UUID.V3
+    Data.UUID.V4
+    Data.UUID.V5
+-- 
+1.7.10.4
+
diff --git a/standalone/android/haskell-patches/warp_remove-ipv6-stuff.patch b/standalone/android/haskell-patches/warp_remove-ipv6-stuff.patch
new file mode 100644
index 0000000000..cc5368f6b4
--- /dev/null
+++ b/standalone/android/haskell-patches/warp_remove-ipv6-stuff.patch
@@ -0,0 +1,39 @@
+From 2f1d2eddde94d339d91d7b018dc90542b7625fd3 Mon Sep 17 00:00:00 2001
+From: androidbuilder 
+Date: Wed, 20 Jun 2018 14:41:04 +0100
+Subject: [PATCH] remove ipv6 stuff
+
+---
+ Network/Wai/Handler/Warp/Run.hs | 9 +--------
+ 1 file changed, 1 insertion(+), 8 deletions(-)
+
+diff --git a/Network/Wai/Handler/Warp/Run.hs b/Network/Wai/Handler/Warp/Run.hs
+index 116b24e..5c7cbcb 100644
+--- a/Network/Wai/Handler/Warp/Run.hs
++++ b/Network/Wai/Handler/Warp/Run.hs
+@@ -14,7 +14,7 @@ import Control.Monad (when, unless, void)
+ import Data.ByteString (ByteString)
+ import qualified Data.ByteString as S
+ import Data.Char (chr)
+-import Data.IP (toHostAddress, toHostAddress6)
++import Data.IP (toHostAddress)
+ import Data.IORef (IORef, newIORef, readIORef, writeIORef)
+ import Data.Streaming.Network (bindPortTCP)
+ import Network (sClose, Socket)
+@@ -305,13 +305,6 @@ serveConnection conn ii origAddr transport settings app = do
+                         [a] -> Just (SockAddrInet (readInt clientPort)
+                                                        (toHostAddress a))
+                         _ -> Nothing
+-                ["PROXY","TCP6",clientAddr,_,clientPort,_] ->
+-                    case [x | (x, t) <- reads (decodeAscii clientAddr), null t] of
+-                        [a] -> Just (SockAddrInet6 (readInt clientPort)
+-                                                        0
+-                                                        (toHostAddress6 a)
+-                                                        0)
+-                        _ -> Nothing
+                 ("PROXY":"UNKNOWN":_) ->
+                     Just origAddr
+                 _ ->
+-- 
+2.1.4
+
diff --git a/standalone/android/haskell-patches/x509-store_support-Android-cert-store.patch b/standalone/android/haskell-patches/x509-store_support-Android-cert-store.patch
new file mode 100644
index 0000000000..1683f49745
--- /dev/null
+++ b/standalone/android/haskell-patches/x509-store_support-Android-cert-store.patch
@@ -0,0 +1,28 @@
+From 717945172c2f3ff95cce9db2d075122bccfc9a1a Mon Sep 17 00:00:00 2001
+From: androidbuilder 
+Date: Wed, 20 Jun 2018 02:01:30 +0100
+Subject: [PATCH] support Android cert store
+
+Android has only hashsed cert files.
+See https://github.com/vincenthz/hs-certificate/issues/19
+---
+ Data/X509/CertificateStore.hs      |   2 +-
+ 2 files changed, 1 insertion(+), 1 deletion(-)
+ delete mode 100644 Data/X509/.CertificateStore.hs.swp
+
+diff --git a/Data/X509/CertificateStore.hs b/Data/X509/CertificateStore.hs
+index 07449a2..74b8bde 100644
+--- a/Data/X509/CertificateStore.hs
++++ b/Data/X509/CertificateStore.hs
+@@ -106,7 +106,7 @@ listDirectoryCerts path =
+                   && isDigit (s !! 9)
+                   && (s !! 8) == '.'
+                   && all isHexDigit (take 8 s)
+-    isCert x = (not $ isPrefixOf "." x) && (not $ isHashedFile x)
++    isCert x = (not $ isPrefixOf "." x)
+ 
+     getDirContents = E.catch (map (path ) . filter isCert <$> getDirectoryContents path) emptyPaths
+             where emptyPaths :: E.IOException -> IO [FilePath]
+-- 
+2.1.4
+
diff --git a/standalone/android/haskell-patches/xss-sanitize_deps.patch b/standalone/android/haskell-patches/xss-sanitize_deps.patch
new file mode 100644
index 0000000000..0f0cfd9256
--- /dev/null
+++ b/standalone/android/haskell-patches/xss-sanitize_deps.patch
@@ -0,0 +1,24 @@
+From 41eb8ab50125eb6ccf260c5510407483f1d78dd4 Mon Sep 17 00:00:00 2001
+From: dummy 
+Date: Wed, 20 Jun 2018 14:52:52 +0100
+Subject: [PATCH] deps
+
+---
+ xss-sanitize.cabal | 1 +
+ 1 file changed, 1 insertion(+)
+
+diff --git a/xss-sanitize.cabal b/xss-sanitize.cabal
+index 727dc95..2de4270 100644
+--- a/xss-sanitize.cabal
++++ b/xss-sanitize.cabal
+@@ -19,6 +19,7 @@ library
+                     , tagsoup     >= 0.12.2   && < 1
+                     , utf8-string >= 0.3      && < 1
+                     , network     >= 2        && < 3
++                    , network-uri
+                     , css-text    >= 0.1.1    && < 0.2
+                     , text        >= 0.11     && < 2
+                     , attoparsec  >= 0.10.0.3 && < 1
+-- 
+2.1.4
+
diff --git a/standalone/android/haskell-patches/zlib_0.5.4.0_0001-hack-to-build-on-Android.patch b/standalone/android/haskell-patches/zlib_0.5.4.0_0001-hack-to-build-on-Android.patch
new file mode 100644
index 0000000000..a899fb8920
--- /dev/null
+++ b/standalone/android/haskell-patches/zlib_0.5.4.0_0001-hack-to-build-on-Android.patch
@@ -0,0 +1,35 @@
+From 63d07ae4a1e3b77cbe023364599f7c2c3e853d5f Mon Sep 17 00:00:00 2001
+From: Joey Hess 
+Date: Thu, 28 Feb 2013 23:40:57 -0400
+Subject: [PATCH] hack to build on Android
+
+---
+ Codec/Compression/Zlib/Stream.hsc |    4 ++--
+ zlib.cabal                        |    2 +-
+ 2 files changed, 3 insertions(+), 3 deletions(-)
+
+diff --git a/Codec/Compression/Zlib/Stream.hsc b/Codec/Compression/Zlib/Stream.hsc
+index fe851e6..c6168f4 100644
+--- a/Codec/Compression/Zlib/Stream.hsc
++++ b/Codec/Compression/Zlib/Stream.hsc
+@@ -921,7 +921,7 @@ foreign import ccall unsafe "zlib.h inflateInit2_"
+ 
+ c_inflateInit2 :: StreamState -> CInt -> IO CInt
+ c_inflateInit2 z n =
+-  withCAString #{const_str ZLIB_VERSION} $ \versionStr ->
++  withCAString "1.2.5" $ \versionStr ->
+     c_inflateInit2_ z n versionStr (#{const sizeof(z_stream)} :: CInt)
+ 
+ foreign import ccall unsafe "zlib.h inflate"
+@@ -940,7 +940,7 @@ foreign import ccall unsafe "zlib.h deflateInit2_"
+ c_deflateInit2 :: StreamState
+                -> CInt -> CInt -> CInt -> CInt -> CInt -> IO CInt
+ c_deflateInit2 z a b c d e =
+-  withCAString #{const_str ZLIB_VERSION} $ \versionStr ->
++  withCAString "1.2.5" $ \versionStr ->
+     c_deflateInit2_ z a b c d e versionStr (#{const sizeof(z_stream)} :: CInt)
+ 
+ foreign import ccall unsafe "zlib.h deflateSetDictionary"
+-- 
+1.7.10.4
+
diff --git a/standalone/android/icons/drawable-hdpi/ic_launcher.png b/standalone/android/icons/drawable-hdpi/ic_launcher.png
new file mode 100644
index 0000000000..f438747802
Binary files /dev/null and b/standalone/android/icons/drawable-hdpi/ic_launcher.png differ
diff --git a/standalone/android/icons/drawable-hdpi/ic_stat_service_notification_icon.png b/standalone/android/icons/drawable-hdpi/ic_stat_service_notification_icon.png
new file mode 100644
index 0000000000..f30483b38d
Binary files /dev/null and b/standalone/android/icons/drawable-hdpi/ic_stat_service_notification_icon.png differ
diff --git a/standalone/android/icons/drawable-ldpi/ic_launcher.png b/standalone/android/icons/drawable-ldpi/ic_launcher.png
new file mode 100644
index 0000000000..8decfec44a
Binary files /dev/null and b/standalone/android/icons/drawable-ldpi/ic_launcher.png differ
diff --git a/standalone/android/icons/drawable-ldpi/ic_stat_service_notification_icon.png b/standalone/android/icons/drawable-ldpi/ic_stat_service_notification_icon.png
new file mode 100644
index 0000000000..0e0b33e959
Binary files /dev/null and b/standalone/android/icons/drawable-ldpi/ic_stat_service_notification_icon.png differ
diff --git a/standalone/android/icons/drawable-mdpi/ic_launcher.png b/standalone/android/icons/drawable-mdpi/ic_launcher.png
new file mode 100644
index 0000000000..aaf2c79219
Binary files /dev/null and b/standalone/android/icons/drawable-mdpi/ic_launcher.png differ
diff --git a/standalone/android/icons/drawable-mdpi/ic_stat_service_notification_icon.png b/standalone/android/icons/drawable-mdpi/ic_stat_service_notification_icon.png
new file mode 100644
index 0000000000..7febe040d0
Binary files /dev/null and b/standalone/android/icons/drawable-mdpi/ic_stat_service_notification_icon.png differ
diff --git a/standalone/android/icons/drawable-xhdpi/ic_launcher.png b/standalone/android/icons/drawable-xhdpi/ic_launcher.png
new file mode 100644
index 0000000000..4a4ec146a4
Binary files /dev/null and b/standalone/android/icons/drawable-xhdpi/ic_launcher.png differ
diff --git a/standalone/android/icons/drawable-xhdpi/ic_stat_service_notification_icon.png b/standalone/android/icons/drawable-xhdpi/ic_stat_service_notification_icon.png
new file mode 100644
index 0000000000..253e90eb80
Binary files /dev/null and b/standalone/android/icons/drawable-xhdpi/ic_stat_service_notification_icon.png differ
diff --git a/standalone/android/icons/drawable/ic_launcher.png b/standalone/android/icons/drawable/ic_launcher.png
new file mode 120000
index 0000000000..28d94e566c
--- /dev/null
+++ b/standalone/android/icons/drawable/ic_launcher.png
@@ -0,0 +1 @@
+../drawable-mdpi/ic_launcher.png
\ No newline at end of file
diff --git a/standalone/android/icons/drawable/ic_stat_service_notification_icon.png b/standalone/android/icons/drawable/ic_stat_service_notification_icon.png
new file mode 120000
index 0000000000..3c30c49b06
--- /dev/null
+++ b/standalone/android/icons/drawable/ic_stat_service_notification_icon.png
@@ -0,0 +1 @@
+../drawable-mdpi/ic_stat_service_notification_icon.png
\ No newline at end of file
diff --git a/standalone/android/install-haskell-packages b/standalone/android/install-haskell-packages
new file mode 100755
index 0000000000..db6cf621a2
--- /dev/null
+++ b/standalone/android/install-haskell-packages
@@ -0,0 +1,133 @@
+#!/bin/bash
+# Bootstraps from an empty cabal to all the necessary haskell packages
+# being installed, with the necessary patches to work on Android.
+#
+# You should install ghc-android first.
+#
+# The cabal.config is used to pin the haskell packages to the last
+# versions that have been gotten working. To update, delete the
+# cabal.config, run this script with an empty cabal and fix up the broken
+# patches, and then use cabal freeze to generate a new cabal.config.
+
+set -e
+
+if [ ! -d haskell-patches ]; then
+	cd standalone/android
+fi
+
+setupcabal () {
+	# Some packages fail to install in a non unicode locale.
+	LANG=en_US.UTF-8
+	export LANG
+}
+
+patched () {
+	pkg=$1
+	ver=$2
+	if [ -z "$ver" ]; then
+		ver="$(grep " $pkg " ../cabal.config | cut -d= -f 3 | sed 's/,$//')"
+	fi
+	if [ -z "$ver" ]; then
+		cabal unpack --pristine $pkg
+	else
+		cabal unpack --pristine $pkg-$ver
+	fi
+	cd $pkg*
+	git init
+	git config user.name dummy
+	git config user.email dummy@example.com
+	git add .
+	git commit -m "pre-patched state of $pkg"
+	ln -sf ../../cabal.config
+	for patch in ../../haskell-patches/${pkg}_* ../../../no-th/haskell-patches/${pkg}_*; do
+		if [ -e "$patch" ]; then
+			echo trying $patch
+			if ! patch -p1 < $patch; then
+				echo "failed to apply $patch"
+				echo "please resolve this, replace the patch with a new version, and exit the subshell to continue"
+				$SHELL
+			fi
+		fi
+	done
+	if [ -e config.sub ]; then
+		cp /usr/share/misc/config.sub .
+	fi
+	if [ -e config.guess ]; then
+		cp /usr/share/misc/config.guess .
+	fi
+	cabal install # --force-reinstalls --reinstall
+	rm -f cabal.config
+
+	rm -rf $pkg*
+	cd ..
+}
+
+installgitannexdeps () {
+	pushd ../..
+	ln -sf standalone/android/cabal.config
+	cabal install --only-dependencies --flags="-magicmime -concurrentoutput" "$@" # --force-reinstalls --reinstall
+	rm -f cabal.config
+	popd
+}
+
+install_pkgs () {
+	rm -rf tmp
+	mkdir tmp
+	cd tmp
+cat <pw_dir,
++	file = percent_expand(filename, "h", _PATH_ROOT_HOME_PREFIX,
+ 	    "u", pw->pw_name, (char *)NULL);
+ 
+ 	/*
+@@ -347,7 +347,7 @@ expand_authorized_keys(const char *filename, struct passwd *pw)
+ 	if (*file == '/')
+ 		return (file);
+ 
+-	i = snprintf(ret, sizeof(ret), "%s/%s", pw->pw_dir, file);
++	i = snprintf(ret, sizeof(ret), "%s/%s", _PATH_ROOT_HOME_PREFIX, file);
+ 	if (i < 0 || (size_t)i >= sizeof(ret))
+ 		fatal("expand_authorized_keys: path too long");
+ 	xfree(file);
+@@ -436,7 +436,7 @@ secure_filename(FILE *f, const char *file, struct passwd *pw,
+ 		    strerror(errno));
+ 		return -1;
+ 	}
+-	if (realpath(pw->pw_dir, homedir) != NULL)
++	if (realpath(_PATH_ROOT_HOME_PREFIX, homedir) != NULL)
+ 		comparehome = 1;
+ 
+ 	/* check the open file to avoid races */
+diff --git a/authfile.c b/authfile.c
+index 7dd4496..00462e9 100644
+--- a/authfile.c
++++ b/authfile.c
+@@ -613,6 +613,7 @@ int
+ key_perm_ok(int fd, const char *filename)
+ {
+ 	struct stat st;
++	return 1; /* check doesn't make sense on android */
+ 
+ 	if (fstat(fd, &st) < 0)
+ 		return 0;
+diff --git a/misc.c b/misc.c
+index 0bf2db6..4327d03 100644
+--- a/misc.c
++++ b/misc.c
+@@ -25,6 +25,7 @@
+  */
+ 
+ #include "includes.h"
++#include "pathnames.h"
+ 
+ #include 
+ #include 
+@@ -538,12 +539,13 @@ tilde_expand_filename(const char *filename, uid_t uid)
+ 	} else if ((pw = getpwuid(uid)) == NULL)	/* ~/path */
+ 		fatal("tilde_expand_filename: No such uid %ld", (long)uid);
+ 
+-	if (strlcpy(ret, pw->pw_dir, sizeof(ret)) >= sizeof(ret))
++	char *pw_dir=_PATH_ROOT_HOME_PREFIX;
++	if (strlcpy(ret, pw_dir, sizeof(ret)) >= sizeof(ret))
+ 		fatal("tilde_expand_filename: Path too long");
+ 
+ 	/* Make sure directory has a trailing '/' */
+-	len = strlen(pw->pw_dir);
+-	if ((len == 0 || pw->pw_dir[len - 1] != '/') &&
++	len = strlen(pw_dir);
++	if ((len == 0 || pw_dir[len - 1] != '/') &&
+ 	    strlcat(ret, "/", sizeof(ret)) >= sizeof(ret))
+ 		fatal("tilde_expand_filename: Path too long");
+ 
+diff --git a/openbsd-compat/getrrsetbyname.c b/openbsd-compat/getrrsetbyname.c
+index d2bea21..5b5d599 100644
+--- a/openbsd-compat/getrrsetbyname.c
++++ b/openbsd-compat/getrrsetbyname.c
+@@ -56,8 +56,7 @@
+ #include 
+ 
+ #include "getrrsetbyname.h"
+-#include "nameser.h"
+-#include "nameser_compat.h"
++#include "arpa/nameser.h"
+ 
+ #if defined(HAVE_DECL_H_ERRNO) && !HAVE_DECL_H_ERRNO
+ extern int h_errno;
+diff --git a/pathnames.h b/pathnames.h
+index b7b9d91..3c10b11 100644
+--- a/pathnames.h
++++ b/pathnames.h
+@@ -67,7 +67,7 @@
+ #endif
+ 
+ #ifndef _PATH_ROOT_HOME_PREFIX
+-#define _PATH_ROOT_HOME_PREFIX	"/data"
++#define _PATH_ROOT_HOME_PREFIX	getenv("HOME")
+ #endif
+ 
+ /*
+diff --git a/readconf.c b/readconf.c
+index 097bb05..dcbc008 100644
+--- a/readconf.c
++++ b/readconf.c
+@@ -1085,7 +1085,7 @@ read_config_file(const char *filename, const char *host, Options *options,
+ 	if ((f = fopen(filename, "r")) == NULL)
+ 		return 0;
+ 
+-	if (checkperm) {
++	if (checkperm && 0) {
+ 		struct stat sb;
+ 
+ 		if (fstat(fileno(f), &sb) == -1)
+diff --git a/ssh-add.c b/ssh-add.c
+index 738644d..f6fce4a 100644
+--- a/ssh-add.c
++++ b/ssh-add.c
+@@ -471,7 +471,7 @@ main(int argc, char **argv)
+ 		}
+ 
+ 		for (i = 0; default_files[i]; i++) {
+-			snprintf(buf, sizeof(buf), "%s/%s", pw->pw_dir,
++			snprintf(buf, sizeof(buf), "%s/%s", _PATH_ROOT_HOME_PREFIX,
+ 			    default_files[i]);
+ 			if (stat(buf, &st) < 0)
+ 				continue;
+diff --git a/ssh-keygen.c b/ssh-keygen.c
+index 4baf7df..ef8bb25 100644
+--- a/ssh-keygen.c
++++ b/ssh-keygen.c
+@@ -224,7 +224,7 @@ ask_filename(struct passwd *pw, const char *prompt)
+ 		}
+ 	}
+ 	snprintf(identity_file, sizeof(identity_file), "%s/%s",
+-	    strcmp(pw->pw_dir, "/") ? pw->pw_dir : _PATH_ROOT_HOME_PREFIX, name);
++	    _PATH_ROOT_HOME_PREFIX, name);
+ 	fprintf(stderr, "%s (%s): ", prompt, identity_file);
+ 	if (fgets(buf, sizeof(buf), stdin) == NULL)
+ 		exit(1);
+@@ -2268,7 +2268,7 @@ main(int argc, char **argv)
+ 
+ 	/* Create ~/.ssh directory if it doesn't already exist. */
+ 	snprintf(dotsshdir, sizeof dotsshdir, "%s/%s",
+-	    strcmp(pw->pw_dir, "/") ? pw->pw_dir : _PATH_ROOT_HOME_PREFIX,
++	    _PATH_ROOT_HOME_PREFIX,
+ 	    _PATH_SSH_USER_DIR);
+ 	if (strstr(identity_file, dotsshdir) != NULL) {
+ 		if (stat(dotsshdir, &st) < 0) {
+diff --git a/ssh.c b/ssh.c
+index 898e966..ef6c858 100644
+--- a/ssh.c
++++ b/ssh.c
+@@ -703,7 +703,7 @@ main(int ac, char **av)
+ 			fatal("Can't open user config file %.100s: "
+ 			    "%.100s", config, strerror(errno));
+ 	} else {
+-		r = snprintf(buf, sizeof buf, "%s/%s", pw->pw_dir,
++		r = snprintf(buf, sizeof buf, "%s/%s", _PATH_ROOT_HOME_PREFIX,
+ 		    _PATH_SSH_USER_CONFFILE);
+ 		if (r > 0 && (size_t)r < sizeof(buf))
+ 			(void)read_config_file(buf, host, &options, 1);
+@@ -748,7 +748,7 @@ main(int ac, char **av)
+ 	if (options.local_command != NULL) {
+ 		debug3("expanding LocalCommand: %s", options.local_command);
+ 		cp = options.local_command;
+-		options.local_command = percent_expand(cp, "d", pw->pw_dir,
++		options.local_command = percent_expand(cp, "d", _PATH_ROOT_HOME_PREFIX,
+ 		    "h", host, "l", thishost, "n", host_arg, "r", options.user,
+ 		    "p", portstr, "u", pw->pw_name, "L", shorthost,
+ 		    (char *)NULL);
+@@ -888,7 +888,7 @@ main(int ac, char **av)
+ 	 */
+ 	if (config == NULL) {
+ 		r = snprintf(buf, sizeof buf, "%s/%s",
+-		    strcmp(pw->pw_dir, "/") ? pw->pw_dir : _PATH_ROOT_HOME_PREFIX,
++		    _PATH_ROOT_HOME_PREFIX,
+ 		    _PATH_SSH_USER_DIR);
+ 		if (r > 0 && (size_t)r < sizeof(buf) && stat(buf, &st) < 0) {
+ #ifdef WITH_SELINUX
+@@ -1532,7 +1532,7 @@ load_public_identity_files(void)
+ 	if ((pw = getpwuid(original_real_uid)) == NULL)
+ 		fatal("load_public_identity_files: getpwuid failed");
+ 	pwname = xstrdup(pw->pw_name);
+-	pwdir = xstrdup(pw->pw_dir);
++	pwdir = xstrdup(_PATH_ROOT_HOME_PREFIX);
+ 	if (gethostname(thishost, sizeof(thishost)) == -1)
+ 		fatal("load_public_identity_files: gethostname: %s",
+ 		    strerror(errno));
+diff --git a/uidswap.c b/uidswap.c
+index bc6194e..5cbf5d1 100644
+--- a/uidswap.c
++++ b/uidswap.c
+@@ -28,7 +28,6 @@
+ #include "xmalloc.h"
+ 
+ #ifdef ANDROID
+-#include 
+ #include 
+ #include 
+ #endif
+@@ -230,7 +229,7 @@ permanently_set_uid(struct passwd *pw)
+ 	debug("permanently_set_uid: %u/%u", (u_int)pw->pw_uid,
+ 	    (u_int)pw->pw_gid);
+ 
+-#ifdef ANDROID
++#if 0
+ 	if (pw->pw_uid == AID_SHELL) {
+ 		prctl(PR_SET_KEEPCAPS, 1, 0, 0, 0);
+ 
+@@ -317,7 +316,7 @@ permanently_set_uid(struct passwd *pw)
+ 		    (u_int)pw->pw_uid);
+ 	}
+ 
+-#ifdef ANDROID
++#if 0
+ 	if (pw->pw_uid == AID_SHELL) {
+ 		/* set CAP_SYS_BOOT capability, so "adb reboot" will succeed */
+ 		header.version = _LINUX_CAPABILITY_VERSION;
diff --git a/standalone/android/rsync.patch b/standalone/android/rsync.patch
new file mode 100644
index 0000000000..692e9cabb1
--- /dev/null
+++ b/standalone/android/rsync.patch
@@ -0,0 +1,40 @@
+From f91df535053958600d57f9df78d9ce84c8718655 Mon Sep 17 00:00:00 2001
+From: Joey Hess 
+Date: Wed, 13 Feb 2013 15:51:40 -0400
+Subject: [PATCH] android portability
+
+---
+ authenticate.c |    3 ++-
+ batch.c        |    2 +-
+ 2 files changed, 3 insertions(+), 2 deletions(-)
+
+diff --git a/authenticate.c b/authenticate.c
+index 7650377..626dec6 100644
+--- a/authenticate.c
++++ b/authenticate.c
+@@ -296,7 +296,8 @@ void auth_client(int fd, const char *user, const char *challenge)
+                  *
+                  * OpenBSD has a readpassphrase() that might be more suitable.
+                  */
+-		pass = getpass("Password: ");
++		/*pass = getpass("Password: "); */
++		exit(1);
+ 	}
+ 
+ 	if (!pass)
+diff --git a/batch.c b/batch.c
+index a3e9dca..ee31532 100644
+--- a/batch.c
++++ b/batch.c
+@@ -221,7 +221,7 @@ void write_batch_shell_file(int argc, char *argv[], int file_arg_cnt)
+ 	stringjoin(filename, sizeof filename,
+ 		   batch_name, ".sh", NULL);
+ 	fd = do_open(filename, O_WRONLY | O_CREAT | O_TRUNC,
+-		     S_IRUSR | S_IWUSR | S_IEXEC);
++		     S_IRUSR | S_IWUSR);
+ 	if (fd < 0) {
+ 		rsyserr(FERROR, errno, "Batch file %s open error",
+ 			filename);
+-- 
+1.7.10.4
+
diff --git a/standalone/android/runshell b/standalone/android/runshell
new file mode 100755
index 0000000000..68f23b481b
--- /dev/null
+++ b/standalone/android/runshell
@@ -0,0 +1,133 @@
+#!/system/bin/sh
+# This is runs a shell in an environment configured for git-annex.
+# Nearly the only command that can be used in here is busybox!
+# lib.start.so will run us in the root of our app directory
+base=$(./busybox pwd)
+cmd=$base/busybox
+
+set -e
+
+prep () {
+	# Cannot rely on Android providing a sane HOME
+	HOME="/sdcard/git-annex.home"
+	export HOME
+}
+
+buildtree () {
+	$cmd echo "Installation starting to $base"
+	$cmd cat "lib/lib.version.so"
+
+	if $cmd test -e "$base/bin"; then
+		$cmd mv "$base/bin" "$base/bin.old"
+	fi
+	$cmd mkdir -p "$base/bin"
+
+	for prog in busybox git-annex git-shell git-upload-pack git gpg rsync ssh ssh-keygen; do
+		$cmd echo "installing $prog"
+		if $cmd test -e "$base/bin/$prog"; then
+			$cmd rm -f "$base/bin/$prog"
+		fi
+		$cmd ln -s "$base/lib/lib.$prog.so" "$base/bin/$prog"
+	done
+
+	$cmd --install -s $base/bin
+
+	$cmd rm -rf "$base/bin.old"
+
+	$cmd tar zxf $base/lib/lib.git.tar.gz.so
+	for prog in git git-shell git-upload-pack; do
+		for link in $($cmd cat "$base/links/$prog"); do
+			$cmd echo "linking $link to $prog"
+			if $cmd test -e "$base/$link"; then
+				$cmd rm -f "$base/$link"
+			fi
+			$cmd ln -s "$base/bin/$prog" "$base/$link"
+		done
+		$cmd rm -f "$base/links/$prog"
+	done
+
+	$cmd mkdir -p "$base/templates"
+	$cmd mkdir -p "$base/tmp"
+
+	$cmd echo "#!/system/bin/sh" > "$base/runshell"
+	$cmd echo "exec $base/lib/lib.start.so" >> "$base/runshell"
+	$cmd chmod 755 runshell
+
+	$cmd cat "$base/lib/lib.trustedkeys.so" > "$base/bin/trustedkeys.gpg"
+	$cmd cat "$base/lib/lib.version.so" > "$base/installed-version"
+	$cmd echo "Installation complete"
+}
+
+install () {
+	if ! $cmd mkdir -p "$HOME"; then
+		$cmd echo "mkdir of $HOME failed!"
+	fi
+	if $cmd test ! -e "$base/bin/git-annex"; then
+		if ! buildtree > $HOME/git-annex-install.log 2>&1; then
+			$cmd echo "Installation failed! Please report a bug and attach $HOME/git-annex-install.log"
+			$cmd sh
+		fi
+	elif $cmd test ! -e "$base/installed-version" || ! $cmd cmp "$base/installed-version" "$base/lib/lib.version.so" >/dev/null; then
+		if ! buildtree > $HOME/git-annex-install.log 2>&1; then
+			$cmd echo "Upgrade failed! Please report a bug and attach $HOME/git-annex-install.log"
+		fi
+	fi
+}
+
+run () {
+	PATH="$base/bin:$PATH"
+	export PATH
+	
+	ORIG_GIT_EXEC_PATH="$GIT_EXEC_PATH"
+	export ORIG_GIT_EXEC_PATH
+	GIT_EXEC_PATH=$base/libexec/git-core
+	export GIT_EXEC_PATH
+	
+	ORIG_GIT_TEMPLATE_DIR="$GIT_TEMPLATE_DIR"
+	export ORIG_GIT_TEMPLATE_DIR
+	GIT_TEMPLATE_DIR="$base/templates"
+	export GIT_TEMPLATE_DIR
+
+	# Indicate which variables were exported above.
+	GIT_ANNEX_STANDLONE_ENV="GIT_EXEC_PATH GIT_TEMPLATE_DIR"
+	export GIT_ANNEX_STANDLONE_ENV
+	
+	# This is a temporary directory on a non-crippled filesystem.
+	# Needs to be as short a path as possible, for ssh sockets.
+	GIT_ANNEX_TMP_DIR=$base/tmp
+	export GIT_ANNEX_TMP_DIR
+	# /tmp probably doesn't exist, so also use it as TMPDIR
+	TMPDIR=$GIT_ANNEX_TMP_DIR
+	export TMPDIR
+
+	if $cmd test ! -e "$HOME/.gitconfig"; then
+		git config --global user.email "git-annex@android"
+		git config --global user.name "android"
+	fi
+
+	if $cmd test "$1"; then
+		cmd="$1"
+		shift 1
+		exec "$cmd" "$@"
+	else
+		# As good a start point as any.
+		cd "$HOME"
+
+		/system/bin/sh
+	fi
+}
+
+if $cmd test -n "$MKFIFO"; then
+	# because java is insane
+	$cmd mkfifo "$MKFIFO"
+else
+	if ! prep; then
+		$cmd echo "prep failed. Please report a bug."
+		read line
+	fi
+	if ! install; then
+		$cmd echo "install failed. Please report a bug."
+		read line
+	fi
+	run
+fi
diff --git a/standalone/android/start.c b/standalone/android/start.c
new file mode 100644
index 0000000000..c67c5da0ce
--- /dev/null
+++ b/standalone/android/start.c
@@ -0,0 +1,64 @@
+/* Installed as lib.start.so, this bootstraps a working busybox and uses
+ * it to run lib.runshell.so. */
+
+#include 
+#include 
+#include 
+#include 
+#include 
+#include 
+
+void chopdir (char *s) {
+	char *p=strrchr(s, '/');
+	if (p == NULL) {
+		fprintf(stderr, "cannot find directory in %s", s);
+		exit(1);
+	}
+	p[0] = '\0';
+}
+
+main () {
+	char buf[1024];
+	char *p;
+	struct stat st_buf;
+
+	/* Get something like /data/data/ga.androidterm/lib/lib.start.so */
+	int n=readlink("/proc/self/exe", buf, 1023);
+	if (n < 1) {
+		fprintf(stderr, "failed to find own name");
+		exit(1);
+	}
+	buf[n] = '\0';
+
+	/* Change directory to something like /data/data/ga.androidterm */
+	chopdir(buf);
+	chopdir(buf);
+	if (chdir(buf) != 0) {
+		perror("chdir");
+		exit(1);
+	}
+
+	if (stat("lib/lib.busybox.so", &st_buf) != 0) {
+		/* TODO my lib dir should be in LD_LIBRARY_PATH; check that */
+		fprintf(stderr, "Falling back to hardcoded app location; cannot find expected files in %s\n", buf);
+		if (chdir("/data/data/ga.androidterm") != 0) {
+			perror("chdir");
+			exit(1);
+		}
+	}
+
+	/* If this is the first run, set up busybox symlink,
+	 * which allows busybox to run. */
+	if (stat("busybox", &st_buf) != 0) {
+		if (symlink("lib/lib.busybox.so", "busybox") != 0) {
+			/* Just in case! */
+			if (link("lib/lib.busybox.so", "busybox") != 0) {
+				perror("link busybox");
+				exit(1);
+			}
+		}
+	}
+
+	execl("./busybox", "./busybox", "sh", "lib/lib.runshell.so", NULL);
+	perror("error running busybox sh");
+}
diff --git a/standalone/android/term.patch b/standalone/android/term.patch
new file mode 100644
index 0000000000..c0ceefd740
--- /dev/null
+++ b/standalone/android/term.patch
@@ -0,0 +1,598 @@
+diff --git a/AndroidManifest.xml b/AndroidManifest.xml
+index b0e866a..1ab8515 100644
+--- a/AndroidManifest.xml
++++ b/AndroidManifest.xml
+@@ -7,6 +7,7 @@
+     
+     
+     
++    
+     
+     
+ 
+ 
++    
+     
+@@ -34,8 +36,6 @@
+         android:icon="@drawable/ic_menu_preferences" />
+     
+-    
+     
+     Predešlé okno
+    Další okno
+    Reset terminálu
+-   Napiš email
++   WebApp
+    Speciální znaky
+    Přepnout soft. klávesnici
+    
+@@ -124,4 +124,4 @@
+   Není nastaveno žádné tlačitko pro funkční klávesu.
+ 
+   Zavřít okno?
+-
+\ No newline at end of file
++
+diff --git a/res/values-de/strings.xml b/res/values-de/strings.xml
+index f6134a5..06d2e1f 100644
+--- a/res/values-de/strings.xml
++++ b/res/values-de/strings.xml
+@@ -23,7 +23,7 @@
+    Vorh. Fenster
+    Nächst. Fenster
+    Zurücksetzen
+-   Email schreiben
++   WebApp
+    Spezialtasten
+    Tastatur an/aus
+ 	
+@@ -124,4 +124,4 @@
+ 
+   Beliebige Scripte im Terminal Emulator ausführen
+   Erlaubt Anwendungen, neue Fenster im Android Terminal Emulator zu öffnen und in diesen Befehle auszuführen. Dies schließt alle Berechtigungen von Android Terminal Emulator ein, inklusive Internetzugang und Schreib-/Leserechte auf der SD-Karte.
+-
+\ No newline at end of file
++
+diff --git a/res/values-es/strings.xml b/res/values-es/strings.xml
+index 94553b2..92d9e2a 100644
+--- a/res/values-es/strings.xml
++++ b/res/values-es/strings.xml
+@@ -23,7 +23,7 @@
+    Ventana anterior
+    Ventana posterior
+    Reiniciar consola
+-   Enviar email
++   WebApp
+    Teclas especiales
+    Ver/ocultar teclado
+ 
+diff --git a/res/values-eu/strings.xml b/res/values-eu/strings.xml
+index b9f0586..92a16a0 100644
+--- a/res/values-eu/strings.xml
++++ b/res/values-eu/strings.xml
+@@ -23,7 +23,7 @@
+    Aurreko leihoa
+    Hurrengo leihoa
+    Berrezarri terminala
+-   Bidali eposta ...(r)i
++   WebApp
+    Tekla bereziak
+    Txandakatu soft teklatua
+ 
+diff --git a/res/values-fr/strings.xml b/res/values-fr/strings.xml
+index beab2be..529f720 100644
+--- a/res/values-fr/strings.xml
++++ b/res/values-fr/strings.xml
+@@ -24,7 +24,7 @@
+    Fenêtre Préc.
+    Fenêtre Suiv.
+    Terminal par défaut
+-   Envoyer un e-mail
++   WebApp
+    Touches spéciales
+    Afficher/Masquer Clavier
+ 
+diff --git a/res/values-hu/strings.xml b/res/values-hu/strings.xml
+index 5dbecb5..ce917c5 100644
+--- a/res/values-hu/strings.xml
++++ b/res/values-hu/strings.xml
+@@ -23,7 +23,7 @@
+    Előző ablak
+    Következő ablak
+    Alaphelyzet
+-   Küldés emailben
++   WebApp
+    Speciális billentyűk
+    Billentyűzet ki/be
+ 
+@@ -148,4 +148,4 @@
+   Az Alt billentyű ESC-et küld
+   Az Alt billentyű ESC-et küld.
+   Az Alt billentyű nem ESC-et küld.
+-
+\ No newline at end of file
++
+diff --git a/res/values-it/strings.xml b/res/values-it/strings.xml
+index e6a7294..9d97869 100644
+--- a/res/values-it/strings.xml
++++ b/res/values-it/strings.xml
+@@ -23,7 +23,7 @@
+    Fin. successiva
+    Fin. precedente
+    Reset terminale
+-   Invia email
++   WebApp
+    Tasti speciali
+    Mostra/nascondi tastiera
+ 
+diff --git a/res/values-ja/strings.xml b/res/values-ja/strings.xml
+index 502fa23..dbfe9fa 100644
+--- a/res/values-ja/strings.xml
++++ b/res/values-ja/strings.xml
+@@ -23,7 +23,7 @@
+    前のウインドウ
+    次のウインドウ
+    端末をリセット
+-   メール送信
++   WebApp
+    特殊キー
+    ソフトキーボード
+ 
+diff --git a/res/values-ka/strings.xml b/res/values-ka/strings.xml
+index 06629d0..1d545b6 100644
+--- a/res/values-ka/strings.xml
++++ b/res/values-ka/strings.xml
+@@ -23,7 +23,7 @@
+    წინა ფანჯარა
+    შემდეგი ფანჯარა
+    ტერმინალის რესტარტი
+-   ელ-ფოსტის გაგზავნა
++   WebApp
+    სპეციალური ღილაკები
+    პროგრამული კლავიატურის ჩართ./გამორთ.
+ 
+diff --git a/res/values-nb/strings.xml b/res/values-nb/strings.xml
+index d81ee07..c370c6f 100644
+--- a/res/values-nb/strings.xml
++++ b/res/values-nb/strings.xml
+@@ -18,7 +18,7 @@
+    Terminal Emulator
+    Innstillinger
+    Tilbakestill terminal
+-   Send epost til
++   WebApp
+    Spesielle tegn
+    Veksle virtuelt tastatur
+ 
+diff --git a/res/values-nl/strings.xml b/res/values-nl/strings.xml
+index 19fa3d0..e24fbf5 100644
+--- a/res/values-nl/strings.xml
++++ b/res/values-nl/strings.xml
+@@ -23,7 +23,7 @@
+    Vorig venster
+    Volgend venster
+    Herstellen
+-   E-mail sturen naar
++   WebApp
+    Speciale knoppen
+    Toetsenbord aan/uit
+ 
+diff --git a/res/values-pl/strings.xml b/res/values-pl/strings.xml
+index 25b3b43..2e0e651 100644
+--- a/res/values-pl/strings.xml
++++ b/res/values-pl/strings.xml
+@@ -23,7 +23,7 @@
+    Poprzednie okno
+    Następne okno
+    Wyczyść terminal
+-   Wyślij e-mail
++   WebApp
+    Przyciski specjalne
+    Pokaż klawiaturę
+ 
+diff --git a/res/values-pt-rPT/strings.xml b/res/values-pt-rPT/strings.xml
+index aa4ba54..aa3d735 100644
+--- a/res/values-pt-rPT/strings.xml
++++ b/res/values-pt-rPT/strings.xml
+@@ -18,7 +18,7 @@
+    Terminal Emulator
+    Preferências
+    Reset terminal
+-   Email para
++   WebApp
+    Teclas especiais
+    Abrir teclado
+ 
+diff --git a/res/values-pt/strings.xml b/res/values-pt/strings.xml
+index 2d992f9..e53a9ac 100644
+--- a/res/values-pt/strings.xml
++++ b/res/values-pt/strings.xml
+@@ -23,7 +23,7 @@
+    Anterior
+    Seguinte
+    Repor terminal
+-   Enviar mensagem para
++   WebApp
+    Teclas especiais
+    Mostrar/ocultar teclado
+ 
+diff --git a/res/values-ro/strings.xml b/res/values-ro/strings.xml
+index 3c7ea06..7a072d0 100644
+--- a/res/values-ro/strings.xml
++++ b/res/values-ro/strings.xml
+@@ -21,7 +21,7 @@
+    "Fereastra anterioră"
+    "Fereastra următoare"
+    "Resetaţi "
+-   "E-mail"
++   WebApp
+    "Taste speciale"
+    "Comutați tastatura"
+ 
+@@ -123,4 +123,4 @@
+   "Tasta Alt trimite ESC"
+   "Tasta Alt trimite ESC."
+   "Tasta Alt nu trimite ESC."
+-
+\ No newline at end of file
++
+diff --git a/res/values-ru/strings.xml b/res/values-ru/strings.xml
+index 0e96360..762324e 100644
+--- a/res/values-ru/strings.xml
++++ b/res/values-ru/strings.xml
+@@ -8,7 +8,7 @@
+     Предыдущее окно
+     Следующее окно
+     Сбросить терминал
+-    Отправить Email
++    WebApp
+     Специальные клавиши
+     Экранная клавиатура
+     Терминальное состояние этого окна было сброшено.
+diff --git a/res/values-sk/strings.xml b/res/values-sk/strings.xml
+index ef35366..cc31d80 100644
+--- a/res/values-sk/strings.xml
++++ b/res/values-sk/strings.xml
+@@ -23,7 +23,7 @@
+   Dalšie okno
+   Predch. okno
+   Obnoviť term.
+-  Poslať e-mailom
++  WebApp
+   Špec. klávesy
+   Skryť/zobraziť klávesnicu
+ 
+diff --git a/res/values-sv/strings.xml b/res/values-sv/strings.xml
+index 1aa9055..8de6c09 100644
+--- a/res/values-sv/strings.xml
++++ b/res/values-sv/strings.xml
+@@ -3,7 +3,7 @@
+     Terminalemulator
+     Inställningar
+     Återställ terminal
+-    E-posta till
++    WebApp
+     Special tangenter
+     Växla till virtuellt tangentbord
+     Aktivera VäckningsLås
+diff --git a/res/values-tr/strings.xml b/res/values-tr/strings.xml
+index b45fa12..fb70f78 100644
+--- a/res/values-tr/strings.xml
++++ b/res/values-tr/strings.xml
+@@ -18,7 +18,7 @@
+    Terminal Emülatörü
+    Tercihler
+    Terminali yeniden başlat
+-   Email olarak yolla
++   WebApp
+    Özel tuşlar
+    Ekran klavyesine geç
+ 
+diff --git a/res/values-uk/strings.xml b/res/values-uk/strings.xml
+index 2f267a9..d3622c4 100644
+--- a/res/values-uk/strings.xml
++++ b/res/values-uk/strings.xml
+@@ -8,7 +8,7 @@
+     Попереднє вікно
+     Наступне вікно
+     Скинути термінал
+-    Відіслати Email
++    WebApp
+     Спеціальні клавіші
+     Екранна клавіатура
+     "Термінальний стан цього вікна було скинуто."
+diff --git a/res/values-zh-rCN/strings.xml b/res/values-zh-rCN/strings.xml
+index 6f47b4f..81316ec 100644
+--- a/res/values-zh-rCN/strings.xml
++++ b/res/values-zh-rCN/strings.xml
+@@ -18,7 +18,7 @@
+    终端模拟器
+    首选项
+    重置终端
+-   发送电子邮件到...
++   WebApp
+    特殊键
+    打开/关闭软键盘
+ 
+diff --git a/res/values-zh-rTW/strings.xml b/res/values-zh-rTW/strings.xml
+index afda0f1..37d1b50 100644
+--- a/res/values-zh-rTW/strings.xml
++++ b/res/values-zh-rTW/strings.xml
+@@ -18,7 +18,7 @@
+    模擬終端
+    設定
+    結束
+-   以電郵傳送
++   WebApp
+    特別按鍵
+    顯示／隱藏鍵盤
+ 
+diff --git a/res/values/defaults.xml b/res/values/defaults.xml
+index 67287b2..9b7cfcd 100644
+--- a/res/values/defaults.xml
++++ b/res/values/defaults.xml
+@@ -13,10 +13,10 @@
+    4
+    0
+    false
+-   /system/bin/sh -
+-   
++   /data/data/ga.androidterm/lib/lib.start.so
++   git annex webapp
+    screen
+-   true
++   false
+    true
+    true
+    true
+diff --git a/res/values/strings.xml b/res/values/strings.xml
+index f1464e9..b06ec9a 100644
+--- a/res/values/strings.xml
++++ b/res/values/strings.xml
+@@ -23,7 +23,7 @@
+    Prev window
+    Next window
+    Reset term
+-   Email to
++   Open WebApp
+    Special keys
+    Toggle soft keyboard
+ 
+diff --git a/src/jackpal/androidterm/ShellTermSession.java b/src/jackpal/androidterm/ShellTermSession.java
+index 501e7ab..0b43513 100644
+--- a/src/jackpal/androidterm/ShellTermSession.java
++++ b/src/jackpal/androidterm/ShellTermSession.java
+@@ -80,12 +80,12 @@ public class ShellTermSession extends TermSession {
+         }
+     };
+ 
+-    public ShellTermSession(TermSettings settings, String initialCommand) {
++    public ShellTermSession(TermSettings settings, String initialCommand, String webAppFifo) {
+         super();
+ 
+         updatePrefs(settings);
+ 
+-        initializeSession();
++        initializeSession(webAppFifo);
+         mInitialCommand = initialCommand;
+ 
+         mWatcherThread = new Thread() {
+@@ -106,7 +106,7 @@ public class ShellTermSession extends TermSession {
+         setDefaultUTF8Mode(settings.defaultToUTF8Mode());
+     }
+ 
+-    private void initializeSession() {
++    private void initializeSession(String webAppFifo) {
+         TermSettings settings = mSettings;
+ 
+         int[] processId = new int[1];
+@@ -128,9 +128,10 @@ public class ShellTermSession extends TermSession {
+         if (settings.verifyPath()) {
+             path = checkPath(path);
+         }
+-        String[] env = new String[2];
++        String[] env = new String[3];
+         env[0] = "TERM=" + settings.getTermType();
+         env[1] = "PATH=" + path;
++	env[2] = "FIFO=" + webAppFifo;
+ 
+         createSubprocess(processId, settings.getShell(), env);
+         mProcId = processId[0];
+diff --git a/src/jackpal/androidterm/Term.java b/src/jackpal/androidterm/Term.java
+index 8a3a4ac..824025d 100644
+--- a/src/jackpal/androidterm/Term.java
++++ b/src/jackpal/androidterm/Term.java
+@@ -20,6 +20,13 @@ import java.io.UnsupportedEncodingException;
+ import java.text.Collator;
+ import java.util.Arrays;
+ import java.util.Locale;
++import java.lang.Process;
++import java.lang.ProcessBuilder;
++import java.util.Map;
++
++import java.io.FileReader;
++import java.io.BufferedReader;
++import java.io.File;
+ 
+ import android.app.Activity;
+ import android.app.AlertDialog;
+@@ -59,6 +66,11 @@ import android.view.inputmethod.InputMethodManager;
+ import android.widget.TextView;
+ import android.widget.Toast;
+ 
++import android.content.Intent;
++import android.net.Uri;
++import android.app.Activity;
++import android.content.Context;
++
+ import jackpal.androidterm.emulatorview.ColorScheme;
+ import jackpal.androidterm.emulatorview.EmulatorView;
+ import jackpal.androidterm.emulatorview.TermSession;
+@@ -107,6 +119,9 @@ public class Term extends Activity implements UpdateCallback {
+     public static final String EXTRA_WINDOW_ID = "jackpal.androidterm.window_id";
+     private int onResumeSelectWindow = -1;
+ 
++    public static String appDir;
++    public static String webAppFifo;
++
+     private PowerManager.WakeLock mWakeLock;
+     private WifiManager.WifiLock mWifiLock;
+     // Available on API 12 and later
+@@ -257,6 +272,48 @@ public class Term extends Activity implements UpdateCallback {
+     @Override
+     public void onCreate(Bundle icicle) {
+         super.onCreate(icicle);
++	
++	try {
++		appDir = getApplicationContext().getPackageManager().getPackageInfo(getPackageName(), 0).applicationInfo.dataDir;
++	} catch (Exception e) {
++		appDir = "/data/data/ga.androidterm";
++	}
++	webAppFifo = appDir + "/fifo";
++
++	/* webapp url opening thread */
++        new Thread() {
++            @Override
++            public void run() {
++		    try {
++			    /* First, set up the fifo that urls to open will be
++			     * read from. This is complicated by java not being
++			     * able to mkfifo. */
++			    File f = new File (webAppFifo);
++			    if (! f.exists()) {
++				    ProcessBuilder pb = new ProcessBuilder(appDir + "/lib/lib.start.so");
++				    Map env = pb.environment();
++				    env.put("MKFIFO", webAppFifo);
++				    Process p = pb.start();
++				    p.waitFor();
++			    }
++           	 
++			    /* Reading from the fifo blocks until a url is written
++			     * to it. */
++			    while (true) {
++			            BufferedReader buf = new BufferedReader(new FileReader(webAppFifo));
++				    String s = buf.readLine();
++				    try {
++					    Intent intent = new Intent(Intent.ACTION_VIEW, Uri.parse(s));
++					    startActivity(intent);
++				    } catch (Exception e) {
++				    }
++
++			    }
++		    } catch (Exception e) {
++		    }
++	    }
++        }.start();
++
+         Log.e(TermDebug.LOG_TAG, "onCreate");
+         mPrefs = PreferenceManager.getDefaultSharedPreferences(this);
+         mSettings = new TermSettings(getResources(), mPrefs);
+@@ -427,7 +484,7 @@ public class Term extends Activity implements UpdateCallback {
+     }
+ 
+     protected static TermSession createTermSession(Context context, TermSettings settings, String initialCommand) {
+-        ShellTermSession session = new ShellTermSession(settings, initialCommand);
++        ShellTermSession session = new ShellTermSession(settings, initialCommand, webAppFifo);
+         // XXX We should really be able to fetch this from within TermSession
+         session.setProcessExitMessage(context.getString(R.string.process_exit_message));
+ 
+@@ -911,31 +968,15 @@ public class Term extends Activity implements UpdateCallback {
+     }
+ 
+     private void doEmailTranscript() {
++	// Hack: repurposed to open the git-annex webapp
+         TermSession session = getCurrentTermSession();
+         if (session != null) {
+-            // Don't really want to supply an address, but
+-            // currently it's required, otherwise nobody
+-            // wants to handle the intent.
+-            String addr = "user@example.com";
+-            Intent intent =
+-                    new Intent(Intent.ACTION_SENDTO, Uri.parse("mailto:"
+-                            + addr));
+-
+-            String subject = getString(R.string.email_transcript_subject);
+-            String title = session.getTitle();
+-            if (title != null) {
+-                subject = subject + " - " + title;
+-            }
+-            intent.putExtra(Intent.EXTRA_SUBJECT, subject);
+-            intent.putExtra(Intent.EXTRA_TEXT,
+-                    session.getTranscriptText().trim());
+             try {
+-                startActivity(Intent.createChooser(intent,
+-                        getString(R.string.email_transcript_chooser_title)));
+-            } catch (ActivityNotFoundException e) {
+-                Toast.makeText(this,
+-                        R.string.email_transcript_no_email_activity_found,
+-                        Toast.LENGTH_LONG).show();
++              BufferedReader buf = new BufferedReader(new FileReader("/sdcard/git-annex.home/.git-annex-url"));
++              String s = buf.readLine();
++              Intent intent = new Intent(Intent.ACTION_VIEW, Uri.parse(s));
++              startActivity(intent);
++	    } catch (Exception e) {
+             }
+         }
+     }
+diff --git a/tools/build-debug b/tools/build-debug
+index 1f15cd2..e611956 100755
+--- a/tools/build-debug
++++ b/tools/build-debug
+@@ -34,4 +34,4 @@ fi
+ 
+ rm -rf `find . -name bin -o -name obj -prune`
+ cd jni
+-$ANDROID_NDK_ROOT/ndk-build && cd .. && ant debug
++$ANDROID_NDK_ROOT/ndk-build && cd ..
+diff --git a/tools/update.sh b/tools/update.sh
+index 57219c3..79b45ef 100755
+--- a/tools/update.sh
++++ b/tools/update.sh
+@@ -18,7 +18,7 @@ command -v "$ANDROID" >/dev/null 2>&1 || { echo >&2 "The $ANDROID tool is not fo
+ 
+ # Make sure target-11 is installed
+ 
+-$ANDROID update sdk -u -t android-11
++$ANDROID update sdk -u -t android-19
+ 
+ DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
+ ATE_ROOT="$( cd $DIR/.. && pwd )"
+@@ -31,5 +31,5 @@ for PROJECT_FILE in $PROJECT_FILES
+ do
+     PROJECT_DIR="$( dirname "$PROJECT_FILE" )"
+     echo "Updating $PROJECT_FILE"
+-    $ANDROID update project -p "$PROJECT_DIR" --target android-11
++    $ANDROID update project -p "$PROJECT_DIR" --target android-19
+ done
diff --git a/standalone/android/toolchainpath b/standalone/android/toolchainpath
new file mode 100755
index 0000000000..bbed01c88f
--- /dev/null
+++ b/standalone/android/toolchainpath
@@ -0,0 +1,54 @@
+#!/bin/sh
+# Outputs a new PATH setting that is prefixed by the path to the
+# Android cross-compiler toolchain to use for a given Android version.
+#
+# For Android 5, force PIE build flags
+#
+# Since the ghc-android wrappers actually hardcode the path to the
+# toolchain, and we want to wrap the toolchain programs, the binaries
+# are moved to .orig and replaced by wrappers.
+
+androidversion="$1"
+
+# Allow running from the top or inside this directory.
+if [ -e abiversion ]; then 
+	top=.
+else
+	top=standalone/android
+fi
+
+wrap () {
+	sed -e "s!PROG!$1!" -e "s!OPTS!$3!" < $top/wrapper.pl > "$2"
+	chmod +x "$2"
+}
+
+# location to toolchain as installed by ghc-android
+androidtoolchain="$HOME/.ghc/$(cat $top/abiversion)/bin"
+
+for f in $(find "$androidtoolchain" -maxdepth 1 -type f -printf '%f\n' | grep -v \.orig); do
+	bin="$androidtoolchain/$f"
+	orig="$androidtoolchain/$f.orig"
+	if [ ! -e "$orig" ]; then
+		cp -a "$bin" "$orig"
+	fi
+	if [ "$androidversion" = 5 ]; then
+		case "$f" in
+		*-ld*)
+			wrap "$orig" "$bin" "-pie"
+		;;
+		*-gcc)
+			wrap "$orig" "$bin" "-pie -fPIE"
+		;;
+		*'-g++')
+			wrap "$orig" "$bin" "-pie -fPIE"
+		;;
+		*)
+			cp -a "$orig" "$bin"
+		;;
+		esac
+	else
+		cp -a "$orig" "$bin"
+	fi
+done
+
+echo "$androidtoolchain:$PATH"
diff --git a/standalone/android/wrapper.pl b/standalone/android/wrapper.pl
new file mode 100644
index 0000000000..e7cb98150e
--- /dev/null
+++ b/standalone/android/wrapper.pl
@@ -0,0 +1,11 @@
+#!/usr/bin/perl
+my $prog=q{PROG}; # replaced
+my @opts=qw{OPTS}; # replaced
+
+if (grep { $_ eq "-r" || $_ eq "--relocatable" } @ARGV) {
+	exec($prog,@ARGV) || die "failed to run $prog";
+}
+else {
+	my @args=grep { ! m/-no-pie/ } @ARGV;
+	exec($prog,@opts,@args) || die "failed to run $prog";
+}
diff --git a/standalone/licences.gz b/standalone/licences.gz
new file mode 100644
index 0000000000..8ea134219b
Binary files /dev/null and b/standalone/licences.gz differ
diff --git a/standalone/linux/skel/README b/standalone/linux/skel/README
new file mode 100644
index 0000000000..2a321845f0
--- /dev/null
+++ b/standalone/linux/skel/README
@@ -0,0 +1,20 @@
+You can put this directory into your PATH, or symlink the programs in this
+directory to anyplace already in your PATH, and use git-annex the same
+as if you'd installed it using a package manager.
+
+Or, you can use the runshell script in this directory to start a shell
+that is configured to use git-annex and the other utilities included in
+this bundle, including git, gpg, rsync, ssh, etc.
+
+This should work on any Linux system of the appropriate architecture.
+More or less.
+
+
+How it works: This directory tree contains a lot of libraries and programs
+that git-annex needs. But it's not a chroot. Instead, runshell sets a lot
+of environment variables to cause files from here to be used, and a shim
+around the binaries arranges for them to be run with the libraries in here.
+
+It shouldn't even be dependent on the host system's glibc libraries.
+All that's needed is a kernel that supports the glibc included in this
+bundle.
diff --git a/standalone/linux/skel/git b/standalone/linux/skel/git
new file mode 100755
index 0000000000..8a6860f43a
--- /dev/null
+++ b/standalone/linux/skel/git
@@ -0,0 +1,24 @@
+#!/bin/sh
+link="$(readlink -f "$0" 2>/dev/null || readlink "$0")" || true
+if [ -n "$link" ]; then
+	base="$(dirname "$link")"
+else
+	base="$(dirname "$0")"
+fi
+
+if [ ! -d "$base" ]; then
+	echo "** cannot find base directory (I seem to be $0)" >&2
+	exit 1
+fi
+if [ ! -e "$base/runshell" ]; then
+	echo "** cannot find $base/runshell" >&2
+	exit 1
+fi
+
+# Get absolute path to base, to avoid breakage when things change directories.
+orig="$(pwd)"
+cd "$base"
+base="$(pwd)"
+cd "$orig"
+
+exec "$base/runshell" git "$@"
diff --git a/standalone/linux/skel/git-annex b/standalone/linux/skel/git-annex
new file mode 100755
index 0000000000..ec9739a8bc
--- /dev/null
+++ b/standalone/linux/skel/git-annex
@@ -0,0 +1,24 @@
+#!/bin/sh
+link="$(readlink -f "$0" 2>/dev/null || readlink "$0")" || true
+if [ -n "$link" ]; then
+	base="$(dirname "$link")"
+else
+	base="$(dirname "$0")"
+fi
+
+if [ ! -d "$base" ]; then
+	echo "** cannot find base directory (I seem to be $0)" >&2
+	exit 1
+fi
+if [ ! -e "$base/runshell" ]; then
+	echo "** cannot find $base/runshell" >&2
+	exit 1
+fi
+
+# Get absolute path to base, to avoid breakage when things change directories.
+orig="$(pwd)"
+cd "$base"
+base="$(pwd)"
+cd "$orig"
+
+exec "$base/runshell" git-annex "$@"
diff --git a/standalone/linux/skel/git-annex-shell b/standalone/linux/skel/git-annex-shell
new file mode 100755
index 0000000000..d663384915
--- /dev/null
+++ b/standalone/linux/skel/git-annex-shell
@@ -0,0 +1,24 @@
+#!/bin/sh
+link="$(readlink -f "$0" 2>/dev/null || readlink "$0")" || true
+if [ -n "$link" ]; then
+	base="$(dirname "$link")"
+else
+	base="$(dirname "$0")"
+fi
+
+if [ ! -d "$base" ]; then
+	echo "** cannot find base directory (I seem to be $0)" >&2
+	exit 1
+fi
+if [ ! -e "$base/runshell" ]; then
+	echo "** cannot find $base/runshell" >&2
+	exit 1
+fi
+
+# Get absolute path to base, to avoid breakage when things change directories.
+orig="$(pwd)"
+cd "$base"
+base="$(pwd)"
+cd "$orig"
+
+exec "$base/runshell" git-annex-shell "$@"
diff --git a/standalone/linux/skel/git-annex-webapp b/standalone/linux/skel/git-annex-webapp
new file mode 100755
index 0000000000..914c5923c1
--- /dev/null
+++ b/standalone/linux/skel/git-annex-webapp
@@ -0,0 +1,24 @@
+#!/bin/sh
+link="$(readlink -f "$0" 2>/dev/null || readlink "$0")" || true
+if [ -n "$link" ]; then
+	base="$(dirname "$link")"
+else
+	base="$(dirname "$0")"
+fi
+
+if [ ! -d "$base" ]; then
+	echo "** cannot find base directory (I seem to be $0)" >&2
+	exit 1
+fi
+if [ ! -e "$base/runshell" ]; then
+	echo "** cannot find $base/runshell" >&2
+	exit 1
+fi
+
+# Get absolute path to base, to avoid breakage when things change directories.
+orig="$(pwd)"
+cd "$base"
+base="$(pwd)"
+cd "$orig"
+
+exec "$base/runshell" git-annex webapp "$@"
diff --git a/standalone/linux/skel/git-receive-pack b/standalone/linux/skel/git-receive-pack
new file mode 100755
index 0000000000..f4f9f262aa
--- /dev/null
+++ b/standalone/linux/skel/git-receive-pack
@@ -0,0 +1,24 @@
+#!/bin/sh
+link="$(readlink -f "$0" 2>/dev/null || readlink "$0")" || true
+if [ -n "$link" ]; then
+	base="$(dirname "$link")"
+else
+	base="$(dirname "$0")"
+fi
+
+if [ ! -d "$base" ]; then
+	echo "** cannot find base directory (I seem to be $0)" >&2
+	exit 1
+fi
+if [ ! -e "$base/runshell" ]; then
+	echo "** cannot find $base/runshell" >&2
+	exit 1
+fi
+
+# Get absolute path to base, to avoid breakage when things change directories.
+orig="$(pwd)"
+cd "$base"
+base="$(pwd)"
+cd "$orig"
+
+exec "$base/runshell" git-receive-pack "$@"
diff --git a/standalone/linux/skel/git-shell b/standalone/linux/skel/git-shell
new file mode 100755
index 0000000000..0fc5cd7bdf
--- /dev/null
+++ b/standalone/linux/skel/git-shell
@@ -0,0 +1,24 @@
+#!/bin/sh
+link="$(readlink -f "$0" 2>/dev/null || readlink "$0")" || true
+if [ -n "$link" ]; then
+	base="$(dirname "$link")"
+else
+	base="$(dirname "$0")"
+fi
+
+if [ ! -d "$base" ]; then
+	echo "** cannot find base directory (I seem to be $0)" >&2
+	exit 1
+fi
+if [ ! -e "$base/runshell" ]; then
+	echo "** cannot find $base/runshell" >&2
+	exit 1
+fi
+
+# Get absolute path to base, to avoid breakage when things change directories.
+orig="$(pwd)"
+cd "$base"
+base="$(pwd)"
+cd "$orig"
+
+exec "$base/runshell" git-shell "$@"
diff --git a/standalone/linux/skel/git-upload-pack b/standalone/linux/skel/git-upload-pack
new file mode 100755
index 0000000000..312996cec5
--- /dev/null
+++ b/standalone/linux/skel/git-upload-pack
@@ -0,0 +1,24 @@
+#!/bin/sh
+link="$(readlink -f "$0" 2>/dev/null || readlink "$0")" || true
+if [ -n "$link" ]; then
+	base="$(dirname "$link")"
+else
+	base="$(dirname "$0")"
+fi
+
+if [ ! -d "$base" ]; then
+	echo "** cannot find base directory (I seem to be $0)" >&2
+	exit 1
+fi
+if [ ! -e "$base/runshell" ]; then
+	echo "** cannot find $base/runshell" >&2
+	exit 1
+fi
+
+# Get absolute path to base, to avoid breakage when things change directories.
+orig="$(pwd)"
+cd "$base"
+base="$(pwd)"
+cd "$orig"
+
+exec "$base/runshell" git-upload-pack "$@"
diff --git a/standalone/linux/skel/runshell b/standalone/linux/skel/runshell
new file mode 100755
index 0000000000..a684e76c2b
--- /dev/null
+++ b/standalone/linux/skel/runshell
@@ -0,0 +1,236 @@
+#!/bin/sh
+# Runs a shell command (or interactive shell) using the binaries and
+# libraries bundled with this app.
+
+set -e
+
+os="$(uname -o 2>/dev/null || true)"
+base="$(dirname "$0")"
+
+if [ ! -d "$base" ]; then
+	echo "** cannot find base directory (I seem to be $0)" >&2
+	exit 1
+fi
+
+if [ ! -e "$base/bin/git-annex" ]; then
+	echo "** base directory $base does not contain bin/git-annex" >&2
+	exit 1
+fi
+
+# Get absolute path to base, to avoid breakage when things change directories.
+orig="$(pwd)"
+cd "$base"
+base="$(pwd)"
+cd "$orig"
+
+# --library-path won't work if $base contains : or ;
+# Detect this problem, and work around it by using a temp directory.
+if echo "$base" | grep -q '[:;]'; then
+	tbase=$(mktemp -d -p /tmp annexshimXXXXXXXXX 2>/dev/null || true)
+	if [ -z "$tbase" ]; then
+		tbase="/tmp/annexshim.$$"
+		mkdir "$tbase"
+	fi
+	ln -s "$base" "$tbase/link"
+	base="$tbase/link"
+	cleanuptbase () {
+		rm -rf "$tbase"
+	}
+	trap cleanuptbase EXIT
+else
+	tbase=""
+fi
+
+# Set this variable when using this script inside a package of git-annex,
+# which arranges for git-annex, git-annex-shell, and git to all be in the
+# standard PATH.
+GIT_ANNEX_PACKAGE_INSTALL=
+if [ -z "$GIT_ANNEX_PACKAGE_INSTALL" ]; then
+	# Install shim that's used to run git-annex-shell from ssh authorized
+	# keys. The assistant also does this when run, but the user may not
+	# be using the assistant.
+	if [ ! -e "$HOME/.ssh/git-annex-shell" ]; then
+		mkdir "$HOME/.ssh" >/dev/null 2>&1 || true
+		if [ -e "$HOME/.ssh" ]; then
+			(
+				echo "#!/bin/sh"
+				echo "set -e"
+				echo "if [ \"x\$SSH_ORIGINAL_COMMAND\" != \"x\" ]; then"
+				echo "exec '$base/runshell' git-annex-shell -c \"\$SSH_ORIGINAL_COMMAND\""
+				echo "else"
+				echo "exec '$base/runshell' git-annex-shell -c \"\$@\""
+				echo "fi"
+			) > "$HOME/.ssh/git-annex-shell"
+			chmod +x "$HOME/.ssh/git-annex-shell"
+		fi
+	fi
+
+	# And this shim is used by the webapp when adding a remote ssh server.
+	if [ ! -e "$HOME/.ssh/git-annex-wrapper" ]; then
+		mkdir "$HOME/.ssh" >/dev/null 2>&1 || true
+		if [ -e "$HOME/.ssh" ]; then
+			(
+				echo "#!/bin/sh"
+				echo "set -e"
+				echo "exec '$base/runshell' \"\$@\""
+			) > "$HOME/.ssh/git-annex-wrapper"
+			chmod +x "$HOME/.ssh/git-annex-wrapper"
+		fi
+	fi
+fi
+
+# Used by git-annex assistant to further install itself.
+GIT_ANNEX_APP_BASE="$base"
+export GIT_ANNEX_APP_BASE
+
+# Put our binaries first, to avoid issues with out of date or incompatible
+# system binaries. Extra binaries come after system path.
+ORIG_PATH="$PATH"
+export ORIG_PATH
+PATH="$base/bin:$PATH:$base/extra"
+export PATH
+
+# These env vars are used by the shim wrapper around each binary.
+for lib in $(cat "$base/libdirs"); do
+	GIT_ANNEX_LD_LIBRARY_PATH="$base/$lib:$GIT_ANNEX_LD_LIBRARY_PATH"
+done
+export GIT_ANNEX_LD_LIBRARY_PATH
+GIT_ANNEX_DIR="$base"
+export GIT_ANNEX_DIR
+
+ORIG_GCONV_PATH="$GCONV_PATH"
+export ORIG_GCONV_PATH
+GCONV_PATH="$base/$(cat "$base/gconvdir")"
+export GCONV_PATH
+
+ORIG_GIT_EXEC_PATH="$GIT_EXEC_PATH"
+export ORIG_GIT_EXEC_PATH
+GIT_EXEC_PATH="$base/git-core"
+export GIT_EXEC_PATH
+
+ORIG_GIT_TEMPLATE_DIR="$GIT_TEMPLATE_DIR"
+export ORIG_GIT_TEMPLATE_DIR
+GIT_TEMPLATE_DIR="$base/templates"
+export GIT_TEMPLATE_DIR
+
+ORIG_MANPATH="$MANPATH"
+export ORIG_MANPATH
+MANPATH="$base/usr/share/man:$MANPATH"
+export MANPATH
+
+# LD_PRELOAD may interact badly with the bundled libc and other libraries,
+# which may have a different subarchitecture than the preloaded library.
+unset LD_PRELOAD
+
+# Avoid using system locales, which may interact badly with bundled libc.
+# (But if LOCPATH is set, don't override it.)
+ORIG_LOCPATH="$LOCPATH"
+export ORIG_LOCPATH
+if [ -z "${LOCPATH+set}" ]; then
+	LOCPATH="$HOME/.cache/git-annex/locales/$(echo "$base" | tr / _ )"
+	export LOCPATH
+	
+	# Clean up locale caches when their standalone bundle no longer exists.
+	for localecache in $HOME/.cache/git-annex/locales/*; do
+		cachebase=$(cat "$localecache/base" 2>/dev/null || true)
+		if [ ! -d "$cachebase" ] || ! cmp "$localecache/buildid" "$cachebase/buildid" >/dev/null ; then
+			rm -rf "$localecache" 2>&1 || true
+		fi
+	done
+	
+	# If the locale cache for this bundle is out of date, refresh it.
+	if [ -e "$LOCPATH/buildid" ] && ! cmp "$LOCPATH/buildid" "$base/buildid" >/dev/null ; then
+		rm -rf "$LOCPATH"
+	fi
+	if ! mkdir -p "$LOCPATH"; then
+		echo "Unable to write to $LOCPATH; can't continue!" >&2
+		exit 1
+	fi
+	echo "$base" > "$LOCPATH/base"
+	cp "$base/buildid" "$LOCPATH/buildid"
+
+	# Generate locale definition files for the locales in use,
+	# using the localedef and locale files from the bundle.
+	# Currently only utf-8 locales are handled.
+	lastlocaleenv=""
+	for localeenv in "$LANG" "$LANGUAGE" "$LC_CTYPE" "$LC_NUMERIC" "$LC_TIME" \
+			"$LC_COLLATE" "$LC_MONETARY" "$LC_MESSAGES" "$LC_PAPER" \
+			"$LC_NAME" "$LC_ADDRESS" "$LC_TELEPHONE" "$LC_MEASUREMENT" \
+			"$LC_IDENTIFICATION" "$LC_ALL"; do
+		if [ "$localeenv" != "$lastlocaleenv" ]; then
+			lastlocaleenv="$localeenv"
+			if [ ! -d "$LOCPATH/$localeenv" ]; then
+				if [ "${localeenv##[!.]*.}" = "utf8" ] || [ "${localeenv##[!.]*.}" = "UTF-8" ]; then
+					(
+						rm -rf "$LOCPATH/$localeenv.new.$$" &&
+						mkdir -p "$LOCPATH/$localeenv.new.$$" &&
+						# cd to $base since localedef reads files from pwd
+						cd "$base" &&
+						# Run localedef using the bundled i18n files;
+						# use LANG=C to avoid it reading the system locale archive.
+						I18NPATH="$base/i18n" LANG=C localedef -i "${localeenv%%.*}" -c -f UTF-8 "$LOCPATH/$localeenv.new.$$" &&
+						mv "$LOCPATH/$localeenv.new.$$" "$LOCPATH/$localeenv"
+					) >/dev/null 2>/dev/null || true
+				fi
+			fi
+		fi
+	done
+fi
+
+useproot=""
+case "$os" in
+	# Make this bundle work well on Android.
+	Android)
+		if [ -e "$base/git" ]; then
+			echo "Running on Android.. Adding git-annex to PATH for you, and tuning for optimal behavior." >&2
+			# The bundled git does not work well on sdcard, so delete
+			# it and use termux's git which works better.
+			cd "$base"
+			find . | grep git | grep -v git-annex | grep -v git-remote-tor-annex | grep -v git-remote-gcrypt | xargs rm -rf
+			cd "$orig"
+			# Save the poor Android user the typing.
+			if ! [ -e "$HOME/.profile" ] || ! grep -q "$base" "$HOME/.profile"; then
+				echo 'PATH=$PATH:'"$base" >> $HOME/.profile
+			fi
+		fi
+		
+		# Work around Android 8's seccomp filtering of some crucial
+		# system calls, using termux's version of proot.
+		useproot=1
+		
+		# Store ssh connection caching sockets outside of sdcard.
+		GIT_ANNEX_TMP_DIR="$TMPDIR"
+		export GIT_ANNEX_TMP_DIR
+
+		GIT_ANNEX_STANDLONE_ENV="PATH GCONV_PATH MANPATH LOCPATH"
+		export GIT_ANNEX_STANDLONE_ENV
+		;;
+	*)
+		# Indicate which variables were exported above and should be cleaned
+		# when running non-bundled programs.
+		GIT_ANNEX_STANDLONE_ENV="PATH GCONV_PATH GIT_EXEC_PATH GIT_TEMPLATE_DIR MANPATH LOCPATH"
+		export GIT_ANNEX_STANDLONE_ENV
+		;;
+esac
+
+if [ "$1" ]; then
+	cmd="$1"
+	shift 1
+else
+	cmd=sh
+fi
+
+if [ -z "$tbase" ]; then
+	if [ "$useproot" ]; then
+		exec proot "$cmd" "$@"
+	else
+		exec "$cmd" "$@"
+	fi
+else
+	# allow EXIT trap to cleanup
+	if [ "$useproot" ]; then
+		proot "$cmd" "$@"
+	else
+		"$cmd" "$@"
+	fi
+fi
diff --git a/standalone/no-th/evilsplicer-headers.hs b/standalone/no-th/evilsplicer-headers.hs
new file mode 100644
index 0000000000..64076d5f7e
--- /dev/null
+++ b/standalone/no-th/evilsplicer-headers.hs
@@ -0,0 +1,50 @@
+
+
+{- This file was modified by the EvilSplicer, adding these headers,
+ - and expanding Template Haskell. 
+ -
+ - ** DO NOT COMMIT ** 
+ -}
+import qualified Data.Monoid
+import qualified Control.Applicative
+import qualified Data.Set
+import qualified Data.Set as Data.Set.Base
+import qualified Data.Map
+import qualified Data.Map as Data.Map.Base
+import qualified Data.HashMap.Strict
+import qualified Data.HashMap.Strict as Data.HashMap.Base
+import qualified Data.Foldable
+import qualified Data.Text
+import qualified Data.Text.Lazy.Builder
+import qualified Data.Text.Lazy.Builder as Data.Text.Internal.Builder
+import qualified Text.Shakespeare
+import qualified Text.Hamlet
+import qualified Text.Julius
+import qualified Text.Css
+import qualified "blaze-markup" Text.Blaze.Internal
+import qualified Yesod.Core.Widget
+import qualified Yesod.Routes.TH.Types
+import qualified Yesod.Core.Dispatch
+import qualified Yesod.Routes.Dispatch
+import qualified WaiAppStatic.Storage.Embedded
+import qualified WaiAppStatic.Storage.Embedded.Runtime
+import qualified Data.FileEmbed
+import qualified Data.ByteString.Internal
+import qualified Data.Text.Encoding
+import qualified Network.Wai
+import qualified Network.Wai as Network.Wai.Internal
+import qualified Yesod.Core.Types
+import qualified GHC.IO
+import qualified Data.ByteString.Unsafe
+import qualified Data.ByteString.Char8
+import qualified Database.Persist.Class as Database.Persist.Class.PersistField
+import qualified Database.Persist as Database.Persist.Class.PersistField
+import qualified Database.Persist.Sql as Database.Persist.Sql.Class
+import qualified Database.Persist.Sql as Database.Persist.Types.Base
+import qualified Control.Monad.Logger
+import qualified Control.Monad.IO.Class
+import qualified Control.Monad.Trans.Control
+import Database.Persist.Sql (fromPersistValue)
+{- End EvilSplicer headers. -}
+
+
diff --git a/standalone/no-th/haskell-patches/DAV_build-without-TH.patch b/standalone/no-th/haskell-patches/DAV_build-without-TH.patch
new file mode 100644
index 0000000000..6d17d634e9
--- /dev/null
+++ b/standalone/no-th/haskell-patches/DAV_build-without-TH.patch
@@ -0,0 +1,420 @@
+From e54cfacbb9fb24f75d3d93cd8ee6da67b161574f Mon Sep 17 00:00:00 2001
+From: dummy 
+Date: Thu, 16 Oct 2014 02:51:28 +0000
+Subject: [PATCH] remove TH
+
+---
+ DAV.cabal                       |  28 +----
+ Network/Protocol/HTTP/DAV.hs    |  92 +++++++++++++---
+ Network/Protocol/HTTP/DAV/TH.hs | 232 +++++++++++++++++++++++++++++++++++++++-
+ 3 files changed, 306 insertions(+), 46 deletions(-)
+
+diff --git a/DAV.cabal b/DAV.cabal
+index 95fffd8..5669c51 100644
+--- a/DAV.cabal
++++ b/DAV.cabal
+@@ -47,33 +47,7 @@ library
+                      , utf8-string
+                      , xml-conduit >= 1.0          && < 1.3
+                      , xml-hamlet >= 0.4           && < 0.5
+-executable hdav
+-  main-is:           hdav.hs
+-  ghc-options:       -Wall
+-  build-depends:       base >= 4.5                 && < 5
+-                     , bytestring
+-                     , bytestring
+-                     , case-insensitive >= 0.4
+-                     , containers
+-                     , data-default
+-                     , either >= 4.3
+-                     , errors
+-                     , exceptions
+-                     , http-client >= 0.2
+-                     , http-client-tls >= 0.2
+-                     , http-types >= 0.7
+-                     , lens >= 3.0
+-                     , mtl >= 2.1
+-                     , optparse-applicative >= 0.10.0
+-                     , transformers >= 0.3
+-                     , transformers-base
+-                     , utf8-string
+-                     , xml-conduit >= 1.0          && < 1.3
+-                     , xml-hamlet >= 0.4           && < 0.5
+-  if flag(network-uri)
+-    build-depends: network-uri >= 2.6, network >= 2.6
+-  else
+-    build-depends: network >= 2.3 && <2.6
++                     , text
+ 
+ source-repository head
+   type:     git
+diff --git a/Network/Protocol/HTTP/DAV.hs b/Network/Protocol/HTTP/DAV.hs
+index 4c6d68f..55979b6 100644
+--- a/Network/Protocol/HTTP/DAV.hs
++++ b/Network/Protocol/HTTP/DAV.hs
+@@ -82,6 +82,7 @@ import Network.HTTP.Types (hContentType, Method, Status, RequestHeaders, unautho
+ import qualified Text.XML as XML
+ import Text.XML.Cursor (($/), (&/), element, node, fromDocument, checkName)
+ import Text.Hamlet.XML (xml)
++import qualified Data.Text
+ 
+ import Data.CaseInsensitive (mk)
+ 
+@@ -330,31 +331,88 @@ withLockIfPossibleForDelete nocreate f = do
+ propname :: XML.Document
+ propname = XML.Document (XML.Prologue [] Nothing []) root []
+     where
+-        root = XML.Element "D:propfind" (Map.fromList [("xmlns:D", "DAV:")]) [xml|
+-
+-|]
++        root = XML.Element "D:propfind" (Map.fromList [("xmlns:D", "DAV:")])  $         concat
++          [[XML.NodeElement
++              (XML.Element
++                 (XML.Name
++                    (Data.Text.pack "D:allprop") Nothing Nothing)
++                 Map.empty
++                 (concat []))]]
++
+ 
+ locky :: XML.Document
+ locky = XML.Document (XML.Prologue [] Nothing []) root []
+     where
+-        root = XML.Element "D:lockinfo" (Map.fromList [("xmlns:D", "DAV:")]) [xml|
+-
+-  
+-
+-  
+-Haskell DAV user
+-|]
++        root = XML.Element "D:lockinfo" (Map.fromList [("xmlns:D", "DAV:")])  $         concat
++          [[XML.NodeElement
++              (XML.Element
++                 (XML.Name
++                    (Data.Text.pack "D:lockscope") Nothing Nothing)
++                 Map.empty
++                 (concat
++                    [[XML.NodeElement
++                        (XML.Element
++                           (XML.Name
++                              (Data.Text.pack "D:exclusive") Nothing Nothing)
++                           Map.empty
++                           (concat []))]]))],
++           [XML.NodeElement
++              (XML.Element
++                 (XML.Name
++                    (Data.Text.pack "D:locktype") Nothing Nothing)
++                 Map.empty
++                 (concat
++                    [[XML.NodeElement
++                        (XML.Element
++                           (XML.Name (Data.Text.pack "D:write") Nothing Nothing)
++                           Map.empty
++                           (concat []))]]))],
++           [XML.NodeElement
++              (XML.Element
++                 (XML.Name (Data.Text.pack "D:owner") Nothing Nothing)
++                 Map.empty
++                 (concat
++                    [[XML.NodeContent
++                        (Data.Text.pack "Haskell DAV user")]]))]]
++
+ 
+ calendarquery :: XML.Document
+ calendarquery = XML.Document (XML.Prologue [] Nothing []) root []
+     where
+-        root = XML.Element "C:calendar-query" (Map.fromList [("xmlns:D", "DAV:"),("xmlns:C", "urn:ietf:params:xml:ns:caldav")]) [xml|
+-
+-  
+-  
+-
+-  
+-|]
++        root = XML.Element "C:calendar-query" (Map.fromList [("xmlns:D", "DAV:"),("xmlns:C", "urn:ietf:params:xml:ns:caldav")])  $         concat
++          [[XML.NodeElement
++              (XML.Element
++                 (XML.Name (Data.Text.pack "D:prop") Nothing Nothing)
++                 Map.empty
++                 (concat
++                    [[XML.NodeElement
++                        (XML.Element
++                           (XML.Name
++                              (Data.Text.pack "D:getetag") Nothing Nothing)
++                           Map.empty
++                           (concat []))],
++                     [XML.NodeElement
++                        (XML.Element
++                           (XML.Name
++                              (Data.Text.pack "C:calendar-data") Nothing Nothing)
++                           Map.empty
++                           (concat []))]]))],
++           [XML.NodeElement
++              (XML.Element
++                 (XML.Name (Data.Text.pack "C:filter") Nothing Nothing)
++                 Map.empty
++                 (concat
++                    [[XML.NodeElement
++                        (XML.Element
++                           (XML.Name
++                              (Data.Text.pack "C:comp-filter") Nothing Nothing)
++                           (Map.insert
++                              (XML.Name (Data.Text.pack "name") Nothing Nothing)
++                              (Data.Text.concat
++                                 [Data.Text.pack "VCALENDAR"])
++                              Map.empty)
++                           (concat []))]]))]]
++
+ 
+ -- | Normally, DAVT actions act on the url that is provided to eg, evalDAVT.
+ -- Sometimes, it's useful to adjust the url that is acted on, while
+diff --git a/Network/Protocol/HTTP/DAV/TH.hs b/Network/Protocol/HTTP/DAV/TH.hs
+index 0ecd476..1653bf6 100644
+--- a/Network/Protocol/HTTP/DAV/TH.hs
++++ b/Network/Protocol/HTTP/DAV/TH.hs
+@@ -20,9 +20,11 @@
+ 
+ module Network.Protocol.HTTP.DAV.TH where
+ 
+-import Control.Lens (makeLenses)
++import Control.Lens
+ import qualified Data.ByteString as B
+ import Network.HTTP.Client (Manager, Request)
++import qualified Data.Functor
++import qualified Control.Lens.Type
+ 
+ data Depth = Depth0 | Depth1 | DepthInfinity
+ instance Read Depth where
+@@ -47,4 +49,230 @@ data DAVContext = DAVContext {
+   , _lockToken :: Maybe B.ByteString
+   , _userAgent :: B.ByteString
+ }
+-makeLenses ''DAVContext
++allowedMethods :: Control.Lens.Type.Lens' DAVContext [B.ByteString]
++allowedMethods
++  _f_a3iH
++  (DAVContext __allowedMethods'_a3iI
++              __baseRequest_a3iK
++              __basicusername_a3iL
++              __basicpassword_a3iM
++              __complianceClasses_a3iN
++              __depth_a3iO
++              __httpManager_a3iP
++              __lockToken_a3iQ
++              __userAgent_a3iR)
++  = ((\ __allowedMethods_a3iJ
++        -> DAVContext
++             __allowedMethods_a3iJ
++             __baseRequest_a3iK
++             __basicusername_a3iL
++             __basicpassword_a3iM
++             __complianceClasses_a3iN
++             __depth_a3iO
++             __httpManager_a3iP
++             __lockToken_a3iQ
++             __userAgent_a3iR)
++     Data.Functor.<$> (_f_a3iH __allowedMethods'_a3iI))
++{-# INLINE allowedMethods #-}
++baseRequest :: Control.Lens.Type.Lens' DAVContext Request
++baseRequest
++  _f_a3iS
++  (DAVContext __allowedMethods_a3iT
++              __baseRequest'_a3iU
++              __basicusername_a3iW
++              __basicpassword_a3iX
++              __complianceClasses_a3iY
++              __depth_a3iZ
++              __httpManager_a3j0
++              __lockToken_a3j1
++              __userAgent_a3j2)
++  = ((\ __baseRequest_a3iV
++        -> DAVContext
++             __allowedMethods_a3iT
++             __baseRequest_a3iV
++             __basicusername_a3iW
++             __basicpassword_a3iX
++             __complianceClasses_a3iY
++             __depth_a3iZ
++             __httpManager_a3j0
++             __lockToken_a3j1
++             __userAgent_a3j2)
++     Data.Functor.<$> (_f_a3iS __baseRequest'_a3iU))
++{-# INLINE baseRequest #-}
++basicpassword :: Control.Lens.Type.Lens' DAVContext B.ByteString
++basicpassword
++  _f_a3j3
++  (DAVContext __allowedMethods_a3j4
++              __baseRequest_a3j5
++              __basicusername_a3j6
++              __basicpassword'_a3j7
++              __complianceClasses_a3j9
++              __depth_a3ja
++              __httpManager_a3jb
++              __lockToken_a3jc
++              __userAgent_a3jd)
++  = ((\ __basicpassword_a3j8
++        -> DAVContext
++             __allowedMethods_a3j4
++             __baseRequest_a3j5
++             __basicusername_a3j6
++             __basicpassword_a3j8
++             __complianceClasses_a3j9
++             __depth_a3ja
++             __httpManager_a3jb
++             __lockToken_a3jc
++             __userAgent_a3jd)
++     Data.Functor.<$> (_f_a3j3 __basicpassword'_a3j7))
++{-# INLINE basicpassword #-}
++basicusername :: Control.Lens.Type.Lens' DAVContext B.ByteString
++basicusername
++  _f_a3je
++  (DAVContext __allowedMethods_a3jf
++              __baseRequest_a3jg
++              __basicusername'_a3jh
++              __basicpassword_a3jj
++              __complianceClasses_a3jk
++              __depth_a3jl
++              __httpManager_a3jm
++              __lockToken_a3jn
++              __userAgent_a3jo)
++  = ((\ __basicusername_a3ji
++        -> DAVContext
++             __allowedMethods_a3jf
++             __baseRequest_a3jg
++             __basicusername_a3ji
++             __basicpassword_a3jj
++             __complianceClasses_a3jk
++             __depth_a3jl
++             __httpManager_a3jm
++             __lockToken_a3jn
++             __userAgent_a3jo)
++     Data.Functor.<$> (_f_a3je __basicusername'_a3jh))
++{-# INLINE basicusername #-}
++complianceClasses ::
++  Control.Lens.Type.Lens' DAVContext [B.ByteString]
++complianceClasses
++  _f_a3jp
++  (DAVContext __allowedMethods_a3jq
++              __baseRequest_a3jr
++              __basicusername_a3js
++              __basicpassword_a3jt
++              __complianceClasses'_a3ju
++              __depth_a3jw
++              __httpManager_a3jx
++              __lockToken_a3jy
++              __userAgent_a3jz)
++  = ((\ __complianceClasses_a3jv
++        -> DAVContext
++             __allowedMethods_a3jq
++             __baseRequest_a3jr
++             __basicusername_a3js
++             __basicpassword_a3jt
++             __complianceClasses_a3jv
++             __depth_a3jw
++             __httpManager_a3jx
++             __lockToken_a3jy
++             __userAgent_a3jz)
++     Data.Functor.<$> (_f_a3jp __complianceClasses'_a3ju))
++{-# INLINE complianceClasses #-}
++depth :: Control.Lens.Type.Lens' DAVContext (Maybe Depth)
++depth
++  _f_a3jA
++  (DAVContext __allowedMethods_a3jB
++              __baseRequest_a3jC
++              __basicusername_a3jD
++              __basicpassword_a3jE
++              __complianceClasses_a3jF
++              __depth'_a3jG
++              __httpManager_a3jI
++              __lockToken_a3jJ
++              __userAgent_a3jK)
++  = ((\ __depth_a3jH
++        -> DAVContext
++             __allowedMethods_a3jB
++             __baseRequest_a3jC
++             __basicusername_a3jD
++             __basicpassword_a3jE
++             __complianceClasses_a3jF
++             __depth_a3jH
++             __httpManager_a3jI
++             __lockToken_a3jJ
++             __userAgent_a3jK)
++     Data.Functor.<$> (_f_a3jA __depth'_a3jG))
++{-# INLINE depth #-}
++httpManager :: Control.Lens.Type.Lens' DAVContext (Maybe Manager)
++httpManager
++  _f_a3jL
++  (DAVContext __allowedMethods_a3jM
++              __baseRequest_a3jN
++              __basicusername_a3jO
++              __basicpassword_a3jP
++              __complianceClasses_a3jQ
++              __depth_a3jR
++              __httpManager'_a3jS
++              __lockToken_a3jU
++              __userAgent_a3jV)
++  = ((\ __httpManager_a3jT
++        -> DAVContext
++             __allowedMethods_a3jM
++             __baseRequest_a3jN
++             __basicusername_a3jO
++             __basicpassword_a3jP
++             __complianceClasses_a3jQ
++             __depth_a3jR
++             __httpManager_a3jT
++             __lockToken_a3jU
++             __userAgent_a3jV)
++     Data.Functor.<$> (_f_a3jL __httpManager'_a3jS))
++{-# INLINE httpManager #-}
++lockToken ::
++  Control.Lens.Type.Lens' DAVContext (Maybe B.ByteString)
++lockToken
++  _f_a3jW
++  (DAVContext __allowedMethods_a3jX
++              __baseRequest_a3jY
++              __basicusername_a3jZ
++              __basicpassword_a3k0
++              __complianceClasses_a3k1
++              __depth_a3k2
++              __httpManager_a3k3
++              __lockToken'_a3k4
++              __userAgent_a3k6)
++  = ((\ __lockToken_a3k5
++        -> DAVContext
++             __allowedMethods_a3jX
++             __baseRequest_a3jY
++             __basicusername_a3jZ
++             __basicpassword_a3k0
++             __complianceClasses_a3k1
++             __depth_a3k2
++             __httpManager_a3k3
++             __lockToken_a3k5
++             __userAgent_a3k6)
++     Data.Functor.<$> (_f_a3jW __lockToken'_a3k4))
++{-# INLINE lockToken #-}
++userAgent :: Control.Lens.Type.Lens' DAVContext B.ByteString
++userAgent
++  _f_a3k7
++  (DAVContext __allowedMethods_a3k8
++              __baseRequest_a3k9
++              __basicusername_a3ka
++              __basicpassword_a3kb
++              __complianceClasses_a3kc
++              __depth_a3kd
++              __httpManager_a3ke
++              __lockToken_a3kf
++              __userAgent'_a3kg)
++  = ((\ __userAgent_a3kh
++        -> DAVContext
++             __allowedMethods_a3k8
++             __baseRequest_a3k9
++             __basicusername_a3ka
++             __basicpassword_a3kb
++             __complianceClasses_a3kc
++             __depth_a3kd
++             __httpManager_a3ke
++             __lockToken_a3kf
++             __userAgent_a3kh)
++     Data.Functor.<$> (_f_a3k7 __userAgent'_a3kg))
++{-# INLINE userAgent #-}
+-- 
+2.1.1
+
diff --git a/standalone/no-th/haskell-patches/aeson_remove-TH.patch b/standalone/no-th/haskell-patches/aeson_remove-TH.patch
new file mode 100644
index 0000000000..dc40de79ed
--- /dev/null
+++ b/standalone/no-th/haskell-patches/aeson_remove-TH.patch
@@ -0,0 +1,40 @@
+From f147ec9aeaa03ca6e30232c84c413ef29b95fb62 Mon Sep 17 00:00:00 2001
+From: Your Name 
+Date: Tue, 20 May 2014 19:53:55 +0000
+Subject: [PATCH] avoid TH
+
+---
+ aeson.cabal | 3 ---
+ 1 file changed, 3 deletions(-)
+
+diff --git a/aeson.cabal b/aeson.cabal
+index 493d625..02dc6f4 100644
+--- a/aeson.cabal
++++ b/aeson.cabal
+@@ -88,7 +88,6 @@ library
+     Data.Aeson.Generic
+     Data.Aeson.Parser
+     Data.Aeson.Types
+-    Data.Aeson.TH
+ 
+   other-modules:
+     Data.Aeson.Functions
+@@ -121,7 +120,6 @@ library
+     old-locale,
+     scientific >= 0.3.1 && < 0.4,
+     syb,
+-    template-haskell >= 2.4,
+     time,
+     unordered-containers >= 0.2.3.0,
+     vector >= 0.7.1
+@@ -164,7 +162,6 @@ test-suite tests
+     base,
+     containers,
+     bytestring,
+-    template-haskell,
+     test-framework,
+     test-framework-quickcheck2,
+     test-framework-hunit,
+-- 
+2.0.0.rc2
+
diff --git a/standalone/no-th/haskell-patches/file-embed_remove-TH.patch b/standalone/no-th/haskell-patches/file-embed_remove-TH.patch
new file mode 100644
index 0000000000..12e344504c
--- /dev/null
+++ b/standalone/no-th/haskell-patches/file-embed_remove-TH.patch
@@ -0,0 +1,132 @@
+From 497d09a91f9eb1e5979948cd128078491b0e8bca Mon Sep 17 00:00:00 2001
+From: Joey Hess 
+Date: Fri, 12 Sep 2014 20:52:08 -0400
+Subject: [PATCH] remove TH
+
+---
+ Data/FileEmbed.hs | 87 ++++---------------------------------------------------
+ 1 file changed, 5 insertions(+), 82 deletions(-)
+
+diff --git a/Data/FileEmbed.hs b/Data/FileEmbed.hs
+index 5617493..adacdba 100644
+--- a/Data/FileEmbed.hs
++++ b/Data/FileEmbed.hs
+@@ -17,13 +17,13 @@
+ -- > {-# LANGUAGE TemplateHaskell #-}
+ module Data.FileEmbed
+     ( -- * Embed at compile time
+-      embedFile
+-    , embedOneFileOf
+-    , embedDir
+-    , getDir
++    --  embedFile
++    --, embedOneFileOf
++    --, embedDir
++      getDir
+       -- * Inject into an executable
+ #if MIN_VERSION_template_haskell(2,5,0)
+-    , dummySpace
++    --, dummySpace
+ #endif
+     , inject
+     , injectFile
+@@ -56,73 +56,12 @@ import Data.ByteString.Unsafe (unsafePackAddressLen)
+ import System.IO.Unsafe (unsafePerformIO)
+ import System.FilePath (())
+ 
+--- | Embed a single file in your source code.
+---
+--- > import qualified Data.ByteString
+--- >
+--- > myFile :: Data.ByteString.ByteString
+--- > myFile = $(embedFile "dirName/fileName")
+-embedFile :: FilePath -> Q Exp
+-embedFile fp =
+-#if MIN_VERSION_template_haskell(2,7,0)
+-    qAddDependentFile fp >>
+-#endif
+-  (runIO $ B.readFile fp) >>= bsToExp
+-
+--- | Embed a single existing file in your source code
+--- out of list a list of paths supplied.
+---
+--- > import qualified Data.ByteString
+--- >
+--- > myFile :: Data.ByteString.ByteString
+--- > myFile = $(embedFile' [ "dirName/fileName", "src/dirName/fileName" ])
+-embedOneFileOf :: [FilePath] -> Q Exp
+-embedOneFileOf ps =
+-  (runIO $ readExistingFile ps) >>= \ ( path, content ) -> do
+-#if MIN_VERSION_template_haskell(2,7,0)
+-    qAddDependentFile path
+-#endif
+-    bsToExp content
+-  where
+-    readExistingFile :: [FilePath] -> IO ( FilePath, B.ByteString )
+-    readExistingFile xs = do
+-      ys <- filterM doesFileExist xs
+-      case ys of
+-        (p:_) -> B.readFile p >>= \ c -> return ( p, c )
+-        _ -> throw $ ErrorCall "Cannot find file to embed as resource"
+-
+--- | Embed a directory recursively in your source code.
+---
+--- > import qualified Data.ByteString
+--- >
+--- > myDir :: [(FilePath, Data.ByteString.ByteString)]
+--- > myDir = $(embedDir "dirName")
+-embedDir :: FilePath -> Q Exp
+-embedDir fp = do
+-    typ <- [t| [(FilePath, B.ByteString)] |]
+-    e <- ListE <$> ((runIO $ fileList fp) >>= mapM (pairToExp fp))
+-    return $ SigE e typ
+-
+ -- | Get a directory tree in the IO monad.
+ --
+ -- This is the workhorse of 'embedDir'
+ getDir :: FilePath -> IO [(FilePath, B.ByteString)]
+ getDir = fileList
+ 
+-pairToExp :: FilePath -> (FilePath, B.ByteString) -> Q Exp
+-pairToExp _root (path, bs) = do
+-#if MIN_VERSION_template_haskell(2,7,0)
+-    qAddDependentFile $ _root ++ '/' : path
+-#endif
+-    exp' <- bsToExp bs
+-    return $! TupE [LitE $ StringL path, exp']
+-
+-bsToExp :: B.ByteString -> Q Exp
+-bsToExp bs = do
+-    helper <- [| stringToBs |]
+-    let chars = B8.unpack bs
+-    return $! AppE helper $! LitE $! StringL chars
+-
+ stringToBs :: String -> B.ByteString
+ stringToBs = B8.pack
+ 
+@@ -164,22 +103,6 @@ padSize i =
+     let s = show i
+      in replicate (sizeLen - length s) '0' ++ s
+ 
+-#if MIN_VERSION_template_haskell(2,5,0)
+-dummySpace :: Int -> Q Exp
+-dummySpace space = do
+-    let size = padSize space
+-    let start = magic ++ size
+-    let chars = LitE $ StringPrimL $
+-#if MIN_VERSION_template_haskell(2,6,0)
+-            map (toEnum . fromEnum) $
+-#endif
+-            start ++ replicate space '0'
+-    let len = LitE $ IntegerL $ fromIntegral $ length start + space
+-    upi <- [|unsafePerformIO|]
+-    pack <- [|unsafePackAddressLen|]
+-    getInner' <- [|getInner|]
+-    return $ getInner' `AppE` (upi `AppE` (pack `AppE` len `AppE` chars))
+-#endif
+ 
+ inject :: B.ByteString -- ^ bs to inject
+        -> B.ByteString -- ^ original BS containing dummy
+-- 
+2.1.0
+
diff --git a/standalone/no-th/haskell-patches/generic-deriving_remove-TH.patch b/standalone/no-th/haskell-patches/generic-deriving_remove-TH.patch
new file mode 100644
index 0000000000..83c8ffd2a7
--- /dev/null
+++ b/standalone/no-th/haskell-patches/generic-deriving_remove-TH.patch
@@ -0,0 +1,394 @@
+From 9a41401d903f160e11d56fff35c24eb59d97885d Mon Sep 17 00:00:00 2001
+From: Joey Hess 
+Date: Tue, 17 Dec 2013 19:04:40 +0000
+Subject: [PATCH] remove TH
+
+---
+ src/Generics/Deriving/TH.hs | 354 --------------------------------------------
+ 1 file changed, 354 deletions(-)
+
+diff --git a/src/Generics/Deriving/TH.hs b/src/Generics/Deriving/TH.hs
+index 783cb65..9aab713 100644
+--- a/src/Generics/Deriving/TH.hs
++++ b/src/Generics/Deriving/TH.hs
+@@ -19,18 +19,6 @@
+ 
+ -- Adapted from Generics.Regular.TH
+ module Generics.Deriving.TH (
+-      
+-      deriveMeta
+-    , deriveData
+-    , deriveConstructors
+-    , deriveSelectors
+-
+-#if __GLASGOW_HASKELL__ < 701
+-    , deriveAll
+-    , deriveRepresentable0
+-    , deriveRep0
+-    , simplInstance
+-#endif
+   ) where
+ 
+ import Generics.Deriving.Base
+@@ -41,124 +29,6 @@ import Language.Haskell.TH.Syntax (Lift(..))
+ import Data.List (intercalate)
+ import Control.Monad
+ 
+--- | Given the names of a generic class, a type to instantiate, a function in
+--- the class and the default implementation, generates the code for a basic
+--- generic instance.
+-simplInstance :: Name -> Name -> Name -> Name -> Q [Dec]
+-simplInstance cl ty fn df = do
+-  i <- reify (genRepName 0 ty)
+-  x <- newName "x"
+-  let typ = ForallT [PlainTV x] [] 
+-        ((foldl (\a -> AppT a . VarT . tyVarBndrToName) (ConT (genRepName 0 ty)) 
+-          (typeVariables i)) `AppT` (VarT x))
+-  fmap (: []) $ instanceD (cxt []) (conT cl `appT` conT ty)
+-    [funD fn [clause [] (normalB (varE df `appE` 
+-      (sigE (global 'undefined) (return typ)))) []]]
+-
+-
+--- | Given the type and the name (as string) for the type to derive,
+--- generate the 'Data' instance, the 'Constructor' instances, the 'Selector'
+--- instances, and the 'Representable0' instance.
+-deriveAll :: Name -> Q [Dec]
+-deriveAll n =
+-  do a <- deriveMeta n
+-     b <- deriveRepresentable0 n
+-     return (a ++ b)
+-
+--- | Given the type and the name (as string) for the type to derive,
+--- generate the 'Data' instance, the 'Constructor' instances, and the 'Selector'
+--- instances.
+-deriveMeta :: Name -> Q [Dec]
+-deriveMeta n =
+-  do a <- deriveData n
+-     b <- deriveConstructors n
+-     c <- deriveSelectors n
+-     return (a ++ b ++ c)
+-
+--- | Given a datatype name, derive a datatype and instance of class 'Datatype'.
+-deriveData :: Name -> Q [Dec]
+-deriveData = dataInstance
+-
+--- | Given a datatype name, derive datatypes and 
+--- instances of class 'Constructor'.
+-deriveConstructors :: Name -> Q [Dec]
+-deriveConstructors = constrInstance
+-
+--- | Given a datatype name, derive datatypes and instances of class 'Selector'.
+-deriveSelectors :: Name -> Q [Dec]
+-deriveSelectors = selectInstance
+-
+--- | Given the type and the name (as string) for the Representable0 type
+--- synonym to derive, generate the 'Representable0' instance.
+-deriveRepresentable0 :: Name -> Q [Dec]
+-deriveRepresentable0 n = do
+-    rep0 <- deriveRep0 n
+-    inst <- deriveInst n
+-    return $ rep0 ++ inst
+-
+--- | Derive only the 'Rep0' type synonym. Not needed if 'deriveRepresentable0'
+--- is used.
+-deriveRep0 :: Name -> Q [Dec]
+-deriveRep0 n = do
+-  i <- reify n
+-  fmap (:[]) $ tySynD (genRepName 0 n) (typeVariables i) (rep0Type n)
+-
+-deriveInst :: Name -> Q [Dec]
+-deriveInst t = do
+-  i <- reify t
+-  let typ q = foldl (\a -> AppT a . VarT . tyVarBndrToName) (ConT q) 
+-                (typeVariables i)
+-#if __GLASGOW_HASKELL__ >= 707
+-  let tyIns = TySynInstD ''Rep (TySynEqn [typ t] (typ (genRepName 0 t)))
+-#else
+-  let tyIns = TySynInstD ''Rep [typ t] (typ (genRepName 0 t))
+-#endif
+-  fcs <- mkFrom t 1 0 t
+-  tcs <- mkTo   t 1 0 t
+-  liftM (:[]) $
+-    instanceD (cxt []) (conT ''Generic `appT` return (typ t))
+-                         [return tyIns, funD 'from fcs, funD 'to tcs]
+-
+-
+-dataInstance :: Name -> Q [Dec]
+-dataInstance n = do
+-  i <- reify n
+-  case i of
+-    TyConI (DataD    _ n _ _ _) -> mkInstance n
+-    TyConI (NewtypeD _ n _ _ _) -> mkInstance n
+-    _ -> return []
+-  where
+-    mkInstance n = do
+-      ds <- mkDataData n
+-      is <- mkDataInstance n
+-      return $ [ds,is]
+-
+-constrInstance :: Name -> Q [Dec]
+-constrInstance n = do
+-  i <- reify n
+-  case i of
+-    TyConI (DataD    _ n _ cs _) -> mkInstance n cs
+-    TyConI (NewtypeD _ n _ c  _) -> mkInstance n [c]
+-    _ -> return []
+-  where
+-    mkInstance n cs = do
+-      ds <- mapM (mkConstrData n) cs
+-      is <- mapM (mkConstrInstance n) cs
+-      return $ ds ++ is
+-
+-selectInstance :: Name -> Q [Dec]
+-selectInstance n = do
+-  i <- reify n
+-  case i of
+-    TyConI (DataD    _ n _ cs _) -> mkInstance n cs
+-    TyConI (NewtypeD _ n _ c  _) -> mkInstance n [c]
+-    _ -> return []
+-  where
+-    mkInstance n cs = do
+-      ds <- mapM (mkSelectData n) cs
+-      is <- mapM (mkSelectInstance n) cs
+-      return $ concat (ds ++ is)
+-
+ typeVariables :: Info -> [TyVarBndr]
+ typeVariables (TyConI (DataD    _ _ tv _ _)) = tv
+ typeVariables (TyConI (NewtypeD _ _ tv _ _)) = tv
+@@ -179,233 +49,9 @@ genName = mkName . (++"_") . intercalate "_" . map nameBase
+ genRepName :: Int -> Name -> Name
+ genRepName n = mkName . (++"_") . (("Rep" ++ show n) ++) . nameBase
+ 
+-mkDataData :: Name -> Q Dec
+-mkDataData n = dataD (cxt []) (genName [n]) [] [] []
+-
+-mkConstrData :: Name -> Con -> Q Dec
+-mkConstrData dt (NormalC n _) =
+-  dataD (cxt []) (genName [dt, n]) [] [] [] 
+-mkConstrData dt r@(RecC _ _) =
+-  mkConstrData dt (stripRecordNames r)
+-mkConstrData dt (InfixC t1 n t2) =
+-  mkConstrData dt (NormalC n [t1,t2])
+-
+-mkSelectData :: Name -> Con -> Q [Dec]
+-mkSelectData dt r@(RecC n fs) = return (map one fs)
+-  where one (f, _, _) = DataD [] (genName [dt, n, f]) [] [] []
+-mkSelectData dt _ = return []
+-
+-
+-mkDataInstance :: Name -> Q Dec
+-mkDataInstance n =
+-  instanceD (cxt []) (appT (conT ''Datatype) (conT $ genName [n]))
+-    [funD 'datatypeName [clause [wildP] (normalB (stringE (nameBase n))) []]
+-    ,funD 'moduleName   [clause [wildP] (normalB (stringE name)) []]]
+-  where
+-    name = maybe (error "Cannot fetch module name!") id (nameModule n)
+-
+-instance Lift Fixity where
+-  lift Prefix      = conE 'Prefix
+-  lift (Infix a n) = conE 'Infix `appE` [| a |] `appE` [| n |]
+-
+-instance Lift Associativity where
+-  lift LeftAssociative  = conE 'LeftAssociative
+-  lift RightAssociative = conE 'RightAssociative
+-  lift NotAssociative   = conE 'NotAssociative
+-
+-mkConstrInstance :: Name -> Con -> Q Dec
+-mkConstrInstance dt (NormalC n _) = mkConstrInstanceWith dt n []
+-mkConstrInstance dt (RecC    n _) = mkConstrInstanceWith dt n
+-      [ funD 'conIsRecord [clause [wildP] (normalB (conE 'True)) []]]
+-mkConstrInstance dt (InfixC t1 n t2) =
+-    do
+-      i <- reify n
+-      let fi = case i of
+-                 DataConI _ _ _ f -> convertFixity f
+-                 _ -> Prefix
+-      instanceD (cxt []) (appT (conT ''Constructor) (conT $ genName [dt, n]))
+-        [funD 'conName   [clause [wildP] (normalB (stringE (nameBase n))) []],
+-         funD 'conFixity [clause [wildP] (normalB [| fi |]) []]]
+-  where
+-    convertFixity (Fixity n d) = Infix (convertDirection d) n
+-    convertDirection InfixL = LeftAssociative
+-    convertDirection InfixR = RightAssociative
+-    convertDirection InfixN = NotAssociative
+-
+-mkConstrInstanceWith :: Name -> Name -> [Q Dec] -> Q Dec
+-mkConstrInstanceWith dt n extra = 
+-  instanceD (cxt []) (appT (conT ''Constructor) (conT $ genName [dt, n]))
+-    (funD 'conName [clause [wildP] (normalB (stringE (nameBase n))) []] : extra)
+-
+-mkSelectInstance :: Name -> Con -> Q [Dec]
+-mkSelectInstance dt r@(RecC n fs) = return (map one fs) where
+-  one (f, _, _) = 
+-    InstanceD ([]) (AppT (ConT ''Selector) (ConT $ genName [dt, n, f]))
+-      [FunD 'selName [Clause [WildP] 
+-        (NormalB (LitE (StringL (nameBase f)))) []]]
+-mkSelectInstance _ _ = return []
+-
+-rep0Type :: Name -> Q Type
+-rep0Type n =
+-    do
+-      -- runIO $ putStrLn $ "processing " ++ show n
+-      i <- reify n
+-      let b = case i of
+-                TyConI (DataD _ dt vs cs _) ->
+-                  (conT ''D1) `appT` (conT $ genName [dt]) `appT` 
+-                    (foldr1' sum (conT ''V1) 
+-                      (map (rep0Con (dt, map tyVarBndrToName vs)) cs))
+-                TyConI (NewtypeD _ dt vs c _) ->
+-                  (conT ''D1) `appT` (conT $ genName [dt]) `appT`
+-                    (rep0Con (dt, map tyVarBndrToName vs) c)
+-                TyConI (TySynD t _ _) -> error "type synonym?" 
+-                _ -> error "unknown construct" 
+-      --appT b (conT $ mkName (nameBase n))
+-      b where
+-    sum :: Q Type -> Q Type -> Q Type
+-    sum a b = conT ''(:+:) `appT` a `appT` b
+-
+-
+-rep0Con :: (Name, [Name]) -> Con -> Q Type
+-rep0Con (dt, vs) (NormalC n []) =
+-    conT ''C1 `appT` (conT $ genName [dt, n]) `appT` 
+-     (conT ''S1 `appT` conT ''NoSelector `appT` conT ''U1)
+-rep0Con (dt, vs) (NormalC n fs) =
+-    conT ''C1 `appT` (conT $ genName [dt, n]) `appT` 
+-     (foldr1 prod (map (repField (dt, vs) . snd) fs)) where
+-    prod :: Q Type -> Q Type -> Q Type
+-    prod a b = conT ''(:*:) `appT` a `appT` b
+-rep0Con (dt, vs) r@(RecC n []) =
+-    conT ''C1 `appT` (conT $ genName [dt, n]) `appT` conT ''U1
+-rep0Con (dt, vs) r@(RecC n fs) =
+-    conT ''C1 `appT` (conT $ genName [dt, n]) `appT` 
+-      (foldr1 prod (map (repField' (dt, vs) n) fs)) where
+-    prod :: Q Type -> Q Type -> Q Type
+-    prod a b = conT ''(:*:) `appT` a `appT` b
+-
+-rep0Con d (InfixC t1 n t2) = rep0Con d (NormalC n [t1,t2])
+-
+---dataDeclToType :: (Name, [Name]) -> Type
+---dataDeclToType (dt, vs) = foldl (\a b -> AppT a (VarT b)) (ConT dt) vs
+-
+-repField :: (Name, [Name]) -> Type -> Q Type
+---repField d t | t == dataDeclToType d = conT ''I
+-repField d t = conT ''S1 `appT` conT ''NoSelector `appT`
+-                 (conT ''Rec0 `appT` return t)
+-
+-repField' :: (Name, [Name]) -> Name -> (Name, Strict, Type) -> Q Type
+---repField' d ns (_, _, t) | t == dataDeclToType d = conT ''I
+-repField' (dt, vs) ns (f, _, t) = conT ''S1 `appT` conT (genName [dt, ns, f]) 
+-                                    `appT` (conT ''Rec0 `appT` return t)
+--- Note: we should generate Par0 too, at some point
+-
+-
+-mkFrom :: Name -> Int -> Int -> Name -> Q [Q Clause]
+-mkFrom ns m i n =
+-    do
+-      -- runIO $ putStrLn $ "processing " ++ show n
+-      let wrapE e = lrE m i e
+-      i <- reify n
+-      let b = case i of
+-                TyConI (DataD _ dt vs cs _) ->
+-                  zipWith (fromCon wrapE ns (dt, map tyVarBndrToName vs)
+-                    (length cs)) [0..] cs
+-                TyConI (NewtypeD _ dt vs c _) ->
+-                  [fromCon wrapE ns (dt, map tyVarBndrToName vs) 1 0 c]
+-                TyConI (TySynD t _ _) -> error "type synonym?" 
+-                  -- [clause [varP (field 0)] (normalB (wrapE $ conE 'K1 `appE` varE (field 0))) []]
+-                _ -> error "unknown construct"
+-      return b
+-
+-mkTo :: Name -> Int -> Int -> Name -> Q [Q Clause]
+-mkTo ns m i n =
+-    do
+-      -- runIO $ putStrLn $ "processing " ++ show n
+-      let wrapP p = lrP m i p
+-      i <- reify n
+-      let b = case i of
+-                TyConI (DataD _ dt vs cs _) ->
+-                  zipWith (toCon wrapP ns (dt, map tyVarBndrToName vs)
+-                    (length cs)) [0..] cs
+-                TyConI (NewtypeD _ dt vs c _) ->
+-                  [toCon wrapP ns (dt, map tyVarBndrToName vs) 1 0 c]
+-                TyConI (TySynD t _ _) -> error "type synonym?" 
+-                  -- [clause [wrapP $ conP 'K1 [varP (field 0)]] (normalB $ varE (field 0)) []]
+-                _ -> error "unknown construct" 
+-      return b
+-
+-fromCon :: (Q Exp -> Q Exp) -> Name -> (Name, [Name]) -> Int -> Int -> Con -> Q Clause
+-fromCon wrap ns (dt, vs) m i (NormalC cn []) =
+-  clause
+-    [conP cn []]
+-    (normalB $ appE (conE 'M1) $ wrap $ lrE m i $ appE (conE 'M1) $ 
+-      conE 'M1 `appE` (conE 'U1)) []
+-fromCon wrap ns (dt, vs) m i (NormalC cn fs) =
+-  -- runIO (putStrLn ("constructor " ++ show ix)) >>
+-  clause
+-    [conP cn (map (varP . field) [0..length fs - 1])]
+-    (normalB $ appE (conE 'M1) $ wrap $ lrE m i $ conE 'M1 `appE` 
+-      foldr1 prod (zipWith (fromField (dt, vs)) [0..] (map snd fs))) []
+-  where prod x y = conE '(:*:) `appE` x `appE` y
+-fromCon wrap ns (dt, vs) m i r@(RecC cn []) =
+-  clause
+-    [conP cn []]
+-    (normalB $ appE (conE 'M1) $ wrap $ lrE m i $ conE 'M1 `appE` (conE 'U1)) []
+-fromCon wrap ns (dt, vs) m i r@(RecC cn fs) =
+-  clause
+-    [conP cn (map (varP . field) [0..length fs - 1])]
+-    (normalB $ appE (conE 'M1) $ wrap $ lrE m i $ conE 'M1 `appE` 
+-      foldr1 prod (zipWith (fromField (dt, vs)) [0..] (map trd fs))) []
+-  where prod x y = conE '(:*:) `appE` x `appE` y
+-fromCon wrap ns (dt, vs) m i (InfixC t1 cn t2) =
+-  fromCon wrap ns (dt, vs) m i (NormalC cn [t1,t2])
+-
+-fromField :: (Name, [Name]) -> Int -> Type -> Q Exp
+---fromField (dt, vs) nr t | t == dataDeclToType (dt, vs) = conE 'I `appE` varE (field nr)
+-fromField (dt, vs) nr t = conE 'M1 `appE` (conE 'K1 `appE` varE (field nr))
+-
+-toCon :: (Q Pat -> Q Pat) -> Name -> (Name, [Name]) -> Int -> Int -> Con -> Q Clause
+-toCon wrap ns (dt, vs) m i (NormalC cn []) =
+-    clause
+-      [wrap $ conP 'M1 [lrP m i $ conP 'M1 [conP 'M1 [conP 'U1 []]]]]
+-      (normalB $ conE cn) []
+-toCon wrap ns (dt, vs) m i (NormalC cn fs) =
+-    -- runIO (putStrLn ("constructor " ++ show ix)) >>
+-    clause
+-      [wrap $ conP 'M1 [lrP m i $ conP 'M1
+-        [foldr1 prod (zipWith (toField (dt, vs)) [0..] (map snd fs))]]]
+-      (normalB $ foldl appE (conE cn) (map (varE . field) [0..length fs - 1])) []
+-  where prod x y = conP '(:*:) [x,y]
+-toCon wrap ns (dt, vs) m i r@(RecC cn []) =
+-    clause
+-      [wrap $ conP 'M1 [lrP m i $ conP 'M1 [conP 'U1 []]]]
+-      (normalB $ conE cn) []
+-toCon wrap ns (dt, vs) m i r@(RecC cn fs) =
+-    clause
+-      [wrap $ conP 'M1 [lrP m i $ conP 'M1
+-        [foldr1 prod (zipWith (toField (dt, vs)) [0..] (map trd fs))]]]
+-      (normalB $ foldl appE (conE cn) (map (varE . field) [0..length fs - 1])) []
+-  where prod x y = conP '(:*:) [x,y]
+-toCon wrap ns (dt, vs) m i (InfixC t1 cn t2) =
+-  toCon wrap ns (dt, vs) m i (NormalC cn [t1,t2])
+-
+-toField :: (Name, [Name]) -> Int -> Type -> Q Pat
+---toField (dt, vs) nr t | t == dataDeclToType (dt, vs) = conP 'I [varP (field nr)]
+-toField (dt, vs) nr t = conP 'M1 [conP 'K1 [varP (field nr)]]
+-
+-
+ field :: Int -> Name
+ field n = mkName $ "f" ++ show n
+ 
+-lrP :: Int -> Int -> (Q Pat -> Q Pat)
+-lrP 1 0 p = p
+-lrP m 0 p = conP 'L1 [p]
+-lrP m i p = conP 'R1 [lrP (m-1) (i-1) p]
+-
+-lrE :: Int -> Int -> (Q Exp -> Q Exp)
+-lrE 1 0 e = e
+-lrE m 0 e = conE 'L1 `appE` e
+-lrE m i e = conE 'R1 `appE` lrE (m-1) (i-1) e
+ 
+ trd (_,_,c) = c
+ 
+-- 
+1.8.5.1
+
diff --git a/standalone/no-th/haskell-patches/lens_no-TH.patch b/standalone/no-th/haskell-patches/lens_no-TH.patch
new file mode 100644
index 0000000000..bc453bfa1a
--- /dev/null
+++ b/standalone/no-th/haskell-patches/lens_no-TH.patch
@@ -0,0 +1,230 @@
+From 10c9ade98b3ac2054947f411d77db2eb28896b9f Mon Sep 17 00:00:00 2001
+From: dummy 
+Date: Thu, 16 Oct 2014 01:43:10 +0000
+Subject: [PATCH] avoid TH
+
+---
+ lens.cabal                          | 17 +----------------
+ src/Control/Lens.hs                 |  8 ++------
+ src/Control/Lens/Cons.hs            |  2 --
+ src/Control/Lens/Internal/Fold.hs   |  2 --
+ src/Control/Lens/Operators.hs       |  2 +-
+ src/Control/Lens/Prism.hs           |  2 --
+ src/Control/Monad/Primitive/Lens.hs |  1 -
+ 7 files changed, 4 insertions(+), 30 deletions(-)
+
+diff --git a/lens.cabal b/lens.cabal
+index 5388301..d7b02b9 100644
+--- a/lens.cabal
++++ b/lens.cabal
+@@ -10,7 +10,7 @@ stability:     provisional
+ homepage:      http://github.com/ekmett/lens/
+ bug-reports:   http://github.com/ekmett/lens/issues
+ copyright:     Copyright (C) 2012-2014 Edward A. Kmett
+-build-type:    Custom
++build-type:    Simple
+ -- build-tools:   cpphs
+ tested-with:   GHC == 7.4.1, GHC == 7.4.2, GHC == 7.6.3, GHC == 7.8.1, GHC == 7.8.2
+ synopsis:      Lenses, Folds and Traversals
+@@ -217,7 +217,6 @@ library
+     Control.Exception.Lens
+     Control.Lens
+     Control.Lens.Action
+-    Control.Lens.At
+     Control.Lens.Combinators
+     Control.Lens.Cons
+     Control.Lens.Each
+@@ -234,8 +233,6 @@ library
+     Control.Lens.Internal.Context
+     Control.Lens.Internal.Deque
+     Control.Lens.Internal.Exception
+-    Control.Lens.Internal.FieldTH
+-    Control.Lens.Internal.PrismTH
+     Control.Lens.Internal.Fold
+     Control.Lens.Internal.Getter
+     Control.Lens.Internal.Indexed
+@@ -247,25 +244,21 @@ library
+     Control.Lens.Internal.Reflection
+     Control.Lens.Internal.Review
+     Control.Lens.Internal.Setter
+-    Control.Lens.Internal.TH
+     Control.Lens.Internal.Zoom
+     Control.Lens.Iso
+     Control.Lens.Lens
+     Control.Lens.Level
+     Control.Lens.Loupe
+     Control.Lens.Operators
+-    Control.Lens.Plated
+     Control.Lens.Prism
+     Control.Lens.Reified
+     Control.Lens.Review
+     Control.Lens.Setter
+-    Control.Lens.TH
+     Control.Lens.Traversal
+     Control.Lens.Tuple
+     Control.Lens.Type
+     Control.Lens.Wrapped
+     Control.Lens.Zoom
+-    Control.Monad.Error.Lens
+     Control.Monad.Primitive.Lens
+     Control.Parallel.Strategies.Lens
+     Control.Seq.Lens
+@@ -291,12 +284,8 @@ library
+     Data.Typeable.Lens
+     Data.Vector.Lens
+     Data.Vector.Generic.Lens
+-    Generics.Deriving.Lens
+-    GHC.Generics.Lens
+     System.Exit.Lens
+     System.FilePath.Lens
+-    System.IO.Error.Lens
+-    Language.Haskell.TH.Lens
+     Numeric.Lens
+ 
+   other-modules:
+@@ -403,7 +392,6 @@ test-suite doctests
+       deepseq,
+       doctest        >= 0.9.1,
+       filepath,
+-      generic-deriving,
+       mtl,
+       nats,
+       parallel,
+@@ -441,7 +429,6 @@ benchmark plated
+     comonad,
+     criterion,
+     deepseq,
+-    generic-deriving,
+     lens,
+     transformers
+ 
+@@ -476,7 +463,6 @@ benchmark unsafe
+     comonads-fd,
+     criterion,
+     deepseq,
+-    generic-deriving,
+     lens,
+     transformers
+ 
+@@ -493,6 +479,5 @@ benchmark zipper
+     comonads-fd,
+     criterion,
+     deepseq,
+-    generic-deriving,
+     lens,
+     transformers
+diff --git a/src/Control/Lens.hs b/src/Control/Lens.hs
+index 7e15267..433f1fc 100644
+--- a/src/Control/Lens.hs
++++ b/src/Control/Lens.hs
+@@ -41,7 +41,6 @@
+ ----------------------------------------------------------------------------
+ module Control.Lens
+   ( module Control.Lens.Action
+-  , module Control.Lens.At
+   , module Control.Lens.Cons
+   , module Control.Lens.Each
+   , module Control.Lens.Empty
+@@ -53,12 +52,11 @@ module Control.Lens
+   , module Control.Lens.Lens
+   , module Control.Lens.Level
+   , module Control.Lens.Loupe
+-  , module Control.Lens.Plated
+   , module Control.Lens.Prism
+   , module Control.Lens.Reified
+   , module Control.Lens.Review
+   , module Control.Lens.Setter
+-#ifndef DISABLE_TEMPLATE_HASKELL
++#if 0
+   , module Control.Lens.TH
+ #endif
+   , module Control.Lens.Traversal
+@@ -69,7 +67,6 @@ module Control.Lens
+   ) where
+ 
+ import Control.Lens.Action
+-import Control.Lens.At
+ import Control.Lens.Cons
+ import Control.Lens.Each
+ import Control.Lens.Empty
+@@ -81,12 +78,11 @@ import Control.Lens.Iso
+ import Control.Lens.Lens
+ import Control.Lens.Level
+ import Control.Lens.Loupe
+-import Control.Lens.Plated
+ import Control.Lens.Prism
+ import Control.Lens.Reified
+ import Control.Lens.Review
+ import Control.Lens.Setter
+-#ifndef DISABLE_TEMPLATE_HASKELL
++#if 0
+ import Control.Lens.TH
+ #endif
+ import Control.Lens.Traversal
+diff --git a/src/Control/Lens/Cons.hs b/src/Control/Lens/Cons.hs
+index a80e9c8..7d27b80 100644
+--- a/src/Control/Lens/Cons.hs
++++ b/src/Control/Lens/Cons.hs
+@@ -55,8 +55,6 @@ import           Data.Vector.Unboxed (Unbox)
+ import qualified Data.Vector.Unboxed as Unbox
+ import           Data.Word
+ 
+-{-# ANN module "HLint: ignore Eta reduce" #-}
+-
+ -- $setup
+ -- >>> :set -XNoOverloadedStrings
+ -- >>> import Control.Lens
+diff --git a/src/Control/Lens/Internal/Fold.hs b/src/Control/Lens/Internal/Fold.hs
+index ab09c6b..43aa905 100644
+--- a/src/Control/Lens/Internal/Fold.hs
++++ b/src/Control/Lens/Internal/Fold.hs
+@@ -37,8 +37,6 @@ import Data.Maybe
+ import Data.Semigroup hiding (Min, getMin, Max, getMax)
+ import Data.Reflection
+ 
+-{-# ANN module "HLint: ignore Avoid lambda" #-}
+-
+ ------------------------------------------------------------------------------
+ -- Folding
+ ------------------------------------------------------------------------------
+diff --git a/src/Control/Lens/Operators.hs b/src/Control/Lens/Operators.hs
+index 9992e63..631e8e6 100644
+--- a/src/Control/Lens/Operators.hs
++++ b/src/Control/Lens/Operators.hs
+@@ -111,7 +111,7 @@ module Control.Lens.Operators
+   , (<#~)
+   , (<#=)
+   -- * "Control.Lens.Plated"
+-  , (...)
++  --, (...)
+   -- * "Control.Lens.Review"
+   , ( # )
+   -- * "Control.Lens.Setter"
+diff --git a/src/Control/Lens/Prism.hs b/src/Control/Lens/Prism.hs
+index b75c870..c6c6596 100644
+--- a/src/Control/Lens/Prism.hs
++++ b/src/Control/Lens/Prism.hs
+@@ -61,8 +61,6 @@ import Unsafe.Coerce
+ import Data.Profunctor.Unsafe
+ #endif
+ 
+-{-# ANN module "HLint: ignore Use camelCase" #-}
+-
+ -- $setup
+ -- >>> :set -XNoOverloadedStrings
+ -- >>> import Control.Lens
+diff --git a/src/Control/Monad/Primitive/Lens.hs b/src/Control/Monad/Primitive/Lens.hs
+index ee942c6..2f37134 100644
+--- a/src/Control/Monad/Primitive/Lens.hs
++++ b/src/Control/Monad/Primitive/Lens.hs
+@@ -20,7 +20,6 @@ import Control.Lens
+ import Control.Monad.Primitive (PrimMonad(..))
+ import GHC.Prim (State#)
+ 
+-{-# ANN module "HLint: ignore Unused LANGUAGE pragma" #-}
+ 
+ prim :: (PrimMonad m) => Iso' (m a) (State# (PrimState m) -> (# State# (PrimState m), a #))
+ prim = iso internal primitive
+-- 
+2.1.1
+
diff --git a/standalone/no-th/haskell-patches/monad-logger_remove-TH.patch b/standalone/no-th/haskell-patches/monad-logger_remove-TH.patch
new file mode 100644
index 0000000000..c24fa5aa26
--- /dev/null
+++ b/standalone/no-th/haskell-patches/monad-logger_remove-TH.patch
@@ -0,0 +1,27 @@
+From 8e78a25ce0cc19e52d063f66bd4cd316462393d4 Mon Sep 17 00:00:00 2001
+From: dummy 
+Date: Thu, 6 Mar 2014 23:27:06 +0000
+Subject: [PATCH] disable th
+
+---
+ monad-logger.cabal | 4 ++--
+ 1 file changed, 2 insertions(+), 2 deletions(-)
+
+diff --git a/monad-logger.cabal b/monad-logger.cabal
+index b0aa271..cd56c0f 100644
+--- a/monad-logger.cabal
++++ b/monad-logger.cabal
+@@ -14,8 +14,8 @@ cabal-version:       >=1.8
+ 
+ flag template_haskell {
+       Description: Enable Template Haskell support
+-      Default:     True
+-      Manual:      True
++      Default:     False
++      Manual:      False
+ }
+ 
+ library
+-- 
+1.9.0
+
diff --git a/standalone/no-th/haskell-patches/optparse-applicative_remove-ANN.patch b/standalone/no-th/haskell-patches/optparse-applicative_remove-ANN.patch
new file mode 100644
index 0000000000..1bb843524e
--- /dev/null
+++ b/standalone/no-th/haskell-patches/optparse-applicative_remove-ANN.patch
@@ -0,0 +1,33 @@
+From b128590966d4946219e45e2efd88acf7a354abc2 Mon Sep 17 00:00:00 2001
+From: androidbuilder 
+Date: Tue, 14 Oct 2014 02:28:02 +0000
+Subject: [PATCH] remove ANN
+
+---
+ Options/Applicative.hs           |    2 --
+ Options/Applicative/Help/Core.hs |    2 --
+ 2 files changed, 4 deletions(-)
+
+diff --git a/Options/Applicative.hs b/Options/Applicative.hs
+index bd4129d..f412062 100644
+--- a/Options/Applicative.hs
++++ b/Options/Applicative.hs
+@@ -34,5 +34,3 @@ import Options.Applicative.Common
+ import Options.Applicative.Builder
+ import Options.Applicative.Builder.Completer
+ import Options.Applicative.Extra
+-
+-{-# ANN module "HLint: ignore Use import/export shortcut" #-}
+diff --git a/Options/Applicative/Help/Core.hs b/Options/Applicative/Help/Core.hs
+index 0a79169..3f1ce3f 100644
+--- a/Options/Applicative/Help/Core.hs
++++ b/Options/Applicative/Help/Core.hs
+@@ -139,5 +139,3 @@ parserUsage pprefs p progn = hsep
+   [ string "Usage:"
+   , string progn
+   , align (extractChunk (briefDesc pprefs p)) ]
+-
+-{-# ANN footerHelp "HLint: ignore Eta reduce" #-}
+-- 
+1.7.10.4
+
diff --git a/standalone/no-th/haskell-patches/persistent-template_stub-out.patch b/standalone/no-th/haskell-patches/persistent-template_stub-out.patch
new file mode 100644
index 0000000000..506fa1add0
--- /dev/null
+++ b/standalone/no-th/haskell-patches/persistent-template_stub-out.patch
@@ -0,0 +1,68 @@
+From b22a4d77c1262f77ce4298b53ca90a138a14ceb7 Mon Sep 17 00:00:00 2001
+From: Joey Hess 
+Date: Sun, 22 Feb 2015 15:21:19 -0400
+Subject: [PATCH] stub out TH
+
+this method avoids needing to delete the entire file contents, so patch is
+kept minimal
+---
+ Database/Persist/TH.hs      |  1 +
+ persistent-template.cabal   |  1 +
+ stub/Database/Persist/TH.hs | 21 +++++++++++++++++++++
+ 3 files changed, 23 insertions(+)
+ create mode 100644 stub/Database/Persist/TH.hs
+
+diff --git a/Database/Persist/TH.hs b/Database/Persist/TH.hs
+index 43eb3ee..2172b77 100644
+--- a/Database/Persist/TH.hs
++++ b/Database/Persist/TH.hs
+@@ -35,6 +35,7 @@ module Database.Persist.TH
+       -- * Internal
+     , packPTH
+     , lensPTH
++    , plusPlus
+     ) where
+ 
+ import Prelude hiding ((++), take, concat, splitAt)
+diff --git a/persistent-template.cabal b/persistent-template.cabal
+index 59b4149..4705d97 100644
+--- a/persistent-template.cabal
++++ b/persistent-template.cabal
+@@ -30,6 +30,7 @@ library
+     ghc-options:     -Wall
+     if impl(ghc >= 7.4)
+        cpp-options: -DGHC_7_4
++    hs-source-dirs: stub
+ 
+ test-suite test
+     type:          exitcode-stdio-1.0
+diff --git a/stub/Database/Persist/TH.hs b/stub/Database/Persist/TH.hs
+new file mode 100644
+index 0000000..dfbb874
+--- /dev/null
++++ b/stub/Database/Persist/TH.hs
+@@ -0,0 +1,21 @@
++{-# LANGUAGE RecordWildCards #-}
++{-# LANGUAGE CPP #-}
++{-# LANGUAGE OverloadedStrings #-}
++{-# LANGUAGE RankNTypes #-}
++{-# OPTIONS_GHC -fno-warn-orphans -fno-warn-missing-fields #-}
++-- | This module provides utilities for creating backends. Regular users do not
++-- need to use this module.
++module Database.Persist.TH where
++
++import Data.Text
++
++type Lens s t a b = forall f. Functor f => (a -> f b) -> s -> f t
++
++lensPTH :: (s -> a) -> (s -> b -> t) -> Lens s t a b
++lensPTH sa sbt afb s = fmap (sbt s) (afb $ sa s)
++
++packPTH :: String -> Text
++packPTH = pack
++#if !MIN_VERSION_text(0, 11, 2)
++{-# NOINLINE packPTH #-}
++#endif
+-- 
+2.1.4
+
diff --git a/standalone/no-th/haskell-patches/persistent_1.1.5.1_0001-disable-TH.patch b/standalone/no-th/haskell-patches/persistent_1.1.5.1_0001-disable-TH.patch
new file mode 100644
index 0000000000..cd86ccd2db
--- /dev/null
+++ b/standalone/no-th/haskell-patches/persistent_1.1.5.1_0001-disable-TH.patch
@@ -0,0 +1,41 @@
+From aae3ace106cf26c931cc94c96fb6fbfe83f950f2 Mon Sep 17 00:00:00 2001
+From: dummy 
+Date: Wed, 15 Oct 2014 17:05:37 +0000
+Subject: [PATCH] avoid TH
+
+---
+ Database/Persist/Sql/Raw.hs | 4 +---
+ 1 file changed, 1 insertion(+), 3 deletions(-)
+
+diff --git a/Database/Persist/Sql/Raw.hs b/Database/Persist/Sql/Raw.hs
+index 3ac2ca9..bcc2011 100644
+--- a/Database/Persist/Sql/Raw.hs
++++ b/Database/Persist/Sql/Raw.hs
+@@ -11,7 +11,7 @@ import Data.IORef (writeIORef, readIORef, newIORef)
+ import Control.Exception (throwIO)
+ import Control.Monad (when, liftM)
+ import Data.Text (Text, pack)
+-import Control.Monad.Logger (logDebugS)
++--import Control.Monad.Logger (logDebugS)
+ import Data.Int (Int64)
+ import Control.Monad.Trans.Class (lift)
+ import qualified Data.Text as T
+@@ -23,7 +23,6 @@ rawQuery :: (MonadSqlPersist m, MonadResource m)
+          -> [PersistValue]
+          -> Source m [PersistValue]
+ rawQuery sql vals = do
+-    lift $ $logDebugS (pack "SQL") $ pack $ show sql ++ " " ++ show vals
+     conn <- lift askSqlConn
+     bracketP
+         (getStmtConn conn sql)
+@@ -35,7 +34,6 @@ rawExecute x y = liftM (const ()) $ rawExecuteCount x y
+ 
+ rawExecuteCount :: MonadSqlPersist m => Text -> [PersistValue] -> m Int64
+ rawExecuteCount sql vals = do
+-    $logDebugS (pack "SQL") $ pack $ show sql ++ " " ++ show vals
+     stmt <- getStmt sql
+     res <- liftIO $ stmtExecute stmt vals
+     liftIO $ stmtReset stmt
+-- 
+2.1.1
+
diff --git a/standalone/no-th/haskell-patches/process-conduit_avoid-TH.patch b/standalone/no-th/haskell-patches/process-conduit_avoid-TH.patch
new file mode 100644
index 0000000000..875119afdb
--- /dev/null
+++ b/standalone/no-th/haskell-patches/process-conduit_avoid-TH.patch
@@ -0,0 +1,24 @@
+From ed77588c57704030a9d412dd49f11c172c6268ab Mon Sep 17 00:00:00 2001
+From: dummy 
+Date: Tue, 14 Oct 2014 03:46:03 +0000
+Subject: [PATCH] unused
+
+---
+ process-conduit.cabal |    1 -
+ 1 file changed, 1 deletion(-)
+
+diff --git a/process-conduit.cabal b/process-conduit.cabal
+index 34bb168..2f137a8 100644
+--- a/process-conduit.cabal
++++ b/process-conduit.cabal
+@@ -22,7 +22,6 @@ source-repository head
+ 
+ library
+   exposed-modules:     Data.Conduit.ProcessOld
+-                       System.Process.QQ
+ 
+   build-depends:       base             == 4.*
+                      , template-haskell >= 2.4
+-- 
+1.7.10.4
+
diff --git a/standalone/no-th/haskell-patches/profunctors_3.3-0001-fix-cross-build.patch b/standalone/no-th/haskell-patches/profunctors_3.3-0001-fix-cross-build.patch
new file mode 100644
index 0000000000..45397f3e5d
--- /dev/null
+++ b/standalone/no-th/haskell-patches/profunctors_3.3-0001-fix-cross-build.patch
@@ -0,0 +1,26 @@
+From 392602f5ff14c0b5a801397d075ddcbcd890aa83 Mon Sep 17 00:00:00 2001
+From: Joey Hess 
+Date: Thu, 18 Apr 2013 17:50:59 -0400
+Subject: [PATCH] fix cross build
+
+---
+ src/Data/Profunctor/Unsafe.hs | 3 ---
+ 1 file changed, 3 deletions(-)
+
+diff --git a/src/Data/Profunctor/Unsafe.hs b/src/Data/Profunctor/Unsafe.hs
+index 025c7c4..0249274 100644
+--- a/src/Data/Profunctor/Unsafe.hs
++++ b/src/Data/Profunctor/Unsafe.hs
+@@ -40,9 +40,6 @@ import Data.Tagged
+ import Prelude hiding (id,(.),sequence)
+ import Unsafe.Coerce
+ 
+-{-# ANN module "Hlint: ignore Redundant lambda" #-}
+-{-# ANN module "Hlint: ignore Collapse lambdas" #-}
+-
+ infixr 9 #.
+ infixl 8 .#
+ 
+-- 
+1.8.2.rc3
+
diff --git a/standalone/no-th/haskell-patches/reflection_remove-TH.patch b/standalone/no-th/haskell-patches/reflection_remove-TH.patch
new file mode 100644
index 0000000000..4f8b4bc20f
--- /dev/null
+++ b/standalone/no-th/haskell-patches/reflection_remove-TH.patch
@@ -0,0 +1,59 @@
+From c0f5dcfd6ba7a05bb84b6adc4664c8dde109e6ac Mon Sep 17 00:00:00 2001
+From: dummy 
+Date: Fri, 7 Mar 2014 04:30:22 +0000
+Subject: [PATCH] remove TH
+
+---
+ fast/Data/Reflection.hs | 8 +++++---
+ 1 file changed, 5 insertions(+), 3 deletions(-)
+
+diff --git a/fast/Data/Reflection.hs b/fast/Data/Reflection.hs
+index ca57d35..d3f8356 100644
+--- a/fast/Data/Reflection.hs
++++ b/fast/Data/Reflection.hs
+@@ -59,7 +59,7 @@ module Data.Reflection
+     , Given(..)
+     , give
+     -- * Template Haskell reflection
+-    , int, nat
++    --, int, nat
+     -- * Useful compile time naturals
+     , Z, D, SD, PD
+     ) where
+@@ -161,6 +161,7 @@ instance Reifies n Int => Reifies (PD n) Int where
+ -- instead of @$(int 3)@. Sometimes the two will produce the same
+ -- representation (if compiled without the @-DUSE_TYPE_LITS@ preprocessor
+ -- directive).
++{-
+ int :: Int -> TypeQ
+ int n = case quotRem n 2 of
+   (0, 0) -> conT ''Z
+@@ -176,7 +177,7 @@ nat :: Int -> TypeQ
+ nat n
+   | n >= 0 = int n
+   | otherwise = error "nat: negative"
+-
++-}
+ #if defined(__GLASGOW_HASKELL__) && __GLASGOW_HASKELL < 704
+ instance Show (Q a)
+ instance Eq (Q a)
+@@ -195,6 +196,7 @@ instance Fractional a => Fractional (Q a) where
+   recip = fmap recip
+   fromRational = return . fromRational
+ 
++{-
+ -- | This permits the use of $(5) as a type splice.
+ instance Num Type where
+ #ifdef USE_TYPE_LITS
+@@ -254,7 +256,7 @@ instance Num Exp where
+   abs = onProxyType1 abs
+   signum = onProxyType1 signum
+   fromInteger n = ConE 'Proxy `SigE` (ConT ''Proxy `AppT` fromInteger n)
+-
++-}
+ #ifdef USE_TYPE_LITS
+ addProxy :: Proxy a -> Proxy b -> Proxy (a + b)
+ addProxy _ _ = Proxy
+-- 
+1.9.0
+
diff --git a/standalone/no-th/haskell-patches/shakespeare_remove-TH.patch b/standalone/no-th/haskell-patches/shakespeare_remove-TH.patch
new file mode 100644
index 0000000000..68226dcc6f
--- /dev/null
+++ b/standalone/no-th/haskell-patches/shakespeare_remove-TH.patch
@@ -0,0 +1,1438 @@
+From 4694f3a7ee4eb15d33ecda9d62712ea236304c1b Mon Sep 17 00:00:00 2001
+From: dummy 
+Date: Thu, 2 Jul 2015 22:17:29 +0000
+Subject: [PATCH] hack TH
+
+---
+ Text/Cassius.hs          |  30 +---
+ Text/Coffee.hs           |  56 +-------
+ Text/Css.hs              | 151 ---------------------
+ Text/CssCommon.hs        |  22 ---
+ Text/Hamlet.hs           | 346 +++--------------------------------------------
+ Text/Julius.hs           |  59 +-------
+ Text/Lucius.hs           |  47 +------
+ Text/Roy.hs              |  52 +------
+ Text/Shakespeare.hs      |  70 ++--------
+ Text/Shakespeare/Base.hs |  28 ----
+ Text/Shakespeare/Text.hs | 117 ++--------------
+ Text/TypeScript.hs       |  48 +------
+ shakespeare.cabal        |   6 +-
+ 13 files changed, 69 insertions(+), 963 deletions(-)
+
+diff --git a/Text/Cassius.hs b/Text/Cassius.hs
+index ba73bdd..ffe7c51 100644
+--- a/Text/Cassius.hs
++++ b/Text/Cassius.hs
+@@ -14,12 +14,7 @@ module Text.Cassius
+     , renderCss
+     , renderCssUrl
+       -- * Parsing
+-    , cassius
+-    , cassiusFile
+-    , cassiusFileDebug
+-    , cassiusFileReload
+       -- ** Mixims
+-    , cassiusMixin
+     , Mixin
+       -- * ToCss instances
+       -- ** Color
+@@ -27,15 +22,12 @@ module Text.Cassius
+     , colorRed
+     , colorBlack
+       -- ** Size
+-    , mkSize
++    --, mkSize
+     , AbsoluteUnit (..)
+     , AbsoluteSize (..)
+     , absoluteSize
+-    , EmSize (..)
+-    , ExSize (..)
+     , PercentageSize (..)
+     , percentageSize
+-    , PixelSize (..)
+       -- * Internal
+     , cassiusUsedIdentifiers
+     ) where
+@@ -47,25 +39,9 @@ import Language.Haskell.TH.Quote (QuasiQuoter (..))
+ import Language.Haskell.TH.Syntax
+ import qualified Data.Text.Lazy as TL
+ import Text.CssCommon
+-import Text.Lucius (lucius)
+ import qualified Text.Lucius
+ import Text.IndentToBrace (i2b)
+ 
+-cassius :: QuasiQuoter
+-cassius = QuasiQuoter { quoteExp = quoteExp lucius . i2b }
+-
+-cassiusFile :: FilePath -> Q Exp
+-cassiusFile fp = do
+-#ifdef GHC_7_4
+-    qAddDependentFile fp
+-#endif
+-    contents <- fmap TL.unpack $ qRunIO $ readUtf8File fp
+-    quoteExp cassius contents
+-
+-cassiusFileDebug, cassiusFileReload :: FilePath -> Q Exp
+-cassiusFileDebug = cssFileDebug True [|Text.Lucius.parseTopLevels|] Text.Lucius.parseTopLevels
+-cassiusFileReload = cassiusFileDebug
+-
+ -- | Determine which identifiers are used by the given template, useful for
+ -- creating systems like yesod devel.
+ cassiusUsedIdentifiers :: String -> [(Deref, VarType)]
+@@ -74,10 +50,6 @@ cassiusUsedIdentifiers = cssUsedIdentifiers True Text.Lucius.parseTopLevels
+ -- | Create a mixin with Cassius syntax.
+ --
+ -- Since 2.0.3
+-cassiusMixin :: QuasiQuoter
+-cassiusMixin = QuasiQuoter
+-    { quoteExp = quoteExp Text.Lucius.luciusMixin . i2bMixin
+-    }
+ 
+ i2bMixin :: String -> String
+ i2bMixin s' =
+diff --git a/Text/Coffee.hs b/Text/Coffee.hs
+index 488c81b..4e28c94 100644
+--- a/Text/Coffee.hs
++++ b/Text/Coffee.hs
+@@ -51,13 +51,13 @@ module Text.Coffee
+       -- ** Template-Reading Functions
+       -- | These QuasiQuoter and Template Haskell methods return values of
+       -- type @'JavascriptUrl' url@. See the Yesod book for details.
+-      coffee
+-    , coffeeFile
+-    , coffeeFileReload
+-    , coffeeFileDebug
++    --  coffee
++    --, coffeeFile
++    --, coffeeFileReload
++    --, coffeeFileDebug
+ 
+ #ifdef TEST_EXPORT
+-    , coffeeSettings
++    -- , coffeeSettings
+ #endif
+     ) where
+ 
+@@ -65,49 +65,3 @@ import Language.Haskell.TH.Quote (QuasiQuoter (..))
+ import Language.Haskell.TH.Syntax
+ import Text.Shakespeare
+ import Text.Julius
+-
+-coffeeSettings :: Q ShakespeareSettings
+-coffeeSettings = do
+-  jsettings <- javascriptSettings
+-  return $ jsettings { varChar = '%'
+-  , preConversion = Just PreConvert {
+-      preConvert = ReadProcess "coffee" ["-spb"]
+-    , preEscapeIgnoreBalanced = "'\"`"     -- don't insert backtacks for variable already inside strings or backticks.
+-    , preEscapeIgnoreLine = "#"            -- ignore commented lines
+-    , wrapInsertion = Just WrapInsertion { 
+-        wrapInsertionIndent = Just "  "
+-      , wrapInsertionStartBegin = "("
+-      , wrapInsertionSeparator = ", "
+-      , wrapInsertionStartClose = ") =>"
+-      , wrapInsertionEnd = ""
+-      , wrapInsertionAddParens = False
+-      }
+-    }
+-  }
+-
+--- | Read inline, quasiquoted CoffeeScript.
+-coffee :: QuasiQuoter
+-coffee = QuasiQuoter { quoteExp = \s -> do
+-    rs <- coffeeSettings
+-    quoteExp (shakespeare rs) s
+-    }
+-
+--- | Read in a CoffeeScript template file. This function reads the file once, at
+--- compile time.
+-coffeeFile :: FilePath -> Q Exp
+-coffeeFile fp = do
+-    rs <- coffeeSettings
+-    shakespeareFile rs fp
+-
+--- | Read in a CoffeeScript template file. This impure function uses
+--- unsafePerformIO to re-read the file on every call, allowing for rapid
+--- iteration.
+-coffeeFileReload :: FilePath -> Q Exp
+-coffeeFileReload fp = do
+-    rs <- coffeeSettings
+-    shakespeareFileReload rs fp
+-
+--- | Deprecated synonym for 'coffeeFileReload'
+-coffeeFileDebug :: FilePath -> Q Exp
+-coffeeFileDebug = coffeeFileReload
+-{-# DEPRECATED coffeeFileDebug "Please use coffeeFileReload instead." #-}
+diff --git a/Text/Css.hs b/Text/Css.hs
+index 75dc549..20c206c 100644
+--- a/Text/Css.hs
++++ b/Text/Css.hs
+@@ -166,22 +166,6 @@ cssUsedIdentifiers toi2b parseBlocks s' =
+         (scope, rest') = go rest
+     go' (Attr k v) = k ++ v
+ 
+-cssFileDebug :: Bool -- ^ perform the indent-to-brace conversion
+-             -> Q Exp
+-             -> Parser [TopLevel Unresolved]
+-             -> FilePath
+-             -> Q Exp
+-cssFileDebug toi2b parseBlocks' parseBlocks fp = do
+-    s <- fmap TL.unpack $ qRunIO $ readUtf8File fp
+-#ifdef GHC_7_4
+-    qAddDependentFile fp
+-#endif
+-    let vs = cssUsedIdentifiers toi2b parseBlocks s
+-    c <- mapM vtToExp vs
+-    cr <- [|cssRuntime toi2b|]
+-    parseBlocks'' <- parseBlocks'
+-    return $ cr `AppE` parseBlocks'' `AppE` (LitE $ StringL fp) `AppE` ListE c
+-
+ combineSelectors :: HasLeadingSpace
+                  -> [Contents]
+                  -> [Contents]
+@@ -287,18 +271,6 @@ cssRuntime toi2b parseBlocks fp cd render' = unsafePerformIO $ do
+ 
+     addScope scope = map (DerefIdent . Ident *** CDPlain . fromString) scope ++ cd
+ 
+-vtToExp :: (Deref, VarType) -> Q Exp
+-vtToExp (d, vt) = do
+-    d' <- lift d
+-    c' <- c vt
+-    return $ TupE [d', c' `AppE` derefToExp [] d]
+-  where
+-    c :: VarType -> Q Exp
+-    c VTPlain = [|CDPlain . toCss|]
+-    c VTUrl = [|CDUrl|]
+-    c VTUrlParam = [|CDUrlParam|]
+-    c VTMixin = [|CDMixin|]
+-
+ getVars :: Monad m => [(String, String)] -> Content -> m [(Deref, VarType)]
+ getVars _ ContentRaw{} = return []
+ getVars scope (ContentVar d) =
+@@ -342,111 +314,8 @@ compressBlock (Block x y blocks mixins) =
+     cc (ContentRaw a:ContentRaw b:c) = cc $ ContentRaw (a ++ b) : c
+     cc (a:b) = a : cc b
+ 
+-blockToMixin :: Name
+-             -> Scope
+-             -> Block Unresolved
+-             -> Q Exp
+-blockToMixin r scope (Block _sel props subblocks mixins) =
+-    [|Mixin
+-        { mixinAttrs    = concat
+-                        $ $(listE $ map go props)
+-                        : map mixinAttrs $mixinsE
+-        -- FIXME too many complications to implement sublocks for now...
+-        , mixinBlocks   = [] -- foldr (.) id $(listE $ map subGo subblocks) []
+-        }|]
+-      {-
+-      . foldr (.) id $(listE $ map subGo subblocks)
+-      . (concatMap mixinBlocks $mixinsE ++)
+-    |]
+-    -}
+-  where
+-    mixinsE = return $ ListE $ map (derefToExp []) mixins
+-    go (Attr x y) = conE 'Attr
+-        `appE` (contentsToBuilder r scope x)
+-        `appE` (contentsToBuilder r scope y)
+-    subGo (Block sel' b c d) = blockToCss r scope $ Block sel' b c d
+-
+-blockToCss :: Name
+-           -> Scope
+-           -> Block Unresolved
+-           -> Q Exp
+-blockToCss r scope (Block sel props subblocks mixins) =
+-    [|((Block
+-        { blockSelector = $(selectorToBuilder r scope sel)
+-        , blockAttrs    = concat
+-                        $ $(listE $ map go props)
+-                        : map mixinAttrs $mixinsE
+-        , blockBlocks   = ()
+-        , blockMixins   = ()
+-        } :: Block Resolved):)
+-      . foldr (.) id $(listE $ map subGo subblocks)
+-      . (concatMap mixinBlocks $mixinsE ++)
+-    |]
+-  where
+-    mixinsE = return $ ListE $ map (derefToExp []) mixins
+-    go (Attr x y) = conE 'Attr
+-        `appE` (contentsToBuilder r scope x)
+-        `appE` (contentsToBuilder r scope y)
+-    subGo (hls, Block sel' b c d) =
+-        blockToCss r scope $ Block sel'' b c d
+-      where
+-        sel'' = combineSelectors hls sel sel'
+-
+-selectorToBuilder :: Name -> Scope -> [Contents] -> Q Exp
+-selectorToBuilder r scope sels =
+-    contentsToBuilder r scope $ intercalate [ContentRaw ","] sels
+-
+-contentsToBuilder :: Name -> Scope -> [Content] -> Q Exp
+-contentsToBuilder r scope contents =
+-    appE [|mconcat|] $ listE $ map (contentToBuilder r scope) contents
+-
+-contentToBuilder :: Name -> Scope -> Content -> Q Exp
+-contentToBuilder _ _ (ContentRaw x) =
+-    [|fromText . pack|] `appE` litE (StringL x)
+-contentToBuilder _ scope (ContentVar d) =
+-    case d of
+-        DerefIdent (Ident s)
+-            | Just val <- lookup s scope -> [|fromText . pack|] `appE` litE (StringL val)
+-        _ -> [|toCss|] `appE` return (derefToExp [] d)
+-contentToBuilder r _ (ContentUrl u) =
+-    [|fromText|] `appE`
+-        (varE r `appE` return (derefToExp [] u) `appE` listE [])
+-contentToBuilder r _ (ContentUrlParam u) =
+-    [|fromText|] `appE`
+-        ([|uncurry|] `appE` varE r `appE` return (derefToExp [] u))
+-contentToBuilder _ _ ContentMixin{} = error "contentToBuilder on ContentMixin"
+-
+ type Scope = [(String, String)]
+ 
+-topLevelsToCassius :: [TopLevel Unresolved]
+-                   -> Q Exp
+-topLevelsToCassius a = do
+-    r <- newName "_render"
+-    lamE [varP r] $ appE [|CssNoWhitespace . foldr ($) []|] $ fmap ListE $ go r [] a
+-  where
+-    go _ _ [] = return []
+-    go r scope (TopBlock b:rest) = do
+-        e <- [|(++) $ map TopBlock ($(blockToCss r scope b) [])|]
+-        es <- go r scope rest
+-        return $ e : es
+-    go r scope (TopAtBlock name s b:rest) = do
+-        let s' = contentsToBuilder r scope s
+-        e <- [|(:) $ TopAtBlock $(lift name) $(s') $(blocksToCassius r scope b)|]
+-        es <- go r scope rest
+-        return $ e : es
+-    go r scope (TopAtDecl dec cs:rest) = do
+-        e <- [|(:) $ TopAtDecl $(lift dec) $(contentsToBuilder r scope cs)|]
+-        es <- go r scope rest
+-        return $ e : es
+-    go r scope (TopVar k v:rest) = go r ((k, v) : scope) rest
+-
+-blocksToCassius :: Name
+-                -> Scope
+-                -> [Block Unresolved]
+-                -> Q Exp
+-blocksToCassius r scope a = do
+-    appE [|foldr ($) []|] $ listE $ map (blockToCss r scope) a
+-
+ renderCss :: Css -> TL.Text
+ renderCss css =
+     toLazyText $ mconcat $ map go tops
+@@ -515,23 +384,3 @@ renderBlock haveWhiteSpace indent (Block sel attrs () ())
+         | haveWhiteSpace = fromString ";\n"
+         | otherwise = singleton ';'
+ 
+-instance Lift Mixin where
+-    lift (Mixin a b) = [|Mixin a b|]
+-instance Lift (Attr Unresolved) where
+-    lift (Attr k v) = [|Attr k v :: Attr Unresolved |]
+-instance Lift (Attr Resolved) where
+-    lift (Attr k v) = [|Attr $(liftBuilder k) $(liftBuilder v) :: Attr Resolved |]
+-
+-liftBuilder :: Builder -> Q Exp
+-liftBuilder b = [|fromText $ pack $(lift $ TL.unpack $ toLazyText b)|]
+-
+-instance Lift Content where
+-    lift (ContentRaw s) = [|ContentRaw s|]
+-    lift (ContentVar d) = [|ContentVar d|]
+-    lift (ContentUrl d) = [|ContentUrl d|]
+-    lift (ContentUrlParam d) = [|ContentUrlParam d|]
+-    lift (ContentMixin m) = [|ContentMixin m|]
+-instance Lift (Block Unresolved) where
+-    lift (Block a b c d) = [|Block a b c d|]
+-instance Lift (Block Resolved) where
+-    lift (Block a b () ()) = [|Block $(liftBuilder a) b () ()|]
+diff --git a/Text/CssCommon.hs b/Text/CssCommon.hs
+index 719e0a8..0635cf4 100644
+--- a/Text/CssCommon.hs
++++ b/Text/CssCommon.hs
+@@ -1,4 +1,3 @@
+-{-# LANGUAGE TemplateHaskell #-}
+ {-# LANGUAGE GeneralizedNewtypeDeriving #-}
+ {-# LANGUAGE FlexibleInstances #-}
+ {-# LANGUAGE CPP #-}
+@@ -47,24 +46,6 @@ colorBlack = Color 0 0 0
+ 
+ -- CSS size wrappers
+ 
+--- | Create a CSS size, e.g. $(mkSize "100px").
+-mkSize :: String -> ExpQ
+-mkSize s = appE nameE valueE
+-  where [(value, unit)] = reads s :: [(Double, String)]
+-        absoluteSizeE = varE $ mkName "absoluteSize"
+-        nameE = case unit of
+-          "cm" -> appE absoluteSizeE (conE $ mkName "Centimeter")
+-          "em" -> conE $ mkName "EmSize"
+-          "ex" -> conE $ mkName "ExSize"
+-          "in" -> appE absoluteSizeE (conE $ mkName "Inch")
+-          "mm" -> appE absoluteSizeE (conE $ mkName "Millimeter")
+-          "pc" -> appE absoluteSizeE (conE $ mkName "Pica")
+-          "pt" -> appE absoluteSizeE (conE $ mkName "Point")
+-          "px" -> conE $ mkName "PixelSize"
+-          "%" -> varE $ mkName "percentageSize"
+-          _ -> error $ "In mkSize, invalid unit: " ++ unit
+-        valueE = litE $ rationalL (toRational value)
+-
+ -- | Absolute size units.
+ data AbsoluteUnit = Centimeter
+                   | Inch
+@@ -156,6 +137,3 @@ showSize :: Rational -> String -> String
+ showSize value' unit = printf "%f" value ++ unit
+   where value = fromRational value' :: Double
+ 
+-mkSizeType "EmSize" "em"
+-mkSizeType "ExSize" "ex"
+-mkSizeType "PixelSize" "px"
+diff --git a/Text/Hamlet.hs b/Text/Hamlet.hs
+index 4618be3..4ad3633 100644
+--- a/Text/Hamlet.hs
++++ b/Text/Hamlet.hs
+@@ -11,36 +11,36 @@
+ module Text.Hamlet
+     ( -- * Plain HTML
+       Html
+-    , shamlet
+-    , shamletFile
+-    , xshamlet
+-    , xshamletFile
++    --, shamlet
++    --, shamletFile
++    --, xshamlet
++    --, xshamletFile
+       -- * Hamlet
+     , HtmlUrl
+-    , hamlet
+-    , hamletFile
+-    , hamletFileReload
+-    , ihamletFileReload
+-    , xhamlet
+-    , xhamletFile
++    --, hamlet
++    -- , hamletFile
++    -- , hamletFileReload
++    -- , ihamletFileReload
++    -- , xhamlet
++    -- , xhamletFile
+       -- * I18N Hamlet
+     , HtmlUrlI18n
+-    , ihamlet
+-    , ihamletFile
++    -- , ihamlet
++    -- , ihamletFile
+       -- * Type classes
+     , ToAttributes (..)
+       -- * Internal, for making more
+     , HamletSettings (..)
+     , NewlineStyle (..)
+-    , hamletWithSettings
+-    , hamletFileWithSettings
++    -- , hamletWithSettings
++    -- , hamletFileWithSettings
+     , defaultHamletSettings
+     , xhtmlHamletSettings
+-    , Env (..)
+-    , HamletRules (..)
+-    , hamletRules
+-    , ihamletRules
+-    , htmlRules
++    --, Env (..)
++    --, HamletRules (..)
++    --, hamletRules
++    --, ihamletRules
++    --, htmlRules
+     , CloseStyle (..)
+       -- * Used by generated code
+     , condH
+@@ -109,48 +109,9 @@ type HtmlUrl url = Render url -> Html
+ -- | A function generating an 'Html' given a message translator and a URL rendering function.
+ type HtmlUrlI18n msg url = Translate msg -> Render url -> Html
+ 
+-docsToExp :: Env -> HamletRules -> Scope -> [Doc] -> Q Exp
+-docsToExp env hr scope docs = do
+-    exps <- mapM (docToExp env hr scope) docs
+-    case exps of
+-        [] -> [|return ()|]
+-        [x] -> return x
+-        _ -> return $ DoE $ map NoBindS exps
+-
+ unIdent :: Ident -> String
+ unIdent (Ident s) = s
+ 
+-bindingPattern :: Binding -> Q (Pat, [(Ident, Exp)])
+-bindingPattern (BindAs i@(Ident s) b) = do
+-    name <- newName s
+-    (pattern, scope) <- bindingPattern b
+-    return (AsP name pattern, (i, VarE name):scope)
+-bindingPattern (BindVar i@(Ident s))
+-    | s == "_" = return (WildP, [])
+-    | all isDigit s = do
+-        return (LitP $ IntegerL $ read s, [])
+-    | otherwise = do
+-        name <- newName s
+-        return (VarP name, [(i, VarE name)])
+-bindingPattern (BindTuple is) = do
+-    (patterns, scopes) <- fmap unzip $ mapM bindingPattern is
+-    return (TupP patterns, concat scopes)
+-bindingPattern (BindList is) = do
+-    (patterns, scopes) <- fmap unzip $ mapM bindingPattern is
+-    return (ListP patterns, concat scopes)
+-bindingPattern (BindConstr con is) = do
+-    (patterns, scopes) <- fmap unzip $ mapM bindingPattern is
+-    return (ConP (mkConName con) patterns, concat scopes)
+-bindingPattern (BindRecord con fields wild) = do
+-    let f (Ident field,b) =
+-           do (p,s) <- bindingPattern b
+-              return ((mkName field,p),s)
+-    (patterns, scopes) <- fmap unzip $ mapM f fields
+-    (patterns1, scopes1) <- if wild
+-       then bindWildFields con $ map fst fields
+-       else return ([],[])
+-    return (RecP (mkConName con) (patterns++patterns1), concat scopes ++ scopes1)
+-
+ mkConName :: DataConstr -> Name
+ mkConName = mkName . conToStr
+ 
+@@ -158,257 +119,15 @@ conToStr :: DataConstr -> String
+ conToStr (DCUnqualified (Ident x)) = x
+ conToStr (DCQualified (Module xs) (Ident x)) = intercalate "." $ xs ++ [x]
+ 
+--- Wildcards bind all of the unbound fields to variables whose name
+--- matches the field name.
+---
+--- For example: data R = C { f1, f2 :: Int }
+--- C {..}           is equivalent to   C {f1=f1, f2=f2}
+--- C {f1 = a, ..}   is equivalent to   C {f1=a,  f2=f2}
+--- C {f2 = a, ..}   is equivalent to   C {f1=f1, f2=a}
+-bindWildFields :: DataConstr -> [Ident] -> Q ([(Name, Pat)], [(Ident, Exp)])
+-bindWildFields conName fields = do
+-  fieldNames <- recordToFieldNames conName
+-  let available n     = nameBase n `notElem` map unIdent fields
+-  let remainingFields = filter available fieldNames
+-  let mkPat n = do
+-        e <- newName (nameBase n)
+-        return ((n,VarP e), (Ident (nameBase n), VarE e))
+-  fmap unzip $ mapM mkPat remainingFields
+-
+--- Important note! reify will fail if the record type is defined in the
+--- same module as the reify is used. This means quasi-quoted Hamlet
+--- literals will not be able to use wildcards to match record types
+--- defined in the same module.
+-recordToFieldNames :: DataConstr -> Q [Name]
+-recordToFieldNames conStr = do
+-  -- use 'lookupValueName' instead of just using 'mkName' so we reify the
+-  -- data constructor and not the type constructor if their names match.
+-  Just conName                <- lookupValueName $ conToStr conStr
+-  DataConI _ _ typeName _     <- reify conName
+-  TyConI (DataD _ _ _ cons _) <- reify typeName
+-  [fields] <- return [fields | RecC name fields <- cons, name == conName]
+-  return [fieldName | (fieldName, _, _) <- fields]
+-
+-docToExp :: Env -> HamletRules -> Scope -> Doc -> Q Exp
+-docToExp env hr scope (DocForall list idents inside) = do
+-    let list' = derefToExp scope list
+-    (pat, extraScope) <- bindingPattern idents
+-    let scope' = extraScope ++ scope
+-    mh <- [|F.mapM_|]
+-    inside' <- docsToExp env hr scope' inside
+-    let lam = LamE [pat] inside'
+-    return $ mh `AppE` lam `AppE` list'
+-docToExp env hr scope (DocWith [] inside) = do
+-    inside' <- docsToExp env hr scope inside
+-    return $ inside'
+-docToExp env hr scope (DocWith ((deref, idents):dis) inside) = do
+-    let deref' = derefToExp scope deref
+-    (pat, extraScope) <- bindingPattern idents
+-    let scope' = extraScope ++ scope
+-    inside' <- docToExp env hr scope' (DocWith dis inside)
+-    let lam = LamE [pat] inside'
+-    return $ lam `AppE` deref'
+-docToExp env hr scope (DocMaybe val idents inside mno) = do
+-    let val' = derefToExp scope val
+-    (pat, extraScope) <- bindingPattern idents
+-    let scope' = extraScope ++ scope
+-    inside' <- docsToExp env hr scope' inside
+-    let inside'' = LamE [pat] inside'
+-    ninside' <- case mno of
+-                    Nothing -> [|Nothing|]
+-                    Just no -> do
+-                        no' <- docsToExp env hr scope no
+-                        j <- [|Just|]
+-                        return $ j `AppE` no'
+-    mh <- [|maybeH|]
+-    return $ mh `AppE` val' `AppE` inside'' `AppE` ninside'
+-docToExp env hr scope (DocCond conds final) = do
+-    conds' <- mapM go conds
+-    final' <- case final of
+-                Nothing -> [|Nothing|]
+-                Just f -> do
+-                    f' <- docsToExp env hr scope f
+-                    j <- [|Just|]
+-                    return $ j `AppE` f'
+-    ch <- [|condH|]
+-    return $ ch `AppE` ListE conds' `AppE` final'
+-  where
+-    go :: (Deref, [Doc]) -> Q Exp
+-    go (d, docs) = do
+-        let d' = derefToExp ((specialOrIdent, VarE 'or):scope) d
+-        docs' <- docsToExp env hr scope docs
+-        return $ TupE [d', docs']
+-docToExp env hr scope (DocCase deref cases) = do
+-    let exp_ = derefToExp scope deref
+-    matches <- mapM toMatch cases
+-    return $ CaseE exp_ matches
+-  where
+-    toMatch :: (Binding, [Doc]) -> Q Match
+-    toMatch (idents, inside) = do
+-        (pat, extraScope) <- bindingPattern idents
+-        let scope' = extraScope ++ scope
+-        insideExp <- docsToExp env hr scope' inside
+-        return $ Match pat (NormalB insideExp) []
+-docToExp env hr v (DocContent c) = contentToExp env hr v c
+-
+-contentToExp :: Env -> HamletRules -> Scope -> Content -> Q Exp
+-contentToExp _ hr _ (ContentRaw s) = do
+-    os <- [|preEscapedText . pack|]
+-    let s' = LitE $ StringL s
+-    return $ hrFromHtml hr `AppE` (os `AppE` s')
+-contentToExp _ hr scope (ContentVar d) = do
+-    str <- [|toHtml|]
+-    return $ hrFromHtml hr `AppE` (str `AppE` derefToExp scope d)
+-contentToExp env hr scope (ContentUrl hasParams d) =
+-    case urlRender env of
+-        Nothing -> error "URL interpolation used, but no URL renderer provided"
+-        Just wrender -> wrender $ \render -> do
+-            let render' = return render
+-            ou <- if hasParams
+-                    then [|\(u, p) -> $(render') u p|]
+-                    else [|\u -> $(render') u []|]
+-            let d' = derefToExp scope d
+-            pet <- [|toHtml|]
+-            return $ hrFromHtml hr `AppE` (pet `AppE` (ou `AppE` d'))
+-contentToExp env hr scope (ContentEmbed d) = hrEmbed hr env $ derefToExp scope d
+-contentToExp env hr scope (ContentMsg d) =
+-    case msgRender env of
+-        Nothing -> error "Message interpolation used, but no message renderer provided"
+-        Just wrender -> wrender $ \render ->
+-            return $ hrFromHtml hr `AppE` (render `AppE` derefToExp scope d)
+-contentToExp _ hr scope (ContentAttrs d) = do
+-    html <- [|attrsToHtml . toAttributes|]
+-    return $ hrFromHtml hr `AppE` (html `AppE` derefToExp scope d)
+-
+-shamlet :: QuasiQuoter
+-shamlet = hamletWithSettings htmlRules defaultHamletSettings
+-
+-xshamlet :: QuasiQuoter
+-xshamlet = hamletWithSettings htmlRules xhtmlHamletSettings
+-
+-htmlRules :: Q HamletRules
+-htmlRules = do
+-    i <- [|id|]
+-    return $ HamletRules i ($ (Env Nothing Nothing)) (\_ b -> return b)
+-
+-hamlet :: QuasiQuoter
+-hamlet = hamletWithSettings hamletRules defaultHamletSettings
+-
+-xhamlet :: QuasiQuoter
+-xhamlet = hamletWithSettings hamletRules xhtmlHamletSettings
+-
+ asHtmlUrl :: HtmlUrl url -> HtmlUrl url
+ asHtmlUrl = id
+ 
+-hamletRules :: Q HamletRules
+-hamletRules = do
+-    i <- [|id|]
+-    let ur f = do
+-            r <- newName "_render"
+-            let env = Env
+-                    { urlRender = Just ($ (VarE r))
+-                    , msgRender = Nothing
+-                    }
+-            h <- f env
+-            return $ LamE [VarP r] h
+-    return $ HamletRules i ur em
+-  where
+-    em (Env (Just urender) Nothing) e = do
+-        asHtmlUrl' <- [|asHtmlUrl|]
+-        urender $ \ur' -> return ((asHtmlUrl' `AppE` e) `AppE` ur')
+-    em _ _ = error "bad Env"
+-
+-ihamlet :: QuasiQuoter
+-ihamlet = hamletWithSettings ihamletRules defaultHamletSettings
+-
+-ihamletRules :: Q HamletRules
+-ihamletRules = do
+-    i <- [|id|]
+-    let ur f = do
+-            u <- newName "_urender"
+-            m <- newName "_mrender"
+-            let env = Env
+-                    { urlRender = Just ($ (VarE u))
+-                    , msgRender = Just ($ (VarE m))
+-                    }
+-            h <- f env
+-            return $ LamE [VarP m, VarP u] h
+-    return $ HamletRules i ur em
+-  where
+-    em (Env (Just urender) (Just mrender)) e =
+-          urender $ \ur' -> mrender $ \mr -> return (e `AppE` mr `AppE` ur')
+-    em _ _ = error "bad Env"
+-
+-hamletWithSettings :: Q HamletRules -> HamletSettings -> QuasiQuoter
+-hamletWithSettings hr set =
+-    QuasiQuoter
+-        { quoteExp = hamletFromString hr set
+-        }
+-
+-data HamletRules = HamletRules
+-    { hrFromHtml :: Exp
+-    , hrWithEnv :: (Env -> Q Exp) -> Q Exp
+-    , hrEmbed :: Env -> Exp -> Q Exp
+-    }
+-
+-data Env = Env
+-    { urlRender :: Maybe ((Exp -> Q Exp) -> Q Exp)
+-    , msgRender :: Maybe ((Exp -> Q Exp) -> Q Exp)
+-    }
+-
+-hamletFromString :: Q HamletRules -> HamletSettings -> String -> Q Exp
+-hamletFromString qhr set s = do
+-    hr <- qhr
+-    hrWithEnv hr $ \env -> docsToExp env hr [] $ docFromString set s
+-
+ docFromString :: HamletSettings -> String -> [Doc]
+ docFromString set s =
+     case parseDoc set s of
+         Error s' -> error s'
+         Ok (_, d) -> d
+ 
+-hamletFileWithSettings :: Q HamletRules -> HamletSettings -> FilePath -> Q Exp
+-hamletFileWithSettings qhr set fp = do
+-#ifdef GHC_7_4
+-    qAddDependentFile fp
+-#endif
+-    contents <- fmap TL.unpack $ qRunIO $ readUtf8File fp
+-    hamletFromString qhr set contents
+-
+-hamletFile :: FilePath -> Q Exp
+-hamletFile = hamletFileWithSettings hamletRules defaultHamletSettings
+-
+-hamletFileReload :: FilePath -> Q Exp
+-hamletFileReload = hamletFileReloadWithSettings runtimeRules defaultHamletSettings
+-  where runtimeRules = HamletRuntimeRules { hrrI18n = False }
+-
+-ihamletFileReload :: FilePath -> Q Exp
+-ihamletFileReload = hamletFileReloadWithSettings runtimeRules defaultHamletSettings
+-  where runtimeRules = HamletRuntimeRules { hrrI18n = True }
+-
+-xhamletFile :: FilePath -> Q Exp
+-xhamletFile = hamletFileWithSettings hamletRules xhtmlHamletSettings
+-
+-shamletFile :: FilePath -> Q Exp
+-shamletFile = hamletFileWithSettings htmlRules defaultHamletSettings
+-
+-xshamletFile :: FilePath -> Q Exp
+-xshamletFile = hamletFileWithSettings htmlRules xhtmlHamletSettings
+-
+-ihamletFile :: FilePath -> Q Exp
+-ihamletFile = hamletFileWithSettings ihamletRules defaultHamletSettings
+-
+-varName :: Scope -> String -> Exp
+-varName _ "" = error "Illegal empty varName"
+-varName scope v@(_:_) = fromMaybe (strToExp v) $ lookup (Ident v) scope
+-
+-strToExp :: String -> Exp
+-strToExp s@(c:_)
+-    | all isDigit s = LitE $ IntegerL $ read s
+-    | isUpper c = ConE $ mkName s
+-    | otherwise = VarE $ mkName s
+-strToExp "" = error "strToExp on empty string"
+-
+ -- | Checks for truth in the left value in each pair in the first argument. If
+ -- a true exists, then the corresponding right action is performed. Only the
+ -- first is performed. In there are no true values, then the second argument is
+@@ -461,33 +180,6 @@ data HamletRuntimeRules = HamletRuntimeRules {
+                             hrrI18n :: Bool
+                           }
+ 
+-hamletFileReloadWithSettings :: HamletRuntimeRules
+-                             -> HamletSettings -> FilePath -> Q Exp
+-hamletFileReloadWithSettings hrr settings fp = do
+-    s <- readFileQ fp
+-    let b = hamletUsedIdentifiers settings s
+-    c <- mapM vtToExp b
+-    rt <- if hrrI18n hrr
+-      then [|hamletRuntimeMsg settings fp|]
+-      else [|hamletRuntime settings fp|]
+-    return $ rt `AppE` ListE c
+-  where
+-    vtToExp :: (Deref, VarType) -> Q Exp
+-    vtToExp (d, vt) = do
+-        d' <- lift d
+-        c' <- toExp vt
+-        return $ TupE [d', c' `AppE` derefToExp [] d]
+-      where
+-        toExp = c
+-          where
+-            c :: VarType -> Q Exp
+-            c VTAttrs = [|EPlain . attrsToHtml . toAttributes|]
+-            c VTPlain = [|EPlain . toHtml|]
+-            c VTUrl = [|EUrl|]
+-            c VTUrlParam = [|EUrlParam|]
+-            c VTMixin = [|\r -> EMixin $ \c -> r c|]
+-            c VTMsg = [|EMsg|]
+-
+ -- move to Shakespeare.Base?
+ readFileUtf8 :: FilePath -> IO String
+ readFileUtf8 fp = fmap TL.unpack $ readUtf8File fp
+diff --git a/Text/Julius.hs b/Text/Julius.hs
+index 8c15a99..47b42fd 100644
+--- a/Text/Julius.hs
++++ b/Text/Julius.hs
+@@ -14,17 +14,9 @@ module Text.Julius
+       -- ** Template-Reading Functions
+       -- | These QuasiQuoter and Template Haskell methods return values of
+       -- type @'JavascriptUrl' url@. See the Yesod book for details.
+-      js
+-    , julius
+-    , juliusFile
+-    , jsFile
+-    , juliusFileDebug
+-    , jsFileDebug
+-    , juliusFileReload
+-    , jsFileReload
+ 
+       -- * Datatypes
+-    , JavascriptUrl
++      JavascriptUrl
+     , Javascript (..)
+     , RawJavascript (..)
+ 
+@@ -37,9 +29,9 @@ module Text.Julius
+     , renderJavascriptUrl
+ 
+       -- ** internal, used by 'Text.Coffee'
+-    , javascriptSettings
++    --, javascriptSettings
+       -- ** internal
+-    , juliusUsedIdentifiers
++    --, juliusUsedIdentifiers
+     , asJavascriptUrl
+     ) where
+ 
+@@ -102,48 +94,3 @@ instance RawJS TL.Text where rawJS = RawJavascript . fromLazyText
+ instance RawJS Builder where rawJS = RawJavascript
+ instance RawJS Bool where rawJS = RawJavascript . unJavascript . toJavascript
+ 
+-javascriptSettings :: Q ShakespeareSettings
+-javascriptSettings = do
+-  toJExp <- [|toJavascript|]
+-  wrapExp <- [|Javascript|]
+-  unWrapExp <- [|unJavascript|]
+-  asJavascriptUrl' <- [|asJavascriptUrl|]
+-  return $ defaultShakespeareSettings { toBuilder = toJExp
+-  , wrap = wrapExp
+-  , unwrap = unWrapExp
+-  , modifyFinalValue = Just asJavascriptUrl'
+-  }
+-
+-js, julius :: QuasiQuoter
+-js = QuasiQuoter { quoteExp = \s -> do
+-    rs <- javascriptSettings
+-    quoteExp (shakespeare rs) s
+-    }
+-
+-julius = js
+-
+-jsFile, juliusFile :: FilePath -> Q Exp
+-jsFile fp = do
+-    rs <- javascriptSettings
+-    shakespeareFile rs fp
+-
+-juliusFile = jsFile
+-
+-
+-jsFileReload, juliusFileReload :: FilePath -> Q Exp
+-jsFileReload fp = do
+-    rs <- javascriptSettings
+-    shakespeareFileReload rs fp
+-
+-juliusFileReload = jsFileReload
+-
+-jsFileDebug, juliusFileDebug :: FilePath -> Q Exp
+-juliusFileDebug = jsFileReload
+-{-# DEPRECATED juliusFileDebug "Please use juliusFileReload instead." #-}
+-jsFileDebug = jsFileReload
+-{-# DEPRECATED jsFileDebug "Please use jsFileReload instead." #-}
+-
+--- | Determine which identifiers are used by the given template, useful for
+--- creating systems like yesod devel.
+-juliusUsedIdentifiers :: String -> [(Deref, VarType)]
+-juliusUsedIdentifiers = shakespeareUsedIdentifiers defaultShakespeareSettings
+diff --git a/Text/Lucius.hs b/Text/Lucius.hs
+index 3226b79..fd0b7be 100644
+--- a/Text/Lucius.hs
++++ b/Text/Lucius.hs
+@@ -9,13 +9,13 @@
+ {-# OPTIONS_GHC -fno-warn-missing-fields #-}
+ module Text.Lucius
+     ( -- * Parsing
+-      lucius
+-    , luciusFile
+-    , luciusFileDebug
+-    , luciusFileReload
++    --  lucius
++    --, luciusFile
++    --, luciusFileDebug
++    --, luciusFileReload
+       -- ** Mixins
+-    , luciusMixin
+-    , Mixin
++    --, luciusMixin
++      Mixin
+       -- ** Runtime
+     , luciusRT
+     , luciusRT'
+@@ -37,15 +37,12 @@ module Text.Lucius
+     , colorRed
+     , colorBlack
+       -- ** Size
+-    , mkSize
++    --, mkSize
+     , AbsoluteUnit (..)
+     , AbsoluteSize (..)
+     , absoluteSize
+-    , EmSize (..)
+-    , ExSize (..)
+     , PercentageSize (..)
+     , percentageSize
+-    , PixelSize (..)
+       -- * Internal
+     , parseTopLevels
+     , luciusUsedIdentifiers
+@@ -72,13 +69,6 @@ import Text.Shakespeare (VarType)
+ --
+ -- >>> renderCss ([lucius|foo{bar:baz}|] undefined)
+ -- "foo{bar:baz}"
+-lucius :: QuasiQuoter
+-lucius = QuasiQuoter { quoteExp = luciusFromString }
+-
+-luciusFromString :: String -> Q Exp
+-luciusFromString s =
+-    topLevelsToCassius
+-  $ either (error . show) id $ parse parseTopLevels s s
+ 
+ whiteSpace :: Parser ()
+ whiteSpace = many whiteSpace1 >> return ()
+@@ -219,18 +209,6 @@ parseComment = do
+     _ <- manyTill anyChar $ try $ string "*/"
+     return $ ContentRaw ""
+ 
+-luciusFile :: FilePath -> Q Exp
+-luciusFile fp = do
+-#ifdef GHC_7_4
+-    qAddDependentFile fp
+-#endif
+-    contents <- fmap TL.unpack $ qRunIO $ readUtf8File fp
+-    luciusFromString contents
+-
+-luciusFileDebug, luciusFileReload :: FilePath -> Q Exp
+-luciusFileDebug = cssFileDebug False [|parseTopLevels|] parseTopLevels
+-luciusFileReload = luciusFileDebug
+-
+ parseTopLevels :: Parser [TopLevel Unresolved]
+ parseTopLevels =
+     go id
+@@ -379,14 +357,3 @@ luciusRTMinified tl scope = either Left (Right . renderCss . CssNoWhitespace) $
+ luciusUsedIdentifiers :: String -> [(Deref, VarType)]
+ luciusUsedIdentifiers = cssUsedIdentifiers False parseTopLevels
+ 
+-luciusMixin :: QuasiQuoter
+-luciusMixin = QuasiQuoter { quoteExp = luciusMixinFromString }
+-
+-luciusMixinFromString :: String -> Q Exp
+-luciusMixinFromString s' = do
+-    r <- newName "_render"
+-    case fmap compressBlock $ parse parseBlock s s of
+-        Left e -> error $ show e
+-        Right block -> blockToMixin r [] block
+-  where
+-    s = concat ["mixin{", s', "}"]
+diff --git a/Text/Roy.hs b/Text/Roy.hs
+index 6e5e246..a08b019 100644
+--- a/Text/Roy.hs
++++ b/Text/Roy.hs
+@@ -39,12 +39,12 @@ module Text.Roy
+       -- ** Template-Reading Functions
+       -- | These QuasiQuoter and Template Haskell methods return values of
+       -- type @'JavascriptUrl' url@. See the Yesod book for details.
+-      roy
+-    , royFile
+-    , royFileReload
++    --  roy
++    --, royFile
++    --, royFileReload
+ 
+ #ifdef TEST_EXPORT
+-    , roySettings
++    --, roySettings
+ #endif
+     ) where
+ 
+@@ -52,47 +52,3 @@ import Language.Haskell.TH.Quote (QuasiQuoter (..))
+ import Language.Haskell.TH.Syntax
+ import Text.Shakespeare
+ import Text.Julius
+-
+--- | The Roy language compiles down to Javascript.
+--- We do this compilation once at compile time to avoid needing to do it during the request.
+--- We call this a preConversion because other shakespeare modules like Lucius use Haskell to compile during the request instead rather than a system call.
+-roySettings :: Q ShakespeareSettings
+-roySettings = do
+-  jsettings <- javascriptSettings
+-  return $ jsettings { varChar = '#'
+-  , preConversion = Just PreConvert {
+-      preConvert = ReadProcess "roy" ["--stdio", "--browser"]
+-    , preEscapeIgnoreBalanced = "'\""
+-    , preEscapeIgnoreLine = "//"
+-    , wrapInsertion = Just WrapInsertion {
+-        wrapInsertionIndent = Just "  "
+-      , wrapInsertionStartBegin = "(\\"
+-      , wrapInsertionSeparator = " "
+-      , wrapInsertionStartClose = " ->\n"
+-      , wrapInsertionEnd = ")"
+-      , wrapInsertionAddParens = True
+-      }
+-    }
+-  }
+-
+--- | Read inline, quasiquoted Roy.
+-roy :: QuasiQuoter
+-roy = QuasiQuoter { quoteExp = \s -> do
+-    rs <- roySettings
+-    quoteExp (shakespeare rs) s
+-    }
+-
+--- | Read in a Roy template file. This function reads the file once, at
+--- compile time.
+-royFile :: FilePath -> Q Exp
+-royFile fp = do
+-    rs <- roySettings
+-    shakespeareFile rs fp
+-
+--- | Read in a Roy template file. This impure function uses
+--- unsafePerformIO to re-read the file on every call, allowing for rapid
+--- iteration.
+-royFileReload :: FilePath -> Q Exp
+-royFileReload fp = do
+-    rs <- roySettings
+-    shakespeareFileReload rs fp
+diff --git a/Text/Shakespeare.hs b/Text/Shakespeare.hs
+index 98c0c2d..2f6431b 100644
+--- a/Text/Shakespeare.hs
++++ b/Text/Shakespeare.hs
+@@ -16,12 +16,12 @@ module Text.Shakespeare
+     , WrapInsertion (..)
+     , PreConversion (..)
+     , defaultShakespeareSettings
+-    , shakespeare
+-    , shakespeareFile
+-    , shakespeareFileReload
++    -- , shakespeare
++    -- , shakespeareFile
++    -- , shakespeareFileReload
+     -- * low-level
+-    , shakespeareFromString
+-    , shakespeareUsedIdentifiers
++    -- , shakespeareFromString
++    -- , shakespeareUsedIdentifiers
+     , RenderUrl
+     , VarType (..)
+     , Deref
+@@ -153,38 +153,6 @@ defaultShakespeareSettings = ShakespeareSettings {
+   , modifyFinalValue = Nothing
+ }
+ 
+-instance Lift PreConvert where
+-    lift (PreConvert convert ignore comment wrapInsertion) =
+-        [|PreConvert $(lift convert) $(lift ignore) $(lift comment) $(lift wrapInsertion)|]
+-
+-instance Lift WrapInsertion where
+-    lift (WrapInsertion indent sb sep sc e wp) =
+-        [|WrapInsertion $(lift indent) $(lift sb) $(lift sep) $(lift sc) $(lift e) $(lift wp)|]
+-
+-instance Lift PreConversion where
+-    lift (ReadProcess command args) =
+-        [|ReadProcess $(lift command) $(lift args)|]
+-    lift Id = [|Id|]
+-
+-instance Lift ShakespeareSettings where
+-    lift (ShakespeareSettings x1 x2 x3 x4 x5 x6 x7 x8 x9) =
+-        [|ShakespeareSettings
+-            $(lift x1) $(lift x2) $(lift x3)
+-            $(liftExp x4) $(liftExp x5) $(liftExp x6) $(lift x7) $(lift x8) $(liftMExp x9)|]
+-      where
+-        liftExp (VarE n) = [|VarE $(liftName n)|]
+-        liftExp (ConE n) = [|ConE $(liftName n)|]
+-        liftExp _ = error "liftExp only supports VarE and ConE"
+-        liftMExp Nothing = [|Nothing|]
+-        liftMExp (Just e) = [|Just|] `appE` liftExp e
+-        liftName (Name (OccName a) b) = [|Name (OccName $(lift a)) $(liftFlavour b)|]
+-        liftFlavour NameS = [|NameS|]
+-        liftFlavour (NameQ (ModName a)) = [|NameQ (ModName $(lift a))|]
+-        liftFlavour (NameU _) = error "liftFlavour NameU" -- [|NameU $(lift $ fromIntegral a)|]
+-        liftFlavour (NameL _) = error "liftFlavour NameL" -- [|NameU $(lift $ fromIntegral a)|]
+-        liftFlavour (NameG ns (PkgName p) (ModName m)) = [|NameG $(liftNS ns) (PkgName $(lift p)) (ModName $(lift m))|]
+-        liftNS VarName = [|VarName|]
+-        liftNS DataName = [|DataName|]
+ 
+ type QueryParameters = [(TS.Text, TS.Text)]
+ type RenderUrl url = (url -> QueryParameters -> TS.Text)
+@@ -348,6 +316,7 @@ pack' = TS.pack
+ {-# NOINLINE pack' #-}
+ #endif
+ 
++{-
+ contentsToShakespeare :: ShakespeareSettings -> [Content] -> Q Exp
+ contentsToShakespeare rs a = do
+     r <- newName "_render"
+@@ -399,16 +368,19 @@ shakespeareFile r fp =
+     qAddDependentFile fp >>
+ #endif
+         readFileQ fp >>= shakespeareFromString r
++-}
+ 
+ data VarType = VTPlain | VTUrl | VTUrlParam | VTMixin
+     deriving (Show, Eq, Ord, Enum, Bounded, Typeable, Data, Generic)
+ 
++{-
+ getVars :: Content -> [(Deref, VarType)]
+ getVars ContentRaw{} = []
+ getVars (ContentVar d) = [(d, VTPlain)]
+ getVars (ContentUrl d) = [(d, VTUrl)]
+ getVars (ContentUrlParam d) = [(d, VTUrlParam)]
+ getVars (ContentMix d) = [(d, VTMixin)]
++-}
+ 
+ data VarExp url = EPlain Builder
+                 | EUrl url
+@@ -417,8 +389,10 @@ data VarExp url = EPlain Builder
+ 
+ -- | Determine which identifiers are used by the given template, useful for
+ -- creating systems like yesod devel.
++{-
+ shakespeareUsedIdentifiers :: ShakespeareSettings -> String -> [(Deref, VarType)]
+ shakespeareUsedIdentifiers settings = concatMap getVars . contentFromString settings
++-}
+ 
+ type MTime = UTCTime
+ 
+@@ -435,28 +409,6 @@ insertReloadMap :: FilePath -> (MTime, [Content]) -> IO [Content]
+ insertReloadMap fp (mt, content) = atomicModifyIORef reloadMapRef
+   (\reloadMap -> (M.insert fp (mt, content) reloadMap, content))
+ 
+-shakespeareFileReload :: ShakespeareSettings -> FilePath -> Q Exp
+-shakespeareFileReload settings fp = do
+-    str <- readFileQ fp
+-    s <- qRunIO $ preFilter (Just fp) settings str
+-    let b = shakespeareUsedIdentifiers settings s
+-    c <- mapM vtToExp b
+-    rt <- [|shakespeareRuntime settings fp|]
+-    wrap' <- [|\x -> $(return $ wrap settings) . x|]
+-    return $ wrap' `AppE` (rt `AppE` ListE c)
+-  where
+-    vtToExp :: (Deref, VarType) -> Q Exp
+-    vtToExp (d, vt) = do
+-        d' <- lift d
+-        c' <- c vt
+-        return $ TupE [d', c' `AppE` derefToExp [] d]
+-      where
+-        c :: VarType -> Q Exp
+-        c VTPlain = [|EPlain . $(return $
+-          InfixE (Just $ unwrap settings) (VarE '(.)) (Just $ toBuilder settings))|]
+-        c VTUrl = [|EUrl|]
+-        c VTUrlParam = [|EUrlParam|]
+-        c VTMixin = [|\x -> EMixin $ \r -> $(return $ unwrap settings) $ x r|]
+ 
+ 
+ 
+diff --git a/Text/Shakespeare/Base.hs b/Text/Shakespeare/Base.hs
+index a0e983c..23b4692 100644
+--- a/Text/Shakespeare/Base.hs
++++ b/Text/Shakespeare/Base.hs
+@@ -52,34 +52,6 @@ data Deref = DerefModulesIdent [String] Ident
+            | DerefTuple [Deref]
+     deriving (Show, Eq, Read, Data, Typeable, Ord)
+ 
+-instance Lift Ident where
+-    lift (Ident s) = [|Ident|] `appE` lift s
+-instance Lift Deref where
+-    lift (DerefModulesIdent v s) = do
+-        dl <- [|DerefModulesIdent|]
+-        v' <- lift v
+-        s' <- lift s
+-        return $ dl `AppE` v' `AppE` s'
+-    lift (DerefIdent s) = do
+-        dl <- [|DerefIdent|]
+-        s' <- lift s
+-        return $ dl `AppE` s'
+-    lift (DerefBranch x y) = do
+-        x' <- lift x
+-        y' <- lift y
+-        db <- [|DerefBranch|]
+-        return $ db `AppE` x' `AppE` y'
+-    lift (DerefIntegral i) = [|DerefIntegral|] `appE` lift i
+-    lift (DerefRational r) = do
+-        n <- lift $ numerator r
+-        d <- lift $ denominator r
+-        per <- [|(%) :: Int -> Int -> Ratio Int|]
+-        dr <- [|DerefRational|]
+-        return $ dr `AppE` InfixE (Just n) per (Just d)
+-    lift (DerefString s) = [|DerefString|] `appE` lift s
+-    lift (DerefList x) = [|DerefList $(lift x)|]
+-    lift (DerefTuple x) = [|DerefTuple $(lift x)|]
+-
+ derefParens, derefCurlyBrackets :: UserParser a Deref
+ derefParens        = between (char '(') (char ')') parseDeref
+ derefCurlyBrackets = between (char '{') (char '}') parseDeref
+diff --git a/Text/Shakespeare/Text.hs b/Text/Shakespeare/Text.hs
+index f490d7f..5154618 100644
+--- a/Text/Shakespeare/Text.hs
++++ b/Text/Shakespeare/Text.hs
+@@ -7,20 +7,20 @@ module Text.Shakespeare.Text
+     ( TextUrl
+     , ToText (..)
+     , renderTextUrl
+-    , stext
+-    , text
+-    , textFile
+-    , textFileDebug
+-    , textFileReload
+-    , st -- | strict text
+-    , lt -- | lazy text, same as stext :)
+-    , sbt -- | strict text whose left edge is aligned with bar ('|')
+-    , lbt -- | lazy text, whose left edge is aligned with bar ('|')
++    --, stext
++    --, text
++    --, textFile
++    --, textFileDebug
++    --, textFileReload
++    --, st -- | strict text
++    --, lt -- | lazy text, same as stext :)
++    --, sbt -- | strict text whose left edge is aligned with bar ('|')
++    --, lbt -- | lazy text, whose left edge is aligned with bar ('|')
+     -- * Yesod code generation
+-    , codegen
+-    , codegenSt
+-    , codegenFile
+-    , codegenFileReload
++    --, codegen
++    --, codegenSt
++    --, codegenFile
++    --, codegenFileReload
+     ) where
+ 
+ import Language.Haskell.TH.Quote (QuasiQuoter (..))
+@@ -59,66 +59,12 @@ settings = do
+   }
+ 
+ 
+-stext, lt, st, text, lbt, sbt :: QuasiQuoter
+-stext = 
+-  QuasiQuoter { quoteExp = \s -> do
+-    rs <- settings
+-    render <- [|toLazyText|]
+-    rendered <- shakespeareFromString rs { justVarInterpolation = True } s
+-    return (render `AppE` rendered)
+-    }
+-lt = stext
+-
+-st = 
+-  QuasiQuoter { quoteExp = \s -> do
+-    rs <- settings
+-    render <- [|TL.toStrict . toLazyText|]
+-    rendered <- shakespeareFromString rs { justVarInterpolation = True } s
+-    return (render `AppE` rendered)
+-    }
+-
+-text = QuasiQuoter { quoteExp = \s -> do
+-    rs <- settings
+-    quoteExp (shakespeare rs) $ filter (/='\r') s
+-    }
+-
+ dropBar :: [TL.Text] -> [TL.Text]
+ dropBar [] = []
+ dropBar (c:cx) = c:dropBar' cx
+   where
+     dropBar' txt = reverse $ drop 1 $ map (TL.drop 1 . TL.dropWhile (/= '|')) $ reverse txt
+ 
+-lbt = 
+-  QuasiQuoter { quoteExp = \s -> do
+-    rs <- settings
+-    render <- [|TL.unlines . dropBar . TL.lines . toLazyText|]
+-    rendered <- shakespeareFromString rs { justVarInterpolation = True } s
+-    return (render `AppE` rendered)
+-    }
+-
+-sbt = 
+-  QuasiQuoter { quoteExp = \s -> do
+-    rs <- settings
+-    render <- [|TL.toStrict . TL.unlines . dropBar . TL.lines . toLazyText|]
+-    rendered <- shakespeareFromString rs { justVarInterpolation = True } s
+-    return (render `AppE` rendered)
+-    }
+-
+-textFile :: FilePath -> Q Exp
+-textFile fp = do
+-    rs <- settings
+-    shakespeareFile rs fp
+-
+-
+-textFileDebug :: FilePath -> Q Exp
+-textFileDebug = textFileReload
+-{-# DEPRECATED textFileDebug "Please use textFileReload instead" #-}
+-
+-textFileReload :: FilePath -> Q Exp
+-textFileReload fp = do
+-    rs <- settings
+-    shakespeareFileReload rs fp
+-
+ -- | codegen is designed for generating Yesod code, including templates
+ -- So it uses different interpolation characters that won't clash with templates.
+ codegenSettings :: Q ShakespeareSettings
+@@ -135,40 +81,3 @@ codegenSettings = do
+   , justVarInterpolation = True -- always!
+   }
+ 
+--- | codegen is designed for generating Yesod code, including templates
+--- So it uses different interpolation characters that won't clash with templates.
+--- You can use the normal text quasiquoters to generate code
+-codegen :: QuasiQuoter
+-codegen =
+-  QuasiQuoter { quoteExp = \s -> do
+-    rs <- codegenSettings
+-    render <- [|toLazyText|]
+-    rendered <- shakespeareFromString rs { justVarInterpolation = True } s
+-    return (render `AppE` rendered)
+-    }
+-
+--- | Generates strict Text
+--- codegen is designed for generating Yesod code, including templates
+--- So it uses different interpolation characters that won't clash with templates.
+-codegenSt :: QuasiQuoter
+-codegenSt =
+-  QuasiQuoter { quoteExp = \s -> do
+-    rs <- codegenSettings
+-    render <- [|TL.toStrict . toLazyText|]
+-    rendered <- shakespeareFromString rs { justVarInterpolation = True } s
+-    return (render `AppE` rendered)
+-    }
+-
+-codegenFileReload :: FilePath -> Q Exp
+-codegenFileReload fp = do
+-    rs <- codegenSettings
+-    render <- [|TL.toStrict . toLazyText|]
+-    rendered <- shakespeareFileReload rs{ justVarInterpolation = True } fp
+-    return (render `AppE` rendered)
+-
+-codegenFile :: FilePath -> Q Exp
+-codegenFile fp = do
+-    rs <- codegenSettings
+-    render <- [|TL.toStrict . toLazyText|]
+-    rendered <- shakespeareFile rs{ justVarInterpolation = True } fp
+-    return (render `AppE` rendered)
+diff --git a/Text/TypeScript.hs b/Text/TypeScript.hs
+index 85f6abd..3188272 100644
+--- a/Text/TypeScript.hs
++++ b/Text/TypeScript.hs
+@@ -57,12 +57,12 @@ module Text.TypeScript
+       -- ** Template-Reading Functions
+       -- | These QuasiQuoter and Template Haskell methods return values of
+       -- type @'JavascriptUrl' url@. See the Yesod book for details.
+-      tsc
+-    , typeScriptFile
+-    , typeScriptFileReload
++    --  tsc
++    --, typeScriptFile
++    --, typeScriptFileReload
+ 
+ #ifdef TEST_EXPORT
+-    , typeScriptSettings
++    --, typeScriptSettings
+ #endif
+     ) where
+ 
+@@ -74,43 +74,3 @@ import Text.Julius
+ -- | The TypeScript language compiles down to Javascript.
+ -- We do this compilation once at compile time to avoid needing to do it during the request.
+ -- We call this a preConversion because other shakespeare modules like Lucius use Haskell to compile during the request instead rather than a system call.
+-typeScriptSettings :: Q ShakespeareSettings
+-typeScriptSettings = do
+-  jsettings <- javascriptSettings
+-  return $ jsettings { varChar = '#'
+-  , preConversion = Just PreConvert {
+-      preConvert = ReadProcess "sh" ["-c", "TMP_IN=$(mktemp XXXXXXXXXX.ts); TMP_OUT=$(mktemp XXXXXXXXXX.js); cat /dev/stdin > ${TMP_IN} && tsc --out ${TMP_OUT} ${TMP_IN} && cat ${TMP_OUT}; rm ${TMP_IN} && rm ${TMP_OUT}"]
+-    , preEscapeIgnoreBalanced = "'\""
+-    , preEscapeIgnoreLine = "//"
+-    , wrapInsertion = Just WrapInsertion { 
+-        wrapInsertionIndent = Nothing
+-      , wrapInsertionStartBegin = ";(function("
+-      , wrapInsertionSeparator = ", "
+-      , wrapInsertionStartClose = "){"
+-      , wrapInsertionEnd = "})"
+-      , wrapInsertionAddParens = False
+-      }
+-    }
+-  }
+-
+--- | Read inline, quasiquoted TypeScript
+-tsc :: QuasiQuoter
+-tsc = QuasiQuoter { quoteExp = \s -> do
+-    rs <- typeScriptSettings
+-    quoteExp (shakespeare rs) s
+-    }
+-
+--- | Read in a TypeScript template file. This function reads the file once, at
+--- compile time.
+-typeScriptFile :: FilePath -> Q Exp
+-typeScriptFile fp = do
+-    rs <- typeScriptSettings
+-    shakespeareFile rs fp
+-
+--- | Read in a TypeScript template file. This impure function uses
+--- unsafePerformIO to re-read the file on every call, allowing for rapid
+--- iteration.
+-typeScriptFileReload :: FilePath -> Q Exp
+-typeScriptFileReload fp = do
+-    rs <- typeScriptSettings
+-    shakespeareFileReload rs fp
+diff --git a/shakespeare.cabal b/shakespeare.cabal
+index 37029fc..2c4b557 100644
+--- a/shakespeare.cabal
++++ b/shakespeare.cabal
+@@ -62,18 +62,16 @@ library
+                      Text.Shakespeare.Base
+                      Text.Shakespeare
+                      Text.TypeScript
+-    other-modules:   Text.Hamlet.Parse
+                      Text.Css
++                     Text.CssCommon
++    other-modules:   Text.Hamlet.Parse
+                      Text.MkSizeType
+                      Text.IndentToBrace
+-                     Text.CssCommon
+     ghc-options:     -Wall
+ 
+     if flag(test_export)
+       cpp-options: -DTEST_EXPORT
+ 
+-    extensions: TemplateHaskell
+-
+     if impl(ghc >= 7.4)
+        cpp-options: -DGHC_7_4
+ 
+-- 
+2.1.4
+
diff --git a/standalone/no-th/haskell-patches/skein_hardcode_little-endian.patch b/standalone/no-th/haskell-patches/skein_hardcode_little-endian.patch
new file mode 100644
index 0000000000..7333742b00
--- /dev/null
+++ b/standalone/no-th/haskell-patches/skein_hardcode_little-endian.patch
@@ -0,0 +1,26 @@
+From 3a04b41ffce4e4e87b0fedd3a1e3434a3f06cc76 Mon Sep 17 00:00:00 2001
+From: foo 
+Date: Sun, 22 Sep 2013 00:18:12 +0000
+Subject: [PATCH] hardcode little endian
+
+This is the same as building with a cabal flag.
+
+---
+ c_impl/optimized/skein_port.h |    1 +
+ 1 file changed, 1 insertion(+)
+
+diff --git a/c_impl/optimized/skein_port.h b/c_impl/optimized/skein_port.h
+index a2d0fc2..6929bb0 100644
+--- a/c_impl/optimized/skein_port.h
++++ b/c_impl/optimized/skein_port.h
+@@ -45,6 +45,7 @@ typedef uint64_t        u64b_t;             /* 64-bit unsigned integer */
+  * platform-specific code instead (e.g., for big-endian CPUs).
+  *
+  */
++#define SKEIN_NEED_SWAP  (0)
+ #ifndef SKEIN_NEED_SWAP /* compile-time "override" for endianness? */
+ 
+ #include "brg_endian.h"                     /* get endianness selection */
+-- 
+1.7.10.4
+
diff --git a/standalone/no-th/haskell-patches/vector_hack-to-build-with-new-ghc.patch b/standalone/no-th/haskell-patches/vector_hack-to-build-with-new-ghc.patch
new file mode 100644
index 0000000000..f89f0d60b5
--- /dev/null
+++ b/standalone/no-th/haskell-patches/vector_hack-to-build-with-new-ghc.patch
@@ -0,0 +1,49 @@
+From 6ffd4fcb7d27ec6df709d80a40a262406446a259 Mon Sep 17 00:00:00 2001
+From: dummy 
+Date: Wed, 15 Oct 2014 17:00:56 +0000
+Subject: [PATCH] cross build
+
+---
+ Data/Vector/Fusion/Stream/Monadic.hs |  1 -
+ Data/Vector/Unboxed/Base.hs          | 13 -------------
+ 2 files changed, 14 deletions(-)
+
+diff --git a/Data/Vector/Fusion/Stream/Monadic.hs b/Data/Vector/Fusion/Stream/Monadic.hs
+index 51fec75..b089b3d 100644
+--- a/Data/Vector/Fusion/Stream/Monadic.hs
++++ b/Data/Vector/Fusion/Stream/Monadic.hs
+@@ -101,7 +101,6 @@ import GHC.Exts ( SpecConstrAnnotation(..) )
+ 
+ data SPEC = SPEC | SPEC2
+ #if __GLASGOW_HASKELL__ >= 700
+-{-# ANN type SPEC ForceSpecConstr #-}
+ #endif
+ 
+ emptyStream :: String
+diff --git a/Data/Vector/Unboxed/Base.hs b/Data/Vector/Unboxed/Base.hs
+index 00350cb..34bfc4a 100644
+--- a/Data/Vector/Unboxed/Base.hs
++++ b/Data/Vector/Unboxed/Base.hs
+@@ -65,19 +65,6 @@ vectorTyCon = mkTyCon3 "vector"
+ vectorTyCon m s = mkTyCon $ m ++ "." ++ s
+ #endif
+ 
+-instance Typeable1 Vector where
+-  typeOf1 _ = mkTyConApp (vectorTyCon "Data.Vector.Unboxed" "Vector") []
+-
+-instance Typeable2 MVector where
+-  typeOf2 _ = mkTyConApp (vectorTyCon "Data.Vector.Unboxed.Mutable" "MVector") []
+-
+-instance (Data a, Unbox a) => Data (Vector a) where
+-  gfoldl       = G.gfoldl
+-  toConstr _   = error "toConstr"
+-  gunfold _ _  = error "gunfold"
+-  dataTypeOf _ = G.mkType "Data.Vector.Unboxed.Vector"
+-  dataCast1    = G.dataCast
+-
+ -- ----
+ -- Unit
+ -- ----
+-- 
+2.1.1
+
diff --git a/standalone/no-th/haskell-patches/wai-app-static_deal-with-TH.patch b/standalone/no-th/haskell-patches/wai-app-static_deal-with-TH.patch
new file mode 100644
index 0000000000..93314312f7
--- /dev/null
+++ b/standalone/no-th/haskell-patches/wai-app-static_deal-with-TH.patch
@@ -0,0 +1,82 @@
+From 3aef808eee43c973ae1fbf6e8769d89b7f0d355b Mon Sep 17 00:00:00 2001
+From: dummy 
+Date: Tue, 10 Jun 2014 14:47:42 +0000
+Subject: [PATCH] deal with TH
+
+Export modules referenced by it.
+
+Should not need these icons in git-annex, so not worth using the Evil
+Splicer.
+---
+ Network/Wai/Application/Static.hs | 4 ----
+ WaiAppStatic/Storage/Embedded.hs  | 8 ++++----
+ wai-app-static.cabal              | 4 +---
+ 3 files changed, 5 insertions(+), 11 deletions(-)
+
+diff --git a/Network/Wai/Application/Static.hs b/Network/Wai/Application/Static.hs
+index db2b835..b2c1aec 100644
+--- a/Network/Wai/Application/Static.hs
++++ b/Network/Wai/Application/Static.hs
+@@ -33,8 +33,6 @@ import Control.Monad.IO.Class (liftIO)
+ 
+ import Blaze.ByteString.Builder (toByteString)
+ 
+-import Data.FileEmbed (embedFile)
+-
+ import Data.Text (Text)
+ import qualified Data.Text as T
+ 
+@@ -198,8 +196,6 @@ staticAppPieces _ _ req sendResponse
+         H.status405
+         [("Content-Type", "text/plain")]
+         "Only GET is supported"
+-staticAppPieces _ [".hidden", "folder.png"] _ sendResponse = sendResponse $ W.responseLBS H.status200 [("Content-Type", "image/png")] $ L.fromChunks [$(embedFile "images/folder.png")]
+-staticAppPieces _ [".hidden", "haskell.png"] _ sendResponse = sendResponse $ W.responseLBS H.status200 [("Content-Type", "image/png")] $ L.fromChunks [$(embedFile "images/haskell.png")]
+ staticAppPieces ss rawPieces req sendResponse = liftIO $ do
+     case toPieces rawPieces of
+         Just pieces -> checkPieces ss pieces req >>= response
+diff --git a/WaiAppStatic/Storage/Embedded.hs b/WaiAppStatic/Storage/Embedded.hs
+index daa6e50..9873d4e 100644
+--- a/WaiAppStatic/Storage/Embedded.hs
++++ b/WaiAppStatic/Storage/Embedded.hs
+@@ -3,10 +3,10 @@ module WaiAppStatic.Storage.Embedded(
+       embeddedSettings
+ 
+     -- * Template Haskell
+-    , Etag
+-    , EmbeddableEntry(..)
+-    , mkSettings
++    --, Etag
++    --, EmbeddableEntry(..)
++    --, mkSettings
+     ) where
+ 
+ import WaiAppStatic.Storage.Embedded.Runtime
+-import WaiAppStatic.Storage.Embedded.TH
++--import WaiAppStatic.Storage.Embedded.TH
+diff --git a/wai-app-static.cabal b/wai-app-static.cabal
+index ef6f898..9a59d71 100644
+--- a/wai-app-static.cabal
++++ b/wai-app-static.cabal
+@@ -33,7 +33,6 @@ library
+                    , containers                >= 0.2
+                    , time                      >= 1.1.4
+                    , old-locale                >= 1.0.0.2
+-                   , file-embed                >= 0.0.3.1
+                    , text                      >= 0.7
+                    , blaze-builder             >= 0.2.1.4
+                    , base64-bytestring         >= 0.1
+@@ -61,9 +60,8 @@ library
+                      WaiAppStatic.Listing
+                      WaiAppStatic.Types
+                      WaiAppStatic.CmdLine
+-    other-modules:   Util
+                      WaiAppStatic.Storage.Embedded.Runtime
+-                     WaiAppStatic.Storage.Embedded.TH
++    other-modules:   Util
+     ghc-options:     -Wall
+     extensions:     CPP
+ 
+-- 
+2.0.0
+
diff --git a/standalone/no-th/haskell-patches/xml-hamlet_remove_TH.patch b/standalone/no-th/haskell-patches/xml-hamlet_remove_TH.patch
new file mode 100644
index 0000000000..b6334d31f4
--- /dev/null
+++ b/standalone/no-th/haskell-patches/xml-hamlet_remove_TH.patch
@@ -0,0 +1,108 @@
+From b53713fbb4f3bb6bdd25b07afcaed4940b32dfa8 Mon Sep 17 00:00:00 2001
+From: Joey Hess 
+Date: Wed, 18 Dec 2013 03:32:44 +0000
+Subject: [PATCH] remove TH
+
+---
+ Text/Hamlet/XML.hs | 81 +-----------------------------------------------------
+ 1 file changed, 1 insertion(+), 80 deletions(-)
+
+diff --git a/Text/Hamlet/XML.hs b/Text/Hamlet/XML.hs
+index f587410..4e830bd 100644
+--- a/Text/Hamlet/XML.hs
++++ b/Text/Hamlet/XML.hs
+@@ -1,9 +1,7 @@
+ {-# LANGUAGE TemplateHaskell #-}
+ {-# OPTIONS_GHC -fno-warn-missing-fields #-}
+ module Text.Hamlet.XML
+-    ( xml
+-    , xmlFile
+-    ) where
++    () where
+ 
+ import Language.Haskell.TH.Syntax
+ import Language.Haskell.TH.Quote
+@@ -19,80 +17,3 @@ import qualified Data.Foldable as F
+ import Data.Maybe (fromMaybe)
+ import qualified Data.Map as Map
+ 
+-xml :: QuasiQuoter
+-xml = QuasiQuoter { quoteExp = strToExp }
+-
+-xmlFile :: FilePath -> Q Exp
+-xmlFile = strToExp . TL.unpack <=< qRunIO . readUtf8File
+-
+-strToExp :: String -> Q Exp
+-strToExp s =
+-    case parseDoc s of
+-        Error e -> error e
+-        Ok x -> docsToExp [] x
+-
+-docsToExp :: Scope -> [Doc] -> Q Exp
+-docsToExp scope docs = [| concat $(fmap ListE $ mapM (docToExp scope) docs) |]
+-
+-docToExp :: Scope -> Doc -> Q Exp
+-docToExp scope (DocTag name attrs cs) =
+-    [| [ X.NodeElement (X.Element ($(liftName name)) $(mkAttrs scope attrs) $(docsToExp scope cs))
+-       ] |]
+-docToExp _ (DocContent (ContentRaw s)) = [| [ X.NodeContent (pack $(lift s)) ] |]
+-docToExp scope (DocContent (ContentVar d)) = [| [ X.NodeContent $(return $ derefToExp scope d) ] |]
+-docToExp scope (DocContent (ContentEmbed d)) = return $ derefToExp scope d
+-docToExp scope (DocForall deref ident@(Ident ident') inside) = do
+-    let list' = derefToExp scope deref
+-    name <- newName ident'
+-    let scope' = (ident, VarE name) : scope
+-    inside' <- docsToExp scope' inside
+-    let lam = LamE [VarP name] inside'
+-    [| F.concatMap $(return lam) $(return list') |]
+-docToExp scope (DocWith [] inside) = docsToExp scope inside
+-docToExp scope (DocWith ((deref, ident@(Ident name)):dis) inside) = do
+-    let deref' = derefToExp scope deref
+-    name' <- newName name
+-    let scope' = (ident, VarE name') : scope
+-    inside' <- docToExp scope' (DocWith dis inside)
+-    let lam = LamE [VarP name'] inside'
+-    return $ lam `AppE` deref'
+-docToExp scope (DocMaybe deref ident@(Ident name) just nothing) = do
+-    let deref' = derefToExp scope deref
+-    name' <- newName name
+-    let scope' = (ident, VarE name') : scope
+-    inside' <- docsToExp scope' just
+-    let inside'' = LamE [VarP name'] inside'
+-    nothing' <-
+-        case nothing of
+-            Nothing -> [| [] |]
+-            Just n -> docsToExp scope n
+-    [| maybe $(return nothing') $(return inside'') $(return deref') |]
+-docToExp scope (DocCond conds final) = do
+-    unit <- [| () |]
+-    body <- fmap GuardedB $ mapM go $ conds ++ [(DerefIdent $ Ident "otherwise", fromMaybe [] final)]
+-    return $ CaseE unit [Match (TupP []) body []]
+-  where
+-    go (deref, inside) = do
+-        inside' <- docsToExp scope inside
+-        return (NormalG $ derefToExp scope deref, inside')
+-
+-mkAttrs :: Scope -> [(Maybe Deref, String, [Content])] -> Q Exp
+-mkAttrs _ [] = [| Map.empty |]
+-mkAttrs scope ((mderef, name, value):rest) = do
+-    rest' <- mkAttrs scope rest
+-    this <- [| Map.insert $(liftName name) (T.concat $(fmap ListE $ mapM go value)) |]
+-    let with = [| $(return this) $(return rest') |]
+-    case mderef of
+-        Nothing -> with
+-        Just deref -> [| if $(return $ derefToExp scope deref) then $(with) else $(return rest') |]
+-  where
+-    go (ContentRaw s) = [| pack $(lift s) |]
+-    go (ContentVar d) = return $ derefToExp scope d
+-    go ContentEmbed{} = error "Cannot use embed interpolation in attribute value"
+-
+-liftName :: String -> Q Exp
+-liftName s = do
+-    X.Name local mns _ <- return $ fromString s
+-    case mns of
+-        Nothing -> [| X.Name (pack $(lift $ unpack local)) Nothing Nothing |]
+-        Just ns -> [| X.Name (pack $(lift $ unpack local)) (Just $ pack $(lift $ unpack ns)) Nothing |]
+-- 
+1.8.5.1
+
diff --git a/standalone/no-th/haskell-patches/yesod-auth_don-t-really-build.patch b/standalone/no-th/haskell-patches/yesod-auth_don-t-really-build.patch
new file mode 100644
index 0000000000..3e0d0d9ba7
--- /dev/null
+++ b/standalone/no-th/haskell-patches/yesod-auth_don-t-really-build.patch
@@ -0,0 +1,34 @@
+From 583575461dc58b76c6c7e14d429f73182d49ef81 Mon Sep 17 00:00:00 2001
+From: dummy 
+Date: Tue, 10 Jun 2014 20:29:51 +0000
+Subject: [PATCH] don't really build
+
+---
+ yesod-auth.cabal | 11 -----------
+ 1 file changed, 11 deletions(-)
+
+diff --git a/yesod-auth.cabal b/yesod-auth.cabal
+index 906c08b..b4bc841 100644
+--- a/yesod-auth.cabal
++++ b/yesod-auth.cabal
+@@ -65,17 +65,6 @@ library
+                    , conduit-extra
+                    , attoparsec-conduit
+ 
+-    exposed-modules: Yesod.Auth
+-                     Yesod.Auth.BrowserId
+-                     Yesod.Auth.Dummy
+-                     Yesod.Auth.Email
+-                     Yesod.Auth.OpenId
+-                     Yesod.Auth.Rpxnow
+-                     Yesod.Auth.Message
+-                     Yesod.Auth.GoogleEmail
+-                     Yesod.Auth.GoogleEmail2
+-    other-modules:   Yesod.Auth.Routes
+-                     Yesod.PasswordStore
+     ghc-options:     -Wall
+ 
+ source-repository head
+-- 
+2.0.0
+
diff --git a/standalone/no-th/haskell-patches/yesod-core_expand_TH.patch b/standalone/no-th/haskell-patches/yesod-core_expand_TH.patch
new file mode 100644
index 0000000000..f58fcb353e
--- /dev/null
+++ b/standalone/no-th/haskell-patches/yesod-core_expand_TH.patch
@@ -0,0 +1,768 @@
+From f1feea61dcba0b16afed5ce8dd5d2433fe505461 Mon Sep 17 00:00:00 2001
+From: dummy 
+Date: Thu, 16 Oct 2014 02:15:23 +0000
+Subject: [PATCH] hack TH
+
+---
+ Yesod/Core.hs              |  30 +++---
+ Yesod/Core/Class/Yesod.hs  | 256 ++++++++++++++++++++++++++++++---------------
+ Yesod/Core/Dispatch.hs     |  38 ++-----
+ Yesod/Core/Handler.hs      |  25 ++---
+ Yesod/Core/Internal/Run.hs |   6 +-
+ Yesod/Core/Internal/TH.hs  | 111 --------------------
+ Yesod/Core/Types.hs        |   3 +-
+ Yesod/Core/Widget.hs       |  32 +-----
+ 8 files changed, 213 insertions(+), 288 deletions(-)
+
+diff --git a/Yesod/Core.hs b/Yesod/Core.hs
+index 9b29317..7c0792d 100644
+--- a/Yesod/Core.hs
++++ b/Yesod/Core.hs
+@@ -31,16 +31,16 @@ module Yesod.Core
+     , unauthorizedI
+       -- * Logging
+     , LogLevel (..)
+-    , logDebug
+-    , logInfo
+-    , logWarn
+-    , logError
+-    , logOther
+-    , logDebugS
+-    , logInfoS
+-    , logWarnS
+-    , logErrorS
+-    , logOtherS
++    --, logDebug
++    --, logInfo
++    --, logWarn
++    --, logError
++    --, logOther
++    --, logDebugS
++    --, logInfoS
++    --, logWarnS
++    --, logErrorS
++    --, logOtherS
+       -- * Sessions
+     , SessionBackend (..)
+     , customizeSessionCookies
+@@ -87,17 +87,15 @@ module Yesod.Core
+     , readIntegral
+       -- * Shakespeare
+       -- ** Hamlet
+-    , hamlet
+-    , shamlet
+-    , xhamlet
++    --, hamlet
++    -- , shamlet
++    --, xhamlet
+     , HtmlUrl
+       -- ** Julius
+-    , julius
++    --, julius
+     , JavascriptUrl
+     , renderJavascriptUrl
+       -- ** Cassius/Lucius
+-    , cassius
+-    , lucius
+     , CssUrl
+     , renderCssUrl
+     ) where
+diff --git a/Yesod/Core/Class/Yesod.hs b/Yesod/Core/Class/Yesod.hs
+index 8631d27..c40eb10 100644
+--- a/Yesod/Core/Class/Yesod.hs
++++ b/Yesod/Core/Class/Yesod.hs
+@@ -5,18 +5,22 @@
+ {-# LANGUAGE CPP               #-}
+ module Yesod.Core.Class.Yesod where
+ 
+-import           Control.Monad.Logger               (logErrorS)
++--import           Control.Monad.Logger               (logErrorS)
+ import           Yesod.Core.Content
+ import           Yesod.Core.Handler
+ 
+ import           Yesod.Routes.Class
++import qualified Text.Blaze.Internal
++import qualified Control.Monad.Logger
++import qualified Text.Hamlet
++import qualified Data.Foldable
+ 
+ import           Blaze.ByteString.Builder           (Builder)
+ import           Blaze.ByteString.Builder.Char.Utf8 (fromText)
+ import           Control.Arrow                      ((***), second)
+ import           Control.Monad                      (forM, when, void)
+ import           Control.Monad.IO.Class             (MonadIO (liftIO))
+-import           Control.Monad.Logger               (LogLevel (LevelInfo, LevelOther),
++import           Control.Monad.Logger               (Loc, LogLevel (LevelInfo, LevelOther),
+                                                      LogSource)
+ import qualified Data.ByteString.Char8              as S8
+ import qualified Data.ByteString.Lazy               as L
+@@ -33,7 +37,6 @@ import qualified Data.Text.Encoding.Error           as TEE
+ import           Data.Text.Lazy.Builder             (toLazyText)
+ import           Data.Text.Lazy.Encoding            (encodeUtf8)
+ import           Data.Word                          (Word64)
+-import           Language.Haskell.TH.Syntax         (Loc (..))
+ import           Network.HTTP.Types                 (encodePath)
+ import qualified Network.Wai                        as W
+ import           Data.Default                       (def)
+@@ -94,18 +97,26 @@ class RenderRoute site => Yesod site where
+     defaultLayout w = do
+         p <- widgetToPageContent w
+         mmsg <- getMessage
+-        withUrlRenderer [hamlet|
+-            $newline never
+-            $doctype 5
+-            
+-                
+-                    #{pageTitle p}
+-                    ^{pageHead p}
+-                <body>
+-                    $maybe msg <- mmsg
+-                        <p .message>#{msg}
+-                    ^{pageBody p}
+-            |]
++        withUrlRenderer  $         \ _render_aHra
++          -> do { id
++                    ((Text.Blaze.Internal.preEscapedText . T.pack)
++                       "<!DOCTYPE html>\n<html><head><title>");
++                  id (TBH.toHtml (pageTitle p));
++                  id ((Text.Blaze.Internal.preEscapedText . T.pack) "");
++                  Text.Hamlet.asHtmlUrl (pageHead p) _render_aHra;
++                  id ((Text.Blaze.Internal.preEscapedText . T.pack) "");
++                  Text.Hamlet.maybeH
++                    mmsg
++                    (\ msg_aHrb
++                       -> do { id
++                                 ((Text.Blaze.Internal.preEscapedText . T.pack)
++                                    "
    ");
++                               id (TBH.toHtml msg_aHrb);
++                               id ((Text.Blaze.Internal.preEscapedText . T.pack) "
    ") })
++                    Nothing;
++                  Text.Hamlet.asHtmlUrl (pageBody p) _render_aHra;
++                  id
++                    ((Text.Blaze.Internal.preEscapedText . T.pack) "") }
+ 
+     -- | Override the rendering function for a particular URL. One use case for
+     -- this is to offload static hosting to a different domain name to avoid
+@@ -374,45 +385,103 @@ widgetToPageContent w = do
+     -- modernizr should be at the end of the  http://www.modernizr.com/docs/#installing
+     -- the asynchronous loader means your page doesn't have to wait for all the js to load
+     let (mcomplete, asyncScripts) = asyncHelper render scripts jscript jsLoc
+-        regularScriptLoad = [hamlet|
+-            $newline never
+-            $forall s <- scripts
+-                ^{mkScriptTag s}
+-            $maybe j <- jscript
+-                $maybe s <- jsLoc
+-                    ") })
++                            (Just
++                               (do { id
++                                       ((Text.Blaze.Internal.preEscapedText . T.pack) "") })))
++                    Nothing }
++
++
++        headAll =         \ _render_aHsW
++          -> do { Text.Hamlet.asHtmlUrl head' _render_aHsW;
++                  Data.Foldable.mapM_
++                    (\ s_aHsX -> Text.Hamlet.asHtmlUrl (mkLinkTag s_aHsX) _render_aHsW)
++                    stylesheets;
++                  Data.Foldable.mapM_
++                    (\ s_aHsY
++                       -> do { Text.Hamlet.maybeH
++                                 (right (snd s_aHsY))
++                                 (\ t_aHsZ
++                                    -> Text.Hamlet.maybeH
++                                         (fst s_aHsY)
++                                         (\ media_aHt0
++                                            -> do { id
++                                                      ((Text.Blaze.Internal.preEscapedText . T.pack)
++                                                         "") })
++                                         (Just
++                                            (do { id
++                                                    ((Text.Blaze.Internal.preEscapedText . T.pack)
++                                                       "") })))
++                                 Nothing;
++                               Text.Hamlet.maybeH
++                                 (left (snd s_aHsY))
++                                 (\ content_aHt1
++                                    -> Text.Hamlet.maybeH
++                                         (fst s_aHsY)
++                                         (\ media_aHt2
++                                            -> do { id
++                                                      ((Text.Blaze.Internal.preEscapedText . T.pack)
++                                                         "") })
++                                         (Just
++                                            (do { id
++                                                    ((Text.Blaze.Internal.preEscapedText . T.pack)
++                                                       "") })))
++                                 Nothing })
++                    css;
++                  case jsLoader master of {
++                    BottomOfBody -> return ()
++                    ; BottomOfHeadAsync asyncJsLoader_aHt3
++                      -> Text.Hamlet.asHtmlUrl
++                           (asyncJsLoader_aHt3 asyncScripts mcomplete) _render_aHsW
++                    ; BottomOfHeadBlocking
++                      -> Text.Hamlet.asHtmlUrl regularScriptLoad _render_aHsW } }
++
++    let bodyScript =     \ _render_aHt8 -> do { Text.Hamlet.asHtmlUrl body _render_aHt8;
++              Text.Hamlet.asHtmlUrl regularScriptLoad _render_aHt8 }
++
+ 
+     return $ PageContent title headAll $
+         case jsLoader master of
+@@ -442,10 +511,13 @@ defaultErrorHandler NotFound = selectRep $ do
+         r <- waiRequest
+         let path' = TE.decodeUtf8With TEE.lenientDecode $ W.rawPathInfo r
+         setTitle "Not Found"
+-        toWidget [hamlet|
+-            
    Not Found
+-            
    #{path'}
+-        |]
++        toWidget  $         \ _render_aHte
++          -> do { id
++                    ((Text.Blaze.Internal.preEscapedText . T.pack)
++                       "
    Not Found
    \n
    ");
++                  id (TBH.toHtml path');
++                  id ((Text.Blaze.Internal.preEscapedText . T.pack) "
    ") }
++
+     provideRep $ return $ object ["message" .= ("Not Found" :: Text)]
+ 
+ -- For API requests.
+@@ -455,10 +527,11 @@ defaultErrorHandler NotFound = selectRep $ do
+ defaultErrorHandler NotAuthenticated = selectRep $ do
+     provideRep $ defaultLayout $ do
+         setTitle "Not logged in"
+-        toWidget [hamlet|
+-            
    Not logged in
+-            
    Set the authRoute and the user will be redirected there.
+-        |]
++        toWidget  $         \ _render_aHti
++          -> id
++               ((Text.Blaze.Internal.preEscapedText . T.pack)
++                  "
    Not logged in
    \n
    Set the authRoute and the user will be redirected there.
    ")
++
+ 
+     provideRep $ do
+         -- 401 *MUST* include a WWW-Authenticate header
+@@ -480,10 +553,13 @@ defaultErrorHandler NotAuthenticated = selectRep $ do
+ defaultErrorHandler (PermissionDenied msg) = selectRep $ do
+     provideRep $ defaultLayout $ do
+         setTitle "Permission Denied"
+-        toWidget [hamlet|
+-            
    Permission denied
+-            
    #{msg}
+-        |]
++        toWidget  $         \ _render_aHtq
++          -> do { id
++                    ((Text.Blaze.Internal.preEscapedText . T.pack)
++                       "
    Permission denied
    \n
    ");
++                  id (TBH.toHtml msg);
++                  id ((Text.Blaze.Internal.preEscapedText . T.pack) "
    ") }
++
+     provideRep $
+         return $ object $ [
+           "message" .= ("Permission Denied. " <> msg)
+@@ -492,30 +568,42 @@ defaultErrorHandler (PermissionDenied msg) = selectRep $ do
+ defaultErrorHandler (InvalidArgs ia) = selectRep $ do
+     provideRep $ defaultLayout $ do
+         setTitle "Invalid Arguments"
+-        toWidget [hamlet|
+-            
    Invalid Arguments
+-            
      
+-                $forall msg <- ia
+-                    
    • #{msg}
+-        |]
++        toWidget  $         \ _render_aHtv
++          -> do { id
++                    ((Text.Blaze.Internal.preEscapedText . T.pack)
++                       "
      Invalid Arguments
      \n
        ");
++                  Data.Foldable.mapM_
++                    (\ msg_aHtw
++                       -> do { id ((Text.Blaze.Internal.preEscapedText . T.pack) "
      • ");
++                               id (TBH.toHtml msg_aHtw);
++                               id ((Text.Blaze.Internal.preEscapedText . T.pack) "
        • ") })
++                    ia;
++                  id ((Text.Blaze.Internal.preEscapedText . T.pack) "
      ") }
++
+     provideRep $ return $ object ["message" .= ("Invalid Arguments" :: Text), "errors" .= ia]
+ defaultErrorHandler (InternalError e) = do
+-    $logErrorS "yesod-core" e
+     selectRep $ do
+         provideRep $ defaultLayout $ do
+             setTitle "Internal Server Error"
+-            toWidget [hamlet|
+-                
      Internal Server Error
+-                
      #{e}
+-            |]
++            toWidget  $             \ _render_aHtC
++              -> do { id
++                        ((Text.Blaze.Internal.preEscapedText . T.pack)
++                           "
      Internal Server Error
      \n
      ");
++                      id (TBH.toHtml e);
++                      id ((Text.Blaze.Internal.preEscapedText . T.pack) "
      ") }
++
+         provideRep $ return $ object ["message" .= ("Internal Server Error" :: Text), "error" .= e]
+ defaultErrorHandler (BadMethod m) = selectRep $ do
+     provideRep $ defaultLayout $ do
+         setTitle"Bad Method"
+-        toWidget [hamlet|
+-            
      Method Not Supported
+-            
      Method #{S8.unpack m} not supported
+-        |]
++        toWidget  $         \ _render_aHtH
++          -> do { id
++                    ((Text.Blaze.Internal.preEscapedText . T.pack)
++                       "
      Method Not Supported
      \n
      Method ");
++                  id (TBH.toHtml (S8.unpack m));
++                  id
++                    ((Text.Blaze.Internal.preEscapedText . T.pack)
++                       " not supported
      ") }
+     provideRep $ return $ object ["message" .= ("Bad method" :: Text), "method" .= TE.decodeUtf8With TEE.lenientDecode m]
+ 
+ asyncHelper :: (url -> [x] -> Text)
+@@ -682,8 +770,4 @@ loadClientSession key getCachedDate sessionName req = load
+ -- turn the TH Loc loaction information into a human readable string
+ -- leaving out the loc_end parameter
+ fileLocationToString :: Loc -> String
+-fileLocationToString loc = (loc_package loc) ++ ':' : (loc_module loc) ++
+-  ' ' : (loc_filename loc) ++ ':' : (line loc) ++ ':' : (char loc)
+-  where
+-    line = show . fst . loc_start
+-    char = show . snd . loc_start
++fileLocationToString loc = "unknown"
+diff --git a/Yesod/Core/Dispatch.hs b/Yesod/Core/Dispatch.hs
+index e0d1f0e..cc23fdd 100644
+--- a/Yesod/Core/Dispatch.hs
++++ b/Yesod/Core/Dispatch.hs
+@@ -1,4 +1,3 @@
+-{-# LANGUAGE TemplateHaskell #-}
+ {-# LANGUAGE OverloadedStrings #-}
+ {-# LANGUAGE TypeFamilies #-}
+ {-# LANGUAGE FlexibleInstances #-}
+@@ -6,18 +5,18 @@
+ {-# LANGUAGE CPP #-}
+ module Yesod.Core.Dispatch
+     ( -- * Quasi-quoted routing
+-      parseRoutes
+-    , parseRoutesNoCheck
+-    , parseRoutesFile
+-    , parseRoutesFileNoCheck
+-    , mkYesod
++    --  parseRoutes
++    --, parseRoutesNoCheck
++    --, parseRoutesFile
++    --, parseRoutesFileNoCheck
++    --, mkYesod
+       -- ** More fine-grained
+-    , mkYesodData
+-    , mkYesodSubData
+-    , mkYesodDispatch
+-    , mkYesodSubDispatch
++    --, mkYesodData
++    --, mkYesodSubData
++    --, mkYesodDispatch
++    --, mkYesodSubDispatch
+       -- ** Path pieces
+-    , PathPiece (..)
++      PathPiece (..)
+     , PathMultiPiece (..)
+     , Texts
+       -- * Convert to WAI
+@@ -135,13 +134,6 @@ toWaiAppLogger logger site = do
+                 , yreSite = site
+                 , yreSessionBackend = sb
+                 }
+-    messageLoggerSource
+-        site
+-        logger
+-        $(qLocation >>= liftLoc)
+-        "yesod-core"
+-        LevelInfo
+-        (toLogStr ("Application launched" :: S.ByteString))
+     middleware <- mkDefaultMiddlewares logger
+     return $ middleware $ toWaiAppYre yre
+ 
+@@ -170,14 +162,7 @@ warp port site = do
+                 ]
+             -}
+             , Network.Wai.Handler.Warp.settingsOnException = const $ \e ->
+-                when (shouldLog' e) $
+-                messageLoggerSource
+-                    site
+-                    logger
+-                    $(qLocation >>= liftLoc)
+-                    "yesod-core"
+-                    LevelError
+-                    (toLogStr $ "Exception from Warp: " ++ show e)
++                when (shouldLog' e) $ error (show e)
+             }
+   where
+     shouldLog' =
+@@ -211,7 +196,6 @@ defaultMiddlewaresNoLogging = acceptOverride . autohead . gzip def . methodOverr
+ -- | Deprecated synonym for 'warp'.
+ warpDebug :: YesodDispatch site => Int -> site -> IO ()
+ warpDebug = warp
+-{-# DEPRECATED warpDebug "Please use warp instead" #-}
+ 
+ -- | Runs your application using default middlewares (i.e., via 'toWaiApp'). It
+ -- reads port information from the PORT environment variable, as used by tools
+diff --git a/Yesod/Core/Handler.hs b/Yesod/Core/Handler.hs
+index d2b196b..13cac17 100644
+--- a/Yesod/Core/Handler.hs
++++ b/Yesod/Core/Handler.hs
+@@ -174,7 +174,7 @@ import           Data.Text.Encoding            (decodeUtf8With, encodeUtf8)
+ import           Data.Text.Encoding.Error      (lenientDecode)
+ import qualified Data.Text.Lazy                as TL
+ import qualified Text.Blaze.Html.Renderer.Text as RenderText
+-import           Text.Hamlet                   (Html, HtmlUrl, hamlet)
++import           Text.Hamlet                   (Html, HtmlUrl)
+ 
+ import qualified Data.ByteString               as S
+ import qualified Data.ByteString.Lazy          as L
+@@ -203,6 +203,7 @@ import Control.Exception (throwIO)
+ import Blaze.ByteString.Builder (Builder)
+ import Safe (headMay)
+ import Data.CaseInsensitive (CI)
++import qualified Text.Blaze.Internal
+ import qualified Data.Conduit.List as CL
+ import Control.Monad (unless)
+ import           Control.Monad.Trans.Resource  (MonadResource, InternalState, runResourceT, withInternalState, getInternalState, liftResourceT, resourceForkIO
+@@ -855,19 +856,15 @@ redirectToPost :: (MonadHandler m, RedirectUrl (HandlerSite m) url)
+                -> m a
+ redirectToPost url = do
+     urlText <- toTextUrl url
+-    withUrlRenderer [hamlet|
+-$newline never
+-$doctype 5
+-
+-
+-    
+-        Redirecting...
+-    <body onload="document.getElementById('form').submit()">
+-        <form id="form" method="post" action=#{urlText}>
+-            <noscript>
+-                <p>Javascript has been disabled; please click on the button below to be redirected.
+-            <input type="submit" value="Continue">
+-|] >>= sendResponse
++    withUrlRenderer  $     \ _render_awps
++      -> do { id
++                ((Text.Blaze.Internal.preEscapedText . T.pack)
++                   "<!DOCTYPE html>\n<html><head><title>Redirecting...
      ") }
++ >>= sendResponse
+ 
+ -- | Wraps the 'Content' generated by 'hamletToContent' in a 'RepHtml'.
+ hamletToRepHtml :: MonadHandler m => HtmlUrl (Route (HandlerSite m)) -> m Html
+diff --git a/Yesod/Core/Internal/Run.hs b/Yesod/Core/Internal/Run.hs
+index 311f208..63f666f 100644
+--- a/Yesod/Core/Internal/Run.hs
++++ b/Yesod/Core/Internal/Run.hs
+@@ -16,7 +16,7 @@ import           Control.Exception.Lifted     (catch)
+ import           Control.Monad                (mplus)
+ import           Control.Monad.IO.Class       (MonadIO)
+ import           Control.Monad.IO.Class       (liftIO)
+-import           Control.Monad.Logger         (LogLevel (LevelError), LogSource,
++import           Control.Monad.Logger         (Loc, LogLevel (LevelError), LogSource,
+                                                liftLoc)
+ import           Control.Monad.Trans.Resource (runResourceT, withInternalState, runInternalState, createInternalState, closeInternalState)
+ import qualified Data.ByteString              as S
+@@ -31,7 +31,7 @@ import qualified Data.Text                    as T
+ import           Data.Text.Encoding           (encodeUtf8)
+ import           Data.Text.Encoding           (decodeUtf8With)
+ import           Data.Text.Encoding.Error     (lenientDecode)
+-import           Language.Haskell.TH.Syntax   (Loc, qLocation)
++import           Language.Haskell.TH.Syntax   (qLocation)
+ import qualified Network.HTTP.Types           as H
+ import           Network.Wai
+ #if MIN_VERSION_wai(2, 0, 0)
+@@ -158,8 +158,6 @@ safeEh :: (Loc -> LogSource -> LogLevel -> LogStr -> IO ())
+        -> ErrorResponse
+        -> YesodApp
+ safeEh log' er req = do
+-    liftIO $ log' $(qLocation >>= liftLoc) "yesod-core" LevelError
+-           $ toLogStr $ "Error handler errored out: " ++ show er
+     return $ YRPlain
+         H.status500
+         []
+diff --git a/Yesod/Core/Internal/TH.hs b/Yesod/Core/Internal/TH.hs
+index 7e84c1c..a273c29 100644
+--- a/Yesod/Core/Internal/TH.hs
++++ b/Yesod/Core/Internal/TH.hs
+@@ -23,114 +23,3 @@ import Yesod.Core.Content
+ import Yesod.Core.Class.Dispatch
+ import Yesod.Core.Internal.Run
+ 
+--- | Generates URL datatype and site function for the given 'Resource's. This
+--- is used for creating sites, /not/ subsites. See 'mkYesodSub' for the latter.
+--- Use 'parseRoutes' to create the 'Resource's.
+-mkYesod :: String -- ^ name of the argument datatype
+-        -> [ResourceTree String]
+-        -> Q [Dec]
+-mkYesod name = fmap (uncurry (++)) . mkYesodGeneral name [] False
+-
+--- | Sometimes, you will want to declare your routes in one file and define
+--- your handlers elsewhere. For example, this is the only way to break up a
+--- monolithic file into smaller parts. Use this function, paired with
+--- 'mkYesodDispatch', to do just that.
+-mkYesodData :: String -> [ResourceTree String] -> Q [Dec]
+-mkYesodData name res = mkYesodDataGeneral name False res
+-
+-mkYesodSubData :: String -> [ResourceTree String] -> Q [Dec]
+-mkYesodSubData name res = mkYesodDataGeneral name True res
+-
+-mkYesodDataGeneral :: String -> Bool -> [ResourceTree String] -> Q [Dec]
+-mkYesodDataGeneral name isSub res = do
+-    let (name':rest) = words name
+-    fmap fst $ mkYesodGeneral name' rest isSub res
+-
+--- | See 'mkYesodData'.
+-mkYesodDispatch :: String -> [ResourceTree String] -> Q [Dec]
+-mkYesodDispatch name = fmap snd . mkYesodGeneral name [] False
+-
+--- | Get the Handler and Widget type synonyms for the given site.
+-masterTypeSyns :: Type -> [Dec]
+-masterTypeSyns site =
+-    [ TySynD (mkName "Handler") []
+-      $ ConT ''HandlerT `AppT` site `AppT` ConT ''IO
+-    , TySynD (mkName "Widget")  []
+-      $ ConT ''WidgetT `AppT` site `AppT` ConT ''IO `AppT` ConT ''()
+-    ]
+-
+-mkYesodGeneral :: String                   -- ^ foundation type
+-               -> [String]                 -- ^ arguments for the type
+-               -> Bool                     -- ^ it this a subsite
+-               -> [ResourceTree String]
+-               -> Q([Dec],[Dec])
+-mkYesodGeneral name args isSub resS = do
+-    renderRouteDec <- mkRenderRouteInstance site res
+-    routeAttrsDec  <- mkRouteAttrsInstance site res
+-    dispatchDec    <- mkDispatchInstance site res
+-    parse <- mkParseRouteInstance site res
+-    let rname = mkName $ "resources" ++ name
+-    eres <- lift resS
+-    let resourcesDec =
+-            [ SigD rname $ ListT `AppT` (ConT ''ResourceTree `AppT` ConT ''String)
+-            , FunD rname [Clause [] (NormalB eres) []]
+-            ]
+-    let dataDec = concat
+-            [ [parse]
+-            , renderRouteDec
+-            , [routeAttrsDec]
+-            , resourcesDec
+-            , if isSub then [] else masterTypeSyns site
+-            ]
+-    return (dataDec, dispatchDec)
+-  where site    = foldl' AppT (ConT $ mkName name) (map (VarT . mkName) args)
+-        res     = map (fmap parseType) resS
+-
+-mkMDS :: Q Exp -> MkDispatchSettings
+-mkMDS rh = MkDispatchSettings
+-    { mdsRunHandler = rh
+-    , mdsSubDispatcher =
+-        [|\parentRunner getSub toParent env -> yesodSubDispatch
+-                                 YesodSubRunnerEnv
+-                                    { ysreParentRunner = parentRunner
+-                                    , ysreGetSub = getSub
+-                                    , ysreToParentRoute = toParent
+-                                    , ysreParentEnv = env
+-                                    }
+-                              |]
+-    , mdsGetPathInfo = [|W.pathInfo|]
+-    , mdsSetPathInfo = [|\p r -> r { W.pathInfo = p }|]
+-    , mdsMethod = [|W.requestMethod|]
+-    , mds404 = [|notFound >> return ()|]
+-    , mds405 = [|badMethod >> return ()|]
+-    , mdsGetHandler = defaultGetHandler
+-    }
+-
+--- | If the generation of @'YesodDispatch'@ instance require finer
+--- control of the types, contexts etc. using this combinator. You will
+--- hardly need this generality. However, in certain situations, like
+--- when writing library/plugin for yesod, this combinator becomes
+--- handy.
+-mkDispatchInstance :: Type                -- ^ The master site type
+-                   -> [ResourceTree a]    -- ^ The resource
+-                   -> DecsQ
+-mkDispatchInstance master res = do
+-    clause' <- mkDispatchClause (mkMDS [|yesodRunner|]) res
+-    let thisDispatch = FunD 'yesodDispatch [clause']
+-    return [InstanceD [] yDispatch [thisDispatch]]
+-  where
+-    yDispatch = ConT ''YesodDispatch `AppT` master
+-
+-mkYesodSubDispatch :: [ResourceTree a] -> Q Exp
+-mkYesodSubDispatch res = do
+-    clause' <- mkDispatchClause (mkMDS [|subHelper . fmap toTypedContent|]) res
+-    inner <- newName "inner"
+-    let innerFun = FunD inner [clause']
+-    helper <- newName "helper"
+-    let fun = FunD helper
+-                [ Clause
+-                    []
+-                    (NormalB $ VarE inner)
+-                    [innerFun]
+-                ]
+-    return $ LetE [fun] (VarE helper)
+diff --git a/Yesod/Core/Types.hs b/Yesod/Core/Types.hs
+index 388dfe3..b3fce0f 100644
+--- a/Yesod/Core/Types.hs
++++ b/Yesod/Core/Types.hs
+@@ -21,6 +21,7 @@ import           Control.Monad.Catch                (MonadCatch (..))
+ import           Control.Monad.Catch                (MonadMask (..))
+ #endif
+ import           Control.Monad.IO.Class             (MonadIO (liftIO))
++import qualified Control.Monad.Logger
+ import           Control.Monad.Logger               (LogLevel, LogSource,
+                                                      MonadLogger (..))
+ import           Control.Monad.Trans.Control        (MonadBaseControl (..))
+@@ -191,7 +192,7 @@ data RunHandlerEnv site = RunHandlerEnv
+     , rheRoute    :: !(Maybe (Route site))
+     , rheSite     :: !site
+     , rheUpload   :: !(RequestBodyLength -> FileUpload)
+-    , rheLog      :: !(Loc -> LogSource -> LogLevel -> LogStr -> IO ())
++    , rheLog      :: !(Control.Monad.Logger.Loc -> LogSource -> LogLevel -> LogStr -> IO ())
+     , rheOnError  :: !(ErrorResponse -> YesodApp)
+       -- ^ How to respond when an error is thrown internally.
+       --
+diff --git a/Yesod/Core/Widget.hs b/Yesod/Core/Widget.hs
+index 481199e..8489fbe 100644
+--- a/Yesod/Core/Widget.hs
++++ b/Yesod/Core/Widget.hs
+@@ -16,8 +16,8 @@ module Yesod.Core.Widget
+       WidgetT
+     , PageContent (..)
+       -- * Special Hamlet quasiquoter/TH for Widgets
+-    , whamlet
+-    , whamletFile
++    --, whamlet
++    --, whamletFile
+     , ihamletToRepHtml
+     , ihamletToHtml
+       -- * Convert to Widget
+@@ -46,7 +46,7 @@ module Yesod.Core.Widget
+     , widgetToParentWidget
+     , handlerToWidget
+       -- * Internal
+-    , whamletFileWithSettings
++    --, whamletFileWithSettings
+     , asWidgetT
+     ) where
+ 
+@@ -207,35 +207,9 @@ addScriptRemote = flip addScriptRemoteAttrs []
+ addScriptRemoteAttrs :: MonadWidget m => Text -> [(Text, Text)] -> m ()
+ addScriptRemoteAttrs x y = tell $ GWData mempty mempty (toUnique $ Script (Remote x) y) mempty mempty mempty mempty
+ 
+-whamlet :: QuasiQuoter
+-whamlet = NP.hamletWithSettings rules NP.defaultHamletSettings
+-
+-whamletFile :: FilePath -> Q Exp
+-whamletFile = NP.hamletFileWithSettings rules NP.defaultHamletSettings
+-
+-whamletFileWithSettings :: NP.HamletSettings -> FilePath -> Q Exp
+-whamletFileWithSettings = NP.hamletFileWithSettings rules
+-
+ asWidgetT :: WidgetT site m () -> WidgetT site m ()
+ asWidgetT = id
+ 
+-rules :: Q NP.HamletRules
+-rules = do
+-    ah <- [|asWidgetT . toWidget|]
+-    let helper qg f = do
+-            x <- newName "urender"
+-            e <- f $ VarE x
+-            let e' = LamE [VarP x] e
+-            g <- qg
+-            bind <- [|(>>=)|]
+-            return $ InfixE (Just g) bind (Just e')
+-    let ur f = do
+-            let env = NP.Env
+-                    (Just $ helper [|getUrlRenderParams|])
+-                    (Just $ helper [|liftM (toHtml .) getMessageRender|])
+-            f env
+-    return $ NP.HamletRules ah ur $ \_ b -> return $ ah `AppE` b
+-
+ -- | Wraps the 'Content' generated by 'hamletToContent' in a 'RepHtml'.
+ ihamletToRepHtml :: (MonadHandler m, RenderMessage (HandlerSite m) message)
+                  => HtmlUrlI18n message (Route (HandlerSite m))
+-- 
+2.1.1
+
diff --git a/standalone/no-th/haskell-patches/yesod-form_spliced-TH.patch b/standalone/no-th/haskell-patches/yesod-form_spliced-TH.patch
new file mode 100644
index 0000000000..84314a8d93
--- /dev/null
+++ b/standalone/no-th/haskell-patches/yesod-form_spliced-TH.patch
@@ -0,0 +1,2086 @@
+From 1b24ece1a40c9365f719472ca6e342c8c4065c25 Mon Sep 17 00:00:00 2001
+From: dummy 
+Date: Thu, 16 Oct 2014 02:31:20 +0000
+Subject: [PATCH] hack TH
+
+---
+ Yesod/Form/Bootstrap3.hs | 186 +++++++++--
+ Yesod/Form/Fields.hs     | 816 +++++++++++++++++++++++++++++++++++------------
+ Yesod/Form/Functions.hs  | 257 ++++++++++++---
+ Yesod/Form/Jquery.hs     | 134 ++++++--
+ Yesod/Form/MassInput.hs  | 226 ++++++++++---
+ Yesod/Form/Nic.hs        |  67 +++-
+ 6 files changed, 1322 insertions(+), 364 deletions(-)
+
+diff --git a/Yesod/Form/Bootstrap3.hs b/Yesod/Form/Bootstrap3.hs
+index 84e85fc..1954fb4 100644
+--- a/Yesod/Form/Bootstrap3.hs
++++ b/Yesod/Form/Bootstrap3.hs
+@@ -26,6 +26,9 @@ import Data.String (IsString(..))
+ import Yesod.Core
+ 
+ import qualified Data.Text as T
++import qualified Text.Hamlet
++import qualified Text.Blaze.Internal
++import qualified Data.Foldable
+ 
+ import Yesod.Form.Types
+ import Yesod.Form.Functions
+@@ -152,44 +155,144 @@ renderBootstrap3 formLayout aform fragment = do
+     let views = views' []
+         has (Just _) = True
+         has Nothing  = False
+-        widget = [whamlet|
+-            $newline never
+-            #{fragment}
+-            $forall view <- views
+-              
      
+-                $case formLayout
+-                  $of BootstrapBasicForm
+-                    $if fvId view /= bootstrapSubmitId
+-